+ + + + diff --git a/lib/genshi/examples/bench/simpletal/template.html b/lib/genshi/examples/bench/simpletal/template.html new file mode 100644 --- /dev/null +++ b/lib/genshi/examples/bench/simpletal/template.html @@ -0,0 +1,29 @@ + + + + Title + + +

+ +

Loop

Item

+ +

+ + diff --git a/lib/genshi/examples/bench/xpath.py b/lib/genshi/examples/bench/xpath.py new file mode 100644 --- /dev/null +++ b/lib/genshi/examples/bench/xpath.py @@ -0,0 +1,170 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2006-2008 Edgewall Software +# All rights reserved. +# +# This software is licensed as described in the file COPYING, which +# you should have received as part of this distribution. The terms +# are also available at http://genshi.edgewall.org/wiki/License. +# +# This software consists of voluntary contributions made by many +# individuals. For the exact contribution history, see the revision +# history and logs, available at http://genshi.edgewall.org/log/. + + +try: + from os import times + def time_func(): + tup = times() + #just user time + return tup[0] # + tup[1] +except ImportError: + from time import time as time_func + +from genshi.core import START, END +from genshi.path import Path +from genshi.input import XML + +def benchmark(f, acurate_time=1): + """Checks how much time does function f work. It runs it as + many times as needed for avoiding inaccuracy""" + + runs = 1 + while True: + start_time = time_func() + for _ in xrange(runs): + f() + dt = time_func() - start_time + if dt >= acurate_time: + break + runs *= 2 + return dt / runs + +def spell(t): + """Returns spelled representation of time""" + units = [(0.000001, 'microsecond', 'microseconds'), + (0.001, 'milisecond', 'miliseconds'), + (1, 'second', 'seconds'), + (60, 'minute', 'minutes'), + (60*60, 'hour', 'hours'), + ] + i = 0 + at = abs(t) + while i + 1 < len(units) and at >= units[i + 1][0]: + i += 1 + t /= units[i][0] + if t >= 2: + name = units[i][2] + else: + name = units[i][1] + return "%f %s"%(t, name) + +def test_paths_in_streams(exprs, streams, test_strategies=False): + for expr in exprs: + print "Testing path %r" % expr + for stream, sname in streams: + print '\tRunning on "%s" example:' % sname + + path = Path(expr) + def f(): + for e in path.select(stream): + pass + t = spell(benchmark(f)) + print "\t\tselect:\t\t%s" % t + + def f(): + path = Path(expr) + for e in path.select(stream): + pass + t = spell(benchmark(f)) + print "\t\tinit + select:\t%s" % t + + if test_strategies and len(path.paths) == 1: + from genshi.path import GenericStrategy, SingleStepStrategy, \ + SimplePathStrategy + from genshi.tests.path import FakePath + strategies = (GenericStrategy, SingleStepStrategy, + SimplePathStrategy) + for strategy in strategies: + if not strategy.supports(path.paths[0]): + continue + print "\t\t%s Strategy"%strategy.__name__ + fp = FakePath(strategy(path.paths[0])) + def f(): + for e in fp.select(stream): + pass + t = spell(benchmark(f)) + print "\t\t\tselect:\t\t%s"%t + + +def test_documents(test_strategies=False): + streams = [] + + s = XML("""\ + + + + Foo + + +

Hello

+ + +""") + streams.append((s, "small document")) + + s = XML("""\ + + + + Foo + + +

Hello

+ + Trac Web-based lightweight project management + system + +

+ + +""") + streams.append((s, "big document")) + + paths = [ + '.', + '*|text()', + 'html', + 'html[@lang="en"]', + 'html/body/h1/text()', + 'html/body/div/a/@href', + 'html/body/div[@id="splash"]/a[@class="b4"]/strong/text()', + 'descendant-or-self::text()', + 'descendant-or-self::h1/text()', + ] + test_paths_in_streams(paths, streams, test_strategies) + +if __name__ == '__main__': + from sys import argv + if "--strategies" in argv: + test_strategies = True + else: + test_strategies = False + test_documents(test_strategies) diff --git a/lib/genshi/examples/tutorial/geddit/__init__.py b/lib/genshi/examples/tutorial/geddit/__init__.py new file mode 100644 diff --git a/lib/genshi/examples/tutorial/geddit/controller.py b/lib/genshi/examples/tutorial/geddit/controller.py new file mode 100755 --- /dev/null +++ b/lib/genshi/examples/tutorial/geddit/controller.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python + +import operator, os, pickle, sys + +import cherrypy +from formencode import Invalid +from genshi.input import HTML +from genshi.filters import HTMLFormFiller, HTMLSanitizer + +from geddit.form import LinkForm, CommentForm +from geddit.lib import ajax, template +from geddit.model import Link, Comment + + +class Root(object): + + def __init__(self, data): + self.data = data + + @cherrypy.expose + @template.output('index.html') + def index(self): + links = sorted(self.data.values(), key=operator.attrgetter('time')) + return template.render(links=links) + + @cherrypy.expose + @template.output('submit.html') + def submit(self, cancel=False, **data): + if cherrypy.request.method == 'POST': + if cancel: + raise cherrypy.HTTPRedirect('/') + form = LinkForm() + try: + data = form.to_python(data) + link = Link(**data) + self.data[link.id] = link + raise cherrypy.HTTPRedirect('/') + except Invalid, e: + errors = e.unpack_errors() + else: + errors = {} + + return template.render(errors=errors) | HTMLFormFiller(data=data) + + @cherrypy.expose + @template.output('info.html') + def info(self, id): + link = self.data.get(id) + if not link: + raise cherrypy.NotFound() + return template.render(link=link) + + @cherrypy.expose + @template.output('comment.html') + def comment(self, id, cancel=False, **data): + link = self.data.get(id) + if not link: + raise cherrypy.NotFound() + if cherrypy.request.method == 'POST': + if cancel: + raise cherrypy.HTTPRedirect('/info/%s' % link.id) + form = CommentForm() + try: + data = form.to_python(data) + markup = HTML(data['content']) | HTMLSanitizer() + data['content'] = markup.render('xhtml') + comment = link.add_comment(**data) + if not ajax.is_xhr(): + raise cherrypy.HTTPRedirect('/info/%s' % link.id) + return template.render('_comment.html', comment=comment, + num=len(link.comments)) + except Invalid, e: + errors = e.unpack_errors() + else: + errors = {} + + if ajax.is_xhr(): + stream = template.render('_form.html', link=link, errors=errors) + else: + stream = template.render(link=link, comment=None, errors=errors) + return stream | HTMLFormFiller(data=data) + + @cherrypy.expose + @template.output('index.xml', method='xml') + def feed(self, id=None): + if id: + link = self.data.get(id) + if not link: + raise cherrypy.NotFound() + return template.render('info.xml', link=link) + else: + links = sorted(self.data.values(), key=operator.attrgetter('time')) + return template.render(links=links) + + +def main(filename): + # load data from the pickle file, or initialize it to an empty list + if os.path.exists(filename): + fileobj = open(filename, 'rb') + try: + data = pickle.load(fileobj) + finally: + fileobj.close() + else: + data = {} + + def _save_data(): + # save data back to the pickle file + fileobj = open(filename, 'wb') + try: + pickle.dump(data, fileobj) + finally: + fileobj.close() + if hasattr(cherrypy.engine, 'subscribe'): # CherryPy >= 3.1 + cherrypy.engine.subscribe('stop', _save_data) + else: + cherrypy.engine.on_stop_engine_list.append(_save_data) + + # Some global configuration; note that this could be moved into a + # configuration file + cherrypy.config.update({ + 'tools.encode.on': True, 'tools.encode.encoding': 'utf-8', + 'tools.decode.on': True, + 'tools.trailing_slash.on': True, + 'tools.staticdir.root': os.path.abspath(os.path.dirname(__file__)), + }) + + cherrypy.quickstart(Root(data), '/', { + '/media': { + 'tools.staticdir.on': True, + 'tools.staticdir.dir': 'static' + } + }) + +if __name__ == '__main__': + import formencode + formencode.api.set_stdtranslation(languages=['en']) + main(sys.argv[1]) diff --git a/lib/genshi/examples/tutorial/geddit/form.py b/lib/genshi/examples/tutorial/geddit/form.py new file mode 100644 --- /dev/null +++ b/lib/genshi/examples/tutorial/geddit/form.py @@ -0,0 +1,12 @@ +from formencode import Schema, validators + + +class LinkForm(Schema): + username = validators.UnicodeString(not_empty=True) + url = validators.URL(not_empty=True, add_http=True, check_exists=False) + title = validators.UnicodeString(not_empty=True) + + +class CommentForm(Schema): + username = validators.UnicodeString(not_empty=True) + content = validators.UnicodeString(not_empty=True) diff --git a/lib/genshi/examples/tutorial/geddit/lib/__init__.py b/lib/genshi/examples/tutorial/geddit/lib/__init__.py new file mode 100644 diff --git a/lib/genshi/examples/tutorial/geddit/lib/ajax.py b/lib/genshi/examples/tutorial/geddit/lib/ajax.py new file mode 100644 --- /dev/null +++ b/lib/genshi/examples/tutorial/geddit/lib/ajax.py @@ -0,0 +1,5 @@ +import cherrypy + +def is_xhr(): + requested_with = cherrypy.request.headers.get('X-Requested-With') + return requested_with and requested_with.lower() == 'xmlhttprequest' diff --git a/lib/genshi/examples/tutorial/geddit/lib/template.py b/lib/genshi/examples/tutorial/geddit/lib/template.py new file mode 100644 --- /dev/null +++ b/lib/genshi/examples/tutorial/geddit/lib/template.py @@ -0,0 +1,47 @@ +import os + +import cherrypy +from genshi.core import Stream +from genshi.output import encode, get_serializer +from genshi.template import Context, TemplateLoader + +from geddit.lib import ajax + +loader = TemplateLoader( + os.path.join(os.path.dirname(__file__), '..', 'templates'), + auto_reload=True +) + +def output(filename, method='html', encoding='utf-8', **options): + """Decorator for exposed methods to specify what template the should use + for rendering, and which serialization method and options should be + applied. + """ + def decorate(func): + def wrapper(*args, **kwargs): + cherrypy.thread_data.template = loader.load(filename) + opt = options.copy() + if not ajax.is_xhr() and method == 'html': + opt.setdefault('doctype', 'html') + serializer = get_serializer(method, **opt) + stream = func(*args, **kwargs) + if not isinstance(stream, Stream): + return stream + return encode(serializer(stream), method=serializer, + encoding=encoding) + return wrapper + return decorate + +def render(*args, **kwargs): + """Function to render the given data to the template specified via the + ``@output`` decorator. + """ + if args: + assert len(args) == 1, \ + 'Expected exactly one argument, but got %r' % (args,) + template = loader.load(args[0]) + else: + template = cherrypy.thread_data.template + ctxt = Context(url=cherrypy.url) + ctxt.push(kwargs) + return template.generate(ctxt) diff --git a/lib/genshi/examples/tutorial/geddit/model.py b/lib/genshi/examples/tutorial/geddit/model.py new file mode 100644 --- /dev/null +++ b/lib/genshi/examples/tutorial/geddit/model.py @@ -0,0 +1,31 @@ +from datetime import datetime + + +class Link(object): + + def __init__(self, username, url, title): + self.username = username + self.url = url + self.title = title + self.time = datetime.utcnow() + self.id = hex(hash(tuple([username, url, title, self.time])))[2:] + self.comments = [] + + def __repr__(self): + return '<%s %r>' % (type(self).__name__, self.title) + + def add_comment(self, username, content): + comment = Comment(username, content) + self.comments.append(comment) + return comment + + +class Comment(object): + + def __init__(self, username, content): + self.username = username + self.content = content + self.time = datetime.utcnow() + + def __repr__(self): + return '<%s by %r>' % (type(self).__name__, self.username) diff --git a/lib/genshi/examples/tutorial/geddit/static/jquery.js b/lib/genshi/examples/tutorial/geddit/static/jquery.js new file mode 100644 --- /dev/null +++ b/lib/genshi/examples/tutorial/geddit/static/jquery.js @@ -0,0 +1,11 @@ +/* + * jQuery 1.1.4 - New Wave Javascript + * + * Copyright (c) 2007 John Resig (jquery.com) + * Dual licensed under the MIT (MIT-LICENSE.txt) + * and GPL (GPL-LICENSE.txt) licenses. + * + * $Date: 2007-08-23 21:49:27 -0400 (Thu, 23 Aug 2007) $ + * $Rev: 2862 $ + */ +eval(function(p,a,c,k,e,r){e=function(c){return(c35?String.fromCharCode(c+29):c.toString(36))};if(!''.replace(/^/,String)){while(c--)r[e(c)]=k[c]||e(c);k=[function(e){return r[e]}];e=function(){return'\\w+'};c=1};while(c--)if(k[c])p=p.replace(new RegExp('\\b'+e(c)+'\\b','g'),k[c]);return p}('(9(){6(1f C!="Q")E v=C;E C=19.16=9(a,c){6(19==7||!7.4a)F 1s C(a,c);F 7.4a(a,c)};6(1f $!="Q")E B=$;19.$=C;E q=/^[^<]*(<(.|\\s)+>)[^>]*$|^#(\\w+)$/;C.15=C.3v={4a:9(a,c){a=a||R;6(1f a=="1E"){E m=q.2d(a);6(m&&(m[1]||!c)){6(m[1])a=C.3c([m[1]]);G{E b=R.37(m[3]);6(b)6(b.2j!=m[3])F C().1F(a);G{7[0]=b;7.H=1;F 7}G a=[]}}G F 1s C(c).1F(a)}G 6(C.1g(a))F 1s C(R)[C.15.1L?"1L":"2f"](a);F 7.5J(a.1b==1K&&a||(a.3w||a.H&&a!=19&&!a.1t&&a[0]!=Q&&a[0].1t)&&C.2V(a)||[a])},3w:"1.1.4",7K:9(){F 7.H},H:0,21:9(a){F a==Q?C.2V(7):7[a]},1O:9(a){E b=C(a);b.5c=7;F b},5J:9(a){7.H=0;1K.3v.Y.T(7,a);F 7},J:9(a,b){F C.J(7,a,b)},45:9(a){E b=-1;7.J(9(i){6(7==a)b=i});F b},1j:9(f,d,e){E c=f;6(f.1b==3n)6(d==Q)F 7.H&&C[e||"1j"](7[0],f)||Q;G{c={};c[f]=d}F 7.J(9(a){I(E b 17 c)C.1j(e?7.S:7,b,C.4Q(7,c[b],e,a,b))})},1h:9(b,a){F 7.1j(b,a,"34")},2Q:9(e){6(1f e!="4P"&&e!=K)F 7.3K().3H(R.60(e));E t="";C.J(e||7,9(){C.J(7.2Z,9(){6(7.1t!=8)t+=7.1t!=1?7.5S:C.15.2Q([7])})});F t},82:9(){E a,2e=1a;F 7.J(9(){6(!a)a=C.3c(2e,7.2I);E b=a[0].3B(O);7.P.2p(b,7);20(b.1k)b=b.1k;b.4p(7)})},3H:9(){F 7.2J(1a,O,1,9(a){7.4p(a)})},5v:9(){F 7.2J(1a,O,-1,9(a){7.2p(a,7.1k)})},5u:9(){F 7.2J(1a,M,1,9(a){7.P.2p(a,7)})},5t:9(){F 7.2J(1a,M,-1,9(a){7.P.2p(a,7.2a)})},3L:9(){F 7.5c||C([])},1F:9(t){E b=C.3M(7,9(a){F C.1F(t,a)});F 7.1O(/[^+>] [^+>]/.1d(t)||t.U("..")>-1?C.4d(b):b)},7o:9(e){e=e!=Q?e:O;E d=7.1r(7.1F("*"));6(C.N.12){d.J(9(){7.2l$1i={};I(E a 17 7.$1i)7.2l$1i[a]=C.14({},7.$1i[a])}).49()}E r=7.1O(C.3M(7,9(a){F a.3B(e)}));6(C.N.12){d.J(9(){E c=7.2l$1i;I(E a 17 c)I(E b 17 c[a])C.1c.1r(7,a,c[a][b],c[a][b].V);7.2l$1i=K})}6(e){E f=r.1r(r.1F(\'*\')).1l(\'2b,39[@L=3i]\');d.1l(\'2b,39[@L=3i]\').J(9(i){6(7.3j)f[i].3j=7.3j;6(7.27)f[i].27=O})}F r},1l:9(t){F 7.1O(C.1g(t)&&C.2B(7,9(b,a){F t.T(b,[a])})||C.2R(t,7))},5l:9(t){F 7.1O(t.1b==3n&&C.2R(t,7,O)||C.2B(7,9(a){F(t.1b==1K||t.3w)?C.4K(a,t)<0:a!=t}))},1r:9(t){F 7.1O(C.29(7.21(),t.1b==3n?C(t).21():t.H!=Q&&(!t.W||t.W=="6s")?t:[t]))},3y:9(a){F a?C.2R(a,7).H>0:M},2G:9(a){F a==Q?(7.H?7[0].2A:K):7.1j("2A",a)},5W:9(a){F a==Q?(7.H?7[0].2W:K):7.3K().3H(a)},3S:9(){F 7.1O(1K.3v.3S.T(7,1a))},2J:9(f,d,g,e){E c=7.H>1,a;F 7.J(9(){6(!a){a=C.3c(f,7.2I);6(g<0)a.8E()}E b=7;6(d&&C.W(7,"1A")&&C.W(a[0],"3O"))b=7.4L("1w")[0]||7.4p(R.6a("1w"));C.J(a,9(){6(C.W(7,"33")){6(7.32)C.31({1G:7.32,2w:M,3G:"33"});G C.4E(7.2Q||7.5Z||7.2W||"")}G e.T(b,[c?7.3B(O):7])})})}};C.14=C.15.14=9(){E c=1a[0]||{},a=1,1M=1a.H,4D=M;6(c.1b==8d){4D=c;c=1a[1]||{}}6(1M==1){c=7;a=0}E b;I(;a<1M;a++)6((b=1a[a])!=K)I(E i 17 b){6(c==b[i])5X;6(4D&&1f b[i]==\'4P\'&&c[i])C.14(c[i],b[i]);G 6(b[i]!=Q)c[i]=b[i]}F c};C.14({8a:9(a){19.$=B;6(a)19.16=v;F C},1g:9(a){F!!a&&1f a!="1E"&&!a.W&&a.1b!=1K&&/9/i.1d(a+"")},3E:9(a){F a.3D&&!a.4z||a.4y&&a.2I&&!a.2I.4z},4E:9(a){a=C.2s(a);6(a){6(19.5N)19.5N(a);G 6(C.N.1H)19.4x(a,0);G 2T.2S(19,a)}},W:9(b,a){F b.W&&b.W.1I()==a.1I()},J:9(a,b,c){6(c){6(a.H==Q)I(E i 17 a)b.T(a[i],c);G I(E i=0,3A=a.H;i<3A;i++)6(b.T(a[i],c)===M)1J}G{6(a.H==Q)I(E i 17 a)b.2S(a[i],i,a[i]);G I(E i=0,3A=a.H,2G=a[0];i<3A&&b.2S(2G,i,2G)!==M;2G=a[++i]){}}F a},4Q:9(c,b,d,e,a){6(C.1g(b))b=b.2S(c,[e]);E f=/z-?45|7S-?7Q|1e|5y|7O-?1u/i;F b&&b.1b==3x&&d=="34"&&!f.1d(a)?b+"4t":b},18:{1r:9(b,c){C.J((c||"").2M(/\\s+/),9(i,a){6(!C.18.2N(b.18,a))b.18+=(b.18?" ":"")+a})},23:9(b,c){b.18=c!=Q?C.2B(b.18.2M(/\\s+/),9(a){F!C.18.2N(c,a)}).5w(" "):""},2N:9(t,c){F C.4K(c,(t.18||t).3s().2M(/\\s+/))>-1}},1V:9(e,o,f){I(E i 17 o){e.S["2U"+i]=e.S[i];e.S[i]=o[i]}f.T(e,[]);I(E i 17 o)e.S[i]=e.S["2U"+i]},1h:9(e,p){6(p=="1u"||p=="24"){E b={},3p,3o,d=["7J","7G","7F","7B"];C.J(d,9(){b["7A"+7]=0;b["7x"+7+"7u"]=0});C.1V(e,b,9(){6(C(e).3y(\':4N\')){3p=e.7t;3o=e.7q}G{e=C(e.3B(O)).1F(":4e").5d("27").3L().1h({3V:"1C",3k:"7n",11:"2m",7h:"0",7e:"0"}).57(e.P)[0];E a=C.1h(e.P,"3k")||"3g";6(a=="3g")e.P.S.3k="76";3p=e.74;3o=e.71;6(a=="3g")e.P.S.3k="3g";e.P.3e(e)}});F p=="1u"?3p:3o}F C.34(e,p)},34:9(h,d,g){E i,1R=[],1V=[];9 2E(a){6(!C.N.1H)F M;E b=R.2L.3b(a,K);F!b||b.44("2E")==""}6(d=="1e"&&C.N.12){i=C.1j(h.S,"1e");F i==""?"1":i}6(d.2k(/3a/i))d=x;6(!g&&h.S[d])i=h.S[d];G 6(R.2L&&R.2L.3b){6(d.2k(/3a/i))d="3a";d=d.1v(/([A-Z])/g,"-$1").2D();E e=R.2L.3b(h,K);6(e&&!2E(h))i=e.44(d);G{I(E a=h;a&&2E(a);a=a.P)1R.42(a);I(a=0;a<1R.H;a++)6(2E(1R[a])){1V[a]=1R[a].S.11;1R[a].S.11="2m"}i=d=="11"&&1V[1R.H-1]!=K?"1T":R.2L.3b(h,K).44(d)||"";I(a=0;a<1V.H;a++)6(1V[a]!=K)1R[a].S.11=1V[a]}6(d=="1e"&&i=="")i="1"}G 6(h.41){E f=d.1v(/\\-(\\w)/g,9(m,c){F c.1I()});i=h.41[d]||h.41[f]}F i},3c:9(a,c){E r=[];c=c||R;C.J(a,9(i,b){6(!b)F;6(b.1b==3x)b=b.3s();6(1f b=="1E"){E s=C.2s(b).2D(),1m=c.6a("1m"),1P=[];E a=!s.U("<1Z")&&[1,"<2b>",""]||!s.U("<6L")&&[1,"<4V>",""]||s.2k(/^<(6I|1w|6H|6F|6D)/)&&[1,"<1A>",""]||!s.U("<3O")&&[2,"<1A><1w>",""]||(!s.U("<6A")||!s.U("<6y"))&&[3,"<1A><1w><3O>",""]||!s.U("<6x")&&[2,"<1A><1w><4T>",""]||C.N.12&&[1,"1m<1m>",""]||[0,"",""];1m.2W=a[1]+b+a[2];20(a[0]--)1m=1m.3Y;6(C.N.12){6(!s.U("<1A")&&s.U("<1w")<0)1P=1m.1k&&1m.1k.2Z;G 6(a[1]=="<1A>"&&s.U("<1w")<0)1P=1m.2Z;I(E n=1P.H-1;n>=0;--n)6(C.W(1P[n],"1w")&&!1P[n].2Z.H)1P[n].P.3e(1P[n]);6(/^\\s/.1d(b))1m.2p(c.60(b.2k(/^\\s*/)[0]),1m.1k)}b=C.2V(1m.2Z)}6(0===b.H&&(!C.W(b,"38")&&!C.W(b,"2b")))F;6(b[0]==Q||C.W(b,"38")||b.6u)r.Y(b);G r=C.29(r,b)});F r},1j:9(c,d,a){E e=C.3E(c)?{}:C.4q;6(d=="28"&&C.N.1H)c.P.3j;6(e[d]){6(a!=Q)c[e[d]]=a;F c[e[d]]}G 6(C.N.12&&d=="S")F C.1j(c.S,"6p",a);G 6(a==Q&&C.N.12&&C.W(c,"38")&&(d=="6n"||d=="6m"))F c.6k(d).5S;G 6(c.4y){6(a!=Q)c.6j(d,a);6(C.N.12&&/5R|32/.1d(d)&&!C.3E(c))F c.3F(d,2);F c.3F(d)}G{6(d=="1e"&&C.N.12){6(a!=Q){c.5y=1;c.1l=(c.1l||"").1v(/5T\$[^)]*\$/,"")+(3m(a).3s()=="6d"?"":"5T(1e="+a*6c+")")}F c.1l?(3m(c.1l.2k(/1e=([^)]*)/)[1])/6c).3s():""}d=d.1v(/-([a-z])/8I,9(z,b){F b.1I()});6(a!=Q)c[d]=a;F c[d]}},2s:9(t){F(t||"").1v(/^\\s+|\\s+$/g,"")},2V:9(a){E r=[];6(1f a!="8H")I(E i=0,1M=a.H;i<1M;i++)r.Y(a[i]);G r=a.3S(0);F r},4K:9(b,a){I(E i=0,1M=a.H;i<1M;i++)6(a[i]==b)F i;F-1},29:9(a,b){6(C.N.12){I(E i=0;b[i];i++)6(b[i].1t!=8)a.Y(b[i])}G I(E i=0;b[i];i++)a.Y(b[i]);F a},4d:9(a){E r=[],4O=C.1q++;2g{I(E i=0,69=a.H;i<69;i++)6(4O!=a[i].1q){a[i].1q=4O;r.Y(a[i])}}2h(e){r=a}F r},1q:0,2B:9(b,a,c){6(1f a=="1E")a=2T("M||9(a,i){F "+a+"}");E d=[];I(E i=0,3P=b.H;i<3P;i++)6(!c&&a(b[i],i)||c&&!a(b[i],i))d.Y(b[i]);F d},3M:9(c,b){6(1f b=="1E")b=2T("M||9(a){F "+b+"}");E d=[];I(E i=0,3P=c.H;i<3P;i++){E a=b(c[i],i);6(a!==K&&a!=Q){6(a.1b!=1K)a=[a];d=d.8x(a)}}F d}});E u=8w.8u.2D();C.N={6b:(u.2k(/.+(?:8s|8q|8p|8o)[\\/: ]([\\d.]+)/)||[])[1],1H:/61/.1d(u),2t:/2t/.1d(u),12:/12/.1d(u)&&!/2t/.1d(u),3J:/3J/.1d(u)&&!/(8n|61)/.1d(u)};E x=C.N.12?"3I":"4G";C.14({8m:!C.N.12||R.8l=="8k",3I:C.N.12?"3I":"4G",4q:{"I":"8j","8i":"18","3a":x,4G:x,3I:x,2W:"2W",18:"18",2A:"2A",30:"30",27:"27",8h:"8g",28:"28",8f:"8e"}});C.J({5Y:"a.P",4C:"16.4C(a)",8c:"16.25(a,2,\'2a\')",8b:"16.25(a,2,\'4B\')",88:"16.4A(a.P.1k,a)",87:"16.4A(a.1k)"},9(i,n){C.15[i]=9(a){E b=C.3M(7,n);6(a&&1f a=="1E")b=C.2R(a,b);F 7.1O(C.4d(b))}});C.J({57:"3H",86:"5v",2p:"5u",85:"5t"},9(i,n){C.15[i]=9(){E a=1a;F 7.J(9(){I(E j=0,1M=a.H;j<1M;j++)C(a[j])[n](7)})}});C.J({5d:9(a){C.1j(7,a,"");7.84(a)},83:9(c){C.18.1r(7,c)},81:9(c){C.18.23(7,c)},80:9(c){C.18[C.18.2N(7,c)?"23":"1r"](7,c)},23:9(a){6(!a||C.1l(a,[7]).r.H)7.P.3e(7)},3K:9(){20(7.1k)7.3e(7.1k)}},9(i,n){C.15[i]=9(){F 7.J(n,1a)}});C.J(["5Q","5P","5M","5L"],9(i,n){C.15[n]=9(a,b){F 7.1l(":"+n+"("+a+")",b)}});C.J(["1u","24"],9(i,n){C.15[n]=9(h){F h==Q?(7.H?C.1h(7[0],n):K):7.1h(n,h.1b==3n?h:h+"4t")}});E A=C.N.1H&&5K(C.N.6b)<7Z?"(?:[\\\\w*2l-]|\\\\\\\\.)":"(?:[\\\\w\\7Y-\\7V*2l-]|\\\\\\\\.)",5I=1s 3C("^[/>]\\\\s*("+A+"+)"),5H=1s 3C("^("+A+"+)(#)("+A+"+)"),5G=1s 3C("^([#.]?)("+A+"*)");C.14({4w:{"":"m[2]==\'*\'||16.W(a,m[2])","#":"a.3F(\'2j\')==m[2]",":":{5P:"im[3]-0",25:"m[3]-0==i",5Q:"m[3]-0==i",2H:"i==0",2P:"i==r.H-1",5E:"i%2==0",5D:"i%2","2H-3z":"a.P.4L(\'*\')[0]==a","2P-3z":"16.25(a.P.3Y,1,\'4B\')==a","7U-3z":"!16.25(a.P.3Y,2,\'4B\')",5Y:"a.1k",3K:"!a.1k",5L:"(a.5Z||a.7T||\'\').U(m[3])>=0",4N:\'"1C"!=a.L&&16.1h(a,"11")!="1T"&&16.1h(a,"3V")!="1C"\',1C:\'"1C"==a.L||16.1h(a,"11")=="1T"||16.1h(a,"3V")=="1C"\',7R:"!a.30",30:"a.30",27:"a.27",28:"a.28||16.1j(a,\'28\')",2Q:"\'2Q\'==a.L",4e:"\'4e\'==a.L",3i:"\'3i\'==a.L",4v:"\'4v\'==a.L",5C:"\'5C\'==a.L",4u:"\'4u\'==a.L",5B:"\'5B\'==a.L",5A:"\'5A\'==a.L",1X:\'"1X"==a.L||16.W(a,"1X")\',39:"/39|2b|7P|1X/i.1d(a.W)",2N:"16.1F(m[3],a).H"},"[":"16.1F(m[2],a).H"},5x:[/^\\[ *(@)([\\w-]+) *([!*$^~=]*) *(\'?"?)(.*?)\\4 *\\]/,/^(\\[)\\s*(.*?(\\[.*?\\])?[^[]*?)\\s*\\]/,/^(:)([\\w-]+)\$"?\'?(.*?(\\(.*?\$)?[^(]*?)"?\'?\\)/,1s 3C("^([:.#]*)("+A+"+)")],2R:9(a,c,b){E d,1Y=[];20(a&&a!=d){d=a;E f=C.1l(a,c,b);a=f.t.1v(/^\\s*,\\s*/,"");1Y=b?c=f.r:C.29(1Y,f.r)}F 1Y},1F:9(t,l){6(1f t!="1E")F[t];6(l&&!l.1t)l=K;l=l||R;6(!t.U("//")){t=t.2K(2,t.H)}G 6(!t.U("/")&&!l.2I){l=l.3D;t=t.2K(1,t.H);6(t.U("/")>=1)t=t.2K(t.U("/"),t.H)}E d=[l],2q=[],2P;20(t&&2P!=t){E r=[];2P=t;t=C.2s(t).1v(/^\\/\\//,"");E k=M;E g=5I;E m=g.2d(t);6(m){E o=m[1].1I();I(E i=0;d[i];i++)I(E c=d[i].1k;c;c=c.2a)6(c.1t==1&&(o=="*"||c.W.1I()==o.1I()))r.Y(c);d=r;t=t.1v(g,"");6(t.U(" ")==0)5X;k=O}G{g=/^((\\/?\\.\\.)|([>\\/+~]))\\s*(\\w*)/i;6((m=g.2d(t))!=K){r=[];E o=m[4],1q=C.1q++;m=m[1];I(E j=0,2o=d.H;j<2o;j++)6(m.U("..")<0){E n=m=="~"||m=="+"?d[j].2a:d[j].1k;I(;n;n=n.2a)6(n.1t==1){6(m=="~"&&n.1q==1q)1J;6(!o||n.W.1I()==o.1I()){6(m=="~")n.1q=1q;r.Y(n)}6(m=="+")1J}}G r.Y(d[j].P);d=r;t=C.2s(t.1v(g,""));k=O}}6(t&&!k){6(!t.U(",")){6(l==d[0])d.4s();2q=C.29(2q,d);r=d=[l];t=" "+t.2K(1,t.H)}G{E h=5H;E m=h.2d(t);6(m){m=[0,m[2],m[3],m[1]]}G{h=5G;m=h.2d(t)}m[2]=m[2].1v(/\\\\/g,"");E f=d[d.H-1];6(m[1]=="#"&&f&&f.37&&!C.3E(f)){E p=f.37(m[2]);6((C.N.12||C.N.2t)&&p&&1f p.2j=="1E"&&p.2j!=m[2])p=C(\'[@2j="\'+m[2]+\'"]\',f)[0];d=r=p&&(!m[3]||C.W(p,m[3]))?[p]:[]}G{I(E i=0;d[i];i++){E a=m[1]!=""||m[0]==""?"*":m[2];6(a=="*"&&d[i].W.2D()=="4P")a="2O";r=C.29(r,d[i].4L(a))}6(m[1]==".")r=C.4r(r,m[2]);6(m[1]=="#"){E e=[];I(E i=0;r[i];i++)6(r[i].3F("2j")==m[2]){e=[r[i]];1J}r=e}d=r}t=t.1v(h,"")}}6(t){E b=C.1l(t,r);d=r=b.r;t=C.2s(b.t)}}6(t)d=[];6(d&&l==d[0])d.4s();2q=C.29(2q,d);F 2q},4r:9(r,m,a){m=" "+m+" ";E c=[];I(E i=0;r[i];i++){E b=(" "+r[i].18+" ").U(m)>=0;6(!a&&b||a&&!b)c.Y(r[i])}F c},1l:9(t,r,h){E d;20(t&&t!=d){d=t;E p=C.5x,m;I(E i=0;p[i];i++){m=p[i].2d(t);6(m){t=t.7N(m[0].H);m[2]=m[2].1v(/\\\\/g,"");1J}}6(!m)1J;6(m[1]==":"&&m[2]=="5l")r=C.1l(m[3],r,O).r;G 6(m[1]==".")r=C.4r(r,m[2],h);G 6(m[1]=="@"){E g=[],L=m[3];I(E i=0,2o=r.H;i<2o;i++){E a=r[i],z=a[C.4q[m[2]]||m[2]];6(z==K||/5R|32|28/.1d(m[2]))z=C.1j(a,m[2])||\'\';6((L==""&&!!z||L=="="&&z==m[5]||L=="!="&&z!=m[5]||L=="^="&&z&&!z.U(m[5])||L=="$="&&z.2K(z.H-m[5].H)==m[5]||(L=="*="||L=="~=")&&z.U(m[5])>=0)^h)g.Y(a)}r=g}G 6(m[1]==":"&&m[2]=="25-3z"){E e=C.1q++,g=[],1d=/(\\d*)n\\+?(\\d*)/.2d(m[3]=="5E"&&"2n"||m[3]=="5D"&&"2n+1"||!/\\D/.1d(m[3])&&"n+"+m[3]||m[3]),2H=(1d[1]||1)-0,d=1d[2]-0;I(E i=0,2o=r.H;i<2o;i++){E j=r[i],P=j.P;6(e!=P.1q){E c=1;I(E n=P.1k;n;n=n.2a)6(n.1t==1)n.4o=c++;P.1q=e}E b=M;6(2H==1){6(d==0||j.4o==d)b=O}G 6((j.4o+d)%2H==0)b=O;6(b^h)g.Y(j)}r=g}G{E f=C.4w[m[1]];6(1f f!="1E")f=C.4w[m[1]][m[2]];f=2T("M||9(a,i){F "+f+"}");r=C.2B(r,f,h)}}F{r:r,t:t}},4C:9(c){E b=[];E a=c.P;20(a&&a!=R){b.Y(a);a=a.P}F b},25:9(a,e,c,b){e=e||1;E d=0;I(;a;a=a[c])6(a.1t==1&&++d==e)1J;F a},4A:9(n,a){E r=[];I(;n;n=n.2a){6(n.1t==1&&(!a||n!=a))r.Y(n)}F r}});C.1c={1r:9(f,d,c,b){6(C.N.12&&f.3t!=Q)f=19;6(!c.22)c.22=7.22++;6(b!=Q){E e=c;c=9(){F e.T(7,1a)};c.V=b;c.22=e.22}6(!f.$1i)f.$1i={};6(!f.$1y)f.$1y=9(){E a;6(1f C=="Q"||C.1c.4n)F a;a=C.1c.1y.T(f,1a);F a};E g=f.$1i[d];6(!g){g=f.$1i[d]={};6(f.4m)f.4m(d,f.$1y,M);G f.7M("3r"+d,f.$1y)}g[c.22]=c;7.1D[d]=O},22:1,1D:{},23:9(c,b,a){E d=c.$1i,2c,45;6(d){6(b&&b.L){a=b.4l;b=b.L}6(!b){I(b 17 d)7.23(c,b)}G 6(d[b]){6(a)4k d[b][a.22];G I(a 17 c.$1i[b])4k d[b][a];I(2c 17 d[b])1J;6(!2c){6(c.4j)c.4j(b,c.$1y,M);G c.7L("3r"+b,c.$1y);2c=K;4k d[b]}}I(2c 17 d)1J;6(!2c)c.$1y=c.$1i=K}},1z:9(c,b,d){b=C.2V(b||[]);6(!d){6(7.1D[c])C("*").1r([19,R]).1z(c,b)}G{E a,2c,15=C.1g(d[c]||K);b.42(7.4i({L:c,1S:d}));6(C.1g(d.$1y))a=d.$1y.T(d,b);6(!15&&d["3r"+c]&&d["3r"+c].T(d,b)===M)a=M;6(15&&a!==M&&!(C.W(d,\'a\')&&c=="4h")){7.4n=O;d[c]()}7.4n=M}},1y:9(b){E a;b=C.1c.4i(b||19.1c||{});E c=7.$1i&&7.$1i[b.L],2e=1K.3v.3S.2S(1a,1);2e.42(b);I(E j 17 c){2e[0].4l=c[j];2e[0].V=c[j].V;6(c[j].T(7,2e)===M){b.2u();b.2X();a=M}}6(C.N.12)b.1S=b.2u=b.2X=b.4l=b.V=K;F a},4i:9(c){E a=c;c=C.14({},a);c.2u=9(){6(a.2u)a.2u();a.7I=M};c.2X=9(){6(a.2X)a.2X();a.7H=O};6(!c.1S&&c.5r)c.1S=c.5r;6(C.N.1H&&c.1S.1t==3)c.1S=a.1S.P;6(!c.4g&&c.4F)c.4g=c.4F==c.1S?c.7C:c.4F;6(c.5p==K&&c.66!=K){E e=R.3D,b=R.4z;c.5p=c.66+(e&&e.5o||b.5o||0);c.7z=c.7v+(e&&e.5m||b.5m||0)}6(!c.3Q&&(c.5k||c.5j))c.3Q=c.5k||c.5j;6(!c.5i&&c.5g)c.5i=c.5g;6(!c.3Q&&c.1X)c.3Q=(c.1X&1?1:(c.1X&2?3:(c.1X&4?2:0)));F c}};C.15.14({3l:9(c,a,b){F c=="5f"?7.5e(c,a,b):7.J(9(){C.1c.1r(7,c,b||a,b&&a)})},5e:9(d,b,c){F 7.J(9(){C.1c.1r(7,d,9(a){C(7).49(a);F(c||b).T(7,1a)},c&&b)})},49:9(a,b){F 7.J(9(){C.1c.23(7,a,b)})},1z:9(a,b){F 7.J(9(){C.1c.1z(a,b,7)})},1W:9(){E a=1a;F 7.4h(9(e){7.3T=0==7.3T?1:0;e.2u();F a[7.3T].T(7,[e])||M})},7p:9(f,g){9 3U(e){E p=e.4g;20(p&&p!=7)2g{p=p.P}2h(e){p=7};6(p==7)F M;F(e.L=="3W"?f:g).T(7,[e])}F 7.3W(3U).5b(3U)},1L:9(f){5a();6(C.36)f.T(R,[C]);G C.2C.Y(9(){F f.T(7,[C])});F 7}});C.14({36:M,2C:[],1L:9(){6(!C.36){C.36=O;6(C.2C){C.J(C.2C,9(){7.T(R)});C.2C=K}6(C.N.3J||C.N.2t)R.4j("59",C.1L,M);6(!19.7m.H)C(19).2f(9(){C("#4b").23()})}}});C.J(("7l,7k,2f,7j,7i,5f,4h,7g,"+"7f,7d,7c,3W,5b,7b,2b,"+"4u,7a,79,78,3f").2M(","),9(i,o){C.15[o]=9(f){F f?7.3l(o,f):7.1z(o)}});E w=M;9 5a(){6(w)F;w=O;6(C.N.3J||C.N.2t)R.4m("59",C.1L,M);G 6(C.N.12){R.75("<73"+"72 2j=4b 70=O "+"32=//:><\\/33>");E a=R.37("4b");6(a)a.6Z=9(){6(R.3d!="1x")F;C.1L()};a=K}G 6(C.N.1H)C.48=3t(9(){6(R.3d=="6Y"||R.3d=="1x"){47(C.48);C.48=K;C.1L()}},10);C.1c.1r(19,"2f",C.1L)}C.15.14({6X:9(c,b,a){7.2f(c,b,a,1)},2f:9(g,e,c,d){6(C.1g(g))F 7.3l("2f",g);c=c||9(){};E f="46";6(e)6(C.1g(e)){c=e;e=K}G{e=C.2O(e);f="55"}E h=7;C.31({1G:g,L:f,V:e,2F:d,1x:9(a,b){6(b=="1U"||!d&&b=="54")h.5W(a.43);4x(9(){h.J(c,[a.43,b,a])},13)}});F 7},6W:9(){F C.2O(7)},6V:9(){}});C.J("53,52,51,50,4Z,5h".2M(","),9(i,o){C.15[o]=9(f){F 7.3l(o,f)}});C.14({21:9(e,c,a,d,b){6(C.1g(c)){a=c;c=K}F C.31({L:"46",1G:e,V:c,1U:a,3G:d,2F:b})},6U:9(d,b,a,c){F C.21(d,b,a,c,1)},6T:9(b,a){F C.21(b,K,a,"33")},77:9(c,b,a){F C.21(c,b,a,"56")},6S:9(d,b,a,c){6(C.1g(b)){a=b;b={}}F C.31({L:"55",1G:d,V:b,1U:a,3G:c})},6R:9(a){C.3u.1Q=a},6Q:9(a){C.14(C.3u,a)},3u:{1D:O,L:"46",1Q:0,4Y:"6P/x-6O-38-6N",4X:O,2w:O,V:K},3h:{},31:9(s){s=C.14(O,s,C.14(O,{},C.3u,s));6(s.V){6(s.4X&&1f s.V!="1E")s.V=C.2O(s.V);6(s.L.2D()=="21"){s.1G+=(s.1G.U("?")>-1?"&":"?")+s.V;s.V=K}}6(s.1D&&!C.40++)C.1c.1z("53");E f=M;E h=19.4W?1s 4W("6M.6K"):1s 58();h.6J(s.L,s.1G,s.2w);6(s.V)h.4c("7r-7s",s.4Y);6(s.2F)h.4c("6G-3Z-6E",C.3h[s.1G]||"7w, 6C 7y 6B 4J:4J:4J 6z");h.4c("X-7D-7E","58");6(s.4U)s.4U(h);6(s.1D)C.1c.1z("5h",[h,s]);E g=9(d){6(!f&&h&&(h.3d==4||d=="1Q")){f=O;6(i){47(i);i=K}E c=d=="1Q"&&"1Q"||!C.5n(h)&&"3f"||s.2F&&C.5s(h,s.1G)&&"54"||"1U";6(c=="1U"){2g{E a=C.5q(h,s.3G)}2h(e){c="4I"}}6(c=="1U"){E b;2g{b=h.4f("4S-3Z")}2h(e){}6(s.2F&&b)C.3h[s.1G]=b;6(s.1U)s.1U(a,c);6(s.1D)C.1c.1z("4Z",[h,s])}G C.3X(s,h,c);6(s.1D)C.1c.1z("51",[h,s]);6(s.1D&&!--C.40)C.1c.1z("52");6(s.1x)s.1x(h,c);6(s.2w)h=K}};6(s.2w){E i=3t(g,13);6(s.1Q>0)4x(9(){6(h){h.6w();6(!f)g("1Q")}},s.1Q)}2g{h.6v(s.V)}2h(e){C.3X(s,h,K,e)}6(!s.2w)g();F h},3X:9(s,a,b,e){6(s.3f)s.3f(a,b,e);6(s.1D)C.1c.1z("50",[a,s,e])},40:0,5n:9(r){2g{F!r.26&&6t.6r=="4v:"||(r.26>=4R&&r.26<6q)||r.26==5z||C.N.1H&&r.26==Q}2h(e){}F M},5s:9(a,c){2g{E b=a.4f("4S-3Z");F a.26==5z||b==C.3h[c]||C.N.1H&&a.26==Q}2h(e){}F M},5q:9(r,a){E b=r.4f("6o-L");E c=a=="5F"||!a&&b&&b.U("5F")>=0;V=c?r.7W:r.43;6(c&&V.3D.4y=="4I")7X"4I";6(a=="33")C.4E(V);6(a=="56")V=2T("("+V+")");F V},2O:9(a){E s=[];6(a.1b==1K||a.3w)C.J(a,9(){s.Y(2y(7.6l)+"="+2y(7.2A))});G I(E j 17 a)6(a[j]&&a[j].1b==1K)C.J(a[j],9(){s.Y(2y(j)+"="+2y(7))});G s.Y(2y(j)+"="+2y(a[j]));F s.5w("&")}});C.15.14({1o:9(b,a){F b?7.1B({1u:"1o",24:"1o",1e:"1o"},b,a):7.1l(":1C").J(9(){7.S.11=7.2r?7.2r:"";6(C.1h(7,"11")=="1T")7.S.11="2m"}).3L()},1p:9(b,a){F b?7.1B({1u:"1p",24:"1p",1e:"1p"},b,a):7.1l(":4N").J(9(){7.2r=7.2r||C.1h(7,"11");6(7.2r=="1T")7.2r="2m";7.S.11="1T"}).3L()},5O:C.15.1W,1W:9(a,b){F C.1g(a)&&C.1g(b)?7.5O(a,b):a?7.1B({1u:"1W",24:"1W",1e:"1W"},a,b):7.J(9(){C(7)[C(7).3y(":1C")?"1o":"1p"]()})},6i:9(b,a){F 7.1B({1u:"1o"},b,a)},6h:9(b,a){F 7.1B({1u:"1p"},b,a)},6g:9(b,a){F 7.1B({1u:"1W"},b,a)},6f:9(b,a){F 7.1B({1e:"1o"},b,a)},89:9(b,a){F 7.1B({1e:"1p"},b,a)},6e:9(c,a,b){F 7.1B({1e:a},c,b)},1B:9(d,h,f,g){F 7.1n(9(){E c=C(7).3y(":1C"),1Z=C.5V(h,f,g),5U=7;I(E p 17 d){6(d[p]=="1p"&&c||d[p]=="1o"&&!c)F C.1g(1Z.1x)&&1Z.1x.T(7);6(p=="1u"||p=="24"){1Z.11=C.1h(7,"11");1Z.2z=7.S.2z}}6(1Z.2z!=K)7.S.2z="1C";7.2v=C.14({},d);C.J(d,9(a,b){E e=1s C.2Y(5U,1Z,a);6(b.1b==3x)e.3R(e.1Y()||0,b);G e[b=="1W"?c?"1o":"1p":b](d)});F O})},1n:9(a,b){6(!b){b=a;a="2Y"}F 7.J(9(){6(!7.1n)7.1n={};6(!7.1n[a])7.1n[a]=[];7.1n[a].Y(b);6(7.1n[a].H==1)b.T(7)})}});C.14({5V:9(b,a,c){E d=b&&b.1b==8G?b:{1x:c||!c&&a||C.1g(b)&&b,1N:b,35:c&&a||a&&a.1b!=8F&&a};d.1N=(d.1N&&d.1N.1b==3x?d.1N:{8D:8C,8B:4R}[d.1N])||8A;d.2U=d.1x;d.1x=9(){C.68(7,"2Y");6(C.1g(d.2U))d.2U.T(7)};F d},35:{62:9(p,n,b,a){F b+a*p},4H:9(p,n,b,a){F((-67.8z(p*67.8y)/2)+0.5)*a+b}},1n:{},68:9(b,a){a=a||"2Y";6(b.1n&&b.1n[a]){b.1n[a].4s();E f=b.1n[a][0];6(f)f.T(b)}},3N:[],2Y:9(f,e,g){E z=7;E y=f.S;z.a=9(){6(e.3q)e.3q.T(f,[z.2x]);6(g=="1e")C.1j(y,"1e",z.2x);G{y[g]=5K(z.2x)+"4t";6(g=="1u"||g=="24")y.11="2m"}};z.65=9(){F 3m(C.1h(f,g))};z.1Y=9(){E r=3m(C.34(f,g));F r&&r>-8v?r:z.65()};z.3R=9(c,b){z.4M=(1s 64()).63();z.2x=c;z.a();C.3N.Y(9(){F z.3q(c,b)});6(C.3N.H==1){E d=3t(9(){E a=C.3N;I(E i=0;ie.1N+z.4M){z.2x=c;z.a();6(f.2v)f.2v[g]=O;E b=O;I(E i 17 f.2v)6(f.2v[i]!==O)b=M;6(b){6(e.11!=K){y.2z=e.2z;y.11=e.11;6(C.1h(f,"11")=="1T")y.11="2m"}6(e.1p)y.11="1T";6(e.1p||e.1o)I(E p 17 f.2v)C.1j(y,p,f.2i[p])}6(b&&C.1g(e.1x))e.1x.T(f);F M}G{E n=t-7.4M;E p=n/e.1N;z.2x=C.35[e.35||(C.35.4H?"4H":"62")](p,n,a,(c-a),e.1N);z.a()}F O}}})})();',62,541,'||||||if|this||function|||||||||||||||||||||||||||||||var|return|else|length|for|each|null|type|false|browser|true|parentNode|undefined|document|style|apply|indexOf|data|nodeName||push|||display|msie||extend|fn|jQuery|in|className|window|arguments|constructor|event|test|opacity|typeof|isFunction|css|events|attr|firstChild|filter|div|queue|show|hide|mergeNum|add|new|nodeType|height|replace|tbody|complete|handle|trigger|table|animate|hidden|global|string|find|url|safari|toUpperCase|break|Array|ready|al|duration|pushStack|tb|timeout|stack|target|none|success|swap|toggle|button|cur|opt|while|get|guid|remove|width|nth|status|checked|selected|merge|nextSibling|select|ret|exec|args|load|try|catch|orig|id|match|_|block||rl|insertBefore|done|oldblock|trim|opera|preventDefault|curAnim|async|now|encodeURIComponent|overflow|value|grep|readyList|toLowerCase|color|ifModified|val|first|ownerDocument|domManip|substr|defaultView|split|has|param|last|text|multiFilter|call|eval|old|makeArray|innerHTML|stopPropagation|fx|childNodes|disabled|ajax|src|script|curCSS|easing|isReady|getElementById|form|input|float|getComputedStyle|clean|readyState|removeChild|error|static|lastModified|checkbox|selectedIndex|position|bind|parseFloat|String|oWidth|oHeight|step|on|toString|setInterval|ajaxSettings|prototype|jquery|Number|is|child|ol|cloneNode|RegExp|documentElement|isXMLDoc|getAttribute|dataType|append|styleFloat|mozilla|empty|end|map|timers|tr|el|which|custom|slice|lastToggle|handleHover|visibility|mouseover|handleError|lastChild|Modified|active|currentStyle|unshift|responseText|getPropertyValue|index|GET|clearInterval|safariTimer|unbind|init|__ie_init|setRequestHeader|unique|radio|getResponseHeader|relatedTarget|click|fix|removeEventListener|delete|handler|addEventListener|triggered|nodeIndex|appendChild|props|classFilter|shift|px|submit|file|expr|setTimeout|tagName|body|sibling|previousSibling|parents|deep|globalEval|fromElement|cssFloat|swing|parsererror|00|inArray|getElementsByTagName|startTime|visible|num|object|prop|200|Last|colgroup|beforeSend|fieldset|ActiveXObject|processData|contentType|ajaxSuccess|ajaxError|ajaxComplete|ajaxStop|ajaxStart|notmodified|POST|json|appendTo|XMLHttpRequest|DOMContentLoaded|bindReady|mouseout|prevObject|removeAttr|one|unload|ctrlKey|ajaxSend|metaKey|keyCode|charCode|not|scrollTop|httpSuccess|scrollLeft|pageX|httpData|srcElement|httpNotModified|after|before|prepend|join|parse|zoom|304|reset|image|password|odd|even|xml|quickClass|quickID|quickChild|setArray|parseInt|contains|gt|execScript|_toggle|lt|eq|href|nodeValue|alpha|self|speed|html|continue|parent|textContent|createTextNode|webkit|linear|getTime|Date|max|clientX|Math|dequeue|fl|createElement|version|100|NaN|fadeTo|fadeIn|slideToggle|slideUp|slideDown|setAttribute|getAttributeNode|name|method|action|content|cssText|300|protocol|FORM|location|options|send|abort|col|th|GMT|td|1970|01|cap|Since|colg|If|tfoot|thead|open|XMLHTTP|leg|Microsoft|urlencoded|www|application|ajaxSetup|ajaxTimeout|post|getScript|getIfModified|evalScripts|serialize|loadIfModified|loaded|onreadystatechange|defer|clientWidth|ipt|scr|clientHeight|write|relative|getJSON|keyup|keypress|keydown|change|mousemove|mouseup|left|mousedown|dblclick|right|scroll|resize|focus|blur|frames|absolute|clone|hover|offsetWidth|Content|Type|offsetHeight|Width|clientY|Thu|border|Jan|pageY|padding|Left|toElement|Requested|With|Right|Bottom|cancelBubble|returnValue|Top|size|detachEvent|attachEvent|substring|line|textarea|weight|enabled|font|innerText|only|uFFFF|responseXML|throw|u0128|417|toggleClass|removeClass|wrap|addClass|removeAttribute|insertAfter|prependTo|children|siblings|fadeOut|noConflict|prev|next|Boolean|maxLength|maxlength|readOnly|readonly|class|htmlFor|CSS1Compat|compatMode|boxModel|compatible|ie|ra|it|1px|rv|splice|userAgent|10000|navigator|concat|PI|cos|400|fast|600|slow|reverse|Function|Object|array|ig'.split('|'),0,{})) \ No newline at end of file diff --git a/lib/genshi/examples/tutorial/geddit/static/layout.css b/lib/genshi/examples/tutorial/geddit/static/layout.css new file mode 100644 --- /dev/null +++ b/lib/genshi/examples/tutorial/geddit/static/layout.css @@ -0,0 +1,42 @@ +html, body { margin: 0; padding: 0; } +body { background: #ddd; color: #333; font: normal 90%/1.3 Arial,Helvetica,sans-serif; } +:link, :visited { color: #c10000; text-decoration: none; } +:link:hover, :visited:hover { text-decoration: underline; } +:link img, :visited img { border: none; } +h1 { color: #666; + font: normal xx-large/1.5 Georgia,serif; margin: 0 0 .5em; +} +blockquote { font-style: italic; } +form table { margin-bottom: 1em; } +form table tbody th { font-weight: normal; padding-top: .3em; text-align: right; + vertical-align: top; +} +form label { color: #666; font-size: 90%; } + +#wrap { background: #fff; width: 600px; margin: 30px auto; } +#content { border-left: 10px solid #b00; min-height: 240px; padding: 10px; } +#header img { float: right; margin-top: -21px; margin-right: -37px; } + +#footer { background: #4A4D4D; border-top: 2px solid #333; font-size: x-small; + padding: 3px; text-align: center; +} +#footer hr { display: none; } +.legalese { color: #999; margin: 0; } + +ol.links li .info { font-size: 85%; } +ol.links li .info :link, ol.links li .info :visited { color: #666; } + +ul.comments { list-style: none; margin: 1em 0; padding: 0 0 0 1em; } +ul.comments li { color: #999; margin: 0 0 1em; } +ul.comments blockquote { color: #333; font-style: normal; margin: 0; + padding: 0; +} +.action:link, .action:visited { background: #f3f3f3; border: 1px outset #ddd; + color: #666; font-size: 90%; padding: 0 .3em; +} +.action:link:hover, .action:visited:hover { background: #e8e8e8; + border-color: #aaa; color: #000; text-decoration: none; +} + +form p.hint { color: #666; font-size: 90%; font-style: italic; margin: 0; } +form .error { color: #b00; } diff --git a/lib/genshi/examples/tutorial/geddit/static/logo.gif b/lib/genshi/examples/tutorial/geddit/static/logo.gif new file mode 100644 index 0000000000000000000000000000000000000000..60267e548999abf7ddb818be82c0e9397ef51b7b GIT binary patch [cut] diff --git a/lib/genshi/examples/tutorial/geddit/templates/_comment.html b/lib/genshi/examples/tutorial/geddit/templates/_comment.html new file mode 100644 --- /dev/null +++ b/lib/genshi/examples/tutorial/geddit/templates/_comment.html @@ -0,0 +1,5 @@ + +

+ ${comment.username} at ${comment.time.strftime('%x %X')} +

${HTML(comment.content)}

diff --git a/lib/genshi/examples/tutorial/geddit/templates/_form.html b/lib/genshi/examples/tutorial/geddit/templates/_form.html new file mode 100644 --- /dev/null +++ b/lib/genshi/examples/tutorial/geddit/templates/_form.html @@ -0,0 +1,24 @@ +

+ + + + + + + + + +

Your name:	+ + ${errors.username} +
Comment:	+ + ${errors.content} + You can use HTML tags here for formatting. +
	+ + +

diff --git a/lib/genshi/examples/tutorial/geddit/templates/comment.html b/lib/genshi/examples/tutorial/geddit/templates/comment.html new file mode 100644 --- /dev/null +++ b/lib/genshi/examples/tutorial/geddit/templates/comment.html @@ -0,0 +1,18 @@ + + + + + Comment on “${link.title}” + + +

Comment on “${link.title}”

+ In reply to ${comment.username} + at ${comment.time.strftime('%x %X')}: +

${comment.content}

+ + + diff --git a/lib/genshi/examples/tutorial/geddit/templates/index.html b/lib/genshi/examples/tutorial/geddit/templates/index.html new file mode 100644 --- /dev/null +++ b/lib/genshi/examples/tutorial/geddit/templates/index.html @@ -0,0 +1,28 @@ + + + + + News + + + +

News

+ +

+ ${link.title} + posted by ${link.username} at ${link.time.strftime('%x %X')} +
+ + ${len(link.comments)} comments + +
+

+ +

Submit new link

+ + diff --git a/lib/genshi/examples/tutorial/geddit/templates/index.xml b/lib/genshi/examples/tutorial/geddit/templates/index.xml new file mode 100644 --- /dev/null +++ b/lib/genshi/examples/tutorial/geddit/templates/index.xml @@ -0,0 +1,23 @@ + + + + Geddit News + + + + ${links[0].time.isoformat()} + + + ${link.url} + + + ${url('/info/%s/' % link.id)} + + ${link.username} + + ${link.time.isoformat()} +

${link.title}

+ + + diff --git a/lib/genshi/examples/tutorial/geddit/templates/info.html b/lib/genshi/examples/tutorial/geddit/templates/info.html new file mode 100644 --- /dev/null +++ b/lib/genshi/examples/tutorial/geddit/templates/info.html @@ -0,0 +1,62 @@ + + + + + ${link.title} + + + + +

${link.title}

+ ${link.url}
+ posted by ${link.username} at ${link.time.strftime('%x %X')}
+ +

+ +

Add comment

+ + diff --git a/lib/genshi/examples/tutorial/geddit/templates/info.xml b/lib/genshi/examples/tutorial/geddit/templates/info.xml new file mode 100644 --- /dev/null +++ b/lib/genshi/examples/tutorial/geddit/templates/info.xml @@ -0,0 +1,28 @@ + + + + Geddit: ${link.title} + + + + + ${time.isoformat()} + + + + + Comment ${len(link.comments) - idx} on “${link.title}” + + ${url('/info/%s/' % link.id)}#comment${idx} + + ${comment.username} + + ${comment.time.isoformat()} +

+ ${HTML(comment.content)} +

+ + + diff --git a/lib/genshi/examples/tutorial/geddit/templates/layout.html b/lib/genshi/examples/tutorial/geddit/templates/layout.html new file mode 100644 --- /dev/null +++ b/lib/genshi/examples/tutorial/geddit/templates/layout.html @@ -0,0 +1,31 @@ + + + + + + + Geddit<py:if test="title">: ${title}</py:if> + + + + ${select('*[local-name()!="title"]')} + + + + +

+ +

+ ${select('*|text()')} +

+ +

+ + + diff --git a/lib/genshi/examples/tutorial/geddit/templates/submit.html b/lib/genshi/examples/tutorial/geddit/templates/submit.html new file mode 100644 --- /dev/null +++ b/lib/genshi/examples/tutorial/geddit/templates/submit.html @@ -0,0 +1,41 @@ + + + + + Submit new link + + +

Submit new link

+ +

+ + + + + + + + + + + + +

Your name:	+ + ${errors.username} +
Link URL:	+ + ${errors.url} +
Title:	+ + ${errors.title} +
	+ + +

+ + + diff --git a/lib/genshi/examples_to_py3k.sh b/lib/genshi/examples_to_py3k.sh new file mode 100644 --- /dev/null +++ b/lib/genshi/examples_to_py3k.sh @@ -0,0 +1,8 @@ +#!/bin/sh +# +# Script to run 2to3 on files not covered by setup.py +# +export PYTHONIOENCODING=utf8 + +# General 2to3 run +2to3 -w --no-diffs examples/ diff --git a/lib/genshi/fixes/__init__.py b/lib/genshi/fixes/__init__.py new file mode 100644 diff --git a/lib/genshi/fixes/fix_unicode_in_strings.py b/lib/genshi/fixes/fix_unicode_in_strings.py new file mode 100644 --- /dev/null +++ b/lib/genshi/fixes/fix_unicode_in_strings.py @@ -0,0 +1,17 @@ +"""Fixer that changes expressions inside strings literals from u"..." to "...". + +""" + +import re +from lib2to3 import fixer_base + +_literal_re = re.compile(r"(.+?)\b[uU]([rR]?[\'\"])") + +class FixUnicodeInStrings(fixer_base.BaseFix): + + PATTERN = "STRING" + + def transform(self, node, results): + new = node.clone() + new.value = _literal_re.sub(r"\1\2", new.value) + return new diff --git a/lib/genshi/genshi/__init__.py b/lib/genshi/genshi/__init__.py new file mode 100644 --- /dev/null +++ b/lib/genshi/genshi/__init__.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2006-2009 Edgewall Software +# All rights reserved. +# +# This software is licensed as described in the file COPYING, which +# you should have received as part of this distribution. The terms +# are also available at http://genshi.edgewall.org/wiki/License. +# +# This software consists of voluntary contributions made by many +# individuals. For the exact contribution history, see the revision +# history and logs, available at http://genshi.edgewall.org/log/. + +"""This package provides various means for generating and processing web markup +(XML or HTML). + +The design is centered around the concept of streams of markup events (similar +in concept to SAX parsing events) which can be processed in a uniform manner +independently of where or how they are produced. +""" + +__docformat__ = 'restructuredtext en' +__version__ = '0.7' + +from genshi.core import * +from genshi.input import ParseError, XML, HTML diff --git a/lib/genshi/genshi/_speedups.c b/lib/genshi/genshi/_speedups.c new file mode 100644 --- /dev/null +++ b/lib/genshi/genshi/_speedups.c @@ -0,0 +1,702 @@ +/* + * Copyright (C) 2006-2008 Edgewall Software + * All rights reserved. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at http://genshi.edgewall.org/wiki/License. + * + * This software consists of voluntary contributions made by many + * individuals. For the exact contribution history, see the revision + * history and logs, available at http://genshi.edgewall.org/log/. + */ + +#include +#include + +#if PY_MAJOR_VERSION > 2 +# define IS_PY3K +#elif PY_VERSION_HEX < 0x02050000 && !defined(PY_SSIZE_T_MIN) + typedef int Py_ssize_t; +# define PY_SSIZE_T_MAX INT_MAX +# define PY_SSIZE_T_MIN INT_MIN +#endif + +/* We only use Unicode Strings in this module */ +#ifndef IS_PY3K +# define PyObject_Str PyObject_Unicode +#endif + +static PyObject *amp1, *amp2, *lt1, *lt2, *gt1, *gt2, *qt1, *qt2; +static PyObject *stripentities, *striptags; + +static void +init_constants(void) +{ + PyObject *util = PyImport_ImportModule("genshi.util"); + stripentities = PyObject_GetAttrString(util, "stripentities"); + striptags = PyObject_GetAttrString(util, "striptags"); + Py_DECREF(util); + + amp1 = PyUnicode_DecodeASCII("&", 1, NULL); + amp2 = PyUnicode_DecodeASCII("&", 5, NULL); + lt1 = PyUnicode_DecodeASCII("<", 1, NULL); + lt2 = PyUnicode_DecodeASCII("<", 4, NULL); + gt1 = PyUnicode_DecodeASCII(">", 1, NULL); + gt2 = PyUnicode_DecodeASCII(">", 4, NULL); + qt1 = PyUnicode_DecodeASCII("\"", 1, NULL); + qt2 = PyUnicode_DecodeASCII(""", 5, NULL); +} + +/* Markup class */ + +PyTypeObject MarkupType; /* declared later */ + +PyDoc_STRVAR(Markup__doc__, +"Marks a string as being safe for inclusion in HTML/XML output without\n\ +needing to be escaped."); + +static PyObject * +escape(PyObject *text, int quotes) +{ + PyObject *args, *ret; + PyUnicodeObject *in, *out; + Py_UNICODE *inp, *outp; + int len, inn, outn; + + if (PyObject_TypeCheck(text, &MarkupType)) { + Py_INCREF(text); + return text; + } + if (PyObject_HasAttrString(text, "__html__")) { + ret = PyObject_CallMethod(text, "__html__", NULL); + args = PyTuple_New(1); + if (args == NULL) { + Py_DECREF(ret); + return NULL; + } + PyTuple_SET_ITEM(args, 0, ret); + ret = MarkupType.tp_new(&MarkupType, args, NULL); + Py_DECREF(args); + return ret; + } + in = (PyUnicodeObject *) PyObject_Str(text); + if (in == NULL) { + return NULL; + } + /* First we need to figure out how long the escaped string will be */ + len = inn = 0; + inp = in->str; + while (*(inp) || in->length > inp - in->str) { + switch (*inp++) { + case '&': len += 5; inn++; break; + case '"': len += quotes ? 5 : 1; inn += quotes ? 1 : 0; break; + case '<': + case '>': len += 4; inn++; break; + default: len++; + } + } + + /* Do we need to escape anything at all? */ + if (!inn) { + args = PyTuple_New(1); + if (args == NULL) { + Py_DECREF((PyObject *) in); + return NULL; + } + PyTuple_SET_ITEM(args, 0, (PyObject *) in); + ret = MarkupType.tp_new(&MarkupType, args, NULL); + Py_DECREF(args); + return ret; + } + + out = (PyUnicodeObject*) PyUnicode_FromUnicode(NULL, len); + if (out == NULL) { + Py_DECREF((PyObject *) in); + return NULL; + } + + outn = 0; + inp = in->str; + outp = out->str; + while (*(inp) || in->length > inp - in->str) { + if (outn == inn) { + /* copy rest of string if we have already replaced everything */ + Py_UNICODE_COPY(outp, inp, in->length - (inp - in->str)); + break; + } + switch (*inp) { + case '&': + Py_UNICODE_COPY(outp, ((PyUnicodeObject *) amp2)->str, 5); + outp += 5; + outn++; + break; + case '"': + if (quotes) { + Py_UNICODE_COPY(outp, ((PyUnicodeObject *) qt2)->str, 5); + outp += 5; + outn++; + } else { + *outp++ = *inp; + } + break; + case '<': + Py_UNICODE_COPY(outp, ((PyUnicodeObject *) lt2)->str, 4); + outp += 4; + outn++; + break; + case '>': + Py_UNICODE_COPY(outp, ((PyUnicodeObject *) gt2)->str, 4); + outp += 4; + outn++; + break; + default: + *outp++ = *inp; + } + inp++; + } + + Py_DECREF((PyObject *) in); + + args = PyTuple_New(1); + if (args == NULL) { + Py_DECREF((PyObject *) out); + return NULL; + } + PyTuple_SET_ITEM(args, 0, (PyObject *) out); + ret = MarkupType.tp_new(&MarkupType, args, NULL); + Py_DECREF(args); + return ret; +} + +PyDoc_STRVAR(escape__doc__, +"Create a Markup instance from a string and escape special characters\n\ +it may contain (<, >, & and \").\n\ +\n\ +>>> escape('\"1 < 2\"')\n\ +\n\ +\n\ +If the `quotes` parameter is set to `False`, the \" character is left\n\ +as is. Escaping quotes is generally only required for strings that are\n\ +to be used in attribute values.\n\ +\n\ +>>> escape('\"1 < 2\"', quotes=False)\n\ +\n\ +\n\ +:param text: the text to escape\n\ +:param quotes: if ``True``, double quote characters are escaped in\n\ + addition to the other special characters\n\ +:return: the escaped `Markup` string\n\ +:rtype: `Markup`\n\ +"); + +static PyObject * +Markup_escape(PyTypeObject* type, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"text", "quotes", 0}; + PyObject *text = NULL; + char quotes = 1; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|b", kwlist, &text, "es)) { + return NULL; + } + if (PyObject_Not(text)) { + args = PyTuple_New(0); + if (args == NULL) + return NULL; + text = type->tp_new(type, args, NULL); + Py_DECREF(args); + return text; + } + if (PyObject_TypeCheck(text, type)) { + Py_INCREF(text); + return text; + } + return escape(text, quotes); +} + +static PyObject * +Markup_html(PyObject *self) +{ + Py_INCREF(self); + return self; +} + +PyDoc_STRVAR(join__doc__, +"Return a `Markup` object which is the concatenation of the strings\n\ +in the given sequence, where this `Markup` object is the separator\n\ +between the joined elements.\n\ +\n\ +Any element in the sequence that is not a `Markup` instance is\n\ +automatically escaped.\n\ +\n\ +:param seq: the sequence of strings to join\n\ +:param escape_quotes: whether double quote characters in the elements\n\ + should be escaped\n\ +:return: the joined `Markup` object\n\ +:rtype: `Markup`\n\ +:see: `escape`\n\ +"); + +static PyObject * +Markup_join(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"seq", "escape_quotes", 0}; + PyObject *seq = NULL, *seq2, *tmp, *tmp2; + char quotes = 1; + Py_ssize_t n; + int i; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|b", kwlist, &seq, "es)) { + return NULL; + } + if (!PySequence_Check(seq)) { + return NULL; + } + n = PySequence_Size(seq); + if (n < 0) { + return NULL; + } + seq2 = PyTuple_New(n); + if (seq2 == NULL) { + return NULL; + } + for (i = 0; i < n; i++) { + tmp = PySequence_GetItem(seq, i); + if (tmp == NULL) { + Py_DECREF(seq2); + return NULL; + } + tmp2 = escape(tmp, quotes); + if (tmp2 == NULL) { + Py_DECREF(seq2); + return NULL; + } + PyTuple_SET_ITEM(seq2, i, tmp2); + Py_DECREF(tmp); + } + tmp = PyUnicode_Join(self, seq2); + Py_DECREF(seq2); + if (tmp == NULL) + return NULL; + args = PyTuple_New(1); + if (args == NULL) { + Py_DECREF(tmp); + return NULL; + } + PyTuple_SET_ITEM(args, 0, tmp); + tmp = MarkupType.tp_new(&MarkupType, args, NULL); + Py_DECREF(args); + return tmp; +} + +static PyObject * +Markup_add(PyObject *self, PyObject *other) +{ + PyObject *tmp, *tmp2, *args, *ret; + if (PyObject_TypeCheck(self, &MarkupType)) { + tmp = escape(other, 1); + if (tmp == NULL) + return NULL; + tmp2 = PyUnicode_Concat(self, tmp); + } else { // __radd__ + tmp = escape(self, 1); + if (tmp == NULL) + return NULL; + tmp2 = PyUnicode_Concat(tmp, other); + } + Py_DECREF(tmp); + if (tmp2 == NULL) + return NULL; + args = PyTuple_New(1); + if (args == NULL) { + Py_DECREF(tmp2); + return NULL; + } + PyTuple_SET_ITEM(args, 0, tmp2); + ret = MarkupType.tp_new(&MarkupType, args, NULL); + Py_DECREF(args); + return ret; +} + +static PyObject * +Markup_mod(PyObject *self, PyObject *args) +{ + PyObject *tmp, *tmp2, *ret, *args2; + int i; + Py_ssize_t nargs = 0; + PyObject *kwds = NULL; + + if (PyDict_Check(args)) { + kwds = args; + } + if (kwds && PyDict_Size(kwds)) { + PyObject *kwcopy, *key, *value; + Py_ssize_t pos = 0; + + kwcopy = PyDict_Copy( kwds ); + if (kwcopy == NULL) { + return NULL; + } + while (PyDict_Next(kwcopy, &pos, &key, &value)) { + tmp = escape(value, 1); + if (tmp == NULL) { + Py_DECREF(kwcopy); + return NULL; + } + if (PyDict_SetItem(kwcopy, key, tmp) < 0) { + Py_DECREF(tmp); + Py_DECREF(kwcopy); + return NULL; + } + } + tmp = PyUnicode_Format(self, kwcopy); + Py_DECREF(kwcopy); + if (tmp == NULL) { + return NULL; + } + } else if (PyTuple_Check(args)) { + nargs = PyTuple_GET_SIZE(args); + args2 = PyTuple_New(nargs); + if (args2 == NULL) { + return NULL; + } + for (i = 0; i < nargs; i++) { + tmp = escape(PyTuple_GET_ITEM(args, i), 1); + if (tmp == NULL) { + Py_DECREF(args2); + return NULL; + } + PyTuple_SET_ITEM(args2, i, tmp); + } + tmp = PyUnicode_Format(self, args2); + Py_DECREF(args2); + if (tmp == NULL) { + return NULL; + } + } else { + tmp2 = escape(args, 1); + if (tmp2 == NULL) { + return NULL; + } + tmp = PyUnicode_Format(self, tmp2); + Py_DECREF(tmp2); + if (tmp == NULL) { + return NULL; + } + } + args = PyTuple_New(1); + if (args == NULL) { + Py_DECREF(tmp); + return NULL; + } + PyTuple_SET_ITEM(args, 0, tmp); + ret = PyUnicode_Type.tp_new(&MarkupType, args, NULL); + Py_DECREF(args); + return ret; +} + +static PyObject * +Markup_mul(PyObject *self, PyObject *num) +{ + PyObject *unicode, *result, *args; + + if (PyObject_TypeCheck(self, &MarkupType)) { + unicode = PyObject_Str(self); + if (unicode == NULL) return NULL; + result = PyNumber_Multiply(unicode, num); + } else { // __rmul__ + unicode = PyObject_Str(num); + if (unicode == NULL) return NULL; + result = PyNumber_Multiply(unicode, self); + } + Py_DECREF(unicode); + + if (result == NULL) return NULL; + args = PyTuple_New(1); + if (args == NULL) { + Py_DECREF(result); + return NULL; + } + PyTuple_SET_ITEM(args, 0, result); + result = PyUnicode_Type.tp_new(&MarkupType, args, NULL); + Py_DECREF(args); + + return result; +} + +static PyObject * +Markup_repr(PyObject *self) +{ + PyObject *format, *result, *args; + +#ifdef IS_PY3K + format = PyUnicode_FromString(""); +#else + format = PyString_FromString(""); +#endif + if (format == NULL) return NULL; + result = PyObject_Str(self); + if (result == NULL) { + Py_DECREF(format); + return NULL; + } + args = PyTuple_New(1); + if (args == NULL) { + Py_DECREF(format); + Py_DECREF(result); + return NULL; + } + PyTuple_SET_ITEM(args, 0, result); +#ifdef IS_PY3K + result = PyUnicode_Format(format, args); +#else + result = PyString_Format(format, args); +#endif + Py_DECREF(format); + Py_DECREF(args); + return result; +} + +PyDoc_STRVAR(unescape__doc__, +"Reverse-escapes &, <, >, and \" and returns a `unicode` object.\n\ +\n\ +>>> Markup('1 < 2').unescape()\n\ +u'1 < 2'\n\ +\n\ +:return: the unescaped string\n\ +:rtype: `unicode`\n\ +:see: `genshi.core.unescape`\n\ +"); + +static PyObject * +Markup_unescape(PyObject* self) +{ + PyObject *tmp, *tmp2; + + tmp = PyUnicode_Replace(self, qt2, qt1, -1); + if (tmp == NULL) return NULL; + tmp2 = PyUnicode_Replace(tmp, gt2, gt1, -1); + Py_DECREF(tmp); + if (tmp2 == NULL) return NULL; + tmp = PyUnicode_Replace(tmp2, lt2, lt1, -1); + Py_DECREF(tmp2); + if (tmp == NULL) return NULL; + tmp2 = PyUnicode_Replace(tmp, amp2, amp1, -1); + Py_DECREF(tmp); + return tmp2; +} + +PyDoc_STRVAR(stripentities__doc__, +"Return a copy of the text with any character or numeric entities\n\ +replaced by the equivalent UTF-8 characters.\n\ +\n\ +If the `keepxmlentities` parameter is provided and evaluates to `True`,\n\ +the core XML entities (``&``, ``'``, ``>``, ``<`` and\n\ +``"``) are not stripped.\n\ +\n\ +:return: a `Markup` instance with entities removed\n\ +:rtype: `Markup`\n\ +:see: `genshi.util.stripentities`\n\ +"); + +static PyObject * +Markup_stripentities(PyObject* self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"keepxmlentities", 0}; + PyObject *result, *args2; + char keepxml = 0; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|b", kwlist, &keepxml)) { + return NULL; + } + + if (stripentities == NULL) return NULL; + result = PyObject_CallFunction(stripentities, "Ob", self, keepxml); + if (result == NULL) return NULL; + args2 = PyTuple_New(1); + if (args2 == NULL) { + Py_DECREF(result); + return NULL; + } + PyTuple_SET_ITEM(args2, 0, result); + result = MarkupType.tp_new(&MarkupType, args2, NULL); + Py_DECREF(args2); + return result; +} + +PyDoc_STRVAR(striptags__doc__, +"""Return a copy of the text with all XML/HTML tags removed.\n\ +\n\ +:return: a `Markup` instance with all tags removed\n\ +:rtype: `Markup`\n\ +:see: `genshi.util.striptags`\n\ +"); + +static PyObject * +Markup_striptags(PyObject* self) +{ + PyObject *result, *args; + + if (striptags == NULL) return NULL; + result = PyObject_CallFunction(striptags, "O", self); + if (result == NULL) return NULL; + args = PyTuple_New(1); + if (args == NULL) { + Py_DECREF(result); + return NULL; + } + PyTuple_SET_ITEM(args, 0, result); + result = MarkupType.tp_new(&MarkupType, args, NULL); + Py_DECREF(args); + return result; +} + +typedef struct { + PyUnicodeObject HEAD; +} MarkupObject; + +static PyMethodDef Markup_methods[] = { + {"__html__", (PyCFunction) Markup_html, METH_NOARGS, NULL}, + {"escape", (PyCFunction) Markup_escape, + METH_VARARGS|METH_CLASS|METH_KEYWORDS, escape__doc__}, + {"join", (PyCFunction)Markup_join, METH_VARARGS|METH_KEYWORDS, join__doc__}, + {"unescape", (PyCFunction)Markup_unescape, METH_NOARGS, unescape__doc__}, + {"stripentities", (PyCFunction) Markup_stripentities, + METH_VARARGS|METH_KEYWORDS, stripentities__doc__}, + {"striptags", (PyCFunction) Markup_striptags, METH_NOARGS, + striptags__doc__}, + {NULL} /* Sentinel */ +}; + +static PyNumberMethods Markup_as_number = { + Markup_add, /*nb_add*/ + 0, /*nb_subtract*/ + Markup_mul, /*nb_multiply*/ +#ifndef IS_PY3K + 0, /*nb_divide*/ +#endif + Markup_mod, /*nb_remainder*/ +}; + +PyTypeObject MarkupType = { +#ifdef IS_PY3K + PyVarObject_HEAD_INIT(NULL, 0) +#else + PyObject_HEAD_INIT(NULL) + 0, +#endif + "genshi._speedups.Markup", + sizeof(MarkupObject), + 0, + 0, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ +#ifdef IS_PY3K + 0, /*tp_reserved*/ +#else + 0, /*tp_compare*/ +#endif + Markup_repr, /*tp_repr*/ + &Markup_as_number, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash */ + + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + +#ifdef IS_PY3K + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_UNICODE_SUBCLASS, /*tp_flags*/ +#elif defined(Py_TPFLAGS_UNICODE_SUBCLASS) + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_CHECKTYPES | Py_TPFLAGS_UNICODE_SUBCLASS, /*tp_flags*/ +#else + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_CHECKTYPES, /*tp_flags*/ +#endif + + Markup__doc__,/*tp_doc*/ + + 0, /*tp_traverse*/ + 0, /*tp_clear*/ + + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + + /* Attribute descriptor and subclassing stuff */ + + Markup_methods,/*tp_methods*/ + 0, /*tp_members*/ + 0, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + 0, /*tp_dictoffset*/ + + 0, /*tp_init*/ + 0, /*tp_alloc will be set to PyType_GenericAlloc in module init*/ + 0, /*tp_new*/ + 0, /*tp_free Low-level free-memory routine */ + 0, /*tp_is_gc For PyObject_IS_GC */ + 0, /*tp_bases*/ + 0, /*tp_mro method resolution order */ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0 /*tp_weaklist*/ +}; + +#ifdef IS_PY3K +struct PyModuleDef module_def = { + PyModuleDef_HEAD_INIT, /*m_base*/ + "_speedups", /*m_name*/ + NULL, /*m_doc*/ + -1, /*m_size*/ + NULL, /*m_methods*/ + NULL, /*m_reload*/ + NULL, /*m_traverse*/ + NULL, /*m_clear*/ + NULL /*m_free*/ +}; + +PyObject * +PyInit__speedups(void) +#else +PyMODINIT_FUNC +init_speedups(void) +#endif +{ + PyObject *module; + + /* Workaround for quirk in Visual Studio, see + */ + MarkupType.tp_base = &PyUnicode_Type; + + if (PyType_Ready(&MarkupType) < 0) +#ifdef IS_PY3K + return NULL; +#else + return; +#endif + + init_constants(); + +#ifdef IS_PY3K + module = PyModule_Create(&module_def); +#else + module = Py_InitModule("_speedups", NULL); +#endif + Py_INCREF(&MarkupType); + PyModule_AddObject(module, "Markup", (PyObject *) &MarkupType); + +#ifdef IS_PY3K + return module; +#endif +} diff --git a/lib/genshi/genshi/builder.py b/lib/genshi/genshi/builder.py new file mode 100644 --- /dev/null +++ b/lib/genshi/genshi/builder.py @@ -0,0 +1,359 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2006-2009 Edgewall Software +# All rights reserved. +# +# This software is licensed as described in the file COPYING, which +# you should have received as part of this distribution. The terms +# are also available at http://genshi.edgewall.org/wiki/License. +# +# This software consists of voluntary contributions made by many +# individuals. For the exact contribution history, see the revision +# history and logs, available at http://genshi.edgewall.org/log/. + +"""Support for programmatically generating markup streams from Python code using +a very simple syntax. The main entry point to this module is the `tag` object +(which is actually an instance of the ``ElementFactory`` class). You should +rarely (if ever) need to directly import and use any of the other classes in +this module. + +Elements can be created using the `tag` object using attribute access. For +example: + +>>> doc = tag.p('Some text and ', tag.a('a link', href='http://example.org/'), '.') +>>> doc + + +This produces an `Element` instance which can be further modified to add child +nodes and attributes. This is done by "calling" the element: positional +arguments are added as child nodes (alternatively, the `Element.append` method +can be used for that purpose), whereas keywords arguments are added as +attributes: + +>>> doc(tag.br) + +>>> print(doc) +

Some text and a link.

+ +If an attribute name collides with a Python keyword, simply append an underscore +to the name: + +>>> doc(class_='intro') + +>>> print(doc) +

Some text and a link.

+ +As shown above, an `Element` can easily be directly rendered to XML text by +printing it or using the Python ``str()`` function. This is basically a +shortcut for converting the `Element` to a stream and serializing that +stream: + +>>> stream = doc.generate() +>>> stream #doctest: +ELLIPSIS + +>>> print(stream) +

Some text and a link.

+ + +The `tag` object also allows creating "fragments", which are basically lists +of nodes (elements or text) that don't have a parent element. This can be useful +for creating snippets of markup that are attached to a parent element later (for +example in a template). Fragments are created by calling the `tag` object, which +returns an object of type `Fragment`: + +>>> fragment = tag('Hello, ', tag.em('world'), '!') +>>> fragment + +>>> print(fragment) +Hello, world! +""" + +from genshi.core import Attrs, Markup, Namespace, QName, Stream, \ + START, END, TEXT + +__all__ = ['Fragment', 'Element', 'ElementFactory', 'tag'] +__docformat__ = 'restructuredtext en' + + +class Fragment(object): + """Represents a markup fragment, which is basically just a list of element + or text nodes. + """ + __slots__ = ['children'] + + def __init__(self): + """Create a new fragment.""" + self.children = [] + + def __add__(self, other): + return Fragment()(self, other) + + def __call__(self, *args): + """Append any positional arguments as child nodes. + + :see: `append` + """ + for arg in args: + self.append(arg) + return self + + def __iter__(self): + return self._generate() + + def __repr__(self): + return '<%s>' % type(self).__name__ + + def __str__(self): + return str(self.generate()) + + def __unicode__(self): + return unicode(self.generate()) + + def __html__(self): + return Markup(self.generate()) + + def append(self, node): + """Append an element or string as child node. + + :param node: the node to append; can be an `Element`, `Fragment`, or a + `Stream`, or a Python string or number + """ + if isinstance(node, (Stream, Element, basestring, int, float, long)): + # For objects of a known/primitive type, we avoid the check for + # whether it is iterable for better performance + self.children.append(node) + elif isinstance(node, Fragment): + self.children.extend(node.children) + elif node is not None: + try: + for child in node: + self.append(child) + except TypeError: + self.children.append(node) + + def _generate(self): + for child in self.children: + if isinstance(child, Fragment): + for event in child._generate(): + yield event + elif isinstance(child, Stream): + for event in child: + yield event + else: + if not isinstance(child, basestring): + child = unicode(child) + yield TEXT, child, (None, -1, -1) + + def generate(self): + """Return a markup event stream for the fragment. + + :rtype: `Stream` + """ + return Stream(self._generate()) + + +def _kwargs_to_attrs(kwargs): + attrs = [] + names = set() + for name, value in kwargs.items(): + name = name.rstrip('_').replace('_', '-') + if value is not None and name not in names: + attrs.append((QName(name), unicode(value))) + names.add(name) + return Attrs(attrs) + + +class Element(Fragment): + """Simple XML output generator based on the builder pattern. + + Construct XML elements by passing the tag name to the constructor: + + >>> print(Element('strong')) + + + Attributes can be specified using keyword arguments. The values of the + arguments will be converted to strings and any special XML characters + escaped: + + >>> print(Element('textarea', rows=10, cols=60)) + + >>> print(Element('span', title='1 < 2')) + <span title="1 < 2"/> + >>> print(Element('span', title='"baz"')) + <span title=""baz""/> + + The " character is escaped using a numerical entity. + The order in which attributes are rendered is undefined. + + If an attribute value evaluates to `None`, that attribute is not included + in the output: + + >>> print(Element('a', name=None)) + <a/> + + Attribute names that conflict with Python keywords can be specified by + appending an underscore: + + >>> print(Element('div', class_='warning')) + <div class="warning"/> + + Nested elements can be added to an element using item access notation. + The call notation can also be used for this and for adding attributes + using keyword arguments, as one would do in the constructor. + + >>> print(Element('ul')(Element('li'), Element('li'))) + <ul><li/><li/></ul> + >>> print(Element('a')('Label')) + <a>Label</a> + >>> print(Element('a')('Label', href="target")) + <a href="target">Label</a> + + Text nodes can be nested in an element by adding strings instead of + elements. Any special characters in the strings are escaped automatically: + + >>> print(Element('em')('Hello world')) + <em>Hello world</em> + >>> print(Element('em')(42)) + <em>42</em> + >>> print(Element('em')('1 < 2')) + <em>1 < 2</em> + + This technique also allows mixed content: + + >>> print(Element('p')('Hello ', Element('b')('world'))) + <p>Hello <b>world</b></p> + + Quotes are not escaped inside text nodes: + >>> print(Element('p')('"Hello"')) + <p>"Hello"</p> + + Elements can also be combined with other elements or strings using the + addition operator, which results in a `Fragment` object that contains the + operands: + + >>> print(Element('br') + 'some text' + Element('br')) + <br/>some text<br/> + + Elements with a namespace can be generated using the `Namespace` and/or + `QName` classes: + + >>> from genshi.core import Namespace + >>> xhtml = Namespace('http://www.w3.org/1999/xhtml') + >>> print(Element(xhtml.html, lang='en')) + <html xmlns="http://www.w3.org/1999/xhtml" lang="en"/> + """ + __slots__ = ['tag', 'attrib'] + + def __init__(self, tag_, **attrib): + Fragment.__init__(self) + self.tag = QName(tag_) + self.attrib = _kwargs_to_attrs(attrib) + + def __call__(self, *args, **kwargs): + """Append any positional arguments as child nodes, and keyword arguments + as attributes. + + :return: the element itself so that calls can be chained + :rtype: `Element` + :see: `Fragment.append` + """ + self.attrib |= _kwargs_to_attrs(kwargs) + Fragment.__call__(self, *args) + return self + + def __repr__(self): + return '<%s "%s">' % (type(self).__name__, self.tag) + + def _generate(self): + yield START, (self.tag, self.attrib), (None, -1, -1) + for kind, data, pos in Fragment._generate(self): + yield kind, data, pos + yield END, self.tag, (None, -1, -1) + + def generate(self): + """Return a markup event stream for the fragment. + + :rtype: `Stream` + """ + return Stream(self._generate()) + + +class ElementFactory(object): + """Factory for `Element` objects. + + A new element is created simply by accessing a correspondingly named + attribute of the factory object: + + >>> factory = ElementFactory() + >>> print(factory.foo) + <foo/> + >>> print(factory.foo(id=2)) + <foo id="2"/> + + Markup fragments (lists of nodes without a parent element) can be created + by calling the factory: + + >>> print(factory('Hello, ', factory.em('world'), '!')) + Hello, <em>world</em>! + + A factory can also be bound to a specific namespace: + + >>> factory = ElementFactory('http://www.w3.org/1999/xhtml') + >>> print(factory.html(lang="en")) + <html xmlns="http://www.w3.org/1999/xhtml" lang="en"/> + + The namespace for a specific element can be altered on an existing factory + by specifying the new namespace using item access: + + >>> factory = ElementFactory() + >>> print(factory.html(factory['http://www.w3.org/2000/svg'].g(id=3))) + <html><g xmlns="http://www.w3.org/2000/svg" id="3"/></html> + + Usually, the `ElementFactory` class is not be used directly. Rather, the + `tag` instance should be used to create elements. + """ + + def __init__(self, namespace=None): + """Create the factory, optionally bound to the given namespace. + + :param namespace: the namespace URI for any created elements, or `None` + for no namespace + """ + if namespace and not isinstance(namespace, Namespace): + namespace = Namespace(namespace) + self.namespace = namespace + + def __call__(self, *args): + """Create a fragment that has the given positional arguments as child + nodes. + + :return: the created `Fragment` + :rtype: `Fragment` + """ + return Fragment()(*args) + + def __getitem__(self, namespace): + """Return a new factory that is bound to the specified namespace. + + :param namespace: the namespace URI or `Namespace` object + :return: an `ElementFactory` that produces elements bound to the given + namespace + :rtype: `ElementFactory` + """ + return ElementFactory(namespace) + + def __getattr__(self, name): + """Create an `Element` with the given name. + + :param name: the tag name of the element to create + :return: an `Element` with the specified name + :rtype: `Element` + """ + return Element(self.namespace and self.namespace[name] or name) + + +tag = ElementFactory() +"""Global `ElementFactory` bound to the default namespace. + +:type: `ElementFactory` +""" diff --git a/lib/genshi/genshi/compat.py b/lib/genshi/genshi/compat.py new file mode 100644 --- /dev/null +++ b/lib/genshi/genshi/compat.py @@ -0,0 +1,115 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2006-2009 Edgewall Software +# All rights reserved. +# +# This software is licensed as described in the file COPYING, which +# you should have received as part of this distribution. The terms +# are also available at http://genshi.edgewall.org/wiki/License. +# +# This software consists of voluntary contributions made by many +# individuals. For the exact contribution history, see the revision +# history and logs, available at http://genshi.edgewall.org/log/. + +"""Various Python version compatibility classes and functions.""" + +import sys +from types import CodeType + + +IS_PYTHON2 = (sys.version_info[0] == 2) + + +# This function should only be called in Python 2, and will fail in Python 3 + +if IS_PYTHON2: + def stringrepr(string): + ascii = string.encode('ascii', 'backslashreplace') + quoted = "'" + ascii.replace("'", "\\'") + "'" + if len(ascii) > len(string): + return 'u' + quoted + return quoted +else: + def stringrepr(string): + raise RuntimeError( + 'Python 2 compatibility function. Not usable in Python 3.') + + +# We need to differentiate between StringIO and BytesIO in places + +if IS_PYTHON2: + from StringIO import StringIO + try: + from cStringIO import StringIO as BytesIO + except ImportError: + BytesIO = StringIO +else: + from io import StringIO, BytesIO + + +# We want to test bytestring input to some stuff. + +if IS_PYTHON2: + def wrapped_bytes(bstr): + assert bstr.startswith('b') + return bstr[1:] +else: + def wrapped_bytes(bstr): + assert bstr.startswith('b') + return bstr + + +# We do some scary stuff with CodeType() in template/eval.py + +if IS_PYTHON2: + def get_code_params(code): + return (code.co_nlocals, code.co_stacksize, code.co_flags, + code.co_code, code.co_consts, code.co_names, code.co_varnames, + code.co_filename, code.co_name, code.co_firstlineno, + code.co_lnotab, (), ()) + + def build_code_chunk(code, filename, name, lineno): + return CodeType(0, code.co_nlocals, code.co_stacksize, + code.co_flags | 0x0040, code.co_code, code.co_consts, + code.co_names, code.co_varnames, filename, name, + lineno, code.co_lnotab, (), ()) +else: + def get_code_params(code): + return (code.co_nlocals, code.co_kwonlyargcount, code.co_stacksize, + code.co_flags, code.co_code, code.co_consts, code.co_names, + code.co_varnames, code.co_filename, code.co_name, + code.co_firstlineno, code.co_lnotab, (), ()) + + def build_code_chunk(code, filename, name, lineno): + return CodeType(0, code.co_nlocals, code.co_kwonlyargcount, + code.co_stacksize, code.co_flags | 0x0040, + code.co_code, code.co_consts, code.co_names, + code.co_varnames, filename, name, lineno, + code.co_lnotab, (), ()) + +# Compatibility fallback implementations for Python < 2.6 + +try: + next = next +except NameError: + def next(iterator): + return iterator.next() + +# Compatibility fallback implementations for Python < 2.5 + +try: + all = all + any = any +except NameError: + def any(S): + for x in S: + if x: + return True + return False + + def all(S): + for x in S: + if not x: + return False + return True + diff --git a/lib/genshi/genshi/core.py b/lib/genshi/genshi/core.py new file mode 100644 --- /dev/null +++ b/lib/genshi/genshi/core.py @@ -0,0 +1,745 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2006-2009 Edgewall Software +# All rights reserved. +# +# This software is licensed as described in the file COPYING, which +# you should have received as part of this distribution. The terms +# are also available at http://genshi.edgewall.org/wiki/License. +# +# This software consists of voluntary contributions made by many +# individuals. For the exact contribution history, see the revision +# history and logs, available at http://genshi.edgewall.org/log/. + +"""Core classes for markup processing.""" + +try: + reduce # builtin in Python < 3 +except NameError: + from functools import reduce +import sys +from itertools import chain +import operator + +from genshi.util import plaintext, stripentities, striptags, stringrepr + +__all__ = ['Stream', 'Markup', 'escape', 'unescape', 'Attrs', 'Namespace', + 'QName'] +__docformat__ = 'restructuredtext en' + + +class StreamEventKind(str): + """A kind of event on a markup stream.""" + __slots__ = [] + _instances = {} + + def __new__(cls, val): + return cls._instances.setdefault(val, str.__new__(cls, val)) + + +class Stream(object): + """Represents a stream of markup events. + + This class is basically an iterator over the events. + + Stream events are tuples of the form:: + + (kind, data, position) + + where ``kind`` is the event kind (such as `START`, `END`, `TEXT`, etc), + ``data`` depends on the kind of event, and ``position`` is a + ``(filename, line, offset)`` tuple that contains the location of the + original element or text in the input. If the original location is unknown, + ``position`` is ``(None, -1, -1)``. + + Also provided are ways to serialize the stream to text. The `serialize()` + method will return an iterator over generated strings, while `render()` + returns the complete generated text at once. Both accept various parameters + that impact the way the stream is serialized. + """ + __slots__ = ['events', 'serializer'] + + START = StreamEventKind('START') #: a start tag + END = StreamEventKind('END') #: an end tag + TEXT = StreamEventKind('TEXT') #: literal text + XML_DECL = StreamEventKind('XML_DECL') #: XML declaration + DOCTYPE = StreamEventKind('DOCTYPE') #: doctype declaration + START_NS = StreamEventKind('START_NS') #: start namespace mapping + END_NS = StreamEventKind('END_NS') #: end namespace mapping + START_CDATA = StreamEventKind('START_CDATA') #: start CDATA section + END_CDATA = StreamEventKind('END_CDATA') #: end CDATA section + PI = StreamEventKind('PI') #: processing instruction + COMMENT = StreamEventKind('COMMENT') #: comment + + def __init__(self, events, serializer=None): + """Initialize the stream with a sequence of markup events. + + :param events: a sequence or iterable providing the events + :param serializer: the default serialization method to use for this + stream + + :note: Changed in 0.5: added the `serializer` argument + """ + self.events = events #: The underlying iterable producing the events + self.serializer = serializer #: The default serializion method + + def __iter__(self): + return iter(self.events) + + def __or__(self, function): + """Override the "bitwise or" operator to apply filters or serializers + to the stream, providing a syntax similar to pipes on Unix shells. + + Assume the following stream produced by the `HTML` function: + + >>> from genshi.input import HTML + >>> html = HTML('''<p onclick="alert('Whoa')">Hello, world!</p>''', encoding='utf-8') + >>> print(html) + <p onclick="alert('Whoa')">Hello, world!</p> + + A filter such as the HTML sanitizer can be applied to that stream using + the pipe notation as follows: + + >>> from genshi.filters import HTMLSanitizer + >>> sanitizer = HTMLSanitizer() + >>> print(html | sanitizer) + <p>Hello, world!</p> + + Filters can be any function that accepts and produces a stream (where + a stream is anything that iterates over events): + + >>> def uppercase(stream): + ... for kind, data, pos in stream: + ... if kind is TEXT: + ... data = data.upper() + ... yield kind, data, pos + >>> print(html | sanitizer | uppercase) + <p>HELLO, WORLD!</p> + + Serializers can also be used with this notation: + + >>> from genshi.output import TextSerializer + >>> output = TextSerializer() + >>> print(html | sanitizer | uppercase | output) + HELLO, WORLD! + + Commonly, serializers should be used at the end of the "pipeline"; + using them somewhere in the middle may produce unexpected results. + + :param function: the callable object that should be applied as a filter + :return: the filtered stream + :rtype: `Stream` + """ + return Stream(_ensure(function(self)), serializer=self.serializer) + + def filter(self, *filters): + """Apply filters to the stream. + + This method returns a new stream with the given filters applied. The + filters must be callables that accept the stream object as parameter, + and return the filtered stream. + + The call:: + + stream.filter(filter1, filter2) + + is equivalent to:: + + stream | filter1 | filter2 + + :param filters: one or more callable objects that should be applied as + filters + :return: the filtered stream + :rtype: `Stream` + """ + return reduce(operator.or_, (self,) + filters) + + def render(self, method=None, encoding=None, out=None, **kwargs): + """Return a string representation of the stream. + + Any additional keyword arguments are passed to the serializer, and thus + depend on the `method` parameter value. + + :param method: determines how the stream is serialized; can be either + "xml", "xhtml", "html", "text", or a custom serializer + class; if `None`, the default serialization method of + the stream is used + :param encoding: how the output string should be encoded; if set to + `None`, this method returns a `unicode` object + :param out: a file-like object that the output should be written to + instead of being returned as one big string; note that if + this is a file or socket (or similar), the `encoding` must + not be `None` (that is, the output must be encoded) + :return: a `str` or `unicode` object (depending on the `encoding` + parameter), or `None` if the `out` parameter is provided + :rtype: `basestring` + + :see: XMLSerializer, XHTMLSerializer, HTMLSerializer, TextSerializer + :note: Changed in 0.5: added the `out` parameter + """ + from genshi.output import encode + if method is None: + method = self.serializer or 'xml' + generator = self.serialize(method=method, **kwargs) + return encode(generator, method=method, encoding=encoding, out=out) + + def select(self, path, namespaces=None, variables=None): + """Return a new stream that contains the events matching the given + XPath expression. + + >>> from genshi import HTML + >>> stream = HTML('<doc><elem>foo</elem><elem>bar</elem></doc>', encoding='utf-8') + >>> print(stream.select('elem')) + <elem>foo</elem><elem>bar</elem> + >>> print(stream.select('elem/text()')) + foobar + + Note that the outermost element of the stream becomes the *context + node* for the XPath test. That means that the expression "doc" would + not match anything in the example above, because it only tests against + child elements of the outermost element: + + >>> print(stream.select('doc')) + <BLANKLINE> + + You can use the "." expression to match the context node itself + (although that usually makes little sense): + + >>> print(stream.select('.')) + <doc><elem>foo</elem><elem>bar</elem></doc> + + :param path: a string containing the XPath expression + :param namespaces: mapping of namespace prefixes used in the path + :param variables: mapping of variable names to values + :return: the selected substream + :rtype: `Stream` + :raises PathSyntaxError: if the given path expression is invalid or not + supported + """ + from genshi.path import Path + return Path(path).select(self, namespaces, variables) + + def serialize(self, method='xml', **kwargs): + """Generate strings corresponding to a specific serialization of the + stream. + + Unlike the `render()` method, this method is a generator that returns + the serialized output incrementally, as opposed to returning a single + string. + + Any additional keyword arguments are passed to the serializer, and thus + depend on the `method` parameter value. + + :param method: determines how the stream is serialized; can be either + "xml", "xhtml", "html", "text", or a custom serializer + class; if `None`, the default serialization method of + the stream is used + :return: an iterator over the serialization results (`Markup` or + `unicode` objects, depending on the serialization method) + :rtype: ``iterator`` + :see: XMLSerializer, XHTMLSerializer, HTMLSerializer, TextSerializer + """ + from genshi.output import get_serializer + if method is None: + method = self.serializer or 'xml' + return get_serializer(method, **kwargs)(_ensure(self)) + + def __str__(self): + return self.render() + + def __unicode__(self): + return self.render(encoding=None) + + def __html__(self): + return self + + +START = Stream.START +END = Stream.END +TEXT = Stream.TEXT +XML_DECL = Stream.XML_DECL +DOCTYPE = Stream.DOCTYPE +START_NS = Stream.START_NS +END_NS = Stream.END_NS +START_CDATA = Stream.START_CDATA +END_CDATA = Stream.END_CDATA +PI = Stream.PI +COMMENT = Stream.COMMENT + + +def _ensure(stream): + """Ensure that every item on the stream is actually a markup event.""" + stream = iter(stream) + event = stream.next() + + # Check whether the iterable is a real markup event stream by examining the + # first item it yields; if it's not we'll need to do some conversion + if type(event) is not tuple or len(event) != 3: + for event in chain([event], stream): + if hasattr(event, 'totuple'): + event = event.totuple() + else: + event = TEXT, unicode(event), (None, -1, -1) + yield event + return + + # This looks like a markup event stream, so we'll just pass it through + # unchanged + yield event + for event in stream: + yield event + + +class Attrs(tuple): + """Immutable sequence type that stores the attributes of an element. + + Ordering of the attributes is preserved, while access by name is also + supported. From noreply at buildbot.pypy.org Sun Jan 22 23:33:04 2012 From: noreply at buildbot.pypy.org (amauryfa) Date: Sun, 22 Jan 2012 23:33:04 +0100 (CET) Subject: [pypy-commit] pypy merge-2.7.2: Buffered IO retries read() and write() when the raw stream raises errno=EINTR. Message-ID: <20120122223304.20DDE71027D@wyvern.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc <amauryfa at gmail.com> Branch: merge-2.7.2 Changeset: r51669:311b0271907b Date: 2012-01-22 23:32 +0100 http://bitbucket.org/pypy/pypy/changeset/311b0271907b/ Log: Buffered IO retries read() and write() when the raw stream raises errno=EINTR. diff --git a/pypy/module/_io/interp_bufferedio.py b/pypy/module/_io/interp_bufferedio.py --- a/pypy/module/_io/interp_bufferedio.py +++ b/pypy/module/_io/interp_bufferedio.py @@ -11,9 +11,23 @@ W_IOBase, DEFAULT_BUFFER_SIZE, convert_size, check_readable_w, check_writable_w, check_seekable_w) from pypy.module._io.interp_io import W_BlockingIOError +import errno STATE_ZERO, STATE_OK, STATE_DETACHED = range(3) +def trap_eintr(space, error): + # Return True if an EnvironmentError with errno == EINTR is set + if not error.match(space, space.w_EnvironmentError): + return False + try: + w_value = error.get_w_value(space) + w_errno = space.getattr(w_value, space.wrap("errno")) + return space.is_true( + space.eq(w_errno, space.wrap(errno.EINTR))) + except OperationError: + return False + + class BlockingIOError(Exception): pass @@ -315,7 +329,16 @@ def _write(self, space, data): w_data = space.wrap(data) - w_written = space.call_method(self.w_raw, "write", w_data) + while True: + try: + w_written = space.call_method(self.w_raw, "write", w_data) + except OperationError, e: + if trap_eintr(space, e): + continue # try again + raise + else: + break + written = space.getindex_w(w_written, space.w_IOError) if not 0 <= written <= len(data): raise OperationError(space.w_IOError, space.wrap( @@ -481,7 +504,16 @@ def _raw_read(self, space, buffer, start, length): length = intmask(length) w_buf = space.wrap(RawBuffer(buffer, start, length)) - w_size = space.call_method(self.w_raw, "readinto", w_buf) + while True: + try: + w_size = space.call_method(self.w_raw, "readinto", w_buf) + except OperationError, e: + if trap_eintr(space, e): + continue # try again + raise + else: + break + if space.is_w(w_size, space.w_None): raise BlockingIOError() size = space.int_w(w_size) diff --git a/pypy/module/_io/test/test_bufferedio.py b/pypy/module/_io/test/test_bufferedio.py --- a/pypy/module/_io/test/test_bufferedio.py +++ b/pypy/module/_io/test/test_bufferedio.py @@ -191,6 +191,26 @@ f = _io.BufferedReader(raw) assert repr(f) == '<_io.BufferedReader name=%r>' % (self.tmpfile,) + def test_read_interrupted(self): + import _io, errno + class MockRawIO(_io._RawIOBase): + def __init__(self): + self.count = 0 + def readable(self): + return True + def readinto(self, buf): + self.count += 1 + if self.count < 3: + raise IOError(errno.EINTR, "interrupted") + else: + buf[:3] = "abc" + return 3 + rawio = MockRawIO() + bufio = _io.BufferedReader(rawio) + r = bufio.read(4) + assert r == "abca" + assert rawio.count == 4 + class AppTestBufferedReaderWithThreads(AppTestBufferedReader): spaceconfig = dict(usemodules=['_io', 'thread']) @@ -386,6 +406,25 @@ assert bufio.read() is None assert bufio.read() == "" + def test_write_interrupted(self): + import _io, errno + class MockRawIO(_io._RawIOBase): + def __init__(self): + self.count = 0 + def writable(self): + return True + def write(self, data): + self.count += 1 + if self.count < 3: + raise IOError(errno.EINTR, "interrupted") + else: + return len(data) + rawio = MockRawIO() + bufio = _io.BufferedWriter(rawio) + assert bufio.write("test") == 4 + bufio.flush() + assert rawio.count == 3 + class AppTestBufferedRWPair: def test_pair(self): import _io From notifications-noreply at bitbucket.org Sun Jan 22 23:49:49 2012 From: notifications-noreply at bitbucket.org (Bitbucket) Date: Sun, 22 Jan 2012 22:49:49 -0000 Subject: [pypy-commit] Notification: pypy Message-ID: <20120122224949.26793.45302@bitbucket13.managed.contegix.com> You have received a notification from David Ripton. Hi, I forked pypy. My fork is at https://bitbucket.org/dripton/pypy. -- Disable notifications at https://bitbucket.org/account/notifications/ From pullrequests-noreply at bitbucket.org Mon Jan 23 00:08:01 2012 From: pullrequests-noreply at bitbucket.org (Bitbucket) Date: Sun, 22 Jan 2012 23:08:01 -0000 Subject: [pypy-commit] [OPEN] Pull request #23 for pypy/pypy: Fix _sqlite3.py fix for issue996, with test Message-ID: <pullreq-80014f10414c@bitbucket.org> A new pull request has been opened by David Ripton. dripton/pypy has changes to be pulled into pypy/pypy. https://bitbucket.org/pypy/pypy/pull-request/23/fix-_sqlite3py-fix-for-issue996-with-test Title: Fix _sqlite3.py fix for issue996, with test 4 commits of trivial cleanups to lib_pypy/_sqlite3.py, a comment to comply with the pysqlite license, a test based on the test case in issue996, and a one-line fix for issue996. lib-python/2.7/test/test_sqlite.py still passes after the change. Changes to be pulled: 80014f10414c by David Ripton: "Do not call statement.reset() after executing DDL. Fixes issue996." 6141b7352630 by David Ripton: "Failing test for issue 996" 01ed002a2b95 by David Ripton: "Add a comment making it clear this file is modified, for license compliance." d49f44c4a62f by David Ripton: "Fix old-style raise statements (illegal in Python 3, ugly in Python 2)" 014a02a9d3df by David Ripton: "Remove unused local variables." d39f5493dcaf by David Ripton: "Remove unused import." 3ed178fb213d by David Ripton: "Remove unreachable statement referring to nonexistant variable." -- This is an issue notification from bitbucket.org. You are receiving this either because you are the participating in a pull request, or you are following it. From notifications-noreply at bitbucket.org Mon Jan 23 00:12:58 2012 From: notifications-noreply at bitbucket.org (Bitbucket) Date: Sun, 22 Jan 2012 23:12:58 -0000 Subject: [pypy-commit] [COMMENT] Pull request #23 for pypy/pypy: Fix _sqlite3.py fix for issue996, with test Message-ID: <20120122231258.8936.91345@bitbucket05.managed.contegix.com> New comment on pull request: https://bitbucket.org/pypy/pypy/pull-request/23/fix-_sqlite3py-fix-for-issue996-with-test#comment-1996 Alex Gaynor (alex_gaynor) said: A few comments: # No need for the new comment at the top of _sqlite3.py, it has already been modified many times :) # The new tests should probably make an assertion about the contents of result, even if it's just that it's an empty list. Thanks! -- This is a pull request comment notification from bitbucket.org. You are receiving this either because you are participating in a pull request, or you are following it. From notifications-noreply at bitbucket.org Mon Jan 23 00:28:10 2012 From: notifications-noreply at bitbucket.org (Bitbucket) Date: Sun, 22 Jan 2012 23:28:10 -0000 Subject: [pypy-commit] [COMMENT] Pull request #23 for pypy/pypy: Fix _sqlite3.py fix for issue996, with test Message-ID: <20120122232810.26350.46169@bitbucket01.managed.contegix.com> New comment on pull request: https://bitbucket.org/pypy/pypy/pull-request/23/fix-_sqlite3py-fix-for-issue996-with-test#comment-1997 David Ripton (dripton) said: Alex, the new comment is there because the pysqlite license specifically requires that modified versions be marked as modified. It's a silly requirement, but it should be obeyed. I will add an assert to the test. -- This is a pull request comment notification from bitbucket.org. You are receiving this either because you are participating in a pull request, or you are following it. From pullrequests-noreply at bitbucket.org Mon Jan 23 00:33:41 2012 From: pullrequests-noreply at bitbucket.org (Bitbucket) Date: Sun, 22 Jan 2012 23:33:41 -0000 Subject: [pypy-commit] [OPEN] Pull request #23 for pypy/pypy: Fix _sqlite3.py fix for issue996, with test In-Reply-To: <pullreq-fd6e2144e652@bitbucket.org> References: <pullreq-fd6e2144e652@bitbucket.org> Message-ID: <20120122233341.17390.62656@bitbucket02.managed.contegix.com> Pull request #23 has been updated by David Ripton to include new changes. https://bitbucket.org/pypy/pypy/pull-request/23/fix-_sqlite3py-fix-for-issue996-with-test Title: Fix _sqlite3.py fix for issue996, with test Creator: David Ripton 4 commits of trivial cleanups to lib_pypy/_sqlite3.py, a comment to comply with the pysqlite license, a test based on the test case in issue996, and a one-line fix for issue996. lib-python/2.7/test/test_sqlite.py still passes after the change. Added an assert to the new test, which previously just looked for exceptions. Updated list of changes: fd6e2144e652 by David Ripton: "Add an assert, as requested by Alex." 80014f10414c by David Ripton: "Do not call statement.reset() after executing DDL. Fixes issue996." 6141b7352630 by David Ripton: "Failing test for issue 996" 01ed002a2b95 by David Ripton: "Add a comment making it clear this file is modified, for license compliance." d49f44c4a62f by David Ripton: "Fix old-style raise statements (illegal in Python 3, ugly in Python 2)" 014a02a9d3df by David Ripton: "Remove unused local variables." d39f5493dcaf by David Ripton: "Remove unused import." 3ed178fb213d by David Ripton: "Remove unreachable statement referring to nonexistant variable." -- This is an issue notification from bitbucket.org. You are receiving this either because you are the participating in a pull request, or you are following it. From noreply at buildbot.pypy.org Mon Jan 23 00:46:34 2012 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 23 Jan 2012 00:46:34 +0100 (CET) Subject: [pypy-commit] buildbot default: some work on numready Message-ID: <20120122234634.A296471027D@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: Changeset: r609:ba9d28fbab20 Date: 2012-01-23 01:46 +0200 http://bitbucket.org/pypy/buildbot/changeset/ba9d28fbab20/ Log: some work on numready diff --git a/numready.py b/numready.py --- a/numready.py +++ b/numready.py @@ -15,7 +15,7 @@ import jinja2 -MODULE_SEARCH_CODE = """ +MODULE_SEARCH_CODE = ''' import types import {modname} as numpy @@ -27,22 +27,22 @@ if isinstance(obj, types.TypeType): kind = "{kinds[TYPE]}" print kind, ":", name -""" +''' -ATTR_SEARCH_CODE = """ +ATTR_SEARCH_CODE = ''' import types import {modname} as numpy obj = getattr(numpy, "{name}") for name in dir(obj): - if name.startswith("_"): - continue + #if name.startswith("_"): + # continue sub_obj = getattr(obj, name) kind = "{kinds[UNKNOWN]}" if isinstance(sub_obj, types.TypeType): kind = "{kinds[TYPE]}" print kind, ":", name -""" +''' KINDS = { "UNKNOWN": "U", @@ -62,6 +62,9 @@ h1 { text-align: center; } + h3 { + text-align: center; + } table { border: 8px solid #DFDECB; margin: 30px auto; @@ -74,7 +77,7 @@ padding: 4px 10px; text-align: center; } - tr.exists { + .exists { background-color: #337792; color: white; border: 1px solid #234F61; @@ -83,27 +86,30 @@ </head> <body> <h1>NumPyPy Status</h1> + <h3>Overall: {{ msg }}</h3> <table> <thead> <tr> <th></th> <th>PyPy</th> + <th></th> + <th>PyPy</th> + <th></th> + <th>PyPy</th> + <th></th> + <th>PyPy</th> + <th></th> + <th>PyPy</th> </tr> </thead> <tbody> - {% for item in all_items %} - <tr{% if item.pypy_exists %} class="exists"{% endif %}> - <th>{{ item.name }}</th> - <td>{% if item.pypy_exists %}✔{% else %}✖{% endif %}</td> + {% for chunk in all_items %} + <tr> + {% for item in chunk %} + <th class='{{ item.cls }}'>{{ item.name }}</th> + <td class='{{ item.cls }}'>{{ item.symbol }}</td> + {% endfor %} </tr> - {% if item.subitems %} - {% for item in item.subitems %} - <tr{% if item.pypy_exists %} class="exists"{% endif %}> - <th> {{ item.name }}</th> - <td>{% if item.pypy_exists %}✔{% else %}✖{% endif %}</td> - </tr> - {% endfor %} - {% endif %} {% endfor %} </tbody> </table> @@ -129,6 +135,9 @@ def add(self, item): self._items[item] = item + def __len__(self): + return len(self._items) + class Item(object): def __init__(self, name, kind, subitems=None): self.name = name @@ -139,14 +148,16 @@ return hash(self.name) def __eq__(self, other): + if isinstance(other, str): + return self.name == other return self.name == other.name class ItemStatus(object): - def __init__(self, name, pypy_exists, subitems=None): + def __init__(self, name, pypy_exists): self.name = name - self.pypy_exists = pypy_exists - self.subitems = subitems + self.cls = 'exists' if pypy_exists else '' + self.symbol = u"✔" if pypy_exists else u'✖' def __lt__(self, other): return self.name < other.name @@ -170,10 +181,21 @@ kind, name = line.split(" : ", 1) subitems = None if kind == KINDS["TYPE"]: - subitems = find_numpy_attrs(python, modname, name) + if name in ['ndarray', 'dtype']: + subitems = find_numpy_attrs(python, modname, name) items.add(Item(name, kind, subitems)) return items +def split(lst): + SPLIT = 5 + lgt = len(lst) // SPLIT + 1 + l = [[] for i in range(lgt)] + for i in range(lgt): + for k in range(SPLIT): + if k * lgt + i < len(lst): + l[i].append(lst[k * lgt + i]) + return l + def main(argv): assert platform.python_implementation() == "PyPy" @@ -181,25 +203,23 @@ pypy_items = find_numpy_items(sys.executable, "numpypy") all_items = [] + msg = '%d/%d names, %d/%d ndarray attributes, %d/%d dtype attributes' % ( + len(pypy_items), len(cpy_items), len(pypy_items['ndarray'].subitems), + len(cpy_items['ndarray'].subitems), len(pypy_items['dtype'].subitems), + len(cpy_items['dtype'].subitems)) for item in cpy_items: pypy_exists = item in pypy_items - subitems = None if item.subitems: - subitems = [] for sub in item.subitems: - subitems.append( - ItemStatus(sub.name, pypy_exists=pypy_exists and pypy_items[item].subitems and sub in pypy_items[item].subitems) + all_items.append( + ItemStatus(item.name + "." + sub.name, pypy_exists=pypy_exists and pypy_items[item].subitems and sub in pypy_items[item].subitems) ) - subitems = sorted(subitems) - all_items.append( - ItemStatus(item.name, pypy_exists=item in pypy_items, subitems=subitems) - ) - - html = jinja2.Template(PAGE_TEMPLATE).render(all_items=sorted(all_items)) + all_items.append(ItemStatus(item.name, pypy_exists=item in pypy_items)) + html = jinja2.Template(PAGE_TEMPLATE).render(all_items=split(sorted(all_items)), msg=msg) with tempfile.NamedTemporaryFile(delete=False) as f: f.write(html.encode("utf-8")) webbrowser.open_new_tab(f.name) if __name__ == '__main__': - main(sys.argv) \ No newline at end of file + main(sys.argv) From noreply at buildbot.pypy.org Mon Jan 23 00:56:38 2012 From: noreply at buildbot.pypy.org (mattip) Date: Mon, 23 Jan 2012 00:56:38 +0100 (CET) Subject: [pypy-commit] pypy numppy-flatitter: redo, add lots of tests, some still fail Message-ID: <20120122235638.0E46771027D@wyvern.cs.uni-duesseldorf.de> Author: mattip Branch: numppy-flatitter Changeset: r51670:ff368a6d0ff7 Date: 2012-01-19 01:16 +0200 http://bitbucket.org/pypy/pypy/changeset/ff368a6d0ff7/ Log: redo, add lots of tests, some still fail diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -1290,53 +1290,100 @@ tolist = interp2app(BaseArray.descr_tolist), ) +#TODO:Move all this to another file after fijal finishes reorganization +def _to_coords(space, arr, w_item_or_slice): + '''Always returns a list of coords. + ''' + start = stop = step = 0 + if space.isinstance_w(w_item_or_slice, space.w_int): + start = space.int_w(w_item_or_slice) + if start < 0: + start += arr.size + stop = start+1 + step = 1 + elif space.isinstance_w(w_item_or_slice, space.w_slice): + start, stop, step, lngth = space.decode_index4(w_item_or_slice,arr.size) + else: + operationerrfmt(space.NotImplementedError,'cannot iterator over %s yet', w_item_or_slice) + retval = [] + for i in range(start, stop, step): + coords = [] + ii = i + if arr.order =='C': + for s in range(len(arr.shape) -1, -1, -1): + coords.insert(0,ii % arr.shape[s]) + ii /= arr.shape[s] + else: + raise NotImplementedError + #untested code. Erase? + for s in range(len(arr.shape)): + coords.append(ii % arr.shape[s]) + ii /= arr.shape[s] + if ii != 0: + raise OperationError(space.w_IndexError, + space.wrap("invalid index")) + + retval.append(space.newtuple([space.wrap(c) for c in coords])) + return retval class W_FlatIterator(ViewArray): @jit.unroll_safe def __init__(self, arr): arr = arr.get_concrete() - size = 1 - for sh in arr.shape: - size *= sh - if arr.strides[-1] <= arr.strides[0]: - self.strides = [arr.strides[-1]] - self.backstrides = [arr.backstrides[-1]] - else: - XXX - # This will not work: getitem and setitem will - # fail. Need to be smarter: calculate the indices from the int - self.strides = [arr.strides[0]] - self.backstrides = [arr.backstrides[0]] - ViewArray.__init__(self, size, [size], arr.dtype, order=arr.order, - parent=arr) self.shapelen = len(arr.shape) sig = arr.find_sig() - #self.iter = OneDimIterator(arr.start, self.strides[0], - # self.shape[0]) self.iter = sig.create_frame(arr).get_final_iter() - self.start = arr.start self.base = arr + self.index = 0 + ViewArray.__init__(self, arr.size, [arr.size], arr.dtype, arr.order, arr) def descr_next(self, space): if self.iter.done(): raise OperationError(space.w_StopIteration, space.w_None) - result = self.getitem(self.iter.offset) + result = self.base.getitem(self.iter.offset) self.iter = self.iter.next(self.shapelen) + self.index += 1 return result def descr_iter(self): return self def descr_index(self, space): - return space.wrap(self.iter.offset) + return space.wrap(self.index) + def descr_coords(self, space): + return _to_coords(space, self.base, space.wrap(self.index))[0] + + def descr_getitem(self, space, w_idx): + coords = _to_coords(space, self.base, w_idx) + if len(coords)>1: + w_result = W_NDimArray(len(coords), [len(coords)], self.base.dtype, + self.base.order) + for i,c in enumerate(coords): + w_val = self.base.descr_getitem(space, c) + w_result.setitem(i,w_val) + return w_result + else: + return self.base.descr_getitem(space, coords[0]) + + def descr_setitem(self, space, w_idx, w_value): + coords = _to_coords(space, self.base, w_idx) + arr = convert_to_array(space, w_value) + ai = 0 + for c in coords: + v = arr.getitem(ai) + self.base.descr_setitem(space, c,v) + ai = (ai + 1) % arr.size + W_FlatIterator.typedef = TypeDef( 'flatiter', next = interp2app(W_FlatIterator.descr_next), __iter__ = interp2app(W_FlatIterator.descr_iter), - __getitem__ = interp2app(BaseArray.descr_getitem), - __setitem__ = interp2app(BaseArray.descr_setitem), + __getitem__ = interp2app(W_FlatIterator.descr_getitem), + __setitem__ = interp2app(W_FlatIterator.descr_setitem), index = GetSetProperty(W_FlatIterator.descr_index), + coords = GetSetProperty(W_FlatIterator.descr_coords), + ) W_FlatIterator.acceptable_as_base_class = False diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -1300,7 +1300,7 @@ assert (a + a).flat[3] == 6 assert a[::2].flat[3] == 6 assert a.reshape(2,5).flat[3] == 3 - b = a.flat + b = a.reshape(2,5).flat b.next() b.next() b.next() @@ -1309,6 +1309,19 @@ raises(IndexError, "b[11]") raises(IndexError, "b[-11]") assert b.index == 3 + assert b.coords == (0,3) + + def test_flatiter_setitem(self): + from _numpypy import arange, array + a = arange(12).reshape(3,4) + b = a.T.flat + b[6::2] = [-1, -2] + assert (a == [[0, 1, -1, 3], [4, 5, 6, -1], [8, 9, -2, 11]]).all() + b[1:2] = [[[100]]] + assert(a[0,0] == 100) + assert(a[1,0] == 100) + b[array([10, 11])] == [-20, -40] + def test_flatiter_view(self): from _numpypy import arange @@ -1323,8 +1336,14 @@ def test_flatiter_transpose(self): from _numpypy import arange - a = arange(10) - assert a.reshape(2,5).T.flat[3] == 6 + a = arange(10).reshape(2,5).T + b = a.flat + assert (b[:5] == [0, 5, 1, 6, 2]).all() + b.next() + b.next() + b.next() + assert b.index == 3 + assert b.coords == (1,1) def test_slice_copy(self): from _numpypy import zeros From noreply at buildbot.pypy.org Mon Jan 23 00:56:39 2012 From: noreply at buildbot.pypy.org (mattip) Date: Mon, 23 Jan 2012 00:56:39 +0100 (CET) Subject: [pypy-commit] pypy numpypy-shape-bug: add more tests Message-ID: <20120122235639.49ADC71027D@wyvern.cs.uni-duesseldorf.de> Author: mattip Branch: numpypy-shape-bug Changeset: r51671:5a8fc969e644 Date: 2012-01-19 01:55 +0200 http://bitbucket.org/pypy/pypy/changeset/5a8fc969e644/ Log: add more tests diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -183,10 +183,9 @@ n_old_elems_to_use *= old_shape[oldI] if n_new_elems_used == n_old_elems_to_use: oldI += 1 - if oldI >= len(old_shape): - break - cur_step = steps[oldI] - n_old_elems_to_use *= old_shape[oldI] + if oldI < len(old_shape): + cur_step = steps[oldI] + n_old_elems_to_use *= old_shape[oldI] elif order == 'C': for i in range(len(old_shape) - 1, -1, -1): steps.insert(0, old_strides[i] / last_step) @@ -206,10 +205,10 @@ n_old_elems_to_use *= old_shape[oldI] if n_new_elems_used == n_old_elems_to_use: oldI -= 1 - if oldI < -len(old_shape): - break - cur_step = steps[oldI] - n_old_elems_to_use *= old_shape[oldI] + if oldI >= -len(old_shape): + cur_step = steps[oldI] + n_old_elems_to_use *= old_shape[oldI] + assert len(new_strides) == len(new_shape) return new_strides class BaseArray(Wrappable): diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -157,6 +157,13 @@ assert calc_new_strides([2, 3, 4], [8, 3], [1, 16], 'F') is None assert calc_new_strides([24], [2, 4, 3], [48, 6, 1], 'C') is None assert calc_new_strides([24], [2, 4, 3], [24, 6, 2], 'C') == [2] + assert calc_new_strides([105, 1], [3, 5, 7], [35, 7, 1],'C') == [1, 1] + assert calc_new_strides([1, 105], [3, 5, 7], [35, 7, 1],'C') == [105, 1] + assert calc_new_strides([1, 105], [3, 5, 7], [35, 7, 1],'F') is None + assert calc_new_strides([1, 1, 1, 105, 1], [15, 7], [7, 1],'C') == \ + [105, 105, 105, 1, 1] + assert calc_new_strides([1, 1, 105, 1, 1], [7, 15], [1, 7],'F') == \ + [1, 1, 1, 105, 105] class AppTestNumArray(BaseNumpyAppTest): @@ -767,7 +774,6 @@ assert (a[:, 1, :].sum(1) == [70, 315, 560]).all() raises (ValueError, 'a[:, 1, :].sum(2)') assert ((a + a).T.sum(2).T == (a + a).sum(0)).all() - skip("Those are broken, fix after removing Scalar") assert (a.reshape(1,-1).sum(0) == range(105)).all() assert (a.reshape(1,-1).sum(1) == 5460) From noreply at buildbot.pypy.org Mon Jan 23 01:05:35 2012 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 23 Jan 2012 01:05:35 +0100 (CET) Subject: [pypy-commit] pypy default: (mattip) merge numpypy-reshape-bug Message-ID: <20120123000535.93B8C71027D@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: Changeset: r51672:d246547e90d8 Date: 2012-01-23 02:04 +0200 http://bitbucket.org/pypy/pypy/changeset/d246547e90d8/ Log: (mattip) merge numpypy-reshape-bug diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -174,15 +174,17 @@ # fits the new shape, using those steps. If there is a shape/step mismatch # (meaning that the realignment of elements crosses from one step into another) # return None so that the caller can raise an exception. -def calc_new_strides(new_shape, old_shape, old_strides): +def calc_new_strides(new_shape, old_shape, old_strides, order): + # Return the proper strides for new_shape, or None if the mapping crosses + # stepping boundaries + # Assumes that prod(old_shape) == prod(new_shape), len(old_shape) > 1, and # len(new_shape) > 0 steps = [] last_step = 1 oldI = 0 new_strides = [] - if old_strides[0] < old_strides[-1]: - #Start at old_shape[0], old_stides[0] + if order == 'F': for i in range(len(old_shape)): steps.append(old_strides[i] / last_step) last_step *= old_shape[i] @@ -199,12 +201,10 @@ n_old_elems_to_use *= old_shape[oldI] if n_new_elems_used == n_old_elems_to_use: oldI += 1 - if oldI >= len(old_shape): - continue - cur_step = steps[oldI] - n_old_elems_to_use *= old_shape[oldI] - else: - #Start at old_shape[-1], old_strides[-1] + if oldI < len(old_shape): + cur_step = steps[oldI] + n_old_elems_to_use *= old_shape[oldI] + elif order == 'C': for i in range(len(old_shape) - 1, -1, -1): steps.insert(0, old_strides[i] / last_step) last_step *= old_shape[i] @@ -223,10 +223,10 @@ n_old_elems_to_use *= old_shape[oldI] if n_new_elems_used == n_old_elems_to_use: oldI -= 1 - if oldI < -len(old_shape): - continue - cur_step = steps[oldI] - n_old_elems_to_use *= old_shape[oldI] + if oldI >= -len(old_shape): + cur_step = steps[oldI] + n_old_elems_to_use *= old_shape[oldI] + assert len(new_strides) == len(new_shape) return new_strides class BaseArray(Wrappable): @@ -631,8 +631,8 @@ concrete = self.get_concrete() new_shape = get_shape_from_iterable(space, concrete.size, w_shape) # Since we got to here, prod(new_shape) == self.size - new_strides = calc_new_strides(new_shape, - concrete.shape, concrete.strides) + new_strides = calc_new_strides(new_shape, concrete.shape, + concrete.strides, concrete.order) if new_strides: # We can create a view, strides somehow match up. ndims = len(new_shape) @@ -1221,7 +1221,8 @@ self.backstrides = backstrides self.shape = new_shape return - new_strides = calc_new_strides(new_shape, self.shape, self.strides) + new_strides = calc_new_strides(new_shape, self.shape, self.strides, + self.order) if new_strides is None: raise OperationError(space.w_AttributeError, space.wrap( "incompatible shape for a non-contiguous array")) diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -154,13 +154,18 @@ def test_calc_new_strides(self): from pypy.module.micronumpy.interp_numarray import calc_new_strides - assert calc_new_strides([2, 4], [4, 2], [4, 2]) == [8, 2] - assert calc_new_strides([2, 4, 3], [8, 3], [1, 16]) == [1, 2, 16] - assert calc_new_strides([2, 3, 4], [8, 3], [1, 16]) is None - assert calc_new_strides([24], [2, 4, 3], [48, 6, 1]) is None - assert calc_new_strides([24], [2, 4, 3], [24, 6, 2]) == [2] - assert calc_new_strides([105, 1], [3, 5, 7], [35, 7, 1]) == [1, 1] - assert calc_new_strides([1, 105], [3, 5, 7], [35, 7, 1]) == [105, 1] + assert calc_new_strides([2, 4], [4, 2], [4, 2], "C") == [8, 2] + assert calc_new_strides([2, 4, 3], [8, 3], [1, 16], 'F') == [1, 2, 16] + assert calc_new_strides([2, 3, 4], [8, 3], [1, 16], 'F') is None + assert calc_new_strides([24], [2, 4, 3], [48, 6, 1], 'C') is None + assert calc_new_strides([24], [2, 4, 3], [24, 6, 2], 'C') == [2] + assert calc_new_strides([105, 1], [3, 5, 7], [35, 7, 1],'C') == [1, 1] + assert calc_new_strides([1, 105], [3, 5, 7], [35, 7, 1],'C') == [105, 1] + assert calc_new_strides([1, 105], [3, 5, 7], [35, 7, 1],'F') is None + assert calc_new_strides([1, 1, 1, 105, 1], [15, 7], [7, 1],'C') == \ + [105, 105, 105, 1, 1] + assert calc_new_strides([1, 1, 105, 1, 1], [7, 15], [1, 7],'F') == \ + [1, 1, 1, 105, 105] class AppTestNumArray(BaseNumpyAppTest): @@ -385,6 +390,8 @@ a.shape = () #numpy allows this a.shape = (1,) + a = array(range(6)).reshape(2,3).T + raises(AttributeError, 'a.shape = 6') def test_reshape(self): from _numpypy import array, zeros From noreply at buildbot.pypy.org Mon Jan 23 01:05:36 2012 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 23 Jan 2012 01:05:36 +0100 (CET) Subject: [pypy-commit] pypy numpypy-shape-bug: close merged-branch Message-ID: <20120123000536.D477171027D@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: numpypy-shape-bug Changeset: r51673:12c695258f06 Date: 2012-01-23 02:05 +0200 http://bitbucket.org/pypy/pypy/changeset/12c695258f06/ Log: close merged-branch From noreply at buildbot.pypy.org Mon Jan 23 01:30:48 2012 From: noreply at buildbot.pypy.org (dripton) Date: Mon, 23 Jan 2012 01:30:48 +0100 (CET) Subject: [pypy-commit] pypy default: Remove unreachable statement referring to nonexistant variable. Message-ID: <20120123003048.4BA4A71027D@wyvern.cs.uni-duesseldorf.de> Author: David Ripton <dripton at ripton.net> Branch: Changeset: r51674:3ed178fb213d Date: 2012-01-21 13:42 -0500 http://bitbucket.org/pypy/pypy/changeset/3ed178fb213d/ Log: Remove unreachable statement referring to nonexistant variable. diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py --- a/lib_pypy/_sqlite3.py +++ b/lib_pypy/_sqlite3.py @@ -861,8 +861,6 @@ except StopIteration: return None - return nextrow - def fetchmany(self, size=None): self._check_closed() self._check_reset() From noreply at buildbot.pypy.org Mon Jan 23 01:30:49 2012 From: noreply at buildbot.pypy.org (dripton) Date: Mon, 23 Jan 2012 01:30:49 +0100 (CET) Subject: [pypy-commit] pypy default: Remove unused import. Message-ID: <20120123003049.7A2A571027D@wyvern.cs.uni-duesseldorf.de> Author: David Ripton <dripton at ripton.net> Branch: Changeset: r51675:d39f5493dcaf Date: 2012-01-21 13:42 -0500 http://bitbucket.org/pypy/pypy/changeset/d39f5493dcaf/ Log: Remove unused import. diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py --- a/lib_pypy/_sqlite3.py +++ b/lib_pypy/_sqlite3.py @@ -27,7 +27,6 @@ from collections import OrderedDict import datetime import sys -import time import weakref from threading import _get_ident as thread_get_ident From noreply at buildbot.pypy.org Mon Jan 23 01:30:50 2012 From: noreply at buildbot.pypy.org (dripton) Date: Mon, 23 Jan 2012 01:30:50 +0100 (CET) Subject: [pypy-commit] pypy default: Remove unused local variables. Message-ID: <20120123003050.A79DC71027D@wyvern.cs.uni-duesseldorf.de> Author: David Ripton <dripton at ripton.net> Branch: Changeset: r51676:014a02a9d3df Date: 2012-01-21 13:43 -0500 http://bitbucket.org/pypy/pypy/changeset/014a02a9d3df/ Log: Remove unused local variables. diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py --- a/lib_pypy/_sqlite3.py +++ b/lib_pypy/_sqlite3.py @@ -605,7 +605,7 @@ def authorizer(userdata, action, arg1, arg2, dbname, source): try: return int(callback(action, arg1, arg2, dbname, source)) - except Exception, e: + except Exception: return SQLITE_DENY c_authorizer = AUTHORIZER(authorizer) @@ -652,7 +652,7 @@ if not aggregate_ptr[0]: try: aggregate = cls() - except Exception, e: + except Exception: msg = ("user-defined aggregate's '__init__' " "method raised error") sqlite.sqlite3_result_error(context, msg, len(msg)) @@ -666,7 +666,7 @@ params = _convert_params(context, argc, c_params) try: aggregate.step(*params) - except Exception, e: + except Exception: msg = ("user-defined aggregate's 'step' " "method raised error") sqlite.sqlite3_result_error(context, msg, len(msg)) @@ -682,7 +682,7 @@ aggregate = self.aggregate_instances[aggregate_ptr[0]] try: val = aggregate.finalize() - except Exception, e: + except Exception: msg = ("user-defined aggregate's 'finalize' " "method raised error") sqlite.sqlite3_result_error(context, msg, len(msg)) @@ -1046,7 +1046,7 @@ param_name = param_name[1:] try: param = params[param_name] - except KeyError, e: + except KeyError: raise ProgrammingError("missing parameter '%s'" %param) self.set_param(idx, param) @@ -1257,7 +1257,7 @@ params = _convert_params(context, nargs, c_params) try: val = real_cb(*params) - except Exception, e: + except Exception: msg = "user-defined function raised exception" sqlite.sqlite3_result_error(context, msg, len(msg)) else: From noreply at buildbot.pypy.org Mon Jan 23 01:30:51 2012 From: noreply at buildbot.pypy.org (dripton) Date: Mon, 23 Jan 2012 01:30:51 +0100 (CET) Subject: [pypy-commit] pypy default: Fix old-style raise statements (illegal in Python 3, ugly in Python 2) Message-ID: <20120123003051.D45C071027D@wyvern.cs.uni-duesseldorf.de> Author: David Ripton <dripton at ripton.net> Branch: Changeset: r51677:d49f44c4a62f Date: 2012-01-21 13:46 -0500 http://bitbucket.org/pypy/pypy/changeset/d49f44c4a62f/ Log: Fix old-style raise statements (illegal in Python 3, ugly in Python 2) diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py --- a/lib_pypy/_sqlite3.py +++ b/lib_pypy/_sqlite3.py @@ -790,7 +790,7 @@ if self.statement.kind == DML: self.connection._begin() else: - raise ProgrammingError, "executemany is only for DML statements" + raise ProgrammingError("executemany is only for DML statements") self.rowcount = 0 for params in many_params: @@ -912,7 +912,7 @@ def __init__(self, connection, sql): self.statement = None if not isinstance(sql, str): - raise ValueError, "sql must be a string" + raise ValueError("sql must be a string") self.con = connection self.sql = sql # DEBUG ONLY first_word = self._statement_kind = sql.lstrip().split(" ")[0].upper() @@ -941,8 +941,8 @@ raise self.con._get_exception(ret) self.con._remember_statement(self) if _check_remaining_sql(next_char.value): - raise Warning, "One and only one statement required: %r" % ( - next_char.value,) + raise Warning("One and only one statement required: %r" % ( + next_char.value,)) # sql_char should remain alive until here self._build_row_cast_map() @@ -1013,7 +1013,7 @@ elif type(param) is buffer: sqlite.sqlite3_bind_blob(self.statement, idx, str(param), len(param), SQLITE_TRANSIENT) else: - raise InterfaceError, "parameter type %s is not supported" % str(type(param)) + raise InterfaceError("parameter type %s is not supported" % str(type(param))) def set_params(self, params): ret = sqlite.sqlite3_reset(self.statement) @@ -1042,7 +1042,7 @@ for idx in range(1, sqlite.sqlite3_bind_parameter_count(self.statement) + 1): param_name = sqlite.sqlite3_bind_parameter_name(self.statement, idx) if param_name is None: - raise ProgrammingError, "need named parameters" + raise ProgrammingError("need named parameters") param_name = param_name[1:] try: param = params[param_name] From noreply at buildbot.pypy.org Mon Jan 23 01:30:53 2012 From: noreply at buildbot.pypy.org (dripton) Date: Mon, 23 Jan 2012 01:30:53 +0100 (CET) Subject: [pypy-commit] pypy default: Add a comment making it clear this file is modified, for license compliance. Message-ID: <20120123003053.0E53D71027D@wyvern.cs.uni-duesseldorf.de> Author: David Ripton <dripton at ripton.net> Branch: Changeset: r51678:01ed002a2b95 Date: 2012-01-21 20:17 -0500 http://bitbucket.org/pypy/pypy/changeset/01ed002a2b95/ Log: Add a comment making it clear this file is modified, for license compliance. diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py --- a/lib_pypy/_sqlite3.py +++ b/lib_pypy/_sqlite3.py @@ -20,6 +20,8 @@ # 2. Altered source versions must be plainly marked as such, and must not be # misrepresented as being the original software. # 3. This notice may not be removed or altered from any source distribution. +# +# Note: This software has been modified for use in PyPy. from ctypes import c_void_p, c_int, c_double, c_int64, c_char_p, cdll from ctypes import POINTER, byref, string_at, CFUNCTYPE, cast From noreply at buildbot.pypy.org Mon Jan 23 01:30:54 2012 From: noreply at buildbot.pypy.org (dripton) Date: Mon, 23 Jan 2012 01:30:54 +0100 (CET) Subject: [pypy-commit] pypy default: Failing test for issue 996 Message-ID: <20120123003054.3BB4971027D@wyvern.cs.uni-duesseldorf.de> Author: David Ripton <dripton at ripton.net> Branch: Changeset: r51679:6141b7352630 Date: 2012-01-22 16:18 -0500 http://bitbucket.org/pypy/pypy/changeset/6141b7352630/ Log: Failing test for issue 996 diff --git a/pypy/module/test_lib_pypy/test_sqlite3.py b/pypy/module/test_lib_pypy/test_sqlite3.py new file mode 100644 --- /dev/null +++ b/pypy/module/test_lib_pypy/test_sqlite3.py @@ -0,0 +1,9 @@ +"""Tests for _sqlite3.py""" + +def test_list_ddl(): + """From issue996. Just looking for lack of exceptions.""" + from sqlite3.dbapi2 import connect + connection = connect(':memory:') + cursor = connection.cursor() + cursor.execute('CREATE TABLE foo (bar INTEGER)') + result = list(cursor) From noreply at buildbot.pypy.org Mon Jan 23 01:30:55 2012 From: noreply at buildbot.pypy.org (dripton) Date: Mon, 23 Jan 2012 01:30:55 +0100 (CET) Subject: [pypy-commit] pypy default: Do not call statement.reset() after executing DDL. Message-ID: <20120123003055.6832D71027D@wyvern.cs.uni-duesseldorf.de> Author: David Ripton <dripton at ripton.net> Branch: Changeset: r51680:80014f10414c Date: 2012-01-22 17:46 -0500 http://bitbucket.org/pypy/pypy/changeset/80014f10414c/ Log: Do not call statement.reset() after executing DDL. Fixes issue996. diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py --- a/lib_pypy/_sqlite3.py +++ b/lib_pypy/_sqlite3.py @@ -772,7 +772,7 @@ self.statement.item = None self.statement.exhausted = True - if self.statement.kind == DML or self.statement.kind == DDL: + if self.statement.kind == DML: self.statement.reset() self.rowcount = -1 From noreply at buildbot.pypy.org Mon Jan 23 01:30:56 2012 From: noreply at buildbot.pypy.org (dripton) Date: Mon, 23 Jan 2012 01:30:56 +0100 (CET) Subject: [pypy-commit] pypy default: Add an assert, as requested by Alex. Message-ID: <20120123003056.934D671027D@wyvern.cs.uni-duesseldorf.de> Author: David Ripton <dripton at ripton.net> Branch: Changeset: r51681:fd6e2144e652 Date: 2012-01-22 18:30 -0500 http://bitbucket.org/pypy/pypy/changeset/fd6e2144e652/ Log: Add an assert, as requested by Alex. diff --git a/pypy/module/test_lib_pypy/test_sqlite3.py b/pypy/module/test_lib_pypy/test_sqlite3.py --- a/pypy/module/test_lib_pypy/test_sqlite3.py +++ b/pypy/module/test_lib_pypy/test_sqlite3.py @@ -1,9 +1,10 @@ """Tests for _sqlite3.py""" def test_list_ddl(): - """From issue996. Just looking for lack of exceptions.""" + """From issue996. Mostly just looking for lack of exceptions.""" from sqlite3.dbapi2 import connect connection = connect(':memory:') cursor = connection.cursor() cursor.execute('CREATE TABLE foo (bar INTEGER)') result = list(cursor) + assert result == [] From notifications-noreply at bitbucket.org Mon Jan 23 03:11:33 2012 From: notifications-noreply at bitbucket.org (Bitbucket) Date: Mon, 23 Jan 2012 02:11:33 -0000 Subject: [pypy-commit] [COMMENT] Pull request #23 for pypy/pypy: Fix _sqlite3.py fix for issue996, with test Message-ID: <20120123021133.10095.42275@bitbucket02.managed.contegix.com> New comment on pull request: https://bitbucket.org/pypy/pypy/pull-request/23/fix-_sqlite3py-fix-for-issue996-with-test#comment-1999 David Ripton (dripton) said: This has been merged. -- This is a pull request comment notification from bitbucket.org. You are receiving this either because you are participating in a pull request, or you are following it. From noreply at buildbot.pypy.org Mon Jan 23 10:36:00 2012 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 23 Jan 2012 10:36:00 +0100 (CET) Subject: [pypy-commit] pypy numpy-back-to-applevel: merge default Message-ID: <20120123093600.215EC82DFE@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: numpy-back-to-applevel Changeset: r51683:0068fa0d5e01 Date: 2012-01-23 11:27 +0200 http://bitbucket.org/pypy/pypy/changeset/0068fa0d5e01/ Log: merge default diff --git a/lib_pypy/datetime.py b/lib_pypy/datetime.py --- a/lib_pypy/datetime.py +++ b/lib_pypy/datetime.py @@ -13,7 +13,7 @@ Sources for time zone and DST data: http://www.twinsun.com/tz/tz-link.htm This was originally copied from the sandbox of the CPython CVS repository. -Thanks to Tim Peters for suggesting using it. +Thanks to Tim Peters for suggesting using it. """ import time as _time @@ -271,6 +271,8 @@ raise ValueError("%s()=%d, must be in -1439..1439" % (name, offset)) def _check_date_fields(year, month, day): + if not isinstance(year, (int, long)): + raise TypeError('int expected') if not MINYEAR <= year <= MAXYEAR: raise ValueError('year must be in %d..%d' % (MINYEAR, MAXYEAR), year) if not 1 <= month <= 12: @@ -280,6 +282,8 @@ raise ValueError('day must be in 1..%d' % dim, day) def _check_time_fields(hour, minute, second, microsecond): + if not isinstance(hour, (int, long)): + raise TypeError('int expected') if not 0 <= hour <= 23: raise ValueError('hour must be in 0..23', hour) if not 0 <= minute <= 59: @@ -543,61 +547,76 @@ self = object.__new__(cls) - self.__days = d - self.__seconds = s - self.__microseconds = us + self._days = d + self._seconds = s + self._microseconds = us if abs(d) > 999999999: raise OverflowError("timedelta # of days is too large: %d" % d) return self def __repr__(self): - if self.__microseconds: + if self._microseconds: return "%s(%d, %d, %d)" % ('datetime.' + self.__class__.__name__, - self.__days, - self.__seconds, - self.__microseconds) - if self.__seconds: + self._days, + self._seconds, + self._microseconds) + if self._seconds: return "%s(%d, %d)" % ('datetime.' + self.__class__.__name__, - self.__days, - self.__seconds) - return "%s(%d)" % ('datetime.' + self.__class__.__name__, self.__days) + self._days, + self._seconds) + return "%s(%d)" % ('datetime.' + self.__class__.__name__, self._days) def __str__(self): - mm, ss = divmod(self.__seconds, 60) + mm, ss = divmod(self._seconds, 60) hh, mm = divmod(mm, 60) s = "%d:%02d:%02d" % (hh, mm, ss) - if self.__days: + if self._days: def plural(n): return n, abs(n) != 1 and "s" or "" - s = ("%d day%s, " % plural(self.__days)) + s - if self.__microseconds: - s = s + ".%06d" % self.__microseconds + s = ("%d day%s, " % plural(self._days)) + s + if self._microseconds: + s = s + ".%06d" % self._microseconds return s - days = property(lambda self: self.__days, doc="days") - seconds = property(lambda self: self.__seconds, doc="seconds") - microseconds = property(lambda self: self.__microseconds, - doc="microseconds") - def total_seconds(self): return ((self.days * 86400 + self.seconds) * 10**6 + self.microseconds) / 1e6 + # Read-only field accessors + @property + def days(self): + """days""" + return self._days + + @property + def seconds(self): + """seconds""" + return self._seconds + + @property + def microseconds(self): + """microseconds""" + return self._microseconds + def __add__(self, other): if isinstance(other, timedelta): # for CPython compatibility, we cannot use # our __class__ here, but need a real timedelta - return timedelta(self.__days + other.__days, - self.__seconds + other.__seconds, - self.__microseconds + other.__microseconds) + return timedelta(self._days + other._days, + self._seconds + other._seconds, + self._microseconds + other._microseconds) return NotImplemented __radd__ = __add__ def __sub__(self, other): if isinstance(other, timedelta): - return self + -other + # for CPython compatibility, we cannot use + # our __class__ here, but need a real timedelta + return timedelta(self._days - other._days, + self._seconds - other._seconds, + self._microseconds - other._microseconds) return NotImplemented def __rsub__(self, other): @@ -606,17 +625,17 @@ return NotImplemented def __neg__(self): - # for CPython compatibility, we cannot use - # our __class__ here, but need a real timedelta - return timedelta(-self.__days, - -self.__seconds, - -self.__microseconds) + # for CPython compatibility, we cannot use + # our __class__ here, but need a real timedelta + return timedelta(-self._days, + -self._seconds, + -self._microseconds) def __pos__(self): return self def __abs__(self): - if self.__days < 0: + if self._days < 0: return -self else: return self @@ -625,81 +644,81 @@ if isinstance(other, (int, long)): # for CPython compatibility, we cannot use # our __class__ here, but need a real timedelta - return timedelta(self.__days * other, - self.__seconds * other, - self.__microseconds * other) + return timedelta(self._days * other, + self._seconds * other, + self._microseconds * other) return NotImplemented __rmul__ = __mul__ def __div__(self, other): if isinstance(other, (int, long)): - usec = ((self.__days * (24*3600L) + self.__seconds) * 1000000 + - self.__microseconds) + usec = ((self._days * (24*3600L) + self._seconds) * 1000000 + + self._microseconds) return timedelta(0, 0, usec // other) return NotImplemented __floordiv__ = __div__ - # Comparisons. + # Comparisons of timedelta objects with other. def __eq__(self, other): if isinstance(other, timedelta): - return self.__cmp(other) == 0 + return self._cmp(other) == 0 else: return False def __ne__(self, other): if isinstance(other, timedelta): - return self.__cmp(other) != 0 + return self._cmp(other) != 0 else: return True def __le__(self, other): if isinstance(other, timedelta): - return self.__cmp(other) <= 0 + return self._cmp(other) <= 0 else: _cmperror(self, other) def __lt__(self, other): if isinstance(other, timedelta): - return self.__cmp(other) < 0 + return self._cmp(other) < 0 else: _cmperror(self, other) def __ge__(self, other): if isinstance(other, timedelta): - return self.__cmp(other) >= 0 + return self._cmp(other) >= 0 else: _cmperror(self, other) def __gt__(self, other): if isinstance(other, timedelta): - return self.__cmp(other) > 0 + return self._cmp(other) > 0 else: _cmperror(self, other) - def __cmp(self, other): + def _cmp(self, other): assert isinstance(other, timedelta) - return cmp(self.__getstate(), other.__getstate()) + return cmp(self._getstate(), other._getstate()) def __hash__(self): - return hash(self.__getstate()) + return hash(self._getstate()) def __nonzero__(self): - return (self.__days != 0 or - self.__seconds != 0 or - self.__microseconds != 0) + return (self._days != 0 or + self._seconds != 0 or + self._microseconds != 0) # Pickle support. __safe_for_unpickling__ = True # For Python 2.2 - def __getstate(self): - return (self.__days, self.__seconds, self.__microseconds) + def _getstate(self): + return (self._days, self._seconds, self._microseconds) def __reduce__(self): - return (self.__class__, self.__getstate()) + return (self.__class__, self._getstate()) timedelta.min = timedelta(-999999999) timedelta.max = timedelta(days=999999999, hours=23, minutes=59, seconds=59, @@ -749,25 +768,26 @@ return self _check_date_fields(year, month, day) self = object.__new__(cls) - self.__year = year - self.__month = month - self.__day = day + self._year = year + self._month = month + self._day = day return self # Additional constructors + @classmethod def fromtimestamp(cls, t): "Construct a date from a POSIX timestamp (like time.time())." y, m, d, hh, mm, ss, weekday, jday, dst = _time.localtime(t) return cls(y, m, d) - fromtimestamp = classmethod(fromtimestamp) + @classmethod def today(cls): "Construct a date from time.time()." t = _time.time() return cls.fromtimestamp(t) - today = classmethod(today) + @classmethod def fromordinal(cls, n): """Contruct a date from a proleptic Gregorian ordinal. @@ -776,16 +796,24 @@ """ y, m, d = _ord2ymd(n) return cls(y, m, d) - fromordinal = classmethod(fromordinal) # Conversions to string def __repr__(self): - "Convert to formal string, for repr()." + """Convert to formal string, for repr(). + + >>> dt = datetime(2010, 1, 1) + >>> repr(dt) + 'datetime.datetime(2010, 1, 1, 0, 0)' + + >>> dt = datetime(2010, 1, 1, tzinfo=timezone.utc) + >>> repr(dt) + 'datetime.datetime(2010, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)' + """ return "%s(%d, %d, %d)" % ('datetime.' + self.__class__.__name__, - self.__year, - self.__month, - self.__day) + self._year, + self._month, + self._day) # XXX These shouldn't depend on time.localtime(), because that # clips the usable dates to [1970 .. 2038). At least ctime() is # easily done without using strftime() -- that's better too because @@ -793,12 +821,20 @@ def ctime(self): "Format a la ctime()." - return tmxxx(self.__year, self.__month, self.__day).ctime() + return tmxxx(self._year, self._month, self._day).ctime() def strftime(self, fmt): "Format using strftime()." return _wrap_strftime(self, fmt, self.timetuple()) + def __format__(self, fmt): + if not isinstance(fmt, (str, unicode)): + raise ValueError("__format__ excepts str or unicode, not %s" % + fmt.__class__.__name__) + if len(fmt) != 0: + return self.strftime(fmt) + return str(self) + def isoformat(self): """Return the date formatted according to ISO. @@ -808,29 +844,31 @@ - http://www.w3.org/TR/NOTE-datetime - http://www.cl.cam.ac.uk/~mgk25/iso-time.html """ - return "%04d-%02d-%02d" % (self.__year, self.__month, self.__day) + return "%04d-%02d-%02d" % (self._year, self._month, self._day) __str__ = isoformat - def __format__(self, format): - if not isinstance(format, (str, unicode)): - raise ValueError("__format__ excepts str or unicode, not %s" % - format.__class__.__name__) - if not format: - return str(self) - return self.strftime(format) + # Read-only field accessors + @property + def year(self): + """year (1-9999)""" + return self._year - # Read-only field accessors - year = property(lambda self: self.__year, - doc="year (%d-%d)" % (MINYEAR, MAXYEAR)) - month = property(lambda self: self.__month, doc="month (1-12)") - day = property(lambda self: self.__day, doc="day (1-31)") + @property + def month(self): + """month (1-12)""" + return self._month + + @property + def day(self): + """day (1-31)""" + return self._day # Standard conversions, __cmp__, __hash__ (and helpers) def timetuple(self): "Return local time tuple compatible with time.localtime()." - return _build_struct_time(self.__year, self.__month, self.__day, + return _build_struct_time(self._year, self._month, self._day, 0, 0, 0, -1) def toordinal(self): @@ -839,24 +877,24 @@ January 1 of year 1 is day 1. Only the year, month and day values contribute to the result. """ - return _ymd2ord(self.__year, self.__month, self.__day) + return _ymd2ord(self._year, self._month, self._day) def replace(self, year=None, month=None, day=None): """Return a new date with new values for the specified fields.""" if year is None: - year = self.__year + year = self._year if month is None: - month = self.__month + month = self._month if day is None: - day = self.__day + day = self._day _check_date_fields(year, month, day) return date(year, month, day) - # Comparisons. + # Comparisons of date objects with other. def __eq__(self, other): if isinstance(other, date): - return self.__cmp(other) == 0 + return self._cmp(other) == 0 elif hasattr(other, "timetuple"): return NotImplemented else: @@ -864,7 +902,7 @@ def __ne__(self, other): if isinstance(other, date): - return self.__cmp(other) != 0 + return self._cmp(other) != 0 elif hasattr(other, "timetuple"): return NotImplemented else: @@ -872,7 +910,7 @@ def __le__(self, other): if isinstance(other, date): - return self.__cmp(other) <= 0 + return self._cmp(other) <= 0 elif hasattr(other, "timetuple"): return NotImplemented else: @@ -880,7 +918,7 @@ def __lt__(self, other): if isinstance(other, date): - return self.__cmp(other) < 0 + return self._cmp(other) < 0 elif hasattr(other, "timetuple"): return NotImplemented else: @@ -888,7 +926,7 @@ def __ge__(self, other): if isinstance(other, date): - return self.__cmp(other) >= 0 + return self._cmp(other) >= 0 elif hasattr(other, "timetuple"): return NotImplemented else: @@ -896,21 +934,21 @@ def __gt__(self, other): if isinstance(other, date): - return self.__cmp(other) > 0 + return self._cmp(other) > 0 elif hasattr(other, "timetuple"): return NotImplemented else: _cmperror(self, other) - def __cmp(self, other): + def _cmp(self, other): assert isinstance(other, date) - y, m, d = self.__year, self.__month, self.__day - y2, m2, d2 = other.__year, other.__month, other.__day + y, m, d = self._year, self._month, self._day + y2, m2, d2 = other._year, other._month, other._day return cmp((y, m, d), (y2, m2, d2)) def __hash__(self): "Hash." - return hash(self.__getstate()) + return hash(self._getstate()) # Computations @@ -922,9 +960,9 @@ def __add__(self, other): "Add a date to a timedelta." if isinstance(other, timedelta): - t = tmxxx(self.__year, - self.__month, - self.__day + other.days) + t = tmxxx(self._year, + self._month, + self._day + other.days) self._checkOverflow(t.year) result = date(t.year, t.month, t.day) return result @@ -966,9 +1004,9 @@ ISO calendar algorithm taken from http://www.phys.uu.nl/~vgent/calendar/isocalendar.htm """ - year = self.__year + year = self._year week1monday = _isoweek1monday(year) - today = _ymd2ord(self.__year, self.__month, self.__day) + today = _ymd2ord(self._year, self._month, self._day) # Internally, week and day have origin 0 week, day = divmod(today - week1monday, 7) if week < 0: @@ -985,18 +1023,18 @@ __safe_for_unpickling__ = True # For Python 2.2 - def __getstate(self): - yhi, ylo = divmod(self.__year, 256) - return ("%c%c%c%c" % (yhi, ylo, self.__month, self.__day), ) + def _getstate(self): + yhi, ylo = divmod(self._year, 256) + return ("%c%c%c%c" % (yhi, ylo, self._month, self._day), ) def __setstate(self, string): if len(string) != 4 or not (1 <= ord(string[2]) <= 12): raise TypeError("not enough arguments") - yhi, ylo, self.__month, self.__day = map(ord, string) - self.__year = yhi * 256 + ylo + yhi, ylo, self._month, self._day = map(ord, string) + self._year = yhi * 256 + ylo def __reduce__(self): - return (self.__class__, self.__getstate()) + return (self.__class__, self._getstate()) _date_class = date # so functions w/ args named "date" can get at the class @@ -1118,62 +1156,80 @@ return self _check_tzinfo_arg(tzinfo) _check_time_fields(hour, minute, second, microsecond) - self.__hour = hour - self.__minute = minute - self.__second = second - self.__microsecond = microsecond + self._hour = hour + self._minute = minute + self._second = second + self._microsecond = microsecond self._tzinfo = tzinfo return self # Read-only field accessors - hour = property(lambda self: self.__hour, doc="hour (0-23)") - minute = property(lambda self: self.__minute, doc="minute (0-59)") - second = property(lambda self: self.__second, doc="second (0-59)") - microsecond = property(lambda self: self.__microsecond, - doc="microsecond (0-999999)") - tzinfo = property(lambda self: self._tzinfo, doc="timezone info object") + @property + def hour(self): + """hour (0-23)""" + return self._hour + + @property + def minute(self): + """minute (0-59)""" + return self._minute + + @property + def second(self): + """second (0-59)""" + return self._second + + @property + def microsecond(self): + """microsecond (0-999999)""" + return self._microsecond + + @property + def tzinfo(self): + """timezone info object""" + return self._tzinfo # Standard conversions, __hash__ (and helpers) - # Comparisons. + # Comparisons of time objects with other. def __eq__(self, other): if isinstance(other, time): - return self.__cmp(other) == 0 + return self._cmp(other) == 0 else: return False def __ne__(self, other): if isinstance(other, time): - return self.__cmp(other) != 0 + return self._cmp(other) != 0 else: return True def __le__(self, other): if isinstance(other, time): - return self.__cmp(other) <= 0 + return self._cmp(other) <= 0 else: _cmperror(self, other) def __lt__(self, other): if isinstance(other, time): - return self.__cmp(other) < 0 + return self._cmp(other) < 0 else: _cmperror(self, other) def __ge__(self, other): if isinstance(other, time): - return self.__cmp(other) >= 0 + return self._cmp(other) >= 0 else: _cmperror(self, other) def __gt__(self, other): if isinstance(other, time): - return self.__cmp(other) > 0 + return self._cmp(other) > 0 else: _cmperror(self, other) - def __cmp(self, other): + def _cmp(self, other): assert isinstance(other, time) mytz = self._tzinfo ottz = other._tzinfo @@ -1187,23 +1243,23 @@ base_compare = myoff == otoff if base_compare: - return cmp((self.__hour, self.__minute, self.__second, - self.__microsecond), - (other.__hour, other.__minute, other.__second, - other.__microsecond)) + return cmp((self._hour, self._minute, self._second, + self._microsecond), + (other._hour, other._minute, other._second, + other._microsecond)) if myoff is None or otoff is None: # XXX Buggy in 2.2.2. raise TypeError("cannot compare naive and aware times") - myhhmm = self.__hour * 60 + self.__minute - myoff - othhmm = other.__hour * 60 + other.__minute - otoff - return cmp((myhhmm, self.__second, self.__microsecond), - (othhmm, other.__second, other.__microsecond)) + myhhmm = self._hour * 60 + self._minute - myoff + othhmm = other._hour * 60 + other._minute - otoff + return cmp((myhhmm, self._second, self._microsecond), + (othhmm, other._second, other._microsecond)) def __hash__(self): """Hash.""" tzoff = self._utcoffset() if not tzoff: # zero or None - return hash(self.__getstate()[0]) + return hash(self._getstate()[0]) h, m = divmod(self.hour * 60 + self.minute - tzoff, 60) if 0 <= h < 24: return hash(time(h, m, self.second, self.microsecond)) @@ -1227,14 +1283,14 @@ def __repr__(self): """Convert to formal string, for repr().""" - if self.__microsecond != 0: - s = ", %d, %d" % (self.__second, self.__microsecond) - elif self.__second != 0: - s = ", %d" % self.__second + if self._microsecond != 0: + s = ", %d, %d" % (self._second, self._microsecond) + elif self._second != 0: + s = ", %d" % self._second else: s = "" s= "%s(%d, %d%s)" % ('datetime.' + self.__class__.__name__, - self.__hour, self.__minute, s) + self._hour, self._minute, s) if self._tzinfo is not None: assert s[-1:] == ")" s = s[:-1] + ", tzinfo=%r" % self._tzinfo + ")" @@ -1246,8 +1302,8 @@ This is 'HH:MM:SS.mmmmmm+zz:zz', or 'HH:MM:SS+zz:zz' if self.microsecond == 0. """ - s = _format_time(self.__hour, self.__minute, self.__second, - self.__microsecond) + s = _format_time(self._hour, self._minute, self._second, + self._microsecond) tz = self._tzstr() if tz: s += tz @@ -1255,14 +1311,6 @@ __str__ = isoformat - def __format__(self, format): - if not isinstance(format, (str, unicode)): - raise ValueError("__format__ excepts str or unicode, not %s" % - format.__class__.__name__) - if not format: - return str(self) - return self.strftime(format) - def strftime(self, fmt): """Format using strftime(). The date part of the timestamp passed to underlying strftime should not be used. @@ -1270,10 +1318,18 @@ # The year must be >= 1900 else Python's strftime implementation # can raise a bogus exception. timetuple = (1900, 1, 1, - self.__hour, self.__minute, self.__second, + self._hour, self._minute, self._second, 0, 1, -1) return _wrap_strftime(self, fmt, timetuple) + def __format__(self, fmt): + if not isinstance(fmt, (str, unicode)): + raise ValueError("__format__ excepts str or unicode, not %s" % + fmt.__class__.__name__) + if len(fmt) != 0: + return self.strftime(fmt) + return str(self) + # Timezone functions def utcoffset(self): @@ -1350,10 +1406,10 @@ __safe_for_unpickling__ = True # For Python 2.2 - def __getstate(self): - us2, us3 = divmod(self.__microsecond, 256) + def _getstate(self): + us2, us3 = divmod(self._microsecond, 256) us1, us2 = divmod(us2, 256) - basestate = ("%c" * 6) % (self.__hour, self.__minute, self.__second, + basestate = ("%c" * 6) % (self._hour, self._minute, self._second, us1, us2, us3) if self._tzinfo is None: return (basestate,) @@ -1363,13 +1419,13 @@ def __setstate(self, string, tzinfo): if len(string) != 6 or ord(string[0]) >= 24: raise TypeError("an integer is required") - self.__hour, self.__minute, self.__second, us1, us2, us3 = \ + self._hour, self._minute, self._second, us1, us2, us3 = \ map(ord, string) - self.__microsecond = (((us1 << 8) | us2) << 8) | us3 + self._microsecond = (((us1 << 8) | us2) << 8) | us3 self._tzinfo = tzinfo def __reduce__(self): - return (time, self.__getstate()) + return (time, self._getstate()) _time_class = time # so functions w/ args named "time" can get at the class @@ -1378,9 +1434,11 @@ time.resolution = timedelta(microseconds=1) class datetime(date): + """datetime(year, month, day[, hour[, minute[, second[, microsecond[,tzinfo]]]]]) - # XXX needs docstrings - # See http://www.zope.org/Members/fdrake/DateTimeWiki/TimeZoneInfo + The year, month and day arguments are required. tzinfo may be None, or an + instance of a tzinfo subclass. The remaining arguments may be ints or longs. + """ def __new__(cls, year, month=None, day=None, hour=0, minute=0, second=0, microsecond=0, tzinfo=None): @@ -1393,24 +1451,43 @@ _check_time_fields(hour, minute, second, microsecond) self = date.__new__(cls, year, month, day) # XXX This duplicates __year, __month, __day for convenience :-( - self.__year = year - self.__month = month - self.__day = day - self.__hour = hour - self.__minute = minute - self.__second = second - self.__microsecond = microsecond + self._year = year + self._month = month + self._day = day + self._hour = hour + self._minute = minute + self._second = second + self._microsecond = microsecond self._tzinfo = tzinfo return self # Read-only field accessors - hour = property(lambda self: self.__hour, doc="hour (0-23)") - minute = property(lambda self: self.__minute, doc="minute (0-59)") - second = property(lambda self: self.__second, doc="second (0-59)") - microsecond = property(lambda self: self.__microsecond, - doc="microsecond (0-999999)") - tzinfo = property(lambda self: self._tzinfo, doc="timezone info object") + @property + def hour(self): + """hour (0-23)""" + return self._hour + @property + def minute(self): + """minute (0-59)""" + return self._minute + + @property + def second(self): + """second (0-59)""" + return self._second + + @property + def microsecond(self): + """microsecond (0-999999)""" + return self._microsecond + + @property + def tzinfo(self): + """timezone info object""" + return self._tzinfo + + @classmethod def fromtimestamp(cls, t, tz=None): """Construct a datetime from a POSIX timestamp (like time.time()). @@ -1438,7 +1515,6 @@ if tz is not None: result = tz.fromutc(result) return result - fromtimestamp = classmethod(fromtimestamp) @classmethod def utcfromtimestamp(cls, t): @@ -1462,18 +1538,19 @@ # XXX uses gettimeofday on platforms that have it, but that isn't # XXX available from Python. So now() may return different results # XXX across the implementations. + @classmethod def now(cls, tz=None): "Construct a datetime from time.time() and optional time zone info." t = _time.time() return cls.fromtimestamp(t, tz) - now = classmethod(now) + @classmethod def utcnow(cls): "Construct a UTC datetime from time.time()." t = _time.time() return cls.utcfromtimestamp(t) - utcnow = classmethod(utcnow) + @classmethod def combine(cls, date, time): "Construct a datetime from a given date and a given time." if not isinstance(date, _date_class): @@ -1483,7 +1560,6 @@ return cls(date.year, date.month, date.day, time.hour, time.minute, time.second, time.microsecond, time.tzinfo) - combine = classmethod(combine) def timetuple(self): "Return local time tuple compatible with time.localtime()." @@ -1509,7 +1585,7 @@ def date(self): "Return the date part." - return date(self.__year, self.__month, self.__day) + return date(self._year, self._month, self._day) def time(self): "Return the time part, with tzinfo None." @@ -1569,8 +1645,8 @@ def ctime(self): "Format a la ctime()." - t = tmxxx(self.__year, self.__month, self.__day, self.__hour, - self.__minute, self.__second) + t = tmxxx(self._year, self._month, self._day, self._hour, + self._minute, self._second) return t.ctime() def isoformat(self, sep='T'): @@ -1585,10 +1661,10 @@ Optional argument sep specifies the separator between date and time, default 'T'. """ - s = ("%04d-%02d-%02d%c" % (self.__year, self.__month, self.__day, + s = ("%04d-%02d-%02d%c" % (self._year, self._month, self._day, sep) + - _format_time(self.__hour, self.__minute, self.__second, - self.__microsecond)) + _format_time(self._hour, self._minute, self._second, + self._microsecond)) off = self._utcoffset() if off is not None: if off < 0: @@ -1601,13 +1677,13 @@ return s def __repr__(self): - "Convert to formal string, for repr()." - L = [self.__year, self.__month, self.__day, # These are never zero - self.__hour, self.__minute, self.__second, self.__microsecond] + """Convert to formal string, for repr().""" + L = [self._year, self._month, self._day, # These are never zero + self._hour, self._minute, self._second, self._microsecond] if L[-1] == 0: del L[-1] if L[-1] == 0: - del L[-1] + del L[-1] s = ", ".join(map(str, L)) s = "%s(%s)" % ('datetime.' + self.__class__.__name__, s) if self._tzinfo is not None: @@ -1680,7 +1756,7 @@ def __eq__(self, other): if isinstance(other, datetime): - return self.__cmp(other) == 0 + return self._cmp(other) == 0 elif hasattr(other, "timetuple") and not isinstance(other, date): return NotImplemented else: @@ -1688,7 +1764,7 @@ def __ne__(self, other): if isinstance(other, datetime): - return self.__cmp(other) != 0 + return self._cmp(other) != 0 elif hasattr(other, "timetuple") and not isinstance(other, date): return NotImplemented else: @@ -1696,7 +1772,7 @@ def __le__(self, other): if isinstance(other, datetime): - return self.__cmp(other) <= 0 + return self._cmp(other) <= 0 elif hasattr(other, "timetuple") and not isinstance(other, date): return NotImplemented else: @@ -1704,7 +1780,7 @@ def __lt__(self, other): if isinstance(other, datetime): - return self.__cmp(other) < 0 + return self._cmp(other) < 0 elif hasattr(other, "timetuple") and not isinstance(other, date): return NotImplemented else: @@ -1712,7 +1788,7 @@ def __ge__(self, other): if isinstance(other, datetime): - return self.__cmp(other) >= 0 + return self._cmp(other) >= 0 elif hasattr(other, "timetuple") and not isinstance(other, date): return NotImplemented else: @@ -1720,13 +1796,13 @@ def __gt__(self, other): if isinstance(other, datetime): - return self.__cmp(other) > 0 + return self._cmp(other) > 0 elif hasattr(other, "timetuple") and not isinstance(other, date): return NotImplemented else: _cmperror(self, other) - def __cmp(self, other): + def _cmp(self, other): assert isinstance(other, datetime) mytz = self._tzinfo ottz = other._tzinfo @@ -1742,12 +1818,12 @@ base_compare = myoff == otoff if base_compare: - return cmp((self.__year, self.__month, self.__day, - self.__hour, self.__minute, self.__second, - self.__microsecond), - (other.__year, other.__month, other.__day, - other.__hour, other.__minute, other.__second, - other.__microsecond)) + return cmp((self._year, self._month, self._day, + self._hour, self._minute, self._second, + self._microsecond), + (other._year, other._month, other._day, + other._hour, other._minute, other._second, + other._microsecond)) if myoff is None or otoff is None: # XXX Buggy in 2.2.2. raise TypeError("cannot compare naive and aware datetimes") @@ -1761,13 +1837,13 @@ "Add a datetime and a timedelta." if not isinstance(other, timedelta): return NotImplemented - t = tmxxx(self.__year, - self.__month, - self.__day + other.days, - self.__hour, - self.__minute, - self.__second + other.seconds, - self.__microsecond + other.microseconds) + t = tmxxx(self._year, + self._month, + self._day + other.days, + self._hour, + self._minute, + self._second + other.seconds, + self._microsecond + other.microseconds) self._checkOverflow(t.year) result = datetime(t.year, t.month, t.day, t.hour, t.minute, t.second, @@ -1785,11 +1861,11 @@ days1 = self.toordinal() days2 = other.toordinal() - secs1 = self.__second + self.__minute * 60 + self.__hour * 3600 - secs2 = other.__second + other.__minute * 60 + other.__hour * 3600 + secs1 = self._second + self._minute * 60 + self._hour * 3600 + secs2 = other._second + other._minute * 60 + other._hour * 3600 base = timedelta(days1 - days2, secs1 - secs2, - self.__microsecond - other.__microsecond) + self._microsecond - other._microsecond) if self._tzinfo is other._tzinfo: return base myoff = self._utcoffset() @@ -1797,13 +1873,13 @@ if myoff == otoff: return base if myoff is None or otoff is None: - raise TypeError, "cannot mix naive and timezone-aware time" + raise TypeError("cannot mix naive and timezone-aware time") return base + timedelta(minutes = otoff-myoff) def __hash__(self): tzoff = self._utcoffset() if tzoff is None: - return hash(self.__getstate()[0]) + return hash(self._getstate()[0]) days = _ymd2ord(self.year, self.month, self.day) seconds = self.hour * 3600 + (self.minute - tzoff) * 60 + self.second return hash(timedelta(days, seconds, self.microsecond)) @@ -1812,12 +1888,12 @@ __safe_for_unpickling__ = True # For Python 2.2 - def __getstate(self): - yhi, ylo = divmod(self.__year, 256) - us2, us3 = divmod(self.__microsecond, 256) + def _getstate(self): + yhi, ylo = divmod(self._year, 256) + us2, us3 = divmod(self._microsecond, 256) us1, us2 = divmod(us2, 256) - basestate = ("%c" * 10) % (yhi, ylo, self.__month, self.__day, - self.__hour, self.__minute, self.__second, + basestate = ("%c" * 10) % (yhi, ylo, self._month, self._day, + self._hour, self._minute, self._second, us1, us2, us3) if self._tzinfo is None: return (basestate,) @@ -1825,14 +1901,14 @@ return (basestate, self._tzinfo) def __setstate(self, string, tzinfo): - (yhi, ylo, self.__month, self.__day, self.__hour, - self.__minute, self.__second, us1, us2, us3) = map(ord, string) - self.__year = yhi * 256 + ylo - self.__microsecond = (((us1 << 8) | us2) << 8) | us3 + (yhi, ylo, self._month, self._day, self._hour, + self._minute, self._second, us1, us2, us3) = map(ord, string) + self._year = yhi * 256 + ylo + self._microsecond = (((us1 << 8) | us2) << 8) | us3 self._tzinfo = tzinfo def __reduce__(self): - return (self.__class__, self.__getstate()) + return (self.__class__, self._getstate()) datetime.min = datetime(1, 1, 1) @@ -2014,7 +2090,7 @@ Because we know z.d said z was in daylight time (else [5] would have held and we would have stopped then), and we know z.d != z'.d (else [8] would have held -and we we have stopped then), and there are only 2 possible values dst() can +and we have stopped then), and there are only 2 possible values dst() can return in Eastern, it follows that z'.d must be 0 (which it is in the example, but the reasoning doesn't depend on the example -- it depends on there being two possible dst() outcomes, one zero and the other non-zero). Therefore diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -491,8 +491,8 @@ concrete = self.get_concrete() new_shape = get_shape_from_iterable(space, concrete.size, w_shape) # Since we got to here, prod(new_shape) == self.size - new_strides = calc_new_strides(new_shape, - concrete.shape, concrete.strides) + new_strides = calc_new_strides(new_shape, concrete.shape, + concrete.strides, concrete.order) if new_strides: # We can create a view, strides somehow match up. ndims = len(new_shape) @@ -1083,7 +1083,8 @@ self.backstrides = backstrides self.shape = new_shape return - new_strides = calc_new_strides(new_shape, self.shape, self.strides) + new_strides = calc_new_strides(new_shape, self.shape, self.strides, + self.order) if new_strides is None: raise OperationError(space.w_AttributeError, space.wrap( "incompatible shape for a non-contiguous array")) diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py --- a/pypy/module/micronumpy/strides.py +++ b/pypy/module/micronumpy/strides.py @@ -38,7 +38,6 @@ rbackstrides = [0] * (len(res_shape) - len(orig_shape)) + rbackstrides return rstrides, rbackstrides - def find_shape_and_elems(space, w_iterable): shape = [space.len_w(w_iterable)] batch = space.listview(w_iterable) @@ -154,15 +153,17 @@ # fits the new shape, using those steps. If there is a shape/step mismatch # (meaning that the realignment of elements crosses from one step into another) # return None so that the caller can raise an exception. -def calc_new_strides(new_shape, old_shape, old_strides): +def calc_new_strides(new_shape, old_shape, old_strides, order): + # Return the proper strides for new_shape, or None if the mapping crosses + # stepping boundaries + # Assumes that prod(old_shape) == prod(new_shape), len(old_shape) > 1, and # len(new_shape) > 0 steps = [] last_step = 1 oldI = 0 new_strides = [] - if old_strides[0] < old_strides[-1]: - #Start at old_shape[0], old_stides[0] + if order == 'F': for i in range(len(old_shape)): steps.append(old_strides[i] / last_step) last_step *= old_shape[i] @@ -179,12 +180,10 @@ n_old_elems_to_use *= old_shape[oldI] if n_new_elems_used == n_old_elems_to_use: oldI += 1 - if oldI >= len(old_shape): - continue - cur_step = steps[oldI] - n_old_elems_to_use *= old_shape[oldI] - else: - #Start at old_shape[-1], old_strides[-1] + if oldI < len(old_shape): + cur_step = steps[oldI] + n_old_elems_to_use *= old_shape[oldI] + elif order == 'C': for i in range(len(old_shape) - 1, -1, -1): steps.insert(0, old_strides[i] / last_step) last_step *= old_shape[i] @@ -203,8 +202,8 @@ n_old_elems_to_use *= old_shape[oldI] if n_new_elems_used == n_old_elems_to_use: oldI -= 1 - if oldI < -len(old_shape): - continue - cur_step = steps[oldI] - n_old_elems_to_use *= old_shape[oldI] + if oldI >= -len(old_shape): + cur_step = steps[oldI] + n_old_elems_to_use *= old_shape[oldI] + assert len(new_strides) == len(new_shape) return new_strides diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -154,13 +154,18 @@ def test_calc_new_strides(self): from pypy.module.micronumpy.interp_numarray import calc_new_strides - assert calc_new_strides([2, 4], [4, 2], [4, 2]) == [8, 2] - assert calc_new_strides([2, 4, 3], [8, 3], [1, 16]) == [1, 2, 16] - assert calc_new_strides([2, 3, 4], [8, 3], [1, 16]) is None - assert calc_new_strides([24], [2, 4, 3], [48, 6, 1]) is None - assert calc_new_strides([24], [2, 4, 3], [24, 6, 2]) == [2] - assert calc_new_strides([105, 1], [3, 5, 7], [35, 7, 1]) == [1, 1] - assert calc_new_strides([1, 105], [3, 5, 7], [35, 7, 1]) == [105, 1] + assert calc_new_strides([2, 4], [4, 2], [4, 2], "C") == [8, 2] + assert calc_new_strides([2, 4, 3], [8, 3], [1, 16], 'F') == [1, 2, 16] + assert calc_new_strides([2, 3, 4], [8, 3], [1, 16], 'F') is None + assert calc_new_strides([24], [2, 4, 3], [48, 6, 1], 'C') is None + assert calc_new_strides([24], [2, 4, 3], [24, 6, 2], 'C') == [2] + assert calc_new_strides([105, 1], [3, 5, 7], [35, 7, 1],'C') == [1, 1] + assert calc_new_strides([1, 105], [3, 5, 7], [35, 7, 1],'C') == [105, 1] + assert calc_new_strides([1, 105], [3, 5, 7], [35, 7, 1],'F') is None + assert calc_new_strides([1, 1, 1, 105, 1], [15, 7], [7, 1],'C') == \ + [105, 105, 105, 1, 1] + assert calc_new_strides([1, 1, 105, 1, 1], [7, 15], [1, 7],'F') == \ + [1, 1, 1, 105, 105] class AppTestNumArray(BaseNumpyAppTest): @@ -385,6 +390,8 @@ a.shape = () #numpy allows this a.shape = (1,) + a = array(range(6)).reshape(2,3).T + raises(AttributeError, 'a.shape = 6') def test_reshape(self): from _numpypy import array, zeros diff --git a/pypy/module/pypyjit/test_pypy_c/test_math.py b/pypy/module/pypyjit/test_pypy_c/test_math.py --- a/pypy/module/pypyjit/test_pypy_c/test_math.py +++ b/pypy/module/pypyjit/test_pypy_c/test_math.py @@ -90,3 +90,19 @@ --TICK-- jump(..., descr=) """) + + def test_pow_two(self): + def main(n): + s = 0.123 + while n > 0: + s -= s ** 2 # ID: pow + n -= 1 + return s + log = self.run(main, [500]) + assert abs(log.result - main(500)) < 1e-9 + loop, = log.loops_by_filename(self.filepath) + assert loop.match_by_id("pow", """ + guard_not_invalidated(descr=...) + f38 = float_mul(f30, f30) + f39 = float_sub(f30, f38) + """) diff --git a/pypy/objspace/std/floatobject.py b/pypy/objspace/std/floatobject.py --- a/pypy/objspace/std/floatobject.py +++ b/pypy/objspace/std/floatobject.py @@ -443,6 +443,8 @@ y = w_float2.floatval # Sort out special cases here instead of relying on pow() + if y == 2.0: # special case for performance: + return W_FloatObject(x * x) # x * x is always correct if y == 0.0: # x**0 is 1, even 0**0 return W_FloatObject(1.0) diff --git a/pypy/rpython/lltypesystem/module/ll_math.py b/pypy/rpython/lltypesystem/module/ll_math.py --- a/pypy/rpython/lltypesystem/module/ll_math.py +++ b/pypy/rpython/lltypesystem/module/ll_math.py @@ -292,10 +292,6 @@ # deal directly with IEEE specials, to cope with problems on various # platforms whose semantics don't exactly match C99 - if y == 2.0: - return x * x # this is always a correct answer, and is relatively - # common in user programs. - if isnan(y): if x == 1.0: return 1.0 # 1**Nan = 1 diff --git a/pypy/translator/c/src/profiling.c b/pypy/translator/c/src/profiling.c --- a/pypy/translator/c/src/profiling.c +++ b/pypy/translator/c/src/profiling.c @@ -29,33 +29,33 @@ profiling_setup = 0; } } - -#elif defined(_WIN32) -#include <windows.h> - -DWORD_PTR base_affinity_mask; -int profiling_setup = 0; - -void pypy_setup_profiling() { - if (!profiling_setup) { - DWORD_PTR affinity_mask, system_affinity_mask; - GetProcessAffinityMask(GetCurrentProcess(), - &base_affinity_mask, &system_affinity_mask); - affinity_mask = 1; - /* Pick one cpu allowed by the system */ - if (system_affinity_mask) - while ((affinity_mask & system_affinity_mask) == 0) - affinity_mask <<= 1; - SetProcessAffinityMask(GetCurrentProcess(), affinity_mask); - profiling_setup = 1; - } -} - -void pypy_teardown_profiling() { - if (profiling_setup) { - SetProcessAffinityMask(GetCurrentProcess(), base_affinity_mask); - profiling_setup = 0; - } + +#elif defined(_WIN32) +#include <windows.h> + +DWORD_PTR base_affinity_mask; +int profiling_setup = 0; + +void pypy_setup_profiling() { + if (!profiling_setup) { + DWORD_PTR affinity_mask, system_affinity_mask; + GetProcessAffinityMask(GetCurrentProcess(), + &base_affinity_mask, &system_affinity_mask); + affinity_mask = 1; + /* Pick one cpu allowed by the system */ + if (system_affinity_mask) + while ((affinity_mask & system_affinity_mask) == 0) + affinity_mask <<= 1; + SetProcessAffinityMask(GetCurrentProcess(), affinity_mask); + profiling_setup = 1; + } +} + +void pypy_teardown_profiling() { + if (profiling_setup) { + SetProcessAffinityMask(GetCurrentProcess(), base_affinity_mask); + profiling_setup = 0; + } } #else From noreply at buildbot.pypy.org Mon Jan 23 11:18:52 2012 From: noreply at buildbot.pypy.org (mattip) Date: Mon, 23 Jan 2012 11:18:52 +0100 (CET) Subject: [pypy-commit] pypy numppy-flatitter: correct faulty tests Message-ID: <20120123101852.AED7E82DFE@wyvern.cs.uni-duesseldorf.de> Author: mattip Branch: numppy-flatitter Changeset: r51684:dd2ec11ec817 Date: 2012-01-23 11:02 +0200 http://bitbucket.org/pypy/pypy/changeset/dd2ec11ec817/ Log: correct faulty tests diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -1304,7 +1304,7 @@ elif space.isinstance_w(w_item_or_slice, space.w_slice): start, stop, step, lngth = space.decode_index4(w_item_or_slice,arr.size) else: - operationerrfmt(space.NotImplementedError,'cannot iterator over %s yet', w_item_or_slice) + raise operationerrfmt(space.w_NotImplementedError,'cannot iterator over %s yet', w_item_or_slice) retval = [] for i in range(start, stop, step): coords = [] diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -1317,10 +1317,10 @@ b = a.T.flat b[6::2] = [-1, -2] assert (a == [[0, 1, -1, 3], [4, 5, 6, -1], [8, 9, -2, 11]]).all() - b[1:2] = [[[100]]] + b[0:2] = [[[100]]] assert(a[0,0] == 100) assert(a[1,0] == 100) - b[array([10, 11])] == [-20, -40] + raises(NotImplementedError, 'b[array([10, 11])] == [-20, -40]') def test_flatiter_view(self): From noreply at buildbot.pypy.org Mon Jan 23 12:37:19 2012 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 23 Jan 2012 12:37:19 +0100 (CET) Subject: [pypy-commit] buildbot default: require an executable Message-ID: <20120123113719.CB39582DFE@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: Changeset: r610:1c946f853e6b Date: 2012-01-23 13:33 +0200 http://bitbucket.org/pypy/buildbot/changeset/1c946f853e6b/ Log: require an executable diff --git a/numready.py b/numready.py --- a/numready.py +++ b/numready.py @@ -197,10 +197,8 @@ return l def main(argv): - assert platform.python_implementation() == "PyPy" - cpy_items = find_numpy_items("/usr/bin/python") - pypy_items = find_numpy_items(sys.executable, "numpypy") + pypy_items = find_numpy_items(argv[1], "numpypy") all_items = [] msg = '%d/%d names, %d/%d ndarray attributes, %d/%d dtype attributes' % ( From noreply at buildbot.pypy.org Mon Jan 23 12:37:20 2012 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 23 Jan 2012 12:37:20 +0100 (CET) Subject: [pypy-commit] buildbot default: minor adjustments Message-ID: <20120123113720.E67F882DFE@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: Changeset: r611:914834b4058a Date: 2012-01-23 13:36 +0200 http://bitbucket.org/pypy/buildbot/changeset/914834b4058a/ Log: minor adjustments diff --git a/numready.py b/numready.py --- a/numready.py +++ b/numready.py @@ -214,10 +214,13 @@ ) all_items.append(ItemStatus(item.name, pypy_exists=item in pypy_items)) html = jinja2.Template(PAGE_TEMPLATE).render(all_items=split(sorted(all_items)), msg=msg) - with tempfile.NamedTemporaryFile(delete=False) as f: - f.write(html.encode("utf-8")) - webbrowser.open_new_tab(f.name) - + if len(argv) > 2: + with open(argv[2], 'w') as f: + f.write(html.encode("utf-8")) + else: + with tempfile.NamedTemporaryFile(delete=False) as f: + f.write(html.encode("utf-8")) + print "Saved in: %s" % f.name if __name__ == '__main__': main(sys.argv) From noreply at buildbot.pypy.org Mon Jan 23 15:28:35 2012 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 23 Jan 2012 15:28:35 +0100 (CET) Subject: [pypy-commit] pypy default: Move these three functions at interp-level: potential threading issues Message-ID: <20120123142835.5441082DFE@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: Changeset: r51685:c32fbc60b64c Date: 2012-01-23 15:28 +0100 http://bitbucket.org/pypy/pypy/changeset/c32fbc60b64c/ Log: Move these three functions at interp-level: potential threading issues if they are written at app-level. diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py --- a/pypy/interpreter/executioncontext.py +++ b/pypy/interpreter/executioncontext.py @@ -445,6 +445,7 @@ AsyncAction.__init__(self, space) self.dying_objects = [] self.finalizers_lock_count = 0 + self.enabled_at_app_level = True def register_callback(self, w_obj, callback, descrname): self.dying_objects.append((w_obj, callback, descrname)) diff --git a/pypy/module/gc/__init__.py b/pypy/module/gc/__init__.py --- a/pypy/module/gc/__init__.py +++ b/pypy/module/gc/__init__.py @@ -1,18 +1,18 @@ from pypy.interpreter.mixedmodule import MixedModule class Module(MixedModule): - appleveldefs = { - 'enable': 'app_gc.enable', - 'disable': 'app_gc.disable', - 'isenabled': 'app_gc.isenabled', - } interpleveldefs = { 'collect': 'interp_gc.collect', + 'enable': 'interp_gc.enable', + 'disable': 'interp_gc.disable', + 'isenabled': 'interp_gc.isenabled', 'enable_finalizers': 'interp_gc.enable_finalizers', 'disable_finalizers': 'interp_gc.disable_finalizers', 'garbage' : 'space.newlist([])', #'dump_heap_stats': 'interp_gc.dump_heap_stats', } + appleveldefs = { + } def __init__(self, space, w_name): if (not space.config.translating or diff --git a/pypy/module/gc/app_gc.py b/pypy/module/gc/app_gc.py deleted file mode 100644 --- a/pypy/module/gc/app_gc.py +++ /dev/null @@ -1,21 +0,0 @@ -# NOT_RPYTHON - -enabled = True - -def isenabled(): - global enabled - return enabled - -def enable(): - global enabled - import gc - if not enabled: - gc.enable_finalizers() - enabled = True - -def disable(): - global enabled - import gc - if enabled: - gc.disable_finalizers() - enabled = False diff --git a/pypy/module/gc/interp_gc.py b/pypy/module/gc/interp_gc.py --- a/pypy/module/gc/interp_gc.py +++ b/pypy/module/gc/interp_gc.py @@ -17,6 +17,26 @@ rgc.collect() return space.wrap(0) +def enable(space): + """Non-recursive version. Enable finalizers now. + If they were already enabled, no-op. + If they were disabled even several times, enable them anyway. + """ + if not space.user_del_action.enabled_at_app_level: + space.user_del_action.enabled_at_app_level = True + enable_finalizers(space) + +def disable(space): + """Non-recursive version. Disable finalizers now. Several calls + to this function are ignored. + """ + if space.user_del_action.enabled_at_app_level: + space.user_del_action.enabled_at_app_level = False + disable_finalizers(space) + +def isenabled(space): + return space.newbool(space.user_del_action.enabled_at_app_level) + def enable_finalizers(space): if space.user_del_action.finalizers_lock_count == 0: raise OperationError(space.w_ValueError, From noreply at buildbot.pypy.org Mon Jan 23 15:34:54 2012 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 23 Jan 2012 15:34:54 +0100 (CET) Subject: [pypy-commit] pypy default: Try to be more careful about free()ing this in case something fails. Message-ID: <20120123143454.B4A1D82DFE@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: Changeset: r51686:598e02bd0eb8 Date: 2012-01-23 15:34 +0100 http://bitbucket.org/pypy/pypy/changeset/598e02bd0eb8/ Log: Try to be more careful about free()ing this in case something fails. diff --git a/pypy/module/_hashlib/interp_hashlib.py b/pypy/module/_hashlib/interp_hashlib.py --- a/pypy/module/_hashlib/interp_hashlib.py +++ b/pypy/module/_hashlib/interp_hashlib.py @@ -34,8 +34,12 @@ ctx = lltype.malloc(ropenssl.EVP_MD_CTX.TO, flavor='raw') rgc.add_memory_pressure(HASH_MALLOC_SIZE + self.digest_size) - ropenssl.EVP_DigestInit(ctx, digest_type) - self.ctx = ctx + try: + ropenssl.EVP_DigestInit(ctx, digest_type) + self.ctx = ctx + except: + lltype.free(ctx, flavor='raw') + raise def __del__(self): # self.lock.free() From noreply at buildbot.pypy.org Mon Jan 23 16:55:38 2012 From: noreply at buildbot.pypy.org (mattip) Date: Mon, 23 Jan 2012 16:55:38 +0100 (CET) Subject: [pypy-commit] pypy matrixmath-dot: added test from numpy docstring Message-ID: <20120123155538.C11F982E07@wyvern.cs.uni-duesseldorf.de> Author: mattip Branch: matrixmath-dot Changeset: r51687:3431bdf3f8a1 Date: 2012-01-22 02:09 +0200 http://bitbucket.org/pypy/pypy/changeset/3431bdf3f8a1/ Log: added test from numpy docstring diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -892,6 +892,10 @@ [[86, 302, 518], [110, 390, 670], [134, 478, 822]]]).all() c = dot(a, b[:, 2]) assert (c == [[62, 214, 366], [518, 670, 822]]).all() + a = arange(3*4*5*6).reshape((3,4,5,6)) + b = arange(3*4*5*6)[::-1].reshape((5,4,6,3)) + assert dot(a, b)[2,3,2,1,2,2] == 499128 + assert sum(a[2,3,2,:] * b[1,2,:,2]) == 499128 def test_dot_constant(self): from _numpypy import array From noreply at buildbot.pypy.org Mon Jan 23 16:55:40 2012 From: noreply at buildbot.pypy.org (mattip) Date: Mon, 23 Jan 2012 16:55:40 +0100 (CET) Subject: [pypy-commit] pypy numpypy-reshape: close useless branch Message-ID: <20120123155540.0A91E82E08@wyvern.cs.uni-duesseldorf.de> Author: mattip Branch: numpypy-reshape Changeset: r51688:1c041f97db71 Date: 2012-01-23 17:54 +0200 http://bitbucket.org/pypy/pypy/changeset/1c041f97db71/ Log: close useless branch From noreply at buildbot.pypy.org Mon Jan 23 17:30:23 2012 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 23 Jan 2012 17:30:23 +0100 (CET) Subject: [pypy-commit] pypy stm: add the stm_thread_id() helper Message-ID: <20120123163023.A053382E07@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: stm Changeset: r51689:6eb15b35a17d Date: 2012-01-23 16:16 +0100 http://bitbucket.org/pypy/pypy/changeset/6eb15b35a17d/ Log: add the stm_thread_id() helper diff --git a/pypy/translator/stm/_rffi_stm.py b/pypy/translator/stm/_rffi_stm.py --- a/pypy/translator/stm/_rffi_stm.py +++ b/pypy/translator/stm/_rffi_stm.py @@ -38,3 +38,4 @@ stm_abort_and_retry = llexternal('stm_abort_and_retry', [], lltype.Void) stm_debug_get_state = llexternal('stm_debug_get_state', [], lltype.Signed) +stm_thread_id = llexternal('stm_thread_id', [], lltype.Signed) diff --git a/pypy/translator/stm/src_stm/et.c b/pypy/translator/stm/src_stm/et.c --- a/pypy/translator/stm/src_stm/et.c +++ b/pypy/translator/stm/src_stm/et.c @@ -71,13 +71,14 @@ unsigned num_aborts[ABORT_REASONS]; unsigned num_spinloops[SPINLOOP_REASONS]; unsigned int spinloop_counter; + int transaction_active; owner_version_t my_lock_word; struct RedoLog redolog; /* last item, because it's the biggest one */ - int transaction_active; }; static const struct tx_descriptor null_tx = { - .transaction_active = 0 + .transaction_active = 0, + .my_lock_word = 0 }; #define NULL_TX ((struct tx_descriptor *)(&null_tx)) @@ -833,4 +834,10 @@ return 2; } +long stm_thread_id(void) +{ + struct tx_descriptor *d = thread_descriptor; + return d->my_lock_word; +} + #endif /* PYPY_NOT_MAIN_FILE */ diff --git a/pypy/translator/stm/src_stm/et.h b/pypy/translator/stm/src_stm/et.h --- a/pypy/translator/stm/src_stm/et.h +++ b/pypy/translator/stm/src_stm/et.h @@ -31,6 +31,8 @@ 0: not in a transaction 1: in a regular transaction 2: in an inevitable transaction */ +long stm_thread_id(void); /* returns a unique thread id, + or 0 if descriptor_init() was not called */ // XXX little-endian only! /* this macro is used if 'base' is a word-aligned pointer and 'offset' diff --git a/pypy/translator/stm/test/test_rffi_stm.py b/pypy/translator/stm/test/test_rffi_stm.py --- a/pypy/translator/stm/test/test_rffi_stm.py +++ b/pypy/translator/stm/test/test_rffi_stm.py @@ -68,3 +68,10 @@ stm_perform_transaction(llhelper(CALLBACK, callback1), lltype.nullptr(rffi.VOIDP.TO)) stm_descriptor_done() + +def test_stm_thread_id(): + assert stm_thread_id() == 0 + stm_descriptor_init() + assert stm_thread_id() != 0 + stm_descriptor_done() + assert stm_thread_id() == 0 From noreply at buildbot.pypy.org Mon Jan 23 17:30:24 2012 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 23 Jan 2012 17:30:24 +0100 (CET) Subject: [pypy-commit] pypy stm: More hacking... Message-ID: <20120123163024.CE90782E07@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: stm Changeset: r51690:3c5c86ba384f Date: 2012-01-23 16:20 +0100 http://bitbucket.org/pypy/pypy/changeset/3c5c86ba384f/ Log: More hacking... diff --git a/pypy/module/transaction/__init__.py b/pypy/module/transaction/__init__.py --- a/pypy/module/transaction/__init__.py +++ b/pypy/module/transaction/__init__.py @@ -21,5 +21,6 @@ def translating_for_checkmodule(self, space): from pypy.module.transaction import interp_transaction + interp_transaction.state._freeze_() interp_transaction.state.space = space interp_transaction.state._freeze_ = lambda: None # hack! From noreply at buildbot.pypy.org Mon Jan 23 17:30:26 2012 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 23 Jan 2012 17:30:26 +0100 (CET) Subject: [pypy-commit] pypy stm: Use thread_id() from stm instead of the one from threads. Message-ID: <20120123163026.0FF3682E07@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: stm Changeset: r51691:db48523657f2 Date: 2012-01-23 16:21 +0100 http://bitbucket.org/pypy/pypy/changeset/db48523657f2/ Log: Use thread_id() from stm instead of the one from threads. diff --git a/pypy/module/transaction/interp_transaction.py b/pypy/module/transaction/interp_transaction.py --- a/pypy/module/transaction/interp_transaction.py +++ b/pypy/module/transaction/interp_transaction.py @@ -7,6 +7,8 @@ NUM_THREADS_DEFAULT = 4 # by default +MAIN_THREAD_ID = 0 + class State(object): @@ -19,7 +21,6 @@ self.ll_lock = threadintf.null_ll_lock self.ll_no_tasks_pending_lock = threadintf.null_ll_lock self.ll_unfinished_lock = threadintf.null_ll_lock - self.main_thread_id = 0 self.w_error = None def startup(self, space): @@ -31,8 +32,7 @@ self.ll_no_tasks_pending_lock = threadintf.allocate_lock() self.ll_unfinished_lock = threadintf.allocate_lock() self.lock_unfinished() - self.main_thread_id = threadintf.thread_id() - self.pending_lists = {self.main_thread_id: self.pending} + self.pending_lists = {MAIN_THREAD_ID: self.pending} def set_num_threads(self, num): if self.running: @@ -88,7 +88,7 @@ self.args = args def register(self): - id = threadintf.thread_id() + id = rstm.thread_id() state.pending_lists[id].append(self) def run(self): @@ -112,7 +112,7 @@ Pending(w_callback, __args__).register() -def add_list(new_pending_list): +def _add_list(new_pending_list): if new_pending_list.is_empty(): return was_empty = state.pending.is_empty() @@ -123,10 +123,11 @@ def _run_thread(): state.lock() + rstm.descriptor_init() my_pending_list = Fifo() - my_thread_id = threadintf.thread_id() + my_thread_id = rstm.thread_id() + assert my_thread_id not in state.pending_lists state.pending_lists[my_thread_id] = my_pending_list - rstm.descriptor_init() # while True: if state.pending.is_empty(): @@ -151,7 +152,7 @@ state.unlock() pending.run() state.lock() - add_list(my_pending_list) + _add_list(my_pending_list) # rstm.descriptor_done() del state.pending_lists[my_thread_id] @@ -180,7 +181,7 @@ # assert state.num_waiting_threads == 0 assert state.pending.is_empty() - assert state.pending_lists.keys() == [state.main_thread_id] + assert state.pending_lists.keys() == [MAIN_THREAD_ID] assert not state.is_locked_no_tasks_pending() state.running = False # diff --git a/pypy/module/transaction/threadintf.py b/pypy/module/transaction/threadintf.py --- a/pypy/module/transaction/threadintf.py +++ b/pypy/module/transaction/threadintf.py @@ -28,9 +28,3 @@ ll_thread.start_new_thread(callback, args) else: thread.start_new_thread(callback, args) - -def thread_id(): - if we_are_translated(): - return ll_thread.get_ident() - else: - return thread.get_ident() diff --git a/pypy/rlib/rstm.py b/pypy/rlib/rstm.py --- a/pypy/rlib/rstm.py +++ b/pypy/rlib/rstm.py @@ -55,3 +55,6 @@ def debug_get_state(): return _rffi_stm.stm_debug_get_state() + +def thread_id(): + return _rffi_stm.stm_thread_id() From noreply at buildbot.pypy.org Mon Jan 23 17:30:27 2012 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 23 Jan 2012 17:30:27 +0100 (CET) Subject: [pypy-commit] pypy stm: Make the ExecutionContext again a thread-local. See comments. Message-ID: <20120123163027.3D7A982E07@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: stm Changeset: r51692:e5329461f8ca Date: 2012-01-23 17:15 +0100 http://bitbucket.org/pypy/pypy/changeset/e5329461f8ca/ Log: Make the ExecutionContext again a thread-local. See comments. diff --git a/pypy/module/transaction/__init__.py b/pypy/module/transaction/__init__.py --- a/pypy/module/transaction/__init__.py +++ b/pypy/module/transaction/__init__.py @@ -15,12 +15,13 @@ 'TransactionError': 'app_transaction.TransactionError', } + def __init__(self, space, *args): + "NOT_RPYTHON: patches space.threadlocals to use real threadlocals" + from pypy.module.transaction import interp_transaction + MixedModule.__init__(self, space, *args) + interp_transaction.state.initialize(space) + space.threadlocals = interp_transaction.state + def startup(self, space): from pypy.module.transaction import interp_transaction - interp_transaction.state.startup(space) - - def translating_for_checkmodule(self, space): - from pypy.module.transaction import interp_transaction - interp_transaction.state._freeze_() - interp_transaction.state.space = space - interp_transaction.state._freeze_ = lambda: None # hack! + interp_transaction.state.startup(space, space.wrap(self)) diff --git a/pypy/module/transaction/interp_transaction.py b/pypy/module/transaction/interp_transaction.py --- a/pypy/module/transaction/interp_transaction.py +++ b/pypy/module/transaction/interp_transaction.py @@ -12,27 +12,73 @@ class State(object): - def _freeze_(self): - self.__dict__.clear() + def initialize(self, space): + self.space = space self.running = False self.num_threads = NUM_THREADS_DEFAULT - self.pending = Fifo() - self.pending_lists = {0: self.pending} + # + self.w_error = None self.ll_lock = threadintf.null_ll_lock self.ll_no_tasks_pending_lock = threadintf.null_ll_lock self.ll_unfinished_lock = threadintf.null_ll_lock - self.w_error = None + self.threadobjs = {} # empty during translation + self.pending = Fifo() - def startup(self, space): - self.space = space - w_module = space.getbuiltinmodule('transaction') - self.w_error = space.getattr(w_module, space.wrap('TransactionError')) - # + def _freeze_(self): + self.threadobjs.clear() + return False + + def startup(self, space, w_module): + assert space is self.space + if w_module is not None: # for tests + self.w_error = space.getattr(w_module, + space.wrap('TransactionError')) self.ll_lock = threadintf.allocate_lock() self.ll_no_tasks_pending_lock = threadintf.allocate_lock() self.ll_unfinished_lock = threadintf.allocate_lock() self.lock_unfinished() - self.pending_lists = {MAIN_THREAD_ID: self.pending} + self.startup_run() + + def startup_run(self): + # this is called at the start of run() too, in order to make + # test_checkmodule happy + main_ec = self.space.getexecutioncontext() # create it if needed + main_ec._transaction_pending = self.pending + + def add_thread(self, id, ec): + # register a new transaction thread + assert id not in self.threadobjs + ec._transaction_pending = Fifo() + self.threadobjs[id] = ec + + def del_thread(self, id): + # un-register a transaction thread + del self.threadobjs[id] + + # ---------- interface for ThreadLocals ---------- + # This works really like a thread-local, which may have slightly + # strange consequences in multiple transactions, because you don't + # know on which thread a transaction will run. The point of this is + # to let every thread get its own ExecutionContext; otherwise, they + # conflict with each other e.g. when setting the 'topframeref' + # attribute. + + def getvalue(self): + id = rstm.thread_id() + return self.threadobjs.get(id, None) + + def setvalue(self, value): + id = rstm.thread_id() + assert id == MAIN_THREAD_ID # should not be used from a transaction + self.threadobjs[id] = value + + def getmainthreadvalue(self): + return self.threadobjs[0] + + def getallvalues(self): + return self.threadobjs + + # ---------- def set_num_threads(self, num): if self.running: @@ -70,7 +116,6 @@ state = State() -state._freeze_() @unwrap_spec(num=int) @@ -88,8 +133,8 @@ self.args = args def register(self): - id = rstm.thread_id() - state.pending_lists[id].append(self) + ec = state.getvalue() + ec._transaction_pending.append(self) def run(self): rstm.perform_transaction(Pending._run_in_transaction, Pending, self) @@ -124,10 +169,9 @@ def _run_thread(): state.lock() rstm.descriptor_init() - my_pending_list = Fifo() my_thread_id = rstm.thread_id() - assert my_thread_id not in state.pending_lists - state.pending_lists[my_thread_id] = my_pending_list + my_ec = state.space.createexecutioncontext() + state.add_thread(my_thread_id, my_ec) # while True: if state.pending.is_empty(): @@ -152,10 +196,10 @@ state.unlock() pending.run() state.lock() - _add_list(my_pending_list) + _add_list(my_ec._transaction_pending) # + state.del_thread(my_thread_id) rstm.descriptor_done() - del state.pending_lists[my_thread_id] if state.num_waiting_threads == 0: # only the last thread to leave state.unlock_unfinished() state.unlock() @@ -166,6 +210,7 @@ raise OperationError( state.w_error, space.wrap("recursive invocation of transaction.run()")) + state.startup_run() assert not state.is_locked_no_tasks_pending() if state.pending.is_empty(): return @@ -181,7 +226,7 @@ # assert state.num_waiting_threads == 0 assert state.pending.is_empty() - assert state.pending_lists.keys() == [MAIN_THREAD_ID] + assert state.threadobjs.keys() == [MAIN_THREAD_ID] assert not state.is_locked_no_tasks_pending() state.running = False # diff --git a/pypy/module/transaction/test/test_interp_transaction.py b/pypy/module/transaction/test/test_interp_transaction.py --- a/pypy/module/transaction/test/test_interp_transaction.py +++ b/pypy/module/transaction/test/test_interp_transaction.py @@ -11,13 +11,29 @@ return 'some stuff from the transaction module' def wrap(self, x): return 'wrapped stuff' + def getexecutioncontext(self): + ec = interp_transaction.state.getvalue() + if ec is None: + ec = self.createexecutioncontext() + interp_transaction.state.setvalue(ec) + return ec + def createexecutioncontext(self): + return FakeEC() def call_args(self, w_callback, args): w_callback(*args) +class FakeEC: + pass + +def make_fake_space(): + space = FakeSpace() + interp_transaction.state.initialize(space) + interp_transaction.state.startup(space, None) + return space + def test_linear_list(): - space = FakeSpace() - interp_transaction.state.startup(space) + space = make_fake_space() seen = [] # def do(n): @@ -32,8 +48,7 @@ def test_tree_of_transactions(): - space = FakeSpace() - interp_transaction.state.startup(space) + space = make_fake_space() seen = [] # def do(level): @@ -51,8 +66,7 @@ def test_transactional_simple(): - space = FakeSpace() - interp_transaction.state.startup(space) + space = make_fake_space() lst = [] def f(n): lst.append(n+0) From noreply at buildbot.pypy.org Mon Jan 23 17:30:28 2012 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 23 Jan 2012 17:30:28 +0100 (CET) Subject: [pypy-commit] pypy default: Improve the docstring and re-export weakref.ref() under the name Message-ID: <20120123163028.6904482E07@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: Changeset: r51693:207092a8b272 Date: 2012-01-23 17:30 +0100 http://bitbucket.org/pypy/pypy/changeset/207092a8b272/ Log: Improve the docstring and re-export weakref.ref() under the name pypy.rlib.rweakref.ref(). diff --git a/pypy/rlib/rweakref.py b/pypy/rlib/rweakref.py --- a/pypy/rlib/rweakref.py +++ b/pypy/rlib/rweakref.py @@ -1,11 +1,14 @@ """ -Weakref support in RPython. Supports ref() without callbacks, +Weakref support in RPython. Basic regular weakrefs without callbacks +are supported. This file contains the following additions: a form of WeakKeyDictionary, and a limited version of WeakValueDictionary. LLType only for now! """ import weakref +ref = weakref.ref # basic regular weakrefs are supported in RPython + class RWeakValueDictionary(object): """A dictionary containing weak values.""" From noreply at buildbot.pypy.org Mon Jan 23 19:20:42 2012 From: noreply at buildbot.pypy.org (amauryfa) Date: Mon, 23 Jan 2012 19:20:42 +0100 (CET) Subject: [pypy-commit] pypy merge-2.7.2: Fix multibytecodec incremental tests: ensure that .encode(final=True) Message-ID: <20120123182042.399FA82E07@wyvern.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc <amauryfa at gmail.com> Branch: merge-2.7.2 Changeset: r51694:9c27a0100918 Date: 2012-01-23 19:19 +0100 http://bitbucket.org/pypy/pypy/changeset/9c27a0100918/ Log: Fix multibytecodec incremental tests: ensure that .encode(final=True) is always called at the end. Also have MultibyteStreamWriter.reset() call .encode(final=True) to flush the pending bytes. diff --git a/pypy/module/_multibytecodec/app_multibytecodec.py b/pypy/module/_multibytecodec/app_multibytecodec.py --- a/pypy/module/_multibytecodec/app_multibytecodec.py +++ b/pypy/module/_multibytecodec/app_multibytecodec.py @@ -40,7 +40,13 @@ return self def write(self, data): - self.stream.write(MultibyteIncrementalEncoder.encode(self, data)) + self.stream.write(MultibyteIncrementalEncoder.encode( + self, data)) + + def reset(self): + self.stream.write(MultibyteIncrementalEncoder.encode( + self, '', final=True)) + MultibyteIncrementalEncoder.reset(self) def writelines(self, lines): for data in lines: diff --git a/pypy/module/_multibytecodec/test/test_app_incremental.py b/pypy/module/_multibytecodec/test/test_app_incremental.py --- a/pypy/module/_multibytecodec/test/test_app_incremental.py +++ b/pypy/module/_multibytecodec/test/test_app_incremental.py @@ -104,11 +104,11 @@ r = e.encode("abcd") assert r == 'abcd' r = e.encode(u"\u5f95\u6c85") - assert r == '~{abcd~}' + assert r == '~{abcd' r = e.encode(u"\u5f50") - assert r == '~{ef~}' - r = e.encode(u"\u73b7") - assert r == '~{gh~}' + assert r == 'ef' + r = e.encode(u"\u73b7", final=True) + assert r == 'gh~}' def test_encode_hz_final(self): e = self.IncrementalHzEncoder() @@ -127,7 +127,9 @@ assert r == 'xyz~{abcd~}' e.reset() r = e.encode(u"xyz\u5f95\u6c85") - assert r == 'xyz~{abcd~}' + assert r == 'xyz~{abcd' + r = e.encode(u'', final=True) + assert r == '~}' def test_encode_hz_noreset(self): text = (u'\u5df1\u6240\u4e0d\u6b32\uff0c\u52ff\u65bd\u65bc\u4eba\u3002' diff --git a/pypy/module/_multibytecodec/test/test_app_stream.py b/pypy/module/_multibytecodec/test/test_app_stream.py --- a/pypy/module/_multibytecodec/test/test_app_stream.py +++ b/pypy/module/_multibytecodec/test/test_app_stream.py @@ -76,8 +76,9 @@ w = self.HzStreamWriter(FakeFile()) for input in u'!\u5f95\u6c85xyz\u5f50\u73b7': w.write(input) - assert w.stream.output == ['!', '~{ab~}', '~{cd~}', 'x', 'y', 'z', - '~{ef~}', '~{gh~}'] + w.reset() + assert w.stream.output == ['!', '~{ab', 'cd', '~}x', 'y', 'z', + '~{ef', 'gh', '~}'] def test_no_flush(self): class FakeFile: From noreply at buildbot.pypy.org Mon Jan 23 20:11:12 2012 From: noreply at buildbot.pypy.org (amauryfa) Date: Mon, 23 Jan 2012 20:11:12 +0100 (CET) Subject: [pypy-commit] pypy merge-2.7.2: Test and fix in mixed read/write/seek/write operations Message-ID: <20120123191112.C4C0B82E07@wyvern.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc <amauryfa at gmail.com> Branch: merge-2.7.2 Changeset: r51695:bf9a09ac155b Date: 2012-01-23 20:10 +0100 http://bitbucket.org/pypy/pypy/changeset/bf9a09ac155b/ Log: Test and fix in mixed read/write/seek/write operations diff --git a/pypy/module/_io/interp_bufferedio.py b/pypy/module/_io/interp_bufferedio.py --- a/pypy/module/_io/interp_bufferedio.py +++ b/pypy/module/_io/interp_bufferedio.py @@ -268,6 +268,7 @@ if pos < 0: raise OperationError(space.w_IOError, space.wrap( "Raw stream returned invalid position")) + self.abs_pos = pos return pos def _closed(self, space): @@ -638,7 +639,7 @@ if size <= available: for i in range(size): self.buffer[self.pos + i] = data[i] - if self.write_end == -1: + if self.write_end == -1 or self.write_pos > self.pos: self.write_pos = self.pos self._adjust_position(self.pos + size) if self.pos > self.write_end: diff --git a/pypy/module/_io/test/test_bufferedio.py b/pypy/module/_io/test/test_bufferedio.py --- a/pypy/module/_io/test/test_bufferedio.py +++ b/pypy/module/_io/test/test_bufferedio.py @@ -467,3 +467,30 @@ f.write('xxxx') f.seek(0) assert f.read() == 'a\nbxxxx' + + def test_write_rewind_write(self): + # Various combinations of reading / writing / seeking + # backwards / writing again + import _io, errno + def mutate(bufio, pos1, pos2): + assert pos2 >= pos1 + # Fill the buffer + bufio.seek(pos1) + bufio.read(pos2 - pos1) + bufio.write(b'\x02') + # This writes earlier than the previous write, but still inside + # the buffer. + bufio.seek(pos1) + bufio.write(b'\x01') + + b = b"\x80\x81\x82\x83\x84" + for i in range(0, len(b)): + for j in range(i, len(b)): + raw = _io.BytesIO(b) + bufio = _io.BufferedRandom(raw, 100) + mutate(bufio, i, j) + bufio.flush() + expected = bytearray(b) + expected[j] = 2 + expected[i] = 1 + assert raw.getvalue() == str(expected) From noreply at buildbot.pypy.org Mon Jan 23 22:14:20 2012 From: noreply at buildbot.pypy.org (amauryfa) Date: Mon, 23 Jan 2012 22:14:20 +0100 (CET) Subject: [pypy-commit] pypy merge-2.7.2: The underlying stream of a BufferedIO can call arbitrary code, Message-ID: <20120123211420.551D182E07@wyvern.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc <amauryfa at gmail.com> Branch: merge-2.7.2 Changeset: r51696:624143b36c8d Date: 2012-01-23 22:13 +0100 http://bitbucket.org/pypy/pypy/changeset/624143b36c8d/ Log: The underlying stream of a BufferedIO can call arbitrary code, or yield to some signal handler; protect against reentrant code. diff --git a/pypy/module/_io/interp_bufferedio.py b/pypy/module/_io/interp_bufferedio.py --- a/pypy/module/_io/interp_bufferedio.py +++ b/pypy/module/_io/interp_bufferedio.py @@ -11,6 +11,7 @@ W_IOBase, DEFAULT_BUFFER_SIZE, convert_size, check_readable_w, check_writable_w, check_seekable_w) from pypy.module._io.interp_io import W_BlockingIOError +from pypy.module.thread import ll_thread import errno STATE_ZERO, STATE_OK, STATE_DETACHED = range(3) @@ -28,6 +29,29 @@ return False +class TryLock(object): + "A Lock that raises RuntimeError when acquired twice by the same thread" + def __init__(self, space): + ## XXX cannot free a Lock? + ## if self.lock: + ## self.lock.free() + self.lock = space.allocate_lock() + self.owner = 0 + self.operr = OperationError(space.w_RuntimeError, + space.wrap("reentrant call")) + + def __enter__(self): + if not self.lock.acquire(False): + if self.owner == ll_thread.get_ident(): + raise self.operr + self.lock.acquire(True) + self.owner = ll_thread.get_ident() + + def __exit__(self,*args): + self.owner = 0 + self.lock.release() + + class BlockingIOError(Exception): pass @@ -130,10 +154,7 @@ self.buffer = ['\0'] * self.buffer_size - ## XXX cannot free a Lock? - ## if self.lock: - ## self.lock.free() - self.lock = space.allocate_lock() + self.lock = TryLock(space) try: self._raw_tell(space) diff --git a/pypy/module/_io/test/test_bufferedio.py b/pypy/module/_io/test/test_bufferedio.py --- a/pypy/module/_io/test/test_bufferedio.py +++ b/pypy/module/_io/test/test_bufferedio.py @@ -1,6 +1,9 @@ from pypy.conftest import gettestobjspace, option from pypy.interpreter.gateway import interp2app from pypy.tool.udir import udir +from pypy.module._io import interp_bufferedio +from pypy.interpreter.error import OperationError +import py.test class AppTestBufferedReader: spaceconfig = dict(usemodules=['_io']) @@ -217,7 +220,7 @@ class AppTestBufferedWriter: def setup_class(cls): - cls.space = gettestobjspace(usemodules=['_io']) + cls.space = gettestobjspace(usemodules=['_io', 'thread']) tmpfile = udir.join('tmpfile') cls.w_tmpfile = cls.space.wrap(str(tmpfile)) if option.runappdirect: @@ -425,6 +428,22 @@ bufio.flush() assert rawio.count == 3 + def test_reentrant_write(self): + import thread # Reentrant-safe is only enabled with threads + import _io, errno + class MockRawIO(_io._RawIOBase): + def writable(self): + return True + def write(self, data): + bufio.write("something else") + return len(data) + + rawio = MockRawIO() + bufio = _io.BufferedWriter(rawio) + bufio.write("test") + exc = raises(RuntimeError, bufio.flush) + assert "reentrant" in str(exc.value) # And not e.g. recursion limit. + class AppTestBufferedRWPair: def test_pair(self): import _io @@ -494,3 +513,15 @@ expected[j] = 2 expected[i] = 1 assert raw.getvalue() == str(expected) + + +class TestNonReentrantLock: + def test_trylock(self): + space = gettestobjspace(usemodules=['thread']) + lock = interp_bufferedio.TryLock(space) + with lock: + pass + with lock: + exc = py.test.raises(OperationError, "with lock: pass") + assert exc.value.match(space, space.w_RuntimeError) + From noreply at buildbot.pypy.org Mon Jan 23 22:49:39 2012 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 23 Jan 2012 22:49:39 +0100 (CET) Subject: [pypy-commit] pypy numpy-back-to-applevel: minor improvements Message-ID: <20120123214939.744A782E07@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: numpy-back-to-applevel Changeset: r51697:9c6ebf53fca0 Date: 2012-01-23 23:47 +0200 http://bitbucket.org/pypy/pypy/changeset/9c6ebf53fca0/ Log: minor improvements diff --git a/lib_pypy/numpypy/core/arrayprint.py b/lib_pypy/numpypy/core/arrayprint.py --- a/lib_pypy/numpypy/core/arrayprint.py +++ b/lib_pypy/numpypy/core/arrayprint.py @@ -551,7 +551,8 @@ def fillFormat(self, data): import numeric as _nc - errstate = _nc.seterr(all='ignore') + # XXX pypy unimplemented + #errstate = _nc.seterr(all='ignore') try: special = isnan(data) | isinf(data) | isna(data) special[isna(data)] = False @@ -570,7 +571,9 @@ or max_val/min_val > 1000.): self.exp_format = True finally: - _nc.seterr(**errstate) + pass + # XXX pypy unimplemented + #_nc.seterr(**errstate) if self.exp_format: self.large_exponent = 0 < min_val < 1e-99 or max_val >= 1e100 diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py --- a/pypy/module/micronumpy/interp_boxes.py +++ b/pypy/module/micronumpy/interp_boxes.py @@ -283,3 +283,4 @@ __new__ = interp2app(W_Float64Box.descr__new__.im_func), ) + diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -567,7 +567,11 @@ backstrides, shape, concrete)) def descr_ravel(self, space, w_order=None): - if not space.is_w(w_order, space.w_None): + if space.is_w(w_order, space.w_None): + order = 'C' + else: + order = space.str_w(w_order) + if order != 'C': raise OperationError(space.w_NotImplementedError, space.wrap( "order not implemented")) return self.descr_reshape(space, [space.wrap(-1)]) From noreply at buildbot.pypy.org Mon Jan 23 22:51:09 2012 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 23 Jan 2012 22:51:09 +0100 (CET) Subject: [pypy-commit] pypy numpy-back-to-applevel: a test commit Message-ID: <20120123215109.AC79F82E07@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: numpy-back-to-applevel Changeset: r51698:f35e55900c6e Date: 2012-01-23 23:50 +0200 http://bitbucket.org/pypy/pypy/changeset/f35e55900c6e/ Log: a test commit diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py --- a/pypy/module/micronumpy/interp_boxes.py +++ b/pypy/module/micronumpy/interp_boxes.py @@ -283,4 +283,3 @@ __new__ = interp2app(W_Float64Box.descr__new__.im_func), ) - From noreply at buildbot.pypy.org Mon Jan 23 22:59:17 2012 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 23 Jan 2012 22:59:17 +0100 (CET) Subject: [pypy-commit] pypy numpy-back-to-applevel: add another test commit Message-ID: <20120123215917.6A59182E07@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: numpy-back-to-applevel Changeset: r51699:27230fad5112 Date: 2012-01-23 23:58 +0200 http://bitbucket.org/pypy/pypy/changeset/27230fad5112/ Log: add another test commit diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py --- a/pypy/module/micronumpy/interp_boxes.py +++ b/pypy/module/micronumpy/interp_boxes.py @@ -283,3 +283,4 @@ __new__ = interp2app(W_Float64Box.descr__new__.im_func), ) + From noreply at buildbot.pypy.org Mon Jan 23 23:41:53 2012 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 23 Jan 2012 23:41:53 +0100 (CET) Subject: [pypy-commit] pypy default: This file belongs here for now Message-ID: <20120123224153.DC6BE82E07@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: Changeset: r51700:a56e47f860e4 Date: 2012-01-24 00:41 +0200 http://bitbucket.org/pypy/pypy/changeset/a56e47f860e4/ Log: This file belongs here for now diff --git a/pypy/module/micronumpy/tool/numready.py b/pypy/module/micronumpy/tool/numready.py new file mode 100644 --- /dev/null +++ b/pypy/module/micronumpy/tool/numready.py @@ -0,0 +1,226 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" +This should be run under PyPy. +""" + +import platform +import subprocess +import sys +import tempfile +import webbrowser +from collections import OrderedDict + +import jinja2 + + +MODULE_SEARCH_CODE = ''' +import types +import {modname} as numpy + +for name in dir(numpy): + if name.startswith("_"): + continue + obj = getattr(numpy, name) + kind = "{kinds[UNKNOWN]}" + if isinstance(obj, types.TypeType): + kind = "{kinds[TYPE]}" + print kind, ":", name +''' + +ATTR_SEARCH_CODE = ''' +import types +import {modname} as numpy + +obj = getattr(numpy, "{name}") +for name in dir(obj): + #if name.startswith("_"): + # continue + sub_obj = getattr(obj, name) + kind = "{kinds[UNKNOWN]}" + if isinstance(sub_obj, types.TypeType): + kind = "{kinds[TYPE]}" + print kind, ":", name +''' + +KINDS = { + "UNKNOWN": "U", + "TYPE": "T", +} + +PAGE_TEMPLATE = u""" +<!DOCTYPE html> +<html lang="en"> + <head> + <title>NumPyPy Status</title> + <meta http-equiv="content-type" content="text/html; charset=utf-8"> + <style type="text/css"> + body { + font-family: 'Consolas', 'Bitstream Vera Sans Mono', monospace; + } + h1 { + text-align: center; + } + h3 { + text-align: center; + } + table { + border: 8px solid #DFDECB; + margin: 30px auto; + font-size: 12px; + } + table th { + text-align: left; + } + table td { + padding: 4px 10px; + text-align: center; + } + .exists { + background-color: #337792; + color: white; + border: 1px solid #234F61; + } + </style> + </head> + <body> + <h1>NumPyPy Status</h1> + <h3>Overall: {{ msg }}</h3> + <table> + <thead> + <tr> + <th></th> + <th>PyPy</th> + <th></th> + <th>PyPy</th> + <th></th> + <th>PyPy</th> + <th></th> + <th>PyPy</th> + <th></th> + <th>PyPy</th> + </tr> + </thead> + <tbody> + {% for chunk in all_items %} + <tr> + {% for item in chunk %} + <th class='{{ item.cls }}'>{{ item.name }}</th> + <td class='{{ item.cls }}'>{{ item.symbol }}</td> + {% endfor %} + </tr> + {% endfor %} + </tbody> + </table> + </body> +</html> +""" + +class SearchableSet(object): + def __init__(self, items=()): + self._items = {} + for item in items: + self.add(item) + + def __iter__(self): + return iter(self._items) + + def __contains__(self, other): + return other in self._items + + def __getitem__(self, idx): + return self._items[idx] + + def add(self, item): + self._items[item] = item + + def __len__(self): + return len(self._items) + +class Item(object): + def __init__(self, name, kind, subitems=None): + self.name = name + self.kind = kind + self.subitems = subitems + + def __hash__(self): + return hash(self.name) + + def __eq__(self, other): + if isinstance(other, str): + return self.name == other + return self.name == other.name + + +class ItemStatus(object): + def __init__(self, name, pypy_exists): + self.name = name + self.cls = 'exists' if pypy_exists else '' + self.symbol = u"✔" if pypy_exists else u'✖' + + def __lt__(self, other): + return self.name < other.name + +def find_numpy_attrs(python, modname, name): + lines = subprocess.check_output( + [python, "-c", ATTR_SEARCH_CODE.format(modname=modname, kinds=KINDS, name=name)] + ).splitlines() + items = SearchableSet() + for line in lines: + kind, name = line.split(" : ", 1) + items.add(Item(name, kind)) + return items + +def find_numpy_items(python, modname="numpy"): + lines = subprocess.check_output( + [python, "-c", MODULE_SEARCH_CODE.format(modname=modname, kinds=KINDS)] + ).splitlines() + items = SearchableSet() + for line in lines: + kind, name = line.split(" : ", 1) + subitems = None + if kind == KINDS["TYPE"]: + if name in ['ndarray', 'dtype']: + subitems = find_numpy_attrs(python, modname, name) + items.add(Item(name, kind, subitems)) + return items + +def split(lst): + SPLIT = 5 + lgt = len(lst) // SPLIT + 1 + l = [[] for i in range(lgt)] + for i in range(lgt): + for k in range(SPLIT): + if k * lgt + i < len(lst): + l[i].append(lst[k * lgt + i]) + return l + +def main(argv): + cpy_items = find_numpy_items("/usr/bin/python") + pypy_items = find_numpy_items(argv[1], "numpypy") + all_items = [] + + msg = '%d/%d names, %d/%d ndarray attributes, %d/%d dtype attributes' % ( + len(pypy_items), len(cpy_items), len(pypy_items['ndarray'].subitems), + len(cpy_items['ndarray'].subitems), len(pypy_items['dtype'].subitems), + len(cpy_items['dtype'].subitems)) + for item in cpy_items: + pypy_exists = item in pypy_items + if item.subitems: + for sub in item.subitems: + all_items.append( + ItemStatus(item.name + "." + sub.name, pypy_exists=pypy_exists and pypy_items[item].subitems and sub in pypy_items[item].subitems) + ) + all_items.append(ItemStatus(item.name, pypy_exists=item in pypy_items)) + html = jinja2.Template(PAGE_TEMPLATE).render(all_items=split(sorted(all_items)), msg=msg) + if len(argv) > 2: + with open(argv[2], 'w') as f: + f.write(html.encode("utf-8")) + else: + with tempfile.NamedTemporaryFile(delete=False) as f: + f.write(html.encode("utf-8")) + print "Saved in: %s" % f.name + +if __name__ == '__main__': + main(sys.argv) From noreply at buildbot.pypy.org Mon Jan 23 23:42:08 2012 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 23 Jan 2012 23:42:08 +0100 (CET) Subject: [pypy-commit] buildbot default: remove numready from here Message-ID: <20120123224208.8090882E07@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: Changeset: r612:fdaf7c879572 Date: 2012-01-24 00:41 +0200 http://bitbucket.org/pypy/buildbot/changeset/fdaf7c879572/ Log: remove numready from here diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py --- a/bot2/pypybuildbot/builds.py +++ b/bot2/pypybuildbot/builds.py @@ -339,6 +339,9 @@ ) ) pypy_c_rel = "../build/pypy/translator/goal/pypy-c" + self.addStep(ShellCmd( + description="measure numpy compatibility", + if postfix: addopts = ['--postfix', postfix] else: diff --git a/numready.py b/numready.py deleted file mode 100644 --- a/numready.py +++ /dev/null @@ -1,226 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -""" -This should be run under PyPy. -""" - -import platform -import subprocess -import sys -import tempfile -import webbrowser -from collections import OrderedDict - -import jinja2 - - -MODULE_SEARCH_CODE = ''' -import types -import {modname} as numpy - -for name in dir(numpy): - if name.startswith("_"): - continue - obj = getattr(numpy, name) - kind = "{kinds[UNKNOWN]}" - if isinstance(obj, types.TypeType): - kind = "{kinds[TYPE]}" - print kind, ":", name -''' - -ATTR_SEARCH_CODE = ''' -import types -import {modname} as numpy - -obj = getattr(numpy, "{name}") -for name in dir(obj): - #if name.startswith("_"): - # continue - sub_obj = getattr(obj, name) - kind = "{kinds[UNKNOWN]}" - if isinstance(sub_obj, types.TypeType): - kind = "{kinds[TYPE]}" - print kind, ":", name -''' - -KINDS = { - "UNKNOWN": "U", - "TYPE": "T", -} - -PAGE_TEMPLATE = u""" -<!DOCTYPE html> -<html lang="en"> - <head> - <title>NumPyPy Status</title> - <meta http-equiv="content-type" content="text/html; charset=utf-8"> - <style type="text/css"> - body { - font-family: 'Consolas', 'Bitstream Vera Sans Mono', monospace; - } - h1 { - text-align: center; - } - h3 { - text-align: center; - } - table { - border: 8px solid #DFDECB; - margin: 30px auto; - font-size: 12px; - } - table th { - text-align: left; - } - table td { - padding: 4px 10px; - text-align: center; - } - .exists { - background-color: #337792; - color: white; - border: 1px solid #234F61; - } - </style> - </head> - <body> - <h1>NumPyPy Status</h1> - <h3>Overall: {{ msg }}</h3> - <table> - <thead> - <tr> - <th></th> - <th>PyPy</th> - <th></th> - <th>PyPy</th> - <th></th> - <th>PyPy</th> - <th></th> - <th>PyPy</th> - <th></th> - <th>PyPy</th> - </tr> - </thead> - <tbody> - {% for chunk in all_items %} - <tr> - {% for item in chunk %} - <th class='{{ item.cls }}'>{{ item.name }}</th> - <td class='{{ item.cls }}'>{{ item.symbol }}</td> - {% endfor %} - </tr> - {% endfor %} - </tbody> - </table> - </body> -</html> -""" - -class SearchableSet(object): - def __init__(self, items=()): - self._items = {} - for item in items: - self.add(item) - - def __iter__(self): - return iter(self._items) - - def __contains__(self, other): - return other in self._items - - def __getitem__(self, idx): - return self._items[idx] - - def add(self, item): - self._items[item] = item - - def __len__(self): - return len(self._items) - -class Item(object): - def __init__(self, name, kind, subitems=None): - self.name = name - self.kind = kind - self.subitems = subitems - - def __hash__(self): - return hash(self.name) - - def __eq__(self, other): - if isinstance(other, str): - return self.name == other - return self.name == other.name - - -class ItemStatus(object): - def __init__(self, name, pypy_exists): - self.name = name - self.cls = 'exists' if pypy_exists else '' - self.symbol = u"✔" if pypy_exists else u'✖' - - def __lt__(self, other): - return self.name < other.name - -def find_numpy_attrs(python, modname, name): - lines = subprocess.check_output( - [python, "-c", ATTR_SEARCH_CODE.format(modname=modname, kinds=KINDS, name=name)] - ).splitlines() - items = SearchableSet() - for line in lines: - kind, name = line.split(" : ", 1) - items.add(Item(name, kind)) - return items - -def find_numpy_items(python, modname="numpy"): - lines = subprocess.check_output( - [python, "-c", MODULE_SEARCH_CODE.format(modname=modname, kinds=KINDS)] - ).splitlines() - items = SearchableSet() - for line in lines: - kind, name = line.split(" : ", 1) - subitems = None - if kind == KINDS["TYPE"]: - if name in ['ndarray', 'dtype']: - subitems = find_numpy_attrs(python, modname, name) - items.add(Item(name, kind, subitems)) - return items - -def split(lst): - SPLIT = 5 - lgt = len(lst) // SPLIT + 1 - l = [[] for i in range(lgt)] - for i in range(lgt): - for k in range(SPLIT): - if k * lgt + i < len(lst): - l[i].append(lst[k * lgt + i]) - return l - -def main(argv): - cpy_items = find_numpy_items("/usr/bin/python") - pypy_items = find_numpy_items(argv[1], "numpypy") - all_items = [] - - msg = '%d/%d names, %d/%d ndarray attributes, %d/%d dtype attributes' % ( - len(pypy_items), len(cpy_items), len(pypy_items['ndarray'].subitems), - len(cpy_items['ndarray'].subitems), len(pypy_items['dtype'].subitems), - len(cpy_items['dtype'].subitems)) - for item in cpy_items: - pypy_exists = item in pypy_items - if item.subitems: - for sub in item.subitems: - all_items.append( - ItemStatus(item.name + "." + sub.name, pypy_exists=pypy_exists and pypy_items[item].subitems and sub in pypy_items[item].subitems) - ) - all_items.append(ItemStatus(item.name, pypy_exists=item in pypy_items)) - html = jinja2.Template(PAGE_TEMPLATE).render(all_items=split(sorted(all_items)), msg=msg) - if len(argv) > 2: - with open(argv[2], 'w') as f: - f.write(html.encode("utf-8")) - else: - with tempfile.NamedTemporaryFile(delete=False) as f: - f.write(html.encode("utf-8")) - print "Saved in: %s" % f.name - -if __name__ == '__main__': - main(sys.argv) From noreply at buildbot.pypy.org Mon Jan 23 23:43:36 2012 From: noreply at buildbot.pypy.org (stefanor) Date: Mon, 23 Jan 2012 23:43:36 +0100 (CET) Subject: [pypy-commit] pypy default: Get additional compiler and linker flags from the environment (rather than ignoring them on linux, and replacing the required flags on freebsd) Message-ID: <20120123224336.1B4D582E07@wyvern.cs.uni-duesseldorf.de> Author: Stefano Rivera <stefano at rivera.za.net> Branch: Changeset: r51701:173aa3e5cde0 Date: 2012-01-24 00:43 +0200 http://bitbucket.org/pypy/pypy/changeset/173aa3e5cde0/ Log: Get additional compiler and linker flags from the environment (rather than ignoring them on linux, and replacing the required flags on freebsd) diff --git a/pypy/translator/platform/freebsd.py b/pypy/translator/platform/freebsd.py --- a/pypy/translator/platform/freebsd.py +++ b/pypy/translator/platform/freebsd.py @@ -18,8 +18,9 @@ class Freebsd(posix.BasePosix): name = "freebsd" - link_flags = get_env_vector("LDFLAGS", '-pthread') - cflags = get_env_vector("CFLAGS", "-O3 -pthread -fomit-frame-pointer") + link_flags = ['-pthread'] + get_env_vector('LDFLAGS', '') + cflags = ['-O3', '-pthread', '-fomit-frame-pointer' + ] + get_env_vector('CFLAGS', '') standalone_only = [] shared_only = [] so_ext = 'so' diff --git a/pypy/translator/platform/linux.py b/pypy/translator/platform/linux.py --- a/pypy/translator/platform/linux.py +++ b/pypy/translator/platform/linux.py @@ -1,15 +1,20 @@ """Support for Linux.""" +import os import sys from pypy.translator.platform.posix import BasePosix class BaseLinux(BasePosix): name = "linux" - link_flags = ('-pthread',) + link_flags = tuple( + ['-pthread',] + + os.environ.get('LDFLAGS', '').split()) extra_libs = ('-lrt',) - cflags = ('-O3', '-pthread', '-fomit-frame-pointer', - '-Wall', '-Wno-unused') + cflags = tuple( + ['-O3', '-pthread', '-fomit-frame-pointer', + '-Wall', '-Wno-unused'] + + os.environ.get('CFLAGS', '').split()) standalone_only = () shared_only = ('-fPIC',) so_ext = 'so' From noreply at buildbot.pypy.org Mon Jan 23 23:55:03 2012 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 23 Jan 2012 23:55:03 +0100 (CET) Subject: [pypy-commit] buildbot default: this was not supposed to be commited Message-ID: <20120123225503.EFB6582E07@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: Changeset: r613:166c710d4406 Date: 2012-01-24 00:54 +0200 http://bitbucket.org/pypy/buildbot/changeset/166c710d4406/ Log: this was not supposed to be commited diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py --- a/bot2/pypybuildbot/builds.py +++ b/bot2/pypybuildbot/builds.py @@ -339,9 +339,6 @@ ) ) pypy_c_rel = "../build/pypy/translator/goal/pypy-c" - self.addStep(ShellCmd( - description="measure numpy compatibility", - if postfix: addopts = ['--postfix', postfix] else: From noreply at buildbot.pypy.org Tue Jan 24 00:01:46 2012 From: noreply at buildbot.pypy.org (amauryfa) Date: Tue, 24 Jan 2012 00:01:46 +0100 (CET) Subject: [pypy-commit] pypy default: a, b = space.unpackiterable(w_obj, 2) is not supposed to raise ValueError Message-ID: <20120123230146.1361882E07@wyvern.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc <amauryfa at gmail.com> Branch: Changeset: r51702:36fefb3d271e Date: 2012-01-23 23:05 +0100 http://bitbucket.org/pypy/pypy/changeset/36fefb3d271e/ Log: a, b = space.unpackiterable(w_obj, 2) is not supposed to raise ValueError diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py --- a/pypy/module/_socket/interp_socket.py +++ b/pypy/module/_socket/interp_socket.py @@ -67,9 +67,6 @@ self.connect(self.addr_from_object(space, w_addr)) except SocketError, e: raise converted_error(space, e) - except TypeError, e: - raise OperationError(space.w_TypeError, - space.wrap(str(e))) def connect_ex_w(self, space, w_addr): """connect_ex(address) -> errno diff --git a/pypy/rlib/rsocket.py b/pypy/rlib/rsocket.py --- a/pypy/rlib/rsocket.py +++ b/pypy/rlib/rsocket.py @@ -310,10 +310,7 @@ def from_object(space, w_address): # Parse an app-level object representing an AF_INET address - try: - w_host, w_port = space.unpackiterable(w_address, 2) - except ValueError: - raise TypeError("AF_INET address must be a tuple of length 2") + w_host, w_port = space.unpackiterable(w_address, 2) host = space.str_w(w_host) port = space.int_w(w_port) port = Address.make_ushort_port(space, port) @@ -544,10 +541,7 @@ space.wrap(self.get_groups())]) def from_object(space, w_address): - try: - w_pid, w_groups = space.unpackiterable(w_address, 2) - except ValueError: - raise TypeError("AF_NETLINK address must be a tuple of length 2") + w_pid, w_groups = space.unpackiterable(w_address, 2) return NETLINKAddress(space.uint_w(w_pid), space.uint_w(w_groups)) from_object = staticmethod(from_object) From noreply at buildbot.pypy.org Tue Jan 24 00:02:37 2012 From: noreply at buildbot.pypy.org (stefanor) Date: Tue, 24 Jan 2012 00:02:37 +0100 (CET) Subject: [pypy-commit] pypy default: Debian kFreeBSD uses glibc Message-ID: <20120123230237.623CE82E07@wyvern.cs.uni-duesseldorf.de> Author: Stefano Rivera <stefano at rivera.za.net> Branch: Changeset: r51703:26b2fb7939ee Date: 2012-01-24 01:01 +0200 http://bitbucket.org/pypy/pypy/changeset/26b2fb7939ee/ Log: Debian kFreeBSD uses glibc Check for sys.platform.startswith("freebsd") when the check is for FreeBSD's userland and 'freebsd' in sys.platform when the check is for a FreeBSD kernel. diff --git a/pypy/module/fcntl/test/test_fcntl.py b/pypy/module/fcntl/test/test_fcntl.py --- a/pypy/module/fcntl/test/test_fcntl.py +++ b/pypy/module/fcntl/test/test_fcntl.py @@ -42,13 +42,9 @@ else: start_len = "qq" - if sys.platform in ('netbsd1', 'netbsd2', 'netbsd3', - 'Darwin1.2', 'darwin', - 'freebsd2', 'freebsd3', 'freebsd4', 'freebsd5', - 'freebsd6', 'freebsd7', 'freebsd8', 'freebsd9', - 'bsdos2', 'bsdos3', 'bsdos4', - 'openbsd', 'openbsd2', 'openbsd3', 'openbsd4', - 'openbsd5'): + if any(substring in sys.platform.lower() + for substring in ('netbsd', 'darwin', 'freebsd', 'bsdos', + 'openbsd')): if struct.calcsize('l') == 8: off_t = 'l' pid_t = 'i' @@ -182,7 +178,8 @@ def test_large_flag(self): import sys - if sys.platform == "darwin" or sys.platform.startswith("openbsd"): + if any(plat in sys.platform + for plat in ('darwin', 'openbsd', 'freebsd')): skip("Mac OS doesn't have any large flag in fcntl.h") import fcntl, sys if sys.maxint == 2147483647: diff --git a/pypy/rlib/rdynload.py b/pypy/rlib/rdynload.py --- a/pypy/rlib/rdynload.py +++ b/pypy/rlib/rdynload.py @@ -14,7 +14,7 @@ _MINGW = platform.name == "mingw32" _WIN32 = _MSVC or _MINGW _MAC_OS = platform.name == "darwin" -_FREEBSD = platform.name == "freebsd" +_FREEBSD = sys.platform.startswith("freebsd") if _WIN32: from pypy.rlib import rwin32 diff --git a/pypy/rpython/lltypesystem/ll2ctypes.py b/pypy/rpython/lltypesystem/ll2ctypes.py --- a/pypy/rpython/lltypesystem/ll2ctypes.py +++ b/pypy/rpython/lltypesystem/ll2ctypes.py @@ -1390,7 +1390,8 @@ def _where_is_errno(): return standard_c_lib.__errno_location() - elif sys.platform in ('darwin', 'freebsd7', 'freebsd8', 'freebsd9'): + elif any(plat in sys.platform + for plat in ('darwin', 'freebsd7', 'freebsd8', 'freebsd9')): standard_c_lib.__error.restype = ctypes.POINTER(ctypes.c_int) def _where_is_errno(): return standard_c_lib.__error() diff --git a/pypy/rpython/memory/gc/env.py b/pypy/rpython/memory/gc/env.py --- a/pypy/rpython/memory/gc/env.py +++ b/pypy/rpython/memory/gc/env.py @@ -116,7 +116,7 @@ def get_total_memory(): return get_total_memory_darwin(get_darwin_sysctl_signed('hw.memsize')) -elif sys.platform.startswith('freebsd'): +elif 'freebsd' in sys.platform: def get_total_memory(): return get_total_memory_darwin(get_darwin_sysctl_signed('hw.usermem')) diff --git a/pypy/rpython/module/ll_time.py b/pypy/rpython/module/ll_time.py --- a/pypy/rpython/module/ll_time.py +++ b/pypy/rpython/module/ll_time.py @@ -41,7 +41,7 @@ RUSAGE = platform.Struct('struct rusage', [('ru_utime', TIMEVAL), ('ru_stime', TIMEVAL)]) -if "freebsd" in sys.platform: +if sys.platform.startswith('freebsd'): libraries = ['compat'] else: libraries = [] From noreply at buildbot.pypy.org Tue Jan 24 00:02:38 2012 From: noreply at buildbot.pypy.org (stefanor) Date: Tue, 24 Jan 2012 00:02:38 +0100 (CET) Subject: [pypy-commit] pypy default: Test hangs on kfreebsd Message-ID: <20120123230238.9519982E07@wyvern.cs.uni-duesseldorf.de> Author: Stefano Rivera <stefano at rivera.za.net> Branch: Changeset: r51704:67ae2044a17f Date: 2012-01-24 01:01 +0200 http://bitbucket.org/pypy/pypy/changeset/67ae2044a17f/ Log: Test hangs on kfreebsd diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py --- a/pypy/module/_ssl/test/test_ssl.py +++ b/pypy/module/_ssl/test/test_ssl.py @@ -64,8 +64,8 @@ def test_sslwrap(self): import _ssl, _socket, sys, gc - if sys.platform == 'darwin': - skip("hangs indefinitely on OSX (also on CPython)") + if sys.platform == 'darwin' or 'freebsd' in sys.platform: + skip("hangs indefinitely on OSX & FreeBSD (also on CPython)") s = _socket.socket() ss = _ssl.sslwrap(s, 0) exc = raises(_socket.error, ss.do_handshake) diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -371,6 +371,8 @@ if hasattr(__import__(os.name), "forkpty"): def test_forkpty(self): import sys + if 'freebsd' in sys.platform: + skip("hangs indifinitly on FreeBSD (also on CPython).") os = self.posix childpid, master_fd = os.forkpty() assert isinstance(childpid, int) From noreply at buildbot.pypy.org Tue Jan 24 00:06:36 2012 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 24 Jan 2012 00:06:36 +0100 (CET) Subject: [pypy-commit] buildbot default: try to upload numpy compatibility website when doing benchmarks Message-ID: <20120123230636.E8C5382E07@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: Changeset: r614:fbcf930f4155 Date: 2012-01-24 01:06 +0200 http://bitbucket.org/pypy/buildbot/changeset/fbcf930f4155/ Log: try to upload numpy compatibility website when doing benchmarks diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py --- a/bot2/pypybuildbot/builds.py +++ b/bot2/pypybuildbot/builds.py @@ -339,6 +339,16 @@ ) ) pypy_c_rel = "../build/pypy/translator/goal/pypy-c" + if host == 'tannit': + self.addStep(ShellCmd( + description="measure numpy compatibility", + command=['python', 'build/pypy/module/micronumpy/tool/numready.py', + pypy_c_rel, 'numpy-compat.html'], + workdir=".")) + resfile = os.path.expanduser("~/numpy_compat/%(got_revision)s.html") + self.addStep(transfer.FileUpload(slavesrc="numpy-compat.html", + masterdest=WithProperties(resfile), + workdir=".")) if postfix: addopts = ['--postfix', postfix] else: From noreply at buildbot.pypy.org Tue Jan 24 01:40:46 2012 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 24 Jan 2012 01:40:46 +0100 (CET) Subject: [pypy-commit] pypy default: bump rev no Message-ID: <20120124004046.4125A82E07@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: Changeset: r51705:fc6c9d14a686 Date: 2012-01-24 02:39 +0200 http://bitbucket.org/pypy/pypy/changeset/fc6c9d14a686/ Log: bump rev no diff --git a/pypy/doc/getting-started.rst b/pypy/doc/getting-started.rst --- a/pypy/doc/getting-started.rst +++ b/pypy/doc/getting-started.rst @@ -53,11 +53,11 @@ PyPy is ready to be executed as soon as you unpack the tarball or the zip file, with no need to install it in any specific location:: - $ tar xf pypy-1.6-linux.tar.bz2 + $ tar xf pypy-1.7-linux.tar.bz2 - $ ./pypy-1.6/bin/pypy + $ ./pypy-1.7/bin/pypy Python 2.7.1 (?, Apr 27 2011, 12:44:21) - [PyPy 1.6.0 with GCC 4.4.3] on linux2 + [PyPy 1.7.0 with GCC 4.4.3] on linux2 Type "help", "copyright", "credits" or "license" for more information. And now for something completely different: ``implementing LOGO in LOGO: "turtles all the way down"'' @@ -75,14 +75,14 @@ $ curl -O https://raw.github.com/pypa/pip/master/contrib/get-pip.py - $ ./pypy-1.6/bin/pypy distribute_setup.py + $ ./pypy-1.7/bin/pypy distribute_setup.py - $ ./pypy-1.6/bin/pypy get-pip.py + $ ./pypy-1.7/bin/pypy get-pip.py - $ ./pypy-1.6/bin/pip install pygments # for example + $ ./pypy-1.7/bin/pip install pygments # for example -3rd party libraries will be installed in ``pypy-1.6/site-packages``, and -the scripts in ``pypy-1.6/bin``. +3rd party libraries will be installed in ``pypy-1.7/site-packages``, and +the scripts in ``pypy-1.7/bin``. Installing using virtualenv --------------------------- From noreply at buildbot.pypy.org Tue Jan 24 01:40:47 2012 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 24 Jan 2012 01:40:47 +0100 (CET) Subject: [pypy-commit] pypy default: merge Message-ID: <20120124004047.847E982E07@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: Changeset: r51706:454ec575a42a Date: 2012-01-24 02:40 +0200 http://bitbucket.org/pypy/pypy/changeset/454ec575a42a/ Log: merge diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py --- a/pypy/module/_socket/interp_socket.py +++ b/pypy/module/_socket/interp_socket.py @@ -67,9 +67,6 @@ self.connect(self.addr_from_object(space, w_addr)) except SocketError, e: raise converted_error(space, e) - except TypeError, e: - raise OperationError(space.w_TypeError, - space.wrap(str(e))) def connect_ex_w(self, space, w_addr): """connect_ex(address) -> errno diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py --- a/pypy/module/_ssl/test/test_ssl.py +++ b/pypy/module/_ssl/test/test_ssl.py @@ -64,8 +64,8 @@ def test_sslwrap(self): import _ssl, _socket, sys, gc - if sys.platform == 'darwin': - skip("hangs indefinitely on OSX (also on CPython)") + if sys.platform == 'darwin' or 'freebsd' in sys.platform: + skip("hangs indefinitely on OSX & FreeBSD (also on CPython)") s = _socket.socket() ss = _ssl.sslwrap(s, 0) exc = raises(_socket.error, ss.do_handshake) diff --git a/pypy/module/fcntl/test/test_fcntl.py b/pypy/module/fcntl/test/test_fcntl.py --- a/pypy/module/fcntl/test/test_fcntl.py +++ b/pypy/module/fcntl/test/test_fcntl.py @@ -42,13 +42,9 @@ else: start_len = "qq" - if sys.platform in ('netbsd1', 'netbsd2', 'netbsd3', - 'Darwin1.2', 'darwin', - 'freebsd2', 'freebsd3', 'freebsd4', 'freebsd5', - 'freebsd6', 'freebsd7', 'freebsd8', 'freebsd9', - 'bsdos2', 'bsdos3', 'bsdos4', - 'openbsd', 'openbsd2', 'openbsd3', 'openbsd4', - 'openbsd5'): + if any(substring in sys.platform.lower() + for substring in ('netbsd', 'darwin', 'freebsd', 'bsdos', + 'openbsd')): if struct.calcsize('l') == 8: off_t = 'l' pid_t = 'i' @@ -182,7 +178,8 @@ def test_large_flag(self): import sys - if sys.platform == "darwin" or sys.platform.startswith("openbsd"): + if any(plat in sys.platform + for plat in ('darwin', 'openbsd', 'freebsd')): skip("Mac OS doesn't have any large flag in fcntl.h") import fcntl, sys if sys.maxint == 2147483647: diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -371,6 +371,8 @@ if hasattr(__import__(os.name), "forkpty"): def test_forkpty(self): import sys + if 'freebsd' in sys.platform: + skip("hangs indifinitly on FreeBSD (also on CPython).") os = self.posix childpid, master_fd = os.forkpty() assert isinstance(childpid, int) diff --git a/pypy/rlib/rdynload.py b/pypy/rlib/rdynload.py --- a/pypy/rlib/rdynload.py +++ b/pypy/rlib/rdynload.py @@ -14,7 +14,7 @@ _MINGW = platform.name == "mingw32" _WIN32 = _MSVC or _MINGW _MAC_OS = platform.name == "darwin" -_FREEBSD = platform.name == "freebsd" +_FREEBSD = sys.platform.startswith("freebsd") if _WIN32: from pypy.rlib import rwin32 diff --git a/pypy/rlib/rsocket.py b/pypy/rlib/rsocket.py --- a/pypy/rlib/rsocket.py +++ b/pypy/rlib/rsocket.py @@ -310,10 +310,7 @@ def from_object(space, w_address): # Parse an app-level object representing an AF_INET address - try: - w_host, w_port = space.unpackiterable(w_address, 2) - except ValueError: - raise TypeError("AF_INET address must be a tuple of length 2") + w_host, w_port = space.unpackiterable(w_address, 2) host = space.str_w(w_host) port = space.int_w(w_port) port = Address.make_ushort_port(space, port) @@ -544,10 +541,7 @@ space.wrap(self.get_groups())]) def from_object(space, w_address): - try: - w_pid, w_groups = space.unpackiterable(w_address, 2) - except ValueError: - raise TypeError("AF_NETLINK address must be a tuple of length 2") + w_pid, w_groups = space.unpackiterable(w_address, 2) return NETLINKAddress(space.uint_w(w_pid), space.uint_w(w_groups)) from_object = staticmethod(from_object) diff --git a/pypy/rpython/lltypesystem/ll2ctypes.py b/pypy/rpython/lltypesystem/ll2ctypes.py --- a/pypy/rpython/lltypesystem/ll2ctypes.py +++ b/pypy/rpython/lltypesystem/ll2ctypes.py @@ -1390,7 +1390,8 @@ def _where_is_errno(): return standard_c_lib.__errno_location() - elif sys.platform in ('darwin', 'freebsd7', 'freebsd8', 'freebsd9'): + elif any(plat in sys.platform + for plat in ('darwin', 'freebsd7', 'freebsd8', 'freebsd9')): standard_c_lib.__error.restype = ctypes.POINTER(ctypes.c_int) def _where_is_errno(): return standard_c_lib.__error() diff --git a/pypy/rpython/memory/gc/env.py b/pypy/rpython/memory/gc/env.py --- a/pypy/rpython/memory/gc/env.py +++ b/pypy/rpython/memory/gc/env.py @@ -116,7 +116,7 @@ def get_total_memory(): return get_total_memory_darwin(get_darwin_sysctl_signed('hw.memsize')) -elif sys.platform.startswith('freebsd'): +elif 'freebsd' in sys.platform: def get_total_memory(): return get_total_memory_darwin(get_darwin_sysctl_signed('hw.usermem')) diff --git a/pypy/rpython/module/ll_time.py b/pypy/rpython/module/ll_time.py --- a/pypy/rpython/module/ll_time.py +++ b/pypy/rpython/module/ll_time.py @@ -41,7 +41,7 @@ RUSAGE = platform.Struct('struct rusage', [('ru_utime', TIMEVAL), ('ru_stime', TIMEVAL)]) -if "freebsd" in sys.platform: +if sys.platform.startswith('freebsd'): libraries = ['compat'] else: libraries = [] diff --git a/pypy/translator/platform/freebsd.py b/pypy/translator/platform/freebsd.py --- a/pypy/translator/platform/freebsd.py +++ b/pypy/translator/platform/freebsd.py @@ -18,8 +18,9 @@ class Freebsd(posix.BasePosix): name = "freebsd" - link_flags = get_env_vector("LDFLAGS", '-pthread') - cflags = get_env_vector("CFLAGS", "-O3 -pthread -fomit-frame-pointer") + link_flags = ['-pthread'] + get_env_vector('LDFLAGS', '') + cflags = ['-O3', '-pthread', '-fomit-frame-pointer' + ] + get_env_vector('CFLAGS', '') standalone_only = [] shared_only = [] so_ext = 'so' diff --git a/pypy/translator/platform/linux.py b/pypy/translator/platform/linux.py --- a/pypy/translator/platform/linux.py +++ b/pypy/translator/platform/linux.py @@ -1,15 +1,20 @@ """Support for Linux.""" +import os import sys from pypy.translator.platform.posix import BasePosix class BaseLinux(BasePosix): name = "linux" - link_flags = ('-pthread',) + link_flags = tuple( + ['-pthread',] + + os.environ.get('LDFLAGS', '').split()) extra_libs = ('-lrt',) - cflags = ('-O3', '-pthread', '-fomit-frame-pointer', - '-Wall', '-Wno-unused') + cflags = tuple( + ['-O3', '-pthread', '-fomit-frame-pointer', + '-Wall', '-Wno-unused'] + + os.environ.get('CFLAGS', '').split()) standalone_only = () shared_only = ('-fPIC',) so_ext = 'so' From noreply at buildbot.pypy.org Tue Jan 24 09:33:15 2012 From: noreply at buildbot.pypy.org (amauryfa) Date: Tue, 24 Jan 2012 09:33:15 +0100 (CET) Subject: [pypy-commit] pypy default: issue1017: In non-blocking mode, read() should return None instead of raising EAGAIN. Message-ID: <20120124083315.922F282CFD@wyvern.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc <amauryfa at gmail.com> Branch: Changeset: r51707:9901f428b3b1 Date: 2012-01-24 09:31 +0100 http://bitbucket.org/pypy/pypy/changeset/9901f428b3b1/ Log: issue1017: In non-blocking mode, read() should return None instead of raising EAGAIN. diff --git a/pypy/module/_io/interp_fileio.py b/pypy/module/_io/interp_fileio.py --- a/pypy/module/_io/interp_fileio.py +++ b/pypy/module/_io/interp_fileio.py @@ -349,6 +349,8 @@ try: s = os.read(self.fd, size) except OSError, e: + if e.errno == errno.EAGAIN: + return space.w_None raise wrap_oserror(space, e, exception_name='w_IOError') @@ -362,6 +364,8 @@ try: buf = os.read(self.fd, length) except OSError, e: + if e.errno == errno.EAGAIN: + return space.w_None raise wrap_oserror(space, e, exception_name='w_IOError') rwbuffer.setslice(0, buf) diff --git a/pypy/module/_io/test/test_fileio.py b/pypy/module/_io/test/test_fileio.py --- a/pypy/module/_io/test/test_fileio.py +++ b/pypy/module/_io/test/test_fileio.py @@ -133,6 +133,19 @@ f.close() assert a == 'a\nbxxxxxxx' + def test_nonblocking_read(self): + import os, fcntl + r_fd, w_fd = os.pipe() + # set nonblocking + fcntl.fcntl(r_fd, fcntl.F_SETFL, os.O_NONBLOCK) + import _io + f = _io.FileIO(r_fd, 'r') + # Read from stream sould return None + assert f.read() is None + assert f.read(10) is None + a = bytearray('x' * 10) + assert f.readinto(a) is None + def test_repr(self): import _io f = _io.FileIO(self.tmpfile, 'r') From noreply at buildbot.pypy.org Tue Jan 24 10:36:46 2012 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 24 Jan 2012 10:36:46 +0100 (CET) Subject: [pypy-commit] buildbot default: oh crap python 2.7 required Message-ID: <20120124093646.EE78982CFD@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: Changeset: r615:af2f8bedb39e Date: 2012-01-24 11:36 +0200 http://bitbucket.org/pypy/buildbot/changeset/af2f8bedb39e/ Log: oh crap python 2.7 required diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py --- a/bot2/pypybuildbot/builds.py +++ b/bot2/pypybuildbot/builds.py @@ -342,7 +342,8 @@ if host == 'tannit': self.addStep(ShellCmd( description="measure numpy compatibility", - command=['python', 'build/pypy/module/micronumpy/tool/numready.py', + command=[pypy_c_rel, + 'build/pypy/module/micronumpy/tool/numready.py', pypy_c_rel, 'numpy-compat.html'], workdir=".")) resfile = os.path.expanduser("~/numpy_compat/%(got_revision)s.html") From noreply at buildbot.pypy.org Tue Jan 24 11:21:09 2012 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 24 Jan 2012 11:21:09 +0100 (CET) Subject: [pypy-commit] buildbot default: workdinr & relative positions Message-ID: <20120124102109.C834C82CFD@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: Changeset: r616:ecb37aa5b739 Date: 2012-01-24 12:20 +0200 http://bitbucket.org/pypy/buildbot/changeset/ecb37aa5b739/ Log: workdinr & relative positions diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py --- a/bot2/pypybuildbot/builds.py +++ b/bot2/pypybuildbot/builds.py @@ -338,8 +338,8 @@ locks=[lock.access('counting')], ) ) - pypy_c_rel = "../build/pypy/translator/goal/pypy-c" if host == 'tannit': + pypy_c_rel = 'build/pypy/translator/goal/pypy-c' self.addStep(ShellCmd( description="measure numpy compatibility", command=[pypy_c_rel, @@ -350,6 +350,7 @@ self.addStep(transfer.FileUpload(slavesrc="numpy-compat.html", masterdest=WithProperties(resfile), workdir=".")) + pypy_c_rel = "../build/pypy/translator/goal/pypy-c" if postfix: addopts = ['--postfix', postfix] else: From noreply at buildbot.pypy.org Tue Jan 24 12:10:11 2012 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 24 Jan 2012 12:10:11 +0100 (CET) Subject: [pypy-commit] benchmarks default: Cheat cheat cheat - add jinja2 to benchmarks (benchmakr coming) Message-ID: <20120124111011.EAFA782CFD@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: Changeset: r162:dac3a66f6c53 Date: 2012-01-24 13:08 +0200 http://bitbucket.org/pypy/benchmarks/changeset/dac3a66f6c53/ Log: Cheat cheat cheat - add jinja2 to benchmarks (benchmakr coming) diff too long, truncating to 10000 out of 22944 lines diff --git a/lib/jinja2/.gitignore b/lib/jinja2/.gitignore new file mode 100644 --- /dev/null +++ b/lib/jinja2/.gitignore @@ -0,0 +1,8 @@ +*.so +docs/_build +*.pyc +*.pyo +*.egg-info +build +dist +.DS_Store diff --git a/lib/jinja2/AUTHORS b/lib/jinja2/AUTHORS new file mode 100644 --- /dev/null +++ b/lib/jinja2/AUTHORS @@ -0,0 +1,31 @@ +Jinja is written and maintained by the Jinja Team and various +contributors: + +Lead Developer: + +- Armin Ronacher <armin.ronacher at active-4.com> + +Developers: + +- Christoph Hack +- Georg Brandl + +Contributors: + +- Bryan McLemore +- Mickaël Guérin <kael at crocobox.org> +- Cameron Knight +- Lawrence Journal-World. +- David Cramer + +Patches and suggestions: + +- Ronny Pfannschmidt +- Axel Böhm +- Alexey Melchakov +- Bryan McLemore +- Clovis Fabricio (nosklo) +- Cameron Knight +- Peter van Dijk (Habbie) +- Stefan Ebner +- Rene Leonhardt diff --git a/lib/jinja2/CHANGES b/lib/jinja2/CHANGES new file mode 100644 --- /dev/null +++ b/lib/jinja2/CHANGES @@ -0,0 +1,282 @@ +Jinja2 Changelog +================ + +Version 2.7 +----------- +(codename to be selected, release date to be announced) + +- Choice and prefix loaders now dispatch source and template lookup + separately in order to work in combination with module loaders as + advertised. +- Fixed filesizeformat. +- Added a non-silent option for babel extraction. +- Added `urlencode` filter that automatically quotes values for + URL safe usage with utf-8 as only supported encoding. If applications + want to change this encoding they can override the filter. + +Version 2.6 +----------- +(codename Convolution, released on July 24th 2011) + +- internal attributes now raise an internal attribute error now instead + of returning an undefined. This fixes problems when passing undefined + objects to Python semantics expecting APIs. +- traceback support now works properly for PyPy. (Tested with 1.4) +- implemented operator intercepting for sandboxed environments. This + allows application developers to disable builtin operators for better + security. (For instance limit the mathematical operators to actual + integers instead of longs) +- groupby filter now supports dotted notation for grouping by attributes + of attributes. +- scoped blocks now properly treat toplevel assignments and imports. + Previously an import suddenly "disappeared" in a scoped block. +- automatically detect newer Python interpreter versions before loading code + from bytecode caches to prevent segfaults on invalid opcodes. The segfault + in earlier Jinja2 versions here was not a Jinja2 bug but a limitation in + the underlying Python interpreter. If you notice Jinja2 segfaulting in + earlier versions after an upgrade of the Python interpreter you don't have + to upgrade, it's enough to flush the bytecode cache. This just no longer + makes this necessary, Jinja2 will automatically detect these cases now. +- the sum filter can now sum up values by attribute. This is a backwards + incompatible change. The argument to the filter previously was the + optional starting index which defaultes to zero. This now became the + second argument to the function because it's rarely used. +- like sum, sort now also makes it possible to order items by attribute. +- like sum and sort, join now also is able to join attributes of objects + as string. +- the internal eval context now has a reference to the environment. +- added a mapping test to see if an object is a dict or an object with + a similar interface. + +Version 2.5.5 +------------- +(re-release of 2.5.4 with built documentation removed for filesize. + Released on October 18th 2010) + +- built documentation is no longer part of release. + +Version 2.5.4 +------------- +(bugfix release, released on October 17th 2010) + +- Fixed extensions not loading properly with overlays. +- Work around a bug in cpython for the debugger that causes segfaults + on 64bit big-endian architectures. + +Version 2.5.3 +------------- +(bugfix release, released on October 17th 2010) + +- fixed an operator precedence error introduced in 2.5.2. Statements + like "-foo.bar" had their implicit parentheses applied around the + first part of the expression ("(-foo).bar") instead of the more + correct "-(foo.bar)". + +Version 2.5.2 +------------- +(bugfix release, released on August 18th 2010) + +- improved setup.py script to better work with assumptions people + might still have from it (``--with-speedups``). +- fixed a packaging error that excluded the new debug support. + +Version 2.5.1 +------------- +(bugfix release, released on August 17th 2010) + +- StopIteration exceptions raised by functions called from templates + are now intercepted and converted to undefineds. This solves a + lot of debugging grief. (StopIteration is used internally to + abort template execution) +- improved performance of macro calls slightly. +- babel extraction can now properly extract newstyle gettext calls. +- using the variable `num` in newstyle gettext for something else + than the pluralize count will no longer raise a :exc:`KeyError`. +- removed builtin markup class and switched to markupsafe. For backwards + compatibility the pure Python implementation still exists but is + pulled from markupsafe by the Jinja2 developers. The debug support + went into a separate feature called "debugsupport" and is disabled + by default because it is only relevant for Python 2.4 +- fixed an issue with unary operators having the wrong precendence. + +Version 2.5 +----------- +(codename Incoherence, relased on May 29th 2010) + +- improved the sort filter (should have worked like this for a + long time) by adding support for case insensitive searches. +- fixed a bug for getattribute constant folding. +- support for newstyle gettext translations which result in a + nicer in-template user interface and more consistent + catalogs. (:ref:`newstyle-gettext`) +- it's now possible to register extensions after an environment + was created. + +Version 2.4.1 +------------- +(bugfix release, released on April 20th 2010) + +- fixed an error reporting bug for undefineds. + +Version 2.4 +----------- +(codename Correlation, released on April 13th 2010) + +- the environment template loading functions now transparently + pass through a template object if it was passed to it. This + makes it possible to import or extend from a template object + that was passed to the template. +- added a :class:`ModuleLoader` that can load templates from + precompiled sources. The environment now features a method + to compile the templates from a configured loader into a zip + file or folder. +- the _speedups C extension now supports Python 3. +- added support for autoescaping toggling sections and support + for evaluation contexts (:ref:`eval-context`). +- extensions have a priority now. + +Version 2.3.1 +------------- +(bugfix release, released on February 19th 2010) + +- fixed an error reporting bug on all python versions +- fixed an error reporting bug on Python 2.4 + +Version 2.3 +----------- +(3000 Pythons, released on February 10th 2010) + +- fixes issue with code generator that causes unbound variables + to be generated if set was used in if-blocks and other small + identifier problems. +- include tags are now able to select between multiple templates + and take the first that exists, if a list of templates is + given. +- fixed a problem with having call blocks in outer scopes that + have an argument that is also used as local variable in an + inner frame (#360). +- greatly improved error message reporting (#339) +- implicit tuple expressions can no longer be totally empty. + This change makes ``{% if %}...{% endif %}`` a syntax error + now. (#364) +- added support for translator comments if extracted via babel. +- added with-statement extension. +- experimental Python 3 support. + +Version 2.2.1 +------------- +(bugfix release, released on September 14th 2009) + +- fixes some smaller problems for Jinja2 on Jython. + +Version 2.2 +----------- +(codename Kong, released on September 13th 2009) + +- Include statements can now be marked with ``ignore missing`` to skip + non existing templates. +- Priority of `not` raised. It's now possible to write `not foo in bar` + as an alias to `foo not in bar` like in python. Previously the grammar + required parentheses (`not (foo in bar)`) which was odd. +- Fixed a bug that caused syntax errors when defining macros or using the + `{% call %}` tag inside loops. +- Fixed a bug in the parser that made ``{{ foo[1, 2] }}`` impossible. +- Made it possible to refer to names from outer scopes in included templates + that were unused in the callers frame (#327) +- Fixed a bug that caused internal errors if names where used as iteration + variable and regular variable *after* the loop if that variable was unused + *before* the loop. (#331) +- Added support for optional `scoped` modifier to blocks. +- Added support for line-comments. +- Added the `meta` module. +- Renamed (undocumented) attribute "overlay" to "overlayed" on the + environment because it was clashing with a method of the same name. +- speedup extension is now disabled by default. + +Version 2.1.1 +------------- +(Bugfix release) + +- Fixed a translation error caused by looping over empty recursive loops. + +Version 2.1 +----------- +(codename Yasuzō, released on November 23rd 2008) + +- fixed a bug with nested loops and the special loop variable. Before the + change an inner loop overwrote the loop variable from the outer one after + iteration. + +- fixed a bug with the i18n extension that caused the explicit pluralization + block to look up the wrong variable. + +- fixed a limitation in the lexer that made ``{{ foo.0.0 }}`` impossible. + +- index based subscribing of variables with a constant value returns an + undefined object now instead of raising an index error. This was a bug + caused by eager optimizing. + +- the i18n extension looks up `foo.ugettext` now followed by `foo.gettext` + if an translations object is installed. This makes dealing with custom + translations classes easier. + +- fixed a confusing behavior with conditional extending. loops were partially + executed under some conditions even though they were not part of a visible + area. + +- added `sort` filter that works like `dictsort` but for arbitrary sequences. + +- fixed a bug with empty statements in macros. + +- implemented a bytecode cache system. (:ref:`bytecode-cache`) + +- the template context is now weakref-able + +- inclusions and imports "with context" forward all variables now, not only + the initial context. + +- added a cycle helper called `cycler`. + +- added a joining helper called `joiner`. + +- added a `compile_expression` method to the environment that allows compiling + of Jinja expressions into callable Python objects. + +- fixed an escaping bug in urlize + +Version 2.0 +----------- +(codename jinjavitus, released on July 17th 2008) + +- the subscribing of objects (looking up attributes and items) changed from + slightly. It's now possible to give attributes or items a higher priority + by either using dot-notation lookup or the bracket syntax. This also + changed the AST slightly. `Subscript` is gone and was replaced with + :class:`~jinja2.nodes.Getitem` and :class:`~jinja2.nodes.Getattr`. + + For more information see :ref:`the implementation details <notes-on-subscriptions>`. + +- added support for preprocessing and token stream filtering for extensions. + This would allow extensions to allow simplified gettext calls in template + data and something similar. + +- added :meth:`jinja2.environment.TemplateStream.dump`. + +- added missing support for implicit string literal concatenation. + ``{{ "foo" "bar" }}`` is equivalent to ``{{ "foobar" }}`` + +- `else` is optional for conditional expressions. If not given it evaluates + to `false`. + +- improved error reporting for undefined values by providing a position. + +- `filesizeformat` filter uses decimal prefixes now per default and can be + set to binary mode with the second parameter. + +- fixed bug in finalizer + +Version 2.0rc1 +-------------- +(no codename, released on June 9th 2008) + +- first release of Jinja2 diff --git a/lib/jinja2/LICENSE b/lib/jinja2/LICENSE new file mode 100644 --- /dev/null +++ b/lib/jinja2/LICENSE @@ -0,0 +1,31 @@ +Copyright (c) 2009 by the Jinja Team, see AUTHORS for more details. + +Some rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + * The names of the contributors may not be used to endorse or + promote products derived from this software without specific + prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/lib/jinja2/MANIFEST.in b/lib/jinja2/MANIFEST.in new file mode 100644 --- /dev/null +++ b/lib/jinja2/MANIFEST.in @@ -0,0 +1,12 @@ +include MANIFEST.in Makefile CHANGES LICENSE AUTHORS jinja2/_debugsupport.c +recursive-include docs * +recursive-include custom_fixers * +recursive-include ext * +recursive-include artwork * +recursive-include examples * +recursive-include jinja2/testsuite/res * +recursive-exclude docs/_build * +recursive-exclude jinja2 *.pyc +recursive-exclude docs *.pyc +recursive-exclude jinja2 *.pyo +recursive-exclude docs *.pyo diff --git a/lib/jinja2/Makefile b/lib/jinja2/Makefile new file mode 100644 --- /dev/null +++ b/lib/jinja2/Makefile @@ -0,0 +1,15 @@ +test: + python setup.py test + +release: + python scripts/make-release.py + +upload-docs: + $(MAKE) -C docs html dirhtml latex + $(MAKE) -C docs/_build/latex all-pdf + cd docs/_build/; mv html jinja-docs; zip -r jinja-docs.zip jinja-docs; mv jinja-docs html + scp -r docs/_build/dirhtml/* pocoo.org:/var/www/jinja.pocoo.org/docs/ + scp -r docs/_build/latex/Jinja2.pdf pocoo.org:/var/www/jinja.pocoo.org/docs/jinja-docs.pdf + scp -r docs/_build/jinja-docs.zip pocoo.org:/var/www/jinja.pocoo.org/docs/ + +.PHONY: test diff --git a/lib/jinja2/README b/lib/jinja2/README new file mode 100644 --- /dev/null +++ b/lib/jinja2/README @@ -0,0 +1,1 @@ +Basic Support for Jinja2 in Textmate diff --git a/lib/jinja2/README.rst b/lib/jinja2/README.rst new file mode 100644 --- /dev/null +++ b/lib/jinja2/README.rst @@ -0,0 +1,38 @@ +Jinja2 +~~~~~~ + +Jinja2 is a template engine written in pure Python. It provides a +`Django`_ inspired non-XML syntax but supports inline expressions and +an optional `sandboxed`_ environment. + +Nutshell +-------- + +Here a small example of a Jinja template:: + + {% extends 'base.html' %} + {% block title %}Memberlist{% endblock %} + {% block content %} + <ul> + {% for user in users %} + <li><a href="{{ user.url }}">{{ user.username }}</a></li> + {% endfor %} + </ul> + {% endblock %} + +Philosophy +---------- + +Application logic is for the controller but don't try to make the life +for the template designer too hard by giving him too few functionality. + +For more informations visit the new `Jinja2 webpage`_ and `documentation`_. + +The `Jinja2 tip`_ is installable via `easy_install` with ``easy_install +Jinja2==dev``. + +.. _sandboxed: http://en.wikipedia.org/wiki/Sandbox_(computer_security) +.. _Django: http://www.djangoproject.com/ +.. _Jinja2 webpage: http://jinja.pocoo.org/ +.. _documentation: http://jinja.pocoo.org/2/documentation/ +.. _Jinja2 tip: http://dev.pocoo.org/hg/jinja2-main/archive/tip.tar.gz#egg=Jinja2-dev diff --git a/lib/jinja2/artwork/jinjalogo.svg b/lib/jinja2/artwork/jinjalogo.svg new file mode 100644 --- /dev/null +++ b/lib/jinja2/artwork/jinjalogo.svg @@ -0,0 +1,132 @@ +<?xml version="1.0" encoding="UTF-8" standalone="no"?> + +<svg + xmlns:dc="http://purl.org/dc/elements/1.1/" + xmlns:cc="http://web.resource.org/cc/" + xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" + xmlns:svg="http://www.w3.org/2000/svg" + xmlns="http://www.w3.org/2000/svg" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd" + xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape" + width="300" + height="120" + id="svg2" + sodipodi:version="0.32" + inkscape:version="0.45.1" + version="1.0" + sodipodi:docbase="/Users/mitsuhiko/Development/jinja2/artwork" + sodipodi:docname="jinjalogo.svg" + inkscape:export-filename="/Users/mitsuhiko/Development/jinja2/docs/_static/jinjabanner.png" + inkscape:export-xdpi="60" + inkscape:export-ydpi="60" + inkscape:output_extension="org.inkscape.output.svg.inkscape"> + <defs + id="defs4"> + <linearGradient + id="linearGradient6558"> + <stop + style="stop-color:#575757;stop-opacity:1;" + offset="0" + id="stop6560" /> + <stop + style="stop-color:#2f2f2f;stop-opacity:1;" + offset="1" + id="stop6562" /> + </linearGradient> + <radialGradient + inkscape:collect="always" + xlink:href="#linearGradient6558" + id="radialGradient6564" + cx="61.297766" + cy="60.910986" + fx="61.297766" + fy="60.910986" + r="44.688254" + gradientTransform="matrix(1,0,0,0.945104,0,3.343747)" + gradientUnits="userSpaceOnUse" /> + <radialGradient + inkscape:collect="always" + xlink:href="#linearGradient6558" + id="radialGradient6580" + gradientUnits="userSpaceOnUse" + gradientTransform="matrix(1,0,0,0.945104,0.355158,3.334402)" + cx="61.297766" + cy="60.910986" + fx="61.297766" + fy="60.910986" + r="44.688254" /> + <linearGradient + inkscape:collect="always" + xlink:href="#linearGradient6558" + id="linearGradient4173" + x1="255.15521" + y1="32.347946" + x2="279.8912" + y2="32.347946" + gradientUnits="userSpaceOnUse" + gradientTransform="matrix(0.8073249,0,0,0.8073249,57.960878,7.4036303)" /> + <linearGradient + inkscape:collect="always" + xlink:href="#linearGradient6558" + id="linearGradient5145" + gradientUnits="userSpaceOnUse" + gradientTransform="matrix(0.7902775,0,0,0.82474,60.019977,8.0684132)" + x1="255.15521" + y1="32.347946" + x2="279.8912" + y2="32.347946" /> + </defs> + <sodipodi:namedview + id="base" + pagecolor="#ffffff" + bordercolor="#666666" + borderopacity="1.0" + gridtolerance="10000" + guidetolerance="10" + objecttolerance="10" + inkscape:pageopacity="0.0" + inkscape:pageshadow="2" + inkscape:zoom="2.8" + inkscape:cx="137.4752" + inkscape:cy="57.574575" + inkscape:document-units="px" + inkscape:current-layer="layer1" + width="300px" + height="120px" + showguides="true" + inkscape:guide-bbox="true" + inkscape:window-width="1396" + inkscape:window-height="900" + inkscape:window-x="0" + inkscape:window-y="22" /> + <metadata + id="metadata7"> + <rdf:RDF> + <cc:Work + rdf:about=""> + <dc:format>image/svg+xml</dc:format> + <dc:type + rdf:resource="http://purl.org/dc/dcmitype/StillImage" /> + </cc:Work> + </rdf:RDF> + </metadata> + <g + inkscape:label="Layer 1" + inkscape:groupmode="layer" + id="layer1"> + <path + style="font-size:12px;font-style:normal;font-weight:normal;fill:#f4f4f4;fill-opacity:1;stroke:#e7e7e7;stroke-width:0.8;stroke-linecap:butt;stroke-linejoin:round;stroke-opacity:1;font-family:Bitstream Vera Sans;stroke-miterlimit:4;stroke-dasharray:none" + d="M 165.36463,80.874808 L 165.36463,80.874808 L 153.32556,80.874808 L 153.32556,81.8344 L 147.64994,81.8344 L 147.64994,36.035583 L 165.36463,36.035583 L 165.36463,20.333129 C 170.58154,21.031083 173.07533,22.077914 172.84609,23.473621 C 172.78871,24.055258 172.21545,24.549594 171.12624,24.956624 L 171.12624,36.035583 L 189.09895,36.035583 L 189.09895,82.532286 L 183.33733,82.532286 L 183.33733,80.874808 L 171.12624,80.874808 L 171.12624,102.94548 L 165.36463,102.94548 L 165.36463,80.874808 M 153.32556,55.489173 L 153.32556,55.489173 L 165.36463,55.489173 L 165.36463,41.793146 L 153.32556,41.793146 L 153.32556,55.489173 M 171.12624,55.489173 L 171.12624,55.489173 L 183.33733,55.489173 L 183.33733,41.793146 L 171.12624,41.793146 L 171.12624,55.489173 M 183.33733,61.333977 L 183.33733,61.333977 L 171.12624,61.333977 L 171.12624,75.030006 L 183.33733,75.030006 L 183.33733,61.333977 M 165.36463,61.333977 L 165.36463,61.333977 L 153.32556,61.333977 L 153.32556,75.030006 L 165.36463,75.030006 L 165.36463,61.333977 M 132.85897,59.414792 C 137.33069,63.136883 140.99969,67.934848 143.86618,73.808701 L 139.13654,77.385372 C 137.24467,72.965445 134.6362,69.12707 131.31114,65.87024 L 131.31114,102.94548 L 125.63554,102.94548 L 125.63554,68.57455 C 122.31042,71.947693 118.52671,74.913707 114.28436,77.47261 L 109.64069,73.372526 C 121.50782,67.091566 130.62312,55.489212 136.98668,38.565417 L 116.26221,38.565417 L 116.26221,32.720615 L 125.80754,32.720615 L 125.80754,20.333129 C 130.85245,21.031083 133.31761,22.048838 133.20299,23.386383 C 133.14561,24.026183 132.57235,24.549594 131.48307,24.956624 L 131.48307,32.720615 L 140.77043,32.720615 L 143.60824,36.733469 C 140.68444,45.51526 137.10137,53.075692 132.85897,59.414792 M 254.11016,49.469901 L 254.11016,49.469901 L 254.11016,20.333129 C 259.21243,21.031083 261.67755,22.048838 261.50562,23.386383 C 261.44823,23.909869 261.04699,24.346044 260.30172,24.694917 C 260.30164,24.694986 260.30164,24.694986 260.30172,24.694917 L 260.30172,24.694917 L 259.78578,24.956624 L 259.78578,49.469901 L 277.15652,49.469901 L 277.15652,55.227471 L 259.78578,55.227471 L 259.78578,93.785712 L 281.45616,93.785712 L 281.45616,99.63051 L 232.35378,99.63051 L 232.35378,93.785712 L 254.11016,93.785712 L 254.11016,55.227471 L 236.22346,55.227471 L 236.22346,49.469901 L 254.11016,49.469901 M 225.5603,59.327554 C 231.12111,63.107798 235.62145,67.876693 239.06127,73.634235 L 234.76157,77.647079 C 231.60845,72.180322 227.82475,67.934848 223.41044,64.910648 L 223.41044,102.94548 L 217.73484,102.94548 L 217.73484,67.44049 C 212.91919,71.627831 207.70222,75.030021 202.084,77.647079 L 197.87027,73.198053 C 212.66118,66.917101 224.01239,55.372897 231.92377,38.565417 L 205.35172,38.565417 L 205.35172,32.720615 L 217.99283,32.720615 L 217.99283,20.333129 C 223.03774,21.031083 225.50291,22.048838 225.38829,23.386383 C 225.33089,24.026183 224.75765,24.549594 223.66837,24.956624 L 223.66837,32.720615 L 236.22346,32.720615 L 238.80326,36.733469 C 235.13421,45.51526 230.71987,53.046611 225.5603,59.327554" + id="text4761" /> + <path + style="font-size:44.09793472px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:start;line-height:125%;writing-mode:lr-tb;text-anchor:start;fill:#b41717;fill-opacity:1;stroke:#7f2828;stroke-width:1;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;font-family:Candara;stroke-miterlimit:4;stroke-dasharray:none" + d="M 149.14708,37.774469 C 148.97807,41.117899 148.84526,44.824225 148.74871,48.893456 C 148.67626,52.962754 148.3818,70.641328 148.38184,75.524422 C 148.3818,79.065795 148.05588,81.991266 147.40406,84.300835 C 146.75219,86.610422 145.72612,88.557071 144.32585,90.140779 C 142.94969,91.724494 141.17522,92.901283 139.00239,93.671139 C 136.82953,94.440996 134.22211,94.825935 131.18014,94.825935 C 128.83828,94.825935 126.73787,94.59498 124.87889,94.133049 L 125.4221,89.31593 C 127.13623,90.0418 128.92278,90.404734 130.78177,90.404733 C 132.85805,90.404734 134.66875,90.140782 136.2139,89.612876 C 137.78315,89.062981 139.02651,88.216133 139.94396,87.072335 C 140.8855,85.928548 141.54942,84.520804 141.93572,82.8491 C 142.34613,81.177412 142.55134,78.988811 142.55136,76.283285 C 142.55134,66.297119 142.62852,44.659257 142.26641,37.774469 L 149.14708,37.774469 M 166.38498,80.732697 L 159.83024,80.732697 C 160.16821,76.333498 160.33723,71.307412 160.33723,65.654424 C 160.33723,59.2976 159.91471,53.963567 159.06973,49.652319 L 166.31257,48.761483 C 166.02284,53.358679 165.87799,58.98965 165.87799,65.654424 C 165.87799,70.933479 166.04699,75.959565 166.38498,80.732697 M 167.90601,39.490159 C 167.90598,40.611994 167.5076,41.590815 166.7109,42.42662 C 165.91418,43.240515 164.79155,43.647442 163.343,43.647399 C 162.11172,43.647442 161.146,43.295504 160.44588,42.591595 C 159.76988,41.865769 159.43188,40.996927 159.43188,39.98507 C 159.43188,38.885304 159.84231,37.928485 160.66315,37.114591 C 161.48399,36.30078 162.61869,35.893853 164.06727,35.893811 C 165.25023,35.893853 166.17975,36.256783 166.85575,36.982609 C 167.55588,37.686526 167.90598,38.522373 167.90601,39.490159 M 206.72748,80.732697 L 200.13651,80.732697 C 200.66763,74.947749 200.93319,68.634899 200.9332,61.794122 C 200.93319,58.406756 200.1727,56.097177 198.65174,54.865371 C 197.15487,53.61163 195.00619,52.984747 192.20564,52.984714 C 188.77731,52.984747 185.61465,54.117535 182.71753,56.383099 C 182.71753,63.883761 182.76583,72.000287 182.86238,80.732697 L 176.27142,80.732697 C 176.68182,73.254058 176.88707,67.843042 176.88707,64.499632 C 176.88707,59.352589 176.3559,54.359493 175.29363,49.520339 L 181.66734,48.695493 L 182.35539,52.720761 L 182.64511,52.720761 C 186.21823,49.773323 190.04483,48.299592 194.12499,48.299567 C 198.13265,48.299592 201.23499,49.113454 203.43201,50.741118 C 205.62895,52.346863 206.72747,55.217334 206.72748,59.352563 C 206.72747,59.770507 206.70331,60.595362 206.65507,61.827118 C 206.60675,63.058915 206.5826,63.883761 206.58262,64.30167 C 206.5826,67.975018 206.63088,73.452022 206.72748,80.732697 M 222.69791,48.695493 C 222.28747,55.514282 222.08225,62.355041 222.08225,69.21778 C 222.08225,71.043461 222.14262,73.463019 222.26332,76.476468 C 222.40822,79.467925 222.4806,81.502559 222.48063,82.580363 C 222.4806,89.685068 219.51105,93.996287 213.57195,95.514024 L 211.76124,93.006484 C 213.90995,91.356766 215.2378,89.597085 215.74478,87.727431 C 216.49321,85.043912 216.86743,79.324953 216.86743,70.570535 C 216.86743,61.178248 216.3846,54.16153 215.41887,49.520339 L 222.69791,48.695493 M 224.2551,39.490159 C 224.2551,40.611994 223.85673,41.590815 223.06006,42.42662 C 222.26332,43.240515 221.14069,43.647442 219.69213,43.647399 C 218.46084,43.647442 217.49515,43.295504 216.795,42.591595 C 216.119,41.865769 215.781,40.996927 215.781,39.98507 C 215.781,38.885304 216.19144,37.928485 217.01231,37.114591 C 217.83316,36.30078 218.96785,35.893853 220.4164,35.893811 C 221.5994,35.893853 222.52889,36.256783 223.20492,36.982609 C 223.90503,37.686526 224.2551,38.522373 224.2551,39.490159 M 259.60008,80.732697 L 253.91446,80.930661 C 253.62473,79.852857 253.47987,78.830045 253.4799,77.862216 L 253.11774,77.862216 C 250.14817,80.325772 246.10427,81.557546 240.98606,81.557547 C 238.20962,81.557546 235.8195,80.820682 233.81563,79.346948 C 231.81178,77.851221 230.80988,75.728607 230.80988,72.979099 C 230.80988,69.591724 232.37914,66.875216 235.51769,64.829574 C 238.65625,62.761967 244.48667,61.67316 253.00913,61.563165 C 253.08155,61.035275 253.11772,60.430386 253.11774,59.748497 C 253.11772,57.043003 252.32104,55.239336 250.72765,54.337474 C 249.15832,53.435661 246.76819,52.984747 243.55721,52.984714 C 239.76681,52.984747 236.03678,53.413668 232.3671,54.271484 L 232.9827,49.718301 C 236.60411,48.77251 240.76873,48.299592 245.47658,48.299567 C 249.77395,48.299592 253.09359,49.113454 255.43545,50.741118 C 257.77728,52.346863 258.94819,55.096363 258.94824,58.989625 C 258.94819,60.023469 258.88785,61.904117 258.76715,64.631608 C 258.67054,67.337133 258.62228,69.140806 258.6223,70.042632 C 258.62228,74.045913 258.94819,77.609265 259.60008,80.732697 M 253.19019,74.331856 C 253.06945,70.988469 253.00909,67.986016 253.00913,65.324484 C 248.47027,65.324498 245.01786,65.632443 242.65187,66.248318 C 238.69248,67.348131 236.71278,69.448748 236.71278,72.550177 C 236.71278,75.541643 239.03044,77.037371 243.66588,77.037366 C 247.64942,77.037371 250.82416,76.135534 253.19019,74.331856" + id="text3736" + sodipodi:nodetypes="ccsscssccscccsccccsccsccsscsssccccscscccsccccscsssccscscccscccsscsssccccccscsccscsccscscsccccssc" /> + <path + style="fill:url(#radialGradient6564);fill-opacity:1.0;fill-rule:evenodd;stroke:#323232;stroke-width:1;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;stroke-miterlimit:4;stroke-dasharray:none" + d="M 105.45673,18.675923 C 105.45673,18.675923 88.211949,26.918461 74.172834,28.737898 C 60.133727,30.557333 33.360434,32.377571 28.045622,31.093256 C 22.730818,29.808941 18.915645,28.309196 18.915645,28.309196 L 20.021441,32.056583 L 16.609513,35.052471 L 17.2144,36.121726 L 18.61792,36.22764 L 22.92773,36.762252 L 23.532621,38.688909 L 25.937975,38.905784 L 27.143021,42.970927 C 27.143021,42.970927 32.254764,43.399628 33.758953,43.399628 C 35.263142,43.399628 38.271966,43.187802 38.271966,43.187802 L 38.371202,44.791657 L 39.477002,45.003495 L 39.477002,46.824227 L 37.066917,48.967759 L 37.671807,49.073671 L 37.671807,49.820127 C 37.671807,49.820127 32.255457,50.252157 30.049301,49.93109 C 27.843157,49.610006 27.440747,49.608286 27.440747,49.608286 L 27.242258,49.820127 L 27.143021,50.783455 L 27.643946,50.783455 L 27.84242,54.959544 L 38.976091,54.530844 L 38.172728,68.980747 L 38.073481,70.796442 L 28.645781,70.261816 L 28.546544,66.408513 L 30.649462,66.408513 L 30.852673,64.910557 L 32.757107,64.481857 L 33.059555,64.058192 L 25.937975,62.343374 L 20.522364,63.947229 L 21.42496,64.698732 L 22.327572,64.698732 L 22.426809,65.984848 L 24.331254,66.09076 L 24.331254,69.838147 L 22.228335,70.372777 L 22.630009,71.225146 L 23.130934,71.547931 L 23.130934,74.437917 L 24.435218,74.437917 L 24.435218,87.813529 L 22.327572,88.13632 L 22.630009,91.989617 L 23.929569,92.206492 L 23.731093,100.98236 L 29.449141,101.08826 L 28.244105,92.418334 L 36.868446,92.206492 L 36.268285,96.912181 L 35.464925,100.23086 L 44.188501,100.33677 L 44.287739,91.777793 L 50.303506,91.243181 L 50.005786,96.700351 L 49.802585,99.90807 L 54.920484,99.90807 L 54.717274,91.132217 L 55.421397,91.243181 L 55.619882,87.067076 L 54.816521,87.067076 L 54.518798,85.352258 L 54.017874,80.429702 L 54.216359,74.760706 L 55.31743,74.760706 L 55.31743,71.336105 L 53.913913,71.442015 L 54.117112,67.402096 L 55.747469,67.240708 L 55.823083,65.929374 L 56.749319,65.793192 L 57.699176,65.071956 L 51.985842,63.896802 L 46.31977,65.15265 L 46.872668,66.060507 L 47.47283,66.010066 L 48.172228,65.984848 L 48.299828,67.639144 L 49.878196,67.563497 L 49.906548,71.144447 L 43.111042,70.988097 L 43.337879,67.160002 L 43.559978,63.679927 L 43.559978,59.105378 L 43.763188,54.288748 L 57.373101,53.592733 L 73.567955,52.659674 L 73.71917,55.736265 L 73.142647,63.120082 L 72.892183,69.9945 L 66.928387,69.888585 L 66.900039,65.071956 L 69.106918,64.991267 L 69.206169,63.629486 L 70.108765,63.493308 L 70.061506,63.226006 L 70.964116,63.175568 L 71.465028,62.504773 L 64.721507,60.926122 L 58.001612,62.368592 L 58.4789,63.200785 L 59.230285,63.1453 L 59.230285,63.523577 L 60.156518,63.523577 L 60.156518,65.046738 L 62.136575,65.071956 L 62.112937,69.298485 L 60.109259,69.298485 L 60.080907,70.261816 L 60.785031,70.342507 L 60.70942,74.009202 L 62.188552,74.089909 L 62.013701,88.620507 L 60.057282,89.018952 L 60.080907,89.714967 L 60.761406,89.714967 L 60.761406,93.437137 L 61.886113,93.437137 L 61.588391,98.52109 L 61.210343,102.95945 L 68.331912,103.14605 L 68.105084,99.29275 L 67.580538,96.085028 L 67.476575,93.300955 L 73.520696,93.195041 L 73.345845,97.502272 L 73.317494,102.05159 L 76.729426,102.3189 L 81.3653,102.1323 L 82.820807,101.70358 L 82.017437,99.26753 L 81.818959,95.439438 L 81.440912,92.710853 L 87.206218,92.499027 L 86.955759,95.842931 L 86.932133,101.08826 L 89.238253,101.30009 L 91.520751,101.24965 L 92.621828,100.90165 L 91.969693,95.923633 L 91.747577,92.176239 L 92.725793,92.070324 L 92.749427,88.726422 L 93.02352,88.670945 L 92.976244,87.949712 L 91.846823,87.949712 L 91.619996,85.488427 L 91.520751,74.811143 L 92.371377,74.785924 L 92.371377,71.280616 L 92.725793,71.336105 L 92.725793,70.640088 L 91.468773,70.529127 L 91.497126,66.463987 L 93.600043,66.277382 L 93.477182,64.910557 L 94.403419,64.829863 L 94.351424,64.562549 L 95.580099,63.947229 L 89.337489,62.69138 L 82.995657,63.977495 L 83.39733,64.723951 L 84.375543,64.643256 L 84.427528,64.966046 L 85.254515,64.966046 L 85.301775,66.569901 L 87.357445,66.544681 L 87.532293,70.478688 L 80.264217,70.423216 L 79.413593,64.512124 L 78.733106,61.380041 L 78.184923,55.761484 L 78.510996,52.473053 L 92.999878,51.373557 L 93.047136,46.476221 L 93.774891,46.289613 L 93.727651,45.543159 L 93.174743,45.220372 C 93.174629,45.220372 85.252181,46.395266 82.745197,46.66284 C 82.0389,46.738209 82.09239,46.733258 81.516524,46.79397 L 81.440912,45.886118 L 78.444837,44.317564 L 78.482644,42.491786 L 79.512842,42.461518 L 79.588444,39.949808 C 79.588444,39.949808 85.728225,39.546834 88.009582,39.0117 C 90.290937,38.476559 93.524432,37.942456 93.524432,37.942456 L 95.055545,33.79662 L 98.089437,32.913987 L 98.339888,32.217972 L 105.20628,30.316548 L 105.98602,29.676006 L 103.37744,23.976741 L 103.62792,22.690624 L 104.95584,21.994611 L 105.91041,19.079404 L 105.45673,18.675923 z M 72.466874,40.403728 L 72.429067,42.476654 L 73.983813,42.542211 L 73.884576,44.509221 L 70.836515,46.506487 L 70.647496,47.081457 L 71.876167,47.091543 L 71.866712,47.575729 L 62.552432,48.029652 L 62.613863,46.652742 L 63.039175,45.966809 L 63.067524,45.528025 L 63.07698,44.579832 L 63.341609,43.949374 L 63.440849,43.439982 L 63.440849,43.076841 L 63.842533,41.47297 L 72.466874,40.403728 z M 52.987688,42.168984 L 52.760853,43.561027 L 53.488599,44.418431 L 53.441349,45.916386 L 54.117112,46.960408 L 53.942262,48.191039 L 54.443185,48.912273 L 44.939872,49.2855 L 44.916247,48.967759 L 46.017333,48.831579 L 46.069307,48.428097 L 43.66394,47.121797 L 43.536351,45.03375 L 44.689411,44.978276 L 44.788661,42.72883 L 52.987688,42.168984 z M 67.051262,74.276518 L 72.81657,74.649742 L 72.618099,82.411833 L 73.36947,88.776857 L 67.254465,88.565018 L 67.051262,74.276518 z M 28.44258,74.599304 L 37.671807,75.078442 L 36.868446,80.429702 L 36.868446,84.928593 L 37.520583,87.440302 L 28.494569,87.869006 L 28.44258,74.599304 z M 87.508658,74.649742 L 87.508658,87.924488 L 81.644113,88.353194 L 81.440912,81.342592 L 80.788764,74.811143 L 87.508658,74.649742 z M 43.087416,74.947312 L 49.906548,74.972531 L 49.977434,87.278902 L 43.611966,87.389863 L 43.285891,83.400379 L 43.262266,79.441156 L 43.087416,74.947312 z " + id="path4735" /> + </g> +</svg> diff --git a/lib/jinja2/custom_fixers/__init__.py b/lib/jinja2/custom_fixers/__init__.py new file mode 100644 diff --git a/lib/jinja2/custom_fixers/fix_alt_unicode.py b/lib/jinja2/custom_fixers/fix_alt_unicode.py new file mode 100644 --- /dev/null +++ b/lib/jinja2/custom_fixers/fix_alt_unicode.py @@ -0,0 +1,13 @@ +from lib2to3 import fixer_base +from lib2to3.fixer_util import Name, BlankLine + + +class FixAltUnicode(fixer_base.BaseFix): + PATTERN = """ + func=funcdef< 'def' name='__unicode__' + parameters< '(' NAME ')' > any+ > + """ + + def transform(self, node, results): + name = results['name'] + name.replace(Name('__str__', prefix=name.prefix)) diff --git a/lib/jinja2/custom_fixers/fix_broken_reraising.py b/lib/jinja2/custom_fixers/fix_broken_reraising.py new file mode 100644 --- /dev/null +++ b/lib/jinja2/custom_fixers/fix_broken_reraising.py @@ -0,0 +1,21 @@ +from lib2to3 import fixer_base, pytree +from lib2to3.fixer_util import Name, BlankLine, Name, Attr, ArgList + + +class FixBrokenReraising(fixer_base.BaseFix): + PATTERN = """ + raise_stmt< 'raise' any ',' val=any ',' tb=any > + """ + + # run before the broken 2to3 checker with the same goal + # tries to rewrite it with a rule that does not work out for jinja + run_order = 1 + + def transform(self, node, results): + tb = results['tb'].clone() + tb.prefix = '' + with_tb = Attr(results['val'].clone(), Name('with_traceback')) + \ + [ArgList([tb])] + new = pytree.Node(self.syms.simple_stmt, [Name("raise")] + with_tb) + new.prefix = node.prefix + return new diff --git a/lib/jinja2/custom_fixers/fix_xrange2.py b/lib/jinja2/custom_fixers/fix_xrange2.py new file mode 100644 --- /dev/null +++ b/lib/jinja2/custom_fixers/fix_xrange2.py @@ -0,0 +1,11 @@ +from lib2to3 import fixer_base +from lib2to3.fixer_util import Name, BlankLine + + +# whyever this is necessary.. + +class FixXrange2(fixer_base.BaseFix): + PATTERN = "'xrange'" + + def transform(self, node, results): + node.replace(Name('range', prefix=node.prefix)) diff --git a/lib/jinja2/docs/Makefile b/lib/jinja2/docs/Makefile new file mode 100644 --- /dev/null +++ b/lib/jinja2/docs/Makefile @@ -0,0 +1,118 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = _build + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . + +.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp epub latex changes linkcheck doctest + +help: + @echo "Please use \`make <target>' where <target> is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " singlehtml to make a single large HTML file" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " devhelp to make HTML files and a Devhelp project" + @echo " epub to make an epub" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " latexpdf to make LaTeX files and run them through pdflatex" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + +clean: + -rm -rf $(BUILDDIR)/* + +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +singlehtml: + $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml + @echo + @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." + +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/Flask.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Flask.qhc" + +devhelp: + $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) _build/devhelp + @echo + @echo "Build finished." + @echo "To view the help file:" + @echo "# mkdir -p $$HOME/.local/share/devhelp/Flask" + @echo "# ln -s _build/devhelp $$HOME/.local/share/devhelp/Flask" + @echo "# devhelp" + +epub: + $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub + @echo + @echo "Build finished. The epub file is in $(BUILDDIR)/epub." + +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \ + "run these through (pdf)latex." + +latexpdf: latex + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) _build/latex + @echo "Running LaTeX files through pdflatex..." + make -C _build/latex all-pdf + @echo "pdflatex finished; the PDF files are in _build/latex." + +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." diff --git a/lib/jinja2/docs/_build/.ignore b/lib/jinja2/docs/_build/.ignore new file mode 100644 diff --git a/lib/jinja2/docs/_static/.ignore b/lib/jinja2/docs/_static/.ignore new file mode 100644 diff --git a/lib/jinja2/docs/_static/jinja-small.png b/lib/jinja2/docs/_static/jinja-small.png new file mode 100644 index 0000000000000000000000000000000000000000..455b4c3c99cb843696c61fd43785ce74f486f6ea GIT binary patch [cut] diff --git a/lib/jinja2/docs/_templates/sidebarintro.html b/lib/jinja2/docs/_templates/sidebarintro.html new file mode 100644 --- /dev/null +++ b/lib/jinja2/docs/_templates/sidebarintro.html @@ -0,0 +1,20 @@ +<h3>About Jinja2</h3> +<p> + Jinja2 is a full featured template engine for Python. It has full unicode + support, an optional integrated sandboxed execution environment, widely used + and BSD licensed. +</p> +<h3>Other Formats</h3> +<p> + You can download the documentation in other formats as well: +</p> +<ul> + <li><a href="http://jinja.pocoo.org/docs/jinja-docs.pdf">as PDF</a> + <li><a href="http://jinja.pocoo.org/docs/jinja-docs.zip">as zipped HTML</a> +</ul> +<h3>Useful Links</h3> +<ul> + <li><a href="http://jinja.pocoo.org/">The Jinja2 Website</a></li> + <li><a href="http://pypi.python.org/pypi/Jinja2">Jinja2 @ PyPI</a></li> + <li><a href="http://github.com/mitsuhiko/jinja2">Jinja2 @ github</a></li> +</ul> diff --git a/lib/jinja2/docs/_templates/sidebarlogo.html b/lib/jinja2/docs/_templates/sidebarlogo.html new file mode 100644 --- /dev/null +++ b/lib/jinja2/docs/_templates/sidebarlogo.html @@ -0,0 +1,3 @@ +<p class="logo"><a href="{{ pathto(master_doc) }}"> + <img class="logo" src="{{ pathto('_static/jinja-small.png', 1) }}" alt="Logo"/> +</a></p> diff --git a/lib/jinja2/docs/_themes/LICENSE b/lib/jinja2/docs/_themes/LICENSE new file mode 100644 --- /dev/null +++ b/lib/jinja2/docs/_themes/LICENSE @@ -0,0 +1,37 @@ +Copyright (c) 2010 by Armin Ronacher. + +Some rights reserved. + +Redistribution and use in source and binary forms of the theme, with or +without modification, are permitted provided that the following conditions +are met: + +* Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + +* The names of the contributors may not be used to endorse or + promote products derived from this software without specific + prior written permission. + +We kindly ask you to only use these themes in an unmodified manner just +for Flask and Flask-related products, not for unrelated projects. If you +like the visual style and want to use it for your own projects, please +consider making some larger changes to the themes (such as changing +font faces, sizes, colors or margins). + +THIS THEME IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS THEME, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. diff --git a/lib/jinja2/docs/_themes/README b/lib/jinja2/docs/_themes/README new file mode 100644 --- /dev/null +++ b/lib/jinja2/docs/_themes/README @@ -0,0 +1,31 @@ +Flask Sphinx Styles +=================== + +This repository contains sphinx styles for Flask and Flask related +projects. To use this style in your Sphinx documentation, follow +this guide: + +1. put this folder as _themes into your docs folder. Alternatively + you can also use git submodules to check out the contents there. +2. add this to your conf.py: + + sys.path.append(os.path.abspath('_themes')) + html_theme_path = ['_themes'] + html_theme = 'flask' + +The following themes exist: + +- 'flask' - the standard flask documentation theme for large + projects +- 'flask_small' - small one-page theme. Intended to be used by + very small addon libraries for flask. + +The following options exist for the flask_small theme: + + [options] + index_logo = '' filename of a picture in _static + to be used as replacement for the + h1 in the index.rst file. + index_logo_height = 120px height of the index logo + github_fork = '' repository name on github for the + "fork me" badge diff --git a/lib/jinja2/docs/_themes/jinja/layout.html b/lib/jinja2/docs/_themes/jinja/layout.html new file mode 100644 --- /dev/null +++ b/lib/jinja2/docs/_themes/jinja/layout.html @@ -0,0 +1,8 @@ +{%- extends "basic/layout.html" %} +{%- block relbar2 %}{% endblock %} +{%- block footer %} + <div class="footer"> + © Copyright {{ copyright }}. + Created using <a href="http://sphinx.pocoo.org/">Sphinx</a>. + </div> +{%- endblock %} diff --git a/lib/jinja2/docs/_themes/jinja/relations.html b/lib/jinja2/docs/_themes/jinja/relations.html new file mode 100644 --- /dev/null +++ b/lib/jinja2/docs/_themes/jinja/relations.html @@ -0,0 +1,19 @@ +<h3>Related Topics</h3> +<ul> + <li><a href="{{ pathto(master_doc) }}">Documentation overview</a><ul> + {%- for parent in parents %} + <li><a href="{{ parent.link|e }}">{{ parent.title }}</a><ul> + {%- endfor %} + {%- if prev %} + <li>Previous: <a href="{{ prev.link|e }}" title="{{ _('previous chapter') + }}">{{ prev.title }}</a></li> + {%- endif %} + {%- if next %} + <li>Next: <a href="{{ next.link|e }}" title="{{ _('next chapter') + }}">{{ next.title }}</a></li> + {%- endif %} + {%- for parent in parents %} + </ul></li> + {%- endfor %} + </ul></li> +</ul> diff --git a/lib/jinja2/docs/_themes/jinja/static/jinja.css_t b/lib/jinja2/docs/_themes/jinja/static/jinja.css_t new file mode 100644 --- /dev/null +++ b/lib/jinja2/docs/_themes/jinja/static/jinja.css_t @@ -0,0 +1,396 @@ +/* + * jinja.css_t + * ~~~~~~~~~~~ + * + * :copyright: Copyright 2011 by Armin Ronacher. + * :license: Flask Design License, see LICENSE for details. + */ + + at import url(http://fonts.googleapis.com/css?family=Crimson+Text); + +{% set page_width = '940px' %} +{% set sidebar_width = '220px' %} +{% set font_family = 'Georgia, serif' %} +{% set header_font_family = 'Crimson Text, ' ~ font_family %} + + at import url("basic.css"); + +/* -- page layout ----------------------------------------------------------- */ + +body { + font-family: {{ font_family }}; + font-size: 17px; + background-color: white; + color: #000; + margin: 0; + padding: 0; +} + +div.document { + width: {{ page_width }}; + margin: 30px auto 0 auto; +} + +div.documentwrapper { + float: left; + width: 100%; +} + +div.bodywrapper { + margin: 0 0 0 {{ sidebar_width }}; +} + +div.sphinxsidebar { + width: {{ sidebar_width }}; +} + +hr { + border: 1px solid #B1B4B6; +} + +div.body { + background-color: #ffffff; + color: #3E4349; + padding: 0 30px 0 30px; +} + +img.floatingflask { + padding: 0 0 10px 10px; + float: right; +} + +div.footer { + width: {{ page_width }}; + margin: 20px auto 30px auto; + font-size: 14px; + color: #888; + text-align: right; +} + +div.footer a { + color: #888; +} + +div.related { + display: none; +} + +div.sphinxsidebar a { + color: #444; + text-decoration: none; + border-bottom: 1px dotted #999; +} + +div.sphinxsidebar a:hover { + border-bottom: 1px solid #999; +} + +div.sphinxsidebar { + font-size: 15px; + line-height: 1.5; +} + +div.sphinxsidebarwrapper { + padding: 18px 10px; +} + +div.sphinxsidebarwrapper p.logo { + padding: 0 0 20px 0; + margin: 0; + text-align: center; +} + +div.sphinxsidebar h3, +div.sphinxsidebar h4 { + font-family: {{ font_family }}; + color: #444; + font-size: 24px; + font-weight: normal; + margin: 0 0 5px 0; + padding: 0; +} + +div.sphinxsidebar h4 { + font-size: 20px; +} + +div.sphinxsidebar h3 a { + color: #444; +} + +div.sphinxsidebar p.logo a, +div.sphinxsidebar h3 a, +div.sphinxsidebar p.logo a:hover, +div.sphinxsidebar h3 a:hover { + border: none; +} + +div.sphinxsidebar p { + color: #555; + margin: 10px 0; +} + +div.sphinxsidebar ul { + margin: 10px 0; + padding: 0; + color: #000; +} + +div.sphinxsidebar input { + border: 1px solid #ccc; + font-family: {{ font_family }}; + font-size: 14px; +} + +div.sphinxsidebar form.search input[name="q"] { + width: 130px; +} + +/* -- body styles ----------------------------------------------------------- */ + +a { + color: #aa0000; + text-decoration: underline; +} + +a:hover { + color: #dd0000; + text-decoration: underline; +} + +div.body h1, +div.body h2, +div.body h3, +div.body h4, +div.body h5, +div.body h6 { + font-family: {{ header_font_family }}; + font-weight: normal; + margin: 30px 0px 10px 0px; + padding: 0; + color: black; +} + +div.body h1 { margin-top: 0; padding-top: 0; font-size: 240%; } +div.body h2 { font-size: 180%; } +div.body h3 { font-size: 150%; } +div.body h4 { font-size: 130%; } +div.body h5 { font-size: 100%; } +div.body h6 { font-size: 100%; } + +a.headerlink { + color: #ddd; + padding: 0 4px; + text-decoration: none; +} + +a.headerlink:hover { + color: #444; + background: #eaeaea; +} + +div.body p, div.body dd, div.body li { + line-height: 1.4em; +} + +div.admonition { + background: #fafafa; + margin: 20px -30px; + padding: 10px 30px; + border-top: 1px solid #ccc; + border-bottom: 1px solid #ccc; +} + +div.admonition tt.xref, div.admonition a tt { + border-bottom: 1px solid #fafafa; +} + +dd div.admonition { + margin-left: -60px; + padding-left: 60px; +} + +div.admonition p.admonition-title { + font-family: {{ font_family }}; + font-weight: normal; + font-size: 24px; + margin: 0 0 10px 0; + padding: 0; + line-height: 1; +} + +div.admonition p.last { + margin-bottom: 0; +} + +div.highlight { + background-color: white; +} + +dt:target, .highlight { + background: #FAF3E8; +} + +div.note { + background-color: #eee; + border: 1px solid #ccc; +} + +div.seealso { + background-color: #ffc; + border: 1px solid #ff6; +} + +div.topic { + background-color: #eee; +} + +p.admonition-title { + display: inline; +} + +p.admonition-title:after { + content: ":"; +} + +pre, tt { + font-family: 'Consolas', 'Menlo', 'Deja Vu Sans Mono', 'Bitstream Vera Sans Mono', monospace; + font-size: 0.85em; +} + +img.screenshot { +} + +tt.descname, tt.descclassname { + font-size: 0.95em; +} + +tt.descname { + padding-right: 0.08em; +} + +img.screenshot { + -moz-box-shadow: 2px 2px 4px #eee; + -webkit-box-shadow: 2px 2px 4px #eee; + box-shadow: 2px 2px 4px #eee; +} + +table.docutils { + border: 1px solid #888; + -moz-box-shadow: 2px 2px 4px #eee; + -webkit-box-shadow: 2px 2px 4px #eee; + box-shadow: 2px 2px 4px #eee; +} + +table.docutils td, table.docutils th { + border: 1px solid #888; + padding: 0.25em 0.7em; +} + +table.field-list, table.footnote { + border: none; + -moz-box-shadow: none; + -webkit-box-shadow: none; + box-shadow: none; +} + +table.footnote { + margin: 15px 0; + width: 100%; + border: 1px solid #eee; + background: #fdfdfd; + font-size: 0.9em; +} + +table.footnote + table.footnote { + margin-top: -15px; + border-top: none; +} + +table.field-list th { + padding: 0 0.8em 0 0; +} + +table.field-list td { + padding: 0; +} + +table.footnote td.label { + width: 0px; + padding: 0.3em 0 0.3em 0.5em; +} + +table.footnote td { + padding: 0.3em 0.5em; +} + +dl { + margin: 0; + padding: 0; +} + +dl dd { + margin-left: 30px; +} + +blockquote { + margin: 0 0 0 30px; + padding: 0; +} + +ul, ol { + margin: 10px 0 10px 30px; + padding: 0; +} + +pre { + background: #eee; + padding: 7px 30px; + margin: 15px -30px; + line-height: 1.3em; +} + +dl pre, blockquote pre, li pre { + margin-left: -60px; + padding-left: 60px; +} + +dl dl pre { + margin-left: -90px; + padding-left: 90px; +} + +tt { + background-color: #E8EFF0; + color: #222; + /* padding: 1px 2px; */ +} + +tt.xref, a tt { + background-color: #E8EFF0; + border-bottom: 1px solid white; +} + +a.reference { + text-decoration: none; + border-bottom: 1px dotted #bb0000; +} + +a.reference:hover { + border-bottom: 1px solid #dd0000; +} + +a.footnote-reference { + text-decoration: none; + font-size: 0.7em; + vertical-align: top; + border-bottom: 1px dotted #bb0000; +} + +a.footnote-reference:hover { + border-bottom: 1px solid #dd0000; +} + +a:hover tt { + background: #EEE; +} diff --git a/lib/jinja2/docs/_themes/jinja/theme.conf b/lib/jinja2/docs/_themes/jinja/theme.conf new file mode 100644 --- /dev/null +++ b/lib/jinja2/docs/_themes/jinja/theme.conf @@ -0,0 +1,3 @@ +[theme] +inherit = basic +stylesheet = jinja.css diff --git a/lib/jinja2/docs/api.rst b/lib/jinja2/docs/api.rst new file mode 100644 --- /dev/null +++ b/lib/jinja2/docs/api.rst @@ -0,0 +1,790 @@ +API +=== + +.. module:: jinja2 + :synopsis: public Jinja2 API + +This document describes the API to Jinja2 and not the template language. It +will be most useful as reference to those implementing the template interface +to the application and not those who are creating Jinja2 templates. + +Basics +------ + +Jinja2 uses a central object called the template :class:`Environment`. +Instances of this class are used to store the configuration, global objects +and are used to load templates from the file system or other locations. +Even if you are creating templates from strings by using the constructor of +:class:`Template` class, an environment is created automatically for you, +albeit a shared one. + +Most applications will create one :class:`Environment` object on application +initialization and use that to load templates. In some cases it's however +useful to have multiple environments side by side, if different configurations +are in use. + +The simplest way to configure Jinja2 to load templates for your application +looks roughly like this:: + + from jinja2 import Environment, PackageLoader + env = Environment(loader=PackageLoader('yourapplication', 'templates')) + +This will create a template environment with the default settings and a +loader that looks up the templates in the `templates` folder inside the +`yourapplication` python package. Different loaders are available +and you can also write your own if you want to load templates from a +database or other resources. + +To load a template from this environment you just have to call the +:meth:`get_template` method which then returns the loaded :class:`Template`:: + + template = env.get_template('mytemplate.html') + +To render it with some variables, just call the :meth:`render` method:: + + print template.render(the='variables', go='here') + +Using a template loader rather then passing strings to :class:`Template` +or :meth:`Environment.from_string` has multiple advantages. Besides being +a lot easier to use it also enables template inheritance. + + +Unicode +------- + +Jinja2 is using Unicode internally which means that you have to pass Unicode +objects to the render function or bytestrings that only consist of ASCII +characters. Additionally newlines are normalized to one end of line +sequence which is per default UNIX style (``\n``). + +Python 2.x supports two ways of representing string objects. One is the +`str` type and the other is the `unicode` type, both of which extend a type +called `basestring`. Unfortunately the default is `str` which should not +be used to store text based information unless only ASCII characters are +used. With Python 2.6 it is possible to make `unicode` the default on a per +module level and with Python 3 it will be the default. + +To explicitly use a Unicode string you have to prefix the string literal +with a `u`: ``u'Hänsel und Gretel sagen Hallo'``. That way Python will +store the string as Unicode by decoding the string with the character +encoding from the current Python module. If no encoding is specified this +defaults to 'ASCII' which means that you can't use any non ASCII identifier. + +To set a better module encoding add the following comment to the first or +second line of the Python module using the Unicode literal:: + + # -*- coding: utf-8 -*- + +We recommend utf-8 as Encoding for Python modules and templates as it's +possible to represent every Unicode character in utf-8 and because it's +backwards compatible to ASCII. For Jinja2 the default encoding of templates +is assumed to be utf-8. + +It is not possible to use Jinja2 to process non-Unicode data. The reason +for this is that Jinja2 uses Unicode already on the language level. For +example Jinja2 treats the non-breaking space as valid whitespace inside +expressions which requires knowledge of the encoding or operating on an +Unicode string. + +For more details about Unicode in Python have a look at the excellent +`Unicode documentation`_. + +Another important thing is how Jinja2 is handling string literals in +templates. A naive implementation would be using Unicode strings for +all string literals but it turned out in the past that this is problematic +as some libraries are typechecking against `str` explicitly. For example +`datetime.strftime` does not accept Unicode arguments. To not break it +completely Jinja2 is returning `str` for strings that fit into ASCII and +for everything else `unicode`: + +>>> m = Template(u"{% set a, b = 'foo', 'föö' %}").module +>>> m.a +'foo' +>>> m.b +u'f\xf6\xf6' + + +.. _Unicode documentation: http://docs.python.org/dev/howto/unicode.html + +High Level API +-------------- + +The high-level API is the API you will use in the application to load and +render Jinja2 templates. The :ref:`low-level-api` on the other side is only +useful if you want to dig deeper into Jinja2 or :ref:`develop extensions +<jinja-extensions>`. + +.. autoclass:: Environment([options]) + :members: from_string, get_template, select_template, + get_or_select_template, join_path, extend, compile_expression, + compile_templates, list_templates, add_extension + + .. attribute:: shared + + If a template was created by using the :class:`Template` constructor + an environment is created automatically. These environments are + created as shared environments which means that multiple templates + may have the same anonymous environment. For all shared environments + this attribute is `True`, else `False`. + + .. attribute:: sandboxed + + If the environment is sandboxed this attribute is `True`. For the + sandbox mode have a look at the documentation for the + :class:`~jinja2.sandbox.SandboxedEnvironment`. + + .. attribute:: filters + + A dict of filters for this environment. As long as no template was + loaded it's safe to add new filters or remove old. For custom filters + see :ref:`writing-filters`. For valid filter names have a look at + :ref:`identifier-naming`. + + .. attribute:: tests + + A dict of test functions for this environment. As long as no + template was loaded it's safe to modify this dict. For custom tests + see :ref:`writing-tests`. For valid test names have a look at + :ref:`identifier-naming`. + + .. attribute:: globals + + A dict of global variables. These variables are always available + in a template. As long as no template was loaded it's safe + to modify this dict. For more details see :ref:`global-namespace`. + For valid object names have a look at :ref:`identifier-naming`. + + .. automethod:: overlay([options]) + + .. method:: undefined([hint, obj, name, exc]) + + Creates a new :class:`Undefined` object for `name`. This is useful + for filters or functions that may return undefined objects for + some operations. All parameters except of `hint` should be provided + as keyword parameters for better readability. The `hint` is used as + error message for the exception if provided, otherwise the error + message will be generated from `obj` and `name` automatically. The exception + provided as `exc` is raised if something with the generated undefined + object is done that the undefined object does not allow. The default + exception is :exc:`UndefinedError`. If a `hint` is provided the + `name` may be ommited. + + The most common way to create an undefined object is by providing + a name only:: + + return environment.undefined(name='some_name') + + This means that the name `some_name` is not defined. If the name + was from an attribute of an object it makes sense to tell the + undefined object the holder object to improve the error message:: + + if not hasattr(obj, 'attr'): + return environment.undefined(obj=obj, name='attr') + + For a more complex example you can provide a hint. For example + the :func:`first` filter creates an undefined object that way:: + + return environment.undefined('no first item, sequence was empty') + + If it the `name` or `obj` is known (for example because an attribute + was accessed) it shold be passed to the undefined object, even if + a custom `hint` is provided. This gives undefined objects the + possibility to enhance the error message. + +.. autoclass:: Template + :members: module, make_module + + .. attribute:: globals + + The dict with the globals of that template. It's unsafe to modify + this dict as it may be shared with other templates or the environment + that loaded the template. + + .. attribute:: name + + The loading name of the template. If the template was loaded from a + string this is `None`. + + .. attribute:: filename + + The filename of the template on the file system if it was loaded from + there. Otherwise this is `None`. + + .. automethod:: render([context]) + + .. automethod:: generate([context]) + + .. automethod:: stream([context]) + + +.. autoclass:: jinja2.environment.TemplateStream() + :members: disable_buffering, enable_buffering, dump + + +Autoescaping +------------ + +.. versionadded:: 2.4 + +As of Jinja 2.4 the preferred way to do autoescaping is to enable the +:ref:`autoescape-extension` and to configure a sensible default for +autoescaping. This makes it possible to enable and disable autoescaping +on a per-template basis (HTML versus text for instance). + +Here a recommended setup that enables autoescaping for templates ending +in ``'.html'``, ``'.htm'`` and ``'.xml'`` and disabling it by default +for all other extensions:: + + def guess_autoescape(template_name): + if template_name is None or '.' not in template_name: + return False + ext = template_name.rsplit('.', 1)[1] + return ext in ('html', 'htm', 'xml') + + env = Environment(autoescape=guess_autoescape, + loader=PackageLoader('mypackage'), + extensions=['jinja2.ext.autoescape']) + +When implementing a guessing autoescape function, make sure you also +accept `None` as valid template name. This will be passed when generating +templates from strings. + +Inside the templates the behaviour can be temporarily changed by using +the `autoescape` block (see :ref:`autoescape-overrides`). + + +.. _identifier-naming: + +Notes on Identifiers +-------------------- + +Jinja2 uses the regular Python 2.x naming rules. Valid identifiers have to +match ``[a-zA-Z_][a-zA-Z0-9_]*``. As a matter of fact non ASCII characters +are currently not allowed. This limitation will probably go away as soon as +unicode identifiers are fully specified for Python 3. + +Filters and tests are looked up in separate namespaces and have slightly +modified identifier syntax. Filters and tests may contain dots to group +filters and tests by topic. For example it's perfectly valid to add a +function into the filter dict and call it `to.unicode`. The regular +expression for filter and test identifiers is +``[a-zA-Z_][a-zA-Z0-9_]*(\.[a-zA-Z_][a-zA-Z0-9_]*)*```. + + +Undefined Types +--------------- + +These classes can be used as undefined types. The :class:`Environment` +constructor takes an `undefined` parameter that can be one of those classes +or a custom subclass of :class:`Undefined`. Whenever the template engine is +unable to look up a name or access an attribute one of those objects is +created and returned. Some operations on undefined values are then allowed, +others fail. + +The closest to regular Python behavior is the `StrictUndefined` which +disallows all operations beside testing if it's an undefined object. + +.. autoclass:: jinja2.Undefined() + + .. attribute:: _undefined_hint + + Either `None` or an unicode string with the error message for + the undefined object. + + .. attribute:: _undefined_obj + + Either `None` or the owner object that caused the undefined object + to be created (for example because an attribute does not exist). + + .. attribute:: _undefined_name + + The name for the undefined variable / attribute or just `None` + if no such information exists. + + .. attribute:: _undefined_exception + + The exception that the undefined object wants to raise. This + is usually one of :exc:`UndefinedError` or :exc:`SecurityError`. + + .. method:: _fail_with_undefined_error(\*args, \**kwargs) + + When called with any arguments this method raises + :attr:`_undefined_exception` with an error message generated + from the undefined hints stored on the undefined object. + +.. autoclass:: jinja2.DebugUndefined() + +.. autoclass:: jinja2.StrictUndefined() + +Undefined objects are created by calling :attr:`undefined`. + +.. admonition:: Implementation + + :class:`Undefined` objects are implemented by overriding the special + `__underscore__` methods. For example the default :class:`Undefined` + class implements `__unicode__` in a way that it returns an empty + string, however `__int__` and others still fail with an exception. To + allow conversion to int by returning ``0`` you can implement your own:: + + class NullUndefined(Undefined): + def __int__(self): + return 0 + def __float__(self): + return 0.0 + + To disallow a method, just override it and raise + :attr:`~Undefined._undefined_exception`. Because this is a very common + idom in undefined objects there is the helper method + :meth:`~Undefined._fail_with_undefined_error` that does the error raising + automatically. Here a class that works like the regular :class:`Undefined` + but chokes on iteration:: + + class NonIterableUndefined(Undefined): + __iter__ = Undefined._fail_with_undefined_error + + +The Context +----------- + +.. autoclass:: jinja2.runtime.Context() + :members: resolve, get_exported, get_all + + .. attribute:: parent + + A dict of read only, global variables the template looks up. These + can either come from another :class:`Context`, from the + :attr:`Environment.globals` or :attr:`Template.globals` or points + to a dict created by combining the globals with the variables + passed to the render function. It must not be altered. + + .. attribute:: vars + + The template local variables. This list contains environment and + context functions from the :attr:`parent` scope as well as local + modifications and exported variables from the template. The template + will modify this dict during template evaluation but filters and + context functions are not allowed to modify it. + + .. attribute:: environment + + The environment that loaded the template. + + .. attribute:: exported_vars + + This set contains all the names the template exports. The values for + the names are in the :attr:`vars` dict. In order to get a copy of the + exported variables as dict, :meth:`get_exported` can be used. + + .. attribute:: name + + The load name of the template owning this context. + + .. attribute:: blocks + + A dict with the current mapping of blocks in the template. The keys + in this dict are the names of the blocks, and the values a list of + blocks registered. The last item in each list is the current active + block (latest in the inheritance chain). + + .. attribute:: eval_ctx + + The current :ref:`eval-context`. + + .. automethod:: jinja2.runtime.Context.call(callable, \*args, \**kwargs) + + +.. admonition:: Implementation + + Context is immutable for the same reason Python's frame locals are + immutable inside functions. Both Jinja2 and Python are not using the + context / frame locals as data storage for variables but only as primary + data source. + + When a template accesses a variable the template does not define, Jinja2 + looks up the variable in the context, after that the variable is treated + as if it was defined in the template. + + +.. _loaders: + +Loaders +------- + +Loaders are responsible for loading templates from a resource such as the +file system. The environment will keep the compiled modules in memory like +Python's `sys.modules`. Unlike `sys.modules` however this cache is limited in +size by default and templates are automatically reloaded. +All loaders are subclasses of :class:`BaseLoader`. If you want to create your +own loader, subclass :class:`BaseLoader` and override `get_source`. + +.. autoclass:: jinja2.BaseLoader + :members: get_source, load + +Here a list of the builtin loaders Jinja2 provides: + +.. autoclass:: jinja2.FileSystemLoader + +.. autoclass:: jinja2.PackageLoader + +.. autoclass:: jinja2.DictLoader + +.. autoclass:: jinja2.FunctionLoader + +.. autoclass:: jinja2.PrefixLoader + +.. autoclass:: jinja2.ChoiceLoader + +.. autoclass:: jinja2.ModuleLoader + + +.. _bytecode-cache: + +Bytecode Cache +-------------- + +Jinja 2.1 and higher support external bytecode caching. Bytecode caches make +it possible to store the generated bytecode on the file system or a different +location to avoid parsing the templates on first use. + +This is especially useful if you have a web application that is initialized on +the first request and Jinja compiles many templates at once which slows down +the application. + +To use a bytecode cache, instanciate it and pass it to the :class:`Environment`. + +.. autoclass:: jinja2.BytecodeCache + :members: load_bytecode, dump_bytecode, clear + +.. autoclass:: jinja2.bccache.Bucket + :members: write_bytecode, load_bytecode, bytecode_from_string, + bytecode_to_string, reset + + .. attribute:: environment + + The :class:`Environment` that created the bucket. + + .. attribute:: key + + The unique cache key for this bucket + + .. attribute:: code + + The bytecode if it's loaded, otherwise `None`. + + +Builtin bytecode caches: + +.. autoclass:: jinja2.FileSystemBytecodeCache + +.. autoclass:: jinja2.MemcachedBytecodeCache + + +Utilities +--------- + +These helper functions and classes are useful if you add custom filters or +functions to a Jinja2 environment. + +.. autofunction:: jinja2.environmentfilter + +.. autofunction:: jinja2.contextfilter + +.. autofunction:: jinja2.evalcontextfilter + +.. autofunction:: jinja2.environmentfunction + +.. autofunction:: jinja2.contextfunction + +.. autofunction:: jinja2.evalcontextfunction + +.. function:: escape(s) + + Convert the characters ``&``, ``<``, ``>``, ``'``, and ``"`` in string `s` + to HTML-safe sequences. Use this if you need to display text that might + contain such characters in HTML. This function will not escaped objects + that do have an HTML representation such as already escaped data. + + The return value is a :class:`Markup` string. + +.. autofunction:: jinja2.clear_caches + +.. autofunction:: jinja2.is_undefined + +.. autoclass:: jinja2.Markup([string]) + :members: escape, unescape, striptags + +.. admonition:: Note + + The Jinja2 :class:`Markup` class is compatible with at least Pylons and + Genshi. It's expected that more template engines and framework will pick + up the `__html__` concept soon. + + +Exceptions +---------- + +.. autoexception:: jinja2.TemplateError + +.. autoexception:: jinja2.UndefinedError + +.. autoexception:: jinja2.TemplateNotFound + +.. autoexception:: jinja2.TemplatesNotFound + +.. autoexception:: jinja2.TemplateSyntaxError + + .. attribute:: message + + The error message as utf-8 bytestring. + + .. attribute:: lineno + + The line number where the error occurred + + .. attribute:: name + + The load name for the template as unicode string. + + .. attribute:: filename + + The filename that loaded the template as bytestring in the encoding + of the file system (most likely utf-8 or mbcs on Windows systems). + + The reason why the filename and error message are bytestrings and not + unicode strings is that Python 2.x is not using unicode for exceptions + and tracebacks as well as the compiler. This will change with Python 3. + +.. autoexception:: jinja2.TemplateAssertionError + + +.. _writing-filters: + +Custom Filters +-------------- + +Custom filters are just regular Python functions that take the left side of +the filter as first argument and the the arguments passed to the filter as +extra arguments or keyword arguments. + +For example in the filter ``{{ 42|myfilter(23) }}`` the function would be +called with ``myfilter(42, 23)``. Here for example a simple filter that can +be applied to datetime objects to format them:: + + def datetimeformat(value, format='%H:%M / %d-%m-%Y'): + return value.strftime(format) + +You can register it on the template environment by updating the +:attr:`~Environment.filters` dict on the environment:: + + environment.filters['datetimeformat'] = datetimeformat + +Inside the template it can then be used as follows: + +.. sourcecode:: jinja + + written on: {{ article.pub_date|datetimeformat }} + publication date: {{ article.pub_date|datetimeformat('%d-%m-%Y') }} + +Filters can also be passed the current template context or environment. This +is useful if a filter wants to return an undefined value or check the current +:attr:`~Environment.autoescape` setting. For this purpose three decorators +exist: :func:`environmentfilter`, :func:`contextfilter` and +:func:`evalcontextfilter`. + +Here a small example filter that breaks a text into HTML line breaks and +paragraphs and marks the return value as safe HTML string if autoescaping is +enabled:: + + import re + from jinja2 import evalcontextfilter, Markup, escape + + _paragraph_re = re.compile(r'(?:\r\n|\r|\n){2,}') + + @evalcontextfilter + def nl2br(eval_ctx, value): + result = u'\n\n'.join(u'<p>%s</p>' % p.replace('\n', '<br>\n') + for p in _paragraph_re.split(escape(value))) + if eval_ctx.autoescape: + result = Markup(result) + return result + +Context filters work the same just that the first argument is the current +active :class:`Context` rather then the environment. + + +.. _eval-context: + +Evaluation Context +------------------ + +The evaluation context (short eval context or eval ctx) is a new object +introducted in Jinja 2.4 that makes it possible to activate and deactivate +compiled features at runtime. + +Currently it is only used to enable and disable the automatic escaping but +can be used for extensions as well. + +In previous Jinja versions filters and functions were marked as +environment callables in order to check for the autoescape status from the +environment. In new versions it's encouraged to check the setting from the +evaluation context instead. + +Previous versions:: + + @environmentfilter + def filter(env, value): + result = do_something(value) + if env.autoescape: + result = Markup(result) + return result + +In new versions you can either use a :func:`contextfilter` and access the +evaluation context from the actual context, or use a +:func:`evalcontextfilter` which directly passes the evaluation context to +the function:: + + @contextfilter + def filter(context, value): + result = do_something(value) + if context.eval_ctx.autoescape: + result = Markup(result) + return result + + @evalcontextfilter + def filter(eval_ctx, value): + result = do_something(value) + if eval_ctx.autoescape: + result = Markup(result) + return result + +The evaluation context must not be modified at runtime. Modifications +must only happen with a :class:`nodes.EvalContextModifier` and +:class:`nodes.ScopedEvalContextModifier` from an extension, not on the +eval context object itself. + +.. autoclass:: jinja2.nodes.EvalContext + + .. attribute:: autoescape + + `True` or `False` depending on if autoescaping is active or not. + + .. attribute:: volatile + + `True` if the compiler cannot evaluate some expressions at compile + time. At runtime this should always be `False`. + + +.. _writing-tests: + +Custom Tests +------------ + +Tests work like filters just that there is no way for a test to get access +to the environment or context and that they can't be chained. The return +value of a test should be `True` or `False`. The purpose of a test is to +give the template designers the possibility to perform type and conformability +checks. + +Here a simple test that checks if a variable is a prime number:: + + import math + + def is_prime(n): + if n == 2: + return True + for i in xrange(2, int(math.ceil(math.sqrt(n))) + 1): + if n % i == 0: + return False + return True + + +You can register it on the template environment by updating the +:attr:`~Environment.tests` dict on the environment:: + + environment.tests['prime'] = is_prime + +A template designer can then use the test like this: + +.. sourcecode:: jinja + + {% if 42 is prime %} + 42 is a prime number + {% else %} + 42 is not a prime number + {% endif %} + + +.. _global-namespace: + +The Global Namespace +-------------------- + +Variables stored in the :attr:`Environment.globals` dict are special as they +are available for imported templates too, even if they are imported without +context. This is the place where you can put variables and functions +that should be available all the time. Additionally :attr:`Template.globals` +exist that are variables available to a specific template that are available +to all :meth:`~Template.render` calls. + + +.. _low-level-api: + +Low Level API +------------- + +The low level API exposes functionality that can be useful to understand some +implementation details, debugging purposes or advanced :ref:`extension +<jinja-extensions>` techniques. Unless you know exactly what you are doing we +don't recommend using any of those. + +.. automethod:: Environment.lex + +.. automethod:: Environment.parse + +.. automethod:: Environment.preprocess + +.. automethod:: Template.new_context + +.. method:: Template.root_render_func(context) + + This is the low level render function. It's passed a :class:`Context` + that has to be created by :meth:`new_context` of the same template or + a compatible template. This render function is generated by the + compiler from the template code and returns a generator that yields + unicode strings. + + If an exception in the template code happens the template engine will + not rewrite the exception but pass through the original one. As a + matter of fact this function should only be called from within a + :meth:`render` / :meth:`generate` / :meth:`stream` call. + +.. attribute:: Template.blocks + + A dict of block render functions. Each of these functions works exactly + like the :meth:`root_render_func` with the same limitations. + +.. attribute:: Template.is_up_to_date + + This attribute is `False` if there is a newer version of the template + available, otherwise `True`. + +.. admonition:: Note + + The low-level API is fragile. Future Jinja2 versions will try not to + change it in a backwards incompatible way but modifications in the Jinja2 + core may shine through. For example if Jinja2 introduces a new AST node + in later versions that may be returned by :meth:`~Environment.parse`. + +The Meta API +------------ + +.. versionadded:: 2.2 + +The meta API returns some information about abstract syntax trees that +could help applications to implement more advanced template concepts. All +the functions of the meta API operate on an abstract syntax tree as +returned by the :meth:`Environment.parse` method. + +.. autofunction:: jinja2.meta.find_undeclared_variables + +.. autofunction:: jinja2.meta.find_referenced_templates diff --git a/lib/jinja2/docs/cache_extension.py b/lib/jinja2/docs/cache_extension.py new file mode 100644 --- /dev/null +++ b/lib/jinja2/docs/cache_extension.py @@ -0,0 +1,56 @@ +from jinja2 import nodes +from jinja2.ext import Extension + + +class FragmentCacheExtension(Extension): + # a set of names that trigger the extension. + tags = set(['cache']) + + def __init__(self, environment): + super(FragmentCacheExtension, self).__init__(environment) + + # add the defaults to the environment + environment.extend( + fragment_cache_prefix='', + fragment_cache=None + ) + + def parse(self, parser): + # the first token is the token that started the tag. In our case + # we only listen to ``'cache'`` so this will be a name token with + # `cache` as value. We get the line number so that we can give + # that line number to the nodes we create by hand. + lineno = parser.stream.next().lineno + + # now we parse a single expression that is used as cache key. + args = [parser.parse_expression()] + + # if there is a comma, the user provided a timeout. If not use + # None as second parameter. + if parser.stream.skip_if('comma'): + args.append(parser.parse_expression()) + else: + args.append(nodes.Const(None)) + + # now we parse the body of the cache block up to `endcache` and + # drop the needle (which would always be `endcache` in that case) + body = parser.parse_statements(['name:endcache'], drop_needle=True) + + # now return a `CallBlock` node that calls our _cache_support + # helper method on this extension. + return nodes.CallBlock(self.call_method('_cache_support', args), + [], [], body).set_lineno(lineno) + + def _cache_support(self, name, timeout, caller): + """Helper callback.""" + key = self.environment.fragment_cache_prefix + name + + # try to load the block from the cache + # if there is no fragment in the cache, render it and store + # it in the cache. + rv = self.environment.fragment_cache.get(key) + if rv is not None: + return rv + rv = caller() + self.environment.fragment_cache.add(key, rv, timeout) + return rv diff --git a/lib/jinja2/docs/changelog.rst b/lib/jinja2/docs/changelog.rst new file mode 100644 --- /dev/null +++ b/lib/jinja2/docs/changelog.rst @@ -0,0 +1,3 @@ +.. module:: jinja2 + +.. include:: ../CHANGES diff --git a/lib/jinja2/docs/conf.py b/lib/jinja2/docs/conf.py new file mode 100644 --- /dev/null +++ b/lib/jinja2/docs/conf.py @@ -0,0 +1,160 @@ +# -*- coding: utf-8 -*- +# +# Jinja2 documentation build configuration file, created by +# sphinx-quickstart on Sun Apr 27 21:42:41 2008. +# +# This file is execfile()d with the current directory set to its containing dir. +# +# The contents of this file are pickled, so don't put values in the namespace +# that aren't pickleable (module imports are okay, they're removed automatically). +# +# All configuration values have a default value; values that are commented out +# serve to show the default value. + +import sys, os + +# If your extensions are in another directory, add it here. If the directory +# is relative to the documentation root, use os.path.abspath to make it +# absolute, like shown here. +sys.path.append(os.path.dirname(os.path.abspath(__file__))) + +# General configuration +# --------------------- + +# Add any Sphinx extension module names here, as strings. They can be extensions +# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = ['sphinx.ext.autodoc', 'jinjaext'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# General substitutions. +project = 'Jinja2' +copyright = '2008, Armin Ronacher' + +# The default replacements for |version| and |release|, also used in various +# other places throughout the built documents. +# +# The short X.Y version. +import pkg_resources +try: + release = pkg_resources.get_distribution('Jinja2').version +except ImportError: + print 'To build the documentation, The distribution information of Jinja2' + print 'Has to be available. Either install the package into your' + print 'development environment or run "setup.py develop" to setup the' + print 'metadata. A virtualenv is recommended!' + sys.exit(1) +if 'dev' in release: + release = release.split('dev')[0] + 'dev' +version = '.'.join(release.split('.')[:2]) + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +today_fmt = '%B %d, %Y' + +# List of documents that shouldn't be included in the build. +#unused_docs = [] + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'jinjaext.JinjaStyle' + + +# Options for HTML output +# ----------------------- + +html_theme = 'jinja' +html_theme_path = ['_themes'] + +# The name for this set of Sphinx documents. If None, it defaults to +# "<project> v<release> documentation". +#html_title = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# no modindex +html_use_modindex = False + +# If true, the reST sources are included in the HTML build as _sources/<name>. +#html_copy_source = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a <link> tag referring to it. +#html_use_opensearch = False + +# Output file base name for HTML help builder. +htmlhelp_basename = 'Jinja2doc' + + +# Options for LaTeX output +# ------------------------ + +# The paper size ('letter' or 'a4'). +latex_paper_size = 'a4' + +# The font size ('10pt', '11pt' or '12pt'). +#latex_font_size = '10pt' + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, document class [howto/manual]). +latex_documents = [ + ('latexindex', 'Jinja2.tex', 'Jinja2 Documentation', 'Armin Ronacher', + 'manual'), +] + +# Additional stuff for LaTeX +latex_elements = { + 'fontpkg': r'\usepackage{mathpazo}', + 'papersize': 'a4paper', + 'pointsize': '12pt', + 'preamble': r''' +\usepackage{jinjastyle} + +% i hate you latex +\DeclareUnicodeCharacter{14D}{o} +''' +} + +latex_use_parts = True + +latex_additional_files = ['jinjastyle.sty', 'logo.pdf'] + +# If false, no module index is generated. +latex_use_modindex = False + +html_sidebars = { + 'index': ['sidebarlogo.html', 'sidebarintro.html', 'sourcelink.html', + 'searchbox.html'], + '**': ['sidebarlogo.html', 'localtoc.html', 'relations.html', + 'sourcelink.html', 'searchbox.html'] +} diff --git a/lib/jinja2/docs/contents.rst.inc b/lib/jinja2/docs/contents.rst.inc new file mode 100644 --- /dev/null +++ b/lib/jinja2/docs/contents.rst.inc @@ -0,0 +1,29 @@ +Jinja2 Documentation +-------------------- + +.. toctree:: + :maxdepth: 2 + + intro + api + sandbox + templates + extensions + integration + switching + tricks + +Additional Information +---------------------- + +.. toctree:: + :maxdepth: 2 + + faq + changelog + +If you can't find the information you're looking for, have a look at the +index or try to find it using the search function: + +* :ref:`genindex` +* :ref:`search` diff --git a/lib/jinja2/docs/extensions.rst b/lib/jinja2/docs/extensions.rst new file mode 100644 --- /dev/null +++ b/lib/jinja2/docs/extensions.rst @@ -0,0 +1,347 @@ +.. _jinja-extensions: + +Extensions +========== + +Jinja2 supports extensions that can add extra filters, tests, globals or even +extend the parser. The main motivation of extensions is it to move often used +code into a reusable class like adding support for internationalization. + + +Adding Extensions +----------------- + +Extensions are added to the Jinja2 environment at creation time. Once the +environment is created additional extensions cannot be added. To add an +extension pass a list of extension classes or import paths to the +`environment` parameter of the :class:`Environment` constructor. The following +example creates a Jinja2 environment with the i18n extension loaded:: + + jinja_env = Environment(extensions=['jinja2.ext.i18n']) + + +.. _i18n-extension: + +i18n Extension +-------------- + +**Import name:** `jinja2.ext.i18n` + +Jinja2 currently comes with one extension, the i18n extension. It can be +used in combination with `gettext`_ or `babel`_. If the i18n extension is +enabled Jinja2 provides a `trans` statement that marks the wrapped string as +translatable and calls `gettext`. + +After enabling dummy `_` function that forwards calls to `gettext` is added +to the environment globals. An internationalized application then has to +provide at least an `gettext` and optionally a `ngettext` function into the +namespace. Either globally or for each rendering. + +Environment Methods +~~~~~~~~~~~~~~~~~~~ + +After enabling of the extension the environment provides the following +additional methods: + +.. method:: jinja2.Environment.install_gettext_translations(translations, newstyle=False) + + Installs a translation globally for that environment. The tranlations + object provided must implement at least `ugettext` and `ungettext`. + The `gettext.NullTranslations` and `gettext.GNUTranslations` classes + as well as `Babel`_\s `Translations` class are supported. + + .. versionchanged:: 2.5 newstyle gettext added + +.. method:: jinja2.Environment.install_null_translations(newstyle=False) + + Install dummy gettext functions. This is useful if you want to prepare + the application for internationalization but don't want to implement the + full internationalization system yet. + + .. versionchanged:: 2.5 newstyle gettext added + +.. method:: jinja2.Environment.install_gettext_callables(gettext, ngettext, newstyle=False) + + Installs the given `gettext` and `ngettext` callables into the + environment as globals. They are supposed to behave exactly like the + standard library's :func:`gettext.ugettext` and + :func:`gettext.ungettext` functions. + + If `newstyle` is activated, the callables are wrapped to work like + newstyle callables. See :ref:`newstyle-gettext` for more information. + + .. versionadded:: 2.5 + +.. method:: jinja2.Environment.uninstall_gettext_translations() + + Uninstall the translations again. + +.. method:: jinja2.Environment.extract_translations(source) + + Extract localizable strings from the given template node or source. + + For every string found this function yields a ``(lineno, function, + message)`` tuple, where: + + * `lineno` is the number of the line on which the string was found, + * `function` is the name of the `gettext` function used (if the + string was extracted from embedded Python code), and + * `message` is the string itself (a `unicode` object, or a tuple + of `unicode` objects for functions with multiple string arguments). + + If `Babel`_ is installed :ref:`the babel integration <babel-integration>` + can be used to extract strings for babel. + +For a web application that is available in multiple languages but gives all +the users the same language (for example a multilingual forum software +installed for a French community) may load the translations once and add the +translation methods to the environment at environment generation time:: + + translations = get_gettext_translations() + env = Environment(extensions=['jinja2.ext.i18n']) + env.install_gettext_translations(translations) + +The `get_gettext_translations` function would return the translator for the +current configuration. (For example by using `gettext.find`) + +The usage of the `i18n` extension for template designers is covered as part +:ref:`of the template documentation <i18n-in-templates>`. + +.. _gettext: http://docs.python.org/dev/library/gettext +.. _Babel: http://babel.edgewall.org/ + +.. _newstyle-gettext: + +Newstyle Gettext +~~~~~~~~~~~~~~~~ + +.. versionadded:: 2.5 + +Starting with version 2.5 you can use newstyle gettext calls. These are +inspired by trac's internal gettext functions and are fully supported by +the babel extraction tool. They might not work as expected by other +extraction tools in case you are not using Babel's. + +What's the big difference between standard and newstyle gettext calls? In +general they are less to type and less error prone. Also if they are used +in an autoescaping environment they better support automatic escaping. +Here some common differences between old and new calls: + +standard gettext: + +.. sourcecode:: html+jinja + + {{ gettext('Hello World!') }} + {{ gettext('Hello %(name)s!')|format(name='World') }} + {{ ngettext('%(num)d apple', '%(num)d apples', apples|count)|format( + num=apples|count + )}} + +newstyle gettext looks like this instead: + +.. sourcecode:: html+jinja + + {{ gettext('Hello World!') }} + {{ gettext('Hello %(name)s!', name='World') }} + {{ ngettext('%(num)d apple', '%(num)d apples', apples|count) }} + +The advantages of newstyle gettext is that you have less to type and that +named placeholders become mandatory. The latter sounds like a +disadvantage but solves a lot of troubles translators are often facing +when they are unable to switch the positions of two placeholder. With +newstyle gettext, all format strings look the same. + +Furthermore with newstyle gettext, string formatting is also used if no +placeholders are used which makes all strings behave exactly the same. +Last but not least are newstyle gettext calls able to properly mark +strings for autoescaping which solves lots of escaping related issues many +templates are experiencing over time when using autoescaping. + +Expression Statement +-------------------- + +**Import name:** `jinja2.ext.do` + +The "do" aka expression-statement extension adds a simple `do` tag to the +template engine that works like a variable expression but ignores the +return value. + +.. _loopcontrols-extension: + +Loop Controls +------------- + +**Import name:** `jinja2.ext.loopcontrols` + +This extension adds support for `break` and `continue` in loops. After +enabling Jinja2 provides those two keywords which work exactly like in +Python. + +.. _with-extension: + +With Statement +-------------- + +**Import name:** `jinja2.ext.with_` + +.. versionadded:: 2.3 + +This extension adds support for the with keyword. Using this keyword it +is possible to enforce a nested scope in a template. Variables can be +declared directly in the opening block of the with statement or using a +standard `set` statement directly within. + +.. _autoescape-extension: + +Autoescape Extension +-------------------- + +**Import name:** `jinja2.ext.autoescape` + +.. versionadded:: 2.4 + +The autoescape extension allows you to toggle the autoescape feature from +within the template. If the environment's :attr:`~Environment.autoescape` +setting is set to `False` it can be activated, if it's `True` it can be +deactivated. The setting overriding is scoped. + + +.. _writing-extensions: + +Writing Extensions +------------------ + +.. module:: jinja2.ext + +By writing extensions you can add custom tags to Jinja2. This is a non trival +task and usually not needed as the default tags and expressions cover all +common use cases. The i18n extension is a good example of why extensions are +useful, another one would be fragment caching. + +When writing extensions you have to keep in mind that you are working with the +Jinja2 template compiler which does not validate the node tree you are possing +to it. If the AST is malformed you will get all kinds of compiler or runtime +errors that are horrible to debug. Always make sure you are using the nodes +you create correctly. The API documentation below shows which nodes exist and +how to use them. + +Example Extension +~~~~~~~~~~~~~~~~~ + +The following example implements a `cache` tag for Jinja2 by using the +`Werkzeug`_ caching contrib module: + +.. literalinclude:: cache_extension.py + :language: python + +And here is how you use it in an environment:: + + from jinja2 import Environment + from werkzeug.contrib.cache import SimpleCache + + env = Environment(extensions=[FragmentCacheExtension]) + env.fragment_cache = SimpleCache() + +Inside the template it's then possible to mark blocks as cacheable. The +following example caches a sidebar for 300 seconds: + +.. sourcecode:: html+jinja + + {% cache 'sidebar', 300 %} + <div class="sidebar"> + ... + </div> + {% endcache %} + +.. _Werkzeug: http://werkzeug.pocoo.org/ + +Extension API +~~~~~~~~~~~~~ + +Extensions always have to extend the :class:`jinja2.ext.Extension` class: + +.. autoclass:: Extension + :members: preprocess, filter_stream, parse, attr, call_method + + .. attribute:: identifier + + The identifier of the extension. This is always the true import name + of the extension class and must not be changed. + + .. attribute:: tags + + If the extension implements custom tags this is a set of tag names + the extension is listening for. + +Parser API +~~~~~~~~~~ + +The parser passed to :meth:`Extension.parse` provides ways to parse +expressions of different types. The following methods may be used by +extensions: + +.. autoclass:: jinja2.parser.Parser + :members: parse_expression, parse_tuple, parse_assign_target, + parse_statements, free_identifier, fail + + .. attribute:: filename + + The filename of the template the parser processes. This is **not** + the load name of the template. For the load name see :attr:`name`. + For templates that were not loaded form the file system this is + `None`. + + .. attribute:: name + + The load name of the template. + + .. attribute:: stream + + The current :class:`~jinja2.lexer.TokenStream` + +.. autoclass:: jinja2.lexer.TokenStream + :members: push, look, eos, skip, next, next_if, skip_if, expect + + .. attribute:: current + + The current :class:`~jinja2.lexer.Token`. + +.. autoclass:: jinja2.lexer.Token + :members: test, test_any + + .. attribute:: lineno + + The line number of the token + + .. attribute:: type + + The type of the token. This string is interned so you may compare + it with arbitrary strings using the `is` operator. + + .. attribute:: value + + The value of the token. + +There is also a utility function in the lexer module that can count newline +characters in strings: + +.. autofunction:: jinja2.lexer.count_newlines + +AST +~~~ + +The AST (Abstract Syntax Tree) is used to represent a template after parsing. +It's build of nodes that the compiler then converts into executable Python +code objects. Extensions that provide custom statements can return nodes to +execute custom Python code. + +The list below describes all nodes that are currently available. The AST may +change between Jinja2 versions but will stay backwards compatible. + +For more information have a look at the repr of :meth:`jinja2.Environment.parse`. + +.. module:: jinja2.nodes + +.. jinjanodes:: + +.. autoexception:: Impossible diff --git a/lib/jinja2/docs/faq.rst b/lib/jinja2/docs/faq.rst new file mode 100644 --- /dev/null +++ b/lib/jinja2/docs/faq.rst @@ -0,0 +1,203 @@ +Frequently Asked Questions +========================== + +This page answers some of the often asked questions about Jinja. + +.. highlight:: html+jinja + +Why is it called Jinja? +----------------------- + +The name Jinja was chosen because it's the name of a Japanese temple and +temple and template share a similar pronunciation. It is not named after +the city in Uganda. + +How fast is it? +--------------- + +We really hate benchmarks especially since they don't reflect much. The +performance of a template depends on many factors and you would have to +benchmark different engines in different situations. The benchmarks from the +testsuite show that Jinja2 has a similar performance to `Mako`_ and is between +10 and 20 times faster than Django's template engine or Genshi. These numbers +should be taken with tons of salt as the benchmarks that took these numbers +only test a few performance related situations such as looping. Generally +speaking the performance of a template engine doesn't matter much as the +usual bottleneck in a web application is either the database or the application +code. + +.. _Mako: http://www.makotemplates.org/ + +How Compatible is Jinja2 with Django? +------------------------------------- + +The default syntax of Jinja2 matches Django syntax in many ways. However +this similarity doesn't mean that you can use a Django template unmodified +in Jinja2. For example filter arguments use a function call syntax rather +than a colon to separate filter name and arguments. Additionally the +extension interface in Jinja is fundamentally different from the Django one +which means that your custom tags won't work any longer. + +Generally speaking you will use much less custom extensions as the Jinja +template system allows you to use a certain subset of Python expressions +which can replace most Django extensions. For example instead of using +something like this:: + + {% load comments %} + {% get_latest_comments 10 as latest_comments %} + {% for comment in latest_comments %} + ... + {% endfor %} + +You will most likely provide an object with attributes to retrieve +comments from the database:: + + {% for comment in models.comments.latest(10) %} + ... + {% endfor %} + +Or directly provide the model for quick testing:: + + {% for comment in Comment.objects.order_by('-pub_date')[:10] %} + ... + {% endfor %} + +Please keep in mind that even though you may put such things into templates +it still isn't a good idea. Queries should go into the view code and not +the template! + +Isn't it a terrible idea to put Logic into Templates? +----------------------------------------------------- + +Without a doubt you should try to remove as much logic from templates as +possible. But templates without any logic mean that you have to do all +the processing in the code which is boring and stupid. A template engine +that does that is shipped with Python and called `string.Template`. Comes +without loops and if conditions and is by far the fastest template engine +you can get for Python. + +So some amount of logic is required in templates to keep everyone happy. +And Jinja leaves it pretty much to you how much logic you want to put into +templates. There are some restrictions in what you can do and what not. + +Jinja2 neither allows you to put arbitrary Python code into templates nor +does it allow all Python expressions. The operators are limited to the +most common ones and more advanced expressions such as list comprehensions +and generator expressions are not supported. This keeps the template engine +easier to maintain and templates more readable. + +Why is Autoescaping not the Default? +------------------------------------ + +There are multiple reasons why automatic escaping is not the default mode +and also not the recommended one. While automatic escaping of variables +means that you will less likely have an XSS problem it also causes a huge +amount of extra processing in the template engine which can cause serious +performance problems. As Python doesn't provide a way to mark strings as +unsafe Jinja has to hack around that limitation by providing a custom +string class (the :class:`Markup` string) that safely interacts with safe +and unsafe strings. + +With explicit escaping however the template engine doesn't have to perform +any safety checks on variables. Also a human knows not to escape integers +or strings that may never contain characters one has to escape or already +HTML markup. For example when iterating over a list over a table of +integers and floats for a table of statistics the template designer can +omit the escaping because he knows that integers or floats don't contain +any unsafe parameters. + +Additionally Jinja2 is a general purpose template engine and not only used +for HTML/XML generation. For example you may generate LaTeX, emails, +CSS, JavaScript, or configuration files. + +Why is the Context immutable? +----------------------------- + +When writing a :func:`contextfunction` or something similar you may have +noticed that the context tries to stop you from modifying it. If you have +managed to modify the context by using an internal context API you may +have noticed that changes in the context don't seem to be visible in the +template. The reason for this is that Jinja uses the context only as +primary data source for template variables for performance reasons. + +If you want to modify the context write a function that returns a variable +instead that one can assign to a variable by using set:: + + {% set comments = get_latest_comments() %} + +What is the speedups module and why is it missing? +-------------------------------------------------- + +To achieve a good performance with automatic escaping enabled, the escaping +function was also implemented in pure C in older Jinja2 releases and used if +Jinja2 was installed with the speedups module. + +Because this feature itself is very useful for non-template engines as +well it was moved into a separate project on PyPI called `MarkupSafe`_. + +Jinja2 no longer ships with a C implementation of it but only the pure +Python implementation. It will however check if MarkupSafe is available +and installed, and if it is, use the Markup class from MarkupSafe. + +So if you want the speedups, just import MarkupSafe. + +.. _MarkupSafe: http://pypi.python.org/pypi/MarkupSafe + +My tracebacks look weird. What's happening? +-------------------------------------------- + +If the debugsupport module is not compiled and you are using a Python +installation without ctypes (Python 2.4 without ctypes, Jython or Google's +AppEngine) Jinja2 is unable to provide correct debugging information and +the traceback may be incomplete. There is currently no good workaround +for Jython or the AppEngine as ctypes is unavailable there and it's not +possible to use the debugsupport extension. + +If you are working in the Google Appengine development server you can +whitelist the ctypes module to restore the tracebacks. This however won't +work in production environments:: + + import os + if os.environ.get('SERVER_SOFTWARE', '').startswith('Dev'): + from google.appengine.tools.dev_appserver import HardenedModulesHook + HardenedModulesHook._WHITE_LIST_C_MODULES += ['_ctypes', 'gestalt'] + +Credit for this snippet goes to `Thomas Johansson +<http://stackoverflow.com/questions/3086091/debug-jinja2-in-google-app-engine/3694434#3694434>`_ + +Why is there no Python 2.3 support? +----------------------------------- + +Python 2.3 is missing a lot of features that are used heavily in Jinja2. This +decision was made as with the upcoming Python 2.6 and 3.0 versions it becomes +harder to maintain the code for older Python versions. If you really need +Python 2.3 support you either have to use `Jinja 1`_ or other templating +engines that still support 2.3. + +My Macros are overriden by something +------------------------------------ + +In some situations the Jinja scoping appears arbitrary: + +layout.tmpl: + +.. sourcecode:: jinja + + {% macro foo() %}LAYOUT{% endmacro %} + {% block body %}{% endblock %} + +child.tmpl: + +.. sourcecode:: jinja + + {% extends 'layout.tmpl' %} + {% macro foo() %}CHILD{% endmacro %} + {% block body %}{{ foo() }}{% endblock %} + +This will print ``LAYOUT`` in Jinja2. This is a side effect of having +the parent template evaluated after the child one. This allows child +templates passing information to the parent template. To avoid this +issue rename the macro or variable in the parent template to have an +uncommon prefix. + +.. _Jinja 1: http://jinja.pocoo.org/1/ diff --git a/lib/jinja2/docs/index.rst b/lib/jinja2/docs/index.rst new file mode 100644 --- /dev/null +++ b/lib/jinja2/docs/index.rst @@ -0,0 +1,34 @@ +Welcome to Jinja2 +================= + +Jinja2 is a modern and designer friendly templating language for Python, +modelled after Django's templates. It is fast, widely used and secure +with the optional sandboxed template execution environment: + +.. sourcecode:: html+jinja + + <title>{% block title %}{% endblock %}</title> + <ul> + {% for user in users %} + <li><a href="{{ user.url }}">{{ user.username }}</a></li> + {% endfor %} + </ul> + +**Features:** + +- sandboxed execution +- powerful automatic HTML escaping system for XSS prevention +- template inheritance +- compiles down to the optimal python code just in time +- optional ahead of time template compilation +- easy to debug. Line numbers of exceptions directly point to + the correct line in the template. +- configurable syntax + +.. include:: contents.rst.inc + +If you can't find the information you're looking for, have a look at the +index or try to find it using the search function: + +* :ref:`genindex` +* :ref:`search` diff --git a/lib/jinja2/docs/integration.rst b/lib/jinja2/docs/integration.rst new file mode 100644 --- /dev/null +++ b/lib/jinja2/docs/integration.rst @@ -0,0 +1,97 @@ +Integration +=========== + +Jinja2 provides some code for integration into other tools such as frameworks, +the `Babel`_ library or your favourite editor for fancy code highlighting. +This is a brief description of whats included. + +.. _babel-integration: + +Babel Integration +----------------- + +Jinja provides support for extracting gettext messages from templates via a +`Babel`_ extractor entry point called `jinja2.ext.babel_extract`. The Babel +support is implemented as part of the :ref:`i18n-extension` extension. + +Gettext messages extracted from both `trans` tags and code expressions. + +To extract gettext messages from templates, the project needs a Jinja2 section +in its Babel extraction method `mapping file`_: + +.. sourcecode:: ini + + [jinja2: **/templates/**.html] + encoding = utf-8 + +The syntax related options of the :class:`Environment` are also available as +configuration values in the mapping file. For example to tell the extraction +that templates use ``%`` as `line_statement_prefix` you can use this code: + +.. sourcecode:: ini + + [jinja2: **/templates/**.html] + encoding = utf-8 + line_statement_prefix = % + +:ref:`jinja-extensions` may also be defined by passing a comma separated list +of import paths as `extensions` value. The i18n extension is added +automatically. + +.. versionchanged:: 2.7 + + Until 2.7 template syntax errors were always ignored. This was done + since many people are dropping non template html files into the + templates folder and it would randomly fail. The assumption was that + testsuites will catch syntax errors in templates anyways. If you don't + want that behavior you can add ``silent=false`` to the settings and + exceptions are propagated. + +.. _mapping file: http://babel.edgewall.org/wiki/Documentation/messages.html#extraction-method-mapping-and-configuration + +Pylons +------ + +With `Pylons`_ 0.9.7 onwards it's incredible easy to integrate Jinja into a +Pylons powered application. + +The template engine is configured in `config/environment.py`. The configuration +for Jinja2 looks something like that:: + + from jinja2 import Environment, PackageLoader + config['pylons.app_globals'].jinja_env = Environment( + loader=PackageLoader('yourapplication', 'templates') + ) + +After that you can render Jinja templates by using the `render_jinja` function +from the `pylons.templating` module. + +Additionally it's a good idea to set the Pylons' `c` object into strict mode. +Per default any attribute to not existing attributes on the `c` object return +an empty string and not an undefined object. To change this just use this +snippet and add it into your `config/environment.py`:: + + config['pylons.strict_c'] = True + +.. _Pylons: http://www.pylonshq.com/ + +TextMate +-------- + +Inside the `ext` folder of Jinja2 there is a bundle for TextMate that supports +syntax highlighting for Jinja1 and Jinja2 for text based templates as well as +HTML. It also contains a few often used snippets. + +Vim +--- + +A syntax plugin for `Vim`_ exists in the Vim-scripts directory as well as the +ext folder of Jinja2. `The script <http://www.vim.org/scripts/script.php?script_id=1856>`_ +supports Jinja1 and Jinja2. Once installed two file types are available `jinja` +and `htmljinja`. The first one for text based templates, the latter for HTML +templates. + +Copy the files into your `syntax` folder. + +.. _Babel: http://babel.edgewall.org/ +.. _Vim: http://www.vim.org/ diff --git a/lib/jinja2/docs/intro.rst b/lib/jinja2/docs/intro.rst new file mode 100644 --- /dev/null +++ b/lib/jinja2/docs/intro.rst @@ -0,0 +1,149 @@ +Introduction +============ + +This is the documentation for the Jinja2 general purpose templating language. +Jinja2 is a library for Python 2.4 and onwards that is designed to be flexible, +fast and secure. + +If you have any exposure to other text-based template languages, such as Smarty or +Django, you should feel right at home with Jinja2. It's both designer and +developer friendly by sticking to Python's principles and adding functionality +useful for templating environments. + +Prerequisites +------------- + +Jinja2 needs at least **Python 2.4** to run. Additionally a working C-compiler +that can create python extensions should be installed for the debugger if you +are using Python 2.4. + +If you don't have a working C-compiler and you are trying to install the source +release with the debugsupport you will get a compiler error. + +.. _ctypes: http://python.net/crew/theller/ctypes/ + + +Installation +------------ + +You have multiple ways to install Jinja2. If you are unsure what to do, go +with the Python egg or tarball. + +As a Python egg (via easy_install) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You can install the most recent Jinja2 version using `easy_install`_ or `pip`_:: + + easy_install Jinja2 + pip install Jinja2 + +This will install a Jinja2 egg in your Python installation's site-packages +directory. + +(If you are installing from the windows command line omit the `sudo` and make +sure to run the command as user with administrator rights) + +From the tarball release +~~~~~~~~~~~~~~~~~~~~~~~~~ + +1. Download the most recent tarball from the `download page`_ +2. Unpack the tarball +3. ``sudo python setup.py install`` + +Note that you either have to have setuptools or `distribute`_ installed, +the latter is preferred. + +This will install Jinja2 into your Python installation's site-packages directory. + +.. _distribute: http://pypi.python.org/pypi/distribute + +Installing the development version +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +1. Install `git`_ +2. ``git clone git://github.com/mitsuhiko/jinja2.git`` +3. ``cd jinja2`` +4. ``ln -s jinja2 /usr/lib/python2.X/site-packages`` + +As an alternative to steps 4 you can also do ``python setup.py develop`` +which will install the package via distribute in development mode. This also +has the advantage that the C extensions are compiled. + +.. _download page: http://pypi.python.org/pypi/Jinja2 +.. _setuptools: http://peak.telecommunity.com/DevCenter/setuptools +.. _easy_install: http://peak.telecommunity.com/DevCenter/EasyInstall +.. _pip: http://pypi.python.org/pypi/pip +.. _git: http://git-scm.org/ + + +More Speed with MarkupSafe +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +As of version 2.5.1 Jinja2 will check for an installed `MarkupSafe`_ +module. If it can find it, it will use the Markup class of that module +instead of the one that comes with Jinja2. `MarkupSafe` replaces the +older speedups module that came with Jinja2 and has the advantage that is +has a better setup script and will automatically attempt to install the C +version and nicely fall back to a pure Python implementation if that is +not possible. + +The C implementation of MarkupSafe is much faster and recommended when +using Jinja2 with autoescaping. + +.. _MarkupSafe: http://pypi.python.org/pypi/MarkupSafe + + +Enable the debug support Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +By default Jinja2 will not compile the debug support module. Enabling this +will fail if you don't have the Python headers or a working compiler. This +is often the case if you are installing Jinja2 from a windows machine. + +Because the debug support is only necessary for Python 2.4 you will not +have to do this unless you run 2.4:: + + sudo python setup.py --with-debugsupport install + + +Basic API Usage +--------------- + +This section gives you a brief introduction to the Python API for Jinja2 +templates. + +The most basic way to create a template and render it is through +:class:`~jinja2.Template`. This however is not the recommended way to +work with it if your templates are not loaded from strings but the file +system or another data source: + +>>> from jinja2 import Template +>>> template = Template('Hello {{ name }}!') +>>> template.render(name='John Doe') +u'Hello John Doe!' + +By creating an instance of :class:`~jinja2.Template` you get back a new template +object that provides a method called :meth:`~jinja2.Template.render` which when +called with a dict or keyword arguments expands the template. The dict +or keywords arguments passed to the template are the so-called "context" +of the template. + +What you can see here is that Jinja2 is using unicode internally and the +return value is an unicode string. So make sure that your application is +indeed using unicode internally. + + +Experimental Python 3 Support +----------------------------- + +Jinja 2.3 brings experimental support for Python 3. It means that all +unittests pass on the new version, but there might still be small bugs in +there and behavior might be inconsistent. If you notice any bugs, please +provide feedback in the `Jinja bug tracker`_. + +Also please keep in mind that the documentation is written with Python 2 +in mind, you will have to adapt the shown code examples to Python 3 syntax +for yourself. + + +.. _Jinja bug tracker: http://github.com/mitsuhiko/jinja2/issues diff --git a/lib/jinja2/docs/jinjaext.py b/lib/jinja2/docs/jinjaext.py new file mode 100644 --- /dev/null +++ b/lib/jinja2/docs/jinjaext.py @@ -0,0 +1,193 @@ +# -*- coding: utf-8 -*- +""" + Jinja Documentation Extensions + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Support for automatically documenting filters and tests. + + :copyright: Copyright 2008 by Armin Ronacher. + :license: BSD. +""" +import os +import re +import inspect +import jinja2 +from itertools import islice +from types import BuiltinFunctionType +from docutils import nodes +from docutils.statemachine import ViewList +from sphinx.ext.autodoc import prepare_docstring +from sphinx.application import TemplateBridge +from pygments.style import Style +from pygments.token import Keyword, Name, Comment, String, Error, \ + Number, Operator, Generic +from jinja2 import Environment, FileSystemLoader + + +def parse_rst(state, content_offset, doc): + node = nodes.section() + # hack around title style bookkeeping + surrounding_title_styles = state.memo.title_styles + surrounding_section_level = state.memo.section_level + state.memo.title_styles = [] + state.memo.section_level = 0 + state.nested_parse(doc, content_offset, node, match_titles=1) + state.memo.title_styles = surrounding_title_styles + state.memo.section_level = surrounding_section_level + return node.children + + +class JinjaStyle(Style): + title = 'Jinja Style' + default_style = "" + styles = { + Comment: 'italic #aaaaaa', + Comment.Preproc: 'noitalic #B11414', + Comment.Special: 'italic #505050', + + Keyword: 'bold #B80000', + Keyword.Type: '#808080', + + Operator.Word: 'bold #B80000', + + Name.Builtin: '#333333', + Name.Function: '#333333', + Name.Class: 'bold #333333', + Name.Namespace: 'bold #333333', + Name.Entity: 'bold #363636', + Name.Attribute: '#686868', + Name.Tag: 'bold #686868', + Name.Decorator: '#686868', + + String: '#AA891C', + Number: '#444444', + + Generic.Heading: 'bold #000080', + Generic.Subheading: 'bold #800080', + Generic.Deleted: '#aa0000', + Generic.Inserted: '#00aa00', + Generic.Error: '#aa0000', + Generic.Emph: 'italic', + Generic.Strong: 'bold', + Generic.Prompt: '#555555', + Generic.Output: '#888888', + Generic.Traceback: '#aa0000', + + Error: '#F00 bg:#FAA' + } + + +_sig_re = re.compile(r'^[a-zA-Z_][a-zA-Z0-9_]*($.*?$)') + + +def format_function(name, aliases, func): + lines = inspect.getdoc(func).splitlines() + signature = '()' + if isinstance(func, BuiltinFunctionType): + match = _sig_re.match(lines[0]) + if match is not None: + del lines[:1 + bool(lines and not lines[0])] + signature = match.group(1) + else: + try: + argspec = inspect.getargspec(func) + if getattr(func, 'environmentfilter', False) or \ + getattr(func, 'contextfilter', False) or \ + getattr(func, 'evalcontextfilter', False): + del argspec[0][0] + signature = inspect.formatargspec(*argspec) + except: + pass + result = ['.. function:: %s%s' % (name, signature), ''] + result.extend(' ' + line for line in lines) + if aliases: + result.extend(('', ' :aliases: %s' % ', '.join( + '``%s``' % x for x in sorted(aliases)))) + return result + + +def dump_functions(mapping): + def directive(dirname, arguments, options, content, lineno, + content_offset, block_text, state, state_machine): + reverse_mapping = {} + for name, func in mapping.iteritems(): + reverse_mapping.setdefault(func, []).append(name) + filters = [] + for func, names in reverse_mapping.iteritems(): + aliases = sorted(names, key=lambda x: len(x)) + name = aliases.pop() + filters.append((name, aliases, func)) + filters.sort() + + result = ViewList() + for name, aliases, func in filters: + for item in format_function(name, aliases, func): + result.append(item, '<jinjaext>') + + node = nodes.paragraph() + state.nested_parse(result, content_offset, node) + return node.children + return directive + + +from jinja2.defaults import DEFAULT_FILTERS, DEFAULT_TESTS +jinja_filters = dump_functions(DEFAULT_FILTERS) +jinja_tests = dump_functions(DEFAULT_TESTS) + + +def jinja_nodes(dirname, arguments, options, content, lineno, + content_offset, block_text, state, state_machine): + from jinja2.nodes import Node + doc = ViewList() + def walk(node, indent): + p = ' ' * indent + sig = ', '.join(node.fields) + doc.append(p + '.. autoclass:: %s(%s)' % (node.__name__, sig), '') + if node.abstract: + members = [] + for key, name in node.__dict__.iteritems(): + if not key.startswith('_') and \ + not hasattr(node.__base__, key) and callable(name): + members.append(key) + if members: + members.sort() + doc.append('%s :members: %s' % (p, ', '.join(members)), '') + if node.__base__ != object: + doc.append('', '') + doc.append('%s :Node type: :class:`%s`' % + (p, node.__base__.__name__), '') + doc.append('', '') + children = node.__subclasses__() + children.sort(key=lambda x: x.__name__.lower()) + for child in children: + walk(child, indent) + walk(Node, 0) + return parse_rst(state, content_offset, doc) + + +def inject_toc(app, doctree, docname): + titleiter = iter(doctree.traverse(nodes.title)) + try: + # skip first title, we are not interested in that one + titleiter.next() + title = titleiter.next() + # and check if there is at least another title + titleiter.next() + except StopIteration: + return + tocnode = nodes.section('') + tocnode['classes'].append('toc') + toctitle = nodes.section('') + toctitle['classes'].append('toctitle') + toctitle.append(nodes.title(text='Table Of Contents')) + tocnode.append(toctitle) + tocnode += doctree.document.settings.env.get_toc_for(docname)[0][1] + title.parent.insert(title.parent.children.index(title), tocnode) + + +def setup(app): + app.add_directive('jinjafilters', jinja_filters, 0, (0, 0, 0)) + app.add_directive('jinjatests', jinja_tests, 0, (0, 0, 0)) + app.add_directive('jinjanodes', jinja_nodes, 0, (0, 0, 0)) + # uncomment for inline toc. links are broken unfortunately + ##app.connect('doctree-resolved', inject_toc) diff --git a/lib/jinja2/docs/jinjastyle.sty b/lib/jinja2/docs/jinjastyle.sty new file mode 100644 --- /dev/null +++ b/lib/jinja2/docs/jinjastyle.sty @@ -0,0 +1,119 @@ +\definecolor{TitleColor}{rgb}{0,0,0} +\definecolor{InnerLinkColor}{rgb}{0,0,0} +\definecolor{OuterLinkColor}{rgb}{0.8,0,0} + +\renewcommand{\maketitle}{% + \begin{titlepage}% + \let\footnotesize\small + \let\footnoterule\relax + \ifsphinxpdfoutput + \begingroup + % This \def is required to deal with multi-line authors; it + % changes \\ to ', ' (comma-space), making it pass muster for + % generating document info in the PDF file. + \def\\{, } + \pdfinfo{ + /Author (\@author) + /Title (\@title) + } + \endgroup + \fi + \begin{flushright}% + %\sphinxlogo% + {\center + \vspace*{3cm} + \includegraphics{logo.pdf} + \vspace{3cm} + \par + {\rm\Huge \@title \par}% + {\em\LARGE \py at release\releaseinfo \par} + {\large + \@date \par + \py at authoraddress \par + }}% + \end{flushright}%\par + \@thanks + \end{titlepage}% + \cleardoublepage% + \setcounter{footnote}{0}% + \let\thanks\relax\let\maketitle\relax + %\gdef\@thanks{}\gdef\@author{}\gdef\@title{} +} + +\fancypagestyle{normal}{ + \fancyhf{} + \fancyfoot[LE,RO]{{\thepage}} + \fancyfoot[LO]{{\nouppercase{\rightmark}}} + \fancyfoot[RE]{{\nouppercase{\leftmark}}} + \fancyhead[LE,RO]{{ \@title, \py at release}} + \renewcommand{\headrulewidth}{0.4pt} + \renewcommand{\footrulewidth}{0.4pt} +} + +\fancypagestyle{plain}{ + \fancyhf{} + \fancyfoot[LE,RO]{{\thepage}} + \renewcommand{\headrulewidth}{0pt} + \renewcommand{\footrulewidth}{0.4pt} +} + +\titleformat{\section}{\Large}% + {\py at TitleColor\thesection}{0.5em}{\py at TitleColor}{\py at NormalColor} +\titleformat{\subsection}{\large}% + {\py at TitleColor\thesubsection}{0.5em}{\py at TitleColor}{\py at NormalColor} +\titleformat{\subsubsection}{}% + {\py at TitleColor\thesubsubsection}{0.5em}{\py at TitleColor}{\py at NormalColor} +\titleformat{\paragraph}{\large}% + {\py at TitleColor}{0em}{\py at TitleColor}{\py at NormalColor} + +\ChNameVar{\raggedleft\normalsize} +\ChNumVar{\raggedleft \bfseries\Large} +\ChTitleVar{\raggedleft \rm\Huge} + +\renewcommand\thepart{\@Roman\c at part} +\renewcommand\part{% + \pagestyle{plain} + \if at noskipsec \leavevmode \fi + \cleardoublepage + \vspace*{6cm}% + \@afterindentfalse + \secdef\@part\@spart} + +\def\@part[#1]#2{% + \ifnum \c at secnumdepth >\m at ne + \refstepcounter{part}% + \addcontentsline{toc}{part}{\thepart\hspace{1em}#1}% + \else + \addcontentsline{toc}{part}{#1}% + \fi + {\parindent \z@ %\center + \interlinepenalty \@M + \normalfont + \ifnum \c at secnumdepth >\m at ne + \rm\Large \partname~\thepart + \par\nobreak + \fi + \MakeUppercase{\rm\Huge #2}% + \markboth{}{}\par}% + \nobreak + \vskip 8ex + \@afterheading} +\def\@spart#1{% + {\parindent \z@ %\center + \interlinepenalty \@M + \normalfont + \huge \bfseries #1\par}% + \nobreak + \vskip 3ex + \@afterheading} + +% use inconsolata font +\usepackage{inconsolata} + +% fix single quotes, for inconsolata. (does not work) +%%\usepackage{textcomp} +%%\begingroup +%% \catcode`'=\active +%% \g at addto@macro\@noligs{\let'\textsinglequote} +%% \endgroup +%%\endinput diff --git a/lib/jinja2/docs/latexindex.rst b/lib/jinja2/docs/latexindex.rst new file mode 100644 --- /dev/null +++ b/lib/jinja2/docs/latexindex.rst @@ -0,0 +1,6 @@ +:orphan: + +Jinja2 Documentation +==================== + +.. include:: contents.rst.inc diff --git a/lib/jinja2/docs/logo.pdf b/lib/jinja2/docs/logo.pdf new file mode 100644 index 0000000000000000000000000000000000000000..3e8a9cf480ccbe4a30911bdb7d78c0b80e767c81 GIT binary patch [cut] diff --git a/lib/jinja2/docs/sandbox.rst b/lib/jinja2/docs/sandbox.rst new file mode 100644 --- /dev/null +++ b/lib/jinja2/docs/sandbox.rst @@ -0,0 +1,94 @@ +Sandbox +======= + +The Jinja2 sandbox can be used to evaluate untrusted code. Access to unsafe +attributes and methods is prohibited. + +Assuming `env` is a :class:`SandboxedEnvironment` in the default configuration +the following piece of code shows how it works: + +>>> env.from_string("{{ func.func_code }}").render(func=lambda:None) +u'' +>>> env.from_string("{{ func.func_code.do_something }}").render(func=lambda:None) +Traceback (most recent call last): + ... +SecurityError: access to attribute 'func_code' of 'function' object is unsafe. + +API +--- + +.. module:: jinja2.sandbox + +.. autoclass:: SandboxedEnvironment([options]) + :members: is_safe_attribute, is_safe_callable, default_binop_table, + default_unop_table, intercepted_binops, intercepted_unops, + call_binop, call_unop + +.. autoclass:: ImmutableSandboxedEnvironment([options]) + +.. autoexception:: SecurityError + +.. autofunction:: unsafe + +.. autofunction:: is_internal_attribute + +.. autofunction:: modifies_known_mutable + +.. admonition:: Note + + The Jinja2 sandbox alone is no solution for perfect security. Especially + for web applications you have to keep in mind that users may create + templates with arbitrary HTML in so it's crucial to ensure that (if you + are running multiple users on the same server) they can't harm each other + via JavaScript insertions and much more. + + Also the sandbox is only as good as the configuration. We stronly + recommend only passing non-shared resources to the template and use + some sort of whitelisting for attributes. + + Also keep in mind that templates may raise runtime or compile time errors, + so make sure to catch them. + +Operator Intercepting +--------------------- + +.. versionadded:: 2.6 + +For maximum performace Jinja2 will let operators call directly the type +specific callback methods. This means that it's not possible to have this +intercepted by overriding :meth:`Environment.call`. Furthermore a +conversion from operator to special method is not always directly possible +due to how operators work. For instance for divisions more than one +special method exist. + +With Jinja 2.6 there is now support for explicit operator intercepting. +This can be used to customize specific operators as necessary. In order +to intercept an operator one has to override the +:attr:`SandboxedEnvironment.intercepted_binops` attribute. Once the +operator that needs to be intercepted is added to that set Jinja2 will +generate bytecode that calls the :meth:`SandboxedEnvironment.call_binop` +function. For unary operators the `unary` attributes and methods have to +be used instead. + +The default implementation of :attr:`SandboxedEnvironment.call_binop` +will use the :attr:`SandboxedEnvironment.binop_table` to translate +operator symbols into callbacks performing the default operator behavior. + +This example shows how the power (``**``) operator can be disabled in +Jinja2:: + + from jinja2.sandbox import SandboxedEnvironment + + + class MyEnvironment(SandboxedEnvironment): + intercepted_binops = frozenset(['**']) + + def call_binop(self, context, operator, left, right): + if operator == '**': + return self.undefined('the power operator is unavailable') + return SandboxedEnvironment.call_binop(self, context, + operator, left, right) + +Make sure to always call into the super method, even if you are not +intercepting the call. Jinja2 might internally call the method to +evaluate expressions. diff --git a/lib/jinja2/docs/switching.rst b/lib/jinja2/docs/switching.rst new file mode 100644 --- /dev/null +++ b/lib/jinja2/docs/switching.rst @@ -0,0 +1,242 @@ +Switching from other Template Engines +===================================== + +.. highlight:: html+jinja + +If you have used a different template engine in the past and want to swtich +to Jinja2 here is a small guide that shows the basic syntatic and semantic +changes between some common, similar text template engines for Python. + +Jinja1 +------ + +Jinja2 is mostly compatible with Jinja1 in terms of API usage and template +syntax. The differences between Jinja1 and 2 are explained in the following +list. + +API +~~~ + +Loaders + Jinja2 uses a different loader API. Because the internal representation + of templates changed there is no longer support for external caching + systems such as memcached. The memory consumed by templates is comparable + with regular Python modules now and external caching doesn't give any + advantage. If you have used a custom loader in the past have a look at + the new :ref:`loader API <loaders>`. + +Loading templates from strings + In the past it was possible to generate templates from a string with the + default environment configuration by using `jinja.from_string`. Jinja2 + provides a :class:`Template` class that can be used to do the same, but + with optional additional configuration. + +Automatic unicode conversion + Jinja1 performed automatic conversion of bytestrings in a given encoding + into unicode objects. This conversion is no longer implemented as it + was inconsistent as most libraries are using the regular Python ASCII + bytestring to Unicode conversion. An application powered by Jinja2 + *has to* use unicode internally everywhere or make sure that Jinja2 only + gets unicode strings passed. + +i18n + Jinja1 used custom translators for internationalization. i18n is now + available as Jinja2 extension and uses a simpler, more gettext friendly + interface and has support for babel. For more details see + :ref:`i18n-extension`. + +Internal methods + Jinja1 exposed a few internal methods on the environment object such + as `call_function`, `get_attribute` and others. While they were marked + as being an internal method it was possible to override them. Jinja2 + doesn't have equivalent methods. + +Sandbox + Jinja1 was running sandbox mode by default. Few applications actually + used that feature so it became optional in Jinja2. For more details + about the sandboxed execution see :class:`SandboxedEnvironment`. + +Context + Jinja1 had a stacked context as storage for variables passed to the + environment. In Jinja2 a similar object exists but it doesn't allow + modifications nor is it a singleton. As inheritance is dynamic now + multiple context objects may exist during template evaluation. + +Filters and Tests + Filters and tests are regular functions now. It's no longer necessary + and allowed to use factory functions. + + +Templates +~~~~~~~~~ + +Jinja2 has mostly the same syntax as Jinja1. What's different is that +macros require parentheses around the argument list now. + +Additionally Jinja2 allows dynamic inheritance now and dynamic includes. +The old helper function `rendertemplate` is gone now, `include` can be used +instead. Includes no longer import macros and variable assignments, for +that the new `import` tag is used. This concept is explained in the +:ref:`import` documentation. + +Another small change happened in the `for`-tag. The special loop variable +doesn't have a `parent` attribute, instead you have to alias the loop +yourself. See :ref:`accessing-the-parent-loop` for more details. + + +Django +------ + +If you have previously worked with Django templates, you should find +Jinja2 very familiar. In fact, most of the syntax elements look and +work the same. + +However, Jinja2 provides some more syntax elements covered in the +documentation and some work a bit different. + +This section covers the template changes. As the API is fundamentally +different we won't cover it here. + +Method Calls +~~~~~~~~~~~~ + +In Django method calls work implicitly. With Jinja2 you have to specify that +you want to call an object. Thus this Django code:: + + {% for page in user.get_created_pages %} + ... + {% endfor %} + +will look like this in Jinja:: + + {% for page in user.get_created_pages() %} + ... + {% endfor %} + +This allows you to pass variables to the function which is also used for macros +which is not possible in Django. + +Conditions +~~~~~~~~~~ + +In Django you can use the following constructs to check for equality:: + + {% ifequal foo "bar" %} + ... + {% else %} + ... + {% endifequal %} + +In Jinja2 you can use the normal if statement in combination with operators:: + + {% if foo == 'bar' %} + ... + {% else %} + ... + {% endif %} + +You can also have multiple elif branches in your template:: + + {% if something %} + ... + {% elif otherthing %} + ... + {% elif foothing %} + ... + {% else %} + ... + {% endif %} + +Filter Arguments +~~~~~~~~~~~~~~~~ + +Jinja2 provides more than one argument for filters. Also the syntax for +argument passing is different. A template that looks like this in Django:: + + {{ items|join:", " }} + +looks like this in Jinja2:: + + {{ items|join(', ') }} + +In fact it's a bit more verbose but it allows different types of arguments - +including variables - and more than one of them. + +Tests +~~~~~ + +In addition to filters there also are tests you can perform using the is +operator. Here are some examples:: + + {% if user.user_id is odd %} + {{ user.username|e }} is odd + {% else %} + hmm. {{ user.username|e }} looks pretty normal + {% endif %} + +Loops +~~~~~ + +For loops work very similar to Django, the only incompatibility is that in +Jinja2 the special variable for the loop context is called `loop` and not +`forloop` like in Django. + +Cycle +~~~~~ + +The ``{% cycle %}`` tag does not exist in Jinja because of it's implicit +nature. However you can achieve mostly the same by using the `cycle` +method on a loop object. + +The following Django template:: + + {% for user in users %} + <li class="{% cycle 'odd' 'even' %}">{{ user }}</li> + {% endfor %} + +Would look like this in Jinja:: + + {% for user in users %} + <li class="{{ loop.cycle('odd', 'even') }}">{{ user }}</li> + {% endfor %} + +There is no equivalent of ``{% cycle ... as variable %}``. + + +Mako +---- + +.. highlight:: html+mako + +If you have used Mako so far and want to switch to Jinja2 you can configure +Jinja2 to look more like Mako: + +.. sourcecode:: python + + env = Environment('<%', '%>', '${', '}', '%') + +Once the environment is configure like that Jinja2 should be able to interpret +a small subset of Mako templates. Jinja2 does not support embedded Python code +so you would have to move that out of the template. The syntax for defs (in +Jinja2 defs are called macros) and template inheritance is different too. The +following Mako template:: + + <%inherit file="layout.html" /> + <%def name="title()">Page Title</%def> + <ul> + % for item in list: + <li>${item}</li> + % endfor + </ul> + +Looks like this in Jinja2 with the above configuration:: + + <% extends "layout.html" %> + <% block title %>Page Title<% endblock %> + <% block body %> + <ul> + % for item in list: + <li>${item}</li> + % endfor + </ul> + <% endblock %> diff --git a/lib/jinja2/docs/templates.rst b/lib/jinja2/docs/templates.rst new file mode 100644 --- /dev/null +++ b/lib/jinja2/docs/templates.rst @@ -0,0 +1,1380 @@ +Template Designer Documentation +=============================== + +.. highlight:: html+jinja + +This document describes the syntax and semantics of the template engine and +will be most useful as reference to those creating Jinja templates. As the +template engine is very flexible the configuration from the application might +be slightly different from here in terms of delimiters and behavior of +undefined values. + + +Synopsis +-------- + +A template is simply a text file. It can generate any text-based format +(HTML, XML, CSV, LaTeX, etc.). It doesn't have a specific extension, +``.html`` or ``.xml`` are just fine. + +A template contains **variables** or **expressions**, which get replaced with +values when the template is evaluated, and tags, which control the logic of +the template. The template syntax is heavily inspired by Django and Python. + +Below is a minimal template that illustrates a few basics. We will cover +the details later in that document:: + + <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN"> + <html lang="en"> + <head> + <title>My Webpage</title> + </head> + <body> + <ul id="navigation"> + {% for item in navigation %} + <li><a href="{{ item.href }}">{{ item.caption }}</a></li> + {% endfor %} + </ul> + + <h1>My Webpage</h1> + {{ a_variable }} + </body> + </html> + +This covers the default settings. The application developer might have +changed the syntax from ``{% foo %}`` to ``<% foo %>`` or something similar. + +There are two kinds of delimiters. ``{% ... %}`` and ``{{ ... }}``. The first +one is used to execute statements such as for-loops or assign values, the +latter prints the result of the expression to the template. + +.. _variables: + +Variables +--------- + +The application passes variables to the templates you can mess around in the +template. Variables may have attributes or elements on them you can access +too. How a variable looks like, heavily depends on the application providing +those. + +You can use a dot (``.``) to access attributes of a variable, alternative the +so-called "subscript" syntax (``[]``) can be used. The following lines do +the same:: + + {{ foo.bar }} + {{ foo['bar'] }} + +It's important to know that the curly braces are *not* part of the variable +but the print statement. If you access variables inside tags don't put the +braces around. + +If a variable or attribute does not exist you will get back an undefined +value. What you can do with that kind of value depends on the application +configuration, the default behavior is that it evaluates to an empty string +if printed and that you can iterate over it, but every other operation fails. + +.. _notes-on-subscriptions: + +.. admonition:: Implementation + + For convenience sake ``foo.bar`` in Jinja2 does the following things on + the Python layer: + + - check if there is an attribute called `bar` on `foo`. + - if there is not, check if there is an item ``'bar'`` in `foo`. + - if there is not, return an undefined object. + + ``foo['bar']`` on the other hand works mostly the same with the a small + difference in the order: + + - check if there is an item ``'bar'`` in `foo`. + - if there is not, check if there is an attribute called `bar` on `foo`. + - if there is not, return an undefined object. + + This is important if an object has an item or attribute with the same + name. Additionally there is the :func:`attr` filter that just looks up + attributes. + +.. _filters: + +Filters +------- + +Variables can be modified by **filters**. Filters are separated from the +variable by a pipe symbol (``|``) and may have optional arguments in +parentheses. Multiple filters can be chained. The output of one filter is +applied to the next. + +``{{ name|striptags|title }}`` for example will remove all HTML Tags from the +`name` and title-cases it. Filters that accept arguments have parentheses +around the arguments, like a function call. This example will join a list +by commas: ``{{ list|join(', ') }}``. + +The :ref:`builtin-filters` below describes all the builtin filters. + +.. _tests: + +Tests +----- + +Beside filters there are also so called "tests" available. Tests can be used +to test a variable against a common expression. To test a variable or +expression you add `is` plus the name of the test after the variable. For +example to find out if a variable is defined you can do ``name is defined`` +which will then return true or false depending on if `name` is defined. + +Tests can accept arguments too. If the test only takes one argument you can +leave out the parentheses to group them. For example the following two +expressions do the same:: + + {% if loop.index is divisibleby 3 %} + {% if loop.index is divisibleby(3) %} + +The :ref:`builtin-tests` below describes all the builtin tests. + + +Comments +-------- + +To comment-out part of a line in a template, use the comment syntax which is +by default set to ``{# ... #}``. This is useful to comment out parts of the +template for debugging or to add information for other template designers or +yourself:: + + {# note: disabled template because we no longer use this + {% for user in users %} + ... + {% endfor %} + #} + + +Whitespace Control +------------------ + +In the default configuration whitespace is not further modified by the +template engine, so each whitespace (spaces, tabs, newlines etc.) is returned +unchanged. If the application configures Jinja to `trim_blocks` the first +newline after a a template tag is removed automatically (like in PHP). + +But you can also strip whitespace in templates by hand. If you put an minus +sign (``-``) to the start or end of an block (for example a for tag), a +comment or variable expression you can remove the whitespaces after or before +that block:: + + {% for item in seq -%} + {{ item }} + {%- endfor %} + +This will yield all elements without whitespace between them. If `seq` was +a list of numbers from ``1`` to ``9`` the output would be ``123456789``. + +If :ref:`line-statements` are enabled they strip leading whitespace +automatically up to the beginning of the line. + +.. admonition:: Note + + You must not use a whitespace between the tag and the minus sign. + + **valid**:: + + {%- if foo -%}...{% endif %} + + **invalid**:: + + {% - if foo - %}...{% endif %} + + +Escaping +-------- + +It is sometimes desirable or even necessary to have Jinja ignore parts it +would otherwise handle as variables or blocks. For example if the default +syntax is used and you want to use ``{{`` as raw string in the template and +not start a variable you have to use a trick. + +The easiest way is to output the variable delimiter (``{{``) by using a +variable expression:: + + {{ '{{' }} + +For bigger sections it makes sense to mark a block `raw`. For example to +put Jinja syntax as example into a template you can use this snippet:: + + {% raw %} + <ul> + {% for item in seq %} + <li>{{ item }}</li> + {% endfor %} + </ul> + {% endraw %} + + +.. _line-statements: + +Line Statements +--------------- + +If line statements are enabled by the application it's possible to mark a +line as a statement. For example if the line statement prefix is configured +to ``#`` the following two examples are equivalent:: + + <ul> + # for item in seq + <li>{{ item }}</li> + # endfor + </ul> + + <ul> + {% for item in seq %} + <li>{{ item }}</li> + {% endfor %} + </ul> + +The line statement prefix can appear anywhere on the line as long as no text +precedes it. For better readability statements that start a block (such as +`for`, `if`, `elif` etc.) may end with a colon:: + + # for item in seq: + ... + # endfor + + +.. admonition:: Note + + Line statements can span multiple lines if there are open parentheses, + braces or brackets:: + + <ul> + # for href, caption in [('index.html', 'Index'), + ('about.html', 'About')]: + <li><a href="{{ href }}">{{ caption }}</a></li> + # endfor + </ul> + +Since Jinja 2.2 line-based comments are available as well. For example if +the line-comment prefix is configured to be ``##`` everything from ``##`` to +the end of the line is ignored (excluding the newline sign):: + + # for item in seq: + <li>{{ item }}</li> ## this comment is ignored + # endfor + + +.. _template-inheritance: + +Template Inheritance +-------------------- + +The most powerful part of Jinja is template inheritance. Template inheritance +allows you to build a base "skeleton" template that contains all the common +elements of your site and defines **blocks** that child templates can override. + +Sounds complicated but is very basic. It's easiest to understand it by starting +with an example. + + +Base Template +~~~~~~~~~~~~~ + +This template, which we'll call ``base.html``, defines a simple HTML skeleton +document that you might use for a simple two-column page. It's the job of +"child" templates to fill the empty blocks with content:: + + <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN"> + <html lang="en"> + <html xmlns="http://www.w3.org/1999/xhtml"> + <head> + {% block head %} + <link rel="stylesheet" href="style.css" /> + <title>{% block title %}{% endblock %} - My Webpage</title> + {% endblock %} + </head> + <body> + <div id="content">{% block content %}{% endblock %}</div> + <div id="footer"> + {% block footer %} + © Copyright 2008 by <a href="http://domain.invalid/">you</a>. + {% endblock %} + </div> + </body> + +In this example, the ``{% block %}`` tags define four blocks that child templates +can fill in. All the `block` tag does is to tell the template engine that a +child template may override those portions of the template. + +Child Template +~~~~~~~~~~~~~~ + +A child template might look like this:: + + {% extends "base.html" %} + {% block title %}Index{% endblock %} + {% block head %} + {{ super() }} + <style type="text/css"> + .important { color: #336699; } + </style> + {% endblock %} + {% block content %} + <h1>Index</h1> + <p class="important"> + Welcome on my awesome homepage. + </p> + {% endblock %} + +The ``{% extends %}`` tag is the key here. It tells the template engine that +this template "extends" another template. When the template system evaluates +this template, first it locates the parent. The extends tag should be the +first tag in the template. Everything before it is printed out normally and +may cause confusion. For details about this behavior and how to take +advantage of it, see :ref:`null-master-fallback`. + +The filename of the template depends on the template loader. For example the +:class:`FileSystemLoader` allows you to access other templates by giving the +filename. You can access templates in subdirectories with an slash:: + + {% extends "layout/default.html" %} + +But this behavior can depend on the application embedding Jinja. Note that +since the child template doesn't define the ``footer`` block, the value from +the parent template is used instead. + +You can't define multiple ``{% block %}`` tags with the same name in the +same template. This limitation exists because a block tag works in "both" +directions. That is, a block tag doesn't just provide a hole to fill - it +also defines the content that fills the hole in the *parent*. If there +were two similarly-named ``{% block %}`` tags in a template, that template's +parent wouldn't know which one of the blocks' content to use. + +If you want to print a block multiple times you can however use the special +`self` variable and call the block with that name:: + + <title>{% block title %}{% endblock %}</title> + <h1>{{ self.title() }}</h1> + {% block body %}{% endblock %} + + +Super Blocks +~~~~~~~~~~~~ + +It's possible to render the contents of the parent block by calling `super`. +This gives back the results of the parent block:: + + {% block sidebar %} + <h3>Table Of Contents</h3> + ... + {{ super() }} + {% endblock %} + + +Named Block End-Tags +~~~~~~~~~~~~~~~~~~~~ + +Jinja2 allows you to put the name of the block after the end tag for better +readability:: + + {% block sidebar %} + {% block inner_sidebar %} + ... + {% endblock inner_sidebar %} + {% endblock sidebar %} + +However the name after the `endblock` word must match the block name. + + +Block Nesting and Scope +~~~~~~~~~~~~~~~~~~~~~~~ + +Blocks can be nested for more complex layouts. However per default blocks +may not access variables from outer scopes:: + + {% for item in seq %} + <li>{% block loop_item %}{{ item }}{% endblock %}</li> + {% endfor %} + +This example would output empty ``<li>`` items because `item` is unavailable +inside the block. The reason for this is that if the block is replaced by +a child template a variable would appear that was not defined in the block or +passed to the context. + +Starting with Jinja 2.2 you can explicitly specify that variables are +available in a block by setting the block to "scoped" by adding the `scoped` +modifier to a block declaration:: + + {% for item in seq %} + <li>{% block loop_item scoped %}{{ item }}{% endblock %}</li> + {% endfor %} + +When overriding a block the `scoped` modifier does not have to be provided. + + +Template Objects +~~~~~~~~~~~~~~~~ + +.. versionchanged:: 2.4 + +If a template object was passed to the template context you can +extend from that object as well. Assuming the calling code passes +a layout template as `layout_template` to the environment, this +code works:: + + {% extends layout_template %} + +Previously the `layout_template` variable had to be a string with +the layout template's filename for this to work. + + +HTML Escaping +------------- + +When generating HTML from templates, there's always a risk that a variable will +include characters that affect the resulting HTML. There are two approaches: +manually escaping each variable or automatically escaping everything by default. + +Jinja supports both, but what is used depends on the application configuration. +The default configuaration is no automatic escaping for various reasons: + +- escaping everything except of safe values will also mean that Jinja is + escaping variables known to not include HTML such as numbers which is + a huge performance hit. + +- The information about the safety of a variable is very fragile. It could + happen that by coercing safe and unsafe values the return value is double + escaped HTML. + +Working with Manual Escaping +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If manual escaping is enabled it's **your** responsibility to escape +variables if needed. What to escape? If you have a variable that *may* +include any of the following chars (``>``, ``<``, ``&``, or ``"``) you +**have to** escape it unless the variable contains well-formed and trusted +HTML. Escaping works by piping the variable through the ``|e`` filter: +``{{ user.username|e }}``. + +Working with Automatic Escaping +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When automatic escaping is enabled everything is escaped by default except +for values explicitly marked as safe. Those can either be marked by the +application or in the template by using the `|safe` filter. The main +problem with this approach is that Python itself doesn't have the concept +of tainted values so the information if a value is safe or unsafe can get +lost. If the information is lost escaping will take place which means that +you could end up with double escaped contents. + +Double escaping is easy to avoid however, just rely on the tools Jinja2 +provides and don't use builtin Python constructs such as the string modulo +operator. + +Functions returning template data (macros, `super`, `self.BLOCKNAME`) return +safe markup always. + +String literals in templates with automatic escaping are considered unsafe +too. The reason for this is that the safe string is an extension to Python +and not every library will work properly with it. + + +List of Control Structures +-------------------------- + +A control structure refers to all those things that control the flow of a +program - conditionals (i.e. if/elif/else), for-loops, as well as things like +macros and blocks. Control structures appear inside ``{% ... %}`` blocks +in the default syntax. + +For +~~~ + +Loop over each item in a sequence. For example, to display a list of users +provided in a variable called `users`:: + + <h1>Members</h1> + <ul> + {% for user in users %} + <li>{{ user.username|e }}</li> + {% endfor %} + </ul> + +As variables in templates retain their object properties, it is possible to +iterate over containers like `dict`:: + + <dl> + {% for key, value in my_dict.iteritems() %} + <dt>{{ key|e }}</dt> + <dd>{{ value|e }}</dd> + {% endfor %} + </dl> + +Note however that dictionaries usually are unordered so you might want to +either pass it as a sorted list to the template or use the `dictsort` +filter. + +Inside of a for-loop block you can access some special variables: + ++-----------------------+---------------------------------------------------+ +| Variable | Description | ++=======================+===================================================+ +| `loop.index` | The current iteration of the loop. (1 indexed) | ++-----------------------+---------------------------------------------------+ +| `loop.index0` | The current iteration of the loop. (0 indexed) | ++-----------------------+---------------------------------------------------+ +| `loop.revindex` | The number of iterations from the end of the loop | +| | (1 indexed) | ++-----------------------+---------------------------------------------------+ +| `loop.revindex0` | The number of iterations from the end of the loop | +| | (0 indexed) | ++-----------------------+---------------------------------------------------+ +| `loop.first` | True if first iteration. | ++-----------------------+---------------------------------------------------+ +| `loop.last` | True if last iteration. | ++-----------------------+---------------------------------------------------+ +| `loop.length` | The number of items in the sequence. | ++-----------------------+---------------------------------------------------+ +| `loop.cycle` | A helper function to cycle between a list of | +| | sequences. See the explanation below. | ++-----------------------+---------------------------------------------------+ + +Within a for-loop, it's possible to cycle among a list of strings/variables +each time through the loop by using the special `loop.cycle` helper:: + + {% for row in rows %} + <li class="{{ loop.cycle('odd', 'even') }}">{{ row }}</li> + {% endfor %} + +Since Jinja 2.1 an extra `cycle` helper exists that allows loop-unbound +cycling. For more information have a look at the :ref:`builtin-globals`. + +.. _loop-filtering: + +Unlike in Python it's not possible to `break` or `continue` in a loop. You +can however filter the sequence during iteration which allows you to skip +items. The following example skips all the users which are hidden:: + + {% for user in users if not user.hidden %} + <li>{{ user.username|e }}</li> + {% endfor %} + +The advantage is that the special `loop` variable will count correctly thus +not counting the users not iterated over. + +If no iteration took place because the sequence was empty or the filtering +removed all the items from the sequence you can render a replacement block +by using `else`:: + + <ul> + {% for user in users %} + <li>{{ user.username|e }}</li> + {% else %} + <li><em>no users found</em></li> + {% endfor %} + </ul> + +It is also possible to use loops recursively. This is useful if you are +dealing with recursive data such as sitemaps. To use loops recursively you +basically have to add the `recursive` modifier to the loop definition and +call the `loop` variable with the new iterable where you want to recurse. + +The following example implements a sitemap with recursive loops:: + + <ul class="sitemap"> + {%- for item in sitemap recursive %} + <li><a href="{{ item.href|e }}">{{ item.title }}</a> + {%- if item.children -%} + <ul class="submenu">{{ loop(item.children) }}</ul> + {%- endif %}</li> + {%- endfor %} + </ul> + + +If +~~ + +The `if` statement in Jinja is comparable with the if statements of Python. +In the simplest form you can use it to test if a variable is defined, not +empty or not false:: + + {% if users %} + <ul> + {% for user in users %} + <li>{{ user.username|e }}</li> + {% endfor %} + </ul> + {% endif %} + +For multiple branches `elif` and `else` can be used like in Python. You can +use more complex :ref:`expressions` there too:: + + {% if kenny.sick %} + Kenny is sick. + {% elif kenny.dead %} + You killed Kenny! You bastard!!! + {% else %} + Kenny looks okay --- so far + {% endif %} + +If can also be used as :ref:`inline expression <if-expression>` and for +:ref:`loop filtering <loop-filtering>`. + + +Macros +~~~~~~ + +Macros are comparable with functions in regular programming languages. They +are useful to put often used idioms into reusable functions to not repeat +yourself. + +Here a small example of a macro that renders a form element:: + + {% macro input(name, value='', type='text', size=20) -%} + <input type="{{ type }}" name="{{ name }}" value="{{ + value|e }}" size="{{ size }}"> + {%- endmacro %} + +The macro can then be called like a function in the namespace:: + + <p>{{ input('username') }}</p> + <p>{{ input('password', type='password') }}</p> + +If the macro was defined in a different template you have to +:ref:`import <import>` it first. + +Inside macros you have access to three special variables: + +`varargs` + If more positional arguments are passed to the macro than accepted by the + macro they end up in the special `varargs` variable as list of values. + +`kwargs` + Like `varargs` but for keyword arguments. All unconsumed keyword + arguments are stored in this special variable. + +`caller` + If the macro was called from a :ref:`call<call>` tag the caller is stored + in this variable as macro which can be called. + +Macros also expose some of their internal details. The following attributes +are available on a macro object: + +`name` + The name of the macro. ``{{ input.name }}`` will print ``input``. + +`arguments` + A tuple of the names of arguments the macro accepts. + +`defaults` + A tuple of default values. + +`catch_kwargs` + This is `true` if the macro accepts extra keyword arguments (ie: accesses + the special `kwargs` variable). + +`catch_varargs` + This is `true` if the macro accepts extra positional arguments (ie: + accesses the special `varargs` variable). + +`caller` + This is `true` if the macro accesses the special `caller` variable and may + be called from a :ref:`call<call>` tag. + +If a macro name starts with an underscore it's not exported and can't +be imported. + + +.. _call: + +Call +~~~~ + +In some cases it can be useful to pass a macro to another macro. For this +purpose you can use the special `call` block. The following example shows +a macro that takes advantage of the call functionality and how it can be +used:: + + {% macro render_dialog(title, class='dialog') -%} + <div class="{{ class }}"> + <h2>{{ title }}</h2> + <div class="contents"> + {{ caller() }} + </div> + </div> + {%- endmacro %} + + {% call render_dialog('Hello World') %} + This is a simple dialog rendered by using a macro and + a call block. + {% endcall %} + +It's also possible to pass arguments back to the call block. This makes it +useful as replacement for loops. Generally speaking a call block works +exactly like an macro, just that it doesn't have a name. + +Here an example of how a call block can be used with arguments:: + + {% macro dump_users(users) -%} + <ul> + {%- for user in users %} + <li><p>{{ user.username|e }}</p>{{ caller(user) }}</li> + {%- endfor %} + </ul> + {%- endmacro %} + + {% call(user) dump_users(list_of_user) %} + <dl> + <dl>Realname</dl> + <dd>{{ user.realname|e }}</dd> + <dl>Description</dl> + <dd>{{ user.description }}</dd> + </dl> + {% endcall %} + + +Filters +~~~~~~~ + +Filter sections allow you to apply regular Jinja2 filters on a block of +template data. Just wrap the code in the special `filter` section:: + + {% filter upper %} + This text becomes uppercase + {% endfilter %} + + +Assignments +~~~~~~~~~~~ + +Inside code blocks you can also assign values to variables. Assignments at +top level (outside of blocks, macros or loops) are exported from the template +like top level macros and can be imported by other templates. + +Assignments use the `set` tag and can have multiple targets:: + + {% set navigation = [('index.html', 'Index'), ('about.html', 'About')] %} + {% set key, value = call_something() %} + + +Extends +~~~~~~~ + +The `extends` tag can be used to extend a template from another one. You +can have multiple of them in a file but only one of them may be executed +at the time. See the section about :ref:`template-inheritance` above. + + +Block +~~~~~ + +Blocks are used for inheritance and act as placeholders and replacements +at the same time. They are documented in detail as part of the section +about :ref:`template-inheritance`. + + +Include +~~~~~~~ + +The `include` statement is useful to include a template and return the +rendered contents of that file into the current namespace:: + + {% include 'header.html' %} + Body + {% include 'footer.html' %} + +Included templates have access to the variables of the active context by +default. For more details about context behavior of imports and includes +see :ref:`import-visibility`. + +From Jinja 2.2 onwards you can mark an include with ``ignore missing`` in +which case Jinja will ignore the statement if the template to be ignored +does not exist. When combined with ``with`` or ``without context`` it has +to be placed *before* the context visibility statement. Here some valid +examples:: + + {% include "sidebar.html" ignore missing %} + {% include "sidebar.html" ignore missing with context %} + {% include "sidebar.html" ignore missing without context %} + +.. versionadded:: 2.2 + +You can also provide a list of templates that are checked for existence +before inclusion. The first template that exists will be included. If +`ignore missing` is given, it will fall back to rendering nothing if +none of the templates exist, otherwise it will raise an exception. + +Example:: + + {% include ['page_detailed.html', 'page.html'] %} + {% include ['special_sidebar.html', 'sidebar.html'] ignore missing %} + +.. versionchanged:: 2.4 + If a template object was passed to the template context you can + include that object using `include`. + +.. _import: + +Import +~~~~~~ + +Jinja2 supports putting often used code into macros. These macros can go into +different templates and get imported from there. This works similar to the +import statements in Python. It's important to know that imports are cached +and imported templates don't have access to the current template variables, +just the globals by defualt. For more details about context behavior of +imports and includes see :ref:`import-visibility`. + +There are two ways to import templates. You can import the complete template +into a variable or request specific macros / exported variables from it. + +Imagine we have a helper module that renders forms (called `forms.html`):: + + {% macro input(name, value='', type='text') -%} + <input type="{{ type }}" value="{{ value|e }}" name="{{ name }}"> + {%- endmacro %} + + {%- macro textarea(name, value='', rows=10, cols=40) -%} + <textarea name="{{ name }}" rows="{{ rows }}" cols="{{ cols + }}">{{ value|e }} + {%- endmacro %} + +The easiest and most flexible is importing the whole module into a variable. +That way you can access the attributes:: + + {% import 'forms.html' as forms %} +
+
Username
+
{{ forms.input('username') }}
+
Password
+
{{ forms.input('password', type='password') }}
+
+
{{ forms.textarea('comment') }}
+ + +Alternatively you can import names from the template into the current +namespace:: + + {% from 'forms.html' import input as input_field, textarea %} +
+
Username
+
{{ input_field('username') }}
+
Password
+
{{ input_field('password', type='password') }}
+
+
{{ textarea('comment') }}
+ +Macros and variables starting with one ore more underscores are private and +cannot be imported. + +.. versionchanged:: 2.4 + If a template object was passed to the template context you can + import from that object. + + +.. _import-visibility: + +Import Context Behavior +----------------------- + +Per default included templates are passed the current context and imported +templates not. The reason for this is that imports unlike includes are +cached as imports are often used just as a module that holds macros. + +This however can be changed of course explicitly. By adding `with context` +or `without context` to the import/include directive the current context +can be passed to the template and caching is disabled automatically. + +Here two examples:: + + {% from 'forms.html' import input with context %} + {% include 'header.html' without context %} + +.. admonition:: Note + + In Jinja 2.0 the context that was passed to the included template + did not include variables defined in the template. As a matter of + fact this did not work:: + + {% for box in boxes %} + {% include "render_box.html" %} + {% endfor %} + + The included template ``render_box.html`` is *not* able to access + `box` in Jinja 2.0. As of Jinja 2.1 ``render_box.html`` *is* able + to do so. + + +.. _expressions: + +Expressions +----------- + +Jinja allows basic expressions everywhere. These work very similar to regular +Python and even if you're not working with Python you should feel comfortable +with it. + +Literals +~~~~~~~~ + +The simplest form of expressions are literals. Literals are representations +for Python objects such as strings and numbers. The following literals exist: + +"Hello World": + Everything between two double or single quotes is a string. They are + useful whenever you need a string in the template (for example as + arguments to function calls, filters or just to extend or include a + template). + +42 / 42.23: + Integers and floating point numbers are created by just writing the + number down. If a dot is present the number is a float, otherwise an + integer. Keep in mind that for Python ``42`` and ``42.0`` is something + different. + +['list', 'of', 'objects']: + Everything between two brackets is a list. Lists are useful to store + sequential data in or to iterate over them. For example you can easily + create a list of links using lists and tuples with a for loop:: + +
+ {% for href, caption in [('index.html', 'Index'), ('about.html', 'About'), + ('downloads.html', 'Downloads')] %} +
{{ caption }}
+ {% endfor %} +
+ +('tuple', 'of', 'values'): + Tuples are like lists, just that you can't modify them. If the tuple + only has one item you have to end it with a comma. Tuples are usually + used to represent items of two or more elements. See the example above + for more details. + +{'dict': 'of', 'key': 'and', 'value': 'pairs'}: + A dict in Python is a structure that combines keys and values. Keys must + be unique and always have exactly one value. Dicts are rarely used in + templates, they are useful in some rare cases such as the :func:`xmlattr` + filter. + +true / false: + true is always true and false is always false. + +.. admonition:: Note + + The special constants `true`, `false` and `none` are indeed lowercase. + Because that caused confusion in the past, when writing `True` expands + to an undefined variable that is considered false, all three of them can + be written in title case too (`True`, `False`, and `None`). However for + consistency (all Jinja identifiers are lowercase) you should use the + lowercase versions. + +Math +~~~~ + +Jinja allows you to calculate with values. This is rarely useful in templates +but exists for completeness' sake. The following operators are supported: + +\+ + Adds two objects together. Usually the objects are numbers but if both are + strings or lists you can concatenate them this way. This however is not + the preferred way to concatenate strings! For string concatenation have + a look at the ``~`` operator. ``{{ 1 + 1 }}`` is ``2``. + +\- + Substract the second number from the first one. ``{{ 3 - 2 }}`` is ``1``. + +/ + Divide two numbers. The return value will be a floating point number. + ``{{ 1 / 2 }}`` is ``{{ 0.5 }}``. + +// + Divide two numbers and return the truncated integer result. + ``{{ 20 // 7 }}`` is ``2``. + +% + Calculate the remainder of an integer division. ``{{ 11 % 7 }}`` is ``4``. + +\* + Multiply the left operand with the right one. ``{{ 2 * 2 }}`` would + return ``4``. This can also be used to repeat a string multiple times. + ``{{ '=' * 80 }}`` would print a bar of 80 equal signs. + +\** + Raise the left operand to the power of the right operand. ``{{ 2**3 }}`` + would return ``8``. + +Comparisons +~~~~~~~~~~~ + +== + Compares two objects for equality. + +!= + Compares two objects for inequality. + +> + `true` if the left hand side is greater than the right hand side. + +>= + `true` if the left hand side is greater or equal to the right hand side. + +< + `true` if the left hand side is lower than the right hand side. + +<= + `true` if the left hand side is lower or equal to the right hand side. + +Logic +~~~~~ + +For `if` statements, `for` filtering or `if` expressions it can be useful to +combine multiple expressions: + +and + Return true if the left and the right operand is true. + +or + Return true if the left or the right operand is true. + +not + negate a statement (see below). + +(expr) + group an expression. + +.. admonition:: Note + + The ``is`` and ``in`` operators support negation using an infix notation + too: ``foo is not bar`` and ``foo not in bar`` instead of ``not foo is bar`` + and ``not foo in bar``. All other expressions require a prefix notation: + ``not (foo and bar).`` + + +Other Operators +~~~~~~~~~~~~~~~ + +The following operators are very useful but don't fit into any of the other +two categories: + +in + Perform sequence / mapping containment test. Returns true if the left + operand is contained in the right. ``{{ 1 in [1, 2, 3] }}`` would for + example return true. + +is + Performs a :ref:`test `. + +\| + Applies a :ref:`filter `. + +~ + Converts all operands into strings and concatenates them. + ``{{ "Hello " ~ name ~ "!" }}`` would return (assuming `name` is + ``'John'``) ``Hello John!``. + +() + Call a callable: ``{{ post.render() }}``. Inside of the parentheses you + can use positional arguments and keyword arguments like in python: + ``{{ post.render(user, full=true) }}``. + +. / [] + Get an attribute of an object. (See :ref:`variables`) + + +.. _if-expression: + +If Expression +~~~~~~~~~~~~~ + +It is also possible to use inline `if` expressions. These are useful in some +situations. For example you can use this to extend from one template if a +variable is defined, otherwise from the default layout template:: + + {% extends layout_template if layout_template is defined else 'master.html' %} + +The general syntax is `` if else ``. + +The `else` part is optional. If not provided the else block implicitly +evaluates into an undefined object:: + + {{ '[%s]' % page.title if page.title }} + + +.. _builtin-filters: + +List of Builtin Filters +----------------------- + +.. jinjafilters:: + + +.. _builtin-tests: + +List of Builtin Tests +--------------------- + +.. jinjatests:: + +.. _builtin-globals: + +List of Global Functions +------------------------ + +The following functions are available in the global scope by default: + +.. function:: range([start,] stop[, step]) + + Return a list containing an arithmetic progression of integers. + range(i, j) returns [i, i+1, i+2, ..., j-1]; start (!) defaults to 0. + When step is given, it specifies the increment (or decrement). + For example, range(4) returns [0, 1, 2, 3]. The end point is omitted! + These are exactly the valid indices for a list of 4 elements. + + This is useful to repeat a template block multiple times for example + to fill a list. Imagine you have 7 users in the list but you want to + render three empty items to enforce a height with CSS:: + +
+ {% for user in users %} +
{{ user.username }}
+ {% endfor %} + {% for number in range(10 - users|count) %} +
...
+ {% endfor %} +
+ +.. function:: lipsum(n=5, html=True, min=20, max=100) + + Generates some lorem ipsum for the template. Per default five paragraphs + with HTML are generated each paragraph between 20 and 100 words. If html + is disabled regular text is returned. This is useful to generate simple + contents for layout testing. + +.. function:: dict(\**items) + + A convenient alternative to dict literals. ``{'foo': 'bar'}`` is the same + as ``dict(foo='bar')``. + +.. class:: cycler(\*items) + + The cycler allows you to cycle among values similar to how `loop.cycle` + works. Unlike `loop.cycle` however you can use this cycler outside of + loops or over multiple loops. + + This is for example very useful if you want to show a list of folders and + files, with the folders on top, but both in the same list with alternating + row colors. + + The following example shows how `cycler` can be used:: + + {% set row_class = cycler('odd', 'even') %} +
+ {% for folder in folders %} +
{{ folder|e }}
+ {% endfor %} + {% for filename in files %} +
{{ filename|e }}
+ {% endfor %} +
+ + A cycler has the following attributes and methods: + + .. method:: reset() + + Resets the cycle to the first item. + + .. method:: next() + + Goes one item a head and returns the then current item. + + .. attribute:: current + + Returns the current item. + + **new in Jinja 2.1** + +.. class:: joiner(sep=', ') + + A tiny helper that can be use to "join" multiple sections. A joiner is + passed a string and will return that string every time it's called, except + the first time in which situation it returns an empty string. You can + use this to join things:: + + {% set pipe = joiner("|") %} + {% if categories %} {{ pipe() }} + Categories: {{ categories|join(", ") }} + {% endif %} + {% if author %} {{ pipe() }} + Author: {{ author() }} + {% endif %} + {% if can_edit %} {{ pipe() }} + Edit + {% endif %} + + **new in Jinja 2.1** + + +Extensions +---------- + +The following sections cover the built-in Jinja2 extensions that may be +enabled by the application. The application could also provide further +extensions not covered by this documentation. In that case there should +be a separate document explaining the extensions. + +.. _i18n-in-templates: + +i18n +~~~~ + +If the i18n extension is enabled it's possible to mark parts in the template +as translatable. To mark a section as translatable you can use `trans`:: + +
{% trans %}Hello {{ user }}!{% endtrans %}
+ +To translate a template expression --- say, using template filters or just +accessing an attribute of an object --- you need to bind the expression to a +name for use within the translation block:: + +
{% trans user=user.username %}Hello {{ user }}!{% endtrans %}
+ +If you need to bind more than one expression inside a `trans` tag, separate +the pieces with a comma (``,``):: + + {% trans book_title=book.title, author=author.name %} + This is {{ book_title }} by {{ author }} + {% endtrans %} + +Inside trans tags no statements are allowed, only variable tags are. + +To pluralize, specify both the singular and plural forms with the `pluralize` +tag, which appears between `trans` and `endtrans`:: + + {% trans count=list|length %} + There is {{ count }} {{ name }} object. + {% pluralize %} + There are {{ count }} {{ name }} objects. + {% endtrans %} + +Per default the first variable in a block is used to determine the correct +singular or plural form. If that doesn't work out you can specify the name +which should be used for pluralizing by adding it as parameter to `pluralize`:: + + {% trans ..., user_count=users|length %}... + {% pluralize user_count %}...{% endtrans %} + +It's also possible to translate strings in expressions. For that purpose +three functions exist: + +_ `gettext`: translate a single string +- `ngettext`: translate a pluralizable string +- `_`: alias for `gettext` + +For example you can print a translated string easily this way:: + + {{ _('Hello World!') }} + +To use placeholders you can use the `format` filter:: + + {{ _('Hello %(user)s!')|format(user=user.username) }} + +For multiple placeholders always use keyword arguments to `format` as other +languages may not use the words in the same order. + +.. versionchanged:: 2.5 + +If newstyle gettext calls are activated (:ref:`newstyle-gettext`), using +placeholders is a lot easier: + +.. sourcecode:: html+jinja + + {{ gettext('Hello World!') }} + {{ gettext('Hello %(name)s!', name='World') }} + {{ ngettext('%(num)d apple', '%(num)d apples', apples|count) }} + +Note that the `ngettext` function's format string automatically receives +the count as `num` parameter additionally to the regular parameters. + + +Expression Statement +~~~~~~~~~~~~~~~~~~~~ + +If the expression-statement extension is loaded a tag called `do` is available +that works exactly like the regular variable expression (``{{ ... }}``) just +that it doesn't print anything. This can be used to modify lists:: + + {% do navigation.append('a string') %} + + +Loop Controls +~~~~~~~~~~~~~ + +If the application enables the :ref:`loopcontrols-extension` it's possible to +use `break` and `continue` in loops. When `break` is reached, the loop is +terminated; if `continue` is reached, the processing is stopped and continues +with the next iteration. + +Here a loop that skips every second item:: + + {% for user in users %} + {%- if loop.index is even %}{% continue %}{% endif %} + ... + {% endfor %} + +Likewise a look that stops processing after the 10th iteration:: + + {% for user in users %} + {%- if loop.index >= 10 %}{% break %}{% endif %} + {%- endfor %} + + +With Statement +~~~~~~~~~~~~~~ + +.. versionadded:: 2.3 + +If the application enables the :ref:`with-extension` it is possible to +use the `with` keyword in templates. This makes it possible to create +a new inner scope. Variables set within this scope are not visible +outside of the scope. + +With in a nutshell:: + + {% with %} + {% set foo = 42 %} + {{ foo }} foo is 42 here + {% endwith %} + foo is not visible here any longer + +Because it is common to set variables at the beginning of the scope +you can do that within the with statement. The following two examples +are equivalent:: + + {% with foo = 42 %} + {{ foo }} + {% endwith %} + + {% with %} + {% set foo = 42 %} + {{ foo }} + {% endwith %} + +.. _autoescape-overrides: + +Autoescape Extension +-------------------- + +.. versionadded:: 2.4 + +If the application enables the :ref:`autoescape-extension` one can +activate and deactivate the autoescaping from within the templates. + +Example:: + + {% autoescape true %} + Autoescaping is active within this block + {% endautoescape %} + + {% autoescape false %} + Autoescaping is inactive within this block + {% endautoescape %} + +After the `endautoescape` the behavior is reverted to what it was before. diff --git a/lib/jinja2/docs/tricks.rst b/lib/jinja2/docs/tricks.rst new file mode 100644 --- /dev/null +++ b/lib/jinja2/docs/tricks.rst @@ -0,0 +1,100 @@ +Tips and Tricks +=============== + +.. highlight:: html+jinja + +This part of the documentation shows some tips and tricks for Jinja2 +templates. + + +.. _null-master-fallback: + +Null-Master Fallback +-------------------- + +Jinja2 supports dynamic inheritance and does not distinguish between parent +and child template as long as no `extends` tag is visited. While this leads +to the surprising behavior that everything before the first `extends` tag +including whitespace is printed out instead of being igored, it can be used +for a neat trick. + +Usually child templates extend from one template that adds a basic HTML +skeleton. However it's possible put the `extends` tag into an `if` tag to +only extend from the layout template if the `standalone` variable evaluates +to false which it does per default if it's not defined. Additionally a very +basic skeleton is added to the file so that if it's indeed rendered with +`standalone` set to `True` a very basic HTML skeleton is added:: + + {% if not standalone %}{% extends 'master.html' %}{% endif -%} + + {% block title %}The Page Title{% endblock %} + + {% block body %} +
This is the page body.
+ {% endblock %} + + +Alternating Rows +---------------- + +If you want to have different styles for each row of a table or +list you can use the `cycle` method on the `loop` object:: + +
+ {% for row in rows %} +
{{ row }}
+ {% endfor %} +
+ +`cycle` can take an unlimited amount of strings. Each time this +tag is encountered the next item from the list is rendered. + + +Highlighting Active Menu Items +------------------------------ + +Often you want to have a navigation bar with an active navigation +item. This is really simple to achieve. Because assignments outside +of `block`\s in child templates are global and executed before the layout +template is evaluated it's possible to define the active menu item in the +child template:: + + {% extends "layout.html" %} + {% set active_page = "index" %} + +The layout template can then access `active_page`. Additionally it makes +sense to defined a default for that variable:: + + {% set navigation_bar = [ + ('/', 'index', 'Index'), + ('/downloads/', 'downloads', 'Downloads'), + ('/about/', 'about', 'About') + ] -%} + {% set active_page = active_page|default('index') -%} + ... +
+ {% for href, id, caption in navigation_bar %} + {{ caption|e }}/li> + {% endfor %} +
+ ... + +.. _accessing-the-parent-loop: + +Accessing the parent Loop +------------------------- + +The special `loop` variable always points to the innermost loop. If it's +desired to have access to an outer loop it's possible to alias it:: + + + {% for row in table %} + + {% set rowloop = loop %} + {% for cell in row %} +
+# for item in range(10) +
${item}
+# endfor +\ +""").render() diff --git a/lib/jinja2/examples/basic/debugger.py b/lib/jinja2/examples/basic/debugger.py new file mode 100644 --- /dev/null +++ b/lib/jinja2/examples/basic/debugger.py @@ -0,0 +1,7 @@ +from jinja2 import Environment +from jinja2.loaders import FileSystemLoader + +env = Environment(loader=FileSystemLoader('templates')) + +tmpl = env.get_template('broken.html') +print tmpl.render(seq=[3, 2, 4, 5, 3, 2, 0, 2, 1]) diff --git a/lib/jinja2/examples/basic/inheritance.py b/lib/jinja2/examples/basic/inheritance.py new file mode 100644 --- /dev/null +++ b/lib/jinja2/examples/basic/inheritance.py @@ -0,0 +1,12 @@ +from jinja2 import Environment +from jinja2.loaders import DictLoader + + +env = Environment(loader=DictLoader({ +'a': '''[A[{% block body %}{% endblock %}]]''', +'b': '''{% extends 'a' %}{% block body %}[B]{% endblock %}''', +'c': '''{% extends 'b' %}{% block body %}###{{ super() }}###{% endblock %}''' +})) + + +print env.get_template('c').render() diff --git a/lib/jinja2/examples/basic/templates/broken.html b/lib/jinja2/examples/basic/templates/broken.html new file mode 100644 --- /dev/null +++ b/lib/jinja2/examples/basic/templates/broken.html @@ -0,0 +1,6 @@ +{% from 'subbroken.html' import may_break %} +
+{% for item in seq %} +
{{ may_break(item) }}
+{% endfor %} +
diff --git a/lib/jinja2/examples/basic/templates/subbroken.html b/lib/jinja2/examples/basic/templates/subbroken.html new file mode 100644 --- /dev/null +++ b/lib/jinja2/examples/basic/templates/subbroken.html @@ -0,0 +1,3 @@ +{% macro may_break(item) -%} + [{{ item / 0 }}] +{%- endmacro %} diff --git a/lib/jinja2/examples/basic/test.py b/lib/jinja2/examples/basic/test.py new file mode 100644 --- /dev/null +++ b/lib/jinja2/examples/basic/test.py @@ -0,0 +1,27 @@ +from jinja2 import Environment +from jinja2.loaders import DictLoader + +env = Environment(loader=DictLoader({ +'child.html': u'''\ +{% extends master_layout or 'master.html' %} +{% include helpers = 'helpers.html' %} +{% macro get_the_answer() %}42{% endmacro %} +{% title = 'Hello World' %} +{% block body %} + {{ get_the_answer() }} + {{ helpers.conspirate() }} +{% endblock %} +''', +'master.html': u'''\ + +{{ title }} +{% block body %}{% endblock %} +''', +'helpers.html': u'''\ +{% macro conspirate() %}23{% endmacro %} +''' +})) + + +tmpl = env.get_template("child.html") +print tmpl.render() diff --git a/lib/jinja2/examples/basic/test_filter_and_linestatements.py b/lib/jinja2/examples/basic/test_filter_and_linestatements.py new file mode 100644 --- /dev/null +++ b/lib/jinja2/examples/basic/test_filter_and_linestatements.py @@ -0,0 +1,25 @@ +from jinja2 import Environment + + +env = Environment(line_statement_prefix='%', variable_start_string="${", variable_end_string="}") +tmpl = env.from_string("""\ +% macro foo() + ${caller(42)} +% endmacro +
+% for item in seq +
${item}
+% endfor +
+% call(var) foo() + [${var}] +% endcall +% filter escape + + % for item in [1, 2, 3] + - ${item} + % endfor +% endfilter +""") + +print tmpl.render(seq=range(10)) diff --git a/lib/jinja2/examples/basic/test_loop_filter.py b/lib/jinja2/examples/basic/test_loop_filter.py new file mode 100644 --- /dev/null +++ b/lib/jinja2/examples/basic/test_loop_filter.py @@ -0,0 +1,12 @@ +from jinja2 import Environment + +tmpl = Environment().from_string("""\ +
+{%- for item in [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] if item % 2 == 0 %} +
{{ loop.index }} / {{ loop.length }}: {{ item }}
+{%- endfor %} +
+if condition: {{ 1 if foo else 0 }} +""") + +print tmpl.render(foo=True) diff --git a/lib/jinja2/examples/basic/translate.py b/lib/jinja2/examples/basic/translate.py new file mode 100644 --- /dev/null +++ b/lib/jinja2/examples/basic/translate.py @@ -0,0 +1,14 @@ +from jinja2 import Environment + +env = Environment(extensions=['jinja2.ext.i18n']) +env.globals['gettext'] = { + 'Hello %(user)s!': 'Hallo %(user)s!' +}.__getitem__ +env.globals['ngettext'] = lambda s, p, n: { + '%(count)s user': '%(count)d Benutzer', + '%(count)s users': '%(count)d Benutzer' +}[n == 1 and s or p] +print env.from_string("""\ +{% trans %}Hello {{ user }}!{% endtrans %} +{% trans count=users|count %}{{ count }} user{% pluralize %}{{ count }} users{% endtrans %} +""").render(user="someone", users=[1, 2, 3]) diff --git a/lib/jinja2/examples/bench.py b/lib/jinja2/examples/bench.py new file mode 100644 --- /dev/null +++ b/lib/jinja2/examples/bench.py @@ -0,0 +1,433 @@ +"""\ + This benchmark compares some python templating engines with Jinja 2 so + that we get a picture of how fast Jinja 2 is for a semi real world + template. If a template engine is not installed the test is skipped.\ +""" +import sys +import cgi +from timeit import Timer +from jinja2 import Environment as JinjaEnvironment + +context = { + 'page_title': 'mitsuhiko\'s benchmark', + 'table': [dict(a=1,b=2,c=3,d=4,e=5,f=6,g=7,h=8,i=9,j=10) for x in range(1000)] +} + +jinja_template = JinjaEnvironment( + line_statement_prefix='%', + variable_start_string="${", + variable_end_string="}" +).from_string("""\ + + + + ${page_title|e} + + +
+
${page_title|e}
+
+
+ % for href, caption in [ + ('index.html', 'Index'), + ('downloads.html', 'Downloads'), + ('products.html', 'Products') + ] +
${caption|e}
+ % endfor +
+
+ + % for row in table + + % for cell in row + + % endfor + + % endfor +
${cell}
+
+ +\ +""") + +def test_jinja(): + jinja_template.render(context) + +try: + from tornado.template import Template +except ImportError: + test_tornado = None +else: + tornado_template = Template("""\ + + + + {{ page_title }} + + +
+
{{ page_title }}
+
+
+ {% for href, caption in [ \ + ('index.html', 'Index'), \ + ('downloads.html', 'Downloads'), \ + ('products.html', 'Products') \ + ] %} +
{{ caption }}
+ {% end %} +
+
+ + {% for row in table %} + + {% for cell in row %} + + {% end %} + + {% end %} +
{{ cell }}
+
+ +\ +""") + + def test_tornado(): + tornado_template.generate(**context) + +try: + from django.conf import settings + settings.configure() + from django.template import Template as DjangoTemplate, Context as DjangoContext +except ImportError: + test_django = None +else: + django_template = DjangoTemplate("""\ + + + + {{ page_title }} + + +
+
{{ page_title }}
+
+
+ {% for href, caption in navigation %} +
{{ caption }}
+ {% endfor %} +
+
+ + {% for row in table %} + + {% for cell in row %} + + {% endfor %} + + {% endfor %} +
{{ cell }}
+
+ +\ +""") + + def test_django(): + c = DjangoContext(context) + c['navigation'] = [('index.html', 'Index'), ('downloads.html', 'Downloads'), + ('products.html', 'Products')] + django_template.render(c) + +try: + from mako.template import Template as MakoTemplate +except ImportError: + test_mako = None +else: + mako_template = MakoTemplate("""\ + + + + ${page_title|h} + + +
+
${page_title|h}
+
+
+ % for href, caption in [('index.html', 'Index'), ('downloads.html', 'Downloads'), ('products.html', 'Products')]: +
${caption|h}
+ % endfor +
+
+ + % for row in table: + + % for cell in row: + + % endfor + + % endfor +
${cell}
+
+ +\ +""") + + def test_mako(): + mako_template.render(**context) + +try: + from genshi.template import MarkupTemplate as GenshiTemplate +except ImportError: + test_genshi = None +else: + genshi_template = GenshiTemplate("""\ + + + ${page_title} + + +
+
${page_title}
+
+
+
${caption}
+
+
+ + + + +
${cell}
+
+ +\ +""") + + def test_genshi(): + genshi_template.generate(**context).render('html', strip_whitespace=False) + +try: + from Cheetah.Template import Template as CheetahTemplate +except ImportError: + test_cheetah = None +else: + cheetah_template = CheetahTemplate("""\ +#import cgi + + + + $cgi.escape($page_title) + + +
+
$cgi.escape($page_title)
+
+
+ #for $href, $caption in [('index.html', 'Index'), ('downloads.html', 'Downloads'), ('products.html', 'Products')]: +
$cgi.escape($caption)
+ #end for +
+
+ + #for $row in $table: + + #for $cell in $row: + + #end for + + #end for +
$cell
+
+ +\ +""", searchList=[dict(context)]) + + def test_cheetah(): + unicode(cheetah_template) + +try: + import tenjin +except ImportError: + test_tenjin = None +else: + tenjin_template = tenjin.Template() + tenjin_template.convert("""\ + + + + ${page_title} + + +
+
${page_title}
+
+
+ +
${caption}
+ +
+
+ + + + + + + + +
#{cell}
+
+ +\ +""") + + def test_tenjin(): + from tenjin.helpers import escape, to_str + tenjin_template.render(context, locals()) + +try: + from spitfire.compiler import util as SpitfireTemplate + from spitfire.compiler.analyzer import o2_options as spitfire_optimizer +except ImportError: + test_spitfire = None +else: + spitfire_template = SpitfireTemplate.load_template("""\ + + + + $cgi.escape($page_title) + + +
+
$cgi.escape($page_title)
+
+
+ #for $href, $caption in [('index.html', 'Index'), ('downloads.html', 'Downloads'), ('products.html', 'Products')] +
$cgi.escape($caption)
+ #end for +
+
+ + #for $row in $table + + #for $cell in $row + + #end for + + #end for +
$cell
+
+ +\ +""", 'spitfire_tmpl', spitfire_optimizer, {'enable_filters': False}) + spitfire_context = dict(context, **{'cgi': cgi}) + + def test_spitfire(): + spitfire_template(search_list=[spitfire_context]).main() + + +try: + from chameleon.zpt.template import PageTemplate +except ImportError: + test_chameleon = None +else: + chameleon_template = PageTemplate("""\ + + + Page Title + + +
+
Page Title
+
+
+
caption
+
+
+ + + + +
cell
+
+ +\ +""") + chameleon_context = dict(context) + chameleon_context['sections'] = [ + ('index.html', 'Index'), + ('downloads.html', 'Downloads'), + ('products.html', 'Products') + ] + def test_chameleon(): + chameleon_template.render(**chameleon_context) + +try: + from chameleon.zpt.template import PageTemplate + from chameleon.genshi import language +except ImportError: + test_chameleon_genshi = None +else: + chameleon_genshi_template = PageTemplate("""\ + + + ${page_title} + + +
+
${page_title}
+
+
+
${info[1]}
+
+
+ + + + +
${row[cell]}
+
+ +\ +""", parser=language.Parser()) + chameleon_genshi_context = dict(context) + chameleon_genshi_context['sections'] = [ + ('index.html', 'Index'), + ('downloads.html', 'Downloads'), + ('products.html', 'Products') + ] + def test_chameleon_genshi(): + chameleon_genshi_template.render(**chameleon_genshi_context) + + +sys.stdout.write('\r' + '\n'.join(( + '=' * 80, + 'Template Engine BigTable Benchmark'.center(80), + '=' * 80, + __doc__, + '-' * 80 +)) + '\n') + + +for test in 'jinja', 'mako', 'tornado', 'tenjin', 'spitfire', 'django', 'genshi', 'cheetah', 'chameleon', 'chameleon_genshi': + if locals()['test_' + test] is None: + sys.stdout.write(' %-20s*not installed*\n' % test) + continue + t = Timer(setup='from __main__ import test_%s as bench' % test, + stmt='bench()') + sys.stdout.write(' >> %-20s' % test) + sys.stdout.flush() + sys.stdout.write('\r %-20s%.4f seconds\n' % (test, t.timeit(number=50) / 50)) +sys.stdout.write('-' * 80 + '\n') +sys.stdout.write('''\ + WARNING: The results of this benchmark are useless to compare the + performance of template engines and should not be taken seriously in any + way. It's testing the performance of simple loops and has no real-world + usefulnes. It only used to check if changes on the Jinja code affect + performance in a good or bad way and how it roughly compares to others. +''' + '=' * 80 + '\n') diff --git a/lib/jinja2/examples/profile.py b/lib/jinja2/examples/profile.py new file mode 100644 --- /dev/null +++ b/lib/jinja2/examples/profile.py @@ -0,0 +1,52 @@ +try: + from cProfile import Profile +except ImportError: + from profile import Profile +from pstats import Stats +from jinja2 import Environment as JinjaEnvironment + +context = { + 'page_title': 'mitsuhiko\'s benchmark', + 'table': [dict(a=1,b=2,c=3,d=4,e=5,f=6,g=7,h=8,i=9,j=10) for x in range(1000)] +} + +source = """\ +% macro testmacro(x) + ${x} +% endmacro + + + + ${page_title|e} + + +
+
${page_title|e}
+
+
+ + % for row in table + + % for cell in row + + % endfor + + % endfor +
${testmacro(cell)}
+
+ +\ +""" +jinja_template = JinjaEnvironment( + line_statement_prefix='%', + variable_start_string="${", + variable_end_string="}" +).from_string(source) +print jinja_template.environment.compile(source, raw=True) + + +p = Profile() +p.runcall(lambda: jinja_template.render(context)) +stats = Stats(p) +stats.sort_stats('time', 'calls') +stats.print_stats() diff --git a/lib/jinja2/examples/rwbench/django/_form.html b/lib/jinja2/examples/rwbench/django/_form.html new file mode 100644 --- /dev/null +++ b/lib/jinja2/examples/rwbench/django/_form.html @@ -0,0 +1,1 @@ +
{{ body }}
diff --git a/lib/jinja2/examples/rwbench/django/_input_field.html b/lib/jinja2/examples/rwbench/django/_input_field.html new file mode 100644 --- /dev/null +++ b/lib/jinja2/examples/rwbench/django/_input_field.html @@ -0,0 +1,1 @@ + diff --git a/lib/jinja2/examples/rwbench/django/_textarea.html b/lib/jinja2/examples/rwbench/django/_textarea.html new file mode 100644 --- /dev/null +++ b/lib/jinja2/examples/rwbench/django/_textarea.html @@ -0,0 +1,1 @@ +{{ value }} diff --git a/lib/jinja2/examples/rwbench/django/index.html b/lib/jinja2/examples/rwbench/django/index.html new file mode 100644 --- /dev/null +++ b/lib/jinja2/examples/rwbench/django/index.html @@ -0,0 +1,29 @@ +{% extends "layout.html" %} +{% block page_title %}Index Page{% endblock %} +{% block body %} + {% for article in articles %} + {% if article.published %} +
+
{{ article.title }}
+
written by {{ article.user.username }} on {{ article.pub_date|dateformat }}
+
{{ article.body|safe }}
+
+ {% endif %} + {% endfor %} + {% form %} +
+
Name
+
{% input_field 'name' %}
+
E-Mail
+
{% input_field 'email' %}
+
URL
+
{% input_field 'url' %}
+
Comment
+
{% textarea 'comment' %}
+
Captcha
+
{% input_field 'captcha' %}
+
+ {% input_field '' 'submit' 'Submit' %} + {% input_field 'cancel' 'submit' 'Cancel' %} + {% endform %} +{% endblock %} diff --git a/lib/jinja2/examples/rwbench/django/layout.html b/lib/jinja2/examples/rwbench/django/layout.html new file mode 100644 --- /dev/null +++ b/lib/jinja2/examples/rwbench/django/layout.html @@ -0,0 +1,29 @@ + + + + {% block page_title %}{% endblock %} | RealWorld Benchmark + + + +
+
+
RealWorld Benchmark
+
+ A less stupid benchmark for Mako and Jinja2 to get an impression how + code changes affect runtime performance. +
+
+
+ {% for href, caption in page_navigation %} +
{{ caption }}
+ {% endfor %} +
+
+ {% block body %}{% endblock %} +
+
+ © Copyright 2008 by I don't know who. +
+
+ + diff --git a/lib/jinja2/examples/rwbench/djangoext.py b/lib/jinja2/examples/rwbench/djangoext.py new file mode 100644 --- /dev/null +++ b/lib/jinja2/examples/rwbench/djangoext.py @@ -0,0 +1,135 @@ +# -*- coding: utf-8 -*- +from rwbench import ROOT +from os.path import join +from django.conf import settings +settings.configure( + TEMPLATE_DIRS=(join(ROOT, 'django'),), + TEMPLATE_LOADERS=( + ('django.template.loaders.cached.Loader', ( + 'django.template.loaders.filesystem.Loader', + )), + ) +) +from django.template import loader as django_loader, Context as DjangoContext, \ + Node, NodeList, Variable, TokenParser +from django import template as django_template_module +from django.template import Library + + +# for django extensions. We monkey patch our extensions in so that +# we don't have to initialize a more complex django setup. +django_extensions = django_template_module.Library() +django_template_module.builtins.append(django_extensions) + + +from rwbench import dateformat +django_extensions.filter(dateformat) + + +def var_or_none(x): + if x is not None: + return Variable(x) + + +# and more django extensions + at django_extensions.tag +def input_field(parser, token): + p = TokenParser(token.contents) + args = [p.value()] + while p.more(): + args.append(p.value()) + return InputFieldNode(*args) + + + at django_extensions.tag +def textarea(parser, token): + p = TokenParser(token.contents) + args = [p.value()] + while p.more(): + args.append(p.value()) + return TextareaNode(*args) + + + at django_extensions.tag +def form(parser, token): + p = TokenParser(token.contents) + args = [] + while p.more(): + args.append(p.value()) + body = parser.parse(('endform',)) + parser.delete_first_token() + return FormNode(body, *args) + + +class InputFieldNode(Node): + + def __init__(self, name, type=None, value=None): + self.name = var_or_none(name) + self.type = var_or_none(type) + self.value = var_or_none(value) + + def render(self, context): + name = self.name.resolve(context) + type = 'text' + value = '' + if self.type is not None: + type = self.type.resolve(context) + if self.value is not None: + value = self.value.resolve(context) + tmpl = django_loader.get_template('_input_field.html') + return tmpl.render(DjangoContext({ + 'name': name, + 'type': type, + 'value': value + })) + + +class TextareaNode(Node): + + def __init__(self, name, rows=None, cols=None, value=None): + self.name = var_or_none(name) + self.rows = var_or_none(rows) + self.cols = var_or_none(cols) + self.value = var_or_none(value) + + def render(self, context): + name = self.name.resolve(context) + rows = 10 + cols = 40 + value = '' + if self.rows is not None: + rows = int(self.rows.resolve(context)) + if self.cols is not None: + cols = int(self.cols.resolve(context)) + if self.value is not None: + value = self.value.resolve(context) + tmpl = django_loader.get_template('_textarea.html') + return tmpl.render(DjangoContext({ + 'name': name, + 'rows': rows, + 'cols': cols, + 'value': value + })) + + +class FormNode(Node): + + def __init__(self, body, action=None, method=None): + self.body = body + self.action = action + self.method = method + + def render(self, context): + body = self.body.render(context) + action = '' + method = 'post' + if self.action is not None: + action = self.action.resolve(context) + if self.method is not None: + method = self.method.resolve(context) + tmpl = django_loader.get_template('_form.html') + return tmpl.render(DjangoContext({ + 'body': body, + 'action': action, + 'method': method + })) diff --git a/lib/jinja2/examples/rwbench/genshi/helpers.html b/lib/jinja2/examples/rwbench/genshi/helpers.html new file mode 100644 --- /dev/null +++ b/lib/jinja2/examples/rwbench/genshi/helpers.html @@ -0,0 +1,12 @@ +
+ + + + + + + $value + + +
diff --git a/lib/jinja2/examples/rwbench/genshi/index.html b/lib/jinja2/examples/rwbench/genshi/index.html new file mode 100644 --- /dev/null +++ b/lib/jinja2/examples/rwbench/genshi/index.html @@ -0,0 +1,41 @@ + + + + + Index Page + +
+ +
${article.title}
+
written by ${article.user.username} on ${dateformat(article.pub_date)}
+
${Markup(article.body)}
+ +
+ +
+
+
Name
+
${input_field('name')}
+
E-Mail
+
${input_field('email')}
+
URL
+
${input_field('url')}
+
Comment
+
${textarea('comment')}
+
Captcha
+
${input_field('captcha')}
+
+ ${input_field(type='submit', value='Submit')} + ${input_field(name='cancel', type='submit', value='Cancel')} +
+ + diff --git a/lib/jinja2/examples/rwbench/genshi/layout.html b/lib/jinja2/examples/rwbench/genshi/layout.html new file mode 100644 --- /dev/null +++ b/lib/jinja2/examples/rwbench/genshi/layout.html @@ -0,0 +1,30 @@ + + + + ${select('title/text()')} | RealWorld Benchmark + + + + + +
+
+
RealWorld Benchmark
+
+ A less stupid benchmark for Mako and Jinja2 to get an impression how + code changes affect runtime performance. +
+
+
+
$caption
+
+
+ ${select('*|text()')} +
+
+ © Copyright 2008 by I don't know who. +
+
+ + + diff --git a/lib/jinja2/examples/rwbench/jinja/helpers.html b/lib/jinja2/examples/rwbench/jinja/helpers.html new file mode 100644 --- /dev/null +++ b/lib/jinja2/examples/rwbench/jinja/helpers.html @@ -0,0 +1,12 @@ +{% macro input_field(name, value='', type='text') -%} + +{%- endmacro %} + +{% macro textarea(name, value='', rows=10, cols=40) -%} + {{ + value|e }} +{%- endmacro %} + +{% macro form(action='', method='post') -%} +
{{ caller() }}
+{%- endmacro %} diff --git a/lib/jinja2/examples/rwbench/jinja/index.html b/lib/jinja2/examples/rwbench/jinja/index.html new file mode 100644 --- /dev/null +++ b/lib/jinja2/examples/rwbench/jinja/index.html @@ -0,0 +1,29 @@ +{% extends "layout.html" %} +{% from "helpers.html" import input_field, textarea, form %} +{% block page_title %}Index Page{% endblock %} +{% block body %} + {%- for article in articles if article.published %} +
+
{{ article.title|e }}
+
written by {{ article.user.username|e }} on {{ article.pub_date|dateformat }}
+
{{ article.body }}
+
+ {%- endfor %} + {%- call form() %} +
+
Name
+
{{ input_field('name') }}
+
E-Mail
+
{{ input_field('email') }}
+
URL
+
{{ input_field('url') }}
+
Comment
+
{{ textarea('comment') }}
+
Captcha
+
{{ input_field('captcha') }}
+
+ {{ input_field(type='submit', value='Submit') }} + {{ input_field('cancel', type='submit', value='Cancel') }} + {%- endcall %} +{% endblock %} diff --git a/lib/jinja2/examples/rwbench/jinja/layout.html b/lib/jinja2/examples/rwbench/jinja/layout.html new file mode 100644 --- /dev/null +++ b/lib/jinja2/examples/rwbench/jinja/layout.html @@ -0,0 +1,29 @@ + + + + {% block page_title %}{% endblock %} | RealWorld Benchmark + + + +
+
+
RealWorld Benchmark
+
+ A less stupid benchmark for Mako and Jinja2 to get an impression how + code changes affect runtime performance. +
+
+
+ {%- for href, caption in page_navigation %} +
{{ caption }}
+ {%- endfor %} +
+
+ {% block body %}{% endblock %} +
+
+ © Copyright 2008 by I don't know who. +
+
+ + diff --git a/lib/jinja2/examples/rwbench/mako/helpers.html b/lib/jinja2/examples/rwbench/mako/helpers.html new file mode 100644 --- /dev/null +++ b/lib/jinja2/examples/rwbench/mako/helpers.html @@ -0,0 +1,11 @@ +<%def name="input_field(name='', value='', type='text')"> + + + +<%def name="textarea(name, value='', rows=10, cols=40)"> + ${value|h} + + +<%def name="form(action='', method='post')"> +
${caller.body()}
+ diff --git a/lib/jinja2/examples/rwbench/mako/index.html b/lib/jinja2/examples/rwbench/mako/index.html new file mode 100644 --- /dev/null +++ b/lib/jinja2/examples/rwbench/mako/index.html @@ -0,0 +1,31 @@ +<%! + from rwbench import dateformat +%> +<%inherit file="layout.html" /> +<%namespace file="helpers.html" import="input_field, textarea, form" /> +<%def name="page_title()">Index Page +% for article in articles: + <% if not article.published: continue %> +
+
${article.title|h}
+
written by ${article.user.username|h} on ${dateformat(article.pub_date)}
+
${article.body}
+
+% endfor +<%call expr="form()"> +
+
Name
+
${input_field('name')}
+
E-Mail
+
${input_field('email')}
+
URL
+
${input_field('url')}
+
Comment
+
${textarea('comment')}
+
Captcha
+
${input_field('captcha')}
+
+ ${input_field(type='submit', value='Submit')} + ${input_field(name='cancel', type='submit', value='Cancel')} + diff --git a/lib/jinja2/examples/rwbench/mako/layout.html b/lib/jinja2/examples/rwbench/mako/layout.html new file mode 100644 --- /dev/null +++ b/lib/jinja2/examples/rwbench/mako/layout.html @@ -0,0 +1,30 @@ + + + + ${self.page_title()} | RealWorld Benchmark + + + +
+
+
RealWorld Benchmark
+
+ A less stupid benchmark for Mako and Jinja2 to get an impression how + code changes affect runtime performance. +
+
+
+ % for href, caption in page_navigation: +
${caption}
+ % endfor +
+
+ ${self.body()} +
+
+ © Copyright 2008 by I don't know who. +
+
+ + +<%def name="page_title()"> diff --git a/lib/jinja2/examples/rwbench/rwbench.py b/lib/jinja2/examples/rwbench/rwbench.py new file mode 100644 --- /dev/null +++ b/lib/jinja2/examples/rwbench/rwbench.py @@ -0,0 +1,112 @@ +# -*- coding: utf-8 -*- +""" + RealWorldish Benchmark + ~~~~~~~~~~~~~~~~~~~~~~ + + A more real-world benchmark of Jinja2. Like the other benchmark in the + Jinja2 repository this has no real-world usefulnes (despite the name). + Just go away and ignore it. NOW! + + :copyright: (c) 2009 by the Jinja Team. + :license: BSD. +""" +import sys +from os.path import join, dirname, abspath +try: + from cProfile import Profile +except ImportError: + from profile import Profile +from pstats import Stats +ROOT = abspath(dirname(__file__)) + +from random import choice, randrange +from datetime import datetime +from timeit import Timer +from jinja2 import Environment, FileSystemLoader +from jinja2.utils import generate_lorem_ipsum +from mako.lookup import TemplateLookup +from genshi.template import TemplateLoader as GenshiTemplateLoader + + +def dateformat(x): + return x.strftime('%Y-%m-%d') + + +jinja_env = Environment(loader=FileSystemLoader(join(ROOT, 'jinja'))) +jinja_env.filters['dateformat'] = dateformat +mako_lookup = TemplateLookup(directories=[join(ROOT, 'mako')]) +genshi_loader = GenshiTemplateLoader([join(ROOT, 'genshi')]) + +class Article(object): + + def __init__(self, id): + self.id = id + self.href = '/article/%d' % self.id + self.title = generate_lorem_ipsum(1, False, 5, 10) + self.user = choice(users) + self.body = generate_lorem_ipsum() + self.pub_date = datetime.utcfromtimestamp(randrange(10 ** 9, 2 * 10 ** 9)) + self.published = True + + +class User(object): + + def __init__(self, username): + self.href = '/user/%s' % username + self.username = username + + +users = map(User, [u'John Doe', u'Jane Doe', u'Peter Somewhat']) +articles = map(Article, range(20)) +navigation = [ + ('index', 'Index'), + ('about', 'About'), + ('foo?bar=1', 'Foo with Bar'), + ('foo?bar=2&s=x', 'Foo with X'), + ('blah', 'Blub Blah'), + ('hehe', 'Haha'), +] * 5 + +context = dict(users=users, articles=articles, page_navigation=navigation) + + +jinja_template = jinja_env.get_template('index.html') +mako_template = mako_lookup.get_template('index.html') +genshi_template = genshi_loader.load('index.html') + + +def test_jinja(): + jinja_template.render(context) + +def test_mako(): + mako_template.render_unicode(**context) + + +from djangoext import django_loader, DjangoContext +def test_django(): + # not cached because django is not thread safe and does + # not cache by itself so it would be unfair to cache it here. + django_template = django_loader.get_template('index.html') + django_template.render(DjangoContext(context)) + + +def test_genshi(): + genshi_template.generate(**context).render('html', doctype='html') + + +if __name__ == '__main__': + sys.stdout.write('Realworldish Benchmark:\n') + for test in 'jinja', 'mako', 'django', 'genshi': + t = Timer(setup='from __main__ import test_%s as bench' % test, + stmt='bench()') + sys.stdout.write(' >> %-20s' % test) + sys.stdout.flush() + sys.stdout.write('\r %-20s%.4f seconds\n' % (test, t.timeit(number=200) / 200)) + + if '-p' in sys.argv: + print 'Jinja profile' + p = Profile() + p.runcall(test_jinja) + stats = Stats(p) + stats.sort_stats('time', 'calls') + stats.print_stats() diff --git a/lib/jinja2/ext/Vim/jinja.vim b/lib/jinja2/ext/Vim/jinja.vim new file mode 100644 --- /dev/null +++ b/lib/jinja2/ext/Vim/jinja.vim @@ -0,0 +1,135 @@ +" Vim syntax file +" Language: Jinja template +" Maintainer: Armin Ronacher +" Last Change: 2008 May 9 +" Version: 1.1 +" +" Known Bugs: +" because of odd limitations dicts and the modulo operator +" appear wrong in the template. +" +" Changes: +" +" 2008 May 9: Added support for Jinja2 changes (new keyword rules) + +" .vimrc variable to disable html highlighting +if !exists('g:jinja_syntax_html') + let g:jinja_syntax_html=1 +endif + +" For version 5.x: Clear all syntax items +" For version 6.x: Quit when a syntax file was already loaded +if !exists("main_syntax") + if version < 600 + syntax clear + elseif exists("b:current_syntax") + finish +endif + let main_syntax = 'jinja' +endif + +" Pull in the HTML syntax. +if g:jinja_syntax_html + if version < 600 + so :p:h/html.vim + else + runtime! syntax/html.vim + unlet b:current_syntax + endif +endif + +syntax case match + +" Jinja template built-in tags and parameters (without filter, macro, is and raw, they +" have special threatment) +syn keyword jinjaStatement containedin=jinjaVarBlock,jinjaTagBlock,jinjaNested contained and if else in not or recursive as import + +syn keyword jinjaStatement containedin=jinjaVarBlock,jinjaTagBlock,jinjaNested contained is filter skipwhite nextgroup=jinjaFilter +syn keyword jinjaStatement containedin=jinjaTagBlock contained macro skipwhite nextgroup=jinjaFunction +syn keyword jinjaStatement containedin=jinjaTagBlock contained block skipwhite nextgroup=jinjaBlockName + +" Variable Names +syn match jinjaVariable containedin=jinjaVarBlock,jinjaTagBlock,jinjaNested contained skipwhite /[a-zA-Z_][a-zA-Z0-9_]*/ +syn keyword jinjaSpecial containedin=jinjaVarBlock,jinjaTagBlock,jinjaNested contained false true none False True None loop super caller varargs kwargs + +" Filters +syn match jinjaOperator "|" containedin=jinjaVarBlock,jinjaTagBlock,jinjaNested contained nextgroup=jinjaFilter +syn match jinjaFilter contained skipwhite /[a-zA-Z_][a-zA-Z0-9_]*/ +syn match jinjaFunction contained skipwhite /[a-zA-Z_][a-zA-Z0-9_]*/ +syn match jinjaBlockName contained skipwhite /[a-zA-Z_][a-zA-Z0-9_]*/ + +" Jinja template constants +syn region jinjaString containedin=jinjaVarBlock,jinjaTagBlock,jinjaNested contained start=/"/ skip=/\\"/ end=/"/ +syn region jinjaString containedin=jinjaVarBlock,jinjaTagBlock,jinjaNested contained start=/'/ skip=/\\'/ end=/'/ +syn match jinjaNumber containedin=jinjaVarBlock,jinjaTagBlock,jinjaNested contained /[0-9]\+$\.[0-9]\+$\?/ + +" Operators +syn match jinjaOperator containedin=jinjaVarBlock,jinjaTagBlock,jinjaNested contained /[+\-*\/<>=!,:]/ +syn match jinjaPunctuation containedin=jinjaVarBlock,jinjaTagBlock,jinjaNested contained /[()\[\]]/ +syn match jinjaOperator containedin=jinjaVarBlock,jinjaTagBlock,jinjaNested contained /\./ nextgroup=jinjaAttribute +syn match jinjaAttribute contained /[a-zA-Z_][a-zA-Z0-9_]*/ + +" Jinja template tag and variable blocks +syn region jinjaNested matchgroup=jinjaOperator start="(" end=")" transparent display containedin=jinjaVarBlock,jinjaTagBlock,jinjaNested contained +syn region jinjaNested matchgroup=jinjaOperator start="\[" end="\]" transparent display containedin=jinjaVarBlock,jinjaTagBlock,jinjaNested contained +syn region jinjaNested matchgroup=jinjaOperator start="{" end="}" transparent display containedin=jinjaVarBlock,jinjaTagBlock,jinjaNested contained +syn region jinjaTagBlock matchgroup=jinjaTagDelim start=/{%-\?/ end=/-\?%}/ skipwhite containedin=ALLBUT,jinjaTagBlock,jinjaVarBlock,jinjaRaw,jinjaString,jinjaNested,jinjaComment + +syn region jinjaVarBlock matchgroup=jinjaVarDelim start=/{{-\?/ end=/-\?}}/ containedin=ALLBUT,jinjaTagBlock,jinjaVarBlock,jinjaRaw,jinjaString,jinjaNested,jinjaComment + +" Jinja template 'raw' tag +syn region jinjaRaw matchgroup=jinjaRawDelim start="{%\s*raw\s*%}" end="{%\s*endraw\s*%}" containedin=ALLBUT,jinjaTagBlock,jinjaVarBlock,jinjaString,jinjaComment + +" Jinja comments +syn region jinjaComment matchgroup=jinjaCommentDelim start="{#" end="#}" containedin=ALLBUT,jinjaTagBlock,jinjaVarBlock,jinjaString + +" Block start keywords. A bit tricker. We only highlight at the start of a +" tag block and only if the name is not followed by a comma or equals sign +" which usually means that we have to deal with an assignment. +syn match jinjaStatement containedin=jinjaTagBlock contained skipwhite /${%-\?\s*$\@<=\<[a-zA-Z_][a-zA-Z0-9_]*\>$\s*[,=]$\@!/ + +" and context modifiers +syn match jinjaStatement containedin=jinjaTagBlock contained /\/ skipwhite + + +" Define the default highlighting. +" For version 5.7 and earlier: only when not done already +" For version 5.8 and later: only when an item doesn't have highlighting yet +if version >= 508 || !exists("did_jinja_syn_inits") + if version < 508 + let did_jinja_syn_inits = 1 + command -nargs=+ HiLink hi link + else + command -nargs=+ HiLink hi def link + endif + + HiLink jinjaPunctuation jinjaOperator + HiLink jinjaAttribute jinjaVariable + HiLink jinjaFunction jinjaFilter + + HiLink jinjaTagDelim jinjaTagBlock + HiLink jinjaVarDelim jinjaVarBlock + HiLink jinjaCommentDelim jinjaComment + HiLink jinjaRawDelim jinja + + HiLink jinjaSpecial Special + HiLink jinjaOperator Normal + HiLink jinjaRaw Normal + HiLink jinjaTagBlock PreProc + HiLink jinjaVarBlock PreProc + HiLink jinjaStatement Statement + HiLink jinjaFilter Function + HiLink jinjaBlockName Function + HiLink jinjaVariable Identifier + HiLink jinjaString Constant + HiLink jinjaNumber Constant + HiLink jinjaComment Comment + + delcommand HiLink +endif + +let b:current_syntax = "jinja" + +if main_syntax == 'jinja' + unlet main_syntax +endif diff --git a/lib/jinja2/ext/django2jinja/django2jinja.py b/lib/jinja2/ext/django2jinja/django2jinja.py new file mode 100644 --- /dev/null +++ b/lib/jinja2/ext/django2jinja/django2jinja.py @@ -0,0 +1,768 @@ +# -*- coding: utf-8 -*- +""" + Django to Jinja + ~~~~~~~~~~~~~~~ + + Helper module that can convert django templates into Jinja2 templates. + + This file is not intended to be used as stand alone application but to + be used as library. To convert templates you basically create your own + writer, add extra conversion logic for your custom template tags, + configure your django environment and run the `convert_templates` + function. + + Here a simple example:: + + # configure django (or use settings.configure) + import os + os.environ['DJANGO_SETTINGS_MODULE'] = 'yourapplication.settings' + from yourapplication.foo.templatetags.bar import MyNode + + from django2jinja import Writer, convert_templates + + def write_my_node(writer, node): + writer.start_variable() + writer.write('myfunc(') + for idx, arg in enumerate(node.args): + if idx: + writer.write(', ') + writer.node(arg) + writer.write(')') + writer.end_variable() + + writer = Writer() + writer.node_handlers[MyNode] = write_my_node + convert_templates('/path/to/output/folder', writer=writer) + + Here is an example hos to automatically translate your django + variables to jinja2:: + + import re + # List of tuple (Match pattern, Replace pattern, Exclusion pattern) + + var_re = ((re.compile(r"(u|user)\.is_authenticated"), r"\1.is_authenticated()", None), + (re.compile(r"\.non_field_errors"), r".non_field_errors()", None), + (re.compile(r"\.label_tag"), r".label_tag()", None), + (re.compile(r"\.as_dl"), r".as_dl()", None), + (re.compile(r"\.as_table"), r".as_table()", None), + (re.compile(r"\.as_widget"), r".as_widget()", None), + (re.compile(r"\.as_hidden"), r".as_hidden()", None), + + (re.compile(r"\.get_([0-9_\w]+)_url"), r".get_\1_url()", None), + (re.compile(r"\.url"), r".url()", re.compile(r"(form|calendar).url")), + (re.compile(r"\.get_([0-9_\w]+)_display"), r".get_\1_display()", None), + (re.compile(r"loop\.counter"), r"loop.index", None), + (re.compile(r"loop\.revcounter"), r"loop.revindex", None), + (re.compile(r"request\.GET\.([0-9_\w]+)"), r"request.GET.get('\1', '')", None), + (re.compile(r"request\.get_host"), r"request.get_host()", None), + + (re.compile(r"\.all(?!_)"), r".all()", None), + (re.compile(r"\.all\.0"), r".all()[0]", None), + (re.compile(r"\.([0-9])($|\s+)"), r"[\1]\2", None), + (re.compile(r"\.items"), r".items()", None), + ) + writer = Writer(var_re=var_re) + + For details about the writing process have a look at the module code. + + :copyright: (c) 2009 by the Jinja Team. + :license: BSD. +""" +import re +import os +import sys +from jinja2.defaults import * +from django.conf import settings +from django.template import defaulttags as core_tags, loader, TextNode, \ + FilterExpression, libraries, Variable, loader_tags, TOKEN_TEXT, \ + TOKEN_VAR +from django.template.debug import DebugVariableNode as VariableNode +from django.templatetags import i18n as i18n_tags +from StringIO import StringIO + + +_node_handlers = {} +_resolved_filters = {} +_newline_re = re.compile(r'(?:\r\n|\r|\n)') + + +# Django stores an itertools object on the cycle node. Not only is this +# thread unsafe but also a problem for the converter which needs the raw +# string values passed to the constructor to create a jinja loop.cycle() +# call from it. +_old_cycle_init = core_tags.CycleNode.__init__ +def _fixed_cycle_init(self, cyclevars, variable_name=None): + self.raw_cycle_vars = map(Variable, cyclevars) + _old_cycle_init(self, cyclevars, variable_name) +core_tags.CycleNode.__init__ = _fixed_cycle_init + + +def node(cls): + def proxy(f): + _node_handlers[cls] = f + return f + return proxy + + +def convert_templates(output_dir, extensions=('.html', '.txt'), writer=None, + callback=None): + """Iterates over all templates in the template dirs configured and + translates them and writes the new templates into the output directory. + """ + if writer is None: + writer = Writer() + + def filter_templates(files): + for filename in files: + ifilename = filename.lower() + for extension in extensions: + if ifilename.endswith(extension): + yield filename + + def translate(f, loadname): + template = loader.get_template(loadname) + original = writer.stream + writer.stream = f + writer.body(template.nodelist) + writer.stream = original + + if callback is None: + def callback(template): + print template + + for directory in settings.TEMPLATE_DIRS: + for dirname, _, files in os.walk(directory): + dirname = dirname[len(directory) + 1:] + for filename in filter_templates(files): + source = os.path.normpath(os.path.join(dirname, filename)) + target = os.path.join(output_dir, dirname, filename) + basetarget = os.path.dirname(target) + if not os.path.exists(basetarget): + os.makedirs(basetarget) + callback(source) + f = file(target, 'w') + try: + translate(f, source) + finally: + f.close() + + +class Writer(object): + """The core writer class.""" + + def __init__(self, stream=None, error_stream=None, + block_start_string=BLOCK_START_STRING, + block_end_string=BLOCK_END_STRING, + variable_start_string=VARIABLE_START_STRING, + variable_end_string=VARIABLE_END_STRING, + comment_start_string=COMMENT_START_STRING, + comment_end_string=COMMENT_END_STRING, + initial_autoescape=True, + use_jinja_autoescape=False, + custom_node_handlers=None, + var_re=[], + env=None): + if stream is None: + stream = sys.stdout + if error_stream is None: + error_stream = sys.stderr + self.stream = stream + self.error_stream = error_stream + self.block_start_string = block_start_string + self.block_end_string = block_end_string + self.variable_start_string = variable_start_string + self.variable_end_string = variable_end_string + self.comment_start_string = comment_start_string + self.comment_end_string = comment_end_string + self.autoescape = initial_autoescape + self.spaceless = False + self.use_jinja_autoescape = use_jinja_autoescape + self.node_handlers = dict(_node_handlers, + **(custom_node_handlers or {})) + self._loop_depth = 0 + self._filters_warned = set() + self.var_re = var_re + self.env = env + + def enter_loop(self): + """Increments the loop depth so that write functions know if they + are in a loop. + """ + self._loop_depth += 1 + + def leave_loop(self): + """Reverse of enter_loop.""" + self._loop_depth -= 1 + + @property + def in_loop(self): + """True if we are in a loop.""" + return self._loop_depth > 0 + + def write(self, s): + """Writes stuff to the stream.""" + self.stream.write(s.encode(settings.FILE_CHARSET)) + + def print_expr(self, expr): + """Open a variable tag, write to the string to the stream and close.""" + self.start_variable() + self.write(expr) + self.end_variable() + + def _post_open(self): + if self.spaceless: + self.write('- ') + else: + self.write(' ') + + def _pre_close(self): + if self.spaceless: + self.write(' -') + else: + self.write(' ') + + def start_variable(self): + """Start a variable.""" + self.write(self.variable_start_string) + self._post_open() + + def end_variable(self, always_safe=False): + """End a variable.""" + if not always_safe and self.autoescape and \ + not self.use_jinja_autoescape: + self.write('|e') + self._pre_close() + self.write(self.variable_end_string) + + def start_block(self): + """Starts a block.""" + self.write(self.block_start_string) + self._post_open() + + def end_block(self): + """Ends a block.""" + self._pre_close() + self.write(self.block_end_string) + + def tag(self, name): + """Like `print_expr` just for blocks.""" + self.start_block() + self.write(name) + self.end_block() + + def variable(self, name): + """Prints a variable. This performs variable name transformation.""" + self.write(self.translate_variable_name(name)) + + def literal(self, value): + """Writes a value as literal.""" + value = repr(value) + if value[:2] in ('u"', "u'"): + value = value[1:] + self.write(value) + + def filters(self, filters, is_block=False): + """Dumps a list of filters.""" + want_pipe = not is_block + for filter, args in filters: + name = self.get_filter_name(filter) + if name is None: + self.warn('Could not find filter %s' % name) + continue + if name not in DEFAULT_FILTERS and \ + name not in self._filters_warned: + self._filters_warned.add(name) + self.warn('Filter %s probably doesn\'t exist in Jinja' % + name) + if not want_pipe: + want_pipe = True + else: + self.write('|') + self.write(name) + if args: + self.write('(') + for idx, (is_var, value) in enumerate(args): + if idx: + self.write(', ') + if is_var: + self.node(value) + else: + self.literal(value) + self.write(')') + + def get_location(self, origin, position): + """Returns the location for an origin and position tuple as name + and lineno. + """ + if hasattr(origin, 'source'): + source = origin.source + name = '' + else: + source = origin.loader(origin.loadname, origin.dirs)[0] + name = origin.loadname + lineno = len(_newline_re.findall(source[:position[0]])) + 1 + return name, lineno + + def warn(self, message, node=None): + """Prints a warning to the error stream.""" + if node is not None and hasattr(node, 'source'): + filename, lineno = self.get_location(*node.source) + message = '[%s:%d] %s' % (filename, lineno, message) + print >> self.error_stream, message + + def translate_variable_name(self, var): + """Performs variable name translation.""" + if self.in_loop and var == 'forloop' or var.startswith('forloop.'): + var = var[3:] + + for reg, rep, unless in self.var_re: + no_unless = unless and unless.search(var) or True + if reg.search(var) and no_unless: + var = reg.sub(rep, var) + break + return var + + def get_filter_name(self, filter): + """Returns the filter name for a filter function or `None` if there + is no such filter. + """ + if filter not in _resolved_filters: + for library in libraries.values(): + for key, value in library.filters.iteritems(): + _resolved_filters[value] = key + return _resolved_filters.get(filter, None) + + def node(self, node): + """Invokes the node handler for a node.""" + for cls, handler in self.node_handlers.iteritems(): + if type(node) is cls or type(node).__name__ == cls: + handler(self, node) + break + else: + self.warn('Untranslatable node %s.%s found' % ( + node.__module__, + node.__class__.__name__ + ), node) + + def body(self, nodes): + """Calls node() for every node in the iterable passed.""" + for node in nodes: + self.node(node) + + + at node(TextNode) +def text_node(writer, node): + writer.write(node.s) + + + at node(Variable) +def variable(writer, node): + if node.translate: + writer.warn('i18n system used, make sure to install translations', node) + writer.write('_(') + if node.literal is not None: + writer.literal(node.literal) + else: + writer.variable(node.var) + if node.translate: + writer.write(')') + + + at node(VariableNode) +def variable_node(writer, node): + writer.start_variable() + if node.filter_expression.var.var == 'block.super' \ + and not node.filter_expression.filters: + writer.write('super()') + else: + writer.node(node.filter_expression) + writer.end_variable() + + + at node(FilterExpression) +def filter_expression(writer, node): + writer.node(node.var) + writer.filters(node.filters) + + + at node(core_tags.CommentNode) +def comment_tag(writer, node): + pass + + + at node(core_tags.DebugNode) +def comment_tag(writer, node): + writer.warn('Debug tag detected. Make sure to add a global function ' + 'called debug to the namespace.', node=node) + writer.print_expr('debug()') + + + at node(core_tags.ForNode) +def for_loop(writer, node): + writer.start_block() + writer.write('for ') + for idx, var in enumerate(node.loopvars): + if idx: + writer.write(', ') + writer.variable(var) + writer.write(' in ') + if node.is_reversed: + writer.write('(') + writer.node(node.sequence) + if node.is_reversed: + writer.write(')|reverse') + writer.end_block() + writer.enter_loop() + writer.body(node.nodelist_loop) + writer.leave_loop() + writer.tag('endfor') + + + at node(core_tags.IfNode) +def if_condition(writer, node): + writer.start_block() + writer.write('if ') + join_with = 'and' + if node.link_type == core_tags.IfNode.LinkTypes.or_: + join_with = 'or' + + for idx, (ifnot, expr) in enumerate(node.bool_exprs): + if idx: + writer.write(' %s ' % join_with) + if ifnot: + writer.write('not ') + writer.node(expr) + writer.end_block() + writer.body(node.nodelist_true) + if node.nodelist_false: + writer.tag('else') + writer.body(node.nodelist_false) + writer.tag('endif') + + + at node(core_tags.IfEqualNode) +def if_equal(writer, node): + writer.start_block() + writer.write('if ') + writer.node(node.var1) + if node.negate: + writer.write(' != ') + else: + writer.write(' == ') + writer.node(node.var2) + writer.end_block() + writer.body(node.nodelist_true) + if node.nodelist_false: + writer.tag('else') + writer.body(node.nodelist_false) + writer.tag('endif') + + + at node(loader_tags.BlockNode) +def block(writer, node): + writer.tag('block ' + node.name.replace('-', '_').rstrip('_')) + node = node + while node.parent is not None: + node = node.parent + writer.body(node.nodelist) + writer.tag('endblock') + + + at node(loader_tags.ExtendsNode) +def extends(writer, node): + writer.start_block() + writer.write('extends ') + if node.parent_name_expr: + writer.node(node.parent_name_expr) + else: + writer.literal(node.parent_name) + writer.end_block() + writer.body(node.nodelist) + + + at node(loader_tags.ConstantIncludeNode) + at node(loader_tags.IncludeNode) +def include(writer, node): + writer.start_block() + writer.write('include ') + if hasattr(node, 'template'): + writer.literal(node.template.name) + else: + writer.node(node.template_name) + writer.end_block() + + + at node(core_tags.CycleNode) +def cycle(writer, node): + if not writer.in_loop: + writer.warn('Untranslatable free cycle (cycle outside loop)', node=node) + return + if node.variable_name is not None: + writer.start_block() + writer.write('set %s = ' % node.variable_name) + else: + writer.start_variable() + writer.write('loop.cycle(') + for idx, var in enumerate(node.raw_cycle_vars): + if idx: + writer.write(', ') + writer.node(var) + writer.write(')') + if node.variable_name is not None: + writer.end_block() + else: + writer.end_variable() + + + at node(core_tags.FilterNode) +def filter(writer, node): + writer.start_block() + writer.write('filter ') + writer.filters(node.filter_expr.filters, True) + writer.end_block() + writer.body(node.nodelist) + writer.tag('endfilter') + + + at node(core_tags.AutoEscapeControlNode) +def autoescape_control(writer, node): + original = writer.autoescape + writer.autoescape = node.setting + writer.body(node.nodelist) + writer.autoescape = original + + + at node(core_tags.SpacelessNode) +def spaceless(writer, node): + original = writer.spaceless + writer.spaceless = True + writer.warn('entering spaceless mode with different semantics', node) + # do the initial stripping + nodelist = list(node.nodelist) + if nodelist: + if isinstance(nodelist[0], TextNode): + nodelist[0] = TextNode(nodelist[0].s.lstrip()) + if isinstance(nodelist[-1], TextNode): + nodelist[-1] = TextNode(nodelist[-1].s.rstrip()) + writer.body(nodelist) + writer.spaceless = original + + + at node(core_tags.TemplateTagNode) +def template_tag(writer, node): + tag = { + 'openblock': writer.block_start_string, + 'closeblock': writer.block_end_string, + 'openvariable': writer.variable_start_string, + 'closevariable': writer.variable_end_string, + 'opencomment': writer.comment_start_string, + 'closecomment': writer.comment_end_string, + 'openbrace': '{', + 'closebrace': '}' + }.get(node.tagtype) + if tag: + writer.start_variable() + writer.literal(tag) + writer.end_variable() + + + at node(core_tags.URLNode) +def url_tag(writer, node): + writer.warn('url node used. make sure to provide a proper url() ' + 'function', node) + if node.asvar: + writer.start_block() + writer.write('set %s = ' % node.asvar) + else: + writer.start_variable() + autoescape = writer.autoescape + writer.write('url(') + writer.literal(node.view_name) + for arg in node.args: + writer.write(', ') + writer.node(arg) + for key, arg in node.kwargs.items(): + writer.write(', %s=' % key) + writer.node(arg) + writer.write(')') + if node.asvar: + writer.end_block() + else: + writer.end_variable() + + + at node(core_tags.WidthRatioNode) +def width_ratio(writer, node): + writer.warn('widthratio expanded into formula. You may want to provide ' + 'a helper function for this calculation', node) + writer.start_variable() + writer.write('(') + writer.node(node.val_expr) + writer.write(' / ') + writer.node(node.max_expr) + writer.write(' * ') + writer.write(str(int(node.max_width))) + writer.write(')|round|int') + writer.end_variable(always_safe=True) + + + at node(core_tags.WithNode) +def with_block(writer, node): + writer.warn('with block expanded into set statement. This could cause ' + 'variables following that block to be overriden.', node) + writer.start_block() + writer.write('set %s = ' % node.name) + writer.node(node.var) + writer.end_block() + writer.body(node.nodelist) + + + at node(core_tags.RegroupNode) +def regroup(writer, node): + if node.expression.var.literal: + writer.warn('literal in groupby filter used. Behavior in that ' + 'situation is undefined and translation is skipped.', node) + return + elif node.expression.filters: + writer.warn('filters in groupby filter used. Behavior in that ' + 'situation is undefined which is most likely a bug ' + 'in your code. Filters were ignored.', node) + writer.start_block() + writer.write('set %s = ' % node.var_name) + writer.node(node.target) + writer.write('|groupby(') + writer.literal(node.expression.var.var) + writer.write(')') + writer.end_block() + + + at node(core_tags.LoadNode) +def warn_load(writer, node): + writer.warn('load statement used which was ignored on conversion', node) + + + at node(i18n_tags.GetAvailableLanguagesNode) +def get_available_languages(writer, node): + writer.warn('make sure to provide a get_available_languages function', node) + writer.tag('set %s = get_available_languages()' % + writer.translate_variable_name(node.variable)) + + + at node(i18n_tags.GetCurrentLanguageNode) +def get_current_language(writer, node): + writer.warn('make sure to provide a get_current_language function', node) + writer.tag('set %s = get_current_language()' % + writer.translate_variable_name(node.variable)) + + + at node(i18n_tags.GetCurrentLanguageBidiNode) +def get_current_language_bidi(writer, node): + writer.warn('make sure to provide a get_current_language_bidi function', node) + writer.tag('set %s = get_current_language_bidi()' % + writer.translate_variable_name(node.variable)) + + + at node(i18n_tags.TranslateNode) +def simple_gettext(writer, node): + writer.warn('i18n system used, make sure to install translations', node) + writer.start_variable() + writer.write('_(') + writer.node(node.value) + writer.write(')') + writer.end_variable() + + + at node(i18n_tags.BlockTranslateNode) +def translate_block(writer, node): + first_var = [] + variables = set() + + def touch_var(name): + variables.add(name) + if not first_var: + first_var.append(name) + + def dump_token_list(tokens): + for token in tokens: + if token.token_type == TOKEN_TEXT: + writer.write(token.contents) + elif token.token_type == TOKEN_VAR: + writer.print_expr(token.contents) + touch_var(token.contents) + + writer.warn('i18n system used, make sure to install translations', node) + writer.start_block() + writer.write('trans') + idx = -1 + for idx, (key, var) in enumerate(node.extra_context.items()): + if idx: + writer.write(',') + writer.write(' %s=' % key) + touch_var(key) + writer.node(var.filter_expression) + + have_plural = False + plural_var = None + if node.plural and node.countervar and node.counter: + have_plural = True + plural_var = node.countervar + if plural_var not in variables: + if idx > -1: + writer.write(',') + touch_var(plural_var) + writer.write(' %s=' % plural_var) + writer.node(node.counter) + + writer.end_block() + dump_token_list(node.singular) + if node.plural and node.countervar and node.counter: + writer.start_block() + writer.write('pluralize') + if node.countervar != first_var[0]: + writer.write(' ' + node.countervar) + writer.end_block() + dump_token_list(node.plural) + writer.tag('endtrans') + + at node("SimpleNode") +def simple_tag(writer, node): + """Check if the simple tag exist as a filter in """ + name = node.tag_name + if writer.env and \ + name not in writer.env.filters and \ + name not in writer._filters_warned: + writer._filters_warned.add(name) + writer.warn('Filter %s probably doesn\'t exist in Jinja' % + name) + + if not node.vars_to_resolve: + # No argument, pass the request + writer.start_variable() + writer.write('request|') + writer.write(name) + writer.end_variable() + return + + first_var = node.vars_to_resolve[0] + args = node.vars_to_resolve[1:] + writer.start_variable() + + # Copied from Writer.filters() + writer.node(first_var) + + writer.write('|') + writer.write(name) + if args: + writer.write('(') + for idx, var in enumerate(args): + if idx: + writer.write(', ') + if var.var: + writer.node(var) + else: + writer.literal(var.literal) + writer.write(')') + writer.end_variable() + +# get rid of node now, it shouldn't be used normally +del node diff --git a/lib/jinja2/ext/django2jinja/example.py b/lib/jinja2/ext/django2jinja/example.py new file mode 100644 --- /dev/null +++ b/lib/jinja2/ext/django2jinja/example.py @@ -0,0 +1,7 @@ +from django.conf import settings +settings.configure(TEMPLATE_DIRS=['templates'], TEMPLATE_DEBUG=True) + +from django2jinja import convert_templates, Writer + +writer = Writer(use_jinja_autoescape=True) +convert_templates('converted', writer=writer) diff --git a/lib/jinja2/ext/django2jinja/templates/index.html b/lib/jinja2/ext/django2jinja/templates/index.html new file mode 100644 --- /dev/null +++ b/lib/jinja2/ext/django2jinja/templates/index.html @@ -0,0 +1,58 @@ +{% extends "layout.html" %} +{% load i18n %} +{% block title %}Foo{% endblock %} +{% block page-body %} + {{ block.super }} + Hello {{ name|cut:"d" }}! + + {% for item in seq reversed %} + {% if forloop.index|divisibleby:2 %} +
{{ item }}
+ {% endif %} + {% endfor %} + {% ifequal foo bar %} + haha + {% else %} + hmm + {% endifequal %} + {% filter upper %} + {% include "subtemplate.html" %} + {% include foo %} + {% endfilter %} + {% spaceless %} + Hello World + {{ foo }} + Hmm + {% endspaceless %} + {% templatetag opencomment %}...{% templatetag closecomment %} + {% url foo a, b, c=d %} + {% url foo a, b, c=d as hmm %} + + {% with object.value as value %} + + {% endwith %} + +
{% debug %}
+ + {% blocktrans with book|title as book_t and author|title as author_t %} + This is {{ book_t }} by {{ author_t }} + {% endblocktrans %} + + {% blocktrans count list|length as counter %} + There is only one {{ name }} object. + {% plural %} + There are {{ counter }} {{ name }} objects. + {% endblocktrans %} + + {% blocktrans with name|escape as name count list|length as counter %} + There is only one {{ name }} object. + {% plural %} + There are {{ counter }} {{ name }} objects. + {% endblocktrans %} + + {% blocktrans %}This string will have {{ value }} inside.{% endblocktrans %} + +
{% trans "This is the title." %}
+ + {% regroup people by gender as grouped %} +{% endblock %} diff --git a/lib/jinja2/ext/django2jinja/templates/layout.html b/lib/jinja2/ext/django2jinja/templates/layout.html new file mode 100644 --- /dev/null +++ b/lib/jinja2/ext/django2jinja/templates/layout.html @@ -0,0 +1,4 @@ +{% block title %}{% endblock %} +
+ {% block page-body %}{% endblock %} +
diff --git a/lib/jinja2/ext/django2jinja/templates/subtemplate.html b/lib/jinja2/ext/django2jinja/templates/subtemplate.html new file mode 100644 --- /dev/null +++ b/lib/jinja2/ext/django2jinja/templates/subtemplate.html @@ -0,0 +1,1 @@ +Hello World! diff --git a/lib/jinja2/ext/djangojinja2.py b/lib/jinja2/ext/djangojinja2.py new file mode 100644 --- /dev/null +++ b/lib/jinja2/ext/djangojinja2.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- +""" + djangojinja2 + ~~~~~~~~~~~~ + + Adds support for Jinja2 to Django. + + Configuration variables: + + ======================= ============================================= + Key Description + ======================= ============================================= + `JINJA2_TEMPLATE_DIRS` List of template folders + `JINJA2_EXTENSIONS` List of Jinja2 extensions to use + `JINJA2_CACHE_SIZE` The size of the Jinja2 template cache. + ======================= ============================================= + + :copyright: (c) 2009 by the Jinja Team. + :license: BSD. +""" +from itertools import chain +from django.conf import settings +from django.http import HttpResponse +from django.core.exceptions import ImproperlyConfigured +from django.template.context import get_standard_processors +from django.template import TemplateDoesNotExist +from jinja2 import Environment, FileSystemLoader, TemplateNotFound +from jinja2.defaults import DEFAULT_NAMESPACE + + +# the environment is unconfigured until the first template is loaded. +_jinja_env = None + + +def get_env(): + """Get the Jinja2 env and initialize it if necessary.""" + global _jinja_env + if _jinja_env is None: + _jinja_env = create_env() + return _jinja_env + + +def create_env(): + """Create a new Jinja2 environment.""" + searchpath = list(settings.JINJA2_TEMPLATE_DIRS) + return Environment(loader=FileSystemLoader(searchpath), + auto_reload=settings.TEMPLATE_DEBUG, + cache_size=getattr(settings, 'JINJA2_CACHE_SIZE', 50), + extensions=getattr(settings, 'JINJA2_EXTENSIONS', ())) + + +def get_template(template_name, globals=None): + """Load a template.""" + try: + return get_env().get_template(template_name, globals=globals) + except TemplateNotFound, e: + raise TemplateDoesNotExist(str(e)) + + +def select_template(templates, globals=None): + """Try to load one of the given templates.""" + env = get_env() + for template in templates: + try: + return env.get_template(template, globals=globals) + except TemplateNotFound: + continue + raise TemplateDoesNotExist(', '.join(templates)) + + +def render_to_string(template_name, context=None, request=None, + processors=None): + """Render a template into a string.""" + context = dict(context or {}) + if request is not None: + context['request'] = request + for processor in chain(get_standard_processors(), processors or ()): + context.update(processor(request)) + return get_template(template_name).render(context) + + +def render_to_response(template_name, context=None, request=None, + processors=None, mimetype=None): + """Render a template into a response object.""" + return HttpResponse(render_to_string(template_name, context, request, + processors), mimetype=mimetype) diff --git a/lib/jinja2/ext/inlinegettext.py b/lib/jinja2/ext/inlinegettext.py new file mode 100644 --- /dev/null +++ b/lib/jinja2/ext/inlinegettext.py @@ -0,0 +1,78 @@ +# -*- coding: utf-8 -*- +""" + Inline Gettext + ~~~~~~~~~~~~~~ + + An example extension for Jinja2 that supports inline gettext calls. + Requires the i18n extension to be loaded. + + :copyright: (c) 2009 by the Jinja Team. + :license: BSD. +""" +import re +from jinja2.ext import Extension +from jinja2.lexer import Token, count_newlines +from jinja2.exceptions import TemplateSyntaxError + + +_outside_re = re.compile(r'\\?(gettext|_)\(') +_inside_re = re.compile(r'\\?[()]') + + +class InlineGettext(Extension): + """This extension implements support for inline gettext blocks:: + +
_(Welcome)
+
_(This is a paragraph)
+ + Requires the i18n extension to be loaded and configured. + """ + + def filter_stream(self, stream): + paren_stack = 0 + + for token in stream: + if token.type is not 'data': + yield token + continue + + pos = 0 + lineno = token.lineno + + while 1: + if not paren_stack: + match = _outside_re.search(token.value, pos) + else: + match = _inside_re.search(token.value, pos) + if match is None: + break + new_pos = match.start() + if new_pos > pos: + preval = token.value[pos:new_pos] + yield Token(lineno, 'data', preval) + lineno += count_newlines(preval) + gtok = match.group() + if gtok[0] == '\\': + yield Token(lineno, 'data', gtok[1:]) + elif not paren_stack: + yield Token(lineno, 'block_begin', None) + yield Token(lineno, 'name', 'trans') + yield Token(lineno, 'block_end', None) + paren_stack = 1 + else: + if gtok == '(' or paren_stack > 1: + yield Token(lineno, 'data', gtok) + paren_stack += gtok == ')' and -1 or 1 + if not paren_stack: + yield Token(lineno, 'block_begin', None) + yield Token(lineno, 'name', 'endtrans') + yield Token(lineno, 'block_end', None) + pos = match.end() + + if pos < len(token.value): + yield Token(lineno, 'data', token.value[pos:]) + + if paren_stack: + raise TemplateSyntaxError('unclosed gettext expression', + token.lineno, stream.name, + stream.filename) diff --git a/lib/jinja2/ext/jinja.el b/lib/jinja2/ext/jinja.el new file mode 100644 --- /dev/null +++ b/lib/jinja2/ext/jinja.el @@ -0,0 +1,128 @@ +;;; jinja.el --- Jinja mode highlighting +;; +;; Author: Georg Brandl +;; Copyright: (c) 2009 by the Jinja Team +;; Last modified: 2008-05-22 23:04 by gbr +;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;;; Commentary: +;; +;; Mostly ripped off django-mode by Lennart Borgman. +;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; This program is free software; you can redistribute it and/or +;; modify it under the terms of the GNU General Public License as +;; published by the Free Software Foundation; either version 2, or +;; (at your option) any later version. +;; +;; This program is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +;; General Public License for more details. +;; +;; You should have received a copy of the GNU General Public License +;; along with this program; see the file COPYING. If not, write to +;; the Free Software Foundation, Inc., 51 Franklin Street, Fifth +;; Floor, Boston, MA 02110-1301, USA. +;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;;; Code: + +(defconst jinja-font-lock-keywords + (list +; (cons (rx "{% comment %}" (submatch (0+ anything)) +; "{% endcomment %}") (list 1 font-lock-comment-face)) + '("{# ?\$.*?\$ ?#}" . (1 font-lock-comment-face)) + '("{%-?\\|-?%}\\|{{\\|}}" . font-lock-preprocessor-face) + '("{#\\|#}" . font-lock-comment-delimiter-face) + ;; first word in a block is a command + '("{%-?[ \t\n]*\$[a-zA-Z_]+\$" . (1 font-lock-keyword-face)) + ;; variables + '("\${{ ?\$\$[^|]*?\$\$|.*?\$? ?}}" . (1 font-lock-variable-name-face)) + ;; keywords and builtins + (cons (rx word-start + (or "in" "as" "recursive" "not" "and" "or" "if" "else" + "import" "with" "without" "context") + word-end) + font-lock-keyword-face) + (cons (rx word-start + (or "true" "false" "none" "loop" "self" "super") + word-end) + font-lock-builtin-face) + ;; tests + '("\$is\$[ \t]*\$not\$[ \t]*\$[a-zA-Z_]+\$" + (1 font-lock-keyword-face) (2 font-lock-keyword-face) + (3 font-lock-function-name-face)) + ;; builtin filters + (cons (rx + "|" (* space) + (submatch + (or "abs" "batch" "capitalize" "capture" "center" "count" "default" + "dformat" "dictsort" "e" "escape" "filesizeformat" "first" + "float" "format" "getattribute" "getitem" "groupby" "indent" + "int" "join" "jsonencode" "last" "length" "lower" "markdown" + "pprint" "random" "replace" "reverse" "round" "rst" "slice" + "sort" "string" "striptags" "sum" "textile" "title" "trim" + "truncate" "upper" "urlencode" "urlize" "wordcount" "wordwrap" + "xmlattr"))) + (list 1 font-lock-builtin-face)) + ) + "Minimal highlighting expressions for Jinja mode") + +(define-derived-mode jinja-mode nil "Jinja" + "Simple Jinja mode for use with `mumamo-mode'. +This mode only provides syntax highlighting." + ;;(set (make-local-variable 'comment-start) "{#") + ;;(set (make-local-variable 'comment-end) "#}") + (setq font-lock-defaults '(jinja-font-lock-keywords))) + +;; mumamo stuff +(when (require 'mumamo nil t) + + (defun mumamo-chunk-jinja3 (pos max) + "Find {# ... #}" + (mumamo-quick-chunk-forward pos max "{#" "#}" 'borders 'jinja-mode)) + + (defun mumamo-chunk-jinja2 (pos max) + "Find {{ ... }}" + (mumamo-quick-chunk-forward pos max "{{" "}}" 'borders 'jinja-mode)) + + (defun mumamo-chunk-jinja (pos max) + "Find {% ... %}" + (mumamo-quick-chunk-forward pos max "{%" "%}" 'borders 'jinja-mode)) + +;;;###autoload + (define-mumamo-multi-major-mode jinja-html-mumamo + "Turn on multiple major modes for Jinja with main mode `html-mode'. +This also covers inlined style and javascript." + ("Jinja HTML Family" html-mode + (mumamo-chunk-jinja + mumamo-chunk-jinja2 + mumamo-chunk-jinja3 + mumamo-chunk-inlined-style + mumamo-chunk-inlined-script + mumamo-chunk-style= + mumamo-chunk-onjs= + ))) + +;;;###autoload + (define-mumamo-multi-major-mode jinja-nxhtml-mumamo + "Turn on multiple major modes for Jinja with main mode `nxhtml-mode'. +This also covers inlined style and javascript." + ("Jinja nXhtml Family" nxhtml-mode + (mumamo-chunk-jinja + mumamo-chunk-jinja2 + mumamo-chunk-jinja3 + mumamo-chunk-inlined-style + mumamo-chunk-inlined-script + mumamo-chunk-style= + mumamo-chunk-onjs= + ))) + ) + +(provide 'jinja) +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; jinja.el ends here diff --git a/lib/jinja2/jinja2-debug.py b/lib/jinja2/jinja2-debug.py new file mode 100755 --- /dev/null +++ b/lib/jinja2/jinja2-debug.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" + Jinja2 Debug Interface + ~~~~~~~~~~~~~~~~~~~~~~ + + Helper script for internal Jinja2 debugging. Requires Werkzeug. + + :copyright: Copyright 2010 by Armin Ronacher. + :license: BSD. +""" +import sys +import jinja2 +from werkzeug import script + +env = jinja2.Environment(extensions=['jinja2.ext.i18n', 'jinja2.ext.do', + 'jinja2.ext.loopcontrols', + 'jinja2.ext.with_', + 'jinja2.ext.autoescape'], + autoescape=True) + +def shell_init_func(): + def _compile(x): + print env.compile(x, raw=True) + result = { + 'e': env, + 'c': _compile, + 't': env.from_string, + 'p': env.parse + } + for key in jinja2.__all__: + result[key] = getattr(jinja2, key) + return result + + +def action_compile(): + print env.compile(sys.stdin.read(), raw=True) + +action_shell = script.make_shell(shell_init_func) + + +if __name__ == '__main__': + script.run() diff --git a/lib/jinja2/jinja2/__init__.py b/lib/jinja2/jinja2/__init__.py new file mode 100644 --- /dev/null +++ b/lib/jinja2/jinja2/__init__.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- +""" + jinja2 + ~~~~~~ + + Jinja2 is a template engine written in pure Python. It provides a + Django inspired non-XML syntax but supports inline expressions and + an optional sandboxed environment. + + Nutshell + -------- + + Here a small example of a Jinja2 template:: + + {% extends 'base.html' %} + {% block title %}Memberlist{% endblock %} + {% block content %} +
+ {% for user in users %} +
{{ user.username }}
+ {% endfor %} +
+ {% endblock %} + + + :copyright: (c) 2010 by the Jinja Team. + :license: BSD, see LICENSE for more details. +""" +__docformat__ = 'restructuredtext en' +__version__ = '2.7-dev' + +# high level interface +from jinja2.environment import Environment, Template + +# loaders +from jinja2.loaders import BaseLoader, FileSystemLoader, PackageLoader, \ + DictLoader, FunctionLoader, PrefixLoader, ChoiceLoader, \ + ModuleLoader + +# bytecode caches +from jinja2.bccache import BytecodeCache, FileSystemBytecodeCache, \ + MemcachedBytecodeCache + +# undefined types +from jinja2.runtime import Undefined, DebugUndefined, StrictUndefined + +# exceptions +from jinja2.exceptions import TemplateError, UndefinedError, \ + TemplateNotFound, TemplatesNotFound, TemplateSyntaxError, \ + TemplateAssertionError + +# decorators and public utilities +from jinja2.filters import environmentfilter, contextfilter, \ + evalcontextfilter +from jinja2.utils import Markup, escape, clear_caches, \ + environmentfunction, evalcontextfunction, contextfunction, \ + is_undefined + +__all__ = [ + 'Environment', 'Template', 'BaseLoader', 'FileSystemLoader', + 'PackageLoader', 'DictLoader', 'FunctionLoader', 'PrefixLoader', + 'ChoiceLoader', 'BytecodeCache', 'FileSystemBytecodeCache', + 'MemcachedBytecodeCache', 'Undefined', 'DebugUndefined', + 'StrictUndefined', 'TemplateError', 'UndefinedError', 'TemplateNotFound', + 'TemplatesNotFound', 'TemplateSyntaxError', 'TemplateAssertionError', + 'ModuleLoader', 'environmentfilter', 'contextfilter', 'Markup', 'escape', + 'environmentfunction', 'contextfunction', 'clear_caches', 'is_undefined', + 'evalcontextfilter', 'evalcontextfunction' +] diff --git a/lib/jinja2/jinja2/_debugsupport.c b/lib/jinja2/jinja2/_debugsupport.c new file mode 100644 --- /dev/null +++ b/lib/jinja2/jinja2/_debugsupport.c @@ -0,0 +1,78 @@ +/** + * jinja2._debugsupport + * ~~~~~~~~~~~~~~~~~~~~ + * + * C implementation of `tb_set_next`. + * + * :copyright: (c) 2010 by the Jinja Team. + * :license: BSD. + */ + +#include + + +static PyObject* +tb_set_next(PyObject *self, PyObject *args) +{ + PyTracebackObject *tb, *old; + PyObject *next; + + if (!PyArg_ParseTuple(args, "O!O:tb_set_next", &PyTraceBack_Type, &tb, &next)) + return NULL; + if (next == Py_None) + next = NULL; + else if (!PyTraceBack_Check(next)) { + PyErr_SetString(PyExc_TypeError, + "tb_set_next arg 2 must be traceback or None"); + return NULL; + } + else + Py_INCREF(next); + + old = tb->tb_next; + tb->tb_next = (PyTracebackObject*)next; + Py_XDECREF(old); + + Py_INCREF(Py_None); + return Py_None; +} + +static PyMethodDef module_methods[] = { + {"tb_set_next", (PyCFunction)tb_set_next, METH_VARARGS, + "Set the tb_next member of a traceback object."}, + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + + +#if PY_MAJOR_VERSION < 3 + +#ifndef PyMODINIT_FUNC /* declarations for DLL import/export */ +#define PyMODINIT_FUNC void +#endif +PyMODINIT_FUNC +init_debugsupport(void) +{ + Py_InitModule3("jinja2._debugsupport", module_methods, ""); +} + +#else /* Python 3.x module initialization */ + +static struct PyModuleDef module_definition = { + PyModuleDef_HEAD_INIT, + "jinja2._debugsupport", + NULL, + -1, + module_methods, + NULL, + NULL, + NULL, + NULL +}; + +PyMODINIT_FUNC +PyInit__debugsupport(void) +{ + return PyModule_Create(&module_definition); +} + +#endif diff --git a/lib/jinja2/jinja2/_markupsafe/__init__.py b/lib/jinja2/jinja2/_markupsafe/__init__.py new file mode 100644 --- /dev/null +++ b/lib/jinja2/jinja2/_markupsafe/__init__.py @@ -0,0 +1,225 @@ +# -*- coding: utf-8 -*- +""" + markupsafe + ~~~~~~~~~~ + + Implements a Markup string. + + :copyright: (c) 2010 by Armin Ronacher. + :license: BSD, see LICENSE for more details. +""" +import re +from itertools import imap + + +__all__ = ['Markup', 'soft_unicode', 'escape', 'escape_silent'] + + +_striptags_re = re.compile(r'(|<[^>]*>)') +_entity_re = re.compile(r'&([^;]+);') + + +class Markup(unicode): + r"""Marks a string as being safe for inclusion in HTML/XML output without + needing to be escaped. This implements the `__html__` interface a couple + of frameworks and web applications use. :class:`Markup` is a direct + subclass of `unicode` and provides all the methods of `unicode` just that + it escapes arguments passed and always returns `Markup`. + + The `escape` function returns markup objects so that double escaping can't + happen. + + The constructor of the :class:`Markup` class can be used for three + different things: When passed an unicode object it's assumed to be safe, + when passed an object with an HTML representation (has an `__html__` + method) that representation is used, otherwise the object passed is + converted into a unicode string and then assumed to be safe: + + >>> Markup("Hello World!") + Markup(u'Hello World!') + >>> class Foo(object): + ... def __html__(self): + ... return 'foo' + ... + >>> Markup(Foo()) + Markup(u'foo') + + If you want object passed being always treated as unsafe you can use the + :meth:`escape` classmethod to create a :class:`Markup` object: + + >>> Markup.escape("Hello World!") + Markup(u'Hello <em>World</em>!') + + Operations on a markup string are markup aware which means that all + arguments are passed through the :func:`escape` function: + + >>> em = Markup("%s") + >>> em % "foo & bar" + Markup(u'foo & bar') + >>> strong = Markup("%(text)s") + >>> strong % {'text': 'hacker here'} + Markup(u'<blink>hacker here</blink>') + >>> Markup("Hello ") + "" + Markup(u'Hello <foo>') + """ + __slots__ = () + + def __new__(cls, base=u'', encoding=None, errors='strict'): + if hasattr(base, '__html__'): + base = base.__html__() + if encoding is None: + return unicode.__new__(cls, base) + return unicode.__new__(cls, base, encoding, errors) + + def __html__(self): + return self + + def __add__(self, other): + if hasattr(other, '__html__') or isinstance(other, basestring): + return self.__class__(unicode(self) + unicode(escape(other))) + return NotImplemented + + def __radd__(self, other): + if hasattr(other, '__html__') or isinstance(other, basestring): + return self.__class__(unicode(escape(other)) + unicode(self)) + return NotImplemented + + def __mul__(self, num): + if isinstance(num, (int, long)): + return self.__class__(unicode.__mul__(self, num)) + return NotImplemented + __rmul__ = __mul__ + + def __mod__(self, arg): + if isinstance(arg, tuple): + arg = tuple(imap(_MarkupEscapeHelper, arg)) + else: + arg = _MarkupEscapeHelper(arg) + return self.__class__(unicode.__mod__(self, arg)) + + def __repr__(self): + return '%s(%s)' % ( + self.__class__.__name__, + unicode.__repr__(self) + ) + + def join(self, seq): + return self.__class__(unicode.join(self, imap(escape, seq))) + join.__doc__ = unicode.join.__doc__ + + def split(self, *args, **kwargs): + return map(self.__class__, unicode.split(self, *args, **kwargs)) + split.__doc__ = unicode.split.__doc__ + + def rsplit(self, *args, **kwargs): + return map(self.__class__, unicode.rsplit(self, *args, **kwargs)) + rsplit.__doc__ = unicode.rsplit.__doc__ + + def splitlines(self, *args, **kwargs): + return map(self.__class__, unicode.splitlines(self, *args, **kwargs)) + splitlines.__doc__ = unicode.splitlines.__doc__ + + def unescape(self): + r"""Unescape markup again into an unicode string. This also resolves + known HTML4 and XHTML entities: + + >>> Markup("Main » About").unescape() + u'Main \xbb About' + """ + from jinja2._markupsafe._constants import HTML_ENTITIES + def handle_match(m): + name = m.group(1) + if name in HTML_ENTITIES: + return unichr(HTML_ENTITIES[name]) + try: + if name[:2] in ('#x', '#X'): + return unichr(int(name[2:], 16)) + elif name.startswith('#'): + return unichr(int(name[1:])) + except ValueError: + pass + return u'' + return _entity_re.sub(handle_match, unicode(self)) + + def striptags(self): + r"""Unescape markup into an unicode string and strip all tags. This + also resolves known HTML4 and XHTML entities. Whitespace is + normalized to one: + + >>> Markup("Main » About").striptags() + u'Main \xbb About' + """ + stripped = u' '.join(_striptags_re.sub('', self).split()) + return Markup(stripped).unescape() + + @classmethod + def escape(cls, s): + """Escape the string. Works like :func:`escape` with the difference + that for subclasses of :class:`Markup` this function would return the + correct subclass. + """ + rv = escape(s) + if rv.__class__ is not cls: + return cls(rv) + return rv + + def make_wrapper(name): + orig = getattr(unicode, name) + def func(self, *args, **kwargs): + args = _escape_argspec(list(args), enumerate(args)) + _escape_argspec(kwargs, kwargs.iteritems()) + return self.__class__(orig(self, *args, **kwargs)) + func.__name__ = orig.__name__ + func.__doc__ = orig.__doc__ + return func + + for method in '__getitem__', 'capitalize', \ + 'title', 'lower', 'upper', 'replace', 'ljust', \ + 'rjust', 'lstrip', 'rstrip', 'center', 'strip', \ + 'translate', 'expandtabs', 'swapcase', 'zfill': + locals()[method] = make_wrapper(method) + + # new in python 2.5 + if hasattr(unicode, 'partition'): + partition = make_wrapper('partition'), + rpartition = make_wrapper('rpartition') + + # new in python 2.6 + if hasattr(unicode, 'format'): + format = make_wrapper('format') + + # not in python 3 + if hasattr(unicode, '__getslice__'): + __getslice__ = make_wrapper('__getslice__') + + del method, make_wrapper + + +def _escape_argspec(obj, iterable): + """Helper for various string-wrapped functions.""" + for key, value in iterable: + if hasattr(value, '__html__') or isinstance(value, basestring): + obj[key] = escape(value) + return obj + + +class _MarkupEscapeHelper(object): + """Helper for Markup.__mod__""" + + def __init__(self, obj): + self.obj = obj + + __getitem__ = lambda s, x: _MarkupEscapeHelper(s.obj[x]) + __str__ = lambda s: str(escape(s.obj)) + __unicode__ = lambda s: unicode(escape(s.obj)) + __repr__ = lambda s: str(escape(repr(s.obj))) + __int__ = lambda s: int(s.obj) + __float__ = lambda s: float(s.obj) + + +# we have to import it down here as the speedups and native +# modules imports the markup type which is define above. +try: + from jinja2._markupsafe._speedups import escape, escape_silent, soft_unicode +except ImportError: + from jinja2._markupsafe._native import escape, escape_silent, soft_unicode diff --git a/lib/jinja2/jinja2/_markupsafe/_bundle.py b/lib/jinja2/jinja2/_markupsafe/_bundle.py new file mode 100644 --- /dev/null +++ b/lib/jinja2/jinja2/_markupsafe/_bundle.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- +""" + jinja2._markupsafe._bundle + ~~~~~~~~~~~~~~~~~~~~~~~~~~ + + This script pulls in markupsafe from a source folder and + bundles it with Jinja2. It does not pull in the speedups + module though. + + :copyright: Copyright 2010 by the Jinja team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" +import sys +import os +import re + + +def rewrite_imports(lines): + for idx, line in enumerate(lines): + new_line = re.sub(r'(import|from)\s+markupsafe\b', + r'\1 jinja2._markupsafe', line) + if new_line != line: + lines[idx] = new_line + + +def main(): + if len(sys.argv) != 2: + print 'error: only argument is path to markupsafe' + sys.exit(1) + basedir = os.path.dirname(__file__) + markupdir = sys.argv[1] + for filename in os.listdir(markupdir): + if filename.endswith('.py'): + f = open(os.path.join(markupdir, filename)) + try: + lines = list(f) + finally: + f.close() + rewrite_imports(lines) + f = open(os.path.join(basedir, filename), 'w') + try: + for line in lines: + f.write(line) + finally: + f.close() + + +if __name__ == '__main__': + main() diff --git a/lib/jinja2/jinja2/_markupsafe/_constants.py b/lib/jinja2/jinja2/_markupsafe/_constants.py new file mode 100644 --- /dev/null +++ b/lib/jinja2/jinja2/_markupsafe/_constants.py @@ -0,0 +1,267 @@ +# -*- coding: utf-8 -*- +""" + markupsafe._constants + ~~~~~~~~~~~~~~~~~~~~~ + + Highlevel implementation of the Markup string. + + :copyright: (c) 2010 by Armin Ronacher. + :license: BSD, see LICENSE for more details. +""" + + +HTML_ENTITIES = { + 'AElig': 198, + 'Aacute': 193, + 'Acirc': 194, + 'Agrave': 192, + 'Alpha': 913, + 'Aring': 197, + 'Atilde': 195, + 'Auml': 196, + 'Beta': 914, + 'Ccedil': 199, + 'Chi': 935, + 'Dagger': 8225, + 'Delta': 916, + 'ETH': 208, + 'Eacute': 201, + 'Ecirc': 202, + 'Egrave': 200, + 'Epsilon': 917, + 'Eta': 919, + 'Euml': 203, + 'Gamma': 915, + 'Iacute': 205, + 'Icirc': 206, + 'Igrave': 204, + 'Iota': 921, + 'Iuml': 207, + 'Kappa': 922, + 'Lambda': 923, + 'Mu': 924, + 'Ntilde': 209, + 'Nu': 925, + 'OElig': 338, + 'Oacute': 211, + 'Ocirc': 212, + 'Ograve': 210, + 'Omega': 937, + 'Omicron': 927, + 'Oslash': 216, + 'Otilde': 213, + 'Ouml': 214, + 'Phi': 934, + 'Pi': 928, + 'Prime': 8243, + 'Psi': 936, + 'Rho': 929, + 'Scaron': 352, + 'Sigma': 931, + 'THORN': 222, + 'Tau': 932, + 'Theta': 920, + 'Uacute': 218, + 'Ucirc': 219, + 'Ugrave': 217, + 'Upsilon': 933, + 'Uuml': 220, + 'Xi': 926, + 'Yacute': 221, + 'Yuml': 376, + 'Zeta': 918, + 'aacute': 225, + 'acirc': 226, + 'acute': 180, + 'aelig': 230, + 'agrave': 224, + 'alefsym': 8501, + 'alpha': 945, + 'amp': 38, + 'and': 8743, + 'ang': 8736, + 'apos': 39, + 'aring': 229, + 'asymp': 8776, + 'atilde': 227, + 'auml': 228, + 'bdquo': 8222, + 'beta': 946, + 'brvbar': 166, + 'bull': 8226, + 'cap': 8745, + 'ccedil': 231, + 'cedil': 184, + 'cent': 162, + 'chi': 967, + 'circ': 710, + 'clubs': 9827, + 'cong': 8773, + 'copy': 169, + 'crarr': 8629, + 'cup': 8746, + 'curren': 164, + 'dArr': 8659, + 'dagger': 8224, + 'darr': 8595, + 'deg': 176, + 'delta': 948, + 'diams': 9830, + 'divide': 247, + 'eacute': 233, + 'ecirc': 234, + 'egrave': 232, + 'empty': 8709, + 'emsp': 8195, + 'ensp': 8194, + 'epsilon': 949, + 'equiv': 8801, + 'eta': 951, + 'eth': 240, + 'euml': 235, + 'euro': 8364, + 'exist': 8707, + 'fnof': 402, + 'forall': 8704, + 'frac12': 189, + 'frac14': 188, + 'frac34': 190, + 'frasl': 8260, + 'gamma': 947, + 'ge': 8805, + 'gt': 62, + 'hArr': 8660, + 'harr': 8596, + 'hearts': 9829, + 'hellip': 8230, + 'iacute': 237, + 'icirc': 238, + 'iexcl': 161, + 'igrave': 236, + 'image': 8465, + 'infin': 8734, + 'int': 8747, + 'iota': 953, + 'iquest': 191, + 'isin': 8712, + 'iuml': 239, + 'kappa': 954, + 'lArr': 8656, + 'lambda': 955, + 'lang': 9001, + 'laquo': 171, + 'larr': 8592, + 'lceil': 8968, + 'ldquo': 8220, + 'le': 8804, + 'lfloor': 8970, + 'lowast': 8727, + 'loz': 9674, + 'lrm': 8206, + 'lsaquo': 8249, + 'lsquo': 8216, + 'lt': 60, + 'macr': 175, + 'mdash': 8212, + 'micro': 181, + 'middot': 183, + 'minus': 8722, + 'mu': 956, + 'nabla': 8711, + 'nbsp': 160, + 'ndash': 8211, + 'ne': 8800, + 'ni': 8715, + 'not': 172, + 'notin': 8713, + 'nsub': 8836, + 'ntilde': 241, + 'nu': 957, + 'oacute': 243, + 'ocirc': 244, + 'oelig': 339, + 'ograve': 242, + 'oline': 8254, + 'omega': 969, + 'omicron': 959, + 'oplus': 8853, + 'or': 8744, + 'ordf': 170, + 'ordm': 186, + 'oslash': 248, + 'otilde': 245, + 'otimes': 8855, + 'ouml': 246, + 'para': 182, + 'part': 8706, + 'permil': 8240, + 'perp': 8869, + 'phi': 966, + 'pi': 960, + 'piv': 982, + 'plusmn': 177, + 'pound': 163, + 'prime': 8242, + 'prod': 8719, + 'prop': 8733, + 'psi': 968, + 'quot': 34, + 'rArr': 8658, + 'radic': 8730, + 'rang': 9002, + 'raquo': 187, + 'rarr': 8594, + 'rceil': 8969, + 'rdquo': 8221, + 'real': 8476, + 'reg': 174, + 'rfloor': 8971, + 'rho': 961, + 'rlm': 8207, + 'rsaquo': 8250, + 'rsquo': 8217, + 'sbquo': 8218, + 'scaron': 353, + 'sdot': 8901, + 'sect': 167, + 'shy': 173, + 'sigma': 963, + 'sigmaf': 962, + 'sim': 8764, + 'spades': 9824, + 'sub': 8834, + 'sube': 8838, + 'sum': 8721, + 'sup': 8835, + 'sup1': 185, + 'sup2': 178, + 'sup3': 179, + 'supe': 8839, + 'szlig': 223, + 'tau': 964, + 'there4': 8756, + 'theta': 952, + 'thetasym': 977, + 'thinsp': 8201, + 'thorn': 254, + 'tilde': 732, + 'times': 215, + 'trade': 8482, + 'uArr': 8657, + 'uacute': 250, + 'uarr': 8593, + 'ucirc': 251, + 'ugrave': 249, + 'uml': 168, + 'upsih': 978, + 'upsilon': 965, + 'uuml': 252, + 'weierp': 8472, + 'xi': 958, + 'yacute': 253, + 'yen': 165, + 'yuml': 255, + 'zeta': 950, + 'zwj': 8205, + 'zwnj': 8204 +} diff --git a/lib/jinja2/jinja2/_markupsafe/_native.py b/lib/jinja2/jinja2/_markupsafe/_native.py new file mode 100644 --- /dev/null +++ b/lib/jinja2/jinja2/_markupsafe/_native.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- +""" + markupsafe._native + ~~~~~~~~~~~~~~~~~~ + + Native Python implementation the C module is not compiled. + + :copyright: (c) 2010 by Armin Ronacher. + :license: BSD, see LICENSE for more details. +""" +from jinja2._markupsafe import Markup + + +def escape(s): + """Convert the characters &, <, >, ' and " in string s to HTML-safe + sequences. Use this if you need to display text that might contain + such characters in HTML. Marks return value as markup string. + """ + if hasattr(s, '__html__'): + return s.__html__() + return Markup(unicode(s) + .replace('&', '&') + .replace('>', '>') + .replace('<', '<') + .replace("'", ''') + .replace('"', '"') + ) + + +def escape_silent(s): + """Like :func:`escape` but converts `None` into an empty + markup string. + """ + if s is None: + return Markup() + return escape(s) + + +def soft_unicode(s): + """Make a string unicode if it isn't already. That way a markup + string is not converted back to unicode. + """ + if not isinstance(s, unicode): + s = unicode(s) + return s diff --git a/lib/jinja2/jinja2/_markupsafe/tests.py b/lib/jinja2/jinja2/_markupsafe/tests.py new file mode 100644 --- /dev/null +++ b/lib/jinja2/jinja2/_markupsafe/tests.py @@ -0,0 +1,80 @@ +import gc +import unittest +from jinja2._markupsafe import Markup, escape, escape_silent + + +class MarkupTestCase(unittest.TestCase): + + def test_markup_operations(self): + # adding two strings should escape the unsafe one + unsafe = '' + safe = Markup('username') + assert unsafe + safe == unicode(escape(unsafe)) + unicode(safe) + + # string interpolations are safe to use too + assert Markup('%s') % '' == \ + '<bad user>' + assert Markup('%(username)s') % { + 'username': '' + } == '<bad user>' + + # an escaped object is markup too + assert type(Markup('foo') + 'bar') is Markup + + # and it implements __html__ by returning itself + x = Markup("foo") + assert x.__html__() is x + + # it also knows how to treat __html__ objects + class Foo(object): + def __html__(self): + return 'awesome' + def __unicode__(self): + return 'awesome' + assert Markup(Foo()) == 'awesome' + assert Markup('%s') % Foo() == \ + 'awesome' + + # escaping and unescaping + assert escape('"<>&\'') == '"<>&'' + assert Markup("Foo & Bar").striptags() == "Foo & Bar" + assert Markup("<test>").unescape() == "" + + def test_all_set(self): + import jinja2._markupsafe as markup + for item in markup.__all__: + getattr(markup, item) + + def test_escape_silent(self): + assert escape_silent(None) == Markup() + assert escape(None) == Markup(None) + assert escape_silent('') == Markup(u'<foo>') + + +class MarkupLeakTestCase(unittest.TestCase): + + def test_markup_leaks(self): + counts = set() + for count in xrange(20): + for item in xrange(1000): + escape("foo") + escape("") + escape(u"foo") + escape(u"") + counts.add(len(gc.get_objects())) + assert len(counts) == 1, 'ouch, c extension seems to leak objects' + + +def suite(): + suite = unittest.TestSuite() + suite.addTest(unittest.makeSuite(MarkupTestCase)) + + # this test only tests the c extension + if not hasattr(escape, 'func_code'): + suite.addTest(unittest.makeSuite(MarkupLeakTestCase)) + + return suite + + +if __name__ == '__main__': + unittest.main(defaultTest='suite') diff --git a/lib/jinja2/jinja2/_stringdefs.py b/lib/jinja2/jinja2/_stringdefs.py new file mode 100644 --- /dev/null +++ b/lib/jinja2/jinja2/_stringdefs.py @@ -0,0 +1,130 @@ +# -*- coding: utf-8 -*- +""" + jinja2._stringdefs + ~~~~~~~~~~~~~~~~~~ + + Strings of all Unicode characters of a certain category. + Used for matching in Unicode-aware languages. Run to regenerate. + + Inspired by chartypes_create.py from the MoinMoin project, original + implementation from Pygments. + + :copyright: Copyright 2006-2009 by the Jinja team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +Cc = u'\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f' + +Cf = u'\xad\u0600\u0601\u0602\u0603\u06dd\u070f\u17b4\u17b5\u200b\u200c\u200d\u200e\u200f\u202a\u202b\u202c\u202d\u202e\u2060\u2061\u2062\u2063\u206a\u206b\u206c\u206d\u206e\u206f\ufeff\ufff9\ufffa\ufffb' + +Cn = u'\u0242\u0243\u0244\u0245\u0246\u0247\u0248\u0249\u024a\u024b\u024c\u024d\u024e\u024f\u0370\u0371\u0372\u0373\u0376\u0377\u0378\u0379\u037b\u037c\u037d\u037f\u0380\u0381\u0382\u0383\u038b\u038d\u03a2\u03cf\u0487\u04cf\u04fa\u04fb\u04fc\u04fd\u04fe\u04ff\u0510\u0511\u0512\u0513\u0514\u0515\u0516\u0517\u0518\u0519\u051a\u051b\u051c\u051d\u051e\u051f\u0520\u0521\u0522\u0523\u0524\u0525\u0526\u0527\u0528\u0529\u052a\u052b\u052c\u052d\u052e\u052f\u0530\u0557\u0558\u0560\u0588\u058b\u058c\u058d\u058e\u058f\u0590\u05ba\u05c8\u05c9\u05ca\u05cb\u05cc\u05cd\u05ce\u05cf\u05eb\u05ec\u05ed\u05ee\u05ef\u05f5\u05f6\u05f7\u05f8\u05f9\u05fa\u05fb\u05fc\u05fd\u05fe\u05ff\u0604\u0605\u0606\u0607\u0608\u0609\u060a\u0616\u0617\u0618\u0619\u061a\u061c\u061d\u0620\u063b\u063c\u063d\u063e\u063f\u065f\u070e\u074b\u074c\u076e\u076f\u0770\u0771\u0772\u0773\u0774\u0775\u0776\u0777\u0778\u0779\u077a\u077b\u077c\u077d\u077e\u077f\u07b2\u07b3\u07b4\u07b5\u07b6\u07b7\u07b8\u07b9\u07ba\u07bb\u07bc\u07bd\u07be\u07bf\u07c0\u07c1\u07c2\u07c3\u07c4\u07c5\u07c6\u07c7\u07c8\u07c9\u07ca\u07cb\u07cc\u07cd\u07ce\u07cf\u07d0\u07d1\u07d2\u07d3\u07d4\u07d5\u07d6\u07d7\u07d8\u07d9\u07da\u07db\u07dc\u07dd\u07de\u07df\u07e0\u07e1\u07e2\u07e3\u07e4\u07e5\u07e6\u07e7\u07e8\u07e9\u07ea\u07eb\u07ec\u07ed\u07ee\u07ef\u07f0\u07f1\u07f2\u07f3\u07f4\u07f5\u07f6\u07f7\u07f8\u07f9\u07fa\u07fb\u07fc\u07fd\u07fe\u07ff\u0800\u0801\u0802\u0803\u0804\u0805\u0806\u0807\u0808\u0809\u080a\u080b\u080c\u080d\u080e\u080f\u0810\u0811\u0812\u0813\u0814\u0815\u0816\u0817\u0818\u0819\u081a\u081b\u081c\u081d\u081e\u081f\u0820\u0821\u0822\u0823\u0824\u0825\u0826\u0827\u0828\u0829\u082a\u082b\u082c\u082d\u082e\u082f\u0830\u0831\u0832\u0833\u0834\u0835\u0836\u0837\u0838\u0839\u083a\u083b\u083c\u083d\u083e\u083f\u0840\u0841\u0842\u0843\u0844\u0845\u0846\u0847\u0848\u0849\u084a\u084b\u084c\u084d\u084e\u084f\u0850\u0851\u0852\u0853\u0854\u0855\u0856\u0857\u0858\u0859\u085a\u085b\u085c\u085d\u085e\u085f\u0860\u0861\u0862\u0863\u0864\u0865\u0866\u0867\u0868\u0869\u086a\u086b\u086c\u086d\u086e\u086f\u0870\u0871\u0872\u0873\u0874\u0875\u0876\u0877\u0878\u0879\u087a\u087b\u087c\u087d\u087e\u087f\u0880\u0881\u0882\u0883\u0884\u0885\u0886\u0887\u0888\u0889\u088a\u088b\u088c\u088d\u088e\u088f\u0890\u0891\u0892\u0893\u0894\u0895\u0896\u0897\u0898\u0899\u089a\u089b\u089c\u089d\u089e\u089f\u08a0\u08a1\u08a2\u08a3\u08a4\u08a5\u08a6\u08a7\u08a8\u08a9\u08aa\u08ab\u08ac\u08ad\u08ae\u08af\u08b0\u08b1\u08b2\u08b3\u08b4\u08b5\u08b6\u08b7\u08b8\u08b9\u08ba\u08bb\u08bc\u08bd\u08be\u08bf\u08c0\u08c1\u08c2\u08c3\u08c4\u08c5\u08c6\u08c7\u08c8\u08c9\u08ca\u08cb\u08cc\u08cd\u08ce\u08cf\u08d0\u08d1\u08d2\u08d3\u08d4\u08d5\u08d6\u08d7\u08d8\u08d9\u08da\u08db\u08dc\u08dd\u08de\u08df\u08e0\u08e1\u08e2\u08e3\u08e4\u08e5\u08e6\u08e7\u08e8\u08e9\u08ea\u08eb\u08ec\u08ed\u08ee\u08ef\u08f0\u08f1\u08f2\u08f3\u08f4\u08f5\u08f6\u08f7\u08f8\u08f9\u08fa\u08fb\u08fc\u08fd\u08fe\u08ff\u0900\u093a\u093b\u094e\u094f\u0955\u0956\u0957\u0971\u0972\u0973\u0974\u0975\u0976\u0977\u0978\u0979\u097a\u097b\u097c\u097e\u097f\u0980\u0984\u098d\u098e\u0991\u0992\u09a9\u09b1\u09b3\u09b4\u09b5\u09ba\u09bb\u09c5\u09c6\u09c9\u09ca\u09cf\u09d0\u09d1\u09d2\u09d3\u09d4\u09d5\u09d6\u09d8\u09d9\u09da\u09db\u09de\u09e4\u09e5\u09fb\u09fc\u09fd\u09fe\u09ff\u0a00\u0a04\u0a0b\u0a0c\u0a0d\u0a0e\u0a11\u0a12\u0a29\u0a31\u0a34\u0a37\u0a3a\u0a3b\u0a3d\u0a43\u0a44\u0a45\u0a46\u0a49\u0a4a\u0a4e\u0a4f\u0a50\u0a51\u0a52\u0a53\u0a54\u0a55\u0a56\u0a57\u0a58\u0a5d\u0a5f\u0a60\u0a61\u0a62\u0a63\u0a64\u0a65\u0a75\u0a76\u0a77\u0a78\u0a79\u0a7a\u0a7b\u0a7c\u0a7d\u0a7e\u0a7f\u0a80\u0a84\u0a8e\u0a92\u0aa9\u0ab1\u0ab4\u0aba\u0abb\u0ac6\u0aca\u0ace\u0acf\u0ad1\u0ad2\u0ad3\u0ad4\u0ad5\u0ad6\u0ad7\u0ad8\u0ad9\u0ada\u0adb\u0adc\u0add\u0ade\u0adf\u0ae4\u0ae5\u0af0\u0af2\u0af3\u0af4\u0af5\u0af6\u0af7\u0af8\u0af9\u0afa\u0afb\u0afc\u0afd\u0afe\u0aff\u0b00\u0b04\u0b0d\u0b0e\u0b11\u0b12\u0b29\u0b31\u0b34\u0b3a\u0b3b\u0b44\u0b45\u0b46\u0b49\u0b4a\u0b4e\u0b4f\u0b50\u0b51\u0b52\u0b53\u0b54\u0b55\u0b58\u0b59\u0b5a\u0b5b\u0b5e\u0b62\u0b63\u0b64\u0b65\u0b72\u0b73\u0b74\u0b75\u0b76\u0b77\u0b78\u0b79\u0b7a\u0b7b\u0b7c\u0b7d\u0b7e\u0b7f\u0b80\u0b81\u0b84\u0b8b\u0b8c\u0b8d\u0b91\u0b96\u0b97\u0b98\u0b9b\u0b9d\u0ba0\u0ba1\u0ba2\u0ba5\u0ba6\u0ba7\u0bab\u0bac\u0bad\u0bba\u0bbb\u0bbc\u0bbd\u0bc3\u0bc4\u0bc5\u0bc9\u0bce\u0bcf\u0bd0\u0bd1\u0bd2\u0bd3\u0bd4\u0bd5\u0bd6\u0bd8\u0bd9\u0bda\u0bdb\u0bdc\u0bdd\u0bde\u0bdf\u0be0\u0be1\u0be2\u0be3\u0be4\u0be5\u0bfb\u0bfc\u0bfd\u0bfe\u0bff\u0c00\u0c04\u0c0d\u0c11\u0c29\u0c34\u0c3a\u0c3b\u0c3c\u0c3d\u0c45\u0c49\u0c4e\u0c4f\u0c50\u0c51\u0c52\u0c53\u0c54\u0c57\u0c58\u0c59\u0c5a\u0c5b\u0c5c\u0c5d\u0c5e\u0c5f\u0c62\u0c63\u0c64\u0c65\u0c70\u0c71\u0c72\u0c73\u0c74\u0c75\u0c76\u0c77\u0c78\u0c79\u0c7a\u0c7b\u0c7c\u0c7d\u0c7e\u0c7f\u0c80\u0c81\u0c84\u0c8d\u0c91\u0ca9\u0cb4\u0cba\u0cbb\u0cc5\u0cc9\u0cce\u0ccf\u0cd0\u0cd1\u0cd2\u0cd3\u0cd4\u0cd7\u0cd8\u0cd9\u0cda\u0cdb\u0cdc\u0cdd\u0cdf\u0ce2\u0ce3\u0ce4\u0ce5\u0cf0\u0cf1\u0cf2\u0cf3\u0cf4\u0cf5\u0cf6\u0cf7\u0cf8\u0cf9\u0cfa\u0cfb\u0cfc\u0cfd\u0cfe\u0cff\u0d00\u0d01\u0d04\u0d0d\u0d11\u0d29\u0d3a\u0d3b\u0d3c\u0d3d\u0d44\u0d45\u0d49\u0d4e\u0d4f\u0d50\u0d51\u0d52\u0d53\u0d54\u0d55\u0d56\u0d58\u0d59\u0d5a\u0d5b\u0d5c\u0d5d\u0d5e\u0d5f\u0d62\u0d63\u0d64\u0d65\u0d70\u0d71\u0d72\u0d73\u0d74\u0d75\u0d76\u0d77\u0d78\u0d79\u0d7a\u0d7b\u0d7c\u0d7d\u0d7e\u0d7f\u0d80\u0d81\u0d84\u0d97\u0d98\u0d99\u0db2\u0dbc\u0dbe\u0dbf\u0dc7\u0dc8\u0dc9\u0dcb\u0dcc\u0dcd\u0dce\u0dd5\u0dd7\u0de0\u0de1\u0de2\u0de3\u0de4\u0de5\u0de6\u0de7\u0de8\u0de9\u0dea\u0deb\u0dec\u0ded\u0dee\u0def\u0df0\u0df1\u0df5\u0df6\u0df7\u0df8\u0df9\u0dfa\u0dfb\u0dfc\u0dfd\u0dfe\u0dff\u0e00\u0e3b\u0e3c\u0e3d\u0e3e\u0e5c\u0e5d\u0e5e\u0e5f\u0e60\u0e61\u0e62\u0e63\u0e64\u0e65\u0e66\u0e67\u0e68\u0e69\u0e6a\u0e6b\u0e6c\u0e6d\u0e6e\u0e6f\u0e70\u0e71\u0e72\u0e73\u0e74\u0e75\u0e76\u0e77\u0e78\u0e79\u0e7a\u0e7b\u0e7c\u0e7d\u0e7e\u0e7f\u0e80\u0e83\u0e85\u0e86\u0e89\u0e8b\u0e8c\u0e8e\u0e8f\u0e90\u0e91\u0e92\u0e93\u0e98\u0ea0\u0ea4\u0ea6\u0ea8\u0ea9\u0eac\u0eba\u0ebe\u0ebf\u0ec5\u0ec7\u0ece\u0ecf\u0eda\u0edb\u0ede\u0edf\u0ee0\u0ee1\u0ee2\u0ee3\u0ee4\u0ee5\u0ee6\u0ee7\u0ee8\u0ee9\u0eea\u0eeb\u0eec\u0eed\u0eee\u0eef\u0ef0\u0ef1\u0ef2\u0ef3\u0ef4\u0ef5\u0ef6\u0ef7\u0ef8\u0ef9\u0efa\u0efb\u0efc\u0efd\u0efe\u0eff\u0f48\u0f6b\u0f6c\u0f6d\u0f6e\u0f6f\u0f70\u0f8c\u0f8d\u0f8e\u0f8f\u0f98\u0fbd\u0fcd\u0fce\u0fd2\u0fd3\u0fd4\u0fd5\u0fd6\u0fd7\u0fd8\u0fd9\u0fda\u0fdb\u0fdc\u0fdd\u0fde\u0fdf\u0fe0\u0fe1\u0fe2\u0fe3\u0fe4\u0fe5\u0fe6\u0fe7\u0fe8\u0fe9\u0fea\u0feb\u0fec\u0fed\u0fee\u0fef\u0ff0\u0ff1\u0ff2\u0ff3\u0ff4\u0ff5\u0ff6\u0ff7\u0ff8\u0ff9\u0ffa\u0ffb\u0ffc\u0ffd\u0ffe\u0fff\u1022\u1028\u102b\u1033\u1034\u1035\u103a\u103b\u103c\u103d\u103e\u103f\u105a\u105b\u105c\u105d\u105e\u105f\u1060\u1061\u1062\u1063\u1064\u1065\u1066\u1067\u1068\u1069\u106a\u106b\u106c\u106d\u106e\u106f\u1070\u1071\u1072\u1073\u1074\u1075\u1076\u1077\u1078\u1079\u107a\u107b\u107c\u107d\u107e\u107f\u1080\u1081\u1082\u1083\u1084\u1085\u1086\u1087\u1088\u1089\u108a\u108b\u108c\u108d\u108e\u108f\u1090\u1091\u1092\u1093\u1094\u1095\u1096\u1097\u1098\u1099\u109a\u109b\u109c\u109d\u109e\u109f\u10c6\u10c7\u10c8\u10c9\u10ca\u10cb\u10cc\u10cd\u10ce\u10cf\u10fd\u10fe\u10ff\u115a\u115b\u115c\u115d\u115e\u11a3\u11a4\u11a5\u11a6\u11a7\u11fa\u11fb\u11fc\u11fd\u11fe\u11ff\u1249\u124e\u124f\u1257\u1259\u125e\u125f\u1289\u128e\u128f\u12b1\u12b6\u12b7\u12bf\u12c1\u12c6\u12c7\u12d7\u1311\u1316\u1317\u135b\u135c\u135d\u135e\u137d\u137e\u137f\u139a\u139b\u139c\u139d\u139e\u139f\u13f5\u13f6\u13f7\u13f8\u13f9\u13fa\u13fb\u13fc\u13fd\u13fe\u13ff\u1400\u1677\u1678\u1679\u167a\u167b\u167c\u167d\u167e\u167f\u169d\u169e\u169f\u16f1\u16f2\u16f3\u16f4\u16f5\u16f6\u16f7\u16f8\u16f9\u16fa\u16fb\u16fc\u16fd\u16fe\u16ff\u170d\u1715\u1716\u1717\u1718\u1719\u171a\u171b\u171c\u171d\u171e\u171f\u1737\u1738\u1739\u173a\u173b\u173c\u173d\u173e\u173f\u1754\u1755\u1756\u1757\u1758\u1759\u175a\u175b\u175c\u175d\u175e\u175f\u176d\u1771\u1774\u1775\u1776\u1777\u1778\u1779\u177a\u177b\u177c\u177d\u177e\u177f\u17de\u17df\u17ea\u17eb\u17ec\u17ed\u17ee\u17ef\u17fa\u17fb\u17fc\u17fd\u17fe\u17ff\u180f\u181a\u181b\u181c\u181d\u181e\u181f\u1878\u1879\u187a\u187b\u187c\u187d\u187e\u187f\u18aa\u18ab\u18ac\u18ad\u18ae\u18af\u18b0\u18b1\u18b2\u18b3\u18b4\u18b5\u18b6\u18b7\u18b8\u18b9\u18ba\u18bb\u18bc\u18bd\u18be\u18bf\u18c0\u18c1\u18c2\u18c3\u18c4\u18c5\u18c6\u18c7\u18c8\u18c9\u18ca\u18cb\u18cc\u18cd\u18ce\u18cf\u18d0\u18d1\u18d2\u18d3\u18d4\u18d5\u18d6\u18d7\u18d8\u18d9\u18da\u18db\u18dc\u18dd\u18de\u18df\u18e0\u18e1\u18e2\u18e3\u18e4\u18e5\u18e6\u18e7\u18e8\u18e9\u18ea\u18eb\u18ec\u18ed\u18ee\u18ef\u18f0\u18f1\u18f2\u18f3\u18f4\u18f5\u18f6\u18f7\u18f8\u18f9\u18fa\u18fb\u18fc\u18fd\u18fe\u18ff\u191d\u191e\u191f\u192c\u192d\u192e\u192f\u193c\u193d\u193e\u193f\u1941\u1942\u1943\u196e\u196f\u1975\u1976\u1977\u1978\u1979\u197a\u197b\u197c\u197d\u197e\u197f\u19aa\u19ab\u19ac\u19ad\u19ae\u19af\u19ca\u19cb\u19cc\u19cd\u19ce\u19cf\u19da\u19db\u19dc\u19dd\u1a1c\u1a1d\u1a20\u1a21\u1a22\u1a23\u1a24\u1a25\u1a26\u1a27\u1a28\u1a29\u1a2a\u1a2b\u1a2c\u1a2d\u1a2e\u1a2f\u1a30\u1a31\u1a32\u1a33\u1a34\u1a35\u1a36\u1a37\u1a38\u1a39\u1a3a\u1a3b\u1a3c\u1a3d\u1a3e\u1a3f\u1a40\u1a41\u1a42\u1a43\u1a44\u1a45\u1a46\u1a47\u1a48\u1a49\u1a4a\u1a4b\u1a4c\u1a4d\u1a4e\u1a4f\u1a50\u1a51\u1a52\u1a53\u1a54\u1a55\u1a56\u1a57\u1a58\u1a59\u1a5a\u1a5b\u1a5c\u1a5d\u1a5e\u1a5f\u1a60\u1a61\u1a62\u1a63\u1a64\u1a65\u1a66\u1a67\u1a68\u1a69\u1a6a\u1a6b\u1a6c\u1a6d\u1a6e\u1a6f\u1a70\u1a71\u1a72\u1a73\u1a74\u1a75\u1a76\u1a77\u1a78\u1a79\u1a7a\u1a7b\u1a7c\u1a7d\u1a7e\u1a7f\u1a80\u1a81\u1a82\u1a83\u1a84\u1a85\u1a86\u1a87\u1a88\u1a89\u1a8a\u1a8b\u1a8c\u1a8d\u1a8e\u1a8f\u1a90\u1a91\u1a92\u1a93\u1a94\u1a95\u1a96\u1a97\u1a98\u1a99\u1a9a\u1a9b\u1a9c\u1a9d\u1a9e\u1a9f\u1aa0\u1aa1\u1aa2\u1aa3\u1aa4\u1aa5\u1aa6\u1aa7\u1aa8\u1aa9\u1aaa\u1aab\u1aac\u1aad\u1aae\u1aaf\u1ab0\u1ab1\u1ab2\u1ab3\u1ab4\u1ab5\u1ab6\u1ab7\u1ab8\u1ab9\u1aba\u1abb\u1abc\u1abd\u1abe\u1abf\u1ac0\u1ac1\u1ac2\u1ac3\u1ac4\u1ac5\u1ac6\u1ac7\u1ac8\u1ac9\u1aca\u1acb\u1acc\u1acd\u1ace\u1acf\u1ad0\u1ad1\u1ad2\u1ad3\u1ad4\u1ad5\u1ad6\u1ad7\u1ad8\u1ad9\u1ada\u1adb\u1adc\u1add\u1ade\u1adf\u1ae0\u1ae1\u1ae2\u1ae3\u1ae4\u1ae5\u1ae6\u1ae7\u1ae8\u1ae9\u1aea\u1aeb\u1aec\u1aed\u1aee\u1aef\u1af0\u1af1\u1af2\u1af3\u1af4\u1af5\u1af6\u1af7\u1af8\u1af9\u1afa\u1afb\u1afc\u1afd\u1afe\u1aff\u1b00\u1b01\u1b02\u1b03\u1b04\u1b05\u1b06\u1b07\u1b08\u1b09\u1b0a\u1b0b\u1b0c\u1b0d\u1b0e\u1b0f\u1b10\u1b11\u1b12\u1b13\u1b14\u1b15\u1b16\u1b17\u1b18\u1b19\u1b1a\u1b1b\u1b1c\u1b1d\u1b1e\u1b1f\u1b20\u1b21\u1b22\u1b23\u1b24\u1b25\u1b26\u1b27\u1b28\u1b29\u1b2a\u1b2b\u1b2c\u1b2d\u1b2e\u1b2f\u1b30\u1b31\u1b32\u1b33\u1b34\u1b35\u1b36\u1b37\u1b38\u1b39\u1b3a\u1b3b\u1b3c\u1b3d\u1b3e\u1b3f\u1b40\u1b41\u1b42\u1b43\u1b44\u1b45\u1b46\u1b47\u1b48\u1b49\u1b4a\u1b4b\u1b4c\u1b4d\u1b4e\u1b4f\u1b50\u1b51\u1b52\u1b53\u1b54\u1b55\u1b56\u1b57\u1b58\u1b59\u1b5a\u1b5b\u1b5c\u1b5d\u1b5e\u1b5f\u1b60\u1b61\u1b62\u1b63\u1b64\u1b65\u1b66\u1b67\u1b68\u1b69\u1b6a\u1b6b\u1b6c\u1b6d\u1b6e\u1b6f\u1b70\u1b71\u1b72\u1b73\u1b74\u1b75\u1b76\u1b77\u1b78\u1b79\u1b7a\u1b7b\u1b7c\u1b7d\u1b7e\u1b7f\u1b80\u1b81\u1b82\u1b83\u1b84\u1b85\u1b86\u1b87\u1b88\u1b89\u1b8a\u1b8b\u1b8c\u1b8d\u1b8e\u1b8f\u1b90\u1b91\u1b92\u1b93\u1b94\u1b95\u1b96\u1b97\u1b98\u1b99\u1b9a\u1b9b\u1b9c\u1b9d\u1b9e\u1b9f\u1ba0\u1ba1\u1ba2\u1ba3\u1ba4\u1ba5\u1ba6\u1ba7\u1ba8\u1ba9\u1baa\u1bab\u1bac\u1bad\u1bae\u1baf\u1bb0\u1bb1\u1bb2\u1bb3\u1bb4\u1bb5\u1bb6\u1bb7\u1bb8\u1bb9\u1bba\u1bbb\u1bbc\u1bbd\u1bbe\u1bbf\u1bc0\u1bc1\u1bc2\u1bc3\u1bc4\u1bc5\u1bc6\u1bc7\u1bc8\u1bc9\u1bca\u1bcb\u1bcc\u1bcd\u1bce\u1bcf\u1bd0\u1bd1\u1bd2\u1bd3\u1bd4\u1bd5\u1bd6\u1bd7\u1bd8\u1bd9\u1bda\u1bdb\u1bdc\u1bdd\u1bde\u1bdf\u1be0\u1be1\u1be2\u1be3\u1be4\u1be5\u1be6\u1be7\u1be8\u1be9\u1bea\u1beb\u1bec\u1bed\u1bee\u1bef\u1bf0\u1bf1\u1bf2\u1bf3\u1bf4\u1bf5\u1bf6\u1bf7\u1bf8\u1bf9\u1bfa\u1bfb\u1bfc\u1bfd\u1bfe\u1bff\u1c00\u1c01\u1c02\u1c03\u1c04\u1c05\u1c06\u1c07\u1c08\u1c09\u1c0a\u1c0b\u1c0c\u1c0d\u1c0e\u1c0f\u1c10\u1c11\u1c12\u1c13\u1c14\u1c15\u1c16\u1c17\u1c18\u1c19\u1c1a\u1c1b\u1c1c\u1c1d\u1c1e\u1c1f\u1c20\u1c21\u1c22\u1c23\u1c24\u1c25\u1c26\u1c27\u1c28\u1c29\u1c2a\u1c2b\u1c2c\u1c2d\u1c2e\u1c2f\u1c30\u1c31\u1c32\u1c33\u1c34\u1c35\u1c36\u1c37\u1c38\u1c39\u1c3a\u1c3b\u1c3c\u1c3d\u1c3e\u1c3f\u1c40\u1c41\u1c42\u1c43\u1c44\u1c45\u1c46\u1c47\u1c48\u1c49\u1c4a\u1c4b\u1c4c\u1c4d\u1c4e\u1c4f\u1c50\u1c51\u1c52\u1c53\u1c54\u1c55\u1c56\u1c57\u1c58\u1c59\u1c5a\u1c5b\u1c5c\u1c5d\u1c5e\u1c5f\u1c60\u1c61\u1c62\u1c63\u1c64\u1c65\u1c66\u1c67\u1c68\u1c69\u1c6a\u1c6b\u1c6c\u1c6d\u1c6e\u1c6f\u1c70\u1c71\u1c72\u1c73\u1c74\u1c75\u1c76\u1c77\u1c78\u1c79\u1c7a\u1c7b\u1c7c\u1c7d\u1c7e\u1c7f\u1c80\u1c81\u1c82\u1c83\u1c84\u1c85\u1c86\u1c87\u1c88\u1c89\u1c8a\u1c8b\u1c8c\u1c8d\u1c8e\u1c8f\u1c90\u1c91\u1c92\u1c93\u1c94\u1c95\u1c96\u1c97\u1c98\u1c99\u1c9a\u1c9b\u1c9c\u1c9d\u1c9e\u1c9f\u1ca0\u1ca1\u1ca2\u1ca3\u1ca4\u1ca5\u1ca6\u1ca7\u1ca8\u1ca9\u1caa\u1cab\u1cac\u1cad\u1cae\u1caf\u1cb0\u1cb1\u1cb2\u1cb3\u1cb4\u1cb5\u1cb6\u1cb7\u1cb8\u1cb9\u1cba\u1cbb\u1cbc\u1cbd\u1cbe\u1cbf\u1cc0\u1cc1\u1cc2\u1cc3\u1cc4\u1cc5\u1cc6\u1cc7\u1cc8\u1cc9\u1cca\u1ccb\u1ccc\u1ccd\u1cce\u1ccf\u1cd0\u1cd1\u1cd2\u1cd3\u1cd4\u1cd5\u1cd6\u1cd7\u1cd8\u1cd9\u1cda\u1cdb\u1cdc\u1cdd\u1cde\u1cdf\u1ce0\u1ce1\u1ce2\u1ce3\u1ce4\u1ce5\u1ce6\u1ce7\u1ce8\u1ce9\u1cea\u1ceb\u1cec\u1ced\u1cee\u1cef\u1cf0\u1cf1\u1cf2\u1cf3\u1cf4\u1cf5\u1cf6\u1cf7\u1cf8\u1cf9\u1cfa\u1cfb\u1cfc\u1cfd\u1cfe\u1cff\u1dc4\u1dc5\u1dc6\u1dc7\u1dc8\u1dc9\u1dca\u1dcb\u1dcc\u1dcd\u1dce\u1dcf\u1dd0\u1dd1\u1dd2\u1dd3\u1dd4\u1dd5\u1dd6\u1dd7\u1dd8\u1dd9\u1dda\u1ddb\u1ddc\u1ddd\u1dde\u1ddf\u1de0\u1de1\u1de2\u1de3\u1de4\u1de5\u1de6\u1de7\u1de8\u1de9\u1dea\u1deb\u1dec\u1ded\u1dee\u1def\u1df0\u1df1\u1df2\u1df3\u1df4\u1df5\u1df6\u1df7\u1df8\u1df9\u1dfa\u1dfb\u1dfc\u1dfd\u1dfe\u1dff\u1e9c\u1e9d\u1e9e\u1e9f\u1efa\u1efb\u1efc\u1efd\u1efe\u1eff\u1f16\u1f17\u1f1e\u1f1f\u1f46\u1f47\u1f4e\u1f4f\u1f58\u1f5a\u1f5c\u1f5e\u1f7e\u1f7f\u1fb5\u1fc5\u1fd4\u1fd5\u1fdc\u1ff0\u1ff1\u1ff5\u1fff\u2064\u2065\u2066\u2067\u2068\u2069\u2072\u2073\u208f\u2095\u2096\u2097\u2098\u2099\u209a\u209b\u209c\u209d\u209e\u209f\u20b6\u20b7\u20b8\u20b9\u20ba\u20bb\u20bc\u20bd\u20be\u20bf\u20c0\u20c1\u20c2\u20c3\u20c4\u20c5\u20c6\u20c7\u20c8\u20c9\u20ca\u20cb\u20cc\u20cd\u20ce\u20cf\u20ec\u20ed\u20ee\u20ef\u20f0\u20f1\u20f2\u20f3\u20f4\u20f5\u20f6\u20f7\u20f8\u20f9\u20fa\u20fb\u20fc\u20fd\u20fe\u20ff\u214d\u214e\u214f\u2150\u2151\u2152\u2184\u2185\u2186\u2187\u2188\u2189\u218a\u218b\u218c\u218d\u218e\u218f\u23dc\u23dd\u23de\u23df\u23e0\u23e1\u23e2\u23e3\u23e4\u23e5\u23e6\u23e7\u23e8\u23e9\u23ea\u23eb\u23ec\u23ed\u23ee\u23ef\u23f0\u23f1\u23f2\u23f3\u23f4\u23f5\u23f6\u23f7\u23f8\u23f9\u23fa\u23fb\u23fc\u23fd\u23fe\u23ff\u2427\u2428\u2429\u242a\u242b\u242c\u242d\u242e\u242f\u2430\u2431\u2432\u2433\u2434\u2435\u2436\u2437\u2438\u2439\u243a\u243b\u243c\u243d\u243e\u243f\u244b\u244c\u244d\u244e\u244f\u2450\u2451\u2452\u2453\u2454\u2455\u2456\u2457\u2458\u2459\u245a\u245b\u245c\u245d\u245e\u245f\u269d\u269e\u269f\u26b2\u26b3\u26b4\u26b5\u26b6\u26b7\u26b8\u26b9\u26ba\u26bb\u26bc\u26bd\u26be\u26bf\u26c0\u26c1\u26c2\u26c3\u26c4\u26c5\u26c6\u26c7\u26c8\u26c9\u26ca\u26cb\u26cc\u26cd\u26ce\u26cf\u26d0\u26d1\u26d2\u26d3\u26d4\u26d5\u26d6\u26d7\u26d8\u26d9\u26da\u26db\u26dc\u26dd\u26de\u26df\u26e0\u26e1\u26e2\u26e3\u26e4\u26e5\u26e6\u26e7\u26e8\u26e9\u26ea\u26eb\u26ec\u26ed\u26ee\u26ef\u26f0\u26f1\u26f2\u26f3\u26f4\u26f5\u26f6\u26f7\u26f8\u26f9\u26fa\u26fb\u26fc\u26fd\u26fe\u26ff\u2700\u2705\u270a\u270b\u2728\u274c\u274e\u2753\u2754\u2755\u2757\u275f\u2760\u2795\u2796\u2797\u27b0\u27bf\u27c7\u27c8\u27c9\u27ca\u27cb\u27cc\u27cd\u27ce\u27cf\u27ec\u27ed\u27ee\u27ef\u2b14\u2b15\u2b16\u2b17\u2b18\u2b19\u2b1a\u2b1b\u2b1c\u2b1d\u2b1e\u2b1f\u2b20\u2b21\u2b22\u2b23\u2b24\u2b25\u2b26\u2b27\u2b28\u2b29\u2b2a\u2b2b\u2b2c\u2b2d\u2b2e\u2b2f\u2b30\u2b31\u2b32\u2b33\u2b34\u2b35\u2b36\u2b37\u2b38\u2b39\u2b3a\u2b3b\u2b3c\u2b3d\u2b3e\u2b3f\u2b40\u2b41\u2b42\u2b43\u2b44\u2b45\u2b46\u2b47\u2b48\u2b49\u2b4a\u2b4b\u2b4c\u2b4d\u2b4e\u2b4f\u2b50\u2b51\u2b52\u2b53\u2b54\u2b55\u2b56\u2b57\u2b58\u2b59\u2b5a\u2b5b\u2b5c\u2b5d\u2b5e\u2b5f\u2b60\u2b61\u2b62\u2b63\u2b64\u2b65\u2b66\u2b67\u2b68\u2b69\u2b6a\u2b6b\u2b6c\u2b6d\u2b6e\u2b6f\u2b70\u2b71\u2b72\u2b73\u2b74\u2b75\u2b76\u2b77\u2b78\u2b79\u2b7a\u2b7b\u2b7c\u2b7d\u2b7e\u2b7f\u2b80\u2b81\u2b82\u2b83\u2b84\u2b85\u2b86\u2b87\u2b88\u2b89\u2b8a\u2b8b\u2b8c\u2b8d\u2b8e\u2b8f\u2b90\u2b91\u2b92\u2b93\u2b94\u2b95\u2b96\u2b97\u2b98\u2b99\u2b9a\u2b9b\u2b9c\u2b9d\u2b9e\u2b9f\u2ba0\u2ba1\u2ba2\u2ba3\u2ba4\u2ba5\u2ba6\u2ba7\u2ba8\u2ba9\u2baa\u2bab\u2bac\u2bad\u2bae\u2baf\u2bb0\u2bb1\u2bb2\u2bb3\u2bb4\u2bb5\u2bb6\u2bb7\u2bb8\u2bb9\u2bba\u2bbb\u2bbc\u2bbd\u2bbe\u2bbf\u2bc0\u2bc1\u2bc2\u2bc3\u2bc4\u2bc5\u2bc6\u2bc7\u2bc8\u2bc9\u2bca\u2bcb\u2bcc\u2bcd\u2bce\u2bcf\u2bd0\u2bd1\u2bd2\u2bd3\u2bd4\u2bd5\u2bd6\u2bd7\u2bd8\u2bd9\u2bda\u2bdb\u2bdc\u2bdd\u2bde\u2bdf\u2be0\u2be1\u2be2\u2be3\u2be4\u2be5\u2be6\u2be7\u2be8\u2be9\u2bea\u2beb\u2bec\u2bed\u2bee\u2bef\u2bf0\u2bf1\u2bf2\u2bf3\u2bf4\u2bf5\u2bf6\u2bf7\u2bf8\u2bf9\u2bfa\u2bfb\u2bfc\u2bfd\u2bfe\u2bff\u2c2f\u2c5f\u2c60\u2c61\u2c62\u2c63\u2c64\u2c65\u2c66\u2c67\u2c68\u2c69\u2c6a\u2c6b\u2c6c\u2c6d\u2c6e\u2c6f\u2c70\u2c71\u2c72\u2c73\u2c74\u2c75\u2c76\u2c77\u2c78\u2c79\u2c7a\u2c7b\u2c7c\u2c7d\u2c7e\u2c7f\u2ceb\u2cec\u2ced\u2cee\u2cef\u2cf0\u2cf1\u2cf2\u2cf3\u2cf4\u2cf5\u2cf6\u2cf7\u2cf8\u2d26\u2d27\u2d28\u2d29\u2d2a\u2d2b\u2d2c\u2d2d\u2d2e\u2d2f\u2d66\u2d67\u2d68\u2d69\u2d6a\u2d6b\u2d6c\u2d6d\u2d6e\u2d70\u2d71\u2d72\u2d73\u2d74\u2d75\u2d76\u2d77\u2d78\u2d79\u2d7a\u2d7b\u2d7c\u2d7d\u2d7e\u2d7f\u2d97\u2d98\u2d99\u2d9a\u2d9b\u2d9c\u2d9d\u2d9e\u2d9f\u2da7\u2daf\u2db7\u2dbf\u2dc7\u2dcf\u2dd7\u2ddf\u2de0\u2de1\u2de2\u2de3\u2de4\u2de5\u2de6\u2de7\u2de8\u2de9\u2dea\u2deb\u2dec\u2ded\u2dee\u2def\u2df0\u2df1\u2df2\u2df3\u2df4\u2df5\u2df6\u2df7\u2df8\u2df9\u2dfa\u2dfb\u2dfc\u2dfd\u2dfe\u2dff\u2e18\u2e19\u2e1a\u2e1b\u2e1e\u2e1f\u2e20\u2e21\u2e22\u2e23\u2e24\u2e25\u2e26\u2e27\u2e28\u2e29\u2e2a\u2e2b\u2e2c\u2e2d\u2e2e\u2e2f\u2e30\u2e31\u2e32\u2e33\u2e34\u2e35\u2e36\u2e37\u2e38\u2e39\u2e3a\u2e3b\u2e3c\u2e3d\u2e3e\u2e3f\u2e40\u2e41\u2e42\u2e43\u2e44\u2e45\u2e46\u2e47\u2e48\u2e49\u2e4a\u2e4b\u2e4c\u2e4d\u2e4e\u2e4f\u2e50\u2e51\u2e52\u2e53\u2e54\u2e55\u2e56\u2e57\u2e58\u2e59\u2e5a\u2e5b\u2e5c\u2e5d\u2e5e\u2e5f\u2e60\u2e61\u2e62\u2e63\u2e64\u2e65\u2e66\u2e67\u2e68\u2e69\u2e6a\u2e6b\u2e6c\u2e6d\u2e6e\u2e6f\u2e70\u2e71\u2e72\u2e73\u2e74\u2e75\u2e76\u2e77\u2e78\u2e79\u2e7a\u2e7b\u2e7c\u2e7d\u2e7e\u2e7f\u2e9a\u2ef4\u2ef5\u2ef6\u2ef7\u2ef8\u2ef9\u2efa\u2efb\u2efc\u2efd\u2efe\u2eff\u2fd6\u2fd7\u2fd8\u2fd9\u2fda\u2fdb\u2fdc\u2fdd\u2fde\u2fdf\u2fe0\u2fe1\u2fe2\u2fe3\u2fe4\u2fe5\u2fe6\u2fe7\u2fe8\u2fe9\u2fea\u2feb\u2fec\u2fed\u2fee\u2fef\u2ffc\u2ffd\u2ffe\u2fff\u3040\u3097\u3098\u3100\u3101\u3102\u3103\u3104\u312d\u312e\u312f\u3130\u318f\u31b8\u31b9\u31ba\u31bb\u31bc\u31bd\u31be\u31bf\u31d0\u31d1\u31d2\u31d3\u31d4\u31d5\u31d6\u31d7\u31d8\u31d9\u31da\u31db\u31dc\u31dd\u31de\u31df\u31e0\u31e1\u31e2\u31e3\u31e4\u31e5\u31e6\u31e7\u31e8\u31e9\u31ea\u31eb\u31ec\u31ed\u31ee\u31ef\u321f\u3244\u3245\u3246\u3247\u3248\u3249\u324a\u324b\u324c\u324d\u324e\u324f\u32ff\u4db6\u4db7\u4db8\u4db9\u4dba\u4dbb\u4dbc\u4dbd\u4dbe\u4dbf\u9fbc\u9fbd\u9fbe\u9fbf\u9fc0\u9fc1\u9fc2\u9fc3\u9fc4\u9fc5\u9fc6\u9fc7\u9fc8\u9fc9\u9fca\u9fcb\u9fcc\u9fcd\u9fce\u9fcf\u9fd0\u9fd1\u9fd2\u9fd3\u9fd4\u9fd5\u9fd6\u9fd7\u9fd8\u9fd9\u9fda\u9fdb\u9fdc\u9fdd\u9fde\u9fdf\u9fe0\u9fe1\u9fe2\u9fe3\u9fe4\u9fe5\u9fe6\u9fe7\u9fe8\u9fe9\u9fea\u9feb\u9fec\u9fed\u9fee\u9fef\u9ff0\u9ff1\u9ff2\u9ff3\u9ff4\u9ff5\u9ff6\u9ff7\u9ff8\u9ff9\u9ffa\u9ffb\u9ffc\u9ffd\u9ffe\u9fff\ua48d\ua48e\ua48f\ua4c7\ua4c8\ua4c9\ua4ca\ua4cb\ua4cc\ua4cd\ua4ce\ua4cf\ua4d0\ua4d1\ua4d2\ua4d3\ua4d4\ua4d5\ua4d6\ua4d7\ua4d8\ua4d9\ua4da\ua4db\ua4dc\ua4dd\ua4de\ua4df\ua4e0\ua4e1\ua4e2\ua4e3\ua4e4\ua4e5\ua4e6\ua4e7\ua4e8\ua4e9\ua4ea\ua4eb\ua4ec\ua4ed\ua4ee\ua4ef\ua4f0\ua4f1\ua4f2\ua4f3\ua4f4\ua4f5\ua4f6\ua4f7\ua4f8\ua4f9\ua4fa\ua4fb\ua4fc\ua4fd\ua4fe\ua4ff\ua500\ua501\ua502\ua503\ua504\ua505\ua506\ua507\ua508\ua509\ua50a\ua50b\ua50c\ua50d\ua50e\ua50f\ua510\ua511\ua512\ua513\ua514\ua515\ua516\ua517\ua518\ua519\ua51a\ua51b\ua51c\ua51d\ua51e\ua51f\ua520\ua521\ua522\ua523\ua524\ua525\ua526\ua527\ua528\ua529\ua52a\ua52b\ua52c\ua52d\ua52e\ua52f\ua530\ua531\ua532\ua533\ua534\ua535\ua536\ua537\ua538\ua539\ua53a\ua53b\ua53c\ua53d\ua53e\ua53f\ua540\ua541\ua542\ua543\ua544\ua545\ua546\ua547\ua548\ua549\ua54a\ua54b\ua54c\ua54d\ua54e\ua54f\ua550\ua551\ua552\ua553\ua554\ua555\ua556\ua557\ua558\ua559\ua55a\ua55b\ua55c\ua55d\ua55e\ua55f\ua560\ua561\ua562\ua563\ua564\ua565\ua566\ua567\ua568\ua569\ua56a\ua56b\ua56c\ua56d\ua56e\ua56f\ua570\ua571\ua572\ua573\ua574\ua575\ua576\ua577\ua578\ua579\ua57a\ua57b\ua57c\ua57d\ua57e\ua57f\ua580\ua581\ua582\ua583\ua584\ua585\ua586\ua587\ua588\ua589\ua58a\ua58b\ua58c\ua58d\ua58e\ua58f\ua590\ua591\ua592\ua593\ua594\ua595\ua596\ua597\ua598\ua599\ua59a\ua59b\ua59c\ua59d\ua59e\ua59f\ua5a0\ua5a1\ua5a2\ua5a3\ua5a4\ua5a5\ua5a6\ua5a7\ua5a8\ua5a9\ua5aa\ua5ab\ua5ac\ua5ad\ua5ae\ua5af\ua5b0\ua5b1\ua5b2\ua5b3\ua5b4\ua5b5\ua5b6\ua5b7\ua5b8\ua5b9\ua5ba\ua5bb\ua5bc\ua5bd\ua5be\ua5bf\ua5c0\ua5c1\ua5c2\ua5c3\ua5c4\ua5c5\ua5c6\ua5c7\ua5c8\ua5c9\ua5ca\ua5cb\ua5cc\ua5cd\ua5ce\ua5cf\ua5d0\ua5d1\ua5d2\ua5d3\ua5d4\ua5d5\ua5d6\ua5d7\ua5d8\ua5d9\ua5da\ua5db\ua5dc\ua5dd\ua5de\ua5df\ua5e0\ua5e1\ua5e2\ua5e3\ua5e4\ua5e5\ua5e6\ua5e7\ua5e8\ua5e9\ua5ea\ua5eb\ua5ec\ua5ed\ua5ee\ua5ef\ua5f0\ua5f1\ua5f2\ua5f3\ua5f4\ua5f5\ua5f6\ua5f7\ua5f8\ua5f9\ua5fa\ua5fb\ua5fc\ua5fd\ua5fe\ua5ff\ua600\ua601\ua602\ua603\ua604\ua605\ua606\ua607\ua608\ua609\ua60a\ua60b\ua60c\ua60d\ua60e\ua60f\ua610\ua611\ua612\ua613\ua614\ua615\ua616\ua617\ua618\ua619\ua61a\ua61b\ua61c\ua61d\ua61e\ua61f\ua620\ua621\ua622\ua623\ua624\ua625\ua626\ua627\ua628\ua629\ua62a\ua62b\ua62c\ua62d\ua62e\ua62f\ua630\ua631\ua632\ua633\ua634\ua635\ua636\ua637\ua638\ua639\ua63a\ua63b\ua63c\ua63d\ua63e\ua63f\ua640\ua641\ua642\ua643\ua644\ua645\ua646\ua647\ua648\ua649\ua64a\ua64b\ua64c\ua64d\ua64e\ua64f\ua650\ua651\ua652\ua653\ua654\ua655\ua656\ua657\ua658\ua659\ua65a\ua65b\ua65c\ua65d\ua65e\ua65f\ua660\ua661\ua662\ua663\ua664\ua665\ua666\ua667\ua668\ua669\ua66a\ua66b\ua66c\ua66d\ua66e\ua66f\ua670\ua671\ua672\ua673\ua674\ua675\ua676\ua677\ua678\ua679\ua67a\ua67b\ua67c\ua67d\ua67e\ua67f\ua680\ua681\ua682\ua683\ua684\ua685\ua686\ua687\ua688\ua689\ua68a\ua68b\ua68c\ua68d\ua68e\ua68f\ua690\ua691\ua692\ua693\ua694\ua695\ua696\ua697\ua698\ua699\ua69a\ua69b\ua69c\ua69d\ua69e\ua69f\ua6a0\ua6a1\ua6a2\ua6a3\ua6a4\ua6a5\ua6a6\ua6a7\ua6a8\ua6a9\ua6aa\ua6ab\ua6ac\ua6ad\ua6ae\ua6af\ua6b0\ua6b1\ua6b2\ua6b3\ua6b4\ua6b5\ua6b6\ua6b7\ua6b8\ua6b9\ua6ba\ua6bb\ua6bc\ua6bd\ua6be\ua6bf\ua6c0\ua6c1\ua6c2\ua6c3\ua6c4\ua6c5\ua6c6\ua6c7\ua6c8\ua6c9\ua6ca\ua6cb\ua6cc\ua6cd\ua6ce\ua6cf\ua6d0\ua6d1\ua6d2\ua6d3\ua6d4\ua6d5\ua6d6\ua6d7\ua6d8\ua6d9\ua6da\ua6db\ua6dc\ua6dd\ua6de\ua6df\ua6e0\ua6e1\ua6e2\ua6e3\ua6e4\ua6e5\ua6e6\ua6e7\ua6e8\ua6e9\ua6ea\ua6eb\ua6ec\ua6ed\ua6ee\ua6ef\ua6f0\ua6f1\ua6f2\ua6f3\ua6f4\ua6f5\ua6f6\ua6f7\ua6f8\ua6f9\ua6fa\ua6fb\ua6fc\ua6fd\ua6fe\ua6ff\ua717\ua718\ua719\ua71a\ua71b\ua71c\ua71d\ua71e\ua71f\ua720\ua721\ua722\ua723\ua724\ua725\ua726\ua727\ua728\ua729\ua72a\ua72b\ua72c\ua72d\ua72e\ua72f\ua730\ua731\ua732\ua733\ua734\ua735\ua736\ua737\ua738\ua739\ua73a\ua73b\ua73c\ua73d\ua73e\ua73f\ua740\ua741\ua742\ua743\ua744\ua745\ua746\ua747\ua748\ua749\ua74a\ua74b\ua74c\ua74d\ua74e\ua74f\ua750\ua751\ua752\ua753\ua754\ua755\ua756\ua757\ua758\ua759\ua75a\ua75b\ua75c\ua75d\ua75e\ua75f\ua760\ua761\ua762\ua763\ua764\ua765\ua766\ua767\ua768\ua769\ua76a\ua76b\ua76c\ua76d\ua76e\ua76f\ua770\ua771\ua772\ua773\ua774\ua775\ua776\ua777\ua778\ua779\ua77a\ua77b\ua77c\ua77d\ua77e\ua77f\ua780\ua781\ua782\ua783\ua784\ua785\ua786\ua787\ua788\ua789\ua78a\ua78b\ua78c\ua78d\ua78e\ua78f\ua790\ua791\ua792\ua793\ua794\ua795\ua796\ua797\ua798\ua799\ua79a\ua79b\ua79c\ua79d\ua79e\ua79f\ua7a0\ua7a1\ua7a2\ua7a3\ua7a4\ua7a5\ua7a6\ua7a7\ua7a8\ua7a9\ua7aa\ua7ab\ua7ac\ua7ad\ua7ae\ua7af\ua7b0\ua7b1\ua7b2\ua7b3\ua7b4\ua7b5\ua7b6\ua7b7\ua7b8\ua7b9\ua7ba\ua7bb\ua7bc\ua7bd\ua7be\ua7bf\ua7c0\ua7c1\ua7c2\ua7c3\ua7c4\ua7c5\ua7c6\ua7c7\ua7c8\ua7c9\ua7ca\ua7cb\ua7cc\ua7cd\ua7ce\ua7cf\ua7d0\ua7d1\ua7d2\ua7d3\ua7d4\ua7d5\ua7d6\ua7d7\ua7d8\ua7d9\ua7da\ua7db\ua7dc\ua7dd\ua7de\ua7df\ua7e0\ua7e1\ua7e2\ua7e3\ua7e4\ua7e5\ua7e6\ua7e7\ua7e8\ua7e9\ua7ea\ua7eb\ua7ec\ua7ed\ua7ee\ua7ef\ua7f0\ua7f1\ua7f2\ua7f3\ua7f4\ua7f5\ua7f6\ua7f7\ua7f8\ua7f9\ua7fa\ua7fb\ua7fc\ua7fd\ua7fe\ua7ff\ua82c\ua82d\ua82e\ua82f\ua830\ua831\ua832\ua833\ua834\ua835\ua836\ua837\ua838\ua839\ua83a\ua83b\ua83c\ua83d\ua83e\ua83f\ua840\ua841\ua842\ua843\ua844\ua845\ua846\ua847\ua848\ua849\ua84a\ua84b\ua84c\ua84d\ua84e\ua84f\ua850\ua851\ua852\ua853\ua854\ua855\ua856\ua857\ua858\ua859\ua85a\ua85b\ua85c\ua85d\ua85e\ua85f\ua860\ua861\ua862\ua863\ua864\ua865\ua866\ua867\ua868\ua869\ua86a\ua86b\ua86c\ua86d\ua86e\ua86f\ua870\ua871\ua872\ua873\ua874\ua875\ua876\ua877\ua878\ua879\ua87a\ua87b\ua87c\ua87d\ua87e\ua87f\ua880\ua881\ua882\ua883\ua884\ua885\ua886\ua887\ua888\ua889\ua88a\ua88b\ua88c\ua88d\ua88e\ua88f\ua890\ua891\ua892\ua893\ua894\ua895\ua896\ua897\ua898\ua899\ua89a\ua89b\ua89c\ua89d\ua89e\ua89f\ua8a0\ua8a1\ua8a2\ua8a3\ua8a4\ua8a5\ua8a6\ua8a7\ua8a8\ua8a9\ua8aa\ua8ab\ua8ac\ua8ad\ua8ae\ua8af\ua8b0\ua8b1\ua8b2\ua8b3\ua8b4\ua8b5\ua8b6\ua8b7\ua8b8\ua8b9\ua8ba\ua8bb\ua8bc\ua8bd\ua8be\ua8bf\ua8c0\ua8c1\ua8c2\ua8c3\ua8c4\ua8c5\ua8c6\ua8c7\ua8c8\ua8c9\ua8ca\ua8cb\ua8cc\ua8cd\ua8ce\ua8cf\ua8d0\ua8d1\ua8d2\ua8d3\ua8d4\ua8d5\ua8d6\ua8d7\ua8d8\ua8d9\ua8da\ua8db\ua8dc\ua8dd\ua8de\ua8df\ua8e0\ua8e1\ua8e2\ua8e3\ua8e4\ua8e5\ua8e6\ua8e7\ua8e8\ua8e9\ua8ea\ua8eb\ua8ec\ua8ed\ua8ee\ua8ef\ua8f0\ua8f1\ua8f2\ua8f3\ua8f4\ua8f5\ua8f6\ua8f7\ua8f8\ua8f9\ua8fa\ua8fb\ua8fc\ua8fd\ua8fe\ua8ff\ua900\ua901\ua902\ua903\ua904\ua905\ua906\ua907\ua908\ua909\ua90a\ua90b\ua90c\ua90d\ua90e\ua90f\ua910\ua911\ua912\ua913\ua914\ua915\ua916\ua917\ua918\ua919\ua91a\ua91b\ua91c\ua91d\ua91e\ua91f\ua920\ua921\ua922\ua923\ua924\ua925\ua926\ua927\ua928\ua929\ua92a\ua92b\ua92c\ua92d\ua92e\ua92f\ua930\ua931\ua932\ua933\ua934\ua935\ua936\ua937\ua938\ua939\ua93a\ua93b\ua93c\ua93d\ua93e\ua93f\ua940\ua941\ua942\ua943\ua944\ua945\ua946\ua947\ua948\ua949\ua94a\ua94b\ua94c\ua94d\ua94e\ua94f\ua950\ua951\ua952\ua953\ua954\ua955\ua956\ua957\ua958\ua959\ua95a\ua95b\ua95c\ua95d\ua95e\ua95f\ua960\ua961\ua962\ua963\ua964\ua965\ua966\ua967\ua968\ua969\ua96a\ua96b\ua96c\ua96d\ua96e\ua96f\ua970\ua971\ua972\ua973\ua974\ua975\ua976\ua977\ua978\ua979\ua97a\ua97b\ua97c\ua97d\ua97e\ua97f\ua980\ua981\ua982\ua983\ua984\ua985\ua986\ua987\ua988\ua989\ua98a\ua98b\ua98c\ua98d\ua98e\ua98f\ua990\ua991\ua992\ua993\ua994\ua995\ua996\ua997\ua998\ua999\ua99a\ua99b\ua99c\ua99d\ua99e\ua99f\ua9a0\ua9a1\ua9a2\ua9a3\ua9a4\ua9a5\ua9a6\ua9a7\ua9a8\ua9a9\ua9aa\ua9ab\ua9ac\ua9ad\ua9ae\ua9af\ua9b0\ua9b1\ua9b2\ua9b3\ua9b4\ua9b5\ua9b6\ua9b7\ua9b8\ua9b9\ua9ba\ua9bb\ua9bc\ua9bd\ua9be\ua9bf\ua9c0\ua9c1\ua9c2\ua9c3\ua9c4\ua9c5\ua9c6\ua9c7\ua9c8\ua9c9\ua9ca\ua9cb\ua9cc\ua9cd\ua9ce\ua9cf\ua9d0\ua9d1\ua9d2\ua9d3\ua9d4\ua9d5\ua9d6\ua9d7\ua9d8\ua9d9\ua9da\ua9db\ua9dc\ua9dd\ua9de\ua9df\ua9e0\ua9e1\ua9e2\ua9e3\ua9e4\ua9e5\ua9e6\ua9e7\ua9e8\ua9e9\ua9ea\ua9eb\ua9ec\ua9ed\ua9ee\ua9ef\ua9f0\ua9f1\ua9f2\ua9f3\ua9f4\ua9f5\ua9f6\ua9f7\ua9f8\ua9f9\ua9fa\ua9fb\ua9fc\ua9fd\ua9fe\ua9ff\uaa00\uaa01\uaa02\uaa03\uaa04\uaa05\uaa06\uaa07\uaa08\uaa09\uaa0a\uaa0b\uaa0c\uaa0d\uaa0e\uaa0f\uaa10\uaa11\uaa12\uaa13\uaa14\uaa15\uaa16\uaa17\uaa18\uaa19\uaa1a\uaa1b\uaa1c\uaa1d\uaa1e\uaa1f\uaa20\uaa21\uaa22\uaa23\uaa24\uaa25\uaa26\uaa27\uaa28\uaa29\uaa2a\uaa2b\uaa2c\uaa2d\uaa2e\uaa2f\uaa30\uaa31\uaa32\uaa33\uaa34\uaa35\uaa36\uaa37\uaa38\uaa39\uaa3a\uaa3b\uaa3c\uaa3d\uaa3e\uaa3f\uaa40\uaa41\uaa42\uaa43\uaa44\uaa45\uaa46\uaa47\uaa48\uaa49\uaa4a\uaa4b\uaa4c\uaa4d\uaa4e\uaa4f\uaa50\uaa51\uaa52\uaa53\uaa54\uaa55\uaa56\uaa57\uaa58\uaa59\uaa5a\uaa5b\uaa5c\uaa5d\uaa5e\uaa5f\uaa60\uaa61\uaa62\uaa63\uaa64\uaa65\uaa66\uaa67\uaa68\uaa69\uaa6a\uaa6b\uaa6c\uaa6d\uaa6e\uaa6f\uaa70\uaa71\uaa72\uaa73\uaa74\uaa75\uaa76\uaa77\uaa78\uaa79\uaa7a\uaa7b\uaa7c\uaa7d\uaa7e\uaa7f\uaa80\uaa81\uaa82\uaa83\uaa84\uaa85\uaa86\uaa87\uaa88\uaa89\uaa8a\uaa8b\uaa8c\uaa8d\uaa8e\uaa8f\uaa90\uaa91\uaa92\uaa93\uaa94\uaa95\uaa96\uaa97\uaa98\uaa99\uaa9a\uaa9b\uaa9c\uaa9d\uaa9e\uaa9f\uaaa0\uaaa1\uaaa2\uaaa3\uaaa4\uaaa5\uaaa6\uaaa7\uaaa8\uaaa9\uaaaa\uaaab\uaaac\uaaad\uaaae\uaaaf\uaab0\uaab1\uaab2\uaab3\uaab4\uaab5\uaab6\uaab7\uaab8\uaab9\uaaba\uaabb\uaabc\uaabd\uaabe\uaabf\uaac0\uaac1\uaac2\uaac3\uaac4\uaac5\uaac6\uaac7\uaac8\uaac9\uaaca\uaacb\uaacc\uaacd\uaace\uaacf\uaad0\uaad1\uaad2\uaad3\uaad4\uaad5\uaad6\uaad7\uaad8\uaad9\uaada\uaadb\uaadc\uaadd\uaade\uaadf\uaae0\uaae1\uaae2\uaae3\uaae4\uaae5\uaae6\uaae7\uaae8\uaae9\uaaea\uaaeb\uaaec\uaaed\uaaee\uaaef\uaaf0\uaaf1\uaaf2\uaaf3\uaaf4\uaaf5\uaaf6\uaaf7\uaaf8\uaaf9\uaafa\uaafb\uaafc\uaafd\uaafe\uaaff\uab00\uab01\uab02\uab03\uab04\uab05\uab06\uab07\uab08\uab09\uab0a\uab0b\uab0c\uab0d\uab0e\uab0f\uab10\uab11\uab12\uab13\uab14\uab15\uab16\uab17\uab18\uab19\uab1a\uab1b\uab1c\uab1d\uab1e\uab1f\uab20\uab21\uab22\uab23\uab24\uab25\uab26\uab27\uab28\uab29\uab2a\uab2b\uab2c\uab2d\uab2e\uab2f\uab30\uab31\uab32\uab33\uab34\uab35\uab36\uab37\uab38\uab39\uab3a\uab3b\uab3c\uab3d\uab3e\uab3f\uab40\uab41\uab42\uab43\uab44\uab45\uab46\uab47\uab48\uab49\uab4a\uab4b\uab4c\uab4d\uab4e\uab4f\uab50\uab51\uab52\uab53\uab54\uab55\uab56\uab57\uab58\uab59\uab5a\uab5b\uab5c\uab5d\uab5e\uab5f\uab60\uab61\uab62\uab63\uab64\uab65\uab66\uab67\uab68\uab69\uab6a\uab6b\uab6c\uab6d\uab6e\uab6f\uab70\uab71\uab72\uab73\uab74\uab75\uab76\uab77\uab78\uab79\uab7a\uab7b\uab7c\uab7d\uab7e\uab7f\uab80\uab81\uab82\uab83\uab84\uab85\uab86\uab87\uab88\uab89\uab8a\uab8b\uab8c\uab8d\uab8e\uab8f\uab90\uab91\uab92\uab93\uab94\uab95\uab96\uab97\uab98\uab99\uab9a\uab9b\uab9c\uab9d\uab9e\uab9f\uaba0\uaba1\uaba2\uaba3\uaba4\uaba5\uaba6\uaba7\uaba8\uaba9\uabaa\uabab\uabac\uabad\uabae\uabaf\uabb0\uabb1\uabb2\uabb3\uabb4\uabb5\uabb6\uabb7\uabb8\uabb9\uabba\uabbb\uabbc\uabbd\uabbe\uabbf\uabc0\uabc1\uabc2\uabc3\uabc4\uabc5\uabc6\uabc7\uabc8\uabc9\uabca\uabcb\uabcc\uabcd\uabce\uabcf\uabd0\uabd1\uabd2\uabd3\uabd4\uabd5\uabd6\uabd7\uabd8\uabd9\uabda\uabdb\uabdc\uabdd\uabde\uabdf\uabe0\uabe1\uabe2\uabe3\uabe4\uabe5\uabe6\uabe7\uabe8\uabe9\uabea\uabeb\uabec\uabed\uabee\uabef\uabf0\uabf1\uabf2\uabf3\uabf4\uabf5\uabf6\uabf7\uabf8\uabf9\uabfa\uabfb\uabfc\uabfd\uabfe\uabff\ud7a4\ud7a5\ud7a6\ud7a7\ud7a8\ud7a9\ud7aa\ud7ab\ud7ac\ud7ad\ud7ae\ud7af\ud7b0\ud7b1\ud7b2\ud7b3\ud7b4\ud7b5\ud7b6\ud7b7\ud7b8\ud7b9\ud7ba\ud7bb\ud7bc\ud7bd\ud7be\ud7bf\ud7c0\ud7c1\ud7c2\ud7c3\ud7c4\ud7c5\ud7c6\ud7c7\ud7c8\ud7c9\ud7ca\ud7cb\ud7cc\ud7cd\ud7ce\ud7cf\ud7d0\ud7d1\ud7d2\ud7d3\ud7d4\ud7d5\ud7d6\ud7d7\ud7d8\ud7d9\ud7da\ud7db\ud7dc\ud7dd\ud7de\ud7df\ud7e0\ud7e1\ud7e2\ud7e3\ud7e4\ud7e5\ud7e6\ud7e7\ud7e8\ud7e9\ud7ea\ud7eb\ud7ec\ud7ed\ud7ee\ud7ef\ud7f0\ud7f1\ud7f2\ud7f3\ud7f4\ud7f5\ud7f6\ud7f7\ud7f8\ud7f9\ud7fa\ud7fb\ud7fc\ud7fd\ud7fe\ud7ff\ufa2e\ufa2f\ufa6b\ufa6c\ufa6d\ufa6e\ufa6f\ufada\ufadb\ufadc\ufadd\ufade\ufadf\ufae0\ufae1\ufae2\ufae3\ufae4\ufae5\ufae6\ufae7\ufae8\ufae9\ufaea\ufaeb\ufaec\ufaed\ufaee\ufaef\ufaf0\ufaf1\ufaf2\ufaf3\ufaf4\ufaf5\ufaf6\ufaf7\ufaf8\ufaf9\ufafa\ufafb\ufafc\ufafd\ufafe\ufaff\ufb07\ufb08\ufb09\ufb0a\ufb0b\ufb0c\ufb0d\ufb0e\ufb0f\ufb10\ufb11\ufb12\ufb18\ufb19\ufb1a\ufb1b\ufb1c\ufb37\ufb3d\ufb3f\ufb42\ufb45\ufbb2\ufbb3\ufbb4\ufbb5\ufbb6\ufbb7\ufbb8\ufbb9\ufbba\ufbbb\ufbbc\ufbbd\ufbbe\ufbbf\ufbc0\ufbc1\ufbc2\ufbc3\ufbc4\ufbc5\ufbc6\ufbc7\ufbc8\ufbc9\ufbca\ufbcb\ufbcc\ufbcd\ufbce\ufbcf\ufbd0\ufbd1\ufbd2\ufd40\ufd41\ufd42\ufd43\ufd44\ufd45\ufd46\ufd47\ufd48\ufd49\ufd4a\ufd4b\ufd4c\ufd4d\ufd4e\ufd4f\ufd90\ufd91\ufdc8\ufdc9\ufdca\ufdcb\ufdcc\ufdcd\ufdce\ufdcf\ufdd0\ufdd1\ufdd2\ufdd3\ufdd4\ufdd5\ufdd6\ufdd7\ufdd8\ufdd9\ufdda\ufddb\ufddc\ufddd\ufdde\ufddf\ufde0\ufde1\ufde2\ufde3\ufde4\ufde5\ufde6\ufde7\ufde8\ufde9\ufdea\ufdeb\ufdec\ufded\ufdee\ufdef\ufdfe\ufdff\ufe1a\ufe1b\ufe1c\ufe1d\ufe1e\ufe1f\ufe24\ufe25\ufe26\ufe27\ufe28\ufe29\ufe2a\ufe2b\ufe2c\ufe2d\ufe2e\ufe2f\ufe53\ufe67\ufe6c\ufe6d\ufe6e\ufe6f\ufe75\ufefd\ufefe\uff00\uffbf\uffc0\uffc1\uffc8\uffc9\uffd0\uffd1\uffd8\uffd9\uffdd\uffde\uffdf\uffe7\uffef\ufff0\ufff1\ufff2\ufff3\ufff4\ufff5\ufff6\ufff7\ufff8\ufffe' + +Co = u'\ue000\ue001\ue002\ue003\ue004\ue005\ue006\ue007\ue008\ue009\ue00a\ue00b\ue00c\ue00d\ue00e\ue00f\ue010\ue011\ue012\ue013\ue014\ue015\ue016\ue017\ue018\ue019\ue01a\ue01b\ue01c\ue01d\ue01e\ue01f\ue020\ue021\ue022\ue023\ue024\ue025\ue026\ue027\ue028\ue029\ue02a\ue02b\ue02c\ue02d\ue02e\ue02f\ue030\ue031\ue032\ue033\ue034\ue035\ue036\ue037\ue038\ue039\ue03a\ue03b\ue03c\ue03d\ue03e\ue03f\ue040\ue041\ue042\ue043\ue044\ue045\ue046\ue047\ue048\ue049\ue04a\ue04b\ue04c\ue04d\ue04e\ue04f\ue050\ue051\ue052\ue053\ue054\ue055\ue056\ue057\ue058\ue059\ue05a\ue05b\ue05c\ue05d\ue05e\ue05f\ue060\ue061\ue062\ue063\ue064\ue065\ue066\ue067\ue068\ue069\ue06a\ue06b\ue06c\ue06d\ue06e\ue06f\ue070\ue071\ue072\ue073\ue074\ue075\ue076\ue077\ue078\ue079\ue07a\ue07b\ue07c\ue07d\ue07e\ue07f\ue080\ue081\ue082\ue083\ue084\ue085\ue086\ue087\ue088\ue089\ue08a\ue08b\ue08c\ue08d\ue08e\ue08f\ue090\ue091\ue092\ue093\ue094\ue095\ue096\ue097\ue098\ue099\ue09a\ue09b\ue09c\ue09d\ue09e\ue09f\ue0a0\ue0a1\ue0a2\ue0a3\ue0a4\ue0a5\ue0a6\ue0a7\ue0a8\ue0a9\ue0aa\ue0ab\ue0ac\ue0ad\ue0ae\ue0af\ue0b0\ue0b1\ue0b2\ue0b3\ue0b4\ue0b5\ue0b6\ue0b7\ue0b8\ue0b9\ue0ba\ue0bb\ue0bc\ue0bd\ue0be\ue0bf\ue0c0\ue0c1\ue0c2\ue0c3\ue0c4\ue0c5\ue0c6\ue0c7\ue0c8\ue0c9\ue0ca\ue0cb\ue0cc\ue0cd\ue0ce\ue0cf\ue0d0\ue0d1\ue0d2\ue0d3\ue0d4\ue0d5\ue0d6\ue0d7\ue0d8\ue0d9\ue0da\ue0db\ue0dc\ue0dd\ue0de\ue0df\ue0e0\ue0e1\ue0e2\ue0e3\ue0e4\ue0e5\ue0e6\ue0e7\ue0e8\ue0e9\ue0ea\ue0eb\ue0ec\ue0ed\ue0ee\ue0ef\ue0f0\ue0f1\ue0f2\ue0f3\ue0f4\ue0f5\ue0f6\ue0f7\ue0f8\ue0f9\ue0fa\ue0fb\ue0fc\ue0fd\ue0fe\ue0ff\ue100\ue101\ue102\ue103\ue104\ue105\ue106\ue107\ue108\ue109\ue10a\ue10b\ue10c\ue10d\ue10e\ue10f\ue110\ue111\ue112\ue113\ue114\ue115\ue116\ue117\ue118\ue119\ue11a\ue11b\ue11c\ue11d\ue11e\ue11f\ue120\ue121\ue122\ue123\ue124\ue125\ue126\ue127\ue128\ue129\ue12a\ue12b\ue12c\ue12d\ue12e\ue12f\ue130\ue131\ue132\ue133\ue134\ue135\ue136\ue137\ue138\ue139\ue13a\ue13b\ue13c\ue13d\ue13e\ue13f\ue140\ue141\ue142\ue143\ue144\ue145\ue146\ue147\ue148\ue149\ue14a\ue14b\ue14c\ue14d\ue14e\ue14f\ue150\ue151\ue152\ue153\ue154\ue155\ue156\ue157\ue158\ue159\ue15a\ue15b\ue15c\ue15d\ue15e\ue15f\ue160\ue161\ue162\ue163\ue164\ue165\ue166\ue167\ue168\ue169\ue16a\ue16b\ue16c\ue16d\ue16e\ue16f\ue170\ue171\ue172\ue173\ue174\ue175\ue176\ue177\ue178\ue179\ue17a\ue17b\ue17c\ue17d\ue17e\ue17f\ue180\ue181\ue182\ue183\ue184\ue185\ue186\ue187\ue188\ue189\ue18a\ue18b\ue18c\ue18d\ue18e\ue18f\ue190\ue191\ue192\ue193\ue194\ue195\ue196\ue197\ue198\ue199\ue19a\ue19b\ue19c\ue19d\ue19e\ue19f\ue1a0\ue1a1\ue1a2\ue1a3\ue1a4\ue1a5\ue1a6\ue1a7\ue1a8\ue1a9\ue1aa\ue1ab\ue1ac\ue1ad\ue1ae\ue1af\ue1b0\ue1b1\ue1b2\ue1b3\ue1b4\ue1b5\ue1b6\ue1b7\ue1b8\ue1b9\ue1ba\ue1bb\ue1bc\ue1bd\ue1be\ue1bf\ue1c0\ue1c1\ue1c2\ue1c3\ue1c4\ue1c5\ue1c6\ue1c7\ue1c8\ue1c9\ue1ca\ue1cb\ue1cc\ue1cd\ue1ce\ue1cf\ue1d0\ue1d1\ue1d2\ue1d3\ue1d4\ue1d5\ue1d6\ue1d7\ue1d8\ue1d9\ue1da\ue1db\ue1dc\ue1dd\ue1de\ue1df\ue1e0\ue1e1\ue1e2\ue1e3\ue1e4\ue1e5\ue1e6\ue1e7\ue1e8\ue1e9\ue1ea\ue1eb\ue1ec\ue1ed\ue1ee\ue1ef\ue1f0\ue1f1\ue1f2\ue1f3\ue1f4\ue1f5\ue1f6\ue1f7\ue1f8\ue1f9\ue1fa\ue1fb\ue1fc\ue1fd\ue1fe\ue1ff\ue200\ue201\ue202\ue203\ue204\ue205\ue206\ue207\ue208\ue209\ue20a\ue20b\ue20c\ue20d\ue20e\ue20f\ue210\ue211\ue212\ue213\ue214\ue215\ue216\ue217\ue218\ue219\ue21a\ue21b\ue21c\ue21d\ue21e\ue21f\ue220\ue221\ue222\ue223\ue224\ue225\ue226\ue227\ue228\ue229\ue22a\ue22b\ue22c\ue22d\ue22e\ue22f\ue230\ue231\ue232\ue233\ue234\ue235\ue236\ue237\ue238\ue239\ue23a\ue23b\ue23c\ue23d\ue23e\ue23f\ue240\ue241\ue242\ue243\ue244\ue245\ue246\ue247\ue248\ue249\ue24a\ue24b\ue24c\ue24d\ue24e\ue24f\ue250\ue251\ue252\ue253\ue254\ue255\ue256\ue257\ue258\ue259\ue25a\ue25b\ue25c\ue25d\ue25e\ue25f\ue260\ue261\ue262\ue263\ue264\ue265\ue266\ue267\ue268\ue269\ue26a\ue26b\ue26c\ue26d\ue26e\ue26f\ue270\ue271\ue272\ue273\ue274\ue275\ue276\ue277\ue278\ue279\ue27a\ue27b\ue27c\ue27d\ue27e\ue27f\ue280\ue281\ue282\ue283\ue284\ue285\ue286\ue287\ue288\ue289\ue28a\ue28b\ue28c\ue28d\ue28e\ue28f\ue290\ue291\ue292\ue293\ue294\ue295\ue296\ue297\ue298\ue299\ue29a\ue29b\ue29c\ue29d\ue29e\ue29f\ue2a0\ue2a1\ue2a2\ue2a3\ue2a4\ue2a5\ue2a6\ue2a7\ue2a8\ue2a9\ue2aa\ue2ab\ue2ac\ue2ad\ue2ae\ue2af\ue2b0\ue2b1\ue2b2\ue2b3\ue2b4\ue2b5\ue2b6\ue2b7\ue2b8\ue2b9\ue2ba\ue2bb\ue2bc\ue2bd\ue2be\ue2bf\ue2c0\ue2c1\ue2c2\ue2c3\ue2c4\ue2c5\ue2c6\ue2c7\ue2c8\ue2c9\ue2ca\ue2cb\ue2cc\ue2cd\ue2ce\ue2cf\ue2d0\ue2d1\ue2d2\ue2d3\ue2d4\ue2d5\ue2d6\ue2d7\ue2d8\ue2d9\ue2da\ue2db\ue2dc\ue2dd\ue2de\ue2df\ue2e0\ue2e1\ue2e2\ue2e3\ue2e4\ue2e5\ue2e6\ue2e7\ue2e8\ue2e9\ue2ea\ue2eb\ue2ec\ue2ed\ue2ee\ue2ef\ue2f0\ue2f1\ue2f2\ue2f3\ue2f4\ue2f5\ue2f6\ue2f7\ue2f8\ue2f9\ue2fa\ue2fb\ue2fc\ue2fd\ue2fe\ue2ff\ue300\ue301\ue302\ue303\ue304\ue305\ue306\ue307\ue308\ue309\ue30a\ue30b\ue30c\ue30d\ue30e\ue30f\ue310\ue311\ue312\ue313\ue314\ue315\ue316\ue317\ue318\ue319\ue31a\ue31b\ue31c\ue31d\ue31e\ue31f\ue320\ue321\ue322\ue323\ue324\ue325\ue326\ue327\ue328\ue329\ue32a\ue32b\ue32c\ue32d\ue32e\ue32f\ue330\ue331\ue332\ue333\ue334\ue335\ue336\ue337\ue338\ue339\ue33a\ue33b\ue33c\ue33d\ue33e\ue33f\ue340\ue341\ue342\ue343\ue344\ue345\ue346\ue347\ue348\ue349\ue34a\ue34b\ue34c\ue34d\ue34e\ue34f\ue350\ue351\ue352\ue353\ue354\ue355\ue356\ue357\ue358\ue359\ue35a\ue35b\ue35c\ue35d\ue35e\ue35f\ue360\ue361\ue362\ue363\ue364\ue365\ue366\ue367\ue368\ue369\ue36a\ue36b\ue36c\ue36d\ue36e\ue36f\ue370\ue371\ue372\ue373\ue374\ue375\ue376\ue377\ue378\ue379\ue37a\ue37b\ue37c\ue37d\ue37e\ue37f\ue380\ue381\ue382\ue383\ue384\ue385\ue386\ue387\ue388\ue389\ue38a\ue38b\ue38c\ue38d\ue38e\ue38f\ue390\ue391\ue392\ue393\ue394\ue395\ue396\ue397\ue398\ue399\ue39a\ue39b\ue39c\ue39d\ue39e\ue39f\ue3a0\ue3a1\ue3a2\ue3a3\ue3a4\ue3a5\ue3a6\ue3a7\ue3a8\ue3a9\ue3aa\ue3ab\ue3ac\ue3ad\ue3ae\ue3af\ue3b0\ue3b1\ue3b2\ue3b3\ue3b4\ue3b5\ue3b6\ue3b7\ue3b8\ue3b9\ue3ba\ue3bb\ue3bc\ue3bd\ue3be\ue3bf\ue3c0\ue3c1\ue3c2\ue3c3\ue3c4\ue3c5\ue3c6\ue3c7\ue3c8\ue3c9\ue3ca\ue3cb\ue3cc\ue3cd\ue3ce\ue3cf\ue3d0\ue3d1\ue3d2\ue3d3\ue3d4\ue3d5\ue3d6\ue3d7\ue3d8\ue3d9\ue3da\ue3db\ue3dc\ue3dd\ue3de\ue3df\ue3e0\ue3e1\ue3e2\ue3e3\ue3e4\ue3e5\ue3e6\ue3e7\ue3e8\ue3e9\ue3ea\ue3eb\ue3ec\ue3ed\ue3ee\ue3ef\ue3f0\ue3f1\ue3f2\ue3f3\ue3f4\ue3f5\ue3f6\ue3f7\ue3f8\ue3f9\ue3fa\ue3fb\ue3fc\ue3fd\ue3fe\ue3ff\ue400\ue401\ue402\ue403\ue404\ue405\ue406\ue407\ue408\ue409\ue40a\ue40b\ue40c\ue40d\ue40e\ue40f\ue410\ue411\ue412\ue413\ue414\ue415\ue416\ue417\ue418\ue419\ue41a\ue41b\ue41c\ue41d\ue41e\ue41f\ue420\ue421\ue422\ue423\ue424\ue425\ue426\ue427\ue428\ue429\ue42a\ue42b\ue42c\ue42d\ue42e\ue42f\ue430\ue431\ue432\ue433\ue434\ue435\ue436\ue437\ue438\ue439\ue43a\ue43b\ue43c\ue43d\ue43e\ue43f\ue440\ue441\ue442\ue443\ue444\ue445\ue446\ue447\ue448\ue449\ue44a\ue44b\ue44c\ue44d\ue44e\ue44f\ue450\ue451\ue452\ue453\ue454\ue455\ue456\ue457\ue458\ue459\ue45a\ue45b\ue45c\ue45d\ue45e\ue45f\ue460\ue461\ue462\ue463\ue464\ue465\ue466\ue467\ue468\ue469\ue46a\ue46b\ue46c\ue46d\ue46e\ue46f\ue470\ue471\ue472\ue473\ue474\ue475\ue476\ue477\ue478\ue479\ue47a\ue47b\ue47c\ue47d\ue47e\ue47f\ue480\ue481\ue482\ue483\ue484\ue485\ue486\ue487\ue488\ue489\ue48a\ue48b\ue48c\ue48d\ue48e\ue48f\ue490\ue491\ue492\ue493\ue494\ue495\ue496\ue497\ue498\ue499\ue49a\ue49b\ue49c\ue49d\ue49e\ue49f\ue4a0\ue4a1\ue4a2\ue4a3\ue4a4\ue4a5\ue4a6\ue4a7\ue4a8\ue4a9\ue4aa\ue4ab\ue4ac\ue4ad\ue4ae\ue4af\ue4b0\ue4b1\ue4b2\ue4b3\ue4b4\ue4b5\ue4b6\ue4b7\ue4b8\ue4b9\ue4ba\ue4bb\ue4bc\ue4bd\ue4be\ue4bf\ue4c0\ue4c1\ue4c2\ue4c3\ue4c4\ue4c5\ue4c6\ue4c7\ue4c8\ue4c9\ue4ca\ue4cb\ue4cc\ue4cd\ue4ce\ue4cf\ue4d0\ue4d1\ue4d2\ue4d3\ue4d4\ue4d5\ue4d6\ue4d7\ue4d8\ue4d9\ue4da\ue4db\ue4dc\ue4dd\ue4de\ue4df\ue4e0\ue4e1\ue4e2\ue4e3\ue4e4\ue4e5\ue4e6\ue4e7\ue4e8\ue4e9\ue4ea\ue4eb\ue4ec\ue4ed\ue4ee\ue4ef\ue4f0\ue4f1\ue4f2\ue4f3\ue4f4\ue4f5\ue4f6\ue4f7\ue4f8\ue4f9\ue4fa\ue4fb\ue4fc\ue4fd\ue4fe\ue4ff\ue500\ue501\ue502\ue503\ue504\ue505\ue506\ue507\ue508\ue509\ue50a\ue50b\ue50c\ue50d\ue50e\ue50f\ue510\ue511\ue512\ue513\ue514\ue515\ue516\ue517\ue518\ue519\ue51a\ue51b\ue51c\ue51d\ue51e\ue51f\ue520\ue521\ue522\ue523\ue524\ue525\ue526\ue527\ue528\ue529\ue52a\ue52b\ue52c\ue52d\ue52e\ue52f\ue530\ue531\ue532\ue533\ue534\ue535\ue536\ue537\ue538\ue539\ue53a\ue53b\ue53c\ue53d\ue53e\ue53f\ue540\ue541\ue542\ue543\ue544\ue545\ue546\ue547\ue548\ue549\ue54a\ue54b\ue54c\ue54d\ue54e\ue54f\ue550\ue551\ue552\ue553\ue554\ue555\ue556\ue557\ue558\ue559\ue55a\ue55b\ue55c\ue55d\ue55e\ue55f\ue560\ue561\ue562\ue563\ue564\ue565\ue566\ue567\ue568\ue569\ue56a\ue56b\ue56c\ue56d\ue56e\ue56f\ue570\ue571\ue572\ue573\ue574\ue575\ue576\ue577\ue578\ue579\ue57a\ue57b\ue57c\ue57d\ue57e\ue57f\ue580\ue581\ue582\ue583\ue584\ue585\ue586\ue587\ue588\ue589\ue58a\ue58b\ue58c\ue58d\ue58e\ue58f\ue590\ue591\ue592\ue593\ue594\ue595\ue596\ue597\ue598\ue599\ue59a\ue59b\ue59c\ue59d\ue59e\ue59f\ue5a0\ue5a1\ue5a2\ue5a3\ue5a4\ue5a5\ue5a6\ue5a7\ue5a8\ue5a9\ue5aa\ue5ab\ue5ac\ue5ad\ue5ae\ue5af\ue5b0\ue5b1\ue5b2\ue5b3\ue5b4\ue5b5\ue5b6\ue5b7\ue5b8\ue5b9\ue5ba\ue5bb\ue5bc\ue5bd\ue5be\ue5bf\ue5c0\ue5c1\ue5c2\ue5c3\ue5c4\ue5c5\ue5c6\ue5c7\ue5c8\ue5c9\ue5ca\ue5cb\ue5cc\ue5cd\ue5ce\ue5cf\ue5d0\ue5d1\ue5d2\ue5d3\ue5d4\ue5d5\ue5d6\ue5d7\ue5d8\ue5d9\ue5da\ue5db\ue5dc\ue5dd\ue5de\ue5df\ue5e0\ue5e1\ue5e2\ue5e3\ue5e4\ue5e5\ue5e6\ue5e7\ue5e8\ue5e9\ue5ea\ue5eb\ue5ec\ue5ed\ue5ee\ue5ef\ue5f0\ue5f1\ue5f2\ue5f3\ue5f4\ue5f5\ue5f6\ue5f7\ue5f8\ue5f9\ue5fa\ue5fb\ue5fc\ue5fd\ue5fe\ue5ff\ue600\ue601\ue602\ue603\ue604\ue605\ue606\ue607\ue608\ue609\ue60a\ue60b\ue60c\ue60d\ue60e\ue60f\ue610\ue611\ue612\ue613\ue614\ue615\ue616\ue617\ue618\ue619\ue61a\ue61b\ue61c\ue61d\ue61e\ue61f\ue620\ue621\ue622\ue623\ue624\ue625\ue626\ue627\ue628\ue629\ue62a\ue62b\ue62c\ue62d\ue62e\ue62f\ue630\ue631\ue632\ue633\ue634\ue635\ue636\ue637\ue638\ue639\ue63a\ue63b\ue63c\ue63d\ue63e\ue63f\ue640\ue641\ue642\ue643\ue644\ue645\ue646\ue647\ue648\ue649\ue64a\ue64b\ue64c\ue64d\ue64e\ue64f\ue650\ue651\ue652\ue653\ue654\ue655\ue656\ue657\ue658\ue659\ue65a\ue65b\ue65c\ue65d\ue65e\ue65f\ue660\ue661\ue662\ue663\ue664\ue665\ue666\ue667\ue668\ue669\ue66a\ue66b\ue66c\ue66d\ue66e\ue66f\ue670\ue671\ue672\ue673\ue674\ue675\ue676\ue677\ue678\ue679\ue67a\ue67b\ue67c\ue67d\ue67e\ue67f\ue680\ue681\ue682\ue683\ue684\ue685\ue686\ue687\ue688\ue689\ue68a\ue68b\ue68c\ue68d\ue68e\ue68f\ue690\ue691\ue692\ue693\ue694\ue695\ue696\ue697\ue698\ue699\ue69a\ue69b\ue69c\ue69d\ue69e\ue69f\ue6a0\ue6a1\ue6a2\ue6a3\ue6a4\ue6a5\ue6a6\ue6a7\ue6a8\ue6a9\ue6aa\ue6ab\ue6ac\ue6ad\ue6ae\ue6af\ue6b0\ue6b1\ue6b2\ue6b3\ue6b4\ue6b5\ue6b6\ue6b7\ue6b8\ue6b9\ue6ba\ue6bb\ue6bc\ue6bd\ue6be\ue6bf\ue6c0\ue6c1\ue6c2\ue6c3\ue6c4\ue6c5\ue6c6\ue6c7\ue6c8\ue6c9\ue6ca\ue6cb\ue6cc\ue6cd\ue6ce\ue6cf\ue6d0\ue6d1\ue6d2\ue6d3\ue6d4\ue6d5\ue6d6\ue6d7\ue6d8\ue6d9\ue6da\ue6db\ue6dc\ue6dd\ue6de\ue6df\ue6e0\ue6e1\ue6e2\ue6e3\ue6e4\ue6e5\ue6e6\ue6e7\ue6e8\ue6e9\ue6ea\ue6eb\ue6ec\ue6ed\ue6ee\ue6ef\ue6f0\ue6f1\ue6f2\ue6f3\ue6f4\ue6f5\ue6f6\ue6f7\ue6f8\ue6f9\ue6fa\ue6fb\ue6fc\ue6fd\ue6fe\ue6ff\ue700\ue701\ue702\ue703\ue704\ue705\ue706\ue707\ue708\ue709\ue70a\ue70b\ue70c\ue70d\ue70e\ue70f\ue710\ue711\ue712\ue713\ue714\ue715\ue716\ue717\ue718\ue719\ue71a\ue71b\ue71c\ue71d\ue71e\ue71f\ue720\ue721\ue722\ue723\ue724\ue725\ue726\ue727\ue728\ue729\ue72a\ue72b\ue72c\ue72d\ue72e\ue72f\ue730\ue731\ue732\ue733\ue734\ue735\ue736\ue737\ue738\ue739\ue73a\ue73b\ue73c\ue73d\ue73e\ue73f\ue740\ue741\ue742\ue743\ue744\ue745\ue746\ue747\ue748\ue749\ue74a\ue74b\ue74c\ue74d\ue74e\ue74f\ue750\ue751\ue752\ue753\ue754\ue755\ue756\ue757\ue758\ue759\ue75a\ue75b\ue75c\ue75d\ue75e\ue75f\ue760\ue761\ue762\ue763\ue764\ue765\ue766\ue767\ue768\ue769\ue76a\ue76b\ue76c\ue76d\ue76e\ue76f\ue770\ue771\ue772\ue773\ue774\ue775\ue776\ue777\ue778\ue779\ue77a\ue77b\ue77c\ue77d\ue77e\ue77f\ue780\ue781\ue782\ue783\ue784\ue785\ue786\ue787\ue788\ue789\ue78a\ue78b\ue78c\ue78d\ue78e\ue78f\ue790\ue791\ue792\ue793\ue794\ue795\ue796\ue797\ue798\ue799\ue79a\ue79b\ue79c\ue79d\ue79e\ue79f\ue7a0\ue7a1\ue7a2\ue7a3\ue7a4\ue7a5\ue7a6\ue7a7\ue7a8\ue7a9\ue7aa\ue7ab\ue7ac\ue7ad\ue7ae\ue7af\ue7b0\ue7b1\ue7b2\ue7b3\ue7b4\ue7b5\ue7b6\ue7b7\ue7b8\ue7b9\ue7ba\ue7bb\ue7bc\ue7bd\ue7be\ue7bf\ue7c0\ue7c1\ue7c2\ue7c3\ue7c4\ue7c5\ue7c6\ue7c7\ue7c8\ue7c9\ue7ca\ue7cb\ue7cc\ue7cd\ue7ce\ue7cf\ue7d0\ue7d1\ue7d2\ue7d3\ue7d4\ue7d5\ue7d6\ue7d7\ue7d8\ue7d9\ue7da\ue7db\ue7dc\ue7dd\ue7de\ue7df\ue7e0\ue7e1\ue7e2\ue7e3\ue7e4\ue7e5\ue7e6\ue7e7\ue7e8\ue7e9\ue7ea\ue7eb\ue7ec\ue7ed\ue7ee\ue7ef\ue7f0\ue7f1\ue7f2\ue7f3\ue7f4\ue7f5\ue7f6\ue7f7\ue7f8\ue7f9\ue7fa\ue7fb\ue7fc\ue7fd\ue7fe\ue7ff\ue800\ue801\ue802\ue803\ue804\ue805\ue806\ue807\ue808\ue809\ue80a\ue80b\ue80c\ue80d\ue80e\ue80f\ue810\ue811\ue812\ue813\ue814\ue815\ue816\ue817\ue818\ue819\ue81a\ue81b\ue81c\ue81d\ue81e\ue81f\ue820\ue821\ue822\ue823\ue824\ue825\ue826\ue827\ue828\ue829\ue82a\ue82b\ue82c\ue82d\ue82e\ue82f\ue830\ue831\ue832\ue833\ue834\ue835\ue836\ue837\ue838\ue839\ue83a\ue83b\ue83c\ue83d\ue83e\ue83f\ue840\ue841\ue842\ue843\ue844\ue845\ue846\ue847\ue848\ue849\ue84a\ue84b\ue84c\ue84d\ue84e\ue84f\ue850\ue851\ue852\ue853\ue854\ue855\ue856\ue857\ue858\ue859\ue85a\ue85b\ue85c\ue85d\ue85e\ue85f\ue860\ue861\ue862\ue863\ue864\ue865\ue866\ue867\ue868\ue869\ue86a\ue86b\ue86c\ue86d\ue86e\ue86f\ue870\ue871\ue872\ue873\ue874\ue875\ue876\ue877\ue878\ue879\ue87a\ue87b\ue87c\ue87d\ue87e\ue87f\ue880\ue881\ue882\ue883\ue884\ue885\ue886\ue887\ue888\ue889\ue88a\ue88b\ue88c\ue88d\ue88e\ue88f\ue890\ue891\ue892\ue893\ue894\ue895\ue896\ue897\ue898\ue899\ue89a\ue89b\ue89c\ue89d\ue89e\ue89f\ue8a0\ue8a1\ue8a2\ue8a3\ue8a4\ue8a5\ue8a6\ue8a7\ue8a8\ue8a9\ue8aa\ue8ab\ue8ac\ue8ad\ue8ae\ue8af\ue8b0\ue8b1\ue8b2\ue8b3\ue8b4\ue8b5\ue8b6\ue8b7\ue8b8\ue8b9\ue8ba\ue8bb\ue8bc\ue8bd\ue8be\ue8bf\ue8c0\ue8c1\ue8c2\ue8c3\ue8c4\ue8c5\ue8c6\ue8c7\ue8c8\ue8c9\ue8ca\ue8cb\ue8cc\ue8cd\ue8ce\ue8cf\ue8d0\ue8d1\ue8d2\ue8d3\ue8d4\ue8d5\ue8d6\ue8d7\ue8d8\ue8d9\ue8da\ue8db\ue8dc\ue8dd\ue8de\ue8df\ue8e0\ue8e1\ue8e2\ue8e3\ue8e4\ue8e5\ue8e6\ue8e7\ue8e8\ue8e9\ue8ea\ue8eb\ue8ec\ue8ed\ue8ee\ue8ef\ue8f0\ue8f1\ue8f2\ue8f3\ue8f4\ue8f5\ue8f6\ue8f7\ue8f8\ue8f9\ue8fa\ue8fb\ue8fc\ue8fd\ue8fe\ue8ff\ue900\ue901\ue902\ue903\ue904\ue905\ue906\ue907\ue908\ue909\ue90a\ue90b\ue90c\ue90d\ue90e\ue90f\ue910\ue911\ue912\ue913\ue914\ue915\ue916\ue917\ue918\ue919\ue91a\ue91b\ue91c\ue91d\ue91e\ue91f\ue920\ue921\ue922\ue923\ue924\ue925\ue926\ue927\ue928\ue929\ue92a\ue92b\ue92c\ue92d\ue92e\ue92f\ue930\ue931\ue932\ue933\ue934\ue935\ue936\ue937\ue938\ue939\ue93a\ue93b\ue93c\ue93d\ue93e\ue93f\ue940\ue941\ue942\ue943\ue944\ue945\ue946\ue947\ue948\ue949\ue94a\ue94b\ue94c\ue94d\ue94e\ue94f\ue950\ue951\ue952\ue953\ue954\ue955\ue956\ue957\ue958\ue959\ue95a\ue95b\ue95c\ue95d\ue95e\ue95f\ue960\ue961\ue962\ue963\ue964\ue965\ue966\ue967\ue968\ue969\ue96a\ue96b\ue96c\ue96d\ue96e\ue96f\ue970\ue971\ue972\ue973\ue974\ue975\ue976\ue977\ue978\ue979\ue97a\ue97b\ue97c\ue97d\ue97e\ue97f\ue980\ue981\ue982\ue983\ue984\ue985\ue986\ue987\ue988\ue989\ue98a\ue98b\ue98c\ue98d\ue98e\ue98f\ue990\ue991\ue992\ue993\ue994\ue995\ue996\ue997\ue998\ue999\ue99a\ue99b\ue99c\ue99d\ue99e\ue99f\ue9a0\ue9a1\ue9a2\ue9a3\ue9a4\ue9a5\ue9a6\ue9a7\ue9a8\ue9a9\ue9aa\ue9ab\ue9ac\ue9ad\ue9ae\ue9af\ue9b0\ue9b1\ue9b2\ue9b3\ue9b4\ue9b5\ue9b6\ue9b7\ue9b8\ue9b9\ue9ba\ue9bb\ue9bc\ue9bd\ue9be\ue9bf\ue9c0\ue9c1\ue9c2\ue9c3\ue9c4\ue9c5\ue9c6\ue9c7\ue9c8\ue9c9\ue9ca\ue9cb\ue9cc\ue9cd\ue9ce\ue9cf\ue9d0\ue9d1\ue9d2\ue9d3\ue9d4\ue9d5\ue9d6\ue9d7\ue9d8\ue9d9\ue9da\ue9db\ue9dc\ue9dd\ue9de\ue9df\ue9e0\ue9e1\ue9e2\ue9e3\ue9e4\ue9e5\ue9e6\ue9e7\ue9e8\ue9e9\ue9ea\ue9eb\ue9ec\ue9ed\ue9ee\ue9ef\ue9f0\ue9f1\ue9f2\ue9f3\ue9f4\ue9f5\ue9f6\ue9f7\ue9f8\ue9f9\ue9fa\ue9fb\ue9fc\ue9fd\ue9fe\ue9ff\uea00\uea01\uea02\uea03\uea04\uea05\uea06\uea07\uea08\uea09\uea0a\uea0b\uea0c\uea0d\uea0e\uea0f\uea10\uea11\uea12\uea13\uea14\uea15\uea16\uea17\uea18\uea19\uea1a\uea1b\uea1c\uea1d\uea1e\uea1f\uea20\uea21\uea22\uea23\uea24\uea25\uea26\uea27\uea28\uea29\uea2a\uea2b\uea2c\uea2d\uea2e\uea2f\uea30\uea31\uea32\uea33\uea34\uea35\uea36\uea37\uea38\uea39\uea3a\uea3b\uea3c\uea3d\uea3e\uea3f\uea40\uea41\uea42\uea43\uea44\uea45\uea46\uea47\uea48\uea49\uea4a\uea4b\uea4c\uea4d\uea4e\uea4f\uea50\uea51\uea52\uea53\uea54\uea55\uea56\uea57\uea58\uea59\uea5a\uea5b\uea5c\uea5d\uea5e\uea5f\uea60\uea61\uea62\uea63\uea64\uea65\uea66\uea67\uea68\uea69\uea6a\uea6b\uea6c\uea6d\uea6e\uea6f\uea70\uea71\uea72\uea73\uea74\uea75\uea76\uea77\uea78\uea79\uea7a\uea7b\uea7c\uea7d\uea7e\uea7f\uea80\uea81\uea82\uea83\uea84\uea85\uea86\uea87\uea88\uea89\uea8a\uea8b\uea8c\uea8d\uea8e\uea8f\uea90\uea91\uea92\uea93\uea94\uea95\uea96\uea97\uea98\uea99\uea9a\uea9b\uea9c\uea9d\uea9e\uea9f\ueaa0\ueaa1\ueaa2\ueaa3\ueaa4\ueaa5\ueaa6\ueaa7\ueaa8\ueaa9\ueaaa\ueaab\ueaac\ueaad\ueaae\ueaaf\ueab0\ueab1\ueab2\ueab3\ueab4\ueab5\ueab6\ueab7\ueab8\ueab9\ueaba\ueabb\ueabc\ueabd\ueabe\ueabf\ueac0\ueac1\ueac2\ueac3\ueac4\ueac5\ueac6\ueac7\ueac8\ueac9\ueaca\ueacb\ueacc\ueacd\ueace\ueacf\uead0\uead1\uead2\uead3\uead4\uead5\uead6\uead7\uead8\uead9\ueada\ueadb\ueadc\ueadd\ueade\ueadf\ueae0\ueae1\ueae2\ueae3\ueae4\ueae5\ueae6\ueae7\ueae8\ueae9\ueaea\ueaeb\ueaec\ueaed\ueaee\ueaef\ueaf0\ueaf1\ueaf2\ueaf3\ueaf4\ueaf5\ueaf6\ueaf7\ueaf8\ueaf9\ueafa\ueafb\ueafc\ueafd\ueafe\ueaff\ueb00\ueb01\ueb02\ueb03\ueb04\ueb05\ueb06\ueb07\ueb08\ueb09\ueb0a\ueb0b\ueb0c\ueb0d\ueb0e\ueb0f\ueb10\ueb11\ueb12\ueb13\ueb14\ueb15\ueb16\ueb17\ueb18\ueb19\ueb1a\ueb1b\ueb1c\ueb1d\ueb1e\ueb1f\ueb20\ueb21\ueb22\ueb23\ueb24\ueb25\ueb26\ueb27\ueb28\ueb29\ueb2a\ueb2b\ueb2c\ueb2d\ueb2e\ueb2f\ueb30\ueb31\ueb32\ueb33\ueb34\ueb35\ueb36\ueb37\ueb38\ueb39\ueb3a\ueb3b\ueb3c\ueb3d\ueb3e\ueb3f\ueb40\ueb41\ueb42\ueb43\ueb44\ueb45\ueb46\ueb47\ueb48\ueb49\ueb4a\ueb4b\ueb4c\ueb4d\ueb4e\ueb4f\ueb50\ueb51\ueb52\ueb53\ueb54\ueb55\ueb56\ueb57\ueb58\ueb59\ueb5a\ueb5b\ueb5c\ueb5d\ueb5e\ueb5f\ueb60\ueb61\ueb62\ueb63\ueb64\ueb65\ueb66\ueb67\ueb68\ueb69\ueb6a\ueb6b\ueb6c\ueb6d\ueb6e\ueb6f\ueb70\ueb71\ueb72\ueb73\ueb74\ueb75\ueb76\ueb77\ueb78\ueb79\ueb7a\ueb7b\ueb7c\ueb7d\ueb7e\ueb7f\ueb80\ueb81\ueb82\ueb83\ueb84\ueb85\ueb86\ueb87\ueb88\ueb89\ueb8a\ueb8b\ueb8c\ueb8d\ueb8e\ueb8f\ueb90\ueb91\ueb92\ueb93\ueb94\ueb95\ueb96\ueb97\ueb98\ueb99\ueb9a\ueb9b\ueb9c\ueb9d\ueb9e\ueb9f\ueba0\ueba1\ueba2\ueba3\ueba4\ueba5\ueba6\ueba7\ueba8\ueba9\uebaa\uebab\uebac\uebad\uebae\uebaf\uebb0\uebb1\uebb2\uebb3\uebb4\uebb5\uebb6\uebb7\uebb8\uebb9\uebba\uebbb\uebbc\uebbd\uebbe\uebbf\uebc0\uebc1\uebc2\uebc3\uebc4\uebc5\uebc6\uebc7\uebc8\uebc9\uebca\uebcb\uebcc\uebcd\uebce\uebcf\uebd0\uebd1\uebd2\uebd3\uebd4\uebd5\uebd6\uebd7\uebd8\uebd9\uebda\uebdb\uebdc\uebdd\uebde\uebdf\uebe0\uebe1\uebe2\uebe3\uebe4\uebe5\uebe6\uebe7\uebe8\uebe9\uebea\uebeb\uebec\uebed\uebee\uebef\uebf0\uebf1\uebf2\uebf3\uebf4\uebf5\uebf6\uebf7\uebf8\uebf9\uebfa\uebfb\uebfc\uebfd\uebfe\uebff\uec00\uec01\uec02\uec03\uec04\uec05\uec06\uec07\uec08\uec09\uec0a\uec0b\uec0c\uec0d\uec0e\uec0f\uec10\uec11\uec12\uec13\uec14\uec15\uec16\uec17\uec18\uec19\uec1a\uec1b\uec1c\uec1d\uec1e\uec1f\uec20\uec21\uec22\uec23\uec24\uec25\uec26\uec27\uec28\uec29\uec2a\uec2b\uec2c\uec2d\uec2e\uec2f\uec30\uec31\uec32\uec33\uec34\uec35\uec36\uec37\uec38\uec39\uec3a\uec3b\uec3c\uec3d\uec3e\uec3f\uec40\uec41\uec42\uec43\uec44\uec45\uec46\uec47\uec48\uec49\uec4a\uec4b\uec4c\uec4d\uec4e\uec4f\uec50\uec51\uec52\uec53\uec54\uec55\uec56\uec57\uec58\uec59\uec5a\uec5b\uec5c\uec5d\uec5e\uec5f\uec60\uec61\uec62\uec63\uec64\uec65\uec66\uec67\uec68\uec69\uec6a\uec6b\uec6c\uec6d\uec6e\uec6f\uec70\uec71\uec72\uec73\uec74\uec75\uec76\uec77\uec78\uec79\uec7a\uec7b\uec7c\uec7d\uec7e\uec7f\uec80\uec81\uec82\uec83\uec84\uec85\uec86\uec87\uec88\uec89\uec8a\uec8b\uec8c\uec8d\uec8e\uec8f\uec90\uec91\uec92\uec93\uec94\uec95\uec96\uec97\uec98\uec99\uec9a\uec9b\uec9c\uec9d\uec9e\uec9f\ueca0\ueca1\ueca2\ueca3\ueca4\ueca5\ueca6\ueca7\ueca8\ueca9\uecaa\uecab\uecac\uecad\uecae\uecaf\uecb0\uecb1\uecb2\uecb3\uecb4\uecb5\uecb6\uecb7\uecb8\uecb9\uecba\uecbb\uecbc\uecbd\uecbe\uecbf\uecc0\uecc1\uecc2\uecc3\uecc4\uecc5\uecc6\uecc7\uecc8\uecc9\uecca\ueccb\ueccc\ueccd\uecce\ueccf\uecd0\uecd1\uecd2\uecd3\uecd4\uecd5\uecd6\uecd7\uecd8\uecd9\uecda\uecdb\uecdc\uecdd\uecde\uecdf\uece0\uece1\uece2\uece3\uece4\uece5\uece6\uece7\uece8\uece9\uecea\ueceb\uecec\ueced\uecee\uecef\uecf0\uecf1\uecf2\uecf3\uecf4\uecf5\uecf6\uecf7\uecf8\uecf9\uecfa\uecfb\uecfc\uecfd\uecfe\uecff\ued00\ued01\ued02\ued03\ued04\ued05\ued06\ued07\ued08\ued09\ued0a\ued0b\ued0c\ued0d\ued0e\ued0f\ued10\ued11\ued12\ued13\ued14\ued15\ued16\ued17\ued18\ued19\ued1a\ued1b\ued1c\ued1d\ued1e\ued1f\ued20\ued21\ued22\ued23\ued24\ued25\ued26\ued27\ued28\ued29\ued2a\ued2b\ued2c\ued2d\ued2e\ued2f\ued30\ued31\ued32\ued33\ued34\ued35\ued36\ued37\ued38\ued39\ued3a\ued3b\ued3c\ued3d\ued3e\ued3f\ued40\ued41\ued42\ued43\ued44\ued45\ued46\ued47\ued48\ued49\ued4a\ued4b\ued4c\ued4d\ued4e\ued4f\ued50\ued51\ued52\ued53\ued54\ued55\ued56\ued57\ued58\ued59\ued5a\ued5b\ued5c\ued5d\ued5e\ued5f\ued60\ued61\ued62\ued63\ued64\ued65\ued66\ued67\ued68\ued69\ued6a\ued6b\ued6c\ued6d\ued6e\ued6f\ued70\ued71\ued72\ued73\ued74\ued75\ued76\ued77\ued78\ued79\ued7a\ued7b\ued7c\ued7d\ued7e\ued7f\ued80\ued81\ued82\ued83\ued84\ued85\ued86\ued87\ued88\ued89\ued8a\ued8b\ued8c\ued8d\ued8e\ued8f\ued90\ued91\ued92\ued93\ued94\ued95\ued96\ued97\ued98\ued99\ued9a\ued9b\ued9c\ued9d\ued9e\ued9f\ueda0\ueda1\ueda2\ueda3\ueda4\ueda5\ueda6\ueda7\ueda8\ueda9\uedaa\uedab\uedac\uedad\uedae\uedaf\uedb0\uedb1\uedb2\uedb3\uedb4\uedb5\uedb6\uedb7\uedb8\uedb9\uedba\uedbb\uedbc\uedbd\uedbe\uedbf\uedc0\uedc1\uedc2\uedc3\uedc4\uedc5\uedc6\uedc7\uedc8\uedc9\uedca\uedcb\uedcc\uedcd\uedce\uedcf\uedd0\uedd1\uedd2\uedd3\uedd4\uedd5\uedd6\uedd7\uedd8\uedd9\uedda\ueddb\ueddc\ueddd\uedde\ueddf\uede0\uede1\uede2\uede3\uede4\uede5\uede6\uede7\uede8\uede9\uedea\uedeb\uedec\ueded\uedee\uedef\uedf0\uedf1\uedf2\uedf3\uedf4\uedf5\uedf6\uedf7\uedf8\uedf9\uedfa\uedfb\uedfc\uedfd\uedfe\uedff\uee00\uee01\uee02\uee03\uee04\uee05\uee06\uee07\uee08\uee09\uee0a\uee0b\uee0c\uee0d\uee0e\uee0f\uee10\uee11\uee12\uee13\uee14\uee15\uee16\uee17\uee18\uee19\uee1a\uee1b\uee1c\uee1d\uee1e\uee1f\uee20\uee21\uee22\uee23\uee24\uee25\uee26\uee27\uee28\uee29\uee2a\uee2b\uee2c\uee2d\uee2e\uee2f\uee30\uee31\uee32\uee33\uee34\uee35\uee36\uee37\uee38\uee39\uee3a\uee3b\uee3c\uee3d\uee3e\uee3f\uee40\uee41\uee42\uee43\uee44\uee45\uee46\uee47\uee48\uee49\uee4a\uee4b\uee4c\uee4d\uee4e\uee4f\uee50\uee51\uee52\uee53\uee54\uee55\uee56\uee57\uee58\uee59\uee5a\uee5b\uee5c\uee5d\uee5e\uee5f\uee60\uee61\uee62\uee63\uee64\uee65\uee66\uee67\uee68\uee69\uee6a\uee6b\uee6c\uee6d\uee6e\uee6f\uee70\uee71\uee72\uee73\uee74\uee75\uee76\uee77\uee78\uee79\uee7a\uee7b\uee7c\uee7d\uee7e\uee7f\uee80\uee81\uee82\uee83\uee84\uee85\uee86\uee87\uee88\uee89\uee8a\uee8b\uee8c\uee8d\uee8e\uee8f\uee90\uee91\uee92\uee93\uee94\uee95\uee96\uee97\uee98\uee99\uee9a\uee9b\uee9c\uee9d\uee9e\uee9f\ueea0\ueea1\ueea2\ueea3\ueea4\ueea5\ueea6\ueea7\ueea8\ueea9\ueeaa\ueeab\ueeac\ueead\ueeae\ueeaf\ueeb0\ueeb1\ueeb2\ueeb3\ueeb4\ueeb5\ueeb6\ueeb7\ueeb8\ueeb9\ueeba\ueebb\ueebc\ueebd\ueebe\ueebf\ueec0\ueec1\ueec2\ueec3\ueec4\ueec5\ueec6\ueec7\ueec8\ueec9\ueeca\ueecb\ueecc\ueecd\ueece\ueecf\ueed0\ueed1\ueed2\ueed3\ueed4\ueed5\ueed6\ueed7\ueed8\ueed9\ueeda\ueedb\ueedc\ueedd\ueede\ueedf\ueee0\ueee1\ueee2\ueee3\ueee4\ueee5\ueee6\ueee7\ueee8\ueee9\ueeea\ueeeb\ueeec\ueeed\ueeee\ueeef\ueef0\ueef1\ueef2\ueef3\ueef4\ueef5\ueef6\ueef7\ueef8\ueef9\ueefa\ueefb\ueefc\ueefd\ueefe\ueeff\uef00\uef01\uef02\uef03\uef04\uef05\uef06\uef07\uef08\uef09\uef0a\uef0b\uef0c\uef0d\uef0e\uef0f\uef10\uef11\uef12\uef13\uef14\uef15\uef16\uef17\uef18\uef19\uef1a\uef1b\uef1c\uef1d\uef1e\uef1f\uef20\uef21\uef22\uef23\uef24\uef25\uef26\uef27\uef28\uef29\uef2a\uef2b\uef2c\uef2d\uef2e\uef2f\uef30\uef31\uef32\uef33\uef34\uef35\uef36\uef37\uef38\uef39\uef3a\uef3b\uef3c\uef3d\uef3e\uef3f\uef40\uef41\uef42\uef43\uef44\uef45\uef46\uef47\uef48\uef49\uef4a\uef4b\uef4c\uef4d\uef4e\uef4f\uef50\uef51\uef52\uef53\uef54\uef55\uef56\uef57\uef58\uef59\uef5a\uef5b\uef5c\uef5d\uef5e\uef5f\uef60\uef61\uef62\uef63\uef64\uef65\uef66\uef67\uef68\uef69\uef6a\uef6b\uef6c\uef6d\uef6e\uef6f\uef70\uef71\uef72\uef73\uef74\uef75\uef76\uef77\uef78\uef79\uef7a\uef7b\uef7c\uef7d\uef7e\uef7f\uef80\uef81\uef82\uef83\uef84\uef85\uef86\uef87\uef88\uef89\uef8a\uef8b\uef8c\uef8d\uef8e\uef8f\uef90\uef91\uef92\uef93\uef94\uef95\uef96\uef97\uef98\uef99\uef9a\uef9b\uef9c\uef9d\uef9e\uef9f\uefa0\uefa1\uefa2\uefa3\uefa4\uefa5\uefa6\uefa7\uefa8\uefa9\uefaa\uefab\uefac\uefad\uefae\uefaf\uefb0\uefb1\uefb2\uefb3\uefb4\uefb5\uefb6\uefb7\uefb8\uefb9\uefba\uefbb\uefbc\uefbd\uefbe\uefbf\uefc0\uefc1\uefc2\uefc3\uefc4\uefc5\uefc6\uefc7\uefc8\uefc9\uefca\uefcb\uefcc\uefcd\uefce\uefcf\uefd0\uefd1\uefd2\uefd3\uefd4\uefd5\uefd6\uefd7\uefd8\uefd9\uefda\uefdb\uefdc\uefdd\uefde\uefdf\uefe0\uefe1\uefe2\uefe3\uefe4\uefe5\uefe6\uefe7\uefe8\uefe9\uefea\uefeb\uefec\uefed\uefee\uefef\ueff0\ueff1\ueff2\ueff3\ueff4\ueff5\ueff6\ueff7\ueff8\ueff9\ueffa\ueffb\ueffc\ueffd\ueffe\uefff\uf000\uf001\uf002\uf003\uf004\uf005\uf006\uf007\uf008\uf009\uf00a\uf00b\uf00c\uf00d\uf00e\uf00f\uf010\uf011\uf012\uf013\uf014\uf015\uf016\uf017\uf018\uf019\uf01a\uf01b\uf01c\uf01d\uf01e\uf01f\uf020\uf021\uf022\uf023\uf024\uf025\uf026\uf027\uf028\uf029\uf02a\uf02b\uf02c\uf02d\uf02e\uf02f\uf030\uf031\uf032\uf033\uf034\uf035\uf036\uf037\uf038\uf039\uf03a\uf03b\uf03c\uf03d\uf03e\uf03f\uf040\uf041\uf042\uf043\uf044\uf045\uf046\uf047\uf048\uf049\uf04a\uf04b\uf04c\uf04d\uf04e\uf04f\uf050\uf051\uf052\uf053\uf054\uf055\uf056\uf057\uf058\uf059\uf05a\uf05b\uf05c\uf05d\uf05e\uf05f\uf060\uf061\uf062\uf063\uf064\uf065\uf066\uf067\uf068\uf069\uf06a\uf06b\uf06c\uf06d\uf06e\uf06f\uf070\uf071\uf072\uf073\uf074\uf075\uf076\uf077\uf078\uf079\uf07a\uf07b\uf07c\uf07d\uf07e\uf07f\uf080\uf081\uf082\uf083\uf084\uf085\uf086\uf087\uf088\uf089\uf08a\uf08b\uf08c\uf08d\uf08e\uf08f\uf090\uf091\uf092\uf093\uf094\uf095\uf096\uf097\uf098\uf099\uf09a\uf09b\uf09c\uf09d\uf09e\uf09f\uf0a0\uf0a1\uf0a2\uf0a3\uf0a4\uf0a5\uf0a6\uf0a7\uf0a8\uf0a9\uf0aa\uf0ab\uf0ac\uf0ad\uf0ae\uf0af\uf0b0\uf0b1\uf0b2\uf0b3\uf0b4\uf0b5\uf0b6\uf0b7\uf0b8\uf0b9\uf0ba\uf0bb\uf0bc\uf0bd\uf0be\uf0bf\uf0c0\uf0c1\uf0c2\uf0c3\uf0c4\uf0c5\uf0c6\uf0c7\uf0c8\uf0c9\uf0ca\uf0cb\uf0cc\uf0cd\uf0ce\uf0cf\uf0d0\uf0d1\uf0d2\uf0d3\uf0d4\uf0d5\uf0d6\uf0d7\uf0d8\uf0d9\uf0da\uf0db\uf0dc\uf0dd\uf0de\uf0df\uf0e0\uf0e1\uf0e2\uf0e3\uf0e4\uf0e5\uf0e6\uf0e7\uf0e8\uf0e9\uf0ea\uf0eb\uf0ec\uf0ed\uf0ee\uf0ef\uf0f0\uf0f1\uf0f2\uf0f3\uf0f4\uf0f5\uf0f6\uf0f7\uf0f8\uf0f9\uf0fa\uf0fb\uf0fc\uf0fd\uf0fe\uf0ff\uf100\uf101\uf102\uf103\uf104\uf105\uf106\uf107\uf108\uf109\uf10a\uf10b\uf10c\uf10d\uf10e\uf10f\uf110\uf111\uf112\uf113\uf114\uf115\uf116\uf117\uf118\uf119\uf11a\uf11b\uf11c\uf11d\uf11e\uf11f\uf120\uf121\uf122\uf123\uf124\uf125\uf126\uf127\uf128\uf129\uf12a\uf12b\uf12c\uf12d\uf12e\uf12f\uf130\uf131\uf132\uf133\uf134\uf135\uf136\uf137\uf138\uf139\uf13a\uf13b\uf13c\uf13d\uf13e\uf13f\uf140\uf141\uf142\uf143\uf144\uf145\uf146\uf147\uf148\uf149\uf14a\uf14b\uf14c\uf14d\uf14e\uf14f\uf150\uf151\uf152\uf153\uf154\uf155\uf156\uf157\uf158\uf159\uf15a\uf15b\uf15c\uf15d\uf15e\uf15f\uf160\uf161\uf162\uf163\uf164\uf165\uf166\uf167\uf168\uf169\uf16a\uf16b\uf16c\uf16d\uf16e\uf16f\uf170\uf171\uf172\uf173\uf174\uf175\uf176\uf177\uf178\uf179\uf17a\uf17b\uf17c\uf17d\uf17e\uf17f\uf180\uf181\uf182\uf183\uf184\uf185\uf186\uf187\uf188\uf189\uf18a\uf18b\uf18c\uf18d\uf18e\uf18f\uf190\uf191\uf192\uf193\uf194\uf195\uf196\uf197\uf198\uf199\uf19a\uf19b\uf19c\uf19d\uf19e\uf19f\uf1a0\uf1a1\uf1a2\uf1a3\uf1a4\uf1a5\uf1a6\uf1a7\uf1a8\uf1a9\uf1aa\uf1ab\uf1ac\uf1ad\uf1ae\uf1af\uf1b0\uf1b1\uf1b2\uf1b3\uf1b4\uf1b5\uf1b6\uf1b7\uf1b8\uf1b9\uf1ba\uf1bb\uf1bc\uf1bd\uf1be\uf1bf\uf1c0\uf1c1\uf1c2\uf1c3\uf1c4\uf1c5\uf1c6\uf1c7\uf1c8\uf1c9\uf1ca\uf1cb\uf1cc\uf1cd\uf1ce\uf1cf\uf1d0\uf1d1\uf1d2\uf1d3\uf1d4\uf1d5\uf1d6\uf1d7\uf1d8\uf1d9\uf1da\uf1db\uf1dc\uf1dd\uf1de\uf1df\uf1e0\uf1e1\uf1e2\uf1e3\uf1e4\uf1e5\uf1e6\uf1e7\uf1e8\uf1e9\uf1ea\uf1eb\uf1ec\uf1ed\uf1ee\uf1ef\uf1f0\uf1f1\uf1f2\uf1f3\uf1f4\uf1f5\uf1f6\uf1f7\uf1f8\uf1f9\uf1fa\uf1fb\uf1fc\uf1fd\uf1fe\uf1ff\uf200\uf201\uf202\uf203\uf204\uf205\uf206\uf207\uf208\uf209\uf20a\uf20b\uf20c\uf20d\uf20e\uf20f\uf210\uf211\uf212\uf213\uf214\uf215\uf216\uf217\uf218\uf219\uf21a\uf21b\uf21c\uf21d\uf21e\uf21f\uf220\uf221\uf222\uf223\uf224\uf225\uf226\uf227\uf228\uf229\uf22a\uf22b\uf22c\uf22d\uf22e\uf22f\uf230\uf231\uf232\uf233\uf234\uf235\uf236\uf237\uf238\uf239\uf23a\uf23b\uf23c\uf23d\uf23e\uf23f\uf240\uf241\uf242\uf243\uf244\uf245\uf246\uf247\uf248\uf249\uf24a\uf24b\uf24c\uf24d\uf24e\uf24f\uf250\uf251\uf252\uf253\uf254\uf255\uf256\uf257\uf258\uf259\uf25a\uf25b\uf25c\uf25d\uf25e\uf25f\uf260\uf261\uf262\uf263\uf264\uf265\uf266\uf267\uf268\uf269\uf26a\uf26b\uf26c\uf26d\uf26e\uf26f\uf270\uf271\uf272\uf273\uf274\uf275\uf276\uf277\uf278\uf279\uf27a\uf27b\uf27c\uf27d\uf27e\uf27f\uf280\uf281\uf282\uf283\uf284\uf285\uf286\uf287\uf288\uf289\uf28a\uf28b\uf28c\uf28d\uf28e\uf28f\uf290\uf291\uf292\uf293\uf294\uf295\uf296\uf297\uf298\uf299\uf29a\uf29b\uf29c\uf29d\uf29e\uf29f\uf2a0\uf2a1\uf2a2\uf2a3\uf2a4\uf2a5\uf2a6\uf2a7\uf2a8\uf2a9\uf2aa\uf2ab\uf2ac\uf2ad\uf2ae\uf2af\uf2b0\uf2b1\uf2b2\uf2b3\uf2b4\uf2b5\uf2b6\uf2b7\uf2b8\uf2b9\uf2ba\uf2bb\uf2bc\uf2bd\uf2be\uf2bf\uf2c0\uf2c1\uf2c2\uf2c3\uf2c4\uf2c5\uf2c6\uf2c7\uf2c8\uf2c9\uf2ca\uf2cb\uf2cc\uf2cd\uf2ce\uf2cf\uf2d0\uf2d1\uf2d2\uf2d3\uf2d4\uf2d5\uf2d6\uf2d7\uf2d8\uf2d9\uf2da\uf2db\uf2dc\uf2dd\uf2de\uf2df\uf2e0\uf2e1\uf2e2\uf2e3\uf2e4\uf2e5\uf2e6\uf2e7\uf2e8\uf2e9\uf2ea\uf2eb\uf2ec\uf2ed\uf2ee\uf2ef\uf2f0\uf2f1\uf2f2\uf2f3\uf2f4\uf2f5\uf2f6\uf2f7\uf2f8\uf2f9\uf2fa\uf2fb\uf2fc\uf2fd\uf2fe\uf2ff\uf300\uf301\uf302\uf303\uf304\uf305\uf306\uf307\uf308\uf309\uf30a\uf30b\uf30c\uf30d\uf30e\uf30f\uf310\uf311\uf312\uf313\uf314\uf315\uf316\uf317\uf318\uf319\uf31a\uf31b\uf31c\uf31d\uf31e\uf31f\uf320\uf321\uf322\uf323\uf324\uf325\uf326\uf327\uf328\uf329\uf32a\uf32b\uf32c\uf32d\uf32e\uf32f\uf330\uf331\uf332\uf333\uf334\uf335\uf336\uf337\uf338\uf339\uf33a\uf33b\uf33c\uf33d\uf33e\uf33f\uf340\uf341\uf342\uf343\uf344\uf345\uf346\uf347\uf348\uf349\uf34a\uf34b\uf34c\uf34d\uf34e\uf34f\uf350\uf351\uf352\uf353\uf354\uf355\uf356\uf357\uf358\uf359\uf35a\uf35b\uf35c\uf35d\uf35e\uf35f\uf360\uf361\uf362\uf363\uf364\uf365\uf366\uf367\uf368\uf369\uf36a\uf36b\uf36c\uf36d\uf36e\uf36f\uf370\uf371\uf372\uf373\uf374\uf375\uf376\uf377\uf378\uf379\uf37a\uf37b\uf37c\uf37d\uf37e\uf37f\uf380\uf381\uf382\uf383\uf384\uf385\uf386\uf387\uf388\uf389\uf38a\uf38b\uf38c\uf38d\uf38e\uf38f\uf390\uf391\uf392\uf393\uf394\uf395\uf396\uf397\uf398\uf399\uf39a\uf39b\uf39c\uf39d\uf39e\uf39f\uf3a0\uf3a1\uf3a2\uf3a3\uf3a4\uf3a5\uf3a6\uf3a7\uf3a8\uf3a9\uf3aa\uf3ab\uf3ac\uf3ad\uf3ae\uf3af\uf3b0\uf3b1\uf3b2\uf3b3\uf3b4\uf3b5\uf3b6\uf3b7\uf3b8\uf3b9\uf3ba\uf3bb\uf3bc\uf3bd\uf3be\uf3bf\uf3c0\uf3c1\uf3c2\uf3c3\uf3c4\uf3c5\uf3c6\uf3c7\uf3c8\uf3c9\uf3ca\uf3cb\uf3cc\uf3cd\uf3ce\uf3cf\uf3d0\uf3d1\uf3d2\uf3d3\uf3d4\uf3d5\uf3d6\uf3d7\uf3d8\uf3d9\uf3da\uf3db\uf3dc\uf3dd\uf3de\uf3df\uf3e0\uf3e1\uf3e2\uf3e3\uf3e4\uf3e5\uf3e6\uf3e7\uf3e8\uf3e9\uf3ea\uf3eb\uf3ec\uf3ed\uf3ee\uf3ef\uf3f0\uf3f1\uf3f2\uf3f3\uf3f4\uf3f5\uf3f6\uf3f7\uf3f8\uf3f9\uf3fa\uf3fb\uf3fc\uf3fd\uf3fe\uf3ff\uf400\uf401\uf402\uf403\uf404\uf405\uf406\uf407\uf408\uf409\uf40a\uf40b\uf40c\uf40d\uf40e\uf40f\uf410\uf411\uf412\uf413\uf414\uf415\uf416\uf417\uf418\uf419\uf41a\uf41b\uf41c\uf41d\uf41e\uf41f\uf420\uf421\uf422\uf423\uf424\uf425\uf426\uf427\uf428\uf429\uf42a\uf42b\uf42c\uf42d\uf42e\uf42f\uf430\uf431\uf432\uf433\uf434\uf435\uf436\uf437\uf438\uf439\uf43a\uf43b\uf43c\uf43d\uf43e\uf43f\uf440\uf441\uf442\uf443\uf444\uf445\uf446\uf447\uf448\uf449\uf44a\uf44b\uf44c\uf44d\uf44e\uf44f\uf450\uf451\uf452\uf453\uf454\uf455\uf456\uf457\uf458\uf459\uf45a\uf45b\uf45c\uf45d\uf45e\uf45f\uf460\uf461\uf462\uf463\uf464\uf465\uf466\uf467\uf468\uf469\uf46a\uf46b\uf46c\uf46d\uf46e\uf46f\uf470\uf471\uf472\uf473\uf474\uf475\uf476\uf477\uf478\uf479\uf47a\uf47b\uf47c\uf47d\uf47e\uf47f\uf480\uf481\uf482\uf483\uf484\uf485\uf486\uf487\uf488\uf489\uf48a\uf48b\uf48c\uf48d\uf48e\uf48f\uf490\uf491\uf492\uf493\uf494\uf495\uf496\uf497\uf498\uf499\uf49a\uf49b\uf49c\uf49d\uf49e\uf49f\uf4a0\uf4a1\uf4a2\uf4a3\uf4a4\uf4a5\uf4a6\uf4a7\uf4a8\uf4a9\uf4aa\uf4ab\uf4ac\uf4ad\uf4ae\uf4af\uf4b0\uf4b1\uf4b2\uf4b3\uf4b4\uf4b5\uf4b6\uf4b7\uf4b8\uf4b9\uf4ba\uf4bb\uf4bc\uf4bd\uf4be\uf4bf\uf4c0\uf4c1\uf4c2\uf4c3\uf4c4\uf4c5\uf4c6\uf4c7\uf4c8\uf4c9\uf4ca\uf4cb\uf4cc\uf4cd\uf4ce\uf4cf\uf4d0\uf4d1\uf4d2\uf4d3\uf4d4\uf4d5\uf4d6\uf4d7\uf4d8\uf4d9\uf4da\uf4db\uf4dc\uf4dd\uf4de\uf4df\uf4e0\uf4e1\uf4e2\uf4e3\uf4e4\uf4e5\uf4e6\uf4e7\uf4e8\uf4e9\uf4ea\uf4eb\uf4ec\uf4ed\uf4ee\uf4ef\uf4f0\uf4f1\uf4f2\uf4f3\uf4f4\uf4f5\uf4f6\uf4f7\uf4f8\uf4f9\uf4fa\uf4fb\uf4fc\uf4fd\uf4fe\uf4ff\uf500\uf501\uf502\uf503\uf504\uf505\uf506\uf507\uf508\uf509\uf50a\uf50b\uf50c\uf50d\uf50e\uf50f\uf510\uf511\uf512\uf513\uf514\uf515\uf516\uf517\uf518\uf519\uf51a\uf51b\uf51c\uf51d\uf51e\uf51f\uf520\uf521\uf522\uf523\uf524\uf525\uf526\uf527\uf528\uf529\uf52a\uf52b\uf52c\uf52d\uf52e\uf52f\uf530\uf531\uf532\uf533\uf534\uf535\uf536\uf537\uf538\uf539\uf53a\uf53b\uf53c\uf53d\uf53e\uf53f\uf540\uf541\uf542\uf543\uf544\uf545\uf546\uf547\uf548\uf549\uf54a\uf54b\uf54c\uf54d\uf54e\uf54f\uf550\uf551\uf552\uf553\uf554\uf555\uf556\uf557\uf558\uf559\uf55a\uf55b\uf55c\uf55d\uf55e\uf55f\uf560\uf561\uf562\uf563\uf564\uf565\uf566\uf567\uf568\uf569\uf56a\uf56b\uf56c\uf56d\uf56e\uf56f\uf570\uf571\uf572\uf573\uf574\uf575\uf576\uf577\uf578\uf579\uf57a\uf57b\uf57c\uf57d\uf57e\uf57f\uf580\uf581\uf582\uf583\uf584\uf585\uf586\uf587\uf588\uf589\uf58a\uf58b\uf58c\uf58d\uf58e\uf58f\uf590\uf591\uf592\uf593\uf594\uf595\uf596\uf597\uf598\uf599\uf59a\uf59b\uf59c\uf59d\uf59e\uf59f\uf5a0\uf5a1\uf5a2\uf5a3\uf5a4\uf5a5\uf5a6\uf5a7\uf5a8\uf5a9\uf5aa\uf5ab\uf5ac\uf5ad\uf5ae\uf5af\uf5b0\uf5b1\uf5b2\uf5b3\uf5b4\uf5b5\uf5b6\uf5b7\uf5b8\uf5b9\uf5ba\uf5bb\uf5bc\uf5bd\uf5be\uf5bf\uf5c0\uf5c1\uf5c2\uf5c3\uf5c4\uf5c5\uf5c6\uf5c7\uf5c8\uf5c9\uf5ca\uf5cb\uf5cc\uf5cd\uf5ce\uf5cf\uf5d0\uf5d1\uf5d2\uf5d3\uf5d4\uf5d5\uf5d6\uf5d7\uf5d8\uf5d9\uf5da\uf5db\uf5dc\uf5dd\uf5de\uf5df\uf5e0\uf5e1\uf5e2\uf5e3\uf5e4\uf5e5\uf5e6\uf5e7\uf5e8\uf5e9\uf5ea\uf5eb\uf5ec\uf5ed\uf5ee\uf5ef\uf5f0\uf5f1\uf5f2\uf5f3\uf5f4\uf5f5\uf5f6\uf5f7\uf5f8\uf5f9\uf5fa\uf5fb\uf5fc\uf5fd\uf5fe\uf5ff\uf600\uf601\uf602\uf603\uf604\uf605\uf606\uf607\uf608\uf609\uf60a\uf60b\uf60c\uf60d\uf60e\uf60f\uf610\uf611\uf612\uf613\uf614\uf615\uf616\uf617\uf618\uf619\uf61a\uf61b\uf61c\uf61d\uf61e\uf61f\uf620\uf621\uf622\uf623\uf624\uf625\uf626\uf627\uf628\uf629\uf62a\uf62b\uf62c\uf62d\uf62e\uf62f\uf630\uf631\uf632\uf633\uf634\uf635\uf636\uf637\uf638\uf639\uf63a\uf63b\uf63c\uf63d\uf63e\uf63f\uf640\uf641\uf642\uf643\uf644\uf645\uf646\uf647\uf648\uf649\uf64a\uf64b\uf64c\uf64d\uf64e\uf64f\uf650\uf651\uf652\uf653\uf654\uf655\uf656\uf657\uf658\uf659\uf65a\uf65b\uf65c\uf65d\uf65e\uf65f\uf660\uf661\uf662\uf663\uf664\uf665\uf666\uf667\uf668\uf669\uf66a\uf66b\uf66c\uf66d\uf66e\uf66f\uf670\uf671\uf672\uf673\uf674\uf675\uf676\uf677\uf678\uf679\uf67a\uf67b\uf67c\uf67d\uf67e\uf67f\uf680\uf681\uf682\uf683\uf684\uf685\uf686\uf687\uf688\uf689\uf68a\uf68b\uf68c\uf68d\uf68e\uf68f\uf690\uf691\uf692\uf693\uf694\uf695\uf696\uf697\uf698\uf699\uf69a\uf69b\uf69c\uf69d\uf69e\uf69f\uf6a0\uf6a1\uf6a2\uf6a3\uf6a4\uf6a5\uf6a6\uf6a7\uf6a8\uf6a9\uf6aa\uf6ab\uf6ac\uf6ad\uf6ae\uf6af\uf6b0\uf6b1\uf6b2\uf6b3\uf6b4\uf6b5\uf6b6\uf6b7\uf6b8\uf6b9\uf6ba\uf6bb\uf6bc\uf6bd\uf6be\uf6bf\uf6c0\uf6c1\uf6c2\uf6c3\uf6c4\uf6c5\uf6c6\uf6c7\uf6c8\uf6c9\uf6ca\uf6cb\uf6cc\uf6cd\uf6ce\uf6cf\uf6d0\uf6d1\uf6d2\uf6d3\uf6d4\uf6d5\uf6d6\uf6d7\uf6d8\uf6d9\uf6da\uf6db\uf6dc\uf6dd\uf6de\uf6df\uf6e0\uf6e1\uf6e2\uf6e3\uf6e4\uf6e5\uf6e6\uf6e7\uf6e8\uf6e9\uf6ea\uf6eb\uf6ec\uf6ed\uf6ee\uf6ef\uf6f0\uf6f1\uf6f2\uf6f3\uf6f4\uf6f5\uf6f6\uf6f7\uf6f8\uf6f9\uf6fa\uf6fb\uf6fc\uf6fd\uf6fe\uf6ff\uf700\uf701\uf702\uf703\uf704\uf705\uf706\uf707\uf708\uf709\uf70a\uf70b\uf70c\uf70d\uf70e\uf70f\uf710\uf711\uf712\uf713\uf714\uf715\uf716\uf717\uf718\uf719\uf71a\uf71b\uf71c\uf71d\uf71e\uf71f\uf720\uf721\uf722\uf723\uf724\uf725\uf726\uf727\uf728\uf729\uf72a\uf72b\uf72c\uf72d\uf72e\uf72f\uf730\uf731\uf732\uf733\uf734\uf735\uf736\uf737\uf738\uf739\uf73a\uf73b\uf73c\uf73d\uf73e\uf73f\uf740\uf741\uf742\uf743\uf744\uf745\uf746\uf747\uf748\uf749\uf74a\uf74b\uf74c\uf74d\uf74e\uf74f\uf750\uf751\uf752\uf753\uf754\uf755\uf756\uf757\uf758\uf759\uf75a\uf75b\uf75c\uf75d\uf75e\uf75f\uf760\uf761\uf762\uf763\uf764\uf765\uf766\uf767\uf768\uf769\uf76a\uf76b\uf76c\uf76d\uf76e\uf76f\uf770\uf771\uf772\uf773\uf774\uf775\uf776\uf777\uf778\uf779\uf77a\uf77b\uf77c\uf77d\uf77e\uf77f\uf780\uf781\uf782\uf783\uf784\uf785\uf786\uf787\uf788\uf789\uf78a\uf78b\uf78c\uf78d\uf78e\uf78f\uf790\uf791\uf792\uf793\uf794\uf795\uf796\uf797\uf798\uf799\uf79a\uf79b\uf79c\uf79d\uf79e\uf79f\uf7a0\uf7a1\uf7a2\uf7a3\uf7a4\uf7a5\uf7a6\uf7a7\uf7a8\uf7a9\uf7aa\uf7ab\uf7ac\uf7ad\uf7ae\uf7af\uf7b0\uf7b1\uf7b2\uf7b3\uf7b4\uf7b5\uf7b6\uf7b7\uf7b8\uf7b9\uf7ba\uf7bb\uf7bc\uf7bd\uf7be\uf7bf\uf7c0\uf7c1\uf7c2\uf7c3\uf7c4\uf7c5\uf7c6\uf7c7\uf7c8\uf7c9\uf7ca\uf7cb\uf7cc\uf7cd\uf7ce\uf7cf\uf7d0\uf7d1\uf7d2\uf7d3\uf7d4\uf7d5\uf7d6\uf7d7\uf7d8\uf7d9\uf7da\uf7db\uf7dc\uf7dd\uf7de\uf7df\uf7e0\uf7e1\uf7e2\uf7e3\uf7e4\uf7e5\uf7e6\uf7e7\uf7e8\uf7e9\uf7ea\uf7eb\uf7ec\uf7ed\uf7ee\uf7ef\uf7f0\uf7f1\uf7f2\uf7f3\uf7f4\uf7f5\uf7f6\uf7f7\uf7f8\uf7f9\uf7fa\uf7fb\uf7fc\uf7fd\uf7fe\uf7ff\uf800\uf801\uf802\uf803\uf804\uf805\uf806\uf807\uf808\uf809\uf80a\uf80b\uf80c\uf80d\uf80e\uf80f\uf810\uf811\uf812\uf813\uf814\uf815\uf816\uf817\uf818\uf819\uf81a\uf81b\uf81c\uf81d\uf81e\uf81f\uf820\uf821\uf822\uf823\uf824\uf825\uf826\uf827\uf828\uf829\uf82a\uf82b\uf82c\uf82d\uf82e\uf82f\uf830\uf831\uf832\uf833\uf834\uf835\uf836\uf837\uf838\uf839\uf83a\uf83b\uf83c\uf83d\uf83e\uf83f\uf840\uf841\uf842\uf843\uf844\uf845\uf846\uf847\uf848\uf849\uf84a\uf84b\uf84c\uf84d\uf84e\uf84f\uf850\uf851\uf852\uf853\uf854\uf855\uf856\uf857\uf858\uf859\uf85a\uf85b\uf85c\uf85d\uf85e\uf85f\uf860\uf861\uf862\uf863\uf864\uf865\uf866\uf867\uf868\uf869\uf86a\uf86b\uf86c\uf86d\uf86e\uf86f\uf870\uf871\uf872\uf873\uf874\uf875\uf876\uf877\uf878\uf879\uf87a\uf87b\uf87c\uf87d\uf87e\uf87f\uf880\uf881\uf882\uf883\uf884\uf885\uf886\uf887\uf888\uf889\uf88a\uf88b\uf88c\uf88d\uf88e\uf88f\uf890\uf891\uf892\uf893\uf894\uf895\uf896\uf897\uf898\uf899\uf89a\uf89b\uf89c\uf89d\uf89e\uf89f\uf8a0\uf8a1\uf8a2\uf8a3\uf8a4\uf8a5\uf8a6\uf8a7\uf8a8\uf8a9\uf8aa\uf8ab\uf8ac\uf8ad\uf8ae\uf8af\uf8b0\uf8b1\uf8b2\uf8b3\uf8b4\uf8b5\uf8b6\uf8b7\uf8b8\uf8b9\uf8ba\uf8bb\uf8bc\uf8bd\uf8be\uf8bf\uf8c0\uf8c1\uf8c2\uf8c3\uf8c4\uf8c5\uf8c6\uf8c7\uf8c8\uf8c9\uf8ca\uf8cb\uf8cc\uf8cd\uf8ce\uf8cf\uf8d0\uf8d1\uf8d2\uf8d3\uf8d4\uf8d5\uf8d6\uf8d7\uf8d8\uf8d9\uf8da\uf8db\uf8dc\uf8dd\uf8de\uf8df\uf8e0\uf8e1\uf8e2\uf8e3\uf8e4\uf8e5\uf8e6\uf8e7\uf8e8\uf8e9\uf8ea\uf8eb\uf8ec\uf8ed\uf8ee\uf8ef\uf8f0\uf8f1\uf8f2\uf8f3\uf8f4\uf8f5\uf8f6\uf8f7\uf8f8\uf8f9\uf8fa\uf8fb\uf8fc\uf8fd\uf8fe\uf8ff' + +try: + Cs = eval(r"'\ud800\ud801\ud802\ud803\ud804\ud805\ud806\ud807\ud808\ud809\ud80a\ud80b\ud80c\ud80d\ud80e\ud80f\ud810\ud811\ud812\ud813\ud814\ud815\ud816\ud817\ud818\ud819\ud81a\ud81b\ud81c\ud81d\ud81e\ud81f\ud820\ud821\ud822\ud823\ud824\ud825\ud826\ud827\ud828\ud829\ud82a\ud82b\ud82c\ud82d\ud82e\ud82f\ud830\ud831\ud832\ud833\ud834\ud835\ud836\ud837\ud838\ud839\ud83a\ud83b\ud83c\ud83d\ud83e\ud83f\ud840\ud841\ud842\ud843\ud844\ud845\ud846\ud847\ud848\ud849\ud84a\ud84b\ud84c\ud84d\ud84e\ud84f\ud850\ud851\ud852\ud853\ud854\ud855\ud856\ud857\ud858\ud859\ud85a\ud85b\ud85c\ud85d\ud85e\ud85f\ud860\ud861\ud862\ud863\ud864\ud865\ud866\ud867\ud868\ud869\ud86a\ud86b\ud86c\ud86d\ud86e\ud86f\ud870\ud871\ud872\ud873\ud874\ud875\ud876\ud877\ud878\ud879\ud87a\ud87b\ud87c\ud87d\ud87e\ud87f\ud880\ud881\ud882\ud883\ud884\ud885\ud886\ud887\ud888\ud889\ud88a\ud88b\ud88c\ud88d\ud88e\ud88f\ud890\ud891\ud892\ud893\ud894\ud895\ud896\ud897\ud898\ud899\ud89a\ud89b\ud89c\ud89d\ud89e\ud89f\ud8a0\ud8a1\ud8a2\ud8a3\ud8a4\ud8a5\ud8a6\ud8a7\ud8a8\ud8a9\ud8aa\ud8ab\ud8ac\ud8ad\ud8ae\ud8af\ud8b0\ud8b1\ud8b2\ud8b3\ud8b4\ud8b5\ud8b6\ud8b7\ud8b8\ud8b9\ud8ba\ud8bb\ud8bc\ud8bd\ud8be\ud8bf\ud8c0\ud8c1\ud8c2\ud8c3\ud8c4\ud8c5\ud8c6\ud8c7\ud8c8\ud8c9\ud8ca\ud8cb\ud8cc\ud8cd\ud8ce\ud8cf\ud8d0\ud8d1\ud8d2\ud8d3\ud8d4\ud8d5\ud8d6\ud8d7\ud8d8\ud8d9\ud8da\ud8db\ud8dc\ud8dd\ud8de\ud8df\ud8e0\ud8e1\ud8e2\ud8e3\ud8e4\ud8e5\ud8e6\ud8e7\ud8e8\ud8e9\ud8ea\ud8eb\ud8ec\ud8ed\ud8ee\ud8ef\ud8f0\ud8f1\ud8f2\ud8f3\ud8f4\ud8f5\ud8f6\ud8f7\ud8f8\ud8f9\ud8fa\ud8fb\ud8fc\ud8fd\ud8fe\ud8ff\ud900\ud901\ud902\ud903\ud904\ud905\ud906\ud907\ud908\ud909\ud90a\ud90b\ud90c\ud90d\ud90e\ud90f\ud910\ud911\ud912\ud913\ud914\ud915\ud916\ud917\ud918\ud919\ud91a\ud91b\ud91c\ud91d\ud91e\ud91f\ud920\ud921\ud922\ud923\ud924\ud925\ud926\ud927\ud928\ud929\ud92a\ud92b\ud92c\ud92d\ud92e\ud92f\ud930\ud931\ud932\ud933\ud934\ud935\ud936\ud937\ud938\ud939\ud93a\ud93b\ud93c\ud93d\ud93e\ud93f\ud940\ud941\ud942\ud943\ud944\ud945\ud946\ud947\ud948\ud949\ud94a\ud94b\ud94c\ud94d\ud94e\ud94f\ud950\ud951\ud952\ud953\ud954\ud955\ud956\ud957\ud958\ud959\ud95a\ud95b\ud95c\ud95d\ud95e\ud95f\ud960\ud961\ud962\ud963\ud964\ud965\ud966\ud967\ud968\ud969\ud96a\ud96b\ud96c\ud96d\ud96e\ud96f\ud970\ud971\ud972\ud973\ud974\ud975\ud976\ud977\ud978\ud979\ud97a\ud97b\ud97c\ud97d\ud97e\ud97f\ud980\ud981\ud982\ud983\ud984\ud985\ud986\ud987\ud988\ud989\ud98a\ud98b\ud98c\ud98d\ud98e\ud98f\ud990\ud991\ud992\ud993\ud994\ud995\ud996\ud997\ud998\ud999\ud99a\ud99b\ud99c\ud99d\ud99e\ud99f\ud9a0\ud9a1\ud9a2\ud9a3\ud9a4\ud9a5\ud9a6\ud9a7\ud9a8\ud9a9\ud9aa\ud9ab\ud9ac\ud9ad\ud9ae\ud9af\ud9b0\ud9b1\ud9b2\ud9b3\ud9b4\ud9b5\ud9b6\ud9b7\ud9b8\ud9b9\ud9ba\ud9bb\ud9bc\ud9bd\ud9be\ud9bf\ud9c0\ud9c1\ud9c2\ud9c3\ud9c4\ud9c5\ud9c6\ud9c7\ud9c8\ud9c9\ud9ca\ud9cb\ud9cc\ud9cd\ud9ce\ud9cf\ud9d0\ud9d1\ud9d2\ud9d3\ud9d4\ud9d5\ud9d6\ud9d7\ud9d8\ud9d9\ud9da\ud9db\ud9dc\ud9dd\ud9de\ud9df\ud9e0\ud9e1\ud9e2\ud9e3\ud9e4\ud9e5\ud9e6\ud9e7\ud9e8\ud9e9\ud9ea\ud9eb\ud9ec\ud9ed\ud9ee\ud9ef\ud9f0\ud9f1\ud9f2\ud9f3\ud9f4\ud9f5\ud9f6\ud9f7\ud9f8\ud9f9\ud9fa\ud9fb\ud9fc\ud9fd\ud9fe\ud9ff\uda00\uda01\uda02\uda03\uda04\uda05\uda06\uda07\uda08\uda09\uda0a\uda0b\uda0c\uda0d\uda0e\uda0f\uda10\uda11\uda12\uda13\uda14\uda15\uda16\uda17\uda18\uda19\uda1a\uda1b\uda1c\uda1d\uda1e\uda1f\uda20\uda21\uda22\uda23\uda24\uda25\uda26\uda27\uda28\uda29\uda2a\uda2b\uda2c\uda2d\uda2e\uda2f\uda30\uda31\uda32\uda33\uda34\uda35\uda36\uda37\uda38\uda39\uda3a\uda3b\uda3c\uda3d\uda3e\uda3f\uda40\uda41\uda42\uda43\uda44\uda45\uda46\uda47\uda48\uda49\uda4a\uda4b\uda4c\uda4d\uda4e\uda4f\uda50\uda51\uda52\uda53\uda54\uda55\uda56\uda57\uda58\uda59\uda5a\uda5b\uda5c\uda5d\uda5e\uda5f\uda60\uda61\uda62\uda63\uda64\uda65\uda66\uda67\uda68\uda69\uda6a\uda6b\uda6c\uda6d\uda6e\uda6f\uda70\uda71\uda72\uda73\uda74\uda75\uda76\uda77\uda78\uda79\uda7a\uda7b\uda7c\uda7d\uda7e\uda7f\uda80\uda81\uda82\uda83\uda84\uda85\uda86\uda87\uda88\uda89\uda8a\uda8b\uda8c\uda8d\uda8e\uda8f\uda90\uda91\uda92\uda93\uda94\uda95\uda96\uda97\uda98\uda99\uda9a\uda9b\uda9c\uda9d\uda9e\uda9f\udaa0\udaa1\udaa2\udaa3\udaa4\udaa5\udaa6\udaa7\udaa8\udaa9\udaaa\udaab\udaac\udaad\udaae\udaaf\udab0\udab1\udab2\udab3\udab4\udab5\udab6\udab7\udab8\udab9\udaba\udabb\udabc\udabd\udabe\udabf\udac0\udac1\udac2\udac3\udac4\udac5\udac6\udac7\udac8\udac9\udaca\udacb\udacc\udacd\udace\udacf\udad0\udad1\udad2\udad3\udad4\udad5\udad6\udad7\udad8\udad9\udada\udadb\udadc\udadd\udade\udadf\udae0\udae1\udae2\udae3\udae4\udae5\udae6\udae7\udae8\udae9\udaea\udaeb\udaec\udaed\udaee\udaef\udaf0\udaf1\udaf2\udaf3\udaf4\udaf5\udaf6\udaf7\udaf8\udaf9\udafa\udafb\udafc\udafd\udafe\udaff\udb00\udb01\udb02\udb03\udb04\udb05\udb06\udb07\udb08\udb09\udb0a\udb0b\udb0c\udb0d\udb0e\udb0f\udb10\udb11\udb12\udb13\udb14\udb15\udb16\udb17\udb18\udb19\udb1a\udb1b\udb1c\udb1d\udb1e\udb1f\udb20\udb21\udb22\udb23\udb24\udb25\udb26\udb27\udb28\udb29\udb2a\udb2b\udb2c\udb2d\udb2e\udb2f\udb30\udb31\udb32\udb33\udb34\udb35\udb36\udb37\udb38\udb39\udb3a\udb3b\udb3c\udb3d\udb3e\udb3f\udb40\udb41\udb42\udb43\udb44\udb45\udb46\udb47\udb48\udb49\udb4a\udb4b\udb4c\udb4d\udb4e\udb4f\udb50\udb51\udb52\udb53\udb54\udb55\udb56\udb57\udb58\udb59\udb5a\udb5b\udb5c\udb5d\udb5e\udb5f\udb60\udb61\udb62\udb63\udb64\udb65\udb66\udb67\udb68\udb69\udb6a\udb6b\udb6c\udb6d\udb6e\udb6f\udb70\udb71\udb72\udb73\udb74\udb75\udb76\udb77\udb78\udb79\udb7a\udb7b\udb7c\udb7d\udb7e\udb7f\udb80\udb81\udb82\udb83\udb84\udb85\udb86\udb87\udb88\udb89\udb8a\udb8b\udb8c\udb8d\udb8e\udb8f\udb90\udb91\udb92\udb93\udb94\udb95\udb96\udb97\udb98\udb99\udb9a\udb9b\udb9c\udb9d\udb9e\udb9f\udba0\udba1\udba2\udba3\udba4\udba5\udba6\udba7\udba8\udba9\udbaa\udbab\udbac\udbad\udbae\udbaf\udbb0\udbb1\udbb2\udbb3\udbb4\udbb5\udbb6\udbb7\udbb8\udbb9\udbba\udbbb\udbbc\udbbd\udbbe\udbbf\udbc0\udbc1\udbc2\udbc3\udbc4\udbc5\udbc6\udbc7\udbc8\udbc9\udbca\udbcb\udbcc\udbcd\udbce\udbcf\udbd0\udbd1\udbd2\udbd3\udbd4\udbd5\udbd6\udbd7\udbd8\udbd9\udbda\udbdb\udbdc\udbdd\udbde\udbdf\udbe0\udbe1\udbe2\udbe3\udbe4\udbe5\udbe6\udbe7\udbe8\udbe9\udbea\udbeb\udbec\udbed\udbee\udbef\udbf0\udbf1\udbf2\udbf3\udbf4\udbf5\udbf6\udbf7\udbf8\udbf9\udbfa\udbfb\udbfc\udbfd\udbfe\U0010fc00\udc01\udc02\udc03\udc04\udc05\udc06\udc07\udc08\udc09\udc0a\udc0b\udc0c\udc0d\udc0e\udc0f\udc10\udc11\udc12\udc13\udc14\udc15\udc16\udc17\udc18\udc19\udc1a\udc1b\udc1c\udc1d\udc1e\udc1f\udc20\udc21\udc22\udc23\udc24\udc25\udc26\udc27\udc28\udc29\udc2a\udc2b\udc2c\udc2d\udc2e\udc2f\udc30\udc31\udc32\udc33\udc34\udc35\udc36\udc37\udc38\udc39\udc3a\udc3b\udc3c\udc3d\udc3e\udc3f\udc40\udc41\udc42\udc43\udc44\udc45\udc46\udc47\udc48\udc49\udc4a\udc4b\udc4c\udc4d\udc4e\udc4f\udc50\udc51\udc52\udc53\udc54\udc55\udc56\udc57\udc58\udc59\udc5a\udc5b\udc5c\udc5d\udc5e\udc5f\udc60\udc61\udc62\udc63\udc64\udc65\udc66\udc67\udc68\udc69\udc6a\udc6b\udc6c\udc6d\udc6e\udc6f\udc70\udc71\udc72\udc73\udc74\udc75\udc76\udc77\udc78\udc79\udc7a\udc7b\udc7c\udc7d\udc7e\udc7f\udc80\udc81\udc82\udc83\udc84\udc85\udc86\udc87\udc88\udc89\udc8a\udc8b\udc8c\udc8d\udc8e\udc8f\udc90\udc91\udc92\udc93\udc94\udc95\udc96\udc97\udc98\udc99\udc9a\udc9b\udc9c\udc9d\udc9e\udc9f\udca0\udca1\udca2\udca3\udca4\udca5\udca6\udca7\udca8\udca9\udcaa\udcab\udcac\udcad\udcae\udcaf\udcb0\udcb1\udcb2\udcb3\udcb4\udcb5\udcb6\udcb7\udcb8\udcb9\udcba\udcbb\udcbc\udcbd\udcbe\udcbf\udcc0\udcc1\udcc2\udcc3\udcc4\udcc5\udcc6\udcc7\udcc8\udcc9\udcca\udccb\udccc\udccd\udcce\udccf\udcd0\udcd1\udcd2\udcd3\udcd4\udcd5\udcd6\udcd7\udcd8\udcd9\udcda\udcdb\udcdc\udcdd\udcde\udcdf\udce0\udce1\udce2\udce3\udce4\udce5\udce6\udce7\udce8\udce9\udcea\udceb\udcec\udced\udcee\udcef\udcf0\udcf1\udcf2\udcf3\udcf4\udcf5\udcf6\udcf7\udcf8\udcf9\udcfa\udcfb\udcfc\udcfd\udcfe\udcff\udd00\udd01\udd02\udd03\udd04\udd05\udd06\udd07\udd08\udd09\udd0a\udd0b\udd0c\udd0d\udd0e\udd0f\udd10\udd11\udd12\udd13\udd14\udd15\udd16\udd17\udd18\udd19\udd1a\udd1b\udd1c\udd1d\udd1e\udd1f\udd20\udd21\udd22\udd23\udd24\udd25\udd26\udd27\udd28\udd29\udd2a\udd2b\udd2c\udd2d\udd2e\udd2f\udd30\udd31\udd32\udd33\udd34\udd35\udd36\udd37\udd38\udd39\udd3a\udd3b\udd3c\udd3d\udd3e\udd3f\udd40\udd41\udd42\udd43\udd44\udd45\udd46\udd47\udd48\udd49\udd4a\udd4b\udd4c\udd4d\udd4e\udd4f\udd50\udd51\udd52\udd53\udd54\udd55\udd56\udd57\udd58\udd59\udd5a\udd5b\udd5c\udd5d\udd5e\udd5f\udd60\udd61\udd62\udd63\udd64\udd65\udd66\udd67\udd68\udd69\udd6a\udd6b\udd6c\udd6d\udd6e\udd6f\udd70\udd71\udd72\udd73\udd74\udd75\udd76\udd77\udd78\udd79\udd7a\udd7b\udd7c\udd7d\udd7e\udd7f\udd80\udd81\udd82\udd83\udd84\udd85\udd86\udd87\udd88\udd89\udd8a\udd8b\udd8c\udd8d\udd8e\udd8f\udd90\udd91\udd92\udd93\udd94\udd95\udd96\udd97\udd98\udd99\udd9a\udd9b\udd9c\udd9d\udd9e\udd9f\udda0\udda1\udda2\udda3\udda4\udda5\udda6\udda7\udda8\udda9\uddaa\uddab\uddac\uddad\uddae\uddaf\uddb0\uddb1\uddb2\uddb3\uddb4\uddb5\uddb6\uddb7\uddb8\uddb9\uddba\uddbb\uddbc\uddbd\uddbe\uddbf\uddc0\uddc1\uddc2\uddc3\uddc4\uddc5\uddc6\uddc7\uddc8\uddc9\uddca\uddcb\uddcc\uddcd\uddce\uddcf\uddd0\uddd1\uddd2\uddd3\uddd4\uddd5\uddd6\uddd7\uddd8\uddd9\uddda\udddb\udddc\udddd\uddde\udddf\udde0\udde1\udde2\udde3\udde4\udde5\udde6\udde7\udde8\udde9\uddea\uddeb\uddec\udded\uddee\uddef\uddf0\uddf1\uddf2\uddf3\uddf4\uddf5\uddf6\uddf7\uddf8\uddf9\uddfa\uddfb\uddfc\uddfd\uddfe\uddff\ude00\ude01\ude02\ude03\ude04\ude05\ude06\ude07\ude08\ude09\ude0a\ude0b\ude0c\ude0d\ude0e\ude0f\ude10\ude11\ude12\ude13\ude14\ude15\ude16\ude17\ude18\ude19\ude1a\ude1b\ude1c\ude1d\ude1e\ude1f\ude20\ude21\ude22\ude23\ude24\ude25\ude26\ude27\ude28\ude29\ude2a\ude2b\ude2c\ude2d\ude2e\ude2f\ude30\ude31\ude32\ude33\ude34\ude35\ude36\ude37\ude38\ude39\ude3a\ude3b\ude3c\ude3d\ude3e\ude3f\ude40\ude41\ude42\ude43\ude44\ude45\ude46\ude47\ude48\ude49\ude4a\ude4b\ude4c\ude4d\ude4e\ude4f\ude50\ude51\ude52\ude53\ude54\ude55\ude56\ude57\ude58\ude59\ude5a\ude5b\ude5c\ude5d\ude5e\ude5f\ude60\ude61\ude62\ude63\ude64\ude65\ude66\ude67\ude68\ude69\ude6a\ude6b\ude6c\ude6d\ude6e\ude6f\ude70\ude71\ude72\ude73\ude74\ude75\ude76\ude77\ude78\ude79\ude7a\ude7b\ude7c\ude7d\ude7e\ude7f\ude80\ude81\ude82\ude83\ude84\ude85\ude86\ude87\ude88\ude89\ude8a\ude8b\ude8c\ude8d\ude8e\ude8f\ude90\ude91\ude92\ude93\ude94\ude95\ude96\ude97\ude98\ude99\ude9a\ude9b\ude9c\ude9d\ude9e\ude9f\udea0\udea1\udea2\udea3\udea4\udea5\udea6\udea7\udea8\udea9\udeaa\udeab\udeac\udead\udeae\udeaf\udeb0\udeb1\udeb2\udeb3\udeb4\udeb5\udeb6\udeb7\udeb8\udeb9\udeba\udebb\udebc\udebd\udebe\udebf\udec0\udec1\udec2\udec3\udec4\udec5\udec6\udec7\udec8\udec9\udeca\udecb\udecc\udecd\udece\udecf\uded0\uded1\uded2\uded3\uded4\uded5\uded6\uded7\uded8\uded9\udeda\udedb\udedc\udedd\udede\udedf\udee0\udee1\udee2\udee3\udee4\udee5\udee6\udee7\udee8\udee9\udeea\udeeb\udeec\udeed\udeee\udeef\udef0\udef1\udef2\udef3\udef4\udef5\udef6\udef7\udef8\udef9\udefa\udefb\udefc\udefd\udefe\udeff\udf00\udf01\udf02\udf03\udf04\udf05\udf06\udf07\udf08\udf09\udf0a\udf0b\udf0c\udf0d\udf0e\udf0f\udf10\udf11\udf12\udf13\udf14\udf15\udf16\udf17\udf18\udf19\udf1a\udf1b\udf1c\udf1d\udf1e\udf1f\udf20\udf21\udf22\udf23\udf24\udf25\udf26\udf27\udf28\udf29\udf2a\udf2b\udf2c\udf2d\udf2e\udf2f\udf30\udf31\udf32\udf33\udf34\udf35\udf36\udf37\udf38\udf39\udf3a\udf3b\udf3c\udf3d\udf3e\udf3f\udf40\udf41\udf42\udf43\udf44\udf45\udf46\udf47\udf48\udf49\udf4a\udf4b\udf4c\udf4d\udf4e\udf4f\udf50\udf51\udf52\udf53\udf54\udf55\udf56\udf57\udf58\udf59\udf5a\udf5b\udf5c\udf5d\udf5e\udf5f\udf60\udf61\udf62\udf63\udf64\udf65\udf66\udf67\udf68\udf69\udf6a\udf6b\udf6c\udf6d\udf6e\udf6f\udf70\udf71\udf72\udf73\udf74\udf75\udf76\udf77\udf78\udf79\udf7a\udf7b\udf7c\udf7d\udf7e\udf7f\udf80\udf81\udf82\udf83\udf84\udf85\udf86\udf87\udf88\udf89\udf8a\udf8b\udf8c\udf8d\udf8e\udf8f\udf90\udf91\udf92\udf93\udf94\udf95\udf96\udf97\udf98\udf99\udf9a\udf9b\udf9c\udf9d\udf9e\udf9f\udfa0\udfa1\udfa2\udfa3\udfa4\udfa5\udfa6\udfa7\udfa8\udfa9\udfaa\udfab\udfac\udfad\udfae\udfaf\udfb0\udfb1\udfb2\udfb3\udfb4\udfb5\udfb6\udfb7\udfb8\udfb9\udfba\udfbb\udfbc\udfbd\udfbe\udfbf\udfc0\udfc1\udfc2\udfc3\udfc4\udfc5\udfc6\udfc7\udfc8\udfc9\udfca\udfcb\udfcc\udfcd\udfce\udfcf\udfd0\udfd1\udfd2\udfd3\udfd4\udfd5\udfd6\udfd7\udfd8\udfd9\udfda\udfdb\udfdc\udfdd\udfde\udfdf\udfe0\udfe1\udfe2\udfe3\udfe4\udfe5\udfe6\udfe7\udfe8\udfe9\udfea\udfeb\udfec\udfed\udfee\udfef\udff0\udff1\udff2\udff3\udff4\udff5\udff6\udff7\udff8\udff9\udffa\udffb\udffc\udffd\udffe\udfff'") +except UnicodeDecodeError: + Cs = '' # Jython can't handle isolated surrogates + +Ll = u'abcdefghijklmnopqrstuvwxyz\xaa\xb5\xba\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff\u0101\u0103\u0105\u0107\u0109\u010b\u010d\u010f\u0111\u0113\u0115\u0117\u0119\u011b\u011d\u011f\u0121\u0123\u0125\u0127\u0129\u012b\u012d\u012f\u0131\u0133\u0135\u0137\u0138\u013a\u013c\u013e\u0140\u0142\u0144\u0146\u0148\u0149\u014b\u014d\u014f\u0151\u0153\u0155\u0157\u0159\u015b\u015d\u015f\u0161\u0163\u0165\u0167\u0169\u016b\u016d\u016f\u0171\u0173\u0175\u0177\u017a\u017c\u017e\u017f\u0180\u0183\u0185\u0188\u018c\u018d\u0192\u0195\u0199\u019a\u019b\u019e\u01a1\u01a3\u01a5\u01a8\u01aa\u01ab\u01ad\u01b0\u01b4\u01b6\u01b9\u01ba\u01bd\u01be\u01bf\u01c6\u01c9\u01cc\u01ce\u01d0\u01d2\u01d4\u01d6\u01d8\u01da\u01dc\u01dd\u01df\u01e1\u01e3\u01e5\u01e7\u01e9\u01eb\u01ed\u01ef\u01f0\u01f3\u01f5\u01f9\u01fb\u01fd\u01ff\u0201\u0203\u0205\u0207\u0209\u020b\u020d\u020f\u0211\u0213\u0215\u0217\u0219\u021b\u021d\u021f\u0221\u0223\u0225\u0227\u0229\u022b\u022d\u022f\u0231\u0233\u0234\u0235\u0236\u0237\u0238\u0239\u023c\u023f\u0240\u0250\u0251\u0252\u0253\u0254\u0255\u0256\u0257\u0258\u0259\u025a\u025b\u025c\u025d\u025e\u025f\u0260\u0261\u0262\u0263\u0264\u0265\u0266\u0267\u0268\u0269\u026a\u026b\u026c\u026d\u026e\u026f\u0270\u0271\u0272\u0273\u0274\u0275\u0276\u0277\u0278\u0279\u027a\u027b\u027c\u027d\u027e\u027f\u0280\u0281\u0282\u0283\u0284\u0285\u0286\u0287\u0288\u0289\u028a\u028b\u028c\u028d\u028e\u028f\u0290\u0291\u0292\u0293\u0294\u0295\u0296\u0297\u0298\u0299\u029a\u029b\u029c\u029d\u029e\u029f\u02a0\u02a1\u02a2\u02a3\u02a4\u02a5\u02a6\u02a7\u02a8\u02a9\u02aa\u02ab\u02ac\u02ad\u02ae\u02af\u0390\u03ac\u03ad\u03ae\u03af\u03b0\u03b1\u03b2\u03b3\u03b4\u03b5\u03b6\u03b7\u03b8\u03b9\u03ba\u03bb\u03bc\u03bd\u03be\u03bf\u03c0\u03c1\u03c2\u03c3\u03c4\u03c5\u03c6\u03c7\u03c8\u03c9\u03ca\u03cb\u03cc\u03cd\u03ce\u03d0\u03d1\u03d5\u03d6\u03d7\u03d9\u03db\u03dd\u03df\u03e1\u03e3\u03e5\u03e7\u03e9\u03eb\u03ed\u03ef\u03f0\u03f1\u03f2\u03f3\u03f5\u03f8\u03fb\u03fc\u0430\u0431\u0432\u0433\u0434\u0435\u0436\u0437\u0438\u0439\u043a\u043b\u043c\u043d\u043e\u043f\u0440\u0441\u0442\u0443\u0444\u0445\u0446\u0447\u0448\u0449\u044a\u044b\u044c\u044d\u044e\u044f\u0450\u0451\u0452\u0453\u0454\u0455\u0456\u0457\u0458\u0459\u045a\u045b\u045c\u045d\u045e\u045f\u0461\u0463\u0465\u0467\u0469\u046b\u046d\u046f\u0471\u0473\u0475\u0477\u0479\u047b\u047d\u047f\u0481\u048b\u048d\u048f\u0491\u0493\u0495\u0497\u0499\u049b\u049d\u049f\u04a1\u04a3\u04a5\u04a7\u04a9\u04ab\u04ad\u04af\u04b1\u04b3\u04b5\u04b7\u04b9\u04bb\u04bd\u04bf\u04c2\u04c4\u04c6\u04c8\u04ca\u04cc\u04ce\u04d1\u04d3\u04d5\u04d7\u04d9\u04db\u04dd\u04df\u04e1\u04e3\u04e5\u04e7\u04e9\u04eb\u04ed\u04ef\u04f1\u04f3\u04f5\u04f7\u04f9\u0501\u0503\u0505\u0507\u0509\u050b\u050d\u050f\u0561\u0562\u0563\u0564\u0565\u0566\u0567\u0568\u0569\u056a\u056b\u056c\u056d\u056e\u056f\u0570\u0571\u0572\u0573\u0574\u0575\u0576\u0577\u0578\u0579\u057a\u057b\u057c\u057d\u057e\u057f\u0580\u0581\u0582\u0583\u0584\u0585\u0586\u0587\u1d00\u1d01\u1d02\u1d03\u1d04\u1d05\u1d06\u1d07\u1d08\u1d09\u1d0a\u1d0b\u1d0c\u1d0d\u1d0e\u1d0f\u1d10\u1d11\u1d12\u1d13\u1d14\u1d15\u1d16\u1d17\u1d18\u1d19\u1d1a\u1d1b\u1d1c\u1d1d\u1d1e\u1d1f\u1d20\u1d21\u1d22\u1d23\u1d24\u1d25\u1d26\u1d27\u1d28\u1d29\u1d2a\u1d2b\u1d62\u1d63\u1d64\u1d65\u1d66\u1d67\u1d68\u1d69\u1d6a\u1d6b\u1d6c\u1d6d\u1d6e\u1d6f\u1d70\u1d71\u1d72\u1d73\u1d74\u1d75\u1d76\u1d77\u1d79\u1d7a\u1d7b\u1d7c\u1d7d\u1d7e\u1d7f\u1d80\u1d81\u1d82\u1d83\u1d84\u1d85\u1d86\u1d87\u1d88\u1d89\u1d8a\u1d8b\u1d8c\u1d8d\u1d8e\u1d8f\u1d90\u1d91\u1d92\u1d93\u1d94\u1d95\u1d96\u1d97\u1d98\u1d99\u1d9a\u1e01\u1e03\u1e05\u1e07\u1e09\u1e0b\u1e0d\u1e0f\u1e11\u1e13\u1e15\u1e17\u1e19\u1e1b\u1e1d\u1e1f\u1e21\u1e23\u1e25\u1e27\u1e29\u1e2b\u1e2d\u1e2f\u1e31\u1e33\u1e35\u1e37\u1e39\u1e3b\u1e3d\u1e3f\u1e41\u1e43\u1e45\u1e47\u1e49\u1e4b\u1e4d\u1e4f\u1e51\u1e53\u1e55\u1e57\u1e59\u1e5b\u1e5d\u1e5f\u1e61\u1e63\u1e65\u1e67\u1e69\u1e6b\u1e6d\u1e6f\u1e71\u1e73\u1e75\u1e77\u1e79\u1e7b\u1e7d\u1e7f\u1e81\u1e83\u1e85\u1e87\u1e89\u1e8b\u1e8d\u1e8f\u1e91\u1e93\u1e95\u1e96\u1e97\u1e98\u1e99\u1e9a\u1e9b\u1ea1\u1ea3\u1ea5\u1ea7\u1ea9\u1eab\u1ead\u1eaf\u1eb1\u1eb3\u1eb5\u1eb7\u1eb9\u1ebb\u1ebd\u1ebf\u1ec1\u1ec3\u1ec5\u1ec7\u1ec9\u1ecb\u1ecd\u1ecf\u1ed1\u1ed3\u1ed5\u1ed7\u1ed9\u1edb\u1edd\u1edf\u1ee1\u1ee3\u1ee5\u1ee7\u1ee9\u1eeb\u1eed\u1eef\u1ef1\u1ef3\u1ef5\u1ef7\u1ef9\u1f00\u1f01\u1f02\u1f03\u1f04\u1f05\u1f06\u1f07\u1f10\u1f11\u1f12\u1f13\u1f14\u1f15\u1f20\u1f21\u1f22\u1f23\u1f24\u1f25\u1f26\u1f27\u1f30\u1f31\u1f32\u1f33\u1f34\u1f35\u1f36\u1f37\u1f40\u1f41\u1f42\u1f43\u1f44\u1f45\u1f50\u1f51\u1f52\u1f53\u1f54\u1f55\u1f56\u1f57\u1f60\u1f61\u1f62\u1f63\u1f64\u1f65\u1f66\u1f67\u1f70\u1f71\u1f72\u1f73\u1f74\u1f75\u1f76\u1f77\u1f78\u1f79\u1f7a\u1f7b\u1f7c\u1f7d\u1f80\u1f81\u1f82\u1f83\u1f84\u1f85\u1f86\u1f87\u1f90\u1f91\u1f92\u1f93\u1f94\u1f95\u1f96\u1f97\u1fa0\u1fa1\u1fa2\u1fa3\u1fa4\u1fa5\u1fa6\u1fa7\u1fb0\u1fb1\u1fb2\u1fb3\u1fb4\u1fb6\u1fb7\u1fbe\u1fc2\u1fc3\u1fc4\u1fc6\u1fc7\u1fd0\u1fd1\u1fd2\u1fd3\u1fd6\u1fd7\u1fe0\u1fe1\u1fe2\u1fe3\u1fe4\u1fe5\u1fe6\u1fe7\u1ff2\u1ff3\u1ff4\u1ff6\u1ff7\u2071\u207f\u210a\u210e\u210f\u2113\u212f\u2134\u2139\u213c\u213d\u2146\u2147\u2148\u2149\u2c30\u2c31\u2c32\u2c33\u2c34\u2c35\u2c36\u2c37\u2c38\u2c39\u2c3a\u2c3b\u2c3c\u2c3d\u2c3e\u2c3f\u2c40\u2c41\u2c42\u2c43\u2c44\u2c45\u2c46\u2c47\u2c48\u2c49\u2c4a\u2c4b\u2c4c\u2c4d\u2c4e\u2c4f\u2c50\u2c51\u2c52\u2c53\u2c54\u2c55\u2c56\u2c57\u2c58\u2c59\u2c5a\u2c5b\u2c5c\u2c5d\u2c5e\u2c81\u2c83\u2c85\u2c87\u2c89\u2c8b\u2c8d\u2c8f\u2c91\u2c93\u2c95\u2c97\u2c99\u2c9b\u2c9d\u2c9f\u2ca1\u2ca3\u2ca5\u2ca7\u2ca9\u2cab\u2cad\u2caf\u2cb1\u2cb3\u2cb5\u2cb7\u2cb9\u2cbb\u2cbd\u2cbf\u2cc1\u2cc3\u2cc5\u2cc7\u2cc9\u2ccb\u2ccd\u2ccf\u2cd1\u2cd3\u2cd5\u2cd7\u2cd9\u2cdb\u2cdd\u2cdf\u2ce1\u2ce3\u2ce4\u2d00\u2d01\u2d02\u2d03\u2d04\u2d05\u2d06\u2d07\u2d08\u2d09\u2d0a\u2d0b\u2d0c\u2d0d\u2d0e\u2d0f\u2d10\u2d11\u2d12\u2d13\u2d14\u2d15\u2d16\u2d17\u2d18\u2d19\u2d1a\u2d1b\u2d1c\u2d1d\u2d1e\u2d1f\u2d20\u2d21\u2d22\u2d23\u2d24\u2d25\ufb00\ufb01\ufb02\ufb03\ufb04\ufb05\ufb06\ufb13\ufb14\ufb15\ufb16\ufb17\uff41\uff42\uff43\uff44\uff45\uff46\uff47\uff48\uff49\uff4a\uff4b\uff4c\uff4d\uff4e\uff4f\uff50\uff51\uff52\uff53\uff54\uff55\uff56\uff57\uff58\uff59\uff5a' + +Lm = u'\u02b0\u02b1\u02b2\u02b3\u02b4\u02b5\u02b6\u02b7\u02b8\u02b9\u02ba\u02bb\u02bc\u02bd\u02be\u02bf\u02c0\u02c1\u02c6\u02c7\u02c8\u02c9\u02ca\u02cb\u02cc\u02cd\u02ce\u02cf\u02d0\u02d1\u02e0\u02e1\u02e2\u02e3\u02e4\u02ee\u037a\u0559\u0640\u06e5\u06e6\u0e46\u0ec6\u10fc\u17d7\u1843\u1d2c\u1d2d\u1d2e\u1d2f\u1d30\u1d31\u1d32\u1d33\u1d34\u1d35\u1d36\u1d37\u1d38\u1d39\u1d3a\u1d3b\u1d3c\u1d3d\u1d3e\u1d3f\u1d40\u1d41\u1d42\u1d43\u1d44\u1d45\u1d46\u1d47\u1d48\u1d49\u1d4a\u1d4b\u1d4c\u1d4d\u1d4e\u1d4f\u1d50\u1d51\u1d52\u1d53\u1d54\u1d55\u1d56\u1d57\u1d58\u1d59\u1d5a\u1d5b\u1d5c\u1d5d\u1d5e\u1d5f\u1d60\u1d61\u1d78\u1d9b\u1d9c\u1d9d\u1d9e\u1d9f\u1da0\u1da1\u1da2\u1da3\u1da4\u1da5\u1da6\u1da7\u1da8\u1da9\u1daa\u1dab\u1dac\u1dad\u1dae\u1daf\u1db0\u1db1\u1db2\u1db3\u1db4\u1db5\u1db6\u1db7\u1db8\u1db9\u1dba\u1dbb\u1dbc\u1dbd\u1dbe\u1dbf\u2090\u2091\u2092\u2093\u2094\u2d6f\u3005\u3031\u3032\u3033\u3034\u3035\u303b\u309d\u309e\u30fc\u30fd\u30fe\ua015\uff70\uff9e\uff9f' + +Lo = u'\u01bb\u01c0\u01c1\u01c2\u01c3\u05d0\u05d1\u05d2\u05d3\u05d4\u05d5\u05d6\u05d7\u05d8\u05d9\u05da\u05db\u05dc\u05dd\u05de\u05df\u05e0\u05e1\u05e2\u05e3\u05e4\u05e5\u05e6\u05e7\u05e8\u05e9\u05ea\u05f0\u05f1\u05f2\u0621\u0622\u0623\u0624\u0625\u0626\u0627\u0628\u0629\u062a\u062b\u062c\u062d\u062e\u062f\u0630\u0631\u0632\u0633\u0634\u0635\u0636\u0637\u0638\u0639\u063a\u0641\u0642\u0643\u0644\u0645\u0646\u0647\u0648\u0649\u064a\u066e\u066f\u0671\u0672\u0673\u0674\u0675\u0676\u0677\u0678\u0679\u067a\u067b\u067c\u067d\u067e\u067f\u0680\u0681\u0682\u0683\u0684\u0685\u0686\u0687\u0688\u0689\u068a\u068b\u068c\u068d\u068e\u068f\u0690\u0691\u0692\u0693\u0694\u0695\u0696\u0697\u0698\u0699\u069a\u069b\u069c\u069d\u069e\u069f\u06a0\u06a1\u06a2\u06a3\u06a4\u06a5\u06a6\u06a7\u06a8\u06a9\u06aa\u06ab\u06ac\u06ad\u06ae\u06af\u06b0\u06b1\u06b2\u06b3\u06b4\u06b5\u06b6\u06b7\u06b8\u06b9\u06ba\u06bb\u06bc\u06bd\u06be\u06bf\u06c0\u06c1\u06c2\u06c3\u06c4\u06c5\u06c6\u06c7\u06c8\u06c9\u06ca\u06cb\u06cc\u06cd\u06ce\u06cf\u06d0\u06d1\u06d2\u06d3\u06d5\u06ee\u06ef\u06fa\u06fb\u06fc\u06ff\u0710\u0712\u0713\u0714\u0715\u0716\u0717\u0718\u0719\u071a\u071b\u071c\u071d\u071e\u071f\u0720\u0721\u0722\u0723\u0724\u0725\u0726\u0727\u0728\u0729\u072a\u072b\u072c\u072d\u072e\u072f\u074d\u074e\u074f\u0750\u0751\u0752\u0753\u0754\u0755\u0756\u0757\u0758\u0759\u075a\u075b\u075c\u075d\u075e\u075f\u0760\u0761\u0762\u0763\u0764\u0765\u0766\u0767\u0768\u0769\u076a\u076b\u076c\u076d\u0780\u0781\u0782\u0783\u0784\u0785\u0786\u0787\u0788\u0789\u078a\u078b\u078c\u078d\u078e\u078f\u0790\u0791\u0792\u0793\u0794\u0795\u0796\u0797\u0798\u0799\u079a\u079b\u079c\u079d\u079e\u079f\u07a0\u07a1\u07a2\u07a3\u07a4\u07a5\u07b1\u0904\u0905\u0906\u0907\u0908\u0909\u090a\u090b\u090c\u090d\u090e\u090f\u0910\u0911\u0912\u0913\u0914\u0915\u0916\u0917\u0918\u0919\u091a\u091b\u091c\u091d\u091e\u091f\u0920\u0921\u0922\u0923\u0924\u0925\u0926\u0927\u0928\u0929\u092a\u092b\u092c\u092d\u092e\u092f\u0930\u0931\u0932\u0933\u0934\u0935\u0936\u0937\u0938\u0939\u093d\u0950\u0958\u0959\u095a\u095b\u095c\u095d\u095e\u095f\u0960\u0961\u097d\u0985\u0986\u0987\u0988\u0989\u098a\u098b\u098c\u098f\u0990\u0993\u0994\u0995\u0996\u0997\u0998\u0999\u099a\u099b\u099c\u099d\u099e\u099f\u09a0\u09a1\u09a2\u09a3\u09a4\u09a5\u09a6\u09a7\u09a8\u09aa\u09ab\u09ac\u09ad\u09ae\u09af\u09b0\u09b2\u09b6\u09b7\u09b8\u09b9\u09bd\u09ce\u09dc\u09dd\u09df\u09e0\u09e1\u09f0\u09f1\u0a05\u0a06\u0a07\u0a08\u0a09\u0a0a\u0a0f\u0a10\u0a13\u0a14\u0a15\u0a16\u0a17\u0a18\u0a19\u0a1a\u0a1b\u0a1c\u0a1d\u0a1e\u0a1f\u0a20\u0a21\u0a22\u0a23\u0a24\u0a25\u0a26\u0a27\u0a28\u0a2a\u0a2b\u0a2c\u0a2d\u0a2e\u0a2f\u0a30\u0a32\u0a33\u0a35\u0a36\u0a38\u0a39\u0a59\u0a5a\u0a5b\u0a5c\u0a5e\u0a72\u0a73\u0a74\u0a85\u0a86\u0a87\u0a88\u0a89\u0a8a\u0a8b\u0a8c\u0a8d\u0a8f\u0a90\u0a91\u0a93\u0a94\u0a95\u0a96\u0a97\u0a98\u0a99\u0a9a\u0a9b\u0a9c\u0a9d\u0a9e\u0a9f\u0aa0\u0aa1\u0aa2\u0aa3\u0aa4\u0aa5\u0aa6\u0aa7\u0aa8\u0aaa\u0aab\u0aac\u0aad\u0aae\u0aaf\u0ab0\u0ab2\u0ab3\u0ab5\u0ab6\u0ab7\u0ab8\u0ab9\u0abd\u0ad0\u0ae0\u0ae1\u0b05\u0b06\u0b07\u0b08\u0b09\u0b0a\u0b0b\u0b0c\u0b0f\u0b10\u0b13\u0b14\u0b15\u0b16\u0b17\u0b18\u0b19\u0b1a\u0b1b\u0b1c\u0b1d\u0b1e\u0b1f\u0b20\u0b21\u0b22\u0b23\u0b24\u0b25\u0b26\u0b27\u0b28\u0b2a\u0b2b\u0b2c\u0b2d\u0b2e\u0b2f\u0b30\u0b32\u0b33\u0b35\u0b36\u0b37\u0b38\u0b39\u0b3d\u0b5c\u0b5d\u0b5f\u0b60\u0b61\u0b71\u0b83\u0b85\u0b86\u0b87\u0b88\u0b89\u0b8a\u0b8e\u0b8f\u0b90\u0b92\u0b93\u0b94\u0b95\u0b99\u0b9a\u0b9c\u0b9e\u0b9f\u0ba3\u0ba4\u0ba8\u0ba9\u0baa\u0bae\u0baf\u0bb0\u0bb1\u0bb2\u0bb3\u0bb4\u0bb5\u0bb6\u0bb7\u0bb8\u0bb9\u0c05\u0c06\u0c07\u0c08\u0c09\u0c0a\u0c0b\u0c0c\u0c0e\u0c0f\u0c10\u0c12\u0c13\u0c14\u0c15\u0c16\u0c17\u0c18\u0c19\u0c1a\u0c1b\u0c1c\u0c1d\u0c1e\u0c1f\u0c20\u0c21\u0c22\u0c23\u0c24\u0c25\u0c26\u0c27\u0c28\u0c2a\u0c2b\u0c2c\u0c2d\u0c2e\u0c2f\u0c30\u0c31\u0c32\u0c33\u0c35\u0c36\u0c37\u0c38\u0c39\u0c60\u0c61\u0c85\u0c86\u0c87\u0c88\u0c89\u0c8a\u0c8b\u0c8c\u0c8e\u0c8f\u0c90\u0c92\u0c93\u0c94\u0c95\u0c96\u0c97\u0c98\u0c99\u0c9a\u0c9b\u0c9c\u0c9d\u0c9e\u0c9f\u0ca0\u0ca1\u0ca2\u0ca3\u0ca4\u0ca5\u0ca6\u0ca7\u0ca8\u0caa\u0cab\u0cac\u0cad\u0cae\u0caf\u0cb0\u0cb1\u0cb2\u0cb3\u0cb5\u0cb6\u0cb7\u0cb8\u0cb9\u0cbd\u0cde\u0ce0\u0ce1\u0d05\u0d06\u0d07\u0d08\u0d09\u0d0a\u0d0b\u0d0c\u0d0e\u0d0f\u0d10\u0d12\u0d13\u0d14\u0d15\u0d16\u0d17\u0d18\u0d19\u0d1a\u0d1b\u0d1c\u0d1d\u0d1e\u0d1f\u0d20\u0d21\u0d22\u0d23\u0d24\u0d25\u0d26\u0d27\u0d28\u0d2a\u0d2b\u0d2c\u0d2d\u0d2e\u0d2f\u0d30\u0d31\u0d32\u0d33\u0d34\u0d35\u0d36\u0d37\u0d38\u0d39\u0d60\u0d61\u0d85\u0d86\u0d87\u0d88\u0d89\u0d8a\u0d8b\u0d8c\u0d8d\u0d8e\u0d8f\u0d90\u0d91\u0d92\u0d93\u0d94\u0d95\u0d96\u0d9a\u0d9b\u0d9c\u0d9d\u0d9e\u0d9f\u0da0\u0da1\u0da2\u0da3\u0da4\u0da5\u0da6\u0da7\u0da8\u0da9\u0daa\u0dab\u0dac\u0dad\u0dae\u0daf\u0db0\u0db1\u0db3\u0db4\u0db5\u0db6\u0db7\u0db8\u0db9\u0dba\u0dbb\u0dbd\u0dc0\u0dc1\u0dc2\u0dc3\u0dc4\u0dc5\u0dc6\u0e01\u0e02\u0e03\u0e04\u0e05\u0e06\u0e07\u0e08\u0e09\u0e0a\u0e0b\u0e0c\u0e0d\u0e0e\u0e0f\u0e10\u0e11\u0e12\u0e13\u0e14\u0e15\u0e16\u0e17\u0e18\u0e19\u0e1a\u0e1b\u0e1c\u0e1d\u0e1e\u0e1f\u0e20\u0e21\u0e22\u0e23\u0e24\u0e25\u0e26\u0e27\u0e28\u0e29\u0e2a\u0e2b\u0e2c\u0e2d\u0e2e\u0e2f\u0e30\u0e32\u0e33\u0e40\u0e41\u0e42\u0e43\u0e44\u0e45\u0e81\u0e82\u0e84\u0e87\u0e88\u0e8a\u0e8d\u0e94\u0e95\u0e96\u0e97\u0e99\u0e9a\u0e9b\u0e9c\u0e9d\u0e9e\u0e9f\u0ea1\u0ea2\u0ea3\u0ea5\u0ea7\u0eaa\u0eab\u0ead\u0eae\u0eaf\u0eb0\u0eb2\u0eb3\u0ebd\u0ec0\u0ec1\u0ec2\u0ec3\u0ec4\u0edc\u0edd\u0f00\u0f40\u0f41\u0f42\u0f43\u0f44\u0f45\u0f46\u0f47\u0f49\u0f4a\u0f4b\u0f4c\u0f4d\u0f4e\u0f4f\u0f50\u0f51\u0f52\u0f53\u0f54\u0f55\u0f56\u0f57\u0f58\u0f59\u0f5a\u0f5b\u0f5c\u0f5d\u0f5e\u0f5f\u0f60\u0f61\u0f62\u0f63\u0f64\u0f65\u0f66\u0f67\u0f68\u0f69\u0f6a\u0f88\u0f89\u0f8a\u0f8b\u1000\u1001\u1002\u1003\u1004\u1005\u1006\u1007\u1008\u1009\u100a\u100b\u100c\u100d\u100e\u100f\u1010\u1011\u1012\u1013\u1014\u1015\u1016\u1017\u1018\u1019\u101a\u101b\u101c\u101d\u101e\u101f\u1020\u1021\u1023\u1024\u1025\u1026\u1027\u1029\u102a\u1050\u1051\u1052\u1053\u1054\u1055\u10d0\u10d1\u10d2\u10d3\u10d4\u10d5\u10d6\u10d7\u10d8\u10d9\u10da\u10db\u10dc\u10dd\u10de\u10df\u10e0\u10e1\u10e2\u10e3\u10e4\u10e5\u10e6\u10e7\u10e8\u10e9\u10ea\u10eb\u10ec\u10ed\u10ee\u10ef\u10f0\u10f1\u10f2\u10f3\u10f4\u10f5\u10f6\u10f7\u10f8\u10f9\u10fa\u1100\u1101\u1102\u1103\u1104\u1105\u1106\u1107\u1108\u1109\u110a\u110b\u110c\u110d\u110e\u110f\u1110\u1111\u1112\u1113\u1114\u1115\u1116\u1117\u1118\u1119\u111a\u111b\u111c\u111d\u111e\u111f\u1120\u1121\u1122\u1123\u1124\u1125\u1126\u1127\u1128\u1129\u112a\u112b\u112c\u112d\u112e\u112f\u1130\u1131\u1132\u1133\u1134\u1135\u1136\u1137\u1138\u1139\u113a\u113b\u113c\u113d\u113e\u113f\u1140\u1141\u1142\u1143\u1144\u1145\u1146\u1147\u1148\u1149\u114a\u114b\u114c\u114d\u114e\u114f\u1150\u1151\u1152\u1153\u1154\u1155\u1156\u1157\u1158\u1159\u115f\u1160\u1161\u1162\u1163\u1164\u1165\u1166\u1167\u1168\u1169\u116a\u116b\u116c\u116d\u116e\u116f\u1170\u1171\u1172\u1173\u1174\u1175\u1176\u1177\u1178\u1179\u117a\u117b\u117c\u117d\u117e\u117f\u1180\u1181\u1182\u1183\u1184\u1185\u1186\u1187\u1188\u1189\u118a\u118b\u118c\u118d\u118e\u118f\u1190\u1191\u1192\u1193\u1194\u1195\u1196\u1197\u1198\u1199\u119a\u119b\u119c\u119d\u119e\u119f\u11a0\u11a1\u11a2\u11a8\u11a9\u11aa\u11ab\u11ac\u11ad\u11ae\u11af\u11b0\u11b1\u11b2\u11b3\u11b4\u11b5\u11b6\u11b7\u11b8\u11b9\u11ba\u11bb\u11bc\u11bd\u11be\u11bf\u11c0\u11c1\u11c2\u11c3\u11c4\u11c5\u11c6\u11c7\u11c8\u11c9\u11ca\u11cb\u11cc\u11cd\u11ce\u11cf\u11d0\u11d1\u11d2\u11d3\u11d4\u11d5\u11d6\u11d7\u11d8\u11d9\u11da\u11db\u11dc\u11dd\u11de\u11df\u11e0\u11e1\u11e2\u11e3\u11e4\u11e5\u11e6\u11e7\u11e8\u11e9\u11ea\u11eb\u11ec\u11ed\u11ee\u11ef\u11f0\u11f1\u11f2\u11f3\u11f4\u11f5\u11f6\u11f7\u11f8\u11f9\u1200\u1201\u1202\u1203\u1204\u1205\u1206\u1207\u1208\u1209\u120a\u120b\u120c\u120d\u120e\u120f\u1210\u1211\u1212\u1213\u1214\u1215\u1216\u1217\u1218\u1219\u121a\u121b\u121c\u121d\u121e\u121f\u1220\u1221\u1222\u1223\u1224\u1225\u1226\u1227\u1228\u1229\u122a\u122b\u122c\u122d\u122e\u122f\u1230\u1231\u1232\u1233\u1234\u1235\u1236\u1237\u1238\u1239\u123a\u123b\u123c\u123d\u123e\u123f\u1240\u1241\u1242\u1243\u1244\u1245\u1246\u1247\u1248\u124a\u124b\u124c\u124d\u1250\u1251\u1252\u1253\u1254\u1255\u1256\u1258\u125a\u125b\u125c\u125d\u1260\u1261\u1262\u1263\u1264\u1265\u1266\u1267\u1268\u1269\u126a\u126b\u126c\u126d\u126e\u126f\u1270\u1271\u1272\u1273\u1274\u1275\u1276\u1277\u1278\u1279\u127a\u127b\u127c\u127d\u127e\u127f\u1280\u1281\u1282\u1283\u1284\u1285\u1286\u1287\u1288\u128a\u128b\u128c\u128d\u1290\u1291\u1292\u1293\u1294\u1295\u1296\u1297\u1298\u1299\u129a\u129b\u129c\u129d\u129e\u129f\u12a0\u12a1\u12a2\u12a3\u12a4\u12a5\u12a6\u12a7\u12a8\u12a9\u12aa\u12ab\u12ac\u12ad\u12ae\u12af\u12b0\u12b2\u12b3\u12b4\u12b5\u12b8\u12b9\u12ba\u12bb\u12bc\u12bd\u12be\u12c0\u12c2\u12c3\u12c4\u12c5\u12c8\u12c9\u12ca\u12cb\u12cc\u12cd\u12ce\u12cf\u12d0\u12d1\u12d2\u12d3\u12d4\u12d5\u12d6\u12d8\u12d9\u12da\u12db\u12dc\u12dd\u12de\u12df\u12e0\u12e1\u12e2\u12e3\u12e4\u12e5\u12e6\u12e7\u12e8\u12e9\u12ea\u12eb\u12ec\u12ed\u12ee\u12ef\u12f0\u12f1\u12f2\u12f3\u12f4\u12f5\u12f6\u12f7\u12f8\u12f9\u12fa\u12fb\u12fc\u12fd\u12fe\u12ff\u1300\u1301\u1302\u1303\u1304\u1305\u1306\u1307\u1308\u1309\u130a\u130b\u130c\u130d\u130e\u130f\u1310\u1312\u1313\u1314\u1315\u1318\u1319\u131a\u131b\u131c\u131d\u131e\u131f\u1320\u1321\u1322\u1323\u1324\u1325\u1326\u1327\u1328\u1329\u132a\u132b\u132c\u132d\u132e\u132f\u1330\u1331\u1332\u1333\u1334\u1335\u1336\u1337\u1338\u1339\u133a\u133b\u133c\u133d\u133e\u133f\u1340\u1341\u1342\u1343\u1344\u1345\u1346\u1347\u1348\u1349\u134a\u134b\u134c\u134d\u134e\u134f\u1350\u1351\u1352\u1353\u1354\u1355\u1356\u1357\u1358\u1359\u135a\u1380\u1381\u1382\u1383\u1384\u1385\u1386\u1387\u1388\u1389\u138a\u138b\u138c\u138d\u138e\u138f\u13a0\u13a1\u13a2\u13a3\u13a4\u13a5\u13a6\u13a7\u13a8\u13a9\u13aa\u13ab\u13ac\u13ad\u13ae\u13af\u13b0\u13b1\u13b2\u13b3\u13b4\u13b5\u13b6\u13b7\u13b8\u13b9\u13ba\u13bb\u13bc\u13bd\u13be\u13bf\u13c0\u13c1\u13c2\u13c3\u13c4\u13c5\u13c6\u13c7\u13c8\u13c9\u13ca\u13cb\u13cc\u13cd\u13ce\u13cf\u13d0\u13d1\u13d2\u13d3\u13d4\u13d5\u13d6\u13d7\u13d8\u13d9\u13da\u13db\u13dc\u13dd\u13de\u13df\u13e0\u13e1\u13e2\u13e3\u13e4\u13e5\u13e6\u13e7\u13e8\u13e9\u13ea\u13eb\u13ec\u13ed\u13ee\u13ef\u13f0\u13f1\u13f2\u13f3\u13f4\u1401\u1402\u1403\u1404\u1405\u1406\u1407\u1408\u1409\u140a\u140b\u140c\u140d\u140e\u140f\u1410\u1411\u1412\u1413\u1414\u1415\u1416\u1417\u1418\u1419\u141a\u141b\u141c\u141d\u141e\u141f\u1420\u1421\u1422\u1423\u1424\u1425\u1426\u1427\u1428\u1429\u142a\u142b\u142c\u142d\u142e\u142f\u1430\u1431\u1432\u1433\u1434\u1435\u1436\u1437\u1438\u1439\u143a\u143b\u143c\u143d\u143e\u143f\u1440\u1441\u1442\u1443\u1444\u1445\u1446\u1447\u1448\u1449\u144a\u144b\u144c\u144d\u144e\u144f\u1450\u1451\u1452\u1453\u1454\u1455\u1456\u1457\u1458\u1459\u145a\u145b\u145c\u145d\u145e\u145f\u1460\u1461\u1462\u1463\u1464\u1465\u1466\u1467\u1468\u1469\u146a\u146b\u146c\u146d\u146e\u146f\u1470\u1471\u1472\u1473\u1474\u1475\u1476\u1477\u1478\u1479\u147a\u147b\u147c\u147d\u147e\u147f\u1480\u1481\u1482\u1483\u1484\u1485\u1486\u1487\u1488\u1489\u148a\u148b\u148c\u148d\u148e\u148f\u1490\u1491\u1492\u1493\u1494\u1495\u1496\u1497\u1498\u1499\u149a\u149b\u149c\u149d\u149e\u149f\u14a0\u14a1\u14a2\u14a3\u14a4\u14a5\u14a6\u14a7\u14a8\u14a9\u14aa\u14ab\u14ac\u14ad\u14ae\u14af\u14b0\u14b1\u14b2\u14b3\u14b4\u14b5\u14b6\u14b7\u14b8\u14b9\u14ba\u14bb\u14bc\u14bd\u14be\u14bf\u14c0\u14c1\u14c2\u14c3\u14c4\u14c5\u14c6\u14c7\u14c8\u14c9\u14ca\u14cb\u14cc\u14cd\u14ce\u14cf\u14d0\u14d1\u14d2\u14d3\u14d4\u14d5\u14d6\u14d7\u14d8\u14d9\u14da\u14db\u14dc\u14dd\u14de\u14df\u14e0\u14e1\u14e2\u14e3\u14e4\u14e5\u14e6\u14e7\u14e8\u14e9\u14ea\u14eb\u14ec\u14ed\u14ee\u14ef\u14f0\u14f1\u14f2\u14f3\u14f4\u14f5\u14f6\u14f7\u14f8\u14f9\u14fa\u14fb\u14fc\u14fd\u14fe\u14ff\u1500\u1501\u1502\u1503\u1504\u1505\u1506\u1507\u1508\u1509\u150a\u150b\u150c\u150d\u150e\u150f\u1510\u1511\u1512\u1513\u1514\u1515\u1516\u1517\u1518\u1519\u151a\u151b\u151c\u151d\u151e\u151f\u1520\u1521\u1522\u1523\u1524\u1525\u1526\u1527\u1528\u1529\u152a\u152b\u152c\u152d\u152e\u152f\u1530\u1531\u1532\u1533\u1534\u1535\u1536\u1537\u1538\u1539\u153a\u153b\u153c\u153d\u153e\u153f\u1540\u1541\u1542\u1543\u1544\u1545\u1546\u1547\u1548\u1549\u154a\u154b\u154c\u154d\u154e\u154f\u1550\u1551\u1552\u1553\u1554\u1555\u1556\u1557\u1558\u1559\u155a\u155b\u155c\u155d\u155e\u155f\u1560\u1561\u1562\u1563\u1564\u1565\u1566\u1567\u1568\u1569\u156a\u156b\u156c\u156d\u156e\u156f\u1570\u1571\u1572\u1573\u1574\u1575\u1576\u1577\u1578\u1579\u157a\u157b\u157c\u157d\u157e\u157f\u1580\u1581\u1582\u1583\u1584\u1585\u1586\u1587\u1588\u1589\u158a\u158b\u158c\u158d\u158e\u158f\u1590\u1591\u1592\u1593\u1594\u1595\u1596\u1597\u1598\u1599\u159a\u159b\u159c\u159d\u159e\u159f\u15a0\u15a1\u15a2\u15a3\u15a4\u15a5\u15a6\u15a7\u15a8\u15a9\u15aa\u15ab\u15ac\u15ad\u15ae\u15af\u15b0\u15b1\u15b2\u15b3\u15b4\u15b5\u15b6\u15b7\u15b8\u15b9\u15ba\u15bb\u15bc\u15bd\u15be\u15bf\u15c0\u15c1\u15c2\u15c3\u15c4\u15c5\u15c6\u15c7\u15c8\u15c9\u15ca\u15cb\u15cc\u15cd\u15ce\u15cf\u15d0\u15d1\u15d2\u15d3\u15d4\u15d5\u15d6\u15d7\u15d8\u15d9\u15da\u15db\u15dc\u15dd\u15de\u15df\u15e0\u15e1\u15e2\u15e3\u15e4\u15e5\u15e6\u15e7\u15e8\u15e9\u15ea\u15eb\u15ec\u15ed\u15ee\u15ef\u15f0\u15f1\u15f2\u15f3\u15f4\u15f5\u15f6\u15f7\u15f8\u15f9\u15fa\u15fb\u15fc\u15fd\u15fe\u15ff\u1600\u1601\u1602\u1603\u1604\u1605\u1606\u1607\u1608\u1609\u160a\u160b\u160c\u160d\u160e\u160f\u1610\u1611\u1612\u1613\u1614\u1615\u1616\u1617\u1618\u1619\u161a\u161b\u161c\u161d\u161e\u161f\u1620\u1621\u1622\u1623\u1624\u1625\u1626\u1627\u1628\u1629\u162a\u162b\u162c\u162d\u162e\u162f\u1630\u1631\u1632\u1633\u1634\u1635\u1636\u1637\u1638\u1639\u163a\u163b\u163c\u163d\u163e\u163f\u1640\u1641\u1642\u1643\u1644\u1645\u1646\u1647\u1648\u1649\u164a\u164b\u164c\u164d\u164e\u164f\u1650\u1651\u1652\u1653\u1654\u1655\u1656\u1657\u1658\u1659\u165a\u165b\u165c\u165d\u165e\u165f\u1660\u1661\u1662\u1663\u1664\u1665\u1666\u1667\u1668\u1669\u166a\u166b\u166c\u166f\u1670\u1671\u1672\u1673\u1674\u1675\u1676\u1681\u1682\u1683\u1684\u1685\u1686\u1687\u1688\u1689\u168a\u168b\u168c\u168d\u168e\u168f\u1690\u1691\u1692\u1693\u1694\u1695\u1696\u1697\u1698\u1699\u169a\u16a0\u16a1\u16a2\u16a3\u16a4\u16a5\u16a6\u16a7\u16a8\u16a9\u16aa\u16ab\u16ac\u16ad\u16ae\u16af\u16b0\u16b1\u16b2\u16b3\u16b4\u16b5\u16b6\u16b7\u16b8\u16b9\u16ba\u16bb\u16bc\u16bd\u16be\u16bf\u16c0\u16c1\u16c2\u16c3\u16c4\u16c5\u16c6\u16c7\u16c8\u16c9\u16ca\u16cb\u16cc\u16cd\u16ce\u16cf\u16d0\u16d1\u16d2\u16d3\u16d4\u16d5\u16d6\u16d7\u16d8\u16d9\u16da\u16db\u16dc\u16dd\u16de\u16df\u16e0\u16e1\u16e2\u16e3\u16e4\u16e5\u16e6\u16e7\u16e8\u16e9\u16ea\u1700\u1701\u1702\u1703\u1704\u1705\u1706\u1707\u1708\u1709\u170a\u170b\u170c\u170e\u170f\u1710\u1711\u1720\u1721\u1722\u1723\u1724\u1725\u1726\u1727\u1728\u1729\u172a\u172b\u172c\u172d\u172e\u172f\u1730\u1731\u1740\u1741\u1742\u1743\u1744\u1745\u1746\u1747\u1748\u1749\u174a\u174b\u174c\u174d\u174e\u174f\u1750\u1751\u1760\u1761\u1762\u1763\u1764\u1765\u1766\u1767\u1768\u1769\u176a\u176b\u176c\u176e\u176f\u1770\u1780\u1781\u1782\u1783\u1784\u1785\u1786\u1787\u1788\u1789\u178a\u178b\u178c\u178d\u178e\u178f\u1790\u1791\u1792\u1793\u1794\u1795\u1796\u1797\u1798\u1799\u179a\u179b\u179c\u179d\u179e\u179f\u17a0\u17a1\u17a2\u17a3\u17a4\u17a5\u17a6\u17a7\u17a8\u17a9\u17aa\u17ab\u17ac\u17ad\u17ae\u17af\u17b0\u17b1\u17b2\u17b3\u17dc\u1820\u1821\u1822\u1823\u1824\u1825\u1826\u1827\u1828\u1829\u182a\u182b\u182c\u182d\u182e\u182f\u1830\u1831\u1832\u1833\u1834\u1835\u1836\u1837\u1838\u1839\u183a\u183b\u183c\u183d\u183e\u183f\u1840\u1841\u1842\u1844\u1845\u1846\u1847\u1848\u1849\u184a\u184b\u184c\u184d\u184e\u184f\u1850\u1851\u1852\u1853\u1854\u1855\u1856\u1857\u1858\u1859\u185a\u185b\u185c\u185d\u185e\u185f\u1860\u1861\u1862\u1863\u1864\u1865\u1866\u1867\u1868\u1869\u186a\u186b\u186c\u186d\u186e\u186f\u1870\u1871\u1872\u1873\u1874\u1875\u1876\u1877\u1880\u1881\u1882\u1883\u1884\u1885\u1886\u1887\u1888\u1889\u188a\u188b\u188c\u188d\u188e\u188f\u1890\u1891\u1892\u1893\u1894\u1895\u1896\u1897\u1898\u1899\u189a\u189b\u189c\u189d\u189e\u189f\u18a0\u18a1\u18a2\u18a3\u18a4\u18a5\u18a6\u18a7\u18a8\u1900\u1901\u1902\u1903\u1904\u1905\u1906\u1907\u1908\u1909\u190a\u190b\u190c\u190d\u190e\u190f\u1910\u1911\u1912\u1913\u1914\u1915\u1916\u1917\u1918\u1919\u191a\u191b\u191c\u1950\u1951\u1952\u1953\u1954\u1955\u1956\u1957\u1958\u1959\u195a\u195b\u195c\u195d\u195e\u195f\u1960\u1961\u1962\u1963\u1964\u1965\u1966\u1967\u1968\u1969\u196a\u196b\u196c\u196d\u1970\u1971\u1972\u1973\u1974\u1980\u1981\u1982\u1983\u1984\u1985\u1986\u1987\u1988\u1989\u198a\u198b\u198c\u198d\u198e\u198f\u1990\u1991\u1992\u1993\u1994\u1995\u1996\u1997\u1998\u1999\u199a\u199b\u199c\u199d\u199e\u199f\u19a0\u19a1\u19a2\u19a3\u19a4\u19a5\u19a6\u19a7\u19a8\u19a9\u19c1\u19c2\u19c3\u19c4\u19c5\u19c6\u19c7\u1a00\u1a01\u1a02\u1a03\u1a04\u1a05\u1a06\u1a07\u1a08\u1a09\u1a0a\u1a0b\u1a0c\u1a0d\u1a0e\u1a0f\u1a10\u1a11\u1a12\u1a13\u1a14\u1a15\u1a16\u2135\u2136\u2137\u2138\u2d30\u2d31\u2d32\u2d33\u2d34\u2d35\u2d36\u2d37\u2d38\u2d39\u2d3a\u2d3b\u2d3c\u2d3d\u2d3e\u2d3f\u2d40\u2d41\u2d42\u2d43\u2d44\u2d45\u2d46\u2d47\u2d48\u2d49\u2d4a\u2d4b\u2d4c\u2d4d\u2d4e\u2d4f\u2d50\u2d51\u2d52\u2d53\u2d54\u2d55\u2d56\u2d57\u2d58\u2d59\u2d5a\u2d5b\u2d5c\u2d5d\u2d5e\u2d5f\u2d60\u2d61\u2d62\u2d63\u2d64\u2d65\u2d80\u2d81\u2d82\u2d83\u2d84\u2d85\u2d86\u2d87\u2d88\u2d89\u2d8a\u2d8b\u2d8c\u2d8d\u2d8e\u2d8f\u2d90\u2d91\u2d92\u2d93\u2d94\u2d95\u2d96\u2da0\u2da1\u2da2\u2da3\u2da4\u2da5\u2da6\u2da8\u2da9\u2daa\u2dab\u2dac\u2dad\u2dae\u2db0\u2db1\u2db2\u2db3\u2db4\u2db5\u2db6\u2db8\u2db9\u2dba\u2dbb\u2dbc\u2dbd\u2dbe\u2dc0\u2dc1\u2dc2\u2dc3\u2dc4\u2dc5\u2dc6\u2dc8\u2dc9\u2dca\u2dcb\u2dcc\u2dcd\u2dce\u2dd0\u2dd1\u2dd2\u2dd3\u2dd4\u2dd5\u2dd6\u2dd8\u2dd9\u2dda\u2ddb\u2ddc\u2ddd\u2dde\u3006\u303c\u3041\u3042\u3043\u3044\u3045\u3046\u3047\u3048\u3049\u304a\u304b\u304c\u304d\u304e\u304f\u3050\u3051\u3052\u3053\u3054\u3055\u3056\u3057\u3058\u3059\u305a\u305b\u305c\u305d\u305e\u305f\u3060\u3061\u3062\u3063\u3064\u3065\u3066\u3067\u3068\u3069\u306a\u306b\u306c\u306d\u306e\u306f\u3070\u3071\u3072\u3073\u3074\u3075\u3076\u3077\u3078\u3079\u307a\u307b\u307c\u307d\u307e\u307f\u3080\u3081\u3082\u3083\u3084\u3085\u3086\u3087\u3088\u3089\u308a\u308b\u308c\u308d\u308e\u308f\u3090\u3091\u3092\u3093\u3094\u3095\u3096\u309f\u30a1\u30a2\u30a3\u30a4\u30a5\u30a6\u30a7\u30a8\u30a9\u30aa\u30ab\u30ac\u30ad\u30ae\u30af\u30b0\u30b1\u30b2\u30b3\u30b4\u30b5\u30b6\u30b7\u30b8\u30b9\u30ba\u30bb\u30bc\u30bd\u30be\u30bf\u30c0\u30c1\u30c2\u30c3\u30c4\u30c5\u30c6\u30c7\u30c8\u30c9\u30ca\u30cb\u30cc\u30cd\u30ce\u30cf\u30d0\u30d1\u30d2\u30d3\u30d4\u30d5\u30d6\u30d7\u30d8\u30d9\u30da\u30db\u30dc\u30dd\u30de\u30df\u30e0\u30e1\u30e2\u30e3\u30e4\u30e5\u30e6\u30e7\u30e8\u30e9\u30ea\u30eb\u30ec\u30ed\u30ee\u30ef\u30f0\u30f1\u30f2\u30f3\u30f4\u30f5\u30f6\u30f7\u30f8\u30f9\u30fa\u30ff\u3105\u3106\u3107\u3108\u3109\u310a\u310b\u310c\u310d\u310e\u310f\u3110\u3111\u3112\u3113\u3114\u3115\u3116\u3117\u3118\u3119\u311a\u311b\u311c\u311d\u311e\u311f\u3120\u3121\u3122\u3123\u3124\u3125\u3126\u3127\u3128\u3129\u312a\u312b\u312c\u3131\u3132\u3133\u3134\u3135\u3136\u3137\u3138\u3139\u313a\u313b\u313c\u313d\u313e\u313f\u3140\u3141\u3142\u3143\u3144\u3145\u3146\u3147\u3148\u3149\u314a\u314b\u314c\u314d\u314e\u314f\u3150\u3151\u3152\u3153\u3154\u3155\u3156\u3157\u3158\u3159\u315a\u315b\u315c\u315d\u315e\u315f\u3160\u3161\u3162\u3163\u3164\u3165\u3166\u3167\u3168\u3169\u316a\u316b\u316c\u316d\u316e\u316f\u3170\u3171\u3172\u3173\u3174\u3175\u3176\u3177\u3178\u3179\u317a\u317b\u317c\u317d\u317e\u317f\u3180\u3181\u3182\u3183\u3184\u3185\u3186\u3187\u3188\u3189\u318a\u318b\u318c\u318d\u318e\u31a0\u31a1\u31a2\u31a3\u31a4\u31a5\u31a6\u31a7\u31a8\u31a9\u31aa\u31ab\u31ac\u31ad\u31ae\u31af\u31b0\u31b1\u31b2\u31b3\u31b4\u31b5\u31b6\u31b7\u31f0\u31f1\u31f2\u31f3\u31f4\u31f5\u31f6\u31f7\u31f8\u31f9\u31fa\u31fb\u31fc\u31fd\u31fe\u31ff\u3400\u3401\u3402\u3403\u3404\u3405\u3406\u3407\u3408\u3409\u340a\u340b\u340c\u340d\u340e\u340f\u3410\u3411\u3412\u3413\u3414\u3415\u3416\u3417\u3418\u3419\u341a\u341b\u341c\u341d\u341e\u341f\u3420\u3421\u3422\u3423\u3424\u3425\u3426\u3427\u3428\u3429\u342a\u342b\u342c\u342d\u342e\u342f\u3430\u3431\u3432\u3433\u3434\u3435\u3436\u3437\u3438\u3439\u343a\u343b\u343c\u343d\u343e\u343f\u3440\u3441\u3442\u3443\u3444\u3445\u3446\u3447\u3448\u3449\u344a\u344b\u344c\u344d\u344e\u344f\u3450\u3451\u3452\u3453\u3454\u3455\u3456\u3457\u3458\u3459\u345a\u345b\u345c\u345d\u345e\u345f\u3460\u3461\u3462\u3463\u3464\u3465\u3466\u3467\u3468\u3469\u346a\u346b\u346c\u346d\u346e\u346f\u3470\u3471\u3472\u3473\u3474\u3475\u3476\u3477\u3478\u3479\u347a\u347b\u347c\u347d\u347e\u347f\u3480\u3481\u3482\u3483\u3484\u3485\u3486\u3487\u3488\u3489\u348a\u348b\u348c\u348d\u348e\u348f\u3490\u3491\u3492\u3493\u3494\u3495\u3496\u3497\u3498\u3499\u349a\u349b\u349c\u349d\u349e\u349f\u34a0\u34a1\u34a2\u34a3\u34a4\u34a5\u34a6\u34a7\u34a8\u34a9\u34aa\u34ab\u34ac\u34ad\u34ae\u34af\u34b0\u34b1\u34b2\u34b3\u34b4\u34b5\u34b6\u34b7\u34b8\u34b9\u34ba\u34bb\u34bc\u34bd\u34be\u34bf\u34c0\u34c1\u34c2\u34c3\u34c4\u34c5\u34c6\u34c7\u34c8\u34c9\u34ca\u34cb\u34cc\u34cd\u34ce\u34cf\u34d0\u34d1\u34d2\u34d3\u34d4\u34d5\u34d6\u34d7\u34d8\u34d9\u34da\u34db\u34dc\u34dd\u34de\u34df\u34e0\u34e1\u34e2\u34e3\u34e4\u34e5\u34e6\u34e7\u34e8\u34e9\u34ea\u34eb\u34ec\u34ed\u34ee\u34ef\u34f0\u34f1\u34f2\u34f3\u34f4\u34f5\u34f6\u34f7\u34f8\u34f9\u34fa\u34fb\u34fc\u34fd\u34fe\u34ff\u3500\u3501\u3502\u3503\u3504\u3505\u3506\u3507\u3508\u3509\u350a\u350b\u350c\u350d\u350e\u350f\u3510\u3511\u3512\u3513\u3514\u3515\u3516\u3517\u3518\u3519\u351a\u351b\u351c\u351d\u351e\u351f\u3520\u3521\u3522\u3523\u3524\u3525\u3526\u3527\u3528\u3529\u352a\u352b\u352c\u352d\u352e\u352f\u3530\u3531\u3532\u3533\u3534\u3535\u3536\u3537\u3538\u3539\u353a\u353b\u353c\u353d\u353e\u353f\u3540\u3541\u3542\u3543\u3544\u3545\u3546\u3547\u3548\u3549\u354a\u354b\u354c\u354d\u354e\u354f\u3550\u3551\u3552\u3553\u3554\u3555\u3556\u3557\u3558\u3559\u355a\u355b\u355c\u355d\u355e\u355f\u3560\u3561\u3562\u3563\u3564\u3565\u3566\u3567\u3568\u3569\u356a\u356b\u356c\u356d\u356e\u356f\u3570\u3571\u3572\u3573\u3574\u3575\u3576\u3577\u3578\u3579\u357a\u357b\u357c\u357d\u357e\u357f\u3580\u3581\u3582\u3583\u3584\u3585\u3586\u3587\u3588\u3589\u358a\u358b\u358c\u358d\u358e\u358f\u3590\u3591\u3592\u3593\u3594\u3595\u3596\u3597\u3598\u3599\u359a\u359b\u359c\u359d\u359e\u359f\u35a0\u35a1\u35a2\u35a3\u35a4\u35a5\u35a6\u35a7\u35a8\u35a9\u35aa\u35ab\u35ac\u35ad\u35ae\u35af\u35b0\u35b1\u35b2\u35b3\u35b4\u35b5\u35b6\u35b7\u35b8\u35b9\u35ba\u35bb\u35bc\u35bd\u35be\u35bf\u35c0\u35c1\u35c2\u35c3\u35c4\u35c5\u35c6\u35c7\u35c8\u35c9\u35ca\u35cb\u35cc\u35cd\u35ce\u35cf\u35d0\u35d1\u35d2\u35d3\u35d4\u35d5\u35d6\u35d7\u35d8\u35d9\u35da\u35db\u35dc\u35dd\u35de\u35df\u35e0\u35e1\u35e2\u35e3\u35e4\u35e5\u35e6\u35e7\u35e8\u35e9\u35ea\u35eb\u35ec\u35ed\u35ee\u35ef\u35f0\u35f1\u35f2\u35f3\u35f4\u35f5\u35f6\u35f7\u35f8\u35f9\u35fa\u35fb\u35fc\u35fd\u35fe\u35ff\u3600\u3601\u3602\u3603\u3604\u3605\u3606\u3607\u3608\u3609\u360a\u360b\u360c\u360d\u360e\u360f\u3610\u3611\u3612\u3613\u3614\u3615\u3616\u3617\u3618\u3619\u361a\u361b\u361c\u361d\u361e\u361f\u3620\u3621\u3622\u3623\u3624\u3625\u3626\u3627\u3628\u3629\u362a\u362b\u362c\u362d\u362e\u362f\u3630\u3631\u3632\u3633\u3634\u3635\u3636\u3637\u3638\u3639\u363a\u363b\u363c\u363d\u363e\u363f\u3640\u3641\u3642\u3643\u3644\u3645\u3646\u3647\u3648\u3649\u364a\u364b\u364c\u364d\u364e\u364f\u3650\u3651\u3652\u3653\u3654\u3655\u3656\u3657\u3658\u3659\u365a\u365b\u365c\u365d\u365e\u365f\u3660\u3661\u3662\u3663\u3664\u3665\u3666\u3667\u3668\u3669\u366a\u366b\u366c\u366d\u366e\u366f\u3670\u3671\u3672\u3673\u3674\u3675\u3676\u3677\u3678\u3679\u367a\u367b\u367c\u367d\u367e\u367f\u3680\u3681\u3682\u3683\u3684\u3685\u3686\u3687\u3688\u3689\u368a\u368b\u368c\u368d\u368e\u368f\u3690\u3691\u3692\u3693\u3694\u3695\u3696\u3697\u3698\u3699\u369a\u369b\u369c\u369d\u369e\u369f\u36a0\u36a1\u36a2\u36a3\u36a4\u36a5\u36a6\u36a7\u36a8\u36a9\u36aa\u36ab\u36ac\u36ad\u36ae\u36af\u36b0\u36b1\u36b2\u36b3\u36b4\u36b5\u36b6\u36b7\u36b8\u36b9\u36ba\u36bb\u36bc\u36bd\u36be\u36bf\u36c0\u36c1\u36c2\u36c3\u36c4\u36c5\u36c6\u36c7\u36c8\u36c9\u36ca\u36cb\u36cc\u36cd\u36ce\u36cf\u36d0\u36d1\u36d2\u36d3\u36d4\u36d5\u36d6\u36d7\u36d8\u36d9\u36da\u36db\u36dc\u36dd\u36de\u36df\u36e0\u36e1\u36e2\u36e3\u36e4\u36e5\u36e6\u36e7\u36e8\u36e9\u36ea\u36eb\u36ec\u36ed\u36ee\u36ef\u36f0\u36f1\u36f2\u36f3\u36f4\u36f5\u36f6\u36f7\u36f8\u36f9\u36fa\u36fb\u36fc\u36fd\u36fe\u36ff\u3700\u3701\u3702\u3703\u3704\u3705\u3706\u3707\u3708\u3709\u370a\u370b\u370c\u370d\u370e\u370f\u3710\u3711\u3712\u3713\u3714\u3715\u3716\u3717\u3718\u3719\u371a\u371b\u371c\u371d\u371e\u371f\u3720\u3721\u3722\u3723\u3724\u3725\u3726\u3727\u3728\u3729\u372a\u372b\u372c\u372d\u372e\u372f\u3730\u3731\u3732\u3733\u3734\u3735\u3736\u3737\u3738\u3739\u373a\u373b\u373c\u373d\u373e\u373f\u3740\u3741\u3742\u3743\u3744\u3745\u3746\u3747\u3748\u3749\u374a\u374b\u374c\u374d\u374e\u374f\u3750\u3751\u3752\u3753\u3754\u3755\u3756\u3757\u3758\u3759\u375a\u375b\u375c\u375d\u375e\u375f\u3760\u3761\u3762\u3763\u3764\u3765\u3766\u3767\u3768\u3769\u376a\u376b\u376c\u376d\u376e\u376f\u3770\u3771\u3772\u3773\u3774\u3775\u3776\u3777\u3778\u3779\u377a\u377b\u377c\u377d\u377e\u377f\u3780\u3781\u3782\u3783\u3784\u3785\u3786\u3787\u3788\u3789\u378a\u378b\u378c\u378d\u378e\u378f\u3790\u3791\u3792\u3793\u3794\u3795\u3796\u3797\u3798\u3799\u379a\u379b\u379c\u379d\u379e\u379f\u37a0\u37a1\u37a2\u37a3\u37a4\u37a5\u37a6\u37a7\u37a8\u37a9\u37aa\u37ab\u37ac\u37ad\u37ae\u37af\u37b0\u37b1\u37b2\u37b3\u37b4\u37b5\u37b6\u37b7\u37b8\u37b9\u37ba\u37bb\u37bc\u37bd\u37be\u37bf\u37c0\u37c1\u37c2\u37c3\u37c4\u37c5\u37c6\u37c7\u37c8\u37c9\u37ca\u37cb\u37cc\u37cd\u37ce\u37cf\u37d0\u37d1\u37d2\u37d3\u37d4\u37d5\u37d6\u37d7\u37d8\u37d9\u37da\u37db\u37dc\u37dd\u37de\u37df\u37e0\u37e1\u37e2\u37e3\u37e4\u37e5\u37e6\u37e7\u37e8\u37e9\u37ea\u37eb\u37ec\u37ed\u37ee\u37ef\u37f0\u37f1\u37f2\u37f3\u37f4\u37f5\u37f6\u37f7\u37f8\u37f9\u37fa\u37fb\u37fc\u37fd\u37fe\u37ff\u3800\u3801\u3802\u3803\u3804\u3805\u3806\u3807\u3808\u3809\u380a\u380b\u380c\u380d\u380e\u380f\u3810\u3811\u3812\u3813\u3814\u3815\u3816\u3817\u3818\u3819\u381a\u381b\u381c\u381d\u381e\u381f\u3820\u3821\u3822\u3823\u3824\u3825\u3826\u3827\u3828\u3829\u382a\u382b\u382c\u382d\u382e\u382f\u3830\u3831\u3832\u3833\u3834\u3835\u3836\u3837\u3838\u3839\u383a\u383b\u383c\u383d\u383e\u383f\u3840\u3841\u3842\u3843\u3844\u3845\u3846\u3847\u3848\u3849\u384a\u384b\u384c\u384d\u384e\u384f\u3850\u3851\u3852\u3853\u3854\u3855\u3856\u3857\u3858\u3859\u385a\u385b\u385c\u385d\u385e\u385f\u3860\u3861\u3862\u3863\u3864\u3865\u3866\u3867\u3868\u3869\u386a\u386b\u386c\u386d\u386e\u386f\u3870\u3871\u3872\u3873\u3874\u3875\u3876\u3877\u3878\u3879\u387a\u387b\u387c\u387d\u387e\u387f\u3880\u3881\u3882\u3883\u3884\u3885\u3886\u3887\u3888\u3889\u388a\u388b\u388c\u388d\u388e\u388f\u3890\u3891\u3892\u3893\u3894\u3895\u3896\u3897\u3898\u3899\u389a\u389b\u389c\u389d\u389e\u389f\u38a0\u38a1\u38a2\u38a3\u38a4\u38a5\u38a6\u38a7\u38a8\u38a9\u38aa\u38ab\u38ac\u38ad\u38ae\u38af\u38b0\u38b1\u38b2\u38b3\u38b4\u38b5\u38b6\u38b7\u38b8\u38b9\u38ba\u38bb\u38bc\u38bd\u38be\u38bf\u38c0\u38c1\u38c2\u38c3\u38c4\u38c5\u38c6\u38c7\u38c8\u38c9\u38ca\u38cb\u38cc\u38cd\u38ce\u38cf\u38d0\u38d1\u38d2\u38d3\u38d4\u38d5\u38d6\u38d7\u38d8\u38d9\u38da\u38db\u38dc\u38dd\u38de\u38df\u38e0\u38e1\u38e2\u38e3\u38e4\u38e5\u38e6\u38e7\u38e8\u38e9\u38ea\u38eb\u38ec\u38ed\u38ee\u38ef\u38f0\u38f1\u38f2\u38f3\u38f4\u38f5\u38f6\u38f7\u38f8\u38f9\u38fa\u38fb\u38fc\u38fd\u38fe\u38ff\u3900\u3901\u3902\u3903\u3904\u3905\u3906\u3907\u3908\u3909\u390a\u390b\u390c\u390d\u390e\u390f\u3910\u3911\u3912\u3913\u3914\u3915\u3916\u3917\u3918\u3919\u391a\u391b\u391c\u391d\u391e\u391f\u3920\u3921\u3922\u3923\u3924\u3925\u3926\u3927\u3928\u3929\u392a\u392b\u392c\u392d\u392e\u392f\u3930\u3931\u3932\u3933\u3934\u3935\u3936\u3937\u3938\u3939\u393a\u393b\u393c\u393d\u393e\u393f\u3940\u3941\u3942\u3943\u3944\u3945\u3946\u3947\u3948\u3949\u394a\u394b\u394c\u394d\u394e\u394f\u3950\u3951\u3952\u3953\u3954\u3955\u3956\u3957\u3958\u3959\u395a\u395b\u395c\u395d\u395e\u395f\u3960\u3961\u3962\u3963\u3964\u3965\u3966\u3967\u3968\u3969\u396a\u396b\u396c\u396d\u396e\u396f\u3970\u3971\u3972\u3973\u3974\u3975\u3976\u3977\u3978\u3979\u397a\u397b\u397c\u397d\u397e\u397f\u3980\u3981\u3982\u3983\u3984\u3985\u3986\u3987\u3988\u3989\u398a\u398b\u398c\u398d\u398e\u398f\u3990\u3991\u3992\u3993\u3994\u3995\u3996\u3997\u3998\u3999\u399a\u399b\u399c\u399d\u399e\u399f\u39a0\u39a1\u39a2\u39a3\u39a4\u39a5\u39a6\u39a7\u39a8\u39a9\u39aa\u39ab\u39ac\u39ad\u39ae\u39af\u39b0\u39b1\u39b2\u39b3\u39b4\u39b5\u39b6\u39b7\u39b8\u39b9\u39ba\u39bb\u39bc\u39bd\u39be\u39bf\u39c0\u39c1\u39c2\u39c3\u39c4\u39c5\u39c6\u39c7\u39c8\u39c9\u39ca\u39cb\u39cc\u39cd\u39ce\u39cf\u39d0\u39d1\u39d2\u39d3\u39d4\u39d5\u39d6\u39d7\u39d8\u39d9\u39da\u39db\u39dc\u39dd\u39de\u39df\u39e0\u39e1\u39e2\u39e3\u39e4\u39e5\u39e6\u39e7\u39e8\u39e9\u39ea\u39eb\u39ec\u39ed\u39ee\u39ef\u39f0\u39f1\u39f2\u39f3\u39f4\u39f5\u39f6\u39f7\u39f8\u39f9\u39fa\u39fb\u39fc\u39fd\u39fe\u39ff\u3a00\u3a01\u3a02\u3a03\u3a04\u3a05\u3a06\u3a07\u3a08\u3a09\u3a0a\u3a0b\u3a0c\u3a0d\u3a0e\u3a0f\u3a10\u3a11\u3a12\u3a13\u3a14\u3a15\u3a16\u3a17\u3a18\u3a19\u3a1a\u3a1b\u3a1c\u3a1d\u3a1e\u3a1f\u3a20\u3a21\u3a22\u3a23\u3a24\u3a25\u3a26\u3a27\u3a28\u3a29\u3a2a\u3a2b\u3a2c\u3a2d\u3a2e\u3a2f\u3a30\u3a31\u3a32\u3a33\u3a34\u3a35\u3a36\u3a37\u3a38\u3a39\u3a3a\u3a3b\u3a3c\u3a3d\u3a3e\u3a3f\u3a40\u3a41\u3a42\u3a43\u3a44\u3a45\u3a46\u3a47\u3a48\u3a49\u3a4a\u3a4b\u3a4c\u3a4d\u3a4e\u3a4f\u3a50\u3a51\u3a52\u3a53\u3a54\u3a55\u3a56\u3a57\u3a58\u3a59\u3a5a\u3a5b\u3a5c\u3a5d\u3a5e\u3a5f\u3a60\u3a61\u3a62\u3a63\u3a64\u3a65\u3a66\u3a67\u3a68\u3a69\u3a6a\u3a6b\u3a6c\u3a6d\u3a6e\u3a6f\u3a70\u3a71\u3a72\u3a73\u3a74\u3a75\u3a76\u3a77\u3a78\u3a79\u3a7a\u3a7b\u3a7c\u3a7d\u3a7e\u3a7f\u3a80\u3a81\u3a82\u3a83\u3a84\u3a85\u3a86\u3a87\u3a88\u3a89\u3a8a\u3a8b\u3a8c\u3a8d\u3a8e\u3a8f\u3a90\u3a91\u3a92\u3a93\u3a94\u3a95\u3a96\u3a97\u3a98\u3a99\u3a9a\u3a9b\u3a9c\u3a9d\u3a9e\u3a9f\u3aa0\u3aa1\u3aa2\u3aa3\u3aa4\u3aa5\u3aa6\u3aa7\u3aa8\u3aa9\u3aaa\u3aab\u3aac\u3aad\u3aae\u3aaf\u3ab0\u3ab1\u3ab2\u3ab3\u3ab4\u3ab5\u3ab6\u3ab7\u3ab8\u3ab9\u3aba\u3abb\u3abc\u3abd\u3abe\u3abf\u3ac0\u3ac1\u3ac2\u3ac3\u3ac4\u3ac5\u3ac6\u3ac7\u3ac8\u3ac9\u3aca\u3acb\u3acc\u3acd\u3ace\u3acf\u3ad0\u3ad1\u3ad2\u3ad3\u3ad4\u3ad5\u3ad6\u3ad7\u3ad8\u3ad9\u3ada\u3adb\u3adc\u3add\u3ade\u3adf\u3ae0\u3ae1\u3ae2\u3ae3\u3ae4\u3ae5\u3ae6\u3ae7\u3ae8\u3ae9\u3aea\u3aeb\u3aec\u3aed\u3aee\u3aef\u3af0\u3af1\u3af2\u3af3\u3af4\u3af5\u3af6\u3af7\u3af8\u3af9\u3afa\u3afb\u3afc\u3afd\u3afe\u3aff\u3b00\u3b01\u3b02\u3b03\u3b04\u3b05\u3b06\u3b07\u3b08\u3b09\u3b0a\u3b0b\u3b0c\u3b0d\u3b0e\u3b0f\u3b10\u3b11\u3b12\u3b13\u3b14\u3b15\u3b16\u3b17\u3b18\u3b19\u3b1a\u3b1b\u3b1c\u3b1d\u3b1e\u3b1f\u3b20\u3b21\u3b22\u3b23\u3b24\u3b25\u3b26\u3b27\u3b28\u3b29\u3b2a\u3b2b\u3b2c\u3b2d\u3b2e\u3b2f\u3b30\u3b31\u3b32\u3b33\u3b34\u3b35\u3b36\u3b37\u3b38\u3b39\u3b3a\u3b3b\u3b3c\u3b3d\u3b3e\u3b3f\u3b40\u3b41\u3b42\u3b43\u3b44\u3b45\u3b46\u3b47\u3b48\u3b49\u3b4a\u3b4b\u3b4c\u3b4d\u3b4e\u3b4f\u3b50\u3b51\u3b52\u3b53\u3b54\u3b55\u3b56\u3b57\u3b58\u3b59\u3b5a\u3b5b\u3b5c\u3b5d\u3b5e\u3b5f\u3b60\u3b61\u3b62\u3b63\u3b64\u3b65\u3b66\u3b67\u3b68\u3b69\u3b6a\u3b6b\u3b6c\u3b6d\u3b6e\u3b6f\u3b70\u3b71\u3b72\u3b73\u3b74\u3b75\u3b76\u3b77\u3b78\u3b79\u3b7a\u3b7b\u3b7c\u3b7d\u3b7e\u3b7f\u3b80\u3b81\u3b82\u3b83\u3b84\u3b85\u3b86\u3b87\u3b88\u3b89\u3b8a\u3b8b\u3b8c\u3b8d\u3b8e\u3b8f\u3b90\u3b91\u3b92\u3b93\u3b94\u3b95\u3b96\u3b97\u3b98\u3b99\u3b9a\u3b9b\u3b9c\u3b9d\u3b9e\u3b9f\u3ba0\u3ba1\u3ba2\u3ba3\u3ba4\u3ba5\u3ba6\u3ba7\u3ba8\u3ba9\u3baa\u3bab\u3bac\u3bad\u3bae\u3baf\u3bb0\u3bb1\u3bb2\u3bb3\u3bb4\u3bb5\u3bb6\u3bb7\u3bb8\u3bb9\u3bba\u3bbb\u3bbc\u3bbd\u3bbe\u3bbf\u3bc0\u3bc1\u3bc2\u3bc3\u3bc4\u3bc5\u3bc6\u3bc7\u3bc8\u3bc9\u3bca\u3bcb\u3bcc\u3bcd\u3bce\u3bcf\u3bd0\u3bd1\u3bd2\u3bd3\u3bd4\u3bd5\u3bd6\u3bd7\u3bd8\u3bd9\u3bda\u3bdb\u3bdc\u3bdd\u3bde\u3bdf\u3be0\u3be1\u3be2\u3be3\u3be4\u3be5\u3be6\u3be7\u3be8\u3be9\u3bea\u3beb\u3bec\u3bed\u3bee\u3bef\u3bf0\u3bf1\u3bf2\u3bf3\u3bf4\u3bf5\u3bf6\u3bf7\u3bf8\u3bf9\u3bfa\u3bfb\u3bfc\u3bfd\u3bfe\u3bff\u3c00\u3c01\u3c02\u3c03\u3c04\u3c05\u3c06\u3c07\u3c08\u3c09\u3c0a\u3c0b\u3c0c\u3c0d\u3c0e\u3c0f\u3c10\u3c11\u3c12\u3c13\u3c14\u3c15\u3c16\u3c17\u3c18\u3c19\u3c1a\u3c1b\u3c1c\u3c1d\u3c1e\u3c1f\u3c20\u3c21\u3c22\u3c23\u3c24\u3c25\u3c26\u3c27\u3c28\u3c29\u3c2a\u3c2b\u3c2c\u3c2d\u3c2e\u3c2f\u3c30\u3c31\u3c32\u3c33\u3c34\u3c35\u3c36\u3c37\u3c38\u3c39\u3c3a\u3c3b\u3c3c\u3c3d\u3c3e\u3c3f\u3c40\u3c41\u3c42\u3c43\u3c44\u3c45\u3c46\u3c47\u3c48\u3c49\u3c4a\u3c4b\u3c4c\u3c4d\u3c4e\u3c4f\u3c50\u3c51\u3c52\u3c53\u3c54\u3c55\u3c56\u3c57\u3c58\u3c59\u3c5a\u3c5b\u3c5c\u3c5d\u3c5e\u3c5f\u3c60\u3c61\u3c62\u3c63\u3c64\u3c65\u3c66\u3c67\u3c68\u3c69\u3c6a\u3c6b\u3c6c\u3c6d\u3c6e\u3c6f\u3c70\u3c71\u3c72\u3c73\u3c74\u3c75\u3c76\u3c77\u3c78\u3c79\u3c7a\u3c7b\u3c7c\u3c7d\u3c7e\u3c7f\u3c80\u3c81\u3c82\u3c83\u3c84\u3c85\u3c86\u3c87\u3c88\u3c89\u3c8a\u3c8b\u3c8c\u3c8d\u3c8e\u3c8f\u3c90\u3c91\u3c92\u3c93\u3c94\u3c95\u3c96\u3c97\u3c98\u3c99\u3c9a\u3c9b\u3c9c\u3c9d\u3c9e\u3c9f\u3ca0\u3ca1\u3ca2\u3ca3\u3ca4\u3ca5\u3ca6\u3ca7\u3ca8\u3ca9\u3caa\u3cab\u3cac\u3cad\u3cae\u3caf\u3cb0\u3cb1\u3cb2\u3cb3\u3cb4\u3cb5\u3cb6\u3cb7\u3cb8\u3cb9\u3cba\u3cbb\u3cbc\u3cbd\u3cbe\u3cbf\u3cc0\u3cc1\u3cc2\u3cc3\u3cc4\u3cc5\u3cc6\u3cc7\u3cc8\u3cc9\u3cca\u3ccb\u3ccc\u3ccd\u3cce\u3ccf\u3cd0\u3cd1\u3cd2\u3cd3\u3cd4\u3cd5\u3cd6\u3cd7\u3cd8\u3cd9\u3cda\u3cdb\u3cdc\u3cdd\u3cde\u3cdf\u3ce0\u3ce1\u3ce2\u3ce3\u3ce4\u3ce5\u3ce6\u3ce7\u3ce8\u3ce9\u3cea\u3ceb\u3cec\u3ced\u3cee\u3cef\u3cf0\u3cf1\u3cf2\u3cf3\u3cf4\u3cf5\u3cf6\u3cf7\u3cf8\u3cf9\u3cfa\u3cfb\u3cfc\u3cfd\u3cfe\u3cff\u3d00\u3d01\u3d02\u3d03\u3d04\u3d05\u3d06\u3d07\u3d08\u3d09\u3d0a\u3d0b\u3d0c\u3d0d\u3d0e\u3d0f\u3d10\u3d11\u3d12\u3d13\u3d14\u3d15\u3d16\u3d17\u3d18\u3d19\u3d1a\u3d1b\u3d1c\u3d1d\u3d1e\u3d1f\u3d20\u3d21\u3d22\u3d23\u3d24\u3d25\u3d26\u3d27\u3d28\u3d29\u3d2a\u3d2b\u3d2c\u3d2d\u3d2e\u3d2f\u3d30\u3d31\u3d32\u3d33\u3d34\u3d35\u3d36\u3d37\u3d38\u3d39\u3d3a\u3d3b\u3d3c\u3d3d\u3d3e\u3d3f\u3d40\u3d41\u3d42\u3d43\u3d44\u3d45\u3d46\u3d47\u3d48\u3d49\u3d4a\u3d4b\u3d4c\u3d4d\u3d4e\u3d4f\u3d50\u3d51\u3d52\u3d53\u3d54\u3d55\u3d56\u3d57\u3d58\u3d59\u3d5a\u3d5b\u3d5c\u3d5d\u3d5e\u3d5f\u3d60\u3d61\u3d62\u3d63\u3d64\u3d65\u3d66\u3d67\u3d68\u3d69\u3d6a\u3d6b\u3d6c\u3d6d\u3d6e\u3d6f\u3d70\u3d71\u3d72\u3d73\u3d74\u3d75\u3d76\u3d77\u3d78\u3d79\u3d7a\u3d7b\u3d7c\u3d7d\u3d7e\u3d7f\u3d80\u3d81\u3d82\u3d83\u3d84\u3d85\u3d86\u3d87\u3d88\u3d89\u3d8a\u3d8b\u3d8c\u3d8d\u3d8e\u3d8f\u3d90\u3d91\u3d92\u3d93\u3d94\u3d95\u3d96\u3d97\u3d98\u3d99\u3d9a\u3d9b\u3d9c\u3d9d\u3d9e\u3d9f\u3da0\u3da1\u3da2\u3da3\u3da4\u3da5\u3da6\u3da7\u3da8\u3da9\u3daa\u3dab\u3dac\u3dad\u3dae\u3daf\u3db0\u3db1\u3db2\u3db3\u3db4\u3db5\u3db6\u3db7\u3db8\u3db9\u3dba\u3dbb\u3dbc\u3dbd\u3dbe\u3dbf\u3dc0\u3dc1\u3dc2\u3dc3\u3dc4\u3dc5\u3dc6\u3dc7\u3dc8\u3dc9\u3dca\u3dcb\u3dcc\u3dcd\u3dce\u3dcf\u3dd0\u3dd1\u3dd2\u3dd3\u3dd4\u3dd5\u3dd6\u3dd7\u3dd8\u3dd9\u3dda\u3ddb\u3ddc\u3ddd\u3dde\u3ddf\u3de0\u3de1\u3de2\u3de3\u3de4\u3de5\u3de6\u3de7\u3de8\u3de9\u3dea\u3deb\u3dec\u3ded\u3dee\u3def\u3df0\u3df1\u3df2\u3df3\u3df4\u3df5\u3df6\u3df7\u3df8\u3df9\u3dfa\u3dfb\u3dfc\u3dfd\u3dfe\u3dff\u3e00\u3e01\u3e02\u3e03\u3e04\u3e05\u3e06\u3e07\u3e08\u3e09\u3e0a\u3e0b\u3e0c\u3e0d\u3e0e\u3e0f\u3e10\u3e11\u3e12\u3e13\u3e14\u3e15\u3e16\u3e17\u3e18\u3e19\u3e1a\u3e1b\u3e1c\u3e1d\u3e1e\u3e1f\u3e20\u3e21\u3e22\u3e23\u3e24\u3e25\u3e26\u3e27\u3e28\u3e29\u3e2a\u3e2b\u3e2c\u3e2d\u3e2e\u3e2f\u3e30\u3e31\u3e32\u3e33\u3e34\u3e35\u3e36\u3e37\u3e38\u3e39\u3e3a\u3e3b\u3e3c\u3e3d\u3e3e\u3e3f\u3e40\u3e41\u3e42\u3e43\u3e44\u3e45\u3e46\u3e47\u3e48\u3e49\u3e4a\u3e4b\u3e4c\u3e4d\u3e4e\u3e4f\u3e50\u3e51\u3e52\u3e53\u3e54\u3e55\u3e56\u3e57\u3e58\u3e59\u3e5a\u3e5b\u3e5c\u3e5d\u3e5e\u3e5f\u3e60\u3e61\u3e62\u3e63\u3e64\u3e65\u3e66\u3e67\u3e68\u3e69\u3e6a\u3e6b\u3e6c\u3e6d\u3e6e\u3e6f\u3e70\u3e71\u3e72\u3e73\u3e74\u3e75\u3e76\u3e77\u3e78\u3e79\u3e7a\u3e7b\u3e7c\u3e7d\u3e7e\u3e7f\u3e80\u3e81\u3e82\u3e83\u3e84\u3e85\u3e86\u3e87\u3e88\u3e89\u3e8a\u3e8b\u3e8c\u3e8d\u3e8e\u3e8f\u3e90\u3e91\u3e92\u3e93\u3e94\u3e95\u3e96\u3e97\u3e98\u3e99\u3e9a\u3e9b\u3e9c\u3e9d\u3e9e\u3e9f\u3ea0\u3ea1\u3ea2\u3ea3\u3ea4\u3ea5\u3ea6\u3ea7\u3ea8\u3ea9\u3eaa\u3eab\u3eac\u3ead\u3eae\u3eaf\u3eb0\u3eb1\u3eb2\u3eb3\u3eb4\u3eb5\u3eb6\u3eb7\u3eb8\u3eb9\u3eba\u3ebb\u3ebc\u3ebd\u3ebe\u3ebf\u3ec0\u3ec1\u3ec2\u3ec3\u3ec4\u3ec5\u3ec6\u3ec7\u3ec8\u3ec9\u3eca\u3ecb\u3ecc\u3ecd\u3ece\u3ecf\u3ed0\u3ed1\u3ed2\u3ed3\u3ed4\u3ed5\u3ed6\u3ed7\u3ed8\u3ed9\u3eda\u3edb\u3edc\u3edd\u3ede\u3edf\u3ee0\u3ee1\u3ee2\u3ee3\u3ee4\u3ee5\u3ee6\u3ee7\u3ee8\u3ee9\u3eea\u3eeb\u3eec\u3eed\u3eee\u3eef\u3ef0\u3ef1\u3ef2\u3ef3\u3ef4\u3ef5\u3ef6\u3ef7\u3ef8\u3ef9\u3efa\u3efb\u3efc\u3efd\u3efe\u3eff\u3f00\u3f01\u3f02\u3f03\u3f04\u3f05\u3f06\u3f07\u3f08\u3f09\u3f0a\u3f0b\u3f0c\u3f0d\u3f0e\u3f0f\u3f10\u3f11\u3f12\u3f13\u3f14\u3f15\u3f16\u3f17\u3f18\u3f19\u3f1a\u3f1b\u3f1c\u3f1d\u3f1e\u3f1f\u3f20\u3f21\u3f22\u3f23\u3f24\u3f25\u3f26\u3f27\u3f28\u3f29\u3f2a\u3f2b\u3f2c\u3f2d\u3f2e\u3f2f\u3f30\u3f31\u3f32\u3f33\u3f34\u3f35\u3f36\u3f37\u3f38\u3f39\u3f3a\u3f3b\u3f3c\u3f3d\u3f3e\u3f3f\u3f40\u3f41\u3f42\u3f43\u3f44\u3f45\u3f46\u3f47\u3f48\u3f49\u3f4a\u3f4b\u3f4c\u3f4d\u3f4e\u3f4f\u3f50\u3f51\u3f52\u3f53\u3f54\u3f55\u3f56\u3f57\u3f58\u3f59\u3f5a\u3f5b\u3f5c\u3f5d\u3f5e\u3f5f\u3f60\u3f61\u3f62\u3f63\u3f64\u3f65\u3f66\u3f67\u3f68\u3f69\u3f6a\u3f6b\u3f6c\u3f6d\u3f6e\u3f6f\u3f70\u3f71\u3f72\u3f73\u3f74\u3f75\u3f76\u3f77\u3f78\u3f79\u3f7a\u3f7b\u3f7c\u3f7d\u3f7e\u3f7f\u3f80\u3f81\u3f82\u3f83\u3f84\u3f85\u3f86\u3f87\u3f88\u3f89\u3f8a\u3f8b\u3f8c\u3f8d\u3f8e\u3f8f\u3f90\u3f91\u3f92\u3f93\u3f94\u3f95\u3f96\u3f97\u3f98\u3f99\u3f9a\u3f9b\u3f9c\u3f9d\u3f9e\u3f9f\u3fa0\u3fa1\u3fa2\u3fa3\u3fa4\u3fa5\u3fa6\u3fa7\u3fa8\u3fa9\u3faa\u3fab\u3fac\u3fad\u3fae\u3faf\u3fb0\u3fb1\u3fb2\u3fb3\u3fb4\u3fb5\u3fb6\u3fb7\u3fb8\u3fb9\u3fba\u3fbb\u3fbc\u3fbd\u3fbe\u3fbf\u3fc0\u3fc1\u3fc2\u3fc3\u3fc4\u3fc5\u3fc6\u3fc7\u3fc8\u3fc9\u3fca\u3fcb\u3fcc\u3fcd\u3fce\u3fcf\u3fd0\u3fd1\u3fd2\u3fd3\u3fd4\u3fd5\u3fd6\u3fd7\u3fd8\u3fd9\u3fda\u3fdb\u3fdc\u3fdd\u3fde\u3fdf\u3fe0\u3fe1\u3fe2\u3fe3\u3fe4\u3fe5\u3fe6\u3fe7\u3fe8\u3fe9\u3fea\u3feb\u3fec\u3fed\u3fee\u3fef\u3ff0\u3ff1\u3ff2\u3ff3\u3ff4\u3ff5\u3ff6\u3ff7\u3ff8\u3ff9\u3ffa\u3ffb\u3ffc\u3ffd\u3ffe\u3fff\u4000\u4001\u4002\u4003\u4004\u4005\u4006\u4007\u4008\u4009\u400a\u400b\u400c\u400d\u400e\u400f\u4010\u4011\u4012\u4013\u4014\u4015\u4016\u4017\u4018\u4019\u401a\u401b\u401c\u401d\u401e\u401f\u4020\u4021\u4022\u4023\u4024\u4025\u4026\u4027\u4028\u4029\u402a\u402b\u402c\u402d\u402e\u402f\u4030\u4031\u4032\u4033\u4034\u4035\u4036\u4037\u4038\u4039\u403a\u403b\u403c\u403d\u403e\u403f\u4040\u4041\u4042\u4043\u4044\u4045\u4046\u4047\u4048\u4049\u404a\u404b\u404c\u404d\u404e\u404f\u4050\u4051\u4052\u4053\u4054\u4055\u4056\u4057\u4058\u4059\u405a\u405b\u405c\u405d\u405e\u405f\u4060\u4061\u4062\u4063\u4064\u4065\u4066\u4067\u4068\u4069\u406a\u406b\u406c\u406d\u406e\u406f\u4070\u4071\u4072\u4073\u4074\u4075\u4076\u4077\u4078\u4079\u407a\u407b\u407c\u407d\u407e\u407f\u4080\u4081\u4082\u4083\u4084\u4085\u4086\u4087\u4088\u4089\u408a\u408b\u408c\u408d\u408e\u408f\u4090\u4091\u4092\u4093\u4094\u4095\u4096\u4097\u4098\u4099\u409a\u409b\u409c\u409d\u409e\u409f\u40a0\u40a1\u40a2\u40a3\u40a4\u40a5\u40a6\u40a7\u40a8\u40a9\u40aa\u40ab\u40ac\u40ad\u40ae\u40af\u40b0\u40b1\u40b2\u40b3\u40b4\u40b5\u40b6\u40b7\u40b8\u40b9\u40ba\u40bb\u40bc\u40bd\u40be\u40bf\u40c0\u40c1\u40c2\u40c3\u40c4\u40c5\u40c6\u40c7\u40c8\u40c9\u40ca\u40cb\u40cc\u40cd\u40ce\u40cf\u40d0\u40d1\u40d2\u40d3\u40d4\u40d5\u40d6\u40d7\u40d8\u40d9\u40da\u40db\u40dc\u40dd\u40de\u40df\u40e0\u40e1\u40e2\u40e3\u40e4\u40e5\u40e6\u40e7\u40e8\u40e9\u40ea\u40eb\u40ec\u40ed\u40ee\u40ef\u40f0\u40f1\u40f2\u40f3\u40f4\u40f5\u40f6\u40f7\u40f8\u40f9\u40fa\u40fb\u40fc\u40fd\u40fe\u40ff\u4100\u4101\u4102\u4103\u4104\u4105\u4106\u4107\u4108\u4109\u410a\u410b\u410c\u410d\u410e\u410f\u4110\u4111\u4112\u4113\u4114\u4115\u4116\u4117\u4118\u4119\u411a\u411b\u411c\u411d\u411e\u411f\u4120\u4121\u4122\u4123\u4124\u4125\u4126\u4127\u4128\u4129\u412a\u412b\u412c\u412d\u412e\u412f\u4130\u4131\u4132\u4133\u4134\u4135\u4136\u4137\u4138\u4139\u413a\u413b\u413c\u413d\u413e\u413f\u4140\u4141\u4142\u4143\u4144\u4145\u4146\u4147\u4148\u4149\u414a\u414b\u414c\u414d\u414e\u414f\u4150\u4151\u4152\u4153\u4154\u4155\u4156\u4157\u4158\u4159\u415a\u415b\u415c\u415d\u415e\u415f\u4160\u4161\u4162\u4163\u4164\u4165\u4166\u4167\u4168\u4169\u416a\u416b\u416c\u416d\u416e\u416f\u4170\u4171\u4172\u4173\u4174\u4175\u4176\u4177\u4178\u4179\u417a\u417b\u417c\u417d\u417e\u417f\u4180\u4181\u4182\u4183\u4184\u4185\u4186\u4187\u4188\u4189\u418a\u418b\u418c\u418d\u418e\u418f\u4190\u4191\u4192\u4193\u4194\u4195\u4196\u4197\u4198\u4199\u419a\u419b\u419c\u419d\u419e\u419f\u41a0\u41a1\u41a2\u41a3\u41a4\u41a5\u41a6\u41a7\u41a8\u41a9\u41aa\u41ab\u41ac\u41ad\u41ae\u41af\u41b0\u41b1\u41b2\u41b3\u41b4\u41b5\u41b6\u41b7\u41b8\u41b9\u41ba\u41bb\u41bc\u41bd\u41be\u41bf\u41c0\u41c1\u41c2\u41c3\u41c4\u41c5\u41c6\u41c7\u41c8\u41c9\u41ca\u41cb\u41cc\u41cd\u41ce\u41cf\u41d0\u41d1\u41d2\u41d3\u41d4\u41d5\u41d6\u41d7\u41d8\u41d9\u41da\u41db\u41dc\u41dd\u41de\u41df\u41e0\u41e1\u41e2\u41e3\u41e4\u41e5\u41e6\u41e7\u41e8\u41e9\u41ea\u41eb\u41ec\u41ed\u41ee\u41ef\u41f0\u41f1\u41f2\u41f3\u41f4\u41f5\u41f6\u41f7\u41f8\u41f9\u41fa\u41fb\u41fc\u41fd\u41fe\u41ff\u4200\u4201\u4202\u4203\u4204\u4205\u4206\u4207\u4208\u4209\u420a\u420b\u420c\u420d\u420e\u420f\u4210\u4211\u4212\u4213\u4214\u4215\u4216\u4217\u4218\u4219\u421a\u421b\u421c\u421d\u421e\u421f\u4220\u4221\u4222\u4223\u4224\u4225\u4226\u4227\u4228\u4229\u422a\u422b\u422c\u422d\u422e\u422f\u4230\u4231\u4232\u4233\u4234\u4235\u4236\u4237\u4238\u4239\u423a\u423b\u423c\u423d\u423e\u423f\u4240\u4241\u4242\u4243\u4244\u4245\u4246\u4247\u4248\u4249\u424a\u424b\u424c\u424d\u424e\u424f\u4250\u4251\u4252\u4253\u4254\u4255\u4256\u4257\u4258\u4259\u425a\u425b\u425c\u425d\u425e\u425f\u4260\u4261\u4262\u4263\u4264\u4265\u4266\u4267\u4268\u4269\u426a\u426b\u426c\u426d\u426e\u426f\u4270\u4271\u4272\u4273\u4274\u4275\u4276\u4277\u4278\u4279\u427a\u427b\u427c\u427d\u427e\u427f\u4280\u4281\u4282\u4283\u4284\u4285\u4286\u4287\u4288\u4289\u428a\u428b\u428c\u428d\u428e\u428f\u4290\u4291\u4292\u4293\u4294\u4295\u4296\u4297\u4298\u4299\u429a\u429b\u429c\u429d\u429e\u429f\u42a0\u42a1\u42a2\u42a3\u42a4\u42a5\u42a6\u42a7\u42a8\u42a9\u42aa\u42ab\u42ac\u42ad\u42ae\u42af\u42b0\u42b1\u42b2\u42b3\u42b4\u42b5\u42b6\u42b7\u42b8\u42b9\u42ba\u42bb\u42bc\u42bd\u42be\u42bf\u42c0\u42c1\u42c2\u42c3\u42c4\u42c5\u42c6\u42c7\u42c8\u42c9\u42ca\u42cb\u42cc\u42cd\u42ce\u42cf\u42d0\u42d1\u42d2\u42d3\u42d4\u42d5\u42d6\u42d7\u42d8\u42d9\u42da\u42db\u42dc\u42dd\u42de\u42df\u42e0\u42e1\u42e2\u42e3\u42e4\u42e5\u42e6\u42e7\u42e8\u42e9\u42ea\u42eb\u42ec\u42ed\u42ee\u42ef\u42f0\u42f1\u42f2\u42f3\u42f4\u42f5\u42f6\u42f7\u42f8\u42f9\u42fa\u42fb\u42fc\u42fd\u42fe\u42ff\u4300\u4301\u4302\u4303\u4304\u4305\u4306\u4307\u4308\u4309\u430a\u430b\u430c\u430d\u430e\u430f\u4310\u4311\u4312\u4313\u4314\u4315\u4316\u4317\u4318\u4319\u431a\u431b\u431c\u431d\u431e\u431f\u4320\u4321\u4322\u4323\u4324\u4325\u4326\u4327\u4328\u4329\u432a\u432b\u432c\u432d\u432e\u432f\u4330\u4331\u4332\u4333\u4334\u4335\u4336\u4337\u4338\u4339\u433a\u433b\u433c\u433d\u433e\u433f\u4340\u4341\u4342\u4343\u4344\u4345\u4346\u4347\u4348\u4349\u434a\u434b\u434c\u434d\u434e\u434f\u4350\u4351\u4352\u4353\u4354\u4355\u4356\u4357\u4358\u4359\u435a\u435b\u435c\u435d\u435e\u435f\u4360\u4361\u4362\u4363\u4364\u4365\u4366\u4367\u4368\u4369\u436a\u436b\u436c\u436d\u436e\u436f\u4370\u4371\u4372\u4373\u4374\u4375\u4376\u4377\u4378\u4379\u437a\u437b\u437c\u437d\u437e\u437f\u4380\u4381\u4382\u4383\u4384\u4385\u4386\u4387\u4388\u4389\u438a\u438b\u438c\u438d\u438e\u438f\u4390\u4391\u4392\u4393\u4394\u4395\u4396\u4397\u4398\u4399\u439a\u439b\u439c\u439d\u439e\u439f\u43a0\u43a1\u43a2\u43a3\u43a4\u43a5\u43a6\u43a7\u43a8\u43a9\u43aa\u43ab\u43ac\u43ad\u43ae\u43af\u43b0\u43b1\u43b2\u43b3\u43b4\u43b5\u43b6\u43b7\u43b8\u43b9\u43ba\u43bb\u43bc\u43bd\u43be\u43bf\u43c0\u43c1\u43c2\u43c3\u43c4\u43c5\u43c6\u43c7\u43c8\u43c9\u43ca\u43cb\u43cc\u43cd\u43ce\u43cf\u43d0\u43d1\u43d2\u43d3\u43d4\u43d5\u43d6\u43d7\u43d8\u43d9\u43da\u43db\u43dc\u43dd\u43de\u43df\u43e0\u43e1\u43e2\u43e3\u43e4\u43e5\u43e6\u43e7\u43e8\u43e9\u43ea\u43eb\u43ec\u43ed\u43ee\u43ef\u43f0\u43f1\u43f2\u43f3\u43f4\u43f5\u43f6\u43f7\u43f8\u43f9\u43fa\u43fb\u43fc\u43fd\u43fe\u43ff\u4400\u4401\u4402\u4403\u4404\u4405\u4406\u4407\u4408\u4409\u440a\u440b\u440c\u440d\u440e\u440f\u4410\u4411\u4412\u4413\u4414\u4415\u4416\u4417\u4418\u4419\u441a\u441b\u441c\u441d\u441e\u441f\u4420\u4421\u4422\u4423\u4424\u4425\u4426\u4427\u4428\u4429\u442a\u442b\u442c\u442d\u442e\u442f\u4430\u4431\u4432\u4433\u4434\u4435\u4436\u4437\u4438\u4439\u443a\u443b\u443c\u443d\u443e\u443f\u4440\u4441\u4442\u4443\u4444\u4445\u4446\u4447\u4448\u4449\u444a\u444b\u444c\u444d\u444e\u444f\u4450\u4451\u4452\u4453\u4454\u4455\u4456\u4457\u4458\u4459\u445a\u445b\u445c\u445d\u445e\u445f\u4460\u4461\u4462\u4463\u4464\u4465\u4466\u4467\u4468\u4469\u446a\u446b\u446c\u446d\u446e\u446f\u4470\u4471\u4472\u4473\u4474\u4475\u4476\u4477\u4478\u4479\u447a\u447b\u447c\u447d\u447e\u447f\u4480\u4481\u4482\u4483\u4484\u4485\u4486\u4487\u4488\u4489\u448a\u448b\u448c\u448d\u448e\u448f\u4490\u4491\u4492\u4493\u4494\u4495\u4496\u4497\u4498\u4499\u449a\u449b\u449c\u449d\u449e\u449f\u44a0\u44a1\u44a2\u44a3\u44a4\u44a5\u44a6\u44a7\u44a8\u44a9\u44aa\u44ab\u44ac\u44ad\u44ae\u44af\u44b0\u44b1\u44b2\u44b3\u44b4\u44b5\u44b6\u44b7\u44b8\u44b9\u44ba\u44bb\u44bc\u44bd\u44be\u44bf\u44c0\u44c1\u44c2\u44c3\u44c4\u44c5\u44c6\u44c7\u44c8\u44c9\u44ca\u44cb\u44cc\u44cd\u44ce\u44cf\u44d0\u44d1\u44d2\u44d3\u44d4\u44d5\u44d6\u44d7\u44d8\u44d9\u44da\u44db\u44dc\u44dd\u44de\u44df\u44e0\u44e1\u44e2\u44e3\u44e4\u44e5\u44e6\u44e7\u44e8\u44e9\u44ea\u44eb\u44ec\u44ed\u44ee\u44ef\u44f0\u44f1\u44f2\u44f3\u44f4\u44f5\u44f6\u44f7\u44f8\u44f9\u44fa\u44fb\u44fc\u44fd\u44fe\u44ff\u4500\u4501\u4502\u4503\u4504\u4505\u4506\u4507\u4508\u4509\u450a\u450b\u450c\u450d\u450e\u450f\u4510\u4511\u4512\u4513\u4514\u4515\u4516\u4517\u4518\u4519\u451a\u451b\u451c\u451d\u451e\u451f\u4520\u4521\u4522\u4523\u4524\u4525\u4526\u4527\u4528\u4529\u452a\u452b\u452c\u452d\u452e\u452f\u4530\u4531\u4532\u4533\u4534\u4535\u4536\u4537\u4538\u4539\u453a\u453b\u453c\u453d\u453e\u453f\u4540\u4541\u4542\u4543\u4544\u4545\u4546\u4547\u4548\u4549\u454a\u454b\u454c\u454d\u454e\u454f\u4550\u4551\u4552\u4553\u4554\u4555\u4556\u4557\u4558\u4559\u455a\u455b\u455c\u455d\u455e\u455f\u4560\u4561\u4562\u4563\u4564\u4565\u4566\u4567\u4568\u4569\u456a\u456b\u456c\u456d\u456e\u456f\u4570\u4571\u4572\u4573\u4574\u4575\u4576\u4577\u4578\u4579\u457a\u457b\u457c\u457d\u457e\u457f\u4580\u4581\u4582\u4583\u4584\u4585\u4586\u4587\u4588\u4589\u458a\u458b\u458c\u458d\u458e\u458f\u4590\u4591\u4592\u4593\u4594\u4595\u4596\u4597\u4598\u4599\u459a\u459b\u459c\u459d\u459e\u459f\u45a0\u45a1\u45a2\u45a3\u45a4\u45a5\u45a6\u45a7\u45a8\u45a9\u45aa\u45ab\u45ac\u45ad\u45ae\u45af\u45b0\u45b1\u45b2\u45b3\u45b4\u45b5\u45b6\u45b7\u45b8\u45b9\u45ba\u45bb\u45bc\u45bd\u45be\u45bf\u45c0\u45c1\u45c2\u45c3\u45c4\u45c5\u45c6\u45c7\u45c8\u45c9\u45ca\u45cb\u45cc\u45cd\u45ce\u45cf\u45d0\u45d1\u45d2\u45d3\u45d4\u45d5\u45d6\u45d7\u45d8\u45d9\u45da\u45db\u45dc\u45dd\u45de\u45df\u45e0\u45e1\u45e2\u45e3\u45e4\u45e5\u45e6\u45e7\u45e8\u45e9\u45ea\u45eb\u45ec\u45ed\u45ee\u45ef\u45f0\u45f1\u45f2\u45f3\u45f4\u45f5\u45f6\u45f7\u45f8\u45f9\u45fa\u45fb\u45fc\u45fd\u45fe\u45ff\u4600\u4601\u4602\u4603\u4604\u4605\u4606\u4607\u4608\u4609\u460a\u460b\u460c\u460d\u460e\u460f\u4610\u4611\u4612\u4613\u4614\u4615\u4616\u4617\u4618\u4619\u461a\u461b\u461c\u461d\u461e\u461f\u4620\u4621\u4622\u4623\u4624\u4625\u4626\u4627\u4628\u4629\u462a\u462b\u462c\u462d\u462e\u462f\u4630\u4631\u4632\u4633\u4634\u4635\u4636\u4637\u4638\u4639\u463a\u463b\u463c\u463d\u463e\u463f\u4640\u4641\u4642\u4643\u4644\u4645\u4646\u4647\u4648\u4649\u464a\u464b\u464c\u464d\u464e\u464f\u4650\u4651\u4652\u4653\u4654\u4655\u4656\u4657\u4658\u4659\u465a\u465b\u465c\u465d\u465e\u465f\u4660\u4661\u4662\u4663\u4664\u4665\u4666\u4667\u4668\u4669\u466a\u466b\u466c\u466d\u466e\u466f\u4670\u4671\u4672\u4673\u4674\u4675\u4676\u4677\u4678\u4679\u467a\u467b\u467c\u467d\u467e\u467f\u4680\u4681\u4682\u4683\u4684\u4685\u4686\u4687\u4688\u4689\u468a\u468b\u468c\u468d\u468e\u468f\u4690\u4691\u4692\u4693\u4694\u4695\u4696\u4697\u4698\u4699\u469a\u469b\u469c\u469d\u469e\u469f\u46a0\u46a1\u46a2\u46a3\u46a4\u46a5\u46a6\u46a7\u46a8\u46a9\u46aa\u46ab\u46ac\u46ad\u46ae\u46af\u46b0\u46b1\u46b2\u46b3\u46b4\u46b5\u46b6\u46b7\u46b8\u46b9\u46ba\u46bb\u46bc\u46bd\u46be\u46bf\u46c0\u46c1\u46c2\u46c3\u46c4\u46c5\u46c6\u46c7\u46c8\u46c9\u46ca\u46cb\u46cc\u46cd\u46ce\u46cf\u46d0\u46d1\u46d2\u46d3\u46d4\u46d5\u46d6\u46d7\u46d8\u46d9\u46da\u46db\u46dc\u46dd\u46de\u46df\u46e0\u46e1\u46e2\u46e3\u46e4\u46e5\u46e6\u46e7\u46e8\u46e9\u46ea\u46eb\u46ec\u46ed\u46ee\u46ef\u46f0\u46f1\u46f2\u46f3\u46f4\u46f5\u46f6\u46f7\u46f8\u46f9\u46fa\u46fb\u46fc\u46fd\u46fe\u46ff\u4700\u4701\u4702\u4703\u4704\u4705\u4706\u4707\u4708\u4709\u470a\u470b\u470c\u470d\u470e\u470f\u4710\u4711\u4712\u4713\u4714\u4715\u4716\u4717\u4718\u4719\u471a\u471b\u471c\u471d\u471e\u471f\u4720\u4721\u4722\u4723\u4724\u4725\u4726\u4727\u4728\u4729\u472a\u472b\u472c\u472d\u472e\u472f\u4730\u4731\u4732\u4733\u4734\u4735\u4736\u4737\u4738\u4739\u473a\u473b\u473c\u473d\u473e\u473f\u4740\u4741\u4742\u4743\u4744\u4745\u4746\u4747\u4748\u4749\u474a\u474b\u474c\u474d\u474e\u474f\u4750\u4751\u4752\u4753\u4754\u4755\u4756\u4757\u4758\u4759\u475a\u475b\u475c\u475d\u475e\u475f\u4760\u4761\u4762\u4763\u4764\u4765\u4766\u4767\u4768\u4769\u476a\u476b\u476c\u476d\u476e\u476f\u4770\u4771\u4772\u4773\u4774\u4775\u4776\u4777\u4778\u4779\u477a\u477b\u477c\u477d\u477e\u477f\u4780\u4781\u4782\u4783\u4784\u4785\u4786\u4787\u4788\u4789\u478a\u478b\u478c\u478d\u478e\u478f\u4790\u4791\u4792\u4793\u4794\u4795\u4796\u4797\u4798\u4799\u479a\u479b\u479c\u479d\u479e\u479f\u47a0\u47a1\u47a2\u47a3\u47a4\u47a5\u47a6\u47a7\u47a8\u47a9\u47aa\u47ab\u47ac\u47ad\u47ae\u47af\u47b0\u47b1\u47b2\u47b3\u47b4\u47b5\u47b6\u47b7\u47b8\u47b9\u47ba\u47bb\u47bc\u47bd\u47be\u47bf\u47c0\u47c1\u47c2\u47c3\u47c4\u47c5\u47c6\u47c7\u47c8\u47c9\u47ca\u47cb\u47cc\u47cd\u47ce\u47cf\u47d0\u47d1\u47d2\u47d3\u47d4\u47d5\u47d6\u47d7\u47d8\u47d9\u47da\u47db\u47dc\u47dd\u47de\u47df\u47e0\u47e1\u47e2\u47e3\u47e4\u47e5\u47e6\u47e7\u47e8\u47e9\u47ea\u47eb\u47ec\u47ed\u47ee\u47ef\u47f0\u47f1\u47f2\u47f3\u47f4\u47f5\u47f6\u47f7\u47f8\u47f9\u47fa\u47fb\u47fc\u47fd\u47fe\u47ff\u4800\u4801\u4802\u4803\u4804\u4805\u4806\u4807\u4808\u4809\u480a\u480b\u480c\u480d\u480e\u480f\u4810\u4811\u4812\u4813\u4814\u4815\u4816\u4817\u4818\u4819\u481a\u481b\u481c\u481d\u481e\u481f\u4820\u4821\u4822\u4823\u4824\u4825\u4826\u4827\u4828\u4829\u482a\u482b\u482c\u482d\u482e\u482f\u4830\u4831\u4832\u4833\u4834\u4835\u4836\u4837\u4838\u4839\u483a\u483b\u483c\u483d\u483e\u483f\u4840\u4841\u4842\u4843\u4844\u4845\u4846\u4847\u4848\u4849\u484a\u484b\u484c\u484d\u484e\u484f\u4850\u4851\u4852\u4853\u4854\u4855\u4856\u4857\u4858\u4859\u485a\u485b\u485c\u485d\u485e\u485f\u4860\u4861\u4862\u4863\u4864\u4865\u4866\u4867\u4868\u4869\u486a\u486b\u486c\u486d\u486e\u486f\u4870\u4871\u4872\u4873\u4874\u4875\u4876\u4877\u4878\u4879\u487a\u487b\u487c\u487d\u487e\u487f\u4880\u4881\u4882\u4883\u4884\u4885\u4886\u4887\u4888\u4889\u488a\u488b\u488c\u488d\u488e\u488f\u4890\u4891\u4892\u4893\u4894\u4895\u4896\u4897\u4898\u4899\u489a\u489b\u489c\u489d\u489e\u489f\u48a0\u48a1\u48a2\u48a3\u48a4\u48a5\u48a6\u48a7\u48a8\u48a9\u48aa\u48ab\u48ac\u48ad\u48ae\u48af\u48b0\u48b1\u48b2\u48b3\u48b4\u48b5\u48b6\u48b7\u48b8\u48b9\u48ba\u48bb\u48bc\u48bd\u48be\u48bf\u48c0\u48c1\u48c2\u48c3\u48c4\u48c5\u48c6\u48c7\u48c8\u48c9\u48ca\u48cb\u48cc\u48cd\u48ce\u48cf\u48d0\u48d1\u48d2\u48d3\u48d4\u48d5\u48d6\u48d7\u48d8\u48d9\u48da\u48db\u48dc\u48dd\u48de\u48df\u48e0\u48e1\u48e2\u48e3\u48e4\u48e5\u48e6\u48e7\u48e8\u48e9\u48ea\u48eb\u48ec\u48ed\u48ee\u48ef\u48f0\u48f1\u48f2\u48f3\u48f4\u48f5\u48f6\u48f7\u48f8\u48f9\u48fa\u48fb\u48fc\u48fd\u48fe\u48ff\u4900\u4901\u4902\u4903\u4904\u4905\u4906\u4907\u4908\u4909\u490a\u490b\u490c\u490d\u490e\u490f\u4910\u4911\u4912\u4913\u4914\u4915\u4916\u4917\u4918\u4919\u491a\u491b\u491c\u491d\u491e\u491f\u4920\u4921\u4922\u4923\u4924\u4925\u4926\u4927\u4928\u4929\u492a\u492b\u492c\u492d\u492e\u492f\u4930\u4931\u4932\u4933\u4934\u4935\u4936\u4937\u4938\u4939\u493a\u493b\u493c\u493d\u493e\u493f\u4940\u4941\u4942\u4943\u4944\u4945\u4946\u4947\u4948\u4949\u494a\u494b\u494c\u494d\u494e\u494f\u4950\u4951\u4952\u4953\u4954\u4955\u4956\u4957\u4958\u4959\u495a\u495b\u495c\u495d\u495e\u495f\u4960\u4961\u4962\u4963\u4964\u4965\u4966\u4967\u4968\u4969\u496a\u496b\u496c\u496d\u496e\u496f\u4970\u4971\u4972\u4973\u4974\u4975\u4976\u4977\u4978\u4979\u497a\u497b\u497c\u497d\u497e\u497f\u4980\u4981\u4982\u4983\u4984\u4985\u4986\u4987\u4988\u4989\u498a\u498b\u498c\u498d\u498e\u498f\u4990\u4991\u4992\u4993\u4994\u4995\u4996\u4997\u4998\u4999\u499a\u499b\u499c\u499d\u499e\u499f\u49a0\u49a1\u49a2\u49a3\u49a4\u49a5\u49a6\u49a7\u49a8\u49a9\u49aa\u49ab\u49ac\u49ad\u49ae\u49af\u49b0\u49b1\u49b2\u49b3\u49b4\u49b5\u49b6\u49b7\u49b8\u49b9\u49ba\u49bb\u49bc\u49bd\u49be\u49bf\u49c0\u49c1\u49c2\u49c3\u49c4\u49c5\u49c6\u49c7\u49c8\u49c9\u49ca\u49cb\u49cc\u49cd\u49ce\u49cf\u49d0\u49d1\u49d2\u49d3\u49d4\u49d5\u49d6\u49d7\u49d8\u49d9\u49da\u49db\u49dc\u49dd\u49de\u49df\u49e0\u49e1\u49e2\u49e3\u49e4\u49e5\u49e6\u49e7\u49e8\u49e9\u49ea\u49eb\u49ec\u49ed\u49ee\u49ef\u49f0\u49f1\u49f2\u49f3\u49f4\u49f5\u49f6\u49f7\u49f8\u49f9\u49fa\u49fb\u49fc\u49fd\u49fe\u49ff\u4a00\u4a01\u4a02\u4a03\u4a04\u4a05\u4a06\u4a07\u4a08\u4a09\u4a0a\u4a0b\u4a0c\u4a0d\u4a0e\u4a0f\u4a10\u4a11\u4a12\u4a13\u4a14\u4a15\u4a16\u4a17\u4a18\u4a19\u4a1a\u4a1b\u4a1c\u4a1d\u4a1e\u4a1f\u4a20\u4a21\u4a22\u4a23\u4a24\u4a25\u4a26\u4a27\u4a28\u4a29\u4a2a\u4a2b\u4a2c\u4a2d\u4a2e\u4a2f\u4a30\u4a31\u4a32\u4a33\u4a34\u4a35\u4a36\u4a37\u4a38\u4a39\u4a3a\u4a3b\u4a3c\u4a3d\u4a3e\u4a3f\u4a40\u4a41\u4a42\u4a43\u4a44\u4a45\u4a46\u4a47\u4a48\u4a49\u4a4a\u4a4b\u4a4c\u4a4d\u4a4e\u4a4f\u4a50\u4a51\u4a52\u4a53\u4a54\u4a55\u4a56\u4a57\u4a58\u4a59\u4a5a\u4a5b\u4a5c\u4a5d\u4a5e\u4a5f\u4a60\u4a61\u4a62\u4a63\u4a64\u4a65\u4a66\u4a67\u4a68\u4a69\u4a6a\u4a6b\u4a6c\u4a6d\u4a6e\u4a6f\u4a70\u4a71\u4a72\u4a73\u4a74\u4a75\u4a76\u4a77\u4a78\u4a79\u4a7a\u4a7b\u4a7c\u4a7d\u4a7e\u4a7f\u4a80\u4a81\u4a82\u4a83\u4a84\u4a85\u4a86\u4a87\u4a88\u4a89\u4a8a\u4a8b\u4a8c\u4a8d\u4a8e\u4a8f\u4a90\u4a91\u4a92\u4a93\u4a94\u4a95\u4a96\u4a97\u4a98\u4a99\u4a9a\u4a9b\u4a9c\u4a9d\u4a9e\u4a9f\u4aa0\u4aa1\u4aa2\u4aa3\u4aa4\u4aa5\u4aa6\u4aa7\u4aa8\u4aa9\u4aaa\u4aab\u4aac\u4aad\u4aae\u4aaf\u4ab0\u4ab1\u4ab2\u4ab3\u4ab4\u4ab5\u4ab6\u4ab7\u4ab8\u4ab9\u4aba\u4abb\u4abc\u4abd\u4abe\u4abf\u4ac0\u4ac1\u4ac2\u4ac3\u4ac4\u4ac5\u4ac6\u4ac7\u4ac8\u4ac9\u4aca\u4acb\u4acc\u4acd\u4ace\u4acf\u4ad0\u4ad1\u4ad2\u4ad3\u4ad4\u4ad5\u4ad6\u4ad7\u4ad8\u4ad9\u4ada\u4adb\u4adc\u4add\u4ade\u4adf\u4ae0\u4ae1\u4ae2\u4ae3\u4ae4\u4ae5\u4ae6\u4ae7\u4ae8\u4ae9\u4aea\u4aeb\u4aec\u4aed\u4aee\u4aef\u4af0\u4af1\u4af2\u4af3\u4af4\u4af5\u4af6\u4af7\u4af8\u4af9\u4afa\u4afb\u4afc\u4afd\u4afe\u4aff\u4b00\u4b01\u4b02\u4b03\u4b04\u4b05\u4b06\u4b07\u4b08\u4b09\u4b0a\u4b0b\u4b0c\u4b0d\u4b0e\u4b0f\u4b10\u4b11\u4b12\u4b13\u4b14\u4b15\u4b16\u4b17\u4b18\u4b19\u4b1a\u4b1b\u4b1c\u4b1d\u4b1e\u4b1f\u4b20\u4b21\u4b22\u4b23\u4b24\u4b25\u4b26\u4b27\u4b28\u4b29\u4b2a\u4b2b\u4b2c\u4b2d\u4b2e\u4b2f\u4b30\u4b31\u4b32\u4b33\u4b34\u4b35\u4b36\u4b37\u4b38\u4b39\u4b3a\u4b3b\u4b3c\u4b3d\u4b3e\u4b3f\u4b40\u4b41\u4b42\u4b43\u4b44\u4b45\u4b46\u4b47\u4b48\u4b49\u4b4a\u4b4b\u4b4c\u4b4d\u4b4e\u4b4f\u4b50\u4b51\u4b52\u4b53\u4b54\u4b55\u4b56\u4b57\u4b58\u4b59\u4b5a\u4b5b\u4b5c\u4b5d\u4b5e\u4b5f\u4b60\u4b61\u4b62\u4b63\u4b64\u4b65\u4b66\u4b67\u4b68\u4b69\u4b6a\u4b6b\u4b6c\u4b6d\u4b6e\u4b6f\u4b70\u4b71\u4b72\u4b73\u4b74\u4b75\u4b76\u4b77\u4b78\u4b79\u4b7a\u4b7b\u4b7c\u4b7d\u4b7e\u4b7f\u4b80\u4b81\u4b82\u4b83\u4b84\u4b85\u4b86\u4b87\u4b88\u4b89\u4b8a\u4b8b\u4b8c\u4b8d\u4b8e\u4b8f\u4b90\u4b91\u4b92\u4b93\u4b94\u4b95\u4b96\u4b97\u4b98\u4b99\u4b9a\u4b9b\u4b9c\u4b9d\u4b9e\u4b9f\u4ba0\u4ba1\u4ba2\u4ba3\u4ba4\u4ba5\u4ba6\u4ba7\u4ba8\u4ba9\u4baa\u4bab\u4bac\u4bad\u4bae\u4baf\u4bb0\u4bb1\u4bb2\u4bb3\u4bb4\u4bb5\u4bb6\u4bb7\u4bb8\u4bb9\u4bba\u4bbb\u4bbc\u4bbd\u4bbe\u4bbf\u4bc0\u4bc1\u4bc2\u4bc3\u4bc4\u4bc5\u4bc6\u4bc7\u4bc8\u4bc9\u4bca\u4bcb\u4bcc\u4bcd\u4bce\u4bcf\u4bd0\u4bd1\u4bd2\u4bd3\u4bd4\u4bd5\u4bd6\u4bd7\u4bd8\u4bd9\u4bda\u4bdb\u4bdc\u4bdd\u4bde\u4bdf\u4be0\u4be1\u4be2\u4be3\u4be4\u4be5\u4be6\u4be7\u4be8\u4be9\u4bea\u4beb\u4bec\u4bed\u4bee\u4bef\u4bf0\u4bf1\u4bf2\u4bf3\u4bf4\u4bf5\u4bf6\u4bf7\u4bf8\u4bf9\u4bfa\u4bfb\u4bfc\u4bfd\u4bfe\u4bff\u4c00\u4c01\u4c02\u4c03\u4c04\u4c05\u4c06\u4c07\u4c08\u4c09\u4c0a\u4c0b\u4c0c\u4c0d\u4c0e\u4c0f\u4c10\u4c11\u4c12\u4c13\u4c14\u4c15\u4c16\u4c17\u4c18\u4c19\u4c1a\u4c1b\u4c1c\u4c1d\u4c1e\u4c1f\u4c20\u4c21\u4c22\u4c23\u4c24\u4c25\u4c26\u4c27\u4c28\u4c29\u4c2a\u4c2b\u4c2c\u4c2d\u4c2e\u4c2f\u4c30\u4c31\u4c32\u4c33\u4c34\u4c35\u4c36\u4c37\u4c38\u4c39\u4c3a\u4c3b\u4c3c\u4c3d\u4c3e\u4c3f\u4c40\u4c41\u4c42\u4c43\u4c44\u4c45\u4c46\u4c47\u4c48\u4c49\u4c4a\u4c4b\u4c4c\u4c4d\u4c4e\u4c4f\u4c50\u4c51\u4c52\u4c53\u4c54\u4c55\u4c56\u4c57\u4c58\u4c59\u4c5a\u4c5b\u4c5c\u4c5d\u4c5e\u4c5f\u4c60\u4c61\u4c62\u4c63\u4c64\u4c65\u4c66\u4c67\u4c68\u4c69\u4c6a\u4c6b\u4c6c\u4c6d\u4c6e\u4c6f\u4c70\u4c71\u4c72\u4c73\u4c74\u4c75\u4c76\u4c77\u4c78\u4c79\u4c7a\u4c7b\u4c7c\u4c7d\u4c7e\u4c7f\u4c80\u4c81\u4c82\u4c83\u4c84\u4c85\u4c86\u4c87\u4c88\u4c89\u4c8a\u4c8b\u4c8c\u4c8d\u4c8e\u4c8f\u4c90\u4c91\u4c92\u4c93\u4c94\u4c95\u4c96\u4c97\u4c98\u4c99\u4c9a\u4c9b\u4c9c\u4c9d\u4c9e\u4c9f\u4ca0\u4ca1\u4ca2\u4ca3\u4ca4\u4ca5\u4ca6\u4ca7\u4ca8\u4ca9\u4caa\u4cab\u4cac\u4cad\u4cae\u4caf\u4cb0\u4cb1\u4cb2\u4cb3\u4cb4\u4cb5\u4cb6\u4cb7\u4cb8\u4cb9\u4cba\u4cbb\u4cbc\u4cbd\u4cbe\u4cbf\u4cc0\u4cc1\u4cc2\u4cc3\u4cc4\u4cc5\u4cc6\u4cc7\u4cc8\u4cc9\u4cca\u4ccb\u4ccc\u4ccd\u4cce\u4ccf\u4cd0\u4cd1\u4cd2\u4cd3\u4cd4\u4cd5\u4cd6\u4cd7\u4cd8\u4cd9\u4cda\u4cdb\u4cdc\u4cdd\u4cde\u4cdf\u4ce0\u4ce1\u4ce2\u4ce3\u4ce4\u4ce5\u4ce6\u4ce7\u4ce8\u4ce9\u4cea\u4ceb\u4cec\u4ced\u4cee\u4cef\u4cf0\u4cf1\u4cf2\u4cf3\u4cf4\u4cf5\u4cf6\u4cf7\u4cf8\u4cf9\u4cfa\u4cfb\u4cfc\u4cfd\u4cfe\u4cff\u4d00\u4d01\u4d02\u4d03\u4d04\u4d05\u4d06\u4d07\u4d08\u4d09\u4d0a\u4d0b\u4d0c\u4d0d\u4d0e\u4d0f\u4d10\u4d11\u4d12\u4d13\u4d14\u4d15\u4d16\u4d17\u4d18\u4d19\u4d1a\u4d1b\u4d1c\u4d1d\u4d1e\u4d1f\u4d20\u4d21\u4d22\u4d23\u4d24\u4d25\u4d26\u4d27\u4d28\u4d29\u4d2a\u4d2b\u4d2c\u4d2d\u4d2e\u4d2f\u4d30\u4d31\u4d32\u4d33\u4d34\u4d35\u4d36\u4d37\u4d38\u4d39\u4d3a\u4d3b\u4d3c\u4d3d\u4d3e\u4d3f\u4d40\u4d41\u4d42\u4d43\u4d44\u4d45\u4d46\u4d47\u4d48\u4d49\u4d4a\u4d4b\u4d4c\u4d4d\u4d4e\u4d4f\u4d50\u4d51\u4d52\u4d53\u4d54\u4d55\u4d56\u4d57\u4d58\u4d59\u4d5a\u4d5b\u4d5c\u4d5d\u4d5e\u4d5f\u4d60\u4d61\u4d62\u4d63\u4d64\u4d65\u4d66\u4d67\u4d68\u4d69\u4d6a\u4d6b\u4d6c\u4d6d\u4d6e\u4d6f\u4d70\u4d71\u4d72\u4d73\u4d74\u4d75\u4d76\u4d77\u4d78\u4d79\u4d7a\u4d7b\u4d7c\u4d7d\u4d7e\u4d7f\u4d80\u4d81\u4d82\u4d83\u4d84\u4d85\u4d86\u4d87\u4d88\u4d89\u4d8a\u4d8b\u4d8c\u4d8d\u4d8e\u4d8f\u4d90\u4d91\u4d92\u4d93\u4d94\u4d95\u4d96\u4d97\u4d98\u4d99\u4d9a\u4d9b\u4d9c\u4d9d\u4d9e\u4d9f\u4da0\u4da1\u4da2\u4da3\u4da4\u4da5\u4da6\u4da7\u4da8\u4da9\u4daa\u4dab\u4dac\u4dad\u4dae\u4daf\u4db0\u4db1\u4db2\u4db3\u4db4\u4db5\u4e00\u4e01\u4e02\u4e03\u4e04\u4e05\u4e06\u4e07\u4e08\u4e09\u4e0a\u4e0b\u4e0c\u4e0d\u4e0e\u4e0f\u4e10\u4e11\u4e12\u4e13\u4e14\u4e15\u4e16\u4e17\u4e18\u4e19\u4e1a\u4e1b\u4e1c\u4e1d\u4e1e\u4e1f\u4e20\u4e21\u4e22\u4e23\u4e24\u4e25\u4e26\u4e27\u4e28\u4e29\u4e2a\u4e2b\u4e2c\u4e2d\u4e2e\u4e2f\u4e30\u4e31\u4e32\u4e33\u4e34\u4e35\u4e36\u4e37\u4e38\u4e39\u4e3a\u4e3b\u4e3c\u4e3d\u4e3e\u4e3f\u4e40\u4e41\u4e42\u4e43\u4e44\u4e45\u4e46\u4e47\u4e48\u4e49\u4e4a\u4e4b\u4e4c\u4e4d\u4e4e\u4e4f\u4e50\u4e51\u4e52\u4e53\u4e54\u4e55\u4e56\u4e57\u4e58\u4e59\u4e5a\u4e5b\u4e5c\u4e5d\u4e5e\u4e5f\u4e60\u4e61\u4e62\u4e63\u4e64\u4e65\u4e66\u4e67\u4e68\u4e69\u4e6a\u4e6b\u4e6c\u4e6d\u4e6e\u4e6f\u4e70\u4e71\u4e72\u4e73\u4e74\u4e75\u4e76\u4e77\u4e78\u4e79\u4e7a\u4e7b\u4e7c\u4e7d\u4e7e\u4e7f\u4e80\u4e81\u4e82\u4e83\u4e84\u4e85\u4e86\u4e87\u4e88\u4e89\u4e8a\u4e8b\u4e8c\u4e8d\u4e8e\u4e8f\u4e90\u4e91\u4e92\u4e93\u4e94\u4e95\u4e96\u4e97\u4e98\u4e99\u4e9a\u4e9b\u4e9c\u4e9d\u4e9e\u4e9f\u4ea0\u4ea1\u4ea2\u4ea3\u4ea4\u4ea5\u4ea6\u4ea7\u4ea8\u4ea9\u4eaa\u4eab\u4eac\u4ead\u4eae\u4eaf\u4eb0\u4eb1\u4eb2\u4eb3\u4eb4\u4eb5\u4eb6\u4eb7\u4eb8\u4eb9\u4eba\u4ebb\u4ebc\u4ebd\u4ebe\u4ebf\u4ec0\u4ec1\u4ec2\u4ec3\u4ec4\u4ec5\u4ec6\u4ec7\u4ec8\u4ec9\u4eca\u4ecb\u4ecc\u4ecd\u4ece\u4ecf\u4ed0\u4ed1\u4ed2\u4ed3\u4ed4\u4ed5\u4ed6\u4ed7\u4ed8\u4ed9\u4eda\u4edb\u4edc\u4edd\u4ede\u4edf\u4ee0\u4ee1\u4ee2\u4ee3\u4ee4\u4ee5\u4ee6\u4ee7\u4ee8\u4ee9\u4eea\u4eeb\u4eec\u4eed\u4eee\u4eef\u4ef0\u4ef1\u4ef2\u4ef3\u4ef4\u4ef5\u4ef6\u4ef7\u4ef8\u4ef9\u4efa\u4efb\u4efc\u4efd\u4efe\u4eff\u4f00\u4f01\u4f02\u4f03\u4f04\u4f05\u4f06\u4f07\u4f08\u4f09\u4f0a\u4f0b\u4f0c\u4f0d\u4f0e\u4f0f\u4f10\u4f11\u4f12\u4f13\u4f14\u4f15\u4f16\u4f17\u4f18\u4f19\u4f1a\u4f1b\u4f1c\u4f1d\u4f1e\u4f1f\u4f20\u4f21\u4f22\u4f23\u4f24\u4f25\u4f26\u4f27\u4f28\u4f29\u4f2a\u4f2b\u4f2c\u4f2d\u4f2e\u4f2f\u4f30\u4f31\u4f32\u4f33\u4f34\u4f35\u4f36\u4f37\u4f38\u4f39\u4f3a\u4f3b\u4f3c\u4f3d\u4f3e\u4f3f\u4f40\u4f41\u4f42\u4f43\u4f44\u4f45\u4f46\u4f47\u4f48\u4f49\u4f4a\u4f4b\u4f4c\u4f4d\u4f4e\u4f4f\u4f50\u4f51\u4f52\u4f53\u4f54\u4f55\u4f56\u4f57\u4f58\u4f59\u4f5a\u4f5b\u4f5c\u4f5d\u4f5e\u4f5f\u4f60\u4f61\u4f62\u4f63\u4f64\u4f65\u4f66\u4f67\u4f68\u4f69\u4f6a\u4f6b\u4f6c\u4f6d\u4f6e\u4f6f\u4f70\u4f71\u4f72\u4f73\u4f74\u4f75\u4f76\u4f77\u4f78\u4f79\u4f7a\u4f7b\u4f7c\u4f7d\u4f7e\u4f7f\u4f80\u4f81\u4f82\u4f83\u4f84\u4f85\u4f86\u4f87\u4f88\u4f89\u4f8a\u4f8b\u4f8c\u4f8d\u4f8e\u4f8f\u4f90\u4f91\u4f92\u4f93\u4f94\u4f95\u4f96\u4f97\u4f98\u4f99\u4f9a\u4f9b\u4f9c\u4f9d\u4f9e\u4f9f\u4fa0\u4fa1\u4fa2\u4fa3\u4fa4\u4fa5\u4fa6\u4fa7\u4fa8\u4fa9\u4faa\u4fab\u4fac\u4fad\u4fae\u4faf\u4fb0\u4fb1\u4fb2\u4fb3\u4fb4\u4fb5\u4fb6\u4fb7\u4fb8\u4fb9\u4fba\u4fbb\u4fbc\u4fbd\u4fbe\u4fbf\u4fc0\u4fc1\u4fc2\u4fc3\u4fc4\u4fc5\u4fc6\u4fc7\u4fc8\u4fc9\u4fca\u4fcb\u4fcc\u4fcd\u4fce\u4fcf\u4fd0\u4fd1\u4fd2\u4fd3\u4fd4\u4fd5\u4fd6\u4fd7\u4fd8\u4fd9\u4fda\u4fdb\u4fdc\u4fdd\u4fde\u4fdf\u4fe0\u4fe1\u4fe2\u4fe3\u4fe4\u4fe5\u4fe6\u4fe7\u4fe8\u4fe9\u4fea\u4feb\u4fec\u4fed\u4fee\u4fef\u4ff0\u4ff1\u4ff2\u4ff3\u4ff4\u4ff5\u4ff6\u4ff7\u4ff8\u4ff9\u4ffa\u4ffb\u4ffc\u4ffd\u4ffe\u4fff\u5000\u5001\u5002\u5003\u5004\u5005\u5006\u5007\u5008\u5009\u500a\u500b\u500c\u500d\u500e\u500f\u5010\u5011\u5012\u5013\u5014\u5015\u5016\u5017\u5018\u5019\u501a\u501b\u501c\u501d\u501e\u501f\u5020\u5021\u5022\u5023\u5024\u5025\u5026\u5027\u5028\u5029\u502a\u502b\u502c\u502d\u502e\u502f\u5030\u5031\u5032\u5033\u5034\u5035\u5036\u5037\u5038\u5039\u503a\u503b\u503c\u503d\u503e\u503f\u5040\u5041\u5042\u5043\u5044\u5045\u5046\u5047\u5048\u5049\u504a\u504b\u504c\u504d\u504e\u504f\u5050\u5051\u5052\u5053\u5054\u5055\u5056\u5057\u5058\u5059\u505a\u505b\u505c\u505d\u505e\u505f\u5060\u5061\u5062\u5063\u5064\u5065\u5066\u5067\u5068\u5069\u506a\u506b\u506c\u506d\u506e\u506f\u5070\u5071\u5072\u5073\u5074\u5075\u5076\u5077\u5078\u5079\u507a\u507b\u507c\u507d\u507e\u507f\u5080\u5081\u5082\u5083\u5084\u5085\u5086\u5087\u5088\u5089\u508a\u508b\u508c\u508d\u508e\u508f\u5090\u5091\u5092\u5093\u5094\u5095\u5096\u5097\u5098\u5099\u509a\u509b\u509c\u509d\u509e\u509f\u50a0\u50a1\u50a2\u50a3\u50a4\u50a5\u50a6\u50a7\u50a8\u50a9\u50aa\u50ab\u50ac\u50ad\u50ae\u50af\u50b0\u50b1\u50b2\u50b3\u50b4\u50b5\u50b6\u50b7\u50b8\u50b9\u50ba\u50bb\u50bc\u50bd\u50be\u50bf\u50c0\u50c1\u50c2\u50c3\u50c4\u50c5\u50c6\u50c7\u50c8\u50c9\u50ca\u50cb\u50cc\u50cd\u50ce\u50cf\u50d0\u50d1\u50d2\u50d3\u50d4\u50d5\u50d6\u50d7\u50d8\u50d9\u50da\u50db\u50dc\u50dd\u50de\u50df\u50e0\u50e1\u50e2\u50e3\u50e4\u50e5\u50e6\u50e7\u50e8\u50e9\u50ea\u50eb\u50ec\u50ed\u50ee\u50ef\u50f0\u50f1\u50f2\u50f3\u50f4\u50f5\u50f6\u50f7\u50f8\u50f9\u50fa\u50fb\u50fc\u50fd\u50fe\u50ff\u5100\u5101\u5102\u5103\u5104\u5105\u5106\u5107\u5108\u5109\u510a\u510b\u510c\u510d\u510e\u510f\u5110\u5111\u5112\u5113\u5114\u5115\u5116\u5117\u5118\u5119\u511a\u511b\u511c\u511d\u511e\u511f\u5120\u5121\u5122\u5123\u5124\u5125\u5126\u5127\u5128\u5129\u512a\u512b\u512c\u512d\u512e\u512f\u5130\u5131\u5132\u5133\u5134\u5135\u5136\u5137\u5138\u5139\u513a\u513b\u513c\u513d\u513e\u513f\u5140\u5141\u5142\u5143\u5144\u5145\u5146\u5147\u5148\u5149\u514a\u514b\u514c\u514d\u514e\u514f\u5150\u5151\u5152\u5153\u5154\u5155\u5156\u5157\u5158\u5159\u515a\u515b\u515c\u515d\u515e\u515f\u5160\u5161\u5162\u5163\u5164\u5165\u5166\u5167\u5168\u5169\u516a\u516b\u516c\u516d\u516e\u516f\u5170\u5171\u5172\u5173\u5174\u5175\u5176\u5177\u5178\u5179\u517a\u517b\u517c\u517d\u517e\u517f\u5180\u5181\u5182\u5183\u5184\u5185\u5186\u5187\u5188\u5189\u518a\u518b\u518c\u518d\u518e\u518f\u5190\u5191\u5192\u5193\u5194\u5195\u5196\u5197\u5198\u5199\u519a\u519b\u519c\u519d\u519e\u519f\u51a0\u51a1\u51a2\u51a3\u51a4\u51a5\u51a6\u51a7\u51a8\u51a9\u51aa\u51ab\u51ac\u51ad\u51ae\u51af\u51b0\u51b1\u51b2\u51b3\u51b4\u51b5\u51b6\u51b7\u51b8\u51b9\u51ba\u51bb\u51bc\u51bd\u51be\u51bf\u51c0\u51c1\u51c2\u51c3\u51c4\u51c5\u51c6\u51c7\u51c8\u51c9\u51ca\u51cb\u51cc\u51cd\u51ce\u51cf\u51d0\u51d1\u51d2\u51d3\u51d4\u51d5\u51d6\u51d7\u51d8\u51d9\u51da\u51db\u51dc\u51dd\u51de\u51df\u51e0\u51e1\u51e2\u51e3\u51e4\u51e5\u51e6\u51e7\u51e8\u51e9\u51ea\u51eb\u51ec\u51ed\u51ee\u51ef\u51f0\u51f1\u51f2\u51f3\u51f4\u51f5\u51f6\u51f7\u51f8\u51f9\u51fa\u51fb\u51fc\u51fd\u51fe\u51ff\u5200\u5201\u5202\u5203\u5204\u5205\u5206\u5207\u5208\u5209\u520a\u520b\u520c\u520d\u520e\u520f\u5210\u5211\u5212\u5213\u5214\u5215\u5216\u5217\u5218\u5219\u521a\u521b\u521c\u521d\u521e\u521f\u5220\u5221\u5222\u5223\u5224\u5225\u5226\u5227\u5228\u5229\u522a\u522b\u522c\u522d\u522e\u522f\u5230\u5231\u5232\u5233\u5234\u5235\u5236\u5237\u5238\u5239\u523a\u523b\u523c\u523d\u523e\u523f\u5240\u5241\u5242\u5243\u5244\u5245\u5246\u5247\u5248\u5249\u524a\u524b\u524c\u524d\u524e\u524f\u5250\u5251\u5252\u5253\u5254\u5255\u5256\u5257\u5258\u5259\u525a\u525b\u525c\u525d\u525e\u525f\u5260\u5261\u5262\u5263\u5264\u5265\u5266\u5267\u5268\u5269\u526a\u526b\u526c\u526d\u526e\u526f\u5270\u5271\u5272\u5273\u5274\u5275\u5276\u5277\u5278\u5279\u527a\u527b\u527c\u527d\u527e\u527f\u5280\u5281\u5282\u5283\u5284\u5285\u5286\u5287\u5288\u5289\u528a\u528b\u528c\u528d\u528e\u528f\u5290\u5291\u5292\u5293\u5294\u5295\u5296\u5297\u5298\u5299\u529a\u529b\u529c\u529d\u529e\u529f\u52a0\u52a1\u52a2\u52a3\u52a4\u52a5\u52a6\u52a7\u52a8\u52a9\u52aa\u52ab\u52ac\u52ad\u52ae\u52af\u52b0\u52b1\u52b2\u52b3\u52b4\u52b5\u52b6\u52b7\u52b8\u52b9\u52ba\u52bb\u52bc\u52bd\u52be\u52bf\u52c0\u52c1\u52c2\u52c3\u52c4\u52c5\u52c6\u52c7\u52c8\u52c9\u52ca\u52cb\u52cc\u52cd\u52ce\u52cf\u52d0\u52d1\u52d2\u52d3\u52d4\u52d5\u52d6\u52d7\u52d8\u52d9\u52da\u52db\u52dc\u52dd\u52de\u52df\u52e0\u52e1\u52e2\u52e3\u52e4\u52e5\u52e6\u52e7\u52e8\u52e9\u52ea\u52eb\u52ec\u52ed\u52ee\u52ef\u52f0\u52f1\u52f2\u52f3\u52f4\u52f5\u52f6\u52f7\u52f8\u52f9\u52fa\u52fb\u52fc\u52fd\u52fe\u52ff\u5300\u5301\u5302\u5303\u5304\u5305\u5306\u5307\u5308\u5309\u530a\u530b\u530c\u530d\u530e\u530f\u5310\u5311\u5312\u5313\u5314\u5315\u5316\u5317\u5318\u5319\u531a\u531b\u531c\u531d\u531e\u531f\u5320\u5321\u5322\u5323\u5324\u5325\u5326\u5327\u5328\u5329\u532a\u532b\u532c\u532d\u532e\u532f\u5330\u5331\u5332\u5333\u5334\u5335\u5336\u5337\u5338\u5339\u533a\u533b\u533c\u533d\u533e\u533f\u5340\u5341\u5342\u5343\u5344\u5345\u5346\u5347\u5348\u5349\u534a\u534b\u534c\u534d\u534e\u534f\u5350\u5351\u5352\u5353\u5354\u5355\u5356\u5357\u5358\u5359\u535a\u535b\u535c\u535d\u535e\u535f\u5360\u5361\u5362\u5363\u5364\u5365\u5366\u5367\u5368\u5369\u536a\u536b\u536c\u536d\u536e\u536f\u5370\u5371\u5372\u5373\u5374\u5375\u5376\u5377\u5378\u5379\u537a\u537b\u537c\u537d\u537e\u537f\u5380\u5381\u5382\u5383\u5384\u5385\u5386\u5387\u5388\u5389\u538a\u538b\u538c\u538d\u538e\u538f\u5390\u5391\u5392\u5393\u5394\u5395\u5396\u5397\u5398\u5399\u539a\u539b\u539c\u539d\u539e\u539f\u53a0\u53a1\u53a2\u53a3\u53a4\u53a5\u53a6\u53a7\u53a8\u53a9\u53aa\u53ab\u53ac\u53ad\u53ae\u53af\u53b0\u53b1\u53b2\u53b3\u53b4\u53b5\u53b6\u53b7\u53b8\u53b9\u53ba\u53bb\u53bc\u53bd\u53be\u53bf\u53c0\u53c1\u53c2\u53c3\u53c4\u53c5\u53c6\u53c7\u53c8\u53c9\u53ca\u53cb\u53cc\u53cd\u53ce\u53cf\u53d0\u53d1\u53d2\u53d3\u53d4\u53d5\u53d6\u53d7\u53d8\u53d9\u53da\u53db\u53dc\u53dd\u53de\u53df\u53e0\u53e1\u53e2\u53e3\u53e4\u53e5\u53e6\u53e7\u53e8\u53e9\u53ea\u53eb\u53ec\u53ed\u53ee\u53ef\u53f0\u53f1\u53f2\u53f3\u53f4\u53f5\u53f6\u53f7\u53f8\u53f9\u53fa\u53fb\u53fc\u53fd\u53fe\u53ff\u5400\u5401\u5402\u5403\u5404\u5405\u5406\u5407\u5408\u5409\u540a\u540b\u540c\u540d\u540e\u540f\u5410\u5411\u5412\u5413\u5414\u5415\u5416\u5417\u5418\u5419\u541a\u541b\u541c\u541d\u541e\u541f\u5420\u5421\u5422\u5423\u5424\u5425\u5426\u5427\u5428\u5429\u542a\u542b\u542c\u542d\u542e\u542f\u5430\u5431\u5432\u5433\u5434\u5435\u5436\u5437\u5438\u5439\u543a\u543b\u543c\u543d\u543e\u543f\u5440\u5441\u5442\u5443\u5444\u5445\u5446\u5447\u5448\u5449\u544a\u544b\u544c\u544d\u544e\u544f\u5450\u5451\u5452\u5453\u5454\u5455\u5456\u5457\u5458\u5459\u545a\u545b\u545c\u545d\u545e\u545f\u5460\u5461\u5462\u5463\u5464\u5465\u5466\u5467\u5468\u5469\u546a\u546b\u546c\u546d\u546e\u546f\u5470\u5471\u5472\u5473\u5474\u5475\u5476\u5477\u5478\u5479\u547a\u547b\u547c\u547d\u547e\u547f\u5480\u5481\u5482\u5483\u5484\u5485\u5486\u5487\u5488\u5489\u548a\u548b\u548c\u548d\u548e\u548f\u5490\u5491\u5492\u5493\u5494\u5495\u5496\u5497\u5498\u5499\u549a\u549b\u549c\u549d\u549e\u549f\u54a0\u54a1\u54a2\u54a3\u54a4\u54a5\u54a6\u54a7\u54a8\u54a9\u54aa\u54ab\u54ac\u54ad\u54ae\u54af\u54b0\u54b1\u54b2\u54b3\u54b4\u54b5\u54b6\u54b7\u54b8\u54b9\u54ba\u54bb\u54bc\u54bd\u54be\u54bf\u54c0\u54c1\u54c2\u54c3\u54c4\u54c5\u54c6\u54c7\u54c8\u54c9\u54ca\u54cb\u54cc\u54cd\u54ce\u54cf\u54d0\u54d1\u54d2\u54d3\u54d4\u54d5\u54d6\u54d7\u54d8\u54d9\u54da\u54db\u54dc\u54dd\u54de\u54df\u54e0\u54e1\u54e2\u54e3\u54e4\u54e5\u54e6\u54e7\u54e8\u54e9\u54ea\u54eb\u54ec\u54ed\u54ee\u54ef\u54f0\u54f1\u54f2\u54f3\u54f4\u54f5\u54f6\u54f7\u54f8\u54f9\u54fa\u54fb\u54fc\u54fd\u54fe\u54ff\u5500\u5501\u5502\u5503\u5504\u5505\u5506\u5507\u5508\u5509\u550a\u550b\u550c\u550d\u550e\u550f\u5510\u5511\u5512\u5513\u5514\u5515\u5516\u5517\u5518\u5519\u551a\u551b\u551c\u551d\u551e\u551f\u5520\u5521\u5522\u5523\u5524\u5525\u5526\u5527\u5528\u5529\u552a\u552b\u552c\u552d\u552e\u552f\u5530\u5531\u5532\u5533\u5534\u5535\u5536\u5537\u5538\u5539\u553a\u553b\u553c\u553d\u553e\u553f\u5540\u5541\u5542\u5543\u5544\u5545\u5546\u5547\u5548\u5549\u554a\u554b\u554c\u554d\u554e\u554f\u5550\u5551\u5552\u5553\u5554\u5555\u5556\u5557\u5558\u5559\u555a\u555b\u555c\u555d\u555e\u555f\u5560\u5561\u5562\u5563\u5564\u5565\u5566\u5567\u5568\u5569\u556a\u556b\u556c\u556d\u556e\u556f\u5570\u5571\u5572\u5573\u5574\u5575\u5576\u5577\u5578\u5579\u557a\u557b\u557c\u557d\u557e\u557f\u5580\u5581\u5582\u5583\u5584\u5585\u5586\u5587\u5588\u5589\u558a\u558b\u558c\u558d\u558e\u558f\u5590\u5591\u5592\u5593\u5594\u5595\u5596\u5597\u5598\u5599\u559a\u559b\u559c\u559d\u559e\u559f\u55a0\u55a1\u55a2\u55a3\u55a4\u55a5\u55a6\u55a7\u55a8\u55a9\u55aa\u55ab\u55ac\u55ad\u55ae\u55af\u55b0\u55b1\u55b2\u55b3\u55b4\u55b5\u55b6\u55b7\u55b8\u55b9\u55ba\u55bb\u55bc\u55bd\u55be\u55bf\u55c0\u55c1\u55c2\u55c3\u55c4\u55c5\u55c6\u55c7\u55c8\u55c9\u55ca\u55cb\u55cc\u55cd\u55ce\u55cf\u55d0\u55d1\u55d2\u55d3\u55d4\u55d5\u55d6\u55d7\u55d8\u55d9\u55da\u55db\u55dc\u55dd\u55de\u55df\u55e0\u55e1\u55e2\u55e3\u55e4\u55e5\u55e6\u55e7\u55e8\u55e9\u55ea\u55eb\u55ec\u55ed\u55ee\u55ef\u55f0\u55f1\u55f2\u55f3\u55f4\u55f5\u55f6\u55f7\u55f8\u55f9\u55fa\u55fb\u55fc\u55fd\u55fe\u55ff\u5600\u5601\u5602\u5603\u5604\u5605\u5606\u5607\u5608\u5609\u560a\u560b\u560c\u560d\u560e\u560f\u5610\u5611\u5612\u5613\u5614\u5615\u5616\u5617\u5618\u5619\u561a\u561b\u561c\u561d\u561e\u561f\u5620\u5621\u5622\u5623\u5624\u5625\u5626\u5627\u5628\u5629\u562a\u562b\u562c\u562d\u562e\u562f\u5630\u5631\u5632\u5633\u5634\u5635\u5636\u5637\u5638\u5639\u563a\u563b\u563c\u563d\u563e\u563f\u5640\u5641\u5642\u5643\u5644\u5645\u5646\u5647\u5648\u5649\u564a\u564b\u564c\u564d\u564e\u564f\u5650\u5651\u5652\u5653\u5654\u5655\u5656\u5657\u5658\u5659\u565a\u565b\u565c\u565d\u565e\u565f\u5660\u5661\u5662\u5663\u5664\u5665\u5666\u5667\u5668\u5669\u566a\u566b\u566c\u566d\u566e\u566f\u5670\u5671\u5672\u5673\u5674\u5675\u5676\u5677\u5678\u5679\u567a\u567b\u567c\u567d\u567e\u567f\u5680\u5681\u5682\u5683\u5684\u5685\u5686\u5687\u5688\u5689\u568a\u568b\u568c\u568d\u568e\u568f\u5690\u5691\u5692\u5693\u5694\u5695\u5696\u5697\u5698\u5699\u569a\u569b\u569c\u569d\u569e\u569f\u56a0\u56a1\u56a2\u56a3\u56a4\u56a5\u56a6\u56a7\u56a8\u56a9\u56aa\u56ab\u56ac\u56ad\u56ae\u56af\u56b0\u56b1\u56b2\u56b3\u56b4\u56b5\u56b6\u56b7\u56b8\u56b9\u56ba\u56bb\u56bc\u56bd\u56be\u56bf\u56c0\u56c1\u56c2\u56c3\u56c4\u56c5\u56c6\u56c7\u56c8\u56c9\u56ca\u56cb\u56cc\u56cd\u56ce\u56cf\u56d0\u56d1\u56d2\u56d3\u56d4\u56d5\u56d6\u56d7\u56d8\u56d9\u56da\u56db\u56dc\u56dd\u56de\u56df\u56e0\u56e1\u56e2\u56e3\u56e4\u56e5\u56e6\u56e7\u56e8\u56e9\u56ea\u56eb\u56ec\u56ed\u56ee\u56ef\u56f0\u56f1\u56f2\u56f3\u56f4\u56f5\u56f6\u56f7\u56f8\u56f9\u56fa\u56fb\u56fc\u56fd\u56fe\u56ff\u5700\u5701\u5702\u5703\u5704\u5705\u5706\u5707\u5708\u5709\u570a\u570b\u570c\u570d\u570e\u570f\u5710\u5711\u5712\u5713\u5714\u5715\u5716\u5717\u5718\u5719\u571a\u571b\u571c\u571d\u571e\u571f\u5720\u5721\u5722\u5723\u5724\u5725\u5726\u5727\u5728\u5729\u572a\u572b\u572c\u572d\u572e\u572f\u5730\u5731\u5732\u5733\u5734\u5735\u5736\u5737\u5738\u5739\u573a\u573b\u573c\u573d\u573e\u573f\u5740\u5741\u5742\u5743\u5744\u5745\u5746\u5747\u5748\u5749\u574a\u574b\u574c\u574d\u574e\u574f\u5750\u5751\u5752\u5753\u5754\u5755\u5756\u5757\u5758\u5759\u575a\u575b\u575c\u575d\u575e\u575f\u5760\u5761\u5762\u5763\u5764\u5765\u5766\u5767\u5768\u5769\u576a\u576b\u576c\u576d\u576e\u576f\u5770\u5771\u5772\u5773\u5774\u5775\u5776\u5777\u5778\u5779\u577a\u577b\u577c\u577d\u577e\u577f\u5780\u5781\u5782\u5783\u5784\u5785\u5786\u5787\u5788\u5789\u578a\u578b\u578c\u578d\u578e\u578f\u5790\u5791\u5792\u5793\u5794\u5795\u5796\u5797\u5798\u5799\u579a\u579b\u579c\u579d\u579e\u579f\u57a0\u57a1\u57a2\u57a3\u57a4\u57a5\u57a6\u57a7\u57a8\u57a9\u57aa\u57ab\u57ac\u57ad\u57ae\u57af\u57b0\u57b1\u57b2\u57b3\u57b4\u57b5\u57b6\u57b7\u57b8\u57b9\u57ba\u57bb\u57bc\u57bd\u57be\u57bf\u57c0\u57c1\u57c2\u57c3\u57c4\u57c5\u57c6\u57c7\u57c8\u57c9\u57ca\u57cb\u57cc\u57cd\u57ce\u57cf\u57d0\u57d1\u57d2\u57d3\u57d4\u57d5\u57d6\u57d7\u57d8\u57d9\u57da\u57db\u57dc\u57dd\u57de\u57df\u57e0\u57e1\u57e2\u57e3\u57e4\u57e5\u57e6\u57e7\u57e8\u57e9\u57ea\u57eb\u57ec\u57ed\u57ee\u57ef\u57f0\u57f1\u57f2\u57f3\u57f4\u57f5\u57f6\u57f7\u57f8\u57f9\u57fa\u57fb\u57fc\u57fd\u57fe\u57ff\u5800\u5801\u5802\u5803\u5804\u5805\u5806\u5807\u5808\u5809\u580a\u580b\u580c\u580d\u580e\u580f\u5810\u5811\u5812\u5813\u5814\u5815\u5816\u5817\u5818\u5819\u581a\u581b\u581c\u581d\u581e\u581f\u5820\u5821\u5822\u5823\u5824\u5825\u5826\u5827\u5828\u5829\u582a\u582b\u582c\u582d\u582e\u582f\u5830\u5831\u5832\u5833\u5834\u5835\u5836\u5837\u5838\u5839\u583a\u583b\u583c\u583d\u583e\u583f\u5840\u5841\u5842\u5843\u5844\u5845\u5846\u5847\u5848\u5849\u584a\u584b\u584c\u584d\u584e\u584f\u5850\u5851\u5852\u5853\u5854\u5855\u5856\u5857\u5858\u5859\u585a\u585b\u585c\u585d\u585e\u585f\u5860\u5861\u5862\u5863\u5864\u5865\u5866\u5867\u5868\u5869\u586a\u586b\u586c\u586d\u586e\u586f\u5870\u5871\u5872\u5873\u5874\u5875\u5876\u5877\u5878\u5879\u587a\u587b\u587c\u587d\u587e\u587f\u5880\u5881\u5882\u5883\u5884\u5885\u5886\u5887\u5888\u5889\u588a\u588b\u588c\u588d\u588e\u588f\u5890\u5891\u5892\u5893\u5894\u5895\u5896\u5897\u5898\u5899\u589a\u589b\u589c\u589d\u589e\u589f\u58a0\u58a1\u58a2\u58a3\u58a4\u58a5\u58a6\u58a7\u58a8\u58a9\u58aa\u58ab\u58ac\u58ad\u58ae\u58af\u58b0\u58b1\u58b2\u58b3\u58b4\u58b5\u58b6\u58b7\u58b8\u58b9\u58ba\u58bb\u58bc\u58bd\u58be\u58bf\u58c0\u58c1\u58c2\u58c3\u58c4\u58c5\u58c6\u58c7\u58c8\u58c9\u58ca\u58cb\u58cc\u58cd\u58ce\u58cf\u58d0\u58d1\u58d2\u58d3\u58d4\u58d5\u58d6\u58d7\u58d8\u58d9\u58da\u58db\u58dc\u58dd\u58de\u58df\u58e0\u58e1\u58e2\u58e3\u58e4\u58e5\u58e6\u58e7\u58e8\u58e9\u58ea\u58eb\u58ec\u58ed\u58ee\u58ef\u58f0\u58f1\u58f2\u58f3\u58f4\u58f5\u58f6\u58f7\u58f8\u58f9\u58fa\u58fb\u58fc\u58fd\u58fe\u58ff\u5900\u5901\u5902\u5903\u5904\u5905\u5906\u5907\u5908\u5909\u590a\u590b\u590c\u590d\u590e\u590f\u5910\u5911\u5912\u5913\u5914\u5915\u5916\u5917\u5918\u5919\u591a\u591b\u591c\u591d\u591e\u591f\u5920\u5921\u5922\u5923\u5924\u5925\u5926\u5927\u5928\u5929\u592a\u592b\u592c\u592d\u592e\u592f\u5930\u5931\u5932\u5933\u5934\u5935\u5936\u5937\u5938\u5939\u593a\u593b\u593c\u593d\u593e\u593f\u5940\u5941\u5942\u5943\u5944\u5945\u5946\u5947\u5948\u5949\u594a\u594b\u594c\u594d\u594e\u594f\u5950\u5951\u5952\u5953\u5954\u5955\u5956\u5957\u5958\u5959\u595a\u595b\u595c\u595d\u595e\u595f\u5960\u5961\u5962\u5963\u5964\u5965\u5966\u5967\u5968\u5969\u596a\u596b\u596c\u596d\u596e\u596f\u5970\u5971\u5972\u5973\u5974\u5975\u5976\u5977\u5978\u5979\u597a\u597b\u597c\u597d\u597e\u597f\u5980\u5981\u5982\u5983\u5984\u5985\u5986\u5987\u5988\u5989\u598a\u598b\u598c\u598d\u598e\u598f\u5990\u5991\u5992\u5993\u5994\u5995\u5996\u5997\u5998\u5999\u599a\u599b\u599c\u599d\u599e\u599f\u59a0\u59a1\u59a2\u59a3\u59a4\u59a5\u59a6\u59a7\u59a8\u59a9\u59aa\u59ab\u59ac\u59ad\u59ae\u59af\u59b0\u59b1\u59b2\u59b3\u59b4\u59b5\u59b6\u59b7\u59b8\u59b9\u59ba\u59bb\u59bc\u59bd\u59be\u59bf\u59c0\u59c1\u59c2\u59c3\u59c4\u59c5\u59c6\u59c7\u59c8\u59c9\u59ca\u59cb\u59cc\u59cd\u59ce\u59cf\u59d0\u59d1\u59d2\u59d3\u59d4\u59d5\u59d6\u59d7\u59d8\u59d9\u59da\u59db\u59dc\u59dd\u59de\u59df\u59e0\u59e1\u59e2\u59e3\u59e4\u59e5\u59e6\u59e7\u59e8\u59e9\u59ea\u59eb\u59ec\u59ed\u59ee\u59ef\u59f0\u59f1\u59f2\u59f3\u59f4\u59f5\u59f6\u59f7\u59f8\u59f9\u59fa\u59fb\u59fc\u59fd\u59fe\u59ff\u5a00\u5a01\u5a02\u5a03\u5a04\u5a05\u5a06\u5a07\u5a08\u5a09\u5a0a\u5a0b\u5a0c\u5a0d\u5a0e\u5a0f\u5a10\u5a11\u5a12\u5a13\u5a14\u5a15\u5a16\u5a17\u5a18\u5a19\u5a1a\u5a1b\u5a1c\u5a1d\u5a1e\u5a1f\u5a20\u5a21\u5a22\u5a23\u5a24\u5a25\u5a26\u5a27\u5a28\u5a29\u5a2a\u5a2b\u5a2c\u5a2d\u5a2e\u5a2f\u5a30\u5a31\u5a32\u5a33\u5a34\u5a35\u5a36\u5a37\u5a38\u5a39\u5a3a\u5a3b\u5a3c\u5a3d\u5a3e\u5a3f\u5a40\u5a41\u5a42\u5a43\u5a44\u5a45\u5a46\u5a47\u5a48\u5a49\u5a4a\u5a4b\u5a4c\u5a4d\u5a4e\u5a4f\u5a50\u5a51\u5a52\u5a53\u5a54\u5a55\u5a56\u5a57\u5a58\u5a59\u5a5a\u5a5b\u5a5c\u5a5d\u5a5e\u5a5f\u5a60\u5a61\u5a62\u5a63\u5a64\u5a65\u5a66\u5a67\u5a68\u5a69\u5a6a\u5a6b\u5a6c\u5a6d\u5a6e\u5a6f\u5a70\u5a71\u5a72\u5a73\u5a74\u5a75\u5a76\u5a77\u5a78\u5a79\u5a7a\u5a7b\u5a7c\u5a7d\u5a7e\u5a7f\u5a80\u5a81\u5a82\u5a83\u5a84\u5a85\u5a86\u5a87\u5a88\u5a89\u5a8a\u5a8b\u5a8c\u5a8d\u5a8e\u5a8f\u5a90\u5a91\u5a92\u5a93\u5a94\u5a95\u5a96\u5a97\u5a98\u5a99\u5a9a\u5a9b\u5a9c\u5a9d\u5a9e\u5a9f\u5aa0\u5aa1\u5aa2\u5aa3\u5aa4\u5aa5\u5aa6\u5aa7\u5aa8\u5aa9\u5aaa\u5aab\u5aac\u5aad\u5aae\u5aaf\u5ab0\u5ab1\u5ab2\u5ab3\u5ab4\u5ab5\u5ab6\u5ab7\u5ab8\u5ab9\u5aba\u5abb\u5abc\u5abd\u5abe\u5abf\u5ac0\u5ac1\u5ac2\u5ac3\u5ac4\u5ac5\u5ac6\u5ac7\u5ac8\u5ac9\u5aca\u5acb\u5acc\u5acd\u5ace\u5acf\u5ad0\u5ad1\u5ad2\u5ad3\u5ad4\u5ad5\u5ad6\u5ad7\u5ad8\u5ad9\u5ada\u5adb\u5adc\u5add\u5ade\u5adf\u5ae0\u5ae1\u5ae2\u5ae3\u5ae4\u5ae5\u5ae6\u5ae7\u5ae8\u5ae9\u5aea\u5aeb\u5aec\u5aed\u5aee\u5aef\u5af0\u5af1\u5af2\u5af3\u5af4\u5af5\u5af6\u5af7\u5af8\u5af9\u5afa\u5afb\u5afc\u5afd\u5afe\u5aff\u5b00\u5b01\u5b02\u5b03\u5b04\u5b05\u5b06\u5b07\u5b08\u5b09\u5b0a\u5b0b\u5b0c\u5b0d\u5b0e\u5b0f\u5b10\u5b11\u5b12\u5b13\u5b14\u5b15\u5b16\u5b17\u5b18\u5b19\u5b1a\u5b1b\u5b1c\u5b1d\u5b1e\u5b1f\u5b20\u5b21\u5b22\u5b23\u5b24\u5b25\u5b26\u5b27\u5b28\u5b29\u5b2a\u5b2b\u5b2c\u5b2d\u5b2e\u5b2f\u5b30\u5b31\u5b32\u5b33\u5b34\u5b35\u5b36\u5b37\u5b38\u5b39\u5b3a\u5b3b\u5b3c\u5b3d\u5b3e\u5b3f\u5b40\u5b41\u5b42\u5b43\u5b44\u5b45\u5b46\u5b47\u5b48\u5b49\u5b4a\u5b4b\u5b4c\u5b4d\u5b4e\u5b4f\u5b50\u5b51\u5b52\u5b53\u5b54\u5b55\u5b56\u5b57\u5b58\u5b59\u5b5a\u5b5b\u5b5c\u5b5d\u5b5e\u5b5f\u5b60\u5b61\u5b62\u5b63\u5b64\u5b65\u5b66\u5b67\u5b68\u5b69\u5b6a\u5b6b\u5b6c\u5b6d\u5b6e\u5b6f\u5b70\u5b71\u5b72\u5b73\u5b74\u5b75\u5b76\u5b77\u5b78\u5b79\u5b7a\u5b7b\u5b7c\u5b7d\u5b7e\u5b7f\u5b80\u5b81\u5b82\u5b83\u5b84\u5b85\u5b86\u5b87\u5b88\u5b89\u5b8a\u5b8b\u5b8c\u5b8d\u5b8e\u5b8f\u5b90\u5b91\u5b92\u5b93\u5b94\u5b95\u5b96\u5b97\u5b98\u5b99\u5b9a\u5b9b\u5b9c\u5b9d\u5b9e\u5b9f\u5ba0\u5ba1\u5ba2\u5ba3\u5ba4\u5ba5\u5ba6\u5ba7\u5ba8\u5ba9\u5baa\u5bab\u5bac\u5bad\u5bae\u5baf\u5bb0\u5bb1\u5bb2\u5bb3\u5bb4\u5bb5\u5bb6\u5bb7\u5bb8\u5bb9\u5bba\u5bbb\u5bbc\u5bbd\u5bbe\u5bbf\u5bc0\u5bc1\u5bc2\u5bc3\u5bc4\u5bc5\u5bc6\u5bc7\u5bc8\u5bc9\u5bca\u5bcb\u5bcc\u5bcd\u5bce\u5bcf\u5bd0\u5bd1\u5bd2\u5bd3\u5bd4\u5bd5\u5bd6\u5bd7\u5bd8\u5bd9\u5bda\u5bdb\u5bdc\u5bdd\u5bde\u5bdf\u5be0\u5be1\u5be2\u5be3\u5be4\u5be5\u5be6\u5be7\u5be8\u5be9\u5bea\u5beb\u5bec\u5bed\u5bee\u5bef\u5bf0\u5bf1\u5bf2\u5bf3\u5bf4\u5bf5\u5bf6\u5bf7\u5bf8\u5bf9\u5bfa\u5bfb\u5bfc\u5bfd\u5bfe\u5bff\u5c00\u5c01\u5c02\u5c03\u5c04\u5c05\u5c06\u5c07\u5c08\u5c09\u5c0a\u5c0b\u5c0c\u5c0d\u5c0e\u5c0f\u5c10\u5c11\u5c12\u5c13\u5c14\u5c15\u5c16\u5c17\u5c18\u5c19\u5c1a\u5c1b\u5c1c\u5c1d\u5c1e\u5c1f\u5c20\u5c21\u5c22\u5c23\u5c24\u5c25\u5c26\u5c27\u5c28\u5c29\u5c2a\u5c2b\u5c2c\u5c2d\u5c2e\u5c2f\u5c30\u5c31\u5c32\u5c33\u5c34\u5c35\u5c36\u5c37\u5c38\u5c39\u5c3a\u5c3b\u5c3c\u5c3d\u5c3e\u5c3f\u5c40\u5c41\u5c42\u5c43\u5c44\u5c45\u5c46\u5c47\u5c48\u5c49\u5c4a\u5c4b\u5c4c\u5c4d\u5c4e\u5c4f\u5c50\u5c51\u5c52\u5c53\u5c54\u5c55\u5c56\u5c57\u5c58\u5c59\u5c5a\u5c5b\u5c5c\u5c5d\u5c5e\u5c5f\u5c60\u5c61\u5c62\u5c63\u5c64\u5c65\u5c66\u5c67\u5c68\u5c69\u5c6a\u5c6b\u5c6c\u5c6d\u5c6e\u5c6f\u5c70\u5c71\u5c72\u5c73\u5c74\u5c75\u5c76\u5c77\u5c78\u5c79\u5c7a\u5c7b\u5c7c\u5c7d\u5c7e\u5c7f\u5c80\u5c81\u5c82\u5c83\u5c84\u5c85\u5c86\u5c87\u5c88\u5c89\u5c8a\u5c8b\u5c8c\u5c8d\u5c8e\u5c8f\u5c90\u5c91\u5c92\u5c93\u5c94\u5c95\u5c96\u5c97\u5c98\u5c99\u5c9a\u5c9b\u5c9c\u5c9d\u5c9e\u5c9f\u5ca0\u5ca1\u5ca2\u5ca3\u5ca4\u5ca5\u5ca6\u5ca7\u5ca8\u5ca9\u5caa\u5cab\u5cac\u5cad\u5cae\u5caf\u5cb0\u5cb1\u5cb2\u5cb3\u5cb4\u5cb5\u5cb6\u5cb7\u5cb8\u5cb9\u5cba\u5cbb\u5cbc\u5cbd\u5cbe\u5cbf\u5cc0\u5cc1\u5cc2\u5cc3\u5cc4\u5cc5\u5cc6\u5cc7\u5cc8\u5cc9\u5cca\u5ccb\u5ccc\u5ccd\u5cce\u5ccf\u5cd0\u5cd1\u5cd2\u5cd3\u5cd4\u5cd5\u5cd6\u5cd7\u5cd8\u5cd9\u5cda\u5cdb\u5cdc\u5cdd\u5cde\u5cdf\u5ce0\u5ce1\u5ce2\u5ce3\u5ce4\u5ce5\u5ce6\u5ce7\u5ce8\u5ce9\u5cea\u5ceb\u5cec\u5ced\u5cee\u5cef\u5cf0\u5cf1\u5cf2\u5cf3\u5cf4\u5cf5\u5cf6\u5cf7\u5cf8\u5cf9\u5cfa\u5cfb\u5cfc\u5cfd\u5cfe\u5cff\u5d00\u5d01\u5d02\u5d03\u5d04\u5d05\u5d06\u5d07\u5d08\u5d09\u5d0a\u5d0b\u5d0c\u5d0d\u5d0e\u5d0f\u5d10\u5d11\u5d12\u5d13\u5d14\u5d15\u5d16\u5d17\u5d18\u5d19\u5d1a\u5d1b\u5d1c\u5d1d\u5d1e\u5d1f\u5d20\u5d21\u5d22\u5d23\u5d24\u5d25\u5d26\u5d27\u5d28\u5d29\u5d2a\u5d2b\u5d2c\u5d2d\u5d2e\u5d2f\u5d30\u5d31\u5d32\u5d33\u5d34\u5d35\u5d36\u5d37\u5d38\u5d39\u5d3a\u5d3b\u5d3c\u5d3d\u5d3e\u5d3f\u5d40\u5d41\u5d42\u5d43\u5d44\u5d45\u5d46\u5d47\u5d48\u5d49\u5d4a\u5d4b\u5d4c\u5d4d\u5d4e\u5d4f\u5d50\u5d51\u5d52\u5d53\u5d54\u5d55\u5d56\u5d57\u5d58\u5d59\u5d5a\u5d5b\u5d5c\u5d5d\u5d5e\u5d5f\u5d60\u5d61\u5d62\u5d63\u5d64\u5d65\u5d66\u5d67\u5d68\u5d69\u5d6a\u5d6b\u5d6c\u5d6d\u5d6e\u5d6f\u5d70\u5d71\u5d72\u5d73\u5d74\u5d75\u5d76\u5d77\u5d78\u5d79\u5d7a\u5d7b\u5d7c\u5d7d\u5d7e\u5d7f\u5d80\u5d81\u5d82\u5d83\u5d84\u5d85\u5d86\u5d87\u5d88\u5d89\u5d8a\u5d8b\u5d8c\u5d8d\u5d8e\u5d8f\u5d90\u5d91\u5d92\u5d93\u5d94\u5d95\u5d96\u5d97\u5d98\u5d99\u5d9a\u5d9b\u5d9c\u5d9d\u5d9e\u5d9f\u5da0\u5da1\u5da2\u5da3\u5da4\u5da5\u5da6\u5da7\u5da8\u5da9\u5daa\u5dab\u5dac\u5dad\u5dae\u5daf\u5db0\u5db1\u5db2\u5db3\u5db4\u5db5\u5db6\u5db7\u5db8\u5db9\u5dba\u5dbb\u5dbc\u5dbd\u5dbe\u5dbf\u5dc0\u5dc1\u5dc2\u5dc3\u5dc4\u5dc5\u5dc6\u5dc7\u5dc8\u5dc9\u5dca\u5dcb\u5dcc\u5dcd\u5dce\u5dcf\u5dd0\u5dd1\u5dd2\u5dd3\u5dd4\u5dd5\u5dd6\u5dd7\u5dd8\u5dd9\u5dda\u5ddb\u5ddc\u5ddd\u5dde\u5ddf\u5de0\u5de1\u5de2\u5de3\u5de4\u5de5\u5de6\u5de7\u5de8\u5de9\u5dea\u5deb\u5dec\u5ded\u5dee\u5def\u5df0\u5df1\u5df2\u5df3\u5df4\u5df5\u5df6\u5df7\u5df8\u5df9\u5dfa\u5dfb\u5dfc\u5dfd\u5dfe\u5dff\u5e00\u5e01\u5e02\u5e03\u5e04\u5e05\u5e06\u5e07\u5e08\u5e09\u5e0a\u5e0b\u5e0c\u5e0d\u5e0e\u5e0f\u5e10\u5e11\u5e12\u5e13\u5e14\u5e15\u5e16\u5e17\u5e18\u5e19\u5e1a\u5e1b\u5e1c\u5e1d\u5e1e\u5e1f\u5e20\u5e21\u5e22\u5e23\u5e24\u5e25\u5e26\u5e27\u5e28\u5e29\u5e2a\u5e2b\u5e2c\u5e2d\u5e2e\u5e2f\u5e30\u5e31\u5e32\u5e33\u5e34\u5e35\u5e36\u5e37\u5e38\u5e39\u5e3a\u5e3b\u5e3c\u5e3d\u5e3e\u5e3f\u5e40\u5e41\u5e42\u5e43\u5e44\u5e45\u5e46\u5e47\u5e48\u5e49\u5e4a\u5e4b\u5e4c\u5e4d\u5e4e\u5e4f\u5e50\u5e51\u5e52\u5e53\u5e54\u5e55\u5e56\u5e57\u5e58\u5e59\u5e5a\u5e5b\u5e5c\u5e5d\u5e5e\u5e5f\u5e60\u5e61\u5e62\u5e63\u5e64\u5e65\u5e66\u5e67\u5e68\u5e69\u5e6a\u5e6b\u5e6c\u5e6d\u5e6e\u5e6f\u5e70\u5e71\u5e72\u5e73\u5e74\u5e75\u5e76\u5e77\u5e78\u5e79\u5e7a\u5e7b\u5e7c\u5e7d\u5e7e\u5e7f\u5e80\u5e81\u5e82\u5e83\u5e84\u5e85\u5e86\u5e87\u5e88\u5e89\u5e8a\u5e8b\u5e8c\u5e8d\u5e8e\u5e8f\u5e90\u5e91\u5e92\u5e93\u5e94\u5e95\u5e96\u5e97\u5e98\u5e99\u5e9a\u5e9b\u5e9c\u5e9d\u5e9e\u5e9f\u5ea0\u5ea1\u5ea2\u5ea3\u5ea4\u5ea5\u5ea6\u5ea7\u5ea8\u5ea9\u5eaa\u5eab\u5eac\u5ead\u5eae\u5eaf\u5eb0\u5eb1\u5eb2\u5eb3\u5eb4\u5eb5\u5eb6\u5eb7\u5eb8\u5eb9\u5eba\u5ebb\u5ebc\u5ebd\u5ebe\u5ebf\u5ec0\u5ec1\u5ec2\u5ec3\u5ec4\u5ec5\u5ec6\u5ec7\u5ec8\u5ec9\u5eca\u5ecb\u5ecc\u5ecd\u5ece\u5ecf\u5ed0\u5ed1\u5ed2\u5ed3\u5ed4\u5ed5\u5ed6\u5ed7\u5ed8\u5ed9\u5eda\u5edb\u5edc\u5edd\u5ede\u5edf\u5ee0\u5ee1\u5ee2\u5ee3\u5ee4\u5ee5\u5ee6\u5ee7\u5ee8\u5ee9\u5eea\u5eeb\u5eec\u5eed\u5eee\u5eef\u5ef0\u5ef1\u5ef2\u5ef3\u5ef4\u5ef5\u5ef6\u5ef7\u5ef8\u5ef9\u5efa\u5efb\u5efc\u5efd\u5efe\u5eff\u5f00\u5f01\u5f02\u5f03\u5f04\u5f05\u5f06\u5f07\u5f08\u5f09\u5f0a\u5f0b\u5f0c\u5f0d\u5f0e\u5f0f\u5f10\u5f11\u5f12\u5f13\u5f14\u5f15\u5f16\u5f17\u5f18\u5f19\u5f1a\u5f1b\u5f1c\u5f1d\u5f1e\u5f1f\u5f20\u5f21\u5f22\u5f23\u5f24\u5f25\u5f26\u5f27\u5f28\u5f29\u5f2a\u5f2b\u5f2c\u5f2d\u5f2e\u5f2f\u5f30\u5f31\u5f32\u5f33\u5f34\u5f35\u5f36\u5f37\u5f38\u5f39\u5f3a\u5f3b\u5f3c\u5f3d\u5f3e\u5f3f\u5f40\u5f41\u5f42\u5f43\u5f44\u5f45\u5f46\u5f47\u5f48\u5f49\u5f4a\u5f4b\u5f4c\u5f4d\u5f4e\u5f4f\u5f50\u5f51\u5f52\u5f53\u5f54\u5f55\u5f56\u5f57\u5f58\u5f59\u5f5a\u5f5b\u5f5c\u5f5d\u5f5e\u5f5f\u5f60\u5f61\u5f62\u5f63\u5f64\u5f65\u5f66\u5f67\u5f68\u5f69\u5f6a\u5f6b\u5f6c\u5f6d\u5f6e\u5f6f\u5f70\u5f71\u5f72\u5f73\u5f74\u5f75\u5f76\u5f77\u5f78\u5f79\u5f7a\u5f7b\u5f7c\u5f7d\u5f7e\u5f7f\u5f80\u5f81\u5f82\u5f83\u5f84\u5f85\u5f86\u5f87\u5f88\u5f89\u5f8a\u5f8b\u5f8c\u5f8d\u5f8e\u5f8f\u5f90\u5f91\u5f92\u5f93\u5f94\u5f95\u5f96\u5f97\u5f98\u5f99\u5f9a\u5f9b\u5f9c\u5f9d\u5f9e\u5f9f\u5fa0\u5fa1\u5fa2\u5fa3\u5fa4\u5fa5\u5fa6\u5fa7\u5fa8\u5fa9\u5faa\u5fab\u5fac\u5fad\u5fae\u5faf\u5fb0\u5fb1\u5fb2\u5fb3\u5fb4\u5fb5\u5fb6\u5fb7\u5fb8\u5fb9\u5fba\u5fbb\u5fbc\u5fbd\u5fbe\u5fbf\u5fc0\u5fc1\u5fc2\u5fc3\u5fc4\u5fc5\u5fc6\u5fc7\u5fc8\u5fc9\u5fca\u5fcb\u5fcc\u5fcd\u5fce\u5fcf\u5fd0\u5fd1\u5fd2\u5fd3\u5fd4\u5fd5\u5fd6\u5fd7\u5fd8\u5fd9\u5fda\u5fdb\u5fdc\u5fdd\u5fde\u5fdf\u5fe0\u5fe1\u5fe2\u5fe3\u5fe4\u5fe5\u5fe6\u5fe7\u5fe8\u5fe9\u5fea\u5feb\u5fec\u5fed\u5fee\u5fef\u5ff0\u5ff1\u5ff2\u5ff3\u5ff4\u5ff5\u5ff6\u5ff7\u5ff8\u5ff9\u5ffa\u5ffb\u5ffc\u5ffd\u5ffe\u5fff\u6000\u6001\u6002\u6003\u6004\u6005\u6006\u6007\u6008\u6009\u600a\u600b\u600c\u600d\u600e\u600f\u6010\u6011\u6012\u6013\u6014\u6015\u6016\u6017\u6018\u6019\u601a\u601b\u601c\u601d\u601e\u601f\u6020\u6021\u6022\u6023\u6024\u6025\u6026\u6027\u6028\u6029\u602a\u602b\u602c\u602d\u602e\u602f\u6030\u6031\u6032\u6033\u6034\u6035\u6036\u6037\u6038\u6039\u603a\u603b\u603c\u603d\u603e\u603f\u6040\u6041\u6042\u6043\u6044\u6045\u6046\u6047\u6048\u6049\u604a\u604b\u604c\u604d\u604e\u604f\u6050\u6051\u6052\u6053\u6054\u6055\u6056\u6057\u6058\u6059\u605a\u605b\u605c\u605d\u605e\u605f\u6060\u6061\u6062\u6063\u6064\u6065\u6066\u6067\u6068\u6069\u606a\u606b\u606c\u606d\u606e\u606f\u6070\u6071\u6072\u6073\u6074\u6075\u6076\u6077\u6078\u6079\u607a\u607b\u607c\u607d\u607e\u607f\u6080\u6081\u6082\u6083\u6084\u6085\u6086\u6087\u6088\u6089\u608a\u608b\u608c\u608d\u608e\u608f\u6090\u6091\u6092\u6093\u6094\u6095\u6096\u6097\u6098\u6099\u609a\u609b\u609c\u609d\u609e\u609f\u60a0\u60a1\u60a2\u60a3\u60a4\u60a5\u60a6\u60a7\u60a8\u60a9\u60aa\u60ab\u60ac\u60ad\u60ae\u60af\u60b0\u60b1\u60b2\u60b3\u60b4\u60b5\u60b6\u60b7\u60b8\u60b9\u60ba\u60bb\u60bc\u60bd\u60be\u60bf\u60c0\u60c1\u60c2\u60c3\u60c4\u60c5\u60c6\u60c7\u60c8\u60c9\u60ca\u60cb\u60cc\u60cd\u60ce\u60cf\u60d0\u60d1\u60d2\u60d3\u60d4\u60d5\u60d6\u60d7\u60d8\u60d9\u60da\u60db\u60dc\u60dd\u60de\u60df\u60e0\u60e1\u60e2\u60e3\u60e4\u60e5\u60e6\u60e7\u60e8\u60e9\u60ea\u60eb\u60ec\u60ed\u60ee\u60ef\u60f0\u60f1\u60f2\u60f3\u60f4\u60f5\u60f6\u60f7\u60f8\u60f9\u60fa\u60fb\u60fc\u60fd\u60fe\u60ff\u6100\u6101\u6102\u6103\u6104\u6105\u6106\u6107\u6108\u6109\u610a\u610b\u610c\u610d\u610e\u610f\u6110\u6111\u6112\u6113\u6114\u6115\u6116\u6117\u6118\u6119\u611a\u611b\u611c\u611d\u611e\u611f\u6120\u6121\u6122\u6123\u6124\u6125\u6126\u6127\u6128\u6129\u612a\u612b\u612c\u612d\u612e\u612f\u6130\u6131\u6132\u6133\u6134\u6135\u6136\u6137\u6138\u6139\u613a\u613b\u613c\u613d\u613e\u613f\u6140\u6141\u6142\u6143\u6144\u6145\u6146\u6147\u6148\u6149\u614a\u614b\u614c\u614d\u614e\u614f\u6150\u6151\u6152\u6153\u6154\u6155\u6156\u6157\u6158\u6159\u615a\u615b\u615c\u615d\u615e\u615f\u6160\u6161\u6162\u6163\u6164\u6165\u6166\u6167\u6168\u6169\u616a\u616b\u616c\u616d\u616e\u616f\u6170\u6171\u6172\u6173\u6174\u6175\u6176\u6177\u6178\u6179\u617a\u617b\u617c\u617d\u617e\u617f\u6180\u6181\u6182\u6183\u6184\u6185\u6186\u6187\u6188\u6189\u618a\u618b\u618c\u618d\u618e\u618f\u6190\u6191\u6192\u6193\u6194\u6195\u6196\u6197\u6198\u6199\u619a\u619b\u619c\u619d\u619e\u619f\u61a0\u61a1\u61a2\u61a3\u61a4\u61a5\u61a6\u61a7\u61a8\u61a9\u61aa\u61ab\u61ac\u61ad\u61ae\u61af\u61b0\u61b1\u61b2\u61b3\u61b4\u61b5\u61b6\u61b7\u61b8\u61b9\u61ba\u61bb\u61bc\u61bd\u61be\u61bf\u61c0\u61c1\u61c2\u61c3\u61c4\u61c5\u61c6\u61c7\u61c8\u61c9\u61ca\u61cb\u61cc\u61cd\u61ce\u61cf\u61d0\u61d1\u61d2\u61d3\u61d4\u61d5\u61d6\u61d7\u61d8\u61d9\u61da\u61db\u61dc\u61dd\u61de\u61df\u61e0\u61e1\u61e2\u61e3\u61e4\u61e5\u61e6\u61e7\u61e8\u61e9\u61ea\u61eb\u61ec\u61ed\u61ee\u61ef\u61f0\u61f1\u61f2\u61f3\u61f4\u61f5\u61f6\u61f7\u61f8\u61f9\u61fa\u61fb\u61fc\u61fd\u61fe\u61ff\u6200\u6201\u6202\u6203\u6204\u6205\u6206\u6207\u6208\u6209\u620a\u620b\u620c\u620d\u620e\u620f\u6210\u6211\u6212\u6213\u6214\u6215\u6216\u6217\u6218\u6219\u621a\u621b\u621c\u621d\u621e\u621f\u6220\u6221\u6222\u6223\u6224\u6225\u6226\u6227\u6228\u6229\u622a\u622b\u622c\u622d\u622e\u622f\u6230\u6231\u6232\u6233\u6234\u6235\u6236\u6237\u6238\u6239\u623a\u623b\u623c\u623d\u623e\u623f\u6240\u6241\u6242\u6243\u6244\u6245\u6246\u6247\u6248\u6249\u624a\u624b\u624c\u624d\u624e\u624f\u6250\u6251\u6252\u6253\u6254\u6255\u6256\u6257\u6258\u6259\u625a\u625b\u625c\u625d\u625e\u625f\u6260\u6261\u6262\u6263\u6264\u6265\u6266\u6267\u6268\u6269\u626a\u626b\u626c\u626d\u626e\u626f\u6270\u6271\u6272\u6273\u6274\u6275\u6276\u6277\u6278\u6279\u627a\u627b\u627c\u627d\u627e\u627f\u6280\u6281\u6282\u6283\u6284\u6285\u6286\u6287\u6288\u6289\u628a\u628b\u628c\u628d\u628e\u628f\u6290\u6291\u6292\u6293\u6294\u6295\u6296\u6297\u6298\u6299\u629a\u629b\u629c\u629d\u629e\u629f\u62a0\u62a1\u62a2\u62a3\u62a4\u62a5\u62a6\u62a7\u62a8\u62a9\u62aa\u62ab\u62ac\u62ad\u62ae\u62af\u62b0\u62b1\u62b2\u62b3\u62b4\u62b5\u62b6\u62b7\u62b8\u62b9\u62ba\u62bb\u62bc\u62bd\u62be\u62bf\u62c0\u62c1\u62c2\u62c3\u62c4\u62c5\u62c6\u62c7\u62c8\u62c9\u62ca\u62cb\u62cc\u62cd\u62ce\u62cf\u62d0\u62d1\u62d2\u62d3\u62d4\u62d5\u62d6\u62d7\u62d8\u62d9\u62da\u62db\u62dc\u62dd\u62de\u62df\u62e0\u62e1\u62e2\u62e3\u62e4\u62e5\u62e6\u62e7\u62e8\u62e9\u62ea\u62eb\u62ec\u62ed\u62ee\u62ef\u62f0\u62f1\u62f2\u62f3\u62f4\u62f5\u62f6\u62f7\u62f8\u62f9\u62fa\u62fb\u62fc\u62fd\u62fe\u62ff\u6300\u6301\u6302\u6303\u6304\u6305\u6306\u6307\u6308\u6309\u630a\u630b\u630c\u630d\u630e\u630f\u6310\u6311\u6312\u6313\u6314\u6315\u6316\u6317\u6318\u6319\u631a\u631b\u631c\u631d\u631e\u631f\u6320\u6321\u6322\u6323\u6324\u6325\u6326\u6327\u6328\u6329\u632a\u632b\u632c\u632d\u632e\u632f\u6330\u6331\u6332\u6333\u6334\u6335\u6336\u6337\u6338\u6339\u633a\u633b\u633c\u633d\u633e\u633f\u6340\u6341\u6342\u6343\u6344\u6345\u6346\u6347\u6348\u6349\u634a\u634b\u634c\u634d\u634e\u634f\u6350\u6351\u6352\u6353\u6354\u6355\u6356\u6357\u6358\u6359\u635a\u635b\u635c\u635d\u635e\u635f\u6360\u6361\u6362\u6363\u6364\u6365\u6366\u6367\u6368\u6369\u636a\u636b\u636c\u636d\u636e\u636f\u6370\u6371\u6372\u6373\u6374\u6375\u6376\u6377\u6378\u6379\u637a\u637b\u637c\u637d\u637e\u637f\u6380\u6381\u6382\u6383\u6384\u6385\u6386\u6387\u6388\u6389\u638a\u638b\u638c\u638d\u638e\u638f\u6390\u6391\u6392\u6393\u6394\u6395\u6396\u6397\u6398\u6399\u639a\u639b\u639c\u639d\u639e\u639f\u63a0\u63a1\u63a2\u63a3\u63a4\u63a5\u63a6\u63a7\u63a8\u63a9\u63aa\u63ab\u63ac\u63ad\u63ae\u63af\u63b0\u63b1\u63b2\u63b3\u63b4\u63b5\u63b6\u63b7\u63b8\u63b9\u63ba\u63bb\u63bc\u63bd\u63be\u63bf\u63c0\u63c1\u63c2\u63c3\u63c4\u63c5\u63c6\u63c7\u63c8\u63c9\u63ca\u63cb\u63cc\u63cd\u63ce\u63cf\u63d0\u63d1\u63d2\u63d3\u63d4\u63d5\u63d6\u63d7\u63d8\u63d9\u63da\u63db\u63dc\u63dd\u63de\u63df\u63e0\u63e1\u63e2\u63e3\u63e4\u63e5\u63e6\u63e7\u63e8\u63e9\u63ea\u63eb\u63ec\u63ed\u63ee\u63ef\u63f0\u63f1\u63f2\u63f3\u63f4\u63f5\u63f6\u63f7\u63f8\u63f9\u63fa\u63fb\u63fc\u63fd\u63fe\u63ff\u6400\u6401\u6402\u6403\u6404\u6405\u6406\u6407\u6408\u6409\u640a\u640b\u640c\u640d\u640e\u640f\u6410\u6411\u6412\u6413\u6414\u6415\u6416\u6417\u6418\u6419\u641a\u641b\u641c\u641d\u641e\u641f\u6420\u6421\u6422\u6423\u6424\u6425\u6426\u6427\u6428\u6429\u642a\u642b\u642c\u642d\u642e\u642f\u6430\u6431\u6432\u6433\u6434\u6435\u6436\u6437\u6438\u6439\u643a\u643b\u643c\u643d\u643e\u643f\u6440\u6441\u6442\u6443\u6444\u6445\u6446\u6447\u6448\u6449\u644a\u644b\u644c\u644d\u644e\u644f\u6450\u6451\u6452\u6453\u6454\u6455\u6456\u6457\u6458\u6459\u645a\u645b\u645c\u645d\u645e\u645f\u6460\u6461\u6462\u6463\u6464\u6465\u6466\u6467\u6468\u6469\u646a\u646b\u646c\u646d\u646e\u646f\u6470\u6471\u6472\u6473\u6474\u6475\u6476\u6477\u6478\u6479\u647a\u647b\u647c\u647d\u647e\u647f\u6480\u6481\u6482\u6483\u6484\u6485\u6486\u6487\u6488\u6489\u648a\u648b\u648c\u648d\u648e\u648f\u6490\u6491\u6492\u6493\u6494\u6495\u6496\u6497\u6498\u6499\u649a\u649b\u649c\u649d\u649e\u649f\u64a0\u64a1\u64a2\u64a3\u64a4\u64a5\u64a6\u64a7\u64a8\u64a9\u64aa\u64ab\u64ac\u64ad\u64ae\u64af\u64b0\u64b1\u64b2\u64b3\u64b4\u64b5\u64b6\u64b7\u64b8\u64b9\u64ba\u64bb\u64bc\u64bd\u64be\u64bf\u64c0\u64c1\u64c2\u64c3\u64c4\u64c5\u64c6\u64c7\u64c8\u64c9\u64ca\u64cb\u64cc\u64cd\u64ce\u64cf\u64d0\u64d1\u64d2\u64d3\u64d4\u64d5\u64d6\u64d7\u64d8\u64d9\u64da\u64db\u64dc\u64dd\u64de\u64df\u64e0\u64e1\u64e2\u64e3\u64e4\u64e5\u64e6\u64e7\u64e8\u64e9\u64ea\u64eb\u64ec\u64ed\u64ee\u64ef\u64f0\u64f1\u64f2\u64f3\u64f4\u64f5\u64f6\u64f7\u64f8\u64f9\u64fa\u64fb\u64fc\u64fd\u64fe\u64ff\u6500\u6501\u6502\u6503\u6504\u6505\u6506\u6507\u6508\u6509\u650a\u650b\u650c\u650d\u650e\u650f\u6510\u6511\u6512\u6513\u6514\u6515\u6516\u6517\u6518\u6519\u651a\u651b\u651c\u651d\u651e\u651f\u6520\u6521\u6522\u6523\u6524\u6525\u6526\u6527\u6528\u6529\u652a\u652b\u652c\u652d\u652e\u652f\u6530\u6531\u6532\u6533\u6534\u6535\u6536\u6537\u6538\u6539\u653a\u653b\u653c\u653d\u653e\u653f\u6540\u6541\u6542\u6543\u6544\u6545\u6546\u6547\u6548\u6549\u654a\u654b\u654c\u654d\u654e\u654f\u6550\u6551\u6552\u6553\u6554\u6555\u6556\u6557\u6558\u6559\u655a\u655b\u655c\u655d\u655e\u655f\u6560\u6561\u6562\u6563\u6564\u6565\u6566\u6567\u6568\u6569\u656a\u656b\u656c\u656d\u656e\u656f\u6570\u6571\u6572\u6573\u6574\u6575\u6576\u6577\u6578\u6579\u657a\u657b\u657c\u657d\u657e\u657f\u6580\u6581\u6582\u6583\u6584\u6585\u6586\u6587\u6588\u6589\u658a\u658b\u658c\u658d\u658e\u658f\u6590\u6591\u6592\u6593\u6594\u6595\u6596\u6597\u6598\u6599\u659a\u659b\u659c\u659d\u659e\u659f\u65a0\u65a1\u65a2\u65a3\u65a4\u65a5\u65a6\u65a7\u65a8\u65a9\u65aa\u65ab\u65ac\u65ad\u65ae\u65af\u65b0\u65b1\u65b2\u65b3\u65b4\u65b5\u65b6\u65b7\u65b8\u65b9\u65ba\u65bb\u65bc\u65bd\u65be\u65bf\u65c0\u65c1\u65c2\u65c3\u65c4\u65c5\u65c6\u65c7\u65c8\u65c9\u65ca\u65cb\u65cc\u65cd\u65ce\u65cf\u65d0\u65d1\u65d2\u65d3\u65d4\u65d5\u65d6\u65d7\u65d8\u65d9\u65da\u65db\u65dc\u65dd\u65de\u65df\u65e0\u65e1\u65e2\u65e3\u65e4\u65e5\u65e6\u65e7\u65e8\u65e9\u65ea\u65eb\u65ec\u65ed\u65ee\u65ef\u65f0\u65f1\u65f2\u65f3\u65f4\u65f5\u65f6\u65f7\u65f8\u65f9\u65fa\u65fb\u65fc\u65fd\u65fe\u65ff\u6600\u6601\u6602\u6603\u6604\u6605\u6606\u6607\u6608\u6609\u660a\u660b\u660c\u660d\u660e\u660f\u6610\u6611\u6612\u6613\u6614\u6615\u6616\u6617\u6618\u6619\u661a\u661b\u661c\u661d\u661e\u661f\u6620\u6621\u6622\u6623\u6624\u6625\u6626\u6627\u6628\u6629\u662a\u662b\u662c\u662d\u662e\u662f\u6630\u6631\u6632\u6633\u6634\u6635\u6636\u6637\u6638\u6639\u663a\u663b\u663c\u663d\u663e\u663f\u6640\u6641\u6642\u6643\u6644\u6645\u6646\u6647\u6648\u6649\u664a\u664b\u664c\u664d\u664e\u664f\u6650\u6651\u6652\u6653\u6654\u6655\u6656\u6657\u6658\u6659\u665a\u665b\u665c\u665d\u665e\u665f\u6660\u6661\u6662\u6663\u6664\u6665\u6666\u6667\u6668\u6669\u666a\u666b\u666c\u666d\u666e\u666f\u6670\u6671\u6672\u6673\u6674\u6675\u6676\u6677\u6678\u6679\u667a\u667b\u667c\u667d\u667e\u667f\u6680\u6681\u6682\u6683\u6684\u6685\u6686\u6687\u6688\u6689\u668a\u668b\u668c\u668d\u668e\u668f\u6690\u6691\u6692\u6693\u6694\u6695\u6696\u6697\u6698\u6699\u669a\u669b\u669c\u669d\u669e\u669f\u66a0\u66a1\u66a2\u66a3\u66a4\u66a5\u66a6\u66a7\u66a8\u66a9\u66aa\u66ab\u66ac\u66ad\u66ae\u66af\u66b0\u66b1\u66b2\u66b3\u66b4\u66b5\u66b6\u66b7\u66b8\u66b9\u66ba\u66bb\u66bc\u66bd\u66be\u66bf\u66c0\u66c1\u66c2\u66c3\u66c4\u66c5\u66c6\u66c7\u66c8\u66c9\u66ca\u66cb\u66cc\u66cd\u66ce\u66cf\u66d0\u66d1\u66d2\u66d3\u66d4\u66d5\u66d6\u66d7\u66d8\u66d9\u66da\u66db\u66dc\u66dd\u66de\u66df\u66e0\u66e1\u66e2\u66e3\u66e4\u66e5\u66e6\u66e7\u66e8\u66e9\u66ea\u66eb\u66ec\u66ed\u66ee\u66ef\u66f0\u66f1\u66f2\u66f3\u66f4\u66f5\u66f6\u66f7\u66f8\u66f9\u66fa\u66fb\u66fc\u66fd\u66fe\u66ff\u6700\u6701\u6702\u6703\u6704\u6705\u6706\u6707\u6708\u6709\u670a\u670b\u670c\u670d\u670e\u670f\u6710\u6711\u6712\u6713\u6714\u6715\u6716\u6717\u6718\u6719\u671a\u671b\u671c\u671d\u671e\u671f\u6720\u6721\u6722\u6723\u6724\u6725\u6726\u6727\u6728\u6729\u672a\u672b\u672c\u672d\u672e\u672f\u6730\u6731\u6732\u6733\u6734\u6735\u6736\u6737\u6738\u6739\u673a\u673b\u673c\u673d\u673e\u673f\u6740\u6741\u6742\u6743\u6744\u6745\u6746\u6747\u6748\u6749\u674a\u674b\u674c\u674d\u674e\u674f\u6750\u6751\u6752\u6753\u6754\u6755\u6756\u6757\u6758\u6759\u675a\u675b\u675c\u675d\u675e\u675f\u6760\u6761\u6762\u6763\u6764\u6765\u6766\u6767\u6768\u6769\u676a\u676b\u676c\u676d\u676e\u676f\u6770\u6771\u6772\u6773\u6774\u6775\u6776\u6777\u6778\u6779\u677a\u677b\u677c\u677d\u677e\u677f\u6780\u6781\u6782\u6783\u6784\u6785\u6786\u6787\u6788\u6789\u678a\u678b\u678c\u678d\u678e\u678f\u6790\u6791\u6792\u6793\u6794\u6795\u6796\u6797\u6798\u6799\u679a\u679b\u679c\u679d\u679e\u679f\u67a0\u67a1\u67a2\u67a3\u67a4\u67a5\u67a6\u67a7\u67a8\u67a9\u67aa\u67ab\u67ac\u67ad\u67ae\u67af\u67b0\u67b1\u67b2\u67b3\u67b4\u67b5\u67b6\u67b7\u67b8\u67b9\u67ba\u67bb\u67bc\u67bd\u67be\u67bf\u67c0\u67c1\u67c2\u67c3\u67c4\u67c5\u67c6\u67c7\u67c8\u67c9\u67ca\u67cb\u67cc\u67cd\u67ce\u67cf\u67d0\u67d1\u67d2\u67d3\u67d4\u67d5\u67d6\u67d7\u67d8\u67d9\u67da\u67db\u67dc\u67dd\u67de\u67df\u67e0\u67e1\u67e2\u67e3\u67e4\u67e5\u67e6\u67e7\u67e8\u67e9\u67ea\u67eb\u67ec\u67ed\u67ee\u67ef\u67f0\u67f1\u67f2\u67f3\u67f4\u67f5\u67f6\u67f7\u67f8\u67f9\u67fa\u67fb\u67fc\u67fd\u67fe\u67ff\u6800\u6801\u6802\u6803\u6804\u6805\u6806\u6807\u6808\u6809\u680a\u680b\u680c\u680d\u680e\u680f\u6810\u6811\u6812\u6813\u6814\u6815\u6816\u6817\u6818\u6819\u681a\u681b\u681c\u681d\u681e\u681f\u6820\u6821\u6822\u6823\u6824\u6825\u6826\u6827\u6828\u6829\u682a\u682b\u682c\u682d\u682e\u682f\u6830\u6831\u6832\u6833\u6834\u6835\u6836\u6837\u6838\u6839\u683a\u683b\u683c\u683d\u683e\u683f\u6840\u6841\u6842\u6843\u6844\u6845\u6846\u6847\u6848\u6849\u684a\u684b\u684c\u684d\u684e\u684f\u6850\u6851\u6852\u6853\u6854\u6855\u6856\u6857\u6858\u6859\u685a\u685b\u685c\u685d\u685e\u685f\u6860\u6861\u6862\u6863\u6864\u6865\u6866\u6867\u6868\u6869\u686a\u686b\u686c\u686d\u686e\u686f\u6870\u6871\u6872\u6873\u6874\u6875\u6876\u6877\u6878\u6879\u687a\u687b\u687c\u687d\u687e\u687f\u6880\u6881\u6882\u6883\u6884\u6885\u6886\u6887\u6888\u6889\u688a\u688b\u688c\u688d\u688e\u688f\u6890\u6891\u6892\u6893\u6894\u6895\u6896\u6897\u6898\u6899\u689a\u689b\u689c\u689d\u689e\u689f\u68a0\u68a1\u68a2\u68a3\u68a4\u68a5\u68a6\u68a7\u68a8\u68a9\u68aa\u68ab\u68ac\u68ad\u68ae\u68af\u68b0\u68b1\u68b2\u68b3\u68b4\u68b5\u68b6\u68b7\u68b8\u68b9\u68ba\u68bb\u68bc\u68bd\u68be\u68bf\u68c0\u68c1\u68c2\u68c3\u68c4\u68c5\u68c6\u68c7\u68c8\u68c9\u68ca\u68cb\u68cc\u68cd\u68ce\u68cf\u68d0\u68d1\u68d2\u68d3\u68d4\u68d5\u68d6\u68d7\u68d8\u68d9\u68da\u68db\u68dc\u68dd\u68de\u68df\u68e0\u68e1\u68e2\u68e3\u68e4\u68e5\u68e6\u68e7\u68e8\u68e9\u68ea\u68eb\u68ec\u68ed\u68ee\u68ef\u68f0\u68f1\u68f2\u68f3\u68f4\u68f5\u68f6\u68f7\u68f8\u68f9\u68fa\u68fb\u68fc\u68fd\u68fe\u68ff\u6900\u6901\u6902\u6903\u6904\u6905\u6906\u6907\u6908\u6909\u690a\u690b\u690c\u690d\u690e\u690f\u6910\u6911\u6912\u6913\u6914\u6915\u6916\u6917\u6918\u6919\u691a\u691b\u691c\u691d\u691e\u691f\u6920\u6921\u6922\u6923\u6924\u6925\u6926\u6927\u6928\u6929\u692a\u692b\u692c\u692d\u692e\u692f\u6930\u6931\u6932\u6933\u6934\u6935\u6936\u6937\u6938\u6939\u693a\u693b\u693c\u693d\u693e\u693f\u6940\u6941\u6942\u6943\u6944\u6945\u6946\u6947\u6948\u6949\u694a\u694b\u694c\u694d\u694e\u694f\u6950\u6951\u6952\u6953\u6954\u6955\u6956\u6957\u6958\u6959\u695a\u695b\u695c\u695d\u695e\u695f\u6960\u6961\u6962\u6963\u6964\u6965\u6966\u6967\u6968\u6969\u696a\u696b\u696c\u696d\u696e\u696f\u6970\u6971\u6972\u6973\u6974\u6975\u6976\u6977\u6978\u6979\u697a\u697b\u697c\u697d\u697e\u697f\u6980\u6981\u6982\u6983\u6984\u6985\u6986\u6987\u6988\u6989\u698a\u698b\u698c\u698d\u698e\u698f\u6990\u6991\u6992\u6993\u6994\u6995\u6996\u6997\u6998\u6999\u699a\u699b\u699c\u699d\u699e\u699f\u69a0\u69a1\u69a2\u69a3\u69a4\u69a5\u69a6\u69a7\u69a8\u69a9\u69aa\u69ab\u69ac\u69ad\u69ae\u69af\u69b0\u69b1\u69b2\u69b3\u69b4\u69b5\u69b6\u69b7\u69b8\u69b9\u69ba\u69bb\u69bc\u69bd\u69be\u69bf\u69c0\u69c1\u69c2\u69c3\u69c4\u69c5\u69c6\u69c7\u69c8\u69c9\u69ca\u69cb\u69cc\u69cd\u69ce\u69cf\u69d0\u69d1\u69d2\u69d3\u69d4\u69d5\u69d6\u69d7\u69d8\u69d9\u69da\u69db\u69dc\u69dd\u69de\u69df\u69e0\u69e1\u69e2\u69e3\u69e4\u69e5\u69e6\u69e7\u69e8\u69e9\u69ea\u69eb\u69ec\u69ed\u69ee\u69ef\u69f0\u69f1\u69f2\u69f3\u69f4\u69f5\u69f6\u69f7\u69f8\u69f9\u69fa\u69fb\u69fc\u69fd\u69fe\u69ff\u6a00\u6a01\u6a02\u6a03\u6a04\u6a05\u6a06\u6a07\u6a08\u6a09\u6a0a\u6a0b\u6a0c\u6a0d\u6a0e\u6a0f\u6a10\u6a11\u6a12\u6a13\u6a14\u6a15\u6a16\u6a17\u6a18\u6a19\u6a1a\u6a1b\u6a1c\u6a1d\u6a1e\u6a1f\u6a20\u6a21\u6a22\u6a23\u6a24\u6a25\u6a26\u6a27\u6a28\u6a29\u6a2a\u6a2b\u6a2c\u6a2d\u6a2e\u6a2f\u6a30\u6a31\u6a32\u6a33\u6a34\u6a35\u6a36\u6a37\u6a38\u6a39\u6a3a\u6a3b\u6a3c\u6a3d\u6a3e\u6a3f\u6a40\u6a41\u6a42\u6a43\u6a44\u6a45\u6a46\u6a47\u6a48\u6a49\u6a4a\u6a4b\u6a4c\u6a4d\u6a4e\u6a4f\u6a50\u6a51\u6a52\u6a53\u6a54\u6a55\u6a56\u6a57\u6a58\u6a59\u6a5a\u6a5b\u6a5c\u6a5d\u6a5e\u6a5f\u6a60\u6a61\u6a62\u6a63\u6a64\u6a65\u6a66\u6a67\u6a68\u6a69\u6a6a\u6a6b\u6a6c\u6a6d\u6a6e\u6a6f\u6a70\u6a71\u6a72\u6a73\u6a74\u6a75\u6a76\u6a77\u6a78\u6a79\u6a7a\u6a7b\u6a7c\u6a7d\u6a7e\u6a7f\u6a80\u6a81\u6a82\u6a83\u6a84\u6a85\u6a86\u6a87\u6a88\u6a89\u6a8a\u6a8b\u6a8c\u6a8d\u6a8e\u6a8f\u6a90\u6a91\u6a92\u6a93\u6a94\u6a95\u6a96\u6a97\u6a98\u6a99\u6a9a\u6a9b\u6a9c\u6a9d\u6a9e\u6a9f\u6aa0\u6aa1\u6aa2\u6aa3\u6aa4\u6aa5\u6aa6\u6aa7\u6aa8\u6aa9\u6aaa\u6aab\u6aac\u6aad\u6aae\u6aaf\u6ab0\u6ab1\u6ab2\u6ab3\u6ab4\u6ab5\u6ab6\u6ab7\u6ab8\u6ab9\u6aba\u6abb\u6abc\u6abd\u6abe\u6abf\u6ac0\u6ac1\u6ac2\u6ac3\u6ac4\u6ac5\u6ac6\u6ac7\u6ac8\u6ac9\u6aca\u6acb\u6acc\u6acd\u6ace\u6acf\u6ad0\u6ad1\u6ad2\u6ad3\u6ad4\u6ad5\u6ad6\u6ad7\u6ad8\u6ad9\u6ada\u6adb\u6adc\u6add\u6ade\u6adf\u6ae0\u6ae1\u6ae2\u6ae3\u6ae4\u6ae5\u6ae6\u6ae7\u6ae8\u6ae9\u6aea\u6aeb\u6aec\u6aed\u6aee\u6aef\u6af0\u6af1\u6af2\u6af3\u6af4\u6af5\u6af6\u6af7\u6af8\u6af9\u6afa\u6afb\u6afc\u6afd\u6afe\u6aff\u6b00\u6b01\u6b02\u6b03\u6b04\u6b05\u6b06\u6b07\u6b08\u6b09\u6b0a\u6b0b\u6b0c\u6b0d\u6b0e\u6b0f\u6b10\u6b11\u6b12\u6b13\u6b14\u6b15\u6b16\u6b17\u6b18\u6b19\u6b1a\u6b1b\u6b1c\u6b1d\u6b1e\u6b1f\u6b20\u6b21\u6b22\u6b23\u6b24\u6b25\u6b26\u6b27\u6b28\u6b29\u6b2a\u6b2b\u6b2c\u6b2d\u6b2e\u6b2f\u6b30\u6b31\u6b32\u6b33\u6b34\u6b35\u6b36\u6b37\u6b38\u6b39\u6b3a\u6b3b\u6b3c\u6b3d\u6b3e\u6b3f\u6b40\u6b41\u6b42\u6b43\u6b44\u6b45\u6b46\u6b47\u6b48\u6b49\u6b4a\u6b4b\u6b4c\u6b4d\u6b4e\u6b4f\u6b50\u6b51\u6b52\u6b53\u6b54\u6b55\u6b56\u6b57\u6b58\u6b59\u6b5a\u6b5b\u6b5c\u6b5d\u6b5e\u6b5f\u6b60\u6b61\u6b62\u6b63\u6b64\u6b65\u6b66\u6b67\u6b68\u6b69\u6b6a\u6b6b\u6b6c\u6b6d\u6b6e\u6b6f\u6b70\u6b71\u6b72\u6b73\u6b74\u6b75\u6b76\u6b77\u6b78\u6b79\u6b7a\u6b7b\u6b7c\u6b7d\u6b7e\u6b7f\u6b80\u6b81\u6b82\u6b83\u6b84\u6b85\u6b86\u6b87\u6b88\u6b89\u6b8a\u6b8b\u6b8c\u6b8d\u6b8e\u6b8f\u6b90\u6b91\u6b92\u6b93\u6b94\u6b95\u6b96\u6b97\u6b98\u6b99\u6b9a\u6b9b\u6b9c\u6b9d\u6b9e\u6b9f\u6ba0\u6ba1\u6ba2\u6ba3\u6ba4\u6ba5\u6ba6\u6ba7\u6ba8\u6ba9\u6baa\u6bab\u6bac\u6bad\u6bae\u6baf\u6bb0\u6bb1\u6bb2\u6bb3\u6bb4\u6bb5\u6bb6\u6bb7\u6bb8\u6bb9\u6bba\u6bbb\u6bbc\u6bbd\u6bbe\u6bbf\u6bc0\u6bc1\u6bc2\u6bc3\u6bc4\u6bc5\u6bc6\u6bc7\u6bc8\u6bc9\u6bca\u6bcb\u6bcc\u6bcd\u6bce\u6bcf\u6bd0\u6bd1\u6bd2\u6bd3\u6bd4\u6bd5\u6bd6\u6bd7\u6bd8\u6bd9\u6bda\u6bdb\u6bdc\u6bdd\u6bde\u6bdf\u6be0\u6be1\u6be2\u6be3\u6be4\u6be5\u6be6\u6be7\u6be8\u6be9\u6bea\u6beb\u6bec\u6bed\u6bee\u6bef\u6bf0\u6bf1\u6bf2\u6bf3\u6bf4\u6bf5\u6bf6\u6bf7\u6bf8\u6bf9\u6bfa\u6bfb\u6bfc\u6bfd\u6bfe\u6bff\u6c00\u6c01\u6c02\u6c03\u6c04\u6c05\u6c06\u6c07\u6c08\u6c09\u6c0a\u6c0b\u6c0c\u6c0d\u6c0e\u6c0f\u6c10\u6c11\u6c12\u6c13\u6c14\u6c15\u6c16\u6c17\u6c18\u6c19\u6c1a\u6c1b\u6c1c\u6c1d\u6c1e\u6c1f\u6c20\u6c21\u6c22\u6c23\u6c24\u6c25\u6c26\u6c27\u6c28\u6c29\u6c2a\u6c2b\u6c2c\u6c2d\u6c2e\u6c2f\u6c30\u6c31\u6c32\u6c33\u6c34\u6c35\u6c36\u6c37\u6c38\u6c39\u6c3a\u6c3b\u6c3c\u6c3d\u6c3e\u6c3f\u6c40\u6c41\u6c42\u6c43\u6c44\u6c45\u6c46\u6c47\u6c48\u6c49\u6c4a\u6c4b\u6c4c\u6c4d\u6c4e\u6c4f\u6c50\u6c51\u6c52\u6c53\u6c54\u6c55\u6c56\u6c57\u6c58\u6c59\u6c5a\u6c5b\u6c5c\u6c5d\u6c5e\u6c5f\u6c60\u6c61\u6c62\u6c63\u6c64\u6c65\u6c66\u6c67\u6c68\u6c69\u6c6a\u6c6b\u6c6c\u6c6d\u6c6e\u6c6f\u6c70\u6c71\u6c72\u6c73\u6c74\u6c75\u6c76\u6c77\u6c78\u6c79\u6c7a\u6c7b\u6c7c\u6c7d\u6c7e\u6c7f\u6c80\u6c81\u6c82\u6c83\u6c84\u6c85\u6c86\u6c87\u6c88\u6c89\u6c8a\u6c8b\u6c8c\u6c8d\u6c8e\u6c8f\u6c90\u6c91\u6c92\u6c93\u6c94\u6c95\u6c96\u6c97\u6c98\u6c99\u6c9a\u6c9b\u6c9c\u6c9d\u6c9e\u6c9f\u6ca0\u6ca1\u6ca2\u6ca3\u6ca4\u6ca5\u6ca6\u6ca7\u6ca8\u6ca9\u6caa\u6cab\u6cac\u6cad\u6cae\u6caf\u6cb0\u6cb1\u6cb2\u6cb3\u6cb4\u6cb5\u6cb6\u6cb7\u6cb8\u6cb9\u6cba\u6cbb\u6cbc\u6cbd\u6cbe\u6cbf\u6cc0\u6cc1\u6cc2\u6cc3\u6cc4\u6cc5\u6cc6\u6cc7\u6cc8\u6cc9\u6cca\u6ccb\u6ccc\u6ccd\u6cce\u6ccf\u6cd0\u6cd1\u6cd2\u6cd3\u6cd4\u6cd5\u6cd6\u6cd7\u6cd8\u6cd9\u6cda\u6cdb\u6cdc\u6cdd\u6cde\u6cdf\u6ce0\u6ce1\u6ce2\u6ce3\u6ce4\u6ce5\u6ce6\u6ce7\u6ce8\u6ce9\u6cea\u6ceb\u6cec\u6ced\u6cee\u6cef\u6cf0\u6cf1\u6cf2\u6cf3\u6cf4\u6cf5\u6cf6\u6cf7\u6cf8\u6cf9\u6cfa\u6cfb\u6cfc\u6cfd\u6cfe\u6cff\u6d00\u6d01\u6d02\u6d03\u6d04\u6d05\u6d06\u6d07\u6d08\u6d09\u6d0a\u6d0b\u6d0c\u6d0d\u6d0e\u6d0f\u6d10\u6d11\u6d12\u6d13\u6d14\u6d15\u6d16\u6d17\u6d18\u6d19\u6d1a\u6d1b\u6d1c\u6d1d\u6d1e\u6d1f\u6d20\u6d21\u6d22\u6d23\u6d24\u6d25\u6d26\u6d27\u6d28\u6d29\u6d2a\u6d2b\u6d2c\u6d2d\u6d2e\u6d2f\u6d30\u6d31\u6d32\u6d33\u6d34\u6d35\u6d36\u6d37\u6d38\u6d39\u6d3a\u6d3b\u6d3c\u6d3d\u6d3e\u6d3f\u6d40\u6d41\u6d42\u6d43\u6d44\u6d45\u6d46\u6d47\u6d48\u6d49\u6d4a\u6d4b\u6d4c\u6d4d\u6d4e\u6d4f\u6d50\u6d51\u6d52\u6d53\u6d54\u6d55\u6d56\u6d57\u6d58\u6d59\u6d5a\u6d5b\u6d5c\u6d5d\u6d5e\u6d5f\u6d60\u6d61\u6d62\u6d63\u6d64\u6d65\u6d66\u6d67\u6d68\u6d69\u6d6a\u6d6b\u6d6c\u6d6d\u6d6e\u6d6f\u6d70\u6d71\u6d72\u6d73\u6d74\u6d75\u6d76\u6d77\u6d78\u6d79\u6d7a\u6d7b\u6d7c\u6d7d\u6d7e\u6d7f\u6d80\u6d81\u6d82\u6d83\u6d84\u6d85\u6d86\u6d87\u6d88\u6d89\u6d8a\u6d8b\u6d8c\u6d8d\u6d8e\u6d8f\u6d90\u6d91\u6d92\u6d93\u6d94\u6d95\u6d96\u6d97\u6d98\u6d99\u6d9a\u6d9b\u6d9c\u6d9d\u6d9e\u6d9f\u6da0\u6da1\u6da2\u6da3\u6da4\u6da5\u6da6\u6da7\u6da8\u6da9\u6daa\u6dab\u6dac\u6dad\u6dae\u6daf\u6db0\u6db1\u6db2\u6db3\u6db4\u6db5\u6db6\u6db7\u6db8\u6db9\u6dba\u6dbb\u6dbc\u6dbd\u6dbe\u6dbf\u6dc0\u6dc1\u6dc2\u6dc3\u6dc4\u6dc5\u6dc6\u6dc7\u6dc8\u6dc9\u6dca\u6dcb\u6dcc\u6dcd\u6dce\u6dcf\u6dd0\u6dd1\u6dd2\u6dd3\u6dd4\u6dd5\u6dd6\u6dd7\u6dd8\u6dd9\u6dda\u6ddb\u6ddc\u6ddd\u6dde\u6ddf\u6de0\u6de1\u6de2\u6de3\u6de4\u6de5\u6de6\u6de7\u6de8\u6de9\u6dea\u6deb\u6dec\u6ded\u6dee\u6def\u6df0\u6df1\u6df2\u6df3\u6df4\u6df5\u6df6\u6df7\u6df8\u6df9\u6dfa\u6dfb\u6dfc\u6dfd\u6dfe\u6dff\u6e00\u6e01\u6e02\u6e03\u6e04\u6e05\u6e06\u6e07\u6e08\u6e09\u6e0a\u6e0b\u6e0c\u6e0d\u6e0e\u6e0f\u6e10\u6e11\u6e12\u6e13\u6e14\u6e15\u6e16\u6e17\u6e18\u6e19\u6e1a\u6e1b\u6e1c\u6e1d\u6e1e\u6e1f\u6e20\u6e21\u6e22\u6e23\u6e24\u6e25\u6e26\u6e27\u6e28\u6e29\u6e2a\u6e2b\u6e2c\u6e2d\u6e2e\u6e2f\u6e30\u6e31\u6e32\u6e33\u6e34\u6e35\u6e36\u6e37\u6e38\u6e39\u6e3a\u6e3b\u6e3c\u6e3d\u6e3e\u6e3f\u6e40\u6e41\u6e42\u6e43\u6e44\u6e45\u6e46\u6e47\u6e48\u6e49\u6e4a\u6e4b\u6e4c\u6e4d\u6e4e\u6e4f\u6e50\u6e51\u6e52\u6e53\u6e54\u6e55\u6e56\u6e57\u6e58\u6e59\u6e5a\u6e5b\u6e5c\u6e5d\u6e5e\u6e5f\u6e60\u6e61\u6e62\u6e63\u6e64\u6e65\u6e66\u6e67\u6e68\u6e69\u6e6a\u6e6b\u6e6c\u6e6d\u6e6e\u6e6f\u6e70\u6e71\u6e72\u6e73\u6e74\u6e75\u6e76\u6e77\u6e78\u6e79\u6e7a\u6e7b\u6e7c\u6e7d\u6e7e\u6e7f\u6e80\u6e81\u6e82\u6e83\u6e84\u6e85\u6e86\u6e87\u6e88\u6e89\u6e8a\u6e8b\u6e8c\u6e8d\u6e8e\u6e8f\u6e90\u6e91\u6e92\u6e93\u6e94\u6e95\u6e96\u6e97\u6e98\u6e99\u6e9a\u6e9b\u6e9c\u6e9d\u6e9e\u6e9f\u6ea0\u6ea1\u6ea2\u6ea3\u6ea4\u6ea5\u6ea6\u6ea7\u6ea8\u6ea9\u6eaa\u6eab\u6eac\u6ead\u6eae\u6eaf\u6eb0\u6eb1\u6eb2\u6eb3\u6eb4\u6eb5\u6eb6\u6eb7\u6eb8\u6eb9\u6eba\u6ebb\u6ebc\u6ebd\u6ebe\u6ebf\u6ec0\u6ec1\u6ec2\u6ec3\u6ec4\u6ec5\u6ec6\u6ec7\u6ec8\u6ec9\u6eca\u6ecb\u6ecc\u6ecd\u6ece\u6ecf\u6ed0\u6ed1\u6ed2\u6ed3\u6ed4\u6ed5\u6ed6\u6ed7\u6ed8\u6ed9\u6eda\u6edb\u6edc\u6edd\u6ede\u6edf\u6ee0\u6ee1\u6ee2\u6ee3\u6ee4\u6ee5\u6ee6\u6ee7\u6ee8\u6ee9\u6eea\u6eeb\u6eec\u6eed\u6eee\u6eef\u6ef0\u6ef1\u6ef2\u6ef3\u6ef4\u6ef5\u6ef6\u6ef7\u6ef8\u6ef9\u6efa\u6efb\u6efc\u6efd\u6efe\u6eff\u6f00\u6f01\u6f02\u6f03\u6f04\u6f05\u6f06\u6f07\u6f08\u6f09\u6f0a\u6f0b\u6f0c\u6f0d\u6f0e\u6f0f\u6f10\u6f11\u6f12\u6f13\u6f14\u6f15\u6f16\u6f17\u6f18\u6f19\u6f1a\u6f1b\u6f1c\u6f1d\u6f1e\u6f1f\u6f20\u6f21\u6f22\u6f23\u6f24\u6f25\u6f26\u6f27\u6f28\u6f29\u6f2a\u6f2b\u6f2c\u6f2d\u6f2e\u6f2f\u6f30\u6f31\u6f32\u6f33\u6f34\u6f35\u6f36\u6f37\u6f38\u6f39\u6f3a\u6f3b\u6f3c\u6f3d\u6f3e\u6f3f\u6f40\u6f41\u6f42\u6f43\u6f44\u6f45\u6f46\u6f47\u6f48\u6f49\u6f4a\u6f4b\u6f4c\u6f4d\u6f4e\u6f4f\u6f50\u6f51\u6f52\u6f53\u6f54\u6f55\u6f56\u6f57\u6f58\u6f59\u6f5a\u6f5b\u6f5c\u6f5d\u6f5e\u6f5f\u6f60\u6f61\u6f62\u6f63\u6f64\u6f65\u6f66\u6f67\u6f68\u6f69\u6f6a\u6f6b\u6f6c\u6f6d\u6f6e\u6f6f\u6f70\u6f71\u6f72\u6f73\u6f74\u6f75\u6f76\u6f77\u6f78\u6f79\u6f7a\u6f7b\u6f7c\u6f7d\u6f7e\u6f7f\u6f80\u6f81\u6f82\u6f83\u6f84\u6f85\u6f86\u6f87\u6f88\u6f89\u6f8a\u6f8b\u6f8c\u6f8d\u6f8e\u6f8f\u6f90\u6f91\u6f92\u6f93\u6f94\u6f95\u6f96\u6f97\u6f98\u6f99\u6f9a\u6f9b\u6f9c\u6f9d\u6f9e\u6f9f\u6fa0\u6fa1\u6fa2\u6fa3\u6fa4\u6fa5\u6fa6\u6fa7\u6fa8\u6fa9\u6faa\u6fab\u6fac\u6fad\u6fae\u6faf\u6fb0\u6fb1\u6fb2\u6fb3\u6fb4\u6fb5\u6fb6\u6fb7\u6fb8\u6fb9\u6fba\u6fbb\u6fbc\u6fbd\u6fbe\u6fbf\u6fc0\u6fc1\u6fc2\u6fc3\u6fc4\u6fc5\u6fc6\u6fc7\u6fc8\u6fc9\u6fca\u6fcb\u6fcc\u6fcd\u6fce\u6fcf\u6fd0\u6fd1\u6fd2\u6fd3\u6fd4\u6fd5\u6fd6\u6fd7\u6fd8\u6fd9\u6fda\u6fdb\u6fdc\u6fdd\u6fde\u6fdf\u6fe0\u6fe1\u6fe2\u6fe3\u6fe4\u6fe5\u6fe6\u6fe7\u6fe8\u6fe9\u6fea\u6feb\u6fec\u6fed\u6fee\u6fef\u6ff0\u6ff1\u6ff2\u6ff3\u6ff4\u6ff5\u6ff6\u6ff7\u6ff8\u6ff9\u6ffa\u6ffb\u6ffc\u6ffd\u6ffe\u6fff\u7000\u7001\u7002\u7003\u7004\u7005\u7006\u7007\u7008\u7009\u700a\u700b\u700c\u700d\u700e\u700f\u7010\u7011\u7012\u7013\u7014\u7015\u7016\u7017\u7018\u7019\u701a\u701b\u701c\u701d\u701e\u701f\u7020\u7021\u7022\u7023\u7024\u7025\u7026\u7027\u7028\u7029\u702a\u702b\u702c\u702d\u702e\u702f\u7030\u7031\u7032\u7033\u7034\u7035\u7036\u7037\u7038\u7039\u703a\u703b\u703c\u703d\u703e\u703f\u7040\u7041\u7042\u7043\u7044\u7045\u7046\u7047\u7048\u7049\u704a\u704b\u704c\u704d\u704e\u704f\u7050\u7051\u7052\u7053\u7054\u7055\u7056\u7057\u7058\u7059\u705a\u705b\u705c\u705d\u705e\u705f\u7060\u7061\u7062\u7063\u7064\u7065\u7066\u7067\u7068\u7069\u706a\u706b\u706c\u706d\u706e\u706f\u7070\u7071\u7072\u7073\u7074\u7075\u7076\u7077\u7078\u7079\u707a\u707b\u707c\u707d\u707e\u707f\u7080\u7081\u7082\u7083\u7084\u7085\u7086\u7087\u7088\u7089\u708a\u708b\u708c\u708d\u708e\u708f\u7090\u7091\u7092\u7093\u7094\u7095\u7096\u7097\u7098\u7099\u709a\u709b\u709c\u709d\u709e\u709f\u70a0\u70a1\u70a2\u70a3\u70a4\u70a5\u70a6\u70a7\u70a8\u70a9\u70aa\u70ab\u70ac\u70ad\u70ae\u70af\u70b0\u70b1\u70b2\u70b3\u70b4\u70b5\u70b6\u70b7\u70b8\u70b9\u70ba\u70bb\u70bc\u70bd\u70be\u70bf\u70c0\u70c1\u70c2\u70c3\u70c4\u70c5\u70c6\u70c7\u70c8\u70c9\u70ca\u70cb\u70cc\u70cd\u70ce\u70cf\u70d0\u70d1\u70d2\u70d3\u70d4\u70d5\u70d6\u70d7\u70d8\u70d9\u70da\u70db\u70dc\u70dd\u70de\u70df\u70e0\u70e1\u70e2\u70e3\u70e4\u70e5\u70e6\u70e7\u70e8\u70e9\u70ea\u70eb\u70ec\u70ed\u70ee\u70ef\u70f0\u70f1\u70f2\u70f3\u70f4\u70f5\u70f6\u70f7\u70f8\u70f9\u70fa\u70fb\u70fc\u70fd\u70fe\u70ff\u7100\u7101\u7102\u7103\u7104\u7105\u7106\u7107\u7108\u7109\u710a\u710b\u710c\u710d\u710e\u710f\u7110\u7111\u7112\u7113\u7114\u7115\u7116\u7117\u7118\u7119\u711a\u711b\u711c\u711d\u711e\u711f\u7120\u7121\u7122\u7123\u7124\u7125\u7126\u7127\u7128\u7129\u712a\u712b\u712c\u712d\u712e\u712f\u7130\u7131\u7132\u7133\u7134\u7135\u7136\u7137\u7138\u7139\u713a\u713b\u713c\u713d\u713e\u713f\u7140\u7141\u7142\u7143\u7144\u7145\u7146\u7147\u7148\u7149\u714a\u714b\u714c\u714d\u714e\u714f\u7150\u7151\u7152\u7153\u7154\u7155\u7156\u7157\u7158\u7159\u715a\u715b\u715c\u715d\u715e\u715f\u7160\u7161\u7162\u7163\u7164\u7165\u7166\u7167\u7168\u7169\u716a\u716b\u716c\u716d\u716e\u716f\u7170\u7171\u7172\u7173\u7174\u7175\u7176\u7177\u7178\u7179\u717a\u717b\u717c\u717d\u717e\u717f\u7180\u7181\u7182\u7183\u7184\u7185\u7186\u7187\u7188\u7189\u718a\u718b\u718c\u718d\u718e\u718f\u7190\u7191\u7192\u7193\u7194\u7195\u7196\u7197\u7198\u7199\u719a\u719b\u719c\u719d\u719e\u719f\u71a0\u71a1\u71a2\u71a3\u71a4\u71a5\u71a6\u71a7\u71a8\u71a9\u71aa\u71ab\u71ac\u71ad\u71ae\u71af\u71b0\u71b1\u71b2\u71b3\u71b4\u71b5\u71b6\u71b7\u71b8\u71b9\u71ba\u71bb\u71bc\u71bd\u71be\u71bf\u71c0\u71c1\u71c2\u71c3\u71c4\u71c5\u71c6\u71c7\u71c8\u71c9\u71ca\u71cb\u71cc\u71cd\u71ce\u71cf\u71d0\u71d1\u71d2\u71d3\u71d4\u71d5\u71d6\u71d7\u71d8\u71d9\u71da\u71db\u71dc\u71dd\u71de\u71df\u71e0\u71e1\u71e2\u71e3\u71e4\u71e5\u71e6\u71e7\u71e8\u71e9\u71ea\u71eb\u71ec\u71ed\u71ee\u71ef\u71f0\u71f1\u71f2\u71f3\u71f4\u71f5\u71f6\u71f7\u71f8\u71f9\u71fa\u71fb\u71fc\u71fd\u71fe\u71ff\u7200\u7201\u7202\u7203\u7204\u7205\u7206\u7207\u7208\u7209\u720a\u720b\u720c\u720d\u720e\u720f\u7210\u7211\u7212\u7213\u7214\u7215\u7216\u7217\u7218\u7219\u721a\u721b\u721c\u721d\u721e\u721f\u7220\u7221\u7222\u7223\u7224\u7225\u7226\u7227\u7228\u7229\u722a\u722b\u722c\u722d\u722e\u722f\u7230\u7231\u7232\u7233\u7234\u7235\u7236\u7237\u7238\u7239\u723a\u723b\u723c\u723d\u723e\u723f\u7240\u7241\u7242\u7243\u7244\u7245\u7246\u7247\u7248\u7249\u724a\u724b\u724c\u724d\u724e\u724f\u7250\u7251\u7252\u7253\u7254\u7255\u7256\u7257\u7258\u7259\u725a\u725b\u725c\u725d\u725e\u725f\u7260\u7261\u7262\u7263\u7264\u7265\u7266\u7267\u7268\u7269\u726a\u726b\u726c\u726d\u726e\u726f\u7270\u7271\u7272\u7273\u7274\u7275\u7276\u7277\u7278\u7279\u727a\u727b\u727c\u727d\u727e\u727f\u7280\u7281\u7282\u7283\u7284\u7285\u7286\u7287\u7288\u7289\u728a\u728b\u728c\u728d\u728e\u728f\u7290\u7291\u7292\u7293\u7294\u7295\u7296\u7297\u7298\u7299\u729a\u729b\u729c\u729d\u729e\u729f\u72a0\u72a1\u72a2\u72a3\u72a4\u72a5\u72a6\u72a7\u72a8\u72a9\u72aa\u72ab\u72ac\u72ad\u72ae\u72af\u72b0\u72b1\u72b2\u72b3\u72b4\u72b5\u72b6\u72b7\u72b8\u72b9\u72ba\u72bb\u72bc\u72bd\u72be\u72bf\u72c0\u72c1\u72c2\u72c3\u72c4\u72c5\u72c6\u72c7\u72c8\u72c9\u72ca\u72cb\u72cc\u72cd\u72ce\u72cf\u72d0\u72d1\u72d2\u72d3\u72d4\u72d5\u72d6\u72d7\u72d8\u72d9\u72da\u72db\u72dc\u72dd\u72de\u72df\u72e0\u72e1\u72e2\u72e3\u72e4\u72e5\u72e6\u72e7\u72e8\u72e9\u72ea\u72eb\u72ec\u72ed\u72ee\u72ef\u72f0\u72f1\u72f2\u72f3\u72f4\u72f5\u72f6\u72f7\u72f8\u72f9\u72fa\u72fb\u72fc\u72fd\u72fe\u72ff\u7300\u7301\u7302\u7303\u7304\u7305\u7306\u7307\u7308\u7309\u730a\u730b\u730c\u730d\u730e\u730f\u7310\u7311\u7312\u7313\u7314\u7315\u7316\u7317\u7318\u7319\u731a\u731b\u731c\u731d\u731e\u731f\u7320\u7321\u7322\u7323\u7324\u7325\u7326\u7327\u7328\u7329\u732a\u732b\u732c\u732d\u732e\u732f\u7330\u7331\u7332\u7333\u7334\u7335\u7336\u7337\u7338\u7339\u733a\u733b\u733c\u733d\u733e\u733f\u7340\u7341\u7342\u7343\u7344\u7345\u7346\u7347\u7348\u7349\u734a\u734b\u734c\u734d\u734e\u734f\u7350\u7351\u7352\u7353\u7354\u7355\u7356\u7357\u7358\u7359\u735a\u735b\u735c\u735d\u735e\u735f\u7360\u7361\u7362\u7363\u7364\u7365\u7366\u7367\u7368\u7369\u736a\u736b\u736c\u736d\u736e\u736f\u7370\u7371\u7372\u7373\u7374\u7375\u7376\u7377\u7378\u7379\u737a\u737b\u737c\u737d\u737e\u737f\u7380\u7381\u7382\u7383\u7384\u7385\u7386\u7387\u7388\u7389\u738a\u738b\u738c\u738d\u738e\u738f\u7390\u7391\u7392\u7393\u7394\u7395\u7396\u7397\u7398\u7399\u739a\u739b\u739c\u739d\u739e\u739f\u73a0\u73a1\u73a2\u73a3\u73a4\u73a5\u73a6\u73a7\u73a8\u73a9\u73aa\u73ab\u73ac\u73ad\u73ae\u73af\u73b0\u73b1\u73b2\u73b3\u73b4\u73b5\u73b6\u73b7\u73b8\u73b9\u73ba\u73bb\u73bc\u73bd\u73be\u73bf\u73c0\u73c1\u73c2\u73c3\u73c4\u73c5\u73c6\u73c7\u73c8\u73c9\u73ca\u73cb\u73cc\u73cd\u73ce\u73cf\u73d0\u73d1\u73d2\u73d3\u73d4\u73d5\u73d6\u73d7\u73d8\u73d9\u73da\u73db\u73dc\u73dd\u73de\u73df\u73e0\u73e1\u73e2\u73e3\u73e4\u73e5\u73e6\u73e7\u73e8\u73e9\u73ea\u73eb\u73ec\u73ed\u73ee\u73ef\u73f0\u73f1\u73f2\u73f3\u73f4\u73f5\u73f6\u73f7\u73f8\u73f9\u73fa\u73fb\u73fc\u73fd\u73fe\u73ff\u7400\u7401\u7402\u7403\u7404\u7405\u7406\u7407\u7408\u7409\u740a\u740b\u740c\u740d\u740e\u740f\u7410\u7411\u7412\u7413\u7414\u7415\u7416\u7417\u7418\u7419\u741a\u741b\u741c\u741d\u741e\u741f\u7420\u7421\u7422\u7423\u7424\u7425\u7426\u7427\u7428\u7429\u742a\u742b\u742c\u742d\u742e\u742f\u7430\u7431\u7432\u7433\u7434\u7435\u7436\u7437\u7438\u7439\u743a\u743b\u743c\u743d\u743e\u743f\u7440\u7441\u7442\u7443\u7444\u7445\u7446\u7447\u7448\u7449\u744a\u744b\u744c\u744d\u744e\u744f\u7450\u7451\u7452\u7453\u7454\u7455\u7456\u7457\u7458\u7459\u745a\u745b\u745c\u745d\u745e\u745f\u7460\u7461\u7462\u7463\u7464\u7465\u7466\u7467\u7468\u7469\u746a\u746b\u746c\u746d\u746e\u746f\u7470\u7471\u7472\u7473\u7474\u7475\u7476\u7477\u7478\u7479\u747a\u747b\u747c\u747d\u747e\u747f\u7480\u7481\u7482\u7483\u7484\u7485\u7486\u7487\u7488\u7489\u748a\u748b\u748c\u748d\u748e\u748f\u7490\u7491\u7492\u7493\u7494\u7495\u7496\u7497\u7498\u7499\u749a\u749b\u749c\u749d\u749e\u749f\u74a0\u74a1\u74a2\u74a3\u74a4\u74a5\u74a6\u74a7\u74a8\u74a9\u74aa\u74ab\u74ac\u74ad\u74ae\u74af\u74b0\u74b1\u74b2\u74b3\u74b4\u74b5\u74b6\u74b7\u74b8\u74b9\u74ba\u74bb\u74bc\u74bd\u74be\u74bf\u74c0\u74c1\u74c2\u74c3\u74c4\u74c5\u74c6\u74c7\u74c8\u74c9\u74ca\u74cb\u74cc\u74cd\u74ce\u74cf\u74d0\u74d1\u74d2\u74d3\u74d4\u74d5\u74d6\u74d7\u74d8\u74d9\u74da\u74db\u74dc\u74dd\u74de\u74df\u74e0\u74e1\u74e2\u74e3\u74e4\u74e5\u74e6\u74e7\u74e8\u74e9\u74ea\u74eb\u74ec\u74ed\u74ee\u74ef\u74f0\u74f1\u74f2\u74f3\u74f4\u74f5\u74f6\u74f7\u74f8\u74f9\u74fa\u74fb\u74fc\u74fd\u74fe\u74ff\u7500\u7501\u7502\u7503\u7504\u7505\u7506\u7507\u7508\u7509\u750a\u750b\u750c\u750d\u750e\u750f\u7510\u7511\u7512\u7513\u7514\u7515\u7516\u7517\u7518\u7519\u751a\u751b\u751c\u751d\u751e\u751f\u7520\u7521\u7522\u7523\u7524\u7525\u7526\u7527\u7528\u7529\u752a\u752b\u752c\u752d\u752e\u752f\u7530\u7531\u7532\u7533\u7534\u7535\u7536\u7537\u7538\u7539\u753a\u753b\u753c\u753d\u753e\u753f\u7540\u7541\u7542\u7543\u7544\u7545\u7546\u7547\u7548\u7549\u754a\u754b\u754c\u754d\u754e\u754f\u7550\u7551\u7552\u7553\u7554\u7555\u7556\u7557\u7558\u7559\u755a\u755b\u755c\u755d\u755e\u755f\u7560\u7561\u7562\u7563\u7564\u7565\u7566\u7567\u7568\u7569\u756a\u756b\u756c\u756d\u756e\u756f\u7570\u7571\u7572\u7573\u7574\u7575\u7576\u7577\u7578\u7579\u757a\u757b\u757c\u757d\u757e\u757f\u7580\u7581\u7582\u7583\u7584\u7585\u7586\u7587\u7588\u7589\u758a\u758b\u758c\u758d\u758e\u758f\u7590\u7591\u7592\u7593\u7594\u7595\u7596\u7597\u7598\u7599\u759a\u759b\u759c\u759d\u759e\u759f\u75a0\u75a1\u75a2\u75a3\u75a4\u75a5\u75a6\u75a7\u75a8\u75a9\u75aa\u75ab\u75ac\u75ad\u75ae\u75af\u75b0\u75b1\u75b2\u75b3\u75b4\u75b5\u75b6\u75b7\u75b8\u75b9\u75ba\u75bb\u75bc\u75bd\u75be\u75bf\u75c0\u75c1\u75c2\u75c3\u75c4\u75c5\u75c6\u75c7\u75c8\u75c9\u75ca\u75cb\u75cc\u75cd\u75ce\u75cf\u75d0\u75d1\u75d2\u75d3\u75d4\u75d5\u75d6\u75d7\u75d8\u75d9\u75da\u75db\u75dc\u75dd\u75de\u75df\u75e0\u75e1\u75e2\u75e3\u75e4\u75e5\u75e6\u75e7\u75e8\u75e9\u75ea\u75eb\u75ec\u75ed\u75ee\u75ef\u75f0\u75f1\u75f2\u75f3\u75f4\u75f5\u75f6\u75f7\u75f8\u75f9\u75fa\u75fb\u75fc\u75fd\u75fe\u75ff\u7600\u7601\u7602\u7603\u7604\u7605\u7606\u7607\u7608\u7609\u760a\u760b\u760c\u760d\u760e\u760f\u7610\u7611\u7612\u7613\u7614\u7615\u7616\u7617\u7618\u7619\u761a\u761b\u761c\u761d\u761e\u761f\u7620\u7621\u7622\u7623\u7624\u7625\u7626\u7627\u7628\u7629\u762a\u762b\u762c\u762d\u762e\u762f\u7630\u7631\u7632\u7633\u7634\u7635\u7636\u7637\u7638\u7639\u763a\u763b\u763c\u763d\u763e\u763f\u7640\u7641\u7642\u7643\u7644\u7645\u7646\u7647\u7648\u7649\u764a\u764b\u764c\u764d\u764e\u764f\u7650\u7651\u7652\u7653\u7654\u7655\u7656\u7657\u7658\u7659\u765a\u765b\u765c\u765d\u765e\u765f\u7660\u7661\u7662\u7663\u7664\u7665\u7666\u7667\u7668\u7669\u766a\u766b\u766c\u766d\u766e\u766f\u7670\u7671\u7672\u7673\u7674\u7675\u7676\u7677\u7678\u7679\u767a\u767b\u767c\u767d\u767e\u767f\u7680\u7681\u7682\u7683\u7684\u7685\u7686\u7687\u7688\u7689\u768a\u768b\u768c\u768d\u768e\u768f\u7690\u7691\u7692\u7693\u7694\u7695\u7696\u7697\u7698\u7699\u769a\u769b\u769c\u769d\u769e\u769f\u76a0\u76a1\u76a2\u76a3\u76a4\u76a5\u76a6\u76a7\u76a8\u76a9\u76aa\u76ab\u76ac\u76ad\u76ae\u76af\u76b0\u76b1\u76b2\u76b3\u76b4\u76b5\u76b6\u76b7\u76b8\u76b9\u76ba\u76bb\u76bc\u76bd\u76be\u76bf\u76c0\u76c1\u76c2\u76c3\u76c4\u76c5\u76c6\u76c7\u76c8\u76c9\u76ca\u76cb\u76cc\u76cd\u76ce\u76cf\u76d0\u76d1\u76d2\u76d3\u76d4\u76d5\u76d6\u76d7\u76d8\u76d9\u76da\u76db\u76dc\u76dd\u76de\u76df\u76e0\u76e1\u76e2\u76e3\u76e4\u76e5\u76e6\u76e7\u76e8\u76e9\u76ea\u76eb\u76ec\u76ed\u76ee\u76ef\u76f0\u76f1\u76f2\u76f3\u76f4\u76f5\u76f6\u76f7\u76f8\u76f9\u76fa\u76fb\u76fc\u76fd\u76fe\u76ff\u7700\u7701\u7702\u7703\u7704\u7705\u7706\u7707\u7708\u7709\u770a\u770b\u770c\u770d\u770e\u770f\u7710\u7711\u7712\u7713\u7714\u7715\u7716\u7717\u7718\u7719\u771a\u771b\u771c\u771d\u771e\u771f\u7720\u7721\u7722\u7723\u7724\u7725\u7726\u7727\u7728\u7729\u772a\u772b\u772c\u772d\u772e\u772f\u7730\u7731\u7732\u7733\u7734\u7735\u7736\u7737\u7738\u7739\u773a\u773b\u773c\u773d\u773e\u773f\u7740\u7741\u7742\u7743\u7744\u7745\u7746\u7747\u7748\u7749\u774a\u774b\u774c\u774d\u774e\u774f\u7750\u7751\u7752\u7753\u7754\u7755\u7756\u7757\u7758\u7759\u775a\u775b\u775c\u775d\u775e\u775f\u7760\u7761\u7762\u7763\u7764\u7765\u7766\u7767\u7768\u7769\u776a\u776b\u776c\u776d\u776e\u776f\u7770\u7771\u7772\u7773\u7774\u7775\u7776\u7777\u7778\u7779\u777a\u777b\u777c\u777d\u777e\u777f\u7780\u7781\u7782\u7783\u7784\u7785\u7786\u7787\u7788\u7789\u778a\u778b\u778c\u778d\u778e\u778f\u7790\u7791\u7792\u7793\u7794\u7795\u7796\u7797\u7798\u7799\u779a\u779b\u779c\u779d\u779e\u779f\u77a0\u77a1\u77a2\u77a3\u77a4\u77a5\u77a6\u77a7\u77a8\u77a9\u77aa\u77ab\u77ac\u77ad\u77ae\u77af\u77b0\u77b1\u77b2\u77b3\u77b4\u77b5\u77b6\u77b7\u77b8\u77b9\u77ba\u77bb\u77bc\u77bd\u77be\u77bf\u77c0\u77c1\u77c2\u77c3\u77c4\u77c5\u77c6\u77c7\u77c8\u77c9\u77ca\u77cb\u77cc\u77cd\u77ce\u77cf\u77d0\u77d1\u77d2\u77d3\u77d4\u77d5\u77d6\u77d7\u77d8\u77d9\u77da\u77db\u77dc\u77dd\u77de\u77df\u77e0\u77e1\u77e2\u77e3\u77e4\u77e5\u77e6\u77e7\u77e8\u77e9\u77ea\u77eb\u77ec\u77ed\u77ee\u77ef\u77f0\u77f1\u77f2\u77f3\u77f4\u77f5\u77f6\u77f7\u77f8\u77f9\u77fa\u77fb\u77fc\u77fd\u77fe\u77ff\u7800\u7801\u7802\u7803\u7804\u7805\u7806\u7807\u7808\u7809\u780a\u780b\u780c\u780d\u780e\u780f\u7810\u7811\u7812\u7813\u7814\u7815\u7816\u7817\u7818\u7819\u781a\u781b\u781c\u781d\u781e\u781f\u7820\u7821\u7822\u7823\u7824\u7825\u7826\u7827\u7828\u7829\u782a\u782b\u782c\u782d\u782e\u782f\u7830\u7831\u7832\u7833\u7834\u7835\u7836\u7837\u7838\u7839\u783a\u783b\u783c\u783d\u783e\u783f\u7840\u7841\u7842\u7843\u7844\u7845\u7846\u7847\u7848\u7849\u784a\u784b\u784c\u784d\u784e\u784f\u7850\u7851\u7852\u7853\u7854\u7855\u7856\u7857\u7858\u7859\u785a\u785b\u785c\u785d\u785e\u785f\u7860\u7861\u7862\u7863\u7864\u7865\u7866\u7867\u7868\u7869\u786a\u786b\u786c\u786d\u786e\u786f\u7870\u7871\u7872\u7873\u7874\u7875\u7876\u7877\u7878\u7879\u787a\u787b\u787c\u787d\u787e\u787f\u7880\u7881\u7882\u7883\u7884\u7885\u7886\u7887\u7888\u7889\u788a\u788b\u788c\u788d\u788e\u788f\u7890\u7891\u7892\u7893\u7894\u7895\u7896\u7897\u7898\u7899\u789a\u789b\u789c\u789d\u789e\u789f\u78a0\u78a1\u78a2\u78a3\u78a4\u78a5\u78a6\u78a7\u78a8\u78a9\u78aa\u78ab\u78ac\u78ad\u78ae\u78af\u78b0\u78b1\u78b2\u78b3\u78b4\u78b5\u78b6\u78b7\u78b8\u78b9\u78ba\u78bb\u78bc\u78bd\u78be\u78bf\u78c0\u78c1\u78c2\u78c3\u78c4\u78c5\u78c6\u78c7\u78c8\u78c9\u78ca\u78cb\u78cc\u78cd\u78ce\u78cf\u78d0\u78d1\u78d2\u78d3\u78d4\u78d5\u78d6\u78d7\u78d8\u78d9\u78da\u78db\u78dc\u78dd\u78de\u78df\u78e0\u78e1\u78e2\u78e3\u78e4\u78e5\u78e6\u78e7\u78e8\u78e9\u78ea\u78eb\u78ec\u78ed\u78ee\u78ef\u78f0\u78f1\u78f2\u78f3\u78f4\u78f5\u78f6\u78f7\u78f8\u78f9\u78fa\u78fb\u78fc\u78fd\u78fe\u78ff\u7900\u7901\u7902\u7903\u7904\u7905\u7906\u7907\u7908\u7909\u790a\u790b\u790c\u790d\u790e\u790f\u7910\u7911\u7912\u7913\u7914\u7915\u7916\u7917\u7918\u7919\u791a\u791b\u791c\u791d\u791e\u791f\u7920\u7921\u7922\u7923\u7924\u7925\u7926\u7927\u7928\u7929\u792a\u792b\u792c\u792d\u792e\u792f\u7930\u7931\u7932\u7933\u7934\u7935\u7936\u7937\u7938\u7939\u793a\u793b\u793c\u793d\u793e\u793f\u7940\u7941\u7942\u7943\u7944\u7945\u7946\u7947\u7948\u7949\u794a\u794b\u794c\u794d\u794e\u794f\u7950\u7951\u7952\u7953\u7954\u7955\u7956\u7957\u7958\u7959\u795a\u795b\u795c\u795d\u795e\u795f\u7960\u7961\u7962\u7963\u7964\u7965\u7966\u7967\u7968\u7969\u796a\u796b\u796c\u796d\u796e\u796f\u7970\u7971\u7972\u7973\u7974\u7975\u7976\u7977\u7978\u7979\u797a\u797b\u797c\u797d\u797e\u797f\u7980\u7981\u7982\u7983\u7984\u7985\u7986\u7987\u7988\u7989\u798a\u798b\u798c\u798d\u798e\u798f\u7990\u7991\u7992\u7993\u7994\u7995\u7996\u7997\u7998\u7999\u799a\u799b\u799c\u799d\u799e\u799f\u79a0\u79a1\u79a2\u79a3\u79a4\u79a5\u79a6\u79a7\u79a8\u79a9\u79aa\u79ab\u79ac\u79ad\u79ae\u79af\u79b0\u79b1\u79b2\u79b3\u79b4\u79b5\u79b6\u79b7\u79b8\u79b9\u79ba\u79bb\u79bc\u79bd\u79be\u79bf\u79c0\u79c1\u79c2\u79c3\u79c4\u79c5\u79c6\u79c7\u79c8\u79c9\u79ca\u79cb\u79cc\u79cd\u79ce\u79cf\u79d0\u79d1\u79d2\u79d3\u79d4\u79d5\u79d6\u79d7\u79d8\u79d9\u79da\u79db\u79dc\u79dd\u79de\u79df\u79e0\u79e1\u79e2\u79e3\u79e4\u79e5\u79e6\u79e7\u79e8\u79e9\u79ea\u79eb\u79ec\u79ed\u79ee\u79ef\u79f0\u79f1\u79f2\u79f3\u79f4\u79f5\u79f6\u79f7\u79f8\u79f9\u79fa\u79fb\u79fc\u79fd\u79fe\u79ff\u7a00\u7a01\u7a02\u7a03\u7a04\u7a05\u7a06\u7a07\u7a08\u7a09\u7a0a\u7a0b\u7a0c\u7a0d\u7a0e\u7a0f\u7a10\u7a11\u7a12\u7a13\u7a14\u7a15\u7a16\u7a17\u7a18\u7a19\u7a1a\u7a1b\u7a1c\u7a1d\u7a1e\u7a1f\u7a20\u7a21\u7a22\u7a23\u7a24\u7a25\u7a26\u7a27\u7a28\u7a29\u7a2a\u7a2b\u7a2c\u7a2d\u7a2e\u7a2f\u7a30\u7a31\u7a32\u7a33\u7a34\u7a35\u7a36\u7a37\u7a38\u7a39\u7a3a\u7a3b\u7a3c\u7a3d\u7a3e\u7a3f\u7a40\u7a41\u7a42\u7a43\u7a44\u7a45\u7a46\u7a47\u7a48\u7a49\u7a4a\u7a4b\u7a4c\u7a4d\u7a4e\u7a4f\u7a50\u7a51\u7a52\u7a53\u7a54\u7a55\u7a56\u7a57\u7a58\u7a59\u7a5a\u7a5b\u7a5c\u7a5d\u7a5e\u7a5f\u7a60\u7a61\u7a62\u7a63\u7a64\u7a65\u7a66\u7a67\u7a68\u7a69\u7a6a\u7a6b\u7a6c\u7a6d\u7a6e\u7a6f\u7a70\u7a71\u7a72\u7a73\u7a74\u7a75\u7a76\u7a77\u7a78\u7a79\u7a7a\u7a7b\u7a7c\u7a7d\u7a7e\u7a7f\u7a80\u7a81\u7a82\u7a83\u7a84\u7a85\u7a86\u7a87\u7a88\u7a89\u7a8a\u7a8b\u7a8c\u7a8d\u7a8e\u7a8f\u7a90\u7a91\u7a92\u7a93\u7a94\u7a95\u7a96\u7a97\u7a98\u7a99\u7a9a\u7a9b\u7a9c\u7a9d\u7a9e\u7a9f\u7aa0\u7aa1\u7aa2\u7aa3\u7aa4\u7aa5\u7aa6\u7aa7\u7aa8\u7aa9\u7aaa\u7aab\u7aac\u7aad\u7aae\u7aaf\u7ab0\u7ab1\u7ab2\u7ab3\u7ab4\u7ab5\u7ab6\u7ab7\u7ab8\u7ab9\u7aba\u7abb\u7abc\u7abd\u7abe\u7abf\u7ac0\u7ac1\u7ac2\u7ac3\u7ac4\u7ac5\u7ac6\u7ac7\u7ac8\u7ac9\u7aca\u7acb\u7acc\u7acd\u7ace\u7acf\u7ad0\u7ad1\u7ad2\u7ad3\u7ad4\u7ad5\u7ad6\u7ad7\u7ad8\u7ad9\u7ada\u7adb\u7adc\u7add\u7ade\u7adf\u7ae0\u7ae1\u7ae2\u7ae3\u7ae4\u7ae5\u7ae6\u7ae7\u7ae8\u7ae9\u7aea\u7aeb\u7aec\u7aed\u7aee\u7aef\u7af0\u7af1\u7af2\u7af3\u7af4\u7af5\u7af6\u7af7\u7af8\u7af9\u7afa\u7afb\u7afc\u7afd\u7afe\u7aff\u7b00\u7b01\u7b02\u7b03\u7b04\u7b05\u7b06\u7b07\u7b08\u7b09\u7b0a\u7b0b\u7b0c\u7b0d\u7b0e\u7b0f\u7b10\u7b11\u7b12\u7b13\u7b14\u7b15\u7b16\u7b17\u7b18\u7b19\u7b1a\u7b1b\u7b1c\u7b1d\u7b1e\u7b1f\u7b20\u7b21\u7b22\u7b23\u7b24\u7b25\u7b26\u7b27\u7b28\u7b29\u7b2a\u7b2b\u7b2c\u7b2d\u7b2e\u7b2f\u7b30\u7b31\u7b32\u7b33\u7b34\u7b35\u7b36\u7b37\u7b38\u7b39\u7b3a\u7b3b\u7b3c\u7b3d\u7b3e\u7b3f\u7b40\u7b41\u7b42\u7b43\u7b44\u7b45\u7b46\u7b47\u7b48\u7b49\u7b4a\u7b4b\u7b4c\u7b4d\u7b4e\u7b4f\u7b50\u7b51\u7b52\u7b53\u7b54\u7b55\u7b56\u7b57\u7b58\u7b59\u7b5a\u7b5b\u7b5c\u7b5d\u7b5e\u7b5f\u7b60\u7b61\u7b62\u7b63\u7b64\u7b65\u7b66\u7b67\u7b68\u7b69\u7b6a\u7b6b\u7b6c\u7b6d\u7b6e\u7b6f\u7b70\u7b71\u7b72\u7b73\u7b74\u7b75\u7b76\u7b77\u7b78\u7b79\u7b7a\u7b7b\u7b7c\u7b7d\u7b7e\u7b7f\u7b80\u7b81\u7b82\u7b83\u7b84\u7b85\u7b86\u7b87\u7b88\u7b89\u7b8a\u7b8b\u7b8c\u7b8d\u7b8e\u7b8f\u7b90\u7b91\u7b92\u7b93\u7b94\u7b95\u7b96\u7b97\u7b98\u7b99\u7b9a\u7b9b\u7b9c\u7b9d\u7b9e\u7b9f\u7ba0\u7ba1\u7ba2\u7ba3\u7ba4\u7ba5\u7ba6\u7ba7\u7ba8\u7ba9\u7baa\u7bab\u7bac\u7bad\u7bae\u7baf\u7bb0\u7bb1\u7bb2\u7bb3\u7bb4\u7bb5\u7bb6\u7bb7\u7bb8\u7bb9\u7bba\u7bbb\u7bbc\u7bbd\u7bbe\u7bbf\u7bc0\u7bc1\u7bc2\u7bc3\u7bc4\u7bc5\u7bc6\u7bc7\u7bc8\u7bc9\u7bca\u7bcb\u7bcc\u7bcd\u7bce\u7bcf\u7bd0\u7bd1\u7bd2\u7bd3\u7bd4\u7bd5\u7bd6\u7bd7\u7bd8\u7bd9\u7bda\u7bdb\u7bdc\u7bdd\u7bde\u7bdf\u7be0\u7be1\u7be2\u7be3\u7be4\u7be5\u7be6\u7be7\u7be8\u7be9\u7bea\u7beb\u7bec\u7bed\u7bee\u7bef\u7bf0\u7bf1\u7bf2\u7bf3\u7bf4\u7bf5\u7bf6\u7bf7\u7bf8\u7bf9\u7bfa\u7bfb\u7bfc\u7bfd\u7bfe\u7bff\u7c00\u7c01\u7c02\u7c03\u7c04\u7c05\u7c06\u7c07\u7c08\u7c09\u7c0a\u7c0b\u7c0c\u7c0d\u7c0e\u7c0f\u7c10\u7c11\u7c12\u7c13\u7c14\u7c15\u7c16\u7c17\u7c18\u7c19\u7c1a\u7c1b\u7c1c\u7c1d\u7c1e\u7c1f\u7c20\u7c21\u7c22\u7c23\u7c24\u7c25\u7c26\u7c27\u7c28\u7c29\u7c2a\u7c2b\u7c2c\u7c2d\u7c2e\u7c2f\u7c30\u7c31\u7c32\u7c33\u7c34\u7c35\u7c36\u7c37\u7c38\u7c39\u7c3a\u7c3b\u7c3c\u7c3d\u7c3e\u7c3f\u7c40\u7c41\u7c42\u7c43\u7c44\u7c45\u7c46\u7c47\u7c48\u7c49\u7c4a\u7c4b\u7c4c\u7c4d\u7c4e\u7c4f\u7c50\u7c51\u7c52\u7c53\u7c54\u7c55\u7c56\u7c57\u7c58\u7c59\u7c5a\u7c5b\u7c5c\u7c5d\u7c5e\u7c5f\u7c60\u7c61\u7c62\u7c63\u7c64\u7c65\u7c66\u7c67\u7c68\u7c69\u7c6a\u7c6b\u7c6c\u7c6d\u7c6e\u7c6f\u7c70\u7c71\u7c72\u7c73\u7c74\u7c75\u7c76\u7c77\u7c78\u7c79\u7c7a\u7c7b\u7c7c\u7c7d\u7c7e\u7c7f\u7c80\u7c81\u7c82\u7c83\u7c84\u7c85\u7c86\u7c87\u7c88\u7c89\u7c8a\u7c8b\u7c8c\u7c8d\u7c8e\u7c8f\u7c90\u7c91\u7c92\u7c93\u7c94\u7c95\u7c96\u7c97\u7c98\u7c99\u7c9a\u7c9b\u7c9c\u7c9d\u7c9e\u7c9f\u7ca0\u7ca1\u7ca2\u7ca3\u7ca4\u7ca5\u7ca6\u7ca7\u7ca8\u7ca9\u7caa\u7cab\u7cac\u7cad\u7cae\u7caf\u7cb0\u7cb1\u7cb2\u7cb3\u7cb4\u7cb5\u7cb6\u7cb7\u7cb8\u7cb9\u7cba\u7cbb\u7cbc\u7cbd\u7cbe\u7cbf\u7cc0\u7cc1\u7cc2\u7cc3\u7cc4\u7cc5\u7cc6\u7cc7\u7cc8\u7cc9\u7cca\u7ccb\u7ccc\u7ccd\u7cce\u7ccf\u7cd0\u7cd1\u7cd2\u7cd3\u7cd4\u7cd5\u7cd6\u7cd7\u7cd8\u7cd9\u7cda\u7cdb\u7cdc\u7cdd\u7cde\u7cdf\u7ce0\u7ce1\u7ce2\u7ce3\u7ce4\u7ce5\u7ce6\u7ce7\u7ce8\u7ce9\u7cea\u7ceb\u7cec\u7ced\u7cee\u7cef\u7cf0\u7cf1\u7cf2\u7cf3\u7cf4\u7cf5\u7cf6\u7cf7\u7cf8\u7cf9\u7cfa\u7cfb\u7cfc\u7cfd\u7cfe\u7cff\u7d00\u7d01\u7d02\u7d03\u7d04\u7d05\u7d06\u7d07\u7d08\u7d09\u7d0a\u7d0b\u7d0c\u7d0d\u7d0e\u7d0f\u7d10\u7d11\u7d12\u7d13\u7d14\u7d15\u7d16\u7d17\u7d18\u7d19\u7d1a\u7d1b\u7d1c\u7d1d\u7d1e\u7d1f\u7d20\u7d21\u7d22\u7d23\u7d24\u7d25\u7d26\u7d27\u7d28\u7d29\u7d2a\u7d2b\u7d2c\u7d2d\u7d2e\u7d2f\u7d30\u7d31\u7d32\u7d33\u7d34\u7d35\u7d36\u7d37\u7d38\u7d39\u7d3a\u7d3b\u7d3c\u7d3d\u7d3e\u7d3f\u7d40\u7d41\u7d42\u7d43\u7d44\u7d45\u7d46\u7d47\u7d48\u7d49\u7d4a\u7d4b\u7d4c\u7d4d\u7d4e\u7d4f\u7d50\u7d51\u7d52\u7d53\u7d54\u7d55\u7d56\u7d57\u7d58\u7d59\u7d5a\u7d5b\u7d5c\u7d5d\u7d5e\u7d5f\u7d60\u7d61\u7d62\u7d63\u7d64\u7d65\u7d66\u7d67\u7d68\u7d69\u7d6a\u7d6b\u7d6c\u7d6d\u7d6e\u7d6f\u7d70\u7d71\u7d72\u7d73\u7d74\u7d75\u7d76\u7d77\u7d78\u7d79\u7d7a\u7d7b\u7d7c\u7d7d\u7d7e\u7d7f\u7d80\u7d81\u7d82\u7d83\u7d84\u7d85\u7d86\u7d87\u7d88\u7d89\u7d8a\u7d8b\u7d8c\u7d8d\u7d8e\u7d8f\u7d90\u7d91\u7d92\u7d93\u7d94\u7d95\u7d96\u7d97\u7d98\u7d99\u7d9a\u7d9b\u7d9c\u7d9d\u7d9e\u7d9f\u7da0\u7da1\u7da2\u7da3\u7da4\u7da5\u7da6\u7da7\u7da8\u7da9\u7daa\u7dab\u7dac\u7dad\u7dae\u7daf\u7db0\u7db1\u7db2\u7db3\u7db4\u7db5\u7db6\u7db7\u7db8\u7db9\u7dba\u7dbb\u7dbc\u7dbd\u7dbe\u7dbf\u7dc0\u7dc1\u7dc2\u7dc3\u7dc4\u7dc5\u7dc6\u7dc7\u7dc8\u7dc9\u7dca\u7dcb\u7dcc\u7dcd\u7dce\u7dcf\u7dd0\u7dd1\u7dd2\u7dd3\u7dd4\u7dd5\u7dd6\u7dd7\u7dd8\u7dd9\u7dda\u7ddb\u7ddc\u7ddd\u7dde\u7ddf\u7de0\u7de1\u7de2\u7de3\u7de4\u7de5\u7de6\u7de7\u7de8\u7de9\u7dea\u7deb\u7dec\u7ded\u7dee\u7def\u7df0\u7df1\u7df2\u7df3\u7df4\u7df5\u7df6\u7df7\u7df8\u7df9\u7dfa\u7dfb\u7dfc\u7dfd\u7dfe\u7dff\u7e00\u7e01\u7e02\u7e03\u7e04\u7e05\u7e06\u7e07\u7e08\u7e09\u7e0a\u7e0b\u7e0c\u7e0d\u7e0e\u7e0f\u7e10\u7e11\u7e12\u7e13\u7e14\u7e15\u7e16\u7e17\u7e18\u7e19\u7e1a\u7e1b\u7e1c\u7e1d\u7e1e\u7e1f\u7e20\u7e21\u7e22\u7e23\u7e24\u7e25\u7e26\u7e27\u7e28\u7e29\u7e2a\u7e2b\u7e2c\u7e2d\u7e2e\u7e2f\u7e30\u7e31\u7e32\u7e33\u7e34\u7e35\u7e36\u7e37\u7e38\u7e39\u7e3a\u7e3b\u7e3c\u7e3d\u7e3e\u7e3f\u7e40\u7e41\u7e42\u7e43\u7e44\u7e45\u7e46\u7e47\u7e48\u7e49\u7e4a\u7e4b\u7e4c\u7e4d\u7e4e\u7e4f\u7e50\u7e51\u7e52\u7e53\u7e54\u7e55\u7e56\u7e57\u7e58\u7e59\u7e5a\u7e5b\u7e5c\u7e5d\u7e5e\u7e5f\u7e60\u7e61\u7e62\u7e63\u7e64\u7e65\u7e66\u7e67\u7e68\u7e69\u7e6a\u7e6b\u7e6c\u7e6d\u7e6e\u7e6f\u7e70\u7e71\u7e72\u7e73\u7e74\u7e75\u7e76\u7e77\u7e78\u7e79\u7e7a\u7e7b\u7e7c\u7e7d\u7e7e\u7e7f\u7e80\u7e81\u7e82\u7e83\u7e84\u7e85\u7e86\u7e87\u7e88\u7e89\u7e8a\u7e8b\u7e8c\u7e8d\u7e8e\u7e8f\u7e90\u7e91\u7e92\u7e93\u7e94\u7e95\u7e96\u7e97\u7e98\u7e99\u7e9a\u7e9b\u7e9c\u7e9d\u7e9e\u7e9f\u7ea0\u7ea1\u7ea2\u7ea3\u7ea4\u7ea5\u7ea6\u7ea7\u7ea8\u7ea9\u7eaa\u7eab\u7eac\u7ead\u7eae\u7eaf\u7eb0\u7eb1\u7eb2\u7eb3\u7eb4\u7eb5\u7eb6\u7eb7\u7eb8\u7eb9\u7eba\u7ebb\u7ebc\u7ebd\u7ebe\u7ebf\u7ec0\u7ec1\u7ec2\u7ec3\u7ec4\u7ec5\u7ec6\u7ec7\u7ec8\u7ec9\u7eca\u7ecb\u7ecc\u7ecd\u7ece\u7ecf\u7ed0\u7ed1\u7ed2\u7ed3\u7ed4\u7ed5\u7ed6\u7ed7\u7ed8\u7ed9\u7eda\u7edb\u7edc\u7edd\u7ede\u7edf\u7ee0\u7ee1\u7ee2\u7ee3\u7ee4\u7ee5\u7ee6\u7ee7\u7ee8\u7ee9\u7eea\u7eeb\u7eec\u7eed\u7eee\u7eef\u7ef0\u7ef1\u7ef2\u7ef3\u7ef4\u7ef5\u7ef6\u7ef7\u7ef8\u7ef9\u7efa\u7efb\u7efc\u7efd\u7efe\u7eff\u7f00\u7f01\u7f02\u7f03\u7f04\u7f05\u7f06\u7f07\u7f08\u7f09\u7f0a\u7f0b\u7f0c\u7f0d\u7f0e\u7f0f\u7f10\u7f11\u7f12\u7f13\u7f14\u7f15\u7f16\u7f17\u7f18\u7f19\u7f1a\u7f1b\u7f1c\u7f1d\u7f1e\u7f1f\u7f20\u7f21\u7f22\u7f23\u7f24\u7f25\u7f26\u7f27\u7f28\u7f29\u7f2a\u7f2b\u7f2c\u7f2d\u7f2e\u7f2f\u7f30\u7f31\u7f32\u7f33\u7f34\u7f35\u7f36\u7f37\u7f38\u7f39\u7f3a\u7f3b\u7f3c\u7f3d\u7f3e\u7f3f\u7f40\u7f41\u7f42\u7f43\u7f44\u7f45\u7f46\u7f47\u7f48\u7f49\u7f4a\u7f4b\u7f4c\u7f4d\u7f4e\u7f4f\u7f50\u7f51\u7f52\u7f53\u7f54\u7f55\u7f56\u7f57\u7f58\u7f59\u7f5a\u7f5b\u7f5c\u7f5d\u7f5e\u7f5f\u7f60\u7f61\u7f62\u7f63\u7f64\u7f65\u7f66\u7f67\u7f68\u7f69\u7f6a\u7f6b\u7f6c\u7f6d\u7f6e\u7f6f\u7f70\u7f71\u7f72\u7f73\u7f74\u7f75\u7f76\u7f77\u7f78\u7f79\u7f7a\u7f7b\u7f7c\u7f7d\u7f7e\u7f7f\u7f80\u7f81\u7f82\u7f83\u7f84\u7f85\u7f86\u7f87\u7f88\u7f89\u7f8a\u7f8b\u7f8c\u7f8d\u7f8e\u7f8f\u7f90\u7f91\u7f92\u7f93\u7f94\u7f95\u7f96\u7f97\u7f98\u7f99\u7f9a\u7f9b\u7f9c\u7f9d\u7f9e\u7f9f\u7fa0\u7fa1\u7fa2\u7fa3\u7fa4\u7fa5\u7fa6\u7fa7\u7fa8\u7fa9\u7faa\u7fab\u7fac\u7fad\u7fae\u7faf\u7fb0\u7fb1\u7fb2\u7fb3\u7fb4\u7fb5\u7fb6\u7fb7\u7fb8\u7fb9\u7fba\u7fbb\u7fbc\u7fbd\u7fbe\u7fbf\u7fc0\u7fc1\u7fc2\u7fc3\u7fc4\u7fc5\u7fc6\u7fc7\u7fc8\u7fc9\u7fca\u7fcb\u7fcc\u7fcd\u7fce\u7fcf\u7fd0\u7fd1\u7fd2\u7fd3\u7fd4\u7fd5\u7fd6\u7fd7\u7fd8\u7fd9\u7fda\u7fdb\u7fdc\u7fdd\u7fde\u7fdf\u7fe0\u7fe1\u7fe2\u7fe3\u7fe4\u7fe5\u7fe6\u7fe7\u7fe8\u7fe9\u7fea\u7feb\u7fec\u7fed\u7fee\u7fef\u7ff0\u7ff1\u7ff2\u7ff3\u7ff4\u7ff5\u7ff6\u7ff7\u7ff8\u7ff9\u7ffa\u7ffb\u7ffc\u7ffd\u7ffe\u7fff\u8000\u8001\u8002\u8003\u8004\u8005\u8006\u8007\u8008\u8009\u800a\u800b\u800c\u800d\u800e\u800f\u8010\u8011\u8012\u8013\u8014\u8015\u8016\u8017\u8018\u8019\u801a\u801b\u801c\u801d\u801e\u801f\u8020\u8021\u8022\u8023\u8024\u8025\u8026\u8027\u8028\u8029\u802a\u802b\u802c\u802d\u802e\u802f\u8030\u8031\u8032\u8033\u8034\u8035\u8036\u8037\u8038\u8039\u803a\u803b\u803c\u803d\u803e\u803f\u8040\u8041\u8042\u8043\u8044\u8045\u8046\u8047\u8048\u8049\u804a\u804b\u804c\u804d\u804e\u804f\u8050\u8051\u8052\u8053\u8054\u8055\u8056\u8057\u8058\u8059\u805a\u805b\u805c\u805d\u805e\u805f\u8060\u8061\u8062\u8063\u8064\u8065\u8066\u8067\u8068\u8069\u806a\u806b\u806c\u806d\u806e\u806f\u8070\u8071\u8072\u8073\u8074\u8075\u8076\u8077\u8078\u8079\u807a\u807b\u807c\u807d\u807e\u807f\u8080\u8081\u8082\u8083\u8084\u8085\u8086\u8087\u8088\u8089\u808a\u808b\u808c\u808d\u808e\u808f\u8090\u8091\u8092\u8093\u8094\u8095\u8096\u8097\u8098\u8099\u809a\u809b\u809c\u809d\u809e\u809f\u80a0\u80a1\u80a2\u80a3\u80a4\u80a5\u80a6\u80a7\u80a8\u80a9\u80aa\u80ab\u80ac\u80ad\u80ae\u80af\u80b0\u80b1\u80b2\u80b3\u80b4\u80b5\u80b6\u80b7\u80b8\u80b9\u80ba\u80bb\u80bc\u80bd\u80be\u80bf\u80c0\u80c1\u80c2\u80c3\u80c4\u80c5\u80c6\u80c7\u80c8\u80c9\u80ca\u80cb\u80cc\u80cd\u80ce\u80cf\u80d0\u80d1\u80d2\u80d3\u80d4\u80d5\u80d6\u80d7\u80d8\u80d9\u80da\u80db\u80dc\u80dd\u80de\u80df\u80e0\u80e1\u80e2\u80e3\u80e4\u80e5\u80e6\u80e7\u80e8\u80e9\u80ea\u80eb\u80ec\u80ed\u80ee\u80ef\u80f0\u80f1\u80f2\u80f3\u80f4\u80f5\u80f6\u80f7\u80f8\u80f9\u80fa\u80fb\u80fc\u80fd\u80fe\u80ff\u8100\u8101\u8102\u8103\u8104\u8105\u8106\u8107\u8108\u8109\u810a\u810b\u810c\u810d\u810e\u810f\u8110\u8111\u8112\u8113\u8114\u8115\u8116\u8117\u8118\u8119\u811a\u811b\u811c\u811d\u811e\u811f\u8120\u8121\u8122\u8123\u8124\u8125\u8126\u8127\u8128\u8129\u812a\u812b\u812c\u812d\u812e\u812f\u8130\u8131\u8132\u8133\u8134\u8135\u8136\u8137\u8138\u8139\u813a\u813b\u813c\u813d\u813e\u813f\u8140\u8141\u8142\u8143\u8144\u8145\u8146\u8147\u8148\u8149\u814a\u814b\u814c\u814d\u814e\u814f\u8150\u8151\u8152\u8153\u8154\u8155\u8156\u8157\u8158\u8159\u815a\u815b\u815c\u815d\u815e\u815f\u8160\u8161\u8162\u8163\u8164\u8165\u8166\u8167\u8168\u8169\u816a\u816b\u816c\u816d\u816e\u816f\u8170\u8171\u8172\u8173\u8174\u8175\u8176\u8177\u8178\u8179\u817a\u817b\u817c\u817d\u817e\u817f\u8180\u8181\u8182\u8183\u8184\u8185\u8186\u8187\u8188\u8189\u818a\u818b\u818c\u818d\u818e\u818f\u8190\u8191\u8192\u8193\u8194\u8195\u8196\u8197\u8198\u8199\u819a\u819b\u819c\u819d\u819e\u819f\u81a0\u81a1\u81a2\u81a3\u81a4\u81a5\u81a6\u81a7\u81a8\u81a9\u81aa\u81ab\u81ac\u81ad\u81ae\u81af\u81b0\u81b1\u81b2\u81b3\u81b4\u81b5\u81b6\u81b7\u81b8\u81b9\u81ba\u81bb\u81bc\u81bd\u81be\u81bf\u81c0\u81c1\u81c2\u81c3\u81c4\u81c5\u81c6\u81c7\u81c8\u81c9\u81ca\u81cb\u81cc\u81cd\u81ce\u81cf\u81d0\u81d1\u81d2\u81d3\u81d4\u81d5\u81d6\u81d7\u81d8\u81d9\u81da\u81db\u81dc\u81dd\u81de\u81df\u81e0\u81e1\u81e2\u81e3\u81e4\u81e5\u81e6\u81e7\u81e8\u81e9\u81ea\u81eb\u81ec\u81ed\u81ee\u81ef\u81f0\u81f1\u81f2\u81f3\u81f4\u81f5\u81f6\u81f7\u81f8\u81f9\u81fa\u81fb\u81fc\u81fd\u81fe\u81ff\u8200\u8201\u8202\u8203\u8204\u8205\u8206\u8207\u8208\u8209\u820a\u820b\u820c\u820d\u820e\u820f\u8210\u8211\u8212\u8213\u8214\u8215\u8216\u8217\u8218\u8219\u821a\u821b\u821c\u821d\u821e\u821f\u8220\u8221\u8222\u8223\u8224\u8225\u8226\u8227\u8228\u8229\u822a\u822b\u822c\u822d\u822e\u822f\u8230\u8231\u8232\u8233\u8234\u8235\u8236\u8237\u8238\u8239\u823a\u823b\u823c\u823d\u823e\u823f\u8240\u8241\u8242\u8243\u8244\u8245\u8246\u8247\u8248\u8249\u824a\u824b\u824c\u824d\u824e\u824f\u8250\u8251\u8252\u8253\u8254\u8255\u8256\u8257\u8258\u8259\u825a\u825b\u825c\u825d\u825e\u825f\u8260\u8261\u8262\u8263\u8264\u8265\u8266\u8267\u8268\u8269\u826a\u826b\u826c\u826d\u826e\u826f\u8270\u8271\u8272\u8273\u8274\u8275\u8276\u8277\u8278\u8279\u827a\u827b\u827c\u827d\u827e\u827f\u8280\u8281\u8282\u8283\u8284\u8285\u8286\u8287\u8288\u8289\u828a\u828b\u828c\u828d\u828e\u828f\u8290\u8291\u8292\u8293\u8294\u8295\u8296\u8297\u8298\u8299\u829a\u829b\u829c\u829d\u829e\u829f\u82a0\u82a1\u82a2\u82a3\u82a4\u82a5\u82a6\u82a7\u82a8\u82a9\u82aa\u82ab\u82ac\u82ad\u82ae\u82af\u82b0\u82b1\u82b2\u82b3\u82b4\u82b5\u82b6\u82b7\u82b8\u82b9\u82ba\u82bb\u82bc\u82bd\u82be\u82bf\u82c0\u82c1\u82c2\u82c3\u82c4\u82c5\u82c6\u82c7\u82c8\u82c9\u82ca\u82cb\u82cc\u82cd\u82ce\u82cf\u82d0\u82d1\u82d2\u82d3\u82d4\u82d5\u82d6\u82d7\u82d8\u82d9\u82da\u82db\u82dc\u82dd\u82de\u82df\u82e0\u82e1\u82e2\u82e3\u82e4\u82e5\u82e6\u82e7\u82e8\u82e9\u82ea\u82eb\u82ec\u82ed\u82ee\u82ef\u82f0\u82f1\u82f2\u82f3\u82f4\u82f5\u82f6\u82f7\u82f8\u82f9\u82fa\u82fb\u82fc\u82fd\u82fe\u82ff\u8300\u8301\u8302\u8303\u8304\u8305\u8306\u8307\u8308\u8309\u830a\u830b\u830c\u830d\u830e\u830f\u8310\u8311\u8312\u8313\u8314\u8315\u8316\u8317\u8318\u8319\u831a\u831b\u831c\u831d\u831e\u831f\u8320\u8321\u8322\u8323\u8324\u8325\u8326\u8327\u8328\u8329\u832a\u832b\u832c\u832d\u832e\u832f\u8330\u8331\u8332\u8333\u8334\u8335\u8336\u8337\u8338\u8339\u833a\u833b\u833c\u833d\u833e\u833f\u8340\u8341\u8342\u8343\u8344\u8345\u8346\u8347\u8348\u8349\u834a\u834b\u834c\u834d\u834e\u834f\u8350\u8351\u8352\u8353\u8354\u8355\u8356\u8357\u8358\u8359\u835a\u835b\u835c\u835d\u835e\u835f\u8360\u8361\u8362\u8363\u8364\u8365\u8366\u8367\u8368\u8369\u836a\u836b\u836c\u836d\u836e\u836f\u8370\u8371\u8372\u8373\u8374\u8375\u8376\u8377\u8378\u8379\u837a\u837b\u837c\u837d\u837e\u837f\u8380\u8381\u8382\u8383\u8384\u8385\u8386\u8387\u8388\u8389\u838a\u838b\u838c\u838d\u838e\u838f\u8390\u8391\u8392\u8393\u8394\u8395\u8396\u8397\u8398\u8399\u839a\u839b\u839c\u839d\u839e\u839f\u83a0\u83a1\u83a2\u83a3\u83a4\u83a5\u83a6\u83a7\u83a8\u83a9\u83aa\u83ab\u83ac\u83ad\u83ae\u83af\u83b0\u83b1\u83b2\u83b3\u83b4\u83b5\u83b6\u83b7\u83b8\u83b9\u83ba\u83bb\u83bc\u83bd\u83be\u83bf\u83c0\u83c1\u83c2\u83c3\u83c4\u83c5\u83c6\u83c7\u83c8\u83c9\u83ca\u83cb\u83cc\u83cd\u83ce\u83cf\u83d0\u83d1\u83d2\u83d3\u83d4\u83d5\u83d6\u83d7\u83d8\u83d9\u83da\u83db\u83dc\u83dd\u83de\u83df\u83e0\u83e1\u83e2\u83e3\u83e4\u83e5\u83e6\u83e7\u83e8\u83e9\u83ea\u83eb\u83ec\u83ed\u83ee\u83ef\u83f0\u83f1\u83f2\u83f3\u83f4\u83f5\u83f6\u83f7\u83f8\u83f9\u83fa\u83fb\u83fc\u83fd\u83fe\u83ff\u8400\u8401\u8402\u8403\u8404\u8405\u8406\u8407\u8408\u8409\u840a\u840b\u840c\u840d\u840e\u840f\u8410\u8411\u8412\u8413\u8414\u8415\u8416\u8417\u8418\u8419\u841a\u841b\u841c\u841d\u841e\u841f\u8420\u8421\u8422\u8423\u8424\u8425\u8426\u8427\u8428\u8429\u842a\u842b\u842c\u842d\u842e\u842f\u8430\u8431\u8432\u8433\u8434\u8435\u8436\u8437\u8438\u8439\u843a\u843b\u843c\u843d\u843e\u843f\u8440\u8441\u8442\u8443\u8444\u8445\u8446\u8447\u8448\u8449\u844a\u844b\u844c\u844d\u844e\u844f\u8450\u8451\u8452\u8453\u8454\u8455\u8456\u8457\u8458\u8459\u845a\u845b\u845c\u845d\u845e\u845f\u8460\u8461\u8462\u8463\u8464\u8465\u8466\u8467\u8468\u8469\u846a\u846b\u846c\u846d\u846e\u846f\u8470\u8471\u8472\u8473\u8474\u8475\u8476\u8477\u8478\u8479\u847a\u847b\u847c\u847d\u847e\u847f\u8480\u8481\u8482\u8483\u8484\u8485\u8486\u8487\u8488\u8489\u848a\u848b\u848c\u848d\u848e\u848f\u8490\u8491\u8492\u8493\u8494\u8495\u8496\u8497\u8498\u8499\u849a\u849b\u849c\u849d\u849e\u849f\u84a0\u84a1\u84a2\u84a3\u84a4\u84a5\u84a6\u84a7\u84a8\u84a9\u84aa\u84ab\u84ac\u84ad\u84ae\u84af\u84b0\u84b1\u84b2\u84b3\u84b4\u84b5\u84b6\u84b7\u84b8\u84b9\u84ba\u84bb\u84bc\u84bd\u84be\u84bf\u84c0\u84c1\u84c2\u84c3\u84c4\u84c5\u84c6\u84c7\u84c8\u84c9\u84ca\u84cb\u84cc\u84cd\u84ce\u84cf\u84d0\u84d1\u84d2\u84d3\u84d4\u84d5\u84d6\u84d7\u84d8\u84d9\u84da\u84db\u84dc\u84dd\u84de\u84df\u84e0\u84e1\u84e2\u84e3\u84e4\u84e5\u84e6\u84e7\u84e8\u84e9\u84ea\u84eb\u84ec\u84ed\u84ee\u84ef\u84f0\u84f1\u84f2\u84f3\u84f4\u84f5\u84f6\u84f7\u84f8\u84f9\u84fa\u84fb\u84fc\u84fd\u84fe\u84ff\u8500\u8501\u8502\u8503\u8504\u8505\u8506\u8507\u8508\u8509\u850a\u850b\u850c\u850d\u850e\u850f\u8510\u8511\u8512\u8513\u8514\u8515\u8516\u8517\u8518\u8519\u851a\u851b\u851c\u851d\u851e\u851f\u8520\u8521\u8522\u8523\u8524\u8525\u8526\u8527\u8528\u8529\u852a\u852b\u852c\u852d\u852e\u852f\u8530\u8531\u8532\u8533\u8534\u8535\u8536\u8537\u8538\u8539\u853a\u853b\u853c\u853d\u853e\u853f\u8540\u8541\u8542\u8543\u8544\u8545\u8546\u8547\u8548\u8549\u854a\u854b\u854c\u854d\u854e\u854f\u8550\u8551\u8552\u8553\u8554\u8555\u8556\u8557\u8558\u8559\u855a\u855b\u855c\u855d\u855e\u855f\u8560\u8561\u8562\u8563\u8564\u8565\u8566\u8567\u8568\u8569\u856a\u856b\u856c\u856d\u856e\u856f\u8570\u8571\u8572\u8573\u8574\u8575\u8576\u8577\u8578\u8579\u857a\u857b\u857c\u857d\u857e\u857f\u8580\u8581\u8582\u8583\u8584\u8585\u8586\u8587\u8588\u8589\u858a\u858b\u858c\u858d\u858e\u858f\u8590\u8591\u8592\u8593\u8594\u8595\u8596\u8597\u8598\u8599\u859a\u859b\u859c\u859d\u859e\u859f\u85a0\u85a1\u85a2\u85a3\u85a4\u85a5\u85a6\u85a7\u85a8\u85a9\u85aa\u85ab\u85ac\u85ad\u85ae\u85af\u85b0\u85b1\u85b2\u85b3\u85b4\u85b5\u85b6\u85b7\u85b8\u85b9\u85ba\u85bb\u85bc\u85bd\u85be\u85bf\u85c0\u85c1\u85c2\u85c3\u85c4\u85c5\u85c6\u85c7\u85c8\u85c9\u85ca\u85cb\u85cc\u85cd\u85ce\u85cf\u85d0\u85d1\u85d2\u85d3\u85d4\u85d5\u85d6\u85d7\u85d8\u85d9\u85da\u85db\u85dc\u85dd\u85de\u85df\u85e0\u85e1\u85e2\u85e3\u85e4\u85e5\u85e6\u85e7\u85e8\u85e9\u85ea\u85eb\u85ec\u85ed\u85ee\u85ef\u85f0\u85f1\u85f2\u85f3\u85f4\u85f5\u85f6\u85f7\u85f8\u85f9\u85fa\u85fb\u85fc\u85fd\u85fe\u85ff\u8600\u8601\u8602\u8603\u8604\u8605\u8606\u8607\u8608\u8609\u860a\u860b\u860c\u860d\u860e\u860f\u8610\u8611\u8612\u8613\u8614\u8615\u8616\u8617\u8618\u8619\u861a\u861b\u861c\u861d\u861e\u861f\u8620\u8621\u8622\u8623\u8624\u8625\u8626\u8627\u8628\u8629\u862a\u862b\u862c\u862d\u862e\u862f\u8630\u8631\u8632\u8633\u8634\u8635\u8636\u8637\u8638\u8639\u863a\u863b\u863c\u863d\u863e\u863f\u8640\u8641\u8642\u8643\u8644\u8645\u8646\u8647\u8648\u8649\u864a\u864b\u864c\u864d\u864e\u864f\u8650\u8651\u8652\u8653\u8654\u8655\u8656\u8657\u8658\u8659\u865a\u865b\u865c\u865d\u865e\u865f\u8660\u8661\u8662\u8663\u8664\u8665\u8666\u8667\u8668\u8669\u866a\u866b\u866c\u866d\u866e\u866f\u8670\u8671\u8672\u8673\u8674\u8675\u8676\u8677\u8678\u8679\u867a\u867b\u867c\u867d\u867e\u867f\u8680\u8681\u8682\u8683\u8684\u8685\u8686\u8687\u8688\u8689\u868a\u868b\u868c\u868d\u868e\u868f\u8690\u8691\u8692\u8693\u8694\u8695\u8696\u8697\u8698\u8699\u869a\u869b\u869c\u869d\u869e\u869f\u86a0\u86a1\u86a2\u86a3\u86a4\u86a5\u86a6\u86a7\u86a8\u86a9\u86aa\u86ab\u86ac\u86ad\u86ae\u86af\u86b0\u86b1\u86b2\u86b3\u86b4\u86b5\u86b6\u86b7\u86b8\u86b9\u86ba\u86bb\u86bc\u86bd\u86be\u86bf\u86c0\u86c1\u86c2\u86c3\u86c4\u86c5\u86c6\u86c7\u86c8\u86c9\u86ca\u86cb\u86cc\u86cd\u86ce\u86cf\u86d0\u86d1\u86d2\u86d3\u86d4\u86d5\u86d6\u86d7\u86d8\u86d9\u86da\u86db\u86dc\u86dd\u86de\u86df\u86e0\u86e1\u86e2\u86e3\u86e4\u86e5\u86e6\u86e7\u86e8\u86e9\u86ea\u86eb\u86ec\u86ed\u86ee\u86ef\u86f0\u86f1\u86f2\u86f3\u86f4\u86f5\u86f6\u86f7\u86f8\u86f9\u86fa\u86fb\u86fc\u86fd\u86fe\u86ff\u8700\u8701\u8702\u8703\u8704\u8705\u8706\u8707\u8708\u8709\u870a\u870b\u870c\u870d\u870e\u870f\u8710\u8711\u8712\u8713\u8714\u8715\u8716\u8717\u8718\u8719\u871a\u871b\u871c\u871d\u871e\u871f\u8720\u8721\u8722\u8723\u8724\u8725\u8726\u8727\u8728\u8729\u872a\u872b\u872c\u872d\u872e\u872f\u8730\u8731\u8732\u8733\u8734\u8735\u8736\u8737\u8738\u8739\u873a\u873b\u873c\u873d\u873e\u873f\u8740\u8741\u8742\u8743\u8744\u8745\u8746\u8747\u8748\u8749\u874a\u874b\u874c\u874d\u874e\u874f\u8750\u8751\u8752\u8753\u8754\u8755\u8756\u8757\u8758\u8759\u875a\u875b\u875c\u875d\u875e\u875f\u8760\u8761\u8762\u8763\u8764\u8765\u8766\u8767\u8768\u8769\u876a\u876b\u876c\u876d\u876e\u876f\u8770\u8771\u8772\u8773\u8774\u8775\u8776\u8777\u8778\u8779\u877a\u877b\u877c\u877d\u877e\u877f\u8780\u8781\u8782\u8783\u8784\u8785\u8786\u8787\u8788\u8789\u878a\u878b\u878c\u878d\u878e\u878f\u8790\u8791\u8792\u8793\u8794\u8795\u8796\u8797\u8798\u8799\u879a\u879b\u879c\u879d\u879e\u879f\u87a0\u87a1\u87a2\u87a3\u87a4\u87a5\u87a6\u87a7\u87a8\u87a9\u87aa\u87ab\u87ac\u87ad\u87ae\u87af\u87b0\u87b1\u87b2\u87b3\u87b4\u87b5\u87b6\u87b7\u87b8\u87b9\u87ba\u87bb\u87bc\u87bd\u87be\u87bf\u87c0\u87c1\u87c2\u87c3\u87c4\u87c5\u87c6\u87c7\u87c8\u87c9\u87ca\u87cb\u87cc\u87cd\u87ce\u87cf\u87d0\u87d1\u87d2\u87d3\u87d4\u87d5\u87d6\u87d7\u87d8\u87d9\u87da\u87db\u87dc\u87dd\u87de\u87df\u87e0\u87e1\u87e2\u87e3\u87e4\u87e5\u87e6\u87e7\u87e8\u87e9\u87ea\u87eb\u87ec\u87ed\u87ee\u87ef\u87f0\u87f1\u87f2\u87f3\u87f4\u87f5\u87f6\u87f7\u87f8\u87f9\u87fa\u87fb\u87fc\u87fd\u87fe\u87ff\u8800\u8801\u8802\u8803\u8804\u8805\u8806\u8807\u8808\u8809\u880a\u880b\u880c\u880d\u880e\u880f\u8810\u8811\u8812\u8813\u8814\u8815\u8816\u8817\u8818\u8819\u881a\u881b\u881c\u881d\u881e\u881f\u8820\u8821\u8822\u8823\u8824\u8825\u8826\u8827\u8828\u8829\u882a\u882b\u882c\u882d\u882e\u882f\u8830\u8831\u8832\u8833\u8834\u8835\u8836\u8837\u8838\u8839\u883a\u883b\u883c\u883d\u883e\u883f\u8840\u8841\u8842\u8843\u8844\u8845\u8846\u8847\u8848\u8849\u884a\u884b\u884c\u884d\u884e\u884f\u8850\u8851\u8852\u8853\u8854\u8855\u8856\u8857\u8858\u8859\u885a\u885b\u885c\u885d\u885e\u885f\u8860\u8861\u8862\u8863\u8864\u8865\u8866\u8867\u8868\u8869\u886a\u886b\u886c\u886d\u886e\u886f\u8870\u8871\u8872\u8873\u8874\u8875\u8876\u8877\u8878\u8879\u887a\u887b\u887c\u887d\u887e\u887f\u8880\u8881\u8882\u8883\u8884\u8885\u8886\u8887\u8888\u8889\u888a\u888b\u888c\u888d\u888e\u888f\u8890\u8891\u8892\u8893\u8894\u8895\u8896\u8897\u8898\u8899\u889a\u889b\u889c\u889d\u889e\u889f\u88a0\u88a1\u88a2\u88a3\u88a4\u88a5\u88a6\u88a7\u88a8\u88a9\u88aa\u88ab\u88ac\u88ad\u88ae\u88af\u88b0\u88b1\u88b2\u88b3\u88b4\u88b5\u88b6\u88b7\u88b8\u88b9\u88ba\u88bb\u88bc\u88bd\u88be\u88bf\u88c0\u88c1\u88c2\u88c3\u88c4\u88c5\u88c6\u88c7\u88c8\u88c9\u88ca\u88cb\u88cc\u88cd\u88ce\u88cf\u88d0\u88d1\u88d2\u88d3\u88d4\u88d5\u88d6\u88d7\u88d8\u88d9\u88da\u88db\u88dc\u88dd\u88de\u88df\u88e0\u88e1\u88e2\u88e3\u88e4\u88e5\u88e6\u88e7\u88e8\u88e9\u88ea\u88eb\u88ec\u88ed\u88ee\u88ef\u88f0\u88f1\u88f2\u88f3\u88f4\u88f5\u88f6\u88f7\u88f8\u88f9\u88fa\u88fb\u88fc\u88fd\u88fe\u88ff\u8900\u8901\u8902\u8903\u8904\u8905\u8906\u8907\u8908\u8909\u890a\u890b\u890c\u890d\u890e\u890f\u8910\u8911\u8912\u8913\u8914\u8915\u8916\u8917\u8918\u8919\u891a\u891b\u891c\u891d\u891e\u891f\u8920\u8921\u8922\u8923\u8924\u8925\u8926\u8927\u8928\u8929\u892a\u892b\u892c\u892d\u892e\u892f\u8930\u8931\u8932\u8933\u8934\u8935\u8936\u8937\u8938\u8939\u893a\u893b\u893c\u893d\u893e\u893f\u8940\u8941\u8942\u8943\u8944\u8945\u8946\u8947\u8948\u8949\u894a\u894b\u894c\u894d\u894e\u894f\u8950\u8951\u8952\u8953\u8954\u8955\u8956\u8957\u8958\u8959\u895a\u895b\u895c\u895d\u895e\u895f\u8960\u8961\u8962\u8963\u8964\u8965\u8966\u8967\u8968\u8969\u896a\u896b\u896c\u896d\u896e\u896f\u8970\u8971\u8972\u8973\u8974\u8975\u8976\u8977\u8978\u8979\u897a\u897b\u897c\u897d\u897e\u897f\u8980\u8981\u8982\u8983\u8984\u8985\u8986\u8987\u8988\u8989\u898a\u898b\u898c\u898d\u898e\u898f\u8990\u8991\u8992\u8993\u8994\u8995\u8996\u8997\u8998\u8999\u899a\u899b\u899c\u899d\u899e\u899f\u89a0\u89a1\u89a2\u89a3\u89a4\u89a5\u89a6\u89a7\u89a8\u89a9\u89aa\u89ab\u89ac\u89ad\u89ae\u89af\u89b0\u89b1\u89b2\u89b3\u89b4\u89b5\u89b6\u89b7\u89b8\u89b9\u89ba\u89bb\u89bc\u89bd\u89be\u89bf\u89c0\u89c1\u89c2\u89c3\u89c4\u89c5\u89c6\u89c7\u89c8\u89c9\u89ca\u89cb\u89cc\u89cd\u89ce\u89cf\u89d0\u89d1\u89d2\u89d3\u89d4\u89d5\u89d6\u89d7\u89d8\u89d9\u89da\u89db\u89dc\u89dd\u89de\u89df\u89e0\u89e1\u89e2\u89e3\u89e4\u89e5\u89e6\u89e7\u89e8\u89e9\u89ea\u89eb\u89ec\u89ed\u89ee\u89ef\u89f0\u89f1\u89f2\u89f3\u89f4\u89f5\u89f6\u89f7\u89f8\u89f9\u89fa\u89fb\u89fc\u89fd\u89fe\u89ff\u8a00\u8a01\u8a02\u8a03\u8a04\u8a05\u8a06\u8a07\u8a08\u8a09\u8a0a\u8a0b\u8a0c\u8a0d\u8a0e\u8a0f\u8a10\u8a11\u8a12\u8a13\u8a14\u8a15\u8a16\u8a17\u8a18\u8a19\u8a1a\u8a1b\u8a1c\u8a1d\u8a1e\u8a1f\u8a20\u8a21\u8a22\u8a23\u8a24\u8a25\u8a26\u8a27\u8a28\u8a29\u8a2a\u8a2b\u8a2c\u8a2d\u8a2e\u8a2f\u8a30\u8a31\u8a32\u8a33\u8a34\u8a35\u8a36\u8a37\u8a38\u8a39\u8a3a\u8a3b\u8a3c\u8a3d\u8a3e\u8a3f\u8a40\u8a41\u8a42\u8a43\u8a44\u8a45\u8a46\u8a47\u8a48\u8a49\u8a4a\u8a4b\u8a4c\u8a4d\u8a4e\u8a4f\u8a50\u8a51\u8a52\u8a53\u8a54\u8a55\u8a56\u8a57\u8a58\u8a59\u8a5a\u8a5b\u8a5c\u8a5d\u8a5e\u8a5f\u8a60\u8a61\u8a62\u8a63\u8a64\u8a65\u8a66\u8a67\u8a68\u8a69\u8a6a\u8a6b\u8a6c\u8a6d\u8a6e\u8a6f\u8a70\u8a71\u8a72\u8a73\u8a74\u8a75\u8a76\u8a77\u8a78\u8a79\u8a7a\u8a7b\u8a7c\u8a7d\u8a7e\u8a7f\u8a80\u8a81\u8a82\u8a83\u8a84\u8a85\u8a86\u8a87\u8a88\u8a89\u8a8a\u8a8b\u8a8c\u8a8d\u8a8e\u8a8f\u8a90\u8a91\u8a92\u8a93\u8a94\u8a95\u8a96\u8a97\u8a98\u8a99\u8a9a\u8a9b\u8a9c\u8a9d\u8a9e\u8a9f\u8aa0\u8aa1\u8aa2\u8aa3\u8aa4\u8aa5\u8aa6\u8aa7\u8aa8\u8aa9\u8aaa\u8aab\u8aac\u8aad\u8aae\u8aaf\u8ab0\u8ab1\u8ab2\u8ab3\u8ab4\u8ab5\u8ab6\u8ab7\u8ab8\u8ab9\u8aba\u8abb\u8abc\u8abd\u8abe\u8abf\u8ac0\u8ac1\u8ac2\u8ac3\u8ac4\u8ac5\u8ac6\u8ac7\u8ac8\u8ac9\u8aca\u8acb\u8acc\u8acd\u8ace\u8acf\u8ad0\u8ad1\u8ad2\u8ad3\u8ad4\u8ad5\u8ad6\u8ad7\u8ad8\u8ad9\u8ada\u8adb\u8adc\u8add\u8ade\u8adf\u8ae0\u8ae1\u8ae2\u8ae3\u8ae4\u8ae5\u8ae6\u8ae7\u8ae8\u8ae9\u8aea\u8aeb\u8aec\u8aed\u8aee\u8aef\u8af0\u8af1\u8af2\u8af3\u8af4\u8af5\u8af6\u8af7\u8af8\u8af9\u8afa\u8afb\u8afc\u8afd\u8afe\u8aff\u8b00\u8b01\u8b02\u8b03\u8b04\u8b05\u8b06\u8b07\u8b08\u8b09\u8b0a\u8b0b\u8b0c\u8b0d\u8b0e\u8b0f\u8b10\u8b11\u8b12\u8b13\u8b14\u8b15\u8b16\u8b17\u8b18\u8b19\u8b1a\u8b1b\u8b1c\u8b1d\u8b1e\u8b1f\u8b20\u8b21\u8b22\u8b23\u8b24\u8b25\u8b26\u8b27\u8b28\u8b29\u8b2a\u8b2b\u8b2c\u8b2d\u8b2e\u8b2f\u8b30\u8b31\u8b32\u8b33\u8b34\u8b35\u8b36\u8b37\u8b38\u8b39\u8b3a\u8b3b\u8b3c\u8b3d\u8b3e\u8b3f\u8b40\u8b41\u8b42\u8b43\u8b44\u8b45\u8b46\u8b47\u8b48\u8b49\u8b4a\u8b4b\u8b4c\u8b4d\u8b4e\u8b4f\u8b50\u8b51\u8b52\u8b53\u8b54\u8b55\u8b56\u8b57\u8b58\u8b59\u8b5a\u8b5b\u8b5c\u8b5d\u8b5e\u8b5f\u8b60\u8b61\u8b62\u8b63\u8b64\u8b65\u8b66\u8b67\u8b68\u8b69\u8b6a\u8b6b\u8b6c\u8b6d\u8b6e\u8b6f\u8b70\u8b71\u8b72\u8b73\u8b74\u8b75\u8b76\u8b77\u8b78\u8b79\u8b7a\u8b7b\u8b7c\u8b7d\u8b7e\u8b7f\u8b80\u8b81\u8b82\u8b83\u8b84\u8b85\u8b86\u8b87\u8b88\u8b89\u8b8a\u8b8b\u8b8c\u8b8d\u8b8e\u8b8f\u8b90\u8b91\u8b92\u8b93\u8b94\u8b95\u8b96\u8b97\u8b98\u8b99\u8b9a\u8b9b\u8b9c\u8b9d\u8b9e\u8b9f\u8ba0\u8ba1\u8ba2\u8ba3\u8ba4\u8ba5\u8ba6\u8ba7\u8ba8\u8ba9\u8baa\u8bab\u8bac\u8bad\u8bae\u8baf\u8bb0\u8bb1\u8bb2\u8bb3\u8bb4\u8bb5\u8bb6\u8bb7\u8bb8\u8bb9\u8bba\u8bbb\u8bbc\u8bbd\u8bbe\u8bbf\u8bc0\u8bc1\u8bc2\u8bc3\u8bc4\u8bc5\u8bc6\u8bc7\u8bc8\u8bc9\u8bca\u8bcb\u8bcc\u8bcd\u8bce\u8bcf\u8bd0\u8bd1\u8bd2\u8bd3\u8bd4\u8bd5\u8bd6\u8bd7\u8bd8\u8bd9\u8bda\u8bdb\u8bdc\u8bdd\u8bde\u8bdf\u8be0\u8be1\u8be2\u8be3\u8be4\u8be5\u8be6\u8be7\u8be8\u8be9\u8bea\u8beb\u8bec\u8bed\u8bee\u8bef\u8bf0\u8bf1\u8bf2\u8bf3\u8bf4\u8bf5\u8bf6\u8bf7\u8bf8\u8bf9\u8bfa\u8bfb\u8bfc\u8bfd\u8bfe\u8bff\u8c00\u8c01\u8c02\u8c03\u8c04\u8c05\u8c06\u8c07\u8c08\u8c09\u8c0a\u8c0b\u8c0c\u8c0d\u8c0e\u8c0f\u8c10\u8c11\u8c12\u8c13\u8c14\u8c15\u8c16\u8c17\u8c18\u8c19\u8c1a\u8c1b\u8c1c\u8c1d\u8c1e\u8c1f\u8c20\u8c21\u8c22\u8c23\u8c24\u8c25\u8c26\u8c27\u8c28\u8c29\u8c2a\u8c2b\u8c2c\u8c2d\u8c2e\u8c2f\u8c30\u8c31\u8c32\u8c33\u8c34\u8c35\u8c36\u8c37\u8c38\u8c39\u8c3a\u8c3b\u8c3c\u8c3d\u8c3e\u8c3f\u8c40\u8c41\u8c42\u8c43\u8c44\u8c45\u8c46\u8c47\u8c48\u8c49\u8c4a\u8c4b\u8c4c\u8c4d\u8c4e\u8c4f\u8c50\u8c51\u8c52\u8c53\u8c54\u8c55\u8c56\u8c57\u8c58\u8c59\u8c5a\u8c5b\u8c5c\u8c5d\u8c5e\u8c5f\u8c60\u8c61\u8c62\u8c63\u8c64\u8c65\u8c66\u8c67\u8c68\u8c69\u8c6a\u8c6b\u8c6c\u8c6d\u8c6e\u8c6f\u8c70\u8c71\u8c72\u8c73\u8c74\u8c75\u8c76\u8c77\u8c78\u8c79\u8c7a\u8c7b\u8c7c\u8c7d\u8c7e\u8c7f\u8c80\u8c81\u8c82\u8c83\u8c84\u8c85\u8c86\u8c87\u8c88\u8c89\u8c8a\u8c8b\u8c8c\u8c8d\u8c8e\u8c8f\u8c90\u8c91\u8c92\u8c93\u8c94\u8c95\u8c96\u8c97\u8c98\u8c99\u8c9a\u8c9b\u8c9c\u8c9d\u8c9e\u8c9f\u8ca0\u8ca1\u8ca2\u8ca3\u8ca4\u8ca5\u8ca6\u8ca7\u8ca8\u8ca9\u8caa\u8cab\u8cac\u8cad\u8cae\u8caf\u8cb0\u8cb1\u8cb2\u8cb3\u8cb4\u8cb5\u8cb6\u8cb7\u8cb8\u8cb9\u8cba\u8cbb\u8cbc\u8cbd\u8cbe\u8cbf\u8cc0\u8cc1\u8cc2\u8cc3\u8cc4\u8cc5\u8cc6\u8cc7\u8cc8\u8cc9\u8cca\u8ccb\u8ccc\u8ccd\u8cce\u8ccf\u8cd0\u8cd1\u8cd2\u8cd3\u8cd4\u8cd5\u8cd6\u8cd7\u8cd8\u8cd9\u8cda\u8cdb\u8cdc\u8cdd\u8cde\u8cdf\u8ce0\u8ce1\u8ce2\u8ce3\u8ce4\u8ce5\u8ce6\u8ce7\u8ce8\u8ce9\u8cea\u8ceb\u8cec\u8ced\u8cee\u8cef\u8cf0\u8cf1\u8cf2\u8cf3\u8cf4\u8cf5\u8cf6\u8cf7\u8cf8\u8cf9\u8cfa\u8cfb\u8cfc\u8cfd\u8cfe\u8cff\u8d00\u8d01\u8d02\u8d03\u8d04\u8d05\u8d06\u8d07\u8d08\u8d09\u8d0a\u8d0b\u8d0c\u8d0d\u8d0e\u8d0f\u8d10\u8d11\u8d12\u8d13\u8d14\u8d15\u8d16\u8d17\u8d18\u8d19\u8d1a\u8d1b\u8d1c\u8d1d\u8d1e\u8d1f\u8d20\u8d21\u8d22\u8d23\u8d24\u8d25\u8d26\u8d27\u8d28\u8d29\u8d2a\u8d2b\u8d2c\u8d2d\u8d2e\u8d2f\u8d30\u8d31\u8d32\u8d33\u8d34\u8d35\u8d36\u8d37\u8d38\u8d39\u8d3a\u8d3b\u8d3c\u8d3d\u8d3e\u8d3f\u8d40\u8d41\u8d42\u8d43\u8d44\u8d45\u8d46\u8d47\u8d48\u8d49\u8d4a\u8d4b\u8d4c\u8d4d\u8d4e\u8d4f\u8d50\u8d51\u8d52\u8d53\u8d54\u8d55\u8d56\u8d57\u8d58\u8d59\u8d5a\u8d5b\u8d5c\u8d5d\u8d5e\u8d5f\u8d60\u8d61\u8d62\u8d63\u8d64\u8d65\u8d66\u8d67\u8d68\u8d69\u8d6a\u8d6b\u8d6c\u8d6d\u8d6e\u8d6f\u8d70\u8d71\u8d72\u8d73\u8d74\u8d75\u8d76\u8d77\u8d78\u8d79\u8d7a\u8d7b\u8d7c\u8d7d\u8d7e\u8d7f\u8d80\u8d81\u8d82\u8d83\u8d84\u8d85\u8d86\u8d87\u8d88\u8d89\u8d8a\u8d8b\u8d8c\u8d8d\u8d8e\u8d8f\u8d90\u8d91\u8d92\u8d93\u8d94\u8d95\u8d96\u8d97\u8d98\u8d99\u8d9a\u8d9b\u8d9c\u8d9d\u8d9e\u8d9f\u8da0\u8da1\u8da2\u8da3\u8da4\u8da5\u8da6\u8da7\u8da8\u8da9\u8daa\u8dab\u8dac\u8dad\u8dae\u8daf\u8db0\u8db1\u8db2\u8db3\u8db4\u8db5\u8db6\u8db7\u8db8\u8db9\u8dba\u8dbb\u8dbc\u8dbd\u8dbe\u8dbf\u8dc0\u8dc1\u8dc2\u8dc3\u8dc4\u8dc5\u8dc6\u8dc7\u8dc8\u8dc9\u8dca\u8dcb\u8dcc\u8dcd\u8dce\u8dcf\u8dd0\u8dd1\u8dd2\u8dd3\u8dd4\u8dd5\u8dd6\u8dd7\u8dd8\u8dd9\u8dda\u8ddb\u8ddc\u8ddd\u8dde\u8ddf\u8de0\u8de1\u8de2\u8de3\u8de4\u8de5\u8de6\u8de7\u8de8\u8de9\u8dea\u8deb\u8dec\u8ded\u8dee\u8def\u8df0\u8df1\u8df2\u8df3\u8df4\u8df5\u8df6\u8df7\u8df8\u8df9\u8dfa\u8dfb\u8dfc\u8dfd\u8dfe\u8dff\u8e00\u8e01\u8e02\u8e03\u8e04\u8e05\u8e06\u8e07\u8e08\u8e09\u8e0a\u8e0b\u8e0c\u8e0d\u8e0e\u8e0f\u8e10\u8e11\u8e12\u8e13\u8e14\u8e15\u8e16\u8e17\u8e18\u8e19\u8e1a\u8e1b\u8e1c\u8e1d\u8e1e\u8e1f\u8e20\u8e21\u8e22\u8e23\u8e24\u8e25\u8e26\u8e27\u8e28\u8e29\u8e2a\u8e2b\u8e2c\u8e2d\u8e2e\u8e2f\u8e30\u8e31\u8e32\u8e33\u8e34\u8e35\u8e36\u8e37\u8e38\u8e39\u8e3a\u8e3b\u8e3c\u8e3d\u8e3e\u8e3f\u8e40\u8e41\u8e42\u8e43\u8e44\u8e45\u8e46\u8e47\u8e48\u8e49\u8e4a\u8e4b\u8e4c\u8e4d\u8e4e\u8e4f\u8e50\u8e51\u8e52\u8e53\u8e54\u8e55\u8e56\u8e57\u8e58\u8e59\u8e5a\u8e5b\u8e5c\u8e5d\u8e5e\u8e5f\u8e60\u8e61\u8e62\u8e63\u8e64\u8e65\u8e66\u8e67\u8e68\u8e69\u8e6a\u8e6b\u8e6c\u8e6d\u8e6e\u8e6f\u8e70\u8e71\u8e72\u8e73\u8e74\u8e75\u8e76\u8e77\u8e78\u8e79\u8e7a\u8e7b\u8e7c\u8e7d\u8e7e\u8e7f\u8e80\u8e81\u8e82\u8e83\u8e84\u8e85\u8e86\u8e87\u8e88\u8e89\u8e8a\u8e8b\u8e8c\u8e8d\u8e8e\u8e8f\u8e90\u8e91\u8e92\u8e93\u8e94\u8e95\u8e96\u8e97\u8e98\u8e99\u8e9a\u8e9b\u8e9c\u8e9d\u8e9e\u8e9f\u8ea0\u8ea1\u8ea2\u8ea3\u8ea4\u8ea5\u8ea6\u8ea7\u8ea8\u8ea9\u8eaa\u8eab\u8eac\u8ead\u8eae\u8eaf\u8eb0\u8eb1\u8eb2\u8eb3\u8eb4\u8eb5\u8eb6\u8eb7\u8eb8\u8eb9\u8eba\u8ebb\u8ebc\u8ebd\u8ebe\u8ebf\u8ec0\u8ec1\u8ec2\u8ec3\u8ec4\u8ec5\u8ec6\u8ec7\u8ec8\u8ec9\u8eca\u8ecb\u8ecc\u8ecd\u8ece\u8ecf\u8ed0\u8ed1\u8ed2\u8ed3\u8ed4\u8ed5\u8ed6\u8ed7\u8ed8\u8ed9\u8eda\u8edb\u8edc\u8edd\u8ede\u8edf\u8ee0\u8ee1\u8ee2\u8ee3\u8ee4\u8ee5\u8ee6\u8ee7\u8ee8\u8ee9\u8eea\u8eeb\u8eec\u8eed\u8eee\u8eef\u8ef0\u8ef1\u8ef2\u8ef3\u8ef4\u8ef5\u8ef6\u8ef7\u8ef8\u8ef9\u8efa\u8efb\u8efc\u8efd\u8efe\u8eff\u8f00\u8f01\u8f02\u8f03\u8f04\u8f05\u8f06\u8f07\u8f08\u8f09\u8f0a\u8f0b\u8f0c\u8f0d\u8f0e\u8f0f\u8f10\u8f11\u8f12\u8f13\u8f14\u8f15\u8f16\u8f17\u8f18\u8f19\u8f1a\u8f1b\u8f1c\u8f1d\u8f1e\u8f1f\u8f20\u8f21\u8f22\u8f23\u8f24\u8f25\u8f26\u8f27\u8f28\u8f29\u8f2a\u8f2b\u8f2c\u8f2d\u8f2e\u8f2f\u8f30\u8f31\u8f32\u8f33\u8f34\u8f35\u8f36\u8f37\u8f38\u8f39\u8f3a\u8f3b\u8f3c\u8f3d\u8f3e\u8f3f\u8f40\u8f41\u8f42\u8f43\u8f44\u8f45\u8f46\u8f47\u8f48\u8f49\u8f4a\u8f4b\u8f4c\u8f4d\u8f4e\u8f4f\u8f50\u8f51\u8f52\u8f53\u8f54\u8f55\u8f56\u8f57\u8f58\u8f59\u8f5a\u8f5b\u8f5c\u8f5d\u8f5e\u8f5f\u8f60\u8f61\u8f62\u8f63\u8f64\u8f65\u8f66\u8f67\u8f68\u8f69\u8f6a\u8f6b\u8f6c\u8f6d\u8f6e\u8f6f\u8f70\u8f71\u8f72\u8f73\u8f74\u8f75\u8f76\u8f77\u8f78\u8f79\u8f7a\u8f7b\u8f7c\u8f7d\u8f7e\u8f7f\u8f80\u8f81\u8f82\u8f83\u8f84\u8f85\u8f86\u8f87\u8f88\u8f89\u8f8a\u8f8b\u8f8c\u8f8d\u8f8e\u8f8f\u8f90\u8f91\u8f92\u8f93\u8f94\u8f95\u8f96\u8f97\u8f98\u8f99\u8f9a\u8f9b\u8f9c\u8f9d\u8f9e\u8f9f\u8fa0\u8fa1\u8fa2\u8fa3\u8fa4\u8fa5\u8fa6\u8fa7\u8fa8\u8fa9\u8faa\u8fab\u8fac\u8fad\u8fae\u8faf\u8fb0\u8fb1\u8fb2\u8fb3\u8fb4\u8fb5\u8fb6\u8fb7\u8fb8\u8fb9\u8fba\u8fbb\u8fbc\u8fbd\u8fbe\u8fbf\u8fc0\u8fc1\u8fc2\u8fc3\u8fc4\u8fc5\u8fc6\u8fc7\u8fc8\u8fc9\u8fca\u8fcb\u8fcc\u8fcd\u8fce\u8fcf\u8fd0\u8fd1\u8fd2\u8fd3\u8fd4\u8fd5\u8fd6\u8fd7\u8fd8\u8fd9\u8fda\u8fdb\u8fdc\u8fdd\u8fde\u8fdf\u8fe0\u8fe1\u8fe2\u8fe3\u8fe4\u8fe5\u8fe6\u8fe7\u8fe8\u8fe9\u8fea\u8feb\u8fec\u8fed\u8fee\u8fef\u8ff0\u8ff1\u8ff2\u8ff3\u8ff4\u8ff5\u8ff6\u8ff7\u8ff8\u8ff9\u8ffa\u8ffb\u8ffc\u8ffd\u8ffe\u8fff\u9000\u9001\u9002\u9003\u9004\u9005\u9006\u9007\u9008\u9009\u900a\u900b\u900c\u900d\u900e\u900f\u9010\u9011\u9012\u9013\u9014\u9015\u9016\u9017\u9018\u9019\u901a\u901b\u901c\u901d\u901e\u901f\u9020\u9021\u9022\u9023\u9024\u9025\u9026\u9027\u9028\u9029\u902a\u902b\u902c\u902d\u902e\u902f\u9030\u9031\u9032\u9033\u9034\u9035\u9036\u9037\u9038\u9039\u903a\u903b\u903c\u903d\u903e\u903f\u9040\u9041\u9042\u9043\u9044\u9045\u9046\u9047\u9048\u9049\u904a\u904b\u904c\u904d\u904e\u904f\u9050\u9051\u9052\u9053\u9054\u9055\u9056\u9057\u9058\u9059\u905a\u905b\u905c\u905d\u905e\u905f\u9060\u9061\u9062\u9063\u9064\u9065\u9066\u9067\u9068\u9069\u906a\u906b\u906c\u906d\u906e\u906f\u9070\u9071\u9072\u9073\u9074\u9075\u9076\u9077\u9078\u9079\u907a\u907b\u907c\u907d\u907e\u907f\u9080\u9081\u9082\u9083\u9084\u9085\u9086\u9087\u9088\u9089\u908a\u908b\u908c\u908d\u908e\u908f\u9090\u9091\u9092\u9093\u9094\u9095\u9096\u9097\u9098\u9099\u909a\u909b\u909c\u909d\u909e\u909f\u90a0\u90a1\u90a2\u90a3\u90a4\u90a5\u90a6\u90a7\u90a8\u90a9\u90aa\u90ab\u90ac\u90ad\u90ae\u90af\u90b0\u90b1\u90b2\u90b3\u90b4\u90b5\u90b6\u90b7\u90b8\u90b9\u90ba\u90bb\u90bc\u90bd\u90be\u90bf\u90c0\u90c1\u90c2\u90c3\u90c4\u90c5\u90c6\u90c7\u90c8\u90c9\u90ca\u90cb\u90cc\u90cd\u90ce\u90cf\u90d0\u90d1\u90d2\u90d3\u90d4\u90d5\u90d6\u90d7\u90d8\u90d9\u90da\u90db\u90dc\u90dd\u90de\u90df\u90e0\u90e1\u90e2\u90e3\u90e4\u90e5\u90e6\u90e7\u90e8\u90e9\u90ea\u90eb\u90ec\u90ed\u90ee\u90ef\u90f0\u90f1\u90f2\u90f3\u90f4\u90f5\u90f6\u90f7\u90f8\u90f9\u90fa\u90fb\u90fc\u90fd\u90fe\u90ff\u9100\u9101\u9102\u9103\u9104\u9105\u9106\u9107\u9108\u9109\u910a\u910b\u910c\u910d\u910e\u910f\u9110\u9111\u9112\u9113\u9114\u9115\u9116\u9117\u9118\u9119\u911a\u911b\u911c\u911d\u911e\u911f\u9120\u9121\u9122\u9123\u9124\u9125\u9126\u9127\u9128\u9129\u912a\u912b\u912c\u912d\u912e\u912f\u9130\u9131\u9132\u9133\u9134\u9135\u9136\u9137\u9138\u9139\u913a\u913b\u913c\u913d\u913e\u913f\u9140\u9141\u9142\u9143\u9144\u9145\u9146\u9147\u9148\u9149\u914a\u914b\u914c\u914d\u914e\u914f\u9150\u9151\u9152\u9153\u9154\u9155\u9156\u9157\u9158\u9159\u915a\u915b\u915c\u915d\u915e\u915f\u9160\u9161\u9162\u9163\u9164\u9165\u9166\u9167\u9168\u9169\u916a\u916b\u916c\u916d\u916e\u916f\u9170\u9171\u9172\u9173\u9174\u9175\u9176\u9177\u9178\u9179\u917a\u917b\u917c\u917d\u917e\u917f\u9180\u9181\u9182\u9183\u9184\u9185\u9186\u9187\u9188\u9189\u918a\u918b\u918c\u918d\u918e\u918f\u9190\u9191\u9192\u9193\u9194\u9195\u9196\u9197\u9198\u9199\u919a\u919b\u919c\u919d\u919e\u919f\u91a0\u91a1\u91a2\u91a3\u91a4\u91a5\u91a6\u91a7\u91a8\u91a9\u91aa\u91ab\u91ac\u91ad\u91ae\u91af\u91b0\u91b1\u91b2\u91b3\u91b4\u91b5\u91b6\u91b7\u91b8\u91b9\u91ba\u91bb\u91bc\u91bd\u91be\u91bf\u91c0\u91c1\u91c2\u91c3\u91c4\u91c5\u91c6\u91c7\u91c8\u91c9\u91ca\u91cb\u91cc\u91cd\u91ce\u91cf\u91d0\u91d1\u91d2\u91d3\u91d4\u91d5\u91d6\u91d7\u91d8\u91d9\u91da\u91db\u91dc\u91dd\u91de\u91df\u91e0\u91e1\u91e2\u91e3\u91e4\u91e5\u91e6\u91e7\u91e8\u91e9\u91ea\u91eb\u91ec\u91ed\u91ee\u91ef\u91f0\u91f1\u91f2\u91f3\u91f4\u91f5\u91f6\u91f7\u91f8\u91f9\u91fa\u91fb\u91fc\u91fd\u91fe\u91ff\u9200\u9201\u9202\u9203\u9204\u9205\u9206\u9207\u9208\u9209\u920a\u920b\u920c\u920d\u920e\u920f\u9210\u9211\u9212\u9213\u9214\u9215\u9216\u9217\u9218\u9219\u921a\u921b\u921c\u921d\u921e\u921f\u9220\u9221\u9222\u9223\u9224\u9225\u9226\u9227\u9228\u9229\u922a\u922b\u922c\u922d\u922e\u922f\u9230\u9231\u9232\u9233\u9234\u9235\u9236\u9237\u9238\u9239\u923a\u923b\u923c\u923d\u923e\u923f\u9240\u9241\u9242\u9243\u9244\u9245\u9246\u9247\u9248\u9249\u924a\u924b\u924c\u924d\u924e\u924f\u9250\u9251\u9252\u9253\u9254\u9255\u9256\u9257\u9258\u9259\u925a\u925b\u925c\u925d\u925e\u925f\u9260\u9261\u9262\u9263\u9264\u9265\u9266\u9267\u9268\u9269\u926a\u926b\u926c\u926d\u926e\u926f\u9270\u9271\u9272\u9273\u9274\u9275\u9276\u9277\u9278\u9279\u927a\u927b\u927c\u927d\u927e\u927f\u9280\u9281\u9282\u9283\u9284\u9285\u9286\u9287\u9288\u9289\u928a\u928b\u928c\u928d\u928e\u928f\u9290\u9291\u9292\u9293\u9294\u9295\u9296\u9297\u9298\u9299\u929a\u929b\u929c\u929d\u929e\u929f\u92a0\u92a1\u92a2\u92a3\u92a4\u92a5\u92a6\u92a7\u92a8\u92a9\u92aa\u92ab\u92ac\u92ad\u92ae\u92af\u92b0\u92b1\u92b2\u92b3\u92b4\u92b5\u92b6\u92b7\u92b8\u92b9\u92ba\u92bb\u92bc\u92bd\u92be\u92bf\u92c0\u92c1\u92c2\u92c3\u92c4\u92c5\u92c6\u92c7\u92c8\u92c9\u92ca\u92cb\u92cc\u92cd\u92ce\u92cf\u92d0\u92d1\u92d2\u92d3\u92d4\u92d5\u92d6\u92d7\u92d8\u92d9\u92da\u92db\u92dc\u92dd\u92de\u92df\u92e0\u92e1\u92e2\u92e3\u92e4\u92e5\u92e6\u92e7\u92e8\u92e9\u92ea\u92eb\u92ec\u92ed\u92ee\u92ef\u92f0\u92f1\u92f2\u92f3\u92f4\u92f5\u92f6\u92f7\u92f8\u92f9\u92fa\u92fb\u92fc\u92fd\u92fe\u92ff\u9300\u9301\u9302\u9303\u9304\u9305\u9306\u9307\u9308\u9309\u930a\u930b\u930c\u930d\u930e\u930f\u9310\u9311\u9312\u9313\u9314\u9315\u9316\u9317\u9318\u9319\u931a\u931b\u931c\u931d\u931e\u931f\u9320\u9321\u9322\u9323\u9324\u9325\u9326\u9327\u9328\u9329\u932a\u932b\u932c\u932d\u932e\u932f\u9330\u9331\u9332\u9333\u9334\u9335\u9336\u9337\u9338\u9339\u933a\u933b\u933c\u933d\u933e\u933f\u9340\u9341\u9342\u9343\u9344\u9345\u9346\u9347\u9348\u9349\u934a\u934b\u934c\u934d\u934e\u934f\u9350\u9351\u9352\u9353\u9354\u9355\u9356\u9357\u9358\u9359\u935a\u935b\u935c\u935d\u935e\u935f\u9360\u9361\u9362\u9363\u9364\u9365\u9366\u9367\u9368\u9369\u936a\u936b\u936c\u936d\u936e\u936f\u9370\u9371\u9372\u9373\u9374\u9375\u9376\u9377\u9378\u9379\u937a\u937b\u937c\u937d\u937e\u937f\u9380\u9381\u9382\u9383\u9384\u9385\u9386\u9387\u9388\u9389\u938a\u938b\u938c\u938d\u938e\u938f\u9390\u9391\u9392\u9393\u9394\u9395\u9396\u9397\u9398\u9399\u939a\u939b\u939c\u939d\u939e\u939f\u93a0\u93a1\u93a2\u93a3\u93a4\u93a5\u93a6\u93a7\u93a8\u93a9\u93aa\u93ab\u93ac\u93ad\u93ae\u93af\u93b0\u93b1\u93b2\u93b3\u93b4\u93b5\u93b6\u93b7\u93b8\u93b9\u93ba\u93bb\u93bc\u93bd\u93be\u93bf\u93c0\u93c1\u93c2\u93c3\u93c4\u93c5\u93c6\u93c7\u93c8\u93c9\u93ca\u93cb\u93cc\u93cd\u93ce\u93cf\u93d0\u93d1\u93d2\u93d3\u93d4\u93d5\u93d6\u93d7\u93d8\u93d9\u93da\u93db\u93dc\u93dd\u93de\u93df\u93e0\u93e1\u93e2\u93e3\u93e4\u93e5\u93e6\u93e7\u93e8\u93e9\u93ea\u93eb\u93ec\u93ed\u93ee\u93ef\u93f0\u93f1\u93f2\u93f3\u93f4\u93f5\u93f6\u93f7\u93f8\u93f9\u93fa\u93fb\u93fc\u93fd\u93fe\u93ff\u9400\u9401\u9402\u9403\u9404\u9405\u9406\u9407\u9408\u9409\u940a\u940b\u940c\u940d\u940e\u940f\u9410\u9411\u9412\u9413\u9414\u9415\u9416\u9417\u9418\u9419\u941a\u941b\u941c\u941d\u941e\u941f\u9420\u9421\u9422\u9423\u9424\u9425\u9426\u9427\u9428\u9429\u942a\u942b\u942c\u942d\u942e\u942f\u9430\u9431\u9432\u9433\u9434\u9435\u9436\u9437\u9438\u9439\u943a\u943b\u943c\u943d\u943e\u943f\u9440\u9441\u9442\u9443\u9444\u9445\u9446\u9447\u9448\u9449\u944a\u944b\u944c\u944d\u944e\u944f\u9450\u9451\u9452\u9453\u9454\u9455\u9456\u9457\u9458\u9459\u945a\u945b\u945c\u945d\u945e\u945f\u9460\u9461\u9462\u9463\u9464\u9465\u9466\u9467\u9468\u9469\u946a\u946b\u946c\u946d\u946e\u946f\u9470\u9471\u9472\u9473\u9474\u9475\u9476\u9477\u9478\u9479\u947a\u947b\u947c\u947d\u947e\u947f\u9480\u9481\u9482\u9483\u9484\u9485\u9486\u9487\u9488\u9489\u948a\u948b\u948c\u948d\u948e\u948f\u9490\u9491\u9492\u9493\u9494\u9495\u9496\u9497\u9498\u9499\u949a\u949b\u949c\u949d\u949e\u949f\u94a0\u94a1\u94a2\u94a3\u94a4\u94a5\u94a6\u94a7\u94a8\u94a9\u94aa\u94ab\u94ac\u94ad\u94ae\u94af\u94b0\u94b1\u94b2\u94b3\u94b4\u94b5\u94b6\u94b7\u94b8\u94b9\u94ba\u94bb\u94bc\u94bd\u94be\u94bf\u94c0\u94c1\u94c2\u94c3\u94c4\u94c5\u94c6\u94c7\u94c8\u94c9\u94ca\u94cb\u94cc\u94cd\u94ce\u94cf\u94d0\u94d1\u94d2\u94d3\u94d4\u94d5\u94d6\u94d7\u94d8\u94d9\u94da\u94db\u94dc\u94dd\u94de\u94df\u94e0\u94e1\u94e2\u94e3\u94e4\u94e5\u94e6\u94e7\u94e8\u94e9\u94ea\u94eb\u94ec\u94ed\u94ee\u94ef\u94f0\u94f1\u94f2\u94f3\u94f4\u94f5\u94f6\u94f7\u94f8\u94f9\u94fa\u94fb\u94fc\u94fd\u94fe\u94ff\u9500\u9501\u9502\u9503\u9504\u9505\u9506\u9507\u9508\u9509\u950a\u950b\u950c\u950d\u950e\u950f\u9510\u9511\u9512\u9513\u9514\u9515\u9516\u9517\u9518\u9519\u951a\u951b\u951c\u951d\u951e\u951f\u9520\u9521\u9522\u9523\u9524\u9525\u9526\u9527\u9528\u9529\u952a\u952b\u952c\u952d\u952e\u952f\u9530\u9531\u9532\u9533\u9534\u9535\u9536\u9537\u9538\u9539\u953a\u953b\u953c\u953d\u953e\u953f\u9540\u9541\u9542\u9543\u9544\u9545\u9546\u9547\u9548\u9549\u954a\u954b\u954c\u954d\u954e\u954f\u9550\u9551\u9552\u9553\u9554\u9555\u9556\u9557\u9558\u9559\u955a\u955b\u955c\u955d\u955e\u955f\u9560\u9561\u9562\u9563\u9564\u9565\u9566\u9567\u9568\u9569\u956a\u956b\u956c\u956d\u956e\u956f\u9570\u9571\u9572\u9573\u9574\u9575\u9576\u9577\u9578\u9579\u957a\u957b\u957c\u957d\u957e\u957f\u9580\u9581\u9582\u9583\u9584\u9585\u9586\u9587\u9588\u9589\u958a\u958b\u958c\u958d\u958e\u958f\u9590\u9591\u9592\u9593\u9594\u9595\u9596\u9597\u9598\u9599\u959a\u959b\u959c\u959d\u959e\u959f\u95a0\u95a1\u95a2\u95a3\u95a4\u95a5\u95a6\u95a7\u95a8\u95a9\u95aa\u95ab\u95ac\u95ad\u95ae\u95af\u95b0\u95b1\u95b2\u95b3\u95b4\u95b5\u95b6\u95b7\u95b8\u95b9\u95ba\u95bb\u95bc\u95bd\u95be\u95bf\u95c0\u95c1\u95c2\u95c3\u95c4\u95c5\u95c6\u95c7\u95c8\u95c9\u95ca\u95cb\u95cc\u95cd\u95ce\u95cf\u95d0\u95d1\u95d2\u95d3\u95d4\u95d5\u95d6\u95d7\u95d8\u95d9\u95da\u95db\u95dc\u95dd\u95de\u95df\u95e0\u95e1\u95e2\u95e3\u95e4\u95e5\u95e6\u95e7\u95e8\u95e9\u95ea\u95eb\u95ec\u95ed\u95ee\u95ef\u95f0\u95f1\u95f2\u95f3\u95f4\u95f5\u95f6\u95f7\u95f8\u95f9\u95fa\u95fb\u95fc\u95fd\u95fe\u95ff\u9600\u9601\u9602\u9603\u9604\u9605\u9606\u9607\u9608\u9609\u960a\u960b\u960c\u960d\u960e\u960f\u9610\u9611\u9612\u9613\u9614\u9615\u9616\u9617\u9618\u9619\u961a\u961b\u961c\u961d\u961e\u961f\u9620\u9621\u9622\u9623\u9624\u9625\u9626\u9627\u9628\u9629\u962a\u962b\u962c\u962d\u962e\u962f\u9630\u9631\u9632\u9633\u9634\u9635\u9636\u9637\u9638\u9639\u963a\u963b\u963c\u963d\u963e\u963f\u9640\u9641\u9642\u9643\u9644\u9645\u9646\u9647\u9648\u9649\u964a\u964b\u964c\u964d\u964e\u964f\u9650\u9651\u9652\u9653\u9654\u9655\u9656\u9657\u9658\u9659\u965a\u965b\u965c\u965d\u965e\u965f\u9660\u9661\u9662\u9663\u9664\u9665\u9666\u9667\u9668\u9669\u966a\u966b\u966c\u966d\u966e\u966f\u9670\u9671\u9672\u9673\u9674\u9675\u9676\u9677\u9678\u9679\u967a\u967b\u967c\u967d\u967e\u967f\u9680\u9681\u9682\u9683\u9684\u9685\u9686\u9687\u9688\u9689\u968a\u968b\u968c\u968d\u968e\u968f\u9690\u9691\u9692\u9693\u9694\u9695\u9696\u9697\u9698\u9699\u969a\u969b\u969c\u969d\u969e\u969f\u96a0\u96a1\u96a2\u96a3\u96a4\u96a5\u96a6\u96a7\u96a8\u96a9\u96aa\u96ab\u96ac\u96ad\u96ae\u96af\u96b0\u96b1\u96b2\u96b3\u96b4\u96b5\u96b6\u96b7\u96b8\u96b9\u96ba\u96bb\u96bc\u96bd\u96be\u96bf\u96c0\u96c1\u96c2\u96c3\u96c4\u96c5\u96c6\u96c7\u96c8\u96c9\u96ca\u96cb\u96cc\u96cd\u96ce\u96cf\u96d0\u96d1\u96d2\u96d3\u96d4\u96d5\u96d6\u96d7\u96d8\u96d9\u96da\u96db\u96dc\u96dd\u96de\u96df\u96e0\u96e1\u96e2\u96e3\u96e4\u96e5\u96e6\u96e7\u96e8\u96e9\u96ea\u96eb\u96ec\u96ed\u96ee\u96ef\u96f0\u96f1\u96f2\u96f3\u96f4\u96f5\u96f6\u96f7\u96f8\u96f9\u96fa\u96fb\u96fc\u96fd\u96fe\u96ff\u9700\u9701\u9702\u9703\u9704\u9705\u9706\u9707\u9708\u9709\u970a\u970b\u970c\u970d\u970e\u970f\u9710\u9711\u9712\u9713\u9714\u9715\u9716\u9717\u9718\u9719\u971a\u971b\u971c\u971d\u971e\u971f\u9720\u9721\u9722\u9723\u9724\u9725\u9726\u9727\u9728\u9729\u972a\u972b\u972c\u972d\u972e\u972f\u9730\u9731\u9732\u9733\u9734\u9735\u9736\u9737\u9738\u9739\u973a\u973b\u973c\u973d\u973e\u973f\u9740\u9741\u9742\u9743\u9744\u9745\u9746\u9747\u9748\u9749\u974a\u974b\u974c\u974d\u974e\u974f\u9750\u9751\u9752\u9753\u9754\u9755\u9756\u9757\u9758\u9759\u975a\u975b\u975c\u975d\u975e\u975f\u9760\u9761\u9762\u9763\u9764\u9765\u9766\u9767\u9768\u9769\u976a\u976b\u976c\u976d\u976e\u976f\u9770\u9771\u9772\u9773\u9774\u9775\u9776\u9777\u9778\u9779\u977a\u977b\u977c\u977d\u977e\u977f\u9780\u9781\u9782\u9783\u9784\u9785\u9786\u9787\u9788\u9789\u978a\u978b\u978c\u978d\u978e\u978f\u9790\u9791\u9792\u9793\u9794\u9795\u9796\u9797\u9798\u9799\u979a\u979b\u979c\u979d\u979e\u979f\u97a0\u97a1\u97a2\u97a3\u97a4\u97a5\u97a6\u97a7\u97a8\u97a9\u97aa\u97ab\u97ac\u97ad\u97ae\u97af\u97b0\u97b1\u97b2\u97b3\u97b4\u97b5\u97b6\u97b7\u97b8\u97b9\u97ba\u97bb\u97bc\u97bd\u97be\u97bf\u97c0\u97c1\u97c2\u97c3\u97c4\u97c5\u97c6\u97c7\u97c8\u97c9\u97ca\u97cb\u97cc\u97cd\u97ce\u97cf\u97d0\u97d1\u97d2\u97d3\u97d4\u97d5\u97d6\u97d7\u97d8\u97d9\u97da\u97db\u97dc\u97dd\u97de\u97df\u97e0\u97e1\u97e2\u97e3\u97e4\u97e5\u97e6\u97e7\u97e8\u97e9\u97ea\u97eb\u97ec\u97ed\u97ee\u97ef\u97f0\u97f1\u97f2\u97f3\u97f4\u97f5\u97f6\u97f7\u97f8\u97f9\u97fa\u97fb\u97fc\u97fd\u97fe\u97ff\u9800\u9801\u9802\u9803\u9804\u9805\u9806\u9807\u9808\u9809\u980a\u980b\u980c\u980d\u980e\u980f\u9810\u9811\u9812\u9813\u9814\u9815\u9816\u9817\u9818\u9819\u981a\u981b\u981c\u981d\u981e\u981f\u9820\u9821\u9822\u9823\u9824\u9825\u9826\u9827\u9828\u9829\u982a\u982b\u982c\u982d\u982e\u982f\u9830\u9831\u9832\u9833\u9834\u9835\u9836\u9837\u9838\u9839\u983a\u983b\u983c\u983d\u983e\u983f\u9840\u9841\u9842\u9843\u9844\u9845\u9846\u9847\u9848\u9849\u984a\u984b\u984c\u984d\u984e\u984f\u9850\u9851\u9852\u9853\u9854\u9855\u9856\u9857\u9858\u9859\u985a\u985b\u985c\u985d\u985e\u985f\u9860\u9861\u9862\u9863\u9864\u9865\u9866\u9867\u9868\u9869\u986a\u986b\u986c\u986d\u986e\u986f\u9870\u9871\u9872\u9873\u9874\u9875\u9876\u9877\u9878\u9879\u987a\u987b\u987c\u987d\u987e\u987f\u9880\u9881\u9882\u9883\u9884\u9885\u9886\u9887\u9888\u9889\u988a\u988b\u988c\u988d\u988e\u988f\u9890\u9891\u9892\u9893\u9894\u9895\u9896\u9897\u9898\u9899\u989a\u989b\u989c\u989d\u989e\u989f\u98a0\u98a1\u98a2\u98a3\u98a4\u98a5\u98a6\u98a7\u98a8\u98a9\u98aa\u98ab\u98ac\u98ad\u98ae\u98af\u98b0\u98b1\u98b2\u98b3\u98b4\u98b5\u98b6\u98b7\u98b8\u98b9\u98ba\u98bb\u98bc\u98bd\u98be\u98bf\u98c0\u98c1\u98c2\u98c3\u98c4\u98c5\u98c6\u98c7\u98c8\u98c9\u98ca\u98cb\u98cc\u98cd\u98ce\u98cf\u98d0\u98d1\u98d2\u98d3\u98d4\u98d5\u98d6\u98d7\u98d8\u98d9\u98da\u98db\u98dc\u98dd\u98de\u98df\u98e0\u98e1\u98e2\u98e3\u98e4\u98e5\u98e6\u98e7\u98e8\u98e9\u98ea\u98eb\u98ec\u98ed\u98ee\u98ef\u98f0\u98f1\u98f2\u98f3\u98f4\u98f5\u98f6\u98f7\u98f8\u98f9\u98fa\u98fb\u98fc\u98fd\u98fe\u98ff\u9900\u9901\u9902\u9903\u9904\u9905\u9906\u9907\u9908\u9909\u990a\u990b\u990c\u990d\u990e\u990f\u9910\u9911\u9912\u9913\u9914\u9915\u9916\u9917\u9918\u9919\u991a\u991b\u991c\u991d\u991e\u991f\u9920\u9921\u9922\u9923\u9924\u9925\u9926\u9927\u9928\u9929\u992a\u992b\u992c\u992d\u992e\u992f\u9930\u9931\u9932\u9933\u9934\u9935\u9936\u9937\u9938\u9939\u993a\u993b\u993c\u993d\u993e\u993f\u9940\u9941\u9942\u9943\u9944\u9945\u9946\u9947\u9948\u9949\u994a\u994b\u994c\u994d\u994e\u994f\u9950\u9951\u9952\u9953\u9954\u9955\u9956\u9957\u9958\u9959\u995a\u995b\u995c\u995d\u995e\u995f\u9960\u9961\u9962\u9963\u9964\u9965\u9966\u9967\u9968\u9969\u996a\u996b\u996c\u996d\u996e\u996f\u9970\u9971\u9972\u9973\u9974\u9975\u9976\u9977\u9978\u9979\u997a\u997b\u997c\u997d\u997e\u997f\u9980\u9981\u9982\u9983\u9984\u9985\u9986\u9987\u9988\u9989\u998a\u998b\u998c\u998d\u998e\u998f\u9990\u9991\u9992\u9993\u9994\u9995\u9996\u9997\u9998\u9999\u999a\u999b\u999c\u999d\u999e\u999f\u99a0\u99a1\u99a2\u99a3\u99a4\u99a5\u99a6\u99a7\u99a8\u99a9\u99aa\u99ab\u99ac\u99ad\u99ae\u99af\u99b0\u99b1\u99b2\u99b3\u99b4\u99b5\u99b6\u99b7\u99b8\u99b9\u99ba\u99bb\u99bc\u99bd\u99be\u99bf\u99c0\u99c1\u99c2\u99c3\u99c4\u99c5\u99c6\u99c7\u99c8\u99c9\u99ca\u99cb\u99cc\u99cd\u99ce\u99cf\u99d0\u99d1\u99d2\u99d3\u99d4\u99d5\u99d6\u99d7\u99d8\u99d9\u99da\u99db\u99dc\u99dd\u99de\u99df\u99e0\u99e1\u99e2\u99e3\u99e4\u99e5\u99e6\u99e7\u99e8\u99e9\u99ea\u99eb\u99ec\u99ed\u99ee\u99ef\u99f0\u99f1\u99f2\u99f3\u99f4\u99f5\u99f6\u99f7\u99f8\u99f9\u99fa\u99fb\u99fc\u99fd\u99fe\u99ff\u9a00\u9a01\u9a02\u9a03\u9a04\u9a05\u9a06\u9a07\u9a08\u9a09\u9a0a\u9a0b\u9a0c\u9a0d\u9a0e\u9a0f\u9a10\u9a11\u9a12\u9a13\u9a14\u9a15\u9a16\u9a17\u9a18\u9a19\u9a1a\u9a1b\u9a1c\u9a1d\u9a1e\u9a1f\u9a20\u9a21\u9a22\u9a23\u9a24\u9a25\u9a26\u9a27\u9a28\u9a29\u9a2a\u9a2b\u9a2c\u9a2d\u9a2e\u9a2f\u9a30\u9a31\u9a32\u9a33\u9a34\u9a35\u9a36\u9a37\u9a38\u9a39\u9a3a\u9a3b\u9a3c\u9a3d\u9a3e\u9a3f\u9a40\u9a41\u9a42\u9a43\u9a44\u9a45\u9a46\u9a47\u9a48\u9a49\u9a4a\u9a4b\u9a4c\u9a4d\u9a4e\u9a4f\u9a50\u9a51\u9a52\u9a53\u9a54\u9a55\u9a56\u9a57\u9a58\u9a59\u9a5a\u9a5b\u9a5c\u9a5d\u9a5e\u9a5f\u9a60\u9a61\u9a62\u9a63\u9a64\u9a65\u9a66\u9a67\u9a68\u9a69\u9a6a\u9a6b\u9a6c\u9a6d\u9a6e\u9a6f\u9a70\u9a71\u9a72\u9a73\u9a74\u9a75\u9a76\u9a77\u9a78\u9a79\u9a7a\u9a7b\u9a7c\u9a7d\u9a7e\u9a7f\u9a80\u9a81\u9a82\u9a83\u9a84\u9a85\u9a86\u9a87\u9a88\u9a89\u9a8a\u9a8b\u9a8c\u9a8d\u9a8e\u9a8f\u9a90\u9a91\u9a92\u9a93\u9a94\u9a95\u9a96\u9a97\u9a98\u9a99\u9a9a\u9a9b\u9a9c\u9a9d\u9a9e\u9a9f\u9aa0\u9aa1\u9aa2\u9aa3\u9aa4\u9aa5\u9aa6\u9aa7\u9aa8\u9aa9\u9aaa\u9aab\u9aac\u9aad\u9aae\u9aaf\u9ab0\u9ab1\u9ab2\u9ab3\u9ab4\u9ab5\u9ab6\u9ab7\u9ab8\u9ab9\u9aba\u9abb\u9abc\u9abd\u9abe\u9abf\u9ac0\u9ac1\u9ac2\u9ac3\u9ac4\u9ac5\u9ac6\u9ac7\u9ac8\u9ac9\u9aca\u9acb\u9acc\u9acd\u9ace\u9acf\u9ad0\u9ad1\u9ad2\u9ad3\u9ad4\u9ad5\u9ad6\u9ad7\u9ad8\u9ad9\u9ada\u9adb\u9adc\u9add\u9ade\u9adf\u9ae0\u9ae1\u9ae2\u9ae3\u9ae4\u9ae5\u9ae6\u9ae7\u9ae8\u9ae9\u9aea\u9aeb\u9aec\u9aed\u9aee\u9aef\u9af0\u9af1\u9af2\u9af3\u9af4\u9af5\u9af6\u9af7\u9af8\u9af9\u9afa\u9afb\u9afc\u9afd\u9afe\u9aff\u9b00\u9b01\u9b02\u9b03\u9b04\u9b05\u9b06\u9b07\u9b08\u9b09\u9b0a\u9b0b\u9b0c\u9b0d\u9b0e\u9b0f\u9b10\u9b11\u9b12\u9b13\u9b14\u9b15\u9b16\u9b17\u9b18\u9b19\u9b1a\u9b1b\u9b1c\u9b1d\u9b1e\u9b1f\u9b20\u9b21\u9b22\u9b23\u9b24\u9b25\u9b26\u9b27\u9b28\u9b29\u9b2a\u9b2b\u9b2c\u9b2d\u9b2e\u9b2f\u9b30\u9b31\u9b32\u9b33\u9b34\u9b35\u9b36\u9b37\u9b38\u9b39\u9b3a\u9b3b\u9b3c\u9b3d\u9b3e\u9b3f\u9b40\u9b41\u9b42\u9b43\u9b44\u9b45\u9b46\u9b47\u9b48\u9b49\u9b4a\u9b4b\u9b4c\u9b4d\u9b4e\u9b4f\u9b50\u9b51\u9b52\u9b53\u9b54\u9b55\u9b56\u9b57\u9b58\u9b59\u9b5a\u9b5b\u9b5c\u9b5d\u9b5e\u9b5f\u9b60\u9b61\u9b62\u9b63\u9b64\u9b65\u9b66\u9b67\u9b68\u9b69\u9b6a\u9b6b\u9b6c\u9b6d\u9b6e\u9b6f\u9b70\u9b71\u9b72\u9b73\u9b74\u9b75\u9b76\u9b77\u9b78\u9b79\u9b7a\u9b7b\u9b7c\u9b7d\u9b7e\u9b7f\u9b80\u9b81\u9b82\u9b83\u9b84\u9b85\u9b86\u9b87\u9b88\u9b89\u9b8a\u9b8b\u9b8c\u9b8d\u9b8e\u9b8f\u9b90\u9b91\u9b92\u9b93\u9b94\u9b95\u9b96\u9b97\u9b98\u9b99\u9b9a\u9b9b\u9b9c\u9b9d\u9b9e\u9b9f\u9ba0\u9ba1\u9ba2\u9ba3\u9ba4\u9ba5\u9ba6\u9ba7\u9ba8\u9ba9\u9baa\u9bab\u9bac\u9bad\u9bae\u9baf\u9bb0\u9bb1\u9bb2\u9bb3\u9bb4\u9bb5\u9bb6\u9bb7\u9bb8\u9bb9\u9bba\u9bbb\u9bbc\u9bbd\u9bbe\u9bbf\u9bc0\u9bc1\u9bc2\u9bc3\u9bc4\u9bc5\u9bc6\u9bc7\u9bc8\u9bc9\u9bca\u9bcb\u9bcc\u9bcd\u9bce\u9bcf\u9bd0\u9bd1\u9bd2\u9bd3\u9bd4\u9bd5\u9bd6\u9bd7\u9bd8\u9bd9\u9bda\u9bdb\u9bdc\u9bdd\u9bde\u9bdf\u9be0\u9be1\u9be2\u9be3\u9be4\u9be5\u9be6\u9be7\u9be8\u9be9\u9bea\u9beb\u9bec\u9bed\u9bee\u9bef\u9bf0\u9bf1\u9bf2\u9bf3\u9bf4\u9bf5\u9bf6\u9bf7\u9bf8\u9bf9\u9bfa\u9bfb\u9bfc\u9bfd\u9bfe\u9bff\u9c00\u9c01\u9c02\u9c03\u9c04\u9c05\u9c06\u9c07\u9c08\u9c09\u9c0a\u9c0b\u9c0c\u9c0d\u9c0e\u9c0f\u9c10\u9c11\u9c12\u9c13\u9c14\u9c15\u9c16\u9c17\u9c18\u9c19\u9c1a\u9c1b\u9c1c\u9c1d\u9c1e\u9c1f\u9c20\u9c21\u9c22\u9c23\u9c24\u9c25\u9c26\u9c27\u9c28\u9c29\u9c2a\u9c2b\u9c2c\u9c2d\u9c2e\u9c2f\u9c30\u9c31\u9c32\u9c33\u9c34\u9c35\u9c36\u9c37\u9c38\u9c39\u9c3a\u9c3b\u9c3c\u9c3d\u9c3e\u9c3f\u9c40\u9c41\u9c42\u9c43\u9c44\u9c45\u9c46\u9c47\u9c48\u9c49\u9c4a\u9c4b\u9c4c\u9c4d\u9c4e\u9c4f\u9c50\u9c51\u9c52\u9c53\u9c54\u9c55\u9c56\u9c57\u9c58\u9c59\u9c5a\u9c5b\u9c5c\u9c5d\u9c5e\u9c5f\u9c60\u9c61\u9c62\u9c63\u9c64\u9c65\u9c66\u9c67\u9c68\u9c69\u9c6a\u9c6b\u9c6c\u9c6d\u9c6e\u9c6f\u9c70\u9c71\u9c72\u9c73\u9c74\u9c75\u9c76\u9c77\u9c78\u9c79\u9c7a\u9c7b\u9c7c\u9c7d\u9c7e\u9c7f\u9c80\u9c81\u9c82\u9c83\u9c84\u9c85\u9c86\u9c87\u9c88\u9c89\u9c8a\u9c8b\u9c8c\u9c8d\u9c8e\u9c8f\u9c90\u9c91\u9c92\u9c93\u9c94\u9c95\u9c96\u9c97\u9c98\u9c99\u9c9a\u9c9b\u9c9c\u9c9d\u9c9e\u9c9f\u9ca0\u9ca1\u9ca2\u9ca3\u9ca4\u9ca5\u9ca6\u9ca7\u9ca8\u9ca9\u9caa\u9cab\u9cac\u9cad\u9cae\u9caf\u9cb0\u9cb1\u9cb2\u9cb3\u9cb4\u9cb5\u9cb6\u9cb7\u9cb8\u9cb9\u9cba\u9cbb\u9cbc\u9cbd\u9cbe\u9cbf\u9cc0\u9cc1\u9cc2\u9cc3\u9cc4\u9cc5\u9cc6\u9cc7\u9cc8\u9cc9\u9cca\u9ccb\u9ccc\u9ccd\u9cce\u9ccf\u9cd0\u9cd1\u9cd2\u9cd3\u9cd4\u9cd5\u9cd6\u9cd7\u9cd8\u9cd9\u9cda\u9cdb\u9cdc\u9cdd\u9cde\u9cdf\u9ce0\u9ce1\u9ce2\u9ce3\u9ce4\u9ce5\u9ce6\u9ce7\u9ce8\u9ce9\u9cea\u9ceb\u9cec\u9ced\u9cee\u9cef\u9cf0\u9cf1\u9cf2\u9cf3\u9cf4\u9cf5\u9cf6\u9cf7\u9cf8\u9cf9\u9cfa\u9cfb\u9cfc\u9cfd\u9cfe\u9cff\u9d00\u9d01\u9d02\u9d03\u9d04\u9d05\u9d06\u9d07\u9d08\u9d09\u9d0a\u9d0b\u9d0c\u9d0d\u9d0e\u9d0f\u9d10\u9d11\u9d12\u9d13\u9d14\u9d15\u9d16\u9d17\u9d18\u9d19\u9d1a\u9d1b\u9d1c\u9d1d\u9d1e\u9d1f\u9d20\u9d21\u9d22\u9d23\u9d24\u9d25\u9d26\u9d27\u9d28\u9d29\u9d2a\u9d2b\u9d2c\u9d2d\u9d2e\u9d2f\u9d30\u9d31\u9d32\u9d33\u9d34\u9d35\u9d36\u9d37\u9d38\u9d39\u9d3a\u9d3b\u9d3c\u9d3d\u9d3e\u9d3f\u9d40\u9d41\u9d42\u9d43\u9d44\u9d45\u9d46\u9d47\u9d48\u9d49\u9d4a\u9d4b\u9d4c\u9d4d\u9d4e\u9d4f\u9d50\u9d51\u9d52\u9d53\u9d54\u9d55\u9d56\u9d57\u9d58\u9d59\u9d5a\u9d5b\u9d5c\u9d5d\u9d5e\u9d5f\u9d60\u9d61\u9d62\u9d63\u9d64\u9d65\u9d66\u9d67\u9d68\u9d69\u9d6a\u9d6b\u9d6c\u9d6d\u9d6e\u9d6f\u9d70\u9d71\u9d72\u9d73\u9d74\u9d75\u9d76\u9d77\u9d78\u9d79\u9d7a\u9d7b\u9d7c\u9d7d\u9d7e\u9d7f\u9d80\u9d81\u9d82\u9d83\u9d84\u9d85\u9d86\u9d87\u9d88\u9d89\u9d8a\u9d8b\u9d8c\u9d8d\u9d8e\u9d8f\u9d90\u9d91\u9d92\u9d93\u9d94\u9d95\u9d96\u9d97\u9d98\u9d99\u9d9a\u9d9b\u9d9c\u9d9d\u9d9e\u9d9f\u9da0\u9da1\u9da2\u9da3\u9da4\u9da5\u9da6\u9da7\u9da8\u9da9\u9daa\u9dab\u9dac\u9dad\u9dae\u9daf\u9db0\u9db1\u9db2\u9db3\u9db4\u9db5\u9db6\u9db7\u9db8\u9db9\u9dba\u9dbb\u9dbc\u9dbd\u9dbe\u9dbf\u9dc0\u9dc1\u9dc2\u9dc3\u9dc4\u9dc5\u9dc6\u9dc7\u9dc8\u9dc9\u9dca\u9dcb\u9dcc\u9dcd\u9dce\u9dcf\u9dd0\u9dd1\u9dd2\u9dd3\u9dd4\u9dd5\u9dd6\u9dd7\u9dd8\u9dd9\u9dda\u9ddb\u9ddc\u9ddd\u9dde\u9ddf\u9de0\u9de1\u9de2\u9de3\u9de4\u9de5\u9de6\u9de7\u9de8\u9de9\u9dea\u9deb\u9dec\u9ded\u9dee\u9def\u9df0\u9df1\u9df2\u9df3\u9df4\u9df5\u9df6\u9df7\u9df8\u9df9\u9dfa\u9dfb\u9dfc\u9dfd\u9dfe\u9dff\u9e00\u9e01\u9e02\u9e03\u9e04\u9e05\u9e06\u9e07\u9e08\u9e09\u9e0a\u9e0b\u9e0c\u9e0d\u9e0e\u9e0f\u9e10\u9e11\u9e12\u9e13\u9e14\u9e15\u9e16\u9e17\u9e18\u9e19\u9e1a\u9e1b\u9e1c\u9e1d\u9e1e\u9e1f\u9e20\u9e21\u9e22\u9e23\u9e24\u9e25\u9e26\u9e27\u9e28\u9e29\u9e2a\u9e2b\u9e2c\u9e2d\u9e2e\u9e2f\u9e30\u9e31\u9e32\u9e33\u9e34\u9e35\u9e36\u9e37\u9e38\u9e39\u9e3a\u9e3b\u9e3c\u9e3d\u9e3e\u9e3f\u9e40\u9e41\u9e42\u9e43\u9e44\u9e45\u9e46\u9e47\u9e48\u9e49\u9e4a\u9e4b\u9e4c\u9e4d\u9e4e\u9e4f\u9e50\u9e51\u9e52\u9e53\u9e54\u9e55\u9e56\u9e57\u9e58\u9e59\u9e5a\u9e5b\u9e5c\u9e5d\u9e5e\u9e5f\u9e60\u9e61\u9e62\u9e63\u9e64\u9e65\u9e66\u9e67\u9e68\u9e69\u9e6a\u9e6b\u9e6c\u9e6d\u9e6e\u9e6f\u9e70\u9e71\u9e72\u9e73\u9e74\u9e75\u9e76\u9e77\u9e78\u9e79\u9e7a\u9e7b\u9e7c\u9e7d\u9e7e\u9e7f\u9e80\u9e81\u9e82\u9e83\u9e84\u9e85\u9e86\u9e87\u9e88\u9e89\u9e8a\u9e8b\u9e8c\u9e8d\u9e8e\u9e8f\u9e90\u9e91\u9e92\u9e93\u9e94\u9e95\u9e96\u9e97\u9e98\u9e99\u9e9a\u9e9b\u9e9c\u9e9d\u9e9e\u9e9f\u9ea0\u9ea1\u9ea2\u9ea3\u9ea4\u9ea5\u9ea6\u9ea7\u9ea8\u9ea9\u9eaa\u9eab\u9eac\u9ead\u9eae\u9eaf\u9eb0\u9eb1\u9eb2\u9eb3\u9eb4\u9eb5\u9eb6\u9eb7\u9eb8\u9eb9\u9eba\u9ebb\u9ebc\u9ebd\u9ebe\u9ebf\u9ec0\u9ec1\u9ec2\u9ec3\u9ec4\u9ec5\u9ec6\u9ec7\u9ec8\u9ec9\u9eca\u9ecb\u9ecc\u9ecd\u9ece\u9ecf\u9ed0\u9ed1\u9ed2\u9ed3\u9ed4\u9ed5\u9ed6\u9ed7\u9ed8\u9ed9\u9eda\u9edb\u9edc\u9edd\u9ede\u9edf\u9ee0\u9ee1\u9ee2\u9ee3\u9ee4\u9ee5\u9ee6\u9ee7\u9ee8\u9ee9\u9eea\u9eeb\u9eec\u9eed\u9eee\u9eef\u9ef0\u9ef1\u9ef2\u9ef3\u9ef4\u9ef5\u9ef6\u9ef7\u9ef8\u9ef9\u9efa\u9efb\u9efc\u9efd\u9efe\u9eff\u9f00\u9f01\u9f02\u9f03\u9f04\u9f05\u9f06\u9f07\u9f08\u9f09\u9f0a\u9f0b\u9f0c\u9f0d\u9f0e\u9f0f\u9f10\u9f11\u9f12\u9f13\u9f14\u9f15\u9f16\u9f17\u9f18\u9f19\u9f1a\u9f1b\u9f1c\u9f1d\u9f1e\u9f1f\u9f20\u9f21\u9f22\u9f23\u9f24\u9f25\u9f26\u9f27\u9f28\u9f29\u9f2a\u9f2b\u9f2c\u9f2d\u9f2e\u9f2f\u9f30\u9f31\u9f32\u9f33\u9f34\u9f35\u9f36\u9f37\u9f38\u9f39\u9f3a\u9f3b\u9f3c\u9f3d\u9f3e\u9f3f\u9f40\u9f41\u9f42\u9f43\u9f44\u9f45\u9f46\u9f47\u9f48\u9f49\u9f4a\u9f4b\u9f4c\u9f4d\u9f4e\u9f4f\u9f50\u9f51\u9f52\u9f53\u9f54\u9f55\u9f56\u9f57\u9f58\u9f59\u9f5a\u9f5b\u9f5c\u9f5d\u9f5e\u9f5f\u9f60\u9f61\u9f62\u9f63\u9f64\u9f65\u9f66\u9f67\u9f68\u9f69\u9f6a\u9f6b\u9f6c\u9f6d\u9f6e\u9f6f\u9f70\u9f71\u9f72\u9f73\u9f74\u9f75\u9f76\u9f77\u9f78\u9f79\u9f7a\u9f7b\u9f7c\u9f7d\u9f7e\u9f7f\u9f80\u9f81\u9f82\u9f83\u9f84\u9f85\u9f86\u9f87\u9f88\u9f89\u9f8a\u9f8b\u9f8c\u9f8d\u9f8e\u9f8f\u9f90\u9f91\u9f92\u9f93\u9f94\u9f95\u9f96\u9f97\u9f98\u9f99\u9f9a\u9f9b\u9f9c\u9f9d\u9f9e\u9f9f\u9fa0\u9fa1\u9fa2\u9fa3\u9fa4\u9fa5\u9fa6\u9fa7\u9fa8\u9fa9\u9faa\u9fab\u9fac\u9fad\u9fae\u9faf\u9fb0\u9fb1\u9fb2\u9fb3\u9fb4\u9fb5\u9fb6\u9fb7\u9fb8\u9fb9\u9fba\u9fbb\ua000\ua001\ua002\ua003\ua004\ua005\ua006\ua007\ua008\ua009\ua00a\ua00b\ua00c\ua00d\ua00e\ua00f\ua010\ua011\ua012\ua013\ua014\ua016\ua017\ua018\ua019\ua01a\ua01b\ua01c\ua01d\ua01e\ua01f\ua020\ua021\ua022\ua023\ua024\ua025\ua026\ua027\ua028\ua029\ua02a\ua02b\ua02c\ua02d\ua02e\ua02f\ua030\ua031\ua032\ua033\ua034\ua035\ua036\ua037\ua038\ua039\ua03a\ua03b\ua03c\ua03d\ua03e\ua03f\ua040\ua041\ua042\ua043\ua044\ua045\ua046\ua047\ua048\ua049\ua04a\ua04b\ua04c\ua04d\ua04e\ua04f\ua050\ua051\ua052\ua053\ua054\ua055\ua056\ua057\ua058\ua059\ua05a\ua05b\ua05c\ua05d\ua05e\ua05f\ua060\ua061\ua062\ua063\ua064\ua065\ua066\ua067\ua068\ua069\ua06a\ua06b\ua06c\ua06d\ua06e\ua06f\ua070\ua071\ua072\ua073\ua074\ua075\ua076\ua077\ua078\ua079\ua07a\ua07b\ua07c\ua07d\ua07e\ua07f\ua080\ua081\ua082\ua083\ua084\ua085\ua086\ua087\ua088\ua089\ua08a\ua08b\ua08c\ua08d\ua08e\ua08f\ua090\ua091\ua092\ua093\ua094\ua095\ua096\ua097\ua098\ua099\ua09a\ua09b\ua09c\ua09d\ua09e\ua09f\ua0a0\ua0a1\ua0a2\ua0a3\ua0a4\ua0a5\ua0a6\ua0a7\ua0a8\ua0a9\ua0aa\ua0ab\ua0ac\ua0ad\ua0ae\ua0af\ua0b0\ua0b1\ua0b2\ua0b3\ua0b4\ua0b5\ua0b6\ua0b7\ua0b8\ua0b9\ua0ba\ua0bb\ua0bc\ua0bd\ua0be\ua0bf\ua0c0\ua0c1\ua0c2\ua0c3\ua0c4\ua0c5\ua0c6\ua0c7\ua0c8\ua0c9\ua0ca\ua0cb\ua0cc\ua0cd\ua0ce\ua0cf\ua0d0\ua0d1\ua0d2\ua0d3\ua0d4\ua0d5\ua0d6\ua0d7\ua0d8\ua0d9\ua0da\ua0db\ua0dc\ua0dd\ua0de\ua0df\ua0e0\ua0e1\ua0e2\ua0e3\ua0e4\ua0e5\ua0e6\ua0e7\ua0e8\ua0e9\ua0ea\ua0eb\ua0ec\ua0ed\ua0ee\ua0ef\ua0f0\ua0f1\ua0f2\ua0f3\ua0f4\ua0f5\ua0f6\ua0f7\ua0f8\ua0f9\ua0fa\ua0fb\ua0fc\ua0fd\ua0fe\ua0ff\ua100\ua101\ua102\ua103\ua104\ua105\ua106\ua107\ua108\ua109\ua10a\ua10b\ua10c\ua10d\ua10e\ua10f\ua110\ua111\ua112\ua113\ua114\ua115\ua116\ua117\ua118\ua119\ua11a\ua11b\ua11c\ua11d\ua11e\ua11f\ua120\ua121\ua122\ua123\ua124\ua125\ua126\ua127\ua128\ua129\ua12a\ua12b\ua12c\ua12d\ua12e\ua12f\ua130\ua131\ua132\ua133\ua134\ua135\ua136\ua137\ua138\ua139\ua13a\ua13b\ua13c\ua13d\ua13e\ua13f\ua140\ua141\ua142\ua143\ua144\ua145\ua146\ua147\ua148\ua149\ua14a\ua14b\ua14c\ua14d\ua14e\ua14f\ua150\ua151\ua152\ua153\ua154\ua155\ua156\ua157\ua158\ua159\ua15a\ua15b\ua15c\ua15d\ua15e\ua15f\ua160\ua161\ua162\ua163\ua164\ua165\ua166\ua167\ua168\ua169\ua16a\ua16b\ua16c\ua16d\ua16e\ua16f\ua170\ua171\ua172\ua173\ua174\ua175\ua176\ua177\ua178\ua179\ua17a\ua17b\ua17c\ua17d\ua17e\ua17f\ua180\ua181\ua182\ua183\ua184\ua185\ua186\ua187\ua188\ua189\ua18a\ua18b\ua18c\ua18d\ua18e\ua18f\ua190\ua191\ua192\ua193\ua194\ua195\ua196\ua197\ua198\ua199\ua19a\ua19b\ua19c\ua19d\ua19e\ua19f\ua1a0\ua1a1\ua1a2\ua1a3\ua1a4\ua1a5\ua1a6\ua1a7\ua1a8\ua1a9\ua1aa\ua1ab\ua1ac\ua1ad\ua1ae\ua1af\ua1b0\ua1b1\ua1b2\ua1b3\ua1b4\ua1b5\ua1b6\ua1b7\ua1b8\ua1b9\ua1ba\ua1bb\ua1bc\ua1bd\ua1be\ua1bf\ua1c0\ua1c1\ua1c2\ua1c3\ua1c4\ua1c5\ua1c6\ua1c7\ua1c8\ua1c9\ua1ca\ua1cb\ua1cc\ua1cd\ua1ce\ua1cf\ua1d0\ua1d1\ua1d2\ua1d3\ua1d4\ua1d5\ua1d6\ua1d7\ua1d8\ua1d9\ua1da\ua1db\ua1dc\ua1dd\ua1de\ua1df\ua1e0\ua1e1\ua1e2\ua1e3\ua1e4\ua1e5\ua1e6\ua1e7\ua1e8\ua1e9\ua1ea\ua1eb\ua1ec\ua1ed\ua1ee\ua1ef\ua1f0\ua1f1\ua1f2\ua1f3\ua1f4\ua1f5\ua1f6\ua1f7\ua1f8\ua1f9\ua1fa\ua1fb\ua1fc\ua1fd\ua1fe\ua1ff\ua200\ua201\ua202\ua203\ua204\ua205\ua206\ua207\ua208\ua209\ua20a\ua20b\ua20c\ua20d\ua20e\ua20f\ua210\ua211\ua212\ua213\ua214\ua215\ua216\ua217\ua218\ua219\ua21a\ua21b\ua21c\ua21d\ua21e\ua21f\ua220\ua221\ua222\ua223\ua224\ua225\ua226\ua227\ua228\ua229\ua22a\ua22b\ua22c\ua22d\ua22e\ua22f\ua230\ua231\ua232\ua233\ua234\ua235\ua236\ua237\ua238\ua239\ua23a\ua23b\ua23c\ua23d\ua23e\ua23f\ua240\ua241\ua242\ua243\ua244\ua245\ua246\ua247\ua248\ua249\ua24a\ua24b\ua24c\ua24d\ua24e\ua24f\ua250\ua251\ua252\ua253\ua254\ua255\ua256\ua257\ua258\ua259\ua25a\ua25b\ua25c\ua25d\ua25e\ua25f\ua260\ua261\ua262\ua263\ua264\ua265\ua266\ua267\ua268\ua269\ua26a\ua26b\ua26c\ua26d\ua26e\ua26f\ua270\ua271\ua272\ua273\ua274\ua275\ua276\ua277\ua278\ua279\ua27a\ua27b\ua27c\ua27d\ua27e\ua27f\ua280\ua281\ua282\ua283\ua284\ua285\ua286\ua287\ua288\ua289\ua28a\ua28b\ua28c\ua28d\ua28e\ua28f\ua290\ua291\ua292\ua293\ua294\ua295\ua296\ua297\ua298\ua299\ua29a\ua29b\ua29c\ua29d\ua29e\ua29f\ua2a0\ua2a1\ua2a2\ua2a3\ua2a4\ua2a5\ua2a6\ua2a7\ua2a8\ua2a9\ua2aa\ua2ab\ua2ac\ua2ad\ua2ae\ua2af\ua2b0\ua2b1\ua2b2\ua2b3\ua2b4\ua2b5\ua2b6\ua2b7\ua2b8\ua2b9\ua2ba\ua2bb\ua2bc\ua2bd\ua2be\ua2bf\ua2c0\ua2c1\ua2c2\ua2c3\ua2c4\ua2c5\ua2c6\ua2c7\ua2c8\ua2c9\ua2ca\ua2cb\ua2cc\ua2cd\ua2ce\ua2cf\ua2d0\ua2d1\ua2d2\ua2d3\ua2d4\ua2d5\ua2d6\ua2d7\ua2d8\ua2d9\ua2da\ua2db\ua2dc\ua2dd\ua2de\ua2df\ua2e0\ua2e1\ua2e2\ua2e3\ua2e4\ua2e5\ua2e6\ua2e7\ua2e8\ua2e9\ua2ea\ua2eb\ua2ec\ua2ed\ua2ee\ua2ef\ua2f0\ua2f1\ua2f2\ua2f3\ua2f4\ua2f5\ua2f6\ua2f7\ua2f8\ua2f9\ua2fa\ua2fb\ua2fc\ua2fd\ua2fe\ua2ff\ua300\ua301\ua302\ua303\ua304\ua305\ua306\ua307\ua308\ua309\ua30a\ua30b\ua30c\ua30d\ua30e\ua30f\ua310\ua311\ua312\ua313\ua314\ua315\ua316\ua317\ua318\ua319\ua31a\ua31b\ua31c\ua31d\ua31e\ua31f\ua320\ua321\ua322\ua323\ua324\ua325\ua326\ua327\ua328\ua329\ua32a\ua32b\ua32c\ua32d\ua32e\ua32f\ua330\ua331\ua332\ua333\ua334\ua335\ua336\ua337\ua338\ua339\ua33a\ua33b\ua33c\ua33d\ua33e\ua33f\ua340\ua341\ua342\ua343\ua344\ua345\ua346\ua347\ua348\ua349\ua34a\ua34b\ua34c\ua34d\ua34e\ua34f\ua350\ua351\ua352\ua353\ua354\ua355\ua356\ua357\ua358\ua359\ua35a\ua35b\ua35c\ua35d\ua35e\ua35f\ua360\ua361\ua362\ua363\ua364\ua365\ua366\ua367\ua368\ua369\ua36a\ua36b\ua36c\ua36d\ua36e\ua36f\ua370\ua371\ua372\ua373\ua374\ua375\ua376\ua377\ua378\ua379\ua37a\ua37b\ua37c\ua37d\ua37e\ua37f\ua380\ua381\ua382\ua383\ua384\ua385\ua386\ua387\ua388\ua389\ua38a\ua38b\ua38c\ua38d\ua38e\ua38f\ua390\ua391\ua392\ua393\ua394\ua395\ua396\ua397\ua398\ua399\ua39a\ua39b\ua39c\ua39d\ua39e\ua39f\ua3a0\ua3a1\ua3a2\ua3a3\ua3a4\ua3a5\ua3a6\ua3a7\ua3a8\ua3a9\ua3aa\ua3ab\ua3ac\ua3ad\ua3ae\ua3af\ua3b0\ua3b1\ua3b2\ua3b3\ua3b4\ua3b5\ua3b6\ua3b7\ua3b8\ua3b9\ua3ba\ua3bb\ua3bc\ua3bd\ua3be\ua3bf\ua3c0\ua3c1\ua3c2\ua3c3\ua3c4\ua3c5\ua3c6\ua3c7\ua3c8\ua3c9\ua3ca\ua3cb\ua3cc\ua3cd\ua3ce\ua3cf\ua3d0\ua3d1\ua3d2\ua3d3\ua3d4\ua3d5\ua3d6\ua3d7\ua3d8\ua3d9\ua3da\ua3db\ua3dc\ua3dd\ua3de\ua3df\ua3e0\ua3e1\ua3e2\ua3e3\ua3e4\ua3e5\ua3e6\ua3e7\ua3e8\ua3e9\ua3ea\ua3eb\ua3ec\ua3ed\ua3ee\ua3ef\ua3f0\ua3f1\ua3f2\ua3f3\ua3f4\ua3f5\ua3f6\ua3f7\ua3f8\ua3f9\ua3fa\ua3fb\ua3fc\ua3fd\ua3fe\ua3ff\ua400\ua401\ua402\ua403\ua404\ua405\ua406\ua407\ua408\ua409\ua40a\ua40b\ua40c\ua40d\ua40e\ua40f\ua410\ua411\ua412\ua413\ua414\ua415\ua416\ua417\ua418\ua419\ua41a\ua41b\ua41c\ua41d\ua41e\ua41f\ua420\ua421\ua422\ua423\ua424\ua425\ua426\ua427\ua428\ua429\ua42a\ua42b\ua42c\ua42d\ua42e\ua42f\ua430\ua431\ua432\ua433\ua434\ua435\ua436\ua437\ua438\ua439\ua43a\ua43b\ua43c\ua43d\ua43e\ua43f\ua440\ua441\ua442\ua443\ua444\ua445\ua446\ua447\ua448\ua449\ua44a\ua44b\ua44c\ua44d\ua44e\ua44f\ua450\ua451\ua452\ua453\ua454\ua455\ua456\ua457\ua458\ua459\ua45a\ua45b\ua45c\ua45d\ua45e\ua45f\ua460\ua461\ua462\ua463\ua464\ua465\ua466\ua467\ua468\ua469\ua46a\ua46b\ua46c\ua46d\ua46e\ua46f\ua470\ua471\ua472\ua473\ua474\ua475\ua476\ua477\ua478\ua479\ua47a\ua47b\ua47c\ua47d\ua47e\ua47f\ua480\ua481\ua482\ua483\ua484\ua485\ua486\ua487\ua488\ua489\ua48a\ua48b\ua48c\ua800\ua801\ua803\ua804\ua805\ua807\ua808\ua809\ua80a\ua80c\ua80d\ua80e\ua80f\ua810\ua811\ua812\ua813\ua814\ua815\ua816\ua817\ua818\ua819\ua81a\ua81b\ua81c\ua81d\ua81e\ua81f\ua820\ua821\ua822\uac00\uac01\uac02\uac03\uac04\uac05\uac06\uac07\uac08\uac09\uac0a\uac0b\uac0c\uac0d\uac0e\uac0f\uac10\uac11\uac12\uac13\uac14\uac15\uac16\uac17\uac18\uac19\uac1a\uac1b\uac1c\uac1d\uac1e\uac1f\uac20\uac21\uac22\uac23\uac24\uac25\uac26\uac27\uac28\uac29\uac2a\uac2b\uac2c\uac2d\uac2e\uac2f\uac30\uac31\uac32\uac33\uac34\uac35\uac36\uac37\uac38\uac39\uac3a\uac3b\uac3c\uac3d\uac3e\uac3f\uac40\uac41\uac42\uac43\uac44\uac45\uac46\uac47\uac48\uac49\uac4a\uac4b\uac4c\uac4d\uac4e\uac4f\uac50\uac51\uac52\uac53\uac54\uac55\uac56\uac57\uac58\uac59\uac5a\uac5b\uac5c\uac5d\uac5e\uac5f\uac60\uac61\uac62\uac63\uac64\uac65\uac66\uac67\uac68\uac69\uac6a\uac6b\uac6c\uac6d\uac6e\uac6f\uac70\uac71\uac72\uac73\uac74\uac75\uac76\uac77\uac78\uac79\uac7a\uac7b\uac7c\uac7d\uac7e\uac7f\uac80\uac81\uac82\uac83\uac84\uac85\uac86\uac87\uac88\uac89\uac8a\uac8b\uac8c\uac8d\uac8e\uac8f\uac90\uac91\uac92\uac93\uac94\uac95\uac96\uac97\uac98\uac99\uac9a\uac9b\uac9c\uac9d\uac9e\uac9f\uaca0\uaca1\uaca2\uaca3\uaca4\uaca5\uaca6\uaca7\uaca8\uaca9\uacaa\uacab\uacac\uacad\uacae\uacaf\uacb0\uacb1\uacb2\uacb3\uacb4\uacb5\uacb6\uacb7\uacb8\uacb9\uacba\uacbb\uacbc\uacbd\uacbe\uacbf\uacc0\uacc1\uacc2\uacc3\uacc4\uacc5\uacc6\uacc7\uacc8\uacc9\uacca\uaccb\uaccc\uaccd\uacce\uaccf\uacd0\uacd1\uacd2\uacd3\uacd4\uacd5\uacd6\uacd7\uacd8\uacd9\uacda\uacdb\uacdc\uacdd\uacde\uacdf\uace0\uace1\uace2\uace3\uace4\uace5\uace6\uace7\uace8\uace9\uacea\uaceb\uacec\uaced\uacee\uacef\uacf0\uacf1\uacf2\uacf3\uacf4\uacf5\uacf6\uacf7\uacf8\uacf9\uacfa\uacfb\uacfc\uacfd\uacfe\uacff\uad00\uad01\uad02\uad03\uad04\uad05\uad06\uad07\uad08\uad09\uad0a\uad0b\uad0c\uad0d\uad0e\uad0f\uad10\uad11\uad12\uad13\uad14\uad15\uad16\uad17\uad18\uad19\uad1a\uad1b\uad1c\uad1d\uad1e\uad1f\uad20\uad21\uad22\uad23\uad24\uad25\uad26\uad27\uad28\uad29\uad2a\uad2b\uad2c\uad2d\uad2e\uad2f\uad30\uad31\uad32\uad33\uad34\uad35\uad36\uad37\uad38\uad39\uad3a\uad3b\uad3c\uad3d\uad3e\uad3f\uad40\uad41\uad42\uad43\uad44\uad45\uad46\uad47\uad48\uad49\uad4a\uad4b\uad4c\uad4d\uad4e\uad4f\uad50\uad51\uad52\uad53\uad54\uad55\uad56\uad57\uad58\uad59\uad5a\uad5b\uad5c\uad5d\uad5e\uad5f\uad60\uad61\uad62\uad63\uad64\uad65\uad66\uad67\uad68\uad69\uad6a\uad6b\uad6c\uad6d\uad6e\uad6f\uad70\uad71\uad72\uad73\uad74\uad75\uad76\uad77\uad78\uad79\uad7a\uad7b\uad7c\uad7d\uad7e\uad7f\uad80\uad81\uad82\uad83\uad84\uad85\uad86\uad87\uad88\uad89\uad8a\uad8b\uad8c\uad8d\uad8e\uad8f\uad90\uad91\uad92\uad93\uad94\uad95\uad96\uad97\uad98\uad99\uad9a\uad9b\uad9c\uad9d\uad9e\uad9f\uada0\uada1\uada2\uada3\uada4\uada5\uada6\uada7\uada8\uada9\uadaa\uadab\uadac\uadad\uadae\uadaf\uadb0\uadb1\uadb2\uadb3\uadb4\uadb5\uadb6\uadb7\uadb8\uadb9\uadba\uadbb\uadbc\uadbd\uadbe\uadbf\uadc0\uadc1\uadc2\uadc3\uadc4\uadc5\uadc6\uadc7\uadc8\uadc9\uadca\uadcb\uadcc\uadcd\uadce\uadcf\uadd0\uadd1\uadd2\uadd3\uadd4\uadd5\uadd6\uadd7\uadd8\uadd9\uadda\uaddb\uaddc\uaddd\uadde\uaddf\uade0\uade1\uade2\uade3\uade4\uade5\uade6\uade7\uade8\uade9\uadea\uadeb\uadec\uaded\uadee\uadef\uadf0\uadf1\uadf2\uadf3\uadf4\uadf5\uadf6\uadf7\uadf8\uadf9\uadfa\uadfb\uadfc\uadfd\uadfe\uadff\uae00\uae01\uae02\uae03\uae04\uae05\uae06\uae07\uae08\uae09\uae0a\uae0b\uae0c\uae0d\uae0e\uae0f\uae10\uae11\uae12\uae13\uae14\uae15\uae16\uae17\uae18\uae19\uae1a\uae1b\uae1c\uae1d\uae1e\uae1f\uae20\uae21\uae22\uae23\uae24\uae25\uae26\uae27\uae28\uae29\uae2a\uae2b\uae2c\uae2d\uae2e\uae2f\uae30\uae31\uae32\uae33\uae34\uae35\uae36\uae37\uae38\uae39\uae3a\uae3b\uae3c\uae3d\uae3e\uae3f\uae40\uae41\uae42\uae43\uae44\uae45\uae46\uae47\uae48\uae49\uae4a\uae4b\uae4c\uae4d\uae4e\uae4f\uae50\uae51\uae52\uae53\uae54\uae55\uae56\uae57\uae58\uae59\uae5a\uae5b\uae5c\uae5d\uae5e\uae5f\uae60\uae61\uae62\uae63\uae64\uae65\uae66\uae67\uae68\uae69\uae6a\uae6b\uae6c\uae6d\uae6e\uae6f\uae70\uae71\uae72\uae73\uae74\uae75\uae76\uae77\uae78\uae79\uae7a\uae7b\uae7c\uae7d\uae7e\uae7f\uae80\uae81\uae82\uae83\uae84\uae85\uae86\uae87\uae88\uae89\uae8a\uae8b\uae8c\uae8d\uae8e\uae8f\uae90\uae91\uae92\uae93\uae94\uae95\uae96\uae97\uae98\uae99\uae9a\uae9b\uae9c\uae9d\uae9e\uae9f\uaea0\uaea1\uaea2\uaea3\uaea4\uaea5\uaea6\uaea7\uaea8\uaea9\uaeaa\uaeab\uaeac\uaead\uaeae\uaeaf\uaeb0\uaeb1\uaeb2\uaeb3\uaeb4\uaeb5\uaeb6\uaeb7\uaeb8\uaeb9\uaeba\uaebb\uaebc\uaebd\uaebe\uaebf\uaec0\uaec1\uaec2\uaec3\uaec4\uaec5\uaec6\uaec7\uaec8\uaec9\uaeca\uaecb\uaecc\uaecd\uaece\uaecf\uaed0\uaed1\uaed2\uaed3\uaed4\uaed5\uaed6\uaed7\uaed8\uaed9\uaeda\uaedb\uaedc\uaedd\uaede\uaedf\uaee0\uaee1\uaee2\uaee3\uaee4\uaee5\uaee6\uaee7\uaee8\uaee9\uaeea\uaeeb\uaeec\uaeed\uaeee\uaeef\uaef0\uaef1\uaef2\uaef3\uaef4\uaef5\uaef6\uaef7\uaef8\uaef9\uaefa\uaefb\uaefc\uaefd\uaefe\uaeff\uaf00\uaf01\uaf02\uaf03\uaf04\uaf05\uaf06\uaf07\uaf08\uaf09\uaf0a\uaf0b\uaf0c\uaf0d\uaf0e\uaf0f\uaf10\uaf11\uaf12\uaf13\uaf14\uaf15\uaf16\uaf17\uaf18\uaf19\uaf1a\uaf1b\uaf1c\uaf1d\uaf1e\uaf1f\uaf20\uaf21\uaf22\uaf23\uaf24\uaf25\uaf26\uaf27\uaf28\uaf29\uaf2a\uaf2b\uaf2c\uaf2d\uaf2e\uaf2f\uaf30\uaf31\uaf32\uaf33\uaf34\uaf35\uaf36\uaf37\uaf38\uaf39\uaf3a\uaf3b\uaf3c\uaf3d\uaf3e\uaf3f\uaf40\uaf41\uaf42\uaf43\uaf44\uaf45\uaf46\uaf47\uaf48\uaf49\uaf4a\uaf4b\uaf4c\uaf4d\uaf4e\uaf4f\uaf50\uaf51\uaf52\uaf53\uaf54\uaf55\uaf56\uaf57\uaf58\uaf59\uaf5a\uaf5b\uaf5c\uaf5d\uaf5e\uaf5f\uaf60\uaf61\uaf62\uaf63\uaf64\uaf65\uaf66\uaf67\uaf68\uaf69\uaf6a\uaf6b\uaf6c\uaf6d\uaf6e\uaf6f\uaf70\uaf71\uaf72\uaf73\uaf74\uaf75\uaf76\uaf77\uaf78\uaf79\uaf7a\uaf7b\uaf7c\uaf7d\uaf7e\uaf7f\uaf80\uaf81\uaf82\uaf83\uaf84\uaf85\uaf86\uaf87\uaf88\uaf89\uaf8a\uaf8b\uaf8c\uaf8d\uaf8e\uaf8f\uaf90\uaf91\uaf92\uaf93\uaf94\uaf95\uaf96\uaf97\uaf98\uaf99\uaf9a\uaf9b\uaf9c\uaf9d\uaf9e\uaf9f\uafa0\uafa1\uafa2\uafa3\uafa4\uafa5\uafa6\uafa7\uafa8\uafa9\uafaa\uafab\uafac\uafad\uafae\uafaf\uafb0\uafb1\uafb2\uafb3\uafb4\uafb5\uafb6\uafb7\uafb8\uafb9\uafba\uafbb\uafbc\uafbd\uafbe\uafbf\uafc0\uafc1\uafc2\uafc3\uafc4\uafc5\uafc6\uafc7\uafc8\uafc9\uafca\uafcb\uafcc\uafcd\uafce\uafcf\uafd0\uafd1\uafd2\uafd3\uafd4\uafd5\uafd6\uafd7\uafd8\uafd9\uafda\uafdb\uafdc\uafdd\uafde\uafdf\uafe0\uafe1\uafe2\uafe3\uafe4\uafe5\uafe6\uafe7\uafe8\uafe9\uafea\uafeb\uafec\uafed\uafee\uafef\uaff0\uaff1\uaff2\uaff3\uaff4\uaff5\uaff6\uaff7\uaff8\uaff9\uaffa\uaffb\uaffc\uaffd\uaffe\uafff\ub000\ub001\ub002\ub003\ub004\ub005\ub006\ub007\ub008\ub009\ub00a\ub00b\ub00c\ub00d\ub00e\ub00f\ub010\ub011\ub012\ub013\ub014\ub015\ub016\ub017\ub018\ub019\ub01a\ub01b\ub01c\ub01d\ub01e\ub01f\ub020\ub021\ub022\ub023\ub024\ub025\ub026\ub027\ub028\ub029\ub02a\ub02b\ub02c\ub02d\ub02e\ub02f\ub030\ub031\ub032\ub033\ub034\ub035\ub036\ub037\ub038\ub039\ub03a\ub03b\ub03c\ub03d\ub03e\ub03f\ub040\ub041\ub042\ub043\ub044\ub045\ub046\ub047\ub048\ub049\ub04a\ub04b\ub04c\ub04d\ub04e\ub04f\ub050\ub051\ub052\ub053\ub054\ub055\ub056\ub057\ub058\ub059\ub05a\ub05b\ub05c\ub05d\ub05e\ub05f\ub060\ub061\ub062\ub063\ub064\ub065\ub066\ub067\ub068\ub069\ub06a\ub06b\ub06c\ub06d\ub06e\ub06f\ub070\ub071\ub072\ub073\ub074\ub075\ub076\ub077\ub078\ub079\ub07a\ub07b\ub07c\ub07d\ub07e\ub07f\ub080\ub081\ub082\ub083\ub084\ub085\ub086\ub087\ub088\ub089\ub08a\ub08b\ub08c\ub08d\ub08e\ub08f\ub090\ub091\ub092\ub093\ub094\ub095\ub096\ub097\ub098\ub099\ub09a\ub09b\ub09c\ub09d\ub09e\ub09f\ub0a0\ub0a1\ub0a2\ub0a3\ub0a4\ub0a5\ub0a6\ub0a7\ub0a8\ub0a9\ub0aa\ub0ab\ub0ac\ub0ad\ub0ae\ub0af\ub0b0\ub0b1\ub0b2\ub0b3\ub0b4\ub0b5\ub0b6\ub0b7\ub0b8\ub0b9\ub0ba\ub0bb\ub0bc\ub0bd\ub0be\ub0bf\ub0c0\ub0c1\ub0c2\ub0c3\ub0c4\ub0c5\ub0c6\ub0c7\ub0c8\ub0c9\ub0ca\ub0cb\ub0cc\ub0cd\ub0ce\ub0cf\ub0d0\ub0d1\ub0d2\ub0d3\ub0d4\ub0d5\ub0d6\ub0d7\ub0d8\ub0d9\ub0da\ub0db\ub0dc\ub0dd\ub0de\ub0df\ub0e0\ub0e1\ub0e2\ub0e3\ub0e4\ub0e5\ub0e6\ub0e7\ub0e8\ub0e9\ub0ea\ub0eb\ub0ec\ub0ed\ub0ee\ub0ef\ub0f0\ub0f1\ub0f2\ub0f3\ub0f4\ub0f5\ub0f6\ub0f7\ub0f8\ub0f9\ub0fa\ub0fb\ub0fc\ub0fd\ub0fe\ub0ff\ub100\ub101\ub102\ub103\ub104\ub105\ub106\ub107\ub108\ub109\ub10a\ub10b\ub10c\ub10d\ub10e\ub10f\ub110\ub111\ub112\ub113\ub114\ub115\ub116\ub117\ub118\ub119\ub11a\ub11b\ub11c\ub11d\ub11e\ub11f\ub120\ub121\ub122\ub123\ub124\ub125\ub126\ub127\ub128\ub129\ub12a\ub12b\ub12c\ub12d\ub12e\ub12f\ub130\ub131\ub132\ub133\ub134\ub135\ub136\ub137\ub138\ub139\ub13a\ub13b\ub13c\ub13d\ub13e\ub13f\ub140\ub141\ub142\ub143\ub144\ub145\ub146\ub147\ub148\ub149\ub14a\ub14b\ub14c\ub14d\ub14e\ub14f\ub150\ub151\ub152\ub153\ub154\ub155\ub156\ub157\ub158\ub159\ub15a\ub15b\ub15c\ub15d\ub15e\ub15f\ub160\ub161\ub162\ub163\ub164\ub165\ub166\ub167\ub168\ub169\ub16a\ub16b\ub16c\ub16d\ub16e\ub16f\ub170\ub171\ub172\ub173\ub174\ub175\ub176\ub177\ub178\ub179\ub17a\ub17b\ub17c\ub17d\ub17e\ub17f\ub180\ub181\ub182\ub183\ub184\ub185\ub186\ub187\ub188\ub189\ub18a\ub18b\ub18c\ub18d\ub18e\ub18f\ub190\ub191\ub192\ub193\ub194\ub195\ub196\ub197\ub198\ub199\ub19a\ub19b\ub19c\ub19d\ub19e\ub19f\ub1a0\ub1a1\ub1a2\ub1a3\ub1a4\ub1a5\ub1a6\ub1a7\ub1a8\ub1a9\ub1aa\ub1ab\ub1ac\ub1ad\ub1ae\ub1af\ub1b0\ub1b1\ub1b2\ub1b3\ub1b4\ub1b5\ub1b6\ub1b7\ub1b8\ub1b9\ub1ba\ub1bb\ub1bc\ub1bd\ub1be\ub1bf\ub1c0\ub1c1\ub1c2\ub1c3\ub1c4\ub1c5\ub1c6\ub1c7\ub1c8\ub1c9\ub1ca\ub1cb\ub1cc\ub1cd\ub1ce\ub1cf\ub1d0\ub1d1\ub1d2\ub1d3\ub1d4\ub1d5\ub1d6\ub1d7\ub1d8\ub1d9\ub1da\ub1db\ub1dc\ub1dd\ub1de\ub1df\ub1e0\ub1e1\ub1e2\ub1e3\ub1e4\ub1e5\ub1e6\ub1e7\ub1e8\ub1e9\ub1ea\ub1eb\ub1ec\ub1ed\ub1ee\ub1ef\ub1f0\ub1f1\ub1f2\ub1f3\ub1f4\ub1f5\ub1f6\ub1f7\ub1f8\ub1f9\ub1fa\ub1fb\ub1fc\ub1fd\ub1fe\ub1ff\ub200\ub201\ub202\ub203\ub204\ub205\ub206\ub207\ub208\ub209\ub20a\ub20b\ub20c\ub20d\ub20e\ub20f\ub210\ub211\ub212\ub213\ub214\ub215\ub216\ub217\ub218\ub219\ub21a\ub21b\ub21c\ub21d\ub21e\ub21f\ub220\ub221\ub222\ub223\ub224\ub225\ub226\ub227\ub228\ub229\ub22a\ub22b\ub22c\ub22d\ub22e\ub22f\ub230\ub231\ub232\ub233\ub234\ub235\ub236\ub237\ub238\ub239\ub23a\ub23b\ub23c\ub23d\ub23e\ub23f\ub240\ub241\ub242\ub243\ub244\ub245\ub246\ub247\ub248\ub249\ub24a\ub24b\ub24c\ub24d\ub24e\ub24f\ub250\ub251\ub252\ub253\ub254\ub255\ub256\ub257\ub258\ub259\ub25a\ub25b\ub25c\ub25d\ub25e\ub25f\ub260\ub261\ub262\ub263\ub264\ub265\ub266\ub267\ub268\ub269\ub26a\ub26b\ub26c\ub26d\ub26e\ub26f\ub270\ub271\ub272\ub273\ub274\ub275\ub276\ub277\ub278\ub279\ub27a\ub27b\ub27c\ub27d\ub27e\ub27f\ub280\ub281\ub282\ub283\ub284\ub285\ub286\ub287\ub288\ub289\ub28a\ub28b\ub28c\ub28d\ub28e\ub28f\ub290\ub291\ub292\ub293\ub294\ub295\ub296\ub297\ub298\ub299\ub29a\ub29b\ub29c\ub29d\ub29e\ub29f\ub2a0\ub2a1\ub2a2\ub2a3\ub2a4\ub2a5\ub2a6\ub2a7\ub2a8\ub2a9\ub2aa\ub2ab\ub2ac\ub2ad\ub2ae\ub2af\ub2b0\ub2b1\ub2b2\ub2b3\ub2b4\ub2b5\ub2b6\ub2b7\ub2b8\ub2b9\ub2ba\ub2bb\ub2bc\ub2bd\ub2be\ub2bf\ub2c0\ub2c1\ub2c2\ub2c3\ub2c4\ub2c5\ub2c6\ub2c7\ub2c8\ub2c9\ub2ca\ub2cb\ub2cc\ub2cd\ub2ce\ub2cf\ub2d0\ub2d1\ub2d2\ub2d3\ub2d4\ub2d5\ub2d6\ub2d7\ub2d8\ub2d9\ub2da\ub2db\ub2dc\ub2dd\ub2de\ub2df\ub2e0\ub2e1\ub2e2\ub2e3\ub2e4\ub2e5\ub2e6\ub2e7\ub2e8\ub2e9\ub2ea\ub2eb\ub2ec\ub2ed\ub2ee\ub2ef\ub2f0\ub2f1\ub2f2\ub2f3\ub2f4\ub2f5\ub2f6\ub2f7\ub2f8\ub2f9\ub2fa\ub2fb\ub2fc\ub2fd\ub2fe\ub2ff\ub300\ub301\ub302\ub303\ub304\ub305\ub306\ub307\ub308\ub309\ub30a\ub30b\ub30c\ub30d\ub30e\ub30f\ub310\ub311\ub312\ub313\ub314\ub315\ub316\ub317\ub318\ub319\ub31a\ub31b\ub31c\ub31d\ub31e\ub31f\ub320\ub321\ub322\ub323\ub324\ub325\ub326\ub327\ub328\ub329\ub32a\ub32b\ub32c\ub32d\ub32e\ub32f\ub330\ub331\ub332\ub333\ub334\ub335\ub336\ub337\ub338\ub339\ub33a\ub33b\ub33c\ub33d\ub33e\ub33f\ub340\ub341\ub342\ub343\ub344\ub345\ub346\ub347\ub348\ub349\ub34a\ub34b\ub34c\ub34d\ub34e\ub34f\ub350\ub351\ub352\ub353\ub354\ub355\ub356\ub357\ub358\ub359\ub35a\ub35b\ub35c\ub35d\ub35e\ub35f\ub360\ub361\ub362\ub363\ub364\ub365\ub366\ub367\ub368\ub369\ub36a\ub36b\ub36c\ub36d\ub36e\ub36f\ub370\ub371\ub372\ub373\ub374\ub375\ub376\ub377\ub378\ub379\ub37a\ub37b\ub37c\ub37d\ub37e\ub37f\ub380\ub381\ub382\ub383\ub384\ub385\ub386\ub387\ub388\ub389\ub38a\ub38b\ub38c\ub38d\ub38e\ub38f\ub390\ub391\ub392\ub393\ub394\ub395\ub396\ub397\ub398\ub399\ub39a\ub39b\ub39c\ub39d\ub39e\ub39f\ub3a0\ub3a1\ub3a2\ub3a3\ub3a4\ub3a5\ub3a6\ub3a7\ub3a8\ub3a9\ub3aa\ub3ab\ub3ac\ub3ad\ub3ae\ub3af\ub3b0\ub3b1\ub3b2\ub3b3\ub3b4\ub3b5\ub3b6\ub3b7\ub3b8\ub3b9\ub3ba\ub3bb\ub3bc\ub3bd\ub3be\ub3bf\ub3c0\ub3c1\ub3c2\ub3c3\ub3c4\ub3c5\ub3c6\ub3c7\ub3c8\ub3c9\ub3ca\ub3cb\ub3cc\ub3cd\ub3ce\ub3cf\ub3d0\ub3d1\ub3d2\ub3d3\ub3d4\ub3d5\ub3d6\ub3d7\ub3d8\ub3d9\ub3da\ub3db\ub3dc\ub3dd\ub3de\ub3df\ub3e0\ub3e1\ub3e2\ub3e3\ub3e4\ub3e5\ub3e6\ub3e7\ub3e8\ub3e9\ub3ea\ub3eb\ub3ec\ub3ed\ub3ee\ub3ef\ub3f0\ub3f1\ub3f2\ub3f3\ub3f4\ub3f5\ub3f6\ub3f7\ub3f8\ub3f9\ub3fa\ub3fb\ub3fc\ub3fd\ub3fe\ub3ff\ub400\ub401\ub402\ub403\ub404\ub405\ub406\ub407\ub408\ub409\ub40a\ub40b\ub40c\ub40d\ub40e\ub40f\ub410\ub411\ub412\ub413\ub414\ub415\ub416\ub417\ub418\ub419\ub41a\ub41b\ub41c\ub41d\ub41e\ub41f\ub420\ub421\ub422\ub423\ub424\ub425\ub426\ub427\ub428\ub429\ub42a\ub42b\ub42c\ub42d\ub42e\ub42f\ub430\ub431\ub432\ub433\ub434\ub435\ub436\ub437\ub438\ub439\ub43a\ub43b\ub43c\ub43d\ub43e\ub43f\ub440\ub441\ub442\ub443\ub444\ub445\ub446\ub447\ub448\ub449\ub44a\ub44b\ub44c\ub44d\ub44e\ub44f\ub450\ub451\ub452\ub453\ub454\ub455\ub456\ub457\ub458\ub459\ub45a\ub45b\ub45c\ub45d\ub45e\ub45f\ub460\ub461\ub462\ub463\ub464\ub465\ub466\ub467\ub468\ub469\ub46a\ub46b\ub46c\ub46d\ub46e\ub46f\ub470\ub471\ub472\ub473\ub474\ub475\ub476\ub477\ub478\ub479\ub47a\ub47b\ub47c\ub47d\ub47e\ub47f\ub480\ub481\ub482\ub483\ub484\ub485\ub486\ub487\ub488\ub489\ub48a\ub48b\ub48c\ub48d\ub48e\ub48f\ub490\ub491\ub492\ub493\ub494\ub495\ub496\ub497\ub498\ub499\ub49a\ub49b\ub49c\ub49d\ub49e\ub49f\ub4a0\ub4a1\ub4a2\ub4a3\ub4a4\ub4a5\ub4a6\ub4a7\ub4a8\ub4a9\ub4aa\ub4ab\ub4ac\ub4ad\ub4ae\ub4af\ub4b0\ub4b1\ub4b2\ub4b3\ub4b4\ub4b5\ub4b6\ub4b7\ub4b8\ub4b9\ub4ba\ub4bb\ub4bc\ub4bd\ub4be\ub4bf\ub4c0\ub4c1\ub4c2\ub4c3\ub4c4\ub4c5\ub4c6\ub4c7\ub4c8\ub4c9\ub4ca\ub4cb\ub4cc\ub4cd\ub4ce\ub4cf\ub4d0\ub4d1\ub4d2\ub4d3\ub4d4\ub4d5\ub4d6\ub4d7\ub4d8\ub4d9\ub4da\ub4db\ub4dc\ub4dd\ub4de\ub4df\ub4e0\ub4e1\ub4e2\ub4e3\ub4e4\ub4e5\ub4e6\ub4e7\ub4e8\ub4e9\ub4ea\ub4eb\ub4ec\ub4ed\ub4ee\ub4ef\ub4f0\ub4f1\ub4f2\ub4f3\ub4f4\ub4f5\ub4f6\ub4f7\ub4f8\ub4f9\ub4fa\ub4fb\ub4fc\ub4fd\ub4fe\ub4ff\ub500\ub501\ub502\ub503\ub504\ub505\ub506\ub507\ub508\ub509\ub50a\ub50b\ub50c\ub50d\ub50e\ub50f\ub510\ub511\ub512\ub513\ub514\ub515\ub516\ub517\ub518\ub519\ub51a\ub51b\ub51c\ub51d\ub51e\ub51f\ub520\ub521\ub522\ub523\ub524\ub525\ub526\ub527\ub528\ub529\ub52a\ub52b\ub52c\ub52d\ub52e\ub52f\ub530\ub531\ub532\ub533\ub534\ub535\ub536\ub537\ub538\ub539\ub53a\ub53b\ub53c\ub53d\ub53e\ub53f\ub540\ub541\ub542\ub543\ub544\ub545\ub546\ub547\ub548\ub549\ub54a\ub54b\ub54c\ub54d\ub54e\ub54f\ub550\ub551\ub552\ub553\ub554\ub555\ub556\ub557\ub558\ub559\ub55a\ub55b\ub55c\ub55d\ub55e\ub55f\ub560\ub561\ub562\ub563\ub564\ub565\ub566\ub567\ub568\ub569\ub56a\ub56b\ub56c\ub56d\ub56e\ub56f\ub570\ub571\ub572\ub573\ub574\ub575\ub576\ub577\ub578\ub579\ub57a\ub57b\ub57c\ub57d\ub57e\ub57f\ub580\ub581\ub582\ub583\ub584\ub585\ub586\ub587\ub588\ub589\ub58a\ub58b\ub58c\ub58d\ub58e\ub58f\ub590\ub591\ub592\ub593\ub594\ub595\ub596\ub597\ub598\ub599\ub59a\ub59b\ub59c\ub59d\ub59e\ub59f\ub5a0\ub5a1\ub5a2\ub5a3\ub5a4\ub5a5\ub5a6\ub5a7\ub5a8\ub5a9\ub5aa\ub5ab\ub5ac\ub5ad\ub5ae\ub5af\ub5b0\ub5b1\ub5b2\ub5b3\ub5b4\ub5b5\ub5b6\ub5b7\ub5b8\ub5b9\ub5ba\ub5bb\ub5bc\ub5bd\ub5be\ub5bf\ub5c0\ub5c1\ub5c2\ub5c3\ub5c4\ub5c5\ub5c6\ub5c7\ub5c8\ub5c9\ub5ca\ub5cb\ub5cc\ub5cd\ub5ce\ub5cf\ub5d0\ub5d1\ub5d2\ub5d3\ub5d4\ub5d5\ub5d6\ub5d7\ub5d8\ub5d9\ub5da\ub5db\ub5dc\ub5dd\ub5de\ub5df\ub5e0\ub5e1\ub5e2\ub5e3\ub5e4\ub5e5\ub5e6\ub5e7\ub5e8\ub5e9\ub5ea\ub5eb\ub5ec\ub5ed\ub5ee\ub5ef\ub5f0\ub5f1\ub5f2\ub5f3\ub5f4\ub5f5\ub5f6\ub5f7\ub5f8\ub5f9\ub5fa\ub5fb\ub5fc\ub5fd\ub5fe\ub5ff\ub600\ub601\ub602\ub603\ub604\ub605\ub606\ub607\ub608\ub609\ub60a\ub60b\ub60c\ub60d\ub60e\ub60f\ub610\ub611\ub612\ub613\ub614\ub615\ub616\ub617\ub618\ub619\ub61a\ub61b\ub61c\ub61d\ub61e\ub61f\ub620\ub621\ub622\ub623\ub624\ub625\ub626\ub627\ub628\ub629\ub62a\ub62b\ub62c\ub62d\ub62e\ub62f\ub630\ub631\ub632\ub633\ub634\ub635\ub636\ub637\ub638\ub639\ub63a\ub63b\ub63c\ub63d\ub63e\ub63f\ub640\ub641\ub642\ub643\ub644\ub645\ub646\ub647\ub648\ub649\ub64a\ub64b\ub64c\ub64d\ub64e\ub64f\ub650\ub651\ub652\ub653\ub654\ub655\ub656\ub657\ub658\ub659\ub65a\ub65b\ub65c\ub65d\ub65e\ub65f\ub660\ub661\ub662\ub663\ub664\ub665\ub666\ub667\ub668\ub669\ub66a\ub66b\ub66c\ub66d\ub66e\ub66f\ub670\ub671\ub672\ub673\ub674\ub675\ub676\ub677\ub678\ub679\ub67a\ub67b\ub67c\ub67d\ub67e\ub67f\ub680\ub681\ub682\ub683\ub684\ub685\ub686\ub687\ub688\ub689\ub68a\ub68b\ub68c\ub68d\ub68e\ub68f\ub690\ub691\ub692\ub693\ub694\ub695\ub696\ub697\ub698\ub699\ub69a\ub69b\ub69c\ub69d\ub69e\ub69f\ub6a0\ub6a1\ub6a2\ub6a3\ub6a4\ub6a5\ub6a6\ub6a7\ub6a8\ub6a9\ub6aa\ub6ab\ub6ac\ub6ad\ub6ae\ub6af\ub6b0\ub6b1\ub6b2\ub6b3\ub6b4\ub6b5\ub6b6\ub6b7\ub6b8\ub6b9\ub6ba\ub6bb\ub6bc\ub6bd\ub6be\ub6bf\ub6c0\ub6c1\ub6c2\ub6c3\ub6c4\ub6c5\ub6c6\ub6c7\ub6c8\ub6c9\ub6ca\ub6cb\ub6cc\ub6cd\ub6ce\ub6cf\ub6d0\ub6d1\ub6d2\ub6d3\ub6d4\ub6d5\ub6d6\ub6d7\ub6d8\ub6d9\ub6da\ub6db\ub6dc\ub6dd\ub6de\ub6df\ub6e0\ub6e1\ub6e2\ub6e3\ub6e4\ub6e5\ub6e6\ub6e7\ub6e8\ub6e9\ub6ea\ub6eb\ub6ec\ub6ed\ub6ee\ub6ef\ub6f0\ub6f1\ub6f2\ub6f3\ub6f4\ub6f5\ub6f6\ub6f7\ub6f8\ub6f9\ub6fa\ub6fb\ub6fc\ub6fd\ub6fe\ub6ff\ub700\ub701\ub702\ub703\ub704\ub705\ub706\ub707\ub708\ub709\ub70a\ub70b\ub70c\ub70d\ub70e\ub70f\ub710\ub711\ub712\ub713\ub714\ub715\ub716\ub717\ub718\ub719\ub71a\ub71b\ub71c\ub71d\ub71e\ub71f\ub720\ub721\ub722\ub723\ub724\ub725\ub726\ub727\ub728\ub729\ub72a\ub72b\ub72c\ub72d\ub72e\ub72f\ub730\ub731\ub732\ub733\ub734\ub735\ub736\ub737\ub738\ub739\ub73a\ub73b\ub73c\ub73d\ub73e\ub73f\ub740\ub741\ub742\ub743\ub744\ub745\ub746\ub747\ub748\ub749\ub74a\ub74b\ub74c\ub74d\ub74e\ub74f\ub750\ub751\ub752\ub753\ub754\ub755\ub756\ub757\ub758\ub759\ub75a\ub75b\ub75c\ub75d\ub75e\ub75f\ub760\ub761\ub762\ub763\ub764\ub765\ub766\ub767\ub768\ub769\ub76a\ub76b\ub76c\ub76d\ub76e\ub76f\ub770\ub771\ub772\ub773\ub774\ub775\ub776\ub777\ub778\ub779\ub77a\ub77b\ub77c\ub77d\ub77e\ub77f\ub780\ub781\ub782\ub783\ub784\ub785\ub786\ub787\ub788\ub789\ub78a\ub78b\ub78c\ub78d\ub78e\ub78f\ub790\ub791\ub792\ub793\ub794\ub795\ub796\ub797\ub798\ub799\ub79a\ub79b\ub79c\ub79d\ub79e\ub79f\ub7a0\ub7a1\ub7a2\ub7a3\ub7a4\ub7a5\ub7a6\ub7a7\ub7a8\ub7a9\ub7aa\ub7ab\ub7ac\ub7ad\ub7ae\ub7af\ub7b0\ub7b1\ub7b2\ub7b3\ub7b4\ub7b5\ub7b6\ub7b7\ub7b8\ub7b9\ub7ba\ub7bb\ub7bc\ub7bd\ub7be\ub7bf\ub7c0\ub7c1\ub7c2\ub7c3\ub7c4\ub7c5\ub7c6\ub7c7\ub7c8\ub7c9\ub7ca\ub7cb\ub7cc\ub7cd\ub7ce\ub7cf\ub7d0\ub7d1\ub7d2\ub7d3\ub7d4\ub7d5\ub7d6\ub7d7\ub7d8\ub7d9\ub7da\ub7db\ub7dc\ub7dd\ub7de\ub7df\ub7e0\ub7e1\ub7e2\ub7e3\ub7e4\ub7e5\ub7e6\ub7e7\ub7e8\ub7e9\ub7ea\ub7eb\ub7ec\ub7ed\ub7ee\ub7ef\ub7f0\ub7f1\ub7f2\ub7f3\ub7f4\ub7f5\ub7f6\ub7f7\ub7f8\ub7f9\ub7fa\ub7fb\ub7fc\ub7fd\ub7fe\ub7ff\ub800\ub801\ub802\ub803\ub804\ub805\ub806\ub807\ub808\ub809\ub80a\ub80b\ub80c\ub80d\ub80e\ub80f\ub810\ub811\ub812\ub813\ub814\ub815\ub816\ub817\ub818\ub819\ub81a\ub81b\ub81c\ub81d\ub81e\ub81f\ub820\ub821\ub822\ub823\ub824\ub825\ub826\ub827\ub828\ub829\ub82a\ub82b\ub82c\ub82d\ub82e\ub82f\ub830\ub831\ub832\ub833\ub834\ub835\ub836\ub837\ub838\ub839\ub83a\ub83b\ub83c\ub83d\ub83e\ub83f\ub840\ub841\ub842\ub843\ub844\ub845\ub846\ub847\ub848\ub849\ub84a\ub84b\ub84c\ub84d\ub84e\ub84f\ub850\ub851\ub852\ub853\ub854\ub855\ub856\ub857\ub858\ub859\ub85a\ub85b\ub85c\ub85d\ub85e\ub85f\ub860\ub861\ub862\ub863\ub864\ub865\ub866\ub867\ub868\ub869\ub86a\ub86b\ub86c\ub86d\ub86e\ub86f\ub870\ub871\ub872\ub873\ub874\ub875\ub876\ub877\ub878\ub879\ub87a\ub87b\ub87c\ub87d\ub87e\ub87f\ub880\ub881\ub882\ub883\ub884\ub885\ub886\ub887\ub888\ub889\ub88a\ub88b\ub88c\ub88d\ub88e\ub88f\ub890\ub891\ub892\ub893\ub894\ub895\ub896\ub897\ub898\ub899\ub89a\ub89b\ub89c\ub89d\ub89e\ub89f\ub8a0\ub8a1\ub8a2\ub8a3\ub8a4\ub8a5\ub8a6\ub8a7\ub8a8\ub8a9\ub8aa\ub8ab\ub8ac\ub8ad\ub8ae\ub8af\ub8b0\ub8b1\ub8b2\ub8b3\ub8b4\ub8b5\ub8b6\ub8b7\ub8b8\ub8b9\ub8ba\ub8bb\ub8bc\ub8bd\ub8be\ub8bf\ub8c0\ub8c1\ub8c2\ub8c3\ub8c4\ub8c5\ub8c6\ub8c7\ub8c8\ub8c9\ub8ca\ub8cb\ub8cc\ub8cd\ub8ce\ub8cf\ub8d0\ub8d1\ub8d2\ub8d3\ub8d4\ub8d5\ub8d6\ub8d7\ub8d8\ub8d9\ub8da\ub8db\ub8dc\ub8dd\ub8de\ub8df\ub8e0\ub8e1\ub8e2\ub8e3\ub8e4\ub8e5\ub8e6\ub8e7\ub8e8\ub8e9\ub8ea\ub8eb\ub8ec\ub8ed\ub8ee\ub8ef\ub8f0\ub8f1\ub8f2\ub8f3\ub8f4\ub8f5\ub8f6\ub8f7\ub8f8\ub8f9\ub8fa\ub8fb\ub8fc\ub8fd\ub8fe\ub8ff\ub900\ub901\ub902\ub903\ub904\ub905\ub906\ub907\ub908\ub909\ub90a\ub90b\ub90c\ub90d\ub90e\ub90f\ub910\ub911\ub912\ub913\ub914\ub915\ub916\ub917\ub918\ub919\ub91a\ub91b\ub91c\ub91d\ub91e\ub91f\ub920\ub921\ub922\ub923\ub924\ub925\ub926\ub927\ub928\ub929\ub92a\ub92b\ub92c\ub92d\ub92e\ub92f\ub930\ub931\ub932\ub933\ub934\ub935\ub936\ub937\ub938\ub939\ub93a\ub93b\ub93c\ub93d\ub93e\ub93f\ub940\ub941\ub942\ub943\ub944\ub945\ub946\ub947\ub948\ub949\ub94a\ub94b\ub94c\ub94d\ub94e\ub94f\ub950\ub951\ub952\ub953\ub954\ub955\ub956\ub957\ub958\ub959\ub95a\ub95b\ub95c\ub95d\ub95e\ub95f\ub960\ub961\ub962\ub963\ub964\ub965\ub966\ub967\ub968\ub969\ub96a\ub96b\ub96c\ub96d\ub96e\ub96f\ub970\ub971\ub972\ub973\ub974\ub975\ub976\ub977\ub978\ub979\ub97a\ub97b\ub97c\ub97d\ub97e\ub97f\ub980\ub981\ub982\ub983\ub984\ub985\ub986\ub987\ub988\ub989\ub98a\ub98b\ub98c\ub98d\ub98e\ub98f\ub990\ub991\ub992\ub993\ub994\ub995\ub996\ub997\ub998\ub999\ub99a\ub99b\ub99c\ub99d\ub99e\ub99f\ub9a0\ub9a1\ub9a2\ub9a3\ub9a4\ub9a5\ub9a6\ub9a7\ub9a8\ub9a9\ub9aa\ub9ab\ub9ac\ub9ad\ub9ae\ub9af\ub9b0\ub9b1\ub9b2\ub9b3\ub9b4\ub9b5\ub9b6\ub9b7\ub9b8\ub9b9\ub9ba\ub9bb\ub9bc\ub9bd\ub9be\ub9bf\ub9c0\ub9c1\ub9c2\ub9c3\ub9c4\ub9c5\ub9c6\ub9c7\ub9c8\ub9c9\ub9ca\ub9cb\ub9cc\ub9cd\ub9ce\ub9cf\ub9d0\ub9d1\ub9d2\ub9d3\ub9d4\ub9d5\ub9d6\ub9d7\ub9d8\ub9d9\ub9da\ub9db\ub9dc\ub9dd\ub9de\ub9df\ub9e0\ub9e1\ub9e2\ub9e3\ub9e4\ub9e5\ub9e6\ub9e7\ub9e8\ub9e9\ub9ea\ub9eb\ub9ec\ub9ed\ub9ee\ub9ef\ub9f0\ub9f1\ub9f2\ub9f3\ub9f4\ub9f5\ub9f6\ub9f7\ub9f8\ub9f9\ub9fa\ub9fb\ub9fc\ub9fd\ub9fe\ub9ff\uba00\uba01\uba02\uba03\uba04\uba05\uba06\uba07\uba08\uba09\uba0a\uba0b\uba0c\uba0d\uba0e\uba0f\uba10\uba11\uba12\uba13\uba14\uba15\uba16\uba17\uba18\uba19\uba1a\uba1b\uba1c\uba1d\uba1e\uba1f\uba20\uba21\uba22\uba23\uba24\uba25\uba26\uba27\uba28\uba29\uba2a\uba2b\uba2c\uba2d\uba2e\uba2f\uba30\uba31\uba32\uba33\uba34\uba35\uba36\uba37\uba38\uba39\uba3a\uba3b\uba3c\uba3d\uba3e\uba3f\uba40\uba41\uba42\uba43\uba44\uba45\uba46\uba47\uba48\uba49\uba4a\uba4b\uba4c\uba4d\uba4e\uba4f\uba50\uba51\uba52\uba53\uba54\uba55\uba56\uba57\uba58\uba59\uba5a\uba5b\uba5c\uba5d\uba5e\uba5f\uba60\uba61\uba62\uba63\uba64\uba65\uba66\uba67\uba68\uba69\uba6a\uba6b\uba6c\uba6d\uba6e\uba6f\uba70\uba71\uba72\uba73\uba74\uba75\uba76\uba77\uba78\uba79\uba7a\uba7b\uba7c\uba7d\uba7e\uba7f\uba80\uba81\uba82\uba83\uba84\uba85\uba86\uba87\uba88\uba89\uba8a\uba8b\uba8c\uba8d\uba8e\uba8f\uba90\uba91\uba92\uba93\uba94\uba95\uba96\uba97\uba98\uba99\uba9a\uba9b\uba9c\uba9d\uba9e\uba9f\ubaa0\ubaa1\ubaa2\ubaa3\ubaa4\ubaa5\ubaa6\ubaa7\ubaa8\ubaa9\ubaaa\ubaab\ubaac\ubaad\ubaae\ubaaf\ubab0\ubab1\ubab2\ubab3\ubab4\ubab5\ubab6\ubab7\ubab8\ubab9\ubaba\ubabb\ubabc\ubabd\ubabe\ubabf\ubac0\ubac1\ubac2\ubac3\ubac4\ubac5\ubac6\ubac7\ubac8\ubac9\ubaca\ubacb\ubacc\ubacd\ubace\ubacf\ubad0\ubad1\ubad2\ubad3\ubad4\ubad5\ubad6\ubad7\ubad8\ubad9\ubada\ubadb\ubadc\ubadd\ubade\ubadf\ubae0\ubae1\ubae2\ubae3\ubae4\ubae5\ubae6\ubae7\ubae8\ubae9\ubaea\ubaeb\ubaec\ubaed\ubaee\ubaef\ubaf0\ubaf1\ubaf2\ubaf3\ubaf4\ubaf5\ubaf6\ubaf7\ubaf8\ubaf9\ubafa\ubafb\ubafc\ubafd\ubafe\ubaff\ubb00\ubb01\ubb02\ubb03\ubb04\ubb05\ubb06\ubb07\ubb08\ubb09\ubb0a\ubb0b\ubb0c\ubb0d\ubb0e\ubb0f\ubb10\ubb11\ubb12\ubb13\ubb14\ubb15\ubb16\ubb17\ubb18\ubb19\ubb1a\ubb1b\ubb1c\ubb1d\ubb1e\ubb1f\ubb20\ubb21\ubb22\ubb23\ubb24\ubb25\ubb26\ubb27\ubb28\ubb29\ubb2a\ubb2b\ubb2c\ubb2d\ubb2e\ubb2f\ubb30\ubb31\ubb32\ubb33\ubb34\ubb35\ubb36\ubb37\ubb38\ubb39\ubb3a\ubb3b\ubb3c\ubb3d\ubb3e\ubb3f\ubb40\ubb41\ubb42\ubb43\ubb44\ubb45\ubb46\ubb47\ubb48\ubb49\ubb4a\ubb4b\ubb4c\ubb4d\ubb4e\ubb4f\ubb50\ubb51\ubb52\ubb53\ubb54\ubb55\ubb56\ubb57\ubb58\ubb59\ubb5a\ubb5b\ubb5c\ubb5d\ubb5e\ubb5f\ubb60\ubb61\ubb62\ubb63\ubb64\ubb65\ubb66\ubb67\ubb68\ubb69\ubb6a\ubb6b\ubb6c\ubb6d\ubb6e\ubb6f\ubb70\ubb71\ubb72\ubb73\ubb74\ubb75\ubb76\ubb77\ubb78\ubb79\ubb7a\ubb7b\ubb7c\ubb7d\ubb7e\ubb7f\ubb80\ubb81\ubb82\ubb83\ubb84\ubb85\ubb86\ubb87\ubb88\ubb89\ubb8a\ubb8b\ubb8c\ubb8d\ubb8e\ubb8f\ubb90\ubb91\ubb92\ubb93\ubb94\ubb95\ubb96\ubb97\ubb98\ubb99\ubb9a\ubb9b\ubb9c\ubb9d\ubb9e\ubb9f\ubba0\ubba1\ubba2\ubba3\ubba4\ubba5\ubba6\ubba7\ubba8\ubba9\ubbaa\ubbab\ubbac\ubbad\ubbae\ubbaf\ubbb0\ubbb1\ubbb2\ubbb3\ubbb4\ubbb5\ubbb6\ubbb7\ubbb8\ubbb9\ubbba\ubbbb\ubbbc\ubbbd\ubbbe\ubbbf\ubbc0\ubbc1\ubbc2\ubbc3\ubbc4\ubbc5\ubbc6\ubbc7\ubbc8\ubbc9\ubbca\ubbcb\ubbcc\ubbcd\ubbce\ubbcf\ubbd0\ubbd1\ubbd2\ubbd3\ubbd4\ubbd5\ubbd6\ubbd7\ubbd8\ubbd9\ubbda\ubbdb\ubbdc\ubbdd\ubbde\ubbdf\ubbe0\ubbe1\ubbe2\ubbe3\ubbe4\ubbe5\ubbe6\ubbe7\ubbe8\ubbe9\ubbea\ubbeb\ubbec\ubbed\ubbee\ubbef\ubbf0\ubbf1\ubbf2\ubbf3\ubbf4\ubbf5\ubbf6\ubbf7\ubbf8\ubbf9\ubbfa\ubbfb\ubbfc\ubbfd\ubbfe\ubbff\ubc00\ubc01\ubc02\ubc03\ubc04\ubc05\ubc06\ubc07\ubc08\ubc09\ubc0a\ubc0b\ubc0c\ubc0d\ubc0e\ubc0f\ubc10\ubc11\ubc12\ubc13\ubc14\ubc15\ubc16\ubc17\ubc18\ubc19\ubc1a\ubc1b\ubc1c\ubc1d\ubc1e\ubc1f\ubc20\ubc21\ubc22\ubc23\ubc24\ubc25\ubc26\ubc27\ubc28\ubc29\ubc2a\ubc2b\ubc2c\ubc2d\ubc2e\ubc2f\ubc30\ubc31\ubc32\ubc33\ubc34\ubc35\ubc36\ubc37\ubc38\ubc39\ubc3a\ubc3b\ubc3c\ubc3d\ubc3e\ubc3f\ubc40\ubc41\ubc42\ubc43\ubc44\ubc45\ubc46\ubc47\ubc48\ubc49\ubc4a\ubc4b\ubc4c\ubc4d\ubc4e\ubc4f\ubc50\ubc51\ubc52\ubc53\ubc54\ubc55\ubc56\ubc57\ubc58\ubc59\ubc5a\ubc5b\ubc5c\ubc5d\ubc5e\ubc5f\ubc60\ubc61\ubc62\ubc63\ubc64\ubc65\ubc66\ubc67\ubc68\ubc69\ubc6a\ubc6b\ubc6c\ubc6d\ubc6e\ubc6f\ubc70\ubc71\ubc72\ubc73\ubc74\ubc75\ubc76\ubc77\ubc78\ubc79\ubc7a\ubc7b\ubc7c\ubc7d\ubc7e\ubc7f\ubc80\ubc81\ubc82\ubc83\ubc84\ubc85\ubc86\ubc87\ubc88\ubc89\ubc8a\ubc8b\ubc8c\ubc8d\ubc8e\ubc8f\ubc90\ubc91\ubc92\ubc93\ubc94\ubc95\ubc96\ubc97\ubc98\ubc99\ubc9a\ubc9b\ubc9c\ubc9d\ubc9e\ubc9f\ubca0\ubca1\ubca2\ubca3\ubca4\ubca5\ubca6\ubca7\ubca8\ubca9\ubcaa\ubcab\ubcac\ubcad\ubcae\ubcaf\ubcb0\ubcb1\ubcb2\ubcb3\ubcb4\ubcb5\ubcb6\ubcb7\ubcb8\ubcb9\ubcba\ubcbb\ubcbc\ubcbd\ubcbe\ubcbf\ubcc0\ubcc1\ubcc2\ubcc3\ubcc4\ubcc5\ubcc6\ubcc7\ubcc8\ubcc9\ubcca\ubccb\ubccc\ubccd\ubcce\ubccf\ubcd0\ubcd1\ubcd2\ubcd3\ubcd4\ubcd5\ubcd6\ubcd7\ubcd8\ubcd9\ubcda\ubcdb\ubcdc\ubcdd\ubcde\ubcdf\ubce0\ubce1\ubce2\ubce3\ubce4\ubce5\ubce6\ubce7\ubce8\ubce9\ubcea\ubceb\ubcec\ubced\ubcee\ubcef\ubcf0\ubcf1\ubcf2\ubcf3\ubcf4\ubcf5\ubcf6\ubcf7\ubcf8\ubcf9\ubcfa\ubcfb\ubcfc\ubcfd\ubcfe\ubcff\ubd00\ubd01\ubd02\ubd03\ubd04\ubd05\ubd06\ubd07\ubd08\ubd09\ubd0a\ubd0b\ubd0c\ubd0d\ubd0e\ubd0f\ubd10\ubd11\ubd12\ubd13\ubd14\ubd15\ubd16\ubd17\ubd18\ubd19\ubd1a\ubd1b\ubd1c\ubd1d\ubd1e\ubd1f\ubd20\ubd21\ubd22\ubd23\ubd24\ubd25\ubd26\ubd27\ubd28\ubd29\ubd2a\ubd2b\ubd2c\ubd2d\ubd2e\ubd2f\ubd30\ubd31\ubd32\ubd33\ubd34\ubd35\ubd36\ubd37\ubd38\ubd39\ubd3a\ubd3b\ubd3c\ubd3d\ubd3e\ubd3f\ubd40\ubd41\ubd42\ubd43\ubd44\ubd45\ubd46\ubd47\ubd48\ubd49\ubd4a\ubd4b\ubd4c\ubd4d\ubd4e\ubd4f\ubd50\ubd51\ubd52\ubd53\ubd54\ubd55\ubd56\ubd57\ubd58\ubd59\ubd5a\ubd5b\ubd5c\ubd5d\ubd5e\ubd5f\ubd60\ubd61\ubd62\ubd63\ubd64\ubd65\ubd66\ubd67\ubd68\ubd69\ubd6a\ubd6b\ubd6c\ubd6d\ubd6e\ubd6f\ubd70\ubd71\ubd72\ubd73\ubd74\ubd75\ubd76\ubd77\ubd78\ubd79\ubd7a\ubd7b\ubd7c\ubd7d\ubd7e\ubd7f\ubd80\ubd81\ubd82\ubd83\ubd84\ubd85\ubd86\ubd87\ubd88\ubd89\ubd8a\ubd8b\ubd8c\ubd8d\ubd8e\ubd8f\ubd90\ubd91\ubd92\ubd93\ubd94\ubd95\ubd96\ubd97\ubd98\ubd99\ubd9a\ubd9b\ubd9c\ubd9d\ubd9e\ubd9f\ubda0\ubda1\ubda2\ubda3\ubda4\ubda5\ubda6\ubda7\ubda8\ubda9\ubdaa\ubdab\ubdac\ubdad\ubdae\ubdaf\ubdb0\ubdb1\ubdb2\ubdb3\ubdb4\ubdb5\ubdb6\ubdb7\ubdb8\ubdb9\ubdba\ubdbb\ubdbc\ubdbd\ubdbe\ubdbf\ubdc0\ubdc1\ubdc2\ubdc3\ubdc4\ubdc5\ubdc6\ubdc7\ubdc8\ubdc9\ubdca\ubdcb\ubdcc\ubdcd\ubdce\ubdcf\ubdd0\ubdd1\ubdd2\ubdd3\ubdd4\ubdd5\ubdd6\ubdd7\ubdd8\ubdd9\ubdda\ubddb\ubddc\ubddd\ubdde\ubddf\ubde0\ubde1\ubde2\ubde3\ubde4\ubde5\ubde6\ubde7\ubde8\ubde9\ubdea\ubdeb\ubdec\ubded\ubdee\ubdef\ubdf0\ubdf1\ubdf2\ubdf3\ubdf4\ubdf5\ubdf6\ubdf7\ubdf8\ubdf9\ubdfa\ubdfb\ubdfc\ubdfd\ubdfe\ubdff\ube00\ube01\ube02\ube03\ube04\ube05\ube06\ube07\ube08\ube09\ube0a\ube0b\ube0c\ube0d\ube0e\ube0f\ube10\ube11\ube12\ube13\ube14\ube15\ube16\ube17\ube18\ube19\ube1a\ube1b\ube1c\ube1d\ube1e\ube1f\ube20\ube21\ube22\ube23\ube24\ube25\ube26\ube27\ube28\ube29\ube2a\ube2b\ube2c\ube2d\ube2e\ube2f\ube30\ube31\ube32\ube33\ube34\ube35\ube36\ube37\ube38\ube39\ube3a\ube3b\ube3c\ube3d\ube3e\ube3f\ube40\ube41\ube42\ube43\ube44\ube45\ube46\ube47\ube48\ube49\ube4a\ube4b\ube4c\ube4d\ube4e\ube4f\ube50\ube51\ube52\ube53\ube54\ube55\ube56\ube57\ube58\ube59\ube5a\ube5b\ube5c\ube5d\ube5e\ube5f\ube60\ube61\ube62\ube63\ube64\ube65\ube66\ube67\ube68\ube69\ube6a\ube6b\ube6c\ube6d\ube6e\ube6f\ube70\ube71\ube72\ube73\ube74\ube75\ube76\ube77\ube78\ube79\ube7a\ube7b\ube7c\ube7d\ube7e\ube7f\ube80\ube81\ube82\ube83\ube84\ube85\ube86\ube87\ube88\ube89\ube8a\ube8b\ube8c\ube8d\ube8e\ube8f\ube90\ube91\ube92\ube93\ube94\ube95\ube96\ube97\ube98\ube99\ube9a\ube9b\ube9c\ube9d\ube9e\ube9f\ubea0\ubea1\ubea2\ubea3\ubea4\ubea5\ubea6\ubea7\ubea8\ubea9\ubeaa\ubeab\ubeac\ubead\ubeae\ubeaf\ubeb0\ubeb1\ubeb2\ubeb3\ubeb4\ubeb5\ubeb6\ubeb7\ubeb8\ubeb9\ubeba\ubebb\ubebc\ubebd\ubebe\ubebf\ubec0\ubec1\ubec2\ubec3\ubec4\ubec5\ubec6\ubec7\ubec8\ubec9\ubeca\ubecb\ubecc\ubecd\ubece\ubecf\ubed0\ubed1\ubed2\ubed3\ubed4\ubed5\ubed6\ubed7\ubed8\ubed9\ubeda\ubedb\ubedc\ubedd\ubede\ubedf\ubee0\ubee1\ubee2\ubee3\ubee4\ubee5\ubee6\ubee7\ubee8\ubee9\ubeea\ubeeb\ubeec\ubeed\ubeee\ubeef\ubef0\ubef1\ubef2\ubef3\ubef4\ubef5\ubef6\ubef7\ubef8\ubef9\ubefa\ubefb\ubefc\ubefd\ubefe\ubeff\ubf00\ubf01\ubf02\ubf03\ubf04\ubf05\ubf06\ubf07\ubf08\ubf09\ubf0a\ubf0b\ubf0c\ubf0d\ubf0e\ubf0f\ubf10\ubf11\ubf12\ubf13\ubf14\ubf15\ubf16\ubf17\ubf18\ubf19\ubf1a\ubf1b\ubf1c\ubf1d\ubf1e\ubf1f\ubf20\ubf21\ubf22\ubf23\ubf24\ubf25\ubf26\ubf27\ubf28\ubf29\ubf2a\ubf2b\ubf2c\ubf2d\ubf2e\ubf2f\ubf30\ubf31\ubf32\ubf33\ubf34\ubf35\ubf36\ubf37\ubf38\ubf39\ubf3a\ubf3b\ubf3c\ubf3d\ubf3e\ubf3f\ubf40\ubf41\ubf42\ubf43\ubf44\ubf45\ubf46\ubf47\ubf48\ubf49\ubf4a\ubf4b\ubf4c\ubf4d\ubf4e\ubf4f\ubf50\ubf51\ubf52\ubf53\ubf54\ubf55\ubf56\ubf57\ubf58\ubf59\ubf5a\ubf5b\ubf5c\ubf5d\ubf5e\ubf5f\ubf60\ubf61\ubf62\ubf63\ubf64\ubf65\ubf66\ubf67\ubf68\ubf69\ubf6a\ubf6b\ubf6c\ubf6d\ubf6e\ubf6f\ubf70\ubf71\ubf72\ubf73\ubf74\ubf75\ubf76\ubf77\ubf78\ubf79\ubf7a\ubf7b\ubf7c\ubf7d\ubf7e\ubf7f\ubf80\ubf81\ubf82\ubf83\ubf84\ubf85\ubf86\ubf87\ubf88\ubf89\ubf8a\ubf8b\ubf8c\ubf8d\ubf8e\ubf8f\ubf90\ubf91\ubf92\ubf93\ubf94\ubf95\ubf96\ubf97\ubf98\ubf99\ubf9a\ubf9b\ubf9c\ubf9d\ubf9e\ubf9f\ubfa0\ubfa1\ubfa2\ubfa3\ubfa4\ubfa5\ubfa6\ubfa7\ubfa8\ubfa9\ubfaa\ubfab\ubfac\ubfad\ubfae\ubfaf\ubfb0\ubfb1\ubfb2\ubfb3\ubfb4\ubfb5\ubfb6\ubfb7\ubfb8\ubfb9\ubfba\ubfbb\ubfbc\ubfbd\ubfbe\ubfbf\ubfc0\ubfc1\ubfc2\ubfc3\ubfc4\ubfc5\ubfc6\ubfc7\ubfc8\ubfc9\ubfca\ubfcb\ubfcc\ubfcd\ubfce\ubfcf\ubfd0\ubfd1\ubfd2\ubfd3\ubfd4\ubfd5\ubfd6\ubfd7\ubfd8\ubfd9\ubfda\ubfdb\ubfdc\ubfdd\ubfde\ubfdf\ubfe0\ubfe1\ubfe2\ubfe3\ubfe4\ubfe5\ubfe6\ubfe7\ubfe8\ubfe9\ubfea\ubfeb\ubfec\ubfed\ubfee\ubfef\ubff0\ubff1\ubff2\ubff3\ubff4\ubff5\ubff6\ubff7\ubff8\ubff9\ubffa\ubffb\ubffc\ubffd\ubffe\ubfff\uc000\uc001\uc002\uc003\uc004\uc005\uc006\uc007\uc008\uc009\uc00a\uc00b\uc00c\uc00d\uc00e\uc00f\uc010\uc011\uc012\uc013\uc014\uc015\uc016\uc017\uc018\uc019\uc01a\uc01b\uc01c\uc01d\uc01e\uc01f\uc020\uc021\uc022\uc023\uc024\uc025\uc026\uc027\uc028\uc029\uc02a\uc02b\uc02c\uc02d\uc02e\uc02f\uc030\uc031\uc032\uc033\uc034\uc035\uc036\uc037\uc038\uc039\uc03a\uc03b\uc03c\uc03d\uc03e\uc03f\uc040\uc041\uc042\uc043\uc044\uc045\uc046\uc047\uc048\uc049\uc04a\uc04b\uc04c\uc04d\uc04e\uc04f\uc050\uc051\uc052\uc053\uc054\uc055\uc056\uc057\uc058\uc059\uc05a\uc05b\uc05c\uc05d\uc05e\uc05f\uc060\uc061\uc062\uc063\uc064\uc065\uc066\uc067\uc068\uc069\uc06a\uc06b\uc06c\uc06d\uc06e\uc06f\uc070\uc071\uc072\uc073\uc074\uc075\uc076\uc077\uc078\uc079\uc07a\uc07b\uc07c\uc07d\uc07e\uc07f\uc080\uc081\uc082\uc083\uc084\uc085\uc086\uc087\uc088\uc089\uc08a\uc08b\uc08c\uc08d\uc08e\uc08f\uc090\uc091\uc092\uc093\uc094\uc095\uc096\uc097\uc098\uc099\uc09a\uc09b\uc09c\uc09d\uc09e\uc09f\uc0a0\uc0a1\uc0a2\uc0a3\uc0a4\uc0a5\uc0a6\uc0a7\uc0a8\uc0a9\uc0aa\uc0ab\uc0ac\uc0ad\uc0ae\uc0af\uc0b0\uc0b1\uc0b2\uc0b3\uc0b4\uc0b5\uc0b6\uc0b7\uc0b8\uc0b9\uc0ba\uc0bb\uc0bc\uc0bd\uc0be\uc0bf\uc0c0\uc0c1\uc0c2\uc0c3\uc0c4\uc0c5\uc0c6\uc0c7\uc0c8\uc0c9\uc0ca\uc0cb\uc0cc\uc0cd\uc0ce\uc0cf\uc0d0\uc0d1\uc0d2\uc0d3\uc0d4\uc0d5\uc0d6\uc0d7\uc0d8\uc0d9\uc0da\uc0db\uc0dc\uc0dd\uc0de\uc0df\uc0e0\uc0e1\uc0e2\uc0e3\uc0e4\uc0e5\uc0e6\uc0e7\uc0e8\uc0e9\uc0ea\uc0eb\uc0ec\uc0ed\uc0ee\uc0ef\uc0f0\uc0f1\uc0f2\uc0f3\uc0f4\uc0f5\uc0f6\uc0f7\uc0f8\uc0f9\uc0fa\uc0fb\uc0fc\uc0fd\uc0fe\uc0ff\uc100\uc101\uc102\uc103\uc104\uc105\uc106\uc107\uc108\uc109\uc10a\uc10b\uc10c\uc10d\uc10e\uc10f\uc110\uc111\uc112\uc113\uc114\uc115\uc116\uc117\uc118\uc119\uc11a\uc11b\uc11c\uc11d\uc11e\uc11f\uc120\uc121\uc122\uc123\uc124\uc125\uc126\uc127\uc128\uc129\uc12a\uc12b\uc12c\uc12d\uc12e\uc12f\uc130\uc131\uc132\uc133\uc134\uc135\uc136\uc137\uc138\uc139\uc13a\uc13b\uc13c\uc13d\uc13e\uc13f\uc140\uc141\uc142\uc143\uc144\uc145\uc146\uc147\uc148\uc149\uc14a\uc14b\uc14c\uc14d\uc14e\uc14f\uc150\uc151\uc152\uc153\uc154\uc155\uc156\uc157\uc158\uc159\uc15a\uc15b\uc15c\uc15d\uc15e\uc15f\uc160\uc161\uc162\uc163\uc164\uc165\uc166\uc167\uc168\uc169\uc16a\uc16b\uc16c\uc16d\uc16e\uc16f\uc170\uc171\uc172\uc173\uc174\uc175\uc176\uc177\uc178\uc179\uc17a\uc17b\uc17c\uc17d\uc17e\uc17f\uc180\uc181\uc182\uc183\uc184\uc185\uc186\uc187\uc188\uc189\uc18a\uc18b\uc18c\uc18d\uc18e\uc18f\uc190\uc191\uc192\uc193\uc194\uc195\uc196\uc197\uc198\uc199\uc19a\uc19b\uc19c\uc19d\uc19e\uc19f\uc1a0\uc1a1\uc1a2\uc1a3\uc1a4\uc1a5\uc1a6\uc1a7\uc1a8\uc1a9\uc1aa\uc1ab\uc1ac\uc1ad\uc1ae\uc1af\uc1b0\uc1b1\uc1b2\uc1b3\uc1b4\uc1b5\uc1b6\uc1b7\uc1b8\uc1b9\uc1ba\uc1bb\uc1bc\uc1bd\uc1be\uc1bf\uc1c0\uc1c1\uc1c2\uc1c3\uc1c4\uc1c5\uc1c6\uc1c7\uc1c8\uc1c9\uc1ca\uc1cb\uc1cc\uc1cd\uc1ce\uc1cf\uc1d0\uc1d1\uc1d2\uc1d3\uc1d4\uc1d5\uc1d6\uc1d7\uc1d8\uc1d9\uc1da\uc1db\uc1dc\uc1dd\uc1de\uc1df\uc1e0\uc1e1\uc1e2\uc1e3\uc1e4\uc1e5\uc1e6\uc1e7\uc1e8\uc1e9\uc1ea\uc1eb\uc1ec\uc1ed\uc1ee\uc1ef\uc1f0\uc1f1\uc1f2\uc1f3\uc1f4\uc1f5\uc1f6\uc1f7\uc1f8\uc1f9\uc1fa\uc1fb\uc1fc\uc1fd\uc1fe\uc1ff\uc200\uc201\uc202\uc203\uc204\uc205\uc206\uc207\uc208\uc209\uc20a\uc20b\uc20c\uc20d\uc20e\uc20f\uc210\uc211\uc212\uc213\uc214\uc215\uc216\uc217\uc218\uc219\uc21a\uc21b\uc21c\uc21d\uc21e\uc21f\uc220\uc221\uc222\uc223\uc224\uc225\uc226\uc227\uc228\uc229\uc22a\uc22b\uc22c\uc22d\uc22e\uc22f\uc230\uc231\uc232\uc233\uc234\uc235\uc236\uc237\uc238\uc239\uc23a\uc23b\uc23c\uc23d\uc23e\uc23f\uc240\uc241\uc242\uc243\uc244\uc245\uc246\uc247\uc248\uc249\uc24a\uc24b\uc24c\uc24d\uc24e\uc24f\uc250\uc251\uc252\uc253\uc254\uc255\uc256\uc257\uc258\uc259\uc25a\uc25b\uc25c\uc25d\uc25e\uc25f\uc260\uc261\uc262\uc263\uc264\uc265\uc266\uc267\uc268\uc269\uc26a\uc26b\uc26c\uc26d\uc26e\uc26f\uc270\uc271\uc272\uc273\uc274\uc275\uc276\uc277\uc278\uc279\uc27a\uc27b\uc27c\uc27d\uc27e\uc27f\uc280\uc281\uc282\uc283\uc284\uc285\uc286\uc287\uc288\uc289\uc28a\uc28b\uc28c\uc28d\uc28e\uc28f\uc290\uc291\uc292\uc293\uc294\uc295\uc296\uc297\uc298\uc299\uc29a\uc29b\uc29c\uc29d\uc29e\uc29f\uc2a0\uc2a1\uc2a2\uc2a3\uc2a4\uc2a5\uc2a6\uc2a7\uc2a8\uc2a9\uc2aa\uc2ab\uc2ac\uc2ad\uc2ae\uc2af\uc2b0\uc2b1\uc2b2\uc2b3\uc2b4\uc2b5\uc2b6\uc2b7\uc2b8\uc2b9\uc2ba\uc2bb\uc2bc\uc2bd\uc2be\uc2bf\uc2c0\uc2c1\uc2c2\uc2c3\uc2c4\uc2c5\uc2c6\uc2c7\uc2c8\uc2c9\uc2ca\uc2cb\uc2cc\uc2cd\uc2ce\uc2cf\uc2d0\uc2d1\uc2d2\uc2d3\uc2d4\uc2d5\uc2d6\uc2d7\uc2d8\uc2d9\uc2da\uc2db\uc2dc\uc2dd\uc2de\uc2df\uc2e0\uc2e1\uc2e2\uc2e3\uc2e4\uc2e5\uc2e6\uc2e7\uc2e8\uc2e9\uc2ea\uc2eb\uc2ec\uc2ed\uc2ee\uc2ef\uc2f0\uc2f1\uc2f2\uc2f3\uc2f4\uc2f5\uc2f6\uc2f7\uc2f8\uc2f9\uc2fa\uc2fb\uc2fc\uc2fd\uc2fe\uc2ff\uc300\uc301\uc302\uc303\uc304\uc305\uc306\uc307\uc308\uc309\uc30a\uc30b\uc30c\uc30d\uc30e\uc30f\uc310\uc311\uc312\uc313\uc314\uc315\uc316\uc317\uc318\uc319\uc31a\uc31b\uc31c\uc31d\uc31e\uc31f\uc320\uc321\uc322\uc323\uc324\uc325\uc326\uc327\uc328\uc329\uc32a\uc32b\uc32c\uc32d\uc32e\uc32f\uc330\uc331\uc332\uc333\uc334\uc335\uc336\uc337\uc338\uc339\uc33a\uc33b\uc33c\uc33d\uc33e\uc33f\uc340\uc341\uc342\uc343\uc344\uc345\uc346\uc347\uc348\uc349\uc34a\uc34b\uc34c\uc34d\uc34e\uc34f\uc350\uc351\uc352\uc353\uc354\uc355\uc356\uc357\uc358\uc359\uc35a\uc35b\uc35c\uc35d\uc35e\uc35f\uc360\uc361\uc362\uc363\uc364\uc365\uc366\uc367\uc368\uc369\uc36a\uc36b\uc36c\uc36d\uc36e\uc36f\uc370\uc371\uc372\uc373\uc374\uc375\uc376\uc377\uc378\uc379\uc37a\uc37b\uc37c\uc37d\uc37e\uc37f\uc380\uc381\uc382\uc383\uc384\uc385\uc386\uc387\uc388\uc389\uc38a\uc38b\uc38c\uc38d\uc38e\uc38f\uc390\uc391\uc392\uc393\uc394\uc395\uc396\uc397\uc398\uc399\uc39a\uc39b\uc39c\uc39d\uc39e\uc39f\uc3a0\uc3a1\uc3a2\uc3a3\uc3a4\uc3a5\uc3a6\uc3a7\uc3a8\uc3a9\uc3aa\uc3ab\uc3ac\uc3ad\uc3ae\uc3af\uc3b0\uc3b1\uc3b2\uc3b3\uc3b4\uc3b5\uc3b6\uc3b7\uc3b8\uc3b9\uc3ba\uc3bb\uc3bc\uc3bd\uc3be\uc3bf\uc3c0\uc3c1\uc3c2\uc3c3\uc3c4\uc3c5\uc3c6\uc3c7\uc3c8\uc3c9\uc3ca\uc3cb\uc3cc\uc3cd\uc3ce\uc3cf\uc3d0\uc3d1\uc3d2\uc3d3\uc3d4\uc3d5\uc3d6\uc3d7\uc3d8\uc3d9\uc3da\uc3db\uc3dc\uc3dd\uc3de\uc3df\uc3e0\uc3e1\uc3e2\uc3e3\uc3e4\uc3e5\uc3e6\uc3e7\uc3e8\uc3e9\uc3ea\uc3eb\uc3ec\uc3ed\uc3ee\uc3ef\uc3f0\uc3f1\uc3f2\uc3f3\uc3f4\uc3f5\uc3f6\uc3f7\uc3f8\uc3f9\uc3fa\uc3fb\uc3fc\uc3fd\uc3fe\uc3ff\uc400\uc401\uc402\uc403\uc404\uc405\uc406\uc407\uc408\uc409\uc40a\uc40b\uc40c\uc40d\uc40e\uc40f\uc410\uc411\uc412\uc413\uc414\uc415\uc416\uc417\uc418\uc419\uc41a\uc41b\uc41c\uc41d\uc41e\uc41f\uc420\uc421\uc422\uc423\uc424\uc425\uc426\uc427\uc428\uc429\uc42a\uc42b\uc42c\uc42d\uc42e\uc42f\uc430\uc431\uc432\uc433\uc434\uc435\uc436\uc437\uc438\uc439\uc43a\uc43b\uc43c\uc43d\uc43e\uc43f\uc440\uc441\uc442\uc443\uc444\uc445\uc446\uc447\uc448\uc449\uc44a\uc44b\uc44c\uc44d\uc44e\uc44f\uc450\uc451\uc452\uc453\uc454\uc455\uc456\uc457\uc458\uc459\uc45a\uc45b\uc45c\uc45d\uc45e\uc45f\uc460\uc461\uc462\uc463\uc464\uc465\uc466\uc467\uc468\uc469\uc46a\uc46b\uc46c\uc46d\uc46e\uc46f\uc470\uc471\uc472\uc473\uc474\uc475\uc476\uc477\uc478\uc479\uc47a\uc47b\uc47c\uc47d\uc47e\uc47f\uc480\uc481\uc482\uc483\uc484\uc485\uc486\uc487\uc488\uc489\uc48a\uc48b\uc48c\uc48d\uc48e\uc48f\uc490\uc491\uc492\uc493\uc494\uc495\uc496\uc497\uc498\uc499\uc49a\uc49b\uc49c\uc49d\uc49e\uc49f\uc4a0\uc4a1\uc4a2\uc4a3\uc4a4\uc4a5\uc4a6\uc4a7\uc4a8\uc4a9\uc4aa\uc4ab\uc4ac\uc4ad\uc4ae\uc4af\uc4b0\uc4b1\uc4b2\uc4b3\uc4b4\uc4b5\uc4b6\uc4b7\uc4b8\uc4b9\uc4ba\uc4bb\uc4bc\uc4bd\uc4be\uc4bf\uc4c0\uc4c1\uc4c2\uc4c3\uc4c4\uc4c5\uc4c6\uc4c7\uc4c8\uc4c9\uc4ca\uc4cb\uc4cc\uc4cd\uc4ce\uc4cf\uc4d0\uc4d1\uc4d2\uc4d3\uc4d4\uc4d5\uc4d6\uc4d7\uc4d8\uc4d9\uc4da\uc4db\uc4dc\uc4dd\uc4de\uc4df\uc4e0\uc4e1\uc4e2\uc4e3\uc4e4\uc4e5\uc4e6\uc4e7\uc4e8\uc4e9\uc4ea\uc4eb\uc4ec\uc4ed\uc4ee\uc4ef\uc4f0\uc4f1\uc4f2\uc4f3\uc4f4\uc4f5\uc4f6\uc4f7\uc4f8\uc4f9\uc4fa\uc4fb\uc4fc\uc4fd\uc4fe\uc4ff\uc500\uc501\uc502\uc503\uc504\uc505\uc506\uc507\uc508\uc509\uc50a\uc50b\uc50c\uc50d\uc50e\uc50f\uc510\uc511\uc512\uc513\uc514\uc515\uc516\uc517\uc518\uc519\uc51a\uc51b\uc51c\uc51d\uc51e\uc51f\uc520\uc521\uc522\uc523\uc524\uc525\uc526\uc527\uc528\uc529\uc52a\uc52b\uc52c\uc52d\uc52e\uc52f\uc530\uc531\uc532\uc533\uc534\uc535\uc536\uc537\uc538\uc539\uc53a\uc53b\uc53c\uc53d\uc53e\uc53f\uc540\uc541\uc542\uc543\uc544\uc545\uc546\uc547\uc548\uc549\uc54a\uc54b\uc54c\uc54d\uc54e\uc54f\uc550\uc551\uc552\uc553\uc554\uc555\uc556\uc557\uc558\uc559\uc55a\uc55b\uc55c\uc55d\uc55e\uc55f\uc560\uc561\uc562\uc563\uc564\uc565\uc566\uc567\uc568\uc569\uc56a\uc56b\uc56c\uc56d\uc56e\uc56f\uc570\uc571\uc572\uc573\uc574\uc575\uc576\uc577\uc578\uc579\uc57a\uc57b\uc57c\uc57d\uc57e\uc57f\uc580\uc581\uc582\uc583\uc584\uc585\uc586\uc587\uc588\uc589\uc58a\uc58b\uc58c\uc58d\uc58e\uc58f\uc590\uc591\uc592\uc593\uc594\uc595\uc596\uc597\uc598\uc599\uc59a\uc59b\uc59c\uc59d\uc59e\uc59f\uc5a0\uc5a1\uc5a2\uc5a3\uc5a4\uc5a5\uc5a6\uc5a7\uc5a8\uc5a9\uc5aa\uc5ab\uc5ac\uc5ad\uc5ae\uc5af\uc5b0\uc5b1\uc5b2\uc5b3\uc5b4\uc5b5\uc5b6\uc5b7\uc5b8\uc5b9\uc5ba\uc5bb\uc5bc\uc5bd\uc5be\uc5bf\uc5c0\uc5c1\uc5c2\uc5c3\uc5c4\uc5c5\uc5c6\uc5c7\uc5c8\uc5c9\uc5ca\uc5cb\uc5cc\uc5cd\uc5ce\uc5cf\uc5d0\uc5d1\uc5d2\uc5d3\uc5d4\uc5d5\uc5d6\uc5d7\uc5d8\uc5d9\uc5da\uc5db\uc5dc\uc5dd\uc5de\uc5df\uc5e0\uc5e1\uc5e2\uc5e3\uc5e4\uc5e5\uc5e6\uc5e7\uc5e8\uc5e9\uc5ea\uc5eb\uc5ec\uc5ed\uc5ee\uc5ef\uc5f0\uc5f1\uc5f2\uc5f3\uc5f4\uc5f5\uc5f6\uc5f7\uc5f8\uc5f9\uc5fa\uc5fb\uc5fc\uc5fd\uc5fe\uc5ff\uc600\uc601\uc602\uc603\uc604\uc605\uc606\uc607\uc608\uc609\uc60a\uc60b\uc60c\uc60d\uc60e\uc60f\uc610\uc611\uc612\uc613\uc614\uc615\uc616\uc617\uc618\uc619\uc61a\uc61b\uc61c\uc61d\uc61e\uc61f\uc620\uc621\uc622\uc623\uc624\uc625\uc626\uc627\uc628\uc629\uc62a\uc62b\uc62c\uc62d\uc62e\uc62f\uc630\uc631\uc632\uc633\uc634\uc635\uc636\uc637\uc638\uc639\uc63a\uc63b\uc63c\uc63d\uc63e\uc63f\uc640\uc641\uc642\uc643\uc644\uc645\uc646\uc647\uc648\uc649\uc64a\uc64b\uc64c\uc64d\uc64e\uc64f\uc650\uc651\uc652\uc653\uc654\uc655\uc656\uc657\uc658\uc659\uc65a\uc65b\uc65c\uc65d\uc65e\uc65f\uc660\uc661\uc662\uc663\uc664\uc665\uc666\uc667\uc668\uc669\uc66a\uc66b\uc66c\uc66d\uc66e\uc66f\uc670\uc671\uc672\uc673\uc674\uc675\uc676\uc677\uc678\uc679\uc67a\uc67b\uc67c\uc67d\uc67e\uc67f\uc680\uc681\uc682\uc683\uc684\uc685\uc686\uc687\uc688\uc689\uc68a\uc68b\uc68c\uc68d\uc68e\uc68f\uc690\uc691\uc692\uc693\uc694\uc695\uc696\uc697\uc698\uc699\uc69a\uc69b\uc69c\uc69d\uc69e\uc69f\uc6a0\uc6a1\uc6a2\uc6a3\uc6a4\uc6a5\uc6a6\uc6a7\uc6a8\uc6a9\uc6aa\uc6ab\uc6ac\uc6ad\uc6ae\uc6af\uc6b0\uc6b1\uc6b2\uc6b3\uc6b4\uc6b5\uc6b6\uc6b7\uc6b8\uc6b9\uc6ba\uc6bb\uc6bc\uc6bd\uc6be\uc6bf\uc6c0\uc6c1\uc6c2\uc6c3\uc6c4\uc6c5\uc6c6\uc6c7\uc6c8\uc6c9\uc6ca\uc6cb\uc6cc\uc6cd\uc6ce\uc6cf\uc6d0\uc6d1\uc6d2\uc6d3\uc6d4\uc6d5\uc6d6\uc6d7\uc6d8\uc6d9\uc6da\uc6db\uc6dc\uc6dd\uc6de\uc6df\uc6e0\uc6e1\uc6e2\uc6e3\uc6e4\uc6e5\uc6e6\uc6e7\uc6e8\uc6e9\uc6ea\uc6eb\uc6ec\uc6ed\uc6ee\uc6ef\uc6f0\uc6f1\uc6f2\uc6f3\uc6f4\uc6f5\uc6f6\uc6f7\uc6f8\uc6f9\uc6fa\uc6fb\uc6fc\uc6fd\uc6fe\uc6ff\uc700\uc701\uc702\uc703\uc704\uc705\uc706\uc707\uc708\uc709\uc70a\uc70b\uc70c\uc70d\uc70e\uc70f\uc710\uc711\uc712\uc713\uc714\uc715\uc716\uc717\uc718\uc719\uc71a\uc71b\uc71c\uc71d\uc71e\uc71f\uc720\uc721\uc722\uc723\uc724\uc725\uc726\uc727\uc728\uc729\uc72a\uc72b\uc72c\uc72d\uc72e\uc72f\uc730\uc731\uc732\uc733\uc734\uc735\uc736\uc737\uc738\uc739\uc73a\uc73b\uc73c\uc73d\uc73e\uc73f\uc740\uc741\uc742\uc743\uc744\uc745\uc746\uc747\uc748\uc749\uc74a\uc74b\uc74c\uc74d\uc74e\uc74f\uc750\uc751\uc752\uc753\uc754\uc755\uc756\uc757\uc758\uc759\uc75a\uc75b\uc75c\uc75d\uc75e\uc75f\uc760\uc761\uc762\uc763\uc764\uc765\uc766\uc767\uc768\uc769\uc76a\uc76b\uc76c\uc76d\uc76e\uc76f\uc770\uc771\uc772\uc773\uc774\uc775\uc776\uc777\uc778\uc779\uc77a\uc77b\uc77c\uc77d\uc77e\uc77f\uc780\uc781\uc782\uc783\uc784\uc785\uc786\uc787\uc788\uc789\uc78a\uc78b\uc78c\uc78d\uc78e\uc78f\uc790\uc791\uc792\uc793\uc794\uc795\uc796\uc797\uc798\uc799\uc79a\uc79b\uc79c\uc79d\uc79e\uc79f\uc7a0\uc7a1\uc7a2\uc7a3\uc7a4\uc7a5\uc7a6\uc7a7\uc7a8\uc7a9\uc7aa\uc7ab\uc7ac\uc7ad\uc7ae\uc7af\uc7b0\uc7b1\uc7b2\uc7b3\uc7b4\uc7b5\uc7b6\uc7b7\uc7b8\uc7b9\uc7ba\uc7bb\uc7bc\uc7bd\uc7be\uc7bf\uc7c0\uc7c1\uc7c2\uc7c3\uc7c4\uc7c5\uc7c6\uc7c7\uc7c8\uc7c9\uc7ca\uc7cb\uc7cc\uc7cd\uc7ce\uc7cf\uc7d0\uc7d1\uc7d2\uc7d3\uc7d4\uc7d5\uc7d6\uc7d7\uc7d8\uc7d9\uc7da\uc7db\uc7dc\uc7dd\uc7de\uc7df\uc7e0\uc7e1\uc7e2\uc7e3\uc7e4\uc7e5\uc7e6\uc7e7\uc7e8\uc7e9\uc7ea\uc7eb\uc7ec\uc7ed\uc7ee\uc7ef\uc7f0\uc7f1\uc7f2\uc7f3\uc7f4\uc7f5\uc7f6\uc7f7\uc7f8\uc7f9\uc7fa\uc7fb\uc7fc\uc7fd\uc7fe\uc7ff\uc800\uc801\uc802\uc803\uc804\uc805\uc806\uc807\uc808\uc809\uc80a\uc80b\uc80c\uc80d\uc80e\uc80f\uc810\uc811\uc812\uc813\uc814\uc815\uc816\uc817\uc818\uc819\uc81a\uc81b\uc81c\uc81d\uc81e\uc81f\uc820\uc821\uc822\uc823\uc824\uc825\uc826\uc827\uc828\uc829\uc82a\uc82b\uc82c\uc82d\uc82e\uc82f\uc830\uc831\uc832\uc833\uc834\uc835\uc836\uc837\uc838\uc839\uc83a\uc83b\uc83c\uc83d\uc83e\uc83f\uc840\uc841\uc842\uc843\uc844\uc845\uc846\uc847\uc848\uc849\uc84a\uc84b\uc84c\uc84d\uc84e\uc84f\uc850\uc851\uc852\uc853\uc854\uc855\uc856\uc857\uc858\uc859\uc85a\uc85b\uc85c\uc85d\uc85e\uc85f\uc860\uc861\uc862\uc863\uc864\uc865\uc866\uc867\uc868\uc869\uc86a\uc86b\uc86c\uc86d\uc86e\uc86f\uc870\uc871\uc872\uc873\uc874\uc875\uc876\uc877\uc878\uc879\uc87a\uc87b\uc87c\uc87d\uc87e\uc87f\uc880\uc881\uc882\uc883\uc884\uc885\uc886\uc887\uc888\uc889\uc88a\uc88b\uc88c\uc88d\uc88e\uc88f\uc890\uc891\uc892\uc893\uc894\uc895\uc896\uc897\uc898\uc899\uc89a\uc89b\uc89c\uc89d\uc89e\uc89f\uc8a0\uc8a1\uc8a2\uc8a3\uc8a4\uc8a5\uc8a6\uc8a7\uc8a8\uc8a9\uc8aa\uc8ab\uc8ac\uc8ad\uc8ae\uc8af\uc8b0\uc8b1\uc8b2\uc8b3\uc8b4\uc8b5\uc8b6\uc8b7\uc8b8\uc8b9\uc8ba\uc8bb\uc8bc\uc8bd\uc8be\uc8bf\uc8c0\uc8c1\uc8c2\uc8c3\uc8c4\uc8c5\uc8c6\uc8c7\uc8c8\uc8c9\uc8ca\uc8cb\uc8cc\uc8cd\uc8ce\uc8cf\uc8d0\uc8d1\uc8d2\uc8d3\uc8d4\uc8d5\uc8d6\uc8d7\uc8d8\uc8d9\uc8da\uc8db\uc8dc\uc8dd\uc8de\uc8df\uc8e0\uc8e1\uc8e2\uc8e3\uc8e4\uc8e5\uc8e6\uc8e7\uc8e8\uc8e9\uc8ea\uc8eb\uc8ec\uc8ed\uc8ee\uc8ef\uc8f0\uc8f1\uc8f2\uc8f3\uc8f4\uc8f5\uc8f6\uc8f7\uc8f8\uc8f9\uc8fa\uc8fb\uc8fc\uc8fd\uc8fe\uc8ff\uc900\uc901\uc902\uc903\uc904\uc905\uc906\uc907\uc908\uc909\uc90a\uc90b\uc90c\uc90d\uc90e\uc90f\uc910\uc911\uc912\uc913\uc914\uc915\uc916\uc917\uc918\uc919\uc91a\uc91b\uc91c\uc91d\uc91e\uc91f\uc920\uc921\uc922\uc923\uc924\uc925\uc926\uc927\uc928\uc929\uc92a\uc92b\uc92c\uc92d\uc92e\uc92f\uc930\uc931\uc932\uc933\uc934\uc935\uc936\uc937\uc938\uc939\uc93a\uc93b\uc93c\uc93d\uc93e\uc93f\uc940\uc941\uc942\uc943\uc944\uc945\uc946\uc947\uc948\uc949\uc94a\uc94b\uc94c\uc94d\uc94e\uc94f\uc950\uc951\uc952\uc953\uc954\uc955\uc956\uc957\uc958\uc959\uc95a\uc95b\uc95c\uc95d\uc95e\uc95f\uc960\uc961\uc962\uc963\uc964\uc965\uc966\uc967\uc968\uc969\uc96a\uc96b\uc96c\uc96d\uc96e\uc96f\uc970\uc971\uc972\uc973\uc974\uc975\uc976\uc977\uc978\uc979\uc97a\uc97b\uc97c\uc97d\uc97e\uc97f\uc980\uc981\uc982\uc983\uc984\uc985\uc986\uc987\uc988\uc989\uc98a\uc98b\uc98c\uc98d\uc98e\uc98f\uc990\uc991\uc992\uc993\uc994\uc995\uc996\uc997\uc998\uc999\uc99a\uc99b\uc99c\uc99d\uc99e\uc99f\uc9a0\uc9a1\uc9a2\uc9a3\uc9a4\uc9a5\uc9a6\uc9a7\uc9a8\uc9a9\uc9aa\uc9ab\uc9ac\uc9ad\uc9ae\uc9af\uc9b0\uc9b1\uc9b2\uc9b3\uc9b4\uc9b5\uc9b6\uc9b7\uc9b8\uc9b9\uc9ba\uc9bb\uc9bc\uc9bd\uc9be\uc9bf\uc9c0\uc9c1\uc9c2\uc9c3\uc9c4\uc9c5\uc9c6\uc9c7\uc9c8\uc9c9\uc9ca\uc9cb\uc9cc\uc9cd\uc9ce\uc9cf\uc9d0\uc9d1\uc9d2\uc9d3\uc9d4\uc9d5\uc9d6\uc9d7\uc9d8\uc9d9\uc9da\uc9db\uc9dc\uc9dd\uc9de\uc9df\uc9e0\uc9e1\uc9e2\uc9e3\uc9e4\uc9e5\uc9e6\uc9e7\uc9e8\uc9e9\uc9ea\uc9eb\uc9ec\uc9ed\uc9ee\uc9ef\uc9f0\uc9f1\uc9f2\uc9f3\uc9f4\uc9f5\uc9f6\uc9f7\uc9f8\uc9f9\uc9fa\uc9fb\uc9fc\uc9fd\uc9fe\uc9ff\uca00\uca01\uca02\uca03\uca04\uca05\uca06\uca07\uca08\uca09\uca0a\uca0b\uca0c\uca0d\uca0e\uca0f\uca10\uca11\uca12\uca13\uca14\uca15\uca16\uca17\uca18\uca19\uca1a\uca1b\uca1c\uca1d\uca1e\uca1f\uca20\uca21\uca22\uca23\uca24\uca25\uca26\uca27\uca28\uca29\uca2a\uca2b\uca2c\uca2d\uca2e\uca2f\uca30\uca31\uca32\uca33\uca34\uca35\uca36\uca37\uca38\uca39\uca3a\uca3b\uca3c\uca3d\uca3e\uca3f\uca40\uca41\uca42\uca43\uca44\uca45\uca46\uca47\uca48\uca49\uca4a\uca4b\uca4c\uca4d\uca4e\uca4f\uca50\uca51\uca52\uca53\uca54\uca55\uca56\uca57\uca58\uca59\uca5a\uca5b\uca5c\uca5d\uca5e\uca5f\uca60\uca61\uca62\uca63\uca64\uca65\uca66\uca67\uca68\uca69\uca6a\uca6b\uca6c\uca6d\uca6e\uca6f\uca70\uca71\uca72\uca73\uca74\uca75\uca76\uca77\uca78\uca79\uca7a\uca7b\uca7c\uca7d\uca7e\uca7f\uca80\uca81\uca82\uca83\uca84\uca85\uca86\uca87\uca88\uca89\uca8a\uca8b\uca8c\uca8d\uca8e\uca8f\uca90\uca91\uca92\uca93\uca94\uca95\uca96\uca97\uca98\uca99\uca9a\uca9b\uca9c\uca9d\uca9e\uca9f\ucaa0\ucaa1\ucaa2\ucaa3\ucaa4\ucaa5\ucaa6\ucaa7\ucaa8\ucaa9\ucaaa\ucaab\ucaac\ucaad\ucaae\ucaaf\ucab0\ucab1\ucab2\ucab3\ucab4\ucab5\ucab6\ucab7\ucab8\ucab9\ucaba\ucabb\ucabc\ucabd\ucabe\ucabf\ucac0\ucac1\ucac2\ucac3\ucac4\ucac5\ucac6\ucac7\ucac8\ucac9\ucaca\ucacb\ucacc\ucacd\ucace\ucacf\ucad0\ucad1\ucad2\ucad3\ucad4\ucad5\ucad6\ucad7\ucad8\ucad9\ucada\ucadb\ucadc\ucadd\ucade\ucadf\ucae0\ucae1\ucae2\ucae3\ucae4\ucae5\ucae6\ucae7\ucae8\ucae9\ucaea\ucaeb\ucaec\ucaed\ucaee\ucaef\ucaf0\ucaf1\ucaf2\ucaf3\ucaf4\ucaf5\ucaf6\ucaf7\ucaf8\ucaf9\ucafa\ucafb\ucafc\ucafd\ucafe\ucaff\ucb00\ucb01\ucb02\ucb03\ucb04\ucb05\ucb06\ucb07\ucb08\ucb09\ucb0a\ucb0b\ucb0c\ucb0d\ucb0e\ucb0f\ucb10\ucb11\ucb12\ucb13\ucb14\ucb15\ucb16\ucb17\ucb18\ucb19\ucb1a\ucb1b\ucb1c\ucb1d\ucb1e\ucb1f\ucb20\ucb21\ucb22\ucb23\ucb24\ucb25\ucb26\ucb27\ucb28\ucb29\ucb2a\ucb2b\ucb2c\ucb2d\ucb2e\ucb2f\ucb30\ucb31\ucb32\ucb33\ucb34\ucb35\ucb36\ucb37\ucb38\ucb39\ucb3a\ucb3b\ucb3c\ucb3d\ucb3e\ucb3f\ucb40\ucb41\ucb42\ucb43\ucb44\ucb45\ucb46\ucb47\ucb48\ucb49\ucb4a\ucb4b\ucb4c\ucb4d\ucb4e\ucb4f\ucb50\ucb51\ucb52\ucb53\ucb54\ucb55\ucb56\ucb57\ucb58\ucb59\ucb5a\ucb5b\ucb5c\ucb5d\ucb5e\ucb5f\ucb60\ucb61\ucb62\ucb63\ucb64\ucb65\ucb66\ucb67\ucb68\ucb69\ucb6a\ucb6b\ucb6c\ucb6d\ucb6e\ucb6f\ucb70\ucb71\ucb72\ucb73\ucb74\ucb75\ucb76\ucb77\ucb78\ucb79\ucb7a\ucb7b\ucb7c\ucb7d\ucb7e\ucb7f\ucb80\ucb81\ucb82\ucb83\ucb84\ucb85\ucb86\ucb87\ucb88\ucb89\ucb8a\ucb8b\ucb8c\ucb8d\ucb8e\ucb8f\ucb90\ucb91\ucb92\ucb93\ucb94\ucb95\ucb96\ucb97\ucb98\ucb99\ucb9a\ucb9b\ucb9c\ucb9d\ucb9e\ucb9f\ucba0\ucba1\ucba2\ucba3\ucba4\ucba5\ucba6\ucba7\ucba8\ucba9\ucbaa\ucbab\ucbac\ucbad\ucbae\ucbaf\ucbb0\ucbb1\ucbb2\ucbb3\ucbb4\ucbb5\ucbb6\ucbb7\ucbb8\ucbb9\ucbba\ucbbb\ucbbc\ucbbd\ucbbe\ucbbf\ucbc0\ucbc1\ucbc2\ucbc3\ucbc4\ucbc5\ucbc6\ucbc7\ucbc8\ucbc9\ucbca\ucbcb\ucbcc\ucbcd\ucbce\ucbcf\ucbd0\ucbd1\ucbd2\ucbd3\ucbd4\ucbd5\ucbd6\ucbd7\ucbd8\ucbd9\ucbda\ucbdb\ucbdc\ucbdd\ucbde\ucbdf\ucbe0\ucbe1\ucbe2\ucbe3\ucbe4\ucbe5\ucbe6\ucbe7\ucbe8\ucbe9\ucbea\ucbeb\ucbec\ucbed\ucbee\ucbef\ucbf0\ucbf1\ucbf2\ucbf3\ucbf4\ucbf5\ucbf6\ucbf7\ucbf8\ucbf9\ucbfa\ucbfb\ucbfc\ucbfd\ucbfe\ucbff\ucc00\ucc01\ucc02\ucc03\ucc04\ucc05\ucc06\ucc07\ucc08\ucc09\ucc0a\ucc0b\ucc0c\ucc0d\ucc0e\ucc0f\ucc10\ucc11\ucc12\ucc13\ucc14\ucc15\ucc16\ucc17\ucc18\ucc19\ucc1a\ucc1b\ucc1c\ucc1d\ucc1e\ucc1f\ucc20\ucc21\ucc22\ucc23\ucc24\ucc25\ucc26\ucc27\ucc28\ucc29\ucc2a\ucc2b\ucc2c\ucc2d\ucc2e\ucc2f\ucc30\ucc31\ucc32\ucc33\ucc34\ucc35\ucc36\ucc37\ucc38\ucc39\ucc3a\ucc3b\ucc3c\ucc3d\ucc3e\ucc3f\ucc40\ucc41\ucc42\ucc43\ucc44\ucc45\ucc46\ucc47\ucc48\ucc49\ucc4a\ucc4b\ucc4c\ucc4d\ucc4e\ucc4f\ucc50\ucc51\ucc52\ucc53\ucc54\ucc55\ucc56\ucc57\ucc58\ucc59\ucc5a\ucc5b\ucc5c\ucc5d\ucc5e\ucc5f\ucc60\ucc61\ucc62\ucc63\ucc64\ucc65\ucc66\ucc67\ucc68\ucc69\ucc6a\ucc6b\ucc6c\ucc6d\ucc6e\ucc6f\ucc70\ucc71\ucc72\ucc73\ucc74\ucc75\ucc76\ucc77\ucc78\ucc79\ucc7a\ucc7b\ucc7c\ucc7d\ucc7e\ucc7f\ucc80\ucc81\ucc82\ucc83\ucc84\ucc85\ucc86\ucc87\ucc88\ucc89\ucc8a\ucc8b\ucc8c\ucc8d\ucc8e\ucc8f\ucc90\ucc91\ucc92\ucc93\ucc94\ucc95\ucc96\ucc97\ucc98\ucc99\ucc9a\ucc9b\ucc9c\ucc9d\ucc9e\ucc9f\ucca0\ucca1\ucca2\ucca3\ucca4\ucca5\ucca6\ucca7\ucca8\ucca9\uccaa\uccab\uccac\uccad\uccae\uccaf\uccb0\uccb1\uccb2\uccb3\uccb4\uccb5\uccb6\uccb7\uccb8\uccb9\uccba\uccbb\uccbc\uccbd\uccbe\uccbf\uccc0\uccc1\uccc2\uccc3\uccc4\uccc5\uccc6\uccc7\uccc8\uccc9\uccca\ucccb\ucccc\ucccd\uccce\ucccf\uccd0\uccd1\uccd2\uccd3\uccd4\uccd5\uccd6\uccd7\uccd8\uccd9\uccda\uccdb\uccdc\uccdd\uccde\uccdf\ucce0\ucce1\ucce2\ucce3\ucce4\ucce5\ucce6\ucce7\ucce8\ucce9\uccea\ucceb\uccec\ucced\uccee\uccef\uccf0\uccf1\uccf2\uccf3\uccf4\uccf5\uccf6\uccf7\uccf8\uccf9\uccfa\uccfb\uccfc\uccfd\uccfe\uccff\ucd00\ucd01\ucd02\ucd03\ucd04\ucd05\ucd06\ucd07\ucd08\ucd09\ucd0a\ucd0b\ucd0c\ucd0d\ucd0e\ucd0f\ucd10\ucd11\ucd12\ucd13\ucd14\ucd15\ucd16\ucd17\ucd18\ucd19\ucd1a\ucd1b\ucd1c\ucd1d\ucd1e\ucd1f\ucd20\ucd21\ucd22\ucd23\ucd24\ucd25\ucd26\ucd27\ucd28\ucd29\ucd2a\ucd2b\ucd2c\ucd2d\ucd2e\ucd2f\ucd30\ucd31\ucd32\ucd33\ucd34\ucd35\ucd36\ucd37\ucd38\ucd39\ucd3a\ucd3b\ucd3c\ucd3d\ucd3e\ucd3f\ucd40\ucd41\ucd42\ucd43\ucd44\ucd45\ucd46\ucd47\ucd48\ucd49\ucd4a\ucd4b\ucd4c\ucd4d\ucd4e\ucd4f\ucd50\ucd51\ucd52\ucd53\ucd54\ucd55\ucd56\ucd57\ucd58\ucd59\ucd5a\ucd5b\ucd5c\ucd5d\ucd5e\ucd5f\ucd60\ucd61\ucd62\ucd63\ucd64\ucd65\ucd66\ucd67\ucd68\ucd69\ucd6a\ucd6b\ucd6c\ucd6d\ucd6e\ucd6f\ucd70\ucd71\ucd72\ucd73\ucd74\ucd75\ucd76\ucd77\ucd78\ucd79\ucd7a\ucd7b\ucd7c\ucd7d\ucd7e\ucd7f\ucd80\ucd81\ucd82\ucd83\ucd84\ucd85\ucd86\ucd87\ucd88\ucd89\ucd8a\ucd8b\ucd8c\ucd8d\ucd8e\ucd8f\ucd90\ucd91\ucd92\ucd93\ucd94\ucd95\ucd96\ucd97\ucd98\ucd99\ucd9a\ucd9b\ucd9c\ucd9d\ucd9e\ucd9f\ucda0\ucda1\ucda2\ucda3\ucda4\ucda5\ucda6\ucda7\ucda8\ucda9\ucdaa\ucdab\ucdac\ucdad\ucdae\ucdaf\ucdb0\ucdb1\ucdb2\ucdb3\ucdb4\ucdb5\ucdb6\ucdb7\ucdb8\ucdb9\ucdba\ucdbb\ucdbc\ucdbd\ucdbe\ucdbf\ucdc0\ucdc1\ucdc2\ucdc3\ucdc4\ucdc5\ucdc6\ucdc7\ucdc8\ucdc9\ucdca\ucdcb\ucdcc\ucdcd\ucdce\ucdcf\ucdd0\ucdd1\ucdd2\ucdd3\ucdd4\ucdd5\ucdd6\ucdd7\ucdd8\ucdd9\ucdda\ucddb\ucddc\ucddd\ucdde\ucddf\ucde0\ucde1\ucde2\ucde3\ucde4\ucde5\ucde6\ucde7\ucde8\ucde9\ucdea\ucdeb\ucdec\ucded\ucdee\ucdef\ucdf0\ucdf1\ucdf2\ucdf3\ucdf4\ucdf5\ucdf6\ucdf7\ucdf8\ucdf9\ucdfa\ucdfb\ucdfc\ucdfd\ucdfe\ucdff\uce00\uce01\uce02\uce03\uce04\uce05\uce06\uce07\uce08\uce09\uce0a\uce0b\uce0c\uce0d\uce0e\uce0f\uce10\uce11\uce12\uce13\uce14\uce15\uce16\uce17\uce18\uce19\uce1a\uce1b\uce1c\uce1d\uce1e\uce1f\uce20\uce21\uce22\uce23\uce24\uce25\uce26\uce27\uce28\uce29\uce2a\uce2b\uce2c\uce2d\uce2e\uce2f\uce30\uce31\uce32\uce33\uce34\uce35\uce36\uce37\uce38\uce39\uce3a\uce3b\uce3c\uce3d\uce3e\uce3f\uce40\uce41\uce42\uce43\uce44\uce45\uce46\uce47\uce48\uce49\uce4a\uce4b\uce4c\uce4d\uce4e\uce4f\uce50\uce51\uce52\uce53\uce54\uce55\uce56\uce57\uce58\uce59\uce5a\uce5b\uce5c\uce5d\uce5e\uce5f\uce60\uce61\uce62\uce63\uce64\uce65\uce66\uce67\uce68\uce69\uce6a\uce6b\uce6c\uce6d\uce6e\uce6f\uce70\uce71\uce72\uce73\uce74\uce75\uce76\uce77\uce78\uce79\uce7a\uce7b\uce7c\uce7d\uce7e\uce7f\uce80\uce81\uce82\uce83\uce84\uce85\uce86\uce87\uce88\uce89\uce8a\uce8b\uce8c\uce8d\uce8e\uce8f\uce90\uce91\uce92\uce93\uce94\uce95\uce96\uce97\uce98\uce99\uce9a\uce9b\uce9c\uce9d\uce9e\uce9f\ucea0\ucea1\ucea2\ucea3\ucea4\ucea5\ucea6\ucea7\ucea8\ucea9\uceaa\uceab\uceac\ucead\uceae\uceaf\uceb0\uceb1\uceb2\uceb3\uceb4\uceb5\uceb6\uceb7\uceb8\uceb9\uceba\ucebb\ucebc\ucebd\ucebe\ucebf\ucec0\ucec1\ucec2\ucec3\ucec4\ucec5\ucec6\ucec7\ucec8\ucec9\uceca\ucecb\ucecc\ucecd\ucece\ucecf\uced0\uced1\uced2\uced3\uced4\uced5\uced6\uced7\uced8\uced9\uceda\ucedb\ucedc\ucedd\ucede\ucedf\ucee0\ucee1\ucee2\ucee3\ucee4\ucee5\ucee6\ucee7\ucee8\ucee9\uceea\uceeb\uceec\uceed\uceee\uceef\ucef0\ucef1\ucef2\ucef3\ucef4\ucef5\ucef6\ucef7\ucef8\ucef9\ucefa\ucefb\ucefc\ucefd\ucefe\uceff\ucf00\ucf01\ucf02\ucf03\ucf04\ucf05\ucf06\ucf07\ucf08\ucf09\ucf0a\ucf0b\ucf0c\ucf0d\ucf0e\ucf0f\ucf10\ucf11\ucf12\ucf13\ucf14\ucf15\ucf16\ucf17\ucf18\ucf19\ucf1a\ucf1b\ucf1c\ucf1d\ucf1e\ucf1f\ucf20\ucf21\ucf22\ucf23\ucf24\ucf25\ucf26\ucf27\ucf28\ucf29\ucf2a\ucf2b\ucf2c\ucf2d\ucf2e\ucf2f\ucf30\ucf31\ucf32\ucf33\ucf34\ucf35\ucf36\ucf37\ucf38\ucf39\ucf3a\ucf3b\ucf3c\ucf3d\ucf3e\ucf3f\ucf40\ucf41\ucf42\ucf43\ucf44\ucf45\ucf46\ucf47\ucf48\ucf49\ucf4a\ucf4b\ucf4c\ucf4d\ucf4e\ucf4f\ucf50\ucf51\ucf52\ucf53\ucf54\ucf55\ucf56\ucf57\ucf58\ucf59\ucf5a\ucf5b\ucf5c\ucf5d\ucf5e\ucf5f\ucf60\ucf61\ucf62\ucf63\ucf64\ucf65\ucf66\ucf67\ucf68\ucf69\ucf6a\ucf6b\ucf6c\ucf6d\ucf6e\ucf6f\ucf70\ucf71\ucf72\ucf73\ucf74\ucf75\ucf76\ucf77\ucf78\ucf79\ucf7a\ucf7b\ucf7c\ucf7d\ucf7e\ucf7f\ucf80\ucf81\ucf82\ucf83\ucf84\ucf85\ucf86\ucf87\ucf88\ucf89\ucf8a\ucf8b\ucf8c\ucf8d\ucf8e\ucf8f\ucf90\ucf91\ucf92\ucf93\ucf94\ucf95\ucf96\ucf97\ucf98\ucf99\ucf9a\ucf9b\ucf9c\ucf9d\ucf9e\ucf9f\ucfa0\ucfa1\ucfa2\ucfa3\ucfa4\ucfa5\ucfa6\ucfa7\ucfa8\ucfa9\ucfaa\ucfab\ucfac\ucfad\ucfae\ucfaf\ucfb0\ucfb1\ucfb2\ucfb3\ucfb4\ucfb5\ucfb6\ucfb7\ucfb8\ucfb9\ucfba\ucfbb\ucfbc\ucfbd\ucfbe\ucfbf\ucfc0\ucfc1\ucfc2\ucfc3\ucfc4\ucfc5\ucfc6\ucfc7\ucfc8\ucfc9\ucfca\ucfcb\ucfcc\ucfcd\ucfce\ucfcf\ucfd0\ucfd1\ucfd2\ucfd3\ucfd4\ucfd5\ucfd6\ucfd7\ucfd8\ucfd9\ucfda\ucfdb\ucfdc\ucfdd\ucfde\ucfdf\ucfe0\ucfe1\ucfe2\ucfe3\ucfe4\ucfe5\ucfe6\ucfe7\ucfe8\ucfe9\ucfea\ucfeb\ucfec\ucfed\ucfee\ucfef\ucff0\ucff1\ucff2\ucff3\ucff4\ucff5\ucff6\ucff7\ucff8\ucff9\ucffa\ucffb\ucffc\ucffd\ucffe\ucfff\ud000\ud001\ud002\ud003\ud004\ud005\ud006\ud007\ud008\ud009\ud00a\ud00b\ud00c\ud00d\ud00e\ud00f\ud010\ud011\ud012\ud013\ud014\ud015\ud016\ud017\ud018\ud019\ud01a\ud01b\ud01c\ud01d\ud01e\ud01f\ud020\ud021\ud022\ud023\ud024\ud025\ud026\ud027\ud028\ud029\ud02a\ud02b\ud02c\ud02d\ud02e\ud02f\ud030\ud031\ud032\ud033\ud034\ud035\ud036\ud037\ud038\ud039\ud03a\ud03b\ud03c\ud03d\ud03e\ud03f\ud040\ud041\ud042\ud043\ud044\ud045\ud046\ud047\ud048\ud049\ud04a\ud04b\ud04c\ud04d\ud04e\ud04f\ud050\ud051\ud052\ud053\ud054\ud055\ud056\ud057\ud058\ud059\ud05a\ud05b\ud05c\ud05d\ud05e\ud05f\ud060\ud061\ud062\ud063\ud064\ud065\ud066\ud067\ud068\ud069\ud06a\ud06b\ud06c\ud06d\ud06e\ud06f\ud070\ud071\ud072\ud073\ud074\ud075\ud076\ud077\ud078\ud079\ud07a\ud07b\ud07c\ud07d\ud07e\ud07f\ud080\ud081\ud082\ud083\ud084\ud085\ud086\ud087\ud088\ud089\ud08a\ud08b\ud08c\ud08d\ud08e\ud08f\ud090\ud091\ud092\ud093\ud094\ud095\ud096\ud097\ud098\ud099\ud09a\ud09b\ud09c\ud09d\ud09e\ud09f\ud0a0\ud0a1\ud0a2\ud0a3\ud0a4\ud0a5\ud0a6\ud0a7\ud0a8\ud0a9\ud0aa\ud0ab\ud0ac\ud0ad\ud0ae\ud0af\ud0b0\ud0b1\ud0b2\ud0b3\ud0b4\ud0b5\ud0b6\ud0b7\ud0b8\ud0b9\ud0ba\ud0bb\ud0bc\ud0bd\ud0be\ud0bf\ud0c0\ud0c1\ud0c2\ud0c3\ud0c4\ud0c5\ud0c6\ud0c7\ud0c8\ud0c9\ud0ca\ud0cb\ud0cc\ud0cd\ud0ce\ud0cf\ud0d0\ud0d1\ud0d2\ud0d3\ud0d4\ud0d5\ud0d6\ud0d7\ud0d8\ud0d9\ud0da\ud0db\ud0dc\ud0dd\ud0de\ud0df\ud0e0\ud0e1\ud0e2\ud0e3\ud0e4\ud0e5\ud0e6\ud0e7\ud0e8\ud0e9\ud0ea\ud0eb\ud0ec\ud0ed\ud0ee\ud0ef\ud0f0\ud0f1\ud0f2\ud0f3\ud0f4\ud0f5\ud0f6\ud0f7\ud0f8\ud0f9\ud0fa\ud0fb\ud0fc\ud0fd\ud0fe\ud0ff\ud100\ud101\ud102\ud103\ud104\ud105\ud106\ud107\ud108\ud109\ud10a\ud10b\ud10c\ud10d\ud10e\ud10f\ud110\ud111\ud112\ud113\ud114\ud115\ud116\ud117\ud118\ud119\ud11a\ud11b\ud11c\ud11d\ud11e\ud11f\ud120\ud121\ud122\ud123\ud124\ud125\ud126\ud127\ud128\ud129\ud12a\ud12b\ud12c\ud12d\ud12e\ud12f\ud130\ud131\ud132\ud133\ud134\ud135\ud136\ud137\ud138\ud139\ud13a\ud13b\ud13c\ud13d\ud13e\ud13f\ud140\ud141\ud142\ud143\ud144\ud145\ud146\ud147\ud148\ud149\ud14a\ud14b\ud14c\ud14d\ud14e\ud14f\ud150\ud151\ud152\ud153\ud154\ud155\ud156\ud157\ud158\ud159\ud15a\ud15b\ud15c\ud15d\ud15e\ud15f\ud160\ud161\ud162\ud163\ud164\ud165\ud166\ud167\ud168\ud169\ud16a\ud16b\ud16c\ud16d\ud16e\ud16f\ud170\ud171\ud172\ud173\ud174\ud175\ud176\ud177\ud178\ud179\ud17a\ud17b\ud17c\ud17d\ud17e\ud17f\ud180\ud181\ud182\ud183\ud184\ud185\ud186\ud187\ud188\ud189\ud18a\ud18b\ud18c\ud18d\ud18e\ud18f\ud190\ud191\ud192\ud193\ud194\ud195\ud196\ud197\ud198\ud199\ud19a\ud19b\ud19c\ud19d\ud19e\ud19f\ud1a0\ud1a1\ud1a2\ud1a3\ud1a4\ud1a5\ud1a6\ud1a7\ud1a8\ud1a9\ud1aa\ud1ab\ud1ac\ud1ad\ud1ae\ud1af\ud1b0\ud1b1\ud1b2\ud1b3\ud1b4\ud1b5\ud1b6\ud1b7\ud1b8\ud1b9\ud1ba\ud1bb\ud1bc\ud1bd\ud1be\ud1bf\ud1c0\ud1c1\ud1c2\ud1c3\ud1c4\ud1c5\ud1c6\ud1c7\ud1c8\ud1c9\ud1ca\ud1cb\ud1cc\ud1cd\ud1ce\ud1cf\ud1d0\ud1d1\ud1d2\ud1d3\ud1d4\ud1d5\ud1d6\ud1d7\ud1d8\ud1d9\ud1da\ud1db\ud1dc\ud1dd\ud1de\ud1df\ud1e0\ud1e1\ud1e2\ud1e3\ud1e4\ud1e5\ud1e6\ud1e7\ud1e8\ud1e9\ud1ea\ud1eb\ud1ec\ud1ed\ud1ee\ud1ef\ud1f0\ud1f1\ud1f2\ud1f3\ud1f4\ud1f5\ud1f6\ud1f7\ud1f8\ud1f9\ud1fa\ud1fb\ud1fc\ud1fd\ud1fe\ud1ff\ud200\ud201\ud202\ud203\ud204\ud205\ud206\ud207\ud208\ud209\ud20a\ud20b\ud20c\ud20d\ud20e\ud20f\ud210\ud211\ud212\ud213\ud214\ud215\ud216\ud217\ud218\ud219\ud21a\ud21b\ud21c\ud21d\ud21e\ud21f\ud220\ud221\ud222\ud223\ud224\ud225\ud226\ud227\ud228\ud229\ud22a\ud22b\ud22c\ud22d\ud22e\ud22f\ud230\ud231\ud232\ud233\ud234\ud235\ud236\ud237\ud238\ud239\ud23a\ud23b\ud23c\ud23d\ud23e\ud23f\ud240\ud241\ud242\ud243\ud244\ud245\ud246\ud247\ud248\ud249\ud24a\ud24b\ud24c\ud24d\ud24e\ud24f\ud250\ud251\ud252\ud253\ud254\ud255\ud256\ud257\ud258\ud259\ud25a\ud25b\ud25c\ud25d\ud25e\ud25f\ud260\ud261\ud262\ud263\ud264\ud265\ud266\ud267\ud268\ud269\ud26a\ud26b\ud26c\ud26d\ud26e\ud26f\ud270\ud271\ud272\ud273\ud274\ud275\ud276\ud277\ud278\ud279\ud27a\ud27b\ud27c\ud27d\ud27e\ud27f\ud280\ud281\ud282\ud283\ud284\ud285\ud286\ud287\ud288\ud289\ud28a\ud28b\ud28c\ud28d\ud28e\ud28f\ud290\ud291\ud292\ud293\ud294\ud295\ud296\ud297\ud298\ud299\ud29a\ud29b\ud29c\ud29d\ud29e\ud29f\ud2a0\ud2a1\ud2a2\ud2a3\ud2a4\ud2a5\ud2a6\ud2a7\ud2a8\ud2a9\ud2aa\ud2ab\ud2ac\ud2ad\ud2ae\ud2af\ud2b0\ud2b1\ud2b2\ud2b3\ud2b4\ud2b5\ud2b6\ud2b7\ud2b8\ud2b9\ud2ba\ud2bb\ud2bc\ud2bd\ud2be\ud2bf\ud2c0\ud2c1\ud2c2\ud2c3\ud2c4\ud2c5\ud2c6\ud2c7\ud2c8\ud2c9\ud2ca\ud2cb\ud2cc\ud2cd\ud2ce\ud2cf\ud2d0\ud2d1\ud2d2\ud2d3\ud2d4\ud2d5\ud2d6\ud2d7\ud2d8\ud2d9\ud2da\ud2db\ud2dc\ud2dd\ud2de\ud2df\ud2e0\ud2e1\ud2e2\ud2e3\ud2e4\ud2e5\ud2e6\ud2e7\ud2e8\ud2e9\ud2ea\ud2eb\ud2ec\ud2ed\ud2ee\ud2ef\ud2f0\ud2f1\ud2f2\ud2f3\ud2f4\ud2f5\ud2f6\ud2f7\ud2f8\ud2f9\ud2fa\ud2fb\ud2fc\ud2fd\ud2fe\ud2ff\ud300\ud301\ud302\ud303\ud304\ud305\ud306\ud307\ud308\ud309\ud30a\ud30b\ud30c\ud30d\ud30e\ud30f\ud310\ud311\ud312\ud313\ud314\ud315\ud316\ud317\ud318\ud319\ud31a\ud31b\ud31c\ud31d\ud31e\ud31f\ud320\ud321\ud322\ud323\ud324\ud325\ud326\ud327\ud328\ud329\ud32a\ud32b\ud32c\ud32d\ud32e\ud32f\ud330\ud331\ud332\ud333\ud334\ud335\ud336\ud337\ud338\ud339\ud33a\ud33b\ud33c\ud33d\ud33e\ud33f\ud340\ud341\ud342\ud343\ud344\ud345\ud346\ud347\ud348\ud349\ud34a\ud34b\ud34c\ud34d\ud34e\ud34f\ud350\ud351\ud352\ud353\ud354\ud355\ud356\ud357\ud358\ud359\ud35a\ud35b\ud35c\ud35d\ud35e\ud35f\ud360\ud361\ud362\ud363\ud364\ud365\ud366\ud367\ud368\ud369\ud36a\ud36b\ud36c\ud36d\ud36e\ud36f\ud370\ud371\ud372\ud373\ud374\ud375\ud376\ud377\ud378\ud379\ud37a\ud37b\ud37c\ud37d\ud37e\ud37f\ud380\ud381\ud382\ud383\ud384\ud385\ud386\ud387\ud388\ud389\ud38a\ud38b\ud38c\ud38d\ud38e\ud38f\ud390\ud391\ud392\ud393\ud394\ud395\ud396\ud397\ud398\ud399\ud39a\ud39b\ud39c\ud39d\ud39e\ud39f\ud3a0\ud3a1\ud3a2\ud3a3\ud3a4\ud3a5\ud3a6\ud3a7\ud3a8\ud3a9\ud3aa\ud3ab\ud3ac\ud3ad\ud3ae\ud3af\ud3b0\ud3b1\ud3b2\ud3b3\ud3b4\ud3b5\ud3b6\ud3b7\ud3b8\ud3b9\ud3ba\ud3bb\ud3bc\ud3bd\ud3be\ud3bf\ud3c0\ud3c1\ud3c2\ud3c3\ud3c4\ud3c5\ud3c6\ud3c7\ud3c8\ud3c9\ud3ca\ud3cb\ud3cc\ud3cd\ud3ce\ud3cf\ud3d0\ud3d1\ud3d2\ud3d3\ud3d4\ud3d5\ud3d6\ud3d7\ud3d8\ud3d9\ud3da\ud3db\ud3dc\ud3dd\ud3de\ud3df\ud3e0\ud3e1\ud3e2\ud3e3\ud3e4\ud3e5\ud3e6\ud3e7\ud3e8\ud3e9\ud3ea\ud3eb\ud3ec\ud3ed\ud3ee\ud3ef\ud3f0\ud3f1\ud3f2\ud3f3\ud3f4\ud3f5\ud3f6\ud3f7\ud3f8\ud3f9\ud3fa\ud3fb\ud3fc\ud3fd\ud3fe\ud3ff\ud400\ud401\ud402\ud403\ud404\ud405\ud406\ud407\ud408\ud409\ud40a\ud40b\ud40c\ud40d\ud40e\ud40f\ud410\ud411\ud412\ud413\ud414\ud415\ud416\ud417\ud418\ud419\ud41a\ud41b\ud41c\ud41d\ud41e\ud41f\ud420\ud421\ud422\ud423\ud424\ud425\ud426\ud427\ud428\ud429\ud42a\ud42b\ud42c\ud42d\ud42e\ud42f\ud430\ud431\ud432\ud433\ud434\ud435\ud436\ud437\ud438\ud439\ud43a\ud43b\ud43c\ud43d\ud43e\ud43f\ud440\ud441\ud442\ud443\ud444\ud445\ud446\ud447\ud448\ud449\ud44a\ud44b\ud44c\ud44d\ud44e\ud44f\ud450\ud451\ud452\ud453\ud454\ud455\ud456\ud457\ud458\ud459\ud45a\ud45b\ud45c\ud45d\ud45e\ud45f\ud460\ud461\ud462\ud463\ud464\ud465\ud466\ud467\ud468\ud469\ud46a\ud46b\ud46c\ud46d\ud46e\ud46f\ud470\ud471\ud472\ud473\ud474\ud475\ud476\ud477\ud478\ud479\ud47a\ud47b\ud47c\ud47d\ud47e\ud47f\ud480\ud481\ud482\ud483\ud484\ud485\ud486\ud487\ud488\ud489\ud48a\ud48b\ud48c\ud48d\ud48e\ud48f\ud490\ud491\ud492\ud493\ud494\ud495\ud496\ud497\ud498\ud499\ud49a\ud49b\ud49c\ud49d\ud49e\ud49f\ud4a0\ud4a1\ud4a2\ud4a3\ud4a4\ud4a5\ud4a6\ud4a7\ud4a8\ud4a9\ud4aa\ud4ab\ud4ac\ud4ad\ud4ae\ud4af\ud4b0\ud4b1\ud4b2\ud4b3\ud4b4\ud4b5\ud4b6\ud4b7\ud4b8\ud4b9\ud4ba\ud4bb\ud4bc\ud4bd\ud4be\ud4bf\ud4c0\ud4c1\ud4c2\ud4c3\ud4c4\ud4c5\ud4c6\ud4c7\ud4c8\ud4c9\ud4ca\ud4cb\ud4cc\ud4cd\ud4ce\ud4cf\ud4d0\ud4d1\ud4d2\ud4d3\ud4d4\ud4d5\ud4d6\ud4d7\ud4d8\ud4d9\ud4da\ud4db\ud4dc\ud4dd\ud4de\ud4df\ud4e0\ud4e1\ud4e2\ud4e3\ud4e4\ud4e5\ud4e6\ud4e7\ud4e8\ud4e9\ud4ea\ud4eb\ud4ec\ud4ed\ud4ee\ud4ef\ud4f0\ud4f1\ud4f2\ud4f3\ud4f4\ud4f5\ud4f6\ud4f7\ud4f8\ud4f9\ud4fa\ud4fb\ud4fc\ud4fd\ud4fe\ud4ff\ud500\ud501\ud502\ud503\ud504\ud505\ud506\ud507\ud508\ud509\ud50a\ud50b\ud50c\ud50d\ud50e\ud50f\ud510\ud511\ud512\ud513\ud514\ud515\ud516\ud517\ud518\ud519\ud51a\ud51b\ud51c\ud51d\ud51e\ud51f\ud520\ud521\ud522\ud523\ud524\ud525\ud526\ud527\ud528\ud529\ud52a\ud52b\ud52c\ud52d\ud52e\ud52f\ud530\ud531\ud532\ud533\ud534\ud535\ud536\ud537\ud538\ud539\ud53a\ud53b\ud53c\ud53d\ud53e\ud53f\ud540\ud541\ud542\ud543\ud544\ud545\ud546\ud547\ud548\ud549\ud54a\ud54b\ud54c\ud54d\ud54e\ud54f\ud550\ud551\ud552\ud553\ud554\ud555\ud556\ud557\ud558\ud559\ud55a\ud55b\ud55c\ud55d\ud55e\ud55f\ud560\ud561\ud562\ud563\ud564\ud565\ud566\ud567\ud568\ud569\ud56a\ud56b\ud56c\ud56d\ud56e\ud56f\ud570\ud571\ud572\ud573\ud574\ud575\ud576\ud577\ud578\ud579\ud57a\ud57b\ud57c\ud57d\ud57e\ud57f\ud580\ud581\ud582\ud583\ud584\ud585\ud586\ud587\ud588\ud589\ud58a\ud58b\ud58c\ud58d\ud58e\ud58f\ud590\ud591\ud592\ud593\ud594\ud595\ud596\ud597\ud598\ud599\ud59a\ud59b\ud59c\ud59d\ud59e\ud59f\ud5a0\ud5a1\ud5a2\ud5a3\ud5a4\ud5a5\ud5a6\ud5a7\ud5a8\ud5a9\ud5aa\ud5ab\ud5ac\ud5ad\ud5ae\ud5af\ud5b0\ud5b1\ud5b2\ud5b3\ud5b4\ud5b5\ud5b6\ud5b7\ud5b8\ud5b9\ud5ba\ud5bb\ud5bc\ud5bd\ud5be\ud5bf\ud5c0\ud5c1\ud5c2\ud5c3\ud5c4\ud5c5\ud5c6\ud5c7\ud5c8\ud5c9\ud5ca\ud5cb\ud5cc\ud5cd\ud5ce\ud5cf\ud5d0\ud5d1\ud5d2\ud5d3\ud5d4\ud5d5\ud5d6\ud5d7\ud5d8\ud5d9\ud5da\ud5db\ud5dc\ud5dd\ud5de\ud5df\ud5e0\ud5e1\ud5e2\ud5e3\ud5e4\ud5e5\ud5e6\ud5e7\ud5e8\ud5e9\ud5ea\ud5eb\ud5ec\ud5ed\ud5ee\ud5ef\ud5f0\ud5f1\ud5f2\ud5f3\ud5f4\ud5f5\ud5f6\ud5f7\ud5f8\ud5f9\ud5fa\ud5fb\ud5fc\ud5fd\ud5fe\ud5ff\ud600\ud601\ud602\ud603\ud604\ud605\ud606\ud607\ud608\ud609\ud60a\ud60b\ud60c\ud60d\ud60e\ud60f\ud610\ud611\ud612\ud613\ud614\ud615\ud616\ud617\ud618\ud619\ud61a\ud61b\ud61c\ud61d\ud61e\ud61f\ud620\ud621\ud622\ud623\ud624\ud625\ud626\ud627\ud628\ud629\ud62a\ud62b\ud62c\ud62d\ud62e\ud62f\ud630\ud631\ud632\ud633\ud634\ud635\ud636\ud637\ud638\ud639\ud63a\ud63b\ud63c\ud63d\ud63e\ud63f\ud640\ud641\ud642\ud643\ud644\ud645\ud646\ud647\ud648\ud649\ud64a\ud64b\ud64c\ud64d\ud64e\ud64f\ud650\ud651\ud652\ud653\ud654\ud655\ud656\ud657\ud658\ud659\ud65a\ud65b\ud65c\ud65d\ud65e\ud65f\ud660\ud661\ud662\ud663\ud664\ud665\ud666\ud667\ud668\ud669\ud66a\ud66b\ud66c\ud66d\ud66e\ud66f\ud670\ud671\ud672\ud673\ud674\ud675\ud676\ud677\ud678\ud679\ud67a\ud67b\ud67c\ud67d\ud67e\ud67f\ud680\ud681\ud682\ud683\ud684\ud685\ud686\ud687\ud688\ud689\ud68a\ud68b\ud68c\ud68d\ud68e\ud68f\ud690\ud691\ud692\ud693\ud694\ud695\ud696\ud697\ud698\ud699\ud69a\ud69b\ud69c\ud69d\ud69e\ud69f\ud6a0\ud6a1\ud6a2\ud6a3\ud6a4\ud6a5\ud6a6\ud6a7\ud6a8\ud6a9\ud6aa\ud6ab\ud6ac\ud6ad\ud6ae\ud6af\ud6b0\ud6b1\ud6b2\ud6b3\ud6b4\ud6b5\ud6b6\ud6b7\ud6b8\ud6b9\ud6ba\ud6bb\ud6bc\ud6bd\ud6be\ud6bf\ud6c0\ud6c1\ud6c2\ud6c3\ud6c4\ud6c5\ud6c6\ud6c7\ud6c8\ud6c9\ud6ca\ud6cb\ud6cc\ud6cd\ud6ce\ud6cf\ud6d0\ud6d1\ud6d2\ud6d3\ud6d4\ud6d5\ud6d6\ud6d7\ud6d8\ud6d9\ud6da\ud6db\ud6dc\ud6dd\ud6de\ud6df\ud6e0\ud6e1\ud6e2\ud6e3\ud6e4\ud6e5\ud6e6\ud6e7\ud6e8\ud6e9\ud6ea\ud6eb\ud6ec\ud6ed\ud6ee\ud6ef\ud6f0\ud6f1\ud6f2\ud6f3\ud6f4\ud6f5\ud6f6\ud6f7\ud6f8\ud6f9\ud6fa\ud6fb\ud6fc\ud6fd\ud6fe\ud6ff\ud700\ud701\ud702\ud703\ud704\ud705\ud706\ud707\ud708\ud709\ud70a\ud70b\ud70c\ud70d\ud70e\ud70f\ud710\ud711\ud712\ud713\ud714\ud715\ud716\ud717\ud718\ud719\ud71a\ud71b\ud71c\ud71d\ud71e\ud71f\ud720\ud721\ud722\ud723\ud724\ud725\ud726\ud727\ud728\ud729\ud72a\ud72b\ud72c\ud72d\ud72e\ud72f\ud730\ud731\ud732\ud733\ud734\ud735\ud736\ud737\ud738\ud739\ud73a\ud73b\ud73c\ud73d\ud73e\ud73f\ud740\ud741\ud742\ud743\ud744\ud745\ud746\ud747\ud748\ud749\ud74a\ud74b\ud74c\ud74d\ud74e\ud74f\ud750\ud751\ud752\ud753\ud754\ud755\ud756\ud757\ud758\ud759\ud75a\ud75b\ud75c\ud75d\ud75e\ud75f\ud760\ud761\ud762\ud763\ud764\ud765\ud766\ud767\ud768\ud769\ud76a\ud76b\ud76c\ud76d\ud76e\ud76f\ud770\ud771\ud772\ud773\ud774\ud775\ud776\ud777\ud778\ud779\ud77a\ud77b\ud77c\ud77d\ud77e\ud77f\ud780\ud781\ud782\ud783\ud784\ud785\ud786\ud787\ud788\ud789\ud78a\ud78b\ud78c\ud78d\ud78e\ud78f\ud790\ud791\ud792\ud793\ud794\ud795\ud796\ud797\ud798\ud799\ud79a\ud79b\ud79c\ud79d\ud79e\ud79f\ud7a0\ud7a1\ud7a2\ud7a3\uf900\uf901\uf902\uf903\uf904\uf905\uf906\uf907\uf908\uf909\uf90a\uf90b\uf90c\uf90d\uf90e\uf90f\uf910\uf911\uf912\uf913\uf914\uf915\uf916\uf917\uf918\uf919\uf91a\uf91b\uf91c\uf91d\uf91e\uf91f\uf920\uf921\uf922\uf923\uf924\uf925\uf926\uf927\uf928\uf929\uf92a\uf92b\uf92c\uf92d\uf92e\uf92f\uf930\uf931\uf932\uf933\uf934\uf935\uf936\uf937\uf938\uf939\uf93a\uf93b\uf93c\uf93d\uf93e\uf93f\uf940\uf941\uf942\uf943\uf944\uf945\uf946\uf947\uf948\uf949\uf94a\uf94b\uf94c\uf94d\uf94e\uf94f\uf950\uf951\uf952\uf953\uf954\uf955\uf956\uf957\uf958\uf959\uf95a\uf95b\uf95c\uf95d\uf95e\uf95f\uf960\uf961\uf962\uf963\uf964\uf965\uf966\uf967\uf968\uf969\uf96a\uf96b\uf96c\uf96d\uf96e\uf96f\uf970\uf971\uf972\uf973\uf974\uf975\uf976\uf977\uf978\uf979\uf97a\uf97b\uf97c\uf97d\uf97e\uf97f\uf980\uf981\uf982\uf983\uf984\uf985\uf986\uf987\uf988\uf989\uf98a\uf98b\uf98c\uf98d\uf98e\uf98f\uf990\uf991\uf992\uf993\uf994\uf995\uf996\uf997\uf998\uf999\uf99a\uf99b\uf99c\uf99d\uf99e\uf99f\uf9a0\uf9a1\uf9a2\uf9a3\uf9a4\uf9a5\uf9a6\uf9a7\uf9a8\uf9a9\uf9aa\uf9ab\uf9ac\uf9ad\uf9ae\uf9af\uf9b0\uf9b1\uf9b2\uf9b3\uf9b4\uf9b5\uf9b6\uf9b7\uf9b8\uf9b9\uf9ba\uf9bb\uf9bc\uf9bd\uf9be\uf9bf\uf9c0\uf9c1\uf9c2\uf9c3\uf9c4\uf9c5\uf9c6\uf9c7\uf9c8\uf9c9\uf9ca\uf9cb\uf9cc\uf9cd\uf9ce\uf9cf\uf9d0\uf9d1\uf9d2\uf9d3\uf9d4\uf9d5\uf9d6\uf9d7\uf9d8\uf9d9\uf9da\uf9db\uf9dc\uf9dd\uf9de\uf9df\uf9e0\uf9e1\uf9e2\uf9e3\uf9e4\uf9e5\uf9e6\uf9e7\uf9e8\uf9e9\uf9ea\uf9eb\uf9ec\uf9ed\uf9ee\uf9ef\uf9f0\uf9f1\uf9f2\uf9f3\uf9f4\uf9f5\uf9f6\uf9f7\uf9f8\uf9f9\uf9fa\uf9fb\uf9fc\uf9fd\uf9fe\uf9ff\ufa00\ufa01\ufa02\ufa03\ufa04\ufa05\ufa06\ufa07\ufa08\ufa09\ufa0a\ufa0b\ufa0c\ufa0d\ufa0e\ufa0f\ufa10\ufa11\ufa12\ufa13\ufa14\ufa15\ufa16\ufa17\ufa18\ufa19\ufa1a\ufa1b\ufa1c\ufa1d\ufa1e\ufa1f\ufa20\ufa21\ufa22\ufa23\ufa24\ufa25\ufa26\ufa27\ufa28\ufa29\ufa2a\ufa2b\ufa2c\ufa2d\ufa30\ufa31\ufa32\ufa33\ufa34\ufa35\ufa36\ufa37\ufa38\ufa39\ufa3a\ufa3b\ufa3c\ufa3d\ufa3e\ufa3f\ufa40\ufa41\ufa42\ufa43\ufa44\ufa45\ufa46\ufa47\ufa48\ufa49\ufa4a\ufa4b\ufa4c\ufa4d\ufa4e\ufa4f\ufa50\ufa51\ufa52\ufa53\ufa54\ufa55\ufa56\ufa57\ufa58\ufa59\ufa5a\ufa5b\ufa5c\ufa5d\ufa5e\ufa5f\ufa60\ufa61\ufa62\ufa63\ufa64\ufa65\ufa66\ufa67\ufa68\ufa69\ufa6a\ufa70\ufa71\ufa72\ufa73\ufa74\ufa75\ufa76\ufa77\ufa78\ufa79\ufa7a\ufa7b\ufa7c\ufa7d\ufa7e\ufa7f\ufa80\ufa81\ufa82\ufa83\ufa84\ufa85\ufa86\ufa87\ufa88\ufa89\ufa8a\ufa8b\ufa8c\ufa8d\ufa8e\ufa8f\ufa90\ufa91\ufa92\ufa93\ufa94\ufa95\ufa96\ufa97\ufa98\ufa99\ufa9a\ufa9b\ufa9c\ufa9d\ufa9e\ufa9f\ufaa0\ufaa1\ufaa2\ufaa3\ufaa4\ufaa5\ufaa6\ufaa7\ufaa8\ufaa9\ufaaa\ufaab\ufaac\ufaad\ufaae\ufaaf\ufab0\ufab1\ufab2\ufab3\ufab4\ufab5\ufab6\ufab7\ufab8\ufab9\ufaba\ufabb\ufabc\ufabd\ufabe\ufabf\ufac0\ufac1\ufac2\ufac3\ufac4\ufac5\ufac6\ufac7\ufac8\ufac9\ufaca\ufacb\ufacc\ufacd\uface\ufacf\ufad0\ufad1\ufad2\ufad3\ufad4\ufad5\ufad6\ufad7\ufad8\ufad9\ufb1d\ufb1f\ufb20\ufb21\ufb22\ufb23\ufb24\ufb25\ufb26\ufb27\ufb28\ufb2a\ufb2b\ufb2c\ufb2d\ufb2e\ufb2f\ufb30\ufb31\ufb32\ufb33\ufb34\ufb35\ufb36\ufb38\ufb39\ufb3a\ufb3b\ufb3c\ufb3e\ufb40\ufb41\ufb43\ufb44\ufb46\ufb47\ufb48\ufb49\ufb4a\ufb4b\ufb4c\ufb4d\ufb4e\ufb4f\ufb50\ufb51\ufb52\ufb53\ufb54\ufb55\ufb56\ufb57\ufb58\ufb59\ufb5a\ufb5b\ufb5c\ufb5d\ufb5e\ufb5f\ufb60\ufb61\ufb62\ufb63\ufb64\ufb65\ufb66\ufb67\ufb68\ufb69\ufb6a\ufb6b\ufb6c\ufb6d\ufb6e\ufb6f\ufb70\ufb71\ufb72\ufb73\ufb74\ufb75\ufb76\ufb77\ufb78\ufb79\ufb7a\ufb7b\ufb7c\ufb7d\ufb7e\ufb7f\ufb80\ufb81\ufb82\ufb83\ufb84\ufb85\ufb86\ufb87\ufb88\ufb89\ufb8a\ufb8b\ufb8c\ufb8d\ufb8e\ufb8f\ufb90\ufb91\ufb92\ufb93\ufb94\ufb95\ufb96\ufb97\ufb98\ufb99\ufb9a\ufb9b\ufb9c\ufb9d\ufb9e\ufb9f\ufba0\ufba1\ufba2\ufba3\ufba4\ufba5\ufba6\ufba7\ufba8\ufba9\ufbaa\ufbab\ufbac\ufbad\ufbae\ufbaf\ufbb0\ufbb1\ufbd3\ufbd4\ufbd5\ufbd6\ufbd7\ufbd8\ufbd9\ufbda\ufbdb\ufbdc\ufbdd\ufbde\ufbdf\ufbe0\ufbe1\ufbe2\ufbe3\ufbe4\ufbe5\ufbe6\ufbe7\ufbe8\ufbe9\ufbea\ufbeb\ufbec\ufbed\ufbee\ufbef\ufbf0\ufbf1\ufbf2\ufbf3\ufbf4\ufbf5\ufbf6\ufbf7\ufbf8\ufbf9\ufbfa\ufbfb\ufbfc\ufbfd\ufbfe\ufbff\ufc00\ufc01\ufc02\ufc03\ufc04\ufc05\ufc06\ufc07\ufc08\ufc09\ufc0a\ufc0b\ufc0c\ufc0d\ufc0e\ufc0f\ufc10\ufc11\ufc12\ufc13\ufc14\ufc15\ufc16\ufc17\ufc18\ufc19\ufc1a\ufc1b\ufc1c\ufc1d\ufc1e\ufc1f\ufc20\ufc21\ufc22\ufc23\ufc24\ufc25\ufc26\ufc27\ufc28\ufc29\ufc2a\ufc2b\ufc2c\ufc2d\ufc2e\ufc2f\ufc30\ufc31\ufc32\ufc33\ufc34\ufc35\ufc36\ufc37\ufc38\ufc39\ufc3a\ufc3b\ufc3c\ufc3d\ufc3e\ufc3f\ufc40\ufc41\ufc42\ufc43\ufc44\ufc45\ufc46\ufc47\ufc48\ufc49\ufc4a\ufc4b\ufc4c\ufc4d\ufc4e\ufc4f\ufc50\ufc51\ufc52\ufc53\ufc54\ufc55\ufc56\ufc57\ufc58\ufc59\ufc5a\ufc5b\ufc5c\ufc5d\ufc5e\ufc5f\ufc60\ufc61\ufc62\ufc63\ufc64\ufc65\ufc66\ufc67\ufc68\ufc69\ufc6a\ufc6b\ufc6c\ufc6d\ufc6e\ufc6f\ufc70\ufc71\ufc72\ufc73\ufc74\ufc75\ufc76\ufc77\ufc78\ufc79\ufc7a\ufc7b\ufc7c\ufc7d\ufc7e\ufc7f\ufc80\ufc81\ufc82\ufc83\ufc84\ufc85\ufc86\ufc87\ufc88\ufc89\ufc8a\ufc8b\ufc8c\ufc8d\ufc8e\ufc8f\ufc90\ufc91\ufc92\ufc93\ufc94\ufc95\ufc96\ufc97\ufc98\ufc99\ufc9a\ufc9b\ufc9c\ufc9d\ufc9e\ufc9f\ufca0\ufca1\ufca2\ufca3\ufca4\ufca5\ufca6\ufca7\ufca8\ufca9\ufcaa\ufcab\ufcac\ufcad\ufcae\ufcaf\ufcb0\ufcb1\ufcb2\ufcb3\ufcb4\ufcb5\ufcb6\ufcb7\ufcb8\ufcb9\ufcba\ufcbb\ufcbc\ufcbd\ufcbe\ufcbf\ufcc0\ufcc1\ufcc2\ufcc3\ufcc4\ufcc5\ufcc6\ufcc7\ufcc8\ufcc9\ufcca\ufccb\ufccc\ufccd\ufcce\ufccf\ufcd0\ufcd1\ufcd2\ufcd3\ufcd4\ufcd5\ufcd6\ufcd7\ufcd8\ufcd9\ufcda\ufcdb\ufcdc\ufcdd\ufcde\ufcdf\ufce0\ufce1\ufce2\ufce3\ufce4\ufce5\ufce6\ufce7\ufce8\ufce9\ufcea\ufceb\ufcec\ufced\ufcee\ufcef\ufcf0\ufcf1\ufcf2\ufcf3\ufcf4\ufcf5\ufcf6\ufcf7\ufcf8\ufcf9\ufcfa\ufcfb\ufcfc\ufcfd\ufcfe\ufcff\ufd00\ufd01\ufd02\ufd03\ufd04\ufd05\ufd06\ufd07\ufd08\ufd09\ufd0a\ufd0b\ufd0c\ufd0d\ufd0e\ufd0f\ufd10\ufd11\ufd12\ufd13\ufd14\ufd15\ufd16\ufd17\ufd18\ufd19\ufd1a\ufd1b\ufd1c\ufd1d\ufd1e\ufd1f\ufd20\ufd21\ufd22\ufd23\ufd24\ufd25\ufd26\ufd27\ufd28\ufd29\ufd2a\ufd2b\ufd2c\ufd2d\ufd2e\ufd2f\ufd30\ufd31\ufd32\ufd33\ufd34\ufd35\ufd36\ufd37\ufd38\ufd39\ufd3a\ufd3b\ufd3c\ufd3d\ufd50\ufd51\ufd52\ufd53\ufd54\ufd55\ufd56\ufd57\ufd58\ufd59\ufd5a\ufd5b\ufd5c\ufd5d\ufd5e\ufd5f\ufd60\ufd61\ufd62\ufd63\ufd64\ufd65\ufd66\ufd67\ufd68\ufd69\ufd6a\ufd6b\ufd6c\ufd6d\ufd6e\ufd6f\ufd70\ufd71\ufd72\ufd73\ufd74\ufd75\ufd76\ufd77\ufd78\ufd79\ufd7a\ufd7b\ufd7c\ufd7d\ufd7e\ufd7f\ufd80\ufd81\ufd82\ufd83\ufd84\ufd85\ufd86\ufd87\ufd88\ufd89\ufd8a\ufd8b\ufd8c\ufd8d\ufd8e\ufd8f\ufd92\ufd93\ufd94\ufd95\ufd96\ufd97\ufd98\ufd99\ufd9a\ufd9b\ufd9c\ufd9d\ufd9e\ufd9f\ufda0\ufda1\ufda2\ufda3\ufda4\ufda5\ufda6\ufda7\ufda8\ufda9\ufdaa\ufdab\ufdac\ufdad\ufdae\ufdaf\ufdb0\ufdb1\ufdb2\ufdb3\ufdb4\ufdb5\ufdb6\ufdb7\ufdb8\ufdb9\ufdba\ufdbb\ufdbc\ufdbd\ufdbe\ufdbf\ufdc0\ufdc1\ufdc2\ufdc3\ufdc4\ufdc5\ufdc6\ufdc7\ufdf0\ufdf1\ufdf2\ufdf3\ufdf4\ufdf5\ufdf6\ufdf7\ufdf8\ufdf9\ufdfa\ufdfb\ufe70\ufe71\ufe72\ufe73\ufe74\ufe76\ufe77\ufe78\ufe79\ufe7a\ufe7b\ufe7c\ufe7d\ufe7e\ufe7f\ufe80\ufe81\ufe82\ufe83\ufe84\ufe85\ufe86\ufe87\ufe88\ufe89\ufe8a\ufe8b\ufe8c\ufe8d\ufe8e\ufe8f\ufe90\ufe91\ufe92\ufe93\ufe94\ufe95\ufe96\ufe97\ufe98\ufe99\ufe9a\ufe9b\ufe9c\ufe9d\ufe9e\ufe9f\ufea0\ufea1\ufea2\ufea3\ufea4\ufea5\ufea6\ufea7\ufea8\ufea9\ufeaa\ufeab\ufeac\ufead\ufeae\ufeaf\ufeb0\ufeb1\ufeb2\ufeb3\ufeb4\ufeb5\ufeb6\ufeb7\ufeb8\ufeb9\ufeba\ufebb\ufebc\ufebd\ufebe\ufebf\ufec0\ufec1\ufec2\ufec3\ufec4\ufec5\ufec6\ufec7\ufec8\ufec9\ufeca\ufecb\ufecc\ufecd\ufece\ufecf\ufed0\ufed1\ufed2\ufed3\ufed4\ufed5\ufed6\ufed7\ufed8\ufed9\ufeda\ufedb\ufedc\ufedd\ufede\ufedf\ufee0\ufee1\ufee2\ufee3\ufee4\ufee5\ufee6\ufee7\ufee8\ufee9\ufeea\ufeeb\ufeec\ufeed\ufeee\ufeef\ufef0\ufef1\ufef2\ufef3\ufef4\ufef5\ufef6\ufef7\ufef8\ufef9\ufefa\ufefb\ufefc\uff66\uff67\uff68\uff69\uff6a\uff6b\uff6c\uff6d\uff6e\uff6f\uff71\uff72\uff73\uff74\uff75\uff76\uff77\uff78\uff79\uff7a\uff7b\uff7c\uff7d\uff7e\uff7f\uff80\uff81\uff82\uff83\uff84\uff85\uff86\uff87\uff88\uff89\uff8a\uff8b\uff8c\uff8d\uff8e\uff8f\uff90\uff91\uff92\uff93\uff94\uff95\uff96\uff97\uff98\uff99\uff9a\uff9b\uff9c\uff9d\uffa0\uffa1\uffa2\uffa3\uffa4\uffa5\uffa6\uffa7\uffa8\uffa9\uffaa\uffab\uffac\uffad\uffae\uffaf\uffb0\uffb1\uffb2\uffb3\uffb4\uffb5\uffb6\uffb7\uffb8\uffb9\uffba\uffbb\uffbc\uffbd\uffbe\uffc2\uffc3\uffc4\uffc5\uffc6\uffc7\uffca\uffcb\uffcc\uffcd\uffce\uffcf\uffd2\uffd3\uffd4\uffd5\uffd6\uffd7\uffda\uffdb\uffdc' + +Lt = u'\u01c5\u01c8\u01cb\u01f2\u1f88\u1f89\u1f8a\u1f8b\u1f8c\u1f8d\u1f8e\u1f8f\u1f98\u1f99\u1f9a\u1f9b\u1f9c\u1f9d\u1f9e\u1f9f\u1fa8\u1fa9\u1faa\u1fab\u1fac\u1fad\u1fae\u1faf\u1fbc\u1fcc\u1ffc' + +Lu = u'ABCDEFGHIJKLMNOPQRSTUVWXYZ\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd8\xd9\xda\xdb\xdc\xdd\xde\u0100\u0102\u0104\u0106\u0108\u010a\u010c\u010e\u0110\u0112\u0114\u0116\u0118\u011a\u011c\u011e\u0120\u0122\u0124\u0126\u0128\u012a\u012c\u012e\u0130\u0132\u0134\u0136\u0139\u013b\u013d\u013f\u0141\u0143\u0145\u0147\u014a\u014c\u014e\u0150\u0152\u0154\u0156\u0158\u015a\u015c\u015e\u0160\u0162\u0164\u0166\u0168\u016a\u016c\u016e\u0170\u0172\u0174\u0176\u0178\u0179\u017b\u017d\u0181\u0182\u0184\u0186\u0187\u0189\u018a\u018b\u018e\u018f\u0190\u0191\u0193\u0194\u0196\u0197\u0198\u019c\u019d\u019f\u01a0\u01a2\u01a4\u01a6\u01a7\u01a9\u01ac\u01ae\u01af\u01b1\u01b2\u01b3\u01b5\u01b7\u01b8\u01bc\u01c4\u01c7\u01ca\u01cd\u01cf\u01d1\u01d3\u01d5\u01d7\u01d9\u01db\u01de\u01e0\u01e2\u01e4\u01e6\u01e8\u01ea\u01ec\u01ee\u01f1\u01f4\u01f6\u01f7\u01f8\u01fa\u01fc\u01fe\u0200\u0202\u0204\u0206\u0208\u020a\u020c\u020e\u0210\u0212\u0214\u0216\u0218\u021a\u021c\u021e\u0220\u0222\u0224\u0226\u0228\u022a\u022c\u022e\u0230\u0232\u023a\u023b\u023d\u023e\u0241\u0386\u0388\u0389\u038a\u038c\u038e\u038f\u0391\u0392\u0393\u0394\u0395\u0396\u0397\u0398\u0399\u039a\u039b\u039c\u039d\u039e\u039f\u03a0\u03a1\u03a3\u03a4\u03a5\u03a6\u03a7\u03a8\u03a9\u03aa\u03ab\u03d2\u03d3\u03d4\u03d8\u03da\u03dc\u03de\u03e0\u03e2\u03e4\u03e6\u03e8\u03ea\u03ec\u03ee\u03f4\u03f7\u03f9\u03fa\u03fd\u03fe\u03ff\u0400\u0401\u0402\u0403\u0404\u0405\u0406\u0407\u0408\u0409\u040a\u040b\u040c\u040d\u040e\u040f\u0410\u0411\u0412\u0413\u0414\u0415\u0416\u0417\u0418\u0419\u041a\u041b\u041c\u041d\u041e\u041f\u0420\u0421\u0422\u0423\u0424\u0425\u0426\u0427\u0428\u0429\u042a\u042b\u042c\u042d\u042e\u042f\u0460\u0462\u0464\u0466\u0468\u046a\u046c\u046e\u0470\u0472\u0474\u0476\u0478\u047a\u047c\u047e\u0480\u048a\u048c\u048e\u0490\u0492\u0494\u0496\u0498\u049a\u049c\u049e\u04a0\u04a2\u04a4\u04a6\u04a8\u04aa\u04ac\u04ae\u04b0\u04b2\u04b4\u04b6\u04b8\u04ba\u04bc\u04be\u04c0\u04c1\u04c3\u04c5\u04c7\u04c9\u04cb\u04cd\u04d0\u04d2\u04d4\u04d6\u04d8\u04da\u04dc\u04de\u04e0\u04e2\u04e4\u04e6\u04e8\u04ea\u04ec\u04ee\u04f0\u04f2\u04f4\u04f6\u04f8\u0500\u0502\u0504\u0506\u0508\u050a\u050c\u050e\u0531\u0532\u0533\u0534\u0535\u0536\u0537\u0538\u0539\u053a\u053b\u053c\u053d\u053e\u053f\u0540\u0541\u0542\u0543\u0544\u0545\u0546\u0547\u0548\u0549\u054a\u054b\u054c\u054d\u054e\u054f\u0550\u0551\u0552\u0553\u0554\u0555\u0556\u10a0\u10a1\u10a2\u10a3\u10a4\u10a5\u10a6\u10a7\u10a8\u10a9\u10aa\u10ab\u10ac\u10ad\u10ae\u10af\u10b0\u10b1\u10b2\u10b3\u10b4\u10b5\u10b6\u10b7\u10b8\u10b9\u10ba\u10bb\u10bc\u10bd\u10be\u10bf\u10c0\u10c1\u10c2\u10c3\u10c4\u10c5\u1e00\u1e02\u1e04\u1e06\u1e08\u1e0a\u1e0c\u1e0e\u1e10\u1e12\u1e14\u1e16\u1e18\u1e1a\u1e1c\u1e1e\u1e20\u1e22\u1e24\u1e26\u1e28\u1e2a\u1e2c\u1e2e\u1e30\u1e32\u1e34\u1e36\u1e38\u1e3a\u1e3c\u1e3e\u1e40\u1e42\u1e44\u1e46\u1e48\u1e4a\u1e4c\u1e4e\u1e50\u1e52\u1e54\u1e56\u1e58\u1e5a\u1e5c\u1e5e\u1e60\u1e62\u1e64\u1e66\u1e68\u1e6a\u1e6c\u1e6e\u1e70\u1e72\u1e74\u1e76\u1e78\u1e7a\u1e7c\u1e7e\u1e80\u1e82\u1e84\u1e86\u1e88\u1e8a\u1e8c\u1e8e\u1e90\u1e92\u1e94\u1ea0\u1ea2\u1ea4\u1ea6\u1ea8\u1eaa\u1eac\u1eae\u1eb0\u1eb2\u1eb4\u1eb6\u1eb8\u1eba\u1ebc\u1ebe\u1ec0\u1ec2\u1ec4\u1ec6\u1ec8\u1eca\u1ecc\u1ece\u1ed0\u1ed2\u1ed4\u1ed6\u1ed8\u1eda\u1edc\u1ede\u1ee0\u1ee2\u1ee4\u1ee6\u1ee8\u1eea\u1eec\u1eee\u1ef0\u1ef2\u1ef4\u1ef6\u1ef8\u1f08\u1f09\u1f0a\u1f0b\u1f0c\u1f0d\u1f0e\u1f0f\u1f18\u1f19\u1f1a\u1f1b\u1f1c\u1f1d\u1f28\u1f29\u1f2a\u1f2b\u1f2c\u1f2d\u1f2e\u1f2f\u1f38\u1f39\u1f3a\u1f3b\u1f3c\u1f3d\u1f3e\u1f3f\u1f48\u1f49\u1f4a\u1f4b\u1f4c\u1f4d\u1f59\u1f5b\u1f5d\u1f5f\u1f68\u1f69\u1f6a\u1f6b\u1f6c\u1f6d\u1f6e\u1f6f\u1fb8\u1fb9\u1fba\u1fbb\u1fc8\u1fc9\u1fca\u1fcb\u1fd8\u1fd9\u1fda\u1fdb\u1fe8\u1fe9\u1fea\u1feb\u1fec\u1ff8\u1ff9\u1ffa\u1ffb\u2102\u2107\u210b\u210c\u210d\u2110\u2111\u2112\u2115\u2119\u211a\u211b\u211c\u211d\u2124\u2126\u2128\u212a\u212b\u212c\u212d\u2130\u2131\u2133\u213e\u213f\u2145\u2c00\u2c01\u2c02\u2c03\u2c04\u2c05\u2c06\u2c07\u2c08\u2c09\u2c0a\u2c0b\u2c0c\u2c0d\u2c0e\u2c0f\u2c10\u2c11\u2c12\u2c13\u2c14\u2c15\u2c16\u2c17\u2c18\u2c19\u2c1a\u2c1b\u2c1c\u2c1d\u2c1e\u2c1f\u2c20\u2c21\u2c22\u2c23\u2c24\u2c25\u2c26\u2c27\u2c28\u2c29\u2c2a\u2c2b\u2c2c\u2c2d\u2c2e\u2c80\u2c82\u2c84\u2c86\u2c88\u2c8a\u2c8c\u2c8e\u2c90\u2c92\u2c94\u2c96\u2c98\u2c9a\u2c9c\u2c9e\u2ca0\u2ca2\u2ca4\u2ca6\u2ca8\u2caa\u2cac\u2cae\u2cb0\u2cb2\u2cb4\u2cb6\u2cb8\u2cba\u2cbc\u2cbe\u2cc0\u2cc2\u2cc4\u2cc6\u2cc8\u2cca\u2ccc\u2cce\u2cd0\u2cd2\u2cd4\u2cd6\u2cd8\u2cda\u2cdc\u2cde\u2ce0\u2ce2\uff21\uff22\uff23\uff24\uff25\uff26\uff27\uff28\uff29\uff2a\uff2b\uff2c\uff2d\uff2e\uff2f\uff30\uff31\uff32\uff33\uff34\uff35\uff36\uff37\uff38\uff39\uff3a' + +Mc = u'\u0903\u093e\u093f\u0940\u0949\u094a\u094b\u094c\u0982\u0983\u09be\u09bf\u09c0\u09c7\u09c8\u09cb\u09cc\u09d7\u0a03\u0a3e\u0a3f\u0a40\u0a83\u0abe\u0abf\u0ac0\u0ac9\u0acb\u0acc\u0b02\u0b03\u0b3e\u0b40\u0b47\u0b48\u0b4b\u0b4c\u0b57\u0bbe\u0bbf\u0bc1\u0bc2\u0bc6\u0bc7\u0bc8\u0bca\u0bcb\u0bcc\u0bd7\u0c01\u0c02\u0c03\u0c41\u0c42\u0c43\u0c44\u0c82\u0c83\u0cbe\u0cc0\u0cc1\u0cc2\u0cc3\u0cc4\u0cc7\u0cc8\u0cca\u0ccb\u0cd5\u0cd6\u0d02\u0d03\u0d3e\u0d3f\u0d40\u0d46\u0d47\u0d48\u0d4a\u0d4b\u0d4c\u0d57\u0d82\u0d83\u0dcf\u0dd0\u0dd1\u0dd8\u0dd9\u0dda\u0ddb\u0ddc\u0ddd\u0dde\u0ddf\u0df2\u0df3\u0f3e\u0f3f\u0f7f\u102c\u1031\u1038\u1056\u1057\u17b6\u17be\u17bf\u17c0\u17c1\u17c2\u17c3\u17c4\u17c5\u17c7\u17c8\u1923\u1924\u1925\u1926\u1929\u192a\u192b\u1930\u1931\u1933\u1934\u1935\u1936\u1937\u1938\u19b0\u19b1\u19b2\u19b3\u19b4\u19b5\u19b6\u19b7\u19b8\u19b9\u19ba\u19bb\u19bc\u19bd\u19be\u19bf\u19c0\u19c8\u19c9\u1a19\u1a1a\u1a1b\ua802\ua823\ua824\ua827' + +Me = u'\u0488\u0489\u06de\u20dd\u20de\u20df\u20e0\u20e2\u20e3\u20e4' + +Mn = u'\u0300\u0301\u0302\u0303\u0304\u0305\u0306\u0307\u0308\u0309\u030a\u030b\u030c\u030d\u030e\u030f\u0310\u0311\u0312\u0313\u0314\u0315\u0316\u0317\u0318\u0319\u031a\u031b\u031c\u031d\u031e\u031f\u0320\u0321\u0322\u0323\u0324\u0325\u0326\u0327\u0328\u0329\u032a\u032b\u032c\u032d\u032e\u032f\u0330\u0331\u0332\u0333\u0334\u0335\u0336\u0337\u0338\u0339\u033a\u033b\u033c\u033d\u033e\u033f\u0340\u0341\u0342\u0343\u0344\u0345\u0346\u0347\u0348\u0349\u034a\u034b\u034c\u034d\u034e\u034f\u0350\u0351\u0352\u0353\u0354\u0355\u0356\u0357\u0358\u0359\u035a\u035b\u035c\u035d\u035e\u035f\u0360\u0361\u0362\u0363\u0364\u0365\u0366\u0367\u0368\u0369\u036a\u036b\u036c\u036d\u036e\u036f\u0483\u0484\u0485\u0486\u0591\u0592\u0593\u0594\u0595\u0596\u0597\u0598\u0599\u059a\u059b\u059c\u059d\u059e\u059f\u05a0\u05a1\u05a2\u05a3\u05a4\u05a5\u05a6\u05a7\u05a8\u05a9\u05aa\u05ab\u05ac\u05ad\u05ae\u05af\u05b0\u05b1\u05b2\u05b3\u05b4\u05b5\u05b6\u05b7\u05b8\u05b9\u05bb\u05bc\u05bd\u05bf\u05c1\u05c2\u05c4\u05c5\u05c7\u0610\u0611\u0612\u0613\u0614\u0615\u064b\u064c\u064d\u064e\u064f\u0650\u0651\u0652\u0653\u0654\u0655\u0656\u0657\u0658\u0659\u065a\u065b\u065c\u065d\u065e\u0670\u06d6\u06d7\u06d8\u06d9\u06da\u06db\u06dc\u06df\u06e0\u06e1\u06e2\u06e3\u06e4\u06e7\u06e8\u06ea\u06eb\u06ec\u06ed\u0711\u0730\u0731\u0732\u0733\u0734\u0735\u0736\u0737\u0738\u0739\u073a\u073b\u073c\u073d\u073e\u073f\u0740\u0741\u0742\u0743\u0744\u0745\u0746\u0747\u0748\u0749\u074a\u07a6\u07a7\u07a8\u07a9\u07aa\u07ab\u07ac\u07ad\u07ae\u07af\u07b0\u0901\u0902\u093c\u0941\u0942\u0943\u0944\u0945\u0946\u0947\u0948\u094d\u0951\u0952\u0953\u0954\u0962\u0963\u0981\u09bc\u09c1\u09c2\u09c3\u09c4\u09cd\u09e2\u09e3\u0a01\u0a02\u0a3c\u0a41\u0a42\u0a47\u0a48\u0a4b\u0a4c\u0a4d\u0a70\u0a71\u0a81\u0a82\u0abc\u0ac1\u0ac2\u0ac3\u0ac4\u0ac5\u0ac7\u0ac8\u0acd\u0ae2\u0ae3\u0b01\u0b3c\u0b3f\u0b41\u0b42\u0b43\u0b4d\u0b56\u0b82\u0bc0\u0bcd\u0c3e\u0c3f\u0c40\u0c46\u0c47\u0c48\u0c4a\u0c4b\u0c4c\u0c4d\u0c55\u0c56\u0cbc\u0cbf\u0cc6\u0ccc\u0ccd\u0d41\u0d42\u0d43\u0d4d\u0dca\u0dd2\u0dd3\u0dd4\u0dd6\u0e31\u0e34\u0e35\u0e36\u0e37\u0e38\u0e39\u0e3a\u0e47\u0e48\u0e49\u0e4a\u0e4b\u0e4c\u0e4d\u0e4e\u0eb1\u0eb4\u0eb5\u0eb6\u0eb7\u0eb8\u0eb9\u0ebb\u0ebc\u0ec8\u0ec9\u0eca\u0ecb\u0ecc\u0ecd\u0f18\u0f19\u0f35\u0f37\u0f39\u0f71\u0f72\u0f73\u0f74\u0f75\u0f76\u0f77\u0f78\u0f79\u0f7a\u0f7b\u0f7c\u0f7d\u0f7e\u0f80\u0f81\u0f82\u0f83\u0f84\u0f86\u0f87\u0f90\u0f91\u0f92\u0f93\u0f94\u0f95\u0f96\u0f97\u0f99\u0f9a\u0f9b\u0f9c\u0f9d\u0f9e\u0f9f\u0fa0\u0fa1\u0fa2\u0fa3\u0fa4\u0fa5\u0fa6\u0fa7\u0fa8\u0fa9\u0faa\u0fab\u0fac\u0fad\u0fae\u0faf\u0fb0\u0fb1\u0fb2\u0fb3\u0fb4\u0fb5\u0fb6\u0fb7\u0fb8\u0fb9\u0fba\u0fbb\u0fbc\u0fc6\u102d\u102e\u102f\u1030\u1032\u1036\u1037\u1039\u1058\u1059\u135f\u1712\u1713\u1714\u1732\u1733\u1734\u1752\u1753\u1772\u1773\u17b7\u17b8\u17b9\u17ba\u17bb\u17bc\u17bd\u17c6\u17c9\u17ca\u17cb\u17cc\u17cd\u17ce\u17cf\u17d0\u17d1\u17d2\u17d3\u17dd\u180b\u180c\u180d\u18a9\u1920\u1921\u1922\u1927\u1928\u1932\u1939\u193a\u193b\u1a17\u1a18\u1dc0\u1dc1\u1dc2\u1dc3\u20d0\u20d1\u20d2\u20d3\u20d4\u20d5\u20d6\u20d7\u20d8\u20d9\u20da\u20db\u20dc\u20e1\u20e5\u20e6\u20e7\u20e8\u20e9\u20ea\u20eb\u302a\u302b\u302c\u302d\u302e\u302f\u3099\u309a\ua806\ua80b\ua825\ua826\ufb1e\ufe00\ufe01\ufe02\ufe03\ufe04\ufe05\ufe06\ufe07\ufe08\ufe09\ufe0a\ufe0b\ufe0c\ufe0d\ufe0e\ufe0f\ufe20\ufe21\ufe22\ufe23' + +Nd = u'0123456789\u0660\u0661\u0662\u0663\u0664\u0665\u0666\u0667\u0668\u0669\u06f0\u06f1\u06f2\u06f3\u06f4\u06f5\u06f6\u06f7\u06f8\u06f9\u0966\u0967\u0968\u0969\u096a\u096b\u096c\u096d\u096e\u096f\u09e6\u09e7\u09e8\u09e9\u09ea\u09eb\u09ec\u09ed\u09ee\u09ef\u0a66\u0a67\u0a68\u0a69\u0a6a\u0a6b\u0a6c\u0a6d\u0a6e\u0a6f\u0ae6\u0ae7\u0ae8\u0ae9\u0aea\u0aeb\u0aec\u0aed\u0aee\u0aef\u0b66\u0b67\u0b68\u0b69\u0b6a\u0b6b\u0b6c\u0b6d\u0b6e\u0b6f\u0be6\u0be7\u0be8\u0be9\u0bea\u0beb\u0bec\u0bed\u0bee\u0bef\u0c66\u0c67\u0c68\u0c69\u0c6a\u0c6b\u0c6c\u0c6d\u0c6e\u0c6f\u0ce6\u0ce7\u0ce8\u0ce9\u0cea\u0ceb\u0cec\u0ced\u0cee\u0cef\u0d66\u0d67\u0d68\u0d69\u0d6a\u0d6b\u0d6c\u0d6d\u0d6e\u0d6f\u0e50\u0e51\u0e52\u0e53\u0e54\u0e55\u0e56\u0e57\u0e58\u0e59\u0ed0\u0ed1\u0ed2\u0ed3\u0ed4\u0ed5\u0ed6\u0ed7\u0ed8\u0ed9\u0f20\u0f21\u0f22\u0f23\u0f24\u0f25\u0f26\u0f27\u0f28\u0f29\u1040\u1041\u1042\u1043\u1044\u1045\u1046\u1047\u1048\u1049\u17e0\u17e1\u17e2\u17e3\u17e4\u17e5\u17e6\u17e7\u17e8\u17e9\u1810\u1811\u1812\u1813\u1814\u1815\u1816\u1817\u1818\u1819\u1946\u1947\u1948\u1949\u194a\u194b\u194c\u194d\u194e\u194f\u19d0\u19d1\u19d2\u19d3\u19d4\u19d5\u19d6\u19d7\u19d8\u19d9\uff10\uff11\uff12\uff13\uff14\uff15\uff16\uff17\uff18\uff19' + +Nl = u'\u16ee\u16ef\u16f0\u2160\u2161\u2162\u2163\u2164\u2165\u2166\u2167\u2168\u2169\u216a\u216b\u216c\u216d\u216e\u216f\u2170\u2171\u2172\u2173\u2174\u2175\u2176\u2177\u2178\u2179\u217a\u217b\u217c\u217d\u217e\u217f\u2180\u2181\u2182\u2183\u3007\u3021\u3022\u3023\u3024\u3025\u3026\u3027\u3028\u3029\u3038\u3039\u303a' + +No = u'\xb2\xb3\xb9\xbc\xbd\xbe\u09f4\u09f5\u09f6\u09f7\u09f8\u09f9\u0bf0\u0bf1\u0bf2\u0f2a\u0f2b\u0f2c\u0f2d\u0f2e\u0f2f\u0f30\u0f31\u0f32\u0f33\u1369\u136a\u136b\u136c\u136d\u136e\u136f\u1370\u1371\u1372\u1373\u1374\u1375\u1376\u1377\u1378\u1379\u137a\u137b\u137c\u17f0\u17f1\u17f2\u17f3\u17f4\u17f5\u17f6\u17f7\u17f8\u17f9\u2070\u2074\u2075\u2076\u2077\u2078\u2079\u2080\u2081\u2082\u2083\u2084\u2085\u2086\u2087\u2088\u2089\u2153\u2154\u2155\u2156\u2157\u2158\u2159\u215a\u215b\u215c\u215d\u215e\u215f\u2460\u2461\u2462\u2463\u2464\u2465\u2466\u2467\u2468\u2469\u246a\u246b\u246c\u246d\u246e\u246f\u2470\u2471\u2472\u2473\u2474\u2475\u2476\u2477\u2478\u2479\u247a\u247b\u247c\u247d\u247e\u247f\u2480\u2481\u2482\u2483\u2484\u2485\u2486\u2487\u2488\u2489\u248a\u248b\u248c\u248d\u248e\u248f\u2490\u2491\u2492\u2493\u2494\u2495\u2496\u2497\u2498\u2499\u249a\u249b\u24ea\u24eb\u24ec\u24ed\u24ee\u24ef\u24f0\u24f1\u24f2\u24f3\u24f4\u24f5\u24f6\u24f7\u24f8\u24f9\u24fa\u24fb\u24fc\u24fd\u24fe\u24ff\u2776\u2777\u2778\u2779\u277a\u277b\u277c\u277d\u277e\u277f\u2780\u2781\u2782\u2783\u2784\u2785\u2786\u2787\u2788\u2789\u278a\u278b\u278c\u278d\u278e\u278f\u2790\u2791\u2792\u2793\u2cfd\u3192\u3193\u3194\u3195\u3220\u3221\u3222\u3223\u3224\u3225\u3226\u3227\u3228\u3229\u3251\u3252\u3253\u3254\u3255\u3256\u3257\u3258\u3259\u325a\u325b\u325c\u325d\u325e\u325f\u3280\u3281\u3282\u3283\u3284\u3285\u3286\u3287\u3288\u3289\u32b1\u32b2\u32b3\u32b4\u32b5\u32b6\u32b7\u32b8\u32b9\u32ba\u32bb\u32bc\u32bd\u32be\u32bf' + +Pc = u'_\u203f\u2040\u2054\ufe33\ufe34\ufe4d\ufe4e\ufe4f\uff3f' + +Pd = u'-\u058a\u1806\u2010\u2011\u2012\u2013\u2014\u2015\u2e17\u301c\u3030\u30a0\ufe31\ufe32\ufe58\ufe63\uff0d' + +Pe = u')]}\u0f3b\u0f3d\u169c\u2046\u207e\u208e\u232a\u23b5\u2769\u276b\u276d\u276f\u2771\u2773\u2775\u27c6\u27e7\u27e9\u27eb\u2984\u2986\u2988\u298a\u298c\u298e\u2990\u2992\u2994\u2996\u2998\u29d9\u29db\u29fd\u3009\u300b\u300d\u300f\u3011\u3015\u3017\u3019\u301b\u301e\u301f\ufd3f\ufe18\ufe36\ufe38\ufe3a\ufe3c\ufe3e\ufe40\ufe42\ufe44\ufe48\ufe5a\ufe5c\ufe5e\uff09\uff3d\uff5d\uff60\uff63' + +Pf = u'\xbb\u2019\u201d\u203a\u2e03\u2e05\u2e0a\u2e0d\u2e1d' + +Pi = u'\xab\u2018\u201b\u201c\u201f\u2039\u2e02\u2e04\u2e09\u2e0c\u2e1c' + +Po = u'!"#%&\'*,./:;?@\\\xa1\xb7\xbf\u037e\u0387\u055a\u055b\u055c\u055d\u055e\u055f\u0589\u05be\u05c0\u05c3\u05c6\u05f3\u05f4\u060c\u060d\u061b\u061e\u061f\u066a\u066b\u066c\u066d\u06d4\u0700\u0701\u0702\u0703\u0704\u0705\u0706\u0707\u0708\u0709\u070a\u070b\u070c\u070d\u0964\u0965\u0970\u0df4\u0e4f\u0e5a\u0e5b\u0f04\u0f05\u0f06\u0f07\u0f08\u0f09\u0f0a\u0f0b\u0f0c\u0f0d\u0f0e\u0f0f\u0f10\u0f11\u0f12\u0f85\u0fd0\u0fd1\u104a\u104b\u104c\u104d\u104e\u104f\u10fb\u1361\u1362\u1363\u1364\u1365\u1366\u1367\u1368\u166d\u166e\u16eb\u16ec\u16ed\u1735\u1736\u17d4\u17d5\u17d6\u17d8\u17d9\u17da\u1800\u1801\u1802\u1803\u1804\u1805\u1807\u1808\u1809\u180a\u1944\u1945\u19de\u19df\u1a1e\u1a1f\u2016\u2017\u2020\u2021\u2022\u2023\u2024\u2025\u2026\u2027\u2030\u2031\u2032\u2033\u2034\u2035\u2036\u2037\u2038\u203b\u203c\u203d\u203e\u2041\u2042\u2043\u2047\u2048\u2049\u204a\u204b\u204c\u204d\u204e\u204f\u2050\u2051\u2053\u2055\u2056\u2057\u2058\u2059\u205a\u205b\u205c\u205d\u205e\u23b6\u2cf9\u2cfa\u2cfb\u2cfc\u2cfe\u2cff\u2e00\u2e01\u2e06\u2e07\u2e08\u2e0b\u2e0e\u2e0f\u2e10\u2e11\u2e12\u2e13\u2e14\u2e15\u2e16\u3001\u3002\u3003\u303d\u30fb\ufe10\ufe11\ufe12\ufe13\ufe14\ufe15\ufe16\ufe19\ufe30\ufe45\ufe46\ufe49\ufe4a\ufe4b\ufe4c\ufe50\ufe51\ufe52\ufe54\ufe55\ufe56\ufe57\ufe5f\ufe60\ufe61\ufe68\ufe6a\ufe6b\uff01\uff02\uff03\uff05\uff06\uff07\uff0a\uff0c\uff0e\uff0f\uff1a\uff1b\uff1f\uff20\uff3c\uff61\uff64\uff65' + +Ps = u'([{\u0f3a\u0f3c\u169b\u201a\u201e\u2045\u207d\u208d\u2329\u23b4\u2768\u276a\u276c\u276e\u2770\u2772\u2774\u27c5\u27e6\u27e8\u27ea\u2983\u2985\u2987\u2989\u298b\u298d\u298f\u2991\u2993\u2995\u2997\u29d8\u29da\u29fc\u3008\u300a\u300c\u300e\u3010\u3014\u3016\u3018\u301a\u301d\ufd3e\ufe17\ufe35\ufe37\ufe39\ufe3b\ufe3d\ufe3f\ufe41\ufe43\ufe47\ufe59\ufe5b\ufe5d\uff08\uff3b\uff5b\uff5f\uff62' + +Sc = u'$\xa2\xa3\xa4\xa5\u060b\u09f2\u09f3\u0af1\u0bf9\u0e3f\u17db\u20a0\u20a1\u20a2\u20a3\u20a4\u20a5\u20a6\u20a7\u20a8\u20a9\u20aa\u20ab\u20ac\u20ad\u20ae\u20af\u20b0\u20b1\u20b2\u20b3\u20b4\u20b5\ufdfc\ufe69\uff04\uffe0\uffe1\uffe5\uffe6' + +Sk = u'^`\xa8\xaf\xb4\xb8\u02c2\u02c3\u02c4\u02c5\u02d2\u02d3\u02d4\u02d5\u02d6\u02d7\u02d8\u02d9\u02da\u02db\u02dc\u02dd\u02de\u02df\u02e5\u02e6\u02e7\u02e8\u02e9\u02ea\u02eb\u02ec\u02ed\u02ef\u02f0\u02f1\u02f2\u02f3\u02f4\u02f5\u02f6\u02f7\u02f8\u02f9\u02fa\u02fb\u02fc\u02fd\u02fe\u02ff\u0374\u0375\u0384\u0385\u1fbd\u1fbf\u1fc0\u1fc1\u1fcd\u1fce\u1fcf\u1fdd\u1fde\u1fdf\u1fed\u1fee\u1fef\u1ffd\u1ffe\u309b\u309c\ua700\ua701\ua702\ua703\ua704\ua705\ua706\ua707\ua708\ua709\ua70a\ua70b\ua70c\ua70d\ua70e\ua70f\ua710\ua711\ua712\ua713\ua714\ua715\ua716\uff3e\uff40\uffe3' + +Sm = u'+<=>|~\xac\xb1\xd7\xf7\u03f6\u2044\u2052\u207a\u207b\u207c\u208a\u208b\u208c\u2140\u2141\u2142\u2143\u2144\u214b\u2190\u2191\u2192\u2193\u2194\u219a\u219b\u21a0\u21a3\u21a6\u21ae\u21ce\u21cf\u21d2\u21d4\u21f4\u21f5\u21f6\u21f7\u21f8\u21f9\u21fa\u21fb\u21fc\u21fd\u21fe\u21ff\u2200\u2201\u2202\u2203\u2204\u2205\u2206\u2207\u2208\u2209\u220a\u220b\u220c\u220d\u220e\u220f\u2210\u2211\u2212\u2213\u2214\u2215\u2216\u2217\u2218\u2219\u221a\u221b\u221c\u221d\u221e\u221f\u2220\u2221\u2222\u2223\u2224\u2225\u2226\u2227\u2228\u2229\u222a\u222b\u222c\u222d\u222e\u222f\u2230\u2231\u2232\u2233\u2234\u2235\u2236\u2237\u2238\u2239\u223a\u223b\u223c\u223d\u223e\u223f\u2240\u2241\u2242\u2243\u2244\u2245\u2246\u2247\u2248\u2249\u224a\u224b\u224c\u224d\u224e\u224f\u2250\u2251\u2252\u2253\u2254\u2255\u2256\u2257\u2258\u2259\u225a\u225b\u225c\u225d\u225e\u225f\u2260\u2261\u2262\u2263\u2264\u2265\u2266\u2267\u2268\u2269\u226a\u226b\u226c\u226d\u226e\u226f\u2270\u2271\u2272\u2273\u2274\u2275\u2276\u2277\u2278\u2279\u227a\u227b\u227c\u227d\u227e\u227f\u2280\u2281\u2282\u2283\u2284\u2285\u2286\u2287\u2288\u2289\u228a\u228b\u228c\u228d\u228e\u228f\u2290\u2291\u2292\u2293\u2294\u2295\u2296\u2297\u2298\u2299\u229a\u229b\u229c\u229d\u229e\u229f\u22a0\u22a1\u22a2\u22a3\u22a4\u22a5\u22a6\u22a7\u22a8\u22a9\u22aa\u22ab\u22ac\u22ad\u22ae\u22af\u22b0\u22b1\u22b2\u22b3\u22b4\u22b5\u22b6\u22b7\u22b8\u22b9\u22ba\u22bb\u22bc\u22bd\u22be\u22bf\u22c0\u22c1\u22c2\u22c3\u22c4\u22c5\u22c6\u22c7\u22c8\u22c9\u22ca\u22cb\u22cc\u22cd\u22ce\u22cf\u22d0\u22d1\u22d2\u22d3\u22d4\u22d5\u22d6\u22d7\u22d8\u22d9\u22da\u22db\u22dc\u22dd\u22de\u22df\u22e0\u22e1\u22e2\u22e3\u22e4\u22e5\u22e6\u22e7\u22e8\u22e9\u22ea\u22eb\u22ec\u22ed\u22ee\u22ef\u22f0\u22f1\u22f2\u22f3\u22f4\u22f5\u22f6\u22f7\u22f8\u22f9\u22fa\u22fb\u22fc\u22fd\u22fe\u22ff\u2308\u2309\u230a\u230b\u2320\u2321\u237c\u239b\u239c\u239d\u239e\u239f\u23a0\u23a1\u23a2\u23a3\u23a4\u23a5\u23a6\u23a7\u23a8\u23a9\u23aa\u23ab\u23ac\u23ad\u23ae\u23af\u23b0\u23b1\u23b2\u23b3\u25b7\u25c1\u25f8\u25f9\u25fa\u25fb\u25fc\u25fd\u25fe\u25ff\u266f\u27c0\u27c1\u27c2\u27c3\u27c4\u27d0\u27d1\u27d2\u27d3\u27d4\u27d5\u27d6\u27d7\u27d8\u27d9\u27da\u27db\u27dc\u27dd\u27de\u27df\u27e0\u27e1\u27e2\u27e3\u27e4\u27e5\u27f0\u27f1\u27f2\u27f3\u27f4\u27f5\u27f6\u27f7\u27f8\u27f9\u27fa\u27fb\u27fc\u27fd\u27fe\u27ff\u2900\u2901\u2902\u2903\u2904\u2905\u2906\u2907\u2908\u2909\u290a\u290b\u290c\u290d\u290e\u290f\u2910\u2911\u2912\u2913\u2914\u2915\u2916\u2917\u2918\u2919\u291a\u291b\u291c\u291d\u291e\u291f\u2920\u2921\u2922\u2923\u2924\u2925\u2926\u2927\u2928\u2929\u292a\u292b\u292c\u292d\u292e\u292f\u2930\u2931\u2932\u2933\u2934\u2935\u2936\u2937\u2938\u2939\u293a\u293b\u293c\u293d\u293e\u293f\u2940\u2941\u2942\u2943\u2944\u2945\u2946\u2947\u2948\u2949\u294a\u294b\u294c\u294d\u294e\u294f\u2950\u2951\u2952\u2953\u2954\u2955\u2956\u2957\u2958\u2959\u295a\u295b\u295c\u295d\u295e\u295f\u2960\u2961\u2962\u2963\u2964\u2965\u2966\u2967\u2968\u2969\u296a\u296b\u296c\u296d\u296e\u296f\u2970\u2971\u2972\u2973\u2974\u2975\u2976\u2977\u2978\u2979\u297a\u297b\u297c\u297d\u297e\u297f\u2980\u2981\u2982\u2999\u299a\u299b\u299c\u299d\u299e\u299f\u29a0\u29a1\u29a2\u29a3\u29a4\u29a5\u29a6\u29a7\u29a8\u29a9\u29aa\u29ab\u29ac\u29ad\u29ae\u29af\u29b0\u29b1\u29b2\u29b3\u29b4\u29b5\u29b6\u29b7\u29b8\u29b9\u29ba\u29bb\u29bc\u29bd\u29be\u29bf\u29c0\u29c1\u29c2\u29c3\u29c4\u29c5\u29c6\u29c7\u29c8\u29c9\u29ca\u29cb\u29cc\u29cd\u29ce\u29cf\u29d0\u29d1\u29d2\u29d3\u29d4\u29d5\u29d6\u29d7\u29dc\u29dd\u29de\u29df\u29e0\u29e1\u29e2\u29e3\u29e4\u29e5\u29e6\u29e7\u29e8\u29e9\u29ea\u29eb\u29ec\u29ed\u29ee\u29ef\u29f0\u29f1\u29f2\u29f3\u29f4\u29f5\u29f6\u29f7\u29f8\u29f9\u29fa\u29fb\u29fe\u29ff\u2a00\u2a01\u2a02\u2a03\u2a04\u2a05\u2a06\u2a07\u2a08\u2a09\u2a0a\u2a0b\u2a0c\u2a0d\u2a0e\u2a0f\u2a10\u2a11\u2a12\u2a13\u2a14\u2a15\u2a16\u2a17\u2a18\u2a19\u2a1a\u2a1b\u2a1c\u2a1d\u2a1e\u2a1f\u2a20\u2a21\u2a22\u2a23\u2a24\u2a25\u2a26\u2a27\u2a28\u2a29\u2a2a\u2a2b\u2a2c\u2a2d\u2a2e\u2a2f\u2a30\u2a31\u2a32\u2a33\u2a34\u2a35\u2a36\u2a37\u2a38\u2a39\u2a3a\u2a3b\u2a3c\u2a3d\u2a3e\u2a3f\u2a40\u2a41\u2a42\u2a43\u2a44\u2a45\u2a46\u2a47\u2a48\u2a49\u2a4a\u2a4b\u2a4c\u2a4d\u2a4e\u2a4f\u2a50\u2a51\u2a52\u2a53\u2a54\u2a55\u2a56\u2a57\u2a58\u2a59\u2a5a\u2a5b\u2a5c\u2a5d\u2a5e\u2a5f\u2a60\u2a61\u2a62\u2a63\u2a64\u2a65\u2a66\u2a67\u2a68\u2a69\u2a6a\u2a6b\u2a6c\u2a6d\u2a6e\u2a6f\u2a70\u2a71\u2a72\u2a73\u2a74\u2a75\u2a76\u2a77\u2a78\u2a79\u2a7a\u2a7b\u2a7c\u2a7d\u2a7e\u2a7f\u2a80\u2a81\u2a82\u2a83\u2a84\u2a85\u2a86\u2a87\u2a88\u2a89\u2a8a\u2a8b\u2a8c\u2a8d\u2a8e\u2a8f\u2a90\u2a91\u2a92\u2a93\u2a94\u2a95\u2a96\u2a97\u2a98\u2a99\u2a9a\u2a9b\u2a9c\u2a9d\u2a9e\u2a9f\u2aa0\u2aa1\u2aa2\u2aa3\u2aa4\u2aa5\u2aa6\u2aa7\u2aa8\u2aa9\u2aaa\u2aab\u2aac\u2aad\u2aae\u2aaf\u2ab0\u2ab1\u2ab2\u2ab3\u2ab4\u2ab5\u2ab6\u2ab7\u2ab8\u2ab9\u2aba\u2abb\u2abc\u2abd\u2abe\u2abf\u2ac0\u2ac1\u2ac2\u2ac3\u2ac4\u2ac5\u2ac6\u2ac7\u2ac8\u2ac9\u2aca\u2acb\u2acc\u2acd\u2ace\u2acf\u2ad0\u2ad1\u2ad2\u2ad3\u2ad4\u2ad5\u2ad6\u2ad7\u2ad8\u2ad9\u2ada\u2adb\u2adc\u2add\u2ade\u2adf\u2ae0\u2ae1\u2ae2\u2ae3\u2ae4\u2ae5\u2ae6\u2ae7\u2ae8\u2ae9\u2aea\u2aeb\u2aec\u2aed\u2aee\u2aef\u2af0\u2af1\u2af2\u2af3\u2af4\u2af5\u2af6\u2af7\u2af8\u2af9\u2afa\u2afb\u2afc\u2afd\u2afe\u2aff\ufb29\ufe62\ufe64\ufe65\ufe66\uff0b\uff1c\uff1d\uff1e\uff5c\uff5e\uffe2\uffe9\uffea\uffeb\uffec' + +So = u'\xa6\xa7\xa9\xae\xb0\xb6\u0482\u060e\u060f\u06e9\u06fd\u06fe\u09fa\u0b70\u0bf3\u0bf4\u0bf5\u0bf6\u0bf7\u0bf8\u0bfa\u0f01\u0f02\u0f03\u0f13\u0f14\u0f15\u0f16\u0f17\u0f1a\u0f1b\u0f1c\u0f1d\u0f1e\u0f1f\u0f34\u0f36\u0f38\u0fbe\u0fbf\u0fc0\u0fc1\u0fc2\u0fc3\u0fc4\u0fc5\u0fc7\u0fc8\u0fc9\u0fca\u0fcb\u0fcc\u0fcf\u1360\u1390\u1391\u1392\u1393\u1394\u1395\u1396\u1397\u1398\u1399\u1940\u19e0\u19e1\u19e2\u19e3\u19e4\u19e5\u19e6\u19e7\u19e8\u19e9\u19ea\u19eb\u19ec\u19ed\u19ee\u19ef\u19f0\u19f1\u19f2\u19f3\u19f4\u19f5\u19f6\u19f7\u19f8\u19f9\u19fa\u19fb\u19fc\u19fd\u19fe\u19ff\u2100\u2101\u2103\u2104\u2105\u2106\u2108\u2109\u2114\u2116\u2117\u2118\u211e\u211f\u2120\u2121\u2122\u2123\u2125\u2127\u2129\u212e\u2132\u213a\u213b\u214a\u214c\u2195\u2196\u2197\u2198\u2199\u219c\u219d\u219e\u219f\u21a1\u21a2\u21a4\u21a5\u21a7\u21a8\u21a9\u21aa\u21ab\u21ac\u21ad\u21af\u21b0\u21b1\u21b2\u21b3\u21b4\u21b5\u21b6\u21b7\u21b8\u21b9\u21ba\u21bb\u21bc\u21bd\u21be\u21bf\u21c0\u21c1\u21c2\u21c3\u21c4\u21c5\u21c6\u21c7\u21c8\u21c9\u21ca\u21cb\u21cc\u21cd\u21d0\u21d1\u21d3\u21d5\u21d6\u21d7\u21d8\u21d9\u21da\u21db\u21dc\u21dd\u21de\u21df\u21e0\u21e1\u21e2\u21e3\u21e4\u21e5\u21e6\u21e7\u21e8\u21e9\u21ea\u21eb\u21ec\u21ed\u21ee\u21ef\u21f0\u21f1\u21f2\u21f3\u2300\u2301\u2302\u2303\u2304\u2305\u2306\u2307\u230c\u230d\u230e\u230f\u2310\u2311\u2312\u2313\u2314\u2315\u2316\u2317\u2318\u2319\u231a\u231b\u231c\u231d\u231e\u231f\u2322\u2323\u2324\u2325\u2326\u2327\u2328\u232b\u232c\u232d\u232e\u232f\u2330\u2331\u2332\u2333\u2334\u2335\u2336\u2337\u2338\u2339\u233a\u233b\u233c\u233d\u233e\u233f\u2340\u2341\u2342\u2343\u2344\u2345\u2346\u2347\u2348\u2349\u234a\u234b\u234c\u234d\u234e\u234f\u2350\u2351\u2352\u2353\u2354\u2355\u2356\u2357\u2358\u2359\u235a\u235b\u235c\u235d\u235e\u235f\u2360\u2361\u2362\u2363\u2364\u2365\u2366\u2367\u2368\u2369\u236a\u236b\u236c\u236d\u236e\u236f\u2370\u2371\u2372\u2373\u2374\u2375\u2376\u2377\u2378\u2379\u237a\u237b\u237d\u237e\u237f\u2380\u2381\u2382\u2383\u2384\u2385\u2386\u2387\u2388\u2389\u238a\u238b\u238c\u238d\u238e\u238f\u2390\u2391\u2392\u2393\u2394\u2395\u2396\u2397\u2398\u2399\u239a\u23b7\u23b8\u23b9\u23ba\u23bb\u23bc\u23bd\u23be\u23bf\u23c0\u23c1\u23c2\u23c3\u23c4\u23c5\u23c6\u23c7\u23c8\u23c9\u23ca\u23cb\u23cc\u23cd\u23ce\u23cf\u23d0\u23d1\u23d2\u23d3\u23d4\u23d5\u23d6\u23d7\u23d8\u23d9\u23da\u23db\u2400\u2401\u2402\u2403\u2404\u2405\u2406\u2407\u2408\u2409\u240a\u240b\u240c\u240d\u240e\u240f\u2410\u2411\u2412\u2413\u2414\u2415\u2416\u2417\u2418\u2419\u241a\u241b\u241c\u241d\u241e\u241f\u2420\u2421\u2422\u2423\u2424\u2425\u2426\u2440\u2441\u2442\u2443\u2444\u2445\u2446\u2447\u2448\u2449\u244a\u249c\u249d\u249e\u249f\u24a0\u24a1\u24a2\u24a3\u24a4\u24a5\u24a6\u24a7\u24a8\u24a9\u24aa\u24ab\u24ac\u24ad\u24ae\u24af\u24b0\u24b1\u24b2\u24b3\u24b4\u24b5\u24b6\u24b7\u24b8\u24b9\u24ba\u24bb\u24bc\u24bd\u24be\u24bf\u24c0\u24c1\u24c2\u24c3\u24c4\u24c5\u24c6\u24c7\u24c8\u24c9\u24ca\u24cb\u24cc\u24cd\u24ce\u24cf\u24d0\u24d1\u24d2\u24d3\u24d4\u24d5\u24d6\u24d7\u24d8\u24d9\u24da\u24db\u24dc\u24dd\u24de\u24df\u24e0\u24e1\u24e2\u24e3\u24e4\u24e5\u24e6\u24e7\u24e8\u24e9\u2500\u2501\u2502\u2503\u2504\u2505\u2506\u2507\u2508\u2509\u250a\u250b\u250c\u250d\u250e\u250f\u2510\u2511\u2512\u2513\u2514\u2515\u2516\u2517\u2518\u2519\u251a\u251b\u251c\u251d\u251e\u251f\u2520\u2521\u2522\u2523\u2524\u2525\u2526\u2527\u2528\u2529\u252a\u252b\u252c\u252d\u252e\u252f\u2530\u2531\u2532\u2533\u2534\u2535\u2536\u2537\u2538\u2539\u253a\u253b\u253c\u253d\u253e\u253f\u2540\u2541\u2542\u2543\u2544\u2545\u2546\u2547\u2548\u2549\u254a\u254b\u254c\u254d\u254e\u254f\u2550\u2551\u2552\u2553\u2554\u2555\u2556\u2557\u2558\u2559\u255a\u255b\u255c\u255d\u255e\u255f\u2560\u2561\u2562\u2563\u2564\u2565\u2566\u2567\u2568\u2569\u256a\u256b\u256c\u256d\u256e\u256f\u2570\u2571\u2572\u2573\u2574\u2575\u2576\u2577\u2578\u2579\u257a\u257b\u257c\u257d\u257e\u257f\u2580\u2581\u2582\u2583\u2584\u2585\u2586\u2587\u2588\u2589\u258a\u258b\u258c\u258d\u258e\u258f\u2590\u2591\u2592\u2593\u2594\u2595\u2596\u2597\u2598\u2599\u259a\u259b\u259c\u259d\u259e\u259f\u25a0\u25a1\u25a2\u25a3\u25a4\u25a5\u25a6\u25a7\u25a8\u25a9\u25aa\u25ab\u25ac\u25ad\u25ae\u25af\u25b0\u25b1\u25b2\u25b3\u25b4\u25b5\u25b6\u25b8\u25b9\u25ba\u25bb\u25bc\u25bd\u25be\u25bf\u25c0\u25c2\u25c3\u25c4\u25c5\u25c6\u25c7\u25c8\u25c9\u25ca\u25cb\u25cc\u25cd\u25ce\u25cf\u25d0\u25d1\u25d2\u25d3\u25d4\u25d5\u25d6\u25d7\u25d8\u25d9\u25da\u25db\u25dc\u25dd\u25de\u25df\u25e0\u25e1\u25e2\u25e3\u25e4\u25e5\u25e6\u25e7\u25e8\u25e9\u25ea\u25eb\u25ec\u25ed\u25ee\u25ef\u25f0\u25f1\u25f2\u25f3\u25f4\u25f5\u25f6\u25f7\u2600\u2601\u2602\u2603\u2604\u2605\u2606\u2607\u2608\u2609\u260a\u260b\u260c\u260d\u260e\u260f\u2610\u2611\u2612\u2613\u2614\u2615\u2616\u2617\u2618\u2619\u261a\u261b\u261c\u261d\u261e\u261f\u2620\u2621\u2622\u2623\u2624\u2625\u2626\u2627\u2628\u2629\u262a\u262b\u262c\u262d\u262e\u262f\u2630\u2631\u2632\u2633\u2634\u2635\u2636\u2637\u2638\u2639\u263a\u263b\u263c\u263d\u263e\u263f\u2640\u2641\u2642\u2643\u2644\u2645\u2646\u2647\u2648\u2649\u264a\u264b\u264c\u264d\u264e\u264f\u2650\u2651\u2652\u2653\u2654\u2655\u2656\u2657\u2658\u2659\u265a\u265b\u265c\u265d\u265e\u265f\u2660\u2661\u2662\u2663\u2664\u2665\u2666\u2667\u2668\u2669\u266a\u266b\u266c\u266d\u266e\u2670\u2671\u2672\u2673\u2674\u2675\u2676\u2677\u2678\u2679\u267a\u267b\u267c\u267d\u267e\u267f\u2680\u2681\u2682\u2683\u2684\u2685\u2686\u2687\u2688\u2689\u268a\u268b\u268c\u268d\u268e\u268f\u2690\u2691\u2692\u2693\u2694\u2695\u2696\u2697\u2698\u2699\u269a\u269b\u269c\u26a0\u26a1\u26a2\u26a3\u26a4\u26a5\u26a6\u26a7\u26a8\u26a9\u26aa\u26ab\u26ac\u26ad\u26ae\u26af\u26b0\u26b1\u2701\u2702\u2703\u2704\u2706\u2707\u2708\u2709\u270c\u270d\u270e\u270f\u2710\u2711\u2712\u2713\u2714\u2715\u2716\u2717\u2718\u2719\u271a\u271b\u271c\u271d\u271e\u271f\u2720\u2721\u2722\u2723\u2724\u2725\u2726\u2727\u2729\u272a\u272b\u272c\u272d\u272e\u272f\u2730\u2731\u2732\u2733\u2734\u2735\u2736\u2737\u2738\u2739\u273a\u273b\u273c\u273d\u273e\u273f\u2740\u2741\u2742\u2743\u2744\u2745\u2746\u2747\u2748\u2749\u274a\u274b\u274d\u274f\u2750\u2751\u2752\u2756\u2758\u2759\u275a\u275b\u275c\u275d\u275e\u2761\u2762\u2763\u2764\u2765\u2766\u2767\u2794\u2798\u2799\u279a\u279b\u279c\u279d\u279e\u279f\u27a0\u27a1\u27a2\u27a3\u27a4\u27a5\u27a6\u27a7\u27a8\u27a9\u27aa\u27ab\u27ac\u27ad\u27ae\u27af\u27b1\u27b2\u27b3\u27b4\u27b5\u27b6\u27b7\u27b8\u27b9\u27ba\u27bb\u27bc\u27bd\u27be\u2800\u2801\u2802\u2803\u2804\u2805\u2806\u2807\u2808\u2809\u280a\u280b\u280c\u280d\u280e\u280f\u2810\u2811\u2812\u2813\u2814\u2815\u2816\u2817\u2818\u2819\u281a\u281b\u281c\u281d\u281e\u281f\u2820\u2821\u2822\u2823\u2824\u2825\u2826\u2827\u2828\u2829\u282a\u282b\u282c\u282d\u282e\u282f\u2830\u2831\u2832\u2833\u2834\u2835\u2836\u2837\u2838\u2839\u283a\u283b\u283c\u283d\u283e\u283f\u2840\u2841\u2842\u2843\u2844\u2845\u2846\u2847\u2848\u2849\u284a\u284b\u284c\u284d\u284e\u284f\u2850\u2851\u2852\u2853\u2854\u2855\u2856\u2857\u2858\u2859\u285a\u285b\u285c\u285d\u285e\u285f\u2860\u2861\u2862\u2863\u2864\u2865\u2866\u2867\u2868\u2869\u286a\u286b\u286c\u286d\u286e\u286f\u2870\u2871\u2872\u2873\u2874\u2875\u2876\u2877\u2878\u2879\u287a\u287b\u287c\u287d\u287e\u287f\u2880\u2881\u2882\u2883\u2884\u2885\u2886\u2887\u2888\u2889\u288a\u288b\u288c\u288d\u288e\u288f\u2890\u2891\u2892\u2893\u2894\u2895\u2896\u2897\u2898\u2899\u289a\u289b\u289c\u289d\u289e\u289f\u28a0\u28a1\u28a2\u28a3\u28a4\u28a5\u28a6\u28a7\u28a8\u28a9\u28aa\u28ab\u28ac\u28ad\u28ae\u28af\u28b0\u28b1\u28b2\u28b3\u28b4\u28b5\u28b6\u28b7\u28b8\u28b9\u28ba\u28bb\u28bc\u28bd\u28be\u28bf\u28c0\u28c1\u28c2\u28c3\u28c4\u28c5\u28c6\u28c7\u28c8\u28c9\u28ca\u28cb\u28cc\u28cd\u28ce\u28cf\u28d0\u28d1\u28d2\u28d3\u28d4\u28d5\u28d6\u28d7\u28d8\u28d9\u28da\u28db\u28dc\u28dd\u28de\u28df\u28e0\u28e1\u28e2\u28e3\u28e4\u28e5\u28e6\u28e7\u28e8\u28e9\u28ea\u28eb\u28ec\u28ed\u28ee\u28ef\u28f0\u28f1\u28f2\u28f3\u28f4\u28f5\u28f6\u28f7\u28f8\u28f9\u28fa\u28fb\u28fc\u28fd\u28fe\u28ff\u2b00\u2b01\u2b02\u2b03\u2b04\u2b05\u2b06\u2b07\u2b08\u2b09\u2b0a\u2b0b\u2b0c\u2b0d\u2b0e\u2b0f\u2b10\u2b11\u2b12\u2b13\u2ce5\u2ce6\u2ce7\u2ce8\u2ce9\u2cea\u2e80\u2e81\u2e82\u2e83\u2e84\u2e85\u2e86\u2e87\u2e88\u2e89\u2e8a\u2e8b\u2e8c\u2e8d\u2e8e\u2e8f\u2e90\u2e91\u2e92\u2e93\u2e94\u2e95\u2e96\u2e97\u2e98\u2e99\u2e9b\u2e9c\u2e9d\u2e9e\u2e9f\u2ea0\u2ea1\u2ea2\u2ea3\u2ea4\u2ea5\u2ea6\u2ea7\u2ea8\u2ea9\u2eaa\u2eab\u2eac\u2ead\u2eae\u2eaf\u2eb0\u2eb1\u2eb2\u2eb3\u2eb4\u2eb5\u2eb6\u2eb7\u2eb8\u2eb9\u2eba\u2ebb\u2ebc\u2ebd\u2ebe\u2ebf\u2ec0\u2ec1\u2ec2\u2ec3\u2ec4\u2ec5\u2ec6\u2ec7\u2ec8\u2ec9\u2eca\u2ecb\u2ecc\u2ecd\u2ece\u2ecf\u2ed0\u2ed1\u2ed2\u2ed3\u2ed4\u2ed5\u2ed6\u2ed7\u2ed8\u2ed9\u2eda\u2edb\u2edc\u2edd\u2ede\u2edf\u2ee0\u2ee1\u2ee2\u2ee3\u2ee4\u2ee5\u2ee6\u2ee7\u2ee8\u2ee9\u2eea\u2eeb\u2eec\u2eed\u2eee\u2eef\u2ef0\u2ef1\u2ef2\u2ef3\u2f00\u2f01\u2f02\u2f03\u2f04\u2f05\u2f06\u2f07\u2f08\u2f09\u2f0a\u2f0b\u2f0c\u2f0d\u2f0e\u2f0f\u2f10\u2f11\u2f12\u2f13\u2f14\u2f15\u2f16\u2f17\u2f18\u2f19\u2f1a\u2f1b\u2f1c\u2f1d\u2f1e\u2f1f\u2f20\u2f21\u2f22\u2f23\u2f24\u2f25\u2f26\u2f27\u2f28\u2f29\u2f2a\u2f2b\u2f2c\u2f2d\u2f2e\u2f2f\u2f30\u2f31\u2f32\u2f33\u2f34\u2f35\u2f36\u2f37\u2f38\u2f39\u2f3a\u2f3b\u2f3c\u2f3d\u2f3e\u2f3f\u2f40\u2f41\u2f42\u2f43\u2f44\u2f45\u2f46\u2f47\u2f48\u2f49\u2f4a\u2f4b\u2f4c\u2f4d\u2f4e\u2f4f\u2f50\u2f51\u2f52\u2f53\u2f54\u2f55\u2f56\u2f57\u2f58\u2f59\u2f5a\u2f5b\u2f5c\u2f5d\u2f5e\u2f5f\u2f60\u2f61\u2f62\u2f63\u2f64\u2f65\u2f66\u2f67\u2f68\u2f69\u2f6a\u2f6b\u2f6c\u2f6d\u2f6e\u2f6f\u2f70\u2f71\u2f72\u2f73\u2f74\u2f75\u2f76\u2f77\u2f78\u2f79\u2f7a\u2f7b\u2f7c\u2f7d\u2f7e\u2f7f\u2f80\u2f81\u2f82\u2f83\u2f84\u2f85\u2f86\u2f87\u2f88\u2f89\u2f8a\u2f8b\u2f8c\u2f8d\u2f8e\u2f8f\u2f90\u2f91\u2f92\u2f93\u2f94\u2f95\u2f96\u2f97\u2f98\u2f99\u2f9a\u2f9b\u2f9c\u2f9d\u2f9e\u2f9f\u2fa0\u2fa1\u2fa2\u2fa3\u2fa4\u2fa5\u2fa6\u2fa7\u2fa8\u2fa9\u2faa\u2fab\u2fac\u2fad\u2fae\u2faf\u2fb0\u2fb1\u2fb2\u2fb3\u2fb4\u2fb5\u2fb6\u2fb7\u2fb8\u2fb9\u2fba\u2fbb\u2fbc\u2fbd\u2fbe\u2fbf\u2fc0\u2fc1\u2fc2\u2fc3\u2fc4\u2fc5\u2fc6\u2fc7\u2fc8\u2fc9\u2fca\u2fcb\u2fcc\u2fcd\u2fce\u2fcf\u2fd0\u2fd1\u2fd2\u2fd3\u2fd4\u2fd5\u2ff0\u2ff1\u2ff2\u2ff3\u2ff4\u2ff5\u2ff6\u2ff7\u2ff8\u2ff9\u2ffa\u2ffb\u3004\u3012\u3013\u3020\u3036\u3037\u303e\u303f\u3190\u3191\u3196\u3197\u3198\u3199\u319a\u319b\u319c\u319d\u319e\u319f\u31c0\u31c1\u31c2\u31c3\u31c4\u31c5\u31c6\u31c7\u31c8\u31c9\u31ca\u31cb\u31cc\u31cd\u31ce\u31cf\u3200\u3201\u3202\u3203\u3204\u3205\u3206\u3207\u3208\u3209\u320a\u320b\u320c\u320d\u320e\u320f\u3210\u3211\u3212\u3213\u3214\u3215\u3216\u3217\u3218\u3219\u321a\u321b\u321c\u321d\u321e\u322a\u322b\u322c\u322d\u322e\u322f\u3230\u3231\u3232\u3233\u3234\u3235\u3236\u3237\u3238\u3239\u323a\u323b\u323c\u323d\u323e\u323f\u3240\u3241\u3242\u3243\u3250\u3260\u3261\u3262\u3263\u3264\u3265\u3266\u3267\u3268\u3269\u326a\u326b\u326c\u326d\u326e\u326f\u3270\u3271\u3272\u3273\u3274\u3275\u3276\u3277\u3278\u3279\u327a\u327b\u327c\u327d\u327e\u327f\u328a\u328b\u328c\u328d\u328e\u328f\u3290\u3291\u3292\u3293\u3294\u3295\u3296\u3297\u3298\u3299\u329a\u329b\u329c\u329d\u329e\u329f\u32a0\u32a1\u32a2\u32a3\u32a4\u32a5\u32a6\u32a7\u32a8\u32a9\u32aa\u32ab\u32ac\u32ad\u32ae\u32af\u32b0\u32c0\u32c1\u32c2\u32c3\u32c4\u32c5\u32c6\u32c7\u32c8\u32c9\u32ca\u32cb\u32cc\u32cd\u32ce\u32cf\u32d0\u32d1\u32d2\u32d3\u32d4\u32d5\u32d6\u32d7\u32d8\u32d9\u32da\u32db\u32dc\u32dd\u32de\u32df\u32e0\u32e1\u32e2\u32e3\u32e4\u32e5\u32e6\u32e7\u32e8\u32e9\u32ea\u32eb\u32ec\u32ed\u32ee\u32ef\u32f0\u32f1\u32f2\u32f3\u32f4\u32f5\u32f6\u32f7\u32f8\u32f9\u32fa\u32fb\u32fc\u32fd\u32fe\u3300\u3301\u3302\u3303\u3304\u3305\u3306\u3307\u3308\u3309\u330a\u330b\u330c\u330d\u330e\u330f\u3310\u3311\u3312\u3313\u3314\u3315\u3316\u3317\u3318\u3319\u331a\u331b\u331c\u331d\u331e\u331f\u3320\u3321\u3322\u3323\u3324\u3325\u3326\u3327\u3328\u3329\u332a\u332b\u332c\u332d\u332e\u332f\u3330\u3331\u3332\u3333\u3334\u3335\u3336\u3337\u3338\u3339\u333a\u333b\u333c\u333d\u333e\u333f\u3340\u3341\u3342\u3343\u3344\u3345\u3346\u3347\u3348\u3349\u334a\u334b\u334c\u334d\u334e\u334f\u3350\u3351\u3352\u3353\u3354\u3355\u3356\u3357\u3358\u3359\u335a\u335b\u335c\u335d\u335e\u335f\u3360\u3361\u3362\u3363\u3364\u3365\u3366\u3367\u3368\u3369\u336a\u336b\u336c\u336d\u336e\u336f\u3370\u3371\u3372\u3373\u3374\u3375\u3376\u3377\u3378\u3379\u337a\u337b\u337c\u337d\u337e\u337f\u3380\u3381\u3382\u3383\u3384\u3385\u3386\u3387\u3388\u3389\u338a\u338b\u338c\u338d\u338e\u338f\u3390\u3391\u3392\u3393\u3394\u3395\u3396\u3397\u3398\u3399\u339a\u339b\u339c\u339d\u339e\u339f\u33a0\u33a1\u33a2\u33a3\u33a4\u33a5\u33a6\u33a7\u33a8\u33a9\u33aa\u33ab\u33ac\u33ad\u33ae\u33af\u33b0\u33b1\u33b2\u33b3\u33b4\u33b5\u33b6\u33b7\u33b8\u33b9\u33ba\u33bb\u33bc\u33bd\u33be\u33bf\u33c0\u33c1\u33c2\u33c3\u33c4\u33c5\u33c6\u33c7\u33c8\u33c9\u33ca\u33cb\u33cc\u33cd\u33ce\u33cf\u33d0\u33d1\u33d2\u33d3\u33d4\u33d5\u33d6\u33d7\u33d8\u33d9\u33da\u33db\u33dc\u33dd\u33de\u33df\u33e0\u33e1\u33e2\u33e3\u33e4\u33e5\u33e6\u33e7\u33e8\u33e9\u33ea\u33eb\u33ec\u33ed\u33ee\u33ef\u33f0\u33f1\u33f2\u33f3\u33f4\u33f5\u33f6\u33f7\u33f8\u33f9\u33fa\u33fb\u33fc\u33fd\u33fe\u33ff\u4dc0\u4dc1\u4dc2\u4dc3\u4dc4\u4dc5\u4dc6\u4dc7\u4dc8\u4dc9\u4dca\u4dcb\u4dcc\u4dcd\u4dce\u4dcf\u4dd0\u4dd1\u4dd2\u4dd3\u4dd4\u4dd5\u4dd6\u4dd7\u4dd8\u4dd9\u4dda\u4ddb\u4ddc\u4ddd\u4dde\u4ddf\u4de0\u4de1\u4de2\u4de3\u4de4\u4de5\u4de6\u4de7\u4de8\u4de9\u4dea\u4deb\u4dec\u4ded\u4dee\u4def\u4df0\u4df1\u4df2\u4df3\u4df4\u4df5\u4df6\u4df7\u4df8\u4df9\u4dfa\u4dfb\u4dfc\u4dfd\u4dfe\u4dff\ua490\ua491\ua492\ua493\ua494\ua495\ua496\ua497\ua498\ua499\ua49a\ua49b\ua49c\ua49d\ua49e\ua49f\ua4a0\ua4a1\ua4a2\ua4a3\ua4a4\ua4a5\ua4a6\ua4a7\ua4a8\ua4a9\ua4aa\ua4ab\ua4ac\ua4ad\ua4ae\ua4af\ua4b0\ua4b1\ua4b2\ua4b3\ua4b4\ua4b5\ua4b6\ua4b7\ua4b8\ua4b9\ua4ba\ua4bb\ua4bc\ua4bd\ua4be\ua4bf\ua4c0\ua4c1\ua4c2\ua4c3\ua4c4\ua4c5\ua4c6\ua828\ua829\ua82a\ua82b\ufdfd\uffe4\uffe8\uffed\uffee\ufffc\ufffd' + +Zl = u'\u2028' + +Zp = u'\u2029' + +Zs = u' \xa0\u1680\u180e\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u202f\u205f\u3000' + +cats = ['Cc', 'Cf', 'Cn', 'Co', 'Cs', 'Ll', 'Lm', 'Lo', 'Lt', 'Lu', 'Mc', 'Me', 'Mn', 'Nd', 'Nl', 'No', 'Pc', 'Pd', 'Pe', 'Pf', 'Pi', 'Po', 'Ps', 'Sc', 'Sk', 'Sm', 'So', 'Zl', 'Zp', 'Zs'] + +def combine(*args): + return u''.join([globals()[cat] for cat in args]) + +xid_start = u'\u0041-\u005A\u005F\u0061-\u007A\u00AA\u00B5\u00BA\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u01BA\u01BB\u01BC-\u01BF\u01C0-\u01C3\u01C4-\u0241\u0250-\u02AF\u02B0-\u02C1\u02C6-\u02D1\u02E0-\u02E4\u02EE\u0386\u0388-\u038A\u038C\u038E-\u03A1\u03A3-\u03CE\u03D0-\u03F5\u03F7-\u0481\u048A-\u04CE\u04D0-\u04F9\u0500-\u050F\u0531-\u0556\u0559\u0561-\u0587\u05D0-\u05EA\u05F0-\u05F2\u0621-\u063A\u0640\u0641-\u064A\u066E-\u066F\u0671-\u06D3\u06D5\u06E5-\u06E6\u06EE-\u06EF\u06FA-\u06FC\u06FF\u0710\u0712-\u072F\u074D-\u076D\u0780-\u07A5\u07B1\u0904-\u0939\u093D\u0950\u0958-\u0961\u097D\u0985-\u098C\u098F-\u0990\u0993-\u09A8\u09AA-\u09B0\u09B2\u09B6-\u09B9\u09BD\u09CE\u09DC-\u09DD\u09DF-\u09E1\u09F0-\u09F1\u0A05-\u0A0A\u0A0F-\u0A10\u0A13-\u0A28\u0A2A-\u0A30\u0A32-\u0A33\u0A35-\u0A36\u0A38-\u0A39\u0A59-\u0A5C\u0A5E\u0A72-\u0A74\u0A85-\u0A8D\u0A8F-\u0A91\u0A93-\u0AA8\u0AAA-\u0AB0\u0AB2-\u0AB3\u0AB5-\u0AB9\u0ABD\u0AD0\u0AE0-\u0AE1\u0B05-\u0B0C\u0B0F-\u0B10\u0B13-\u0B28\u0B2A-\u0B30\u0B32-\u0B33\u0B35-\u0B39\u0B3D\u0B5C-\u0B5D\u0B5F-\u0B61\u0B71\u0B83\u0B85-\u0B8A\u0B8E-\u0B90\u0B92-\u0B95\u0B99-\u0B9A\u0B9C\u0B9E-\u0B9F\u0BA3-\u0BA4\u0BA8-\u0BAA\u0BAE-\u0BB9\u0C05-\u0C0C\u0C0E-\u0C10\u0C12-\u0C28\u0C2A-\u0C33\u0C35-\u0C39\u0C60-\u0C61\u0C85-\u0C8C\u0C8E-\u0C90\u0C92-\u0CA8\u0CAA-\u0CB3\u0CB5-\u0CB9\u0CBD\u0CDE\u0CE0-\u0CE1\u0D05-\u0D0C\u0D0E-\u0D10\u0D12-\u0D28\u0D2A-\u0D39\u0D60-\u0D61\u0D85-\u0D96\u0D9A-\u0DB1\u0DB3-\u0DBB\u0DBD\u0DC0-\u0DC6\u0E01-\u0E30\u0E32\u0E40-\u0E45\u0E46\u0E81-\u0E82\u0E84\u0E87-\u0E88\u0E8A\u0E8D\u0E94-\u0E97\u0E99-\u0E9F\u0EA1-\u0EA3\u0EA5\u0EA7\u0EAA-\u0EAB\u0EAD-\u0EB0\u0EB2\u0EBD\u0EC0-\u0EC4\u0EC6\u0EDC-\u0EDD\u0F00\u0F40-\u0F47\u0F49-\u0F6A\u0F88-\u0F8B\u1000-\u1021\u1023-\u1027\u1029-\u102A\u1050-\u1055\u10A0-\u10C5\u10D0-\u10FA\u10FC\u1100-\u1159\u115F-\u11A2\u11A8-\u11F9\u1200-\u1248\u124A-\u124D\u1250-\u1256\u1258\u125A-\u125D\u1260-\u1288\u128A-\u128D\u1290-\u12B0\u12B2-\u12B5\u12B8-\u12BE\u12C0\u12C2-\u12C5\u12C8-\u12D6\u12D8-\u1310\u1312-\u1315\u1318-\u135A\u1380-\u138F\u13A0-\u13F4\u1401-\u166C\u166F-\u1676\u1681-\u169A\u16A0-\u16EA\u16EE-\u16F0\u1700-\u170C\u170E-\u1711\u1720-\u1731\u1740-\u1751\u1760-\u176C\u176E-\u1770\u1780-\u17B3\u17D7\u17DC\u1820-\u1842\u1843\u1844-\u1877\u1880-\u18A8\u1900-\u191C\u1950-\u196D\u1970-\u1974\u1980-\u19A9\u19C1-\u19C7\u1A00-\u1A16\u1D00-\u1D2B\u1D2C-\u1D61\u1D62-\u1D77\u1D78\u1D79-\u1D9A\u1D9B-\u1DBF\u1E00-\u1E9B\u1EA0-\u1EF9\u1F00-\u1F15\u1F18-\u1F1D\u1F20-\u1F45\u1F48-\u1F4D\u1F50-\u1F57\u1F59\u1F5B\u1F5D\u1F5F-\u1F7D\u1F80-\u1FB4\u1FB6-\u1FBC\u1FBE\u1FC2-\u1FC4\u1FC6-\u1FCC\u1FD0-\u1FD3\u1FD6-\u1FDB\u1FE0-\u1FEC\u1FF2-\u1FF4\u1FF6-\u1FFC\u2071\u207F\u2090-\u2094\u2102\u2107\u210A-\u2113\u2115\u2118\u2119-\u211D\u2124\u2126\u2128\u212A-\u212D\u212E\u212F-\u2131\u2133-\u2134\u2135-\u2138\u2139\u213C-\u213F\u2145-\u2149\u2160-\u2183\u2C00-\u2C2E\u2C30-\u2C5E\u2C80-\u2CE4\u2D00-\u2D25\u2D30-\u2D65\u2D6F\u2D80-\u2D96\u2DA0-\u2DA6\u2DA8-\u2DAE\u2DB0-\u2DB6\u2DB8-\u2DBE\u2DC0-\u2DC6\u2DC8-\u2DCE\u2DD0-\u2DD6\u2DD8-\u2DDE\u3005\u3006\u3007\u3021-\u3029\u3031-\u3035\u3038-\u303A\u303B\u303C\u3041-\u3096\u309D-\u309E\u309F\u30A1-\u30FA\u30FC-\u30FE\u30FF\u3105-\u312C\u3131-\u318E\u31A0-\u31B7\u31F0-\u31FF\u3400-\u4DB5\u4E00-\u9FBB\uA000-\uA014\uA015\uA016-\uA48C\uA800-\uA801\uA803-\uA805\uA807-\uA80A\uA80C-\uA822\uAC00-\uD7A3\uF900-\uFA2D\uFA30-\uFA6A\uFA70-\uFAD9\uFB00-\uFB06\uFB13-\uFB17\uFB1D\uFB1F-\uFB28\uFB2A-\uFB36\uFB38-\uFB3C\uFB3E\uFB40-\uFB41\uFB43-\uFB44\uFB46-\uFBB1\uFBD3-\uFC5D\uFC64-\uFD3D\uFD50-\uFD8F\uFD92-\uFDC7\uFDF0-\uFDF9\uFE71\uFE73\uFE77\uFE79\uFE7B\uFE7D\uFE7F-\uFEFC\uFF21-\uFF3A\uFF41-\uFF5A\uFF66-\uFF6F\uFF70\uFF71-\uFF9D\uFFA0-\uFFBE\uFFC2-\uFFC7\uFFCA-\uFFCF\uFFD2-\uFFD7\uFFDA-\uFFDC' + +xid_continue = u'\u0030-\u0039\u0041-\u005A\u005F\u0061-\u007A\u00AA\u00B5\u00B7\u00BA\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u01BA\u01BB\u01BC-\u01BF\u01C0-\u01C3\u01C4-\u0241\u0250-\u02AF\u02B0-\u02C1\u02C6-\u02D1\u02E0-\u02E4\u02EE\u0300-\u036F\u0386\u0388-\u038A\u038C\u038E-\u03A1\u03A3-\u03CE\u03D0-\u03F5\u03F7-\u0481\u0483-\u0486\u048A-\u04CE\u04D0-\u04F9\u0500-\u050F\u0531-\u0556\u0559\u0561-\u0587\u0591-\u05B9\u05BB-\u05BD\u05BF\u05C1-\u05C2\u05C4-\u05C5\u05C7\u05D0-\u05EA\u05F0-\u05F2\u0610-\u0615\u0621-\u063A\u0640\u0641-\u064A\u064B-\u065E\u0660-\u0669\u066E-\u066F\u0670\u0671-\u06D3\u06D5\u06D6-\u06DC\u06DF-\u06E4\u06E5-\u06E6\u06E7-\u06E8\u06EA-\u06ED\u06EE-\u06EF\u06F0-\u06F9\u06FA-\u06FC\u06FF\u0710\u0711\u0712-\u072F\u0730-\u074A\u074D-\u076D\u0780-\u07A5\u07A6-\u07B0\u07B1\u0901-\u0902\u0903\u0904-\u0939\u093C\u093D\u093E-\u0940\u0941-\u0948\u0949-\u094C\u094D\u0950\u0951-\u0954\u0958-\u0961\u0962-\u0963\u0966-\u096F\u097D\u0981\u0982-\u0983\u0985-\u098C\u098F-\u0990\u0993-\u09A8\u09AA-\u09B0\u09B2\u09B6-\u09B9\u09BC\u09BD\u09BE-\u09C0\u09C1-\u09C4\u09C7-\u09C8\u09CB-\u09CC\u09CD\u09CE\u09D7\u09DC-\u09DD\u09DF-\u09E1\u09E2-\u09E3\u09E6-\u09EF\u09F0-\u09F1\u0A01-\u0A02\u0A03\u0A05-\u0A0A\u0A0F-\u0A10\u0A13-\u0A28\u0A2A-\u0A30\u0A32-\u0A33\u0A35-\u0A36\u0A38-\u0A39\u0A3C\u0A3E-\u0A40\u0A41-\u0A42\u0A47-\u0A48\u0A4B-\u0A4D\u0A59-\u0A5C\u0A5E\u0A66-\u0A6F\u0A70-\u0A71\u0A72-\u0A74\u0A81-\u0A82\u0A83\u0A85-\u0A8D\u0A8F-\u0A91\u0A93-\u0AA8\u0AAA-\u0AB0\u0AB2-\u0AB3\u0AB5-\u0AB9\u0ABC\u0ABD\u0ABE-\u0AC0\u0AC1-\u0AC5\u0AC7-\u0AC8\u0AC9\u0ACB-\u0ACC\u0ACD\u0AD0\u0AE0-\u0AE1\u0AE2-\u0AE3\u0AE6-\u0AEF\u0B01\u0B02-\u0B03\u0B05-\u0B0C\u0B0F-\u0B10\u0B13-\u0B28\u0B2A-\u0B30\u0B32-\u0B33\u0B35-\u0B39\u0B3C\u0B3D\u0B3E\u0B3F\u0B40\u0B41-\u0B43\u0B47-\u0B48\u0B4B-\u0B4C\u0B4D\u0B56\u0B57\u0B5C-\u0B5D\u0B5F-\u0B61\u0B66-\u0B6F\u0B71\u0B82\u0B83\u0B85-\u0B8A\u0B8E-\u0B90\u0B92-\u0B95\u0B99-\u0B9A\u0B9C\u0B9E-\u0B9F\u0BA3-\u0BA4\u0BA8-\u0BAA\u0BAE-\u0BB9\u0BBE-\u0BBF\u0BC0\u0BC1-\u0BC2\u0BC6-\u0BC8\u0BCA-\u0BCC\u0BCD\u0BD7\u0BE6-\u0BEF\u0C01-\u0C03\u0C05-\u0C0C\u0C0E-\u0C10\u0C12-\u0C28\u0C2A-\u0C33\u0C35-\u0C39\u0C3E-\u0C40\u0C41-\u0C44\u0C46-\u0C48\u0C4A-\u0C4D\u0C55-\u0C56\u0C60-\u0C61\u0C66-\u0C6F\u0C82-\u0C83\u0C85-\u0C8C\u0C8E-\u0C90\u0C92-\u0CA8\u0CAA-\u0CB3\u0CB5-\u0CB9\u0CBC\u0CBD\u0CBE\u0CBF\u0CC0-\u0CC4\u0CC6\u0CC7-\u0CC8\u0CCA-\u0CCB\u0CCC-\u0CCD\u0CD5-\u0CD6\u0CDE\u0CE0-\u0CE1\u0CE6-\u0CEF\u0D02-\u0D03\u0D05-\u0D0C\u0D0E-\u0D10\u0D12-\u0D28\u0D2A-\u0D39\u0D3E-\u0D40\u0D41-\u0D43\u0D46-\u0D48\u0D4A-\u0D4C\u0D4D\u0D57\u0D60-\u0D61\u0D66-\u0D6F\u0D82-\u0D83\u0D85-\u0D96\u0D9A-\u0DB1\u0DB3-\u0DBB\u0DBD\u0DC0-\u0DC6\u0DCA\u0DCF-\u0DD1\u0DD2-\u0DD4\u0DD6\u0DD8-\u0DDF\u0DF2-\u0DF3\u0E01-\u0E30\u0E31\u0E32-\u0E33\u0E34-\u0E3A\u0E40-\u0E45\u0E46\u0E47-\u0E4E\u0E50-\u0E59\u0E81-\u0E82\u0E84\u0E87-\u0E88\u0E8A\u0E8D\u0E94-\u0E97\u0E99-\u0E9F\u0EA1-\u0EA3\u0EA5\u0EA7\u0EAA-\u0EAB\u0EAD-\u0EB0\u0EB1\u0EB2-\u0EB3\u0EB4-\u0EB9\u0EBB-\u0EBC\u0EBD\u0EC0-\u0EC4\u0EC6\u0EC8-\u0ECD\u0ED0-\u0ED9\u0EDC-\u0EDD\u0F00\u0F18-\u0F19\u0F20-\u0F29\u0F35\u0F37\u0F39\u0F3E-\u0F3F\u0F40-\u0F47\u0F49-\u0F6A\u0F71-\u0F7E\u0F7F\u0F80-\u0F84\u0F86-\u0F87\u0F88-\u0F8B\u0F90-\u0F97\u0F99-\u0FBC\u0FC6\u1000-\u1021\u1023-\u1027\u1029-\u102A\u102C\u102D-\u1030\u1031\u1032\u1036-\u1037\u1038\u1039\u1040-\u1049\u1050-\u1055\u1056-\u1057\u1058-\u1059\u10A0-\u10C5\u10D0-\u10FA\u10FC\u1100-\u1159\u115F-\u11A2\u11A8-\u11F9\u1200-\u1248\u124A-\u124D\u1250-\u1256\u1258\u125A-\u125D\u1260-\u1288\u128A-\u128D\u1290-\u12B0\u12B2-\u12B5\u12B8-\u12BE\u12C0\u12C2-\u12C5\u12C8-\u12D6\u12D8-\u1310\u1312-\u1315\u1318-\u135A\u135F\u1369-\u1371\u1380-\u138F\u13A0-\u13F4\u1401-\u166C\u166F-\u1676\u1681-\u169A\u16A0-\u16EA\u16EE-\u16F0\u1700-\u170C\u170E-\u1711\u1712-\u1714\u1720-\u1731\u1732-\u1734\u1740-\u1751\u1752-\u1753\u1760-\u176C\u176E-\u1770\u1772-\u1773\u1780-\u17B3\u17B6\u17B7-\u17BD\u17BE-\u17C5\u17C6\u17C7-\u17C8\u17C9-\u17D3\u17D7\u17DC\u17DD\u17E0-\u17E9\u180B-\u180D\u1810-\u1819\u1820-\u1842\u1843\u1844-\u1877\u1880-\u18A8\u18A9\u1900-\u191C\u1920-\u1922\u1923-\u1926\u1927-\u1928\u1929-\u192B\u1930-\u1931\u1932\u1933-\u1938\u1939-\u193B\u1946-\u194F\u1950-\u196D\u1970-\u1974\u1980-\u19A9\u19B0-\u19C0\u19C1-\u19C7\u19C8-\u19C9\u19D0-\u19D9\u1A00-\u1A16\u1A17-\u1A18\u1A19-\u1A1B\u1D00-\u1D2B\u1D2C-\u1D61\u1D62-\u1D77\u1D78\u1D79-\u1D9A\u1D9B-\u1DBF\u1DC0-\u1DC3\u1E00-\u1E9B\u1EA0-\u1EF9\u1F00-\u1F15\u1F18-\u1F1D\u1F20-\u1F45\u1F48-\u1F4D\u1F50-\u1F57\u1F59\u1F5B\u1F5D\u1F5F-\u1F7D\u1F80-\u1FB4\u1FB6-\u1FBC\u1FBE\u1FC2-\u1FC4\u1FC6-\u1FCC\u1FD0-\u1FD3\u1FD6-\u1FDB\u1FE0-\u1FEC\u1FF2-\u1FF4\u1FF6-\u1FFC\u203F-\u2040\u2054\u2071\u207F\u2090-\u2094\u20D0-\u20DC\u20E1\u20E5-\u20EB\u2102\u2107\u210A-\u2113\u2115\u2118\u2119-\u211D\u2124\u2126\u2128\u212A-\u212D\u212E\u212F-\u2131\u2133-\u2134\u2135-\u2138\u2139\u213C-\u213F\u2145-\u2149\u2160-\u2183\u2C00-\u2C2E\u2C30-\u2C5E\u2C80-\u2CE4\u2D00-\u2D25\u2D30-\u2D65\u2D6F\u2D80-\u2D96\u2DA0-\u2DA6\u2DA8-\u2DAE\u2DB0-\u2DB6\u2DB8-\u2DBE\u2DC0-\u2DC6\u2DC8-\u2DCE\u2DD0-\u2DD6\u2DD8-\u2DDE\u3005\u3006\u3007\u3021-\u3029\u302A-\u302F\u3031-\u3035\u3038-\u303A\u303B\u303C\u3041-\u3096\u3099-\u309A\u309D-\u309E\u309F\u30A1-\u30FA\u30FC-\u30FE\u30FF\u3105-\u312C\u3131-\u318E\u31A0-\u31B7\u31F0-\u31FF\u3400-\u4DB5\u4E00-\u9FBB\uA000-\uA014\uA015\uA016-\uA48C\uA800-\uA801\uA802\uA803-\uA805\uA806\uA807-\uA80A\uA80B\uA80C-\uA822\uA823-\uA824\uA825-\uA826\uA827\uAC00-\uD7A3\uF900-\uFA2D\uFA30-\uFA6A\uFA70-\uFAD9\uFB00-\uFB06\uFB13-\uFB17\uFB1D\uFB1E\uFB1F-\uFB28\uFB2A-\uFB36\uFB38-\uFB3C\uFB3E\uFB40-\uFB41\uFB43-\uFB44\uFB46-\uFBB1\uFBD3-\uFC5D\uFC64-\uFD3D\uFD50-\uFD8F\uFD92-\uFDC7\uFDF0-\uFDF9\uFE00-\uFE0F\uFE20-\uFE23\uFE33-\uFE34\uFE4D-\uFE4F\uFE71\uFE73\uFE77\uFE79\uFE7B\uFE7D\uFE7F-\uFEFC\uFF10-\uFF19\uFF21-\uFF3A\uFF3F\uFF41-\uFF5A\uFF66-\uFF6F\uFF70\uFF71-\uFF9D\uFF9E-\uFF9F\uFFA0-\uFFBE\uFFC2-\uFFC7\uFFCA-\uFFCF\uFFD2-\uFFD7\uFFDA-\uFFDC' + +def allexcept(*args): + newcats = cats[:] + for arg in args: + newcats.remove(arg) + return u''.join([globals()[cat] for cat in newcats]) + +if __name__ == '__main__': + import unicodedata + + categories = {} + + f = open(__file__.rstrip('co')) + try: + content = f.read() + finally: + f.close() + + header = content[:content.find('Cc =')] + footer = content[content.find("def combine("):] + + for code in range(65535): + c = unichr(code) + cat = unicodedata.category(c) + categories.setdefault(cat, []).append(c) + + f = open(__file__, 'w') + f.write(header) + + for cat in sorted(categories): + val = u''.join(categories[cat]) + if cat == 'Cs': + # Jython can't handle isolated surrogates + f.write("""\ +try: + Cs = eval(r"%r") +except UnicodeDecodeError: + Cs = '' # Jython can't handle isolated surrogates\n\n""" % val) + else: + f.write('%s = %r\n\n' % (cat, val)) + f.write('cats = %r\n\n' % sorted(categories.keys())) + + f.write(footer) + f.close() diff --git a/lib/jinja2/jinja2/bccache.py b/lib/jinja2/jinja2/bccache.py new file mode 100644 --- /dev/null +++ b/lib/jinja2/jinja2/bccache.py @@ -0,0 +1,301 @@ +# -*- coding: utf-8 -*- +""" + jinja2.bccache + ~~~~~~~~~~~~~~ + + This module implements the bytecode cache system Jinja is optionally + using. This is useful if you have very complex template situations and + the compiliation of all those templates slow down your application too + much. + + Situations where this is useful are often forking web applications that + are initialized on the first request. + + :copyright: (c) 2010 by the Jinja Team. + :license: BSD. +""" +from os import path, listdir +import sys +import marshal +import tempfile +import cPickle as pickle +import fnmatch +try: + from hashlib import sha1 +except ImportError: + from sha import new as sha1 +from jinja2.utils import open_if_exists + + +# marshal works better on 3.x, one hack less required +if sys.version_info > (3, 0): + from io import BytesIO + marshal_dump = marshal.dump + marshal_load = marshal.load +else: + from cStringIO import StringIO as BytesIO + + def marshal_dump(code, f): + if isinstance(f, file): + marshal.dump(code, f) + else: + f.write(marshal.dumps(code)) + + def marshal_load(f): + if isinstance(f, file): + return marshal.load(f) + return marshal.loads(f.read()) + + +bc_version = 2 + +# magic version used to only change with new jinja versions. With 2.6 +# we change this to also take Python version changes into account. The +# reason for this is that Python tends to segfault if fed earlier bytecode +# versions because someone thought it would be a good idea to reuse opcodes +# or make Python incompatible with earlier versions. +bc_magic = 'j2'.encode('ascii') + \ + pickle.dumps(bc_version, 2) + \ + pickle.dumps((sys.version_info[0] << 24) | sys.version_info[1]) + + +class Bucket(object): + """Buckets are used to store the bytecode for one template. It's created + and initialized by the bytecode cache and passed to the loading functions. + + The buckets get an internal checksum from the cache assigned and use this + to automatically reject outdated cache material. Individual bytecode + cache subclasses don't have to care about cache invalidation. + """ + + def __init__(self, environment, key, checksum): + self.environment = environment + self.key = key + self.checksum = checksum + self.reset() + + def reset(self): + """Resets the bucket (unloads the bytecode).""" + self.code = None + + def load_bytecode(self, f): + """Loads bytecode from a file or file like object.""" + # make sure the magic header is correct + magic = f.read(len(bc_magic)) + if magic != bc_magic: + self.reset() + return + # the source code of the file changed, we need to reload + checksum = pickle.load(f) + if self.checksum != checksum: + self.reset() + return + self.code = marshal_load(f) + + def write_bytecode(self, f): + """Dump the bytecode into the file or file like object passed.""" + if self.code is None: + raise TypeError('can\'t write empty bucket') + f.write(bc_magic) + pickle.dump(self.checksum, f, 2) + marshal_dump(self.code, f) + + def bytecode_from_string(self, string): + """Load bytecode from a string.""" + self.load_bytecode(BytesIO(string)) + + def bytecode_to_string(self): + """Return the bytecode as string.""" + out = BytesIO() + self.write_bytecode(out) + return out.getvalue() + + +class BytecodeCache(object): + """To implement your own bytecode cache you have to subclass this class + and override :meth:`load_bytecode` and :meth:`dump_bytecode`. Both of + these methods are passed a :class:`~jinja2.bccache.Bucket`. + + A very basic bytecode cache that saves the bytecode on the file system:: + + from os import path + + class MyCache(BytecodeCache): + + def __init__(self, directory): + self.directory = directory + + def load_bytecode(self, bucket): + filename = path.join(self.directory, bucket.key) + if path.exists(filename): + with open(filename, 'rb') as f: + bucket.load_bytecode(f) + + def dump_bytecode(self, bucket): + filename = path.join(self.directory, bucket.key) + with open(filename, 'wb') as f: + bucket.write_bytecode(f) + + A more advanced version of a filesystem based bytecode cache is part of + Jinja2. + """ + + def load_bytecode(self, bucket): + """Subclasses have to override this method to load bytecode into a + bucket. If they are not able to find code in the cache for the + bucket, it must not do anything. + """ + raise NotImplementedError() + + def dump_bytecode(self, bucket): + """Subclasses have to override this method to write the bytecode + from a bucket back to the cache. If it unable to do so it must not + fail silently but raise an exception. + """ + raise NotImplementedError() + + def clear(self): + """Clears the cache. This method is not used by Jinja2 but should be + implemented to allow applications to clear the bytecode cache used + by a particular environment. + """ + + def get_cache_key(self, name, filename=None): + """Returns the unique hash key for this template name.""" + hash = sha1(name.encode('utf-8')) + if filename is not None: + filename = '|' + filename + if isinstance(filename, unicode): + filename = filename.encode('utf-8') + hash.update(filename) + return hash.hexdigest() + + def get_source_checksum(self, source): + """Returns a checksum for the source.""" + return sha1(source.encode('utf-8')).hexdigest() + + def get_bucket(self, environment, name, filename, source): + """Return a cache bucket for the given template. All arguments are + mandatory but filename may be `None`. + """ + key = self.get_cache_key(name, filename) + checksum = self.get_source_checksum(source) + bucket = Bucket(environment, key, checksum) + self.load_bytecode(bucket) + return bucket + + def set_bucket(self, bucket): + """Put the bucket into the cache.""" + self.dump_bytecode(bucket) + + +class FileSystemBytecodeCache(BytecodeCache): + """A bytecode cache that stores bytecode on the filesystem. It accepts + two arguments: The directory where the cache items are stored and a + pattern string that is used to build the filename. + + If no directory is specified the system temporary items folder is used. + + The pattern can be used to have multiple separate caches operate on the + same directory. The default pattern is ``'__jinja2_%s.cache'``. ``%s`` + is replaced with the cache key. + + >>> bcc = FileSystemBytecodeCache('/tmp/jinja_cache', '%s.cache') + + This bytecode cache supports clearing of the cache using the clear method. + """ + + def __init__(self, directory=None, pattern='__jinja2_%s.cache'): + if directory is None: + directory = tempfile.gettempdir() + self.directory = directory + self.pattern = pattern + + def _get_cache_filename(self, bucket): + return path.join(self.directory, self.pattern % bucket.key) + + def load_bytecode(self, bucket): + f = open_if_exists(self._get_cache_filename(bucket), 'rb') + if f is not None: + try: + bucket.load_bytecode(f) + finally: + f.close() + + def dump_bytecode(self, bucket): + f = open(self._get_cache_filename(bucket), 'wb') + try: + bucket.write_bytecode(f) + finally: + f.close() + + def clear(self): + # imported lazily here because google app-engine doesn't support + # write access on the file system and the function does not exist + # normally. + from os import remove + files = fnmatch.filter(listdir(self.directory), self.pattern % '*') + for filename in files: + try: + remove(path.join(self.directory, filename)) + except OSError: + pass + + +class MemcachedBytecodeCache(BytecodeCache): + """This class implements a bytecode cache that uses a memcache cache for + storing the information. It does not enforce a specific memcache library + (tummy's memcache or cmemcache) but will accept any class that provides + the minimal interface required. + + Libraries compatible with this class: + + - `werkzeug `_.contrib.cache + - `python-memcached `_ + - `cmemcache `_ + + (Unfortunately the django cache interface is not compatible because it + does not support storing binary data, only unicode. You can however pass + the underlying cache client to the bytecode cache which is available + as `django.core.cache.cache._client`.) + + The minimal interface for the client passed to the constructor is this: + + .. class:: MinimalClientInterface + + .. method:: set(key, value[, timeout]) + + Stores the bytecode in the cache. `value` is a string and + `timeout` the timeout of the key. If timeout is not provided + a default timeout or no timeout should be assumed, if it's + provided it's an integer with the number of seconds the cache + item should exist. + + .. method:: get(key) + + Returns the value for the cache key. If the item does not + exist in the cache the return value must be `None`. + + The other arguments to the constructor are the prefix for all keys that + is added before the actual cache key and the timeout for the bytecode in + the cache system. We recommend a high (or no) timeout. + + This bytecode cache does not support clearing of used items in the cache. + The clear method is a no-operation function. + """ + + def __init__(self, client, prefix='jinja2/bytecode/', timeout=None): + self.client = client + self.prefix = prefix + self.timeout = timeout + + def load_bytecode(self, bucket): + code = self.client.get(self.prefix + bucket.key) + if code is not None: + bucket.bytecode_from_string(code) + + def dump_bytecode(self, bucket): + args = (self.prefix + bucket.key, bucket.bytecode_to_string()) + if self.timeout is not None: + args += (self.timeout,) + self.client.set(*args) diff --git a/lib/jinja2/jinja2/compiler.py b/lib/jinja2/jinja2/compiler.py new file mode 100644 --- /dev/null +++ b/lib/jinja2/jinja2/compiler.py @@ -0,0 +1,1649 @@ +# -*- coding: utf-8 -*- +""" + jinja2.compiler + ~~~~~~~~~~~~~~~ + + Compiles nodes into python code. + + :copyright: (c) 2010 by the Jinja Team. + :license: BSD, see LICENSE for more details. +""" +from cStringIO import StringIO +from itertools import chain +from copy import deepcopy +from jinja2 import nodes +from jinja2.nodes import EvalContext +from jinja2.visitor import NodeVisitor +from jinja2.exceptions import TemplateAssertionError +from jinja2.utils import Markup, concat, escape, is_python_keyword, next + + +operators = { + 'eq': '==', + 'ne': '!=', + 'gt': '>', + 'gteq': '>=', + 'lt': '<', + 'lteq': '<=', + 'in': 'in', + 'notin': 'not in' +} + +try: + exec '(0 if 0 else 0)' +except SyntaxError: + have_condexpr = False +else: + have_condexpr = True + + +# what method to iterate over items do we want to use for dict iteration +# in generated code? on 2.x let's go with iteritems, on 3.x with items +if hasattr(dict, 'iteritems'): + dict_item_iter = 'iteritems' +else: + dict_item_iter = 'items' + + +# does if 0: dummy(x) get us x into the scope? +def unoptimize_before_dead_code(): + x = 42 + def f(): + if 0: dummy(x) + return f +unoptimize_before_dead_code = bool(unoptimize_before_dead_code().func_closure) + + +def generate(node, environment, name, filename, stream=None, + defer_init=False): + """Generate the python source for a node tree.""" + if not isinstance(node, nodes.Template): + raise TypeError('Can\'t compile non template nodes') + generator = CodeGenerator(environment, name, filename, stream, defer_init) + generator.visit(node) + if stream is None: + return generator.stream.getvalue() + + +def has_safe_repr(value): + """Does the node have a safe representation?""" + if value is None or value is NotImplemented or value is Ellipsis: + return True + if isinstance(value, (bool, int, long, float, complex, basestring, + xrange, Markup)): + return True + if isinstance(value, (tuple, list, set, frozenset)): + for item in value: + if not has_safe_repr(item): + return False + return True + elif isinstance(value, dict): + for key, value in value.iteritems(): + if not has_safe_repr(key): + return False + if not has_safe_repr(value): + return False + return True + return False + + +def find_undeclared(nodes, names): + """Check if the names passed are accessed undeclared. The return value + is a set of all the undeclared names from the sequence of names found. + """ + visitor = UndeclaredNameVisitor(names) + try: + for node in nodes: + visitor.visit(node) + except VisitorExit: + pass + return visitor.undeclared + + +class Identifiers(object): + """Tracks the status of identifiers in frames.""" + + def __init__(self): + # variables that are known to be declared (probably from outer + # frames or because they are special for the frame) + self.declared = set() + + # undeclared variables from outer scopes + self.outer_undeclared = set() + + # names that are accessed without being explicitly declared by + # this one or any of the outer scopes. Names can appear both in + # declared and undeclared. + self.undeclared = set() + + # names that are declared locally + self.declared_locally = set() + + # names that are declared by parameters + self.declared_parameter = set() + + def add_special(self, name): + """Register a special name like `loop`.""" + self.undeclared.discard(name) + self.declared.add(name) + + def is_declared(self, name): + """Check if a name is declared in this or an outer scope.""" + if name in self.declared_locally or name in self.declared_parameter: + return True + return name in self.declared + + def copy(self): + return deepcopy(self) + + +class Frame(object): + """Holds compile time information for us.""" + + def __init__(self, eval_ctx, parent=None): + self.eval_ctx = eval_ctx + self.identifiers = Identifiers() + + # a toplevel frame is the root + soft frames such as if conditions. + self.toplevel = False + + # the root frame is basically just the outermost frame, so no if + # conditions. This information is used to optimize inheritance + # situations. + self.rootlevel = False + + # in some dynamic inheritance situations the compiler needs to add + # write tests around output statements. + self.require_output_check = parent and parent.require_output_check + + # inside some tags we are using a buffer rather than yield statements. + # this for example affects {% filter %} or {% macro %}. If a frame + # is buffered this variable points to the name of the list used as + # buffer. + self.buffer = None + + # the name of the block we're in, otherwise None. + self.block = parent and parent.block or None + + # a set of actually assigned names + self.assigned_names = set() + + # the parent of this frame + self.parent = parent + + if parent is not None: + self.identifiers.declared.update( + parent.identifiers.declared | + parent.identifiers.declared_parameter | + parent.assigned_names + ) + self.identifiers.outer_undeclared.update( + parent.identifiers.undeclared - + self.identifiers.declared + ) + self.buffer = parent.buffer + + def copy(self): + """Create a copy of the current one.""" + rv = object.__new__(self.__class__) + rv.__dict__.update(self.__dict__) + rv.identifiers = object.__new__(self.identifiers.__class__) + rv.identifiers.__dict__.update(self.identifiers.__dict__) + return rv + + def inspect(self, nodes): + """Walk the node and check for identifiers. If the scope is hard (eg: + enforce on a python level) overrides from outer scopes are tracked + differently. + """ + visitor = FrameIdentifierVisitor(self.identifiers) + for node in nodes: + visitor.visit(node) + + def find_shadowed(self, extra=()): + """Find all the shadowed names. extra is an iterable of variables + that may be defined with `add_special` which may occour scoped. + """ + i = self.identifiers + return (i.declared | i.outer_undeclared) & \ + (i.declared_locally | i.declared_parameter) | \ + set(x for x in extra if i.is_declared(x)) + + def inner(self): + """Return an inner frame.""" + return Frame(self.eval_ctx, self) + + def soft(self): + """Return a soft frame. A soft frame may not be modified as + standalone thing as it shares the resources with the frame it + was created of, but it's not a rootlevel frame any longer. + """ + rv = self.copy() + rv.rootlevel = False + return rv + + __copy__ = copy + + +class VisitorExit(RuntimeError): + """Exception used by the `UndeclaredNameVisitor` to signal a stop.""" + + +class DependencyFinderVisitor(NodeVisitor): + """A visitor that collects filter and test calls.""" + + def __init__(self): + self.filters = set() + self.tests = set() + + def visit_Filter(self, node): + self.generic_visit(node) + self.filters.add(node.name) + + def visit_Test(self, node): + self.generic_visit(node) + self.tests.add(node.name) + + def visit_Block(self, node): + """Stop visiting at blocks.""" + + +class UndeclaredNameVisitor(NodeVisitor): + """A visitor that checks if a name is accessed without being + declared. This is different from the frame visitor as it will + not stop at closure frames. + """ + + def __init__(self, names): + self.names = set(names) + self.undeclared = set() + + def visit_Name(self, node): + if node.ctx == 'load' and node.name in self.names: + self.undeclared.add(node.name) + if self.undeclared == self.names: + raise VisitorExit() + else: + self.names.discard(node.name) + + def visit_Block(self, node): + """Stop visiting a blocks.""" + + +class FrameIdentifierVisitor(NodeVisitor): + """A visitor for `Frame.inspect`.""" + + def __init__(self, identifiers): + self.identifiers = identifiers + + def visit_Name(self, node): + """All assignments to names go through this function.""" + if node.ctx == 'store': + self.identifiers.declared_locally.add(node.name) + elif node.ctx == 'param': + self.identifiers.declared_parameter.add(node.name) + elif node.ctx == 'load' and not \ + self.identifiers.is_declared(node.name): + self.identifiers.undeclared.add(node.name) + + def visit_If(self, node): + self.visit(node.test) + real_identifiers = self.identifiers + + old_names = real_identifiers.declared_locally | \ + real_identifiers.declared_parameter + + def inner_visit(nodes): + if not nodes: + return set() + self.identifiers = real_identifiers.copy() + for subnode in nodes: + self.visit(subnode) + rv = self.identifiers.declared_locally - old_names + # we have to remember the undeclared variables of this branch + # because we will have to pull them. + real_identifiers.undeclared.update(self.identifiers.undeclared) + self.identifiers = real_identifiers + return rv + + body = inner_visit(node.body) + else_ = inner_visit(node.else_ or ()) + + # the differences between the two branches are also pulled as + # undeclared variables + real_identifiers.undeclared.update(body.symmetric_difference(else_) - + real_identifiers.declared) + + # remember those that are declared. + real_identifiers.declared_locally.update(body | else_) + + def visit_Macro(self, node): + self.identifiers.declared_locally.add(node.name) + + def visit_Import(self, node): + self.generic_visit(node) + self.identifiers.declared_locally.add(node.target) + + def visit_FromImport(self, node): + self.generic_visit(node) + for name in node.names: + if isinstance(name, tuple): + self.identifiers.declared_locally.add(name[1]) + else: + self.identifiers.declared_locally.add(name) + + def visit_Assign(self, node): + """Visit assignments in the correct order.""" + self.visit(node.node) + self.visit(node.target) + + def visit_For(self, node): + """Visiting stops at for blocks. However the block sequence + is visited as part of the outer scope. + """ + self.visit(node.iter) + + def visit_CallBlock(self, node): + self.visit(node.call) + + def visit_FilterBlock(self, node): + self.visit(node.filter) + + def visit_Scope(self, node): + """Stop visiting at scopes.""" + + def visit_Block(self, node): + """Stop visiting at blocks.""" + + +class CompilerExit(Exception): + """Raised if the compiler encountered a situation where it just + doesn't make sense to further process the code. Any block that + raises such an exception is not further processed. + """ + + +class CodeGenerator(NodeVisitor): + + def __init__(self, environment, name, filename, stream=None, + defer_init=False): + if stream is None: + stream = StringIO() + self.environment = environment + self.name = name + self.filename = filename + self.stream = stream + self.created_block_context = False + self.defer_init = defer_init + + # aliases for imports + self.import_aliases = {} + + # a registry for all blocks. Because blocks are moved out + # into the global python scope they are registered here + self.blocks = {} + + # the number of extends statements so far + self.extends_so_far = 0 + + # some templates have a rootlevel extends. In this case we + # can safely assume that we're a child template and do some + # more optimizations. + self.has_known_extends = False + + # the current line number + self.code_lineno = 1 + + # registry of all filters and tests (global, not block local) + self.tests = {} + self.filters = {} + + # the debug information + self.debug_info = [] + self._write_debug_info = None + + # the number of new lines before the next write() + self._new_lines = 0 + + # the line number of the last written statement + self._last_line = 0 + + # true if nothing was written so far. + self._first_write = True + + # used by the `temporary_identifier` method to get new + # unique, temporary identifier + self._last_identifier = 0 + + # the current indentation + self._indentation = 0 + + # -- Various compilation helpers + + def fail(self, msg, lineno): + """Fail with a :exc:`TemplateAssertionError`.""" + raise TemplateAssertionError(msg, lineno, self.name, self.filename) + + def temporary_identifier(self): + """Get a new unique identifier.""" + self._last_identifier += 1 + return 't_%d' % self._last_identifier + + def buffer(self, frame): + """Enable buffering for the frame from that point onwards.""" + frame.buffer = self.temporary_identifier() + self.writeline('%s = []' % frame.buffer) + + def return_buffer_contents(self, frame): + """Return the buffer contents of the frame.""" + if frame.eval_ctx.volatile: + self.writeline('if context.eval_ctx.autoescape:') + self.indent() + self.writeline('return Markup(concat(%s))' % frame.buffer) + self.outdent() + self.writeline('else:') + self.indent() + self.writeline('return concat(%s)' % frame.buffer) + self.outdent() + elif frame.eval_ctx.autoescape: + self.writeline('return Markup(concat(%s))' % frame.buffer) + else: + self.writeline('return concat(%s)' % frame.buffer) + + def indent(self): + """Indent by one.""" + self._indentation += 1 + + def outdent(self, step=1): + """Outdent by step.""" + self._indentation -= step + + def start_write(self, frame, node=None): + """Yield or write into the frame buffer.""" + if frame.buffer is None: + self.writeline('yield ', node) + else: + self.writeline('%s.append(' % frame.buffer, node) + + def end_write(self, frame): + """End the writing process started by `start_write`.""" + if frame.buffer is not None: + self.write(')') + + def simple_write(self, s, frame, node=None): + """Simple shortcut for start_write + write + end_write.""" + self.start_write(frame, node) + self.write(s) + self.end_write(frame) + + def blockvisit(self, nodes, frame): + """Visit a list of nodes as block in a frame. If the current frame + is no buffer a dummy ``if 0: yield None`` is written automatically + unless the force_generator parameter is set to False. + """ + if frame.buffer is None: + self.writeline('if 0: yield None') + else: + self.writeline('pass') + try: + for node in nodes: + self.visit(node, frame) + except CompilerExit: + pass + + def write(self, x): + """Write a string into the output stream.""" + if self._new_lines: + if not self._first_write: + self.stream.write('\n' * self._new_lines) + self.code_lineno += self._new_lines + if self._write_debug_info is not None: + self.debug_info.append((self._write_debug_info, + self.code_lineno)) + self._write_debug_info = None + self._first_write = False + self.stream.write(' ' * self._indentation) + self._new_lines = 0 + self.stream.write(x) + + def writeline(self, x, node=None, extra=0): + """Combination of newline and write.""" + self.newline(node, extra) + self.write(x) + + def newline(self, node=None, extra=0): + """Add one or more newlines before the next write.""" + self._new_lines = max(self._new_lines, 1 + extra) + if node is not None and node.lineno != self._last_line: + self._write_debug_info = node.lineno + self._last_line = node.lineno + + def signature(self, node, frame, extra_kwargs=None): + """Writes a function call to the stream for the current node. + A leading comma is added automatically. The extra keyword + arguments may not include python keywords otherwise a syntax + error could occour. The extra keyword arguments should be given + as python dict. + """ + # if any of the given keyword arguments is a python keyword + # we have to make sure that no invalid call is created. + kwarg_workaround = False + for kwarg in chain((x.key for x in node.kwargs), extra_kwargs or ()): + if is_python_keyword(kwarg): + kwarg_workaround = True + break + + for arg in node.args: + self.write(', ') + self.visit(arg, frame) + + if not kwarg_workaround: + for kwarg in node.kwargs: + self.write(', ') + self.visit(kwarg, frame) + if extra_kwargs is not None: + for key, value in extra_kwargs.iteritems(): + self.write(', %s=%s' % (key, value)) + if node.dyn_args: + self.write(', *') + self.visit(node.dyn_args, frame) + + if kwarg_workaround: + if node.dyn_kwargs is not None: + self.write(', **dict({') + else: + self.write(', **{') + for kwarg in node.kwargs: + self.write('%r: ' % kwarg.key) + self.visit(kwarg.value, frame) + self.write(', ') + if extra_kwargs is not None: + for key, value in extra_kwargs.iteritems(): + self.write('%r: %s, ' % (key, value)) + if node.dyn_kwargs is not None: + self.write('}, **') + self.visit(node.dyn_kwargs, frame) + self.write(')') + else: + self.write('}') + + elif node.dyn_kwargs is not None: + self.write(', **') + self.visit(node.dyn_kwargs, frame) + + def pull_locals(self, frame): + """Pull all the references identifiers into the local scope.""" + for name in frame.identifiers.undeclared: + self.writeline('l_%s = context.resolve(%r)' % (name, name)) + + def pull_dependencies(self, nodes): + """Pull all the dependencies.""" + visitor = DependencyFinderVisitor() + for node in nodes: + visitor.visit(node) + for dependency in 'filters', 'tests': + mapping = getattr(self, dependency) + for name in getattr(visitor, dependency): + if name not in mapping: + mapping[name] = self.temporary_identifier() + self.writeline('%s = environment.%s[%r]' % + (mapping[name], dependency, name)) + + def unoptimize_scope(self, frame): + """Disable Python optimizations for the frame.""" + # XXX: this is not that nice but it has no real overhead. It + # mainly works because python finds the locals before dead code + # is removed. If that breaks we have to add a dummy function + # that just accepts the arguments and does nothing. + if frame.identifiers.declared: + self.writeline('%sdummy(%s)' % ( + unoptimize_before_dead_code and 'if 0: ' or '', + ', '.join('l_' + name for name in frame.identifiers.declared) + )) + + def push_scope(self, frame, extra_vars=()): + """This function returns all the shadowed variables in a dict + in the form name: alias and will write the required assignments + into the current scope. No indentation takes place. + + This also predefines locally declared variables from the loop + body because under some circumstances it may be the case that + + `extra_vars` is passed to `Frame.find_shadowed`. + """ + aliases = {} + for name in frame.find_shadowed(extra_vars): + aliases[name] = ident = self.temporary_identifier() + self.writeline('%s = l_%s' % (ident, name)) + to_declare = set() + for name in frame.identifiers.declared_locally: + if name not in aliases: + to_declare.add('l_' + name) + if to_declare: + self.writeline(' = '.join(to_declare) + ' = missing') + return aliases + + def pop_scope(self, aliases, frame): + """Restore all aliases and delete unused variables.""" + for name, alias in aliases.iteritems(): + self.writeline('l_%s = %s' % (name, alias)) + to_delete = set() + for name in frame.identifiers.declared_locally: + if name not in aliases: + to_delete.add('l_' + name) + if to_delete: + # we cannot use the del statement here because enclosed + # scopes can trigger a SyntaxError: From noreply at buildbot.pypy.org Tue Jan 24 12:10:15 2012 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 24 Jan 2012 12:10:15 +0100 (CET) Subject: [pypy-commit] benchmarks default: merge Message-ID: <20120124111015.4E21A82CFD@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r163:f4a31718810d Date: 2012-01-24 13:09 +0200 http://bitbucket.org/pypy/benchmarks/changeset/f4a31718810d/ Log: merge diff too long, truncating to 10000 out of 32654 lines diff --git a/benchmarks.py b/benchmarks.py --- a/benchmarks.py +++ b/benchmarks.py @@ -52,6 +52,11 @@ extra_args=['--benchmark=' + name], iteration_scaling=0.1) +for name in ['xml', 'text']: + _register_new_bm('bm_genshi', 'genshi_' + name, + globals(), bm_env={'PYTHONPATH': relative('lib/genshi')}, + extra_args=['--benchmark=' + name]) + for name in ['float', 'nbody_modified', 'meteor-contest', 'fannkuch', 'spectral-norm', 'chaos', 'telco', 'go', 'pyflate-fast', 'raytrace-simple', 'crypto_pyaes', 'bm_mako', 'bm_chameleon', diff --git a/lib/genshi/COPYING b/lib/genshi/COPYING new file mode 100644 --- /dev/null +++ b/lib/genshi/COPYING @@ -0,0 +1,28 @@ +Copyright (C) 2006-2010 Edgewall Software +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + 3. The name of the author may not be used to endorse or promote + products derived from this software without specific prior + written permission. + +THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS +OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE +GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER +IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR +OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN +IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/lib/genshi/ChangeLog b/lib/genshi/ChangeLog new file mode 100644 --- /dev/null +++ b/lib/genshi/ChangeLog @@ -0,0 +1,472 @@ +Version 0.6.1 +http://svn.edgewall.org/repos/genshi/tags/0.6.1/ +(???, from branches/stable/0.6.x) + + * Fix for error in how `HTMLFormFiller` would handle `textarea` elements if + no value was not supplied form them. + * The `HTMLFormFiller` now correctly handles check boxes and radio buttons + with an empty `value` attribute. + + +Version 0.6 +http://svn.edgewall.org/repos/genshi/tags/0.6.0/ +(Apr 22 2010, from branches/stable/0.6.x) + + * Support for Python 2.3 has been dropped. + * Rewrite of the XPath evaluation engine for better performance and improved + correctness. This is the result of integrating work done by Marcin Kurczych + during GSoC 2008. + * Updated the Python AST processing for template code evaluation to use the + `_ast` module instead of the deprecated `compiler` package, including an + adapter layer for Python 2.4. This, too, is the result of integrating work + done by Marcin Kurczych during GSoC 2008. + * Added caching in the serialization stage for improved performance in some + cases. + * Various improvements to the HTML sanitization filter. + * Fix problem with I18n filter that would get confused by expressions in + attribute values when inside an `i18n:msg` block (ticket #250). + * Fix problem with the transformation filter dropping events after the + selection (ticket #290). + * `for` loops in template code blocks no longer establish their own locals + scope, meaning you can now access variables assigned in the loop outside + of the loop, just as you can in regular Python code (ticket #259). + * Import statements inside function definitions in template code blocks no + longer result in an UndefinedError when the imported name is accessed + (ticket #276). + * Fixed handling of relative URLs with fragment identifiers containing colons + in the `HTMLSanitizer` (ticket #274). + * Added an option to the `HTMLFiller` to also populate password fields. + * Match template processing no longer produces unwanted duplicate output in + some cases (ticket #254). + * Templates instantiated without a loader now get an implicit loader based on + their file path, or the current directory as a fallback (ticket #320). + * Added documentation for the `TemplateLoader`. + * Enhanced documentation for internationalization. + + +Version 0.5.1 +http://svn.edgewall.org/repos/genshi/tags/0.5.1/ +(Jul 9 2008, from branches/stable/0.5.x) + + * Fix problem with nested match templates not being applied when buffering + on the outer `py:match` is disabled. Thanks to Erik Bray for reporting the + problem and providing a test case! + * Fix problem in `Translator` filter that would cause the translation of + text nodes to fail if the translation function returned an object that was + not directly a string, but rather something like an instance of the + `LazyProxy` class in Babel (ticket #145). + * Fix problem with match templates incorrectly being applied multiple times. + * Includes from templates loaded via an absolute path now include the correct + file in nested directories as long if no search path has been configured + (ticket #240). + * Unbuffered match templates could result in parts of the matched content + being included in the output if the match template didn't actually consume + it via one or more calls to the `select()` function (ticket #243). + + +Version 0.5 +http://svn.edgewall.org/repos/genshi/tags/0.5.0/ +(Jun 9 2008, from branches/stable/0.5.x) + + * Added #include directive for text templates (ticket #115). + * Added new markup transformation filter contributed by Alec Thomas. This + provides gorgeous jQuery-inspired stream transformation capabilities based + on XPath expressions. + * When using HTML or XHTML serialization, the `xml:lang` attribute is + automatically translated to the `lang` attribute which HTML user agents + understand. + * Added support for the XPath 2 `matches()` function in XPath expressions, + which allow matching against regular expressions. + * Support for Python code blocks in templates can now be disabled + (ticket #123). + * Includes are now processed when the template is parsed if possible, but + only if the template loader is not set to do automatic reloading. Included + templates are basically inlined into the including template, which can + speed up rendering of that template a bit. + * Added new syntax for text templates, which is more powerful and flexible + with respect to white-space and line breaks. It also supports Python code + blocks. The old syntax is still available and the default for now, but in a + future release the new syntax will become the default, and some time after + that the old syntax will be removed. + * Added support for passing optimization hints to `` directives, + which can speed up match templates in many cases, for example when a match + template should only be applied once to a stream, or when it should not be + applied recursively. + * Text templates now default to rendering as plain text; it is no longer + necessary to explicitly specify the "text" method to the `render()` or + `serialize()` method of the generated markup stream. + * XInclude elements in markup templates now support the `parse` attribute; + when set to "xml" (the default), the include is processed as before, but + when set to "text", the included template is parsed as a text template using + the new syntax (ticket #101). + * Python code blocks inside match templates are now executed (ticket #155). + * The template engine plugin no longer adds the `default_doctype` when the + `fragment` parameter is `True`. + * The `striptags` function now also removes HTML/XML-style comments (ticket + #150). + * The `py:replace` directive can now also be used as an element, with an + attribute named `value` (ticket #144). + * The `TextSerializer` class no longer strips all markup in text by default, + so that it is still possible to use the Genshi `escape` function even with + text templates. The old behavior is available via the `strip_markup` option + of the serializer (ticket #146). + * Assigning to a variable named `data` in a Python code block no longer + breaks context lookup. + * The `Stream.render` now accepts an optional `out` parameter that can be + used to pass in a writable file-like object to use for assembling the + output, instead of building a big string and returning it. + * The XHTML serializer now strips `xml:space` attributes as they are only + allowed on very few tags. + * Match templates are now applied in a more controlled fashion: in the order + they are declared in the template source, all match templates up to (and + including) the matching template itself are applied to the matched content, + whereas the match templates declared after the matching template are only + applied to the generated content (ticket #186). + * The `TemplateLoader` class now provides an `_instantiate()` method that can + be overridden by subclasses to implement advanced template instantiation + logic (ticket #204). + * The search path of the `TemplateLoader` class can now contain ''load + functions'' in addition to path strings. A load function is passed the + name of the requested template file, and should return a file-like object + and some metadata. New load functions are supplied for loading from egg + package data, and loading from different loaders depending on the path + prefix of the requested filename (ticket #182). + * Match templates can now be processed without keeping the complete matched + content in memory, which could cause excessive memory use on long pages. + The buffering can be disabled using the new `buffer` optimization hint on + the `` directive. + * Improve error reporting when accessing an attribute in a Python expression + raises an `AttributeError` (ticket #191). + * The `Markup` class now supports mappings for right hand of the `%` (modulo) + operator in the same way the Python string classes do, except that the + substituted values are escape. Also, the special constructor which took + positional arguments that would be substituted was removed. Thus the + `Markup` class now supports the same arguments as that of its `unicode` + base class (ticket #211). + * The `Template` class and its subclasses, as well as the interpolation API, + now take an `filepath` parameter instead of `basedir` (ticket #207). + * The `XHTMLSerializer` now has a `drop_xml_decl` option that defaults to + `True`. Setting it to `False` will cause any XML decl in the serialized + stream to be included in the output as it would for XML serialization. + * Add support for a protocol that would allow interoperability of different + Python packages that generate and/or consume markup, based on the special + `__html__()` method (ticket #202). + + +Version 0.4.4 +http://svn.edgewall.org/repos/genshi/tags/0.4.4/ +(Aug 14, 2007, from branches/stable/0.4.x) + + * Fixed augmented assignment to local variables in Python code blocks. + * Fixed handling of nested function and class definitions in Python code + blocks. + * Includes were not raising `TemplateNotFound` exceptions even when no + fallback has been specified. That has been corrected. + * The template loader now raises a `TemplateNotFound` error when a previously + cached template is removed or renamed, where it previously was passing up + an `OSError`. + * The Genshi I18n filter can be configured to only extract messages found in + `gettext` function calls, ignoring any text nodes and attribute values + (ticket #138). + + +Version 0.4.3 +http://svn.edgewall.org/repos/genshi/tags/0.4.3/ +(Jul 17 2007, from branches/stable/0.4.x) + + * The I18n filter no longer extracts or translates literal strings in + attribute values that also contain expressions. + * Added `loader_callback` option to plugin interface, which allows specifying + a callback function that the template loader should invoke whenever a new + template is loaded (ticket #130). Note that the value for this option can + not be specified as a string, only as an actual function object, which means + it is not available for use through configuration files. + * The I18n filter now extracts messages from gettext functions even inside + ignored tags (ticket #132). + * The HTML sanitizer now strips any CSS comments in style attributes, which + could previously be used to hide malicious property values. + * The HTML sanitizer now also removes any HTML comments encountered, as those + may be used to hide malicious payloads targetting a certain "innovative" + browser that goes and interprets the content of specially prepared comments. + * Attribute access in template expressions no longer silently ignores + exceptions other than `AttributeError` raised in the attribute accessor. + + +Version 0.4.2 +http://svn.edgewall.org/repos/genshi/tags/0.4.2/ +(Jun 20 2007, from branches/stable/0.4.x) + + * The `doctype` parameter of the markup serializers now also accepts the "name" + of the doctype as string, in addition to the `(name, pubid, sysid)` tuple. + * The I18n filter was not replacing the original attributes with the + translation, but instead adding a second attribute with the same name. + * `TextTemplate` can now handle unicode source (ticket #125). + * A `` processing instruction containing trailing whitespace no + longer causes a syntax error (ticket #127). + * The I18n filter now skips the content of elements that have an `xml:lang` + attribute with a fixed string value. Basically, `xml:lang` can now be used + as a flag to mark specific sections as not needing localization. + * Added plugin for message extraction via Babel (http://babel.edgewall.org/). + + +Version 0.4.1 +http://svn.edgewall.org/repos/genshi/tags/0.4.1/ +(May 21 2007, from branches/stable/0.4.x) + + * Fix incorrect reference to translation function in the I18N filter. + * The `ET()` function now correctly handles attributes with a namespace. + * XML declarations are now processed internally, as well as written to the + output when XML serialization is used (ticket #111). + * Added the functions `encode()` and `get_serializer()` to the `genshi.output` + module, which provide a lower-level API to the functionality previously only + available through `Stream.render()` and `Stream.serialize()`. + * The `DocType` class now has a `get(name)` function that returns a `DOCTYPE` + tuple for a given string. + * Added frameset variants to the `DocType` constants for HTML 4.01 and XHTML + 1.0. + * Improved I18n extraction for pluralizable messages: for any translation + function with multiple string arguments (such as ``ngettext``), a single + item with a tuple of strings is yielded, instead an item for each string + argument. + * The `HTMLFormFiller` stream filter no longer alters form elements for which + the data element contains no corresponding item. + * Code in `` processing instructions no longer gets the special + treatment as Python code in template expressions, i.e. item and attribute + access are no longer interchangeable (which was broken in a number of ways + anyway, see ticket #113). This change does not affect expressions. + * Numerous fixes for the execution of Python code in `` processing + instructions (tickets #113 and #114). + * The `py:def` (and `#def`) directive now supports "star args" (i.e. `*args` + and `**kwargs`) in the function declaration (ticket #116). + + +Version 0.4 +http://svn.edgewall.org/repos/genshi/tags/0.4.0/ +(Apr 16 2007, from branches/stable/0.4.x) + + * New example applications for CherryPy and web.py. + * The template loader now uses a LRU cache to limit the number of cached + templates to a configurable maximum. Also, a race condition in the template + loader was fixed by adding locking. + * A new filter (genshi.filters.HTMLFormFiller) was added, which can populate + HTML forms based on a dictionary of values. + * The set of permitted tag and attribute names for the HTMLSanitizer can now + be configured per instance. + * The template engine plugin now supports a range of options for + configuration, for example to set the default serialization method, the + default output encoding, or the default DOCTYPE. + * The ElementTree adaptation function `ET()` has moved into the `genshi.input` + module. + * Allow `when` directives to omit the test expression as long as the + associated choose directive does have one. In that case, the when branch is + followed if the expression of the choose directive evaluates to a truth + value. + * Unsuccessful attribute or item lookups now return `Undefined` objects for + nicer error messages. + * Split up the `genshi.template` module into multiple modules inside the new + `genshi.template` package. + * Results of expression evaluation are no longer implicitly called if they + are callable. + * Instances of the `genshi.core.Attrs` class are now immutable (they are + subclasses of `tuple` instead of `list`). + * `MarkupTemplate`s can now be instantiated from markup streams, in addition + to strings and file-like objects (ticket #69). + * Improve handling of incorrectly nested tags in the HTML parser. + * Template includes can now be nested inside fallback content. + * Expressions can now contain dict literals (ticket #37). + * It is now possible to have one or more escaped dollar signs in front of a + full expression (ticket #92). + * The `Markup` class is now available by default in template expressions + (ticket #67). + * The handling of namespace declarations in XML/XHTML output has been improved. + * The `Attrs` class no longer automatically wraps all attribute names in + `QName` objects. This is now the responsibility of whoever is instantiating + `Attrs` objects (for example, stream filters and generators). + * Python code blocks are now supported using the `` processing + instruction (ticket #84). + * The way errors in template expressions are handled can now be configured. The + option `LenientLookup` provides the same forgiving mode used in previous + Genshi versions, while `StrictLookup` raises exceptions when undefined + variables or members are accessed. The lenient mode is still the default in + this version, but that may change in the future. (ticket #88) + * If a variable is not necessarily defined at the top level of the template + data, the new built-in functions `defined(key)` and `value_of(key, default)` + can be used so that the template also works in strict lookup mode. These + functions were previously only available when using Genshi via the template + engine plugin (for compatibility with Kid). + * `style` attributes are no longer allowed by the `HTMLSanitizer` by default. + If they are explicitly added to the set of safe attributes, any unicode + escapes in the attribute value are now handled properly. + * Namespace declarations on conditional elements (for example using a `py:if` + directive`) are no longer moved to the following element when the element + originally carrying the declaration is removed from the stream (ticket #107). + * Added basic built-in support for internationalizing templates by providing + a new `Translator` class that can both extract localizable strings from a + stream, and replace those strings with their localizations at render time. + The code for this was largely taken from previous work done by Matt Good + and David Fraser. + + +Version 0.3.6 +http://svn.edgewall.org/repos/genshi/tags/0.3.6/ +(Dec 11 2006, from branches/stable/0.3.x) + + * The builder API now accepts streams as children of elements and fragments. + + +Version 0.3.5 +http://svn.edgewall.org/repos/genshi/tags/0.3.5/ +(Nov 22 2006, from branches/stable/0.3.x) + + * Fix XPath traversal in match templates. Previously, `div/p` would be treated + the same as `div//p`, i.e. it would match all descendants and not just the + immediate children. + * Preserve whitespace in HTML `
` elements also when they contain child + elements. + * Match templates no longer match their own output (ticket #77). + * Blank lines before directives in text templates are now preserved as + expected (ticket #62). + + +Version 0.3.4 +http://svn.edgewall.org/repos/genshi/tags/0.3.4/ +(Nov 2 2006, from branches/stable/0.3.x) + + * The encoding of HTML and XML files, as well as markup and text templates, + can now be specified. Also, the encoding specified in XML declarations is + now respected unless an expiclit encoding is requested. + * Expressions used as arguments for `py:with`, `py:def`, and `py:for` + directives can now contain non-ASCII strings. + + +Version 0.3.3 +http://svn.edgewall.org/repos/genshi/tags/0.3.3/ +(Oct 16 2006, from branches/stable/0.3.x) + + * Fixed bug introduced in 0.3.2 that broke the parsing of templates which + declare the same namespace more than once in a nested fashion. + * Fixed the parsing of HTML entity references inside attribute values, both + in the `XMLParser` and the `HTMLParser` classes. + * Some changes to usage of absolute vs. relative template paths to ensure that + the filenamed-keyed cache employed by the TemplateLoader doesn't mix up + templates with the same name, but from different subdirectories. + + +Version 0.3.2 +http://svn.edgewall.org/repos/genshi/tags/0.3.2/ +(Oct 12 2006, from branches/stable/0.3.x) + + * Exceptions from templates now contain the absolute path to the template file + when a search path is used. This enables tracebacks to display the line in + question. + * The template engine plugin now provides three different types: "genshi" and + "genshi-markup" map to markup templates, while "genshi-text" maps to text + templates. + * Fixed the namespace context used by XPath patterns in py:match templates. + The were erroneously using the namespace context of the elements being + matched, where they should rather use the context in which they were + defined. + * The contents of ` + ... """) + >>> sanitize = HTMLSanitizer() + >>> print(html | sanitize) +
+
Innocent looking text.
+
+ +In this example, the ```` element. Both ````. Also, it will normalize any boolean attributes values + that are minimized in HTML, so that for example ``
`` + becomes ``
``. + + This serializer supports the use of namespaces for compound documents, for + example to use inline SVG inside an XHTML document. + +``html`` + The ``HTMLSerializer`` produces proper HTML markup. The main differences + compared to ``xhtml`` serialization are that boolean attributes are + minimized, empty tags are not self-closing (so it's ``
`` instead of + ``
``), and that the contents of ``') + >>> print(tmpl.generate()) + + +On the other hand, Genshi will always replace two dollar signs in text with a +single dollar sign, so you'll need to use three dollar signs to get two in the +output: + +.. code-block:: pycon + + >>> tmpl = MarkupTemplate('') + >>> print(tmpl.generate()) + + + +.. _`code blocks`: + +Code Blocks +=========== + +Templates also support full Python code blocks, using the ```` +processing instruction in XML templates: + +.. code-block:: genshi + +
+ + ${greeting('world')} +
+ +This will produce the following output: + +.. code-block:: xml + +
+ Hello, world! +
+ +In text templates (although only those using the new syntax introduced in +Genshi 0.5), code blocks use the special ``{% python %}`` directive: + +.. code-block:: genshitext + + {% python + from genshi.builder import tag + def greeting(name): + return 'Hello, %s!' % name + %} + ${greeting('world')} + +This will produce the following output:: + + Hello, world! + + +Code blocks can import modules, define classes and functions, and basically do +anything you can do in normal Python code. What code blocks can *not* do is to +produce content that is emitted directly tp the generated output. + +.. note:: Using the ``print`` statement will print to the standard output + stream, just as it does for other Python code in your application. + +Unlike expressions, Python code in ```` processing instructions can +not use item and attribute access in an interchangeable manner. That means that +“dotted notation” is always attribute access, and vice-versa. + +The support for Python code blocks in templates is not supposed to encourage +mixing application code into templates, which is generally considered bad +design. If you're using many code blocks, that may be a sign that you should +move such code into separate Python modules. + +If you'd rather not allow the use of Python code blocks in templates, you can +simply set the ``allow_exec`` parameter (available on the ``Template`` and the +``TemplateLoader`` initializers) to ``False``. In that case Genshi will raise +a syntax error when a ```` processing instruction is encountered. +But please note that disallowing code blocks in templates does not turn Genshi +into a sandboxable template engine; there are sufficient ways to do harm even +using plain expressions. + + +.. _`error handling`: + +Error Handling +============== + +By default, Genshi raises an ``UndefinedError`` if a template expression +attempts to access a variable that is not defined: + +.. code-block:: pycon + + >>> from genshi.template import MarkupTemplate + >>> tmpl = MarkupTemplate('
${doh}
') + >>> tmpl.generate().render('xhtml') + Traceback (most recent call last): + ... + UndefinedError: "doh" not defined + +You can change this behavior by setting the variable lookup mode to "lenient". +In that case, accessing undefined variables returns an `Undefined` object, +meaning that the expression does not fail immediately. See below for details. + +If you need to check whether a variable exists in the template context, use the +defined_ or the value_of_ function described below. To check for existence of +attributes on an object, or keys in a dictionary, use the ``hasattr()``, +``getattr()`` or ``get()`` functions, or the ``in`` operator, just as you would +in regular Python code: + + >>> from genshi.template import MarkupTemplate + >>> tmpl = MarkupTemplate('
${defined("doh")}
') + >>> print(tmpl.generate().render('xhtml')) +
False
+ +.. note:: Lenient error handling was the default in Genshi prior to version 0.5. + Strict mode was introduced in version 0.4, and became the default in + 0.5. The reason for this change was that the lenient error handling + was masking actual errors in templates, thereby also making it harder + to debug some problems. + + +.. _`lenient`: + +Lenient Mode +------------ + +If you instruct Genshi to use the lenient variable lookup mode, it allows you +to access variables that are not defined, without raising an ``UndefinedError``. + +This mode can be chosen by passing the ``lookup='lenient'`` keyword argument to +the template initializer, or by passing the ``variable_lookup='lenient'`` +keyword argument to the ``TemplateLoader`` initializer: + +.. code-block:: pycon + + >>> from genshi.template import MarkupTemplate + >>> tmpl = MarkupTemplate('
${doh}
', lookup='lenient') + >>> print(tmpl.generate().render('xhtml')) +
+ +You *will* however get an exception if you try to call an undefined variable, or +do anything else with it, such as accessing its attributes: + +.. code-block:: pycon + + >>> from genshi.template import MarkupTemplate + >>> tmpl = MarkupTemplate('
${doh.oops}
', lookup='lenient') + >>> print(tmpl.generate().render('xhtml')) + Traceback (most recent call last): + ... + UndefinedError: "doh" not defined + +If you need to know whether a variable is defined, you can check its type +against the ``Undefined`` class, for example in a conditional directive: + +.. code-block:: pycon + + >>> from genshi.template import MarkupTemplate + >>> tmpl = MarkupTemplate('
${type(doh) is not Undefined}
', + ... lookup='lenient') + >>> print(tmpl.generate().render('xhtml')) +
False
+ +Alternatively, the built-in functions defined_ or value_of_ can be used in this +case. + +Custom Modes +------------ + +In addition to the built-in "lenient" and "strict" modes, it is also possible to +use a custom error handling mode. For example, you could use lenient error +handling in a production environment, while also logging a warning when an +undefined variable is referenced. + +See the API documentation of the ``genshi.template.eval`` module for details. + + +Built-in Functions & Types +========================== + +The following functions and types are available by default in template code, in +addition to the standard built-ins that are available to all Python code. + +.. _`defined`: + +``defined(name)`` +----------------- +This function determines whether a variable of the specified name exists in +the context data, and returns ``True`` if it does. + +.. _`value_of`: + +``value_of(name, default=None)`` +-------------------------------- +This function returns the value of the variable with the specified name if +such a variable is defined, and returns the value of the ``default`` +parameter if no such variable is defined. + +.. _`Markup`: + +``Markup(text)`` +---------------- +The ``Markup`` type marks a given string as being safe for inclusion in markup, +meaning it will *not* be escaped in the serialization stage. Use this with care, +as not escaping a user-provided string may allow malicious users to open your +web site to cross-site scripting attacks. + +.. _`Undefined`: + +``Undefined`` +---------------- +The ``Undefined`` type can be used to check whether a reference variable is +defined, as explained in `error handling`_. + + +.. _`directives`: + +------------------- +Template Directives +------------------- + +Directives provide control flow functionality for templates, such as conditions +or iteration. As the syntax for directives depends on whether you're using +markup or text templates, refer to the +`XML Template Language `_ or +`Text Template Language `_ pages for information. diff --git a/lib/genshi/doc/text-templates.txt b/lib/genshi/doc/text-templates.txt new file mode 100644 --- /dev/null +++ b/lib/genshi/doc/text-templates.txt @@ -0,0 +1,360 @@ +.. -*- mode: rst; encoding: utf-8 -*- + +============================= +Genshi Text Template Language +============================= + +In addition to the XML-based template language, Genshi provides a simple +text-based template language, intended for basic plain text generation needs. +The language is similar to the Django_ template language. + +This document describes the template language and will be most useful as +reference to those developing Genshi text templates. Templates are text files of +some kind that include processing directives_ that affect how the template is +rendered, and template expressions that are dynamically substituted by +variable data. + +See `Genshi Templating Basics `_ for general information on +embedding Python code in templates. + +.. note:: Actually, Genshi currently has two different syntaxes for text + templates languages: One implemented by the class ``OldTextTemplate`` + and another implemented by ``NewTextTemplate``. This documentation + concentrates on the latter, which is planned to completely replace the + older syntax. The older syntax is briefly described under legacy_. + +.. _django: http://www.djangoproject.com/ + +.. contents:: Contents + :depth: 3 +.. sectnum:: + + +.. _`directives`: + +------------------- +Template Directives +------------------- + +Directives are template commands enclosed by ``{% ... %}`` characters. They can +affect how the template is rendered in a number of ways: Genshi provides +directives for conditionals and looping, among others. + +Each directive must be terminated using an ``{% end %}`` marker. You can add +a string inside the ``{% end %}`` marker, for example to document which +directive is being closed, or even the expression associated with that +directive. Any text after ``end`` inside the delimiters is ignored, and +effectively treated as a comment. + +If you want to include a literal delimiter in the output, you need to escape it +by prepending a backslash character (``\``). + + +Conditional Sections +==================== + +.. _`if`: + +``{% if %}`` +------------ + +The content is only rendered if the expression evaluates to a truth value: + +.. code-block:: genshitext + + {% if foo %} + ${bar} + {% end %} + +Given the data ``foo=True`` and ``bar='Hello'`` in the template context, this +would produce:: + + Hello + + +.. _`choose`: +.. _`when`: +.. _`otherwise`: + +``{% choose %}`` +---------------- + +The ``choose`` directive, in combination with the directives ``when`` and +``otherwise``, provides advanced contional processing for rendering one of +several alternatives. The first matching ``when`` branch is rendered, or, if +no ``when`` branch matches, the ``otherwise`` branch is be rendered. + +If the ``choose`` directive has no argument the nested ``when`` directives will +be tested for truth: + +.. code-block:: genshitext + + The answer is: + {% choose %} + {% when 0 == 1 %}0{% end %} + {% when 1 == 1 %}1{% end %} + {% otherwise %}2{% end %} + {% end %} + +This would produce the following output:: + + The answer is: + 1 + +If the ``choose`` does have an argument, the nested ``when`` directives will +be tested for equality to the parent ``choose`` value: + +.. code-block:: genshitext + + The answer is: + {% choose 1 %}\ + {% when 0 %}0{% end %}\ + {% when 1 %}1{% end %}\ + {% otherwise %}2{% end %}\ + {% end %} + +This would produce the following output:: + + The answer is: + 1 + + +Looping +======= + +.. _`for`: + +``{% for %}`` +------------- + +The content is repeated for every item in an iterable: + +.. code-block:: genshitext + + Your items: + {% for item in items %}\ + * ${item} + {% end %} + +Given ``items=[1, 2, 3]`` in the context data, this would produce:: + + Your items + * 1 + * 2 + * 3 + + +Snippet Reuse +============= + +.. _`def`: +.. _`macros`: + +``{% def %}`` +------------- + +The ``def`` directive can be used to create macros, i.e. snippets of template +text that have a name and optionally some parameters, and that can be inserted +in other places: + +.. code-block:: genshitext + + {% def greeting(name) %} + Hello, ${name}! + {% end %} + ${greeting('world')} + ${greeting('everyone else')} + +The above would be rendered to:: + + Hello, world! + Hello, everyone else! + +If a macro doesn't require parameters, it can be defined without the +parenthesis. For example: + +.. code-block:: genshitext + + {% def greeting %} + Hello, world! + {% end %} + ${greeting()} + +The above would be rendered to:: + + Hello, world! + + +.. _includes: +.. _`include`: + +``{% include %}`` +----------------- + +To reuse common parts of template text across template files, you can include +other files using the ``include`` directive: + +.. code-block:: genshitext + + {% include base.txt %} + +Any content included this way is inserted into the generated output. The +included template sees the context data as it exists at the point of the +include. `Macros`_ in the included template are also available to the including +template after the point it was included. + +Include paths are relative to the filename of the template currently being +processed. So if the example above was in the file "``myapp/mail.txt``" +(relative to the template search path), the include directive would look for +the included file at "``myapp/base.txt``". You can also use Unix-style +relative paths, for example "``../base.txt``" to look in the parent directory. + +Just like other directives, the argument to the ``include`` directive accepts +any Python expression, so the path to the included template can be determined +dynamically: + +.. code-block:: genshitext + + {% include ${'%s.txt' % filename} %} + +Note that a ``TemplateNotFound`` exception is raised if an included file can't +be found. + +.. note:: The include directive for text templates was added in Genshi 0.5. + + +Variable Binding +================ + +.. _`with`: + +``{% with %}`` +-------------- + +The ``{% with %}`` directive lets you assign expressions to variables, which can +be used to make expressions inside the directive less verbose and more +efficient. For example, if you need use the expression ``author.posts`` more +than once, and that actually results in a database query, assigning the results +to a variable using this directive would probably help. + +For example: + +.. code-block:: genshitext + + Magic numbers! + {% with y=7; z=x+10 %} + $x $y $z + {% end %} + +Given ``x=42`` in the context data, this would produce:: + + Magic numbers! + 42 7 52 + +Note that if a variable of the same name already existed outside of the scope +of the ``with`` directive, it will **not** be overwritten. Instead, it will +have the same value it had prior to the ``with`` assignment. Effectively, +this means that variables are immutable in Genshi. + + +.. _whitespace: + +--------------------------- +White-space and Line Breaks +--------------------------- + +Note that space or line breaks around directives is never automatically removed. +Consider the following example: + +.. code-block:: genshitext + + {% for item in items %} + {% if item.visible %} + ${item} + {% end %} + {% end %} + +This will result in two empty lines above and beneath every item, plus the +spaces used for indentation. If you want to supress a line break, simply end +the line with a backslash: + +.. code-block:: genshitext + + {% for item in items %}\ + {% if item.visible %}\ + ${item} + {% end %}\ + {% end %}\ + +Now there would be no empty lines between the items in the output. But you still +get the spaces used for indentation, and because the line breaks are removed, +they actually continue and add up between lines. There are numerous ways to +control white-space in the output while keeping the template readable, such as +moving the indentation into the delimiters, or moving the end delimiter on the +next line, and so on. + + +.. _comments: + +-------- +Comments +-------- + +Parts in templates can be commented out using the delimiters ``{# ... #}``. +Any content in comments are removed from the output. + +.. code-block:: genshitext + + {# This won't end up in the output #} + This will. + +Just like directive delimiters, these can be escaped by prefixing with a +backslash. + +.. code-block:: genshitext + + \{# This *will* end up in the output, including delimiters #} + This too. + + +.. _legacy: + +--------------------------- +Legacy Text Template Syntax +--------------------------- + +The syntax for text templates was redesigned in version 0.5 of Genshi to make +the language more flexible and powerful. The older syntax is based on line +starting with dollar signs, similar to e.g. Cheetah_ or Velocity_. + +.. _cheetah: http://cheetahtemplate.org/ +.. _velocity: http://jakarta.apache.org/velocity/ + +A simple template using the old syntax looked like this: + +.. code-block:: genshitext + + Dear $name, + + We have the following items for you: + #for item in items + * $item + #end + + All the best, + Foobar + +Beyond the requirement of putting directives on separate lines prefixed with +dollar signs, the language itself is very similar to the new one. Except that +comments are lines that start with two ``#`` characters, and a line-break at the +end of a directive is removed automatically. + +.. note:: If you're using this old syntax, it is strongly recommended to + migrate to the new syntax. Simply replace any references to + ``TextTemplate`` by ``NewTextTemplate`` (and also change the + text templates, of course). On the other hand, if you want to stick + with the old syntax for a while longer, replace references to + ``TextTemplate`` by ``OldTextTemplate``; while ``TextTemplate`` is + still an alias for the old language at this point, that will change + in a future release. But also note that the old syntax may be + dropped entirely in a future release. diff --git a/lib/genshi/doc/upgrade.txt b/lib/genshi/doc/upgrade.txt new file mode 100644 --- /dev/null +++ b/lib/genshi/doc/upgrade.txt @@ -0,0 +1,176 @@ +================ +Upgrading Genshi +================ + + +.. contents:: Contents + :depth: 2 +.. sectnum:: + +------------------------------------------------------ +Upgrading from Genshi 0.6.x to the development version +------------------------------------------------------ + +The Genshi development version now supports both Python 2 and Python 3. + +The most noticable API change in the Genshi development version is that the +default encoding in numerous places is now None (i.e. unicode) instead +of UTF-8. This change was made in order to ease the transition to Python 3 +where strings are unicode strings by default. + +If your application relies on the default UTF-8 encoding a simple way to +have it work both with Genshi 0.6.x and the development version is to specify the +encoding explicitly in calls to the following classes, methods and functions: + +* genshi.HTML +* genshi.Stream.render +* genshi.input.HTMLParser +* genshi.template.MarkupTemplate +* genshi.template.TemplateLoader +* genshi.template.TextTemplate (and genshi.template.NewTextTemplate) +* genshi.template.OldTextTemplate + +Whether you explicitly specify UTF-8 or explicitly specify None (unicode) is +a matter of personal taste, although working with unicode may make porting +your own application to Python 3 easier. + + +------------------------------------ +Upgrading from Genshi 0.5.x to 0.6.x +------------------------------------ + +Required Python Version +----------------------- + +Support for Python 2.3 has been dropped in this release. Python 2.4 is +now the minimum version of Python required to run Genshi. + +The XPath engine has been completely overhauled for this version. Some +patterns that previously matched incorrectly will no longer match, and +the other way around. In such cases, the XPath expressions need to be +fixed in your application and templates. + + +------------------------------------ +Upgrading from Genshi 0.4.x to 0.5.x +------------------------------------ + +Error Handling +-------------- + +The default error handling mode has been changed to "strict". This +means that accessing variables not defined in the template data will +now generate an immediate exception, as will accessing object +attributes or dictionary keys that don't exist. If your templates rely +on the old lenient behavior, you can configure Genshi to use that +instead. See the documentation for details on how to do that. But be +warned that lenient error handling may be removed completely in a +future release. + +Match Template Processing +------------------------- + +There has also been a subtle change to how ``py:match`` templates are +processed: in previous versions, all match templates would be applied +to the content generated by the matching template, and only the +matching template itself was applied recursively to the original +content. This behavior resulted in problems with many kinds of +recursive matching, and hence was changed for 0.5: now, all match +templates declared before the matching template are applied to the +original content, and match templates declared after the matching +template are applied to the generated content. This change should not +have any effect on most applications, but you may want to check your +use of match templates to make sure. + +Text Templates +-------------- + +Genshi 0.5 introduces a new, alternative syntax for text templates, +which is more flexible and powerful compared to the old syntax. For +backwards compatibility, this new syntax is not used by default, +though it will be in a future version. It is recommended that you +migrate to using this new syntax. To do so, simply rename any +references in your code to ``TextTemplate`` to ``NewTextTemplate``. To +explicitly use the old syntax, use ``OldTextTemplate`` instead, so +that you can be sure you'll be using the same language when the +default in Genshi is changed (at least until the old implementation is +completely removed). + +``Markup`` Constructor +---------------------- + +The ``Markup`` class no longer has a specialized constructor. The old +(undocumented) constructor provided a shorthand for doing positional +substitutions. If you have code like this: + +.. code-block:: python + + Markup('%s', name) + +You must replace it by the more explicit: + +.. code-block:: python + + Markup('%s') % name + +``Template`` Constructor +------------------------ + +The constructor of the ``Template`` class and its subclasses has changed +slightly: instead of the optional ``basedir`` parameter, it now expects +an (also optional) ``filepath`` parameter, which specifies the absolute +path to the template. You probably aren't using those constructors +directly, anyway, but using the ``TemplateLoader`` API instead. + + +------------------------------------ +Upgrading from Genshi 0.3.x to 0.4.x +------------------------------------ + +The modules ``genshi.filters`` and ``genshi.template`` have been +refactored into packages containing multiple modules. While code using +the regular APIs should continue to work without problems, you should +make sure to remove any leftover traces of the files ``filters.py`` +and ``template.py`` in the ``genshi`` package on the installation +path (including the corresponding ``.pyc`` files). This is not +necessary when Genshi was installed as a Python egg. + +Results of evaluating template expressions are no longer implicitly +called if they are callable. If you have been using that feature, you +will need to add the parenthesis to actually call the function. + +Instances of ``genshi.core.Attrs`` are now immutable. Filters +manipulating the attributes in a stream may need to be updated. Also, +the ``Attrs`` class no longer automatically wraps all attribute names +in ``QName`` objects, so users of the ``Attrs`` class need to do this +themselves. See the documentation of the ``Attrs`` class for more +information. + + +--------------------- +Upgrading from Markup +--------------------- + +Prior to version 0.3, the name of the Genshi project was "Markup". The +name change means that you will have to adjust your import statements +and the namespace URI of XML templates, among other things: + +* The package name was changed from "markup" to "genshi". Please + adjust any import statements referring to the old package name. +* The namespace URI for directives in Genshi XML templates has changed + from ``http://markup.edgewall.org/`` to + ``http://genshi.edgewall.org/``. Please update the ``xmlns:py`` + declaration in your template files accordingly. + +Furthermore, due to the inclusion of a text-based template language, +the class:: + + markup.template.Template + +has been renamed to:: + + genshi.template.MarkupTemplate + +If you've been using the Template class directly, you'll need to +update your code (a simple find/replace should do—the API itself +did not change). diff --git a/lib/genshi/doc/xml-templates.txt b/lib/genshi/doc/xml-templates.txt new file mode 100644 --- /dev/null +++ b/lib/genshi/doc/xml-templates.txt @@ -0,0 +1,702 @@ +.. -*- mode: rst; encoding: utf-8 -*- + +============================ +Genshi XML Template Language +============================ + +Genshi provides a XML-based template language that is heavily inspired by Kid_, +which in turn was inspired by a number of existing template languages, namely +XSLT_, TAL_, and PHP_. + +.. _kid: http://kid-templating.org/ +.. _python: http://www.python.org/ +.. _xslt: http://www.w3.org/TR/xslt +.. _tal: http://www.zope.org/Wikis/DevSite/Projects/ZPT/TAL +.. _php: http://www.php.net/ + +This document describes the template language and will be most useful as +reference to those developing Genshi XML templates. Templates are XML files of +some kind (such as XHTML) that include processing directives_ (elements or +attributes identified by a separate namespace) that affect how the template is +rendered, and template expressions that are dynamically substituted by +variable data. + +See `Genshi Templating Basics `_ for general information on +embedding Python code in templates. + + +.. contents:: Contents + :depth: 3 +.. sectnum:: + + +.. _`directives`: + +------------------- +Template Directives +------------------- + +Directives are elements and/or attributes in the template that are identified +by the namespace ``http://genshi.edgewall.org/``. They can affect how the +template is rendered in a number of ways: Genshi provides directives for +conditionals and looping, among others. + +To use directives in a template, the namespace must be declared, which is +usually done on the root element: + +.. code-block:: genshi + + + ... + + +In this example, the default namespace is set to the XHTML namespace, and the +namespace for Genshi directives is bound to the prefix “py”. + +All directives can be applied as attributes, and some can also be used as +elements. The ``if`` directives for conditionals, for example, can be used in +both ways: + +.. code-block:: genshi + + + ... +
+
Bar
+
+ ... + + +This is basically equivalent to the following: + +.. code-block:: genshi + + + ... + +
+
Bar
+
+ + ... + + +The rationale behind the second form is that directives do not always map +naturally to elements in the template. In such cases, the ``py:strip`` +directive can be used to strip off the unwanted element, or the directive can +simply be used as an element. + + +Conditional Sections +==================== + +.. _`py:if`: + +``py:if`` +--------- + +The element and its content is only rendered if the expression evaluates to a +truth value: + +.. code-block:: genshi + +
+ ${bar} +
+ +Given the data ``foo=True`` and ``bar='Hello'`` in the template context, this +would produce: + +.. code-block:: xml + +
+ Hello +
+ +But setting ``foo=False`` would result in the following output: + +.. code-block:: xml + +
+
+ +This directive can also be used as an element: + +.. code-block:: genshi + +
+ + ${bar} + +
+ +.. _`py:choose`: +.. _`py:when`: +.. _`py:otherwise`: + +``py:choose`` +------------- + +The ``py:choose`` directive, in combination with the directives ``py:when`` +and ``py:otherwise`` provides advanced conditional processing for rendering one +of several alternatives. The first matching ``py:when`` branch is rendered, or, +if no ``py:when`` branch matches, the ``py:otherwise`` branch is rendered. + +If the ``py:choose`` directive is empty the nested ``py:when`` directives will +be tested for truth: + +.. code-block:: genshi + +
+ 0 + 1 + 2 +
+ +This would produce the following output: + +.. code-block:: xml + +
+ 1 +
+ +If the ``py:choose`` directive contains an expression the nested ``py:when`` +directives will be tested for equality to the parent ``py:choose`` value: + +.. code-block:: genshi + +
+ 0 + 1 + 2 +
+ +This would produce the following output: + +.. code-block:: xml + +
+ 1 +
+ +These directives can also be used as elements: + +.. code-block:: genshi + + + 0 + 1 + 2 + + +Looping +======= + +.. _`py:for`: + +``py:for`` +---------- + +The element is repeated for every item in an iterable: + +.. code-block:: genshi + +
+
${item}
+
+ +Given ``items=[1, 2, 3]`` in the context data, this would produce: + +.. code-block:: xml + +
+
1
2
3
+
+ +This directive can also be used as an element: + +.. code-block:: genshi + +
+ +
${item}
+ +
+ + +Snippet Reuse +============= + +.. _`py:def`: +.. _`macros`: + +``py:def`` +---------- + +The ``py:def`` directive can be used to create macros, i.e. snippets of +template code that have a name and optionally some parameters, and that can be +inserted in other places: + +.. code-block:: genshi + +
+
+ Hello, ${name}! +
+ ${greeting('world')} + ${greeting('everyone else')} +
+ +The above would be rendered to: + +.. code-block:: xml + +
+
+ Hello, world! +
+
+ Hello, everyone else! +
+
+ +If a macro doesn't require parameters, it can be defined without the +parenthesis. For example: + +.. code-block:: genshi + +
+
+ Hello, world! +
+ ${greeting()} +
+ +The above would be rendered to: + +.. code-block:: xml + +
+
+ Hello, world! +
+
+ +This directive can also be used as an element: + +.. code-block:: genshi + +
+ +
Hello, ${name}!
+ +
+ + +.. _Match Templates: +.. _`py:match`: + +``py:match`` +------------ + +This directive defines a *match template*: given an XPath expression, it +replaces any element in the template that matches the expression with its own +content. + +For example, the match template defined in the following template matches any +element with the tag name “greeting”: + +.. code-block:: genshi + +
+ + Hello ${select('@name')} + + +
+ +This would result in the following output: + +.. code-block:: xml + +
+ + Hello Dude + +
+ +Inside the body of a ``py:match`` directive, the ``select(path)`` function is +made available so that parts or all of the original element can be incorporated +in the output of the match template. See `Using XPath`_ for more information +about this function. + +.. _`Using XPath`: streams.html#using-xpath + +Match templates are applied both to the original markup as well to the +generated markup. The order in which they are applied depends on the order +they are declared in the template source: a match template defined after +another match template is applied to the output generated by the first match +template. The match templates basically form a pipeline. + +This directive can also be used as an element: + +.. code-block:: genshi + +
+ + Hello ${select('@name')} + + +
+ +When used this way, the ``py:match`` directive can also be annotated with a +couple of optimization hints. For example, the following informs the matching +engine that the match should only be applied once: + +.. code-block:: genshi + + + +
...
+ ${select("*|text()")} +
...
+ + + +The following optimization hints are recognized: + ++---------------+-----------+-----------------------------------------------+ +| Attribute | Default | Description | ++===============+===========+===============================================+ +| ``buffer`` | ``true`` | Whether the matched content should be | +| | | buffered in memory. Buffering can improve | +| | | performance a bit at the cost of needing more | +| | | memory during rendering. Buffering is | +| | | ''required'' for match templates that contain | +| | | more than one invocation of the ``select()`` | +| | | function. If there is only one call, and the | +| | | matched content can potentially be very long, | +| | | consider disabling buffering to avoid | +| | | excessive memory use. | ++---------------+-----------+-----------------------------------------------+ +| ``once`` | ``false`` | Whether the engine should stop looking for | +| | | more matching elements after the first match. | +| | | Use this on match templates that match | +| | | elements that can only occur once in the | +| | | stream, such as the ```` or ```` | +| | | elements in an HTML template, or elements | +| | | with a specific ID. | ++---------------+-----------+-----------------------------------------------+ +| ``recursive`` | ``true`` | Whether the match template should be applied | +| | | to its own output. Note that ``once`` implies | +| | | non-recursive behavior, so this attribute | +| | | only needs to be set for match templates that | +| | | don't also have ``once`` set. | ++---------------+-----------+-----------------------------------------------+ + +.. note:: The ``py:match`` optimization hints were added in the 0.5 release. In + earlier versions, the attributes have no effect. + + +Variable Binding +================ + +.. _`with`: + +``py:with`` +----------- + +The ``py:with`` directive lets you assign expressions to variables, which can +be used to make expressions inside the directive less verbose and more +efficient. For example, if you need use the expression ``author.posts`` more +than once, and that actually results in a database query, assigning the results +to a variable using this directive would probably help. + +For example: + +.. code-block:: genshi + +
+ $x $y $z +
+ +Given ``x=42`` in the context data, this would produce: + +.. code-block:: xml + +
+ 42 7 52 +
+ +This directive can also be used as an element: + +.. code-block:: genshi + +
+ $x $y $z +
+ +Note that if a variable of the same name already existed outside of the scope +of the ``py:with`` directive, it will **not** be overwritten. Instead, it +will have the same value it had prior to the ``py:with`` assignment. +Effectively, this means that variables are immutable in Genshi. + + +Structure Manipulation +====================== + +.. _`py:attrs`: + +``py:attrs`` +------------ + +This directive adds, modifies or removes attributes from the element: + +.. code-block:: genshi + +
+
Bar
+
+ +Given ``foo={'class': 'collapse'}`` in the template context, this would +produce: + +.. code-block:: xml + +
+
Bar
+
+ +Attributes with the value ``None`` are omitted, so given ``foo={'class': None}`` +in the context for the same template this would produce: + +.. code-block:: xml + +
+
Bar
+
+ +This directive can only be used as an attribute. + + +.. _`py:content`: + +``py:content`` +-------------- + +This directive replaces any nested content with the result of evaluating the +expression: + +.. code-block:: genshi + +
+
Hello
+
+ +Given ``bar='Bye'`` in the context data, this would produce: + +.. code-block:: xml + +
+
Bye
+
+ +This directive can only be used as an attribute. + + +.. _`py:replace`: + +``py:replace`` +-------------- + +This directive replaces the element itself with the result of evaluating the +expression: + +.. code-block:: genshi + +
+ Hello +
+ +Given ``bar='Bye'`` in the context data, this would produce: + +.. code-block:: xml + +
+ Bye +
+ +This directive can also be used as an element (since version 0.5): + +.. code-block:: genshi + +
+ Placeholder +
+ + + +.. _`py:strip`: + +``py:strip`` +------------ + +This directive conditionally strips the top-level element from the output. When +the value of the ``py:strip`` attribute evaluates to ``True``, the element is +stripped from the output: + +.. code-block:: genshi + +
+
foo
+
+ +This would be rendered as: + +.. code-block:: xml + +
+ foo +
+ +As a shorthand, if the value of the ``py:strip`` attribute is empty, that has +the same effect as using a truth value (i.e. the element is stripped). + + +.. _order: + +Processing Order +================ + +It is possible to attach multiple directives to a single element, although not +all combinations make sense. When multiple directives are encountered, they are +processed in the following order: + +#. `py:def`_ +#. `py:match`_ +#. `py:when`_ +#. `py:otherwise`_ +#. `py:for`_ +#. `py:if`_ +#. `py:choose`_ +#. `py:with`_ +#. `py:replace`_ +#. `py:content`_ +#. `py:attrs`_ +#. `py:strip`_ + + +.. _includes: + +-------- +Includes +-------- + +To reuse common snippets of template code, you can include other files using +XInclude_. + +.. _xinclude: http://www.w3.org/TR/xinclude/ + +For this, you need to declare the XInclude namespace (commonly bound to the +prefix “xi”) and use the ```` element where you want the external +file to be pulled in: + +.. code-block:: genshi + + + + ... + + +Include paths are relative to the filename of the template currently being +processed. So if the example above was in the file "``myapp/index.html``" +(relative to the template search path), the XInclude processor would look for +the included file at "``myapp/base.html``". You can also use Unix-style +relative paths, for example "``../base.html``" to look in the parent directory. + +Any content included this way is inserted into the generated output instead of +the ```` element. The included template sees the same context data. +`Match templates`_ and `macros`_ in the included template are also available to +the including template after the point it was included. + +By default, an error will be raised if an included file is not found. If that's +not what you want, you can specify fallback content that should be used if the +include fails. For example, to to make the include above fail silently, you'd +write: + +.. code-block:: genshi + + + +See the `XInclude specification`_ for more about fallback content. Note though +that Genshi currently only supports a small subset of XInclude. + +.. _`xinclude specification`: http://www.w3.org/TR/xinclude/ + + +Dynamic Includes +================ + +Incudes in Genshi are fully dynamic: Just like normal attributes, the `href` +attribute accepts expressions, and directives_ can be used on the +```` element just as on any other element, meaning you can do +things like conditional includes: + +.. code-block:: genshi + + + + +Including Text Templates +======================== + +The ``parse`` attribute of the ```` element can be used to specify +whether the included template is an XML template or a text template (using the +new syntax added in Genshi 0.5): + +.. code-block:: genshi + + + +This example would load the ``myscript.js`` file as a ``NewTextTemplate``. See +`text templates`_ for details on the syntax of text templates. + +.. _`text templates`: text-templates.html + + +.. _comments: + +-------- +Comments +-------- + +Normal XML/HTML comment syntax can be used in templates: + +.. code-block:: genshi + + + +However, such comments get passed through the processing pipeline and are by +default included in the final output. If that's not desired, prefix the comment +text with an exclamation mark: + +.. code-block:: genshi + + + +Note that it does not matter whether there's whitespace before or after the +exclamation mark, so the above could also be written as follows: + +.. code-block:: genshi + + diff --git a/lib/genshi/doc/xpath.txt b/lib/genshi/doc/xpath.txt new file mode 100644 --- /dev/null +++ b/lib/genshi/doc/xpath.txt @@ -0,0 +1,101 @@ +.. -*- mode: rst; encoding: utf-8 -*- + +===================== +Using XPath in Genshi +===================== + +Genshi provides basic XPath_ support for matching and querying event streams. + +.. _xpath: http://www.w3.org/TR/xpath + + +.. contents:: Contents + :depth: 2 +.. sectnum:: + + +----------- +Limitations +----------- + +Due to the streaming nature of the processing model, Genshi uses only a subset +of the `XPath 1.0`_ language. + +.. _`XPath 1.0`: http://www.w3.org/TR/xpath + +In particular, only the following axes are supported: + +* ``attribute`` +* ``child`` +* ``descendant`` +* ``descendant-or-self`` +* ``self`` + +This means you can't use the ``parent``, ancestor, or sibling axes in Genshi +(the ``namespace`` axis isn't supported either, but what you'd ever need that +for I don't know). Basically, any path expression that would require buffering +of the stream is not supported. + +Predicates are of course supported, but path expressions *inside* predicates +are restricted to attribute lookups (again due to the lack of buffering). + +Most of the XPath functions and operators are supported, however they +(currently) only work inside predicates. The following functions are **not** +supported: + +* ``count()`` +* ``id()`` +* ``lang()`` +* ``last()`` +* ``position()`` +* ``string()`` +* ``sum()`` + +The mathematical operators (``+``, ``-``, ``*``, ``div``, and ``mod``) are not +yet supported, whereas sub-expressions and the various comparison and logical +operators should work as expected. + +You can also use XPath variable references (``$var``) inside predicates. + + +---------------- +Querying Streams +---------------- + +The ``Stream`` class provides a ``select(path)`` function that can be used to +retrieve subsets of the stream: + +.. code-block:: pycon + + >>> from genshi.input import XML + + >>> doc = XML(''' + ... + ... + ...
Foo
+ ... + ... + ...
Bar
+ ... + ... + ...
Baz
+ ... + ... + ...
Waz
+ ... + ... + ... ''') + + >>> print(doc.select('items/item[@status="closed" and ' + ... '(@resolution="invalid" or not(@resolution))]/summary/text()')) + BarBaz + + + +--------------------- +Matching in Templates +--------------------- + +See the directive ``py:match`` in the `XML Template Language Specification`_. + +.. _`XML Template Language Specification`: xml-templates.html diff --git a/lib/genshi/examples/basic/kidrun.py b/lib/genshi/examples/basic/kidrun.py new file mode 100755 --- /dev/null +++ b/lib/genshi/examples/basic/kidrun.py @@ -0,0 +1,48 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +import os +import sys +import time + +import kid + +def test(): + base_path = os.path.dirname(os.path.abspath(__file__)) + kid.path = kid.TemplatePath([base_path]) + + ctxt = dict(hello='', hey='ZYX', bozz=None, + items=['Number %d' % num for num in range(1, 15)], + prefix='#') + + start = time.clock() + template = kid.Template(file='test.kid', **ctxt) + print ' --> parse stage: %.4f ms' % ((time.clock() - start) * 1000) + + for output in template.generate(): + sys.stdout.write(output) + print + + times = [] + for i in range(1000): + start = time.clock() + list(template.generate()) + times.append(time.clock() - start) + sys.stdout.write('.') + sys.stdout.flush() + print + + print ' --> render stage: %s ms (average)' % ( + (sum(times) / len(times) * 1000)) + +if __name__ == '__main__': + if '-p' in sys.argv: + import hotshot, hotshot.stats + prof = hotshot.Profile("template.prof") + benchtime = prof.runcall(test) + stats = hotshot.stats.load("template.prof") + stats.strip_dirs() + stats.sort_stats('time', 'calls') + stats.print_stats() + else: + test() diff --git a/lib/genshi/examples/basic/layout.html b/lib/genshi/examples/basic/layout.html new file mode 100644 --- /dev/null +++ b/lib/genshi/examples/basic/layout.html @@ -0,0 +1,15 @@ +
+ + Hello ${hello} + + +
reference me, please
+
+ Hello ${name.title()} +
+ + Hello ${select('@name')} + + +
diff --git a/lib/genshi/examples/basic/layout.kid b/lib/genshi/examples/basic/layout.kid new file mode 100644 --- /dev/null +++ b/lib/genshi/examples/basic/layout.kid @@ -0,0 +1,15 @@ +
+ + Hello ${hello} + + +
reference me, please
+
+ Hello ${name.title()} +
+ + Hello ${item.get('name')} + + +
diff --git a/lib/genshi/examples/basic/run.py b/lib/genshi/examples/basic/run.py new file mode 100755 --- /dev/null +++ b/lib/genshi/examples/basic/run.py @@ -0,0 +1,46 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +import os +import sys +import time + +from genshi.template import TemplateLoader + +def test(): + base_path = os.path.dirname(os.path.abspath(__file__)) + loader = TemplateLoader([base_path], auto_reload=True) + + start = time.clock() + tmpl = loader.load('test.html') + print ' --> parse stage: %.4f ms' % ((time.clock() - start) * 1000) + + data = dict(hello='', skin='default', hey='ZYX', bozz=None, + items=['Number %d' % num for num in range(1, 15)], + prefix='#') + + print tmpl.generate(**data).render(method='html') + + times = [] + for i in range(1000): + start = time.clock() + list(tmpl.generate(**data)) + times.append(time.clock() - start) + sys.stdout.write('.') + sys.stdout.flush() + print + + print ' --> render stage: %s ms (average)' % ( + (sum(times) / len(times) * 1000)) + +if __name__ == '__main__': + if '-p' in sys.argv: + import hotshot, hotshot.stats + prof = hotshot.Profile("template.prof") + benchtime = prof.runcall(test) + stats = hotshot.stats.load("template.prof") + stats.strip_dirs() + stats.sort_stats('time', 'calls') + stats.print_stats() + else: + test() diff --git a/lib/genshi/examples/basic/test.html b/lib/genshi/examples/basic/test.html new file mode 100644 --- /dev/null +++ b/lib/genshi/examples/basic/test.html @@ -0,0 +1,22 @@ + + + + + +
+
Item $prefix${item.split()[-1]}
+
+ ${macro1()} ${macro1()} ${macro1()} + ${macro2('john')} + ${macro2('kate', classname='collapsed')} +
Replace me
+ + + Hello Silicon + + diff --git a/lib/genshi/examples/basic/test.kid b/lib/genshi/examples/basic/test.kid new file mode 100644 --- /dev/null +++ b/lib/genshi/examples/basic/test.kid @@ -0,0 +1,21 @@ + + + +
+
Item $prefix${item.split()[-1]}
+ XYZ ${hey} +
+ ${macro1()} ${macro1()} ${macro1()} + ${macro2('john')} + ${macro2('kate', classname='collapsed')} +
Replace me
+ + + Hello Silicon + + diff --git a/lib/genshi/examples/bench/basic.py b/lib/genshi/examples/bench/basic.py new file mode 100644 --- /dev/null +++ b/lib/genshi/examples/bench/basic.py @@ -0,0 +1,204 @@ +# -*- encoding: utf-8 -*- +# Template language benchmarks +# +# Objective: Test general templating features using a small template + +from cgi import escape +import os +from StringIO import StringIO +import sys +import timeit + +__all__ = ['clearsilver', 'mako', 'django', 'kid', 'genshi', 'genshi_text', + 'simpletal'] + +def genshi(dirname, verbose=False): + from genshi.template import TemplateLoader + loader = TemplateLoader([dirname], auto_reload=False) + template = loader.load('template.html') + def render(): + data = dict(title='Just a test', user='joe', + items=['Number %d' % num for num in range(1, 15)]) + return template.generate(**data).render('xhtml') + + if verbose: + print render() + return render + +def genshi_text(dirname, verbose=False): + from genshi.core import escape + from genshi.template import TemplateLoader, NewTextTemplate + loader = TemplateLoader([dirname], auto_reload=False) + template = loader.load('template.txt', cls=NewTextTemplate) + def render(): + data = dict(escape=escape, title='Just a test', user='joe', + items=['Number %d' % num for num in range(1, 15)]) + return template.generate(**data).render('text') + + if verbose: + print render() + return render + +def mako(dirname, verbose=False): + from mako.lookup import TemplateLookup + lookup = TemplateLookup(directories=[dirname], filesystem_checks=False) + template = lookup.get_template('template.html') + def render(): + data = dict(title='Just a test', user='joe', + list_items=['Number %d' % num for num in range(1, 15)]) + return template.render(**data) + if verbose: + print render() + return render + +def cheetah(dirname, verbose=False): + # FIXME: infinite recursion somewhere... WTF? + try: + from Cheetah.Template import Template + except ImportError: + print>>sys.stderr, 'Cheetah not installed, skipping' + return lambda: None + class MyTemplate(Template): + def serverSidePath(self, path): return os.path.join(dirname, path) + filename = os.path.join(dirname, 'template.tmpl') + template = MyTemplate(file=filename) + + def render(): + template = MyTemplate(file=filename, + searchList=[{'title': 'Just a test', 'user': 'joe', + 'items': [u'Number %d' % num for num in range(1, 15)]}]) + return template.respond() + + if verbose: + print render() + return render + +def clearsilver(dirname, verbose=False): + try: + import neo_cgi + except ImportError: + print>>sys.stderr, 'ClearSilver not installed, skipping' + return lambda: None + neo_cgi.update() + import neo_util + import neo_cs + def render(): + hdf = neo_util.HDF() + hdf.setValue('hdf.loadpaths.0', dirname) + hdf.setValue('title', escape('Just a test')) + hdf.setValue('user', escape('joe')) + for num in range(1, 15): + hdf.setValue('items.%d' % (num - 1), escape('Number %d' % num)) + cs = neo_cs.CS(hdf) + cs.parseFile('template.cs') + return cs.render() + + if verbose: + print render() + return render + +def django(dirname, verbose=False): + try: + from django.conf import settings + settings.configure(TEMPLATE_DIRS=[os.path.join(dirname, 'templates')]) + except ImportError: + print>>sys.stderr, 'Django not installed, skipping' + return lambda: None + from django import template, templatetags + from django.template import loader + templatetags.__path__.append(os.path.join(dirname, 'templatetags')) + tmpl = loader.get_template('template.html') + + def render(): + data = {'title': 'Just a test', 'user': 'joe', + 'items': ['Number %d' % num for num in range(1, 15)]} + return tmpl.render(template.Context(data)) + + if verbose: + print render() + return render + +def kid(dirname, verbose=False): + try: + import kid + except ImportError: + print>>sys.stderr, "Kid not installed, skipping" + return lambda: None + kid.path = kid.TemplatePath([dirname]) + template = kid.load_template('template.kid').Template + def render(): + return template( + title='Just a test', user='joe', + items=['Number %d' % num for num in range(1, 15)] + ).serialize(output='xhtml') + + if verbose: + print render() + return render + +def simpletal(dirname, verbose=False): + try: + from simpletal import simpleTAL, simpleTALES + except ImportError: + print>>sys.stderr, "SimpleTAL not installed, skipping" + return lambda: None + fileobj = open(os.path.join(dirname, 'base.html')) + base = simpleTAL.compileHTMLTemplate(fileobj) + fileobj.close() + fileobj = open(os.path.join(dirname, 'template.html')) + template = simpleTAL.compileHTMLTemplate(fileobj) + fileobj.close() + def render(): + ctxt = simpleTALES.Context(allowPythonPath=1) + ctxt.addGlobal('base', base) + ctxt.addGlobal('title', 'Just a test') + ctxt.addGlobal('user', 'joe') + ctxt.addGlobal('items', ['Number %d' % num for num in range(1, 15)]) + buf = StringIO() + template.expand(ctxt, buf) + return buf.getvalue() + + if verbose: + print render() + return render + +def run(engines, number=2000, verbose=False): + basepath = os.path.abspath(os.path.dirname(__file__)) + for engine in engines: + dirname = os.path.join(basepath, engine) + if verbose: + print '%s:' % engine.capitalize() + print '--------------------------------------------------------' + else: + print '%s:' % engine.capitalize(), + t = timeit.Timer(setup='from __main__ import %s; render = %s(r"%s", %s)' + % (engine, engine, dirname, verbose), + stmt='render()') + time = t.timeit(number=number) / number + if verbose: + print '--------------------------------------------------------' + print '%.2f ms' % (1000 * time) + if verbose: + print '--------------------------------------------------------' + + +if __name__ == '__main__': + engines = [arg for arg in sys.argv[1:] if arg[0] != '-'] + if not engines: + engines = __all__ + + verbose = '-v' in sys.argv + + if '-p' in sys.argv: + import cProfile, pstats + prof = cProfile.Profile() + prof.run('run(%r, number=200, verbose=%r)' % (engines, verbose)) + stats = pstats.Stats(prof) + stats.strip_dirs() + stats.sort_stats('calls') + stats.print_stats(25) + if verbose: + stats.print_callees() + stats.print_callers() + else: + run(engines, verbose=verbose) diff --git a/lib/genshi/examples/bench/bigtable.py b/lib/genshi/examples/bench/bigtable.py new file mode 100644 --- /dev/null +++ b/lib/genshi/examples/bench/bigtable.py @@ -0,0 +1,233 @@ +# -*- encoding: utf-8 -*- +# Template language benchmarks +# +# Objective: Generate a 1000x10 HTML table as fast as possible. +# +# Author: Jonas Borgström + +import cgi +import sys +import timeit +from StringIO import StringIO +from genshi.builder import tag +from genshi.template import MarkupTemplate, NewTextTemplate + +try: + from elementtree import ElementTree as et +except ImportError: + et = None + +try: + import cElementTree as cet +except ImportError: + cet = None + +try: + import neo_cgi, neo_cs, neo_util +except ImportError: + neo_cgi = None + +try: + import kid +except ImportError: + kid = None + +try: + from django.conf import settings + settings.configure() + from django.template import Context as DjangoContext + from django.template import Template as DjangoTemplate +except ImportError: + DjangoContext = DjangoTemplate = None + +try: + from mako.template import Template as MakoTemplate +except ImportError: + MakoTemplate = None + +table = [dict(a=1,b=2,c=3,d=4,e=5,f=6,g=7,h=8,i=9,j=10) + for x in range(1000)] + +genshi_tmpl = MarkupTemplate(""" + + + +
+
+""") + +genshi_tmpl2 = MarkupTemplate(""" +$table
+""") + +genshi_text_tmpl = NewTextTemplate(""" + +{% for row in table %} +{% for c in row.values() %}{% end %} +{% end %} +
$c
+""") + +if DjangoTemplate: + django_tmpl = DjangoTemplate(""" + + {% for row in table %} + {% for col in row.values %}{{ col|escape }}{% endfor %} + {% endfor %} +
+ """) + + def test_django(): + """Djange template""" + context = DjangoContext({'table': table}) + django_tmpl.render(context) + +if MakoTemplate: + mako_tmpl = MakoTemplate(""" + + % for row in table: + + % for col in row.values(): + + % endfor + + % endfor +
${ col | h }
+""") + def test_mako(): + """Mako Template""" + mako_tmpl.render(table=table) + +def test_genshi(): + """Genshi template""" + stream = genshi_tmpl.generate(table=table) + stream.render('html', strip_whitespace=False) + +def test_genshi_text(): + """Genshi text template""" + stream = genshi_text_tmpl.generate(table=table) + stream.render('text') + +def test_genshi_builder(): + """Genshi template + tag builder""" + stream = tag.TABLE([ + tag.tr([tag.td(c) for c in row.values()]) + for row in table + ]).generate() + stream = genshi_tmpl2.generate(table=stream) + stream.render('html', strip_whitespace=False) + +def test_builder(): + """Genshi tag builder""" + stream = tag.TABLE([ + tag.tr([ + tag.td(c) for c in row.values() + ]) + for row in table + ]).generate() + stream.render('html', strip_whitespace=False) + +if kid: + kid_tmpl = kid.Template(""" + + + +
+
+ """) + + kid_tmpl2 = kid.Template(""" + $table + """) + + def test_kid(): + """Kid template""" + kid_tmpl.table = table + kid_tmpl.serialize(output='html') + + if cet: + def test_kid_et(): + """Kid template + cElementTree""" + _table = cet.Element('table') + for row in table: + td = cet.SubElement(_table, 'tr') + for c in row.values(): + cet.SubElement(td, 'td').text=str(c) + kid_tmpl2.table = _table + kid_tmpl2.serialize(output='html') + +if et: + def test_et(): + """ElementTree""" + _table = et.Element('table') + for row in table: + tr = et.SubElement(_table, 'tr') + for c in row.values(): + et.SubElement(tr, 'td').text=str(c) + et.tostring(_table) + +if cet: + def test_cet(): + """cElementTree""" + _table = cet.Element('table') + for row in table: + tr = cet.SubElement(_table, 'tr') + for c in row.values(): + cet.SubElement(tr, 'td').text=str(c) + cet.tostring(_table) + +if neo_cgi: + def test_clearsilver(): + """ClearSilver""" + hdf = neo_util.HDF() + for i, row in enumerate(table): + for j, c in enumerate(row.values()): + hdf.setValue("rows.%d.cell.%d" % (i, j), cgi.escape(str(c))) + + cs = neo_cs.CS(hdf) + cs.parseStr(""" + +
""") + cs.render() + + +def run(which=None, number=10): + tests = ['test_builder', 'test_genshi', 'test_genshi_text', + 'test_genshi_builder', 'test_mako', 'test_kid', 'test_kid_et', + 'test_et', 'test_cet', 'test_clearsilver', 'test_django'] + + if which: + tests = filter(lambda n: n[5:] in which, tests) + + for test in [t for t in tests if hasattr(sys.modules[__name__], t)]: + t = timeit.Timer(setup='from __main__ import %s;' % test, + stmt='%s()' % test) + time = t.timeit(number=number) / number + + if time < 0.00001: + result = ' (not installed?)' + else: + result = '%16.2f ms' % (1000 * time) + print '%-35s %s' % (getattr(sys.modules[__name__], test).__doc__, result) + + +if __name__ == '__main__': + which = [arg for arg in sys.argv[1:] if arg[0] != '-'] + + if '-p' in sys.argv: + import cProfile, pstats + prof = cProfile.Profile() + prof.run('run(%r, number=1)' % which) + stats = pstats.Stats(prof) + stats.strip_dirs() + stats.sort_stats('time', 'calls') + stats.print_stats(25) + if '-v' in sys.argv: + stats.print_callees() + stats.print_callers() + else: + run(which) diff --git a/lib/genshi/examples/bench/cheetah/footer.tmpl b/lib/genshi/examples/bench/cheetah/footer.tmpl new file mode 100644 --- /dev/null +++ b/lib/genshi/examples/bench/cheetah/footer.tmpl @@ -0,0 +1,2 @@ +
+
diff --git a/lib/genshi/examples/bench/cheetah/header.tmpl b/lib/genshi/examples/bench/cheetah/header.tmpl new file mode 100644 --- /dev/null +++ b/lib/genshi/examples/bench/cheetah/header.tmpl @@ -0,0 +1,3 @@ +
+
$title
+
diff --git a/lib/genshi/examples/bench/cheetah/template.tmpl b/lib/genshi/examples/bench/cheetah/template.tmpl new file mode 100644 --- /dev/null +++ b/lib/genshi/examples/bench/cheetah/template.tmpl @@ -0,0 +1,22 @@ + + + + ${title} + + + #include "cheetah/header.tmpl" + +
Loop
+ #if $items +
+ #for $item in $items +
$item
+ #end for +
+ #end if + + #include "cheetah/footer.tmpl" + + diff --git a/lib/genshi/examples/bench/clearsilver/footer.cs b/lib/genshi/examples/bench/clearsilver/footer.cs new file mode 100644 --- /dev/null +++ b/lib/genshi/examples/bench/clearsilver/footer.cs @@ -0,0 +1,2 @@ +
+
diff --git a/lib/genshi/examples/bench/clearsilver/header.cs b/lib/genshi/examples/bench/clearsilver/header.cs new file mode 100644 --- /dev/null +++ b/lib/genshi/examples/bench/clearsilver/header.cs @@ -0,0 +1,7 @@ +
+
+
+ + +
Hello, !
+ diff --git a/lib/genshi/examples/bench/clearsilver/template.cs b/lib/genshi/examples/bench/clearsilver/template.cs new file mode 100644 --- /dev/null +++ b/lib/genshi/examples/bench/clearsilver/template.cs @@ -0,0 +1,25 @@ + + + + <?cs var:title ?> + + + + + + + +
Loop
+ +
+ + class="last"> + +
+ + + + + diff --git a/lib/genshi/examples/bench/django/templates/base.html b/lib/genshi/examples/bench/django/templates/base.html new file mode 100644 --- /dev/null +++ b/lib/genshi/examples/bench/django/templates/base.html @@ -0,0 +1,18 @@ + + + + {% block content %} + {% block header %} +
+
{{ title|escape }}
+
+ {% endblock %} + + {% block footer %} +
+ {% endblock %} + {% endblock %} + + diff --git a/lib/genshi/examples/bench/django/templates/template.html b/lib/genshi/examples/bench/django/templates/template.html new file mode 100644 --- /dev/null +++ b/lib/genshi/examples/bench/django/templates/template.html @@ -0,0 +1,27 @@ +{% extends "base.html" %} +{% load bench %} + +{% block content %} + + {{title|escape}} + + + + {% block header %}{% endblock %} + +
{% greeting user %}
+
{% greeting "me" %}
+
{% greeting "world" %}
+ +
Loop
+ {% if items %} +
+ {% for item in items %} + {{ item|escape }} + {% endfor %} +
+ {% endif %} + + {% block footer %}{% endblock %} + +{% endblock %} diff --git a/lib/genshi/examples/bench/django/templatetags/__init__.py b/lib/genshi/examples/bench/django/templatetags/__init__.py new file mode 100644 diff --git a/lib/genshi/examples/bench/django/templatetags/bench.py b/lib/genshi/examples/bench/django/templatetags/bench.py new file mode 100644 --- /dev/null +++ b/lib/genshi/examples/bench/django/templatetags/bench.py @@ -0,0 +1,8 @@ +from django.template import Library, Node, resolve_variable +from django.utils.html import escape + +register = Library() + +def greeting(name): + return 'Hello, %s!' % escape(name) +greeting = register.simple_tag(greeting) diff --git a/lib/genshi/examples/bench/genshi/base.html b/lib/genshi/examples/bench/genshi/base.html new file mode 100644 --- /dev/null +++ b/lib/genshi/examples/bench/genshi/base.html @@ -0,0 +1,17 @@ + + +
+ Hello, ${name}! +
+ + +
+
${title}
+
+ ${select('*')} +
+ + + diff --git a/lib/genshi/examples/bench/genshi/template.html b/lib/genshi/examples/bench/genshi/template.html new file mode 100644 --- /dev/null +++ b/lib/genshi/examples/bench/genshi/template.html @@ -0,0 +1,24 @@ + + + + + ${title} + + +
${greeting(user)}
+
${greeting('me')}
+
${greeting('world')}
+ +
Loop
+
+
+
+ + + diff --git a/lib/genshi/examples/bench/genshi_text/footer.txt b/lib/genshi/examples/bench/genshi_text/footer.txt new file mode 100644 --- /dev/null +++ b/lib/genshi/examples/bench/genshi_text/footer.txt @@ -0,0 +1,1 @@ +
diff --git a/lib/genshi/examples/bench/genshi_text/header.txt b/lib/genshi/examples/bench/genshi_text/header.txt new file mode 100644 --- /dev/null +++ b/lib/genshi/examples/bench/genshi_text/header.txt @@ -0,0 +1,3 @@ +
+
${escape(title)}
+
diff --git a/lib/genshi/examples/bench/genshi_text/template.txt b/lib/genshi/examples/bench/genshi_text/template.txt new file mode 100644 --- /dev/null +++ b/lib/genshi/examples/bench/genshi_text/template.txt @@ -0,0 +1,28 @@ + + + + + ${escape(title)} + + + + {% include header.txt %} + {% def greeting(name) %} + Hello, ${name}! + {% end %} + +
${greeting(user)}
+
${greeting("me")}
+
${greeting("world")}
+ +
Loop
+ {% if items %}
+ {% for idx, item in enumerate(items) %}\ + ${escape(item)} + {% end %} +
{% end %} + + {% include footer.txt %} + diff --git a/lib/genshi/examples/bench/kid/base.kid b/lib/genshi/examples/bench/kid/base.kid new file mode 100644 --- /dev/null +++ b/lib/genshi/examples/bench/kid/base.kid @@ -0,0 +1,15 @@ + + +
+ Hello, ${name}! +
+ + +
+
${title}
+
+ ${item} +
+ + diff --git a/lib/genshi/examples/bench/kid/template.kid b/lib/genshi/examples/bench/kid/template.kid new file mode 100644 --- /dev/null +++ b/lib/genshi/examples/bench/kid/template.kid @@ -0,0 +1,22 @@ + + + + ${title} + + +
${greeting(user)}
+
${greeting('me')}
+
${greeting('world')}
+ +
Loop
+
+
+
+ + diff --git a/lib/genshi/examples/bench/mako/footer.html b/lib/genshi/examples/bench/mako/footer.html new file mode 100644 --- /dev/null +++ b/lib/genshi/examples/bench/mako/footer.html @@ -0,0 +1,2 @@ +
+
diff --git a/lib/genshi/examples/bench/mako/header.html b/lib/genshi/examples/bench/mako/header.html new file mode 100644 --- /dev/null +++ b/lib/genshi/examples/bench/mako/header.html @@ -0,0 +1,5 @@ +
+
${title | h}
+
+ + diff --git a/lib/genshi/examples/bench/mako/template.html b/lib/genshi/examples/bench/mako/template.html new file mode 100644 --- /dev/null +++ b/lib/genshi/examples/bench/mako/template.html @@ -0,0 +1,29 @@ + + + + ${title} + + + <%def name="greeting(name)"> +
Hello, ${name | h}!
+ + + <%include file="header.html"/> + + ${greeting(user)} + ${greeting('me')} + ${greeting('world')} + +
Loop
+ % if items: +
+ % for idx, item in enumerate(items): +
${item | h}
+ % endfor +
+ % endif + <%include file="footer.html"/> + + diff --git a/lib/genshi/examples/bench/myghty/base.myt b/lib/genshi/examples/bench/myghty/base.myt new file mode 100644 --- /dev/null +++ b/lib/genshi/examples/bench/myghty/base.myt @@ -0,0 +1,26 @@ + + + +<%method wrap> +<%args scope="request"> + title + +
+
<% title %>
+
+ +<% m.content() %> + + + +
+ + +<%method greeting> +<%args> + name + +Hello, <% name | h %> + diff --git a/lib/genshi/examples/bench/myghty/template.myt b/lib/genshi/examples/bench/myghty/template.myt new file mode 100644 --- /dev/null +++ b/lib/genshi/examples/bench/myghty/template.myt @@ -0,0 +1,25 @@ +<%args> + title + items + user + + + + <% title %> + + + <&|base.myt:wrap &> +
<& base.myt:greeting, name=user &>
+
<& base.myt:greeting, name="me"&>
+
<& base.myt:greeting, name="world" &>
+ +
Loop
+%if items: +
+% for item in items: +
<% item %>
+% +
+% + + diff --git a/lib/genshi/examples/bench/simpletal/base.html b/lib/genshi/examples/bench/simpletal/base.html new file mode 100644 --- /dev/null +++ b/lib/genshi/examples/bench/simpletal/base.html @@ -0,0 +1,15 @@ + + +
+ Hello, name +
+ + +
+
+
+
+
+ + + diff --git a/lib/genshi/examples/bench/simpletal/template.html b/lib/genshi/examples/bench/simpletal/template.html new file mode 100644 --- /dev/null +++ b/lib/genshi/examples/bench/simpletal/template.html @@ -0,0 +1,29 @@ + + + + Title + + +
+ +
+ +
+
+ me +
+
+ world +
+ +
Loop
+
+
Item
+
+ +
+ + diff --git a/lib/genshi/examples/bench/xpath.py b/lib/genshi/examples/bench/xpath.py new file mode 100644 --- /dev/null +++ b/lib/genshi/examples/bench/xpath.py @@ -0,0 +1,170 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2006-2008 Edgewall Software +# All rights reserved. +# +# This software is licensed as described in the file COPYING, which +# you should have received as part of this distribution. The terms +# are also available at http://genshi.edgewall.org/wiki/License. +# +# This software consists of voluntary contributions made by many +# individuals. For the exact contribution history, see the revision +# history and logs, available at http://genshi.edgewall.org/log/. + + +try: + from os import times + def time_func(): + tup = times() + #just user time + return tup[0] # + tup[1] +except ImportError: + from time import time as time_func + +from genshi.core import START, END +from genshi.path import Path +from genshi.input import XML + +def benchmark(f, acurate_time=1): + """Checks how much time does function f work. It runs it as + many times as needed for avoiding inaccuracy""" + + runs = 1 + while True: + start_time = time_func() + for _ in xrange(runs): + f() + dt = time_func() - start_time + if dt >= acurate_time: + break + runs *= 2 + return dt / runs + +def spell(t): + """Returns spelled representation of time""" + units = [(0.000001, 'microsecond', 'microseconds'), + (0.001, 'milisecond', 'miliseconds'), + (1, 'second', 'seconds'), + (60, 'minute', 'minutes'), + (60*60, 'hour', 'hours'), + ] + i = 0 + at = abs(t) + while i + 1 < len(units) and at >= units[i + 1][0]: + i += 1 + t /= units[i][0] + if t >= 2: + name = units[i][2] + else: + name = units[i][1] + return "%f %s"%(t, name) + +def test_paths_in_streams(exprs, streams, test_strategies=False): + for expr in exprs: + print "Testing path %r" % expr + for stream, sname in streams: + print '\tRunning on "%s" example:' % sname + + path = Path(expr) + def f(): + for e in path.select(stream): + pass + t = spell(benchmark(f)) + print "\t\tselect:\t\t%s" % t + + def f(): + path = Path(expr) + for e in path.select(stream): + pass + t = spell(benchmark(f)) + print "\t\tinit + select:\t%s" % t + + if test_strategies and len(path.paths) == 1: + from genshi.path import GenericStrategy, SingleStepStrategy, \ + SimplePathStrategy + from genshi.tests.path import FakePath + strategies = (GenericStrategy, SingleStepStrategy, + SimplePathStrategy) + for strategy in strategies: + if not strategy.supports(path.paths[0]): + continue + print "\t\t%s Strategy"%strategy.__name__ + fp = FakePath(strategy(path.paths[0])) + def f(): + for e in fp.select(stream): + pass + t = spell(benchmark(f)) + print "\t\t\tselect:\t\t%s"%t + + +def test_documents(test_strategies=False): + streams = [] + + s = XML("""\ + + + + Foo + + +
Hello
+ + +""") + streams.append((s, "small document")) + + s = XML("""\ + + + + Foo + + +
Hello
+
+
+
+ Genshi + Python toolkit for generating output for the web
+
+ Babel + Python library for I18n/L10n in web applications
+
+ Bitten + Continuous integration plugin for Trac
+
+ Posterity + Web-based email system
+
+
+ + Trac Web-based lightweight project management + system + +
+
+ + +""") + streams.append((s, "big document")) + + paths = [ + '.', + '*|text()', + 'html', + 'html[@lang="en"]', + 'html/body/h1/text()', + 'html/body/div/a/@href', + 'html/body/div[@id="splash"]/a[@class="b4"]/strong/text()', + 'descendant-or-self::text()', + 'descendant-or-self::h1/text()', + ] + test_paths_in_streams(paths, streams, test_strategies) + +if __name__ == '__main__': + from sys import argv + if "--strategies" in argv: + test_strategies = True + else: + test_strategies = False + test_documents(test_strategies) diff --git a/lib/genshi/examples/tutorial/geddit/__init__.py b/lib/genshi/examples/tutorial/geddit/__init__.py new file mode 100644 diff --git a/lib/genshi/examples/tutorial/geddit/controller.py b/lib/genshi/examples/tutorial/geddit/controller.py new file mode 100755 --- /dev/null +++ b/lib/genshi/examples/tutorial/geddit/controller.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python + +import operator, os, pickle, sys + +import cherrypy +from formencode import Invalid +from genshi.input import HTML +from genshi.filters import HTMLFormFiller, HTMLSanitizer + +from geddit.form import LinkForm, CommentForm +from geddit.lib import ajax, template +from geddit.model import Link, Comment + + +class Root(object): + + def __init__(self, data): + self.data = data + + @cherrypy.expose + @template.output('index.html') + def index(self): + links = sorted(self.data.values(), key=operator.attrgetter('time')) + return template.render(links=links) + + @cherrypy.expose + @template.output('submit.html') + def submit(self, cancel=False, **data): + if cherrypy.request.method == 'POST': + if cancel: + raise cherrypy.HTTPRedirect('/') + form = LinkForm() + try: + data = form.to_python(data) + link = Link(**data) + self.data[link.id] = link + raise cherrypy.HTTPRedirect('/') + except Invalid, e: + errors = e.unpack_errors() + else: + errors = {} + + return template.render(errors=errors) | HTMLFormFiller(data=data) + + @cherrypy.expose + @template.output('info.html') + def info(self, id): + link = self.data.get(id) + if not link: + raise cherrypy.NotFound() + return template.render(link=link) + + @cherrypy.expose + @template.output('comment.html') + def comment(self, id, cancel=False, **data): + link = self.data.get(id) + if not link: + raise cherrypy.NotFound() + if cherrypy.request.method == 'POST': + if cancel: + raise cherrypy.HTTPRedirect('/info/%s' % link.id) + form = CommentForm() + try: + data = form.to_python(data) + markup = HTML(data['content']) | HTMLSanitizer() + data['content'] = markup.render('xhtml') + comment = link.add_comment(**data) + if not ajax.is_xhr(): + raise cherrypy.HTTPRedirect('/info/%s' % link.id) + return template.render('_comment.html', comment=comment, + num=len(link.comments)) + except Invalid, e: + errors = e.unpack_errors() + else: + errors = {} + + if ajax.is_xhr(): + stream = template.render('_form.html', link=link, errors=errors) + else: + stream = template.render(link=link, comment=None, errors=errors) + return stream | HTMLFormFiller(data=data) + + @cherrypy.expose + @template.output('index.xml', method='xml') + def feed(self, id=None): + if id: + link = self.data.get(id) + if not link: + raise cherrypy.NotFound() + return template.render('info.xml', link=link) + else: + links = sorted(self.data.values(), key=operator.attrgetter('time')) + return template.render(links=links) + + +def main(filename): + # load data from the pickle file, or initialize it to an empty list + if os.path.exists(filename): + fileobj = open(filename, 'rb') + try: + data = pickle.load(fileobj) + finally: + fileobj.close() + else: + data = {} + + def _save_data(): + # save data back to the pickle file + fileobj = open(filename, 'wb') + try: + pickle.dump(data, fileobj) + finally: + fileobj.close() + if hasattr(cherrypy.engine, 'subscribe'): # CherryPy >= 3.1 + cherrypy.engine.subscribe('stop', _save_data) + else: + cherrypy.engine.on_stop_engine_list.append(_save_data) + + # Some global configuration; note that this could be moved into a + # configuration file + cherrypy.config.update({ + 'tools.encode.on': True, 'tools.encode.encoding': 'utf-8', + 'tools.decode.on': True, + 'tools.trailing_slash.on': True, + 'tools.staticdir.root': os.path.abspath(os.path.dirname(__file__)), + }) + + cherrypy.quickstart(Root(data), '/', { + '/media': { + 'tools.staticdir.on': True, + 'tools.staticdir.dir': 'static' + } + }) + +if __name__ == '__main__': + import formencode + formencode.api.set_stdtranslation(languages=['en']) + main(sys.argv[1]) diff --git a/lib/genshi/examples/tutorial/geddit/form.py b/lib/genshi/examples/tutorial/geddit/form.py new file mode 100644 --- /dev/null +++ b/lib/genshi/examples/tutorial/geddit/form.py @@ -0,0 +1,12 @@ +from formencode import Schema, validators + + +class LinkForm(Schema): + username = validators.UnicodeString(not_empty=True) + url = validators.URL(not_empty=True, add_http=True, check_exists=False) + title = validators.UnicodeString(not_empty=True) + + +class CommentForm(Schema): + username = validators.UnicodeString(not_empty=True) + content = validators.UnicodeString(not_empty=True) diff --git a/lib/genshi/examples/tutorial/geddit/lib/__init__.py b/lib/genshi/examples/tutorial/geddit/lib/__init__.py new file mode 100644 diff --git a/lib/genshi/examples/tutorial/geddit/lib/ajax.py b/lib/genshi/examples/tutorial/geddit/lib/ajax.py new file mode 100644 --- /dev/null +++ b/lib/genshi/examples/tutorial/geddit/lib/ajax.py @@ -0,0 +1,5 @@ +import cherrypy + +def is_xhr(): + requested_with = cherrypy.request.headers.get('X-Requested-With') + return requested_with and requested_with.lower() == 'xmlhttprequest' diff --git a/lib/genshi/examples/tutorial/geddit/lib/template.py b/lib/genshi/examples/tutorial/geddit/lib/template.py new file mode 100644 --- /dev/null +++ b/lib/genshi/examples/tutorial/geddit/lib/template.py @@ -0,0 +1,47 @@ +import os + +import cherrypy +from genshi.core import Stream +from genshi.output import encode, get_serializer +from genshi.template import Context, TemplateLoader + +from geddit.lib import ajax + +loader = TemplateLoader( + os.path.join(os.path.dirname(__file__), '..', 'templates'), + auto_reload=True +) + +def output(filename, method='html', encoding='utf-8', **options): + """Decorator for exposed methods to specify what template the should use + for rendering, and which serialization method and options should be + applied. + """ + def decorate(func): + def wrapper(*args, **kwargs): + cherrypy.thread_data.template = loader.load(filename) + opt = options.copy() + if not ajax.is_xhr() and method == 'html': + opt.setdefault('doctype', 'html') + serializer = get_serializer(method, **opt) + stream = func(*args, **kwargs) + if not isinstance(stream, Stream): + return stream + return encode(serializer(stream), method=serializer, + encoding=encoding) + return wrapper + return decorate + +def render(*args, **kwargs): + """Function to render the given data to the template specified via the + ``@output`` decorator. + """ + if args: + assert len(args) == 1, \ + 'Expected exactly one argument, but got %r' % (args,) + template = loader.load(args[0]) + else: + template = cherrypy.thread_data.template + ctxt = Context(url=cherrypy.url) + ctxt.push(kwargs) + return template.generate(ctxt) diff --git a/lib/genshi/examples/tutorial/geddit/model.py b/lib/genshi/examples/tutorial/geddit/model.py new file mode 100644 --- /dev/null +++ b/lib/genshi/examples/tutorial/geddit/model.py @@ -0,0 +1,31 @@ +from datetime import datetime + + +class Link(object): + + def __init__(self, username, url, title): + self.username = username + self.url = url + self.title = title + self.time = datetime.utcnow() + self.id = hex(hash(tuple([username, url, title, self.time])))[2:] + self.comments = [] + + def __repr__(self): + return '<%s %r>' % (type(self).__name__, self.title) + + def add_comment(self, username, content): + comment = Comment(username, content) + self.comments.append(comment) + return comment + + +class Comment(object): + + def __init__(self, username, content): + self.username = username + self.content = content + self.time = datetime.utcnow() + + def __repr__(self): + return '<%s by %r>' % (type(self).__name__, self.username) diff --git a/lib/genshi/examples/tutorial/geddit/static/jquery.js b/lib/genshi/examples/tutorial/geddit/static/jquery.js new file mode 100644 --- /dev/null +++ b/lib/genshi/examples/tutorial/geddit/static/jquery.js @@ -0,0 +1,11 @@ +/* + * jQuery 1.1.4 - New Wave Javascript + * + * Copyright (c) 2007 John Resig (jquery.com) + * Dual licensed under the MIT (MIT-LICENSE.txt) + * and GPL (GPL-LICENSE.txt) licenses. + * + * $Date: 2007-08-23 21:49:27 -0400 (Thu, 23 Aug 2007) $ + * $Rev: 2862 $ + */ +eval(function(p,a,c,k,e,r){e=function(c){return(c35?String.fromCharCode(c+29):c.toString(36))};if(!''.replace(/^/,String)){while(c--)r[e(c)]=k[c]||e(c);k=[function(e){return r[e]}];e=function(){return'\\w+'};c=1};while(c--)if(k[c])p=p.replace(new RegExp('\\b'+e(c)+'\\b','g'),k[c]);return p}('(9(){6(1f C!="Q")E v=C;E C=19.16=9(a,c){6(19==7||!7.4a)F 1s C(a,c);F 7.4a(a,c)};6(1f $!="Q")E B=$;19.$=C;E q=/^[^<]*(<(.|\\s)+>)[^>]*$|^#(\\w+)$/;C.15=C.3v={4a:9(a,c){a=a||R;6(1f a=="1E"){E m=q.2d(a);6(m&&(m[1]||!c)){6(m[1])a=C.3c([m[1]]);G{E b=R.37(m[3]);6(b)6(b.2j!=m[3])F C().1F(a);G{7[0]=b;7.H=1;F 7}G a=[]}}G F 1s C(c).1F(a)}G 6(C.1g(a))F 1s C(R)[C.15.1L?"1L":"2f"](a);F 7.5J(a.1b==1K&&a||(a.3w||a.H&&a!=19&&!a.1t&&a[0]!=Q&&a[0].1t)&&C.2V(a)||[a])},3w:"1.1.4",7K:9(){F 7.H},H:0,21:9(a){F a==Q?C.2V(7):7[a]},1O:9(a){E b=C(a);b.5c=7;F b},5J:9(a){7.H=0;1K.3v.Y.T(7,a);F 7},J:9(a,b){F C.J(7,a,b)},45:9(a){E b=-1;7.J(9(i){6(7==a)b=i});F b},1j:9(f,d,e){E c=f;6(f.1b==3n)6(d==Q)F 7.H&&C[e||"1j"](7[0],f)||Q;G{c={};c[f]=d}F 7.J(9(a){I(E b 17 c)C.1j(e?7.S:7,b,C.4Q(7,c[b],e,a,b))})},1h:9(b,a){F 7.1j(b,a,"34")},2Q:9(e){6(1f e!="4P"&&e!=K)F 7.3K().3H(R.60(e));E t="";C.J(e||7,9(){C.J(7.2Z,9(){6(7.1t!=8)t+=7.1t!=1?7.5S:C.15.2Q([7])})});F t},82:9(){E a,2e=1a;F 7.J(9(){6(!a)a=C.3c(2e,7.2I);E b=a[0].3B(O);7.P.2p(b,7);20(b.1k)b=b.1k;b.4p(7)})},3H:9(){F 7.2J(1a,O,1,9(a){7.4p(a)})},5v:9(){F 7.2J(1a,O,-1,9(a){7.2p(a,7.1k)})},5u:9(){F 7.2J(1a,M,1,9(a){7.P.2p(a,7)})},5t:9(){F 7.2J(1a,M,-1,9(a){7.P.2p(a,7.2a)})},3L:9(){F 7.5c||C([])},1F:9(t){E b=C.3M(7,9(a){F C.1F(t,a)});F 7.1O(/[^+>] [^+>]/.1d(t)||t.U("..")>-1?C.4d(b):b)},7o:9(e){e=e!=Q?e:O;E d=7.1r(7.1F("*"));6(C.N.12){d.J(9(){7.2l$1i={};I(E a 17 7.$1i)7.2l$1i[a]=C.14({},7.$1i[a])}).49()}E r=7.1O(C.3M(7,9(a){F a.3B(e)}));6(C.N.12){d.J(9(){E c=7.2l$1i;I(E a 17 c)I(E b 17 c[a])C.1c.1r(7,a,c[a][b],c[a][b].V);7.2l$1i=K})}6(e){E f=r.1r(r.1F(\'*\')).1l(\'2b,39[@L=3i]\');d.1l(\'2b,39[@L=3i]\').J(9(i){6(7.3j)f[i].3j=7.3j;6(7.27)f[i].27=O})}F r},1l:9(t){F 7.1O(C.1g(t)&&C.2B(7,9(b,a){F t.T(b,[a])})||C.2R(t,7))},5l:9(t){F 7.1O(t.1b==3n&&C.2R(t,7,O)||C.2B(7,9(a){F(t.1b==1K||t.3w)?C.4K(a,t)<0:a!=t}))},1r:9(t){F 7.1O(C.29(7.21(),t.1b==3n?C(t).21():t.H!=Q&&(!t.W||t.W=="6s")?t:[t]))},3y:9(a){F a?C.2R(a,7).H>0:M},2G:9(a){F a==Q?(7.H?7[0].2A:K):7.1j("2A",a)},5W:9(a){F a==Q?(7.H?7[0].2W:K):7.3K().3H(a)},3S:9(){F 7.1O(1K.3v.3S.T(7,1a))},2J:9(f,d,g,e){E c=7.H>1,a;F 7.J(9(){6(!a){a=C.3c(f,7.2I);6(g<0)a.8E()}E b=7;6(d&&C.W(7,"1A")&&C.W(a[0],"3O"))b=7.4L("1w")[0]||7.4p(R.6a("1w"));C.J(a,9(){6(C.W(7,"33")){6(7.32)C.31({1G:7.32,2w:M,3G:"33"});G C.4E(7.2Q||7.5Z||7.2W||"")}G e.T(b,[c?7.3B(O):7])})})}};C.14=C.15.14=9(){E c=1a[0]||{},a=1,1M=1a.H,4D=M;6(c.1b==8d){4D=c;c=1a[1]||{}}6(1M==1){c=7;a=0}E b;I(;a<1M;a++)6((b=1a[a])!=K)I(E i 17 b){6(c==b[i])5X;6(4D&&1f b[i]==\'4P\'&&c[i])C.14(c[i],b[i]);G 6(b[i]!=Q)c[i]=b[i]}F c};C.14({8a:9(a){19.$=B;6(a)19.16=v;F C},1g:9(a){F!!a&&1f a!="1E"&&!a.W&&a.1b!=1K&&/9/i.1d(a+"")},3E:9(a){F a.3D&&!a.4z||a.4y&&a.2I&&!a.2I.4z},4E:9(a){a=C.2s(a);6(a){6(19.5N)19.5N(a);G 6(C.N.1H)19.4x(a,0);G 2T.2S(19,a)}},W:9(b,a){F b.W&&b.W.1I()==a.1I()},J:9(a,b,c){6(c){6(a.H==Q)I(E i 17 a)b.T(a[i],c);G I(E i=0,3A=a.H;i<3A;i++)6(b.T(a[i],c)===M)1J}G{6(a.H==Q)I(E i 17 a)b.2S(a[i],i,a[i]);G I(E i=0,3A=a.H,2G=a[0];i<3A&&b.2S(2G,i,2G)!==M;2G=a[++i]){}}F a},4Q:9(c,b,d,e,a){6(C.1g(b))b=b.2S(c,[e]);E f=/z-?45|7S-?7Q|1e|5y|7O-?1u/i;F b&&b.1b==3x&&d=="34"&&!f.1d(a)?b+"4t":b},18:{1r:9(b,c){C.J((c||"").2M(/\\s+/),9(i,a){6(!C.18.2N(b.18,a))b.18+=(b.18?" ":"")+a})},23:9(b,c){b.18=c!=Q?C.2B(b.18.2M(/\\s+/),9(a){F!C.18.2N(c,a)}).5w(" "):""},2N:9(t,c){F C.4K(c,(t.18||t).3s().2M(/\\s+/))>-1}},1V:9(e,o,f){I(E i 17 o){e.S["2U"+i]=e.S[i];e.S[i]=o[i]}f.T(e,[]);I(E i 17 o)e.S[i]=e.S["2U"+i]},1h:9(e,p){6(p=="1u"||p=="24"){E b={},3p,3o,d=["7J","7G","7F","7B"];C.J(d,9(){b["7A"+7]=0;b["7x"+7+"7u"]=0});C.1V(e,b,9(){6(C(e).3y(\':4N\')){3p=e.7t;3o=e.7q}G{e=C(e.3B(O)).1F(":4e").5d("27").3L().1h({3V:"1C",3k:"7n",11:"2m",7h:"0",7e:"0"}).57(e.P)[0];E a=C.1h(e.P,"3k")||"3g";6(a=="3g")e.P.S.3k="76";3p=e.74;3o=e.71;6(a=="3g")e.P.S.3k="3g";e.P.3e(e)}});F p=="1u"?3p:3o}F C.34(e,p)},34:9(h,d,g){E i,1R=[],1V=[];9 2E(a){6(!C.N.1H)F M;E b=R.2L.3b(a,K);F!b||b.44("2E")==""}6(d=="1e"&&C.N.12){i=C.1j(h.S,"1e");F i==""?"1":i}6(d.2k(/3a/i))d=x;6(!g&&h.S[d])i=h.S[d];G 6(R.2L&&R.2L.3b){6(d.2k(/3a/i))d="3a";d=d.1v(/([A-Z])/g,"-$1").2D();E e=R.2L.3b(h,K);6(e&&!2E(h))i=e.44(d);G{I(E a=h;a&&2E(a);a=a.P)1R.42(a);I(a=0;a<1R.H;a++)6(2E(1R[a])){1V[a]=1R[a].S.11;1R[a].S.11="2m"}i=d=="11"&&1V[1R.H-1]!=K?"1T":R.2L.3b(h,K).44(d)||"";I(a=0;a<1V.H;a++)6(1V[a]!=K)1R[a].S.11=1V[a]}6(d=="1e"&&i=="")i="1"}G 6(h.41){E f=d.1v(/\\-(\\w)/g,9(m,c){F c.1I()});i=h.41[d]||h.41[f]}F i},3c:9(a,c){E r=[];c=c||R;C.J(a,9(i,b){6(!b)F;6(b.1b==3x)b=b.3s();6(1f b=="1E"){E s=C.2s(b).2D(),1m=c.6a("1m"),1P=[];E a=!s.U("<1Z")&&[1,"<2b>",""]||!s.U("<6L")&&[1,"<4V>",""]||s.2k(/^<(6I|1w|6H|6F|6D)/)&&[1,"<1A>",""]||!s.U("<3O")&&[2,"<1A><1w>",""]||(!s.U("<6A")||!s.U("<6y"))&&[3,"<1A><1w><3O>",""]||!s.U("<6x")&&[2,"<1A><1w><4T>",""]||C.N.12&&[1,"1m<1m>",""]||[0,"",""];1m.2W=a[1]+b+a[2];20(a[0]--)1m=1m.3Y;6(C.N.12){6(!s.U("<1A")&&s.U("<1w")<0)1P=1m.1k&&1m.1k.2Z;G 6(a[1]=="<1A>"&&s.U("<1w")<0)1P=1m.2Z;I(E n=1P.H-1;n>=0;--n)6(C.W(1P[n],"1w")&&!1P[n].2Z.H)1P[n].P.3e(1P[n]);6(/^\\s/.1d(b))1m.2p(c.60(b.2k(/^\\s*/)[0]),1m.1k)}b=C.2V(1m.2Z)}6(0===b.H&&(!C.W(b,"38")&&!C.W(b,"2b")))F;6(b[0]==Q||C.W(b,"38")||b.6u)r.Y(b);G r=C.29(r,b)});F r},1j:9(c,d,a){E e=C.3E(c)?{}:C.4q;6(d=="28"&&C.N.1H)c.P.3j;6(e[d]){6(a!=Q)c[e[d]]=a;F c[e[d]]}G 6(C.N.12&&d=="S")F C.1j(c.S,"6p",a);G 6(a==Q&&C.N.12&&C.W(c,"38")&&(d=="6n"||d=="6m"))F c.6k(d).5S;G 6(c.4y){6(a!=Q)c.6j(d,a);6(C.N.12&&/5R|32/.1d(d)&&!C.3E(c))F c.3F(d,2);F c.3F(d)}G{6(d=="1e"&&C.N.12){6(a!=Q){c.5y=1;c.1l=(c.1l||"").1v(/5T\$[^)]*\$/,"")+(3m(a).3s()=="6d"?"":"5T(1e="+a*6c+")")}F c.1l?(3m(c.1l.2k(/1e=([^)]*)/)[1])/6c).3s():""}d=d.1v(/-([a-z])/8I,9(z,b){F b.1I()});6(a!=Q)c[d]=a;F c[d]}},2s:9(t){F(t||"").1v(/^\\s+|\\s+$/g,"")},2V:9(a){E r=[];6(1f a!="8H")I(E i=0,1M=a.H;i<1M;i++)r.Y(a[i]);G r=a.3S(0);F r},4K:9(b,a){I(E i=0,1M=a.H;i<1M;i++)6(a[i]==b)F i;F-1},29:9(a,b){6(C.N.12){I(E i=0;b[i];i++)6(b[i].1t!=8)a.Y(b[i])}G I(E i=0;b[i];i++)a.Y(b[i]);F a},4d:9(a){E r=[],4O=C.1q++;2g{I(E i=0,69=a.H;i<69;i++)6(4O!=a[i].1q){a[i].1q=4O;r.Y(a[i])}}2h(e){r=a}F r},1q:0,2B:9(b,a,c){6(1f a=="1E")a=2T("M||9(a,i){F "+a+"}");E d=[];I(E i=0,3P=b.H;i<3P;i++)6(!c&&a(b[i],i)||c&&!a(b[i],i))d.Y(b[i]);F d},3M:9(c,b){6(1f b=="1E")b=2T("M||9(a){F "+b+"}");E d=[];I(E i=0,3P=c.H;i<3P;i++){E a=b(c[i],i);6(a!==K&&a!=Q){6(a.1b!=1K)a=[a];d=d.8x(a)}}F d}});E u=8w.8u.2D();C.N={6b:(u.2k(/.+(?:8s|8q|8p|8o)[\\/: ]([\\d.]+)/)||[])[1],1H:/61/.1d(u),2t:/2t/.1d(u),12:/12/.1d(u)&&!/2t/.1d(u),3J:/3J/.1d(u)&&!/(8n|61)/.1d(u)};E x=C.N.12?"3I":"4G";C.14({8m:!C.N.12||R.8l=="8k",3I:C.N.12?"3I":"4G",4q:{"I":"8j","8i":"18","3a":x,4G:x,3I:x,2W:"2W",18:"18",2A:"2A",30:"30",27:"27",8h:"8g",28:"28",8f:"8e"}});C.J({5Y:"a.P",4C:"16.4C(a)",8c:"16.25(a,2,\'2a\')",8b:"16.25(a,2,\'4B\')",88:"16.4A(a.P.1k,a)",87:"16.4A(a.1k)"},9(i,n){C.15[i]=9(a){E b=C.3M(7,n);6(a&&1f a=="1E")b=C.2R(a,b);F 7.1O(C.4d(b))}});C.J({57:"3H",86:"5v",2p:"5u",85:"5t"},9(i,n){C.15[i]=9(){E a=1a;F 7.J(9(){I(E j=0,1M=a.H;j<1M;j++)C(a[j])[n](7)})}});C.J({5d:9(a){C.1j(7,a,"");7.84(a)},83:9(c){C.18.1r(7,c)},81:9(c){C.18.23(7,c)},80:9(c){C.18[C.18.2N(7,c)?"23":"1r"](7,c)},23:9(a){6(!a||C.1l(a,[7]).r.H)7.P.3e(7)},3K:9(){20(7.1k)7.3e(7.1k)}},9(i,n){C.15[i]=9(){F 7.J(n,1a)}});C.J(["5Q","5P","5M","5L"],9(i,n){C.15[n]=9(a,b){F 7.1l(":"+n+"("+a+")",b)}});C.J(["1u","24"],9(i,n){C.15[n]=9(h){F h==Q?(7.H?C.1h(7[0],n):K):7.1h(n,h.1b==3n?h:h+"4t")}});E A=C.N.1H&&5K(C.N.6b)<7Z?"(?:[\\\\w*2l-]|\\\\\\\\.)":"(?:[\\\\w\\7Y-\\7V*2l-]|\\\\\\\\.)",5I=1s 3C("^[/>]\\\\s*("+A+"+)"),5H=1s 3C("^("+A+"+)(#)("+A+"+)"),5G=1s 3C("^([#.]?)("+A+"*)");C.14({4w:{"":"m[2]==\'*\'||16.W(a,m[2])","#":"a.3F(\'2j\')==m[2]",":":{5P:"im[3]-0",25:"m[3]-0==i",5Q:"m[3]-0==i",2H:"i==0",2P:"i==r.H-1",5E:"i%2==0",5D:"i%2","2H-3z":"a.P.4L(\'*\')[0]==a","2P-3z":"16.25(a.P.3Y,1,\'4B\')==a","7U-3z":"!16.25(a.P.3Y,2,\'4B\')",5Y:"a.1k",3K:"!a.1k",5L:"(a.5Z||a.7T||\'\').U(m[3])>=0",4N:\'"1C"!=a.L&&16.1h(a,"11")!="1T"&&16.1h(a,"3V")!="1C"\',1C:\'"1C"==a.L||16.1h(a,"11")=="1T"||16.1h(a,"3V")=="1C"\',7R:"!a.30",30:"a.30",27:"a.27",28:"a.28||16.1j(a,\'28\')",2Q:"\'2Q\'==a.L",4e:"\'4e\'==a.L",3i:"\'3i\'==a.L",4v:"\'4v\'==a.L",5C:"\'5C\'==a.L",4u:"\'4u\'==a.L",5B:"\'5B\'==a.L",5A:"\'5A\'==a.L",1X:\'"1X"==a.L||16.W(a,"1X")\',39:"/39|2b|7P|1X/i.1d(a.W)",2N:"16.1F(m[3],a).H"},"[":"16.1F(m[2],a).H"},5x:[/^\\[ *(@)([\\w-]+) *([!*$^~=]*) *(\'?"?)(.*?)\\4 *\\]/,/^(\\[)\\s*(.*?(\\[.*?\\])?[^[]*?)\\s*\\]/,/^(:)([\\w-]+)\$"?\'?(.*?(\\(.*?\$)?[^(]*?)"?\'?\\)/,1s 3C("^([:.#]*)("+A+"+)")],2R:9(a,c,b){E d,1Y=[];20(a&&a!=d){d=a;E f=C.1l(a,c,b);a=f.t.1v(/^\\s*,\\s*/,"");1Y=b?c=f.r:C.29(1Y,f.r)}F 1Y},1F:9(t,l){6(1f t!="1E")F[t];6(l&&!l.1t)l=K;l=l||R;6(!t.U("//")){t=t.2K(2,t.H)}G 6(!t.U("/")&&!l.2I){l=l.3D;t=t.2K(1,t.H);6(t.U("/")>=1)t=t.2K(t.U("/"),t.H)}E d=[l],2q=[],2P;20(t&&2P!=t){E r=[];2P=t;t=C.2s(t).1v(/^\\/\\//,"");E k=M;E g=5I;E m=g.2d(t);6(m){E o=m[1].1I();I(E i=0;d[i];i++)I(E c=d[i].1k;c;c=c.2a)6(c.1t==1&&(o=="*"||c.W.1I()==o.1I()))r.Y(c);d=r;t=t.1v(g,"");6(t.U(" ")==0)5X;k=O}G{g=/^((\\/?\\.\\.)|([>\\/+~]))\\s*(\\w*)/i;6((m=g.2d(t))!=K){r=[];E o=m[4],1q=C.1q++;m=m[1];I(E j=0,2o=d.H;j<2o;j++)6(m.U("..")<0){E n=m=="~"||m=="+"?d[j].2a:d[j].1k;I(;n;n=n.2a)6(n.1t==1){6(m=="~"&&n.1q==1q)1J;6(!o||n.W.1I()==o.1I()){6(m=="~")n.1q=1q;r.Y(n)}6(m=="+")1J}}G r.Y(d[j].P);d=r;t=C.2s(t.1v(g,""));k=O}}6(t&&!k){6(!t.U(",")){6(l==d[0])d.4s();2q=C.29(2q,d);r=d=[l];t=" "+t.2K(1,t.H)}G{E h=5H;E m=h.2d(t);6(m){m=[0,m[2],m[3],m[1]]}G{h=5G;m=h.2d(t)}m[2]=m[2].1v(/\\\\/g,"");E f=d[d.H-1];6(m[1]=="#"&&f&&f.37&&!C.3E(f)){E p=f.37(m[2]);6((C.N.12||C.N.2t)&&p&&1f p.2j=="1E"&&p.2j!=m[2])p=C(\'[@2j="\'+m[2]+\'"]\',f)[0];d=r=p&&(!m[3]||C.W(p,m[3]))?[p]:[]}G{I(E i=0;d[i];i++){E a=m[1]!=""||m[0]==""?"*":m[2];6(a=="*"&&d[i].W.2D()=="4P")a="2O";r=C.29(r,d[i].4L(a))}6(m[1]==".")r=C.4r(r,m[2]);6(m[1]=="#"){E e=[];I(E i=0;r[i];i++)6(r[i].3F("2j")==m[2]){e=[r[i]];1J}r=e}d=r}t=t.1v(h,"")}}6(t){E b=C.1l(t,r);d=r=b.r;t=C.2s(b.t)}}6(t)d=[];6(d&&l==d[0])d.4s();2q=C.29(2q,d);F 2q},4r:9(r,m,a){m=" "+m+" ";E c=[];I(E i=0;r[i];i++){E b=(" "+r[i].18+" ").U(m)>=0;6(!a&&b||a&&!b)c.Y(r[i])}F c},1l:9(t,r,h){E d;20(t&&t!=d){d=t;E p=C.5x,m;I(E i=0;p[i];i++){m=p[i].2d(t);6(m){t=t.7N(m[0].H);m[2]=m[2].1v(/\\\\/g,"");1J}}6(!m)1J;6(m[1]==":"&&m[2]=="5l")r=C.1l(m[3],r,O).r;G 6(m[1]==".")r=C.4r(r,m[2],h);G 6(m[1]=="@"){E g=[],L=m[3];I(E i=0,2o=r.H;i<2o;i++){E a=r[i],z=a[C.4q[m[2]]||m[2]];6(z==K||/5R|32|28/.1d(m[2]))z=C.1j(a,m[2])||\'\';6((L==""&&!!z||L=="="&&z==m[5]||L=="!="&&z!=m[5]||L=="^="&&z&&!z.U(m[5])||L=="$="&&z.2K(z.H-m[5].H)==m[5]||(L=="*="||L=="~=")&&z.U(m[5])>=0)^h)g.Y(a)}r=g}G 6(m[1]==":"&&m[2]=="25-3z"){E e=C.1q++,g=[],1d=/(\\d*)n\\+?(\\d*)/.2d(m[3]=="5E"&&"2n"||m[3]=="5D"&&"2n+1"||!/\\D/.1d(m[3])&&"n+"+m[3]||m[3]),2H=(1d[1]||1)-0,d=1d[2]-0;I(E i=0,2o=r.H;i<2o;i++){E j=r[i],P=j.P;6(e!=P.1q){E c=1;I(E n=P.1k;n;n=n.2a)6(n.1t==1)n.4o=c++;P.1q=e}E b=M;6(2H==1){6(d==0||j.4o==d)b=O}G 6((j.4o+d)%2H==0)b=O;6(b^h)g.Y(j)}r=g}G{E f=C.4w[m[1]];6(1f f!="1E")f=C.4w[m[1]][m[2]];f=2T("M||9(a,i){F "+f+"}");r=C.2B(r,f,h)}}F{r:r,t:t}},4C:9(c){E b=[];E a=c.P;20(a&&a!=R){b.Y(a);a=a.P}F b},25:9(a,e,c,b){e=e||1;E d=0;I(;a;a=a[c])6(a.1t==1&&++d==e)1J;F a},4A:9(n,a){E r=[];I(;n;n=n.2a){6(n.1t==1&&(!a||n!=a))r.Y(n)}F r}});C.1c={1r:9(f,d,c,b){6(C.N.12&&f.3t!=Q)f=19;6(!c.22)c.22=7.22++;6(b!=Q){E e=c;c=9(){F e.T(7,1a)};c.V=b;c.22=e.22}6(!f.$1i)f.$1i={};6(!f.$1y)f.$1y=9(){E a;6(1f C=="Q"||C.1c.4n)F a;a=C.1c.1y.T(f,1a);F a};E g=f.$1i[d];6(!g){g=f.$1i[d]={};6(f.4m)f.4m(d,f.$1y,M);G f.7M("3r"+d,f.$1y)}g[c.22]=c;7.1D[d]=O},22:1,1D:{},23:9(c,b,a){E d=c.$1i,2c,45;6(d){6(b&&b.L){a=b.4l;b=b.L}6(!b){I(b 17 d)7.23(c,b)}G 6(d[b]){6(a)4k d[b][a.22];G I(a 17 c.$1i[b])4k d[b][a];I(2c 17 d[b])1J;6(!2c){6(c.4j)c.4j(b,c.$1y,M);G c.7L("3r"+b,c.$1y);2c=K;4k d[b]}}I(2c 17 d)1J;6(!2c)c.$1y=c.$1i=K}},1z:9(c,b,d){b=C.2V(b||[]);6(!d){6(7.1D[c])C("*").1r([19,R]).1z(c,b)}G{E a,2c,15=C.1g(d[c]||K);b.42(7.4i({L:c,1S:d}));6(C.1g(d.$1y))a=d.$1y.T(d,b);6(!15&&d["3r"+c]&&d["3r"+c].T(d,b)===M)a=M;6(15&&a!==M&&!(C.W(d,\'a\')&&c=="4h")){7.4n=O;d[c]()}7.4n=M}},1y:9(b){E a;b=C.1c.4i(b||19.1c||{});E c=7.$1i&&7.$1i[b.L],2e=1K.3v.3S.2S(1a,1);2e.42(b);I(E j 17 c){2e[0].4l=c[j];2e[0].V=c[j].V;6(c[j].T(7,2e)===M){b.2u();b.2X();a=M}}6(C.N.12)b.1S=b.2u=b.2X=b.4l=b.V=K;F a},4i:9(c){E a=c;c=C.14({},a);c.2u=9(){6(a.2u)a.2u();a.7I=M};c.2X=9(){6(a.2X)a.2X();a.7H=O};6(!c.1S&&c.5r)c.1S=c.5r;6(C.N.1H&&c.1S.1t==3)c.1S=a.1S.P;6(!c.4g&&c.4F)c.4g=c.4F==c.1S?c.7C:c.4F;6(c.5p==K&&c.66!=K){E e=R.3D,b=R.4z;c.5p=c.66+(e&&e.5o||b.5o||0);c.7z=c.7v+(e&&e.5m||b.5m||0)}6(!c.3Q&&(c.5k||c.5j))c.3Q=c.5k||c.5j;6(!c.5i&&c.5g)c.5i=c.5g;6(!c.3Q&&c.1X)c.3Q=(c.1X&1?1:(c.1X&2?3:(c.1X&4?2:0)));F c}};C.15.14({3l:9(c,a,b){F c=="5f"?7.5e(c,a,b):7.J(9(){C.1c.1r(7,c,b||a,b&&a)})},5e:9(d,b,c){F 7.J(9(){C.1c.1r(7,d,9(a){C(7).49(a);F(c||b).T(7,1a)},c&&b)})},49:9(a,b){F 7.J(9(){C.1c.23(7,a,b)})},1z:9(a,b){F 7.J(9(){C.1c.1z(a,b,7)})},1W:9(){E a=1a;F 7.4h(9(e){7.3T=0==7.3T?1:0;e.2u();F a[7.3T].T(7,[e])||M})},7p:9(f,g){9 3U(e){E p=e.4g;20(p&&p!=7)2g{p=p.P}2h(e){p=7};6(p==7)F M;F(e.L=="3W"?f:g).T(7,[e])}F 7.3W(3U).5b(3U)},1L:9(f){5a();6(C.36)f.T(R,[C]);G C.2C.Y(9(){F f.T(7,[C])});F 7}});C.14({36:M,2C:[],1L:9(){6(!C.36){C.36=O;6(C.2C){C.J(C.2C,9(){7.T(R)});C.2C=K}6(C.N.3J||C.N.2t)R.4j("59",C.1L,M);6(!19.7m.H)C(19).2f(9(){C("#4b").23()})}}});C.J(("7l,7k,2f,7j,7i,5f,4h,7g,"+"7f,7d,7c,3W,5b,7b,2b,"+"4u,7a,79,78,3f").2M(","),9(i,o){C.15[o]=9(f){F f?7.3l(o,f):7.1z(o)}});E w=M;9 5a(){6(w)F;w=O;6(C.N.3J||C.N.2t)R.4m("59",C.1L,M);G 6(C.N.12){R.75("<73"+"72 2j=4b 70=O "+"32=//:><\\/33>");E a=R.37("4b");6(a)a.6Z=9(){6(R.3d!="1x")F;C.1L()};a=K}G 6(C.N.1H)C.48=3t(9(){6(R.3d=="6Y"||R.3d=="1x"){47(C.48);C.48=K;C.1L()}},10);C.1c.1r(19,"2f",C.1L)}C.15.14({6X:9(c,b,a){7.2f(c,b,a,1)},2f:9(g,e,c,d){6(C.1g(g))F 7.3l("2f",g);c=c||9(){};E f="46";6(e)6(C.1g(e)){c=e;e=K}G{e=C.2O(e);f="55"}E h=7;C.31({1G:g,L:f,V:e,2F:d,1x:9(a,b){6(b=="1U"||!d&&b=="54")h.5W(a.43);4x(9(){h.J(c,[a.43,b,a])},13)}});F 7},6W:9(){F C.2O(7)},6V:9(){}});C.J("53,52,51,50,4Z,5h".2M(","),9(i,o){C.15[o]=9(f){F 7.3l(o,f)}});C.14({21:9(e,c,a,d,b){6(C.1g(c)){a=c;c=K}F C.31({L:"46",1G:e,V:c,1U:a,3G:d,2F:b})},6U:9(d,b,a,c){F C.21(d,b,a,c,1)},6T:9(b,a){F C.21(b,K,a,"33")},77:9(c,b,a){F C.21(c,b,a,"56")},6S:9(d,b,a,c){6(C.1g(b)){a=b;b={}}F C.31({L:"55",1G:d,V:b,1U:a,3G:c})},6R:9(a){C.3u.1Q=a},6Q:9(a){C.14(C.3u,a)},3u:{1D:O,L:"46",1Q:0,4Y:"6P/x-6O-38-6N",4X:O,2w:O,V:K},3h:{},31:9(s){s=C.14(O,s,C.14(O,{},C.3u,s));6(s.V){6(s.4X&&1f s.V!="1E")s.V=C.2O(s.V);6(s.L.2D()=="21"){s.1G+=(s.1G.U("?")>-1?"&":"?")+s.V;s.V=K}}6(s.1D&&!C.40++)C.1c.1z("53");E f=M;E h=19.4W?1s 4W("6M.6K"):1s 58();h.6J(s.L,s.1G,s.2w);6(s.V)h.4c("7r-7s",s.4Y);6(s.2F)h.4c("6G-3Z-6E",C.3h[s.1G]||"7w, 6C 7y 6B 4J:4J:4J 6z");h.4c("X-7D-7E","58");6(s.4U)s.4U(h);6(s.1D)C.1c.1z("5h",[h,s]);E g=9(d){6(!f&&h&&(h.3d==4||d=="1Q")){f=O;6(i){47(i);i=K}E c=d=="1Q"&&"1Q"||!C.5n(h)&&"3f"||s.2F&&C.5s(h,s.1G)&&"54"||"1U";6(c=="1U"){2g{E a=C.5q(h,s.3G)}2h(e){c="4I"}}6(c=="1U"){E b;2g{b=h.4f("4S-3Z")}2h(e){}6(s.2F&&b)C.3h[s.1G]=b;6(s.1U)s.1U(a,c);6(s.1D)C.1c.1z("4Z",[h,s])}G C.3X(s,h,c);6(s.1D)C.1c.1z("51",[h,s]);6(s.1D&&!--C.40)C.1c.1z("52");6(s.1x)s.1x(h,c);6(s.2w)h=K}};6(s.2w){E i=3t(g,13);6(s.1Q>0)4x(9(){6(h){h.6w();6(!f)g("1Q")}},s.1Q)}2g{h.6v(s.V)}2h(e){C.3X(s,h,K,e)}6(!s.2w)g();F h},3X:9(s,a,b,e){6(s.3f)s.3f(a,b,e);6(s.1D)C.1c.1z("50",[a,s,e])},40:0,5n:9(r){2g{F!r.26&&6t.6r=="4v:"||(r.26>=4R&&r.26<6q)||r.26==5z||C.N.1H&&r.26==Q}2h(e){}F M},5s:9(a,c){2g{E b=a.4f("4S-3Z");F a.26==5z||b==C.3h[c]||C.N.1H&&a.26==Q}2h(e){}F M},5q:9(r,a){E b=r.4f("6o-L");E c=a=="5F"||!a&&b&&b.U("5F")>=0;V=c?r.7W:r.43;6(c&&V.3D.4y=="4I")7X"4I";6(a=="33")C.4E(V);6(a=="56")V=2T("("+V+")");F V},2O:9(a){E s=[];6(a.1b==1K||a.3w)C.J(a,9(){s.Y(2y(7.6l)+"="+2y(7.2A))});G I(E j 17 a)6(a[j]&&a[j].1b==1K)C.J(a[j],9(){s.Y(2y(j)+"="+2y(7))});G s.Y(2y(j)+"="+2y(a[j]));F s.5w("&")}});C.15.14({1o:9(b,a){F b?7.1B({1u:"1o",24:"1o",1e:"1o"},b,a):7.1l(":1C").J(9(){7.S.11=7.2r?7.2r:"";6(C.1h(7,"11")=="1T")7.S.11="2m"}).3L()},1p:9(b,a){F b?7.1B({1u:"1p",24:"1p",1e:"1p"},b,a):7.1l(":4N").J(9(){7.2r=7.2r||C.1h(7,"11");6(7.2r=="1T")7.2r="2m";7.S.11="1T"}).3L()},5O:C.15.1W,1W:9(a,b){F C.1g(a)&&C.1g(b)?7.5O(a,b):a?7.1B({1u:"1W",24:"1W",1e:"1W"},a,b):7.J(9(){C(7)[C(7).3y(":1C")?"1o":"1p"]()})},6i:9(b,a){F 7.1B({1u:"1o"},b,a)},6h:9(b,a){F 7.1B({1u:"1p"},b,a)},6g:9(b,a){F 7.1B({1u:"1W"},b,a)},6f:9(b,a){F 7.1B({1e:"1o"},b,a)},89:9(b,a){F 7.1B({1e:"1p"},b,a)},6e:9(c,a,b){F 7.1B({1e:a},c,b)},1B:9(d,h,f,g){F 7.1n(9(){E c=C(7).3y(":1C"),1Z=C.5V(h,f,g),5U=7;I(E p 17 d){6(d[p]=="1p"&&c||d[p]=="1o"&&!c)F C.1g(1Z.1x)&&1Z.1x.T(7);6(p=="1u"||p=="24"){1Z.11=C.1h(7,"11");1Z.2z=7.S.2z}}6(1Z.2z!=K)7.S.2z="1C";7.2v=C.14({},d);C.J(d,9(a,b){E e=1s C.2Y(5U,1Z,a);6(b.1b==3x)e.3R(e.1Y()||0,b);G e[b=="1W"?c?"1o":"1p":b](d)});F O})},1n:9(a,b){6(!b){b=a;a="2Y"}F 7.J(9(){6(!7.1n)7.1n={};6(!7.1n[a])7.1n[a]=[];7.1n[a].Y(b);6(7.1n[a].H==1)b.T(7)})}});C.14({5V:9(b,a,c){E d=b&&b.1b==8G?b:{1x:c||!c&&a||C.1g(b)&&b,1N:b,35:c&&a||a&&a.1b!=8F&&a};d.1N=(d.1N&&d.1N.1b==3x?d.1N:{8D:8C,8B:4R}[d.1N])||8A;d.2U=d.1x;d.1x=9(){C.68(7,"2Y");6(C.1g(d.2U))d.2U.T(7)};F d},35:{62:9(p,n,b,a){F b+a*p},4H:9(p,n,b,a){F((-67.8z(p*67.8y)/2)+0.5)*a+b}},1n:{},68:9(b,a){a=a||"2Y";6(b.1n&&b.1n[a]){b.1n[a].4s();E f=b.1n[a][0];6(f)f.T(b)}},3N:[],2Y:9(f,e,g){E z=7;E y=f.S;z.a=9(){6(e.3q)e.3q.T(f,[z.2x]);6(g=="1e")C.1j(y,"1e",z.2x);G{y[g]=5K(z.2x)+"4t";6(g=="1u"||g=="24")y.11="2m"}};z.65=9(){F 3m(C.1h(f,g))};z.1Y=9(){E r=3m(C.34(f,g));F r&&r>-8v?r:z.65()};z.3R=9(c,b){z.4M=(1s 64()).63();z.2x=c;z.a();C.3N.Y(9(){F z.3q(c,b)});6(C.3N.H==1){E d=3t(9(){E a=C.3N;I(E i=0;ie.1N+z.4M){z.2x=c;z.a();6(f.2v)f.2v[g]=O;E b=O;I(E i 17 f.2v)6(f.2v[i]!==O)b=M;6(b){6(e.11!=K){y.2z=e.2z;y.11=e.11;6(C.1h(f,"11")=="1T")y.11="2m"}6(e.1p)y.11="1T";6(e.1p||e.1o)I(E p 17 f.2v)C.1j(y,p,f.2i[p])}6(b&&C.1g(e.1x))e.1x.T(f);F M}G{E n=t-7.4M;E p=n/e.1N;z.2x=C.35[e.35||(C.35.4H?"4H":"62")](p,n,a,(c-a),e.1N);z.a()}F O}}})})();',62,541,'||||||if|this||function|||||||||||||||||||||||||||||||var|return|else|length|for|each|null|type|false|browser|true|parentNode|undefined|document|style|apply|indexOf|data|nodeName||push|||display|msie||extend|fn|jQuery|in|className|window|arguments|constructor|event|test|opacity|typeof|isFunction|css|events|attr|firstChild|filter|div|queue|show|hide|mergeNum|add|new|nodeType|height|replace|tbody|complete|handle|trigger|table|animate|hidden|global|string|find|url|safari|toUpperCase|break|Array|ready|al|duration|pushStack|tb|timeout|stack|target|none|success|swap|toggle|button|cur|opt|while|get|guid|remove|width|nth|status|checked|selected|merge|nextSibling|select|ret|exec|args|load|try|catch|orig|id|match|_|block||rl|insertBefore|done|oldblock|trim|opera|preventDefault|curAnim|async|now|encodeURIComponent|overflow|value|grep|readyList|toLowerCase|color|ifModified|val|first|ownerDocument|domManip|substr|defaultView|split|has|param|last|text|multiFilter|call|eval|old|makeArray|innerHTML|stopPropagation|fx|childNodes|disabled|ajax|src|script|curCSS|easing|isReady|getElementById|form|input|float|getComputedStyle|clean|readyState|removeChild|error|static|lastModified|checkbox|selectedIndex|position|bind|parseFloat|String|oWidth|oHeight|step|on|toString|setInterval|ajaxSettings|prototype|jquery|Number|is|child|ol|cloneNode|RegExp|documentElement|isXMLDoc|getAttribute|dataType|append|styleFloat|mozilla|empty|end|map|timers|tr|el|which|custom|slice|lastToggle|handleHover|visibility|mouseover|handleError|lastChild|Modified|active|currentStyle|unshift|responseText|getPropertyValue|index|GET|clearInterval|safariTimer|unbind|init|__ie_init|setRequestHeader|unique|radio|getResponseHeader|relatedTarget|click|fix|removeEventListener|delete|handler|addEventListener|triggered|nodeIndex|appendChild|props|classFilter|shift|px|submit|file|expr|setTimeout|tagName|body|sibling|previousSibling|parents|deep|globalEval|fromElement|cssFloat|swing|parsererror|00|inArray|getElementsByTagName|startTime|visible|num|object|prop|200|Last|colgroup|beforeSend|fieldset|ActiveXObject|processData|contentType|ajaxSuccess|ajaxError|ajaxComplete|ajaxStop|ajaxStart|notmodified|POST|json|appendTo|XMLHttpRequest|DOMContentLoaded|bindReady|mouseout|prevObject|removeAttr|one|unload|ctrlKey|ajaxSend|metaKey|keyCode|charCode|not|scrollTop|httpSuccess|scrollLeft|pageX|httpData|srcElement|httpNotModified|after|before|prepend|join|parse|zoom|304|reset|image|password|odd|even|xml|quickClass|quickID|quickChild|setArray|parseInt|contains|gt|execScript|_toggle|lt|eq|href|nodeValue|alpha|self|speed|html|continue|parent|textContent|createTextNode|webkit|linear|getTime|Date|max|clientX|Math|dequeue|fl|createElement|version|100|NaN|fadeTo|fadeIn|slideToggle|slideUp|slideDown|setAttribute|getAttributeNode|name|method|action|content|cssText|300|protocol|FORM|location|options|send|abort|col|th|GMT|td|1970|01|cap|Since|colg|If|tfoot|thead|open|XMLHTTP|leg|Microsoft|urlencoded|www|application|ajaxSetup|ajaxTimeout|post|getScript|getIfModified|evalScripts|serialize|loadIfModified|loaded|onreadystatechange|defer|clientWidth|ipt|scr|clientHeight|write|relative|getJSON|keyup|keypress|keydown|change|mousemove|mouseup|left|mousedown|dblclick|right|scroll|resize|focus|blur|frames|absolute|clone|hover|offsetWidth|Content|Type|offsetHeight|Width|clientY|Thu|border|Jan|pageY|padding|Left|toElement|Requested|With|Right|Bottom|cancelBubble|returnValue|Top|size|detachEvent|attachEvent|substring|line|textarea|weight|enabled|font|innerText|only|uFFFF|responseXML|throw|u0128|417|toggleClass|removeClass|wrap|addClass|removeAttribute|insertAfter|prependTo|children|siblings|fadeOut|noConflict|prev|next|Boolean|maxLength|maxlength|readOnly|readonly|class|htmlFor|CSS1Compat|compatMode|boxModel|compatible|ie|ra|it|1px|rv|splice|userAgent|10000|navigator|concat|PI|cos|400|fast|600|slow|reverse|Function|Object|array|ig'.split('|'),0,{})) \ No newline at end of file diff --git a/lib/genshi/examples/tutorial/geddit/static/layout.css b/lib/genshi/examples/tutorial/geddit/static/layout.css new file mode 100644 --- /dev/null +++ b/lib/genshi/examples/tutorial/geddit/static/layout.css @@ -0,0 +1,42 @@ +html, body { margin: 0; padding: 0; } +body { background: #ddd; color: #333; font: normal 90%/1.3 Arial,Helvetica,sans-serif; } +:link, :visited { color: #c10000; text-decoration: none; } +:link:hover, :visited:hover { text-decoration: underline; } +:link img, :visited img { border: none; } +h1 { color: #666; + font: normal xx-large/1.5 Georgia,serif; margin: 0 0 .5em; +} +blockquote { font-style: italic; } +form table { margin-bottom: 1em; } +form table tbody th { font-weight: normal; padding-top: .3em; text-align: right; + vertical-align: top; +} +form label { color: #666; font-size: 90%; } + +#wrap { background: #fff; width: 600px; margin: 30px auto; } +#content { border-left: 10px solid #b00; min-height: 240px; padding: 10px; } +#header img { float: right; margin-top: -21px; margin-right: -37px; } + +#footer { background: #4A4D4D; border-top: 2px solid #333; font-size: x-small; + padding: 3px; text-align: center; +} +#footer hr { display: none; } +.legalese { color: #999; margin: 0; } + +ol.links li .info { font-size: 85%; } +ol.links li .info :link, ol.links li .info :visited { color: #666; } + +ul.comments { list-style: none; margin: 1em 0; padding: 0 0 0 1em; } +ul.comments li { color: #999; margin: 0 0 1em; } +ul.comments blockquote { color: #333; font-style: normal; margin: 0; + padding: 0; +} +.action:link, .action:visited { background: #f3f3f3; border: 1px outset #ddd; + color: #666; font-size: 90%; padding: 0 .3em; +} +.action:link:hover, .action:visited:hover { background: #e8e8e8; + border-color: #aaa; color: #000; text-decoration: none; +} + +form p.hint { color: #666; font-size: 90%; font-style: italic; margin: 0; } +form .error { color: #b00; } diff --git a/lib/genshi/examples/tutorial/geddit/static/logo.gif b/lib/genshi/examples/tutorial/geddit/static/logo.gif new file mode 100644 index 0000000000000000000000000000000000000000..60267e548999abf7ddb818be82c0e9397ef51b7b GIT binary patch [cut] diff --git a/lib/genshi/examples/tutorial/geddit/templates/_comment.html b/lib/genshi/examples/tutorial/geddit/templates/_comment.html new file mode 100644 --- /dev/null +++ b/lib/genshi/examples/tutorial/geddit/templates/_comment.html @@ -0,0 +1,5 @@ + +
+ ${comment.username} at ${comment.time.strftime('%x %X')} +
${HTML(comment.content)}
+
diff --git a/lib/genshi/examples/tutorial/geddit/templates/_form.html b/lib/genshi/examples/tutorial/geddit/templates/_form.html new file mode 100644 --- /dev/null +++ b/lib/genshi/examples/tutorial/geddit/templates/_form.html @@ -0,0 +1,24 @@ +
+ + + + + + + + + +
Your name: + + ${errors.username} +
Comment: + +
${errors.content} +
You can use HTML tags here for formatting.
+
+ + +
+
diff --git a/lib/genshi/examples/tutorial/geddit/templates/comment.html b/lib/genshi/examples/tutorial/geddit/templates/comment.html new file mode 100644 --- /dev/null +++ b/lib/genshi/examples/tutorial/geddit/templates/comment.html @@ -0,0 +1,18 @@ + + + + + Comment on “${link.title}” + + +
Comment on “${link.title}”
+
+ In reply to ${comment.username} + at ${comment.time.strftime('%x %X')}: +
${comment.content}
+
+ + + diff --git a/lib/genshi/examples/tutorial/geddit/templates/index.html b/lib/genshi/examples/tutorial/geddit/templates/index.html new file mode 100644 --- /dev/null +++ b/lib/genshi/examples/tutorial/geddit/templates/index.html @@ -0,0 +1,28 @@ + + + + + News + + + +
News
+ +
+
+ ${link.title} + posted by ${link.username} at ${link.time.strftime('%x %X')} +
+ + ${len(link.comments)} comments + +
+
+
+ +
Submit new link
+ + diff --git a/lib/genshi/examples/tutorial/geddit/templates/index.xml b/lib/genshi/examples/tutorial/geddit/templates/index.xml new file mode 100644 --- /dev/null +++ b/lib/genshi/examples/tutorial/geddit/templates/index.xml @@ -0,0 +1,23 @@ + + + + Geddit News + + + + ${links[0].time.isoformat()} + + + ${link.url} + + + ${url('/info/%s/' % link.id)} + + ${link.username} + + ${link.time.isoformat()} +
${link.title}
+ + + diff --git a/lib/genshi/examples/tutorial/geddit/templates/info.html b/lib/genshi/examples/tutorial/geddit/templates/info.html new file mode 100644 --- /dev/null +++ b/lib/genshi/examples/tutorial/geddit/templates/info.html @@ -0,0 +1,62 @@ + + + + + ${link.title} + + + + +
${link.title}
+ ${link.url}
+ posted by ${link.username} at ${link.time.strftime('%x %X')}
+ +
+ +
+ +
Add comment
+ + diff --git a/lib/genshi/examples/tutorial/geddit/templates/info.xml b/lib/genshi/examples/tutorial/geddit/templates/info.xml new file mode 100644 --- /dev/null +++ b/lib/genshi/examples/tutorial/geddit/templates/info.xml @@ -0,0 +1,28 @@ + + + + Geddit: ${link.title} + + + + + ${time.isoformat()} + + + + + Comment ${len(link.comments) - idx} on “${link.title}” + + ${url('/info/%s/' % link.id)}#comment${idx} + + ${comment.username} + + ${comment.time.isoformat()} +
+ ${HTML(comment.content)} +
+ + + diff --git a/lib/genshi/examples/tutorial/geddit/templates/layout.html b/lib/genshi/examples/tutorial/geddit/templates/layout.html new file mode 100644 --- /dev/null +++ b/lib/genshi/examples/tutorial/geddit/templates/layout.html @@ -0,0 +1,31 @@ + + + + + + + Geddit<py:if test="title">: ${title}</py:if> + + + + ${select('*[local-name()!="title"]')} + + + + +
+
+ +
+
+ ${select('*|text()')} +
+
+
+
© 2007 Edgewall Software
+
+
+ + + diff --git a/lib/genshi/examples/tutorial/geddit/templates/submit.html b/lib/genshi/examples/tutorial/geddit/templates/submit.html new file mode 100644 --- /dev/null +++ b/lib/genshi/examples/tutorial/geddit/templates/submit.html @@ -0,0 +1,41 @@ + + + + + Submit new link + + +
Submit new link
+ +
+ + + + + + + + + + + + +
Your name: + + ${errors.username} +
Link URL: + + ${errors.url} +
Title: + + ${errors.title} +
+ + +
+
+ + + diff --git a/lib/genshi/examples_to_py3k.sh b/lib/genshi/examples_to_py3k.sh new file mode 100644 --- /dev/null +++ b/lib/genshi/examples_to_py3k.sh @@ -0,0 +1,8 @@ +#!/bin/sh +# +# Script to run 2to3 on files not covered by setup.py +# +export PYTHONIOENCODING=utf8 + +# General 2to3 run +2to3 -w --no-diffs examples/ diff --git a/lib/genshi/fixes/__init__.py b/lib/genshi/fixes/__init__.py new file mode 100644 diff --git a/lib/genshi/fixes/fix_unicode_in_strings.py b/lib/genshi/fixes/fix_unicode_in_strings.py new file mode 100644 --- /dev/null +++ b/lib/genshi/fixes/fix_unicode_in_strings.py @@ -0,0 +1,17 @@ +"""Fixer that changes expressions inside strings literals from u"..." to "...". + +""" + +import re +from lib2to3 import fixer_base + +_literal_re = re.compile(r"(.+?)\b[uU]([rR]?[\'\"])") + +class FixUnicodeInStrings(fixer_base.BaseFix): + + PATTERN = "STRING" + + def transform(self, node, results): + new = node.clone() + new.value = _literal_re.sub(r"\1\2", new.value) + return new diff --git a/lib/genshi/genshi/__init__.py b/lib/genshi/genshi/__init__.py new file mode 100644 --- /dev/null +++ b/lib/genshi/genshi/__init__.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2006-2009 Edgewall Software +# All rights reserved. +# +# This software is licensed as described in the file COPYING, which +# you should have received as part of this distribution. The terms +# are also available at http://genshi.edgewall.org/wiki/License. +# +# This software consists of voluntary contributions made by many +# individuals. For the exact contribution history, see the revision +# history and logs, available at http://genshi.edgewall.org/log/. + +"""This package provides various means for generating and processing web markup +(XML or HTML). + +The design is centered around the concept of streams of markup events (similar +in concept to SAX parsing events) which can be processed in a uniform manner +independently of where or how they are produced. +""" + +__docformat__ = 'restructuredtext en' +__version__ = '0.7' + +from genshi.core import * +from genshi.input import ParseError, XML, HTML diff --git a/lib/genshi/genshi/_speedups.c b/lib/genshi/genshi/_speedups.c new file mode 100644 --- /dev/null +++ b/lib/genshi/genshi/_speedups.c @@ -0,0 +1,702 @@ +/* + * Copyright (C) 2006-2008 Edgewall Software + * All rights reserved. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at http://genshi.edgewall.org/wiki/License. + * + * This software consists of voluntary contributions made by many + * individuals. For the exact contribution history, see the revision + * history and logs, available at http://genshi.edgewall.org/log/. + */ + +#include +#include + +#if PY_MAJOR_VERSION > 2 +# define IS_PY3K +#elif PY_VERSION_HEX < 0x02050000 && !defined(PY_SSIZE_T_MIN) + typedef int Py_ssize_t; +# define PY_SSIZE_T_MAX INT_MAX +# define PY_SSIZE_T_MIN INT_MIN +#endif + +/* We only use Unicode Strings in this module */ +#ifndef IS_PY3K +# define PyObject_Str PyObject_Unicode +#endif + +static PyObject *amp1, *amp2, *lt1, *lt2, *gt1, *gt2, *qt1, *qt2; +static PyObject *stripentities, *striptags; + +static void +init_constants(void) +{ + PyObject *util = PyImport_ImportModule("genshi.util"); + stripentities = PyObject_GetAttrString(util, "stripentities"); + striptags = PyObject_GetAttrString(util, "striptags"); + Py_DECREF(util); + + amp1 = PyUnicode_DecodeASCII("&", 1, NULL); + amp2 = PyUnicode_DecodeASCII("&", 5, NULL); + lt1 = PyUnicode_DecodeASCII("<", 1, NULL); + lt2 = PyUnicode_DecodeASCII("<", 4, NULL); + gt1 = PyUnicode_DecodeASCII(">", 1, NULL); + gt2 = PyUnicode_DecodeASCII(">", 4, NULL); + qt1 = PyUnicode_DecodeASCII("\"", 1, NULL); + qt2 = PyUnicode_DecodeASCII(""", 5, NULL); +} + +/* Markup class */ + +PyTypeObject MarkupType; /* declared later */ + +PyDoc_STRVAR(Markup__doc__, +"Marks a string as being safe for inclusion in HTML/XML output without\n\ +needing to be escaped."); + +static PyObject * +escape(PyObject *text, int quotes) +{ + PyObject *args, *ret; + PyUnicodeObject *in, *out; + Py_UNICODE *inp, *outp; + int len, inn, outn; + + if (PyObject_TypeCheck(text, &MarkupType)) { + Py_INCREF(text); + return text; + } + if (PyObject_HasAttrString(text, "__html__")) { + ret = PyObject_CallMethod(text, "__html__", NULL); + args = PyTuple_New(1); + if (args == NULL) { + Py_DECREF(ret); + return NULL; + } + PyTuple_SET_ITEM(args, 0, ret); + ret = MarkupType.tp_new(&MarkupType, args, NULL); + Py_DECREF(args); + return ret; + } + in = (PyUnicodeObject *) PyObject_Str(text); + if (in == NULL) { + return NULL; + } + /* First we need to figure out how long the escaped string will be */ + len = inn = 0; + inp = in->str; + while (*(inp) || in->length > inp - in->str) { + switch (*inp++) { + case '&': len += 5; inn++; break; + case '"': len += quotes ? 5 : 1; inn += quotes ? 1 : 0; break; + case '<': + case '>': len += 4; inn++; break; + default: len++; + } + } + + /* Do we need to escape anything at all? */ + if (!inn) { + args = PyTuple_New(1); + if (args == NULL) { + Py_DECREF((PyObject *) in); + return NULL; + } + PyTuple_SET_ITEM(args, 0, (PyObject *) in); + ret = MarkupType.tp_new(&MarkupType, args, NULL); + Py_DECREF(args); + return ret; + } + + out = (PyUnicodeObject*) PyUnicode_FromUnicode(NULL, len); + if (out == NULL) { + Py_DECREF((PyObject *) in); + return NULL; + } + + outn = 0; + inp = in->str; + outp = out->str; + while (*(inp) || in->length > inp - in->str) { + if (outn == inn) { + /* copy rest of string if we have already replaced everything */ + Py_UNICODE_COPY(outp, inp, in->length - (inp - in->str)); + break; + } + switch (*inp) { + case '&': + Py_UNICODE_COPY(outp, ((PyUnicodeObject *) amp2)->str, 5); + outp += 5; + outn++; + break; + case '"': + if (quotes) { + Py_UNICODE_COPY(outp, ((PyUnicodeObject *) qt2)->str, 5); + outp += 5; + outn++; + } else { + *outp++ = *inp; + } + break; + case '<': + Py_UNICODE_COPY(outp, ((PyUnicodeObject *) lt2)->str, 4); + outp += 4; + outn++; + break; + case '>': + Py_UNICODE_COPY(outp, ((PyUnicodeObject *) gt2)->str, 4); + outp += 4; + outn++; + break; + default: + *outp++ = *inp; + } + inp++; + } + + Py_DECREF((PyObject *) in); + + args = PyTuple_New(1); + if (args == NULL) { + Py_DECREF((PyObject *) out); + return NULL; + } + PyTuple_SET_ITEM(args, 0, (PyObject *) out); + ret = MarkupType.tp_new(&MarkupType, args, NULL); + Py_DECREF(args); + return ret; +} + +PyDoc_STRVAR(escape__doc__, +"Create a Markup instance from a string and escape special characters\n\ +it may contain (<, >, & and \").\n\ +\n\ +>>> escape('\"1 < 2\"')\n\ +\n\ +\n\ +If the `quotes` parameter is set to `False`, the \" character is left\n\ +as is. Escaping quotes is generally only required for strings that are\n\ +to be used in attribute values.\n\ +\n\ +>>> escape('\"1 < 2\"', quotes=False)\n\ +\n\ +\n\ +:param text: the text to escape\n\ +:param quotes: if ``True``, double quote characters are escaped in\n\ + addition to the other special characters\n\ +:return: the escaped `Markup` string\n\ +:rtype: `Markup`\n\ +"); + +static PyObject * +Markup_escape(PyTypeObject* type, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"text", "quotes", 0}; + PyObject *text = NULL; + char quotes = 1; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|b", kwlist, &text, "es)) { + return NULL; + } + if (PyObject_Not(text)) { + args = PyTuple_New(0); + if (args == NULL) + return NULL; + text = type->tp_new(type, args, NULL); + Py_DECREF(args); + return text; + } + if (PyObject_TypeCheck(text, type)) { + Py_INCREF(text); + return text; + } + return escape(text, quotes); +} + +static PyObject * +Markup_html(PyObject *self) +{ + Py_INCREF(self); + return self; +} + +PyDoc_STRVAR(join__doc__, +"Return a `Markup` object which is the concatenation of the strings\n\ +in the given sequence, where this `Markup` object is the separator\n\ +between the joined elements.\n\ +\n\ +Any element in the sequence that is not a `Markup` instance is\n\ +automatically escaped.\n\ +\n\ +:param seq: the sequence of strings to join\n\ +:param escape_quotes: whether double quote characters in the elements\n\ + should be escaped\n\ +:return: the joined `Markup` object\n\ +:rtype: `Markup`\n\ +:see: `escape`\n\ +"); + +static PyObject * +Markup_join(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"seq", "escape_quotes", 0}; + PyObject *seq = NULL, *seq2, *tmp, *tmp2; + char quotes = 1; + Py_ssize_t n; + int i; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|b", kwlist, &seq, "es)) { + return NULL; + } + if (!PySequence_Check(seq)) { + return NULL; + } + n = PySequence_Size(seq); + if (n < 0) { + return NULL; + } + seq2 = PyTuple_New(n); + if (seq2 == NULL) { + return NULL; + } + for (i = 0; i < n; i++) { + tmp = PySequence_GetItem(seq, i); + if (tmp == NULL) { + Py_DECREF(seq2); + return NULL; + } + tmp2 = escape(tmp, quotes); + if (tmp2 == NULL) { + Py_DECREF(seq2); + return NULL; + } + PyTuple_SET_ITEM(seq2, i, tmp2); + Py_DECREF(tmp); + } + tmp = PyUnicode_Join(self, seq2); + Py_DECREF(seq2); + if (tmp == NULL) + return NULL; + args = PyTuple_New(1); + if (args == NULL) { + Py_DECREF(tmp); + return NULL; + } + PyTuple_SET_ITEM(args, 0, tmp); + tmp = MarkupType.tp_new(&MarkupType, args, NULL); + Py_DECREF(args); + return tmp; +} + +static PyObject * +Markup_add(PyObject *self, PyObject *other) +{ + PyObject *tmp, *tmp2, *args, *ret; + if (PyObject_TypeCheck(self, &MarkupType)) { + tmp = escape(other, 1); + if (tmp == NULL) + return NULL; + tmp2 = PyUnicode_Concat(self, tmp); + } else { // __radd__ + tmp = escape(self, 1); + if (tmp == NULL) + return NULL; + tmp2 = PyUnicode_Concat(tmp, other); + } + Py_DECREF(tmp); + if (tmp2 == NULL) + return NULL; + args = PyTuple_New(1); + if (args == NULL) { + Py_DECREF(tmp2); + return NULL; + } + PyTuple_SET_ITEM(args, 0, tmp2); + ret = MarkupType.tp_new(&MarkupType, args, NULL); + Py_DECREF(args); + return ret; +} + +static PyObject * +Markup_mod(PyObject *self, PyObject *args) +{ + PyObject *tmp, *tmp2, *ret, *args2; + int i; + Py_ssize_t nargs = 0; + PyObject *kwds = NULL; + + if (PyDict_Check(args)) { + kwds = args; + } + if (kwds && PyDict_Size(kwds)) { + PyObject *kwcopy, *key, *value; + Py_ssize_t pos = 0; + + kwcopy = PyDict_Copy( kwds ); + if (kwcopy == NULL) { + return NULL; + } + while (PyDict_Next(kwcopy, &pos, &key, &value)) { + tmp = escape(value, 1); + if (tmp == NULL) { + Py_DECREF(kwcopy); + return NULL; + } + if (PyDict_SetItem(kwcopy, key, tmp) < 0) { + Py_DECREF(tmp); + Py_DECREF(kwcopy); + return NULL; + } + } + tmp = PyUnicode_Format(self, kwcopy); + Py_DECREF(kwcopy); + if (tmp == NULL) { + return NULL; + } + } else if (PyTuple_Check(args)) { + nargs = PyTuple_GET_SIZE(args); + args2 = PyTuple_New(nargs); + if (args2 == NULL) { + return NULL; + } + for (i = 0; i < nargs; i++) { + tmp = escape(PyTuple_GET_ITEM(args, i), 1); + if (tmp == NULL) { + Py_DECREF(args2); + return NULL; + } + PyTuple_SET_ITEM(args2, i, tmp); + } + tmp = PyUnicode_Format(self, args2); + Py_DECREF(args2); + if (tmp == NULL) { + return NULL; + } + } else { + tmp2 = escape(args, 1); + if (tmp2 == NULL) { + return NULL; + } + tmp = PyUnicode_Format(self, tmp2); + Py_DECREF(tmp2); + if (tmp == NULL) { + return NULL; + } + } + args = PyTuple_New(1); + if (args == NULL) { + Py_DECREF(tmp); + return NULL; + } + PyTuple_SET_ITEM(args, 0, tmp); + ret = PyUnicode_Type.tp_new(&MarkupType, args, NULL); + Py_DECREF(args); + return ret; +} + +static PyObject * +Markup_mul(PyObject *self, PyObject *num) +{ + PyObject *unicode, *result, *args; + + if (PyObject_TypeCheck(self, &MarkupType)) { + unicode = PyObject_Str(self); + if (unicode == NULL) return NULL; + result = PyNumber_Multiply(unicode, num); + } else { // __rmul__ + unicode = PyObject_Str(num); + if (unicode == NULL) return NULL; + result = PyNumber_Multiply(unicode, self); + } + Py_DECREF(unicode); + + if (result == NULL) return NULL; + args = PyTuple_New(1); + if (args == NULL) { + Py_DECREF(result); + return NULL; + } + PyTuple_SET_ITEM(args, 0, result); + result = PyUnicode_Type.tp_new(&MarkupType, args, NULL); + Py_DECREF(args); + + return result; +} + +static PyObject * +Markup_repr(PyObject *self) +{ + PyObject *format, *result, *args; + +#ifdef IS_PY3K + format = PyUnicode_FromString(""); +#else + format = PyString_FromString(""); +#endif + if (format == NULL) return NULL; + result = PyObject_Str(self); + if (result == NULL) { + Py_DECREF(format); + return NULL; + } + args = PyTuple_New(1); + if (args == NULL) { + Py_DECREF(format); + Py_DECREF(result); + return NULL; + } + PyTuple_SET_ITEM(args, 0, result); +#ifdef IS_PY3K + result = PyUnicode_Format(format, args); +#else + result = PyString_Format(format, args); +#endif + Py_DECREF(format); + Py_DECREF(args); + return result; +} + +PyDoc_STRVAR(unescape__doc__, +"Reverse-escapes &, <, >, and \" and returns a `unicode` object.\n\ +\n\ +>>> Markup('1 < 2').unescape()\n\ +u'1 < 2'\n\ +\n\ +:return: the unescaped string\n\ +:rtype: `unicode`\n\ +:see: `genshi.core.unescape`\n\ +"); + +static PyObject * +Markup_unescape(PyObject* self) +{ + PyObject *tmp, *tmp2; + + tmp = PyUnicode_Replace(self, qt2, qt1, -1); + if (tmp == NULL) return NULL; + tmp2 = PyUnicode_Replace(tmp, gt2, gt1, -1); + Py_DECREF(tmp); + if (tmp2 == NULL) return NULL; + tmp = PyUnicode_Replace(tmp2, lt2, lt1, -1); + Py_DECREF(tmp2); + if (tmp == NULL) return NULL; + tmp2 = PyUnicode_Replace(tmp, amp2, amp1, -1); + Py_DECREF(tmp); + return tmp2; +} + +PyDoc_STRVAR(stripentities__doc__, +"Return a copy of the text with any character or numeric entities\n\ +replaced by the equivalent UTF-8 characters.\n\ +\n\ +If the `keepxmlentities` parameter is provided and evaluates to `True`,\n\ +the core XML entities (``&``, ``'``, ``>``, ``<`` and\n\ +``"``) are not stripped.\n\ +\n\ +:return: a `Markup` instance with entities removed\n\ +:rtype: `Markup`\n\ +:see: `genshi.util.stripentities`\n\ +"); + +static PyObject * +Markup_stripentities(PyObject* self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"keepxmlentities", 0}; + PyObject *result, *args2; + char keepxml = 0; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|b", kwlist, &keepxml)) { + return NULL; + } + + if (stripentities == NULL) return NULL; + result = PyObject_CallFunction(stripentities, "Ob", self, keepxml); + if (result == NULL) return NULL; + args2 = PyTuple_New(1); + if (args2 == NULL) { + Py_DECREF(result); + return NULL; + } + PyTuple_SET_ITEM(args2, 0, result); + result = MarkupType.tp_new(&MarkupType, args2, NULL); + Py_DECREF(args2); + return result; +} + +PyDoc_STRVAR(striptags__doc__, +"""Return a copy of the text with all XML/HTML tags removed.\n\ +\n\ +:return: a `Markup` instance with all tags removed\n\ +:rtype: `Markup`\n\ +:see: `genshi.util.striptags`\n\ +"); + +static PyObject * +Markup_striptags(PyObject* self) +{ + PyObject *result, *args; + + if (striptags == NULL) return NULL; + result = PyObject_CallFunction(striptags, "O", self); + if (result == NULL) return NULL; + args = PyTuple_New(1); + if (args == NULL) { + Py_DECREF(result); + return NULL; + } + PyTuple_SET_ITEM(args, 0, result); + result = MarkupType.tp_new(&MarkupType, args, NULL); + Py_DECREF(args); + return result; +} + +typedef struct { + PyUnicodeObject HEAD; +} MarkupObject; + +static PyMethodDef Markup_methods[] = { + {"__html__", (PyCFunction) Markup_html, METH_NOARGS, NULL}, + {"escape", (PyCFunction) Markup_escape, + METH_VARARGS|METH_CLASS|METH_KEYWORDS, escape__doc__}, + {"join", (PyCFunction)Markup_join, METH_VARARGS|METH_KEYWORDS, join__doc__}, + {"unescape", (PyCFunction)Markup_unescape, METH_NOARGS, unescape__doc__}, + {"stripentities", (PyCFunction) Markup_stripentities, + METH_VARARGS|METH_KEYWORDS, stripentities__doc__}, + {"striptags", (PyCFunction) Markup_striptags, METH_NOARGS, + striptags__doc__}, + {NULL} /* Sentinel */ +}; + +static PyNumberMethods Markup_as_number = { + Markup_add, /*nb_add*/ + 0, /*nb_subtract*/ + Markup_mul, /*nb_multiply*/ +#ifndef IS_PY3K + 0, /*nb_divide*/ +#endif + Markup_mod, /*nb_remainder*/ +}; + +PyTypeObject MarkupType = { +#ifdef IS_PY3K + PyVarObject_HEAD_INIT(NULL, 0) +#else + PyObject_HEAD_INIT(NULL) + 0, +#endif + "genshi._speedups.Markup", + sizeof(MarkupObject), + 0, + 0, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ +#ifdef IS_PY3K + 0, /*tp_reserved*/ +#else + 0, /*tp_compare*/ +#endif + Markup_repr, /*tp_repr*/ + &Markup_as_number, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash */ + + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + +#ifdef IS_PY3K + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_UNICODE_SUBCLASS, /*tp_flags*/ +#elif defined(Py_TPFLAGS_UNICODE_SUBCLASS) + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_CHECKTYPES | Py_TPFLAGS_UNICODE_SUBCLASS, /*tp_flags*/ +#else + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_CHECKTYPES, /*tp_flags*/ +#endif + + Markup__doc__,/*tp_doc*/ + + 0, /*tp_traverse*/ + 0, /*tp_clear*/ + + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + + /* Attribute descriptor and subclassing stuff */ + + Markup_methods,/*tp_methods*/ + 0, /*tp_members*/ + 0, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + 0, /*tp_dictoffset*/ + + 0, /*tp_init*/ + 0, /*tp_alloc will be set to PyType_GenericAlloc in module init*/ + 0, /*tp_new*/ + 0, /*tp_free Low-level free-memory routine */ + 0, /*tp_is_gc For PyObject_IS_GC */ + 0, /*tp_bases*/ + 0, /*tp_mro method resolution order */ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0 /*tp_weaklist*/ +}; + +#ifdef IS_PY3K +struct PyModuleDef module_def = { + PyModuleDef_HEAD_INIT, /*m_base*/ + "_speedups", /*m_name*/ + NULL, /*m_doc*/ + -1, /*m_size*/ + NULL, /*m_methods*/ + NULL, /*m_reload*/ + NULL, /*m_traverse*/ + NULL, /*m_clear*/ + NULL /*m_free*/ +}; + +PyObject * +PyInit__speedups(void) +#else +PyMODINIT_FUNC +init_speedups(void) +#endif +{ + PyObject *module; + + /* Workaround for quirk in Visual Studio, see + */ + MarkupType.tp_base = &PyUnicode_Type; + + if (PyType_Ready(&MarkupType) < 0) +#ifdef IS_PY3K + return NULL; +#else + return; +#endif + + init_constants(); + +#ifdef IS_PY3K + module = PyModule_Create(&module_def); +#else + module = Py_InitModule("_speedups", NULL); +#endif + Py_INCREF(&MarkupType); + PyModule_AddObject(module, "Markup", (PyObject *) &MarkupType); + +#ifdef IS_PY3K + return module; +#endif +} diff --git a/lib/genshi/genshi/builder.py b/lib/genshi/genshi/builder.py new file mode 100644 --- /dev/null +++ b/lib/genshi/genshi/builder.py @@ -0,0 +1,359 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2006-2009 Edgewall Software +# All rights reserved. +# +# This software is licensed as described in the file COPYING, which +# you should have received as part of this distribution. The terms +# are also available at http://genshi.edgewall.org/wiki/License. +# +# This software consists of voluntary contributions made by many +# individuals. For the exact contribution history, see the revision +# history and logs, available at http://genshi.edgewall.org/log/. + +"""Support for programmatically generating markup streams from Python code using +a very simple syntax. The main entry point to this module is the `tag` object +(which is actually an instance of the ``ElementFactory`` class). You should +rarely (if ever) need to directly import and use any of the other classes in +this module. + +Elements can be created using the `tag` object using attribute access. For +example: + +>>> doc = tag.p('Some text and ', tag.a('a link', href='http://example.org/'), '.') +>>> doc + + +This produces an `Element` instance which can be further modified to add child +nodes and attributes. This is done by "calling" the element: positional +arguments are added as child nodes (alternatively, the `Element.append` method +can be used for that purpose), whereas keywords arguments are added as +attributes: + +>>> doc(tag.br) + +>>> print(doc) +
Some text and a link.
+ +If an attribute name collides with a Python keyword, simply append an underscore +to the name: + +>>> doc(class_='intro') + +>>> print(doc) +
Some text and a link.
+ +As shown above, an `Element` can easily be directly rendered to XML text by +printing it or using the Python ``str()`` function. This is basically a +shortcut for converting the `Element` to a stream and serializing that +stream: + +>>> stream = doc.generate() +>>> stream #doctest: +ELLIPSIS + +>>> print(stream) +
Some text and a link.
+ + +The `tag` object also allows creating "fragments", which are basically lists +of nodes (elements or text) that don't have a parent element. This can be useful +for creating snippets of markup that are attached to a parent element later (for +example in a template). Fragments are created by calling the `tag` object, which +returns an object of type `Fragment`: + +>>> fragment = tag('Hello, ', tag.em('world'), '!') +>>> fragment + +>>> print(fragment) +Hello, world! +""" + +from genshi.core import Attrs, Markup, Namespace, QName, Stream, \ + START, END, TEXT + +__all__ = ['Fragment', 'Element', 'ElementFactory', 'tag'] +__docformat__ = 'restructuredtext en' + + +class Fragment(object): + """Represents a markup fragment, which is basically just a list of element + or text nodes. + """ + __slots__ = ['children'] + + def __init__(self): + """Create a new fragment.""" + self.children = [] + + def __add__(self, other): + return Fragment()(self, other) + + def __call__(self, *args): + """Append any positional arguments as child nodes. + + :see: `append` + """ + for arg in args: + self.append(arg) + return self + + def __iter__(self): + return self._generate() + + def __repr__(self): + return '<%s>' % type(self).__name__ + + def __str__(self): + return str(self.generate()) + + def __unicode__(self): + return unicode(self.generate()) + + def __html__(self): + return Markup(self.generate()) + + def append(self, node): + """Append an element or string as child node. + + :param node: the node to append; can be an `Element`, `Fragment`, or a + `Stream`, or a Python string or number + """ + if isinstance(node, (Stream, Element, basestring, int, float, long)): + # For objects of a known/primitive type, we avoid the check for + # whether it is iterable for better performance + self.children.append(node) + elif isinstance(node, Fragment): + self.children.extend(node.children) + elif node is not None: + try: + for child in node: + self.append(child) + except TypeError: + self.children.append(node) + + def _generate(self): + for child in self.children: + if isinstance(child, Fragment): + for event in child._generate(): + yield event + elif isinstance(child, Stream): + for event in child: + yield event + else: + if not isinstance(child, basestring): + child = unicode(child) + yield TEXT, child, (None, -1, -1) + + def generate(self): + """Return a markup event stream for the fragment. + + :rtype: `Stream` + """ + return Stream(self._generate()) + + +def _kwargs_to_attrs(kwargs): + attrs = [] + names = set() + for name, value in kwargs.items(): + name = name.rstrip('_').replace('_', '-') + if value is not None and name not in names: + attrs.append((QName(name), unicode(value))) + names.add(name) + return Attrs(attrs) + + +class Element(Fragment): + """Simple XML output generator based on the builder pattern. + + Construct XML elements by passing the tag name to the constructor: + + >>> print(Element('strong')) + + + Attributes can be specified using keyword arguments. The values of the + arguments will be converted to strings and any special XML characters + escaped: + + >>> print(Element('textarea', rows=10, cols=60)) + + >>> print(Element('span', title='1 < 2')) + <span title="1 < 2"/> + >>> print(Element('span', title='"baz"')) + <span title=""baz""/> + + The " character is escaped using a numerical entity. + The order in which attributes are rendered is undefined. + + If an attribute value evaluates to `None`, that attribute is not included + in the output: + + >>> print(Element('a', name=None)) + <a/> + + Attribute names that conflict with Python keywords can be specified by + appending an underscore: + + >>> print(Element('div', class_='warning')) + <div class="warning"/> + + Nested elements can be added to an element using item access notation. + The call notation can also be used for this and for adding attributes + using keyword arguments, as one would do in the constructor. + + >>> print(Element('ul')(Element('li'), Element('li'))) + <ul><li/><li/></ul> + >>> print(Element('a')('Label')) + <a>Label</a> + >>> print(Element('a')('Label', href="target")) + <a href="target">Label</a> + + Text nodes can be nested in an element by adding strings instead of + elements. Any special characters in the strings are escaped automatically: + + >>> print(Element('em')('Hello world')) + <em>Hello world</em> + >>> print(Element('em')(42)) + <em>42</em> + >>> print(Element('em')('1 < 2')) + <em>1 < 2</em> + + This technique also allows mixed content: + + >>> print(Element('p')('Hello ', Element('b')('world'))) + <p>Hello <b>world</b></p> + + Quotes are not escaped inside text nodes: + >>> print(Element('p')('"Hello"')) + <p>"Hello"</p> + + Elements can also be combined with other elements or strings using the + addition operator, which results in a `Fragment` object that contains the + operands: + + >>> print(Element('br') + 'some text' + Element('br')) + <br/>some text<br/> + + Elements with a namespace can be generated using the `Namespace` and/or + `QName` classes: + + >>> from genshi.core import Namespace + >>> xhtml = Namespace('http://www.w3.org/1999/xhtml') + >>> print(Element(xhtml.html, lang='en')) + <html xmlns="http://www.w3.org/1999/xhtml" lang="en"/> + """ + __slots__ = ['tag', 'attrib'] + + def __init__(self, tag_, **attrib): + Fragment.__init__(self) + self.tag = QName(tag_) + self.attrib = _kwargs_to_attrs(attrib) + + def __call__(self, *args, **kwargs): + """Append any positional arguments as child nodes, and keyword arguments + as attributes. + + :return: the element itself so that calls can be chained + :rtype: `Element` + :see: `Fragment.append` + """ + self.attrib |= _kwargs_to_attrs(kwargs) + Fragment.__call__(self, *args) + return self + + def __repr__(self): + return '<%s "%s">' % (type(self).__name__, self.tag) + + def _generate(self): + yield START, (self.tag, self.attrib), (None, -1, -1) + for kind, data, pos in Fragment._generate(self): + yield kind, data, pos + yield END, self.tag, (None, -1, -1) + + def generate(self): + """Return a markup event stream for the fragment. + + :rtype: `Stream` + """ + return Stream(self._generate()) + + +class ElementFactory(object): + """Factory for `Element` objects. + + A new element is created simply by accessing a correspondingly named + attribute of the factory object: + + >>> factory = ElementFactory() + >>> print(factory.foo) + <foo/> + >>> print(factory.foo(id=2)) + <foo id="2"/> + + Markup fragments (lists of nodes without a parent element) can be created + by calling the factory: + + >>> print(factory('Hello, ', factory.em('world'), '!')) + Hello, <em>world</em>! + + A factory can also be bound to a specific namespace: + + >>> factory = ElementFactory('http://www.w3.org/1999/xhtml') + >>> print(factory.html(lang="en")) + <html xmlns="http://www.w3.org/1999/xhtml" lang="en"/> + + The namespace for a specific element can be altered on an existing factory + by specifying the new namespace using item access: + + >>> factory = ElementFactory() + >>> print(factory.html(factory['http://www.w3.org/2000/svg'].g(id=3))) + <html><g xmlns="http://www.w3.org/2000/svg" id="3"/></html> + + Usually, the `ElementFactory` class is not be used directly. Rather, the + `tag` instance should be used to create elements. + """ + + def __init__(self, namespace=None): + """Create the factory, optionally bound to the given namespace. + + :param namespace: the namespace URI for any created elements, or `None` + for no namespace + """ + if namespace and not isinstance(namespace, Namespace): + namespace = Namespace(namespace) + self.namespace = namespace + + def __call__(self, *args): + """Create a fragment that has the given positional arguments as child + nodes. + + :return: the created `Fragment` + :rtype: `Fragment` + """ + return Fragment()(*args) + + def __getitem__(self, namespace): + """Return a new factory that is bound to the specified namespace. + + :param namespace: the namespace URI or `Namespace` object + :return: an `ElementFactory` that produces elements bound to the given + namespace + :rtype: `ElementFactory` + """ + return ElementFactory(namespace) + + def __getattr__(self, name): + """Create an `Element` with the given name. + + :param name: the tag name of the element to create + :return: an `Element` with the specified name + :rtype: `Element` + """ + return Element(self.namespace and self.namespace[name] or name) + + +tag = ElementFactory() +"""Global `ElementFactory` bound to the default namespace. + +:type: `ElementFactory` +""" diff --git a/lib/genshi/genshi/compat.py b/lib/genshi/genshi/compat.py new file mode 100644 --- /dev/null +++ b/lib/genshi/genshi/compat.py @@ -0,0 +1,115 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2006-2009 Edgewall Software +# All rights reserved. +# +# This software is licensed as described in the file COPYING, which +# you should have received as part of this distribution. The terms +# are also available at http://genshi.edgewall.org/wiki/License. +# +# This software consists of voluntary contributions made by many +# individuals. For the exact contribution history, see the revision +# history and logs, available at http://genshi.edgewall.org/log/. + +"""Various Python version compatibility classes and functions.""" + +import sys +from types import CodeType + + +IS_PYTHON2 = (sys.version_info[0] == 2) + + +# This function should only be called in Python 2, and will fail in Python 3 + +if IS_PYTHON2: + def stringrepr(string): + ascii = string.encode('ascii', 'backslashreplace') + quoted = "'" + ascii.replace("'", "\\'") + "'" + if len(ascii) > len(string): + return 'u' + quoted + return quoted +else: + def stringrepr(string): + raise RuntimeError( + 'Python 2 compatibility function. Not usable in Python 3.') + + +# We need to differentiate between StringIO and BytesIO in places + +if IS_PYTHON2: + from StringIO import StringIO + try: + from cStringIO import StringIO as BytesIO + except ImportError: + BytesIO = StringIO +else: + from io import StringIO, BytesIO + + +# We want to test bytestring input to some stuff. + +if IS_PYTHON2: + def wrapped_bytes(bstr): + assert bstr.startswith('b') + return bstr[1:] +else: + def wrapped_bytes(bstr): + assert bstr.startswith('b') + return bstr + + +# We do some scary stuff with CodeType() in template/eval.py + +if IS_PYTHON2: + def get_code_params(code): + return (code.co_nlocals, code.co_stacksize, code.co_flags, + code.co_code, code.co_consts, code.co_names, code.co_varnames, + code.co_filename, code.co_name, code.co_firstlineno, + code.co_lnotab, (), ()) + + def build_code_chunk(code, filename, name, lineno): + return CodeType(0, code.co_nlocals, code.co_stacksize, + code.co_flags | 0x0040, code.co_code, code.co_consts, + code.co_names, code.co_varnames, filename, name, + lineno, code.co_lnotab, (), ()) +else: + def get_code_params(code): + return (code.co_nlocals, code.co_kwonlyargcount, code.co_stacksize, + code.co_flags, code.co_code, code.co_consts, code.co_names, + code.co_varnames, code.co_filename, code.co_name, + code.co_firstlineno, code.co_lnotab, (), ()) + + def build_code_chunk(code, filename, name, lineno): + return CodeType(0, code.co_nlocals, code.co_kwonlyargcount, + code.co_stacksize, code.co_flags | 0x0040, + code.co_code, code.co_consts, code.co_names, + code.co_varnames, filename, name, lineno, + code.co_lnotab, (), ()) + +# Compatibility fallback implementations for Python < 2.6 + +try: + next = next +except NameError: + def next(iterator): + return iterator.next() + +# Compatibility fallback implementations for Python < 2.5 + +try: + all = all + any = any +except NameError: + def any(S): + for x in S: + if x: + return True + return False + + def all(S): + for x in S: + if not x: + return False + return True + diff --git a/lib/genshi/genshi/core.py b/lib/genshi/genshi/core.py new file mode 100644 --- /dev/null +++ b/lib/genshi/genshi/core.py @@ -0,0 +1,745 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2006-2009 Edgewall Software +# All rights reserved. +# +# This software is licensed as described in the file COPYING, which +# you should have received as part of this distribution. The terms +# are also available at http://genshi.edgewall.org/wiki/License. +# +# This software consists of voluntary contributions made by many +# individuals. For the exact contribution history, see the revision +# history and logs, available at http://genshi.edgewall.org/log/. + +"""Core classes for markup processing.""" + +try: + reduce # builtin in Python < 3 +except NameError: + from functools import reduce +import sys +from itertools import chain +import operator + +from genshi.util import plaintext, stripentities, striptags, stringrepr + +__all__ = ['Stream', 'Markup', 'escape', 'unescape', 'Attrs', 'Namespace', + 'QName'] +__docformat__ = 'restructuredtext en' + + +class StreamEventKind(str): + """A kind of event on a markup stream.""" + __slots__ = [] + _instances = {} + + def __new__(cls, val): + return cls._instances.setdefault(val, str.__new__(cls, val)) + + +class Stream(object): + """Represents a stream of markup events. + + This class is basically an iterator over the events. + + Stream events are tuples of the form:: + + (kind, data, position) + + where ``kind`` is the event kind (such as `START`, `END`, `TEXT`, etc), + ``data`` depends on the kind of event, and ``position`` is a + ``(filename, line, offset)`` tuple that contains the location of the + original element or text in the input. If the original location is unknown, + ``position`` is ``(None, -1, -1)``. + + Also provided are ways to serialize the stream to text. The `serialize()` + method will return an iterator over generated strings, while `render()` + returns the complete generated text at once. Both accept various parameters + that impact the way the stream is serialized. + """ + __slots__ = ['events', 'serializer'] + + START = StreamEventKind('START') #: a start tag + END = StreamEventKind('END') #: an end tag + TEXT = StreamEventKind('TEXT') #: literal text + XML_DECL = StreamEventKind('XML_DECL') #: XML declaration + DOCTYPE = StreamEventKind('DOCTYPE') #: doctype declaration + START_NS = StreamEventKind('START_NS') #: start namespace mapping + END_NS = StreamEventKind('END_NS') #: end namespace mapping + START_CDATA = StreamEventKind('START_CDATA') #: start CDATA section + END_CDATA = StreamEventKind('END_CDATA') #: end CDATA section + PI = StreamEventKind('PI') #: processing instruction + COMMENT = StreamEventKind('COMMENT') #: comment + + def __init__(self, events, serializer=None): + """Initialize the stream with a sequence of markup events. + + :param events: a sequence or iterable providing the events + :param serializer: the default serialization method to use for this + stream + + :note: Changed in 0.5: added the `serializer` argument + """ + self.events = events #: The underlying iterable producing the events + self.serializer = serializer #: The default serializion method + + def __iter__(self): + return iter(self.events) + + def __or__(self, function): + """Override the "bitwise or" operator to apply filters or serializers + to the stream, providing a syntax similar to pipes on Unix shells. + + Assume the following stream produced by the `HTML` function: + + >>> from genshi.input import HTML + >>> html = HTML('''<p onclick="alert('Whoa')">Hello, world!</p>''', encoding='utf-8') + >>> print(html) + <p onclick="alert('Whoa')">Hello, world!</p> + + A filter such as the HTML sanitizer can be applied to that stream using + the pipe notation as follows: + + >>> from genshi.filters import HTMLSanitizer + >>> sanitizer = HTMLSanitizer() + >>> print(html | sanitizer) + <p>Hello, world!</p> + + Filters can be any function that accepts and produces a stream (where + a stream is anything that iterates over events): + + >>> def uppercase(stream): + ... for kind, data, pos in stream: + ... if kind is TEXT: + ... data = data.upper() + ... yield kind, data, pos + >>> print(html | sanitizer | uppercase) + <p>HELLO, WORLD!</p> + + Serializers can also be used with this notation: + + >>> from genshi.output import TextSerializer + >>> output = TextSerializer() + >>> print(html | sanitizer | uppercase | output) + HELLO, WORLD! + + Commonly, serializers should be used at the end of the "pipeline"; + using them somewhere in the middle may produce unexpected results. + + :param function: the callable object that should be applied as a filter + :return: the filtered stream + :rtype: `Stream` + """ + return Stream(_ensure(function(self)), serializer=self.serializer) + + def filter(self, *filters): + """Apply filters to the stream. + + This method returns a new stream with the given filters applied. The + filters must be callables that accept the stream object as parameter, + and return the filtered stream. + + The call:: + + stream.filter(filter1, filter2) + + is equivalent to:: + + stream | filter1 | filter2 + + :param filters: one or more callable objects that should be applied as + filters + :return: the filtered stream + :rtype: `Stream` + """ + return reduce(operator.or_, (self,) + filters) + + def render(self, method=None, encoding=None, out=None, **kwargs): + """Return a string representation of the stream. + + Any additional keyword arguments are passed to the serializer, and thus + depend on the `method` parameter value. + + :param method: determines how the stream is serialized; can be either + "xml", "xhtml", "html", "text", or a custom serializer + class; if `None`, the default serialization method of + the stream is used + :param encoding: how the output string should be encoded; if set to + `None`, this method returns a `unicode` object + :param out: a file-like object that the output should be written to + instead of being returned as one big string; note that if + this is a file or socket (or similar), the `encoding` must + not be `None` (that is, the output must be encoded) + :return: a `str` or `unicode` object (depending on the `encoding` + parameter), or `None` if the `out` parameter is provided + :rtype: `basestring` + + :see: XMLSerializer, XHTMLSerializer, HTMLSerializer, TextSerializer + :note: Changed in 0.5: added the `out` parameter + """ + from genshi.output import encode + if method is None: + method = self.serializer or 'xml' + generator = self.serialize(method=method, **kwargs) + return encode(generator, method=method, encoding=encoding, out=out) + + def select(self, path, namespaces=None, variables=None): + """Return a new stream that contains the events matching the given + XPath expression. + + >>> from genshi import HTML + >>> stream = HTML('<doc><elem>foo</elem><elem>bar</elem></doc>', encoding='utf-8') + >>> print(stream.select('elem')) + <elem>foo</elem><elem>bar</elem> + >>> print(stream.select('elem/text()')) + foobar + + Note that the outermost element of the stream becomes the *context + node* for the XPath test. That means that the expression "doc" would + not match anything in the example above, because it only tests against + child elements of the outermost element: + + >>> print(stream.select('doc')) + <BLANKLINE> + + You can use the "." expression to match the context node itself + (although that usually makes little sense): + + >>> print(stream.select('.')) + <doc><elem>foo</elem><elem>bar</elem></doc> + + :param path: a string containing the XPath expression + :param namespaces: mapping of namespace prefixes used in the path + :param variables: mapping of variable names to values + :return: the selected substream + :rtype: `Stream` + :raises PathSyntaxError: if the given path expression is invalid or not + supported + """ + from genshi.path import Path + return Path(path).select(self, namespaces, variables) + + def serialize(self, method='xml', **kwargs): + """Generate strings corresponding to a specific serialization of the + stream. + + Unlike the `render()` method, this method is a generator that returns + the serialized output incrementally, as opposed to returning a single + string. + + Any additional keyword arguments are passed to the serializer, and thus + depend on the `method` parameter value. + + :param method: determines how the stream is serialized; can be either + "xml", "xhtml", "html", "text", or a custom serializer + class; if `None`, the default serialization method of + the stream is used + :return: an iterator over the serialization results (`Markup` or + `unicode` objects, depending on the serialization method) + :rtype: ``iterator`` + :see: XMLSerializer, XHTMLSerializer, HTMLSerializer, TextSerializer + """ + from genshi.output import get_serializer + if method is None: + method = self.serializer or 'xml' + return get_serializer(method, **kwargs)(_ensure(self)) + + def __str__(self): + return self.render() + + def __unicode__(self): + return self.render(encoding=None) + + def __html__(self): + return self + + +START = Stream.START +END = Stream.END +TEXT = Stream.TEXT +XML_DECL = Stream.XML_DECL +DOCTYPE = Stream.DOCTYPE +START_NS = Stream.START_NS +END_NS = Stream.END_NS +START_CDATA = Stream.START_CDATA +END_CDATA = Stream.END_CDATA +PI = Stream.PI +COMMENT = Stream.COMMENT + + +def _ensure(stream): + """Ensure that every item on the stream is actually a markup event.""" + stream = iter(stream) + event = stream.next() + + # Check whether the iterable is a real markup event stream by examining the + # first item it yields; if it's not we'll need to do some conversion + if type(event) is not tuple or len(event) != 3: + for event in chain([event], stream): + if hasattr(event, 'totuple'): + event = event.totuple() + else: + event = TEXT, unicode(event), (None, -1, -1) + yield event + return + + # This looks like a markup event stream, so we'll just pass it through + # unchanged + yield event + for event in stream: + yield event + + +class Attrs(tuple): + """Immutable sequence type that stores the attributes of an element. + + Ordering of the attributes is preserved, while access by name is also + supported. From noreply at buildbot.pypy.org Tue Jan 24 12:13:03 2012 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 24 Jan 2012 12:13:03 +0100 (CET) Subject: [pypy-commit] buildbot default: cheat, we use benchmarks checkout of jinja, what can go wrong... Message-ID: <20120124111303.C16AE82CFD@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: Changeset: r617:d4dc199b5ee5 Date: 2012-01-24 13:12 +0200 http://bitbucket.org/pypy/buildbot/changeset/d4dc199b5ee5/ Log: cheat, we use benchmarks checkout of jinja, what can go wrong... diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py --- a/bot2/pypybuildbot/builds.py +++ b/bot2/pypybuildbot/builds.py @@ -341,6 +341,7 @@ if host == 'tannit': pypy_c_rel = 'build/pypy/translator/goal/pypy-c' self.addStep(ShellCmd( + env={'PYTHONPATH': './benchmarks/lib/jinja2'}, description="measure numpy compatibility", command=[pypy_c_rel, 'build/pypy/module/micronumpy/tool/numready.py', From noreply at buildbot.pypy.org Tue Jan 24 16:44:25 2012 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 24 Jan 2012 16:44:25 +0100 (CET) Subject: [pypy-commit] buildbot default: an attempt to expose numpy readiness Message-ID: <20120124154425.7AF5C82CFD@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: Changeset: r618:98e265ee14d4 Date: 2012-01-24 17:44 +0200 http://bitbucket.org/pypy/buildbot/changeset/98e265ee14d4/ Log: an attempt to expose numpy readiness diff --git a/bot2/pypybuildbot/master.py b/bot2/pypybuildbot/master.py --- a/bot2/pypybuildbot/master.py +++ b/bot2/pypybuildbot/master.py @@ -1,3 +1,5 @@ + +import os import getpass from buildbot.scheduler import Nightly from buildbot.buildslave import BuildSlave @@ -7,6 +9,8 @@ from pypybuildbot.pypylist import PyPyList from pypybuildbot.ircbot import IRC # side effects +from twisted.web.static import File, DirectoryLister + # Forbid "force build" with empty user name from buildbot.status.web.builder import StatusResourceBuilder def my_force(self, req, *args, **kwds): @@ -40,6 +44,7 @@ 'freebsd'])) status.putChild('nightly', PyPyList(os.path.expanduser('~/nightly'), defaultType='application/octet-stream')) +status.putChild('numpy-status', DirectoryLister(os.path.expanduser('~/numpy_compat'))) pypybuilds = load('pypybuildbot.builds') From noreply at buildbot.pypy.org Tue Jan 24 16:49:03 2012 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 24 Jan 2012 16:49:03 +0100 (CET) Subject: [pypy-commit] pypy stm: More tests. Message-ID: <20120124154903.2BD0782CFD@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: stm Changeset: r51708:9e1713213887 Date: 2012-01-23 18:02 +0100 http://bitbucket.org/pypy/pypy/changeset/9e1713213887/ Log: More tests. diff --git a/pypy/rpython/lltypesystem/test/test_lltype.py b/pypy/rpython/lltypesystem/test/test_lltype.py --- a/pypy/rpython/lltypesystem/test/test_lltype.py +++ b/pypy/rpython/lltypesystem/test/test_lltype.py @@ -796,6 +796,13 @@ S = GcStruct('S', ('x', lltype.Signed), hints={'immutable_fields': FieldListAccessor({'x': 1234})}) assert S._immutable_field('x') == 1234 + # + S = GcStruct('S', ('c', lltype.Array(lltype.Char))) + assert S.c._immutable_field() == False + # + S = GcStruct('S', ('d', lltype.Array(lltype.Char, + hints={'immutable': True}))) + assert S.d._immutable_field() == True def test_typedef(): T = Typedef(Signed, 'T') From noreply at buildbot.pypy.org Tue Jan 24 16:49:04 2012 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 24 Jan 2012 16:49:04 +0100 (CET) Subject: [pypy-commit] pypy stm: Add an official way to ask some Struct whether a Message-ID: <20120124154904.679B482CFD@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: stm Changeset: r51709:1fd090dc1127 Date: 2012-01-23 18:14 +0100 http://bitbucket.org/pypy/pypy/changeset/1fd090dc1127/ Log: Add an official way to ask some Struct whether a {get,set}interiorfield() targets an immutable field or not. diff --git a/pypy/rpython/lltypesystem/lltype.py b/pypy/rpython/lltypesystem/lltype.py --- a/pypy/rpython/lltypesystem/lltype.py +++ b/pypy/rpython/lltypesystem/lltype.py @@ -352,6 +352,20 @@ pass return False + def _immutable_interiorfield(self, fields_v): + T = self + for v_field in fields_v: + fieldname = getattr(v_field, 'value', None) + if T._immutable_field(fieldname): + return True + if isinstance(fieldname, str): + assert isinstance(T, Struct) + T = getattr(T, fieldname) + else: + assert isinstance(T, Array) + T = T.OF + return False + class RttiStruct(Struct): _runtime_type_info = None diff --git a/pypy/rpython/lltypesystem/test/test_lltype.py b/pypy/rpython/lltypesystem/test/test_lltype.py --- a/pypy/rpython/lltypesystem/test/test_lltype.py +++ b/pypy/rpython/lltypesystem/test/test_lltype.py @@ -804,6 +804,44 @@ hints={'immutable': True}))) assert S.d._immutable_field() == True +def test_immutable_interiorfield(): + class Constant: + def __init__(self, value): + self.value = value + # + for hints in [{}, {'immutable': True}]: + expected = 'immutable' in hints + # + T = Struct('T', ('a', lltype.Signed), hints=hints) + S = GcStruct('S', ('t', T)) + immut = S._immutable_interiorfield([Constant('t'), Constant('a')]) + assert immut == expected + # + T = Struct('T', ('a', lltype.Signed)) + S = GcStruct('S', ('t', T), hints=hints) + immut = S._immutable_interiorfield([Constant('t'), Constant('a')]) + assert immut == expected + # + A = Array(lltype.Signed, hints=hints) + S = GcStruct('S', ('ar', A)) + immut = S._immutable_interiorfield([Constant('ar'), Constant(4)]) + assert immut == expected + # + T = Struct('T', ('a', lltype.Signed), hints=hints) + A = Array(T) + S = GcStruct('S', ('ar', A)) + immut = S._immutable_interiorfield([Constant('ar'), Constant(4), + Constant('a')]) + assert immut == expected + # + T = Struct('T', ('a', lltype.Signed)) + A = Array(T, hints=hints) + S = GcStruct('S', ('ar', A)) + immut = S._immutable_interiorfield([Constant('ar'), Constant(4), + Constant('a')]) + assert immut == expected + + def test_typedef(): T = Typedef(Signed, 'T') assert T == Signed From noreply at buildbot.pypy.org Tue Jan 24 16:49:05 2012 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 24 Jan 2012 16:49:05 +0100 (CET) Subject: [pypy-commit] pypy stm: Simplify. Message-ID: <20120124154905.99D6282CFD@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: stm Changeset: r51710:6a0e53c03ecb Date: 2012-01-23 18:18 +0100 http://bitbucket.org/pypy/pypy/changeset/6a0e53c03ecb/ Log: Simplify. diff --git a/pypy/rpython/lltypesystem/lltype.py b/pypy/rpython/lltypesystem/lltype.py --- a/pypy/rpython/lltypesystem/lltype.py +++ b/pypy/rpython/lltypesystem/lltype.py @@ -352,10 +352,9 @@ pass return False - def _immutable_interiorfield(self, fields_v): + def _immutable_interiorfield(self, fields): T = self - for v_field in fields_v: - fieldname = getattr(v_field, 'value', None) + for fieldname in fields: # may also be None or an integer, for arrays if T._immutable_field(fieldname): return True if isinstance(fieldname, str): diff --git a/pypy/rpython/lltypesystem/test/test_lltype.py b/pypy/rpython/lltypesystem/test/test_lltype.py --- a/pypy/rpython/lltypesystem/test/test_lltype.py +++ b/pypy/rpython/lltypesystem/test/test_lltype.py @@ -805,40 +805,35 @@ assert S.d._immutable_field() == True def test_immutable_interiorfield(): - class Constant: - def __init__(self, value): - self.value = value # for hints in [{}, {'immutable': True}]: expected = 'immutable' in hints # T = Struct('T', ('a', lltype.Signed), hints=hints) S = GcStruct('S', ('t', T)) - immut = S._immutable_interiorfield([Constant('t'), Constant('a')]) + immut = S._immutable_interiorfield(['t', 'a']) assert immut == expected # T = Struct('T', ('a', lltype.Signed)) S = GcStruct('S', ('t', T), hints=hints) - immut = S._immutable_interiorfield([Constant('t'), Constant('a')]) + immut = S._immutable_interiorfield(['t', 'a']) assert immut == expected # A = Array(lltype.Signed, hints=hints) S = GcStruct('S', ('ar', A)) - immut = S._immutable_interiorfield([Constant('ar'), Constant(4)]) + immut = S._immutable_interiorfield(['ar', 4]) assert immut == expected # T = Struct('T', ('a', lltype.Signed), hints=hints) A = Array(T) S = GcStruct('S', ('ar', A)) - immut = S._immutable_interiorfield([Constant('ar'), Constant(4), - Constant('a')]) + immut = S._immutable_interiorfield(['ar', 4, 'a']) assert immut == expected # T = Struct('T', ('a', lltype.Signed)) A = Array(T, hints=hints) S = GcStruct('S', ('ar', A)) - immut = S._immutable_interiorfield([Constant('ar'), Constant(4), - Constant('a')]) + immut = S._immutable_interiorfield(['ar', 4, 'a']) assert immut == expected From noreply at buildbot.pypy.org Tue Jan 24 16:49:06 2012 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 24 Jan 2012 16:49:06 +0100 (CET) Subject: [pypy-commit] pypy stm: Must also be supported if the outermost container is an array. Message-ID: <20120124154906.CCCD082CFD@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: stm Changeset: r51711:6776525c81d5 Date: 2012-01-23 18:24 +0100 http://bitbucket.org/pypy/pypy/changeset/6776525c81d5/ Log: Must also be supported if the outermost container is an array. diff --git a/pypy/rpython/lltypesystem/lltype.py b/pypy/rpython/lltypesystem/lltype.py --- a/pypy/rpython/lltypesystem/lltype.py +++ b/pypy/rpython/lltypesystem/lltype.py @@ -197,6 +197,19 @@ def _container_example(self): raise NotImplementedError + def _immutable_interiorfield(self, fields): + T = self + for fieldname in fields: # may also be None or an integer, for arrays + if T._immutable_field(fieldname): + return True + if isinstance(fieldname, str): + assert isinstance(T, Struct) + T = getattr(T, fieldname) + else: + assert isinstance(T, Array) + T = T.OF + return False + class Typedef(LowLevelType): """A typedef is just another name for an existing type""" @@ -352,19 +365,6 @@ pass return False - def _immutable_interiorfield(self, fields): - T = self - for fieldname in fields: # may also be None or an integer, for arrays - if T._immutable_field(fieldname): - return True - if isinstance(fieldname, str): - assert isinstance(T, Struct) - T = getattr(T, fieldname) - else: - assert isinstance(T, Array) - T = T.OF - return False - class RttiStruct(Struct): _runtime_type_info = None diff --git a/pypy/rpython/lltypesystem/test/test_lltype.py b/pypy/rpython/lltypesystem/test/test_lltype.py --- a/pypy/rpython/lltypesystem/test/test_lltype.py +++ b/pypy/rpython/lltypesystem/test/test_lltype.py @@ -829,12 +829,16 @@ S = GcStruct('S', ('ar', A)) immut = S._immutable_interiorfield(['ar', 4, 'a']) assert immut == expected + immut = A._immutable_interiorfield([4, 'a']) + assert immut == expected # T = Struct('T', ('a', lltype.Signed)) A = Array(T, hints=hints) S = GcStruct('S', ('ar', A)) immut = S._immutable_interiorfield(['ar', 4, 'a']) assert immut == expected + immut = A._immutable_interiorfield([4, 'a']) + assert immut == expected def test_typedef(): From noreply at buildbot.pypy.org Tue Jan 24 16:49:08 2012 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 24 Jan 2012 16:49:08 +0100 (CET) Subject: [pypy-commit] pypy stm: Implement proper immutability detection in {get, set}interiorfield(). Message-ID: <20120124154908.0829682CFD@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: stm Changeset: r51712:d8e6a730661d Date: 2012-01-23 18:26 +0100 http://bitbucket.org/pypy/pypy/changeset/d8e6a730661d/ Log: Implement proper immutability detection in {get,set}interiorfield(). This lets *all* our RPython-level 'string[index]' be considered as immutable reads, which is likely essential for performance. diff --git a/pypy/translator/stm/llstminterp.py b/pypy/translator/stm/llstminterp.py --- a/pypy/translator/stm/llstminterp.py +++ b/pypy/translator/stm/llstminterp.py @@ -124,6 +124,38 @@ self.check_stm_mode(lambda m: False) assert 0 + def opstm_getinteriorfield(self, struct, *fields): + STRUCT = lltype.typeOf(struct).TO + if STRUCT._immutable_interiorfield(fields): + # immutable field reads are always allowed + return LLFrame.op_getinteriorfield(self, struct, *fields) + elif STRUCT._gckind == 'raw': + # raw getfields are allowed outside a regular transaction + self.check_stm_mode(lambda m: m != "regular_transaction") + return LLFrame.op_getinteriorfield(self, struct, *fields) + else: + # mutable 'getfields' are always forbidden for now + self.check_stm_mode(lambda m: False) + assert 0 + + def opstm_setinteriorfield(self, struct, *fields_and_newvalue): + fields = fields_and_newvalue[:-1] + newvalue = fields_and_newvalue[-1] + STRUCT = lltype.typeOf(struct).TO + if STRUCT._immutable_interiorfield(fields): + # immutable field writes (i.e. initializing writes) should + # always be fine, because they should occur into newly malloced + # structures + LLFrame.op_setinteriorfield(self, struct, *fields_and_newvalue) + elif STRUCT._gckind == 'raw': + # raw setfields are allowed outside a regular transaction + self.check_stm_mode(lambda m: m != "regular_transaction") + LLFrame.op_setinteriorfield(self, struct, *fields_and_newvalue) + else: + # mutable 'setfields' are always forbidden for now + self.check_stm_mode(lambda m: False) + assert 0 + def opstm_malloc(self, TYPE, flags): # non-GC must not occur in a regular transaction, # but can occur in inevitable mode or outside a transaction diff --git a/pypy/translator/stm/test/test_transform.py b/pypy/translator/stm/test/test_transform.py --- a/pypy/translator/stm/test/test_transform.py +++ b/pypy/translator/stm/test/test_transform.py @@ -98,8 +98,11 @@ assert summary(graph) == {'stm_setarrayitem': 1} eval_stm_graph(interp, graph, [p], stm_mode="regular_transaction") +STRLIKE = lltype.GcStruct('STRLIKE', # no 'immutable' in this version + ('chars', lltype.Array(lltype.Char))) + def test_getinteriorfield(): - p = lltype.malloc(rstr.STR, 100, immortal=True) + p = lltype.malloc(STRLIKE, 100, immortal=True) p.chars[42] = 'X' def func(p): return p.chars[42] @@ -110,12 +113,32 @@ assert res == 'X' def test_setinteriorfield(): + p = lltype.malloc(STRLIKE, 100, immortal=True) + def func(p): + p.chars[42] = 'Y' + interp, graph = get_interpreter(func, [p]) + transform_graph(graph) + assert summary(graph) == {'stm_setinteriorfield': 1} + res = eval_stm_graph(interp, graph, [p], stm_mode="regular_transaction") + +def test_getstrchar(): + p = lltype.malloc(rstr.STR, 100, immortal=True) + p.chars[42] = 'X' + def func(p): + return p.chars[42] + interp, graph = get_interpreter(func, [p]) + transform_graph(graph) + assert summary(graph) == {'getinteriorfield': 1} + res = eval_stm_graph(interp, graph, [p], stm_mode="regular_transaction") + assert res == 'X' + +def test_setstrchar(): p = lltype.malloc(rstr.STR, 100, immortal=True) def func(p): p.chars[42] = 'Y' interp, graph = get_interpreter(func, [p]) transform_graph(graph) - assert summary(graph) == {'stm_setinteriorfield': 1} + assert summary(graph) == {'setinteriorfield': 1} res = eval_stm_graph(interp, graph, [p], stm_mode="regular_transaction") def test_unsupported_operation(): diff --git a/pypy/translator/stm/transform.py b/pypy/translator/stm/transform.py --- a/pypy/translator/stm/transform.py +++ b/pypy/translator/stm/transform.py @@ -164,7 +164,7 @@ OUTER = op.args[0].concretetype.TO if op.result.concretetype is lltype.Void: op1 = op - elif OUTER._hints.get('immutable'): + elif OUTER._immutable_interiorfield(unwraplist(op.args[1:])): op1 = op elif OUTER._gckind == 'raw': turn_inevitable(newoperations, "getinteriorfield-raw") @@ -177,7 +177,7 @@ OUTER = op.args[0].concretetype.TO if op.args[-1].concretetype is lltype.Void: op1 = op - elif OUTER._hints.get('immutable'): + elif OUTER._immutable_interiorfield(unwraplist(op.args[1:-1])): op1 = op elif OUTER._gckind == 'raw': turn_inevitable(newoperations, "setinteriorfield-raw") @@ -232,3 +232,12 @@ def turn_inevitable_and_proceed(newoperations, op): turn_inevitable(newoperations, op.opname) newoperations.append(op) + +def unwraplist(list_v): + for v in list_v: + if isinstance(v, Constant): + yield v.value + elif isinstance(v, Variable): + yield None # unknown + else: + raise AssertionError(v) From noreply at buildbot.pypy.org Tue Jan 24 16:49:09 2012 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 24 Jan 2012 16:49:09 +0100 (CET) Subject: [pypy-commit] pypy stm: Propagate this flag from the RPython class. Message-ID: <20120124154909.38E3F82CFD@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: stm Changeset: r51713:1471ad7b2ff2 Date: 2012-01-24 15:28 +0100 http://bitbucket.org/pypy/pypy/changeset/1471ad7b2ff2/ Log: Propagate this flag from the RPython class. diff --git a/pypy/rpython/rclass.py b/pypy/rpython/rclass.py --- a/pypy/rpython/rclass.py +++ b/pypy/rpython/rclass.py @@ -206,6 +206,8 @@ self.immutable_field_set = set(immutable_fields.value) accessor = FieldListAccessor() hints['immutable_fields'] = accessor + if self.classdef.classdesc.lookup('_stm_access_directly_') is not None: + hints['stm_access_directly'] = True return hints def __repr__(self): diff --git a/pypy/rpython/test/test_rclass.py b/pypy/rpython/test/test_rclass.py --- a/pypy/rpython/test/test_rclass.py +++ b/pypy/rpython/test/test_rclass.py @@ -1142,6 +1142,35 @@ 'cast_pointer': 1, 'setfield': 1} + def test_stm_access_directly(self): + for ret in ['a', 'b', 'c']: + class A(object): + pass + class B(A): + _stm_access_directly_ = True + class C(B): + pass + def f(n): + a = b = c = None + if n < 5: + a = A() + a.a = n + elif n < 10: + b = B() + b.b = n + else: + c = C() + c.c = n + if ret == 'a': return a + if ret == 'b': return b + if ret == 'c': return c + t, typer, graph = self.gengraph(f, [int]) + TYPE = graph.getreturnvar().concretetype.TO + if ret == 'a': + assert "stm_access_directly" not in TYPE._hints + else: + assert TYPE._hints["stm_access_directly"] == True + class TestOOtype(BaseTestRclass, OORtypeMixin): From noreply at buildbot.pypy.org Tue Jan 24 16:49:10 2012 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 24 Jan 2012 16:49:10 +0100 (CET) Subject: [pypy-commit] pypy stm: Use _stm_access_directly_ to mean that all field accesses Message-ID: <20120124154910.6721D82CFD@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: stm Changeset: r51714:0774830dfdeb Date: 2012-01-24 15:29 +0100 http://bitbucket.org/pypy/pypy/changeset/0774830dfdeb/ Log: Use _stm_access_directly_ to mean that all field accesses should be direct. Should be combined with some form of locking. diff --git a/pypy/translator/stm/llstminterp.py b/pypy/translator/stm/llstminterp.py --- a/pypy/translator/stm/llstminterp.py +++ b/pypy/translator/stm/llstminterp.py @@ -66,7 +66,8 @@ def opstm_getfield(self, struct, fieldname): STRUCT = lltype.typeOf(struct).TO - if STRUCT._immutable_field(fieldname): + if (STRUCT._immutable_field(fieldname) or + 'stm_access_directly' in STRUCT._hints): # immutable field reads are always allowed return LLFrame.op_getfield(self, struct, fieldname) elif STRUCT._gckind == 'raw': @@ -80,7 +81,8 @@ def opstm_setfield(self, struct, fieldname, newvalue): STRUCT = lltype.typeOf(struct).TO - if STRUCT._immutable_field(fieldname): + if (STRUCT._immutable_field(fieldname) or + 'stm_access_directly' in STRUCT._hints): # immutable field writes (i.e. initializing writes) should # always be fine, because they should occur into newly malloced # structures diff --git a/pypy/translator/stm/test/test_transform.py b/pypy/translator/stm/test/test_transform.py --- a/pypy/translator/stm/test/test_transform.py +++ b/pypy/translator/stm/test/test_transform.py @@ -141,6 +141,19 @@ assert summary(graph) == {'setinteriorfield': 1} res = eval_stm_graph(interp, graph, [p], stm_mode="regular_transaction") +def test_getfield_access_directly(): + class P: + x = 42 + _stm_access_directly_ = True + def func(): + p = P() + p.x += 1 + interp, graph = get_interpreter(func, []) + transform_graph(graph) + assert summary(graph) == {'malloc': 1, 'cast_pointer': 1, + 'getfield': 1, 'setfield': 3, 'int_add': 1} + res = eval_stm_graph(interp, graph, [], stm_mode="regular_transaction") + def test_unsupported_operation(): def func(n): n += 1 diff --git a/pypy/translator/stm/transform.py b/pypy/translator/stm/transform.py --- a/pypy/translator/stm/transform.py +++ b/pypy/translator/stm/transform.py @@ -112,7 +112,8 @@ STRUCT = op.args[0].concretetype.TO if op.result.concretetype is lltype.Void: op1 = op - elif STRUCT._immutable_field(op.args[1].value): + elif (STRUCT._immutable_field(op.args[1].value) or + 'stm_access_directly' in STRUCT._hints): op1 = op elif STRUCT._gckind == 'raw': turn_inevitable(newoperations, "getfield-raw") @@ -125,7 +126,8 @@ STRUCT = op.args[0].concretetype.TO if op.args[2].concretetype is lltype.Void: op1 = op - elif STRUCT._immutable_field(op.args[1].value): + elif (STRUCT._immutable_field(op.args[1].value) or + 'stm_access_directly' in STRUCT._hints): op1 = op elif STRUCT._gckind == 'raw': turn_inevitable(newoperations, "setfield-raw") From noreply at buildbot.pypy.org Tue Jan 24 16:49:11 2012 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 24 Jan 2012 16:49:11 +0100 (CET) Subject: [pypy-commit] pypy stm: Give this flag to PyFrames. Write in TODO.txt the issues that Message-ID: <20120124154911.959B182CFD@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: stm Changeset: r51715:cd57e12abc24 Date: 2012-01-24 15:30 +0100 http://bitbucket.org/pypy/pypy/changeset/cd57e12abc24/ Log: Give this flag to PyFrames. Write in TODO.txt the issues that this opens. diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py --- a/pypy/interpreter/pyframe.py +++ b/pypy/interpreter/pyframe.py @@ -39,6 +39,8 @@ __metaclass__ = extendabletype + _stm_access_directly_ = True + frame_finished_execution = False last_instr = -1 last_exception = None diff --git a/pypy/translator/stm/TODO.txt b/pypy/translator/stm/TODO.txt --- a/pypy/translator/stm/TODO.txt +++ b/pypy/translator/stm/TODO.txt @@ -7,3 +7,9 @@ if (d) x = stm_read_word(a); else x = *a; if (d) y = stm_read_word(b); else y = *b; to check !d only once if it is null + + +* PyFrames are accessed directly: + + - make the case of concurrent accesses to the same generator safe + - make the rare other cases of concurrent accesses safe From noreply at buildbot.pypy.org Tue Jan 24 16:49:12 2012 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 24 Jan 2012 16:49:12 +0100 (CET) Subject: [pypy-commit] pypy stm: Directly access also any array present as a field on the Message-ID: <20120124154912.CDE8782CFD@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: stm Changeset: r51716:00a838f5088a Date: 2012-01-24 15:50 +0100 http://bitbucket.org/pypy/pypy/changeset/00a838f5088a/ Log: Directly access also any array present as a field on the stm_access_directly object. diff --git a/pypy/translator/stm/test/test_transform.py b/pypy/translator/stm/test/test_transform.py --- a/pypy/translator/stm/test/test_transform.py +++ b/pypy/translator/stm/test/test_transform.py @@ -141,7 +141,7 @@ assert summary(graph) == {'setinteriorfield': 1} res = eval_stm_graph(interp, graph, [p], stm_mode="regular_transaction") -def test_getfield_access_directly(): +def test_getfield_setfield_access_directly(): class P: x = 42 _stm_access_directly_ = True @@ -154,6 +154,42 @@ 'getfield': 1, 'setfield': 3, 'int_add': 1} res = eval_stm_graph(interp, graph, [], stm_mode="regular_transaction") +def test_arrayitem_access_directly(): + class P1: + pass + class P2: + _stm_access_directly_ = True + for P in [P1, P2]: + def func(n): + p = P() + p.lst = [0] + p.lst[0] = n + return p.lst[0] + interp, graph = get_interpreter(func, [42]) + # + from pypy.translator.backendopt.inline import auto_inline_graphs + translator = interp.typer.annotator.translator + auto_inline_graphs(translator, translator.graphs, 16.0) + if option.view: + graph.show() + # + transform_graph(graph) + if P is P1: + assert 'stm_getfield' in summary(graph) + assert 'stm_setfield' in summary(graph) + assert 'stm_getarrayitem' in summary(graph) + assert 'stm_setarrayitem' in summary(graph) + elif P is P2: + assert 'stm_getfield' not in summary(graph) + assert 'stm_setfield' not in summary(graph) + assert 'stm_getarrayitem' not in summary(graph) + assert 'stm_setarrayitem' not in summary(graph) + else: + assert 0 + res = eval_stm_graph(interp, graph, [42], + stm_mode="regular_transaction") + assert res == 42 + def test_unsupported_operation(): def func(n): n += 1 diff --git a/pypy/translator/stm/transform.py b/pypy/translator/stm/transform.py --- a/pypy/translator/stm/transform.py +++ b/pypy/translator/stm/transform.py @@ -43,6 +43,7 @@ return newoperations = [] self.current_block = block + self.array_of_stm_access_directly = set() for i, op in enumerate(block.operations): self.current_op_index = i try: @@ -64,6 +65,7 @@ assert res is None block.operations = newoperations self.current_block = None + self.array_of_stm_access_directly = None def transform_graph(self, graph): for block in graph.iterblocks(): @@ -112,8 +114,10 @@ STRUCT = op.args[0].concretetype.TO if op.result.concretetype is lltype.Void: op1 = op - elif (STRUCT._immutable_field(op.args[1].value) or - 'stm_access_directly' in STRUCT._hints): + elif STRUCT._immutable_field(op.args[1].value): + op1 = op + elif 'stm_access_directly' in STRUCT._hints: + self.array_of_stm_access_directly.add(op.result) op1 = op elif STRUCT._gckind == 'raw': turn_inevitable(newoperations, "getfield-raw") @@ -142,6 +146,8 @@ op1 = op elif ARRAY._immutable_field(): op1 = op + elif op.args[0] in self.array_of_stm_access_directly: + op1 = op elif ARRAY._gckind == 'raw': turn_inevitable(newoperations, "getarrayitem-raw") op1 = op @@ -155,6 +161,8 @@ op1 = op elif ARRAY._immutable_field(): op1 = op + elif op.args[0] in self.array_of_stm_access_directly: + op1 = op elif ARRAY._gckind == 'raw': turn_inevitable(newoperations, "setarrayitem-raw") op1 = op @@ -219,6 +227,13 @@ ## self.seen_gc_stack_bottom = True newoperations.append(op) + def stt_same_as(self, newoperations, op): + if op.args[0] in self.array_of_stm_access_directly: + self.array_of_stm_access_directly.add(op.result) + newoperations.append(op) + + stt_cast_pointer = stt_same_as + def transform_graph(graph): # for tests: only transforms one graph From noreply at buildbot.pypy.org Tue Jan 24 16:49:14 2012 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 24 Jan 2012 16:49:14 +0100 (CET) Subject: [pypy-commit] pypy stm: Finish the previous checkin. Message-ID: <20120124154914.0888282CFD@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: stm Changeset: r51717:312bf884e018 Date: 2012-01-24 15:56 +0100 http://bitbucket.org/pypy/pypy/changeset/312bf884e018/ Log: Finish the previous checkin. diff --git a/pypy/translator/stm/llstminterp.py b/pypy/translator/stm/llstminterp.py --- a/pypy/translator/stm/llstminterp.py +++ b/pypy/translator/stm/llstminterp.py @@ -32,6 +32,7 @@ class LLSTMFrame(LLFrame): def eval(self): + self.array_of_stm_access_directly = set() try: res = LLFrame.eval(self) except LLException, e: @@ -66,10 +67,16 @@ def opstm_getfield(self, struct, fieldname): STRUCT = lltype.typeOf(struct).TO - if (STRUCT._immutable_field(fieldname) or - 'stm_access_directly' in STRUCT._hints): + if STRUCT._immutable_field(fieldname): # immutable field reads are always allowed return LLFrame.op_getfield(self, struct, fieldname) + elif 'stm_access_directly' in STRUCT._hints: + # stm_access_directly fields are allowed + # (XXX should check for locks!) + res = LLFrame.op_getfield(self, struct, fieldname) + if isinstance(res, lltype._ptr): + self.array_of_stm_access_directly.add(res._obj) + return res elif STRUCT._gckind == 'raw': # raw getfields are allowed outside a regular transaction self.check_stm_mode(lambda m: m != "regular_transaction") @@ -81,12 +88,15 @@ def opstm_setfield(self, struct, fieldname, newvalue): STRUCT = lltype.typeOf(struct).TO - if (STRUCT._immutable_field(fieldname) or - 'stm_access_directly' in STRUCT._hints): + if STRUCT._immutable_field(fieldname): # immutable field writes (i.e. initializing writes) should # always be fine, because they should occur into newly malloced # structures LLFrame.op_setfield(self, struct, fieldname, newvalue) + elif 'stm_access_directly' in STRUCT._hints: + # field writes in a stm_access_directly are allowed + # (XXX should check for locks!) + LLFrame.op_setfield(self, struct, fieldname, newvalue) elif STRUCT._gckind == 'raw': # raw setfields are allowed outside a regular transaction self.check_stm_mode(lambda m: m != "regular_transaction") @@ -101,6 +111,10 @@ if ARRAY._immutable_field(): # immutable item reads are always allowed return LLFrame.op_getarrayitem(self, array, index) + elif array._obj in self.array_of_stm_access_directly: + # an array read out of an stm_access_directly structure is ok + # (XXX should check for locks!) + return LLFrame.op_getarrayitem(self, array, index) elif ARRAY._gckind == 'raw': # raw getfields are allowed outside a regular transaction self.check_stm_mode(lambda m: m != "regular_transaction") @@ -111,12 +125,16 @@ assert 0 def opstm_setarrayitem(self, array, index, newvalue): - ARRAY = lltype.typeOf(struct).TO + ARRAY = lltype.typeOf(array).TO if ARRAY._immutable_field(): # immutable item writes (i.e. initializing writes) should # always be fine, because they should occur into newly malloced # arrays LLFrame.op_setarrayitem(self, array, index, newvalue) + elif array._obj in self.array_of_stm_access_directly: + # an array read out of an stm_access_directly structure is ok + # (XXX should check for locks!) + LLFrame.op_setarrayitem(self, array, index, newvalue) elif ARRAY._gckind == 'raw': # raw setarrayitems are allowed outside a regular transaction self.check_stm_mode(lambda m: m != "regular_transaction") diff --git a/pypy/translator/stm/test/test_transform.py b/pypy/translator/stm/test/test_transform.py --- a/pypy/translator/stm/test/test_transform.py +++ b/pypy/translator/stm/test/test_transform.py @@ -183,7 +183,7 @@ assert 'stm_getfield' not in summary(graph) assert 'stm_setfield' not in summary(graph) assert 'stm_getarrayitem' not in summary(graph) - assert 'stm_setarrayitem' not in summary(graph) + #assert 'stm_setarrayitem' not in summary(graph) --- xxx else: assert 0 res = eval_stm_graph(interp, graph, [42], From noreply at buildbot.pypy.org Tue Jan 24 16:49:15 2012 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 24 Jan 2012 16:49:15 +0100 (CET) Subject: [pypy-commit] pypy stm: Add a new type of immutability: "mutable owned". Message-ID: <20120124154915.419E182CFD@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: stm Changeset: r51718:b6c8cd8207c6 Date: 2012-01-24 16:20 +0100 http://bitbucket.org/pypy/pypy/changeset/b6c8cd8207c6/ Log: Add a new type of immutability: "mutable owned". diff --git a/pypy/rpython/lltypesystem/lltype.py b/pypy/rpython/lltypesystem/lltype.py --- a/pypy/rpython/lltypesystem/lltype.py +++ b/pypy/rpython/lltypesystem/lltype.py @@ -360,7 +360,7 @@ return True if 'immutable_fields' in self._hints: try: - return self._hints['immutable_fields'].fields[field] + return self._hints['immutable_fields']._fields[field] except KeyError: pass return False diff --git a/pypy/rpython/ootypesystem/ootype.py b/pypy/rpython/ootypesystem/ootype.py --- a/pypy/rpython/ootypesystem/ootype.py +++ b/pypy/rpython/ootypesystem/ootype.py @@ -272,7 +272,7 @@ return True if 'immutable_fields' in self._hints: try: - return self._hints['immutable_fields'].fields[field] + return self._hints['immutable_fields']._fields[field] except KeyError: pass return False diff --git a/pypy/rpython/rclass.py b/pypy/rpython/rclass.py --- a/pypy/rpython/rclass.py +++ b/pypy/rpython/rclass.py @@ -12,13 +12,13 @@ def initialize(self, TYPE, fields): assert type(fields) is dict self.TYPE = TYPE - self.fields = fields + self._fields = fields for x in fields.itervalues(): assert isinstance(x, ImmutableRanking) def all_immutable_fields(self): result = set() - for key, value in self.fields.iteritems(): + for key, value in self._fields.iteritems(): if value in (IR_IMMUTABLE, IR_IMMUTABLE_ARRAY): result.add(key) return result @@ -38,7 +38,11 @@ def __repr__(self): return '<%s>' % self.name +# IR_MUTABLE_OWNED means that the field is a regular mutable pointer field, +# but as a pointer it is the only reference to whatever it points to. This +# is useful for STM support in PyFrame. IR_MUTABLE = ImmutableRanking('mutable', False) +IR_MUTABLE_OWNED = ImmutableRanking("mutable_owned", False) IR_IMMUTABLE = ImmutableRanking('immutable', True) IR_IMMUTABLE_ARRAY = ImmutableRanking('immutable_array', True) IR_QUASIIMMUTABLE = ImmutableRanking('quasiimmutable', False) @@ -252,6 +256,9 @@ elif name.endswith('?'): # a quasi-immutable field name = name[:-1] rank = IR_QUASIIMMUTABLE + elif name.endswith('->...'): # for stm_access_directly: + name = name[:-5] # a mutable but owned object + rank = IR_MUTABLE_OWNED else: # a regular immutable/green field rank = IR_IMMUTABLE try: diff --git a/pypy/rpython/test/test_rclass.py b/pypy/rpython/test/test_rclass.py --- a/pypy/rpython/test/test_rclass.py +++ b/pypy/rpython/test/test_rclass.py @@ -7,6 +7,7 @@ from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin from pypy.rpython.rclass import IR_IMMUTABLE, IR_IMMUTABLE_ARRAY from pypy.rpython.rclass import IR_QUASIIMMUTABLE, IR_QUASIIMMUTABLE_ARRAY +from pypy.rpython.rclass import IR_MUTABLE_OWNED from pypy.rpython.error import TyperError from pypy.objspace.flow.model import summary @@ -753,10 +754,10 @@ t, typer, graph = self.gengraph(f, []) A_TYPE = deref(graph.getreturnvar().concretetype) accessor = A_TYPE._hints["immutable_fields"] - assert accessor.fields == {"inst_x": IR_IMMUTABLE, - "inst_y": IR_IMMUTABLE_ARRAY} or \ - accessor.fields == {"ox": IR_IMMUTABLE, - "oy": IR_IMMUTABLE_ARRAY} # for ootype + assert accessor._fields == {"inst_x": IR_IMMUTABLE, + "inst_y": IR_IMMUTABLE_ARRAY} or \ + accessor._fields == {"ox": IR_IMMUTABLE, + "oy": IR_IMMUTABLE_ARRAY} # for ootype def test_immutable_fields_subclass_1(self): from pypy.jit.metainterp.typesystem import deref @@ -774,8 +775,8 @@ t, typer, graph = self.gengraph(f, []) B_TYPE = deref(graph.getreturnvar().concretetype) accessor = B_TYPE._hints["immutable_fields"] - assert accessor.fields == {"inst_x": IR_IMMUTABLE} or \ - accessor.fields == {"ox": IR_IMMUTABLE} # for ootype + assert accessor._fields == {"inst_x": IR_IMMUTABLE} or \ + accessor._fields == {"ox": IR_IMMUTABLE} # for ootype def test_immutable_fields_subclass_2(self): from pypy.jit.metainterp.typesystem import deref @@ -794,10 +795,10 @@ t, typer, graph = self.gengraph(f, []) B_TYPE = deref(graph.getreturnvar().concretetype) accessor = B_TYPE._hints["immutable_fields"] - assert accessor.fields == {"inst_x": IR_IMMUTABLE, - "inst_y": IR_IMMUTABLE} or \ - accessor.fields == {"ox": IR_IMMUTABLE, - "oy": IR_IMMUTABLE} # for ootype + assert accessor._fields == {"inst_x": IR_IMMUTABLE, + "inst_y": IR_IMMUTABLE} or \ + accessor._fields == {"ox": IR_IMMUTABLE, + "oy": IR_IMMUTABLE} # for ootype def test_immutable_fields_only_in_subclass(self): from pypy.jit.metainterp.typesystem import deref @@ -815,8 +816,8 @@ t, typer, graph = self.gengraph(f, []) B_TYPE = deref(graph.getreturnvar().concretetype) accessor = B_TYPE._hints["immutable_fields"] - assert accessor.fields == {"inst_y": IR_IMMUTABLE} or \ - accessor.fields == {"oy": IR_IMMUTABLE} # for ootype + assert accessor._fields == {"inst_y": IR_IMMUTABLE} or \ + accessor._fields == {"oy": IR_IMMUTABLE} # for ootype def test_immutable_forbidden_inheritance_1(self): from pypy.rpython.rclass import ImmutableConflictError @@ -860,8 +861,8 @@ except AttributeError: A_TYPE = B_TYPE._superclass # for ootype accessor = A_TYPE._hints["immutable_fields"] - assert accessor.fields == {"inst_v": IR_IMMUTABLE} or \ - accessor.fields == {"ov": IR_IMMUTABLE} # for ootype + assert accessor._fields == {"inst_v": IR_IMMUTABLE} or \ + accessor._fields == {"ov": IR_IMMUTABLE} # for ootype def test_immutable_subclass_1(self): from pypy.rpython.rclass import ImmutableConflictError @@ -925,12 +926,12 @@ t, typer, graph = self.gengraph(f, []) B_TYPE = deref(graph.getreturnvar().concretetype) accessor = B_TYPE._hints["immutable_fields"] - assert accessor.fields == {"inst_y": IR_IMMUTABLE, - "inst_b": IR_QUASIIMMUTABLE} or \ - accessor.fields == {"ox": IR_IMMUTABLE, - "oy": IR_IMMUTABLE, - "oa": IR_QUASIIMMUTABLE, - "ob": IR_QUASIIMMUTABLE} # for ootype + assert accessor._fields == {"inst_y": IR_IMMUTABLE, + "inst_b": IR_QUASIIMMUTABLE} or \ + accessor._fields == {"ox": IR_IMMUTABLE, + "oy": IR_IMMUTABLE, + "oa": IR_QUASIIMMUTABLE, + "ob": IR_QUASIIMMUTABLE} # for ootype found = [] for op in graph.startblock.operations: if op.opname == 'jit_force_quasi_immutable': @@ -950,8 +951,8 @@ t, typer, graph = self.gengraph(f, []) A_TYPE = deref(graph.getreturnvar().concretetype) accessor = A_TYPE._hints["immutable_fields"] - assert accessor.fields == {"inst_c": IR_QUASIIMMUTABLE_ARRAY} or \ - accessor.fields == {"oc": IR_QUASIIMMUTABLE_ARRAY} # for ootype + assert accessor._fields == {"inst_c": IR_QUASIIMMUTABLE_ARRAY} or \ + accessor._fields == {"oc": IR_QUASIIMMUTABLE_ARRAY} # for ootype found = [] for op in graph.startblock.operations: if op.opname == 'jit_force_quasi_immutable': @@ -1171,6 +1172,20 @@ else: assert TYPE._hints["stm_access_directly"] == True + def test_mutable_but_owned(self): + class A(object): + _immutable_fields_ = ['a->...'] + class B(object): + pass + def f(n): + a = A() + a.a = B() + return a + t, typer, graph = self.gengraph(f, [int]) + A_TYPE = graph.getreturnvar().concretetype.TO + accessor = A_TYPE._hints["immutable_fields"] + assert accessor._fields == {"inst_a": IR_MUTABLE_OWNED} + class TestOOtype(BaseTestRclass, OORtypeMixin): From noreply at buildbot.pypy.org Tue Jan 24 16:49:16 2012 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 24 Jan 2012 16:49:16 +0100 (CET) Subject: [pypy-commit] pypy stm: Adjust PyFrame and the stm transformer. Message-ID: <20120124154916.7130782CFD@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: stm Changeset: r51719:3ae2aedede04 Date: 2012-01-24 16:33 +0100 http://bitbucket.org/pypy/pypy/changeset/3ae2aedede04/ Log: Adjust PyFrame and the stm transformer. diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py --- a/pypy/interpreter/pyframe.py +++ b/pypy/interpreter/pyframe.py @@ -40,6 +40,7 @@ __metaclass__ = extendabletype _stm_access_directly_ = True + _immutable_fields_ = ['locals_stack_w->...', 'cells->...'] frame_finished_execution = False last_instr = -1 diff --git a/pypy/translator/stm/test/test_transform.py b/pypy/translator/stm/test/test_transform.py --- a/pypy/translator/stm/test/test_transform.py +++ b/pypy/translator/stm/test/test_transform.py @@ -159,7 +159,10 @@ pass class P2: _stm_access_directly_ = True - for P in [P1, P2]: + class P3: + _stm_access_directly_ = True + _immutable_fields_ = ['lst->...'] + for P in [P1, P2, P2]: def func(n): p = P() p.lst = [0] @@ -174,18 +177,10 @@ graph.show() # transform_graph(graph) - if P is P1: - assert 'stm_getfield' in summary(graph) - assert 'stm_setfield' in summary(graph) - assert 'stm_getarrayitem' in summary(graph) - assert 'stm_setarrayitem' in summary(graph) - elif P is P2: - assert 'stm_getfield' not in summary(graph) - assert 'stm_setfield' not in summary(graph) - assert 'stm_getarrayitem' not in summary(graph) - #assert 'stm_setarrayitem' not in summary(graph) --- xxx - else: - assert 0 + assert ('stm_getfield' in summary(graph)) == (P is P1) + assert ('stm_setfield' in summary(graph)) == (P is P1) + assert ('stm_getarrayitem' in summary(graph)) == (P is not P3) + #assert ('stm_setarrayitem' in summary(graph)) == (P is not P3) -- xxx res = eval_stm_graph(interp, graph, [42], stm_mode="regular_transaction") assert res == 42 diff --git a/pypy/translator/stm/transform.py b/pypy/translator/stm/transform.py --- a/pypy/translator/stm/transform.py +++ b/pypy/translator/stm/transform.py @@ -4,6 +4,7 @@ from pypy.translator.stm import _rffi_stm from pypy.translator.unsimplify import varoftype, copyvar from pypy.rpython.lltypesystem import lltype, lloperation +from pypy.rpython import rclass ALWAYS_ALLOW_OPERATIONS = set([ @@ -43,7 +44,7 @@ return newoperations = [] self.current_block = block - self.array_of_stm_access_directly = set() + self.access_directly = set() for i, op in enumerate(block.operations): self.current_op_index = i try: @@ -65,7 +66,7 @@ assert res is None block.operations = newoperations self.current_block = None - self.array_of_stm_access_directly = None + self.access_directly = None def transform_graph(self, graph): for block in graph.iterblocks(): @@ -117,7 +118,14 @@ elif STRUCT._immutable_field(op.args[1].value): op1 = op elif 'stm_access_directly' in STRUCT._hints: - self.array_of_stm_access_directly.add(op.result) + try: + immfld = STRUCT._hints['immutable_fields'] + except KeyError: + pass + else: + rank = immfld._fields.get(op.args[1].value, None) + if rank is rclass.IR_MUTABLE_OWNED: + self.access_directly.add(op.result) op1 = op elif STRUCT._gckind == 'raw': turn_inevitable(newoperations, "getfield-raw") @@ -146,7 +154,7 @@ op1 = op elif ARRAY._immutable_field(): op1 = op - elif op.args[0] in self.array_of_stm_access_directly: + elif op.args[0] in self.access_directly: op1 = op elif ARRAY._gckind == 'raw': turn_inevitable(newoperations, "getarrayitem-raw") @@ -161,7 +169,7 @@ op1 = op elif ARRAY._immutable_field(): op1 = op - elif op.args[0] in self.array_of_stm_access_directly: + elif op.args[0] in self.access_directly: op1 = op elif ARRAY._gckind == 'raw': turn_inevitable(newoperations, "setarrayitem-raw") @@ -228,8 +236,8 @@ newoperations.append(op) def stt_same_as(self, newoperations, op): - if op.args[0] in self.array_of_stm_access_directly: - self.array_of_stm_access_directly.add(op.result) + if op.args[0] in self.access_directly: + self.access_directly.add(op.result) newoperations.append(op) stt_cast_pointer = stt_same_as From noreply at buildbot.pypy.org Tue Jan 24 16:49:17 2012 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 24 Jan 2012 16:49:17 +0100 (CET) Subject: [pypy-commit] pypy stm: Write down a next test, not implemented so far. Message-ID: <20120124154917.9EDEA82CFD@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: stm Changeset: r51720:539ec3f053ef Date: 2012-01-24 16:48 +0100 http://bitbucket.org/pypy/pypy/changeset/539ec3f053ef/ Log: Write down a next test, not implemented so far. diff --git a/pypy/translator/stm/test/test_transform.py b/pypy/translator/stm/test/test_transform.py --- a/pypy/translator/stm/test/test_transform.py +++ b/pypy/translator/stm/test/test_transform.py @@ -185,6 +185,17 @@ stm_mode="regular_transaction") assert res == 42 +def test_setfield_freshly_allocated(): + py.test.skip("XXX not implemented") + S = lltype.GcStruct('S', ('x', lltype.Signed)) + def func(n): + p = lltype.malloc(S) + p.x = n + interp, graph = get_interpreter(func, [42]) + transform_graph(graph) + assert summary(graph) == {'malloc': 1, 'setfield': 1} + res = eval_stm_graph(interp, graph, [42], stm_mode="regular_transaction") + def test_unsupported_operation(): def func(n): n += 1 From noreply at buildbot.pypy.org Tue Jan 24 16:51:06 2012 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 24 Jan 2012 16:51:06 +0100 (CET) Subject: [pypy-commit] pypy closed-branches: Merge closed head 4d9eda6790fd on branch better-jit-hooks Message-ID: <20120124155106.D910182CFD@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: closed-branches Changeset: r51721:334cec07b153 Date: 2012-01-24 16:49 +0100 http://bitbucket.org/pypy/pypy/changeset/334cec07b153/ Log: Merge closed head 4d9eda6790fd on branch better-jit-hooks From noreply at buildbot.pypy.org Tue Jan 24 16:51:07 2012 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 24 Jan 2012 16:51:07 +0100 (CET) Subject: [pypy-commit] pypy closed-branches: Merge closed head f36626591158 on branch numpypy-frompyfunc Message-ID: <20120124155107.F199F82CFD@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: closed-branches Changeset: r51722:69db50c53708 Date: 2012-01-24 16:49 +0100 http://bitbucket.org/pypy/pypy/changeset/69db50c53708/ Log: Merge closed head f36626591158 on branch numpypy-frompyfunc From noreply at buildbot.pypy.org Tue Jan 24 16:51:09 2012 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 24 Jan 2012 16:51:09 +0100 (CET) Subject: [pypy-commit] pypy closed-branches: Merge closed head e5b79894ad1e on branch numpypy-is_contiguous Message-ID: <20120124155109.1A58582CFD@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: closed-branches Changeset: r51723:b9621500bbe4 Date: 2012-01-24 16:49 +0100 http://bitbucket.org/pypy/pypy/changeset/b9621500bbe4/ Log: Merge closed head e5b79894ad1e on branch numpypy-is_contiguous From noreply at buildbot.pypy.org Tue Jan 24 16:51:10 2012 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 24 Jan 2012 16:51:10 +0100 (CET) Subject: [pypy-commit] pypy closed-branches: Merge closed head b5b4522440a5 on branch numpypy-axisops Message-ID: <20120124155110.3AD1482CFD@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: closed-branches Changeset: r51724:d0e6dbc1c951 Date: 2012-01-24 16:49 +0100 http://bitbucket.org/pypy/pypy/changeset/d0e6dbc1c951/ Log: Merge closed head b5b4522440a5 on branch numpypy-axisops From noreply at buildbot.pypy.org Tue Jan 24 16:51:11 2012 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 24 Jan 2012 16:51:11 +0100 (CET) Subject: [pypy-commit] pypy closed-branches: Merge closed head 08302909741e on branch numpy-indexing-by-arrays Message-ID: <20120124155111.519F182CFD@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: closed-branches Changeset: r51725:51aed7023c03 Date: 2012-01-24 16:49 +0100 http://bitbucket.org/pypy/pypy/changeset/51aed7023c03/ Log: Merge closed head 08302909741e on branch numpy-indexing-by-arrays From noreply at buildbot.pypy.org Tue Jan 24 16:51:12 2012 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 24 Jan 2012 16:51:12 +0100 (CET) Subject: [pypy-commit] pypy closed-branches: Merge closed head 8e57f6c0502b on branch numpypy-ufuncs Message-ID: <20120124155112.684AB82CFD@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: closed-branches Changeset: r51726:8102a8ae2713 Date: 2012-01-24 16:49 +0100 http://bitbucket.org/pypy/pypy/changeset/8102a8ae2713/ Log: Merge closed head 8e57f6c0502b on branch numpypy-ufuncs From noreply at buildbot.pypy.org Tue Jan 24 16:51:13 2012 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 24 Jan 2012 16:51:13 +0100 (CET) Subject: [pypy-commit] pypy closed-branches: Merge closed head 12c695258f06 on branch numpypy-shape-bug Message-ID: <20120124155113.7ED5182CFD@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: closed-branches Changeset: r51727:bb014819a808 Date: 2012-01-24 16:49 +0100 http://bitbucket.org/pypy/pypy/changeset/bb014819a808/ Log: Merge closed head 12c695258f06 on branch numpypy-shape-bug From noreply at buildbot.pypy.org Tue Jan 24 16:51:14 2012 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 24 Jan 2012 16:51:14 +0100 (CET) Subject: [pypy-commit] pypy closed-branches: re-close this branch Message-ID: <20120124155114.9B55182CFD@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: closed-branches Changeset: r51728:72656112790d Date: 2012-01-24 16:49 +0100 http://bitbucket.org/pypy/pypy/changeset/72656112790d/ Log: re-close this branch From noreply at buildbot.pypy.org Tue Jan 24 16:52:10 2012 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 24 Jan 2012 16:52:10 +0100 (CET) Subject: [pypy-commit] pypy stm: Missing import. Message-ID: <20120124155210.7BE8282CFD@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: stm Changeset: r51729:655478ab5af1 Date: 2012-01-24 16:51 +0100 http://bitbucket.org/pypy/pypy/changeset/655478ab5af1/ Log: Missing import. diff --git a/pypy/translator/stm/test/test_transform.py b/pypy/translator/stm/test/test_transform.py --- a/pypy/translator/stm/test/test_transform.py +++ b/pypy/translator/stm/test/test_transform.py @@ -1,3 +1,4 @@ +import py from pypy.rpython.lltypesystem import lltype, llmemory, rstr from pypy.rpython.test.test_llinterp import get_interpreter from pypy.objspace.flow.model import summary From noreply at buildbot.pypy.org Tue Jan 24 16:56:10 2012 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 24 Jan 2012 16:56:10 +0100 (CET) Subject: [pypy-commit] buildbot default: make it a subclass of File instead Message-ID: <20120124155610.7308C82CFD@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: Changeset: r619:17f934fc60f2 Date: 2012-01-24 17:55 +0200 http://bitbucket.org/pypy/buildbot/changeset/17f934fc60f2/ Log: make it a subclass of File instead diff --git a/bot2/pypybuildbot/master.py b/bot2/pypybuildbot/master.py --- a/bot2/pypybuildbot/master.py +++ b/bot2/pypybuildbot/master.py @@ -6,11 +6,9 @@ from buildbot.status.html import WebStatus from buildbot.process.builder import Builder #from buildbot import manhole -from pypybuildbot.pypylist import PyPyList +from pypybuildbot.pypylist import PyPyList, NumpyStatusList from pypybuildbot.ircbot import IRC # side effects -from twisted.web.static import File, DirectoryLister - # Forbid "force build" with empty user name from buildbot.status.web.builder import StatusResourceBuilder def my_force(self, req, *args, **kwds): @@ -44,7 +42,7 @@ 'freebsd'])) status.putChild('nightly', PyPyList(os.path.expanduser('~/nightly'), defaultType='application/octet-stream')) -status.putChild('numpy-status', DirectoryLister(os.path.expanduser('~/numpy_compat'))) +status.putChild('numpy-status', NumpyStatusList(os.path.expanduser('~/numpy_compat'))) pypybuilds = load('pypybuildbot.builds') diff --git a/bot2/pypybuildbot/pypylist.py b/bot2/pypybuildbot/pypylist.py --- a/bot2/pypybuildbot/pypylist.py +++ b/bot2/pypybuildbot/pypylist.py @@ -126,7 +126,8 @@ self.contentEncodings, self.defaultType) - +class NumpyStatusList(File): + pass class PyPyDirectoryLister(DirectoryLister): template = """<html> From noreply at buildbot.pypy.org Tue Jan 24 17:11:40 2012 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 24 Jan 2012 17:11:40 +0100 (CET) Subject: [pypy-commit] buildbot default: add symlinking Message-ID: <20120124161140.BAB8882CFD@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: Changeset: r620:8d18c2d75424 Date: 2012-01-24 18:11 +0200 http://bitbucket.org/pypy/buildbot/changeset/8d18c2d75424/ Log: add symlinking diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py --- a/bot2/pypybuildbot/builds.py +++ b/bot2/pypybuildbot/builds.py @@ -68,6 +68,18 @@ except OSError: pass +class NumpyStatusUpload(transfer.FileUpload): + def finished(self, *args, **kwds): + transfer.FileUpload.finished(self, *args, **kwds) + try: + os.chmod(self.masterdest, 0644) + except OSError: + pass + try: + symlink_force(os.path.basename(self.masterdest), 'latest.html') + except OSError: + pass + class Translate(ShellCmd): name = "translate" description = ["translating"] @@ -348,9 +360,10 @@ pypy_c_rel, 'numpy-compat.html'], workdir=".")) resfile = os.path.expanduser("~/numpy_compat/%(got_revision)s.html") - self.addStep(transfer.FileUpload(slavesrc="numpy-compat.html", - masterdest=WithProperties(resfile), - workdir=".")) + self.addStep(NumpyStatusUpload( + slavesrc="numpy-compat.html", + masterdest=WithProperties(resfile), + workdir=".")) pypy_c_rel = "../build/pypy/translator/goal/pypy-c" if postfix: addopts = ['--postfix', postfix] From noreply at buildbot.pypy.org Tue Jan 24 17:44:57 2012 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Tue, 24 Jan 2012 17:44:57 +0100 (CET) Subject: [pypy-commit] buildbot default: this is a dir now Message-ID: <20120124164457.AF72D82CFD@wyvern.cs.uni-duesseldorf.de> Author: Alex Gaynor <alex.gaynor at gmail.com> Branch: Changeset: r621:6148676b97bb Date: 2012-01-24 11:44 -0500 http://bitbucket.org/pypy/buildbot/changeset/6148676b97bb/ Log: this is a dir now diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py --- a/bot2/pypybuildbot/builds.py +++ b/bot2/pypybuildbot/builds.py @@ -356,7 +356,7 @@ env={'PYTHONPATH': './benchmarks/lib/jinja2'}, description="measure numpy compatibility", command=[pypy_c_rel, - 'build/pypy/module/micronumpy/tool/numready.py', + 'build/pypy/module/micronumpy/tool/numready/', pypy_c_rel, 'numpy-compat.html'], workdir=".")) resfile = os.path.expanduser("~/numpy_compat/%(got_revision)s.html") From noreply at buildbot.pypy.org Tue Jan 24 17:45:12 2012 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Tue, 24 Jan 2012 17:45:12 +0100 (CET) Subject: [pypy-commit] pypy numppy-flatitter: merged default in Message-ID: <20120124164512.5A31B82CFD@wyvern.cs.uni-duesseldorf.de> Author: Alex Gaynor <alex.gaynor at gmail.com> Branch: numppy-flatitter Changeset: r51730:9d3652817442 Date: 2012-01-23 16:18 -0500 http://bitbucket.org/pypy/pypy/changeset/9d3652817442/ Log: merged default in diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py --- a/lib_pypy/_sqlite3.py +++ b/lib_pypy/_sqlite3.py @@ -20,6 +20,8 @@ # 2. Altered source versions must be plainly marked as such, and must not be # misrepresented as being the original software. # 3. This notice may not be removed or altered from any source distribution. +# +# Note: This software has been modified for use in PyPy. from ctypes import c_void_p, c_int, c_double, c_int64, c_char_p, cdll from ctypes import POINTER, byref, string_at, CFUNCTYPE, cast @@ -27,7 +29,6 @@ from collections import OrderedDict import datetime import sys -import time import weakref from threading import _get_ident as thread_get_ident @@ -606,7 +607,7 @@ def authorizer(userdata, action, arg1, arg2, dbname, source): try: return int(callback(action, arg1, arg2, dbname, source)) - except Exception, e: + except Exception: return SQLITE_DENY c_authorizer = AUTHORIZER(authorizer) @@ -653,7 +654,7 @@ if not aggregate_ptr[0]: try: aggregate = cls() - except Exception, e: + except Exception: msg = ("user-defined aggregate's '__init__' " "method raised error") sqlite.sqlite3_result_error(context, msg, len(msg)) @@ -667,7 +668,7 @@ params = _convert_params(context, argc, c_params) try: aggregate.step(*params) - except Exception, e: + except Exception: msg = ("user-defined aggregate's 'step' " "method raised error") sqlite.sqlite3_result_error(context, msg, len(msg)) @@ -683,7 +684,7 @@ aggregate = self.aggregate_instances[aggregate_ptr[0]] try: val = aggregate.finalize() - except Exception, e: + except Exception: msg = ("user-defined aggregate's 'finalize' " "method raised error") sqlite.sqlite3_result_error(context, msg, len(msg)) @@ -771,7 +772,7 @@ self.statement.item = None self.statement.exhausted = True - if self.statement.kind == DML or self.statement.kind == DDL: + if self.statement.kind == DML: self.statement.reset() self.rowcount = -1 @@ -791,7 +792,7 @@ if self.statement.kind == DML: self.connection._begin() else: - raise ProgrammingError, "executemany is only for DML statements" + raise ProgrammingError("executemany is only for DML statements") self.rowcount = 0 for params in many_params: @@ -861,8 +862,6 @@ except StopIteration: return None - return nextrow - def fetchmany(self, size=None): self._check_closed() self._check_reset() @@ -915,7 +914,7 @@ def __init__(self, connection, sql): self.statement = None if not isinstance(sql, str): - raise ValueError, "sql must be a string" + raise ValueError("sql must be a string") self.con = connection self.sql = sql # DEBUG ONLY first_word = self._statement_kind = sql.lstrip().split(" ")[0].upper() @@ -944,8 +943,8 @@ raise self.con._get_exception(ret) self.con._remember_statement(self) if _check_remaining_sql(next_char.value): - raise Warning, "One and only one statement required: %r" % ( - next_char.value,) + raise Warning("One and only one statement required: %r" % ( + next_char.value,)) # sql_char should remain alive until here self._build_row_cast_map() @@ -1016,7 +1015,7 @@ elif type(param) is buffer: sqlite.sqlite3_bind_blob(self.statement, idx, str(param), len(param), SQLITE_TRANSIENT) else: - raise InterfaceError, "parameter type %s is not supported" % str(type(param)) + raise InterfaceError("parameter type %s is not supported" % str(type(param))) def set_params(self, params): ret = sqlite.sqlite3_reset(self.statement) @@ -1045,11 +1044,11 @@ for idx in range(1, sqlite.sqlite3_bind_parameter_count(self.statement) + 1): param_name = sqlite.sqlite3_bind_parameter_name(self.statement, idx) if param_name is None: - raise ProgrammingError, "need named parameters" + raise ProgrammingError("need named parameters") param_name = param_name[1:] try: param = params[param_name] - except KeyError, e: + except KeyError: raise ProgrammingError("missing parameter '%s'" %param) self.set_param(idx, param) @@ -1260,7 +1259,7 @@ params = _convert_params(context, nargs, c_params) try: val = real_cb(*params) - except Exception, e: + except Exception: msg = "user-defined function raised exception" sqlite.sqlite3_result_error(context, msg, len(msg)) else: diff --git a/lib_pypy/datetime.py b/lib_pypy/datetime.py --- a/lib_pypy/datetime.py +++ b/lib_pypy/datetime.py @@ -13,7 +13,7 @@ Sources for time zone and DST data: http://www.twinsun.com/tz/tz-link.htm This was originally copied from the sandbox of the CPython CVS repository. -Thanks to Tim Peters for suggesting using it. +Thanks to Tim Peters for suggesting using it. """ import time as _time @@ -271,6 +271,8 @@ raise ValueError("%s()=%d, must be in -1439..1439" % (name, offset)) def _check_date_fields(year, month, day): + if not isinstance(year, (int, long)): + raise TypeError('int expected') if not MINYEAR <= year <= MAXYEAR: raise ValueError('year must be in %d..%d' % (MINYEAR, MAXYEAR), year) if not 1 <= month <= 12: @@ -280,6 +282,8 @@ raise ValueError('day must be in 1..%d' % dim, day) def _check_time_fields(hour, minute, second, microsecond): + if not isinstance(hour, (int, long)): + raise TypeError('int expected') if not 0 <= hour <= 23: raise ValueError('hour must be in 0..23', hour) if not 0 <= minute <= 59: @@ -543,61 +547,76 @@ self = object.__new__(cls) - self.__days = d - self.__seconds = s - self.__microseconds = us + self._days = d + self._seconds = s + self._microseconds = us if abs(d) > 999999999: raise OverflowError("timedelta # of days is too large: %d" % d) return self def __repr__(self): - if self.__microseconds: + if self._microseconds: return "%s(%d, %d, %d)" % ('datetime.' + self.__class__.__name__, - self.__days, - self.__seconds, - self.__microseconds) - if self.__seconds: + self._days, + self._seconds, + self._microseconds) + if self._seconds: return "%s(%d, %d)" % ('datetime.' + self.__class__.__name__, - self.__days, - self.__seconds) - return "%s(%d)" % ('datetime.' + self.__class__.__name__, self.__days) + self._days, + self._seconds) + return "%s(%d)" % ('datetime.' + self.__class__.__name__, self._days) def __str__(self): - mm, ss = divmod(self.__seconds, 60) + mm, ss = divmod(self._seconds, 60) hh, mm = divmod(mm, 60) s = "%d:%02d:%02d" % (hh, mm, ss) - if self.__days: + if self._days: def plural(n): return n, abs(n) != 1 and "s" or "" - s = ("%d day%s, " % plural(self.__days)) + s - if self.__microseconds: - s = s + ".%06d" % self.__microseconds + s = ("%d day%s, " % plural(self._days)) + s + if self._microseconds: + s = s + ".%06d" % self._microseconds return s - days = property(lambda self: self.__days, doc="days") - seconds = property(lambda self: self.__seconds, doc="seconds") - microseconds = property(lambda self: self.__microseconds, - doc="microseconds") - def total_seconds(self): return ((self.days * 86400 + self.seconds) * 10**6 + self.microseconds) / 1e6 + # Read-only field accessors + @property + def days(self): + """days""" + return self._days + + @property + def seconds(self): + """seconds""" + return self._seconds + + @property + def microseconds(self): + """microseconds""" + return self._microseconds + def __add__(self, other): if isinstance(other, timedelta): # for CPython compatibility, we cannot use # our __class__ here, but need a real timedelta - return timedelta(self.__days + other.__days, - self.__seconds + other.__seconds, - self.__microseconds + other.__microseconds) + return timedelta(self._days + other._days, + self._seconds + other._seconds, + self._microseconds + other._microseconds) return NotImplemented __radd__ = __add__ def __sub__(self, other): if isinstance(other, timedelta): - return self + -other + # for CPython compatibility, we cannot use + # our __class__ here, but need a real timedelta + return timedelta(self._days - other._days, + self._seconds - other._seconds, + self._microseconds - other._microseconds) return NotImplemented def __rsub__(self, other): @@ -606,17 +625,17 @@ return NotImplemented def __neg__(self): - # for CPython compatibility, we cannot use - # our __class__ here, but need a real timedelta - return timedelta(-self.__days, - -self.__seconds, - -self.__microseconds) + # for CPython compatibility, we cannot use + # our __class__ here, but need a real timedelta + return timedelta(-self._days, + -self._seconds, + -self._microseconds) def __pos__(self): return self def __abs__(self): - if self.__days < 0: + if self._days < 0: return -self else: return self @@ -625,81 +644,81 @@ if isinstance(other, (int, long)): # for CPython compatibility, we cannot use # our __class__ here, but need a real timedelta - return timedelta(self.__days * other, - self.__seconds * other, - self.__microseconds * other) + return timedelta(self._days * other, + self._seconds * other, + self._microseconds * other) return NotImplemented __rmul__ = __mul__ def __div__(self, other): if isinstance(other, (int, long)): - usec = ((self.__days * (24*3600L) + self.__seconds) * 1000000 + - self.__microseconds) + usec = ((self._days * (24*3600L) + self._seconds) * 1000000 + + self._microseconds) return timedelta(0, 0, usec // other) return NotImplemented __floordiv__ = __div__ - # Comparisons. + # Comparisons of timedelta objects with other. def __eq__(self, other): if isinstance(other, timedelta): - return self.__cmp(other) == 0 + return self._cmp(other) == 0 else: return False def __ne__(self, other): if isinstance(other, timedelta): - return self.__cmp(other) != 0 + return self._cmp(other) != 0 else: return True def __le__(self, other): if isinstance(other, timedelta): - return self.__cmp(other) <= 0 + return self._cmp(other) <= 0 else: _cmperror(self, other) def __lt__(self, other): if isinstance(other, timedelta): - return self.__cmp(other) < 0 + return self._cmp(other) < 0 else: _cmperror(self, other) def __ge__(self, other): if isinstance(other, timedelta): - return self.__cmp(other) >= 0 + return self._cmp(other) >= 0 else: _cmperror(self, other) def __gt__(self, other): if isinstance(other, timedelta): - return self.__cmp(other) > 0 + return self._cmp(other) > 0 else: _cmperror(self, other) - def __cmp(self, other): + def _cmp(self, other): assert isinstance(other, timedelta) - return cmp(self.__getstate(), other.__getstate()) + return cmp(self._getstate(), other._getstate()) def __hash__(self): - return hash(self.__getstate()) + return hash(self._getstate()) def __nonzero__(self): - return (self.__days != 0 or - self.__seconds != 0 or - self.__microseconds != 0) + return (self._days != 0 or + self._seconds != 0 or + self._microseconds != 0) # Pickle support. __safe_for_unpickling__ = True # For Python 2.2 - def __getstate(self): - return (self.__days, self.__seconds, self.__microseconds) + def _getstate(self): + return (self._days, self._seconds, self._microseconds) def __reduce__(self): - return (self.__class__, self.__getstate()) + return (self.__class__, self._getstate()) timedelta.min = timedelta(-999999999) timedelta.max = timedelta(days=999999999, hours=23, minutes=59, seconds=59, @@ -749,25 +768,26 @@ return self _check_date_fields(year, month, day) self = object.__new__(cls) - self.__year = year - self.__month = month - self.__day = day + self._year = year + self._month = month + self._day = day return self # Additional constructors + @classmethod def fromtimestamp(cls, t): "Construct a date from a POSIX timestamp (like time.time())." y, m, d, hh, mm, ss, weekday, jday, dst = _time.localtime(t) return cls(y, m, d) - fromtimestamp = classmethod(fromtimestamp) + @classmethod def today(cls): "Construct a date from time.time()." t = _time.time() return cls.fromtimestamp(t) - today = classmethod(today) + @classmethod def fromordinal(cls, n): """Contruct a date from a proleptic Gregorian ordinal. @@ -776,16 +796,24 @@ """ y, m, d = _ord2ymd(n) return cls(y, m, d) - fromordinal = classmethod(fromordinal) # Conversions to string def __repr__(self): - "Convert to formal string, for repr()." + """Convert to formal string, for repr(). + + >>> dt = datetime(2010, 1, 1) + >>> repr(dt) + 'datetime.datetime(2010, 1, 1, 0, 0)' + + >>> dt = datetime(2010, 1, 1, tzinfo=timezone.utc) + >>> repr(dt) + 'datetime.datetime(2010, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)' + """ return "%s(%d, %d, %d)" % ('datetime.' + self.__class__.__name__, - self.__year, - self.__month, - self.__day) + self._year, + self._month, + self._day) # XXX These shouldn't depend on time.localtime(), because that # clips the usable dates to [1970 .. 2038). At least ctime() is # easily done without using strftime() -- that's better too because @@ -793,12 +821,20 @@ def ctime(self): "Format a la ctime()." - return tmxxx(self.__year, self.__month, self.__day).ctime() + return tmxxx(self._year, self._month, self._day).ctime() def strftime(self, fmt): "Format using strftime()." return _wrap_strftime(self, fmt, self.timetuple()) + def __format__(self, fmt): + if not isinstance(fmt, (str, unicode)): + raise ValueError("__format__ excepts str or unicode, not %s" % + fmt.__class__.__name__) + if len(fmt) != 0: + return self.strftime(fmt) + return str(self) + def isoformat(self): """Return the date formatted according to ISO. @@ -808,29 +844,31 @@ - http://www.w3.org/TR/NOTE-datetime - http://www.cl.cam.ac.uk/~mgk25/iso-time.html """ - return "%04d-%02d-%02d" % (self.__year, self.__month, self.__day) + return "%04d-%02d-%02d" % (self._year, self._month, self._day) __str__ = isoformat - def __format__(self, format): - if not isinstance(format, (str, unicode)): - raise ValueError("__format__ excepts str or unicode, not %s" % - format.__class__.__name__) - if not format: - return str(self) - return self.strftime(format) + # Read-only field accessors + @property + def year(self): + """year (1-9999)""" + return self._year - # Read-only field accessors - year = property(lambda self: self.__year, - doc="year (%d-%d)" % (MINYEAR, MAXYEAR)) - month = property(lambda self: self.__month, doc="month (1-12)") - day = property(lambda self: self.__day, doc="day (1-31)") + @property + def month(self): + """month (1-12)""" + return self._month + + @property + def day(self): + """day (1-31)""" + return self._day # Standard conversions, __cmp__, __hash__ (and helpers) def timetuple(self): "Return local time tuple compatible with time.localtime()." - return _build_struct_time(self.__year, self.__month, self.__day, + return _build_struct_time(self._year, self._month, self._day, 0, 0, 0, -1) def toordinal(self): @@ -839,24 +877,24 @@ January 1 of year 1 is day 1. Only the year, month and day values contribute to the result. """ - return _ymd2ord(self.__year, self.__month, self.__day) + return _ymd2ord(self._year, self._month, self._day) def replace(self, year=None, month=None, day=None): """Return a new date with new values for the specified fields.""" if year is None: - year = self.__year + year = self._year if month is None: - month = self.__month + month = self._month if day is None: - day = self.__day + day = self._day _check_date_fields(year, month, day) return date(year, month, day) - # Comparisons. + # Comparisons of date objects with other. def __eq__(self, other): if isinstance(other, date): - return self.__cmp(other) == 0 + return self._cmp(other) == 0 elif hasattr(other, "timetuple"): return NotImplemented else: @@ -864,7 +902,7 @@ def __ne__(self, other): if isinstance(other, date): - return self.__cmp(other) != 0 + return self._cmp(other) != 0 elif hasattr(other, "timetuple"): return NotImplemented else: @@ -872,7 +910,7 @@ def __le__(self, other): if isinstance(other, date): - return self.__cmp(other) <= 0 + return self._cmp(other) <= 0 elif hasattr(other, "timetuple"): return NotImplemented else: @@ -880,7 +918,7 @@ def __lt__(self, other): if isinstance(other, date): - return self.__cmp(other) < 0 + return self._cmp(other) < 0 elif hasattr(other, "timetuple"): return NotImplemented else: @@ -888,7 +926,7 @@ def __ge__(self, other): if isinstance(other, date): - return self.__cmp(other) >= 0 + return self._cmp(other) >= 0 elif hasattr(other, "timetuple"): return NotImplemented else: @@ -896,21 +934,21 @@ def __gt__(self, other): if isinstance(other, date): - return self.__cmp(other) > 0 + return self._cmp(other) > 0 elif hasattr(other, "timetuple"): return NotImplemented else: _cmperror(self, other) - def __cmp(self, other): + def _cmp(self, other): assert isinstance(other, date) - y, m, d = self.__year, self.__month, self.__day - y2, m2, d2 = other.__year, other.__month, other.__day + y, m, d = self._year, self._month, self._day + y2, m2, d2 = other._year, other._month, other._day return cmp((y, m, d), (y2, m2, d2)) def __hash__(self): "Hash." - return hash(self.__getstate()) + return hash(self._getstate()) # Computations @@ -922,9 +960,9 @@ def __add__(self, other): "Add a date to a timedelta." if isinstance(other, timedelta): - t = tmxxx(self.__year, - self.__month, - self.__day + other.days) + t = tmxxx(self._year, + self._month, + self._day + other.days) self._checkOverflow(t.year) result = date(t.year, t.month, t.day) return result @@ -966,9 +1004,9 @@ ISO calendar algorithm taken from http://www.phys.uu.nl/~vgent/calendar/isocalendar.htm """ - year = self.__year + year = self._year week1monday = _isoweek1monday(year) - today = _ymd2ord(self.__year, self.__month, self.__day) + today = _ymd2ord(self._year, self._month, self._day) # Internally, week and day have origin 0 week, day = divmod(today - week1monday, 7) if week < 0: @@ -985,18 +1023,18 @@ __safe_for_unpickling__ = True # For Python 2.2 - def __getstate(self): - yhi, ylo = divmod(self.__year, 256) - return ("%c%c%c%c" % (yhi, ylo, self.__month, self.__day), ) + def _getstate(self): + yhi, ylo = divmod(self._year, 256) + return ("%c%c%c%c" % (yhi, ylo, self._month, self._day), ) def __setstate(self, string): if len(string) != 4 or not (1 <= ord(string[2]) <= 12): raise TypeError("not enough arguments") - yhi, ylo, self.__month, self.__day = map(ord, string) - self.__year = yhi * 256 + ylo + yhi, ylo, self._month, self._day = map(ord, string) + self._year = yhi * 256 + ylo def __reduce__(self): - return (self.__class__, self.__getstate()) + return (self.__class__, self._getstate()) _date_class = date # so functions w/ args named "date" can get at the class @@ -1118,62 +1156,80 @@ return self _check_tzinfo_arg(tzinfo) _check_time_fields(hour, minute, second, microsecond) - self.__hour = hour - self.__minute = minute - self.__second = second - self.__microsecond = microsecond + self._hour = hour + self._minute = minute + self._second = second + self._microsecond = microsecond self._tzinfo = tzinfo return self # Read-only field accessors - hour = property(lambda self: self.__hour, doc="hour (0-23)") - minute = property(lambda self: self.__minute, doc="minute (0-59)") - second = property(lambda self: self.__second, doc="second (0-59)") - microsecond = property(lambda self: self.__microsecond, - doc="microsecond (0-999999)") - tzinfo = property(lambda self: self._tzinfo, doc="timezone info object") + @property + def hour(self): + """hour (0-23)""" + return self._hour + + @property + def minute(self): + """minute (0-59)""" + return self._minute + + @property + def second(self): + """second (0-59)""" + return self._second + + @property + def microsecond(self): + """microsecond (0-999999)""" + return self._microsecond + + @property + def tzinfo(self): + """timezone info object""" + return self._tzinfo # Standard conversions, __hash__ (and helpers) - # Comparisons. + # Comparisons of time objects with other. def __eq__(self, other): if isinstance(other, time): - return self.__cmp(other) == 0 + return self._cmp(other) == 0 else: return False def __ne__(self, other): if isinstance(other, time): - return self.__cmp(other) != 0 + return self._cmp(other) != 0 else: return True def __le__(self, other): if isinstance(other, time): - return self.__cmp(other) <= 0 + return self._cmp(other) <= 0 else: _cmperror(self, other) def __lt__(self, other): if isinstance(other, time): - return self.__cmp(other) < 0 + return self._cmp(other) < 0 else: _cmperror(self, other) def __ge__(self, other): if isinstance(other, time): - return self.__cmp(other) >= 0 + return self._cmp(other) >= 0 else: _cmperror(self, other) def __gt__(self, other): if isinstance(other, time): - return self.__cmp(other) > 0 + return self._cmp(other) > 0 else: _cmperror(self, other) - def __cmp(self, other): + def _cmp(self, other): assert isinstance(other, time) mytz = self._tzinfo ottz = other._tzinfo @@ -1187,23 +1243,23 @@ base_compare = myoff == otoff if base_compare: - return cmp((self.__hour, self.__minute, self.__second, - self.__microsecond), - (other.__hour, other.__minute, other.__second, - other.__microsecond)) + return cmp((self._hour, self._minute, self._second, + self._microsecond), + (other._hour, other._minute, other._second, + other._microsecond)) if myoff is None or otoff is None: # XXX Buggy in 2.2.2. raise TypeError("cannot compare naive and aware times") - myhhmm = self.__hour * 60 + self.__minute - myoff - othhmm = other.__hour * 60 + other.__minute - otoff - return cmp((myhhmm, self.__second, self.__microsecond), - (othhmm, other.__second, other.__microsecond)) + myhhmm = self._hour * 60 + self._minute - myoff + othhmm = other._hour * 60 + other._minute - otoff + return cmp((myhhmm, self._second, self._microsecond), + (othhmm, other._second, other._microsecond)) def __hash__(self): """Hash.""" tzoff = self._utcoffset() if not tzoff: # zero or None - return hash(self.__getstate()[0]) + return hash(self._getstate()[0]) h, m = divmod(self.hour * 60 + self.minute - tzoff, 60) if 0 <= h < 24: return hash(time(h, m, self.second, self.microsecond)) @@ -1227,14 +1283,14 @@ def __repr__(self): """Convert to formal string, for repr().""" - if self.__microsecond != 0: - s = ", %d, %d" % (self.__second, self.__microsecond) - elif self.__second != 0: - s = ", %d" % self.__second + if self._microsecond != 0: + s = ", %d, %d" % (self._second, self._microsecond) + elif self._second != 0: + s = ", %d" % self._second else: s = "" s= "%s(%d, %d%s)" % ('datetime.' + self.__class__.__name__, - self.__hour, self.__minute, s) + self._hour, self._minute, s) if self._tzinfo is not None: assert s[-1:] == ")" s = s[:-1] + ", tzinfo=%r" % self._tzinfo + ")" @@ -1246,8 +1302,8 @@ This is 'HH:MM:SS.mmmmmm+zz:zz', or 'HH:MM:SS+zz:zz' if self.microsecond == 0. """ - s = _format_time(self.__hour, self.__minute, self.__second, - self.__microsecond) + s = _format_time(self._hour, self._minute, self._second, + self._microsecond) tz = self._tzstr() if tz: s += tz @@ -1255,14 +1311,6 @@ __str__ = isoformat - def __format__(self, format): - if not isinstance(format, (str, unicode)): - raise ValueError("__format__ excepts str or unicode, not %s" % - format.__class__.__name__) - if not format: - return str(self) - return self.strftime(format) - def strftime(self, fmt): """Format using strftime(). The date part of the timestamp passed to underlying strftime should not be used. @@ -1270,10 +1318,18 @@ # The year must be >= 1900 else Python's strftime implementation # can raise a bogus exception. timetuple = (1900, 1, 1, - self.__hour, self.__minute, self.__second, + self._hour, self._minute, self._second, 0, 1, -1) return _wrap_strftime(self, fmt, timetuple) + def __format__(self, fmt): + if not isinstance(fmt, (str, unicode)): + raise ValueError("__format__ excepts str or unicode, not %s" % + fmt.__class__.__name__) + if len(fmt) != 0: + return self.strftime(fmt) + return str(self) + # Timezone functions def utcoffset(self): @@ -1350,10 +1406,10 @@ __safe_for_unpickling__ = True # For Python 2.2 - def __getstate(self): - us2, us3 = divmod(self.__microsecond, 256) + def _getstate(self): + us2, us3 = divmod(self._microsecond, 256) us1, us2 = divmod(us2, 256) - basestate = ("%c" * 6) % (self.__hour, self.__minute, self.__second, + basestate = ("%c" * 6) % (self._hour, self._minute, self._second, us1, us2, us3) if self._tzinfo is None: return (basestate,) @@ -1363,13 +1419,13 @@ def __setstate(self, string, tzinfo): if len(string) != 6 or ord(string[0]) >= 24: raise TypeError("an integer is required") - self.__hour, self.__minute, self.__second, us1, us2, us3 = \ + self._hour, self._minute, self._second, us1, us2, us3 = \ map(ord, string) - self.__microsecond = (((us1 << 8) | us2) << 8) | us3 + self._microsecond = (((us1 << 8) | us2) << 8) | us3 self._tzinfo = tzinfo def __reduce__(self): - return (time, self.__getstate()) + return (time, self._getstate()) _time_class = time # so functions w/ args named "time" can get at the class @@ -1378,9 +1434,11 @@ time.resolution = timedelta(microseconds=1) class datetime(date): + """datetime(year, month, day[, hour[, minute[, second[, microsecond[,tzinfo]]]]]) - # XXX needs docstrings - # See http://www.zope.org/Members/fdrake/DateTimeWiki/TimeZoneInfo + The year, month and day arguments are required. tzinfo may be None, or an + instance of a tzinfo subclass. The remaining arguments may be ints or longs. + """ def __new__(cls, year, month=None, day=None, hour=0, minute=0, second=0, microsecond=0, tzinfo=None): @@ -1393,24 +1451,43 @@ _check_time_fields(hour, minute, second, microsecond) self = date.__new__(cls, year, month, day) # XXX This duplicates __year, __month, __day for convenience :-( - self.__year = year - self.__month = month - self.__day = day - self.__hour = hour - self.__minute = minute - self.__second = second - self.__microsecond = microsecond + self._year = year + self._month = month + self._day = day + self._hour = hour + self._minute = minute + self._second = second + self._microsecond = microsecond self._tzinfo = tzinfo return self # Read-only field accessors - hour = property(lambda self: self.__hour, doc="hour (0-23)") - minute = property(lambda self: self.__minute, doc="minute (0-59)") - second = property(lambda self: self.__second, doc="second (0-59)") - microsecond = property(lambda self: self.__microsecond, - doc="microsecond (0-999999)") - tzinfo = property(lambda self: self._tzinfo, doc="timezone info object") + @property + def hour(self): + """hour (0-23)""" + return self._hour + @property + def minute(self): + """minute (0-59)""" + return self._minute + + @property + def second(self): + """second (0-59)""" + return self._second + + @property + def microsecond(self): + """microsecond (0-999999)""" + return self._microsecond + + @property + def tzinfo(self): + """timezone info object""" + return self._tzinfo + + @classmethod def fromtimestamp(cls, t, tz=None): """Construct a datetime from a POSIX timestamp (like time.time()). @@ -1438,37 +1515,42 @@ if tz is not None: result = tz.fromutc(result) return result - fromtimestamp = classmethod(fromtimestamp) + @classmethod def utcfromtimestamp(cls, t): "Construct a UTC datetime from a POSIX timestamp (like time.time())." - if 1 - (t % 1.0) < 0.0000005: - t = float(int(t)) + 1 - if t < 0: - t -= 1 + t, frac = divmod(t, 1.0) + us = round(frac * 1e6) + + # If timestamp is less than one microsecond smaller than a + # full second, us can be rounded up to 1000000. In this case, + # roll over to seconds, otherwise, ValueError is raised + # by the constructor. + if us == 1000000: + t += 1 + us = 0 y, m, d, hh, mm, ss, weekday, jday, dst = _time.gmtime(t) - us = int((t % 1.0) * 1000000) ss = min(ss, 59) # clamp out leap seconds if the platform has them return cls(y, m, d, hh, mm, ss, us) - utcfromtimestamp = classmethod(utcfromtimestamp) # XXX This is supposed to do better than we *can* do by using time.time(), # XXX if the platform supports a more accurate way. The C implementation # XXX uses gettimeofday on platforms that have it, but that isn't # XXX available from Python. So now() may return different results # XXX across the implementations. + @classmethod def now(cls, tz=None): "Construct a datetime from time.time() and optional time zone info." t = _time.time() return cls.fromtimestamp(t, tz) - now = classmethod(now) + @classmethod def utcnow(cls): "Construct a UTC datetime from time.time()." t = _time.time() return cls.utcfromtimestamp(t) - utcnow = classmethod(utcnow) + @classmethod def combine(cls, date, time): "Construct a datetime from a given date and a given time." if not isinstance(date, _date_class): @@ -1478,7 +1560,6 @@ return cls(date.year, date.month, date.day, time.hour, time.minute, time.second, time.microsecond, time.tzinfo) - combine = classmethod(combine) def timetuple(self): "Return local time tuple compatible with time.localtime()." @@ -1504,7 +1585,7 @@ def date(self): "Return the date part." - return date(self.__year, self.__month, self.__day) + return date(self._year, self._month, self._day) def time(self): "Return the time part, with tzinfo None." @@ -1564,8 +1645,8 @@ def ctime(self): "Format a la ctime()." - t = tmxxx(self.__year, self.__month, self.__day, self.__hour, - self.__minute, self.__second) + t = tmxxx(self._year, self._month, self._day, self._hour, + self._minute, self._second) return t.ctime() def isoformat(self, sep='T'): @@ -1580,10 +1661,10 @@ Optional argument sep specifies the separator between date and time, default 'T'. """ - s = ("%04d-%02d-%02d%c" % (self.__year, self.__month, self.__day, + s = ("%04d-%02d-%02d%c" % (self._year, self._month, self._day, sep) + - _format_time(self.__hour, self.__minute, self.__second, - self.__microsecond)) + _format_time(self._hour, self._minute, self._second, + self._microsecond)) off = self._utcoffset() if off is not None: if off < 0: @@ -1596,13 +1677,13 @@ return s def __repr__(self): - "Convert to formal string, for repr()." - L = [self.__year, self.__month, self.__day, # These are never zero - self.__hour, self.__minute, self.__second, self.__microsecond] + """Convert to formal string, for repr().""" + L = [self._year, self._month, self._day, # These are never zero + self._hour, self._minute, self._second, self._microsecond] if L[-1] == 0: del L[-1] if L[-1] == 0: - del L[-1] + del L[-1] s = ", ".join(map(str, L)) s = "%s(%s)" % ('datetime.' + self.__class__.__name__, s) if self._tzinfo is not None: @@ -1675,7 +1756,7 @@ def __eq__(self, other): if isinstance(other, datetime): - return self.__cmp(other) == 0 + return self._cmp(other) == 0 elif hasattr(other, "timetuple") and not isinstance(other, date): return NotImplemented else: @@ -1683,7 +1764,7 @@ def __ne__(self, other): if isinstance(other, datetime): - return self.__cmp(other) != 0 + return self._cmp(other) != 0 elif hasattr(other, "timetuple") and not isinstance(other, date): return NotImplemented else: @@ -1691,7 +1772,7 @@ def __le__(self, other): if isinstance(other, datetime): - return self.__cmp(other) <= 0 + return self._cmp(other) <= 0 elif hasattr(other, "timetuple") and not isinstance(other, date): return NotImplemented else: @@ -1699,7 +1780,7 @@ def __lt__(self, other): if isinstance(other, datetime): - return self.__cmp(other) < 0 + return self._cmp(other) < 0 elif hasattr(other, "timetuple") and not isinstance(other, date): return NotImplemented else: @@ -1707,7 +1788,7 @@ def __ge__(self, other): if isinstance(other, datetime): - return self.__cmp(other) >= 0 + return self._cmp(other) >= 0 elif hasattr(other, "timetuple") and not isinstance(other, date): return NotImplemented else: @@ -1715,13 +1796,13 @@ def __gt__(self, other): if isinstance(other, datetime): - return self.__cmp(other) > 0 + return self._cmp(other) > 0 elif hasattr(other, "timetuple") and not isinstance(other, date): return NotImplemented else: _cmperror(self, other) - def __cmp(self, other): + def _cmp(self, other): assert isinstance(other, datetime) mytz = self._tzinfo ottz = other._tzinfo @@ -1737,12 +1818,12 @@ base_compare = myoff == otoff if base_compare: - return cmp((self.__year, self.__month, self.__day, - self.__hour, self.__minute, self.__second, - self.__microsecond), - (other.__year, other.__month, other.__day, - other.__hour, other.__minute, other.__second, - other.__microsecond)) + return cmp((self._year, self._month, self._day, + self._hour, self._minute, self._second, + self._microsecond), + (other._year, other._month, other._day, + other._hour, other._minute, other._second, + other._microsecond)) if myoff is None or otoff is None: # XXX Buggy in 2.2.2. raise TypeError("cannot compare naive and aware datetimes") @@ -1756,13 +1837,13 @@ "Add a datetime and a timedelta." if not isinstance(other, timedelta): return NotImplemented - t = tmxxx(self.__year, - self.__month, - self.__day + other.days, - self.__hour, - self.__minute, - self.__second + other.seconds, - self.__microsecond + other.microseconds) + t = tmxxx(self._year, + self._month, + self._day + other.days, + self._hour, + self._minute, + self._second + other.seconds, + self._microsecond + other.microseconds) self._checkOverflow(t.year) result = datetime(t.year, t.month, t.day, t.hour, t.minute, t.second, @@ -1780,11 +1861,11 @@ days1 = self.toordinal() days2 = other.toordinal() - secs1 = self.__second + self.__minute * 60 + self.__hour * 3600 - secs2 = other.__second + other.__minute * 60 + other.__hour * 3600 + secs1 = self._second + self._minute * 60 + self._hour * 3600 + secs2 = other._second + other._minute * 60 + other._hour * 3600 base = timedelta(days1 - days2, secs1 - secs2, - self.__microsecond - other.__microsecond) + self._microsecond - other._microsecond) if self._tzinfo is other._tzinfo: return base myoff = self._utcoffset() @@ -1792,13 +1873,13 @@ if myoff == otoff: return base if myoff is None or otoff is None: - raise TypeError, "cannot mix naive and timezone-aware time" + raise TypeError("cannot mix naive and timezone-aware time") return base + timedelta(minutes = otoff-myoff) def __hash__(self): tzoff = self._utcoffset() if tzoff is None: - return hash(self.__getstate()[0]) + return hash(self._getstate()[0]) days = _ymd2ord(self.year, self.month, self.day) seconds = self.hour * 3600 + (self.minute - tzoff) * 60 + self.second return hash(timedelta(days, seconds, self.microsecond)) @@ -1807,12 +1888,12 @@ __safe_for_unpickling__ = True # For Python 2.2 - def __getstate(self): - yhi, ylo = divmod(self.__year, 256) - us2, us3 = divmod(self.__microsecond, 256) + def _getstate(self): + yhi, ylo = divmod(self._year, 256) + us2, us3 = divmod(self._microsecond, 256) us1, us2 = divmod(us2, 256) - basestate = ("%c" * 10) % (yhi, ylo, self.__month, self.__day, - self.__hour, self.__minute, self.__second, + basestate = ("%c" * 10) % (yhi, ylo, self._month, self._day, + self._hour, self._minute, self._second, us1, us2, us3) if self._tzinfo is None: return (basestate,) @@ -1820,14 +1901,14 @@ return (basestate, self._tzinfo) def __setstate(self, string, tzinfo): - (yhi, ylo, self.__month, self.__day, self.__hour, - self.__minute, self.__second, us1, us2, us3) = map(ord, string) - self.__year = yhi * 256 + ylo - self.__microsecond = (((us1 << 8) | us2) << 8) | us3 + (yhi, ylo, self._month, self._day, self._hour, + self._minute, self._second, us1, us2, us3) = map(ord, string) + self._year = yhi * 256 + ylo + self._microsecond = (((us1 << 8) | us2) << 8) | us3 self._tzinfo = tzinfo def __reduce__(self): - return (self.__class__, self.__getstate()) + return (self.__class__, self._getstate()) datetime.min = datetime(1, 1, 1) @@ -2009,7 +2090,7 @@ Because we know z.d said z was in daylight time (else [5] would have held and we would have stopped then), and we know z.d != z'.d (else [8] would have held -and we we have stopped then), and there are only 2 possible values dst() can +and we have stopped then), and there are only 2 possible values dst() can return in Eastern, it follows that z'.d must be 0 (which it is in the example, but the reasoning doesn't depend on the example -- it depends on there being two possible dst() outcomes, one zero and the other non-zero). Therefore diff --git a/lib_pypy/numpypy/core/fromnumeric.py b/lib_pypy/numpypy/core/fromnumeric.py --- a/lib_pypy/numpypy/core/fromnumeric.py +++ b/lib_pypy/numpypy/core/fromnumeric.py @@ -149,6 +149,7 @@ [5, 6]]) """ + assert order == 'C' if not hasattr(a, 'reshape'): a = numpypy.array(a) return a.reshape(newshape) @@ -690,6 +691,7 @@ 1 """ + assert axis is None if not hasattr(a, 'argmax'): a = numpypy.array(a) return a.argmax() @@ -705,6 +707,7 @@ documentation. """ + assert axis is None if not hasattr(a, 'argmin'): a = numpypy.array(a) return a.argmin() @@ -1354,9 +1357,11 @@ -128 """ + assert dtype is None + assert out is None if not hasattr(a, "sum"): a = numpypy.array(a) - return a.sum() + return a.sum(axis=axis) def product (a, axis=None, dtype=None, out=None): @@ -1382,6 +1387,8 @@ any : equivalent function """ + assert axis is None + assert out is None if not hasattr(a, 'any'): a = numpypy.array(a) return a.any() @@ -1396,6 +1403,8 @@ numpy.all : Equivalent function; see for details. """ + assert axis is None + assert out is None if not hasattr(a, 'all'): a = numpypy.array(a) return a.all() @@ -1464,6 +1473,8 @@ (191614240, 191614240) """ + assert axis is None + assert out is None if not hasattr(a, 'any'): a = numpypy.array(a) return a.any() @@ -1526,6 +1537,8 @@ (28293632, 28293632, array([ True], dtype=bool)) """ + assert axis is None + assert out is None if not hasattr(a, 'all'): a = numpypy.array(a) return a.all() @@ -1705,6 +1718,8 @@ 4.0 """ + assert axis is None + assert out is None if not hasattr(a, "max"): a = numpypy.array(a) return a.max() @@ -1766,6 +1781,8 @@ """ # amin() is equivalent to min() + assert axis is None + assert out is None if not hasattr(a, 'min'): a = numpypy.array(a) return a.min() @@ -2214,9 +2231,11 @@ 0.55000000074505806 """ + assert dtype is None + assert out is None if not hasattr(a, "mean"): a = numpypy.array(a) - return a.mean() + return a.mean(axis=axis) def std(a, axis=None, dtype=None, out=None, ddof=0): @@ -2305,6 +2324,10 @@ 0.44999999925552653 """ + assert axis is None + assert dtype is None + assert out is None + assert ddof == 0 if not hasattr(a, "std"): a = numpypy.array(a) return a.std() @@ -2398,6 +2421,10 @@ 0.20250000000000001 """ + assert axis is None + assert dtype is None + assert out is None + assert ddof == 0 if not hasattr(a, "var"): a = numpypy.array(a) return a.var() diff --git a/pypy/doc/translation.rst b/pypy/doc/translation.rst --- a/pypy/doc/translation.rst +++ b/pypy/doc/translation.rst @@ -155,7 +155,7 @@ function. The two input variables are the exception class and the exception value, respectively. (No other block will actually link to the exceptblock if the function does not - explicitely raise exceptions.) + explicitly raise exceptions.) ``Block`` @@ -325,7 +325,7 @@ Mutable objects need special treatment during annotation, because the annotation of contained values needs to be possibly updated to account for mutation operations, and consequently the annotation information -reflown through the relevant parts of the flow the graphs. +reflown through the relevant parts of the flow graphs. * ``SomeList`` stands for a list of homogeneous type (i.e. all the elements of the list are represented by a single common ``SomeXxx`` @@ -503,8 +503,8 @@ Since RPython is a garbage collected language there is a lot of heap memory allocation going on all the time, which would either not occur at all in a more -traditional explicitely managed language or results in an object which dies at -a time known in advance and can thus be explicitely deallocated. For example a +traditional explicitly managed language or results in an object which dies at +a time known in advance and can thus be explicitly deallocated. For example a loop of the following form:: for i in range(n): @@ -696,7 +696,7 @@ So far it is the second most mature high level backend after GenCLI: it still can't translate the full Standard Interpreter, but after the -Leysin sprint we were able to compile and run the rpytstone and +Leysin sprint we were able to compile and run the rpystone and richards benchmarks. GenJVM is almost entirely the work of Niko Matsakis, who worked on it diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py --- a/pypy/interpreter/executioncontext.py +++ b/pypy/interpreter/executioncontext.py @@ -445,6 +445,7 @@ AsyncAction.__init__(self, space) self.dying_objects = [] self.finalizers_lock_count = 0 + self.enabled_at_app_level = True def register_callback(self, w_obj, callback, descrname): self.dying_objects.append((w_obj, callback, descrname)) diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py --- a/pypy/jit/backend/test/runner_test.py +++ b/pypy/jit/backend/test/runner_test.py @@ -551,6 +551,28 @@ res = self.execute_operation(rop.CALL, [funcbox] + map(BoxInt, args), 'int', descr=calldescr) assert res.value == func(*args) + def test_call_box_func(self): + def a(a1, a2): + return a1 + a2 + def b(b1, b2): + return b1 * b2 + + arg1 = 40 + arg2 = 2 + for f in [a, b]: + TP = lltype.Signed + FPTR = self.Ptr(self.FuncType([TP, TP], TP)) + func_ptr = llhelper(FPTR, f) + FUNC = deref(FPTR) + funcconst = self.get_funcbox(self.cpu, func_ptr) + funcbox = funcconst.clonebox() + calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, + EffectInfo.MOST_GENERAL) + res = self.execute_operation(rop.CALL, + [funcbox, BoxInt(arg1), BoxInt(arg2)], + 'int', descr=calldescr) + assert res.getint() == f(arg1, arg2) + def test_call_stack_alignment(self): # test stack alignment issues, notably for Mac OS/X. # also test the ordering of the arguments. @@ -1872,6 +1894,7 @@ values.append(descr) values.append(self.cpu.get_latest_value_int(0)) values.append(self.cpu.get_latest_value_int(1)) + values.append(token) FUNC = self.FuncType([lltype.Signed, lltype.Signed], lltype.Void) func_ptr = llhelper(lltype.Ptr(FUNC), maybe_force) @@ -1902,7 +1925,8 @@ assert fail.identifier == 1 assert self.cpu.get_latest_value_int(0) == 1 assert self.cpu.get_latest_value_int(1) == 10 - assert values == [faildescr, 1, 10] + token = self.cpu.get_latest_force_token() + assert values == [faildescr, 1, 10, token] def test_force_operations_returning_int(self): values = [] @@ -1911,6 +1935,7 @@ self.cpu.force(token) values.append(self.cpu.get_latest_value_int(0)) values.append(self.cpu.get_latest_value_int(2)) + values.append(token) return 42 FUNC = self.FuncType([lltype.Signed, lltype.Signed], lltype.Signed) @@ -1944,7 +1969,8 @@ assert self.cpu.get_latest_value_int(0) == 1 assert self.cpu.get_latest_value_int(1) == 42 assert self.cpu.get_latest_value_int(2) == 10 - assert values == [1, 10] + token = self.cpu.get_latest_force_token() + assert values == [1, 10, token] def test_force_operations_returning_float(self): values = [] @@ -1953,6 +1979,7 @@ self.cpu.force(token) values.append(self.cpu.get_latest_value_int(0)) values.append(self.cpu.get_latest_value_int(2)) + values.append(token) return 42.5 FUNC = self.FuncType([lltype.Signed, lltype.Signed], lltype.Float) @@ -1988,7 +2015,8 @@ x = self.cpu.get_latest_value_float(1) assert longlong.getrealfloat(x) == 42.5 assert self.cpu.get_latest_value_int(2) == 10 - assert values == [1, 10] + token = self.cpu.get_latest_force_token() + assert values == [1, 10, token] def test_call_to_c_function(self): from pypy.rlib.libffi import CDLL, types, ArgChain, FUNCFLAG_CDECL diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py --- a/pypy/jit/backend/x86/assembler.py +++ b/pypy/jit/backend/x86/assembler.py @@ -1118,6 +1118,12 @@ for src, dst in singlefloats: self.mc.MOVD(dst, src) # Finally remap the arguments in the main regs + # If x is a register and is in dst_locs, then oups, it needs to + # be moved away: + if x in dst_locs: + src_locs.append(x) + dst_locs.append(r10) + x = r10 remap_frame_layout(self, src_locs, dst_locs, X86_64_SCRATCH_REG) self._regalloc.reserve_param(len(pass_on_stack)) @@ -2042,10 +2048,7 @@ size = sizeloc.value signloc = arglocs[1] - if isinstance(op.getarg(0), Const): - x = imm(op.getarg(0).getint()) - else: - x = arglocs[2] + x = arglocs[2] # the function address if x is eax: tmp = ecx else: diff --git a/pypy/jit/backend/x86/runner.py b/pypy/jit/backend/x86/runner.py --- a/pypy/jit/backend/x86/runner.py +++ b/pypy/jit/backend/x86/runner.py @@ -6,7 +6,7 @@ from pypy.jit.codewriter import longlong from pypy.jit.metainterp import history, compile from pypy.jit.backend.x86.assembler import Assembler386 -from pypy.jit.backend.x86.arch import FORCE_INDEX_OFS +from pypy.jit.backend.x86.arch import FORCE_INDEX_OFS, IS_X86_32 from pypy.jit.backend.x86.profagent import ProfileAgent from pypy.jit.backend.llsupport.llmodel import AbstractLLCPU from pypy.jit.backend.x86 import regloc @@ -142,7 +142,9 @@ cast_ptr_to_int._annspecialcase_ = 'specialize:arglltype(0)' cast_ptr_to_int = staticmethod(cast_ptr_to_int) - all_null_registers = lltype.malloc(rffi.LONGP.TO, 24, + all_null_registers = lltype.malloc(rffi.LONGP.TO, + IS_X86_32 and (16+8) # 16 + 8 regs + or (16+16), # 16 + 16 regs flavor='raw', zero=True, immortal=True) diff --git a/pypy/module/_codecs/test/test_ztranslation.py b/pypy/module/_codecs/test/test_ztranslation.py new file mode 100644 --- /dev/null +++ b/pypy/module/_codecs/test/test_ztranslation.py @@ -0,0 +1,5 @@ +from pypy.objspace.fake.checkmodule import checkmodule + + +def test__codecs_translates(): + checkmodule('_codecs') diff --git a/pypy/module/_hashlib/interp_hashlib.py b/pypy/module/_hashlib/interp_hashlib.py --- a/pypy/module/_hashlib/interp_hashlib.py +++ b/pypy/module/_hashlib/interp_hashlib.py @@ -34,8 +34,12 @@ ctx = lltype.malloc(ropenssl.EVP_MD_CTX.TO, flavor='raw') rgc.add_memory_pressure(HASH_MALLOC_SIZE + self.digest_size) - ropenssl.EVP_DigestInit(ctx, digest_type) - self.ctx = ctx + try: + ropenssl.EVP_DigestInit(ctx, digest_type) + self.ctx = ctx + except: + lltype.free(ctx, flavor='raw') + raise def __del__(self): # self.lock.free() diff --git a/pypy/module/gc/__init__.py b/pypy/module/gc/__init__.py --- a/pypy/module/gc/__init__.py +++ b/pypy/module/gc/__init__.py @@ -1,18 +1,18 @@ from pypy.interpreter.mixedmodule import MixedModule class Module(MixedModule): - appleveldefs = { - 'enable': 'app_gc.enable', - 'disable': 'app_gc.disable', - 'isenabled': 'app_gc.isenabled', - } interpleveldefs = { 'collect': 'interp_gc.collect', + 'enable': 'interp_gc.enable', + 'disable': 'interp_gc.disable', + 'isenabled': 'interp_gc.isenabled', 'enable_finalizers': 'interp_gc.enable_finalizers', 'disable_finalizers': 'interp_gc.disable_finalizers', 'garbage' : 'space.newlist([])', #'dump_heap_stats': 'interp_gc.dump_heap_stats', } + appleveldefs = { + } def __init__(self, space, w_name): if (not space.config.translating or diff --git a/pypy/module/gc/app_gc.py b/pypy/module/gc/app_gc.py deleted file mode 100644 --- a/pypy/module/gc/app_gc.py +++ /dev/null @@ -1,21 +0,0 @@ -# NOT_RPYTHON - -enabled = True - -def isenabled(): - global enabled - return enabled - -def enable(): - global enabled - import gc - if not enabled: - gc.enable_finalizers() - enabled = True - -def disable(): - global enabled - import gc - if enabled: - gc.disable_finalizers() - enabled = False diff --git a/pypy/module/gc/interp_gc.py b/pypy/module/gc/interp_gc.py --- a/pypy/module/gc/interp_gc.py +++ b/pypy/module/gc/interp_gc.py @@ -17,6 +17,26 @@ rgc.collect() return space.wrap(0) +def enable(space): + """Non-recursive version. Enable finalizers now. + If they were already enabled, no-op. + If they were disabled even several times, enable them anyway. + """ + if not space.user_del_action.enabled_at_app_level: + space.user_del_action.enabled_at_app_level = True + enable_finalizers(space) + +def disable(space): + """Non-recursive version. Disable finalizers now. Several calls + to this function are ignored. + """ + if space.user_del_action.enabled_at_app_level: + space.user_del_action.enabled_at_app_level = False + disable_finalizers(space) + +def isenabled(space): + return space.newbool(space.user_del_action.enabled_at_app_level) + def enable_finalizers(space): if space.user_del_action.finalizers_lock_count == 0: raise OperationError(space.w_ValueError, diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py --- a/pypy/module/micronumpy/__init__.py +++ b/pypy/module/micronumpy/__init__.py @@ -70,6 +70,7 @@ ("exp", "exp"), ("fabs", "fabs"), ("floor", "floor"), + ("ceil", "ceil"), ("greater", "greater"), ("greater_equal", "greater_equal"), ("less", "less"), @@ -85,6 +86,8 @@ ("subtract", "subtract"), ('sqrt', 'sqrt'), ("tan", "tan"), + ('bitwise_and', 'bitwise_and'), + ('bitwise_or', 'bitwise_or'), ]: interpleveldefs[exposed] = "interp_ufuncs.get(space).%s" % impl diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py --- a/pypy/module/micronumpy/interp_dtype.py +++ b/pypy/module/micronumpy/interp_dtype.py @@ -47,6 +47,10 @@ def getitem(self, storage, i): return self.itemtype.read(storage, self.itemtype.get_element_size(), i, 0) + def getitem_bool(self, storage, i): + isize = self.itemtype.get_element_size() + return self.itemtype.read_bool(storage, isize, i, 0) + def setitem(self, storage, i, box): self.itemtype.store(storage, self.itemtype.get_element_size(), i, 0, box) @@ -85,6 +89,12 @@ def descr_get_shape(self, space): return space.newtuple([]) + def is_int_type(self): + return self.kind == SIGNEDLTR or self.kind == UNSIGNEDLTR + + def is_bool_type(self): + return self.kind == BOOLLTR + W_Dtype.typedef = TypeDef("dtype", __module__ = "numpypy", __new__ = interp2app(W_Dtype.descr__new__.im_func), diff --git a/pypy/module/micronumpy/interp_iter.py b/pypy/module/micronumpy/interp_iter.py --- a/pypy/module/micronumpy/interp_iter.py +++ b/pypy/module/micronumpy/interp_iter.py @@ -4,6 +4,19 @@ from pypy.module.micronumpy.strides import calculate_broadcast_strides,\ calculate_slice_strides +# structures to describe slicing + +class Chunk(object): + def __init__(self, start, stop, step, lgt): + self.start = start + self.stop = stop + self.step = step + self.lgt = lgt + + def extend_shape(self, shape): + if self.step != 0: + shape.append(self.lgt) + class BaseTransform(object): pass @@ -38,11 +51,18 @@ self.size = size def next(self, shapelen): + return self._next(1) + + def _next(self, ofs): arr = instantiate(ArrayIterator) arr.size = self.size - arr.offset = self.offset + 1 + arr.offset = self.offset + ofs return arr + def next_no_increase(self, shapelen): + # a hack to make JIT believe this is always virtual + return self._next(0) + def done(self): return self.offset >= self.size diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -2,14 +2,15 @@ from pypy.interpreter.error import OperationError, operationerrfmt from pypy.interpreter.gateway import interp2app, NoneNotWrapped from pypy.interpreter.typedef import TypeDef, GetSetProperty -from pypy.module.micronumpy import interp_ufuncs, interp_dtype, signature +from pypy.module.micronumpy import interp_ufuncs, interp_dtype, signature,\ + interp_boxes from pypy.module.micronumpy.strides import calculate_slice_strides from pypy.rlib import jit from pypy.rpython.lltypesystem import lltype, rffi from pypy.tool.sourcetools import func_with_new_name from pypy.rlib.rstring import StringBuilder from pypy.module.micronumpy.interp_iter import ArrayIterator, OneDimIterator,\ - SkipLastAxisIterator + SkipLastAxisIterator, Chunk, ViewIterator numpy_driver = jit.JitDriver( greens=['shapelen', 'sig'], @@ -39,7 +40,24 @@ get_printable_location=signature.new_printable_location('slice'), name='numpy_slice', ) - +count_driver = jit.JitDriver( + greens=['shapelen'], + virtualizables=['frame'], + reds=['s', 'frame', 'iter', 'arr'], + name='numpy_count' +) +filter_driver = jit.JitDriver( + greens=['shapelen', 'sig'], + virtualizables=['frame'], + reds=['concr', 'argi', 'ri', 'frame', 'v', 'res', 'self'], + name='numpy_filter', +) +filter_set_driver = jit.JitDriver( + greens=['shapelen', 'sig'], + virtualizables=['frame'], + reds=['idx', 'idxi', 'frame', 'arr'], + name='numpy_filterset', +) def _find_shape_and_elems(space, w_iterable): shape = [space.len_w(w_iterable)] @@ -156,15 +174,17 @@ # fits the new shape, using those steps. If there is a shape/step mismatch # (meaning that the realignment of elements crosses from one step into another) # return None so that the caller can raise an exception. -def calc_new_strides(new_shape, old_shape, old_strides): +def calc_new_strides(new_shape, old_shape, old_strides, order): + # Return the proper strides for new_shape, or None if the mapping crosses + # stepping boundaries + # Assumes that prod(old_shape) == prod(new_shape), len(old_shape) > 1, and # len(new_shape) > 0 steps = [] last_step = 1 oldI = 0 new_strides = [] - if old_strides[0] < old_strides[-1]: - #Start at old_shape[0], old_stides[0] + if order == 'F': for i in range(len(old_shape)): steps.append(old_strides[i] / last_step) last_step *= old_shape[i] @@ -181,12 +201,10 @@ n_old_elems_to_use *= old_shape[oldI] if n_new_elems_used == n_old_elems_to_use: oldI += 1 - if oldI >= len(old_shape): - continue - cur_step = steps[oldI] - n_old_elems_to_use *= old_shape[oldI] - else: - #Start at old_shape[-1], old_strides[-1] + if oldI < len(old_shape): + cur_step = steps[oldI] + n_old_elems_to_use *= old_shape[oldI] + elif order == 'C': for i in range(len(old_shape) - 1, -1, -1): steps.insert(0, old_strides[i] / last_step) last_step *= old_shape[i] @@ -205,10 +223,10 @@ n_old_elems_to_use *= old_shape[oldI] if n_new_elems_used == n_old_elems_to_use: oldI -= 1 - if oldI < -len(old_shape): - continue - cur_step = steps[oldI] - n_old_elems_to_use *= old_shape[oldI] + if oldI >= -len(old_shape): + cur_step = steps[oldI] + n_old_elems_to_use *= old_shape[oldI] + assert len(new_strides) == len(new_shape) return new_strides class BaseArray(Wrappable): @@ -270,6 +288,9 @@ descr_gt = _binop_impl("greater") descr_ge = _binop_impl("greater_equal") + descr_and = _binop_impl("bitwise_and") + descr_or = _binop_impl("bitwise_or") + def _binop_right_impl(ufunc_name): def impl(self, space, w_other): w_other = scalar_w(space, @@ -409,9 +430,15 @@ def descr_copy(self, space): return self.copy(space) + def descr_flatten(self, space): + return self.flatten(space) + def copy(self, space): return self.get_concrete().copy(space) + def flatten(self, space): + return self.get_concrete().flatten(space) + def descr_len(self, space): if len(self.shape): return space.wrap(self.shape[0]) @@ -479,11 +506,69 @@ def _prepare_slice_args(self, space, w_idx): if (space.isinstance_w(w_idx, space.w_int) or space.isinstance_w(w_idx, space.w_slice)): - return [space.decode_index4(w_idx, self.shape[0])] - return [space.decode_index4(w_item, self.shape[i]) for i, w_item in + return [Chunk(*space.decode_index4(w_idx, self.shape[0]))] + return [Chunk(*space.decode_index4(w_item, self.shape[i])) for i, w_item in enumerate(space.fixedview(w_idx))] + def count_all_true(self, arr): + sig = arr.find_sig() + frame = sig.create_frame(self) + shapelen = len(arr.shape) + s = 0 + iter = None + while not frame.done(): + count_driver.jit_merge_point(arr=arr, frame=frame, iter=iter, s=s, + shapelen=shapelen) + iter = frame.get_final_iter() + s += arr.dtype.getitem_bool(arr.storage, iter.offset) + frame.next(shapelen) + return s + + def getitem_filter(self, space, arr): + concr = arr.get_concrete() + size = self.count_all_true(concr) + res = W_NDimArray(size, [size], self.find_dtype()) + ri = ArrayIterator(size) + shapelen = len(self.shape) + argi = concr.create_iter() + sig = self.find_sig() + frame = sig.create_frame(self) + v = None + while not frame.done(): + filter_driver.jit_merge_point(concr=concr, argi=argi, ri=ri, + frame=frame, v=v, res=res, sig=sig, + shapelen=shapelen, self=self) + if concr.dtype.getitem_bool(concr.storage, argi.offset): + v = sig.eval(frame, self) + res.setitem(ri.offset, v) + ri = ri.next(1) + else: + ri = ri.next_no_increase(1) + argi = argi.next(shapelen) + frame.next(shapelen) + return res + + def setitem_filter(self, space, idx, val): + size = self.count_all_true(idx) + arr = SliceArray([size], self.dtype, self, val) + sig = arr.find_sig() + shapelen = len(self.shape) + frame = sig.create_frame(arr) + idxi = idx.create_iter() + while not frame.done(): + filter_set_driver.jit_merge_point(idx=idx, idxi=idxi, sig=sig, + frame=frame, arr=arr, + shapelen=shapelen) + if idx.dtype.getitem_bool(idx.storage, idxi.offset): + sig.eval(frame, arr) + frame.next_from_second(1) + frame.next_first(shapelen) + idxi = idxi.next(shapelen) + def descr_getitem(self, space, w_idx): + if (isinstance(w_idx, BaseArray) and w_idx.shape == self.shape and + w_idx.find_dtype().is_bool_type()): + return self.getitem_filter(space, w_idx) if self._single_item_result(space, w_idx): concrete = self.get_concrete() item = concrete._index_of_single_item(space, w_idx) @@ -493,6 +578,11 @@ def descr_setitem(self, space, w_idx, w_value): self.invalidated() + if (isinstance(w_idx, BaseArray) and w_idx.shape == self.shape and + w_idx.find_dtype().is_bool_type()): + return self.get_concrete().setitem_filter(space, + w_idx.get_concrete(), + convert_to_array(space, w_value)) if self._single_item_result(space, w_idx): concrete = self.get_concrete() item = concrete._index_of_single_item(space, w_idx) @@ -509,9 +599,8 @@ def create_slice(self, chunks): shape = [] i = -1 - for i, (start_, stop, step, lgt) in enumerate(chunks): - if step != 0: - shape.append(lgt) + for i, chunk in enumerate(chunks): + chunk.extend_shape(shape) s = i + 1 assert s >= 0 shape += self.shape[s:] @@ -542,8 +631,8 @@ concrete = self.get_concrete() new_shape = get_shape_from_iterable(space, concrete.size, w_shape) # Since we got to here, prod(new_shape) == self.size - new_strides = calc_new_strides(new_shape, - concrete.shape, concrete.strides) + new_strides = calc_new_strides(new_shape, concrete.shape, + concrete.strides, concrete.order) if new_strides: # We can create a view, strides somehow match up. ndims = len(new_shape) @@ -686,6 +775,11 @@ def copy(self, space): return Scalar(self.dtype, self.value) + def flatten(self, space): + array = W_NDimArray(self.size, [self.size], self.dtype) + array.setitem(0, self.value) + return array + def fill(self, space, w_value): self.value = self.dtype.coerce(space, w_value) @@ -724,8 +818,7 @@ frame=frame, ri=ri, self=self, result=result) - result.dtype.setitem(result.storage, ri.offset, - sig.eval(frame, self)) + result.setitem(ri.offset, sig.eval(frame, self)) frame.next(shapelen) ri = ri.next(shapelen) return result @@ -832,14 +925,15 @@ self.left.create_sig(), self.right.create_sig()) class SliceArray(Call2): - def __init__(self, shape, dtype, left, right): + def __init__(self, shape, dtype, left, right, no_broadcast=False): + self.no_broadcast = no_broadcast Call2.__init__(self, None, 'sliceloop', shape, dtype, dtype, left, right) def create_sig(self): lsig = self.left.create_sig() rsig = self.right.create_sig() - if self.shape != self.right.shape: + if not self.no_broadcast and self.shape != self.right.shape: return signature.SliceloopBroadcastSignature(self.ufunc, self.name, self.calc_dtype, @@ -945,7 +1039,7 @@ builder.append('\n' + indent) else: builder.append(indent) - view = self.create_slice([(i, 0, 0, 1)]).get_concrete() + view = self.create_slice([Chunk(i, 0, 0, 1)]).get_concrete() view.to_str(space, comma, builder, indent=indent + ' ', use_ellipsis=use_ellipsis) if i < self.shape[0] - 1: @@ -962,7 +1056,7 @@ builder.append(indent) # create_slice requires len(chunks) > 1 in order to reduce # shape - view = self.create_slice([(i, 0, 0, 1)]).get_concrete() + view = self.create_slice([Chunk(i, 0, 0, 1)]).get_concrete() view.to_str(space, comma, builder, indent=indent + ' ', use_ellipsis=use_ellipsis) i += 1 @@ -1068,6 +1162,15 @@ array.setslice(space, self) return array + def flatten(self, space): + array = W_NDimArray(self.size, [self.size], self.dtype, self.order) + if self.supports_fast_slicing(): + array._fast_setslice(space, self) + else: + arr = SliceArray(array.shape, array.dtype, array, self, no_broadcast=True) + array._sliceloop(arr) + return array + def fill(self, space, w_value): self.setslice(space, scalar_w(space, self.dtype, w_value)) @@ -1091,6 +1194,10 @@ parent) self.start = start + def create_iter(self): + return ViewIterator(self.start, self.strides, self.backstrides, + self.shape) + def setshape(self, space, new_shape): if len(self.shape) < 1: return @@ -1114,7 +1221,8 @@ self.backstrides = backstrides self.shape = new_shape return - new_strides = calc_new_strides(new_shape, self.shape, self.strides) + new_strides = calc_new_strides(new_shape, self.shape, self.strides, + self.order) if new_strides is None: raise OperationError(space.w_AttributeError, space.wrap( "incompatible shape for a non-contiguous array")) @@ -1137,6 +1245,9 @@ self.shape = new_shape self.calc_strides(new_shape) + def create_iter(self): + return ArrayIterator(self.size) + def create_sig(self): return signature.ArraySignature(self.dtype) @@ -1191,6 +1302,7 @@ arr = W_NDimArray(size, shape[:], dtype=dtype, order=order) shapelen = len(shape) arr_iter = ArrayIterator(arr.size) + # XXX we might want to have a jitdriver here for i in range(len(elems_w)): w_elem = elems_w[i] dtype.setitem(arr.storage, arr_iter.offset, @@ -1257,6 +1369,9 @@ __gt__ = interp2app(BaseArray.descr_gt), __ge__ = interp2app(BaseArray.descr_ge), + __and__ = interp2app(BaseArray.descr_and), + __or__ = interp2app(BaseArray.descr_or), + __repr__ = interp2app(BaseArray.descr_repr), __str__ = interp2app(BaseArray.descr_str), __array_interface__ = GetSetProperty(BaseArray.descr_array_iface), @@ -1286,6 +1401,7 @@ fill = interp2app(BaseArray.descr_fill), copy = interp2app(BaseArray.descr_copy), + flatten = interp2app(BaseArray.descr_flatten), reshape = interp2app(BaseArray.descr_reshape), tolist = interp2app(BaseArray.descr_tolist), ) diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py --- a/pypy/module/micronumpy/interp_ufuncs.py +++ b/pypy/module/micronumpy/interp_ufuncs.py @@ -249,15 +249,16 @@ class W_Ufunc2(W_Ufunc): - _immutable_fields_ = ["comparison_func", "func", "name"] + _immutable_fields_ = ["comparison_func", "func", "name", "int_only"] argcount = 2 def __init__(self, func, name, promote_to_float=False, promote_bools=False, - identity=None, comparison_func=False): + identity=None, comparison_func=False, int_only=False): W_Ufunc.__init__(self, name, promote_to_float, promote_bools, identity) self.func = func self.comparison_func = comparison_func + self.int_only = int_only def call(self, space, args_w): from pypy.module.micronumpy.interp_numarray import (Call2, @@ -268,6 +269,7 @@ w_rhs = convert_to_array(space, w_rhs) calc_dtype = find_binop_result_dtype(space, w_lhs.find_dtype(), w_rhs.find_dtype(), + int_only=self.int_only, promote_to_float=self.promote_to_float, promote_bools=self.promote_bools, ) @@ -304,10 +306,12 @@ def find_binop_result_dtype(space, dt1, dt2, promote_to_float=False, - promote_bools=False): + promote_bools=False, int_only=False): # dt1.num should be <= dt2.num if dt1.num > dt2.num: dt1, dt2 = dt2, dt1 + if int_only and (not dt1.is_int_type() or not dt2.is_int_type()): + raise OperationError(space.w_TypeError, space.wrap("Unsupported types")) # Some operations promote op(bool, bool) to return int8, rather than bool if promote_bools and (dt1.kind == dt2.kind == interp_dtype.BOOLLTR): return interp_dtype.get_dtype_cache(space).w_int8dtype @@ -425,6 +429,10 @@ ("add", "add", 2, {"identity": 0}), ("subtract", "sub", 2), ("multiply", "mul", 2, {"identity": 1}), + ("bitwise_and", "bitwise_and", 2, {"identity": 1, + 'int_only': True}), + ("bitwise_or", "bitwise_or", 2, {"identity": 0, + 'int_only': True}), ("divide", "div", 2, {"promote_bools": True}), ("mod", "mod", 2, {"promote_bools": True}), ("power", "pow", 2, {"promote_bools": True}), @@ -449,6 +457,7 @@ ("fabs", "fabs", 1, {"promote_to_float": True}), ("floor", "floor", 1, {"promote_to_float": True}), + ("ceil", "ceil", 1, {"promote_to_float": True}), ("exp", "exp", 1, {"promote_to_float": True}), ('sqrt', 'sqrt', 1, {'promote_to_float': True}), @@ -475,7 +484,7 @@ extra_kwargs["identity"] = identity func = ufunc_dtype_caller(space, ufunc_name, op_name, argcount, - comparison_func=extra_kwargs.get("comparison_func", False) + comparison_func=extra_kwargs.get("comparison_func", False), ) if argcount == 1: ufunc = W_Ufunc1(func, ufunc_name, **extra_kwargs) diff --git a/pypy/module/micronumpy/signature.py b/pypy/module/micronumpy/signature.py --- a/pypy/module/micronumpy/signature.py +++ b/pypy/module/micronumpy/signature.py @@ -82,6 +82,16 @@ for i in range(len(self.iterators)): self.iterators[i] = self.iterators[i].next(shapelen) + @unroll_safe + def next_from_second(self, shapelen): + """ Don't increase the first iterator + """ + for i in range(1, len(self.iterators)): + self.iterators[i] = self.iterators[i].next(shapelen) + + def next_first(self, shapelen): + self.iterators[0] = self.iterators[0].next(shapelen) + def get_final_iter(self): final_iter = promote(self.final_iter) if final_iter < 0: diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py --- a/pypy/module/micronumpy/strides.py +++ b/pypy/module/micronumpy/strides.py @@ -10,12 +10,12 @@ rstart = start rshape = [] i = -1 - for i, (start_, stop, step, lgt) in enumerate(chunks): - if step != 0: - rstrides.append(strides[i] * step) - rbackstrides.append(strides[i] * (lgt - 1) * step) - rshape.append(lgt) - rstart += strides[i] * start_ + for i, chunk in enumerate(chunks): + if chunk.step != 0: + rstrides.append(strides[i] * chunk.step) + rbackstrides.append(strides[i] * (chunk.lgt - 1) * chunk.step) + rshape.append(chunk.lgt) + rstart += strides[i] * chunk.start # add a reminder s = i + 1 assert s >= 0 diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -2,6 +2,7 @@ import py from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest from pypy.module.micronumpy.interp_numarray import W_NDimArray, shape_agreement +from pypy.module.micronumpy.interp_iter import Chunk from pypy.module.micronumpy import signature from pypy.interpreter.error import OperationError from pypy.conftest import gettestobjspace @@ -37,53 +38,54 @@ def test_create_slice_f(self): a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'F') - s = a.create_slice([(3, 0, 0, 1)]) + s = a.create_slice([Chunk(3, 0, 0, 1)]) assert s.start == 3 assert s.strides == [10, 50] assert s.backstrides == [40, 100] - s = a.create_slice([(1, 9, 2, 4)]) + s = a.create_slice([Chunk(1, 9, 2, 4)]) assert s.start == 1 assert s.strides == [2, 10, 50] assert s.backstrides == [6, 40, 100] - s = a.create_slice([(1, 5, 3, 2), (1, 2, 1, 1), (1, 0, 0, 1)]) + s = a.create_slice([Chunk(1, 5, 3, 2), Chunk(1, 2, 1, 1), Chunk(1, 0, 0, 1)]) assert s.shape == [2, 1] assert s.strides == [3, 10] assert s.backstrides == [3, 0] - s = a.create_slice([(0, 10, 1, 10), (2, 0, 0, 1)]) + s = a.create_slice([Chunk(0, 10, 1, 10), Chunk(2, 0, 0, 1)]) assert s.start == 20 assert s.shape == [10, 3] def test_create_slice_c(self): a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'C') - s = a.create_slice([(3, 0, 0, 1)]) + s = a.create_slice([Chunk(3, 0, 0, 1)]) assert s.start == 45 assert s.strides == [3, 1] assert s.backstrides == [12, 2] - s = a.create_slice([(1, 9, 2, 4)]) + s = a.create_slice([Chunk(1, 9, 2, 4)]) assert s.start == 15 assert s.strides == [30, 3, 1] assert s.backstrides == [90, 12, 2] - s = a.create_slice([(1, 5, 3, 2), (1, 2, 1, 1), (1, 0, 0, 1)]) + s = a.create_slice([Chunk(1, 5, 3, 2), Chunk(1, 2, 1, 1), + Chunk(1, 0, 0, 1)]) assert s.start == 19 assert s.shape == [2, 1] assert s.strides == [45, 3] assert s.backstrides == [45, 0] - s = a.create_slice([(0, 10, 1, 10), (2, 0, 0, 1)]) + s = a.create_slice([Chunk(0, 10, 1, 10), Chunk(2, 0, 0, 1)]) assert s.start == 6 assert s.shape == [10, 3] def test_slice_of_slice_f(self): a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'F') - s = a.create_slice([(5, 0, 0, 1)]) + s = a.create_slice([Chunk(5, 0, 0, 1)]) assert s.start == 5 - s2 = s.create_slice([(3, 0, 0, 1)]) + s2 = s.create_slice([Chunk(3, 0, 0, 1)]) assert s2.shape == [3] assert s2.strides == [50] assert s2.parent is a assert s2.backstrides == [100] assert s2.start == 35 - s = a.create_slice([(1, 5, 3, 2)]) - s2 = s.create_slice([(0, 2, 1, 2), (2, 0, 0, 1)]) + s = a.create_slice([Chunk(1, 5, 3, 2)]) + s2 = s.create_slice([Chunk(0, 2, 1, 2), Chunk(2, 0, 0, 1)]) assert s2.shape == [2, 3] assert s2.strides == [3, 50] assert s2.backstrides == [3, 100] @@ -91,16 +93,16 @@ def test_slice_of_slice_c(self): a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), order='C') - s = a.create_slice([(5, 0, 0, 1)]) + s = a.create_slice([Chunk(5, 0, 0, 1)]) assert s.start == 15 * 5 - s2 = s.create_slice([(3, 0, 0, 1)]) + s2 = s.create_slice([Chunk(3, 0, 0, 1)]) assert s2.shape == [3] assert s2.strides == [1] assert s2.parent is a assert s2.backstrides == [2] assert s2.start == 5 * 15 + 3 * 3 - s = a.create_slice([(1, 5, 3, 2)]) - s2 = s.create_slice([(0, 2, 1, 2), (2, 0, 0, 1)]) + s = a.create_slice([Chunk(1, 5, 3, 2)]) + s2 = s.create_slice([Chunk(0, 2, 1, 2), Chunk(2, 0, 0, 1)]) assert s2.shape == [2, 3] assert s2.strides == [45, 1] assert s2.backstrides == [45, 2] @@ -108,14 +110,14 @@ def test_negative_step_f(self): a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'F') - s = a.create_slice([(9, -1, -2, 5)]) + s = a.create_slice([Chunk(9, -1, -2, 5)]) assert s.start == 9 assert s.strides == [-2, 10, 50] assert s.backstrides == [-8, 40, 100] def test_negative_step_c(self): a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), order='C') - s = a.create_slice([(9, -1, -2, 5)]) + s = a.create_slice([Chunk(9, -1, -2, 5)]) assert s.start == 135 assert s.strides == [-30, 3, 1] assert s.backstrides == [-120, 12, 2] @@ -124,7 +126,7 @@ a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'F') r = a._index_of_single_item(self.space, self.newtuple(1, 2, 2)) assert r == 1 + 2 * 10 + 2 * 50 - s = a.create_slice([(0, 10, 1, 10), (2, 0, 0, 1)]) + s = a.create_slice([Chunk(0, 10, 1, 10), Chunk(2, 0, 0, 1)]) r = s._index_of_single_item(self.space, self.newtuple(1, 0)) assert r == a._index_of_single_item(self.space, self.newtuple(1, 2, 0)) r = s._index_of_single_item(self.space, self.newtuple(1, 1)) @@ -134,7 +136,7 @@ a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'C') r = a._index_of_single_item(self.space, self.newtuple(1, 2, 2)) assert r == 1 * 3 * 5 + 2 * 3 + 2 - s = a.create_slice([(0, 10, 1, 10), (2, 0, 0, 1)]) + s = a.create_slice([Chunk(0, 10, 1, 10), Chunk(2, 0, 0, 1)]) r = s._index_of_single_item(self.space, self.newtuple(1, 0)) assert r == a._index_of_single_item(self.space, self.newtuple(1, 2, 0)) r = s._index_of_single_item(self.space, self.newtuple(1, 1)) @@ -152,13 +154,18 @@ def test_calc_new_strides(self): from pypy.module.micronumpy.interp_numarray import calc_new_strides - assert calc_new_strides([2, 4], [4, 2], [4, 2]) == [8, 2] - assert calc_new_strides([2, 4, 3], [8, 3], [1, 16]) == [1, 2, 16] - assert calc_new_strides([2, 3, 4], [8, 3], [1, 16]) is None - assert calc_new_strides([24], [2, 4, 3], [48, 6, 1]) is None - assert calc_new_strides([24], [2, 4, 3], [24, 6, 2]) == [2] - assert calc_new_strides([105, 1], [3, 5, 7], [35, 7, 1]) == [1, 1] - assert calc_new_strides([1, 105], [3, 5, 7], [35, 7, 1]) == [105, 1] + assert calc_new_strides([2, 4], [4, 2], [4, 2], "C") == [8, 2] + assert calc_new_strides([2, 4, 3], [8, 3], [1, 16], 'F') == [1, 2, 16] + assert calc_new_strides([2, 3, 4], [8, 3], [1, 16], 'F') is None + assert calc_new_strides([24], [2, 4, 3], [48, 6, 1], 'C') is None + assert calc_new_strides([24], [2, 4, 3], [24, 6, 2], 'C') == [2] + assert calc_new_strides([105, 1], [3, 5, 7], [35, 7, 1],'C') == [1, 1] + assert calc_new_strides([1, 105], [3, 5, 7], [35, 7, 1],'C') == [105, 1] + assert calc_new_strides([1, 105], [3, 5, 7], [35, 7, 1],'F') is None + assert calc_new_strides([1, 1, 1, 105, 1], [15, 7], [7, 1],'C') == \ + [105, 105, 105, 1, 1] + assert calc_new_strides([1, 1, 105, 1, 1], [7, 15], [1, 7],'F') == \ + [1, 1, 1, 105, 105] class AppTestNumArray(BaseNumpyAppTest): @@ -389,6 +396,8 @@ a.shape = () #numpy allows this a.shape = (1,) + a = array(range(6)).reshape(2,3).T + raises(AttributeError, 'a.shape = 6') def test_reshape(self): from _numpypy import array, zeros @@ -732,7 +741,7 @@ assert d[1] == 12 def test_mean(self): - from _numpypy import array + from _numpypy import array, arange a = array(range(5)) assert a.mean() == 2.0 assert a[:4].mean() == 1.5 @@ -742,6 +751,7 @@ assert a.mean(axis=0)[0, 0] == 35 assert (b == array(range(35, 70), dtype=float).reshape(5, 7)).all() assert (a.mean(2) == array(range(0, 15), dtype=float).reshape(3, 5) * 7 + 3).all() + assert (arange(10).reshape(5, 2).mean(axis=1) == [0.5, 2.5, 4.5, 6.5, 8.5]).all() def test_sum(self): from _numpypy import array @@ -1025,11 +1035,15 @@ assert a[0].tolist() == [17.1, 27.2] def test_var(self): - from _numpypy import array + from _numpypy import array, arange a = array(range(10)) assert a.var() == 8.25 a = array([5.0]) assert a.var() == 0.0 + a = arange(10).reshape(5, 2) + assert a.var() == 8.25 + #assert (a.var(0) == [8, 8]).all() + #assert (a.var(1) == [.25] * 5).all() def test_std(self): from _numpypy import array @@ -1038,6 +1052,22 @@ a = array([5.0]) assert a.std() == 0.0 + def test_flatten(self): + from _numpypy import array + + a = array([[1, 2, 3], [4, 5, 6]]) + assert (a.flatten() == [1, 2, 3, 4, 5, 6]).all() + a = array([[[1, 2], [3, 4]], [[5, 6], [7, 8]]]) + assert (a.flatten() == [1, 2, 3, 4, 5, 6, 7, 8]).all() + a = array([1, 2, 3, 4, 5, 6, 7, 8]) + assert (a[::2].flatten() == [1, 3, 5, 7]).all() + a = array([1, 2, 3]) + assert ((a + a).flatten() == [2, 4, 6]).all() + a = array(2) + assert (a.flatten() == [2]).all() + a = array([[1, 2], [3, 4]]) + assert (a.T.flatten() == [1, 3, 2, 4]).all() + class AppTestMultiDim(BaseNumpyAppTest): def test_init(self): @@ -1361,15 +1391,25 @@ assert isinstance(i['data'][0], int) raises(TypeError, getattr, array(3), '__array_interface__') + def test_array_indexing_one_elem(self): + skip("not yet") + from _numpypy import array, arange + raises(IndexError, 'arange(3)[array([3.5])]') + a = arange(3)[array([1])] + assert a == 1 + assert a[0] == 1 + raises(IndexError,'arange(3)[array([15])]') + assert arange(3)[array([-3])] == 0 + raises(IndexError,'arange(3)[array([-15])]') + assert arange(3)[array(1)] == 1 + def test_fill(self): from _numpypy import array - a = array([1, 2, 3]) a.fill(10) assert (a == [10, 10, 10]).all() a.fill(False) assert (a == [0, 0, 0]).all() - b = a[:1] b.fill(4) assert (b == [4]).all() @@ -1383,6 +1423,24 @@ d.fill(100) assert d == 100 + def test_array_indexing_bool(self): + from _numpypy import arange + a = arange(10) + assert (a[a > 3] == [4, 5, 6, 7, 8, 9]).all() + a = arange(10).reshape(5, 2) + assert (a[a > 3] == [4, 5, 6, 7, 8, 9]).all() + assert (a[a & 1 == 1] == [1, 3, 5, 7, 9]).all() + + def test_array_indexing_bool_setitem(self): + from _numpypy import arange, array + a = arange(6) + a[a > 3] = 15 + assert (a == [0, 1, 2, 3, 15, 15]).all() + a = arange(6).reshape(3, 2) + a[a & 1 == 1] = array([8, 9, 10]) + assert (a == [[0, 8], [2, 9], [4, 10]]).all() + + class AppTestSupport(BaseNumpyAppTest): def setup_class(cls): diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py --- a/pypy/module/micronumpy/test/test_ufuncs.py +++ b/pypy/module/micronumpy/test/test_ufuncs.py @@ -190,14 +190,24 @@ for i in range(3): assert c[i] == a[i] - b[i] - def test_floor(self): - from _numpypy import array, floor - - reference = [-2.0, -1.0, 0.0, 1.0, 1.0] - a = array([-1.4, -1.0, 0.0, 1.0, 1.4]) + def test_floorceil(self): + from _numpypy import array, floor, ceil + import math + reference = [-2.0, -2.0, -1.0, 0.0, 1.0, 1.0, 0] + a = array([-1.4, -1.5, -1.0, 0.0, 1.0, 1.4, 0.5]) b = floor(a) for i in range(5): assert b[i] == reference[i] + reference = [-1.0, -1.0, -1.0, 0.0, 1.0, 2.0, 1.0] + a = array([-1.4, -1.5, -1.0, 0.0, 1.0, 1.4, 0.5]) + b = ceil(a) + assert (reference == b).all() + inf = float("inf") + data = [1.5, 2.9999, -1.999, inf] + results = [math.floor(x) for x in data] + assert (floor(data) == results).all() + results = [math.ceil(x) for x in data] + assert (ceil(data) == results).all() def test_copysign(self): from _numpypy import array, copysign @@ -238,7 +248,7 @@ assert b[i] == math.sin(a[i]) a = sin(array([True, False], dtype=bool)) - assert abs(a[0] - sin(1)) < 1e-7 # a[0] will be less precise + assert abs(a[0] - sin(1)) < 1e-7 # a[0] will be less precise assert a[1] == 0.0 def test_cos(self): @@ -259,7 +269,6 @@ for i in range(len(a)): assert b[i] == math.tan(a[i]) - def test_arcsin(self): import math from _numpypy import array, arcsin @@ -283,7 +292,6 @@ for i in range(len(a)): assert b[i] == math.acos(a[i]) - a = array([-10, -1.5, -1.01, 1.01, 1.5, 10, float('nan'), float('inf'), float('-inf')]) b = arccos(a) for f in b: @@ -347,11 +355,20 @@ raises(ValueError, maximum.reduce, []) def test_reduceND(self): - from numpypy import add, arange + from _numpypy import add, arange a = arange(12).reshape(3, 4) assert (add.reduce(a, 0) == [12, 15, 18, 21]).all() assert (add.reduce(a, 1) == [6.0, 22.0, 38.0]).all() + def test_bitwise(self): + from _numpypy import bitwise_and, bitwise_or, arange, array + a = arange(6).reshape(2, 3) + assert (a & 1 == [[0, 1, 0], [1, 0, 1]]).all() + assert (a & 1 == bitwise_and(a, 1)).all() + assert (a | 1 == [[1, 1, 3], [3, 5, 5]]).all() + assert (a | 1 == bitwise_or(a, 1)).all() + raises(TypeError, 'array([1.0]) & 1') + def test_comparisons(self): import operator from _numpypy import equal, not_equal, less, less_equal, greater, greater_equal diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py --- a/pypy/module/micronumpy/test/test_zjit.py +++ b/pypy/module/micronumpy/test/test_zjit.py @@ -217,6 +217,7 @@ # This is the sum of the ops for both loops, however if you remove the # optimization then you end up with 2 float_adds, so we can still be # sure it was optimized correctly. + py.test.skip("too fragile") self.check_resops({'setinteriorfield_raw': 4, 'getfield_gc': 22, 'getarrayitem_gc': 4, 'getarrayitem_gc_pure': 2, 'getfield_gc_pure': 8, diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py --- a/pypy/module/micronumpy/types.py +++ b/pypy/module/micronumpy/types.py @@ -94,6 +94,9 @@ width, storage, i, offset )) + def read_bool(self, storage, width, i, offset): + raise NotImplementedError + def store(self, storage, width, i, offset, box): value = self.unbox(box) libffi.array_setitem(clibffi.cast_type_to_ffitype(self.T), @@ -168,6 +171,7 @@ @simple_binary_op def min(self, v1, v2): return min(v1, v2) + class Bool(BaseType, Primitive): T = lltype.Bool @@ -185,6 +189,11 @@ else: return self.False + + def read_bool(self, storage, width, i, offset): + return libffi.array_getitem(clibffi.cast_type_to_ffitype(self.T), + width, storage, i, offset) + def coerce_subtype(self, space, w_subtype, w_item): # Doesn't return subclasses so it can return the constants. return self._coerce(space, w_item) @@ -253,6 +262,14 @@ assert v == 0 return 0 + @simple_binary_op + def bitwise_and(self, v1, v2): + return v1 & v2 + + @simple_binary_op + def bitwise_or(self, v1, v2): + return v1 | v2 + class Int8(BaseType, Integer): T = rffi.SIGNEDCHAR BoxType = interp_boxes.W_Int8Box @@ -374,6 +391,10 @@ return math.floor(v) @simple_unary_op + def ceil(self, v): + return math.ceil(v) + + @simple_unary_op def exp(self, v): try: return math.exp(v) @@ -436,4 +457,4 @@ class Float64(BaseType, Float): T = rffi.DOUBLE BoxType = interp_boxes.W_Float64Box - format_code = "d" \ No newline at end of file + format_code = "d" diff --git a/pypy/module/pypyjit/test_pypy_c/test_math.py b/pypy/module/pypyjit/test_pypy_c/test_math.py --- a/pypy/module/pypyjit/test_pypy_c/test_math.py +++ b/pypy/module/pypyjit/test_pypy_c/test_math.py @@ -90,3 +90,19 @@ --TICK-- jump(..., descr=) """) + + def test_pow_two(self): + def main(n): + s = 0.123 + while n > 0: + s -= s ** 2 # ID: pow + n -= 1 + return s + log = self.run(main, [500]) + assert abs(log.result - main(500)) < 1e-9 + loop, = log.loops_by_filename(self.filepath) + assert loop.match_by_id("pow", """ + guard_not_invalidated(descr=...) + f38 = float_mul(f30, f30) + f39 = float_sub(f30, f38) + """) diff --git a/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py --- a/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py +++ b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py @@ -18,8 +18,8 @@ from numpypy import array, arange, argmin a = arange(6).reshape((2,3)) assert argmin(a) == 0 - assert (argmin(a, axis=0) == array([0, 0, 0])).all() - assert (argmin(a, axis=1) == array([0, 0])).all() + #assert (argmin(a, axis=0) == array([0, 0, 0])).all() + #assert (argmin(a, axis=1) == array([0, 0])).all() b = arange(6) b[1] = 0 assert argmin(b) == 0 @@ -40,8 +40,8 @@ assert sum([0.5, 1.5])== 2.0 assert sum([[0, 1], [0, 5]]) == 6 # assert sum([0.5, 0.7, 0.2, 1.5], dtype=int32) == 1 - # assert (sum([[0, 1], [0, 5]], axis=0) == array([0, 6])).all() - # assert (sum([[0, 1], [0, 5]], axis=1) == array([1, 5])).all() + assert (sum([[0, 1], [0, 5]], axis=0) == array([0, 6])).all() + assert (sum([[0, 1], [0, 5]], axis=1) == array([1, 5])).all() # If the accumulator is too small, overflow occurs: # assert ones(128, dtype=int8).sum(dtype=int8) == -128 @@ -98,20 +98,22 @@ from numpypy import array, var a = array([[1,2],[3,4]]) assert var(a) == 1.25 - # assert (np.var(a,0) == array([ 1., 1.])).all() - # assert (np.var(a,1) == array([ 0.25, 0.25])).all() + #assert (var(a,0) == array([ 1., 1.])).all() + #assert (var(a,1) == array([ 0.25, 0.25])).all() def test_std(self): from numpypy import array, std a = array([[1, 2], [3, 4]]) assert std(a) == 1.1180339887498949 - # assert (std(a, axis=0) == array([ 1., 1.])).all() - # assert (std(a, axis=1) == array([ 0.5, 0.5]).all() + #assert (std(a, axis=0) == array([ 1., 1.])).all() + #assert (std(a, axis=1) == array([ 0.5, 0.5])).all() def test_mean(self): - from numpypy import array, mean + from numpypy import array, mean, arange assert mean(array(range(5))) == 2.0 assert mean(range(5)) == 2.0 + assert (mean(arange(10).reshape(5, 2), axis=0) == [4, 5]).all() + assert (mean(arange(10).reshape(5, 2), axis=1) == [0.5, 2.5, 4.5, 6.5, 8.5]).all() def test_reshape(self): from numpypy import arange, array, dtype, reshape diff --git a/pypy/module/test_lib_pypy/test_datetime.py b/pypy/module/test_lib_pypy/test_datetime.py new file mode 100644 --- /dev/null +++ b/pypy/module/test_lib_pypy/test_datetime.py @@ -0,0 +1,24 @@ +"""Additional tests for datetime.""" + +import time +import datetime +import os + +def test_utcfromtimestamp(): + """Confirm that utcfromtimestamp and fromtimestamp give consistent results. + + Based on danchr's test script in https://bugs.pypy.org/issue986 + """ + try: + prev_tz = os.environ.get("TZ") + os.environ["TZ"] = "GMT" + for unused in xrange(100): + now = time.time() + delta = (datetime.datetime.utcfromtimestamp(now) - + datetime.datetime.fromtimestamp(now)) + assert delta.days * 86400 + delta.seconds == 0 + finally: + if prev_tz is None: + del os.environ["TZ"] + else: + os.environ["TZ"] = prev_tz diff --git a/pypy/module/test_lib_pypy/test_sqlite3.py b/pypy/module/test_lib_pypy/test_sqlite3.py new file mode 100644 --- /dev/null +++ b/pypy/module/test_lib_pypy/test_sqlite3.py @@ -0,0 +1,10 @@ +"""Tests for _sqlite3.py""" + +def test_list_ddl(): + """From issue996. Mostly just looking for lack of exceptions.""" + from sqlite3.dbapi2 import connect + connection = connect(':memory:') + cursor = connection.cursor() + cursor.execute('CREATE TABLE foo (bar INTEGER)') + result = list(cursor) + assert result == [] diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py --- a/pypy/objspace/std/bytearrayobject.py +++ b/pypy/objspace/std/bytearrayobject.py @@ -624,8 +624,8 @@ if step == 1: assert start >= 0 - assert slicelength >= 0 - del items[start:start+slicelength] + if slicelength > 0: + del items[start:start+slicelength] else: n = len(items) i = start @@ -662,10 +662,11 @@ while i >= lim: items[i] = items[i-delta] i -= 1 - elif start >= 0: + elif delta == 0: + pass + else: + assert start >= 0 # start<0 is only possible with slicelength==0 del items[start:start+delta] - else: - assert delta==0 # start<0 is only possible with slicelength==0 elif len2 != slicelength: # No resize for extended slices raise operationerrfmt(space.w_ValueError, "attempt to " "assign sequence of size %d to extended slice of size %d", diff --git a/pypy/objspace/std/complexobject.py b/pypy/objspace/std/complexobject.py --- a/pypy/objspace/std/complexobject.py +++ b/pypy/objspace/std/complexobject.py @@ -8,6 +8,7 @@ from pypy.rlib.rbigint import rbigint from pypy.rlib.rfloat import ( formatd, DTSF_STR_PRECISION, isinf, isnan, copysign) +from pypy.rlib import jit import math @@ -129,10 +130,10 @@ ir = len * math.sin(phase) return W_ComplexObject(rr, ir) - def pow_int(self, n): - if n > 100 or n < -100: - return self.pow(W_ComplexObject(1.0 * n, 0.0)) - elif n > 0: + def pow_small_int(self, n): + if n >= 0: + if jit.isconstant(n) and n == 2: + return self.mul(self) return self.pow_positive_int(n) else: return w_one.div(self.pow_positive_int(-n)) @@ -217,10 +218,10 @@ def pow__Complex_Complex_ANY(space, w_complex, w_exponent, thirdArg): if not space.is_w(thirdArg, space.w_None): raise OperationError(space.w_ValueError, space.wrap('complex modulo')) - int_exponent = int(w_exponent.realval) try: - if w_exponent.imagval == 0.0 and w_exponent.realval == int_exponent: - w_p = w_complex.pow_int(int_exponent) + r = w_exponent.realval + if w_exponent.imagval == 0.0 and -100.0 <= r <= 100.0 and r == int(r): + w_p = w_complex.pow_small_int(int(r)) else: w_p = w_complex.pow(w_exponent) except ZeroDivisionError: diff --git a/pypy/objspace/std/floatobject.py b/pypy/objspace/std/floatobject.py --- a/pypy/objspace/std/floatobject.py +++ b/pypy/objspace/std/floatobject.py @@ -443,6 +443,8 @@ y = w_float2.floatval # Sort out special cases here instead of relying on pow() + if y == 2.0: # special case for performance: + return W_FloatObject(x * x) # x * x is always correct if y == 0.0: # x**0 is 1, even 0**0 return W_FloatObject(1.0) diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py --- a/pypy/objspace/std/listobject.py +++ b/pypy/objspace/std/listobject.py @@ -804,10 +804,11 @@ while i >= lim: items[i] = items[i-delta] i -= 1 - elif start >= 0: + elif delta == 0: + pass + else: + assert start >= 0 # start<0 is only possible with slicelength==0 del items[start:start+delta] - else: - assert delta==0 # start<0 is only possible with slicelength==0 elif len2 != slicelength: # No resize for extended slices raise operationerrfmt(self.space.w_ValueError, "attempt to " "assign sequence of size %d to extended slice of size %d", @@ -851,8 +852,8 @@ if step == 1: assert start >= 0 - assert slicelength >= 0 - del items[start:start+slicelength] + if slicelength > 0: + del items[start:start+slicelength] else: n = len(items) i = start diff --git a/pypy/objspace/std/test/test_bytearrayobject.py b/pypy/objspace/std/test/test_bytearrayobject.py --- a/pypy/objspace/std/test/test_bytearrayobject.py +++ b/pypy/objspace/std/test/test_bytearrayobject.py @@ -1,5 +1,9 @@ +from pypy import conftest class AppTestBytesArray: + def setup_class(cls): + cls.w_runappdirect = cls.space.wrap(conftest.option.runappdirect) + def test_basics(self): b = bytearray() assert type(b) is bytearray @@ -439,3 +443,15 @@ def test_reduce(self): assert bytearray('caf\xe9').__reduce__() == ( bytearray, (u'caf\xe9', 'latin-1'), None) + + def test_setitem_slice_performance(self): + # because of a complexity bug, this used to take forever on a + # translated pypy. On CPython2.6 -A, it takes around 8 seconds. + if self.runappdirect: + count = 16*1024*1024 + else: + count = 1024 + b = bytearray(count) + for i in range(count): + b[i:i+1] = 'y' + assert str(b) == 'y' * count diff --git a/pypy/objspace/std/test/test_complexobject.py b/pypy/objspace/std/test/test_complexobject.py --- a/pypy/objspace/std/test/test_complexobject.py +++ b/pypy/objspace/std/test/test_complexobject.py @@ -71,7 +71,7 @@ assert _powu((0.0,1.0),2) == (-1.0,0.0) def _powi((r1, i1), n): - w_res = W_ComplexObject(r1, i1).pow_int(n) + w_res = W_ComplexObject(r1, i1).pow_small_int(n) return w_res.realval, w_res.imagval assert _powi((0.0,2.0),0) == (1.0,0.0) assert _powi((0.0,0.0),2) == (0.0,0.0) @@ -213,6 +213,7 @@ assert a ** 105 == a ** 105 assert a ** -105 == a ** -105 assert a ** -30 == a ** -30 + assert a ** 2 == a * a assert 0.0j ** 0 == 1 diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py --- a/pypy/objspace/std/test/test_listobject.py +++ b/pypy/objspace/std/test/test_listobject.py @@ -397,6 +397,7 @@ on_cpython = (option.runappdirect and not hasattr(sys, 'pypy_translation_info')) cls.w_on_cpython = cls.space.wrap(on_cpython) + cls.w_runappdirect = cls.space.wrap(option.runappdirect) def test_getstrategyfromlist_w(self): l0 = ["a", "2", "a", True] @@ -898,6 +899,18 @@ l[::-1] = l assert l == [6,5,4,3,2,1] + def test_setitem_slice_performance(self): + # because of a complexity bug, this used to take forever on a + # translated pypy. On CPython2.6 -A, it takes around 5 seconds. + if self.runappdirect: + count = 16*1024*1024 + else: + count = 1024 + b = [None] * count + for i in range(count): + b[i:i+1] = ['y'] + assert b == ['y'] * count + def test_recursive_repr(self): l = [] assert repr(l) == '[]' diff --git a/pypy/rlib/rweakref.py b/pypy/rlib/rweakref.py --- a/pypy/rlib/rweakref.py +++ b/pypy/rlib/rweakref.py @@ -1,11 +1,14 @@ """ -Weakref support in RPython. Supports ref() without callbacks, +Weakref support in RPython. Basic regular weakrefs without callbacks +are supported. This file contains the following additions: a form of WeakKeyDictionary, and a limited version of WeakValueDictionary. LLType only for now! """ import weakref +ref = weakref.ref # basic regular weakrefs are supported in RPython + class RWeakValueDictionary(object): """A dictionary containing weak values.""" diff --git a/pypy/tool/test/test_version.py b/pypy/tool/test/test_version.py --- a/pypy/tool/test/test_version.py +++ b/pypy/tool/test/test_version.py @@ -1,6 +1,22 @@ import os, sys import py -from pypy.tool.version import get_repo_version_info +from pypy.tool.version import get_repo_version_info, _get_hg_archive_version + +def test_hg_archival_version(tmpdir): + def version_for(name, **kw): + path = tmpdir.join(name) + path.write('\n'.join('%s: %s' % x for x in kw.items())) + return _get_hg_archive_version(str(path)) + + assert version_for('release', + tag='release-123', + node='000', + ) == ('PyPy', 'release-123', '000') + assert version_for('somebranch', + node='000', + branch='something', + ) == ('PyPy', 'something', '000') + def test_get_repo_version_info(): assert get_repo_version_info(None) diff --git a/pypy/tool/version.py b/pypy/tool/version.py --- a/pypy/tool/version.py +++ b/pypy/tool/version.py @@ -3,111 +3,139 @@ from subprocess import Popen, PIPE import pypy pypydir = os.path.dirname(os.path.abspath(pypy.__file__)) +pypyroot = os.path.dirname(pypydir) +default_retval = 'PyPy', '?', '?' + +def maywarn(err, repo_type='Mercurial'): + if not err: + return + + from pypy.tool.ansi_print import ansi_log + log = py.log.Producer("version") + py.log.setconsumer("version", ansi_log) + log.WARNING('Errors getting %s information: %s' % (repo_type, err)) def get_repo_version_info(hgexe=None): '''Obtain version information by invoking the 'hg' or 'git' commands.''' - # TODO: support extracting from .hg_archival.txt - - default_retval = 'PyPy', '?', '?' - pypyroot = os.path.abspath(os.path.join(pypydir, '..')) - - def maywarn(err, repo_type='Mercurial'): - if not err: - return - - from pypy.tool.ansi_print import ansi_log - log = py.log.Producer("version") - py.log.setconsumer("version", ansi_log) - log.WARNING('Errors getting %s information: %s' % (repo_type, err)) # Try to see if we can get info from Git if hgexe is not specified. if not hgexe: if os.path.isdir(os.path.join(pypyroot, '.git')): - gitexe = py.path.local.sysfind('git') - if gitexe: - try: - p = Popen( - [str(gitexe), 'rev-parse', 'HEAD'], - stdout=PIPE, stderr=PIPE, cwd=pypyroot - ) - except OSError, e: - maywarn(e, 'Git') - return default_retval - if p.wait() != 0: - maywarn(p.stderr.read(), 'Git') - return default_retval - revision_id = p.stdout.read().strip()[:12] - p = Popen( - [str(gitexe), 'describe', '--tags', '--exact-match'], - stdout=PIPE, stderr=PIPE, cwd=pypyroot - ) - if p.wait() != 0: - p = Popen( - [str(gitexe), 'branch'], stdout=PIPE, stderr=PIPE, - cwd=pypyroot - ) - if p.wait() != 0: - maywarn(p.stderr.read(), 'Git') - return 'PyPy', '?', revision_id - branch = '?' - for line in p.stdout.read().strip().split('\n'): - if line.startswith('* '): - branch = line[1:].strip() - if branch == '(no branch)': - branch = '?' - break - return 'PyPy', branch, revision_id - return 'PyPy', p.stdout.read().strip(), revision_id + return _get_git_version() # Fallback to trying Mercurial. if hgexe is None: hgexe = py.path.local.sysfind('hg') - if not os.path.isdir(os.path.join(pypyroot, '.hg')): + if os.path.isfile(os.path.join(pypyroot, '.hg_archival.txt')): + return _get_hg_archive_version(os.path.join(pypyroot, '.hg_archival.txt')) + elif not os.path.isdir(os.path.join(pypyroot, '.hg')): maywarn('Not running from a Mercurial repository!') return default_retval elif not hgexe: maywarn('Cannot find Mercurial command!') return default_retval else: - env = dict(os.environ) - # get Mercurial into scripting mode - env['HGPLAIN'] = '1' - # disable user configuration, extensions, etc. - env['HGRCPATH'] = os.devnull + return _get_hg_version(hgexe) - try: - p = Popen([str(hgexe), 'version', '-q'], - stdout=PIPE, stderr=PIPE, env=env) - except OSError, e: - maywarn(e) - return default_retval - if not p.stdout.read().startswith('Mercurial Distributed SCM'): - maywarn('command does not identify itself as Mercurial') - return default_retval +def _get_hg_version(hgexe): + env = dict(os.environ) + # get Mercurial into scripting mode + env['HGPLAIN'] = '1' + # disable user configuration, extensions, etc. + env['HGRCPATH'] = os.devnull - p = Popen([str(hgexe), 'id', '-i', pypyroot], + try: + p = Popen([str(hgexe), 'version', '-q'], stdout=PIPE, stderr=PIPE, env=env) - hgid = p.stdout.read().strip() + except OSError, e: + maywarn(e) + return default_retval + + if not p.stdout.read().startswith('Mercurial Distributed SCM'): + maywarn('command does not identify itself as Mercurial') + return default_retval + + p = Popen([str(hgexe), 'id', '-i', pypyroot], + stdout=PIPE, stderr=PIPE, env=env) + hgid = p.stdout.read().strip() + maywarn(p.stderr.read()) + if p.wait() != 0: + hgid = '?' + + p = Popen([str(hgexe), 'id', '-t', pypyroot], + stdout=PIPE, stderr=PIPE, env=env) + hgtags = [t for t in p.stdout.read().strip().split() if t != 'tip'] + maywarn(p.stderr.read()) + if p.wait() != 0: + hgtags = ['?'] + + if hgtags: + return 'PyPy', hgtags[0], hgid + else: + # use the branch instead + p = Popen([str(hgexe), 'id', '-b', pypyroot], + stdout=PIPE, stderr=PIPE, env=env) + hgbranch = p.stdout.read().strip() maywarn(p.stderr.read()) + + return 'PyPy', hgbranch, hgid + + +def _get_hg_archive_version(path): + fp = open(path) + try: + data = dict(x.split(': ', 1) for x in fp.read().splitlines()) + finally: + fp.close() + if 'tag' in data: + return 'PyPy', data['tag'], data['node'] + else: + return 'PyPy', data['branch'], data['node'] + + +def _get_git_version(): + #XXX: this function is a untested hack, + # so the git mirror tav made will work + gitexe = py.path.local.sysfind('git') + if not gitexe: + return default_retval + + try: + p = Popen( + [str(gitexe), 'rev-parse', 'HEAD'], + stdout=PIPE, stderr=PIPE, cwd=pypyroot + ) + except OSError, e: + maywarn(e, 'Git') + return default_retval + if p.wait() != 0: + maywarn(p.stderr.read(), 'Git') + return default_retval + revision_id = p.stdout.read().strip()[:12] + p = Popen( + [str(gitexe), 'describe', '--tags', '--exact-match'], + stdout=PIPE, stderr=PIPE, cwd=pypyroot + ) + if p.wait() != 0: + p = Popen( + [str(gitexe), 'branch'], stdout=PIPE, stderr=PIPE, + cwd=pypyroot + ) if p.wait() != 0: - hgid = '?' + maywarn(p.stderr.read(), 'Git') + return 'PyPy', '?', revision_id + branch = '?' + for line in p.stdout.read().strip().split('\n'): + if line.startswith('* '): + branch = line[1:].strip() + if branch == '(no branch)': + branch = '?' + break + return 'PyPy', branch, revision_id + return 'PyPy', p.stdout.read().strip(), revision_id - p = Popen([str(hgexe), 'id', '-t', pypyroot], - stdout=PIPE, stderr=PIPE, env=env) - hgtags = [t for t in p.stdout.read().strip().split() if t != 'tip'] - maywarn(p.stderr.read()) - if p.wait() != 0: - hgtags = ['?'] - if hgtags: - return 'PyPy', hgtags[0], hgid - else: - # use the branch instead - p = Popen([str(hgexe), 'id', '-b', pypyroot], - stdout=PIPE, stderr=PIPE, env=env) - hgbranch = p.stdout.read().strip() - maywarn(p.stderr.read()) - - return 'PyPy', hgbranch, hgid +if __name__ == '__main__': + print get_repo_version_info() diff --git a/pypy/translator/c/src/profiling.c b/pypy/translator/c/src/profiling.c --- a/pypy/translator/c/src/profiling.c +++ b/pypy/translator/c/src/profiling.c @@ -29,33 +29,33 @@ profiling_setup = 0; } } - -#elif defined(_WIN32) -#include <windows.h> - -DWORD_PTR base_affinity_mask; -int profiling_setup = 0; - -void pypy_setup_profiling() { - if (!profiling_setup) { - DWORD_PTR affinity_mask, system_affinity_mask; - GetProcessAffinityMask(GetCurrentProcess(), - &base_affinity_mask, &system_affinity_mask); - affinity_mask = 1; - /* Pick one cpu allowed by the system */ - if (system_affinity_mask) - while ((affinity_mask & system_affinity_mask) == 0) - affinity_mask <<= 1; - SetProcessAffinityMask(GetCurrentProcess(), affinity_mask); - profiling_setup = 1; - } -} - -void pypy_teardown_profiling() { - if (profiling_setup) { - SetProcessAffinityMask(GetCurrentProcess(), base_affinity_mask); - profiling_setup = 0; - } + +#elif defined(_WIN32) +#include <windows.h> + +DWORD_PTR base_affinity_mask; +int profiling_setup = 0; + +void pypy_setup_profiling() { + if (!profiling_setup) { + DWORD_PTR affinity_mask, system_affinity_mask; + GetProcessAffinityMask(GetCurrentProcess(), + &base_affinity_mask, &system_affinity_mask); + affinity_mask = 1; + /* Pick one cpu allowed by the system */ + if (system_affinity_mask) + while ((affinity_mask & system_affinity_mask) == 0) + affinity_mask <<= 1; + SetProcessAffinityMask(GetCurrentProcess(), affinity_mask); + profiling_setup = 1; + } +} + +void pypy_teardown_profiling() { + if (profiling_setup) { + SetProcessAffinityMask(GetCurrentProcess(), base_affinity_mask); + profiling_setup = 0; + } } #else diff --git a/pypy/translator/goal/translate.py b/pypy/translator/goal/translate.py --- a/pypy/translator/goal/translate.py +++ b/pypy/translator/goal/translate.py @@ -293,17 +293,18 @@ drv.exe_name = targetspec_dic['__name__'] + '-%(backend)s' # Double check to ensure we are not overwriting the current interpreter - try: - this_exe = py.path.local(sys.executable).new(ext='') - exe_name = drv.compute_exe_name() - samefile = this_exe.samefile(exe_name) - assert not samefile, ( - 'Output file %s is the currently running ' - 'interpreter (use --output=...)'% exe_name) - except EnvironmentError: - pass + goals = translateconfig.goals + if not goals or 'compile' in goals: + try: + this_exe = py.path.local(sys.executable).new(ext='') + exe_name = drv.compute_exe_name() + samefile = this_exe.samefile(exe_name) + assert not samefile, ( + 'Output file %s is the currently running ' + 'interpreter (use --output=...)'% exe_name) + except EnvironmentError: + pass - goals = translateconfig.goals try: drv.proceed(goals) finally: From noreply at buildbot.pypy.org Tue Jan 24 17:45:13 2012 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Tue, 24 Jan 2012 17:45:13 +0100 (CET) Subject: [pypy-commit] pypy default: Make numready less horrible. Message-ID: <20120124164513.A19BA82CFD@wyvern.cs.uni-duesseldorf.de> Author: Alex Gaynor <alex.gaynor at gmail.com> Branch: Changeset: r51731:578b2ef8d345 Date: 2012-01-24 11:44 -0500 http://bitbucket.org/pypy/pypy/changeset/578b2ef8d345/ Log: Make numready less horrible. diff --git a/pypy/module/micronumpy/tool/numready.py b/pypy/module/micronumpy/tool/numready.py deleted file mode 100644 --- a/pypy/module/micronumpy/tool/numready.py +++ /dev/null @@ -1,226 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -""" -This should be run under PyPy. -""" - -import platform -import subprocess -import sys -import tempfile -import webbrowser -from collections import OrderedDict - -import jinja2 - - -MODULE_SEARCH_CODE = ''' -import types -import {modname} as numpy - -for name in dir(numpy): - if name.startswith("_"): - continue - obj = getattr(numpy, name) - kind = "{kinds[UNKNOWN]}" - if isinstance(obj, types.TypeType): - kind = "{kinds[TYPE]}" - print kind, ":", name -''' - -ATTR_SEARCH_CODE = ''' -import types -import {modname} as numpy - -obj = getattr(numpy, "{name}") -for name in dir(obj): - #if name.startswith("_"): - # continue - sub_obj = getattr(obj, name) - kind = "{kinds[UNKNOWN]}" - if isinstance(sub_obj, types.TypeType): - kind = "{kinds[TYPE]}" - print kind, ":", name -''' - -KINDS = { - "UNKNOWN": "U", - "TYPE": "T", -} - -PAGE_TEMPLATE = u""" -<!DOCTYPE html> -<html lang="en"> - <head> - <title>NumPyPy Status</title> - <meta http-equiv="content-type" content="text/html; charset=utf-8"> - <style type="text/css"> - body { - font-family: 'Consolas', 'Bitstream Vera Sans Mono', monospace; - } - h1 { - text-align: center; - } - h3 { - text-align: center; - } - table { - border: 8px solid #DFDECB; - margin: 30px auto; - font-size: 12px; - } - table th { - text-align: left; - } - table td { - padding: 4px 10px; - text-align: center; - } - .exists { - background-color: #337792; - color: white; - border: 1px solid #234F61; - } - </style> - </head> - <body> - <h1>NumPyPy Status</h1> - <h3>Overall: {{ msg }}</h3> - <table> - <thead> - <tr> - <th></th> - <th>PyPy</th> - <th></th> - <th>PyPy</th> - <th></th> - <th>PyPy</th> - <th></th> - <th>PyPy</th> - <th></th> - <th>PyPy</th> - </tr> - </thead> - <tbody> - {% for chunk in all_items %} - <tr> - {% for item in chunk %} - <th class='{{ item.cls }}'>{{ item.name }}</th> - <td class='{{ item.cls }}'>{{ item.symbol }}</td> - {% endfor %} - </tr> - {% endfor %} - </tbody> - </table> - </body> -</html> -""" - -class SearchableSet(object): - def __init__(self, items=()): - self._items = {} - for item in items: - self.add(item) - - def __iter__(self): - return iter(self._items) - - def __contains__(self, other): - return other in self._items - - def __getitem__(self, idx): - return self._items[idx] - - def add(self, item): - self._items[item] = item - - def __len__(self): - return len(self._items) - -class Item(object): - def __init__(self, name, kind, subitems=None): - self.name = name - self.kind = kind - self.subitems = subitems - - def __hash__(self): - return hash(self.name) - - def __eq__(self, other): - if isinstance(other, str): - return self.name == other - return self.name == other.name - - -class ItemStatus(object): - def __init__(self, name, pypy_exists): - self.name = name - self.cls = 'exists' if pypy_exists else '' - self.symbol = u"✔" if pypy_exists else u'✖' - - def __lt__(self, other): - return self.name < other.name - -def find_numpy_attrs(python, modname, name): - lines = subprocess.check_output( - [python, "-c", ATTR_SEARCH_CODE.format(modname=modname, kinds=KINDS, name=name)] - ).splitlines() - items = SearchableSet() - for line in lines: - kind, name = line.split(" : ", 1) - items.add(Item(name, kind)) - return items - -def find_numpy_items(python, modname="numpy"): - lines = subprocess.check_output( - [python, "-c", MODULE_SEARCH_CODE.format(modname=modname, kinds=KINDS)] - ).splitlines() - items = SearchableSet() - for line in lines: - kind, name = line.split(" : ", 1) - subitems = None - if kind == KINDS["TYPE"]: - if name in ['ndarray', 'dtype']: - subitems = find_numpy_attrs(python, modname, name) - items.add(Item(name, kind, subitems)) - return items - -def split(lst): - SPLIT = 5 - lgt = len(lst) // SPLIT + 1 - l = [[] for i in range(lgt)] - for i in range(lgt): - for k in range(SPLIT): - if k * lgt + i < len(lst): - l[i].append(lst[k * lgt + i]) - return l - -def main(argv): - cpy_items = find_numpy_items("/usr/bin/python") - pypy_items = find_numpy_items(argv[1], "numpypy") - all_items = [] - - msg = '%d/%d names, %d/%d ndarray attributes, %d/%d dtype attributes' % ( - len(pypy_items), len(cpy_items), len(pypy_items['ndarray'].subitems), - len(cpy_items['ndarray'].subitems), len(pypy_items['dtype'].subitems), - len(cpy_items['dtype'].subitems)) - for item in cpy_items: - pypy_exists = item in pypy_items - if item.subitems: - for sub in item.subitems: - all_items.append( - ItemStatus(item.name + "." + sub.name, pypy_exists=pypy_exists and pypy_items[item].subitems and sub in pypy_items[item].subitems) - ) - all_items.append(ItemStatus(item.name, pypy_exists=item in pypy_items)) - html = jinja2.Template(PAGE_TEMPLATE).render(all_items=split(sorted(all_items)), msg=msg) - if len(argv) > 2: - with open(argv[2], 'w') as f: - f.write(html.encode("utf-8")) - else: - with tempfile.NamedTemporaryFile(delete=False) as f: - f.write(html.encode("utf-8")) - print "Saved in: %s" % f.name - -if __name__ == '__main__': - main(sys.argv) diff --git a/pypy/module/micronumpy/tool/numready/__init__.py b/pypy/module/micronumpy/tool/numready/__init__.py new file mode 100644 diff --git a/pypy/module/micronumpy/tool/numready/__main__.py b/pypy/module/micronumpy/tool/numready/__main__.py new file mode 100644 --- /dev/null +++ b/pypy/module/micronumpy/tool/numready/__main__.py @@ -0,0 +1,6 @@ +import sys + +from .main import main + + +main(sys.argv) diff --git a/pypy/module/micronumpy/tool/numready/kinds.py b/pypy/module/micronumpy/tool/numready/kinds.py new file mode 100644 --- /dev/null +++ b/pypy/module/micronumpy/tool/numready/kinds.py @@ -0,0 +1,4 @@ +KINDS = { + "UNKNOWN": "U", + "TYPE": "T", +} diff --git a/pypy/module/micronumpy/tool/numready/main.py b/pypy/module/micronumpy/tool/numready/main.py new file mode 100644 --- /dev/null +++ b/pypy/module/micronumpy/tool/numready/main.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" +This should be run under PyPy. +""" + +import os +import platform +import subprocess +import tempfile +import webbrowser +from collections import OrderedDict + +import jinja2 + +from .kinds import KINDS + + +class SearchableSet(object): + def __init__(self, items=()): + self._items = {} + for item in items: + self.add(item) + + def __iter__(self): + return iter(self._items) + + def __contains__(self, other): + return other in self._items + + def __getitem__(self, idx): + return self._items[idx] + + def add(self, item): + self._items[item] = item + + def __len__(self): + return len(self._items) + +class Item(object): + def __init__(self, name, kind, subitems=None): + self.name = name + self.kind = kind + self.subitems = subitems + + def __hash__(self): + return hash(self.name) + + def __eq__(self, other): + if isinstance(other, str): + return self.name == other + return self.name == other.name + + +class ItemStatus(object): + def __init__(self, name, pypy_exists): + self.name = name + self.cls = 'exists' if pypy_exists else '' + self.symbol = u"✔" if pypy_exists else u'✖' + + def __lt__(self, other): + return self.name < other.name + +def find_numpy_items(python, modname="numpy", attr=None): + args = [ + python, os.path.join(os.path.dirname(__file__), "search.py"), modname + ] + if attr is not None: + args.append(attr) + lines = subprocess.check_output(args).splitlines() + items = SearchableSet() + for line in lines: + kind, name = line.split(" : ", 1) + subitems = None + if kind == KINDS["TYPE"]: + if name in ['ndarray', 'dtype']: + subitems = find_numpy_items(python, modname, name) + items.add(Item(name, kind, subitems)) + return items + +def split(lst): + SPLIT = 5 + lgt = len(lst) // SPLIT + 1 + l = [[] for i in range(lgt)] + for i in range(lgt): + for k in range(SPLIT): + if k * lgt + i < len(lst): + l[i].append(lst[k * lgt + i]) + return l + +def main(argv): + cpy_items = find_numpy_items("/usr/bin/python") + pypy_items = find_numpy_items(argv[1], "numpypy") + all_items = [] + + msg = '%d/%d names, %d/%d ndarray attributes, %d/%d dtype attributes' % ( + len(pypy_items), len(cpy_items), len(pypy_items['ndarray'].subitems), + len(cpy_items['ndarray'].subitems), len(pypy_items['dtype'].subitems), + len(cpy_items['dtype'].subitems)) + for item in cpy_items: + pypy_exists = item in pypy_items + if item.subitems: + for sub in item.subitems: + all_items.append( + ItemStatus(item.name + "." + sub.name, pypy_exists=pypy_exists and pypy_items[item].subitems and sub in pypy_items[item].subitems) + ) + all_items.append(ItemStatus(item.name, pypy_exists=item in pypy_items)) + env = jinja2.Environment( + loader=jinja2.FileSystemLoader(os.path.dirname(__file__)) + ) + html = env.get_template("page.html").render(all_items=split(sorted(all_items)), msg=msg) + if len(argv) > 2: + with open(argv[2], 'w') as f: + f.write(html.encode("utf-8")) + else: + with tempfile.NamedTemporaryFile(delete=False) as f: + f.write(html.encode("utf-8")) + print "Saved in: %s" % f.name diff --git a/pypy/module/micronumpy/tool/numready/page.html b/pypy/module/micronumpy/tool/numready/page.html new file mode 100644 --- /dev/null +++ b/pypy/module/micronumpy/tool/numready/page.html @@ -0,0 +1,65 @@ +<!DOCTYPE html> +<html lang="en"> + <head> + <title>NumPyPy Status</title> + <meta http-equiv="content-type" content="text/html; charset=utf-8"> + <style type="text/css"> + body { + font-family: 'Consolas', 'Bitstream Vera Sans Mono', monospace; + } + h1 { + text-align: center; + } + h3 { + text-align: center; + } + table { + border: 8px solid #DFDECB; + margin: 30px auto; + font-size: 12px; + } + table th { + text-align: left; + } + table td { + padding: 4px 10px; + text-align: center; + } + .exists { + background-color: #337792; + color: white; + border: 1px solid #234F61; + } + </style> + </head> + <body> + <h1>NumPyPy Status</h1> + <h3>Overall: {{ msg }}</h3> + <table> + <thead> + <tr> + <th></th> + <th>PyPy</th> + <th></th> + <th>PyPy</th> + <th></th> + <th>PyPy</th> + <th></th> + <th>PyPy</th> + <th></th> + <th>PyPy</th> + </tr> + </thead> + <tbody> + {% for chunk in all_items %} + <tr> + {% for item in chunk %} + <th class='{{ item.cls }}'>{{ item.name }}</th> + <td class='{{ item.cls }}'>{{ item.symbol }}</td> + {% endfor %} + </tr> + {% endfor %} + </tbody> + </table> + </body> +</html> diff --git a/pypy/module/micronumpy/tool/numready/search.py b/pypy/module/micronumpy/tool/numready/search.py new file mode 100644 --- /dev/null +++ b/pypy/module/micronumpy/tool/numready/search.py @@ -0,0 +1,33 @@ +import sys +import types + +# Evil implicit relative import. +from kinds import KINDS + + +def main(argv): + if len(argv) == 2: + [_, modname] = argv + attr = None + elif len(argv) == 3: + [_, modname, attr] = argv + else: + sys.exit("Wrong number of args") + __import__(modname) + obj = sys.modules[modname] + + if attr is not None: + obj = getattr(obj, attr) + + for name in dir(obj): + if attr is None and name.startswith("_"): + continue + subobj = getattr(obj, name) + if isinstance(subobj, types.TypeType): + kind = KINDS["TYPE"] + else: + kind = KINDS["UNKNOWN"] + print kind, ":", name + +if __name__ == "__main__": + main(sys.argv) \ No newline at end of file From noreply at buildbot.pypy.org Tue Jan 24 18:28:28 2012 From: noreply at buildbot.pypy.org (antocuni) Date: Tue, 24 Jan 2012 18:28:28 +0100 (CET) Subject: [pypy-commit] pypy py3k: hardcode the sys.std{in, out, err} encoding to ascii when running the tests. Two reasons: Message-ID: <20120124172828.A7F6B82CFD@wyvern.cs.uni-duesseldorf.de> Author: Antonio Cuni <anto.cuni at gmail.com> Branch: py3k Changeset: r51732:698aff2e6ca7 Date: 2012-01-24 16:36 +0100 http://bitbucket.org/pypy/pypy/changeset/698aff2e6ca7/ Log: hardcode the sys.std{in,out,err} encoding to ascii when running the tests. Two reasons: 1. if you don't specificy the encoding, it ends up using locate.getpreferredlocale() to guess one. I don't think it's a good idea that the behavior of the tests depend on the locale settings 2. importing locale is very slow on py.py. By using the explicit encoding, initializing the space takes ~3 seconds instead of ~10 on my machine diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py --- a/pypy/module/sys/__init__.py +++ b/pypy/module/sys/__init__.py @@ -116,11 +116,14 @@ # Install standard streams for tests that don't call app_main space.appexec([], """(): import sys, io - sys.stdin = sys.__stdin__ = io.open(0, "r", closefd=False) + sys.stdin = sys.__stdin__ = io.open(0, "r", encoding="ascii", + closefd=False) sys.stdin.buffer.raw.name = "<stdin>" - sys.stdout = sys.__stdout__ = io.open(1, "w", closefd=False) + sys.stdout = sys.__stdout__ = io.open(1, "w", encoding="ascii", + closefd=False) sys.stdout.buffer.raw.name = "<stdout>" - sys.stderr = sys.__stderr__ = io.open(2, "w", closefd=False) + sys.stderr = sys.__stderr__ = io.open(2, "w", encoding="ascii", + closefd=False) sys.stderr.buffer.raw.name = "<stderr>" """) From noreply at buildbot.pypy.org Tue Jan 24 18:28:29 2012 From: noreply at buildbot.pypy.org (antocuni) Date: Tue, 24 Jan 2012 18:28:29 +0100 (CET) Subject: [pypy-commit] pypy py3k: the repr of builtin types changed in python3, so we can no longer rely on the host python to know the exepected repr of some expressions Message-ID: <20120124172829.DC6D482CFD@wyvern.cs.uni-duesseldorf.de> Author: Antonio Cuni <anto.cuni at gmail.com> Branch: py3k Changeset: r51733:0b17ddaa8c4f Date: 2012-01-24 18:25 +0100 http://bitbucket.org/pypy/pypy/changeset/0b17ddaa8c4f/ Log: the repr of builtin types changed in python3, so we can no longer rely on the host python to know the exepected repr of some expressions diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py --- a/pypy/interpreter/astcompiler/test/test_compiler.py +++ b/pypy/interpreter/astcompiler/test/test_compiler.py @@ -53,6 +53,20 @@ code.exec_code(space, w_dict, w_dict) return w_dict + # on Python3 some reprs are different than Python2. Here is a collection + # of how the repr should be on on Python3 for some objects + PY3_REPR = { + int: "<class 'int'>", + float: "<class 'float'>", + } + + def get_py3_repr(self, val): + try: + return self.PY3_REPR.get(val, repr(val)) + except TypeError: + # e.g., for unhashable types + return repr(val) + def check(self, w_dict, evalexpr, expected): # for now, we compile evalexpr with CPython's compiler but run # it with our own interpreter to extract the data from w_dict @@ -61,8 +75,9 @@ pyco_expr = PyCode._from_code(space, co_expr) w_res = pyco_expr.exec_host_bytecode(w_dict, w_dict) res = space.str_w(space.repr(w_res)) + expected_repr = self.get_py3_repr(expected) if not isinstance(expected, float): - assert res == repr(expected) + assert res == expected_repr else: # Float representation can vary a bit between interpreter # versions, compare the numbers instead. From noreply at buildbot.pypy.org Tue Jan 24 18:28:31 2012 From: noreply at buildbot.pypy.org (antocuni) Date: Tue, 24 Jan 2012 18:28:31 +0100 (CET) Subject: [pypy-commit] pypy py3k: kill code to support tuple unpacking inside function arguments, which is no longer supported in python3 Message-ID: <20120124172831.1CFFC82CFD@wyvern.cs.uni-duesseldorf.de> Author: Antonio Cuni <anto.cuni at gmail.com> Branch: py3k Changeset: r51734:0f9987682c05 Date: 2012-01-24 18:26 +0100 http://bitbucket.org/pypy/pypy/changeset/0f9987682c05/ Log: kill code to support tuple unpacking inside function arguments, which is no longer supported in python3 diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py --- a/pypy/interpreter/astcompiler/codegen.py +++ b/pypy/interpreter/astcompiler/codegen.py @@ -1126,14 +1126,6 @@ class AbstractFunctionCodeGenerator(PythonCodeGenerator): - def _handle_nested_args(self, args): - for i in range(len(args)): - arg = args[i] - if isinstance(arg, ast.Tuple): - self.update_position(arg.lineno) - self.name_op(".%d" % (i,), ast.Load) - arg.walkabout(self) - def _get_code_flags(self): scope = self.scope assert isinstance(scope, symtable.FunctionScope) @@ -1171,10 +1163,8 @@ args = func.args assert isinstance(args, ast.arguments) if args.args: - self._handle_nested_args(args.args) self.argcount = len(args.args) if args.kwonlyargs: - self._handle_nested_args(args.kwonlyargs) self.kwonlyargcount = len(args.kwonlyargs) if func.body: for i in range(start, len(func.body)): @@ -1188,7 +1178,6 @@ args = lam.args assert isinstance(args, ast.arguments) if args.args: - self._handle_nested_args(args.args) self.argcount = len(args.args) # Prevent a string from being the first constant and thus a docstring. self.add_const(self.space.w_None) diff --git a/pypy/interpreter/astcompiler/symtable.py b/pypy/interpreter/astcompiler/symtable.py --- a/pypy/interpreter/astcompiler/symtable.py +++ b/pypy/interpreter/astcompiler/symtable.py @@ -508,28 +508,14 @@ if arguments.kwarg: self.note_symbol(arguments.kwarg, SYM_PARAM) scope.note_keywords_arg(arguments.kwarg) - if arguments.args: - self._handle_nested_params(arguments.args) def _handle_params(self, params, is_toplevel): for i in range(len(params)): arg = params[i] if isinstance(arg, ast.Name): self.note_symbol(arg.id, SYM_PARAM) - elif isinstance(arg, ast.Tuple): - # Tuple unpacking in the argument list. Add a secret variable - # name to recieve the tuple with. - if is_toplevel: - self.implicit_arg(i) else: raise AssertionError("unknown parameter type") - if not is_toplevel: - self._handle_nested_params(params) - - def _handle_nested_params(self, params): - for param in params: - if isinstance(param, ast.Tuple): - self._handle_params(param.elts, False) def visit_Name(self, name): if name.ctx == ast.Load: diff --git a/pypy/interpreter/astcompiler/test/test_symtable.py b/pypy/interpreter/astcompiler/test/test_symtable.py --- a/pypy/interpreter/astcompiler/test/test_symtable.py +++ b/pypy/interpreter/astcompiler/test/test_symtable.py @@ -124,12 +124,6 @@ assert scp.has_keywords_arg assert not scp.has_variable_arg assert scp.lookup("x") == symtable.SCOPE_LOCAL - scp = self.func_scope("def f((x, y), a): pass") - for name in ("x", "y", "a"): - assert scp.lookup(name) == symtable.SCOPE_LOCAL - scp = self.func_scope("def f(((a, b), c)): pass") - for name in ("a", "b", "c"): - assert scp.lookup(name) == symtable.SCOPE_LOCAL def test_arguments_kwonly(self): scp = self.func_scope("def f(a, *, b): pass") From noreply at buildbot.pypy.org Tue Jan 24 19:21:31 2012 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 24 Jan 2012 19:21:31 +0100 (CET) Subject: [pypy-commit] pypy numpy-back-to-applevel: implement isnan/isinf Message-ID: <20120124182131.19EC482CFD@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: numpy-back-to-applevel Changeset: r51735:200be74121ce Date: 2012-01-24 20:21 +0200 http://bitbucket.org/pypy/pypy/changeset/200be74121ce/ Log: implement isnan/isinf diff --git a/lib_pypy/numpypy/core/arrayprint.py b/lib_pypy/numpypy/core/arrayprint.py --- a/lib_pypy/numpypy/core/arrayprint.py +++ b/lib_pypy/numpypy/core/arrayprint.py @@ -14,9 +14,9 @@ import sys import _numpypy as _nt -from _numpypy import maximum, minimum, absolute, not_equal #, isnan, isinf +from _numpypy import maximum, minimum, absolute, not_equal, isinf, isnan #from _numpypy import format_longfloat, datetime_as_string, datetime_data, isna -from fromnumeric import ravel +from .fromnumeric import ravel def product(x, y): return x*y diff --git a/lib_pypy/numpypy/core/fromnumeric.py b/lib_pypy/numpypy/core/fromnumeric.py --- a/lib_pypy/numpypy/core/fromnumeric.py +++ b/lib_pypy/numpypy/core/fromnumeric.py @@ -30,7 +30,7 @@ 'rank', 'size', 'around', 'round_', 'mean', 'std', 'var', 'squeeze', 'amax', 'amin', ] - + def take(a, indices, axis=None, out=None, mode='raise'): """ Take elements from an array along an axis. diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py --- a/pypy/module/micronumpy/__init__.py +++ b/pypy/module/micronumpy/__init__.py @@ -93,6 +93,8 @@ ("tan", "tan"), ('bitwise_and', 'bitwise_and'), ('bitwise_or', 'bitwise_or'), + ('isnan', 'isnan'), + ('isinf', 'isinf'), ]: interpleveldefs[exposed] = "interp_ufuncs.get(space).%s" % impl diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -735,11 +735,12 @@ class Call1(VirtualArray): - def __init__(self, ufunc, name, shape, res_dtype, values): + def __init__(self, ufunc, name, shape, calc_dtype, res_dtype, values): VirtualArray.__init__(self, name, shape, res_dtype) self.values = values self.size = values.size self.ufunc = ufunc + self.calc_dtype = calc_dtype def _del_sources(self): self.values = None diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py --- a/pypy/module/micronumpy/interp_ufuncs.py +++ b/pypy/module/micronumpy/interp_ufuncs.py @@ -248,10 +248,11 @@ _immutable_fields_ = ["func", "name"] def __init__(self, func, name, promote_to_float=False, promote_bools=False, - identity=None): + identity=None, bool_result=False): W_Ufunc.__init__(self, name, promote_to_float, promote_bools, identity) self.func = func + self.bool_result = bool_result def call(self, space, args_w): from pypy.module.micronumpy.interp_numarray import (Call1, @@ -259,15 +260,19 @@ [w_obj] = args_w w_obj = convert_to_array(space, w_obj) - res_dtype = find_unaryop_result_dtype(space, - w_obj.find_dtype(), - promote_to_float=self.promote_to_float, - promote_bools=self.promote_bools, - ) + calc_dtype = find_unaryop_result_dtype(space, + w_obj.find_dtype(), + promote_to_float=self.promote_to_float, + promote_bools=self.promote_bools) + if self.bool_result: + res_dtype = interp_dtype.get_dtype_cache(space).w_booldtype + else: + res_dtype = calc_dtype if isinstance(w_obj, Scalar): - return self.func(res_dtype, w_obj.value.convert_to(res_dtype)) + return self.func(calc_dtype, w_obj.value.convert_to(calc_dtype)) - w_res = Call1(self.func, self.name, w_obj.shape, res_dtype, w_obj) + w_res = Call1(self.func, self.name, w_obj.shape, calc_dtype, res_dtype, + w_obj) w_obj.add_invalidates(w_res) return w_res @@ -433,12 +438,16 @@ return interp_dtype.get_dtype_cache(space).w_float64dtype -def ufunc_dtype_caller(space, ufunc_name, op_name, argcount, comparison_func): +def ufunc_dtype_caller(space, ufunc_name, op_name, argcount, comparison_func, + bool_result): + dtype_cache = interp_dtype.get_dtype_cache(space) if argcount == 1: def impl(res_dtype, value): - return getattr(res_dtype.itemtype, op_name)(value) + res = getattr(res_dtype.itemtype, op_name)(value) + if bool_result: + return dtype_cache.w_booldtype.box(res) + return res elif argcount == 2: - dtype_cache = interp_dtype.get_dtype_cache(space) def impl(res_dtype, lvalue, rvalue): res = getattr(res_dtype.itemtype, op_name)(lvalue, rvalue) if comparison_func: @@ -468,6 +477,8 @@ ("less_equal", "le", 2, {"comparison_func": True}), ("greater", "gt", 2, {"comparison_func": True}), ("greater_equal", "ge", 2, {"comparison_func": True}), + ("isnan", "isnan", 1, {"bool_result": True}), + ("isinf", "isinf", 1, {"bool_result": True}), ("maximum", "max", 2), ("minimum", "min", 2), @@ -510,6 +521,7 @@ func = ufunc_dtype_caller(space, ufunc_name, op_name, argcount, comparison_func=extra_kwargs.get("comparison_func", False), + bool_result=extra_kwargs.get("bool_result", False), ) if argcount == 1: ufunc = W_Ufunc1(func, ufunc_name, **extra_kwargs) diff --git a/pypy/module/micronumpy/signature.py b/pypy/module/micronumpy/signature.py --- a/pypy/module/micronumpy/signature.py +++ b/pypy/module/micronumpy/signature.py @@ -293,8 +293,8 @@ def eval(self, frame, arr): from pypy.module.micronumpy.interp_numarray import Call1 assert isinstance(arr, Call1) - v = self.child.eval(frame, arr.values).convert_to(arr.res_dtype) - return self.unfunc(arr.res_dtype, v) + v = self.child.eval(frame, arr.values).convert_to(arr.calc_dtype) + return self.unfunc(arr.calc_dtype, v) class Call2(Signature): _immutable_fields_ = ['binfunc', 'name', 'calc_dtype', 'left', 'right'] diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py --- a/pypy/module/micronumpy/test/test_ufuncs.py +++ b/pypy/module/micronumpy/test/test_ufuncs.py @@ -413,3 +413,17 @@ def test_true_divide(self): from _numpypy import arange, array, true_divide assert (true_divide(arange(3), array([2, 2, 2])) == array([0, 0.5, 1])).all() + + def test_isnan_isinf(self): + from _numpypy import isnan, isinf, float64, array + assert isnan(float('nan')) + assert isnan(float64(float('nan'))) + assert not isnan(3) + assert isinf(float('inf')) + assert not isnan(3.5) + assert not isinf(3.5) + assert not isnan(float('inf')) + assert not isinf(float('nan')) + assert (isnan(array([0.2, float('inf'), float('nan')])) == [False, False, True]).all() + assert (isinf(array([0.2, float('inf'), float('nan')])) == [False, True, False]).all() + assert isinf(array([0.2])).dtype.kind == 'b' diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py --- a/pypy/module/micronumpy/types.py +++ b/pypy/module/micronumpy/types.py @@ -23,6 +23,16 @@ ) return dispatcher +def raw_unary_op(func): + specialize.argtype(1)(func) + @functools.wraps(func) + def dispatcher(self, v): + return func( + self, + self.for_computation(self.unbox(v)) + ) + return dispatcher + def simple_binary_op(func): specialize.argtype(1, 2)(func) @functools.wraps(func) @@ -137,6 +147,14 @@ def abs(self, v): return abs(v) + @raw_unary_op + def isnan(self, v): + return False + + @raw_unary_op + def isinf(self, v): + return False + @raw_binary_op def eq(self, v1, v2): return v1 == v2 @@ -448,6 +466,14 @@ except ValueError: return rfloat.NAN + @raw_unary_op + def isnan(self, v): + return rfloat.isnan(v) + + @raw_unary_op + def isinf(self, v): + return rfloat.isinf(v) + class Float32(BaseType, Float): T = rffi.FLOAT From noreply at buildbot.pypy.org Tue Jan 24 19:31:59 2012 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 24 Jan 2012 19:31:59 +0100 (CET) Subject: [pypy-commit] pypy numpy-back-to-applevel: disable things we don't support Message-ID: <20120124183159.5EF9882CFD@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: numpy-back-to-applevel Changeset: r51736:1af82ff8a172 Date: 2012-01-24 20:31 +0200 http://bitbucket.org/pypy/pypy/changeset/1af82ff8a172/ Log: disable things we don't support diff --git a/lib_pypy/numpypy/core/arrayprint.py b/lib_pypy/numpypy/core/arrayprint.py --- a/lib_pypy/numpypy/core/arrayprint.py +++ b/lib_pypy/numpypy/core/arrayprint.py @@ -259,11 +259,11 @@ 'int' : IntegerFormat(data), 'float' : FloatFormat(data, precision, suppress_small), 'longfloat' : LongFloatFormat(precision), - 'complexfloat' : ComplexFormat(data, precision, - suppress_small), - 'longcomplexfloat' : LongComplexFormat(precision), - 'datetime' : DatetimeFormat(data), - 'timedelta' : TimedeltaFormat(data), + #'complexfloat' : ComplexFormat(data, precision, + # suppress_small), + #'longcomplexfloat' : LongComplexFormat(precision), + #'datetime' : DatetimeFormat(data), + #'timedelta' : TimedeltaFormat(data), 'numpystr' : repr_format, 'str' : str} @@ -300,10 +300,10 @@ if issubclass(dtypeobj, _nt.bool_): format_function = formatdict['bool'] elif issubclass(dtypeobj, _nt.integer): - if issubclass(dtypeobj, _nt.timedelta64): - format_function = formatdict['timedelta'] - else: - format_function = formatdict['int'] + #if issubclass(dtypeobj, _nt.timedelta64): + # format_function = formatdict['timedelta'] + #else: + format_function = formatdict['int'] elif issubclass(dtypeobj, _nt.floating): if issubclass(dtypeobj, _nt.longfloat): format_function = formatdict['longfloat'] From noreply at buildbot.pypy.org Tue Jan 24 19:42:38 2012 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 24 Jan 2012 19:42:38 +0100 (CET) Subject: [pypy-commit] pypy default: a dumb implementation of isna Message-ID: <20120124184238.2572082CFD@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: Changeset: r51737:24f7e3520fb4 Date: 2012-01-24 20:41 +0200 http://bitbucket.org/pypy/pypy/changeset/24f7e3520fb4/ Log: a dumb implementation of isna diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py --- a/pypy/module/micronumpy/__init__.py +++ b/pypy/module/micronumpy/__init__.py @@ -27,6 +27,7 @@ 'dot': 'interp_numarray.dot', 'fromstring': 'interp_support.fromstring', 'flatiter': 'interp_numarray.W_FlatIterator', + 'isna': 'interp_numarray.isna', 'True_': 'types.Bool.True', 'False_': 'types.Bool.False', diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -436,6 +436,13 @@ def copy(self, space): return self.get_concrete().copy(space) + def empty_copy(self, space, dtype): + shape = self.shape + size = 1 + for elem in shape: + size *= elem + return W_NDimArray(size, shape[:], dtype, 'C') + def flatten(self, space): return self.get_concrete().flatten(space) @@ -1439,3 +1446,11 @@ __iter__ = interp2app(W_FlatIterator.descr_iter), ) W_FlatIterator.acceptable_as_base_class = False + +def isna(space, w_obj): + if isinstance(w_obj, BaseArray): + arr = w_obj.empty_copy(space, + interp_dtype.get_dtype_cache(space).w_booldtype) + arr.fill(space, space.wrap(False)) + return arr + return space.wrap(False) diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -1381,6 +1381,11 @@ a[a & 1 == 1] = array([8, 9, 10]) assert (a == [[0, 8], [2, 9], [4, 10]]).all() + def test_isna(self): + from _numpypy import isna, array + # XXX for now + assert not isna(3) + assert (isna(array([1, 2, 3, 4])) == [False, False, False, False]).all() class AppTestSupport(BaseNumpyAppTest): From noreply at buildbot.pypy.org Tue Jan 24 19:42:39 2012 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 24 Jan 2012 19:42:39 +0100 (CET) Subject: [pypy-commit] pypy default: merge Message-ID: <20120124184239.576F082CFD@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: Changeset: r51738:45af9fa4aed0 Date: 2012-01-24 20:42 +0200 http://bitbucket.org/pypy/pypy/changeset/45af9fa4aed0/ Log: merge diff --git a/pypy/module/_io/interp_fileio.py b/pypy/module/_io/interp_fileio.py --- a/pypy/module/_io/interp_fileio.py +++ b/pypy/module/_io/interp_fileio.py @@ -349,6 +349,8 @@ try: s = os.read(self.fd, size) except OSError, e: + if e.errno == errno.EAGAIN: + return space.w_None raise wrap_oserror(space, e, exception_name='w_IOError') @@ -362,6 +364,8 @@ try: buf = os.read(self.fd, length) except OSError, e: + if e.errno == errno.EAGAIN: + return space.w_None raise wrap_oserror(space, e, exception_name='w_IOError') rwbuffer.setslice(0, buf) diff --git a/pypy/module/_io/test/test_fileio.py b/pypy/module/_io/test/test_fileio.py --- a/pypy/module/_io/test/test_fileio.py +++ b/pypy/module/_io/test/test_fileio.py @@ -133,6 +133,19 @@ f.close() assert a == 'a\nbxxxxxxx' + def test_nonblocking_read(self): + import os, fcntl + r_fd, w_fd = os.pipe() + # set nonblocking + fcntl.fcntl(r_fd, fcntl.F_SETFL, os.O_NONBLOCK) + import _io + f = _io.FileIO(r_fd, 'r') + # Read from stream sould return None + assert f.read() is None + assert f.read(10) is None + a = bytearray('x' * 10) + assert f.readinto(a) is None + def test_repr(self): import _io f = _io.FileIO(self.tmpfile, 'r') diff --git a/pypy/module/micronumpy/tool/numready.py b/pypy/module/micronumpy/tool/numready.py deleted file mode 100644 --- a/pypy/module/micronumpy/tool/numready.py +++ /dev/null @@ -1,226 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -""" -This should be run under PyPy. -""" - -import platform -import subprocess -import sys -import tempfile -import webbrowser -from collections import OrderedDict - -import jinja2 - - -MODULE_SEARCH_CODE = ''' -import types -import {modname} as numpy - -for name in dir(numpy): - if name.startswith("_"): - continue - obj = getattr(numpy, name) - kind = "{kinds[UNKNOWN]}" - if isinstance(obj, types.TypeType): - kind = "{kinds[TYPE]}" - print kind, ":", name -''' - -ATTR_SEARCH_CODE = ''' -import types -import {modname} as numpy - -obj = getattr(numpy, "{name}") -for name in dir(obj): - #if name.startswith("_"): - # continue - sub_obj = getattr(obj, name) - kind = "{kinds[UNKNOWN]}" - if isinstance(sub_obj, types.TypeType): - kind = "{kinds[TYPE]}" - print kind, ":", name -''' - -KINDS = { - "UNKNOWN": "U", - "TYPE": "T", -} - -PAGE_TEMPLATE = u""" -<!DOCTYPE html> -<html lang="en"> - <head> - <title>NumPyPy Status</title> - <meta http-equiv="content-type" content="text/html; charset=utf-8"> - <style type="text/css"> - body { - font-family: 'Consolas', 'Bitstream Vera Sans Mono', monospace; - } - h1 { - text-align: center; - } - h3 { - text-align: center; - } - table { - border: 8px solid #DFDECB; - margin: 30px auto; - font-size: 12px; - } - table th { - text-align: left; - } - table td { - padding: 4px 10px; - text-align: center; - } - .exists { - background-color: #337792; - color: white; - border: 1px solid #234F61; - } - </style> - </head> - <body> - <h1>NumPyPy Status</h1> - <h3>Overall: {{ msg }}</h3> - <table> - <thead> - <tr> - <th></th> - <th>PyPy</th> - <th></th> - <th>PyPy</th> - <th></th> - <th>PyPy</th> - <th></th> - <th>PyPy</th> - <th></th> - <th>PyPy</th> - </tr> - </thead> - <tbody> - {% for chunk in all_items %} - <tr> - {% for item in chunk %} - <th class='{{ item.cls }}'>{{ item.name }}</th> - <td class='{{ item.cls }}'>{{ item.symbol }}</td> - {% endfor %} - </tr> - {% endfor %} - </tbody> - </table> - </body> -</html> -""" - -class SearchableSet(object): - def __init__(self, items=()): - self._items = {} - for item in items: - self.add(item) - - def __iter__(self): - return iter(self._items) - - def __contains__(self, other): - return other in self._items - - def __getitem__(self, idx): - return self._items[idx] - - def add(self, item): - self._items[item] = item - - def __len__(self): - return len(self._items) - -class Item(object): - def __init__(self, name, kind, subitems=None): - self.name = name - self.kind = kind - self.subitems = subitems - - def __hash__(self): - return hash(self.name) - - def __eq__(self, other): - if isinstance(other, str): - return self.name == other - return self.name == other.name - - -class ItemStatus(object): - def __init__(self, name, pypy_exists): - self.name = name - self.cls = 'exists' if pypy_exists else '' - self.symbol = u"✔" if pypy_exists else u'✖' - - def __lt__(self, other): - return self.name < other.name - -def find_numpy_attrs(python, modname, name): - lines = subprocess.check_output( - [python, "-c", ATTR_SEARCH_CODE.format(modname=modname, kinds=KINDS, name=name)] - ).splitlines() - items = SearchableSet() - for line in lines: - kind, name = line.split(" : ", 1) - items.add(Item(name, kind)) - return items - -def find_numpy_items(python, modname="numpy"): - lines = subprocess.check_output( - [python, "-c", MODULE_SEARCH_CODE.format(modname=modname, kinds=KINDS)] - ).splitlines() - items = SearchableSet() - for line in lines: - kind, name = line.split(" : ", 1) - subitems = None - if kind == KINDS["TYPE"]: - if name in ['ndarray', 'dtype']: - subitems = find_numpy_attrs(python, modname, name) - items.add(Item(name, kind, subitems)) - return items - -def split(lst): - SPLIT = 5 - lgt = len(lst) // SPLIT + 1 - l = [[] for i in range(lgt)] - for i in range(lgt): - for k in range(SPLIT): - if k * lgt + i < len(lst): - l[i].append(lst[k * lgt + i]) - return l - -def main(argv): - cpy_items = find_numpy_items("/usr/bin/python") - pypy_items = find_numpy_items(argv[1], "numpypy") - all_items = [] - - msg = '%d/%d names, %d/%d ndarray attributes, %d/%d dtype attributes' % ( - len(pypy_items), len(cpy_items), len(pypy_items['ndarray'].subitems), - len(cpy_items['ndarray'].subitems), len(pypy_items['dtype'].subitems), - len(cpy_items['dtype'].subitems)) - for item in cpy_items: - pypy_exists = item in pypy_items - if item.subitems: - for sub in item.subitems: - all_items.append( - ItemStatus(item.name + "." + sub.name, pypy_exists=pypy_exists and pypy_items[item].subitems and sub in pypy_items[item].subitems) - ) - all_items.append(ItemStatus(item.name, pypy_exists=item in pypy_items)) - html = jinja2.Template(PAGE_TEMPLATE).render(all_items=split(sorted(all_items)), msg=msg) - if len(argv) > 2: - with open(argv[2], 'w') as f: - f.write(html.encode("utf-8")) - else: - with tempfile.NamedTemporaryFile(delete=False) as f: - f.write(html.encode("utf-8")) - print "Saved in: %s" % f.name - -if __name__ == '__main__': - main(sys.argv) diff --git a/pypy/module/micronumpy/tool/numready/__init__.py b/pypy/module/micronumpy/tool/numready/__init__.py new file mode 100644 diff --git a/pypy/module/micronumpy/tool/numready/__main__.py b/pypy/module/micronumpy/tool/numready/__main__.py new file mode 100644 --- /dev/null +++ b/pypy/module/micronumpy/tool/numready/__main__.py @@ -0,0 +1,6 @@ +import sys + +from .main import main + + +main(sys.argv) diff --git a/pypy/module/micronumpy/tool/numready/kinds.py b/pypy/module/micronumpy/tool/numready/kinds.py new file mode 100644 --- /dev/null +++ b/pypy/module/micronumpy/tool/numready/kinds.py @@ -0,0 +1,4 @@ +KINDS = { + "UNKNOWN": "U", + "TYPE": "T", +} diff --git a/pypy/module/micronumpy/tool/numready/main.py b/pypy/module/micronumpy/tool/numready/main.py new file mode 100644 --- /dev/null +++ b/pypy/module/micronumpy/tool/numready/main.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" +This should be run under PyPy. +""" + +import os +import platform +import subprocess +import tempfile +import webbrowser +from collections import OrderedDict + +import jinja2 + +from .kinds import KINDS + + +class SearchableSet(object): + def __init__(self, items=()): + self._items = {} + for item in items: + self.add(item) + + def __iter__(self): + return iter(self._items) + + def __contains__(self, other): + return other in self._items + + def __getitem__(self, idx): + return self._items[idx] + + def add(self, item): + self._items[item] = item + + def __len__(self): + return len(self._items) + +class Item(object): + def __init__(self, name, kind, subitems=None): + self.name = name + self.kind = kind + self.subitems = subitems + + def __hash__(self): + return hash(self.name) + + def __eq__(self, other): + if isinstance(other, str): + return self.name == other + return self.name == other.name + + +class ItemStatus(object): + def __init__(self, name, pypy_exists): + self.name = name + self.cls = 'exists' if pypy_exists else '' + self.symbol = u"✔" if pypy_exists else u'✖' + + def __lt__(self, other): + return self.name < other.name + +def find_numpy_items(python, modname="numpy", attr=None): + args = [ + python, os.path.join(os.path.dirname(__file__), "search.py"), modname + ] + if attr is not None: + args.append(attr) + lines = subprocess.check_output(args).splitlines() + items = SearchableSet() + for line in lines: + kind, name = line.split(" : ", 1) + subitems = None + if kind == KINDS["TYPE"]: + if name in ['ndarray', 'dtype']: + subitems = find_numpy_items(python, modname, name) + items.add(Item(name, kind, subitems)) + return items + +def split(lst): + SPLIT = 5 + lgt = len(lst) // SPLIT + 1 + l = [[] for i in range(lgt)] + for i in range(lgt): + for k in range(SPLIT): + if k * lgt + i < len(lst): + l[i].append(lst[k * lgt + i]) + return l + +def main(argv): + cpy_items = find_numpy_items("/usr/bin/python") + pypy_items = find_numpy_items(argv[1], "numpypy") + all_items = [] + + msg = '%d/%d names, %d/%d ndarray attributes, %d/%d dtype attributes' % ( + len(pypy_items), len(cpy_items), len(pypy_items['ndarray'].subitems), + len(cpy_items['ndarray'].subitems), len(pypy_items['dtype'].subitems), + len(cpy_items['dtype'].subitems)) + for item in cpy_items: + pypy_exists = item in pypy_items + if item.subitems: + for sub in item.subitems: + all_items.append( + ItemStatus(item.name + "." + sub.name, pypy_exists=pypy_exists and pypy_items[item].subitems and sub in pypy_items[item].subitems) + ) + all_items.append(ItemStatus(item.name, pypy_exists=item in pypy_items)) + env = jinja2.Environment( + loader=jinja2.FileSystemLoader(os.path.dirname(__file__)) + ) + html = env.get_template("page.html").render(all_items=split(sorted(all_items)), msg=msg) + if len(argv) > 2: + with open(argv[2], 'w') as f: + f.write(html.encode("utf-8")) + else: + with tempfile.NamedTemporaryFile(delete=False) as f: + f.write(html.encode("utf-8")) + print "Saved in: %s" % f.name diff --git a/pypy/module/micronumpy/tool/numready/page.html b/pypy/module/micronumpy/tool/numready/page.html new file mode 100644 --- /dev/null +++ b/pypy/module/micronumpy/tool/numready/page.html @@ -0,0 +1,65 @@ +<!DOCTYPE html> +<html lang="en"> + <head> + <title>NumPyPy Status</title> + <meta http-equiv="content-type" content="text/html; charset=utf-8"> + <style type="text/css"> + body { + font-family: 'Consolas', 'Bitstream Vera Sans Mono', monospace; + } + h1 { + text-align: center; + } + h3 { + text-align: center; + } + table { + border: 8px solid #DFDECB; + margin: 30px auto; + font-size: 12px; + } + table th { + text-align: left; + } + table td { + padding: 4px 10px; + text-align: center; + } + .exists { + background-color: #337792; + color: white; + border: 1px solid #234F61; + } + </style> + </head> + <body> + <h1>NumPyPy Status</h1> + <h3>Overall: {{ msg }}</h3> + <table> + <thead> + <tr> + <th></th> + <th>PyPy</th> + <th></th> + <th>PyPy</th> + <th></th> + <th>PyPy</th> + <th></th> + <th>PyPy</th> + <th></th> + <th>PyPy</th> + </tr> + </thead> + <tbody> + {% for chunk in all_items %} + <tr> + {% for item in chunk %} + <th class='{{ item.cls }}'>{{ item.name }}</th> + <td class='{{ item.cls }}'>{{ item.symbol }}</td> + {% endfor %} + </tr> + {% endfor %} + </tbody> + </table> + </body> +</html> diff --git a/pypy/module/micronumpy/tool/numready/search.py b/pypy/module/micronumpy/tool/numready/search.py new file mode 100644 --- /dev/null +++ b/pypy/module/micronumpy/tool/numready/search.py @@ -0,0 +1,33 @@ +import sys +import types + +# Evil implicit relative import. +from kinds import KINDS + + +def main(argv): + if len(argv) == 2: + [_, modname] = argv + attr = None + elif len(argv) == 3: + [_, modname, attr] = argv + else: + sys.exit("Wrong number of args") + __import__(modname) + obj = sys.modules[modname] + + if attr is not None: + obj = getattr(obj, attr) + + for name in dir(obj): + if attr is None and name.startswith("_"): + continue + subobj = getattr(obj, name) + if isinstance(subobj, types.TypeType): + kind = KINDS["TYPE"] + else: + kind = KINDS["UNKNOWN"] + print kind, ":", name + +if __name__ == "__main__": + main(sys.argv) \ No newline at end of file From noreply at buildbot.pypy.org Tue Jan 24 19:44:55 2012 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 24 Jan 2012 19:44:55 +0100 (CET) Subject: [pypy-commit] pypy numpy-back-to-applevel: merge Message-ID: <20120124184455.6E15382CFD@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: numpy-back-to-applevel Changeset: r51739:c77051b2f51a Date: 2012-01-24 20:43 +0200 http://bitbucket.org/pypy/pypy/changeset/c77051b2f51a/ Log: merge diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py --- a/lib_pypy/_sqlite3.py +++ b/lib_pypy/_sqlite3.py @@ -20,6 +20,8 @@ # 2. Altered source versions must be plainly marked as such, and must not be # misrepresented as being the original software. # 3. This notice may not be removed or altered from any source distribution. +# +# Note: This software has been modified for use in PyPy. from ctypes import c_void_p, c_int, c_double, c_int64, c_char_p, cdll from ctypes import POINTER, byref, string_at, CFUNCTYPE, cast @@ -27,7 +29,6 @@ from collections import OrderedDict import datetime import sys -import time import weakref from threading import _get_ident as thread_get_ident @@ -606,7 +607,7 @@ def authorizer(userdata, action, arg1, arg2, dbname, source): try: return int(callback(action, arg1, arg2, dbname, source)) - except Exception, e: + except Exception: return SQLITE_DENY c_authorizer = AUTHORIZER(authorizer) @@ -653,7 +654,7 @@ if not aggregate_ptr[0]: try: aggregate = cls() - except Exception, e: + except Exception: msg = ("user-defined aggregate's '__init__' " "method raised error") sqlite.sqlite3_result_error(context, msg, len(msg)) @@ -667,7 +668,7 @@ params = _convert_params(context, argc, c_params) try: aggregate.step(*params) - except Exception, e: + except Exception: msg = ("user-defined aggregate's 'step' " "method raised error") sqlite.sqlite3_result_error(context, msg, len(msg)) @@ -683,7 +684,7 @@ aggregate = self.aggregate_instances[aggregate_ptr[0]] try: val = aggregate.finalize() - except Exception, e: + except Exception: msg = ("user-defined aggregate's 'finalize' " "method raised error") sqlite.sqlite3_result_error(context, msg, len(msg)) @@ -771,7 +772,7 @@ self.statement.item = None self.statement.exhausted = True - if self.statement.kind == DML or self.statement.kind == DDL: + if self.statement.kind == DML: self.statement.reset() self.rowcount = -1 @@ -791,7 +792,7 @@ if self.statement.kind == DML: self.connection._begin() else: - raise ProgrammingError, "executemany is only for DML statements" + raise ProgrammingError("executemany is only for DML statements") self.rowcount = 0 for params in many_params: @@ -861,8 +862,6 @@ except StopIteration: return None - return nextrow - def fetchmany(self, size=None): self._check_closed() self._check_reset() @@ -915,7 +914,7 @@ def __init__(self, connection, sql): self.statement = None if not isinstance(sql, str): - raise ValueError, "sql must be a string" + raise ValueError("sql must be a string") self.con = connection self.sql = sql # DEBUG ONLY first_word = self._statement_kind = sql.lstrip().split(" ")[0].upper() @@ -944,8 +943,8 @@ raise self.con._get_exception(ret) self.con._remember_statement(self) if _check_remaining_sql(next_char.value): - raise Warning, "One and only one statement required: %r" % ( - next_char.value,) + raise Warning("One and only one statement required: %r" % ( + next_char.value,)) # sql_char should remain alive until here self._build_row_cast_map() @@ -1016,7 +1015,7 @@ elif type(param) is buffer: sqlite.sqlite3_bind_blob(self.statement, idx, str(param), len(param), SQLITE_TRANSIENT) else: - raise InterfaceError, "parameter type %s is not supported" % str(type(param)) + raise InterfaceError("parameter type %s is not supported" % str(type(param))) def set_params(self, params): ret = sqlite.sqlite3_reset(self.statement) @@ -1045,11 +1044,11 @@ for idx in range(1, sqlite.sqlite3_bind_parameter_count(self.statement) + 1): param_name = sqlite.sqlite3_bind_parameter_name(self.statement, idx) if param_name is None: - raise ProgrammingError, "need named parameters" + raise ProgrammingError("need named parameters") param_name = param_name[1:] try: param = params[param_name] - except KeyError, e: + except KeyError: raise ProgrammingError("missing parameter '%s'" %param) self.set_param(idx, param) @@ -1260,7 +1259,7 @@ params = _convert_params(context, nargs, c_params) try: val = real_cb(*params) - except Exception, e: + except Exception: msg = "user-defined function raised exception" sqlite.sqlite3_result_error(context, msg, len(msg)) else: diff --git a/pypy/doc/getting-started.rst b/pypy/doc/getting-started.rst --- a/pypy/doc/getting-started.rst +++ b/pypy/doc/getting-started.rst @@ -53,11 +53,11 @@ PyPy is ready to be executed as soon as you unpack the tarball or the zip file, with no need to install it in any specific location:: - $ tar xf pypy-1.6-linux.tar.bz2 + $ tar xf pypy-1.7-linux.tar.bz2 - $ ./pypy-1.6/bin/pypy + $ ./pypy-1.7/bin/pypy Python 2.7.1 (?, Apr 27 2011, 12:44:21) - [PyPy 1.6.0 with GCC 4.4.3] on linux2 + [PyPy 1.7.0 with GCC 4.4.3] on linux2 Type "help", "copyright", "credits" or "license" for more information. And now for something completely different: ``implementing LOGO in LOGO: "turtles all the way down"'' @@ -75,14 +75,14 @@ $ curl -O https://raw.github.com/pypa/pip/master/contrib/get-pip.py - $ ./pypy-1.6/bin/pypy distribute_setup.py + $ ./pypy-1.7/bin/pypy distribute_setup.py - $ ./pypy-1.6/bin/pypy get-pip.py + $ ./pypy-1.7/bin/pypy get-pip.py - $ ./pypy-1.6/bin/pip install pygments # for example + $ ./pypy-1.7/bin/pip install pygments # for example -3rd party libraries will be installed in ``pypy-1.6/site-packages``, and -the scripts in ``pypy-1.6/bin``. +3rd party libraries will be installed in ``pypy-1.7/site-packages``, and +the scripts in ``pypy-1.7/bin``. Installing using virtualenv --------------------------- diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py --- a/pypy/interpreter/executioncontext.py +++ b/pypy/interpreter/executioncontext.py @@ -445,6 +445,7 @@ AsyncAction.__init__(self, space) self.dying_objects = [] self.finalizers_lock_count = 0 + self.enabled_at_app_level = True def register_callback(self, w_obj, callback, descrname): self.dying_objects.append((w_obj, callback, descrname)) diff --git a/pypy/module/_hashlib/interp_hashlib.py b/pypy/module/_hashlib/interp_hashlib.py --- a/pypy/module/_hashlib/interp_hashlib.py +++ b/pypy/module/_hashlib/interp_hashlib.py @@ -34,8 +34,12 @@ ctx = lltype.malloc(ropenssl.EVP_MD_CTX.TO, flavor='raw') rgc.add_memory_pressure(HASH_MALLOC_SIZE + self.digest_size) - ropenssl.EVP_DigestInit(ctx, digest_type) - self.ctx = ctx + try: + ropenssl.EVP_DigestInit(ctx, digest_type) + self.ctx = ctx + except: + lltype.free(ctx, flavor='raw') + raise def __del__(self): # self.lock.free() diff --git a/pypy/module/_io/interp_fileio.py b/pypy/module/_io/interp_fileio.py --- a/pypy/module/_io/interp_fileio.py +++ b/pypy/module/_io/interp_fileio.py @@ -349,6 +349,8 @@ try: s = os.read(self.fd, size) except OSError, e: + if e.errno == errno.EAGAIN: + return space.w_None raise wrap_oserror(space, e, exception_name='w_IOError') @@ -362,6 +364,8 @@ try: buf = os.read(self.fd, length) except OSError, e: + if e.errno == errno.EAGAIN: + return space.w_None raise wrap_oserror(space, e, exception_name='w_IOError') rwbuffer.setslice(0, buf) diff --git a/pypy/module/_io/test/test_fileio.py b/pypy/module/_io/test/test_fileio.py --- a/pypy/module/_io/test/test_fileio.py +++ b/pypy/module/_io/test/test_fileio.py @@ -133,6 +133,19 @@ f.close() assert a == 'a\nbxxxxxxx' + def test_nonblocking_read(self): + import os, fcntl + r_fd, w_fd = os.pipe() + # set nonblocking + fcntl.fcntl(r_fd, fcntl.F_SETFL, os.O_NONBLOCK) + import _io + f = _io.FileIO(r_fd, 'r') + # Read from stream sould return None + assert f.read() is None + assert f.read(10) is None + a = bytearray('x' * 10) + assert f.readinto(a) is None + def test_repr(self): import _io f = _io.FileIO(self.tmpfile, 'r') diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py --- a/pypy/module/_socket/interp_socket.py +++ b/pypy/module/_socket/interp_socket.py @@ -67,9 +67,6 @@ self.connect(self.addr_from_object(space, w_addr)) except SocketError, e: raise converted_error(space, e) - except TypeError, e: - raise OperationError(space.w_TypeError, - space.wrap(str(e))) def connect_ex_w(self, space, w_addr): """connect_ex(address) -> errno diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py --- a/pypy/module/_ssl/test/test_ssl.py +++ b/pypy/module/_ssl/test/test_ssl.py @@ -64,8 +64,8 @@ def test_sslwrap(self): import _ssl, _socket, sys, gc - if sys.platform == 'darwin': - skip("hangs indefinitely on OSX (also on CPython)") + if sys.platform == 'darwin' or 'freebsd' in sys.platform: + skip("hangs indefinitely on OSX & FreeBSD (also on CPython)") s = _socket.socket() ss = _ssl.sslwrap(s, 0) exc = raises(_socket.error, ss.do_handshake) diff --git a/pypy/module/fcntl/test/test_fcntl.py b/pypy/module/fcntl/test/test_fcntl.py --- a/pypy/module/fcntl/test/test_fcntl.py +++ b/pypy/module/fcntl/test/test_fcntl.py @@ -42,13 +42,9 @@ else: start_len = "qq" - if sys.platform in ('netbsd1', 'netbsd2', 'netbsd3', - 'Darwin1.2', 'darwin', - 'freebsd2', 'freebsd3', 'freebsd4', 'freebsd5', - 'freebsd6', 'freebsd7', 'freebsd8', 'freebsd9', - 'bsdos2', 'bsdos3', 'bsdos4', - 'openbsd', 'openbsd2', 'openbsd3', 'openbsd4', - 'openbsd5'): + if any(substring in sys.platform.lower() + for substring in ('netbsd', 'darwin', 'freebsd', 'bsdos', + 'openbsd')): if struct.calcsize('l') == 8: off_t = 'l' pid_t = 'i' @@ -182,7 +178,8 @@ def test_large_flag(self): import sys - if sys.platform == "darwin" or sys.platform.startswith("openbsd"): + if any(plat in sys.platform + for plat in ('darwin', 'openbsd', 'freebsd')): skip("Mac OS doesn't have any large flag in fcntl.h") import fcntl, sys if sys.maxint == 2147483647: diff --git a/pypy/module/gc/__init__.py b/pypy/module/gc/__init__.py --- a/pypy/module/gc/__init__.py +++ b/pypy/module/gc/__init__.py @@ -1,18 +1,18 @@ from pypy.interpreter.mixedmodule import MixedModule class Module(MixedModule): - appleveldefs = { - 'enable': 'app_gc.enable', - 'disable': 'app_gc.disable', - 'isenabled': 'app_gc.isenabled', - } interpleveldefs = { 'collect': 'interp_gc.collect', + 'enable': 'interp_gc.enable', + 'disable': 'interp_gc.disable', + 'isenabled': 'interp_gc.isenabled', 'enable_finalizers': 'interp_gc.enable_finalizers', 'disable_finalizers': 'interp_gc.disable_finalizers', 'garbage' : 'space.newlist([])', #'dump_heap_stats': 'interp_gc.dump_heap_stats', } + appleveldefs = { + } def __init__(self, space, w_name): if (not space.config.translating or diff --git a/pypy/module/gc/app_gc.py b/pypy/module/gc/app_gc.py deleted file mode 100644 --- a/pypy/module/gc/app_gc.py +++ /dev/null @@ -1,21 +0,0 @@ -# NOT_RPYTHON - -enabled = True - -def isenabled(): - global enabled - return enabled - -def enable(): - global enabled - import gc - if not enabled: - gc.enable_finalizers() - enabled = True - -def disable(): - global enabled - import gc - if enabled: - gc.disable_finalizers() - enabled = False diff --git a/pypy/module/gc/interp_gc.py b/pypy/module/gc/interp_gc.py --- a/pypy/module/gc/interp_gc.py +++ b/pypy/module/gc/interp_gc.py @@ -17,6 +17,26 @@ rgc.collect() return space.wrap(0) +def enable(space): + """Non-recursive version. Enable finalizers now. + If they were already enabled, no-op. + If they were disabled even several times, enable them anyway. + """ + if not space.user_del_action.enabled_at_app_level: + space.user_del_action.enabled_at_app_level = True + enable_finalizers(space) + +def disable(space): + """Non-recursive version. Disable finalizers now. Several calls + to this function are ignored. + """ + if space.user_del_action.enabled_at_app_level: + space.user_del_action.enabled_at_app_level = False + disable_finalizers(space) + +def isenabled(space): + return space.newbool(space.user_del_action.enabled_at_app_level) + def enable_finalizers(space): if space.user_del_action.finalizers_lock_count == 0: raise OperationError(space.w_ValueError, diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py --- a/pypy/module/micronumpy/__init__.py +++ b/pypy/module/micronumpy/__init__.py @@ -27,6 +27,7 @@ 'dot': 'interp_numarray.dot', 'fromstring': 'interp_support.fromstring', 'flatiter': 'interp_numarray.W_FlatIterator', + 'isna': 'interp_numarray.isna', 'set_string_function': 'appbridge.set_string_function', diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -270,6 +270,13 @@ def copy(self, space): return self.get_concrete().copy(space) + def empty_copy(self, space, dtype): + shape = self.shape + size = 1 + for elem in shape: + size *= elem + return W_NDimArray(size, shape[:], dtype, 'C') + def flatten(self, space): return self.get_concrete().flatten(space) @@ -1339,3 +1346,11 @@ __iter__ = interp2app(W_FlatIterator.descr_iter), ) W_FlatIterator.acceptable_as_base_class = False + +def isna(space, w_obj): + if isinstance(w_obj, BaseArray): + arr = w_obj.empty_copy(space, + interp_dtype.get_dtype_cache(space).w_booldtype) + arr.fill(space, space.wrap(False)) + return arr + return space.wrap(False) diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -1389,6 +1389,12 @@ assert array(x, copy=False) is x assert array(x, copy=True) is not x + def test_isna(self): + from _numpypy import isna, array + # XXX for now + assert not isna(3) + assert (isna(array([1, 2, 3, 4])) == [False, False, False, False]).all() + def test_ravel(self): from _numpypy import arange assert (arange(3).ravel() == arange(3)).all() diff --git a/pypy/module/micronumpy/tool/numready/__init__.py b/pypy/module/micronumpy/tool/numready/__init__.py new file mode 100644 diff --git a/pypy/module/micronumpy/tool/numready/__main__.py b/pypy/module/micronumpy/tool/numready/__main__.py new file mode 100644 --- /dev/null +++ b/pypy/module/micronumpy/tool/numready/__main__.py @@ -0,0 +1,6 @@ +import sys + +from .main import main + + +main(sys.argv) diff --git a/pypy/module/micronumpy/tool/numready/kinds.py b/pypy/module/micronumpy/tool/numready/kinds.py new file mode 100644 --- /dev/null +++ b/pypy/module/micronumpy/tool/numready/kinds.py @@ -0,0 +1,4 @@ +KINDS = { + "UNKNOWN": "U", + "TYPE": "T", +} diff --git a/pypy/module/micronumpy/tool/numready/main.py b/pypy/module/micronumpy/tool/numready/main.py new file mode 100644 --- /dev/null +++ b/pypy/module/micronumpy/tool/numready/main.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" +This should be run under PyPy. +""" + +import os +import platform +import subprocess +import tempfile +import webbrowser +from collections import OrderedDict + +import jinja2 + +from .kinds import KINDS + + +class SearchableSet(object): + def __init__(self, items=()): + self._items = {} + for item in items: + self.add(item) + + def __iter__(self): + return iter(self._items) + + def __contains__(self, other): + return other in self._items + + def __getitem__(self, idx): + return self._items[idx] + + def add(self, item): + self._items[item] = item + + def __len__(self): + return len(self._items) + +class Item(object): + def __init__(self, name, kind, subitems=None): + self.name = name + self.kind = kind + self.subitems = subitems + + def __hash__(self): + return hash(self.name) + + def __eq__(self, other): + if isinstance(other, str): + return self.name == other + return self.name == other.name + + +class ItemStatus(object): + def __init__(self, name, pypy_exists): + self.name = name + self.cls = 'exists' if pypy_exists else '' + self.symbol = u"✔" if pypy_exists else u'✖' + + def __lt__(self, other): + return self.name < other.name + +def find_numpy_items(python, modname="numpy", attr=None): + args = [ + python, os.path.join(os.path.dirname(__file__), "search.py"), modname + ] + if attr is not None: + args.append(attr) + lines = subprocess.check_output(args).splitlines() + items = SearchableSet() + for line in lines: + kind, name = line.split(" : ", 1) + subitems = None + if kind == KINDS["TYPE"]: + if name in ['ndarray', 'dtype']: + subitems = find_numpy_items(python, modname, name) + items.add(Item(name, kind, subitems)) + return items + +def split(lst): + SPLIT = 5 + lgt = len(lst) // SPLIT + 1 + l = [[] for i in range(lgt)] + for i in range(lgt): + for k in range(SPLIT): + if k * lgt + i < len(lst): + l[i].append(lst[k * lgt + i]) + return l + +def main(argv): + cpy_items = find_numpy_items("/usr/bin/python") + pypy_items = find_numpy_items(argv[1], "numpypy") + all_items = [] + + msg = '%d/%d names, %d/%d ndarray attributes, %d/%d dtype attributes' % ( + len(pypy_items), len(cpy_items), len(pypy_items['ndarray'].subitems), + len(cpy_items['ndarray'].subitems), len(pypy_items['dtype'].subitems), + len(cpy_items['dtype'].subitems)) + for item in cpy_items: + pypy_exists = item in pypy_items + if item.subitems: + for sub in item.subitems: + all_items.append( + ItemStatus(item.name + "." + sub.name, pypy_exists=pypy_exists and pypy_items[item].subitems and sub in pypy_items[item].subitems) + ) + all_items.append(ItemStatus(item.name, pypy_exists=item in pypy_items)) + env = jinja2.Environment( + loader=jinja2.FileSystemLoader(os.path.dirname(__file__)) + ) + html = env.get_template("page.html").render(all_items=split(sorted(all_items)), msg=msg) + if len(argv) > 2: + with open(argv[2], 'w') as f: + f.write(html.encode("utf-8")) + else: + with tempfile.NamedTemporaryFile(delete=False) as f: + f.write(html.encode("utf-8")) + print "Saved in: %s" % f.name diff --git a/pypy/module/micronumpy/tool/numready/page.html b/pypy/module/micronumpy/tool/numready/page.html new file mode 100644 --- /dev/null +++ b/pypy/module/micronumpy/tool/numready/page.html @@ -0,0 +1,65 @@ +<!DOCTYPE html> +<html lang="en"> + <head> + <title>NumPyPy Status</title> + <meta http-equiv="content-type" content="text/html; charset=utf-8"> + <style type="text/css"> + body { + font-family: 'Consolas', 'Bitstream Vera Sans Mono', monospace; + } + h1 { + text-align: center; + } + h3 { + text-align: center; + } + table { + border: 8px solid #DFDECB; + margin: 30px auto; + font-size: 12px; + } + table th { + text-align: left; + } + table td { + padding: 4px 10px; + text-align: center; + } + .exists { + background-color: #337792; + color: white; + border: 1px solid #234F61; + } + </style> + </head> + <body> + <h1>NumPyPy Status</h1> + <h3>Overall: {{ msg }}</h3> + <table> + <thead> + <tr> + <th></th> + <th>PyPy</th> + <th></th> + <th>PyPy</th> + <th></th> + <th>PyPy</th> + <th></th> + <th>PyPy</th> + <th></th> + <th>PyPy</th> + </tr> + </thead> + <tbody> + {% for chunk in all_items %} + <tr> + {% for item in chunk %} + <th class='{{ item.cls }}'>{{ item.name }}</th> + <td class='{{ item.cls }}'>{{ item.symbol }}</td> + {% endfor %} + </tr> + {% endfor %} + </tbody> + </table> + </body> +</html> diff --git a/pypy/module/micronumpy/tool/numready/search.py b/pypy/module/micronumpy/tool/numready/search.py new file mode 100644 --- /dev/null +++ b/pypy/module/micronumpy/tool/numready/search.py @@ -0,0 +1,33 @@ +import sys +import types + +# Evil implicit relative import. +from kinds import KINDS + + +def main(argv): + if len(argv) == 2: + [_, modname] = argv + attr = None + elif len(argv) == 3: + [_, modname, attr] = argv + else: + sys.exit("Wrong number of args") + __import__(modname) + obj = sys.modules[modname] + + if attr is not None: + obj = getattr(obj, attr) + + for name in dir(obj): + if attr is None and name.startswith("_"): + continue + subobj = getattr(obj, name) + if isinstance(subobj, types.TypeType): + kind = KINDS["TYPE"] + else: + kind = KINDS["UNKNOWN"] + print kind, ":", name + +if __name__ == "__main__": + main(sys.argv) \ No newline at end of file diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -371,6 +371,8 @@ if hasattr(__import__(os.name), "forkpty"): def test_forkpty(self): import sys + if 'freebsd' in sys.platform: + skip("hangs indifinitly on FreeBSD (also on CPython).") os = self.posix childpid, master_fd = os.forkpty() assert isinstance(childpid, int) diff --git a/pypy/module/test_lib_pypy/test_sqlite3.py b/pypy/module/test_lib_pypy/test_sqlite3.py new file mode 100644 --- /dev/null +++ b/pypy/module/test_lib_pypy/test_sqlite3.py @@ -0,0 +1,10 @@ +"""Tests for _sqlite3.py""" + +def test_list_ddl(): + """From issue996. Mostly just looking for lack of exceptions.""" + from sqlite3.dbapi2 import connect + connection = connect(':memory:') + cursor = connection.cursor() + cursor.execute('CREATE TABLE foo (bar INTEGER)') + result = list(cursor) + assert result == [] diff --git a/pypy/rlib/rdynload.py b/pypy/rlib/rdynload.py --- a/pypy/rlib/rdynload.py +++ b/pypy/rlib/rdynload.py @@ -14,7 +14,7 @@ _MINGW = platform.name == "mingw32" _WIN32 = _MSVC or _MINGW _MAC_OS = platform.name == "darwin" -_FREEBSD = platform.name == "freebsd" +_FREEBSD = sys.platform.startswith("freebsd") if _WIN32: from pypy.rlib import rwin32 diff --git a/pypy/rlib/rsocket.py b/pypy/rlib/rsocket.py --- a/pypy/rlib/rsocket.py +++ b/pypy/rlib/rsocket.py @@ -310,10 +310,7 @@ def from_object(space, w_address): # Parse an app-level object representing an AF_INET address - try: - w_host, w_port = space.unpackiterable(w_address, 2) - except ValueError: - raise TypeError("AF_INET address must be a tuple of length 2") + w_host, w_port = space.unpackiterable(w_address, 2) host = space.str_w(w_host) port = space.int_w(w_port) port = Address.make_ushort_port(space, port) @@ -544,10 +541,7 @@ space.wrap(self.get_groups())]) def from_object(space, w_address): - try: - w_pid, w_groups = space.unpackiterable(w_address, 2) - except ValueError: - raise TypeError("AF_NETLINK address must be a tuple of length 2") + w_pid, w_groups = space.unpackiterable(w_address, 2) return NETLINKAddress(space.uint_w(w_pid), space.uint_w(w_groups)) from_object = staticmethod(from_object) diff --git a/pypy/rlib/rweakref.py b/pypy/rlib/rweakref.py --- a/pypy/rlib/rweakref.py +++ b/pypy/rlib/rweakref.py @@ -1,11 +1,14 @@ """ -Weakref support in RPython. Supports ref() without callbacks, +Weakref support in RPython. Basic regular weakrefs without callbacks +are supported. This file contains the following additions: a form of WeakKeyDictionary, and a limited version of WeakValueDictionary. LLType only for now! """ import weakref +ref = weakref.ref # basic regular weakrefs are supported in RPython + class RWeakValueDictionary(object): """A dictionary containing weak values.""" diff --git a/pypy/rpython/lltypesystem/ll2ctypes.py b/pypy/rpython/lltypesystem/ll2ctypes.py --- a/pypy/rpython/lltypesystem/ll2ctypes.py +++ b/pypy/rpython/lltypesystem/ll2ctypes.py @@ -1390,7 +1390,8 @@ def _where_is_errno(): return standard_c_lib.__errno_location() - elif sys.platform in ('darwin', 'freebsd7', 'freebsd8', 'freebsd9'): + elif any(plat in sys.platform + for plat in ('darwin', 'freebsd7', 'freebsd8', 'freebsd9')): standard_c_lib.__error.restype = ctypes.POINTER(ctypes.c_int) def _where_is_errno(): return standard_c_lib.__error() diff --git a/pypy/rpython/memory/gc/env.py b/pypy/rpython/memory/gc/env.py --- a/pypy/rpython/memory/gc/env.py +++ b/pypy/rpython/memory/gc/env.py @@ -116,7 +116,7 @@ def get_total_memory(): return get_total_memory_darwin(get_darwin_sysctl_signed('hw.memsize')) -elif sys.platform.startswith('freebsd'): +elif 'freebsd' in sys.platform: def get_total_memory(): return get_total_memory_darwin(get_darwin_sysctl_signed('hw.usermem')) diff --git a/pypy/rpython/module/ll_time.py b/pypy/rpython/module/ll_time.py --- a/pypy/rpython/module/ll_time.py +++ b/pypy/rpython/module/ll_time.py @@ -41,7 +41,7 @@ RUSAGE = platform.Struct('struct rusage', [('ru_utime', TIMEVAL), ('ru_stime', TIMEVAL)]) -if "freebsd" in sys.platform: +if sys.platform.startswith('freebsd'): libraries = ['compat'] else: libraries = [] diff --git a/pypy/translator/platform/freebsd.py b/pypy/translator/platform/freebsd.py --- a/pypy/translator/platform/freebsd.py +++ b/pypy/translator/platform/freebsd.py @@ -18,8 +18,9 @@ class Freebsd(posix.BasePosix): name = "freebsd" - link_flags = get_env_vector("LDFLAGS", '-pthread') - cflags = get_env_vector("CFLAGS", "-O3 -pthread -fomit-frame-pointer") + link_flags = ['-pthread'] + get_env_vector('LDFLAGS', '') + cflags = ['-O3', '-pthread', '-fomit-frame-pointer' + ] + get_env_vector('CFLAGS', '') standalone_only = [] shared_only = [] so_ext = 'so' diff --git a/pypy/translator/platform/linux.py b/pypy/translator/platform/linux.py --- a/pypy/translator/platform/linux.py +++ b/pypy/translator/platform/linux.py @@ -1,15 +1,20 @@ """Support for Linux.""" +import os import sys from pypy.translator.platform.posix import BasePosix class BaseLinux(BasePosix): name = "linux" - link_flags = ('-pthread',) + link_flags = tuple( + ['-pthread',] + + os.environ.get('LDFLAGS', '').split()) extra_libs = ('-lrt',) - cflags = ('-O3', '-pthread', '-fomit-frame-pointer', - '-Wall', '-Wno-unused') + cflags = tuple( + ['-O3', '-pthread', '-fomit-frame-pointer', + '-Wall', '-Wno-unused'] + + os.environ.get('CFLAGS', '').split()) standalone_only = () shared_only = ('-fPIC',) so_ext = 'so' From noreply at buildbot.pypy.org Tue Jan 24 19:44:56 2012 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 24 Jan 2012 19:44:56 +0100 (CET) Subject: [pypy-commit] pypy numpy-back-to-applevel: hooray applevel print works Message-ID: <20120124184456.9F56F82CFD@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: numpy-back-to-applevel Changeset: r51740:f11a97418c3f Date: 2012-01-24 20:44 +0200 http://bitbucket.org/pypy/pypy/changeset/f11a97418c3f/ Log: hooray applevel print works diff --git a/lib_pypy/numpypy/core/arrayprint.py b/lib_pypy/numpypy/core/arrayprint.py --- a/lib_pypy/numpypy/core/arrayprint.py +++ b/lib_pypy/numpypy/core/arrayprint.py @@ -14,8 +14,8 @@ import sys import _numpypy as _nt -from _numpypy import maximum, minimum, absolute, not_equal, isinf, isnan -#from _numpypy import format_longfloat, datetime_as_string, datetime_data, isna +from _numpypy import maximum, minimum, absolute, not_equal, isinf, isnan, isna +#from _numpypy import format_longfloat, datetime_as_string, datetime_data from .fromnumeric import ravel diff --git a/lib_pypy/numpypy/core/numeric.py b/lib_pypy/numpypy/core/numeric.py --- a/lib_pypy/numpypy/core/numeric.py +++ b/lib_pypy/numpypy/core/numeric.py @@ -180,7 +180,8 @@ skipdtype = (arr.dtype.type in _typelessdata) and arr.size > 0 - if arr.flags.maskna: + # XXX pypy lacks support + if 0 and arr.flags.maskna: whichna = isna(arr) # If nothing is NA, explicitly signal the NA-mask if not any(whichna): From noreply at buildbot.pypy.org Tue Jan 24 19:57:41 2012 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 24 Jan 2012 19:57:41 +0100 (CET) Subject: [pypy-commit] pypy numpy-back-to-applevel: move test and make it pass on CPython. besides, trick numpy into believing it's Message-ID: <20120124185741.7375582CFD@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: numpy-back-to-applevel Changeset: r51741:838a81530121 Date: 2012-01-24 20:57 +0200 http://bitbucket.org/pypy/pypy/changeset/838a81530121/ Log: move test and make it pass on CPython. besides, trick numpy into believing it's numpypy when running with -A on CPython diff --git a/lib_pypy/numpypy/core/arrayprint.py b/lib_pypy/numpypy/core/arrayprint.py --- a/lib_pypy/numpypy/core/arrayprint.py +++ b/lib_pypy/numpypy/core/arrayprint.py @@ -305,10 +305,10 @@ #else: format_function = formatdict['int'] elif issubclass(dtypeobj, _nt.floating): - if issubclass(dtypeobj, _nt.longfloat): - format_function = formatdict['longfloat'] - else: - format_function = formatdict['float'] + #if issubclass(dtypeobj, _nt.longfloat): + # format_function = formatdict['longfloat'] + #else: + format_function = formatdict['float'] elif issubclass(dtypeobj, _nt.complexfloating): if issubclass(dtypeobj, _nt.clongfloat): format_function = formatdict['longcomplexfloat'] @@ -610,7 +610,7 @@ def __call__(self, x, strip_zeros=True): import numeric as _nc - err = _nc.seterr(invalid='ignore') + #err = _nc.seterr(invalid='ignore') try: if isna(x): return self.special_fmt % (str(x).replace('NA', _na_str, 1),) @@ -628,7 +628,8 @@ else: return self.special_fmt % ('-' + _inf_str,) finally: - _nc.seterr(**err) + pass + #_nc.seterr(**err) s = self.format % x if self.large_exponent: diff --git a/pypy/module/micronumpy/test/test_base.py b/pypy/module/micronumpy/test/test_base.py --- a/pypy/module/micronumpy/test/test_base.py +++ b/pypy/module/micronumpy/test/test_base.py @@ -3,9 +3,15 @@ from pypy.module.micronumpy.interp_numarray import W_NDimArray, Scalar from pypy.module.micronumpy.interp_ufuncs import (find_binop_result_dtype, find_unaryop_result_dtype) +from pypy.conftest import option +import sys class BaseNumpyAppTest(object): def setup_class(cls): + if option.runappdirect: + if '__pypy__' not in sys.builtin_module_names: + import numpy + sys.modules['numpypy'] = numpy cls.space = gettestobjspace(usemodules=['micronumpy']) class TestSignature(object): diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -1513,128 +1513,6 @@ raises(ValueError, fromstring, "\x01\x02\x03", count=5, dtype=uint8) -class AppTestRepr(BaseNumpyAppTest): - def test_repr(self): - from _numpypy import array, zeros - int_size = array(5).dtype.itemsize - a = array(range(5), float) - assert repr(a) == "array([0.0, 1.0, 2.0, 3.0, 4.0])" - a = array([], float) - assert repr(a) == "array([], dtype=float64)" - a = zeros(1001) - assert repr(a) == "array([0.0, 0.0, 0.0, ..., 0.0, 0.0, 0.0])" - a = array(range(5), long) - if a.dtype.itemsize == int_size: - assert repr(a) == "array([0, 1, 2, 3, 4])" - else: - assert repr(a) == "array([0, 1, 2, 3, 4], dtype=int64)" - a = array(range(5), 'int32') - if a.dtype.itemsize == int_size: - assert repr(a) == "array([0, 1, 2, 3, 4])" - else: - assert repr(a) == "array([0, 1, 2, 3, 4], dtype=int32)" - a = array([], long) - assert repr(a) == "array([], dtype=int64)" - a = array([True, False, True, False], "?") - assert repr(a) == "array([True, False, True, False], dtype=bool)" - a = zeros([]) - assert repr(a) == "array(0.0)" - a = array(0.2) - assert repr(a) == "array(0.2)" - a = array([2]) - assert repr(a) == "array([2])" - - def test_repr_multi(self): - from _numpypy import arange, zeros, array - a = zeros((3, 4)) - assert repr(a) == '''array([[0.0, 0.0, 0.0, 0.0], - [0.0, 0.0, 0.0, 0.0], - [0.0, 0.0, 0.0, 0.0]])''' - a = zeros((2, 3, 4)) - assert repr(a) == '''array([[[0.0, 0.0, 0.0, 0.0], - [0.0, 0.0, 0.0, 0.0], - [0.0, 0.0, 0.0, 0.0]], - - [[0.0, 0.0, 0.0, 0.0], - [0.0, 0.0, 0.0, 0.0], - [0.0, 0.0, 0.0, 0.0]]])''' - a = arange(1002).reshape((2, 501)) - assert repr(a) == '''array([[0, 1, 2, ..., 498, 499, 500], - [501, 502, 503, ..., 999, 1000, 1001]])''' - assert repr(a.T) == '''array([[0, 501], - [1, 502], - [2, 503], - ..., - [498, 999], - [499, 1000], - [500, 1001]])''' - a = arange(2).reshape((2,1)) - assert repr(a) == '''array([[0], - [1]])''' - - def test_repr_slice(self): - from _numpypy import array, zeros - a = array(range(5), float) - b = a[1::2] - assert repr(b) == "array([1.0, 3.0])" - a = zeros(2002) - b = a[::2] - assert repr(b) == "array([0.0, 0.0, 0.0, ..., 0.0, 0.0, 0.0])" - a = array((range(5), range(5, 10)), dtype="int16") - b = a[1, 2:] - assert repr(b) == "array([7, 8, 9], dtype=int16)" - # an empty slice prints its shape - b = a[2:1, ] - assert repr(b) == "array([], shape=(0, 5), dtype=int16)" - - def test_str(self): - from _numpypy import array, zeros - a = array(range(5), float) - assert str(a) == "[0.0 1.0 2.0 3.0 4.0]" - assert str((2 * a)[:]) == "[0.0 2.0 4.0 6.0 8.0]" - a = zeros(1001) - assert str(a) == "[0.0 0.0 0.0 ..., 0.0 0.0 0.0]" - - a = array(range(5), dtype=long) - assert str(a) == "[0 1 2 3 4]" - a = array([True, False, True, False], dtype="?") - assert str(a) == "[True False True False]" - - a = array(range(5), dtype="int8") - assert str(a) == "[0 1 2 3 4]" - - a = array(range(5), dtype="int16") - assert str(a) == "[0 1 2 3 4]" - - a = array((range(5), range(5, 10)), dtype="int16") - assert str(a) == "[[0 1 2 3 4]\n [5 6 7 8 9]]" - - a = array(3, dtype=int) - assert str(a) == "3" - - a = zeros((400, 400), dtype=int) - assert str(a) == "[[0 0 0 ..., 0 0 0]\n [0 0 0 ..., 0 0 0]\n" \ - " [0 0 0 ..., 0 0 0]\n ...,\n [0 0 0 ..., 0 0 0]\n" \ - " [0 0 0 ..., 0 0 0]\n [0 0 0 ..., 0 0 0]]" - a = zeros((2, 2, 2)) - r = str(a) - assert r == '[[[0.0 0.0]\n [0.0 0.0]]\n\n [[0.0 0.0]\n [0.0 0.0]]]' - - def test_str_slice(self): - from _numpypy import array, zeros - a = array(range(5), float) - b = a[1::2] - assert str(b) == "[1.0 3.0]" - a = zeros(2002) - b = a[::2] - assert str(b) == "[0.0 0.0 0.0 ..., 0.0 0.0 0.0]" - a = array((range(5), range(5, 10)), dtype="int16") - b = a[1, 2:] - assert str(b) == "[7 8 9]" - b = a[2:1, ] - assert str(b) == "[]" - - class AppTestRanges(BaseNumpyAppTest): def test_arange(self): from _numpypy import arange, array, dtype diff --git a/pypy/module/test_lib_pypy/numpypy/core/test_numeric.py b/pypy/module/test_lib_pypy/numpypy/core/test_numeric.py --- a/pypy/module/test_lib_pypy/numpypy/core/test_numeric.py +++ b/pypy/module/test_lib_pypy/numpypy/core/test_numeric.py @@ -24,3 +24,121 @@ def test_repr(self): from numpypy import array assert repr(array([1, 2, 3, 4])) == 'array([1, 2, 3, 4])' + + def test_repr_2(self): + from numpypy import array, zeros + int_size = array(5).dtype.itemsize + a = array(range(5), float) + assert repr(a) == "array([ 0., 1., 2., 3., 4.])" + a = array([], float) + assert repr(a) == "array([], dtype=float64)" + a = zeros(1001) + assert repr(a) == "array([ 0., 0., 0., ..., 0., 0., 0.])" + a = array(range(5), long) + if a.dtype.itemsize == int_size: + assert repr(a) == "array([0, 1, 2, 3, 4])" + else: + assert repr(a) == "array([0, 1, 2, 3, 4], dtype=int64)" + a = array(range(5), 'int32') + if a.dtype.itemsize == int_size: + assert repr(a) == "array([0, 1, 2, 3, 4])" + else: + assert repr(a) == "array([0, 1, 2, 3, 4], dtype=int32)" + a = array([], long) + assert repr(a) == "array([], dtype=int64)" + a = array([True, False, True, False], "?") + assert repr(a) == "array([ True, False, True, False], dtype=bool)" + a = zeros([]) + assert repr(a) == "array(0.0)" + a = array(0.2) + assert repr(a) == "array(0.2)" + a = array([2]) + assert repr(a) == "array([2])" + + def test_repr_multi(self): + from numpypy import arange, zeros, array + a = zeros((3, 4)) + assert repr(a) == '''array([[ 0., 0., 0., 0.], + [ 0., 0., 0., 0.], + [ 0., 0., 0., 0.]])''' + a = zeros((2, 3, 4)) + assert repr(a) == '''array([[[ 0., 0., 0., 0.], + [ 0., 0., 0., 0.], + [ 0., 0., 0., 0.]], + + [[ 0., 0., 0., 0.], + [ 0., 0., 0., 0.], + [ 0., 0., 0., 0.]]])''' + a = arange(1002).reshape((2, 501)) + assert repr(a) == '''array([[ 0, 1, 2, ..., 498, 499, 500], + [ 501, 502, 503, ..., 999, 1000, 1001]])''' + assert repr(a.T) == '''array([[ 0, 501], + [ 1, 502], + [ 2, 503], + ..., + [ 498, 999], + [ 499, 1000], + [ 500, 1001]])''' + a = arange(2).reshape((2,1)) + assert repr(a) == '''array([[0], + [1]])''' + + def test_repr_slice(self): + from numpypy import array, zeros + a = array(range(5), float) + b = a[1::2] + assert repr(b) == "array([ 1., 3.])" + a = zeros(2002) + b = a[::2] + assert repr(b) == "array([ 0., 0., 0., ..., 0., 0., 0.])" + a = array((range(5), range(5, 10)), dtype="int16") + b = a[1, 2:] + assert repr(b) == "array([7, 8, 9], dtype=int16)" + # an empty slice prints its shape + b = a[2:1, ] + assert repr(b) == "array([], shape=(0, 5), dtype=int16)" + + def test_str(self): + from numpypy import array, zeros + a = array(range(5), float) + assert str(a) == "[ 0. 1. 2. 3. 4.]" + assert str((2 * a)[:]) == "[ 0. 2. 4. 6. 8.]" + a = zeros(1001) + assert str(a) == "[ 0. 0. 0. ..., 0. 0. 0.]" + + a = array(range(5), dtype=long) + assert str(a) == "[0 1 2 3 4]" + a = array([True, False, True, False], dtype="?") + assert str(a) == "[ True False True False]" + + a = array(range(5), dtype="int8") + assert str(a) == "[0 1 2 3 4]" + + a = array(range(5), dtype="int16") + assert str(a) == "[0 1 2 3 4]" + + a = array((range(5), range(5, 10)), dtype="int16") + assert str(a) == "[[0 1 2 3 4]\n [5 6 7 8 9]]" + + a = array(3, dtype=int) + assert str(a) == "3" + + a = zeros((400, 400), dtype=int) + assert str(a) == '[[0 0 0 ..., 0 0 0]\n [0 0 0 ..., 0 0 0]\n [0 0 0 ..., 0 0 0]\n ..., \n [0 0 0 ..., 0 0 0]\n [0 0 0 ..., 0 0 0]\n [0 0 0 ..., 0 0 0]]' + a = zeros((2, 2, 2)) + r = str(a) + assert r == '[[[ 0. 0.]\n [ 0. 0.]]\n\n [[ 0. 0.]\n [ 0. 0.]]]' + + def test_str_slice(self): + from numpypy import array, zeros + a = array(range(5), float) + b = a[1::2] + assert str(b) == "[ 1. 3.]" + a = zeros(2002) + b = a[::2] + assert str(b) == "[ 0. 0. 0. ..., 0. 0. 0.]" + a = array((range(5), range(5, 10)), dtype="int16") + b = a[1, 2:] + assert str(b) == "[7 8 9]" + b = a[2:1, ] + assert str(b) == "[]" From noreply at buildbot.pypy.org Tue Jan 24 20:43:16 2012 From: noreply at buildbot.pypy.org (stefanor) Date: Tue, 24 Jan 2012 20:43:16 +0100 (CET) Subject: [pypy-commit] pypy default: Avoid a AF_UNIX path that's too long Message-ID: <20120124194316.E8E6D82CFD@wyvern.cs.uni-duesseldorf.de> Author: Stefano Rivera <stefano at rivera.za.net> Branch: Changeset: r51742:522114ac6c4a Date: 2012-01-24 21:42 +0200 http://bitbucket.org/pypy/pypy/changeset/522114ac6c4a/ Log: Avoid a AF_UNIX path that's too long AppTestSocket.test_unix_socket_connect tries to connect to a socket within TMPDIR. TMPDIR can be quite long, and socket.bind() is length limited, so chdir and use a relative path. diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py --- a/pypy/module/_socket/test/test_sock_app.py +++ b/pypy/module/_socket/test/test_sock_app.py @@ -529,26 +529,31 @@ import _socket, os if not hasattr(_socket, 'AF_UNIX'): skip('AF_UNIX not supported.') - sockpath = os.path.join(self.udir, 'app_test_unix_socket_connect') + oldcwd = os.getcwd() + os.chdir(self.udir) + try: + sockpath = 'app_test_unix_socket_connect' - serversock = _socket.socket(_socket.AF_UNIX) - serversock.bind(sockpath) - serversock.listen(1) + serversock = _socket.socket(_socket.AF_UNIX) + serversock.bind(sockpath) + serversock.listen(1) - clientsock = _socket.socket(_socket.AF_UNIX) - clientsock.connect(sockpath) - s, addr = serversock.accept() - assert not addr + clientsock = _socket.socket(_socket.AF_UNIX) + clientsock.connect(sockpath) + s, addr = serversock.accept() + assert not addr - s.send('X') - data = clientsock.recv(100) - assert data == 'X' - clientsock.send('Y') - data = s.recv(100) - assert data == 'Y' + s.send('X') + data = clientsock.recv(100) + assert data == 'X' + clientsock.send('Y') + data = s.recv(100) + assert data == 'Y' - clientsock.close() - s.close() + clientsock.close() + s.close() + finally: + os.chdir(oldcwd) class AppTestSocketTCP: From noreply at buildbot.pypy.org Tue Jan 24 20:46:21 2012 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 24 Jan 2012 20:46:21 +0100 (CET) Subject: [pypy-commit] pypy numpy-back-to-applevel: inversion and don't swallow exceptions too much Message-ID: <20120124194621.8C8E882CFD@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: numpy-back-to-applevel Changeset: r51743:6a5df7612571 Date: 2012-01-24 21:45 +0200 http://bitbucket.org/pypy/pypy/changeset/6a5df7612571/ Log: inversion and don't swallow exceptions too much diff --git a/lib_pypy/numpypy/core/arrayprint.py b/lib_pypy/numpypy/core/arrayprint.py --- a/lib_pypy/numpypy/core/arrayprint.py +++ b/lib_pypy/numpypy/core/arrayprint.py @@ -542,12 +542,12 @@ self.exp_format = False self.large_exponent = False self.max_str_len = 0 - try: - self.fillFormat(data) - except (TypeError, NotImplementedError): + #try: + self.fillFormat(data) + #except (TypeError, NotImplementedError): # if reduce(data) fails, this instance will not be called, just # instantiated in formatdict. - pass + #pass def fillFormat(self, data): import numeric as _nc diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py --- a/pypy/module/micronumpy/__init__.py +++ b/pypy/module/micronumpy/__init__.py @@ -94,6 +94,7 @@ ("tan", "tan"), ('bitwise_and', 'bitwise_and'), ('bitwise_or', 'bitwise_or'), + ('bitwise_not', 'invert'), ('isnan', 'isnan'), ('isinf', 'isinf'), ]: diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py --- a/pypy/module/micronumpy/interp_dtype.py +++ b/pypy/module/micronumpy/interp_dtype.py @@ -90,7 +90,8 @@ return space.newtuple([]) def is_int_type(self): - return self.kind == SIGNEDLTR or self.kind == UNSIGNEDLTR + return (self.kind == SIGNEDLTR or self.kind == UNSIGNEDLTR or + self.kind == BOOLLTR) def is_bool_type(self): return self.kind == BOOLLTR diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -100,6 +100,7 @@ descr_pos = _unaryop_impl("positive") descr_neg = _unaryop_impl("negative") descr_abs = _unaryop_impl("absolute") + descr_invert = _unaryop_impl("invert") def _binop_impl(ufunc_name): def impl(self, space, w_other): @@ -1277,6 +1278,7 @@ __and__ = interp2app(BaseArray.descr_and), __or__ = interp2app(BaseArray.descr_or), + __invert__ = interp2app(BaseArray.descr_invert), __repr__ = interp2app(BaseArray.descr_repr), __str__ = interp2app(BaseArray.descr_str), diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py --- a/pypy/module/micronumpy/interp_ufuncs.py +++ b/pypy/module/micronumpy/interp_ufuncs.py @@ -30,12 +30,14 @@ _attrs_ = ["name", "promote_to_float", "promote_bools", "identity"] _immutable_fields_ = ["promote_to_float", "promote_bools", "name"] - def __init__(self, name, promote_to_float, promote_bools, identity): + def __init__(self, name, promote_to_float, promote_bools, identity, + int_only): self.name = name self.promote_to_float = promote_to_float self.promote_bools = promote_bools self.identity = identity + self.int_only = int_only def descr_repr(self, space): return space.wrap("<ufunc '%s'>" % self.name) @@ -248,9 +250,10 @@ _immutable_fields_ = ["func", "name"] def __init__(self, func, name, promote_to_float=False, promote_bools=False, - identity=None, bool_result=False): + identity=None, bool_result=False, int_only=False): - W_Ufunc.__init__(self, name, promote_to_float, promote_bools, identity) + W_Ufunc.__init__(self, name, promote_to_float, promote_bools, identity, + int_only) self.func = func self.bool_result = bool_result @@ -284,10 +287,10 @@ def __init__(self, func, name, promote_to_float=False, promote_bools=False, identity=None, comparison_func=False, int_only=False): - W_Ufunc.__init__(self, name, promote_to_float, promote_bools, identity) + W_Ufunc.__init__(self, name, promote_to_float, promote_bools, identity, + int_only) self.func = func self.comparison_func = comparison_func - self.int_only = int_only def call(self, space, args_w): from pypy.module.micronumpy.interp_numarray import (Call2, @@ -466,6 +469,7 @@ 'int_only': True}), ("bitwise_or", "bitwise_or", 2, {"identity": 0, 'int_only': True}), + ("invert", "invert", 1, {"int_only": True}), ("divide", "div", 2, {"promote_bools": True}), ("true_divide", "div", 2, {"promote_to_float": True}), ("mod", "mod", 2, {"promote_bools": True}), diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py --- a/pypy/module/micronumpy/test/test_ufuncs.py +++ b/pypy/module/micronumpy/test/test_ufuncs.py @@ -377,6 +377,12 @@ assert (a | 1 == bitwise_or(a, 1)).all() raises(TypeError, 'array([1.0]) & 1') + def test_unary_bitops(self): + from _numpypy import bitwise_not, array + a = array([1, 2, 3, 4]) + assert (~a == [-2, -3, -4, -5]).all() + assert (bitwise_not(a) == ~a).all() + def test_comparisons(self): import operator from _numpypy import equal, not_equal, less, less_equal, greater, greater_equal diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py --- a/pypy/module/micronumpy/types.py +++ b/pypy/module/micronumpy/types.py @@ -189,7 +189,7 @@ @simple_binary_op def min(self, v1, v2): return min(v1, v2) - + class Bool(BaseType, Primitive): T = lltype.Bool @@ -232,6 +232,20 @@ def default_fromstring(self, space): return self.box(False) + # XXX check rpythonization + + @simple_binary_op + def bitwise_and(self, v1, v2): + return v1 & v2 + + @simple_binary_op + def bitwise_or(self, v1, v2): + return v1 | v2 + + @simple_unary_op + def invert(self, v): + return ~v + class Integer(Primitive): _mixin_ = True @@ -288,6 +302,10 @@ def bitwise_or(self, v1, v2): return v1 | v2 + @simple_unary_op + def invert(self, v): + return ~v + class Int8(BaseType, Integer): T = rffi.SIGNEDCHAR BoxType = interp_boxes.W_Int8Box From noreply at buildbot.pypy.org Tue Jan 24 22:16:42 2012 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 24 Jan 2012 22:16:42 +0100 (CET) Subject: [pypy-commit] pypy default: Test and fix for "Ambiguous low-level helper specialization" when Message-ID: <20120124211642.2A3EB82CFD@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: Changeset: r51744:80d054b4615d Date: 2012-01-24 22:07 +0100 http://bitbucket.org/pypy/pypy/changeset/80d054b4615d/ Log: Test and fix for "Ambiguous low-level helper specialization" when the same RPython program contains several calls to start_new_thread(). diff --git a/pypy/module/thread/ll_thread.py b/pypy/module/thread/ll_thread.py --- a/pypy/module/thread/ll_thread.py +++ b/pypy/module/thread/ll_thread.py @@ -4,7 +4,7 @@ import py from pypy.rlib import jit, rgc from pypy.rlib.debug import ll_assert -from pypy.rlib.objectmodel import we_are_translated +from pypy.rlib.objectmodel import we_are_translated, specialize from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.tool import rffi_platform from pypy.tool import autopath @@ -85,6 +85,7 @@ def allocate_lock(): return Lock(allocate_ll_lock()) + at specialize.arg(0) def ll_start_new_thread(func): ident = c_thread_start(func) if ident == -1: @@ -97,6 +98,7 @@ def get_ident(): return rffi.cast(lltype.Signed, c_thread_get_ident()) + at specialize.arg(0) def start_new_thread(x, y): """In RPython, no argument can be passed. You have to use global variables to pass information to the new thread. That's not very diff --git a/pypy/module/thread/test/test_ll_thread.py b/pypy/module/thread/test/test_ll_thread.py --- a/pypy/module/thread/test/test_ll_thread.py +++ b/pypy/module/thread/test/test_ll_thread.py @@ -34,72 +34,34 @@ use_threads = True def test_start_new_thread(self): - py.test.skip("xxx ideally, investigate why it fails randomly") - # xxx but in practice start_new_thread() is also tested by the - # next test, and it's a mess to test start_new_thread() without - # the proper GIL to protect the GC import time class State: pass state = State() - class Z: - def __init__(self, value): - self.value = value - def __del__(self): - state.freed_counter += 1 - - def bootstrap(): - state.my_thread_ident = state.z.ident = get_ident() - assert state.my_thread_ident == get_ident() - assert get_ident() == state.z.ident - state.seen_value = state.z.value - state.z = None - # I think that we would need here a memory barrier in order - # to make the test pass reliably. The issue is that the - # main thread may see 'state.done = 1' before seeing the - # effect of the other assignments done above. For now let's - # emulate the write barrier by doing a system call and - # waiting a bit... - time.sleep(0.012) - state.done = 1 - - def g(i): - state.z = Z(i) - return start_new_thread(bootstrap, ()) - g._dont_inline_ = True + def bootstrap1(): + state.my_thread_ident1 = get_ident() + def bootstrap2(): + state.my_thread_ident2 = get_ident() def f(): - main_ident = get_ident() - assert main_ident == get_ident() - state.freed_counter = 0 - for i in range(50): - state.done = 0 - state.seen_value = 0 - ident = g(i) - gc.collect() - willing_to_wait_more = 1000 - while not state.done: - willing_to_wait_more -= 1 - if not willing_to_wait_more: - raise Exception("thread didn't start?") - time.sleep(0.01) - assert state.my_thread_ident != main_ident - assert state.my_thread_ident == ident - assert state.seen_value == i - # try to force Boehm to do some freeing - for i in range(3): - gc.collect() - return state.freed_counter + state.my_thread_ident1 = get_ident() + state.my_thread_ident2 = get_ident() + start_new_thread(bootstrap1, ()) + start_new_thread(bootstrap2, ()) + willing_to_wait_more = 1000 + while (state.my_thread_ident1 == get_ident() or + state.my_thread_ident2 == get_ident()): + willing_to_wait_more -= 1 + if not willing_to_wait_more: + raise Exception("thread didn't start?") + time.sleep(0.01) + return 42 fn = self.getcompiled(f, []) - freed_counter = fn() - print freed_counter - if self.gcpolicy == 'boehm': - assert freed_counter > 0 - else: - assert freed_counter == 50 + res = fn() + assert res == 42 def test_gc_locking(self): import time From noreply at buildbot.pypy.org Tue Jan 24 22:16:43 2012 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 24 Jan 2012 22:16:43 +0100 (CET) Subject: [pypy-commit] pypy default: Deadlocks occasionally -- why??? Message-ID: <20120124211643.5731F82CFD@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: Changeset: r51745:29a811af16dc Date: 2012-01-24 22:13 +0100 http://bitbucket.org/pypy/pypy/changeset/29a811af16dc/ Log: Deadlocks occasionally -- why??? diff --git a/pypy/module/thread/test/test_ll_thread.py b/pypy/module/thread/test/test_ll_thread.py --- a/pypy/module/thread/test/test_ll_thread.py +++ b/pypy/module/thread/test/test_ll_thread.py @@ -164,6 +164,9 @@ def getcompiled(self, f, argtypes): return f + def test_start_new_thread(self): + py.test.skip("deadlocks occasionally -- why???") + class TestUsingBoehm(AbstractThreadTests): gcpolicy = 'boehm' From noreply at buildbot.pypy.org Tue Jan 24 22:16:46 2012 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 24 Jan 2012 22:16:46 +0100 (CET) Subject: [pypy-commit] pypy stm: hg merge default Message-ID: <20120124211646.A117182CFD@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: stm Changeset: r51746:776034ae21ee Date: 2012-01-24 22:16 +0100 http://bitbucket.org/pypy/pypy/changeset/776034ae21ee/ Log: hg merge default diff --git a/.hgignore b/.hgignore --- a/.hgignore +++ b/.hgignore @@ -2,6 +2,9 @@ *.py[co] *~ .*.swp +.idea +.project +.pydevproject syntax: regexp ^testresult$ diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py --- a/lib_pypy/_sqlite3.py +++ b/lib_pypy/_sqlite3.py @@ -20,6 +20,8 @@ # 2. Altered source versions must be plainly marked as such, and must not be # misrepresented as being the original software. # 3. This notice may not be removed or altered from any source distribution. +# +# Note: This software has been modified for use in PyPy. from ctypes import c_void_p, c_int, c_double, c_int64, c_char_p, cdll from ctypes import POINTER, byref, string_at, CFUNCTYPE, cast @@ -27,7 +29,6 @@ from collections import OrderedDict import datetime import sys -import time import weakref from threading import _get_ident as thread_get_ident @@ -606,7 +607,7 @@ def authorizer(userdata, action, arg1, arg2, dbname, source): try: return int(callback(action, arg1, arg2, dbname, source)) - except Exception, e: + except Exception: return SQLITE_DENY c_authorizer = AUTHORIZER(authorizer) @@ -653,7 +654,7 @@ if not aggregate_ptr[0]: try: aggregate = cls() - except Exception, e: + except Exception: msg = ("user-defined aggregate's '__init__' " "method raised error") sqlite.sqlite3_result_error(context, msg, len(msg)) @@ -667,7 +668,7 @@ params = _convert_params(context, argc, c_params) try: aggregate.step(*params) - except Exception, e: + except Exception: msg = ("user-defined aggregate's 'step' " "method raised error") sqlite.sqlite3_result_error(context, msg, len(msg)) @@ -683,7 +684,7 @@ aggregate = self.aggregate_instances[aggregate_ptr[0]] try: val = aggregate.finalize() - except Exception, e: + except Exception: msg = ("user-defined aggregate's 'finalize' " "method raised error") sqlite.sqlite3_result_error(context, msg, len(msg)) @@ -771,7 +772,7 @@ self.statement.item = None self.statement.exhausted = True - if self.statement.kind == DML or self.statement.kind == DDL: + if self.statement.kind == DML: self.statement.reset() self.rowcount = -1 @@ -791,7 +792,7 @@ if self.statement.kind == DML: self.connection._begin() else: - raise ProgrammingError, "executemany is only for DML statements" + raise ProgrammingError("executemany is only for DML statements") self.rowcount = 0 for params in many_params: @@ -861,8 +862,6 @@ except StopIteration: return None - return nextrow - def fetchmany(self, size=None): self._check_closed() self._check_reset() @@ -915,7 +914,7 @@ def __init__(self, connection, sql): self.statement = None if not isinstance(sql, str): - raise ValueError, "sql must be a string" + raise ValueError("sql must be a string") self.con = connection self.sql = sql # DEBUG ONLY first_word = self._statement_kind = sql.lstrip().split(" ")[0].upper() @@ -944,8 +943,8 @@ raise self.con._get_exception(ret) self.con._remember_statement(self) if _check_remaining_sql(next_char.value): - raise Warning, "One and only one statement required: %r" % ( - next_char.value,) + raise Warning("One and only one statement required: %r" % ( + next_char.value,)) # sql_char should remain alive until here self._build_row_cast_map() @@ -1016,7 +1015,7 @@ elif type(param) is buffer: sqlite.sqlite3_bind_blob(self.statement, idx, str(param), len(param), SQLITE_TRANSIENT) else: - raise InterfaceError, "parameter type %s is not supported" % str(type(param)) + raise InterfaceError("parameter type %s is not supported" % str(type(param))) def set_params(self, params): ret = sqlite.sqlite3_reset(self.statement) @@ -1045,11 +1044,11 @@ for idx in range(1, sqlite.sqlite3_bind_parameter_count(self.statement) + 1): param_name = sqlite.sqlite3_bind_parameter_name(self.statement, idx) if param_name is None: - raise ProgrammingError, "need named parameters" + raise ProgrammingError("need named parameters") param_name = param_name[1:] try: param = params[param_name] - except KeyError, e: + except KeyError: raise ProgrammingError("missing parameter '%s'" %param) self.set_param(idx, param) @@ -1260,7 +1259,7 @@ params = _convert_params(context, nargs, c_params) try: val = real_cb(*params) - except Exception, e: + except Exception: msg = "user-defined function raised exception" sqlite.sqlite3_result_error(context, msg, len(msg)) else: diff --git a/lib_pypy/datetime.py b/lib_pypy/datetime.py --- a/lib_pypy/datetime.py +++ b/lib_pypy/datetime.py @@ -13,7 +13,7 @@ Sources for time zone and DST data: http://www.twinsun.com/tz/tz-link.htm This was originally copied from the sandbox of the CPython CVS repository. -Thanks to Tim Peters for suggesting using it. +Thanks to Tim Peters for suggesting using it. """ import time as _time @@ -271,6 +271,8 @@ raise ValueError("%s()=%d, must be in -1439..1439" % (name, offset)) def _check_date_fields(year, month, day): + if not isinstance(year, (int, long)): + raise TypeError('int expected') if not MINYEAR <= year <= MAXYEAR: raise ValueError('year must be in %d..%d' % (MINYEAR, MAXYEAR), year) if not 1 <= month <= 12: @@ -280,6 +282,8 @@ raise ValueError('day must be in 1..%d' % dim, day) def _check_time_fields(hour, minute, second, microsecond): + if not isinstance(hour, (int, long)): + raise TypeError('int expected') if not 0 <= hour <= 23: raise ValueError('hour must be in 0..23', hour) if not 0 <= minute <= 59: @@ -543,61 +547,76 @@ self = object.__new__(cls) - self.__days = d - self.__seconds = s - self.__microseconds = us + self._days = d + self._seconds = s + self._microseconds = us if abs(d) > 999999999: raise OverflowError("timedelta # of days is too large: %d" % d) return self def __repr__(self): - if self.__microseconds: + if self._microseconds: return "%s(%d, %d, %d)" % ('datetime.' + self.__class__.__name__, - self.__days, - self.__seconds, - self.__microseconds) - if self.__seconds: + self._days, + self._seconds, + self._microseconds) + if self._seconds: return "%s(%d, %d)" % ('datetime.' + self.__class__.__name__, - self.__days, - self.__seconds) - return "%s(%d)" % ('datetime.' + self.__class__.__name__, self.__days) + self._days, + self._seconds) + return "%s(%d)" % ('datetime.' + self.__class__.__name__, self._days) def __str__(self): - mm, ss = divmod(self.__seconds, 60) + mm, ss = divmod(self._seconds, 60) hh, mm = divmod(mm, 60) s = "%d:%02d:%02d" % (hh, mm, ss) - if self.__days: + if self._days: def plural(n): return n, abs(n) != 1 and "s" or "" - s = ("%d day%s, " % plural(self.__days)) + s - if self.__microseconds: - s = s + ".%06d" % self.__microseconds + s = ("%d day%s, " % plural(self._days)) + s + if self._microseconds: + s = s + ".%06d" % self._microseconds return s - days = property(lambda self: self.__days, doc="days") - seconds = property(lambda self: self.__seconds, doc="seconds") - microseconds = property(lambda self: self.__microseconds, - doc="microseconds") - def total_seconds(self): return ((self.days * 86400 + self.seconds) * 10**6 + self.microseconds) / 1e6 + # Read-only field accessors + @property + def days(self): + """days""" + return self._days + + @property + def seconds(self): + """seconds""" + return self._seconds + + @property + def microseconds(self): + """microseconds""" + return self._microseconds + def __add__(self, other): if isinstance(other, timedelta): # for CPython compatibility, we cannot use # our __class__ here, but need a real timedelta - return timedelta(self.__days + other.__days, - self.__seconds + other.__seconds, - self.__microseconds + other.__microseconds) + return timedelta(self._days + other._days, + self._seconds + other._seconds, + self._microseconds + other._microseconds) return NotImplemented __radd__ = __add__ def __sub__(self, other): if isinstance(other, timedelta): - return self + -other + # for CPython compatibility, we cannot use + # our __class__ here, but need a real timedelta + return timedelta(self._days - other._days, + self._seconds - other._seconds, + self._microseconds - other._microseconds) return NotImplemented def __rsub__(self, other): @@ -606,17 +625,17 @@ return NotImplemented def __neg__(self): - # for CPython compatibility, we cannot use - # our __class__ here, but need a real timedelta - return timedelta(-self.__days, - -self.__seconds, - -self.__microseconds) + # for CPython compatibility, we cannot use + # our __class__ here, but need a real timedelta + return timedelta(-self._days, + -self._seconds, + -self._microseconds) def __pos__(self): return self def __abs__(self): - if self.__days < 0: + if self._days < 0: return -self else: return self @@ -625,81 +644,81 @@ if isinstance(other, (int, long)): # for CPython compatibility, we cannot use # our __class__ here, but need a real timedelta - return timedelta(self.__days * other, - self.__seconds * other, - self.__microseconds * other) + return timedelta(self._days * other, + self._seconds * other, + self._microseconds * other) return NotImplemented __rmul__ = __mul__ def __div__(self, other): if isinstance(other, (int, long)): - usec = ((self.__days * (24*3600L) + self.__seconds) * 1000000 + - self.__microseconds) + usec = ((self._days * (24*3600L) + self._seconds) * 1000000 + + self._microseconds) return timedelta(0, 0, usec // other) return NotImplemented __floordiv__ = __div__ - # Comparisons. + # Comparisons of timedelta objects with other. def __eq__(self, other): if isinstance(other, timedelta): - return self.__cmp(other) == 0 + return self._cmp(other) == 0 else: return False def __ne__(self, other): if isinstance(other, timedelta): - return self.__cmp(other) != 0 + return self._cmp(other) != 0 else: return True def __le__(self, other): if isinstance(other, timedelta): - return self.__cmp(other) <= 0 + return self._cmp(other) <= 0 else: _cmperror(self, other) def __lt__(self, other): if isinstance(other, timedelta): - return self.__cmp(other) < 0 + return self._cmp(other) < 0 else: _cmperror(self, other) def __ge__(self, other): if isinstance(other, timedelta): - return self.__cmp(other) >= 0 + return self._cmp(other) >= 0 else: _cmperror(self, other) def __gt__(self, other): if isinstance(other, timedelta): - return self.__cmp(other) > 0 + return self._cmp(other) > 0 else: _cmperror(self, other) - def __cmp(self, other): + def _cmp(self, other): assert isinstance(other, timedelta) - return cmp(self.__getstate(), other.__getstate()) + return cmp(self._getstate(), other._getstate()) def __hash__(self): - return hash(self.__getstate()) + return hash(self._getstate()) def __nonzero__(self): - return (self.__days != 0 or - self.__seconds != 0 or - self.__microseconds != 0) + return (self._days != 0 or + self._seconds != 0 or + self._microseconds != 0) # Pickle support. __safe_for_unpickling__ = True # For Python 2.2 - def __getstate(self): - return (self.__days, self.__seconds, self.__microseconds) + def _getstate(self): + return (self._days, self._seconds, self._microseconds) def __reduce__(self): - return (self.__class__, self.__getstate()) + return (self.__class__, self._getstate()) timedelta.min = timedelta(-999999999) timedelta.max = timedelta(days=999999999, hours=23, minutes=59, seconds=59, @@ -749,25 +768,26 @@ return self _check_date_fields(year, month, day) self = object.__new__(cls) - self.__year = year - self.__month = month - self.__day = day + self._year = year + self._month = month + self._day = day return self # Additional constructors + @classmethod def fromtimestamp(cls, t): "Construct a date from a POSIX timestamp (like time.time())." y, m, d, hh, mm, ss, weekday, jday, dst = _time.localtime(t) return cls(y, m, d) - fromtimestamp = classmethod(fromtimestamp) + @classmethod def today(cls): "Construct a date from time.time()." t = _time.time() return cls.fromtimestamp(t) - today = classmethod(today) + @classmethod def fromordinal(cls, n): """Contruct a date from a proleptic Gregorian ordinal. @@ -776,16 +796,24 @@ """ y, m, d = _ord2ymd(n) return cls(y, m, d) - fromordinal = classmethod(fromordinal) # Conversions to string def __repr__(self): - "Convert to formal string, for repr()." + """Convert to formal string, for repr(). + + >>> dt = datetime(2010, 1, 1) + >>> repr(dt) + 'datetime.datetime(2010, 1, 1, 0, 0)' + + >>> dt = datetime(2010, 1, 1, tzinfo=timezone.utc) + >>> repr(dt) + 'datetime.datetime(2010, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)' + """ return "%s(%d, %d, %d)" % ('datetime.' + self.__class__.__name__, - self.__year, - self.__month, - self.__day) + self._year, + self._month, + self._day) # XXX These shouldn't depend on time.localtime(), because that # clips the usable dates to [1970 .. 2038). At least ctime() is # easily done without using strftime() -- that's better too because @@ -793,12 +821,20 @@ def ctime(self): "Format a la ctime()." - return tmxxx(self.__year, self.__month, self.__day).ctime() + return tmxxx(self._year, self._month, self._day).ctime() def strftime(self, fmt): "Format using strftime()." return _wrap_strftime(self, fmt, self.timetuple()) + def __format__(self, fmt): + if not isinstance(fmt, (str, unicode)): + raise ValueError("__format__ excepts str or unicode, not %s" % + fmt.__class__.__name__) + if len(fmt) != 0: + return self.strftime(fmt) + return str(self) + def isoformat(self): """Return the date formatted according to ISO. @@ -808,29 +844,31 @@ - http://www.w3.org/TR/NOTE-datetime - http://www.cl.cam.ac.uk/~mgk25/iso-time.html """ - return "%04d-%02d-%02d" % (self.__year, self.__month, self.__day) + return "%04d-%02d-%02d" % (self._year, self._month, self._day) __str__ = isoformat - def __format__(self, format): - if not isinstance(format, (str, unicode)): - raise ValueError("__format__ excepts str or unicode, not %s" % - format.__class__.__name__) - if not format: - return str(self) - return self.strftime(format) + # Read-only field accessors + @property + def year(self): + """year (1-9999)""" + return self._year - # Read-only field accessors - year = property(lambda self: self.__year, - doc="year (%d-%d)" % (MINYEAR, MAXYEAR)) - month = property(lambda self: self.__month, doc="month (1-12)") - day = property(lambda self: self.__day, doc="day (1-31)") + @property + def month(self): + """month (1-12)""" + return self._month + + @property + def day(self): + """day (1-31)""" + return self._day # Standard conversions, __cmp__, __hash__ (and helpers) def timetuple(self): "Return local time tuple compatible with time.localtime()." - return _build_struct_time(self.__year, self.__month, self.__day, + return _build_struct_time(self._year, self._month, self._day, 0, 0, 0, -1) def toordinal(self): @@ -839,24 +877,24 @@ January 1 of year 1 is day 1. Only the year, month and day values contribute to the result. """ - return _ymd2ord(self.__year, self.__month, self.__day) + return _ymd2ord(self._year, self._month, self._day) def replace(self, year=None, month=None, day=None): """Return a new date with new values for the specified fields.""" if year is None: - year = self.__year + year = self._year if month is None: - month = self.__month + month = self._month if day is None: - day = self.__day + day = self._day _check_date_fields(year, month, day) return date(year, month, day) - # Comparisons. + # Comparisons of date objects with other. def __eq__(self, other): if isinstance(other, date): - return self.__cmp(other) == 0 + return self._cmp(other) == 0 elif hasattr(other, "timetuple"): return NotImplemented else: @@ -864,7 +902,7 @@ def __ne__(self, other): if isinstance(other, date): - return self.__cmp(other) != 0 + return self._cmp(other) != 0 elif hasattr(other, "timetuple"): return NotImplemented else: @@ -872,7 +910,7 @@ def __le__(self, other): if isinstance(other, date): - return self.__cmp(other) <= 0 + return self._cmp(other) <= 0 elif hasattr(other, "timetuple"): return NotImplemented else: @@ -880,7 +918,7 @@ def __lt__(self, other): if isinstance(other, date): - return self.__cmp(other) < 0 + return self._cmp(other) < 0 elif hasattr(other, "timetuple"): return NotImplemented else: @@ -888,7 +926,7 @@ def __ge__(self, other): if isinstance(other, date): - return self.__cmp(other) >= 0 + return self._cmp(other) >= 0 elif hasattr(other, "timetuple"): return NotImplemented else: @@ -896,21 +934,21 @@ def __gt__(self, other): if isinstance(other, date): - return self.__cmp(other) > 0 + return self._cmp(other) > 0 elif hasattr(other, "timetuple"): return NotImplemented else: _cmperror(self, other) - def __cmp(self, other): + def _cmp(self, other): assert isinstance(other, date) - y, m, d = self.__year, self.__month, self.__day - y2, m2, d2 = other.__year, other.__month, other.__day + y, m, d = self._year, self._month, self._day + y2, m2, d2 = other._year, other._month, other._day return cmp((y, m, d), (y2, m2, d2)) def __hash__(self): "Hash." - return hash(self.__getstate()) + return hash(self._getstate()) # Computations @@ -922,9 +960,9 @@ def __add__(self, other): "Add a date to a timedelta." if isinstance(other, timedelta): - t = tmxxx(self.__year, - self.__month, - self.__day + other.days) + t = tmxxx(self._year, + self._month, + self._day + other.days) self._checkOverflow(t.year) result = date(t.year, t.month, t.day) return result @@ -966,9 +1004,9 @@ ISO calendar algorithm taken from http://www.phys.uu.nl/~vgent/calendar/isocalendar.htm """ - year = self.__year + year = self._year week1monday = _isoweek1monday(year) - today = _ymd2ord(self.__year, self.__month, self.__day) + today = _ymd2ord(self._year, self._month, self._day) # Internally, week and day have origin 0 week, day = divmod(today - week1monday, 7) if week < 0: @@ -985,18 +1023,18 @@ __safe_for_unpickling__ = True # For Python 2.2 - def __getstate(self): - yhi, ylo = divmod(self.__year, 256) - return ("%c%c%c%c" % (yhi, ylo, self.__month, self.__day), ) + def _getstate(self): + yhi, ylo = divmod(self._year, 256) + return ("%c%c%c%c" % (yhi, ylo, self._month, self._day), ) def __setstate(self, string): if len(string) != 4 or not (1 <= ord(string[2]) <= 12): raise TypeError("not enough arguments") - yhi, ylo, self.__month, self.__day = map(ord, string) - self.__year = yhi * 256 + ylo + yhi, ylo, self._month, self._day = map(ord, string) + self._year = yhi * 256 + ylo def __reduce__(self): - return (self.__class__, self.__getstate()) + return (self.__class__, self._getstate()) _date_class = date # so functions w/ args named "date" can get at the class @@ -1118,62 +1156,80 @@ return self _check_tzinfo_arg(tzinfo) _check_time_fields(hour, minute, second, microsecond) - self.__hour = hour - self.__minute = minute - self.__second = second - self.__microsecond = microsecond + self._hour = hour + self._minute = minute + self._second = second + self._microsecond = microsecond self._tzinfo = tzinfo return self # Read-only field accessors - hour = property(lambda self: self.__hour, doc="hour (0-23)") - minute = property(lambda self: self.__minute, doc="minute (0-59)") - second = property(lambda self: self.__second, doc="second (0-59)") - microsecond = property(lambda self: self.__microsecond, - doc="microsecond (0-999999)") - tzinfo = property(lambda self: self._tzinfo, doc="timezone info object") + @property + def hour(self): + """hour (0-23)""" + return self._hour + + @property + def minute(self): + """minute (0-59)""" + return self._minute + + @property + def second(self): + """second (0-59)""" + return self._second + + @property + def microsecond(self): + """microsecond (0-999999)""" + return self._microsecond + + @property + def tzinfo(self): + """timezone info object""" + return self._tzinfo # Standard conversions, __hash__ (and helpers) - # Comparisons. + # Comparisons of time objects with other. def __eq__(self, other): if isinstance(other, time): - return self.__cmp(other) == 0 + return self._cmp(other) == 0 else: return False def __ne__(self, other): if isinstance(other, time): - return self.__cmp(other) != 0 + return self._cmp(other) != 0 else: return True def __le__(self, other): if isinstance(other, time): - return self.__cmp(other) <= 0 + return self._cmp(other) <= 0 else: _cmperror(self, other) def __lt__(self, other): if isinstance(other, time): - return self.__cmp(other) < 0 + return self._cmp(other) < 0 else: _cmperror(self, other) def __ge__(self, other): if isinstance(other, time): - return self.__cmp(other) >= 0 + return self._cmp(other) >= 0 else: _cmperror(self, other) def __gt__(self, other): if isinstance(other, time): - return self.__cmp(other) > 0 + return self._cmp(other) > 0 else: _cmperror(self, other) - def __cmp(self, other): + def _cmp(self, other): assert isinstance(other, time) mytz = self._tzinfo ottz = other._tzinfo @@ -1187,23 +1243,23 @@ base_compare = myoff == otoff if base_compare: - return cmp((self.__hour, self.__minute, self.__second, - self.__microsecond), - (other.__hour, other.__minute, other.__second, - other.__microsecond)) + return cmp((self._hour, self._minute, self._second, + self._microsecond), + (other._hour, other._minute, other._second, + other._microsecond)) if myoff is None or otoff is None: # XXX Buggy in 2.2.2. raise TypeError("cannot compare naive and aware times") - myhhmm = self.__hour * 60 + self.__minute - myoff - othhmm = other.__hour * 60 + other.__minute - otoff - return cmp((myhhmm, self.__second, self.__microsecond), - (othhmm, other.__second, other.__microsecond)) + myhhmm = self._hour * 60 + self._minute - myoff + othhmm = other._hour * 60 + other._minute - otoff + return cmp((myhhmm, self._second, self._microsecond), + (othhmm, other._second, other._microsecond)) def __hash__(self): """Hash.""" tzoff = self._utcoffset() if not tzoff: # zero or None - return hash(self.__getstate()[0]) + return hash(self._getstate()[0]) h, m = divmod(self.hour * 60 + self.minute - tzoff, 60) if 0 <= h < 24: return hash(time(h, m, self.second, self.microsecond)) @@ -1227,14 +1283,14 @@ def __repr__(self): """Convert to formal string, for repr().""" - if self.__microsecond != 0: - s = ", %d, %d" % (self.__second, self.__microsecond) - elif self.__second != 0: - s = ", %d" % self.__second + if self._microsecond != 0: + s = ", %d, %d" % (self._second, self._microsecond) + elif self._second != 0: + s = ", %d" % self._second else: s = "" s= "%s(%d, %d%s)" % ('datetime.' + self.__class__.__name__, - self.__hour, self.__minute, s) + self._hour, self._minute, s) if self._tzinfo is not None: assert s[-1:] == ")" s = s[:-1] + ", tzinfo=%r" % self._tzinfo + ")" @@ -1246,8 +1302,8 @@ This is 'HH:MM:SS.mmmmmm+zz:zz', or 'HH:MM:SS+zz:zz' if self.microsecond == 0. """ - s = _format_time(self.__hour, self.__minute, self.__second, - self.__microsecond) + s = _format_time(self._hour, self._minute, self._second, + self._microsecond) tz = self._tzstr() if tz: s += tz @@ -1255,14 +1311,6 @@ __str__ = isoformat - def __format__(self, format): - if not isinstance(format, (str, unicode)): - raise ValueError("__format__ excepts str or unicode, not %s" % - format.__class__.__name__) - if not format: - return str(self) - return self.strftime(format) - def strftime(self, fmt): """Format using strftime(). The date part of the timestamp passed to underlying strftime should not be used. @@ -1270,10 +1318,18 @@ # The year must be >= 1900 else Python's strftime implementation # can raise a bogus exception. timetuple = (1900, 1, 1, - self.__hour, self.__minute, self.__second, + self._hour, self._minute, self._second, 0, 1, -1) return _wrap_strftime(self, fmt, timetuple) + def __format__(self, fmt): + if not isinstance(fmt, (str, unicode)): + raise ValueError("__format__ excepts str or unicode, not %s" % + fmt.__class__.__name__) + if len(fmt) != 0: + return self.strftime(fmt) + return str(self) + # Timezone functions def utcoffset(self): @@ -1350,10 +1406,10 @@ __safe_for_unpickling__ = True # For Python 2.2 - def __getstate(self): - us2, us3 = divmod(self.__microsecond, 256) + def _getstate(self): + us2, us3 = divmod(self._microsecond, 256) us1, us2 = divmod(us2, 256) - basestate = ("%c" * 6) % (self.__hour, self.__minute, self.__second, + basestate = ("%c" * 6) % (self._hour, self._minute, self._second, us1, us2, us3) if self._tzinfo is None: return (basestate,) @@ -1363,13 +1419,13 @@ def __setstate(self, string, tzinfo): if len(string) != 6 or ord(string[0]) >= 24: raise TypeError("an integer is required") - self.__hour, self.__minute, self.__second, us1, us2, us3 = \ + self._hour, self._minute, self._second, us1, us2, us3 = \ map(ord, string) - self.__microsecond = (((us1 << 8) | us2) << 8) | us3 + self._microsecond = (((us1 << 8) | us2) << 8) | us3 self._tzinfo = tzinfo def __reduce__(self): - return (time, self.__getstate()) + return (time, self._getstate()) _time_class = time # so functions w/ args named "time" can get at the class @@ -1378,9 +1434,11 @@ time.resolution = timedelta(microseconds=1) class datetime(date): + """datetime(year, month, day[, hour[, minute[, second[, microsecond[,tzinfo]]]]]) - # XXX needs docstrings - # See http://www.zope.org/Members/fdrake/DateTimeWiki/TimeZoneInfo + The year, month and day arguments are required. tzinfo may be None, or an + instance of a tzinfo subclass. The remaining arguments may be ints or longs. + """ def __new__(cls, year, month=None, day=None, hour=0, minute=0, second=0, microsecond=0, tzinfo=None): @@ -1393,24 +1451,43 @@ _check_time_fields(hour, minute, second, microsecond) self = date.__new__(cls, year, month, day) # XXX This duplicates __year, __month, __day for convenience :-( - self.__year = year - self.__month = month - self.__day = day - self.__hour = hour - self.__minute = minute - self.__second = second - self.__microsecond = microsecond + self._year = year + self._month = month + self._day = day + self._hour = hour + self._minute = minute + self._second = second + self._microsecond = microsecond self._tzinfo = tzinfo return self # Read-only field accessors - hour = property(lambda self: self.__hour, doc="hour (0-23)") - minute = property(lambda self: self.__minute, doc="minute (0-59)") - second = property(lambda self: self.__second, doc="second (0-59)") - microsecond = property(lambda self: self.__microsecond, - doc="microsecond (0-999999)") - tzinfo = property(lambda self: self._tzinfo, doc="timezone info object") + @property + def hour(self): + """hour (0-23)""" + return self._hour + @property + def minute(self): + """minute (0-59)""" + return self._minute + + @property + def second(self): + """second (0-59)""" + return self._second + + @property + def microsecond(self): + """microsecond (0-999999)""" + return self._microsecond + + @property + def tzinfo(self): + """timezone info object""" + return self._tzinfo + + @classmethod def fromtimestamp(cls, t, tz=None): """Construct a datetime from a POSIX timestamp (like time.time()). @@ -1438,37 +1515,42 @@ if tz is not None: result = tz.fromutc(result) return result - fromtimestamp = classmethod(fromtimestamp) + @classmethod def utcfromtimestamp(cls, t): "Construct a UTC datetime from a POSIX timestamp (like time.time())." - if 1 - (t % 1.0) < 0.0000005: - t = float(int(t)) + 1 - if t < 0: - t -= 1 + t, frac = divmod(t, 1.0) + us = round(frac * 1e6) + + # If timestamp is less than one microsecond smaller than a + # full second, us can be rounded up to 1000000. In this case, + # roll over to seconds, otherwise, ValueError is raised + # by the constructor. + if us == 1000000: + t += 1 + us = 0 y, m, d, hh, mm, ss, weekday, jday, dst = _time.gmtime(t) - us = int((t % 1.0) * 1000000) ss = min(ss, 59) # clamp out leap seconds if the platform has them return cls(y, m, d, hh, mm, ss, us) - utcfromtimestamp = classmethod(utcfromtimestamp) # XXX This is supposed to do better than we *can* do by using time.time(), # XXX if the platform supports a more accurate way. The C implementation # XXX uses gettimeofday on platforms that have it, but that isn't # XXX available from Python. So now() may return different results # XXX across the implementations. + @classmethod def now(cls, tz=None): "Construct a datetime from time.time() and optional time zone info." t = _time.time() return cls.fromtimestamp(t, tz) - now = classmethod(now) + @classmethod def utcnow(cls): "Construct a UTC datetime from time.time()." t = _time.time() return cls.utcfromtimestamp(t) - utcnow = classmethod(utcnow) + @classmethod def combine(cls, date, time): "Construct a datetime from a given date and a given time." if not isinstance(date, _date_class): @@ -1478,7 +1560,6 @@ return cls(date.year, date.month, date.day, time.hour, time.minute, time.second, time.microsecond, time.tzinfo) - combine = classmethod(combine) def timetuple(self): "Return local time tuple compatible with time.localtime()." @@ -1504,7 +1585,7 @@ def date(self): "Return the date part." - return date(self.__year, self.__month, self.__day) + return date(self._year, self._month, self._day) def time(self): "Return the time part, with tzinfo None." @@ -1564,8 +1645,8 @@ def ctime(self): "Format a la ctime()." - t = tmxxx(self.__year, self.__month, self.__day, self.__hour, - self.__minute, self.__second) + t = tmxxx(self._year, self._month, self._day, self._hour, + self._minute, self._second) return t.ctime() def isoformat(self, sep='T'): @@ -1580,10 +1661,10 @@ Optional argument sep specifies the separator between date and time, default 'T'. """ - s = ("%04d-%02d-%02d%c" % (self.__year, self.__month, self.__day, + s = ("%04d-%02d-%02d%c" % (self._year, self._month, self._day, sep) + - _format_time(self.__hour, self.__minute, self.__second, - self.__microsecond)) + _format_time(self._hour, self._minute, self._second, + self._microsecond)) off = self._utcoffset() if off is not None: if off < 0: @@ -1596,13 +1677,13 @@ return s def __repr__(self): - "Convert to formal string, for repr()." - L = [self.__year, self.__month, self.__day, # These are never zero - self.__hour, self.__minute, self.__second, self.__microsecond] + """Convert to formal string, for repr().""" + L = [self._year, self._month, self._day, # These are never zero + self._hour, self._minute, self._second, self._microsecond] if L[-1] == 0: del L[-1] if L[-1] == 0: - del L[-1] + del L[-1] s = ", ".join(map(str, L)) s = "%s(%s)" % ('datetime.' + self.__class__.__name__, s) if self._tzinfo is not None: @@ -1675,7 +1756,7 @@ def __eq__(self, other): if isinstance(other, datetime): - return self.__cmp(other) == 0 + return self._cmp(other) == 0 elif hasattr(other, "timetuple") and not isinstance(other, date): return NotImplemented else: @@ -1683,7 +1764,7 @@ def __ne__(self, other): if isinstance(other, datetime): - return self.__cmp(other) != 0 + return self._cmp(other) != 0 elif hasattr(other, "timetuple") and not isinstance(other, date): return NotImplemented else: @@ -1691,7 +1772,7 @@ def __le__(self, other): if isinstance(other, datetime): - return self.__cmp(other) <= 0 + return self._cmp(other) <= 0 elif hasattr(other, "timetuple") and not isinstance(other, date): return NotImplemented else: @@ -1699,7 +1780,7 @@ def __lt__(self, other): if isinstance(other, datetime): - return self.__cmp(other) < 0 + return self._cmp(other) < 0 elif hasattr(other, "timetuple") and not isinstance(other, date): return NotImplemented else: @@ -1707,7 +1788,7 @@ def __ge__(self, other): if isinstance(other, datetime): - return self.__cmp(other) >= 0 + return self._cmp(other) >= 0 elif hasattr(other, "timetuple") and not isinstance(other, date): return NotImplemented else: @@ -1715,13 +1796,13 @@ def __gt__(self, other): if isinstance(other, datetime): - return self.__cmp(other) > 0 + return self._cmp(other) > 0 elif hasattr(other, "timetuple") and not isinstance(other, date): return NotImplemented else: _cmperror(self, other) - def __cmp(self, other): + def _cmp(self, other): assert isinstance(other, datetime) mytz = self._tzinfo ottz = other._tzinfo @@ -1737,12 +1818,12 @@ base_compare = myoff == otoff if base_compare: - return cmp((self.__year, self.__month, self.__day, - self.__hour, self.__minute, self.__second, - self.__microsecond), - (other.__year, other.__month, other.__day, - other.__hour, other.__minute, other.__second, - other.__microsecond)) + return cmp((self._year, self._month, self._day, + self._hour, self._minute, self._second, + self._microsecond), + (other._year, other._month, other._day, + other._hour, other._minute, other._second, + other._microsecond)) if myoff is None or otoff is None: # XXX Buggy in 2.2.2. raise TypeError("cannot compare naive and aware datetimes") @@ -1756,13 +1837,13 @@ "Add a datetime and a timedelta." if not isinstance(other, timedelta): return NotImplemented - t = tmxxx(self.__year, - self.__month, - self.__day + other.days, - self.__hour, - self.__minute, - self.__second + other.seconds, - self.__microsecond + other.microseconds) + t = tmxxx(self._year, + self._month, + self._day + other.days, + self._hour, + self._minute, + self._second + other.seconds, + self._microsecond + other.microseconds) self._checkOverflow(t.year) result = datetime(t.year, t.month, t.day, t.hour, t.minute, t.second, @@ -1780,11 +1861,11 @@ days1 = self.toordinal() days2 = other.toordinal() - secs1 = self.__second + self.__minute * 60 + self.__hour * 3600 - secs2 = other.__second + other.__minute * 60 + other.__hour * 3600 + secs1 = self._second + self._minute * 60 + self._hour * 3600 + secs2 = other._second + other._minute * 60 + other._hour * 3600 base = timedelta(days1 - days2, secs1 - secs2, - self.__microsecond - other.__microsecond) + self._microsecond - other._microsecond) if self._tzinfo is other._tzinfo: return base myoff = self._utcoffset() @@ -1792,13 +1873,13 @@ if myoff == otoff: return base if myoff is None or otoff is None: - raise TypeError, "cannot mix naive and timezone-aware time" + raise TypeError("cannot mix naive and timezone-aware time") return base + timedelta(minutes = otoff-myoff) def __hash__(self): tzoff = self._utcoffset() if tzoff is None: - return hash(self.__getstate()[0]) + return hash(self._getstate()[0]) days = _ymd2ord(self.year, self.month, self.day) seconds = self.hour * 3600 + (self.minute - tzoff) * 60 + self.second return hash(timedelta(days, seconds, self.microsecond)) @@ -1807,12 +1888,12 @@ __safe_for_unpickling__ = True # For Python 2.2 - def __getstate(self): - yhi, ylo = divmod(self.__year, 256) - us2, us3 = divmod(self.__microsecond, 256) + def _getstate(self): + yhi, ylo = divmod(self._year, 256) + us2, us3 = divmod(self._microsecond, 256) us1, us2 = divmod(us2, 256) - basestate = ("%c" * 10) % (yhi, ylo, self.__month, self.__day, - self.__hour, self.__minute, self.__second, + basestate = ("%c" * 10) % (yhi, ylo, self._month, self._day, + self._hour, self._minute, self._second, us1, us2, us3) if self._tzinfo is None: return (basestate,) @@ -1820,14 +1901,14 @@ return (basestate, self._tzinfo) def __setstate(self, string, tzinfo): - (yhi, ylo, self.__month, self.__day, self.__hour, - self.__minute, self.__second, us1, us2, us3) = map(ord, string) - self.__year = yhi * 256 + ylo - self.__microsecond = (((us1 << 8) | us2) << 8) | us3 + (yhi, ylo, self._month, self._day, self._hour, + self._minute, self._second, us1, us2, us3) = map(ord, string) + self._year = yhi * 256 + ylo + self._microsecond = (((us1 << 8) | us2) << 8) | us3 self._tzinfo = tzinfo def __reduce__(self): - return (self.__class__, self.__getstate()) + return (self.__class__, self._getstate()) datetime.min = datetime(1, 1, 1) @@ -2009,7 +2090,7 @@ Because we know z.d said z was in daylight time (else [5] would have held and we would have stopped then), and we know z.d != z'.d (else [8] would have held -and we we have stopped then), and there are only 2 possible values dst() can +and we have stopped then), and there are only 2 possible values dst() can return in Eastern, it follows that z'.d must be 0 (which it is in the example, but the reasoning doesn't depend on the example -- it depends on there being two possible dst() outcomes, one zero and the other non-zero). Therefore diff --git a/lib_pypy/numpypy/__init__.py b/lib_pypy/numpypy/__init__.py --- a/lib_pypy/numpypy/__init__.py +++ b/lib_pypy/numpypy/__init__.py @@ -1,2 +1,2 @@ from _numpypy import * -from .fromnumeric import * +from .core import * diff --git a/lib_pypy/numpypy/core/__init__.py b/lib_pypy/numpypy/core/__init__.py new file mode 100644 --- /dev/null +++ b/lib_pypy/numpypy/core/__init__.py @@ -0,0 +1,1 @@ +from .fromnumeric import * diff --git a/lib_pypy/numpypy/fromnumeric.py b/lib_pypy/numpypy/core/fromnumeric.py rename from lib_pypy/numpypy/fromnumeric.py rename to lib_pypy/numpypy/core/fromnumeric.py --- a/lib_pypy/numpypy/fromnumeric.py +++ b/lib_pypy/numpypy/core/fromnumeric.py @@ -85,7 +85,7 @@ array([4, 3, 6]) """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') # not deprecated --- copy if necessary, view otherwise @@ -149,6 +149,7 @@ [5, 6]]) """ + assert order == 'C' if not hasattr(a, 'reshape'): a = numpypy.array(a) return a.reshape(newshape) @@ -273,7 +274,7 @@ [-1, -2, -3, -4, -5]]]) """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def repeat(a, repeats, axis=None): @@ -315,7 +316,7 @@ [3, 4]]) """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def put(a, ind, v, mode='raise'): @@ -366,7 +367,7 @@ array([ 0, 1, 2, 3, -5]) """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def swapaxes(a, axis1, axis2): @@ -410,7 +411,7 @@ [3, 7]]]) """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def transpose(a, axes=None): @@ -451,8 +452,11 @@ (2, 1, 3) """ - raise NotImplemented('Waiting on interp level method') - + if axes is not None: + raise NotImplementedError('No "axes" arg yet.') + if not hasattr(a, 'T'): + a = numpypy.array(a) + return a.T def sort(a, axis=-1, kind='quicksort', order=None): """ @@ -553,7 +557,7 @@ dtype=[('name', '|S10'), ('height', '<f8'), ('age', '<i4')]) """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def argsort(a, axis=-1, kind='quicksort', order=None): @@ -634,7 +638,7 @@ array([0, 1]) """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def argmax(a, axis=None): @@ -687,6 +691,7 @@ 1 """ + assert axis is None if not hasattr(a, 'argmax'): a = numpypy.array(a) return a.argmax() @@ -702,6 +707,7 @@ documentation. """ + assert axis is None if not hasattr(a, 'argmin'): a = numpypy.array(a) return a.argmin() @@ -753,7 +759,7 @@ array([0, 5, 1, 2]) """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def resize(a, new_shape): @@ -794,7 +800,7 @@ [0, 1, 2, 3]]) """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def squeeze(a): @@ -821,7 +827,7 @@ (3,) """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def diagonal(a, offset=0, axis1=0, axis2=1): @@ -905,7 +911,7 @@ [5, 7]]) """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def trace(a, offset=0, axis1=0, axis2=1, dtype=None, out=None): @@ -964,7 +970,7 @@ (2, 3) """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def ravel(a, order='C'): """ @@ -1048,7 +1054,7 @@ array([ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]) """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def nonzero(a): @@ -1124,7 +1130,7 @@ (array([1, 1, 1, 2, 2, 2]), array([0, 1, 2, 0, 1, 2])) """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def shape(a): @@ -1229,7 +1235,7 @@ array([2]) """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def clip(a, a_min, a_max, out=None): @@ -1282,7 +1288,7 @@ array([3, 4, 2, 3, 4, 5, 6, 7, 8, 8]) """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def sum(a, axis=None, dtype=None, out=None): @@ -1351,9 +1357,11 @@ -128 """ + assert dtype is None + assert out is None if not hasattr(a, "sum"): a = numpypy.array(a) - return a.sum() + return a.sum(axis=axis) def product (a, axis=None, dtype=None, out=None): @@ -1365,7 +1373,7 @@ prod : equivalent function; see for details. """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def sometrue(a, axis=None, out=None): @@ -1379,6 +1387,8 @@ any : equivalent function """ + assert axis is None + assert out is None if not hasattr(a, 'any'): a = numpypy.array(a) return a.any() @@ -1393,6 +1403,8 @@ numpy.all : Equivalent function; see for details. """ + assert axis is None + assert out is None if not hasattr(a, 'all'): a = numpypy.array(a) return a.all() @@ -1461,6 +1473,8 @@ (191614240, 191614240) """ + assert axis is None + assert out is None if not hasattr(a, 'any'): a = numpypy.array(a) return a.any() @@ -1523,6 +1537,8 @@ (28293632, 28293632, array([ True], dtype=bool)) """ + assert axis is None + assert out is None if not hasattr(a, 'all'): a = numpypy.array(a) return a.all() @@ -1590,7 +1606,7 @@ [ 4, 9, 15]]) """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def cumproduct(a, axis=None, dtype=None, out=None): @@ -1603,7 +1619,7 @@ cumprod : equivalent function; see for details. """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def ptp(a, axis=None, out=None): @@ -1644,7 +1660,7 @@ array([1, 1]) """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def amax(a, axis=None, out=None): @@ -1702,6 +1718,8 @@ 4.0 """ + assert axis is None + assert out is None if not hasattr(a, "max"): a = numpypy.array(a) return a.max() @@ -1763,6 +1781,8 @@ """ # amin() is equivalent to min() + assert axis is None + assert out is None if not hasattr(a, 'min'): a = numpypy.array(a) return a.min() @@ -1873,7 +1893,7 @@ True """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def cumprod(a, axis=None, dtype=None, out=None): @@ -1936,7 +1956,7 @@ [ 4, 20, 120]]) """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def ndim(a): @@ -2054,7 +2074,7 @@ 2 """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def around(a, decimals=0, out=None): @@ -2123,7 +2143,7 @@ array([ 0, 0, 0, 10]) """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def round_(a, decimals=0, out=None): @@ -2137,7 +2157,7 @@ around : equivalent function """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def mean(a, axis=None, dtype=None, out=None): @@ -2211,9 +2231,11 @@ 0.55000000074505806 """ + assert dtype is None + assert out is None if not hasattr(a, "mean"): a = numpypy.array(a) - return a.mean() + return a.mean(axis=axis) def std(a, axis=None, dtype=None, out=None, ddof=0): @@ -2302,6 +2324,10 @@ 0.44999999925552653 """ + assert axis is None + assert dtype is None + assert out is None + assert ddof == 0 if not hasattr(a, "std"): a = numpypy.array(a) return a.std() @@ -2395,6 +2421,10 @@ 0.20250000000000001 """ + assert axis is None + assert dtype is None + assert out is None + assert ddof == 0 if not hasattr(a, "var"): a = numpypy.array(a) return a.var() diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py --- a/pypy/config/pypyoption.py +++ b/pypy/config/pypyoption.py @@ -371,6 +371,7 @@ config.objspace.std.suggest(newshortcut=True) config.objspace.std.suggest(withidentitydict=True) config.objspace.std.suggest(withspecialisedtuple=True) + config.objspace.std.suggest(withidentitydict=True) #if not IS_64_BITS: # config.objspace.std.suggest(withsmalllong=True) diff --git a/pypy/doc/getting-started.rst b/pypy/doc/getting-started.rst --- a/pypy/doc/getting-started.rst +++ b/pypy/doc/getting-started.rst @@ -53,11 +53,11 @@ PyPy is ready to be executed as soon as you unpack the tarball or the zip file, with no need to install it in any specific location:: - $ tar xf pypy-1.6-linux.tar.bz2 + $ tar xf pypy-1.7-linux.tar.bz2 - $ ./pypy-1.6/bin/pypy + $ ./pypy-1.7/bin/pypy Python 2.7.1 (?, Apr 27 2011, 12:44:21) - [PyPy 1.6.0 with GCC 4.4.3] on linux2 + [PyPy 1.7.0 with GCC 4.4.3] on linux2 Type "help", "copyright", "credits" or "license" for more information. And now for something completely different: ``implementing LOGO in LOGO: "turtles all the way down"'' @@ -75,14 +75,14 @@ $ curl -O https://raw.github.com/pypa/pip/master/contrib/get-pip.py - $ ./pypy-1.6/bin/pypy distribute_setup.py + $ ./pypy-1.7/bin/pypy distribute_setup.py - $ ./pypy-1.6/bin/pypy get-pip.py + $ ./pypy-1.7/bin/pypy get-pip.py - $ ./pypy-1.6/bin/pip install pygments # for example + $ ./pypy-1.7/bin/pip install pygments # for example -3rd party libraries will be installed in ``pypy-1.6/site-packages``, and -the scripts in ``pypy-1.6/bin``. +3rd party libraries will be installed in ``pypy-1.7/site-packages``, and +the scripts in ``pypy-1.7/bin``. Installing using virtualenv --------------------------- diff --git a/pypy/doc/translation.rst b/pypy/doc/translation.rst --- a/pypy/doc/translation.rst +++ b/pypy/doc/translation.rst @@ -155,7 +155,7 @@ function. The two input variables are the exception class and the exception value, respectively. (No other block will actually link to the exceptblock if the function does not - explicitely raise exceptions.) + explicitly raise exceptions.) ``Block`` @@ -325,7 +325,7 @@ Mutable objects need special treatment during annotation, because the annotation of contained values needs to be possibly updated to account for mutation operations, and consequently the annotation information -reflown through the relevant parts of the flow the graphs. +reflown through the relevant parts of the flow graphs. * ``SomeList`` stands for a list of homogeneous type (i.e. all the elements of the list are represented by a single common ``SomeXxx`` @@ -503,8 +503,8 @@ Since RPython is a garbage collected language there is a lot of heap memory allocation going on all the time, which would either not occur at all in a more -traditional explicitely managed language or results in an object which dies at -a time known in advance and can thus be explicitely deallocated. For example a +traditional explicitly managed language or results in an object which dies at +a time known in advance and can thus be explicitly deallocated. For example a loop of the following form:: for i in range(n): @@ -696,7 +696,7 @@ So far it is the second most mature high level backend after GenCLI: it still can't translate the full Standard Interpreter, but after the -Leysin sprint we were able to compile and run the rpytstone and +Leysin sprint we were able to compile and run the rpystone and richards benchmarks. GenJVM is almost entirely the work of Niko Matsakis, who worked on it diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py --- a/pypy/interpreter/executioncontext.py +++ b/pypy/interpreter/executioncontext.py @@ -445,6 +445,7 @@ AsyncAction.__init__(self, space) self.dying_objects = [] self.finalizers_lock_count = 0 + self.enabled_at_app_level = True def register_callback(self, w_obj, callback, descrname): self.dying_objects.append((w_obj, callback, descrname)) diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py --- a/pypy/jit/backend/test/runner_test.py +++ b/pypy/jit/backend/test/runner_test.py @@ -551,6 +551,28 @@ res = self.execute_operation(rop.CALL, [funcbox] + map(BoxInt, args), 'int', descr=calldescr) assert res.value == func(*args) + def test_call_box_func(self): + def a(a1, a2): + return a1 + a2 + def b(b1, b2): + return b1 * b2 + + arg1 = 40 + arg2 = 2 + for f in [a, b]: + TP = lltype.Signed + FPTR = self.Ptr(self.FuncType([TP, TP], TP)) + func_ptr = llhelper(FPTR, f) + FUNC = deref(FPTR) + funcconst = self.get_funcbox(self.cpu, func_ptr) + funcbox = funcconst.clonebox() + calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, + EffectInfo.MOST_GENERAL) + res = self.execute_operation(rop.CALL, + [funcbox, BoxInt(arg1), BoxInt(arg2)], + 'int', descr=calldescr) + assert res.getint() == f(arg1, arg2) + def test_call_stack_alignment(self): # test stack alignment issues, notably for Mac OS/X. # also test the ordering of the arguments. @@ -1872,6 +1894,7 @@ values.append(descr) values.append(self.cpu.get_latest_value_int(0)) values.append(self.cpu.get_latest_value_int(1)) + values.append(token) FUNC = self.FuncType([lltype.Signed, lltype.Signed], lltype.Void) func_ptr = llhelper(lltype.Ptr(FUNC), maybe_force) @@ -1902,7 +1925,8 @@ assert fail.identifier == 1 assert self.cpu.get_latest_value_int(0) == 1 assert self.cpu.get_latest_value_int(1) == 10 - assert values == [faildescr, 1, 10] + token = self.cpu.get_latest_force_token() + assert values == [faildescr, 1, 10, token] def test_force_operations_returning_int(self): values = [] @@ -1911,6 +1935,7 @@ self.cpu.force(token) values.append(self.cpu.get_latest_value_int(0)) values.append(self.cpu.get_latest_value_int(2)) + values.append(token) return 42 FUNC = self.FuncType([lltype.Signed, lltype.Signed], lltype.Signed) @@ -1944,7 +1969,8 @@ assert self.cpu.get_latest_value_int(0) == 1 assert self.cpu.get_latest_value_int(1) == 42 assert self.cpu.get_latest_value_int(2) == 10 - assert values == [1, 10] + token = self.cpu.get_latest_force_token() + assert values == [1, 10, token] def test_force_operations_returning_float(self): values = [] @@ -1953,6 +1979,7 @@ self.cpu.force(token) values.append(self.cpu.get_latest_value_int(0)) values.append(self.cpu.get_latest_value_int(2)) + values.append(token) return 42.5 FUNC = self.FuncType([lltype.Signed, lltype.Signed], lltype.Float) @@ -1988,7 +2015,8 @@ x = self.cpu.get_latest_value_float(1) assert longlong.getrealfloat(x) == 42.5 assert self.cpu.get_latest_value_int(2) == 10 - assert values == [1, 10] + token = self.cpu.get_latest_force_token() + assert values == [1, 10, token] def test_call_to_c_function(self): from pypy.rlib.libffi import CDLL, types, ArgChain, FUNCFLAG_CDECL diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py --- a/pypy/jit/backend/x86/assembler.py +++ b/pypy/jit/backend/x86/assembler.py @@ -1118,6 +1118,12 @@ for src, dst in singlefloats: self.mc.MOVD(dst, src) # Finally remap the arguments in the main regs + # If x is a register and is in dst_locs, then oups, it needs to + # be moved away: + if x in dst_locs: + src_locs.append(x) + dst_locs.append(r10) + x = r10 remap_frame_layout(self, src_locs, dst_locs, X86_64_SCRATCH_REG) self._regalloc.reserve_param(len(pass_on_stack)) @@ -2042,10 +2048,7 @@ size = sizeloc.value signloc = arglocs[1] - if isinstance(op.getarg(0), Const): - x = imm(op.getarg(0).getint()) - else: - x = arglocs[2] + x = arglocs[2] # the function address if x is eax: tmp = ecx else: diff --git a/pypy/jit/backend/x86/runner.py b/pypy/jit/backend/x86/runner.py --- a/pypy/jit/backend/x86/runner.py +++ b/pypy/jit/backend/x86/runner.py @@ -6,7 +6,7 @@ from pypy.jit.codewriter import longlong from pypy.jit.metainterp import history, compile from pypy.jit.backend.x86.assembler import Assembler386 -from pypy.jit.backend.x86.arch import FORCE_INDEX_OFS +from pypy.jit.backend.x86.arch import FORCE_INDEX_OFS, IS_X86_32 from pypy.jit.backend.x86.profagent import ProfileAgent from pypy.jit.backend.llsupport.llmodel import AbstractLLCPU from pypy.jit.backend.x86 import regloc @@ -142,7 +142,9 @@ cast_ptr_to_int._annspecialcase_ = 'specialize:arglltype(0)' cast_ptr_to_int = staticmethod(cast_ptr_to_int) - all_null_registers = lltype.malloc(rffi.LONGP.TO, 24, + all_null_registers = lltype.malloc(rffi.LONGP.TO, + IS_X86_32 and (16+8) # 16 + 8 regs + or (16+16), # 16 + 16 regs flavor='raw', zero=True, immortal=True) diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py --- a/pypy/module/_codecs/interp_codecs.py +++ b/pypy/module/_codecs/interp_codecs.py @@ -108,6 +108,10 @@ w_result = state.codec_search_cache.get(normalized_encoding, None) if w_result is not None: return w_result + return _lookup_codec_loop(space, encoding, normalized_encoding) + +def _lookup_codec_loop(space, encoding, normalized_encoding): + state = space.fromcache(CodecState) if state.codec_need_encodings: w_import = space.getattr(space.builtin, space.wrap("__import__")) # registers new codecs diff --git a/pypy/module/_codecs/test/test_ztranslation.py b/pypy/module/_codecs/test/test_ztranslation.py new file mode 100644 --- /dev/null +++ b/pypy/module/_codecs/test/test_ztranslation.py @@ -0,0 +1,5 @@ +from pypy.objspace.fake.checkmodule import checkmodule + + +def test__codecs_translates(): + checkmodule('_codecs') diff --git a/pypy/module/_hashlib/interp_hashlib.py b/pypy/module/_hashlib/interp_hashlib.py --- a/pypy/module/_hashlib/interp_hashlib.py +++ b/pypy/module/_hashlib/interp_hashlib.py @@ -34,8 +34,12 @@ ctx = lltype.malloc(ropenssl.EVP_MD_CTX.TO, flavor='raw') rgc.add_memory_pressure(HASH_MALLOC_SIZE + self.digest_size) - ropenssl.EVP_DigestInit(ctx, digest_type) - self.ctx = ctx + try: + ropenssl.EVP_DigestInit(ctx, digest_type) + self.ctx = ctx + except: + lltype.free(ctx, flavor='raw') + raise def __del__(self): # self.lock.free() diff --git a/pypy/module/_io/interp_fileio.py b/pypy/module/_io/interp_fileio.py --- a/pypy/module/_io/interp_fileio.py +++ b/pypy/module/_io/interp_fileio.py @@ -349,6 +349,8 @@ try: s = os.read(self.fd, size) except OSError, e: + if e.errno == errno.EAGAIN: + return space.w_None raise wrap_oserror(space, e, exception_name='w_IOError') @@ -362,6 +364,8 @@ try: buf = os.read(self.fd, length) except OSError, e: + if e.errno == errno.EAGAIN: + return space.w_None raise wrap_oserror(space, e, exception_name='w_IOError') rwbuffer.setslice(0, buf) diff --git a/pypy/module/_io/test/test_fileio.py b/pypy/module/_io/test/test_fileio.py --- a/pypy/module/_io/test/test_fileio.py +++ b/pypy/module/_io/test/test_fileio.py @@ -133,6 +133,19 @@ f.close() assert a == 'a\nbxxxxxxx' + def test_nonblocking_read(self): + import os, fcntl + r_fd, w_fd = os.pipe() + # set nonblocking + fcntl.fcntl(r_fd, fcntl.F_SETFL, os.O_NONBLOCK) + import _io + f = _io.FileIO(r_fd, 'r') + # Read from stream sould return None + assert f.read() is None + assert f.read(10) is None + a = bytearray('x' * 10) + assert f.readinto(a) is None + def test_repr(self): import _io f = _io.FileIO(self.tmpfile, 'r') diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py --- a/pypy/module/_socket/interp_socket.py +++ b/pypy/module/_socket/interp_socket.py @@ -67,9 +67,6 @@ self.connect(self.addr_from_object(space, w_addr)) except SocketError, e: raise converted_error(space, e) - except TypeError, e: - raise OperationError(space.w_TypeError, - space.wrap(str(e))) def connect_ex_w(self, space, w_addr): """connect_ex(address) -> errno diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py --- a/pypy/module/_socket/test/test_sock_app.py +++ b/pypy/module/_socket/test/test_sock_app.py @@ -529,26 +529,31 @@ import _socket, os if not hasattr(_socket, 'AF_UNIX'): skip('AF_UNIX not supported.') - sockpath = os.path.join(self.udir, 'app_test_unix_socket_connect') + oldcwd = os.getcwd() + os.chdir(self.udir) + try: + sockpath = 'app_test_unix_socket_connect' - serversock = _socket.socket(_socket.AF_UNIX) - serversock.bind(sockpath) - serversock.listen(1) + serversock = _socket.socket(_socket.AF_UNIX) + serversock.bind(sockpath) + serversock.listen(1) - clientsock = _socket.socket(_socket.AF_UNIX) - clientsock.connect(sockpath) - s, addr = serversock.accept() - assert not addr + clientsock = _socket.socket(_socket.AF_UNIX) + clientsock.connect(sockpath) + s, addr = serversock.accept() + assert not addr - s.send('X') - data = clientsock.recv(100) - assert data == 'X' - clientsock.send('Y') - data = s.recv(100) - assert data == 'Y' + s.send('X') + data = clientsock.recv(100) + assert data == 'X' + clientsock.send('Y') + data = s.recv(100) + assert data == 'Y' - clientsock.close() - s.close() + clientsock.close() + s.close() + finally: + os.chdir(oldcwd) class AppTestSocketTCP: diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py --- a/pypy/module/_ssl/test/test_ssl.py +++ b/pypy/module/_ssl/test/test_ssl.py @@ -64,8 +64,8 @@ def test_sslwrap(self): import _ssl, _socket, sys, gc - if sys.platform == 'darwin': - skip("hangs indefinitely on OSX (also on CPython)") + if sys.platform == 'darwin' or 'freebsd' in sys.platform: + skip("hangs indefinitely on OSX & FreeBSD (also on CPython)") s = _socket.socket() ss = _ssl.sslwrap(s, 0) exc = raises(_socket.error, ss.do_handshake) diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h --- a/pypy/module/cpyext/include/patchlevel.h +++ b/pypy/module/cpyext/include/patchlevel.h @@ -29,7 +29,7 @@ #define PY_VERSION "2.7.1" /* PyPy version as a string */ -#define PYPY_VERSION "1.7.1" +#define PYPY_VERSION "1.8.1" /* Subversion Revision number of this file (not of the repository). * Empty since Mercurial migration. */ diff --git a/pypy/module/fcntl/test/test_fcntl.py b/pypy/module/fcntl/test/test_fcntl.py --- a/pypy/module/fcntl/test/test_fcntl.py +++ b/pypy/module/fcntl/test/test_fcntl.py @@ -42,13 +42,9 @@ else: start_len = "qq" - if sys.platform in ('netbsd1', 'netbsd2', 'netbsd3', - 'Darwin1.2', 'darwin', - 'freebsd2', 'freebsd3', 'freebsd4', 'freebsd5', - 'freebsd6', 'freebsd7', 'freebsd8', 'freebsd9', - 'bsdos2', 'bsdos3', 'bsdos4', - 'openbsd', 'openbsd2', 'openbsd3', 'openbsd4', - 'openbsd5'): + if any(substring in sys.platform.lower() + for substring in ('netbsd', 'darwin', 'freebsd', 'bsdos', + 'openbsd')): if struct.calcsize('l') == 8: off_t = 'l' pid_t = 'i' @@ -182,7 +178,8 @@ def test_large_flag(self): import sys - if sys.platform == "darwin" or sys.platform.startswith("openbsd"): + if any(plat in sys.platform + for plat in ('darwin', 'openbsd', 'freebsd')): skip("Mac OS doesn't have any large flag in fcntl.h") import fcntl, sys if sys.maxint == 2147483647: diff --git a/pypy/module/gc/__init__.py b/pypy/module/gc/__init__.py --- a/pypy/module/gc/__init__.py +++ b/pypy/module/gc/__init__.py @@ -1,18 +1,18 @@ from pypy.interpreter.mixedmodule import MixedModule class Module(MixedModule): - appleveldefs = { - 'enable': 'app_gc.enable', - 'disable': 'app_gc.disable', - 'isenabled': 'app_gc.isenabled', - } interpleveldefs = { 'collect': 'interp_gc.collect', + 'enable': 'interp_gc.enable', + 'disable': 'interp_gc.disable', + 'isenabled': 'interp_gc.isenabled', 'enable_finalizers': 'interp_gc.enable_finalizers', 'disable_finalizers': 'interp_gc.disable_finalizers', 'garbage' : 'space.newlist([])', #'dump_heap_stats': 'interp_gc.dump_heap_stats', } + appleveldefs = { + } def __init__(self, space, w_name): if (not space.config.translating or diff --git a/pypy/module/gc/app_gc.py b/pypy/module/gc/app_gc.py deleted file mode 100644 --- a/pypy/module/gc/app_gc.py +++ /dev/null @@ -1,21 +0,0 @@ -# NOT_RPYTHON - -enabled = True - -def isenabled(): - global enabled - return enabled - -def enable(): - global enabled - import gc - if not enabled: - gc.enable_finalizers() - enabled = True - -def disable(): - global enabled - import gc - if enabled: - gc.disable_finalizers() - enabled = False diff --git a/pypy/module/gc/interp_gc.py b/pypy/module/gc/interp_gc.py --- a/pypy/module/gc/interp_gc.py +++ b/pypy/module/gc/interp_gc.py @@ -17,6 +17,26 @@ rgc.collect() return space.wrap(0) +def enable(space): + """Non-recursive version. Enable finalizers now. + If they were already enabled, no-op. + If they were disabled even several times, enable them anyway. + """ + if not space.user_del_action.enabled_at_app_level: + space.user_del_action.enabled_at_app_level = True + enable_finalizers(space) + +def disable(space): + """Non-recursive version. Disable finalizers now. Several calls + to this function are ignored. + """ + if space.user_del_action.enabled_at_app_level: + space.user_del_action.enabled_at_app_level = False + disable_finalizers(space) + +def isenabled(space): + return space.newbool(space.user_del_action.enabled_at_app_level) + def enable_finalizers(space): if space.user_del_action.finalizers_lock_count == 0: raise OperationError(space.w_ValueError, diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py --- a/pypy/module/micronumpy/__init__.py +++ b/pypy/module/micronumpy/__init__.py @@ -27,6 +27,7 @@ 'dot': 'interp_numarray.dot', 'fromstring': 'interp_support.fromstring', 'flatiter': 'interp_numarray.W_FlatIterator', + 'isna': 'interp_numarray.isna', 'True_': 'types.Bool.True', 'False_': 'types.Bool.False', @@ -70,6 +71,7 @@ ("exp", "exp"), ("fabs", "fabs"), ("floor", "floor"), + ("ceil", "ceil"), ("greater", "greater"), ("greater_equal", "greater_equal"), ("less", "less"), @@ -85,12 +87,13 @@ ("subtract", "subtract"), ('sqrt', 'sqrt'), ("tan", "tan"), + ('bitwise_and', 'bitwise_and'), + ('bitwise_or', 'bitwise_or'), ]: interpleveldefs[exposed] = "interp_ufuncs.get(space).%s" % impl appleveldefs = { 'average': 'app_numpy.average', - 'mean': 'app_numpy.mean', 'sum': 'app_numpy.sum', 'min': 'app_numpy.min', 'identity': 'app_numpy.identity', @@ -99,5 +102,4 @@ 'e': 'app_numpy.e', 'pi': 'app_numpy.pi', 'arange': 'app_numpy.arange', - 'reshape': 'app_numpy.reshape', } diff --git a/pypy/module/micronumpy/app_numpy.py b/pypy/module/micronumpy/app_numpy.py --- a/pypy/module/micronumpy/app_numpy.py +++ b/pypy/module/micronumpy/app_numpy.py @@ -11,23 +11,20 @@ def average(a): # This implements a weighted average, for now we don't implement the # weighting, just the average part! - return mean(a) + if not hasattr(a, "mean"): + a = _numpypy.array(a) + return a.mean() def identity(n, dtype=None): - a = _numpypy.zeros((n,n), dtype=dtype) + a = _numpypy.zeros((n, n), dtype=dtype) for i in range(n): a[i][i] = 1 return a -def mean(a, axis=None): - if not hasattr(a, "mean"): - a = _numpypy.array(a) - return a.mean(axis) - def sum(a,axis=None): '''sum(a, axis=None) Sum of array elements over a given axis. - + Parameters ---------- a : array_like @@ -35,7 +32,7 @@ axis : integer, optional Axis over which the sum is taken. By default `axis` is None, and all elements are summed. - + Returns ------- sum_along_axis : ndarray @@ -43,7 +40,7 @@ axis removed. If `a` is a 0-d array, or if `axis` is None, a scalar is returned. If an output array is specified, a reference to `out` is returned. - + See Also -------- ndarray.sum : Equivalent method. @@ -79,40 +76,3 @@ arr[j] = i i += step return arr - - -def reshape(a, shape): - '''reshape(a, newshape) - Gives a new shape to an array without changing its data. - - Parameters - ---------- - a : array_like - Array to be reshaped. - newshape : int or tuple of ints - The new shape should be compatible with the original shape. If - an integer, then the result will be a 1-D array of that length. - One shape dimension can be -1. In this case, the value is inferred - from the length of the array and remaining dimensions. - - Returns - ------- - reshaped_array : ndarray - This will be a new view object if possible; otherwise, it will - be a copy. - - - See Also - -------- - ndarray.reshape : Equivalent method. - - Notes - ----- - - It is not always possible to change the shape of an array without - copying the data. If you want an error to be raise if the data is copied, - you should assign the new shape to the shape attribute of the array -''' - if not hasattr(a, 'reshape'): - a = _numpypy.array(a) - return a.reshape(shape) diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py --- a/pypy/module/micronumpy/interp_dtype.py +++ b/pypy/module/micronumpy/interp_dtype.py @@ -20,7 +20,7 @@ class W_Dtype(Wrappable): _immutable_fields_ = ["itemtype", "num", "kind"] - def __init__(self, itemtype, num, kind, name, char, w_box_type, alternate_constructors=[]): + def __init__(self, itemtype, num, kind, name, char, w_box_type, alternate_constructors=[], aliases=[]): self.itemtype = itemtype self.num = num self.kind = kind @@ -28,6 +28,7 @@ self.char = char self.w_box_type = w_box_type self.alternate_constructors = alternate_constructors + self.aliases = aliases def malloc(self, length): # XXX find out why test_zjit explodes with tracking of allocations @@ -46,6 +47,10 @@ def getitem(self, storage, i): return self.itemtype.read(storage, self.itemtype.get_element_size(), i, 0) + def getitem_bool(self, storage, i): + isize = self.itemtype.get_element_size() + return self.itemtype.read_bool(storage, isize, i, 0) + def setitem(self, storage, i, box): self.itemtype.store(storage, self.itemtype.get_element_size(), i, 0, box) @@ -62,7 +67,7 @@ elif space.isinstance_w(w_dtype, space.w_str): name = space.str_w(w_dtype) for dtype in cache.builtin_dtypes: - if dtype.name == name or dtype.char == name: + if dtype.name == name or dtype.char == name or name in dtype.aliases: return dtype else: for dtype in cache.builtin_dtypes: @@ -84,6 +89,12 @@ def descr_get_shape(self, space): return space.newtuple([]) + def is_int_type(self): + return self.kind == SIGNEDLTR or self.kind == UNSIGNEDLTR + + def is_bool_type(self): + return self.kind == BOOLLTR + W_Dtype.typedef = TypeDef("dtype", __module__ = "numpypy", __new__ = interp2app(W_Dtype.descr__new__.im_func), @@ -107,7 +118,7 @@ kind=BOOLLTR, name="bool", char="?", - w_box_type = space.gettypefor(interp_boxes.W_BoolBox), + w_box_type=space.gettypefor(interp_boxes.W_BoolBox), alternate_constructors=[space.w_bool], ) self.w_int8dtype = W_Dtype( @@ -116,7 +127,7 @@ kind=SIGNEDLTR, name="int8", char="b", - w_box_type = space.gettypefor(interp_boxes.W_Int8Box) + w_box_type=space.gettypefor(interp_boxes.W_Int8Box) ) self.w_uint8dtype = W_Dtype( types.UInt8(), @@ -124,7 +135,7 @@ kind=UNSIGNEDLTR, name="uint8", char="B", - w_box_type = space.gettypefor(interp_boxes.W_UInt8Box), + w_box_type=space.gettypefor(interp_boxes.W_UInt8Box), ) self.w_int16dtype = W_Dtype( types.Int16(), @@ -132,7 +143,7 @@ kind=SIGNEDLTR, name="int16", char="h", - w_box_type = space.gettypefor(interp_boxes.W_Int16Box), + w_box_type=space.gettypefor(interp_boxes.W_Int16Box), ) self.w_uint16dtype = W_Dtype( types.UInt16(), @@ -140,7 +151,7 @@ kind=UNSIGNEDLTR, name="uint16", char="H", - w_box_type = space.gettypefor(interp_boxes.W_UInt16Box), + w_box_type=space.gettypefor(interp_boxes.W_UInt16Box), ) self.w_int32dtype = W_Dtype( types.Int32(), @@ -148,7 +159,7 @@ kind=SIGNEDLTR, name="int32", char="i", - w_box_type = space.gettypefor(interp_boxes.W_Int32Box), + w_box_type=space.gettypefor(interp_boxes.W_Int32Box), ) self.w_uint32dtype = W_Dtype( types.UInt32(), @@ -156,7 +167,7 @@ kind=UNSIGNEDLTR, name="uint32", char="I", - w_box_type = space.gettypefor(interp_boxes.W_UInt32Box), + w_box_type=space.gettypefor(interp_boxes.W_UInt32Box), ) if LONG_BIT == 32: name = "int32" @@ -168,7 +179,7 @@ kind=SIGNEDLTR, name=name, char="l", - w_box_type = space.gettypefor(interp_boxes.W_LongBox), + w_box_type=space.gettypefor(interp_boxes.W_LongBox), alternate_constructors=[space.w_int], ) self.w_ulongdtype = W_Dtype( @@ -177,7 +188,7 @@ kind=UNSIGNEDLTR, name="u" + name, char="L", - w_box_type = space.gettypefor(interp_boxes.W_ULongBox), + w_box_type=space.gettypefor(interp_boxes.W_ULongBox), ) self.w_int64dtype = W_Dtype( types.Int64(), @@ -185,7 +196,7 @@ kind=SIGNEDLTR, name="int64", char="q", - w_box_type = space.gettypefor(interp_boxes.W_Int64Box), + w_box_type=space.gettypefor(interp_boxes.W_Int64Box), alternate_constructors=[space.w_long], ) self.w_uint64dtype = W_Dtype( @@ -194,7 +205,7 @@ kind=UNSIGNEDLTR, name="uint64", char="Q", - w_box_type = space.gettypefor(interp_boxes.W_UInt64Box), + w_box_type=space.gettypefor(interp_boxes.W_UInt64Box), ) self.w_float32dtype = W_Dtype( types.Float32(), @@ -202,7 +213,7 @@ kind=FLOATINGLTR, name="float32", char="f", - w_box_type = space.gettypefor(interp_boxes.W_Float32Box), + w_box_type=space.gettypefor(interp_boxes.W_Float32Box), ) self.w_float64dtype = W_Dtype( types.Float64(), @@ -212,6 +223,7 @@ char="d", w_box_type = space.gettypefor(interp_boxes.W_Float64Box), alternate_constructors=[space.w_float], + aliases=["float"], ) self.builtin_dtypes = [ diff --git a/pypy/module/micronumpy/interp_iter.py b/pypy/module/micronumpy/interp_iter.py --- a/pypy/module/micronumpy/interp_iter.py +++ b/pypy/module/micronumpy/interp_iter.py @@ -4,6 +4,19 @@ from pypy.module.micronumpy.strides import calculate_broadcast_strides,\ calculate_slice_strides +# structures to describe slicing + +class Chunk(object): + def __init__(self, start, stop, step, lgt): + self.start = start + self.stop = stop + self.step = step + self.lgt = lgt + + def extend_shape(self, shape): + if self.step != 0: + shape.append(self.lgt) + class BaseTransform(object): pass @@ -38,11 +51,18 @@ self.size = size def next(self, shapelen): + return self._next(1) + + def _next(self, ofs): arr = instantiate(ArrayIterator) arr.size = self.size - arr.offset = self.offset + 1 + arr.offset = self.offset + ofs return arr + def next_no_increase(self, shapelen): + # a hack to make JIT believe this is always virtual + return self._next(0) + def done(self): return self.offset >= self.size @@ -157,6 +177,8 @@ offset += self.strides[i] break else: + if i == self.dim: + first_line = True indices[i] = 0 offset -= self.backstrides[i] else: diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -2,14 +2,15 @@ from pypy.interpreter.error import OperationError, operationerrfmt from pypy.interpreter.gateway import interp2app, NoneNotWrapped from pypy.interpreter.typedef import TypeDef, GetSetProperty -from pypy.module.micronumpy import interp_ufuncs, interp_dtype, signature +from pypy.module.micronumpy import interp_ufuncs, interp_dtype, signature,\ + interp_boxes from pypy.module.micronumpy.strides import calculate_slice_strides from pypy.rlib import jit from pypy.rpython.lltypesystem import lltype, rffi from pypy.tool.sourcetools import func_with_new_name from pypy.rlib.rstring import StringBuilder from pypy.module.micronumpy.interp_iter import ArrayIterator, OneDimIterator,\ - SkipLastAxisIterator + SkipLastAxisIterator, Chunk, ViewIterator numpy_driver = jit.JitDriver( greens=['shapelen', 'sig'], @@ -39,7 +40,24 @@ get_printable_location=signature.new_printable_location('slice'), name='numpy_slice', ) - +count_driver = jit.JitDriver( + greens=['shapelen'], + virtualizables=['frame'], + reds=['s', 'frame', 'iter', 'arr'], + name='numpy_count' +) +filter_driver = jit.JitDriver( + greens=['shapelen', 'sig'], + virtualizables=['frame'], + reds=['concr', 'argi', 'ri', 'frame', 'v', 'res', 'self'], + name='numpy_filter', +) +filter_set_driver = jit.JitDriver( + greens=['shapelen', 'sig'], + virtualizables=['frame'], + reds=['idx', 'idxi', 'frame', 'arr'], + name='numpy_filterset', +) def _find_shape_and_elems(space, w_iterable): shape = [space.len_w(w_iterable)] @@ -156,15 +174,17 @@ # fits the new shape, using those steps. If there is a shape/step mismatch # (meaning that the realignment of elements crosses from one step into another) # return None so that the caller can raise an exception. -def calc_new_strides(new_shape, old_shape, old_strides): +def calc_new_strides(new_shape, old_shape, old_strides, order): + # Return the proper strides for new_shape, or None if the mapping crosses + # stepping boundaries + # Assumes that prod(old_shape) == prod(new_shape), len(old_shape) > 1, and # len(new_shape) > 0 steps = [] last_step = 1 oldI = 0 new_strides = [] - if old_strides[0] < old_strides[-1]: - #Start at old_shape[0], old_stides[0] + if order == 'F': for i in range(len(old_shape)): steps.append(old_strides[i] / last_step) last_step *= old_shape[i] @@ -181,12 +201,10 @@ n_old_elems_to_use *= old_shape[oldI] if n_new_elems_used == n_old_elems_to_use: oldI += 1 - if oldI >= len(old_shape): - continue - cur_step = steps[oldI] - n_old_elems_to_use *= old_shape[oldI] - else: - #Start at old_shape[-1], old_strides[-1] + if oldI < len(old_shape): + cur_step = steps[oldI] + n_old_elems_to_use *= old_shape[oldI] + elif order == 'C': for i in range(len(old_shape) - 1, -1, -1): steps.insert(0, old_strides[i] / last_step) last_step *= old_shape[i] @@ -205,10 +223,10 @@ n_old_elems_to_use *= old_shape[oldI] if n_new_elems_used == n_old_elems_to_use: oldI -= 1 - if oldI < -len(old_shape): - continue - cur_step = steps[oldI] - n_old_elems_to_use *= old_shape[oldI] + if oldI >= -len(old_shape): + cur_step = steps[oldI] + n_old_elems_to_use *= old_shape[oldI] + assert len(new_strides) == len(new_shape) return new_strides class BaseArray(Wrappable): @@ -270,6 +288,9 @@ descr_gt = _binop_impl("greater") descr_ge = _binop_impl("greater_equal") + descr_and = _binop_impl("bitwise_and") + descr_or = _binop_impl("bitwise_or") + def _binop_right_impl(ufunc_name): def impl(self, space, w_other): w_other = scalar_w(space, @@ -409,9 +430,22 @@ def descr_copy(self, space): return self.copy(space) + def descr_flatten(self, space): + return self.flatten(space) + def copy(self, space): return self.get_concrete().copy(space) + def empty_copy(self, space, dtype): + shape = self.shape + size = 1 + for elem in shape: + size *= elem + return W_NDimArray(size, shape[:], dtype, 'C') + + def flatten(self, space): + return self.get_concrete().flatten(space) + def descr_len(self, space): if len(self.shape): return space.wrap(self.shape[0]) @@ -479,11 +513,69 @@ def _prepare_slice_args(self, space, w_idx): if (space.isinstance_w(w_idx, space.w_int) or space.isinstance_w(w_idx, space.w_slice)): - return [space.decode_index4(w_idx, self.shape[0])] - return [space.decode_index4(w_item, self.shape[i]) for i, w_item in + return [Chunk(*space.decode_index4(w_idx, self.shape[0]))] + return [Chunk(*space.decode_index4(w_item, self.shape[i])) for i, w_item in enumerate(space.fixedview(w_idx))] + def count_all_true(self, arr): + sig = arr.find_sig() + frame = sig.create_frame(self) + shapelen = len(arr.shape) + s = 0 + iter = None + while not frame.done(): + count_driver.jit_merge_point(arr=arr, frame=frame, iter=iter, s=s, + shapelen=shapelen) + iter = frame.get_final_iter() + s += arr.dtype.getitem_bool(arr.storage, iter.offset) + frame.next(shapelen) + return s + + def getitem_filter(self, space, arr): + concr = arr.get_concrete() + size = self.count_all_true(concr) + res = W_NDimArray(size, [size], self.find_dtype()) + ri = ArrayIterator(size) + shapelen = len(self.shape) + argi = concr.create_iter() + sig = self.find_sig() + frame = sig.create_frame(self) + v = None + while not frame.done(): + filter_driver.jit_merge_point(concr=concr, argi=argi, ri=ri, + frame=frame, v=v, res=res, sig=sig, + shapelen=shapelen, self=self) + if concr.dtype.getitem_bool(concr.storage, argi.offset): + v = sig.eval(frame, self) + res.setitem(ri.offset, v) + ri = ri.next(1) + else: + ri = ri.next_no_increase(1) + argi = argi.next(shapelen) + frame.next(shapelen) + return res + + def setitem_filter(self, space, idx, val): + size = self.count_all_true(idx) + arr = SliceArray([size], self.dtype, self, val) + sig = arr.find_sig() + shapelen = len(self.shape) + frame = sig.create_frame(arr) + idxi = idx.create_iter() + while not frame.done(): + filter_set_driver.jit_merge_point(idx=idx, idxi=idxi, sig=sig, + frame=frame, arr=arr, + shapelen=shapelen) + if idx.dtype.getitem_bool(idx.storage, idxi.offset): + sig.eval(frame, arr) + frame.next_from_second(1) + frame.next_first(shapelen) + idxi = idxi.next(shapelen) + def descr_getitem(self, space, w_idx): + if (isinstance(w_idx, BaseArray) and w_idx.shape == self.shape and + w_idx.find_dtype().is_bool_type()): + return self.getitem_filter(space, w_idx) if self._single_item_result(space, w_idx): concrete = self.get_concrete() item = concrete._index_of_single_item(space, w_idx) @@ -493,6 +585,11 @@ def descr_setitem(self, space, w_idx, w_value): self.invalidated() + if (isinstance(w_idx, BaseArray) and w_idx.shape == self.shape and + w_idx.find_dtype().is_bool_type()): + return self.get_concrete().setitem_filter(space, + w_idx.get_concrete(), + convert_to_array(space, w_value)) if self._single_item_result(space, w_idx): concrete = self.get_concrete() item = concrete._index_of_single_item(space, w_idx) @@ -509,9 +606,8 @@ def create_slice(self, chunks): shape = [] i = -1 - for i, (start_, stop, step, lgt) in enumerate(chunks): - if step != 0: - shape.append(lgt) + for i, chunk in enumerate(chunks): + chunk.extend_shape(shape) s = i + 1 assert s >= 0 shape += self.shape[s:] @@ -542,8 +638,8 @@ concrete = self.get_concrete() new_shape = get_shape_from_iterable(space, concrete.size, w_shape) # Since we got to here, prod(new_shape) == self.size - new_strides = calc_new_strides(new_shape, - concrete.shape, concrete.strides) + new_strides = calc_new_strides(new_shape, concrete.shape, + concrete.strides, concrete.order) if new_strides: # We can create a view, strides somehow match up. ndims = len(new_shape) @@ -581,7 +677,7 @@ def descr_var(self, space): # var = mean((values - mean(values)) ** 2) w_res = self.descr_sub(space, self.descr_mean(space, space.w_None)) - assert isinstance(w_res, BaseArray) + assert isinstance(w_res, BaseArray) w_res = w_res.descr_pow(space, space.wrap(2)) assert isinstance(w_res, BaseArray) return w_res.descr_mean(space, space.w_None) @@ -590,6 +686,10 @@ # std(v) = sqrt(var(v)) return interp_ufuncs.get(space).sqrt.call(space, [self.descr_var(space)]) + def descr_fill(self, space, w_value): + concr = self.get_concrete_or_scalar() + concr.fill(space, w_value) + def descr_nonzero(self, space): if self.size > 1: raise OperationError(space.w_ValueError, space.wrap( @@ -682,6 +782,14 @@ def copy(self, space): return Scalar(self.dtype, self.value) + def flatten(self, space): + array = W_NDimArray(self.size, [self.size], self.dtype) + array.setitem(0, self.value) + return array + + def fill(self, space, w_value): + self.value = self.dtype.coerce(space, w_value) + def create_sig(self): return signature.ScalarSignature(self.dtype) @@ -717,8 +825,7 @@ frame=frame, ri=ri, self=self, result=result) - result.dtype.setitem(result.storage, ri.offset, - sig.eval(frame, self)) + result.setitem(ri.offset, sig.eval(frame, self)) frame.next(shapelen) ri = ri.next(shapelen) return result @@ -788,7 +895,7 @@ Intermediate class for performing binary operations. """ _immutable_fields_ = ['left', 'right'] - + def __init__(self, ufunc, name, shape, calc_dtype, res_dtype, left, right): VirtualArray.__init__(self, name, shape, res_dtype) self.ufunc = ufunc @@ -825,14 +932,15 @@ self.left.create_sig(), self.right.create_sig()) class SliceArray(Call2): - def __init__(self, shape, dtype, left, right): + def __init__(self, shape, dtype, left, right, no_broadcast=False): + self.no_broadcast = no_broadcast Call2.__init__(self, None, 'sliceloop', shape, dtype, dtype, left, right) - + def create_sig(self): lsig = self.left.create_sig() rsig = self.right.create_sig() - if self.shape != self.right.shape: + if not self.no_broadcast and self.shape != self.right.shape: return signature.SliceloopBroadcastSignature(self.ufunc, self.name, self.calc_dtype, @@ -847,7 +955,7 @@ when we'll make AxisReduce lazy """ _immutable_fields_ = ['left', 'right'] - + def __init__(self, ufunc, name, shape, dtype, left, right, dim): Call2.__init__(self, ufunc, name, shape, dtype, dtype, left, right) @@ -919,14 +1027,14 @@ if size < 1: builder.append('[]') return - elif size == 1: - builder.append(dtype.itemtype.str_format(self.getitem(0))) - return if size > 1000: # Once this goes True it does not go back to False for recursive # calls use_ellipsis = True ndims = len(self.shape) + if ndims == 0: + builder.append(dtype.itemtype.str_format(self.getitem(0))) + return i = 0 builder.append('[') if ndims > 1: @@ -938,7 +1046,7 @@ builder.append('\n' + indent) else: builder.append(indent) - view = self.create_slice([(i, 0, 0, 1)]).get_concrete() + view = self.create_slice([Chunk(i, 0, 0, 1)]).get_concrete() view.to_str(space, comma, builder, indent=indent + ' ', use_ellipsis=use_ellipsis) if i < self.shape[0] - 1: @@ -955,7 +1063,7 @@ builder.append(indent) # create_slice requires len(chunks) > 1 in order to reduce # shape - view = self.create_slice([(i, 0, 0, 1)]).get_concrete() + view = self.create_slice([Chunk(i, 0, 0, 1)]).get_concrete() view.to_str(space, comma, builder, indent=indent + ' ', use_ellipsis=use_ellipsis) i += 1 @@ -1061,6 +1169,18 @@ array.setslice(space, self) return array + def flatten(self, space): + array = W_NDimArray(self.size, [self.size], self.dtype, self.order) + if self.supports_fast_slicing(): + array._fast_setslice(space, self) + else: + arr = SliceArray(array.shape, array.dtype, array, self, no_broadcast=True) + array._sliceloop(arr) + return array + + def fill(self, space, w_value): + self.setslice(space, scalar_w(space, self.dtype, w_value)) + class ViewArray(ConcreteArray): def create_sig(self): @@ -1081,6 +1201,10 @@ parent) self.start = start + def create_iter(self): + return ViewIterator(self.start, self.strides, self.backstrides, + self.shape) + def setshape(self, space, new_shape): if len(self.shape) < 1: return @@ -1104,7 +1228,8 @@ self.backstrides = backstrides self.shape = new_shape return - new_strides = calc_new_strides(new_shape, self.shape, self.strides) + new_strides = calc_new_strides(new_shape, self.shape, self.strides, + self.order) if new_strides is None: raise OperationError(space.w_AttributeError, space.wrap( "incompatible shape for a non-contiguous array")) @@ -1127,6 +1252,9 @@ self.shape = new_shape self.calc_strides(new_shape) + def create_iter(self): + return ArrayIterator(self.size) + def create_sig(self): return signature.ArraySignature(self.dtype) @@ -1181,6 +1309,7 @@ arr = W_NDimArray(size, shape[:], dtype=dtype, order=order) shapelen = len(shape) arr_iter = ArrayIterator(arr.size) + # XXX we might want to have a jitdriver here for i in range(len(elems_w)): w_elem = elems_w[i] dtype.setitem(arr.storage, arr_iter.offset, @@ -1247,6 +1376,9 @@ __gt__ = interp2app(BaseArray.descr_gt), __ge__ = interp2app(BaseArray.descr_ge), + __and__ = interp2app(BaseArray.descr_and), + __or__ = interp2app(BaseArray.descr_or), + __repr__ = interp2app(BaseArray.descr_repr), __str__ = interp2app(BaseArray.descr_str), __array_interface__ = GetSetProperty(BaseArray.descr_array_iface), @@ -1273,7 +1405,10 @@ var = interp2app(BaseArray.descr_var), std = interp2app(BaseArray.descr_std), + fill = interp2app(BaseArray.descr_fill), + copy = interp2app(BaseArray.descr_copy), + flatten = interp2app(BaseArray.descr_flatten), reshape = interp2app(BaseArray.descr_reshape), tolist = interp2app(BaseArray.descr_tolist), ) @@ -1311,3 +1446,11 @@ __iter__ = interp2app(W_FlatIterator.descr_iter), ) W_FlatIterator.acceptable_as_base_class = False + +def isna(space, w_obj): + if isinstance(w_obj, BaseArray): + arr = w_obj.empty_copy(space, + interp_dtype.get_dtype_cache(space).w_booldtype) + arr.fill(space, space.wrap(False)) + return arr + return space.wrap(False) diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py --- a/pypy/module/micronumpy/interp_ufuncs.py +++ b/pypy/module/micronumpy/interp_ufuncs.py @@ -249,15 +249,16 @@ class W_Ufunc2(W_Ufunc): - _immutable_fields_ = ["comparison_func", "func", "name"] + _immutable_fields_ = ["comparison_func", "func", "name", "int_only"] argcount = 2 def __init__(self, func, name, promote_to_float=False, promote_bools=False, - identity=None, comparison_func=False): + identity=None, comparison_func=False, int_only=False): W_Ufunc.__init__(self, name, promote_to_float, promote_bools, identity) self.func = func self.comparison_func = comparison_func + self.int_only = int_only def call(self, space, args_w): from pypy.module.micronumpy.interp_numarray import (Call2, @@ -268,6 +269,7 @@ w_rhs = convert_to_array(space, w_rhs) calc_dtype = find_binop_result_dtype(space, w_lhs.find_dtype(), w_rhs.find_dtype(), + int_only=self.int_only, promote_to_float=self.promote_to_float, promote_bools=self.promote_bools, ) @@ -304,10 +306,12 @@ def find_binop_result_dtype(space, dt1, dt2, promote_to_float=False, - promote_bools=False): + promote_bools=False, int_only=False): # dt1.num should be <= dt2.num if dt1.num > dt2.num: dt1, dt2 = dt2, dt1 + if int_only and (not dt1.is_int_type() or not dt2.is_int_type()): + raise OperationError(space.w_TypeError, space.wrap("Unsupported types")) # Some operations promote op(bool, bool) to return int8, rather than bool if promote_bools and (dt1.kind == dt2.kind == interp_dtype.BOOLLTR): return interp_dtype.get_dtype_cache(space).w_int8dtype @@ -425,6 +429,10 @@ ("add", "add", 2, {"identity": 0}), ("subtract", "sub", 2), ("multiply", "mul", 2, {"identity": 1}), + ("bitwise_and", "bitwise_and", 2, {"identity": 1, + 'int_only': True}), + ("bitwise_or", "bitwise_or", 2, {"identity": 0, + 'int_only': True}), ("divide", "div", 2, {"promote_bools": True}), ("mod", "mod", 2, {"promote_bools": True}), ("power", "pow", 2, {"promote_bools": True}), @@ -449,6 +457,7 @@ ("fabs", "fabs", 1, {"promote_to_float": True}), ("floor", "floor", 1, {"promote_to_float": True}), + ("ceil", "ceil", 1, {"promote_to_float": True}), ("exp", "exp", 1, {"promote_to_float": True}), ('sqrt', 'sqrt', 1, {'promote_to_float': True}), @@ -475,7 +484,7 @@ extra_kwargs["identity"] = identity func = ufunc_dtype_caller(space, ufunc_name, op_name, argcount, - comparison_func=extra_kwargs.get("comparison_func", False) + comparison_func=extra_kwargs.get("comparison_func", False), ) if argcount == 1: ufunc = W_Ufunc1(func, ufunc_name, **extra_kwargs) diff --git a/pypy/module/micronumpy/signature.py b/pypy/module/micronumpy/signature.py --- a/pypy/module/micronumpy/signature.py +++ b/pypy/module/micronumpy/signature.py @@ -82,6 +82,16 @@ for i in range(len(self.iterators)): self.iterators[i] = self.iterators[i].next(shapelen) + @unroll_safe + def next_from_second(self, shapelen): + """ Don't increase the first iterator + """ + for i in range(1, len(self.iterators)): + self.iterators[i] = self.iterators[i].next(shapelen) + + def next_first(self, shapelen): + self.iterators[0] = self.iterators[0].next(shapelen) + def get_final_iter(self): final_iter = promote(self.final_iter) if final_iter < 0: diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py --- a/pypy/module/micronumpy/strides.py +++ b/pypy/module/micronumpy/strides.py @@ -10,12 +10,12 @@ rstart = start rshape = [] i = -1 - for i, (start_, stop, step, lgt) in enumerate(chunks): - if step != 0: - rstrides.append(strides[i] * step) - rbackstrides.append(strides[i] * (lgt - 1) * step) - rshape.append(lgt) - rstart += strides[i] * start_ + for i, chunk in enumerate(chunks): + if chunk.step != 0: + rstrides.append(strides[i] * chunk.step) + rbackstrides.append(strides[i] * (chunk.lgt - 1) * chunk.step) + rshape.append(chunk.lgt) + rstart += strides[i] * chunk.start # add a reminder s = i + 1 assert s >= 0 diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py --- a/pypy/module/micronumpy/test/test_dtypes.py +++ b/pypy/module/micronumpy/test/test_dtypes.py @@ -166,14 +166,11 @@ # You can't subclass dtype raises(TypeError, type, "Foo", (dtype,), {}) - def test_new(self): - import _numpypy as np - assert np.int_(4) == 4 - assert np.float_(3.4) == 3.4 + def test_aliases(self): + from _numpypy import dtype - def test_pow(self): - from _numpypy import int_ - assert int_(4) ** 2 == 16 + assert dtype("float") is dtype(float) + class AppTestTypes(BaseNumpyAppTest): def test_abstract_types(self): @@ -189,6 +186,15 @@ raises(TypeError, numpy.floating, 0) raises(TypeError, numpy.inexact, 0) + def test_new(self): + import _numpypy as np + assert np.int_(4) == 4 + assert np.float_(3.4) == 3.4 + + def test_pow(self): + from _numpypy import int_ + assert int_(4) ** 2 == 16 + def test_bool(self): import _numpypy as numpy @@ -318,7 +324,7 @@ else: raises(OverflowError, numpy.int64, 9223372036854775807) raises(OverflowError, numpy.int64, '9223372036854775807') - + raises(OverflowError, numpy.int64, 9223372036854775808) raises(OverflowError, numpy.int64, '9223372036854775808') diff --git a/pypy/module/micronumpy/test/test_module.py b/pypy/module/micronumpy/test/test_module.py --- a/pypy/module/micronumpy/test/test_module.py +++ b/pypy/module/micronumpy/test/test_module.py @@ -2,16 +2,11 @@ class AppTestNumPyModule(BaseNumpyAppTest): - def test_mean(self): - from _numpypy import array, mean - assert mean(array(range(5))) == 2.0 - assert mean(range(5)) == 2.0 - def test_average(self): from _numpypy import array, average assert average(range(10)) == 4.5 assert average(array(range(10))) == 4.5 - + def test_sum(self): from _numpypy import array, sum assert sum(range(10)) == 45 @@ -21,7 +16,7 @@ from _numpypy import array, min assert min(range(10)) == 0 assert min(array(range(10))) == 0 - + def test_max(self): from _numpypy import array, max assert max(range(10)) == 9 diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -2,6 +2,7 @@ import py from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest from pypy.module.micronumpy.interp_numarray import W_NDimArray, shape_agreement +from pypy.module.micronumpy.interp_iter import Chunk from pypy.module.micronumpy import signature from pypy.interpreter.error import OperationError from pypy.conftest import gettestobjspace @@ -37,53 +38,54 @@ def test_create_slice_f(self): a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'F') - s = a.create_slice([(3, 0, 0, 1)]) + s = a.create_slice([Chunk(3, 0, 0, 1)]) assert s.start == 3 assert s.strides == [10, 50] assert s.backstrides == [40, 100] - s = a.create_slice([(1, 9, 2, 4)]) + s = a.create_slice([Chunk(1, 9, 2, 4)]) assert s.start == 1 assert s.strides == [2, 10, 50] assert s.backstrides == [6, 40, 100] - s = a.create_slice([(1, 5, 3, 2), (1, 2, 1, 1), (1, 0, 0, 1)]) + s = a.create_slice([Chunk(1, 5, 3, 2), Chunk(1, 2, 1, 1), Chunk(1, 0, 0, 1)]) assert s.shape == [2, 1] assert s.strides == [3, 10] assert s.backstrides == [3, 0] - s = a.create_slice([(0, 10, 1, 10), (2, 0, 0, 1)]) + s = a.create_slice([Chunk(0, 10, 1, 10), Chunk(2, 0, 0, 1)]) assert s.start == 20 assert s.shape == [10, 3] def test_create_slice_c(self): a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'C') - s = a.create_slice([(3, 0, 0, 1)]) + s = a.create_slice([Chunk(3, 0, 0, 1)]) assert s.start == 45 assert s.strides == [3, 1] assert s.backstrides == [12, 2] - s = a.create_slice([(1, 9, 2, 4)]) + s = a.create_slice([Chunk(1, 9, 2, 4)]) assert s.start == 15 assert s.strides == [30, 3, 1] assert s.backstrides == [90, 12, 2] - s = a.create_slice([(1, 5, 3, 2), (1, 2, 1, 1), (1, 0, 0, 1)]) + s = a.create_slice([Chunk(1, 5, 3, 2), Chunk(1, 2, 1, 1), + Chunk(1, 0, 0, 1)]) assert s.start == 19 assert s.shape == [2, 1] assert s.strides == [45, 3] assert s.backstrides == [45, 0] - s = a.create_slice([(0, 10, 1, 10), (2, 0, 0, 1)]) + s = a.create_slice([Chunk(0, 10, 1, 10), Chunk(2, 0, 0, 1)]) assert s.start == 6 assert s.shape == [10, 3] def test_slice_of_slice_f(self): a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'F') - s = a.create_slice([(5, 0, 0, 1)]) + s = a.create_slice([Chunk(5, 0, 0, 1)]) assert s.start == 5 - s2 = s.create_slice([(3, 0, 0, 1)]) + s2 = s.create_slice([Chunk(3, 0, 0, 1)]) assert s2.shape == [3] assert s2.strides == [50] assert s2.parent is a assert s2.backstrides == [100] assert s2.start == 35 - s = a.create_slice([(1, 5, 3, 2)]) - s2 = s.create_slice([(0, 2, 1, 2), (2, 0, 0, 1)]) + s = a.create_slice([Chunk(1, 5, 3, 2)]) + s2 = s.create_slice([Chunk(0, 2, 1, 2), Chunk(2, 0, 0, 1)]) assert s2.shape == [2, 3] assert s2.strides == [3, 50] assert s2.backstrides == [3, 100] @@ -91,16 +93,16 @@ def test_slice_of_slice_c(self): a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), order='C') - s = a.create_slice([(5, 0, 0, 1)]) + s = a.create_slice([Chunk(5, 0, 0, 1)]) assert s.start == 15 * 5 - s2 = s.create_slice([(3, 0, 0, 1)]) + s2 = s.create_slice([Chunk(3, 0, 0, 1)]) assert s2.shape == [3] assert s2.strides == [1] assert s2.parent is a assert s2.backstrides == [2] assert s2.start == 5 * 15 + 3 * 3 - s = a.create_slice([(1, 5, 3, 2)]) - s2 = s.create_slice([(0, 2, 1, 2), (2, 0, 0, 1)]) + s = a.create_slice([Chunk(1, 5, 3, 2)]) + s2 = s.create_slice([Chunk(0, 2, 1, 2), Chunk(2, 0, 0, 1)]) assert s2.shape == [2, 3] assert s2.strides == [45, 1] assert s2.backstrides == [45, 2] @@ -108,14 +110,14 @@ def test_negative_step_f(self): a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'F') - s = a.create_slice([(9, -1, -2, 5)]) + s = a.create_slice([Chunk(9, -1, -2, 5)]) assert s.start == 9 assert s.strides == [-2, 10, 50] assert s.backstrides == [-8, 40, 100] def test_negative_step_c(self): a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), order='C') - s = a.create_slice([(9, -1, -2, 5)]) + s = a.create_slice([Chunk(9, -1, -2, 5)]) assert s.start == 135 assert s.strides == [-30, 3, 1] assert s.backstrides == [-120, 12, 2] @@ -124,7 +126,7 @@ a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'F') r = a._index_of_single_item(self.space, self.newtuple(1, 2, 2)) assert r == 1 + 2 * 10 + 2 * 50 - s = a.create_slice([(0, 10, 1, 10), (2, 0, 0, 1)]) + s = a.create_slice([Chunk(0, 10, 1, 10), Chunk(2, 0, 0, 1)]) r = s._index_of_single_item(self.space, self.newtuple(1, 0)) assert r == a._index_of_single_item(self.space, self.newtuple(1, 2, 0)) r = s._index_of_single_item(self.space, self.newtuple(1, 1)) @@ -134,7 +136,7 @@ a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'C') r = a._index_of_single_item(self.space, self.newtuple(1, 2, 2)) assert r == 1 * 3 * 5 + 2 * 3 + 2 - s = a.create_slice([(0, 10, 1, 10), (2, 0, 0, 1)]) + s = a.create_slice([Chunk(0, 10, 1, 10), Chunk(2, 0, 0, 1)]) r = s._index_of_single_item(self.space, self.newtuple(1, 0)) assert r == a._index_of_single_item(self.space, self.newtuple(1, 2, 0)) r = s._index_of_single_item(self.space, self.newtuple(1, 1)) @@ -152,13 +154,18 @@ def test_calc_new_strides(self): from pypy.module.micronumpy.interp_numarray import calc_new_strides - assert calc_new_strides([2, 4], [4, 2], [4, 2]) == [8, 2] - assert calc_new_strides([2, 4, 3], [8, 3], [1, 16]) == [1, 2, 16] - assert calc_new_strides([2, 3, 4], [8, 3], [1, 16]) is None - assert calc_new_strides([24], [2, 4, 3], [48, 6, 1]) is None - assert calc_new_strides([24], [2, 4, 3], [24, 6, 2]) == [2] - assert calc_new_strides([105, 1], [3, 5, 7], [35, 7, 1]) == [1, 1] - assert calc_new_strides([1, 105], [3, 5, 7], [35, 7, 1]) == [105, 1] + assert calc_new_strides([2, 4], [4, 2], [4, 2], "C") == [8, 2] + assert calc_new_strides([2, 4, 3], [8, 3], [1, 16], 'F') == [1, 2, 16] + assert calc_new_strides([2, 3, 4], [8, 3], [1, 16], 'F') is None + assert calc_new_strides([24], [2, 4, 3], [48, 6, 1], 'C') is None + assert calc_new_strides([24], [2, 4, 3], [24, 6, 2], 'C') == [2] + assert calc_new_strides([105, 1], [3, 5, 7], [35, 7, 1],'C') == [1, 1] + assert calc_new_strides([1, 105], [3, 5, 7], [35, 7, 1],'C') == [105, 1] + assert calc_new_strides([1, 105], [3, 5, 7], [35, 7, 1],'F') is None + assert calc_new_strides([1, 1, 1, 105, 1], [15, 7], [7, 1],'C') == \ + [105, 105, 105, 1, 1] + assert calc_new_strides([1, 1, 105, 1, 1], [7, 15], [1, 7],'F') == \ + [1, 1, 1, 105, 105] class AppTestNumArray(BaseNumpyAppTest): @@ -383,6 +390,8 @@ a.shape = () #numpy allows this a.shape = (1,) + a = array(range(6)).reshape(2,3).T + raises(AttributeError, 'a.shape = 6') def test_reshape(self): from _numpypy import array, zeros @@ -726,16 +735,17 @@ assert d[1] == 12 def test_mean(self): - from _numpypy import array, mean + from _numpypy import array, arange a = array(range(5)) assert a.mean() == 2.0 assert a[:4].mean() == 1.5 a = array(range(105)).reshape(3, 5, 7) - b = mean(a, axis=0) - b[0,0]==35. + b = a.mean(axis=0) + b[0, 0]==35. assert a.mean(axis=0)[0, 0] == 35 assert (b == array(range(35, 70), dtype=float).reshape(5, 7)).all() - assert (mean(a, 2) == array(range(0, 15), dtype=float).reshape(3, 5) * 7 + 3).all() + assert (a.mean(2) == array(range(0, 15), dtype=float).reshape(3, 5) * 7 + 3).all() + assert (arange(10).reshape(5, 2).mean(axis=1) == [0.5, 2.5, 4.5, 6.5, 8.5]).all() def test_sum(self): from _numpypy import array @@ -771,6 +781,8 @@ assert ((a + a).T.sum(2).T == (a + a).sum(0)).all() assert (a.reshape(1,-1).sum(0) == range(105)).all() assert (a.reshape(1,-1).sum(1) == 5460) + assert (array([[1,2],[3,4]]).prod(0) == [3, 8]).all() + assert (array([[1,2],[3,4]]).prod(1) == [2, 12]).all() def test_identity(self): from _numpypy import identity, array @@ -1017,11 +1029,15 @@ assert a[0].tolist() == [17.1, 27.2] def test_var(self): - from _numpypy import array + from _numpypy import array, arange a = array(range(10)) assert a.var() == 8.25 a = array([5.0]) assert a.var() == 0.0 + a = arange(10).reshape(5, 2) + assert a.var() == 8.25 + #assert (a.var(0) == [8, 8]).all() + #assert (a.var(1) == [.25] * 5).all() def test_std(self): from _numpypy import array @@ -1030,6 +1046,22 @@ a = array([5.0]) assert a.std() == 0.0 + def test_flatten(self): + from _numpypy import array + + a = array([[1, 2, 3], [4, 5, 6]]) + assert (a.flatten() == [1, 2, 3, 4, 5, 6]).all() + a = array([[[1, 2], [3, 4]], [[5, 6], [7, 8]]]) + assert (a.flatten() == [1, 2, 3, 4, 5, 6, 7, 8]).all() + a = array([1, 2, 3, 4, 5, 6, 7, 8]) + assert (a[::2].flatten() == [1, 3, 5, 7]).all() + a = array([1, 2, 3]) + assert ((a + a).flatten() == [2, 4, 6]).all() + a = array(2) + assert (a.flatten() == [2]).all() + a = array([[1, 2], [3, 4]]) + assert (a.T.flatten() == [1, 3, 2, 4]).all() + class AppTestMultiDim(BaseNumpyAppTest): def test_init(self): @@ -1300,6 +1332,61 @@ assert isinstance(i['data'][0], int) raises(TypeError, getattr, array(3), '__array_interface__') + def test_array_indexing_one_elem(self): + skip("not yet") + from _numpypy import array, arange + raises(IndexError, 'arange(3)[array([3.5])]') + a = arange(3)[array([1])] + assert a == 1 + assert a[0] == 1 + raises(IndexError,'arange(3)[array([15])]') + assert arange(3)[array([-3])] == 0 + raises(IndexError,'arange(3)[array([-15])]') + assert arange(3)[array(1)] == 1 + + def test_fill(self): + from _numpypy import array + a = array([1, 2, 3]) + a.fill(10) + assert (a == [10, 10, 10]).all() + a.fill(False) + assert (a == [0, 0, 0]).all() + b = a[:1] + b.fill(4) + assert (b == [4]).all() + assert (a == [4, 0, 0]).all() + + c = b + b + c.fill(27) + assert (c == [27]).all() + + d = array(10) + d.fill(100) + assert d == 100 + + def test_array_indexing_bool(self): + from _numpypy import arange + a = arange(10) + assert (a[a > 3] == [4, 5, 6, 7, 8, 9]).all() + a = arange(10).reshape(5, 2) + assert (a[a > 3] == [4, 5, 6, 7, 8, 9]).all() + assert (a[a & 1 == 1] == [1, 3, 5, 7, 9]).all() + + def test_array_indexing_bool_setitem(self): + from _numpypy import arange, array + a = arange(6) + a[a > 3] = 15 + assert (a == [0, 1, 2, 3, 15, 15]).all() + a = arange(6).reshape(3, 2) + a[a & 1 == 1] = array([8, 9, 10]) + assert (a == [[0, 8], [2, 9], [4, 10]]).all() + + def test_isna(self): + from _numpypy import isna, array + # XXX for now + assert not isna(3) + assert (isna(array([1, 2, 3, 4])) == [False, False, False, False]).all() + class AppTestSupport(BaseNumpyAppTest): def setup_class(cls): @@ -1439,9 +1526,11 @@ assert repr(a) == "array(0.0)" a = array(0.2) assert repr(a) == "array(0.2)" + a = array([2]) + assert repr(a) == "array([2])" def test_repr_multi(self): - from _numpypy import arange, zeros + from _numpypy import arange, zeros, array a = zeros((3, 4)) assert repr(a) == '''array([[0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0], @@ -1464,6 +1553,9 @@ [498, 999], [499, 1000], [500, 1001]])''' + a = arange(2).reshape((2,1)) + assert repr(a) == '''array([[0], + [1]])''' def test_repr_slice(self): from _numpypy import array, zeros @@ -1548,18 +1640,3 @@ a = arange(0, 0.8, 0.1) assert len(a) == 8 assert arange(False, True, True).dtype is dtype(int) - - -class AppTestRanges(BaseNumpyAppTest): - def test_app_reshape(self): - from _numpypy import arange, array, dtype, reshape - a = arange(12) - b = reshape(a, (3, 4)) - assert b.shape == (3, 4) - a = range(12) - b = reshape(a, (3, 4)) - assert b.shape == (3, 4) - a = array(range(105)).reshape(3, 5, 7) - assert a.reshape(1, -1).shape == (1, 105) - assert a.reshape(1, 1, -1).shape == (1, 1, 105) - assert a.reshape(-1, 1, 1).shape == (105, 1, 1) diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py --- a/pypy/module/micronumpy/test/test_ufuncs.py +++ b/pypy/module/micronumpy/test/test_ufuncs.py @@ -190,14 +190,24 @@ for i in range(3): assert c[i] == a[i] - b[i] - def test_floor(self): - from _numpypy import array, floor - - reference = [-2.0, -1.0, 0.0, 1.0, 1.0] - a = array([-1.4, -1.0, 0.0, 1.0, 1.4]) + def test_floorceil(self): + from _numpypy import array, floor, ceil + import math + reference = [-2.0, -2.0, -1.0, 0.0, 1.0, 1.0, 0] + a = array([-1.4, -1.5, -1.0, 0.0, 1.0, 1.4, 0.5]) b = floor(a) for i in range(5): assert b[i] == reference[i] + reference = [-1.0, -1.0, -1.0, 0.0, 1.0, 2.0, 1.0] + a = array([-1.4, -1.5, -1.0, 0.0, 1.0, 1.4, 0.5]) + b = ceil(a) + assert (reference == b).all() + inf = float("inf") + data = [1.5, 2.9999, -1.999, inf] + results = [math.floor(x) for x in data] + assert (floor(data) == results).all() + results = [math.ceil(x) for x in data] + assert (ceil(data) == results).all() def test_copysign(self): from _numpypy import array, copysign @@ -238,7 +248,7 @@ assert b[i] == math.sin(a[i]) a = sin(array([True, False], dtype=bool)) - assert abs(a[0] - sin(1)) < 1e-7 # a[0] will be less precise + assert abs(a[0] - sin(1)) < 1e-7 # a[0] will be less precise assert a[1] == 0.0 def test_cos(self): @@ -259,7 +269,6 @@ for i in range(len(a)): assert b[i] == math.tan(a[i]) - def test_arcsin(self): import math from _numpypy import array, arcsin @@ -283,7 +292,6 @@ for i in range(len(a)): assert b[i] == math.acos(a[i]) - a = array([-10, -1.5, -1.01, 1.01, 1.5, 10, float('nan'), float('inf'), float('-inf')]) b = arccos(a) for f in b: @@ -347,11 +355,20 @@ raises(ValueError, maximum.reduce, []) def test_reduceND(self): - from numpypy import add, arange + from _numpypy import add, arange a = arange(12).reshape(3, 4) assert (add.reduce(a, 0) == [12, 15, 18, 21]).all() assert (add.reduce(a, 1) == [6.0, 22.0, 38.0]).all() + def test_bitwise(self): + from _numpypy import bitwise_and, bitwise_or, arange, array + a = arange(6).reshape(2, 3) + assert (a & 1 == [[0, 1, 0], [1, 0, 1]]).all() + assert (a & 1 == bitwise_and(a, 1)).all() + assert (a | 1 == [[1, 1, 3], [3, 5, 5]]).all() + assert (a | 1 == bitwise_or(a, 1)).all() + raises(TypeError, 'array([1.0]) & 1') + def test_comparisons(self): import operator from _numpypy import equal, not_equal, less, less_equal, greater, greater_equal diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py --- a/pypy/module/micronumpy/test/test_zjit.py +++ b/pypy/module/micronumpy/test/test_zjit.py @@ -217,6 +217,7 @@ # This is the sum of the ops for both loops, however if you remove the # optimization then you end up with 2 float_adds, so we can still be # sure it was optimized correctly. + py.test.skip("too fragile") self.check_resops({'setinteriorfield_raw': 4, 'getfield_gc': 22, 'getarrayitem_gc': 4, 'getarrayitem_gc_pure': 2, 'getfield_gc_pure': 8, @@ -349,7 +350,8 @@ self.check_trace_count(1) self.check_simple_loop({'getinteriorfield_raw': 2, 'float_add': 1, 'setinteriorfield_raw': 1, 'int_add': 2, - 'int_eq': 1, 'guard_false': 1, 'jump': 1}) + 'int_eq': 1, 'guard_false': 1, 'jump': 1, + 'arraylen_gc': 1}) def define_virtual_slice(): return """ diff --git a/pypy/module/micronumpy/tool/numready/__init__.py b/pypy/module/micronumpy/tool/numready/__init__.py new file mode 100644 diff --git a/pypy/module/micronumpy/tool/numready/__main__.py b/pypy/module/micronumpy/tool/numready/__main__.py new file mode 100644 --- /dev/null +++ b/pypy/module/micronumpy/tool/numready/__main__.py @@ -0,0 +1,6 @@ +import sys + +from .main import main + + +main(sys.argv) diff --git a/pypy/module/micronumpy/tool/numready/kinds.py b/pypy/module/micronumpy/tool/numready/kinds.py new file mode 100644 --- /dev/null +++ b/pypy/module/micronumpy/tool/numready/kinds.py @@ -0,0 +1,4 @@ +KINDS = { + "UNKNOWN": "U", + "TYPE": "T", +} diff --git a/pypy/module/micronumpy/tool/numready/main.py b/pypy/module/micronumpy/tool/numready/main.py new file mode 100644 --- /dev/null +++ b/pypy/module/micronumpy/tool/numready/main.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" +This should be run under PyPy. +""" + +import os +import platform +import subprocess +import tempfile +import webbrowser +from collections import OrderedDict + +import jinja2 + +from .kinds import KINDS + + +class SearchableSet(object): + def __init__(self, items=()): + self._items = {} + for item in items: + self.add(item) + + def __iter__(self): + return iter(self._items) + + def __contains__(self, other): + return other in self._items + + def __getitem__(self, idx): + return self._items[idx] + + def add(self, item): + self._items[item] = item + + def __len__(self): + return len(self._items) + +class Item(object): + def __init__(self, name, kind, subitems=None): + self.name = name + self.kind = kind + self.subitems = subitems + + def __hash__(self): + return hash(self.name) + + def __eq__(self, other): + if isinstance(other, str): + return self.name == other + return self.name == other.name + + +class ItemStatus(object): + def __init__(self, name, pypy_exists): + self.name = name + self.cls = 'exists' if pypy_exists else '' + self.symbol = u"✔" if pypy_exists else u'✖' + + def __lt__(self, other): + return self.name < other.name + +def find_numpy_items(python, modname="numpy", attr=None): + args = [ + python, os.path.join(os.path.dirname(__file__), "search.py"), modname + ] + if attr is not None: + args.append(attr) + lines = subprocess.check_output(args).splitlines() + items = SearchableSet() + for line in lines: + kind, name = line.split(" : ", 1) + subitems = None + if kind == KINDS["TYPE"]: + if name in ['ndarray', 'dtype']: + subitems = find_numpy_items(python, modname, name) + items.add(Item(name, kind, subitems)) + return items + +def split(lst): + SPLIT = 5 + lgt = len(lst) // SPLIT + 1 + l = [[] for i in range(lgt)] + for i in range(lgt): + for k in range(SPLIT): + if k * lgt + i < len(lst): + l[i].append(lst[k * lgt + i]) + return l + +def main(argv): + cpy_items = find_numpy_items("/usr/bin/python") + pypy_items = find_numpy_items(argv[1], "numpypy") + all_items = [] + + msg = '%d/%d names, %d/%d ndarray attributes, %d/%d dtype attributes' % ( + len(pypy_items), len(cpy_items), len(pypy_items['ndarray'].subitems), + len(cpy_items['ndarray'].subitems), len(pypy_items['dtype'].subitems), + len(cpy_items['dtype'].subitems)) + for item in cpy_items: + pypy_exists = item in pypy_items + if item.subitems: + for sub in item.subitems: + all_items.append( + ItemStatus(item.name + "." + sub.name, pypy_exists=pypy_exists and pypy_items[item].subitems and sub in pypy_items[item].subitems) + ) + all_items.append(ItemStatus(item.name, pypy_exists=item in pypy_items)) + env = jinja2.Environment( + loader=jinja2.FileSystemLoader(os.path.dirname(__file__)) + ) + html = env.get_template("page.html").render(all_items=split(sorted(all_items)), msg=msg) + if len(argv) > 2: + with open(argv[2], 'w') as f: + f.write(html.encode("utf-8")) + else: + with tempfile.NamedTemporaryFile(delete=False) as f: + f.write(html.encode("utf-8")) + print "Saved in: %s" % f.name diff --git a/pypy/module/micronumpy/tool/numready/page.html b/pypy/module/micronumpy/tool/numready/page.html new file mode 100644 --- /dev/null +++ b/pypy/module/micronumpy/tool/numready/page.html @@ -0,0 +1,65 @@ +<!DOCTYPE html> +<html lang="en"> + <head> + <title>NumPyPy Status</title> + <meta http-equiv="content-type" content="text/html; charset=utf-8"> + <style type="text/css"> + body { + font-family: 'Consolas', 'Bitstream Vera Sans Mono', monospace; + } + h1 { + text-align: center; + } + h3 { + text-align: center; + } + table { + border: 8px solid #DFDECB; + margin: 30px auto; + font-size: 12px; + } + table th { + text-align: left; + } + table td { + padding: 4px 10px; + text-align: center; + } + .exists { + background-color: #337792; + color: white; + border: 1px solid #234F61; + } + </style> + </head> + <body> + <h1>NumPyPy Status</h1> + <h3>Overall: {{ msg }}</h3> + <table> + <thead> + <tr> + <th></th> + <th>PyPy</th> + <th></th> + <th>PyPy</th> + <th></th> + <th>PyPy</th> + <th></th> + <th>PyPy</th> + <th></th> + <th>PyPy</th> + </tr> + </thead> + <tbody> + {% for chunk in all_items %} + <tr> + {% for item in chunk %} + <th class='{{ item.cls }}'>{{ item.name }}</th> + <td class='{{ item.cls }}'>{{ item.symbol }}</td> + {% endfor %} + </tr> + {% endfor %} + </tbody> + </table> + </body> +</html> diff --git a/pypy/module/micronumpy/tool/numready/search.py b/pypy/module/micronumpy/tool/numready/search.py new file mode 100644 --- /dev/null +++ b/pypy/module/micronumpy/tool/numready/search.py @@ -0,0 +1,33 @@ +import sys +import types + +# Evil implicit relative import. +from kinds import KINDS + + +def main(argv): + if len(argv) == 2: + [_, modname] = argv + attr = None + elif len(argv) == 3: + [_, modname, attr] = argv + else: + sys.exit("Wrong number of args") + __import__(modname) + obj = sys.modules[modname] + + if attr is not None: + obj = getattr(obj, attr) + + for name in dir(obj): + if attr is None and name.startswith("_"): + continue + subobj = getattr(obj, name) + if isinstance(subobj, types.TypeType): + kind = KINDS["TYPE"] + else: + kind = KINDS["UNKNOWN"] + print kind, ":", name + +if __name__ == "__main__": + main(sys.argv) \ No newline at end of file diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py --- a/pypy/module/micronumpy/types.py +++ b/pypy/module/micronumpy/types.py @@ -94,6 +94,9 @@ width, storage, i, offset )) + def read_bool(self, storage, width, i, offset): + raise NotImplementedError + def store(self, storage, width, i, offset, box): value = self.unbox(box) libffi.array_setitem(clibffi.cast_type_to_ffitype(self.T), @@ -168,6 +171,7 @@ @simple_binary_op def min(self, v1, v2): return min(v1, v2) + class Bool(BaseType, Primitive): T = lltype.Bool @@ -185,6 +189,11 @@ else: return self.False + + def read_bool(self, storage, width, i, offset): + return libffi.array_getitem(clibffi.cast_type_to_ffitype(self.T), + width, storage, i, offset) + def coerce_subtype(self, space, w_subtype, w_item): # Doesn't return subclasses so it can return the constants. return self._coerce(space, w_item) @@ -253,6 +262,14 @@ assert v == 0 return 0 + @simple_binary_op + def bitwise_and(self, v1, v2): + return v1 & v2 + + @simple_binary_op + def bitwise_or(self, v1, v2): + return v1 | v2 + class Int8(BaseType, Integer): T = rffi.SIGNEDCHAR BoxType = interp_boxes.W_Int8Box @@ -374,6 +391,10 @@ return math.floor(v) @simple_unary_op + def ceil(self, v): + return math.ceil(v) + + @simple_unary_op def exp(self, v): try: return math.exp(v) @@ -436,4 +457,4 @@ class Float64(BaseType, Float): T = rffi.DOUBLE BoxType = interp_boxes.W_Float64Box - format_code = "d" \ No newline at end of file + format_code = "d" diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -371,6 +371,8 @@ if hasattr(__import__(os.name), "forkpty"): def test_forkpty(self): import sys + if 'freebsd' in sys.platform: + skip("hangs indifinitly on FreeBSD (also on CPython).") os = self.posix childpid, master_fd = os.forkpty() assert isinstance(childpid, int) diff --git a/pypy/module/pypyjit/__init__.py b/pypy/module/pypyjit/__init__.py --- a/pypy/module/pypyjit/__init__.py +++ b/pypy/module/pypyjit/__init__.py @@ -11,6 +11,7 @@ 'set_optimize_hook': 'interp_resop.set_optimize_hook', 'set_abort_hook': 'interp_resop.set_abort_hook', 'ResOperation': 'interp_resop.WrappedOp', + 'DebugMergePoint': 'interp_resop.DebugMergePoint', 'Box': 'interp_resop.WrappedBox', } diff --git a/pypy/module/pypyjit/interp_resop.py b/pypy/module/pypyjit/interp_resop.py --- a/pypy/module/pypyjit/interp_resop.py +++ b/pypy/module/pypyjit/interp_resop.py @@ -1,5 +1,6 @@ -from pypy.interpreter.typedef import TypeDef, GetSetProperty +from pypy.interpreter.typedef import (TypeDef, GetSetProperty, + interp_attrproperty, interp_attrproperty_w) from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.gateway import unwrap_spec, interp2app, NoneNotWrapped from pypy.interpreter.pycode import PyCode @@ -10,6 +11,7 @@ from pypy.jit.metainterp.resoperation import rop, AbstractResOp from pypy.rlib.nonconst import NonConstant from pypy.rlib import jit_hooks +from pypy.module.pypyjit.interp_jit import pypyjitdriver class Cache(object): in_recursion = False @@ -111,13 +113,24 @@ def wrap_oplist(space, logops, operations, ops_offset=None): l_w = [] + jitdrivers_sd = logops.metainterp_sd.jitdrivers_sd for op in operations: if ops_offset is None: ofs = -1 else: ofs = ops_offset.get(op, 0) - l_w.append(WrappedOp(jit_hooks._cast_to_gcref(op), ofs, - logops.repr_of_resop(op))) + if op.opnum == rop.DEBUG_MERGE_POINT: + jd_sd = jitdrivers_sd[op.getarg(0).getint()] + greenkey = op.getarglist()[2:] + repr = jd_sd.warmstate.get_location_str(greenkey) + w_greenkey = wrap_greenkey(space, jd_sd.jitdriver, greenkey, repr) + l_w.append(DebugMergePoint(space, jit_hooks._cast_to_gcref(op), + logops.repr_of_resop(op), + jd_sd.jitdriver.name, + w_greenkey)) + else: + l_w.append(WrappedOp(jit_hooks._cast_to_gcref(op), ofs, + logops.repr_of_resop(op))) return l_w class WrappedBox(Wrappable): @@ -150,6 +163,15 @@ llres = res.llbox return WrappedOp(jit_hooks.resop_new(num, args, llres), offset, repr) + at unwrap_spec(repr=str, jd_name=str) +def descr_new_dmp(space, w_tp, w_args, repr, jd_name, w_greenkey): + args = [space.interp_w(WrappedBox, w_arg).llbox for w_arg in + space.listview(w_args)] + num = rop.DEBUG_MERGE_POINT + return DebugMergePoint(space, + jit_hooks.resop_new(num, args, jit_hooks.emptyval()), + repr, jd_name, w_greenkey) + class WrappedOp(Wrappable): """ A class representing a single ResOperation, wrapped nicely """ @@ -182,6 +204,25 @@ box = space.interp_w(WrappedBox, w_box) jit_hooks.resop_setresult(self.op, box.llbox) +class DebugMergePoint(WrappedOp): + def __init__(self, space, op, repr_of_resop, jd_name, w_greenkey): + WrappedOp.__init__(self, op, -1, repr_of_resop) + self.w_greenkey = w_greenkey + self.jd_name = jd_name + + def get_pycode(self, space): + if self.jd_name == pypyjitdriver.name: + return space.getitem(self.w_greenkey, space.wrap(0)) + raise OperationError(space.w_AttributeError, space.wrap("This DebugMergePoint doesn't belong to the main Python JitDriver")) + + def get_bytecode_no(self, space): + if self.jd_name == pypyjitdriver.name: + return space.getitem(self.w_greenkey, space.wrap(1)) + raise OperationError(space.w_AttributeError, space.wrap("This DebugMergePoint doesn't belong to the main Python JitDriver")) + + def get_jitdriver_name(self, space): + return space.wrap(self.jd_name) + WrappedOp.typedef = TypeDef( 'ResOperation', __doc__ = WrappedOp.__doc__, @@ -195,3 +236,15 @@ WrappedOp.descr_setresult) ) WrappedOp.acceptable_as_base_class = False + +DebugMergePoint.typedef = TypeDef( + 'DebugMergePoint', WrappedOp.typedef, + __new__ = interp2app(descr_new_dmp), + greenkey = interp_attrproperty_w("w_greenkey", cls=DebugMergePoint), + pycode = GetSetProperty(DebugMergePoint.get_pycode), + bytecode_no = GetSetProperty(DebugMergePoint.get_bytecode_no), + jitdriver_name = GetSetProperty(DebugMergePoint.get_jitdriver_name), +) +DebugMergePoint.acceptable_as_base_class = False + + diff --git a/pypy/module/pypyjit/policy.py b/pypy/module/pypyjit/policy.py --- a/pypy/module/pypyjit/policy.py +++ b/pypy/module/pypyjit/policy.py @@ -127,7 +127,7 @@ 'imp', 'sys', 'array', '_ffi', 'itertools', 'operator', 'posix', '_socket', '_sre', '_lsprof', '_weakref', '__pypy__', 'cStringIO', '_collections', 'struct', - 'mmap', 'marshal']: + 'mmap', 'marshal', '_codecs']: if modname == 'pypyjit' and 'interp_resop' in rest: return False return True diff --git a/pypy/module/pypyjit/test/test_jit_hook.py b/pypy/module/pypyjit/test/test_jit_hook.py --- a/pypy/module/pypyjit/test/test_jit_hook.py +++ b/pypy/module/pypyjit/test/test_jit_hook.py @@ -92,6 +92,7 @@ cls.w_on_compile_bridge = space.wrap(interp2app(interp_on_compile_bridge)) cls.w_on_abort = space.wrap(interp2app(interp_on_abort)) cls.w_int_add_num = space.wrap(rop.INT_ADD) + cls.w_dmp_num = space.wrap(rop.DEBUG_MERGE_POINT) cls.w_on_optimize = space.wrap(interp2app(interp_on_optimize)) cls.orig_oplist = oplist @@ -117,6 +118,10 @@ assert elem[2][2] == False assert len(elem[3]) == 4 int_add = elem[3][0] + dmp = elem[3][1] + assert isinstance(dmp, pypyjit.DebugMergePoint) + assert dmp.pycode is self.f.func_code + assert dmp.greenkey == (self.f.func_code, 0, False) #assert int_add.name == 'int_add' assert int_add.num == self.int_add_num self.on_compile_bridge() @@ -211,3 +216,18 @@ assert op.getarg(0).getint() == 4 op.result = box assert op.result.getint() == 1 + + def test_creation_dmp(self): + from pypyjit import DebugMergePoint, Box + + def f(): + pass + + op = DebugMergePoint([Box(0)], 'repr', 'pypyjit', (f.func_code, 0, 0)) + assert op.bytecode_no == 0 + assert op.pycode is f.func_code + assert repr(op) == 'repr' + assert op.jitdriver_name == 'pypyjit' + assert op.num == self.dmp_num + op = DebugMergePoint([Box(0)], 'repr', 'notmain', ('str',)) + raises(AttributeError, 'op.pycode') diff --git a/pypy/module/pypyjit/test_pypy_c/test_math.py b/pypy/module/pypyjit/test_pypy_c/test_math.py --- a/pypy/module/pypyjit/test_pypy_c/test_math.py +++ b/pypy/module/pypyjit/test_pypy_c/test_math.py @@ -90,3 +90,19 @@ --TICK-- jump(..., descr=) """) + + def test_pow_two(self): + def main(n): + s = 0.123 + while n > 0: + s -= s ** 2 # ID: pow + n -= 1 + return s + log = self.run(main, [500]) + assert abs(log.result - main(500)) < 1e-9 + loop, = log.loops_by_filename(self.filepath) + assert loop.match_by_id("pow", """ + guard_not_invalidated(descr=...) + f38 = float_mul(f30, f30) + f39 = float_sub(f30, f38) + """) diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py --- a/pypy/module/sys/version.py +++ b/pypy/module/sys/version.py @@ -10,7 +10,7 @@ CPYTHON_VERSION = (2, 7, 1, "final", 42) #XXX # sync patchlevel.h CPYTHON_API_VERSION = 1013 #XXX # sync with include/modsupport.h -PYPY_VERSION = (1, 7, 1, "dev", 0) #XXX # sync patchlevel.h +PYPY_VERSION = (1, 8, 1, "dev", 0) #XXX # sync patchlevel.h if platform.name == 'msvc': COMPILER_INFO = 'MSC v.%d 32 bit' % (platform.version * 10 + 600) diff --git a/lib_pypy/numpypy/test/test_fromnumeric.py b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py rename from lib_pypy/numpypy/test/test_fromnumeric.py rename to pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py --- a/lib_pypy/numpypy/test/test_fromnumeric.py +++ b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py @@ -1,7 +1,7 @@ - from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest -class AppTestFromNumeric(BaseNumpyAppTest): + +class AppTestFromNumeric(BaseNumpyAppTest): def test_argmax(self): # tests taken from numpy/core/fromnumeric.py docstring from numpypy import array, arange, argmax @@ -18,12 +18,12 @@ from numpypy import array, arange, argmin a = arange(6).reshape((2,3)) assert argmin(a) == 0 - # assert (argmax(a, axis=0) == array([0, 0, 0])).all() - # assert (argmax(a, axis=1) == array([0, 0])).all() + #assert (argmin(a, axis=0) == array([0, 0, 0])).all() + #assert (argmin(a, axis=1) == array([0, 0])).all() b = arange(6) b[1] = 0 assert argmin(b) == 0 - + def test_shape(self): # tests taken from numpy/core/fromnumeric.py docstring from numpypy import array, identity, shape @@ -40,11 +40,11 @@ assert sum([0.5, 1.5])== 2.0 assert sum([[0, 1], [0, 5]]) == 6 # assert sum([0.5, 0.7, 0.2, 1.5], dtype=int32) == 1 - # assert (sum([[0, 1], [0, 5]], axis=0) == array([0, 6])).all() - # assert (sum([[0, 1], [0, 5]], axis=1) == array([1, 5])).all() + assert (sum([[0, 1], [0, 5]], axis=0) == array([0, 6])).all() + assert (sum([[0, 1], [0, 5]], axis=1) == array([1, 5])).all() # If the accumulator is too small, overflow occurs: # assert ones(128, dtype=int8).sum(dtype=int8) == -128 - + def test_amin(self): # tests taken from numpy/core/fromnumeric.py docstring from numpypy import array, arange, amin @@ -86,24 +86,54 @@ assert ndim([[1,2,3],[4,5,6]]) == 2 assert ndim(array([[1,2,3],[4,5,6]])) == 2 assert ndim(1) == 0 - + def test_rank(self): # tests taken from numpy/core/fromnumeric.py docstring from numpypy import array, rank assert rank([[1,2,3],[4,5,6]]) == 2 assert rank(array([[1,2,3],[4,5,6]])) == 2 assert rank(1) == 0 - + def test_var(self): from numpypy import array, var a = array([[1,2],[3,4]]) assert var(a) == 1.25 - # assert (np.var(a,0) == array([ 1., 1.])).all() - # assert (np.var(a,1) == array([ 0.25, 0.25])).all() + #assert (var(a,0) == array([ 1., 1.])).all() + #assert (var(a,1) == array([ 0.25, 0.25])).all() def test_std(self): from numpypy import array, std a = array([[1, 2], [3, 4]]) assert std(a) == 1.1180339887498949 - # assert (std(a, axis=0) == array([ 1., 1.])).all() - # assert (std(a, axis=1) == array([ 0.5, 0.5]).all() + #assert (std(a, axis=0) == array([ 1., 1.])).all() + #assert (std(a, axis=1) == array([ 0.5, 0.5])).all() + + def test_mean(self): + from numpypy import array, mean, arange + assert mean(array(range(5))) == 2.0 + assert mean(range(5)) == 2.0 + assert (mean(arange(10).reshape(5, 2), axis=0) == [4, 5]).all() + assert (mean(arange(10).reshape(5, 2), axis=1) == [0.5, 2.5, 4.5, 6.5, 8.5]).all() + + def test_reshape(self): + from numpypy import arange, array, dtype, reshape + a = arange(12) + b = reshape(a, (3, 4)) + assert b.shape == (3, 4) + a = range(12) + b = reshape(a, (3, 4)) + assert b.shape == (3, 4) + a = array(range(105)).reshape(3, 5, 7) + assert reshape(a, (1, -1)).shape == (1, 105) + assert reshape(a, (1, 1, -1)).shape == (1, 1, 105) + assert reshape(a, (-1, 1, 1)).shape == (105, 1, 1) + + def test_transpose(self): + from numpypy import arange, array, transpose, ones + x = arange(4).reshape((2,2)) + assert (transpose(x) == array([[0, 2],[1, 3]])).all() + # Once axes argument is implemented, add more tests + raises(NotImplementedError, "transpose(x, axes=(1, 0, 2))") + # x = ones((1, 2, 3)) + # assert transpose(x, (1, 0, 2)).shape == (2, 1, 3) + diff --git a/pypy/module/test_lib_pypy/test_datetime.py b/pypy/module/test_lib_pypy/test_datetime.py new file mode 100644 --- /dev/null +++ b/pypy/module/test_lib_pypy/test_datetime.py @@ -0,0 +1,24 @@ +"""Additional tests for datetime.""" + +import time +import datetime +import os + +def test_utcfromtimestamp(): + """Confirm that utcfromtimestamp and fromtimestamp give consistent results. + + Based on danchr's test script in https://bugs.pypy.org/issue986 + """ + try: + prev_tz = os.environ.get("TZ") + os.environ["TZ"] = "GMT" + for unused in xrange(100): + now = time.time() + delta = (datetime.datetime.utcfromtimestamp(now) - + datetime.datetime.fromtimestamp(now)) + assert delta.days * 86400 + delta.seconds == 0 + finally: + if prev_tz is None: + del os.environ["TZ"] + else: + os.environ["TZ"] = prev_tz diff --git a/pypy/module/test_lib_pypy/test_sqlite3.py b/pypy/module/test_lib_pypy/test_sqlite3.py new file mode 100644 --- /dev/null +++ b/pypy/module/test_lib_pypy/test_sqlite3.py @@ -0,0 +1,10 @@ +"""Tests for _sqlite3.py""" + +def test_list_ddl(): + """From issue996. Mostly just looking for lack of exceptions.""" + from sqlite3.dbapi2 import connect + connection = connect(':memory:') + cursor = connection.cursor() + cursor.execute('CREATE TABLE foo (bar INTEGER)') + result = list(cursor) + assert result == [] diff --git a/pypy/module/thread/ll_thread.py b/pypy/module/thread/ll_thread.py --- a/pypy/module/thread/ll_thread.py +++ b/pypy/module/thread/ll_thread.py @@ -4,7 +4,7 @@ import py from pypy.rlib import jit, rgc from pypy.rlib.debug import ll_assert -from pypy.rlib.objectmodel import we_are_translated +from pypy.rlib.objectmodel import we_are_translated, specialize from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.tool import rffi_platform from pypy.tool import autopath @@ -85,6 +85,7 @@ def allocate_lock(): return Lock(allocate_ll_lock()) + at specialize.arg(0) def ll_start_new_thread(func): ident = c_thread_start(func) if ident == -1: @@ -97,6 +98,7 @@ def get_ident(): return rffi.cast(lltype.Signed, c_thread_get_ident()) + at specialize.arg(0) def start_new_thread(x, y): """In RPython, no argument can be passed. You have to use global variables to pass information to the new thread. That's not very diff --git a/pypy/module/thread/test/test_ll_thread.py b/pypy/module/thread/test/test_ll_thread.py --- a/pypy/module/thread/test/test_ll_thread.py +++ b/pypy/module/thread/test/test_ll_thread.py @@ -34,72 +34,34 @@ use_threads = True def test_start_new_thread(self): - py.test.skip("xxx ideally, investigate why it fails randomly") - # xxx but in practice start_new_thread() is also tested by the - # next test, and it's a mess to test start_new_thread() without - # the proper GIL to protect the GC import time class State: pass state = State() - class Z: - def __init__(self, value): - self.value = value - def __del__(self): - state.freed_counter += 1 - - def bootstrap(): - state.my_thread_ident = state.z.ident = get_ident() - assert state.my_thread_ident == get_ident() - assert get_ident() == state.z.ident - state.seen_value = state.z.value - state.z = None - # I think that we would need here a memory barrier in order - # to make the test pass reliably. The issue is that the - # main thread may see 'state.done = 1' before seeing the - # effect of the other assignments done above. For now let's - # emulate the write barrier by doing a system call and - # waiting a bit... - time.sleep(0.012) - state.done = 1 - - def g(i): - state.z = Z(i) - return start_new_thread(bootstrap, ()) - g._dont_inline_ = True + def bootstrap1(): + state.my_thread_ident1 = get_ident() + def bootstrap2(): + state.my_thread_ident2 = get_ident() def f(): - main_ident = get_ident() - assert main_ident == get_ident() - state.freed_counter = 0 - for i in range(50): - state.done = 0 - state.seen_value = 0 - ident = g(i) - gc.collect() - willing_to_wait_more = 1000 - while not state.done: - willing_to_wait_more -= 1 - if not willing_to_wait_more: - raise Exception("thread didn't start?") - time.sleep(0.01) - assert state.my_thread_ident != main_ident - assert state.my_thread_ident == ident - assert state.seen_value == i - # try to force Boehm to do some freeing - for i in range(3): - gc.collect() - return state.freed_counter + state.my_thread_ident1 = get_ident() + state.my_thread_ident2 = get_ident() + start_new_thread(bootstrap1, ()) + start_new_thread(bootstrap2, ()) + willing_to_wait_more = 1000 + while (state.my_thread_ident1 == get_ident() or + state.my_thread_ident2 == get_ident()): + willing_to_wait_more -= 1 + if not willing_to_wait_more: + raise Exception("thread didn't start?") + time.sleep(0.01) + return 42 fn = self.getcompiled(f, []) - freed_counter = fn() - print freed_counter - if self.gcpolicy == 'boehm': - assert freed_counter > 0 - else: - assert freed_counter == 50 + res = fn() + assert res == 42 def test_gc_locking(self): import time @@ -202,6 +164,9 @@ def getcompiled(self, f, argtypes): return f + def test_start_new_thread(self): + py.test.skip("deadlocks occasionally -- why???") + class TestUsingBoehm(AbstractThreadTests): gcpolicy = 'boehm' diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py --- a/pypy/objspace/std/bytearrayobject.py +++ b/pypy/objspace/std/bytearrayobject.py @@ -624,8 +624,8 @@ if step == 1: assert start >= 0 - assert slicelength >= 0 - del items[start:start+slicelength] + if slicelength > 0: + del items[start:start+slicelength] else: n = len(items) i = start @@ -662,10 +662,11 @@ while i >= lim: items[i] = items[i-delta] i -= 1 - elif start >= 0: + elif delta == 0: + pass + else: + assert start >= 0 # start<0 is only possible with slicelength==0 del items[start:start+delta] - else: - assert delta==0 # start<0 is only possible with slicelength==0 elif len2 != slicelength: # No resize for extended slices raise operationerrfmt(space.w_ValueError, "attempt to " "assign sequence of size %d to extended slice of size %d", diff --git a/pypy/objspace/std/complexobject.py b/pypy/objspace/std/complexobject.py --- a/pypy/objspace/std/complexobject.py +++ b/pypy/objspace/std/complexobject.py @@ -8,6 +8,7 @@ from pypy.rlib.rbigint import rbigint from pypy.rlib.rfloat import ( formatd, DTSF_STR_PRECISION, isinf, isnan, copysign) +from pypy.rlib import jit import math @@ -129,10 +130,10 @@ ir = len * math.sin(phase) return W_ComplexObject(rr, ir) - def pow_int(self, n): - if n > 100 or n < -100: - return self.pow(W_ComplexObject(1.0 * n, 0.0)) - elif n > 0: + def pow_small_int(self, n): + if n >= 0: + if jit.isconstant(n) and n == 2: + return self.mul(self) return self.pow_positive_int(n) else: return w_one.div(self.pow_positive_int(-n)) @@ -217,10 +218,10 @@ def pow__Complex_Complex_ANY(space, w_complex, w_exponent, thirdArg): if not space.is_w(thirdArg, space.w_None): raise OperationError(space.w_ValueError, space.wrap('complex modulo')) - int_exponent = int(w_exponent.realval) try: - if w_exponent.imagval == 0.0 and w_exponent.realval == int_exponent: - w_p = w_complex.pow_int(int_exponent) + r = w_exponent.realval + if w_exponent.imagval == 0.0 and -100.0 <= r <= 100.0 and r == int(r): + w_p = w_complex.pow_small_int(int(r)) else: w_p = w_complex.pow(w_exponent) except ZeroDivisionError: diff --git a/pypy/objspace/std/floatobject.py b/pypy/objspace/std/floatobject.py --- a/pypy/objspace/std/floatobject.py +++ b/pypy/objspace/std/floatobject.py @@ -443,6 +443,8 @@ y = w_float2.floatval # Sort out special cases here instead of relying on pow() + if y == 2.0: # special case for performance: + return W_FloatObject(x * x) # x * x is always correct if y == 0.0: # x**0 is 1, even 0**0 return W_FloatObject(1.0) diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py --- a/pypy/objspace/std/listobject.py +++ b/pypy/objspace/std/listobject.py @@ -804,10 +804,11 @@ while i >= lim: items[i] = items[i-delta] i -= 1 - elif start >= 0: + elif delta == 0: + pass + else: + assert start >= 0 # start<0 is only possible with slicelength==0 del items[start:start+delta] - else: - assert delta==0 # start<0 is only possible with slicelength==0 elif len2 != slicelength: # No resize for extended slices raise operationerrfmt(self.space.w_ValueError, "attempt to " "assign sequence of size %d to extended slice of size %d", @@ -851,8 +852,8 @@ if step == 1: assert start >= 0 - assert slicelength >= 0 - del items[start:start+slicelength] + if slicelength > 0: + del items[start:start+slicelength] else: n = len(items) i = start diff --git a/pypy/objspace/std/test/test_bytearrayobject.py b/pypy/objspace/std/test/test_bytearrayobject.py --- a/pypy/objspace/std/test/test_bytearrayobject.py +++ b/pypy/objspace/std/test/test_bytearrayobject.py @@ -1,5 +1,9 @@ +from pypy import conftest class AppTestBytesArray: + def setup_class(cls): + cls.w_runappdirect = cls.space.wrap(conftest.option.runappdirect) + def test_basics(self): b = bytearray() assert type(b) is bytearray @@ -439,3 +443,15 @@ def test_reduce(self): assert bytearray('caf\xe9').__reduce__() == ( bytearray, (u'caf\xe9', 'latin-1'), None) + + def test_setitem_slice_performance(self): + # because of a complexity bug, this used to take forever on a + # translated pypy. On CPython2.6 -A, it takes around 8 seconds. + if self.runappdirect: + count = 16*1024*1024 + else: + count = 1024 + b = bytearray(count) + for i in range(count): + b[i:i+1] = 'y' + assert str(b) == 'y' * count diff --git a/pypy/objspace/std/test/test_complexobject.py b/pypy/objspace/std/test/test_complexobject.py --- a/pypy/objspace/std/test/test_complexobject.py +++ b/pypy/objspace/std/test/test_complexobject.py @@ -71,7 +71,7 @@ assert _powu((0.0,1.0),2) == (-1.0,0.0) def _powi((r1, i1), n): - w_res = W_ComplexObject(r1, i1).pow_int(n) + w_res = W_ComplexObject(r1, i1).pow_small_int(n) return w_res.realval, w_res.imagval assert _powi((0.0,2.0),0) == (1.0,0.0) assert _powi((0.0,0.0),2) == (0.0,0.0) @@ -213,6 +213,7 @@ assert a ** 105 == a ** 105 assert a ** -105 == a ** -105 assert a ** -30 == a ** -30 + assert a ** 2 == a * a assert 0.0j ** 0 == 1 diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py --- a/pypy/objspace/std/test/test_listobject.py +++ b/pypy/objspace/std/test/test_listobject.py @@ -397,6 +397,7 @@ on_cpython = (option.runappdirect and not hasattr(sys, 'pypy_translation_info')) cls.w_on_cpython = cls.space.wrap(on_cpython) + cls.w_runappdirect = cls.space.wrap(option.runappdirect) def test_getstrategyfromlist_w(self): l0 = ["a", "2", "a", True] @@ -898,6 +899,18 @@ l[::-1] = l assert l == [6,5,4,3,2,1] + def test_setitem_slice_performance(self): + # because of a complexity bug, this used to take forever on a + # translated pypy. On CPython2.6 -A, it takes around 5 seconds. + if self.runappdirect: + count = 16*1024*1024 + else: + count = 1024 + b = [None] * count + for i in range(count): + b[i:i+1] = ['y'] + assert b == ['y'] * count + def test_recursive_repr(self): l = [] assert repr(l) == '[]' diff --git a/pypy/objspace/std/test/test_unicodeobject.py b/pypy/objspace/std/test/test_unicodeobject.py --- a/pypy/objspace/std/test/test_unicodeobject.py +++ b/pypy/objspace/std/test/test_unicodeobject.py @@ -64,6 +64,12 @@ check(', '.join([u'a']), u'a') check(', '.join(['a', u'b']), u'a, b') check(u', '.join(['a', 'b']), u'a, b') + try: + u''.join([u'a', 2, 3]) + except TypeError, e: + assert 'sequence item 1' in str(e) + else: + raise Exception("DID NOT RAISE") if sys.version_info >= (2,3): def test_contains_ex(self): diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -201,7 +201,7 @@ return space.newbool(container.find(item) != -1) def unicode_join__Unicode_ANY(space, w_self, w_list): - list_w = space.unpackiterable(w_list) + list_w = space.listview(w_list) size = len(list_w) if size == 0: @@ -216,22 +216,21 @@ def _unicode_join_many_items(space, w_self, list_w, size): self = w_self._value - sb = UnicodeBuilder() + prealloc_size = len(self) * (size - 1) + for i in range(size): + try: + prealloc_size += len(space.unicode_w(list_w[i])) + except OperationError, e: + if not e.match(space, space.w_TypeError): + raise + raise operationerrfmt(space.w_TypeError, + "sequence item %d: expected string or Unicode", i) + sb = UnicodeBuilder(prealloc_size) for i in range(size): if self and i != 0: sb.append(self) w_s = list_w[i] - if isinstance(w_s, W_UnicodeObject): - # shortcut for performance - sb.append(w_s._value) - else: - try: - sb.append(space.unicode_w(w_s)) - except OperationError, e: - if not e.match(space, space.w_TypeError): - raise - raise operationerrfmt(space.w_TypeError, - "sequence item %d: expected string or Unicode", i) + sb.append(space.unicode_w(w_s)) return space.wrap(sb.build()) def hash__Unicode(space, w_uni): diff --git a/pypy/rlib/jit_hooks.py b/pypy/rlib/jit_hooks.py --- a/pypy/rlib/jit_hooks.py +++ b/pypy/rlib/jit_hooks.py @@ -52,7 +52,10 @@ from pypy.jit.metainterp.history import ResOperation args = [_cast_to_box(llargs[i]) for i in range(len(llargs))] - res = _cast_to_box(llres) + if llres: + res = _cast_to_box(llres) + else: + res = None return _cast_to_gcref(ResOperation(no, args, res)) @register_helper(annmodel.SomePtr(llmemory.GCREF)) diff --git a/pypy/rlib/longlong2float.py b/pypy/rlib/longlong2float.py --- a/pypy/rlib/longlong2float.py +++ b/pypy/rlib/longlong2float.py @@ -79,19 +79,23 @@ longlong2float = rffi.llexternal( "pypy__longlong2float", [rffi.LONGLONG], rffi.DOUBLE, _callable=longlong2float_emulator, compilation_info=eci, - _nowrapper=True, elidable_function=True, sandboxsafe=True) + _nowrapper=True, elidable_function=True, sandboxsafe=True, + oo_primitive="pypy__longlong2float") float2longlong = rffi.llexternal( "pypy__float2longlong", [rffi.DOUBLE], rffi.LONGLONG, _callable=float2longlong_emulator, compilation_info=eci, - _nowrapper=True, elidable_function=True, sandboxsafe=True) + _nowrapper=True, elidable_function=True, sandboxsafe=True, + oo_primitive="pypy__float2longlong") uint2singlefloat = rffi.llexternal( "pypy__uint2singlefloat", [rffi.UINT], rffi.FLOAT, _callable=uint2singlefloat_emulator, compilation_info=eci, - _nowrapper=True, elidable_function=True, sandboxsafe=True) + _nowrapper=True, elidable_function=True, sandboxsafe=True, + oo_primitive="pypy__uint2singlefloat") singlefloat2uint = rffi.llexternal( "pypy__singlefloat2uint", [rffi.FLOAT], rffi.UINT, _callable=singlefloat2uint_emulator, compilation_info=eci, - _nowrapper=True, elidable_function=True, sandboxsafe=True) + _nowrapper=True, elidable_function=True, sandboxsafe=True, + oo_primitive="pypy__singlefloat2uint") diff --git a/pypy/rlib/objectmodel.py b/pypy/rlib/objectmodel.py --- a/pypy/rlib/objectmodel.py +++ b/pypy/rlib/objectmodel.py @@ -420,7 +420,7 @@ vobj.concretetype.TO._gckind == 'gc') else: from pypy.rpython.ootypesystem import ootype - ok = isinstance(vobj.concretetype, ootype.Instance) + ok = isinstance(vobj.concretetype, (ootype.Instance, ootype.BuiltinType)) if not ok: from pypy.rpython.error import TyperError raise TyperError("compute_unique_id() cannot be applied to" diff --git a/pypy/rlib/rdynload.py b/pypy/rlib/rdynload.py --- a/pypy/rlib/rdynload.py +++ b/pypy/rlib/rdynload.py @@ -14,7 +14,7 @@ _MINGW = platform.name == "mingw32" _WIN32 = _MSVC or _MINGW _MAC_OS = platform.name == "darwin" -_FREEBSD = platform.name == "freebsd" +_FREEBSD = sys.platform.startswith("freebsd") if _WIN32: from pypy.rlib import rwin32 diff --git a/pypy/rlib/rsocket.py b/pypy/rlib/rsocket.py --- a/pypy/rlib/rsocket.py +++ b/pypy/rlib/rsocket.py @@ -310,10 +310,7 @@ def from_object(space, w_address): # Parse an app-level object representing an AF_INET address - try: - w_host, w_port = space.unpackiterable(w_address, 2) - except ValueError: - raise TypeError("AF_INET address must be a tuple of length 2") + w_host, w_port = space.unpackiterable(w_address, 2) host = space.str_w(w_host) port = space.int_w(w_port) port = Address.make_ushort_port(space, port) @@ -544,10 +541,7 @@ space.wrap(self.get_groups())]) def from_object(space, w_address): - try: - w_pid, w_groups = space.unpackiterable(w_address, 2) - except ValueError: - raise TypeError("AF_NETLINK address must be a tuple of length 2") + w_pid, w_groups = space.unpackiterable(w_address, 2) return NETLINKAddress(space.uint_w(w_pid), space.uint_w(w_groups)) from_object = staticmethod(from_object) diff --git a/pypy/rlib/rweakref.py b/pypy/rlib/rweakref.py --- a/pypy/rlib/rweakref.py +++ b/pypy/rlib/rweakref.py @@ -1,11 +1,14 @@ """ -Weakref support in RPython. Supports ref() without callbacks, +Weakref support in RPython. Basic regular weakrefs without callbacks +are supported. This file contains the following additions: a form of WeakKeyDictionary, and a limited version of WeakValueDictionary. LLType only for now! """ import weakref +ref = weakref.ref # basic regular weakrefs are supported in RPython + class RWeakValueDictionary(object): """A dictionary containing weak values.""" diff --git a/pypy/rpython/lltypesystem/ll2ctypes.py b/pypy/rpython/lltypesystem/ll2ctypes.py --- a/pypy/rpython/lltypesystem/ll2ctypes.py +++ b/pypy/rpython/lltypesystem/ll2ctypes.py @@ -1390,7 +1390,8 @@ def _where_is_errno(): return standard_c_lib.__errno_location() - elif sys.platform in ('darwin', 'freebsd7', 'freebsd8', 'freebsd9'): + elif any(plat in sys.platform + for plat in ('darwin', 'freebsd7', 'freebsd8', 'freebsd9')): standard_c_lib.__error.restype = ctypes.POINTER(ctypes.c_int) def _where_is_errno(): return standard_c_lib.__error() diff --git a/pypy/rpython/lltypesystem/rtuple.py b/pypy/rpython/lltypesystem/rtuple.py --- a/pypy/rpython/lltypesystem/rtuple.py +++ b/pypy/rpython/lltypesystem/rtuple.py @@ -27,6 +27,10 @@ def newtuple(cls, llops, r_tuple, items_v): # items_v should have the lowleveltype of the internal reprs + assert len(r_tuple.items_r) == len(items_v) + for r_item, v_item in zip(r_tuple.items_r, items_v): + assert r_item.lowleveltype == v_item.concretetype + # if len(r_tuple.items_r) == 0: return inputconst(Void, ()) # a Void empty tuple c1 = inputconst(Void, r_tuple.lowleveltype.TO) diff --git a/pypy/rpython/memory/gc/env.py b/pypy/rpython/memory/gc/env.py --- a/pypy/rpython/memory/gc/env.py +++ b/pypy/rpython/memory/gc/env.py @@ -116,7 +116,7 @@ def get_total_memory(): return get_total_memory_darwin(get_darwin_sysctl_signed('hw.memsize')) -elif sys.platform.startswith('freebsd'): +elif 'freebsd' in sys.platform: def get_total_memory(): return get_total_memory_darwin(get_darwin_sysctl_signed('hw.usermem')) diff --git a/pypy/rpython/module/ll_time.py b/pypy/rpython/module/ll_time.py --- a/pypy/rpython/module/ll_time.py +++ b/pypy/rpython/module/ll_time.py @@ -41,7 +41,7 @@ RUSAGE = platform.Struct('struct rusage', [('ru_utime', TIMEVAL), ('ru_stime', TIMEVAL)]) -if "freebsd" in sys.platform: +if sys.platform.startswith('freebsd'): libraries = ['compat'] else: libraries = [] diff --git a/pypy/rpython/ootypesystem/ootype.py b/pypy/rpython/ootypesystem/ootype.py --- a/pypy/rpython/ootypesystem/ootype.py +++ b/pypy/rpython/ootypesystem/ootype.py @@ -512,6 +512,7 @@ "ll_append_char": Meth([CHARTP], Void), "ll_append": Meth([STRINGTP], Void), "ll_build": Meth([], STRINGTP), + "ll_getlength": Meth([], Signed), }) self._setup_methods({}) @@ -1376,6 +1377,9 @@ def _cast_to_object(self): return make_object(self) + def _identityhash(self): + return object.__hash__(self) + class _string(_builtin_type): def __init__(self, STRING, value = ''): @@ -1543,6 +1547,9 @@ else: return make_unicode(u''.join(self._buf)) + def ll_getlength(self): + return self.ll_build().ll_strlen() + class _null_string_builder(_null_mixin(_string_builder), _string_builder): def __init__(self, STRING_BUILDER): self.__dict__["_TYPE"] = STRING_BUILDER diff --git a/pypy/rpython/ootypesystem/rbuilder.py b/pypy/rpython/ootypesystem/rbuilder.py --- a/pypy/rpython/ootypesystem/rbuilder.py +++ b/pypy/rpython/ootypesystem/rbuilder.py @@ -21,6 +21,10 @@ builder.ll_append_char(char) @staticmethod + def ll_getlength(builder): + return builder.ll_getlength() + + @staticmethod def ll_append(builder, string): builder.ll_append(string) diff --git a/pypy/rpython/rrange.py b/pypy/rpython/rrange.py --- a/pypy/rpython/rrange.py +++ b/pypy/rpython/rrange.py @@ -204,7 +204,10 @@ v_index = hop.gendirectcall(self.ll_getnextindex, v_enumerate) hop2 = hop.copy() hop2.args_r = [self.r_baseiter] + r_item_src = self.r_baseiter.r_list.external_item_repr + r_item_dst = hop.r_result.items_r[1] v_item = self.r_baseiter.rtype_next(hop2) + v_item = hop.llops.convertvar(v_item, r_item_src, r_item_dst) return hop.r_result.newtuple(hop.llops, hop.r_result, [v_index, v_item]) diff --git a/pypy/rpython/test/test_rbuilder.py b/pypy/rpython/test/test_rbuilder.py --- a/pypy/rpython/test/test_rbuilder.py +++ b/pypy/rpython/test/test_rbuilder.py @@ -124,9 +124,5 @@ pass class TestOOtype(BaseTestStringBuilder, OORtypeMixin): - def test_string_getlength(self): - py.test.skip("getlength(): not implemented on ootype") - def test_unicode_getlength(self): - py.test.skip("getlength(): not implemented on ootype") def test_append_charpsize(self): py.test.skip("append_charpsize(): not implemented on ootype") diff --git a/pypy/rpython/test/test_rbuiltin.py b/pypy/rpython/test/test_rbuiltin.py --- a/pypy/rpython/test/test_rbuiltin.py +++ b/pypy/rpython/test/test_rbuiltin.py @@ -463,6 +463,31 @@ assert x1 == intmask(x0) assert x3 == intmask(x2) + def test_id_on_builtins(self): + from pypy.rlib.objectmodel import compute_unique_id + from pypy.rlib.rstring import StringBuilder, UnicodeBuilder + def fn(): + return (compute_unique_id("foo"), + compute_unique_id(u"bar"), + compute_unique_id([1]), + compute_unique_id({"foo": 3}), + compute_unique_id(StringBuilder()), + compute_unique_id(UnicodeBuilder())) + res = self.interpret(fn, []) + for id in self.ll_unpack_tuple(res, 6): + assert isinstance(id, (int, r_longlong)) + + def test_uniqueness_of_id_on_strings(self): + from pypy.rlib.objectmodel import compute_unique_id + def fn(s1, s2): + return (compute_unique_id(s1), compute_unique_id(s2)) + + s1 = "foo" + s2 = ''.join(['f','oo']) + res = self.interpret(fn, [self.string_to_ll(s1), self.string_to_ll(s2)]) + i1, i2 = self.ll_unpack_tuple(res, 2) + assert i1 != i2 + def test_cast_primitive(self): from pypy.rpython.annlowlevel import LowLevelAnnotatorPolicy def llf(u): diff --git a/pypy/rpython/test/test_rrange.py b/pypy/rpython/test/test_rrange.py --- a/pypy/rpython/test/test_rrange.py +++ b/pypy/rpython/test/test_rrange.py @@ -169,6 +169,22 @@ res = self.interpret(fn, [2]) assert res == 789 + def test_enumerate_instances(self): + class A: + pass + def fn(n): + a = A() + b = A() + a.k = 10 + b.k = 20 + for i, x in enumerate([a, b]): + if i == n: + return x.k + return 5 + res = self.interpret(fn, [1]) + assert res == 20 + + class TestLLtype(BaseTestRrange, LLRtypeMixin): from pypy.rpython.lltypesystem import rrange diff --git a/pypy/tool/jitlogparser/parser.py b/pypy/tool/jitlogparser/parser.py --- a/pypy/tool/jitlogparser/parser.py +++ b/pypy/tool/jitlogparser/parser.py @@ -140,13 +140,15 @@ bytecode_name = None is_bytecode = True inline_level = None + has_dmp = False def parse_code_data(self, arg): m = re.search('<code object ([<>\w]+)[\.,] file \'(.+?)\'[\.,] line (\d+)> #(\d+) (\w+)', arg) if m is None: # a non-code loop, like StrLiteralSearch or something - self.bytecode_name = arg + if arg: + self.bytecode_name = arg else: self.name, self.filename, lineno, bytecode_no, self.bytecode_name = m.groups() self.startlineno = int(lineno) @@ -218,7 +220,7 @@ self.inputargs = inputargs self.chunks = chunks for chunk in self.chunks: - if chunk.filename is not None: + if chunk.bytecode_name is not None: self.startlineno = chunk.startlineno self.filename = chunk.filename self.name = chunk.name diff --git a/pypy/tool/jitlogparser/test/test_parser.py b/pypy/tool/jitlogparser/test/test_parser.py --- a/pypy/tool/jitlogparser/test/test_parser.py +++ b/pypy/tool/jitlogparser/test/test_parser.py @@ -283,3 +283,13 @@ assert loops[-1].count == 1234 assert loops[1].count == 123 assert loops[2].count == 12 + +def test_parse_nonpython(): + loop = parse(""" + [] + debug_merge_point(0, 'random') + debug_merge_point(0, '<code object f. file 'x.py'. line 2> #15 COMPARE_OP') + """) + f = Function.from_operations(loop.operations, LoopStorage()) + assert f.chunks[-1].filename == 'x.py' + assert f.filename is None diff --git a/pypy/tool/test/test_version.py b/pypy/tool/test/test_version.py --- a/pypy/tool/test/test_version.py +++ b/pypy/tool/test/test_version.py @@ -1,6 +1,22 @@ import os, sys import py -from pypy.tool.version import get_repo_version_info +from pypy.tool.version import get_repo_version_info, _get_hg_archive_version + +def test_hg_archival_version(tmpdir): + def version_for(name, **kw): + path = tmpdir.join(name) + path.write('\n'.join('%s: %s' % x for x in kw.items())) + return _get_hg_archive_version(str(path)) + + assert version_for('release', + tag='release-123', + node='000', + ) == ('PyPy', 'release-123', '000') + assert version_for('somebranch', + node='000', + branch='something', + ) == ('PyPy', 'something', '000') + def test_get_repo_version_info(): assert get_repo_version_info(None) diff --git a/pypy/tool/version.py b/pypy/tool/version.py --- a/pypy/tool/version.py +++ b/pypy/tool/version.py @@ -3,111 +3,139 @@ from subprocess import Popen, PIPE import pypy pypydir = os.path.dirname(os.path.abspath(pypy.__file__)) +pypyroot = os.path.dirname(pypydir) +default_retval = 'PyPy', '?', '?' + +def maywarn(err, repo_type='Mercurial'): + if not err: + return + + from pypy.tool.ansi_print import ansi_log + log = py.log.Producer("version") + py.log.setconsumer("version", ansi_log) + log.WARNING('Errors getting %s information: %s' % (repo_type, err)) def get_repo_version_info(hgexe=None): '''Obtain version information by invoking the 'hg' or 'git' commands.''' - # TODO: support extracting from .hg_archival.txt - - default_retval = 'PyPy', '?', '?' - pypyroot = os.path.abspath(os.path.join(pypydir, '..')) - - def maywarn(err, repo_type='Mercurial'): - if not err: - return - - from pypy.tool.ansi_print import ansi_log - log = py.log.Producer("version") - py.log.setconsumer("version", ansi_log) - log.WARNING('Errors getting %s information: %s' % (repo_type, err)) # Try to see if we can get info from Git if hgexe is not specified. if not hgexe: if os.path.isdir(os.path.join(pypyroot, '.git')): - gitexe = py.path.local.sysfind('git') - if gitexe: - try: - p = Popen( - [str(gitexe), 'rev-parse', 'HEAD'], - stdout=PIPE, stderr=PIPE, cwd=pypyroot - ) - except OSError, e: - maywarn(e, 'Git') - return default_retval - if p.wait() != 0: - maywarn(p.stderr.read(), 'Git') - return default_retval - revision_id = p.stdout.read().strip()[:12] - p = Popen( - [str(gitexe), 'describe', '--tags', '--exact-match'], - stdout=PIPE, stderr=PIPE, cwd=pypyroot - ) - if p.wait() != 0: - p = Popen( - [str(gitexe), 'branch'], stdout=PIPE, stderr=PIPE, - cwd=pypyroot - ) - if p.wait() != 0: - maywarn(p.stderr.read(), 'Git') - return 'PyPy', '?', revision_id - branch = '?' - for line in p.stdout.read().strip().split('\n'): - if line.startswith('* '): - branch = line[1:].strip() - if branch == '(no branch)': - branch = '?' - break - return 'PyPy', branch, revision_id - return 'PyPy', p.stdout.read().strip(), revision_id + return _get_git_version() # Fallback to trying Mercurial. if hgexe is None: hgexe = py.path.local.sysfind('hg') - if not os.path.isdir(os.path.join(pypyroot, '.hg')): + if os.path.isfile(os.path.join(pypyroot, '.hg_archival.txt')): + return _get_hg_archive_version(os.path.join(pypyroot, '.hg_archival.txt')) + elif not os.path.isdir(os.path.join(pypyroot, '.hg')): maywarn('Not running from a Mercurial repository!') return default_retval elif not hgexe: maywarn('Cannot find Mercurial command!') return default_retval else: - env = dict(os.environ) - # get Mercurial into scripting mode - env['HGPLAIN'] = '1' - # disable user configuration, extensions, etc. - env['HGRCPATH'] = os.devnull + return _get_hg_version(hgexe) - try: - p = Popen([str(hgexe), 'version', '-q'], - stdout=PIPE, stderr=PIPE, env=env) - except OSError, e: - maywarn(e) - return default_retval - if not p.stdout.read().startswith('Mercurial Distributed SCM'): - maywarn('command does not identify itself as Mercurial') - return default_retval +def _get_hg_version(hgexe): + env = dict(os.environ) + # get Mercurial into scripting mode + env['HGPLAIN'] = '1' + # disable user configuration, extensions, etc. + env['HGRCPATH'] = os.devnull - p = Popen([str(hgexe), 'id', '-i', pypyroot], + try: + p = Popen([str(hgexe), 'version', '-q'], stdout=PIPE, stderr=PIPE, env=env) - hgid = p.stdout.read().strip() + except OSError, e: + maywarn(e) + return default_retval + + if not p.stdout.read().startswith('Mercurial Distributed SCM'): + maywarn('command does not identify itself as Mercurial') + return default_retval + + p = Popen([str(hgexe), 'id', '-i', pypyroot], + stdout=PIPE, stderr=PIPE, env=env) + hgid = p.stdout.read().strip() + maywarn(p.stderr.read()) + if p.wait() != 0: + hgid = '?' + + p = Popen([str(hgexe), 'id', '-t', pypyroot], + stdout=PIPE, stderr=PIPE, env=env) + hgtags = [t for t in p.stdout.read().strip().split() if t != 'tip'] + maywarn(p.stderr.read()) + if p.wait() != 0: + hgtags = ['?'] + + if hgtags: + return 'PyPy', hgtags[0], hgid + else: + # use the branch instead + p = Popen([str(hgexe), 'id', '-b', pypyroot], + stdout=PIPE, stderr=PIPE, env=env) + hgbranch = p.stdout.read().strip() maywarn(p.stderr.read()) + + return 'PyPy', hgbranch, hgid + + +def _get_hg_archive_version(path): + fp = open(path) + try: + data = dict(x.split(': ', 1) for x in fp.read().splitlines()) + finally: + fp.close() + if 'tag' in data: + return 'PyPy', data['tag'], data['node'] + else: + return 'PyPy', data['branch'], data['node'] + + +def _get_git_version(): + #XXX: this function is a untested hack, + # so the git mirror tav made will work + gitexe = py.path.local.sysfind('git') + if not gitexe: + return default_retval + + try: + p = Popen( + [str(gitexe), 'rev-parse', 'HEAD'], + stdout=PIPE, stderr=PIPE, cwd=pypyroot + ) + except OSError, e: + maywarn(e, 'Git') + return default_retval + if p.wait() != 0: + maywarn(p.stderr.read(), 'Git') + return default_retval + revision_id = p.stdout.read().strip()[:12] + p = Popen( + [str(gitexe), 'describe', '--tags', '--exact-match'], + stdout=PIPE, stderr=PIPE, cwd=pypyroot + ) + if p.wait() != 0: + p = Popen( + [str(gitexe), 'branch'], stdout=PIPE, stderr=PIPE, + cwd=pypyroot + ) if p.wait() != 0: - hgid = '?' + maywarn(p.stderr.read(), 'Git') + return 'PyPy', '?', revision_id + branch = '?' + for line in p.stdout.read().strip().split('\n'): + if line.startswith('* '): + branch = line[1:].strip() + if branch == '(no branch)': + branch = '?' + break + return 'PyPy', branch, revision_id + return 'PyPy', p.stdout.read().strip(), revision_id - p = Popen([str(hgexe), 'id', '-t', pypyroot], - stdout=PIPE, stderr=PIPE, env=env) - hgtags = [t for t in p.stdout.read().strip().split() if t != 'tip'] - maywarn(p.stderr.read()) - if p.wait() != 0: - hgtags = ['?'] - if hgtags: - return 'PyPy', hgtags[0], hgid - else: - # use the branch instead - p = Popen([str(hgexe), 'id', '-b', pypyroot], - stdout=PIPE, stderr=PIPE, env=env) - hgbranch = p.stdout.read().strip() - maywarn(p.stderr.read()) - - return 'PyPy', hgbranch, hgid +if __name__ == '__main__': + print get_repo_version_info() diff --git a/pypy/translator/c/src/profiling.c b/pypy/translator/c/src/profiling.c --- a/pypy/translator/c/src/profiling.c +++ b/pypy/translator/c/src/profiling.c @@ -29,33 +29,33 @@ profiling_setup = 0; } } - -#elif defined(_WIN32) -#include <windows.h> - -DWORD_PTR base_affinity_mask; -int profiling_setup = 0; - -void pypy_setup_profiling() { - if (!profiling_setup) { - DWORD_PTR affinity_mask, system_affinity_mask; - GetProcessAffinityMask(GetCurrentProcess(), - &base_affinity_mask, &system_affinity_mask); - affinity_mask = 1; - /* Pick one cpu allowed by the system */ - if (system_affinity_mask) - while ((affinity_mask & system_affinity_mask) == 0) - affinity_mask <<= 1; - SetProcessAffinityMask(GetCurrentProcess(), affinity_mask); - profiling_setup = 1; - } -} - -void pypy_teardown_profiling() { - if (profiling_setup) { - SetProcessAffinityMask(GetCurrentProcess(), base_affinity_mask); - profiling_setup = 0; - } + +#elif defined(_WIN32) +#include <windows.h> + +DWORD_PTR base_affinity_mask; +int profiling_setup = 0; + +void pypy_setup_profiling() { + if (!profiling_setup) { + DWORD_PTR affinity_mask, system_affinity_mask; + GetProcessAffinityMask(GetCurrentProcess(), + &base_affinity_mask, &system_affinity_mask); + affinity_mask = 1; + /* Pick one cpu allowed by the system */ + if (system_affinity_mask) + while ((affinity_mask & system_affinity_mask) == 0) + affinity_mask <<= 1; + SetProcessAffinityMask(GetCurrentProcess(), affinity_mask); + profiling_setup = 1; + } +} + +void pypy_teardown_profiling() { + if (profiling_setup) { + SetProcessAffinityMask(GetCurrentProcess(), base_affinity_mask); + profiling_setup = 0; + } } #else diff --git a/pypy/translator/goal/translate.py b/pypy/translator/goal/translate.py --- a/pypy/translator/goal/translate.py +++ b/pypy/translator/goal/translate.py @@ -293,17 +293,18 @@ drv.exe_name = targetspec_dic['__name__'] + '-%(backend)s' # Double check to ensure we are not overwriting the current interpreter - try: - this_exe = py.path.local(sys.executable).new(ext='') - exe_name = drv.compute_exe_name() - samefile = this_exe.samefile(exe_name) - assert not samefile, ( - 'Output file %s is the currently running ' - 'interpreter (use --output=...)'% exe_name) - except EnvironmentError: - pass + goals = translateconfig.goals + if not goals or 'compile' in goals: + try: + this_exe = py.path.local(sys.executable).new(ext='') + exe_name = drv.compute_exe_name() + samefile = this_exe.samefile(exe_name) + assert not samefile, ( + 'Output file %s is the currently running ' + 'interpreter (use --output=...)'% exe_name) + except EnvironmentError: + pass - goals = translateconfig.goals try: drv.proceed(goals) finally: diff --git a/pypy/translator/jvm/builtin.py b/pypy/translator/jvm/builtin.py --- a/pypy/translator/jvm/builtin.py +++ b/pypy/translator/jvm/builtin.py @@ -84,6 +84,9 @@ (ootype.StringBuilder.__class__, "ll_build"): jvm.Method.v(jStringBuilder, "toString", (), jString), + (ootype.StringBuilder.__class__, "ll_getlength"): + jvm.Method.v(jStringBuilder, "length", (), jInt), + (ootype.String.__class__, "ll_hash"): jvm.Method.v(jString, "hashCode", (), jInt), diff --git a/pypy/translator/jvm/database.py b/pypy/translator/jvm/database.py --- a/pypy/translator/jvm/database.py +++ b/pypy/translator/jvm/database.py @@ -358,7 +358,7 @@ ootype.Unsigned:jvm.PYPYSERIALIZEUINT, ootype.SignedLongLong:jvm.LONGTOSTRINGL, ootype.UnsignedLongLong: jvm.PYPYSERIALIZEULONG, - ootype.Float:jvm.DOUBLETOSTRINGD, + ootype.Float:jvm.PYPYSERIALIZEDOUBLE, ootype.Bool:jvm.PYPYSERIALIZEBOOLEAN, ootype.Void:jvm.PYPYSERIALIZEVOID, ootype.Char:jvm.PYPYESCAPEDCHAR, diff --git a/pypy/translator/jvm/metavm.py b/pypy/translator/jvm/metavm.py --- a/pypy/translator/jvm/metavm.py +++ b/pypy/translator/jvm/metavm.py @@ -92,6 +92,7 @@ CASTS = { # FROM TO (ootype.Signed, ootype.UnsignedLongLong): jvm.I2L, + (ootype.Unsigned, ootype.UnsignedLongLong): jvm.I2L, (ootype.SignedLongLong, ootype.Signed): jvm.L2I, (ootype.UnsignedLongLong, ootype.Unsigned): jvm.L2I, (ootype.UnsignedLongLong, ootype.Signed): jvm.L2I, diff --git a/pypy/translator/jvm/opcodes.py b/pypy/translator/jvm/opcodes.py --- a/pypy/translator/jvm/opcodes.py +++ b/pypy/translator/jvm/opcodes.py @@ -101,6 +101,7 @@ 'jit_force_virtualizable': Ignore, 'jit_force_virtual': DoNothing, 'jit_force_quasi_immutable': Ignore, + 'jit_is_virtual': PushPrimitive(ootype.Bool, False), 'debug_assert': [], # TODO: implement? 'debug_start_traceback': Ignore, diff --git a/pypy/translator/jvm/src/pypy/PyPy.java b/pypy/translator/jvm/src/pypy/PyPy.java --- a/pypy/translator/jvm/src/pypy/PyPy.java +++ b/pypy/translator/jvm/src/pypy/PyPy.java @@ -283,6 +283,14 @@ } } + public double pypy__longlong2float(long l) { + return Double.longBitsToDouble(l); + } + + public long pypy__float2longlong(double d) { + return Double.doubleToRawLongBits(d); + } + public double ooparse_float(String s) { try { return Double.parseDouble(s); @@ -353,6 +361,19 @@ return "False"; } + public static String serialize_double(double d) { + if (Double.isNaN(d)) { + return "float(\"nan\")"; + } else if (Double.isInfinite(d)) { + if (d > 0) + return "float(\"inf\")"; + else + return "float(\"-inf\")"; + } else { + return Double.toString(d); + } + } + private static String format_char(char c) { String res = "\\x"; if (c <= 0x0F) res = res + "0"; diff --git a/pypy/translator/jvm/test/runtest.py b/pypy/translator/jvm/test/runtest.py --- a/pypy/translator/jvm/test/runtest.py +++ b/pypy/translator/jvm/test/runtest.py @@ -56,6 +56,7 @@ # CLI could-be duplicate class JvmGeneratedSourceWrapper(object): + def __init__(self, gensrc): """ gensrc is an instance of JvmGeneratedSource """ self.gensrc = gensrc diff --git a/pypy/translator/jvm/test/test_builder.py b/pypy/translator/jvm/test/test_builder.py new file mode 100644 --- /dev/null +++ b/pypy/translator/jvm/test/test_builder.py @@ -0,0 +1,7 @@ +from pypy.translator.jvm.test.runtest import JvmTest +from pypy.rpython.test.test_rbuilder import BaseTestStringBuilder +import py + +class TestJvmStringBuilder(JvmTest, BaseTestStringBuilder): + def test_append_charpsize(self): + py.test.skip("append_charpsize(): not implemented on ootype") diff --git a/pypy/translator/jvm/test/test_longlong2float.py b/pypy/translator/jvm/test/test_longlong2float.py new file mode 100644 --- /dev/null +++ b/pypy/translator/jvm/test/test_longlong2float.py @@ -0,0 +1,20 @@ +from pypy.translator.jvm.test.runtest import JvmTest +from pypy.rlib.longlong2float import * +from pypy.rlib.test.test_longlong2float import enum_floats +from pypy.rlib.test.test_longlong2float import fn as float2longlong2float +import py + +class TestLongLong2Float(JvmTest): + + def test_float2longlong_and_longlong2float(self): + def func(f): + return float2longlong2float(f) + + for f in enum_floats(): + assert repr(f) == repr(self.interpret(func, [f])) + + def test_uint2singlefloat(self): + py.test.skip("uint2singlefloat is not implemented in ootype") + + def test_singlefloat2uint(self): + py.test.skip("singlefloat2uint is not implemented in ootype") diff --git a/pypy/translator/jvm/typesystem.py b/pypy/translator/jvm/typesystem.py --- a/pypy/translator/jvm/typesystem.py +++ b/pypy/translator/jvm/typesystem.py @@ -955,6 +955,7 @@ PYPYSERIALIZEUINT = Method.s(jPyPy, 'serialize_uint', (jInt,), jString) PYPYSERIALIZEULONG = Method.s(jPyPy, 'serialize_ulonglong', (jLong,),jString) PYPYSERIALIZEVOID = Method.s(jPyPy, 'serialize_void', (), jString) +PYPYSERIALIZEDOUBLE = Method.s(jPyPy, 'serialize_double', (jDouble,), jString) PYPYESCAPEDCHAR = Method.s(jPyPy, 'escaped_char', (jChar,), jString) PYPYESCAPEDUNICHAR = Method.s(jPyPy, 'escaped_unichar', (jChar,), jString) PYPYESCAPEDSTRING = Method.s(jPyPy, 'escaped_string', (jString,), jString) diff --git a/pypy/translator/oosupport/test_template/cast.py b/pypy/translator/oosupport/test_template/cast.py --- a/pypy/translator/oosupport/test_template/cast.py +++ b/pypy/translator/oosupport/test_template/cast.py @@ -13,6 +13,9 @@ def to_longlong(x): return r_longlong(x) +def to_ulonglong(x): + return r_ulonglong(x) + def uint_to_int(x): return intmask(x) @@ -56,6 +59,9 @@ def test_unsignedlonglong_to_unsigned4(self): self.check(to_uint, [r_ulonglong(18446744073709551615l)]) # max 64 bit num + def test_unsigned_to_usignedlonglong(self): + self.check(to_ulonglong, [r_uint(42)]) + def test_uint_to_int(self): self.check(uint_to_int, [r_uint(sys.maxint+1)]) diff --git a/pypy/translator/platform/freebsd.py b/pypy/translator/platform/freebsd.py --- a/pypy/translator/platform/freebsd.py +++ b/pypy/translator/platform/freebsd.py @@ -18,8 +18,9 @@ class Freebsd(posix.BasePosix): name = "freebsd" - link_flags = get_env_vector("LDFLAGS", '-pthread') - cflags = get_env_vector("CFLAGS", "-O3 -pthread -fomit-frame-pointer") + link_flags = ['-pthread'] + get_env_vector('LDFLAGS', '') + cflags = ['-O3', '-pthread', '-fomit-frame-pointer' + ] + get_env_vector('CFLAGS', '') standalone_only = [] shared_only = [] so_ext = 'so' diff --git a/pypy/translator/platform/linux.py b/pypy/translator/platform/linux.py --- a/pypy/translator/platform/linux.py +++ b/pypy/translator/platform/linux.py @@ -1,15 +1,20 @@ """Support for Linux.""" +import os import sys from pypy.translator.platform.posix import BasePosix class BaseLinux(BasePosix): name = "linux" - link_flags = ('-pthread',) + link_flags = tuple( + ['-pthread',] + + os.environ.get('LDFLAGS', '').split()) extra_libs = ('-lrt',) - cflags = ('-O3', '-pthread', '-fomit-frame-pointer', - '-Wall', '-Wno-unused') + cflags = tuple( + ['-O3', '-pthread', '-fomit-frame-pointer', + '-Wall', '-Wno-unused'] + + os.environ.get('CFLAGS', '').split()) standalone_only = () shared_only = ('-fPIC',) so_ext = 'so' From noreply at buildbot.pypy.org Tue Jan 24 23:12:24 2012 From: noreply at buildbot.pypy.org (mattip) Date: Tue, 24 Jan 2012 23:12:24 +0100 (CET) Subject: [pypy-commit] pypy numppy-flatitter: attempt at faster getitem, breaks setitem Message-ID: <20120124221224.0806982CFD@wyvern.cs.uni-duesseldorf.de> Author: mattip Branch: numppy-flatitter Changeset: r51747:f8f03c1f35e0 Date: 2012-01-25 00:11 +0200 http://bitbucket.org/pypy/pypy/changeset/f8f03c1f35e0/ Log: attempt at faster getitem, breaks setitem diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -1408,38 +1408,36 @@ #TODO:Move all this to another file after fijal finishes reorganization def _to_coords(space, arr, w_item_or_slice): - '''Always returns a list of coords. + '''Returns a start coord, step, and length. ''' - start = stop = step = 0 + start = lngth = step = 0 if space.isinstance_w(w_item_or_slice, space.w_int): start = space.int_w(w_item_or_slice) if start < 0: start += arr.size - stop = start+1 + lngth = 1 step = 1 elif space.isinstance_w(w_item_or_slice, space.w_slice): start, stop, step, lngth = space.decode_index4(w_item_or_slice,arr.size) else: - raise operationerrfmt(space.w_NotImplementedError,'cannot iterator over %s yet', w_item_or_slice) - retval = [] - for i in range(start, stop, step): - coords = [] - ii = i - if arr.order =='C': - for s in range(len(arr.shape) -1, -1, -1): - coords.insert(0,ii % arr.shape[s]) - ii /= arr.shape[s] - else: - raise NotImplementedError - #untested code. Erase? - for s in range(len(arr.shape)): - coords.append(ii % arr.shape[s]) - ii /= arr.shape[s] - if ii != 0: - raise OperationError(space.w_IndexError, - space.wrap("invalid index")) + raise OperationError(space.w_IndexError, + space.wrap('unsupported iterator index')) + coords = [] + i = start + if arr.order =='C': + for s in range(len(arr.shape) -1, -1, -1): + coords.insert(0, i % arr.shape[s]) + i /= arr.shape[s] + else: + raise NotImplementedError + #untested code. Erase? + for s in range(len(arr.shape)): + coords.append(i % arr.shape[s]) + i /= arr.shape[s] + if i != 0: + raise OperationError(space.w_IndexError, space.wrap("invalid index")) - retval.append(space.newtuple([space.wrap(c) for c in coords])) + retval = [coords, step, lngth] return retval class W_FlatIterator(ViewArray): @@ -1469,27 +1467,61 @@ return space.wrap(self.index) def descr_coords(self, space): - return _to_coords(space, self.base, space.wrap(self.index))[0] + coords = _to_coords(space, self.base, space.wrap(self.index))[0] + return space.newtuple([space.wrap(c) for c in coords]) def descr_getitem(self, space, w_idx): - coords = _to_coords(space, self.base, w_idx) - if len(coords)>1: - w_result = W_NDimArray(len(coords), [len(coords)], self.base.dtype, + coords,step,lngth = _to_coords(space, self.base, w_idx) + if lngth > 1: + res = W_NDimArray(lngth, [lngth], self.base.dtype, self.base.order) - for i,c in enumerate(coords): - w_val = self.base.descr_getitem(space, c) - w_result.setitem(i,w_val) - return w_result + # setslice would have been better, but flat[6:9] for arbitrary + # shapes of array a cannot be represented as a[x1:x2, y1:y2] + basei = ViewIterator(self.base.start, self.base.strides, + self.base.backstrides,self.base.shape) + basei.indices = coords + ri = ArrayIterator(lngth) + itemsize = self.base.dtype.itemtype.get_element_size() + shapelen = len(self.base.shape) + delta = 0 + ri_start = ri.offset + basei_start = basei.offset + bstorage = self.base.storage + while not ri.done(): + # TODO: add a jit_merge_point + + # This is an O(N) approach + # w_val = self.base.getitemspace, c) + # w_result.setitem(ri.offset,w_val) + # Try to do better by checking when we can use memcpy + # from basei_start to basei.offset + + while basei.offset - delta - basei_start == 0: + basei = basei.next(shapelen) + ri = ri.next(shapelen) + delta += 1 + rffi.c_memcpy( + rffi.ptradd(res.storage, ri_start* itemsize), + rffi.ptradd(bstorage, basei_start * itemsize), + delta * itemsize, + ) + delta = 0 + ri_start = ri.offset + basei_start = basei.offset + return res else: - return self.base.descr_getitem(space, coords[0]) + w_coords = space.newtuple([space.wrap(c) for c in coords]) + return self.base.descr_getitem(space, w_coords) def descr_setitem(self, space, w_idx, w_value): - coords = _to_coords(space, self.base, w_idx) + coords, step, lngth = _to_coords(space, self.base, w_idx) arr = convert_to_array(space, w_value) ai = 0 + return for c in coords: v = arr.getitem(ai) - self.base.descr_setitem(space, c,v) + self.base.descr_setitem(space, c, v) + # need to repeat input values until all assignments are done ai = (ai + 1) % arr.size W_FlatIterator.typedef = TypeDef( From noreply at buildbot.pypy.org Wed Jan 25 00:31:26 2012 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 25 Jan 2012 00:31:26 +0100 (CET) Subject: [pypy-commit] pypy default: seems to be necessary for running continuation stuff Message-ID: <20120124233126.6C5A982CFE@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: Changeset: r51749:e3fbdc682ed9 Date: 2012-01-25 01:30 +0200 http://bitbucket.org/pypy/pypy/changeset/e3fbdc682ed9/ Log: seems to be necessary for running continuation stuff diff --git a/pypy/bin/py.py b/pypy/bin/py.py --- a/pypy/bin/py.py +++ b/pypy/bin/py.py @@ -76,6 +76,8 @@ config.objspace.suggest(allworkingmodules=False) if config.objspace.allworkingmodules: pypyoption.enable_allworkingmodules(config) + if config.objspace.usemodules._continuation: + config.translation.continuation = True if config.objspace.usemodules.thread: config.translation.thread = True From noreply at buildbot.pypy.org Wed Jan 25 00:31:25 2012 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 25 Jan 2012 00:31:25 +0100 (CET) Subject: [pypy-commit] pypy numpy-back-to-applevel: enable even more tests on -A Message-ID: <20120124233125.40A9E82CFD@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: numpy-back-to-applevel Changeset: r51748:15cfdca400bb Date: 2012-01-25 01:30 +0200 http://bitbucket.org/pypy/pypy/changeset/15cfdca400bb/ Log: enable even more tests on -A diff --git a/pypy/module/micronumpy/test/test_base.py b/pypy/module/micronumpy/test/test_base.py --- a/pypy/module/micronumpy/test/test_base.py +++ b/pypy/module/micronumpy/test/test_base.py @@ -12,6 +12,7 @@ if '__pypy__' not in sys.builtin_module_names: import numpy sys.modules['numpypy'] = numpy + sys.modules['_numpypy'] = numpy cls.space = gettestobjspace(usemodules=['micronumpy']) class TestSignature(object): diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py --- a/pypy/module/micronumpy/test/test_dtypes.py +++ b/pypy/module/micronumpy/test/test_dtypes.py @@ -30,7 +30,7 @@ def test_repr_str(self): from _numpypy import dtype - assert repr(dtype) == "<type 'numpypy.dtype'>" + assert '.dtype' in repr(dtype) d = dtype('?') assert repr(d) == "dtype('bool')" assert str(d) == "bool" From noreply at buildbot.pypy.org Wed Jan 25 02:02:39 2012 From: noreply at buildbot.pypy.org (mattip) Date: Wed, 25 Jan 2012 02:02:39 +0100 (CET) Subject: [pypy-commit] pypy numppy-flatitter: setitem, getitem work, add failing tests for comparison ops Message-ID: <20120125010239.AE6F382CFD@wyvern.cs.uni-duesseldorf.de> Author: mattip Branch: numppy-flatitter Changeset: r51750:d73d63090af7 Date: 2012-01-25 03:00 +0200 http://bitbucket.org/pypy/pypy/changeset/d73d63090af7/ Log: setitem, getitem work, add failing tests for comparison ops diff --git a/pypy/module/micronumpy/interp_iter.py b/pypy/module/micronumpy/interp_iter.py --- a/pypy/module/micronumpy/interp_iter.py +++ b/pypy/module/micronumpy/interp_iter.py @@ -133,6 +133,13 @@ res._done = done return res + @jit.unroll_safe + def next_skip_x(self, shapelen, step): + res = self.next(shapelen) + for x in range(step-1): + res = res.next(shapelen) + return res + def apply_transformations(self, arr, transformations): v = BaseIterator.apply_transformations(self, arr, transformations) if len(arr.shape) == 1: diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -898,6 +898,7 @@ self.size = 1 for s in self.shape: self.size *= s + xxx def _del_sources(self): self.left = None @@ -1407,6 +1408,7 @@ ) #TODO:Move all this to another file after fijal finishes reorganization + at jit.unroll_safe def _to_coords(space, arr, w_item_or_slice): '''Returns a start coord, step, and length. ''' @@ -1420,7 +1422,7 @@ elif space.isinstance_w(w_item_or_slice, space.w_slice): start, stop, step, lngth = space.decode_index4(w_item_or_slice,arr.size) else: - raise OperationError(space.w_IndexError, + raise OperationError(space.w_IndexError, space.wrap('unsupported iterator index')) coords = [] i = start @@ -1436,9 +1438,10 @@ i /= arr.shape[s] if i != 0: raise OperationError(space.w_IndexError, space.wrap("invalid index")) - - retval = [coords, step, lngth] - return retval + + retval = [coords, start, step, lngth] + return retval + class W_FlatIterator(ViewArray): @@ -1471,67 +1474,69 @@ return space.newtuple([space.wrap(c) for c in coords]) def descr_getitem(self, space, w_idx): - coords,step,lngth = _to_coords(space, self.base, w_idx) - if lngth > 1: - res = W_NDimArray(lngth, [lngth], self.base.dtype, - self.base.order) - # setslice would have been better, but flat[6:9] for arbitrary - # shapes of array a cannot be represented as a[x1:x2, y1:y2] - basei = ViewIterator(self.base.start, self.base.strides, - self.base.backstrides,self.base.shape) - basei.indices = coords - ri = ArrayIterator(lngth) - itemsize = self.base.dtype.itemtype.get_element_size() - shapelen = len(self.base.shape) - delta = 0 - ri_start = ri.offset - basei_start = basei.offset - bstorage = self.base.storage - while not ri.done(): - # TODO: add a jit_merge_point - - # This is an O(N) approach - # w_val = self.base.getitemspace, c) - # w_result.setitem(ri.offset,w_val) - # Try to do better by checking when we can use memcpy - # from basei_start to basei.offset - - while basei.offset - delta - basei_start == 0: - basei = basei.next(shapelen) - ri = ri.next(shapelen) - delta += 1 - rffi.c_memcpy( - rffi.ptradd(res.storage, ri_start* itemsize), - rffi.ptradd(bstorage, basei_start * itemsize), - delta * itemsize, - ) - delta = 0 - ri_start = ri.offset - basei_start = basei.offset - return res - else: + coords, start, step, lngth = _to_coords(space, self.base, w_idx) + if lngth <2: w_coords = space.newtuple([space.wrap(c) for c in coords]) return self.base.descr_getitem(space, w_coords) + res = W_NDimArray(lngth, [lngth], self.base.dtype, + self.base.order) + # setslice would have been better, but flat[u:v] for arbitrary + # shapes of array a cannot be represented as a[x1:x2, y1:y2] + basei = ViewIterator(self.base.start, self.base.strides, + self.base.backstrides,self.base.shape) + shapelen = len(self.base.shape) + if start > 0: + basei = basei.next_skip_x(shapelen, start) + ri = ArrayIterator(lngth) + while not ri.done(): + # TODO: add a jit_merge_point + w_val = self.base.getitem(basei.offset) + res.setitem(ri.offset,w_val) + basei = basei.next_skip_x(shapelen, step) + ri = ri.next(shapelen) + return res + def descr_setitem(self, space, w_idx, w_value): - coords, step, lngth = _to_coords(space, self.base, w_idx) + coords, start, step, lngth = _to_coords(space, self.base, w_idx) arr = convert_to_array(space, w_value) ai = 0 - return - for c in coords: - v = arr.getitem(ai) - self.base.descr_setitem(space, c, v) + basei = ViewIterator(self.base.start, self.base.strides, + self.base.backstrides,self.base.shape) + shapelen = len(self.base.shape) + if start > 0: + basei = basei.next_skip_x(shapelen, start) + for i in range(lngth): + # TODO: add jit_merge_point + v = arr.getitem(ai).convert_to(self.base.dtype) + print i, basei.offset, basei.indices, v.value + if basei.offset >= self.base.size: + xxx + self.base.setitem(basei.offset, v) # need to repeat input values until all assignments are done ai = (ai + 1) % arr.size - + basei = basei.next_skip_x(shapelen, step) + + def descr_base(self, space): + return space.wrap(self.base) + W_FlatIterator.typedef = TypeDef( 'flatiter', - next = interp2app(W_FlatIterator.descr_next), + #__array__ = #MISSING __iter__ = interp2app(W_FlatIterator.descr_iter), __getitem__ = interp2app(W_FlatIterator.descr_getitem), __setitem__ = interp2app(W_FlatIterator.descr_setitem), + __eq__ = interp2app(BaseArray.descr_eq), + __ne__ = interp2app(BaseArray.descr_ne), + __lt__ = interp2app(BaseArray.descr_lt), + __le__ = interp2app(BaseArray.descr_le), + __gt__ = interp2app(BaseArray.descr_gt), + __ge__ = interp2app(BaseArray.descr_ge), + #__sizeof__ #MISSING + base = GetSetProperty(W_FlatIterator.descr_base), index = GetSetProperty(W_FlatIterator.descr_index), coords = GetSetProperty(W_FlatIterator.descr_coords), + next = interp2app(W_FlatIterator.descr_next), ) W_FlatIterator.acceptable_as_base_class = False diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -1335,6 +1335,8 @@ b.next() b.next() assert b[3] == 3 + assert (b[::3] == [0, 3, 6, 9]).all() + assert (b[2::5] == [2, 7]).all() assert b[-2] == 8 raises(IndexError, "b[11]") raises(IndexError, "b[-11]") @@ -1350,9 +1352,22 @@ b[0:2] = [[[100]]] assert(a[0,0] == 100) assert(a[1,0] == 100) - raises(NotImplementedError, 'b[array([10, 11])] == [-20, -40]') - + raises(IndexError, 'b[array([10, 11])] == [-20, -40]') + def test_flatiter_ops(self): + from _numpypy import arange, array + a = arange(12).reshape(3,4) + b = a.T.flat + assert (b == [0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11]).all() + assert not (b != [0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11]).any() + assert ((b >= range(12)) == [True, True, True,False, True, True, + False, False, True, False, False, True]).all() + assert ((b < range(12)) != [True, True, True,False, True, True, + False, False, True, False, False, True]).all() + assert ((b <= range(12)) != [False, True, True,False, True, True, + False, False, True, False, False, False]).all() + assert ((b > range(12)) == [False, True, True,False, True, True, + False, False, True, False, False, False]).all() def test_flatiter_view(self): from _numpypy import arange a = arange(10).reshape(5, 2) From noreply at buildbot.pypy.org Wed Jan 25 02:02:40 2012 From: noreply at buildbot.pypy.org (mattip) Date: Wed, 25 Jan 2012 02:02:40 +0100 (CET) Subject: [pypy-commit] pypy numppy-flatitter: remove debug cruft Message-ID: <20120125010240.DF71982CFD@wyvern.cs.uni-duesseldorf.de> Author: mattip Branch: numppy-flatitter Changeset: r51751:d1a6dc38d5b5 Date: 2012-01-25 03:02 +0200 http://bitbucket.org/pypy/pypy/changeset/d1a6dc38d5b5/ Log: remove debug cruft diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -1509,9 +1509,6 @@ for i in range(lngth): # TODO: add jit_merge_point v = arr.getitem(ai).convert_to(self.base.dtype) - print i, basei.offset, basei.indices, v.value - if basei.offset >= self.base.size: - xxx self.base.setitem(basei.offset, v) # need to repeat input values until all assignments are done ai = (ai + 1) % arr.size From noreply at buildbot.pypy.org Wed Jan 25 11:52:06 2012 From: noreply at buildbot.pypy.org (antocuni) Date: Wed, 25 Jan 2012 11:52:06 +0100 (CET) Subject: [pypy-commit] buildbot default: try to run at least the own tests in the py3k branch nightly. The locks should take care of scheduling them after all the others Message-ID: <20120125105206.E298D829FF@wyvern.cs.uni-duesseldorf.de> Author: Antonio Cuni <anto.cuni at gmail.com> Branch: Changeset: r622:19bba7abc55c Date: 2012-01-25 11:51 +0100 http://bitbucket.org/pypy/buildbot/changeset/19bba7abc55c/ Log: try to run at least the own tests in the py3k branch nightly. The locks should take care of scheduling them after all the others diff --git a/bot2/pypybuildbot/master.py b/bot2/pypybuildbot/master.py --- a/bot2/pypybuildbot/master.py +++ b/bot2/pypybuildbot/master.py @@ -225,7 +225,12 @@ JITWIN32, # on bigboard #JITFREEBSD64, # on headless JITMACOSX64, # on mvt's machine - ], branch=None, hour=3, minute=0) + ], branch=None, hour=3, minute=0), + + Nightly("nighly-4-00-py3k", [ + LINUX32, # on tannit32, uses 4 cores + ], branch='py3k', hour=4, minute=0), + ], 'status': [status, ircbot], From noreply at buildbot.pypy.org Wed Jan 25 15:36:18 2012 From: noreply at buildbot.pypy.org (l.diekmann) Date: Wed, 25 Jan 2012 15:36:18 +0100 (CET) Subject: [pypy-commit] pypy type-specialized-instances: merge default Message-ID: <20120125143618.B2AF3829FF@wyvern.cs.uni-duesseldorf.de> Author: Lukas Diekmann <lukas.diekmann at uni-duesseldorf.de> Branch: type-specialized-instances Changeset: r51752:7636a491c009 Date: 2012-01-25 15:34 +0100 http://bitbucket.org/pypy/pypy/changeset/7636a491c009/ Log: merge default diff too long, truncating to 10000 out of 43426 lines diff --git a/.hgignore b/.hgignore --- a/.hgignore +++ b/.hgignore @@ -2,6 +2,9 @@ *.py[co] *~ .*.swp +.idea +.project +.pydevproject syntax: regexp ^testresult$ diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -27,7 +27,7 @@ DEALINGS IN THE SOFTWARE. -PyPy Copyright holders 2003-2011 +PyPy Copyright holders 2003-2012 ----------------------------------- Except when otherwise stated (look for LICENSE files or information at @@ -37,43 +37,47 @@ Armin Rigo Maciej Fijalkowski Carl Friedrich Bolz + Amaury Forgeot d'Arc Antonio Cuni - Amaury Forgeot d'Arc Samuele Pedroni Michael Hudson Holger Krekel - Benjamin Peterson + Alex Gaynor Christian Tismer Hakan Ardo - Alex Gaynor + Benjamin Peterson + David Schneider Eric van Riet Paap Anders Chrigstrom - David Schneider Richard Emslie Dan Villiom Podlaski Christiansen Alexander Schremmer + Lukas Diekmann Aurelien Campeas Anders Lehmann Camillo Bruni Niklaus Haldimann + Sven Hager Leonardo Santagada Toon Verwaest Seo Sanghyeon + Justin Peel Lawrence Oluyede Bartosz Skowron Jakub Gustak Guido Wesdorp Daniel Roberts + Laura Creighton Adrien Di Mascio - Laura Creighton Ludovic Aubry Niko Matsakis + Wim Lavrijsen + Matti Picus Jason Creighton Jacob Hallen Alex Martelli Anders Hammarquist Jan de Mooij - Wim Lavrijsen Stephan Diehl Michael Foord Stefan Schwarzer @@ -84,34 +88,36 @@ Alexandre Fayolle Marius Gedminas Simon Burton - Justin Peel + David Edelsohn Jean-Paul Calderone John Witulski - Lukas Diekmann + Timo Paulssen holger krekel - Wim Lavrijsen Dario Bertini + Mark Pearse Andreas Stührk Jean-Philippe St. Pierre Guido van Rossum Pavel Vinogradov Valentino Volonghi Paul deGrandis + Ilya Osadchiy + Ronny Pfannschmidt Adrian Kuhn tav Georg Brandl + Philip Jenvey Gerald Klix Wanja Saatkamp - Ronny Pfannschmidt Boris Feigin Oscar Nierstrasz David Malcolm Eugene Oden Henry Mason - Sven Hager + Jeff Terrace Lukas Renggli - Ilya Osadchiy Guenter Jantzen + Ned Batchelder Bert Freudenberg Amit Regmi Ben Young @@ -142,7 +148,6 @@ Anders Qvist Beatrice During Alexander Sedov - Timo Paulssen Corbin Simpson Vincent Legoll Romain Guillebert @@ -165,9 +170,10 @@ Lucio Torre Lene Wagner Miguel de Val Borro + Artur Lisiecki + Bruno Gola Ignas Mikalajunas - Artur Lisiecki - Philip Jenvey + Stefano Rivera Joshua Gilbert Godefroid Chappelle Yusei Tahara @@ -179,17 +185,17 @@ Kristjan Valur Jonsson Bobby Impollonia Michael Hudson-Doyle + Laurence Tratt + Yasir Suhail Andrew Thompson Anders Sigfridsson Floris Bruynooghe Jacek Generowicz Dan Colish Zooko Wilcox-O Hearn - Dan Villiom Podlaski Christiansen - Anders Hammarquist + Dan Loewenherz Chris Lambacher Dinu Gherman - Dan Colish Brett Cannon Daniel Neuhäuser Michael Chermside diff --git a/lib-python/modified-2.7/ctypes/__init__.py b/lib-python/modified-2.7/ctypes/__init__.py --- a/lib-python/modified-2.7/ctypes/__init__.py +++ b/lib-python/modified-2.7/ctypes/__init__.py @@ -351,7 +351,7 @@ self._FuncPtr = _FuncPtr if handle is None: - self._handle = _ffi.CDLL(name) + self._handle = _ffi.CDLL(name, mode) else: self._handle = handle diff --git a/lib-python/modified-2.7/ctypes/test/test_callbacks.py b/lib-python/modified-2.7/ctypes/test/test_callbacks.py --- a/lib-python/modified-2.7/ctypes/test/test_callbacks.py +++ b/lib-python/modified-2.7/ctypes/test/test_callbacks.py @@ -1,5 +1,6 @@ import unittest from ctypes import * +from ctypes.test import xfail import _ctypes_test class Callbacks(unittest.TestCase): @@ -98,6 +99,7 @@ ## self.check_type(c_char_p, "abc") ## self.check_type(c_char_p, "def") + @xfail def test_pyobject(self): o = () from sys import getrefcount as grc diff --git a/lib-python/modified-2.7/ctypes/test/test_libc.py b/lib-python/modified-2.7/ctypes/test/test_libc.py --- a/lib-python/modified-2.7/ctypes/test/test_libc.py +++ b/lib-python/modified-2.7/ctypes/test/test_libc.py @@ -25,7 +25,10 @@ lib.my_qsort(chars, len(chars)-1, sizeof(c_char), comparefunc(sort)) self.assertEqual(chars.raw, " ,,aaaadmmmnpppsss\x00") - def test_no_more_xfail(self): + def SKIPPED_test_no_more_xfail(self): + # We decided to not explicitly support the whole ctypes-2.7 + # and instead go for a case-by-case, demand-driven approach. + # So this test is skipped instead of failing. import socket import ctypes.test self.assertTrue(not hasattr(ctypes.test, 'xfail'), diff --git a/lib_pypy/_collections.py b/lib_pypy/_collections.py --- a/lib_pypy/_collections.py +++ b/lib_pypy/_collections.py @@ -406,7 +406,7 @@ recurse.remove(id(self)) def copy(self): - return type(self)(self, default_factory=self.default_factory) + return type(self)(self.default_factory, self) def __copy__(self): return self.copy() diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py --- a/lib_pypy/_ctypes/structure.py +++ b/lib_pypy/_ctypes/structure.py @@ -73,8 +73,12 @@ class Field(object): def __init__(self, name, offset, size, ctype, num, is_bitfield): - for k in ('name', 'offset', 'size', 'ctype', 'num', 'is_bitfield'): - self.__dict__[k] = locals()[k] + self.__dict__['name'] = name + self.__dict__['offset'] = offset + self.__dict__['size'] = size + self.__dict__['ctype'] = ctype + self.__dict__['num'] = num + self.__dict__['is_bitfield'] = is_bitfield def __setattr__(self, name, value): raise AttributeError(name) diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py --- a/lib_pypy/_sqlite3.py +++ b/lib_pypy/_sqlite3.py @@ -20,6 +20,8 @@ # 2. Altered source versions must be plainly marked as such, and must not be # misrepresented as being the original software. # 3. This notice may not be removed or altered from any source distribution. +# +# Note: This software has been modified for use in PyPy. from ctypes import c_void_p, c_int, c_double, c_int64, c_char_p, cdll from ctypes import POINTER, byref, string_at, CFUNCTYPE, cast @@ -27,7 +29,6 @@ from collections import OrderedDict import datetime import sys -import time import weakref from threading import _get_ident as thread_get_ident @@ -231,8 +232,10 @@ sqlite.sqlite3_result_text.argtypes = [c_void_p, c_char_p, c_int, c_void_p] sqlite.sqlite3_result_text.restype = None -sqlite.sqlite3_enable_load_extension.argtypes = [c_void_p, c_int] -sqlite.sqlite3_enable_load_extension.restype = c_int +HAS_LOAD_EXTENSION = hasattr(sqlite, "sqlite3_enable_load_extension") +if HAS_LOAD_EXTENSION: + sqlite.sqlite3_enable_load_extension.argtypes = [c_void_p, c_int] + sqlite.sqlite3_enable_load_extension.restype = c_int ########################################## # END Wrapped SQLite C API and constants @@ -604,7 +607,7 @@ def authorizer(userdata, action, arg1, arg2, dbname, source): try: return int(callback(action, arg1, arg2, dbname, source)) - except Exception, e: + except Exception: return SQLITE_DENY c_authorizer = AUTHORIZER(authorizer) @@ -651,7 +654,7 @@ if not aggregate_ptr[0]: try: aggregate = cls() - except Exception, e: + except Exception: msg = ("user-defined aggregate's '__init__' " "method raised error") sqlite.sqlite3_result_error(context, msg, len(msg)) @@ -665,7 +668,7 @@ params = _convert_params(context, argc, c_params) try: aggregate.step(*params) - except Exception, e: + except Exception: msg = ("user-defined aggregate's 'step' " "method raised error") sqlite.sqlite3_result_error(context, msg, len(msg)) @@ -681,7 +684,7 @@ aggregate = self.aggregate_instances[aggregate_ptr[0]] try: val = aggregate.finalize() - except Exception, e: + except Exception: msg = ("user-defined aggregate's 'finalize' " "method raised error") sqlite.sqlite3_result_error(context, msg, len(msg)) @@ -708,13 +711,14 @@ from sqlite3.dump import _iterdump return _iterdump(self) - def enable_load_extension(self, enabled): - self._check_thread() - self._check_closed() + if HAS_LOAD_EXTENSION: + def enable_load_extension(self, enabled): + self._check_thread() + self._check_closed() - rc = sqlite.sqlite3_enable_load_extension(self.db, int(enabled)) - if rc != SQLITE_OK: - raise OperationalError("Error enabling load extension") + rc = sqlite.sqlite3_enable_load_extension(self.db, int(enabled)) + if rc != SQLITE_OK: + raise OperationalError("Error enabling load extension") DML, DQL, DDL = range(3) @@ -768,7 +772,7 @@ self.statement.item = None self.statement.exhausted = True - if self.statement.kind == DML or self.statement.kind == DDL: + if self.statement.kind == DML: self.statement.reset() self.rowcount = -1 @@ -788,7 +792,7 @@ if self.statement.kind == DML: self.connection._begin() else: - raise ProgrammingError, "executemany is only for DML statements" + raise ProgrammingError("executemany is only for DML statements") self.rowcount = 0 for params in many_params: @@ -858,8 +862,6 @@ except StopIteration: return None - return nextrow - def fetchmany(self, size=None): self._check_closed() self._check_reset() @@ -912,7 +914,7 @@ def __init__(self, connection, sql): self.statement = None if not isinstance(sql, str): - raise ValueError, "sql must be a string" + raise ValueError("sql must be a string") self.con = connection self.sql = sql # DEBUG ONLY first_word = self._statement_kind = sql.lstrip().split(" ")[0].upper() @@ -941,8 +943,8 @@ raise self.con._get_exception(ret) self.con._remember_statement(self) if _check_remaining_sql(next_char.value): - raise Warning, "One and only one statement required: %r" % ( - next_char.value,) + raise Warning("One and only one statement required: %r" % ( + next_char.value,)) # sql_char should remain alive until here self._build_row_cast_map() @@ -1013,7 +1015,7 @@ elif type(param) is buffer: sqlite.sqlite3_bind_blob(self.statement, idx, str(param), len(param), SQLITE_TRANSIENT) else: - raise InterfaceError, "parameter type %s is not supported" % str(type(param)) + raise InterfaceError("parameter type %s is not supported" % str(type(param))) def set_params(self, params): ret = sqlite.sqlite3_reset(self.statement) @@ -1042,11 +1044,11 @@ for idx in range(1, sqlite.sqlite3_bind_parameter_count(self.statement) + 1): param_name = sqlite.sqlite3_bind_parameter_name(self.statement, idx) if param_name is None: - raise ProgrammingError, "need named parameters" + raise ProgrammingError("need named parameters") param_name = param_name[1:] try: param = params[param_name] - except KeyError, e: + except KeyError: raise ProgrammingError("missing parameter '%s'" %param) self.set_param(idx, param) @@ -1257,7 +1259,7 @@ params = _convert_params(context, nargs, c_params) try: val = real_cb(*params) - except Exception, e: + except Exception: msg = "user-defined function raised exception" sqlite.sqlite3_result_error(context, msg, len(msg)) else: diff --git a/lib_pypy/datetime.py b/lib_pypy/datetime.py --- a/lib_pypy/datetime.py +++ b/lib_pypy/datetime.py @@ -13,7 +13,7 @@ Sources for time zone and DST data: http://www.twinsun.com/tz/tz-link.htm This was originally copied from the sandbox of the CPython CVS repository. -Thanks to Tim Peters for suggesting using it. +Thanks to Tim Peters for suggesting using it. """ import time as _time @@ -271,6 +271,8 @@ raise ValueError("%s()=%d, must be in -1439..1439" % (name, offset)) def _check_date_fields(year, month, day): + if not isinstance(year, (int, long)): + raise TypeError('int expected') if not MINYEAR <= year <= MAXYEAR: raise ValueError('year must be in %d..%d' % (MINYEAR, MAXYEAR), year) if not 1 <= month <= 12: @@ -280,6 +282,8 @@ raise ValueError('day must be in 1..%d' % dim, day) def _check_time_fields(hour, minute, second, microsecond): + if not isinstance(hour, (int, long)): + raise TypeError('int expected') if not 0 <= hour <= 23: raise ValueError('hour must be in 0..23', hour) if not 0 <= minute <= 59: @@ -543,61 +547,76 @@ self = object.__new__(cls) - self.__days = d - self.__seconds = s - self.__microseconds = us + self._days = d + self._seconds = s + self._microseconds = us if abs(d) > 999999999: raise OverflowError("timedelta # of days is too large: %d" % d) return self def __repr__(self): - if self.__microseconds: + if self._microseconds: return "%s(%d, %d, %d)" % ('datetime.' + self.__class__.__name__, - self.__days, - self.__seconds, - self.__microseconds) - if self.__seconds: + self._days, + self._seconds, + self._microseconds) + if self._seconds: return "%s(%d, %d)" % ('datetime.' + self.__class__.__name__, - self.__days, - self.__seconds) - return "%s(%d)" % ('datetime.' + self.__class__.__name__, self.__days) + self._days, + self._seconds) + return "%s(%d)" % ('datetime.' + self.__class__.__name__, self._days) def __str__(self): - mm, ss = divmod(self.__seconds, 60) + mm, ss = divmod(self._seconds, 60) hh, mm = divmod(mm, 60) s = "%d:%02d:%02d" % (hh, mm, ss) - if self.__days: + if self._days: def plural(n): return n, abs(n) != 1 and "s" or "" - s = ("%d day%s, " % plural(self.__days)) + s - if self.__microseconds: - s = s + ".%06d" % self.__microseconds + s = ("%d day%s, " % plural(self._days)) + s + if self._microseconds: + s = s + ".%06d" % self._microseconds return s - days = property(lambda self: self.__days, doc="days") - seconds = property(lambda self: self.__seconds, doc="seconds") - microseconds = property(lambda self: self.__microseconds, - doc="microseconds") - def total_seconds(self): return ((self.days * 86400 + self.seconds) * 10**6 + self.microseconds) / 1e6 + # Read-only field accessors + @property + def days(self): + """days""" + return self._days + + @property + def seconds(self): + """seconds""" + return self._seconds + + @property + def microseconds(self): + """microseconds""" + return self._microseconds + def __add__(self, other): if isinstance(other, timedelta): # for CPython compatibility, we cannot use # our __class__ here, but need a real timedelta - return timedelta(self.__days + other.__days, - self.__seconds + other.__seconds, - self.__microseconds + other.__microseconds) + return timedelta(self._days + other._days, + self._seconds + other._seconds, + self._microseconds + other._microseconds) return NotImplemented __radd__ = __add__ def __sub__(self, other): if isinstance(other, timedelta): - return self + -other + # for CPython compatibility, we cannot use + # our __class__ here, but need a real timedelta + return timedelta(self._days - other._days, + self._seconds - other._seconds, + self._microseconds - other._microseconds) return NotImplemented def __rsub__(self, other): @@ -606,17 +625,17 @@ return NotImplemented def __neg__(self): - # for CPython compatibility, we cannot use - # our __class__ here, but need a real timedelta - return timedelta(-self.__days, - -self.__seconds, - -self.__microseconds) + # for CPython compatibility, we cannot use + # our __class__ here, but need a real timedelta + return timedelta(-self._days, + -self._seconds, + -self._microseconds) def __pos__(self): return self def __abs__(self): - if self.__days < 0: + if self._days < 0: return -self else: return self @@ -625,81 +644,81 @@ if isinstance(other, (int, long)): # for CPython compatibility, we cannot use # our __class__ here, but need a real timedelta - return timedelta(self.__days * other, - self.__seconds * other, - self.__microseconds * other) + return timedelta(self._days * other, + self._seconds * other, + self._microseconds * other) return NotImplemented __rmul__ = __mul__ def __div__(self, other): if isinstance(other, (int, long)): - usec = ((self.__days * (24*3600L) + self.__seconds) * 1000000 + - self.__microseconds) + usec = ((self._days * (24*3600L) + self._seconds) * 1000000 + + self._microseconds) return timedelta(0, 0, usec // other) return NotImplemented __floordiv__ = __div__ - # Comparisons. + # Comparisons of timedelta objects with other. def __eq__(self, other): if isinstance(other, timedelta): - return self.__cmp(other) == 0 + return self._cmp(other) == 0 else: return False def __ne__(self, other): if isinstance(other, timedelta): - return self.__cmp(other) != 0 + return self._cmp(other) != 0 else: return True def __le__(self, other): if isinstance(other, timedelta): - return self.__cmp(other) <= 0 + return self._cmp(other) <= 0 else: _cmperror(self, other) def __lt__(self, other): if isinstance(other, timedelta): - return self.__cmp(other) < 0 + return self._cmp(other) < 0 else: _cmperror(self, other) def __ge__(self, other): if isinstance(other, timedelta): - return self.__cmp(other) >= 0 + return self._cmp(other) >= 0 else: _cmperror(self, other) def __gt__(self, other): if isinstance(other, timedelta): - return self.__cmp(other) > 0 + return self._cmp(other) > 0 else: _cmperror(self, other) - def __cmp(self, other): + def _cmp(self, other): assert isinstance(other, timedelta) - return cmp(self.__getstate(), other.__getstate()) + return cmp(self._getstate(), other._getstate()) def __hash__(self): - return hash(self.__getstate()) + return hash(self._getstate()) def __nonzero__(self): - return (self.__days != 0 or - self.__seconds != 0 or - self.__microseconds != 0) + return (self._days != 0 or + self._seconds != 0 or + self._microseconds != 0) # Pickle support. __safe_for_unpickling__ = True # For Python 2.2 - def __getstate(self): - return (self.__days, self.__seconds, self.__microseconds) + def _getstate(self): + return (self._days, self._seconds, self._microseconds) def __reduce__(self): - return (self.__class__, self.__getstate()) + return (self.__class__, self._getstate()) timedelta.min = timedelta(-999999999) timedelta.max = timedelta(days=999999999, hours=23, minutes=59, seconds=59, @@ -749,25 +768,26 @@ return self _check_date_fields(year, month, day) self = object.__new__(cls) - self.__year = year - self.__month = month - self.__day = day + self._year = year + self._month = month + self._day = day return self # Additional constructors + @classmethod def fromtimestamp(cls, t): "Construct a date from a POSIX timestamp (like time.time())." y, m, d, hh, mm, ss, weekday, jday, dst = _time.localtime(t) return cls(y, m, d) - fromtimestamp = classmethod(fromtimestamp) + @classmethod def today(cls): "Construct a date from time.time()." t = _time.time() return cls.fromtimestamp(t) - today = classmethod(today) + @classmethod def fromordinal(cls, n): """Contruct a date from a proleptic Gregorian ordinal. @@ -776,16 +796,24 @@ """ y, m, d = _ord2ymd(n) return cls(y, m, d) - fromordinal = classmethod(fromordinal) # Conversions to string def __repr__(self): - "Convert to formal string, for repr()." + """Convert to formal string, for repr(). + + >>> dt = datetime(2010, 1, 1) + >>> repr(dt) + 'datetime.datetime(2010, 1, 1, 0, 0)' + + >>> dt = datetime(2010, 1, 1, tzinfo=timezone.utc) + >>> repr(dt) + 'datetime.datetime(2010, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)' + """ return "%s(%d, %d, %d)" % ('datetime.' + self.__class__.__name__, - self.__year, - self.__month, - self.__day) + self._year, + self._month, + self._day) # XXX These shouldn't depend on time.localtime(), because that # clips the usable dates to [1970 .. 2038). At least ctime() is # easily done without using strftime() -- that's better too because @@ -793,12 +821,20 @@ def ctime(self): "Format a la ctime()." - return tmxxx(self.__year, self.__month, self.__day).ctime() + return tmxxx(self._year, self._month, self._day).ctime() def strftime(self, fmt): "Format using strftime()." return _wrap_strftime(self, fmt, self.timetuple()) + def __format__(self, fmt): + if not isinstance(fmt, (str, unicode)): + raise ValueError("__format__ excepts str or unicode, not %s" % + fmt.__class__.__name__) + if len(fmt) != 0: + return self.strftime(fmt) + return str(self) + def isoformat(self): """Return the date formatted according to ISO. @@ -808,29 +844,31 @@ - http://www.w3.org/TR/NOTE-datetime - http://www.cl.cam.ac.uk/~mgk25/iso-time.html """ - return "%04d-%02d-%02d" % (self.__year, self.__month, self.__day) + return "%04d-%02d-%02d" % (self._year, self._month, self._day) __str__ = isoformat - def __format__(self, format): - if not isinstance(format, (str, unicode)): - raise ValueError("__format__ excepts str or unicode, not %s" % - format.__class__.__name__) - if not format: - return str(self) - return self.strftime(format) + # Read-only field accessors + @property + def year(self): + """year (1-9999)""" + return self._year - # Read-only field accessors - year = property(lambda self: self.__year, - doc="year (%d-%d)" % (MINYEAR, MAXYEAR)) - month = property(lambda self: self.__month, doc="month (1-12)") - day = property(lambda self: self.__day, doc="day (1-31)") + @property + def month(self): + """month (1-12)""" + return self._month + + @property + def day(self): + """day (1-31)""" + return self._day # Standard conversions, __cmp__, __hash__ (and helpers) def timetuple(self): "Return local time tuple compatible with time.localtime()." - return _build_struct_time(self.__year, self.__month, self.__day, + return _build_struct_time(self._year, self._month, self._day, 0, 0, 0, -1) def toordinal(self): @@ -839,24 +877,24 @@ January 1 of year 1 is day 1. Only the year, month and day values contribute to the result. """ - return _ymd2ord(self.__year, self.__month, self.__day) + return _ymd2ord(self._year, self._month, self._day) def replace(self, year=None, month=None, day=None): """Return a new date with new values for the specified fields.""" if year is None: - year = self.__year + year = self._year if month is None: - month = self.__month + month = self._month if day is None: - day = self.__day + day = self._day _check_date_fields(year, month, day) return date(year, month, day) - # Comparisons. + # Comparisons of date objects with other. def __eq__(self, other): if isinstance(other, date): - return self.__cmp(other) == 0 + return self._cmp(other) == 0 elif hasattr(other, "timetuple"): return NotImplemented else: @@ -864,7 +902,7 @@ def __ne__(self, other): if isinstance(other, date): - return self.__cmp(other) != 0 + return self._cmp(other) != 0 elif hasattr(other, "timetuple"): return NotImplemented else: @@ -872,7 +910,7 @@ def __le__(self, other): if isinstance(other, date): - return self.__cmp(other) <= 0 + return self._cmp(other) <= 0 elif hasattr(other, "timetuple"): return NotImplemented else: @@ -880,7 +918,7 @@ def __lt__(self, other): if isinstance(other, date): - return self.__cmp(other) < 0 + return self._cmp(other) < 0 elif hasattr(other, "timetuple"): return NotImplemented else: @@ -888,7 +926,7 @@ def __ge__(self, other): if isinstance(other, date): - return self.__cmp(other) >= 0 + return self._cmp(other) >= 0 elif hasattr(other, "timetuple"): return NotImplemented else: @@ -896,21 +934,21 @@ def __gt__(self, other): if isinstance(other, date): - return self.__cmp(other) > 0 + return self._cmp(other) > 0 elif hasattr(other, "timetuple"): return NotImplemented else: _cmperror(self, other) - def __cmp(self, other): + def _cmp(self, other): assert isinstance(other, date) - y, m, d = self.__year, self.__month, self.__day - y2, m2, d2 = other.__year, other.__month, other.__day + y, m, d = self._year, self._month, self._day + y2, m2, d2 = other._year, other._month, other._day return cmp((y, m, d), (y2, m2, d2)) def __hash__(self): "Hash." - return hash(self.__getstate()) + return hash(self._getstate()) # Computations @@ -922,9 +960,9 @@ def __add__(self, other): "Add a date to a timedelta." if isinstance(other, timedelta): - t = tmxxx(self.__year, - self.__month, - self.__day + other.days) + t = tmxxx(self._year, + self._month, + self._day + other.days) self._checkOverflow(t.year) result = date(t.year, t.month, t.day) return result @@ -966,9 +1004,9 @@ ISO calendar algorithm taken from http://www.phys.uu.nl/~vgent/calendar/isocalendar.htm """ - year = self.__year + year = self._year week1monday = _isoweek1monday(year) - today = _ymd2ord(self.__year, self.__month, self.__day) + today = _ymd2ord(self._year, self._month, self._day) # Internally, week and day have origin 0 week, day = divmod(today - week1monday, 7) if week < 0: @@ -985,18 +1023,18 @@ __safe_for_unpickling__ = True # For Python 2.2 - def __getstate(self): - yhi, ylo = divmod(self.__year, 256) - return ("%c%c%c%c" % (yhi, ylo, self.__month, self.__day), ) + def _getstate(self): + yhi, ylo = divmod(self._year, 256) + return ("%c%c%c%c" % (yhi, ylo, self._month, self._day), ) def __setstate(self, string): if len(string) != 4 or not (1 <= ord(string[2]) <= 12): raise TypeError("not enough arguments") - yhi, ylo, self.__month, self.__day = map(ord, string) - self.__year = yhi * 256 + ylo + yhi, ylo, self._month, self._day = map(ord, string) + self._year = yhi * 256 + ylo def __reduce__(self): - return (self.__class__, self.__getstate()) + return (self.__class__, self._getstate()) _date_class = date # so functions w/ args named "date" can get at the class @@ -1118,62 +1156,80 @@ return self _check_tzinfo_arg(tzinfo) _check_time_fields(hour, minute, second, microsecond) - self.__hour = hour - self.__minute = minute - self.__second = second - self.__microsecond = microsecond + self._hour = hour + self._minute = minute + self._second = second + self._microsecond = microsecond self._tzinfo = tzinfo return self # Read-only field accessors - hour = property(lambda self: self.__hour, doc="hour (0-23)") - minute = property(lambda self: self.__minute, doc="minute (0-59)") - second = property(lambda self: self.__second, doc="second (0-59)") - microsecond = property(lambda self: self.__microsecond, - doc="microsecond (0-999999)") - tzinfo = property(lambda self: self._tzinfo, doc="timezone info object") + @property + def hour(self): + """hour (0-23)""" + return self._hour + + @property + def minute(self): + """minute (0-59)""" + return self._minute + + @property + def second(self): + """second (0-59)""" + return self._second + + @property + def microsecond(self): + """microsecond (0-999999)""" + return self._microsecond + + @property + def tzinfo(self): + """timezone info object""" + return self._tzinfo # Standard conversions, __hash__ (and helpers) - # Comparisons. + # Comparisons of time objects with other. def __eq__(self, other): if isinstance(other, time): - return self.__cmp(other) == 0 + return self._cmp(other) == 0 else: return False def __ne__(self, other): if isinstance(other, time): - return self.__cmp(other) != 0 + return self._cmp(other) != 0 else: return True def __le__(self, other): if isinstance(other, time): - return self.__cmp(other) <= 0 + return self._cmp(other) <= 0 else: _cmperror(self, other) def __lt__(self, other): if isinstance(other, time): - return self.__cmp(other) < 0 + return self._cmp(other) < 0 else: _cmperror(self, other) def __ge__(self, other): if isinstance(other, time): - return self.__cmp(other) >= 0 + return self._cmp(other) >= 0 else: _cmperror(self, other) def __gt__(self, other): if isinstance(other, time): - return self.__cmp(other) > 0 + return self._cmp(other) > 0 else: _cmperror(self, other) - def __cmp(self, other): + def _cmp(self, other): assert isinstance(other, time) mytz = self._tzinfo ottz = other._tzinfo @@ -1187,23 +1243,23 @@ base_compare = myoff == otoff if base_compare: - return cmp((self.__hour, self.__minute, self.__second, - self.__microsecond), - (other.__hour, other.__minute, other.__second, - other.__microsecond)) + return cmp((self._hour, self._minute, self._second, + self._microsecond), + (other._hour, other._minute, other._second, + other._microsecond)) if myoff is None or otoff is None: # XXX Buggy in 2.2.2. raise TypeError("cannot compare naive and aware times") - myhhmm = self.__hour * 60 + self.__minute - myoff - othhmm = other.__hour * 60 + other.__minute - otoff - return cmp((myhhmm, self.__second, self.__microsecond), - (othhmm, other.__second, other.__microsecond)) + myhhmm = self._hour * 60 + self._minute - myoff + othhmm = other._hour * 60 + other._minute - otoff + return cmp((myhhmm, self._second, self._microsecond), + (othhmm, other._second, other._microsecond)) def __hash__(self): """Hash.""" tzoff = self._utcoffset() if not tzoff: # zero or None - return hash(self.__getstate()[0]) + return hash(self._getstate()[0]) h, m = divmod(self.hour * 60 + self.minute - tzoff, 60) if 0 <= h < 24: return hash(time(h, m, self.second, self.microsecond)) @@ -1227,14 +1283,14 @@ def __repr__(self): """Convert to formal string, for repr().""" - if self.__microsecond != 0: - s = ", %d, %d" % (self.__second, self.__microsecond) - elif self.__second != 0: - s = ", %d" % self.__second + if self._microsecond != 0: + s = ", %d, %d" % (self._second, self._microsecond) + elif self._second != 0: + s = ", %d" % self._second else: s = "" s= "%s(%d, %d%s)" % ('datetime.' + self.__class__.__name__, - self.__hour, self.__minute, s) + self._hour, self._minute, s) if self._tzinfo is not None: assert s[-1:] == ")" s = s[:-1] + ", tzinfo=%r" % self._tzinfo + ")" @@ -1246,8 +1302,8 @@ This is 'HH:MM:SS.mmmmmm+zz:zz', or 'HH:MM:SS+zz:zz' if self.microsecond == 0. """ - s = _format_time(self.__hour, self.__minute, self.__second, - self.__microsecond) + s = _format_time(self._hour, self._minute, self._second, + self._microsecond) tz = self._tzstr() if tz: s += tz @@ -1255,14 +1311,6 @@ __str__ = isoformat - def __format__(self, format): - if not isinstance(format, (str, unicode)): - raise ValueError("__format__ excepts str or unicode, not %s" % - format.__class__.__name__) - if not format: - return str(self) - return self.strftime(format) - def strftime(self, fmt): """Format using strftime(). The date part of the timestamp passed to underlying strftime should not be used. @@ -1270,10 +1318,18 @@ # The year must be >= 1900 else Python's strftime implementation # can raise a bogus exception. timetuple = (1900, 1, 1, - self.__hour, self.__minute, self.__second, + self._hour, self._minute, self._second, 0, 1, -1) return _wrap_strftime(self, fmt, timetuple) + def __format__(self, fmt): + if not isinstance(fmt, (str, unicode)): + raise ValueError("__format__ excepts str or unicode, not %s" % + fmt.__class__.__name__) + if len(fmt) != 0: + return self.strftime(fmt) + return str(self) + # Timezone functions def utcoffset(self): @@ -1350,10 +1406,10 @@ __safe_for_unpickling__ = True # For Python 2.2 - def __getstate(self): - us2, us3 = divmod(self.__microsecond, 256) + def _getstate(self): + us2, us3 = divmod(self._microsecond, 256) us1, us2 = divmod(us2, 256) - basestate = ("%c" * 6) % (self.__hour, self.__minute, self.__second, + basestate = ("%c" * 6) % (self._hour, self._minute, self._second, us1, us2, us3) if self._tzinfo is None: return (basestate,) @@ -1363,13 +1419,13 @@ def __setstate(self, string, tzinfo): if len(string) != 6 or ord(string[0]) >= 24: raise TypeError("an integer is required") - self.__hour, self.__minute, self.__second, us1, us2, us3 = \ + self._hour, self._minute, self._second, us1, us2, us3 = \ map(ord, string) - self.__microsecond = (((us1 << 8) | us2) << 8) | us3 + self._microsecond = (((us1 << 8) | us2) << 8) | us3 self._tzinfo = tzinfo def __reduce__(self): - return (time, self.__getstate()) + return (time, self._getstate()) _time_class = time # so functions w/ args named "time" can get at the class @@ -1378,9 +1434,11 @@ time.resolution = timedelta(microseconds=1) class datetime(date): + """datetime(year, month, day[, hour[, minute[, second[, microsecond[,tzinfo]]]]]) - # XXX needs docstrings - # See http://www.zope.org/Members/fdrake/DateTimeWiki/TimeZoneInfo + The year, month and day arguments are required. tzinfo may be None, or an + instance of a tzinfo subclass. The remaining arguments may be ints or longs. + """ def __new__(cls, year, month=None, day=None, hour=0, minute=0, second=0, microsecond=0, tzinfo=None): @@ -1393,24 +1451,43 @@ _check_time_fields(hour, minute, second, microsecond) self = date.__new__(cls, year, month, day) # XXX This duplicates __year, __month, __day for convenience :-( - self.__year = year - self.__month = month - self.__day = day - self.__hour = hour - self.__minute = minute - self.__second = second - self.__microsecond = microsecond + self._year = year + self._month = month + self._day = day + self._hour = hour + self._minute = minute + self._second = second + self._microsecond = microsecond self._tzinfo = tzinfo return self # Read-only field accessors - hour = property(lambda self: self.__hour, doc="hour (0-23)") - minute = property(lambda self: self.__minute, doc="minute (0-59)") - second = property(lambda self: self.__second, doc="second (0-59)") - microsecond = property(lambda self: self.__microsecond, - doc="microsecond (0-999999)") - tzinfo = property(lambda self: self._tzinfo, doc="timezone info object") + @property + def hour(self): + """hour (0-23)""" + return self._hour + @property + def minute(self): + """minute (0-59)""" + return self._minute + + @property + def second(self): + """second (0-59)""" + return self._second + + @property + def microsecond(self): + """microsecond (0-999999)""" + return self._microsecond + + @property + def tzinfo(self): + """timezone info object""" + return self._tzinfo + + @classmethod def fromtimestamp(cls, t, tz=None): """Construct a datetime from a POSIX timestamp (like time.time()). @@ -1438,37 +1515,42 @@ if tz is not None: result = tz.fromutc(result) return result - fromtimestamp = classmethod(fromtimestamp) + @classmethod def utcfromtimestamp(cls, t): "Construct a UTC datetime from a POSIX timestamp (like time.time())." - if 1 - (t % 1.0) < 0.0000005: - t = float(int(t)) + 1 - if t < 0: - t -= 1 + t, frac = divmod(t, 1.0) + us = round(frac * 1e6) + + # If timestamp is less than one microsecond smaller than a + # full second, us can be rounded up to 1000000. In this case, + # roll over to seconds, otherwise, ValueError is raised + # by the constructor. + if us == 1000000: + t += 1 + us = 0 y, m, d, hh, mm, ss, weekday, jday, dst = _time.gmtime(t) - us = int((t % 1.0) * 1000000) ss = min(ss, 59) # clamp out leap seconds if the platform has them return cls(y, m, d, hh, mm, ss, us) - utcfromtimestamp = classmethod(utcfromtimestamp) # XXX This is supposed to do better than we *can* do by using time.time(), # XXX if the platform supports a more accurate way. The C implementation # XXX uses gettimeofday on platforms that have it, but that isn't # XXX available from Python. So now() may return different results # XXX across the implementations. + @classmethod def now(cls, tz=None): "Construct a datetime from time.time() and optional time zone info." t = _time.time() return cls.fromtimestamp(t, tz) - now = classmethod(now) + @classmethod def utcnow(cls): "Construct a UTC datetime from time.time()." t = _time.time() return cls.utcfromtimestamp(t) - utcnow = classmethod(utcnow) + @classmethod def combine(cls, date, time): "Construct a datetime from a given date and a given time." if not isinstance(date, _date_class): @@ -1478,7 +1560,6 @@ return cls(date.year, date.month, date.day, time.hour, time.minute, time.second, time.microsecond, time.tzinfo) - combine = classmethod(combine) def timetuple(self): "Return local time tuple compatible with time.localtime()." @@ -1504,7 +1585,7 @@ def date(self): "Return the date part." - return date(self.__year, self.__month, self.__day) + return date(self._year, self._month, self._day) def time(self): "Return the time part, with tzinfo None." @@ -1564,8 +1645,8 @@ def ctime(self): "Format a la ctime()." - t = tmxxx(self.__year, self.__month, self.__day, self.__hour, - self.__minute, self.__second) + t = tmxxx(self._year, self._month, self._day, self._hour, + self._minute, self._second) return t.ctime() def isoformat(self, sep='T'): @@ -1580,10 +1661,10 @@ Optional argument sep specifies the separator between date and time, default 'T'. """ - s = ("%04d-%02d-%02d%c" % (self.__year, self.__month, self.__day, + s = ("%04d-%02d-%02d%c" % (self._year, self._month, self._day, sep) + - _format_time(self.__hour, self.__minute, self.__second, - self.__microsecond)) + _format_time(self._hour, self._minute, self._second, + self._microsecond)) off = self._utcoffset() if off is not None: if off < 0: @@ -1596,13 +1677,13 @@ return s def __repr__(self): - "Convert to formal string, for repr()." - L = [self.__year, self.__month, self.__day, # These are never zero - self.__hour, self.__minute, self.__second, self.__microsecond] + """Convert to formal string, for repr().""" + L = [self._year, self._month, self._day, # These are never zero + self._hour, self._minute, self._second, self._microsecond] if L[-1] == 0: del L[-1] if L[-1] == 0: - del L[-1] + del L[-1] s = ", ".join(map(str, L)) s = "%s(%s)" % ('datetime.' + self.__class__.__name__, s) if self._tzinfo is not None: @@ -1675,7 +1756,7 @@ def __eq__(self, other): if isinstance(other, datetime): - return self.__cmp(other) == 0 + return self._cmp(other) == 0 elif hasattr(other, "timetuple") and not isinstance(other, date): return NotImplemented else: @@ -1683,7 +1764,7 @@ def __ne__(self, other): if isinstance(other, datetime): - return self.__cmp(other) != 0 + return self._cmp(other) != 0 elif hasattr(other, "timetuple") and not isinstance(other, date): return NotImplemented else: @@ -1691,7 +1772,7 @@ def __le__(self, other): if isinstance(other, datetime): - return self.__cmp(other) <= 0 + return self._cmp(other) <= 0 elif hasattr(other, "timetuple") and not isinstance(other, date): return NotImplemented else: @@ -1699,7 +1780,7 @@ def __lt__(self, other): if isinstance(other, datetime): - return self.__cmp(other) < 0 + return self._cmp(other) < 0 elif hasattr(other, "timetuple") and not isinstance(other, date): return NotImplemented else: @@ -1707,7 +1788,7 @@ def __ge__(self, other): if isinstance(other, datetime): - return self.__cmp(other) >= 0 + return self._cmp(other) >= 0 elif hasattr(other, "timetuple") and not isinstance(other, date): return NotImplemented else: @@ -1715,13 +1796,13 @@ def __gt__(self, other): if isinstance(other, datetime): - return self.__cmp(other) > 0 + return self._cmp(other) > 0 elif hasattr(other, "timetuple") and not isinstance(other, date): return NotImplemented else: _cmperror(self, other) - def __cmp(self, other): + def _cmp(self, other): assert isinstance(other, datetime) mytz = self._tzinfo ottz = other._tzinfo @@ -1737,12 +1818,12 @@ base_compare = myoff == otoff if base_compare: - return cmp((self.__year, self.__month, self.__day, - self.__hour, self.__minute, self.__second, - self.__microsecond), - (other.__year, other.__month, other.__day, - other.__hour, other.__minute, other.__second, - other.__microsecond)) + return cmp((self._year, self._month, self._day, + self._hour, self._minute, self._second, + self._microsecond), + (other._year, other._month, other._day, + other._hour, other._minute, other._second, + other._microsecond)) if myoff is None or otoff is None: # XXX Buggy in 2.2.2. raise TypeError("cannot compare naive and aware datetimes") @@ -1756,13 +1837,13 @@ "Add a datetime and a timedelta." if not isinstance(other, timedelta): return NotImplemented - t = tmxxx(self.__year, - self.__month, - self.__day + other.days, - self.__hour, - self.__minute, - self.__second + other.seconds, - self.__microsecond + other.microseconds) + t = tmxxx(self._year, + self._month, + self._day + other.days, + self._hour, + self._minute, + self._second + other.seconds, + self._microsecond + other.microseconds) self._checkOverflow(t.year) result = datetime(t.year, t.month, t.day, t.hour, t.minute, t.second, @@ -1780,11 +1861,11 @@ days1 = self.toordinal() days2 = other.toordinal() - secs1 = self.__second + self.__minute * 60 + self.__hour * 3600 - secs2 = other.__second + other.__minute * 60 + other.__hour * 3600 + secs1 = self._second + self._minute * 60 + self._hour * 3600 + secs2 = other._second + other._minute * 60 + other._hour * 3600 base = timedelta(days1 - days2, secs1 - secs2, - self.__microsecond - other.__microsecond) + self._microsecond - other._microsecond) if self._tzinfo is other._tzinfo: return base myoff = self._utcoffset() @@ -1792,13 +1873,13 @@ if myoff == otoff: return base if myoff is None or otoff is None: - raise TypeError, "cannot mix naive and timezone-aware time" + raise TypeError("cannot mix naive and timezone-aware time") return base + timedelta(minutes = otoff-myoff) def __hash__(self): tzoff = self._utcoffset() if tzoff is None: - return hash(self.__getstate()[0]) + return hash(self._getstate()[0]) days = _ymd2ord(self.year, self.month, self.day) seconds = self.hour * 3600 + (self.minute - tzoff) * 60 + self.second return hash(timedelta(days, seconds, self.microsecond)) @@ -1807,12 +1888,12 @@ __safe_for_unpickling__ = True # For Python 2.2 - def __getstate(self): - yhi, ylo = divmod(self.__year, 256) - us2, us3 = divmod(self.__microsecond, 256) + def _getstate(self): + yhi, ylo = divmod(self._year, 256) + us2, us3 = divmod(self._microsecond, 256) us1, us2 = divmod(us2, 256) - basestate = ("%c" * 10) % (yhi, ylo, self.__month, self.__day, - self.__hour, self.__minute, self.__second, + basestate = ("%c" * 10) % (yhi, ylo, self._month, self._day, + self._hour, self._minute, self._second, us1, us2, us3) if self._tzinfo is None: return (basestate,) @@ -1820,14 +1901,14 @@ return (basestate, self._tzinfo) def __setstate(self, string, tzinfo): - (yhi, ylo, self.__month, self.__day, self.__hour, - self.__minute, self.__second, us1, us2, us3) = map(ord, string) - self.__year = yhi * 256 + ylo - self.__microsecond = (((us1 << 8) | us2) << 8) | us3 + (yhi, ylo, self._month, self._day, self._hour, + self._minute, self._second, us1, us2, us3) = map(ord, string) + self._year = yhi * 256 + ylo + self._microsecond = (((us1 << 8) | us2) << 8) | us3 self._tzinfo = tzinfo def __reduce__(self): - return (self.__class__, self.__getstate()) + return (self.__class__, self._getstate()) datetime.min = datetime(1, 1, 1) @@ -2009,7 +2090,7 @@ Because we know z.d said z was in daylight time (else [5] would have held and we would have stopped then), and we know z.d != z'.d (else [8] would have held -and we we have stopped then), and there are only 2 possible values dst() can +and we have stopped then), and there are only 2 possible values dst() can return in Eastern, it follows that z'.d must be 0 (which it is in the example, but the reasoning doesn't depend on the example -- it depends on there being two possible dst() outcomes, one zero and the other non-zero). Therefore diff --git a/lib_pypy/distributed/socklayer.py b/lib_pypy/distributed/socklayer.py --- a/lib_pypy/distributed/socklayer.py +++ b/lib_pypy/distributed/socklayer.py @@ -2,7 +2,7 @@ import py from socket import socket -XXX needs import adaptation as 'green' is removed from py lib for years +raise ImportError("XXX needs import adaptation as 'green' is removed from py lib for years") from py.impl.green.msgstruct import decodemessage, message from socket import socket, AF_INET, SOCK_STREAM import marshal diff --git a/lib_pypy/numpypy/__init__.py b/lib_pypy/numpypy/__init__.py new file mode 100644 --- /dev/null +++ b/lib_pypy/numpypy/__init__.py @@ -0,0 +1,2 @@ +from _numpypy import * +from .core import * diff --git a/lib_pypy/numpypy/core/__init__.py b/lib_pypy/numpypy/core/__init__.py new file mode 100644 --- /dev/null +++ b/lib_pypy/numpypy/core/__init__.py @@ -0,0 +1,1 @@ +from .fromnumeric import * diff --git a/lib_pypy/numpypy/core/fromnumeric.py b/lib_pypy/numpypy/core/fromnumeric.py new file mode 100644 --- /dev/null +++ b/lib_pypy/numpypy/core/fromnumeric.py @@ -0,0 +1,2430 @@ +###################################################################### +# This is a copy of numpy/core/fromnumeric.py modified for numpypy +###################################################################### +# Each name in __all__ was a function in 'numeric' that is now +# a method in 'numpy'. +# When the corresponding method is added to numpypy BaseArray +# each function should be added as a module function +# at the applevel +# This can be as simple as doing the following +# +# def func(a, ...): +# if not hasattr(a, 'func') +# a = numpypy.array(a) +# return a.func(...) +# +###################################################################### + +import numpypy + +# Module containing non-deprecated functions borrowed from Numeric. +__docformat__ = "restructuredtext en" + +# functions that are now methods +__all__ = ['take', 'reshape', 'choose', 'repeat', 'put', + 'swapaxes', 'transpose', 'sort', 'argsort', 'argmax', 'argmin', + 'searchsorted', 'alen', + 'resize', 'diagonal', 'trace', 'ravel', 'nonzero', 'shape', + 'compress', 'clip', 'sum', 'product', 'prod', 'sometrue', 'alltrue', + 'any', 'all', 'cumsum', 'cumproduct', 'cumprod', 'ptp', 'ndim', + 'rank', 'size', 'around', 'round_', 'mean', 'std', 'var', 'squeeze', + 'amax', 'amin', + ] + +def take(a, indices, axis=None, out=None, mode='raise'): + """ + Take elements from an array along an axis. + + This function does the same thing as "fancy" indexing (indexing arrays + using arrays); however, it can be easier to use if you need elements + along a given axis. + + Parameters + ---------- + a : array_like + The source array. + indices : array_like + The indices of the values to extract. + axis : int, optional + The axis over which to select values. By default, the flattened + input array is used. + out : ndarray, optional + If provided, the result will be placed in this array. It should + be of the appropriate shape and dtype. + mode : {'raise', 'wrap', 'clip'}, optional + Specifies how out-of-bounds indices will behave. + + * 'raise' -- raise an error (default) + * 'wrap' -- wrap around + * 'clip' -- clip to the range + + 'clip' mode means that all indices that are too large are replaced + by the index that addresses the last element along that axis. Note + that this disables indexing with negative numbers. + + Returns + ------- + subarray : ndarray + The returned array has the same type as `a`. + + See Also + -------- + ndarray.take : equivalent method + + Examples + -------- + >>> a = [4, 3, 5, 7, 6, 8] + >>> indices = [0, 1, 4] + >>> np.take(a, indices) + array([4, 3, 6]) + + In this example if `a` is an ndarray, "fancy" indexing can be used. + + >>> a = np.array(a) + >>> a[indices] + array([4, 3, 6]) + + """ + raise NotImplementedError('Waiting on interp level method') + + +# not deprecated --- copy if necessary, view otherwise +def reshape(a, newshape, order='C'): + """ + Gives a new shape to an array without changing its data. + + Parameters + ---------- + a : array_like + Array to be reshaped. + newshape : int or tuple of ints + The new shape should be compatible with the original shape. If + an integer, then the result will be a 1-D array of that length. + One shape dimension can be -1. In this case, the value is inferred + from the length of the array and remaining dimensions. + order : {'C', 'F', 'A'}, optional + Determines whether the array data should be viewed as in C + (row-major) order, FORTRAN (column-major) order, or the C/FORTRAN + order should be preserved. + + Returns + ------- + reshaped_array : ndarray + This will be a new view object if possible; otherwise, it will + be a copy. + + + See Also + -------- + ndarray.reshape : Equivalent method. + + Notes + ----- + + It is not always possible to change the shape of an array without + copying the data. If you want an error to be raise if the data is copied, + you should assign the new shape to the shape attribute of the array:: + + >>> a = np.zeros((10, 2)) + # A transpose make the array non-contiguous + >>> b = a.T + # Taking a view makes it possible to modify the shape without modiying the + # initial object. + >>> c = b.view() + >>> c.shape = (20) + AttributeError: incompatible shape for a non-contiguous array + + + Examples + -------- + >>> a = np.array([[1,2,3], [4,5,6]]) + >>> np.reshape(a, 6) + array([1, 2, 3, 4, 5, 6]) + >>> np.reshape(a, 6, order='F') + array([1, 4, 2, 5, 3, 6]) + + >>> np.reshape(a, (3,-1)) # the unspecified value is inferred to be 2 + array([[1, 2], + [3, 4], + [5, 6]]) + + """ + assert order == 'C' + if not hasattr(a, 'reshape'): + a = numpypy.array(a) + return a.reshape(newshape) + + +def choose(a, choices, out=None, mode='raise'): + """ + Construct an array from an index array and a set of arrays to choose from. + + First of all, if confused or uncertain, definitely look at the Examples - + in its full generality, this function is less simple than it might + seem from the following code description (below ndi = + `numpy.lib.index_tricks`): + + ``np.choose(a,c) == np.array([c[a[I]][I] for I in ndi.ndindex(a.shape)])``. + + But this omits some subtleties. Here is a fully general summary: + + Given an "index" array (`a`) of integers and a sequence of `n` arrays + (`choices`), `a` and each choice array are first broadcast, as necessary, + to arrays of a common shape; calling these *Ba* and *Bchoices[i], i = + 0,...,n-1* we have that, necessarily, ``Ba.shape == Bchoices[i].shape`` + for each `i`. Then, a new array with shape ``Ba.shape`` is created as + follows: + + * if ``mode=raise`` (the default), then, first of all, each element of + `a` (and thus `Ba`) must be in the range `[0, n-1]`; now, suppose that + `i` (in that range) is the value at the `(j0, j1, ..., jm)` position + in `Ba` - then the value at the same position in the new array is the + value in `Bchoices[i]` at that same position; + + * if ``mode=wrap``, values in `a` (and thus `Ba`) may be any (signed) + integer; modular arithmetic is used to map integers outside the range + `[0, n-1]` back into that range; and then the new array is constructed + as above; + + * if ``mode=clip``, values in `a` (and thus `Ba`) may be any (signed) + integer; negative integers are mapped to 0; values greater than `n-1` + are mapped to `n-1`; and then the new array is constructed as above. + + Parameters + ---------- + a : int array + This array must contain integers in `[0, n-1]`, where `n` is the number + of choices, unless ``mode=wrap`` or ``mode=clip``, in which cases any + integers are permissible. + choices : sequence of arrays + Choice arrays. `a` and all of the choices must be broadcastable to the + same shape. If `choices` is itself an array (not recommended), then + its outermost dimension (i.e., the one corresponding to + ``choices.shape[0]``) is taken as defining the "sequence". + out : array, optional + If provided, the result will be inserted into this array. It should + be of the appropriate shape and dtype. + mode : {'raise' (default), 'wrap', 'clip'}, optional + Specifies how indices outside `[0, n-1]` will be treated: + + * 'raise' : an exception is raised + * 'wrap' : value becomes value mod `n` + * 'clip' : values < 0 are mapped to 0, values > n-1 are mapped to n-1 + + Returns + ------- + merged_array : array + The merged result. + + Raises + ------ + ValueError: shape mismatch + If `a` and each choice array are not all broadcastable to the same + shape. + + See Also + -------- + ndarray.choose : equivalent method + + Notes + ----- + To reduce the chance of misinterpretation, even though the following + "abuse" is nominally supported, `choices` should neither be, nor be + thought of as, a single array, i.e., the outermost sequence-like container + should be either a list or a tuple. + + Examples + -------- + + >>> choices = [[0, 1, 2, 3], [10, 11, 12, 13], + ... [20, 21, 22, 23], [30, 31, 32, 33]] + >>> np.choose([2, 3, 1, 0], choices + ... # the first element of the result will be the first element of the + ... # third (2+1) "array" in choices, namely, 20; the second element + ... # will be the second element of the fourth (3+1) choice array, i.e., + ... # 31, etc. + ... ) + array([20, 31, 12, 3]) + >>> np.choose([2, 4, 1, 0], choices, mode='clip') # 4 goes to 3 (4-1) + array([20, 31, 12, 3]) + >>> # because there are 4 choice arrays + >>> np.choose([2, 4, 1, 0], choices, mode='wrap') # 4 goes to (4 mod 4) + array([20, 1, 12, 3]) + >>> # i.e., 0 + + A couple examples illustrating how choose broadcasts: + + >>> a = [[1, 0, 1], [0, 1, 0], [1, 0, 1]] + >>> choices = [-10, 10] + >>> np.choose(a, choices) + array([[ 10, -10, 10], + [-10, 10, -10], + [ 10, -10, 10]]) + + >>> # With thanks to Anne Archibald + >>> a = np.array([0, 1]).reshape((2,1,1)) + >>> c1 = np.array([1, 2, 3]).reshape((1,3,1)) + >>> c2 = np.array([-1, -2, -3, -4, -5]).reshape((1,1,5)) + >>> np.choose(a, (c1, c2)) # result is 2x3x5, res[0,:,:]=c1, res[1,:,:]=c2 + array([[[ 1, 1, 1, 1, 1], + [ 2, 2, 2, 2, 2], + [ 3, 3, 3, 3, 3]], + [[-1, -2, -3, -4, -5], + [-1, -2, -3, -4, -5], + [-1, -2, -3, -4, -5]]]) + + """ + raise NotImplementedError('Waiting on interp level method') + + +def repeat(a, repeats, axis=None): + """ + Repeat elements of an array. + + Parameters + ---------- + a : array_like + Input array. + repeats : {int, array of ints} + The number of repetitions for each element. `repeats` is broadcasted + to fit the shape of the given axis. + axis : int, optional + The axis along which to repeat values. By default, use the + flattened input array, and return a flat output array. + + Returns + ------- + repeated_array : ndarray + Output array which has the same shape as `a`, except along + the given axis. + + See Also + -------- + tile : Tile an array. + + Examples + -------- + >>> x = np.array([[1,2],[3,4]]) + >>> np.repeat(x, 2) + array([1, 1, 2, 2, 3, 3, 4, 4]) + >>> np.repeat(x, 3, axis=1) + array([[1, 1, 1, 2, 2, 2], + [3, 3, 3, 4, 4, 4]]) + >>> np.repeat(x, [1, 2], axis=0) + array([[1, 2], + [3, 4], + [3, 4]]) + + """ + raise NotImplementedError('Waiting on interp level method') + + +def put(a, ind, v, mode='raise'): + """ + Replaces specified elements of an array with given values. + + The indexing works on the flattened target array. `put` is roughly + equivalent to: + + :: + + a.flat[ind] = v + + Parameters + ---------- + a : ndarray + Target array. + ind : array_like + Target indices, interpreted as integers. + v : array_like + Values to place in `a` at target indices. If `v` is shorter than + `ind` it will be repeated as necessary. + mode : {'raise', 'wrap', 'clip'}, optional + Specifies how out-of-bounds indices will behave. + + * 'raise' -- raise an error (default) + * 'wrap' -- wrap around + * 'clip' -- clip to the range + + 'clip' mode means that all indices that are too large are replaced + by the index that addresses the last element along that axis. Note + that this disables indexing with negative numbers. + + See Also + -------- + putmask, place + + Examples + -------- + >>> a = np.arange(5) + >>> np.put(a, [0, 2], [-44, -55]) + >>> a + array([-44, 1, -55, 3, 4]) + + >>> a = np.arange(5) + >>> np.put(a, 22, -5, mode='clip') + >>> a + array([ 0, 1, 2, 3, -5]) + + """ + raise NotImplementedError('Waiting on interp level method') + + +def swapaxes(a, axis1, axis2): + """ + Interchange two axes of an array. + + Parameters + ---------- + a : array_like + Input array. + axis1 : int + First axis. + axis2 : int + Second axis. + + Returns + ------- + a_swapped : ndarray + If `a` is an ndarray, then a view of `a` is returned; otherwise + a new array is created. + + Examples + -------- + >>> x = np.array([[1,2,3]]) + >>> np.swapaxes(x,0,1) + array([[1], + [2], + [3]]) + + >>> x = np.array([[[0,1],[2,3]],[[4,5],[6,7]]]) + >>> x + array([[[0, 1], + [2, 3]], + [[4, 5], + [6, 7]]]) + + >>> np.swapaxes(x,0,2) + array([[[0, 4], + [2, 6]], + [[1, 5], + [3, 7]]]) + + """ + raise NotImplementedError('Waiting on interp level method') + + +def transpose(a, axes=None): + """ + Permute the dimensions of an array. + + Parameters + ---------- + a : array_like + Input array. + axes : list of ints, optional + By default, reverse the dimensions, otherwise permute the axes + according to the values given. + + Returns + ------- + p : ndarray + `a` with its axes permuted. A view is returned whenever + possible. + + See Also + -------- + rollaxis + + Examples + -------- + >>> x = np.arange(4).reshape((2,2)) + >>> x + array([[0, 1], + [2, 3]]) + + >>> np.transpose(x) + array([[0, 2], + [1, 3]]) + + >>> x = np.ones((1, 2, 3)) + >>> np.transpose(x, (1, 0, 2)).shape + (2, 1, 3) + + """ + if axes is not None: + raise NotImplementedError('No "axes" arg yet.') + if not hasattr(a, 'T'): + a = numpypy.array(a) + return a.T + +def sort(a, axis=-1, kind='quicksort', order=None): + """ + Return a sorted copy of an array. + + Parameters + ---------- + a : array_like + Array to be sorted. + axis : int or None, optional + Axis along which to sort. If None, the array is flattened before + sorting. The default is -1, which sorts along the last axis. + kind : {'quicksort', 'mergesort', 'heapsort'}, optional + Sorting algorithm. Default is 'quicksort'. + order : list, optional + When `a` is a structured array, this argument specifies which fields + to compare first, second, and so on. This list does not need to + include all of the fields. + + Returns + ------- + sorted_array : ndarray + Array of the same type and shape as `a`. + + See Also + -------- + ndarray.sort : Method to sort an array in-place. + argsort : Indirect sort. + lexsort : Indirect stable sort on multiple keys. + searchsorted : Find elements in a sorted array. + + Notes + ----- + The various sorting algorithms are characterized by their average speed, + worst case performance, work space size, and whether they are stable. A + stable sort keeps items with the same key in the same relative + order. The three available algorithms have the following + properties: + + =========== ======= ============= ============ ======= + kind speed worst case work space stable + =========== ======= ============= ============ ======= + 'quicksort' 1 O(n^2) 0 no + 'mergesort' 2 O(n*log(n)) ~n/2 yes + 'heapsort' 3 O(n*log(n)) 0 no + =========== ======= ============= ============ ======= + + All the sort algorithms make temporary copies of the data when + sorting along any but the last axis. Consequently, sorting along + the last axis is faster and uses less space than sorting along + any other axis. + + The sort order for complex numbers is lexicographic. If both the real + and imaginary parts are non-nan then the order is determined by the + real parts except when they are equal, in which case the order is + determined by the imaginary parts. + + Previous to numpy 1.4.0 sorting real and complex arrays containing nan + values led to undefined behaviour. In numpy versions >= 1.4.0 nan + values are sorted to the end. The extended sort order is: + + * Real: [R, nan] + * Complex: [R + Rj, R + nanj, nan + Rj, nan + nanj] + + where R is a non-nan real value. Complex values with the same nan + placements are sorted according to the non-nan part if it exists. + Non-nan values are sorted as before. + + Examples + -------- + >>> a = np.array([[1,4],[3,1]]) + >>> np.sort(a) # sort along the last axis + array([[1, 4], + [1, 3]]) + >>> np.sort(a, axis=None) # sort the flattened array + array([1, 1, 3, 4]) + >>> np.sort(a, axis=0) # sort along the first axis + array([[1, 1], + [3, 4]]) + + Use the `order` keyword to specify a field to use when sorting a + structured array: + + >>> dtype = [('name', 'S10'), ('height', float), ('age', int)] + >>> values = [('Arthur', 1.8, 41), ('Lancelot', 1.9, 38), + ... ('Galahad', 1.7, 38)] + >>> a = np.array(values, dtype=dtype) # create a structured array + >>> np.sort(a, order='height') # doctest: +SKIP + array([('Galahad', 1.7, 38), ('Arthur', 1.8, 41), + ('Lancelot', 1.8999999999999999, 38)], + dtype=[('name', '|S10'), ('height', '<f8'), ('age', '<i4')]) + + Sort by age, then height if ages are equal: + + >>> np.sort(a, order=['age', 'height']) # doctest: +SKIP + array([('Galahad', 1.7, 38), ('Lancelot', 1.8999999999999999, 38), + ('Arthur', 1.8, 41)], + dtype=[('name', '|S10'), ('height', '<f8'), ('age', '<i4')]) + + """ + raise NotImplementedError('Waiting on interp level method') + + +def argsort(a, axis=-1, kind='quicksort', order=None): + """ + Returns the indices that would sort an array. + + Perform an indirect sort along the given axis using the algorithm specified + by the `kind` keyword. It returns an array of indices of the same shape as + `a` that index data along the given axis in sorted order. + + Parameters + ---------- + a : array_like + Array to sort. + axis : int or None, optional + Axis along which to sort. The default is -1 (the last axis). If None, + the flattened array is used. + kind : {'quicksort', 'mergesort', 'heapsort'}, optional + Sorting algorithm. + order : list, optional + When `a` is an array with fields defined, this argument specifies + which fields to compare first, second, etc. Not all fields need be + specified. + + Returns + ------- + index_array : ndarray, int + Array of indices that sort `a` along the specified axis. + In other words, ``a[index_array]`` yields a sorted `a`. + + See Also + -------- + sort : Describes sorting algorithms used. + lexsort : Indirect stable sort with multiple keys. + ndarray.sort : Inplace sort. + + Notes + ----- + See `sort` for notes on the different sorting algorithms. + + As of NumPy 1.4.0 `argsort` works with real/complex arrays containing + nan values. The enhanced sort order is documented in `sort`. + + Examples + -------- + One dimensional array: + + >>> x = np.array([3, 1, 2]) + >>> np.argsort(x) + array([1, 2, 0]) + + Two-dimensional array: + + >>> x = np.array([[0, 3], [2, 2]]) + >>> x + array([[0, 3], + [2, 2]]) + + >>> np.argsort(x, axis=0) + array([[0, 1], + [1, 0]]) + + >>> np.argsort(x, axis=1) + array([[0, 1], + [0, 1]]) + + Sorting with keys: + + >>> x = np.array([(1, 0), (0, 1)], dtype=[('x', '<i4'), ('y', '<i4')]) + >>> x + array([(1, 0), (0, 1)], + dtype=[('x', '<i4'), ('y', '<i4')]) + + >>> np.argsort(x, order=('x','y')) + array([1, 0]) + + >>> np.argsort(x, order=('y','x')) + array([0, 1]) + + """ + raise NotImplementedError('Waiting on interp level method') + + +def argmax(a, axis=None): + """ + Indices of the maximum values along an axis. + + Parameters + ---------- + a : array_like + Input array. + axis : int, optional + By default, the index is into the flattened array, otherwise + along the specified axis. + + Returns + ------- + index_array : ndarray of ints + Array of indices into the array. It has the same shape as `a.shape` + with the dimension along `axis` removed. + + See Also + -------- + ndarray.argmax, argmin + amax : The maximum value along a given axis. + unravel_index : Convert a flat index into an index tuple. + + Notes + ----- + In case of multiple occurrences of the maximum values, the indices + corresponding to the first occurrence are returned. + + Examples + -------- + >>> a = np.arange(6).reshape(2,3) + >>> a + array([[0, 1, 2], + [3, 4, 5]]) + >>> np.argmax(a) + 5 + >>> np.argmax(a, axis=0) + array([1, 1, 1]) + >>> np.argmax(a, axis=1) + array([2, 2]) + + >>> b = np.arange(6) + >>> b[1] = 5 + >>> b + array([0, 5, 2, 3, 4, 5]) + >>> np.argmax(b) # Only the first occurrence is returned. + 1 + + """ + assert axis is None + if not hasattr(a, 'argmax'): + a = numpypy.array(a) + return a.argmax() + + +def argmin(a, axis=None): + """ + Return the indices of the minimum values along an axis. + + See Also + -------- + argmax : Similar function. Please refer to `numpy.argmax` for detailed + documentation. + + """ + assert axis is None + if not hasattr(a, 'argmin'): + a = numpypy.array(a) + return a.argmin() + + +def searchsorted(a, v, side='left'): + """ + Find indices where elements should be inserted to maintain order. + + Find the indices into a sorted array `a` such that, if the corresponding + elements in `v` were inserted before the indices, the order of `a` would + be preserved. + + Parameters + ---------- + a : 1-D array_like + Input array, sorted in ascending order. + v : array_like + Values to insert into `a`. + side : {'left', 'right'}, optional + If 'left', the index of the first suitable location found is given. If + 'right', return the last such index. If there is no suitable + index, return either 0 or N (where N is the length of `a`). + + Returns + ------- + indices : array of ints + Array of insertion points with the same shape as `v`. + + See Also + -------- + sort : Return a sorted copy of an array. + histogram : Produce histogram from 1-D data. + + Notes + ----- + Binary search is used to find the required insertion points. + + As of Numpy 1.4.0 `searchsorted` works with real/complex arrays containing + `nan` values. The enhanced sort order is documented in `sort`. + + Examples + -------- + >>> np.searchsorted([1,2,3,4,5], 3) + 2 + >>> np.searchsorted([1,2,3,4,5], 3, side='right') + 3 + >>> np.searchsorted([1,2,3,4,5], [-10, 10, 2, 3]) + array([0, 5, 1, 2]) + + """ + raise NotImplementedError('Waiting on interp level method') + + +def resize(a, new_shape): + """ + Return a new array with the specified shape. + + If the new array is larger than the original array, then the new + array is filled with repeated copies of `a`. Note that this behavior + is different from a.resize(new_shape) which fills with zeros instead + of repeated copies of `a`. + + Parameters + ---------- + a : array_like + Array to be resized. + + new_shape : int or tuple of int + Shape of resized array. + + Returns + ------- + reshaped_array : ndarray + The new array is formed from the data in the old array, repeated + if necessary to fill out the required number of elements. The + data are repeated in the order that they are stored in memory. + + See Also + -------- + ndarray.resize : resize an array in-place. + + Examples + -------- + >>> a=np.array([[0,1],[2,3]]) + >>> np.resize(a,(1,4)) + array([[0, 1, 2, 3]]) + >>> np.resize(a,(2,4)) + array([[0, 1, 2, 3], + [0, 1, 2, 3]]) + + """ + raise NotImplementedError('Waiting on interp level method') + + +def squeeze(a): + """ + Remove single-dimensional entries from the shape of an array. + + Parameters + ---------- + a : array_like + Input data. + + Returns + ------- + squeezed : ndarray + The input array, but with with all dimensions of length 1 + removed. Whenever possible, a view on `a` is returned. + + Examples + -------- + >>> x = np.array([[[0], [1], [2]]]) + >>> x.shape + (1, 3, 1) + >>> np.squeeze(x).shape + (3,) + + """ + raise NotImplementedError('Waiting on interp level method') + + +def diagonal(a, offset=0, axis1=0, axis2=1): + """ + Return specified diagonals. + + If `a` is 2-D, returns the diagonal of `a` with the given offset, + i.e., the collection of elements of the form ``a[i, i+offset]``. If + `a` has more than two dimensions, then the axes specified by `axis1` + and `axis2` are used to determine the 2-D sub-array whose diagonal is + returned. The shape of the resulting array can be determined by + removing `axis1` and `axis2` and appending an index to the right equal + to the size of the resulting diagonals. + + Parameters + ---------- + a : array_like + Array from which the diagonals are taken. + offset : int, optional + Offset of the diagonal from the main diagonal. Can be positive or + negative. Defaults to main diagonal (0). + axis1 : int, optional + Axis to be used as the first axis of the 2-D sub-arrays from which + the diagonals should be taken. Defaults to first axis (0). + axis2 : int, optional + Axis to be used as the second axis of the 2-D sub-arrays from + which the diagonals should be taken. Defaults to second axis (1). + + Returns + ------- + array_of_diagonals : ndarray + If `a` is 2-D, a 1-D array containing the diagonal is returned. + If the dimension of `a` is larger, then an array of diagonals is + returned, "packed" from left-most dimension to right-most (e.g., + if `a` is 3-D, then the diagonals are "packed" along rows). + + Raises + ------ + ValueError + If the dimension of `a` is less than 2. + + See Also + -------- + diag : MATLAB work-a-like for 1-D and 2-D arrays. + diagflat : Create diagonal arrays. + trace : Sum along diagonals. + + Examples + -------- + >>> a = np.arange(4).reshape(2,2) + >>> a + array([[0, 1], + [2, 3]]) + >>> a.diagonal() + array([0, 3]) + >>> a.diagonal(1) + array([1]) + + A 3-D example: + + >>> a = np.arange(8).reshape(2,2,2); a + array([[[0, 1], + [2, 3]], + [[4, 5], + [6, 7]]]) + >>> a.diagonal(0, # Main diagonals of two arrays created by skipping + ... 0, # across the outer(left)-most axis last and + ... 1) # the "middle" (row) axis first. + array([[0, 6], + [1, 7]]) + + The sub-arrays whose main diagonals we just obtained; note that each + corresponds to fixing the right-most (column) axis, and that the + diagonals are "packed" in rows. + + >>> a[:,:,0] # main diagonal is [0 6] + array([[0, 2], + [4, 6]]) + >>> a[:,:,1] # main diagonal is [1 7] + array([[1, 3], + [5, 7]]) + + """ + raise NotImplementedError('Waiting on interp level method') + + +def trace(a, offset=0, axis1=0, axis2=1, dtype=None, out=None): + """ + Return the sum along diagonals of the array. + + If `a` is 2-D, the sum along its diagonal with the given offset + is returned, i.e., the sum of elements ``a[i,i+offset]`` for all i. + + If `a` has more than two dimensions, then the axes specified by axis1 and + axis2 are used to determine the 2-D sub-arrays whose traces are returned. + The shape of the resulting array is the same as that of `a` with `axis1` + and `axis2` removed. + + Parameters + ---------- + a : array_like + Input array, from which the diagonals are taken. + offset : int, optional + Offset of the diagonal from the main diagonal. Can be both positive + and negative. Defaults to 0. + axis1, axis2 : int, optional + Axes to be used as the first and second axis of the 2-D sub-arrays + from which the diagonals should be taken. Defaults are the first two + axes of `a`. + dtype : dtype, optional + Determines the data-type of the returned array and of the accumulator + where the elements are summed. If dtype has the value None and `a` is + of integer type of precision less than the default integer + precision, then the default integer precision is used. Otherwise, + the precision is the same as that of `a`. + out : ndarray, optional + Array into which the output is placed. Its type is preserved and + it must be of the right shape to hold the output. + + Returns + ------- + sum_along_diagonals : ndarray + If `a` is 2-D, the sum along the diagonal is returned. If `a` has + larger dimensions, then an array of sums along diagonals is returned. + + See Also + -------- + diag, diagonal, diagflat + + Examples + -------- + >>> np.trace(np.eye(3)) + 3.0 + >>> a = np.arange(8).reshape((2,2,2)) + >>> np.trace(a) + array([6, 8]) + + >>> a = np.arange(24).reshape((2,2,2,3)) + >>> np.trace(a).shape + (2, 3) + + """ + raise NotImplementedError('Waiting on interp level method') + +def ravel(a, order='C'): + """ + Return a flattened array. + + A 1-D array, containing the elements of the input, is returned. A copy is + made only if needed. + + Parameters + ---------- + a : array_like + Input array. The elements in ``a`` are read in the order specified by + `order`, and packed as a 1-D array. + order : {'C','F', 'A', 'K'}, optional + The elements of ``a`` are read in this order. 'C' means to view + the elements in C (row-major) order. 'F' means to view the elements + in Fortran (column-major) order. 'A' means to view the elements + in 'F' order if a is Fortran contiguous, 'C' order otherwise. + 'K' means to view the elements in the order they occur in memory, + except for reversing the data when strides are negative. + By default, 'C' order is used. + + Returns + ------- + 1d_array : ndarray + Output of the same dtype as `a`, and of shape ``(a.size(),)``. + + See Also + -------- + ndarray.flat : 1-D iterator over an array. + ndarray.flatten : 1-D array copy of the elements of an array + in row-major order. + + Notes + ----- + In row-major order, the row index varies the slowest, and the column + index the quickest. This can be generalized to multiple dimensions, + where row-major order implies that the index along the first axis + varies slowest, and the index along the last quickest. The opposite holds + for Fortran-, or column-major, mode. + + Examples + -------- + It is equivalent to ``reshape(-1, order=order)``. + + >>> x = np.array([[1, 2, 3], [4, 5, 6]]) + >>> print np.ravel(x) + [1 2 3 4 5 6] + + >>> print x.reshape(-1) + [1 2 3 4 5 6] + + >>> print np.ravel(x, order='F') + [1 4 2 5 3 6] + + When ``order`` is 'A', it will preserve the array's 'C' or 'F' ordering: + + >>> print np.ravel(x.T) + [1 4 2 5 3 6] + >>> print np.ravel(x.T, order='A') + [1 2 3 4 5 6] + + When ``order`` is 'K', it will preserve orderings that are neither 'C' + nor 'F', but won't reverse axes: + + >>> a = np.arange(3)[::-1]; a + array([2, 1, 0]) + >>> a.ravel(order='C') + array([2, 1, 0]) + >>> a.ravel(order='K') + array([2, 1, 0]) + + >>> a = np.arange(12).reshape(2,3,2).swapaxes(1,2); a + array([[[ 0, 2, 4], + [ 1, 3, 5]], + [[ 6, 8, 10], + [ 7, 9, 11]]]) + >>> a.ravel(order='C') + array([ 0, 2, 4, 1, 3, 5, 6, 8, 10, 7, 9, 11]) + >>> a.ravel(order='K') + array([ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]) + + """ + raise NotImplementedError('Waiting on interp level method') + + +def nonzero(a): + """ + Return the indices of the elements that are non-zero. + + Returns a tuple of arrays, one for each dimension of `a`, containing + the indices of the non-zero elements in that dimension. The + corresponding non-zero values can be obtained with:: + + a[nonzero(a)] + + To group the indices by element, rather than dimension, use:: + + transpose(nonzero(a)) + + The result of this is always a 2-D array, with a row for + each non-zero element. + + Parameters + ---------- + a : array_like + Input array. + + Returns + ------- + tuple_of_arrays : tuple + Indices of elements that are non-zero. + + See Also + -------- + flatnonzero : + Return indices that are non-zero in the flattened version of the input + array. + ndarray.nonzero : + Equivalent ndarray method. + count_nonzero : + Counts the number of non-zero elements in the input array. + + Examples + -------- + >>> x = np.eye(3) + >>> x + array([[ 1., 0., 0.], + [ 0., 1., 0.], + [ 0., 0., 1.]]) + >>> np.nonzero(x) + (array([0, 1, 2]), array([0, 1, 2])) + + >>> x[np.nonzero(x)] + array([ 1., 1., 1.]) + >>> np.transpose(np.nonzero(x)) + array([[0, 0], + [1, 1], + [2, 2]]) + + A common use for ``nonzero`` is to find the indices of an array, where + a condition is True. Given an array `a`, the condition `a` > 3 is a + boolean array and since False is interpreted as 0, np.nonzero(a > 3) + yields the indices of the `a` where the condition is true. + + >>> a = np.array([[1,2,3],[4,5,6],[7,8,9]]) + >>> a > 3 + array([[False, False, False], + [ True, True, True], + [ True, True, True]], dtype=bool) + >>> np.nonzero(a > 3) + (array([1, 1, 1, 2, 2, 2]), array([0, 1, 2, 0, 1, 2])) + + The ``nonzero`` method of the boolean array can also be called. + + >>> (a > 3).nonzero() + (array([1, 1, 1, 2, 2, 2]), array([0, 1, 2, 0, 1, 2])) + + """ + raise NotImplementedError('Waiting on interp level method') + + +def shape(a): + """ + Return the shape of an array. + + Parameters + ---------- + a : array_like + Input array. + + Returns + ------- + shape : tuple of ints + The elements of the shape tuple give the lengths of the + corresponding array dimensions. + + See Also + -------- + alen + ndarray.shape : Equivalent array method. + + Examples + -------- + >>> np.shape(np.eye(3)) + (3, 3) + >>> np.shape([[1, 2]]) + (1, 2) + >>> np.shape([0]) + (1,) + >>> np.shape(0) + () + + >>> a = np.array([(1, 2), (3, 4)], dtype=[('x', 'i4'), ('y', 'i4')]) + >>> np.shape(a) + (2,) + >>> a.shape + (2,) + + """ + if not hasattr(a, 'shape'): + a = numpypy.array(a) + return a.shape + + +def compress(condition, a, axis=None, out=None): + """ + Return selected slices of an array along given axis. + + When working along a given axis, a slice along that axis is returned in + `output` for each index where `condition` evaluates to True. When + working on a 1-D array, `compress` is equivalent to `extract`. + + Parameters + ---------- + condition : 1-D array of bools + Array that selects which entries to return. If len(condition) + is less than the size of `a` along the given axis, then output is + truncated to the length of the condition array. + a : array_like + Array from which to extract a part. + axis : int, optional + Axis along which to take slices. If None (default), work on the + flattened array. + out : ndarray, optional + Output array. Its type is preserved and it must be of the right + shape to hold the output. + + Returns + ------- + compressed_array : ndarray + A copy of `a` without the slices along axis for which `condition` + is false. + + See Also + -------- + take, choose, diag, diagonal, select + ndarray.compress : Equivalent method. + numpy.doc.ufuncs : Section "Output arguments" + + Examples + -------- + >>> a = np.array([[1, 2], [3, 4], [5, 6]]) + >>> a + array([[1, 2], + [3, 4], + [5, 6]]) + >>> np.compress([0, 1], a, axis=0) + array([[3, 4]]) + >>> np.compress([False, True, True], a, axis=0) + array([[3, 4], + [5, 6]]) + >>> np.compress([False, True], a, axis=1) + array([[2], + [4], + [6]]) + + Working on the flattened array does not return slices along an axis but + selects elements. + + >>> np.compress([False, True], a) + array([2]) + + """ + raise NotImplementedError('Waiting on interp level method') + + +def clip(a, a_min, a_max, out=None): + """ + Clip (limit) the values in an array. + + Given an interval, values outside the interval are clipped to + the interval edges. For example, if an interval of ``[0, 1]`` + is specified, values smaller than 0 become 0, and values larger + than 1 become 1. + + Parameters + ---------- + a : array_like + Array containing elements to clip. + a_min : scalar or array_like + Minimum value. + a_max : scalar or array_like + Maximum value. If `a_min` or `a_max` are array_like, then they will + be broadcasted to the shape of `a`. + out : ndarray, optional + The results will be placed in this array. It may be the input + array for in-place clipping. `out` must be of the right shape + to hold the output. Its type is preserved. + + Returns + ------- + clipped_array : ndarray + An array with the elements of `a`, but where values + < `a_min` are replaced with `a_min`, and those > `a_max` + with `a_max`. + + See Also + -------- + numpy.doc.ufuncs : Section "Output arguments" + + Examples + -------- + >>> a = np.arange(10) + >>> np.clip(a, 1, 8) + array([1, 1, 2, 3, 4, 5, 6, 7, 8, 8]) + >>> a + array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) + >>> np.clip(a, 3, 6, out=a) + array([3, 3, 3, 3, 4, 5, 6, 6, 6, 6]) + >>> a = np.arange(10) + >>> a + array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) + >>> np.clip(a, [3,4,1,1,1,4,4,4,4,4], 8) + array([3, 4, 2, 3, 4, 5, 6, 7, 8, 8]) + + """ + raise NotImplementedError('Waiting on interp level method') + + +def sum(a, axis=None, dtype=None, out=None): + """ + Sum of array elements over a given axis. + + Parameters + ---------- + a : array_like + Elements to sum. + axis : integer, optional + Axis over which the sum is taken. By default `axis` is None, + and all elements are summed. + dtype : dtype, optional + The type of the returned array and of the accumulator in which + the elements are summed. By default, the dtype of `a` is used. + An exception is when `a` has an integer type with less precision + than the default platform integer. In that case, the default + platform integer is used instead. + out : ndarray, optional + Array into which the output is placed. By default, a new array is + created. If `out` is given, it must be of the appropriate shape + (the shape of `a` with `axis` removed, i.e., + ``numpy.delete(a.shape, axis)``). Its type is preserved. See + `doc.ufuncs` (Section "Output arguments") for more details. + + Returns + ------- + sum_along_axis : ndarray + An array with the same shape as `a`, with the specified + axis removed. If `a` is a 0-d array, or if `axis` is None, a scalar + is returned. If an output array is specified, a reference to + `out` is returned. + + See Also + -------- + ndarray.sum : Equivalent method. + + cumsum : Cumulative sum of array elements. + + trapz : Integration of array values using the composite trapezoidal rule. + + mean, average + + Notes + ----- + Arithmetic is modular when using integer types, and no error is + raised on overflow. + + Examples + -------- + >>> np.sum([0.5, 1.5]) + 2.0 + >>> np.sum([0.5, 0.7, 0.2, 1.5], dtype=np.int32) + 1 + >>> np.sum([[0, 1], [0, 5]]) + 6 + >>> np.sum([[0, 1], [0, 5]], axis=0) + array([0, 6]) + >>> np.sum([[0, 1], [0, 5]], axis=1) + array([1, 5]) + + If the accumulator is too small, overflow occurs: + + >>> np.ones(128, dtype=np.int8).sum(dtype=np.int8) + -128 + + """ + assert dtype is None + assert out is None + if not hasattr(a, "sum"): + a = numpypy.array(a) + return a.sum(axis=axis) + + +def product (a, axis=None, dtype=None, out=None): + """ + Return the product of array elements over a given axis. + + See Also + -------- + prod : equivalent function; see for details. + + """ + raise NotImplementedError('Waiting on interp level method') + + +def sometrue(a, axis=None, out=None): + """ + Check whether some values are true. + + Refer to `any` for full documentation. + + See Also + -------- + any : equivalent function + + """ + assert axis is None + assert out is None + if not hasattr(a, 'any'): + a = numpypy.array(a) + return a.any() + + +def alltrue (a, axis=None, out=None): + """ + Check if all elements of input array are true. + + See Also + -------- + numpy.all : Equivalent function; see for details. + + """ + assert axis is None + assert out is None + if not hasattr(a, 'all'): + a = numpypy.array(a) + return a.all() + +def any(a,axis=None, out=None): + """ + Test whether any array element along a given axis evaluates to True. + + Returns single boolean unless `axis` is not ``None`` + + Parameters + ---------- + a : array_like + Input array or object that can be converted to an array. + axis : int, optional + Axis along which a logical OR is performed. The default + (`axis` = `None`) is to perform a logical OR over a flattened + input array. `axis` may be negative, in which case it counts + from the last to the first axis. + out : ndarray, optional + Alternate output array in which to place the result. It must have + the same shape as the expected output and its type is preserved + (e.g., if it is of type float, then it will remain so, returning + 1.0 for True and 0.0 for False, regardless of the type of `a`). + See `doc.ufuncs` (Section "Output arguments") for details. + + Returns + ------- + any : bool or ndarray + A new boolean or `ndarray` is returned unless `out` is specified, + in which case a reference to `out` is returned. + + See Also + -------- + ndarray.any : equivalent method + + all : Test whether all elements along a given axis evaluate to True. + + Notes + ----- + Not a Number (NaN), positive infinity and negative infinity evaluate + to `True` because these are not equal to zero. + + Examples + -------- + >>> np.any([[True, False], [True, True]]) + True + + >>> np.any([[True, False], [False, False]], axis=0) + array([ True, False], dtype=bool) + + >>> np.any([-1, 0, 5]) + True + + >>> np.any(np.nan) + True + + >>> o=np.array([False]) + >>> z=np.any([-1, 4, 5], out=o) + >>> z, o + (array([ True], dtype=bool), array([ True], dtype=bool)) + >>> # Check now that z is a reference to o + >>> z is o + True + >>> id(z), id(o) # identity of z and o # doctest: +SKIP + (191614240, 191614240) + + """ + assert axis is None + assert out is None + if not hasattr(a, 'any'): + a = numpypy.array(a) + return a.any() + + +def all(a,axis=None, out=None): + """ + Test whether all array elements along a given axis evaluate to True. + + Parameters + ---------- + a : array_like + Input array or object that can be converted to an array. + axis : int, optional + Axis along which a logical AND is performed. + The default (`axis` = `None`) is to perform a logical AND + over a flattened input array. `axis` may be negative, in which + case it counts from the last to the first axis. + out : ndarray, optional + Alternate output array in which to place the result. + It must have the same shape as the expected output and its + type is preserved (e.g., if ``dtype(out)`` is float, the result + will consist of 0.0's and 1.0's). See `doc.ufuncs` (Section + "Output arguments") for more details. + + Returns + ------- + all : ndarray, bool + A new boolean or array is returned unless `out` is specified, + in which case a reference to `out` is returned. + + See Also + -------- + ndarray.all : equivalent method + + any : Test whether any element along a given axis evaluates to True. + + Notes + ----- + Not a Number (NaN), positive infinity and negative infinity + evaluate to `True` because these are not equal to zero. + + Examples + -------- + >>> np.all([[True,False],[True,True]]) + False + + >>> np.all([[True,False],[True,True]], axis=0) + array([ True, False], dtype=bool) + + >>> np.all([-1, 4, 5]) + True + + >>> np.all([1.0, np.nan]) + True + + >>> o=np.array([False]) + >>> z=np.all([-1, 4, 5], out=o) + >>> id(z), id(o), z # doctest: +SKIP + (28293632, 28293632, array([ True], dtype=bool)) + + """ + assert axis is None + assert out is None + if not hasattr(a, 'all'): + a = numpypy.array(a) + return a.all() + + +def cumsum (a, axis=None, dtype=None, out=None): + """ + Return the cumulative sum of the elements along a given axis. + + Parameters + ---------- + a : array_like + Input array. + axis : int, optional + Axis along which the cumulative sum is computed. The default + (None) is to compute the cumsum over the flattened array. + dtype : dtype, optional + Type of the returned array and of the accumulator in which the + elements are summed. If `dtype` is not specified, it defaults + to the dtype of `a`, unless `a` has an integer dtype with a + precision less than that of the default platform integer. In + that case, the default platform integer is used. + out : ndarray, optional + Alternative output array in which to place the result. It must + have the same shape and buffer length as the expected output + but the type will be cast if necessary. See `doc.ufuncs` + (Section "Output arguments") for more details. + + Returns + ------- + cumsum_along_axis : ndarray. + A new array holding the result is returned unless `out` is + specified, in which case a reference to `out` is returned. The + result has the same size as `a`, and the same shape as `a` if + `axis` is not None or `a` is a 1-d array. + + + See Also + -------- + sum : Sum array elements. + + trapz : Integration of array values using the composite trapezoidal rule. + + Notes + ----- + Arithmetic is modular when using integer types, and no error is + raised on overflow. + + Examples + -------- + >>> a = np.array([[1,2,3], [4,5,6]]) + >>> a + array([[1, 2, 3], + [4, 5, 6]]) + >>> np.cumsum(a) + array([ 1, 3, 6, 10, 15, 21]) + >>> np.cumsum(a, dtype=float) # specifies type of output value(s) + array([ 1., 3., 6., 10., 15., 21.]) + + >>> np.cumsum(a,axis=0) # sum over rows for each of the 3 columns + array([[1, 2, 3], + [5, 7, 9]]) + >>> np.cumsum(a,axis=1) # sum over columns for each of the 2 rows + array([[ 1, 3, 6], + [ 4, 9, 15]]) + + """ + raise NotImplementedError('Waiting on interp level method') + + +def cumproduct(a, axis=None, dtype=None, out=None): + """ + Return the cumulative product over the given axis. + + + See Also + -------- + cumprod : equivalent function; see for details. + + """ + raise NotImplementedError('Waiting on interp level method') + + +def ptp(a, axis=None, out=None): + """ + Range of values (maximum - minimum) along an axis. + + The name of the function comes from the acronym for 'peak to peak'. + + Parameters + ---------- + a : array_like + Input values. + axis : int, optional + Axis along which to find the peaks. By default, flatten the + array. + out : array_like + Alternative output array in which to place the result. It must + have the same shape and buffer length as the expected output, + but the type of the output values will be cast if necessary. + + Returns + ------- + ptp : ndarray + A new array holding the result, unless `out` was + specified, in which case a reference to `out` is returned. + + Examples + -------- + >>> x = np.arange(4).reshape((2,2)) + >>> x + array([[0, 1], + [2, 3]]) + + >>> np.ptp(x, axis=0) + array([2, 2]) + + >>> np.ptp(x, axis=1) + array([1, 1]) + + """ + raise NotImplementedError('Waiting on interp level method') + + +def amax(a, axis=None, out=None): + """ + Return the maximum of an array or maximum along an axis. + + Parameters + ---------- + a : array_like + Input data. + axis : int, optional + Axis along which to operate. By default flattened input is used. + out : ndarray, optional + Alternate output array in which to place the result. Must be of + the same shape and buffer length as the expected output. See + `doc.ufuncs` (Section "Output arguments") for more details. + + Returns + ------- + amax : ndarray or scalar + Maximum of `a`. If `axis` is None, the result is a scalar value. + If `axis` is given, the result is an array of dimension + ``a.ndim - 1``. + + See Also + -------- + nanmax : NaN values are ignored instead of being propagated. + fmax : same behavior as the C99 fmax function. + argmax : indices of the maximum values. + + Notes + ----- + NaN values are propagated, that is if at least one item is NaN, the + corresponding max value will be NaN as well. To ignore NaN values + (MATLAB behavior), please use nanmax. + + Examples + -------- + >>> a = np.arange(4).reshape((2,2)) + >>> a + array([[0, 1], + [2, 3]]) + >>> np.amax(a) + 3 + >>> np.amax(a, axis=0) + array([2, 3]) + >>> np.amax(a, axis=1) + array([1, 3]) + + >>> b = np.arange(5, dtype=np.float) + >>> b[2] = np.NaN + >>> np.amax(b) + nan + >>> np.nanmax(b) + 4.0 + + """ + assert axis is None + assert out is None + if not hasattr(a, "max"): + a = numpypy.array(a) + return a.max() + + +def amin(a, axis=None, out=None): + """ + Return the minimum of an array or minimum along an axis. + + Parameters + ---------- + a : array_like + Input data. + axis : int, optional + Axis along which to operate. By default a flattened input is used. + out : ndarray, optional + Alternative output array in which to place the result. Must + be of the same shape and buffer length as the expected output. + See `doc.ufuncs` (Section "Output arguments") for more details. + + Returns + ------- + amin : ndarray + A new array or a scalar array with the result. + + See Also + -------- + nanmin: nan values are ignored instead of being propagated + fmin: same behavior as the C99 fmin function + argmin: Return the indices of the minimum values. + + amax, nanmax, fmax + + Notes + ----- + NaN values are propagated, that is if at least one item is nan, the + corresponding min value will be nan as well. To ignore NaN values (matlab + behavior), please use nanmin. + + Examples + -------- + >>> a = np.arange(4).reshape((2,2)) + >>> a + array([[0, 1], + [2, 3]]) + >>> np.amin(a) # Minimum of the flattened array + 0 + >>> np.amin(a, axis=0) # Minima along the first axis + array([0, 1]) + >>> np.amin(a, axis=1) # Minima along the second axis + array([0, 2]) + + >>> b = np.arange(5, dtype=np.float) + >>> b[2] = np.NaN + >>> np.amin(b) + nan + >>> np.nanmin(b) + 0.0 + + """ + # amin() is equivalent to min() + assert axis is None + assert out is None + if not hasattr(a, 'min'): + a = numpypy.array(a) + return a.min() + +def alen(a): + """ + Return the length of the first dimension of the input array. + + Parameters + ---------- + a : array_like + Input array. + + Returns + ------- + l : int + Length of the first dimension of `a`. + + See Also + -------- + shape, size + + Examples + -------- + >>> a = np.zeros((7,4,5)) + >>> a.shape[0] + 7 + >>> np.alen(a) + 7 + + """ + if not hasattr(a, 'shape'): + a = numpypy.array(a) + return a.shape[0] + + +def prod(a, axis=None, dtype=None, out=None): + """ + Return the product of array elements over a given axis. + + Parameters + ---------- + a : array_like + Input data. + axis : int, optional + Axis over which the product is taken. By default, the product + of all elements is calculated. + dtype : data-type, optional + The data-type of the returned array, as well as of the accumulator + in which the elements are multiplied. By default, if `a` is of + integer type, `dtype` is the default platform integer. (Note: if + the type of `a` is unsigned, then so is `dtype`.) Otherwise, + the dtype is the same as that of `a`. + out : ndarray, optional + Alternative output array in which to place the result. It must have + the same shape as the expected output, but the type of the + output values will be cast if necessary. + + Returns + ------- + product_along_axis : ndarray, see `dtype` parameter above. + An array shaped as `a` but with the specified axis removed. + Returns a reference to `out` if specified. + + See Also + -------- + ndarray.prod : equivalent method + numpy.doc.ufuncs : Section "Output arguments" + + Notes + ----- + Arithmetic is modular when using integer types, and no error is + raised on overflow. That means that, on a 32-bit platform: + + >>> x = np.array([536870910, 536870910, 536870910, 536870910]) + >>> np.prod(x) #random + 16 + + Examples + -------- + By default, calculate the product of all elements: + + >>> np.prod([1.,2.]) + 2.0 + + Even when the input array is two-dimensional: + + >>> np.prod([[1.,2.],[3.,4.]]) + 24.0 + + But we can also specify the axis over which to multiply: + + >>> np.prod([[1.,2.],[3.,4.]], axis=1) + array([ 2., 12.]) + + If the type of `x` is unsigned, then the output type is + the unsigned platform integer: + + >>> x = np.array([1, 2, 3], dtype=np.uint8) + >>> np.prod(x).dtype == np.uint + True + + If `x` is of a signed integer type, then the output type + is the default platform integer: + + >>> x = np.array([1, 2, 3], dtype=np.int8) + >>> np.prod(x).dtype == np.int + True + + """ + raise NotImplementedError('Waiting on interp level method') + + +def cumprod(a, axis=None, dtype=None, out=None): + """ + Return the cumulative product of elements along a given axis. + + Parameters + ---------- + a : array_like + Input array. + axis : int, optional + Axis along which the cumulative product is computed. By default + the input is flattened. + dtype : dtype, optional + Type of the returned array, as well as of the accumulator in which + the elements are multiplied. If *dtype* is not specified, it + defaults to the dtype of `a`, unless `a` has an integer dtype with + a precision less than that of the default platform integer. In + that case, the default platform integer is used instead. + out : ndarray, optional + Alternative output array in which to place the result. It must + have the same shape and buffer length as the expected output + but the type of the resulting values will be cast if necessary. + + Returns + ------- + cumprod : ndarray + A new array holding the result is returned unless `out` is + specified, in which case a reference to out is returned. + + See Also + -------- + numpy.doc.ufuncs : Section "Output arguments" + + Notes + ----- + Arithmetic is modular when using integer types, and no error is + raised on overflow. + + Examples + -------- + >>> a = np.array([1,2,3]) + >>> np.cumprod(a) # intermediate results 1, 1*2 + ... # total product 1*2*3 = 6 + array([1, 2, 6]) + >>> a = np.array([[1, 2, 3], [4, 5, 6]]) + >>> np.cumprod(a, dtype=float) # specify type of output + array([ 1., 2., 6., 24., 120., 720.]) + + The cumulative product for each column (i.e., over the rows) of `a`: + + >>> np.cumprod(a, axis=0) + array([[ 1, 2, 3], + [ 4, 10, 18]]) + + The cumulative product for each row (i.e. over the columns) of `a`: + + >>> np.cumprod(a,axis=1) + array([[ 1, 2, 6], + [ 4, 20, 120]]) + + """ + raise NotImplementedError('Waiting on interp level method') + + +def ndim(a): + """ + Return the number of dimensions of an array. + + Parameters + ---------- + a : array_like + Input array. If it is not already an ndarray, a conversion is + attempted. + + Returns + ------- + number_of_dimensions : int + The number of dimensions in `a`. Scalars are zero-dimensional. + + See Also + -------- + ndarray.ndim : equivalent method + shape : dimensions of array + ndarray.shape : dimensions of array + + Examples + -------- + >>> np.ndim([[1,2,3],[4,5,6]]) + 2 + >>> np.ndim(np.array([[1,2,3],[4,5,6]])) + 2 + >>> np.ndim(1) + 0 + + """ + if not hasattr(a, 'ndim'): + a = numpypy.array(a) + return a.ndim + + +def rank(a): + """ + Return the number of dimensions of an array. + + If `a` is not already an array, a conversion is attempted. + Scalars are zero dimensional. + + Parameters + ---------- + a : array_like + Array whose number of dimensions is desired. If `a` is not an array, + a conversion is attempted. + + Returns + ------- + number_of_dimensions : int + The number of dimensions in the array. + + See Also + -------- + ndim : equivalent function + ndarray.ndim : equivalent property + shape : dimensions of array + ndarray.shape : dimensions of array + + Notes + ----- + In the old Numeric package, `rank` was the term used for the number of + dimensions, but in Numpy `ndim` is used instead. + + Examples + -------- + >>> np.rank([1,2,3]) + 1 + >>> np.rank(np.array([[1,2,3],[4,5,6]])) + 2 + >>> np.rank(1) + 0 + + """ + if not hasattr(a, 'ndim'): + a = numpypy.array(a) + return a.ndim + + +def size(a, axis=None): + """ + Return the number of elements along a given axis. + + Parameters + ---------- + a : array_like + Input data. + axis : int, optional + Axis along which the elements are counted. By default, give + the total number of elements. + + Returns + ------- + element_count : int + Number of elements along the specified axis. + + See Also + -------- + shape : dimensions of array + ndarray.shape : dimensions of array + ndarray.size : number of elements in array + + Examples + -------- + >>> a = np.array([[1,2,3],[4,5,6]]) + >>> np.size(a) + 6 + >>> np.size(a,1) + 3 + >>> np.size(a,0) + 2 + + """ + raise NotImplementedError('Waiting on interp level method') + + +def around(a, decimals=0, out=None): + """ + Evenly round to the given number of decimals. + + Parameters + ---------- + a : array_like + Input data. + decimals : int, optional + Number of decimal places to round to (default: 0). If + decimals is negative, it specifies the number of positions to + the left of the decimal point. + out : ndarray, optional + Alternative output array in which to place the result. It must have + the same shape as the expected output, but the type of the output + values will be cast if necessary. See `doc.ufuncs` (Section + "Output arguments") for details. + + Returns + ------- + rounded_array : ndarray + An array of the same type as `a`, containing the rounded values. + Unless `out` was specified, a new array is created. A reference to + the result is returned. + + The real and imaginary parts of complex numbers are rounded + separately. The result of rounding a float is a float. + + See Also + -------- + ndarray.round : equivalent method + + ceil, fix, floor, rint, trunc + + + Notes + ----- + For values exactly halfway between rounded decimal values, Numpy + rounds to the nearest even value. Thus 1.5 and 2.5 round to 2.0, + -0.5 and 0.5 round to 0.0, etc. Results may also be surprising due + to the inexact representation of decimal fractions in the IEEE + floating point standard [1]_ and errors introduced when scaling + by powers of ten. + + References + ---------- + .. [1] "Lecture Notes on the Status of IEEE 754", William Kahan, + http://www.cs.berkeley.edu/~wkahan/ieee754status/IEEE754.PDF + .. [2] "How Futile are Mindless Assessments of + Roundoff in Floating-Point Computation?", William Kahan, + http://www.cs.berkeley.edu/~wkahan/Mindless.pdf + + Examples + -------- + >>> np.around([0.37, 1.64]) + array([ 0., 2.]) + >>> np.around([0.37, 1.64], decimals=1) + array([ 0.4, 1.6]) + >>> np.around([.5, 1.5, 2.5, 3.5, 4.5]) # rounds to nearest even value + array([ 0., 2., 2., 4., 4.]) + >>> np.around([1,2,3,11], decimals=1) # ndarray of ints is returned + array([ 1, 2, 3, 11]) + >>> np.around([1,2,3,11], decimals=-1) + array([ 0, 0, 0, 10]) + + """ + raise NotImplementedError('Waiting on interp level method') + + +def round_(a, decimals=0, out=None): + """ + Round an array to the given number of decimals. + + Refer to `around` for full documentation. + + See Also + -------- + around : equivalent function + + """ + raise NotImplementedError('Waiting on interp level method') + + +def mean(a, axis=None, dtype=None, out=None): + """ + Compute the arithmetic mean along the specified axis. + + Returns the average of the array elements. The average is taken over + the flattened array by default, otherwise over the specified axis. + `float64` intermediate and return values are used for integer inputs. + + Parameters + ---------- + a : array_like + Array containing numbers whose mean is desired. If `a` is not an + array, a conversion is attempted. + axis : int, optional + Axis along which the means are computed. The default is to compute + the mean of the flattened array. + dtype : data-type, optional + Type to use in computing the mean. For integer inputs, the default + is `float64`; for floating point inputs, it is the same as the + input dtype. + out : ndarray, optional + Alternate output array in which to place the result. The default + is ``None``; if provided, it must have the same shape as the + expected output, but the type will be cast if necessary. + See `doc.ufuncs` for details. + + Returns + ------- + m : ndarray, see dtype parameter above + If `out=None`, returns a new array containing the mean values, + otherwise a reference to the output array is returned. + + See Also + -------- + average : Weighted average + + Notes + ----- + The arithmetic mean is the sum of the elements along the axis divided + by the number of elements. + + Note that for floating-point input, the mean is computed using the + same precision the input has. Depending on the input data, this can + cause the results to be inaccurate, especially for `float32` (see + example below). Specifying a higher-precision accumulator using the + `dtype` keyword can alleviate this issue. + + Examples + -------- + >>> a = np.array([[1, 2], [3, 4]]) + >>> np.mean(a) + 2.5 + >>> np.mean(a, axis=0) + array([ 2., 3.]) + >>> np.mean(a, axis=1) + array([ 1.5, 3.5]) + + In single precision, `mean` can be inaccurate: + + >>> a = np.zeros((2, 512*512), dtype=np.float32) + >>> a[0, :] = 1.0 + >>> a[1, :] = 0.1 + >>> np.mean(a) + 0.546875 + + Computing the mean in float64 is more accurate: + + >>> np.mean(a, dtype=np.float64) + 0.55000000074505806 + + """ + assert dtype is None + assert out is None + if not hasattr(a, "mean"): + a = numpypy.array(a) + return a.mean(axis=axis) + + +def std(a, axis=None, dtype=None, out=None, ddof=0): + """ + Compute the standard deviation along the specified axis. + + Returns the standard deviation, a measure of the spread of a distribution, + of the array elements. The standard deviation is computed for the + flattened array by default, otherwise over the specified axis. + + Parameters + ---------- + a : array_like + Calculate the standard deviation of these values. + axis : int, optional + Axis along which the standard deviation is computed. The default is + to compute the standard deviation of the flattened array. + dtype : dtype, optional + Type to use in computing the standard deviation. For arrays of + integer type the default is float64, for arrays of float types it is + the same as the array type. + out : ndarray, optional + Alternative output array in which to place the result. It must have + the same shape as the expected output but the type (of the calculated + values) will be cast if necessary. + ddof : int, optional + Means Delta Degrees of Freedom. The divisor used in calculations + is ``N - ddof``, where ``N`` represents the number of elements. + By default `ddof` is zero. + + Returns + ------- + standard_deviation : ndarray, see dtype parameter above. + If `out` is None, return a new array containing the standard deviation, + otherwise return a reference to the output array. + + See Also + -------- + var, mean + numpy.doc.ufuncs : Section "Output arguments" + + Notes + ----- + The standard deviation is the square root of the average of the squared + deviations from the mean, i.e., ``std = sqrt(mean(abs(x - x.mean())**2))``. + + The average squared deviation is normally calculated as ``x.sum() / N``, where + ``N = len(x)``. If, however, `ddof` is specified, the divisor ``N - ddof`` + is used instead. In standard statistical practice, ``ddof=1`` provides an + unbiased estimator of the variance of the infinite population. ``ddof=0`` + provides a maximum likelihood estimate of the variance for normally + distributed variables. The standard deviation computed in this function + is the square root of the estimated variance, so even with ``ddof=1``, it + will not be an unbiased estimate of the standard deviation per se. + + Note that, for complex numbers, `std` takes the absolute + value before squaring, so that the result is always real and nonnegative. + + For floating-point input, the *std* is computed using the same + precision the input has. Depending on the input data, this can cause + the results to be inaccurate, especially for float32 (see example below). + Specifying a higher-accuracy accumulator using the `dtype` keyword can + alleviate this issue. + + Examples + -------- + >>> a = np.array([[1, 2], [3, 4]]) + >>> np.std(a) + 1.1180339887498949 + >>> np.std(a, axis=0) + array([ 1., 1.]) + >>> np.std(a, axis=1) + array([ 0.5, 0.5]) + + In single precision, std() can be inaccurate: + + >>> a = np.zeros((2,512*512), dtype=np.float32) + >>> a[0,:] = 1.0 + >>> a[1,:] = 0.1 + >>> np.std(a) + 0.45172946707416706 + + Computing the standard deviation in float64 is more accurate: + + >>> np.std(a, dtype=np.float64) + 0.44999999925552653 + + """ + assert axis is None + assert dtype is None + assert out is None + assert ddof == 0 + if not hasattr(a, "std"): + a = numpypy.array(a) + return a.std() + + +def var(a, axis=None, dtype=None, out=None, ddof=0): + """ + Compute the variance along the specified axis. + + Returns the variance of the array elements, a measure of the spread of a + distribution. The variance is computed for the flattened array by + default, otherwise over the specified axis. + + Parameters + ---------- + a : array_like + Array containing numbers whose variance is desired. If `a` is not an + array, a conversion is attempted. + axis : int, optional + Axis along which the variance is computed. The default is to compute + the variance of the flattened array. + dtype : data-type, optional + Type to use in computing the variance. For arrays of integer type + the default is `float32`; for arrays of float types it is the same as + the array type. + out : ndarray, optional + Alternate output array in which to place the result. It must have + the same shape as the expected output, but the type is cast if + necessary. + ddof : int, optional + "Delta Degrees of Freedom": the divisor used in the calculation is + ``N - ddof``, where ``N`` represents the number of elements. By + default `ddof` is zero. + + Returns + ------- + variance : ndarray, see dtype parameter above + If ``out=None``, returns a new array containing the variance; + otherwise, a reference to the output array is returned. + + See Also + -------- + std : Standard deviation + mean : Average + numpy.doc.ufuncs : Section "Output arguments" + + Notes + ----- + The variance is the average of the squared deviations from the mean, + i.e., ``var = mean(abs(x - x.mean())**2)``. + + The mean is normally calculated as ``x.sum() / N``, where ``N = len(x)``. + If, however, `ddof` is specified, the divisor ``N - ddof`` is used + instead. In standard statistical practice, ``ddof=1`` provides an + unbiased estimator of the variance of a hypothetical infinite population. + ``ddof=0`` provides a maximum likelihood estimate of the variance for + normally distributed variables. + + Note that for complex numbers, the absolute value is taken before + squaring, so that the result is always real and nonnegative. + + For floating-point input, the variance is computed using the same + precision the input has. Depending on the input data, this can cause + the results to be inaccurate, especially for `float32` (see example + below). Specifying a higher-accuracy accumulator using the ``dtype`` + keyword can alleviate this issue. + + Examples + -------- + >>> a = np.array([[1,2],[3,4]]) + >>> np.var(a) + 1.25 + >>> np.var(a,0) + array([ 1., 1.]) + >>> np.var(a,1) + array([ 0.25, 0.25]) + + In single precision, var() can be inaccurate: + + >>> a = np.zeros((2,512*512), dtype=np.float32) + >>> a[0,:] = 1.0 + >>> a[1,:] = 0.1 + >>> np.var(a) + 0.20405951142311096 + + Computing the standard deviation in float64 is more accurate: + + >>> np.var(a, dtype=np.float64) + 0.20249999932997387 + >>> ((1-0.55)**2 + (0.1-0.55)**2)/2 + 0.20250000000000001 + + """ + assert axis is None + assert dtype is None + assert out is None + assert ddof == 0 + if not hasattr(a, "var"): + a = numpypy.array(a) + return a.var() diff --git a/py/_code/code.py b/py/_code/code.py --- a/py/_code/code.py +++ b/py/_code/code.py @@ -164,6 +164,7 @@ # if something: # assume this causes a NameError # # _this_ lines and the one # below we don't want from entry.getsource() + end = min(end, len(source)) for i in range(self.lineno, end): if source[i].rstrip().endswith(':'): end = i + 1 diff --git a/pypy/annotation/description.py b/pypy/annotation/description.py --- a/pypy/annotation/description.py +++ b/pypy/annotation/description.py @@ -180,7 +180,12 @@ if name is None: name = pyobj.func_name if signature is None: - signature = cpython_code_signature(pyobj.func_code) + if hasattr(pyobj, '_generator_next_method_of_'): + from pypy.interpreter.argument import Signature + signature = Signature(['entry']) # haaaaaack + defaults = () + else: + signature = cpython_code_signature(pyobj.func_code) if defaults is None: defaults = pyobj.func_defaults self.name = name @@ -252,7 +257,8 @@ try: inputcells = args.match_signature(signature, defs_s) except ArgErr, e: - raise TypeError, "signature mismatch: %s" % e.getmsg(self.name) + raise TypeError("signature mismatch: %s() %s" % + (self.name, e.getmsg())) return inputcells def specialize(self, inputcells, op=None): diff --git a/pypy/bin/checkmodule.py b/pypy/bin/checkmodule.py --- a/pypy/bin/checkmodule.py +++ b/pypy/bin/checkmodule.py @@ -1,43 +1,45 @@ #! /usr/bin/env python """ -Usage: checkmodule.py [-b backend] <module-name> +Usage: checkmodule.py <module-name> -Compiles the PyPy extension module from pypy/module/<module-name>/ -into a fake program which does nothing. Useful for testing whether a -modules compiles without doing a full translation. Default backend is cli. - -WARNING: this is still incomplete: there are chances that the -compilation fails with strange errors not due to the module. If a -module is known to compile during a translation but don't pass -checkmodule.py, please report the bug (or, better, correct it :-). +Check annotation and rtyping of the PyPy extension module from +pypy/module/<module-name>/. Useful for testing whether a +modules compiles without doing a full translation. """ import autopath -import sys +import sys, os from pypy.objspace.fake.checkmodule import checkmodule def main(argv): - try: - assert len(argv) in (2, 4) - if len(argv) == 2: - backend = 'cli' - modname = argv[1] - if modname in ('-h', '--help'): - print >> sys.stderr, __doc__ - sys.exit(0) - if modname.startswith('-'): - print >> sys.stderr, "Bad command line" - print >> sys.stderr, __doc__ - sys.exit(1) - else: - _, b, backend, modname = argv - assert b == '-b' - except AssertionError: + if len(argv) != 2: print >> sys.stderr, __doc__ sys.exit(2) + modname = argv[1] + if modname in ('-h', '--help'): + print >> sys.stderr, __doc__ + sys.exit(0) + if modname.startswith('-'): + print >> sys.stderr, "Bad command line" + print >> sys.stderr, __doc__ + sys.exit(1) + if os.path.sep in modname: + if os.path.basename(modname) == '': + modname = os.path.dirname(modname) + if os.path.basename(os.path.dirname(modname)) != 'module': + print >> sys.stderr, "Must give '../module/xxx', or just 'xxx'." + sys.exit(1) + modname = os.path.basename(modname) + try: + checkmodule(modname) + except Exception, e: + import traceback, pdb + traceback.print_exc() + pdb.post_mortem(sys.exc_info()[2]) + return 1 else: - checkmodule(modname, backend, interactive=True) - print 'Module compiled succesfully' + print 'Passed.' + return 0 if __name__ == '__main__': - main(sys.argv) + sys.exit(main(sys.argv)) diff --git a/pypy/bin/py.py b/pypy/bin/py.py --- a/pypy/bin/py.py +++ b/pypy/bin/py.py @@ -76,6 +76,8 @@ config.objspace.suggest(allworkingmodules=False) if config.objspace.allworkingmodules: pypyoption.enable_allworkingmodules(config) + if config.objspace.usemodules._continuation: + config.translation.continuation = True if config.objspace.usemodules.thread: config.translation.thread = True diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py --- a/pypy/config/pypyoption.py +++ b/pypy/config/pypyoption.py @@ -252,6 +252,10 @@ "use small tuples", default=False), + BoolOption("withspecialisedtuple", + "use specialised tuples", + default=False), + BoolOption("withrope", "use ropes as the string implementation", default=False, requires=[("objspace.std.withstrslice", False), @@ -336,7 +340,7 @@ requires=[("objspace.std.builtinshortcut", True)]), BoolOption("withidentitydict", "track types that override __hash__, __eq__ or __cmp__ and use a special dict strategy for those which do not", - default=True), + default=False), ]), ]) @@ -365,6 +369,8 @@ config.objspace.std.suggest(optimized_list_getitem=True) config.objspace.std.suggest(getattributeshortcut=True) config.objspace.std.suggest(newshortcut=True) + config.objspace.std.suggest(withspecialisedtuple=True) + config.objspace.std.suggest(withidentitydict=True) #if not IS_64_BITS: # config.objspace.std.suggest(withsmalllong=True) diff --git a/pypy/conftest.py b/pypy/conftest.py --- a/pypy/conftest.py +++ b/pypy/conftest.py @@ -496,6 +496,17 @@ def setup(self): super(AppClassCollector, self).setup() cls = self.obj + # + # <hack> + for name in dir(cls): + if name.startswith('test_'): + func = getattr(cls, name, None) + code = getattr(func, 'func_code', None) + if code and code.co_flags & 32: + raise AssertionError("unsupported: %r is a generator " + "app-level test method" % (name,)) + # </hack> + # space = cls.space clsname = cls.__name__ if self.config.option.runappdirect: diff --git a/pypy/doc/Makefile b/pypy/doc/Makefile --- a/pypy/doc/Makefile +++ b/pypy/doc/Makefile @@ -12,7 +12,7 @@ PAPEROPT_letter = -D latex_paper_size=letter ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . -.PHONY: help clean html dirhtml pickle json htmlhelp qthelp latex changes linkcheck doctest +.PHONY: help clean html dirhtml pickle json htmlhelp qthelp latex man changes linkcheck doctest help: @echo "Please use \`make <target>' where <target> is one of" @@ -23,6 +23,7 @@ @echo " htmlhelp to make HTML files and a HTML help project" @echo " qthelp to make HTML files and a qthelp project" @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " man to make manual pages" @echo " changes to make an overview of all changed/added/deprecated items" @echo " linkcheck to check all external links for integrity" @echo " doctest to run all doctests embedded in the documentation (if enabled)" @@ -79,6 +80,11 @@ @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \ "run these through (pdf)latex." +man: + $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man + @echo + @echo "Build finished. The manual pages are in $(BUILDDIR)/man" + changes: python config/generate.py $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes diff --git a/pypy/doc/coding-guide.rst b/pypy/doc/coding-guide.rst --- a/pypy/doc/coding-guide.rst +++ b/pypy/doc/coding-guide.rst @@ -175,15 +175,15 @@ RPython ================= -RPython Definition, not ------------------------ +RPython Definition +------------------ -The list and exact details of the "RPython" restrictions are a somewhat -evolving topic. In particular, we have no formal language definition -as we find it more practical to discuss and evolve the set of -restrictions while working on the whole program analysis. If you -have any questions about the restrictions below then please feel -free to mail us at pypy-dev at codespeak net. +RPython is a restricted subset of Python that is amenable to static analysis. +Although there are additions to the language and some things might surprisingly +work, this is a rough list of restrictions that should be considered. Note +that there are tons of special cased restrictions that you'll encounter +as you go. The exact definition is "RPython is everything that our translation +toolchain can accept" :) .. _`wrapped object`: coding-guide.html#wrapping-rules @@ -198,7 +198,7 @@ contain both a string and a int must be avoided. It is allowed to mix None (basically with the role of a null pointer) with many other types: `wrapped objects`, class instances, lists, dicts, strings, etc. - but *not* with int and floats. + but *not* with int, floats or tuples. **constants** @@ -209,9 +209,12 @@ have this restriction, so if you need mutable global state, store it in the attributes of some prebuilt singleton instance. + + **control structures** - all allowed but yield, ``for`` loops restricted to builtin types + all allowed, ``for`` loops restricted to builtin types, generators + very restricted. **range** @@ -226,7 +229,8 @@ **generators** - generators are not supported. + generators are supported, but their exact scope is very limited. you can't + merge two different generator in one control point. **exceptions** @@ -245,22 +249,27 @@ **strings** - a lot of, but not all string methods are supported. Indexes can be + a lot of, but not all string methods are supported and those that are + supported, not necesarilly accept all arguments. Indexes can be negative. In case they are not, then you get slightly more efficient code if the translator can prove that they are non-negative. When slicing a string it is necessary to prove that the slice start and - stop indexes are non-negative. + stop indexes are non-negative. There is no implicit str-to-unicode cast + anywhere. **tuples** no variable-length tuples; use them to store or return pairs or n-tuples of - values. Each combination of types for elements and length constitute a separate - and not mixable type. + values. Each combination of types for elements and length constitute + a separate and not mixable type. **lists** lists are used as an allocated array. Lists are over-allocated, so list.append() - is reasonably fast. Negative or out-of-bound indexes are only allowed for the + is reasonably fast. However, if you use a fixed-size list, the code + is more efficient. Annotator can figure out most of the time that your + list is fixed-size, even when you use list comprehension. + Negative or out-of-bound indexes are only allowed for the most common operations, as follows: - *indexing*: @@ -287,16 +296,14 @@ **dicts** - dicts with a unique key type only, provided it is hashable. - String keys have been the only allowed key types for a while, but this was generalized. - After some re-optimization, - the implementation could safely decide that all string dict keys should be interned. + dicts with a unique key type only, provided it is hashable. Custom + hash functions and custom equality will not be honored. + Use ``pypy.rlib.objectmodel.r_dict`` for custom hash functions. **list comprehensions** - may be used to create allocated, initialized arrays. - After list over-allocation was introduced, there is no longer any restriction. + May be used to create allocated, initialized arrays. **functions** @@ -334,9 +341,8 @@ **objects** - in PyPy, wrapped objects are borrowed from the object space. Just like - in CPython, code that needs e.g. a dictionary can use a wrapped dict - and the object space operations on it. + Normal rules apply. Special methods are not honoured, except ``__init__`` and + ``__del__``. This layout makes the number of types to take care about quite limited. diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py --- a/pypy/doc/conf.py +++ b/pypy/doc/conf.py @@ -45,9 +45,9 @@ # built documents. # # The short X.Y version. -version = '1.6' +version = '1.7' # The full version, including alpha/beta/rc tags. -release = '1.6' +release = '1.7' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -197,3 +197,10 @@ # Example configuration for intersphinx: refer to the Python standard library. intersphinx_mapping = {'http://docs.python.org/': None} +# -- Options for manpage output------------------------------------------------- + +man_pages = [ + ('man/pypy.1', 'pypy', + u'fast, compliant alternative implementation of the Python language', + u'The PyPy Project', 1) +] diff --git a/pypy/doc/config/objspace.std.withspecialisedtuple.txt b/pypy/doc/config/objspace.std.withspecialisedtuple.txt new file mode 100644 --- /dev/null +++ b/pypy/doc/config/objspace.std.withspecialisedtuple.txt @@ -0,0 +1,3 @@ +Use "specialized tuples", a custom implementation for some common kinds +of tuples. Currently limited to tuples of length 2, in three variants: +(int, int), (float, float), and a generic (object, object). diff --git a/pypy/doc/extradoc.rst b/pypy/doc/extradoc.rst --- a/pypy/doc/extradoc.rst +++ b/pypy/doc/extradoc.rst @@ -8,6 +8,9 @@ *Articles about PyPy published so far, most recent first:* (bibtex_ file) +* `Runtime Feedback in a Meta-Tracing JIT for Efficient Dynamic Languages`_, + C.F. Bolz, A. Cuni, M. Fijalkowski, M. Leuschel, S. Pedroni, A. Rigo + * `Allocation Removal by Partial Evaluation in a Tracing JIT`_, C.F. Bolz, A. Cuni, M. Fijalkowski, M. Leuschel, S. Pedroni, A. Rigo @@ -50,6 +53,9 @@ *Other research using PyPy (as far as we know it):* +* `Hardware Transactional Memory Support for Lightweight Dynamic Language Evolution`_, + N. Riley and C. Zilles + * `PyGirl: Generating Whole-System VMs from High-Level Prototypes using PyPy`_, C. Bruni and T. Verwaest @@ -65,6 +71,7 @@ .. _bibtex: https://bitbucket.org/pypy/extradoc/raw/tip/talk/bibtex.bib +.. _`Runtime Feedback in a Meta-Tracing JIT for Efficient Dynamic Languages`: https://bitbucket.org/pypy/extradoc/raw/extradoc/talk/icooolps2011/jit-hints.pdf .. _`Allocation Removal by Partial Evaluation in a Tracing JIT`: http://codespeak.net/svn/pypy/extradoc/talk/pepm2011/bolz-allocation-removal.pdf .. _`Towards a Jitting VM for Prolog Execution`: http://www.stups.uni-duesseldorf.de/publications/bolz-prolog-jit.pdf .. _`High performance implementation of Python for CLI/.NET with JIT compiler generation for dynamic languages`: http://buildbot.pypy.org/misc/antocuni-thesis.pdf @@ -74,6 +81,7 @@ .. _`Automatic JIT Compiler Generation with Runtime Partial Evaluation`: http://www.stups.uni-duesseldorf.de/thesis/final-master.pdf .. _`RPython: A Step towards Reconciling Dynamically and Statically Typed OO Languages`: http://www.disi.unige.it/person/AnconaD/papers/Recent_abstracts.html#AACM-DLS07 .. _`EU Reports`: index-report.html +.. _`Hardware Transactional Memory Support for Lightweight Dynamic Language Evolution`: http://sabi.net/nriley/pubs/dls6-riley.pdf .. _`PyGirl: Generating Whole-System VMs from High-Level Prototypes using PyPy`: http://scg.unibe.ch/archive/papers/Brun09cPyGirl.pdf .. _`Representation-Based Just-in-Time Specialization and the Psyco Prototype for Python`: http://psyco.sourceforge.net/psyco-pepm-a.ps.gz .. _`Back to the Future in One Week -- Implementing a Smalltalk VM in PyPy`: http://dx.doi.org/10.1007/978-3-540-89275-5_7 diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst --- a/pypy/doc/faq.rst +++ b/pypy/doc/faq.rst @@ -112,10 +112,32 @@ You might be interested in our `benchmarking site`_ and our `jit documentation`_. +Note that the JIT has a very high warm-up cost, meaning that the +programs are slow at the beginning. If you want to compare the timings +with CPython, even relatively simple programs need to run *at least* one +second, preferrably at least a few seconds. Large, complicated programs +need even more time to warm-up the JIT. + .. _`benchmarking site`: http://speed.pypy.org .. _`jit documentation`: jit/index.html +--------------------------------------------------------------- +Couldn't the JIT dump and reload already-compiled machine code? +--------------------------------------------------------------- + +No, we found no way of doing that. The JIT generates machine code +containing a large number of constant addresses --- constant at the time +the machine code is written. The vast majority is probably not at all +constants that you find in the executable, with a nice link name. E.g. +the addresses of Python classes are used all the time, but Python +classes don't come statically from the executable; they are created anew +every time you restart your program. This makes saving and reloading +machine code completely impossible without some very advanced way of +mapping addresses in the old (now-dead) process to addresses in the new +process, including checking that all the previous assumptions about the +(now-dead) object are still true about the new object. + .. _`prolog and javascript`: diff --git a/pypy/doc/getting-started.rst b/pypy/doc/getting-started.rst --- a/pypy/doc/getting-started.rst +++ b/pypy/doc/getting-started.rst @@ -53,11 +53,11 @@ PyPy is ready to be executed as soon as you unpack the tarball or the zip file, with no need to install it in any specific location:: - $ tar xf pypy-1.6-linux.tar.bz2 + $ tar xf pypy-1.7-linux.tar.bz2 - $ ./pypy-1.6/bin/pypy + $ ./pypy-1.7/bin/pypy Python 2.7.1 (?, Apr 27 2011, 12:44:21) - [PyPy 1.6.0 with GCC 4.4.3] on linux2 + [PyPy 1.7.0 with GCC 4.4.3] on linux2 Type "help", "copyright", "credits" or "license" for more information. And now for something completely different: ``implementing LOGO in LOGO: "turtles all the way down"'' @@ -75,14 +75,14 @@ $ curl -O https://raw.github.com/pypa/pip/master/contrib/get-pip.py - $ ./pypy-1.6/bin/pypy distribute_setup.py + $ ./pypy-1.7/bin/pypy distribute_setup.py - $ ./pypy-1.6/bin/pypy get-pip.py + $ ./pypy-1.7/bin/pypy get-pip.py - $ ./pypy-1.6/bin/pip install pygments # for example + $ ./pypy-1.7/bin/pip install pygments # for example -3rd party libraries will be installed in ``pypy-1.6/site-packages``, and -the scripts in ``pypy-1.6/bin``. +3rd party libraries will be installed in ``pypy-1.7/site-packages``, and +the scripts in ``pypy-1.7/bin``. Installing using virtualenv --------------------------- diff --git a/pypy/doc/man/pypy.1.rst b/pypy/doc/man/pypy.1.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/man/pypy.1.rst @@ -0,0 +1,90 @@ +====== + pypy +====== + +SYNOPSIS +======== + +``pypy`` [*options*] +[``-c`` *cmd*\ \|\ ``-m`` *mod*\ \|\ *file.py*\ \|\ ``-``\ ] +[*arg*\ ...] + +OPTIONS +======= + +-i + Inspect interactively after running script. + +-O + Dummy optimization flag for compatibility with C Python. + +-c *cmd* + Program passed in as CMD (terminates option list). + +-S + Do not ``import site`` on initialization. + +-u + Unbuffered binary ``stdout`` and ``stderr``. + +-h, --help + Show a help message and exit. + +-m *mod* + Library module to be run as a script (terminates option list). + +-W *arg* + Warning control (*arg* is *action*:*message*:*category*:*module*:*lineno*). + +-E + Ignore environment variables (such as ``PYTHONPATH``). + +--version + Print the PyPy version. + +--info + Print translation information about this PyPy executable. + +--jit *arg* + Low level JIT parameters. Format is + *arg*\ ``=``\ *value*\ [``,``\ *arg*\ ``=``\ *value*\ ...] + + ``off`` + Disable the JIT. + + ``threshold=``\ *value* + Number of times a loop has to run for it to become hot. + + ``function_threshold=``\ *value* + Number of times a function must run for it to become traced from + start. + + ``inlining=``\ *value* + Inline python functions or not (``1``/``0``). + + ``loop_longevity=``\ *value* + A parameter controlling how long loops will be kept before being + freed, an estimate. + + ``max_retrace_guards=``\ *value* + Number of extra guards a retrace can cause. + + ``retrace_limit=``\ *value* + How many times we can try retracing before giving up. + + ``trace_eagerness=``\ *value* + Number of times a guard has to fail before we start compiling a + bridge. + + ``trace_limit=``\ *value* + Number of recorded operations before we abort tracing with + ``ABORT_TRACE_TOO_LONG``. + + ``enable_opts=``\ *value* + Optimizations to enabled or ``all``. + Warning, this option is dangerous, and should be avoided. + +SEE ALSO +======== + +**python**\ (1) diff --git a/pypy/doc/tool/makecontributor.py b/pypy/doc/tool/makecontributor.py deleted file mode 100644 --- a/pypy/doc/tool/makecontributor.py +++ /dev/null @@ -1,47 +0,0 @@ -""" - -generates a contributor list - -""" -import py - -# this file is useless, use the following commandline instead: -# hg churn -c -t "{author}" | sed -e 's/ <.*//' - -try: - path = py.std.sys.argv[1] -except IndexError: - print "usage: %s ROOTPATH" %(py.std.sys.argv[0]) - raise SystemExit, 1 - -d = {} - -for logentry in py.path.svnwc(path).log(): - a = logentry.author - if a in d: - d[a] += 1 - else: - d[a] = 1 - -items = d.items() -items.sort(lambda x,y: -cmp(x[1], y[1])) - -import uconf # http://codespeak.net/svn/uconf/dist/uconf - -# Authors that don't want to be listed -excluded = set("anna gintas ignas".split()) -cutoff = 5 # cutoff for authors in the LICENSE file -mark = False -for author, count in items: - if author in excluded: - continue - user = uconf.system.User(author) - try: - realname = user.realname.strip() - except KeyError: - realname = author - if not mark and count < cutoff: - mark = True - print '-'*60 - print " ", realname - #print count, " ", author diff --git a/pypy/doc/translation.rst b/pypy/doc/translation.rst --- a/pypy/doc/translation.rst +++ b/pypy/doc/translation.rst @@ -155,7 +155,7 @@ function. The two input variables are the exception class and the exception value, respectively. (No other block will actually link to the exceptblock if the function does not - explicitely raise exceptions.) + explicitly raise exceptions.) ``Block`` @@ -325,7 +325,7 @@ Mutable objects need special treatment during annotation, because the annotation of contained values needs to be possibly updated to account for mutation operations, and consequently the annotation information -reflown through the relevant parts of the flow the graphs. +reflown through the relevant parts of the flow graphs. * ``SomeList`` stands for a list of homogeneous type (i.e. all the elements of the list are represented by a single common ``SomeXxx`` @@ -503,8 +503,8 @@ Since RPython is a garbage collected language there is a lot of heap memory allocation going on all the time, which would either not occur at all in a more -traditional explicitely managed language or results in an object which dies at -a time known in advance and can thus be explicitely deallocated. For example a +traditional explicitly managed language or results in an object which dies at +a time known in advance and can thus be explicitly deallocated. For example a loop of the following form:: for i in range(n): @@ -696,7 +696,7 @@ So far it is the second most mature high level backend after GenCLI: it still can't translate the full Standard Interpreter, but after the -Leysin sprint we were able to compile and run the rpytstone and +Leysin sprint we were able to compile and run the rpystone and richards benchmarks. GenJVM is almost entirely the work of Niko Matsakis, who worked on it diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py --- a/pypy/interpreter/argument.py +++ b/pypy/interpreter/argument.py @@ -428,8 +428,8 @@ return self._match_signature(w_firstarg, scope_w, signature, defaults_w, 0) except ArgErr, e: - raise OperationError(self.space.w_TypeError, - self.space.wrap(e.getmsg(fnname))) + raise operationerrfmt(self.space.w_TypeError, + "%s() %s", fnname, e.getmsg()) def _parse(self, w_firstarg, signature, defaults_w, blindargs=0): """Parse args and kwargs according to the signature of a code object, @@ -450,8 +450,8 @@ try: return self._parse(w_firstarg, signature, defaults_w, blindargs) except ArgErr, e: - raise OperationError(self.space.w_TypeError, - self.space.wrap(e.getmsg(fnname))) + raise operationerrfmt(self.space.w_TypeError, + "%s() %s", fnname, e.getmsg()) @staticmethod def frompacked(space, w_args=None, w_kwds=None): @@ -626,7 +626,7 @@ class ArgErr(Exception): - def getmsg(self, fnname): + def getmsg(self): raise NotImplementedError class ArgErrCount(ArgErr): @@ -642,11 +642,10 @@ self.num_args = got_nargs self.num_kwds = nkwds - def getmsg(self, fnname): + def getmsg(self): n = self.expected_nargs if n == 0: - msg = "%s() takes no arguments (%d given)" % ( - fnname, + msg = "takes no arguments (%d given)" % ( self.num_args + self.num_kwds) else: defcount = self.num_defaults @@ -672,8 +671,7 @@ msg2 = " non-keyword" else: msg2 = "" - msg = "%s() takes %s %d%s argument%s (%d given)" % ( - fnname, + msg = "takes %s %d%s argument%s (%d given)" % ( msg1, n, msg2, @@ -686,9 +684,8 @@ def __init__(self, argname): self.argname = argname - def getmsg(self, fnname): - msg = "%s() got multiple values for keyword argument '%s'" % ( - fnname, + def getmsg(self): + msg = "got multiple values for keyword argument '%s'" % ( self.argname) return msg @@ -722,13 +719,11 @@ break self.kwd_name = name - def getmsg(self, fnname): + def getmsg(self): if self.num_kwds == 1: - msg = "%s() got an unexpected keyword argument '%s'" % ( - fnname, + msg = "got an unexpected keyword argument '%s'" % ( self.kwd_name) else: - msg = "%s() got %d unexpected keyword arguments" % ( - fnname, + msg = "got %d unexpected keyword arguments" % ( self.num_kwds) return msg diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py --- a/pypy/interpreter/astcompiler/ast.py +++ b/pypy/interpreter/astcompiler/ast.py @@ -51,6 +51,24 @@ space.setattr(self, w_name, space.getitem(w_state, w_name)) + def missing_field(self, space, required, host): + "Find which required field is missing." + state = self.initialization_state + for i in range(len(required)): + if (state >> i) & 1: + continue # field is present + missing = required[i] + if missing is None: + continue # field is optional + w_obj = self.getdictvalue(space, missing) + if w_obj is None: + err = "required field \"%s\" missing from %s" + raise operationerrfmt(space.w_TypeError, err, missing, host) + else: + err = "incorrect type for field \"%s\" in %s" + raise operationerrfmt(space.w_TypeError, err, missing, host) + raise AssertionError("should not reach here") + class NodeVisitorNotImplemented(Exception): pass @@ -94,17 +112,6 @@ ) -def missing_field(space, state, required, host): - "Find which required field is missing." - for i in range(len(required)): - if not (state >> i) & 1: - missing = required[i] - if missing is not None: - err = "required field \"%s\" missing from %s" - err = err % (missing, host) - w_err = space.wrap(err) - raise OperationError(space.w_TypeError, w_err) - raise AssertionError("should not reach here") class mod(AST): @@ -112,7 +119,6 @@ class Module(mod): - def __init__(self, body): self.body = body self.w_body = None @@ -128,7 +134,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 1: - missing_field(space, self.initialization_state, ['body'], 'Module') + self.missing_field(space, ['body'], 'Module') else: pass w_list = self.w_body @@ -145,7 +151,6 @@ class Interactive(mod): - def __init__(self, body): self.body = body self.w_body = None @@ -161,7 +166,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 1: - missing_field(space, self.initialization_state, ['body'], 'Interactive') + self.missing_field(space, ['body'], 'Interactive') else: pass w_list = self.w_body @@ -178,7 +183,6 @@ class Expression(mod): - def __init__(self, body): self.body = body self.initialization_state = 1 @@ -192,7 +196,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 1: - missing_field(space, self.initialization_state, ['body'], 'Expression') + self.missing_field(space, ['body'], 'Expression') else: pass self.body.sync_app_attrs(space) @@ -200,7 +204,6 @@ class Suite(mod): - def __init__(self, body): self.body = body self.w_body = None @@ -216,7 +219,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 1: - missing_field(space, self.initialization_state, ['body'], 'Suite') + self.missing_field(space, ['body'], 'Suite') else: pass w_list = self.w_body @@ -232,15 +235,13 @@ class stmt(AST): + def __init__(self, lineno, col_offset): self.lineno = lineno self.col_offset = col_offset class FunctionDef(stmt): - _lineno_mask = 16 - _col_offset_mask = 32 - def __init__(self, name, args, body, decorator_list, lineno, col_offset): self.name = name self.args = args @@ -264,7 +265,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 63: - missing_field(space, self.initialization_state, ['name', 'args', 'body', 'decorator_list', 'lineno', 'col_offset'], 'FunctionDef') + self.missing_field(space, ['lineno', 'col_offset', 'name', 'args', 'body', 'decorator_list'], 'FunctionDef') else: pass self.args.sync_app_attrs(space) @@ -292,9 +293,6 @@ class ClassDef(stmt): - _lineno_mask = 16 - _col_offset_mask = 32 - def __init__(self, name, bases, body, decorator_list, lineno, col_offset): self.name = name self.bases = bases @@ -320,7 +318,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 63: - missing_field(space, self.initialization_state, ['name', 'bases', 'body', 'decorator_list', 'lineno', 'col_offset'], 'ClassDef') + self.missing_field(space, ['lineno', 'col_offset', 'name', 'bases', 'body', 'decorator_list'], 'ClassDef') else: pass w_list = self.w_bases @@ -357,9 +355,6 @@ class Return(stmt): - _lineno_mask = 2 - _col_offset_mask = 4 - def __init__(self, value, lineno, col_offset): self.value = value stmt.__init__(self, lineno, col_offset) @@ -374,10 +369,10 @@ return visitor.visit_Return(self) def sync_app_attrs(self, space): - if (self.initialization_state & ~1) ^ 6: - missing_field(space, self.initialization_state, [None, 'lineno', 'col_offset'], 'Return') + if (self.initialization_state & ~4) ^ 3: + self.missing_field(space, ['lineno', 'col_offset', None], 'Return') else: - if not self.initialization_state & 1: + if not self.initialization_state & 4: self.value = None if self.value: self.value.sync_app_attrs(space) @@ -385,9 +380,6 @@ class Delete(stmt): - _lineno_mask = 2 - _col_offset_mask = 4 - def __init__(self, targets, lineno, col_offset): self.targets = targets self.w_targets = None @@ -404,7 +396,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 7: - missing_field(space, self.initialization_state, ['targets', 'lineno', 'col_offset'], 'Delete') + self.missing_field(space, ['lineno', 'col_offset', 'targets'], 'Delete') else: pass w_list = self.w_targets @@ -421,9 +413,6 @@ class Assign(stmt): - _lineno_mask = 4 - _col_offset_mask = 8 - def __init__(self, targets, value, lineno, col_offset): self.targets = targets self.w_targets = None @@ -442,7 +431,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 15: - missing_field(space, self.initialization_state, ['targets', 'value', 'lineno', 'col_offset'], 'Assign') + self.missing_field(space, ['lineno', 'col_offset', 'targets', 'value'], 'Assign') else: pass w_list = self.w_targets @@ -460,9 +449,6 @@ class AugAssign(stmt): - _lineno_mask = 8 - _col_offset_mask = 16 - def __init__(self, target, op, value, lineno, col_offset): self.target = target self.op = op @@ -480,7 +466,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 31: - missing_field(space, self.initialization_state, ['target', 'op', 'value', 'lineno', 'col_offset'], 'AugAssign') + self.missing_field(space, ['lineno', 'col_offset', 'target', 'op', 'value'], 'AugAssign') else: pass self.target.sync_app_attrs(space) @@ -489,9 +475,6 @@ class Print(stmt): - _lineno_mask = 8 - _col_offset_mask = 16 - def __init__(self, dest, values, nl, lineno, col_offset): self.dest = dest self.values = values @@ -511,10 +494,10 @@ return visitor.visit_Print(self) def sync_app_attrs(self, space): - if (self.initialization_state & ~1) ^ 30: - missing_field(space, self.initialization_state, [None, 'values', 'nl', 'lineno', 'col_offset'], 'Print') + if (self.initialization_state & ~4) ^ 27: + self.missing_field(space, ['lineno', 'col_offset', None, 'values', 'nl'], 'Print') else: - if not self.initialization_state & 1: + if not self.initialization_state & 4: self.dest = None if self.dest: self.dest.sync_app_attrs(space) @@ -532,9 +515,6 @@ class For(stmt): - _lineno_mask = 16 - _col_offset_mask = 32 - def __init__(self, target, iter, body, orelse, lineno, col_offset): self.target = target self.iter = iter @@ -559,7 +539,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 63: - missing_field(space, self.initialization_state, ['target', 'iter', 'body', 'orelse', 'lineno', 'col_offset'], 'For') + self.missing_field(space, ['lineno', 'col_offset', 'target', 'iter', 'body', 'orelse'], 'For') else: pass self.target.sync_app_attrs(space) @@ -588,9 +568,6 @@ class While(stmt): - _lineno_mask = 8 - _col_offset_mask = 16 - def __init__(self, test, body, orelse, lineno, col_offset): self.test = test self.body = body @@ -613,7 +590,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 31: - missing_field(space, self.initialization_state, ['test', 'body', 'orelse', 'lineno', 'col_offset'], 'While') + self.missing_field(space, ['lineno', 'col_offset', 'test', 'body', 'orelse'], 'While') else: pass self.test.sync_app_attrs(space) @@ -641,9 +618,6 @@ class If(stmt): - _lineno_mask = 8 - _col_offset_mask = 16 - def __init__(self, test, body, orelse, lineno, col_offset): self.test = test self.body = body @@ -666,7 +640,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 31: - missing_field(space, self.initialization_state, ['test', 'body', 'orelse', 'lineno', 'col_offset'], 'If') + self.missing_field(space, ['lineno', 'col_offset', 'test', 'body', 'orelse'], 'If') else: pass self.test.sync_app_attrs(space) @@ -694,9 +668,6 @@ class With(stmt): - _lineno_mask = 8 - _col_offset_mask = 16 - def __init__(self, context_expr, optional_vars, body, lineno, col_offset): self.context_expr = context_expr self.optional_vars = optional_vars @@ -717,10 +688,10 @@ return visitor.visit_With(self) def sync_app_attrs(self, space): - if (self.initialization_state & ~2) ^ 29: - missing_field(space, self.initialization_state, ['context_expr', None, 'body', 'lineno', 'col_offset'], 'With') + if (self.initialization_state & ~8) ^ 23: + self.missing_field(space, ['lineno', 'col_offset', 'context_expr', None, 'body'], 'With') else: - if not self.initialization_state & 2: + if not self.initialization_state & 8: self.optional_vars = None self.context_expr.sync_app_attrs(space) if self.optional_vars: @@ -739,9 +710,6 @@ class Raise(stmt): - _lineno_mask = 8 - _col_offset_mask = 16 - def __init__(self, type, inst, tback, lineno, col_offset): self.type = type self.inst = inst @@ -762,14 +730,14 @@ return visitor.visit_Raise(self) def sync_app_attrs(self, space): - if (self.initialization_state & ~7) ^ 24: - missing_field(space, self.initialization_state, [None, None, None, 'lineno', 'col_offset'], 'Raise') + if (self.initialization_state & ~28) ^ 3: + self.missing_field(space, ['lineno', 'col_offset', None, None, None], 'Raise') else: - if not self.initialization_state & 1: + if not self.initialization_state & 4: self.type = None - if not self.initialization_state & 2: + if not self.initialization_state & 8: self.inst = None - if not self.initialization_state & 4: + if not self.initialization_state & 16: self.tback = None if self.type: self.type.sync_app_attrs(space) @@ -781,9 +749,6 @@ class TryExcept(stmt): - _lineno_mask = 8 - _col_offset_mask = 16 - def __init__(self, body, handlers, orelse, lineno, col_offset): self.body = body self.w_body = None @@ -808,7 +773,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 31: - missing_field(space, self.initialization_state, ['body', 'handlers', 'orelse', 'lineno', 'col_offset'], 'TryExcept') + self.missing_field(space, ['lineno', 'col_offset', 'body', 'handlers', 'orelse'], 'TryExcept') else: pass w_list = self.w_body @@ -845,9 +810,6 @@ class TryFinally(stmt): - _lineno_mask = 4 - _col_offset_mask = 8 - def __init__(self, body, finalbody, lineno, col_offset): self.body = body self.w_body = None @@ -868,7 +830,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 15: - missing_field(space, self.initialization_state, ['body', 'finalbody', 'lineno', 'col_offset'], 'TryFinally') + self.missing_field(space, ['lineno', 'col_offset', 'body', 'finalbody'], 'TryFinally') else: pass w_list = self.w_body @@ -895,9 +857,6 @@ class Assert(stmt): - _lineno_mask = 4 - _col_offset_mask = 8 - def __init__(self, test, msg, lineno, col_offset): self.test = test self.msg = msg @@ -914,10 +873,10 @@ return visitor.visit_Assert(self) def sync_app_attrs(self, space): - if (self.initialization_state & ~2) ^ 13: - missing_field(space, self.initialization_state, ['test', None, 'lineno', 'col_offset'], 'Assert') + if (self.initialization_state & ~8) ^ 7: + self.missing_field(space, ['lineno', 'col_offset', 'test', None], 'Assert') else: - if not self.initialization_state & 2: + if not self.initialization_state & 8: self.msg = None self.test.sync_app_attrs(space) if self.msg: @@ -926,9 +885,6 @@ class Import(stmt): - _lineno_mask = 2 - _col_offset_mask = 4 - def __init__(self, names, lineno, col_offset): self.names = names self.w_names = None @@ -945,7 +901,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 7: - missing_field(space, self.initialization_state, ['names', 'lineno', 'col_offset'], 'Import') + self.missing_field(space, ['lineno', 'col_offset', 'names'], 'Import') else: pass w_list = self.w_names @@ -962,9 +918,6 @@ class ImportFrom(stmt): - _lineno_mask = 8 - _col_offset_mask = 16 - def __init__(self, module, names, level, lineno, col_offset): self.module = module self.names = names @@ -982,12 +935,12 @@ return visitor.visit_ImportFrom(self) def sync_app_attrs(self, space): - if (self.initialization_state & ~5) ^ 26: - missing_field(space, self.initialization_state, [None, 'names', None, 'lineno', 'col_offset'], 'ImportFrom') + if (self.initialization_state & ~20) ^ 11: + self.missing_field(space, ['lineno', 'col_offset', None, 'names', None], 'ImportFrom') else: - if not self.initialization_state & 1: + if not self.initialization_state & 4: self.module = None - if not self.initialization_state & 4: + if not self.initialization_state & 16: self.level = 0 w_list = self.w_names if w_list is not None: @@ -1003,9 +956,6 @@ class Exec(stmt): - _lineno_mask = 8 - _col_offset_mask = 16 - def __init__(self, body, globals, locals, lineno, col_offset): self.body = body self.globals = globals @@ -1025,12 +975,12 @@ return visitor.visit_Exec(self) def sync_app_attrs(self, space): - if (self.initialization_state & ~6) ^ 25: - missing_field(space, self.initialization_state, ['body', None, None, 'lineno', 'col_offset'], 'Exec') + if (self.initialization_state & ~24) ^ 7: + self.missing_field(space, ['lineno', 'col_offset', 'body', None, None], 'Exec') else: - if not self.initialization_state & 2: + if not self.initialization_state & 8: self.globals = None - if not self.initialization_state & 4: + if not self.initialization_state & 16: self.locals = None self.body.sync_app_attrs(space) if self.globals: @@ -1041,9 +991,6 @@ class Global(stmt): - _lineno_mask = 2 - _col_offset_mask = 4 - def __init__(self, names, lineno, col_offset): self.names = names self.w_names = None @@ -1058,7 +1005,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 7: - missing_field(space, self.initialization_state, ['names', 'lineno', 'col_offset'], 'Global') + self.missing_field(space, ['lineno', 'col_offset', 'names'], 'Global') else: pass w_list = self.w_names @@ -1072,9 +1019,6 @@ class Expr(stmt): - _lineno_mask = 2 - _col_offset_mask = 4 - def __init__(self, value, lineno, col_offset): self.value = value stmt.__init__(self, lineno, col_offset) @@ -1089,7 +1033,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 7: - missing_field(space, self.initialization_state, ['value', 'lineno', 'col_offset'], 'Expr') + self.missing_field(space, ['lineno', 'col_offset', 'value'], 'Expr') else: pass self.value.sync_app_attrs(space) @@ -1097,9 +1041,6 @@ class Pass(stmt): - _lineno_mask = 1 - _col_offset_mask = 2 - def __init__(self, lineno, col_offset): stmt.__init__(self, lineno, col_offset) self.initialization_state = 3 @@ -1112,16 +1053,13 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 3: - missing_field(space, self.initialization_state, ['lineno', 'col_offset'], 'Pass') + self.missing_field(space, ['lineno', 'col_offset'], 'Pass') else: pass class Break(stmt): - _lineno_mask = 1 - _col_offset_mask = 2 - def __init__(self, lineno, col_offset): stmt.__init__(self, lineno, col_offset) self.initialization_state = 3 @@ -1134,16 +1072,13 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 3: - missing_field(space, self.initialization_state, ['lineno', 'col_offset'], 'Break') + self.missing_field(space, ['lineno', 'col_offset'], 'Break') else: pass class Continue(stmt): - _lineno_mask = 1 - _col_offset_mask = 2 - def __init__(self, lineno, col_offset): stmt.__init__(self, lineno, col_offset) self.initialization_state = 3 @@ -1156,21 +1091,19 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 3: - missing_field(space, self.initialization_state, ['lineno', 'col_offset'], 'Continue') + self.missing_field(space, ['lineno', 'col_offset'], 'Continue') else: pass class expr(AST): + def __init__(self, lineno, col_offset): self.lineno = lineno self.col_offset = col_offset class BoolOp(expr): - _lineno_mask = 4 - _col_offset_mask = 8 - def __init__(self, op, values, lineno, col_offset): self.op = op self.values = values @@ -1188,7 +1121,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 15: - missing_field(space, self.initialization_state, ['op', 'values', 'lineno', 'col_offset'], 'BoolOp') + self.missing_field(space, ['lineno', 'col_offset', 'op', 'values'], 'BoolOp') else: pass w_list = self.w_values @@ -1205,9 +1138,6 @@ class BinOp(expr): - _lineno_mask = 8 - _col_offset_mask = 16 - def __init__(self, left, op, right, lineno, col_offset): self.left = left self.op = op @@ -1225,7 +1155,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 31: - missing_field(space, self.initialization_state, ['left', 'op', 'right', 'lineno', 'col_offset'], 'BinOp') + self.missing_field(space, ['lineno', 'col_offset', 'left', 'op', 'right'], 'BinOp') else: pass self.left.sync_app_attrs(space) @@ -1234,9 +1164,6 @@ class UnaryOp(expr): - _lineno_mask = 4 - _col_offset_mask = 8 - def __init__(self, op, operand, lineno, col_offset): self.op = op self.operand = operand @@ -1252,7 +1179,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 15: - missing_field(space, self.initialization_state, ['op', 'operand', 'lineno', 'col_offset'], 'UnaryOp') + self.missing_field(space, ['lineno', 'col_offset', 'op', 'operand'], 'UnaryOp') else: pass self.operand.sync_app_attrs(space) @@ -1260,9 +1187,6 @@ class Lambda(expr): - _lineno_mask = 4 - _col_offset_mask = 8 - def __init__(self, args, body, lineno, col_offset): self.args = args self.body = body @@ -1279,7 +1203,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 15: - missing_field(space, self.initialization_state, ['args', 'body', 'lineno', 'col_offset'], 'Lambda') + self.missing_field(space, ['lineno', 'col_offset', 'args', 'body'], 'Lambda') else: pass self.args.sync_app_attrs(space) @@ -1288,9 +1212,6 @@ class IfExp(expr): - _lineno_mask = 8 - _col_offset_mask = 16 - def __init__(self, test, body, orelse, lineno, col_offset): self.test = test self.body = body @@ -1309,7 +1230,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 31: - missing_field(space, self.initialization_state, ['test', 'body', 'orelse', 'lineno', 'col_offset'], 'IfExp') + self.missing_field(space, ['lineno', 'col_offset', 'test', 'body', 'orelse'], 'IfExp') else: pass self.test.sync_app_attrs(space) @@ -1319,9 +1240,6 @@ class Dict(expr): - _lineno_mask = 4 - _col_offset_mask = 8 - def __init__(self, keys, values, lineno, col_offset): self.keys = keys self.w_keys = None @@ -1342,7 +1260,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 15: - missing_field(space, self.initialization_state, ['keys', 'values', 'lineno', 'col_offset'], 'Dict') + self.missing_field(space, ['lineno', 'col_offset', 'keys', 'values'], 'Dict') else: pass w_list = self.w_keys @@ -1369,9 +1287,6 @@ class Set(expr): - _lineno_mask = 2 - _col_offset_mask = 4 - def __init__(self, elts, lineno, col_offset): self.elts = elts self.w_elts = None @@ -1388,7 +1303,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 7: - missing_field(space, self.initialization_state, ['elts', 'lineno', 'col_offset'], 'Set') + self.missing_field(space, ['lineno', 'col_offset', 'elts'], 'Set') else: pass w_list = self.w_elts @@ -1405,9 +1320,6 @@ class ListComp(expr): - _lineno_mask = 4 - _col_offset_mask = 8 - def __init__(self, elt, generators, lineno, col_offset): self.elt = elt self.generators = generators @@ -1426,7 +1338,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 15: - missing_field(space, self.initialization_state, ['elt', 'generators', 'lineno', 'col_offset'], 'ListComp') + self.missing_field(space, ['lineno', 'col_offset', 'elt', 'generators'], 'ListComp') else: pass self.elt.sync_app_attrs(space) @@ -1444,9 +1356,6 @@ class SetComp(expr): - _lineno_mask = 4 - _col_offset_mask = 8 - def __init__(self, elt, generators, lineno, col_offset): self.elt = elt self.generators = generators @@ -1465,7 +1374,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 15: - missing_field(space, self.initialization_state, ['elt', 'generators', 'lineno', 'col_offset'], 'SetComp') + self.missing_field(space, ['lineno', 'col_offset', 'elt', 'generators'], 'SetComp') else: pass self.elt.sync_app_attrs(space) @@ -1483,9 +1392,6 @@ class DictComp(expr): - _lineno_mask = 8 - _col_offset_mask = 16 - def __init__(self, key, value, generators, lineno, col_offset): self.key = key self.value = value @@ -1506,7 +1412,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 31: - missing_field(space, self.initialization_state, ['key', 'value', 'generators', 'lineno', 'col_offset'], 'DictComp') + self.missing_field(space, ['lineno', 'col_offset', 'key', 'value', 'generators'], 'DictComp') else: pass self.key.sync_app_attrs(space) @@ -1525,9 +1431,6 @@ class GeneratorExp(expr): - _lineno_mask = 4 - _col_offset_mask = 8 - def __init__(self, elt, generators, lineno, col_offset): self.elt = elt self.generators = generators @@ -1546,7 +1449,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 15: - missing_field(space, self.initialization_state, ['elt', 'generators', 'lineno', 'col_offset'], 'GeneratorExp') + self.missing_field(space, ['lineno', 'col_offset', 'elt', 'generators'], 'GeneratorExp') else: pass self.elt.sync_app_attrs(space) @@ -1564,9 +1467,6 @@ class Yield(expr): - _lineno_mask = 2 - _col_offset_mask = 4 - def __init__(self, value, lineno, col_offset): self.value = value expr.__init__(self, lineno, col_offset) @@ -1581,10 +1481,10 @@ return visitor.visit_Yield(self) def sync_app_attrs(self, space): - if (self.initialization_state & ~1) ^ 6: - missing_field(space, self.initialization_state, [None, 'lineno', 'col_offset'], 'Yield') + if (self.initialization_state & ~4) ^ 3: + self.missing_field(space, ['lineno', 'col_offset', None], 'Yield') else: - if not self.initialization_state & 1: + if not self.initialization_state & 4: self.value = None if self.value: self.value.sync_app_attrs(space) @@ -1592,9 +1492,6 @@ class Compare(expr): - _lineno_mask = 8 - _col_offset_mask = 16 - def __init__(self, left, ops, comparators, lineno, col_offset): self.left = left self.ops = ops @@ -1615,7 +1512,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 31: - missing_field(space, self.initialization_state, ['left', 'ops', 'comparators', 'lineno', 'col_offset'], 'Compare') + self.missing_field(space, ['lineno', 'col_offset', 'left', 'ops', 'comparators'], 'Compare') else: pass self.left.sync_app_attrs(space) @@ -1640,9 +1537,6 @@ class Call(expr): - _lineno_mask = 32 - _col_offset_mask = 64 - def __init__(self, func, args, keywords, starargs, kwargs, lineno, col_offset): self.func = func self.args = args @@ -1670,12 +1564,12 @@ return visitor.visit_Call(self) def sync_app_attrs(self, space): - if (self.initialization_state & ~24) ^ 103: - missing_field(space, self.initialization_state, ['func', 'args', 'keywords', None, None, 'lineno', 'col_offset'], 'Call') + if (self.initialization_state & ~96) ^ 31: + self.missing_field(space, ['lineno', 'col_offset', 'func', 'args', 'keywords', None, None], 'Call') else: - if not self.initialization_state & 8: + if not self.initialization_state & 32: self.starargs = None - if not self.initialization_state & 16: + if not self.initialization_state & 64: self.kwargs = None self.func.sync_app_attrs(space) w_list = self.w_args @@ -1706,9 +1600,6 @@ class Repr(expr): - _lineno_mask = 2 - _col_offset_mask = 4 - def __init__(self, value, lineno, col_offset): self.value = value expr.__init__(self, lineno, col_offset) @@ -1723,7 +1614,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 7: - missing_field(space, self.initialization_state, ['value', 'lineno', 'col_offset'], 'Repr') + self.missing_field(space, ['lineno', 'col_offset', 'value'], 'Repr') else: pass self.value.sync_app_attrs(space) @@ -1731,9 +1622,6 @@ class Num(expr): - _lineno_mask = 2 - _col_offset_mask = 4 - def __init__(self, n, lineno, col_offset): self.n = n expr.__init__(self, lineno, col_offset) @@ -1747,16 +1635,13 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 7: - missing_field(space, self.initialization_state, ['n', 'lineno', 'col_offset'], 'Num') + self.missing_field(space, ['lineno', 'col_offset', 'n'], 'Num') else: pass class Str(expr): - _lineno_mask = 2 - _col_offset_mask = 4 - def __init__(self, s, lineno, col_offset): self.s = s expr.__init__(self, lineno, col_offset) @@ -1770,16 +1655,13 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 7: - missing_field(space, self.initialization_state, ['s', 'lineno', 'col_offset'], 'Str') + self.missing_field(space, ['lineno', 'col_offset', 's'], 'Str') else: pass class Attribute(expr): - _lineno_mask = 8 - _col_offset_mask = 16 - def __init__(self, value, attr, ctx, lineno, col_offset): self.value = value self.attr = attr @@ -1796,7 +1678,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 31: - missing_field(space, self.initialization_state, ['value', 'attr', 'ctx', 'lineno', 'col_offset'], 'Attribute') + self.missing_field(space, ['lineno', 'col_offset', 'value', 'attr', 'ctx'], 'Attribute') else: pass self.value.sync_app_attrs(space) @@ -1804,9 +1686,6 @@ class Subscript(expr): - _lineno_mask = 8 - _col_offset_mask = 16 - def __init__(self, value, slice, ctx, lineno, col_offset): self.value = value self.slice = slice @@ -1824,7 +1703,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 31: - missing_field(space, self.initialization_state, ['value', 'slice', 'ctx', 'lineno', 'col_offset'], 'Subscript') + self.missing_field(space, ['lineno', 'col_offset', 'value', 'slice', 'ctx'], 'Subscript') else: pass self.value.sync_app_attrs(space) @@ -1833,9 +1712,6 @@ class Name(expr): - _lineno_mask = 4 - _col_offset_mask = 8 - def __init__(self, id, ctx, lineno, col_offset): self.id = id self.ctx = ctx @@ -1850,16 +1726,13 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 15: - missing_field(space, self.initialization_state, ['id', 'ctx', 'lineno', 'col_offset'], 'Name') + self.missing_field(space, ['lineno', 'col_offset', 'id', 'ctx'], 'Name') else: pass class List(expr): - _lineno_mask = 4 - _col_offset_mask = 8 - def __init__(self, elts, ctx, lineno, col_offset): self.elts = elts self.w_elts = None @@ -1877,7 +1750,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 15: - missing_field(space, self.initialization_state, ['elts', 'ctx', 'lineno', 'col_offset'], 'List') + self.missing_field(space, ['lineno', 'col_offset', 'elts', 'ctx'], 'List') else: pass w_list = self.w_elts @@ -1894,9 +1767,6 @@ class Tuple(expr): - _lineno_mask = 4 - _col_offset_mask = 8 - def __init__(self, elts, ctx, lineno, col_offset): self.elts = elts self.w_elts = None @@ -1914,7 +1784,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 15: - missing_field(space, self.initialization_state, ['elts', 'ctx', 'lineno', 'col_offset'], 'Tuple') + self.missing_field(space, ['lineno', 'col_offset', 'elts', 'ctx'], 'Tuple') else: pass w_list = self.w_elts @@ -1931,9 +1801,6 @@ class Const(expr): - _lineno_mask = 2 - _col_offset_mask = 4 - def __init__(self, value, lineno, col_offset): self.value = value expr.__init__(self, lineno, col_offset) @@ -1947,7 +1814,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 7: - missing_field(space, self.initialization_state, ['value', 'lineno', 'col_offset'], 'Const') + self.missing_field(space, ['lineno', 'col_offset', 'value'], 'Const') else: pass @@ -2009,7 +1876,6 @@ class Ellipsis(slice): - def __init__(self): self.initialization_state = 0 @@ -2021,14 +1887,13 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 0: - missing_field(space, self.initialization_state, [], 'Ellipsis') + self.missing_field(space, [], 'Ellipsis') else: pass class Slice(slice): - def __init__(self, lower, upper, step): self.lower = lower self.upper = upper @@ -2049,7 +1914,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~7) ^ 0: - missing_field(space, self.initialization_state, [None, None, None], 'Slice') + self.missing_field(space, [None, None, None], 'Slice') else: if not self.initialization_state & 1: self.lower = None @@ -2067,7 +1932,6 @@ class ExtSlice(slice): - def __init__(self, dims): self.dims = dims self.w_dims = None @@ -2083,7 +1947,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 1: - missing_field(space, self.initialization_state, ['dims'], 'ExtSlice') + self.missing_field(space, ['dims'], 'ExtSlice') else: pass w_list = self.w_dims @@ -2100,7 +1964,6 @@ class Index(slice): - def __init__(self, value): self.value = value self.initialization_state = 1 @@ -2114,7 +1977,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 1: - missing_field(space, self.initialization_state, ['value'], 'Index') + self.missing_field(space, ['value'], 'Index') else: pass self.value.sync_app_attrs(space) @@ -2377,7 +2240,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 7: - missing_field(space, self.initialization_state, ['target', 'iter', 'ifs'], 'comprehension') + self.missing_field(space, ['target', 'iter', 'ifs'], 'comprehension') else: pass self.target.sync_app_attrs(space) @@ -2394,15 +2257,13 @@ node.sync_app_attrs(space) class excepthandler(AST): + def __init__(self, lineno, col_offset): self.lineno = lineno self.col_offset = col_offset class ExceptHandler(excepthandler): - _lineno_mask = 8 - _col_offset_mask = 16 - def __init__(self, type, name, body, lineno, col_offset): self.type = type self.name = name @@ -2424,12 +2285,12 @@ return visitor.visit_ExceptHandler(self) def sync_app_attrs(self, space): - if (self.initialization_state & ~3) ^ 28: - missing_field(space, self.initialization_state, [None, None, 'body', 'lineno', 'col_offset'], 'ExceptHandler') + if (self.initialization_state & ~12) ^ 19: + self.missing_field(space, ['lineno', 'col_offset', None, None, 'body'], 'ExceptHandler') else: - if not self.initialization_state & 1: + if not self.initialization_state & 4: self.type = None - if not self.initialization_state & 2: + if not self.initialization_state & 8: self.name = None if self.type: self.type.sync_app_attrs(space) @@ -2470,7 +2331,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~6) ^ 9: - missing_field(space, self.initialization_state, ['args', None, None, 'defaults'], 'arguments') + self.missing_field(space, ['args', None, None, 'defaults'], 'arguments') else: if not self.initialization_state & 2: self.vararg = None @@ -2513,7 +2374,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 3: - missing_field(space, self.initialization_state, ['arg', 'value'], 'keyword') + self.missing_field(space, ['arg', 'value'], 'keyword') else: pass self.value.sync_app_attrs(space) @@ -2533,7 +2394,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~2) ^ 1: - missing_field(space, self.initialization_state, ['name', None], 'alias') + self.missing_field(space, ['name', None], 'alias') else: if not self.initialization_state & 2: self.asname = None @@ -3019,6 +2880,8 @@ def Expression_set_body(space, w_self, w_new_value): try: w_self.body = space.interp_w(expr, w_new_value, False) + if type(w_self.body) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise @@ -3098,7 +2961,7 @@ w_obj = w_self.getdictvalue(space, 'lineno') if w_obj is not None: return w_obj - if not w_self.initialization_state & w_self._lineno_mask: + if not w_self.initialization_state & 1: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'lineno') return space.wrap(w_self.lineno) @@ -3112,14 +2975,14 @@ w_self.setdictvalue(space, 'lineno', w_new_value) return w_self.deldictvalue(space, 'lineno') - w_self.initialization_state |= w_self._lineno_mask + w_self.initialization_state |= 1 def stmt_get_col_offset(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'col_offset') if w_obj is not None: return w_obj - if not w_self.initialization_state & w_self._col_offset_mask: + if not w_self.initialization_state & 2: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'col_offset') return space.wrap(w_self.col_offset) @@ -3133,7 +2996,7 @@ w_self.setdictvalue(space, 'col_offset', w_new_value) return w_self.deldictvalue(space, 'col_offset') - w_self.initialization_state |= w_self._col_offset_mask + w_self.initialization_state |= 2 stmt.typedef = typedef.TypeDef("stmt", AST.typedef, @@ -3149,7 +3012,7 @@ w_obj = w_self.getdictvalue(space, 'name') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'name') return space.wrap(w_self.name) @@ -3163,14 +3026,14 @@ w_self.setdictvalue(space, 'name', w_new_value) return w_self.deldictvalue(space, 'name') - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 def FunctionDef_get_args(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'args') if w_obj is not None: return w_obj - if not w_self.initialization_state & 2: + if not w_self.initialization_state & 8: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'args') return space.wrap(w_self.args) @@ -3184,10 +3047,10 @@ w_self.setdictvalue(space, 'args', w_new_value) return w_self.deldictvalue(space, 'args') - w_self.initialization_state |= 2 + w_self.initialization_state |= 8 def FunctionDef_get_body(space, w_self): - if not w_self.initialization_state & 4: + if not w_self.initialization_state & 16: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body') if w_self.w_body is None: @@ -3201,10 +3064,10 @@ def FunctionDef_set_body(space, w_self, w_new_value): w_self.w_body = w_new_value - w_self.initialization_state |= 4 + w_self.initialization_state |= 16 def FunctionDef_get_decorator_list(space, w_self): - if not w_self.initialization_state & 8: + if not w_self.initialization_state & 32: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'decorator_list') if w_self.w_decorator_list is None: @@ -3218,7 +3081,7 @@ def FunctionDef_set_decorator_list(space, w_self, w_new_value): w_self.w_decorator_list = w_new_value - w_self.initialization_state |= 8 + w_self.initialization_state |= 32 _FunctionDef_field_unroller = unrolling_iterable(['name', 'args', 'body', 'decorator_list']) def FunctionDef_init(space, w_self, __args__): @@ -3254,7 +3117,7 @@ w_obj = w_self.getdictvalue(space, 'name') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'name') return space.wrap(w_self.name) @@ -3268,10 +3131,10 @@ w_self.setdictvalue(space, 'name', w_new_value) return w_self.deldictvalue(space, 'name') - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 def ClassDef_get_bases(space, w_self): - if not w_self.initialization_state & 2: + if not w_self.initialization_state & 8: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'bases') if w_self.w_bases is None: @@ -3285,10 +3148,10 @@ def ClassDef_set_bases(space, w_self, w_new_value): w_self.w_bases = w_new_value - w_self.initialization_state |= 2 + w_self.initialization_state |= 8 def ClassDef_get_body(space, w_self): - if not w_self.initialization_state & 4: + if not w_self.initialization_state & 16: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body') if w_self.w_body is None: @@ -3302,10 +3165,10 @@ def ClassDef_set_body(space, w_self, w_new_value): w_self.w_body = w_new_value - w_self.initialization_state |= 4 + w_self.initialization_state |= 16 def ClassDef_get_decorator_list(space, w_self): - if not w_self.initialization_state & 8: + if not w_self.initialization_state & 32: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'decorator_list') if w_self.w_decorator_list is None: @@ -3319,7 +3182,7 @@ def ClassDef_set_decorator_list(space, w_self, w_new_value): w_self.w_decorator_list = w_new_value - w_self.initialization_state |= 8 + w_self.initialization_state |= 32 _ClassDef_field_unroller = unrolling_iterable(['name', 'bases', 'body', 'decorator_list']) def ClassDef_init(space, w_self, __args__): @@ -3356,7 +3219,7 @@ w_obj = w_self.getdictvalue(space, 'value') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value') return space.wrap(w_self.value) @@ -3364,13 +3227,15 @@ def Return_set_value(space, w_self, w_new_value): try: w_self.value = space.interp_w(expr, w_new_value, True) + if type(w_self.value) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'value', w_new_value) return w_self.deldictvalue(space, 'value') - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 _Return_field_unroller = unrolling_iterable(['value']) def Return_init(space, w_self, __args__): @@ -3397,7 +3262,7 @@ ) def Delete_get_targets(space, w_self): - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'targets') if w_self.w_targets is None: @@ -3411,7 +3276,7 @@ def Delete_set_targets(space, w_self, w_new_value): w_self.w_targets = w_new_value - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 _Delete_field_unroller = unrolling_iterable(['targets']) def Delete_init(space, w_self, __args__): @@ -3439,7 +3304,7 @@ ) def Assign_get_targets(space, w_self): - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'targets') if w_self.w_targets is None: @@ -3453,14 +3318,14 @@ def Assign_set_targets(space, w_self, w_new_value): w_self.w_targets = w_new_value - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 def Assign_get_value(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'value') if w_obj is not None: return w_obj - if not w_self.initialization_state & 2: + if not w_self.initialization_state & 8: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value') return space.wrap(w_self.value) @@ -3468,13 +3333,15 @@ def Assign_set_value(space, w_self, w_new_value): try: w_self.value = space.interp_w(expr, w_new_value, False) + if type(w_self.value) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'value', w_new_value) return w_self.deldictvalue(space, 'value') - w_self.initialization_state |= 2 + w_self.initialization_state |= 8 _Assign_field_unroller = unrolling_iterable(['targets', 'value']) def Assign_init(space, w_self, __args__): @@ -3507,7 +3374,7 @@ w_obj = w_self.getdictvalue(space, 'target') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'target') return space.wrap(w_self.target) @@ -3515,20 +3382,22 @@ def AugAssign_set_target(space, w_self, w_new_value): try: w_self.target = space.interp_w(expr, w_new_value, False) + if type(w_self.target) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'target', w_new_value) return w_self.deldictvalue(space, 'target') - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 def AugAssign_get_op(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'op') if w_obj is not None: return w_obj - if not w_self.initialization_state & 2: + if not w_self.initialization_state & 8: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'op') return operator_to_class[w_self.op - 1]() @@ -3544,14 +3413,14 @@ return # need to save the original object too w_self.setdictvalue(space, 'op', w_new_value) - w_self.initialization_state |= 2 + w_self.initialization_state |= 8 def AugAssign_get_value(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'value') if w_obj is not None: return w_obj - if not w_self.initialization_state & 4: + if not w_self.initialization_state & 16: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value') return space.wrap(w_self.value) @@ -3559,13 +3428,15 @@ def AugAssign_set_value(space, w_self, w_new_value): try: w_self.value = space.interp_w(expr, w_new_value, False) + if type(w_self.value) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'value', w_new_value) return w_self.deldictvalue(space, 'value') - w_self.initialization_state |= 4 + w_self.initialization_state |= 16 _AugAssign_field_unroller = unrolling_iterable(['target', 'op', 'value']) def AugAssign_init(space, w_self, __args__): @@ -3598,7 +3469,7 @@ w_obj = w_self.getdictvalue(space, 'dest') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'dest') return space.wrap(w_self.dest) @@ -3606,16 +3477,18 @@ def Print_set_dest(space, w_self, w_new_value): try: w_self.dest = space.interp_w(expr, w_new_value, True) + if type(w_self.dest) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'dest', w_new_value) return w_self.deldictvalue(space, 'dest') - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 def Print_get_values(space, w_self): - if not w_self.initialization_state & 2: + if not w_self.initialization_state & 8: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'values') if w_self.w_values is None: @@ -3629,14 +3502,14 @@ def Print_set_values(space, w_self, w_new_value): w_self.w_values = w_new_value - w_self.initialization_state |= 2 + w_self.initialization_state |= 8 def Print_get_nl(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'nl') if w_obj is not None: return w_obj - if not w_self.initialization_state & 4: + if not w_self.initialization_state & 16: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'nl') return space.wrap(w_self.nl) @@ -3650,7 +3523,7 @@ w_self.setdictvalue(space, 'nl', w_new_value) return w_self.deldictvalue(space, 'nl') - w_self.initialization_state |= 4 + w_self.initialization_state |= 16 _Print_field_unroller = unrolling_iterable(['dest', 'values', 'nl']) def Print_init(space, w_self, __args__): @@ -3684,7 +3557,7 @@ w_obj = w_self.getdictvalue(space, 'target') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'target') return space.wrap(w_self.target) @@ -3692,20 +3565,22 @@ def For_set_target(space, w_self, w_new_value): try: w_self.target = space.interp_w(expr, w_new_value, False) + if type(w_self.target) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'target', w_new_value) return w_self.deldictvalue(space, 'target') - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 def For_get_iter(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'iter') if w_obj is not None: return w_obj - if not w_self.initialization_state & 2: + if not w_self.initialization_state & 8: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'iter') return space.wrap(w_self.iter) @@ -3713,16 +3588,18 @@ def For_set_iter(space, w_self, w_new_value): try: w_self.iter = space.interp_w(expr, w_new_value, False) + if type(w_self.iter) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'iter', w_new_value) return w_self.deldictvalue(space, 'iter') - w_self.initialization_state |= 2 + w_self.initialization_state |= 8 def For_get_body(space, w_self): - if not w_self.initialization_state & 4: + if not w_self.initialization_state & 16: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body') if w_self.w_body is None: @@ -3736,10 +3613,10 @@ def For_set_body(space, w_self, w_new_value): w_self.w_body = w_new_value - w_self.initialization_state |= 4 + w_self.initialization_state |= 16 def For_get_orelse(space, w_self): - if not w_self.initialization_state & 8: + if not w_self.initialization_state & 32: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse') if w_self.w_orelse is None: @@ -3753,7 +3630,7 @@ def For_set_orelse(space, w_self, w_new_value): w_self.w_orelse = w_new_value - w_self.initialization_state |= 8 + w_self.initialization_state |= 32 _For_field_unroller = unrolling_iterable(['target', 'iter', 'body', 'orelse']) def For_init(space, w_self, __args__): @@ -3789,7 +3666,7 @@ w_obj = w_self.getdictvalue(space, 'test') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'test') return space.wrap(w_self.test) @@ -3797,16 +3674,18 @@ def While_set_test(space, w_self, w_new_value): try: w_self.test = space.interp_w(expr, w_new_value, False) + if type(w_self.test) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'test', w_new_value) return w_self.deldictvalue(space, 'test') - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 def While_get_body(space, w_self): - if not w_self.initialization_state & 2: + if not w_self.initialization_state & 8: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body') if w_self.w_body is None: @@ -3820,10 +3699,10 @@ def While_set_body(space, w_self, w_new_value): w_self.w_body = w_new_value - w_self.initialization_state |= 2 + w_self.initialization_state |= 8 def While_get_orelse(space, w_self): - if not w_self.initialization_state & 4: + if not w_self.initialization_state & 16: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse') if w_self.w_orelse is None: @@ -3837,7 +3716,7 @@ def While_set_orelse(space, w_self, w_new_value): w_self.w_orelse = w_new_value - w_self.initialization_state |= 4 + w_self.initialization_state |= 16 _While_field_unroller = unrolling_iterable(['test', 'body', 'orelse']) def While_init(space, w_self, __args__): @@ -3872,7 +3751,7 @@ w_obj = w_self.getdictvalue(space, 'test') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'test') return space.wrap(w_self.test) @@ -3880,16 +3759,18 @@ def If_set_test(space, w_self, w_new_value): try: w_self.test = space.interp_w(expr, w_new_value, False) + if type(w_self.test) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'test', w_new_value) return w_self.deldictvalue(space, 'test') - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 def If_get_body(space, w_self): - if not w_self.initialization_state & 2: + if not w_self.initialization_state & 8: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body') if w_self.w_body is None: @@ -3903,10 +3784,10 @@ def If_set_body(space, w_self, w_new_value): w_self.w_body = w_new_value - w_self.initialization_state |= 2 + w_self.initialization_state |= 8 def If_get_orelse(space, w_self): - if not w_self.initialization_state & 4: + if not w_self.initialization_state & 16: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse') if w_self.w_orelse is None: @@ -3920,7 +3801,7 @@ def If_set_orelse(space, w_self, w_new_value): w_self.w_orelse = w_new_value - w_self.initialization_state |= 4 + w_self.initialization_state |= 16 _If_field_unroller = unrolling_iterable(['test', 'body', 'orelse']) def If_init(space, w_self, __args__): @@ -3955,7 +3836,7 @@ w_obj = w_self.getdictvalue(space, 'context_expr') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'context_expr') return space.wrap(w_self.context_expr) @@ -3963,20 +3844,22 @@ def With_set_context_expr(space, w_self, w_new_value): try: w_self.context_expr = space.interp_w(expr, w_new_value, False) + if type(w_self.context_expr) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'context_expr', w_new_value) return w_self.deldictvalue(space, 'context_expr') - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 def With_get_optional_vars(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'optional_vars') if w_obj is not None: return w_obj - if not w_self.initialization_state & 2: + if not w_self.initialization_state & 8: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'optional_vars') return space.wrap(w_self.optional_vars) @@ -3984,16 +3867,18 @@ def With_set_optional_vars(space, w_self, w_new_value): try: w_self.optional_vars = space.interp_w(expr, w_new_value, True) + if type(w_self.optional_vars) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'optional_vars', w_new_value) return w_self.deldictvalue(space, 'optional_vars') - w_self.initialization_state |= 2 + w_self.initialization_state |= 8 def With_get_body(space, w_self): - if not w_self.initialization_state & 4: + if not w_self.initialization_state & 16: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body') if w_self.w_body is None: @@ -4007,7 +3892,7 @@ def With_set_body(space, w_self, w_new_value): w_self.w_body = w_new_value - w_self.initialization_state |= 4 + w_self.initialization_state |= 16 _With_field_unroller = unrolling_iterable(['context_expr', 'optional_vars', 'body']) def With_init(space, w_self, __args__): @@ -4041,7 +3926,7 @@ w_obj = w_self.getdictvalue(space, 'type') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'type') return space.wrap(w_self.type) @@ -4049,20 +3934,22 @@ def Raise_set_type(space, w_self, w_new_value): try: w_self.type = space.interp_w(expr, w_new_value, True) + if type(w_self.type) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'type', w_new_value) return w_self.deldictvalue(space, 'type') - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 def Raise_get_inst(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'inst') if w_obj is not None: return w_obj - if not w_self.initialization_state & 2: + if not w_self.initialization_state & 8: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'inst') return space.wrap(w_self.inst) @@ -4070,20 +3957,22 @@ def Raise_set_inst(space, w_self, w_new_value): try: w_self.inst = space.interp_w(expr, w_new_value, True) + if type(w_self.inst) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'inst', w_new_value) return w_self.deldictvalue(space, 'inst') - w_self.initialization_state |= 2 + w_self.initialization_state |= 8 def Raise_get_tback(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'tback') if w_obj is not None: return w_obj - if not w_self.initialization_state & 4: + if not w_self.initialization_state & 16: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'tback') return space.wrap(w_self.tback) @@ -4091,13 +3980,15 @@ def Raise_set_tback(space, w_self, w_new_value): try: w_self.tback = space.interp_w(expr, w_new_value, True) + if type(w_self.tback) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'tback', w_new_value) return w_self.deldictvalue(space, 'tback') - w_self.initialization_state |= 4 + w_self.initialization_state |= 16 _Raise_field_unroller = unrolling_iterable(['type', 'inst', 'tback']) def Raise_init(space, w_self, __args__): @@ -4126,7 +4017,7 @@ ) def TryExcept_get_body(space, w_self): - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body') if w_self.w_body is None: @@ -4140,10 +4031,10 @@ def TryExcept_set_body(space, w_self, w_new_value): w_self.w_body = w_new_value - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 def TryExcept_get_handlers(space, w_self): - if not w_self.initialization_state & 2: + if not w_self.initialization_state & 8: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'handlers') if w_self.w_handlers is None: @@ -4157,10 +4048,10 @@ def TryExcept_set_handlers(space, w_self, w_new_value): w_self.w_handlers = w_new_value - w_self.initialization_state |= 2 + w_self.initialization_state |= 8 def TryExcept_get_orelse(space, w_self): - if not w_self.initialization_state & 4: + if not w_self.initialization_state & 16: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse') if w_self.w_orelse is None: @@ -4174,7 +4065,7 @@ def TryExcept_set_orelse(space, w_self, w_new_value): w_self.w_orelse = w_new_value - w_self.initialization_state |= 4 + w_self.initialization_state |= 16 _TryExcept_field_unroller = unrolling_iterable(['body', 'handlers', 'orelse']) def TryExcept_init(space, w_self, __args__): @@ -4206,7 +4097,7 @@ ) def TryFinally_get_body(space, w_self): - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body') if w_self.w_body is None: @@ -4220,10 +4111,10 @@ def TryFinally_set_body(space, w_self, w_new_value): w_self.w_body = w_new_value - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 def TryFinally_get_finalbody(space, w_self): - if not w_self.initialization_state & 2: + if not w_self.initialization_state & 8: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'finalbody') if w_self.w_finalbody is None: @@ -4237,7 +4128,7 @@ def TryFinally_set_finalbody(space, w_self, w_new_value): w_self.w_finalbody = w_new_value - w_self.initialization_state |= 2 + w_self.initialization_state |= 8 _TryFinally_field_unroller = unrolling_iterable(['body', 'finalbody']) def TryFinally_init(space, w_self, __args__): @@ -4271,7 +4162,7 @@ w_obj = w_self.getdictvalue(space, 'test') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'test') return space.wrap(w_self.test) @@ -4279,20 +4170,22 @@ def Assert_set_test(space, w_self, w_new_value): try: w_self.test = space.interp_w(expr, w_new_value, False) + if type(w_self.test) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'test', w_new_value) return w_self.deldictvalue(space, 'test') - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 def Assert_get_msg(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'msg') if w_obj is not None: return w_obj - if not w_self.initialization_state & 2: + if not w_self.initialization_state & 8: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'msg') return space.wrap(w_self.msg) @@ -4300,13 +4193,15 @@ def Assert_set_msg(space, w_self, w_new_value): try: w_self.msg = space.interp_w(expr, w_new_value, True) + if type(w_self.msg) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'msg', w_new_value) return w_self.deldictvalue(space, 'msg') - w_self.initialization_state |= 2 + w_self.initialization_state |= 8 _Assert_field_unroller = unrolling_iterable(['test', 'msg']) def Assert_init(space, w_self, __args__): @@ -4334,7 +4229,7 @@ ) def Import_get_names(space, w_self): - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'names') if w_self.w_names is None: @@ -4348,7 +4243,7 @@ def Import_set_names(space, w_self, w_new_value): w_self.w_names = w_new_value - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 _Import_field_unroller = unrolling_iterable(['names']) def Import_init(space, w_self, __args__): @@ -4380,7 +4275,7 @@ w_obj = w_self.getdictvalue(space, 'module') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'module') return space.wrap(w_self.module) @@ -4397,10 +4292,10 @@ w_self.setdictvalue(space, 'module', w_new_value) return w_self.deldictvalue(space, 'module') - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 def ImportFrom_get_names(space, w_self): - if not w_self.initialization_state & 2: + if not w_self.initialization_state & 8: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'names') if w_self.w_names is None: @@ -4414,14 +4309,14 @@ def ImportFrom_set_names(space, w_self, w_new_value): w_self.w_names = w_new_value - w_self.initialization_state |= 2 + w_self.initialization_state |= 8 def ImportFrom_get_level(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'level') if w_obj is not None: return w_obj - if not w_self.initialization_state & 4: + if not w_self.initialization_state & 16: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'level') return space.wrap(w_self.level) @@ -4435,7 +4330,7 @@ w_self.setdictvalue(space, 'level', w_new_value) return w_self.deldictvalue(space, 'level') - w_self.initialization_state |= 4 + w_self.initialization_state |= 16 _ImportFrom_field_unroller = unrolling_iterable(['module', 'names', 'level']) def ImportFrom_init(space, w_self, __args__): @@ -4469,7 +4364,7 @@ w_obj = w_self.getdictvalue(space, 'body') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body') return space.wrap(w_self.body) @@ -4477,20 +4372,22 @@ def Exec_set_body(space, w_self, w_new_value): try: w_self.body = space.interp_w(expr, w_new_value, False) + if type(w_self.body) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'body', w_new_value) return w_self.deldictvalue(space, 'body') - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 def Exec_get_globals(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'globals') if w_obj is not None: return w_obj - if not w_self.initialization_state & 2: + if not w_self.initialization_state & 8: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'globals') return space.wrap(w_self.globals) @@ -4498,20 +4395,22 @@ def Exec_set_globals(space, w_self, w_new_value): try: w_self.globals = space.interp_w(expr, w_new_value, True) + if type(w_self.globals) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'globals', w_new_value) return w_self.deldictvalue(space, 'globals') - w_self.initialization_state |= 2 + w_self.initialization_state |= 8 def Exec_get_locals(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'locals') if w_obj is not None: return w_obj - if not w_self.initialization_state & 4: + if not w_self.initialization_state & 16: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'locals') return space.wrap(w_self.locals) @@ -4519,13 +4418,15 @@ def Exec_set_locals(space, w_self, w_new_value): try: w_self.locals = space.interp_w(expr, w_new_value, True) + if type(w_self.locals) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'locals', w_new_value) return w_self.deldictvalue(space, 'locals') - w_self.initialization_state |= 4 + w_self.initialization_state |= 16 _Exec_field_unroller = unrolling_iterable(['body', 'globals', 'locals']) def Exec_init(space, w_self, __args__): @@ -4554,7 +4455,7 @@ ) def Global_get_names(space, w_self): - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'names') if w_self.w_names is None: @@ -4568,7 +4469,7 @@ def Global_set_names(space, w_self, w_new_value): w_self.w_names = w_new_value - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 _Global_field_unroller = unrolling_iterable(['names']) def Global_init(space, w_self, __args__): @@ -4600,7 +4501,7 @@ w_obj = w_self.getdictvalue(space, 'value') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value') return space.wrap(w_self.value) @@ -4608,13 +4509,15 @@ def Expr_set_value(space, w_self, w_new_value): try: w_self.value = space.interp_w(expr, w_new_value, False) + if type(w_self.value) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'value', w_new_value) return w_self.deldictvalue(space, 'value') - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 _Expr_field_unroller = unrolling_iterable(['value']) def Expr_init(space, w_self, __args__): @@ -4696,7 +4599,7 @@ w_obj = w_self.getdictvalue(space, 'lineno') if w_obj is not None: return w_obj - if not w_self.initialization_state & w_self._lineno_mask: + if not w_self.initialization_state & 1: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'lineno') return space.wrap(w_self.lineno) @@ -4710,14 +4613,14 @@ w_self.setdictvalue(space, 'lineno', w_new_value) return w_self.deldictvalue(space, 'lineno') - w_self.initialization_state |= w_self._lineno_mask + w_self.initialization_state |= 1 def expr_get_col_offset(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'col_offset') if w_obj is not None: return w_obj - if not w_self.initialization_state & w_self._col_offset_mask: + if not w_self.initialization_state & 2: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'col_offset') return space.wrap(w_self.col_offset) @@ -4731,7 +4634,7 @@ w_self.setdictvalue(space, 'col_offset', w_new_value) return w_self.deldictvalue(space, 'col_offset') - w_self.initialization_state |= w_self._col_offset_mask + w_self.initialization_state |= 2 expr.typedef = typedef.TypeDef("expr", AST.typedef, @@ -4747,7 +4650,7 @@ w_obj = w_self.getdictvalue(space, 'op') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'op') return boolop_to_class[w_self.op - 1]() @@ -4763,10 +4666,10 @@ return # need to save the original object too w_self.setdictvalue(space, 'op', w_new_value) - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 def BoolOp_get_values(space, w_self): - if not w_self.initialization_state & 2: + if not w_self.initialization_state & 8: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'values') if w_self.w_values is None: @@ -4780,7 +4683,7 @@ def BoolOp_set_values(space, w_self, w_new_value): w_self.w_values = w_new_value - w_self.initialization_state |= 2 + w_self.initialization_state |= 8 _BoolOp_field_unroller = unrolling_iterable(['op', 'values']) def BoolOp_init(space, w_self, __args__): @@ -4813,7 +4716,7 @@ w_obj = w_self.getdictvalue(space, 'left') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'left') return space.wrap(w_self.left) @@ -4821,20 +4724,22 @@ def BinOp_set_left(space, w_self, w_new_value): try: w_self.left = space.interp_w(expr, w_new_value, False) + if type(w_self.left) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'left', w_new_value) return w_self.deldictvalue(space, 'left') - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 def BinOp_get_op(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'op') if w_obj is not None: return w_obj - if not w_self.initialization_state & 2: + if not w_self.initialization_state & 8: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'op') return operator_to_class[w_self.op - 1]() @@ -4850,14 +4755,14 @@ return # need to save the original object too w_self.setdictvalue(space, 'op', w_new_value) - w_self.initialization_state |= 2 + w_self.initialization_state |= 8 def BinOp_get_right(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'right') if w_obj is not None: return w_obj - if not w_self.initialization_state & 4: + if not w_self.initialization_state & 16: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'right') return space.wrap(w_self.right) @@ -4865,13 +4770,15 @@ def BinOp_set_right(space, w_self, w_new_value): try: w_self.right = space.interp_w(expr, w_new_value, False) + if type(w_self.right) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'right', w_new_value) return w_self.deldictvalue(space, 'right') - w_self.initialization_state |= 4 + w_self.initialization_state |= 16 _BinOp_field_unroller = unrolling_iterable(['left', 'op', 'right']) def BinOp_init(space, w_self, __args__): @@ -4904,7 +4811,7 @@ w_obj = w_self.getdictvalue(space, 'op') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'op') return unaryop_to_class[w_self.op - 1]() @@ -4920,14 +4827,14 @@ return # need to save the original object too w_self.setdictvalue(space, 'op', w_new_value) - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 def UnaryOp_get_operand(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'operand') if w_obj is not None: return w_obj - if not w_self.initialization_state & 2: + if not w_self.initialization_state & 8: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'operand') return space.wrap(w_self.operand) @@ -4935,13 +4842,15 @@ def UnaryOp_set_operand(space, w_self, w_new_value): try: w_self.operand = space.interp_w(expr, w_new_value, False) + if type(w_self.operand) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'operand', w_new_value) return w_self.deldictvalue(space, 'operand') - w_self.initialization_state |= 2 + w_self.initialization_state |= 8 _UnaryOp_field_unroller = unrolling_iterable(['op', 'operand']) def UnaryOp_init(space, w_self, __args__): @@ -4973,7 +4882,7 @@ w_obj = w_self.getdictvalue(space, 'args') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'args') return space.wrap(w_self.args) @@ -4987,14 +4896,14 @@ w_self.setdictvalue(space, 'args', w_new_value) return w_self.deldictvalue(space, 'args') - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 def Lambda_get_body(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'body') if w_obj is not None: return w_obj - if not w_self.initialization_state & 2: + if not w_self.initialization_state & 8: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body') return space.wrap(w_self.body) @@ -5002,13 +4911,15 @@ def Lambda_set_body(space, w_self, w_new_value): try: w_self.body = space.interp_w(expr, w_new_value, False) + if type(w_self.body) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'body', w_new_value) return w_self.deldictvalue(space, 'body') - w_self.initialization_state |= 2 + w_self.initialization_state |= 8 _Lambda_field_unroller = unrolling_iterable(['args', 'body']) def Lambda_init(space, w_self, __args__): @@ -5040,7 +4951,7 @@ w_obj = w_self.getdictvalue(space, 'test') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'test') return space.wrap(w_self.test) @@ -5048,20 +4959,22 @@ def IfExp_set_test(space, w_self, w_new_value): try: w_self.test = space.interp_w(expr, w_new_value, False) + if type(w_self.test) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'test', w_new_value) return w_self.deldictvalue(space, 'test') - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 def IfExp_get_body(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'body') if w_obj is not None: return w_obj - if not w_self.initialization_state & 2: + if not w_self.initialization_state & 8: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body') return space.wrap(w_self.body) @@ -5069,20 +4982,22 @@ def IfExp_set_body(space, w_self, w_new_value): try: w_self.body = space.interp_w(expr, w_new_value, False) + if type(w_self.body) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'body', w_new_value) return w_self.deldictvalue(space, 'body') - w_self.initialization_state |= 2 + w_self.initialization_state |= 8 def IfExp_get_orelse(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'orelse') if w_obj is not None: return w_obj - if not w_self.initialization_state & 4: + if not w_self.initialization_state & 16: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse') return space.wrap(w_self.orelse) @@ -5090,13 +5005,15 @@ def IfExp_set_orelse(space, w_self, w_new_value): try: w_self.orelse = space.interp_w(expr, w_new_value, False) + if type(w_self.orelse) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'orelse', w_new_value) return w_self.deldictvalue(space, 'orelse') - w_self.initialization_state |= 4 + w_self.initialization_state |= 16 _IfExp_field_unroller = unrolling_iterable(['test', 'body', 'orelse']) def IfExp_init(space, w_self, __args__): @@ -5125,7 +5042,7 @@ ) def Dict_get_keys(space, w_self): - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'keys') if w_self.w_keys is None: @@ -5139,10 +5056,10 @@ def Dict_set_keys(space, w_self, w_new_value): w_self.w_keys = w_new_value - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 def Dict_get_values(space, w_self): - if not w_self.initialization_state & 2: + if not w_self.initialization_state & 8: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'values') if w_self.w_values is None: @@ -5156,7 +5073,7 @@ def Dict_set_values(space, w_self, w_new_value): w_self.w_values = w_new_value - w_self.initialization_state |= 2 + w_self.initialization_state |= 8 _Dict_field_unroller = unrolling_iterable(['keys', 'values']) def Dict_init(space, w_self, __args__): @@ -5186,7 +5103,7 @@ ) def Set_get_elts(space, w_self): - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elts') if w_self.w_elts is None: @@ -5200,7 +5117,7 @@ def Set_set_elts(space, w_self, w_new_value): w_self.w_elts = w_new_value - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 _Set_field_unroller = unrolling_iterable(['elts']) def Set_init(space, w_self, __args__): @@ -5232,7 +5149,7 @@ w_obj = w_self.getdictvalue(space, 'elt') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elt') return space.wrap(w_self.elt) @@ -5240,16 +5157,18 @@ def ListComp_set_elt(space, w_self, w_new_value): try: w_self.elt = space.interp_w(expr, w_new_value, False) + if type(w_self.elt) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'elt', w_new_value) return w_self.deldictvalue(space, 'elt') - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 def ListComp_get_generators(space, w_self): - if not w_self.initialization_state & 2: + if not w_self.initialization_state & 8: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'generators') if w_self.w_generators is None: @@ -5263,7 +5182,7 @@ def ListComp_set_generators(space, w_self, w_new_value): w_self.w_generators = w_new_value - w_self.initialization_state |= 2 + w_self.initialization_state |= 8 _ListComp_field_unroller = unrolling_iterable(['elt', 'generators']) def ListComp_init(space, w_self, __args__): @@ -5296,7 +5215,7 @@ w_obj = w_self.getdictvalue(space, 'elt') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elt') return space.wrap(w_self.elt) @@ -5304,16 +5223,18 @@ def SetComp_set_elt(space, w_self, w_new_value): try: w_self.elt = space.interp_w(expr, w_new_value, False) + if type(w_self.elt) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'elt', w_new_value) return w_self.deldictvalue(space, 'elt') - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 def SetComp_get_generators(space, w_self): - if not w_self.initialization_state & 2: + if not w_self.initialization_state & 8: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'generators') if w_self.w_generators is None: @@ -5327,7 +5248,7 @@ def SetComp_set_generators(space, w_self, w_new_value): w_self.w_generators = w_new_value - w_self.initialization_state |= 2 + w_self.initialization_state |= 8 _SetComp_field_unroller = unrolling_iterable(['elt', 'generators']) def SetComp_init(space, w_self, __args__): @@ -5360,7 +5281,7 @@ w_obj = w_self.getdictvalue(space, 'key') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'key') return space.wrap(w_self.key) @@ -5368,20 +5289,22 @@ def DictComp_set_key(space, w_self, w_new_value): try: w_self.key = space.interp_w(expr, w_new_value, False) + if type(w_self.key) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'key', w_new_value) return w_self.deldictvalue(space, 'key') - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 def DictComp_get_value(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'value') if w_obj is not None: return w_obj - if not w_self.initialization_state & 2: + if not w_self.initialization_state & 8: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value') return space.wrap(w_self.value) @@ -5389,16 +5312,18 @@ def DictComp_set_value(space, w_self, w_new_value): try: w_self.value = space.interp_w(expr, w_new_value, False) + if type(w_self.value) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'value', w_new_value) return w_self.deldictvalue(space, 'value') - w_self.initialization_state |= 2 + w_self.initialization_state |= 8 def DictComp_get_generators(space, w_self): - if not w_self.initialization_state & 4: + if not w_self.initialization_state & 16: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'generators') if w_self.w_generators is None: @@ -5412,7 +5337,7 @@ def DictComp_set_generators(space, w_self, w_new_value): w_self.w_generators = w_new_value - w_self.initialization_state |= 4 + w_self.initialization_state |= 16 _DictComp_field_unroller = unrolling_iterable(['key', 'value', 'generators']) def DictComp_init(space, w_self, __args__): @@ -5446,7 +5371,7 @@ w_obj = w_self.getdictvalue(space, 'elt') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elt') return space.wrap(w_self.elt) @@ -5454,16 +5379,18 @@ def GeneratorExp_set_elt(space, w_self, w_new_value): try: w_self.elt = space.interp_w(expr, w_new_value, False) + if type(w_self.elt) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'elt', w_new_value) return w_self.deldictvalue(space, 'elt') - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 def GeneratorExp_get_generators(space, w_self): - if not w_self.initialization_state & 2: + if not w_self.initialization_state & 8: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'generators') if w_self.w_generators is None: @@ -5477,7 +5404,7 @@ def GeneratorExp_set_generators(space, w_self, w_new_value): w_self.w_generators = w_new_value - w_self.initialization_state |= 2 + w_self.initialization_state |= 8 _GeneratorExp_field_unroller = unrolling_iterable(['elt', 'generators']) def GeneratorExp_init(space, w_self, __args__): @@ -5510,7 +5437,7 @@ w_obj = w_self.getdictvalue(space, 'value') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value') return space.wrap(w_self.value) @@ -5518,13 +5445,15 @@ def Yield_set_value(space, w_self, w_new_value): try: w_self.value = space.interp_w(expr, w_new_value, True) + if type(w_self.value) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'value', w_new_value) return w_self.deldictvalue(space, 'value') - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 _Yield_field_unroller = unrolling_iterable(['value']) def Yield_init(space, w_self, __args__): @@ -5555,7 +5484,7 @@ w_obj = w_self.getdictvalue(space, 'left') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'left') return space.wrap(w_self.left) @@ -5563,16 +5492,18 @@ def Compare_set_left(space, w_self, w_new_value): try: w_self.left = space.interp_w(expr, w_new_value, False) + if type(w_self.left) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'left', w_new_value) return w_self.deldictvalue(space, 'left') - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 def Compare_get_ops(space, w_self): - if not w_self.initialization_state & 2: + if not w_self.initialization_state & 8: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ops') if w_self.w_ops is None: @@ -5586,10 +5517,10 @@ def Compare_set_ops(space, w_self, w_new_value): w_self.w_ops = w_new_value - w_self.initialization_state |= 2 + w_self.initialization_state |= 8 def Compare_get_comparators(space, w_self): - if not w_self.initialization_state & 4: + if not w_self.initialization_state & 16: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'comparators') if w_self.w_comparators is None: @@ -5603,7 +5534,7 @@ def Compare_set_comparators(space, w_self, w_new_value): w_self.w_comparators = w_new_value - w_self.initialization_state |= 4 + w_self.initialization_state |= 16 _Compare_field_unroller = unrolling_iterable(['left', 'ops', 'comparators']) def Compare_init(space, w_self, __args__): @@ -5638,7 +5569,7 @@ w_obj = w_self.getdictvalue(space, 'func') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'func') return space.wrap(w_self.func) @@ -5646,16 +5577,18 @@ def Call_set_func(space, w_self, w_new_value): try: w_self.func = space.interp_w(expr, w_new_value, False) + if type(w_self.func) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'func', w_new_value) return w_self.deldictvalue(space, 'func') - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 def Call_get_args(space, w_self): - if not w_self.initialization_state & 2: + if not w_self.initialization_state & 8: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'args') if w_self.w_args is None: @@ -5669,10 +5602,10 @@ def Call_set_args(space, w_self, w_new_value): w_self.w_args = w_new_value - w_self.initialization_state |= 2 + w_self.initialization_state |= 8 def Call_get_keywords(space, w_self): - if not w_self.initialization_state & 4: + if not w_self.initialization_state & 16: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'keywords') if w_self.w_keywords is None: @@ -5686,14 +5619,14 @@ def Call_set_keywords(space, w_self, w_new_value): w_self.w_keywords = w_new_value - w_self.initialization_state |= 4 + w_self.initialization_state |= 16 def Call_get_starargs(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'starargs') if w_obj is not None: return w_obj - if not w_self.initialization_state & 8: + if not w_self.initialization_state & 32: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'starargs') return space.wrap(w_self.starargs) @@ -5701,20 +5634,22 @@ def Call_set_starargs(space, w_self, w_new_value): try: w_self.starargs = space.interp_w(expr, w_new_value, True) + if type(w_self.starargs) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'starargs', w_new_value) return w_self.deldictvalue(space, 'starargs') - w_self.initialization_state |= 8 + w_self.initialization_state |= 32 def Call_get_kwargs(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'kwargs') if w_obj is not None: return w_obj - if not w_self.initialization_state & 16: + if not w_self.initialization_state & 64: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'kwargs') return space.wrap(w_self.kwargs) @@ -5722,13 +5657,15 @@ def Call_set_kwargs(space, w_self, w_new_value): try: w_self.kwargs = space.interp_w(expr, w_new_value, True) + if type(w_self.kwargs) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'kwargs', w_new_value) return w_self.deldictvalue(space, 'kwargs') - w_self.initialization_state |= 16 + w_self.initialization_state |= 64 _Call_field_unroller = unrolling_iterable(['func', 'args', 'keywords', 'starargs', 'kwargs']) def Call_init(space, w_self, __args__): @@ -5765,7 +5702,7 @@ w_obj = w_self.getdictvalue(space, 'value') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value') return space.wrap(w_self.value) @@ -5773,13 +5710,15 @@ def Repr_set_value(space, w_self, w_new_value): try: w_self.value = space.interp_w(expr, w_new_value, False) + if type(w_self.value) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'value', w_new_value) return w_self.deldictvalue(space, 'value') - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 _Repr_field_unroller = unrolling_iterable(['value']) def Repr_init(space, w_self, __args__): @@ -5810,7 +5749,7 @@ w_obj = w_self.getdictvalue(space, 'n') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'n') return w_self.n @@ -5824,7 +5763,7 @@ w_self.setdictvalue(space, 'n', w_new_value) return w_self.deldictvalue(space, 'n') - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 _Num_field_unroller = unrolling_iterable(['n']) def Num_init(space, w_self, __args__): @@ -5855,7 +5794,7 @@ w_obj = w_self.getdictvalue(space, 's') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 's') return w_self.s @@ -5869,7 +5808,7 @@ w_self.setdictvalue(space, 's', w_new_value) return w_self.deldictvalue(space, 's') - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 _Str_field_unroller = unrolling_iterable(['s']) def Str_init(space, w_self, __args__): @@ -5900,7 +5839,7 @@ w_obj = w_self.getdictvalue(space, 'value') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value') return space.wrap(w_self.value) @@ -5908,20 +5847,22 @@ def Attribute_set_value(space, w_self, w_new_value): try: w_self.value = space.interp_w(expr, w_new_value, False) + if type(w_self.value) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'value', w_new_value) return w_self.deldictvalue(space, 'value') - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 def Attribute_get_attr(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'attr') if w_obj is not None: return w_obj - if not w_self.initialization_state & 2: + if not w_self.initialization_state & 8: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'attr') return space.wrap(w_self.attr) @@ -5935,14 +5876,14 @@ w_self.setdictvalue(space, 'attr', w_new_value) return w_self.deldictvalue(space, 'attr') - w_self.initialization_state |= 2 + w_self.initialization_state |= 8 def Attribute_get_ctx(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'ctx') if w_obj is not None: return w_obj - if not w_self.initialization_state & 4: + if not w_self.initialization_state & 16: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx') return expr_context_to_class[w_self.ctx - 1]() @@ -5958,7 +5899,7 @@ return # need to save the original object too w_self.setdictvalue(space, 'ctx', w_new_value) - w_self.initialization_state |= 4 + w_self.initialization_state |= 16 _Attribute_field_unroller = unrolling_iterable(['value', 'attr', 'ctx']) def Attribute_init(space, w_self, __args__): @@ -5991,7 +5932,7 @@ w_obj = w_self.getdictvalue(space, 'value') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value') return space.wrap(w_self.value) @@ -5999,20 +5940,22 @@ def Subscript_set_value(space, w_self, w_new_value): try: w_self.value = space.interp_w(expr, w_new_value, False) + if type(w_self.value) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'value', w_new_value) return w_self.deldictvalue(space, 'value') - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 def Subscript_get_slice(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'slice') if w_obj is not None: return w_obj - if not w_self.initialization_state & 2: + if not w_self.initialization_state & 8: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'slice') return space.wrap(w_self.slice) @@ -6020,20 +5963,22 @@ def Subscript_set_slice(space, w_self, w_new_value): try: w_self.slice = space.interp_w(slice, w_new_value, False) + if type(w_self.slice) is slice: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'slice', w_new_value) return w_self.deldictvalue(space, 'slice') - w_self.initialization_state |= 2 + w_self.initialization_state |= 8 def Subscript_get_ctx(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'ctx') if w_obj is not None: return w_obj - if not w_self.initialization_state & 4: + if not w_self.initialization_state & 16: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx') return expr_context_to_class[w_self.ctx - 1]() @@ -6049,7 +5994,7 @@ return # need to save the original object too w_self.setdictvalue(space, 'ctx', w_new_value) - w_self.initialization_state |= 4 + w_self.initialization_state |= 16 _Subscript_field_unroller = unrolling_iterable(['value', 'slice', 'ctx']) def Subscript_init(space, w_self, __args__): @@ -6082,7 +6027,7 @@ w_obj = w_self.getdictvalue(space, 'id') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'id') return space.wrap(w_self.id) @@ -6096,14 +6041,14 @@ w_self.setdictvalue(space, 'id', w_new_value) return w_self.deldictvalue(space, 'id') - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 def Name_get_ctx(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'ctx') if w_obj is not None: return w_obj - if not w_self.initialization_state & 2: + if not w_self.initialization_state & 8: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx') return expr_context_to_class[w_self.ctx - 1]() @@ -6119,7 +6064,7 @@ return # need to save the original object too w_self.setdictvalue(space, 'ctx', w_new_value) - w_self.initialization_state |= 2 + w_self.initialization_state |= 8 _Name_field_unroller = unrolling_iterable(['id', 'ctx']) def Name_init(space, w_self, __args__): @@ -6147,7 +6092,7 @@ ) def List_get_elts(space, w_self): - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elts') if w_self.w_elts is None: @@ -6161,14 +6106,14 @@ def List_set_elts(space, w_self, w_new_value): w_self.w_elts = w_new_value - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 def List_get_ctx(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'ctx') if w_obj is not None: return w_obj - if not w_self.initialization_state & 2: + if not w_self.initialization_state & 8: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx') return expr_context_to_class[w_self.ctx - 1]() @@ -6184,7 +6129,7 @@ return # need to save the original object too w_self.setdictvalue(space, 'ctx', w_new_value) - w_self.initialization_state |= 2 + w_self.initialization_state |= 8 _List_field_unroller = unrolling_iterable(['elts', 'ctx']) def List_init(space, w_self, __args__): @@ -6213,7 +6158,7 @@ ) def Tuple_get_elts(space, w_self): - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elts') if w_self.w_elts is None: @@ -6227,14 +6172,14 @@ def Tuple_set_elts(space, w_self, w_new_value): w_self.w_elts = w_new_value - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 def Tuple_get_ctx(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'ctx') if w_obj is not None: return w_obj - if not w_self.initialization_state & 2: + if not w_self.initialization_state & 8: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx') return expr_context_to_class[w_self.ctx - 1]() @@ -6250,7 +6195,7 @@ return # need to save the original object too w_self.setdictvalue(space, 'ctx', w_new_value) - w_self.initialization_state |= 2 + w_self.initialization_state |= 8 _Tuple_field_unroller = unrolling_iterable(['elts', 'ctx']) def Tuple_init(space, w_self, __args__): @@ -6283,7 +6228,7 @@ w_obj = w_self.getdictvalue(space, 'value') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value') return w_self.value @@ -6297,7 +6242,7 @@ w_self.setdictvalue(space, 'value', w_new_value) return w_self.deldictvalue(space, 'value') - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 _Const_field_unroller = unrolling_iterable(['value']) def Const_init(space, w_self, __args__): @@ -6409,6 +6354,8 @@ def Slice_set_lower(space, w_self, w_new_value): try: w_self.lower = space.interp_w(expr, w_new_value, True) + if type(w_self.lower) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise @@ -6430,6 +6377,8 @@ def Slice_set_upper(space, w_self, w_new_value): try: w_self.upper = space.interp_w(expr, w_new_value, True) + if type(w_self.upper) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise @@ -6451,6 +6400,8 @@ def Slice_set_step(space, w_self, w_new_value): try: w_self.step = space.interp_w(expr, w_new_value, True) + if type(w_self.step) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise @@ -6540,6 +6491,8 @@ def Index_set_value(space, w_self, w_new_value): try: w_self.value = space.interp_w(expr, w_new_value, False) + if type(w_self.value) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise @@ -6809,6 +6762,8 @@ def comprehension_set_target(space, w_self, w_new_value): try: w_self.target = space.interp_w(expr, w_new_value, False) + if type(w_self.target) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise @@ -6830,6 +6785,8 @@ def comprehension_set_iter(space, w_self, w_new_value): try: w_self.iter = space.interp_w(expr, w_new_value, False) + if type(w_self.iter) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise @@ -6887,7 +6844,7 @@ w_obj = w_self.getdictvalue(space, 'lineno') if w_obj is not None: return w_obj - if not w_self.initialization_state & w_self._lineno_mask: + if not w_self.initialization_state & 1: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'lineno') return space.wrap(w_self.lineno) @@ -6901,14 +6858,14 @@ w_self.setdictvalue(space, 'lineno', w_new_value) return w_self.deldictvalue(space, 'lineno') - w_self.initialization_state |= w_self._lineno_mask + w_self.initialization_state |= 1 def excepthandler_get_col_offset(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'col_offset') if w_obj is not None: return w_obj - if not w_self.initialization_state & w_self._col_offset_mask: + if not w_self.initialization_state & 2: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'col_offset') return space.wrap(w_self.col_offset) @@ -6922,7 +6879,7 @@ w_self.setdictvalue(space, 'col_offset', w_new_value) return w_self.deldictvalue(space, 'col_offset') - w_self.initialization_state |= w_self._col_offset_mask + w_self.initialization_state |= 2 excepthandler.typedef = typedef.TypeDef("excepthandler", AST.typedef, @@ -6938,7 +6895,7 @@ w_obj = w_self.getdictvalue(space, 'type') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'type') return space.wrap(w_self.type) @@ -6946,20 +6903,22 @@ def ExceptHandler_set_type(space, w_self, w_new_value): try: w_self.type = space.interp_w(expr, w_new_value, True) + if type(w_self.type) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'type', w_new_value) return w_self.deldictvalue(space, 'type') - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 def ExceptHandler_get_name(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'name') if w_obj is not None: return w_obj - if not w_self.initialization_state & 2: + if not w_self.initialization_state & 8: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'name') return space.wrap(w_self.name) @@ -6967,16 +6926,18 @@ def ExceptHandler_set_name(space, w_self, w_new_value): try: w_self.name = space.interp_w(expr, w_new_value, True) + if type(w_self.name) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'name', w_new_value) return w_self.deldictvalue(space, 'name') - w_self.initialization_state |= 2 + w_self.initialization_state |= 8 def ExceptHandler_get_body(space, w_self): - if not w_self.initialization_state & 4: + if not w_self.initialization_state & 16: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body') if w_self.w_body is None: @@ -6990,7 +6951,7 @@ def ExceptHandler_set_body(space, w_self, w_new_value): w_self.w_body = w_new_value - w_self.initialization_state |= 4 + w_self.initialization_state |= 16 _ExceptHandler_field_unroller = unrolling_iterable(['type', 'name', 'body']) def ExceptHandler_init(space, w_self, __args__): @@ -7164,6 +7125,8 @@ def keyword_set_value(space, w_self, w_new_value): try: w_self.value = space.interp_w(expr, w_new_value, False) + if type(w_self.value) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise diff --git a/pypy/interpreter/astcompiler/optimize.py b/pypy/interpreter/astcompiler/optimize.py --- a/pypy/interpreter/astcompiler/optimize.py +++ b/pypy/interpreter/astcompiler/optimize.py @@ -1,6 +1,5 @@ """codegen helpers and AST constant folding.""" import sys -import itertools from pypy.interpreter.astcompiler import ast, consts, misc from pypy.tool import stdlib_opcode as ops @@ -146,8 +145,7 @@ } unrolling_unary_folders = unrolling_iterable(unary_folders.items()) -for folder in itertools.chain(binary_folders.itervalues(), - unary_folders.itervalues()): +for folder in binary_folders.values() + unary_folders.values(): folder._always_inline_ = True del folder diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py --- a/pypy/interpreter/astcompiler/tools/asdl_py.py +++ b/pypy/interpreter/astcompiler/tools/asdl_py.py @@ -79,6 +79,7 @@ else: self.emit("class %s(AST):" % (base,)) if sum.attributes: + self.emit("") args = ", ".join(attr.name.value for attr in sum.attributes) self.emit("def __init__(self, %s):" % (args,), 1) for attr in sum.attributes: @@ -114,7 +115,7 @@ else: names.append(repr(field.name.value)) sub = (", ".join(names), name.value) - self.emit("missing_field(space, self.initialization_state, [%s], %r)" + self.emit("self.missing_field(space, [%s], %r)" % sub, 3) self.emit("else:", 2) # Fill in all the default fields. @@ -195,17 +196,13 @@ def visitConstructor(self, cons, base, extra_attributes): self.emit("class %s(%s):" % (cons.name, base)) self.emit("") - for field in self.data.cons_attributes[cons]: - subst = (field.name, self.data.field_masks[field]) - self.emit("_%s_mask = %i" % subst, 1) - self.emit("") self.make_constructor(cons.fields, cons, extra_attributes, base) self.emit("") self.emit("def walkabout(self, visitor):", 1) self.emit("visitor.visit_%s(self)" % (cons.name,), 2) self.emit("") self.make_mutate_over(cons, cons.name) - self.make_var_syncer(cons.fields + self.data.cons_attributes[cons], + self.make_var_syncer(self.data.cons_attributes[cons] + cons.fields, cons, cons.name) def visitField(self, field): @@ -324,7 +321,7 @@ def visitSum(self, sum, name): for field in sum.attributes: - self.make_property(field, name, True) + self.make_property(field, name) self.make_typedef(name, "AST", sum.attributes, fields_name="_attributes") if not is_simple_sum(sum): @@ -400,13 +397,10 @@ def visitField(self, field, name): self.make_property(field, name) - def make_property(self, field, name, different_masks=False): + def make_property(self, field, name): func = "def %s_get_%s(space, w_self):" % (name, field.name) self.emit(func) - if different_masks: - flag = "w_self._%s_mask" % (field.name,) - else: - flag = self.data.field_masks[field] + flag = self.data.field_masks[field] if not field.seq: self.emit("if w_self.w_dict is not None:", 1) self.emit(" w_obj = w_self.getdictvalue(space, '%s')" % (field.name,), 1) @@ -458,6 +452,11 @@ config = (field.name, field.type, repr(field.opt)) self.emit("w_self.%s = space.interp_w(%s, w_new_value, %s)" % config, 2) + if field.type.value not in self.data.prod_simple: + self.emit("if type(w_self.%s) is %s:" % ( + field.name, field.type), 2) + self.emit("raise OperationError(space.w_TypeError, " + "space.w_None)", 3) else: level = 2 if field.opt and field.type.value != "int": @@ -505,7 +504,10 @@ optional_mask = 0 for i, field in enumerate(fields): flag = 1 << i - field_masks[field] = flag + if field not in field_masks: + field_masks[field] = flag + else: + assert field_masks[field] == flag if field.opt: optional_mask |= flag else: @@ -518,9 +520,9 @@ if is_simple_sum(sum): simple_types.add(tp.name.value) else: + attrs = [field for field in sum.attributes] for cons in sum.types: - attrs = [copy_field(field) for field in sum.attributes] - add_masks(cons.fields + attrs, cons) + add_masks(attrs + cons.fields, cons) cons_attributes[cons] = attrs else: prod = tp.value @@ -588,6 +590,24 @@ space.setattr(self, w_name, space.getitem(w_state, w_name)) + def missing_field(self, space, required, host): + "Find which required field is missing." + state = self.initialization_state + for i in range(len(required)): + if (state >> i) & 1: + continue # field is present + missing = required[i] + if missing is None: + continue # field is optional + w_obj = self.getdictvalue(space, missing) + if w_obj is None: + err = "required field \\"%s\\" missing from %s" + raise operationerrfmt(space.w_TypeError, err, missing, host) + else: + err = "incorrect type for field \\"%s\\" in %s" + raise operationerrfmt(space.w_TypeError, err, missing, host) + raise AssertionError("should not reach here") + class NodeVisitorNotImplemented(Exception): pass @@ -631,15 +651,6 @@ ) -def missing_field(space, state, required, host): - "Find which required field is missing." - for i in range(len(required)): - if not (state >> i) & 1: - missing = required[i] - if missing is not None: - err = "required field \\"%s\\" missing from %s" - raise operationerrfmt(space.w_TypeError, err, missing, host) - raise AssertionError("should not reach here") """ diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -1,4 +1,3 @@ -import itertools import pypy from pypy.interpreter.executioncontext import ExecutionContext, ActionFlag from pypy.interpreter.executioncontext import UserDelAction, FrameTraceAction @@ -191,8 +190,8 @@ def is_w(self, space, w_other): return self is w_other - def unique_id(self, space): - return space.wrap(compute_unique_id(self)) + def immutable_unique_id(self, space): + return None def str_w(self, space): w_msg = typed_unwrap_error_msg(space, "string", self) @@ -488,6 +487,16 @@ 'parser', 'fcntl', '_codecs', 'binascii' ] + # These modules are treated like CPython treats built-in modules, + # i.e. they always shadow any xx.py. The other modules are treated + # like CPython treats extension modules, and are loaded in sys.path + # order by the fake entry '.../lib_pypy/__extensions__'. + MODULES_THAT_ALWAYS_SHADOW = dict.fromkeys([ + '__builtin__', '__pypy__', '_ast', '_codecs', '_sre', '_warnings', + '_weakref', 'errno', 'exceptions', 'gc', 'imp', 'marshal', + 'posix', 'nt', 'pwd', 'signal', 'sys', 'thread', 'zipimport', + ], None) + def make_builtins(self): "NOT_RPYTHON: only for initializing the space." @@ -519,8 +528,8 @@ exception_types_w = self.export_builtin_exceptions() # initialize with "bootstrap types" from objspace (e.g. w_None) - types_w = itertools.chain(self.get_builtin_types().iteritems(), - exception_types_w.iteritems()) + types_w = (self.get_builtin_types().items() + + exception_types_w.items()) for name, w_type in types_w: self.setitem(self.builtin.w_dict, self.wrap(name), w_type) @@ -697,7 +706,10 @@ return w_two.is_w(self, w_one) def id(self, w_obj): - return w_obj.unique_id(self) + w_result = w_obj.immutable_unique_id(self) + if w_result is None: + w_result = self.wrap(compute_unique_id(w_obj)) + return w_result def hash_w(self, w_obj): """shortcut for space.int_w(space.hash(w_obj))""" @@ -1579,12 +1591,15 @@ 'ArithmeticError', 'AssertionError', 'AttributeError', + 'BaseException', + 'DeprecationWarning', 'EOFError', 'EnvironmentError', 'Exception', 'FloatingPointError', 'IOError', 'ImportError', + 'ImportWarning', 'IndentationError', 'IndexError', 'KeyError', @@ -1605,9 +1620,14 @@ 'TabError', 'TypeError', 'UnboundLocalError', + 'UnicodeDecodeError', 'UnicodeError', + 'UnicodeEncodeError', + 'UnicodeTranslateError', 'ValueError', 'ZeroDivisionError', + 'UnicodeEncodeError', + 'UnicodeDecodeError', ] ## Irregular part of the interface: diff --git a/pypy/interpreter/eval.py b/pypy/interpreter/eval.py --- a/pypy/interpreter/eval.py +++ b/pypy/interpreter/eval.py @@ -2,7 +2,6 @@ This module defines the abstract base classes that support execution: Code and Frame. """ -from pypy.rlib import jit from pypy.interpreter.error import OperationError from pypy.interpreter.baseobjspace import Wrappable @@ -98,7 +97,6 @@ "Abstract. Get the expected number of locals." raise TypeError, "abstract" - @jit.dont_look_inside def fast2locals(self): # Copy values from the fastlocals to self.w_locals if self.w_locals is None: @@ -112,7 +110,6 @@ w_name = self.space.wrap(name) self.space.setitem(self.w_locals, w_name, w_value) - @jit.dont_look_inside def locals2fast(self): # Copy values from self.w_locals to the fastlocals assert self.w_locals is not None diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py --- a/pypy/interpreter/executioncontext.py +++ b/pypy/interpreter/executioncontext.py @@ -445,6 +445,7 @@ AsyncAction.__init__(self, space) self.dying_objects = [] self.finalizers_lock_count = 0 + self.enabled_at_app_level = True def register_callback(self, w_obj, callback, descrname): self.dying_objects.append((w_obj, callback, descrname)) diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py --- a/pypy/interpreter/gateway.py +++ b/pypy/interpreter/gateway.py @@ -619,7 +619,8 @@ self.descr_reqcls, args) except Exception, e: - raise self.handle_exception(space, e) + self.handle_exception(space, e) + w_result = None if w_result is None: w_result = space.w_None return w_result @@ -655,7 +656,8 @@ self.descr_reqcls, args) except Exception, e: - raise self.handle_exception(space, e) + self.handle_exception(space, e) + w_result = None if w_result is None: w_result = space.w_None return w_result @@ -674,7 +676,8 @@ self.descr_reqcls, args.prepend(w_obj)) except Exception, e: - raise self.handle_exception(space, e) + self.handle_exception(space, e) + w_result = None if w_result is None: w_result = space.w_None return w_result @@ -690,7 +693,8 @@ raise OperationError(space.w_SystemError, space.wrap("unexpected DescrMismatch error")) except Exception, e: - raise self.handle_exception(space, e) + self.handle_exception(space, e) + w_result = None if w_result is None: w_result = space.w_None return w_result @@ -708,7 +712,8 @@ self.descr_reqcls, Arguments(space, [w1])) except Exception, e: - raise self.handle_exception(space, e) + self.handle_exception(space, e) + w_result = None if w_result is None: w_result = space.w_None return w_result @@ -726,7 +731,8 @@ self.descr_reqcls, Arguments(space, [w1, w2])) except Exception, e: - raise self.handle_exception(space, e) + self.handle_exception(space, e) + w_result = None if w_result is None: w_result = space.w_None return w_result @@ -744,7 +750,8 @@ self.descr_reqcls, Arguments(space, [w1, w2, w3])) except Exception, e: - raise self.handle_exception(space, e) + self.handle_exception(space, e) + w_result = None if w_result is None: w_result = space.w_None return w_result @@ -763,7 +770,8 @@ Arguments(space, [w1, w2, w3, w4])) except Exception, e: - raise self.handle_exception(space, e) + self.handle_exception(space, e) + w_result = None if w_result is None: w_result = space.w_None return w_result diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py --- a/pypy/interpreter/generator.py +++ b/pypy/interpreter/generator.py @@ -162,7 +162,8 @@ # generate 2 versions of the function and 2 jit drivers. def _create_unpack_into(): jitdriver = jit.JitDriver(greens=['pycode'], - reds=['self', 'frame', 'results']) + reds=['self', 'frame', 'results'], + name='unpack_into') def unpack_into(self, results): """This is a hack for performance: runs the generator and collects all produced items in a list.""" @@ -196,4 +197,4 @@ self.frame = None return unpack_into unpack_into = _create_unpack_into() - unpack_into_w = _create_unpack_into() \ No newline at end of file + unpack_into_w = _create_unpack_into() diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py --- a/pypy/interpreter/test/test_argument.py +++ b/pypy/interpreter/test/test_argument.py @@ -393,8 +393,8 @@ class FakeArgErr(ArgErr): - def getmsg(self, fname): - return "msg "+fname + def getmsg(self): + return "msg" def _match_signature(*args): raise FakeArgErr() @@ -404,7 +404,7 @@ excinfo = py.test.raises(OperationError, args.parse_obj, "obj", "foo", Signature(["a", "b"], None, None)) assert excinfo.value.w_type is TypeError - assert excinfo.value._w_value == "msg foo" + assert excinfo.value.get_w_value(space) == "foo() msg" def test_args_parsing_into_scope(self): @@ -448,8 +448,8 @@ class FakeArgErr(ArgErr): - def getmsg(self, fname): - return "msg "+fname + def getmsg(self): + return "msg" def _match_signature(*args): raise FakeArgErr() @@ -460,7 +460,7 @@ "obj", [None, None], "foo", Signature(["a", "b"], None, None)) assert excinfo.value.w_type is TypeError - assert excinfo.value._w_value == "msg foo" + assert excinfo.value.get_w_value(space) == "foo() msg" def test_topacked_frompacked(self): space = DummySpace() @@ -493,35 +493,35 @@ # got_nargs, nkwds, expected_nargs, has_vararg, has_kwarg, # defaults_w, missing_args err = ArgErrCount(1, 0, 0, False, False, None, 0) - s = err.getmsg('foo') - assert s == "foo() takes no arguments (1 given)" + s = err.getmsg() + assert s == "takes no arguments (1 given)" err = ArgErrCount(0, 0, 1, False, False, [], 1) - s = err.getmsg('foo') - assert s == "foo() takes exactly 1 argument (0 given)" + s = err.getmsg() + assert s == "takes exactly 1 argument (0 given)" err = ArgErrCount(3, 0, 2, False, False, [], 0) - s = err.getmsg('foo') - assert s == "foo() takes exactly 2 arguments (3 given)" + s = err.getmsg() + assert s == "takes exactly 2 arguments (3 given)" err = ArgErrCount(3, 0, 2, False, False, ['a'], 0) - s = err.getmsg('foo') - assert s == "foo() takes at most 2 arguments (3 given)" + s = err.getmsg() + assert s == "takes at most 2 arguments (3 given)" err = ArgErrCount(1, 0, 2, True, False, [], 1) - s = err.getmsg('foo') - assert s == "foo() takes at least 2 arguments (1 given)" + s = err.getmsg() + assert s == "takes at least 2 arguments (1 given)" err = ArgErrCount(0, 1, 2, True, False, ['a'], 1) - s = err.getmsg('foo') - assert s == "foo() takes at least 1 non-keyword argument (0 given)" + s = err.getmsg() + assert s == "takes at least 1 non-keyword argument (0 given)" err = ArgErrCount(2, 1, 1, False, True, [], 0) - s = err.getmsg('foo') - assert s == "foo() takes exactly 1 non-keyword argument (2 given)" + s = err.getmsg() + assert s == "takes exactly 1 non-keyword argument (2 given)" err = ArgErrCount(0, 1, 1, False, True, [], 1) - s = err.getmsg('foo') - assert s == "foo() takes exactly 1 non-keyword argument (0 given)" + s = err.getmsg() + assert s == "takes exactly 1 non-keyword argument (0 given)" err = ArgErrCount(0, 1, 1, True, True, [], 1) - s = err.getmsg('foo') - assert s == "foo() takes at least 1 non-keyword argument (0 given)" + s = err.getmsg() + assert s == "takes at least 1 non-keyword argument (0 given)" err = ArgErrCount(2, 1, 1, False, True, ['a'], 0) - s = err.getmsg('foo') - assert s == "foo() takes at most 1 non-keyword argument (2 given)" + s = err.getmsg() + assert s == "takes at most 1 non-keyword argument (2 given)" def test_bad_type_for_star(self): space = self.space @@ -543,12 +543,12 @@ def test_unknown_keywords(self): space = DummySpace() err = ArgErrUnknownKwds(space, 1, ['a', 'b'], [True, False], None) - s = err.getmsg('foo') - assert s == "foo() got an unexpected keyword argument 'b'" + s = err.getmsg() + assert s == "got an unexpected keyword argument 'b'" err = ArgErrUnknownKwds(space, 2, ['a', 'b', 'c'], [True, False, False], None) - s = err.getmsg('foo') - assert s == "foo() got 2 unexpected keyword arguments" + s = err.getmsg() + assert s == "got 2 unexpected keyword arguments" def test_unknown_unicode_keyword(self): class DummySpaceUnicode(DummySpace): @@ -558,13 +558,13 @@ err = ArgErrUnknownKwds(space, 1, ['a', None, 'b', 'c'], [True, False, True, True], [unichr(0x1234), u'b', u'c']) - s = err.getmsg('foo') - assert s == "foo() got an unexpected keyword argument '\xe1\x88\xb4'" + s = err.getmsg() + assert s == "got an unexpected keyword argument '\xe1\x88\xb4'" def test_multiple_values(self): err = ArgErrMultipleValues('bla') - s = err.getmsg('foo') - assert s == "foo() got multiple values for keyword argument 'bla'" + s = err.getmsg() + assert s == "got multiple values for keyword argument 'bla'" class AppTestArgument: def test_error_message(self): diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py --- a/pypy/interpreter/typedef.py +++ b/pypy/interpreter/typedef.py @@ -54,7 +54,11 @@ # Hash support def default_identity_hash(space, w_obj): - return space.wrap(compute_identity_hash(w_obj)) + w_unique_id = w_obj.immutable_unique_id(space) + if w_unique_id is None: # common case + return space.wrap(compute_identity_hash(w_obj)) + else: + return space.hash(w_unique_id) # ____________________________________________________________ # diff --git a/pypy/jit/backend/llgraph/llimpl.py b/pypy/jit/backend/llgraph/llimpl.py --- a/pypy/jit/backend/llgraph/llimpl.py +++ b/pypy/jit/backend/llgraph/llimpl.py @@ -8,6 +8,7 @@ from pypy.objspace.flow.model import Variable, Constant from pypy.annotation import model as annmodel from pypy.jit.metainterp.history import REF, INT, FLOAT +from pypy.jit.metainterp import history from pypy.jit.codewriter import heaptracker from pypy.rpython.lltypesystem import lltype, llmemory, rclass, rstr, rffi from pypy.rpython.ootypesystem import ootype @@ -48,6 +49,11 @@ value._the_opaque_pointer = op return op +def _normalize(value): + if isinstance(value, lltype._ptr): + value = lltype.top_container(value._obj) + return value + def from_opaque_string(s): if isinstance(s, str): return s @@ -322,6 +328,14 @@ _variables.append(v) return r +def compile_started_vars(clt): + if not hasattr(clt, '_debug_argtypes'): # only when compiling the loop + argtypes = [v.concretetype for v in _variables] + try: + clt._debug_argtypes = argtypes + except AttributeError: # when 'clt' is actually a translated + pass # GcStruct + def compile_add(loop, opnum): loop = _from_opaque(loop) loop.operations.append(Operation(opnum)) @@ -347,6 +361,16 @@ op = loop.operations[-1] op.descr = weakref.ref(descr) +TARGET_TOKENS = weakref.WeakKeyDictionary() + +def compile_add_target_token(loop, descr, clt): + # here, 'clt' is the compiled_loop_token of the original loop that + # we are compiling + loop = _from_opaque(loop) + op = loop.operations[-1] + descrobj = _normalize(descr) + TARGET_TOKENS[descrobj] = loop, len(loop.operations), op.args, clt + def compile_add_var(loop, intvar): loop = _from_opaque(loop) op = loop.operations[-1] @@ -381,13 +405,25 @@ _variables.append(v) return r -def compile_add_jump_target(loop, loop_target): +def compile_add_jump_target(loop, targettoken, source_clt): loop = _from_opaque(loop) - loop_target = _from_opaque(loop_target) + descrobj = _normalize(targettoken) + (loop_target, target_opindex, target_inputargs, target_clt + ) = TARGET_TOKENS[descrobj] + # + try: + assert source_clt._debug_argtypes == target_clt._debug_argtypes + except AttributeError: # when translated + pass + # op = loop.operations[-1] op.jump_target = loop_target + op.jump_target_opindex = target_opindex + op.jump_target_inputargs = target_inputargs assert op.opnum == rop.JUMP - assert len(op.args) == len(loop_target.inputargs) + assert [v.concretetype for v in op.args] == ( + [v.concretetype for v in target_inputargs]) + # if loop_target == loop: log.info("compiling new loop") else: @@ -521,10 +557,11 @@ self.opindex += 1 continue if op.opnum == rop.JUMP: - assert len(op.jump_target.inputargs) == len(args) - self.env = dict(zip(op.jump_target.inputargs, args)) + inputargs = op.jump_target_inputargs + assert len(inputargs) == len(args) + self.env = dict(zip(inputargs, args)) self.loop = op.jump_target - self.opindex = 0 + self.opindex = op.jump_target_opindex _stats.exec_jumps += 1 elif op.opnum == rop.FINISH: if self.verbose: @@ -617,6 +654,15 @@ # return _op_default_implementation + def op_label(self, _, *args): + op = self.loop.operations[self.opindex] + assert op.opnum == rop.LABEL + assert len(op.args) == len(args) + newenv = {} + for v, value in zip(op.args, args): + newenv[v] = value + self.env = newenv + def op_debug_merge_point(self, _, *args): from pypy.jit.metainterp.warmspot import get_stats try: @@ -959,6 +1005,7 @@ self._may_force = self.opindex try: inpargs = _from_opaque(ctl.compiled_version).inputargs + assert len(inpargs) == len(args) for i, inparg in enumerate(inpargs): TYPE = inparg.concretetype if TYPE is lltype.Signed: @@ -1788,9 +1835,11 @@ setannotation(compile_start_int_var, annmodel.SomeInteger()) setannotation(compile_start_ref_var, annmodel.SomeInteger()) setannotation(compile_start_float_var, annmodel.SomeInteger()) +setannotation(compile_started_vars, annmodel.s_None) setannotation(compile_add, annmodel.s_None) setannotation(compile_add_descr, annmodel.s_None) setannotation(compile_add_descr_arg, annmodel.s_None) +setannotation(compile_add_target_token, annmodel.s_None) setannotation(compile_add_var, annmodel.s_None) setannotation(compile_add_int_const, annmodel.s_None) setannotation(compile_add_ref_const, annmodel.s_None) diff --git a/pypy/jit/backend/llgraph/runner.py b/pypy/jit/backend/llgraph/runner.py --- a/pypy/jit/backend/llgraph/runner.py +++ b/pypy/jit/backend/llgraph/runner.py @@ -37,7 +37,7 @@ def get_arg_types(self): return self.arg_types - def get_return_type(self): + def get_result_type(self): return self.typeinfo def get_extra_info(self): @@ -138,29 +138,30 @@ clt = original_loop_token.compiled_loop_token clt.loop_and_bridges.append(c) clt.compiling_a_bridge() - self._compile_loop_or_bridge(c, inputargs, operations) + self._compile_loop_or_bridge(c, inputargs, operations, clt) old, oldindex = faildescr._compiled_fail llimpl.compile_redirect_fail(old, oldindex, c) - def compile_loop(self, inputargs, operations, looptoken, log=True, name=''): + def compile_loop(self, inputargs, operations, jitcell_token, + log=True, name=''): """In a real assembler backend, this should assemble the given list of operations. Here we just generate a similar CompiledLoop instance. The code here is RPython, whereas the code in llimpl is not. """ c = llimpl.compile_start() - clt = model.CompiledLoopToken(self, looptoken.number) + clt = model.CompiledLoopToken(self, jitcell_token.number) clt.loop_and_bridges = [c] clt.compiled_version = c - looptoken.compiled_loop_token = clt - self._compile_loop_or_bridge(c, inputargs, operations) + jitcell_token.compiled_loop_token = clt + self._compile_loop_or_bridge(c, inputargs, operations, clt) def free_loop_and_bridges(self, compiled_loop_token): for c in compiled_loop_token.loop_and_bridges: llimpl.mark_as_free(c) model.AbstractCPU.free_loop_and_bridges(self, compiled_loop_token) - def _compile_loop_or_bridge(self, c, inputargs, operations): + def _compile_loop_or_bridge(self, c, inputargs, operations, clt): var2index = {} for box in inputargs: if isinstance(box, history.BoxInt): @@ -172,10 +173,11 @@ var2index[box] = llimpl.compile_start_float_var(c) else: raise Exception("box is: %r" % (box,)) - self._compile_operations(c, operations, var2index) + llimpl.compile_started_vars(clt) + self._compile_operations(c, operations, var2index, clt) return c - def _compile_operations(self, c, operations, var2index): + def _compile_operations(self, c, operations, var2index, clt): for op in operations: llimpl.compile_add(c, op.getopnum()) descr = op.getdescr() @@ -183,9 +185,11 @@ llimpl.compile_add_descr(c, descr.ofs, descr.typeinfo, descr.arg_types, descr.extrainfo, descr.width) - if (isinstance(descr, history.LoopToken) and - op.getopnum() != rop.JUMP): + if isinstance(descr, history.JitCellToken): + assert op.getopnum() != rop.JUMP llimpl.compile_add_loop_token(c, descr) + if isinstance(descr, history.TargetToken) and op.getopnum() == rop.LABEL: + llimpl.compile_add_target_token(c, descr, clt) if self.is_oo and isinstance(descr, (OODescr, MethDescr)): # hack hack, not rpython c._obj.externalobj.operations[-1].setdescr(descr) @@ -239,9 +243,7 @@ assert op.is_final() if op.getopnum() == rop.JUMP: targettoken = op.getdescr() - assert isinstance(targettoken, history.LoopToken) - compiled_version = targettoken.compiled_loop_token.compiled_version - llimpl.compile_add_jump_target(c, compiled_version) + llimpl.compile_add_jump_target(c, targettoken, clt) elif op.getopnum() == rop.FINISH: faildescr = op.getdescr() index = self.get_fail_descr_number(faildescr) @@ -260,21 +262,28 @@ self.latest_frame = frame return fail_index - def execute_token(self, loop_token): - """Calls the assembler generated for the given loop. - Returns the ResOperation that failed, of type rop.FAIL. - """ - fail_index = self._execute_token(loop_token) - return self.get_fail_descr_from_number(fail_index) - - def set_future_value_int(self, index, intvalue): - llimpl.set_future_value_int(index, intvalue) - - def set_future_value_ref(self, index, objvalue): - llimpl.set_future_value_ref(index, objvalue) - - def set_future_value_float(self, index, floatvalue): - llimpl.set_future_value_float(index, floatvalue) + def make_execute_token(self, *argtypes): + nb_args = len(argtypes) + unroll_argtypes = unrolling_iterable(list(enumerate(argtypes))) + # + def execute_token(loop_token, *args): + assert len(args) == nb_args + for index, TYPE in unroll_argtypes: + x = args[index] + assert TYPE == lltype.typeOf(x) + if TYPE == lltype.Signed: + llimpl.set_future_value_int(index, x) + elif TYPE == llmemory.GCREF: + llimpl.set_future_value_ref(index, x) + elif TYPE == longlong.FLOATSTORAGE: + llimpl.set_future_value_float(index, x) + else: + assert 0 + # + fail_index = self._execute_token(loop_token) + return self.get_fail_descr_from_number(fail_index) + # + return execute_token def get_latest_value_int(self, index): return llimpl.frame_int_getvalue(self.latest_frame, index) diff --git a/pypy/jit/backend/llsupport/descr.py b/pypy/jit/backend/llsupport/descr.py --- a/pypy/jit/backend/llsupport/descr.py +++ b/pypy/jit/backend/llsupport/descr.py @@ -5,11 +5,7 @@ from pypy.jit.metainterp.history import AbstractDescr, getkind from pypy.jit.metainterp import history from pypy.jit.codewriter import heaptracker, longlong - -# The point of the class organization in this file is to make instances -# as compact as possible. This is done by not storing the field size or -# the 'is_pointer_field' flag in the instance itself but in the class -# (in methods actually) using a few classes instead of just one. +from pypy.jit.codewriter.longlong import is_longlong class GcCache(object): @@ -19,6 +15,7 @@ self._cache_size = {} self._cache_field = {} self._cache_array = {} + self._cache_arraylen = {} self._cache_call = {} self._cache_interiorfield = {} @@ -26,24 +23,15 @@ assert isinstance(STRUCT, lltype.GcStruct) def init_array_descr(self, ARRAY, arraydescr): - assert isinstance(ARRAY, lltype.GcArray) + assert (isinstance(ARRAY, lltype.GcArray) or + isinstance(ARRAY, lltype.GcStruct) and ARRAY._arrayfld) -if lltype.SignedLongLong is lltype.Signed: - def is_longlong(TYPE): - return False -else: - assert rffi.sizeof(lltype.SignedLongLong) == rffi.sizeof(lltype.Float) - def is_longlong(TYPE): - return TYPE in (lltype.SignedLongLong, lltype.UnsignedLongLong) - # ____________________________________________________________ # SizeDescrs class SizeDescr(AbstractDescr): size = 0 # help translation - is_immutable = False - tid = llop.combine_ushort(lltype.Signed, 0, 0) def __init__(self, size, count_fields_if_immut=-1): @@ -77,265 +65,247 @@ cache[STRUCT] = sizedescr return sizedescr + # ____________________________________________________________ # FieldDescrs -class BaseFieldDescr(AbstractDescr): +FLAG_POINTER = 'P' +FLAG_FLOAT = 'F' +FLAG_UNSIGNED = 'U' +FLAG_SIGNED = 'S' +FLAG_STRUCT = 'X' +FLAG_VOID = 'V' + +class FieldDescr(AbstractDescr): + name = '' offset = 0 # help translation - name = '' - _clsname = '' + field_size = 0 + flag = '\x00' - def __init__(self, name, offset): + def __init__(self, name, offset, field_size, flag): self.name = name self.offset = offset + self.field_size = field_size + self.flag = flag + + def is_pointer_field(self): + return self.flag == FLAG_POINTER + + def is_float_field(self): + return self.flag == FLAG_FLOAT + + def is_field_signed(self): + return self.flag == FLAG_SIGNED def sort_key(self): return self.offset - def get_field_size(self, translate_support_code): - raise NotImplementedError + def repr_of_descr(self): + return '<Field%s %s %s>' % (self.flag, self.name, self.offset) - _is_pointer_field = False # unless overridden by GcPtrFieldDescr - _is_float_field = False # unless overridden by FloatFieldDescr - _is_field_signed = False # unless overridden by XxxFieldDescr - - def is_pointer_field(self): - return self._is_pointer_field - - def is_float_field(self): - return self._is_float_field - - def is_field_signed(self): - return self._is_field_signed - - def repr_of_descr(self): - return '<%s %s %s>' % (self._clsname, self.name, self.offset) - -class DynamicFieldDescr(BaseFieldDescr): - def __init__(self, offset, fieldsize, is_pointer, is_float, is_signed): - self.offset = offset - self._fieldsize = fieldsize - self._is_pointer_field = is_pointer - self._is_float_field = is_float - self._is_field_signed = is_signed - - def get_field_size(self, translate_support_code): - return self._fieldsize - -class NonGcPtrFieldDescr(BaseFieldDescr): - _clsname = 'NonGcPtrFieldDescr' - def get_field_size(self, translate_support_code): - return symbolic.get_size_of_ptr(translate_support_code) - -class GcPtrFieldDescr(NonGcPtrFieldDescr): - _clsname = 'GcPtrFieldDescr' - _is_pointer_field = True - -def getFieldDescrClass(TYPE): - return getDescrClass(TYPE, BaseFieldDescr, GcPtrFieldDescr, - NonGcPtrFieldDescr, 'Field', 'get_field_size', - '_is_float_field', '_is_field_signed') def get_field_descr(gccache, STRUCT, fieldname): cache = gccache._cache_field try: return cache[STRUCT][fieldname] except KeyError: - offset, _ = symbolic.get_field_token(STRUCT, fieldname, - gccache.translate_support_code) + offset, size = symbolic.get_field_token(STRUCT, fieldname, + gccache.translate_support_code) FIELDTYPE = getattr(STRUCT, fieldname) + flag = get_type_flag(FIELDTYPE) name = '%s.%s' % (STRUCT._name, fieldname) - fielddescr = getFieldDescrClass(FIELDTYPE)(name, offset) + fielddescr = FieldDescr(name, offset, size, flag) cachedict = cache.setdefault(STRUCT, {}) cachedict[fieldname] = fielddescr return fielddescr +def get_type_flag(TYPE): + if isinstance(TYPE, lltype.Ptr): + if TYPE.TO._gckind == 'gc': + return FLAG_POINTER + else: + return FLAG_UNSIGNED + if isinstance(TYPE, lltype.Struct): + return FLAG_STRUCT + if TYPE is lltype.Float or is_longlong(TYPE): + return FLAG_FLOAT + if (TYPE is not lltype.Bool and isinstance(TYPE, lltype.Number) and + rffi.cast(TYPE, -1) == -1): + return FLAG_SIGNED + return FLAG_UNSIGNED + +def get_field_arraylen_descr(gccache, ARRAY_OR_STRUCT): + cache = gccache._cache_arraylen + try: + return cache[ARRAY_OR_STRUCT] + except KeyError: + tsc = gccache.translate_support_code + (_, _, ofs) = symbolic.get_array_token(ARRAY_OR_STRUCT, tsc) + size = symbolic.get_size(lltype.Signed, tsc) + result = FieldDescr("len", ofs, size, get_type_flag(lltype.Signed)) + cache[ARRAY_OR_STRUCT] = result + return result + + # ____________________________________________________________ # ArrayDescrs -_A = lltype.GcArray(lltype.Signed) # a random gcarray -_AF = lltype.GcArray(lltype.Float) # an array of C doubles +class ArrayDescr(AbstractDescr): + tid = 0 + basesize = 0 # workaround for the annotator + itemsize = 0 + lendescr = None + flag = '\x00' - -class BaseArrayDescr(AbstractDescr): - _clsname = '' - tid = llop.combine_ushort(lltype.Signed, 0, 0) - - def get_base_size(self, translate_support_code): - basesize, _, _ = symbolic.get_array_token(_A, translate_support_code) - return basesize - - def get_ofs_length(self, translate_support_code): - _, _, ofslength = symbolic.get_array_token(_A, translate_support_code) - return ofslength - - def get_item_size(self, translate_support_code): - raise NotImplementedError - - _is_array_of_pointers = False # unless overridden by GcPtrArrayDescr - _is_array_of_floats = False # unless overridden by FloatArrayDescr - _is_array_of_structs = False # unless overridden by StructArrayDescr - _is_item_signed = False # unless overridden by XxxArrayDescr + def __init__(self, basesize, itemsize, lendescr, flag): + self.basesize = basesize + self.itemsize = itemsize + self.lendescr = lendescr # or None, if no length + self.flag = flag def is_array_of_pointers(self): - return self._is_array_of_pointers + return self.flag == FLAG_POINTER def is_array_of_floats(self): - return self._is_array_of_floats + return self.flag == FLAG_FLOAT + + def is_item_signed(self): + return self.flag == FLAG_SIGNED def is_array_of_structs(self): - return self._is_array_of_structs - - def is_item_signed(self): - return self._is_item_signed + return self.flag == FLAG_STRUCT def repr_of_descr(self): - return '<%s>' % self._clsname + return '<Array%s %s>' % (self.flag, self.itemsize) -class NonGcPtrArrayDescr(BaseArrayDescr): - _clsname = 'NonGcPtrArrayDescr' - def get_item_size(self, translate_support_code): - return symbolic.get_size_of_ptr(translate_support_code) - -class GcPtrArrayDescr(NonGcPtrArrayDescr): - _clsname = 'GcPtrArrayDescr' - _is_array_of_pointers = True - -class FloatArrayDescr(BaseArrayDescr): - _clsname = 'FloatArrayDescr' - _is_array_of_floats = True - def get_base_size(self, translate_support_code): - basesize, _, _ = symbolic.get_array_token(_AF, translate_support_code) - return basesize - def get_item_size(self, translate_support_code): - return symbolic.get_size(lltype.Float, translate_support_code) - -class StructArrayDescr(BaseArrayDescr): - _clsname = 'StructArrayDescr' - _is_array_of_structs = True - -class BaseArrayNoLengthDescr(BaseArrayDescr): - def get_base_size(self, translate_support_code): - return 0 - - def get_ofs_length(self, translate_support_code): - return -1 - -class DynamicArrayNoLengthDescr(BaseArrayNoLengthDescr): - def __init__(self, itemsize): - self.itemsize = itemsize - - def get_item_size(self, translate_support_code): - return self.itemsize - -class NonGcPtrArrayNoLengthDescr(BaseArrayNoLengthDescr): - _clsname = 'NonGcPtrArrayNoLengthDescr' - def get_item_size(self, translate_support_code): - return symbolic.get_size_of_ptr(translate_support_code) - -class GcPtrArrayNoLengthDescr(NonGcPtrArrayNoLengthDescr): - _clsname = 'GcPtrArrayNoLengthDescr' - _is_array_of_pointers = True - -def getArrayDescrClass(ARRAY): - if ARRAY.OF is lltype.Float: - return FloatArrayDescr - elif isinstance(ARRAY.OF, lltype.Struct): - class Descr(StructArrayDescr): - _clsname = '%sArrayDescr' % ARRAY.OF._name - def get_item_size(self, translate_support_code): - return symbolic.get_size(ARRAY.OF, translate_support_code) - Descr.__name__ = Descr._clsname - return Descr - return getDescrClass(ARRAY.OF, BaseArrayDescr, GcPtrArrayDescr, - NonGcPtrArrayDescr, 'Array', 'get_item_size', - '_is_array_of_floats', '_is_item_signed') - -def getArrayNoLengthDescrClass(ARRAY): - return getDescrClass(ARRAY.OF, BaseArrayNoLengthDescr, GcPtrArrayNoLengthDescr, - NonGcPtrArrayNoLengthDescr, 'ArrayNoLength', 'get_item_size', - '_is_array_of_floats', '_is_item_signed') - -def get_array_descr(gccache, ARRAY): +def get_array_descr(gccache, ARRAY_OR_STRUCT): cache = gccache._cache_array try: - return cache[ARRAY] + return cache[ARRAY_OR_STRUCT] except KeyError: - # we only support Arrays that are either GcArrays, or raw no-length - # non-gc Arrays. - if ARRAY._hints.get('nolength', False): - assert not isinstance(ARRAY, lltype.GcArray) - arraydescr = getArrayNoLengthDescrClass(ARRAY)() + tsc = gccache.translate_support_code + basesize, itemsize, _ = symbolic.get_array_token(ARRAY_OR_STRUCT, tsc) + if isinstance(ARRAY_OR_STRUCT, lltype.Array): + ARRAY_INSIDE = ARRAY_OR_STRUCT else: - assert isinstance(ARRAY, lltype.GcArray) - arraydescr = getArrayDescrClass(ARRAY)() - # verify basic assumption that all arrays' basesize and ofslength - # are equal - basesize, itemsize, ofslength = symbolic.get_array_token(ARRAY, False) - assert basesize == arraydescr.get_base_size(False) - assert itemsize == arraydescr.get_item_size(False) - if not ARRAY._hints.get('nolength', False): - assert ofslength == arraydescr.get_ofs_length(False) - if isinstance(ARRAY, lltype.GcArray): - gccache.init_array_descr(ARRAY, arraydescr) - cache[ARRAY] = arraydescr + ARRAY_INSIDE = ARRAY_OR_STRUCT._flds[ARRAY_OR_STRUCT._arrayfld] + if ARRAY_INSIDE._hints.get('nolength', False): + lendescr = None + else: + lendescr = get_field_arraylen_descr(gccache, ARRAY_OR_STRUCT) + flag = get_type_flag(ARRAY_INSIDE.OF) + arraydescr = ArrayDescr(basesize, itemsize, lendescr, flag) + if ARRAY_OR_STRUCT._gckind == 'gc': + gccache.init_array_descr(ARRAY_OR_STRUCT, arraydescr) + cache[ARRAY_OR_STRUCT] = arraydescr return arraydescr + # ____________________________________________________________ # InteriorFieldDescr class InteriorFieldDescr(AbstractDescr): - arraydescr = BaseArrayDescr() # workaround for the annotator - fielddescr = BaseFieldDescr('', 0) + arraydescr = ArrayDescr(0, 0, None, '\x00') # workaround for the annotator + fielddescr = FieldDescr('', 0, 0, '\x00') def __init__(self, arraydescr, fielddescr): + assert arraydescr.flag == FLAG_STRUCT self.arraydescr = arraydescr self.fielddescr = fielddescr + def sort_key(self): + return self.fielddescr.sort_key() + def is_pointer_field(self): return self.fielddescr.is_pointer_field() def is_float_field(self): return self.fielddescr.is_float_field() - def sort_key(self): - return self.fielddescr.sort_key() - def repr_of_descr(self): return '<InteriorFieldDescr %s>' % self.fielddescr.repr_of_descr() -def get_interiorfield_descr(gc_ll_descr, ARRAY, FIELDTP, name): +def get_interiorfield_descr(gc_ll_descr, ARRAY, name): cache = gc_ll_descr._cache_interiorfield try: - return cache[(ARRAY, FIELDTP, name)] + return cache[(ARRAY, name)] except KeyError: arraydescr = get_array_descr(gc_ll_descr, ARRAY) - fielddescr = get_field_descr(gc_ll_descr, FIELDTP, name) + fielddescr = get_field_descr(gc_ll_descr, ARRAY.OF, name) descr = InteriorFieldDescr(arraydescr, fielddescr) - cache[(ARRAY, FIELDTP, name)] = descr + cache[(ARRAY, name)] = descr return descr +def get_dynamic_interiorfield_descr(gc_ll_descr, offset, width, fieldsize, + is_pointer, is_float, is_signed): + arraydescr = ArrayDescr(0, width, None, FLAG_STRUCT) + if is_pointer: + assert not is_float + flag = FLAG_POINTER + elif is_float: + flag = FLAG_FLOAT + elif is_signed: + flag = FLAG_SIGNED + else: + flag = FLAG_UNSIGNED + fielddescr = FieldDescr('dynamic', offset, fieldsize, flag) + return InteriorFieldDescr(arraydescr, fielddescr) + + # ____________________________________________________________ # CallDescrs -class BaseCallDescr(AbstractDescr): - _clsname = '' - loop_token = None +class CallDescr(AbstractDescr): arg_classes = '' # <-- annotation hack + result_type = '\x00' + result_flag = '\x00' ffi_flags = 1 + call_stub_i = staticmethod(lambda func, args_i, args_r, args_f: + 0) + call_stub_r = staticmethod(lambda func, args_i, args_r, args_f: + lltype.nullptr(llmemory.GCREF.TO)) + call_stub_f = staticmethod(lambda func,args_i,args_r,args_f: + longlong.ZEROF) - def __init__(self, arg_classes, extrainfo=None, ffi_flags=1): - self.arg_classes = arg_classes # string of "r" and "i" (ref/int) + def __init__(self, arg_classes, result_type, result_signed, result_size, + extrainfo=None, ffi_flags=1): + """ + 'arg_classes' is a string of characters, one per argument: + 'i', 'r', 'f', 'L', 'S' + + 'result_type' is one character from the same list or 'v' + + 'result_signed' is a boolean True/False + """ + self.arg_classes = arg_classes + self.result_type = result_type + self.result_size = result_size self.extrainfo = extrainfo self.ffi_flags = ffi_flags # NB. the default ffi_flags is 1, meaning FUNCFLAG_CDECL, which # makes sense on Windows as it's the one for all the C functions # we are compiling together with the JIT. On non-Windows platforms # it is just ignored anyway. + if result_type == 'v': + result_flag = FLAG_VOID + elif result_type == 'i': + if result_signed: + result_flag = FLAG_SIGNED + else: + result_flag = FLAG_UNSIGNED + elif result_type == history.REF: + result_flag = FLAG_POINTER + elif result_type == history.FLOAT or result_type == 'L': + result_flag = FLAG_FLOAT + elif result_type == 'S': + result_flag = FLAG_UNSIGNED + else: + raise NotImplementedError("result_type = '%s'" % (result_type,)) + self.result_flag = result_flag def __repr__(self): - res = '%s(%s)' % (self.__class__.__name__, self.arg_classes) + res = 'CallDescr(%s)' % (self.arg_classes,) extraeffect = getattr(self.extrainfo, 'extraeffect', None) if extraeffect is not None: res += ' EF=%r' % extraeffect @@ -363,14 +333,14 @@ def get_arg_types(self): return self.arg_classes - def get_return_type(self): - return self._return_type + def get_result_type(self): + return self.result_type - def get_result_size(self, translate_support_code): - raise NotImplementedError + def get_result_size(self): + return self.result_size def is_result_signed(self): - return False # unless overridden + return self.result_flag == FLAG_SIGNED def create_call_stub(self, rtyper, RESULT): from pypy.rlib.clibffi import FFI_DEFAULT_ABI @@ -408,18 +378,26 @@ seen = {'i': 0, 'r': 0, 'f': 0} args = ", ".join([process(c) for c in self.arg_classes]) - if self.get_return_type() == history.INT: + result_type = self.get_result_type() + if result_type == history.INT: result = 'rffi.cast(lltype.Signed, res)' - elif self.get_return_type() == history.REF: + category = 'i' + elif result_type == history.REF: + assert RESULT == llmemory.GCREF # should be ensured by the caller result = 'lltype.cast_opaque_ptr(llmemory.GCREF, res)' - elif self.get_return_type() == history.FLOAT: + category = 'r' + elif result_type == history.FLOAT: result = 'longlong.getfloatstorage(res)' - elif self.get_return_type() == 'L': + category = 'f' + elif result_type == 'L': result = 'rffi.cast(lltype.SignedLongLong, res)' - elif self.get_return_type() == history.VOID: - result = 'None' - elif self.get_return_type() == 'S': + category = 'f' + elif result_type == history.VOID: + result = '0' + category = 'i' + elif result_type == 'S': result = 'longlong.singlefloat2int(res)' + category = 'i' else: assert 0 source = py.code.Source(""" @@ -433,10 +411,13 @@ d = globals().copy() d.update(locals()) exec source.compile() in d - self.call_stub = d['call_stub'] + call_stub = d['call_stub'] + # store the function into one of three attributes, to preserve + # type-correctness of the return value + setattr(self, 'call_stub_%s' % category, call_stub) def verify_types(self, args_i, args_r, args_f, return_type): - assert self._return_type in return_type + assert self.result_type in return_type assert (self.arg_classes.count('i') + self.arg_classes.count('S')) == len(args_i or ()) assert self.arg_classes.count('r') == len(args_r or ()) @@ -444,161 +425,56 @@ self.arg_classes.count('L')) == len(args_f or ()) def repr_of_descr(self): - return '<%s>' % self._clsname + res = 'Call%s %d' % (self.result_type, self.result_size) + if self.arg_classes: + res += ' ' + self.arg_classes + if self.extrainfo: + res += ' EF=%d' % self.extrainfo.extraeffect + oopspecindex = self.extrainfo.oopspecindex + if oopspecindex: + res += ' OS=%d' % oopspecindex + return '<%s>' % res -class BaseIntCallDescr(BaseCallDescr): - # Base class of the various subclasses of descrs corresponding to - # calls having a return kind of 'int' (including non-gc pointers). - # The inheritance hierarchy is a bit different than with other Descr - # classes because of the 'call_stub' attribute, which is of type - # - # lambda func, args_i, args_r, args_f --> int/ref/float/void - # - # The purpose of BaseIntCallDescr is to be the parent of all classes - # in which 'call_stub' has a return kind of 'int'. - _return_type = history.INT - call_stub = staticmethod(lambda func, args_i, args_r, args_f: 0) - - _is_result_signed = False # can be overridden in XxxCallDescr - def is_result_signed(self): - return self._is_result_signed - -class DynamicIntCallDescr(BaseIntCallDescr): - """ - calldescr that works for every integer type, by explicitly passing it the - size of the result. Used only by get_call_descr_dynamic - """ - _clsname = 'DynamicIntCallDescr' - - def __init__(self, arg_classes, result_size, result_sign, extrainfo, ffi_flags): - BaseIntCallDescr.__init__(self, arg_classes, extrainfo, ffi_flags) - assert isinstance(result_sign, bool) - self._result_size = chr(result_size) - self._result_sign = result_sign - - def get_result_size(self, translate_support_code): - return ord(self._result_size) - - def is_result_signed(self): - return self._result_sign - - -class NonGcPtrCallDescr(BaseIntCallDescr): - _clsname = 'NonGcPtrCallDescr' - def get_result_size(self, translate_support_code): - return symbolic.get_size_of_ptr(translate_support_code) - -class GcPtrCallDescr(BaseCallDescr): - _clsname = 'GcPtrCallDescr' - _return_type = history.REF - call_stub = staticmethod(lambda func, args_i, args_r, args_f: - lltype.nullptr(llmemory.GCREF.TO)) - def get_result_size(self, translate_support_code): - return symbolic.get_size_of_ptr(translate_support_code) - -class FloatCallDescr(BaseCallDescr): - _clsname = 'FloatCallDescr' - _return_type = history.FLOAT - call_stub = staticmethod(lambda func,args_i,args_r,args_f: longlong.ZEROF) - def get_result_size(self, translate_support_code): - return symbolic.get_size(lltype.Float, translate_support_code) - -class LongLongCallDescr(FloatCallDescr): - _clsname = 'LongLongCallDescr' - _return_type = 'L' - -class VoidCallDescr(BaseCallDescr): - _clsname = 'VoidCallDescr' - _return_type = history.VOID - call_stub = staticmethod(lambda func, args_i, args_r, args_f: None) - def get_result_size(self, translate_support_code): - return 0 - -_SingleFloatCallDescr = None # built lazily - -def getCallDescrClass(RESULT): - if RESULT is lltype.Void: - return VoidCallDescr - if RESULT is lltype.Float: - return FloatCallDescr - if RESULT is lltype.SingleFloat: - global _SingleFloatCallDescr - if _SingleFloatCallDescr is None: - assert rffi.sizeof(rffi.UINT) == rffi.sizeof(RESULT) - class SingleFloatCallDescr(getCallDescrClass(rffi.UINT)): - _clsname = 'SingleFloatCallDescr' - _return_type = 'S' - _SingleFloatCallDescr = SingleFloatCallDescr - return _SingleFloatCallDescr - if is_longlong(RESULT): - return LongLongCallDescr - return getDescrClass(RESULT, BaseIntCallDescr, GcPtrCallDescr, - NonGcPtrCallDescr, 'Call', 'get_result_size', - Ellipsis, # <= floatattrname should not be used here - '_is_result_signed') -getCallDescrClass._annspecialcase_ = 'specialize:memo' +def map_type_to_argclass(ARG, accept_void=False): + kind = getkind(ARG) + if kind == 'int': + if ARG is lltype.SingleFloat: return 'S' + else: return 'i' + elif kind == 'ref': return 'r' + elif kind == 'float': + if is_longlong(ARG): return 'L' + else: return 'f' + elif kind == 'void': + if accept_void: return 'v' + raise NotImplementedError('ARG = %r' % (ARG,)) def get_call_descr(gccache, ARGS, RESULT, extrainfo=None): - arg_classes = [] - for ARG in ARGS: - kind = getkind(ARG) - if kind == 'int': - if ARG is lltype.SingleFloat: - arg_classes.append('S') + arg_classes = map(map_type_to_argclass, ARGS) + arg_classes = ''.join(arg_classes) + result_type = map_type_to_argclass(RESULT, accept_void=True) + RESULT_ERASED = RESULT + if RESULT is lltype.Void: + result_size = 0 + result_signed = False + else: + if isinstance(RESULT, lltype.Ptr): + # avoid too many CallDescrs + if result_type == 'r': + RESULT_ERASED = llmemory.GCREF else: - arg_classes.append('i') - elif kind == 'ref': arg_classes.append('r') - elif kind == 'float': - if is_longlong(ARG): - arg_classes.append('L') - else: - arg_classes.append('f') - else: - raise NotImplementedError('ARG = %r' % (ARG,)) - arg_classes = ''.join(arg_classes) - cls = getCallDescrClass(RESULT) - key = (cls, arg_classes, extrainfo) + RESULT_ERASED = llmemory.Address + result_size = symbolic.get_size(RESULT_ERASED, + gccache.translate_support_code) + result_signed = get_type_flag(RESULT) == FLAG_SIGNED + key = (arg_classes, result_type, result_signed, RESULT_ERASED, extrainfo) cache = gccache._cache_call try: - return cache[key] + calldescr = cache[key] except KeyError: - calldescr = cls(arg_classes, extrainfo) - calldescr.create_call_stub(gccache.rtyper, RESULT) + calldescr = CallDescr(arg_classes, result_type, result_signed, + result_size, extrainfo) + calldescr.create_call_stub(gccache.rtyper, RESULT_ERASED) cache[key] = calldescr - return calldescr - - -# ____________________________________________________________ - -def getDescrClass(TYPE, BaseDescr, GcPtrDescr, NonGcPtrDescr, - nameprefix, methodname, floatattrname, signedattrname, - _cache={}): - if isinstance(TYPE, lltype.Ptr): - if TYPE.TO._gckind == 'gc': - return GcPtrDescr - else: - return NonGcPtrDescr - if TYPE is lltype.SingleFloat: - assert rffi.sizeof(rffi.UINT) == rffi.sizeof(TYPE) - TYPE = rffi.UINT - try: - return _cache[nameprefix, TYPE] - except KeyError: - # - class Descr(BaseDescr): - _clsname = '%s%sDescr' % (TYPE._name, nameprefix) - Descr.__name__ = Descr._clsname - # - def method(self, translate_support_code): - return symbolic.get_size(TYPE, translate_support_code) - setattr(Descr, methodname, method) - # - if TYPE is lltype.Float or is_longlong(TYPE): - setattr(Descr, floatattrname, True) - elif (TYPE is not lltype.Bool and isinstance(TYPE, lltype.Number) and - rffi.cast(TYPE, -1) == -1): - setattr(Descr, signedattrname, True) - # - _cache[nameprefix, TYPE] = Descr - return Descr + assert repr(calldescr.result_size) == repr(result_size) + return calldescr diff --git a/pypy/jit/backend/llsupport/ffisupport.py b/pypy/jit/backend/llsupport/ffisupport.py --- a/pypy/jit/backend/llsupport/ffisupport.py +++ b/pypy/jit/backend/llsupport/ffisupport.py @@ -1,9 +1,7 @@ from pypy.rlib.rarithmetic import intmask from pypy.jit.metainterp import history from pypy.rpython.lltypesystem import rffi -from pypy.jit.backend.llsupport.descr import ( - DynamicIntCallDescr, NonGcPtrCallDescr, FloatCallDescr, VoidCallDescr, - LongLongCallDescr, getCallDescrClass) +from pypy.jit.backend.llsupport.descr import CallDescr class UnsupportedKind(Exception): pass @@ -16,29 +14,13 @@ argkinds = [get_ffi_type_kind(cpu, arg) for arg in ffi_args] except UnsupportedKind: return None - arg_classes = ''.join(argkinds) - if reskind == history.INT: - size = intmask(ffi_result.c_size) - signed = is_ffi_type_signed(ffi_result) - return DynamicIntCallDescr(arg_classes, size, signed, extrainfo, - ffi_flags=ffi_flags) - elif reskind == history.REF: - return NonGcPtrCallDescr(arg_classes, extrainfo, - ffi_flags=ffi_flags) - elif reskind == history.FLOAT: - return FloatCallDescr(arg_classes, extrainfo, - ffi_flags=ffi_flags) - elif reskind == history.VOID: - return VoidCallDescr(arg_classes, extrainfo, - ffi_flags=ffi_flags) - elif reskind == 'L': - return LongLongCallDescr(arg_classes, extrainfo, - ffi_flags=ffi_flags) - elif reskind == 'S': - SingleFloatCallDescr = getCallDescrClass(rffi.FLOAT) - return SingleFloatCallDescr(arg_classes, extrainfo, - ffi_flags=ffi_flags) - assert False + if reskind == history.VOID: + result_size = 0 + else: + result_size = intmask(ffi_result.c_size) + argkinds = ''.join(argkinds) + return CallDescr(argkinds, reskind, is_ffi_type_signed(ffi_result), + result_size, extrainfo, ffi_flags=ffi_flags) def get_ffi_type_kind(cpu, ffi_type): from pypy.rlib.libffi import types diff --git a/pypy/jit/backend/llsupport/gc.py b/pypy/jit/backend/llsupport/gc.py --- a/pypy/jit/backend/llsupport/gc.py +++ b/pypy/jit/backend/llsupport/gc.py @@ -1,6 +1,6 @@ import os from pypy.rlib import rgc -from pypy.rlib.objectmodel import we_are_translated +from pypy.rlib.objectmodel import we_are_translated, specialize from pypy.rlib.debug import fatalerror from pypy.rlib.rarithmetic import ovfcheck from pypy.rpython.lltypesystem import lltype, llmemory, rffi, rclass, rstr @@ -8,52 +8,93 @@ from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.annlowlevel import llhelper from pypy.translator.tool.cbuild import ExternalCompilationInfo -from pypy.jit.metainterp.history import BoxInt, BoxPtr, ConstInt, ConstPtr -from pypy.jit.metainterp.history import AbstractDescr +from pypy.jit.codewriter import heaptracker +from pypy.jit.metainterp.history import ConstPtr, AbstractDescr from pypy.jit.metainterp.resoperation import ResOperation, rop from pypy.jit.backend.llsupport import symbolic from pypy.jit.backend.llsupport.symbolic import WORD -from pypy.jit.backend.llsupport.descr import BaseSizeDescr, BaseArrayDescr +from pypy.jit.backend.llsupport.descr import SizeDescr, ArrayDescr from pypy.jit.backend.llsupport.descr import GcCache, get_field_descr -from pypy.jit.backend.llsupport.descr import GcPtrFieldDescr +from pypy.jit.backend.llsupport.descr import get_array_descr from pypy.jit.backend.llsupport.descr import get_call_descr +from pypy.jit.backend.llsupport.rewrite import GcRewriterAssembler from pypy.rpython.memory.gctransform import asmgcroot # ____________________________________________________________ class GcLLDescription(GcCache): - minimal_size_in_nursery = 0 - get_malloc_slowpath_addr = None def __init__(self, gcdescr, translator=None, rtyper=None): GcCache.__init__(self, translator is not None, rtyper) self.gcdescr = gcdescr + if translator and translator.config.translation.gcremovetypeptr: + self.fielddescr_vtable = None + else: + self.fielddescr_vtable = get_field_descr(self, rclass.OBJECT, + 'typeptr') + self._generated_functions = [] + + def _setup_str(self): + self.str_descr = get_array_descr(self, rstr.STR) + self.unicode_descr = get_array_descr(self, rstr.UNICODE) + + def generate_function(self, funcname, func, ARGS, RESULT=llmemory.GCREF): + """Generates a variant of malloc with the given name and the given + arguments. It should return NULL if out of memory. If it raises + anything, it must be an optional MemoryError. + """ + FUNCPTR = lltype.Ptr(lltype.FuncType(ARGS, RESULT)) + descr = get_call_descr(self, ARGS, RESULT) + setattr(self, funcname, func) + setattr(self, funcname + '_FUNCPTR', FUNCPTR) + setattr(self, funcname + '_descr', descr) + self._generated_functions.append(funcname) + + @specialize.arg(1) + def get_malloc_fn(self, funcname): + func = getattr(self, funcname) + FUNC = getattr(self, funcname + '_FUNCPTR') + return llhelper(FUNC, func) + + @specialize.arg(1) + def get_malloc_fn_addr(self, funcname): + ll_func = self.get_malloc_fn(funcname) + return heaptracker.adr2int(llmemory.cast_ptr_to_adr(ll_func)) + def _freeze_(self): return True def initialize(self): pass def do_write_barrier(self, gcref_struct, gcref_newptr): pass - def rewrite_assembler(self, cpu, operations, gcrefs_output_list): - return operations - def can_inline_malloc(self, descr): - return False - def can_inline_malloc_varsize(self, descr, num_elem): + def can_use_nursery_malloc(self, size): return False def has_write_barrier_class(self): return None def freeing_block(self, start, stop): pass + def get_nursery_free_addr(self): + raise NotImplementedError + def get_nursery_top_addr(self): + raise NotImplementedError - def get_funcptr_for_newarray(self): - return llhelper(self.GC_MALLOC_ARRAY, self.malloc_array) - def get_funcptr_for_newstr(self): - return llhelper(self.GC_MALLOC_STR_UNICODE, self.malloc_str) - def get_funcptr_for_newunicode(self): - return llhelper(self.GC_MALLOC_STR_UNICODE, self.malloc_unicode) + def gc_malloc(self, sizedescr): + """Blackhole: do a 'bh_new'. Also used for 'bh_new_with_vtable', + with the vtable pointer set manually afterwards.""" + assert isinstance(sizedescr, SizeDescr) + return self._bh_malloc(sizedescr) + def gc_malloc_array(self, arraydescr, num_elem): + assert isinstance(arraydescr, ArrayDescr) + return self._bh_malloc_array(arraydescr, num_elem) - def record_constptrs(self, op, gcrefs_output_list): + def gc_malloc_str(self, num_elem): + return self._bh_malloc_array(self.str_descr, num_elem) + + def gc_malloc_unicode(self, num_elem): + return self._bh_malloc_array(self.unicode_descr, num_elem) + + def _record_constptrs(self, op, gcrefs_output_list): for i in range(op.numargs()): v = op.getarg(i) if isinstance(v, ConstPtr) and bool(v.value): @@ -61,11 +102,27 @@ rgc._make_sure_does_not_move(p) gcrefs_output_list.append(p) + def rewrite_assembler(self, cpu, operations, gcrefs_output_list): + rewriter = GcRewriterAssembler(self, cpu) + newops = rewriter.rewrite(operations) + # record all GCREFs, because the GC (or Boehm) cannot see them and + # keep them alive if they end up as constants in the assembler + for op in newops: + self._record_constptrs(op, gcrefs_output_list) + return newops + # ____________________________________________________________ class GcLLDescr_boehm(GcLLDescription): - moving_gc = False - gcrootmap = None + kind = 'boehm' + moving_gc = False + round_up = False + gcrootmap = None + write_barrier_descr = None + fielddescr_tid = None + str_type_id = 0 + unicode_type_id = 0 + get_malloc_slowpath_addr = None @classmethod def configure_boehm_once(cls): @@ -76,6 +133,16 @@ from pypy.rpython.tool import rffi_platform compilation_info = rffi_platform.configure_boehm() + # on some platform GC_init is required before any other + # GC_* functions, call it here for the benefit of tests + # XXX move this to tests + init_fn_ptr = rffi.llexternal("GC_init", + [], lltype.Void, + compilation_info=compilation_info, + sandboxsafe=True, + _nowrapper=True) + init_fn_ptr() + # Versions 6.x of libgc needs to use GC_local_malloc(). # Versions 7.x of libgc removed this function; GC_malloc() has # the same behavior if libgc was compiled with @@ -95,96 +162,42 @@ sandboxsafe=True, _nowrapper=True) cls.malloc_fn_ptr = malloc_fn_ptr - cls.compilation_info = compilation_info return malloc_fn_ptr def __init__(self, gcdescr, translator, rtyper): GcLLDescription.__init__(self, gcdescr, translator, rtyper) # grab a pointer to the Boehm 'malloc' function - malloc_fn_ptr = self.configure_boehm_once() - self.funcptr_for_new = malloc_fn_ptr + self.malloc_fn_ptr = self.configure_boehm_once() + self._setup_str() + self._make_functions() - def malloc_array(basesize, itemsize, ofs_length, num_elem): + def _make_functions(self): + + def malloc_fixedsize(size): + return self.malloc_fn_ptr(size) + self.generate_function('malloc_fixedsize', malloc_fixedsize, + [lltype.Signed]) + + def malloc_array(basesize, num_elem, itemsize, ofs_length): try: - size = ovfcheck(basesize + ovfcheck(itemsize * num_elem)) + totalsize = ovfcheck(basesize + ovfcheck(itemsize * num_elem)) except OverflowError: return lltype.nullptr(llmemory.GCREF.TO) - res = self.funcptr_for_new(size) - if not res: - return res - rffi.cast(rffi.CArrayPtr(lltype.Signed), res)[ofs_length/WORD] = num_elem + res = self.malloc_fn_ptr(totalsize) + if res: + arrayptr = rffi.cast(rffi.CArrayPtr(lltype.Signed), res) + arrayptr[ofs_length/WORD] = num_elem return res - self.malloc_array = malloc_array - self.GC_MALLOC_ARRAY = lltype.Ptr(lltype.FuncType( - [lltype.Signed] * 4, llmemory.GCREF)) + self.generate_function('malloc_array', malloc_array, + [lltype.Signed] * 4) + def _bh_malloc(self, sizedescr): + return self.malloc_fixedsize(sizedescr.size) - (str_basesize, str_itemsize, str_ofs_length - ) = symbolic.get_array_token(rstr.STR, self.translate_support_code) - (unicode_basesize, unicode_itemsize, unicode_ofs_length - ) = symbolic.get_array_token(rstr.UNICODE, self.translate_support_code) - def malloc_str(length): - return self.malloc_array( - str_basesize, str_itemsize, str_ofs_length, length - ) - def malloc_unicode(length): - return self.malloc_array( - unicode_basesize, unicode_itemsize, unicode_ofs_length, length - ) - self.malloc_str = malloc_str - self.malloc_unicode = malloc_unicode - self.GC_MALLOC_STR_UNICODE = lltype.Ptr(lltype.FuncType( - [lltype.Signed], llmemory.GCREF)) - - - # on some platform GC_init is required before any other - # GC_* functions, call it here for the benefit of tests - # XXX move this to tests - init_fn_ptr = rffi.llexternal("GC_init", - [], lltype.Void, - compilation_info=self.compilation_info, - sandboxsafe=True, - _nowrapper=True) - - init_fn_ptr() - - def gc_malloc(self, sizedescr): - assert isinstance(sizedescr, BaseSizeDescr) - return self.funcptr_for_new(sizedescr.size) - - def gc_malloc_array(self, arraydescr, num_elem): - assert isinstance(arraydescr, BaseArrayDescr) - ofs_length = arraydescr.get_ofs_length(self.translate_support_code) - basesize = arraydescr.get_base_size(self.translate_support_code) - itemsize = arraydescr.get_item_size(self.translate_support_code) - return self.malloc_array(basesize, itemsize, ofs_length, num_elem) From noreply at buildbot.pypy.org Wed Jan 25 15:36:20 2012 From: noreply at buildbot.pypy.org (l.diekmann) Date: Wed, 25 Jan 2012 15:36:20 +0100 (CET) Subject: [pypy-commit] pypy type-specialized-instances: (cfbolz, l.diekmann): merge int-tag-untag-as-operationus to get the necessary Message-ID: <20120125143620.7888B829FF@wyvern.cs.uni-duesseldorf.de> Author: Lukas Diekmann <lukas.diekmann at uni-duesseldorf.de> Branch: type-specialized-instances Changeset: r51753:b15e618e3d82 Date: 2012-01-25 15:35 +0100 http://bitbucket.org/pypy/pypy/changeset/b15e618e3d82/ Log: (cfbolz, l.diekmann): merge int-tag-untag-as-operationus to get the necessary optimizations to make this branch useful diff --git a/pypy/jit/backend/llgraph/llimpl.py b/pypy/jit/backend/llgraph/llimpl.py --- a/pypy/jit/backend/llgraph/llimpl.py +++ b/pypy/jit/backend/llgraph/llimpl.py @@ -92,10 +92,13 @@ 'int_is_true' : (('int',), 'bool'), 'int_is_zero' : (('int',), 'bool'), 'int_neg' : (('int',), 'int'), + 'int_tag' : (('int', ), 'int'), + 'int_untag' : (('int', ), 'int'), 'int_invert' : (('int',), 'int'), 'int_add_ovf' : (('int', 'int'), 'int'), 'int_sub_ovf' : (('int', 'int'), 'int'), 'int_mul_ovf' : (('int', 'int'), 'int'), + 'int_tag_ovf' : (('int', ), 'int'), 'uint_add' : (('int', 'int'), 'int'), 'uint_sub' : (('int', 'int'), 'int'), 'uint_mul' : (('int', 'int'), 'int'), @@ -747,6 +750,17 @@ if not flag: raise GuardFailed + def op_int_tag_ovf(self, _, x): + try: + z = ovfcheck(x << 1) + 1 + except OverflowError: + ovf = True + z = 0 + else: + ovf = False + self.overflow_flag = ovf + return z + def op_int_add_ovf(self, _, x, y): try: z = ovfcheck(x + y) diff --git a/pypy/jit/backend/test/test_random.py b/pypy/jit/backend/test/test_random.py --- a/pypy/jit/backend/test/test_random.py +++ b/pypy/jit/backend/test/test_random.py @@ -294,6 +294,9 @@ class BinaryOvfOperation(AbstractOvfOperation, BinaryOperation): pass +class UnaryOvfOperation(AbstractOvfOperation, UnaryOperation): + pass + class AbstractFloatOperation(AbstractOperation): def filter(self, builder): if not builder.cpu.supports_floats: @@ -422,6 +425,8 @@ for _op in [rop.INT_NEG, rop.INT_INVERT, + rop.INT_TAG, + rop.INT_UNTAG, ]: OPERATIONS.append(UnaryOperation(_op)) @@ -434,6 +439,7 @@ rop.INT_MUL_OVF, ]: OPERATIONS.append(BinaryOvfOperation(_op)) +OPERATIONS.append(UnaryOvfOperation(rop.INT_TAG_OVF)) for _op in [rop.FLOAT_ADD, rop.FLOAT_SUB, diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py --- a/pypy/jit/backend/x86/assembler.py +++ b/pypy/jit/backend/x86/assembler.py @@ -1208,6 +1208,18 @@ genop_guard_float_gt = _cmpop_guard_float("A", "B", "BE","AE") genop_guard_float_ge = _cmpop_guard_float("AE","BE", "B", "A") + + def genop_int_tag(self, op, arglocs, resloc): + loc, = arglocs + assert isinstance(loc, RegLoc) + assert isinstance(resloc, RegLoc) + # res = loc + (loc << 0) + 1 + self.mc.LEA_ra(resloc.value, (loc.value, loc.value, 0, 1)) + + def genop_int_untag(self, op, arglocs, resloc): + loc, = arglocs + self.mc.SAR(loc, imm1) + def genop_math_sqrt(self, op, arglocs, resloc): self.mc.SQRTSD(arglocs[0], resloc) @@ -1628,6 +1640,11 @@ self.genop_int_mul(op, arglocs, result_loc) return self._gen_guard_overflow(guard_op, guard_token) + def genop_guard_int_tag_ovf(self, op, guard_op, guard_token, arglocs, result_loc): + self.mc.STC() + self.mc.ADC(arglocs[0], arglocs[0]) + return self._gen_guard_overflow(guard_op, guard_token) + def genop_guard_guard_false(self, ign_1, guard_op, guard_token, locs, ign_2): loc = locs[0] self.mc.TEST(loc, loc) diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py --- a/pypy/jit/backend/x86/regalloc.py +++ b/pypy/jit/backend/x86/regalloc.py @@ -619,11 +619,22 @@ consider_int_sub_ovf = _consider_binop_with_guard consider_int_add_ovf = _consider_binop_with_guard + def consider_int_tag_ovf(self, op, guard_op): + loc = self.rm.force_result_in_reg(op.result, op.getarg(0)) + self.perform_with_guard(op, guard_op, [loc], loc) + + def consider_int_tag(self, op): + loc = self.rm.make_sure_var_in_reg(op.getarg(0)) + self.rm.possibly_free_vars_for_op(op) + res = self.rm.force_allocate_reg(op.result) + self.Perform(op, [loc], res) + def consider_int_neg(self, op): res = self.rm.force_result_in_reg(op.result, op.getarg(0)) self.Perform(op, [res], res) consider_int_invert = consider_int_neg + consider_int_untag = consider_int_neg def consider_int_lshift(self, op): if isinstance(op.getarg(1), Const): diff --git a/pypy/jit/backend/x86/regloc.py b/pypy/jit/backend/x86/regloc.py --- a/pypy/jit/backend/x86/regloc.py +++ b/pypy/jit/backend/x86/regloc.py @@ -530,6 +530,7 @@ BTS = _binaryop('BTS') ADD = _binaryop('ADD') + ADC = _binaryop('ADC') SUB = _binaryop('SUB') IMUL = _binaryop('IMUL') NEG = _unaryop('NEG') diff --git a/pypy/jit/backend/x86/rx86.py b/pypy/jit/backend/x86/rx86.py --- a/pypy/jit/backend/x86/rx86.py +++ b/pypy/jit/backend/x86/rx86.py @@ -470,6 +470,7 @@ # ------------------------------ Arithmetic ------------------------------ ADD_ri,ADD_rr,ADD_rb,_,_,ADD_rm,ADD_rj,_,_ = common_modes(0) + ADC_ri,ADC_rr,ADC_rb,_,_,ADC_rm,ADC_rj,_,_ = common_modes(2) OR_ri, OR_rr, OR_rb, _,_,OR_rm, OR_rj, _,_ = common_modes(1) AND_ri,AND_rr,AND_rb,_,_,AND_rm,AND_rj,_,_ = common_modes(4) SUB_ri,SUB_rr,SUB_rb,_,_,SUB_rm,SUB_rj,SUB_ji8,SUB_mi8 = common_modes(5) @@ -577,6 +578,8 @@ FSTPL_b = insn('\xDD', orbyte(3<<3), stack_bp(1)) # rffi.DOUBLE ('as' wants L??) FSTPS_s = insn('\xD9', orbyte(3<<3), stack_sp(1)) # lltype.SingleFloat + STC = insn('\xF9') + # ------------------------------ Random mess ----------------------- RDTSC = insn('\x0F\x31') diff --git a/pypy/jit/backend/x86/test/test_regalloc.py b/pypy/jit/backend/x86/test/test_regalloc.py --- a/pypy/jit/backend/x86/test/test_regalloc.py +++ b/pypy/jit/backend/x86/test/test_regalloc.py @@ -692,3 +692,27 @@ self.run(loop, 4, 7) assert self.getint(0) == 29 + def test_int_untag(self): + ops = ''' + [i0] + i1 = int_untag(i0) + finish(i1) + ''' + self.interpret(ops, [1129]) + assert self.getint(0) == 564 + self.interpret(ops, [-1129]) + assert self.getint(0) == -565 + + def test_int_tag(self): + ops = ''' + [i0] + i1 = int_tag(i0) + i2 = int_tag(i0) + finish(i1, i2) + ''' + self.interpret(ops, [1129]) + assert self.getint(0) == 1129 * 2 + 1 + assert self.getint(1) == 1129 * 2 + 1 + self.interpret(ops, [-1129]) + assert self.getint(0) == -1129 * 2 + 1 + assert self.getint(1) == -1129 * 2 + 1 diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py --- a/pypy/jit/codewriter/jtransform.py +++ b/pypy/jit/codewriter/jtransform.py @@ -290,6 +290,10 @@ op1 = SpaceOperation('-live-', [], None) return [op, op1] + def rewrite_op_int_tag_ovf(self, op): + op1 = SpaceOperation('-live-', [], None) + return [op, op1] + # ---------- # Various kinds of calls diff --git a/pypy/jit/metainterp/blackhole.py b/pypy/jit/metainterp/blackhole.py --- a/pypy/jit/metainterp/blackhole.py +++ b/pypy/jit/metainterp/blackhole.py @@ -446,6 +446,18 @@ def bhimpl_int_invert(a): return intmask(~a) + @arguments("i", returns="i") + def bhimpl_int_untag(a): + ll_assert((a & 1) == 1, "bhimpl_int_untag: not an odd int") + return a >> 1 + @arguments("i", returns="i") + def bhimpl_int_tag(a): + return intmask(a << 1) + 1 # mostly there for test_random + @arguments("i", returns="i") + def bhimpl_int_tag_ovf(a): + return ovfcheck(a << 1) + 1 + + @arguments("i", "i", returns="i") def bhimpl_int_lt(a, b): return a < b @@ -508,7 +520,6 @@ @arguments("r", returns="i") def bhimpl_cast_ptr_to_int(a): i = lltype.cast_ptr_to_int(a) - ll_assert((i & 1) == 1, "bhimpl_cast_ptr_to_int: not an odd int") return i @arguments("i", returns="r") def bhimpl_cast_int_to_ptr(i): diff --git a/pypy/jit/metainterp/executor.py b/pypy/jit/metainterp/executor.py --- a/pypy/jit/metainterp/executor.py +++ b/pypy/jit/metainterp/executor.py @@ -224,6 +224,18 @@ z = 0 return BoxInt(z) +def do_int_tag_ovf(cpu, metainterp, box1): + # the overflow operations can be called without a metainterp, if an + # overflow cannot occur + a = box1.getint() + try: + z = ovfcheck(a << 1) + except OverflowError: + assert metainterp is not None + metainterp.execute_raised(OverflowError(), constant=True) + z = 0 + return BoxInt(z + 1) + def do_same_as(cpu, _, box): return box.clonebox() diff --git a/pypy/jit/metainterp/optimizeopt/intbounds.py b/pypy/jit/metainterp/optimizeopt/intbounds.py --- a/pypy/jit/metainterp/optimizeopt/intbounds.py +++ b/pypy/jit/metainterp/optimizeopt/intbounds.py @@ -62,6 +62,14 @@ def optimize_INT_AND(self, op): v1 = self.getvalue(op.getarg(0)) v2 = self.getvalue(op.getarg(1)) + if (self.optimizer.metainterp_sd.config.translation.taggedpointers and + v2.is_constant() and v2.box.getint() == 1): + if self.has_pure_result(rop.INT_UNTAG, [v1.box], None): + # the result of untagging the int is known, so the box must be + # tagged, so int_and(x, 1) == 1 + value = self.getvalue(ConstInt(1)) + self.optimizer.make_equal_to(op.result, value) + return self.emit_operation(op) r = self.getvalue(op.result) @@ -163,18 +171,16 @@ def optimize_GUARD_NO_OVERFLOW(self, op): lastop = self.last_emitted_operation if lastop is not None: + # If the INT_xxx_OVF was replaced with INT_xxx, then we can kill + # the GUARD_NO_OVERFLOW. + if not lastop.is_ovf(): + return opnum = lastop.getopnum() args = lastop.getarglist() result = lastop.result - # If the INT_xxx_OVF was replaced with INT_xxx, then we can kill - # the GUARD_NO_OVERFLOW. - if (opnum == rop.INT_ADD or - opnum == rop.INT_SUB or - opnum == rop.INT_MUL): - return # Else, synthesize the non overflowing op for optimize_default to - # reuse, as well as the reverse op - elif opnum == rop.INT_ADD_OVF: + # reuse, as well as the reverse ops + if opnum == rop.INT_ADD_OVF: self.pure(rop.INT_ADD, args[:], result) self.pure(rop.INT_SUB, [result, args[1]], args[0]) self.pure(rop.INT_SUB, [result, args[0]], args[1]) @@ -184,6 +190,11 @@ self.pure(rop.INT_SUB, [args[0], result], args[1]) elif opnum == rop.INT_MUL_OVF: self.pure(rop.INT_MUL, args[:], result) + elif opnum == rop.INT_TAG_OVF: + v1 = self.getvalue(lastop.getarg(0)) + maxbounds = IntBound((-sys.maxint-1) >> 1, sys.maxint >> 1) + v1.intbound.intersect(maxbounds) + self.pure(rop.INT_UNTAG, [result], args[0]) self.emit_operation(op) def optimize_GUARD_OVERFLOW(self, op): @@ -193,7 +204,7 @@ if lastop is None: raise InvalidLoop opnum = lastop.getopnum() - if opnum not in (rop.INT_ADD_OVF, rop.INT_SUB_OVF, rop.INT_MUL_OVF): + if not lastop.is_ovf(): raise InvalidLoop self.emit_operation(op) @@ -294,6 +305,32 @@ else: self.emit_operation(op) + def optimize_INT_TAG_OVF(self, op): + v1 = self.getvalue(op.getarg(0)) + resbound = v1.intbound.mul(2).add(1) + if resbound.bounded(): + op = op.copy_and_change(rop.INT_TAG) + self.optimize_INT_TAG(op) # emit the op + else: + self.emit_operation(op) + + def optimize_INT_TAG(self, op): + v1 = self.getvalue(op.getarg(0)) + self.emit_operation(op) + r = self.getvalue(op.result) + resbound = v1.intbound.mul(2).add(1) + r.intbound.intersect(resbound) + maxbounds = IntBound((-sys.maxint-1) >> 1, sys.maxint >> 1) + v1.intbound.intersect(maxbounds) + self.pure(rop.INT_UNTAG, [op.result], op.getarg(0)) + + def optimize_INT_UNTAG(self, op): + v1 = self.getvalue(op.getarg(0)) + self.pure(rop.INT_TAG, [op.result], op.getarg(0)) + self.emit_operation(op) + r = self.getvalue(op.result) + r.intbound.intersect(v1.intbound.rshift_bound(IntBound(1, 1))) + def optimize_ARRAYLEN_GC(self, op): self.emit_operation(op) array = self.getvalue(op.getarg(0)) diff --git a/pypy/jit/metainterp/optimizeopt/intutils.py b/pypy/jit/metainterp/optimizeopt/intutils.py --- a/pypy/jit/metainterp/optimizeopt/intutils.py +++ b/pypy/jit/metainterp/optimizeopt/intutils.py @@ -101,8 +101,32 @@ return res def mul(self, value): - return self.mul_bound(IntBound(value, value)) - + upper = 0 + has_upper = False + if self.has_upper: + try: + upper = ovfcheck(self.upper * value) + except OverflowError: + pass + else: + has_upper = True + lower = 0 + has_lower = False + if self.has_lower: + try: + lower = ovfcheck(self.lower * value) + except OverflowError: + pass + else: + has_lower = True + if value < 0: + has_upper, has_lower = has_lower, has_upper + upper, lower = lower, upper + result = IntBound(lower, upper) + result.has_lower = has_lower + result.has_upper = has_upper + return result + def add_bound(self, other): res = self.clone() if other.has_upper: diff --git a/pypy/jit/metainterp/optimizeopt/test/test_intutils.py b/pypy/jit/metainterp/optimizeopt/test/test_intutils.py new file mode 100644 --- /dev/null +++ b/pypy/jit/metainterp/optimizeopt/test/test_intutils.py @@ -0,0 +1,29 @@ +from pypy.jit.metainterp.optimizeopt import intutils + +# XXX this file should really be filled with tests for all operations! + +def test_mul_with_constant(): + x = intutils.IntBound(0, 100) + y = x.mul(2) + assert y.has_lower + assert y.has_upper + assert y.lower == 0 + assert y.upper == 200 + + y = x.mul(-5) + assert y.has_lower + assert y.has_upper + assert y.lower == -500 + assert y.upper == 0 + + x = intutils.IntUpperBound(100) + y = x.mul(2) + assert not y.has_lower + assert y.has_upper + assert y.upper == 200 + + y = x.mul(-5) + assert y.has_lower + assert not y.has_upper + assert y.lower == -500 + assert y.upper == 0 diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py --- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py +++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py @@ -232,7 +232,10 @@ args = [] for argtype in argtypes: assert argtype in ('int', 'bool') - args.append(random.randrange(1, 20)) + arg = random.randrange(1, 20) + if opnum == rop.INT_UNTAG: + arg = arg | 1 # must be an odd int + args.append(arg) assert restype in ('int', 'bool') ops = """ [] diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -205,7 +205,10 @@ args = [] for argtype in argtypes: assert argtype in ('int', 'bool') - args.append(random.randrange(1, 20)) + arg = random.randrange(1, 20) + if opnum == rop.INT_UNTAG: + arg = arg | 1 # must be an odd int + args.append(arg) assert restype in ('int', 'bool') ops = """ [] @@ -5217,6 +5220,137 @@ """ self.optimize_loop(ops, ops, ops) + def test_int_tag_untag_reverses(self): + ops = """ + [i0] + i1 = int_tag_ovf(i0) + guard_no_overflow() [] + i2 = int_untag(i1) + i3 = int_add(i2, 1) + jump(i3) + """ + expected = """ + [i0] + i1 = int_tag_ovf(i0) + guard_no_overflow() [] + i2 = int_add(i0, 1) + jump(i2) + """ + self.optimize_loop(ops, expected) + ops = """ + [i0] + i1 = int_untag(i0) + i2 = int_tag_ovf(i1) + guard_no_overflow() [] + i3 = int_untag(i2) + i4 = int_add(i3, 1) + jump(i4) + """ + expected = """ + [i0] + i1 = int_untag(i0) + i2 = int_add(i1, 1) + jump(i2) + """ + self.optimize_loop(ops, expected) + + def test_int_tag_remove_overflow_checking(self): + ops = """ + [i0] + i1 = int_lt(i0, 1000) + guard_true(i1), [] + i2 = int_tag_ovf(i0) + guard_no_overflow() [] + i3 = int_untag(i2) + i4 = int_add_ovf(i3, 1) + guard_no_overflow() [] + jump(i4) + """ + expected = """ + [i0] + i1 = int_lt(i0, 1000) + guard_true(i1), [] + i2 = int_tag(i0) + i3 = int_add(i0, 1) + jump(i3) + """ + preamble = """ + [i0] + i1 = int_lt(i0, 1000) + guard_true(i1), [] + i2 = int_tag_ovf(i0) + guard_no_overflow() [] + i3 = int_add(i0, 1) + jump(i3) + """ + self.optimize_loop(ops, expected, preamble) + + def test_int_tag_remove_overflow_checking2(self): + ops = """ + [i0] + i1 = int_lt(i0, 1000) + guard_true(i1), [] + i2 = int_gt(i0, 0) + guard_true(i2), [] + i3 = int_tag_ovf(i0) + guard_no_overflow() [] + i4 = escape(i3) + jump(i4) + """ + expected = """ + [i0] + i1 = int_lt(i0, 1000) + guard_true(i1), [] + i2 = int_gt(i0, 0) + guard_true(i2), [] + i3 = int_tag(i0) + i4 = escape(i3) + jump(i4) + """ + self.optimize_loop(ops, expected) + + def test_int_tag_removes_int_and(self): + ops = """ + [i0] + i1 = int_tag_ovf(i0) + guard_no_overflow() [] + i2 = int_and(i1, 1) + i3 = int_is_true(i2) + guard_true(i3) [] + i4 = int_untag(i1) + i5 = int_add(i4, 1) + jump(i5) + """ + expected = """ + [i0] + i1 = int_tag_ovf(i0) + guard_no_overflow() [] + i2 = int_add(i0, 1) + jump(i2) + """ + self.optimize_loop(ops, expected) + + def test_int_tag_constfold(self): + ops = """ + [i0] + i1 = int_tag_ovf(1) + guard_no_overflow() [] + i4 = int_untag(i1) + i5 = int_add(i4, 1) + escape(i5) + jump(i5) + """ + expected = """ + [] + escape(2) + jump() + """ + preamble = """ + [i0] + escape(2) + jump() + """ + self.optimize_loop(ops, expected, preamble) def test_mul_ovf(self): ops = """ diff --git a/pypy/jit/metainterp/optimizeopt/test/test_util.py b/pypy/jit/metainterp/optimizeopt/test/test_util.py --- a/pypy/jit/metainterp/optimizeopt/test/test_util.py +++ b/pypy/jit/metainterp/optimizeopt/test/test_util.py @@ -346,6 +346,7 @@ self.globaldata = Fake() self.config = get_pypy_config(translating=True) self.config.translation.jit_ffi = True + self.config.translation.taggedpointers = True class logger_noopt: @classmethod diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py --- a/pypy/jit/metainterp/pyjitpl.py +++ b/pypy/jit/metainterp/pyjitpl.py @@ -218,7 +218,7 @@ return resbox ''' % (_opimpl, _opimpl.upper())).compile() - for _opimpl in ['int_is_true', 'int_is_zero', 'int_neg', 'int_invert', + for _opimpl in ['int_is_true', 'int_is_zero', 'int_neg', 'int_invert', 'int_untag', 'cast_float_to_int', 'cast_int_to_float', 'cast_float_to_singlefloat', 'cast_singlefloat_to_float', 'float_neg', 'float_abs', @@ -231,6 +231,15 @@ ''' % (_opimpl, _opimpl.upper())).compile() @arguments("box") + def opimpl_int_tag_ovf(self, b1): + self.metainterp.clear_exception() + resbox = self.execute(rop.INT_TAG_OVF, b1) + self.make_result_of_lastop(resbox) + if not isinstance(resbox, Const): + self.metainterp.handle_possible_overflow_error() + return resbox + + @arguments("box") def opimpl_ptr_nonzero(self, box): return self.execute(rop.PTR_NE, box, history.CONST_NULL) diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py --- a/pypy/jit/metainterp/resoperation.py +++ b/pypy/jit/metainterp/resoperation.py @@ -441,6 +441,8 @@ 'INT_IS_TRUE/1b', 'INT_NEG/1', 'INT_INVERT/1', + 'INT_UNTAG/1', + 'INT_TAG/1', # 'SAME_AS/1', # gets a Const or a Box, turns it into another Box 'CAST_PTR_TO_INT/1', @@ -523,6 +525,7 @@ 'INT_ADD_OVF/2', 'INT_SUB_OVF/2', 'INT_MUL_OVF/2', + 'INT_TAG_OVF/1', '_OVF_LAST', # ----- end of is_ovf operations ----- '_LAST', # for the backend to add more internal operations ] diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py --- a/pypy/jit/metainterp/test/test_ajit.py +++ b/pypy/jit/metainterp/test/test_ajit.py @@ -3569,7 +3569,6 @@ translationoptions = {'withsmallfuncsets': 3} self.interp_operations(f, [5], translationoptions=translationoptions) - def test_annotation_gives_class_knowledge_to_tracer(self): py.test.skip("disabled") class Base(object): @@ -3696,7 +3695,6 @@ self.check_operations_history(guard_class=0, record_known_class=1) -class TestLLtype(BaseLLtypeTests, LLJitMixin): def test_tagged(self): from pypy.rlib.objectmodel import UnboxedValue class Base(object): @@ -3821,3 +3819,6 @@ return 42 self.interp_operations(f, [1, 2, 3]) self.check_operations_history(call=1, guard_no_exception=0) + +class TestLLtype(BaseLLtypeTests, LLJitMixin): + pass # should be empty diff --git a/pypy/jit/metainterp/test/test_immutable.py b/pypy/jit/metainterp/test/test_immutable.py --- a/pypy/jit/metainterp/test/test_immutable.py +++ b/pypy/jit/metainterp/test/test_immutable.py @@ -64,7 +64,7 @@ l[2] = 30 a = escape(X(l)) return a.y[index] - res = self.interp_operations(f, [2], listops=True) + res = self.interp_operations(f, [2]) assert res == 30 self.check_operations_history(getfield_gc=0, getfield_gc_pure=1, getarrayitem_gc=0, getarrayitem_gc_pure=1) @@ -82,7 +82,7 @@ def f(x, index): y = escape(X([x], x+1)) return y.lst[index] + y.y + 5 - res = self.interp_operations(f, [23, 0], listops=True) + res = self.interp_operations(f, [23, 0]) assert res == 23 + 24 + 5 self.check_operations_history(getfield_gc=0, getfield_gc_pure=2, getarrayitem_gc=0, getarrayitem_gc_pure=1, diff --git a/pypy/jit/metainterp/test/test_list.py b/pypy/jit/metainterp/test/test_list.py --- a/pypy/jit/metainterp/test/test_list.py +++ b/pypy/jit/metainterp/test/test_list.py @@ -124,7 +124,7 @@ l2 = l[:] return l2[0] + l2[1] + l2[2] + l2[3] - res = self.interp_operations(f, [], listops=True) + res = self.interp_operations(f, []) assert res == 10 def test_arraycopy_full(self): diff --git a/pypy/jit/metainterp/test/test_slist.py b/pypy/jit/metainterp/test/test_slist.py --- a/pypy/jit/metainterp/test/test_slist.py +++ b/pypy/jit/metainterp/test/test_slist.py @@ -31,7 +31,7 @@ lst2.append(FooBar(5)) m += lst2.pop().z # 49 return m - res = self.interp_operations(f, [11], listops=True) + res = self.interp_operations(f, [11]) assert res == 49 self.check_operations_history(call=3) diff --git a/pypy/rlib/rerased.py b/pypy/rlib/rerased.py --- a/pypy/rlib/rerased.py +++ b/pypy/rlib/rerased.py @@ -160,7 +160,7 @@ from pypy.rlib.debug import ll_assert x = llop.cast_ptr_to_int(lltype.Signed, gcref) ll_assert((x&1) != 0, "unerased_int(): not an integer") - return x >> 1 + return llop.int_untag(lltype.Signed, x) class Entry(ExtRegistryEntry): @@ -220,11 +220,9 @@ [v_value] = hop.inputargs(lltype.Signed) c_one = hop.inputconst(lltype.Signed, 1) hop.exception_is_here() - v2 = hop.genop('int_add_ovf', [v_value, v_value], + v2 = hop.genop('int_tag_ovf', [v_value], resulttype = lltype.Signed) - v2p1 = hop.genop('int_add', [v2, c_one], - resulttype = lltype.Signed) - v_instance = hop.genop('cast_int_to_ptr', [v2p1], + v_instance = hop.genop('cast_int_to_ptr', [v2], resulttype=self.lowleveltype) return v_instance diff --git a/pypy/rpython/llinterp.py b/pypy/rpython/llinterp.py --- a/pypy/rpython/llinterp.py +++ b/pypy/rpython/llinterp.py @@ -1101,6 +1101,16 @@ assert y >= 0 return self.op_int_add_ovf(x, y) + def op_int_tag_ovf(self, x): + try: + return ovfcheck(x + x + 1) + except OverflowError: + self.make_llexception() + + def op_int_untag(self, x): + assert x & 1, "argument has to be tagged!" + return x >> 1 + def op_int_is_true(self, x): # special case if type(x) is CDefinedIntSymbolic: diff --git a/pypy/rpython/lltypesystem/lloperation.py b/pypy/rpython/lltypesystem/lloperation.py --- a/pypy/rpython/lltypesystem/lloperation.py +++ b/pypy/rpython/lltypesystem/lloperation.py @@ -207,6 +207,8 @@ 'int_abs': LLOp(canfold=True), 'int_abs_ovf': LLOp(canraise=(OverflowError,), tryfold=True), 'int_invert': LLOp(canfold=True), + 'int_tag_ovf': LLOp(canraise=(OverflowError, ), tryfold=True), + 'int_untag': LLOp(canfold=True), 'int_add': LLOp(canfold=True), 'int_sub': LLOp(canfold=True), diff --git a/pypy/rpython/lltypesystem/rtagged.py b/pypy/rpython/lltypesystem/rtagged.py --- a/pypy/rpython/lltypesystem/rtagged.py +++ b/pypy/rpython/lltypesystem/rtagged.py @@ -4,6 +4,7 @@ from pypy.rpython.lltypesystem.rclass import InstanceRepr, CLASSTYPE, ll_inst_type from pypy.rpython.lltypesystem.rclass import MissingRTypeAttribute from pypy.rpython.lltypesystem.rclass import ll_issubclass_const +from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.rmodel import TyperError, inputconst @@ -43,24 +44,22 @@ v_value = hop.inputarg(lltype.Signed, arg=1) c_one = hop.inputconst(lltype.Signed, 1) hop.exception_is_here() - v2 = hop.genop('int_add_ovf', [v_value, v_value], + v2 = hop.genop('int_tag_ovf', [v_value], resulttype = lltype.Signed) - v2p1 = hop.genop('int_add', [v2, c_one], - resulttype = lltype.Signed) - v_instance = hop.genop('cast_int_to_ptr', [v2p1], + v_instance = hop.genop('cast_int_to_ptr', [v2], resulttype = self.lowleveltype) return v_instance, False # don't call __init__ def convert_const_exact(self, value): self.setup() number = value.get_untagged_value() - return ll_int_to_unboxed(self.lowleveltype, number) + tagged = number * 2 + 1 + return lltype.cast_int_to_ptr(self.lowleveltype, tagged) def getvalue_from_unboxed(self, llops, vinst): assert not self.is_parent v2 = llops.genop('cast_ptr_to_int', [vinst], resulttype=lltype.Signed) - c_one = inputconst(lltype.Signed, 1) - return llops.genop('int_rshift', [v2, c_one], resulttype=lltype.Signed) + return llops.genop('int_untag', [v2], resulttype=lltype.Signed) def gettype_from_unboxed(self, llops, vinst, can_be_none=False): unboxedclass_repr = getclassrepr(self.rtyper, self.unboxedclassdef) @@ -142,11 +141,9 @@ minid, maxid, c_answer_if_unboxed) -def ll_int_to_unboxed(PTRTYPE, value): - return lltype.cast_int_to_ptr(PTRTYPE, value*2+1) def ll_unboxed_to_int(p): - return lltype.cast_ptr_to_int(p) >> 1 + return llop.int_untag(lltype.Signed, lltype.cast_ptr_to_int(p)) def ll_unboxed_getclass_canbenone(instance, class_if_unboxed): if instance: diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py --- a/pypy/rpython/memory/gc/minimark.py +++ b/pypy/rpython/memory/gc/minimark.py @@ -756,6 +756,8 @@ def can_move(self, obj): """Overrides the parent can_move().""" + if not self.is_valid_gc_object(obj): + return False return self.is_in_nursery(obj) diff --git a/pypy/translator/c/src/int.h b/pypy/translator/c/src/int.h --- a/pypy/translator/c/src/int.h +++ b/pypy/translator/c/src/int.h @@ -19,6 +19,13 @@ if ((x) == LONG_MIN) FAIL_OVF("integer absolute"); \ OP_INT_ABS(x,r) +#define OP_INT_TAG_OVF(x, r) \ + r = (long)((unsigned long)x << 1); \ + if ((r ^ x) < 0) FAIL_OVF("integer tagging"); \ + r = r + 1 +#define OP_INT_UNTAG(x, r) \ + r = x >> 1 + /*** binary operations ***/ #define OP_INT_EQ(x,y,r) r = ((x) == (y)) From noreply at buildbot.pypy.org Wed Jan 25 15:46:04 2012 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 25 Jan 2012 15:46:04 +0100 (CET) Subject: [pypy-commit] pypy numpy-back-to-applevel: progress on take Message-ID: <20120125144604.B6F7A829FF@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: numpy-back-to-applevel Changeset: r51754:96b51eda2096 Date: 2012-01-25 16:45 +0200 http://bitbucket.org/pypy/pypy/changeset/96b51eda2096/ Log: progress on take diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py --- a/pypy/module/micronumpy/interp_boxes.py +++ b/pypy/module/micronumpy/interp_boxes.py @@ -6,7 +6,7 @@ from pypy.objspace.std.inttype import int_typedef from pypy.rlib.rarithmetic import LONG_BIT from pypy.tool.sourcetools import func_with_new_name - +from pypy.rpython.lltypesystem import rffi, lltype MIXIN_64 = (int_typedef,) if LONG_BIT == 64 else () @@ -104,7 +104,8 @@ _attrs_ = () class W_IntegerBox(W_NumberBox): - pass + def int_w(self, space): + return rffi.cast(lltype.Signed, self.value) class W_SignedIntegerBox(W_IntegerBox): pass diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -613,6 +613,27 @@ def supports_fast_slicing(self): return False + def descr_take(self, space, w_obj): + index = convert_to_array(space, w_obj).get_concrete() + if len(self.shape) > 1: + xxx + index_i = index.create_iter() + res_shape = index.shape + size = 1 + for elem in res_shape: + size *= elem + res = W_NDimArray(size, res_shape[:], self.dtype, self.order) + res_i = res.create_iter() + longdtype = interp_dtype.get_dtype_cache(space).w_longdtype + while not index_i.done(): + w_item = index.getitem(index_i.offset).convert_to(longdtype) + import pdb + pdb.set_trace() + res.setitem(res_i.offset, self.descr_getitem(space, w_item)) + index_i = index_i.next() + res_i = res_i.next() + return res + def convert_to_array(space, w_obj): if isinstance(w_obj, BaseArray): return w_obj @@ -1313,6 +1334,7 @@ flatten = interp2app(BaseArray.descr_flatten), reshape = interp2app(BaseArray.descr_reshape), tolist = interp2app(BaseArray.descr_tolist), + take = interp2app(BaseArray.descr_take), ) diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py --- a/pypy/module/micronumpy/test/test_dtypes.py +++ b/pypy/module/micronumpy/test/test_dtypes.py @@ -376,3 +376,7 @@ b = X(10) assert type(b) is X assert b.m() == 12 + + def test_long_as_index(self): + from _numpypy import int_ + assert (1, 2, 3)[int_(1)] == 2 diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -1401,7 +1401,10 @@ assert (arange(6).reshape(2, 3).ravel() == arange(6)).all() print arange(6).reshape(2, 3).T.ravel() assert (arange(6).reshape(2, 3).T.ravel() == [0, 3, 1, 4, 2, 5]).all() - + + def test_take(self): + from _numpypy import arange + assert(arange(10).take([1, 2, 1, 1]) == [1, 2, 1, 1]) class AppTestSupport(BaseNumpyAppTest): def setup_class(cls): From noreply at buildbot.pypy.org Wed Jan 25 15:50:12 2012 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 25 Jan 2012 15:50:12 +0100 (CET) Subject: [pypy-commit] pypy kill-more-multimethods: a branch to perform some experiments on killing some multimethods. a performance experiment (Completely invalid) Message-ID: <20120125145012.ECB12829FF@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: kill-more-multimethods Changeset: r51755:ef9ea1a64a8e Date: 2012-01-25 16:49 +0200 http://bitbucket.org/pypy/pypy/changeset/ef9ea1a64a8e/ Log: a branch to perform some experiments on killing some multimethods. a performance experiment (Completely invalid) diff --git a/pypy/objspace/std/tupleobject.py b/pypy/objspace/std/tupleobject.py --- a/pypy/objspace/std/tupleobject.py +++ b/pypy/objspace/std/tupleobject.py @@ -54,11 +54,12 @@ def getitem__Tuple_ANY(space, w_tuple, w_index): # getindex_w should get a second argument space.w_IndexError, # but that doesn't exist the first time this is called. - try: - w_IndexError = space.w_IndexError - except AttributeError: - w_IndexError = None - index = space.getindex_w(w_index, w_IndexError, "tuple index") + #try: + # w_IndexError = space.w_IndexError + #except AttributeError: + # w_IndexError = None + #index = space.getindex_w(w_index, w_IndexError, "tuple index") + index = space.int_w(w_index) try: return w_tuple.wrappeditems[index] except IndexError: From noreply at buildbot.pypy.org Wed Jan 25 17:12:13 2012 From: noreply at buildbot.pypy.org (antocuni) Date: Wed, 25 Jan 2012 17:12:13 +0100 (CET) Subject: [pypy-commit] pypy py3k: fix the test, which was broken by the implementation of nonlocal in d6d9ea7ba306 Message-ID: <20120125161213.3E7AD829FF@wyvern.cs.uni-duesseldorf.de> Author: Antonio Cuni <anto.cuni at gmail.com> Branch: py3k Changeset: r51756:b83de607e7cc Date: 2012-01-25 10:13 +0100 http://bitbucket.org/pypy/pypy/changeset/b83de607e7cc/ Log: fix the test, which was broken by the implementation of nonlocal in d6d9ea7ba306 diff --git a/pypy/interpreter/astcompiler/test/test_symtable.py b/pypy/interpreter/astcompiler/test/test_symtable.py --- a/pypy/interpreter/astcompiler/test/test_symtable.py +++ b/pypy/interpreter/astcompiler/test/test_symtable.py @@ -272,7 +272,7 @@ assert xscp.lookup("y") == symtable.SCOPE_GLOBAL_EXPLICIT assert zscp.lookup("y") == symtable.SCOPE_FREE exc = py.test.raises(SyntaxError, self.func_scope, input).value - assert exc.msg == "name 'x' is local and global" + assert exc.msg == "name 'x' is parameter and global" def test_optimization(self): assert not self.mod_scope("").can_be_optimized From noreply at buildbot.pypy.org Wed Jan 25 17:12:14 2012 From: noreply at buildbot.pypy.org (antocuni) Date: Wed, 25 Jan 2012 17:12:14 +0100 (CET) Subject: [pypy-commit] pypy py3k: add more tests about the nonlocal declaration introduced by d6d9ea7ba306 Message-ID: <20120125161214.7B2CA82CFC@wyvern.cs.uni-duesseldorf.de> Author: Antonio Cuni <anto.cuni at gmail.com> Branch: py3k Changeset: r51757:4df3b47d0b13 Date: 2012-01-25 10:17 +0100 http://bitbucket.org/pypy/pypy/changeset/4df3b47d0b13/ Log: add more tests about the nonlocal declaration introduced by d6d9ea7ba306 diff --git a/pypy/interpreter/astcompiler/test/test_symtable.py b/pypy/interpreter/astcompiler/test/test_symtable.py --- a/pypy/interpreter/astcompiler/test/test_symtable.py +++ b/pypy/interpreter/astcompiler/test/test_symtable.py @@ -274,6 +274,34 @@ exc = py.test.raises(SyntaxError, self.func_scope, input).value assert exc.msg == "name 'x' is parameter and global" + def test_nonlocal(self): + src = str(py.code.Source(""" + def f(): + nonlocal x + global x + """)) + exc = py.test.raises(SyntaxError, self.func_scope, src).value + assert exc.msg == "name 'x' is nonlocal and global" + # + src = str(py.code.Source(""" + def f(x): + nonlocal x + """)) + exc = py.test.raises(SyntaxError, self.func_scope, src).value + assert exc.msg == "name 'x' is parameter and nonlocal" + # + src = str(py.code.Source(""" + def f(): + nonlocal x + """)) + exc = py.test.raises(SyntaxError, self.func_scope, src).value + assert exc.msg == "no binding for nonlocal 'x' found" + # + src = "nonlocal x" + exc = py.test.raises(SyntaxError, self.func_scope, src).value + assert exc.msg == "nonlocal declaration not allowed at module level" + + def test_optimization(self): assert not self.mod_scope("").can_be_optimized assert not self.class_scope("class x: pass").can_be_optimized From noreply at buildbot.pypy.org Wed Jan 25 17:12:19 2012 From: noreply at buildbot.pypy.org (antocuni) Date: Wed, 25 Jan 2012 17:12:19 +0100 (CET) Subject: [pypy-commit] pypy py3k: fix these tests which were broken by the previous commit (because now the docstring tests are unselected) Message-ID: <20120125161219.6298F82D01@wyvern.cs.uni-duesseldorf.de> Author: Antonio Cuni <anto.cuni at gmail.com> Branch: py3k Changeset: r51761:443723ebd5e0 Date: 2012-01-25 16:45 +0100 http://bitbucket.org/pypy/pypy/changeset/443723ebd5e0/ Log: fix these tests which were broken by the previous commit (because now the docstring tests are unselected) diff --git a/pypy/tool/pytest/test/test_conftest1.py b/pypy/tool/pytest/test/test_conftest1.py --- a/pypy/tool/pytest/test/test_conftest1.py +++ b/pypy/tool/pytest/test/test_conftest1.py @@ -27,7 +27,7 @@ sorter = testdir.inline_run(innertest, '-k', 'applevel -docstring', '--runappdirect') passed, skipped, failed = sorter.listoutcomes() - assert len(passed) == 3 + assert len(passed) == 2 print passed assert "app_test_something" in passed[0].nodeid assert "test_method_app" in passed[1].nodeid @@ -36,7 +36,7 @@ sorter = testdir.inline_run(innertest, '-k', 'applevel -docstring', '--appdirect=%s' % (sys.executable,)) passed, skipped, failed = sorter.listoutcomes() - assert len(passed) == 3 + assert len(passed) == 2 print passed assert "app_test_something" in passed[0].nodeid assert "test_method_app" in passed[1].nodeid From noreply at buildbot.pypy.org Wed Jan 25 17:12:20 2012 From: noreply at buildbot.pypy.org (antocuni) Date: Wed, 25 Jan 2012 17:12:20 +0100 (CET) Subject: [pypy-commit] pypy py3k: support docstring-tests also in functions Message-ID: <20120125161220.9993D829FF@wyvern.cs.uni-duesseldorf.de> Author: Antonio Cuni <anto.cuni at gmail.com> Branch: py3k Changeset: r51762:d3e2c1dbfd08 Date: 2012-01-25 16:46 +0100 http://bitbucket.org/pypy/pypy/changeset/d3e2c1dbfd08/ Log: support docstring-tests also in functions diff --git a/pypy/conftest.py b/pypy/conftest.py --- a/pypy/conftest.py +++ b/pypy/conftest.py @@ -428,7 +428,8 @@ return target() space = gettestobjspace() filename = self._getdynfilename(target) - func = app2interp_temp(target, filename=filename) + src = extract_docstring_if_empty_function(target) + func = app2interp_temp(src, filename=filename) print "executing", func self.execute_appex(space, func, space) @@ -482,9 +483,11 @@ def empty_func(): "" pass - co_code = fn.func_code.co_code - if co_code == empty_func.func_code.co_code and fn.__doc__ is not None: - head = '%s(self):' % fn.func_name + empty_func_code = empty_func.func_code + fn_code = fn.func_code + if fn_code.co_code == empty_func_code.co_code and fn.__doc__ is not None: + fnargs = py.std.inspect.getargs(fn_code).args + head = '%s(%s):' % (fn.func_name, ', '.join(fnargs)) body = py.code.Source(fn.__doc__) return head + str(body.indent()) else: diff --git a/pypy/tool/pytest/test/conftest1_innertest.py b/pypy/tool/pytest/test/conftest1_innertest.py --- a/pypy/tool/pytest/test/conftest1_innertest.py +++ b/pypy/tool/pytest/test/conftest1_innertest.py @@ -5,6 +5,11 @@ def app_test_something(): assert 42 == 42 +def app_test_code_in_docstring_failing(): + """ + assert False + """ + class AppTestSomething: def test_method_app(self): assert 23 == 23 diff --git a/pypy/tool/pytest/test/test_conftest1.py b/pypy/tool/pytest/test/test_conftest1.py --- a/pypy/tool/pytest/test/test_conftest1.py +++ b/pypy/tool/pytest/test/test_conftest1.py @@ -42,10 +42,20 @@ assert "test_method_app" in passed[1].nodeid def test_docstring_in_methods(self, testdir): - sorter = testdir.inline_run("-k", "test_code_in_docstring", innertest) + sorter = testdir.inline_run("-k", "AppTestSomething test_code_in_docstring", + innertest) passed, skipped, failed = sorter.listoutcomes() assert len(passed) == 1 assert len(failed) == 1 assert skipped == [] assert "test_code_in_docstring_ignored" in passed[0].nodeid assert "test_code_in_docstring_failing" in failed[0].nodeid + + def test_docstring_in_functions(self, testdir): + sorter = testdir.inline_run("-k", "app_test_code_in_docstring", innertest) + passed, skipped, failed = sorter.listoutcomes() + assert passed == [] + assert len(failed) == 1 + assert skipped == [] + assert "app_test_code_in_docstring_failing" in failed[0].nodeid + From noreply at buildbot.pypy.org Wed Jan 25 17:12:15 2012 From: noreply at buildbot.pypy.org (antocuni) Date: Wed, 25 Jan 2012 17:12:15 +0100 (CET) Subject: [pypy-commit] pypy py3k: make 'from x import *' always an error if it's not at the module level instead of a warning. Adapt or kill the old tests Message-ID: <20120125161215.AF32D82CFE@wyvern.cs.uni-duesseldorf.de> Author: Antonio Cuni <anto.cuni at gmail.com> Branch: py3k Changeset: r51758:f1dde39ee4a3 Date: 2012-01-25 10:55 +0100 http://bitbucket.org/pypy/pypy/changeset/f1dde39ee4a3/ Log: make 'from x import *' always an error if it's not at the module level instead of a warning. Adapt or kill the old tests diff --git a/pypy/interpreter/astcompiler/symtable.py b/pypy/interpreter/astcompiler/symtable.py --- a/pypy/interpreter/astcompiler/symtable.py +++ b/pypy/interpreter/astcompiler/symtable.py @@ -396,9 +396,8 @@ if self._visit_alias(alias): if self.scope.note_import_star(imp): msg = "import * only allowed at module level" - misc.syntax_warning( - self.space, msg, self.compile_info.filename, - imp.lineno, imp.col_offset) + raise SyntaxError(msg, imp.lineno, imp.col_offset, + filename=self.compile_info.filename) def _visit_alias(self, alias): assert isinstance(alias, ast.alias) diff --git a/pypy/interpreter/astcompiler/test/test_symtable.py b/pypy/interpreter/astcompiler/test/test_symtable.py --- a/pypy/interpreter/astcompiler/test/test_symtable.py +++ b/pypy/interpreter/astcompiler/test/test_symtable.py @@ -251,10 +251,6 @@ self.check_unknown(scp, "a", "b") scp = self.mod_scope("from x import *") self.check_unknown("x") - scp = self.func_scope("def f(): from x import *") - self.check_unknown(scp, "x") - assert not scp.optimized - assert scp.import_star def test_global(self): scp = self.func_scope("def f():\n global x\n x = 4") @@ -307,45 +303,28 @@ assert not self.class_scope("class x: pass").can_be_optimized assert self.func_scope("def f(): pass").can_be_optimized - def test_unoptimization_with_nested_scopes(self): - table = ( - ("from x import *", "import * is not allowed in function 'f' " \ - "because it"), - ) - for line, error in table: - input = """def n(): - x = 4 - def f(): - %s - return x""" % (line,) - exc = py.test.raises(SyntaxError, self.mod_scope, input).value - assert exc.msg == error + " is a nested function" - input = """def f(): - %s - x = 4 - def n(): - return x""" % (line,) - exc = py.test.raises(SyntaxError, self.mod_scope, input).value - assert exc.msg == error + " contains a nested function with free variables" - input = """def f(): - %s - x = 4 - class Y: - def n(): - return x""" % (line,) - exc = py.test.raises(SyntaxError, self.mod_scope, input).value - assert exc.msg == error + " contains a nested function with free variables" + def test_importstar_nonglobal(self): + src = str(py.code.Source(""" + def f(): + from re import * + """)) + exc = py.test.raises(SyntaxError, self.mod_scope, src) + assert exc.value.msg == "import * only allowed at module level" + # + src = str(py.code.Source(""" + def f(): + def g(): + from re import * + """)) + exc = py.test.raises(SyntaxError, self.mod_scope, src) + assert exc.value.msg == "import * only allowed at module level" - def test_importstar_warning(self, capfd): - self.mod_scope("def f():\n from re import *") - _, err1 = capfd.readouterr() - - self.mod_scope("if 1:\n from re import *") - _, err2 = capfd.readouterr() - - capfd.close() - assert "import * only allowed at module level" in err1 - assert not "import * only allowed at module level" in err2 + src = str(py.code.Source(""" + if True: + from re import * + """)) + scp = self.mod_scope(src) + assert scp # did not raise def test_yield(self): scp = self.func_scope("def f(): yield x") From noreply at buildbot.pypy.org Wed Jan 25 17:12:21 2012 From: noreply at buildbot.pypy.org (antocuni) Date: Wed, 25 Jan 2012 17:12:21 +0100 (CET) Subject: [pypy-commit] pypy py3k: skip docstring-tests when running with -A Message-ID: <20120125161221.D1C5C829FF@wyvern.cs.uni-duesseldorf.de> Author: Antonio Cuni <anto.cuni at gmail.com> Branch: py3k Changeset: r51763:1b6b08120524 Date: 2012-01-25 17:00 +0100 http://bitbucket.org/pypy/pypy/changeset/1b6b08120524/ Log: skip docstring-tests when running with -A diff --git a/pypy/conftest.py b/pypy/conftest.py --- a/pypy/conftest.py +++ b/pypy/conftest.py @@ -422,13 +422,11 @@ def runtest(self): target = self.obj + src = extract_docstring_if_empty_function(target) if self.config.option.runappdirect: - if self.config.option.appdirect: - return run_with_python(self.config.option.appdirect, target) - return target() + return run_appdirect_or_skip(self.config, target, src) space = gettestobjspace() filename = self._getdynfilename(target) - src = extract_docstring_if_empty_function(target) func = app2interp_temp(src, filename=filename) print "executing", func self.execute_appex(space, func, space) @@ -468,17 +466,27 @@ def runtest(self): target = self.obj + src = extract_docstring_if_empty_function(target.im_func) if self.config.option.runappdirect: - if self.config.option.appdirect: - return run_with_python(self.config.option.appdirect, target) - return target() + return run_appdirect_or_skip(self.config, target, src) space = target.im_self.space filename = self._getdynfilename(target) - src = extract_docstring_if_empty_function(target.im_func) func = app2interp_temp(src, filename=filename) w_instance = self.parent.w_instance self.execute_appex(space, func, space, w_instance) +def run_appdirect_or_skip(config, target, src): + if config.option.appdirect: + return run_with_python(self.config.option.appdirect, target) + if isinstance(src, str): + # we are trying to directly run a test whose code is inside + # the docstring. This cannot work because the code might + # contain py3k-only syntax, while we are on a python2 hosting + # python. So, we just skip the test + py.test.skip('Cannot run docstring-tests with -A') + return target() + + def extract_docstring_if_empty_function(fn): def empty_func(): "" diff --git a/pypy/tool/pytest/test/test_conftest1.py b/pypy/tool/pytest/test/test_conftest1.py --- a/pypy/tool/pytest/test/test_conftest1.py +++ b/pypy/tool/pytest/test/test_conftest1.py @@ -59,3 +59,13 @@ assert skipped == [] assert "app_test_code_in_docstring_failing" in failed[0].nodeid + def test_docstring_runappdirect(self, testdir): + sorter = testdir.inline_run(innertest, + '-k', 'test_code_in_docstring', + '--runappdirect') + passed, skipped, failed = sorter.listoutcomes() + assert len(passed) == 1 + assert len(skipped) == 2 + assert "test_code_in_docstring_ignored" in passed[0].nodeid + assert "app_test_code_in_docstring" in skipped[0].nodeid + assert "test_code_in_docstring_failing" in skipped[1].nodeid From noreply at buildbot.pypy.org Wed Jan 25 17:12:16 2012 From: noreply at buildbot.pypy.org (antocuni) Date: Wed, 25 Jan 2012 17:12:16 +0100 (CET) Subject: [pypy-commit] pypy py3k: add the possibility to write applevel tests inside the docstring (only if the body of the method is empty. This way we can write tests using py3k only syntax and still run them on top of cpython 2 Message-ID: <20120125161216.E651582CFF@wyvern.cs.uni-duesseldorf.de> Author: Antonio Cuni <anto.cuni at gmail.com> Branch: py3k Changeset: r51759:def902951d4f Date: 2012-01-25 16:02 +0100 http://bitbucket.org/pypy/pypy/changeset/def902951d4f/ Log: add the possibility to write applevel tests inside the docstring (only if the body of the method is empty. This way we can write tests using py3k only syntax and still run them on top of cpython 2 diff --git a/pypy/conftest.py b/pypy/conftest.py --- a/pypy/conftest.py +++ b/pypy/conftest.py @@ -473,10 +473,24 @@ return target() space = target.im_self.space filename = self._getdynfilename(target) - func = app2interp_temp(target.im_func, filename=filename) + src = extract_docstring_if_empty_function(target.im_func) + func = app2interp_temp(src, filename=filename) w_instance = self.parent.w_instance self.execute_appex(space, func, space, w_instance) +def extract_docstring_if_empty_function(fn): + def empty_func(): + "" + pass + co_code = fn.func_code.co_code + if co_code == empty_func.func_code.co_code and fn.__doc__ is not None: + head = '%s(self):' % fn.func_name + body = py.code.Source(fn.__doc__) + return head + str(body.indent()) + else: + return fn + + class PyPyClassCollector(py.test.collect.Class): def setup(self): cls = self.obj diff --git a/pypy/tool/pytest/test/conftest1_innertest.py b/pypy/tool/pytest/test/conftest1_innertest.py --- a/pypy/tool/pytest/test/conftest1_innertest.py +++ b/pypy/tool/pytest/test/conftest1_innertest.py @@ -7,7 +7,20 @@ class AppTestSomething: def test_method_app(self): - assert 23 == 23 + assert 23 == 23 + + def test_code_in_docstring_failing(self): + """ + assert False # failing test + """ + + def test_code_in_docstring_ignored(self): + """ + this docstring is not parsed as code because the function body is not + empty + """ + assert True + class TestSomething: def test_method(self): diff --git a/pypy/tool/pytest/test/test_conftest1.py b/pypy/tool/pytest/test/test_conftest1.py --- a/pypy/tool/pytest/test/test_conftest1.py +++ b/pypy/tool/pytest/test/test_conftest1.py @@ -17,15 +17,18 @@ def test_selection_by_keyword_app(self, testdir): sorter = testdir.inline_run("-k", "applevel", innertest) passed, skipped, failed = sorter.listoutcomes() - assert len(passed) == 2 - assert not skipped and not failed + assert len(passed) == 3 + assert len(failed) == 1 + assert skipped == [] assert "app_test_something" in passed[0].nodeid assert "test_method_app" in passed[1].nodeid - + assert "test_code_in_docstring_ignored" in passed[2].nodeid + assert "test_code_in_docstring_failing" in failed[0].nodeid + def test_runappdirect(self, testdir): sorter = testdir.inline_run(innertest, '-k', 'applevel', '--runappdirect') passed, skipped, failed = sorter.listoutcomes() - assert len(passed) == 2 + assert len(passed) == 3 print passed assert "app_test_something" in passed[0].nodeid assert "test_method_app" in passed[1].nodeid @@ -34,8 +37,16 @@ sorter = testdir.inline_run(innertest, '-k', 'applevel', '--appdirect=%s' % (sys.executable,)) passed, skipped, failed = sorter.listoutcomes() - assert len(passed) == 2 + assert len(passed) == 3 print passed assert "app_test_something" in passed[0].nodeid assert "test_method_app" in passed[1].nodeid + def test_docstrings(self, testdir): + sorter = testdir.inline_run("-k", "test_code_in_docstring", innertest) + passed, skipped, failed = sorter.listoutcomes() + assert len(passed) == 1 + assert len(failed) == 1 + assert skipped == [] + assert "test_code_in_docstring_ignored" in passed[0].nodeid + assert "test_code_in_docstring_failing" in failed[0].nodeid From noreply at buildbot.pypy.org Wed Jan 25 17:12:23 2012 From: noreply at buildbot.pypy.org (antocuni) Date: Wed, 25 Jan 2012 17:12:23 +0100 (CET) Subject: [pypy-commit] pypy py3k: correctly run docstring-tests when using --appdirect=... Message-ID: <20120125161223.12416829FF@wyvern.cs.uni-duesseldorf.de> Author: Antonio Cuni <anto.cuni at gmail.com> Branch: py3k Changeset: r51764:9868585f9314 Date: 2012-01-25 17:10 +0100 http://bitbucket.org/pypy/pypy/changeset/9868585f9314/ Log: correctly run docstring-tests when using --appdirect=... diff --git a/pypy/conftest.py b/pypy/conftest.py --- a/pypy/conftest.py +++ b/pypy/conftest.py @@ -477,7 +477,7 @@ def run_appdirect_or_skip(config, target, src): if config.option.appdirect: - return run_with_python(self.config.option.appdirect, target) + return run_with_python(config.option.appdirect, src) if isinstance(src, str): # we are trying to directly run a test whose code is inside # the docstring. This cannot work because the code might diff --git a/pypy/tool/pytest/test/test_conftest1.py b/pypy/tool/pytest/test/test_conftest1.py --- a/pypy/tool/pytest/test/test_conftest1.py +++ b/pypy/tool/pytest/test/test_conftest1.py @@ -67,5 +67,16 @@ assert len(passed) == 1 assert len(skipped) == 2 assert "test_code_in_docstring_ignored" in passed[0].nodeid - assert "app_test_code_in_docstring" in skipped[0].nodeid + assert "app_test_code_in_docstring_failing" in skipped[0].nodeid assert "test_code_in_docstring_failing" in skipped[1].nodeid + + def test_docstring_appdirect(self, testdir): + sorter = testdir.inline_run(innertest, + '-k', 'test_code_in_docstring', + '--appdirect=%s' % (sys.executable,)) + passed, skipped, failed = sorter.listoutcomes() + assert len(passed) == 1 + assert len(failed) == 2 + assert "test_code_in_docstring_ignored" in passed[0].nodeid + assert "app_test_code_in_docstring_failing" in failed[0].nodeid + assert "test_code_in_docstring_failing" in failed[1].nodeid From noreply at buildbot.pypy.org Wed Jan 25 17:12:18 2012 From: noreply at buildbot.pypy.org (antocuni) Date: Wed, 25 Jan 2012 17:12:18 +0100 (CET) Subject: [pypy-commit] pypy py3k: test docstrings separately from the other applevel tests Message-ID: <20120125161218.2553C82D00@wyvern.cs.uni-duesseldorf.de> Author: Antonio Cuni <anto.cuni at gmail.com> Branch: py3k Changeset: r51760:b34f81eba785 Date: 2012-01-25 16:35 +0100 http://bitbucket.org/pypy/pypy/changeset/b34f81eba785/ Log: test docstrings separately from the other applevel tests diff --git a/pypy/tool/pytest/test/test_conftest1.py b/pypy/tool/pytest/test/test_conftest1.py --- a/pypy/tool/pytest/test/test_conftest1.py +++ b/pypy/tool/pytest/test/test_conftest1.py @@ -15,18 +15,17 @@ assert "test_method" in passed[1].nodeid def test_selection_by_keyword_app(self, testdir): - sorter = testdir.inline_run("-k", "applevel", innertest) + sorter = testdir.inline_run("-k", "applevel -docstring", innertest) passed, skipped, failed = sorter.listoutcomes() - assert len(passed) == 3 - assert len(failed) == 1 + assert len(passed) == 2 + assert failed == [] assert skipped == [] assert "app_test_something" in passed[0].nodeid assert "test_method_app" in passed[1].nodeid - assert "test_code_in_docstring_ignored" in passed[2].nodeid - assert "test_code_in_docstring_failing" in failed[0].nodeid - + def test_runappdirect(self, testdir): - sorter = testdir.inline_run(innertest, '-k', 'applevel', '--runappdirect') + sorter = testdir.inline_run(innertest, '-k', 'applevel -docstring', + '--runappdirect') passed, skipped, failed = sorter.listoutcomes() assert len(passed) == 3 print passed @@ -34,7 +33,7 @@ assert "test_method_app" in passed[1].nodeid def test_appdirect(self, testdir): - sorter = testdir.inline_run(innertest, '-k', 'applevel', + sorter = testdir.inline_run(innertest, '-k', 'applevel -docstring', '--appdirect=%s' % (sys.executable,)) passed, skipped, failed = sorter.listoutcomes() assert len(passed) == 3 @@ -42,7 +41,7 @@ assert "app_test_something" in passed[0].nodeid assert "test_method_app" in passed[1].nodeid - def test_docstrings(self, testdir): + def test_docstring_in_methods(self, testdir): sorter = testdir.inline_run("-k", "test_code_in_docstring", innertest) passed, skipped, failed = sorter.listoutcomes() assert len(passed) == 1 From noreply at buildbot.pypy.org Wed Jan 25 17:15:22 2012 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 25 Jan 2012 17:15:22 +0100 (CET) Subject: [pypy-commit] pypy kill-more-multimethods: remove performance hack Message-ID: <20120125161522.F1AA0829FF@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: kill-more-multimethods Changeset: r51765:5ab71bed7f64 Date: 2012-01-25 18:14 +0200 http://bitbucket.org/pypy/pypy/changeset/5ab71bed7f64/ Log: remove performance hack diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py --- a/pypy/objspace/std/objspace.py +++ b/pypy/objspace/std/objspace.py @@ -544,19 +544,6 @@ else: self.setitem(w_obj, self.wrap(key), w_value) - def getindex_w(self, w_obj, w_exception, objdescr=None): - # Performance shortcut for the common case of w_obj being an int. - # If withsmallint is disabled, we check for W_IntObject. - # If withsmallint is enabled, we only check for W_SmallIntObject - it's - # probably not useful to have a shortcut for W_IntObject at all then. - if self.config.objspace.std.withsmallint: - if type(w_obj) is W_SmallIntObject: - return w_obj.intval - else: - if type(w_obj) is W_IntObject: - return w_obj.intval - return ObjSpace.getindex_w(self, w_obj, w_exception, objdescr) - def call_method(self, w_obj, methname, *arg_w): if self.config.objspace.opcodes.CALL_METHOD: return callmethod.call_method_opt(self, w_obj, methname, *arg_w) From noreply at buildbot.pypy.org Wed Jan 25 17:28:52 2012 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 25 Jan 2012 17:28:52 +0100 (CET) Subject: [pypy-commit] pypy default: a test and a fix for boxes identity Message-ID: <20120125162852.C29C6829FF@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: Changeset: r51766:3890a6e149fa Date: 2012-01-25 18:28 +0200 http://bitbucket.org/pypy/pypy/changeset/3890a6e149fa/ Log: a test and a fix for boxes identity diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py --- a/pypy/module/micronumpy/interp_boxes.py +++ b/pypy/module/micronumpy/interp_boxes.py @@ -9,6 +9,7 @@ MIXIN_64 = (int_typedef,) if LONG_BIT == 64 else () +MIXIN_32 = () if LONG_BIT == 64 else (int_typedef,) def new_dtype_getter(name): def get_dtype(space): @@ -231,7 +232,7 @@ __new__ = interp2app(W_UInt16Box.descr__new__.im_func), ) -W_Int32Box.typedef = TypeDef("int32", W_SignedIntegerBox.typedef, +W_Int32Box.typedef = TypeDef("int32", (W_SignedIntegerBox.typedef,) + MIXIN_32, __module__ = "numpypy", __new__ = interp2app(W_Int32Box.descr__new__.im_func), ) @@ -241,24 +242,18 @@ __new__ = interp2app(W_UInt32Box.descr__new__.im_func), ) -if LONG_BIT == 32: - long_name = "int32" -elif LONG_BIT == 64: - long_name = "int64" -W_LongBox.typedef = TypeDef(long_name, (W_SignedIntegerBox.typedef, int_typedef,), - __module__ = "numpypy", - __new__ = interp2app(W_LongBox.descr__new__.im_func), -) - -W_ULongBox.typedef = TypeDef("u" + long_name, W_UnsignedIntegerBox.typedef, - __module__ = "numpypy", -) - W_Int64Box.typedef = TypeDef("int64", (W_SignedIntegerBox.typedef,) + MIXIN_64, __module__ = "numpypy", __new__ = interp2app(W_Int64Box.descr__new__.im_func), ) +if LONG_BIT == 32: + W_LongBox = W_Int32Box + W_ULongBox = W_UInt32Box +elif LONG_BIT == 64: + W_LongBox = W_Int64Box + W_ULongBox = W_UInt64Box + W_UInt64Box.typedef = TypeDef("uint64", W_UnsignedIntegerBox.typedef, __module__ = "numpypy", __new__ = interp2app(W_UInt64Box.descr__new__.im_func), diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py --- a/pypy/module/micronumpy/test/test_dtypes.py +++ b/pypy/module/micronumpy/test/test_dtypes.py @@ -376,3 +376,14 @@ b = X(10) assert type(b) is X assert b.m() == 12 + + def test_int(self): + import sys + from _numpypy import int32, int64, int_ + assert issubclass(int_, int) + if sys.maxint == (1<<31) - 1: + assert issubclass(int32, int) + assert int_ is int32 + else: + assert issubclass(int64, int) + assert int_ is int64 From noreply at buildbot.pypy.org Wed Jan 25 21:37:29 2012 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 25 Jan 2012 21:37:29 +0100 (CET) Subject: [pypy-commit] pypy stm: Comment out this, as it doesn't work reliably. Message-ID: <20120125203729.20E8B829FF@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: stm Changeset: r51767:de5b100916b6 Date: 2012-01-25 21:36 +0100 http://bitbucket.org/pypy/pypy/changeset/de5b100916b6/ Log: Comment out this, as it doesn't work reliably. diff --git a/pypy/translator/stm/transform.py b/pypy/translator/stm/transform.py --- a/pypy/translator/stm/transform.py +++ b/pypy/translator/stm/transform.py @@ -44,7 +44,7 @@ return newoperations = [] self.current_block = block - self.access_directly = set() + #self.access_directly = set() for i, op in enumerate(block.operations): self.current_op_index = i try: @@ -66,7 +66,7 @@ assert res is None block.operations = newoperations self.current_block = None - self.access_directly = None + #self.access_directly = None def transform_graph(self, graph): for block in graph.iterblocks(): @@ -118,14 +118,14 @@ elif STRUCT._immutable_field(op.args[1].value): op1 = op elif 'stm_access_directly' in STRUCT._hints: - try: - immfld = STRUCT._hints['immutable_fields'] - except KeyError: - pass - else: - rank = immfld._fields.get(op.args[1].value, None) - if rank is rclass.IR_MUTABLE_OWNED: - self.access_directly.add(op.result) + #try: + # immfld = STRUCT._hints['immutable_fields'] + #except KeyError: + # pass + #else: + # rank = immfld._fields.get(op.args[1].value, None) + # if rank is rclass.IR_MUTABLE_OWNED: + # self.access_directly.add(op.result) op1 = op elif STRUCT._gckind == 'raw': turn_inevitable(newoperations, "getfield-raw") @@ -154,8 +154,8 @@ op1 = op elif ARRAY._immutable_field(): op1 = op - elif op.args[0] in self.access_directly: - op1 = op + #elif op.args[0] in self.access_directly: + # op1 = op elif ARRAY._gckind == 'raw': turn_inevitable(newoperations, "getarrayitem-raw") op1 = op @@ -169,8 +169,8 @@ op1 = op elif ARRAY._immutable_field(): op1 = op - elif op.args[0] in self.access_directly: - op1 = op + #elif op.args[0] in self.access_directly: + # op1 = op elif ARRAY._gckind == 'raw': turn_inevitable(newoperations, "setarrayitem-raw") op1 = op @@ -235,12 +235,12 @@ ## self.seen_gc_stack_bottom = True newoperations.append(op) - def stt_same_as(self, newoperations, op): - if op.args[0] in self.access_directly: - self.access_directly.add(op.result) - newoperations.append(op) - - stt_cast_pointer = stt_same_as + #def stt_same_as(self, newoperations, op): + # if op.args[0] in self.access_directly: + # self.access_directly.add(op.result) + # newoperations.append(op) + # + #stt_cast_pointer = stt_same_as def transform_graph(graph): From noreply at buildbot.pypy.org Wed Jan 25 21:37:30 2012 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 25 Jan 2012 21:37:30 +0100 (CET) Subject: [pypy-commit] pypy stm: merge heads Message-ID: <20120125203730.D6B07829FF@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: stm Changeset: r51768:f0ee46ba0dfb Date: 2012-01-25 21:37 +0100 http://bitbucket.org/pypy/pypy/changeset/f0ee46ba0dfb/ Log: merge heads diff --git a/.hgignore b/.hgignore --- a/.hgignore +++ b/.hgignore @@ -2,6 +2,9 @@ *.py[co] *~ .*.swp +.idea +.project +.pydevproject syntax: regexp ^testresult$ diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py --- a/lib_pypy/_sqlite3.py +++ b/lib_pypy/_sqlite3.py @@ -20,6 +20,8 @@ # 2. Altered source versions must be plainly marked as such, and must not be # misrepresented as being the original software. # 3. This notice may not be removed or altered from any source distribution. +# +# Note: This software has been modified for use in PyPy. from ctypes import c_void_p, c_int, c_double, c_int64, c_char_p, cdll from ctypes import POINTER, byref, string_at, CFUNCTYPE, cast @@ -27,7 +29,6 @@ from collections import OrderedDict import datetime import sys -import time import weakref from threading import _get_ident as thread_get_ident @@ -606,7 +607,7 @@ def authorizer(userdata, action, arg1, arg2, dbname, source): try: return int(callback(action, arg1, arg2, dbname, source)) - except Exception, e: + except Exception: return SQLITE_DENY c_authorizer = AUTHORIZER(authorizer) @@ -653,7 +654,7 @@ if not aggregate_ptr[0]: try: aggregate = cls() - except Exception, e: + except Exception: msg = ("user-defined aggregate's '__init__' " "method raised error") sqlite.sqlite3_result_error(context, msg, len(msg)) @@ -667,7 +668,7 @@ params = _convert_params(context, argc, c_params) try: aggregate.step(*params) - except Exception, e: + except Exception: msg = ("user-defined aggregate's 'step' " "method raised error") sqlite.sqlite3_result_error(context, msg, len(msg)) @@ -683,7 +684,7 @@ aggregate = self.aggregate_instances[aggregate_ptr[0]] try: val = aggregate.finalize() - except Exception, e: + except Exception: msg = ("user-defined aggregate's 'finalize' " "method raised error") sqlite.sqlite3_result_error(context, msg, len(msg)) @@ -771,7 +772,7 @@ self.statement.item = None self.statement.exhausted = True - if self.statement.kind == DML or self.statement.kind == DDL: + if self.statement.kind == DML: self.statement.reset() self.rowcount = -1 @@ -791,7 +792,7 @@ if self.statement.kind == DML: self.connection._begin() else: - raise ProgrammingError, "executemany is only for DML statements" + raise ProgrammingError("executemany is only for DML statements") self.rowcount = 0 for params in many_params: @@ -861,8 +862,6 @@ except StopIteration: return None - return nextrow - def fetchmany(self, size=None): self._check_closed() self._check_reset() @@ -915,7 +914,7 @@ def __init__(self, connection, sql): self.statement = None if not isinstance(sql, str): - raise ValueError, "sql must be a string" + raise ValueError("sql must be a string") self.con = connection self.sql = sql # DEBUG ONLY first_word = self._statement_kind = sql.lstrip().split(" ")[0].upper() @@ -944,8 +943,8 @@ raise self.con._get_exception(ret) self.con._remember_statement(self) if _check_remaining_sql(next_char.value): - raise Warning, "One and only one statement required: %r" % ( - next_char.value,) + raise Warning("One and only one statement required: %r" % ( + next_char.value,)) # sql_char should remain alive until here self._build_row_cast_map() @@ -1016,7 +1015,7 @@ elif type(param) is buffer: sqlite.sqlite3_bind_blob(self.statement, idx, str(param), len(param), SQLITE_TRANSIENT) else: - raise InterfaceError, "parameter type %s is not supported" % str(type(param)) + raise InterfaceError("parameter type %s is not supported" % str(type(param))) def set_params(self, params): ret = sqlite.sqlite3_reset(self.statement) @@ -1045,11 +1044,11 @@ for idx in range(1, sqlite.sqlite3_bind_parameter_count(self.statement) + 1): param_name = sqlite.sqlite3_bind_parameter_name(self.statement, idx) if param_name is None: - raise ProgrammingError, "need named parameters" + raise ProgrammingError("need named parameters") param_name = param_name[1:] try: param = params[param_name] - except KeyError, e: + except KeyError: raise ProgrammingError("missing parameter '%s'" %param) self.set_param(idx, param) @@ -1260,7 +1259,7 @@ params = _convert_params(context, nargs, c_params) try: val = real_cb(*params) - except Exception, e: + except Exception: msg = "user-defined function raised exception" sqlite.sqlite3_result_error(context, msg, len(msg)) else: diff --git a/lib_pypy/datetime.py b/lib_pypy/datetime.py --- a/lib_pypy/datetime.py +++ b/lib_pypy/datetime.py @@ -13,7 +13,7 @@ Sources for time zone and DST data: http://www.twinsun.com/tz/tz-link.htm This was originally copied from the sandbox of the CPython CVS repository. -Thanks to Tim Peters for suggesting using it. +Thanks to Tim Peters for suggesting using it. """ import time as _time @@ -271,6 +271,8 @@ raise ValueError("%s()=%d, must be in -1439..1439" % (name, offset)) def _check_date_fields(year, month, day): + if not isinstance(year, (int, long)): + raise TypeError('int expected') if not MINYEAR <= year <= MAXYEAR: raise ValueError('year must be in %d..%d' % (MINYEAR, MAXYEAR), year) if not 1 <= month <= 12: @@ -280,6 +282,8 @@ raise ValueError('day must be in 1..%d' % dim, day) def _check_time_fields(hour, minute, second, microsecond): + if not isinstance(hour, (int, long)): + raise TypeError('int expected') if not 0 <= hour <= 23: raise ValueError('hour must be in 0..23', hour) if not 0 <= minute <= 59: @@ -543,61 +547,76 @@ self = object.__new__(cls) - self.__days = d - self.__seconds = s - self.__microseconds = us + self._days = d + self._seconds = s + self._microseconds = us if abs(d) > 999999999: raise OverflowError("timedelta # of days is too large: %d" % d) return self def __repr__(self): - if self.__microseconds: + if self._microseconds: return "%s(%d, %d, %d)" % ('datetime.' + self.__class__.__name__, - self.__days, - self.__seconds, - self.__microseconds) - if self.__seconds: + self._days, + self._seconds, + self._microseconds) + if self._seconds: return "%s(%d, %d)" % ('datetime.' + self.__class__.__name__, - self.__days, - self.__seconds) - return "%s(%d)" % ('datetime.' + self.__class__.__name__, self.__days) + self._days, + self._seconds) + return "%s(%d)" % ('datetime.' + self.__class__.__name__, self._days) def __str__(self): - mm, ss = divmod(self.__seconds, 60) + mm, ss = divmod(self._seconds, 60) hh, mm = divmod(mm, 60) s = "%d:%02d:%02d" % (hh, mm, ss) - if self.__days: + if self._days: def plural(n): return n, abs(n) != 1 and "s" or "" - s = ("%d day%s, " % plural(self.__days)) + s - if self.__microseconds: - s = s + ".%06d" % self.__microseconds + s = ("%d day%s, " % plural(self._days)) + s + if self._microseconds: + s = s + ".%06d" % self._microseconds return s - days = property(lambda self: self.__days, doc="days") - seconds = property(lambda self: self.__seconds, doc="seconds") - microseconds = property(lambda self: self.__microseconds, - doc="microseconds") - def total_seconds(self): return ((self.days * 86400 + self.seconds) * 10**6 + self.microseconds) / 1e6 + # Read-only field accessors + @property + def days(self): + """days""" + return self._days + + @property + def seconds(self): + """seconds""" + return self._seconds + + @property + def microseconds(self): + """microseconds""" + return self._microseconds + def __add__(self, other): if isinstance(other, timedelta): # for CPython compatibility, we cannot use # our __class__ here, but need a real timedelta - return timedelta(self.__days + other.__days, - self.__seconds + other.__seconds, - self.__microseconds + other.__microseconds) + return timedelta(self._days + other._days, + self._seconds + other._seconds, + self._microseconds + other._microseconds) return NotImplemented __radd__ = __add__ def __sub__(self, other): if isinstance(other, timedelta): - return self + -other + # for CPython compatibility, we cannot use + # our __class__ here, but need a real timedelta + return timedelta(self._days - other._days, + self._seconds - other._seconds, + self._microseconds - other._microseconds) return NotImplemented def __rsub__(self, other): @@ -606,17 +625,17 @@ return NotImplemented def __neg__(self): - # for CPython compatibility, we cannot use - # our __class__ here, but need a real timedelta - return timedelta(-self.__days, - -self.__seconds, - -self.__microseconds) + # for CPython compatibility, we cannot use + # our __class__ here, but need a real timedelta + return timedelta(-self._days, + -self._seconds, + -self._microseconds) def __pos__(self): return self def __abs__(self): - if self.__days < 0: + if self._days < 0: return -self else: return self @@ -625,81 +644,81 @@ if isinstance(other, (int, long)): # for CPython compatibility, we cannot use # our __class__ here, but need a real timedelta - return timedelta(self.__days * other, - self.__seconds * other, - self.__microseconds * other) + return timedelta(self._days * other, + self._seconds * other, + self._microseconds * other) return NotImplemented __rmul__ = __mul__ def __div__(self, other): if isinstance(other, (int, long)): - usec = ((self.__days * (24*3600L) + self.__seconds) * 1000000 + - self.__microseconds) + usec = ((self._days * (24*3600L) + self._seconds) * 1000000 + + self._microseconds) return timedelta(0, 0, usec // other) return NotImplemented __floordiv__ = __div__ - # Comparisons. + # Comparisons of timedelta objects with other. def __eq__(self, other): if isinstance(other, timedelta): - return self.__cmp(other) == 0 + return self._cmp(other) == 0 else: return False def __ne__(self, other): if isinstance(other, timedelta): - return self.__cmp(other) != 0 + return self._cmp(other) != 0 else: return True def __le__(self, other): if isinstance(other, timedelta): - return self.__cmp(other) <= 0 + return self._cmp(other) <= 0 else: _cmperror(self, other) def __lt__(self, other): if isinstance(other, timedelta): - return self.__cmp(other) < 0 + return self._cmp(other) < 0 else: _cmperror(self, other) def __ge__(self, other): if isinstance(other, timedelta): - return self.__cmp(other) >= 0 + return self._cmp(other) >= 0 else: _cmperror(self, other) def __gt__(self, other): if isinstance(other, timedelta): - return self.__cmp(other) > 0 + return self._cmp(other) > 0 else: _cmperror(self, other) - def __cmp(self, other): + def _cmp(self, other): assert isinstance(other, timedelta) - return cmp(self.__getstate(), other.__getstate()) + return cmp(self._getstate(), other._getstate()) def __hash__(self): - return hash(self.__getstate()) + return hash(self._getstate()) def __nonzero__(self): - return (self.__days != 0 or - self.__seconds != 0 or - self.__microseconds != 0) + return (self._days != 0 or + self._seconds != 0 or + self._microseconds != 0) # Pickle support. __safe_for_unpickling__ = True # For Python 2.2 - def __getstate(self): - return (self.__days, self.__seconds, self.__microseconds) + def _getstate(self): + return (self._days, self._seconds, self._microseconds) def __reduce__(self): - return (self.__class__, self.__getstate()) + return (self.__class__, self._getstate()) timedelta.min = timedelta(-999999999) timedelta.max = timedelta(days=999999999, hours=23, minutes=59, seconds=59, @@ -749,25 +768,26 @@ return self _check_date_fields(year, month, day) self = object.__new__(cls) - self.__year = year - self.__month = month - self.__day = day + self._year = year + self._month = month + self._day = day return self # Additional constructors + @classmethod def fromtimestamp(cls, t): "Construct a date from a POSIX timestamp (like time.time())." y, m, d, hh, mm, ss, weekday, jday, dst = _time.localtime(t) return cls(y, m, d) - fromtimestamp = classmethod(fromtimestamp) + @classmethod def today(cls): "Construct a date from time.time()." t = _time.time() return cls.fromtimestamp(t) - today = classmethod(today) + @classmethod def fromordinal(cls, n): """Contruct a date from a proleptic Gregorian ordinal. @@ -776,16 +796,24 @@ """ y, m, d = _ord2ymd(n) return cls(y, m, d) - fromordinal = classmethod(fromordinal) # Conversions to string def __repr__(self): - "Convert to formal string, for repr()." + """Convert to formal string, for repr(). + + >>> dt = datetime(2010, 1, 1) + >>> repr(dt) + 'datetime.datetime(2010, 1, 1, 0, 0)' + + >>> dt = datetime(2010, 1, 1, tzinfo=timezone.utc) + >>> repr(dt) + 'datetime.datetime(2010, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)' + """ return "%s(%d, %d, %d)" % ('datetime.' + self.__class__.__name__, - self.__year, - self.__month, - self.__day) + self._year, + self._month, + self._day) # XXX These shouldn't depend on time.localtime(), because that # clips the usable dates to [1970 .. 2038). At least ctime() is # easily done without using strftime() -- that's better too because @@ -793,12 +821,20 @@ def ctime(self): "Format a la ctime()." - return tmxxx(self.__year, self.__month, self.__day).ctime() + return tmxxx(self._year, self._month, self._day).ctime() def strftime(self, fmt): "Format using strftime()." return _wrap_strftime(self, fmt, self.timetuple()) + def __format__(self, fmt): + if not isinstance(fmt, (str, unicode)): + raise ValueError("__format__ excepts str or unicode, not %s" % + fmt.__class__.__name__) + if len(fmt) != 0: + return self.strftime(fmt) + return str(self) + def isoformat(self): """Return the date formatted according to ISO. @@ -808,29 +844,31 @@ - http://www.w3.org/TR/NOTE-datetime - http://www.cl.cam.ac.uk/~mgk25/iso-time.html """ - return "%04d-%02d-%02d" % (self.__year, self.__month, self.__day) + return "%04d-%02d-%02d" % (self._year, self._month, self._day) __str__ = isoformat - def __format__(self, format): - if not isinstance(format, (str, unicode)): - raise ValueError("__format__ excepts str or unicode, not %s" % - format.__class__.__name__) - if not format: - return str(self) - return self.strftime(format) + # Read-only field accessors + @property + def year(self): + """year (1-9999)""" + return self._year - # Read-only field accessors - year = property(lambda self: self.__year, - doc="year (%d-%d)" % (MINYEAR, MAXYEAR)) - month = property(lambda self: self.__month, doc="month (1-12)") - day = property(lambda self: self.__day, doc="day (1-31)") + @property + def month(self): + """month (1-12)""" + return self._month + + @property + def day(self): + """day (1-31)""" + return self._day # Standard conversions, __cmp__, __hash__ (and helpers) def timetuple(self): "Return local time tuple compatible with time.localtime()." - return _build_struct_time(self.__year, self.__month, self.__day, + return _build_struct_time(self._year, self._month, self._day, 0, 0, 0, -1) def toordinal(self): @@ -839,24 +877,24 @@ January 1 of year 1 is day 1. Only the year, month and day values contribute to the result. """ - return _ymd2ord(self.__year, self.__month, self.__day) + return _ymd2ord(self._year, self._month, self._day) def replace(self, year=None, month=None, day=None): """Return a new date with new values for the specified fields.""" if year is None: - year = self.__year + year = self._year if month is None: - month = self.__month + month = self._month if day is None: - day = self.__day + day = self._day _check_date_fields(year, month, day) return date(year, month, day) - # Comparisons. + # Comparisons of date objects with other. def __eq__(self, other): if isinstance(other, date): - return self.__cmp(other) == 0 + return self._cmp(other) == 0 elif hasattr(other, "timetuple"): return NotImplemented else: @@ -864,7 +902,7 @@ def __ne__(self, other): if isinstance(other, date): - return self.__cmp(other) != 0 + return self._cmp(other) != 0 elif hasattr(other, "timetuple"): return NotImplemented else: @@ -872,7 +910,7 @@ def __le__(self, other): if isinstance(other, date): - return self.__cmp(other) <= 0 + return self._cmp(other) <= 0 elif hasattr(other, "timetuple"): return NotImplemented else: @@ -880,7 +918,7 @@ def __lt__(self, other): if isinstance(other, date): - return self.__cmp(other) < 0 + return self._cmp(other) < 0 elif hasattr(other, "timetuple"): return NotImplemented else: @@ -888,7 +926,7 @@ def __ge__(self, other): if isinstance(other, date): - return self.__cmp(other) >= 0 + return self._cmp(other) >= 0 elif hasattr(other, "timetuple"): return NotImplemented else: @@ -896,21 +934,21 @@ def __gt__(self, other): if isinstance(other, date): - return self.__cmp(other) > 0 + return self._cmp(other) > 0 elif hasattr(other, "timetuple"): return NotImplemented else: _cmperror(self, other) - def __cmp(self, other): + def _cmp(self, other): assert isinstance(other, date) - y, m, d = self.__year, self.__month, self.__day - y2, m2, d2 = other.__year, other.__month, other.__day + y, m, d = self._year, self._month, self._day + y2, m2, d2 = other._year, other._month, other._day return cmp((y, m, d), (y2, m2, d2)) def __hash__(self): "Hash." - return hash(self.__getstate()) + return hash(self._getstate()) # Computations @@ -922,9 +960,9 @@ def __add__(self, other): "Add a date to a timedelta." if isinstance(other, timedelta): - t = tmxxx(self.__year, - self.__month, - self.__day + other.days) + t = tmxxx(self._year, + self._month, + self._day + other.days) self._checkOverflow(t.year) result = date(t.year, t.month, t.day) return result @@ -966,9 +1004,9 @@ ISO calendar algorithm taken from http://www.phys.uu.nl/~vgent/calendar/isocalendar.htm """ - year = self.__year + year = self._year week1monday = _isoweek1monday(year) - today = _ymd2ord(self.__year, self.__month, self.__day) + today = _ymd2ord(self._year, self._month, self._day) # Internally, week and day have origin 0 week, day = divmod(today - week1monday, 7) if week < 0: @@ -985,18 +1023,18 @@ __safe_for_unpickling__ = True # For Python 2.2 - def __getstate(self): - yhi, ylo = divmod(self.__year, 256) - return ("%c%c%c%c" % (yhi, ylo, self.__month, self.__day), ) + def _getstate(self): + yhi, ylo = divmod(self._year, 256) + return ("%c%c%c%c" % (yhi, ylo, self._month, self._day), ) def __setstate(self, string): if len(string) != 4 or not (1 <= ord(string[2]) <= 12): raise TypeError("not enough arguments") - yhi, ylo, self.__month, self.__day = map(ord, string) - self.__year = yhi * 256 + ylo + yhi, ylo, self._month, self._day = map(ord, string) + self._year = yhi * 256 + ylo def __reduce__(self): - return (self.__class__, self.__getstate()) + return (self.__class__, self._getstate()) _date_class = date # so functions w/ args named "date" can get at the class @@ -1118,62 +1156,80 @@ return self _check_tzinfo_arg(tzinfo) _check_time_fields(hour, minute, second, microsecond) - self.__hour = hour - self.__minute = minute - self.__second = second - self.__microsecond = microsecond + self._hour = hour + self._minute = minute + self._second = second + self._microsecond = microsecond self._tzinfo = tzinfo return self # Read-only field accessors - hour = property(lambda self: self.__hour, doc="hour (0-23)") - minute = property(lambda self: self.__minute, doc="minute (0-59)") - second = property(lambda self: self.__second, doc="second (0-59)") - microsecond = property(lambda self: self.__microsecond, - doc="microsecond (0-999999)") - tzinfo = property(lambda self: self._tzinfo, doc="timezone info object") + @property + def hour(self): + """hour (0-23)""" + return self._hour + + @property + def minute(self): + """minute (0-59)""" + return self._minute + + @property + def second(self): + """second (0-59)""" + return self._second + + @property + def microsecond(self): + """microsecond (0-999999)""" + return self._microsecond + + @property + def tzinfo(self): + """timezone info object""" + return self._tzinfo # Standard conversions, __hash__ (and helpers) - # Comparisons. + # Comparisons of time objects with other. def __eq__(self, other): if isinstance(other, time): - return self.__cmp(other) == 0 + return self._cmp(other) == 0 else: return False def __ne__(self, other): if isinstance(other, time): - return self.__cmp(other) != 0 + return self._cmp(other) != 0 else: return True def __le__(self, other): if isinstance(other, time): - return self.__cmp(other) <= 0 + return self._cmp(other) <= 0 else: _cmperror(self, other) def __lt__(self, other): if isinstance(other, time): - return self.__cmp(other) < 0 + return self._cmp(other) < 0 else: _cmperror(self, other) def __ge__(self, other): if isinstance(other, time): - return self.__cmp(other) >= 0 + return self._cmp(other) >= 0 else: _cmperror(self, other) def __gt__(self, other): if isinstance(other, time): - return self.__cmp(other) > 0 + return self._cmp(other) > 0 else: _cmperror(self, other) - def __cmp(self, other): + def _cmp(self, other): assert isinstance(other, time) mytz = self._tzinfo ottz = other._tzinfo @@ -1187,23 +1243,23 @@ base_compare = myoff == otoff if base_compare: - return cmp((self.__hour, self.__minute, self.__second, - self.__microsecond), - (other.__hour, other.__minute, other.__second, - other.__microsecond)) + return cmp((self._hour, self._minute, self._second, + self._microsecond), + (other._hour, other._minute, other._second, + other._microsecond)) if myoff is None or otoff is None: # XXX Buggy in 2.2.2. raise TypeError("cannot compare naive and aware times") - myhhmm = self.__hour * 60 + self.__minute - myoff - othhmm = other.__hour * 60 + other.__minute - otoff - return cmp((myhhmm, self.__second, self.__microsecond), - (othhmm, other.__second, other.__microsecond)) + myhhmm = self._hour * 60 + self._minute - myoff + othhmm = other._hour * 60 + other._minute - otoff + return cmp((myhhmm, self._second, self._microsecond), + (othhmm, other._second, other._microsecond)) def __hash__(self): """Hash.""" tzoff = self._utcoffset() if not tzoff: # zero or None - return hash(self.__getstate()[0]) + return hash(self._getstate()[0]) h, m = divmod(self.hour * 60 + self.minute - tzoff, 60) if 0 <= h < 24: return hash(time(h, m, self.second, self.microsecond)) @@ -1227,14 +1283,14 @@ def __repr__(self): """Convert to formal string, for repr().""" - if self.__microsecond != 0: - s = ", %d, %d" % (self.__second, self.__microsecond) - elif self.__second != 0: - s = ", %d" % self.__second + if self._microsecond != 0: + s = ", %d, %d" % (self._second, self._microsecond) + elif self._second != 0: + s = ", %d" % self._second else: s = "" s= "%s(%d, %d%s)" % ('datetime.' + self.__class__.__name__, - self.__hour, self.__minute, s) + self._hour, self._minute, s) if self._tzinfo is not None: assert s[-1:] == ")" s = s[:-1] + ", tzinfo=%r" % self._tzinfo + ")" @@ -1246,8 +1302,8 @@ This is 'HH:MM:SS.mmmmmm+zz:zz', or 'HH:MM:SS+zz:zz' if self.microsecond == 0. """ - s = _format_time(self.__hour, self.__minute, self.__second, - self.__microsecond) + s = _format_time(self._hour, self._minute, self._second, + self._microsecond) tz = self._tzstr() if tz: s += tz @@ -1255,14 +1311,6 @@ __str__ = isoformat - def __format__(self, format): - if not isinstance(format, (str, unicode)): - raise ValueError("__format__ excepts str or unicode, not %s" % - format.__class__.__name__) - if not format: - return str(self) - return self.strftime(format) - def strftime(self, fmt): """Format using strftime(). The date part of the timestamp passed to underlying strftime should not be used. @@ -1270,10 +1318,18 @@ # The year must be >= 1900 else Python's strftime implementation # can raise a bogus exception. timetuple = (1900, 1, 1, - self.__hour, self.__minute, self.__second, + self._hour, self._minute, self._second, 0, 1, -1) return _wrap_strftime(self, fmt, timetuple) + def __format__(self, fmt): + if not isinstance(fmt, (str, unicode)): + raise ValueError("__format__ excepts str or unicode, not %s" % + fmt.__class__.__name__) + if len(fmt) != 0: + return self.strftime(fmt) + return str(self) + # Timezone functions def utcoffset(self): @@ -1350,10 +1406,10 @@ __safe_for_unpickling__ = True # For Python 2.2 - def __getstate(self): - us2, us3 = divmod(self.__microsecond, 256) + def _getstate(self): + us2, us3 = divmod(self._microsecond, 256) us1, us2 = divmod(us2, 256) - basestate = ("%c" * 6) % (self.__hour, self.__minute, self.__second, + basestate = ("%c" * 6) % (self._hour, self._minute, self._second, us1, us2, us3) if self._tzinfo is None: return (basestate,) @@ -1363,13 +1419,13 @@ def __setstate(self, string, tzinfo): if len(string) != 6 or ord(string[0]) >= 24: raise TypeError("an integer is required") - self.__hour, self.__minute, self.__second, us1, us2, us3 = \ + self._hour, self._minute, self._second, us1, us2, us3 = \ map(ord, string) - self.__microsecond = (((us1 << 8) | us2) << 8) | us3 + self._microsecond = (((us1 << 8) | us2) << 8) | us3 self._tzinfo = tzinfo def __reduce__(self): - return (time, self.__getstate()) + return (time, self._getstate()) _time_class = time # so functions w/ args named "time" can get at the class @@ -1378,9 +1434,11 @@ time.resolution = timedelta(microseconds=1) class datetime(date): + """datetime(year, month, day[, hour[, minute[, second[, microsecond[,tzinfo]]]]]) - # XXX needs docstrings - # See http://www.zope.org/Members/fdrake/DateTimeWiki/TimeZoneInfo + The year, month and day arguments are required. tzinfo may be None, or an + instance of a tzinfo subclass. The remaining arguments may be ints or longs. + """ def __new__(cls, year, month=None, day=None, hour=0, minute=0, second=0, microsecond=0, tzinfo=None): @@ -1393,24 +1451,43 @@ _check_time_fields(hour, minute, second, microsecond) self = date.__new__(cls, year, month, day) # XXX This duplicates __year, __month, __day for convenience :-( - self.__year = year - self.__month = month - self.__day = day - self.__hour = hour - self.__minute = minute - self.__second = second - self.__microsecond = microsecond + self._year = year + self._month = month + self._day = day + self._hour = hour + self._minute = minute + self._second = second + self._microsecond = microsecond self._tzinfo = tzinfo return self # Read-only field accessors - hour = property(lambda self: self.__hour, doc="hour (0-23)") - minute = property(lambda self: self.__minute, doc="minute (0-59)") - second = property(lambda self: self.__second, doc="second (0-59)") - microsecond = property(lambda self: self.__microsecond, - doc="microsecond (0-999999)") - tzinfo = property(lambda self: self._tzinfo, doc="timezone info object") + @property + def hour(self): + """hour (0-23)""" + return self._hour + @property + def minute(self): + """minute (0-59)""" + return self._minute + + @property + def second(self): + """second (0-59)""" + return self._second + + @property + def microsecond(self): + """microsecond (0-999999)""" + return self._microsecond + + @property + def tzinfo(self): + """timezone info object""" + return self._tzinfo + + @classmethod def fromtimestamp(cls, t, tz=None): """Construct a datetime from a POSIX timestamp (like time.time()). @@ -1438,37 +1515,42 @@ if tz is not None: result = tz.fromutc(result) return result - fromtimestamp = classmethod(fromtimestamp) + @classmethod def utcfromtimestamp(cls, t): "Construct a UTC datetime from a POSIX timestamp (like time.time())." - if 1 - (t % 1.0) < 0.0000005: - t = float(int(t)) + 1 - if t < 0: - t -= 1 + t, frac = divmod(t, 1.0) + us = round(frac * 1e6) + + # If timestamp is less than one microsecond smaller than a + # full second, us can be rounded up to 1000000. In this case, + # roll over to seconds, otherwise, ValueError is raised + # by the constructor. + if us == 1000000: + t += 1 + us = 0 y, m, d, hh, mm, ss, weekday, jday, dst = _time.gmtime(t) - us = int((t % 1.0) * 1000000) ss = min(ss, 59) # clamp out leap seconds if the platform has them return cls(y, m, d, hh, mm, ss, us) - utcfromtimestamp = classmethod(utcfromtimestamp) # XXX This is supposed to do better than we *can* do by using time.time(), # XXX if the platform supports a more accurate way. The C implementation # XXX uses gettimeofday on platforms that have it, but that isn't # XXX available from Python. So now() may return different results # XXX across the implementations. + @classmethod def now(cls, tz=None): "Construct a datetime from time.time() and optional time zone info." t = _time.time() return cls.fromtimestamp(t, tz) - now = classmethod(now) + @classmethod def utcnow(cls): "Construct a UTC datetime from time.time()." t = _time.time() return cls.utcfromtimestamp(t) - utcnow = classmethod(utcnow) + @classmethod def combine(cls, date, time): "Construct a datetime from a given date and a given time." if not isinstance(date, _date_class): @@ -1478,7 +1560,6 @@ return cls(date.year, date.month, date.day, time.hour, time.minute, time.second, time.microsecond, time.tzinfo) - combine = classmethod(combine) def timetuple(self): "Return local time tuple compatible with time.localtime()." @@ -1504,7 +1585,7 @@ def date(self): "Return the date part." - return date(self.__year, self.__month, self.__day) + return date(self._year, self._month, self._day) def time(self): "Return the time part, with tzinfo None." @@ -1564,8 +1645,8 @@ def ctime(self): "Format a la ctime()." - t = tmxxx(self.__year, self.__month, self.__day, self.__hour, - self.__minute, self.__second) + t = tmxxx(self._year, self._month, self._day, self._hour, + self._minute, self._second) return t.ctime() def isoformat(self, sep='T'): @@ -1580,10 +1661,10 @@ Optional argument sep specifies the separator between date and time, default 'T'. """ - s = ("%04d-%02d-%02d%c" % (self.__year, self.__month, self.__day, + s = ("%04d-%02d-%02d%c" % (self._year, self._month, self._day, sep) + - _format_time(self.__hour, self.__minute, self.__second, - self.__microsecond)) + _format_time(self._hour, self._minute, self._second, + self._microsecond)) off = self._utcoffset() if off is not None: if off < 0: @@ -1596,13 +1677,13 @@ return s def __repr__(self): - "Convert to formal string, for repr()." - L = [self.__year, self.__month, self.__day, # These are never zero - self.__hour, self.__minute, self.__second, self.__microsecond] + """Convert to formal string, for repr().""" + L = [self._year, self._month, self._day, # These are never zero + self._hour, self._minute, self._second, self._microsecond] if L[-1] == 0: del L[-1] if L[-1] == 0: - del L[-1] + del L[-1] s = ", ".join(map(str, L)) s = "%s(%s)" % ('datetime.' + self.__class__.__name__, s) if self._tzinfo is not None: @@ -1675,7 +1756,7 @@ def __eq__(self, other): if isinstance(other, datetime): - return self.__cmp(other) == 0 + return self._cmp(other) == 0 elif hasattr(other, "timetuple") and not isinstance(other, date): return NotImplemented else: @@ -1683,7 +1764,7 @@ def __ne__(self, other): if isinstance(other, datetime): - return self.__cmp(other) != 0 + return self._cmp(other) != 0 elif hasattr(other, "timetuple") and not isinstance(other, date): return NotImplemented else: @@ -1691,7 +1772,7 @@ def __le__(self, other): if isinstance(other, datetime): - return self.__cmp(other) <= 0 + return self._cmp(other) <= 0 elif hasattr(other, "timetuple") and not isinstance(other, date): return NotImplemented else: @@ -1699,7 +1780,7 @@ def __lt__(self, other): if isinstance(other, datetime): - return self.__cmp(other) < 0 + return self._cmp(other) < 0 elif hasattr(other, "timetuple") and not isinstance(other, date): return NotImplemented else: @@ -1707,7 +1788,7 @@ def __ge__(self, other): if isinstance(other, datetime): - return self.__cmp(other) >= 0 + return self._cmp(other) >= 0 elif hasattr(other, "timetuple") and not isinstance(other, date): return NotImplemented else: @@ -1715,13 +1796,13 @@ def __gt__(self, other): if isinstance(other, datetime): - return self.__cmp(other) > 0 + return self._cmp(other) > 0 elif hasattr(other, "timetuple") and not isinstance(other, date): return NotImplemented else: _cmperror(self, other) - def __cmp(self, other): + def _cmp(self, other): assert isinstance(other, datetime) mytz = self._tzinfo ottz = other._tzinfo @@ -1737,12 +1818,12 @@ base_compare = myoff == otoff if base_compare: - return cmp((self.__year, self.__month, self.__day, - self.__hour, self.__minute, self.__second, - self.__microsecond), - (other.__year, other.__month, other.__day, - other.__hour, other.__minute, other.__second, - other.__microsecond)) + return cmp((self._year, self._month, self._day, + self._hour, self._minute, self._second, + self._microsecond), + (other._year, other._month, other._day, + other._hour, other._minute, other._second, + other._microsecond)) if myoff is None or otoff is None: # XXX Buggy in 2.2.2. raise TypeError("cannot compare naive and aware datetimes") @@ -1756,13 +1837,13 @@ "Add a datetime and a timedelta." if not isinstance(other, timedelta): return NotImplemented - t = tmxxx(self.__year, - self.__month, - self.__day + other.days, - self.__hour, - self.__minute, - self.__second + other.seconds, - self.__microsecond + other.microseconds) + t = tmxxx(self._year, + self._month, + self._day + other.days, + self._hour, + self._minute, + self._second + other.seconds, + self._microsecond + other.microseconds) self._checkOverflow(t.year) result = datetime(t.year, t.month, t.day, t.hour, t.minute, t.second, @@ -1780,11 +1861,11 @@ days1 = self.toordinal() days2 = other.toordinal() - secs1 = self.__second + self.__minute * 60 + self.__hour * 3600 - secs2 = other.__second + other.__minute * 60 + other.__hour * 3600 + secs1 = self._second + self._minute * 60 + self._hour * 3600 + secs2 = other._second + other._minute * 60 + other._hour * 3600 base = timedelta(days1 - days2, secs1 - secs2, - self.__microsecond - other.__microsecond) + self._microsecond - other._microsecond) if self._tzinfo is other._tzinfo: return base myoff = self._utcoffset() @@ -1792,13 +1873,13 @@ if myoff == otoff: return base if myoff is None or otoff is None: - raise TypeError, "cannot mix naive and timezone-aware time" + raise TypeError("cannot mix naive and timezone-aware time") return base + timedelta(minutes = otoff-myoff) def __hash__(self): tzoff = self._utcoffset() if tzoff is None: - return hash(self.__getstate()[0]) + return hash(self._getstate()[0]) days = _ymd2ord(self.year, self.month, self.day) seconds = self.hour * 3600 + (self.minute - tzoff) * 60 + self.second return hash(timedelta(days, seconds, self.microsecond)) @@ -1807,12 +1888,12 @@ __safe_for_unpickling__ = True # For Python 2.2 - def __getstate(self): - yhi, ylo = divmod(self.__year, 256) - us2, us3 = divmod(self.__microsecond, 256) + def _getstate(self): + yhi, ylo = divmod(self._year, 256) + us2, us3 = divmod(self._microsecond, 256) us1, us2 = divmod(us2, 256) - basestate = ("%c" * 10) % (yhi, ylo, self.__month, self.__day, - self.__hour, self.__minute, self.__second, + basestate = ("%c" * 10) % (yhi, ylo, self._month, self._day, + self._hour, self._minute, self._second, us1, us2, us3) if self._tzinfo is None: return (basestate,) @@ -1820,14 +1901,14 @@ return (basestate, self._tzinfo) def __setstate(self, string, tzinfo): - (yhi, ylo, self.__month, self.__day, self.__hour, - self.__minute, self.__second, us1, us2, us3) = map(ord, string) - self.__year = yhi * 256 + ylo - self.__microsecond = (((us1 << 8) | us2) << 8) | us3 + (yhi, ylo, self._month, self._day, self._hour, + self._minute, self._second, us1, us2, us3) = map(ord, string) + self._year = yhi * 256 + ylo + self._microsecond = (((us1 << 8) | us2) << 8) | us3 self._tzinfo = tzinfo def __reduce__(self): - return (self.__class__, self.__getstate()) + return (self.__class__, self._getstate()) datetime.min = datetime(1, 1, 1) @@ -2009,7 +2090,7 @@ Because we know z.d said z was in daylight time (else [5] would have held and we would have stopped then), and we know z.d != z'.d (else [8] would have held -and we we have stopped then), and there are only 2 possible values dst() can +and we have stopped then), and there are only 2 possible values dst() can return in Eastern, it follows that z'.d must be 0 (which it is in the example, but the reasoning doesn't depend on the example -- it depends on there being two possible dst() outcomes, one zero and the other non-zero). Therefore diff --git a/lib_pypy/numpypy/__init__.py b/lib_pypy/numpypy/__init__.py --- a/lib_pypy/numpypy/__init__.py +++ b/lib_pypy/numpypy/__init__.py @@ -1,2 +1,2 @@ from _numpypy import * -from .fromnumeric import * +from .core import * diff --git a/lib_pypy/numpypy/core/__init__.py b/lib_pypy/numpypy/core/__init__.py new file mode 100644 --- /dev/null +++ b/lib_pypy/numpypy/core/__init__.py @@ -0,0 +1,1 @@ +from .fromnumeric import * diff --git a/lib_pypy/numpypy/fromnumeric.py b/lib_pypy/numpypy/core/fromnumeric.py rename from lib_pypy/numpypy/fromnumeric.py rename to lib_pypy/numpypy/core/fromnumeric.py --- a/lib_pypy/numpypy/fromnumeric.py +++ b/lib_pypy/numpypy/core/fromnumeric.py @@ -85,7 +85,7 @@ array([4, 3, 6]) """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') # not deprecated --- copy if necessary, view otherwise @@ -149,6 +149,7 @@ [5, 6]]) """ + assert order == 'C' if not hasattr(a, 'reshape'): a = numpypy.array(a) return a.reshape(newshape) @@ -273,7 +274,7 @@ [-1, -2, -3, -4, -5]]]) """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def repeat(a, repeats, axis=None): @@ -315,7 +316,7 @@ [3, 4]]) """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def put(a, ind, v, mode='raise'): @@ -366,7 +367,7 @@ array([ 0, 1, 2, 3, -5]) """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def swapaxes(a, axis1, axis2): @@ -410,7 +411,7 @@ [3, 7]]]) """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def transpose(a, axes=None): @@ -451,8 +452,11 @@ (2, 1, 3) """ - raise NotImplemented('Waiting on interp level method') - + if axes is not None: + raise NotImplementedError('No "axes" arg yet.') + if not hasattr(a, 'T'): + a = numpypy.array(a) + return a.T def sort(a, axis=-1, kind='quicksort', order=None): """ @@ -553,7 +557,7 @@ dtype=[('name', '|S10'), ('height', '<f8'), ('age', '<i4')]) """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def argsort(a, axis=-1, kind='quicksort', order=None): @@ -634,7 +638,7 @@ array([0, 1]) """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def argmax(a, axis=None): @@ -687,6 +691,7 @@ 1 """ + assert axis is None if not hasattr(a, 'argmax'): a = numpypy.array(a) return a.argmax() @@ -702,6 +707,7 @@ documentation. """ + assert axis is None if not hasattr(a, 'argmin'): a = numpypy.array(a) return a.argmin() @@ -753,7 +759,7 @@ array([0, 5, 1, 2]) """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def resize(a, new_shape): @@ -794,7 +800,7 @@ [0, 1, 2, 3]]) """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def squeeze(a): @@ -821,7 +827,7 @@ (3,) """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def diagonal(a, offset=0, axis1=0, axis2=1): @@ -905,7 +911,7 @@ [5, 7]]) """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def trace(a, offset=0, axis1=0, axis2=1, dtype=None, out=None): @@ -964,7 +970,7 @@ (2, 3) """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def ravel(a, order='C'): """ @@ -1048,7 +1054,7 @@ array([ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]) """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def nonzero(a): @@ -1124,7 +1130,7 @@ (array([1, 1, 1, 2, 2, 2]), array([0, 1, 2, 0, 1, 2])) """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def shape(a): @@ -1229,7 +1235,7 @@ array([2]) """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def clip(a, a_min, a_max, out=None): @@ -1282,7 +1288,7 @@ array([3, 4, 2, 3, 4, 5, 6, 7, 8, 8]) """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def sum(a, axis=None, dtype=None, out=None): @@ -1351,9 +1357,11 @@ -128 """ + assert dtype is None + assert out is None if not hasattr(a, "sum"): a = numpypy.array(a) - return a.sum() + return a.sum(axis=axis) def product (a, axis=None, dtype=None, out=None): @@ -1365,7 +1373,7 @@ prod : equivalent function; see for details. """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def sometrue(a, axis=None, out=None): @@ -1379,6 +1387,8 @@ any : equivalent function """ + assert axis is None + assert out is None if not hasattr(a, 'any'): a = numpypy.array(a) return a.any() @@ -1393,6 +1403,8 @@ numpy.all : Equivalent function; see for details. """ + assert axis is None + assert out is None if not hasattr(a, 'all'): a = numpypy.array(a) return a.all() @@ -1461,6 +1473,8 @@ (191614240, 191614240) """ + assert axis is None + assert out is None if not hasattr(a, 'any'): a = numpypy.array(a) return a.any() @@ -1523,6 +1537,8 @@ (28293632, 28293632, array([ True], dtype=bool)) """ + assert axis is None + assert out is None if not hasattr(a, 'all'): a = numpypy.array(a) return a.all() @@ -1590,7 +1606,7 @@ [ 4, 9, 15]]) """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def cumproduct(a, axis=None, dtype=None, out=None): @@ -1603,7 +1619,7 @@ cumprod : equivalent function; see for details. """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def ptp(a, axis=None, out=None): @@ -1644,7 +1660,7 @@ array([1, 1]) """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def amax(a, axis=None, out=None): @@ -1702,6 +1718,8 @@ 4.0 """ + assert axis is None + assert out is None if not hasattr(a, "max"): a = numpypy.array(a) return a.max() @@ -1763,6 +1781,8 @@ """ # amin() is equivalent to min() + assert axis is None + assert out is None if not hasattr(a, 'min'): a = numpypy.array(a) return a.min() @@ -1873,7 +1893,7 @@ True """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def cumprod(a, axis=None, dtype=None, out=None): @@ -1936,7 +1956,7 @@ [ 4, 20, 120]]) """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def ndim(a): @@ -2054,7 +2074,7 @@ 2 """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def around(a, decimals=0, out=None): @@ -2123,7 +2143,7 @@ array([ 0, 0, 0, 10]) """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def round_(a, decimals=0, out=None): @@ -2137,7 +2157,7 @@ around : equivalent function """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def mean(a, axis=None, dtype=None, out=None): @@ -2211,9 +2231,11 @@ 0.55000000074505806 """ + assert dtype is None + assert out is None if not hasattr(a, "mean"): a = numpypy.array(a) - return a.mean() + return a.mean(axis=axis) def std(a, axis=None, dtype=None, out=None, ddof=0): @@ -2302,6 +2324,10 @@ 0.44999999925552653 """ + assert axis is None + assert dtype is None + assert out is None + assert ddof == 0 if not hasattr(a, "std"): a = numpypy.array(a) return a.std() @@ -2395,6 +2421,10 @@ 0.20250000000000001 """ + assert axis is None + assert dtype is None + assert out is None + assert ddof == 0 if not hasattr(a, "var"): a = numpypy.array(a) return a.var() diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py --- a/pypy/config/pypyoption.py +++ b/pypy/config/pypyoption.py @@ -371,6 +371,7 @@ config.objspace.std.suggest(newshortcut=True) config.objspace.std.suggest(withidentitydict=True) config.objspace.std.suggest(withspecialisedtuple=True) + config.objspace.std.suggest(withidentitydict=True) #if not IS_64_BITS: # config.objspace.std.suggest(withsmalllong=True) diff --git a/pypy/doc/getting-started.rst b/pypy/doc/getting-started.rst --- a/pypy/doc/getting-started.rst +++ b/pypy/doc/getting-started.rst @@ -53,11 +53,11 @@ PyPy is ready to be executed as soon as you unpack the tarball or the zip file, with no need to install it in any specific location:: - $ tar xf pypy-1.6-linux.tar.bz2 + $ tar xf pypy-1.7-linux.tar.bz2 - $ ./pypy-1.6/bin/pypy + $ ./pypy-1.7/bin/pypy Python 2.7.1 (?, Apr 27 2011, 12:44:21) - [PyPy 1.6.0 with GCC 4.4.3] on linux2 + [PyPy 1.7.0 with GCC 4.4.3] on linux2 Type "help", "copyright", "credits" or "license" for more information. And now for something completely different: ``implementing LOGO in LOGO: "turtles all the way down"'' @@ -75,14 +75,14 @@ $ curl -O https://raw.github.com/pypa/pip/master/contrib/get-pip.py - $ ./pypy-1.6/bin/pypy distribute_setup.py + $ ./pypy-1.7/bin/pypy distribute_setup.py - $ ./pypy-1.6/bin/pypy get-pip.py + $ ./pypy-1.7/bin/pypy get-pip.py - $ ./pypy-1.6/bin/pip install pygments # for example + $ ./pypy-1.7/bin/pip install pygments # for example -3rd party libraries will be installed in ``pypy-1.6/site-packages``, and -the scripts in ``pypy-1.6/bin``. +3rd party libraries will be installed in ``pypy-1.7/site-packages``, and +the scripts in ``pypy-1.7/bin``. Installing using virtualenv --------------------------- diff --git a/pypy/doc/translation.rst b/pypy/doc/translation.rst --- a/pypy/doc/translation.rst +++ b/pypy/doc/translation.rst @@ -155,7 +155,7 @@ function. The two input variables are the exception class and the exception value, respectively. (No other block will actually link to the exceptblock if the function does not - explicitely raise exceptions.) + explicitly raise exceptions.) ``Block`` @@ -325,7 +325,7 @@ Mutable objects need special treatment during annotation, because the annotation of contained values needs to be possibly updated to account for mutation operations, and consequently the annotation information -reflown through the relevant parts of the flow the graphs. +reflown through the relevant parts of the flow graphs. * ``SomeList`` stands for a list of homogeneous type (i.e. all the elements of the list are represented by a single common ``SomeXxx`` @@ -503,8 +503,8 @@ Since RPython is a garbage collected language there is a lot of heap memory allocation going on all the time, which would either not occur at all in a more -traditional explicitely managed language or results in an object which dies at -a time known in advance and can thus be explicitely deallocated. For example a +traditional explicitly managed language or results in an object which dies at +a time known in advance and can thus be explicitly deallocated. For example a loop of the following form:: for i in range(n): @@ -696,7 +696,7 @@ So far it is the second most mature high level backend after GenCLI: it still can't translate the full Standard Interpreter, but after the -Leysin sprint we were able to compile and run the rpytstone and +Leysin sprint we were able to compile and run the rpystone and richards benchmarks. GenJVM is almost entirely the work of Niko Matsakis, who worked on it diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py --- a/pypy/interpreter/executioncontext.py +++ b/pypy/interpreter/executioncontext.py @@ -445,6 +445,7 @@ AsyncAction.__init__(self, space) self.dying_objects = [] self.finalizers_lock_count = 0 + self.enabled_at_app_level = True def register_callback(self, w_obj, callback, descrname): self.dying_objects.append((w_obj, callback, descrname)) diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py --- a/pypy/jit/backend/test/runner_test.py +++ b/pypy/jit/backend/test/runner_test.py @@ -551,6 +551,28 @@ res = self.execute_operation(rop.CALL, [funcbox] + map(BoxInt, args), 'int', descr=calldescr) assert res.value == func(*args) + def test_call_box_func(self): + def a(a1, a2): + return a1 + a2 + def b(b1, b2): + return b1 * b2 + + arg1 = 40 + arg2 = 2 + for f in [a, b]: + TP = lltype.Signed + FPTR = self.Ptr(self.FuncType([TP, TP], TP)) + func_ptr = llhelper(FPTR, f) + FUNC = deref(FPTR) + funcconst = self.get_funcbox(self.cpu, func_ptr) + funcbox = funcconst.clonebox() + calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, + EffectInfo.MOST_GENERAL) + res = self.execute_operation(rop.CALL, + [funcbox, BoxInt(arg1), BoxInt(arg2)], + 'int', descr=calldescr) + assert res.getint() == f(arg1, arg2) + def test_call_stack_alignment(self): # test stack alignment issues, notably for Mac OS/X. # also test the ordering of the arguments. @@ -1872,6 +1894,7 @@ values.append(descr) values.append(self.cpu.get_latest_value_int(0)) values.append(self.cpu.get_latest_value_int(1)) + values.append(token) FUNC = self.FuncType([lltype.Signed, lltype.Signed], lltype.Void) func_ptr = llhelper(lltype.Ptr(FUNC), maybe_force) @@ -1902,7 +1925,8 @@ assert fail.identifier == 1 assert self.cpu.get_latest_value_int(0) == 1 assert self.cpu.get_latest_value_int(1) == 10 - assert values == [faildescr, 1, 10] + token = self.cpu.get_latest_force_token() + assert values == [faildescr, 1, 10, token] def test_force_operations_returning_int(self): values = [] @@ -1911,6 +1935,7 @@ self.cpu.force(token) values.append(self.cpu.get_latest_value_int(0)) values.append(self.cpu.get_latest_value_int(2)) + values.append(token) return 42 FUNC = self.FuncType([lltype.Signed, lltype.Signed], lltype.Signed) @@ -1944,7 +1969,8 @@ assert self.cpu.get_latest_value_int(0) == 1 assert self.cpu.get_latest_value_int(1) == 42 assert self.cpu.get_latest_value_int(2) == 10 - assert values == [1, 10] + token = self.cpu.get_latest_force_token() + assert values == [1, 10, token] def test_force_operations_returning_float(self): values = [] @@ -1953,6 +1979,7 @@ self.cpu.force(token) values.append(self.cpu.get_latest_value_int(0)) values.append(self.cpu.get_latest_value_int(2)) + values.append(token) return 42.5 FUNC = self.FuncType([lltype.Signed, lltype.Signed], lltype.Float) @@ -1988,7 +2015,8 @@ x = self.cpu.get_latest_value_float(1) assert longlong.getrealfloat(x) == 42.5 assert self.cpu.get_latest_value_int(2) == 10 - assert values == [1, 10] + token = self.cpu.get_latest_force_token() + assert values == [1, 10, token] def test_call_to_c_function(self): from pypy.rlib.libffi import CDLL, types, ArgChain, FUNCFLAG_CDECL diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py --- a/pypy/jit/backend/x86/assembler.py +++ b/pypy/jit/backend/x86/assembler.py @@ -1118,6 +1118,12 @@ for src, dst in singlefloats: self.mc.MOVD(dst, src) # Finally remap the arguments in the main regs + # If x is a register and is in dst_locs, then oups, it needs to + # be moved away: + if x in dst_locs: + src_locs.append(x) + dst_locs.append(r10) + x = r10 remap_frame_layout(self, src_locs, dst_locs, X86_64_SCRATCH_REG) self._regalloc.reserve_param(len(pass_on_stack)) @@ -2042,10 +2048,7 @@ size = sizeloc.value signloc = arglocs[1] - if isinstance(op.getarg(0), Const): - x = imm(op.getarg(0).getint()) - else: - x = arglocs[2] + x = arglocs[2] # the function address if x is eax: tmp = ecx else: diff --git a/pypy/jit/backend/x86/runner.py b/pypy/jit/backend/x86/runner.py --- a/pypy/jit/backend/x86/runner.py +++ b/pypy/jit/backend/x86/runner.py @@ -6,7 +6,7 @@ from pypy.jit.codewriter import longlong from pypy.jit.metainterp import history, compile from pypy.jit.backend.x86.assembler import Assembler386 -from pypy.jit.backend.x86.arch import FORCE_INDEX_OFS +from pypy.jit.backend.x86.arch import FORCE_INDEX_OFS, IS_X86_32 from pypy.jit.backend.x86.profagent import ProfileAgent from pypy.jit.backend.llsupport.llmodel import AbstractLLCPU from pypy.jit.backend.x86 import regloc @@ -142,7 +142,9 @@ cast_ptr_to_int._annspecialcase_ = 'specialize:arglltype(0)' cast_ptr_to_int = staticmethod(cast_ptr_to_int) - all_null_registers = lltype.malloc(rffi.LONGP.TO, 24, + all_null_registers = lltype.malloc(rffi.LONGP.TO, + IS_X86_32 and (16+8) # 16 + 8 regs + or (16+16), # 16 + 16 regs flavor='raw', zero=True, immortal=True) diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py --- a/pypy/module/_codecs/interp_codecs.py +++ b/pypy/module/_codecs/interp_codecs.py @@ -108,6 +108,10 @@ w_result = state.codec_search_cache.get(normalized_encoding, None) if w_result is not None: return w_result + return _lookup_codec_loop(space, encoding, normalized_encoding) + +def _lookup_codec_loop(space, encoding, normalized_encoding): + state = space.fromcache(CodecState) if state.codec_need_encodings: w_import = space.getattr(space.builtin, space.wrap("__import__")) # registers new codecs diff --git a/pypy/module/_codecs/test/test_ztranslation.py b/pypy/module/_codecs/test/test_ztranslation.py new file mode 100644 --- /dev/null +++ b/pypy/module/_codecs/test/test_ztranslation.py @@ -0,0 +1,5 @@ +from pypy.objspace.fake.checkmodule import checkmodule + + +def test__codecs_translates(): + checkmodule('_codecs') diff --git a/pypy/module/_hashlib/interp_hashlib.py b/pypy/module/_hashlib/interp_hashlib.py --- a/pypy/module/_hashlib/interp_hashlib.py +++ b/pypy/module/_hashlib/interp_hashlib.py @@ -34,8 +34,12 @@ ctx = lltype.malloc(ropenssl.EVP_MD_CTX.TO, flavor='raw') rgc.add_memory_pressure(HASH_MALLOC_SIZE + self.digest_size) - ropenssl.EVP_DigestInit(ctx, digest_type) - self.ctx = ctx + try: + ropenssl.EVP_DigestInit(ctx, digest_type) + self.ctx = ctx + except: + lltype.free(ctx, flavor='raw') + raise def __del__(self): # self.lock.free() diff --git a/pypy/module/_io/interp_fileio.py b/pypy/module/_io/interp_fileio.py --- a/pypy/module/_io/interp_fileio.py +++ b/pypy/module/_io/interp_fileio.py @@ -349,6 +349,8 @@ try: s = os.read(self.fd, size) except OSError, e: + if e.errno == errno.EAGAIN: + return space.w_None raise wrap_oserror(space, e, exception_name='w_IOError') @@ -362,6 +364,8 @@ try: buf = os.read(self.fd, length) except OSError, e: + if e.errno == errno.EAGAIN: + return space.w_None raise wrap_oserror(space, e, exception_name='w_IOError') rwbuffer.setslice(0, buf) diff --git a/pypy/module/_io/test/test_fileio.py b/pypy/module/_io/test/test_fileio.py --- a/pypy/module/_io/test/test_fileio.py +++ b/pypy/module/_io/test/test_fileio.py @@ -133,6 +133,19 @@ f.close() assert a == 'a\nbxxxxxxx' + def test_nonblocking_read(self): + import os, fcntl + r_fd, w_fd = os.pipe() + # set nonblocking + fcntl.fcntl(r_fd, fcntl.F_SETFL, os.O_NONBLOCK) + import _io + f = _io.FileIO(r_fd, 'r') + # Read from stream sould return None + assert f.read() is None + assert f.read(10) is None + a = bytearray('x' * 10) + assert f.readinto(a) is None + def test_repr(self): import _io f = _io.FileIO(self.tmpfile, 'r') diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py --- a/pypy/module/_socket/interp_socket.py +++ b/pypy/module/_socket/interp_socket.py @@ -67,9 +67,6 @@ self.connect(self.addr_from_object(space, w_addr)) except SocketError, e: raise converted_error(space, e) - except TypeError, e: - raise OperationError(space.w_TypeError, - space.wrap(str(e))) def connect_ex_w(self, space, w_addr): """connect_ex(address) -> errno diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py --- a/pypy/module/_socket/test/test_sock_app.py +++ b/pypy/module/_socket/test/test_sock_app.py @@ -529,26 +529,31 @@ import _socket, os if not hasattr(_socket, 'AF_UNIX'): skip('AF_UNIX not supported.') - sockpath = os.path.join(self.udir, 'app_test_unix_socket_connect') + oldcwd = os.getcwd() + os.chdir(self.udir) + try: + sockpath = 'app_test_unix_socket_connect' - serversock = _socket.socket(_socket.AF_UNIX) - serversock.bind(sockpath) - serversock.listen(1) + serversock = _socket.socket(_socket.AF_UNIX) + serversock.bind(sockpath) + serversock.listen(1) - clientsock = _socket.socket(_socket.AF_UNIX) - clientsock.connect(sockpath) - s, addr = serversock.accept() - assert not addr + clientsock = _socket.socket(_socket.AF_UNIX) + clientsock.connect(sockpath) + s, addr = serversock.accept() + assert not addr - s.send('X') - data = clientsock.recv(100) - assert data == 'X' - clientsock.send('Y') - data = s.recv(100) - assert data == 'Y' + s.send('X') + data = clientsock.recv(100) + assert data == 'X' + clientsock.send('Y') + data = s.recv(100) + assert data == 'Y' - clientsock.close() - s.close() + clientsock.close() + s.close() + finally: + os.chdir(oldcwd) class AppTestSocketTCP: diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py --- a/pypy/module/_ssl/test/test_ssl.py +++ b/pypy/module/_ssl/test/test_ssl.py @@ -64,8 +64,8 @@ def test_sslwrap(self): import _ssl, _socket, sys, gc - if sys.platform == 'darwin': - skip("hangs indefinitely on OSX (also on CPython)") + if sys.platform == 'darwin' or 'freebsd' in sys.platform: + skip("hangs indefinitely on OSX & FreeBSD (also on CPython)") s = _socket.socket() ss = _ssl.sslwrap(s, 0) exc = raises(_socket.error, ss.do_handshake) diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h --- a/pypy/module/cpyext/include/patchlevel.h +++ b/pypy/module/cpyext/include/patchlevel.h @@ -29,7 +29,7 @@ #define PY_VERSION "2.7.1" /* PyPy version as a string */ -#define PYPY_VERSION "1.7.1" +#define PYPY_VERSION "1.8.1" /* Subversion Revision number of this file (not of the repository). * Empty since Mercurial migration. */ diff --git a/pypy/module/fcntl/test/test_fcntl.py b/pypy/module/fcntl/test/test_fcntl.py --- a/pypy/module/fcntl/test/test_fcntl.py +++ b/pypy/module/fcntl/test/test_fcntl.py @@ -42,13 +42,9 @@ else: start_len = "qq" - if sys.platform in ('netbsd1', 'netbsd2', 'netbsd3', - 'Darwin1.2', 'darwin', - 'freebsd2', 'freebsd3', 'freebsd4', 'freebsd5', - 'freebsd6', 'freebsd7', 'freebsd8', 'freebsd9', - 'bsdos2', 'bsdos3', 'bsdos4', - 'openbsd', 'openbsd2', 'openbsd3', 'openbsd4', - 'openbsd5'): + if any(substring in sys.platform.lower() + for substring in ('netbsd', 'darwin', 'freebsd', 'bsdos', + 'openbsd')): if struct.calcsize('l') == 8: off_t = 'l' pid_t = 'i' @@ -182,7 +178,8 @@ def test_large_flag(self): import sys - if sys.platform == "darwin" or sys.platform.startswith("openbsd"): + if any(plat in sys.platform + for plat in ('darwin', 'openbsd', 'freebsd')): skip("Mac OS doesn't have any large flag in fcntl.h") import fcntl, sys if sys.maxint == 2147483647: diff --git a/pypy/module/gc/__init__.py b/pypy/module/gc/__init__.py --- a/pypy/module/gc/__init__.py +++ b/pypy/module/gc/__init__.py @@ -1,18 +1,18 @@ from pypy.interpreter.mixedmodule import MixedModule class Module(MixedModule): - appleveldefs = { - 'enable': 'app_gc.enable', - 'disable': 'app_gc.disable', - 'isenabled': 'app_gc.isenabled', - } interpleveldefs = { 'collect': 'interp_gc.collect', + 'enable': 'interp_gc.enable', + 'disable': 'interp_gc.disable', + 'isenabled': 'interp_gc.isenabled', 'enable_finalizers': 'interp_gc.enable_finalizers', 'disable_finalizers': 'interp_gc.disable_finalizers', 'garbage' : 'space.newlist([])', #'dump_heap_stats': 'interp_gc.dump_heap_stats', } + appleveldefs = { + } def __init__(self, space, w_name): if (not space.config.translating or diff --git a/pypy/module/gc/app_gc.py b/pypy/module/gc/app_gc.py deleted file mode 100644 --- a/pypy/module/gc/app_gc.py +++ /dev/null @@ -1,21 +0,0 @@ -# NOT_RPYTHON - -enabled = True - -def isenabled(): - global enabled - return enabled - -def enable(): - global enabled - import gc - if not enabled: - gc.enable_finalizers() - enabled = True - -def disable(): - global enabled - import gc - if enabled: - gc.disable_finalizers() - enabled = False diff --git a/pypy/module/gc/interp_gc.py b/pypy/module/gc/interp_gc.py --- a/pypy/module/gc/interp_gc.py +++ b/pypy/module/gc/interp_gc.py @@ -17,6 +17,26 @@ rgc.collect() return space.wrap(0) +def enable(space): + """Non-recursive version. Enable finalizers now. + If they were already enabled, no-op. + If they were disabled even several times, enable them anyway. + """ + if not space.user_del_action.enabled_at_app_level: + space.user_del_action.enabled_at_app_level = True + enable_finalizers(space) + +def disable(space): + """Non-recursive version. Disable finalizers now. Several calls + to this function are ignored. + """ + if space.user_del_action.enabled_at_app_level: + space.user_del_action.enabled_at_app_level = False + disable_finalizers(space) + +def isenabled(space): + return space.newbool(space.user_del_action.enabled_at_app_level) + def enable_finalizers(space): if space.user_del_action.finalizers_lock_count == 0: raise OperationError(space.w_ValueError, diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py --- a/pypy/module/micronumpy/__init__.py +++ b/pypy/module/micronumpy/__init__.py @@ -27,6 +27,7 @@ 'dot': 'interp_numarray.dot', 'fromstring': 'interp_support.fromstring', 'flatiter': 'interp_numarray.W_FlatIterator', + 'isna': 'interp_numarray.isna', 'True_': 'types.Bool.True', 'False_': 'types.Bool.False', @@ -70,6 +71,7 @@ ("exp", "exp"), ("fabs", "fabs"), ("floor", "floor"), + ("ceil", "ceil"), ("greater", "greater"), ("greater_equal", "greater_equal"), ("less", "less"), @@ -85,12 +87,13 @@ ("subtract", "subtract"), ('sqrt', 'sqrt'), ("tan", "tan"), + ('bitwise_and', 'bitwise_and'), + ('bitwise_or', 'bitwise_or'), ]: interpleveldefs[exposed] = "interp_ufuncs.get(space).%s" % impl appleveldefs = { 'average': 'app_numpy.average', - 'mean': 'app_numpy.mean', 'sum': 'app_numpy.sum', 'min': 'app_numpy.min', 'identity': 'app_numpy.identity', @@ -99,5 +102,4 @@ 'e': 'app_numpy.e', 'pi': 'app_numpy.pi', 'arange': 'app_numpy.arange', - 'reshape': 'app_numpy.reshape', } diff --git a/pypy/module/micronumpy/app_numpy.py b/pypy/module/micronumpy/app_numpy.py --- a/pypy/module/micronumpy/app_numpy.py +++ b/pypy/module/micronumpy/app_numpy.py @@ -11,23 +11,20 @@ def average(a): # This implements a weighted average, for now we don't implement the # weighting, just the average part! - return mean(a) + if not hasattr(a, "mean"): + a = _numpypy.array(a) + return a.mean() def identity(n, dtype=None): - a = _numpypy.zeros((n,n), dtype=dtype) + a = _numpypy.zeros((n, n), dtype=dtype) for i in range(n): a[i][i] = 1 return a -def mean(a, axis=None): - if not hasattr(a, "mean"): - a = _numpypy.array(a) - return a.mean(axis) - def sum(a,axis=None): '''sum(a, axis=None) Sum of array elements over a given axis. - + Parameters ---------- a : array_like @@ -35,7 +32,7 @@ axis : integer, optional Axis over which the sum is taken. By default `axis` is None, and all elements are summed. - + Returns ------- sum_along_axis : ndarray @@ -43,7 +40,7 @@ axis removed. If `a` is a 0-d array, or if `axis` is None, a scalar is returned. If an output array is specified, a reference to `out` is returned. - + See Also -------- ndarray.sum : Equivalent method. @@ -79,40 +76,3 @@ arr[j] = i i += step return arr - - -def reshape(a, shape): - '''reshape(a, newshape) - Gives a new shape to an array without changing its data. - - Parameters - ---------- - a : array_like - Array to be reshaped. - newshape : int or tuple of ints - The new shape should be compatible with the original shape. If - an integer, then the result will be a 1-D array of that length. - One shape dimension can be -1. In this case, the value is inferred - from the length of the array and remaining dimensions. - - Returns - ------- - reshaped_array : ndarray - This will be a new view object if possible; otherwise, it will - be a copy. - - - See Also - -------- - ndarray.reshape : Equivalent method. - - Notes - ----- - - It is not always possible to change the shape of an array without - copying the data. If you want an error to be raise if the data is copied, - you should assign the new shape to the shape attribute of the array -''' - if not hasattr(a, 'reshape'): - a = _numpypy.array(a) - return a.reshape(shape) diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py --- a/pypy/module/micronumpy/interp_dtype.py +++ b/pypy/module/micronumpy/interp_dtype.py @@ -20,7 +20,7 @@ class W_Dtype(Wrappable): _immutable_fields_ = ["itemtype", "num", "kind"] - def __init__(self, itemtype, num, kind, name, char, w_box_type, alternate_constructors=[]): + def __init__(self, itemtype, num, kind, name, char, w_box_type, alternate_constructors=[], aliases=[]): self.itemtype = itemtype self.num = num self.kind = kind @@ -28,6 +28,7 @@ self.char = char self.w_box_type = w_box_type self.alternate_constructors = alternate_constructors + self.aliases = aliases def malloc(self, length): # XXX find out why test_zjit explodes with tracking of allocations @@ -46,6 +47,10 @@ def getitem(self, storage, i): return self.itemtype.read(storage, self.itemtype.get_element_size(), i, 0) + def getitem_bool(self, storage, i): + isize = self.itemtype.get_element_size() + return self.itemtype.read_bool(storage, isize, i, 0) + def setitem(self, storage, i, box): self.itemtype.store(storage, self.itemtype.get_element_size(), i, 0, box) @@ -62,7 +67,7 @@ elif space.isinstance_w(w_dtype, space.w_str): name = space.str_w(w_dtype) for dtype in cache.builtin_dtypes: - if dtype.name == name or dtype.char == name: + if dtype.name == name or dtype.char == name or name in dtype.aliases: return dtype else: for dtype in cache.builtin_dtypes: @@ -84,6 +89,12 @@ def descr_get_shape(self, space): return space.newtuple([]) + def is_int_type(self): + return self.kind == SIGNEDLTR or self.kind == UNSIGNEDLTR + + def is_bool_type(self): + return self.kind == BOOLLTR + W_Dtype.typedef = TypeDef("dtype", __module__ = "numpypy", __new__ = interp2app(W_Dtype.descr__new__.im_func), @@ -107,7 +118,7 @@ kind=BOOLLTR, name="bool", char="?", - w_box_type = space.gettypefor(interp_boxes.W_BoolBox), + w_box_type=space.gettypefor(interp_boxes.W_BoolBox), alternate_constructors=[space.w_bool], ) self.w_int8dtype = W_Dtype( @@ -116,7 +127,7 @@ kind=SIGNEDLTR, name="int8", char="b", - w_box_type = space.gettypefor(interp_boxes.W_Int8Box) + w_box_type=space.gettypefor(interp_boxes.W_Int8Box) ) self.w_uint8dtype = W_Dtype( types.UInt8(), @@ -124,7 +135,7 @@ kind=UNSIGNEDLTR, name="uint8", char="B", - w_box_type = space.gettypefor(interp_boxes.W_UInt8Box), + w_box_type=space.gettypefor(interp_boxes.W_UInt8Box), ) self.w_int16dtype = W_Dtype( types.Int16(), @@ -132,7 +143,7 @@ kind=SIGNEDLTR, name="int16", char="h", - w_box_type = space.gettypefor(interp_boxes.W_Int16Box), + w_box_type=space.gettypefor(interp_boxes.W_Int16Box), ) self.w_uint16dtype = W_Dtype( types.UInt16(), @@ -140,7 +151,7 @@ kind=UNSIGNEDLTR, name="uint16", char="H", - w_box_type = space.gettypefor(interp_boxes.W_UInt16Box), + w_box_type=space.gettypefor(interp_boxes.W_UInt16Box), ) self.w_int32dtype = W_Dtype( types.Int32(), @@ -148,7 +159,7 @@ kind=SIGNEDLTR, name="int32", char="i", - w_box_type = space.gettypefor(interp_boxes.W_Int32Box), + w_box_type=space.gettypefor(interp_boxes.W_Int32Box), ) self.w_uint32dtype = W_Dtype( types.UInt32(), @@ -156,7 +167,7 @@ kind=UNSIGNEDLTR, name="uint32", char="I", - w_box_type = space.gettypefor(interp_boxes.W_UInt32Box), + w_box_type=space.gettypefor(interp_boxes.W_UInt32Box), ) if LONG_BIT == 32: name = "int32" @@ -168,7 +179,7 @@ kind=SIGNEDLTR, name=name, char="l", - w_box_type = space.gettypefor(interp_boxes.W_LongBox), + w_box_type=space.gettypefor(interp_boxes.W_LongBox), alternate_constructors=[space.w_int], ) self.w_ulongdtype = W_Dtype( @@ -177,7 +188,7 @@ kind=UNSIGNEDLTR, name="u" + name, char="L", - w_box_type = space.gettypefor(interp_boxes.W_ULongBox), + w_box_type=space.gettypefor(interp_boxes.W_ULongBox), ) self.w_int64dtype = W_Dtype( types.Int64(), @@ -185,7 +196,7 @@ kind=SIGNEDLTR, name="int64", char="q", - w_box_type = space.gettypefor(interp_boxes.W_Int64Box), + w_box_type=space.gettypefor(interp_boxes.W_Int64Box), alternate_constructors=[space.w_long], ) self.w_uint64dtype = W_Dtype( @@ -194,7 +205,7 @@ kind=UNSIGNEDLTR, name="uint64", char="Q", - w_box_type = space.gettypefor(interp_boxes.W_UInt64Box), + w_box_type=space.gettypefor(interp_boxes.W_UInt64Box), ) self.w_float32dtype = W_Dtype( types.Float32(), @@ -202,7 +213,7 @@ kind=FLOATINGLTR, name="float32", char="f", - w_box_type = space.gettypefor(interp_boxes.W_Float32Box), + w_box_type=space.gettypefor(interp_boxes.W_Float32Box), ) self.w_float64dtype = W_Dtype( types.Float64(), @@ -212,6 +223,7 @@ char="d", w_box_type = space.gettypefor(interp_boxes.W_Float64Box), alternate_constructors=[space.w_float], + aliases=["float"], ) self.builtin_dtypes = [ diff --git a/pypy/module/micronumpy/interp_iter.py b/pypy/module/micronumpy/interp_iter.py --- a/pypy/module/micronumpy/interp_iter.py +++ b/pypy/module/micronumpy/interp_iter.py @@ -4,6 +4,19 @@ from pypy.module.micronumpy.strides import calculate_broadcast_strides,\ calculate_slice_strides +# structures to describe slicing + +class Chunk(object): + def __init__(self, start, stop, step, lgt): + self.start = start + self.stop = stop + self.step = step + self.lgt = lgt + + def extend_shape(self, shape): + if self.step != 0: + shape.append(self.lgt) + class BaseTransform(object): pass @@ -38,11 +51,18 @@ self.size = size def next(self, shapelen): + return self._next(1) + + def _next(self, ofs): arr = instantiate(ArrayIterator) arr.size = self.size - arr.offset = self.offset + 1 + arr.offset = self.offset + ofs return arr + def next_no_increase(self, shapelen): + # a hack to make JIT believe this is always virtual + return self._next(0) + def done(self): return self.offset >= self.size @@ -157,6 +177,8 @@ offset += self.strides[i] break else: + if i == self.dim: + first_line = True indices[i] = 0 offset -= self.backstrides[i] else: diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -2,14 +2,15 @@ from pypy.interpreter.error import OperationError, operationerrfmt from pypy.interpreter.gateway import interp2app, NoneNotWrapped from pypy.interpreter.typedef import TypeDef, GetSetProperty -from pypy.module.micronumpy import interp_ufuncs, interp_dtype, signature +from pypy.module.micronumpy import interp_ufuncs, interp_dtype, signature,\ + interp_boxes from pypy.module.micronumpy.strides import calculate_slice_strides from pypy.rlib import jit from pypy.rpython.lltypesystem import lltype, rffi from pypy.tool.sourcetools import func_with_new_name from pypy.rlib.rstring import StringBuilder from pypy.module.micronumpy.interp_iter import ArrayIterator, OneDimIterator,\ - SkipLastAxisIterator + SkipLastAxisIterator, Chunk, ViewIterator numpy_driver = jit.JitDriver( greens=['shapelen', 'sig'], @@ -39,7 +40,24 @@ get_printable_location=signature.new_printable_location('slice'), name='numpy_slice', ) - +count_driver = jit.JitDriver( + greens=['shapelen'], + virtualizables=['frame'], + reds=['s', 'frame', 'iter', 'arr'], + name='numpy_count' +) +filter_driver = jit.JitDriver( + greens=['shapelen', 'sig'], + virtualizables=['frame'], + reds=['concr', 'argi', 'ri', 'frame', 'v', 'res', 'self'], + name='numpy_filter', +) +filter_set_driver = jit.JitDriver( + greens=['shapelen', 'sig'], + virtualizables=['frame'], + reds=['idx', 'idxi', 'frame', 'arr'], + name='numpy_filterset', +) def _find_shape_and_elems(space, w_iterable): shape = [space.len_w(w_iterable)] @@ -156,15 +174,17 @@ # fits the new shape, using those steps. If there is a shape/step mismatch # (meaning that the realignment of elements crosses from one step into another) # return None so that the caller can raise an exception. -def calc_new_strides(new_shape, old_shape, old_strides): +def calc_new_strides(new_shape, old_shape, old_strides, order): + # Return the proper strides for new_shape, or None if the mapping crosses + # stepping boundaries + # Assumes that prod(old_shape) == prod(new_shape), len(old_shape) > 1, and # len(new_shape) > 0 steps = [] last_step = 1 oldI = 0 new_strides = [] - if old_strides[0] < old_strides[-1]: - #Start at old_shape[0], old_stides[0] + if order == 'F': for i in range(len(old_shape)): steps.append(old_strides[i] / last_step) last_step *= old_shape[i] @@ -181,12 +201,10 @@ n_old_elems_to_use *= old_shape[oldI] if n_new_elems_used == n_old_elems_to_use: oldI += 1 - if oldI >= len(old_shape): - continue - cur_step = steps[oldI] - n_old_elems_to_use *= old_shape[oldI] - else: - #Start at old_shape[-1], old_strides[-1] + if oldI < len(old_shape): + cur_step = steps[oldI] + n_old_elems_to_use *= old_shape[oldI] + elif order == 'C': for i in range(len(old_shape) - 1, -1, -1): steps.insert(0, old_strides[i] / last_step) last_step *= old_shape[i] @@ -205,10 +223,10 @@ n_old_elems_to_use *= old_shape[oldI] if n_new_elems_used == n_old_elems_to_use: oldI -= 1 - if oldI < -len(old_shape): - continue - cur_step = steps[oldI] - n_old_elems_to_use *= old_shape[oldI] + if oldI >= -len(old_shape): + cur_step = steps[oldI] + n_old_elems_to_use *= old_shape[oldI] + assert len(new_strides) == len(new_shape) return new_strides class BaseArray(Wrappable): @@ -270,6 +288,9 @@ descr_gt = _binop_impl("greater") descr_ge = _binop_impl("greater_equal") + descr_and = _binop_impl("bitwise_and") + descr_or = _binop_impl("bitwise_or") + def _binop_right_impl(ufunc_name): def impl(self, space, w_other): w_other = scalar_w(space, @@ -409,9 +430,22 @@ def descr_copy(self, space): return self.copy(space) + def descr_flatten(self, space): + return self.flatten(space) + def copy(self, space): return self.get_concrete().copy(space) + def empty_copy(self, space, dtype): + shape = self.shape + size = 1 + for elem in shape: + size *= elem + return W_NDimArray(size, shape[:], dtype, 'C') + + def flatten(self, space): + return self.get_concrete().flatten(space) + def descr_len(self, space): if len(self.shape): return space.wrap(self.shape[0]) @@ -479,11 +513,69 @@ def _prepare_slice_args(self, space, w_idx): if (space.isinstance_w(w_idx, space.w_int) or space.isinstance_w(w_idx, space.w_slice)): - return [space.decode_index4(w_idx, self.shape[0])] - return [space.decode_index4(w_item, self.shape[i]) for i, w_item in + return [Chunk(*space.decode_index4(w_idx, self.shape[0]))] + return [Chunk(*space.decode_index4(w_item, self.shape[i])) for i, w_item in enumerate(space.fixedview(w_idx))] + def count_all_true(self, arr): + sig = arr.find_sig() + frame = sig.create_frame(self) + shapelen = len(arr.shape) + s = 0 + iter = None + while not frame.done(): + count_driver.jit_merge_point(arr=arr, frame=frame, iter=iter, s=s, + shapelen=shapelen) + iter = frame.get_final_iter() + s += arr.dtype.getitem_bool(arr.storage, iter.offset) + frame.next(shapelen) + return s + + def getitem_filter(self, space, arr): + concr = arr.get_concrete() + size = self.count_all_true(concr) + res = W_NDimArray(size, [size], self.find_dtype()) + ri = ArrayIterator(size) + shapelen = len(self.shape) + argi = concr.create_iter() + sig = self.find_sig() + frame = sig.create_frame(self) + v = None + while not frame.done(): + filter_driver.jit_merge_point(concr=concr, argi=argi, ri=ri, + frame=frame, v=v, res=res, sig=sig, + shapelen=shapelen, self=self) + if concr.dtype.getitem_bool(concr.storage, argi.offset): + v = sig.eval(frame, self) + res.setitem(ri.offset, v) + ri = ri.next(1) + else: + ri = ri.next_no_increase(1) + argi = argi.next(shapelen) + frame.next(shapelen) + return res + + def setitem_filter(self, space, idx, val): + size = self.count_all_true(idx) + arr = SliceArray([size], self.dtype, self, val) + sig = arr.find_sig() + shapelen = len(self.shape) + frame = sig.create_frame(arr) + idxi = idx.create_iter() + while not frame.done(): + filter_set_driver.jit_merge_point(idx=idx, idxi=idxi, sig=sig, + frame=frame, arr=arr, + shapelen=shapelen) + if idx.dtype.getitem_bool(idx.storage, idxi.offset): + sig.eval(frame, arr) + frame.next_from_second(1) + frame.next_first(shapelen) + idxi = idxi.next(shapelen) + def descr_getitem(self, space, w_idx): + if (isinstance(w_idx, BaseArray) and w_idx.shape == self.shape and + w_idx.find_dtype().is_bool_type()): + return self.getitem_filter(space, w_idx) if self._single_item_result(space, w_idx): concrete = self.get_concrete() item = concrete._index_of_single_item(space, w_idx) @@ -493,6 +585,11 @@ def descr_setitem(self, space, w_idx, w_value): self.invalidated() + if (isinstance(w_idx, BaseArray) and w_idx.shape == self.shape and + w_idx.find_dtype().is_bool_type()): + return self.get_concrete().setitem_filter(space, + w_idx.get_concrete(), + convert_to_array(space, w_value)) if self._single_item_result(space, w_idx): concrete = self.get_concrete() item = concrete._index_of_single_item(space, w_idx) @@ -509,9 +606,8 @@ def create_slice(self, chunks): shape = [] i = -1 - for i, (start_, stop, step, lgt) in enumerate(chunks): - if step != 0: - shape.append(lgt) + for i, chunk in enumerate(chunks): + chunk.extend_shape(shape) s = i + 1 assert s >= 0 shape += self.shape[s:] @@ -542,8 +638,8 @@ concrete = self.get_concrete() new_shape = get_shape_from_iterable(space, concrete.size, w_shape) # Since we got to here, prod(new_shape) == self.size - new_strides = calc_new_strides(new_shape, - concrete.shape, concrete.strides) + new_strides = calc_new_strides(new_shape, concrete.shape, + concrete.strides, concrete.order) if new_strides: # We can create a view, strides somehow match up. ndims = len(new_shape) @@ -581,7 +677,7 @@ def descr_var(self, space): # var = mean((values - mean(values)) ** 2) w_res = self.descr_sub(space, self.descr_mean(space, space.w_None)) - assert isinstance(w_res, BaseArray) + assert isinstance(w_res, BaseArray) w_res = w_res.descr_pow(space, space.wrap(2)) assert isinstance(w_res, BaseArray) return w_res.descr_mean(space, space.w_None) @@ -590,6 +686,10 @@ # std(v) = sqrt(var(v)) return interp_ufuncs.get(space).sqrt.call(space, [self.descr_var(space)]) + def descr_fill(self, space, w_value): + concr = self.get_concrete_or_scalar() + concr.fill(space, w_value) + def descr_nonzero(self, space): if self.size > 1: raise OperationError(space.w_ValueError, space.wrap( @@ -682,6 +782,14 @@ def copy(self, space): return Scalar(self.dtype, self.value) + def flatten(self, space): + array = W_NDimArray(self.size, [self.size], self.dtype) + array.setitem(0, self.value) + return array + + def fill(self, space, w_value): + self.value = self.dtype.coerce(space, w_value) + def create_sig(self): return signature.ScalarSignature(self.dtype) @@ -717,8 +825,7 @@ frame=frame, ri=ri, self=self, result=result) - result.dtype.setitem(result.storage, ri.offset, - sig.eval(frame, self)) + result.setitem(ri.offset, sig.eval(frame, self)) frame.next(shapelen) ri = ri.next(shapelen) return result @@ -788,7 +895,7 @@ Intermediate class for performing binary operations. """ _immutable_fields_ = ['left', 'right'] - + def __init__(self, ufunc, name, shape, calc_dtype, res_dtype, left, right): VirtualArray.__init__(self, name, shape, res_dtype) self.ufunc = ufunc @@ -825,14 +932,15 @@ self.left.create_sig(), self.right.create_sig()) class SliceArray(Call2): - def __init__(self, shape, dtype, left, right): + def __init__(self, shape, dtype, left, right, no_broadcast=False): + self.no_broadcast = no_broadcast Call2.__init__(self, None, 'sliceloop', shape, dtype, dtype, left, right) - + def create_sig(self): lsig = self.left.create_sig() rsig = self.right.create_sig() - if self.shape != self.right.shape: + if not self.no_broadcast and self.shape != self.right.shape: return signature.SliceloopBroadcastSignature(self.ufunc, self.name, self.calc_dtype, @@ -847,7 +955,7 @@ when we'll make AxisReduce lazy """ _immutable_fields_ = ['left', 'right'] - + def __init__(self, ufunc, name, shape, dtype, left, right, dim): Call2.__init__(self, ufunc, name, shape, dtype, dtype, left, right) @@ -919,14 +1027,14 @@ if size < 1: builder.append('[]') return - elif size == 1: - builder.append(dtype.itemtype.str_format(self.getitem(0))) - return if size > 1000: # Once this goes True it does not go back to False for recursive # calls use_ellipsis = True ndims = len(self.shape) + if ndims == 0: + builder.append(dtype.itemtype.str_format(self.getitem(0))) + return i = 0 builder.append('[') if ndims > 1: @@ -938,7 +1046,7 @@ builder.append('\n' + indent) else: builder.append(indent) - view = self.create_slice([(i, 0, 0, 1)]).get_concrete() + view = self.create_slice([Chunk(i, 0, 0, 1)]).get_concrete() view.to_str(space, comma, builder, indent=indent + ' ', use_ellipsis=use_ellipsis) if i < self.shape[0] - 1: @@ -955,7 +1063,7 @@ builder.append(indent) # create_slice requires len(chunks) > 1 in order to reduce # shape - view = self.create_slice([(i, 0, 0, 1)]).get_concrete() + view = self.create_slice([Chunk(i, 0, 0, 1)]).get_concrete() view.to_str(space, comma, builder, indent=indent + ' ', use_ellipsis=use_ellipsis) i += 1 @@ -1061,6 +1169,18 @@ array.setslice(space, self) return array + def flatten(self, space): + array = W_NDimArray(self.size, [self.size], self.dtype, self.order) + if self.supports_fast_slicing(): + array._fast_setslice(space, self) + else: + arr = SliceArray(array.shape, array.dtype, array, self, no_broadcast=True) + array._sliceloop(arr) + return array + + def fill(self, space, w_value): + self.setslice(space, scalar_w(space, self.dtype, w_value)) + class ViewArray(ConcreteArray): def create_sig(self): @@ -1081,6 +1201,10 @@ parent) self.start = start + def create_iter(self): + return ViewIterator(self.start, self.strides, self.backstrides, + self.shape) + def setshape(self, space, new_shape): if len(self.shape) < 1: return @@ -1104,7 +1228,8 @@ self.backstrides = backstrides self.shape = new_shape return - new_strides = calc_new_strides(new_shape, self.shape, self.strides) + new_strides = calc_new_strides(new_shape, self.shape, self.strides, + self.order) if new_strides is None: raise OperationError(space.w_AttributeError, space.wrap( "incompatible shape for a non-contiguous array")) @@ -1127,6 +1252,9 @@ self.shape = new_shape self.calc_strides(new_shape) + def create_iter(self): + return ArrayIterator(self.size) + def create_sig(self): return signature.ArraySignature(self.dtype) @@ -1181,6 +1309,7 @@ arr = W_NDimArray(size, shape[:], dtype=dtype, order=order) shapelen = len(shape) arr_iter = ArrayIterator(arr.size) + # XXX we might want to have a jitdriver here for i in range(len(elems_w)): w_elem = elems_w[i] dtype.setitem(arr.storage, arr_iter.offset, @@ -1247,6 +1376,9 @@ __gt__ = interp2app(BaseArray.descr_gt), __ge__ = interp2app(BaseArray.descr_ge), + __and__ = interp2app(BaseArray.descr_and), + __or__ = interp2app(BaseArray.descr_or), + __repr__ = interp2app(BaseArray.descr_repr), __str__ = interp2app(BaseArray.descr_str), __array_interface__ = GetSetProperty(BaseArray.descr_array_iface), @@ -1273,7 +1405,10 @@ var = interp2app(BaseArray.descr_var), std = interp2app(BaseArray.descr_std), + fill = interp2app(BaseArray.descr_fill), + copy = interp2app(BaseArray.descr_copy), + flatten = interp2app(BaseArray.descr_flatten), reshape = interp2app(BaseArray.descr_reshape), tolist = interp2app(BaseArray.descr_tolist), ) @@ -1311,3 +1446,11 @@ __iter__ = interp2app(W_FlatIterator.descr_iter), ) W_FlatIterator.acceptable_as_base_class = False + +def isna(space, w_obj): + if isinstance(w_obj, BaseArray): + arr = w_obj.empty_copy(space, + interp_dtype.get_dtype_cache(space).w_booldtype) + arr.fill(space, space.wrap(False)) + return arr + return space.wrap(False) diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py --- a/pypy/module/micronumpy/interp_ufuncs.py +++ b/pypy/module/micronumpy/interp_ufuncs.py @@ -249,15 +249,16 @@ class W_Ufunc2(W_Ufunc): - _immutable_fields_ = ["comparison_func", "func", "name"] + _immutable_fields_ = ["comparison_func", "func", "name", "int_only"] argcount = 2 def __init__(self, func, name, promote_to_float=False, promote_bools=False, - identity=None, comparison_func=False): + identity=None, comparison_func=False, int_only=False): W_Ufunc.__init__(self, name, promote_to_float, promote_bools, identity) self.func = func self.comparison_func = comparison_func + self.int_only = int_only def call(self, space, args_w): from pypy.module.micronumpy.interp_numarray import (Call2, @@ -268,6 +269,7 @@ w_rhs = convert_to_array(space, w_rhs) calc_dtype = find_binop_result_dtype(space, w_lhs.find_dtype(), w_rhs.find_dtype(), + int_only=self.int_only, promote_to_float=self.promote_to_float, promote_bools=self.promote_bools, ) @@ -304,10 +306,12 @@ def find_binop_result_dtype(space, dt1, dt2, promote_to_float=False, - promote_bools=False): + promote_bools=False, int_only=False): # dt1.num should be <= dt2.num if dt1.num > dt2.num: dt1, dt2 = dt2, dt1 + if int_only and (not dt1.is_int_type() or not dt2.is_int_type()): + raise OperationError(space.w_TypeError, space.wrap("Unsupported types")) # Some operations promote op(bool, bool) to return int8, rather than bool if promote_bools and (dt1.kind == dt2.kind == interp_dtype.BOOLLTR): return interp_dtype.get_dtype_cache(space).w_int8dtype @@ -425,6 +429,10 @@ ("add", "add", 2, {"identity": 0}), ("subtract", "sub", 2), ("multiply", "mul", 2, {"identity": 1}), + ("bitwise_and", "bitwise_and", 2, {"identity": 1, + 'int_only': True}), + ("bitwise_or", "bitwise_or", 2, {"identity": 0, + 'int_only': True}), ("divide", "div", 2, {"promote_bools": True}), ("mod", "mod", 2, {"promote_bools": True}), ("power", "pow", 2, {"promote_bools": True}), @@ -449,6 +457,7 @@ ("fabs", "fabs", 1, {"promote_to_float": True}), ("floor", "floor", 1, {"promote_to_float": True}), + ("ceil", "ceil", 1, {"promote_to_float": True}), ("exp", "exp", 1, {"promote_to_float": True}), ('sqrt', 'sqrt', 1, {'promote_to_float': True}), @@ -475,7 +484,7 @@ extra_kwargs["identity"] = identity func = ufunc_dtype_caller(space, ufunc_name, op_name, argcount, - comparison_func=extra_kwargs.get("comparison_func", False) + comparison_func=extra_kwargs.get("comparison_func", False), ) if argcount == 1: ufunc = W_Ufunc1(func, ufunc_name, **extra_kwargs) diff --git a/pypy/module/micronumpy/signature.py b/pypy/module/micronumpy/signature.py --- a/pypy/module/micronumpy/signature.py +++ b/pypy/module/micronumpy/signature.py @@ -82,6 +82,16 @@ for i in range(len(self.iterators)): self.iterators[i] = self.iterators[i].next(shapelen) + @unroll_safe + def next_from_second(self, shapelen): + """ Don't increase the first iterator + """ + for i in range(1, len(self.iterators)): + self.iterators[i] = self.iterators[i].next(shapelen) + + def next_first(self, shapelen): + self.iterators[0] = self.iterators[0].next(shapelen) + def get_final_iter(self): final_iter = promote(self.final_iter) if final_iter < 0: diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py --- a/pypy/module/micronumpy/strides.py +++ b/pypy/module/micronumpy/strides.py @@ -10,12 +10,12 @@ rstart = start rshape = [] i = -1 - for i, (start_, stop, step, lgt) in enumerate(chunks): - if step != 0: - rstrides.append(strides[i] * step) - rbackstrides.append(strides[i] * (lgt - 1) * step) - rshape.append(lgt) - rstart += strides[i] * start_ + for i, chunk in enumerate(chunks): + if chunk.step != 0: + rstrides.append(strides[i] * chunk.step) + rbackstrides.append(strides[i] * (chunk.lgt - 1) * chunk.step) + rshape.append(chunk.lgt) + rstart += strides[i] * chunk.start # add a reminder s = i + 1 assert s >= 0 diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py --- a/pypy/module/micronumpy/test/test_dtypes.py +++ b/pypy/module/micronumpy/test/test_dtypes.py @@ -166,14 +166,11 @@ # You can't subclass dtype raises(TypeError, type, "Foo", (dtype,), {}) - def test_new(self): - import _numpypy as np - assert np.int_(4) == 4 - assert np.float_(3.4) == 3.4 + def test_aliases(self): + from _numpypy import dtype - def test_pow(self): - from _numpypy import int_ - assert int_(4) ** 2 == 16 + assert dtype("float") is dtype(float) + class AppTestTypes(BaseNumpyAppTest): def test_abstract_types(self): @@ -189,6 +186,15 @@ raises(TypeError, numpy.floating, 0) raises(TypeError, numpy.inexact, 0) + def test_new(self): + import _numpypy as np + assert np.int_(4) == 4 + assert np.float_(3.4) == 3.4 + + def test_pow(self): + from _numpypy import int_ + assert int_(4) ** 2 == 16 + def test_bool(self): import _numpypy as numpy @@ -318,7 +324,7 @@ else: raises(OverflowError, numpy.int64, 9223372036854775807) raises(OverflowError, numpy.int64, '9223372036854775807') - + raises(OverflowError, numpy.int64, 9223372036854775808) raises(OverflowError, numpy.int64, '9223372036854775808') diff --git a/pypy/module/micronumpy/test/test_module.py b/pypy/module/micronumpy/test/test_module.py --- a/pypy/module/micronumpy/test/test_module.py +++ b/pypy/module/micronumpy/test/test_module.py @@ -2,16 +2,11 @@ class AppTestNumPyModule(BaseNumpyAppTest): - def test_mean(self): - from _numpypy import array, mean - assert mean(array(range(5))) == 2.0 - assert mean(range(5)) == 2.0 - def test_average(self): from _numpypy import array, average assert average(range(10)) == 4.5 assert average(array(range(10))) == 4.5 - + def test_sum(self): from _numpypy import array, sum assert sum(range(10)) == 45 @@ -21,7 +16,7 @@ from _numpypy import array, min assert min(range(10)) == 0 assert min(array(range(10))) == 0 - + def test_max(self): from _numpypy import array, max assert max(range(10)) == 9 diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -2,6 +2,7 @@ import py from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest from pypy.module.micronumpy.interp_numarray import W_NDimArray, shape_agreement +from pypy.module.micronumpy.interp_iter import Chunk from pypy.module.micronumpy import signature from pypy.interpreter.error import OperationError from pypy.conftest import gettestobjspace @@ -37,53 +38,54 @@ def test_create_slice_f(self): a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'F') - s = a.create_slice([(3, 0, 0, 1)]) + s = a.create_slice([Chunk(3, 0, 0, 1)]) assert s.start == 3 assert s.strides == [10, 50] assert s.backstrides == [40, 100] - s = a.create_slice([(1, 9, 2, 4)]) + s = a.create_slice([Chunk(1, 9, 2, 4)]) assert s.start == 1 assert s.strides == [2, 10, 50] assert s.backstrides == [6, 40, 100] - s = a.create_slice([(1, 5, 3, 2), (1, 2, 1, 1), (1, 0, 0, 1)]) + s = a.create_slice([Chunk(1, 5, 3, 2), Chunk(1, 2, 1, 1), Chunk(1, 0, 0, 1)]) assert s.shape == [2, 1] assert s.strides == [3, 10] assert s.backstrides == [3, 0] - s = a.create_slice([(0, 10, 1, 10), (2, 0, 0, 1)]) + s = a.create_slice([Chunk(0, 10, 1, 10), Chunk(2, 0, 0, 1)]) assert s.start == 20 assert s.shape == [10, 3] def test_create_slice_c(self): a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'C') - s = a.create_slice([(3, 0, 0, 1)]) + s = a.create_slice([Chunk(3, 0, 0, 1)]) assert s.start == 45 assert s.strides == [3, 1] assert s.backstrides == [12, 2] - s = a.create_slice([(1, 9, 2, 4)]) + s = a.create_slice([Chunk(1, 9, 2, 4)]) assert s.start == 15 assert s.strides == [30, 3, 1] assert s.backstrides == [90, 12, 2] - s = a.create_slice([(1, 5, 3, 2), (1, 2, 1, 1), (1, 0, 0, 1)]) + s = a.create_slice([Chunk(1, 5, 3, 2), Chunk(1, 2, 1, 1), + Chunk(1, 0, 0, 1)]) assert s.start == 19 assert s.shape == [2, 1] assert s.strides == [45, 3] assert s.backstrides == [45, 0] - s = a.create_slice([(0, 10, 1, 10), (2, 0, 0, 1)]) + s = a.create_slice([Chunk(0, 10, 1, 10), Chunk(2, 0, 0, 1)]) assert s.start == 6 assert s.shape == [10, 3] def test_slice_of_slice_f(self): a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'F') - s = a.create_slice([(5, 0, 0, 1)]) + s = a.create_slice([Chunk(5, 0, 0, 1)]) assert s.start == 5 - s2 = s.create_slice([(3, 0, 0, 1)]) + s2 = s.create_slice([Chunk(3, 0, 0, 1)]) assert s2.shape == [3] assert s2.strides == [50] assert s2.parent is a assert s2.backstrides == [100] assert s2.start == 35 - s = a.create_slice([(1, 5, 3, 2)]) - s2 = s.create_slice([(0, 2, 1, 2), (2, 0, 0, 1)]) + s = a.create_slice([Chunk(1, 5, 3, 2)]) + s2 = s.create_slice([Chunk(0, 2, 1, 2), Chunk(2, 0, 0, 1)]) assert s2.shape == [2, 3] assert s2.strides == [3, 50] assert s2.backstrides == [3, 100] @@ -91,16 +93,16 @@ def test_slice_of_slice_c(self): a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), order='C') - s = a.create_slice([(5, 0, 0, 1)]) + s = a.create_slice([Chunk(5, 0, 0, 1)]) assert s.start == 15 * 5 - s2 = s.create_slice([(3, 0, 0, 1)]) + s2 = s.create_slice([Chunk(3, 0, 0, 1)]) assert s2.shape == [3] assert s2.strides == [1] assert s2.parent is a assert s2.backstrides == [2] assert s2.start == 5 * 15 + 3 * 3 - s = a.create_slice([(1, 5, 3, 2)]) - s2 = s.create_slice([(0, 2, 1, 2), (2, 0, 0, 1)]) + s = a.create_slice([Chunk(1, 5, 3, 2)]) + s2 = s.create_slice([Chunk(0, 2, 1, 2), Chunk(2, 0, 0, 1)]) assert s2.shape == [2, 3] assert s2.strides == [45, 1] assert s2.backstrides == [45, 2] @@ -108,14 +110,14 @@ def test_negative_step_f(self): a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'F') - s = a.create_slice([(9, -1, -2, 5)]) + s = a.create_slice([Chunk(9, -1, -2, 5)]) assert s.start == 9 assert s.strides == [-2, 10, 50] assert s.backstrides == [-8, 40, 100] def test_negative_step_c(self): a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), order='C') - s = a.create_slice([(9, -1, -2, 5)]) + s = a.create_slice([Chunk(9, -1, -2, 5)]) assert s.start == 135 assert s.strides == [-30, 3, 1] assert s.backstrides == [-120, 12, 2] @@ -124,7 +126,7 @@ a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'F') r = a._index_of_single_item(self.space, self.newtuple(1, 2, 2)) assert r == 1 + 2 * 10 + 2 * 50 - s = a.create_slice([(0, 10, 1, 10), (2, 0, 0, 1)]) + s = a.create_slice([Chunk(0, 10, 1, 10), Chunk(2, 0, 0, 1)]) r = s._index_of_single_item(self.space, self.newtuple(1, 0)) assert r == a._index_of_single_item(self.space, self.newtuple(1, 2, 0)) r = s._index_of_single_item(self.space, self.newtuple(1, 1)) @@ -134,7 +136,7 @@ a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'C') r = a._index_of_single_item(self.space, self.newtuple(1, 2, 2)) assert r == 1 * 3 * 5 + 2 * 3 + 2 - s = a.create_slice([(0, 10, 1, 10), (2, 0, 0, 1)]) + s = a.create_slice([Chunk(0, 10, 1, 10), Chunk(2, 0, 0, 1)]) r = s._index_of_single_item(self.space, self.newtuple(1, 0)) assert r == a._index_of_single_item(self.space, self.newtuple(1, 2, 0)) r = s._index_of_single_item(self.space, self.newtuple(1, 1)) @@ -152,13 +154,18 @@ def test_calc_new_strides(self): from pypy.module.micronumpy.interp_numarray import calc_new_strides - assert calc_new_strides([2, 4], [4, 2], [4, 2]) == [8, 2] - assert calc_new_strides([2, 4, 3], [8, 3], [1, 16]) == [1, 2, 16] - assert calc_new_strides([2, 3, 4], [8, 3], [1, 16]) is None - assert calc_new_strides([24], [2, 4, 3], [48, 6, 1]) is None - assert calc_new_strides([24], [2, 4, 3], [24, 6, 2]) == [2] - assert calc_new_strides([105, 1], [3, 5, 7], [35, 7, 1]) == [1, 1] - assert calc_new_strides([1, 105], [3, 5, 7], [35, 7, 1]) == [105, 1] + assert calc_new_strides([2, 4], [4, 2], [4, 2], "C") == [8, 2] + assert calc_new_strides([2, 4, 3], [8, 3], [1, 16], 'F') == [1, 2, 16] + assert calc_new_strides([2, 3, 4], [8, 3], [1, 16], 'F') is None + assert calc_new_strides([24], [2, 4, 3], [48, 6, 1], 'C') is None + assert calc_new_strides([24], [2, 4, 3], [24, 6, 2], 'C') == [2] + assert calc_new_strides([105, 1], [3, 5, 7], [35, 7, 1],'C') == [1, 1] + assert calc_new_strides([1, 105], [3, 5, 7], [35, 7, 1],'C') == [105, 1] + assert calc_new_strides([1, 105], [3, 5, 7], [35, 7, 1],'F') is None + assert calc_new_strides([1, 1, 1, 105, 1], [15, 7], [7, 1],'C') == \ + [105, 105, 105, 1, 1] + assert calc_new_strides([1, 1, 105, 1, 1], [7, 15], [1, 7],'F') == \ + [1, 1, 1, 105, 105] class AppTestNumArray(BaseNumpyAppTest): @@ -383,6 +390,8 @@ a.shape = () #numpy allows this a.shape = (1,) + a = array(range(6)).reshape(2,3).T + raises(AttributeError, 'a.shape = 6') def test_reshape(self): from _numpypy import array, zeros @@ -726,16 +735,17 @@ assert d[1] == 12 def test_mean(self): - from _numpypy import array, mean + from _numpypy import array, arange a = array(range(5)) assert a.mean() == 2.0 assert a[:4].mean() == 1.5 a = array(range(105)).reshape(3, 5, 7) - b = mean(a, axis=0) - b[0,0]==35. + b = a.mean(axis=0) + b[0, 0]==35. assert a.mean(axis=0)[0, 0] == 35 assert (b == array(range(35, 70), dtype=float).reshape(5, 7)).all() - assert (mean(a, 2) == array(range(0, 15), dtype=float).reshape(3, 5) * 7 + 3).all() + assert (a.mean(2) == array(range(0, 15), dtype=float).reshape(3, 5) * 7 + 3).all() + assert (arange(10).reshape(5, 2).mean(axis=1) == [0.5, 2.5, 4.5, 6.5, 8.5]).all() def test_sum(self): from _numpypy import array @@ -771,6 +781,8 @@ assert ((a + a).T.sum(2).T == (a + a).sum(0)).all() assert (a.reshape(1,-1).sum(0) == range(105)).all() assert (a.reshape(1,-1).sum(1) == 5460) + assert (array([[1,2],[3,4]]).prod(0) == [3, 8]).all() + assert (array([[1,2],[3,4]]).prod(1) == [2, 12]).all() def test_identity(self): from _numpypy import identity, array @@ -1017,11 +1029,15 @@ assert a[0].tolist() == [17.1, 27.2] def test_var(self): - from _numpypy import array + from _numpypy import array, arange a = array(range(10)) assert a.var() == 8.25 a = array([5.0]) assert a.var() == 0.0 + a = arange(10).reshape(5, 2) + assert a.var() == 8.25 + #assert (a.var(0) == [8, 8]).all() + #assert (a.var(1) == [.25] * 5).all() def test_std(self): from _numpypy import array @@ -1030,6 +1046,22 @@ a = array([5.0]) assert a.std() == 0.0 + def test_flatten(self): + from _numpypy import array + + a = array([[1, 2, 3], [4, 5, 6]]) + assert (a.flatten() == [1, 2, 3, 4, 5, 6]).all() + a = array([[[1, 2], [3, 4]], [[5, 6], [7, 8]]]) + assert (a.flatten() == [1, 2, 3, 4, 5, 6, 7, 8]).all() + a = array([1, 2, 3, 4, 5, 6, 7, 8]) + assert (a[::2].flatten() == [1, 3, 5, 7]).all() + a = array([1, 2, 3]) + assert ((a + a).flatten() == [2, 4, 6]).all() + a = array(2) + assert (a.flatten() == [2]).all() + a = array([[1, 2], [3, 4]]) + assert (a.T.flatten() == [1, 3, 2, 4]).all() + class AppTestMultiDim(BaseNumpyAppTest): def test_init(self): @@ -1300,6 +1332,61 @@ assert isinstance(i['data'][0], int) raises(TypeError, getattr, array(3), '__array_interface__') + def test_array_indexing_one_elem(self): + skip("not yet") + from _numpypy import array, arange + raises(IndexError, 'arange(3)[array([3.5])]') + a = arange(3)[array([1])] + assert a == 1 + assert a[0] == 1 + raises(IndexError,'arange(3)[array([15])]') + assert arange(3)[array([-3])] == 0 + raises(IndexError,'arange(3)[array([-15])]') + assert arange(3)[array(1)] == 1 + + def test_fill(self): + from _numpypy import array + a = array([1, 2, 3]) + a.fill(10) + assert (a == [10, 10, 10]).all() + a.fill(False) + assert (a == [0, 0, 0]).all() + b = a[:1] + b.fill(4) + assert (b == [4]).all() + assert (a == [4, 0, 0]).all() + + c = b + b + c.fill(27) + assert (c == [27]).all() + + d = array(10) + d.fill(100) + assert d == 100 + + def test_array_indexing_bool(self): + from _numpypy import arange + a = arange(10) + assert (a[a > 3] == [4, 5, 6, 7, 8, 9]).all() + a = arange(10).reshape(5, 2) + assert (a[a > 3] == [4, 5, 6, 7, 8, 9]).all() + assert (a[a & 1 == 1] == [1, 3, 5, 7, 9]).all() + + def test_array_indexing_bool_setitem(self): + from _numpypy import arange, array + a = arange(6) + a[a > 3] = 15 + assert (a == [0, 1, 2, 3, 15, 15]).all() + a = arange(6).reshape(3, 2) + a[a & 1 == 1] = array([8, 9, 10]) + assert (a == [[0, 8], [2, 9], [4, 10]]).all() + + def test_isna(self): + from _numpypy import isna, array + # XXX for now + assert not isna(3) + assert (isna(array([1, 2, 3, 4])) == [False, False, False, False]).all() + class AppTestSupport(BaseNumpyAppTest): def setup_class(cls): @@ -1439,9 +1526,11 @@ assert repr(a) == "array(0.0)" a = array(0.2) assert repr(a) == "array(0.2)" + a = array([2]) + assert repr(a) == "array([2])" def test_repr_multi(self): - from _numpypy import arange, zeros + from _numpypy import arange, zeros, array a = zeros((3, 4)) assert repr(a) == '''array([[0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0], @@ -1464,6 +1553,9 @@ [498, 999], [499, 1000], [500, 1001]])''' + a = arange(2).reshape((2,1)) + assert repr(a) == '''array([[0], + [1]])''' def test_repr_slice(self): from _numpypy import array, zeros @@ -1548,18 +1640,3 @@ a = arange(0, 0.8, 0.1) assert len(a) == 8 assert arange(False, True, True).dtype is dtype(int) - - -class AppTestRanges(BaseNumpyAppTest): - def test_app_reshape(self): - from _numpypy import arange, array, dtype, reshape - a = arange(12) - b = reshape(a, (3, 4)) - assert b.shape == (3, 4) - a = range(12) - b = reshape(a, (3, 4)) - assert b.shape == (3, 4) - a = array(range(105)).reshape(3, 5, 7) - assert a.reshape(1, -1).shape == (1, 105) - assert a.reshape(1, 1, -1).shape == (1, 1, 105) - assert a.reshape(-1, 1, 1).shape == (105, 1, 1) diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py --- a/pypy/module/micronumpy/test/test_ufuncs.py +++ b/pypy/module/micronumpy/test/test_ufuncs.py @@ -190,14 +190,24 @@ for i in range(3): assert c[i] == a[i] - b[i] - def test_floor(self): - from _numpypy import array, floor - - reference = [-2.0, -1.0, 0.0, 1.0, 1.0] - a = array([-1.4, -1.0, 0.0, 1.0, 1.4]) + def test_floorceil(self): + from _numpypy import array, floor, ceil + import math + reference = [-2.0, -2.0, -1.0, 0.0, 1.0, 1.0, 0] + a = array([-1.4, -1.5, -1.0, 0.0, 1.0, 1.4, 0.5]) b = floor(a) for i in range(5): assert b[i] == reference[i] + reference = [-1.0, -1.0, -1.0, 0.0, 1.0, 2.0, 1.0] + a = array([-1.4, -1.5, -1.0, 0.0, 1.0, 1.4, 0.5]) + b = ceil(a) + assert (reference == b).all() + inf = float("inf") + data = [1.5, 2.9999, -1.999, inf] + results = [math.floor(x) for x in data] + assert (floor(data) == results).all() + results = [math.ceil(x) for x in data] + assert (ceil(data) == results).all() def test_copysign(self): from _numpypy import array, copysign @@ -238,7 +248,7 @@ assert b[i] == math.sin(a[i]) a = sin(array([True, False], dtype=bool)) - assert abs(a[0] - sin(1)) < 1e-7 # a[0] will be less precise + assert abs(a[0] - sin(1)) < 1e-7 # a[0] will be less precise assert a[1] == 0.0 def test_cos(self): @@ -259,7 +269,6 @@ for i in range(len(a)): assert b[i] == math.tan(a[i]) - def test_arcsin(self): import math from _numpypy import array, arcsin @@ -283,7 +292,6 @@ for i in range(len(a)): assert b[i] == math.acos(a[i]) - a = array([-10, -1.5, -1.01, 1.01, 1.5, 10, float('nan'), float('inf'), float('-inf')]) b = arccos(a) for f in b: @@ -347,11 +355,20 @@ raises(ValueError, maximum.reduce, []) def test_reduceND(self): - from numpypy import add, arange + from _numpypy import add, arange a = arange(12).reshape(3, 4) assert (add.reduce(a, 0) == [12, 15, 18, 21]).all() assert (add.reduce(a, 1) == [6.0, 22.0, 38.0]).all() + def test_bitwise(self): + from _numpypy import bitwise_and, bitwise_or, arange, array + a = arange(6).reshape(2, 3) + assert (a & 1 == [[0, 1, 0], [1, 0, 1]]).all() + assert (a & 1 == bitwise_and(a, 1)).all() + assert (a | 1 == [[1, 1, 3], [3, 5, 5]]).all() + assert (a | 1 == bitwise_or(a, 1)).all() + raises(TypeError, 'array([1.0]) & 1') + def test_comparisons(self): import operator from _numpypy import equal, not_equal, less, less_equal, greater, greater_equal diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py --- a/pypy/module/micronumpy/test/test_zjit.py +++ b/pypy/module/micronumpy/test/test_zjit.py @@ -217,6 +217,7 @@ # This is the sum of the ops for both loops, however if you remove the # optimization then you end up with 2 float_adds, so we can still be # sure it was optimized correctly. + py.test.skip("too fragile") self.check_resops({'setinteriorfield_raw': 4, 'getfield_gc': 22, 'getarrayitem_gc': 4, 'getarrayitem_gc_pure': 2, 'getfield_gc_pure': 8, @@ -349,7 +350,8 @@ self.check_trace_count(1) self.check_simple_loop({'getinteriorfield_raw': 2, 'float_add': 1, 'setinteriorfield_raw': 1, 'int_add': 2, - 'int_eq': 1, 'guard_false': 1, 'jump': 1}) + 'int_eq': 1, 'guard_false': 1, 'jump': 1, + 'arraylen_gc': 1}) def define_virtual_slice(): return """ diff --git a/pypy/module/micronumpy/tool/numready/__init__.py b/pypy/module/micronumpy/tool/numready/__init__.py new file mode 100644 diff --git a/pypy/module/micronumpy/tool/numready/__main__.py b/pypy/module/micronumpy/tool/numready/__main__.py new file mode 100644 --- /dev/null +++ b/pypy/module/micronumpy/tool/numready/__main__.py @@ -0,0 +1,6 @@ +import sys + +from .main import main + + +main(sys.argv) diff --git a/pypy/module/micronumpy/tool/numready/kinds.py b/pypy/module/micronumpy/tool/numready/kinds.py new file mode 100644 --- /dev/null +++ b/pypy/module/micronumpy/tool/numready/kinds.py @@ -0,0 +1,4 @@ +KINDS = { + "UNKNOWN": "U", + "TYPE": "T", +} diff --git a/pypy/module/micronumpy/tool/numready/main.py b/pypy/module/micronumpy/tool/numready/main.py new file mode 100644 --- /dev/null +++ b/pypy/module/micronumpy/tool/numready/main.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" +This should be run under PyPy. +""" + +import os +import platform +import subprocess +import tempfile +import webbrowser +from collections import OrderedDict + +import jinja2 + +from .kinds import KINDS + + +class SearchableSet(object): + def __init__(self, items=()): + self._items = {} + for item in items: + self.add(item) + + def __iter__(self): + return iter(self._items) + + def __contains__(self, other): + return other in self._items + + def __getitem__(self, idx): + return self._items[idx] + + def add(self, item): + self._items[item] = item + + def __len__(self): + return len(self._items) + +class Item(object): + def __init__(self, name, kind, subitems=None): + self.name = name + self.kind = kind + self.subitems = subitems + + def __hash__(self): + return hash(self.name) + + def __eq__(self, other): + if isinstance(other, str): + return self.name == other + return self.name == other.name + + +class ItemStatus(object): + def __init__(self, name, pypy_exists): + self.name = name + self.cls = 'exists' if pypy_exists else '' + self.symbol = u"✔" if pypy_exists else u'✖' + + def __lt__(self, other): + return self.name < other.name + +def find_numpy_items(python, modname="numpy", attr=None): + args = [ + python, os.path.join(os.path.dirname(__file__), "search.py"), modname + ] + if attr is not None: + args.append(attr) + lines = subprocess.check_output(args).splitlines() + items = SearchableSet() + for line in lines: + kind, name = line.split(" : ", 1) + subitems = None + if kind == KINDS["TYPE"]: + if name in ['ndarray', 'dtype']: + subitems = find_numpy_items(python, modname, name) + items.add(Item(name, kind, subitems)) + return items + +def split(lst): + SPLIT = 5 + lgt = len(lst) // SPLIT + 1 + l = [[] for i in range(lgt)] + for i in range(lgt): + for k in range(SPLIT): + if k * lgt + i < len(lst): + l[i].append(lst[k * lgt + i]) + return l + +def main(argv): + cpy_items = find_numpy_items("/usr/bin/python") + pypy_items = find_numpy_items(argv[1], "numpypy") + all_items = [] + + msg = '%d/%d names, %d/%d ndarray attributes, %d/%d dtype attributes' % ( + len(pypy_items), len(cpy_items), len(pypy_items['ndarray'].subitems), + len(cpy_items['ndarray'].subitems), len(pypy_items['dtype'].subitems), + len(cpy_items['dtype'].subitems)) + for item in cpy_items: + pypy_exists = item in pypy_items + if item.subitems: + for sub in item.subitems: + all_items.append( + ItemStatus(item.name + "." + sub.name, pypy_exists=pypy_exists and pypy_items[item].subitems and sub in pypy_items[item].subitems) + ) + all_items.append(ItemStatus(item.name, pypy_exists=item in pypy_items)) + env = jinja2.Environment( + loader=jinja2.FileSystemLoader(os.path.dirname(__file__)) + ) + html = env.get_template("page.html").render(all_items=split(sorted(all_items)), msg=msg) + if len(argv) > 2: + with open(argv[2], 'w') as f: + f.write(html.encode("utf-8")) + else: + with tempfile.NamedTemporaryFile(delete=False) as f: + f.write(html.encode("utf-8")) + print "Saved in: %s" % f.name diff --git a/pypy/module/micronumpy/tool/numready/page.html b/pypy/module/micronumpy/tool/numready/page.html new file mode 100644 --- /dev/null +++ b/pypy/module/micronumpy/tool/numready/page.html @@ -0,0 +1,65 @@ +<!DOCTYPE html> +<html lang="en"> + <head> + <title>NumPyPy Status</title> + <meta http-equiv="content-type" content="text/html; charset=utf-8"> + <style type="text/css"> + body { + font-family: 'Consolas', 'Bitstream Vera Sans Mono', monospace; + } + h1 { + text-align: center; + } + h3 { + text-align: center; + } + table { + border: 8px solid #DFDECB; + margin: 30px auto; + font-size: 12px; + } + table th { + text-align: left; + } + table td { + padding: 4px 10px; + text-align: center; + } + .exists { + background-color: #337792; + color: white; + border: 1px solid #234F61; + } + </style> + </head> + <body> + <h1>NumPyPy Status</h1> + <h3>Overall: {{ msg }}</h3> + <table> + <thead> + <tr> + <th></th> + <th>PyPy</th> + <th></th> + <th>PyPy</th> + <th></th> + <th>PyPy</th> + <th></th> + <th>PyPy</th> + <th></th> + <th>PyPy</th> + </tr> + </thead> + <tbody> + {% for chunk in all_items %} + <tr> + {% for item in chunk %} + <th class='{{ item.cls }}'>{{ item.name }}</th> + <td class='{{ item.cls }}'>{{ item.symbol }}</td> + {% endfor %} + </tr> + {% endfor %} + </tbody> + </table> + </body> +</html> diff --git a/pypy/module/micronumpy/tool/numready/search.py b/pypy/module/micronumpy/tool/numready/search.py new file mode 100644 --- /dev/null +++ b/pypy/module/micronumpy/tool/numready/search.py @@ -0,0 +1,33 @@ +import sys +import types + +# Evil implicit relative import. +from kinds import KINDS + + +def main(argv): + if len(argv) == 2: + [_, modname] = argv + attr = None + elif len(argv) == 3: + [_, modname, attr] = argv + else: + sys.exit("Wrong number of args") + __import__(modname) + obj = sys.modules[modname] + + if attr is not None: + obj = getattr(obj, attr) + + for name in dir(obj): + if attr is None and name.startswith("_"): + continue + subobj = getattr(obj, name) + if isinstance(subobj, types.TypeType): + kind = KINDS["TYPE"] + else: + kind = KINDS["UNKNOWN"] + print kind, ":", name + +if __name__ == "__main__": + main(sys.argv) \ No newline at end of file diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py --- a/pypy/module/micronumpy/types.py +++ b/pypy/module/micronumpy/types.py @@ -94,6 +94,9 @@ width, storage, i, offset )) + def read_bool(self, storage, width, i, offset): + raise NotImplementedError + def store(self, storage, width, i, offset, box): value = self.unbox(box) libffi.array_setitem(clibffi.cast_type_to_ffitype(self.T), @@ -168,6 +171,7 @@ @simple_binary_op def min(self, v1, v2): return min(v1, v2) + class Bool(BaseType, Primitive): T = lltype.Bool @@ -185,6 +189,11 @@ else: return self.False + + def read_bool(self, storage, width, i, offset): + return libffi.array_getitem(clibffi.cast_type_to_ffitype(self.T), + width, storage, i, offset) + def coerce_subtype(self, space, w_subtype, w_item): # Doesn't return subclasses so it can return the constants. return self._coerce(space, w_item) @@ -253,6 +262,14 @@ assert v == 0 return 0 + @simple_binary_op + def bitwise_and(self, v1, v2): + return v1 & v2 + + @simple_binary_op + def bitwise_or(self, v1, v2): + return v1 | v2 + class Int8(BaseType, Integer): T = rffi.SIGNEDCHAR BoxType = interp_boxes.W_Int8Box @@ -374,6 +391,10 @@ return math.floor(v) @simple_unary_op + def ceil(self, v): + return math.ceil(v) + + @simple_unary_op def exp(self, v): try: return math.exp(v) @@ -436,4 +457,4 @@ class Float64(BaseType, Float): T = rffi.DOUBLE BoxType = interp_boxes.W_Float64Box - format_code = "d" \ No newline at end of file + format_code = "d" diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -371,6 +371,8 @@ if hasattr(__import__(os.name), "forkpty"): def test_forkpty(self): import sys + if 'freebsd' in sys.platform: + skip("hangs indifinitly on FreeBSD (also on CPython).") os = self.posix childpid, master_fd = os.forkpty() assert isinstance(childpid, int) diff --git a/pypy/module/pypyjit/__init__.py b/pypy/module/pypyjit/__init__.py --- a/pypy/module/pypyjit/__init__.py +++ b/pypy/module/pypyjit/__init__.py @@ -11,6 +11,7 @@ 'set_optimize_hook': 'interp_resop.set_optimize_hook', 'set_abort_hook': 'interp_resop.set_abort_hook', 'ResOperation': 'interp_resop.WrappedOp', + 'DebugMergePoint': 'interp_resop.DebugMergePoint', 'Box': 'interp_resop.WrappedBox', } diff --git a/pypy/module/pypyjit/interp_resop.py b/pypy/module/pypyjit/interp_resop.py --- a/pypy/module/pypyjit/interp_resop.py +++ b/pypy/module/pypyjit/interp_resop.py @@ -1,5 +1,6 @@ -from pypy.interpreter.typedef import TypeDef, GetSetProperty +from pypy.interpreter.typedef import (TypeDef, GetSetProperty, + interp_attrproperty, interp_attrproperty_w) from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.gateway import unwrap_spec, interp2app, NoneNotWrapped from pypy.interpreter.pycode import PyCode @@ -10,6 +11,7 @@ from pypy.jit.metainterp.resoperation import rop, AbstractResOp from pypy.rlib.nonconst import NonConstant from pypy.rlib import jit_hooks +from pypy.module.pypyjit.interp_jit import pypyjitdriver class Cache(object): in_recursion = False @@ -111,13 +113,24 @@ def wrap_oplist(space, logops, operations, ops_offset=None): l_w = [] + jitdrivers_sd = logops.metainterp_sd.jitdrivers_sd for op in operations: if ops_offset is None: ofs = -1 else: ofs = ops_offset.get(op, 0) - l_w.append(WrappedOp(jit_hooks._cast_to_gcref(op), ofs, - logops.repr_of_resop(op))) + if op.opnum == rop.DEBUG_MERGE_POINT: + jd_sd = jitdrivers_sd[op.getarg(0).getint()] + greenkey = op.getarglist()[2:] + repr = jd_sd.warmstate.get_location_str(greenkey) + w_greenkey = wrap_greenkey(space, jd_sd.jitdriver, greenkey, repr) + l_w.append(DebugMergePoint(space, jit_hooks._cast_to_gcref(op), + logops.repr_of_resop(op), + jd_sd.jitdriver.name, + w_greenkey)) + else: + l_w.append(WrappedOp(jit_hooks._cast_to_gcref(op), ofs, + logops.repr_of_resop(op))) return l_w class WrappedBox(Wrappable): @@ -150,6 +163,15 @@ llres = res.llbox return WrappedOp(jit_hooks.resop_new(num, args, llres), offset, repr) + at unwrap_spec(repr=str, jd_name=str) +def descr_new_dmp(space, w_tp, w_args, repr, jd_name, w_greenkey): + args = [space.interp_w(WrappedBox, w_arg).llbox for w_arg in + space.listview(w_args)] + num = rop.DEBUG_MERGE_POINT + return DebugMergePoint(space, + jit_hooks.resop_new(num, args, jit_hooks.emptyval()), + repr, jd_name, w_greenkey) + class WrappedOp(Wrappable): """ A class representing a single ResOperation, wrapped nicely """ @@ -182,6 +204,25 @@ box = space.interp_w(WrappedBox, w_box) jit_hooks.resop_setresult(self.op, box.llbox) +class DebugMergePoint(WrappedOp): + def __init__(self, space, op, repr_of_resop, jd_name, w_greenkey): + WrappedOp.__init__(self, op, -1, repr_of_resop) + self.w_greenkey = w_greenkey + self.jd_name = jd_name + + def get_pycode(self, space): + if self.jd_name == pypyjitdriver.name: + return space.getitem(self.w_greenkey, space.wrap(0)) + raise OperationError(space.w_AttributeError, space.wrap("This DebugMergePoint doesn't belong to the main Python JitDriver")) + + def get_bytecode_no(self, space): + if self.jd_name == pypyjitdriver.name: + return space.getitem(self.w_greenkey, space.wrap(1)) + raise OperationError(space.w_AttributeError, space.wrap("This DebugMergePoint doesn't belong to the main Python JitDriver")) + + def get_jitdriver_name(self, space): + return space.wrap(self.jd_name) + WrappedOp.typedef = TypeDef( 'ResOperation', __doc__ = WrappedOp.__doc__, @@ -195,3 +236,15 @@ WrappedOp.descr_setresult) ) WrappedOp.acceptable_as_base_class = False + +DebugMergePoint.typedef = TypeDef( + 'DebugMergePoint', WrappedOp.typedef, + __new__ = interp2app(descr_new_dmp), + greenkey = interp_attrproperty_w("w_greenkey", cls=DebugMergePoint), + pycode = GetSetProperty(DebugMergePoint.get_pycode), + bytecode_no = GetSetProperty(DebugMergePoint.get_bytecode_no), + jitdriver_name = GetSetProperty(DebugMergePoint.get_jitdriver_name), +) +DebugMergePoint.acceptable_as_base_class = False + + diff --git a/pypy/module/pypyjit/policy.py b/pypy/module/pypyjit/policy.py --- a/pypy/module/pypyjit/policy.py +++ b/pypy/module/pypyjit/policy.py @@ -127,7 +127,7 @@ 'imp', 'sys', 'array', '_ffi', 'itertools', 'operator', 'posix', '_socket', '_sre', '_lsprof', '_weakref', '__pypy__', 'cStringIO', '_collections', 'struct', - 'mmap', 'marshal']: + 'mmap', 'marshal', '_codecs']: if modname == 'pypyjit' and 'interp_resop' in rest: return False return True diff --git a/pypy/module/pypyjit/test/test_jit_hook.py b/pypy/module/pypyjit/test/test_jit_hook.py --- a/pypy/module/pypyjit/test/test_jit_hook.py +++ b/pypy/module/pypyjit/test/test_jit_hook.py @@ -92,6 +92,7 @@ cls.w_on_compile_bridge = space.wrap(interp2app(interp_on_compile_bridge)) cls.w_on_abort = space.wrap(interp2app(interp_on_abort)) cls.w_int_add_num = space.wrap(rop.INT_ADD) + cls.w_dmp_num = space.wrap(rop.DEBUG_MERGE_POINT) cls.w_on_optimize = space.wrap(interp2app(interp_on_optimize)) cls.orig_oplist = oplist @@ -117,6 +118,10 @@ assert elem[2][2] == False assert len(elem[3]) == 4 int_add = elem[3][0] + dmp = elem[3][1] + assert isinstance(dmp, pypyjit.DebugMergePoint) + assert dmp.pycode is self.f.func_code + assert dmp.greenkey == (self.f.func_code, 0, False) #assert int_add.name == 'int_add' assert int_add.num == self.int_add_num self.on_compile_bridge() @@ -211,3 +216,18 @@ assert op.getarg(0).getint() == 4 op.result = box assert op.result.getint() == 1 + + def test_creation_dmp(self): + from pypyjit import DebugMergePoint, Box + + def f(): + pass + + op = DebugMergePoint([Box(0)], 'repr', 'pypyjit', (f.func_code, 0, 0)) + assert op.bytecode_no == 0 + assert op.pycode is f.func_code + assert repr(op) == 'repr' + assert op.jitdriver_name == 'pypyjit' + assert op.num == self.dmp_num + op = DebugMergePoint([Box(0)], 'repr', 'notmain', ('str',)) + raises(AttributeError, 'op.pycode') diff --git a/pypy/module/pypyjit/test_pypy_c/test_math.py b/pypy/module/pypyjit/test_pypy_c/test_math.py --- a/pypy/module/pypyjit/test_pypy_c/test_math.py +++ b/pypy/module/pypyjit/test_pypy_c/test_math.py @@ -90,3 +90,19 @@ --TICK-- jump(..., descr=) """) + + def test_pow_two(self): + def main(n): + s = 0.123 + while n > 0: + s -= s ** 2 # ID: pow + n -= 1 + return s + log = self.run(main, [500]) + assert abs(log.result - main(500)) < 1e-9 + loop, = log.loops_by_filename(self.filepath) + assert loop.match_by_id("pow", """ + guard_not_invalidated(descr=...) + f38 = float_mul(f30, f30) + f39 = float_sub(f30, f38) + """) diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py --- a/pypy/module/sys/version.py +++ b/pypy/module/sys/version.py @@ -10,7 +10,7 @@ CPYTHON_VERSION = (2, 7, 1, "final", 42) #XXX # sync patchlevel.h CPYTHON_API_VERSION = 1013 #XXX # sync with include/modsupport.h -PYPY_VERSION = (1, 7, 1, "dev", 0) #XXX # sync patchlevel.h +PYPY_VERSION = (1, 8, 1, "dev", 0) #XXX # sync patchlevel.h if platform.name == 'msvc': COMPILER_INFO = 'MSC v.%d 32 bit' % (platform.version * 10 + 600) diff --git a/lib_pypy/numpypy/test/test_fromnumeric.py b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py rename from lib_pypy/numpypy/test/test_fromnumeric.py rename to pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py --- a/lib_pypy/numpypy/test/test_fromnumeric.py +++ b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py @@ -1,7 +1,7 @@ - from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest -class AppTestFromNumeric(BaseNumpyAppTest): + +class AppTestFromNumeric(BaseNumpyAppTest): def test_argmax(self): # tests taken from numpy/core/fromnumeric.py docstring from numpypy import array, arange, argmax @@ -18,12 +18,12 @@ from numpypy import array, arange, argmin a = arange(6).reshape((2,3)) assert argmin(a) == 0 - # assert (argmax(a, axis=0) == array([0, 0, 0])).all() - # assert (argmax(a, axis=1) == array([0, 0])).all() + #assert (argmin(a, axis=0) == array([0, 0, 0])).all() + #assert (argmin(a, axis=1) == array([0, 0])).all() b = arange(6) b[1] = 0 assert argmin(b) == 0 - + def test_shape(self): # tests taken from numpy/core/fromnumeric.py docstring from numpypy import array, identity, shape @@ -40,11 +40,11 @@ assert sum([0.5, 1.5])== 2.0 assert sum([[0, 1], [0, 5]]) == 6 # assert sum([0.5, 0.7, 0.2, 1.5], dtype=int32) == 1 - # assert (sum([[0, 1], [0, 5]], axis=0) == array([0, 6])).all() - # assert (sum([[0, 1], [0, 5]], axis=1) == array([1, 5])).all() + assert (sum([[0, 1], [0, 5]], axis=0) == array([0, 6])).all() + assert (sum([[0, 1], [0, 5]], axis=1) == array([1, 5])).all() # If the accumulator is too small, overflow occurs: # assert ones(128, dtype=int8).sum(dtype=int8) == -128 - + def test_amin(self): # tests taken from numpy/core/fromnumeric.py docstring from numpypy import array, arange, amin @@ -86,24 +86,54 @@ assert ndim([[1,2,3],[4,5,6]]) == 2 assert ndim(array([[1,2,3],[4,5,6]])) == 2 assert ndim(1) == 0 - + def test_rank(self): # tests taken from numpy/core/fromnumeric.py docstring from numpypy import array, rank assert rank([[1,2,3],[4,5,6]]) == 2 assert rank(array([[1,2,3],[4,5,6]])) == 2 assert rank(1) == 0 - + def test_var(self): from numpypy import array, var a = array([[1,2],[3,4]]) assert var(a) == 1.25 - # assert (np.var(a,0) == array([ 1., 1.])).all() - # assert (np.var(a,1) == array([ 0.25, 0.25])).all() + #assert (var(a,0) == array([ 1., 1.])).all() + #assert (var(a,1) == array([ 0.25, 0.25])).all() def test_std(self): from numpypy import array, std a = array([[1, 2], [3, 4]]) assert std(a) == 1.1180339887498949 - # assert (std(a, axis=0) == array([ 1., 1.])).all() - # assert (std(a, axis=1) == array([ 0.5, 0.5]).all() + #assert (std(a, axis=0) == array([ 1., 1.])).all() + #assert (std(a, axis=1) == array([ 0.5, 0.5])).all() + + def test_mean(self): + from numpypy import array, mean, arange + assert mean(array(range(5))) == 2.0 + assert mean(range(5)) == 2.0 + assert (mean(arange(10).reshape(5, 2), axis=0) == [4, 5]).all() + assert (mean(arange(10).reshape(5, 2), axis=1) == [0.5, 2.5, 4.5, 6.5, 8.5]).all() + + def test_reshape(self): + from numpypy import arange, array, dtype, reshape + a = arange(12) + b = reshape(a, (3, 4)) + assert b.shape == (3, 4) + a = range(12) + b = reshape(a, (3, 4)) + assert b.shape == (3, 4) + a = array(range(105)).reshape(3, 5, 7) + assert reshape(a, (1, -1)).shape == (1, 105) + assert reshape(a, (1, 1, -1)).shape == (1, 1, 105) + assert reshape(a, (-1, 1, 1)).shape == (105, 1, 1) + + def test_transpose(self): + from numpypy import arange, array, transpose, ones + x = arange(4).reshape((2,2)) + assert (transpose(x) == array([[0, 2],[1, 3]])).all() + # Once axes argument is implemented, add more tests + raises(NotImplementedError, "transpose(x, axes=(1, 0, 2))") + # x = ones((1, 2, 3)) + # assert transpose(x, (1, 0, 2)).shape == (2, 1, 3) + diff --git a/pypy/module/test_lib_pypy/test_datetime.py b/pypy/module/test_lib_pypy/test_datetime.py new file mode 100644 --- /dev/null +++ b/pypy/module/test_lib_pypy/test_datetime.py @@ -0,0 +1,24 @@ +"""Additional tests for datetime.""" + +import time +import datetime +import os + +def test_utcfromtimestamp(): + """Confirm that utcfromtimestamp and fromtimestamp give consistent results. + + Based on danchr's test script in https://bugs.pypy.org/issue986 + """ + try: + prev_tz = os.environ.get("TZ") + os.environ["TZ"] = "GMT" + for unused in xrange(100): + now = time.time() + delta = (datetime.datetime.utcfromtimestamp(now) - + datetime.datetime.fromtimestamp(now)) + assert delta.days * 86400 + delta.seconds == 0 + finally: + if prev_tz is None: + del os.environ["TZ"] + else: + os.environ["TZ"] = prev_tz diff --git a/pypy/module/test_lib_pypy/test_sqlite3.py b/pypy/module/test_lib_pypy/test_sqlite3.py new file mode 100644 --- /dev/null +++ b/pypy/module/test_lib_pypy/test_sqlite3.py @@ -0,0 +1,10 @@ +"""Tests for _sqlite3.py""" + +def test_list_ddl(): + """From issue996. Mostly just looking for lack of exceptions.""" + from sqlite3.dbapi2 import connect + connection = connect(':memory:') + cursor = connection.cursor() + cursor.execute('CREATE TABLE foo (bar INTEGER)') + result = list(cursor) + assert result == [] diff --git a/pypy/module/thread/ll_thread.py b/pypy/module/thread/ll_thread.py --- a/pypy/module/thread/ll_thread.py +++ b/pypy/module/thread/ll_thread.py @@ -4,7 +4,7 @@ import py from pypy.rlib import jit, rgc from pypy.rlib.debug import ll_assert -from pypy.rlib.objectmodel import we_are_translated +from pypy.rlib.objectmodel import we_are_translated, specialize from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.tool import rffi_platform from pypy.tool import autopath @@ -85,6 +85,7 @@ def allocate_lock(): return Lock(allocate_ll_lock()) + at specialize.arg(0) def ll_start_new_thread(func): ident = c_thread_start(func) if ident == -1: @@ -97,6 +98,7 @@ def get_ident(): return rffi.cast(lltype.Signed, c_thread_get_ident()) + at specialize.arg(0) def start_new_thread(x, y): """In RPython, no argument can be passed. You have to use global variables to pass information to the new thread. That's not very diff --git a/pypy/module/thread/test/test_ll_thread.py b/pypy/module/thread/test/test_ll_thread.py --- a/pypy/module/thread/test/test_ll_thread.py +++ b/pypy/module/thread/test/test_ll_thread.py @@ -34,72 +34,34 @@ use_threads = True def test_start_new_thread(self): - py.test.skip("xxx ideally, investigate why it fails randomly") - # xxx but in practice start_new_thread() is also tested by the - # next test, and it's a mess to test start_new_thread() without - # the proper GIL to protect the GC import time class State: pass state = State() - class Z: - def __init__(self, value): - self.value = value - def __del__(self): - state.freed_counter += 1 - - def bootstrap(): - state.my_thread_ident = state.z.ident = get_ident() - assert state.my_thread_ident == get_ident() - assert get_ident() == state.z.ident - state.seen_value = state.z.value - state.z = None - # I think that we would need here a memory barrier in order - # to make the test pass reliably. The issue is that the - # main thread may see 'state.done = 1' before seeing the - # effect of the other assignments done above. For now let's - # emulate the write barrier by doing a system call and - # waiting a bit... - time.sleep(0.012) - state.done = 1 - - def g(i): - state.z = Z(i) - return start_new_thread(bootstrap, ()) - g._dont_inline_ = True + def bootstrap1(): + state.my_thread_ident1 = get_ident() + def bootstrap2(): + state.my_thread_ident2 = get_ident() def f(): - main_ident = get_ident() - assert main_ident == get_ident() - state.freed_counter = 0 - for i in range(50): - state.done = 0 - state.seen_value = 0 - ident = g(i) - gc.collect() - willing_to_wait_more = 1000 - while not state.done: - willing_to_wait_more -= 1 - if not willing_to_wait_more: - raise Exception("thread didn't start?") - time.sleep(0.01) - assert state.my_thread_ident != main_ident - assert state.my_thread_ident == ident - assert state.seen_value == i - # try to force Boehm to do some freeing - for i in range(3): - gc.collect() - return state.freed_counter + state.my_thread_ident1 = get_ident() + state.my_thread_ident2 = get_ident() + start_new_thread(bootstrap1, ()) + start_new_thread(bootstrap2, ()) + willing_to_wait_more = 1000 + while (state.my_thread_ident1 == get_ident() or + state.my_thread_ident2 == get_ident()): + willing_to_wait_more -= 1 + if not willing_to_wait_more: + raise Exception("thread didn't start?") + time.sleep(0.01) + return 42 fn = self.getcompiled(f, []) - freed_counter = fn() - print freed_counter - if self.gcpolicy == 'boehm': - assert freed_counter > 0 - else: - assert freed_counter == 50 + res = fn() + assert res == 42 def test_gc_locking(self): import time @@ -202,6 +164,9 @@ def getcompiled(self, f, argtypes): return f + def test_start_new_thread(self): + py.test.skip("deadlocks occasionally -- why???") + class TestUsingBoehm(AbstractThreadTests): gcpolicy = 'boehm' diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py --- a/pypy/objspace/std/bytearrayobject.py +++ b/pypy/objspace/std/bytearrayobject.py @@ -624,8 +624,8 @@ if step == 1: assert start >= 0 - assert slicelength >= 0 - del items[start:start+slicelength] + if slicelength > 0: + del items[start:start+slicelength] else: n = len(items) i = start @@ -662,10 +662,11 @@ while i >= lim: items[i] = items[i-delta] i -= 1 - elif start >= 0: + elif delta == 0: + pass + else: + assert start >= 0 # start<0 is only possible with slicelength==0 del items[start:start+delta] - else: - assert delta==0 # start<0 is only possible with slicelength==0 elif len2 != slicelength: # No resize for extended slices raise operationerrfmt(space.w_ValueError, "attempt to " "assign sequence of size %d to extended slice of size %d", diff --git a/pypy/objspace/std/complexobject.py b/pypy/objspace/std/complexobject.py --- a/pypy/objspace/std/complexobject.py +++ b/pypy/objspace/std/complexobject.py @@ -8,6 +8,7 @@ from pypy.rlib.rbigint import rbigint from pypy.rlib.rfloat import ( formatd, DTSF_STR_PRECISION, isinf, isnan, copysign) +from pypy.rlib import jit import math @@ -129,10 +130,10 @@ ir = len * math.sin(phase) return W_ComplexObject(rr, ir) - def pow_int(self, n): - if n > 100 or n < -100: - return self.pow(W_ComplexObject(1.0 * n, 0.0)) - elif n > 0: + def pow_small_int(self, n): + if n >= 0: + if jit.isconstant(n) and n == 2: + return self.mul(self) return self.pow_positive_int(n) else: return w_one.div(self.pow_positive_int(-n)) @@ -217,10 +218,10 @@ def pow__Complex_Complex_ANY(space, w_complex, w_exponent, thirdArg): if not space.is_w(thirdArg, space.w_None): raise OperationError(space.w_ValueError, space.wrap('complex modulo')) - int_exponent = int(w_exponent.realval) try: - if w_exponent.imagval == 0.0 and w_exponent.realval == int_exponent: - w_p = w_complex.pow_int(int_exponent) + r = w_exponent.realval + if w_exponent.imagval == 0.0 and -100.0 <= r <= 100.0 and r == int(r): + w_p = w_complex.pow_small_int(int(r)) else: w_p = w_complex.pow(w_exponent) except ZeroDivisionError: diff --git a/pypy/objspace/std/floatobject.py b/pypy/objspace/std/floatobject.py --- a/pypy/objspace/std/floatobject.py +++ b/pypy/objspace/std/floatobject.py @@ -443,6 +443,8 @@ y = w_float2.floatval # Sort out special cases here instead of relying on pow() + if y == 2.0: # special case for performance: + return W_FloatObject(x * x) # x * x is always correct if y == 0.0: # x**0 is 1, even 0**0 return W_FloatObject(1.0) diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py --- a/pypy/objspace/std/listobject.py +++ b/pypy/objspace/std/listobject.py @@ -804,10 +804,11 @@ while i >= lim: items[i] = items[i-delta] i -= 1 - elif start >= 0: + elif delta == 0: + pass + else: + assert start >= 0 # start<0 is only possible with slicelength==0 del items[start:start+delta] - else: - assert delta==0 # start<0 is only possible with slicelength==0 elif len2 != slicelength: # No resize for extended slices raise operationerrfmt(self.space.w_ValueError, "attempt to " "assign sequence of size %d to extended slice of size %d", @@ -851,8 +852,8 @@ if step == 1: assert start >= 0 - assert slicelength >= 0 - del items[start:start+slicelength] + if slicelength > 0: + del items[start:start+slicelength] else: n = len(items) i = start diff --git a/pypy/objspace/std/test/test_bytearrayobject.py b/pypy/objspace/std/test/test_bytearrayobject.py --- a/pypy/objspace/std/test/test_bytearrayobject.py +++ b/pypy/objspace/std/test/test_bytearrayobject.py @@ -1,5 +1,9 @@ +from pypy import conftest class AppTestBytesArray: + def setup_class(cls): + cls.w_runappdirect = cls.space.wrap(conftest.option.runappdirect) + def test_basics(self): b = bytearray() assert type(b) is bytearray @@ -439,3 +443,15 @@ def test_reduce(self): assert bytearray('caf\xe9').__reduce__() == ( bytearray, (u'caf\xe9', 'latin-1'), None) + + def test_setitem_slice_performance(self): + # because of a complexity bug, this used to take forever on a + # translated pypy. On CPython2.6 -A, it takes around 8 seconds. + if self.runappdirect: + count = 16*1024*1024 + else: + count = 1024 + b = bytearray(count) + for i in range(count): + b[i:i+1] = 'y' + assert str(b) == 'y' * count diff --git a/pypy/objspace/std/test/test_complexobject.py b/pypy/objspace/std/test/test_complexobject.py --- a/pypy/objspace/std/test/test_complexobject.py +++ b/pypy/objspace/std/test/test_complexobject.py @@ -71,7 +71,7 @@ assert _powu((0.0,1.0),2) == (-1.0,0.0) def _powi((r1, i1), n): - w_res = W_ComplexObject(r1, i1).pow_int(n) + w_res = W_ComplexObject(r1, i1).pow_small_int(n) return w_res.realval, w_res.imagval assert _powi((0.0,2.0),0) == (1.0,0.0) assert _powi((0.0,0.0),2) == (0.0,0.0) @@ -213,6 +213,7 @@ assert a ** 105 == a ** 105 assert a ** -105 == a ** -105 assert a ** -30 == a ** -30 + assert a ** 2 == a * a assert 0.0j ** 0 == 1 diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py --- a/pypy/objspace/std/test/test_listobject.py +++ b/pypy/objspace/std/test/test_listobject.py @@ -397,6 +397,7 @@ on_cpython = (option.runappdirect and not hasattr(sys, 'pypy_translation_info')) cls.w_on_cpython = cls.space.wrap(on_cpython) + cls.w_runappdirect = cls.space.wrap(option.runappdirect) def test_getstrategyfromlist_w(self): l0 = ["a", "2", "a", True] @@ -898,6 +899,18 @@ l[::-1] = l assert l == [6,5,4,3,2,1] + def test_setitem_slice_performance(self): + # because of a complexity bug, this used to take forever on a + # translated pypy. On CPython2.6 -A, it takes around 5 seconds. + if self.runappdirect: + count = 16*1024*1024 + else: + count = 1024 + b = [None] * count + for i in range(count): + b[i:i+1] = ['y'] + assert b == ['y'] * count + def test_recursive_repr(self): l = [] assert repr(l) == '[]' diff --git a/pypy/objspace/std/test/test_unicodeobject.py b/pypy/objspace/std/test/test_unicodeobject.py --- a/pypy/objspace/std/test/test_unicodeobject.py +++ b/pypy/objspace/std/test/test_unicodeobject.py @@ -64,6 +64,12 @@ check(', '.join([u'a']), u'a') check(', '.join(['a', u'b']), u'a, b') check(u', '.join(['a', 'b']), u'a, b') + try: + u''.join([u'a', 2, 3]) + except TypeError, e: + assert 'sequence item 1' in str(e) + else: + raise Exception("DID NOT RAISE") if sys.version_info >= (2,3): def test_contains_ex(self): diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -201,7 +201,7 @@ return space.newbool(container.find(item) != -1) def unicode_join__Unicode_ANY(space, w_self, w_list): - list_w = space.unpackiterable(w_list) + list_w = space.listview(w_list) size = len(list_w) if size == 0: @@ -216,22 +216,21 @@ def _unicode_join_many_items(space, w_self, list_w, size): self = w_self._value - sb = UnicodeBuilder() + prealloc_size = len(self) * (size - 1) + for i in range(size): + try: + prealloc_size += len(space.unicode_w(list_w[i])) + except OperationError, e: + if not e.match(space, space.w_TypeError): + raise + raise operationerrfmt(space.w_TypeError, + "sequence item %d: expected string or Unicode", i) + sb = UnicodeBuilder(prealloc_size) for i in range(size): if self and i != 0: sb.append(self) w_s = list_w[i] - if isinstance(w_s, W_UnicodeObject): - # shortcut for performance - sb.append(w_s._value) - else: - try: - sb.append(space.unicode_w(w_s)) - except OperationError, e: - if not e.match(space, space.w_TypeError): - raise - raise operationerrfmt(space.w_TypeError, - "sequence item %d: expected string or Unicode", i) + sb.append(space.unicode_w(w_s)) return space.wrap(sb.build()) def hash__Unicode(space, w_uni): diff --git a/pypy/rlib/jit_hooks.py b/pypy/rlib/jit_hooks.py --- a/pypy/rlib/jit_hooks.py +++ b/pypy/rlib/jit_hooks.py @@ -52,7 +52,10 @@ from pypy.jit.metainterp.history import ResOperation args = [_cast_to_box(llargs[i]) for i in range(len(llargs))] - res = _cast_to_box(llres) + if llres: + res = _cast_to_box(llres) + else: + res = None return _cast_to_gcref(ResOperation(no, args, res)) @register_helper(annmodel.SomePtr(llmemory.GCREF)) diff --git a/pypy/rlib/longlong2float.py b/pypy/rlib/longlong2float.py --- a/pypy/rlib/longlong2float.py +++ b/pypy/rlib/longlong2float.py @@ -79,19 +79,23 @@ longlong2float = rffi.llexternal( "pypy__longlong2float", [rffi.LONGLONG], rffi.DOUBLE, _callable=longlong2float_emulator, compilation_info=eci, - _nowrapper=True, elidable_function=True, sandboxsafe=True) + _nowrapper=True, elidable_function=True, sandboxsafe=True, + oo_primitive="pypy__longlong2float") float2longlong = rffi.llexternal( "pypy__float2longlong", [rffi.DOUBLE], rffi.LONGLONG, _callable=float2longlong_emulator, compilation_info=eci, - _nowrapper=True, elidable_function=True, sandboxsafe=True) + _nowrapper=True, elidable_function=True, sandboxsafe=True, + oo_primitive="pypy__float2longlong") uint2singlefloat = rffi.llexternal( "pypy__uint2singlefloat", [rffi.UINT], rffi.FLOAT, _callable=uint2singlefloat_emulator, compilation_info=eci, - _nowrapper=True, elidable_function=True, sandboxsafe=True) + _nowrapper=True, elidable_function=True, sandboxsafe=True, + oo_primitive="pypy__uint2singlefloat") singlefloat2uint = rffi.llexternal( "pypy__singlefloat2uint", [rffi.FLOAT], rffi.UINT, _callable=singlefloat2uint_emulator, compilation_info=eci, - _nowrapper=True, elidable_function=True, sandboxsafe=True) + _nowrapper=True, elidable_function=True, sandboxsafe=True, + oo_primitive="pypy__singlefloat2uint") diff --git a/pypy/rlib/objectmodel.py b/pypy/rlib/objectmodel.py --- a/pypy/rlib/objectmodel.py +++ b/pypy/rlib/objectmodel.py @@ -420,7 +420,7 @@ vobj.concretetype.TO._gckind == 'gc') else: from pypy.rpython.ootypesystem import ootype - ok = isinstance(vobj.concretetype, ootype.Instance) + ok = isinstance(vobj.concretetype, (ootype.Instance, ootype.BuiltinType)) if not ok: from pypy.rpython.error import TyperError raise TyperError("compute_unique_id() cannot be applied to" diff --git a/pypy/rlib/rdynload.py b/pypy/rlib/rdynload.py --- a/pypy/rlib/rdynload.py +++ b/pypy/rlib/rdynload.py @@ -14,7 +14,7 @@ _MINGW = platform.name == "mingw32" _WIN32 = _MSVC or _MINGW _MAC_OS = platform.name == "darwin" -_FREEBSD = platform.name == "freebsd" +_FREEBSD = sys.platform.startswith("freebsd") if _WIN32: from pypy.rlib import rwin32 diff --git a/pypy/rlib/rsocket.py b/pypy/rlib/rsocket.py --- a/pypy/rlib/rsocket.py +++ b/pypy/rlib/rsocket.py @@ -310,10 +310,7 @@ def from_object(space, w_address): # Parse an app-level object representing an AF_INET address - try: - w_host, w_port = space.unpackiterable(w_address, 2) - except ValueError: - raise TypeError("AF_INET address must be a tuple of length 2") + w_host, w_port = space.unpackiterable(w_address, 2) host = space.str_w(w_host) port = space.int_w(w_port) port = Address.make_ushort_port(space, port) @@ -544,10 +541,7 @@ space.wrap(self.get_groups())]) def from_object(space, w_address): - try: - w_pid, w_groups = space.unpackiterable(w_address, 2) - except ValueError: - raise TypeError("AF_NETLINK address must be a tuple of length 2") + w_pid, w_groups = space.unpackiterable(w_address, 2) return NETLINKAddress(space.uint_w(w_pid), space.uint_w(w_groups)) from_object = staticmethod(from_object) diff --git a/pypy/rlib/rweakref.py b/pypy/rlib/rweakref.py --- a/pypy/rlib/rweakref.py +++ b/pypy/rlib/rweakref.py @@ -1,11 +1,14 @@ """ -Weakref support in RPython. Supports ref() without callbacks, +Weakref support in RPython. Basic regular weakrefs without callbacks +are supported. This file contains the following additions: a form of WeakKeyDictionary, and a limited version of WeakValueDictionary. LLType only for now! """ import weakref +ref = weakref.ref # basic regular weakrefs are supported in RPython + class RWeakValueDictionary(object): """A dictionary containing weak values.""" diff --git a/pypy/rpython/lltypesystem/ll2ctypes.py b/pypy/rpython/lltypesystem/ll2ctypes.py --- a/pypy/rpython/lltypesystem/ll2ctypes.py +++ b/pypy/rpython/lltypesystem/ll2ctypes.py @@ -1390,7 +1390,8 @@ def _where_is_errno(): return standard_c_lib.__errno_location() - elif sys.platform in ('darwin', 'freebsd7', 'freebsd8', 'freebsd9'): + elif any(plat in sys.platform + for plat in ('darwin', 'freebsd7', 'freebsd8', 'freebsd9')): standard_c_lib.__error.restype = ctypes.POINTER(ctypes.c_int) def _where_is_errno(): return standard_c_lib.__error() diff --git a/pypy/rpython/lltypesystem/rtuple.py b/pypy/rpython/lltypesystem/rtuple.py --- a/pypy/rpython/lltypesystem/rtuple.py +++ b/pypy/rpython/lltypesystem/rtuple.py @@ -27,6 +27,10 @@ def newtuple(cls, llops, r_tuple, items_v): # items_v should have the lowleveltype of the internal reprs + assert len(r_tuple.items_r) == len(items_v) + for r_item, v_item in zip(r_tuple.items_r, items_v): + assert r_item.lowleveltype == v_item.concretetype + # if len(r_tuple.items_r) == 0: return inputconst(Void, ()) # a Void empty tuple c1 = inputconst(Void, r_tuple.lowleveltype.TO) diff --git a/pypy/rpython/memory/gc/env.py b/pypy/rpython/memory/gc/env.py --- a/pypy/rpython/memory/gc/env.py +++ b/pypy/rpython/memory/gc/env.py @@ -116,7 +116,7 @@ def get_total_memory(): return get_total_memory_darwin(get_darwin_sysctl_signed('hw.memsize')) -elif sys.platform.startswith('freebsd'): +elif 'freebsd' in sys.platform: def get_total_memory(): return get_total_memory_darwin(get_darwin_sysctl_signed('hw.usermem')) diff --git a/pypy/rpython/module/ll_time.py b/pypy/rpython/module/ll_time.py --- a/pypy/rpython/module/ll_time.py +++ b/pypy/rpython/module/ll_time.py @@ -41,7 +41,7 @@ RUSAGE = platform.Struct('struct rusage', [('ru_utime', TIMEVAL), ('ru_stime', TIMEVAL)]) -if "freebsd" in sys.platform: +if sys.platform.startswith('freebsd'): libraries = ['compat'] else: libraries = [] diff --git a/pypy/rpython/ootypesystem/ootype.py b/pypy/rpython/ootypesystem/ootype.py --- a/pypy/rpython/ootypesystem/ootype.py +++ b/pypy/rpython/ootypesystem/ootype.py @@ -512,6 +512,7 @@ "ll_append_char": Meth([CHARTP], Void), "ll_append": Meth([STRINGTP], Void), "ll_build": Meth([], STRINGTP), + "ll_getlength": Meth([], Signed), }) self._setup_methods({}) @@ -1376,6 +1377,9 @@ def _cast_to_object(self): return make_object(self) + def _identityhash(self): + return object.__hash__(self) + class _string(_builtin_type): def __init__(self, STRING, value = ''): @@ -1543,6 +1547,9 @@ else: return make_unicode(u''.join(self._buf)) + def ll_getlength(self): + return self.ll_build().ll_strlen() + class _null_string_builder(_null_mixin(_string_builder), _string_builder): def __init__(self, STRING_BUILDER): self.__dict__["_TYPE"] = STRING_BUILDER diff --git a/pypy/rpython/ootypesystem/rbuilder.py b/pypy/rpython/ootypesystem/rbuilder.py --- a/pypy/rpython/ootypesystem/rbuilder.py +++ b/pypy/rpython/ootypesystem/rbuilder.py @@ -21,6 +21,10 @@ builder.ll_append_char(char) @staticmethod + def ll_getlength(builder): + return builder.ll_getlength() + + @staticmethod def ll_append(builder, string): builder.ll_append(string) diff --git a/pypy/rpython/rrange.py b/pypy/rpython/rrange.py --- a/pypy/rpython/rrange.py +++ b/pypy/rpython/rrange.py @@ -204,7 +204,10 @@ v_index = hop.gendirectcall(self.ll_getnextindex, v_enumerate) hop2 = hop.copy() hop2.args_r = [self.r_baseiter] + r_item_src = self.r_baseiter.r_list.external_item_repr + r_item_dst = hop.r_result.items_r[1] v_item = self.r_baseiter.rtype_next(hop2) + v_item = hop.llops.convertvar(v_item, r_item_src, r_item_dst) return hop.r_result.newtuple(hop.llops, hop.r_result, [v_index, v_item]) diff --git a/pypy/rpython/test/test_rbuilder.py b/pypy/rpython/test/test_rbuilder.py --- a/pypy/rpython/test/test_rbuilder.py +++ b/pypy/rpython/test/test_rbuilder.py @@ -124,9 +124,5 @@ pass class TestOOtype(BaseTestStringBuilder, OORtypeMixin): - def test_string_getlength(self): - py.test.skip("getlength(): not implemented on ootype") - def test_unicode_getlength(self): - py.test.skip("getlength(): not implemented on ootype") def test_append_charpsize(self): py.test.skip("append_charpsize(): not implemented on ootype") diff --git a/pypy/rpython/test/test_rbuiltin.py b/pypy/rpython/test/test_rbuiltin.py --- a/pypy/rpython/test/test_rbuiltin.py +++ b/pypy/rpython/test/test_rbuiltin.py @@ -463,6 +463,31 @@ assert x1 == intmask(x0) assert x3 == intmask(x2) + def test_id_on_builtins(self): + from pypy.rlib.objectmodel import compute_unique_id + from pypy.rlib.rstring import StringBuilder, UnicodeBuilder + def fn(): + return (compute_unique_id("foo"), + compute_unique_id(u"bar"), + compute_unique_id([1]), + compute_unique_id({"foo": 3}), + compute_unique_id(StringBuilder()), + compute_unique_id(UnicodeBuilder())) + res = self.interpret(fn, []) + for id in self.ll_unpack_tuple(res, 6): + assert isinstance(id, (int, r_longlong)) + + def test_uniqueness_of_id_on_strings(self): + from pypy.rlib.objectmodel import compute_unique_id + def fn(s1, s2): + return (compute_unique_id(s1), compute_unique_id(s2)) + + s1 = "foo" + s2 = ''.join(['f','oo']) + res = self.interpret(fn, [self.string_to_ll(s1), self.string_to_ll(s2)]) + i1, i2 = self.ll_unpack_tuple(res, 2) + assert i1 != i2 + def test_cast_primitive(self): from pypy.rpython.annlowlevel import LowLevelAnnotatorPolicy def llf(u): diff --git a/pypy/rpython/test/test_rrange.py b/pypy/rpython/test/test_rrange.py --- a/pypy/rpython/test/test_rrange.py +++ b/pypy/rpython/test/test_rrange.py @@ -169,6 +169,22 @@ res = self.interpret(fn, [2]) assert res == 789 + def test_enumerate_instances(self): + class A: + pass + def fn(n): + a = A() + b = A() + a.k = 10 + b.k = 20 + for i, x in enumerate([a, b]): + if i == n: + return x.k + return 5 + res = self.interpret(fn, [1]) + assert res == 20 + + class TestLLtype(BaseTestRrange, LLRtypeMixin): from pypy.rpython.lltypesystem import rrange diff --git a/pypy/tool/jitlogparser/parser.py b/pypy/tool/jitlogparser/parser.py --- a/pypy/tool/jitlogparser/parser.py +++ b/pypy/tool/jitlogparser/parser.py @@ -140,13 +140,15 @@ bytecode_name = None is_bytecode = True inline_level = None + has_dmp = False def parse_code_data(self, arg): m = re.search('<code object ([<>\w]+)[\.,] file \'(.+?)\'[\.,] line (\d+)> #(\d+) (\w+)', arg) if m is None: # a non-code loop, like StrLiteralSearch or something - self.bytecode_name = arg + if arg: + self.bytecode_name = arg else: self.name, self.filename, lineno, bytecode_no, self.bytecode_name = m.groups() self.startlineno = int(lineno) @@ -218,7 +220,7 @@ self.inputargs = inputargs self.chunks = chunks for chunk in self.chunks: - if chunk.filename is not None: + if chunk.bytecode_name is not None: self.startlineno = chunk.startlineno self.filename = chunk.filename self.name = chunk.name diff --git a/pypy/tool/jitlogparser/test/test_parser.py b/pypy/tool/jitlogparser/test/test_parser.py --- a/pypy/tool/jitlogparser/test/test_parser.py +++ b/pypy/tool/jitlogparser/test/test_parser.py @@ -283,3 +283,13 @@ assert loops[-1].count == 1234 assert loops[1].count == 123 assert loops[2].count == 12 + +def test_parse_nonpython(): + loop = parse(""" + [] + debug_merge_point(0, 'random') + debug_merge_point(0, '<code object f. file 'x.py'. line 2> #15 COMPARE_OP') + """) + f = Function.from_operations(loop.operations, LoopStorage()) + assert f.chunks[-1].filename == 'x.py' + assert f.filename is None diff --git a/pypy/tool/test/test_version.py b/pypy/tool/test/test_version.py --- a/pypy/tool/test/test_version.py +++ b/pypy/tool/test/test_version.py @@ -1,6 +1,22 @@ import os, sys import py -from pypy.tool.version import get_repo_version_info +from pypy.tool.version import get_repo_version_info, _get_hg_archive_version + +def test_hg_archival_version(tmpdir): + def version_for(name, **kw): + path = tmpdir.join(name) + path.write('\n'.join('%s: %s' % x for x in kw.items())) + return _get_hg_archive_version(str(path)) + + assert version_for('release', + tag='release-123', + node='000', + ) == ('PyPy', 'release-123', '000') + assert version_for('somebranch', + node='000', + branch='something', + ) == ('PyPy', 'something', '000') + def test_get_repo_version_info(): assert get_repo_version_info(None) diff --git a/pypy/tool/version.py b/pypy/tool/version.py --- a/pypy/tool/version.py +++ b/pypy/tool/version.py @@ -3,111 +3,139 @@ from subprocess import Popen, PIPE import pypy pypydir = os.path.dirname(os.path.abspath(pypy.__file__)) +pypyroot = os.path.dirname(pypydir) +default_retval = 'PyPy', '?', '?' + +def maywarn(err, repo_type='Mercurial'): + if not err: + return + + from pypy.tool.ansi_print import ansi_log + log = py.log.Producer("version") + py.log.setconsumer("version", ansi_log) + log.WARNING('Errors getting %s information: %s' % (repo_type, err)) def get_repo_version_info(hgexe=None): '''Obtain version information by invoking the 'hg' or 'git' commands.''' - # TODO: support extracting from .hg_archival.txt - - default_retval = 'PyPy', '?', '?' - pypyroot = os.path.abspath(os.path.join(pypydir, '..')) - - def maywarn(err, repo_type='Mercurial'): - if not err: - return - - from pypy.tool.ansi_print import ansi_log - log = py.log.Producer("version") - py.log.setconsumer("version", ansi_log) - log.WARNING('Errors getting %s information: %s' % (repo_type, err)) # Try to see if we can get info from Git if hgexe is not specified. if not hgexe: if os.path.isdir(os.path.join(pypyroot, '.git')): - gitexe = py.path.local.sysfind('git') - if gitexe: - try: - p = Popen( - [str(gitexe), 'rev-parse', 'HEAD'], - stdout=PIPE, stderr=PIPE, cwd=pypyroot - ) - except OSError, e: - maywarn(e, 'Git') - return default_retval - if p.wait() != 0: - maywarn(p.stderr.read(), 'Git') - return default_retval - revision_id = p.stdout.read().strip()[:12] - p = Popen( - [str(gitexe), 'describe', '--tags', '--exact-match'], - stdout=PIPE, stderr=PIPE, cwd=pypyroot - ) - if p.wait() != 0: - p = Popen( - [str(gitexe), 'branch'], stdout=PIPE, stderr=PIPE, - cwd=pypyroot - ) - if p.wait() != 0: - maywarn(p.stderr.read(), 'Git') - return 'PyPy', '?', revision_id - branch = '?' - for line in p.stdout.read().strip().split('\n'): - if line.startswith('* '): - branch = line[1:].strip() - if branch == '(no branch)': - branch = '?' - break - return 'PyPy', branch, revision_id - return 'PyPy', p.stdout.read().strip(), revision_id + return _get_git_version() # Fallback to trying Mercurial. if hgexe is None: hgexe = py.path.local.sysfind('hg') - if not os.path.isdir(os.path.join(pypyroot, '.hg')): + if os.path.isfile(os.path.join(pypyroot, '.hg_archival.txt')): + return _get_hg_archive_version(os.path.join(pypyroot, '.hg_archival.txt')) + elif not os.path.isdir(os.path.join(pypyroot, '.hg')): maywarn('Not running from a Mercurial repository!') return default_retval elif not hgexe: maywarn('Cannot find Mercurial command!') return default_retval else: - env = dict(os.environ) - # get Mercurial into scripting mode - env['HGPLAIN'] = '1' - # disable user configuration, extensions, etc. - env['HGRCPATH'] = os.devnull + return _get_hg_version(hgexe) - try: - p = Popen([str(hgexe), 'version', '-q'], - stdout=PIPE, stderr=PIPE, env=env) - except OSError, e: - maywarn(e) - return default_retval - if not p.stdout.read().startswith('Mercurial Distributed SCM'): - maywarn('command does not identify itself as Mercurial') - return default_retval +def _get_hg_version(hgexe): + env = dict(os.environ) + # get Mercurial into scripting mode + env['HGPLAIN'] = '1' + # disable user configuration, extensions, etc. + env['HGRCPATH'] = os.devnull - p = Popen([str(hgexe), 'id', '-i', pypyroot], + try: + p = Popen([str(hgexe), 'version', '-q'], stdout=PIPE, stderr=PIPE, env=env) - hgid = p.stdout.read().strip() + except OSError, e: + maywarn(e) + return default_retval + + if not p.stdout.read().startswith('Mercurial Distributed SCM'): + maywarn('command does not identify itself as Mercurial') + return default_retval + + p = Popen([str(hgexe), 'id', '-i', pypyroot], + stdout=PIPE, stderr=PIPE, env=env) + hgid = p.stdout.read().strip() + maywarn(p.stderr.read()) + if p.wait() != 0: + hgid = '?' + + p = Popen([str(hgexe), 'id', '-t', pypyroot], + stdout=PIPE, stderr=PIPE, env=env) + hgtags = [t for t in p.stdout.read().strip().split() if t != 'tip'] + maywarn(p.stderr.read()) + if p.wait() != 0: + hgtags = ['?'] + + if hgtags: + return 'PyPy', hgtags[0], hgid + else: + # use the branch instead + p = Popen([str(hgexe), 'id', '-b', pypyroot], + stdout=PIPE, stderr=PIPE, env=env) + hgbranch = p.stdout.read().strip() maywarn(p.stderr.read()) + + return 'PyPy', hgbranch, hgid + + +def _get_hg_archive_version(path): + fp = open(path) + try: + data = dict(x.split(': ', 1) for x in fp.read().splitlines()) + finally: + fp.close() + if 'tag' in data: + return 'PyPy', data['tag'], data['node'] + else: + return 'PyPy', data['branch'], data['node'] + + +def _get_git_version(): + #XXX: this function is a untested hack, + # so the git mirror tav made will work + gitexe = py.path.local.sysfind('git') + if not gitexe: + return default_retval + + try: + p = Popen( + [str(gitexe), 'rev-parse', 'HEAD'], + stdout=PIPE, stderr=PIPE, cwd=pypyroot + ) + except OSError, e: + maywarn(e, 'Git') + return default_retval + if p.wait() != 0: + maywarn(p.stderr.read(), 'Git') + return default_retval + revision_id = p.stdout.read().strip()[:12] + p = Popen( + [str(gitexe), 'describe', '--tags', '--exact-match'], + stdout=PIPE, stderr=PIPE, cwd=pypyroot + ) + if p.wait() != 0: + p = Popen( + [str(gitexe), 'branch'], stdout=PIPE, stderr=PIPE, + cwd=pypyroot + ) if p.wait() != 0: - hgid = '?' + maywarn(p.stderr.read(), 'Git') + return 'PyPy', '?', revision_id + branch = '?' + for line in p.stdout.read().strip().split('\n'): + if line.startswith('* '): + branch = line[1:].strip() + if branch == '(no branch)': + branch = '?' + break + return 'PyPy', branch, revision_id + return 'PyPy', p.stdout.read().strip(), revision_id - p = Popen([str(hgexe), 'id', '-t', pypyroot], - stdout=PIPE, stderr=PIPE, env=env) - hgtags = [t for t in p.stdout.read().strip().split() if t != 'tip'] - maywarn(p.stderr.read()) - if p.wait() != 0: - hgtags = ['?'] - if hgtags: - return 'PyPy', hgtags[0], hgid - else: - # use the branch instead - p = Popen([str(hgexe), 'id', '-b', pypyroot], - stdout=PIPE, stderr=PIPE, env=env) - hgbranch = p.stdout.read().strip() - maywarn(p.stderr.read()) - - return 'PyPy', hgbranch, hgid +if __name__ == '__main__': + print get_repo_version_info() diff --git a/pypy/translator/c/src/profiling.c b/pypy/translator/c/src/profiling.c --- a/pypy/translator/c/src/profiling.c +++ b/pypy/translator/c/src/profiling.c @@ -29,33 +29,33 @@ profiling_setup = 0; } } - -#elif defined(_WIN32) -#include <windows.h> - -DWORD_PTR base_affinity_mask; -int profiling_setup = 0; - -void pypy_setup_profiling() { - if (!profiling_setup) { - DWORD_PTR affinity_mask, system_affinity_mask; - GetProcessAffinityMask(GetCurrentProcess(), - &base_affinity_mask, &system_affinity_mask); - affinity_mask = 1; - /* Pick one cpu allowed by the system */ - if (system_affinity_mask) - while ((affinity_mask & system_affinity_mask) == 0) - affinity_mask <<= 1; - SetProcessAffinityMask(GetCurrentProcess(), affinity_mask); - profiling_setup = 1; - } -} - -void pypy_teardown_profiling() { - if (profiling_setup) { - SetProcessAffinityMask(GetCurrentProcess(), base_affinity_mask); - profiling_setup = 0; - } + +#elif defined(_WIN32) +#include <windows.h> + +DWORD_PTR base_affinity_mask; +int profiling_setup = 0; + +void pypy_setup_profiling() { + if (!profiling_setup) { + DWORD_PTR affinity_mask, system_affinity_mask; + GetProcessAffinityMask(GetCurrentProcess(), + &base_affinity_mask, &system_affinity_mask); + affinity_mask = 1; + /* Pick one cpu allowed by the system */ + if (system_affinity_mask) + while ((affinity_mask & system_affinity_mask) == 0) + affinity_mask <<= 1; + SetProcessAffinityMask(GetCurrentProcess(), affinity_mask); + profiling_setup = 1; + } +} + +void pypy_teardown_profiling() { + if (profiling_setup) { + SetProcessAffinityMask(GetCurrentProcess(), base_affinity_mask); + profiling_setup = 0; + } } #else diff --git a/pypy/translator/goal/translate.py b/pypy/translator/goal/translate.py --- a/pypy/translator/goal/translate.py +++ b/pypy/translator/goal/translate.py @@ -293,17 +293,18 @@ drv.exe_name = targetspec_dic['__name__'] + '-%(backend)s' # Double check to ensure we are not overwriting the current interpreter - try: - this_exe = py.path.local(sys.executable).new(ext='') - exe_name = drv.compute_exe_name() - samefile = this_exe.samefile(exe_name) - assert not samefile, ( - 'Output file %s is the currently running ' - 'interpreter (use --output=...)'% exe_name) - except EnvironmentError: - pass + goals = translateconfig.goals + if not goals or 'compile' in goals: + try: + this_exe = py.path.local(sys.executable).new(ext='') + exe_name = drv.compute_exe_name() + samefile = this_exe.samefile(exe_name) + assert not samefile, ( + 'Output file %s is the currently running ' + 'interpreter (use --output=...)'% exe_name) + except EnvironmentError: + pass - goals = translateconfig.goals try: drv.proceed(goals) finally: diff --git a/pypy/translator/jvm/builtin.py b/pypy/translator/jvm/builtin.py --- a/pypy/translator/jvm/builtin.py +++ b/pypy/translator/jvm/builtin.py @@ -84,6 +84,9 @@ (ootype.StringBuilder.__class__, "ll_build"): jvm.Method.v(jStringBuilder, "toString", (), jString), + (ootype.StringBuilder.__class__, "ll_getlength"): + jvm.Method.v(jStringBuilder, "length", (), jInt), + (ootype.String.__class__, "ll_hash"): jvm.Method.v(jString, "hashCode", (), jInt), diff --git a/pypy/translator/jvm/database.py b/pypy/translator/jvm/database.py --- a/pypy/translator/jvm/database.py +++ b/pypy/translator/jvm/database.py @@ -358,7 +358,7 @@ ootype.Unsigned:jvm.PYPYSERIALIZEUINT, ootype.SignedLongLong:jvm.LONGTOSTRINGL, ootype.UnsignedLongLong: jvm.PYPYSERIALIZEULONG, - ootype.Float:jvm.DOUBLETOSTRINGD, + ootype.Float:jvm.PYPYSERIALIZEDOUBLE, ootype.Bool:jvm.PYPYSERIALIZEBOOLEAN, ootype.Void:jvm.PYPYSERIALIZEVOID, ootype.Char:jvm.PYPYESCAPEDCHAR, diff --git a/pypy/translator/jvm/metavm.py b/pypy/translator/jvm/metavm.py --- a/pypy/translator/jvm/metavm.py +++ b/pypy/translator/jvm/metavm.py @@ -92,6 +92,7 @@ CASTS = { # FROM TO (ootype.Signed, ootype.UnsignedLongLong): jvm.I2L, + (ootype.Unsigned, ootype.UnsignedLongLong): jvm.I2L, (ootype.SignedLongLong, ootype.Signed): jvm.L2I, (ootype.UnsignedLongLong, ootype.Unsigned): jvm.L2I, (ootype.UnsignedLongLong, ootype.Signed): jvm.L2I, diff --git a/pypy/translator/jvm/opcodes.py b/pypy/translator/jvm/opcodes.py --- a/pypy/translator/jvm/opcodes.py +++ b/pypy/translator/jvm/opcodes.py @@ -101,6 +101,7 @@ 'jit_force_virtualizable': Ignore, 'jit_force_virtual': DoNothing, 'jit_force_quasi_immutable': Ignore, + 'jit_is_virtual': PushPrimitive(ootype.Bool, False), 'debug_assert': [], # TODO: implement? 'debug_start_traceback': Ignore, diff --git a/pypy/translator/jvm/src/pypy/PyPy.java b/pypy/translator/jvm/src/pypy/PyPy.java --- a/pypy/translator/jvm/src/pypy/PyPy.java +++ b/pypy/translator/jvm/src/pypy/PyPy.java @@ -283,6 +283,14 @@ } } + public double pypy__longlong2float(long l) { + return Double.longBitsToDouble(l); + } + + public long pypy__float2longlong(double d) { + return Double.doubleToRawLongBits(d); + } + public double ooparse_float(String s) { try { return Double.parseDouble(s); @@ -353,6 +361,19 @@ return "False"; } + public static String serialize_double(double d) { + if (Double.isNaN(d)) { + return "float(\"nan\")"; + } else if (Double.isInfinite(d)) { + if (d > 0) + return "float(\"inf\")"; + else + return "float(\"-inf\")"; + } else { + return Double.toString(d); + } + } + private static String format_char(char c) { String res = "\\x"; if (c <= 0x0F) res = res + "0"; diff --git a/pypy/translator/jvm/test/runtest.py b/pypy/translator/jvm/test/runtest.py --- a/pypy/translator/jvm/test/runtest.py +++ b/pypy/translator/jvm/test/runtest.py @@ -56,6 +56,7 @@ # CLI could-be duplicate class JvmGeneratedSourceWrapper(object): + def __init__(self, gensrc): """ gensrc is an instance of JvmGeneratedSource """ self.gensrc = gensrc diff --git a/pypy/translator/jvm/test/test_builder.py b/pypy/translator/jvm/test/test_builder.py new file mode 100644 --- /dev/null +++ b/pypy/translator/jvm/test/test_builder.py @@ -0,0 +1,7 @@ +from pypy.translator.jvm.test.runtest import JvmTest +from pypy.rpython.test.test_rbuilder import BaseTestStringBuilder +import py + +class TestJvmStringBuilder(JvmTest, BaseTestStringBuilder): + def test_append_charpsize(self): + py.test.skip("append_charpsize(): not implemented on ootype") diff --git a/pypy/translator/jvm/test/test_longlong2float.py b/pypy/translator/jvm/test/test_longlong2float.py new file mode 100644 --- /dev/null +++ b/pypy/translator/jvm/test/test_longlong2float.py @@ -0,0 +1,20 @@ +from pypy.translator.jvm.test.runtest import JvmTest +from pypy.rlib.longlong2float import * +from pypy.rlib.test.test_longlong2float import enum_floats +from pypy.rlib.test.test_longlong2float import fn as float2longlong2float +import py + +class TestLongLong2Float(JvmTest): + + def test_float2longlong_and_longlong2float(self): + def func(f): + return float2longlong2float(f) + + for f in enum_floats(): + assert repr(f) == repr(self.interpret(func, [f])) + + def test_uint2singlefloat(self): + py.test.skip("uint2singlefloat is not implemented in ootype") + + def test_singlefloat2uint(self): + py.test.skip("singlefloat2uint is not implemented in ootype") diff --git a/pypy/translator/jvm/typesystem.py b/pypy/translator/jvm/typesystem.py --- a/pypy/translator/jvm/typesystem.py +++ b/pypy/translator/jvm/typesystem.py @@ -955,6 +955,7 @@ PYPYSERIALIZEUINT = Method.s(jPyPy, 'serialize_uint', (jInt,), jString) PYPYSERIALIZEULONG = Method.s(jPyPy, 'serialize_ulonglong', (jLong,),jString) PYPYSERIALIZEVOID = Method.s(jPyPy, 'serialize_void', (), jString) +PYPYSERIALIZEDOUBLE = Method.s(jPyPy, 'serialize_double', (jDouble,), jString) PYPYESCAPEDCHAR = Method.s(jPyPy, 'escaped_char', (jChar,), jString) PYPYESCAPEDUNICHAR = Method.s(jPyPy, 'escaped_unichar', (jChar,), jString) PYPYESCAPEDSTRING = Method.s(jPyPy, 'escaped_string', (jString,), jString) diff --git a/pypy/translator/oosupport/test_template/cast.py b/pypy/translator/oosupport/test_template/cast.py --- a/pypy/translator/oosupport/test_template/cast.py +++ b/pypy/translator/oosupport/test_template/cast.py @@ -13,6 +13,9 @@ def to_longlong(x): return r_longlong(x) +def to_ulonglong(x): + return r_ulonglong(x) + def uint_to_int(x): return intmask(x) @@ -56,6 +59,9 @@ def test_unsignedlonglong_to_unsigned4(self): self.check(to_uint, [r_ulonglong(18446744073709551615l)]) # max 64 bit num + def test_unsigned_to_usignedlonglong(self): + self.check(to_ulonglong, [r_uint(42)]) + def test_uint_to_int(self): self.check(uint_to_int, [r_uint(sys.maxint+1)]) diff --git a/pypy/translator/platform/freebsd.py b/pypy/translator/platform/freebsd.py --- a/pypy/translator/platform/freebsd.py +++ b/pypy/translator/platform/freebsd.py @@ -18,8 +18,9 @@ class Freebsd(posix.BasePosix): name = "freebsd" - link_flags = get_env_vector("LDFLAGS", '-pthread') - cflags = get_env_vector("CFLAGS", "-O3 -pthread -fomit-frame-pointer") + link_flags = ['-pthread'] + get_env_vector('LDFLAGS', '') + cflags = ['-O3', '-pthread', '-fomit-frame-pointer' + ] + get_env_vector('CFLAGS', '') standalone_only = [] shared_only = [] so_ext = 'so' diff --git a/pypy/translator/platform/linux.py b/pypy/translator/platform/linux.py --- a/pypy/translator/platform/linux.py +++ b/pypy/translator/platform/linux.py @@ -1,15 +1,20 @@ """Support for Linux.""" +import os import sys from pypy.translator.platform.posix import BasePosix class BaseLinux(BasePosix): name = "linux" - link_flags = ('-pthread',) + link_flags = tuple( + ['-pthread',] + + os.environ.get('LDFLAGS', '').split()) extra_libs = ('-lrt',) - cflags = ('-O3', '-pthread', '-fomit-frame-pointer', - '-Wall', '-Wno-unused') + cflags = tuple( + ['-O3', '-pthread', '-fomit-frame-pointer', + '-Wall', '-Wno-unused'] + + os.environ.get('CFLAGS', '').split()) standalone_only = () shared_only = ('-fPIC',) so_ext = 'so' From noreply at buildbot.pypy.org Wed Jan 25 21:50:05 2012 From: noreply at buildbot.pypy.org (mattip) Date: Wed, 25 Jan 2012 21:50:05 +0100 (CET) Subject: [pypy-commit] pypy numppy-flatitter: added signature, all tests now pass Message-ID: <20120125205005.DC344829FF@wyvern.cs.uni-duesseldorf.de> Author: mattip Branch: numppy-flatitter Changeset: r51769:dd14fbe5d0a4 Date: 2012-01-25 22:34 +0200 http://bitbucket.org/pypy/pypy/changeset/dd14fbe5d0a4/ Log: added signature, all tests now pass diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -898,7 +898,6 @@ self.size = 1 for s in self.shape: self.size *= s - xxx def _del_sources(self): self.left = None @@ -1514,6 +1513,9 @@ ai = (ai + 1) % arr.size basei = basei.next_skip_x(shapelen, step) + def create_sig(self): + return signature.FlatSignature(self.base.dtype) + def descr_base(self, space): return space.wrap(self.base) diff --git a/pypy/module/micronumpy/signature.py b/pypy/module/micronumpy/signature.py --- a/pypy/module/micronumpy/signature.py +++ b/pypy/module/micronumpy/signature.py @@ -224,6 +224,18 @@ return ViewIterator(arr.start, arr.strides, arr.backstrides, arr.shape).apply_transformations(arr, transforms) +class FlatSignature(ViewSignature): + def debug_repr(self): + return 'Flat' + + def allocate_iter(self, arr, transforms): + from pypy.module.micronumpy.interp_numarray import W_FlatIterator + assert isinstance(arr, W_FlatIterator) + return ViewIterator(arr.base.start, arr.base.strides, + arr.base.backstrides, + arr.base.shape).apply_transformations(arr.base, + transforms) + class VirtualSliceSignature(Signature): def __init__(self, child): self.child = child diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -962,7 +962,7 @@ assert debug_repr(a + a) == 'Call2(add, Array, Array)' assert debug_repr(a[::2]) == 'Slice' assert debug_repr(a + 2) == 'Call2(add, Array, Scalar)' - assert debug_repr(a + a.flat) == 'Call2(add, Array, Slice)' + assert debug_repr(a + a.flat) == 'Call2(add, Array, Flat)' assert debug_repr(sin(a)) == 'Call1(sin, Array)' b = a + a From noreply at buildbot.pypy.org Wed Jan 25 21:50:07 2012 From: noreply at buildbot.pypy.org (mattip) Date: Wed, 25 Jan 2012 21:50:07 +0100 (CET) Subject: [pypy-commit] pypy numppy-flatitter: translation passes Message-ID: <20120125205007.2169D829FF@wyvern.cs.uni-duesseldorf.de> Author: mattip Branch: numppy-flatitter Changeset: r51770:35b17e8dce50 Date: 2012-01-25 22:49 +0200 http://bitbucket.org/pypy/pypy/changeset/35b17e8dce50/ Log: translation passes diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -1438,8 +1438,7 @@ if i != 0: raise OperationError(space.w_IndexError, space.wrap("invalid index")) - retval = [coords, start, step, lngth] - return retval + return coords, start, step, lngth class W_FlatIterator(ViewArray): @@ -1469,7 +1468,7 @@ return space.wrap(self.index) def descr_coords(self, space): - coords = _to_coords(space, self.base, space.wrap(self.index))[0] + coords, start, step, lngth = _to_coords(space, self.base, space.wrap(self.index)) return space.newtuple([space.wrap(c) for c in coords]) def descr_getitem(self, space, w_idx): From noreply at buildbot.pypy.org Thu Jan 26 10:00:13 2012 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 26 Jan 2012 10:00:13 +0100 (CET) Subject: [pypy-commit] pypy default: Add some reprs to Symbolic subclasses, and show them in the error Message-ID: <20120126090013.C783182B1F@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: Changeset: r51771:7b1eec04c0f9 Date: 2012-01-26 09:52 +0100 http://bitbucket.org/pypy/pypy/changeset/7b1eec04c0f9/ Log: Add some reprs to Symbolic subclasses, and show them in the error messages. diff --git a/pypy/rlib/objectmodel.py b/pypy/rlib/objectmodel.py --- a/pypy/rlib/objectmodel.py +++ b/pypy/rlib/objectmodel.py @@ -130,19 +130,23 @@ if self is other: return 0 else: - raise TypeError("Symbolics can not be compared!") + raise TypeError("Symbolics can not be compared! (%r, %r)" + % (self, other)) def __hash__(self): - raise TypeError("Symbolics are not hashable!") + raise TypeError("Symbolics are not hashable! %r" % (self,)) def __nonzero__(self): - raise TypeError("Symbolics are not comparable") + raise TypeError("Symbolics are not comparable! %r" % (self,)) class ComputedIntSymbolic(Symbolic): def __init__(self, compute_fn): self.compute_fn = compute_fn + def __repr__(self): + return '%s(%r)' % (self.__class__.__name__, self.compute_fn) + def annotation(self): from pypy.annotation import model return model.SomeInteger() @@ -157,6 +161,9 @@ self.expr = expr self.default = default + def __repr__(self): + return '%s(%r)' % (self.__class__.__name__, self.expr) + def annotation(self): from pypy.annotation import model return model.SomeInteger() diff --git a/pypy/rpython/lltypesystem/llgroup.py b/pypy/rpython/lltypesystem/llgroup.py --- a/pypy/rpython/lltypesystem/llgroup.py +++ b/pypy/rpython/lltypesystem/llgroup.py @@ -76,6 +76,10 @@ self.index = memberindex self.member = grp.members[memberindex]._as_ptr() + def __repr__(self): + return '%s(%s, %s)' % (self.__class__.__name__, + self.grpptr, self.index) + def __nonzero__(self): return True diff --git a/pypy/rpython/lltypesystem/rffi.py b/pypy/rpython/lltypesystem/rffi.py --- a/pypy/rpython/lltypesystem/rffi.py +++ b/pypy/rpython/lltypesystem/rffi.py @@ -27,6 +27,10 @@ self.c_name = c_name self.TP = TP + def __repr__(self): + return '%s(%r, %s)' % (self.__class__.__name__, + self.c_name, self.TP) + def annotation(self): return lltype_to_annotation(self.TP) From noreply at buildbot.pypy.org Thu Jan 26 10:00:15 2012 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 26 Jan 2012 10:00:15 +0100 (CET) Subject: [pypy-commit] pypy default: Fix. Message-ID: <20120126090015.01ED182B1F@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: Changeset: r51772:66a54c1c81de Date: 2012-01-26 09:59 +0100 http://bitbucket.org/pypy/pypy/changeset/66a54c1c81de/ Log: Fix. diff --git a/pypy/rlib/objectmodel.py b/pypy/rlib/objectmodel.py --- a/pypy/rlib/objectmodel.py +++ b/pypy/rlib/objectmodel.py @@ -145,7 +145,13 @@ self.compute_fn = compute_fn def __repr__(self): - return '%s(%r)' % (self.__class__.__name__, self.compute_fn) + # repr(self.compute_fn) can arrive back here in an + # infinite recursion + try: + name = self.compute_fn.__name__ + except (AttributeError, TypeError): + name = hex(id(self.compute_fn)) + return '%s(%r)' % (self.__class__.__name__, name) def annotation(self): from pypy.annotation import model From noreply at buildbot.pypy.org Thu Jan 26 10:45:16 2012 From: noreply at buildbot.pypy.org (bivab) Date: Thu, 26 Jan 2012 10:45:16 +0100 (CET) Subject: [pypy-commit] pypy arm-backend-2: remove some XXX and update some comments Message-ID: <20120126094516.B3A2882B1F@wyvern.cs.uni-duesseldorf.de> Author: David Schneider <david.schneider at picle.org> Branch: arm-backend-2 Changeset: r51773:1aea54f9875c Date: 2012-01-23 12:29 +0100 http://bitbucket.org/pypy/pypy/changeset/1aea54f9875c/ Log: remove some XXX and update some comments diff --git a/pypy/jit/backend/arm/assembler.py b/pypy/jit/backend/arm/assembler.py --- a/pypy/jit/backend/arm/assembler.py +++ b/pypy/jit/backend/arm/assembler.py @@ -32,8 +32,8 @@ from pypy.jit.backend.x86.support import values_array, memcpy_fn DEBUG_COUNTER = lltype.Struct('DEBUG_COUNTER', ('i', lltype.Signed), - ('type', lltype.Char), # 'b'ridge, 'l'abel or - # 'e'ntry point + ('type', lltype.Char), # 'b'ridge, 'l'abel or + # 'e'ntry point ('number', lltype.Signed)) @@ -384,7 +384,6 @@ # are stored in r0 and r1. mc.SUB_rr(r.r0.value, r.r1.value, r.r0.value) addr = self.cpu.gc_ll_descr.get_malloc_slowpath_addr() - # XXX replace with an STMxx operation for reg, ofs in ARMv7RegisterManager.REGLOC_TO_COPY_AREA_OFS.items(): mc.STR_ri(reg.value, r.fp.value, imm=ofs) mc.BL(addr) @@ -531,8 +530,7 @@ def gen_shadowstack_header(self, gcrootmap): # we need to put two words into the shadowstack: the MARKER - # and the address of the frame (ebp, actually) - # XXX add some comments + # and the address of the frame (fp, actually) rst = gcrootmap.get_root_stack_top_addr() self.mc.gen_load_int(r.ip.value, rst) self.mc.LDR_ri(r.r4.value, r.ip.value) # LDR r4, [rootstacktop] @@ -1121,23 +1119,18 @@ self.mc.gen_load_int(r.r1.value, size) self.mc.ADD_rr(r.r1.value, r.r0.value, r.r1.value) - # XXX maybe use an offset from the value nursery_free_addr self.mc.gen_load_int(r.ip.value, nursery_top_adr) self.mc.LDR_ri(r.ip.value, r.ip.value) self.mc.CMP_rr(r.r1.value, r.ip.value) - # XXX update - # See comments in _build_malloc_slowpath for the - # details of the two helper functions that we are calling below. - # First, we need to call two of them and not just one because we - # need to have a mark_gc_roots() in between. Then the calling - # convention of slowpath_addr{1,2} are tweaked a lot to allow - # the code here to be just two CALLs: slowpath_addr1 gets the - # size of the object to allocate from (EDX-EAX) and returns the - # result in EAX; self.malloc_slowpath additionally returns in EDX a - # copy of heap(nursery_free_adr), so that the final MOV below is - # a no-op. + # We load into r0 the address stored at nursery_free_adr We calculate + # the new value for nursery_free_adr and store in r1 The we load the + # address stored in nursery_top_adr into IP If the value in r1 is + # (unsigned) bigger than the one in ip we conditionally call + # malloc_slowpath in case we called malloc_slowpath, which returns the + # new value of nursery_free_adr in r1 and the adr of the new object in + # r0. self.mark_gc_roots(self.write_new_force_index(), use_copy_area=True) self.mc.BL(self.malloc_slowpath, c=c.HI) diff --git a/pypy/jit/backend/arm/opassembler.py b/pypy/jit/backend/arm/opassembler.py --- a/pypy/jit/backend/arm/opassembler.py +++ b/pypy/jit/backend/arm/opassembler.py @@ -1158,8 +1158,8 @@ def call_release_gil(self, gcrootmap, save_registers, fcond): # First, we need to save away the registers listed in - # 'save_registers' that are not callee-save. XXX We assume that - # the floating point registers won't be modified. + # 'save_registers' that are not callee-save. + # NOTE: We assume that the floating point registers won't be modified. regs_to_save = [] for reg in self._regalloc.rm.save_around_call_regs: if reg in save_registers: @@ -1170,8 +1170,8 @@ def call_reacquire_gil(self, gcrootmap, save_loc, fcond): # save the previous result into the stack temporarily. - # XXX like with call_release_gil(), we assume that we don't need - # to save vfp regs in this case. Besides the result location + # NOTE: like with call_release_gil(), we assume that we don't need to + # save vfp regs in this case. Besides the result location regs_to_save = [] vfp_regs_to_save = [] if save_loc.is_reg(): diff --git a/pypy/jit/backend/arm/regalloc.py b/pypy/jit/backend/arm/regalloc.py --- a/pypy/jit/backend/arm/regalloc.py +++ b/pypy/jit/backend/arm/regalloc.py @@ -66,9 +66,6 @@ def __init__(self): FrameManager.__init__(self) - #self.used = [True] # keep first slot free - # XXX refactor frame to avoid this issue of keeping the first slot - # reserved @staticmethod def frame_pos(i, box_type): @@ -1009,7 +1006,6 @@ return [res_loc] def prepare_op_label(self, op, fcond): - # XXX big refactoring needed? descr = op.getdescr() assert isinstance(descr, TargetToken) inputargs = op.getarglist() From noreply at buildbot.pypy.org Thu Jan 26 10:45:17 2012 From: noreply at buildbot.pypy.org (bivab) Date: Thu, 26 Jan 2012 10:45:17 +0100 (CET) Subject: [pypy-commit] pypy arm-backend-2: implement the BX operation for jumps Message-ID: <20120126094517.EF8BA82B1F@wyvern.cs.uni-duesseldorf.de> Author: David Schneider <david.schneider at picle.org> Branch: arm-backend-2 Changeset: r51774:b2886fd16756 Date: 2012-01-23 14:45 +0100 http://bitbucket.org/pypy/pypy/changeset/b2886fd16756/ Log: implement the BX operation for jumps diff --git a/pypy/jit/backend/arm/codebuilder.py b/pypy/jit/backend/arm/codebuilder.py --- a/pypy/jit/backend/arm/codebuilder.py +++ b/pypy/jit/backend/arm/codebuilder.py @@ -163,12 +163,11 @@ self.write32(cond << 28 | 0xEF1FA10) def B(self, target, c=cond.AL): - if c == cond.AL: - self.LDR_ri(reg.pc.value, reg.pc.value, -arch.PC_OFFSET / 2) - self.write32(target) - else: - self.gen_load_int(reg.ip.value, target, cond=c) - self.MOV_rr(reg.pc.value, reg.ip.value, cond=c) + self.gen_load_int(reg.ip.value, target, cond=c) + self.BX(reg.ip.value, c=c) + + def BX(self, reg, c=cond.AL): + self.write32(c << 28 | 0x12FFF1 << 4 | (reg & 0xF)) def B_offs(self, target_ofs, c=cond.AL): pos = self.currpos() From noreply at buildbot.pypy.org Thu Jan 26 10:45:19 2012 From: noreply at buildbot.pypy.org (bivab) Date: Thu, 26 Jan 2012 10:45:19 +0100 (CET) Subject: [pypy-commit] pypy arm-backend-2: backout b2fff2c474ff Message-ID: <20120126094519.3AD9482B1F@wyvern.cs.uni-duesseldorf.de> Author: David Schneider <david.schneider at picle.org> Branch: arm-backend-2 Changeset: r51775:25e1511591e1 Date: 2012-01-23 14:47 +0100 http://bitbucket.org/pypy/pypy/changeset/25e1511591e1/ Log: backout b2fff2c474ff diff --git a/pypy/jit/backend/x86/test/test_runner.py b/pypy/jit/backend/x86/test/test_runner.py --- a/pypy/jit/backend/x86/test/test_runner.py +++ b/pypy/jit/backend/x86/test/test_runner.py @@ -582,52 +582,3 @@ self.cpu.execute_token(looptoken, 0) assert looptoken._x86_debug_checksum == sum([op.getopnum() for op in ops.operations]) - - def test_compile_asmlen(self): - from pypy.jit.backend.llsupport.llmodel import AbstractLLCPU - if not isinstance(self.cpu, AbstractLLCPU): - py.test.skip("pointless test on non-asm") - from pypy.jit.backend.x86.tool.viewcode import machine_code_dump - import ctypes - ops = """ - [i2] - i0 = same_as(i2) # but forced to be in a register - label(i0, descr=1) - i1 = int_add(i0, i0) - guard_true(i1, descr=faildesr) [i1] - jump(i1, descr=1) - """ - faildescr = BasicFailDescr(2) - loop = parse(ops, self.cpu, namespace=locals()) - faildescr = loop.operations[-2].getdescr() - jumpdescr = loop.operations[-1].getdescr() - bridge_ops = """ - [i0] - jump(i0, descr=jumpdescr) - """ - bridge = parse(bridge_ops, self.cpu, namespace=locals()) - looptoken = JitCellToken() - self.cpu.assembler.set_debug(False) - info = self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken) - bridge_info = self.cpu.compile_bridge(faildescr, bridge.inputargs, - bridge.operations, - looptoken) - self.cpu.assembler.set_debug(True) # always on untranslated - assert info.asmlen != 0 - cpuname = autodetect_main_model_and_size() - # XXX we have to check the precise assembler, otherwise - # we don't quite know if borders are correct - - def checkops(mc, ops): - assert len(mc) == len(ops) - for i in range(len(mc)): - assert mc[i].split("\t")[-1].startswith(ops[i]) - - data = ctypes.string_at(info.asmaddr, info.asmlen) - mc = list(machine_code_dump(data, info.asmaddr, cpuname)) - lines = [line for line in mc if line.count('\t') == 2] - checkops(lines, self.add_loop_instructions) - data = ctypes.string_at(bridge_info.asmaddr, bridge_info.asmlen) - mc = list(machine_code_dump(data, bridge_info.asmaddr, cpuname)) - lines = [line for line in mc if line.count('\t') == 2] - checkops(lines, self.bridge_loop_instructions) From noreply at buildbot.pypy.org Thu Jan 26 10:45:20 2012 From: noreply at buildbot.pypy.org (bivab) Date: Thu, 26 Jan 2012 10:45:20 +0100 (CET) Subject: [pypy-commit] pypy arm-backend-2: Backout 839659291f03 Message-ID: <20120126094520.713B082B1F@wyvern.cs.uni-duesseldorf.de> Author: David Schneider <david.schneider at picle.org> Branch: arm-backend-2 Changeset: r51776:e61421452c3f Date: 2012-01-23 14:48 +0100 http://bitbucket.org/pypy/pypy/changeset/e61421452c3f/ Log: Backout 839659291f03 diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py --- a/pypy/jit/backend/test/runner_test.py +++ b/pypy/jit/backend/test/runner_test.py @@ -3216,6 +3216,55 @@ res = self.cpu.get_latest_value_int(0) assert res == -10 + def test_compile_asmlen(self): + from pypy.jit.backend.llsupport.llmodel import AbstractLLCPU + if not isinstance(self.cpu, AbstractLLCPU): + py.test.skip("pointless test on non-asm") + from pypy.jit.backend.x86.tool.viewcode import machine_code_dump + import ctypes + ops = """ + [i2] + i0 = same_as(i2) # but forced to be in a register + label(i0, descr=1) + i1 = int_add(i0, i0) + guard_true(i1, descr=faildesr) [i1] + jump(i1, descr=1) + """ + faildescr = BasicFailDescr(2) + loop = parse(ops, self.cpu, namespace=locals()) + faildescr = loop.operations[-2].getdescr() + jumpdescr = loop.operations[-1].getdescr() + bridge_ops = """ + [i0] + jump(i0, descr=jumpdescr) + """ + bridge = parse(bridge_ops, self.cpu, namespace=locals()) + looptoken = JitCellToken() + self.cpu.assembler.set_debug(False) + info = self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken) + bridge_info = self.cpu.compile_bridge(faildescr, bridge.inputargs, + bridge.operations, + looptoken) + self.cpu.assembler.set_debug(True) # always on untranslated + assert info.asmlen != 0 + cpuname = autodetect_main_model_and_size() + # XXX we have to check the precise assembler, otherwise + # we don't quite know if borders are correct + + def checkops(mc, ops): + assert len(mc) == len(ops) + for i in range(len(mc)): + assert mc[i].split("\t")[-1].startswith(ops[i]) + + data = ctypes.string_at(info.asmaddr, info.asmlen) + mc = list(machine_code_dump(data, info.asmaddr, cpuname)) + lines = [line for line in mc if line.count('\t') == 2] + checkops(lines, self.add_loop_instructions) + data = ctypes.string_at(bridge_info.asmaddr, bridge_info.asmlen) + mc = list(machine_code_dump(data, bridge_info.asmaddr, cpuname)) + lines = [line for line in mc if line.count('\t') == 2] + checkops(lines, self.bridge_loop_instructions) + def test_compile_bridge_with_target(self): # This test creates a loopy piece of code in a bridge, and builds another From noreply at buildbot.pypy.org Thu Jan 26 10:45:21 2012 From: noreply at buildbot.pypy.org (bivab) Date: Thu, 26 Jan 2012 10:45:21 +0100 (CET) Subject: [pypy-commit] pypy arm-backend-2: refactor test_compile_asmlen a bit to make it work the ARM backend Message-ID: <20120126094521.A48B582B1F@wyvern.cs.uni-duesseldorf.de> Author: David Schneider <david.schneider at picle.org> Branch: arm-backend-2 Changeset: r51777:fb43a79417b7 Date: 2012-01-23 17:06 +0100 http://bitbucket.org/pypy/pypy/changeset/fb43a79417b7/ Log: refactor test_compile_asmlen a bit to make it work the ARM backend diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py --- a/pypy/jit/backend/test/runner_test.py +++ b/pypy/jit/backend/test/runner_test.py @@ -3174,7 +3174,6 @@ 'float', descr=calldescr) assert res.getfloat() == expected - def test_compile_loop_with_target(self): i0 = BoxInt() i1 = BoxInt() @@ -3220,10 +3219,10 @@ from pypy.jit.backend.llsupport.llmodel import AbstractLLCPU if not isinstance(self.cpu, AbstractLLCPU): py.test.skip("pointless test on non-asm") - from pypy.jit.backend.x86.tool.viewcode import machine_code_dump + machine_code_dump = self.get_machine_code_dump_func() import ctypes ops = """ - [i2] + [i3, i2] i0 = same_as(i2) # but forced to be in a register label(i0, descr=1) i1 = int_add(i0, i0) @@ -3254,18 +3253,17 @@ def checkops(mc, ops): assert len(mc) == len(ops) for i in range(len(mc)): - assert mc[i].split("\t")[-1].startswith(ops[i]) - + assert mc[i].split("\t")[2].startswith(ops[i]) + data = ctypes.string_at(info.asmaddr, info.asmlen) mc = list(machine_code_dump(data, info.asmaddr, cpuname)) - lines = [line for line in mc if line.count('\t') == 2] + lines = [line for line in mc if line.count('\t') >= 2] checkops(lines, self.add_loop_instructions) data = ctypes.string_at(bridge_info.asmaddr, bridge_info.asmlen) mc = list(machine_code_dump(data, bridge_info.asmaddr, cpuname)) - lines = [line for line in mc if line.count('\t') == 2] + lines = [line for line in mc if line.count('\t') >= 2] checkops(lines, self.bridge_loop_instructions) - def test_compile_bridge_with_target(self): # This test creates a loopy piece of code in a bridge, and builds another # unrelated loop that ends in a jump directly to this loopy bit of code. diff --git a/pypy/jit/backend/x86/test/test_runner.py b/pypy/jit/backend/x86/test/test_runner.py --- a/pypy/jit/backend/x86/test/test_runner.py +++ b/pypy/jit/backend/x86/test/test_runner.py @@ -40,6 +40,10 @@ # the 'mov' is part of the 'jmp' so far bridge_loop_instructions = ['lea', 'mov', 'jmp'] + def get_machine_code_dump_func(self): + from pypy.jit.backend.x86.tool.viewcode import machine_code_dump + return machine_code_dump + def setup_method(self, meth): self.cpu = CPU(rtyper=None, stats=FakeStats()) self.cpu.setup_once() From noreply at buildbot.pypy.org Thu Jan 26 10:45:22 2012 From: noreply at buildbot.pypy.org (bivab) Date: Thu, 26 Jan 2012 10:45:22 +0100 (CET) Subject: [pypy-commit] pypy arm-backend-2: Add methods and attributes required for test_compile_asmlen in the ARM test clases Message-ID: <20120126094522.E264A82B1F@wyvern.cs.uni-duesseldorf.de> Author: David Schneider <david.schneider at picle.org> Branch: arm-backend-2 Changeset: r51778:e13d8ab613df Date: 2012-01-23 17:07 +0100 http://bitbucket.org/pypy/pypy/changeset/e13d8ab613df/ Log: Add methods and attributes required for test_compile_asmlen in the ARM test clases diff --git a/pypy/jit/backend/arm/test/test_runner.py b/pypy/jit/backend/arm/test/test_runner.py --- a/pypy/jit/backend/arm/test/test_runner.py +++ b/pypy/jit/backend/arm/test/test_runner.py @@ -1,5 +1,6 @@ import py from pypy.jit.backend.arm.runner import ArmCPU +from pypy.jit.backend.arm.arch import WORD from pypy.jit.backend.test.runner_test import LLtypeBackendTest, \ boxfloat, \ constfloat @@ -23,12 +24,20 @@ class TestARM(LLtypeBackendTest): + # for the individual tests see + # ====> ../../test/runner_test.py + + add_loop_instructions = ['mov', 'adds', 'cmp', 'beq', 'b'] + bridge_loop_instructions = ['movw', 'movt', 'bx'] + + def get_machine_code_dump_func(self): + from pypy.jit.backend.arm.tool.objdump import machine_code_dump + return machine_code_dump + def setup_method(self, meth): self.cpu = ArmCPU(rtyper=None, stats=FakeStats()) self.cpu.setup_once() - # for the individual tests see - # ====> ../../test/runner_test.py def test_result_is_spilled(self): cpu = self.cpu inp = [BoxInt(i) for i in range(1, 15)] diff --git a/pypy/jit/backend/arm/tool/objdump.py b/pypy/jit/backend/arm/tool/objdump.py --- a/pypy/jit/backend/arm/tool/objdump.py +++ b/pypy/jit/backend/arm/tool/objdump.py @@ -5,9 +5,53 @@ ./objdump.py --decode dumpfile """ import os, sys, py +import subprocess + +def machine_code_dump(data, originaddr, backend_name, label_list=None): + assert backend_name == 'arm_32' + tmpfile = get_tmp_file() + objdump = 'objdump -M reg-names-std --adjust-vma=%(origin)d -D --architecture=arm --target=binary %(file)s' + # + f = open(tmpfile, 'wb') + f.write(data) + f.close() + p = subprocess.Popen(objdump % { + 'file': tmpfile, + 'origin': originaddr, + }, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = p.communicate() + assert not p.returncode, ('Encountered an error running objdump: %s' % + stderr) + # drop some objdump cruft + lines = stdout.splitlines(True)[6:] # drop some objdump cruft + return format_code_dump_with_labels(originaddr, lines, label_list) + +def format_code_dump_with_labels(originaddr, lines, label_list): + from pypy.rlib.rarithmetic import r_uint + if not label_list: + label_list = [] + originaddr = r_uint(originaddr) + itlines = iter(lines) + yield itlines.next() # don't process the first line + for lbl_start, lbl_name in label_list: + for line in itlines: + addr, _ = line.split(':', 1) + addr = int(addr, 16) + if addr >= originaddr+lbl_start: + yield '\n' + if lbl_name is None: + yield '--end of the loop--\n' + else: + yield str(lbl_name) + '\n' + yield line + break + yield line + # yield all the remaining lines + for line in itlines: + yield line def objdump(input): - os.system('objdump -D --architecture=arm --target=binary %s' % input) + os.system('objdump -D -M reg-names-std --architecture=arm --target=binary %s' % input) def get_tmp_file(): From noreply at buildbot.pypy.org Thu Jan 26 10:45:24 2012 From: noreply at buildbot.pypy.org (bivab) Date: Thu, 26 Jan 2012 10:45:24 +0100 (CET) Subject: [pypy-commit] pypy arm-backend-2: move code a bit around to make things clearer Message-ID: <20120126094524.31E4182B1F@wyvern.cs.uni-duesseldorf.de> Author: David Schneider <david.schneider at picle.org> Branch: arm-backend-2 Changeset: r51779:2d0405522a48 Date: 2012-01-23 17:08 +0100 http://bitbucket.org/pypy/pypy/changeset/2d0405522a48/ Log: move code a bit around to make things clearer diff --git a/pypy/jit/backend/arm/assembler.py b/pypy/jit/backend/arm/assembler.py --- a/pypy/jit/backend/arm/assembler.py +++ b/pypy/jit/backend/arm/assembler.py @@ -582,20 +582,21 @@ regalloc = Regalloc(assembler=self, frame_manager=ARMFrameManager()) regalloc.prepare_loop(inputargs, operations) - loop_head = self.mc.currpos() + loop_head = self.mc.get_relative_pos() looptoken._arm_loop_code = loop_head - + # clt.frame_depth = -1 frame_depth = self._assemble(operations, regalloc) clt.frame_depth = frame_depth + # + size_excluding_failure_stuff = self.mc.get_relative_pos() + self._patch_sp_offset(sp_patch_location, frame_depth) - self.write_pending_failure_recoveries() rawstart = self.materialize_loop(looptoken) looptoken._arm_func_addr = rawstart - size_excluding_failure_stuff = self.mc.get_relative_pos() self.process_pending_guards(rawstart) self.fixup_target_tokens(rawstart) @@ -613,7 +614,6 @@ def _assemble(self, operations, regalloc): regalloc.compute_hint_frame_locations(operations) - #self.mc.BKPT() self._walk_operations(operations, regalloc) frame_depth = regalloc.frame_manager.get_frame_depth() jump_target_descr = regalloc.jump_target_descr @@ -637,12 +637,13 @@ if not we_are_translated(): assert len(inputargs) == len(arglocs) - startpos = self.mc.get_relative_pos() regalloc = Regalloc(assembler=self, frame_manager=ARMFrameManager()) regalloc.prepare_bridge(inputargs, arglocs, operations) sp_patch_location = self._prepare_sp_patch_position() + startpos = self.mc.get_relative_pos() + frame_depth = self._assemble(operations, regalloc) codeendpos = self.mc.get_relative_pos() @@ -746,8 +747,6 @@ def _patch_sp_offset(self, pos, frame_depth): cb = OverwritingBuilder(self.mc, pos, OverwritingBuilder.size_of_gen_load_int + WORD) - # Note: the frame_depth is one less than the value stored in the frame - # manager n = frame_depth * WORD # ensure the sp is 8 byte aligned when patching it From noreply at buildbot.pypy.org Thu Jan 26 10:45:25 2012 From: noreply at buildbot.pypy.org (bivab) Date: Thu, 26 Jan 2012 10:45:25 +0100 (CET) Subject: [pypy-commit] pypy arm-backend-2: refactor call_assembler to use conditional_execution and the B instruction for the relative jumps Message-ID: <20120126094525.64CB382B1F@wyvern.cs.uni-duesseldorf.de> Author: David Schneider <david.schneider at picle.org> Branch: arm-backend-2 Changeset: r51780:bdd2be87eb46 Date: 2012-01-24 15:21 +0100 http://bitbucket.org/pypy/pypy/changeset/bdd2be87eb46/ Log: refactor call_assembler to use conditional_execution and the B instruction for the relative jumps diff --git a/pypy/jit/backend/arm/opassembler.py b/pypy/jit/backend/arm/opassembler.py --- a/pypy/jit/backend/arm/opassembler.py +++ b/pypy/jit/backend/arm/opassembler.py @@ -1024,38 +1024,17 @@ self.mc.gen_load_int(r.ip.value, value) self.mc.CMP_rr(tmploc.value, r.ip.value) - fast_jmp_pos = self.mc.currpos() - self.mc.BKPT() - - # Path A: use assembler helper #if values are equal we take the fast path # Slow path, calling helper # jump to merge point + jd = descr.outermost_jitdriver_sd assert jd is not None - asm_helper_adr = self.cpu.cast_adr_to_int(jd.assembler_helper_adr) - with saved_registers(self.mc, r.caller_resp[1:] + [r.ip], - r.caller_vfp_resp): - # result of previous call is in r0 - self.mov_loc_loc(arglocs[0], r.r1) - self.mc.BL(asm_helper_adr) - if op.result and resloc.is_vfp_reg(): - # move result to the allocated register - self.mov_to_vfp_loc(r.r0, r.r1, resloc) - # jump to merge point - jmp_pos = self.mc.currpos() - # This location is not necessarily patched later, depending on how many - # instructions we emit from here to the merge point below. - self.mc.NOP() + # Path A: load return value and reset token + # Fast Path using result boxes - # Path B: load return value and reset token - # Fast Path using result boxes - # patch the jump to the fast path - offset = self.mc.currpos() - fast_jmp_pos - pmc = OverwritingBuilder(self.mc, fast_jmp_pos, WORD) - pmc.ADD_ri(r.pc.value, r.pc.value, offset - PC_OFFSET, cond=c.EQ) - + fast_path_cond=c.EQ # Reset the vable token --- XXX really too much special logic here:-( if jd.index_of_virtualizable >= 0: from pypy.jit.backend.llsupport.descr import FieldDescr @@ -1063,9 +1042,9 @@ assert isinstance(fielddescr, FieldDescr) ofs = fielddescr.offset tmploc = regalloc.get_scratch_reg(INT) - self.mov_loc_loc(arglocs[0], r.ip) - self.mc.MOV_ri(tmploc.value, 0) - self.mc.STR_ri(tmploc.value, r.ip.value, ofs) + self.mov_loc_loc(arglocs[0], r.ip, cond=fast_path_cond) + self.mc.MOV_ri(tmploc.value, 0, cond=fast_path_cond) + self.mc.STR_ri(tmploc.value, r.ip.value, ofs, cond=fast_path_cond) if op.result is not None: # load the return value from fail_boxes_xxx[0] @@ -1078,17 +1057,30 @@ adr = self.fail_boxes_float.get_addr_for_num(0) else: raise AssertionError(kind) - self.mc.gen_load_int(r.ip.value, adr) + self.mc.gen_load_int(r.ip.value, adr, cond=fast_path_cond) if op.result.type == FLOAT: - self.mc.VLDR(resloc.value, r.ip.value) + self.mc.VLDR(resloc.value, r.ip.value, cond=fast_path_cond) else: - self.mc.LDR_ri(resloc.value, r.ip.value) + self.mc.LDR_ri(resloc.value, r.ip.value, cond=fast_path_cond) + # jump to merge point + jmp_pos = self.mc.currpos() + self.mc.BKPT() + + # Path B: use assembler helper + asm_helper_adr = self.cpu.cast_adr_to_int(jd.assembler_helper_adr) + with saved_registers(self.mc, r.caller_resp[1:] + [r.ip], + r.caller_vfp_resp): + # result of previous call is in r0 + self.mov_loc_loc(arglocs[0], r.r1) + self.mc.BL(asm_helper_adr) + if op.result and resloc.is_vfp_reg(): + # move result to the allocated register + self.mov_to_vfp_loc(r.r0, r.r1, resloc) # merge point - offset = self.mc.currpos() - jmp_pos - if offset - PC_OFFSET >= 0: - pmc = OverwritingBuilder(self.mc, jmp_pos, WORD) - pmc.ADD_ri(r.pc.value, r.pc.value, offset - PC_OFFSET) + currpos = self.mc.currpos() + pmc = OverwritingBuilder(self.mc, jmp_pos, WORD) + pmc.B_offs(currpos, fast_path_cond) self.mc.LDR_ri(r.ip.value, r.fp.value) self.mc.CMP_ri(r.ip.value, 0) From noreply at buildbot.pypy.org Thu Jan 26 10:45:26 2012 From: noreply at buildbot.pypy.org (bivab) Date: Thu, 26 Jan 2012 10:45:26 +0100 (CET) Subject: [pypy-commit] pypy arm-backend-2: kill unused code Message-ID: <20120126094526.9751B82B1F@wyvern.cs.uni-duesseldorf.de> Author: David Schneider <david.schneider at picle.org> Branch: arm-backend-2 Changeset: r51781:375bd2545643 Date: 2012-01-24 15:22 +0100 http://bitbucket.org/pypy/pypy/changeset/375bd2545643/ Log: kill unused code diff --git a/pypy/jit/backend/arm/helper/assembler.py b/pypy/jit/backend/arm/helper/assembler.py --- a/pypy/jit/backend/arm/helper/assembler.py +++ b/pypy/jit/backend/arm/helper/assembler.py @@ -141,18 +141,14 @@ f.__name__ = 'emit_guard_%s' % name return f + class saved_registers(object): - def __init__(self, assembler, regs_to_save, vfp_regs_to_save=None, regalloc=None): - assert regalloc is None + def __init__(self, assembler, regs_to_save, vfp_regs_to_save=None): self.assembler = assembler - self.regalloc = None if vfp_regs_to_save is None: vfp_regs_to_save = [] - if self.regalloc: - self._filter_regs(regs_to_save, vfp_regs_to_save) - else: - self.regs = regs_to_save - self.vfp_regs = vfp_regs_to_save + self.regs = regs_to_save + self.vfp_regs = vfp_regs_to_save def __enter__(self): if len(self.regs) > 0: @@ -166,17 +162,6 @@ if len(self.regs) > 0: self.assembler.POP([r.value for r in self.regs]) - def _filter_regs(self, regs_to_save, vfp_regs_to_save): - regs = [] - for box, reg in self.regalloc.rm.reg_bindings.iteritems(): - if reg is r.ip or (reg in regs_to_save and self.regalloc.stays_alive(box)): - regs.append(reg) - self.regs = regs - regs = [] - for box, reg in self.regalloc.vfprm.reg_bindings.iteritems(): - if reg in vfp_regs_to_save and self.regalloc.stays_alive(box): - regs.append(reg) - self.vfp_regs = regs def count_reg_args(args): reg_args = 0 words = 0 From noreply at buildbot.pypy.org Thu Jan 26 10:45:27 2012 From: noreply at buildbot.pypy.org (bivab) Date: Thu, 26 Jan 2012 10:45:27 +0100 (CET) Subject: [pypy-commit] pypy arm-backend-2: add spaces Message-ID: <20120126094527.CAECC82B1F@wyvern.cs.uni-duesseldorf.de> Author: David Schneider <david.schneider at picle.org> Branch: arm-backend-2 Changeset: r51782:9c12b4103f60 Date: 2012-01-26 09:17 +0100 http://bitbucket.org/pypy/pypy/changeset/9c12b4103f60/ Log: add spaces diff --git a/pypy/jit/backend/arm/opassembler.py b/pypy/jit/backend/arm/opassembler.py --- a/pypy/jit/backend/arm/opassembler.py +++ b/pypy/jit/backend/arm/opassembler.py @@ -1034,7 +1034,7 @@ # Path A: load return value and reset token # Fast Path using result boxes - fast_path_cond=c.EQ + fast_path_cond = c.EQ # Reset the vable token --- XXX really too much special logic here:-( if jd.index_of_virtualizable >= 0: from pypy.jit.backend.llsupport.descr import FieldDescr From noreply at buildbot.pypy.org Thu Jan 26 10:45:29 2012 From: noreply at buildbot.pypy.org (bivab) Date: Thu, 26 Jan 2012 10:45:29 +0100 (CET) Subject: [pypy-commit] pypy arm-backend-2: replace a conditional jump with conditional execution Message-ID: <20120126094529.0643082B1F@wyvern.cs.uni-duesseldorf.de> Author: David Schneider <david.schneider at picle.org> Branch: arm-backend-2 Changeset: r51783:5e16783d7519 Date: 2012-01-26 09:17 +0100 http://bitbucket.org/pypy/pypy/changeset/5e16783d7519/ Log: replace a conditional jump with conditional execution diff --git a/pypy/jit/backend/arm/opassembler.py b/pypy/jit/backend/arm/opassembler.py --- a/pypy/jit/backend/arm/opassembler.py +++ b/pypy/jit/backend/arm/opassembler.py @@ -559,9 +559,9 @@ self.mc.BL(func) # patch the JZ above - offset = self.mc.currpos() - jz_location + offset = self.mc.currpos() pmc = OverwritingBuilder(self.mc, jz_location, WORD) - pmc.ADD_ri(r.pc.value, r.pc.value, offset - PC_OFFSET, cond=c.EQ) + pmc.B_offs(offset, c.EQ) return fcond emit_op_cond_call_gc_wb_array = emit_op_cond_call_gc_wb From noreply at buildbot.pypy.org Thu Jan 26 10:45:30 2012 From: noreply at buildbot.pypy.org (bivab) Date: Thu, 26 Jan 2012 10:45:30 +0100 (CET) Subject: [pypy-commit] pypy arm-backend-2: remove redundant saves Message-ID: <20120126094530.3573682B1F@wyvern.cs.uni-duesseldorf.de> Author: David Schneider <david.schneider at picle.org> Branch: arm-backend-2 Changeset: r51784:fc0c1822721d Date: 2012-01-26 10:18 +0100 http://bitbucket.org/pypy/pypy/changeset/fc0c1822721d/ Log: remove redundant saves diff --git a/pypy/jit/backend/arm/codebuilder.py b/pypy/jit/backend/arm/codebuilder.py --- a/pypy/jit/backend/arm/codebuilder.py +++ b/pypy/jit/backend/arm/codebuilder.py @@ -23,12 +23,7 @@ """Generates a call to a helper function, takes its arguments in r0 and r1, result is placed in r0""" addr = rffi.cast(lltype.Signed, function) - if c == cond.AL: - self.BL(addr) - else: - self.PUSH(range(2, 4), cond=c) - self.BL(addr, c) - self.POP(range(2, 4), cond=c) + self.BL(addr, c) return f From noreply at buildbot.pypy.org Thu Jan 26 10:45:31 2012 From: noreply at buildbot.pypy.org (bivab) Date: Thu, 26 Jan 2012 10:45:31 +0100 (CET) Subject: [pypy-commit] pypy arm-backend-2: add more checks for floating point support Message-ID: <20120126094531.69FEE82B1F@wyvern.cs.uni-duesseldorf.de> Author: David Schneider <david.schneider at picle.org> Branch: arm-backend-2 Changeset: r51785:db27ab55d51b Date: 2012-01-26 10:20 +0100 http://bitbucket.org/pypy/pypy/changeset/db27ab55d51b/ Log: add more checks for floating point support diff --git a/pypy/jit/backend/arm/assembler.py b/pypy/jit/backend/arm/assembler.py --- a/pypy/jit/backend/arm/assembler.py +++ b/pypy/jit/backend/arm/assembler.py @@ -207,8 +207,11 @@ def _gen_leave_jitted_hook_code(self, save_exc): mc = ARMv7Builder() - # XXX add a check if cpu supports floats - with saved_registers(mc, r.caller_resp + [r.lr], r.caller_vfp_resp): + if self.cpu.supports_floats: + floats = r.caller_vfp_resp + else: + floats = [] + with saved_registers(mc, r.caller_resp + [r.lr], floats): addr = self.cpu.get_on_leave_jitted_int(save_exception=save_exc) mc.BL(addr) assert self._exit_code_addr != 0 @@ -222,8 +225,11 @@ # mc = ARMv7Builder() # call on_leave_jitted_save_exc() - # XXX add a check if cpu supports floats - with saved_registers(mc, r.caller_resp + [r.ip], r.caller_vfp_resp): + if self.cpu.supports_floats: + floats = r.caller_vfp_resp + else: + floats = [] + with saved_registers(mc, r.caller_resp + [r.lr], floats): addr = self.cpu.get_on_leave_jitted_int(save_exception=True, default_to_memoryerror=True) mc.BL(addr) diff --git a/pypy/jit/backend/arm/helper/assembler.py b/pypy/jit/backend/arm/helper/assembler.py --- a/pypy/jit/backend/arm/helper/assembler.py +++ b/pypy/jit/backend/arm/helper/assembler.py @@ -145,6 +145,7 @@ class saved_registers(object): def __init__(self, assembler, regs_to_save, vfp_regs_to_save=None): self.assembler = assembler + self.supports_floats = assembler.cpu.supports_floats if vfp_regs_to_save is None: vfp_regs_to_save = [] self.regs = regs_to_save @@ -153,11 +154,11 @@ def __enter__(self): if len(self.regs) > 0: self.assembler.PUSH([r.value for r in self.regs]) - if len(self.vfp_regs) > 0: + if self.supports_floats and len(self.vfp_regs) > 0: self.assembler.VPUSH([r.value for r in self.vfp_regs]) def __exit__(self, *args): - if len(self.vfp_regs) > 0: + if self.supports_floats and len(self.vfp_regs) > 0: self.assembler.VPOP([r.value for r in self.vfp_regs]) if len(self.regs) > 0: self.assembler.POP([r.value for r in self.regs]) diff --git a/pypy/jit/backend/arm/opassembler.py b/pypy/jit/backend/arm/opassembler.py --- a/pypy/jit/backend/arm/opassembler.py +++ b/pypy/jit/backend/arm/opassembler.py @@ -1068,8 +1068,11 @@ # Path B: use assembler helper asm_helper_adr = self.cpu.cast_adr_to_int(jd.assembler_helper_adr) - with saved_registers(self.mc, r.caller_resp[1:] + [r.ip], - r.caller_vfp_resp): + if self.cpu.supports_floats: + floats = r.caller_vfp_resp + else: + floats = [] + with saved_registers(self.mc, r.caller_resp[1:] + [r.ip], floats): # result of previous call is in r0 self.mov_loc_loc(arglocs[0], r.r1) self.mc.BL(asm_helper_adr) From noreply at buildbot.pypy.org Thu Jan 26 10:45:32 2012 From: noreply at buildbot.pypy.org (bivab) Date: Thu, 26 Jan 2012 10:45:32 +0100 (CET) Subject: [pypy-commit] pypy arm-backend-2: remove unused import Message-ID: <20120126094532.9A36C82B1F@wyvern.cs.uni-duesseldorf.de> Author: David Schneider <david.schneider at picle.org> Branch: arm-backend-2 Changeset: r51786:23ebba29e2fc Date: 2012-01-26 10:20 +0100 http://bitbucket.org/pypy/pypy/changeset/23ebba29e2fc/ Log: remove unused import diff --git a/pypy/jit/backend/arm/opassembler.py b/pypy/jit/backend/arm/opassembler.py --- a/pypy/jit/backend/arm/opassembler.py +++ b/pypy/jit/backend/arm/opassembler.py @@ -2,7 +2,7 @@ from pypy.jit.backend.arm import conditions as c from pypy.jit.backend.arm import registers as r from pypy.jit.backend.arm import shift -from pypy.jit.backend.arm.arch import WORD, PC_OFFSET +from pypy.jit.backend.arm.arch import WORD from pypy.jit.backend.arm.helper.assembler import (gen_emit_op_by_helper_call, gen_emit_op_unary_cmp, From noreply at buildbot.pypy.org Thu Jan 26 10:45:33 2012 From: noreply at buildbot.pypy.org (bivab) Date: Thu, 26 Jan 2012 10:45:33 +0100 (CET) Subject: [pypy-commit] pypy arm-backend-2: add asserts Message-ID: <20120126094533.C966882B1F@wyvern.cs.uni-duesseldorf.de> Author: David Schneider <david.schneider at picle.org> Branch: arm-backend-2 Changeset: r51787:1f07ea8076c9 Date: 2012-01-26 10:20 +0100 http://bitbucket.org/pypy/pypy/changeset/1f07ea8076c9/ Log: add asserts diff --git a/pypy/jit/backend/arm/helper/regalloc.py b/pypy/jit/backend/arm/helper/regalloc.py --- a/pypy/jit/backend/arm/helper/regalloc.py +++ b/pypy/jit/backend/arm/helper/regalloc.py @@ -90,6 +90,8 @@ assert fcond is not None a0 = op.getarg(0) a1 = op.getarg(1) + assert isinstance(a0, Box) + assert isinstance(a1, Box) arg1 = self.rm.make_sure_var_in_reg(a0, selected_reg=r.r0) arg2 = self.rm.make_sure_var_in_reg(a1, selected_reg=r.r1) assert arg1 == r.r0 From noreply at buildbot.pypy.org Thu Jan 26 12:46:55 2012 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 26 Jan 2012 12:46:55 +0100 (CET) Subject: [pypy-commit] pypy numppy-flatitter: review Message-ID: <20120126114655.10F0382B1F@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: numppy-flatitter Changeset: r51788:977157a7ea64 Date: 2012-01-26 13:46 +0200 http://bitbucket.org/pypy/pypy/changeset/977157a7ea64/ Log: review diff --git a/pypy/module/micronumpy/REVIEW b/pypy/module/micronumpy/REVIEW --- a/pypy/module/micronumpy/REVIEW +++ b/pypy/module/micronumpy/REVIEW @@ -1,3 +1,5 @@ +* this file should go away before asking for a review :) + * I think we should wait for indexing-by-arrays-2, since this would clean up the iterator interface @@ -7,3 +9,17 @@ duplication, since the indexing is getting slowly fairly complex. * iterating over a transposed array still fails. + +* next_skip_x is implemented badly. It still iterates over items (calls + next_skip n times) and is jit.unroll_safe. This means that the JIT will assume + skip is a constant and compile a linear trace unrolling all iterations. This + is *BAD*. Instead it would be something like: + + for arrays: + + res.offset = self.offset + skip + + for views something slightly more complex that does appropriate modulo. + + Note that next() has unroll_safe, because the loop only dependas on the + length of shape which is a constant for assembler. \ No newline at end of file From noreply at buildbot.pypy.org Thu Jan 26 12:59:09 2012 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 26 Jan 2012 12:59:09 +0100 (CET) Subject: [pypy-commit] pypy numpy-back-to-applevel: skip that test Message-ID: <20120126115910.014BF82B1F@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: numpy-back-to-applevel Changeset: r51789:e96e525cc103 Date: 2012-01-26 13:49 +0200 http://bitbucket.org/pypy/pypy/changeset/e96e525cc103/ Log: skip that test diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py --- a/pypy/module/micronumpy/test/test_dtypes.py +++ b/pypy/module/micronumpy/test/test_dtypes.py @@ -378,5 +378,6 @@ assert b.m() == 12 def test_long_as_index(self): + skip("waiting for removal of multimethods of __index__") from _numpypy import int_ assert (1, 2, 3)[int_(1)] == 2 From noreply at buildbot.pypy.org Thu Jan 26 12:59:11 2012 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 26 Jan 2012 12:59:11 +0100 (CET) Subject: [pypy-commit] pypy numpy-back-to-applevel: take for 1d arrays Message-ID: <20120126115911.3693682B1F@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: numpy-back-to-applevel Changeset: r51790:4209800a8a51 Date: 2012-01-26 13:58 +0200 http://bitbucket.org/pypy/pypy/changeset/4209800a8a51/ Log: take for 1d arrays diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -625,13 +625,12 @@ res = W_NDimArray(size, res_shape[:], self.dtype, self.order) res_i = res.create_iter() longdtype = interp_dtype.get_dtype_cache(space).w_longdtype + shapelen = len(index.shape) while not index_i.done(): w_item = index.getitem(index_i.offset).convert_to(longdtype) - import pdb - pdb.set_trace() res.setitem(res_i.offset, self.descr_getitem(space, w_item)) - index_i = index_i.next() - res_i = res_i.next() + index_i = index_i.next(shapelen) + res_i = res_i.next(shapelen) return res def convert_to_array(space, w_obj): diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -1404,7 +1404,7 @@ def test_take(self): from _numpypy import arange - assert(arange(10).take([1, 2, 1, 1]) == [1, 2, 1, 1]) + assert (arange(10).take([1, 2, 1, 1]) == [1, 2, 1, 1]).all() class AppTestSupport(BaseNumpyAppTest): def setup_class(cls): From noreply at buildbot.pypy.org Thu Jan 26 13:48:57 2012 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 26 Jan 2012 13:48:57 +0100 (CET) Subject: [pypy-commit] buildbot default: merge Message-ID: <20120126124857.75E7282B20@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: Changeset: r624:2e2a10de4ae5 Date: 2012-01-26 14:48 +0200 http://bitbucket.org/pypy/buildbot/changeset/2e2a10de4ae5/ Log: merge diff --git a/bot2/pypybuildbot/master.py b/bot2/pypybuildbot/master.py --- a/bot2/pypybuildbot/master.py +++ b/bot2/pypybuildbot/master.py @@ -225,7 +225,12 @@ JITWIN32, # on bigboard #JITFREEBSD64, # on headless JITMACOSX64, # on mvt's machine - ], branch=None, hour=3, minute=0) + ], branch=None, hour=3, minute=0), + + Nightly("nighly-4-00-py3k", [ + LINUX32, # on tannit32, uses 4 cores + ], branch='py3k', hour=4, minute=0), + ], 'status': [status, ircbot], From noreply at buildbot.pypy.org Thu Jan 26 13:48:56 2012 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 26 Jan 2012 13:48:56 +0100 (CET) Subject: [pypy-commit] buildbot default: I believe this is correct :) Message-ID: <20120126124856.59A8682B1F@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: Changeset: r623:29b4d8f8ccf0 Date: 2012-01-26 14:47 +0200 http://bitbucket.org/pypy/buildbot/changeset/29b4d8f8ccf0/ Log: I believe this is correct :) diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py --- a/bot2/pypybuildbot/builds.py +++ b/bot2/pypybuildbot/builds.py @@ -76,7 +76,9 @@ except OSError: pass try: - symlink_force(os.path.basename(self.masterdest), 'latest.html') + symname = os.path.join(os.path.dirname(self.masterdest), + 'latest.html') + symlink_force(self.masterdest, symname) except OSError: pass From noreply at buildbot.pypy.org Thu Jan 26 15:15:34 2012 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 26 Jan 2012 15:15:34 +0100 (CET) Subject: [pypy-commit] pypy numpy-back-to-applevel: more tests Message-ID: <20120126141534.C5F9682B1F@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: numpy-back-to-applevel Changeset: r51791:5ab418b60c8c Date: 2012-01-26 14:00 +0200 http://bitbucket.org/pypy/pypy/changeset/5ab418b60c8c/ Log: more tests diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -1405,6 +1405,7 @@ def test_take(self): from _numpypy import arange assert (arange(10).take([1, 2, 1, 1]) == [1, 2, 1, 1]).all() + raises(IndexError, "arange(3).take([15])") class AppTestSupport(BaseNumpyAppTest): def setup_class(cls): From noreply at buildbot.pypy.org Thu Jan 26 15:15:36 2012 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 26 Jan 2012 15:15:36 +0100 (CET) Subject: [pypy-commit] pypy numpy-back-to-applevel: least effort take for no axis Message-ID: <20120126141536.0504282B20@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: numpy-back-to-applevel Changeset: r51792:0dc95b060dca Date: 2012-01-26 14:54 +0200 http://bitbucket.org/pypy/pypy/changeset/0dc95b060dca/ Log: least effort take for no axis diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -575,7 +575,7 @@ backstrides, shape, concrete)) def descr_ravel(self, space, w_order=None): - if space.is_w(w_order, space.w_None): + if w_order is None or space.is_w(w_order, space.w_None): order = 'C' else: order = space.str_w(w_order) @@ -613,22 +613,25 @@ def supports_fast_slicing(self): return False - def descr_take(self, space, w_obj): + def descr_take(self, space, w_obj, w_axis=None): index = convert_to_array(space, w_obj).get_concrete() - if len(self.shape) > 1: + concr = self.get_concrete() + if space.is_w(w_axis, space.w_None): + concr = concr.descr_ravel(space) + if len(concr.shape) > 1: xxx index_i = index.create_iter() res_shape = index.shape size = 1 for elem in res_shape: size *= elem - res = W_NDimArray(size, res_shape[:], self.dtype, self.order) + res = W_NDimArray(size, res_shape[:], concr.dtype, concr.order) res_i = res.create_iter() longdtype = interp_dtype.get_dtype_cache(space).w_longdtype shapelen = len(index.shape) while not index_i.done(): w_item = index.getitem(index_i.offset).convert_to(longdtype) - res.setitem(res_i.offset, self.descr_getitem(space, w_item)) + res.setitem(res_i.offset, concr.descr_getitem(space, w_item)) index_i = index_i.next(shapelen) res_i = res_i.next(shapelen) return res diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -1406,6 +1406,8 @@ from _numpypy import arange assert (arange(10).take([1, 2, 1, 1]) == [1, 2, 1, 1]).all() raises(IndexError, "arange(3).take([15])") + a = arange(6).reshape(2, 3) + assert (a.take([1, 0, 3]) == [1, 0, 3]).all() class AppTestSupport(BaseNumpyAppTest): def setup_class(cls): From noreply at buildbot.pypy.org Thu Jan 26 15:15:37 2012 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 26 Jan 2012 15:15:37 +0100 (CET) Subject: [pypy-commit] pypy numpy-back-to-applevel: more tests Message-ID: <20120126141537.36F5782B22@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: numpy-back-to-applevel Changeset: r51793:233034cea174 Date: 2012-01-26 14:56 +0200 http://bitbucket.org/pypy/pypy/changeset/233034cea174/ Log: more tests diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -618,7 +618,7 @@ concr = self.get_concrete() if space.is_w(w_axis, space.w_None): concr = concr.descr_ravel(space) - if len(concr.shape) > 1: + else: xxx index_i = index.create_iter() res_shape = index.shape diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -1408,6 +1408,10 @@ raises(IndexError, "arange(3).take([15])") a = arange(6).reshape(2, 3) assert (a.take([1, 0, 3]) == [1, 0, 3]).all() + assert ((a + a).take([3]) == [6]).all() + a = arange(12).reshape(2, 6) + assert (a[:,::2].take([3, 2, 1]) == [6, 4, 2]).all() + class AppTestSupport(BaseNumpyAppTest): def setup_class(cls): From noreply at buildbot.pypy.org Thu Jan 26 15:15:38 2012 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 26 Jan 2012 15:15:38 +0100 (CET) Subject: [pypy-commit] pypy numpy-back-to-applevel: proper error Message-ID: <20120126141538.65F1382B23@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: numpy-back-to-applevel Changeset: r51794:3c1c293c3fa4 Date: 2012-01-26 14:56 +0200 http://bitbucket.org/pypy/pypy/changeset/3c1c293c3fa4/ Log: proper error diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -619,7 +619,8 @@ if space.is_w(w_axis, space.w_None): concr = concr.descr_ravel(space) else: - xxx + raise OperationError(space.w_NotImplementedError, + space.wrap("axis unsupported for take")) index_i = index.create_iter() res_shape = index.shape size = 1 From noreply at buildbot.pypy.org Thu Jan 26 15:15:39 2012 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 26 Jan 2012 15:15:39 +0100 (CET) Subject: [pypy-commit] pypy numpy-back-to-applevel: basic compress + fix a weird bug Message-ID: <20120126141539.9927982B24@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: numpy-back-to-applevel Changeset: r51795:ecec9d981978 Date: 2012-01-26 16:15 +0200 http://bitbucket.org/pypy/pypy/changeset/ecec9d981978/ Log: basic compress + fix a weird bug diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -613,6 +613,10 @@ def supports_fast_slicing(self): return False + def descr_compress(self, space, w_obj, w_axis=None): + index = convert_to_array(space, w_obj) + return self.getitem_filter(space, index) + def descr_take(self, space, w_obj, w_axis=None): index = convert_to_array(space, w_obj).get_concrete() concr = self.get_concrete() @@ -631,6 +635,7 @@ longdtype = interp_dtype.get_dtype_cache(space).w_longdtype shapelen = len(index.shape) while not index_i.done(): + # XXX jitdriver + test_zjit w_item = index.getitem(index_i.offset).convert_to(longdtype) res.setitem(res_i.offset, concr.descr_getitem(space, w_item)) index_i = index_i.next(shapelen) @@ -1176,7 +1181,7 @@ ownmaskna): raise OperationError(space.w_NotImplementedError, space.wrap("Unsupported args")) if not space.issequence_w(w_item_or_iterable): - if space.is_w(w_dtype, space.w_None): + if w_dtype is None or space.is_w(w_dtype, space.w_None): w_dtype = interp_ufuncs.find_dtype_for_scalar(space, w_item_or_iterable) dtype = space.interp_w(interp_dtype.W_Dtype, @@ -1201,7 +1206,7 @@ shape, elems_w = find_shape_and_elems(space, w_item_or_iterable) # they come back in C order size = len(elems_w) - if space.is_w(w_dtype, space.w_None): + if w_dtype is None or space.is_w(w_dtype, space.w_None): w_dtype = None for w_elem in elems_w: w_dtype = interp_ufuncs.find_dtype_for_scalar(space, w_elem, @@ -1338,6 +1343,7 @@ reshape = interp2app(BaseArray.descr_reshape), tolist = interp2app(BaseArray.descr_tolist), take = interp2app(BaseArray.descr_take), + compress = interp2app(BaseArray.descr_compress), ) diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -1411,7 +1411,11 @@ assert ((a + a).take([3]) == [6]).all() a = arange(12).reshape(2, 6) assert (a[:,::2].take([3, 2, 1]) == [6, 4, 2]).all() - + + def test_compress(self): + from _numpypy import arange + a = arange(10) + assert (a.compress([True, False, True]) == [0, 2]).all() class AppTestSupport(BaseNumpyAppTest): def setup_class(cls): From noreply at buildbot.pypy.org Thu Jan 26 15:25:18 2012 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 26 Jan 2012 15:25:18 +0100 (CET) Subject: [pypy-commit] pypy numpy-back-to-applevel: compress support Message-ID: <20120126142518.E8B3282B1F@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: numpy-back-to-applevel Changeset: r51796:b70485f468be Date: 2012-01-26 16:24 +0200 http://bitbucket.org/pypy/pypy/changeset/b70485f468be/ Log: compress support diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -380,7 +380,7 @@ def count_all_true(self, arr): sig = arr.find_sig() - frame = sig.create_frame(self) + frame = sig.create_frame(arr) shapelen = len(arr.shape) s = 0 iter = None @@ -394,6 +394,9 @@ def getitem_filter(self, space, arr): concr = arr.get_concrete() + if concr.size > self.size: + raise OperationError(space.w_IndexError, + space.wrap("index out of range for array")) size = self.count_all_true(concr) res = W_NDimArray(size, [size], self.find_dtype()) ri = ArrayIterator(size) @@ -402,7 +405,7 @@ sig = self.find_sig() frame = sig.create_frame(self) v = None - while not frame.done(): + while not ri.done(): filter_driver.jit_merge_point(concr=concr, argi=argi, ri=ri, frame=frame, v=v, res=res, sig=sig, shapelen=shapelen, self=self) diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -1416,6 +1416,9 @@ from _numpypy import arange a = arange(10) assert (a.compress([True, False, True]) == [0, 2]).all() + assert (a.compress([1, 0, 13]) == [0, 2]).all() + assert (a.compress([1, 0, 13.5]) == [0, 2]).all() + raises(IndexError, "a.compress([1] * 100)") class AppTestSupport(BaseNumpyAppTest): def setup_class(cls): diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py --- a/pypy/module/micronumpy/types.py +++ b/pypy/module/micronumpy/types.py @@ -105,7 +105,8 @@ )) def read_bool(self, storage, width, i, offset): - raise NotImplementedError + return bool(libffi.array_getitem(clibffi.cast_type_to_ffitype(self.T), + width, storage, i, offset)) def store(self, storage, width, i, offset, box): value = self.unbox(box) @@ -207,11 +208,6 @@ else: return self.False - - def read_bool(self, storage, width, i, offset): - return libffi.array_getitem(clibffi.cast_type_to_ffitype(self.T), - width, storage, i, offset) - def coerce_subtype(self, space, w_subtype, w_item): # Doesn't return subclasses so it can return the constants. return self._coerce(space, w_item) From noreply at buildbot.pypy.org Thu Jan 26 15:31:00 2012 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 26 Jan 2012 15:31:00 +0100 (CET) Subject: [pypy-commit] pypy numpy-back-to-applevel: cheat a tiny bit. One test passes (we have to export any and all though Message-ID: <20120126143100.228BA82B1F@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: numpy-back-to-applevel Changeset: r51797:1a4c04c232fb Date: 2012-01-26 16:30 +0200 http://bitbucket.org/pypy/pypy/changeset/1a4c04c232fb/ Log: cheat a tiny bit. One test passes (we have to export any and all though and implement axis there) diff --git a/lib_pypy/numpypy/core/arrayprint.py b/lib_pypy/numpypy/core/arrayprint.py --- a/lib_pypy/numpypy/core/arrayprint.py +++ b/lib_pypy/numpypy/core/arrayprint.py @@ -594,7 +594,7 @@ precision = 0 precision = min(self.precision, precision) self.max_str_len = len(str(int(max_val))) + precision + 2 - if _nc.any(special): + if special.any(): self.max_str_len = max(self.max_str_len, len(_nan_str), len(_inf_str)+1, diff --git a/lib_pypy/numpypy/core/numeric.py b/lib_pypy/numpypy/core/numeric.py --- a/lib_pypy/numpypy/core/numeric.py +++ b/lib_pypy/numpypy/core/numeric.py @@ -4,6 +4,8 @@ import _numpypy as multiarray # ARGH from numpypy.core.arrayprint import array2string + + def asanyarray(a, dtype=None, order=None, maskna=None, ownmaskna=False): """ Convert the input to an ndarray, but pass ndarray subclasses through. From noreply at buildbot.pypy.org Thu Jan 26 15:48:51 2012 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 26 Jan 2012 15:48:51 +0100 (CET) Subject: [pypy-commit] pypy numpy-back-to-applevel: pfff Message-ID: <20120126144851.5863882B20@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: numpy-back-to-applevel Changeset: r51799:72b8bf6e9019 Date: 2012-01-26 16:48 +0200 http://bitbucket.org/pypy/pypy/changeset/72b8bf6e9019/ Log: pfff diff --git a/lib_pypy/numpypy/core/numeric.py b/lib_pypy/numpypy/core/numeric.py --- a/lib_pypy/numpypy/core/numeric.py +++ b/lib_pypy/numpypy/core/numeric.py @@ -201,7 +201,7 @@ typename = "'%s'" % typename lf = '' - if 0 or issubclass(arr.dtype.type, flexible): + if 0: # or issubclass(arr.dtype.type, flexible): if arr.dtype.names: typename = "%s" % str(arr.dtype) else: From noreply at buildbot.pypy.org Thu Jan 26 15:48:50 2012 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 26 Jan 2012 15:48:50 +0100 (CET) Subject: [pypy-commit] pypy numpy-back-to-applevel: expose name Message-ID: <20120126144850.2A3C682B1F@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: numpy-back-to-applevel Changeset: r51798:ecfe72e6275e Date: 2012-01-26 16:37 +0200 http://bitbucket.org/pypy/pypy/changeset/ecfe72e6275e/ Log: expose name diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py --- a/pypy/module/micronumpy/interp_dtype.py +++ b/pypy/module/micronumpy/interp_dtype.py @@ -108,6 +108,7 @@ type = interp_attrproperty_w("w_box_type", cls=W_Dtype), itemsize = GetSetProperty(W_Dtype.descr_get_itemsize), shape = GetSetProperty(W_Dtype.descr_get_shape), + name = interp_attrproperty('name', cls=W_Dtype), ) W_Dtype.typedef.acceptable_as_base_class = False diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py --- a/pypy/module/micronumpy/test/test_dtypes.py +++ b/pypy/module/micronumpy/test/test_dtypes.py @@ -11,6 +11,7 @@ assert dtype('int8').num == 1 assert dtype(d) is d assert dtype(None) is dtype(float) + assert dtype('int8').name == 'int8' raises(TypeError, dtype, 1042) def test_dtype_with_types(self): From noreply at buildbot.pypy.org Thu Jan 26 16:22:19 2012 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 26 Jan 2012 16:22:19 +0100 (CET) Subject: [pypy-commit] pypy default: merge numpy-concatenate and move it to interp-level Message-ID: <20120126152219.CB90F82B1F@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: Changeset: r51800:8d9c1bd7e3be Date: 2012-01-26 17:21 +0200 http://bitbucket.org/pypy/pypy/changeset/8d9c1bd7e3be/ Log: merge numpy-concatenate and move it to interp-level diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py --- a/pypy/module/micronumpy/__init__.py +++ b/pypy/module/micronumpy/__init__.py @@ -28,6 +28,7 @@ 'fromstring': 'interp_support.fromstring', 'flatiter': 'interp_numarray.W_FlatIterator', 'isna': 'interp_numarray.isna', + 'concatenate': 'interp_numarray.concatenate', 'True_': 'types.Bool.True', 'False_': 'types.Bool.False', diff --git a/pypy/module/micronumpy/app_numpy.py b/pypy/module/micronumpy/app_numpy.py --- a/pypy/module/micronumpy/app_numpy.py +++ b/pypy/module/micronumpy/app_numpy.py @@ -59,7 +59,7 @@ if not hasattr(a, "max"): a = _numpypy.array(a) return a.max(axis) - + def arange(start, stop=None, step=1, dtype=None): '''arange([start], stop[, step], dtype=None) Generate values in the half-interval [start, stop). diff --git a/pypy/module/micronumpy/interp_iter.py b/pypy/module/micronumpy/interp_iter.py --- a/pypy/module/micronumpy/interp_iter.py +++ b/pypy/module/micronumpy/interp_iter.py @@ -17,6 +17,10 @@ if self.step != 0: shape.append(self.lgt) + def __repr__(self): + return 'Chunk(%d, %d, %d, %d)' % (self.start, self.stop, self.step, + self.lgt) + class BaseTransform(object): pass diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -1,6 +1,6 @@ from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.error import OperationError, operationerrfmt -from pypy.interpreter.gateway import interp2app, NoneNotWrapped +from pypy.interpreter.gateway import interp2app, NoneNotWrapped, unwrap_spec from pypy.interpreter.typedef import TypeDef, GetSetProperty from pypy.module.micronumpy import interp_ufuncs, interp_dtype, signature,\ interp_boxes @@ -1341,6 +1341,42 @@ return convert_to_array(space, w_obj2).descr_dot(space, w_arr) return w_arr.descr_dot(space, w_obj2) + at unwrap_spec(axis=int) +def concatenate(space, w_args, axis=0): + args_w = space.listview(w_args) + if len(args_w) == 0: + raise OperationError(space.w_ValueError, space.wrap("concatenation of zero-length sequences is impossible")) + args_w = [convert_to_array(space, w_arg) for w_arg in args_w] + dtype = args_w[0].find_dtype() + shape = args_w[0].shape[:] + if len(shape) <= axis: + raise OperationError(space.w_ValueError, + space.wrap("bad axis argument")) + for arr in args_w[1:]: + dtype = interp_ufuncs.find_binop_result_dtype(space, dtype, + arr.find_dtype()) + if len(arr.shape) <= axis: + raise OperationError(space.w_ValueError, + space.wrap("bad axis argument")) + for i, axis_size in enumerate(arr.shape): + if len(arr.shape) != len(shape) or (i != axis and axis_size != shape[i]): + raise OperationError(space.w_ValueError, space.wrap( + "array dimensions must agree except for axis being concatenated")) + elif i == axis: + shape[i] += axis_size + size = 1 + for elem in shape: + size *= elem + res = W_NDimArray(size, shape, dtype, 'C') + chunks = [Chunk(0, i, 1, i) for i in shape] + axis_start = 0 + for arr in args_w: + chunks[axis] = Chunk(axis_start, axis_start + arr.shape[axis], 1, + arr.shape[axis]) + res.create_slice(chunks).setslice(space, arr) + axis_start += arr.shape[axis] + return res + BaseArray.typedef = TypeDef( 'ndarray', __module__ = "numpypy", diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -1039,6 +1039,40 @@ #assert (a.var(0) == [8, 8]).all() #assert (a.var(1) == [.25] * 5).all() + def test_concatenate(self): + from numpypy import array, concatenate, dtype + a1 = array([0,1,2]) + a2 = array([3,4,5]) + a = concatenate((a1, a2)) + assert len(a) == 6 + assert (a == [0,1,2,3,4,5]).all() + assert a.dtype is dtype(int) + b1 = array([[1, 2], [3, 4]]) + b2 = array([[5, 6]]) + b = concatenate((b1, b2), axis=0) + assert (b == [[1, 2],[3, 4],[5, 6]]).all() + c = concatenate((b1, b2.T), axis=1) + assert (c == [[1, 2, 5],[3, 4, 6]]).all() + d = concatenate(([0],[1])) + assert (d == [0,1]).all() + e1 = array([[0,1],[2,3]]) + e = concatenate(e1) + assert (e == [0,1,2,3]).all() + f1 = array([0,1]) + f = concatenate((f1, [2], f1, [7])) + assert (f == [0,1,2,0,1,7]).all() + + bad_axis = raises(ValueError, concatenate, (a1,a2), axis=1) + assert str(bad_axis.value) == "bad axis argument" + + concat_zero = raises(ValueError, concatenate, ()) + assert str(concat_zero.value) == \ + "concatenation of zero-length sequences is impossible" + + dims_disagree = raises(ValueError, concatenate, (a1, b1), axis=0) + assert str(dims_disagree.value) == \ + "array dimensions must agree except for axis being concatenated" + def test_std(self): from _numpypy import array a = array(range(10)) @@ -1062,7 +1096,6 @@ a = array([[1, 2], [3, 4]]) assert (a.T.flatten() == [1, 3, 2, 4]).all() - class AppTestMultiDim(BaseNumpyAppTest): def test_init(self): import _numpypy From noreply at buildbot.pypy.org Thu Jan 26 18:13:49 2012 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 26 Jan 2012 18:13:49 +0100 (CET) Subject: [pypy-commit] pypy numpy-back-to-applevel: make sure it works on multidim arrays as well Message-ID: <20120126171349.7EBD982B20@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: numpy-back-to-applevel Changeset: r51802:349ebdfc7cde Date: 2012-01-26 17:29 +0200 http://bitbucket.org/pypy/pypy/changeset/349ebdfc7cde/ Log: make sure it works on multidim arrays as well diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -1451,8 +1451,11 @@ assert (a.compress([True, False, True]) == [0, 2]).all() assert (a.compress([1, 0, 13]) == [0, 2]).all() assert (a.compress([1, 0, 13.5]) == [0, 2]).all() + a = arange(10).reshape(2, 5) + assert (a.compress([True, False, True]) == [0, 2]).all() raises(IndexError, "a.compress([1] * 100)") + class AppTestSupport(BaseNumpyAppTest): def setup_class(cls): import struct From noreply at buildbot.pypy.org Thu Jan 26 18:13:48 2012 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 26 Jan 2012 18:13:48 +0100 (CET) Subject: [pypy-commit] pypy numpy-back-to-applevel: merge default Message-ID: <20120126171348.4ABEE82B1F@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: numpy-back-to-applevel Changeset: r51801:2b3f50dad1dd Date: 2012-01-26 17:25 +0200 http://bitbucket.org/pypy/pypy/changeset/2b3f50dad1dd/ Log: merge default diff --git a/pypy/bin/py.py b/pypy/bin/py.py --- a/pypy/bin/py.py +++ b/pypy/bin/py.py @@ -76,6 +76,8 @@ config.objspace.suggest(allworkingmodules=False) if config.objspace.allworkingmodules: pypyoption.enable_allworkingmodules(config) + if config.objspace.usemodules._continuation: + config.translation.continuation = True if config.objspace.usemodules.thread: config.translation.thread = True diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py --- a/pypy/module/_socket/test/test_sock_app.py +++ b/pypy/module/_socket/test/test_sock_app.py @@ -529,26 +529,31 @@ import _socket, os if not hasattr(_socket, 'AF_UNIX'): skip('AF_UNIX not supported.') - sockpath = os.path.join(self.udir, 'app_test_unix_socket_connect') + oldcwd = os.getcwd() + os.chdir(self.udir) + try: + sockpath = 'app_test_unix_socket_connect' - serversock = _socket.socket(_socket.AF_UNIX) - serversock.bind(sockpath) - serversock.listen(1) + serversock = _socket.socket(_socket.AF_UNIX) + serversock.bind(sockpath) + serversock.listen(1) - clientsock = _socket.socket(_socket.AF_UNIX) - clientsock.connect(sockpath) - s, addr = serversock.accept() - assert not addr + clientsock = _socket.socket(_socket.AF_UNIX) + clientsock.connect(sockpath) + s, addr = serversock.accept() + assert not addr - s.send('X') - data = clientsock.recv(100) - assert data == 'X' - clientsock.send('Y') - data = s.recv(100) - assert data == 'Y' + s.send('X') + data = clientsock.recv(100) + assert data == 'X' + clientsock.send('Y') + data = s.recv(100) + assert data == 'Y' - clientsock.close() - s.close() + clientsock.close() + s.close() + finally: + os.chdir(oldcwd) class AppTestSocketTCP: diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py --- a/pypy/module/micronumpy/__init__.py +++ b/pypy/module/micronumpy/__init__.py @@ -28,6 +28,7 @@ 'fromstring': 'interp_support.fromstring', 'flatiter': 'interp_numarray.W_FlatIterator', 'isna': 'interp_numarray.isna', + 'concatenate': 'interp_numarray.concatenate', 'set_string_function': 'appbridge.set_string_function', diff --git a/pypy/module/micronumpy/app_numpy.py b/pypy/module/micronumpy/app_numpy.py --- a/pypy/module/micronumpy/app_numpy.py +++ b/pypy/module/micronumpy/app_numpy.py @@ -59,7 +59,7 @@ if not hasattr(a, "max"): a = _numpypy.array(a) return a.max(axis) - + def arange(start, stop=None, step=1, dtype=None): '''arange([start], stop[, step], dtype=None) Generate values in the half-interval [start, stop). diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py --- a/pypy/module/micronumpy/interp_boxes.py +++ b/pypy/module/micronumpy/interp_boxes.py @@ -9,6 +9,7 @@ from pypy.rpython.lltypesystem import rffi, lltype MIXIN_64 = (int_typedef,) if LONG_BIT == 64 else () +MIXIN_32 = () if LONG_BIT == 64 else (int_typedef,) def new_dtype_getter(name): def get_dtype(space): @@ -232,7 +233,7 @@ __new__ = interp2app(W_UInt16Box.descr__new__.im_func), ) -W_Int32Box.typedef = TypeDef("int32", W_SignedIntegerBox.typedef, +W_Int32Box.typedef = TypeDef("int32", (W_SignedIntegerBox.typedef,) + MIXIN_32, __module__ = "numpypy", __new__ = interp2app(W_Int32Box.descr__new__.im_func), ) @@ -242,24 +243,18 @@ __new__ = interp2app(W_UInt32Box.descr__new__.im_func), ) -if LONG_BIT == 32: - long_name = "int32" -elif LONG_BIT == 64: - long_name = "int64" -W_LongBox.typedef = TypeDef(long_name, (W_SignedIntegerBox.typedef, int_typedef,), - __module__ = "numpypy", - __new__ = interp2app(W_LongBox.descr__new__.im_func), -) - -W_ULongBox.typedef = TypeDef("u" + long_name, W_UnsignedIntegerBox.typedef, - __module__ = "numpypy", -) - W_Int64Box.typedef = TypeDef("int64", (W_SignedIntegerBox.typedef,) + MIXIN_64, __module__ = "numpypy", __new__ = interp2app(W_Int64Box.descr__new__.im_func), ) +if LONG_BIT == 32: + W_LongBox = W_Int32Box + W_ULongBox = W_UInt32Box +elif LONG_BIT == 64: + W_LongBox = W_Int64Box + W_ULongBox = W_UInt64Box + W_UInt64Box.typedef = TypeDef("uint64", W_UnsignedIntegerBox.typedef, __module__ = "numpypy", __new__ = interp2app(W_UInt64Box.descr__new__.im_func), diff --git a/pypy/module/micronumpy/interp_iter.py b/pypy/module/micronumpy/interp_iter.py --- a/pypy/module/micronumpy/interp_iter.py +++ b/pypy/module/micronumpy/interp_iter.py @@ -17,6 +17,10 @@ if self.step != 0: shape.append(self.lgt) + def __repr__(self): + return 'Chunk(%d, %d, %d, %d)' % (self.start, self.stop, self.step, + self.lgt) + class BaseTransform(object): pass diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -1273,6 +1273,42 @@ return convert_to_array(space, w_obj2).descr_dot(space, w_arr) return w_arr.descr_dot(space, w_obj2) + at unwrap_spec(axis=int) +def concatenate(space, w_args, axis=0): + args_w = space.listview(w_args) + if len(args_w) == 0: + raise OperationError(space.w_ValueError, space.wrap("concatenation of zero-length sequences is impossible")) + args_w = [convert_to_array(space, w_arg) for w_arg in args_w] + dtype = args_w[0].find_dtype() + shape = args_w[0].shape[:] + if len(shape) <= axis: + raise OperationError(space.w_ValueError, + space.wrap("bad axis argument")) + for arr in args_w[1:]: + dtype = interp_ufuncs.find_binop_result_dtype(space, dtype, + arr.find_dtype()) + if len(arr.shape) <= axis: + raise OperationError(space.w_ValueError, + space.wrap("bad axis argument")) + for i, axis_size in enumerate(arr.shape): + if len(arr.shape) != len(shape) or (i != axis and axis_size != shape[i]): + raise OperationError(space.w_ValueError, space.wrap( + "array dimensions must agree except for axis being concatenated")) + elif i == axis: + shape[i] += axis_size + size = 1 + for elem in shape: + size *= elem + res = W_NDimArray(size, shape, dtype, 'C') + chunks = [Chunk(0, i, 1, i) for i in shape] + axis_start = 0 + for arr in args_w: + chunks[axis] = Chunk(axis_start, axis_start + arr.shape[axis], 1, + arr.shape[axis]) + res.create_slice(chunks).setslice(space, arr) + axis_start += arr.shape[axis] + return res + BaseArray.typedef = TypeDef( 'ndarray', __module__ = "numpypy", diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py --- a/pypy/module/micronumpy/test/test_dtypes.py +++ b/pypy/module/micronumpy/test/test_dtypes.py @@ -382,3 +382,14 @@ skip("waiting for removal of multimethods of __index__") from _numpypy import int_ assert (1, 2, 3)[int_(1)] == 2 + + def test_int(self): + import sys + from _numpypy import int32, int64, int_ + assert issubclass(int_, int) + if sys.maxint == (1<<31) - 1: + assert issubclass(int32, int) + assert int_ is int32 + else: + assert issubclass(int64, int) + assert int_ is int64 diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -1039,6 +1039,40 @@ assert (a.var(0) == [8, 8]).all() assert (a.var(1) == [.25] * 5).all() + def test_concatenate(self): + from numpypy import array, concatenate, dtype + a1 = array([0,1,2]) + a2 = array([3,4,5]) + a = concatenate((a1, a2)) + assert len(a) == 6 + assert (a == [0,1,2,3,4,5]).all() + assert a.dtype is dtype(int) + b1 = array([[1, 2], [3, 4]]) + b2 = array([[5, 6]]) + b = concatenate((b1, b2), axis=0) + assert (b == [[1, 2],[3, 4],[5, 6]]).all() + c = concatenate((b1, b2.T), axis=1) + assert (c == [[1, 2, 5],[3, 4, 6]]).all() + d = concatenate(([0],[1])) + assert (d == [0,1]).all() + e1 = array([[0,1],[2,3]]) + e = concatenate(e1) + assert (e == [0,1,2,3]).all() + f1 = array([0,1]) + f = concatenate((f1, [2], f1, [7])) + assert (f == [0,1,2,0,1,7]).all() + + bad_axis = raises(ValueError, concatenate, (a1,a2), axis=1) + assert str(bad_axis.value) == "bad axis argument" + + concat_zero = raises(ValueError, concatenate, ()) + assert str(concat_zero.value) == \ + "concatenation of zero-length sequences is impossible" + + dims_disagree = raises(ValueError, concatenate, (a1, b1), axis=0) + assert str(dims_disagree.value) == \ + "array dimensions must agree except for axis being concatenated" + def test_std(self): from _numpypy import array a = array(range(10)) @@ -1062,7 +1096,6 @@ a = array([[1, 2], [3, 4]]) assert (a.T.flatten() == [1, 3, 2, 4]).all() - class AppTestMultiDim(BaseNumpyAppTest): def test_init(self): import _numpypy diff --git a/pypy/module/thread/ll_thread.py b/pypy/module/thread/ll_thread.py --- a/pypy/module/thread/ll_thread.py +++ b/pypy/module/thread/ll_thread.py @@ -4,7 +4,7 @@ import py from pypy.rlib import jit, rgc from pypy.rlib.debug import ll_assert -from pypy.rlib.objectmodel import we_are_translated +from pypy.rlib.objectmodel import we_are_translated, specialize from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.tool import rffi_platform from pypy.tool import autopath @@ -85,6 +85,7 @@ def allocate_lock(): return Lock(allocate_ll_lock()) + at specialize.arg(0) def ll_start_new_thread(func): ident = c_thread_start(func) if ident == -1: @@ -97,6 +98,7 @@ def get_ident(): return rffi.cast(lltype.Signed, c_thread_get_ident()) + at specialize.arg(0) def start_new_thread(x, y): """In RPython, no argument can be passed. You have to use global variables to pass information to the new thread. That's not very diff --git a/pypy/module/thread/test/test_ll_thread.py b/pypy/module/thread/test/test_ll_thread.py --- a/pypy/module/thread/test/test_ll_thread.py +++ b/pypy/module/thread/test/test_ll_thread.py @@ -34,72 +34,34 @@ use_threads = True def test_start_new_thread(self): - py.test.skip("xxx ideally, investigate why it fails randomly") - # xxx but in practice start_new_thread() is also tested by the - # next test, and it's a mess to test start_new_thread() without - # the proper GIL to protect the GC import time class State: pass state = State() - class Z: - def __init__(self, value): - self.value = value - def __del__(self): - state.freed_counter += 1 - - def bootstrap(): - state.my_thread_ident = state.z.ident = get_ident() - assert state.my_thread_ident == get_ident() - assert get_ident() == state.z.ident - state.seen_value = state.z.value - state.z = None - # I think that we would need here a memory barrier in order - # to make the test pass reliably. The issue is that the - # main thread may see 'state.done = 1' before seeing the - # effect of the other assignments done above. For now let's - # emulate the write barrier by doing a system call and - # waiting a bit... - time.sleep(0.012) - state.done = 1 - - def g(i): - state.z = Z(i) - return start_new_thread(bootstrap, ()) - g._dont_inline_ = True + def bootstrap1(): + state.my_thread_ident1 = get_ident() + def bootstrap2(): + state.my_thread_ident2 = get_ident() def f(): - main_ident = get_ident() - assert main_ident == get_ident() - state.freed_counter = 0 - for i in range(50): - state.done = 0 - state.seen_value = 0 - ident = g(i) - gc.collect() - willing_to_wait_more = 1000 - while not state.done: - willing_to_wait_more -= 1 - if not willing_to_wait_more: - raise Exception("thread didn't start?") - time.sleep(0.01) - assert state.my_thread_ident != main_ident - assert state.my_thread_ident == ident - assert state.seen_value == i - # try to force Boehm to do some freeing - for i in range(3): - gc.collect() - return state.freed_counter + state.my_thread_ident1 = get_ident() + state.my_thread_ident2 = get_ident() + start_new_thread(bootstrap1, ()) + start_new_thread(bootstrap2, ()) + willing_to_wait_more = 1000 + while (state.my_thread_ident1 == get_ident() or + state.my_thread_ident2 == get_ident()): + willing_to_wait_more -= 1 + if not willing_to_wait_more: + raise Exception("thread didn't start?") + time.sleep(0.01) + return 42 fn = self.getcompiled(f, []) - freed_counter = fn() - print freed_counter - if self.gcpolicy == 'boehm': - assert freed_counter > 0 - else: - assert freed_counter == 50 + res = fn() + assert res == 42 def test_gc_locking(self): import time @@ -202,6 +164,9 @@ def getcompiled(self, f, argtypes): return f + def test_start_new_thread(self): + py.test.skip("deadlocks occasionally -- why???") + class TestUsingBoehm(AbstractThreadTests): gcpolicy = 'boehm' diff --git a/pypy/rlib/objectmodel.py b/pypy/rlib/objectmodel.py --- a/pypy/rlib/objectmodel.py +++ b/pypy/rlib/objectmodel.py @@ -130,19 +130,29 @@ if self is other: return 0 else: - raise TypeError("Symbolics can not be compared!") + raise TypeError("Symbolics can not be compared! (%r, %r)" + % (self, other)) def __hash__(self): - raise TypeError("Symbolics are not hashable!") + raise TypeError("Symbolics are not hashable! %r" % (self,)) def __nonzero__(self): - raise TypeError("Symbolics are not comparable") + raise TypeError("Symbolics are not comparable! %r" % (self,)) class ComputedIntSymbolic(Symbolic): def __init__(self, compute_fn): self.compute_fn = compute_fn + def __repr__(self): + # repr(self.compute_fn) can arrive back here in an + # infinite recursion + try: + name = self.compute_fn.__name__ + except (AttributeError, TypeError): + name = hex(id(self.compute_fn)) + return '%s(%r)' % (self.__class__.__name__, name) + def annotation(self): from pypy.annotation import model return model.SomeInteger() @@ -157,6 +167,9 @@ self.expr = expr self.default = default + def __repr__(self): + return '%s(%r)' % (self.__class__.__name__, self.expr) + def annotation(self): from pypy.annotation import model return model.SomeInteger() diff --git a/pypy/rpython/lltypesystem/llgroup.py b/pypy/rpython/lltypesystem/llgroup.py --- a/pypy/rpython/lltypesystem/llgroup.py +++ b/pypy/rpython/lltypesystem/llgroup.py @@ -76,6 +76,10 @@ self.index = memberindex self.member = grp.members[memberindex]._as_ptr() + def __repr__(self): + return '%s(%s, %s)' % (self.__class__.__name__, + self.grpptr, self.index) + def __nonzero__(self): return True diff --git a/pypy/rpython/lltypesystem/rffi.py b/pypy/rpython/lltypesystem/rffi.py --- a/pypy/rpython/lltypesystem/rffi.py +++ b/pypy/rpython/lltypesystem/rffi.py @@ -27,6 +27,10 @@ self.c_name = c_name self.TP = TP + def __repr__(self): + return '%s(%r, %s)' % (self.__class__.__name__, + self.c_name, self.TP) + def annotation(self): return lltype_to_annotation(self.TP) From noreply at buildbot.pypy.org Thu Jan 26 18:13:50 2012 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 26 Jan 2012 18:13:50 +0100 (CET) Subject: [pypy-commit] pypy numpy-back-to-applevel: start implementing iter Message-ID: <20120126171350.B334682B22@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: numpy-back-to-applevel Changeset: r51803:9698baa9d6b5 Date: 2012-01-26 19:13 +0200 http://bitbucket.org/pypy/pypy/changeset/9698baa9d6b5/ Log: start implementing iter diff --git a/lib_pypy/numpypy/core/numeric.py b/lib_pypy/numpypy/core/numeric.py --- a/lib_pypy/numpypy/core/numeric.py +++ b/lib_pypy/numpypy/core/numeric.py @@ -1,5 +1,6 @@ from _numpypy import array, ndarray, int_, float_ #, complex_# , longlong +from _numpypy import concatenate import sys import _numpypy as multiarray # ARGH from numpypy.core.arrayprint import array2string diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -645,6 +645,13 @@ res_i = res_i.next(shapelen) return res + def descr_item(self, space, w_arg=None): + if space.is_w(w_arg, space.w_None): + if not isinstance(self, Scalar): + raise OperationError(space.w_ValueError, space.wrap("index out of bounds")) + return self.value.wrap(space) + xxx + def convert_to_array(space, w_obj): if isinstance(w_obj, BaseArray): return w_obj @@ -1357,6 +1364,7 @@ BaseArray.descr_set_shape), size = GetSetProperty(BaseArray.descr_get_size), ndim = GetSetProperty(BaseArray.descr_get_ndim), + item = interp2app(BaseArray.descr_item), T = GetSetProperty(BaseArray.descr_get_transpose), flat = GetSetProperty(BaseArray.descr_get_flatiter), diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -1455,6 +1455,15 @@ assert (a.compress([True, False, True]) == [0, 2]).all() raises(IndexError, "a.compress([1] * 100)") + def test_item(self): + from _numpypy import array + assert array(3).item() == 3 + assert type(array(3).item()) is int + assert type(array(True).item()) is bool + assert type(array(3.5).item()) is float + raises((ValueError, IndexError), "array(3).item(15)") + raises(ValueError, "array([1, 2, 3]).item()") + xxx class AppTestSupport(BaseNumpyAppTest): def setup_class(cls): From noreply at buildbot.pypy.org Thu Jan 26 18:21:59 2012 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 26 Jan 2012 18:21:59 +0100 (CET) Subject: [pypy-commit] pypy default: obscure dtype equality Message-ID: <20120126172159.886CA82B1F@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: Changeset: r51804:b4faea8e796e Date: 2012-01-26 19:21 +0200 http://bitbucket.org/pypy/pypy/changeset/b4faea8e796e/ Log: obscure dtype equality diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py --- a/pypy/module/micronumpy/interp_dtype.py +++ b/pypy/module/micronumpy/interp_dtype.py @@ -89,6 +89,19 @@ def descr_get_shape(self, space): return space.newtuple([]) + def eq(self, space, w_other): + if space.is_w(self, w_other): + return True + if space.isinstance_w(w_other, space.w_str): + return self.name == space.str_w(w_other) + return False + + def descr_eq(self, space, w_other): + return space.wrap(self.eq(space, w_other)) + + def descr_ne(self, space, w_other): + return space.wrap(not self.eq(space, w_other)) + def is_int_type(self): return self.kind == SIGNEDLTR or self.kind == UNSIGNEDLTR @@ -101,6 +114,8 @@ __str__= interp2app(W_Dtype.descr_str), __repr__ = interp2app(W_Dtype.descr_repr), + __eq__ = interp2app(W_Dtype.descr_eq), + __ne__ = interp2app(W_Dtype.descr_ne), num = interp_attrproperty("num", cls=W_Dtype), kind = interp_attrproperty("kind", cls=W_Dtype), diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py --- a/pypy/module/micronumpy/test/test_dtypes.py +++ b/pypy/module/micronumpy/test/test_dtypes.py @@ -9,6 +9,9 @@ assert d.num == 0 assert d.kind == 'b' assert dtype('int8').num == 1 + assert dtype('int8') == 'int8' + assert 'int8' == dtype('int8') + assert dtype('int8') != 3 assert dtype(d) is d assert dtype(None) is dtype(float) raises(TypeError, dtype, 1042) From noreply at buildbot.pypy.org Thu Jan 26 19:19:39 2012 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 26 Jan 2012 19:19:39 +0100 (CET) Subject: [pypy-commit] extradoc extradoc: add an optimization Message-ID: <20120126181939.C4CE282B1F@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: extradoc Changeset: r4051:4acf71058742 Date: 2012-01-26 20:17 +0200 http://bitbucket.org/pypy/extradoc/changeset/4acf71058742/ Log: add an optimization diff --git a/planning/micronumpy.txt b/planning/micronumpy.txt --- a/planning/micronumpy.txt +++ b/planning/micronumpy.txt @@ -36,3 +36,7 @@ in order to improve though - iterators should possibly not require reinstantiating them. + +- things like take/item/fancy indexing can use some knowledge about the density + of data and either evaluate interesting points (without forcing) or + do what they do now. From noreply at buildbot.pypy.org Thu Jan 26 19:19:40 2012 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 26 Jan 2012 19:19:40 +0100 (CET) Subject: [pypy-commit] extradoc extradoc: merge Message-ID: <20120126181940.F200882B1F@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: extradoc Changeset: r4052:da88aafb0de1 Date: 2012-01-26 20:19 +0200 http://bitbucket.org/pypy/extradoc/changeset/da88aafb0de1/ Log: merge diff --git a/blog/draft/pypy-2011.rst b/blog/draft/pypy-2011.rst --- a/blog/draft/pypy-2011.rst +++ b/blog/draft/pypy-2011.rst @@ -3,31 +3,44 @@ Hello everyone. -PyPy development is fast, sometimes very fast. We try to do 3-4 releases a year -and yet a lot of time when someone complains about the last release being slow -we usually say "oh, but it's the release, that was AGES ago". That makes -something as big as summarizing a year next to impossible, but using my -external IQ amplifiers like internet and hg log, I will try to provide you -with some semi-serious statistics: +PyPy's development is fast, sometimes very fast. We try to do 3-4 releases a +year and still many times when someone complains about the latest release being +slow we usually say "oh, that release was AGES ago". This makes something as +big as summarizing a year next to impossible, but using my external IQ +amplifiers, like internet and hg log, I will try to provide you with some +semi-serious statistics: -* We made 3 releases last year, boringly, 1.5, 1.6 and 1.7 +* We did 3 releases last year, boringly named, 1.5, 1.6 and 1.7 -* Number of visitors on pypy.org grew from about 200 to 800 per day (averaged - weekly) +* The number of visitors on pypy.org grew from about 200 to 800 per day + (averaged weekly) * We spoke at 15 conferences/meetups + (XXX: can we put a list in a comment to be sure it's correct) -* We made 10660 commits, or 29 per day (mostly typos ;-) +* We made 10660 commits, or 29 per day (mostly typos ;-)) * We published 43 blog posts, keeping you entertained almost weekly -* We made PyPy over 2x faster (on a select set of benchmarks), +* We made PyPy over 2x faster (on a select set of benchmarks), see the `1.4 and nightly`_ comparison -* We made PyPy 17x more compatible (of course this is unmeasurable, so why - not just claim it) +* 71 brand new people contributed at least one commit to the PyPy repository. + (XXX: this is what I got by doing wc on contributors.txt at 2011-01-01 and + the current contributors.rst, but we still need to update the latter) -* We probably have [XXX insert unicode for infinity] infinitely many times more users now, since noone used - 1.4 in production +* We made PyPy 17x more compatible (of course this is not measurable, so we're + going to claim it!) + +* We have [XXX insert unicode for infinity] infinitely many times more users + now, since no one used 1.4 in production + +2011 was a very exciting year for us. But we're pretty sure 2012 is going to be +even more exciting! So watch this space for more exciting news, try PyPy out on +your projects, and, as always, we invite you to join us and contribute to PyPy +in any way you can. + +Signed, +The PyPy Developers .. _`1.4 and nightly` http://speed.pypy.org/comparison/?exe=1%2B172%2C1%2BL%2Bdefault&ben=1%2C34%2C27%2C2%2C25%2C3%2C4%2C5%2C22%2C6%2C39%2C7%2C8%2C23%2C24%2C9%2C10%2C11%2C12%2C13%2C14%2C15%2C35%2C36%2C37%2C38%2C16%2C28%2C30%2C32%2C29%2C33%2C17%2C18%2C19%2C20&env=1%2C2&hor=true&bas=1%2B172&chart=normal+bars diff --git a/planning/micronumpy.txt b/planning/micronumpy.txt --- a/planning/micronumpy.txt +++ b/planning/micronumpy.txt @@ -19,6 +19,8 @@ - expose ndarray.ctypes +- Make .T not force a lazy array. + - subclassing ndarray (instantiating subcalsses curently returns the wrong type) * keep subclass type when slicing, __array_finalize__ diff --git a/sprintinfo/leysin-winter-2012/planning.txt b/sprintinfo/leysin-winter-2012/planning.txt --- a/sprintinfo/leysin-winter-2012/planning.txt +++ b/sprintinfo/leysin-winter-2012/planning.txt @@ -2,26 +2,24 @@ People present -------------- -* Antonio Cuni * Armin Rigo * Romain Guillebert +* David Schneider Things we want to do -------------------- -* some skiing: wednesday (last day of sun :-/ ) +* some skiing (DONE) * review the JVM backend pull request (DONE) -* py3k (romain, anto later) +* py3k (romain) (little bit of progress) -* ffistruct - -* Cython backend +* Debug the ARM backend (bivab, armin around) (some more progress) * STM - - refactored the RPython API: mostly done, must adapt targetdemo*.py - (anto, arigo) - - start work on the GC (arigo later?) + - refactored the RPython API (DONE) + - app-level transaction module (DONE) + - start work on the GC (armin) -* concurrent-marksweep GC +* lightweight tracing experiment (DONE but still experimental) From noreply at buildbot.pypy.org Thu Jan 26 19:37:49 2012 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 26 Jan 2012 19:37:49 +0100 (CET) Subject: [pypy-commit] pypy default: import to_coords from numppy-flatitter branch and modify it. Thanks mattip! Message-ID: <20120126183749.2A96482B1F@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: Changeset: r51805:9183b721128a Date: 2012-01-26 20:37 +0200 http://bitbucket.org/pypy/pypy/changeset/9183b721128a/ Log: import to_coords from numppy-flatitter branch and modify it. Thanks mattip! diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py --- a/pypy/module/micronumpy/strides.py +++ b/pypy/module/micronumpy/strides.py @@ -1,4 +1,5 @@ from pypy.rlib import jit +from pypy.interpreter.error import OperationError @jit.look_inside_iff(lambda shape, start, strides, backstrides, chunks: @@ -37,3 +38,27 @@ rstrides = [0] * (len(res_shape) - len(orig_shape)) + rstrides rbackstrides = [0] * (len(res_shape) - len(orig_shape)) + rbackstrides return rstrides, rbackstrides + +def to_coords(space, shape, size, order, w_item_or_slice): + '''Returns a start coord, step, and length. + ''' + start = lngth = step = 0 + if not (space.isinstance_w(w_item_or_slice, space.w_int) or + space.isinstance_w(w_item_or_slice, space.w_slice)): + raise OperationError(space.w_IndexError, + space.wrap('unsupported iterator index')) + + start, stop, step, lngth = space.decode_index4(w_item_or_slice, size) + + coords = [0] * len(shape) + i = start + if order == 'C': + for s in range(len(shape) -1, -1, -1): + coords[s] = i % shape[s] + i //= shape[s] + else: + for s in range(len(shape)): + coords[s] = i % shape[s] + i //= shape[s] + + return coords, step, lngth diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -167,6 +167,24 @@ assert calc_new_strides([1, 1, 105, 1, 1], [7, 15], [1, 7],'F') == \ [1, 1, 1, 105, 105] + def test_to_coords(self): + from pypy.module.micronumpy.strides import to_coords + + def _to_coords(index, order): + return to_coords(self.space, [2, 3, 4], 24, order, + self.space.wrap(index))[0] + + assert _to_coords(0, 'C') == [0, 0, 0] + assert _to_coords(1, 'C') == [0, 0, 1] + assert _to_coords(-1, 'C') == [1, 2, 3] + assert _to_coords(5, 'C') == [0, 1, 1] + assert _to_coords(13, 'C') == [1, 0, 1] + assert _to_coords(0, 'F') == [0, 0, 0] + assert _to_coords(1, 'F') == [1, 0, 0] + assert _to_coords(-1, 'F') == [1, 2, 3] + assert _to_coords(5, 'F') == [1, 2, 0] + assert _to_coords(13, 'F') == [1, 0, 2] + class AppTestNumArray(BaseNumpyAppTest): def test_ndarray(self): From noreply at buildbot.pypy.org Thu Jan 26 20:25:20 2012 From: noreply at buildbot.pypy.org (mattip) Date: Thu, 26 Jan 2012 20:25:20 +0100 (CET) Subject: [pypy-commit] pypy numppy-flatitter: rework next_skip_x, there still is an AnnotationError Message-ID: <20120126192520.246BC82B1F@wyvern.cs.uni-duesseldorf.de> Author: mattip Branch: numppy-flatitter Changeset: r51806:6d23b2d09136 Date: 2012-01-19 07:29 +0200 http://bitbucket.org/pypy/pypy/changeset/6d23b2d09136/ Log: rework next_skip_x, there still is an AnnotationError diff --git a/pypy/module/micronumpy/REVIEW b/pypy/module/micronumpy/REVIEW deleted file mode 100644 --- a/pypy/module/micronumpy/REVIEW +++ /dev/null @@ -1,25 +0,0 @@ -* this file should go away before asking for a review :) - -* I think we should wait for indexing-by-arrays-2, since this would clean up - the iterator interface - -* For getitem, we need to reuse parent getitem, but with some hook - that recomputes indexes. That hook would be used for any sort of access, - be it slices or be it integers, but in general I would like to avoid code - duplication, since the indexing is getting slowly fairly complex. - -* iterating over a transposed array still fails. - -* next_skip_x is implemented badly. It still iterates over items (calls - next_skip n times) and is jit.unroll_safe. This means that the JIT will assume - skip is a constant and compile a linear trace unrolling all iterations. This - is *BAD*. Instead it would be something like: - - for arrays: - - res.offset = self.offset + skip - - for views something slightly more complex that does appropriate modulo. - - Note that next() has unroll_safe, because the loop only dependas on the - length of shape which is a constant for assembler. \ No newline at end of file diff --git a/pypy/module/micronumpy/interp_iter.py b/pypy/module/micronumpy/interp_iter.py --- a/pypy/module/micronumpy/interp_iter.py +++ b/pypy/module/micronumpy/interp_iter.py @@ -51,9 +51,9 @@ self.size = size def next(self, shapelen): - return self._next(1) + return self.next_skip_x(1) - def _next(self, ofs): + def next_skip_x(self, ofs): arr = instantiate(ArrayIterator) arr.size = self.size arr.offset = self.offset + ofs @@ -61,7 +61,7 @@ def next_no_increase(self, shapelen): # a hack to make JIT believe this is always virtual - return self._next(0) + return self.next_skip_x(0) def done(self): return self.offset >= self.size @@ -133,11 +133,34 @@ res._done = done return res - @jit.unroll_safe + #@jit.unroll_safe def next_skip_x(self, shapelen, step): - res = self.next(shapelen) - for x in range(step-1): - res = res.next(shapelen) + shapelen = jit.promote(len(self.res_shape)) + offset = self.offset + indices = [0] * shapelen + for i in range(shapelen): + indices[i] = self.indices[i] + done = False + for i in range(shapelen - 1, -1, -1): + if indices[i] < self.res_shape[i] - step: + indices[i] += step + offset += self.strides[i] * step + break + else: + remaining_step = (indices[i] + step) // self.res_shape[i] + this_i_step = step - remaining_step * self.res_shape[i] + offset += self.strides[i] * this_i_step + indices[i] = indices[i] + this_i_step + step = remaining_step + else: + done = True + res = instantiate(ViewIterator) + res.offset = offset + res.indices = indices + res.strides = self.strides + res.backstrides = self.backstrides + res.res_shape = self.res_shape + res._done = done return res def apply_transformations(self, arr, transformations): diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -1484,8 +1484,7 @@ basei = ViewIterator(self.base.start, self.base.strides, self.base.backstrides,self.base.shape) shapelen = len(self.base.shape) - if start > 0: - basei = basei.next_skip_x(shapelen, start) + basei = basei.next_skip_x(shapelen, start) ri = ArrayIterator(lngth) while not ri.done(): # TODO: add a jit_merge_point @@ -1502,8 +1501,7 @@ basei = ViewIterator(self.base.start, self.base.strides, self.base.backstrides,self.base.shape) shapelen = len(self.base.shape) - if start > 0: - basei = basei.next_skip_x(shapelen, start) + basei = basei.next_skip_x(shapelen, start) for i in range(lngth): # TODO: add jit_merge_point v = arr.getitem(ai).convert_to(self.base.dtype) From noreply at buildbot.pypy.org Thu Jan 26 20:31:42 2012 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 26 Jan 2012 20:31:42 +0100 (CET) Subject: [pypy-commit] extradoc extradoc: Start a draft about STM+GC. Message-ID: <20120126193142.2EE3C82B1F@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: extradoc Changeset: r4053:12399d963afe Date: 2012-01-26 20:31 +0100 http://bitbucket.org/pypy/extradoc/changeset/12399d963afe/ Log: Start a draft about STM+GC. diff --git a/planning/stm.txt b/planning/stm.txt new file mode 100644 --- /dev/null +++ b/planning/stm.txt @@ -0,0 +1,61 @@ +============ +STM planning +============ + +Overview +-------- + +A saner approach (and likely better results that now): integrate with +the GC. Here is the basic plan. + +Let T be the number of threads. Use a custom GC, with T nurseries and +one "global area." Every object in the nursery t is only visible to +thread t. Every object in the global area is shared but read-only. +Changes to global objects are only done by committing. + +Every thread t allocates new objects in the nursery t. Accesses to +nursery objects are the fastest, not monitored at all. When we need +read access to a global object, we can read it directly, but we need to +record the version of the object that we read. When we need write +access to a global object, we need to make a whole copy of it into our +nursery. + +The RPython program should have at least one hint: "force for writing", +which is like writing to an object in the sense that it forces a local +copy. + +We need annotator support to track which variables contain objects that +are known to be local. It lets us avoid the run-time check. That's +useful for all freshly malloc'ed objects, which we know are always +local; and that's useful for special cases like the PyFrames, on which +we would use the "force for writing" hint before running the +interpreter. In both cases the result is: no STM code is needed any +more. + +When a transaction commits, we do a "minor collection"-like process: we +move all surviving objects from the nursery to the global area, either +as new objects, or as overwrites of their previous version. So there is +one "minor collection" at the end of every transaction. Unlike the +minor collections in other GCs, this one occurs at a well-defined time, +with no stack roots to scan. + +Later we'll need to consider what occurs if a nursery grows too big +while the transaction is still not finished. Probably somehow run a +collection of the nursery itself, not touching the global area. + +Of course we also need to do from time to time a major collection. We +will need at some point some concurrency here, to be able to run the +major collection in a random thread t but detecting changes done by the +other threads overwriting objects during their own minor collections. + + +GC flags +-------- + +Still open to consideration, but the basic GC flags could be: + + * GC_GLOBAL if the object is in the global area + + * GC_WAS_COPIED on a global object: it has at least one local copy + (then we need to look it up in some local dictionary) + on a local object: it comes from a global object From noreply at buildbot.pypy.org Thu Jan 26 20:55:24 2012 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 26 Jan 2012 20:55:24 +0100 (CET) Subject: [pypy-commit] pypy numpy-back-to-applevel: item, basic support Message-ID: <20120126195524.B707682B20@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: numpy-back-to-applevel Changeset: r51808:b1cb2bf3f898 Date: 2012-01-26 21:40 +0200 http://bitbucket.org/pypy/pypy/changeset/b1cb2bf3f898/ Log: item, basic support diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py --- a/pypy/module/micronumpy/interp_boxes.py +++ b/pypy/module/micronumpy/interp_boxes.py @@ -101,6 +101,9 @@ class W_BoolBox(W_GenericBox, PrimitiveBox): descr__new__, get_dtype = new_dtype_getter("bool") + def wrap(self, space): + return space.wrap(self.value) + class W_NumberBox(W_GenericBox): _attrs_ = () @@ -108,6 +111,9 @@ def int_w(self, space): return rffi.cast(lltype.Signed, self.value) + def wrap(self, space): + return space.wrap(rffi.cast(lltype.Signed, self.value)) + class W_SignedIntegerBox(W_IntegerBox): pass @@ -150,6 +156,9 @@ class W_FloatingBox(W_InexactBox): _attrs_ = () + def wrap(self, space): + return space.wrap(rffi.cast(lltype.Float, self.value)) + class W_Float32Box(W_FloatingBox, PrimitiveBox): descr__new__, get_dtype = new_dtype_getter("float32") diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -5,7 +5,7 @@ from pypy.module.micronumpy import interp_ufuncs, interp_dtype, signature from pypy.module.micronumpy.strides import calculate_slice_strides,\ shape_agreement, find_shape_and_elems, get_shape_from_iterable,\ - calc_new_strides + calc_new_strides, to_coords from pypy.rlib import jit from pypy.rpython.lltypesystem import lltype, rffi from pypy.tool.sourcetools import func_with_new_name @@ -650,6 +650,11 @@ if not isinstance(self, Scalar): raise OperationError(space.w_ValueError, space.wrap("index out of bounds")) return self.value.wrap(space) + if space.isinstance_w(w_arg, space.w_int): + i = to_coords(space, self.shape, self.size, self.order, w_arg)[0] + # XXX a bit around + return self.descr_getitem(space.newtuple([space.wrap(x) + for x in i])) xxx def convert_to_array(space, w_obj): @@ -671,6 +676,7 @@ Intermediate class representing a literal. """ size = 1 + order = 'C' _attrs_ = ["dtype", "value", "shape"] def __init__(self, dtype, value): diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -1481,7 +1481,6 @@ assert type(array(3.5).item()) is float raises((ValueError, IndexError), "array(3).item(15)") raises(ValueError, "array([1, 2, 3]).item()") - xxx class AppTestSupport(BaseNumpyAppTest): def setup_class(cls): From noreply at buildbot.pypy.org Thu Jan 26 20:55:23 2012 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 26 Jan 2012 20:55:23 +0100 (CET) Subject: [pypy-commit] pypy numpy-back-to-applevel: merge default. why oh why so many conflicts??? Message-ID: <20120126195523.7856082B1F@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: numpy-back-to-applevel Changeset: r51807:e74b4e330798 Date: 2012-01-26 21:24 +0200 http://bitbucket.org/pypy/pypy/changeset/e74b4e330798/ Log: merge default. why oh why so many conflicts??? diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py --- a/pypy/module/micronumpy/interp_dtype.py +++ b/pypy/module/micronumpy/interp_dtype.py @@ -89,6 +89,19 @@ def descr_get_shape(self, space): return space.newtuple([]) + def eq(self, space, w_other): + if space.is_w(self, w_other): + return True + if space.isinstance_w(w_other, space.w_str): + return self.name == space.str_w(w_other) + return False + + def descr_eq(self, space, w_other): + return space.wrap(self.eq(space, w_other)) + + def descr_ne(self, space, w_other): + return space.wrap(not self.eq(space, w_other)) + def is_int_type(self): return (self.kind == SIGNEDLTR or self.kind == UNSIGNEDLTR or self.kind == BOOLLTR) @@ -102,6 +115,8 @@ __str__= interp2app(W_Dtype.descr_str), __repr__ = interp2app(W_Dtype.descr_repr), + __eq__ = interp2app(W_Dtype.descr_eq), + __ne__ = interp2app(W_Dtype.descr_ne), num = interp_attrproperty("num", cls=W_Dtype), kind = interp_attrproperty("kind", cls=W_Dtype), diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py --- a/pypy/module/micronumpy/strides.py +++ b/pypy/module/micronumpy/strides.py @@ -60,6 +60,29 @@ shape.append(size) batch = new_batch +def to_coords(space, shape, size, order, w_item_or_slice): + '''Returns a start coord, step, and length. + ''' + start = lngth = step = 0 + if not (space.isinstance_w(w_item_or_slice, space.w_int) or + space.isinstance_w(w_item_or_slice, space.w_slice)): + raise OperationError(space.w_IndexError, + space.wrap('unsupported iterator index')) + + start, stop, step, lngth = space.decode_index4(w_item_or_slice, size) + + coords = [0] * len(shape) + i = start + if order == 'C': + for s in range(len(shape) -1, -1, -1): + coords[s] = i % shape[s] + i //= shape[s] + else: + for s in range(len(shape)): + coords[s] = i % shape[s] + i //= shape[s] + return coords, step, lngth + def shape_agreement(space, shape1, shape2): ret = _shape_agreement(shape1, shape2) if len(ret) < max(len(shape1), len(shape2)): diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py --- a/pypy/module/micronumpy/test/test_dtypes.py +++ b/pypy/module/micronumpy/test/test_dtypes.py @@ -9,6 +9,9 @@ assert d.num == 0 assert d.kind == 'b' assert dtype('int8').num == 1 + assert dtype('int8') == 'int8' + assert 'int8' == dtype('int8') + assert dtype('int8') != 3 assert dtype(d) is d assert dtype(None) is dtype(float) assert dtype('int8').name == 'int8' diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -167,6 +167,24 @@ assert calc_new_strides([1, 1, 105, 1, 1], [7, 15], [1, 7],'F') == \ [1, 1, 1, 105, 105] + def test_to_coords(self): + from pypy.module.micronumpy.strides import to_coords + + def _to_coords(index, order): + return to_coords(self.space, [2, 3, 4], 24, order, + self.space.wrap(index))[0] + + assert _to_coords(0, 'C') == [0, 0, 0] + assert _to_coords(1, 'C') == [0, 0, 1] + assert _to_coords(-1, 'C') == [1, 2, 3] + assert _to_coords(5, 'C') == [0, 1, 1] + assert _to_coords(13, 'C') == [1, 0, 1] + assert _to_coords(0, 'F') == [0, 0, 0] + assert _to_coords(1, 'F') == [1, 0, 0] + assert _to_coords(-1, 'F') == [1, 2, 3] + assert _to_coords(5, 'F') == [1, 2, 0] + assert _to_coords(13, 'F') == [1, 0, 2] + class AppTestNumArray(BaseNumpyAppTest): def test_ndarray(self): From noreply at buildbot.pypy.org Thu Jan 26 20:55:25 2012 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 26 Jan 2012 20:55:25 +0100 (CET) Subject: [pypy-commit] pypy numpy-back-to-applevel: make tests pass with -A and return scalars on zeros(()). A bit hard to test Message-ID: <20120126195525.EAAFA82B22@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: numpy-back-to-applevel Changeset: r51809:ae72e452c1fc Date: 2012-01-26 21:54 +0200 http://bitbucket.org/pypy/pypy/changeset/ae72e452c1fc/ Log: make tests pass with -A and return scalars on zeros(()). A bit hard to test without is_scalar, but zeros is going back to applevel at some point soon diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -1250,6 +1250,8 @@ space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype) ) size, shape = _find_size_and_shape(space, w_size) + if not shape: + return scalar_w(space, dtype, space.wrap(0)) return space.wrap(W_NDimArray(size, shape[:], dtype=dtype)) def ones(space, w_size, w_dtype=None): @@ -1258,6 +1260,8 @@ ) size, shape = _find_size_and_shape(space, w_size) + if not shape: + return scalar_w(space, dtype, space.wrap(1)) arr = W_NDimArray(size, shape[:], dtype=dtype) one = dtype.box(1) arr.dtype.fill(arr.storage, one, 0, size) diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -229,6 +229,7 @@ # And check that changes stick. a[13] = 5.3 assert a[13] == 5.3 + assert zeros(()).shape == () def test_size(self): from _numpypy import array @@ -388,6 +389,7 @@ assert b.shape == (5,) c = a[:3] assert c.shape == (3,) + assert array([]).shape == (0,) def test_set_shape(self): from _numpypy import array, zeros @@ -1128,7 +1130,7 @@ assert _numpypy.array([[1], [2], [3]]).shape == (3, 1) assert len(_numpypy.zeros((3, 1, 2))) == 3 raises(TypeError, len, _numpypy.zeros(())) - raises(ValueError, _numpypy.array, [[1, 2], 3]) + raises(ValueError, _numpypy.array, [[1, 2], 3], dtype=float) def test_getsetitem(self): import _numpypy From noreply at buildbot.pypy.org Thu Jan 26 20:59:34 2012 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 26 Jan 2012 20:59:34 +0100 (CET) Subject: [pypy-commit] pypy numpy-back-to-applevel: a test and a fix Message-ID: <20120126195934.2D9B682B1F@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: numpy-back-to-applevel Changeset: r51810:48f4c0d75310 Date: 2012-01-26 21:59 +0200 http://bitbucket.org/pypy/pypy/changeset/48f4c0d75310/ Log: a test and a fix diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -511,7 +511,7 @@ for nd in range(ndims): new_backstrides[nd] = (new_shape[nd] - 1) * new_strides[nd] arr = W_NDimSlice(concrete.start, new_strides, new_backstrides, - new_shape, self) + new_shape, concrete) else: # Create copy with contiguous data arr = concrete.copy(space) diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -425,6 +425,7 @@ assert (a == [1000, 1, 2, 3, 1000, 5, 6, 7, 1000, 9, 10, 11]).all() a = zeros((4, 2, 3)) a.shape = (12, 2) + (a + a).reshape(2, 12) # assert did not explode def test_slice_reshape(self): from _numpypy import zeros, arange From noreply at buildbot.pypy.org Thu Jan 26 21:02:05 2012 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 26 Jan 2012 21:02:05 +0100 (CET) Subject: [pypy-commit] pypy numpy-back-to-applevel: make the unsupported case applevel Message-ID: <20120126200205.41B2A82B1F@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: numpy-back-to-applevel Changeset: r51811:ddf5b962aa39 Date: 2012-01-26 22:01 +0200 http://bitbucket.org/pypy/pypy/changeset/ddf5b962aa39/ Log: make the unsupported case applevel diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -655,7 +655,8 @@ # XXX a bit around return self.descr_getitem(space.newtuple([space.wrap(x) for x in i])) - xxx + raise OperationError(space.w_NotImplementedError, space.wrap( + "non-int arg not supported")) def convert_to_array(space, w_obj): if isinstance(w_obj, BaseArray): From noreply at buildbot.pypy.org Thu Jan 26 21:09:34 2012 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 26 Jan 2012 21:09:34 +0100 (CET) Subject: [pypy-commit] pypy numpy-back-to-applevel: rename wrap to item and merge with tolist. Improve item() and test Message-ID: <20120126200934.CD9C482B1F@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: numpy-back-to-applevel Changeset: r51812:c3680933a008 Date: 2012-01-26 22:09 +0200 http://bitbucket.org/pypy/pypy/changeset/c3680933a008/ Log: rename wrap to item and merge with tolist. Improve item() and test diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py --- a/pypy/module/micronumpy/interp_boxes.py +++ b/pypy/module/micronumpy/interp_boxes.py @@ -94,16 +94,13 @@ descr_neg = _unaryop_impl("negative") descr_abs = _unaryop_impl("absolute") - def descr_tolist(self, space): + def item(self, space): return self.get_dtype(space).itemtype.to_builtin_type(space, self) class W_BoolBox(W_GenericBox, PrimitiveBox): descr__new__, get_dtype = new_dtype_getter("bool") - def wrap(self, space): - return space.wrap(self.value) - class W_NumberBox(W_GenericBox): _attrs_ = () @@ -111,9 +108,6 @@ def int_w(self, space): return rffi.cast(lltype.Signed, self.value) - def wrap(self, space): - return space.wrap(rffi.cast(lltype.Signed, self.value)) - class W_SignedIntegerBox(W_IntegerBox): pass @@ -156,9 +150,6 @@ class W_FloatingBox(W_InexactBox): _attrs_ = () - def wrap(self, space): - return space.wrap(rffi.cast(lltype.Float, self.value)) - class W_Float32Box(W_FloatingBox, PrimitiveBox): descr__new__, get_dtype = new_dtype_getter("float32") @@ -198,7 +189,7 @@ __neg__ = interp2app(W_GenericBox.descr_neg), __abs__ = interp2app(W_GenericBox.descr_abs), - tolist = interp2app(W_GenericBox.descr_tolist), + tolist = interp2app(W_GenericBox.item), ) W_BoolBox.typedef = TypeDef("bool_", W_GenericBox.typedef, diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -649,12 +649,12 @@ if space.is_w(w_arg, space.w_None): if not isinstance(self, Scalar): raise OperationError(space.w_ValueError, space.wrap("index out of bounds")) - return self.value.wrap(space) + return self.value.item(space) if space.isinstance_w(w_arg, space.w_int): i = to_coords(space, self.shape, self.size, self.order, w_arg)[0] # XXX a bit around - return self.descr_getitem(space.newtuple([space.wrap(x) - for x in i])) + return self.descr_getitem(space, space.newtuple([space.wrap(x) + for x in i])).item(space) raise OperationError(space.w_NotImplementedError, space.wrap( "non-int arg not supported")) diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -1484,6 +1484,9 @@ assert type(array(3.5).item()) is float raises((ValueError, IndexError), "array(3).item(15)") raises(ValueError, "array([1, 2, 3]).item()") + assert array([3]).item(0) == 3 + assert type(array([3]).item(0)) is int + assert array([1, 2, 3]).item(-1) == 3 class AppTestSupport(BaseNumpyAppTest): def setup_class(cls): From noreply at buildbot.pypy.org Thu Jan 26 21:13:32 2012 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Thu, 26 Jan 2012 21:13:32 +0100 (CET) Subject: [pypy-commit] pypy default: Fix dtype.__eq__ Message-ID: <20120126201332.619D082B1F@wyvern.cs.uni-duesseldorf.de> Author: Alex Gaynor <alex.gaynor at gmail.com> Branch: Changeset: r51813:0e63f32cf93c Date: 2012-01-26 15:13 -0500 http://bitbucket.org/pypy/pypy/changeset/0e63f32cf93c/ Log: Fix dtype.__eq__ diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py --- a/pypy/module/micronumpy/interp_dtype.py +++ b/pypy/module/micronumpy/interp_dtype.py @@ -90,11 +90,8 @@ return space.newtuple([]) def eq(self, space, w_other): - if space.is_w(self, w_other): - return True - if space.isinstance_w(w_other, space.w_str): - return self.name == space.str_w(w_other) - return False + w_other = space.call_function(space.gettypefor(W_Dtype), w_other) + return space.is_w(self, w_other) def descr_eq(self, space, w_other): return space.wrap(self.eq(space, w_other)) diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py --- a/pypy/module/micronumpy/test/test_dtypes.py +++ b/pypy/module/micronumpy/test/test_dtypes.py @@ -9,13 +9,18 @@ assert d.num == 0 assert d.kind == 'b' assert dtype('int8').num == 1 - assert dtype('int8') == 'int8' - assert 'int8' == dtype('int8') - assert dtype('int8') != 3 assert dtype(d) is d assert dtype(None) is dtype(float) raises(TypeError, dtype, 1042) + def test_dtype_eq(self): + from _numpypy import dtype + + assert dtype("int8") == "int8" + assert "int8" == dtype("int8") + raises(TypeError, lambda: dtype("int8") == 3) + assert dtype(bool) == bool + def test_dtype_with_types(self): from _numpypy import dtype From noreply at buildbot.pypy.org Thu Jan 26 21:14:25 2012 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 26 Jan 2012 21:14:25 +0100 (CET) Subject: [pypy-commit] extradoc extradoc: More. Message-ID: <20120126201425.573A782B1F@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: extradoc Changeset: r4054:79baef447541 Date: 2012-01-26 21:14 +0100 http://bitbucket.org/pypy/extradoc/changeset/79baef447541/ Log: More. diff --git a/planning/stm.txt b/planning/stm.txt --- a/planning/stm.txt +++ b/planning/stm.txt @@ -20,7 +20,7 @@ access to a global object, we need to make a whole copy of it into our nursery. -The RPython program should have at least one hint: "force for writing", +The RPython program should have at least one hint: "force local copy", which is like writing to an object in the sense that it forces a local copy. @@ -28,16 +28,16 @@ are known to be local. It lets us avoid the run-time check. That's useful for all freshly malloc'ed objects, which we know are always local; and that's useful for special cases like the PyFrames, on which -we would use the "force for writing" hint before running the +we would use the "force local copy" hint before running the interpreter. In both cases the result is: no STM code is needed any more. -When a transaction commits, we do a "minor collection"-like process: we -move all surviving objects from the nursery to the global area, either -as new objects, or as overwrites of their previous version. So there is -one "minor collection" at the end of every transaction. Unlike the -minor collections in other GCs, this one occurs at a well-defined time, -with no stack roots to scan. +When a transaction commits, we do a "minor collection"-like process, +called an "end-of-transaction collection": we move all surviving objects +from the nursery to the global area, either as new objects, or as +overwrites of their previous version. Unlike the minor collections in +other GCs, this one occurs at a well-defined time, with no stack roots +to scan. Later we'll need to consider what occurs if a nursery grows too big while the transaction is still not finished. Probably somehow run a @@ -46,7 +46,8 @@ Of course we also need to do from time to time a major collection. We will need at some point some concurrency here, to be able to run the major collection in a random thread t but detecting changes done by the -other threads overwriting objects during their own minor collections. +other threads overwriting objects during their own end-of-transaction +collections. GC flags @@ -59,3 +60,87 @@ * GC_WAS_COPIED on a global object: it has at least one local copy (then we need to look it up in some local dictionary) on a local object: it comes from a global object + + * and one complete word (for now?) for the version number, see below + +(Optimization: objects declared immutable don't need a version number.) + + +stm_read +-------- + +The STM read operation is potentially a complex operation. (That's why +it's useful to remove it as much as possible.) + +stm_read(obj, offset) -> field value + +- If obj is not GC_GLOBAL, then read directly and be done. + +- Otherwise, if GC_WAS_COPIED, and if we find 'localobj' in this + thread's local dictionary, then read directly from 'localobj' and + be done. (Ideally we should also use 'localobj' instead of 'obj' + in future references to this object, but unclear how.) + +- Otherwise, we need to do a global read. This is a real STM read. + Done (on x86 [1]) by reading the version number, then the actual field, + then *again* the version number. If the version number didn't change + and if it is not more recent than the transaction start, then the read + is accepted; otherwise not (we might retry or abort the transaction, + depending on cases). And if the read is accepted then we need to + remember in a local list that we've read that object. + + +stm_write +--------- + +- If obj is GC_GLOBAL, we need to find or make a local copy + +- Then we just perform the write. + +This means that stm_write could be implemented with a write barrier that +returns potentially a copy of the object, and which is followed by a +regular write to that copy. + +Note that "making a local copy" implies the same rules as stm_read: read +the version number, copy all fields, then read *again* the version +number [1]. If it didn't change, then we know that we got at least a +consistent copy (i.e. nobody changed the object in the middle of us +reading it). If it is too recent, then we might have to abort. + + +End-of-transaction collections +------------------------------ + +Start from the "roots" being all local copies of global objects. (These +are the only roots: if there are none, then it means we didn't write +anything in any global object, so there is no new object that can +survive.) From the roots, scan and move all fresh new objects to the +global area. Add the GC_GLOBAL flag to them, of course. Then we need, +atomically (in the STM sense), to overwrite the old global objects with +their local copies. This is done by temporarily locking the global +objects with a special value in their "version" field that will cause +concurrent reads to spin-loop. + + +Annotator support +----------------- + +To get good performance, we should as much as possible use the +'localobj' version of every object instead of the 'obj' one. At least +after a write barrier we should replace the local variable 'obj' with +'localobj', and someone (the annotator? or later?) should propagate the +fact that it is now a localobj that doesn't need special stm support +any longer. Similarly, all mallocs return a localobj. + +The "force local copy" hint should be used on PyFrame before the main +interpreter loop, so that we can then be sure that all accesses to +'frame' are to a local obj. Ideally, we could even track which fields +of a localobj are themselves localobjs. This would be useful for +'PyFrame.fastlocals_w': it should also be known to always be a localobj. + + +notes +----- + +[1] this relies on a property guaranteed so far by the x86, but not, + say, by PowerPCs. (XXX find a reference again) From noreply at buildbot.pypy.org Thu Jan 26 21:21:46 2012 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 26 Jan 2012 21:21:46 +0100 (CET) Subject: [pypy-commit] pypy numpy-back-to-applevel: oops Message-ID: <20120126202146.4469782B1F@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: numpy-back-to-applevel Changeset: r51814:0db7c34519db Date: 2012-01-26 22:21 +0200 http://bitbucket.org/pypy/pypy/changeset/0db7c34519db/ Log: oops diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -521,7 +521,7 @@ def descr_tolist(self, space): if len(self.shape) == 0: assert isinstance(self, Scalar) - return self.value.descr_tolist(space) + return self.value.item(space) w_result = space.newlist([]) for i in range(self.shape[0]): space.call_method(w_result, "append", From noreply at buildbot.pypy.org Thu Jan 26 21:27:08 2012 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 26 Jan 2012 21:27:08 +0100 (CET) Subject: [pypy-commit] extradoc extradoc: Add para Message-ID: <20120126202708.6102982B1F@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: extradoc Changeset: r4055:65d56ccf97b0 Date: 2012-01-26 21:26 +0100 http://bitbucket.org/pypy/extradoc/changeset/65d56ccf97b0/ Log: Add para diff --git a/planning/stm.txt b/planning/stm.txt --- a/planning/stm.txt +++ b/planning/stm.txt @@ -121,6 +121,10 @@ objects with a special value in their "version" field that will cause concurrent reads to spin-loop. +This is also where we need the list of global objects that we've read. +We need to check that each of these global objects' versions have not +been modified in the meantime. + Annotator support ----------------- From noreply at buildbot.pypy.org Thu Jan 26 21:32:16 2012 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Thu, 26 Jan 2012 21:32:16 +0100 (CET) Subject: [pypy-commit] pypy numpy-back-to-applevel: make this method RPython Message-ID: <20120126203216.43FBF82B1F@wyvern.cs.uni-duesseldorf.de> Author: Alex Gaynor <alex.gaynor at gmail.com> Branch: numpy-back-to-applevel Changeset: r51815:6d4db9f0618a Date: 2012-01-26 15:31 -0500 http://bitbucket.org/pypy/pypy/changeset/6d4db9f0618a/ Log: make this method RPython diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py --- a/pypy/module/micronumpy/interp_boxes.py +++ b/pypy/module/micronumpy/interp_boxes.py @@ -29,6 +29,7 @@ def convert_to(self, dtype): return dtype.box(self.value) + class W_GenericBox(Wrappable): _attrs_ = () @@ -106,7 +107,7 @@ class W_IntegerBox(W_NumberBox): def int_w(self, space): - return rffi.cast(lltype.Signed, self.value) + return space.int_w(self.descr_int(space)) class W_SignedIntegerBox(W_IntegerBox): pass From noreply at buildbot.pypy.org Thu Jan 26 21:35:30 2012 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 26 Jan 2012 21:35:30 +0100 (CET) Subject: [pypy-commit] pypy default: simplify targetnopstandalone Message-ID: <20120126203530.18D7482B1F@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: Changeset: r51816:ddbfdbe2d419 Date: 2012-01-26 22:33 +0200 http://bitbucket.org/pypy/pypy/changeset/ddbfdbe2d419/ Log: simplify targetnopstandalone diff --git a/pypy/translator/goal/targetnopstandalone.py b/pypy/translator/goal/targetnopstandalone.py --- a/pypy/translator/goal/targetnopstandalone.py +++ b/pypy/translator/goal/targetnopstandalone.py @@ -7,10 +7,8 @@ actually implementing argv of the executable. """ -import os, sys - -def debug(msg): - os.write(2, "debug: " + msg + '\n') +def debug(msg): + print "debug:", msg # __________ Entry point __________ From noreply at buildbot.pypy.org Thu Jan 26 21:35:31 2012 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 26 Jan 2012 21:35:31 +0100 (CET) Subject: [pypy-commit] pypy default: merge Message-ID: <20120126203531.4891782B1F@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: Changeset: r51817:0586c5404983 Date: 2012-01-26 22:34 +0200 http://bitbucket.org/pypy/pypy/changeset/0586c5404983/ Log: merge diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py --- a/pypy/module/micronumpy/interp_dtype.py +++ b/pypy/module/micronumpy/interp_dtype.py @@ -90,11 +90,8 @@ return space.newtuple([]) def eq(self, space, w_other): - if space.is_w(self, w_other): - return True - if space.isinstance_w(w_other, space.w_str): - return self.name == space.str_w(w_other) - return False + w_other = space.call_function(space.gettypefor(W_Dtype), w_other) + return space.is_w(self, w_other) def descr_eq(self, space, w_other): return space.wrap(self.eq(space, w_other)) diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py --- a/pypy/module/micronumpy/test/test_dtypes.py +++ b/pypy/module/micronumpy/test/test_dtypes.py @@ -9,13 +9,18 @@ assert d.num == 0 assert d.kind == 'b' assert dtype('int8').num == 1 - assert dtype('int8') == 'int8' - assert 'int8' == dtype('int8') - assert dtype('int8') != 3 assert dtype(d) is d assert dtype(None) is dtype(float) raises(TypeError, dtype, 1042) + def test_dtype_eq(self): + from _numpypy import dtype + + assert dtype("int8") == "int8" + assert "int8" == dtype("int8") + raises(TypeError, lambda: dtype("int8") == 3) + assert dtype(bool) == bool + def test_dtype_with_types(self): from _numpypy import dtype From noreply at buildbot.pypy.org Thu Jan 26 21:59:54 2012 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 26 Jan 2012 21:59:54 +0100 (CET) Subject: [pypy-commit] pypy numpy-back-to-applevel: a pretty important fix for item and more tests, bad fijal Message-ID: <20120126205954.6E4BF82B1F@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: numpy-back-to-applevel Changeset: r51818:f5834bdc4464 Date: 2012-01-26 22:40 +0200 http://bitbucket.org/pypy/pypy/changeset/f5834bdc4464/ Log: a pretty important fix for item and more tests, bad fijal diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -651,7 +651,10 @@ raise OperationError(space.w_ValueError, space.wrap("index out of bounds")) return self.value.item(space) if space.isinstance_w(w_arg, space.w_int): - i = to_coords(space, self.shape, self.size, self.order, w_arg)[0] + if isinstance(self, Scalar): + raise OperationError(space.w_ValueError, space.wrap("index out of bounds")) + concr = self.get_concrete() + i = to_coords(space, self.shape, concr.size, concr.order, w_arg)[0] # XXX a bit around return self.descr_getitem(space, space.newtuple([space.wrap(x) for x in i])).item(space) @@ -677,7 +680,6 @@ Intermediate class representing a literal. """ size = 1 - order = 'C' _attrs_ = ["dtype", "value", "shape"] def __init__(self, dtype, value): diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -1487,6 +1487,10 @@ assert array([3]).item(0) == 3 assert type(array([3]).item(0)) is int assert array([1, 2, 3]).item(-1) == 3 + a = array([1, 2, 3]) + assert a[::2].item(1) == 3 + assert (a + a).item(1) == 4 + raises(ValueError, "array(5).item(1)") class AppTestSupport(BaseNumpyAppTest): def setup_class(cls): diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py --- a/pypy/module/micronumpy/types.py +++ b/pypy/module/micronumpy/types.py @@ -181,7 +181,7 @@ return v1 >= v2 def bool(self, v): - return bool(self.for_computation(self.unbox(v))) + return self.for_computation(self.unbox(v)) != 0 @simple_binary_op def max(self, v1, v2): From noreply at buildbot.pypy.org Thu Jan 26 21:59:55 2012 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 26 Jan 2012 21:59:55 +0100 (CET) Subject: [pypy-commit] pypy numpy-back-to-applevel: soem rpython fixes Message-ID: <20120126205955.A7DF382B20@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: numpy-back-to-applevel Changeset: r51819:101fcf0a6557 Date: 2012-01-26 22:59 +0200 http://bitbucket.org/pypy/pypy/changeset/101fcf0a6557/ Log: soem rpython fixes diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py --- a/pypy/module/micronumpy/interp_boxes.py +++ b/pypy/module/micronumpy/interp_boxes.py @@ -6,7 +6,6 @@ from pypy.objspace.std.inttype import int_typedef from pypy.rlib.rarithmetic import LONG_BIT from pypy.tool.sourcetools import func_with_new_name -from pypy.rpython.lltypesystem import rffi, lltype MIXIN_64 = (int_typedef,) if LONG_BIT == 64 else () MIXIN_32 = () if LONG_BIT == 64 else (int_typedef,) diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -2,7 +2,8 @@ from pypy.interpreter.error import OperationError, operationerrfmt from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.interpreter.typedef import TypeDef, GetSetProperty -from pypy.module.micronumpy import interp_ufuncs, interp_dtype, signature +from pypy.module.micronumpy import interp_ufuncs, interp_dtype, signature,\ + interp_boxes from pypy.module.micronumpy.strides import calculate_slice_strides,\ shape_agreement, find_shape_and_elems, get_shape_from_iterable,\ calc_new_strides, to_coords @@ -445,7 +446,7 @@ item = concrete._index_of_single_item(space, w_idx) return concrete.getitem(item) chunks = self._prepare_slice_args(space, w_idx) - return space.wrap(self.create_slice(chunks)) + return self.create_slice(chunks) def descr_setitem(self, space, w_idx, w_value): self.invalidated() @@ -656,8 +657,10 @@ concr = self.get_concrete() i = to_coords(space, self.shape, concr.size, concr.order, w_arg)[0] # XXX a bit around - return self.descr_getitem(space, space.newtuple([space.wrap(x) - for x in i])).item(space) + item = self.descr_getitem(space, space.newtuple([space.wrap(x) + for x in i])) + assert isinstance(item, interp_boxes.W_GenericBox) + return item.item(space) raise OperationError(space.w_NotImplementedError, space.wrap( "non-int arg not supported")) @@ -686,6 +689,7 @@ self.shape = [] BaseArray.__init__(self, []) self.dtype = dtype + assert isinstance(value, interp_boxes.W_GenericBox) self.value = value def find_dtype(self): diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py --- a/pypy/module/micronumpy/types.py +++ b/pypy/module/micronumpy/types.py @@ -105,8 +105,9 @@ )) def read_bool(self, storage, width, i, offset): - return bool(libffi.array_getitem(clibffi.cast_type_to_ffitype(self.T), - width, storage, i, offset)) + return self.for_computation( + libffi.array_getitem(clibffi.cast_type_to_ffitype(self.T), + width, storage, i, offset)) != 0 def store(self, storage, width, i, offset, box): value = self.unbox(box) From noreply at buildbot.pypy.org Thu Jan 26 22:29:52 2012 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 26 Jan 2012 22:29:52 +0100 (CET) Subject: [pypy-commit] pypy numpy-back-to-applevel: fix translation? Message-ID: <20120126212952.16F9A82B1F@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: numpy-back-to-applevel Changeset: r51820:ecbc34a30eca Date: 2012-01-26 23:02 +0200 http://bitbucket.org/pypy/pypy/changeset/ecbc34a30eca/ Log: fix translation? diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py --- a/pypy/module/micronumpy/interp_ufuncs.py +++ b/pypy/module/micronumpy/interp_ufuncs.py @@ -272,7 +272,7 @@ else: res_dtype = calc_dtype if isinstance(w_obj, Scalar): - return self.func(calc_dtype, w_obj.value.convert_to(calc_dtype)) + return space.wrap(self.func(calc_dtype, w_obj.value.convert_to(calc_dtype))) w_res = Call1(self.func, self.name, w_obj.shape, calc_dtype, res_dtype, w_obj) From noreply at buildbot.pypy.org Thu Jan 26 22:29:53 2012 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 26 Jan 2012 22:29:53 +0100 (CET) Subject: [pypy-commit] pypy numpy-back-to-applevel: another fix? Message-ID: <20120126212953.4F46982B1F@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: numpy-back-to-applevel Changeset: r51821:77d1d7f10210 Date: 2012-01-26 23:10 +0200 http://bitbucket.org/pypy/pypy/changeset/77d1d7f10210/ Log: another fix? diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py --- a/pypy/module/micronumpy/interp_ufuncs.py +++ b/pypy/module/micronumpy/interp_ufuncs.py @@ -310,10 +310,10 @@ else: res_dtype = calc_dtype if isinstance(w_lhs, Scalar) and isinstance(w_rhs, Scalar): - return self.func(calc_dtype, + return space.wrap(self.func(calc_dtype, w_lhs.value.convert_to(calc_dtype), w_rhs.value.convert_to(calc_dtype) - ) + )) new_shape = shape_agreement(space, w_lhs.shape, w_rhs.shape) w_res = Call2(self.func, self.name, From noreply at buildbot.pypy.org Thu Jan 26 22:29:54 2012 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 26 Jan 2012 22:29:54 +0100 (CET) Subject: [pypy-commit] pypy numpy-back-to-applevel: cheat Message-ID: <20120126212954.80EF082B1F@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: numpy-back-to-applevel Changeset: r51822:aa22b6b02b7e Date: 2012-01-26 23:29 +0200 http://bitbucket.org/pypy/pypy/changeset/aa22b6b02b7e/ Log: cheat diff --git a/pypy/rpython/lltypesystem/rdict.py b/pypy/rpython/lltypesystem/rdict.py --- a/pypy/rpython/lltypesystem/rdict.py +++ b/pypy/rpython/lltypesystem/rdict.py @@ -321,7 +321,8 @@ r_tuple = hop.r_result cTUPLE = hop.inputconst(lltype.Void, r_tuple.lowleveltype) hop.exception_is_here() - return hop.gendirectcall(ll_popitem, cTUPLE, v_dict) + v_res = hop.gendirectcall(ll_popitem, cTUPLE, v_dict) + return self.recast_value(hop.llops, v_res) def rtype_method_pop(self, hop): if hop.nb_args == 2: @@ -331,7 +332,8 @@ v_args = hop.inputargs(self, self.key_repr, self.value_repr) target = ll_pop_default hop.exception_is_here() - return hop.gendirectcall(target, *v_args) + v_res = hop.gendirectcall(target, *v_args) + return self.recast_value(hop.llops, v_res) class __extend__(pairtype(DictRepr, rmodel.Repr)): From noreply at buildbot.pypy.org Thu Jan 26 22:35:49 2012 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Thu, 26 Jan 2012 22:35:49 +0100 (CET) Subject: [pypy-commit] pypy numpy-back-to-applevel: merged upstream Message-ID: <20120126213549.623C982B20@wyvern.cs.uni-duesseldorf.de> Author: Alex Gaynor <alex.gaynor at gmail.com> Branch: numpy-back-to-applevel Changeset: r51824:8fe3c92a0d5e Date: 2012-01-26 16:34 -0500 http://bitbucket.org/pypy/pypy/changeset/8fe3c92a0d5e/ Log: merged upstream diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py --- a/pypy/module/micronumpy/interp_ufuncs.py +++ b/pypy/module/micronumpy/interp_ufuncs.py @@ -272,7 +272,7 @@ else: res_dtype = calc_dtype if isinstance(w_obj, Scalar): - return self.func(calc_dtype, w_obj.value.convert_to(calc_dtype)) + return space.wrap(self.func(calc_dtype, w_obj.value.convert_to(calc_dtype))) w_res = Call1(self.func, self.name, w_obj.shape, calc_dtype, res_dtype, w_obj) @@ -310,10 +310,10 @@ else: res_dtype = calc_dtype if isinstance(w_lhs, Scalar) and isinstance(w_rhs, Scalar): - return self.func(calc_dtype, + return space.wrap(self.func(calc_dtype, w_lhs.value.convert_to(calc_dtype), w_rhs.value.convert_to(calc_dtype) - ) + )) new_shape = shape_agreement(space, w_lhs.shape, w_rhs.shape) w_res = Call2(self.func, self.name, diff --git a/pypy/rpython/lltypesystem/rdict.py b/pypy/rpython/lltypesystem/rdict.py --- a/pypy/rpython/lltypesystem/rdict.py +++ b/pypy/rpython/lltypesystem/rdict.py @@ -321,7 +321,8 @@ r_tuple = hop.r_result cTUPLE = hop.inputconst(lltype.Void, r_tuple.lowleveltype) hop.exception_is_here() - return hop.gendirectcall(ll_popitem, cTUPLE, v_dict) + v_res = hop.gendirectcall(ll_popitem, cTUPLE, v_dict) + return self.recast_value(hop.llops, v_res) def rtype_method_pop(self, hop): if hop.nb_args == 2: From noreply at buildbot.pypy.org Thu Jan 26 22:35:48 2012 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Thu, 26 Jan 2012 22:35:48 +0100 (CET) Subject: [pypy-commit] pypy numpy-back-to-applevel: fix dict.pop() in RPython with instances. Message-ID: <20120126213548.3304E82B1F@wyvern.cs.uni-duesseldorf.de> Author: Alex Gaynor <alex.gaynor at gmail.com> Branch: numpy-back-to-applevel Changeset: r51823:45f2ee87bb74 Date: 2012-01-26 16:34 -0500 http://bitbucket.org/pypy/pypy/changeset/45f2ee87bb74/ Log: fix dict.pop() in RPython with instances. diff --git a/pypy/rpython/lltypesystem/rdict.py b/pypy/rpython/lltypesystem/rdict.py --- a/pypy/rpython/lltypesystem/rdict.py +++ b/pypy/rpython/lltypesystem/rdict.py @@ -331,7 +331,8 @@ v_args = hop.inputargs(self, self.key_repr, self.value_repr) target = ll_pop_default hop.exception_is_here() - return hop.gendirectcall(target, *v_args) + v_res = hop.gendirectcall(target, *v_args) + return self.recast_value(hop.llops, v_res) class __extend__(pairtype(DictRepr, rmodel.Repr)): diff --git a/pypy/rpython/test/test_rdict.py b/pypy/rpython/test/test_rdict.py --- a/pypy/rpython/test/test_rdict.py +++ b/pypy/rpython/test/test_rdict.py @@ -25,37 +25,37 @@ class BaseTestRdict(BaseRtypingTest): def test_dict_creation(self): - def createdict(i): + def createdict(i): d = {'hello' : i} return d['hello'] res = self.interpret(createdict, [42]) assert res == 42 - def test_dict_getitem_setitem(self): - def func(i): + def test_dict_getitem_setitem(self): + def func(i): d = {'hello' : i} d['world'] = i + 1 - return d['hello'] * d['world'] + return d['hello'] * d['world'] res = self.interpret(func, [6]) assert res == 42 - def test_dict_getitem_keyerror(self): - def func(i): + def test_dict_getitem_keyerror(self): + def func(i): d = {'hello' : i} try: return d['world'] except KeyError: - return 0 + return 0 res = self.interpret(func, [6]) assert res == 0 def test_dict_del_simple(self): - def func(i): + def func(i): d = {'hello' : i} d['world'] = i + 1 del d['hello'] - return len(d) + return len(d) res = self.interpret(func, [6]) assert res == 1 @@ -71,7 +71,7 @@ assert res == True def test_empty_strings(self): - def func(i): + def func(i): d = {'' : i} del d[''] try: @@ -83,7 +83,7 @@ res = self.interpret(func, [6]) assert res == 1 - def func(i): + def func(i): d = {'' : i} del d[''] d[''] = i + 1 @@ -146,8 +146,8 @@ d1 = {} d1['hello'] = i + 1 d2 = {} - d2['world'] = d1 - return d2['world']['hello'] + d2['world'] = d1 + return d2['world']['hello'] res = self.interpret(func, [5]) assert res == 6 @@ -297,7 +297,7 @@ a = 0 for k, v in items: b += isinstance(k, B) - a += isinstance(v, A) + a += isinstance(v, A) return 3*b+a res = self.interpret(func, []) assert res == 8 @@ -316,7 +316,7 @@ a = 0 for k, v in dic.iteritems(): b += isinstance(k, B) - a += isinstance(v, A) + a += isinstance(v, A) return 3*b+a res = self.interpret(func, []) assert res == 8 @@ -342,11 +342,11 @@ def test_dict_contains_with_constant_dict(self): dic = {'4':1000, ' 8':200} def func(i): - return chr(i) in dic - res = self.interpret(func, [ord('4')]) + return chr(i) in dic + res = self.interpret(func, [ord('4')]) assert res is True - res = self.interpret(func, [1]) - assert res is False + res = self.interpret(func, [1]) + assert res is False def test_dict_or_none(self): class A: @@ -413,7 +413,7 @@ return g(get) res = self.interpret(f, []) - assert res == 2 + assert res == 2 def test_specific_obscure_bug(self): class A: pass @@ -642,6 +642,22 @@ res = self.interpret(f, [2, 5]) assert res == 31 + def test_dict_pop_instance(self): + class A(object): + pass + def f(n): + d = {} + d[2] = A() + x = d.pop(n, None) + if x is None: + return 12 + else: + return 15 + res = self.interpret(f, [2]) + assert res == 15 + res = self.interpret(f, [700]) + assert res == 12 + class TestLLtype(BaseTestRdict, LLRtypeMixin): def test_dict_but_not_with_char_keys(self): def func(i): @@ -653,19 +669,19 @@ res = self.interpret(func, [6]) assert res == 0 - def test_deleted_entry_reusage_with_colliding_hashes(self): - def lowlevelhash(value): + def test_deleted_entry_reusage_with_colliding_hashes(self): + def lowlevelhash(value): p = rstr.mallocstr(len(value)) for i in range(len(value)): p.chars[i] = value[i] - return rstr.LLHelpers.ll_strhash(p) + return rstr.LLHelpers.ll_strhash(p) - def func(c1, c2): - c1 = chr(c1) - c2 = chr(c2) + def func(c1, c2): + c1 = chr(c1) + c2 = chr(c2) d = {} d[c1] = 1 - d[c2] = 2 + d[c2] = 2 del d[c1] return d[c2] @@ -673,7 +689,7 @@ base = rdict.DICT_INITSIZE for y in range(0, 256): y = chr(y) - y_hash = lowlevelhash(y) % base + y_hash = lowlevelhash(y) % base char_by_hash.setdefault(y_hash, []).append(y) x, y = char_by_hash[0][:2] # find a collision @@ -681,18 +697,18 @@ res = self.interpret(func, [ord(x), ord(y)]) assert res == 2 - def func2(c1, c2): - c1 = chr(c1) - c2 = chr(c2) + def func2(c1, c2): + c1 = chr(c1) + c2 = chr(c2) d = {} d[c1] = 1 - d[c2] = 2 + d[c2] = 2 del d[c1] d[c1] = 3 - return d + return d res = self.interpret(func2, [ord(x), ord(y)]) - for i in range(len(res.entries)): + for i in range(len(res.entries)): assert not (res.entries.everused(i) and not res.entries.valid(i)) def func3(c0, c1, c2, c3, c4, c5, c6, c7): @@ -707,9 +723,9 @@ c7 = chr(c7) ; d[c7] = 1; del d[c7] return d - if rdict.DICT_INITSIZE != 8: + if rdict.DICT_INITSIZE != 8: py.test.skip("make dict tests more indepdent from initsize") - res = self.interpret(func3, [ord(char_by_hash[i][0]) + res = self.interpret(func3, [ord(char_by_hash[i][0]) for i in range(rdict.DICT_INITSIZE)]) count_frees = 0 for i in range(len(res.entries)): @@ -727,9 +743,9 @@ del d[chr(ord('a') + i)] return d res = self.interpret(func, [0]) - assert len(res.entries) > rdict.DICT_INITSIZE + assert len(res.entries) > rdict.DICT_INITSIZE res = self.interpret(func, [1]) - assert len(res.entries) == rdict.DICT_INITSIZE + assert len(res.entries) == rdict.DICT_INITSIZE def test_dict_valid_resize(self): # see if we find our keys after resize @@ -864,7 +880,7 @@ def test_prebuilt_list_of_addresses(self): from pypy.rpython.lltypesystem import llmemory - + TP = lltype.Struct('x', ('y', lltype.Signed)) a = lltype.malloc(TP, flavor='raw', immortal=True) b = lltype.malloc(TP, flavor='raw', immortal=True) @@ -878,7 +894,7 @@ d = {a_a: 3, a_b: 4, a_c: 5} d[a0] = 8 - + def func(i): if i == 0: ptr = a @@ -908,7 +924,7 @@ return a == b def rhash(a): return 3 - + def func(i): d = r_dict(eq, rhash, force_non_null=True) if not i: From noreply at buildbot.pypy.org Thu Jan 26 22:35:50 2012 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Thu, 26 Jan 2012 22:35:50 +0100 (CET) Subject: [pypy-commit] pypy numpy-back-to-applevel: revert this part of the change, ti's wrong Message-ID: <20120126213550.8FA5E82B1F@wyvern.cs.uni-duesseldorf.de> Author: Alex Gaynor <alex.gaynor at gmail.com> Branch: numpy-back-to-applevel Changeset: r51825:ff1328acfae8 Date: 2012-01-26 16:35 -0500 http://bitbucket.org/pypy/pypy/changeset/ff1328acfae8/ Log: revert this part of the change, ti's wrong diff --git a/pypy/rpython/lltypesystem/rdict.py b/pypy/rpython/lltypesystem/rdict.py --- a/pypy/rpython/lltypesystem/rdict.py +++ b/pypy/rpython/lltypesystem/rdict.py @@ -321,8 +321,7 @@ r_tuple = hop.r_result cTUPLE = hop.inputconst(lltype.Void, r_tuple.lowleveltype) hop.exception_is_here() - v_res = hop.gendirectcall(ll_popitem, cTUPLE, v_dict) - return self.recast_value(hop.llops, v_res) + return hop.gendirectcall(ll_popitem, cTUPLE, v_dict) def rtype_method_pop(self, hop): if hop.nb_args == 2: From noreply at buildbot.pypy.org Thu Jan 26 22:39:36 2012 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 26 Jan 2012 22:39:36 +0100 (CET) Subject: [pypy-commit] pypy numpy-back-to-applevel: fix test_base Message-ID: <20120126213936.6313D82B1F@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: numpy-back-to-applevel Changeset: r51826:857a38cda57f Date: 2012-01-26 23:39 +0200 http://bitbucket.org/pypy/pypy/changeset/857a38cda57f/ Log: fix test_base diff --git a/pypy/module/micronumpy/test/test_base.py b/pypy/module/micronumpy/test/test_base.py --- a/pypy/module/micronumpy/test/test_base.py +++ b/pypy/module/micronumpy/test/test_base.py @@ -3,6 +3,7 @@ from pypy.module.micronumpy.interp_numarray import W_NDimArray, Scalar from pypy.module.micronumpy.interp_ufuncs import (find_binop_result_dtype, find_unaryop_result_dtype) +from pypy.module.micronumpy.interp_boxes import W_Float64Box from pypy.conftest import option import sys @@ -23,7 +24,7 @@ ar = W_NDimArray(10, [10], dtype=float64_dtype) ar2 = W_NDimArray(10, [10], dtype=float64_dtype) v1 = ar.descr_add(space, ar) - v2 = ar.descr_add(space, Scalar(float64_dtype, 2.0)) + v2 = ar.descr_add(space, Scalar(float64_dtype, W_Float64Box(2.0))) sig1 = v1.find_sig() sig2 = v2.find_sig() assert v1 is not v2 @@ -33,7 +34,7 @@ sig1b = ar2.descr_add(space, ar).find_sig() assert sig1b.left.array_no != sig1b.right.array_no assert sig1b is not sig1 - v3 = ar.descr_add(space, Scalar(float64_dtype, 1.0)) + v3 = ar.descr_add(space, Scalar(float64_dtype, W_Float64Box(1.0))) sig3 = v3.find_sig() assert sig2 is sig3 v4 = ar.descr_add(space, ar) From noreply at buildbot.pypy.org Thu Jan 26 22:47:13 2012 From: noreply at buildbot.pypy.org (mattip) Date: Thu, 26 Jan 2012 22:47:13 +0100 (CET) Subject: [pypy-commit] pypy numppy-flatitter: merge with default Message-ID: <20120126214713.CD6CA82B1F@wyvern.cs.uni-duesseldorf.de> Author: mattip Branch: numppy-flatitter Changeset: r51827:4bd2c28ea1e4 Date: 2012-01-19 07:50 +0200 http://bitbucket.org/pypy/pypy/changeset/4bd2c28ea1e4/ Log: merge with default diff --git a/pypy/bin/py.py b/pypy/bin/py.py --- a/pypy/bin/py.py +++ b/pypy/bin/py.py @@ -76,6 +76,8 @@ config.objspace.suggest(allworkingmodules=False) if config.objspace.allworkingmodules: pypyoption.enable_allworkingmodules(config) + if config.objspace.usemodules._continuation: + config.translation.continuation = True if config.objspace.usemodules.thread: config.translation.thread = True diff --git a/pypy/doc/getting-started.rst b/pypy/doc/getting-started.rst --- a/pypy/doc/getting-started.rst +++ b/pypy/doc/getting-started.rst @@ -53,11 +53,11 @@ PyPy is ready to be executed as soon as you unpack the tarball or the zip file, with no need to install it in any specific location:: - $ tar xf pypy-1.6-linux.tar.bz2 + $ tar xf pypy-1.7-linux.tar.bz2 - $ ./pypy-1.6/bin/pypy + $ ./pypy-1.7/bin/pypy Python 2.7.1 (?, Apr 27 2011, 12:44:21) - [PyPy 1.6.0 with GCC 4.4.3] on linux2 + [PyPy 1.7.0 with GCC 4.4.3] on linux2 Type "help", "copyright", "credits" or "license" for more information. And now for something completely different: ``implementing LOGO in LOGO: "turtles all the way down"'' @@ -75,14 +75,14 @@ $ curl -O https://raw.github.com/pypa/pip/master/contrib/get-pip.py - $ ./pypy-1.6/bin/pypy distribute_setup.py + $ ./pypy-1.7/bin/pypy distribute_setup.py - $ ./pypy-1.6/bin/pypy get-pip.py + $ ./pypy-1.7/bin/pypy get-pip.py - $ ./pypy-1.6/bin/pip install pygments # for example + $ ./pypy-1.7/bin/pip install pygments # for example -3rd party libraries will be installed in ``pypy-1.6/site-packages``, and -the scripts in ``pypy-1.6/bin``. +3rd party libraries will be installed in ``pypy-1.7/site-packages``, and +the scripts in ``pypy-1.7/bin``. Installing using virtualenv --------------------------- diff --git a/pypy/module/_io/interp_fileio.py b/pypy/module/_io/interp_fileio.py --- a/pypy/module/_io/interp_fileio.py +++ b/pypy/module/_io/interp_fileio.py @@ -349,6 +349,8 @@ try: s = os.read(self.fd, size) except OSError, e: + if e.errno == errno.EAGAIN: + return space.w_None raise wrap_oserror(space, e, exception_name='w_IOError') @@ -362,6 +364,8 @@ try: buf = os.read(self.fd, length) except OSError, e: + if e.errno == errno.EAGAIN: + return space.w_None raise wrap_oserror(space, e, exception_name='w_IOError') rwbuffer.setslice(0, buf) diff --git a/pypy/module/_io/test/test_fileio.py b/pypy/module/_io/test/test_fileio.py --- a/pypy/module/_io/test/test_fileio.py +++ b/pypy/module/_io/test/test_fileio.py @@ -133,6 +133,19 @@ f.close() assert a == 'a\nbxxxxxxx' + def test_nonblocking_read(self): + import os, fcntl + r_fd, w_fd = os.pipe() + # set nonblocking + fcntl.fcntl(r_fd, fcntl.F_SETFL, os.O_NONBLOCK) + import _io + f = _io.FileIO(r_fd, 'r') + # Read from stream sould return None + assert f.read() is None + assert f.read(10) is None + a = bytearray('x' * 10) + assert f.readinto(a) is None + def test_repr(self): import _io f = _io.FileIO(self.tmpfile, 'r') diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py --- a/pypy/module/_socket/interp_socket.py +++ b/pypy/module/_socket/interp_socket.py @@ -67,9 +67,6 @@ self.connect(self.addr_from_object(space, w_addr)) except SocketError, e: raise converted_error(space, e) - except TypeError, e: - raise OperationError(space.w_TypeError, - space.wrap(str(e))) def connect_ex_w(self, space, w_addr): """connect_ex(address) -> errno diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py --- a/pypy/module/_socket/test/test_sock_app.py +++ b/pypy/module/_socket/test/test_sock_app.py @@ -529,26 +529,31 @@ import _socket, os if not hasattr(_socket, 'AF_UNIX'): skip('AF_UNIX not supported.') - sockpath = os.path.join(self.udir, 'app_test_unix_socket_connect') + oldcwd = os.getcwd() + os.chdir(self.udir) + try: + sockpath = 'app_test_unix_socket_connect' - serversock = _socket.socket(_socket.AF_UNIX) - serversock.bind(sockpath) - serversock.listen(1) + serversock = _socket.socket(_socket.AF_UNIX) + serversock.bind(sockpath) + serversock.listen(1) - clientsock = _socket.socket(_socket.AF_UNIX) - clientsock.connect(sockpath) - s, addr = serversock.accept() - assert not addr + clientsock = _socket.socket(_socket.AF_UNIX) + clientsock.connect(sockpath) + s, addr = serversock.accept() + assert not addr - s.send('X') - data = clientsock.recv(100) - assert data == 'X' - clientsock.send('Y') - data = s.recv(100) - assert data == 'Y' + s.send('X') + data = clientsock.recv(100) + assert data == 'X' + clientsock.send('Y') + data = s.recv(100) + assert data == 'Y' - clientsock.close() - s.close() + clientsock.close() + s.close() + finally: + os.chdir(oldcwd) class AppTestSocketTCP: diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py --- a/pypy/module/_ssl/test/test_ssl.py +++ b/pypy/module/_ssl/test/test_ssl.py @@ -64,8 +64,8 @@ def test_sslwrap(self): import _ssl, _socket, sys, gc - if sys.platform == 'darwin': - skip("hangs indefinitely on OSX (also on CPython)") + if sys.platform == 'darwin' or 'freebsd' in sys.platform: + skip("hangs indefinitely on OSX & FreeBSD (also on CPython)") s = _socket.socket() ss = _ssl.sslwrap(s, 0) exc = raises(_socket.error, ss.do_handshake) diff --git a/pypy/module/fcntl/test/test_fcntl.py b/pypy/module/fcntl/test/test_fcntl.py --- a/pypy/module/fcntl/test/test_fcntl.py +++ b/pypy/module/fcntl/test/test_fcntl.py @@ -42,13 +42,9 @@ else: start_len = "qq" - if sys.platform in ('netbsd1', 'netbsd2', 'netbsd3', - 'Darwin1.2', 'darwin', - 'freebsd2', 'freebsd3', 'freebsd4', 'freebsd5', - 'freebsd6', 'freebsd7', 'freebsd8', 'freebsd9', - 'bsdos2', 'bsdos3', 'bsdos4', - 'openbsd', 'openbsd2', 'openbsd3', 'openbsd4', - 'openbsd5'): + if any(substring in sys.platform.lower() + for substring in ('netbsd', 'darwin', 'freebsd', 'bsdos', + 'openbsd')): if struct.calcsize('l') == 8: off_t = 'l' pid_t = 'i' @@ -182,7 +178,8 @@ def test_large_flag(self): import sys - if sys.platform == "darwin" or sys.platform.startswith("openbsd"): + if any(plat in sys.platform + for plat in ('darwin', 'openbsd', 'freebsd')): skip("Mac OS doesn't have any large flag in fcntl.h") import fcntl, sys if sys.maxint == 2147483647: diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py --- a/pypy/module/micronumpy/__init__.py +++ b/pypy/module/micronumpy/__init__.py @@ -27,6 +27,8 @@ 'dot': 'interp_numarray.dot', 'fromstring': 'interp_support.fromstring', 'flatiter': 'interp_numarray.W_FlatIterator', + 'isna': 'interp_numarray.isna', + 'concatenate': 'interp_numarray.concatenate', 'True_': 'types.Bool.True', 'False_': 'types.Bool.False', diff --git a/pypy/module/micronumpy/app_numpy.py b/pypy/module/micronumpy/app_numpy.py --- a/pypy/module/micronumpy/app_numpy.py +++ b/pypy/module/micronumpy/app_numpy.py @@ -59,7 +59,7 @@ if not hasattr(a, "max"): a = _numpypy.array(a) return a.max(axis) - + def arange(start, stop=None, step=1, dtype=None): '''arange([start], stop[, step], dtype=None) Generate values in the half-interval [start, stop). diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py --- a/pypy/module/micronumpy/interp_boxes.py +++ b/pypy/module/micronumpy/interp_boxes.py @@ -9,6 +9,7 @@ MIXIN_64 = (int_typedef,) if LONG_BIT == 64 else () +MIXIN_32 = () if LONG_BIT == 64 else (int_typedef,) def new_dtype_getter(name): def get_dtype(space): @@ -231,7 +232,7 @@ __new__ = interp2app(W_UInt16Box.descr__new__.im_func), ) -W_Int32Box.typedef = TypeDef("int32", W_SignedIntegerBox.typedef, +W_Int32Box.typedef = TypeDef("int32", (W_SignedIntegerBox.typedef,) + MIXIN_32, __module__ = "numpypy", __new__ = interp2app(W_Int32Box.descr__new__.im_func), ) @@ -241,24 +242,18 @@ __new__ = interp2app(W_UInt32Box.descr__new__.im_func), ) -if LONG_BIT == 32: - long_name = "int32" -elif LONG_BIT == 64: - long_name = "int64" -W_LongBox.typedef = TypeDef(long_name, (W_SignedIntegerBox.typedef, int_typedef,), - __module__ = "numpypy", - __new__ = interp2app(W_LongBox.descr__new__.im_func), -) - -W_ULongBox.typedef = TypeDef("u" + long_name, W_UnsignedIntegerBox.typedef, - __module__ = "numpypy", -) - W_Int64Box.typedef = TypeDef("int64", (W_SignedIntegerBox.typedef,) + MIXIN_64, __module__ = "numpypy", __new__ = interp2app(W_Int64Box.descr__new__.im_func), ) +if LONG_BIT == 32: + W_LongBox = W_Int32Box + W_ULongBox = W_UInt32Box +elif LONG_BIT == 64: + W_LongBox = W_Int64Box + W_ULongBox = W_UInt64Box + W_UInt64Box.typedef = TypeDef("uint64", W_UnsignedIntegerBox.typedef, __module__ = "numpypy", __new__ = interp2app(W_UInt64Box.descr__new__.im_func), diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py --- a/pypy/module/micronumpy/interp_dtype.py +++ b/pypy/module/micronumpy/interp_dtype.py @@ -89,6 +89,19 @@ def descr_get_shape(self, space): return space.newtuple([]) + def eq(self, space, w_other): + if space.is_w(self, w_other): + return True + if space.isinstance_w(w_other, space.w_str): + return self.name == space.str_w(w_other) + return False + + def descr_eq(self, space, w_other): + return space.wrap(self.eq(space, w_other)) + + def descr_ne(self, space, w_other): + return space.wrap(not self.eq(space, w_other)) + def is_int_type(self): return self.kind == SIGNEDLTR or self.kind == UNSIGNEDLTR @@ -101,6 +114,8 @@ __str__= interp2app(W_Dtype.descr_str), __repr__ = interp2app(W_Dtype.descr_repr), + __eq__ = interp2app(W_Dtype.descr_eq), + __ne__ = interp2app(W_Dtype.descr_ne), num = interp_attrproperty("num", cls=W_Dtype), kind = interp_attrproperty("kind", cls=W_Dtype), diff --git a/pypy/module/micronumpy/interp_iter.py b/pypy/module/micronumpy/interp_iter.py --- a/pypy/module/micronumpy/interp_iter.py +++ b/pypy/module/micronumpy/interp_iter.py @@ -17,6 +17,10 @@ if self.step != 0: shape.append(self.lgt) + def __repr__(self): + return 'Chunk(%d, %d, %d, %d)' % (self.start, self.stop, self.step, + self.lgt) + class BaseTransform(object): pass diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -1,6 +1,6 @@ from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.error import OperationError, operationerrfmt -from pypy.interpreter.gateway import interp2app, NoneNotWrapped +from pypy.interpreter.gateway import interp2app, NoneNotWrapped, unwrap_spec from pypy.interpreter.typedef import TypeDef, GetSetProperty from pypy.module.micronumpy import interp_ufuncs, interp_dtype, signature,\ interp_boxes @@ -436,6 +436,13 @@ def copy(self, space): return self.get_concrete().copy(space) + def empty_copy(self, space, dtype): + shape = self.shape + size = 1 + for elem in shape: + size *= elem + return W_NDimArray(size, shape[:], dtype, 'C') + def flatten(self, space): return self.get_concrete().flatten(space) @@ -1334,6 +1341,42 @@ return convert_to_array(space, w_obj2).descr_dot(space, w_arr) return w_arr.descr_dot(space, w_obj2) + at unwrap_spec(axis=int) +def concatenate(space, w_args, axis=0): + args_w = space.listview(w_args) + if len(args_w) == 0: + raise OperationError(space.w_ValueError, space.wrap("concatenation of zero-length sequences is impossible")) + args_w = [convert_to_array(space, w_arg) for w_arg in args_w] + dtype = args_w[0].find_dtype() + shape = args_w[0].shape[:] + if len(shape) <= axis: + raise OperationError(space.w_ValueError, + space.wrap("bad axis argument")) + for arr in args_w[1:]: + dtype = interp_ufuncs.find_binop_result_dtype(space, dtype, + arr.find_dtype()) + if len(arr.shape) <= axis: + raise OperationError(space.w_ValueError, + space.wrap("bad axis argument")) + for i, axis_size in enumerate(arr.shape): + if len(arr.shape) != len(shape) or (i != axis and axis_size != shape[i]): + raise OperationError(space.w_ValueError, space.wrap( + "array dimensions must agree except for axis being concatenated")) + elif i == axis: + shape[i] += axis_size + size = 1 + for elem in shape: + size *= elem + res = W_NDimArray(size, shape, dtype, 'C') + chunks = [Chunk(0, i, 1, i) for i in shape] + axis_start = 0 + for arr in args_w: + chunks[axis] = Chunk(axis_start, axis_start + arr.shape[axis], 1, + arr.shape[axis]) + res.create_slice(chunks).setslice(space, arr) + axis_start += arr.shape[axis] + return res + BaseArray.typedef = TypeDef( 'ndarray', __module__ = "numpypy", @@ -1536,3 +1579,11 @@ ) W_FlatIterator.acceptable_as_base_class = False + +def isna(space, w_obj): + if isinstance(w_obj, BaseArray): + arr = w_obj.empty_copy(space, + interp_dtype.get_dtype_cache(space).w_booldtype) + arr.fill(space, space.wrap(False)) + return arr + return space.wrap(False) diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py --- a/pypy/module/micronumpy/strides.py +++ b/pypy/module/micronumpy/strides.py @@ -1,4 +1,5 @@ from pypy.rlib import jit +from pypy.interpreter.error import OperationError @jit.look_inside_iff(lambda shape, start, strides, backstrides, chunks: @@ -37,3 +38,27 @@ rstrides = [0] * (len(res_shape) - len(orig_shape)) + rstrides rbackstrides = [0] * (len(res_shape) - len(orig_shape)) + rbackstrides return rstrides, rbackstrides + +def to_coords(space, shape, size, order, w_item_or_slice): + '''Returns a start coord, step, and length. + ''' + start = lngth = step = 0 + if not (space.isinstance_w(w_item_or_slice, space.w_int) or + space.isinstance_w(w_item_or_slice, space.w_slice)): + raise OperationError(space.w_IndexError, + space.wrap('unsupported iterator index')) + + start, stop, step, lngth = space.decode_index4(w_item_or_slice, size) + + coords = [0] * len(shape) + i = start + if order == 'C': + for s in range(len(shape) -1, -1, -1): + coords[s] = i % shape[s] + i //= shape[s] + else: + for s in range(len(shape)): + coords[s] = i % shape[s] + i //= shape[s] + + return coords, step, lngth diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py --- a/pypy/module/micronumpy/test/test_dtypes.py +++ b/pypy/module/micronumpy/test/test_dtypes.py @@ -9,6 +9,9 @@ assert d.num == 0 assert d.kind == 'b' assert dtype('int8').num == 1 + assert dtype('int8') == 'int8' + assert 'int8' == dtype('int8') + assert dtype('int8') != 3 assert dtype(d) is d assert dtype(None) is dtype(float) raises(TypeError, dtype, 1042) @@ -376,3 +379,14 @@ b = X(10) assert type(b) is X assert b.m() == 12 + + def test_int(self): + import sys + from _numpypy import int32, int64, int_ + assert issubclass(int_, int) + if sys.maxint == (1<<31) - 1: + assert issubclass(int32, int) + assert int_ is int32 + else: + assert issubclass(int64, int) + assert int_ is int64 diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -167,6 +167,24 @@ assert calc_new_strides([1, 1, 105, 1, 1], [7, 15], [1, 7],'F') == \ [1, 1, 1, 105, 105] + def test_to_coords(self): + from pypy.module.micronumpy.strides import to_coords + + def _to_coords(index, order): + return to_coords(self.space, [2, 3, 4], 24, order, + self.space.wrap(index))[0] + + assert _to_coords(0, 'C') == [0, 0, 0] + assert _to_coords(1, 'C') == [0, 0, 1] + assert _to_coords(-1, 'C') == [1, 2, 3] + assert _to_coords(5, 'C') == [0, 1, 1] + assert _to_coords(13, 'C') == [1, 0, 1] + assert _to_coords(0, 'F') == [0, 0, 0] + assert _to_coords(1, 'F') == [1, 0, 0] + assert _to_coords(-1, 'F') == [1, 2, 3] + assert _to_coords(5, 'F') == [1, 2, 0] + assert _to_coords(13, 'F') == [1, 0, 2] + class AppTestNumArray(BaseNumpyAppTest): def test_ndarray(self): @@ -1045,6 +1063,40 @@ #assert (a.var(0) == [8, 8]).all() #assert (a.var(1) == [.25] * 5).all() + def test_concatenate(self): + from numpypy import array, concatenate, dtype + a1 = array([0,1,2]) + a2 = array([3,4,5]) + a = concatenate((a1, a2)) + assert len(a) == 6 + assert (a == [0,1,2,3,4,5]).all() + assert a.dtype is dtype(int) + b1 = array([[1, 2], [3, 4]]) + b2 = array([[5, 6]]) + b = concatenate((b1, b2), axis=0) + assert (b == [[1, 2],[3, 4],[5, 6]]).all() + c = concatenate((b1, b2.T), axis=1) + assert (c == [[1, 2, 5],[3, 4, 6]]).all() + d = concatenate(([0],[1])) + assert (d == [0,1]).all() + e1 = array([[0,1],[2,3]]) + e = concatenate(e1) + assert (e == [0,1,2,3]).all() + f1 = array([0,1]) + f = concatenate((f1, [2], f1, [7])) + assert (f == [0,1,2,0,1,7]).all() + + bad_axis = raises(ValueError, concatenate, (a1,a2), axis=1) + assert str(bad_axis.value) == "bad axis argument" + + concat_zero = raises(ValueError, concatenate, ()) + assert str(concat_zero.value) == \ + "concatenation of zero-length sequences is impossible" + + dims_disagree = raises(ValueError, concatenate, (a1, b1), axis=0) + assert str(dims_disagree.value) == \ + "array dimensions must agree except for axis being concatenated" + def test_std(self): from _numpypy import array a = array(range(10)) @@ -1068,7 +1120,6 @@ a = array([[1, 2], [3, 4]]) assert (a.T.flatten() == [1, 3, 2, 4]).all() - class AppTestMultiDim(BaseNumpyAppTest): def test_init(self): import _numpypy @@ -1455,6 +1506,11 @@ a[a & 1 == 1] = array([8, 9, 10]) assert (a == [[0, 8], [2, 9], [4, 10]]).all() + def test_isna(self): + from _numpypy import isna, array + # XXX for now + assert not isna(3) + assert (isna(array([1, 2, 3, 4])) == [False, False, False, False]).all() class AppTestSupport(BaseNumpyAppTest): diff --git a/pypy/module/micronumpy/tool/numready/__init__.py b/pypy/module/micronumpy/tool/numready/__init__.py new file mode 100644 diff --git a/pypy/module/micronumpy/tool/numready/__main__.py b/pypy/module/micronumpy/tool/numready/__main__.py new file mode 100644 --- /dev/null +++ b/pypy/module/micronumpy/tool/numready/__main__.py @@ -0,0 +1,6 @@ +import sys + +from .main import main + + +main(sys.argv) diff --git a/pypy/module/micronumpy/tool/numready/kinds.py b/pypy/module/micronumpy/tool/numready/kinds.py new file mode 100644 --- /dev/null +++ b/pypy/module/micronumpy/tool/numready/kinds.py @@ -0,0 +1,4 @@ +KINDS = { + "UNKNOWN": "U", + "TYPE": "T", +} diff --git a/pypy/module/micronumpy/tool/numready/main.py b/pypy/module/micronumpy/tool/numready/main.py new file mode 100644 --- /dev/null +++ b/pypy/module/micronumpy/tool/numready/main.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" +This should be run under PyPy. +""" + +import os +import platform +import subprocess +import tempfile +import webbrowser +from collections import OrderedDict + +import jinja2 + +from .kinds import KINDS + + +class SearchableSet(object): + def __init__(self, items=()): + self._items = {} + for item in items: + self.add(item) + + def __iter__(self): + return iter(self._items) + + def __contains__(self, other): + return other in self._items + + def __getitem__(self, idx): + return self._items[idx] + + def add(self, item): + self._items[item] = item + + def __len__(self): + return len(self._items) + +class Item(object): + def __init__(self, name, kind, subitems=None): + self.name = name + self.kind = kind + self.subitems = subitems + + def __hash__(self): + return hash(self.name) + + def __eq__(self, other): + if isinstance(other, str): + return self.name == other + return self.name == other.name + + +class ItemStatus(object): + def __init__(self, name, pypy_exists): + self.name = name + self.cls = 'exists' if pypy_exists else '' + self.symbol = u"✔" if pypy_exists else u'✖' + + def __lt__(self, other): + return self.name < other.name + +def find_numpy_items(python, modname="numpy", attr=None): + args = [ + python, os.path.join(os.path.dirname(__file__), "search.py"), modname + ] + if attr is not None: + args.append(attr) + lines = subprocess.check_output(args).splitlines() + items = SearchableSet() + for line in lines: + kind, name = line.split(" : ", 1) + subitems = None + if kind == KINDS["TYPE"]: + if name in ['ndarray', 'dtype']: + subitems = find_numpy_items(python, modname, name) + items.add(Item(name, kind, subitems)) + return items + +def split(lst): + SPLIT = 5 + lgt = len(lst) // SPLIT + 1 + l = [[] for i in range(lgt)] + for i in range(lgt): + for k in range(SPLIT): + if k * lgt + i < len(lst): + l[i].append(lst[k * lgt + i]) + return l + +def main(argv): + cpy_items = find_numpy_items("/usr/bin/python") + pypy_items = find_numpy_items(argv[1], "numpypy") + all_items = [] + + msg = '%d/%d names, %d/%d ndarray attributes, %d/%d dtype attributes' % ( + len(pypy_items), len(cpy_items), len(pypy_items['ndarray'].subitems), + len(cpy_items['ndarray'].subitems), len(pypy_items['dtype'].subitems), + len(cpy_items['dtype'].subitems)) + for item in cpy_items: + pypy_exists = item in pypy_items + if item.subitems: + for sub in item.subitems: + all_items.append( + ItemStatus(item.name + "." + sub.name, pypy_exists=pypy_exists and pypy_items[item].subitems and sub in pypy_items[item].subitems) + ) + all_items.append(ItemStatus(item.name, pypy_exists=item in pypy_items)) + env = jinja2.Environment( + loader=jinja2.FileSystemLoader(os.path.dirname(__file__)) + ) + html = env.get_template("page.html").render(all_items=split(sorted(all_items)), msg=msg) + if len(argv) > 2: + with open(argv[2], 'w') as f: + f.write(html.encode("utf-8")) + else: + with tempfile.NamedTemporaryFile(delete=False) as f: + f.write(html.encode("utf-8")) + print "Saved in: %s" % f.name diff --git a/pypy/module/micronumpy/tool/numready/page.html b/pypy/module/micronumpy/tool/numready/page.html new file mode 100644 --- /dev/null +++ b/pypy/module/micronumpy/tool/numready/page.html @@ -0,0 +1,65 @@ +<!DOCTYPE html> +<html lang="en"> + <head> + <title>NumPyPy Status</title> + <meta http-equiv="content-type" content="text/html; charset=utf-8"> + <style type="text/css"> + body { + font-family: 'Consolas', 'Bitstream Vera Sans Mono', monospace; + } + h1 { + text-align: center; + } + h3 { + text-align: center; + } + table { + border: 8px solid #DFDECB; + margin: 30px auto; + font-size: 12px; + } + table th { + text-align: left; + } + table td { + padding: 4px 10px; + text-align: center; + } + .exists { + background-color: #337792; + color: white; + border: 1px solid #234F61; + } + </style> + </head> + <body> + <h1>NumPyPy Status</h1> + <h3>Overall: {{ msg }}</h3> + <table> + <thead> + <tr> + <th></th> + <th>PyPy</th> + <th></th> + <th>PyPy</th> + <th></th> + <th>PyPy</th> + <th></th> + <th>PyPy</th> + <th></th> + <th>PyPy</th> + </tr> + </thead> + <tbody> + {% for chunk in all_items %} + <tr> + {% for item in chunk %} + <th class='{{ item.cls }}'>{{ item.name }}</th> + <td class='{{ item.cls }}'>{{ item.symbol }}</td> + {% endfor %} + </tr> + {% endfor %} + </tbody> + </table> + </body> +</html> diff --git a/pypy/module/micronumpy/tool/numready/search.py b/pypy/module/micronumpy/tool/numready/search.py new file mode 100644 --- /dev/null +++ b/pypy/module/micronumpy/tool/numready/search.py @@ -0,0 +1,33 @@ +import sys +import types + +# Evil implicit relative import. +from kinds import KINDS + + +def main(argv): + if len(argv) == 2: + [_, modname] = argv + attr = None + elif len(argv) == 3: + [_, modname, attr] = argv + else: + sys.exit("Wrong number of args") + __import__(modname) + obj = sys.modules[modname] + + if attr is not None: + obj = getattr(obj, attr) + + for name in dir(obj): + if attr is None and name.startswith("_"): + continue + subobj = getattr(obj, name) + if isinstance(subobj, types.TypeType): + kind = KINDS["TYPE"] + else: + kind = KINDS["UNKNOWN"] + print kind, ":", name + +if __name__ == "__main__": + main(sys.argv) \ No newline at end of file diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -371,6 +371,8 @@ if hasattr(__import__(os.name), "forkpty"): def test_forkpty(self): import sys + if 'freebsd' in sys.platform: + skip("hangs indifinitly on FreeBSD (also on CPython).") os = self.posix childpid, master_fd = os.forkpty() assert isinstance(childpid, int) diff --git a/pypy/module/thread/ll_thread.py b/pypy/module/thread/ll_thread.py --- a/pypy/module/thread/ll_thread.py +++ b/pypy/module/thread/ll_thread.py @@ -4,7 +4,7 @@ import py from pypy.rlib import jit, rgc from pypy.rlib.debug import ll_assert -from pypy.rlib.objectmodel import we_are_translated +from pypy.rlib.objectmodel import we_are_translated, specialize from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.tool import rffi_platform from pypy.tool import autopath @@ -85,6 +85,7 @@ def allocate_lock(): return Lock(allocate_ll_lock()) + at specialize.arg(0) def ll_start_new_thread(func): ident = c_thread_start(func) if ident == -1: @@ -97,6 +98,7 @@ def get_ident(): return rffi.cast(lltype.Signed, c_thread_get_ident()) + at specialize.arg(0) def start_new_thread(x, y): """In RPython, no argument can be passed. You have to use global variables to pass information to the new thread. That's not very diff --git a/pypy/module/thread/test/test_ll_thread.py b/pypy/module/thread/test/test_ll_thread.py --- a/pypy/module/thread/test/test_ll_thread.py +++ b/pypy/module/thread/test/test_ll_thread.py @@ -34,72 +34,34 @@ use_threads = True def test_start_new_thread(self): - py.test.skip("xxx ideally, investigate why it fails randomly") - # xxx but in practice start_new_thread() is also tested by the - # next test, and it's a mess to test start_new_thread() without - # the proper GIL to protect the GC import time class State: pass state = State() - class Z: - def __init__(self, value): - self.value = value - def __del__(self): - state.freed_counter += 1 - - def bootstrap(): - state.my_thread_ident = state.z.ident = get_ident() - assert state.my_thread_ident == get_ident() - assert get_ident() == state.z.ident - state.seen_value = state.z.value - state.z = None - # I think that we would need here a memory barrier in order - # to make the test pass reliably. The issue is that the - # main thread may see 'state.done = 1' before seeing the - # effect of the other assignments done above. For now let's - # emulate the write barrier by doing a system call and - # waiting a bit... - time.sleep(0.012) - state.done = 1 - - def g(i): - state.z = Z(i) - return start_new_thread(bootstrap, ()) - g._dont_inline_ = True + def bootstrap1(): + state.my_thread_ident1 = get_ident() + def bootstrap2(): + state.my_thread_ident2 = get_ident() def f(): - main_ident = get_ident() - assert main_ident == get_ident() - state.freed_counter = 0 - for i in range(50): - state.done = 0 - state.seen_value = 0 - ident = g(i) - gc.collect() - willing_to_wait_more = 1000 - while not state.done: - willing_to_wait_more -= 1 - if not willing_to_wait_more: - raise Exception("thread didn't start?") - time.sleep(0.01) - assert state.my_thread_ident != main_ident - assert state.my_thread_ident == ident - assert state.seen_value == i - # try to force Boehm to do some freeing - for i in range(3): - gc.collect() - return state.freed_counter + state.my_thread_ident1 = get_ident() + state.my_thread_ident2 = get_ident() + start_new_thread(bootstrap1, ()) + start_new_thread(bootstrap2, ()) + willing_to_wait_more = 1000 + while (state.my_thread_ident1 == get_ident() or + state.my_thread_ident2 == get_ident()): + willing_to_wait_more -= 1 + if not willing_to_wait_more: + raise Exception("thread didn't start?") + time.sleep(0.01) + return 42 fn = self.getcompiled(f, []) - freed_counter = fn() - print freed_counter - if self.gcpolicy == 'boehm': - assert freed_counter > 0 - else: - assert freed_counter == 50 + res = fn() + assert res == 42 def test_gc_locking(self): import time @@ -202,6 +164,9 @@ def getcompiled(self, f, argtypes): return f + def test_start_new_thread(self): + py.test.skip("deadlocks occasionally -- why???") + class TestUsingBoehm(AbstractThreadTests): gcpolicy = 'boehm' diff --git a/pypy/rlib/objectmodel.py b/pypy/rlib/objectmodel.py --- a/pypy/rlib/objectmodel.py +++ b/pypy/rlib/objectmodel.py @@ -130,19 +130,29 @@ if self is other: return 0 else: - raise TypeError("Symbolics can not be compared!") + raise TypeError("Symbolics can not be compared! (%r, %r)" + % (self, other)) def __hash__(self): - raise TypeError("Symbolics are not hashable!") + raise TypeError("Symbolics are not hashable! %r" % (self,)) def __nonzero__(self): - raise TypeError("Symbolics are not comparable") + raise TypeError("Symbolics are not comparable! %r" % (self,)) class ComputedIntSymbolic(Symbolic): def __init__(self, compute_fn): self.compute_fn = compute_fn + def __repr__(self): + # repr(self.compute_fn) can arrive back here in an + # infinite recursion + try: + name = self.compute_fn.__name__ + except (AttributeError, TypeError): + name = hex(id(self.compute_fn)) + return '%s(%r)' % (self.__class__.__name__, name) + def annotation(self): from pypy.annotation import model return model.SomeInteger() @@ -157,6 +167,9 @@ self.expr = expr self.default = default + def __repr__(self): + return '%s(%r)' % (self.__class__.__name__, self.expr) + def annotation(self): from pypy.annotation import model return model.SomeInteger() diff --git a/pypy/rlib/rdynload.py b/pypy/rlib/rdynload.py --- a/pypy/rlib/rdynload.py +++ b/pypy/rlib/rdynload.py @@ -14,7 +14,7 @@ _MINGW = platform.name == "mingw32" _WIN32 = _MSVC or _MINGW _MAC_OS = platform.name == "darwin" -_FREEBSD = platform.name == "freebsd" +_FREEBSD = sys.platform.startswith("freebsd") if _WIN32: from pypy.rlib import rwin32 diff --git a/pypy/rlib/rsocket.py b/pypy/rlib/rsocket.py --- a/pypy/rlib/rsocket.py +++ b/pypy/rlib/rsocket.py @@ -310,10 +310,7 @@ def from_object(space, w_address): # Parse an app-level object representing an AF_INET address - try: - w_host, w_port = space.unpackiterable(w_address, 2) - except ValueError: - raise TypeError("AF_INET address must be a tuple of length 2") + w_host, w_port = space.unpackiterable(w_address, 2) host = space.str_w(w_host) port = space.int_w(w_port) port = Address.make_ushort_port(space, port) @@ -544,10 +541,7 @@ space.wrap(self.get_groups())]) def from_object(space, w_address): - try: - w_pid, w_groups = space.unpackiterable(w_address, 2) - except ValueError: - raise TypeError("AF_NETLINK address must be a tuple of length 2") + w_pid, w_groups = space.unpackiterable(w_address, 2) return NETLINKAddress(space.uint_w(w_pid), space.uint_w(w_groups)) from_object = staticmethod(from_object) diff --git a/pypy/rpython/lltypesystem/ll2ctypes.py b/pypy/rpython/lltypesystem/ll2ctypes.py --- a/pypy/rpython/lltypesystem/ll2ctypes.py +++ b/pypy/rpython/lltypesystem/ll2ctypes.py @@ -1390,7 +1390,8 @@ def _where_is_errno(): return standard_c_lib.__errno_location() - elif sys.platform in ('darwin', 'freebsd7', 'freebsd8', 'freebsd9'): + elif any(plat in sys.platform + for plat in ('darwin', 'freebsd7', 'freebsd8', 'freebsd9')): standard_c_lib.__error.restype = ctypes.POINTER(ctypes.c_int) def _where_is_errno(): return standard_c_lib.__error() diff --git a/pypy/rpython/lltypesystem/llgroup.py b/pypy/rpython/lltypesystem/llgroup.py --- a/pypy/rpython/lltypesystem/llgroup.py +++ b/pypy/rpython/lltypesystem/llgroup.py @@ -76,6 +76,10 @@ self.index = memberindex self.member = grp.members[memberindex]._as_ptr() + def __repr__(self): + return '%s(%s, %s)' % (self.__class__.__name__, + self.grpptr, self.index) + def __nonzero__(self): return True diff --git a/pypy/rpython/lltypesystem/rffi.py b/pypy/rpython/lltypesystem/rffi.py --- a/pypy/rpython/lltypesystem/rffi.py +++ b/pypy/rpython/lltypesystem/rffi.py @@ -27,6 +27,10 @@ self.c_name = c_name self.TP = TP + def __repr__(self): + return '%s(%r, %s)' % (self.__class__.__name__, + self.c_name, self.TP) + def annotation(self): return lltype_to_annotation(self.TP) diff --git a/pypy/rpython/memory/gc/env.py b/pypy/rpython/memory/gc/env.py --- a/pypy/rpython/memory/gc/env.py +++ b/pypy/rpython/memory/gc/env.py @@ -116,7 +116,7 @@ def get_total_memory(): return get_total_memory_darwin(get_darwin_sysctl_signed('hw.memsize')) -elif sys.platform.startswith('freebsd'): +elif 'freebsd' in sys.platform: def get_total_memory(): return get_total_memory_darwin(get_darwin_sysctl_signed('hw.usermem')) diff --git a/pypy/rpython/module/ll_time.py b/pypy/rpython/module/ll_time.py --- a/pypy/rpython/module/ll_time.py +++ b/pypy/rpython/module/ll_time.py @@ -41,7 +41,7 @@ RUSAGE = platform.Struct('struct rusage', [('ru_utime', TIMEVAL), ('ru_stime', TIMEVAL)]) -if "freebsd" in sys.platform: +if sys.platform.startswith('freebsd'): libraries = ['compat'] else: libraries = [] diff --git a/pypy/translator/platform/freebsd.py b/pypy/translator/platform/freebsd.py --- a/pypy/translator/platform/freebsd.py +++ b/pypy/translator/platform/freebsd.py @@ -18,8 +18,9 @@ class Freebsd(posix.BasePosix): name = "freebsd" - link_flags = get_env_vector("LDFLAGS", '-pthread') - cflags = get_env_vector("CFLAGS", "-O3 -pthread -fomit-frame-pointer") + link_flags = ['-pthread'] + get_env_vector('LDFLAGS', '') + cflags = ['-O3', '-pthread', '-fomit-frame-pointer' + ] + get_env_vector('CFLAGS', '') standalone_only = [] shared_only = [] so_ext = 'so' diff --git a/pypy/translator/platform/linux.py b/pypy/translator/platform/linux.py --- a/pypy/translator/platform/linux.py +++ b/pypy/translator/platform/linux.py @@ -1,15 +1,20 @@ """Support for Linux.""" +import os import sys from pypy.translator.platform.posix import BasePosix class BaseLinux(BasePosix): name = "linux" - link_flags = ('-pthread',) + link_flags = tuple( + ['-pthread',] + + os.environ.get('LDFLAGS', '').split()) extra_libs = ('-lrt',) - cflags = ('-O3', '-pthread', '-fomit-frame-pointer', - '-Wall', '-Wno-unused') + cflags = tuple( + ['-O3', '-pthread', '-fomit-frame-pointer', + '-Wall', '-Wno-unused'] + + os.environ.get('CFLAGS', '').split()) standalone_only = () shared_only = ('-fPIC',) so_ext = 'so' From noreply at buildbot.pypy.org Thu Jan 26 22:47:15 2012 From: noreply at buildbot.pypy.org (mattip) Date: Thu, 26 Jan 2012 22:47:15 +0100 (CET) Subject: [pypy-commit] pypy numppy-flatitter: cleanup merge, add failing test Message-ID: <20120126214715.0F52182B1F@wyvern.cs.uni-duesseldorf.de> Author: mattip Branch: numppy-flatitter Changeset: r51828:88475cf6b4ab Date: 2012-01-19 09:51 +0200 http://bitbucket.org/pypy/pypy/changeset/88475cf6b4ab/ Log: cleanup merge, add failing test diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -4,7 +4,7 @@ from pypy.interpreter.typedef import TypeDef, GetSetProperty from pypy.module.micronumpy import interp_ufuncs, interp_dtype, signature,\ interp_boxes -from pypy.module.micronumpy.strides import calculate_slice_strides +from pypy.module.micronumpy.strides import calculate_slice_strides, to_coords from pypy.rlib import jit from pypy.rpython.lltypesystem import lltype, rffi from pypy.tool.sourcetools import func_with_new_name @@ -1449,40 +1449,6 @@ tolist = interp2app(BaseArray.descr_tolist), ) -#TODO:Move all this to another file after fijal finishes reorganization - at jit.unroll_safe -def _to_coords(space, arr, w_item_or_slice): - '''Returns a start coord, step, and length. - ''' - start = lngth = step = 0 - if space.isinstance_w(w_item_or_slice, space.w_int): - start = space.int_w(w_item_or_slice) - if start < 0: - start += arr.size - lngth = 1 - step = 1 - elif space.isinstance_w(w_item_or_slice, space.w_slice): - start, stop, step, lngth = space.decode_index4(w_item_or_slice,arr.size) - else: - raise OperationError(space.w_IndexError, - space.wrap('unsupported iterator index')) - coords = [] - i = start - if arr.order =='C': - for s in range(len(arr.shape) -1, -1, -1): - coords.insert(0, i % arr.shape[s]) - i /= arr.shape[s] - else: - raise NotImplementedError - #untested code. Erase? - for s in range(len(arr.shape)): - coords.append(i % arr.shape[s]) - i /= arr.shape[s] - if i != 0: - raise OperationError(space.w_IndexError, space.wrap("invalid index")) - - return coords, start, step, lngth - class W_FlatIterator(ViewArray): @@ -1511,26 +1477,30 @@ return space.wrap(self.index) def descr_coords(self, space): - coords, start, step, lngth = _to_coords(space, self.base, space.wrap(self.index)) + coords, step, lngth = to_coords(space, self.base.shape, + self.base.size, self.base.order, + space.wrap(self.index)) return space.newtuple([space.wrap(c) for c in coords]) def descr_getitem(self, space, w_idx): - coords, start, step, lngth = _to_coords(space, self.base, w_idx) - if lngth <2: - w_coords = space.newtuple([space.wrap(c) for c in coords]) - return self.base.descr_getitem(space, w_coords) - - res = W_NDimArray(lngth, [lngth], self.base.dtype, - self.base.order) + if not (space.isinstance_w(w_idx, space.w_int) or + space.isinstance_w(w_idx, space.w_slice)): + raise OperationError(space.w_IndexError, + space.wrap('unsupported iterator index')) + start, stop, step, lngth = space.decode_index4(w_idx, self.base.size) # setslice would have been better, but flat[u:v] for arbitrary # shapes of array a cannot be represented as a[x1:x2, y1:y2] basei = ViewIterator(self.base.start, self.base.strides, self.base.backstrides,self.base.shape) shapelen = len(self.base.shape) basei = basei.next_skip_x(shapelen, start) + if lngth <2: + return self.base.getitem(basei.offset) ri = ArrayIterator(lngth) + res = W_NDimArray(lngth, [lngth], self.base.dtype, + self.base.order) while not ri.done(): - # TODO: add a jit_merge_point + # TODO: add a jit_merge_point? w_val = self.base.getitem(basei.offset) res.setitem(ri.offset,w_val) basei = basei.next_skip_x(shapelen, step) @@ -1538,7 +1508,11 @@ return res def descr_setitem(self, space, w_idx, w_value): - coords, start, step, lngth = _to_coords(space, self.base, w_idx) + if not (space.isinstance_w(w_idx, space.w_int) or + space.isinstance_w(w_idx, space.w_slice)): + raise OperationError(space.w_IndexError, + space.wrap('unsupported iterator index')) + start, stop, step, lngth = space.decode_index4(w_idx, self.base.size) arr = convert_to_array(space, w_value) ai = 0 basei = ViewIterator(self.base.start, self.base.strides, @@ -1546,7 +1520,7 @@ shapelen = len(self.base.shape) basei = basei.next_skip_x(shapelen, start) for i in range(lngth): - # TODO: add jit_merge_point + # TODO: add jit_merge_point? v = arr.getitem(ai).convert_to(self.base.dtype) self.base.setitem(basei.offset, v) # need to repeat input values until all assignments are done diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -1391,6 +1391,7 @@ assert b[-2] == 8 raises(IndexError, "b[11]") raises(IndexError, "b[-11]") + raises(IndexError, 'b[0, 1]') assert b.index == 3 assert b.coords == (0,3) diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py --- a/pypy/module/micronumpy/test/test_zjit.py +++ b/pypy/module/micronumpy/test/test_zjit.py @@ -12,7 +12,7 @@ from pypy.module.micronumpy.compile import (FakeSpace, IntObject, Parser, InterpreterState) from pypy.module.micronumpy.interp_numarray import (W_NDimArray, - BaseArray) + BaseArray, W_FlatIterator) from pypy.rlib.nonconst import NonConstant @@ -50,6 +50,8 @@ if not len(interp.results): raise Exception("need results") w_res = interp.results[-1] + if isinstance(w_res, W_FlatIterator): + w_res = w_res.next() if isinstance(w_res, BaseArray): concr = w_res.get_concrete_or_scalar() sig = concr.find_sig() @@ -369,7 +371,28 @@ 'setinteriorfield_raw': 1, 'int_add': 2, 'int_ge': 1, 'guard_false': 1, 'jump': 1, 'arraylen_gc': 1}) + def define_flat_iter(): + return ''' + a = |30| + a -> flat + ''' + def test_flat_iter(self): + result = self.run("flat_iter") + assert result == 0 + + def define_flat_getitem(): + return ''' + a = |30| + b = a -> flat + b -> 6 + ''' + + def test_flat_getitem(self): + result = self.run("flat_getitem") + assert result == 4 + self.check_trace_count(1) + self.check_simple_loop({}) class TestNumpyOld(LLJitMixin): def setup_class(cls): From noreply at buildbot.pypy.org Thu Jan 26 23:03:51 2012 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 26 Jan 2012 23:03:51 +0100 (CET) Subject: [pypy-commit] pypy numpy-back-to-applevel: no longer needed Message-ID: <20120126220351.4BB0E82B1F@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: numpy-back-to-applevel Changeset: r51829:1d6e533a0aaa Date: 2012-01-27 00:03 +0200 http://bitbucket.org/pypy/pypy/changeset/1d6e533a0aaa/ Log: no longer needed diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -294,28 +294,6 @@ return space.wrap(self.dump_data()) return space.call_function(cache.w_array_repr, self) - res = StringBuilder() - res.append("array(") - concrete = self.get_concrete_or_scalar() - dtype = concrete.find_dtype() - if not concrete.size: - res.append('[]') - if len(self.shape) > 1: - # An empty slice reports its shape - res.append(", shape=(") - self_shape = str(self.shape) - res.append_slice(str(self_shape), 1, len(self_shape) - 1) - res.append(')') - else: - concrete.to_str(space, 1, res, indent=' ') - if (dtype is not interp_dtype.get_dtype_cache(space).w_float64dtype and - not (dtype.kind == interp_dtype.SIGNEDLTR and - dtype.itemtype.get_element_size() == rffi.sizeof(lltype.Signed)) or - not self.size): - res.append(", dtype=" + dtype.name) - res.append(")") - return space.wrap(res.build()) - def dump_data(self): concr = self.get_concrete() i = concr.create_iter() @@ -337,11 +315,6 @@ if cache.w_array_str is None: return space.wrap(self.dump_data()) return space.call_function(cache.w_array_str, self) - - ret = StringBuilder() - concrete = self.get_concrete_or_scalar() - concrete.to_str(space, 0, ret, ' ') - return space.wrap(ret.build()) @jit.unroll_safe def _single_item_result(self, space, w_idx): @@ -695,9 +668,6 @@ def find_dtype(self): return self.dtype - def to_str(self, space, comma, builder, indent=' ', use_ellipsis=False): - builder.append(self.dtype.itemtype.str_format(self.value)) - def copy(self, space): return Scalar(self.dtype, self.value) @@ -934,89 +904,6 @@ self.strides = strides self.backstrides = backstrides - def to_str(self, space, comma, builder, indent=' ', use_ellipsis=False): - '''Modifies builder with a representation of the array/slice - The items will be seperated by a comma if comma is 1 - Multidimensional arrays/slices will span a number of lines, - each line will begin with indent. - ''' - size = self.size - ccomma = ',' * comma - ncomma = ',' * (1 - comma) - dtype = self.find_dtype() - if size < 1: - builder.append('[]') - return - if size > 1000: - # Once this goes True it does not go back to False for recursive - # calls - use_ellipsis = True - ndims = len(self.shape) - if ndims == 0: - builder.append(dtype.itemtype.str_format(self.getitem(0))) - return - i = 0 - builder.append('[') - if ndims > 1: - if use_ellipsis: - for i in range(min(3, self.shape[0])): - if i > 0: - builder.append(ccomma + '\n') - if ndims >= 3: - builder.append('\n' + indent) - else: - builder.append(indent) - view = self.create_slice([Chunk(i, 0, 0, 1)]).get_concrete() - view.to_str(space, comma, builder, indent=indent + ' ', - use_ellipsis=use_ellipsis) - if i < self.shape[0] - 1: - builder.append(ccomma + '\n' + indent + '...' + ncomma) - i = self.shape[0] - 3 - else: - i += 1 - while i < self.shape[0]: - if i > 0: - builder.append(ccomma + '\n') - if ndims >= 3: - builder.append('\n' + indent) - else: - builder.append(indent) - # create_slice requires len(chunks) > 1 in order to reduce - # shape - view = self.create_slice([Chunk(i, 0, 0, 1)]).get_concrete() - view.to_str(space, comma, builder, indent=indent + ' ', - use_ellipsis=use_ellipsis) - i += 1 - elif ndims == 1: - spacer = ccomma + ' ' - item = self.start - # An iterator would be a nicer way to walk along the 1d array, but - # how do I reset it if printing ellipsis? iterators have no - # "set_offset()" - i = 0 - if use_ellipsis: - for i in range(min(3, self.shape[0])): - if i > 0: - builder.append(spacer) - builder.append(dtype.itemtype.str_format(self.getitem(item))) - item += self.strides[0] - if i < self.shape[0] - 1: - # Add a comma only if comma is False - this prevents adding - # two commas - builder.append(spacer + '...' + ncomma) - # Ugly, but can this be done with an iterator? - item = self.start + self.backstrides[0] - 2 * self.strides[0] - i = self.shape[0] - 3 - else: - i += 1 - while i < self.shape[0]: - if i > 0: - builder.append(spacer) - builder.append(dtype.itemtype.str_format(self.getitem(item))) - item += self.strides[0] - i += 1 - builder.append(']') - @jit.unroll_safe def _index_of_single_item(self, space, w_idx): if space.isinstance_w(w_idx, space.w_int): From noreply at buildbot.pypy.org Fri Jan 27 00:31:34 2012 From: noreply at buildbot.pypy.org (antocuni) Date: Fri, 27 Jan 2012 00:31:34 +0100 (CET) Subject: [pypy-commit] pypy py3k: after one day of investigation, add support for unicode characters inside the 'raises' function in apptests. The test still fails because we don't fully support unicode error messages Message-ID: <20120126233134.A0EA682B1F@wyvern.cs.uni-duesseldorf.de> Author: Antonio Cuni <anto.cuni at gmail.com> Branch: py3k Changeset: r51830:f93127358305 Date: 2012-01-26 16:28 +0100 http://bitbucket.org/pypy/pypy/changeset/f93127358305/ Log: after one day of investigation, add support for unicode characters inside the 'raises' function in apptests. The test still fails because we don't fully support unicode error messages diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py --- a/pypy/interpreter/test/test_argument.py +++ b/pypy/interpreter/test/test_argument.py @@ -589,12 +589,18 @@ assert exc.value.message == "<lambda>() takes exactly 2 non-keyword arguments (0 given)" def test_unicode_keywords(self): + """ def f(**kwargs): - assert kwargs[u"美"] == 42 - f(**{u"美" : 42}) + assert kwargs["美"] == 42 + f(**{"美" : 42}) + # + # XXX: the following test fails because we cannot have error messages + # with unicode characters yet, and it tries to build a message like: + # "f() got an unexpected keyword argument 'ü'" def f(x): pass - e = raises(TypeError, "f(**{u'ü' : 19})") + e = raises(TypeError, "f(**{'ü' : 19})") assert "?" in str(e.value) + """ def make_arguments_for_translation(space, args_w, keywords_w={}, w_stararg=None, w_starstararg=None): diff --git a/pypy/tool/pytest/appsupport.py b/pypy/tool/pytest/appsupport.py --- a/pypy/tool/pytest/appsupport.py +++ b/pypy/tool/pytest/appsupport.py @@ -236,7 +236,7 @@ #if filename.endswith("pyc"): # filename = filename[:-1] try: - space.exec_(str(source), frame.w_globals, w_locals, + space.exec_(unicode(source).encode('utf-8'), frame.w_globals, w_locals, filename=filename) except OperationError, e: if e.match(space, w_ExpectedException): From noreply at buildbot.pypy.org Fri Jan 27 00:31:35 2012 From: noreply at buildbot.pypy.org (antocuni) Date: Fri, 27 Jan 2012 00:31:35 +0100 (CET) Subject: [pypy-commit] pypy py3k: fix test_constants on 32bit, by ignoring the L suffix that we get in python2 in the repr of longs Message-ID: <20120126233135.D69D182B1F@wyvern.cs.uni-duesseldorf.de> Author: Antonio Cuni <anto.cuni at gmail.com> Branch: py3k Changeset: r51831:c08972109bbb Date: 2012-01-26 15:33 +0000 http://bitbucket.org/pypy/pypy/changeset/c08972109bbb/ Log: fix test_constants on 32bit, by ignoring the L suffix that we get in python2 in the repr of longs diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py --- a/pypy/interpreter/astcompiler/test/test_compiler.py +++ b/pypy/interpreter/astcompiler/test/test_compiler.py @@ -76,12 +76,15 @@ w_res = pyco_expr.exec_host_bytecode(w_dict, w_dict) res = space.str_w(space.repr(w_res)) expected_repr = self.get_py3_repr(expected) - if not isinstance(expected, float): - assert res == expected_repr - else: + if isinstance(expected, float): # Float representation can vary a bit between interpreter # versions, compare the numbers instead. assert eval(res) == expected + elif isinstance(expected, long): + assert expected_repr.endswith('L') + assert res == expected_repr[:-1] # in py3 we don't have the L suffix + else: + assert res == expected_repr def simple_test(self, source, evalexpr, expected): w_g = self.run(source) From noreply at buildbot.pypy.org Fri Jan 27 00:31:37 2012 From: noreply at buildbot.pypy.org (antocuni) Date: Fri, 27 Jan 2012 00:31:37 +0100 (CET) Subject: [pypy-commit] pypy py3k: kill all the tests which use import * from within a function, which is no longer syntactically valid. Leave just one test in the INVALID case Message-ID: <20120126233137.16DAE82B1F@wyvern.cs.uni-duesseldorf.de> Author: Antonio Cuni <anto.cuni at gmail.com> Branch: py3k Changeset: r51832:7c5c365ec355 Date: 2012-01-26 16:44 +0100 http://bitbucket.org/pypy/pypy/changeset/7c5c365ec355/ Log: kill all the tests which use import * from within a function, which is no longer syntactically valid. Leave just one test in the INVALID case diff --git a/pypy/interpreter/test/test_syntax.py b/pypy/interpreter/test/test_syntax.py --- a/pypy/interpreter/test/test_syntax.py +++ b/pypy/interpreter/test/test_syntax.py @@ -19,81 +19,39 @@ exec "hi" x - def f(): - def g(): - global x - from a import * - x - def f(x): def g(): global x exec "hi" x - def f(x): - def g(): - global x - from a import * - x - - def f(): - def g(): - from a import * - def f(): def g(): exec "hi" def f(): - from a import * - - def f(): exec "hi" def f(): - from a import * - def g(): - global x - x - - def f(): exec "hi" def g(): global x x def f(): - from a import * - def g(x): - x - - def f(): exec "hi" def g(x): x def f(): - from a import * - lambda x: x - - def f(): exec "hi" lambda x: x def f(): - from a import * - x - - def f(): exec "hi" x def f(): - from a import * - (i for i in x) - - def f(): exec "hi" (i for i in x) @@ -102,27 +60,21 @@ exec "hi" x - def f(): - class g: - from a import * - x - """) -## --- the following ones are valid in CPython, but not sensibly so: +## --- the following one is valid in CPython, but not sensibly so: ## --- if x is rebound, then it is even rebound in the parent scope! ## def f(x): ## class g: -## from a import * -## x -## def f(x): -## class g: ## exec "x=41" ## x INVALID = splitcases(""" def f(): + from x import * + + def f(): def g(): exec "hi" x @@ -134,17 +86,6 @@ x def f(): - def g(): - from a import * - x - # NB. the above one is invalid in CPython, but there is no real reason - - def f(x): - def g(): - from a import * - x - - def f(): exec "hi" def g(): x @@ -154,40 +95,17 @@ lambda x: y def f(): - from a import * - x - def g(): - x - - def f(): - from a import * - lambda x: y - - def f(): exec "hi" class g: x def f(): - from a import * - x - class g: - x - - def f(): exec "hi" x = 5 class g: def h(): x - def f(): - from a import * - x = 4 - class g: - def h(): - x - def f(x): exec "hi" x = 4 @@ -195,33 +113,15 @@ x def f(x): - from a import * - class g: - x - - def f(x): exec "hi" x = 5 class g: def h(): x - def f(x): - from a import * - x = 5 - class g: - def h(): - x - def f(): (i for i in x) = 10 - def f(x): - def g(): - from a import * - def k(): - return x - """) From noreply at buildbot.pypy.org Fri Jan 27 00:31:38 2012 From: noreply at buildbot.pypy.org (antocuni) Date: Fri, 27 Jan 2012 00:31:38 +0100 (CET) Subject: [pypy-commit] pypy py3k: kill code to deal with 'import *' inside functions, as it's not longer valid syntax Message-ID: <20120126233138.491B482B1F@wyvern.cs.uni-duesseldorf.de> Author: Antonio Cuni <anto.cuni at gmail.com> Branch: py3k Changeset: r51833:d882e23cc8ae Date: 2012-01-26 16:55 +0100 http://bitbucket.org/pypy/pypy/changeset/d882e23cc8ae/ Log: kill code to deal with 'import *' inside functions, as it's not longer valid syntax diff --git a/pypy/interpreter/astcompiler/symtable.py b/pypy/interpreter/astcompiler/symtable.py --- a/pypy/interpreter/astcompiler/symtable.py +++ b/pypy/interpreter/astcompiler/symtable.py @@ -246,9 +246,7 @@ self.return_with_value = True self.ret = ret - def note_import_star(self, imp): - self.optimized = False - self.import_star = imp + def note_import_star(self): return True def note_variable_arg(self, vararg): @@ -273,19 +271,7 @@ def _check_optimization(self): if (self.has_free or self.child_has_free) and not self.optimized: - err = None - if self.child_has_free: - trailer = "contains a nested function with free variables" - else: - trailer = "is a nested function" - name = self.name - if self.import_star: - node = self.import_star - err = "import * is not allowed in function '%s' because " \ - "it %s" % (name, trailer) - else: - raise AssertionError("unknown reason for unoptimization") - raise SyntaxError(err, node.lineno, node.col_offset) + raise AssertionError("unknown reason for unoptimization") class ClassScope(Scope): From noreply at buildbot.pypy.org Fri Jan 27 00:31:39 2012 From: noreply at buildbot.pypy.org (antocuni) Date: Fri, 27 Jan 2012 00:31:39 +0100 (CET) Subject: [pypy-commit] pypy py3k: extend these test to make sure that the valid snippets are actually valid with cpython3. So far it expects to find the python3 executable in the path Message-ID: <20120126233139.7BD6982B1F@wyvern.cs.uni-duesseldorf.de> Author: Antonio Cuni <anto.cuni at gmail.com> Branch: py3k Changeset: r51834:a8eb581e9e9a Date: 2012-01-26 17:58 +0100 http://bitbucket.org/pypy/pypy/changeset/a8eb581e9e9a/ Log: extend these test to make sure that the valid snippets are actually valid with cpython3. So far it expects to find the python3 executable in the path diff --git a/pypy/interpreter/test/test_syntax.py b/pypy/interpreter/test/test_syntax.py --- a/pypy/interpreter/test/test_syntax.py +++ b/pypy/interpreter/test/test_syntax.py @@ -1,4 +1,5 @@ import py +import commands from pypy.conftest import gettestobjspace def splitcases(s): @@ -126,9 +127,10 @@ for i in range(len(VALID)): - exec """def test_valid_%d(space): + exec """def test_valid_%d(space, tmpdir): + checkvalid_cpython(tmpdir, %d, %r) checkvalid(space, %r) -""" % (i, VALID[i]) +""" % (i, i, VALID[i], VALID[i]) for i in range(len(INVALID)): exec """def test_invalid_%d(space): @@ -136,6 +138,27 @@ """ % (i, INVALID[i]) +def checkvalid_cpython(tmpdir, i, s): + src = ''' +try: + exec("""%s +""") +except SyntaxError as e: + print(e) + raise SystemExit(1) +else: + print('OK') +''' % s + pyfile = tmpdir.join('checkvalid_%d.py' % i) + pyfile.write(src) + res = commands.getoutput('python3 "%s"' % pyfile) + if res != 'OK': + print s + print + print res + assert False, 'checkvalid_cpython failed' + + def checkvalid(space, s): try: space.call_function(space.builtin.get('compile'), From noreply at buildbot.pypy.org Fri Jan 27 00:31:40 2012 From: noreply at buildbot.pypy.org (antocuni) Date: Fri, 27 Jan 2012 00:31:40 +0100 (CET) Subject: [pypy-commit] pypy py3k: check for invalid syntax also on top of cpython3 Message-ID: <20120126233140.AE07D82B1F@wyvern.cs.uni-duesseldorf.de> Author: Antonio Cuni <anto.cuni at gmail.com> Branch: py3k Changeset: r51835:64a00c269dce Date: 2012-01-27 00:02 +0100 http://bitbucket.org/pypy/pypy/changeset/64a00c269dce/ Log: check for invalid syntax also on top of cpython3 diff --git a/pypy/interpreter/test/test_syntax.py b/pypy/interpreter/test/test_syntax.py --- a/pypy/interpreter/test/test_syntax.py +++ b/pypy/interpreter/test/test_syntax.py @@ -133,12 +133,12 @@ """ % (i, i, VALID[i], VALID[i]) for i in range(len(INVALID)): - exec """def test_invalid_%d(space): + exec """def test_invalid_%d(space, tmpdir): + checkinvalid_cpython(tmpdir, %d, %r) checkinvalid(space, %r) -""" % (i, INVALID[i]) +""" % (i, i, INVALID[i], INVALID[i]) - -def checkvalid_cpython(tmpdir, i, s): +def checksyntax_cpython(tmpdir, i, s): src = ''' try: exec("""%s @@ -152,12 +152,24 @@ pyfile = tmpdir.join('checkvalid_%d.py' % i) pyfile.write(src) res = commands.getoutput('python3 "%s"' % pyfile) + return res + +def checkvalid_cpython(tmpdir, i, s): + res = checksyntax_cpython(tmpdir, i, s) if res != 'OK': print s print print res assert False, 'checkvalid_cpython failed' +def checkinvalid_cpython(tmpdir, i, s): + res = checksyntax_cpython(tmpdir, i, s) + if res == 'OK': + print s + print + print res + assert False, 'checkinvalid_cpython failed, did not raise SyntaxError' + def checkvalid(space, s): try: From noreply at buildbot.pypy.org Fri Jan 27 00:31:41 2012 From: noreply at buildbot.pypy.org (antocuni) Date: Fri, 27 Jan 2012 00:31:41 +0100 (CET) Subject: [pypy-commit] pypy py3k: don't try to run python3 if we cannot find it Message-ID: <20120126233141.DFED982B1F@wyvern.cs.uni-duesseldorf.de> Author: Antonio Cuni <anto.cuni at gmail.com> Branch: py3k Changeset: r51836:003f9177cdb4 Date: 2012-01-27 00:06 +0100 http://bitbucket.org/pypy/pypy/changeset/003f9177cdb4/ Log: don't try to run python3 if we cannot find it diff --git a/pypy/interpreter/test/test_syntax.py b/pypy/interpreter/test/test_syntax.py --- a/pypy/interpreter/test/test_syntax.py +++ b/pypy/interpreter/test/test_syntax.py @@ -138,7 +138,13 @@ checkinvalid(space, %r) """ % (i, i, INVALID[i], INVALID[i]) + +PYTHON3 = py.path.local.sysfind('python3') def checksyntax_cpython(tmpdir, i, s): + if PYTHON3 is None: + print 'Warning: cannot run python3 to check syntax' + return + src = ''' try: exec("""%s @@ -151,12 +157,12 @@ ''' % s pyfile = tmpdir.join('checkvalid_%d.py' % i) pyfile.write(src) - res = commands.getoutput('python3 "%s"' % pyfile) + res = commands.getoutput('"%s" "%s"' % (PYTHON3, pyfile)) return res def checkvalid_cpython(tmpdir, i, s): res = checksyntax_cpython(tmpdir, i, s) - if res != 'OK': + if res is not None and res != 'OK': print s print print res @@ -164,7 +170,7 @@ def checkinvalid_cpython(tmpdir, i, s): res = checksyntax_cpython(tmpdir, i, s) - if res == 'OK': + if res is not None and res == 'OK': print s print print res From noreply at buildbot.pypy.org Fri Jan 27 00:31:43 2012 From: noreply at buildbot.pypy.org (antocuni) Date: Fri, 27 Jan 2012 00:31:43 +0100 (CET) Subject: [pypy-commit] pypy py3k: exec is a function in python3, not a statement Message-ID: <20120126233143.21ADA82B1F@wyvern.cs.uni-duesseldorf.de> Author: Antonio Cuni <anto.cuni at gmail.com> Branch: py3k Changeset: r51837:4a4a774f432f Date: 2012-01-27 00:10 +0100 http://bitbucket.org/pypy/pypy/changeset/4a4a774f432f/ Log: exec is a function in python3, not a statement diff --git a/pypy/interpreter/test/test_syntax.py b/pypy/interpreter/test/test_syntax.py --- a/pypy/interpreter/test/test_syntax.py +++ b/pypy/interpreter/test/test_syntax.py @@ -17,48 +17,48 @@ def f(): def g(): global x - exec "hi" + exec("hi") x def f(x): def g(): global x - exec "hi" + exec("hi") x def f(): def g(): - exec "hi" + exec("hi") def f(): - exec "hi" + exec("hi") def f(): - exec "hi" + exec("hi") def g(): global x x def f(): - exec "hi" + exec("hi") def g(x): x def f(): - exec "hi" + exec("hi") lambda x: x def f(): - exec "hi" + exec("hi") x def f(): - exec "hi" + exec("hi") (i for i in x) def f(): class g: - exec "hi" + exec("hi") x """) From noreply at buildbot.pypy.org Fri Jan 27 00:31:44 2012 From: noreply at buildbot.pypy.org (antocuni) Date: Fri, 27 Jan 2012 00:31:44 +0100 (CET) Subject: [pypy-commit] pypy py3k: exec is a function in python3, not a statement (also in the INVALID cases, although some still fail) Message-ID: <20120126233144.5448A82B1F@wyvern.cs.uni-duesseldorf.de> Author: Antonio Cuni <anto.cuni at gmail.com> Branch: py3k Changeset: r51838:cbba65e7057f Date: 2012-01-27 00:13 +0100 http://bitbucket.org/pypy/pypy/changeset/cbba65e7057f/ Log: exec is a function in python3, not a statement (also in the INVALID cases, although some still fail) diff --git a/pypy/interpreter/test/test_syntax.py b/pypy/interpreter/test/test_syntax.py --- a/pypy/interpreter/test/test_syntax.py +++ b/pypy/interpreter/test/test_syntax.py @@ -77,44 +77,44 @@ def f(): def g(): - exec "hi" + exec("hi") x # NB. the above one is invalid in CPython, but there is no real reason def f(x): def g(): - exec "hi" + exec("hi") x def f(): - exec "hi" + exec("hi") def g(): x def f(): - exec "hi" + exec("hi") lambda x: y def f(): - exec "hi" + exec("hi") class g: x def f(): - exec "hi" + exec("hi") x = 5 class g: def h(): x def f(x): - exec "hi" + exec("hi") x = 4 class g: x def f(x): - exec "hi" + exec("hi") x = 5 class g: def h(): From noreply at buildbot.pypy.org Fri Jan 27 00:32:47 2012 From: noreply at buildbot.pypy.org (mattip) Date: Fri, 27 Jan 2012 00:32:47 +0100 (CET) Subject: [pypy-commit] pypy numppy-flatitter: fix for z_jit test, now passes Message-ID: <20120126233247.8478D82B1F@wyvern.cs.uni-duesseldorf.de> Author: mattip Branch: numppy-flatitter Changeset: r51839:5a2c53898154 Date: 2012-01-19 11:30 +0200 http://bitbucket.org/pypy/pypy/changeset/5a2c53898154/ Log: fix for z_jit test, now passes diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py --- a/pypy/module/micronumpy/compile.py +++ b/pypy/module/micronumpy/compile.py @@ -9,7 +9,7 @@ from pypy.module.micronumpy import interp_boxes from pypy.module.micronumpy.interp_dtype import get_dtype_cache from pypy.module.micronumpy.interp_numarray import (Scalar, BaseArray, - scalar_w, W_NDimArray, array) + scalar_w, W_NDimArray, array, W_FlatIterator) from pypy.module.micronumpy import interp_ufuncs from pypy.rlib.objectmodel import specialize, instantiate @@ -313,6 +313,21 @@ def __repr__(self): return 'Range(%s)' % self.v +class FlatIterFromRange(Node): + def __init__(self, v): + self.v = int(v) + + def execute(self, interp): + w_list = interp.space.newlist( + [interp.space.wrap(float(i)) for i in range(self.v)] + ) + dtype = get_dtype_cache(interp.space).w_float64dtype + return W_FlatIterator(array(interp.space, w_list, w_dtype=dtype, + w_order=None)) + + def __repr__(self): + return 'FlatFromRange(%s)' % self.v + class Code(Node): def __init__(self, statements): self.statements = statements @@ -422,6 +437,7 @@ ('=', 'assign'), (',', 'comma'), ('\|', 'pipe'), + ('\!', 'exclaim'), ('$', 'paren_left'), ('$', 'paren_right'), ] @@ -528,6 +544,10 @@ stack.append(RangeConstant(tokens.pop().v)) end = tokens.pop() assert end.name == 'pipe' + elif token.name == 'exclaim': + stack.append(FlatIterFromRange(tokens.pop().v)) + end = tokens.pop() + assert end.name == 'exclaim' elif accept_comma and token.name == 'comma': continue else: diff --git a/pypy/module/micronumpy/interp_iter.py b/pypy/module/micronumpy/interp_iter.py --- a/pypy/module/micronumpy/interp_iter.py +++ b/pypy/module/micronumpy/interp_iter.py @@ -137,7 +137,7 @@ res._done = done return res - #@jit.unroll_safe + @jit.unroll_safe def next_skip_x(self, shapelen, step): shapelen = jit.promote(len(self.res_shape)) offset = self.offset diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py --- a/pypy/module/micronumpy/test/test_zjit.py +++ b/pypy/module/micronumpy/test/test_zjit.py @@ -50,8 +50,6 @@ if not len(interp.results): raise Exception("need results") w_res = interp.results[-1] - if isinstance(w_res, W_FlatIterator): - w_res = w_res.next() if isinstance(w_res, BaseArray): concr = w_res.get_concrete_or_scalar() sig = concr.find_sig() @@ -373,8 +371,7 @@ 'arraylen_gc': 1}) def define_flat_iter(): return ''' - a = |30| - a -> flat + !30! ''' def test_flat_iter(self): @@ -383,16 +380,14 @@ def define_flat_getitem(): return ''' - a = |30| - b = a -> flat - b -> 6 + a = !30! + a -> 6 ''' def test_flat_getitem(self): result = self.run("flat_getitem") - assert result == 4 - self.check_trace_count(1) - self.check_simple_loop({}) + assert result == 6.0 + self.check_trace_count(0) class TestNumpyOld(LLJitMixin): def setup_class(cls): From noreply at buildbot.pypy.org Fri Jan 27 00:32:48 2012 From: noreply at buildbot.pypy.org (mattip) Date: Fri, 27 Jan 2012 00:32:48 +0100 (CET) Subject: [pypy-commit] pypy numppy-flatitter: add iter test, more to come Message-ID: <20120126233248.B19B782B1F@wyvern.cs.uni-duesseldorf.de> Author: mattip Branch: numppy-flatitter Changeset: r51840:fc5fe1efe542 Date: 2012-01-19 11:37 +0200 http://bitbucket.org/pypy/pypy/changeset/fc5fe1efe542/ Log: add iter test, more to come diff --git a/pypy/module/micronumpy/test/test_iter.py b/pypy/module/micronumpy/test/test_iter.py new file mode 100644 --- /dev/null +++ b/pypy/module/micronumpy/test/test_iter.py @@ -0,0 +1,12 @@ +import py +from pypy.module.micronumpy.interp_iter import ViewIterator + +class TestIterDirect(object): + def test_viewiterator(self): + i = ViewIterator(0, [5, 1], [10, 4], [3, 5]) + i = i.next(2) + i = i.next(2) + i = i.next(2) + assert i.offset == 3 + assert not i.done() + assert i.indices == [0,3] From noreply at buildbot.pypy.org Fri Jan 27 01:36:09 2012 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Fri, 27 Jan 2012 01:36:09 +0100 (CET) Subject: [pypy-commit] pypy default: better organization, and show data for numpy.generic as well Message-ID: <20120127003609.5E63982B1F@wyvern.cs.uni-duesseldorf.de> Author: Alex Gaynor <alex.gaynor at gmail.com> Branch: Changeset: r51841:8276de5ece4c Date: 2012-01-26 19:35 -0500 http://bitbucket.org/pypy/pypy/changeset/8276de5ece4c/ Log: better organization, and show data for numpy.generic as well diff --git a/pypy/module/micronumpy/tool/numready/main.py b/pypy/module/micronumpy/tool/numready/main.py --- a/pypy/module/micronumpy/tool/numready/main.py +++ b/pypy/module/micronumpy/tool/numready/main.py @@ -73,9 +73,8 @@ for line in lines: kind, name = line.split(" : ", 1) subitems = None - if kind == KINDS["TYPE"]: - if name in ['ndarray', 'dtype']: - subitems = find_numpy_items(python, modname, name) + if kind == KINDS["TYPE"] and name in SPECIAL_NAMES and attr is None: + subitems = find_numpy_items(python, modname, name) items.add(Item(name, kind, subitems)) return items @@ -89,15 +88,20 @@ l[i].append(lst[k * lgt + i]) return l +SPECIAL_NAMES = ["ndarray", "dtype", "generic"] + def main(argv): cpy_items = find_numpy_items("/usr/bin/python") pypy_items = find_numpy_items(argv[1], "numpypy") all_items = [] - msg = '%d/%d names, %d/%d ndarray attributes, %d/%d dtype attributes' % ( - len(pypy_items), len(cpy_items), len(pypy_items['ndarray'].subitems), - len(cpy_items['ndarray'].subitems), len(pypy_items['dtype'].subitems), - len(cpy_items['dtype'].subitems)) + msg = "{:d}/{:d} names".format(len(pypy_items), len(cpy_items)) + msg += ", ".join( + "{:d}/{:d} {} attributes".format( + len(pypy_items[name].subitems), len(cpy_items[name].subitems), name + ) + for name in SPECIAL_NAMES + ) for item in cpy_items: pypy_exists = item in pypy_items if item.subitems: From noreply at buildbot.pypy.org Fri Jan 27 03:11:19 2012 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Fri, 27 Jan 2012 03:11:19 +0100 (CET) Subject: [pypy-commit] pypy default: use a .html extension Message-ID: <20120127021119.87BEF82B1F@wyvern.cs.uni-duesseldorf.de> Author: Alex Gaynor <alex.gaynor at gmail.com> Branch: Changeset: r51842:457e2166648c Date: 2012-01-26 21:11 -0500 http://bitbucket.org/pypy/pypy/changeset/457e2166648c/ Log: use a .html extension diff --git a/pypy/module/micronumpy/tool/numready/main.py b/pypy/module/micronumpy/tool/numready/main.py --- a/pypy/module/micronumpy/tool/numready/main.py +++ b/pypy/module/micronumpy/tool/numready/main.py @@ -118,6 +118,6 @@ with open(argv[2], 'w') as f: f.write(html.encode("utf-8")) else: - with tempfile.NamedTemporaryFile(delete=False) as f: + with tempfile.NamedTemporaryFile(delete=False, suffix=".html") as f: f.write(html.encode("utf-8")) print "Saved in: %s" % f.name From noreply at buildbot.pypy.org Fri Jan 27 09:57:18 2012 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 27 Jan 2012 09:57:18 +0100 (CET) Subject: [pypy-commit] extradoc extradoc: Update Message-ID: <20120127085718.0A33382114@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: extradoc Changeset: r4056:6d59f50abf58 Date: 2012-01-27 09:57 +0100 http://bitbucket.org/pypy/extradoc/changeset/6d59f50abf58/ Log: Update diff --git a/planning/stm.txt b/planning/stm.txt --- a/planning/stm.txt +++ b/planning/stm.txt @@ -65,6 +65,11 @@ (Optimization: objects declared immutable don't need a version number.) +GC_WAS_COPIED should rather be some counter, counting how many threads +have a local copy; something like 2 or 3 bits, where the maximum value +means "overflowed" and is sticky (maybe until some global +synchronization point, if we have one). + stm_read -------- From noreply at buildbot.pypy.org Fri Jan 27 10:06:45 2012 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 27 Jan 2012 10:06:45 +0100 (CET) Subject: [pypy-commit] extradoc extradoc: update Message-ID: <20120127090645.5996B82114@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: extradoc Changeset: r4057:d2cbf8878e65 Date: 2012-01-27 10:06 +0100 http://bitbucket.org/pypy/extradoc/changeset/d2cbf8878e65/ Log: update diff --git a/planning/stm.txt b/planning/stm.txt --- a/planning/stm.txt +++ b/planning/stm.txt @@ -68,7 +68,9 @@ GC_WAS_COPIED should rather be some counter, counting how many threads have a local copy; something like 2 or 3 bits, where the maximum value means "overflowed" and is sticky (maybe until some global -synchronization point, if we have one). +synchronization point, if we have one). Or, we can be more advanced and +use 4-5 bits, where in addition we use some "thread hash" value if there +is only one copy. stm_read From noreply at buildbot.pypy.org Fri Jan 27 10:43:32 2012 From: noreply at buildbot.pypy.org (mattip) Date: Fri, 27 Jan 2012 10:43:32 +0100 (CET) Subject: [pypy-commit] pypy numppy-flatitter: more tests Message-ID: <20120127094332.0EA4D82114@wyvern.cs.uni-duesseldorf.de> Author: mattip Branch: numppy-flatitter Changeset: r51843:794461ec9ef3 Date: 2012-01-27 11:24 +0200 http://bitbucket.org/pypy/pypy/changeset/794461ec9ef3/ Log: more tests diff --git a/pypy/module/micronumpy/test/test_iter.py b/pypy/module/micronumpy/test/test_iter.py --- a/pypy/module/micronumpy/test/test_iter.py +++ b/pypy/module/micronumpy/test/test_iter.py @@ -1,12 +1,132 @@ import py from pypy.module.micronumpy.interp_iter import ViewIterator +# This is both a test and a mini-tutorial on iterators, strides, and +# memory layout. It assumes you are familiar with the terms, see +# http://docs.scipy.org/doc/numpy/reference/arrays.ndarray.html +# for a more gentle introduction. +# +# Given an array x: x.shape == [5,6], +# +# At which byte in x.data does the item x[3,4] begin? +# if x.strides==[1,5]: +# pData = x.pData + (x.start + 3*1 + 4*5)*sizeof(x.pData[0]) +# pData = x.pData + (x.start + 24) * sizeof(x.pData[0]) +# so the offset of the element is 24 elements after the first +# +# What is the next element in x after coordinates [3,4]? +# if x.order =='C': +# next == [3,5] => offset is 28 +# if x.order =='F': +# next == [4,4] => offset is 24 +# so for the strides [1,5] x is 'F' contiguous +# likewise, for the strides [6,1] x would be 'C' contiguous. +# +# Iterators have an internal representation of the current coordinates +# (indices), the array, strides, and backstrides. A short digression to +# explain backstrides: what is the coordinate and offset after [3,5] in +# the example above? +# if x.order == 'C': +# next == [4,0] => offset is 4 +# if x.order == 'F': +# next == [4,5] => offset is 25 +# Note that in 'C' order we stepped BACKWARDS 24 while 'overflowing' a +# shape dimension +# which is back 25 and forward 1, +# which is x.strides[1] * (x.shape[1] - 1) + x.strides[0] +# so if we precalculate the overflow backstride as +# [x.strides[i] * (x.shape[i] - 1) for i in range(len(x.shape))] +# we can go faster. +# All the calculations happen in next() +# +# next_step_x() tries to do the iteration for a number of steps at once, +# but then we cannot gaurentee that we only overflow one single shape +# dimension, perhaps we could overflow times in one big step. +# + class TestIterDirect(object): - def test_viewiterator(self): - i = ViewIterator(0, [5, 1], [10, 4], [3, 5]) + def test_C_viewiterator(self): + #Let's get started, simple iteration in C order with + #contiguous layout => strides[-1] is 1 + start = 0 + shape = [3, 5] + strides = [5, 1] + backstrides = [x * (y - 1) for x,y in zip(strides, shape)] + assert backstrides == [10, 4] + i = ViewIterator(start, strides, backstrides, shape) i = i.next(2) i = i.next(2) i = i.next(2) assert i.offset == 3 assert not i.done() assert i.indices == [0,3] + #cause a dimension overflow + i = i.next(2) + i = i.next(2) + assert i.offset == 5 + assert i.indices == [1,0] + + #Now what happens if the array is transposed? strides[-1] != 1 + # therefore layout is non-contiguous + strides = [1, 3] + backstrides = [x * (y - 1) for x,y in zip(strides, shape)] + assert backstrides == [2, 12] + i = ViewIterator(start, strides, backstrides, shape) + i = i.next(2) + i = i.next(2) + i = i.next(2) + assert i.offset == 9 + assert not i.done() + assert i.indices == [0,3] + #cause a dimension overflow + i = i.next(2) + i = i.next(2) + assert i.offset == 1 + assert i.indices == [1,0] + + def test_C_viewiterator_step(self): + #iteration in C order with #contiguous layout => strides[-1] is 1 + #skip less than the shape + start = 0 + shape = [3, 5] + strides = [5, 1] + backstrides = [x * (y - 1) for x,y in zip(strides, shape)] + assert backstrides == [10, 4] + i = ViewIterator(start, strides, backstrides, shape) + i = i.next_skip_x(2,2) + i = i.next_skip_x(2,2) + i = i.next_skip_x(2,2) + assert i.offset == 6 + assert not i.done() + assert i.indices == [1,1] + #And for some big skips + i = i.next_skip_x(2,5) + assert i.offset == 11 + assert i.indices == [2,1] + i = i.next_skip_x(2,5) + # Note: the offset does not overflow but recycles, + # this is good for broadcast + assert i.offset == 1 + assert i.indices == [0,1] + assert i.done() + + #Now what happens if the array is transposed? strides[-1] != 1 + # therefore layout is non-contiguous + strides = [1, 3] + backstrides = [x * (y - 1) for x,y in zip(strides, shape)] + assert backstrides == [2, 12] + i = ViewIterator(start, strides, backstrides, shape) + i = i.next_skip_x(2,2) + i = i.next_skip_x(2,2) + i = i.next_skip_x(2,2) + assert i.offset == 4 + assert i.indices == [1,1] + assert not i.done() + i = i.next_skip_x(2,5) + assert i.offset == 5 + assert i.indices == [2,1] + assert not i.done() + i = i.next_skip_x(2,5) + assert i.indices == [0,1] + assert i.offset == 3 + assert i.done() From noreply at buildbot.pypy.org Fri Jan 27 10:43:33 2012 From: noreply at buildbot.pypy.org (mattip) Date: Fri, 27 Jan 2012 10:43:33 +0100 (CET) Subject: [pypy-commit] pypy numppy-flatitter: failed attempt to redo compile.py Message-ID: <20120127094333.3FD2882114@wyvern.cs.uni-duesseldorf.de> Author: mattip Branch: numppy-flatitter Changeset: r51844:32b53b45a4c3 Date: 2012-01-27 11:42 +0200 http://bitbucket.org/pypy/pypy/changeset/32b53b45a4c3/ Log: failed attempt to redo compile.py diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py --- a/pypy/module/micronumpy/compile.py +++ b/pypy/module/micronumpy/compile.py @@ -9,7 +9,7 @@ from pypy.module.micronumpy import interp_boxes from pypy.module.micronumpy.interp_dtype import get_dtype_cache from pypy.module.micronumpy.interp_numarray import (Scalar, BaseArray, - scalar_w, W_NDimArray, array, W_FlatIterator) + scalar_w, W_NDimArray, array) from pypy.module.micronumpy import interp_ufuncs from pypy.rlib.objectmodel import specialize, instantiate @@ -32,7 +32,7 @@ class BadToken(Exception): pass -SINGLE_ARG_FUNCTIONS = ["sum", "prod", "max", "min", "all", "any", "unegative"] +SINGLE_ARG_FUNCTIONS = ["sum", "prod", "max", "min", "all", "any", "unegative", "flat"] class FakeSpace(object): w_ValueError = None @@ -313,21 +313,6 @@ def __repr__(self): return 'Range(%s)' % self.v -class FlatIterFromRange(Node): - def __init__(self, v): - self.v = int(v) - - def execute(self, interp): - w_list = interp.space.newlist( - [interp.space.wrap(float(i)) for i in range(self.v)] - ) - dtype = get_dtype_cache(interp.space).w_float64dtype - return W_FlatIterator(array(interp.space, w_list, w_dtype=dtype, - w_order=None)) - - def __repr__(self): - return 'FlatFromRange(%s)' % self.v - class Code(Node): def __init__(self, statements): self.statements = statements @@ -411,6 +396,8 @@ elif self.name == "unegative": neg = interp_ufuncs.get(interp.space).negative w_res = neg.call(interp.space, [arr]) + elif self.name == "flat": + w_res = arr.descr_flat(interp.space) else: assert False # unreachable code if isinstance(w_res, BaseArray): @@ -437,7 +424,6 @@ ('=', 'assign'), (',', 'comma'), ('\|', 'pipe'), - ('\!', 'exclaim'), ('$', 'paren_left'), ('$', 'paren_right'), ] @@ -544,10 +530,6 @@ stack.append(RangeConstant(tokens.pop().v)) end = tokens.pop() assert end.name == 'pipe' - elif token.name == 'exclaim': - stack.append(FlatIterFromRange(tokens.pop().v)) - end = tokens.pop() - assert end.name == 'exclaim' elif accept_comma and token.name == 'comma': continue else: diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py --- a/pypy/module/micronumpy/test/test_zjit.py +++ b/pypy/module/micronumpy/test/test_zjit.py @@ -371,7 +371,8 @@ 'arraylen_gc': 1}) def define_flat_iter(): return ''' - !30! + a = |30| + flat(a) ''' def test_flat_iter(self): @@ -380,8 +381,9 @@ def define_flat_getitem(): return ''' - a = !30! - a -> 6 + a = |30| + b = flat(a) + b -> 6 ''' def test_flat_getitem(self): From noreply at buildbot.pypy.org Fri Jan 27 10:59:36 2012 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 27 Jan 2012 10:59:36 +0100 (CET) Subject: [pypy-commit] pypy.org extradoc: update progress bars Message-ID: <20120127095936.5F9DC82114@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: extradoc Changeset: r305:347134143935 Date: 2012-01-27 11:56 +0200 http://bitbucket.org/pypy/pypy.org/changeset/347134143935/ Log: update progress bars diff --git a/don1.html b/don1.html --- a/don1.html +++ b/don1.html @@ -8,12 +8,12 @@ <script> $(function() { $("#progressbar").progressbar({ - value: 4.2 + value: 38.0 }); }); </script> - $4882 of $105000 (4.6%) + $39882 of $105000 (38.0%) <div id="progressbar"> </div> diff --git a/don3.html b/don3.html --- a/don3.html +++ b/don3.html @@ -8,12 +8,12 @@ <script> $(function() { $("#progressbar").progressbar({ - value: 4.2 + value: 68.0 }); }); </script> - $5820 of $60000 (9.7%) + $40820 of $60000 (68.0%) <div id="progressbar"> </div> From noreply at buildbot.pypy.org Fri Jan 27 10:59:37 2012 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 27 Jan 2012 10:59:37 +0100 (CET) Subject: [pypy-commit] pypy.org extradoc: merge Message-ID: <20120127095937.8887982114@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: extradoc Changeset: r306:0721767ab411 Date: 2012-01-27 11:59 +0200 http://bitbucket.org/pypy/pypy.org/changeset/0721767ab411/ Log: merge diff --git a/download.html b/download.html --- a/download.html +++ b/download.html @@ -183,9 +183,11 @@ ceb8dfe7d9d1aeb558553b91b381a1a8 pypy-1.7-linux64.tar.bz2 8a6e2583902bc6f2661eb3c96b45f4e3 pypy-1.7-linux.tar.bz2 ff979054fc8e17b4973ffebb9844b159 pypy-1.7-osx64.tar.bz2 +fd0ad58b92ca0933c087bb93a82fda9e release-1.7.tar.bz2 d364e3aa0dd5e0e1ad7f1800a0bfa7e87250c8bb pypy-1.7-linux64.tar.bz2 68554c4cbcc20b03ff56b6a1495a6ecf8f24b23a pypy-1.7-linux.tar.bz2 cedeb1d6bf0431589f62e8c95b71fbfe6c4e7b96 pypy-1.7-osx64.tar.bz2 +b4be3a8dc69cd838a49382867db3c41864b9e8d9 release-1.7.tar.bz2 </pre> </div> </div> diff --git a/features.html b/features.html --- a/features.html +++ b/features.html @@ -87,10 +87,10 @@ </div> <div class="section" id="stackless"> <h1>Stackless</h1> -<p>PyPy is also available in a separate <a class="reference external" href="download.html#stackless-version">Stackless version</a> that includes -support for micro-threads for massive concurrency. Read more about -it at the <a class="reference external" href="http://www.stackless.com/">Stackless</a> main site (we provide the same interface as the -standard Stackless Python), and at the <a class="reference external" href="http://codespeak.net/svn/greenlet/trunk/doc/greenlet.txt">greenlets</a> page.</p> +<p>Support for <a class="reference external" href="http://www.stackless.com/">Stackless</a> and greenlets are now integrated in the normal +PyPy. More detailed information is available <a class="reference external" href="http://doc.pypy.org/en/latest/stackless.html">here</a>.</p> +<p>Note that there is still an important performance hit for programs using +Stackless features.</p> </div> <div class="section" id="other-features"> <h1>Other features</h1> diff --git a/index.html b/index.html --- a/index.html +++ b/index.html @@ -58,7 +58,7 @@ and <a class="reference external" href="http://www.djangoproject.com/">django</a>.</li> <li><strong>Sandboxing:</strong> PyPy provides the ability to <a class="reference external" href="features.html#sandboxing">run untrusted code</a> in a fully secure way.</li> -<li><strong>Stackless:</strong> PyPy can be configured to run in <a class="reference external" href="http://www.stackless.com/">stackless</a> mode, +<li><strong>Stackless:</strong> PyPy comes by default with support for <a class="reference external" href="features.html#stackless">stackless mode</a>, providing micro-threads for massive concurrency.</li> <li>As well as other <a class="reference external" href="features.html">features</a>.</li> </ul> diff --git a/source/download.txt b/source/download.txt --- a/source/download.txt +++ b/source/download.txt @@ -182,7 +182,9 @@ ceb8dfe7d9d1aeb558553b91b381a1a8 pypy-1.7-linux64.tar.bz2 8a6e2583902bc6f2661eb3c96b45f4e3 pypy-1.7-linux.tar.bz2 ff979054fc8e17b4973ffebb9844b159 pypy-1.7-osx64.tar.bz2 + fd0ad58b92ca0933c087bb93a82fda9e release-1.7.tar.bz2 d364e3aa0dd5e0e1ad7f1800a0bfa7e87250c8bb pypy-1.7-linux64.tar.bz2 68554c4cbcc20b03ff56b6a1495a6ecf8f24b23a pypy-1.7-linux.tar.bz2 cedeb1d6bf0431589f62e8c95b71fbfe6c4e7b96 pypy-1.7-osx64.tar.bz2 + b4be3a8dc69cd838a49382867db3c41864b9e8d9 release-1.7.tar.bz2 diff --git a/source/features.txt b/source/features.txt --- a/source/features.txt +++ b/source/features.txt @@ -70,14 +70,14 @@ Stackless -------------------------- -PyPy is also available in a separate `Stackless version`_ that includes -support for micro-threads for massive concurrency. Read more about -it at the Stackless_ main site (we provide the same interface as the -standard Stackless Python), and at the greenlets_ page. +Support for Stackless_ and greenlets are now integrated in the normal +PyPy. More detailed information is available here__. -.. _`Stackless version`: download.html#stackless-version -.. _`stackless`: http://www.stackless.com/ -.. _`greenlets`: http://codespeak.net/svn/greenlet/trunk/doc/greenlet.txt +Note that there is still an important performance hit for programs using +Stackless features. + +.. _Stackless: http://www.stackless.com/ +.. __: http://doc.pypy.org/en/latest/stackless.html Other features diff --git a/source/index.txt b/source/index.txt --- a/source/index.txt +++ b/source/index.txt @@ -19,7 +19,7 @@ * **Sandboxing:** PyPy provides the ability to `run untrusted code`_ in a fully secure way. - * **Stackless:** PyPy can be configured to run in `stackless`_ mode, + * **Stackless:** PyPy comes by default with support for `stackless mode`_, providing micro-threads for massive concurrency. * As well as other `features`_. @@ -33,7 +33,7 @@ Want to know more? A good place to start is our detailed `speed`_ and `compatibility`_ reports! -.. _`stackless`: http://www.stackless.com/ +.. _`stackless mode`: features.html#stackless .. _`Python`: http://python.org/ .. _`fast`: http://speed.pypy.org/ .. _`faster`: http://speed.pypy.org/ From noreply at buildbot.pypy.org Fri Jan 27 11:01:10 2012 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 27 Jan 2012 11:01:10 +0100 (CET) Subject: [pypy-commit] pypy.org extradoc: make it not cache Message-ID: <20120127100110.D514282114@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: extradoc Changeset: r307:f663f346f8dc Date: 2012-01-27 12:00 +0200 http://bitbucket.org/pypy/pypy.org/changeset/f663f346f8dc/ Log: make it not cache diff --git a/js/script2.js b/js/script2.js --- a/js/script2.js +++ b/js/script2.js @@ -1,18 +1,18 @@ function py3k_donate() { - $.get("don1.html", function (html) { + $.get("don1.html#", function (html) { $("#sidebar").html(html); }); } function general_donate() { - $.get("don2.html", function (html) { + $.get("don2.html#", function (html) { $("#sidebar").html(html); }); } function numpy_donate() { - $.get("don3.html", function (html) { + $.get("don3.html#", function (html) { $("#sidebar").html(html); }); } From noreply at buildbot.pypy.org Fri Jan 27 11:20:18 2012 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 27 Jan 2012 11:20:18 +0100 (CET) Subject: [pypy-commit] pypy numppy-flatitter: an absolutely crucial test Message-ID: <20120127102018.4A42C82114@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: numppy-flatitter Changeset: r51845:93ed7361747f Date: 2012-01-27 12:19 +0200 http://bitbucket.org/pypy/pypy/changeset/93ed7361747f/ Log: an absolutely crucial test diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py --- a/pypy/module/micronumpy/compile.py +++ b/pypy/module/micronumpy/compile.py @@ -32,7 +32,8 @@ class BadToken(Exception): pass -SINGLE_ARG_FUNCTIONS = ["sum", "prod", "max", "min", "all", "any", "unegative", "flat"] +SINGLE_ARG_FUNCTIONS = ["sum", "prod", "max", "min", "all", "any", + "unegative", "flat"] class FakeSpace(object): w_ValueError = None diff --git a/pypy/module/micronumpy/test/test_compile.py b/pypy/module/micronumpy/test/test_compile.py --- a/pypy/module/micronumpy/test/test_compile.py +++ b/pypy/module/micronumpy/test/test_compile.py @@ -245,3 +245,11 @@ a -> 3 """) assert interp.results[0].value == 11 + + def test_flat_iter(self): + interp = self.run(''' + a = |30| + b = flat(a) + b -> 3 + ''') + assert interp.results[0].value == 3 diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py --- a/pypy/module/micronumpy/test/test_zjit.py +++ b/pypy/module/micronumpy/test/test_zjit.py @@ -372,12 +372,13 @@ def define_flat_iter(): return ''' a = |30| - flat(a) + b = flat(a) + b -> 3 ''' def test_flat_iter(self): result = self.run("flat_iter") - assert result == 0 + assert result == 3 def define_flat_getitem(): return ''' From notifications-noreply at bitbucket.org Fri Jan 27 11:24:26 2012 From: notifications-noreply at bitbucket.org (Bitbucket) Date: Fri, 27 Jan 2012 10:24:26 -0000 Subject: [pypy-commit] Notification: pypy Message-ID: <20120127102426.1592.55710@bitbucket05.managed.contegix.com> You have received a notification from Stefano Parmesan. Hi, I forked pypy. My fork is at https://bitbucket.org/armisael/pypy. -- Disable notifications at https://bitbucket.org/account/notifications/ From noreply at buildbot.pypy.org Fri Jan 27 11:25:29 2012 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 27 Jan 2012 11:25:29 +0100 (CET) Subject: [pypy-commit] pypy numppy-flatitter: fix the wrong name Message-ID: <20120127102529.1703D82114@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: numppy-flatitter Changeset: r51846:5a27b974638a Date: 2012-01-27 12:24 +0200 http://bitbucket.org/pypy/pypy/changeset/5a27b974638a/ Log: fix the wrong name diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py --- a/pypy/module/micronumpy/compile.py +++ b/pypy/module/micronumpy/compile.py @@ -398,7 +398,7 @@ neg = interp_ufuncs.get(interp.space).negative w_res = neg.call(interp.space, [arr]) elif self.name == "flat": - w_res = arr.descr_flat(interp.space) + w_res = arr.descr_get_flatiter(interp.space) else: assert False # unreachable code if isinstance(w_res, BaseArray): From noreply at buildbot.pypy.org Fri Jan 27 11:50:44 2012 From: noreply at buildbot.pypy.org (RonnyPfannschmidt) Date: Fri, 27 Jan 2012 11:50:44 +0100 (CET) Subject: [pypy-commit] jitviewer default: shuffle around app code, so we have templates+static in the package Message-ID: <20120127105044.E06FA82114@wyvern.cs.uni-duesseldorf.de> Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de> Branch: Changeset: r186:34292f6ef484 Date: 2012-01-27 11:47 +0100 http://bitbucket.org/pypy/jitviewer/changeset/34292f6ef484/ Log: shuffle around app code, so we have templates+static in the package diff --git a/bin/jitviewer.py b/_jitviewer/app.py copy from bin/jitviewer.py copy to _jitviewer/app.py --- a/bin/jitviewer.py +++ b/_jitviewer/app.py @@ -152,10 +152,8 @@ class OverrideFlask(flask.Flask): - root_path = property(lambda self: self._root_path, lambda *args: None) def __init__(self, *args, **kwargs): - self._root_path = kwargs.pop('root_path') self.servers = [] self.evil_monkeypatch() flask.Flask.__init__(self, *args, **kwargs) @@ -172,7 +170,6 @@ BaseServer.__init__ = __init__ def main(): - PATH = os.path.dirname(os.path.dirname((_jitviewer.__file__))) if not '__pypy__' in sys.builtin_module_names: print "Please run it using pypy-c" sys.exit(1) @@ -195,7 +192,7 @@ log, loops = import_log(filename, ParserWithHtmlRepr) parse_log_counts(extract_category(log, 'jit-backend-count'), loops) storage.reconnect_loops(loops) - app = OverrideFlask('jitviewer', root_path=PATH) + app = OverrideFlask('_jitviewer') server = Server(filename, storage) app.debug = True app.route('/')(server.index) diff --git a/static/DroidSansMono.ttf b/_jitviewer/static/DroidSansMono.ttf rename from static/DroidSansMono.ttf rename to _jitviewer/static/DroidSansMono.ttf diff --git a/static/jquery-1.2.6.min.js b/_jitviewer/static/jquery-1.2.6.min.js rename from static/jquery-1.2.6.min.js rename to _jitviewer/static/jquery-1.2.6.min.js diff --git a/static/jquery.min.js b/_jitviewer/static/jquery.min.js rename from static/jquery.min.js rename to _jitviewer/static/jquery.min.js diff --git a/static/jquery.scrollTo-1.4.1.js b/_jitviewer/static/jquery.scrollTo-1.4.1.js rename from static/jquery.scrollTo-1.4.1.js rename to _jitviewer/static/jquery.scrollTo-1.4.1.js diff --git a/static/jquery.scrollTo-1.4.2-min.js b/_jitviewer/static/jquery.scrollTo-1.4.2-min.js rename from static/jquery.scrollTo-1.4.2-min.js rename to _jitviewer/static/jquery.scrollTo-1.4.2-min.js diff --git a/static/loop.js b/_jitviewer/static/loop.js rename from static/loop.js rename to _jitviewer/static/loop.js diff --git a/static/pygments.css b/_jitviewer/static/pygments.css rename from static/pygments.css rename to _jitviewer/static/pygments.css diff --git a/static/qt_workaround.css b/_jitviewer/static/qt_workaround.css rename from static/qt_workaround.css rename to _jitviewer/static/qt_workaround.css diff --git a/static/script.js b/_jitviewer/static/script.js rename from static/script.js rename to _jitviewer/static/script.js diff --git a/static/style.css b/_jitviewer/static/style.css rename from static/style.css rename to _jitviewer/static/style.css diff --git a/templates/index.html b/_jitviewer/templates/index.html rename from templates/index.html rename to _jitviewer/templates/index.html diff --git a/templates/loop.html b/_jitviewer/templates/loop.html rename from templates/loop.html rename to _jitviewer/templates/loop.html diff --git a/bin/jitviewer.py b/bin/jitviewer.py old mode 100755 new mode 100644 --- a/bin/jitviewer.py +++ b/bin/jitviewer.py @@ -1,241 +1,3 @@ #!/usr/bin/env pypy -""" A web-based browser of your log files. Run by - - jitviewer.py <path to your log file> [port] [--qt] - -By default the script will run a web server, point your browser to -http://localhost:5000 - -If you pass --qt, this script will also start a lightweight PyQT/QWebKit based -browser pointing at the jitviewer. This assumes that CPython is installed in -/usr/bin/python, and that PyQT with WebKit support is installed. - -Demo logfile available in this directory as 'log'. - -To produce the logfile for your program, run: - - PYPYLOG=jit-log-opt,jit-backend-counts:mylogfile.log pypy myapp.py -""" - -import sys -import os.path - -try: - import _jitviewer -except ImportError: - sys.path.insert(0, os.path.abspath(os.path.join(__file__, '..', '..'))) - -try: - import pypy -except ImportError: - import __pypy__ - sys.path.append(os.path.join(__pypy__.__file__, '..', '..', '..')) - try: - import pypy - except ImportError: - raise ImportError('Could not import pypy module, make sure to ' - 'add the pypy module to PYTHONPATH') - -import jinja2 -if jinja2.__version__ < '2.6': - raise ImportError("Required jinja version is 2.6 (the git tip), older versions might segfault PyPy") - -import flask -import inspect -import threading -import time -from pypy.tool.logparser import extract_category -from pypy.tool.jitlogparser.storage import LoopStorage -from pypy.tool.jitlogparser.parser import adjust_bridges, import_log,\ - parse_log_counts -# -from _jitviewer.parser import ParserWithHtmlRepr, FunctionHtml -from _jitviewer.display import CodeRepr, CodeReprNoFile -import _jitviewer - -CUTOFF = 30 - -class CannotFindFile(Exception): - pass - -class DummyFunc(object): - def repr(self): - return '???' - -class Server(object): - def __init__(self, filename, storage): - self.filename = filename - self.storage = storage - - def index(self): - all = flask.request.args.get('all', None) - loops = [] - for index, loop in enumerate(self.storage.loops): - is_entry = False - try: - start, stop = loop.comment.find('('), loop.comment.rfind(')') - name = loop.comment[start + 1:stop] - func = FunctionHtml.from_operations(loop.operations, self.storage, - limit=1, - inputargs=loop.inputargs, - loopname=name) - except CannotFindFile: - func = DummyFunc() - func.count = getattr(loop, 'count', '?') - loops.append((is_entry, index, func)) - loops.sort(lambda a, b: cmp(b[2].count, a[2].count)) - if len(loops) > CUTOFF: - extra_data = "Show all (%d) loops" % len(loops) - else: - extra_data = "" - if not all: - loops = loops[:CUTOFF] - - qt_workaround = ('Qt/4.7.2' in flask.request.user_agent.string) - return flask.render_template('index.html', loops=loops, - filename=self.filename, - qt_workaround=qt_workaround, - extra_data=extra_data) - - def loop(self): - no = int(flask.request.args.get('no', '0')) - orig_loop = self.storage.loops[no] - if hasattr(orig_loop, 'force_asm'): - orig_loop.force_asm() - ops = adjust_bridges(orig_loop, flask.request.args) - loop = FunctionHtml.from_operations(ops, self.storage, - inputargs=orig_loop.inputargs) - path = flask.request.args.get('path', '').split(',') - if path: - up = '"' + ','.join(path[:-1]) + '"' - else: - up = '""' - callstack = [] - path_so_far = [] - for e in path: - if e: - callstack.append((','.join(path_so_far), - '%s in %s at %d' % (loop.name, - loop.filename, - loop.startlineno))) - loop = loop.chunks[int(e)] - path_so_far.append(e) - callstack.append((','.join(path_so_far), '%s in %s at %d' % (loop.name, - loop.filename, loop.startlineno))) - - if not loop.has_valid_code() or loop.filename is None: - startline = 0 - source = CodeReprNoFile(loop) - else: - startline, endline = loop.linerange - code = self.storage.load_code(loop.filename)[(loop.startlineno, - loop.name)] - if code.co_name == '<module>' and code.co_firstlineno == 1: - try: - with open(code.co_filename) as f: - source = CodeRepr(f.read(), code, loop) - except (IOError, OSError): - source = CodeReprNoFile(loop) - else: - #try: - source = CodeRepr(inspect.getsource(code), code, loop) - #except: - # source = CodeReprNoFile(loop) - d = {'html': flask.render_template('loop.html', - source=source, - current_loop=no, - upper_path=up, - show_upper_path=bool(path)), - 'scrollto': startline, - 'callstack': callstack} - return flask.jsonify(d) - - -class OverrideFlask(flask.Flask): - root_path = property(lambda self: self._root_path, lambda *args: None) - - def __init__(self, *args, **kwargs): - self._root_path = kwargs.pop('root_path') - self.servers = [] - self.evil_monkeypatch() - flask.Flask.__init__(self, *args, **kwargs) - - def evil_monkeypatch(self): - """ - Evil way to fish the server started by flask, necessary to be able to shut - it down cleanly.""" - from SocketServer import BaseServer - orig___init__ = BaseServer.__init__ - def __init__(self2, *args, **kwds): - self.servers.append(self2) - orig___init__(self2, *args, **kwds) - BaseServer.__init__ = __init__ - -def main(): - PATH = os.path.dirname(os.path.dirname((_jitviewer.__file__))) - if not '__pypy__' in sys.builtin_module_names: - print "Please run it using pypy-c" - sys.exit(1) - # - server_mode = True - if '--qt' in sys.argv: - server_mode = False - sys.argv.remove('--qt') - # - if len(sys.argv) != 2 and len(sys.argv) != 3: - print __doc__ - sys.exit(1) - filename = sys.argv[1] - extra_path = os.path.dirname(filename) - if len(sys.argv) != 3: - port = 5000 - else: - port = int(sys.argv[2]) - storage = LoopStorage(extra_path) - log, loops = import_log(filename, ParserWithHtmlRepr) - parse_log_counts(extract_category(log, 'jit-backend-count'), loops) - storage.reconnect_loops(loops) - app = OverrideFlask('jitviewer', root_path=PATH) - server = Server(filename, storage) - app.debug = True - app.route('/')(server.index) - app.route('/loop')(server.loop) - def run(): - app.run(use_reloader=False, host='0.0.0.0', port=port) - - if server_mode: - run() - else: - url = "http://localhost:%d/" % port - run_server_and_browser(app, run, url, filename) - -def run_server_and_browser(app, run, url, filename): - try: - # start the HTTP server in another thread - th = threading.Thread(target=run) - th.start() - # - # start the webkit browser in the main thread (actually, it's a subprocess) - time.sleep(0.5) # give the server some time to start - start_browser(url, filename) - finally: - # shutdown the HTPP server and wait until it completes - app.servers[0].shutdown() - th.join() - -def start_browser(url, filename): - import subprocess - qwebview_py = os.path.join(os.path.dirname(__file__), 'qwebview.py') - title = "jitviewer: " + filename - try: - return subprocess.check_call(['/usr/bin/python', qwebview_py, url, title]) - except Exception, e: - print 'Cannot start the builtin browser: %s' % e - print "Please point your browser to: %s" % url - try: - raw_input("Press enter to quit and kill the server") - except KeyboardInterrupt: - pass - -if __name__ == '__main__': - main() +from _jitviewer.app import main +main() From noreply at buildbot.pypy.org Fri Jan 27 11:50:45 2012 From: noreply at buildbot.pypy.org (RonnyPfannschmidt) Date: Fri, 27 Jan 2012 11:50:45 +0100 (CET) Subject: [pypy-commit] jitviewer default: some additions to .hgignore Message-ID: <20120127105045.F1CCD82114@wyvern.cs.uni-duesseldorf.de> Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de> Branch: Changeset: r187:5b778b5ea015 Date: 2012-01-27 11:50 +0100 http://bitbucket.org/pypy/jitviewer/changeset/5b778b5ea015/ Log: some additions to .hgignore diff --git a/.hgignore b/.hgignore --- a/.hgignore +++ b/.hgignore @@ -1,3 +1,6 @@ syntax: glob *.pyc +build +dist +*.egg-info From noreply at buildbot.pypy.org Fri Jan 27 12:02:59 2012 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 27 Jan 2012 12:02:59 +0100 (CET) Subject: [pypy-commit] pypy numppy-flatitter: finish the test Message-ID: <20120127110259.AD8AB82114@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: numppy-flatitter Changeset: r51847:e0d5bd060533 Date: 2012-01-27 12:32 +0200 http://bitbucket.org/pypy/pypy/changeset/e0d5bd060533/ Log: finish the test diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py --- a/pypy/module/micronumpy/test/test_zjit.py +++ b/pypy/module/micronumpy/test/test_zjit.py @@ -373,12 +373,17 @@ return ''' a = |30| b = flat(a) - b -> 3 + c = b + a + c -> 3 ''' def test_flat_iter(self): result = self.run("flat_iter") - assert result == 3 + assert result == 6 + self.check_simple_loop({'getinteriorfield_raw': 2, 'float_add': 1, + 'setinteriorfield_raw': 1, 'int_add': 3, + 'int_ge': 1, 'guard_false': 1, + 'arraylen_gc': 1}) def define_flat_getitem(): return ''' From noreply at buildbot.pypy.org Fri Jan 27 12:03:00 2012 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 27 Jan 2012 12:03:00 +0100 (CET) Subject: [pypy-commit] pypy numpy-back-to-applevel: implement take for test_compile, also be more explicit about wrapping Message-ID: <20120127110300.F07F282114@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: numpy-back-to-applevel Changeset: r51848:2ba6d0106f54 Date: 2012-01-27 13:02 +0200 http://bitbucket.org/pypy/pypy/changeset/2ba6d0106f54/ Log: implement take for test_compile, also be more explicit about wrapping diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py --- a/pypy/module/micronumpy/compile.py +++ b/pypy/module/micronumpy/compile.py @@ -33,6 +33,7 @@ pass SINGLE_ARG_FUNCTIONS = ["sum", "prod", "max", "min", "all", "any", "unegative"] +TWO_ARG_FUNCTIONS = ['take'] class FakeSpace(object): w_ValueError = None @@ -372,12 +373,12 @@ for arg in self.args])) def execute(self, interp): + arr = self.args[0].execute(interp) + if not isinstance(arr, BaseArray): + raise ArgumentNotAnArray if self.name in SINGLE_ARG_FUNCTIONS: if len(self.args) != 1 and self.name != 'sum': raise ArgumentMismatch - arr = self.args[0].execute(interp) - if not isinstance(arr, BaseArray): - raise ArgumentNotAnArray if self.name == "sum": if len(self.args)>1: w_res = arr.descr_sum(interp.space, @@ -399,19 +400,27 @@ w_res = neg.call(interp.space, [arr]) else: assert False # unreachable code - if isinstance(w_res, BaseArray): - return w_res - if isinstance(w_res, FloatObject): - dtype = get_dtype_cache(interp.space).w_float64dtype - elif isinstance(w_res, BoolObject): - dtype = get_dtype_cache(interp.space).w_booldtype - elif isinstance(w_res, interp_boxes.W_GenericBox): - dtype = w_res.get_dtype(interp.space) + elif self.name in TWO_ARG_FUNCTIONS: + arg = self.args[1].execute(interp) + if not isinstance(arg, BaseArray): + raise ArgumentNotAnArray + if self.name == 'take': + w_res = arr.descr_take(interp.space, arg) else: - dtype = None - return scalar_w(interp.space, dtype, w_res) + assert False # unreachable else: raise WrongFunctionName + if isinstance(w_res, BaseArray): + return w_res + if isinstance(w_res, FloatObject): + dtype = get_dtype_cache(interp.space).w_float64dtype + elif isinstance(w_res, BoolObject): + dtype = get_dtype_cache(interp.space).w_booldtype + elif isinstance(w_res, interp_boxes.W_GenericBox): + dtype = w_res.get_dtype(interp.space) + else: + dtype = None + return scalar_w(interp.space, dtype, w_res) _REGEXES = [ ('-?[\d\.]+', 'number'), diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -61,6 +61,11 @@ reds=['idx', 'idxi', 'frame', 'arr'], name='numpy_filterset', ) +take_driver = jit.JitDriver( + greens=['shapelen', 'sig'], + reds=['index_i', 'res_i', 'concr'], + name='numpy_take', +) class BaseArray(Wrappable): _attrs_ = ["invalidates", "shape", 'size'] @@ -611,9 +616,14 @@ res_i = res.create_iter() longdtype = interp_dtype.get_dtype_cache(space).w_longdtype shapelen = len(index.shape) + sig = concr.find_sig() while not index_i.done(): + take_driver.jit_merge_point(index_i=index_i, + res_i=res_i, concr=concr, + shapelen=shapelen, sig=sig) # XXX jitdriver + test_zjit - w_item = index.getitem(index_i.offset).convert_to(longdtype) + w_item = index.getitem(index_i.offset).convert_to(longdtype).item( + space) res.setitem(res_i.offset, concr.descr_getitem(space, w_item)) index_i = index_i.next(shapelen) res_i = res_i.next(shapelen) diff --git a/pypy/module/micronumpy/test/test_compile.py b/pypy/module/micronumpy/test/test_compile.py --- a/pypy/module/micronumpy/test/test_compile.py +++ b/pypy/module/micronumpy/test/test_compile.py @@ -245,3 +245,11 @@ a -> 3 """) assert interp.results[0].value == 11 + + def test_take(self): + interp = self.run(""" + a = |10| + b = take(a, [1, 1, 3, 2]) + b -> 2 + """) + assert interp.results[0].value == 3 From noreply at buildbot.pypy.org Fri Jan 27 12:32:05 2012 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 27 Jan 2012 12:32:05 +0100 (CET) Subject: [pypy-commit] extradoc extradoc: Add more thinking. Message-ID: <20120127113205.2D36982114@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: extradoc Changeset: r4058:f2a69fdb7e80 Date: 2012-01-27 12:31 +0100 http://bitbucket.org/pypy/extradoc/changeset/f2a69fdb7e80/ Log: Add more thinking. diff --git a/planning/stm.txt b/planning/stm.txt --- a/planning/stm.txt +++ b/planning/stm.txt @@ -150,6 +150,120 @@ 'PyFrame.fastlocals_w': it should also be known to always be a localobj. +Local collections +----------------- + +If the nursery fills up too much during a transaction, it needs to be +locally collected. This is supposed to be a generally rare occurrance. +Unlike end-of-transaction collections, we need to have the stack roots +of the current transaction. Because such collections are more rare than +in previous GCs, we could use for performance a completely different +approach: conservatively scan the stack, finding everything that looks +like a pointer to an object in the nursery; mark these objects as roots; +and do a local collection from there. We need either a non-moving GC or +at least to pin the potential roots. Pinning is better in the sense +that it should ideally pin a small number of objects, and all other +objects can move away; this would free most of the nursery again. +Afterwards we can still use a bump-pointer allocation technique, to +allocate within each area between the pinned objects. The objects are +pinned just for one local collection, which means that number of such +pinned objects should remain roughly constant as time passes. + +The local collection is also a good time to compress the local list of +all global reads done --- "compress" in the sense of removing +duplicates. + + +Global collections +------------------ + +We will sometimes need to do a "major" collection, called global +collection here. The issue with it is that there might be live +references to global objects in the local objects of any thread. The +problem becomes even harder as some threads may be currently blocked in +some system call. As an intermediate solution that should work well +enough, we could try to acquire a lock for every thread, a kind of LIL +(local interpreter lock). Every thread releases its LIL around +potentially-blocking system calls. At the end of a transaction and +maybe once per local collection, we also do the equivalent of a +release-and-require-the-LIL. + +The major collection could be orchestrated by either the thread that +noticed one should start, or by its own thread. We first acquire all +the LILs, and for every LIL, we ask the corresponding thread to do a +local marking, starting from their own stacks. Out of this, we obtain a +list of global objects. + +Then we can resume running the threads while at the same time doing a +mark-n-sweep collection of the global objects. There is never any +pointer from a global object to a local object, but some global objects +are duplicated in one or several local nurseries. To simplify, these +duplicates should be considered as additional roots for local marking, +and the original objects should be additional roots for global marking. +At some point we might figure out a way to allow duplicated objects to +be freed too. + +The global objects are read-only, at least if there is no commit. If we +don't want to block the other threads we need support for detecting +commit-time concurrent writes. Alternatively, we can ask the threads to +do all together a parallel global marking; this would have a +stop-the-world effect, but require no concurrency detection mechanism. + +Note: standard terminology: + +* Concurrency: there is one thread that does something GC-related, + like scan the heap, and at the same time another thread changes + some object from the heap. + +* Parallelism: there are multiple threads all going something GC-related, + like all scan the heap together. + + +When not running transactively +------------------------------ + +The above describes the mode during which there is a main thread blocked +in transaction.run(). The other mode is mostly that of "start-up", +before we call transaction.run(). Of course no STM is needed in that +mode, but it's still running the same STM-enabled interpreter. We need +to figure out how to tweak the above concepts for that mode. + +We can probably abuse the notion of nursery above, by running with one +nursery (corresponding to the only thread running, the main thread). We +would need to do collections that are some intermediate between "local +collections" and "end-of-transaction collections". Likely, a scheme +that might work would be similar to local collections (with some pinned +objects) but where surviving non-pinned objects are moved to become +global objects. + +This needs a bit more thinking: the issue is that when transaction.run() +is called, we can try to do such a collection, but what about the pinned +objects? + +Some intermediate solution would be to let this mode be rather slow: +have only global objects, and have the stm_write barrier of 'obj' return +'obj'. Do only global collections. Allocation would allocate +immediately a global object, mostly without being able to benefit from +bump-pointer allocation. + + +Pointer equality +---------------- + +Another (traditionally messy) issue is that by having several copies of +the same object, we need to take care of all pointer comparisons too. +This is all llops of the form ``ptr_eq(x, y)`` or ``ptr_ne(x, y)``. + +If we know statically that both copies are local copies, then we can +just compare the pointers. Otherwise we need to check their GC_GLOBAL +and GC_WAS_COPIED flag, and potentially if they both have GC_WAS_COPIED +but only one of them as GC_GLOBAL, we need to check in local dictionary +if they map to each other. And we need to take care of the cases of +NULL pointers. + + + + notes ----- From noreply at buildbot.pypy.org Fri Jan 27 12:42:06 2012 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 27 Jan 2012 12:42:06 +0100 (CET) Subject: [pypy-commit] pypy numpy-back-to-applevel: write a jitdriver for take Message-ID: <20120127114206.1858782114@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: numpy-back-to-applevel Changeset: r51849:e656f86d3017 Date: 2012-01-27 13:41 +0200 http://bitbucket.org/pypy/pypy/changeset/e656f86d3017/ Log: write a jitdriver for take diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py --- a/pypy/module/micronumpy/compile.py +++ b/pypy/module/micronumpy/compile.py @@ -55,6 +55,9 @@ """NOT_RPYTHON""" self.fromcache = InternalSpaceCache(self).getorbuild + def _freeze_(self): + return True + def issequence_w(self, w_obj): return isinstance(w_obj, ListObject) or isinstance(w_obj, W_NDimArray) @@ -146,6 +149,9 @@ def allocate_instance(self, klass, w_subtype): return instantiate(klass) + def newtuple(self, list_w): + raise ValueError + def len_w(self, w_obj): if isinstance(w_obj, ListObject): return len(w_obj.items) diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -63,7 +63,7 @@ ) take_driver = jit.JitDriver( greens=['shapelen', 'sig'], - reds=['index_i', 'res_i', 'concr'], + reds=['index_i', 'res_i', 'concr', 'index', 'res'], name='numpy_take', ) @@ -614,21 +614,25 @@ size *= elem res = W_NDimArray(size, res_shape[:], concr.dtype, concr.order) res_i = res.create_iter() - longdtype = interp_dtype.get_dtype_cache(space).w_longdtype shapelen = len(index.shape) sig = concr.find_sig() while not index_i.done(): - take_driver.jit_merge_point(index_i=index_i, + take_driver.jit_merge_point(index_i=index_i, index=index, res_i=res_i, concr=concr, + res=res, shapelen=shapelen, sig=sig) - # XXX jitdriver + test_zjit - w_item = index.getitem(index_i.offset).convert_to(longdtype).item( - space) + w_item = index._getitem_long(space, index_i.offset) res.setitem(res_i.offset, concr.descr_getitem(space, w_item)) index_i = index_i.next(shapelen) res_i = res_i.next(shapelen) return res + def _getitem_long(self, space, offset): + # an obscure hack to not have longdtype inside a jitted loop + longdtype = interp_dtype.get_dtype_cache(space).w_longdtype + return self.getitem(offset).convert_to(longdtype).item( + space) + def descr_item(self, space, w_arg=None): if space.is_w(w_arg, space.w_None): if not isinstance(self, Scalar): diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py --- a/pypy/module/micronumpy/test/test_zjit.py +++ b/pypy/module/micronumpy/test/test_zjit.py @@ -287,6 +287,27 @@ 'jump': 1, 'arraylen_gc': 1}) + def define_take(): + return """ + a = |10| + b = take(a, [1, 1, 3, 2]) + b -> 2 + """ + + def test_take(self): + result = self.run("take") + assert result == 3 + self.check_simple_loop({'getinteriorfield_raw': 2, + 'cast_float_to_int': 1, + 'int_lt': 1, + 'int_ge': 2, + 'guard_false': 3, + 'setinteriorfield_raw': 1, + 'int_mul': 1, + 'int_add': 3, + 'jump': 1, + 'arraylen_gc': 2}) + def define_multidim(): return """ a = [[1, 2], [3, 4], [5, 6], [7, 8], [9, 10]] From noreply at buildbot.pypy.org Fri Jan 27 13:00:36 2012 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 27 Jan 2012 13:00:36 +0100 (CET) Subject: [pypy-commit] pypy default: Bah, Windows fails on another place that raises the same message. Message-ID: <20120127120036.6786A82114@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: Changeset: r51850:4451c9eca510 Date: 2012-01-27 13:00 +0100 http://bitbucket.org/pypy/pypy/changeset/4451c9eca510/ Log: Bah, Windows fails on another place that raises the same message. diff --git a/pypy/rlib/objectmodel.py b/pypy/rlib/objectmodel.py --- a/pypy/rlib/objectmodel.py +++ b/pypy/rlib/objectmodel.py @@ -130,7 +130,7 @@ if self is other: return 0 else: - raise TypeError("Symbolics can not be compared! (%r, %r)" + raise TypeError("Symbolics cannot be compared! (%r, %r)" % (self, other)) def __hash__(self): diff --git a/pypy/rpython/lltypesystem/llmemory.py b/pypy/rpython/lltypesystem/llmemory.py --- a/pypy/rpython/lltypesystem/llmemory.py +++ b/pypy/rpython/lltypesystem/llmemory.py @@ -32,7 +32,8 @@ self.known_nonneg()): return True else: - raise TypeError("Symbolics can not be compared!") + raise TypeError("Symbolics cannot be compared! (%r, %r)" + % (self, other)) def __lt__(self, other): return not self.__ge__(other) From noreply at buildbot.pypy.org Fri Jan 27 13:21:57 2012 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 27 Jan 2012 13:21:57 +0100 (CET) Subject: [pypy-commit] pypy numpy-back-to-applevel: increase test coverage Message-ID: <20120127122157.2531282114@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: numpy-back-to-applevel Changeset: r51851:b8cb7e93e732 Date: 2012-01-27 14:21 +0200 http://bitbucket.org/pypy/pypy/changeset/b8cb7e93e732/ Log: increase test coverage diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -184,7 +184,6 @@ assert _to_coords(-1, 'F') == [1, 2, 3] assert _to_coords(5, 'F') == [1, 2, 0] assert _to_coords(13, 'F') == [1, 0, 2] - class AppTestNumArray(BaseNumpyAppTest): def test_ndarray(self): @@ -273,6 +272,7 @@ b = a[::2] c = b.copy() assert (c == b).all() + assert ((a + a).copy() == (a + a)).all() a = arange(15).reshape(5,3) b = a.copy() @@ -1622,3 +1622,24 @@ a = arange(0, 0.8, 0.1) assert len(a) == 8 assert arange(False, True, True).dtype is dtype(int) + +from pypy.module.micronumpy.appbridge import get_appbridge_cache + +class AppTestRepr(BaseNumpyAppTest): + def setup_class(cls): + BaseNumpyAppTest.setup_class.im_func(cls) + cache = get_appbridge_cache(cls.space) + cls.old_array_repr = cache.w_array_repr + cls.old_array_str = cache.w_array_str + cache.w_array_str = None + cache.w_array_repr = None + + def test_repr_str(self): + from _numpypy import array + assert repr(array([1, 2, 3])) == 'array([1, 2, 3])' + assert str(array([1, 2, 3])) == 'array([1, 2, 3])' + + def teardown_class(cls): + cache = get_appbridge_cache(cls.space) + cache.w_array_repr = cls.old_array_repr + cache.w_array_str = cls.old_array_str From noreply at buildbot.pypy.org Fri Jan 27 14:20:16 2012 From: noreply at buildbot.pypy.org (mattip) Date: Fri, 27 Jan 2012 14:20:16 +0100 (CET) Subject: [pypy-commit] pypy numppy-flatitter: add jit_merge_point s, add tests for them, tests fail Message-ID: <20120127132016.5AC7882114@wyvern.cs.uni-duesseldorf.de> Author: mattip Branch: numppy-flatitter Changeset: r51852:ef26cbf01db4 Date: 2012-01-27 15:19 +0200 http://bitbucket.org/pypy/pypy/changeset/ef26cbf01db4/ Log: add jit_merge_point s, add tests for them, tests fail diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -59,6 +59,18 @@ name='numpy_filterset', ) +flat_get_driver = jit.JitDriver( + greens=['shapelen', 'base'], + reds=['step', 'ri', 'basei', 'res'], + name='numpy_flatget', +) + +flat_set_driver = jit.JitDriver( + greens=['shapelen', 'base'], + reds=['step', 'ai', 'lngth', 'arr', 'basei'], + name='numpy_flatset', +) + def _find_shape_and_elems(space, w_iterable): shape = [space.len_w(w_iterable)] batch = space.listview(w_iterable) @@ -1482,26 +1494,34 @@ space.wrap(self.index)) return space.newtuple([space.wrap(c) for c in coords]) + @jit.unroll_safe def descr_getitem(self, space, w_idx): if not (space.isinstance_w(w_idx, space.w_int) or space.isinstance_w(w_idx, space.w_slice)): raise OperationError(space.w_IndexError, space.wrap('unsupported iterator index')) - start, stop, step, lngth = space.decode_index4(w_idx, self.base.size) + base = self.base + start, stop, step, lngth = space.decode_index4(w_idx, base.size) # setslice would have been better, but flat[u:v] for arbitrary # shapes of array a cannot be represented as a[x1:x2, y1:y2] - basei = ViewIterator(self.base.start, self.base.strides, - self.base.backstrides,self.base.shape) - shapelen = len(self.base.shape) + basei = ViewIterator(base.start, base.strides, + base.backstrides,base.shape) + shapelen = len(base.shape) basei = basei.next_skip_x(shapelen, start) if lngth <2: - return self.base.getitem(basei.offset) + return base.getitem(basei.offset) ri = ArrayIterator(lngth) - res = W_NDimArray(lngth, [lngth], self.base.dtype, - self.base.order) + res = W_NDimArray(lngth, [lngth], base.dtype, + base.order) while not ri.done(): - # TODO: add a jit_merge_point? - w_val = self.base.getitem(basei.offset) + flat_get_driver.jit_merge_point(shapelen=shapelen, + base=base, + basei=basei, + step=step, + res=res, + ri=ri, + ) + w_val = base.getitem(basei.offset) res.setitem(ri.offset,w_val) basei = basei.next_skip_x(shapelen, step) ri = ri.next(shapelen) @@ -1512,20 +1532,30 @@ space.isinstance_w(w_idx, space.w_slice)): raise OperationError(space.w_IndexError, space.wrap('unsupported iterator index')) - start, stop, step, lngth = space.decode_index4(w_idx, self.base.size) + base = self.base + start, stop, step, lngth = space.decode_index4(w_idx, base.size) arr = convert_to_array(space, w_value) ai = 0 - basei = ViewIterator(self.base.start, self.base.strides, - self.base.backstrides,self.base.shape) - shapelen = len(self.base.shape) + basei = ViewIterator(base.start, base.strides, + base.backstrides,base.shape) + shapelen = len(base.shape) basei = basei.next_skip_x(shapelen, start) - for i in range(lngth): + while lngth > 0: + flat_set_driver.jit_merge_point(shapelen=shapelen, + basei=basei, + base=base, + step=step, + arr=arr, + ai=ai, + lngth=lngth, + ) # TODO: add jit_merge_point? - v = arr.getitem(ai).convert_to(self.base.dtype) - self.base.setitem(basei.offset, v) + v = arr.getitem(ai).convert_to(base.dtype) + base.setitem(basei.offset, v) # need to repeat input values until all assignments are done ai = (ai + 1) % arr.size basei = basei.next_skip_x(shapelen, step) + lngth -= 1 def create_sig(self): return signature.FlatSignature(self.base.dtype) diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py --- a/pypy/module/micronumpy/test/test_zjit.py +++ b/pypy/module/micronumpy/test/test_zjit.py @@ -389,13 +389,28 @@ return ''' a = |30| b = flat(a) - b -> 6 + b -> 4: -> 6 ''' def test_flat_getitem(self): result = self.run("flat_getitem") - assert result == 6.0 - self.check_trace_count(0) + assert result == 10.0 + #self.check_trace_count(1) + #self.check_simple_loop({}) + + def define_flat_setitem(): + return ''' + a = |30| + b = flat(a) + b -> 4: = a->:26 + a -> 5 + ''' + + def test_flat_setitem(self): + result = self.run("flat_setitem") + assert result == 1.0 + #self.check_trace_count(1) + #self.check_simple_loop({}) class TestNumpyOld(LLJitMixin): def setup_class(cls): From noreply at buildbot.pypy.org Fri Jan 27 19:23:31 2012 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 27 Jan 2012 19:23:31 +0100 (CET) Subject: [pypy-commit] pypy numppy-flatitter: fix tests Message-ID: <20120127182331.A5C5882114@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: numppy-flatitter Changeset: r51853:2adecee1eb73 Date: 2012-01-27 20:23 +0200 http://bitbucket.org/pypy/pypy/changeset/2adecee1eb73/ Log: fix tests diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py --- a/pypy/module/micronumpy/test/test_zjit.py +++ b/pypy/module/micronumpy/test/test_zjit.py @@ -383,7 +383,7 @@ self.check_simple_loop({'getinteriorfield_raw': 2, 'float_add': 1, 'setinteriorfield_raw': 1, 'int_add': 3, 'int_ge': 1, 'guard_false': 1, - 'arraylen_gc': 1}) + 'arraylen_gc': 1, 'jump': 1}) def define_flat_getitem(): return ''' From noreply at buildbot.pypy.org Fri Jan 27 19:49:05 2012 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 27 Jan 2012 19:49:05 +0100 (CET) Subject: [pypy-commit] pypy numppy-flatitter: fix tests Message-ID: <20120127184905.4040A82114@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: numppy-flatitter Changeset: r51854:d4ba294094ac Date: 2012-01-27 20:48 +0200 http://bitbucket.org/pypy/pypy/changeset/d4ba294094ac/ Log: fix tests diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -1549,7 +1549,6 @@ ai=ai, lngth=lngth, ) - # TODO: add jit_merge_point? v = arr.getitem(ai).convert_to(base.dtype) base.setitem(basei.offset, v) # need to repeat input values until all assignments are done diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py --- a/pypy/module/micronumpy/test/test_zjit.py +++ b/pypy/module/micronumpy/test/test_zjit.py @@ -380,6 +380,7 @@ def test_flat_iter(self): result = self.run("flat_iter") assert result == 6 + self.check_trace_count(1) self.check_simple_loop({'getinteriorfield_raw': 2, 'float_add': 1, 'setinteriorfield_raw': 1, 'int_add': 3, 'int_ge': 1, 'guard_false': 1, @@ -395,22 +396,44 @@ def test_flat_getitem(self): result = self.run("flat_getitem") assert result == 10.0 - #self.check_trace_count(1) - #self.check_simple_loop({}) + self.check_trace_count(1) + self.check_simple_loop({'getinteriorfield_raw': 1, + 'setinteriorfield_raw': 1, + 'int_lt': 1, + 'int_ge': 1, + 'int_add': 3, + 'guard_true': 1, + 'guard_false': 1, + 'arraylen_gc': 2, + 'jump': 1}) def define_flat_setitem(): return ''' a = |30| b = flat(a) - b -> 4: = a->:26 + b[4:] = a->:26 a -> 5 ''' def test_flat_setitem(self): result = self.run("flat_setitem") assert result == 1.0 - #self.check_trace_count(1) - #self.check_simple_loop({}) + self.check_trace_count(1) + # XXX not ideal, but hey, let's ignore it for now + self.check_simple_loop({'getinteriorfield_raw': 1, + 'setinteriorfield_raw': 1, + 'int_lt': 1, + 'int_gt': 1, + 'int_add': 4, + 'guard_true': 2, + 'arraylen_gc': 2, + 'jump': 1, + 'int_sub': 1, + # XXX bad part + 'int_and': 1, + 'int_mod': 1, + 'int_rshift': 1, + }) class TestNumpyOld(LLJitMixin): def setup_class(cls): From noreply at buildbot.pypy.org Fri Jan 27 20:01:11 2012 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 27 Jan 2012 20:01:11 +0100 (CET) Subject: [pypy-commit] pypy numppy-flatitter: a few missing tests Message-ID: <20120127190111.79A1382114@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: numppy-flatitter Changeset: r51855:13420231bfda Date: 2012-01-27 20:55 +0200 http://bitbucket.org/pypy/pypy/changeset/13420231bfda/ Log: a few missing tests diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -1349,7 +1349,7 @@ assert(b[:, 0] == a[0, :]).all() def test_flatiter(self): - from _numpypy import array, flatiter + from _numpypy import array, flatiter, arange a = array([[10, 30], [40, 60]]) f_iter = a.flat assert f_iter.next() == 10 @@ -1362,6 +1362,9 @@ for k in a.flat: s += k assert s == 140 + a = arange(10).reshape(5, 2) + raises(IndexError, 'a.flat[(1, 2)]') + assert a.flat.base is a def test_flatiter_array_conv(self): from _numpypy import array, dot From noreply at buildbot.pypy.org Fri Jan 27 20:01:12 2012 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 27 Jan 2012 20:01:12 +0100 (CET) Subject: [pypy-commit] pypy numppy-flatitter: move around a very useful comment Message-ID: <20120127190112.ACE4382114@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: numppy-flatitter Changeset: r51856:35aaba0342c7 Date: 2012-01-27 20:58 +0200 http://bitbucket.org/pypy/pypy/changeset/35aaba0342c7/ Log: move around a very useful comment diff --git a/pypy/module/micronumpy/interp_iter.py b/pypy/module/micronumpy/interp_iter.py --- a/pypy/module/micronumpy/interp_iter.py +++ b/pypy/module/micronumpy/interp_iter.py @@ -4,6 +4,49 @@ from pypy.module.micronumpy.strides import calculate_broadcast_strides,\ calculate_slice_strides +""" This is a mini-tutorial on iterators, strides, and +memory layout. It assumes you are familiar with the terms, see +http://docs.scipy.org/doc/numpy/reference/arrays.ndarray.html +for a more gentle introduction. + +Given an array x: x.shape == [5,6], + +At which byte in x.data does the item x[3,4] begin? +if x.strides==[1,5]: + pData = x.pData + (x.start + 3*1 + 4*5)*sizeof(x.pData[0]) + pData = x.pData + (x.start + 24) * sizeof(x.pData[0]) +so the offset of the element is 24 elements after the first + +What is the next element in x after coordinates [3,4]? +if x.order =='C': + next == [3,5] => offset is 28 +if x.order =='F': + next == [4,4] => offset is 24 +so for the strides [1,5] x is 'F' contiguous +likewise, for the strides [6,1] x would be 'C' contiguous. + +Iterators have an internal representation of the current coordinates +(indices), the array, strides, and backstrides. A short digression to +explain backstrides: what is the coordinate and offset after [3,5] in +the example above? +if x.order == 'C': + next == [4,0] => offset is 4 +if x.order == 'F': + next == [4,5] => offset is 25 +Note that in 'C' order we stepped BACKWARDS 24 while 'overflowing' a +shape dimension + which is back 25 and forward 1, + which is x.strides[1] * (x.shape[1] - 1) + x.strides[0] +so if we precalculate the overflow backstride as +[x.strides[i] * (x.shape[i] - 1) for i in range(len(x.shape))] +we can go faster. +All the calculations happen in next() + +next_step_x() tries to do the iteration for a number of steps at once, +but then we cannot gaurentee that we only overflow one single shape +dimension, perhaps we could overflow times in one big step. +""" + # structures to describe slicing class Chunk(object): diff --git a/pypy/module/micronumpy/test/test_iter.py b/pypy/module/micronumpy/test/test_iter.py --- a/pypy/module/micronumpy/test/test_iter.py +++ b/pypy/module/micronumpy/test/test_iter.py @@ -1,49 +1,5 @@ -import py from pypy.module.micronumpy.interp_iter import ViewIterator -# This is both a test and a mini-tutorial on iterators, strides, and -# memory layout. It assumes you are familiar with the terms, see -# http://docs.scipy.org/doc/numpy/reference/arrays.ndarray.html -# for a more gentle introduction. -# -# Given an array x: x.shape == [5,6], -# -# At which byte in x.data does the item x[3,4] begin? -# if x.strides==[1,5]: -# pData = x.pData + (x.start + 3*1 + 4*5)*sizeof(x.pData[0]) -# pData = x.pData + (x.start + 24) * sizeof(x.pData[0]) -# so the offset of the element is 24 elements after the first -# -# What is the next element in x after coordinates [3,4]? -# if x.order =='C': -# next == [3,5] => offset is 28 -# if x.order =='F': -# next == [4,4] => offset is 24 -# so for the strides [1,5] x is 'F' contiguous -# likewise, for the strides [6,1] x would be 'C' contiguous. -# -# Iterators have an internal representation of the current coordinates -# (indices), the array, strides, and backstrides. A short digression to -# explain backstrides: what is the coordinate and offset after [3,5] in -# the example above? -# if x.order == 'C': -# next == [4,0] => offset is 4 -# if x.order == 'F': -# next == [4,5] => offset is 25 -# Note that in 'C' order we stepped BACKWARDS 24 while 'overflowing' a -# shape dimension -# which is back 25 and forward 1, -# which is x.strides[1] * (x.shape[1] - 1) + x.strides[0] -# so if we precalculate the overflow backstride as -# [x.strides[i] * (x.shape[i] - 1) for i in range(len(x.shape))] -# we can go faster. -# All the calculations happen in next() -# -# next_step_x() tries to do the iteration for a number of steps at once, -# but then we cannot gaurentee that we only overflow one single shape -# dimension, perhaps we could overflow times in one big step. -# - class TestIterDirect(object): def test_C_viewiterator(self): #Let's get started, simple iteration in C order with From noreply at buildbot.pypy.org Fri Jan 27 20:01:13 2012 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 27 Jan 2012 20:01:13 +0100 (CET) Subject: [pypy-commit] pypy numppy-flatitter: close merged branch Message-ID: <20120127190113.D6DE982114@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: numppy-flatitter Changeset: r51857:81af4f4c5c21 Date: 2012-01-27 20:59 +0200 http://bitbucket.org/pypy/pypy/changeset/81af4f4c5c21/ Log: close merged branch From noreply at buildbot.pypy.org Fri Jan 27 20:01:15 2012 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 27 Jan 2012 20:01:15 +0100 (CET) Subject: [pypy-commit] pypy default: (mattip, fijal reviewing) Merge numppy-flatitter branch (yes, with all the Message-ID: <20120127190115.4D7B082114@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: Changeset: r51858:caa2d47782ba Date: 2012-01-27 21:00 +0200 http://bitbucket.org/pypy/pypy/changeset/caa2d47782ba/ Log: (mattip, fijal reviewing) Merge numppy-flatitter branch (yes, with all the typos). This improves flat iterator and adds __getitem__ and __setitem__ to it. diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py --- a/pypy/module/micronumpy/compile.py +++ b/pypy/module/micronumpy/compile.py @@ -32,7 +32,8 @@ class BadToken(Exception): pass -SINGLE_ARG_FUNCTIONS = ["sum", "prod", "max", "min", "all", "any", "unegative"] +SINGLE_ARG_FUNCTIONS = ["sum", "prod", "max", "min", "all", "any", + "unegative", "flat"] class FakeSpace(object): w_ValueError = None @@ -396,6 +397,8 @@ elif self.name == "unegative": neg = interp_ufuncs.get(interp.space).negative w_res = neg.call(interp.space, [arr]) + elif self.name == "flat": + w_res = arr.descr_get_flatiter(interp.space) else: assert False # unreachable code if isinstance(w_res, BaseArray): diff --git a/pypy/module/micronumpy/interp_iter.py b/pypy/module/micronumpy/interp_iter.py --- a/pypy/module/micronumpy/interp_iter.py +++ b/pypy/module/micronumpy/interp_iter.py @@ -4,6 +4,49 @@ from pypy.module.micronumpy.strides import calculate_broadcast_strides,\ calculate_slice_strides +""" This is a mini-tutorial on iterators, strides, and +memory layout. It assumes you are familiar with the terms, see +http://docs.scipy.org/doc/numpy/reference/arrays.ndarray.html +for a more gentle introduction. + +Given an array x: x.shape == [5,6], + +At which byte in x.data does the item x[3,4] begin? +if x.strides==[1,5]: + pData = x.pData + (x.start + 3*1 + 4*5)*sizeof(x.pData[0]) + pData = x.pData + (x.start + 24) * sizeof(x.pData[0]) +so the offset of the element is 24 elements after the first + +What is the next element in x after coordinates [3,4]? +if x.order =='C': + next == [3,5] => offset is 28 +if x.order =='F': + next == [4,4] => offset is 24 +so for the strides [1,5] x is 'F' contiguous +likewise, for the strides [6,1] x would be 'C' contiguous. + +Iterators have an internal representation of the current coordinates +(indices), the array, strides, and backstrides. A short digression to +explain backstrides: what is the coordinate and offset after [3,5] in +the example above? +if x.order == 'C': + next == [4,0] => offset is 4 +if x.order == 'F': + next == [4,5] => offset is 25 +Note that in 'C' order we stepped BACKWARDS 24 while 'overflowing' a +shape dimension + which is back 25 and forward 1, + which is x.strides[1] * (x.shape[1] - 1) + x.strides[0] +so if we precalculate the overflow backstride as +[x.strides[i] * (x.shape[i] - 1) for i in range(len(x.shape))] +we can go faster. +All the calculations happen in next() + +next_step_x() tries to do the iteration for a number of steps at once, +but then we cannot gaurentee that we only overflow one single shape +dimension, perhaps we could overflow times in one big step. +""" + # structures to describe slicing class Chunk(object): @@ -55,9 +98,9 @@ self.size = size def next(self, shapelen): - return self._next(1) + return self.next_skip_x(1) - def _next(self, ofs): + def next_skip_x(self, ofs): arr = instantiate(ArrayIterator) arr.size = self.size arr.offset = self.offset + ofs @@ -65,7 +108,7 @@ def next_no_increase(self, shapelen): # a hack to make JIT believe this is always virtual - return self._next(0) + return self.next_skip_x(0) def done(self): return self.offset >= self.size @@ -137,6 +180,36 @@ res._done = done return res + @jit.unroll_safe + def next_skip_x(self, shapelen, step): + shapelen = jit.promote(len(self.res_shape)) + offset = self.offset + indices = [0] * shapelen + for i in range(shapelen): + indices[i] = self.indices[i] + done = False + for i in range(shapelen - 1, -1, -1): + if indices[i] < self.res_shape[i] - step: + indices[i] += step + offset += self.strides[i] * step + break + else: + remaining_step = (indices[i] + step) // self.res_shape[i] + this_i_step = step - remaining_step * self.res_shape[i] + offset += self.strides[i] * this_i_step + indices[i] = indices[i] + this_i_step + step = remaining_step + else: + done = True + res = instantiate(ViewIterator) + res.offset = offset + res.indices = indices + res.strides = self.strides + res.backstrides = self.backstrides + res.res_shape = self.res_shape + res._done = done + return res + def apply_transformations(self, arr, transformations): v = BaseIterator.apply_transformations(self, arr, transformations) if len(arr.shape) == 1: diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -4,7 +4,7 @@ from pypy.interpreter.typedef import TypeDef, GetSetProperty from pypy.module.micronumpy import interp_ufuncs, interp_dtype, signature,\ interp_boxes -from pypy.module.micronumpy.strides import calculate_slice_strides +from pypy.module.micronumpy.strides import calculate_slice_strides, to_coords from pypy.rlib import jit from pypy.rpython.lltypesystem import lltype, rffi from pypy.tool.sourcetools import func_with_new_name @@ -59,6 +59,18 @@ name='numpy_filterset', ) +flat_get_driver = jit.JitDriver( + greens=['shapelen', 'base'], + reds=['step', 'ri', 'basei', 'res'], + name='numpy_flatget', +) + +flat_set_driver = jit.JitDriver( + greens=['shapelen', 'base'], + reds=['step', 'ai', 'lngth', 'arr', 'basei'], + name='numpy_flatset', +) + def _find_shape_and_elems(space, w_iterable): shape = [space.len_w(w_iterable)] batch = space.listview(w_iterable) @@ -1455,31 +1467,119 @@ @jit.unroll_safe def __init__(self, arr): arr = arr.get_concrete() - size = 1 - for sh in arr.shape: - size *= sh - self.strides = [arr.strides[-1]] - self.backstrides = [arr.backstrides[-1]] - ViewArray.__init__(self, size, [size], arr.dtype, arr.order, - arr) self.shapelen = len(arr.shape) - self.iter = OneDimIterator(arr.start, self.strides[0], - self.shape[0]) + sig = arr.find_sig() + self.iter = sig.create_frame(arr).get_final_iter() + self.base = arr + self.index = 0 + ViewArray.__init__(self, arr.size, [arr.size], arr.dtype, arr.order, arr) def descr_next(self, space): if self.iter.done(): raise OperationError(space.w_StopIteration, space.w_None) - result = self.getitem(self.iter.offset) + result = self.base.getitem(self.iter.offset) self.iter = self.iter.next(self.shapelen) + self.index += 1 return result def descr_iter(self): return self + def descr_index(self, space): + return space.wrap(self.index) + + def descr_coords(self, space): + coords, step, lngth = to_coords(space, self.base.shape, + self.base.size, self.base.order, + space.wrap(self.index)) + return space.newtuple([space.wrap(c) for c in coords]) + + @jit.unroll_safe + def descr_getitem(self, space, w_idx): + if not (space.isinstance_w(w_idx, space.w_int) or + space.isinstance_w(w_idx, space.w_slice)): + raise OperationError(space.w_IndexError, + space.wrap('unsupported iterator index')) + base = self.base + start, stop, step, lngth = space.decode_index4(w_idx, base.size) + # setslice would have been better, but flat[u:v] for arbitrary + # shapes of array a cannot be represented as a[x1:x2, y1:y2] + basei = ViewIterator(base.start, base.strides, + base.backstrides,base.shape) + shapelen = len(base.shape) + basei = basei.next_skip_x(shapelen, start) + if lngth <2: + return base.getitem(basei.offset) + ri = ArrayIterator(lngth) + res = W_NDimArray(lngth, [lngth], base.dtype, + base.order) + while not ri.done(): + flat_get_driver.jit_merge_point(shapelen=shapelen, + base=base, + basei=basei, + step=step, + res=res, + ri=ri, + ) + w_val = base.getitem(basei.offset) + res.setitem(ri.offset,w_val) + basei = basei.next_skip_x(shapelen, step) + ri = ri.next(shapelen) + return res + + def descr_setitem(self, space, w_idx, w_value): + if not (space.isinstance_w(w_idx, space.w_int) or + space.isinstance_w(w_idx, space.w_slice)): + raise OperationError(space.w_IndexError, + space.wrap('unsupported iterator index')) + base = self.base + start, stop, step, lngth = space.decode_index4(w_idx, base.size) + arr = convert_to_array(space, w_value) + ai = 0 + basei = ViewIterator(base.start, base.strides, + base.backstrides,base.shape) + shapelen = len(base.shape) + basei = basei.next_skip_x(shapelen, start) + while lngth > 0: + flat_set_driver.jit_merge_point(shapelen=shapelen, + basei=basei, + base=base, + step=step, + arr=arr, + ai=ai, + lngth=lngth, + ) + v = arr.getitem(ai).convert_to(base.dtype) + base.setitem(basei.offset, v) + # need to repeat input values until all assignments are done + ai = (ai + 1) % arr.size + basei = basei.next_skip_x(shapelen, step) + lngth -= 1 + + def create_sig(self): + return signature.FlatSignature(self.base.dtype) + + def descr_base(self, space): + return space.wrap(self.base) + W_FlatIterator.typedef = TypeDef( 'flatiter', + #__array__ = #MISSING + __iter__ = interp2app(W_FlatIterator.descr_iter), + __getitem__ = interp2app(W_FlatIterator.descr_getitem), + __setitem__ = interp2app(W_FlatIterator.descr_setitem), + __eq__ = interp2app(BaseArray.descr_eq), + __ne__ = interp2app(BaseArray.descr_ne), + __lt__ = interp2app(BaseArray.descr_lt), + __le__ = interp2app(BaseArray.descr_le), + __gt__ = interp2app(BaseArray.descr_gt), + __ge__ = interp2app(BaseArray.descr_ge), + #__sizeof__ #MISSING + base = GetSetProperty(W_FlatIterator.descr_base), + index = GetSetProperty(W_FlatIterator.descr_index), + coords = GetSetProperty(W_FlatIterator.descr_coords), next = interp2app(W_FlatIterator.descr_next), - __iter__ = interp2app(W_FlatIterator.descr_iter), + ) W_FlatIterator.acceptable_as_base_class = False diff --git a/pypy/module/micronumpy/signature.py b/pypy/module/micronumpy/signature.py --- a/pypy/module/micronumpy/signature.py +++ b/pypy/module/micronumpy/signature.py @@ -224,6 +224,18 @@ return ViewIterator(arr.start, arr.strides, arr.backstrides, arr.shape).apply_transformations(arr, transforms) +class FlatSignature(ViewSignature): + def debug_repr(self): + return 'Flat' + + def allocate_iter(self, arr, transforms): + from pypy.module.micronumpy.interp_numarray import W_FlatIterator + assert isinstance(arr, W_FlatIterator) + return ViewIterator(arr.base.start, arr.base.strides, + arr.base.backstrides, + arr.base.shape).apply_transformations(arr.base, + transforms) + class VirtualSliceSignature(Signature): def __init__(self, child): self.child = child diff --git a/pypy/module/micronumpy/test/test_compile.py b/pypy/module/micronumpy/test/test_compile.py --- a/pypy/module/micronumpy/test/test_compile.py +++ b/pypy/module/micronumpy/test/test_compile.py @@ -245,3 +245,11 @@ a -> 3 """) assert interp.results[0].value == 11 + + def test_flat_iter(self): + interp = self.run(''' + a = |30| + b = flat(a) + b -> 3 + ''') + assert interp.results[0].value == 3 diff --git a/pypy/module/micronumpy/test/test_iter.py b/pypy/module/micronumpy/test/test_iter.py new file mode 100644 --- /dev/null +++ b/pypy/module/micronumpy/test/test_iter.py @@ -0,0 +1,88 @@ +from pypy.module.micronumpy.interp_iter import ViewIterator + +class TestIterDirect(object): + def test_C_viewiterator(self): + #Let's get started, simple iteration in C order with + #contiguous layout => strides[-1] is 1 + start = 0 + shape = [3, 5] + strides = [5, 1] + backstrides = [x * (y - 1) for x,y in zip(strides, shape)] + assert backstrides == [10, 4] + i = ViewIterator(start, strides, backstrides, shape) + i = i.next(2) + i = i.next(2) + i = i.next(2) + assert i.offset == 3 + assert not i.done() + assert i.indices == [0,3] + #cause a dimension overflow + i = i.next(2) + i = i.next(2) + assert i.offset == 5 + assert i.indices == [1,0] + + #Now what happens if the array is transposed? strides[-1] != 1 + # therefore layout is non-contiguous + strides = [1, 3] + backstrides = [x * (y - 1) for x,y in zip(strides, shape)] + assert backstrides == [2, 12] + i = ViewIterator(start, strides, backstrides, shape) + i = i.next(2) + i = i.next(2) + i = i.next(2) + assert i.offset == 9 + assert not i.done() + assert i.indices == [0,3] + #cause a dimension overflow + i = i.next(2) + i = i.next(2) + assert i.offset == 1 + assert i.indices == [1,0] + + def test_C_viewiterator_step(self): + #iteration in C order with #contiguous layout => strides[-1] is 1 + #skip less than the shape + start = 0 + shape = [3, 5] + strides = [5, 1] + backstrides = [x * (y - 1) for x,y in zip(strides, shape)] + assert backstrides == [10, 4] + i = ViewIterator(start, strides, backstrides, shape) + i = i.next_skip_x(2,2) + i = i.next_skip_x(2,2) + i = i.next_skip_x(2,2) + assert i.offset == 6 + assert not i.done() + assert i.indices == [1,1] + #And for some big skips + i = i.next_skip_x(2,5) + assert i.offset == 11 + assert i.indices == [2,1] + i = i.next_skip_x(2,5) + # Note: the offset does not overflow but recycles, + # this is good for broadcast + assert i.offset == 1 + assert i.indices == [0,1] + assert i.done() + + #Now what happens if the array is transposed? strides[-1] != 1 + # therefore layout is non-contiguous + strides = [1, 3] + backstrides = [x * (y - 1) for x,y in zip(strides, shape)] + assert backstrides == [2, 12] + i = ViewIterator(start, strides, backstrides, shape) + i = i.next_skip_x(2,2) + i = i.next_skip_x(2,2) + i = i.next_skip_x(2,2) + assert i.offset == 4 + assert i.indices == [1,1] + assert not i.done() + i = i.next_skip_x(2,5) + assert i.offset == 5 + assert i.indices == [2,1] + assert not i.done() + i = i.next_skip_x(2,5) + assert i.indices == [0,1] + assert i.offset == 3 + assert i.done() diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -301,6 +301,12 @@ for i in xrange(5): assert a[i] == b[i] + def test_getitem_nd(self): + from _numpypy import arange + a = arange(15).reshape(3, 5) + assert a[1, 3] == 8 + assert a.T[1, 2] == 11 + def test_setitem(self): from _numpypy import array a = array(range(5)) @@ -974,7 +980,7 @@ assert debug_repr(a + a) == 'Call2(add, Array, Array)' assert debug_repr(a[::2]) == 'Slice' assert debug_repr(a + 2) == 'Call2(add, Array, Scalar)' - assert debug_repr(a + a.flat) == 'Call2(add, Array, Slice)' + assert debug_repr(a + a.flat) == 'Call2(add, Array, Flat)' assert debug_repr(sin(a)) == 'Call1(sin, Array)' b = a + a @@ -1343,7 +1349,7 @@ assert(b[:, 0] == a[0, :]).all() def test_flatiter(self): - from _numpypy import array, flatiter + from _numpypy import array, flatiter, arange a = array([[10, 30], [40, 60]]) f_iter = a.flat assert f_iter.next() == 10 @@ -1356,6 +1362,9 @@ for k in a.flat: s += k assert s == 140 + a = arange(10).reshape(5, 2) + raises(IndexError, 'a.flat[(1, 2)]') + assert a.flat.base is a def test_flatiter_array_conv(self): from _numpypy import array, dot @@ -1367,6 +1376,75 @@ a = ones((2, 2)) assert list(((a + a).flat)) == [2, 2, 2, 2] + def test_flatiter_getitem(self): + from _numpypy import arange + a = arange(10) + assert a.flat[3] == 3 + assert a[2:].flat[3] == 5 + assert (a + a).flat[3] == 6 + assert a[::2].flat[3] == 6 + assert a.reshape(2,5).flat[3] == 3 + b = a.reshape(2,5).flat + b.next() + b.next() + b.next() + assert b[3] == 3 + assert (b[::3] == [0, 3, 6, 9]).all() + assert (b[2::5] == [2, 7]).all() + assert b[-2] == 8 + raises(IndexError, "b[11]") + raises(IndexError, "b[-11]") + raises(IndexError, 'b[0, 1]') + assert b.index == 3 + assert b.coords == (0,3) + + def test_flatiter_setitem(self): + from _numpypy import arange, array + a = arange(12).reshape(3,4) + b = a.T.flat + b[6::2] = [-1, -2] + assert (a == [[0, 1, -1, 3], [4, 5, 6, -1], [8, 9, -2, 11]]).all() + b[0:2] = [[[100]]] + assert(a[0,0] == 100) + assert(a[1,0] == 100) + raises(IndexError, 'b[array([10, 11])] == [-20, -40]') + + def test_flatiter_ops(self): + from _numpypy import arange, array + a = arange(12).reshape(3,4) + b = a.T.flat + assert (b == [0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11]).all() + assert not (b != [0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11]).any() + assert ((b >= range(12)) == [True, True, True,False, True, True, + False, False, True, False, False, True]).all() + assert ((b < range(12)) != [True, True, True,False, True, True, + False, False, True, False, False, True]).all() + assert ((b <= range(12)) != [False, True, True,False, True, True, + False, False, True, False, False, False]).all() + assert ((b > range(12)) == [False, True, True,False, True, True, + False, False, True, False, False, False]).all() + def test_flatiter_view(self): + from _numpypy import arange + a = arange(10).reshape(5, 2) + #no == yet. + # a[::2].flat == [0, 1, 4, 5, 8, 9] + isequal = True + for y,z in zip(a[::2].flat, [0, 1, 4, 5, 8, 9]): + if y != z: + isequal = False + assert isequal == True + + def test_flatiter_transpose(self): + from _numpypy import arange + a = arange(10).reshape(2,5).T + b = a.flat + assert (b[:5] == [0, 5, 1, 6, 2]).all() + b.next() + b.next() + b.next() + assert b.index == 3 + assert b.coords == (1,1) + def test_slice_copy(self): from _numpypy import zeros a = zeros((10, 10)) diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py --- a/pypy/module/micronumpy/test/test_zjit.py +++ b/pypy/module/micronumpy/test/test_zjit.py @@ -12,7 +12,7 @@ from pypy.module.micronumpy.compile import (FakeSpace, IntObject, Parser, InterpreterState) from pypy.module.micronumpy.interp_numarray import (W_NDimArray, - BaseArray) + BaseArray, W_FlatIterator) from pypy.rlib.nonconst import NonConstant @@ -369,7 +369,71 @@ 'setinteriorfield_raw': 1, 'int_add': 2, 'int_ge': 1, 'guard_false': 1, 'jump': 1, 'arraylen_gc': 1}) + def define_flat_iter(): + return ''' + a = |30| + b = flat(a) + c = b + a + c -> 3 + ''' + def test_flat_iter(self): + result = self.run("flat_iter") + assert result == 6 + self.check_trace_count(1) + self.check_simple_loop({'getinteriorfield_raw': 2, 'float_add': 1, + 'setinteriorfield_raw': 1, 'int_add': 3, + 'int_ge': 1, 'guard_false': 1, + 'arraylen_gc': 1, 'jump': 1}) + + def define_flat_getitem(): + return ''' + a = |30| + b = flat(a) + b -> 4: -> 6 + ''' + + def test_flat_getitem(self): + result = self.run("flat_getitem") + assert result == 10.0 + self.check_trace_count(1) + self.check_simple_loop({'getinteriorfield_raw': 1, + 'setinteriorfield_raw': 1, + 'int_lt': 1, + 'int_ge': 1, + 'int_add': 3, + 'guard_true': 1, + 'guard_false': 1, + 'arraylen_gc': 2, + 'jump': 1}) + + def define_flat_setitem(): + return ''' + a = |30| + b = flat(a) + b[4:] = a->:26 + a -> 5 + ''' + + def test_flat_setitem(self): + result = self.run("flat_setitem") + assert result == 1.0 + self.check_trace_count(1) + # XXX not ideal, but hey, let's ignore it for now + self.check_simple_loop({'getinteriorfield_raw': 1, + 'setinteriorfield_raw': 1, + 'int_lt': 1, + 'int_gt': 1, + 'int_add': 4, + 'guard_true': 2, + 'arraylen_gc': 2, + 'jump': 1, + 'int_sub': 1, + # XXX bad part + 'int_and': 1, + 'int_mod': 1, + 'int_rshift': 1, + }) class TestNumpyOld(LLJitMixin): def setup_class(cls): From noreply at buildbot.pypy.org Fri Jan 27 20:05:48 2012 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 27 Jan 2012 20:05:48 +0100 (CET) Subject: [pypy-commit] pypy numpy-back-to-applevel: remove cruft from type.py Message-ID: <20120127190548.9689482114@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: numpy-back-to-applevel Changeset: r51859:a113c99b0ec2 Date: 2012-01-27 21:05 +0200 http://bitbucket.org/pypy/pypy/changeset/a113c99b0ec2/ Log: remove cruft from type.py diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py --- a/pypy/module/micronumpy/types.py +++ b/pypy/module/micronumpy/types.py @@ -105,9 +105,9 @@ )) def read_bool(self, storage, width, i, offset): - return self.for_computation( + return bool(self.for_computation( libffi.array_getitem(clibffi.cast_type_to_ffitype(self.T), - width, storage, i, offset)) != 0 + width, storage, i, offset))) def store(self, storage, width, i, offset, box): value = self.unbox(box) @@ -182,7 +182,7 @@ return v1 >= v2 def bool(self, v): - return self.for_computation(self.unbox(v)) != 0 + return bool(self.for_computation(self.unbox(v))) @simple_binary_op def max(self, v1, v2): @@ -229,8 +229,6 @@ def default_fromstring(self, space): return self.box(False) - # XXX check rpythonization - @simple_binary_op def bitwise_and(self, v1, v2): return v1 & v2 From noreply at buildbot.pypy.org Fri Jan 27 20:08:18 2012 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 27 Jan 2012 20:08:18 +0100 (CET) Subject: [pypy-commit] pypy numpy-back-to-applevel: remove print Message-ID: <20120127190818.8350782114@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: numpy-back-to-applevel Changeset: r51860:b6f8f63b46ec Date: 2012-01-27 21:07 +0200 http://bitbucket.org/pypy/pypy/changeset/b6f8f63b46ec/ Log: remove print diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -1453,7 +1453,6 @@ from _numpypy import arange assert (arange(3).ravel() == arange(3)).all() assert (arange(6).reshape(2, 3).ravel() == arange(6)).all() - print arange(6).reshape(2, 3).T.ravel() assert (arange(6).reshape(2, 3).T.ravel() == [0, 3, 1, 4, 2, 5]).all() def test_take(self): From noreply at buildbot.pypy.org Fri Jan 27 20:16:54 2012 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Fri, 27 Jan 2012 20:16:54 +0100 (CET) Subject: [pypy-commit] pypy numpy-back-to-applevel: merged default Message-ID: <20120127191654.7C65282114@wyvern.cs.uni-duesseldorf.de> Author: Alex Gaynor <alex.gaynor at gmail.com> Branch: numpy-back-to-applevel Changeset: r51861:4ecd878caff2 Date: 2012-01-27 13:28 -0500 http://bitbucket.org/pypy/pypy/changeset/4ecd878caff2/ Log: merged default diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py --- a/pypy/module/micronumpy/interp_dtype.py +++ b/pypy/module/micronumpy/interp_dtype.py @@ -90,11 +90,8 @@ return space.newtuple([]) def eq(self, space, w_other): - if space.is_w(self, w_other): - return True - if space.isinstance_w(w_other, space.w_str): - return self.name == space.str_w(w_other) - return False + w_other = space.call_function(space.gettypefor(W_Dtype), w_other) + return space.is_w(self, w_other) def descr_eq(self, space, w_other): return space.wrap(self.eq(space, w_other)) diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py --- a/pypy/module/micronumpy/test/test_dtypes.py +++ b/pypy/module/micronumpy/test/test_dtypes.py @@ -9,14 +9,19 @@ assert d.num == 0 assert d.kind == 'b' assert dtype('int8').num == 1 - assert dtype('int8') == 'int8' - assert 'int8' == dtype('int8') - assert dtype('int8') != 3 assert dtype(d) is d assert dtype(None) is dtype(float) assert dtype('int8').name == 'int8' raises(TypeError, dtype, 1042) + def test_dtype_eq(self): + from _numpypy import dtype + + assert dtype("int8") == "int8" + assert "int8" == dtype("int8") + raises(TypeError, lambda: dtype("int8") == 3) + assert dtype(bool) == bool + def test_dtype_with_types(self): from _numpypy import dtype diff --git a/pypy/module/micronumpy/tool/numready/main.py b/pypy/module/micronumpy/tool/numready/main.py --- a/pypy/module/micronumpy/tool/numready/main.py +++ b/pypy/module/micronumpy/tool/numready/main.py @@ -73,9 +73,8 @@ for line in lines: kind, name = line.split(" : ", 1) subitems = None - if kind == KINDS["TYPE"]: - if name in ['ndarray', 'dtype']: - subitems = find_numpy_items(python, modname, name) + if kind == KINDS["TYPE"] and name in SPECIAL_NAMES and attr is None: + subitems = find_numpy_items(python, modname, name) items.add(Item(name, kind, subitems)) return items @@ -89,15 +88,20 @@ l[i].append(lst[k * lgt + i]) return l +SPECIAL_NAMES = ["ndarray", "dtype", "generic"] + def main(argv): cpy_items = find_numpy_items("/usr/bin/python") pypy_items = find_numpy_items(argv[1], "numpypy") all_items = [] - msg = '%d/%d names, %d/%d ndarray attributes, %d/%d dtype attributes' % ( - len(pypy_items), len(cpy_items), len(pypy_items['ndarray'].subitems), - len(cpy_items['ndarray'].subitems), len(pypy_items['dtype'].subitems), - len(cpy_items['dtype'].subitems)) + msg = "{:d}/{:d} names".format(len(pypy_items), len(cpy_items)) + msg += ", ".join( + "{:d}/{:d} {} attributes".format( + len(pypy_items[name].subitems), len(cpy_items[name].subitems), name + ) + for name in SPECIAL_NAMES + ) for item in cpy_items: pypy_exists = item in pypy_items if item.subitems: @@ -114,6 +118,6 @@ with open(argv[2], 'w') as f: f.write(html.encode("utf-8")) else: - with tempfile.NamedTemporaryFile(delete=False) as f: + with tempfile.NamedTemporaryFile(delete=False, suffix=".html") as f: f.write(html.encode("utf-8")) print "Saved in: %s" % f.name diff --git a/pypy/rlib/objectmodel.py b/pypy/rlib/objectmodel.py --- a/pypy/rlib/objectmodel.py +++ b/pypy/rlib/objectmodel.py @@ -130,7 +130,7 @@ if self is other: return 0 else: - raise TypeError("Symbolics can not be compared! (%r, %r)" + raise TypeError("Symbolics cannot be compared! (%r, %r)" % (self, other)) def __hash__(self): diff --git a/pypy/rpython/lltypesystem/llmemory.py b/pypy/rpython/lltypesystem/llmemory.py --- a/pypy/rpython/lltypesystem/llmemory.py +++ b/pypy/rpython/lltypesystem/llmemory.py @@ -32,7 +32,8 @@ self.known_nonneg()): return True else: - raise TypeError("Symbolics can not be compared!") + raise TypeError("Symbolics cannot be compared! (%r, %r)" + % (self, other)) def __lt__(self, other): return not self.__ge__(other) diff --git a/pypy/translator/goal/targetnopstandalone.py b/pypy/translator/goal/targetnopstandalone.py --- a/pypy/translator/goal/targetnopstandalone.py +++ b/pypy/translator/goal/targetnopstandalone.py @@ -7,10 +7,8 @@ actually implementing argv of the executable. """ -import os, sys - -def debug(msg): - os.write(2, "debug: " + msg + '\n') +def debug(msg): + print "debug:", msg # __________ Entry point __________ From noreply at buildbot.pypy.org Fri Jan 27 20:16:55 2012 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Fri, 27 Jan 2012 20:16:55 +0100 (CET) Subject: [pypy-commit] pypy numpy-back-to-applevel: Don't use __import__ with lots of arguments, it can results in double imports and other silly stuff. Message-ID: <20120127191655.AB96E82114@wyvern.cs.uni-duesseldorf.de> Author: Alex Gaynor <alex.gaynor at gmail.com> Branch: numpy-back-to-applevel Changeset: r51862:6deca798be13 Date: 2012-01-27 13:53 -0500 http://bitbucket.org/pypy/pypy/changeset/6deca798be13/ Log: Don't use __import__ with lots of arguments, it can results in double imports and other silly stuff. diff --git a/pypy/module/micronumpy/appbridge.py b/pypy/module/micronumpy/appbridge.py --- a/pypy/module/micronumpy/appbridge.py +++ b/pypy/module/micronumpy/appbridge.py @@ -11,8 +11,9 @@ def __init__(self, space): self.w_import = space.appexec([], """(): def f(): - mod = __import__('numpypy.core._methods', {}, {}, ['']) - return mod + import sys + __import__('numpypy.core._methods') + return sys.modules['numpypy.core._methods'] return f """) From noreply at buildbot.pypy.org Fri Jan 27 20:16:56 2012 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Fri, 27 Jan 2012 20:16:56 +0100 (CET) Subject: [pypy-commit] pypy numpy-back-to-applevel: cleanup Message-ID: <20120127191656.DE61F82114@wyvern.cs.uni-duesseldorf.de> Author: Alex Gaynor <alex.gaynor at gmail.com> Branch: numpy-back-to-applevel Changeset: r51863:dc4cacb0a280 Date: 2012-01-27 14:15 -0500 http://bitbucket.org/pypy/pypy/changeset/dc4cacb0a280/ Log: cleanup diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py --- a/pypy/module/micronumpy/interp_boxes.py +++ b/pypy/module/micronumpy/interp_boxes.py @@ -7,8 +7,9 @@ from pypy.rlib.rarithmetic import LONG_BIT from pypy.tool.sourcetools import func_with_new_name + +MIXIN_32 = (int_typedef,) if LONG_BIT == 32 else () MIXIN_64 = (int_typedef,) if LONG_BIT == 64 else () -MIXIN_32 = () if LONG_BIT == 64 else (int_typedef,) def new_dtype_getter(name): def get_dtype(space): diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -2,19 +2,20 @@ from pypy.interpreter.error import OperationError, operationerrfmt from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.interpreter.typedef import TypeDef, GetSetProperty -from pypy.module.micronumpy import interp_ufuncs, interp_dtype, signature,\ - interp_boxes -from pypy.module.micronumpy.strides import calculate_slice_strides,\ - shape_agreement, find_shape_and_elems, get_shape_from_iterable,\ - calc_new_strides, to_coords +from pypy.module.micronumpy import (interp_ufuncs, interp_dtype, interp_boxes, + signature, support) +from pypy.module.micronumpy.strides import (calculate_slice_strides, + shape_agreement, find_shape_and_elems, get_shape_from_iterable, + calc_new_strides, to_coords) from pypy.rlib import jit from pypy.rpython.lltypesystem import lltype, rffi from pypy.tool.sourcetools import func_with_new_name from pypy.rlib.rstring import StringBuilder -from pypy.module.micronumpy.interp_iter import ArrayIterator, OneDimIterator,\ - SkipLastAxisIterator, Chunk, ViewIterator +from pypy.module.micronumpy.interp_iter import (ArrayIterator, OneDimIterator, + SkipLastAxisIterator, Chunk, ViewIterator) from pypy.module.micronumpy.appbridge import get_appbridge_cache + numpy_driver = jit.JitDriver( greens=['shapelen', 'sig'], virtualizables=['frame'], @@ -279,10 +280,7 @@ def empty_copy(self, space, dtype): shape = self.shape - size = 1 - for elem in shape: - size *= elem - return W_NDimArray(size, shape[:], dtype, 'C') + return W_NDimArray(support.product(shape), shape[:], dtype, 'C') def flatten(self, space): return self.get_concrete().flatten(space) @@ -609,9 +607,7 @@ space.wrap("axis unsupported for take")) index_i = index.create_iter() res_shape = index.shape - size = 1 - for elem in res_shape: - size *= elem + size = support.product(res_shape) res = W_NDimArray(size, res_shape[:], concr.dtype, concr.order) res_i = res.create_iter() shapelen = len(index.shape) @@ -755,12 +751,9 @@ class VirtualSlice(VirtualArray): def __init__(self, child, chunks, shape): - size = 1 - for sh in shape: - size *= sh self.child = child self.chunks = chunks - self.size = size + self.size = support.product(shape) VirtualArray.__init__(self, 'slice', shape, child.find_dtype()) def create_sig(self): @@ -806,9 +799,7 @@ self.left = left self.right = right self.calc_dtype = calc_dtype - self.size = 1 - for s in self.shape: - self.size *= s + self.size = support.product(self.shape) def _del_sources(self): self.left = None @@ -1013,13 +1004,10 @@ assert isinstance(parent, ConcreteArray) if isinstance(parent, W_NDimSlice): parent = parent.parent - size = 1 - for sh in shape: - size *= sh self.strides = strides self.backstrides = backstrides - ViewArray.__init__(self, size, shape, parent.dtype, parent.order, - parent) + ViewArray.__init__(self, support.product(shape), shape, parent.dtype, + parent.order, parent) self.start = start def create_iter(self): @@ -1180,10 +1168,7 @@ if not keepdims: raise OperationError(space.w_NotImplementedError, space.wrap("unsupported")) if space.is_w(w_axis, space.w_None): - s = 1 - for elem in arr.shape: - s *= elem - return space.wrap(s) + return space.wrap(support.product(arr.shape)) if space.isinstance_w(w_axis, space.w_int): return space.wrap(arr.shape[space.int_w(w_axis)]) s = 1 @@ -1221,10 +1206,7 @@ "array dimensions must agree except for axis being concatenated")) elif i == axis: shape[i] += axis_size - size = 1 - for elem in shape: - size *= elem - res = W_NDimArray(size, shape, dtype, 'C') + res = W_NDimArray(support.product(shape), shape, dtype, 'C') chunks = [Chunk(0, i, 1, i) for i in shape] axis_start = 0 for arr in args_w: @@ -1317,9 +1299,7 @@ @jit.unroll_safe def __init__(self, arr): arr = arr.get_concrete() - size = 1 - for sh in arr.shape: - size *= sh + size = support.product(arr.shape) self.strides = [arr.strides[-1]] self.backstrides = [arr.backstrides[-1]] ViewArray.__init__(self, size, [size], arr.dtype, arr.order, diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py --- a/pypy/module/micronumpy/interp_ufuncs.py +++ b/pypy/module/micronumpy/interp_ufuncs.py @@ -2,13 +2,14 @@ from pypy.interpreter.error import OperationError, operationerrfmt from pypy.interpreter.gateway import interp2app, unwrap_spec, NoneNotWrapped from pypy.interpreter.typedef import TypeDef, GetSetProperty, interp_attrproperty -from pypy.module.micronumpy import interp_boxes, interp_dtype -from pypy.module.micronumpy.signature import ReduceSignature,\ - find_sig, new_printable_location, AxisReduceSignature, ScalarSignature +from pypy.module.micronumpy import interp_boxes, interp_dtype, support +from pypy.module.micronumpy.signature import (ReduceSignature, find_sig, + new_printable_location, AxisReduceSignature, ScalarSignature) from pypy.rlib import jit from pypy.rlib.rarithmetic import LONG_BIT from pypy.tool.sourcetools import func_with_new_name + reduce_driver = jit.JitDriver( greens=['shapelen', "sig"], virtualizables=["frame"], @@ -185,10 +186,7 @@ shape = obj.shape[:dim] + [1] + obj.shape[dim + 1:] else: shape = obj.shape[:dim] + obj.shape[dim + 1:] - size = 1 - for s in shape: - size *= s - result = W_NDimArray(size, shape, dtype) + result = W_NDimArray(support.product(shape), shape, dtype) rightsig = obj.create_sig() # note - this is just a wrapper so signature can fetch # both left and right, nothing more, especially From noreply at buildbot.pypy.org Fri Jan 27 20:16:58 2012 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Fri, 27 Jan 2012 20:16:58 +0100 (CET) Subject: [pypy-commit] pypy numpy-back-to-applevel: merged upstream Message-ID: <20120127191658.1D28882114@wyvern.cs.uni-duesseldorf.de> Author: Alex Gaynor <alex.gaynor at gmail.com> Branch: numpy-back-to-applevel Changeset: r51864:6ab531465c38 Date: 2012-01-27 14:16 -0500 http://bitbucket.org/pypy/pypy/changeset/6ab531465c38/ Log: merged upstream diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -1453,7 +1453,6 @@ from _numpypy import arange assert (arange(3).ravel() == arange(3)).all() assert (arange(6).reshape(2, 3).ravel() == arange(6)).all() - print arange(6).reshape(2, 3).T.ravel() assert (arange(6).reshape(2, 3).T.ravel() == [0, 3, 1, 4, 2, 5]).all() def test_take(self): diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py --- a/pypy/module/micronumpy/types.py +++ b/pypy/module/micronumpy/types.py @@ -105,9 +105,9 @@ )) def read_bool(self, storage, width, i, offset): - return self.for_computation( + return bool(self.for_computation( libffi.array_getitem(clibffi.cast_type_to_ffitype(self.T), - width, storage, i, offset)) != 0 + width, storage, i, offset))) def store(self, storage, width, i, offset, box): value = self.unbox(box) @@ -182,7 +182,7 @@ return v1 >= v2 def bool(self, v): - return self.for_computation(self.unbox(v)) != 0 + return bool(self.for_computation(self.unbox(v))) @simple_binary_op def max(self, v1, v2): @@ -229,8 +229,6 @@ def default_fromstring(self, space): return self.box(False) - # XXX check rpythonization - @simple_binary_op def bitwise_and(self, v1, v2): return v1 & v2 From noreply at buildbot.pypy.org Fri Jan 27 20:20:04 2012 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Fri, 27 Jan 2012 20:20:04 +0100 (CET) Subject: [pypy-commit] pypy numpy-back-to-applevel: forgotten file Message-ID: <20120127192004.BA4BC82114@wyvern.cs.uni-duesseldorf.de> Author: Alex Gaynor <alex.gaynor at gmail.com> Branch: numpy-back-to-applevel Changeset: r51865:706c23fe89c5 Date: 2012-01-27 14:19 -0500 http://bitbucket.org/pypy/pypy/changeset/706c23fe89c5/ Log: forgotten file diff --git a/pypy/module/micronumpy/support.py b/pypy/module/micronumpy/support.py new file mode 100644 --- /dev/null +++ b/pypy/module/micronumpy/support.py @@ -0,0 +1,5 @@ +def product(s): + i = 1 + for x in s: + i *= x + return i \ No newline at end of file From noreply at buildbot.pypy.org Fri Jan 27 20:24:30 2012 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 27 Jan 2012 20:24:30 +0100 (CET) Subject: [pypy-commit] pypy numpy-back-to-applevel: make flatten use ravel, remove previous version, improve tests Message-ID: <20120127192430.3EFBC82114@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: numpy-back-to-applevel Changeset: r51866:938d4dcbaa69 Date: 2012-01-27 21:23 +0200 http://bitbucket.org/pypy/pypy/changeset/938d4dcbaa69/ Log: make flatten use ravel, remove previous version, improve tests diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -272,8 +272,14 @@ def descr_copy(self, space): return self.copy(space) - def descr_flatten(self, space): - return self.flatten(space) + def descr_flatten(self, space, w_order=None): + if isinstance(self, Scalar): + return self.copy(space) + concr = self.get_concrete() + w_res = concr.descr_ravel(space, w_order) + if w_res.storage == concr.storage: + return w_res.copy(space) + return w_res def copy(self, space): return self.get_concrete().copy(space) @@ -282,9 +288,6 @@ shape = self.shape return W_NDimArray(support.product(shape), shape[:], dtype, 'C') - def flatten(self, space): - return self.get_concrete().flatten(space) - def descr_len(self, space): if len(self.shape): return space.wrap(self.shape[0]) @@ -681,11 +684,6 @@ def copy(self, space): return Scalar(self.dtype, self.value) - def flatten(self, space): - array = W_NDimArray(self.size, [self.size], self.dtype) - array.setitem(0, self.value) - return array - def fill(self, space, w_value): self.value = self.dtype.coerce(space, w_value) @@ -981,15 +979,6 @@ array.setslice(space, self) return array - def flatten(self, space): - array = W_NDimArray(self.size, [self.size], self.dtype, self.order) - if self.supports_fast_slicing(): - array._fast_setslice(space, self) - else: - arr = SliceArray(array.shape, array.dtype, array, self, no_broadcast=True) - array._sliceloop(arr) - return array - def fill(self, space, w_value): self.setslice(space, scalar_w(space, self.dtype, w_value)) diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -1104,6 +1104,12 @@ def test_flatten(self): from _numpypy import array + a = array([[1, 2], [3, 4]]) + b = a.flatten() + c = a.ravel() + a[0, 0] = 15 + assert b[0] == 1 + assert c[0] == 15 a = array([[1, 2, 3], [4, 5, 6]]) assert (a.flatten() == [1, 2, 3, 4, 5, 6]).all() a = array([[[1, 2], [3, 4]], [[5, 6], [7, 8]]]) From noreply at buildbot.pypy.org Fri Jan 27 20:24:31 2012 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 27 Jan 2012 20:24:31 +0100 (CET) Subject: [pypy-commit] pypy numpy-back-to-applevel: remove an empty test Message-ID: <20120127192431.733F482114@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: numpy-back-to-applevel Changeset: r51867:9c98420734a5 Date: 2012-01-27 21:23 +0200 http://bitbucket.org/pypy/pypy/changeset/9c98420734a5/ Log: remove an empty test diff --git a/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py --- a/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py +++ b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py @@ -136,7 +136,4 @@ raises(NotImplementedError, "transpose(x, axes=(1, 0, 2))") # x = ones((1, 2, 3)) # assert transpose(x, (1, 0, 2)).shape == (2, 1, 3) - - def test_ravel(self): - from numpypy import ravel From noreply at buildbot.pypy.org Fri Jan 27 20:29:10 2012 From: noreply at buildbot.pypy.org (antocuni) Date: Fri, 27 Jan 2012 20:29:10 +0100 (CET) Subject: [pypy-commit] pypy py3k: bah, fix the signature of note_import_star, which I broke in d882e23cc8ae Message-ID: <20120127192910.C921082114@wyvern.cs.uni-duesseldorf.de> Author: Antonio Cuni <anto.cuni at gmail.com> Branch: py3k Changeset: r51868:742faaa192e9 Date: 2012-01-27 10:34 +0100 http://bitbucket.org/pypy/pypy/changeset/742faaa192e9/ Log: bah, fix the signature of note_import_star, which I broke in d882e23cc8ae diff --git a/pypy/interpreter/astcompiler/symtable.py b/pypy/interpreter/astcompiler/symtable.py --- a/pypy/interpreter/astcompiler/symtable.py +++ b/pypy/interpreter/astcompiler/symtable.py @@ -246,7 +246,7 @@ self.return_with_value = True self.ret = ret - def note_import_star(self): + def note_import_star(self, imp): return True def note_variable_arg(self, vararg): From noreply at buildbot.pypy.org Fri Jan 27 20:29:12 2012 From: noreply at buildbot.pypy.org (antocuni) Date: Fri, 27 Jan 2012 20:29:12 +0100 (CET) Subject: [pypy-commit] pypy py3k: kill a lot of snippets from the invalid cases: in python3 it is now possible to call exec() inside a function which creates nested functions with free variables, because exec() cannot modify the locals anyway Message-ID: <20120127192912.1176D82114@wyvern.cs.uni-duesseldorf.de> Author: Antonio Cuni <anto.cuni at gmail.com> Branch: py3k Changeset: r51869:116575861a5d Date: 2012-01-27 11:21 +0100 http://bitbucket.org/pypy/pypy/changeset/116575861a5d/ Log: kill a lot of snippets from the invalid cases: in python3 it is now possible to call exec() inside a function which creates nested functions with free variables, because exec() cannot modify the locals anyway diff --git a/pypy/interpreter/test/test_syntax.py b/pypy/interpreter/test/test_syntax.py --- a/pypy/interpreter/test/test_syntax.py +++ b/pypy/interpreter/test/test_syntax.py @@ -76,51 +76,6 @@ from x import * def f(): - def g(): - exec("hi") - x - # NB. the above one is invalid in CPython, but there is no real reason - - def f(x): - def g(): - exec("hi") - x - - def f(): - exec("hi") - def g(): - x - - def f(): - exec("hi") - lambda x: y - - def f(): - exec("hi") - class g: - x - - def f(): - exec("hi") - x = 5 - class g: - def h(): - x - - def f(x): - exec("hi") - x = 4 - class g: - x - - def f(x): - exec("hi") - x = 5 - class g: - def h(): - x - - def f(): (i for i in x) = 10 """) From noreply at buildbot.pypy.org Fri Jan 27 20:29:13 2012 From: noreply at buildbot.pypy.org (antocuni) Date: Fri, 27 Jan 2012 20:29:13 +0100 (CET) Subject: [pypy-commit] pypy py3k: port the syntax of some AppLevel tests to python3 Message-ID: <20120127192913.4D65782114@wyvern.cs.uni-duesseldorf.de> Author: Antonio Cuni <anto.cuni at gmail.com> Branch: py3k Changeset: r51870:b421a4de9302 Date: 2012-01-27 11:50 +0100 http://bitbucket.org/pypy/pypy/changeset/b421a4de9302/ Log: port the syntax of some AppLevel tests to python3 diff --git a/pypy/interpreter/test/test_syntax.py b/pypy/interpreter/test/test_syntax.py --- a/pypy/interpreter/test/test_syntax.py +++ b/pypy/interpreter/test/test_syntax.py @@ -164,39 +164,44 @@ def test_condexpr(self): for s, expected in [("x = 1 if True else 2", 1), ("x = 1 if False else 2", 2)]: - exec s + exec(s) assert x == expected def test_condexpr_no_warning(self): import warnings warnings.simplefilter('error', SyntaxWarning) - exec "1 if True else 2" + exec("1 if True else 2") warnings.simplefilter('default', SyntaxWarning) class AppTestYield: def test_bare_yield(self): s = "def f():\n yield" - exec s + exec(s) class AppTestDecorators: def test_function_decorators(self): + ''' def other(): return 4 def dec(f): return other ns = {} ns["dec"] = dec - exec """@dec -def g(): - pass""" in ns + exec("""if 1: + @dec + def g(): + pass + """, ns) assert ns["g"] is other assert ns["g"]() == 4 + ''' def test_application_order(self): + ''' def dec1(f): record.append(1) return f @@ -205,21 +210,28 @@ return f record = [] ns = {"dec1" : dec1, "dec2" : dec2} - exec """@dec1 - at dec2 -def g(): pass""" in ns + exec("""if 1: + @dec1 + @dec2 + def g(): + pass + """, ns) assert record == [2, 1] del record[:] - exec """@dec1 - at dec2 -class x: pass""" in ns + exec("""if 1: + @dec1 + @dec2 + class x: + pass + """, ns) assert record == [2, 1] + ''' def test_class_decorators(self): s = """@func class x: pass""" ns = {"func" : lambda cls: 4} - exec s in ns + exec(s, ns) assert ns["x"] == 4 @@ -231,7 +243,7 @@ x = print """ ns = {} - exec s in ns + exec(s, ns) assert ns["x"] is getattr(__builtin__, "print") def test_print(self): @@ -241,7 +253,7 @@ print("Hello,", "person", file=s) """ ns = {} - exec s in ns + exec(s, ns) assert ns["s"].getvalue() == "Hello, person\n" @@ -255,7 +267,7 @@ b = b'u' c = br'u'""" ns = {} - exec s in ns + exec(s, ns) assert isinstance(ns["x"], unicode) assert isinstance(ns["y"], unicode) assert isinstance(ns["z"], unicode) @@ -308,7 +320,7 @@ with acontext: pass """ - exec s + exec(s) assert acontext.calls == '__enter__ __exit__'.split() @@ -328,7 +340,7 @@ pass """ ns = {} - exec s in ns + exec(s, ns) assert ns["v1"] == "blah" assert ns["v2"] == "bling" assert ns["c1"].record == [("__enter__", "blah"), ("__exit__", "blah")] @@ -353,7 +365,7 @@ with acontext: pass """ - exec s + exec(s) assert acontext.calls == '__enter__ __exit__'.split() def test_raw_doc_string(self): @@ -367,7 +379,7 @@ exit = True with Context() as w: pass""" - exec s + exec(s) assert enter assert exit @@ -392,7 +404,7 @@ avar.append('__body__') pass """ - exec s + exec(s) assert acontextfact.exit_params == (None, None, None) assert acontextfact.calls == '__enter__ __body__ __exit__'.split() @@ -425,7 +437,7 @@ else: raise AssertionError('With did not raise RuntimeError') """ - exec s + exec(s) assert acontextfact.calls == '__enter__ __body__ __exit__'.split() assert acontextfact.exit_params[0:2] == (RuntimeError, error) @@ -456,7 +468,7 @@ raise error avar.append('__after_raise__') """ - exec s + exec(s) assert acontextfact.calls == '__enter__ __body__ __exit__'.split() assert acontextfact.exit_params[0:2] == (RuntimeError, error) @@ -489,7 +501,7 @@ else: raise AssertionError('Break failed with With, reached else clause') """ - exec s + exec(s) assert acontextfact.calls == '__enter__ __body__ __exit__'.split() assert acontextfact.exit_params == (None, None, None) @@ -520,7 +532,7 @@ else: avar.append('__continue__') """ - exec s + exec(s) assert acontextfact.calls == '__enter__ __body__ __exit__ __continue__'.split() assert acontextfact.exit_params == (None, None, None) @@ -549,14 +561,14 @@ avar.append('__after_return__') acontextfact.calls.append(g(acontextfact)) """ - exec s + exec(s) assert acontextfact.calls == '__enter__ __body__ __exit__ __return__'.split() assert acontextfact.exit_params == (None, None, None) def test_with_as_keyword(self): try: - exec "with = 9" + exec("with = 9") except SyntaxError: pass else: @@ -564,7 +576,7 @@ def test_with_as_keyword_compound(self): try: - exec "from __future__ import generators, with_statement\nwith = 9" + exec("from __future__ import generators, with_statement\nwith = 9") except SyntaxError: pass else: @@ -580,7 +592,7 @@ """] for snippet in snippets: try: - exec snippet + exec(snippet) except SyntaxError: pass else: @@ -593,14 +605,14 @@ compile('''with x: pass''', '', 'exec') """ - exec s + exec(s) class AppTestSyntaxError: def test_tokenizer_error_location(self): line4 = "if ?: pass\n" try: - exec "print\nprint\nprint\n" + line4 + exec("print\nprint\nprint\n" + line4) except SyntaxError, e: assert e.lineno == 4 assert e.text == line4 @@ -610,12 +622,12 @@ def test_grammar_error_location(self): try: - exec """if 1: + exec("""if 1: class Foo: bla a b c d e bar - """ + """) except SyntaxError, e: assert e.lineno == 4 assert e.text.endswith('a b c d e\n') @@ -626,7 +638,7 @@ def test_astbuilder_error_location(self): program = "(1, 2) += (3, 4)\n" try: - exec program + exec(program) except SyntaxError, e: assert e.lineno == 1 assert e.text is None @@ -638,7 +650,7 @@ # -*- coding: uft-8 -*- pass """ - raises(SyntaxError, "exec program") + raises(SyntaxError, "exec(program)") if __name__ == '__main__': From noreply at buildbot.pypy.org Fri Jan 27 20:29:14 2012 From: noreply at buildbot.pypy.org (antocuni) Date: Fri, 27 Jan 2012 20:29:14 +0100 (CET) Subject: [pypy-commit] pypy py3k: we no longer need to import print_function from __future__. Also, s/__builtin__/builtins Message-ID: <20120127192914.83EE682114@wyvern.cs.uni-duesseldorf.de> Author: Antonio Cuni <anto.cuni at gmail.com> Branch: py3k Changeset: r51871:7586f1183218 Date: 2012-01-27 11:51 +0100 http://bitbucket.org/pypy/pypy/changeset/7586f1183218/ Log: we no longer need to import print_function from __future__. Also, s/__builtin__/builtins diff --git a/pypy/interpreter/test/test_syntax.py b/pypy/interpreter/test/test_syntax.py --- a/pypy/interpreter/test/test_syntax.py +++ b/pypy/interpreter/test/test_syntax.py @@ -238,13 +238,13 @@ class AppTestPrintFunction: def test_simple_print(self): - import __builtin__ - s = """from __future__ import print_function -x = print -""" + """ + import builtins + s = "x = print" ns = {} exec(s, ns) - assert ns["x"] is getattr(__builtin__, "print") + assert ns["x"] is builtins.print + """ def test_print(self): s = """from __future__ import print_function From noreply at buildbot.pypy.org Fri Jan 27 20:29:15 2012 From: noreply at buildbot.pypy.org (antocuni) Date: Fri, 27 Jan 2012 20:29:15 +0100 (CET) Subject: [pypy-commit] pypy py3k: kill import print_function from __future__ also here, and adjust the import of StringIO Message-ID: <20120127192915.BA3DB82114@wyvern.cs.uni-duesseldorf.de> Author: Antonio Cuni <anto.cuni at gmail.com> Branch: py3k Changeset: r51872:c5b8818fe586 Date: 2012-01-27 11:53 +0100 http://bitbucket.org/pypy/pypy/changeset/c5b8818fe586/ Log: kill import print_function from __future__ also here, and adjust the import of StringIO diff --git a/pypy/interpreter/test/test_syntax.py b/pypy/interpreter/test/test_syntax.py --- a/pypy/interpreter/test/test_syntax.py +++ b/pypy/interpreter/test/test_syntax.py @@ -247,9 +247,9 @@ """ def test_print(self): - s = """from __future__ import print_function -import StringIO -s = StringIO.StringIO() + s = """ +from io import StringIO +s = StringIO() print("Hello,", "person", file=s) """ ns = {} From noreply at buildbot.pypy.org Fri Jan 27 20:29:17 2012 From: noreply at buildbot.pypy.org (antocuni) Date: Fri, 27 Jan 2012 20:29:17 +0100 (CET) Subject: [pypy-commit] pypy py3k: don't import unicode_literals from __future__, and adapt unicode-->str and str-->bytes Message-ID: <20120127192917.07DFB82114@wyvern.cs.uni-duesseldorf.de> Author: Antonio Cuni <anto.cuni at gmail.com> Branch: py3k Changeset: r51873:58915e03655e Date: 2012-01-27 11:56 +0100 http://bitbucket.org/pypy/pypy/changeset/58915e03655e/ Log: don't import unicode_literals from __future__, and adapt unicode-->str and str-->bytes diff --git a/pypy/interpreter/test/test_syntax.py b/pypy/interpreter/test/test_syntax.py --- a/pypy/interpreter/test/test_syntax.py +++ b/pypy/interpreter/test/test_syntax.py @@ -260,19 +260,17 @@ class AppTestUnicodeLiterals: def test_simple(self): - s = """from __future__ import unicode_literals + s = """ x = 'u' y = r'u' -z = u'u' b = b'u' c = br'u'""" ns = {} exec(s, ns) - assert isinstance(ns["x"], unicode) - assert isinstance(ns["y"], unicode) - assert isinstance(ns["z"], unicode) - assert isinstance(ns["b"], str) - assert isinstance(ns["c"], str) + assert isinstance(ns["x"], str) + assert isinstance(ns["y"], str) + assert isinstance(ns["b"], bytes) + assert isinstance(ns["c"], bytes) class AppTestComprehensions: From noreply at buildbot.pypy.org Fri Jan 27 20:29:18 2012 From: noreply at buildbot.pypy.org (antocuni) Date: Fri, 27 Jan 2012 20:29:18 +0100 (CET) Subject: [pypy-commit] pypy py3k: fix the syntax of exception clauses. Now all applevel tests pass with --appdirect=/usr/bin/python3 Message-ID: <20120127192918.413CB82114@wyvern.cs.uni-duesseldorf.de> Author: Antonio Cuni <anto.cuni at gmail.com> Branch: py3k Changeset: r51874:59514c43c5c0 Date: 2012-01-27 12:08 +0100 http://bitbucket.org/pypy/pypy/changeset/59514c43c5c0/ Log: fix the syntax of exception clauses. Now all applevel tests pass with --appdirect=/usr/bin/python3 diff --git a/pypy/interpreter/test/test_syntax.py b/pypy/interpreter/test/test_syntax.py --- a/pypy/interpreter/test/test_syntax.py +++ b/pypy/interpreter/test/test_syntax.py @@ -611,7 +611,7 @@ line4 = "if ?: pass\n" try: exec("print\nprint\nprint\n" + line4) - except SyntaxError, e: + except SyntaxError as e: assert e.lineno == 4 assert e.text == line4 assert e.offset == e.text.index('?') + 1 @@ -626,10 +626,11 @@ a b c d e bar """) - except SyntaxError, e: + except SyntaxError as e: assert e.lineno == 4 assert e.text.endswith('a b c d e\n') - assert e.offset == e.text.index('b') + b_pos = e.text.index('b') + assert e.offset in (b_pos, b_pos+1) # b_pos in pypy, b_pos+1 in CPython. else: raise Exception("no SyntaxError??") @@ -637,19 +638,20 @@ program = "(1, 2) += (3, 4)\n" try: exec(program) - except SyntaxError, e: + except SyntaxError as e: assert e.lineno == 1 assert e.text is None else: raise Exception("no SyntaxError??") def test_bad_encoding(self): + ''' program = """ # -*- coding: uft-8 -*- pass """ - raises(SyntaxError, "exec(program)") - + raises(SyntaxError, exec, program) + ''' if __name__ == '__main__': # only to check on top of CPython (you need 2.4) From noreply at buildbot.pypy.org Fri Jan 27 20:29:19 2012 From: noreply at buildbot.pypy.org (antocuni) Date: Fri, 27 Jan 2012 20:29:19 +0100 (CET) Subject: [pypy-commit] pypy py3k: exec can no longer modify the local scope, so use an explicit scope instead Message-ID: <20120127192919.7815A82114@wyvern.cs.uni-duesseldorf.de> Author: Antonio Cuni <anto.cuni at gmail.com> Branch: py3k Changeset: r51875:ba117ac345c3 Date: 2012-01-27 12:32 +0100 http://bitbucket.org/pypy/pypy/changeset/ba117ac345c3/ Log: exec can no longer modify the local scope, so use an explicit scope instead diff --git a/pypy/interpreter/test/test_syntax.py b/pypy/interpreter/test/test_syntax.py --- a/pypy/interpreter/test/test_syntax.py +++ b/pypy/interpreter/test/test_syntax.py @@ -164,8 +164,9 @@ def test_condexpr(self): for s, expected in [("x = 1 if True else 2", 1), ("x = 1 if False else 2", 2)]: - exec(s) - assert x == expected + ns = {} + exec(s, ns) + assert ns['x'] == expected def test_condexpr_no_warning(self): import warnings From noreply at buildbot.pypy.org Fri Jan 27 20:29:20 2012 From: noreply at buildbot.pypy.org (antocuni) Date: Fri, 27 Jan 2012 20:29:20 +0100 (CET) Subject: [pypy-commit] pypy py3k: kill this test about warnings and conditional expressions. We stopped issuing the warning long time ago Message-ID: <20120127192920.AF47382114@wyvern.cs.uni-duesseldorf.de> Author: Antonio Cuni <anto.cuni at gmail.com> Branch: py3k Changeset: r51876:1b76cb58345b Date: 2012-01-27 12:35 +0100 http://bitbucket.org/pypy/pypy/changeset/1b76cb58345b/ Log: kill this test about warnings and conditional expressions. We stopped issuing the warning long time ago diff --git a/pypy/interpreter/test/test_syntax.py b/pypy/interpreter/test/test_syntax.py --- a/pypy/interpreter/test/test_syntax.py +++ b/pypy/interpreter/test/test_syntax.py @@ -168,13 +168,6 @@ exec(s, ns) assert ns['x'] == expected - def test_condexpr_no_warning(self): - import warnings - - warnings.simplefilter('error', SyntaxWarning) - exec("1 if True else 2") - warnings.simplefilter('default', SyntaxWarning) - class AppTestYield: def test_bare_yield(self): s = "def f():\n yield" From noreply at buildbot.pypy.org Fri Jan 27 20:29:21 2012 From: noreply at buildbot.pypy.org (antocuni) Date: Fri, 27 Jan 2012 20:29:21 +0100 (CET) Subject: [pypy-commit] pypy py3k: exec() can no longer modify the local scope, so we need to explicitly pass a namespace and fish the interesting things out of it Message-ID: <20120127192921.F023F82114@wyvern.cs.uni-duesseldorf.de> Author: Antonio Cuni <anto.cuni at gmail.com> Branch: py3k Changeset: r51877:5127c7fa289c Date: 2012-01-27 14:50 +0100 http://bitbucket.org/pypy/pypy/changeset/5127c7fa289c/ Log: exec() can no longer modify the local scope, so we need to explicitly pass a namespace and fish the interesting things out of it diff --git a/pypy/interpreter/test/test_syntax.py b/pypy/interpreter/test/test_syntax.py --- a/pypy/interpreter/test/test_syntax.py +++ b/pypy/interpreter/test/test_syntax.py @@ -312,8 +312,9 @@ with acontext: pass """ - exec(s) - + ns = {} + exec(s, ns) + acontext = ns['acontext'] assert acontext.calls == '__enter__ __exit__'.split() def test_compound_with(self): @@ -357,7 +358,9 @@ with acontext: pass """ - exec(s) + ns = {} + exec(s, ns) + acontext = ns['acontext'] assert acontext.calls == '__enter__ __exit__'.split() def test_raw_doc_string(self): @@ -371,9 +374,10 @@ exit = True with Context() as w: pass""" - exec(s) - assert enter - assert exit + ns = {} + exec(s, ns) + assert ns['enter'] + assert ns['exit'] def test_with_as_var(self): @@ -396,8 +400,9 @@ avar.append('__body__') pass """ - exec(s) - + ns = {} + exec(s, ns) + acontextfact = ns['acontextfact'] assert acontextfact.exit_params == (None, None, None) assert acontextfact.calls == '__enter__ __body__ __exit__'.split() @@ -429,8 +434,10 @@ else: raise AssertionError('With did not raise RuntimeError') """ - exec(s) - + ns = {} + exec(s, ns) + acontextfact = ns['acontextfact'] + error = ns['error'] assert acontextfact.calls == '__enter__ __body__ __exit__'.split() assert acontextfact.exit_params[0:2] == (RuntimeError, error) import types @@ -460,8 +467,10 @@ raise error avar.append('__after_raise__') """ - exec(s) - + ns = {} + exec(s, ns) + acontextfact = ns['acontextfact'] + error = ns['error'] assert acontextfact.calls == '__enter__ __body__ __exit__'.split() assert acontextfact.exit_params[0:2] == (RuntimeError, error) import types @@ -493,8 +502,9 @@ else: raise AssertionError('Break failed with With, reached else clause') """ - exec(s) - + ns = {} + exec(s, ns) + acontextfact = ns['acontextfact'] assert acontextfact.calls == '__enter__ __body__ __exit__'.split() assert acontextfact.exit_params == (None, None, None) @@ -524,8 +534,9 @@ else: avar.append('__continue__') """ - exec(s) - + ns = {} + exec(s, ns) + acontextfact = ns['acontextfact'] assert acontextfact.calls == '__enter__ __body__ __exit__ __continue__'.split() assert acontextfact.exit_params == (None, None, None) @@ -553,8 +564,9 @@ avar.append('__after_return__') acontextfact.calls.append(g(acontextfact)) """ - exec(s) - + ns = {} + exec(s, ns) + acontextfact = ns['acontextfact'] assert acontextfact.calls == '__enter__ __body__ __exit__ __return__'.split() assert acontextfact.exit_params == (None, None, None) From noreply at buildbot.pypy.org Fri Jan 27 20:29:23 2012 From: noreply at buildbot.pypy.org (antocuni) Date: Fri, 27 Jan 2012 20:29:23 +0100 (CET) Subject: [pypy-commit] pypy py3k: kill the --appdirect option and leave only --runappdirect, which by default tries to run the tests with the python3 executable found in the path, unless you specify another one with --python Message-ID: <20120127192923.3249682114@wyvern.cs.uni-duesseldorf.de> Author: Antonio Cuni <anto.cuni at gmail.com> Branch: py3k Changeset: r51878:f3e4991575d5 Date: 2012-01-27 15:25 +0100 http://bitbucket.org/pypy/pypy/changeset/f3e4991575d5/ Log: kill the --appdirect option and leave only --runappdirect, which by default tries to run the tests with the python3 executable found in the path, unless you specify another one with --python diff --git a/pypy/conftest.py b/pypy/conftest.py --- a/pypy/conftest.py +++ b/pypy/conftest.py @@ -10,6 +10,10 @@ from pypy.tool.autopath import pypydir from pypy.tool import leakfinder, runsubprocess +PYTHON3 = py.path.local.sysfind('python3') +if PYTHON3 is not None: + PYTHON3 = str(PYTHON3) + # pytest settings rsyncdirs = ['.', '../lib-python', '../lib_pypy', '../demo'] rsyncignore = ['_cache'] @@ -25,8 +29,6 @@ def pytest_configure(config): global option option = config.option - if option.appdirect: - option.runappdirect = True def _set_platform(opt, opt_str, value, parser): from pypy.config.translationoption import PLATFORMS @@ -41,9 +43,10 @@ help="view translation tests' flow graphs with Pygame") group.addoption('-A', '--runappdirect', action="store_true", default=False, dest="runappdirect", - help="run applevel tests directly on python interpreter (not through PyPy)") - group.addoption('--appdirect', type="string", - help="run applevel tests directly with the specified interpreter") + help="run applevel tests directly on the python interpreter " + + "specified by --python") + group.addoption('--python', type="string", default=PYTHON3, + help="python interpreter to run appdirect tests with") group.addoption('--direct', action="store_true", default=False, dest="rundirect", help="run pexpect tests directly") @@ -195,6 +198,8 @@ py.test.skip("translation test, skipped for appdirect") def run_with_python(python, target): + if python is None: + py.test.skip("Cannot find the default python3 interpreter to run with -A") helpers = """if 1: def skip(message): print(message) @@ -424,7 +429,7 @@ target = self.obj src = extract_docstring_if_empty_function(target) if self.config.option.runappdirect: - return run_appdirect_or_skip(self.config, target, src) + return run_with_python(self.config.option.python, src) space = gettestobjspace() filename = self._getdynfilename(target) func = app2interp_temp(src, filename=filename) @@ -468,24 +473,13 @@ target = self.obj src = extract_docstring_if_empty_function(target.im_func) if self.config.option.runappdirect: - return run_appdirect_or_skip(self.config, target, src) + return run_with_python(self.config.option.python, src) space = target.im_self.space filename = self._getdynfilename(target) func = app2interp_temp(src, filename=filename) w_instance = self.parent.w_instance self.execute_appex(space, func, space, w_instance) -def run_appdirect_or_skip(config, target, src): - if config.option.appdirect: - return run_with_python(config.option.appdirect, src) - if isinstance(src, str): - # we are trying to directly run a test whose code is inside - # the docstring. This cannot work because the code might - # contain py3k-only syntax, while we are on a python2 hosting - # python. So, we just skip the test - py.test.skip('Cannot run docstring-tests with -A') - return target() - def extract_docstring_if_empty_function(fn): def empty_func(): diff --git a/pypy/tool/pytest/test/test_conftest1.py b/pypy/tool/pytest/test/test_conftest1.py --- a/pypy/tool/pytest/test/test_conftest1.py +++ b/pypy/tool/pytest/test/test_conftest1.py @@ -32,15 +32,6 @@ assert "app_test_something" in passed[0].nodeid assert "test_method_app" in passed[1].nodeid - def test_appdirect(self, testdir): - sorter = testdir.inline_run(innertest, '-k', 'applevel -docstring', - '--appdirect=%s' % (sys.executable,)) - passed, skipped, failed = sorter.listoutcomes() - assert len(passed) == 2 - print passed - assert "app_test_something" in passed[0].nodeid - assert "test_method_app" in passed[1].nodeid - def test_docstring_in_methods(self, testdir): sorter = testdir.inline_run("-k", "AppTestSomething test_code_in_docstring", innertest) @@ -65,17 +56,6 @@ '--runappdirect') passed, skipped, failed = sorter.listoutcomes() assert len(passed) == 1 - assert len(skipped) == 2 - assert "test_code_in_docstring_ignored" in passed[0].nodeid - assert "app_test_code_in_docstring_failing" in skipped[0].nodeid - assert "test_code_in_docstring_failing" in skipped[1].nodeid - - def test_docstring_appdirect(self, testdir): - sorter = testdir.inline_run(innertest, - '-k', 'test_code_in_docstring', - '--appdirect=%s' % (sys.executable,)) - passed, skipped, failed = sorter.listoutcomes() - assert len(passed) == 1 assert len(failed) == 2 assert "test_code_in_docstring_ignored" in passed[0].nodeid assert "app_test_code_in_docstring_failing" in failed[0].nodeid From noreply at buildbot.pypy.org Fri Jan 27 20:29:24 2012 From: noreply at buildbot.pypy.org (antocuni) Date: Fri, 27 Jan 2012 20:29:24 +0100 (CET) Subject: [pypy-commit] pypy py3k: fish the name of the python3 executable from the conftest Message-ID: <20120127192924.69C8C82114@wyvern.cs.uni-duesseldorf.de> Author: Antonio Cuni <anto.cuni at gmail.com> Branch: py3k Changeset: r51879:7aeb630cb68e Date: 2012-01-27 15:28 +0100 http://bitbucket.org/pypy/pypy/changeset/7aeb630cb68e/ Log: fish the name of the python3 executable from the conftest diff --git a/pypy/interpreter/test/test_syntax.py b/pypy/interpreter/test/test_syntax.py --- a/pypy/interpreter/test/test_syntax.py +++ b/pypy/interpreter/test/test_syntax.py @@ -1,5 +1,6 @@ import py import commands +import pypy.conftest from pypy.conftest import gettestobjspace def splitcases(s): @@ -94,9 +95,9 @@ """ % (i, i, INVALID[i], INVALID[i]) -PYTHON3 = py.path.local.sysfind('python3') def checksyntax_cpython(tmpdir, i, s): - if PYTHON3 is None: + python3 = pypy.conftest.option.python + if python3 is None: print 'Warning: cannot run python3 to check syntax' return @@ -112,7 +113,7 @@ ''' % s pyfile = tmpdir.join('checkvalid_%d.py' % i) pyfile.write(src) - res = commands.getoutput('"%s" "%s"' % (PYTHON3, pyfile)) + res = commands.getoutput('"%s" "%s"' % (python3, pyfile)) return res def checkvalid_cpython(tmpdir, i, s): From noreply at buildbot.pypy.org Fri Jan 27 20:29:25 2012 From: noreply at buildbot.pypy.org (antocuni) Date: Fri, 27 Jan 2012 20:29:25 +0100 (CET) Subject: [pypy-commit] pypy py3k: py3k-ify the syntax of this test Message-ID: <20120127192925.A2D4382114@wyvern.cs.uni-duesseldorf.de> Author: Antonio Cuni <anto.cuni at gmail.com> Branch: py3k Changeset: r51880:a47a0d7d1ec5 Date: 2012-01-27 16:45 +0100 http://bitbucket.org/pypy/pypy/changeset/a47a0d7d1ec5/ Log: py3k-ify the syntax of this test diff --git a/pypy/interpreter/test/test_compiler.py b/pypy/interpreter/test/test_compiler.py --- a/pypy/interpreter/test/test_compiler.py +++ b/pypy/interpreter/test/test_compiler.py @@ -30,7 +30,7 @@ def test_compile_command(self): for mode in ('single', 'exec'): c0 = self.compiler.compile_command('\t # hello\n ', '?', mode, 0) - c1 = self.compiler.compile_command('print 6*7', '?', mode, 0) + c1 = self.compiler.compile_command('print(6*7)', '?', mode, 0) c2 = self.compiler.compile_command('if 1:\n x\n', '?', mode, 0) c8 = self.compiler.compile_command('x = 5', '?', mode, 0) c9 = self.compiler.compile_command('x = 5 ', '?', mode, 0) From noreply at buildbot.pypy.org Fri Jan 27 20:29:26 2012 From: noreply at buildbot.pypy.org (antocuni) Date: Fri, 27 Jan 2012 20:29:26 +0100 (CET) Subject: [pypy-commit] pypy py3k: py3k-ify the syntax of this test as well Message-ID: <20120127192926.D84F982114@wyvern.cs.uni-duesseldorf.de> Author: Antonio Cuni <anto.cuni at gmail.com> Branch: py3k Changeset: r51881:1089c2a88694 Date: 2012-01-27 16:47 +0100 http://bitbucket.org/pypy/pypy/changeset/1089c2a88694/ Log: py3k-ify the syntax of this test as well diff --git a/pypy/interpreter/test/test_compiler.py b/pypy/interpreter/test/test_compiler.py --- a/pypy/interpreter/test/test_compiler.py +++ b/pypy/interpreter/test/test_compiler.py @@ -71,8 +71,8 @@ def test_syntaxerror_attrs(self): w_args = self.space.appexec([], r"""(): try: - exec 'if 1:\n x\n y\n' - except SyntaxError, e: + exec('if 1:\n x\n y\n') + except SyntaxError as e: return e.args """) assert self.space.unwrap(w_args) == ( From noreply at buildbot.pypy.org Fri Jan 27 20:29:28 2012 From: noreply at buildbot.pypy.org (antocuni) Date: Fri, 27 Jan 2012 20:29:28 +0100 (CET) Subject: [pypy-commit] pypy py3k: py3k-ify the syntax of these three tests Message-ID: <20120127192928.2482A82114@wyvern.cs.uni-duesseldorf.de> Author: Antonio Cuni <anto.cuni at gmail.com> Branch: py3k Changeset: r51882:cf23b6ea0ec0 Date: 2012-01-27 16:55 +0100 http://bitbucket.org/pypy/pypy/changeset/cf23b6ea0ec0/ Log: py3k-ify the syntax of these three tests diff --git a/pypy/interpreter/test/test_compiler.py b/pypy/interpreter/test/test_compiler.py --- a/pypy/interpreter/test/test_compiler.py +++ b/pypy/interpreter/test/test_compiler.py @@ -85,7 +85,7 @@ flags = self.compiler.getcodeflags(code) assert flags & __future__.division.compiler_flag # check that we don't get more flags than the compiler can accept back - code2 = self.compiler.compile('print 6*7', '<hello>', 'exec', flags) + code2 = self.compiler.compile('print(6*7)', '<hello>', 'exec', flags) # check that the flag remains in force flags2 = self.compiler.getcodeflags(code2) assert flags == flags2 @@ -235,7 +235,7 @@ def test_unicode_docstring(self): space = self.space - code = self.compiler.compile('u"hello"\n', '<hello>', 'exec', 0) + code = self.compiler.compile('"hello"\n', '<hello>', 'exec', 0) assert space.eq_w(code.co_consts_w[0], space.wrap("hello")) assert space.is_w(space.type(code.co_consts_w[0]), space.w_unicode) @@ -420,7 +420,7 @@ __NameError = NameError try: yield "found: " + __g - except __NameError, __e: + except __NameError as __e: yield "not found: " + str(__e) del __NameError for __i in range(__self.__u * __n): From noreply at buildbot.pypy.org Fri Jan 27 20:29:29 2012 From: noreply at buildbot.pypy.org (antocuni) Date: Fri, 27 Jan 2012 20:29:29 +0100 (CET) Subject: [pypy-commit] pypy py3k: we can no longer index the keys() of a dict, use next(iter(...)) instead Message-ID: <20120127192929.602A982114@wyvern.cs.uni-duesseldorf.de> Author: Antonio Cuni <anto.cuni at gmail.com> Branch: py3k Changeset: r51883:3014f5e44e49 Date: 2012-01-27 16:56 +0100 http://bitbucket.org/pypy/pypy/changeset/3014f5e44e49/ Log: we can no longer index the keys() of a dict, use next(iter(...)) instead diff --git a/pypy/interpreter/test/test_compiler.py b/pypy/interpreter/test/test_compiler.py --- a/pypy/interpreter/test/test_compiler.py +++ b/pypy/interpreter/test/test_compiler.py @@ -443,7 +443,7 @@ snippet = str(py.code.Source(r''' d = {} d[...] = 12 - assert d.keys()[0] is Ellipsis + assert next(iter(d.keys())) is Ellipsis ''')) code = self.compiler.compile(snippet, '<tmp>', 'exec', 0) space = self.space From noreply at buildbot.pypy.org Fri Jan 27 20:29:30 2012 From: noreply at buildbot.pypy.org (antocuni) Date: Fri, 27 Jan 2012 20:29:30 +0100 (CET) Subject: [pypy-commit] pypy py3k: sanitize the test class hierarchy: now we have only one compiler, so there is no need to have a base class and two subclasses, because they are actually the same Message-ID: <20120127192930.94EA582114@wyvern.cs.uni-duesseldorf.de> Author: Antonio Cuni <anto.cuni at gmail.com> Branch: py3k Changeset: r51884:131fb90722ed Date: 2012-01-27 17:39 +0100 http://bitbucket.org/pypy/pypy/changeset/131fb90722ed/ Log: sanitize the test class hierarchy: now we have only one compiler, so there is no need to have a base class and two subclasses, because they are actually the same diff --git a/pypy/interpreter/test/test_compiler.py b/pypy/interpreter/test/test_compiler.py --- a/pypy/interpreter/test/test_compiler.py +++ b/pypy/interpreter/test/test_compiler.py @@ -6,7 +6,7 @@ from pypy.interpreter.argument import Arguments from pypy.conftest import gettestobjspace -class BaseTestCompiler: +class TestPythonAstCompiler: def setup_method(self, method): self.compiler = self.space.createcompiler() @@ -652,11 +652,6 @@ assert ex.match(space, space.w_SyntaxError) assert 'hello_world' in space.str_w(space.str(ex.get_w_value(space))) - -class TestPythonAstCompiler_25_grammar(BaseTestCompiler): - def setup_method(self, method): - self.compiler = PythonAstCompiler(self.space, "2.5") - def test_from_future_import(self): source = """from __future__ import with_statement with somtehing as stuff: @@ -707,10 +702,6 @@ else: py.test.fail("Did not raise") -class TestECCompiler(BaseTestCompiler): - def setup_method(self, method): - self.compiler = self.space.getexecutioncontext().compiler - class AppTestCompiler: @@ -757,13 +748,6 @@ assert math.copysign(1., c[1]) == -1.0 -##class TestPythonAstCompiler(BaseTestCompiler): -## def setup_method(self, method): -## self.compiler = PythonAstCompiler(self.space, "2.4") - -## def test_try_except_finally(self): -## py.test.skip("unsupported") - class AppTestOptimizer: def test_remove_ending(self): From noreply at buildbot.pypy.org Fri Jan 27 20:31:35 2012 From: noreply at buildbot.pypy.org (edelsohn) Date: Fri, 27 Jan 2012 20:31:35 +0100 (CET) Subject: [pypy-commit] pypy ppc-jit-backend: Move definition of self.asm from __init__() to setup(). Message-ID: <20120127193135.3701382114@wyvern.cs.uni-duesseldorf.de> Author: edelsohn Branch: ppc-jit-backend Changeset: r51885:2743ff5a00f1 Date: 2012-01-27 14:30 -0500 http://bitbucket.org/pypy/pypy/changeset/2743ff5a00f1/ Log: Move definition of self.asm from __init__() to setup(). Define get_latest_force_token() and get_on_leave_jitted_hook(). diff --git a/pypy/jit/backend/ppc/runner.py b/pypy/jit/backend/ppc/runner.py --- a/pypy/jit/backend/ppc/runner.py +++ b/pypy/jit/backend/ppc/runner.py @@ -37,6 +37,8 @@ self.supports_floats = False self.total_compiled_loops = 0 self.total_compiled_bridges = 0 + + def setup(self): self.asm = AssemblerPPC(self) def setup_once(self): @@ -113,7 +115,13 @@ def get_latest_value_ref(self, index): return self.asm.fail_boxes_ptr.getitem(index) + + def get_latest_force_token(self): + return self.asm.fail_force_index + def get_on_leave_jitted_hook(self): + return self.asm.leave_jitted_hook + # walk through the given trace and generate machine code def _walk_trace_ops(self, codebuilder, operations): for op in operations: From noreply at buildbot.pypy.org Fri Jan 27 20:39:35 2012 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Fri, 27 Jan 2012 20:39:35 +0100 (CET) Subject: [pypy-commit] pypy py3k: tiny beautification Message-ID: <20120127193935.173AE82114@wyvern.cs.uni-duesseldorf.de> Author: Alex Gaynor <alex.gaynor at gmail.com> Branch: py3k Changeset: r51886:d41f30d6c1ca Date: 2012-01-27 14:39 -0500 http://bitbucket.org/pypy/pypy/changeset/d41f30d6c1ca/ Log: tiny beautification diff --git a/pypy/interpreter/test/test_compiler.py b/pypy/interpreter/test/test_compiler.py --- a/pypy/interpreter/test/test_compiler.py +++ b/pypy/interpreter/test/test_compiler.py @@ -443,7 +443,7 @@ snippet = str(py.code.Source(r''' d = {} d[...] = 12 - assert next(iter(d.keys())) is Ellipsis + assert next(iter(d)) is Ellipsis ''')) code = self.compiler.compile(snippet, '<tmp>', 'exec', 0) space = self.space From noreply at buildbot.pypy.org Fri Jan 27 21:08:20 2012 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 27 Jan 2012 21:08:20 +0100 (CET) Subject: [pypy-commit] pypy numpy-back-to-applevel: clean up scalar reshape and ravel Message-ID: <20120127200820.8A3CF82114@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: numpy-back-to-applevel Changeset: r51887:891a2ea64919 Date: 2012-01-27 22:07 +0200 http://bitbucket.org/pypy/pypy/changeset/891a2ea64919/ Log: clean up scalar reshape and ravel diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -274,7 +274,8 @@ def descr_flatten(self, space, w_order=None): if isinstance(self, Scalar): - return self.copy(space) + # scalars have no storage + return self.descr_reshape(space, [space.wrap([1])]) concr = self.get_concrete() w_res = concr.descr_ravel(space, w_order) if w_res.storage == concr.storage: @@ -479,8 +480,11 @@ w_shape = args_w[0] else: w_shape = space.newtuple(args_w) + new_shape = get_shape_from_iterable(space, self.size, w_shape) + return self.reshape(space, new_shape) + + def reshape(self, space, new_shape): concrete = self.get_concrete() - new_shape = get_shape_from_iterable(space, concrete.size, w_shape) # Since we got to here, prod(new_shape) == self.size new_strides = calc_new_strides(new_shape, concrete.shape, concrete.strides, concrete.order) @@ -693,6 +697,11 @@ def get_concrete_or_scalar(self): return self + def reshape(self, space, new_shape): + size = support.product(new_shape) + res = W_NDimArray(size, new_shape, self.dtype, 'C') + res.setitem(0, self.value) + return res class VirtualArray(BaseArray): """ diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -469,6 +469,13 @@ y = z.reshape(4, 3, 8) assert y.shape == (4, 3, 8) + def test_scalar_reshape(self): + from numpypy import array + a = array(3) + assert a.reshape([1, 1]).shape == (1, 1) + assert a.reshape([1]).shape == (1,) + raises(ValueError, "a.reshape(3)") + def test_add(self): from _numpypy import array a = array(range(5)) @@ -1104,6 +1111,7 @@ def test_flatten(self): from _numpypy import array + assert array(3).flatten().shape == (1,) a = array([[1, 2], [3, 4]]) b = a.flatten() c = a.ravel() From noreply at buildbot.pypy.org Fri Jan 27 21:24:34 2012 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 27 Jan 2012 21:24:34 +0100 (CET) Subject: [pypy-commit] pypy numpy-back-to-applevel: close merged branch Message-ID: <20120127202434.774F182114@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: numpy-back-to-applevel Changeset: r51888:a49c9222bf0e Date: 2012-01-27 22:13 +0200 http://bitbucket.org/pypy/pypy/changeset/a49c9222bf0e/ Log: close merged branch From noreply at buildbot.pypy.org Fri Jan 27 21:24:36 2012 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 27 Jan 2012 21:24:36 +0100 (CET) Subject: [pypy-commit] pypy default: (fijal, alex) Merge numpy-back-to-applevel branch. This branch removes Message-ID: <20120127202436.4567682114@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: Changeset: r51889:75784011b73f Date: 2012-01-27 22:23 +0200 http://bitbucket.org/pypy/pypy/changeset/75784011b73f/ Log: (fijal, alex) Merge numpy-back-to-applevel branch. This branch removes a bunch of functionality from micronumpy to put it back into applevel numpy. The main feature is printing of arrays, which required implementing quite a few features to work. diff --git a/lib_pypy/numpypy/core/__init__.py b/lib_pypy/numpypy/core/__init__.py --- a/lib_pypy/numpypy/core/__init__.py +++ b/lib_pypy/numpypy/core/__init__.py @@ -1,1 +1,2 @@ from .fromnumeric import * +from .numeric import * diff --git a/lib_pypy/numpypy/core/_methods.py b/lib_pypy/numpypy/core/_methods.py new file mode 100644 --- /dev/null +++ b/lib_pypy/numpypy/core/_methods.py @@ -0,0 +1,98 @@ +# Array methods which are called by the both the C-code for the method +# and the Python code for the NumPy-namespace function + +import _numpypy as mu +um = mu +#from numpypy.core import umath as um +from numpypy.core.numeric import asanyarray + +def _amax(a, axis=None, out=None, skipna=False, keepdims=False): + return um.maximum.reduce(a, axis=axis, + out=out, skipna=skipna, keepdims=keepdims) + +def _amin(a, axis=None, out=None, skipna=False, keepdims=False): + return um.minimum.reduce(a, axis=axis, + out=out, skipna=skipna, keepdims=keepdims) + +def _sum(a, axis=None, dtype=None, out=None, skipna=False, keepdims=False): + return um.add.reduce(a, axis=axis, dtype=dtype, + out=out, skipna=skipna, keepdims=keepdims) + +def _prod(a, axis=None, dtype=None, out=None, skipna=False, keepdims=False): + return um.multiply.reduce(a, axis=axis, dtype=dtype, + out=out, skipna=skipna, keepdims=keepdims) + +def _mean(a, axis=None, dtype=None, out=None, skipna=False, keepdims=False): + arr = asanyarray(a) + + # Upgrade bool, unsigned int, and int to float64 + if dtype is None and arr.dtype.kind in ['b','u','i']: + ret = um.add.reduce(arr, axis=axis, dtype='f8', + out=out, skipna=skipna, keepdims=keepdims) + else: + ret = um.add.reduce(arr, axis=axis, dtype=dtype, + out=out, skipna=skipna, keepdims=keepdims) + rcount = mu.count_reduce_items(arr, axis=axis, + skipna=skipna, keepdims=keepdims) + if isinstance(ret, mu.ndarray): + ret = um.true_divide(ret, rcount, + casting='unsafe', subok=False) + else: + ret = ret / float(rcount) + return ret + +def _var(a, axis=None, dtype=None, out=None, ddof=0, + skipna=False, keepdims=False): + arr = asanyarray(a) + + # First compute the mean, saving 'rcount' for reuse later + if dtype is None and arr.dtype.kind in ['b','u','i']: + arrmean = um.add.reduce(arr, axis=axis, dtype='f8', + skipna=skipna, keepdims=True) + else: + arrmean = um.add.reduce(arr, axis=axis, dtype=dtype, + skipna=skipna, keepdims=True) + rcount = mu.count_reduce_items(arr, axis=axis, + skipna=skipna, keepdims=True) + if isinstance(arrmean, mu.ndarray): + arrmean = um.true_divide(arrmean, rcount, + casting='unsafe', subok=False) + else: + arrmean = arrmean / float(rcount) + + # arr - arrmean + x = arr - arrmean + + # (arr - arrmean) ** 2 + if arr.dtype.kind == 'c': + x = um.multiply(x, um.conjugate(x)).real + else: + x = um.multiply(x, x) + + # add.reduce((arr - arrmean) ** 2, axis) + ret = um.add.reduce(x, axis=axis, dtype=dtype, out=out, + skipna=skipna, keepdims=keepdims) + + # add.reduce((arr - arrmean) ** 2, axis) / (n - ddof) + if not keepdims and isinstance(rcount, mu.ndarray): + rcount = rcount.squeeze(axis=axis) + rcount -= ddof + if isinstance(ret, mu.ndarray): + ret = um.true_divide(ret, rcount, + casting='unsafe', subok=False) + else: + ret = ret / float(rcount) + + return ret + +def _std(a, axis=None, dtype=None, out=None, ddof=0, + skipna=False, keepdims=False): + ret = _var(a, axis=axis, dtype=dtype, out=out, ddof=ddof, + skipna=skipna, keepdims=keepdims) + + if isinstance(ret, mu.ndarray): + ret = um.sqrt(ret) + else: + ret = um.sqrt(ret) + + return ret diff --git a/lib_pypy/numpypy/core/arrayprint.py b/lib_pypy/numpypy/core/arrayprint.py new file mode 100644 --- /dev/null +++ b/lib_pypy/numpypy/core/arrayprint.py @@ -0,0 +1,789 @@ +"""Array printing function + +$Id: arrayprint.py,v 1.9 2005/09/13 13:58:44 teoliphant Exp $ +""" +__all__ = ["array2string", "set_printoptions", "get_printoptions"] +__docformat__ = 'restructuredtext' + +# +# Written by Konrad Hinsen <hinsenk at ere.umontreal.ca> +# last revision: 1996-3-13 +# modified by Jim Hugunin 1997-3-3 for repr's and str's (and other details) +# and by Perry Greenfield 2000-4-1 for numarray +# and by Travis Oliphant 2005-8-22 for numpy + +import sys +import _numpypy as _nt +from _numpypy import maximum, minimum, absolute, not_equal, isinf, isnan, isna +#from _numpypy import format_longfloat, datetime_as_string, datetime_data +from .fromnumeric import ravel + + +def product(x, y): return x*y + +_summaryEdgeItems = 3 # repr N leading and trailing items of each dimension +_summaryThreshold = 1000 # total items > triggers array summarization + +_float_output_precision = 8 +_float_output_suppress_small = False +_line_width = 75 +_nan_str = 'nan' +_inf_str = 'inf' +_na_str = 'NA' +_formatter = None # formatting function for array elements + +if sys.version_info[0] >= 3: + from functools import reduce + +def set_printoptions(precision=None, threshold=None, edgeitems=None, + linewidth=None, suppress=None, + nanstr=None, infstr=None, nastr=None, + formatter=None): + """ + Set printing options. + + These options determine the way floating point numbers, arrays and + other NumPy objects are displayed. + + Parameters + ---------- + precision : int, optional + Number of digits of precision for floating point output (default 8). + threshold : int, optional + Total number of array elements which trigger summarization + rather than full repr (default 1000). + edgeitems : int, optional + Number of array items in summary at beginning and end of + each dimension (default 3). + linewidth : int, optional + The number of characters per line for the purpose of inserting + line breaks (default 75). + suppress : bool, optional + Whether or not suppress printing of small floating point values + using scientific notation (default False). + nanstr : str, optional + String representation of floating point not-a-number (default nan). + infstr : str, optional + String representation of floating point infinity (default inf). + nastr : str, optional + String representation of NA missing value (default NA). + formatter : dict of callables, optional + If not None, the keys should indicate the type(s) that the respective + formatting function applies to. Callables should return a string. + Types that are not specified (by their corresponding keys) are handled + by the default formatters. Individual types for which a formatter + can be set are:: + + - 'bool' + - 'int' + - 'timedelta' : a `numpy.timedelta64` + - 'datetime' : a `numpy.datetime64` + - 'float' + - 'longfloat' : 128-bit floats + - 'complexfloat' + - 'longcomplexfloat' : composed of two 128-bit floats + - 'numpy_str' : types `numpy.string_` and `numpy.unicode_` + - 'str' : all other strings + + Other keys that can be used to set a group of types at once are:: + + - 'all' : sets all types + - 'int_kind' : sets 'int' + - 'float_kind' : sets 'float' and 'longfloat' + - 'complex_kind' : sets 'complexfloat' and 'longcomplexfloat' + - 'str_kind' : sets 'str' and 'numpystr' + + See Also + -------- + get_printoptions, set_string_function, array2string + + Notes + ----- + `formatter` is always reset with a call to `set_printoptions`. + + Examples + -------- + Floating point precision can be set: + + >>> np.set_printoptions(precision=4) + >>> print np.array([1.123456789]) + [ 1.1235] + + Long arrays can be summarised: + + >>> np.set_printoptions(threshold=5) + >>> print np.arange(10) + [0 1 2 ..., 7 8 9] + + Small results can be suppressed: + + >>> eps = np.finfo(float).eps + >>> x = np.arange(4.) + >>> x**2 - (x + eps)**2 + array([ -4.9304e-32, -4.4409e-16, 0.0000e+00, 0.0000e+00]) + >>> np.set_printoptions(suppress=True) + >>> x**2 - (x + eps)**2 + array([-0., -0., 0., 0.]) + + A custom formatter can be used to display array elements as desired: + + >>> np.set_printoptions(formatter={'all':lambda x: 'int: '+str(-x)}) + >>> x = np.arange(3) + >>> x + array([int: 0, int: -1, int: -2]) + >>> np.set_printoptions() # formatter gets reset + >>> x + array([0, 1, 2]) + + To put back the default options, you can use: + + >>> np.set_printoptions(edgeitems=3,infstr='inf', + ... linewidth=75, nanstr='nan', precision=8, + ... suppress=False, threshold=1000, formatter=None) + """ + + global _summaryThreshold, _summaryEdgeItems, _float_output_precision, \ + _line_width, _float_output_suppress_small, _nan_str, _inf_str, \ + _na_str, _formatter + if linewidth is not None: + _line_width = linewidth + if threshold is not None: + _summaryThreshold = threshold + if edgeitems is not None: + _summaryEdgeItems = edgeitems + if precision is not None: + _float_output_precision = precision + if suppress is not None: + _float_output_suppress_small = not not suppress + if nanstr is not None: + _nan_str = nanstr + if infstr is not None: + _inf_str = infstr + if nastr is not None: + _na_str = nastr + _formatter = formatter + +def get_printoptions(): + """ + Return the current print options. + + Returns + ------- + print_opts : dict + Dictionary of current print options with keys + + - precision : int + - threshold : int + - edgeitems : int + - linewidth : int + - suppress : bool + - nanstr : str + - infstr : str + - formatter : dict of callables + + For a full description of these options, see `set_printoptions`. + + See Also + -------- + set_printoptions, set_string_function + + """ + d = dict(precision=_float_output_precision, + threshold=_summaryThreshold, + edgeitems=_summaryEdgeItems, + linewidth=_line_width, + suppress=_float_output_suppress_small, + nanstr=_nan_str, + infstr=_inf_str, + nastr=_na_str, + formatter=_formatter) + return d + +def _leading_trailing(a): + import numeric as _nc + if a.ndim == 1: + if len(a) > 2*_summaryEdgeItems: + b = _nc.concatenate((a[:_summaryEdgeItems], + a[-_summaryEdgeItems:])) + else: + b = a + else: + if len(a) > 2*_summaryEdgeItems: + l = [_leading_trailing(a[i]) for i in range( + min(len(a), _summaryEdgeItems))] + l.extend([_leading_trailing(a[-i]) for i in range( + min(len(a), _summaryEdgeItems),0,-1)]) + else: + l = [_leading_trailing(a[i]) for i in range(0, len(a))] + b = _nc.concatenate(tuple(l)) + return b + +def _boolFormatter(x): + if isna(x): + return str(x).replace('NA', _na_str, 1) + elif x: + return ' True' + else: + return 'False' + + +def repr_format(x): + if isna(x): + return str(x).replace('NA', _na_str, 1) + else: + return repr(x) + +def _array2string(a, max_line_width, precision, suppress_small, separator=' ', + prefix="", formatter=None): + + if max_line_width is None: + max_line_width = _line_width + + if precision is None: + precision = _float_output_precision + + if suppress_small is None: + suppress_small = _float_output_suppress_small + + if formatter is None: + formatter = _formatter + + if a.size > _summaryThreshold: + summary_insert = "..., " + data = _leading_trailing(a) + else: + summary_insert = "" + data = ravel(a) + + formatdict = {'bool' : _boolFormatter, + 'int' : IntegerFormat(data), + 'float' : FloatFormat(data, precision, suppress_small), + 'longfloat' : LongFloatFormat(precision), + #'complexfloat' : ComplexFormat(data, precision, + # suppress_small), + #'longcomplexfloat' : LongComplexFormat(precision), + #'datetime' : DatetimeFormat(data), + #'timedelta' : TimedeltaFormat(data), + 'numpystr' : repr_format, + 'str' : str} + + if formatter is not None: + fkeys = [k for k in formatter.keys() if formatter[k] is not None] + if 'all' in fkeys: + for key in formatdict.keys(): + formatdict[key] = formatter['all'] + if 'int_kind' in fkeys: + for key in ['int']: + formatdict[key] = formatter['int_kind'] + if 'float_kind' in fkeys: + for key in ['float', 'longfloat']: + formatdict[key] = formatter['float_kind'] + if 'complex_kind' in fkeys: + for key in ['complexfloat', 'longcomplexfloat']: + formatdict[key] = formatter['complex_kind'] + if 'str_kind' in fkeys: + for key in ['numpystr', 'str']: + formatdict[key] = formatter['str_kind'] + for key in formatdict.keys(): + if key in fkeys: + formatdict[key] = formatter[key] + + try: + format_function = a._format + msg = "The `_format` attribute is deprecated in Numpy 2.0 and " \ + "will be removed in 2.1. Use the `formatter` kw instead." + import warnings + warnings.warn(msg, DeprecationWarning) + except AttributeError: + # find the right formatting function for the array + dtypeobj = a.dtype.type + if issubclass(dtypeobj, _nt.bool_): + format_function = formatdict['bool'] + elif issubclass(dtypeobj, _nt.integer): + #if issubclass(dtypeobj, _nt.timedelta64): + # format_function = formatdict['timedelta'] + #else: + format_function = formatdict['int'] + elif issubclass(dtypeobj, _nt.floating): + #if issubclass(dtypeobj, _nt.longfloat): + # format_function = formatdict['longfloat'] + #else: + format_function = formatdict['float'] + elif issubclass(dtypeobj, _nt.complexfloating): + if issubclass(dtypeobj, _nt.clongfloat): + format_function = formatdict['longcomplexfloat'] + else: + format_function = formatdict['complexfloat'] + elif issubclass(dtypeobj, (_nt.unicode_, _nt.string_)): + format_function = formatdict['numpystr'] + elif issubclass(dtypeobj, _nt.datetime64): + format_function = formatdict['datetime'] + else: + format_function = formatdict['str'] + + # skip over "[" + next_line_prefix = " " + # skip over array( + next_line_prefix += " "*len(prefix) + + lst = _formatArray(a, format_function, len(a.shape), max_line_width, + next_line_prefix, separator, + _summaryEdgeItems, summary_insert)[:-1] + return lst + +def _convert_arrays(obj): + import numeric as _nc + newtup = [] + for k in obj: + if isinstance(k, _nc.ndarray): + k = k.tolist() + elif isinstance(k, tuple): + k = _convert_arrays(k) + newtup.append(k) + return tuple(newtup) + + +def array2string(a, max_line_width=None, precision=None, + suppress_small=None, separator=' ', prefix="", + style=repr, formatter=None): + """ + Return a string representation of an array. + + Parameters + ---------- + a : ndarray + Input array. + max_line_width : int, optional + The maximum number of columns the string should span. Newline + characters splits the string appropriately after array elements. + precision : int, optional + Floating point precision. Default is the current printing + precision (usually 8), which can be altered using `set_printoptions`. + suppress_small : bool, optional + Represent very small numbers as zero. A number is "very small" if it + is smaller than the current printing precision. + separator : str, optional + Inserted between elements. + prefix : str, optional + An array is typically printed as:: + + 'prefix(' + array2string(a) + ')' + + The length of the prefix string is used to align the + output correctly. + style : function, optional + A function that accepts an ndarray and returns a string. Used only + when the shape of `a` is equal to ``()``, i.e. for 0-D arrays. + formatter : dict of callables, optional + If not None, the keys should indicate the type(s) that the respective + formatting function applies to. Callables should return a string. + Types that are not specified (by their corresponding keys) are handled + by the default formatters. Individual types for which a formatter + can be set are:: + + - 'bool' + - 'int' + - 'timedelta' : a `numpy.timedelta64` + - 'datetime' : a `numpy.datetime64` + - 'float' + - 'longfloat' : 128-bit floats + - 'complexfloat' + - 'longcomplexfloat' : composed of two 128-bit floats + - 'numpy_str' : types `numpy.string_` and `numpy.unicode_` + - 'str' : all other strings + + Other keys that can be used to set a group of types at once are:: + + - 'all' : sets all types + - 'int_kind' : sets 'int' + - 'float_kind' : sets 'float' and 'longfloat' + - 'complex_kind' : sets 'complexfloat' and 'longcomplexfloat' + - 'str_kind' : sets 'str' and 'numpystr' + + Returns + ------- + array_str : str + String representation of the array. + + Raises + ------ + TypeError : if a callable in `formatter` does not return a string. + + See Also + -------- + array_str, array_repr, set_printoptions, get_printoptions + + Notes + ----- + If a formatter is specified for a certain type, the `precision` keyword is + ignored for that type. + + Examples + -------- + >>> x = np.array([1e-16,1,2,3]) + >>> print np.array2string(x, precision=2, separator=',', + ... suppress_small=True) + [ 0., 1., 2., 3.] + + >>> x = np.arange(3.) + >>> np.array2string(x, formatter={'float_kind':lambda x: "%.2f" % x}) + '[0.00 1.00 2.00]' + + >>> x = np.arange(3) + >>> np.array2string(x, formatter={'int':lambda x: hex(x)}) + '[0x0L 0x1L 0x2L]' + + """ + + if a.shape == (): + x = a.item() + if isna(x): + lst = str(x).replace('NA', _na_str, 1) + else: + try: + lst = a._format(x) + msg = "The `_format` attribute is deprecated in Numpy " \ + "2.0 and will be removed in 2.1. Use the " \ + "`formatter` kw instead." + import warnings + warnings.warn(msg, DeprecationWarning) + except AttributeError: + if isinstance(x, tuple): + x = _convert_arrays(x) + lst = style(x) + elif reduce(product, a.shape) == 0: + # treat as a null array if any of shape elements == 0 + lst = "[]" + else: + lst = _array2string(a, max_line_width, precision, suppress_small, + separator, prefix, formatter=formatter) + return lst + +def _extendLine(s, line, word, max_line_len, next_line_prefix): + if len(line.rstrip()) + len(word.rstrip()) >= max_line_len: + s += line.rstrip() + "\n" + line = next_line_prefix + line += word + return s, line + + +def _formatArray(a, format_function, rank, max_line_len, + next_line_prefix, separator, edge_items, summary_insert): + """formatArray is designed for two modes of operation: + + 1. Full output + + 2. Summarized output + + """ + if rank == 0: + obj = a.item() + if isinstance(obj, tuple): + obj = _convert_arrays(obj) + return str(obj) + + if summary_insert and 2*edge_items < len(a): + leading_items, trailing_items, summary_insert1 = \ + edge_items, edge_items, summary_insert + else: + leading_items, trailing_items, summary_insert1 = 0, len(a), "" + + if rank == 1: + s = "" + line = next_line_prefix + for i in xrange(leading_items): + word = format_function(a[i]) + separator + s, line = _extendLine(s, line, word, max_line_len, next_line_prefix) + + if summary_insert1: + s, line = _extendLine(s, line, summary_insert1, max_line_len, next_line_prefix) + + for i in xrange(trailing_items, 1, -1): + word = format_function(a[-i]) + separator + s, line = _extendLine(s, line, word, max_line_len, next_line_prefix) + + word = format_function(a[-1]) + s, line = _extendLine(s, line, word, max_line_len, next_line_prefix) + s += line + "]\n" + s = '[' + s[len(next_line_prefix):] + else: + s = '[' + sep = separator.rstrip() + for i in xrange(leading_items): + if i > 0: + s += next_line_prefix + s += _formatArray(a[i], format_function, rank-1, max_line_len, + " " + next_line_prefix, separator, edge_items, + summary_insert) + s = s.rstrip() + sep.rstrip() + '\n'*max(rank-1,1) + + if summary_insert1: + s += next_line_prefix + summary_insert1 + "\n" + + for i in xrange(trailing_items, 1, -1): + if leading_items or i != trailing_items: + s += next_line_prefix + s += _formatArray(a[-i], format_function, rank-1, max_line_len, + " " + next_line_prefix, separator, edge_items, + summary_insert) + s = s.rstrip() + sep.rstrip() + '\n'*max(rank-1,1) + if leading_items or trailing_items > 1: + s += next_line_prefix + s += _formatArray(a[-1], format_function, rank-1, max_line_len, + " " + next_line_prefix, separator, edge_items, + summary_insert).rstrip()+']\n' + return s + +class FloatFormat(object): + def __init__(self, data, precision, suppress_small, sign=False): + self.precision = precision + self.suppress_small = suppress_small + self.sign = sign + self.exp_format = False + self.large_exponent = False + self.max_str_len = 0 + #try: + self.fillFormat(data) + #except (TypeError, NotImplementedError): + # if reduce(data) fails, this instance will not be called, just + # instantiated in formatdict. + #pass + + def fillFormat(self, data): + import numeric as _nc + # XXX pypy unimplemented + #errstate = _nc.seterr(all='ignore') + try: + special = isnan(data) | isinf(data) | isna(data) + special[isna(data)] = False + valid = not_equal(data, 0) & ~special + valid[isna(data)] = False + non_zero = absolute(data.compress(valid)) + if len(non_zero) == 0: + max_val = 0. + min_val = 0. + else: + max_val = maximum.reduce(non_zero, skipna=True) + min_val = minimum.reduce(non_zero, skipna=True) + if max_val >= 1.e8: + self.exp_format = True + if not self.suppress_small and (min_val < 0.0001 + or max_val/min_val > 1000.): + self.exp_format = True + finally: + pass + # XXX pypy unimplemented + #_nc.seterr(**errstate) + + if self.exp_format: + self.large_exponent = 0 < min_val < 1e-99 or max_val >= 1e100 + self.max_str_len = 8 + self.precision + if self.large_exponent: + self.max_str_len += 1 + if self.sign: + format = '%+' + else: + format = '%' + format = format + '%d.%de' % (self.max_str_len, self.precision) + else: + format = '%%.%df' % (self.precision,) + if len(non_zero): + precision = max([_digits(x, self.precision, format) + for x in non_zero]) + else: + precision = 0 + precision = min(self.precision, precision) + self.max_str_len = len(str(int(max_val))) + precision + 2 + if special.any(): + self.max_str_len = max(self.max_str_len, + len(_nan_str), + len(_inf_str)+1, + len(_na_str)) + if self.sign: + format = '%#+' + else: + format = '%#' + format = format + '%d.%df' % (self.max_str_len, precision) + + self.special_fmt = '%%%ds' % (self.max_str_len,) + self.format = format + + def __call__(self, x, strip_zeros=True): + import numeric as _nc + #err = _nc.seterr(invalid='ignore') + try: + if isna(x): + return self.special_fmt % (str(x).replace('NA', _na_str, 1),) + elif isnan(x): + if self.sign: + return self.special_fmt % ('+' + _nan_str,) + else: + return self.special_fmt % (_nan_str,) + elif isinf(x): + if x > 0: + if self.sign: + return self.special_fmt % ('+' + _inf_str,) + else: + return self.special_fmt % (_inf_str,) + else: + return self.special_fmt % ('-' + _inf_str,) + finally: + pass + #_nc.seterr(**err) + + s = self.format % x + if self.large_exponent: + # 3-digit exponent + expsign = s[-3] + if expsign == '+' or expsign == '-': + s = s[1:-2] + '0' + s[-2:] + elif self.exp_format: + # 2-digit exponent + if s[-3] == '0': + s = ' ' + s[:-3] + s[-2:] + elif strip_zeros: + z = s.rstrip('0') + s = z + ' '*(len(s)-len(z)) + return s + + +def _digits(x, precision, format): + s = format % x + z = s.rstrip('0') + return precision - len(s) + len(z) + + +_MAXINT = sys.maxint +_MININT = -sys.maxint-1 +class IntegerFormat(object): + def __init__(self, data): + try: + max_str_len = max(len(str(maximum.reduce(data, skipna=True))), + len(str(minimum.reduce(data, skipna=True)))) + self.format = '%' + str(max_str_len) + 'd' + except TypeError, NotImplementedError: + # if reduce(data) fails, this instance will not be called, just + # instantiated in formatdict. + pass + except ValueError: + # this occurs when everything is NA + pass + + def __call__(self, x): + if isna(x): + return str(x).replace('NA', _na_str, 1) + elif _MININT < x < _MAXINT: + return self.format % x + else: + return "%s" % x + +class LongFloatFormat(object): + # XXX Have to add something to determine the width to use a la FloatFormat + # Right now, things won't line up properly + def __init__(self, precision, sign=False): + self.precision = precision + self.sign = sign + + def __call__(self, x): + if isna(x): + return str(x).replace('NA', _na_str, 1) + elif isnan(x): + if self.sign: + return '+' + _nan_str + else: + return ' ' + _nan_str + elif isinf(x): + if x > 0: + if self.sign: + return '+' + _inf_str + else: + return ' ' + _inf_str + else: + return '-' + _inf_str + elif x >= 0: + if self.sign: + return '+' + format_longfloat(x, self.precision) + else: + return ' ' + format_longfloat(x, self.precision) + else: + return format_longfloat(x, self.precision) + + +class LongComplexFormat(object): + def __init__(self, precision): + self.real_format = LongFloatFormat(precision) + self.imag_format = LongFloatFormat(precision, sign=True) + + def __call__(self, x): + if isna(x): + return str(x).replace('NA', _na_str, 1) + else: + r = self.real_format(x.real) + i = self.imag_format(x.imag) + return r + i + 'j' + + +class ComplexFormat(object): + def __init__(self, x, precision, suppress_small): + self.real_format = FloatFormat(x.real, precision, suppress_small) + self.imag_format = FloatFormat(x.imag, precision, suppress_small, + sign=True) + + def __call__(self, x): + if isna(x): + return str(x).replace('NA', _na_str, 1) + else: + r = self.real_format(x.real, strip_zeros=False) + i = self.imag_format(x.imag, strip_zeros=False) + if not self.imag_format.exp_format: + z = i.rstrip('0') + i = z + 'j' + ' '*(len(i)-len(z)) + else: + i = i + 'j' + return r + i + +class DatetimeFormat(object): + def __init__(self, x, unit=None, + timezone=None, casting='same_kind'): + # Get the unit from the dtype + if unit is None: + if x.dtype.kind == 'M': + unit = datetime_data(x.dtype)[0] + else: + unit = 's' + + # If timezone is default, make it 'local' or 'UTC' based on the unit + if timezone is None: + # Date units -> UTC, time units -> local + if unit in ('Y', 'M', 'W', 'D'): + self.timezone = 'UTC' + else: + self.timezone = 'local' + else: + self.timezone = timezone + self.unit = unit + self.casting = casting + + def __call__(self, x): + if isna(x): + return str(x).replace('NA', _na_str, 1) + else: + return "'%s'" % datetime_as_string(x, + unit=self.unit, + timezone=self.timezone, + casting=self.casting) + +class TimedeltaFormat(object): + def __init__(self, data): + if data.dtype.kind == 'm': + v = data.view('i8') + max_str_len = max(len(str(maximum.reduce(v))), + len(str(minimum.reduce(v)))) + self.format = '%' + str(max_str_len) + 'd' + + def __call__(self, x): + if isna(x): + return str(x).replace('NA', _na_str, 1) + else: + return self.format % x.astype('i8') + diff --git a/lib_pypy/numpypy/core/fromnumeric.py b/lib_pypy/numpypy/core/fromnumeric.py --- a/lib_pypy/numpypy/core/fromnumeric.py +++ b/lib_pypy/numpypy/core/fromnumeric.py @@ -30,7 +30,7 @@ 'rank', 'size', 'around', 'round_', 'mean', 'std', 'var', 'squeeze', 'amax', 'amin', ] - + def take(a, indices, axis=None, out=None, mode='raise'): """ Take elements from an array along an axis. @@ -1054,8 +1054,9 @@ array([ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]) """ - raise NotImplementedError('Waiting on interp level method') - + if not hasattr(a, 'ravel'): + a = numpypy.array(a) + return a.ravel(order=order) def nonzero(a): """ @@ -2324,13 +2325,12 @@ 0.44999999925552653 """ - assert axis is None assert dtype is None assert out is None assert ddof == 0 if not hasattr(a, "std"): a = numpypy.array(a) - return a.std() + return a.std(axis=axis) def var(a, axis=None, dtype=None, out=None, ddof=0): @@ -2421,10 +2421,9 @@ 0.20250000000000001 """ - assert axis is None assert dtype is None assert out is None assert ddof == 0 if not hasattr(a, "var"): a = numpypy.array(a) - return a.var() + return a.var(axis=axis) diff --git a/lib_pypy/numpypy/core/numeric.py b/lib_pypy/numpypy/core/numeric.py new file mode 100644 --- /dev/null +++ b/lib_pypy/numpypy/core/numeric.py @@ -0,0 +1,311 @@ + +from _numpypy import array, ndarray, int_, float_ #, complex_# , longlong +from _numpypy import concatenate +import sys +import _numpypy as multiarray # ARGH +from numpypy.core.arrayprint import array2string + + + +def asanyarray(a, dtype=None, order=None, maskna=None, ownmaskna=False): + """ + Convert the input to an ndarray, but pass ndarray subclasses through. + + Parameters + ---------- + a : array_like + Input data, in any form that can be converted to an array. This + includes scalars, lists, lists of tuples, tuples, tuples of tuples, + tuples of lists, and ndarrays. + dtype : data-type, optional + By default, the data-type is inferred from the input data. + order : {'C', 'F'}, optional + Whether to use row-major ('C') or column-major ('F') memory + representation. Defaults to 'C'. + maskna : bool or None, optional + If this is set to True, it forces the array to have an NA mask. + If this is set to False, it forces the array to not have an NA + mask. + ownmaskna : bool, optional + If this is set to True, forces the array to have a mask which + it owns. + + Returns + ------- + out : ndarray or an ndarray subclass + Array interpretation of `a`. If `a` is an ndarray or a subclass + of ndarray, it is returned as-is and no copy is performed. + + See Also + -------- + asarray : Similar function which always returns ndarrays. + ascontiguousarray : Convert input to a contiguous array. + asfarray : Convert input to a floating point ndarray. + asfortranarray : Convert input to an ndarray with column-major + memory order. + asarray_chkfinite : Similar function which checks input for NaNs and + Infs. + fromiter : Create an array from an iterator. + fromfunction : Construct an array by executing a function on grid + positions. + + Examples + -------- + Convert a list into an array: + + >>> a = [1, 2] + >>> np.asanyarray(a) + array([1, 2]) + + Instances of `ndarray` subclasses are passed through as-is: + + >>> a = np.matrix([1, 2]) + >>> np.asanyarray(a) is a + True + + """ + return array(a, dtype, copy=False, order=order, subok=True, + maskna=maskna, ownmaskna=ownmaskna) + +def base_repr(number, base=2, padding=0): + """ + Return a string representation of a number in the given base system. + + Parameters + ---------- + number : int + The value to convert. Only positive values are handled. + base : int, optional + Convert `number` to the `base` number system. The valid range is 2-36, + the default value is 2. + padding : int, optional + Number of zeros padded on the left. Default is 0 (no padding). + + Returns + ------- + out : str + String representation of `number` in `base` system. + + See Also + -------- + binary_repr : Faster version of `base_repr` for base 2. + + Examples + -------- + >>> np.base_repr(5) + '101' + >>> np.base_repr(6, 5) + '11' + >>> np.base_repr(7, base=5, padding=3) + '00012' + + >>> np.base_repr(10, base=16) + 'A' + >>> np.base_repr(32, base=16) + '20' + + """ + digits = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ' + if base > len(digits): + raise ValueError("Bases greater than 36 not handled in base_repr.") + + num = abs(number) + res = [] + while num: + res.append(digits[num % base]) + num //= base + if padding: + res.append('0' * padding) + if number < 0: + res.append('-') + return ''.join(reversed(res or '0')) + +_typelessdata = [int_, float_]#, complex_] +# XXX +#if issubclass(intc, int): +# _typelessdata.append(intc) + +#if issubclass(longlong, int): +# _typelessdata.append(longlong) + +def array_repr(arr, max_line_width=None, precision=None, suppress_small=None): + """ + Return the string representation of an array. + + Parameters + ---------- + arr : ndarray + Input array. + max_line_width : int, optional + The maximum number of columns the string should span. Newline + characters split the string appropriately after array elements. + precision : int, optional + Floating point precision. Default is the current printing precision + (usually 8), which can be altered using `set_printoptions`. + suppress_small : bool, optional + Represent very small numbers as zero, default is False. Very small + is defined by `precision`, if the precision is 8 then + numbers smaller than 5e-9 are represented as zero. + + Returns + ------- + string : str + The string representation of an array. + + See Also + -------- + array_str, array2string, set_printoptions + + Examples + -------- + >>> np.array_repr(np.array([1,2])) + 'array([1, 2])' + >>> np.array_repr(np.ma.array([0.])) + 'MaskedArray([ 0.])' + >>> np.array_repr(np.array([], np.int32)) + 'array([], dtype=int32)' + + >>> x = np.array([1e-6, 4e-7, 2, 3]) + >>> np.array_repr(x, precision=6, suppress_small=True) + 'array([ 0.000001, 0. , 2. , 3. ])' + + """ + if arr.size > 0 or arr.shape==(0,): + lst = array2string(arr, max_line_width, precision, suppress_small, + ', ', "array(") + else: # show zero-length shape unless it is (0,) + lst = "[], shape=%s" % (repr(arr.shape),) + + if arr.__class__ is not ndarray: + cName= arr.__class__.__name__ + else: + cName = "array" + + skipdtype = (arr.dtype.type in _typelessdata) and arr.size > 0 + + # XXX pypy lacks support + if 0 and arr.flags.maskna: + whichna = isna(arr) + # If nothing is NA, explicitly signal the NA-mask + if not any(whichna): + lst += ", maskna=True" + # If everything is NA, can't skip the dtype + if skipdtype and all(whichna): + skipdtype = False + + if skipdtype: + return "%s(%s)" % (cName, lst) + else: + typename = arr.dtype.name + # Quote typename in the output if it is "complex". + if typename and not (typename[0].isalpha() and typename.isalnum()): + typename = "'%s'" % typename + + lf = '' + if 0: # or issubclass(arr.dtype.type, flexible): + if arr.dtype.names: + typename = "%s" % str(arr.dtype) + else: + typename = "'%s'" % str(arr.dtype) + lf = '\n'+' '*len("array(") + return cName + "(%s, %sdtype=%s)" % (lst, lf, typename) + +def array_str(a, max_line_width=None, precision=None, suppress_small=None): + """ + Return a string representation of the data in an array. + + The data in the array is returned as a single string. This function is + similar to `array_repr`, the difference being that `array_repr` also + returns information on the kind of array and its data type. + + Parameters + ---------- + a : ndarray + Input array. + max_line_width : int, optional + Inserts newlines if text is longer than `max_line_width`. The + default is, indirectly, 75. + precision : int, optional + Floating point precision. Default is the current printing precision + (usually 8), which can be altered using `set_printoptions`. + suppress_small : bool, optional + Represent numbers "very close" to zero as zero; default is False. + Very close is defined by precision: if the precision is 8, e.g., + numbers smaller (in absolute value) than 5e-9 are represented as + zero. + + See Also + -------- + array2string, array_repr, set_printoptions + + Examples + -------- + >>> np.array_str(np.arange(3)) + '[0 1 2]' + + """ + return array2string(a, max_line_width, precision, suppress_small, ' ', "", str) + +def set_string_function(f, repr=True): + """ + Set a Python function to be used when pretty printing arrays. + + Parameters + ---------- + f : function or None + Function to be used to pretty print arrays. The function should expect + a single array argument and return a string of the representation of + the array. If None, the function is reset to the default NumPy function + to print arrays. + repr : bool, optional + If True (default), the function for pretty printing (``__repr__``) + is set, if False the function that returns the default string + representation (``__str__``) is set. + + See Also + -------- + set_printoptions, get_printoptions + + Examples + -------- + >>> def pprint(arr): + ... return 'HA! - What are you going to do now?' + ... + >>> np.set_string_function(pprint) + >>> a = np.arange(10) + >>> a + HA! - What are you going to do now? + >>> print a + [0 1 2 3 4 5 6 7 8 9] + + We can reset the function to the default: + + >>> np.set_string_function(None) + >>> a + array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) + + `repr` affects either pretty printing or normal string representation. + Note that ``__repr__`` is still affected by setting ``__str__`` + because the width of each array element in the returned string becomes + equal to the length of the result of ``__str__()``. + + >>> x = np.arange(4) + >>> np.set_string_function(lambda x:'random', repr=False) + >>> x.__str__() + 'random' + >>> x.__repr__() + 'array([ 0, 1, 2, 3])' + + """ + if f is None: + if repr: + return multiarray.set_string_function(array_repr, 1) + else: + return multiarray.set_string_function(array_str, 0) + else: + return multiarray.set_string_function(f, repr) + +set_string_function(array_str, 0) +set_string_function(array_repr, 1) + +little_endian = (sys.byteorder == 'little') diff --git a/pypy/annotation/unaryop.py b/pypy/annotation/unaryop.py --- a/pypy/annotation/unaryop.py +++ b/pypy/annotation/unaryop.py @@ -440,6 +440,12 @@ def method_popitem(dct): return dct.getanyitem('items') + def method_pop(dct, s_key, s_dfl=None): + dct.dictdef.generalize_key(s_key) + if s_dfl is not None: + dct.dictdef.generalize_value(s_dfl) + return dct.dictdef.read_value() + def _can_only_throw(dic, *ignore): if dic1.dictdef.dictkey.custom_eq_hash: return None # r_dict: can throw anything diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py --- a/pypy/module/micronumpy/__init__.py +++ b/pypy/module/micronumpy/__init__.py @@ -30,6 +30,10 @@ 'isna': 'interp_numarray.isna', 'concatenate': 'interp_numarray.concatenate', + 'set_string_function': 'appbridge.set_string_function', + + 'count_reduce_items': 'interp_numarray.count_reduce_items', + 'True_': 'types.Bool.True', 'False_': 'types.Bool.False', @@ -68,6 +72,7 @@ ("copysign", "copysign"), ("cos", "cos"), ("divide", "divide"), + ("true_divide", "true_divide"), ("equal", "equal"), ("exp", "exp"), ("fabs", "fabs"), @@ -90,6 +95,9 @@ ("tan", "tan"), ('bitwise_and', 'bitwise_and'), ('bitwise_or', 'bitwise_or'), + ('bitwise_not', 'invert'), + ('isnan', 'isnan'), + ('isinf', 'isinf'), ]: interpleveldefs[exposed] = "interp_ufuncs.get(space).%s" % impl diff --git a/pypy/module/micronumpy/appbridge.py b/pypy/module/micronumpy/appbridge.py new file mode 100644 --- /dev/null +++ b/pypy/module/micronumpy/appbridge.py @@ -0,0 +1,38 @@ + +from pypy.rlib.objectmodel import specialize + +class AppBridgeCache(object): + w__var = None + w__std = None + w_module = None + w_array_repr = None + w_array_str = None + + def __init__(self, space): + self.w_import = space.appexec([], """(): + def f(): + import sys + __import__('numpypy.core._methods') + return sys.modules['numpypy.core._methods'] + return f + """) + + @specialize.arg(2) + def call_method(self, space, name, *args): + w_meth = getattr(self, 'w_' + name) + if w_meth is None: + if self.w_module is None: + self.w_module = space.call_function(self.w_import) + w_meth = space.getattr(self.w_module, space.wrap(name)) + setattr(self, 'w_' + name, w_meth) + return space.call_function(w_meth, *args) + +def set_string_function(space, w_f, w_repr): + cache = get_appbridge_cache(space) + if space.is_true(w_repr): + cache.w_array_repr = w_f + else: + cache.w_array_str = w_f + +def get_appbridge_cache(space): + return space.fromcache(AppBridgeCache) diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py --- a/pypy/module/micronumpy/compile.py +++ b/pypy/module/micronumpy/compile.py @@ -34,12 +34,14 @@ SINGLE_ARG_FUNCTIONS = ["sum", "prod", "max", "min", "all", "any", "unegative", "flat"] +TWO_ARG_FUNCTIONS = ['take'] class FakeSpace(object): w_ValueError = None w_TypeError = None w_IndexError = None w_OverflowError = None + w_NotImplementedError = None w_None = None w_bool = "bool" @@ -54,6 +56,9 @@ """NOT_RPYTHON""" self.fromcache = InternalSpaceCache(self).getorbuild + def _freeze_(self): + return True + def issequence_w(self, w_obj): return isinstance(w_obj, ListObject) or isinstance(w_obj, W_NDimArray) @@ -145,6 +150,9 @@ def allocate_instance(self, klass, w_subtype): return instantiate(klass) + def newtuple(self, list_w): + raise ValueError + def len_w(self, w_obj): if isinstance(w_obj, ListObject): return len(w_obj.items) @@ -372,12 +380,12 @@ for arg in self.args])) def execute(self, interp): + arr = self.args[0].execute(interp) + if not isinstance(arr, BaseArray): + raise ArgumentNotAnArray if self.name in SINGLE_ARG_FUNCTIONS: if len(self.args) != 1 and self.name != 'sum': raise ArgumentMismatch - arr = self.args[0].execute(interp) - if not isinstance(arr, BaseArray): - raise ArgumentNotAnArray if self.name == "sum": if len(self.args)>1: w_res = arr.descr_sum(interp.space, @@ -401,19 +409,27 @@ w_res = arr.descr_get_flatiter(interp.space) else: assert False # unreachable code - if isinstance(w_res, BaseArray): - return w_res - if isinstance(w_res, FloatObject): - dtype = get_dtype_cache(interp.space).w_float64dtype - elif isinstance(w_res, BoolObject): - dtype = get_dtype_cache(interp.space).w_booldtype - elif isinstance(w_res, interp_boxes.W_GenericBox): - dtype = w_res.get_dtype(interp.space) + elif self.name in TWO_ARG_FUNCTIONS: + arg = self.args[1].execute(interp) + if not isinstance(arg, BaseArray): + raise ArgumentNotAnArray + if self.name == 'take': + w_res = arr.descr_take(interp.space, arg) else: - dtype = None - return scalar_w(interp.space, dtype, w_res) + assert False # unreachable else: raise WrongFunctionName + if isinstance(w_res, BaseArray): + return w_res + if isinstance(w_res, FloatObject): + dtype = get_dtype_cache(interp.space).w_float64dtype + elif isinstance(w_res, BoolObject): + dtype = get_dtype_cache(interp.space).w_booldtype + elif isinstance(w_res, interp_boxes.W_GenericBox): + dtype = w_res.get_dtype(interp.space) + else: + dtype = None + return scalar_w(interp.space, dtype, w_res) _REGEXES = [ ('-?[\d\.]+', 'number'), diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py --- a/pypy/module/micronumpy/interp_boxes.py +++ b/pypy/module/micronumpy/interp_boxes.py @@ -8,8 +8,8 @@ from pypy.tool.sourcetools import func_with_new_name +MIXIN_32 = (int_typedef,) if LONG_BIT == 32 else () MIXIN_64 = (int_typedef,) if LONG_BIT == 64 else () -MIXIN_32 = () if LONG_BIT == 64 else (int_typedef,) def new_dtype_getter(name): def get_dtype(space): @@ -29,6 +29,7 @@ def convert_to(self, dtype): return dtype.box(self.value) + class W_GenericBox(Wrappable): _attrs_ = () @@ -94,7 +95,7 @@ descr_neg = _unaryop_impl("negative") descr_abs = _unaryop_impl("absolute") - def descr_tolist(self, space): + def item(self, space): return self.get_dtype(space).itemtype.to_builtin_type(space, self) @@ -105,7 +106,8 @@ _attrs_ = () class W_IntegerBox(W_NumberBox): - pass + def int_w(self, space): + return space.int_w(self.descr_int(space)) class W_SignedIntegerBox(W_IntegerBox): pass @@ -188,7 +190,7 @@ __neg__ = interp2app(W_GenericBox.descr_neg), __abs__ = interp2app(W_GenericBox.descr_abs), - tolist = interp2app(W_GenericBox.descr_tolist), + tolist = interp2app(W_GenericBox.item), ) W_BoolBox.typedef = TypeDef("bool_", W_GenericBox.typedef, @@ -278,3 +280,4 @@ __new__ = interp2app(W_Float64Box.descr__new__.im_func), ) + diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py --- a/pypy/module/micronumpy/interp_dtype.py +++ b/pypy/module/micronumpy/interp_dtype.py @@ -100,7 +100,8 @@ return space.wrap(not self.eq(space, w_other)) def is_int_type(self): - return self.kind == SIGNEDLTR or self.kind == UNSIGNEDLTR + return (self.kind == SIGNEDLTR or self.kind == UNSIGNEDLTR or + self.kind == BOOLLTR) def is_bool_type(self): return self.kind == BOOLLTR @@ -119,6 +120,7 @@ type = interp_attrproperty_w("w_box_type", cls=W_Dtype), itemsize = GetSetProperty(W_Dtype.descr_get_itemsize), shape = GetSetProperty(W_Dtype.descr_get_shape), + name = interp_attrproperty('name', cls=W_Dtype), ) W_Dtype.typedef.acceptable_as_base_class = False diff --git a/pypy/module/micronumpy/interp_iter.py b/pypy/module/micronumpy/interp_iter.py --- a/pypy/module/micronumpy/interp_iter.py +++ b/pypy/module/micronumpy/interp_iter.py @@ -230,8 +230,13 @@ class AxisIterator(BaseIterator): def __init__(self, start, dim, shape, strides, backstrides): self.res_shape = shape[:] - self.strides = strides[:dim] + [0] + strides[dim:] - self.backstrides = backstrides[:dim] + [0] + backstrides[dim:] + if len(shape) == len(strides): + # keepdims = True + self.strides = strides[:dim] + [0] + strides[dim + 1:] + self.backstrides = backstrides[:dim] + [0] + backstrides[dim + 1:] + else: + self.strides = strides[:dim] + [0] + strides[dim:] + self.backstrides = backstrides[:dim] + [0] + backstrides[dim:] self.first_line = True self.indices = [0] * len(shape) self._done = False diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -1,16 +1,20 @@ from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.error import OperationError, operationerrfmt -from pypy.interpreter.gateway import interp2app, NoneNotWrapped, unwrap_spec +from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.interpreter.typedef import TypeDef, GetSetProperty -from pypy.module.micronumpy import interp_ufuncs, interp_dtype, signature,\ - interp_boxes -from pypy.module.micronumpy.strides import calculate_slice_strides, to_coords +from pypy.module.micronumpy import (interp_ufuncs, interp_dtype, interp_boxes, + signature, support) +from pypy.module.micronumpy.strides import (calculate_slice_strides, + shape_agreement, find_shape_and_elems, get_shape_from_iterable, + calc_new_strides, to_coords) from pypy.rlib import jit from pypy.rpython.lltypesystem import lltype, rffi from pypy.tool.sourcetools import func_with_new_name from pypy.rlib.rstring import StringBuilder -from pypy.module.micronumpy.interp_iter import ArrayIterator, OneDimIterator,\ - SkipLastAxisIterator, Chunk, ViewIterator +from pypy.module.micronumpy.interp_iter import (ArrayIterator, OneDimIterator, + SkipLastAxisIterator, Chunk, ViewIterator) +from pypy.module.micronumpy.appbridge import get_appbridge_cache + numpy_driver = jit.JitDriver( greens=['shapelen', 'sig'], @@ -58,189 +62,22 @@ reds=['idx', 'idxi', 'frame', 'arr'], name='numpy_filterset', ) - +take_driver = jit.JitDriver( + greens=['shapelen', 'sig'], + reds=['index_i', 'res_i', 'concr', 'index', 'res'], + name='numpy_take', +) flat_get_driver = jit.JitDriver( greens=['shapelen', 'base'], reds=['step', 'ri', 'basei', 'res'], name='numpy_flatget', ) - flat_set_driver = jit.JitDriver( greens=['shapelen', 'base'], reds=['step', 'ai', 'lngth', 'arr', 'basei'], name='numpy_flatset', ) -def _find_shape_and_elems(space, w_iterable): - shape = [space.len_w(w_iterable)] - batch = space.listview(w_iterable) - while True: - new_batch = [] - if not batch: - return shape, [] - if not space.issequence_w(batch[0]): - for elem in batch: - if space.issequence_w(elem): - raise OperationError(space.w_ValueError, space.wrap( - "setting an array element with a sequence")) - return shape, batch - size = space.len_w(batch[0]) - for w_elem in batch: - if not space.issequence_w(w_elem) or space.len_w(w_elem) != size: - raise OperationError(space.w_ValueError, space.wrap( - "setting an array element with a sequence")) - new_batch += space.listview(w_elem) - shape.append(size) - batch = new_batch - -def shape_agreement(space, shape1, shape2): - ret = _shape_agreement(shape1, shape2) - if len(ret) < max(len(shape1), len(shape2)): - raise OperationError(space.w_ValueError, - space.wrap("operands could not be broadcast together with shapes (%s) (%s)" % ( - ",".join([str(x) for x in shape1]), - ",".join([str(x) for x in shape2]), - )) - ) - return ret - -def _shape_agreement(shape1, shape2): - """ Checks agreement about two shapes with respect to broadcasting. Returns - the resulting shape. - """ - lshift = 0 - rshift = 0 - if len(shape1) > len(shape2): - m = len(shape1) - n = len(shape2) - rshift = len(shape2) - len(shape1) - remainder = shape1 - else: - m = len(shape2) - n = len(shape1) - lshift = len(shape1) - len(shape2) - remainder = shape2 - endshape = [0] * m - indices1 = [True] * m - indices2 = [True] * m - for i in range(m - 1, m - n - 1, -1): - left = shape1[i + lshift] - right = shape2[i + rshift] - if left == right: - endshape[i] = left - elif left == 1: - endshape[i] = right - indices1[i + lshift] = False - elif right == 1: - endshape[i] = left - indices2[i + rshift] = False - else: - return [] - #raise OperationError(space.w_ValueError, space.wrap( - # "frames are not aligned")) - for i in range(m - n): - endshape[i] = remainder[i] - return endshape - -def get_shape_from_iterable(space, old_size, w_iterable): - new_size = 0 - new_shape = [] - if space.isinstance_w(w_iterable, space.w_int): - new_size = space.int_w(w_iterable) - if new_size < 0: - new_size = old_size - new_shape = [new_size] - else: - neg_dim = -1 - batch = space.listview(w_iterable) - new_size = 1 - if len(batch) < 1: - if old_size == 1: - # Scalars can have an empty size. - new_size = 1 - else: - new_size = 0 - new_shape = [] - i = 0 - for elem in batch: - s = space.int_w(elem) - if s < 0: - if neg_dim >= 0: - raise OperationError(space.w_ValueError, space.wrap( - "can only specify one unknown dimension")) - s = 1 - neg_dim = i - new_size *= s - new_shape.append(s) - i += 1 - if neg_dim >= 0: - new_shape[neg_dim] = old_size / new_size - new_size *= new_shape[neg_dim] - if new_size != old_size: - raise OperationError(space.w_ValueError, - space.wrap("total size of new array must be unchanged")) - return new_shape - -# Recalculating strides. Find the steps that the iteration does for each -# dimension, given the stride and shape. Then try to create a new stride that -# fits the new shape, using those steps. If there is a shape/step mismatch -# (meaning that the realignment of elements crosses from one step into another) -# return None so that the caller can raise an exception. -def calc_new_strides(new_shape, old_shape, old_strides, order): - # Return the proper strides for new_shape, or None if the mapping crosses - # stepping boundaries - - # Assumes that prod(old_shape) == prod(new_shape), len(old_shape) > 1, and - # len(new_shape) > 0 - steps = [] - last_step = 1 - oldI = 0 - new_strides = [] - if order == 'F': - for i in range(len(old_shape)): - steps.append(old_strides[i] / last_step) - last_step *= old_shape[i] - cur_step = steps[0] - n_new_elems_used = 1 - n_old_elems_to_use = old_shape[0] - for s in new_shape: - new_strides.append(cur_step * n_new_elems_used) - n_new_elems_used *= s - while n_new_elems_used > n_old_elems_to_use: - oldI += 1 - if steps[oldI] != steps[oldI - 1]: - return None - n_old_elems_to_use *= old_shape[oldI] - if n_new_elems_used == n_old_elems_to_use: - oldI += 1 - if oldI < len(old_shape): - cur_step = steps[oldI] - n_old_elems_to_use *= old_shape[oldI] - elif order == 'C': - for i in range(len(old_shape) - 1, -1, -1): - steps.insert(0, old_strides[i] / last_step) - last_step *= old_shape[i] - cur_step = steps[-1] - n_new_elems_used = 1 - oldI = -1 - n_old_elems_to_use = old_shape[-1] - for i in range(len(new_shape) - 1, -1, -1): - s = new_shape[i] - new_strides.insert(0, cur_step * n_new_elems_used) - n_new_elems_used *= s - while n_new_elems_used > n_old_elems_to_use: - oldI -= 1 - if steps[oldI] != steps[oldI + 1]: - return None - n_old_elems_to_use *= old_shape[oldI] - if n_new_elems_used == n_old_elems_to_use: - oldI -= 1 - if oldI >= -len(old_shape): - cur_step = steps[oldI] - n_old_elems_to_use *= old_shape[oldI] - assert len(new_strides) == len(new_shape) - return new_strides - class BaseArray(Wrappable): _attrs_ = ["invalidates", "shape", 'size'] @@ -280,6 +117,7 @@ descr_pos = _unaryop_impl("positive") descr_neg = _unaryop_impl("negative") descr_abs = _unaryop_impl("absolute") + descr_invert = _unaryop_impl("invert") def _binop_impl(ufunc_name): def impl(self, space, w_other): @@ -322,9 +160,11 @@ def _reduce_ufunc_impl(ufunc_name, promote_to_largest=False): def impl(self, space, w_axis=None): if space.is_w(w_axis, space.w_None): - w_axis = space.wrap(-1) + axis = -1 + else: + axis = space.int_w(w_axis) return getattr(interp_ufuncs.get(space), ufunc_name).reduce(space, - self, True, promote_to_largest, w_axis) + self, True, promote_to_largest, axis) return func_with_new_name(impl, "reduce_%s_impl" % ufunc_name) descr_sum = _reduce_ufunc_impl("add") @@ -442,21 +282,22 @@ def descr_copy(self, space): return self.copy(space) - def descr_flatten(self, space): - return self.flatten(space) + def descr_flatten(self, space, w_order=None): + if isinstance(self, Scalar): + # scalars have no storage + return self.descr_reshape(space, [space.wrap([1])]) + concr = self.get_concrete() + w_res = concr.descr_ravel(space, w_order) + if w_res.storage == concr.storage: + return w_res.copy(space) + return w_res def copy(self, space): return self.get_concrete().copy(space) def empty_copy(self, space, dtype): shape = self.shape - size = 1 - for elem in shape: - size *= elem - return W_NDimArray(size, shape[:], dtype, 'C') - - def flatten(self, space): - return self.get_concrete().flatten(space) + return W_NDimArray(support.product(shape), shape[:], dtype, 'C') def descr_len(self, space): if len(self.shape): @@ -465,33 +306,32 @@ "len() of unsized object")) def descr_repr(self, space): - res = StringBuilder() - res.append("array(") - concrete = self.get_concrete_or_scalar() - dtype = concrete.find_dtype() - if not concrete.size: - res.append('[]') - if len(self.shape) > 1: - # An empty slice reports its shape - res.append(", shape=(") - self_shape = str(self.shape) - res.append_slice(str(self_shape), 1, len(self_shape) - 1) - res.append(')') - else: - concrete.to_str(space, 1, res, indent=' ') - if (dtype is not interp_dtype.get_dtype_cache(space).w_float64dtype and - not (dtype.kind == interp_dtype.SIGNEDLTR and - dtype.itemtype.get_element_size() == rffi.sizeof(lltype.Signed)) or - not self.size): - res.append(", dtype=" + dtype.name) - res.append(")") - return space.wrap(res.build()) + cache = get_appbridge_cache(space) + if cache.w_array_repr is None: + return space.wrap(self.dump_data()) + return space.call_function(cache.w_array_repr, self) + + def dump_data(self): + concr = self.get_concrete() + i = concr.create_iter() + first = True + s = StringBuilder() + s.append('array([') + while not i.done(): + if first: + first = False + else: + s.append(', ') + s.append(concr.dtype.itemtype.str_format(concr.getitem(i.offset))) + i = i.next(len(concr.shape)) + s.append('])') + return s.build() def descr_str(self, space): - ret = StringBuilder() - concrete = self.get_concrete_or_scalar() - concrete.to_str(space, 0, ret, ' ') - return space.wrap(ret.build()) + cache = get_appbridge_cache(space) + if cache.w_array_str is None: + return space.wrap(self.dump_data()) + return space.call_function(cache.w_array_str, self) @jit.unroll_safe def _single_item_result(self, space, w_idx): @@ -531,7 +371,7 @@ def count_all_true(self, arr): sig = arr.find_sig() - frame = sig.create_frame(self) + frame = sig.create_frame(arr) shapelen = len(arr.shape) s = 0 iter = None @@ -545,6 +385,9 @@ def getitem_filter(self, space, arr): concr = arr.get_concrete() + if concr.size > self.size: + raise OperationError(space.w_IndexError, + space.wrap("index out of range for array")) size = self.count_all_true(concr) res = W_NDimArray(size, [size], self.find_dtype()) ri = ArrayIterator(size) @@ -553,7 +396,7 @@ sig = self.find_sig() frame = sig.create_frame(self) v = None - while not frame.done(): + while not ri.done(): filter_driver.jit_merge_point(concr=concr, argi=argi, ri=ri, frame=frame, v=v, res=res, sig=sig, shapelen=shapelen, self=self) @@ -593,7 +436,7 @@ item = concrete._index_of_single_item(space, w_idx) return concrete.getitem(item) chunks = self._prepare_slice_args(space, w_idx) - return space.wrap(self.create_slice(chunks)) + return self.create_slice(chunks) def descr_setitem(self, space, w_idx, w_value): self.invalidated() @@ -647,8 +490,11 @@ w_shape = args_w[0] else: w_shape = space.newtuple(args_w) + new_shape = get_shape_from_iterable(space, self.size, w_shape) + return self.reshape(space, new_shape) + + def reshape(self, space, new_shape): concrete = self.get_concrete() - new_shape = get_shape_from_iterable(space, concrete.size, w_shape) # Since we got to here, prod(new_shape) == self.size new_strides = calc_new_strides(new_shape, concrete.shape, concrete.strides, concrete.order) @@ -659,7 +505,7 @@ for nd in range(ndims): new_backstrides[nd] = (new_shape[nd] - 1) * new_strides[nd] arr = W_NDimSlice(concrete.start, new_strides, new_backstrides, - new_shape, self) + new_shape, concrete) else: # Create copy with contiguous data arr = concrete.copy(space) @@ -669,7 +515,7 @@ def descr_tolist(self, space): if len(self.shape) == 0: assert isinstance(self, Scalar) - return self.value.descr_tolist(space) + return self.value.item(space) w_result = space.newlist([]) for i in range(self.shape[0]): space.call_method(w_result, "append", @@ -686,17 +532,13 @@ w_denom = space.wrap(self.shape[dim]) return space.div(self.descr_sum_promote(space, w_axis), w_denom) - def descr_var(self, space): - # var = mean((values - mean(values)) ** 2) - w_res = self.descr_sub(space, self.descr_mean(space, space.w_None)) - assert isinstance(w_res, BaseArray) - w_res = w_res.descr_pow(space, space.wrap(2)) - assert isinstance(w_res, BaseArray) - return w_res.descr_mean(space, space.w_None) + def descr_var(self, space, w_axis=None): + return get_appbridge_cache(space).call_method(space, '_var', self, + w_axis) - def descr_std(self, space): - # std(v) = sqrt(var(v)) - return interp_ufuncs.get(space).sqrt.call(space, [self.descr_var(space)]) + def descr_std(self, space, w_axis=None): + return get_appbridge_cache(space).call_method(space, '_std', self, + w_axis) def descr_fill(self, space, w_value): concr = self.get_concrete_or_scalar() @@ -729,6 +571,16 @@ return space.wrap(W_NDimSlice(concrete.start, strides, backstrides, shape, concrete)) + def descr_ravel(self, space, w_order=None): + if w_order is None or space.is_w(w_order, space.w_None): + order = 'C' + else: + order = space.str_w(w_order) + if order != 'C': + raise OperationError(space.w_NotImplementedError, space.wrap( + "order not implemented")) + return self.descr_reshape(space, [space.wrap(-1)]) + def descr_get_flatiter(self, space): return space.wrap(W_FlatIterator(self)) @@ -758,6 +610,60 @@ def supports_fast_slicing(self): return False + def descr_compress(self, space, w_obj, w_axis=None): + index = convert_to_array(space, w_obj) + return self.getitem_filter(space, index) + + def descr_take(self, space, w_obj, w_axis=None): + index = convert_to_array(space, w_obj).get_concrete() + concr = self.get_concrete() + if space.is_w(w_axis, space.w_None): + concr = concr.descr_ravel(space) + else: + raise OperationError(space.w_NotImplementedError, + space.wrap("axis unsupported for take")) + index_i = index.create_iter() + res_shape = index.shape + size = support.product(res_shape) + res = W_NDimArray(size, res_shape[:], concr.dtype, concr.order) + res_i = res.create_iter() + shapelen = len(index.shape) + sig = concr.find_sig() + while not index_i.done(): + take_driver.jit_merge_point(index_i=index_i, index=index, + res_i=res_i, concr=concr, + res=res, + shapelen=shapelen, sig=sig) + w_item = index._getitem_long(space, index_i.offset) + res.setitem(res_i.offset, concr.descr_getitem(space, w_item)) + index_i = index_i.next(shapelen) + res_i = res_i.next(shapelen) + return res + + def _getitem_long(self, space, offset): + # an obscure hack to not have longdtype inside a jitted loop + longdtype = interp_dtype.get_dtype_cache(space).w_longdtype + return self.getitem(offset).convert_to(longdtype).item( + space) + + def descr_item(self, space, w_arg=None): + if space.is_w(w_arg, space.w_None): + if not isinstance(self, Scalar): + raise OperationError(space.w_ValueError, space.wrap("index out of bounds")) + return self.value.item(space) + if space.isinstance_w(w_arg, space.w_int): + if isinstance(self, Scalar): + raise OperationError(space.w_ValueError, space.wrap("index out of bounds")) + concr = self.get_concrete() + i = to_coords(space, self.shape, concr.size, concr.order, w_arg)[0] + # XXX a bit around + item = self.descr_getitem(space, space.newtuple([space.wrap(x) + for x in i])) + assert isinstance(item, interp_boxes.W_GenericBox) + return item.item(space) + raise OperationError(space.w_NotImplementedError, space.wrap( + "non-int arg not supported")) + def convert_to_array(space, w_obj): if isinstance(w_obj, BaseArray): return w_obj @@ -783,22 +689,15 @@ self.shape = [] BaseArray.__init__(self, []) self.dtype = dtype + assert isinstance(value, interp_boxes.W_GenericBox) self.value = value def find_dtype(self): return self.dtype - def to_str(self, space, comma, builder, indent=' ', use_ellipsis=False): - builder.append(self.dtype.itemtype.str_format(self.value)) - def copy(self, space): return Scalar(self.dtype, self.value) - def flatten(self, space): - array = W_NDimArray(self.size, [self.size], self.dtype) - array.setitem(0, self.value) - return array - def fill(self, space, w_value): self.value = self.dtype.coerce(space, w_value) @@ -808,6 +707,11 @@ def get_concrete_or_scalar(self): return self + def reshape(self, space, new_shape): + size = support.product(new_shape) + res = W_NDimArray(size, new_shape, self.dtype, 'C') + res.setitem(0, self.value) + return res class VirtualArray(BaseArray): """ @@ -864,12 +768,9 @@ class VirtualSlice(VirtualArray): def __init__(self, child, chunks, shape): - size = 1 - for sh in shape: - size *= sh self.child = child self.chunks = chunks - self.size = size + self.size = support.product(shape) VirtualArray.__init__(self, 'slice', shape, child.find_dtype()) def create_sig(self): @@ -888,11 +789,12 @@ class Call1(VirtualArray): - def __init__(self, ufunc, name, shape, res_dtype, values): + def __init__(self, ufunc, name, shape, calc_dtype, res_dtype, values): VirtualArray.__init__(self, name, shape, res_dtype) self.values = values self.size = values.size self.ufunc = ufunc + self.calc_dtype = calc_dtype def _del_sources(self): self.values = None @@ -914,9 +816,7 @@ self.left = left self.right = right self.calc_dtype = calc_dtype - self.size = 1 - for s in self.shape: - self.size *= s + self.size = support.product(self.shape) def _del_sources(self): self.left = None @@ -1026,89 +926,6 @@ self.strides = strides self.backstrides = backstrides - def to_str(self, space, comma, builder, indent=' ', use_ellipsis=False): - '''Modifies builder with a representation of the array/slice - The items will be seperated by a comma if comma is 1 - Multidimensional arrays/slices will span a number of lines, - each line will begin with indent. - ''' - size = self.size - ccomma = ',' * comma - ncomma = ',' * (1 - comma) - dtype = self.find_dtype() - if size < 1: - builder.append('[]') - return - if size > 1000: - # Once this goes True it does not go back to False for recursive - # calls - use_ellipsis = True - ndims = len(self.shape) - if ndims == 0: - builder.append(dtype.itemtype.str_format(self.getitem(0))) - return - i = 0 - builder.append('[') - if ndims > 1: - if use_ellipsis: - for i in range(min(3, self.shape[0])): - if i > 0: - builder.append(ccomma + '\n') - if ndims >= 3: - builder.append('\n' + indent) - else: - builder.append(indent) - view = self.create_slice([Chunk(i, 0, 0, 1)]).get_concrete() - view.to_str(space, comma, builder, indent=indent + ' ', - use_ellipsis=use_ellipsis) - if i < self.shape[0] - 1: - builder.append(ccomma + '\n' + indent + '...' + ncomma) - i = self.shape[0] - 3 - else: - i += 1 - while i < self.shape[0]: - if i > 0: - builder.append(ccomma + '\n') - if ndims >= 3: - builder.append('\n' + indent) - else: - builder.append(indent) - # create_slice requires len(chunks) > 1 in order to reduce - # shape - view = self.create_slice([Chunk(i, 0, 0, 1)]).get_concrete() - view.to_str(space, comma, builder, indent=indent + ' ', - use_ellipsis=use_ellipsis) - i += 1 - elif ndims == 1: - spacer = ccomma + ' ' - item = self.start - # An iterator would be a nicer way to walk along the 1d array, but - # how do I reset it if printing ellipsis? iterators have no - # "set_offset()" - i = 0 - if use_ellipsis: - for i in range(min(3, self.shape[0])): - if i > 0: - builder.append(spacer) - builder.append(dtype.itemtype.str_format(self.getitem(item))) - item += self.strides[0] - if i < self.shape[0] - 1: - # Add a comma only if comma is False - this prevents adding - # two commas - builder.append(spacer + '...' + ncomma) - # Ugly, but can this be done with an iterator? - item = self.start + self.backstrides[0] - 2 * self.strides[0] - i = self.shape[0] - 3 - else: - i += 1 - while i < self.shape[0]: - if i > 0: - builder.append(spacer) - builder.append(dtype.itemtype.str_format(self.getitem(item))) - item += self.strides[0] - i += 1 - builder.append(']') - @jit.unroll_safe def _index_of_single_item(self, space, w_idx): if space.isinstance_w(w_idx, space.w_int): @@ -1181,15 +998,6 @@ array.setslice(space, self) return array - def flatten(self, space): - array = W_NDimArray(self.size, [self.size], self.dtype, self.order) - if self.supports_fast_slicing(): - array._fast_setslice(space, self) - else: - arr = SliceArray(array.shape, array.dtype, array, self, no_broadcast=True) - array._sliceloop(arr) - return array - def fill(self, space, w_value): self.setslice(space, scalar_w(space, self.dtype, w_value)) @@ -1204,13 +1012,10 @@ assert isinstance(parent, ConcreteArray) if isinstance(parent, W_NDimSlice): parent = parent.parent - size = 1 - for sh in shape: - size *= sh self.strides = strides self.backstrides = backstrides - ViewArray.__init__(self, size, shape, parent.dtype, parent.order, - parent) + ViewArray.__init__(self, support.product(shape), shape, parent.dtype, + parent.order, parent) self.start = start def create_iter(self): @@ -1286,27 +1091,42 @@ shape.append(item) return size, shape -def array(space, w_item_or_iterable, w_dtype=None, w_order=NoneNotWrapped): + at unwrap_spec(subok=bool, copy=bool, ownmaskna=bool) +def array(space, w_item_or_iterable, w_dtype=None, w_order=None, + subok=True, copy=True, w_maskna=None, ownmaskna=False): # find scalar + if w_maskna is None: + w_maskna = space.w_None + if (not subok or not space.is_w(w_maskna, space.w_None) or + ownmaskna): + raise OperationError(space.w_NotImplementedError, space.wrap("Unsupported args")) if not space.issequence_w(w_item_or_iterable): - if space.is_w(w_dtype, space.w_None): + if w_dtype is None or space.is_w(w_dtype, space.w_None): w_dtype = interp_ufuncs.find_dtype_for_scalar(space, w_item_or_iterable) dtype = space.interp_w(interp_dtype.W_Dtype, space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype) ) return scalar_w(space, dtype, w_item_or_iterable) - if w_order is None: + if space.is_w(w_order, space.w_None) or w_order is None: order = 'C' else: order = space.str_w(w_order) if order != 'C': # or order != 'F': raise operationerrfmt(space.w_ValueError, "Unknown order: %s", order) - shape, elems_w = _find_shape_and_elems(space, w_item_or_iterable) + if isinstance(w_item_or_iterable, BaseArray): + if (not space.is_w(w_dtype, space.w_None) and + w_item_or_iterable.find_dtype() is not w_dtype): + raise OperationError(space.w_NotImplementedError, space.wrap( + "copying over different dtypes unsupported")) + if copy: + return w_item_or_iterable.copy(space) + return w_item_or_iterable + shape, elems_w = find_shape_and_elems(space, w_item_or_iterable) # they come back in C order size = len(elems_w) - if space.is_w(w_dtype, space.w_None): + if w_dtype is None or space.is_w(w_dtype, space.w_None): w_dtype = None for w_elem in elems_w: w_dtype = interp_ufuncs.find_dtype_for_scalar(space, w_elem, @@ -1334,6 +1154,8 @@ space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype) ) size, shape = _find_size_and_shape(space, w_size) + if not shape: + return scalar_w(space, dtype, space.wrap(0)) return space.wrap(W_NDimArray(size, shape[:], dtype=dtype)) def ones(space, w_size, w_dtype=None): @@ -1342,11 +1164,27 @@ ) size, shape = _find_size_and_shape(space, w_size) + if not shape: + return scalar_w(space, dtype, space.wrap(1)) arr = W_NDimArray(size, shape[:], dtype=dtype) one = dtype.box(1) arr.dtype.fill(arr.storage, one, 0, size) return space.wrap(arr) + at unwrap_spec(arr=BaseArray, skipna=bool, keepdims=bool) +def count_reduce_items(space, arr, w_axis=None, skipna=False, keepdims=True): + if not keepdims: + raise OperationError(space.w_NotImplementedError, space.wrap("unsupported")) + if space.is_w(w_axis, space.w_None): + return space.wrap(support.product(arr.shape)) + if space.isinstance_w(w_axis, space.w_int): + return space.wrap(arr.shape[space.int_w(w_axis)]) + s = 1 + elems = space.fixedview(w_axis) + for w_elem in elems: + s *= arr.shape[space.int_w(w_elem)] + return space.wrap(s) + def dot(space, w_obj, w_obj2): w_arr = convert_to_array(space, w_obj) if isinstance(w_arr, Scalar): @@ -1376,10 +1214,7 @@ "array dimensions must agree except for axis being concatenated")) elif i == axis: shape[i] += axis_size - size = 1 - for elem in shape: - size *= elem - res = W_NDimArray(size, shape, dtype, 'C') + res = W_NDimArray(support.product(shape), shape, dtype, 'C') chunks = [Chunk(0, i, 1, i) for i in shape] axis_start = 0 for arr in args_w: @@ -1426,6 +1261,7 @@ __and__ = interp2app(BaseArray.descr_and), __or__ = interp2app(BaseArray.descr_or), + __invert__ = interp2app(BaseArray.descr_invert), __repr__ = interp2app(BaseArray.descr_repr), __str__ = interp2app(BaseArray.descr_str), @@ -1436,9 +1272,11 @@ BaseArray.descr_set_shape), size = GetSetProperty(BaseArray.descr_get_size), ndim = GetSetProperty(BaseArray.descr_get_ndim), + item = interp2app(BaseArray.descr_item), T = GetSetProperty(BaseArray.descr_get_transpose), flat = GetSetProperty(BaseArray.descr_get_flatiter), + ravel = interp2app(BaseArray.descr_ravel), mean = interp2app(BaseArray.descr_mean), sum = interp2app(BaseArray.descr_sum), @@ -1459,6 +1297,8 @@ flatten = interp2app(BaseArray.descr_flatten), reshape = interp2app(BaseArray.descr_reshape), tolist = interp2app(BaseArray.descr_tolist), + take = interp2app(BaseArray.descr_take), + compress = interp2app(BaseArray.descr_compress), ) @@ -1467,12 +1307,15 @@ @jit.unroll_safe def __init__(self, arr): arr = arr.get_concrete() + self.strides = [arr.strides[-1]] + self.backstrides = [arr.backstrides[-1]] self.shapelen = len(arr.shape) sig = arr.find_sig() self.iter = sig.create_frame(arr).get_final_iter() self.base = arr self.index = 0 - ViewArray.__init__(self, arr.size, [arr.size], arr.dtype, arr.order, arr) + ViewArray.__init__(self, arr.size, [arr.size], arr.dtype, arr.order, + arr) def descr_next(self, space): if self.iter.done(): diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py --- a/pypy/module/micronumpy/interp_ufuncs.py +++ b/pypy/module/micronumpy/interp_ufuncs.py @@ -1,14 +1,15 @@ from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.error import OperationError, operationerrfmt -from pypy.interpreter.gateway import interp2app +from pypy.interpreter.gateway import interp2app, unwrap_spec, NoneNotWrapped from pypy.interpreter.typedef import TypeDef, GetSetProperty, interp_attrproperty -from pypy.module.micronumpy import interp_boxes, interp_dtype -from pypy.module.micronumpy.signature import ReduceSignature,\ - find_sig, new_printable_location, AxisReduceSignature, ScalarSignature +from pypy.module.micronumpy import interp_boxes, interp_dtype, support +from pypy.module.micronumpy.signature import (ReduceSignature, find_sig, + new_printable_location, AxisReduceSignature, ScalarSignature) from pypy.rlib import jit from pypy.rlib.rarithmetic import LONG_BIT from pypy.tool.sourcetools import func_with_new_name + reduce_driver = jit.JitDriver( greens=['shapelen', "sig"], virtualizables=["frame"], @@ -30,12 +31,14 @@ _attrs_ = ["name", "promote_to_float", "promote_bools", "identity"] _immutable_fields_ = ["promote_to_float", "promote_bools", "name"] - def __init__(self, name, promote_to_float, promote_bools, identity): + def __init__(self, name, promote_to_float, promote_bools, identity, + int_only): self.name = name self.promote_to_float = promote_to_float self.promote_bools = promote_bools self.identity = identity + self.int_only = int_only def descr_repr(self, space): return space.wrap("<ufunc '%s'>" % self.name) @@ -46,19 +49,31 @@ return self.identity def descr_call(self, space, __args__): - if __args__.keywords or len(__args__.arguments_w) < self.argcount: + args_w, kwds_w = __args__.unpack() + # it occurs to me that we don't support any datatypes that + # require casting, change it later when we do + kwds_w.pop('casting', None) + w_subok = kwds_w.pop('subok', None) + w_out = kwds_w.pop('out', space.w_None) + if ((w_subok is not None and space.is_true(w_subok)) or + not space.is_w(w_out, space.w_None)): + raise OperationError(space.w_NotImplementedError, + space.wrap("parameters unsupported")) + if kwds_w or len(args_w) < self.argcount: raise OperationError(space.w_ValueError, space.wrap("invalid number of arguments") ) - elif len(__args__.arguments_w) > self.argcount: + elif len(args_w) > self.argcount: # The extra arguments should actually be the output array, but we # don't support that yet. raise OperationError(space.w_TypeError, space.wrap("invalid number of arguments") ) - return self.call(space, __args__.arguments_w) + return self.call(space, args_w) - def descr_reduce(self, space, w_obj, w_dim=0): + @unwrap_spec(skipna=bool, keepdims=bool) + def descr_reduce(self, space, w_obj, w_axis=NoneNotWrapped, w_dtype=None, + skipna=False, keepdims=False, w_out=None): """reduce(...) reduce(a, axis=0) @@ -111,15 +126,24 @@ array([[ 1, 5], [ 9, 13]]) """ - return self.reduce(space, w_obj, False, False, w_dim) + if not space.is_w(w_out, space.w_None): + raise OperationError(space.w_NotImplementedError, space.wrap( + "out not supported")) + if w_axis is None: + axis = 0 + elif space.is_w(w_axis, space.w_None): + axis = -1 + else: + axis = space.int_w(w_axis) + return self.reduce(space, w_obj, False, False, axis, keepdims) - def reduce(self, space, w_obj, multidim, promote_to_largest, w_dim): + def reduce(self, space, w_obj, multidim, promote_to_largest, dim, + keepdims=False): from pypy.module.micronumpy.interp_numarray import convert_to_array, \ Scalar if self.argcount != 2: raise OperationError(space.w_ValueError, space.wrap("reduce only " "supported for binary functions")) - dim = space.int_w(w_dim) assert isinstance(self, W_Ufunc2) obj = convert_to_array(space, w_obj) if dim >= len(obj.shape): @@ -140,7 +164,7 @@ raise operationerrfmt(space.w_ValueError, "zero-size array to " "%s.reduce without identity", self.name) if shapelen > 1 and dim >= 0: - res = self.do_axis_reduce(obj, dtype, dim) + res = self.do_axis_reduce(obj, dtype, dim, keepdims) return space.wrap(res) scalarsig = ScalarSignature(dtype) sig = find_sig(ReduceSignature(self.func, self.name, dtype, @@ -154,15 +178,15 @@ value = self.identity.convert_to(dtype) return self.reduce_loop(shapelen, sig, frame, value, obj, dtype) - def do_axis_reduce(self, obj, dtype, dim): + def do_axis_reduce(self, obj, dtype, dim, keepdims): from pypy.module.micronumpy.interp_numarray import AxisReduce,\ W_NDimArray - - shape = obj.shape[0:dim] + obj.shape[dim + 1:len(obj.shape)] - size = 1 - for s in shape: - size *= s - result = W_NDimArray(size, shape, dtype) + + if keepdims: + shape = obj.shape[:dim] + [1] + obj.shape[dim + 1:] + else: + shape = obj.shape[:dim] + obj.shape[dim + 1:] + result = W_NDimArray(support.product(shape), shape, dtype) rightsig = obj.create_sig() # note - this is just a wrapper so signature can fetch # both left and right, nothing more, especially @@ -224,10 +248,12 @@ _immutable_fields_ = ["func", "name"] def __init__(self, func, name, promote_to_float=False, promote_bools=False, - identity=None): + identity=None, bool_result=False, int_only=False): - W_Ufunc.__init__(self, name, promote_to_float, promote_bools, identity) + W_Ufunc.__init__(self, name, promote_to_float, promote_bools, identity, + int_only) self.func = func + self.bool_result = bool_result def call(self, space, args_w): from pypy.module.micronumpy.interp_numarray import (Call1, @@ -235,15 +261,19 @@ [w_obj] = args_w w_obj = convert_to_array(space, w_obj) - res_dtype = find_unaryop_result_dtype(space, - w_obj.find_dtype(), - promote_to_float=self.promote_to_float, - promote_bools=self.promote_bools, - ) + calc_dtype = find_unaryop_result_dtype(space, + w_obj.find_dtype(), + promote_to_float=self.promote_to_float, + promote_bools=self.promote_bools) + if self.bool_result: + res_dtype = interp_dtype.get_dtype_cache(space).w_booldtype + else: + res_dtype = calc_dtype if isinstance(w_obj, Scalar): - return self.func(res_dtype, w_obj.value.convert_to(res_dtype)) + return space.wrap(self.func(calc_dtype, w_obj.value.convert_to(calc_dtype))) - w_res = Call1(self.func, self.name, w_obj.shape, res_dtype, w_obj) + w_res = Call1(self.func, self.name, w_obj.shape, calc_dtype, res_dtype, + w_obj) w_obj.add_invalidates(w_res) return w_res @@ -255,10 +285,10 @@ def __init__(self, func, name, promote_to_float=False, promote_bools=False, identity=None, comparison_func=False, int_only=False): - W_Ufunc.__init__(self, name, promote_to_float, promote_bools, identity) + W_Ufunc.__init__(self, name, promote_to_float, promote_bools, identity, + int_only) self.func = func self.comparison_func = comparison_func - self.int_only = int_only def call(self, space, args_w): from pypy.module.micronumpy.interp_numarray import (Call2, @@ -278,10 +308,10 @@ else: res_dtype = calc_dtype if isinstance(w_lhs, Scalar) and isinstance(w_rhs, Scalar): - return self.func(calc_dtype, + return space.wrap(self.func(calc_dtype, w_lhs.value.convert_to(calc_dtype), w_rhs.value.convert_to(calc_dtype) - ) + )) new_shape = shape_agreement(space, w_lhs.shape, w_rhs.shape) w_res = Call2(self.func, self.name, @@ -409,12 +439,16 @@ return interp_dtype.get_dtype_cache(space).w_float64dtype -def ufunc_dtype_caller(space, ufunc_name, op_name, argcount, comparison_func): +def ufunc_dtype_caller(space, ufunc_name, op_name, argcount, comparison_func, + bool_result): + dtype_cache = interp_dtype.get_dtype_cache(space) if argcount == 1: def impl(res_dtype, value): - return getattr(res_dtype.itemtype, op_name)(value) + res = getattr(res_dtype.itemtype, op_name)(value) + if bool_result: + return dtype_cache.w_booldtype.box(res) + return res elif argcount == 2: - dtype_cache = interp_dtype.get_dtype_cache(space) def impl(res_dtype, lvalue, rvalue): res = getattr(res_dtype.itemtype, op_name)(lvalue, rvalue) if comparison_func: @@ -433,7 +467,9 @@ 'int_only': True}), ("bitwise_or", "bitwise_or", 2, {"identity": 0, 'int_only': True}), + ("invert", "invert", 1, {"int_only": True}), ("divide", "div", 2, {"promote_bools": True}), + ("true_divide", "div", 2, {"promote_to_float": True}), ("mod", "mod", 2, {"promote_bools": True}), ("power", "pow", 2, {"promote_bools": True}), @@ -443,6 +479,8 @@ ("less_equal", "le", 2, {"comparison_func": True}), ("greater", "gt", 2, {"comparison_func": True}), ("greater_equal", "ge", 2, {"comparison_func": True}), + ("isnan", "isnan", 1, {"bool_result": True}), + ("isinf", "isinf", 1, {"bool_result": True}), ("maximum", "max", 2), ("minimum", "min", 2), @@ -485,6 +523,7 @@ func = ufunc_dtype_caller(space, ufunc_name, op_name, argcount, comparison_func=extra_kwargs.get("comparison_func", False), + bool_result=extra_kwargs.get("bool_result", False), ) if argcount == 1: ufunc = W_Ufunc1(func, ufunc_name, **extra_kwargs) @@ -494,3 +533,4 @@ def get(space): return space.fromcache(UfuncState) + diff --git a/pypy/module/micronumpy/signature.py b/pypy/module/micronumpy/signature.py --- a/pypy/module/micronumpy/signature.py +++ b/pypy/module/micronumpy/signature.py @@ -305,8 +305,8 @@ def eval(self, frame, arr): from pypy.module.micronumpy.interp_numarray import Call1 assert isinstance(arr, Call1) - v = self.child.eval(frame, arr.values).convert_to(arr.res_dtype) - return self.unfunc(arr.res_dtype, v) + v = self.child.eval(frame, arr.values).convert_to(arr.calc_dtype) + return self.unfunc(arr.calc_dtype, v) class Call2(Signature): _immutable_fields_ = ['binfunc', 'name', 'calc_dtype', 'left', 'right'] diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py --- a/pypy/module/micronumpy/strides.py +++ b/pypy/module/micronumpy/strides.py @@ -1,7 +1,6 @@ from pypy.rlib import jit from pypy.interpreter.error import OperationError - @jit.look_inside_iff(lambda shape, start, strides, backstrides, chunks: jit.isconstant(len(chunks)) ) @@ -39,6 +38,28 @@ rbackstrides = [0] * (len(res_shape) - len(orig_shape)) + rbackstrides return rstrides, rbackstrides +def find_shape_and_elems(space, w_iterable): + shape = [space.len_w(w_iterable)] + batch = space.listview(w_iterable) + while True: + new_batch = [] + if not batch: + return shape, [] + if not space.issequence_w(batch[0]): + for elem in batch: + if space.issequence_w(elem): + raise OperationError(space.w_ValueError, space.wrap( + "setting an array element with a sequence")) + return shape, batch + size = space.len_w(batch[0]) + for w_elem in batch: + if not space.issequence_w(w_elem) or space.len_w(w_elem) != size: + raise OperationError(space.w_ValueError, space.wrap( + "setting an array element with a sequence")) + new_batch += space.listview(w_elem) + shape.append(size) + batch = new_batch + def to_coords(space, shape, size, order, w_item_or_slice): '''Returns a start coord, step, and length. ''' @@ -60,5 +81,152 @@ for s in range(len(shape)): coords[s] = i % shape[s] i //= shape[s] + return coords, step, lngth - return coords, step, lngth +def shape_agreement(space, shape1, shape2): + ret = _shape_agreement(shape1, shape2) + if len(ret) < max(len(shape1), len(shape2)): + raise OperationError(space.w_ValueError, + space.wrap("operands could not be broadcast together with shapes (%s) (%s)" % ( + ",".join([str(x) for x in shape1]), + ",".join([str(x) for x in shape2]), + )) + ) + return ret + +def _shape_agreement(shape1, shape2): + """ Checks agreement about two shapes with respect to broadcasting. Returns + the resulting shape. + """ + lshift = 0 + rshift = 0 + if len(shape1) > len(shape2): + m = len(shape1) + n = len(shape2) + rshift = len(shape2) - len(shape1) + remainder = shape1 + else: + m = len(shape2) + n = len(shape1) + lshift = len(shape1) - len(shape2) + remainder = shape2 + endshape = [0] * m + indices1 = [True] * m + indices2 = [True] * m + for i in range(m - 1, m - n - 1, -1): + left = shape1[i + lshift] + right = shape2[i + rshift] + if left == right: + endshape[i] = left + elif left == 1: + endshape[i] = right + indices1[i + lshift] = False + elif right == 1: + endshape[i] = left + indices2[i + rshift] = False + else: + return [] + #raise OperationError(space.w_ValueError, space.wrap( + # "frames are not aligned")) + for i in range(m - n): + endshape[i] = remainder[i] + return endshape + +def get_shape_from_iterable(space, old_size, w_iterable): + new_size = 0 + new_shape = [] + if space.isinstance_w(w_iterable, space.w_int): + new_size = space.int_w(w_iterable) + if new_size < 0: + new_size = old_size + new_shape = [new_size] + else: + neg_dim = -1 + batch = space.listview(w_iterable) + new_size = 1 + if len(batch) < 1: + if old_size == 1: + # Scalars can have an empty size. + new_size = 1 + else: + new_size = 0 + new_shape = [] + i = 0 + for elem in batch: + s = space.int_w(elem) + if s < 0: + if neg_dim >= 0: + raise OperationError(space.w_ValueError, space.wrap( + "can only specify one unknown dimension")) + s = 1 + neg_dim = i + new_size *= s + new_shape.append(s) + i += 1 + if neg_dim >= 0: + new_shape[neg_dim] = old_size / new_size + new_size *= new_shape[neg_dim] + if new_size != old_size: + raise OperationError(space.w_ValueError, + space.wrap("total size of new array must be unchanged")) + return new_shape + +# Recalculating strides. Find the steps that the iteration does for each +# dimension, given the stride and shape. Then try to create a new stride that +# fits the new shape, using those steps. If there is a shape/step mismatch +# (meaning that the realignment of elements crosses from one step into another) +# return None so that the caller can raise an exception. +def calc_new_strides(new_shape, old_shape, old_strides, order): + # Return the proper strides for new_shape, or None if the mapping crosses + # stepping boundaries + + # Assumes that prod(old_shape) == prod(new_shape), len(old_shape) > 1, and + # len(new_shape) > 0 + steps = [] + last_step = 1 + oldI = 0 + new_strides = [] + if order == 'F': + for i in range(len(old_shape)): + steps.append(old_strides[i] / last_step) + last_step *= old_shape[i] + cur_step = steps[0] + n_new_elems_used = 1 + n_old_elems_to_use = old_shape[0] + for s in new_shape: + new_strides.append(cur_step * n_new_elems_used) + n_new_elems_used *= s + while n_new_elems_used > n_old_elems_to_use: + oldI += 1 + if steps[oldI] != steps[oldI - 1]: + return None + n_old_elems_to_use *= old_shape[oldI] + if n_new_elems_used == n_old_elems_to_use: + oldI += 1 + if oldI < len(old_shape): + cur_step = steps[oldI] + n_old_elems_to_use *= old_shape[oldI] + elif order == 'C': + for i in range(len(old_shape) - 1, -1, -1): + steps.insert(0, old_strides[i] / last_step) + last_step *= old_shape[i] + cur_step = steps[-1] + n_new_elems_used = 1 + oldI = -1 + n_old_elems_to_use = old_shape[-1] + for i in range(len(new_shape) - 1, -1, -1): + s = new_shape[i] + new_strides.insert(0, cur_step * n_new_elems_used) + n_new_elems_used *= s + while n_new_elems_used > n_old_elems_to_use: + oldI -= 1 + if steps[oldI] != steps[oldI + 1]: + return None + n_old_elems_to_use *= old_shape[oldI] + if n_new_elems_used == n_old_elems_to_use: + oldI -= 1 + if oldI >= -len(old_shape): + cur_step = steps[oldI] + n_old_elems_to_use *= old_shape[oldI] + assert len(new_strides) == len(new_shape) + return new_strides diff --git a/pypy/module/micronumpy/support.py b/pypy/module/micronumpy/support.py new file mode 100644 --- /dev/null +++ b/pypy/module/micronumpy/support.py @@ -0,0 +1,5 @@ +def product(s): + i = 1 + for x in s: + i *= x + return i \ No newline at end of file diff --git a/pypy/module/micronumpy/test/test_base.py b/pypy/module/micronumpy/test/test_base.py --- a/pypy/module/micronumpy/test/test_base.py +++ b/pypy/module/micronumpy/test/test_base.py @@ -3,9 +3,17 @@ from pypy.module.micronumpy.interp_numarray import W_NDimArray, Scalar from pypy.module.micronumpy.interp_ufuncs import (find_binop_result_dtype, find_unaryop_result_dtype) +from pypy.module.micronumpy.interp_boxes import W_Float64Box +from pypy.conftest import option +import sys class BaseNumpyAppTest(object): def setup_class(cls): + if option.runappdirect: + if '__pypy__' not in sys.builtin_module_names: + import numpy + sys.modules['numpypy'] = numpy + sys.modules['_numpypy'] = numpy cls.space = gettestobjspace(usemodules=['micronumpy']) class TestSignature(object): @@ -16,7 +24,7 @@ ar = W_NDimArray(10, [10], dtype=float64_dtype) ar2 = W_NDimArray(10, [10], dtype=float64_dtype) v1 = ar.descr_add(space, ar) - v2 = ar.descr_add(space, Scalar(float64_dtype, 2.0)) + v2 = ar.descr_add(space, Scalar(float64_dtype, W_Float64Box(2.0))) sig1 = v1.find_sig() sig2 = v2.find_sig() assert v1 is not v2 @@ -26,7 +34,7 @@ sig1b = ar2.descr_add(space, ar).find_sig() assert sig1b.left.array_no != sig1b.right.array_no assert sig1b is not sig1 - v3 = ar.descr_add(space, Scalar(float64_dtype, 1.0)) + v3 = ar.descr_add(space, Scalar(float64_dtype, W_Float64Box(1.0))) sig3 = v3.find_sig() assert sig2 is sig3 v4 = ar.descr_add(space, ar) diff --git a/pypy/module/micronumpy/test/test_compile.py b/pypy/module/micronumpy/test/test_compile.py --- a/pypy/module/micronumpy/test/test_compile.py +++ b/pypy/module/micronumpy/test/test_compile.py @@ -253,3 +253,11 @@ b -> 3 ''') assert interp.results[0].value == 3 + + def test_take(self): + interp = self.run(""" + a = |10| + b = take(a, [1, 1, 3, 2]) + b -> 2 + """) + assert interp.results[0].value == 3 diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py --- a/pypy/module/micronumpy/test/test_dtypes.py +++ b/pypy/module/micronumpy/test/test_dtypes.py @@ -11,6 +11,7 @@ assert dtype('int8').num == 1 assert dtype(d) is d assert dtype(None) is dtype(float) + assert dtype('int8').name == 'int8' raises(TypeError, dtype, 1042) def test_dtype_eq(self): @@ -38,7 +39,7 @@ def test_repr_str(self): from _numpypy import dtype - assert repr(dtype) == "<type 'numpypy.dtype'>" + assert '.dtype' in repr(dtype) d = dtype('?') assert repr(d) == "dtype('bool')" assert str(d) == "bool" @@ -385,6 +386,11 @@ assert type(b) is X assert b.m() == 12 + def test_long_as_index(self): + skip("waiting for removal of multimethods of __index__") + from _numpypy import int_ + assert (1, 2, 3)[int_(1)] == 2 + def test_int(self): import sys from _numpypy import int32, int64, int_ diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -184,7 +184,6 @@ assert _to_coords(-1, 'F') == [1, 2, 3] assert _to_coords(5, 'F') == [1, 2, 0] assert _to_coords(13, 'F') == [1, 0, 2] - class AppTestNumArray(BaseNumpyAppTest): def test_ndarray(self): @@ -229,6 +228,7 @@ # And check that changes stick. a[13] = 5.3 assert a[13] == 5.3 + assert zeros(()).shape == () def test_size(self): from _numpypy import array @@ -272,6 +272,7 @@ b = a[::2] c = b.copy() assert (c == b).all() + assert ((a + a).copy() == (a + a)).all() a = arange(15).reshape(5,3) b = a.copy() @@ -394,6 +395,7 @@ assert b.shape == (5,) c = a[:3] assert c.shape == (3,) + assert array([]).shape == (0,) def test_set_shape(self): from _numpypy import array, zeros @@ -429,6 +431,7 @@ assert (a == [1000, 1, 2, 3, 1000, 5, 6, 7, 1000, 9, 10, 11]).all() a = zeros((4, 2, 3)) a.shape = (12, 2) + (a + a).reshape(2, 12) # assert did not explode def test_slice_reshape(self): from _numpypy import zeros, arange @@ -472,6 +475,13 @@ y = z.reshape(4, 3, 8) assert y.shape == (4, 3, 8) + def test_scalar_reshape(self): + from numpypy import array + a = array(3) + assert a.reshape([1, 1]).shape == (1, 1) + assert a.reshape([1]).shape == (1,) + raises(ValueError, "a.reshape(3)") + def test_add(self): from _numpypy import array a = array(range(5)) @@ -1060,8 +1070,8 @@ assert a.var() == 0.0 a = arange(10).reshape(5, 2) assert a.var() == 8.25 - #assert (a.var(0) == [8, 8]).all() - #assert (a.var(1) == [.25] * 5).all() + assert (a.var(0) == [8, 8]).all() + assert (a.var(1) == [.25] * 5).all() def test_concatenate(self): from numpypy import array, concatenate, dtype @@ -1107,6 +1117,13 @@ def test_flatten(self): from _numpypy import array + assert array(3).flatten().shape == (1,) + a = array([[1, 2], [3, 4]]) + b = a.flatten() + c = a.ravel() + a[0, 0] = 15 + assert b[0] == 1 + assert c[0] == 15 a = array([[1, 2, 3], [4, 5, 6]]) assert (a.flatten() == [1, 2, 3, 4, 5, 6]).all() a = array([[[1, 2], [3, 4]], [[5, 6], [7, 8]]]) @@ -1134,7 +1151,7 @@ assert _numpypy.array([[1], [2], [3]]).shape == (3, 1) assert len(_numpypy.zeros((3, 1, 2))) == 3 raises(TypeError, len, _numpypy.zeros(())) - raises(ValueError, _numpypy.array, [[1, 2], 3]) + raises(ValueError, _numpypy.array, [[1, 2], 3], dtype=float) def test_getsetitem(self): import _numpypy @@ -1510,12 +1527,61 @@ a[a & 1 == 1] = array([8, 9, 10]) assert (a == [[0, 8], [2, 9], [4, 10]]).all() + def test_copy_kwarg(self): + from _numpypy import array + x = array([1, 2, 3]) + assert (array(x) == x).all() + assert array(x) is not x + assert array(x, copy=False) is x + assert array(x, copy=True) is not x + def test_isna(self): from _numpypy import isna, array # XXX for now assert not isna(3) assert (isna(array([1, 2, 3, 4])) == [False, False, False, False]).all() + def test_ravel(self): + from _numpypy import arange + assert (arange(3).ravel() == arange(3)).all() + assert (arange(6).reshape(2, 3).ravel() == arange(6)).all() + assert (arange(6).reshape(2, 3).T.ravel() == [0, 3, 1, 4, 2, 5]).all() + + def test_take(self): + from _numpypy import arange + assert (arange(10).take([1, 2, 1, 1]) == [1, 2, 1, 1]).all() + raises(IndexError, "arange(3).take([15])") + a = arange(6).reshape(2, 3) + assert (a.take([1, 0, 3]) == [1, 0, 3]).all() + assert ((a + a).take([3]) == [6]).all() + a = arange(12).reshape(2, 6) + assert (a[:,::2].take([3, 2, 1]) == [6, 4, 2]).all() + + def test_compress(self): + from _numpypy import arange + a = arange(10) + assert (a.compress([True, False, True]) == [0, 2]).all() + assert (a.compress([1, 0, 13]) == [0, 2]).all() + assert (a.compress([1, 0, 13.5]) == [0, 2]).all() + a = arange(10).reshape(2, 5) + assert (a.compress([True, False, True]) == [0, 2]).all() + raises(IndexError, "a.compress([1] * 100)") + + def test_item(self): + from _numpypy import array + assert array(3).item() == 3 + assert type(array(3).item()) is int + assert type(array(True).item()) is bool + assert type(array(3.5).item()) is float + raises((ValueError, IndexError), "array(3).item(15)") + raises(ValueError, "array([1, 2, 3]).item()") + assert array([3]).item(0) == 3 + assert type(array([3]).item(0)) is int + assert array([1, 2, 3]).item(-1) == 3 + a = array([1, 2, 3]) + assert a[::2].item(1) == 3 + assert (a + a).item(1) == 4 + raises(ValueError, "array(5).item(1)") class AppTestSupport(BaseNumpyAppTest): def setup_class(cls): @@ -1627,128 +1693,6 @@ raises(ValueError, fromstring, "\x01\x02\x03", count=5, dtype=uint8) -class AppTestRepr(BaseNumpyAppTest): - def test_repr(self): - from _numpypy import array, zeros - int_size = array(5).dtype.itemsize - a = array(range(5), float) - assert repr(a) == "array([0.0, 1.0, 2.0, 3.0, 4.0])" - a = array([], float) - assert repr(a) == "array([], dtype=float64)" - a = zeros(1001) - assert repr(a) == "array([0.0, 0.0, 0.0, ..., 0.0, 0.0, 0.0])" - a = array(range(5), long) - if a.dtype.itemsize == int_size: - assert repr(a) == "array([0, 1, 2, 3, 4])" - else: - assert repr(a) == "array([0, 1, 2, 3, 4], dtype=int64)" - a = array(range(5), 'int32') - if a.dtype.itemsize == int_size: - assert repr(a) == "array([0, 1, 2, 3, 4])" - else: - assert repr(a) == "array([0, 1, 2, 3, 4], dtype=int32)" - a = array([], long) - assert repr(a) == "array([], dtype=int64)" - a = array([True, False, True, False], "?") - assert repr(a) == "array([True, False, True, False], dtype=bool)" - a = zeros([]) - assert repr(a) == "array(0.0)" - a = array(0.2) - assert repr(a) == "array(0.2)" - a = array([2]) - assert repr(a) == "array([2])" - - def test_repr_multi(self): - from _numpypy import arange, zeros, array - a = zeros((3, 4)) - assert repr(a) == '''array([[0.0, 0.0, 0.0, 0.0], - [0.0, 0.0, 0.0, 0.0], - [0.0, 0.0, 0.0, 0.0]])''' - a = zeros((2, 3, 4)) - assert repr(a) == '''array([[[0.0, 0.0, 0.0, 0.0], - [0.0, 0.0, 0.0, 0.0], - [0.0, 0.0, 0.0, 0.0]], - - [[0.0, 0.0, 0.0, 0.0], - [0.0, 0.0, 0.0, 0.0], - [0.0, 0.0, 0.0, 0.0]]])''' - a = arange(1002).reshape((2, 501)) - assert repr(a) == '''array([[0, 1, 2, ..., 498, 499, 500], - [501, 502, 503, ..., 999, 1000, 1001]])''' - assert repr(a.T) == '''array([[0, 501], - [1, 502], - [2, 503], - ..., - [498, 999], - [499, 1000], - [500, 1001]])''' - a = arange(2).reshape((2,1)) - assert repr(a) == '''array([[0], - [1]])''' - - def test_repr_slice(self): - from _numpypy import array, zeros - a = array(range(5), float) - b = a[1::2] - assert repr(b) == "array([1.0, 3.0])" - a = zeros(2002) - b = a[::2] - assert repr(b) == "array([0.0, 0.0, 0.0, ..., 0.0, 0.0, 0.0])" - a = array((range(5), range(5, 10)), dtype="int16") - b = a[1, 2:] - assert repr(b) == "array([7, 8, 9], dtype=int16)" - # an empty slice prints its shape - b = a[2:1, ] - assert repr(b) == "array([], shape=(0, 5), dtype=int16)" - - def test_str(self): - from _numpypy import array, zeros - a = array(range(5), float) - assert str(a) == "[0.0 1.0 2.0 3.0 4.0]" - assert str((2 * a)[:]) == "[0.0 2.0 4.0 6.0 8.0]" - a = zeros(1001) - assert str(a) == "[0.0 0.0 0.0 ..., 0.0 0.0 0.0]" - - a = array(range(5), dtype=long) - assert str(a) == "[0 1 2 3 4]" - a = array([True, False, True, False], dtype="?") - assert str(a) == "[True False True False]" - - a = array(range(5), dtype="int8") - assert str(a) == "[0 1 2 3 4]" - - a = array(range(5), dtype="int16") - assert str(a) == "[0 1 2 3 4]" - - a = array((range(5), range(5, 10)), dtype="int16") - assert str(a) == "[[0 1 2 3 4]\n [5 6 7 8 9]]" - - a = array(3, dtype=int) - assert str(a) == "3" - - a = zeros((400, 400), dtype=int) - assert str(a) == "[[0 0 0 ..., 0 0 0]\n [0 0 0 ..., 0 0 0]\n" \ - " [0 0 0 ..., 0 0 0]\n ...,\n [0 0 0 ..., 0 0 0]\n" \ - " [0 0 0 ..., 0 0 0]\n [0 0 0 ..., 0 0 0]]" - a = zeros((2, 2, 2)) - r = str(a) - assert r == '[[[0.0 0.0]\n [0.0 0.0]]\n\n [[0.0 0.0]\n [0.0 0.0]]]' - - def test_str_slice(self): - from _numpypy import array, zeros - a = array(range(5), float) - b = a[1::2] - assert str(b) == "[1.0 3.0]" - a = zeros(2002) - b = a[::2] - assert str(b) == "[0.0 0.0 0.0 ..., 0.0 0.0 0.0]" - a = array((range(5), range(5, 10)), dtype="int16") - b = a[1, 2:] - assert str(b) == "[7 8 9]" - b = a[2:1, ] - assert str(b) == "[]" - - class AppTestRanges(BaseNumpyAppTest): def test_arange(self): from _numpypy import arange, array, dtype @@ -1769,3 +1713,24 @@ a = arange(0, 0.8, 0.1) assert len(a) == 8 assert arange(False, True, True).dtype is dtype(int) + +from pypy.module.micronumpy.appbridge import get_appbridge_cache + +class AppTestRepr(BaseNumpyAppTest): + def setup_class(cls): + BaseNumpyAppTest.setup_class.im_func(cls) + cache = get_appbridge_cache(cls.space) + cls.old_array_repr = cache.w_array_repr + cls.old_array_str = cache.w_array_str + cache.w_array_str = None + cache.w_array_repr = None + + def test_repr_str(self): + from _numpypy import array + assert repr(array([1, 2, 3])) == 'array([1, 2, 3])' + assert str(array([1, 2, 3])) == 'array([1, 2, 3])' + + def teardown_class(cls): + cache = get_appbridge_cache(cls.space) + cache.w_array_repr = cls.old_array_repr + cache.w_array_str = cls.old_array_str diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py --- a/pypy/module/micronumpy/test/test_ufuncs.py +++ b/pypy/module/micronumpy/test/test_ufuncs.py @@ -344,7 +344,7 @@ from _numpypy import sin, add raises(ValueError, sin.reduce, [1, 2, 3]) - raises(ValueError, add.reduce, 1) + raises((ValueError, TypeError), add.reduce, 1) def test_reduce_1d(self): from _numpypy import add, maximum @@ -360,6 +360,14 @@ assert (add.reduce(a, 0) == [12, 15, 18, 21]).all() assert (add.reduce(a, 1) == [6.0, 22.0, 38.0]).all() + def test_reduce_keepdims(self): + from _numpypy import add, arange + a = arange(12).reshape(3, 4) + b = add.reduce(a, 0, keepdims=True) + assert b.shape == (1, 4) + assert (add.reduce(a, 0, keepdims=True) == [12, 15, 18, 21]).all() + + def test_bitwise(self): from _numpypy import bitwise_and, bitwise_or, arange, array a = arange(6).reshape(2, 3) @@ -369,6 +377,12 @@ assert (a | 1 == bitwise_or(a, 1)).all() raises(TypeError, 'array([1.0]) & 1') + def test_unary_bitops(self): + from _numpypy import bitwise_not, array + a = array([1, 2, 3, 4]) + assert (~a == [-2, -3, -4, -5]).all() + assert (bitwise_not(a) == ~a).all() + def test_comparisons(self): import operator from _numpypy import equal, not_equal, less, less_equal, greater, greater_equal @@ -394,3 +408,28 @@ (3, 3.5), ]: assert ufunc(a, b) == func(a, b) + + def test_count_reduce_items(self): + from _numpypy import count_reduce_items, arange + a = arange(24).reshape(2, 3, 4) + assert count_reduce_items(a) == 24 + assert count_reduce_items(a, 1) == 3 + assert count_reduce_items(a, (1, 2)) == 3 * 4 + + def test_true_divide(self): + from _numpypy import arange, array, true_divide + assert (true_divide(arange(3), array([2, 2, 2])) == array([0, 0.5, 1])).all() + + def test_isnan_isinf(self): + from _numpypy import isnan, isinf, float64, array + assert isnan(float('nan')) + assert isnan(float64(float('nan'))) + assert not isnan(3) + assert isinf(float('inf')) + assert not isnan(3.5) + assert not isinf(3.5) + assert not isnan(float('inf')) + assert not isinf(float('nan')) + assert (isnan(array([0.2, float('inf'), float('nan')])) == [False, False, True]).all() + assert (isinf(array([0.2, float('inf'), float('nan')])) == [False, True, False]).all() + assert isinf(array([0.2])).dtype.kind == 'b' diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py --- a/pypy/module/micronumpy/test/test_zjit.py +++ b/pypy/module/micronumpy/test/test_zjit.py @@ -287,6 +287,27 @@ 'jump': 1, 'arraylen_gc': 1}) + def define_take(): + return """ + a = |10| + b = take(a, [1, 1, 3, 2]) + b -> 2 + """ + + def test_take(self): + result = self.run("take") + assert result == 3 + self.check_simple_loop({'getinteriorfield_raw': 2, + 'cast_float_to_int': 1, + 'int_lt': 1, + 'int_ge': 2, + 'guard_false': 3, + 'setinteriorfield_raw': 1, + 'int_mul': 1, + 'int_add': 3, + 'jump': 1, + 'arraylen_gc': 2}) + def define_multidim(): return """ a = [[1, 2], [3, 4], [5, 6], [7, 8], [9, 10]] diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py --- a/pypy/module/micronumpy/types.py +++ b/pypy/module/micronumpy/types.py @@ -23,6 +23,16 @@ ) return dispatcher +def raw_unary_op(func): + specialize.argtype(1)(func) + @functools.wraps(func) + def dispatcher(self, v): + return func( + self, + self.for_computation(self.unbox(v)) + ) + return dispatcher + def simple_binary_op(func): specialize.argtype(1, 2)(func) @functools.wraps(func) @@ -95,7 +105,9 @@ )) def read_bool(self, storage, width, i, offset): - raise NotImplementedError + return bool(self.for_computation( + libffi.array_getitem(clibffi.cast_type_to_ffitype(self.T), + width, storage, i, offset))) def store(self, storage, width, i, offset, box): value = self.unbox(box) @@ -137,6 +149,14 @@ def abs(self, v): return abs(v) + @raw_unary_op + def isnan(self, v): + return False + + @raw_unary_op + def isinf(self, v): + return False + @raw_binary_op def eq(self, v1, v2): return v1 == v2 @@ -171,7 +191,7 @@ @simple_binary_op def min(self, v1, v2): return min(v1, v2) - + class Bool(BaseType, Primitive): T = lltype.Bool @@ -189,11 +209,6 @@ else: return self.False - - def read_bool(self, storage, width, i, offset): - return libffi.array_getitem(clibffi.cast_type_to_ffitype(self.T), - width, storage, i, offset) - def coerce_subtype(self, space, w_subtype, w_item): # Doesn't return subclasses so it can return the constants. return self._coerce(space, w_item) @@ -214,6 +229,18 @@ def default_fromstring(self, space): return self.box(False) + @simple_binary_op + def bitwise_and(self, v1, v2): + return v1 & v2 + + @simple_binary_op + def bitwise_or(self, v1, v2): + return v1 | v2 + + @simple_unary_op + def invert(self, v): + return ~v + class Integer(Primitive): _mixin_ = True @@ -270,6 +297,10 @@ def bitwise_or(self, v1, v2): return v1 | v2 + @simple_unary_op + def invert(self, v): + return ~v + class Int8(BaseType, Integer): T = rffi.SIGNEDCHAR BoxType = interp_boxes.W_Int8Box @@ -448,6 +479,14 @@ except ValueError: return rfloat.NAN + @raw_unary_op + def isnan(self, v): + return rfloat.isnan(v) + + @raw_unary_op + def isinf(self, v): + return rfloat.isinf(v) + class Float32(BaseType, Float): T = rffi.FLOAT diff --git a/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py --- a/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py +++ b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py @@ -98,15 +98,15 @@ from numpypy import array, var a = array([[1,2],[3,4]]) assert var(a) == 1.25 - #assert (var(a,0) == array([ 1., 1.])).all() - #assert (var(a,1) == array([ 0.25, 0.25])).all() + assert (var(a,0) == array([ 1., 1.])).all() + assert (var(a,1) == array([ 0.25, 0.25])).all() def test_std(self): from numpypy import array, std a = array([[1, 2], [3, 4]]) assert std(a) == 1.1180339887498949 - #assert (std(a, axis=0) == array([ 1., 1.])).all() - #assert (std(a, axis=1) == array([ 0.5, 0.5])).all() + assert (std(a, axis=0) == array([ 1., 1.])).all() + assert (std(a, axis=1) == array([ 0.5, 0.5])).all() def test_mean(self): from numpypy import array, mean, arange @@ -136,4 +136,4 @@ raises(NotImplementedError, "transpose(x, axes=(1, 0, 2))") # x = ones((1, 2, 3)) # assert transpose(x, (1, 0, 2)).shape == (2, 1, 3) - + diff --git a/pypy/module/test_lib_pypy/numpypy/core/test_numeric.py b/pypy/module/test_lib_pypy/numpypy/core/test_numeric.py new file mode 100644 --- /dev/null +++ b/pypy/module/test_lib_pypy/numpypy/core/test_numeric.py @@ -0,0 +1,144 @@ + +from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest + + +class AppTestBaseRepr(BaseNumpyAppTest): + def test_base3(self): + from numpypy import base_repr + assert base_repr(3**5, 3) == '100000' + + def test_positive(self): + from numpypy import base_repr + assert base_repr(12, 10) == '12' + assert base_repr(12, 10, 4) == '000012' + assert base_repr(12, 4) == '30' + assert base_repr(3731624803700888, 36) == '10QR0ROFCEW' + + def test_negative(self): + from numpypy import base_repr + assert base_repr(-12, 10) == '-12' + assert base_repr(-12, 10, 4) == '-000012' + assert base_repr(-12, 4) == '-30' + +class AppTestRepr(BaseNumpyAppTest): + def test_repr(self): + from numpypy import array + assert repr(array([1, 2, 3, 4])) == 'array([1, 2, 3, 4])' + + def test_repr_2(self): + from numpypy import array, zeros + int_size = array(5).dtype.itemsize + a = array(range(5), float) + assert repr(a) == "array([ 0., 1., 2., 3., 4.])" + a = array([], float) + assert repr(a) == "array([], dtype=float64)" + a = zeros(1001) + assert repr(a) == "array([ 0., 0., 0., ..., 0., 0., 0.])" + a = array(range(5), long) + if a.dtype.itemsize == int_size: + assert repr(a) == "array([0, 1, 2, 3, 4])" + else: + assert repr(a) == "array([0, 1, 2, 3, 4], dtype=int64)" + a = array(range(5), 'int32') + if a.dtype.itemsize == int_size: + assert repr(a) == "array([0, 1, 2, 3, 4])" + else: + assert repr(a) == "array([0, 1, 2, 3, 4], dtype=int32)" + a = array([], long) + assert repr(a) == "array([], dtype=int64)" + a = array([True, False, True, False], "?") + assert repr(a) == "array([ True, False, True, False], dtype=bool)" + a = zeros([]) + assert repr(a) == "array(0.0)" + a = array(0.2) + assert repr(a) == "array(0.2)" + a = array([2]) + assert repr(a) == "array([2])" + + def test_repr_multi(self): + from numpypy import arange, zeros, array + a = zeros((3, 4)) + assert repr(a) == '''array([[ 0., 0., 0., 0.], + [ 0., 0., 0., 0.], + [ 0., 0., 0., 0.]])''' + a = zeros((2, 3, 4)) + assert repr(a) == '''array([[[ 0., 0., 0., 0.], + [ 0., 0., 0., 0.], + [ 0., 0., 0., 0.]], + + [[ 0., 0., 0., 0.], + [ 0., 0., 0., 0.], + [ 0., 0., 0., 0.]]])''' + a = arange(1002).reshape((2, 501)) + assert repr(a) == '''array([[ 0, 1, 2, ..., 498, 499, 500], + [ 501, 502, 503, ..., 999, 1000, 1001]])''' + assert repr(a.T) == '''array([[ 0, 501], + [ 1, 502], + [ 2, 503], + ..., + [ 498, 999], + [ 499, 1000], + [ 500, 1001]])''' + a = arange(2).reshape((2,1)) + assert repr(a) == '''array([[0], + [1]])''' + + def test_repr_slice(self): + from numpypy import array, zeros + a = array(range(5), float) + b = a[1::2] + assert repr(b) == "array([ 1., 3.])" + a = zeros(2002) + b = a[::2] + assert repr(b) == "array([ 0., 0., 0., ..., 0., 0., 0.])" + a = array((range(5), range(5, 10)), dtype="int16") + b = a[1, 2:] + assert repr(b) == "array([7, 8, 9], dtype=int16)" + # an empty slice prints its shape + b = a[2:1, ] + assert repr(b) == "array([], shape=(0, 5), dtype=int16)" + + def test_str(self): + from numpypy import array, zeros + a = array(range(5), float) + assert str(a) == "[ 0. 1. 2. 3. 4.]" + assert str((2 * a)[:]) == "[ 0. 2. 4. 6. 8.]" + a = zeros(1001) + assert str(a) == "[ 0. 0. 0. ..., 0. 0. 0.]" + + a = array(range(5), dtype=long) + assert str(a) == "[0 1 2 3 4]" + a = array([True, False, True, False], dtype="?") + assert str(a) == "[ True False True False]" + + a = array(range(5), dtype="int8") + assert str(a) == "[0 1 2 3 4]" + + a = array(range(5), dtype="int16") + assert str(a) == "[0 1 2 3 4]" + + a = array((range(5), range(5, 10)), dtype="int16") + assert str(a) == "[[0 1 2 3 4]\n [5 6 7 8 9]]" + + a = array(3, dtype=int) + assert str(a) == "3" + + a = zeros((400, 400), dtype=int) + assert str(a) == '[[0 0 0 ..., 0 0 0]\n [0 0 0 ..., 0 0 0]\n [0 0 0 ..., 0 0 0]\n ..., \n [0 0 0 ..., 0 0 0]\n [0 0 0 ..., 0 0 0]\n [0 0 0 ..., 0 0 0]]' + a = zeros((2, 2, 2)) + r = str(a) + assert r == '[[[ 0. 0.]\n [ 0. 0.]]\n\n [[ 0. 0.]\n [ 0. 0.]]]' + + def test_str_slice(self): + from numpypy import array, zeros + a = array(range(5), float) + b = a[1::2] + assert str(b) == "[ 1. 3.]" + a = zeros(2002) + b = a[::2] + assert str(b) == "[ 0. 0. 0. ..., 0. 0. 0.]" + a = array((range(5), range(5, 10)), dtype="int16") + b = a[1, 2:] + assert str(b) == "[7 8 9]" + b = a[2:1, ] + assert str(b) == "[]" diff --git a/pypy/rpython/lltypesystem/rdict.py b/pypy/rpython/lltypesystem/rdict.py --- a/pypy/rpython/lltypesystem/rdict.py +++ b/pypy/rpython/lltypesystem/rdict.py @@ -323,6 +323,17 @@ hop.exception_is_here() return hop.gendirectcall(ll_popitem, cTUPLE, v_dict) + def rtype_method_pop(self, hop): + if hop.nb_args == 2: + v_args = hop.inputargs(self, self.key_repr) + target = ll_pop + elif hop.nb_args == 3: + v_args = hop.inputargs(self, self.key_repr, self.value_repr) + target = ll_pop_default + hop.exception_is_here() + v_res = hop.gendirectcall(target, *v_args) + return self.recast_value(hop.llops, v_res) + class __extend__(pairtype(DictRepr, rmodel.Repr)): def rtype_getitem((r_dict, r_key), hop): @@ -874,3 +885,18 @@ r.item1 = recast(ELEM.TO.item1, entry.value) _ll_dict_del(dic, i) return r + +def ll_pop(dic, key): + i = ll_dict_lookup(dic, key, dic.keyhash(key)) + if not i & HIGHEST_BIT: + value = ll_get_value(dic, i) + _ll_dict_del(dic, i) + return value + else: + raise KeyError + +def ll_pop_default(dic, key, dfl): + try: + return ll_pop(dic, key) + except KeyError: + return dfl diff --git a/pypy/rpython/ootypesystem/rdict.py b/pypy/rpython/ootypesystem/rdict.py --- a/pypy/rpython/ootypesystem/rdict.py +++ b/pypy/rpython/ootypesystem/rdict.py @@ -160,6 +160,16 @@ hop.exception_is_here() return hop.gendirectcall(ll_popitem, cTUPLE, v_dict) + def rtype_method_pop(self, hop): + if hop.nb_args == 2: + v_args = hop.inputargs(self, self.key_repr) + target = ll_pop + elif hop.nb_args == 3: + v_args = hop.inputargs(self, self.key_repr, self.value_repr) + target = ll_pop_default + hop.exception_is_here() + return hop.gendirectcall(target, *v_args) + def __get_func(self, interp, r_func, fn, TYPE): if isinstance(r_func, MethodOfFrozenPBCRepr): obj = r_func.r_im_self.convert_const(fn.im_self) @@ -370,6 +380,20 @@ return res raise KeyError +def ll_pop(d, key): + if d.ll_contains(key): + value = d.ll_get(key) + d.ll_remove(key) + return value + else: + raise KeyError + +def ll_pop_default(d, key, dfl): + try: + return ll_pop(d, key) + except KeyError: + return dfl + # ____________________________________________________________ # # Iteration. diff --git a/pypy/rpython/test/test_rdict.py b/pypy/rpython/test/test_rdict.py --- a/pypy/rpython/test/test_rdict.py +++ b/pypy/rpython/test/test_rdict.py @@ -25,37 +25,37 @@ class BaseTestRdict(BaseRtypingTest): def test_dict_creation(self): - def createdict(i): + def createdict(i): d = {'hello' : i} return d['hello'] res = self.interpret(createdict, [42]) assert res == 42 - def test_dict_getitem_setitem(self): - def func(i): + def test_dict_getitem_setitem(self): + def func(i): d = {'hello' : i} d['world'] = i + 1 - return d['hello'] * d['world'] + return d['hello'] * d['world'] res = self.interpret(func, [6]) assert res == 42 - def test_dict_getitem_keyerror(self): - def func(i): + def test_dict_getitem_keyerror(self): + def func(i): d = {'hello' : i} try: return d['world'] except KeyError: - return 0 + return 0 res = self.interpret(func, [6]) assert res == 0 def test_dict_del_simple(self): - def func(i): + def func(i): d = {'hello' : i} d['world'] = i + 1 del d['hello'] - return len(d) + return len(d) res = self.interpret(func, [6]) assert res == 1 @@ -71,7 +71,7 @@ assert res == True def test_empty_strings(self): - def func(i): + def func(i): d = {'' : i} del d[''] try: @@ -83,7 +83,7 @@ res = self.interpret(func, [6]) assert res == 1 - def func(i): + def func(i): d = {'' : i} del d[''] d[''] = i + 1 @@ -146,8 +146,8 @@ d1 = {} d1['hello'] = i + 1 d2 = {} - d2['world'] = d1 - return d2['world']['hello'] + d2['world'] = d1 + return d2['world']['hello'] res = self.interpret(func, [5]) assert res == 6 @@ -297,7 +297,7 @@ a = 0 for k, v in items: b += isinstance(k, B) - a += isinstance(v, A) + a += isinstance(v, A) return 3*b+a res = self.interpret(func, []) assert res == 8 @@ -316,7 +316,7 @@ a = 0 for k, v in dic.iteritems(): b += isinstance(k, B) - a += isinstance(v, A) + a += isinstance(v, A) return 3*b+a res = self.interpret(func, []) assert res == 8 @@ -342,11 +342,11 @@ def test_dict_contains_with_constant_dict(self): dic = {'4':1000, ' 8':200} def func(i): - return chr(i) in dic - res = self.interpret(func, [ord('4')]) + return chr(i) in dic + res = self.interpret(func, [ord('4')]) assert res is True - res = self.interpret(func, [1]) - assert res is False + res = self.interpret(func, [1]) + assert res is False def test_dict_or_none(self): class A: @@ -413,7 +413,7 @@ return g(get) res = self.interpret(f, []) - assert res == 2 + assert res == 2 def test_specific_obscure_bug(self): class A: pass @@ -622,6 +622,42 @@ res = self.interpret(func, []) assert res in [5263, 6352] + def test_dict_pop(self): + def f(n, default): + d = {} + d[2] = 3 + d[4] = 5 + if default == -1: + try: + x = d.pop(n) + except KeyError: + x = -1 + else: + x = d.pop(n, default) + return x * 10 + len(d) + res = self.interpret(f, [2, -1]) + assert res == 31 + res = self.interpret(f, [3, -1]) + assert res == -8 + res = self.interpret(f, [2, 5]) + assert res == 31 + + def test_dict_pop_instance(self): + class A(object): + pass + def f(n): + d = {} + d[2] = A() + x = d.pop(n, None) + if x is None: + return 12 + else: + return 15 + res = self.interpret(f, [2]) + assert res == 15 + res = self.interpret(f, [700]) + assert res == 12 + class TestLLtype(BaseTestRdict, LLRtypeMixin): def test_dict_but_not_with_char_keys(self): def func(i): @@ -633,19 +669,19 @@ res = self.interpret(func, [6]) assert res == 0 - def test_deleted_entry_reusage_with_colliding_hashes(self): - def lowlevelhash(value): + def test_deleted_entry_reusage_with_colliding_hashes(self): + def lowlevelhash(value): p = rstr.mallocstr(len(value)) for i in range(len(value)): p.chars[i] = value[i] - return rstr.LLHelpers.ll_strhash(p) + return rstr.LLHelpers.ll_strhash(p) - def func(c1, c2): - c1 = chr(c1) - c2 = chr(c2) + def func(c1, c2): + c1 = chr(c1) + c2 = chr(c2) d = {} d[c1] = 1 - d[c2] = 2 + d[c2] = 2 del d[c1] return d[c2] @@ -653,7 +689,7 @@ base = rdict.DICT_INITSIZE for y in range(0, 256): y = chr(y) - y_hash = lowlevelhash(y) % base + y_hash = lowlevelhash(y) % base char_by_hash.setdefault(y_hash, []).append(y) x, y = char_by_hash[0][:2] # find a collision @@ -661,18 +697,18 @@ res = self.interpret(func, [ord(x), ord(y)]) assert res == 2 - def func2(c1, c2): - c1 = chr(c1) - c2 = chr(c2) + def func2(c1, c2): + c1 = chr(c1) + c2 = chr(c2) d = {} d[c1] = 1 - d[c2] = 2 + d[c2] = 2 del d[c1] d[c1] = 3 - return d + return d res = self.interpret(func2, [ord(x), ord(y)]) - for i in range(len(res.entries)): + for i in range(len(res.entries)): assert not (res.entries.everused(i) and not res.entries.valid(i)) def func3(c0, c1, c2, c3, c4, c5, c6, c7): @@ -687,9 +723,9 @@ c7 = chr(c7) ; d[c7] = 1; del d[c7] return d - if rdict.DICT_INITSIZE != 8: + if rdict.DICT_INITSIZE != 8: py.test.skip("make dict tests more indepdent from initsize") - res = self.interpret(func3, [ord(char_by_hash[i][0]) + res = self.interpret(func3, [ord(char_by_hash[i][0]) for i in range(rdict.DICT_INITSIZE)]) count_frees = 0 for i in range(len(res.entries)): @@ -707,9 +743,9 @@ del d[chr(ord('a') + i)] return d res = self.interpret(func, [0]) - assert len(res.entries) > rdict.DICT_INITSIZE + assert len(res.entries) > rdict.DICT_INITSIZE res = self.interpret(func, [1]) - assert len(res.entries) == rdict.DICT_INITSIZE + assert len(res.entries) == rdict.DICT_INITSIZE def test_dict_valid_resize(self): # see if we find our keys after resize @@ -844,7 +880,7 @@ def test_prebuilt_list_of_addresses(self): from pypy.rpython.lltypesystem import llmemory - + TP = lltype.Struct('x', ('y', lltype.Signed)) a = lltype.malloc(TP, flavor='raw', immortal=True) b = lltype.malloc(TP, flavor='raw', immortal=True) @@ -858,7 +894,7 @@ d = {a_a: 3, a_b: 4, a_c: 5} d[a0] = 8 - + def func(i): if i == 0: ptr = a @@ -888,7 +924,7 @@ return a == b def rhash(a): return 3 - + def func(i): d = r_dict(eq, rhash, force_non_null=True) if not i: From noreply at buildbot.pypy.org Fri Jan 27 21:55:01 2012 From: noreply at buildbot.pypy.org (edelsohn) Date: Fri, 27 Jan 2012 21:55:01 +0100 (CET) Subject: [pypy-commit] pypy ppc-jit-backend: Add ppc64_64 backend name. Message-ID: <20120127205501.9A09982114@wyvern.cs.uni-duesseldorf.de> Author: edelsohn Branch: ppc-jit-backend Changeset: r51890:1a4cf2e87fd7 Date: 2012-01-27 15:53 -0500 http://bitbucket.org/pypy/pypy/changeset/1a4cf2e87fd7/ Log: Add ppc64_64 backend name. diff --git a/pypy/jit/backend/detect_cpu.py b/pypy/jit/backend/detect_cpu.py --- a/pypy/jit/backend/detect_cpu.py +++ b/pypy/jit/backend/detect_cpu.py @@ -79,6 +79,8 @@ return "pypy.jit.backend.arm.runner", "ArmCPU" elif backend_name == 'ppc64': return "pypy.jit.backend.ppc.runner", "PPC_64_CPU" + elif backend_name == 'ppc64_64': + return "pypy.jit.backend.ppc.runner", "PPC_64_CPU" else: raise ProcessorAutodetectError, ( "we have no JIT backend for this cpu: '%s'" % backend_name) From noreply at buildbot.pypy.org Fri Jan 27 21:55:02 2012 From: noreply at buildbot.pypy.org (edelsohn) Date: Fri, 27 Jan 2012 21:55:02 +0100 (CET) Subject: [pypy-commit] pypy ppc-jit-backend: PPC version of test_ztranslation Message-ID: <20120127205502.CE79D82115@wyvern.cs.uni-duesseldorf.de> Author: edelsohn Branch: ppc-jit-backend Changeset: r51891:d52428409628 Date: 2012-01-27 15:54 -0500 http://bitbucket.org/pypy/pypy/changeset/d52428409628/ Log: PPC version of test_ztranslation diff --git a/pypy/jit/backend/ppc/test/test_ztranslation.py b/pypy/jit/backend/ppc/test/test_ztranslation.py new file mode 100644 --- /dev/null +++ b/pypy/jit/backend/ppc/test/test_ztranslation.py @@ -0,0 +1,255 @@ +import py, os, sys +from pypy.tool.udir import udir +from pypy.rlib.jit import JitDriver, unroll_parameters, set_param +from pypy.rlib.jit import PARAMETERS, dont_look_inside +from pypy.rlib.jit import promote +from pypy.jit.metainterp.jitprof import Profiler +from pypy.jit.backend.detect_cpu import getcpuclass +from pypy.jit.backend.test.support import CCompiledMixin +from pypy.jit.codewriter.policy import StopAtXPolicy +from pypy.translator.translator import TranslationContext +from pypy.jit.backend.ppc.ppcgen.arch import IS_PPC_32, IS_PPC_64 +from pypy.config.translationoption import DEFL_GC +from pypy.rlib import rgc + +class TestTranslationPPC(CCompiledMixin): + CPUClass = getcpuclass() + + def _check_cbuilder(self, cbuilder): + # We assume here that we have sse2. If not, the CPUClass + # needs to be changed to CPU386_NO_SSE2, but well. + assert '-msse2' in cbuilder.eci.compile_extra + assert '-mfpmath=sse' in cbuilder.eci.compile_extra + + def test_stuff_translates(self): + # this is a basic test that tries to hit a number of features and their + # translation: + # - jitting of loops and bridges + # - virtualizables + # - set_param interface + # - profiler + # - full optimizer + # - floats neg and abs + + class Frame(object): + _virtualizable2_ = ['i'] + + def __init__(self, i): + self.i = i + + @dont_look_inside + def myabs(x): + return abs(x) + + jitdriver = JitDriver(greens = [], + reds = ['total', 'frame', 'j'], + virtualizables = ['frame']) + def f(i, j): + for param, _ in unroll_parameters: + defl = PARAMETERS[param] + set_param(jitdriver, param, defl) + set_param(jitdriver, "threshold", 3) + set_param(jitdriver, "trace_eagerness", 2) + total = 0 + frame = Frame(i) + while frame.i > 3: + jitdriver.can_enter_jit(frame=frame, total=total, j=j) + jitdriver.jit_merge_point(frame=frame, total=total, j=j) + total += frame.i + if frame.i >= 20: + frame.i -= 2 + frame.i -= 1 + j *= -0.712 + if j + (-j): raise ValueError + k = myabs(j) + if k - abs(j): raise ValueError + if k - abs(-j): raise ValueError + return chr(total % 253) + # + from pypy.rpython.lltypesystem import lltype, rffi + from pypy.rlib.libffi import types, CDLL, ArgChain + from pypy.rlib.test.test_libffi import get_libm_name + libm_name = get_libm_name(sys.platform) + jitdriver2 = JitDriver(greens=[], reds = ['i', 'func', 'res', 'x']) + def libffi_stuff(i, j): + lib = CDLL(libm_name) + func = lib.getpointer('fabs', [types.double], types.double) + res = 0.0 + x = float(j) + while i > 0: + jitdriver2.jit_merge_point(i=i, res=res, func=func, x=x) + promote(func) + argchain = ArgChain() + argchain.arg(x) + res = func.call(argchain, rffi.DOUBLE) + i -= 1 + return res + # + def main(i, j): + a_char = f(i, j) + a_float = libffi_stuff(i, j) + return ord(a_char) * 10 + int(a_float) + expected = main(40, -49) + res = self.meta_interp(main, [40, -49]) + assert res == expected + + def test_direct_assembler_call_translates(self): + """Test CALL_ASSEMBLER and the recursion limit""" + from pypy.rlib.rstackovf import StackOverflow + + class Thing(object): + def __init__(self, val): + self.val = val + + class Frame(object): + _virtualizable2_ = ['thing'] + + driver = JitDriver(greens = ['codeno'], reds = ['i', 'frame'], + virtualizables = ['frame'], + get_printable_location = lambda codeno: str(codeno)) + class SomewhereElse(object): + pass + + somewhere_else = SomewhereElse() + + def change(newthing): + somewhere_else.frame.thing = newthing + + def main(codeno): + frame = Frame() + somewhere_else.frame = frame + frame.thing = Thing(0) + portal(codeno, frame) + return frame.thing.val + + def portal(codeno, frame): + i = 0 + while i < 10: + driver.can_enter_jit(frame=frame, codeno=codeno, i=i) + driver.jit_merge_point(frame=frame, codeno=codeno, i=i) + nextval = frame.thing.val + if codeno == 0: + subframe = Frame() + subframe.thing = Thing(nextval) + nextval = portal(1, subframe) + elif frame.thing.val > 40: + change(Thing(13)) + nextval = 13 + frame.thing = Thing(nextval + 1) + i += 1 + return frame.thing.val + + driver2 = JitDriver(greens = [], reds = ['n']) + + def main2(bound): + try: + while portal2(bound) == -bound+1: + bound *= 2 + except StackOverflow: + pass + return bound + + def portal2(n): + while True: + driver2.jit_merge_point(n=n) + n -= 1 + if n <= 0: + return n + n = portal2(n) + assert portal2(10) == -9 + + def mainall(codeno, bound): + return main(codeno) + main2(bound) + + res = self.meta_interp(mainall, [0, 1], inline=True, + policy=StopAtXPolicy(change)) + print hex(res) + assert res & 255 == main(0) + bound = res & ~255 + assert 1024 <= bound <= 131072 + assert bound & (bound-1) == 0 # a power of two + + +class TestTranslationRemoveTypePtrPPC(CCompiledMixin): + CPUClass = getcpuclass() + + def _get_TranslationContext(self): + t = TranslationContext() + t.config.translation.gc = DEFL_GC # 'hybrid' or 'minimark' + t.config.translation.gcrootfinder = 'asmgcc' + t.config.translation.list_comprehension_operations = True + t.config.translation.gcremovetypeptr = True + return t + + def test_external_exception_handling_translates(self): + jitdriver = JitDriver(greens = [], reds = ['n', 'total']) + + class ImDone(Exception): + def __init__(self, resvalue): + self.resvalue = resvalue + + @dont_look_inside + def f(x, total): + if x <= 30: + raise ImDone(total * 10) + if x > 200: + return 2 + raise ValueError + @dont_look_inside + def g(x): + if x > 150: + raise ValueError + return 2 + class Base: + def meth(self): + return 2 + class Sub(Base): + def meth(self): + return 1 + @dont_look_inside + def h(x): + if x < 20000: + return Sub() + else: + return Base() + def myportal(i): + set_param(jitdriver, "threshold", 3) + set_param(jitdriver, "trace_eagerness", 2) + total = 0 + n = i + while True: + jitdriver.can_enter_jit(n=n, total=total) + jitdriver.jit_merge_point(n=n, total=total) + try: + total += f(n, total) + except ValueError: + total += 1 + try: + total += g(n) + except ValueError: + total -= 1 + n -= h(n).meth() # this is to force a GUARD_CLASS + def main(i): + try: + myportal(i) + except ImDone, e: + return e.resvalue + + # XXX custom fishing, depends on the exact env var and format + logfile = udir.join('test_ztranslation.log') + os.environ['PYPYLOG'] = 'jit-log-opt:%s' % (logfile,) + try: + res = self.meta_interp(main, [400]) + assert res == main(400) + finally: + del os.environ['PYPYLOG'] + + guard_class = 0 + for line in open(str(logfile)): + if 'guard_class' in line: + guard_class += 1 + # if we get many more guard_classes, it means that we generate + # guards that always fail (the following assert's original purpose + # is to catch the following case: each GUARD_CLASS is misgenerated + # and always fails with "gcremovetypeptr") + assert 0 < guard_class < 10 From noreply at buildbot.pypy.org Fri Jan 27 22:20:12 2012 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 27 Jan 2012 22:20:12 +0100 (CET) Subject: [pypy-commit] pypy ppc-jit-backend: kill some already supposed to be dead people Message-ID: <20120127212012.1DC7082114@wyvern.cs.uni-duesseldorf.de> Author: fijal Branch: ppc-jit-backend Changeset: r51892:086f6c58e2e9 Date: 2012-01-27 13:17 -0800 http://bitbucket.org/pypy/pypy/changeset/086f6c58e2e9/ Log: kill some already supposed to be dead people diff --git a/pypy/jit/backend/detect_cpu.py b/pypy/jit/backend/detect_cpu.py --- a/pypy/jit/backend/detect_cpu.py +++ b/pypy/jit/backend/detect_cpu.py @@ -32,6 +32,7 @@ 'x86': 'x86', # Apple 'Power Macintosh': 'ppc', 'ppc64': 'ppc64', + 'ppc64_64': 'ppc64', 'x86_64': 'x86', 'amd64': 'x86', # freebsd 'AMD64': 'x86', # win64 @@ -79,6 +80,8 @@ return "pypy.jit.backend.arm.runner", "ArmCPU" elif backend_name == 'ppc64': return "pypy.jit.backend.ppc.runner", "PPC_64_CPU" + elif backend_name == 'ppc64_64': + return "pypy.jit.backend.ppc.runner", "PPC_64_CPU" else: raise ProcessorAutodetectError, ( "we have no JIT backend for this cpu: '%s'" % backend_name) diff --git a/pypy/jit/backend/ppc/ppcgen/form.py b/pypy/jit/backend/ppc/ppcgen/form.py --- a/pypy/jit/backend/ppc/ppcgen/form.py +++ b/pypy/jit/backend/ppc/ppcgen/form.py @@ -84,7 +84,7 @@ self.defaults = self.defaults + ((self.fieldmap[f], v),) return self - def __call__(self, **more_specializatons): + def call(self, **more_specializatons): s = self.specializations.copy() ms = {} ds = {} diff --git a/pypy/jit/backend/ppc/test/test_genc_ts.py b/pypy/jit/backend/ppc/test/test_genc_ts.py deleted file mode 100644 --- a/pypy/jit/backend/ppc/test/test_genc_ts.py +++ /dev/null @@ -1,16 +0,0 @@ -import py -from pypy.jit.codegen.i386.test.test_genc_ts import I386TimeshiftingTestMixin -from pypy.jit.timeshifter.test import test_timeshift -from pypy.jit.codegen.ppc.rgenop import RPPCGenOp - -class PPCTimeshiftingTestMixin(I386TimeshiftingTestMixin): - RGenOp = RPPCGenOp - -class TestTimeshiftPPC(PPCTimeshiftingTestMixin, - test_timeshift.TestLLType): - - # for the individual tests see - # ====> ../../../timeshifter/test/test_timeshift.py - - pass - From noreply at buildbot.pypy.org Fri Jan 27 22:20:13 2012 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 27 Jan 2012 22:20:13 +0100 (CET) Subject: [pypy-commit] pypy ppc-jit-backend: merge default Message-ID: <20120127212013.531BE82114@wyvern.cs.uni-duesseldorf.de> Author: fijal Branch: ppc-jit-backend Changeset: r51893:20a7afe2c42d Date: 2012-01-27 13:18 -0800 http://bitbucket.org/pypy/pypy/changeset/20a7afe2c42d/ Log: merge default diff --git a/pypy/jit/backend/ppc/test/test_ztranslation.py b/pypy/jit/backend/ppc/test/test_ztranslation.py new file mode 100644 --- /dev/null +++ b/pypy/jit/backend/ppc/test/test_ztranslation.py @@ -0,0 +1,255 @@ +import py, os, sys +from pypy.tool.udir import udir +from pypy.rlib.jit import JitDriver, unroll_parameters, set_param +from pypy.rlib.jit import PARAMETERS, dont_look_inside +from pypy.rlib.jit import promote +from pypy.jit.metainterp.jitprof import Profiler +from pypy.jit.backend.detect_cpu import getcpuclass +from pypy.jit.backend.test.support import CCompiledMixin +from pypy.jit.codewriter.policy import StopAtXPolicy +from pypy.translator.translator import TranslationContext +from pypy.jit.backend.ppc.ppcgen.arch import IS_PPC_32, IS_PPC_64 +from pypy.config.translationoption import DEFL_GC +from pypy.rlib import rgc + +class TestTranslationPPC(CCompiledMixin): + CPUClass = getcpuclass() + + def _check_cbuilder(self, cbuilder): + # We assume here that we have sse2. If not, the CPUClass + # needs to be changed to CPU386_NO_SSE2, but well. + assert '-msse2' in cbuilder.eci.compile_extra + assert '-mfpmath=sse' in cbuilder.eci.compile_extra + + def test_stuff_translates(self): + # this is a basic test that tries to hit a number of features and their + # translation: + # - jitting of loops and bridges + # - virtualizables + # - set_param interface + # - profiler + # - full optimizer + # - floats neg and abs + + class Frame(object): + _virtualizable2_ = ['i'] + + def __init__(self, i): + self.i = i + + @dont_look_inside + def myabs(x): + return abs(x) + + jitdriver = JitDriver(greens = [], + reds = ['total', 'frame', 'j'], + virtualizables = ['frame']) + def f(i, j): + for param, _ in unroll_parameters: + defl = PARAMETERS[param] + set_param(jitdriver, param, defl) + set_param(jitdriver, "threshold", 3) + set_param(jitdriver, "trace_eagerness", 2) + total = 0 + frame = Frame(i) + while frame.i > 3: + jitdriver.can_enter_jit(frame=frame, total=total, j=j) + jitdriver.jit_merge_point(frame=frame, total=total, j=j) + total += frame.i + if frame.i >= 20: + frame.i -= 2 + frame.i -= 1 + j *= -0.712 + if j + (-j): raise ValueError + k = myabs(j) + if k - abs(j): raise ValueError + if k - abs(-j): raise ValueError + return chr(total % 253) + # + from pypy.rpython.lltypesystem import lltype, rffi + from pypy.rlib.libffi import types, CDLL, ArgChain + from pypy.rlib.test.test_libffi import get_libm_name + libm_name = get_libm_name(sys.platform) + jitdriver2 = JitDriver(greens=[], reds = ['i', 'func', 'res', 'x']) + def libffi_stuff(i, j): + lib = CDLL(libm_name) + func = lib.getpointer('fabs', [types.double], types.double) + res = 0.0 + x = float(j) + while i > 0: + jitdriver2.jit_merge_point(i=i, res=res, func=func, x=x) + promote(func) + argchain = ArgChain() + argchain.arg(x) + res = func.call(argchain, rffi.DOUBLE) + i -= 1 + return res + # + def main(i, j): + a_char = f(i, j) + a_float = libffi_stuff(i, j) + return ord(a_char) * 10 + int(a_float) + expected = main(40, -49) + res = self.meta_interp(main, [40, -49]) + assert res == expected + + def test_direct_assembler_call_translates(self): + """Test CALL_ASSEMBLER and the recursion limit""" + from pypy.rlib.rstackovf import StackOverflow + + class Thing(object): + def __init__(self, val): + self.val = val + + class Frame(object): + _virtualizable2_ = ['thing'] + + driver = JitDriver(greens = ['codeno'], reds = ['i', 'frame'], + virtualizables = ['frame'], + get_printable_location = lambda codeno: str(codeno)) + class SomewhereElse(object): + pass + + somewhere_else = SomewhereElse() + + def change(newthing): + somewhere_else.frame.thing = newthing + + def main(codeno): + frame = Frame() + somewhere_else.frame = frame + frame.thing = Thing(0) + portal(codeno, frame) + return frame.thing.val + + def portal(codeno, frame): + i = 0 + while i < 10: + driver.can_enter_jit(frame=frame, codeno=codeno, i=i) + driver.jit_merge_point(frame=frame, codeno=codeno, i=i) + nextval = frame.thing.val + if codeno == 0: + subframe = Frame() + subframe.thing = Thing(nextval) + nextval = portal(1, subframe) + elif frame.thing.val > 40: + change(Thing(13)) + nextval = 13 + frame.thing = Thing(nextval + 1) + i += 1 + return frame.thing.val + + driver2 = JitDriver(greens = [], reds = ['n']) + + def main2(bound): + try: + while portal2(bound) == -bound+1: + bound *= 2 + except StackOverflow: + pass + return bound + + def portal2(n): + while True: + driver2.jit_merge_point(n=n) + n -= 1 + if n <= 0: + return n + n = portal2(n) + assert portal2(10) == -9 + + def mainall(codeno, bound): + return main(codeno) + main2(bound) + + res = self.meta_interp(mainall, [0, 1], inline=True, + policy=StopAtXPolicy(change)) + print hex(res) + assert res & 255 == main(0) + bound = res & ~255 + assert 1024 <= bound <= 131072 + assert bound & (bound-1) == 0 # a power of two + + +class TestTranslationRemoveTypePtrPPC(CCompiledMixin): + CPUClass = getcpuclass() + + def _get_TranslationContext(self): + t = TranslationContext() + t.config.translation.gc = DEFL_GC # 'hybrid' or 'minimark' + t.config.translation.gcrootfinder = 'asmgcc' + t.config.translation.list_comprehension_operations = True + t.config.translation.gcremovetypeptr = True + return t + + def test_external_exception_handling_translates(self): + jitdriver = JitDriver(greens = [], reds = ['n', 'total']) + + class ImDone(Exception): + def __init__(self, resvalue): + self.resvalue = resvalue + + @dont_look_inside + def f(x, total): + if x <= 30: + raise ImDone(total * 10) + if x > 200: + return 2 + raise ValueError + @dont_look_inside + def g(x): + if x > 150: + raise ValueError + return 2 + class Base: + def meth(self): + return 2 + class Sub(Base): + def meth(self): + return 1 + @dont_look_inside + def h(x): + if x < 20000: + return Sub() + else: + return Base() + def myportal(i): + set_param(jitdriver, "threshold", 3) + set_param(jitdriver, "trace_eagerness", 2) + total = 0 + n = i + while True: + jitdriver.can_enter_jit(n=n, total=total) + jitdriver.jit_merge_point(n=n, total=total) + try: + total += f(n, total) + except ValueError: + total += 1 + try: + total += g(n) + except ValueError: + total -= 1 + n -= h(n).meth() # this is to force a GUARD_CLASS + def main(i): + try: + myportal(i) + except ImDone, e: + return e.resvalue + + # XXX custom fishing, depends on the exact env var and format + logfile = udir.join('test_ztranslation.log') + os.environ['PYPYLOG'] = 'jit-log-opt:%s' % (logfile,) + try: + res = self.meta_interp(main, [400]) + assert res == main(400) + finally: + del os.environ['PYPYLOG'] + + guard_class = 0 + for line in open(str(logfile)): + if 'guard_class' in line: + guard_class += 1 + # if we get many more guard_classes, it means that we generate + # guards that always fail (the following assert's original purpose + # is to catch the following case: each GUARD_CLASS is misgenerated + # and always fails with "gcremovetypeptr") + assert 0 < guard_class < 10 From noreply at buildbot.pypy.org Fri Jan 27 23:30:09 2012 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 27 Jan 2012 23:30:09 +0100 (CET) Subject: [pypy-commit] pypy ppc-jit-backend: this was definitely not supposed to be commited Message-ID: <20120127223009.199A582114@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: ppc-jit-backend Changeset: r51894:77891e3dee7b Date: 2012-01-28 00:29 +0200 http://bitbucket.org/pypy/pypy/changeset/77891e3dee7b/ Log: this was definitely not supposed to be commited diff --git a/pypy/jit/backend/ppc/ppcgen/form.py b/pypy/jit/backend/ppc/ppcgen/form.py --- a/pypy/jit/backend/ppc/ppcgen/form.py +++ b/pypy/jit/backend/ppc/ppcgen/form.py @@ -84,14 +84,14 @@ self.defaults = self.defaults + ((self.fieldmap[f], v),) return self - def call(self, **more_specializatons): + def __call__(self, **more_specializatons): s = self.specializations.copy() ms = {} ds = {} for fname, v in more_specializatons.iteritems(): field = self.fieldmap[fname] if field not in self.fields: - raise FormException, "don't know about '%s' here"%k + raise FormException, "don't know about '%s' here" % field if isinstance(v, str): ds[field] = self.fieldmap[v] else: From noreply at buildbot.pypy.org Fri Jan 27 23:45:14 2012 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 27 Jan 2012 23:45:14 +0100 (CET) Subject: [pypy-commit] pypy ppc-jit-backend: rpythonize the assembler Message-ID: <20120127224514.2804482114@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: ppc-jit-backend Changeset: r51895:71425577dae9 Date: 2012-01-28 00:44 +0200 http://bitbucket.org/pypy/pypy/changeset/71425577dae9/ Log: rpythonize the assembler diff --git a/pypy/jit/backend/ppc/ppcgen/codebuilder.py b/pypy/jit/backend/ppc/ppcgen/codebuilder.py --- a/pypy/jit/backend/ppc/ppcgen/codebuilder.py +++ b/pypy/jit/backend/ppc/ppcgen/codebuilder.py @@ -25,6 +25,7 @@ from pypy.jit.backend.x86.support import values_array from pypy.tool.udir import udir from pypy.rlib.objectmodel import we_are_translated +from pypy.jit.backend.ppc.ppcgen.rassemblermaker import make_rassembler from pypy.translator.tool.cbuild import ExternalCompilationInfo @@ -960,7 +961,7 @@ for i, new_inst in enumerate(self.insts): self.cb.insts[i + startindex] = new_inst -class PPCBuilder(BlockBuilderMixin, PPCAssembler): +class PPCBuilder(BlockBuilderMixin, make_rassembler(PPCAssembler)): def __init__(self, failargs_limit=1000, r0_in_use=False): PPCAssembler.__init__(self) self.init_block_builder() From noreply at buildbot.pypy.org Fri Jan 27 23:50:52 2012 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 27 Jan 2012 23:50:52 +0100 (CET) Subject: [pypy-commit] pypy ppc-jit-backend: Backed out changeset 71425577dae9 Message-ID: <20120127225052.CB69182114@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: ppc-jit-backend Changeset: r51896:8c2aa655427d Date: 2012-01-28 00:50 +0200 http://bitbucket.org/pypy/pypy/changeset/8c2aa655427d/ Log: Backed out changeset 71425577dae9 diff --git a/pypy/jit/backend/ppc/ppcgen/codebuilder.py b/pypy/jit/backend/ppc/ppcgen/codebuilder.py --- a/pypy/jit/backend/ppc/ppcgen/codebuilder.py +++ b/pypy/jit/backend/ppc/ppcgen/codebuilder.py @@ -25,7 +25,6 @@ from pypy.jit.backend.x86.support import values_array from pypy.tool.udir import udir from pypy.rlib.objectmodel import we_are_translated -from pypy.jit.backend.ppc.ppcgen.rassemblermaker import make_rassembler from pypy.translator.tool.cbuild import ExternalCompilationInfo @@ -961,7 +960,7 @@ for i, new_inst in enumerate(self.insts): self.cb.insts[i + startindex] = new_inst -class PPCBuilder(BlockBuilderMixin, make_rassembler(PPCAssembler)): +class PPCBuilder(BlockBuilderMixin, PPCAssembler): def __init__(self, failargs_limit=1000, r0_in_use=False): PPCAssembler.__init__(self) self.init_block_builder() From noreply at buildbot.pypy.org Sat Jan 28 00:08:02 2012 From: noreply at buildbot.pypy.org (edelsohn) Date: Sat, 28 Jan 2012 00:08:02 +0100 (CET) Subject: [pypy-commit] pypy ppc-jit-backend: Add test_zll_random for PPC. Message-ID: <20120127230802.DD09982114@wyvern.cs.uni-duesseldorf.de> Author: edelsohn Branch: ppc-jit-backend Changeset: r51897:0696f19f1bba Date: 2012-01-27 17:15 -0500 http://bitbucket.org/pypy/pypy/changeset/0696f19f1bba/ Log: Add test_zll_random for PPC. diff --git a/pypy/jit/backend/ppc/test/test_zll_random.py b/pypy/jit/backend/ppc/test/test_zll_random.py new file mode 100644 --- /dev/null +++ b/pypy/jit/backend/ppc/test/test_zll_random.py @@ -0,0 +1,12 @@ +from pypy.jit.backend.test.test_random import check_random_function, Random +from pypy.jit.backend.test.test_ll_random import LLtypeOperationBuilder +from pypy.jit.backend.detect_cpu import getcpuclass + +CPU = getcpuclass() + +def test_stress(): + cpu = CPU(None, None) + cpu.setup_once() + r = Random() + for i in range(1000): + check_random_function(cpu, LLtypeOperationBuilder, r, i, 1000) From noreply at buildbot.pypy.org Sat Jan 28 00:08:04 2012 From: noreply at buildbot.pypy.org (edelsohn) Date: Sat, 28 Jan 2012 00:08:04 +0100 (CET) Subject: [pypy-commit] pypy ppc-jit-backend: merge Message-ID: <20120127230804.2172A82114@wyvern.cs.uni-duesseldorf.de> Author: edelsohn Branch: ppc-jit-backend Changeset: r51898:8e5ed1cad830 Date: 2012-01-27 18:07 -0500 http://bitbucket.org/pypy/pypy/changeset/8e5ed1cad830/ Log: merge diff --git a/pypy/jit/backend/ppc/test/test_zll_random.py b/pypy/jit/backend/ppc/test/test_zll_random.py new file mode 100644 --- /dev/null +++ b/pypy/jit/backend/ppc/test/test_zll_random.py @@ -0,0 +1,12 @@ +from pypy.jit.backend.test.test_random import check_random_function, Random +from pypy.jit.backend.test.test_ll_random import LLtypeOperationBuilder +from pypy.jit.backend.detect_cpu import getcpuclass + +CPU = getcpuclass() + +def test_stress(): + cpu = CPU(None, None) + cpu.setup_once() + r = Random() + for i in range(1000): + check_random_function(cpu, LLtypeOperationBuilder, r, i, 1000) From noreply at buildbot.pypy.org Sat Jan 28 00:09:58 2012 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 28 Jan 2012 00:09:58 +0100 (CET) Subject: [pypy-commit] pypy default: improve fake objspace and fix translation Message-ID: <20120127230958.9200A82114@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: Changeset: r51899:c9343ef21049 Date: 2012-01-28 01:09 +0200 http://bitbucket.org/pypy/pypy/changeset/c9343ef21049/ Log: improve fake objspace and fix translation diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -285,7 +285,7 @@ def descr_flatten(self, space, w_order=None): if isinstance(self, Scalar): # scalars have no storage - return self.descr_reshape(space, [space.wrap([1])]) + return self.descr_reshape(space, [space.wrap(1)]) concr = self.get_concrete() w_res = concr.descr_ravel(space, w_order) if w_res.storage == concr.storage: diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py --- a/pypy/objspace/fake/objspace.py +++ b/pypy/objspace/fake/objspace.py @@ -150,6 +150,8 @@ self._see_getsetproperty(x) if isinstance(x, r_singlefloat): self._wrap_not_rpython(x) + if isinstance(x, list): + self._wrap_not_rpython(x) return w_some_obj() wrap._annspecialcase_ = "specialize:argtype(1)" From noreply at buildbot.pypy.org Sat Jan 28 10:26:21 2012 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 28 Jan 2012 10:26:21 +0100 (CET) Subject: [pypy-commit] pypy default: Tweak to ignore all Symbolics, not just some of them, Message-ID: <20120128092621.CF70682114@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: Changeset: r51900:15600a1bc7ca Date: 2012-01-28 09:21 +0100 http://bitbucket.org/pypy/pypy/changeset/15600a1bc7ca/ Log: Tweak to ignore all Symbolics, not just some of them, by ignoring the TypeError when trying to compare them with small numbers. diff --git a/pypy/jit/codewriter/assembler.py b/pypy/jit/codewriter/assembler.py --- a/pypy/jit/codewriter/assembler.py +++ b/pypy/jit/codewriter/assembler.py @@ -78,10 +78,13 @@ value = heaptracker.adr2int(value) if TYPE is lltype.SingleFloat: value = longlong.singlefloat2int(value) - if not isinstance(value, (llmemory.AddressAsInt, - ComputedIntSymbolic)): - value = lltype.cast_primitive(lltype.Signed, value) - if allow_short and -128 <= value <= 127: + value = lltype.cast_primitive(lltype.Signed, value) + if allow_short: + try: + short_num = -128 <= value <= 127 + except TypeError: # "Symbolics cannot be compared!" + short_num = False + if short_num: # emit the constant as a small integer self.code.append(chr(value & 0xFF)) return True From noreply at buildbot.pypy.org Sat Jan 28 10:26:23 2012 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 28 Jan 2012 10:26:23 +0100 (CET) Subject: [pypy-commit] pypy default: Fix: keep the original special-cases. Needed at least for Message-ID: <20120128092623.0ECFD82114@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: Changeset: r51901:7cdc8cea91ab Date: 2012-01-28 08:56 +0000 http://bitbucket.org/pypy/pypy/changeset/7cdc8cea91ab/ Log: Fix: keep the original special-cases. Needed at least for ComputedIntSymbolics, otherwise they might force computation of their value, which is not what we want. diff --git a/pypy/jit/codewriter/assembler.py b/pypy/jit/codewriter/assembler.py --- a/pypy/jit/codewriter/assembler.py +++ b/pypy/jit/codewriter/assembler.py @@ -78,16 +78,18 @@ value = heaptracker.adr2int(value) if TYPE is lltype.SingleFloat: value = longlong.singlefloat2int(value) - value = lltype.cast_primitive(lltype.Signed, value) - if allow_short: - try: - short_num = -128 <= value <= 127 - except TypeError: # "Symbolics cannot be compared!" - short_num = False - if short_num: - # emit the constant as a small integer - self.code.append(chr(value & 0xFF)) - return True + if not isinstance(value, (llmemory.AddressAsInt, + ComputedIntSymbolic)): + value = lltype.cast_primitive(lltype.Signed, value) + if allow_short: + try: + short_num = -128 <= value <= 127 + except TypeError: # "Symbolics cannot be compared!" + short_num = False + if short_num: + # emit the constant as a small integer + self.code.append(chr(value & 0xFF)) + return True constants = self.constants_i elif kind == 'ref': value = lltype.cast_opaque_ptr(llmemory.GCREF, value) From noreply at buildbot.pypy.org Sat Jan 28 10:26:24 2012 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 28 Jan 2012 10:26:24 +0100 (CET) Subject: [pypy-commit] pypy default: Baaaaaah. "Fix" issue1020. Message-ID: <20120128092624.3FF8E82114@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: Changeset: r51902:451261ed7b89 Date: 2012-01-28 09:26 +0000 http://bitbucket.org/pypy/pypy/changeset/451261ed7b89/ Log: Baaaaaah. "Fix" issue1020. diff --git a/pypy/rlib/rstacklet.py b/pypy/rlib/rstacklet.py --- a/pypy/rlib/rstacklet.py +++ b/pypy/rlib/rstacklet.py @@ -1,3 +1,4 @@ +import sys from pypy.rlib import _rffi_stacklet as _c from pypy.rlib import jit from pypy.rlib.objectmodel import we_are_translated @@ -72,6 +73,11 @@ if translated: assert config is not None, ("you have to pass a valid config, " "e.g. from 'driver.config'") + elif '__pypy__' in sys.builtin_module_names: + import py + py.test.skip("cannot run the stacklet tests on top of pypy: " + "calling directly the C function stacklet_switch() " + "will crash, depending on details of your config") if config is not None: assert config.translation.continuation, ( "stacklet: you have to translate with --continuation") From noreply at buildbot.pypy.org Sat Jan 28 11:20:09 2012 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 28 Jan 2012 11:20:09 +0100 (CET) Subject: [pypy-commit] pypy miniscan: A branch to try to play with conservative root scanning, Message-ID: <20120128102009.2635082114@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: miniscan Changeset: r51903:e50b47052be5 Date: 2012-01-28 09:26 +0100 http://bitbucket.org/pypy/pypy/changeset/e50b47052be5/ Log: A branch to try to play with conservative root scanning, as a 3rd option to asmgcc-vs-shadowstack. From noreply at buildbot.pypy.org Sat Jan 28 11:20:10 2012 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 28 Jan 2012 11:20:10 +0100 (CET) Subject: [pypy-commit] pypy miniscan: In-progress. Message-ID: <20120128102010.6961B82114@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: miniscan Changeset: r51904:31b709ec2998 Date: 2012-01-28 11:19 +0100 http://bitbucket.org/pypy/pypy/changeset/31b709ec2998/ Log: In-progress. diff --git a/pypy/config/translationoption.py b/pypy/config/translationoption.py --- a/pypy/config/translationoption.py +++ b/pypy/config/translationoption.py @@ -90,13 +90,14 @@ default=IS_64_BITS, cmdline="--gcremovetypeptr"), ChoiceOption("gcrootfinder", "Strategy for finding GC Roots (framework GCs only)", - ["n/a", "shadowstack", "asmgcc"], + ["n/a", "shadowstack", "asmgcc", "scan"], "shadowstack", cmdline="--gcrootfinder", requires={ "shadowstack": [("translation.gctransformer", "framework")], "asmgcc": [("translation.gctransformer", "framework"), ("translation.backend", "c")], + "scan": [("translation.gctransformer", "framework")], }), # other noticeable options diff --git a/pypy/rpython/lltypesystem/lltype.py b/pypy/rpython/lltypesystem/lltype.py --- a/pypy/rpython/lltypesystem/lltype.py +++ b/pypy/rpython/lltypesystem/lltype.py @@ -403,6 +403,17 @@ class GcStruct(RttiStruct): _gckind = 'gc' + def __init__(self, *args, **kwds): + RttiStruct.__init__(self, *args, **kwds) + # for --gcrootfinder=scan: we need a way to ensure that most gcrefs + # in a program cannot point to varsized stuff. The easiest is to + # make sure that GcStructs with no _arrayfld cannot be extended into + # GcStructs with an _arrayfld. If this property is needed later, + # think harder about pypy.rpython.memory.gctransform.scan. + if self._arrayfld is not None and self._first_struct() != (None, None): + raise TypeError("GcStruct has both an inlined first struct and " + "an Array at the end") + STRUCT_BY_FLAVOR = {'raw': Struct, 'gc': GcStruct} diff --git a/pypy/rpython/memory/gctransform/scan.py b/pypy/rpython/memory/gctransform/scan.py new file mode 100644 --- /dev/null +++ b/pypy/rpython/memory/gctransform/scan.py @@ -0,0 +1,89 @@ +from pypy.rpython.memory.gctransform.framework import FrameworkGCTransformer +from pypy.rpython.memory.gctransform.framework import BaseRootWalker +from pypy.translator.tool.cbuild import ExternalCompilationInfo +from pypy.rpython.lltypesystem import lltype, rffi +from pypy.rpython.annlowlevel import llhelper + + +class ScanFrameworkGCTransformer(FrameworkGCTransformer): + + def push_roots(self, hop, keep_current_args=False): + livevars = self.get_livevars_for_roots(hop, keep_current_args) + self.num_pushs += len(livevars) + return livevars + + def pop_roots(self, hop, livevars): + if not livevars: + return + # mark the values as keepalives if they can point to var-sized objs + for var in livevars: + if self.can_point_to_varsized(var.concretetype): + hop.genop("keepalive", [var]) + + def can_point_to_varsized(self, TYPE): + if not isinstance(TYPE, lltype.Ptr) or TYPE.TO._gckind != "gc": + return False # not a ptr-to-gc type at all + # this is where we use the fact that a GcStruct cannot inherit another + # GcStruct *and* add an _arrayfld: + if isinstance(TYPE.TO, lltype.GcStruct) and TYPE.TO._arrayfld is None: + return False # can point only to a GcStruct with no _arrayfld + else: + return True # other, including GCREF + + def build_root_walker(self): + return ScanStackRootWalker(self) + + +class ScanStackRootWalker(BaseRootWalker): + + def __init__(self, gctransformer): + BaseRootWalker.__init__(self, gctransformer) + + def _asm_callback(): + self.walk_stack_from() + self._asm_callback = _asm_callback + + #def need_stacklet_support(self, gctransformer, getfn): + # anything needed? + + #def need_thread_support(self, gctransformer, getfn): + # xxx + + def walk_stack_roots(self, collect_stack_root): + gcdata = self.gcdata + gcdata._gc_collect_stack_root = collect_stack_root + pypy_asm_close_for_scanning( + llhelper(ASM_CALLBACK_PTR, self._asm_callback)) + + def walk_stack_from(self): + raise NotImplementedError + + +eci = ExternalCompilationInfo( + post_include_bits = ["extern void pypy_asm_close_for_scanning(void*);\n"], + separate_module_sources = [''' + +void pypy_asm_close_for_scanning(void *fn) +{ + /* We have to do the call by clobbering all registers. This is + needed to ensure that all GC pointers are forced on the stack. */ +#if defined(__amd64__) + asm volatile("call *%%rsi" : : "rsi"(fn) : + "memory", "rax", "rbx", "rcx", "rdx", "rbp", "rdi", + "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15"); +#else + asm volatile("call *%%eax" : : "eax"(fn) : + "memory", "ebx", "ecx", "edx", "ebp", "esi", "edi"); +#endif +} +'''], + ) + +ASM_CALLBACK_PTR = lltype.Ptr(lltype.FuncType([], lltype.Void)) + +pypy_asm_close_for_scanning = rffi.llexternal('pypy_asm_close_for_scanning', + [ASM_CALLBACK_PTR], lltype.Void, + sandboxsafe=True, + _nowrapper=True, + random_effects_on_gcobjs=True, + compilation_info=eci) diff --git a/pypy/translator/c/gc.py b/pypy/translator/c/gc.py --- a/pypy/translator/c/gc.py +++ b/pypy/translator/c/gc.py @@ -6,7 +6,7 @@ typeOf, Ptr, ContainerType, RttiStruct, \ RuntimeTypeInfo, getRuntimeTypeInfo, top_container from pypy.rpython.memory.gctransform import \ - refcounting, boehm, framework, asmgcroot + refcounting, boehm, framework, asmgcroot, scan from pypy.rpython.lltypesystem import lltype, llmemory from pypy.translator.tool.cbuild import ExternalCompilationInfo @@ -406,6 +406,15 @@ def OP_GC_STACK_BOTTOM(self, funcgen, op): return 'pypy_asm_stack_bottom();' +class ScanFrameworkGcPolicy(FrameworkGcPolicy): + transformerclass = scan.ScanFrameworkGCTransformer + + def GC_KEEPALIVE(self, funcgen, v): + return 'pypy_asm_keepalive(%s);' % funcgen.expr(v) + + def OP_GC_RELOAD_POSSIBLY_MOVED(self, funcgen, op): + raise Exception("should not be produced with --gcrootfinder=scan") + name_to_gcpolicy = { 'boehm': BoehmGcPolicy, @@ -413,6 +422,5 @@ 'none': NoneGcPolicy, 'framework': FrameworkGcPolicy, 'framework+asmgcroot': AsmGcRootFrameworkGcPolicy, + 'framework+scan': ScanFrameworkGcPolicy, } - - diff --git a/pypy/translator/c/genc.py b/pypy/translator/c/genc.py --- a/pypy/translator/c/genc.py +++ b/pypy/translator/c/genc.py @@ -191,6 +191,8 @@ name = self.config.translation.gctransformer if self.config.translation.gcrootfinder == "asmgcc": name = "%s+asmgcroot" % (name,) + if self.config.translation.gcrootfinder == "scan": + name = "%s+scan" % (name,) return gc.name_to_gcpolicy[name] return self.gcpolicy diff --git a/pypy/translator/c/test/test_newgc.py b/pypy/translator/c/test/test_newgc.py --- a/pypy/translator/c/test/test_newgc.py +++ b/pypy/translator/c/test/test_newgc.py @@ -15,6 +15,7 @@ class TestUsingFramework(object): gcpolicy = "marksweep" + gcrootfinder = "shadowstack" should_be_moving = False removetypeptr = False taggedpointers = False @@ -41,7 +42,8 @@ t = Translation(main, standalone=True, gc=cls.gcpolicy, policy=annpolicy.StrictAnnotatorPolicy(), taggedpointers=cls.taggedpointers, - gcremovetypeptr=cls.removetypeptr) + gcremovetypeptr=cls.removetypeptr, + gcrootfinder=cls.gcrootfinder) t.disable(['backendopt']) t.set_backend_extra_options(c_debug_defines=True) t.rtype() diff --git a/pypy/translator/c/test/test_scan.py b/pypy/translator/c/test/test_scan.py new file mode 100644 --- /dev/null +++ b/pypy/translator/c/test/test_scan.py @@ -0,0 +1,5 @@ +from pypy.translator.c.test import test_newgc + + +class TestMiniMarkGC(test_newgc.TestMiniMarkGC): + gcrootfinder = "scan" From noreply at buildbot.pypy.org Sat Jan 28 12:48:00 2012 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 28 Jan 2012 12:48:00 +0100 (CET) Subject: [pypy-commit] pypy default: a missing space Message-ID: <20120128114800.6DC6982114@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: Changeset: r51905:534391d69807 Date: 2012-01-28 13:47 +0200 http://bitbucket.org/pypy/pypy/changeset/534391d69807/ Log: a missing space diff --git a/pypy/module/micronumpy/tool/numready/main.py b/pypy/module/micronumpy/tool/numready/main.py --- a/pypy/module/micronumpy/tool/numready/main.py +++ b/pypy/module/micronumpy/tool/numready/main.py @@ -95,7 +95,7 @@ pypy_items = find_numpy_items(argv[1], "numpypy") all_items = [] - msg = "{:d}/{:d} names".format(len(pypy_items), len(cpy_items)) + msg = "{:d}/{:d} names".format(len(pypy_items), len(cpy_items)) + " " msg += ", ".join( "{:d}/{:d} {} attributes".format( len(pypy_items[name].subitems), len(cpy_items[name].subitems), name From noreply at buildbot.pypy.org Sat Jan 28 13:31:20 2012 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 28 Jan 2012 13:31:20 +0100 (CET) Subject: [pypy-commit] extradoc extradoc: Another numpy update Message-ID: <20120128123120.2B29A82114@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: extradoc Changeset: r4059:8191a7475814 Date: 2012-01-28 14:29 +0200 http://bitbucket.org/pypy/extradoc/changeset/8191a7475814/ Log: Another numpy update diff --git a/blog/draft/numpy-status-update-2.rst b/blog/draft/numpy-status-update-2.rst new file mode 100644 --- /dev/null +++ b/blog/draft/numpy-status-update-2.rst @@ -0,0 +1,39 @@ +Hello. + +This is just a quick status update on the NumPy in PyPy project that as of very +recently became my day job. I probably should thank once again to Getco and +Nate Lawson who donated $35000 towards this goal and individual contributors +who donated above $5000. + +Recently we (Alex Gaynor, Matti Picus and me) implemented few interesting things +that a lot of people use: + +* more ufuncs + +* most ufuncs now accept axis parameter (except all and any) + +* fixed string representation of arrays, now it's identical to numpy (uses + pretty much the same code) + +* ``ndarray.flat`` should be working correctly + +* ``ndarray.flatten``, ``ndarray.ravel``, ``ndarray.take`` + +* indexing arrays by boolean arrays of the same size + +* bugfixes + +We would also like to introduce the `nightly report`_ of numpy status. This +is an automated tool that does package introspection. While it gives some +sort of idea how much of numpy is implemented, it's not by far the authority. +Your tests should be the authority. It won't report whether functions +support all kinds of parameters (for example masked arrays and ``out`` parameter +are completely unsupported) or that functions work **at all**. + +We hope to provide you with more frequent updates on the progress and in +meantime I would like to encourage people to donate towards the step 2, which +is making it **really** fast. + +Cheers, +fijal & the PyPy tem + From noreply at buildbot.pypy.org Sat Jan 28 13:31:21 2012 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 28 Jan 2012 13:31:21 +0100 (CET) Subject: [pypy-commit] extradoc extradoc: fix link & update Message-ID: <20120128123121.628D382114@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: extradoc Changeset: r4060:3745452a2f67 Date: 2012-01-28 14:31 +0200 http://bitbucket.org/pypy/extradoc/changeset/3745452a2f67/ Log: fix link & update diff --git a/blog/draft/numpy-status-update-2.rst b/blog/draft/numpy-status-update-2.rst --- a/blog/draft/numpy-status-update-2.rst +++ b/blog/draft/numpy-status-update-2.rst @@ -28,7 +28,9 @@ sort of idea how much of numpy is implemented, it's not by far the authority. Your tests should be the authority. It won't report whether functions support all kinds of parameters (for example masked arrays and ``out`` parameter -are completely unsupported) or that functions work **at all**. +are completely unsupported) or that functions work **at all**. We also +reserve the right to incorporate jokes in that website, so don't treat it +that seriously overall :-) We hope to provide you with more frequent updates on the progress and in meantime I would like to encourage people to donate towards the step 2, which @@ -37,3 +39,4 @@ Cheers, fijal & the PyPy tem +.. `nightly report`: http://buildbot.pypy.org/numpy-status/latest.html From noreply at buildbot.pypy.org Sat Jan 28 13:37:58 2012 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 28 Jan 2012 13:37:58 +0100 (CET) Subject: [pypy-commit] extradoc extradoc: remove info Message-ID: <20120128123758.1997082114@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: extradoc Changeset: r4061:d119c145bdd7 Date: 2012-01-28 14:37 +0200 http://bitbucket.org/pypy/extradoc/changeset/d119c145bdd7/ Log: remove info diff --git a/blog/draft/numpy-status-update-2.rst b/blog/draft/numpy-status-update-2.rst --- a/blog/draft/numpy-status-update-2.rst +++ b/blog/draft/numpy-status-update-2.rst @@ -1,9 +1,8 @@ Hello. This is just a quick status update on the NumPy in PyPy project that as of very -recently became my day job. I probably should thank once again to Getco and -Nate Lawson who donated $35000 towards this goal and individual contributors -who donated above $5000. +recently became my day job. I probably should thank once again to Getco, +Nate Lawson and other contributors who donated above $40000. Recently we (Alex Gaynor, Matti Picus and me) implemented few interesting things that a lot of people use: From noreply at buildbot.pypy.org Sat Jan 28 13:38:48 2012 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 28 Jan 2012 13:38:48 +0100 (CET) Subject: [pypy-commit] extradoc extradoc: oops, finish the sentence Message-ID: <20120128123848.A7F1382114@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: extradoc Changeset: r4062:684e5e859da4 Date: 2012-01-28 14:38 +0200 http://bitbucket.org/pypy/extradoc/changeset/684e5e859da4/ Log: oops, finish the sentence diff --git a/blog/draft/numpy-status-update-2.rst b/blog/draft/numpy-status-update-2.rst --- a/blog/draft/numpy-status-update-2.rst +++ b/blog/draft/numpy-status-update-2.rst @@ -2,7 +2,7 @@ This is just a quick status update on the NumPy in PyPy project that as of very recently became my day job. I probably should thank once again to Getco, -Nate Lawson and other contributors who donated above $40000. +Nate Lawson and other contributors who donated above $40000 towards the goal. Recently we (Alex Gaynor, Matti Picus and me) implemented few interesting things that a lot of people use: From noreply at buildbot.pypy.org Sat Jan 28 13:44:22 2012 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 28 Jan 2012 13:44:22 +0100 (CET) Subject: [pypy-commit] extradoc extradoc: typotypotypos but otherwise sounds good. Message-ID: <20120128124422.1184282114@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: extradoc Changeset: r4063:4d7cc9f25be7 Date: 2012-01-28 13:44 +0100 http://bitbucket.org/pypy/extradoc/changeset/4d7cc9f25be7/ Log: typotypotypos but otherwise sounds good. diff --git a/blog/draft/numpy-status-update-2.rst b/blog/draft/numpy-status-update-2.rst --- a/blog/draft/numpy-status-update-2.rst +++ b/blog/draft/numpy-status-update-2.rst @@ -1,15 +1,15 @@ Hello. -This is just a quick status update on the NumPy in PyPy project that as of very -recently became my day job. I probably should thank once again to Getco, +This is just a quick status update on the NumPy in PyPy project that very +recently became my day job. I should give my thanks once again to Getco, Nate Lawson and other contributors who donated above $40000 towards the goal. -Recently we (Alex Gaynor, Matti Picus and me) implemented few interesting things +Recently we (Alex Gaynor, Matti Picus and me) implemented a few interesting things that a lot of people use: * more ufuncs -* most ufuncs now accept axis parameter (except all and any) +* most ufuncs now accept axis parameter (except ``all`` and ``any``) * fixed string representation of arrays, now it's identical to numpy (uses pretty much the same code) @@ -31,11 +31,11 @@ reserve the right to incorporate jokes in that website, so don't treat it that seriously overall :-) -We hope to provide you with more frequent updates on the progress and in +We hope to provide you with more frequent updates on the progress, and in the meantime I would like to encourage people to donate towards the step 2, which -is making it **really** fast. +is about making numpypy **really** fast. Cheers, -fijal & the PyPy tem +fijal & the PyPy team .. `nightly report`: http://buildbot.pypy.org/numpy-status/latest.html From noreply at buildbot.pypy.org Sat Jan 28 13:45:50 2012 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 28 Jan 2012 13:45:50 +0100 (CET) Subject: [pypy-commit] extradoc extradoc: one more Message-ID: <20120128124550.31C6D82114@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: extradoc Changeset: r4064:d95b8fd7f2dd Date: 2012-01-28 13:45 +0100 http://bitbucket.org/pypy/extradoc/changeset/d95b8fd7f2dd/ Log: one more diff --git a/blog/draft/numpy-status-update-2.rst b/blog/draft/numpy-status-update-2.rst --- a/blog/draft/numpy-status-update-2.rst +++ b/blog/draft/numpy-status-update-2.rst @@ -9,7 +9,7 @@ * more ufuncs -* most ufuncs now accept axis parameter (except ``all`` and ``any``) +* most ufuncs now accept the ``axis`` parameter (except ``all`` and ``any``) * fixed string representation of arrays, now it's identical to numpy (uses pretty much the same code) From noreply at buildbot.pypy.org Sat Jan 28 13:53:59 2012 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 28 Jan 2012 13:53:59 +0100 (CET) Subject: [pypy-commit] extradoc extradoc: the last para is a bit pushy, we just started Message-ID: <20120128125359.A0CD782114@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: extradoc Changeset: r4065:96966961df4d Date: 2012-01-28 14:53 +0200 http://bitbucket.org/pypy/extradoc/changeset/96966961df4d/ Log: the last para is a bit pushy, we just started diff --git a/blog/draft/numpy-status-update-2.rst b/blog/draft/numpy-status-update-2.rst --- a/blog/draft/numpy-status-update-2.rst +++ b/blog/draft/numpy-status-update-2.rst @@ -31,10 +31,6 @@ reserve the right to incorporate jokes in that website, so don't treat it that seriously overall :-) -We hope to provide you with more frequent updates on the progress, and in the -meantime I would like to encourage people to donate towards the step 2, which -is about making numpypy **really** fast. - Cheers, fijal & the PyPy team From noreply at buildbot.pypy.org Sat Jan 28 13:57:21 2012 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 28 Jan 2012 13:57:21 +0100 (CET) Subject: [pypy-commit] extradoc extradoc: this probably wasn't supposed to be killed Message-ID: <20120128125721.3693F82114@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: extradoc Changeset: r4066:898fac7cc21a Date: 2012-01-28 14:57 +0200 http://bitbucket.org/pypy/extradoc/changeset/898fac7cc21a/ Log: this probably wasn't supposed to be killed diff --git a/blog/draft/numpy-status-update-2.rst b/blog/draft/numpy-status-update-2.rst --- a/blog/draft/numpy-status-update-2.rst +++ b/blog/draft/numpy-status-update-2.rst @@ -31,6 +31,8 @@ reserve the right to incorporate jokes in that website, so don't treat it that seriously overall :-) +We hope to provide you with more frequent updates on the progress. + Cheers, fijal & the PyPy team From noreply at buildbot.pypy.org Sat Jan 28 14:01:00 2012 From: noreply at buildbot.pypy.org (l.diekmann) Date: Sat, 28 Jan 2012 14:01:00 +0100 (CET) Subject: [pypy-commit] pypy default: test and fix for cached empty list with deactivated strategies Message-ID: <20120128130100.91B4E82114@wyvern.cs.uni-duesseldorf.de> Author: Lukas Diekmann <lukas.diekmann at uni-duesseldorf.de> Branch: Changeset: r51906:2cb8ade3aef2 Date: 2012-01-27 17:42 +0100 http://bitbucket.org/pypy/pypy/changeset/2cb8ade3aef2/ Log: test and fix for cached empty list with deactivated strategies diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py --- a/pypy/objspace/std/listobject.py +++ b/pypy/objspace/std/listobject.py @@ -110,7 +110,10 @@ return list(items) def switch_to_object_strategy(self): - list_w = self.getitems() + if self.strategy is self.space.fromcache(EmptyListStrategy): + list_w = [] + else: + list_w = self.getitems() self.strategy = self.space.fromcache(ObjectListStrategy) # XXX this is quite indirect self.init_from_list_w(list_w) diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py --- a/pypy/objspace/std/test/test_listobject.py +++ b/pypy/objspace/std/test/test_listobject.py @@ -1251,6 +1251,20 @@ l.reverse() assert l == [2,1,0] +class AppTestWithoutStrategies(object): + + def setup_class(cls): + cls.space = gettestobjspace(**{"objspace.std.withliststrategies" : + False}) + + def test_no_shared_empty_list(self): + l = [] + copy = l[:] + copy.append({}) + assert copy == [{}] + + notshared = l[:] + assert notshared == [] class AppTestListFastSubscr: From noreply at buildbot.pypy.org Sat Jan 28 14:01:01 2012 From: noreply at buildbot.pypy.org (l.diekmann) Date: Sat, 28 Jan 2012 14:01:01 +0100 (CET) Subject: [pypy-commit] pypy default: test and fix for cpyext/listobject.py: PyList_GET_SIZE and PyList_SetItem Message-ID: <20120128130101.CBA0582114@wyvern.cs.uni-duesseldorf.de> Author: Lukas Diekmann <lukas.diekmann at uni-duesseldorf.de> Branch: Changeset: r51907:a384864c5d6c Date: 2012-01-28 12:39 +0100 http://bitbucket.org/pypy/pypy/changeset/a384864c5d6c/ Log: test and fix for cpyext/listobject.py: PyList_GET_SIZE and PyList_SetItem diff --git a/pypy/module/cpyext/listobject.py b/pypy/module/cpyext/listobject.py --- a/pypy/module/cpyext/listobject.py +++ b/pypy/module/cpyext/listobject.py @@ -32,11 +32,10 @@ Py_DecRef(space, w_item) if not isinstance(w_list, W_ListObject): PyErr_BadInternalCall(space) - wrappeditems = w_list.getitems() - if index < 0 or index >= len(wrappeditems): + if index < 0 or index >= w_list.length(): raise OperationError(space.w_IndexError, space.wrap( "list assignment index out of range")) - wrappeditems[index] = w_item + w_list.setitem(index, w_item) return 0 @cpython_api([PyObject, Py_ssize_t], PyObject) @@ -74,7 +73,7 @@ """Macro form of PyList_Size() without error checking. """ assert isinstance(w_list, W_ListObject) - return len(w_list.getitems()) + return w_list.length() @cpython_api([PyObject], Py_ssize_t, error=-1) diff --git a/pypy/module/cpyext/test/test_listobject.py b/pypy/module/cpyext/test/test_listobject.py --- a/pypy/module/cpyext/test/test_listobject.py +++ b/pypy/module/cpyext/test/test_listobject.py @@ -128,3 +128,6 @@ module.setslice(l, None) assert l == [0, 4, 5] + l = [1, 2, 3] + module.setlistitem(l,0) + assert l == [None, 2, 3] From noreply at buildbot.pypy.org Sat Jan 28 14:01:03 2012 From: noreply at buildbot.pypy.org (l.diekmann) Date: Sat, 28 Jan 2012 14:01:03 +0100 (CET) Subject: [pypy-commit] pypy default: decided that a cached empty list is too dangerous and that is is better to live Message-ID: <20120128130103.0ABD582114@wyvern.cs.uni-duesseldorf.de> Author: Lukas Diekmann <lukas.diekmann at uni-duesseldorf.de> Branch: Changeset: r51908:3607ba0bb92d Date: 2012-01-28 13:08 +0100 http://bitbucket.org/pypy/pypy/changeset/3607ba0bb92d/ Log: decided that a cached empty list is too dangerous and that is is better to live with a sligthly slower empty list creation than with unexpected behaviour if we are not careful enough with the getitems method diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py --- a/pypy/objspace/std/listobject.py +++ b/pypy/objspace/std/listobject.py @@ -347,8 +347,6 @@ def __init__(self, space): ListStrategy.__init__(self, space) - # cache an empty list that is used whenever getitems is called (i.e. sorting) - self.cached_emptylist_w = [] def init_from_list_w(self, w_list, list_w): assert len(list_w) == 0 @@ -376,10 +374,10 @@ def getslice(self, w_list, start, stop, step, length): # will never be called because the empty list case is already caught in # getslice__List_ANY_ANY and getitem__List_Slice - return W_ListObject(self.space, self.cached_emptylist_w) + return W_ListObject(self.space, []) def getitems(self, w_list): - return self.cached_emptylist_w + return [] def getitems_copy(self, w_list): return [] From noreply at buildbot.pypy.org Sat Jan 28 14:01:05 2012 From: noreply at buildbot.pypy.org (l.diekmann) Date: Sat, 28 Jan 2012 14:01:05 +0100 (CET) Subject: [pypy-commit] pypy default: merge Message-ID: <20120128130105.F098482114@wyvern.cs.uni-duesseldorf.de> Author: Lukas Diekmann <lukas.diekmann at uni-duesseldorf.de> Branch: Changeset: r51909:51eb80cfd760 Date: 2012-01-28 14:00 +0100 http://bitbucket.org/pypy/pypy/changeset/51eb80cfd760/ Log: merge diff --git a/lib_pypy/numpypy/core/__init__.py b/lib_pypy/numpypy/core/__init__.py --- a/lib_pypy/numpypy/core/__init__.py +++ b/lib_pypy/numpypy/core/__init__.py @@ -1,1 +1,2 @@ from .fromnumeric import * +from .numeric import * diff --git a/lib_pypy/numpypy/core/_methods.py b/lib_pypy/numpypy/core/_methods.py new file mode 100644 --- /dev/null +++ b/lib_pypy/numpypy/core/_methods.py @@ -0,0 +1,98 @@ +# Array methods which are called by the both the C-code for the method +# and the Python code for the NumPy-namespace function + +import _numpypy as mu +um = mu +#from numpypy.core import umath as um +from numpypy.core.numeric import asanyarray + +def _amax(a, axis=None, out=None, skipna=False, keepdims=False): + return um.maximum.reduce(a, axis=axis, + out=out, skipna=skipna, keepdims=keepdims) + +def _amin(a, axis=None, out=None, skipna=False, keepdims=False): + return um.minimum.reduce(a, axis=axis, + out=out, skipna=skipna, keepdims=keepdims) + +def _sum(a, axis=None, dtype=None, out=None, skipna=False, keepdims=False): + return um.add.reduce(a, axis=axis, dtype=dtype, + out=out, skipna=skipna, keepdims=keepdims) + +def _prod(a, axis=None, dtype=None, out=None, skipna=False, keepdims=False): + return um.multiply.reduce(a, axis=axis, dtype=dtype, + out=out, skipna=skipna, keepdims=keepdims) + +def _mean(a, axis=None, dtype=None, out=None, skipna=False, keepdims=False): + arr = asanyarray(a) + + # Upgrade bool, unsigned int, and int to float64 + if dtype is None and arr.dtype.kind in ['b','u','i']: + ret = um.add.reduce(arr, axis=axis, dtype='f8', + out=out, skipna=skipna, keepdims=keepdims) + else: + ret = um.add.reduce(arr, axis=axis, dtype=dtype, + out=out, skipna=skipna, keepdims=keepdims) + rcount = mu.count_reduce_items(arr, axis=axis, + skipna=skipna, keepdims=keepdims) + if isinstance(ret, mu.ndarray): + ret = um.true_divide(ret, rcount, + casting='unsafe', subok=False) + else: + ret = ret / float(rcount) + return ret + +def _var(a, axis=None, dtype=None, out=None, ddof=0, + skipna=False, keepdims=False): + arr = asanyarray(a) + + # First compute the mean, saving 'rcount' for reuse later + if dtype is None and arr.dtype.kind in ['b','u','i']: + arrmean = um.add.reduce(arr, axis=axis, dtype='f8', + skipna=skipna, keepdims=True) + else: + arrmean = um.add.reduce(arr, axis=axis, dtype=dtype, + skipna=skipna, keepdims=True) + rcount = mu.count_reduce_items(arr, axis=axis, + skipna=skipna, keepdims=True) + if isinstance(arrmean, mu.ndarray): + arrmean = um.true_divide(arrmean, rcount, + casting='unsafe', subok=False) + else: + arrmean = arrmean / float(rcount) + + # arr - arrmean + x = arr - arrmean + + # (arr - arrmean) ** 2 + if arr.dtype.kind == 'c': + x = um.multiply(x, um.conjugate(x)).real + else: + x = um.multiply(x, x) + + # add.reduce((arr - arrmean) ** 2, axis) + ret = um.add.reduce(x, axis=axis, dtype=dtype, out=out, + skipna=skipna, keepdims=keepdims) + + # add.reduce((arr - arrmean) ** 2, axis) / (n - ddof) + if not keepdims and isinstance(rcount, mu.ndarray): + rcount = rcount.squeeze(axis=axis) + rcount -= ddof + if isinstance(ret, mu.ndarray): + ret = um.true_divide(ret, rcount, + casting='unsafe', subok=False) + else: + ret = ret / float(rcount) + + return ret + +def _std(a, axis=None, dtype=None, out=None, ddof=0, + skipna=False, keepdims=False): + ret = _var(a, axis=axis, dtype=dtype, out=out, ddof=ddof, + skipna=skipna, keepdims=keepdims) + + if isinstance(ret, mu.ndarray): + ret = um.sqrt(ret) + else: + ret = um.sqrt(ret) + + return ret diff --git a/lib_pypy/numpypy/core/arrayprint.py b/lib_pypy/numpypy/core/arrayprint.py new file mode 100644 --- /dev/null +++ b/lib_pypy/numpypy/core/arrayprint.py @@ -0,0 +1,789 @@ +"""Array printing function + +$Id: arrayprint.py,v 1.9 2005/09/13 13:58:44 teoliphant Exp $ +""" +__all__ = ["array2string", "set_printoptions", "get_printoptions"] +__docformat__ = 'restructuredtext' + +# +# Written by Konrad Hinsen <hinsenk at ere.umontreal.ca> +# last revision: 1996-3-13 +# modified by Jim Hugunin 1997-3-3 for repr's and str's (and other details) +# and by Perry Greenfield 2000-4-1 for numarray +# and by Travis Oliphant 2005-8-22 for numpy + +import sys +import _numpypy as _nt +from _numpypy import maximum, minimum, absolute, not_equal, isinf, isnan, isna +#from _numpypy import format_longfloat, datetime_as_string, datetime_data +from .fromnumeric import ravel + + +def product(x, y): return x*y + +_summaryEdgeItems = 3 # repr N leading and trailing items of each dimension +_summaryThreshold = 1000 # total items > triggers array summarization + +_float_output_precision = 8 +_float_output_suppress_small = False +_line_width = 75 +_nan_str = 'nan' +_inf_str = 'inf' +_na_str = 'NA' +_formatter = None # formatting function for array elements + +if sys.version_info[0] >= 3: + from functools import reduce + +def set_printoptions(precision=None, threshold=None, edgeitems=None, + linewidth=None, suppress=None, + nanstr=None, infstr=None, nastr=None, + formatter=None): + """ + Set printing options. + + These options determine the way floating point numbers, arrays and + other NumPy objects are displayed. + + Parameters + ---------- + precision : int, optional + Number of digits of precision for floating point output (default 8). + threshold : int, optional + Total number of array elements which trigger summarization + rather than full repr (default 1000). + edgeitems : int, optional + Number of array items in summary at beginning and end of + each dimension (default 3). + linewidth : int, optional + The number of characters per line for the purpose of inserting + line breaks (default 75). + suppress : bool, optional + Whether or not suppress printing of small floating point values + using scientific notation (default False). + nanstr : str, optional + String representation of floating point not-a-number (default nan). + infstr : str, optional + String representation of floating point infinity (default inf). + nastr : str, optional + String representation of NA missing value (default NA). + formatter : dict of callables, optional + If not None, the keys should indicate the type(s) that the respective + formatting function applies to. Callables should return a string. + Types that are not specified (by their corresponding keys) are handled + by the default formatters. Individual types for which a formatter + can be set are:: + + - 'bool' + - 'int' + - 'timedelta' : a `numpy.timedelta64` + - 'datetime' : a `numpy.datetime64` + - 'float' + - 'longfloat' : 128-bit floats + - 'complexfloat' + - 'longcomplexfloat' : composed of two 128-bit floats + - 'numpy_str' : types `numpy.string_` and `numpy.unicode_` + - 'str' : all other strings + + Other keys that can be used to set a group of types at once are:: + + - 'all' : sets all types + - 'int_kind' : sets 'int' + - 'float_kind' : sets 'float' and 'longfloat' + - 'complex_kind' : sets 'complexfloat' and 'longcomplexfloat' + - 'str_kind' : sets 'str' and 'numpystr' + + See Also + -------- + get_printoptions, set_string_function, array2string + + Notes + ----- + `formatter` is always reset with a call to `set_printoptions`. + + Examples + -------- + Floating point precision can be set: + + >>> np.set_printoptions(precision=4) + >>> print np.array([1.123456789]) + [ 1.1235] + + Long arrays can be summarised: + + >>> np.set_printoptions(threshold=5) + >>> print np.arange(10) + [0 1 2 ..., 7 8 9] + + Small results can be suppressed: + + >>> eps = np.finfo(float).eps + >>> x = np.arange(4.) + >>> x**2 - (x + eps)**2 + array([ -4.9304e-32, -4.4409e-16, 0.0000e+00, 0.0000e+00]) + >>> np.set_printoptions(suppress=True) + >>> x**2 - (x + eps)**2 + array([-0., -0., 0., 0.]) + + A custom formatter can be used to display array elements as desired: + + >>> np.set_printoptions(formatter={'all':lambda x: 'int: '+str(-x)}) + >>> x = np.arange(3) + >>> x + array([int: 0, int: -1, int: -2]) + >>> np.set_printoptions() # formatter gets reset + >>> x + array([0, 1, 2]) + + To put back the default options, you can use: + + >>> np.set_printoptions(edgeitems=3,infstr='inf', + ... linewidth=75, nanstr='nan', precision=8, + ... suppress=False, threshold=1000, formatter=None) + """ + + global _summaryThreshold, _summaryEdgeItems, _float_output_precision, \ + _line_width, _float_output_suppress_small, _nan_str, _inf_str, \ + _na_str, _formatter + if linewidth is not None: + _line_width = linewidth + if threshold is not None: + _summaryThreshold = threshold + if edgeitems is not None: + _summaryEdgeItems = edgeitems + if precision is not None: + _float_output_precision = precision + if suppress is not None: + _float_output_suppress_small = not not suppress + if nanstr is not None: + _nan_str = nanstr + if infstr is not None: + _inf_str = infstr + if nastr is not None: + _na_str = nastr + _formatter = formatter + +def get_printoptions(): + """ + Return the current print options. + + Returns + ------- + print_opts : dict + Dictionary of current print options with keys + + - precision : int + - threshold : int + - edgeitems : int + - linewidth : int + - suppress : bool + - nanstr : str + - infstr : str + - formatter : dict of callables + + For a full description of these options, see `set_printoptions`. + + See Also + -------- + set_printoptions, set_string_function + + """ + d = dict(precision=_float_output_precision, + threshold=_summaryThreshold, + edgeitems=_summaryEdgeItems, + linewidth=_line_width, + suppress=_float_output_suppress_small, + nanstr=_nan_str, + infstr=_inf_str, + nastr=_na_str, + formatter=_formatter) + return d + +def _leading_trailing(a): + import numeric as _nc + if a.ndim == 1: + if len(a) > 2*_summaryEdgeItems: + b = _nc.concatenate((a[:_summaryEdgeItems], + a[-_summaryEdgeItems:])) + else: + b = a + else: + if len(a) > 2*_summaryEdgeItems: + l = [_leading_trailing(a[i]) for i in range( + min(len(a), _summaryEdgeItems))] + l.extend([_leading_trailing(a[-i]) for i in range( + min(len(a), _summaryEdgeItems),0,-1)]) + else: + l = [_leading_trailing(a[i]) for i in range(0, len(a))] + b = _nc.concatenate(tuple(l)) + return b + +def _boolFormatter(x): + if isna(x): + return str(x).replace('NA', _na_str, 1) + elif x: + return ' True' + else: + return 'False' + + +def repr_format(x): + if isna(x): + return str(x).replace('NA', _na_str, 1) + else: + return repr(x) + +def _array2string(a, max_line_width, precision, suppress_small, separator=' ', + prefix="", formatter=None): + + if max_line_width is None: + max_line_width = _line_width + + if precision is None: + precision = _float_output_precision + + if suppress_small is None: + suppress_small = _float_output_suppress_small + + if formatter is None: + formatter = _formatter + + if a.size > _summaryThreshold: + summary_insert = "..., " + data = _leading_trailing(a) + else: + summary_insert = "" + data = ravel(a) + + formatdict = {'bool' : _boolFormatter, + 'int' : IntegerFormat(data), + 'float' : FloatFormat(data, precision, suppress_small), + 'longfloat' : LongFloatFormat(precision), + #'complexfloat' : ComplexFormat(data, precision, + # suppress_small), + #'longcomplexfloat' : LongComplexFormat(precision), + #'datetime' : DatetimeFormat(data), + #'timedelta' : TimedeltaFormat(data), + 'numpystr' : repr_format, + 'str' : str} + + if formatter is not None: + fkeys = [k for k in formatter.keys() if formatter[k] is not None] + if 'all' in fkeys: + for key in formatdict.keys(): + formatdict[key] = formatter['all'] + if 'int_kind' in fkeys: + for key in ['int']: + formatdict[key] = formatter['int_kind'] + if 'float_kind' in fkeys: + for key in ['float', 'longfloat']: + formatdict[key] = formatter['float_kind'] + if 'complex_kind' in fkeys: + for key in ['complexfloat', 'longcomplexfloat']: + formatdict[key] = formatter['complex_kind'] + if 'str_kind' in fkeys: + for key in ['numpystr', 'str']: + formatdict[key] = formatter['str_kind'] + for key in formatdict.keys(): + if key in fkeys: + formatdict[key] = formatter[key] + + try: + format_function = a._format + msg = "The `_format` attribute is deprecated in Numpy 2.0 and " \ + "will be removed in 2.1. Use the `formatter` kw instead." + import warnings + warnings.warn(msg, DeprecationWarning) + except AttributeError: + # find the right formatting function for the array + dtypeobj = a.dtype.type + if issubclass(dtypeobj, _nt.bool_): + format_function = formatdict['bool'] + elif issubclass(dtypeobj, _nt.integer): + #if issubclass(dtypeobj, _nt.timedelta64): + # format_function = formatdict['timedelta'] + #else: + format_function = formatdict['int'] + elif issubclass(dtypeobj, _nt.floating): + #if issubclass(dtypeobj, _nt.longfloat): + # format_function = formatdict['longfloat'] + #else: + format_function = formatdict['float'] + elif issubclass(dtypeobj, _nt.complexfloating): + if issubclass(dtypeobj, _nt.clongfloat): + format_function = formatdict['longcomplexfloat'] + else: + format_function = formatdict['complexfloat'] + elif issubclass(dtypeobj, (_nt.unicode_, _nt.string_)): + format_function = formatdict['numpystr'] + elif issubclass(dtypeobj, _nt.datetime64): + format_function = formatdict['datetime'] + else: + format_function = formatdict['str'] + + # skip over "[" + next_line_prefix = " " + # skip over array( + next_line_prefix += " "*len(prefix) + + lst = _formatArray(a, format_function, len(a.shape), max_line_width, + next_line_prefix, separator, + _summaryEdgeItems, summary_insert)[:-1] + return lst + +def _convert_arrays(obj): + import numeric as _nc + newtup = [] + for k in obj: + if isinstance(k, _nc.ndarray): + k = k.tolist() + elif isinstance(k, tuple): + k = _convert_arrays(k) + newtup.append(k) + return tuple(newtup) + + +def array2string(a, max_line_width=None, precision=None, + suppress_small=None, separator=' ', prefix="", + style=repr, formatter=None): + """ + Return a string representation of an array. + + Parameters + ---------- + a : ndarray + Input array. + max_line_width : int, optional + The maximum number of columns the string should span. Newline + characters splits the string appropriately after array elements. + precision : int, optional + Floating point precision. Default is the current printing + precision (usually 8), which can be altered using `set_printoptions`. + suppress_small : bool, optional + Represent very small numbers as zero. A number is "very small" if it + is smaller than the current printing precision. + separator : str, optional + Inserted between elements. + prefix : str, optional + An array is typically printed as:: + + 'prefix(' + array2string(a) + ')' + + The length of the prefix string is used to align the + output correctly. + style : function, optional + A function that accepts an ndarray and returns a string. Used only + when the shape of `a` is equal to ``()``, i.e. for 0-D arrays. + formatter : dict of callables, optional + If not None, the keys should indicate the type(s) that the respective + formatting function applies to. Callables should return a string. + Types that are not specified (by their corresponding keys) are handled + by the default formatters. Individual types for which a formatter + can be set are:: + + - 'bool' + - 'int' + - 'timedelta' : a `numpy.timedelta64` + - 'datetime' : a `numpy.datetime64` + - 'float' + - 'longfloat' : 128-bit floats + - 'complexfloat' + - 'longcomplexfloat' : composed of two 128-bit floats + - 'numpy_str' : types `numpy.string_` and `numpy.unicode_` + - 'str' : all other strings + + Other keys that can be used to set a group of types at once are:: + + - 'all' : sets all types + - 'int_kind' : sets 'int' + - 'float_kind' : sets 'float' and 'longfloat' + - 'complex_kind' : sets 'complexfloat' and 'longcomplexfloat' + - 'str_kind' : sets 'str' and 'numpystr' + + Returns + ------- + array_str : str + String representation of the array. + + Raises + ------ + TypeError : if a callable in `formatter` does not return a string. + + See Also + -------- + array_str, array_repr, set_printoptions, get_printoptions + + Notes + ----- + If a formatter is specified for a certain type, the `precision` keyword is + ignored for that type. + + Examples + -------- + >>> x = np.array([1e-16,1,2,3]) + >>> print np.array2string(x, precision=2, separator=',', + ... suppress_small=True) + [ 0., 1., 2., 3.] + + >>> x = np.arange(3.) + >>> np.array2string(x, formatter={'float_kind':lambda x: "%.2f" % x}) + '[0.00 1.00 2.00]' + + >>> x = np.arange(3) + >>> np.array2string(x, formatter={'int':lambda x: hex(x)}) + '[0x0L 0x1L 0x2L]' + + """ + + if a.shape == (): + x = a.item() + if isna(x): + lst = str(x).replace('NA', _na_str, 1) + else: + try: + lst = a._format(x) + msg = "The `_format` attribute is deprecated in Numpy " \ + "2.0 and will be removed in 2.1. Use the " \ + "`formatter` kw instead." + import warnings + warnings.warn(msg, DeprecationWarning) + except AttributeError: + if isinstance(x, tuple): + x = _convert_arrays(x) + lst = style(x) + elif reduce(product, a.shape) == 0: + # treat as a null array if any of shape elements == 0 + lst = "[]" + else: + lst = _array2string(a, max_line_width, precision, suppress_small, + separator, prefix, formatter=formatter) + return lst + +def _extendLine(s, line, word, max_line_len, next_line_prefix): + if len(line.rstrip()) + len(word.rstrip()) >= max_line_len: + s += line.rstrip() + "\n" + line = next_line_prefix + line += word + return s, line + + +def _formatArray(a, format_function, rank, max_line_len, + next_line_prefix, separator, edge_items, summary_insert): + """formatArray is designed for two modes of operation: + + 1. Full output + + 2. Summarized output + + """ + if rank == 0: + obj = a.item() + if isinstance(obj, tuple): + obj = _convert_arrays(obj) + return str(obj) + + if summary_insert and 2*edge_items < len(a): + leading_items, trailing_items, summary_insert1 = \ + edge_items, edge_items, summary_insert + else: + leading_items, trailing_items, summary_insert1 = 0, len(a), "" + + if rank == 1: + s = "" + line = next_line_prefix + for i in xrange(leading_items): + word = format_function(a[i]) + separator + s, line = _extendLine(s, line, word, max_line_len, next_line_prefix) + + if summary_insert1: + s, line = _extendLine(s, line, summary_insert1, max_line_len, next_line_prefix) + + for i in xrange(trailing_items, 1, -1): + word = format_function(a[-i]) + separator + s, line = _extendLine(s, line, word, max_line_len, next_line_prefix) + + word = format_function(a[-1]) + s, line = _extendLine(s, line, word, max_line_len, next_line_prefix) + s += line + "]\n" + s = '[' + s[len(next_line_prefix):] + else: + s = '[' + sep = separator.rstrip() + for i in xrange(leading_items): + if i > 0: + s += next_line_prefix + s += _formatArray(a[i], format_function, rank-1, max_line_len, + " " + next_line_prefix, separator, edge_items, + summary_insert) + s = s.rstrip() + sep.rstrip() + '\n'*max(rank-1,1) + + if summary_insert1: + s += next_line_prefix + summary_insert1 + "\n" + + for i in xrange(trailing_items, 1, -1): + if leading_items or i != trailing_items: + s += next_line_prefix + s += _formatArray(a[-i], format_function, rank-1, max_line_len, + " " + next_line_prefix, separator, edge_items, + summary_insert) + s = s.rstrip() + sep.rstrip() + '\n'*max(rank-1,1) + if leading_items or trailing_items > 1: + s += next_line_prefix + s += _formatArray(a[-1], format_function, rank-1, max_line_len, + " " + next_line_prefix, separator, edge_items, + summary_insert).rstrip()+']\n' + return s + +class FloatFormat(object): + def __init__(self, data, precision, suppress_small, sign=False): + self.precision = precision + self.suppress_small = suppress_small + self.sign = sign + self.exp_format = False + self.large_exponent = False + self.max_str_len = 0 + #try: + self.fillFormat(data) + #except (TypeError, NotImplementedError): + # if reduce(data) fails, this instance will not be called, just + # instantiated in formatdict. + #pass + + def fillFormat(self, data): + import numeric as _nc + # XXX pypy unimplemented + #errstate = _nc.seterr(all='ignore') + try: + special = isnan(data) | isinf(data) | isna(data) + special[isna(data)] = False + valid = not_equal(data, 0) & ~special + valid[isna(data)] = False + non_zero = absolute(data.compress(valid)) + if len(non_zero) == 0: + max_val = 0. + min_val = 0. + else: + max_val = maximum.reduce(non_zero, skipna=True) + min_val = minimum.reduce(non_zero, skipna=True) + if max_val >= 1.e8: + self.exp_format = True + if not self.suppress_small and (min_val < 0.0001 + or max_val/min_val > 1000.): + self.exp_format = True + finally: + pass + # XXX pypy unimplemented + #_nc.seterr(**errstate) + + if self.exp_format: + self.large_exponent = 0 < min_val < 1e-99 or max_val >= 1e100 + self.max_str_len = 8 + self.precision + if self.large_exponent: + self.max_str_len += 1 + if self.sign: + format = '%+' + else: + format = '%' + format = format + '%d.%de' % (self.max_str_len, self.precision) + else: + format = '%%.%df' % (self.precision,) + if len(non_zero): + precision = max([_digits(x, self.precision, format) + for x in non_zero]) + else: + precision = 0 + precision = min(self.precision, precision) + self.max_str_len = len(str(int(max_val))) + precision + 2 + if special.any(): + self.max_str_len = max(self.max_str_len, + len(_nan_str), + len(_inf_str)+1, + len(_na_str)) + if self.sign: + format = '%#+' + else: + format = '%#' + format = format + '%d.%df' % (self.max_str_len, precision) + + self.special_fmt = '%%%ds' % (self.max_str_len,) + self.format = format + + def __call__(self, x, strip_zeros=True): + import numeric as _nc + #err = _nc.seterr(invalid='ignore') + try: + if isna(x): + return self.special_fmt % (str(x).replace('NA', _na_str, 1),) + elif isnan(x): + if self.sign: + return self.special_fmt % ('+' + _nan_str,) + else: + return self.special_fmt % (_nan_str,) + elif isinf(x): + if x > 0: + if self.sign: + return self.special_fmt % ('+' + _inf_str,) + else: + return self.special_fmt % (_inf_str,) + else: + return self.special_fmt % ('-' + _inf_str,) + finally: + pass + #_nc.seterr(**err) + + s = self.format % x + if self.large_exponent: + # 3-digit exponent + expsign = s[-3] + if expsign == '+' or expsign == '-': + s = s[1:-2] + '0' + s[-2:] + elif self.exp_format: + # 2-digit exponent + if s[-3] == '0': + s = ' ' + s[:-3] + s[-2:] + elif strip_zeros: + z = s.rstrip('0') + s = z + ' '*(len(s)-len(z)) + return s + + +def _digits(x, precision, format): + s = format % x + z = s.rstrip('0') + return precision - len(s) + len(z) + + +_MAXINT = sys.maxint +_MININT = -sys.maxint-1 +class IntegerFormat(object): + def __init__(self, data): + try: + max_str_len = max(len(str(maximum.reduce(data, skipna=True))), + len(str(minimum.reduce(data, skipna=True)))) + self.format = '%' + str(max_str_len) + 'd' + except TypeError, NotImplementedError: + # if reduce(data) fails, this instance will not be called, just + # instantiated in formatdict. + pass + except ValueError: + # this occurs when everything is NA + pass + + def __call__(self, x): + if isna(x): + return str(x).replace('NA', _na_str, 1) + elif _MININT < x < _MAXINT: + return self.format % x + else: + return "%s" % x + +class LongFloatFormat(object): + # XXX Have to add something to determine the width to use a la FloatFormat + # Right now, things won't line up properly + def __init__(self, precision, sign=False): + self.precision = precision + self.sign = sign + + def __call__(self, x): + if isna(x): + return str(x).replace('NA', _na_str, 1) + elif isnan(x): + if self.sign: + return '+' + _nan_str + else: + return ' ' + _nan_str + elif isinf(x): + if x > 0: + if self.sign: + return '+' + _inf_str + else: + return ' ' + _inf_str + else: + return '-' + _inf_str + elif x >= 0: + if self.sign: + return '+' + format_longfloat(x, self.precision) + else: + return ' ' + format_longfloat(x, self.precision) + else: + return format_longfloat(x, self.precision) + + +class LongComplexFormat(object): + def __init__(self, precision): + self.real_format = LongFloatFormat(precision) + self.imag_format = LongFloatFormat(precision, sign=True) + + def __call__(self, x): + if isna(x): + return str(x).replace('NA', _na_str, 1) + else: + r = self.real_format(x.real) + i = self.imag_format(x.imag) + return r + i + 'j' + + +class ComplexFormat(object): + def __init__(self, x, precision, suppress_small): + self.real_format = FloatFormat(x.real, precision, suppress_small) + self.imag_format = FloatFormat(x.imag, precision, suppress_small, + sign=True) + + def __call__(self, x): + if isna(x): + return str(x).replace('NA', _na_str, 1) + else: + r = self.real_format(x.real, strip_zeros=False) + i = self.imag_format(x.imag, strip_zeros=False) + if not self.imag_format.exp_format: + z = i.rstrip('0') + i = z + 'j' + ' '*(len(i)-len(z)) + else: + i = i + 'j' + return r + i + +class DatetimeFormat(object): + def __init__(self, x, unit=None, + timezone=None, casting='same_kind'): + # Get the unit from the dtype + if unit is None: + if x.dtype.kind == 'M': + unit = datetime_data(x.dtype)[0] + else: + unit = 's' + + # If timezone is default, make it 'local' or 'UTC' based on the unit + if timezone is None: + # Date units -> UTC, time units -> local + if unit in ('Y', 'M', 'W', 'D'): + self.timezone = 'UTC' + else: + self.timezone = 'local' + else: + self.timezone = timezone + self.unit = unit + self.casting = casting + + def __call__(self, x): + if isna(x): + return str(x).replace('NA', _na_str, 1) + else: + return "'%s'" % datetime_as_string(x, + unit=self.unit, + timezone=self.timezone, + casting=self.casting) + +class TimedeltaFormat(object): + def __init__(self, data): + if data.dtype.kind == 'm': + v = data.view('i8') + max_str_len = max(len(str(maximum.reduce(v))), + len(str(minimum.reduce(v)))) + self.format = '%' + str(max_str_len) + 'd' + + def __call__(self, x): + if isna(x): + return str(x).replace('NA', _na_str, 1) + else: + return self.format % x.astype('i8') + diff --git a/lib_pypy/numpypy/core/fromnumeric.py b/lib_pypy/numpypy/core/fromnumeric.py --- a/lib_pypy/numpypy/core/fromnumeric.py +++ b/lib_pypy/numpypy/core/fromnumeric.py @@ -30,7 +30,7 @@ 'rank', 'size', 'around', 'round_', 'mean', 'std', 'var', 'squeeze', 'amax', 'amin', ] - + def take(a, indices, axis=None, out=None, mode='raise'): """ Take elements from an array along an axis. @@ -1054,8 +1054,9 @@ array([ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]) """ - raise NotImplementedError('Waiting on interp level method') - + if not hasattr(a, 'ravel'): + a = numpypy.array(a) + return a.ravel(order=order) def nonzero(a): """ @@ -2324,13 +2325,12 @@ 0.44999999925552653 """ - assert axis is None assert dtype is None assert out is None assert ddof == 0 if not hasattr(a, "std"): a = numpypy.array(a) - return a.std() + return a.std(axis=axis) def var(a, axis=None, dtype=None, out=None, ddof=0): @@ -2421,10 +2421,9 @@ 0.20250000000000001 """ - assert axis is None assert dtype is None assert out is None assert ddof == 0 if not hasattr(a, "var"): a = numpypy.array(a) - return a.var() + return a.var(axis=axis) diff --git a/lib_pypy/numpypy/core/numeric.py b/lib_pypy/numpypy/core/numeric.py new file mode 100644 --- /dev/null +++ b/lib_pypy/numpypy/core/numeric.py @@ -0,0 +1,311 @@ + +from _numpypy import array, ndarray, int_, float_ #, complex_# , longlong +from _numpypy import concatenate +import sys +import _numpypy as multiarray # ARGH +from numpypy.core.arrayprint import array2string + + + +def asanyarray(a, dtype=None, order=None, maskna=None, ownmaskna=False): + """ + Convert the input to an ndarray, but pass ndarray subclasses through. + + Parameters + ---------- + a : array_like + Input data, in any form that can be converted to an array. This + includes scalars, lists, lists of tuples, tuples, tuples of tuples, + tuples of lists, and ndarrays. + dtype : data-type, optional + By default, the data-type is inferred from the input data. + order : {'C', 'F'}, optional + Whether to use row-major ('C') or column-major ('F') memory + representation. Defaults to 'C'. + maskna : bool or None, optional + If this is set to True, it forces the array to have an NA mask. + If this is set to False, it forces the array to not have an NA + mask. + ownmaskna : bool, optional + If this is set to True, forces the array to have a mask which + it owns. + + Returns + ------- + out : ndarray or an ndarray subclass + Array interpretation of `a`. If `a` is an ndarray or a subclass + of ndarray, it is returned as-is and no copy is performed. + + See Also + -------- + asarray : Similar function which always returns ndarrays. + ascontiguousarray : Convert input to a contiguous array. + asfarray : Convert input to a floating point ndarray. + asfortranarray : Convert input to an ndarray with column-major + memory order. + asarray_chkfinite : Similar function which checks input for NaNs and + Infs. + fromiter : Create an array from an iterator. + fromfunction : Construct an array by executing a function on grid + positions. + + Examples + -------- + Convert a list into an array: + + >>> a = [1, 2] + >>> np.asanyarray(a) + array([1, 2]) + + Instances of `ndarray` subclasses are passed through as-is: + + >>> a = np.matrix([1, 2]) + >>> np.asanyarray(a) is a + True + + """ + return array(a, dtype, copy=False, order=order, subok=True, + maskna=maskna, ownmaskna=ownmaskna) + +def base_repr(number, base=2, padding=0): + """ + Return a string representation of a number in the given base system. + + Parameters + ---------- + number : int + The value to convert. Only positive values are handled. + base : int, optional + Convert `number` to the `base` number system. The valid range is 2-36, + the default value is 2. + padding : int, optional + Number of zeros padded on the left. Default is 0 (no padding). + + Returns + ------- + out : str + String representation of `number` in `base` system. + + See Also + -------- + binary_repr : Faster version of `base_repr` for base 2. + + Examples + -------- + >>> np.base_repr(5) + '101' + >>> np.base_repr(6, 5) + '11' + >>> np.base_repr(7, base=5, padding=3) + '00012' + + >>> np.base_repr(10, base=16) + 'A' + >>> np.base_repr(32, base=16) + '20' + + """ + digits = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ' + if base > len(digits): + raise ValueError("Bases greater than 36 not handled in base_repr.") + + num = abs(number) + res = [] + while num: + res.append(digits[num % base]) + num //= base + if padding: + res.append('0' * padding) + if number < 0: + res.append('-') + return ''.join(reversed(res or '0')) + +_typelessdata = [int_, float_]#, complex_] +# XXX +#if issubclass(intc, int): +# _typelessdata.append(intc) + +#if issubclass(longlong, int): +# _typelessdata.append(longlong) + +def array_repr(arr, max_line_width=None, precision=None, suppress_small=None): + """ + Return the string representation of an array. + + Parameters + ---------- + arr : ndarray + Input array. + max_line_width : int, optional + The maximum number of columns the string should span. Newline + characters split the string appropriately after array elements. + precision : int, optional + Floating point precision. Default is the current printing precision + (usually 8), which can be altered using `set_printoptions`. + suppress_small : bool, optional + Represent very small numbers as zero, default is False. Very small + is defined by `precision`, if the precision is 8 then + numbers smaller than 5e-9 are represented as zero. + + Returns + ------- + string : str + The string representation of an array. + + See Also + -------- + array_str, array2string, set_printoptions + + Examples + -------- + >>> np.array_repr(np.array([1,2])) + 'array([1, 2])' + >>> np.array_repr(np.ma.array([0.])) + 'MaskedArray([ 0.])' + >>> np.array_repr(np.array([], np.int32)) + 'array([], dtype=int32)' + + >>> x = np.array([1e-6, 4e-7, 2, 3]) + >>> np.array_repr(x, precision=6, suppress_small=True) + 'array([ 0.000001, 0. , 2. , 3. ])' + + """ + if arr.size > 0 or arr.shape==(0,): + lst = array2string(arr, max_line_width, precision, suppress_small, + ', ', "array(") + else: # show zero-length shape unless it is (0,) + lst = "[], shape=%s" % (repr(arr.shape),) + + if arr.__class__ is not ndarray: + cName= arr.__class__.__name__ + else: + cName = "array" + + skipdtype = (arr.dtype.type in _typelessdata) and arr.size > 0 + + # XXX pypy lacks support + if 0 and arr.flags.maskna: + whichna = isna(arr) + # If nothing is NA, explicitly signal the NA-mask + if not any(whichna): + lst += ", maskna=True" + # If everything is NA, can't skip the dtype + if skipdtype and all(whichna): + skipdtype = False + + if skipdtype: + return "%s(%s)" % (cName, lst) + else: + typename = arr.dtype.name + # Quote typename in the output if it is "complex". + if typename and not (typename[0].isalpha() and typename.isalnum()): + typename = "'%s'" % typename + + lf = '' + if 0: # or issubclass(arr.dtype.type, flexible): + if arr.dtype.names: + typename = "%s" % str(arr.dtype) + else: + typename = "'%s'" % str(arr.dtype) + lf = '\n'+' '*len("array(") + return cName + "(%s, %sdtype=%s)" % (lst, lf, typename) + +def array_str(a, max_line_width=None, precision=None, suppress_small=None): + """ + Return a string representation of the data in an array. + + The data in the array is returned as a single string. This function is + similar to `array_repr`, the difference being that `array_repr` also + returns information on the kind of array and its data type. + + Parameters + ---------- + a : ndarray + Input array. + max_line_width : int, optional + Inserts newlines if text is longer than `max_line_width`. The + default is, indirectly, 75. + precision : int, optional + Floating point precision. Default is the current printing precision + (usually 8), which can be altered using `set_printoptions`. + suppress_small : bool, optional + Represent numbers "very close" to zero as zero; default is False. + Very close is defined by precision: if the precision is 8, e.g., + numbers smaller (in absolute value) than 5e-9 are represented as + zero. + + See Also + -------- + array2string, array_repr, set_printoptions + + Examples + -------- + >>> np.array_str(np.arange(3)) + '[0 1 2]' + + """ + return array2string(a, max_line_width, precision, suppress_small, ' ', "", str) + +def set_string_function(f, repr=True): + """ + Set a Python function to be used when pretty printing arrays. + + Parameters + ---------- + f : function or None + Function to be used to pretty print arrays. The function should expect + a single array argument and return a string of the representation of + the array. If None, the function is reset to the default NumPy function + to print arrays. + repr : bool, optional + If True (default), the function for pretty printing (``__repr__``) + is set, if False the function that returns the default string + representation (``__str__``) is set. + + See Also + -------- + set_printoptions, get_printoptions + + Examples + -------- + >>> def pprint(arr): + ... return 'HA! - What are you going to do now?' + ... + >>> np.set_string_function(pprint) + >>> a = np.arange(10) + >>> a + HA! - What are you going to do now? + >>> print a + [0 1 2 3 4 5 6 7 8 9] + + We can reset the function to the default: + + >>> np.set_string_function(None) + >>> a + array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) + + `repr` affects either pretty printing or normal string representation. + Note that ``__repr__`` is still affected by setting ``__str__`` + because the width of each array element in the returned string becomes + equal to the length of the result of ``__str__()``. + + >>> x = np.arange(4) + >>> np.set_string_function(lambda x:'random', repr=False) + >>> x.__str__() + 'random' + >>> x.__repr__() + 'array([ 0, 1, 2, 3])' + + """ + if f is None: + if repr: + return multiarray.set_string_function(array_repr, 1) + else: + return multiarray.set_string_function(array_str, 0) + else: + return multiarray.set_string_function(f, repr) + +set_string_function(array_str, 0) +set_string_function(array_repr, 1) + +little_endian = (sys.byteorder == 'little') diff --git a/pypy/annotation/unaryop.py b/pypy/annotation/unaryop.py --- a/pypy/annotation/unaryop.py +++ b/pypy/annotation/unaryop.py @@ -440,6 +440,12 @@ def method_popitem(dct): return dct.getanyitem('items') + def method_pop(dct, s_key, s_dfl=None): + dct.dictdef.generalize_key(s_key) + if s_dfl is not None: + dct.dictdef.generalize_value(s_dfl) + return dct.dictdef.read_value() + def _can_only_throw(dic, *ignore): if dic1.dictdef.dictkey.custom_eq_hash: return None # r_dict: can throw anything diff --git a/pypy/jit/codewriter/assembler.py b/pypy/jit/codewriter/assembler.py --- a/pypy/jit/codewriter/assembler.py +++ b/pypy/jit/codewriter/assembler.py @@ -81,10 +81,15 @@ if not isinstance(value, (llmemory.AddressAsInt, ComputedIntSymbolic)): value = lltype.cast_primitive(lltype.Signed, value) - if allow_short and -128 <= value <= 127: - # emit the constant as a small integer - self.code.append(chr(value & 0xFF)) - return True + if allow_short: + try: + short_num = -128 <= value <= 127 + except TypeError: # "Symbolics cannot be compared!" + short_num = False + if short_num: + # emit the constant as a small integer + self.code.append(chr(value & 0xFF)) + return True constants = self.constants_i elif kind == 'ref': value = lltype.cast_opaque_ptr(llmemory.GCREF, value) diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py --- a/pypy/module/micronumpy/__init__.py +++ b/pypy/module/micronumpy/__init__.py @@ -28,6 +28,11 @@ 'fromstring': 'interp_support.fromstring', 'flatiter': 'interp_numarray.W_FlatIterator', 'isna': 'interp_numarray.isna', + 'concatenate': 'interp_numarray.concatenate', + + 'set_string_function': 'appbridge.set_string_function', + + 'count_reduce_items': 'interp_numarray.count_reduce_items', 'True_': 'types.Bool.True', 'False_': 'types.Bool.False', @@ -67,6 +72,7 @@ ("copysign", "copysign"), ("cos", "cos"), ("divide", "divide"), + ("true_divide", "true_divide"), ("equal", "equal"), ("exp", "exp"), ("fabs", "fabs"), @@ -89,6 +95,9 @@ ("tan", "tan"), ('bitwise_and', 'bitwise_and'), ('bitwise_or', 'bitwise_or'), + ('bitwise_not', 'invert'), + ('isnan', 'isnan'), + ('isinf', 'isinf'), ]: interpleveldefs[exposed] = "interp_ufuncs.get(space).%s" % impl diff --git a/pypy/module/micronumpy/app_numpy.py b/pypy/module/micronumpy/app_numpy.py --- a/pypy/module/micronumpy/app_numpy.py +++ b/pypy/module/micronumpy/app_numpy.py @@ -59,7 +59,7 @@ if not hasattr(a, "max"): a = _numpypy.array(a) return a.max(axis) - + def arange(start, stop=None, step=1, dtype=None): '''arange([start], stop[, step], dtype=None) Generate values in the half-interval [start, stop). diff --git a/pypy/module/micronumpy/appbridge.py b/pypy/module/micronumpy/appbridge.py new file mode 100644 --- /dev/null +++ b/pypy/module/micronumpy/appbridge.py @@ -0,0 +1,38 @@ + +from pypy.rlib.objectmodel import specialize + +class AppBridgeCache(object): + w__var = None + w__std = None + w_module = None + w_array_repr = None + w_array_str = None + + def __init__(self, space): + self.w_import = space.appexec([], """(): + def f(): + import sys + __import__('numpypy.core._methods') + return sys.modules['numpypy.core._methods'] + return f + """) + + @specialize.arg(2) + def call_method(self, space, name, *args): + w_meth = getattr(self, 'w_' + name) + if w_meth is None: + if self.w_module is None: + self.w_module = space.call_function(self.w_import) + w_meth = space.getattr(self.w_module, space.wrap(name)) + setattr(self, 'w_' + name, w_meth) + return space.call_function(w_meth, *args) + +def set_string_function(space, w_f, w_repr): + cache = get_appbridge_cache(space) + if space.is_true(w_repr): + cache.w_array_repr = w_f + else: + cache.w_array_str = w_f + +def get_appbridge_cache(space): + return space.fromcache(AppBridgeCache) diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py --- a/pypy/module/micronumpy/compile.py +++ b/pypy/module/micronumpy/compile.py @@ -32,13 +32,16 @@ class BadToken(Exception): pass -SINGLE_ARG_FUNCTIONS = ["sum", "prod", "max", "min", "all", "any", "unegative"] +SINGLE_ARG_FUNCTIONS = ["sum", "prod", "max", "min", "all", "any", + "unegative", "flat"] +TWO_ARG_FUNCTIONS = ['take'] class FakeSpace(object): w_ValueError = None w_TypeError = None w_IndexError = None w_OverflowError = None + w_NotImplementedError = None w_None = None w_bool = "bool" @@ -53,6 +56,9 @@ """NOT_RPYTHON""" self.fromcache = InternalSpaceCache(self).getorbuild + def _freeze_(self): + return True + def issequence_w(self, w_obj): return isinstance(w_obj, ListObject) or isinstance(w_obj, W_NDimArray) @@ -144,6 +150,9 @@ def allocate_instance(self, klass, w_subtype): return instantiate(klass) + def newtuple(self, list_w): + raise ValueError + def len_w(self, w_obj): if isinstance(w_obj, ListObject): return len(w_obj.items) @@ -371,12 +380,12 @@ for arg in self.args])) def execute(self, interp): + arr = self.args[0].execute(interp) + if not isinstance(arr, BaseArray): + raise ArgumentNotAnArray if self.name in SINGLE_ARG_FUNCTIONS: if len(self.args) != 1 and self.name != 'sum': raise ArgumentMismatch - arr = self.args[0].execute(interp) - if not isinstance(arr, BaseArray): - raise ArgumentNotAnArray if self.name == "sum": if len(self.args)>1: w_res = arr.descr_sum(interp.space, @@ -396,21 +405,31 @@ elif self.name == "unegative": neg = interp_ufuncs.get(interp.space).negative w_res = neg.call(interp.space, [arr]) + elif self.name == "flat": + w_res = arr.descr_get_flatiter(interp.space) else: assert False # unreachable code - if isinstance(w_res, BaseArray): - return w_res - if isinstance(w_res, FloatObject): - dtype = get_dtype_cache(interp.space).w_float64dtype - elif isinstance(w_res, BoolObject): - dtype = get_dtype_cache(interp.space).w_booldtype - elif isinstance(w_res, interp_boxes.W_GenericBox): - dtype = w_res.get_dtype(interp.space) + elif self.name in TWO_ARG_FUNCTIONS: + arg = self.args[1].execute(interp) + if not isinstance(arg, BaseArray): + raise ArgumentNotAnArray + if self.name == 'take': + w_res = arr.descr_take(interp.space, arg) else: - dtype = None - return scalar_w(interp.space, dtype, w_res) + assert False # unreachable else: raise WrongFunctionName + if isinstance(w_res, BaseArray): + return w_res + if isinstance(w_res, FloatObject): + dtype = get_dtype_cache(interp.space).w_float64dtype + elif isinstance(w_res, BoolObject): + dtype = get_dtype_cache(interp.space).w_booldtype + elif isinstance(w_res, interp_boxes.W_GenericBox): + dtype = w_res.get_dtype(interp.space) + else: + dtype = None + return scalar_w(interp.space, dtype, w_res) _REGEXES = [ ('-?[\d\.]+', 'number'), diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py --- a/pypy/module/micronumpy/interp_boxes.py +++ b/pypy/module/micronumpy/interp_boxes.py @@ -8,6 +8,7 @@ from pypy.tool.sourcetools import func_with_new_name +MIXIN_32 = (int_typedef,) if LONG_BIT == 32 else () MIXIN_64 = (int_typedef,) if LONG_BIT == 64 else () def new_dtype_getter(name): @@ -28,6 +29,7 @@ def convert_to(self, dtype): return dtype.box(self.value) + class W_GenericBox(Wrappable): _attrs_ = () @@ -93,7 +95,7 @@ descr_neg = _unaryop_impl("negative") descr_abs = _unaryop_impl("absolute") - def descr_tolist(self, space): + def item(self, space): return self.get_dtype(space).itemtype.to_builtin_type(space, self) @@ -104,7 +106,8 @@ _attrs_ = () class W_IntegerBox(W_NumberBox): - pass + def int_w(self, space): + return space.int_w(self.descr_int(space)) class W_SignedIntegerBox(W_IntegerBox): pass @@ -187,7 +190,7 @@ __neg__ = interp2app(W_GenericBox.descr_neg), __abs__ = interp2app(W_GenericBox.descr_abs), - tolist = interp2app(W_GenericBox.descr_tolist), + tolist = interp2app(W_GenericBox.item), ) W_BoolBox.typedef = TypeDef("bool_", W_GenericBox.typedef, @@ -231,7 +234,7 @@ __new__ = interp2app(W_UInt16Box.descr__new__.im_func), ) -W_Int32Box.typedef = TypeDef("int32", W_SignedIntegerBox.typedef, +W_Int32Box.typedef = TypeDef("int32", (W_SignedIntegerBox.typedef,) + MIXIN_32, __module__ = "numpypy", __new__ = interp2app(W_Int32Box.descr__new__.im_func), ) @@ -241,24 +244,18 @@ __new__ = interp2app(W_UInt32Box.descr__new__.im_func), ) -if LONG_BIT == 32: - long_name = "int32" -elif LONG_BIT == 64: - long_name = "int64" -W_LongBox.typedef = TypeDef(long_name, (W_SignedIntegerBox.typedef, int_typedef,), - __module__ = "numpypy", - __new__ = interp2app(W_LongBox.descr__new__.im_func), -) - -W_ULongBox.typedef = TypeDef("u" + long_name, W_UnsignedIntegerBox.typedef, - __module__ = "numpypy", -) - W_Int64Box.typedef = TypeDef("int64", (W_SignedIntegerBox.typedef,) + MIXIN_64, __module__ = "numpypy", __new__ = interp2app(W_Int64Box.descr__new__.im_func), ) +if LONG_BIT == 32: + W_LongBox = W_Int32Box + W_ULongBox = W_UInt32Box +elif LONG_BIT == 64: + W_LongBox = W_Int64Box + W_ULongBox = W_UInt64Box + W_UInt64Box.typedef = TypeDef("uint64", W_UnsignedIntegerBox.typedef, __module__ = "numpypy", __new__ = interp2app(W_UInt64Box.descr__new__.im_func), @@ -283,3 +280,4 @@ __new__ = interp2app(W_Float64Box.descr__new__.im_func), ) + diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py --- a/pypy/module/micronumpy/interp_dtype.py +++ b/pypy/module/micronumpy/interp_dtype.py @@ -89,8 +89,19 @@ def descr_get_shape(self, space): return space.newtuple([]) + def eq(self, space, w_other): + w_other = space.call_function(space.gettypefor(W_Dtype), w_other) + return space.is_w(self, w_other) + + def descr_eq(self, space, w_other): + return space.wrap(self.eq(space, w_other)) + + def descr_ne(self, space, w_other): + return space.wrap(not self.eq(space, w_other)) + def is_int_type(self): - return self.kind == SIGNEDLTR or self.kind == UNSIGNEDLTR + return (self.kind == SIGNEDLTR or self.kind == UNSIGNEDLTR or + self.kind == BOOLLTR) def is_bool_type(self): return self.kind == BOOLLTR @@ -101,12 +112,15 @@ __str__= interp2app(W_Dtype.descr_str), __repr__ = interp2app(W_Dtype.descr_repr), + __eq__ = interp2app(W_Dtype.descr_eq), + __ne__ = interp2app(W_Dtype.descr_ne), num = interp_attrproperty("num", cls=W_Dtype), kind = interp_attrproperty("kind", cls=W_Dtype), type = interp_attrproperty_w("w_box_type", cls=W_Dtype), itemsize = GetSetProperty(W_Dtype.descr_get_itemsize), shape = GetSetProperty(W_Dtype.descr_get_shape), + name = interp_attrproperty('name', cls=W_Dtype), ) W_Dtype.typedef.acceptable_as_base_class = False diff --git a/pypy/module/micronumpy/interp_iter.py b/pypy/module/micronumpy/interp_iter.py --- a/pypy/module/micronumpy/interp_iter.py +++ b/pypy/module/micronumpy/interp_iter.py @@ -4,6 +4,49 @@ from pypy.module.micronumpy.strides import calculate_broadcast_strides,\ calculate_slice_strides +""" This is a mini-tutorial on iterators, strides, and +memory layout. It assumes you are familiar with the terms, see +http://docs.scipy.org/doc/numpy/reference/arrays.ndarray.html +for a more gentle introduction. + +Given an array x: x.shape == [5,6], + +At which byte in x.data does the item x[3,4] begin? +if x.strides==[1,5]: + pData = x.pData + (x.start + 3*1 + 4*5)*sizeof(x.pData[0]) + pData = x.pData + (x.start + 24) * sizeof(x.pData[0]) +so the offset of the element is 24 elements after the first + +What is the next element in x after coordinates [3,4]? +if x.order =='C': + next == [3,5] => offset is 28 +if x.order =='F': + next == [4,4] => offset is 24 +so for the strides [1,5] x is 'F' contiguous +likewise, for the strides [6,1] x would be 'C' contiguous. + +Iterators have an internal representation of the current coordinates +(indices), the array, strides, and backstrides. A short digression to +explain backstrides: what is the coordinate and offset after [3,5] in +the example above? +if x.order == 'C': + next == [4,0] => offset is 4 +if x.order == 'F': + next == [4,5] => offset is 25 +Note that in 'C' order we stepped BACKWARDS 24 while 'overflowing' a +shape dimension + which is back 25 and forward 1, + which is x.strides[1] * (x.shape[1] - 1) + x.strides[0] +so if we precalculate the overflow backstride as +[x.strides[i] * (x.shape[i] - 1) for i in range(len(x.shape))] +we can go faster. +All the calculations happen in next() + +next_step_x() tries to do the iteration for a number of steps at once, +but then we cannot gaurentee that we only overflow one single shape +dimension, perhaps we could overflow times in one big step. +""" + # structures to describe slicing class Chunk(object): @@ -17,6 +60,10 @@ if self.step != 0: shape.append(self.lgt) + def __repr__(self): + return 'Chunk(%d, %d, %d, %d)' % (self.start, self.stop, self.step, + self.lgt) + class BaseTransform(object): pass @@ -51,9 +98,9 @@ self.size = size def next(self, shapelen): - return self._next(1) + return self.next_skip_x(1) - def _next(self, ofs): + def next_skip_x(self, ofs): arr = instantiate(ArrayIterator) arr.size = self.size arr.offset = self.offset + ofs @@ -61,7 +108,7 @@ def next_no_increase(self, shapelen): # a hack to make JIT believe this is always virtual - return self._next(0) + return self.next_skip_x(0) def done(self): return self.offset >= self.size @@ -133,6 +180,36 @@ res._done = done return res + @jit.unroll_safe + def next_skip_x(self, shapelen, step): + shapelen = jit.promote(len(self.res_shape)) + offset = self.offset + indices = [0] * shapelen + for i in range(shapelen): + indices[i] = self.indices[i] + done = False + for i in range(shapelen - 1, -1, -1): + if indices[i] < self.res_shape[i] - step: + indices[i] += step + offset += self.strides[i] * step + break + else: + remaining_step = (indices[i] + step) // self.res_shape[i] + this_i_step = step - remaining_step * self.res_shape[i] + offset += self.strides[i] * this_i_step + indices[i] = indices[i] + this_i_step + step = remaining_step + else: + done = True + res = instantiate(ViewIterator) + res.offset = offset + res.indices = indices + res.strides = self.strides + res.backstrides = self.backstrides + res.res_shape = self.res_shape + res._done = done + return res + def apply_transformations(self, arr, transformations): v = BaseIterator.apply_transformations(self, arr, transformations) if len(arr.shape) == 1: @@ -153,8 +230,13 @@ class AxisIterator(BaseIterator): def __init__(self, start, dim, shape, strides, backstrides): self.res_shape = shape[:] - self.strides = strides[:dim] + [0] + strides[dim:] - self.backstrides = backstrides[:dim] + [0] + backstrides[dim:] + if len(shape) == len(strides): + # keepdims = True + self.strides = strides[:dim] + [0] + strides[dim + 1:] + self.backstrides = backstrides[:dim] + [0] + backstrides[dim + 1:] + else: + self.strides = strides[:dim] + [0] + strides[dim:] + self.backstrides = backstrides[:dim] + [0] + backstrides[dim:] self.first_line = True self.indices = [0] * len(shape) self._done = False diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -1,16 +1,20 @@ from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.error import OperationError, operationerrfmt -from pypy.interpreter.gateway import interp2app, NoneNotWrapped +from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.interpreter.typedef import TypeDef, GetSetProperty -from pypy.module.micronumpy import interp_ufuncs, interp_dtype, signature,\ - interp_boxes -from pypy.module.micronumpy.strides import calculate_slice_strides +from pypy.module.micronumpy import (interp_ufuncs, interp_dtype, interp_boxes, + signature, support) +from pypy.module.micronumpy.strides import (calculate_slice_strides, + shape_agreement, find_shape_and_elems, get_shape_from_iterable, + calc_new_strides, to_coords) from pypy.rlib import jit from pypy.rpython.lltypesystem import lltype, rffi from pypy.tool.sourcetools import func_with_new_name from pypy.rlib.rstring import StringBuilder -from pypy.module.micronumpy.interp_iter import ArrayIterator, OneDimIterator,\ - SkipLastAxisIterator, Chunk, ViewIterator +from pypy.module.micronumpy.interp_iter import (ArrayIterator, OneDimIterator, + SkipLastAxisIterator, Chunk, ViewIterator) +from pypy.module.micronumpy.appbridge import get_appbridge_cache + numpy_driver = jit.JitDriver( greens=['shapelen', 'sig'], @@ -58,176 +62,21 @@ reds=['idx', 'idxi', 'frame', 'arr'], name='numpy_filterset', ) - -def _find_shape_and_elems(space, w_iterable): - shape = [space.len_w(w_iterable)] - batch = space.listview(w_iterable) - while True: - new_batch = [] - if not batch: - return shape, [] - if not space.issequence_w(batch[0]): - for elem in batch: - if space.issequence_w(elem): - raise OperationError(space.w_ValueError, space.wrap( - "setting an array element with a sequence")) - return shape, batch - size = space.len_w(batch[0]) - for w_elem in batch: - if not space.issequence_w(w_elem) or space.len_w(w_elem) != size: - raise OperationError(space.w_ValueError, space.wrap( - "setting an array element with a sequence")) - new_batch += space.listview(w_elem) - shape.append(size) - batch = new_batch - -def shape_agreement(space, shape1, shape2): - ret = _shape_agreement(shape1, shape2) - if len(ret) < max(len(shape1), len(shape2)): - raise OperationError(space.w_ValueError, - space.wrap("operands could not be broadcast together with shapes (%s) (%s)" % ( - ",".join([str(x) for x in shape1]), - ",".join([str(x) for x in shape2]), - )) - ) - return ret - -def _shape_agreement(shape1, shape2): - """ Checks agreement about two shapes with respect to broadcasting. Returns - the resulting shape. - """ - lshift = 0 - rshift = 0 - if len(shape1) > len(shape2): - m = len(shape1) - n = len(shape2) - rshift = len(shape2) - len(shape1) - remainder = shape1 - else: - m = len(shape2) - n = len(shape1) - lshift = len(shape1) - len(shape2) - remainder = shape2 - endshape = [0] * m - indices1 = [True] * m - indices2 = [True] * m - for i in range(m - 1, m - n - 1, -1): - left = shape1[i + lshift] - right = shape2[i + rshift] - if left == right: - endshape[i] = left - elif left == 1: - endshape[i] = right - indices1[i + lshift] = False - elif right == 1: - endshape[i] = left - indices2[i + rshift] = False - else: - return [] - #raise OperationError(space.w_ValueError, space.wrap( - # "frames are not aligned")) - for i in range(m - n): - endshape[i] = remainder[i] - return endshape - -def get_shape_from_iterable(space, old_size, w_iterable): - new_size = 0 - new_shape = [] - if space.isinstance_w(w_iterable, space.w_int): - new_size = space.int_w(w_iterable) - if new_size < 0: - new_size = old_size - new_shape = [new_size] - else: - neg_dim = -1 - batch = space.listview(w_iterable) - new_size = 1 - if len(batch) < 1: - if old_size == 1: - # Scalars can have an empty size. - new_size = 1 - else: - new_size = 0 - new_shape = [] - i = 0 - for elem in batch: - s = space.int_w(elem) - if s < 0: - if neg_dim >= 0: - raise OperationError(space.w_ValueError, space.wrap( - "can only specify one unknown dimension")) - s = 1 - neg_dim = i - new_size *= s - new_shape.append(s) - i += 1 - if neg_dim >= 0: - new_shape[neg_dim] = old_size / new_size - new_size *= new_shape[neg_dim] - if new_size != old_size: - raise OperationError(space.w_ValueError, - space.wrap("total size of new array must be unchanged")) - return new_shape - -# Recalculating strides. Find the steps that the iteration does for each -# dimension, given the stride and shape. Then try to create a new stride that -# fits the new shape, using those steps. If there is a shape/step mismatch -# (meaning that the realignment of elements crosses from one step into another) -# return None so that the caller can raise an exception. -def calc_new_strides(new_shape, old_shape, old_strides, order): - # Return the proper strides for new_shape, or None if the mapping crosses - # stepping boundaries - - # Assumes that prod(old_shape) == prod(new_shape), len(old_shape) > 1, and - # len(new_shape) > 0 - steps = [] - last_step = 1 - oldI = 0 - new_strides = [] - if order == 'F': - for i in range(len(old_shape)): - steps.append(old_strides[i] / last_step) - last_step *= old_shape[i] - cur_step = steps[0] - n_new_elems_used = 1 - n_old_elems_to_use = old_shape[0] - for s in new_shape: - new_strides.append(cur_step * n_new_elems_used) - n_new_elems_used *= s - while n_new_elems_used > n_old_elems_to_use: - oldI += 1 - if steps[oldI] != steps[oldI - 1]: - return None - n_old_elems_to_use *= old_shape[oldI] - if n_new_elems_used == n_old_elems_to_use: - oldI += 1 - if oldI < len(old_shape): - cur_step = steps[oldI] - n_old_elems_to_use *= old_shape[oldI] - elif order == 'C': - for i in range(len(old_shape) - 1, -1, -1): - steps.insert(0, old_strides[i] / last_step) - last_step *= old_shape[i] - cur_step = steps[-1] - n_new_elems_used = 1 - oldI = -1 - n_old_elems_to_use = old_shape[-1] - for i in range(len(new_shape) - 1, -1, -1): - s = new_shape[i] - new_strides.insert(0, cur_step * n_new_elems_used) - n_new_elems_used *= s - while n_new_elems_used > n_old_elems_to_use: - oldI -= 1 - if steps[oldI] != steps[oldI + 1]: - return None - n_old_elems_to_use *= old_shape[oldI] - if n_new_elems_used == n_old_elems_to_use: - oldI -= 1 - if oldI >= -len(old_shape): - cur_step = steps[oldI] - n_old_elems_to_use *= old_shape[oldI] - assert len(new_strides) == len(new_shape) - return new_strides +take_driver = jit.JitDriver( + greens=['shapelen', 'sig'], + reds=['index_i', 'res_i', 'concr', 'index', 'res'], + name='numpy_take', +) +flat_get_driver = jit.JitDriver( + greens=['shapelen', 'base'], + reds=['step', 'ri', 'basei', 'res'], + name='numpy_flatget', +) +flat_set_driver = jit.JitDriver( + greens=['shapelen', 'base'], + reds=['step', 'ai', 'lngth', 'arr', 'basei'], + name='numpy_flatset', +) class BaseArray(Wrappable): _attrs_ = ["invalidates", "shape", 'size'] @@ -268,6 +117,7 @@ descr_pos = _unaryop_impl("positive") descr_neg = _unaryop_impl("negative") descr_abs = _unaryop_impl("absolute") + descr_invert = _unaryop_impl("invert") def _binop_impl(ufunc_name): def impl(self, space, w_other): @@ -310,9 +160,11 @@ def _reduce_ufunc_impl(ufunc_name, promote_to_largest=False): def impl(self, space, w_axis=None): if space.is_w(w_axis, space.w_None): - w_axis = space.wrap(-1) + axis = -1 + else: + axis = space.int_w(w_axis) return getattr(interp_ufuncs.get(space), ufunc_name).reduce(space, - self, True, promote_to_largest, w_axis) + self, True, promote_to_largest, axis) return func_with_new_name(impl, "reduce_%s_impl" % ufunc_name) descr_sum = _reduce_ufunc_impl("add") @@ -430,21 +282,22 @@ def descr_copy(self, space): return self.copy(space) - def descr_flatten(self, space): - return self.flatten(space) + def descr_flatten(self, space, w_order=None): + if isinstance(self, Scalar): + # scalars have no storage + return self.descr_reshape(space, [space.wrap(1)]) + concr = self.get_concrete() + w_res = concr.descr_ravel(space, w_order) + if w_res.storage == concr.storage: + return w_res.copy(space) + return w_res def copy(self, space): return self.get_concrete().copy(space) def empty_copy(self, space, dtype): shape = self.shape - size = 1 - for elem in shape: - size *= elem - return W_NDimArray(size, shape[:], dtype, 'C') - - def flatten(self, space): - return self.get_concrete().flatten(space) + return W_NDimArray(support.product(shape), shape[:], dtype, 'C') def descr_len(self, space): if len(self.shape): @@ -453,33 +306,32 @@ "len() of unsized object")) def descr_repr(self, space): - res = StringBuilder() - res.append("array(") - concrete = self.get_concrete_or_scalar() - dtype = concrete.find_dtype() - if not concrete.size: - res.append('[]') - if len(self.shape) > 1: - # An empty slice reports its shape - res.append(", shape=(") - self_shape = str(self.shape) - res.append_slice(str(self_shape), 1, len(self_shape) - 1) - res.append(')') - else: - concrete.to_str(space, 1, res, indent=' ') - if (dtype is not interp_dtype.get_dtype_cache(space).w_float64dtype and - not (dtype.kind == interp_dtype.SIGNEDLTR and - dtype.itemtype.get_element_size() == rffi.sizeof(lltype.Signed)) or - not self.size): - res.append(", dtype=" + dtype.name) - res.append(")") - return space.wrap(res.build()) + cache = get_appbridge_cache(space) + if cache.w_array_repr is None: + return space.wrap(self.dump_data()) + return space.call_function(cache.w_array_repr, self) + + def dump_data(self): + concr = self.get_concrete() + i = concr.create_iter() + first = True + s = StringBuilder() + s.append('array([') + while not i.done(): + if first: + first = False + else: + s.append(', ') + s.append(concr.dtype.itemtype.str_format(concr.getitem(i.offset))) + i = i.next(len(concr.shape)) + s.append('])') + return s.build() def descr_str(self, space): - ret = StringBuilder() - concrete = self.get_concrete_or_scalar() - concrete.to_str(space, 0, ret, ' ') - return space.wrap(ret.build()) + cache = get_appbridge_cache(space) + if cache.w_array_str is None: + return space.wrap(self.dump_data()) + return space.call_function(cache.w_array_str, self) @jit.unroll_safe def _single_item_result(self, space, w_idx): @@ -519,7 +371,7 @@ def count_all_true(self, arr): sig = arr.find_sig() - frame = sig.create_frame(self) + frame = sig.create_frame(arr) shapelen = len(arr.shape) s = 0 iter = None @@ -533,6 +385,9 @@ def getitem_filter(self, space, arr): concr = arr.get_concrete() + if concr.size > self.size: + raise OperationError(space.w_IndexError, + space.wrap("index out of range for array")) size = self.count_all_true(concr) res = W_NDimArray(size, [size], self.find_dtype()) ri = ArrayIterator(size) @@ -541,7 +396,7 @@ sig = self.find_sig() frame = sig.create_frame(self) v = None - while not frame.done(): + while not ri.done(): filter_driver.jit_merge_point(concr=concr, argi=argi, ri=ri, frame=frame, v=v, res=res, sig=sig, shapelen=shapelen, self=self) @@ -581,7 +436,7 @@ item = concrete._index_of_single_item(space, w_idx) return concrete.getitem(item) chunks = self._prepare_slice_args(space, w_idx) - return space.wrap(self.create_slice(chunks)) + return self.create_slice(chunks) def descr_setitem(self, space, w_idx, w_value): self.invalidated() @@ -635,8 +490,11 @@ w_shape = args_w[0] else: w_shape = space.newtuple(args_w) + new_shape = get_shape_from_iterable(space, self.size, w_shape) + return self.reshape(space, new_shape) + + def reshape(self, space, new_shape): concrete = self.get_concrete() - new_shape = get_shape_from_iterable(space, concrete.size, w_shape) # Since we got to here, prod(new_shape) == self.size new_strides = calc_new_strides(new_shape, concrete.shape, concrete.strides, concrete.order) @@ -647,7 +505,7 @@ for nd in range(ndims): new_backstrides[nd] = (new_shape[nd] - 1) * new_strides[nd] arr = W_NDimSlice(concrete.start, new_strides, new_backstrides, - new_shape, self) + new_shape, concrete) else: # Create copy with contiguous data arr = concrete.copy(space) @@ -657,7 +515,7 @@ def descr_tolist(self, space): if len(self.shape) == 0: assert isinstance(self, Scalar) - return self.value.descr_tolist(space) + return self.value.item(space) w_result = space.newlist([]) for i in range(self.shape[0]): space.call_method(w_result, "append", @@ -674,17 +532,13 @@ w_denom = space.wrap(self.shape[dim]) return space.div(self.descr_sum_promote(space, w_axis), w_denom) - def descr_var(self, space): - # var = mean((values - mean(values)) ** 2) - w_res = self.descr_sub(space, self.descr_mean(space, space.w_None)) - assert isinstance(w_res, BaseArray) - w_res = w_res.descr_pow(space, space.wrap(2)) - assert isinstance(w_res, BaseArray) - return w_res.descr_mean(space, space.w_None) + def descr_var(self, space, w_axis=None): + return get_appbridge_cache(space).call_method(space, '_var', self, + w_axis) - def descr_std(self, space): - # std(v) = sqrt(var(v)) - return interp_ufuncs.get(space).sqrt.call(space, [self.descr_var(space)]) + def descr_std(self, space, w_axis=None): + return get_appbridge_cache(space).call_method(space, '_std', self, + w_axis) def descr_fill(self, space, w_value): concr = self.get_concrete_or_scalar() @@ -717,6 +571,16 @@ return space.wrap(W_NDimSlice(concrete.start, strides, backstrides, shape, concrete)) + def descr_ravel(self, space, w_order=None): + if w_order is None or space.is_w(w_order, space.w_None): + order = 'C' + else: + order = space.str_w(w_order) + if order != 'C': + raise OperationError(space.w_NotImplementedError, space.wrap( + "order not implemented")) + return self.descr_reshape(space, [space.wrap(-1)]) + def descr_get_flatiter(self, space): return space.wrap(W_FlatIterator(self)) @@ -746,6 +610,60 @@ def supports_fast_slicing(self): return False + def descr_compress(self, space, w_obj, w_axis=None): + index = convert_to_array(space, w_obj) + return self.getitem_filter(space, index) + + def descr_take(self, space, w_obj, w_axis=None): + index = convert_to_array(space, w_obj).get_concrete() + concr = self.get_concrete() + if space.is_w(w_axis, space.w_None): + concr = concr.descr_ravel(space) + else: + raise OperationError(space.w_NotImplementedError, + space.wrap("axis unsupported for take")) + index_i = index.create_iter() + res_shape = index.shape + size = support.product(res_shape) + res = W_NDimArray(size, res_shape[:], concr.dtype, concr.order) + res_i = res.create_iter() + shapelen = len(index.shape) + sig = concr.find_sig() + while not index_i.done(): + take_driver.jit_merge_point(index_i=index_i, index=index, + res_i=res_i, concr=concr, + res=res, + shapelen=shapelen, sig=sig) + w_item = index._getitem_long(space, index_i.offset) + res.setitem(res_i.offset, concr.descr_getitem(space, w_item)) + index_i = index_i.next(shapelen) + res_i = res_i.next(shapelen) + return res + + def _getitem_long(self, space, offset): + # an obscure hack to not have longdtype inside a jitted loop + longdtype = interp_dtype.get_dtype_cache(space).w_longdtype + return self.getitem(offset).convert_to(longdtype).item( + space) + + def descr_item(self, space, w_arg=None): + if space.is_w(w_arg, space.w_None): + if not isinstance(self, Scalar): + raise OperationError(space.w_ValueError, space.wrap("index out of bounds")) + return self.value.item(space) + if space.isinstance_w(w_arg, space.w_int): + if isinstance(self, Scalar): + raise OperationError(space.w_ValueError, space.wrap("index out of bounds")) + concr = self.get_concrete() + i = to_coords(space, self.shape, concr.size, concr.order, w_arg)[0] + # XXX a bit around + item = self.descr_getitem(space, space.newtuple([space.wrap(x) + for x in i])) + assert isinstance(item, interp_boxes.W_GenericBox) + return item.item(space) + raise OperationError(space.w_NotImplementedError, space.wrap( + "non-int arg not supported")) + def convert_to_array(space, w_obj): if isinstance(w_obj, BaseArray): return w_obj @@ -771,22 +689,15 @@ self.shape = [] BaseArray.__init__(self, []) self.dtype = dtype + assert isinstance(value, interp_boxes.W_GenericBox) self.value = value def find_dtype(self): return self.dtype - def to_str(self, space, comma, builder, indent=' ', use_ellipsis=False): - builder.append(self.dtype.itemtype.str_format(self.value)) - def copy(self, space): return Scalar(self.dtype, self.value) - def flatten(self, space): - array = W_NDimArray(self.size, [self.size], self.dtype) - array.setitem(0, self.value) - return array - def fill(self, space, w_value): self.value = self.dtype.coerce(space, w_value) @@ -796,6 +707,11 @@ def get_concrete_or_scalar(self): return self + def reshape(self, space, new_shape): + size = support.product(new_shape) + res = W_NDimArray(size, new_shape, self.dtype, 'C') + res.setitem(0, self.value) + return res class VirtualArray(BaseArray): """ @@ -852,12 +768,9 @@ class VirtualSlice(VirtualArray): def __init__(self, child, chunks, shape): - size = 1 - for sh in shape: - size *= sh self.child = child self.chunks = chunks - self.size = size + self.size = support.product(shape) VirtualArray.__init__(self, 'slice', shape, child.find_dtype()) def create_sig(self): @@ -876,11 +789,12 @@ class Call1(VirtualArray): - def __init__(self, ufunc, name, shape, res_dtype, values): + def __init__(self, ufunc, name, shape, calc_dtype, res_dtype, values): VirtualArray.__init__(self, name, shape, res_dtype) self.values = values self.size = values.size self.ufunc = ufunc + self.calc_dtype = calc_dtype def _del_sources(self): self.values = None @@ -902,9 +816,7 @@ self.left = left self.right = right self.calc_dtype = calc_dtype - self.size = 1 - for s in self.shape: - self.size *= s + self.size = support.product(self.shape) def _del_sources(self): self.left = None @@ -1014,89 +926,6 @@ self.strides = strides self.backstrides = backstrides - def to_str(self, space, comma, builder, indent=' ', use_ellipsis=False): - '''Modifies builder with a representation of the array/slice - The items will be seperated by a comma if comma is 1 - Multidimensional arrays/slices will span a number of lines, - each line will begin with indent. - ''' - size = self.size - ccomma = ',' * comma - ncomma = ',' * (1 - comma) - dtype = self.find_dtype() - if size < 1: - builder.append('[]') - return - if size > 1000: - # Once this goes True it does not go back to False for recursive - # calls - use_ellipsis = True - ndims = len(self.shape) - if ndims == 0: - builder.append(dtype.itemtype.str_format(self.getitem(0))) - return - i = 0 - builder.append('[') - if ndims > 1: - if use_ellipsis: - for i in range(min(3, self.shape[0])): - if i > 0: - builder.append(ccomma + '\n') - if ndims >= 3: - builder.append('\n' + indent) - else: - builder.append(indent) - view = self.create_slice([Chunk(i, 0, 0, 1)]).get_concrete() - view.to_str(space, comma, builder, indent=indent + ' ', - use_ellipsis=use_ellipsis) - if i < self.shape[0] - 1: - builder.append(ccomma + '\n' + indent + '...' + ncomma) - i = self.shape[0] - 3 - else: - i += 1 - while i < self.shape[0]: - if i > 0: - builder.append(ccomma + '\n') - if ndims >= 3: - builder.append('\n' + indent) - else: - builder.append(indent) - # create_slice requires len(chunks) > 1 in order to reduce - # shape - view = self.create_slice([Chunk(i, 0, 0, 1)]).get_concrete() - view.to_str(space, comma, builder, indent=indent + ' ', - use_ellipsis=use_ellipsis) - i += 1 - elif ndims == 1: - spacer = ccomma + ' ' - item = self.start - # An iterator would be a nicer way to walk along the 1d array, but - # how do I reset it if printing ellipsis? iterators have no - # "set_offset()" - i = 0 - if use_ellipsis: - for i in range(min(3, self.shape[0])): - if i > 0: - builder.append(spacer) - builder.append(dtype.itemtype.str_format(self.getitem(item))) - item += self.strides[0] - if i < self.shape[0] - 1: - # Add a comma only if comma is False - this prevents adding - # two commas - builder.append(spacer + '...' + ncomma) - # Ugly, but can this be done with an iterator? - item = self.start + self.backstrides[0] - 2 * self.strides[0] - i = self.shape[0] - 3 - else: - i += 1 - while i < self.shape[0]: - if i > 0: - builder.append(spacer) - builder.append(dtype.itemtype.str_format(self.getitem(item))) - item += self.strides[0] - i += 1 - builder.append(']') - @jit.unroll_safe def _index_of_single_item(self, space, w_idx): if space.isinstance_w(w_idx, space.w_int): @@ -1169,15 +998,6 @@ array.setslice(space, self) return array - def flatten(self, space): - array = W_NDimArray(self.size, [self.size], self.dtype, self.order) - if self.supports_fast_slicing(): - array._fast_setslice(space, self) - else: - arr = SliceArray(array.shape, array.dtype, array, self, no_broadcast=True) - array._sliceloop(arr) - return array - def fill(self, space, w_value): self.setslice(space, scalar_w(space, self.dtype, w_value)) @@ -1192,13 +1012,10 @@ assert isinstance(parent, ConcreteArray) if isinstance(parent, W_NDimSlice): parent = parent.parent - size = 1 - for sh in shape: - size *= sh self.strides = strides self.backstrides = backstrides - ViewArray.__init__(self, size, shape, parent.dtype, parent.order, - parent) + ViewArray.__init__(self, support.product(shape), shape, parent.dtype, + parent.order, parent) self.start = start def create_iter(self): @@ -1274,27 +1091,42 @@ shape.append(item) return size, shape -def array(space, w_item_or_iterable, w_dtype=None, w_order=NoneNotWrapped): + at unwrap_spec(subok=bool, copy=bool, ownmaskna=bool) +def array(space, w_item_or_iterable, w_dtype=None, w_order=None, + subok=True, copy=True, w_maskna=None, ownmaskna=False): # find scalar + if w_maskna is None: + w_maskna = space.w_None + if (not subok or not space.is_w(w_maskna, space.w_None) or + ownmaskna): + raise OperationError(space.w_NotImplementedError, space.wrap("Unsupported args")) if not space.issequence_w(w_item_or_iterable): - if space.is_w(w_dtype, space.w_None): + if w_dtype is None or space.is_w(w_dtype, space.w_None): w_dtype = interp_ufuncs.find_dtype_for_scalar(space, w_item_or_iterable) dtype = space.interp_w(interp_dtype.W_Dtype, space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype) ) return scalar_w(space, dtype, w_item_or_iterable) - if w_order is None: + if space.is_w(w_order, space.w_None) or w_order is None: order = 'C' else: order = space.str_w(w_order) if order != 'C': # or order != 'F': raise operationerrfmt(space.w_ValueError, "Unknown order: %s", order) - shape, elems_w = _find_shape_and_elems(space, w_item_or_iterable) + if isinstance(w_item_or_iterable, BaseArray): + if (not space.is_w(w_dtype, space.w_None) and + w_item_or_iterable.find_dtype() is not w_dtype): + raise OperationError(space.w_NotImplementedError, space.wrap( + "copying over different dtypes unsupported")) + if copy: + return w_item_or_iterable.copy(space) + return w_item_or_iterable + shape, elems_w = find_shape_and_elems(space, w_item_or_iterable) # they come back in C order size = len(elems_w) - if space.is_w(w_dtype, space.w_None): + if w_dtype is None or space.is_w(w_dtype, space.w_None): w_dtype = None for w_elem in elems_w: w_dtype = interp_ufuncs.find_dtype_for_scalar(space, w_elem, @@ -1322,6 +1154,8 @@ space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype) ) size, shape = _find_size_and_shape(space, w_size) + if not shape: + return scalar_w(space, dtype, space.wrap(0)) return space.wrap(W_NDimArray(size, shape[:], dtype=dtype)) def ones(space, w_size, w_dtype=None): @@ -1330,17 +1164,66 @@ ) size, shape = _find_size_and_shape(space, w_size) + if not shape: + return scalar_w(space, dtype, space.wrap(1)) arr = W_NDimArray(size, shape[:], dtype=dtype) one = dtype.box(1) arr.dtype.fill(arr.storage, one, 0, size) return space.wrap(arr) + at unwrap_spec(arr=BaseArray, skipna=bool, keepdims=bool) +def count_reduce_items(space, arr, w_axis=None, skipna=False, keepdims=True): + if not keepdims: + raise OperationError(space.w_NotImplementedError, space.wrap("unsupported")) + if space.is_w(w_axis, space.w_None): + return space.wrap(support.product(arr.shape)) + if space.isinstance_w(w_axis, space.w_int): + return space.wrap(arr.shape[space.int_w(w_axis)]) + s = 1 + elems = space.fixedview(w_axis) + for w_elem in elems: + s *= arr.shape[space.int_w(w_elem)] + return space.wrap(s) + def dot(space, w_obj, w_obj2): w_arr = convert_to_array(space, w_obj) if isinstance(w_arr, Scalar): return convert_to_array(space, w_obj2).descr_dot(space, w_arr) return w_arr.descr_dot(space, w_obj2) + at unwrap_spec(axis=int) +def concatenate(space, w_args, axis=0): + args_w = space.listview(w_args) + if len(args_w) == 0: + raise OperationError(space.w_ValueError, space.wrap("concatenation of zero-length sequences is impossible")) + args_w = [convert_to_array(space, w_arg) for w_arg in args_w] + dtype = args_w[0].find_dtype() + shape = args_w[0].shape[:] + if len(shape) <= axis: + raise OperationError(space.w_ValueError, + space.wrap("bad axis argument")) + for arr in args_w[1:]: + dtype = interp_ufuncs.find_binop_result_dtype(space, dtype, + arr.find_dtype()) + if len(arr.shape) <= axis: + raise OperationError(space.w_ValueError, + space.wrap("bad axis argument")) + for i, axis_size in enumerate(arr.shape): + if len(arr.shape) != len(shape) or (i != axis and axis_size != shape[i]): + raise OperationError(space.w_ValueError, space.wrap( + "array dimensions must agree except for axis being concatenated")) + elif i == axis: + shape[i] += axis_size + res = W_NDimArray(support.product(shape), shape, dtype, 'C') + chunks = [Chunk(0, i, 1, i) for i in shape] + axis_start = 0 + for arr in args_w: + chunks[axis] = Chunk(axis_start, axis_start + arr.shape[axis], 1, + arr.shape[axis]) + res.create_slice(chunks).setslice(space, arr) + axis_start += arr.shape[axis] + return res + BaseArray.typedef = TypeDef( 'ndarray', __module__ = "numpypy", @@ -1378,6 +1261,7 @@ __and__ = interp2app(BaseArray.descr_and), __or__ = interp2app(BaseArray.descr_or), + __invert__ = interp2app(BaseArray.descr_invert), __repr__ = interp2app(BaseArray.descr_repr), __str__ = interp2app(BaseArray.descr_str), @@ -1388,9 +1272,11 @@ BaseArray.descr_set_shape), size = GetSetProperty(BaseArray.descr_get_size), ndim = GetSetProperty(BaseArray.descr_get_ndim), + item = interp2app(BaseArray.descr_item), T = GetSetProperty(BaseArray.descr_get_transpose), flat = GetSetProperty(BaseArray.descr_get_flatiter), + ravel = interp2app(BaseArray.descr_ravel), mean = interp2app(BaseArray.descr_mean), sum = interp2app(BaseArray.descr_sum), @@ -1411,6 +1297,8 @@ flatten = interp2app(BaseArray.descr_flatten), reshape = interp2app(BaseArray.descr_reshape), tolist = interp2app(BaseArray.descr_tolist), + take = interp2app(BaseArray.descr_take), + compress = interp2app(BaseArray.descr_compress), ) @@ -1419,31 +1307,122 @@ @jit.unroll_safe def __init__(self, arr): arr = arr.get_concrete() - size = 1 - for sh in arr.shape: - size *= sh self.strides = [arr.strides[-1]] self.backstrides = [arr.backstrides[-1]] - ViewArray.__init__(self, size, [size], arr.dtype, arr.order, - arr) self.shapelen = len(arr.shape) - self.iter = OneDimIterator(arr.start, self.strides[0], - self.shape[0]) + sig = arr.find_sig() + self.iter = sig.create_frame(arr).get_final_iter() + self.base = arr + self.index = 0 + ViewArray.__init__(self, arr.size, [arr.size], arr.dtype, arr.order, + arr) def descr_next(self, space): if self.iter.done(): raise OperationError(space.w_StopIteration, space.w_None) - result = self.getitem(self.iter.offset) + result = self.base.getitem(self.iter.offset) self.iter = self.iter.next(self.shapelen) + self.index += 1 return result def descr_iter(self): return self + def descr_index(self, space): + return space.wrap(self.index) + + def descr_coords(self, space): + coords, step, lngth = to_coords(space, self.base.shape, + self.base.size, self.base.order, + space.wrap(self.index)) + return space.newtuple([space.wrap(c) for c in coords]) + + @jit.unroll_safe + def descr_getitem(self, space, w_idx): + if not (space.isinstance_w(w_idx, space.w_int) or + space.isinstance_w(w_idx, space.w_slice)): + raise OperationError(space.w_IndexError, + space.wrap('unsupported iterator index')) + base = self.base + start, stop, step, lngth = space.decode_index4(w_idx, base.size) + # setslice would have been better, but flat[u:v] for arbitrary + # shapes of array a cannot be represented as a[x1:x2, y1:y2] + basei = ViewIterator(base.start, base.strides, + base.backstrides,base.shape) + shapelen = len(base.shape) + basei = basei.next_skip_x(shapelen, start) + if lngth <2: + return base.getitem(basei.offset) + ri = ArrayIterator(lngth) + res = W_NDimArray(lngth, [lngth], base.dtype, + base.order) + while not ri.done(): + flat_get_driver.jit_merge_point(shapelen=shapelen, + base=base, + basei=basei, + step=step, + res=res, + ri=ri, + ) + w_val = base.getitem(basei.offset) + res.setitem(ri.offset,w_val) + basei = basei.next_skip_x(shapelen, step) + ri = ri.next(shapelen) + return res + + def descr_setitem(self, space, w_idx, w_value): + if not (space.isinstance_w(w_idx, space.w_int) or + space.isinstance_w(w_idx, space.w_slice)): + raise OperationError(space.w_IndexError, + space.wrap('unsupported iterator index')) + base = self.base + start, stop, step, lngth = space.decode_index4(w_idx, base.size) + arr = convert_to_array(space, w_value) + ai = 0 + basei = ViewIterator(base.start, base.strides, + base.backstrides,base.shape) + shapelen = len(base.shape) + basei = basei.next_skip_x(shapelen, start) + while lngth > 0: + flat_set_driver.jit_merge_point(shapelen=shapelen, + basei=basei, + base=base, + step=step, + arr=arr, + ai=ai, + lngth=lngth, + ) + v = arr.getitem(ai).convert_to(base.dtype) + base.setitem(basei.offset, v) + # need to repeat input values until all assignments are done + ai = (ai + 1) % arr.size + basei = basei.next_skip_x(shapelen, step) + lngth -= 1 + + def create_sig(self): + return signature.FlatSignature(self.base.dtype) + + def descr_base(self, space): + return space.wrap(self.base) + W_FlatIterator.typedef = TypeDef( 'flatiter', + #__array__ = #MISSING + __iter__ = interp2app(W_FlatIterator.descr_iter), + __getitem__ = interp2app(W_FlatIterator.descr_getitem), + __setitem__ = interp2app(W_FlatIterator.descr_setitem), + __eq__ = interp2app(BaseArray.descr_eq), + __ne__ = interp2app(BaseArray.descr_ne), + __lt__ = interp2app(BaseArray.descr_lt), + __le__ = interp2app(BaseArray.descr_le), + __gt__ = interp2app(BaseArray.descr_gt), + __ge__ = interp2app(BaseArray.descr_ge), + #__sizeof__ #MISSING + base = GetSetProperty(W_FlatIterator.descr_base), + index = GetSetProperty(W_FlatIterator.descr_index), + coords = GetSetProperty(W_FlatIterator.descr_coords), next = interp2app(W_FlatIterator.descr_next), - __iter__ = interp2app(W_FlatIterator.descr_iter), + ) W_FlatIterator.acceptable_as_base_class = False diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py --- a/pypy/module/micronumpy/interp_ufuncs.py +++ b/pypy/module/micronumpy/interp_ufuncs.py @@ -1,14 +1,15 @@ from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.error import OperationError, operationerrfmt -from pypy.interpreter.gateway import interp2app +from pypy.interpreter.gateway import interp2app, unwrap_spec, NoneNotWrapped from pypy.interpreter.typedef import TypeDef, GetSetProperty, interp_attrproperty -from pypy.module.micronumpy import interp_boxes, interp_dtype -from pypy.module.micronumpy.signature import ReduceSignature,\ - find_sig, new_printable_location, AxisReduceSignature, ScalarSignature +from pypy.module.micronumpy import interp_boxes, interp_dtype, support +from pypy.module.micronumpy.signature import (ReduceSignature, find_sig, + new_printable_location, AxisReduceSignature, ScalarSignature) from pypy.rlib import jit from pypy.rlib.rarithmetic import LONG_BIT from pypy.tool.sourcetools import func_with_new_name + reduce_driver = jit.JitDriver( greens=['shapelen', "sig"], virtualizables=["frame"], @@ -30,12 +31,14 @@ _attrs_ = ["name", "promote_to_float", "promote_bools", "identity"] _immutable_fields_ = ["promote_to_float", "promote_bools", "name"] - def __init__(self, name, promote_to_float, promote_bools, identity): + def __init__(self, name, promote_to_float, promote_bools, identity, + int_only): self.name = name self.promote_to_float = promote_to_float self.promote_bools = promote_bools self.identity = identity + self.int_only = int_only def descr_repr(self, space): return space.wrap("<ufunc '%s'>" % self.name) @@ -46,19 +49,31 @@ return self.identity def descr_call(self, space, __args__): - if __args__.keywords or len(__args__.arguments_w) < self.argcount: + args_w, kwds_w = __args__.unpack() + # it occurs to me that we don't support any datatypes that + # require casting, change it later when we do + kwds_w.pop('casting', None) + w_subok = kwds_w.pop('subok', None) + w_out = kwds_w.pop('out', space.w_None) + if ((w_subok is not None and space.is_true(w_subok)) or + not space.is_w(w_out, space.w_None)): + raise OperationError(space.w_NotImplementedError, + space.wrap("parameters unsupported")) + if kwds_w or len(args_w) < self.argcount: raise OperationError(space.w_ValueError, space.wrap("invalid number of arguments") ) - elif len(__args__.arguments_w) > self.argcount: + elif len(args_w) > self.argcount: # The extra arguments should actually be the output array, but we # don't support that yet. raise OperationError(space.w_TypeError, space.wrap("invalid number of arguments") ) - return self.call(space, __args__.arguments_w) + return self.call(space, args_w) - def descr_reduce(self, space, w_obj, w_dim=0): + @unwrap_spec(skipna=bool, keepdims=bool) + def descr_reduce(self, space, w_obj, w_axis=NoneNotWrapped, w_dtype=None, + skipna=False, keepdims=False, w_out=None): """reduce(...) reduce(a, axis=0) @@ -111,15 +126,24 @@ array([[ 1, 5], [ 9, 13]]) """ - return self.reduce(space, w_obj, False, False, w_dim) + if not space.is_w(w_out, space.w_None): + raise OperationError(space.w_NotImplementedError, space.wrap( + "out not supported")) + if w_axis is None: + axis = 0 + elif space.is_w(w_axis, space.w_None): + axis = -1 + else: + axis = space.int_w(w_axis) + return self.reduce(space, w_obj, False, False, axis, keepdims) - def reduce(self, space, w_obj, multidim, promote_to_largest, w_dim): + def reduce(self, space, w_obj, multidim, promote_to_largest, dim, + keepdims=False): from pypy.module.micronumpy.interp_numarray import convert_to_array, \ Scalar if self.argcount != 2: raise OperationError(space.w_ValueError, space.wrap("reduce only " "supported for binary functions")) - dim = space.int_w(w_dim) assert isinstance(self, W_Ufunc2) obj = convert_to_array(space, w_obj) if dim >= len(obj.shape): @@ -140,7 +164,7 @@ raise operationerrfmt(space.w_ValueError, "zero-size array to " "%s.reduce without identity", self.name) if shapelen > 1 and dim >= 0: - res = self.do_axis_reduce(obj, dtype, dim) + res = self.do_axis_reduce(obj, dtype, dim, keepdims) return space.wrap(res) scalarsig = ScalarSignature(dtype) sig = find_sig(ReduceSignature(self.func, self.name, dtype, @@ -154,15 +178,15 @@ value = self.identity.convert_to(dtype) return self.reduce_loop(shapelen, sig, frame, value, obj, dtype) - def do_axis_reduce(self, obj, dtype, dim): + def do_axis_reduce(self, obj, dtype, dim, keepdims): from pypy.module.micronumpy.interp_numarray import AxisReduce,\ W_NDimArray - - shape = obj.shape[0:dim] + obj.shape[dim + 1:len(obj.shape)] - size = 1 - for s in shape: - size *= s - result = W_NDimArray(size, shape, dtype) + + if keepdims: + shape = obj.shape[:dim] + [1] + obj.shape[dim + 1:] + else: + shape = obj.shape[:dim] + obj.shape[dim + 1:] + result = W_NDimArray(support.product(shape), shape, dtype) rightsig = obj.create_sig() # note - this is just a wrapper so signature can fetch # both left and right, nothing more, especially @@ -224,10 +248,12 @@ _immutable_fields_ = ["func", "name"] def __init__(self, func, name, promote_to_float=False, promote_bools=False, - identity=None): + identity=None, bool_result=False, int_only=False): - W_Ufunc.__init__(self, name, promote_to_float, promote_bools, identity) + W_Ufunc.__init__(self, name, promote_to_float, promote_bools, identity, + int_only) self.func = func + self.bool_result = bool_result def call(self, space, args_w): from pypy.module.micronumpy.interp_numarray import (Call1, @@ -235,15 +261,19 @@ [w_obj] = args_w w_obj = convert_to_array(space, w_obj) - res_dtype = find_unaryop_result_dtype(space, - w_obj.find_dtype(), - promote_to_float=self.promote_to_float, - promote_bools=self.promote_bools, - ) + calc_dtype = find_unaryop_result_dtype(space, + w_obj.find_dtype(), + promote_to_float=self.promote_to_float, + promote_bools=self.promote_bools) + if self.bool_result: + res_dtype = interp_dtype.get_dtype_cache(space).w_booldtype + else: + res_dtype = calc_dtype if isinstance(w_obj, Scalar): - return self.func(res_dtype, w_obj.value.convert_to(res_dtype)) + return space.wrap(self.func(calc_dtype, w_obj.value.convert_to(calc_dtype))) - w_res = Call1(self.func, self.name, w_obj.shape, res_dtype, w_obj) + w_res = Call1(self.func, self.name, w_obj.shape, calc_dtype, res_dtype, + w_obj) w_obj.add_invalidates(w_res) return w_res @@ -255,10 +285,10 @@ def __init__(self, func, name, promote_to_float=False, promote_bools=False, identity=None, comparison_func=False, int_only=False): - W_Ufunc.__init__(self, name, promote_to_float, promote_bools, identity) + W_Ufunc.__init__(self, name, promote_to_float, promote_bools, identity, + int_only) self.func = func self.comparison_func = comparison_func - self.int_only = int_only def call(self, space, args_w): from pypy.module.micronumpy.interp_numarray import (Call2, @@ -278,10 +308,10 @@ else: res_dtype = calc_dtype if isinstance(w_lhs, Scalar) and isinstance(w_rhs, Scalar): - return self.func(calc_dtype, + return space.wrap(self.func(calc_dtype, w_lhs.value.convert_to(calc_dtype), w_rhs.value.convert_to(calc_dtype) - ) + )) new_shape = shape_agreement(space, w_lhs.shape, w_rhs.shape) w_res = Call2(self.func, self.name, @@ -409,12 +439,16 @@ return interp_dtype.get_dtype_cache(space).w_float64dtype -def ufunc_dtype_caller(space, ufunc_name, op_name, argcount, comparison_func): +def ufunc_dtype_caller(space, ufunc_name, op_name, argcount, comparison_func, + bool_result): + dtype_cache = interp_dtype.get_dtype_cache(space) if argcount == 1: def impl(res_dtype, value): - return getattr(res_dtype.itemtype, op_name)(value) + res = getattr(res_dtype.itemtype, op_name)(value) + if bool_result: + return dtype_cache.w_booldtype.box(res) + return res elif argcount == 2: - dtype_cache = interp_dtype.get_dtype_cache(space) def impl(res_dtype, lvalue, rvalue): res = getattr(res_dtype.itemtype, op_name)(lvalue, rvalue) if comparison_func: @@ -433,7 +467,9 @@ 'int_only': True}), ("bitwise_or", "bitwise_or", 2, {"identity": 0, 'int_only': True}), + ("invert", "invert", 1, {"int_only": True}), ("divide", "div", 2, {"promote_bools": True}), + ("true_divide", "div", 2, {"promote_to_float": True}), ("mod", "mod", 2, {"promote_bools": True}), ("power", "pow", 2, {"promote_bools": True}), @@ -443,6 +479,8 @@ ("less_equal", "le", 2, {"comparison_func": True}), ("greater", "gt", 2, {"comparison_func": True}), ("greater_equal", "ge", 2, {"comparison_func": True}), + ("isnan", "isnan", 1, {"bool_result": True}), + ("isinf", "isinf", 1, {"bool_result": True}), ("maximum", "max", 2), ("minimum", "min", 2), @@ -485,6 +523,7 @@ func = ufunc_dtype_caller(space, ufunc_name, op_name, argcount, comparison_func=extra_kwargs.get("comparison_func", False), + bool_result=extra_kwargs.get("bool_result", False), ) if argcount == 1: ufunc = W_Ufunc1(func, ufunc_name, **extra_kwargs) @@ -494,3 +533,4 @@ def get(space): return space.fromcache(UfuncState) + diff --git a/pypy/module/micronumpy/signature.py b/pypy/module/micronumpy/signature.py --- a/pypy/module/micronumpy/signature.py +++ b/pypy/module/micronumpy/signature.py @@ -224,6 +224,18 @@ return ViewIterator(arr.start, arr.strides, arr.backstrides, arr.shape).apply_transformations(arr, transforms) +class FlatSignature(ViewSignature): + def debug_repr(self): + return 'Flat' + + def allocate_iter(self, arr, transforms): + from pypy.module.micronumpy.interp_numarray import W_FlatIterator + assert isinstance(arr, W_FlatIterator) + return ViewIterator(arr.base.start, arr.base.strides, + arr.base.backstrides, + arr.base.shape).apply_transformations(arr.base, + transforms) + class VirtualSliceSignature(Signature): def __init__(self, child): self.child = child @@ -293,8 +305,8 @@ def eval(self, frame, arr): from pypy.module.micronumpy.interp_numarray import Call1 assert isinstance(arr, Call1) - v = self.child.eval(frame, arr.values).convert_to(arr.res_dtype) - return self.unfunc(arr.res_dtype, v) + v = self.child.eval(frame, arr.values).convert_to(arr.calc_dtype) + return self.unfunc(arr.calc_dtype, v) class Call2(Signature): _immutable_fields_ = ['binfunc', 'name', 'calc_dtype', 'left', 'right'] diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py --- a/pypy/module/micronumpy/strides.py +++ b/pypy/module/micronumpy/strides.py @@ -1,5 +1,5 @@ from pypy.rlib import jit - +from pypy.interpreter.error import OperationError @jit.look_inside_iff(lambda shape, start, strides, backstrides, chunks: jit.isconstant(len(chunks)) @@ -37,3 +37,196 @@ rstrides = [0] * (len(res_shape) - len(orig_shape)) + rstrides rbackstrides = [0] * (len(res_shape) - len(orig_shape)) + rbackstrides return rstrides, rbackstrides + +def find_shape_and_elems(space, w_iterable): + shape = [space.len_w(w_iterable)] + batch = space.listview(w_iterable) + while True: + new_batch = [] + if not batch: + return shape, [] + if not space.issequence_w(batch[0]): + for elem in batch: + if space.issequence_w(elem): + raise OperationError(space.w_ValueError, space.wrap( + "setting an array element with a sequence")) + return shape, batch + size = space.len_w(batch[0]) + for w_elem in batch: + if not space.issequence_w(w_elem) or space.len_w(w_elem) != size: + raise OperationError(space.w_ValueError, space.wrap( + "setting an array element with a sequence")) + new_batch += space.listview(w_elem) + shape.append(size) + batch = new_batch + +def to_coords(space, shape, size, order, w_item_or_slice): + '''Returns a start coord, step, and length. + ''' + start = lngth = step = 0 + if not (space.isinstance_w(w_item_or_slice, space.w_int) or + space.isinstance_w(w_item_or_slice, space.w_slice)): + raise OperationError(space.w_IndexError, + space.wrap('unsupported iterator index')) + + start, stop, step, lngth = space.decode_index4(w_item_or_slice, size) + + coords = [0] * len(shape) + i = start + if order == 'C': + for s in range(len(shape) -1, -1, -1): + coords[s] = i % shape[s] + i //= shape[s] + else: + for s in range(len(shape)): + coords[s] = i % shape[s] + i //= shape[s] + return coords, step, lngth + +def shape_agreement(space, shape1, shape2): + ret = _shape_agreement(shape1, shape2) + if len(ret) < max(len(shape1), len(shape2)): + raise OperationError(space.w_ValueError, + space.wrap("operands could not be broadcast together with shapes (%s) (%s)" % ( + ",".join([str(x) for x in shape1]), + ",".join([str(x) for x in shape2]), + )) + ) + return ret + +def _shape_agreement(shape1, shape2): + """ Checks agreement about two shapes with respect to broadcasting. Returns + the resulting shape. + """ + lshift = 0 + rshift = 0 + if len(shape1) > len(shape2): + m = len(shape1) + n = len(shape2) + rshift = len(shape2) - len(shape1) + remainder = shape1 + else: + m = len(shape2) + n = len(shape1) + lshift = len(shape1) - len(shape2) + remainder = shape2 + endshape = [0] * m + indices1 = [True] * m + indices2 = [True] * m + for i in range(m - 1, m - n - 1, -1): + left = shape1[i + lshift] + right = shape2[i + rshift] + if left == right: + endshape[i] = left + elif left == 1: + endshape[i] = right + indices1[i + lshift] = False + elif right == 1: + endshape[i] = left + indices2[i + rshift] = False + else: + return [] + #raise OperationError(space.w_ValueError, space.wrap( + # "frames are not aligned")) + for i in range(m - n): + endshape[i] = remainder[i] + return endshape + +def get_shape_from_iterable(space, old_size, w_iterable): + new_size = 0 + new_shape = [] + if space.isinstance_w(w_iterable, space.w_int): + new_size = space.int_w(w_iterable) + if new_size < 0: + new_size = old_size + new_shape = [new_size] + else: + neg_dim = -1 + batch = space.listview(w_iterable) + new_size = 1 + if len(batch) < 1: + if old_size == 1: + # Scalars can have an empty size. + new_size = 1 + else: + new_size = 0 + new_shape = [] + i = 0 + for elem in batch: + s = space.int_w(elem) + if s < 0: + if neg_dim >= 0: + raise OperationError(space.w_ValueError, space.wrap( + "can only specify one unknown dimension")) + s = 1 + neg_dim = i + new_size *= s + new_shape.append(s) + i += 1 + if neg_dim >= 0: + new_shape[neg_dim] = old_size / new_size + new_size *= new_shape[neg_dim] + if new_size != old_size: + raise OperationError(space.w_ValueError, + space.wrap("total size of new array must be unchanged")) + return new_shape + +# Recalculating strides. Find the steps that the iteration does for each +# dimension, given the stride and shape. Then try to create a new stride that +# fits the new shape, using those steps. If there is a shape/step mismatch +# (meaning that the realignment of elements crosses from one step into another) +# return None so that the caller can raise an exception. +def calc_new_strides(new_shape, old_shape, old_strides, order): + # Return the proper strides for new_shape, or None if the mapping crosses + # stepping boundaries + + # Assumes that prod(old_shape) == prod(new_shape), len(old_shape) > 1, and + # len(new_shape) > 0 + steps = [] + last_step = 1 + oldI = 0 + new_strides = [] + if order == 'F': + for i in range(len(old_shape)): + steps.append(old_strides[i] / last_step) + last_step *= old_shape[i] + cur_step = steps[0] + n_new_elems_used = 1 + n_old_elems_to_use = old_shape[0] + for s in new_shape: + new_strides.append(cur_step * n_new_elems_used) + n_new_elems_used *= s + while n_new_elems_used > n_old_elems_to_use: + oldI += 1 + if steps[oldI] != steps[oldI - 1]: + return None + n_old_elems_to_use *= old_shape[oldI] + if n_new_elems_used == n_old_elems_to_use: + oldI += 1 + if oldI < len(old_shape): + cur_step = steps[oldI] + n_old_elems_to_use *= old_shape[oldI] + elif order == 'C': + for i in range(len(old_shape) - 1, -1, -1): + steps.insert(0, old_strides[i] / last_step) + last_step *= old_shape[i] + cur_step = steps[-1] + n_new_elems_used = 1 + oldI = -1 + n_old_elems_to_use = old_shape[-1] + for i in range(len(new_shape) - 1, -1, -1): + s = new_shape[i] + new_strides.insert(0, cur_step * n_new_elems_used) + n_new_elems_used *= s + while n_new_elems_used > n_old_elems_to_use: + oldI -= 1 + if steps[oldI] != steps[oldI + 1]: + return None + n_old_elems_to_use *= old_shape[oldI] + if n_new_elems_used == n_old_elems_to_use: + oldI -= 1 + if oldI >= -len(old_shape): + cur_step = steps[oldI] + n_old_elems_to_use *= old_shape[oldI] + assert len(new_strides) == len(new_shape) + return new_strides diff --git a/pypy/module/micronumpy/support.py b/pypy/module/micronumpy/support.py new file mode 100644 --- /dev/null +++ b/pypy/module/micronumpy/support.py @@ -0,0 +1,5 @@ +def product(s): + i = 1 + for x in s: + i *= x + return i \ No newline at end of file diff --git a/pypy/module/micronumpy/test/test_base.py b/pypy/module/micronumpy/test/test_base.py --- a/pypy/module/micronumpy/test/test_base.py +++ b/pypy/module/micronumpy/test/test_base.py @@ -3,9 +3,17 @@ from pypy.module.micronumpy.interp_numarray import W_NDimArray, Scalar from pypy.module.micronumpy.interp_ufuncs import (find_binop_result_dtype, find_unaryop_result_dtype) +from pypy.module.micronumpy.interp_boxes import W_Float64Box +from pypy.conftest import option +import sys class BaseNumpyAppTest(object): def setup_class(cls): + if option.runappdirect: + if '__pypy__' not in sys.builtin_module_names: + import numpy + sys.modules['numpypy'] = numpy + sys.modules['_numpypy'] = numpy cls.space = gettestobjspace(usemodules=['micronumpy']) class TestSignature(object): @@ -16,7 +24,7 @@ ar = W_NDimArray(10, [10], dtype=float64_dtype) ar2 = W_NDimArray(10, [10], dtype=float64_dtype) v1 = ar.descr_add(space, ar) - v2 = ar.descr_add(space, Scalar(float64_dtype, 2.0)) + v2 = ar.descr_add(space, Scalar(float64_dtype, W_Float64Box(2.0))) sig1 = v1.find_sig() sig2 = v2.find_sig() assert v1 is not v2 @@ -26,7 +34,7 @@ sig1b = ar2.descr_add(space, ar).find_sig() assert sig1b.left.array_no != sig1b.right.array_no assert sig1b is not sig1 - v3 = ar.descr_add(space, Scalar(float64_dtype, 1.0)) + v3 = ar.descr_add(space, Scalar(float64_dtype, W_Float64Box(1.0))) sig3 = v3.find_sig() assert sig2 is sig3 v4 = ar.descr_add(space, ar) diff --git a/pypy/module/micronumpy/test/test_compile.py b/pypy/module/micronumpy/test/test_compile.py --- a/pypy/module/micronumpy/test/test_compile.py +++ b/pypy/module/micronumpy/test/test_compile.py @@ -245,3 +245,19 @@ a -> 3 """) assert interp.results[0].value == 11 + + def test_flat_iter(self): + interp = self.run(''' + a = |30| + b = flat(a) + b -> 3 + ''') + assert interp.results[0].value == 3 + + def test_take(self): + interp = self.run(""" + a = |10| + b = take(a, [1, 1, 3, 2]) + b -> 2 + """) + assert interp.results[0].value == 3 diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py --- a/pypy/module/micronumpy/test/test_dtypes.py +++ b/pypy/module/micronumpy/test/test_dtypes.py @@ -11,8 +11,17 @@ assert dtype('int8').num == 1 assert dtype(d) is d assert dtype(None) is dtype(float) + assert dtype('int8').name == 'int8' raises(TypeError, dtype, 1042) + def test_dtype_eq(self): + from _numpypy import dtype + + assert dtype("int8") == "int8" + assert "int8" == dtype("int8") + raises(TypeError, lambda: dtype("int8") == 3) + assert dtype(bool) == bool + def test_dtype_with_types(self): from _numpypy import dtype @@ -30,7 +39,7 @@ def test_repr_str(self): from _numpypy import dtype - assert repr(dtype) == "<type 'numpypy.dtype'>" + assert '.dtype' in repr(dtype) d = dtype('?') assert repr(d) == "dtype('bool')" assert str(d) == "bool" @@ -376,3 +385,19 @@ b = X(10) assert type(b) is X assert b.m() == 12 + + def test_long_as_index(self): + skip("waiting for removal of multimethods of __index__") + from _numpypy import int_ + assert (1, 2, 3)[int_(1)] == 2 + + def test_int(self): + import sys + from _numpypy import int32, int64, int_ + assert issubclass(int_, int) + if sys.maxint == (1<<31) - 1: + assert issubclass(int32, int) + assert int_ is int32 + else: + assert issubclass(int64, int) + assert int_ is int64 diff --git a/pypy/module/micronumpy/test/test_iter.py b/pypy/module/micronumpy/test/test_iter.py new file mode 100644 --- /dev/null +++ b/pypy/module/micronumpy/test/test_iter.py @@ -0,0 +1,88 @@ +from pypy.module.micronumpy.interp_iter import ViewIterator + +class TestIterDirect(object): + def test_C_viewiterator(self): + #Let's get started, simple iteration in C order with + #contiguous layout => strides[-1] is 1 + start = 0 + shape = [3, 5] + strides = [5, 1] + backstrides = [x * (y - 1) for x,y in zip(strides, shape)] + assert backstrides == [10, 4] + i = ViewIterator(start, strides, backstrides, shape) + i = i.next(2) + i = i.next(2) + i = i.next(2) + assert i.offset == 3 + assert not i.done() + assert i.indices == [0,3] + #cause a dimension overflow + i = i.next(2) + i = i.next(2) + assert i.offset == 5 + assert i.indices == [1,0] + + #Now what happens if the array is transposed? strides[-1] != 1 + # therefore layout is non-contiguous + strides = [1, 3] + backstrides = [x * (y - 1) for x,y in zip(strides, shape)] + assert backstrides == [2, 12] + i = ViewIterator(start, strides, backstrides, shape) + i = i.next(2) + i = i.next(2) + i = i.next(2) + assert i.offset == 9 + assert not i.done() + assert i.indices == [0,3] + #cause a dimension overflow + i = i.next(2) + i = i.next(2) + assert i.offset == 1 + assert i.indices == [1,0] + + def test_C_viewiterator_step(self): + #iteration in C order with #contiguous layout => strides[-1] is 1 + #skip less than the shape + start = 0 + shape = [3, 5] + strides = [5, 1] + backstrides = [x * (y - 1) for x,y in zip(strides, shape)] + assert backstrides == [10, 4] + i = ViewIterator(start, strides, backstrides, shape) + i = i.next_skip_x(2,2) + i = i.next_skip_x(2,2) + i = i.next_skip_x(2,2) + assert i.offset == 6 + assert not i.done() + assert i.indices == [1,1] + #And for some big skips + i = i.next_skip_x(2,5) + assert i.offset == 11 + assert i.indices == [2,1] + i = i.next_skip_x(2,5) + # Note: the offset does not overflow but recycles, + # this is good for broadcast + assert i.offset == 1 + assert i.indices == [0,1] + assert i.done() + + #Now what happens if the array is transposed? strides[-1] != 1 + # therefore layout is non-contiguous + strides = [1, 3] + backstrides = [x * (y - 1) for x,y in zip(strides, shape)] + assert backstrides == [2, 12] + i = ViewIterator(start, strides, backstrides, shape) + i = i.next_skip_x(2,2) + i = i.next_skip_x(2,2) + i = i.next_skip_x(2,2) + assert i.offset == 4 + assert i.indices == [1,1] + assert not i.done() + i = i.next_skip_x(2,5) + assert i.offset == 5 + assert i.indices == [2,1] + assert not i.done() + i = i.next_skip_x(2,5) + assert i.indices == [0,1] + assert i.offset == 3 + assert i.done() diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -167,6 +167,23 @@ assert calc_new_strides([1, 1, 105, 1, 1], [7, 15], [1, 7],'F') == \ [1, 1, 1, 105, 105] + def test_to_coords(self): + from pypy.module.micronumpy.strides import to_coords + + def _to_coords(index, order): + return to_coords(self.space, [2, 3, 4], 24, order, + self.space.wrap(index))[0] + + assert _to_coords(0, 'C') == [0, 0, 0] + assert _to_coords(1, 'C') == [0, 0, 1] + assert _to_coords(-1, 'C') == [1, 2, 3] + assert _to_coords(5, 'C') == [0, 1, 1] + assert _to_coords(13, 'C') == [1, 0, 1] + assert _to_coords(0, 'F') == [0, 0, 0] + assert _to_coords(1, 'F') == [1, 0, 0] + assert _to_coords(-1, 'F') == [1, 2, 3] + assert _to_coords(5, 'F') == [1, 2, 0] + assert _to_coords(13, 'F') == [1, 0, 2] class AppTestNumArray(BaseNumpyAppTest): def test_ndarray(self): @@ -211,6 +228,7 @@ # And check that changes stick. a[13] = 5.3 assert a[13] == 5.3 + assert zeros(()).shape == () def test_size(self): from _numpypy import array @@ -254,6 +272,7 @@ b = a[::2] c = b.copy() assert (c == b).all() + assert ((a + a).copy() == (a + a)).all() a = arange(15).reshape(5,3) b = a.copy() @@ -283,6 +302,12 @@ for i in xrange(5): assert a[i] == b[i] + def test_getitem_nd(self): + from _numpypy import arange + a = arange(15).reshape(3, 5) + assert a[1, 3] == 8 + assert a.T[1, 2] == 11 + def test_setitem(self): from _numpypy import array a = array(range(5)) @@ -370,6 +395,7 @@ assert b.shape == (5,) c = a[:3] assert c.shape == (3,) + assert array([]).shape == (0,) def test_set_shape(self): from _numpypy import array, zeros @@ -405,6 +431,7 @@ assert (a == [1000, 1, 2, 3, 1000, 5, 6, 7, 1000, 9, 10, 11]).all() a = zeros((4, 2, 3)) a.shape = (12, 2) + (a + a).reshape(2, 12) # assert did not explode def test_slice_reshape(self): from _numpypy import zeros, arange @@ -448,6 +475,13 @@ y = z.reshape(4, 3, 8) assert y.shape == (4, 3, 8) + def test_scalar_reshape(self): + from numpypy import array + a = array(3) + assert a.reshape([1, 1]).shape == (1, 1) + assert a.reshape([1]).shape == (1,) + raises(ValueError, "a.reshape(3)") + def test_add(self): from _numpypy import array a = array(range(5)) @@ -956,7 +990,7 @@ assert debug_repr(a + a) == 'Call2(add, Array, Array)' assert debug_repr(a[::2]) == 'Slice' assert debug_repr(a + 2) == 'Call2(add, Array, Scalar)' - assert debug_repr(a + a.flat) == 'Call2(add, Array, Slice)' + assert debug_repr(a + a.flat) == 'Call2(add, Array, Flat)' assert debug_repr(sin(a)) == 'Call1(sin, Array)' b = a + a @@ -1036,8 +1070,42 @@ assert a.var() == 0.0 a = arange(10).reshape(5, 2) assert a.var() == 8.25 - #assert (a.var(0) == [8, 8]).all() - #assert (a.var(1) == [.25] * 5).all() + assert (a.var(0) == [8, 8]).all() + assert (a.var(1) == [.25] * 5).all() + + def test_concatenate(self): + from numpypy import array, concatenate, dtype + a1 = array([0,1,2]) + a2 = array([3,4,5]) + a = concatenate((a1, a2)) + assert len(a) == 6 + assert (a == [0,1,2,3,4,5]).all() + assert a.dtype is dtype(int) + b1 = array([[1, 2], [3, 4]]) + b2 = array([[5, 6]]) + b = concatenate((b1, b2), axis=0) + assert (b == [[1, 2],[3, 4],[5, 6]]).all() + c = concatenate((b1, b2.T), axis=1) + assert (c == [[1, 2, 5],[3, 4, 6]]).all() + d = concatenate(([0],[1])) + assert (d == [0,1]).all() + e1 = array([[0,1],[2,3]]) + e = concatenate(e1) + assert (e == [0,1,2,3]).all() + f1 = array([0,1]) + f = concatenate((f1, [2], f1, [7])) + assert (f == [0,1,2,0,1,7]).all() + + bad_axis = raises(ValueError, concatenate, (a1,a2), axis=1) + assert str(bad_axis.value) == "bad axis argument" + + concat_zero = raises(ValueError, concatenate, ()) + assert str(concat_zero.value) == \ + "concatenation of zero-length sequences is impossible" + + dims_disagree = raises(ValueError, concatenate, (a1, b1), axis=0) + assert str(dims_disagree.value) == \ + "array dimensions must agree except for axis being concatenated" def test_std(self): from _numpypy import array @@ -1049,6 +1117,13 @@ def test_flatten(self): from _numpypy import array + assert array(3).flatten().shape == (1,) + a = array([[1, 2], [3, 4]]) + b = a.flatten() + c = a.ravel() + a[0, 0] = 15 + assert b[0] == 1 + assert c[0] == 15 a = array([[1, 2, 3], [4, 5, 6]]) assert (a.flatten() == [1, 2, 3, 4, 5, 6]).all() a = array([[[1, 2], [3, 4]], [[5, 6], [7, 8]]]) @@ -1062,7 +1137,6 @@ a = array([[1, 2], [3, 4]]) assert (a.T.flatten() == [1, 3, 2, 4]).all() - class AppTestMultiDim(BaseNumpyAppTest): def test_init(self): import _numpypy @@ -1077,7 +1151,7 @@ assert _numpypy.array([[1], [2], [3]]).shape == (3, 1) assert len(_numpypy.zeros((3, 1, 2))) == 3 raises(TypeError, len, _numpypy.zeros(())) - raises(ValueError, _numpypy.array, [[1, 2], 3]) + raises(ValueError, _numpypy.array, [[1, 2], 3], dtype=float) def test_getsetitem(self): import _numpypy @@ -1292,7 +1366,7 @@ assert(b[:, 0] == a[0, :]).all() def test_flatiter(self): - from _numpypy import array, flatiter + from _numpypy import array, flatiter, arange a = array([[10, 30], [40, 60]]) f_iter = a.flat assert f_iter.next() == 10 @@ -1305,6 +1379,9 @@ for k in a.flat: s += k assert s == 140 + a = arange(10).reshape(5, 2) + raises(IndexError, 'a.flat[(1, 2)]') + assert a.flat.base is a def test_flatiter_array_conv(self): from _numpypy import array, dot @@ -1316,6 +1393,75 @@ a = ones((2, 2)) assert list(((a + a).flat)) == [2, 2, 2, 2] + def test_flatiter_getitem(self): + from _numpypy import arange + a = arange(10) + assert a.flat[3] == 3 + assert a[2:].flat[3] == 5 + assert (a + a).flat[3] == 6 + assert a[::2].flat[3] == 6 + assert a.reshape(2,5).flat[3] == 3 + b = a.reshape(2,5).flat + b.next() + b.next() + b.next() + assert b[3] == 3 + assert (b[::3] == [0, 3, 6, 9]).all() + assert (b[2::5] == [2, 7]).all() + assert b[-2] == 8 + raises(IndexError, "b[11]") + raises(IndexError, "b[-11]") + raises(IndexError, 'b[0, 1]') + assert b.index == 3 + assert b.coords == (0,3) + + def test_flatiter_setitem(self): + from _numpypy import arange, array + a = arange(12).reshape(3,4) + b = a.T.flat + b[6::2] = [-1, -2] + assert (a == [[0, 1, -1, 3], [4, 5, 6, -1], [8, 9, -2, 11]]).all() + b[0:2] = [[[100]]] + assert(a[0,0] == 100) + assert(a[1,0] == 100) + raises(IndexError, 'b[array([10, 11])] == [-20, -40]') + + def test_flatiter_ops(self): + from _numpypy import arange, array + a = arange(12).reshape(3,4) + b = a.T.flat + assert (b == [0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11]).all() + assert not (b != [0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11]).any() + assert ((b >= range(12)) == [True, True, True,False, True, True, + False, False, True, False, False, True]).all() + assert ((b < range(12)) != [True, True, True,False, True, True, + False, False, True, False, False, True]).all() + assert ((b <= range(12)) != [False, True, True,False, True, True, + False, False, True, False, False, False]).all() + assert ((b > range(12)) == [False, True, True,False, True, True, + False, False, True, False, False, False]).all() + def test_flatiter_view(self): + from _numpypy import arange + a = arange(10).reshape(5, 2) + #no == yet. + # a[::2].flat == [0, 1, 4, 5, 8, 9] + isequal = True + for y,z in zip(a[::2].flat, [0, 1, 4, 5, 8, 9]): + if y != z: + isequal = False + assert isequal == True + + def test_flatiter_transpose(self): + from _numpypy import arange + a = arange(10).reshape(2,5).T + b = a.flat + assert (b[:5] == [0, 5, 1, 6, 2]).all() + b.next() + b.next() + b.next() + assert b.index == 3 + assert b.coords == (1,1) + def test_slice_copy(self): from _numpypy import zeros a = zeros((10, 10)) @@ -1381,12 +1527,61 @@ a[a & 1 == 1] = array([8, 9, 10]) assert (a == [[0, 8], [2, 9], [4, 10]]).all() + def test_copy_kwarg(self): + from _numpypy import array + x = array([1, 2, 3]) + assert (array(x) == x).all() + assert array(x) is not x + assert array(x, copy=False) is x + assert array(x, copy=True) is not x + def test_isna(self): from _numpypy import isna, array # XXX for now assert not isna(3) assert (isna(array([1, 2, 3, 4])) == [False, False, False, False]).all() + def test_ravel(self): + from _numpypy import arange + assert (arange(3).ravel() == arange(3)).all() + assert (arange(6).reshape(2, 3).ravel() == arange(6)).all() + assert (arange(6).reshape(2, 3).T.ravel() == [0, 3, 1, 4, 2, 5]).all() + + def test_take(self): + from _numpypy import arange + assert (arange(10).take([1, 2, 1, 1]) == [1, 2, 1, 1]).all() + raises(IndexError, "arange(3).take([15])") + a = arange(6).reshape(2, 3) + assert (a.take([1, 0, 3]) == [1, 0, 3]).all() + assert ((a + a).take([3]) == [6]).all() + a = arange(12).reshape(2, 6) + assert (a[:,::2].take([3, 2, 1]) == [6, 4, 2]).all() + + def test_compress(self): + from _numpypy import arange + a = arange(10) + assert (a.compress([True, False, True]) == [0, 2]).all() + assert (a.compress([1, 0, 13]) == [0, 2]).all() + assert (a.compress([1, 0, 13.5]) == [0, 2]).all() + a = arange(10).reshape(2, 5) + assert (a.compress([True, False, True]) == [0, 2]).all() + raises(IndexError, "a.compress([1] * 100)") + + def test_item(self): + from _numpypy import array + assert array(3).item() == 3 + assert type(array(3).item()) is int + assert type(array(True).item()) is bool + assert type(array(3.5).item()) is float + raises((ValueError, IndexError), "array(3).item(15)") + raises(ValueError, "array([1, 2, 3]).item()") + assert array([3]).item(0) == 3 + assert type(array([3]).item(0)) is int + assert array([1, 2, 3]).item(-1) == 3 + a = array([1, 2, 3]) + assert a[::2].item(1) == 3 + assert (a + a).item(1) == 4 + raises(ValueError, "array(5).item(1)") class AppTestSupport(BaseNumpyAppTest): def setup_class(cls): @@ -1498,128 +1693,6 @@ raises(ValueError, fromstring, "\x01\x02\x03", count=5, dtype=uint8) -class AppTestRepr(BaseNumpyAppTest): - def test_repr(self): - from _numpypy import array, zeros - int_size = array(5).dtype.itemsize - a = array(range(5), float) - assert repr(a) == "array([0.0, 1.0, 2.0, 3.0, 4.0])" - a = array([], float) - assert repr(a) == "array([], dtype=float64)" - a = zeros(1001) - assert repr(a) == "array([0.0, 0.0, 0.0, ..., 0.0, 0.0, 0.0])" - a = array(range(5), long) - if a.dtype.itemsize == int_size: - assert repr(a) == "array([0, 1, 2, 3, 4])" - else: - assert repr(a) == "array([0, 1, 2, 3, 4], dtype=int64)" - a = array(range(5), 'int32') - if a.dtype.itemsize == int_size: - assert repr(a) == "array([0, 1, 2, 3, 4])" - else: - assert repr(a) == "array([0, 1, 2, 3, 4], dtype=int32)" - a = array([], long) - assert repr(a) == "array([], dtype=int64)" - a = array([True, False, True, False], "?") - assert repr(a) == "array([True, False, True, False], dtype=bool)" - a = zeros([]) - assert repr(a) == "array(0.0)" - a = array(0.2) - assert repr(a) == "array(0.2)" - a = array([2]) - assert repr(a) == "array([2])" - - def test_repr_multi(self): - from _numpypy import arange, zeros, array - a = zeros((3, 4)) - assert repr(a) == '''array([[0.0, 0.0, 0.0, 0.0], - [0.0, 0.0, 0.0, 0.0], - [0.0, 0.0, 0.0, 0.0]])''' - a = zeros((2, 3, 4)) - assert repr(a) == '''array([[[0.0, 0.0, 0.0, 0.0], - [0.0, 0.0, 0.0, 0.0], - [0.0, 0.0, 0.0, 0.0]], - - [[0.0, 0.0, 0.0, 0.0], - [0.0, 0.0, 0.0, 0.0], - [0.0, 0.0, 0.0, 0.0]]])''' - a = arange(1002).reshape((2, 501)) - assert repr(a) == '''array([[0, 1, 2, ..., 498, 499, 500], - [501, 502, 503, ..., 999, 1000, 1001]])''' - assert repr(a.T) == '''array([[0, 501], - [1, 502], - [2, 503], - ..., - [498, 999], - [499, 1000], - [500, 1001]])''' - a = arange(2).reshape((2,1)) - assert repr(a) == '''array([[0], - [1]])''' - - def test_repr_slice(self): - from _numpypy import array, zeros - a = array(range(5), float) - b = a[1::2] - assert repr(b) == "array([1.0, 3.0])" - a = zeros(2002) - b = a[::2] - assert repr(b) == "array([0.0, 0.0, 0.0, ..., 0.0, 0.0, 0.0])" - a = array((range(5), range(5, 10)), dtype="int16") - b = a[1, 2:] - assert repr(b) == "array([7, 8, 9], dtype=int16)" - # an empty slice prints its shape - b = a[2:1, ] - assert repr(b) == "array([], shape=(0, 5), dtype=int16)" - - def test_str(self): - from _numpypy import array, zeros - a = array(range(5), float) - assert str(a) == "[0.0 1.0 2.0 3.0 4.0]" - assert str((2 * a)[:]) == "[0.0 2.0 4.0 6.0 8.0]" - a = zeros(1001) - assert str(a) == "[0.0 0.0 0.0 ..., 0.0 0.0 0.0]" - - a = array(range(5), dtype=long) - assert str(a) == "[0 1 2 3 4]" - a = array([True, False, True, False], dtype="?") - assert str(a) == "[True False True False]" - - a = array(range(5), dtype="int8") - assert str(a) == "[0 1 2 3 4]" - - a = array(range(5), dtype="int16") - assert str(a) == "[0 1 2 3 4]" - - a = array((range(5), range(5, 10)), dtype="int16") - assert str(a) == "[[0 1 2 3 4]\n [5 6 7 8 9]]" - - a = array(3, dtype=int) - assert str(a) == "3" - - a = zeros((400, 400), dtype=int) - assert str(a) == "[[0 0 0 ..., 0 0 0]\n [0 0 0 ..., 0 0 0]\n" \ - " [0 0 0 ..., 0 0 0]\n ...,\n [0 0 0 ..., 0 0 0]\n" \ - " [0 0 0 ..., 0 0 0]\n [0 0 0 ..., 0 0 0]]" - a = zeros((2, 2, 2)) - r = str(a) - assert r == '[[[0.0 0.0]\n [0.0 0.0]]\n\n [[0.0 0.0]\n [0.0 0.0]]]' - - def test_str_slice(self): - from _numpypy import array, zeros - a = array(range(5), float) - b = a[1::2] - assert str(b) == "[1.0 3.0]" - a = zeros(2002) - b = a[::2] - assert str(b) == "[0.0 0.0 0.0 ..., 0.0 0.0 0.0]" - a = array((range(5), range(5, 10)), dtype="int16") - b = a[1, 2:] - assert str(b) == "[7 8 9]" - b = a[2:1, ] - assert str(b) == "[]" - - class AppTestRanges(BaseNumpyAppTest): def test_arange(self): from _numpypy import arange, array, dtype @@ -1640,3 +1713,24 @@ a = arange(0, 0.8, 0.1) assert len(a) == 8 assert arange(False, True, True).dtype is dtype(int) + +from pypy.module.micronumpy.appbridge import get_appbridge_cache + +class AppTestRepr(BaseNumpyAppTest): + def setup_class(cls): + BaseNumpyAppTest.setup_class.im_func(cls) + cache = get_appbridge_cache(cls.space) + cls.old_array_repr = cache.w_array_repr + cls.old_array_str = cache.w_array_str + cache.w_array_str = None + cache.w_array_repr = None + + def test_repr_str(self): + from _numpypy import array + assert repr(array([1, 2, 3])) == 'array([1, 2, 3])' + assert str(array([1, 2, 3])) == 'array([1, 2, 3])' + + def teardown_class(cls): + cache = get_appbridge_cache(cls.space) + cache.w_array_repr = cls.old_array_repr + cache.w_array_str = cls.old_array_str diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py --- a/pypy/module/micronumpy/test/test_ufuncs.py +++ b/pypy/module/micronumpy/test/test_ufuncs.py @@ -344,7 +344,7 @@ from _numpypy import sin, add raises(ValueError, sin.reduce, [1, 2, 3]) - raises(ValueError, add.reduce, 1) + raises((ValueError, TypeError), add.reduce, 1) def test_reduce_1d(self): from _numpypy import add, maximum @@ -360,6 +360,14 @@ assert (add.reduce(a, 0) == [12, 15, 18, 21]).all() assert (add.reduce(a, 1) == [6.0, 22.0, 38.0]).all() + def test_reduce_keepdims(self): + from _numpypy import add, arange + a = arange(12).reshape(3, 4) + b = add.reduce(a, 0, keepdims=True) + assert b.shape == (1, 4) + assert (add.reduce(a, 0, keepdims=True) == [12, 15, 18, 21]).all() + + def test_bitwise(self): from _numpypy import bitwise_and, bitwise_or, arange, array a = arange(6).reshape(2, 3) @@ -369,6 +377,12 @@ assert (a | 1 == bitwise_or(a, 1)).all() raises(TypeError, 'array([1.0]) & 1') + def test_unary_bitops(self): + from _numpypy import bitwise_not, array + a = array([1, 2, 3, 4]) + assert (~a == [-2, -3, -4, -5]).all() + assert (bitwise_not(a) == ~a).all() + def test_comparisons(self): import operator from _numpypy import equal, not_equal, less, less_equal, greater, greater_equal @@ -394,3 +408,28 @@ (3, 3.5), ]: assert ufunc(a, b) == func(a, b) + + def test_count_reduce_items(self): + from _numpypy import count_reduce_items, arange + a = arange(24).reshape(2, 3, 4) + assert count_reduce_items(a) == 24 + assert count_reduce_items(a, 1) == 3 + assert count_reduce_items(a, (1, 2)) == 3 * 4 + + def test_true_divide(self): + from _numpypy import arange, array, true_divide + assert (true_divide(arange(3), array([2, 2, 2])) == array([0, 0.5, 1])).all() + + def test_isnan_isinf(self): + from _numpypy import isnan, isinf, float64, array + assert isnan(float('nan')) + assert isnan(float64(float('nan'))) + assert not isnan(3) + assert isinf(float('inf')) + assert not isnan(3.5) + assert not isinf(3.5) + assert not isnan(float('inf')) + assert not isinf(float('nan')) + assert (isnan(array([0.2, float('inf'), float('nan')])) == [False, False, True]).all() + assert (isinf(array([0.2, float('inf'), float('nan')])) == [False, True, False]).all() + assert isinf(array([0.2])).dtype.kind == 'b' diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py --- a/pypy/module/micronumpy/test/test_zjit.py +++ b/pypy/module/micronumpy/test/test_zjit.py @@ -12,7 +12,7 @@ from pypy.module.micronumpy.compile import (FakeSpace, IntObject, Parser, InterpreterState) from pypy.module.micronumpy.interp_numarray import (W_NDimArray, - BaseArray) + BaseArray, W_FlatIterator) from pypy.rlib.nonconst import NonConstant @@ -287,6 +287,27 @@ 'jump': 1, 'arraylen_gc': 1}) + def define_take(): + return """ + a = |10| + b = take(a, [1, 1, 3, 2]) + b -> 2 + """ + + def test_take(self): + result = self.run("take") + assert result == 3 + self.check_simple_loop({'getinteriorfield_raw': 2, + 'cast_float_to_int': 1, + 'int_lt': 1, + 'int_ge': 2, + 'guard_false': 3, + 'setinteriorfield_raw': 1, + 'int_mul': 1, + 'int_add': 3, + 'jump': 1, + 'arraylen_gc': 2}) + def define_multidim(): return """ a = [[1, 2], [3, 4], [5, 6], [7, 8], [9, 10]] @@ -369,7 +390,71 @@ 'setinteriorfield_raw': 1, 'int_add': 2, 'int_ge': 1, 'guard_false': 1, 'jump': 1, 'arraylen_gc': 1}) + def define_flat_iter(): + return ''' + a = |30| + b = flat(a) + c = b + a + c -> 3 + ''' + def test_flat_iter(self): + result = self.run("flat_iter") + assert result == 6 + self.check_trace_count(1) + self.check_simple_loop({'getinteriorfield_raw': 2, 'float_add': 1, + 'setinteriorfield_raw': 1, 'int_add': 3, + 'int_ge': 1, 'guard_false': 1, + 'arraylen_gc': 1, 'jump': 1}) + + def define_flat_getitem(): + return ''' + a = |30| + b = flat(a) + b -> 4: -> 6 + ''' + + def test_flat_getitem(self): + result = self.run("flat_getitem") + assert result == 10.0 + self.check_trace_count(1) + self.check_simple_loop({'getinteriorfield_raw': 1, + 'setinteriorfield_raw': 1, + 'int_lt': 1, + 'int_ge': 1, + 'int_add': 3, + 'guard_true': 1, + 'guard_false': 1, + 'arraylen_gc': 2, + 'jump': 1}) + + def define_flat_setitem(): + return ''' + a = |30| + b = flat(a) + b[4:] = a->:26 + a -> 5 + ''' + + def test_flat_setitem(self): + result = self.run("flat_setitem") + assert result == 1.0 + self.check_trace_count(1) + # XXX not ideal, but hey, let's ignore it for now + self.check_simple_loop({'getinteriorfield_raw': 1, + 'setinteriorfield_raw': 1, + 'int_lt': 1, + 'int_gt': 1, + 'int_add': 4, + 'guard_true': 2, + 'arraylen_gc': 2, + 'jump': 1, + 'int_sub': 1, + # XXX bad part + 'int_and': 1, + 'int_mod': 1, + 'int_rshift': 1, + }) class TestNumpyOld(LLJitMixin): def setup_class(cls): diff --git a/pypy/module/micronumpy/tool/numready/main.py b/pypy/module/micronumpy/tool/numready/main.py --- a/pypy/module/micronumpy/tool/numready/main.py +++ b/pypy/module/micronumpy/tool/numready/main.py @@ -73,9 +73,8 @@ for line in lines: kind, name = line.split(" : ", 1) subitems = None - if kind == KINDS["TYPE"]: - if name in ['ndarray', 'dtype']: - subitems = find_numpy_items(python, modname, name) + if kind == KINDS["TYPE"] and name in SPECIAL_NAMES and attr is None: + subitems = find_numpy_items(python, modname, name) items.add(Item(name, kind, subitems)) return items @@ -89,15 +88,20 @@ l[i].append(lst[k * lgt + i]) return l +SPECIAL_NAMES = ["ndarray", "dtype", "generic"] + def main(argv): cpy_items = find_numpy_items("/usr/bin/python") pypy_items = find_numpy_items(argv[1], "numpypy") all_items = [] - msg = '%d/%d names, %d/%d ndarray attributes, %d/%d dtype attributes' % ( - len(pypy_items), len(cpy_items), len(pypy_items['ndarray'].subitems), - len(cpy_items['ndarray'].subitems), len(pypy_items['dtype'].subitems), - len(cpy_items['dtype'].subitems)) + msg = "{:d}/{:d} names".format(len(pypy_items), len(cpy_items)) + " " + msg += ", ".join( + "{:d}/{:d} {} attributes".format( + len(pypy_items[name].subitems), len(cpy_items[name].subitems), name + ) + for name in SPECIAL_NAMES + ) for item in cpy_items: pypy_exists = item in pypy_items if item.subitems: @@ -114,6 +118,6 @@ with open(argv[2], 'w') as f: f.write(html.encode("utf-8")) else: - with tempfile.NamedTemporaryFile(delete=False) as f: + with tempfile.NamedTemporaryFile(delete=False, suffix=".html") as f: f.write(html.encode("utf-8")) print "Saved in: %s" % f.name diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py --- a/pypy/module/micronumpy/types.py +++ b/pypy/module/micronumpy/types.py @@ -23,6 +23,16 @@ ) return dispatcher +def raw_unary_op(func): + specialize.argtype(1)(func) + @functools.wraps(func) + def dispatcher(self, v): + return func( + self, + self.for_computation(self.unbox(v)) + ) + return dispatcher + def simple_binary_op(func): specialize.argtype(1, 2)(func) @functools.wraps(func) @@ -95,7 +105,9 @@ )) def read_bool(self, storage, width, i, offset): - raise NotImplementedError + return bool(self.for_computation( + libffi.array_getitem(clibffi.cast_type_to_ffitype(self.T), + width, storage, i, offset))) def store(self, storage, width, i, offset, box): value = self.unbox(box) @@ -137,6 +149,14 @@ def abs(self, v): return abs(v) + @raw_unary_op + def isnan(self, v): + return False + + @raw_unary_op + def isinf(self, v): + return False + @raw_binary_op def eq(self, v1, v2): return v1 == v2 @@ -171,7 +191,7 @@ @simple_binary_op def min(self, v1, v2): return min(v1, v2) - + class Bool(BaseType, Primitive): T = lltype.Bool @@ -189,11 +209,6 @@ else: return self.False - - def read_bool(self, storage, width, i, offset): - return libffi.array_getitem(clibffi.cast_type_to_ffitype(self.T), - width, storage, i, offset) - def coerce_subtype(self, space, w_subtype, w_item): # Doesn't return subclasses so it can return the constants. return self._coerce(space, w_item) @@ -214,6 +229,18 @@ def default_fromstring(self, space): return self.box(False) + @simple_binary_op + def bitwise_and(self, v1, v2): + return v1 & v2 + + @simple_binary_op + def bitwise_or(self, v1, v2): + return v1 | v2 + + @simple_unary_op + def invert(self, v): + return ~v + class Integer(Primitive): _mixin_ = True @@ -270,6 +297,10 @@ def bitwise_or(self, v1, v2): return v1 | v2 + @simple_unary_op + def invert(self, v): + return ~v + class Int8(BaseType, Integer): T = rffi.SIGNEDCHAR BoxType = interp_boxes.W_Int8Box @@ -448,6 +479,14 @@ except ValueError: return rfloat.NAN + @raw_unary_op + def isnan(self, v): + return rfloat.isnan(v) + + @raw_unary_op + def isinf(self, v): + return rfloat.isinf(v) + class Float32(BaseType, Float): T = rffi.FLOAT diff --git a/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py --- a/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py +++ b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py @@ -98,15 +98,15 @@ from numpypy import array, var a = array([[1,2],[3,4]]) assert var(a) == 1.25 - #assert (var(a,0) == array([ 1., 1.])).all() - #assert (var(a,1) == array([ 0.25, 0.25])).all() + assert (var(a,0) == array([ 1., 1.])).all() + assert (var(a,1) == array([ 0.25, 0.25])).all() def test_std(self): from numpypy import array, std a = array([[1, 2], [3, 4]]) assert std(a) == 1.1180339887498949 - #assert (std(a, axis=0) == array([ 1., 1.])).all() - #assert (std(a, axis=1) == array([ 0.5, 0.5])).all() + assert (std(a, axis=0) == array([ 1., 1.])).all() + assert (std(a, axis=1) == array([ 0.5, 0.5])).all() def test_mean(self): from numpypy import array, mean, arange @@ -136,4 +136,4 @@ raises(NotImplementedError, "transpose(x, axes=(1, 0, 2))") # x = ones((1, 2, 3)) # assert transpose(x, (1, 0, 2)).shape == (2, 1, 3) - + diff --git a/pypy/module/test_lib_pypy/numpypy/core/test_numeric.py b/pypy/module/test_lib_pypy/numpypy/core/test_numeric.py new file mode 100644 --- /dev/null +++ b/pypy/module/test_lib_pypy/numpypy/core/test_numeric.py @@ -0,0 +1,144 @@ + +from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest + + +class AppTestBaseRepr(BaseNumpyAppTest): + def test_base3(self): + from numpypy import base_repr + assert base_repr(3**5, 3) == '100000' + + def test_positive(self): + from numpypy import base_repr + assert base_repr(12, 10) == '12' + assert base_repr(12, 10, 4) == '000012' + assert base_repr(12, 4) == '30' + assert base_repr(3731624803700888, 36) == '10QR0ROFCEW' + + def test_negative(self): + from numpypy import base_repr + assert base_repr(-12, 10) == '-12' + assert base_repr(-12, 10, 4) == '-000012' + assert base_repr(-12, 4) == '-30' + +class AppTestRepr(BaseNumpyAppTest): + def test_repr(self): + from numpypy import array + assert repr(array([1, 2, 3, 4])) == 'array([1, 2, 3, 4])' + + def test_repr_2(self): + from numpypy import array, zeros + int_size = array(5).dtype.itemsize + a = array(range(5), float) + assert repr(a) == "array([ 0., 1., 2., 3., 4.])" + a = array([], float) + assert repr(a) == "array([], dtype=float64)" + a = zeros(1001) + assert repr(a) == "array([ 0., 0., 0., ..., 0., 0., 0.])" + a = array(range(5), long) + if a.dtype.itemsize == int_size: + assert repr(a) == "array([0, 1, 2, 3, 4])" + else: + assert repr(a) == "array([0, 1, 2, 3, 4], dtype=int64)" + a = array(range(5), 'int32') + if a.dtype.itemsize == int_size: + assert repr(a) == "array([0, 1, 2, 3, 4])" + else: + assert repr(a) == "array([0, 1, 2, 3, 4], dtype=int32)" + a = array([], long) + assert repr(a) == "array([], dtype=int64)" + a = array([True, False, True, False], "?") + assert repr(a) == "array([ True, False, True, False], dtype=bool)" + a = zeros([]) + assert repr(a) == "array(0.0)" + a = array(0.2) + assert repr(a) == "array(0.2)" + a = array([2]) + assert repr(a) == "array([2])" + + def test_repr_multi(self): + from numpypy import arange, zeros, array + a = zeros((3, 4)) + assert repr(a) == '''array([[ 0., 0., 0., 0.], + [ 0., 0., 0., 0.], + [ 0., 0., 0., 0.]])''' + a = zeros((2, 3, 4)) + assert repr(a) == '''array([[[ 0., 0., 0., 0.], + [ 0., 0., 0., 0.], + [ 0., 0., 0., 0.]], + + [[ 0., 0., 0., 0.], + [ 0., 0., 0., 0.], + [ 0., 0., 0., 0.]]])''' + a = arange(1002).reshape((2, 501)) + assert repr(a) == '''array([[ 0, 1, 2, ..., 498, 499, 500], + [ 501, 502, 503, ..., 999, 1000, 1001]])''' + assert repr(a.T) == '''array([[ 0, 501], + [ 1, 502], + [ 2, 503], + ..., + [ 498, 999], + [ 499, 1000], + [ 500, 1001]])''' + a = arange(2).reshape((2,1)) + assert repr(a) == '''array([[0], + [1]])''' + + def test_repr_slice(self): + from numpypy import array, zeros + a = array(range(5), float) + b = a[1::2] + assert repr(b) == "array([ 1., 3.])" + a = zeros(2002) + b = a[::2] + assert repr(b) == "array([ 0., 0., 0., ..., 0., 0., 0.])" + a = array((range(5), range(5, 10)), dtype="int16") + b = a[1, 2:] + assert repr(b) == "array([7, 8, 9], dtype=int16)" + # an empty slice prints its shape + b = a[2:1, ] + assert repr(b) == "array([], shape=(0, 5), dtype=int16)" + + def test_str(self): + from numpypy import array, zeros + a = array(range(5), float) + assert str(a) == "[ 0. 1. 2. 3. 4.]" + assert str((2 * a)[:]) == "[ 0. 2. 4. 6. 8.]" + a = zeros(1001) + assert str(a) == "[ 0. 0. 0. ..., 0. 0. 0.]" + + a = array(range(5), dtype=long) + assert str(a) == "[0 1 2 3 4]" + a = array([True, False, True, False], dtype="?") + assert str(a) == "[ True False True False]" + + a = array(range(5), dtype="int8") + assert str(a) == "[0 1 2 3 4]" + + a = array(range(5), dtype="int16") + assert str(a) == "[0 1 2 3 4]" + + a = array((range(5), range(5, 10)), dtype="int16") + assert str(a) == "[[0 1 2 3 4]\n [5 6 7 8 9]]" + + a = array(3, dtype=int) + assert str(a) == "3" + + a = zeros((400, 400), dtype=int) + assert str(a) == '[[0 0 0 ..., 0 0 0]\n [0 0 0 ..., 0 0 0]\n [0 0 0 ..., 0 0 0]\n ..., \n [0 0 0 ..., 0 0 0]\n [0 0 0 ..., 0 0 0]\n [0 0 0 ..., 0 0 0]]' + a = zeros((2, 2, 2)) + r = str(a) + assert r == '[[[ 0. 0.]\n [ 0. 0.]]\n\n [[ 0. 0.]\n [ 0. 0.]]]' + + def test_str_slice(self): + from numpypy import array, zeros + a = array(range(5), float) + b = a[1::2] + assert str(b) == "[ 1. 3.]" + a = zeros(2002) + b = a[::2] + assert str(b) == "[ 0. 0. 0. ..., 0. 0. 0.]" + a = array((range(5), range(5, 10)), dtype="int16") + b = a[1, 2:] + assert str(b) == "[7 8 9]" + b = a[2:1, ] + assert str(b) == "[]" diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py --- a/pypy/objspace/fake/objspace.py +++ b/pypy/objspace/fake/objspace.py @@ -150,6 +150,8 @@ self._see_getsetproperty(x) if isinstance(x, r_singlefloat): self._wrap_not_rpython(x) + if isinstance(x, list): + self._wrap_not_rpython(x) return w_some_obj() wrap._annspecialcase_ = "specialize:argtype(1)" diff --git a/pypy/rlib/objectmodel.py b/pypy/rlib/objectmodel.py --- a/pypy/rlib/objectmodel.py +++ b/pypy/rlib/objectmodel.py @@ -130,19 +130,29 @@ if self is other: return 0 else: - raise TypeError("Symbolics can not be compared!") + raise TypeError("Symbolics cannot be compared! (%r, %r)" + % (self, other)) def __hash__(self): - raise TypeError("Symbolics are not hashable!") + raise TypeError("Symbolics are not hashable! %r" % (self,)) def __nonzero__(self): - raise TypeError("Symbolics are not comparable") + raise TypeError("Symbolics are not comparable! %r" % (self,)) class ComputedIntSymbolic(Symbolic): def __init__(self, compute_fn): self.compute_fn = compute_fn + def __repr__(self): + # repr(self.compute_fn) can arrive back here in an + # infinite recursion + try: + name = self.compute_fn.__name__ + except (AttributeError, TypeError): + name = hex(id(self.compute_fn)) + return '%s(%r)' % (self.__class__.__name__, name) + def annotation(self): from pypy.annotation import model return model.SomeInteger() @@ -157,6 +167,9 @@ self.expr = expr self.default = default + def __repr__(self): + return '%s(%r)' % (self.__class__.__name__, self.expr) + def annotation(self): from pypy.annotation import model return model.SomeInteger() diff --git a/pypy/rlib/rstacklet.py b/pypy/rlib/rstacklet.py --- a/pypy/rlib/rstacklet.py +++ b/pypy/rlib/rstacklet.py @@ -1,3 +1,4 @@ +import sys from pypy.rlib import _rffi_stacklet as _c from pypy.rlib import jit from pypy.rlib.objectmodel import we_are_translated @@ -72,6 +73,11 @@ if translated: assert config is not None, ("you have to pass a valid config, " "e.g. from 'driver.config'") + elif '__pypy__' in sys.builtin_module_names: + import py + py.test.skip("cannot run the stacklet tests on top of pypy: " + "calling directly the C function stacklet_switch() " + "will crash, depending on details of your config") if config is not None: assert config.translation.continuation, ( "stacklet: you have to translate with --continuation") diff --git a/pypy/rpython/lltypesystem/llgroup.py b/pypy/rpython/lltypesystem/llgroup.py --- a/pypy/rpython/lltypesystem/llgroup.py +++ b/pypy/rpython/lltypesystem/llgroup.py @@ -76,6 +76,10 @@ self.index = memberindex self.member = grp.members[memberindex]._as_ptr() + def __repr__(self): + return '%s(%s, %s)' % (self.__class__.__name__, + self.grpptr, self.index) + def __nonzero__(self): return True diff --git a/pypy/rpython/lltypesystem/llmemory.py b/pypy/rpython/lltypesystem/llmemory.py --- a/pypy/rpython/lltypesystem/llmemory.py +++ b/pypy/rpython/lltypesystem/llmemory.py @@ -32,7 +32,8 @@ self.known_nonneg()): return True else: - raise TypeError("Symbolics can not be compared!") + raise TypeError("Symbolics cannot be compared! (%r, %r)" + % (self, other)) def __lt__(self, other): return not self.__ge__(other) diff --git a/pypy/rpython/lltypesystem/rdict.py b/pypy/rpython/lltypesystem/rdict.py --- a/pypy/rpython/lltypesystem/rdict.py +++ b/pypy/rpython/lltypesystem/rdict.py @@ -323,6 +323,17 @@ hop.exception_is_here() return hop.gendirectcall(ll_popitem, cTUPLE, v_dict) + def rtype_method_pop(self, hop): + if hop.nb_args == 2: + v_args = hop.inputargs(self, self.key_repr) + target = ll_pop + elif hop.nb_args == 3: + v_args = hop.inputargs(self, self.key_repr, self.value_repr) + target = ll_pop_default + hop.exception_is_here() + v_res = hop.gendirectcall(target, *v_args) + return self.recast_value(hop.llops, v_res) + class __extend__(pairtype(DictRepr, rmodel.Repr)): def rtype_getitem((r_dict, r_key), hop): @@ -874,3 +885,18 @@ r.item1 = recast(ELEM.TO.item1, entry.value) _ll_dict_del(dic, i) return r + +def ll_pop(dic, key): + i = ll_dict_lookup(dic, key, dic.keyhash(key)) + if not i & HIGHEST_BIT: + value = ll_get_value(dic, i) + _ll_dict_del(dic, i) + return value + else: + raise KeyError + +def ll_pop_default(dic, key, dfl): + try: + return ll_pop(dic, key) + except KeyError: + return dfl diff --git a/pypy/rpython/lltypesystem/rffi.py b/pypy/rpython/lltypesystem/rffi.py --- a/pypy/rpython/lltypesystem/rffi.py +++ b/pypy/rpython/lltypesystem/rffi.py @@ -27,6 +27,10 @@ self.c_name = c_name self.TP = TP + def __repr__(self): + return '%s(%r, %s)' % (self.__class__.__name__, + self.c_name, self.TP) + def annotation(self): return lltype_to_annotation(self.TP) diff --git a/pypy/rpython/ootypesystem/rdict.py b/pypy/rpython/ootypesystem/rdict.py --- a/pypy/rpython/ootypesystem/rdict.py +++ b/pypy/rpython/ootypesystem/rdict.py @@ -160,6 +160,16 @@ hop.exception_is_here() return hop.gendirectcall(ll_popitem, cTUPLE, v_dict) + def rtype_method_pop(self, hop): + if hop.nb_args == 2: + v_args = hop.inputargs(self, self.key_repr) + target = ll_pop + elif hop.nb_args == 3: + v_args = hop.inputargs(self, self.key_repr, self.value_repr) + target = ll_pop_default + hop.exception_is_here() + return hop.gendirectcall(target, *v_args) + def __get_func(self, interp, r_func, fn, TYPE): if isinstance(r_func, MethodOfFrozenPBCRepr): obj = r_func.r_im_self.convert_const(fn.im_self) @@ -370,6 +380,20 @@ return res raise KeyError +def ll_pop(d, key): + if d.ll_contains(key): + value = d.ll_get(key) + d.ll_remove(key) + return value + else: + raise KeyError + +def ll_pop_default(d, key, dfl): + try: + return ll_pop(d, key) + except KeyError: + return dfl + # ____________________________________________________________ # # Iteration. diff --git a/pypy/rpython/test/test_rdict.py b/pypy/rpython/test/test_rdict.py --- a/pypy/rpython/test/test_rdict.py +++ b/pypy/rpython/test/test_rdict.py @@ -25,37 +25,37 @@ class BaseTestRdict(BaseRtypingTest): def test_dict_creation(self): - def createdict(i): + def createdict(i): d = {'hello' : i} return d['hello'] res = self.interpret(createdict, [42]) assert res == 42 - def test_dict_getitem_setitem(self): - def func(i): + def test_dict_getitem_setitem(self): + def func(i): d = {'hello' : i} d['world'] = i + 1 - return d['hello'] * d['world'] + return d['hello'] * d['world'] res = self.interpret(func, [6]) assert res == 42 - def test_dict_getitem_keyerror(self): - def func(i): + def test_dict_getitem_keyerror(self): + def func(i): d = {'hello' : i} try: return d['world'] except KeyError: - return 0 + return 0 res = self.interpret(func, [6]) assert res == 0 def test_dict_del_simple(self): - def func(i): + def func(i): d = {'hello' : i} d['world'] = i + 1 del d['hello'] - return len(d) + return len(d) res = self.interpret(func, [6]) assert res == 1 @@ -71,7 +71,7 @@ assert res == True def test_empty_strings(self): - def func(i): + def func(i): d = {'' : i} del d[''] try: @@ -83,7 +83,7 @@ res = self.interpret(func, [6]) assert res == 1 - def func(i): + def func(i): d = {'' : i} del d[''] d[''] = i + 1 @@ -146,8 +146,8 @@ d1 = {} d1['hello'] = i + 1 d2 = {} - d2['world'] = d1 - return d2['world']['hello'] + d2['world'] = d1 + return d2['world']['hello'] res = self.interpret(func, [5]) assert res == 6 @@ -297,7 +297,7 @@ a = 0 for k, v in items: b += isinstance(k, B) - a += isinstance(v, A) + a += isinstance(v, A) return 3*b+a res = self.interpret(func, []) assert res == 8 @@ -316,7 +316,7 @@ a = 0 for k, v in dic.iteritems(): b += isinstance(k, B) - a += isinstance(v, A) + a += isinstance(v, A) return 3*b+a res = self.interpret(func, []) assert res == 8 @@ -342,11 +342,11 @@ def test_dict_contains_with_constant_dict(self): dic = {'4':1000, ' 8':200} def func(i): - return chr(i) in dic - res = self.interpret(func, [ord('4')]) + return chr(i) in dic + res = self.interpret(func, [ord('4')]) assert res is True - res = self.interpret(func, [1]) - assert res is False + res = self.interpret(func, [1]) + assert res is False def test_dict_or_none(self): class A: @@ -413,7 +413,7 @@ return g(get) res = self.interpret(f, []) - assert res == 2 + assert res == 2 def test_specific_obscure_bug(self): class A: pass @@ -622,6 +622,42 @@ res = self.interpret(func, []) assert res in [5263, 6352] + def test_dict_pop(self): + def f(n, default): + d = {} + d[2] = 3 + d[4] = 5 + if default == -1: + try: + x = d.pop(n) + except KeyError: + x = -1 + else: + x = d.pop(n, default) + return x * 10 + len(d) + res = self.interpret(f, [2, -1]) + assert res == 31 + res = self.interpret(f, [3, -1]) + assert res == -8 + res = self.interpret(f, [2, 5]) + assert res == 31 + + def test_dict_pop_instance(self): + class A(object): + pass + def f(n): + d = {} + d[2] = A() + x = d.pop(n, None) + if x is None: + return 12 + else: + return 15 + res = self.interpret(f, [2]) + assert res == 15 + res = self.interpret(f, [700]) + assert res == 12 + class TestLLtype(BaseTestRdict, LLRtypeMixin): def test_dict_but_not_with_char_keys(self): def func(i): @@ -633,19 +669,19 @@ res = self.interpret(func, [6]) assert res == 0 - def test_deleted_entry_reusage_with_colliding_hashes(self): - def lowlevelhash(value): + def test_deleted_entry_reusage_with_colliding_hashes(self): + def lowlevelhash(value): p = rstr.mallocstr(len(value)) for i in range(len(value)): p.chars[i] = value[i] - return rstr.LLHelpers.ll_strhash(p) + return rstr.LLHelpers.ll_strhash(p) - def func(c1, c2): - c1 = chr(c1) - c2 = chr(c2) + def func(c1, c2): + c1 = chr(c1) + c2 = chr(c2) d = {} d[c1] = 1 - d[c2] = 2 + d[c2] = 2 del d[c1] return d[c2] @@ -653,7 +689,7 @@ base = rdict.DICT_INITSIZE for y in range(0, 256): y = chr(y) - y_hash = lowlevelhash(y) % base + y_hash = lowlevelhash(y) % base char_by_hash.setdefault(y_hash, []).append(y) x, y = char_by_hash[0][:2] # find a collision @@ -661,18 +697,18 @@ res = self.interpret(func, [ord(x), ord(y)]) assert res == 2 - def func2(c1, c2): - c1 = chr(c1) - c2 = chr(c2) + def func2(c1, c2): + c1 = chr(c1) + c2 = chr(c2) d = {} d[c1] = 1 - d[c2] = 2 + d[c2] = 2 del d[c1] d[c1] = 3 - return d + return d res = self.interpret(func2, [ord(x), ord(y)]) - for i in range(len(res.entries)): + for i in range(len(res.entries)): assert not (res.entries.everused(i) and not res.entries.valid(i)) def func3(c0, c1, c2, c3, c4, c5, c6, c7): @@ -687,9 +723,9 @@ c7 = chr(c7) ; d[c7] = 1; del d[c7] return d - if rdict.DICT_INITSIZE != 8: + if rdict.DICT_INITSIZE != 8: py.test.skip("make dict tests more indepdent from initsize") - res = self.interpret(func3, [ord(char_by_hash[i][0]) + res = self.interpret(func3, [ord(char_by_hash[i][0]) for i in range(rdict.DICT_INITSIZE)]) count_frees = 0 for i in range(len(res.entries)): @@ -707,9 +743,9 @@ del d[chr(ord('a') + i)] return d res = self.interpret(func, [0]) - assert len(res.entries) > rdict.DICT_INITSIZE + assert len(res.entries) > rdict.DICT_INITSIZE res = self.interpret(func, [1]) - assert len(res.entries) == rdict.DICT_INITSIZE + assert len(res.entries) == rdict.DICT_INITSIZE def test_dict_valid_resize(self): # see if we find our keys after resize @@ -844,7 +880,7 @@ def test_prebuilt_list_of_addresses(self): from pypy.rpython.lltypesystem import llmemory - + TP = lltype.Struct('x', ('y', lltype.Signed)) a = lltype.malloc(TP, flavor='raw', immortal=True) b = lltype.malloc(TP, flavor='raw', immortal=True) @@ -858,7 +894,7 @@ d = {a_a: 3, a_b: 4, a_c: 5} d[a0] = 8 - + def func(i): if i == 0: ptr = a @@ -888,7 +924,7 @@ return a == b def rhash(a): return 3 - + def func(i): d = r_dict(eq, rhash, force_non_null=True) if not i: diff --git a/pypy/translator/goal/targetnopstandalone.py b/pypy/translator/goal/targetnopstandalone.py --- a/pypy/translator/goal/targetnopstandalone.py +++ b/pypy/translator/goal/targetnopstandalone.py @@ -7,10 +7,8 @@ actually implementing argv of the executable. """ -import os, sys - -def debug(msg): - os.write(2, "debug: " + msg + '\n') +def debug(msg): + print "debug:", msg # __________ Entry point __________ From noreply at buildbot.pypy.org Sat Jan 28 14:02:37 2012 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 28 Jan 2012 14:02:37 +0100 (CET) Subject: [pypy-commit] extradoc extradoc: This way maybe? Message-ID: <20120128130237.0C63682114@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: extradoc Changeset: r4067:f58b1b95e973 Date: 2012-01-28 14:02 +0100 http://bitbucket.org/pypy/extradoc/changeset/f58b1b95e973/ Log: This way maybe? diff --git a/blog/draft/numpy-status-update-2.rst b/blog/draft/numpy-status-update-2.rst --- a/blog/draft/numpy-status-update-2.rst +++ b/blog/draft/numpy-status-update-2.rst @@ -20,7 +20,7 @@ * indexing arrays by boolean arrays of the same size -* bugfixes +* and various bugfixes. We would also like to introduce the `nightly report`_ of numpy status. This is an automated tool that does package introspection. While it gives some @@ -31,7 +31,8 @@ reserve the right to incorporate jokes in that website, so don't treat it that seriously overall :-) -We hope to provide you with more frequent updates on the progress. +Thanks, and stay tuned. We hope to post here regular updates on the +progress. Cheers, fijal & the PyPy team From noreply at buildbot.pypy.org Sat Jan 28 14:05:28 2012 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 28 Jan 2012 14:05:28 +0100 (CET) Subject: [pypy-commit] extradoc extradoc: rst fix; Message-ID: <20120128130528.AE83482114@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: extradoc Changeset: r4068:67dcf04c0397 Date: 2012-01-28 15:05 +0200 http://bitbucket.org/pypy/extradoc/changeset/67dcf04c0397/ Log: rst fix; diff --git a/blog/draft/numpy-status-update-2.rst b/blog/draft/numpy-status-update-2.rst --- a/blog/draft/numpy-status-update-2.rst +++ b/blog/draft/numpy-status-update-2.rst @@ -37,4 +37,4 @@ Cheers, fijal & the PyPy team -.. `nightly report`: http://buildbot.pypy.org/numpy-status/latest.html +.. _`nightly report`: http://buildbot.pypy.org/numpy-status/latest.html From noreply at buildbot.pypy.org Sat Jan 28 16:55:48 2012 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 28 Jan 2012 16:55:48 +0100 (CET) Subject: [pypy-commit] pypy stackroot-speedup: (fijal, arigo) Message-ID: <20120128155548.798F482115@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: stackroot-speedup Changeset: r51911:ecd5a39e368a Date: 2012-01-28 16:42 +0100 http://bitbucket.org/pypy/pypy/changeset/ecd5a39e368a/ Log: (fijal, arigo) Pass the flag "is_minor" around. diff --git a/pypy/rpython/memory/gc/base.py b/pypy/rpython/memory/gc/base.py --- a/pypy/rpython/memory/gc/base.py +++ b/pypy/rpython/memory/gc/base.py @@ -299,7 +299,8 @@ # list of last generation roots callback2, attrname = _convert_callback_formats(callback) # :-/ setattr(self, attrname, arg) - self.root_walker.walk_roots(callback2, callback2, callback2) + self.root_walker.walk_roots(callback2, callback2, callback2, + is_minor=False) self.run_finalizers.foreach(callback, arg) enumerate_all_roots._annspecialcase_ = 'specialize:arg(1)' diff --git a/pypy/rpython/memory/gc/generation.py b/pypy/rpython/memory/gc/generation.py --- a/pypy/rpython/memory/gc/generation.py +++ b/pypy/rpython/memory/gc/generation.py @@ -318,7 +318,8 @@ self.root_walker.walk_roots( SemiSpaceGC._collect_root, # stack roots SemiSpaceGC._collect_root, # static in prebuilt non-gc structures - None) # we don't need the static in prebuilt gc objects + None, # we don't need the static in prebuilt gc objects + is_minor=False) def collect_last_generation_roots(self): stack = self.last_generation_root_objects @@ -414,7 +415,8 @@ self.root_walker.walk_roots( GenerationGC._collect_root_in_nursery, # stack roots GenerationGC._collect_root_in_nursery, # static in prebuilt non-gc - None) # static in prebuilt gc + None, # static in prebuilt gc + is_minor=True) def _collect_root_in_nursery(self, root): obj = root.address[0] diff --git a/pypy/rpython/memory/gc/hybrid.py b/pypy/rpython/memory/gc/hybrid.py --- a/pypy/rpython/memory/gc/hybrid.py +++ b/pypy/rpython/memory/gc/hybrid.py @@ -305,7 +305,8 @@ self.root_walker.walk_roots( SemiSpaceGC._collect_root, # stack roots SemiSpaceGC._collect_root, # static in prebuilt non-gc structs - SemiSpaceGC._collect_root) # static in prebuilt gc objects + SemiSpaceGC._collect_root, # static in prebuilt gc objects + is_minor=False) def surviving(self, obj): # To use during a collection. The objects that survive are the diff --git a/pypy/rpython/memory/gc/markcompact.py b/pypy/rpython/memory/gc/markcompact.py --- a/pypy/rpython/memory/gc/markcompact.py +++ b/pypy/rpython/memory/gc/markcompact.py @@ -367,7 +367,8 @@ self.root_walker.walk_roots( MarkCompactGC._mark_root, # stack roots MarkCompactGC._mark_root, # static in prebuilt non-gc structures - MarkCompactGC._mark_root) # static in prebuilt gc objects + MarkCompactGC._mark_root, # static in prebuilt gc objects + is_minor=False) if (self.objects_with_finalizers.non_empty() or self.run_finalizers.non_empty()): self.trace_from_objects_with_finalizers() @@ -461,7 +462,8 @@ self.root_walker.walk_roots( MarkCompactGC._update_ref, # stack roots MarkCompactGC._update_ref, # static in prebuilt non-gc structures - MarkCompactGC._update_ref) # static in prebuilt gc objects + MarkCompactGC._update_ref, # static in prebuilt gc objects + is_minor=False) self.walk_marked_objects(MarkCompactGC.trace_and_update_ref) def walk_marked_objects(self, callback): diff --git a/pypy/rpython/memory/gc/marksweep.py b/pypy/rpython/memory/gc/marksweep.py --- a/pypy/rpython/memory/gc/marksweep.py +++ b/pypy/rpython/memory/gc/marksweep.py @@ -253,7 +253,8 @@ self.root_walker.walk_roots( MarkSweepGC._mark_root, # stack roots MarkSweepGC._mark_root, # static in prebuilt non-gc structures - MarkSweepGC._mark_root) # static in prebuilt gc objects + MarkSweepGC._mark_root, # static in prebuilt gc objects + is_minor=False) # from this point onwards, no more mallocs should be possible old_malloced = self.bytes_malloced diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py --- a/pypy/rpython/memory/gc/minimark.py +++ b/pypy/rpython/memory/gc/minimark.py @@ -1303,7 +1303,8 @@ self.root_walker.walk_roots( MiniMarkGC._trace_drag_out1, # stack roots MiniMarkGC._trace_drag_out1, # static in prebuilt non-gc - None) # static in prebuilt gc + None, # static in prebuilt gc + is_minor=True) debug_stop("gc-minor-walkroots") def collect_cardrefs_to_nursery(self): @@ -1711,7 +1712,8 @@ self.root_walker.walk_roots( MiniMarkGC._collect_ref_stk, # stack roots MiniMarkGC._collect_ref_stk, # static in prebuilt non-gc structures - None) # we don't need the static in all prebuilt gc objects + None, # we don't need the static in all prebuilt gc objects + is_minor=False) # # If we are in an inner collection caused by a call to a finalizer, # the 'run_finalizers' objects also need to be kept alive. diff --git a/pypy/rpython/memory/gc/semispace.py b/pypy/rpython/memory/gc/semispace.py --- a/pypy/rpython/memory/gc/semispace.py +++ b/pypy/rpython/memory/gc/semispace.py @@ -350,7 +350,8 @@ self.root_walker.walk_roots( SemiSpaceGC._collect_root, # stack roots SemiSpaceGC._collect_root, # static in prebuilt non-gc structures - SemiSpaceGC._collect_root) # static in prebuilt gc objects + SemiSpaceGC._collect_root, # static in prebuilt gc objects + is_minor=False) def _collect_root(self, root): root.address[0] = self.copy(root.address[0]) diff --git a/pypy/rpython/memory/gctransform/framework.py b/pypy/rpython/memory/gctransform/framework.py --- a/pypy/rpython/memory/gctransform/framework.py +++ b/pypy/rpython/memory/gctransform/framework.py @@ -1355,7 +1355,8 @@ def walk_roots(self, collect_stack_root, collect_static_in_prebuilt_nongc, - collect_static_in_prebuilt_gc): + collect_static_in_prebuilt_gc, + is_minor): gcdata = self.gcdata gc = self.gc if collect_static_in_prebuilt_nongc: @@ -1375,7 +1376,8 @@ collect_static_in_prebuilt_gc(gc, result) addr += sizeofaddr if collect_stack_root: - self.walk_stack_roots(collect_stack_root) # abstract + self.walk_stack_roots(collect_stack_root, # abstract + is_minor=is_minor) def need_stacklet_support(self): raise Exception("%s does not support stacklets" % ( diff --git a/pypy/rpython/memory/gctransform/shadowstack.py b/pypy/rpython/memory/gctransform/shadowstack.py --- a/pypy/rpython/memory/gctransform/shadowstack.py +++ b/pypy/rpython/memory/gctransform/shadowstack.py @@ -94,10 +94,11 @@ self.shadow_stack_pool.initial_setup() BaseRootWalker.setup_root_walker(self) - def walk_stack_roots(self, collect_stack_root): + def walk_stack_roots(self, collect_stack_root, is_minor): gcdata = self.gcdata self.rootstackhook(collect_stack_root, - gcdata.root_stack_base, gcdata.root_stack_top) + gcdata.root_stack_base, gcdata.root_stack_top, + is_minor=is_minor) def need_thread_support(self, gctransformer, getfn): from pypy.module.thread import ll_thread # xxx fish From noreply at buildbot.pypy.org Sat Jan 28 16:55:49 2012 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 28 Jan 2012 16:55:49 +0100 (CET) Subject: [pypy-commit] pypy stackroot-speedup: (fijal, arigo) Message-ID: <20120128155549.AB91582116@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: stackroot-speedup Changeset: r51912:20b7b00aefee Date: 2012-01-28 16:55 +0100 http://bitbucket.org/pypy/pypy/changeset/20b7b00aefee/ Log: (fijal, arigo) Test in progress. diff --git a/pypy/rpython/memory/gctransform/test/test_framework.py b/pypy/rpython/memory/gctransform/test/test_framework.py --- a/pypy/rpython/memory/gctransform/test/test_framework.py +++ b/pypy/rpython/memory/gctransform/test/test_framework.py @@ -332,3 +332,33 @@ clean_setarrayitems = find_clean_setarrayitems(collect_analyzer, t.graphs[0]) assert len(clean_setarrayitems) == 1 + +def test_contains_stack_check(): + from pypy.translator.transform import insert_ll_stackcheck + from pypy.translator.c.genc import CStandaloneBuilder + # + class A: + pass + def f_non_recursive(n): + A() + def f_recursive(n): + A() + if n > 5: + f_recursive(n-1) + def f(n): + a = A() + f_non_recursive(n) + f_recursive(n) + return a + + t = rtype(f, [int]) + insert_ll_stackcheck(t) + g_non_recursive = t._graphof(f_non_recursive) + assert not hasattr(g_non_recursive, 'contains_stack_check') + g_recursive = t._graphof(f_recursive) + assert g_recursive.contains_stack_check is True + # + cbuild = CStandaloneBuilder(t, f, t.config, + gcpolicy=FrameworkGcPolicy2) + db = cbuild.generate_graphs_for_llinterp() + import pdb; pdb.set_trace() diff --git a/pypy/translator/transform.py b/pypy/translator/transform.py --- a/pypy/translator/transform.py +++ b/pypy/translator/transform.py @@ -214,6 +214,8 @@ # not consume any stack, so would turn into potentially infinite loops graph = block2graph[block] graph.inhibit_tail_call = True + # this other attribute is used by shadowstack.py + graph.contains_stack_check = True return len(insert_in) From noreply at buildbot.pypy.org Sat Jan 28 16:55:47 2012 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 28 Jan 2012 16:55:47 +0100 (CET) Subject: [pypy-commit] pypy stackroot-speedup: (fijal, arigo) Message-ID: <20120128155547.347EE82114@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: stackroot-speedup Changeset: r51910:641b24e593c6 Date: 2012-01-28 16:30 +0100 http://bitbucket.org/pypy/pypy/changeset/641b24e593c6/ Log: (fijal, arigo) Trying to speed up the stack root walking step of minor collections. The idea is to mark and detect the limit of the previous stack root walking, so that we don't need to repeat tracing over the already- scanned, already-old objects. diff --git a/pypy/rpython/lltypesystem/ll2ctypes.py b/pypy/rpython/lltypesystem/ll2ctypes.py --- a/pypy/rpython/lltypesystem/ll2ctypes.py +++ b/pypy/rpython/lltypesystem/ll2ctypes.py @@ -1266,6 +1266,9 @@ def _cast_to_ptr(self, TP): return force_cast(TP, self.intval) + def _cast_to_int(self, symbolic="ignored"): + return long(self) # 'self' is already a subclass of long... + def __repr__(self): return '<_lladdress %s>' % (self.void_p,) diff --git a/pypy/rpython/memory/gctransform/shadowstack.py b/pypy/rpython/memory/gctransform/shadowstack.py --- a/pypy/rpython/memory/gctransform/shadowstack.py +++ b/pypy/rpython/memory/gctransform/shadowstack.py @@ -1,7 +1,7 @@ from pypy.rpython.memory.gctransform.framework import BaseRootWalker from pypy.rpython.memory.gctransform.framework import sizeofaddr from pypy.rpython.annlowlevel import llhelper -from pypy.rpython.lltypesystem import lltype, llmemory +from pypy.rpython.lltypesystem import lltype, llmemory, rffi from pypy.rlib.debug import ll_assert from pypy.rlib.nonconst import NonConstant from pypy.annotation import model as annmodel @@ -10,6 +10,11 @@ class ShadowStackRootWalker(BaseRootWalker): need_root_stack = True + # special values in the shadowstack (different values than + # the jit's MARKER, which is 8): + MARKER_NOT_TRACED = 16 + MARKER_TRACED = 24 + def __init__(self, gctransformer): BaseRootWalker.__init__(self, gctransformer) # NB. 'self' is frozen, but we can use self.gcdata to store state @@ -42,12 +47,34 @@ addr += sizeofaddr self.rootstackhook = jit_walk_stack_root else: - def default_walk_stack_root(callback, addr, end): + def default_walk_stack_root(callback, start, end, is_minor): gc = self.gc - while addr != end: + addr = end + while addr != start: + addr -= sizeofaddr + value = llmemory.cast_adr_to_int(addr.address[0]) + # + # If myaddr contains MARKER_TRACED, and if we are doing + # a minor collection, then stop here. The previous items + # have all been traced at least once already, so they + # cannot contain young pointers. + if value == self.MARKER_TRACED: + if is_minor: + break + continue # ignore the marker and continue + # + # If myaddr contains MARKER_NOT_TRACED, replace it by + # MARKER_TRACED. + if value == self.MARKER_NOT_TRACED: + addr.address[0] = rffi.cast(llmemory.Address, + self.MARKER_TRACED) + continue + # + # Regular part follows: invoke 'callback' if myaddr really + # points to an object if gc.points_to_valid_gc_object(addr): callback(gc, addr) - addr += sizeofaddr + # self.rootstackhook = default_walk_stack_root self.shadow_stack_pool = ShadowStackPool(gcdata) diff --git a/pypy/rpython/memory/gctransform/test/test_shadowstack.py b/pypy/rpython/memory/gctransform/test/test_shadowstack.py new file mode 100644 --- /dev/null +++ b/pypy/rpython/memory/gctransform/test/test_shadowstack.py @@ -0,0 +1,54 @@ +from pypy.rpython.memory.gctransform.shadowstack import ShadowStackRootWalker +from pypy.rpython.memory.gctransform.shadowstack import sizeofaddr +from pypy.rpython.lltypesystem import lltype, llmemory, rffi + + +class MockGC: + def points_to_valid_gc_object(self, addr): + n = rffi.cast(lltype.Signed, addr.address[0]) + if n > 0x1000: + return True + if n == 0: + return False + assert 0, "oups, bogus address: 0x%x" % n + +class MockGCData: + gc = MockGC() + +class MockGcTransformer: + gcdata = MockGCData() + translator = None + root_stack_depth = None + + +def test_default_walk_stack_root(): + for (minor, already_traced) in [(False, False), + (False, True), + (True, False), + (True, True)]: + root_stack_size = sizeofaddr * 5 + a = llmemory.raw_malloc(root_stack_size) + a.address[0] = llmemory.NULL + a.address[1] = rffi.cast(llmemory.Address, 0x1234) + if already_traced: + marker = ShadowStackRootWalker.MARKER_TRACED + else: + marker = ShadowStackRootWalker.MARKER_NOT_TRACED + a.address[2] = rffi.cast(llmemory.Address, marker) + a.address[3] = rffi.cast(llmemory.Address, 0x5678) + a.address[4] = llmemory.NULL + walker = ShadowStackRootWalker(MockGcTransformer()) + seen = [] + def callback(gc, addr): + assert gc == walker.gc + seen.append(rffi.cast(lltype.Signed, addr.address[0])) + aend = a + 5 * sizeofaddr + walker.rootstackhook(callback, a, aend, is_minor=minor) + marker = rffi.cast(lltype.Signed, a.address[2]) + assert marker == ShadowStackRootWalker.MARKER_TRACED + if minor and already_traced: + assert seen == [0x5678] + else: + assert seen == [0x5678, 0x1234] + llmemory.raw_free(a) + From noreply at buildbot.pypy.org Sat Jan 28 17:57:22 2012 From: noreply at buildbot.pypy.org (amauryfa) Date: Sat, 28 Jan 2012 17:57:22 +0100 (CET) Subject: [pypy-commit] pypy py3k: range() is not a list anymore, and objspace.std.withrangelist was removed Message-ID: <20120128165722.1A46282114@wyvern.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc <amauryfa at gmail.com> Branch: py3k Changeset: r51913:07a52ec2616d Date: 2012-01-28 09:33 +0100 http://bitbucket.org/pypy/pypy/changeset/07a52ec2616d/ Log: range() is not a list anymore, and objspace.std.withrangelist was removed diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py --- a/pypy/objspace/std/test/test_listobject.py +++ b/pypy/objspace/std/test/test_listobject.py @@ -1148,97 +1148,6 @@ assert l == [] assert list(g) == [] -class AppTestForRangeLists(AppTestW_ListObject): - - def setup_class(cls): - cls.space = gettestobjspace(**{"objspace.std.withrangelist" : - True}) - - def test_range_simple_backwards(self): - x = range(5,1) - assert x == [] - - def test_range_big_start(self): - x = range(1,10) - x[22:0:-1] == range(1,10) - - def test_range_list_invalid_slice(self): - x = [1,2,3,4] - assert x[10:0] == [] - assert x[10:0:None] == [] - - x = range(1,5) - assert x[10:0] == [] - assert x[10:0:None] == [] - - assert x[0:22] == [1,2,3,4] - assert x[-1:10] == [4] - - assert x[0:22:None] == [1,2,3,4] - assert x[-1:10:None] == [4] - - def test_range_backwards(self): - x = range(1,10) - assert x[22:-10] == [] - assert x[22:-10:-1] == [9,8,7,6,5,4,3,2,1] - assert x[10:3:-1] == [9,8,7,6,5] - assert x[10:3:-2] == [9,7,5] - assert x[1:5:-1] == [] - - def test_sort_range(self): - l = range(3,10,3) - l.sort() - assert l == [3, 6, 9] - l.sort(reverse = True) - assert l == [9, 6, 3] - l.sort(reverse = True) - assert l == [9, 6, 3] - l.sort() - assert l == [3, 6, 9] - - def test_slice(self): - l = [] - l2 = range(3) - l.__setslice__(0,3,l2) - assert l == [0,1,2] - - def test_getitem(self): - l = range(5) - raises(IndexError, "l[-10]") - - def test_append(self): - l = range(5) - l.append(26) - assert l == [0,1,2,3,4,26] - - l = range(5) - l.append("a") - assert l == [0,1,2,3,4,"a"] - - l = range(5) - l.append(5) - assert l == [0,1,2,3,4,5] - - def test_pop(self): - l = range(3) - assert l.pop(0) == 0 - - def test_setitem(self): - l = range(3) - l[0] = 1 - assert l == [1,1,2] - - def test_inset(self): - l = range(3) - l.insert(1,5) - assert l == [0,5,1,2] - - def test_reverse(self): - l = range(3) - l.reverse() - assert l == [2,1,0] - - class AppTestListFastSubscr: def setup_class(cls): From noreply at buildbot.pypy.org Sat Jan 28 17:57:23 2012 From: noreply at buildbot.pypy.org (amauryfa) Date: Sat, 28 Jan 2012 17:57:23 +0100 (CET) Subject: [pypy-commit] pypy py3k: Fix remaining failures in test_listobject. Message-ID: <20120128165723.58E6882114@wyvern.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc <amauryfa at gmail.com> Branch: py3k Changeset: r51914:732d082457bf Date: 2012-01-28 09:55 +0100 http://bitbucket.org/pypy/pypy/changeset/732d082457bf/ Log: Fix remaining failures in test_listobject. diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py --- a/pypy/objspace/std/test/test_listobject.py +++ b/pypy/objspace/std/test/test_listobject.py @@ -460,7 +460,6 @@ l = [1] assert list(l) is not l assert list(l) == l - assert list(range(10)) == range(10) def test_explicit_new_init(self): l = l0 = list.__new__(list) @@ -485,9 +484,9 @@ l = ['a'] l.extend([0]) assert l == ['a', 0] - l = range(10) + l = list(range(10)) l.extend([10]) - assert l == range(11) + assert l == list(range(11)) l = [] m = [1,2,3] @@ -568,9 +567,9 @@ assert l == [ 'a' ] def test_sort_reversed(self): - l = range(10) + l = list(range(10)) l.sort(reverse = True) - assert l == range(9, -1, -1) + assert l == list(range(9, -1, -1)) l = [] l.sort(reverse = True) assert l == [] @@ -656,12 +655,12 @@ assert l is l0 raises(IndexError, "del l[0]") - l = range(10) + l = list(range(10)) del l[5] assert l == [0, 1, 2, 3, 4, 6, 7, 8, 9] def test_getitem_slice(self): - l = range(10) + l = list(range(10)) assert l[::] == l del l[::2] assert l == [1,3,5,7,9] @@ -703,11 +702,7 @@ l = [1.1, 2.2] del l[:] assert l == [] - - l = range(5) - del l[:] - assert l == [] - + def test_iadd(self): l = l0 = [1,2,3] l += [4,5] @@ -748,10 +743,6 @@ l3 = l1 + l2 assert l3 == [1,2,3,4,5,6] - l4 = range(3) - l5 = l4 + l2 - assert l5 == [0,1,2,4,5,6] - def test_imul(self): l = l0 = [4,3] l *= 2 @@ -781,10 +772,6 @@ assert l is l0 assert l == [1.1, 2.2, 1.1, 2.2] - l = range(2) - l *= 2 - assert l == [0, 1, 0, 1] - def test_mul_errors(self): try: [1, 2, 3] * (3,) @@ -792,7 +779,7 @@ pass def test_index(self): - c = range(10) + c = list(range(10)) assert c.index(0) == 0 raises(ValueError, c.index, 10) @@ -828,13 +815,13 @@ assert c.index('l', None, 4) == 2 def test_ass_slice(self): - l = range(6) + l = list(range(6)) l[1:3] = 'abc' assert l == [0, 'a', 'b', 'c', 3, 4, 5] l = [] l[:-3] = [] assert l == [] - l = range(6) + l = list(range(6)) l[:] = [] assert l == [] @@ -863,7 +850,7 @@ raises(ValueError, "l[0:2:2] = [1,2,3,4]") raises(ValueError, "l[::2] = []") - l = range(6) + l = list(range(6)) l[::3] = ('a', 'b') assert l == ['a', 1, 2, 'b', 4, 5] @@ -917,11 +904,11 @@ assert l == [0] for x in range(1, 5): l.append(x) - assert l == range(5) + assert l == list(range(5)) - l = range(4) + l = list(range(4)) l.append(4) - assert l == range(5) + assert l == list(range(5)) l = [1,2,3] l.append("a") @@ -967,9 +954,9 @@ raises(IndexError, c.pop) assert len(c) == 0 - l = range(10) + l = list(range(10)) l.pop() - assert l == range(9) + assert l == list(range(9)) l = [1.1, 2.2, 3.3] l.pop() @@ -986,16 +973,16 @@ def __int__(self): return self.x - l = range(10) + l = list(range(10)) x = l.pop(A(-1)) assert x == 9 - assert l == range(9) - raises(TypeError, range(10).pop, 1.0) + assert l == list(range(9)) + raises(TypeError, list(range(10)).pop, 1.0) def test_pop_negative(self): l1 = [1,2,3,4] l2 = ["1", "2", "3", "4"] - l3 = range(5) + l3 = list(range(5)) l4 = [1, 2, 3, "4"] l5 = [1.1, 2.2, 3.3, 4.4] @@ -1022,7 +1009,7 @@ raises(ValueError, c.remove, 'l') assert ''.join(c) == 'heo word' - l = range(5) + l = list(range(5)) l.remove(2) assert l == [0, 1, 3, 4] l = [0, 3, 5] @@ -1090,7 +1077,7 @@ skip("not reliable on top of Boehm") class A(object): def __del__(self): - print 'del' + print('del') del lst[:] for i in range(10): keepalive = [] @@ -1099,40 +1086,18 @@ while lst: keepalive.append(lst[:]) - def test___getslice__(self): - l = [1,2,3,4] - res = l.__getslice__(0, 2) - assert res == [1, 2] - - l = [] - assert l.__getslice__(0,2) == [] - - def test___setslice__(self): - l = [1,2,3,4] - l.__setslice__(0, 2, [5, 6]) - assert l == [5, 6, 3, 4] - - l = [] - l.__setslice__(0,0,[3,4,5]) - assert l == [3,4,5] - - def test___delslice__(self): - l = [1,2,3,4] - l.__delslice__(0, 2) - assert l == [3, 4] - def test_unicode(self): - s = u"\ufffd\ufffd\ufffd" - assert s.encode("ascii", "replace") == "???" - assert s.encode("ascii", "ignore") == "" + s = "\ufffd\ufffd\ufffd" + assert s.encode("ascii", "replace") == b"???" + assert s.encode("ascii", "ignore") == b"" l1 = [s.encode("ascii", "replace")] - assert l1[0] == "???" + assert l1[0] == b"???" l2 = [s.encode("ascii", "ignore")] - assert l2[0] == "" + assert l2[0] == b"" l3 = [s] - assert l1[0].encode("ascii", "replace") == "???" + assert l3[0].encode("ascii", "replace") == b"???" def test_list_from_set(self): l = ['a'] @@ -1157,7 +1122,7 @@ def test_getitem(self): import operator l = [0, 1, 2, 3, 4] - for i in xrange(5): + for i in range(5): assert l[i] == i assert l[3:] == [3, 4] raises(TypeError, operator.getitem, l, "str") From noreply at buildbot.pypy.org Sat Jan 28 17:57:24 2012 From: noreply at buildbot.pypy.org (amauryfa) Date: Sat, 28 Jan 2012 17:57:24 +0100 (CET) Subject: [pypy-commit] pypy py3k: Fix geninterp tests, and give up with the unicode tests: Message-ID: <20120128165724.9918B82114@wyvern.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc <amauryfa at gmail.com> Branch: py3k Changeset: r51915:d6dfa54f243c Date: 2012-01-28 10:04 +0100 http://bitbucket.org/pypy/pypy/changeset/d6dfa54f243c/ Log: Fix geninterp tests, and give up with the unicode tests: Parsing 2.x code to build 3.x operations is not a good idea anyway. diff --git a/pypy/translator/test/snippet.py b/pypy/translator/test/snippet.py --- a/pypy/translator/test/snippet.py +++ b/pypy/translator/test/snippet.py @@ -657,7 +657,7 @@ return issubclass(type(x), (int, long)) and issubclass(type(y), int) def t_neg_long(): - return -132L + return -132 def t_attrerror(x): try: diff --git a/pypy/translator/test/test_geninterp.py b/pypy/translator/test/test_geninterp.py --- a/pypy/translator/test/test_geninterp.py +++ b/pypy/translator/test/test_geninterp.py @@ -43,9 +43,6 @@ def import_sys_func(): import sys return sys.__name__ - - def unicode_test(x): - return unicode(x, 'ascii') """ def setup_class(cls): @@ -288,12 +285,7 @@ def test_negative_long(self): fn = self.build_interpfunc(snippet.t_neg_long) result = fn() - assert result == -132L and type(result) is long - - def test_unicode_with_encoding(self): - fn = self.build_interpfunc(snippet.unicode_test) - result = fn("abc") - assert result == u"abc" and type(result) is unicode + assert result == -132 and type(result) is int def test_attributeerror(self): fn = self.build_interpfunc(snippet.t_attrerror) From noreply at buildbot.pypy.org Sat Jan 28 17:57:25 2012 From: noreply at buildbot.pypy.org (amauryfa) Date: Sat, 28 Jan 2012 17:57:25 +0100 (CET) Subject: [pypy-commit] pypy py3k: Fix tests for the Thunk Object Space Message-ID: <20120128165725.D2B7382114@wyvern.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc <amauryfa at gmail.com> Branch: py3k Changeset: r51916:8f7805107a60 Date: 2012-01-28 10:08 +0100 http://bitbucket.org/pypy/pypy/changeset/8f7805107a60/ Log: Fix tests for the Thunk Object Space diff --git a/pypy/objspace/test/test_thunkobjspace.py b/pypy/objspace/test/test_thunkobjspace.py --- a/pypy/objspace/test/test_thunkobjspace.py +++ b/pypy/objspace/test/test_thunkobjspace.py @@ -170,12 +170,12 @@ from __pypy__ import thunk d = {} # need the method to use the pypy compiler - exec """if 1: + exec("""if 1: def f(x): return [x] def g(l): l.append(1) - """ in d + """, d) l = thunk(d['f'], 10) d['g'](l) assert l == [10, 1] diff --git a/pypy/objspace/thunk.py b/pypy/objspace/thunk.py --- a/pypy/objspace/thunk.py +++ b/pypy/objspace/thunk.py @@ -129,8 +129,7 @@ def lazy(space, w_callable): """Decorator to make a callable return its results wrapped in a thunk.""" - meth = Method(space, space.w_fn_thunk, - w_callable, space.type(w_callable)) + meth = Method(space, space.w_fn_thunk, w_callable) return space.wrap(meth) app_lazy = gateway.interp2app(lazy) @@ -146,6 +145,7 @@ # ---- irregular operations ---- 'wrap': 0, 'str_w': 1, + 'bytes_w': 1, 'int_w': 1, 'float_w': 1, 'uint_w': 1, From noreply at buildbot.pypy.org Sat Jan 28 17:57:27 2012 From: noreply at buildbot.pypy.org (amauryfa) Date: Sat, 28 Jan 2012 17:57:27 +0100 (CET) Subject: [pypy-commit] pypy py3k: Fix tuple tests Message-ID: <20120128165727.169BA82114@wyvern.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc <amauryfa at gmail.com> Branch: py3k Changeset: r51917:975692f436ef Date: 2012-01-28 13:46 +0100 http://bitbucket.org/pypy/pypy/changeset/975692f436ef/ Log: Fix tuple tests diff --git a/pypy/objspace/std/test/test_specialisedtupleobject.py b/pypy/objspace/std/test/test_specialisedtupleobject.py --- a/pypy/objspace/std/test/test_specialisedtupleobject.py +++ b/pypy/objspace/std/test/test_specialisedtupleobject.py @@ -79,7 +79,7 @@ def w_isspecialised(self, obj, expected=''): import __pypy__ r = __pypy__.internal_repr(obj) - print obj, '==>', r, ' (expected: %r)' % expected + print(obj, '==>', r, ' (expected: %r)' % expected) return ("SpecialisedTupleObject" + expected) in r def test_createspecialisedtuple(self): @@ -105,7 +105,7 @@ def test_delegation(self): t = self.forbid_delegation((42, 43)) - raises(ReferenceError, t.__getslice__, 0, 1) + raises(ReferenceError, t.__contains__, 0) def test_len(self): t = self.forbid_delegation((42,43)) @@ -147,7 +147,7 @@ b = (1,3,2) assert not a == b - values = [2, 2L, 2.0, 1, 1L, 1.0] + values = [2, 2.0, 1, 1.0] for x in values: for y in values: assert ((1,2) == (x,y)) == (1 == x and 2 == y) @@ -204,7 +204,7 @@ c = (2,4) assert hash(a) != hash(c) - assert hash(a) == hash((1L, 2L)) == hash((1.0, 2.0)) == hash((1.0, 2L)) + assert hash(a) == hash((1, 2)) == hash((1.0, 2.0)) == hash((1.0, 2)) def test_getitem(self): t = self.forbid_delegation((5,3)) diff --git a/pypy/objspace/std/test/test_tupleobject.py b/pypy/objspace/std/test/test_tupleobject.py --- a/pypy/objspace/std/test/test_tupleobject.py +++ b/pypy/objspace/std/test/test_tupleobject.py @@ -257,10 +257,10 @@ def test_iter(self): t = (5,3,99) i = iter(t) - assert i.next() == 5 - assert i.next() == 3 - assert i.next() == 99 - raises(StopIteration, i.next) + assert next(i) == 5 + assert next(i) == 3 + assert next(i) == 99 + raises(StopIteration, next, i) def test_contains(self): t = (5,3,99) @@ -332,7 +332,7 @@ assert repr((1,2,3)) == '(1, 2, 3)' def test_getslice(self): - assert ('a', 'b', 'c').__getslice__(-17, 2) == ('a', 'b') + assert ('a', 'b', 'c')[-17: 2] == ('a', 'b') def test_count(self): assert ().count(4) == 0 From noreply at buildbot.pypy.org Sat Jan 28 17:57:28 2012 From: noreply at buildbot.pypy.org (amauryfa) Date: Sat, 28 Jan 2012 17:57:28 +0100 (CET) Subject: [pypy-commit] pypy py3k: Fix most of test_long Message-ID: <20120128165728.50EF082114@wyvern.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc <amauryfa at gmail.com> Branch: py3k Changeset: r51918:71d4eb2be749 Date: 2012-01-28 13:52 +0100 http://bitbucket.org/pypy/pypy/changeset/71d4eb2be749/ Log: Fix most of test_long diff --git a/pypy/objspace/std/test/test_longobject.py b/pypy/objspace/std/test/test_longobject.py --- a/pypy/objspace/std/test/test_longobject.py +++ b/pypy/objspace/std/test/test_longobject.py @@ -212,7 +212,7 @@ assert repr(12345678901234567890) == '12345678901234567890' assert str(12345678901234567890) == '12345678901234567890' assert hex(0x1234567890ABCDEF) == '0x1234567890abcdef' - assert oct(01234567012345670) == '01234567012345670' + assert oct(0o1234567012345670) == '0o1234567012345670' def test_bits(self): x = 0xAAAAAAAA diff --git a/pypy/objspace/std/test/test_smalllongobject.py b/pypy/objspace/std/test/test_smalllongobject.py --- a/pypy/objspace/std/test/test_smalllongobject.py +++ b/pypy/objspace/std/test/test_smalllongobject.py @@ -46,22 +46,22 @@ def test_sl_simple(self): import __pypy__ - s = __pypy__.internal_repr(5L) + s = __pypy__.internal_repr(5) assert 'SmallLong' in s def test_sl_hash(self): import __pypy__ - x = 5L + x = 5 assert 'SmallLong' in __pypy__.internal_repr(x) assert hash(5) == hash(x) - biglong = 5L + biglong = 5 biglong ^= 2**100 # hack based on the fact that xor__Long_Long biglong ^= 2**100 # does not call newlong() - assert biglong == 5L + assert biglong == 5 assert 'SmallLong' not in __pypy__.internal_repr(biglong) assert hash(5) == hash(biglong) # - x = 0x123456789ABCDEFL + x = 0x123456789ABCDEF assert 'SmallLong' in __pypy__.internal_repr(x) biglong = x biglong ^= 2**100 @@ -71,7 +71,7 @@ assert hash(biglong) == hash(x) def test_sl_int(self): - x = 0x123456789ABCDEFL + x = 0x123456789ABCDEF two = 2 assert int(x) == x assert type(int(x)) == type(0x1234567 ** two) @@ -81,42 +81,42 @@ def test_sl_long(self): import __pypy__ - x = long(0) + x = int(0) assert 'SmallLong' in __pypy__.internal_repr(x) def test_sl_add(self): import __pypy__ - x = 0x123456789ABCDEFL - assert x + x == 0x2468ACF13579BDEL + x = 0x123456789ABCDEF + assert x + x == 0x2468ACF13579BDE assert 'SmallLong' in __pypy__.internal_repr(x + x) - x = -0x123456789ABCDEFL - assert x + x == -0x2468ACF13579BDEL + x = -0x123456789ABCDEF + assert x + x == -0x2468ACF13579BDE assert 'SmallLong' in __pypy__.internal_repr(x + x) - x = 0x723456789ABCDEF0L - assert x + x == 0xE468ACF13579BDE0L + x = 0x723456789ABCDEF0 + assert x + x == 0xE468ACF13579BDE0 assert 'SmallLong' not in __pypy__.internal_repr(x + x) - x = -0x723456789ABCDEF0L - assert x + x == -0xE468ACF13579BDE0L + x = -0x723456789ABCDEF0 + assert x + x == -0xE468ACF13579BDE0 assert 'SmallLong' not in __pypy__.internal_repr(x + x) def test_sl_add_32(self): import sys, __pypy__ if sys.maxint == 2147483647: x = 2147483647 - assert x + x == 4294967294L + assert x + x == 4294967294 assert 'SmallLong' in __pypy__.internal_repr(x + x) y = -1 - assert x - y == 2147483648L + assert x - y == 2147483648 assert 'SmallLong' in __pypy__.internal_repr(x - y) def test_sl_lshift(self): - for x in [1, 1L]: - assert x << 1 == 2L - assert x << 30 == 1073741824L - assert x << 31 == 2147483648L - assert x << 32 == 4294967296L - assert x << 62 == 4611686018427387904L - assert x << 63 == 9223372036854775808L - assert x << 64 == 18446744073709551616L - assert (x << 31) << 31 == 4611686018427387904L - assert (x << 32) << 32 == 18446744073709551616L + x = 1 + assert x << 1 == 2 + assert x << 30 == 1073741824 + assert x << 31 == 2147483648 + assert x << 32 == 4294967296 + assert x << 62 == 4611686018427387904 + assert x << 63 == 9223372036854775808 + assert x << 64 == 18446744073709551616 + assert (x << 31) << 31 == 4611686018427387904 + assert (x << 32) << 32 == 18446744073709551616 From noreply at buildbot.pypy.org Sat Jan 28 17:57:29 2012 From: noreply at buildbot.pypy.org (amauryfa) Date: Sat, 28 Jan 2012 17:57:29 +0100 (CET) Subject: [pypy-commit] pypy py3k: Easy fixes in test_index Message-ID: <20120128165729.881D082114@wyvern.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc <amauryfa at gmail.com> Branch: py3k Changeset: r51919:26e3b9127ae8 Date: 2012-01-28 14:00 +0100 http://bitbucket.org/pypy/pypy/changeset/26e3b9127ae8/ Log: Easy fixes in test_index diff --git a/pypy/objspace/std/test/test_index.py b/pypy/objspace/std/test/test_index.py --- a/pypy/objspace/std/test/test_index.py +++ b/pypy/objspace/std/test/test_index.py @@ -32,12 +32,6 @@ return self return TrapInt""") - w_TrapLong = self.space.appexec([], """(): - class TrapLong(long): - def __index__(self): - return self - return TrapLong""") - self.w_oldstyle = w_oldstyle self.w_o = self.space.call_function(w_oldstyle) self.w_o_no_index = self.space.call_function(w_oldstyle_no_index) @@ -46,7 +40,6 @@ self.w_n_no_index = self.space.call_function(w_newstyle_no_index) self.w_TrapInt = w_TrapInt - self.w_TrapLong = w_TrapLong def test_basic(self): self.o.ind = -2 @@ -78,16 +71,14 @@ self.o.ind = 4 self.n.ind = 5 assert 6 .__index__() == 6 - assert -7L.__index__() == -7 + assert -7 .__index__() == -7 assert self.o.__index__() == 4 assert self.n.__index__() == 5 def test_subclasses(self): r = range(10) assert r[self.TrapInt(5):self.TrapInt(10)] == r[5:10] - assert r[self.TrapLong(5):self.TrapLong(10)] == r[5:10] assert slice(self.TrapInt()).indices(0) == (0,0,1) - assert slice(self.TrapLong(0)).indices(0) == (0,0,1) def test_error(self): self.o.ind = 'dumb' @@ -121,19 +112,12 @@ return self return TrapInt""") - w_TrapLong = self.space.appexec([], """(): - class TrapLong(long): - def __index__(self): - return self - return TrapLong""") - self.w_o = self.space.call_function(w_oldstyle) self.w_n = self.space.call_function(w_newstyle) self.w_o2 = self.space.call_function(w_oldstyle) self.w_n2 = self.space.call_function(w_newstyle) self.w_TrapInt = w_TrapInt - self.w_TrapLong = w_TrapLong def test_index(self): self.o.ind = -2 @@ -169,7 +153,6 @@ def test_subclasses(self): assert self.seq[self.TrapInt()] == self.seq[0] - assert self.seq[self.TrapLong()] == self.seq[0] def test_error(self): self.o.ind = 'dumb' @@ -261,12 +244,6 @@ SeqTestCase.setup_method(self, method) self.w_seq = self.space.wrap("this is a test") self.w_const = self.space.appexec([], """(): return str""") - -class AppTest_UnicodeTestCase(SeqTestCase, StringTestCase): - def setup_method(self, method): - SeqTestCase.setup_method(self, method) - self.w_seq = self.space.wrap(u"this is a test") - self.w_const = self.space.appexec([], """(): return unicode""") class AppTest_RangeTestCase: @@ -297,8 +274,6 @@ return maxint def __getitem__(self, key): return key - def __getslice__(self, i, j): - return i, j x = GetItem() assert x[self.pos] == self.pos assert x[self.neg] == self.neg @@ -307,8 +282,8 @@ def test_getslice_nolength(self): class X(object): - def __getslice__(self, i, j): - return (i, j) + def __getitem__(self, slice): + return slice assert X()[-2:1] == (-2, 1) @@ -320,8 +295,6 @@ return maxint def __getitem__(self, key): return key - def __getslice__(self, i, j): - return i, j x = GetItem() assert x[self.pos] == self.pos assert x[self.neg] == self.neg From noreply at buildbot.pypy.org Sat Jan 28 17:57:30 2012 From: noreply at buildbot.pypy.org (amauryfa) Date: Sat, 28 Jan 2012 17:57:30 +0100 (CET) Subject: [pypy-commit] pypy py3k: Fixes in test_sysmodule Message-ID: <20120128165730.C8AC482114@wyvern.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc <amauryfa at gmail.com> Branch: py3k Changeset: r51920:f665d2b4c34d Date: 2012-01-28 14:22 +0100 http://bitbucket.org/pypy/pypy/changeset/f665d2b4c34d/ Log: Fixes in test_sysmodule diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py --- a/pypy/module/sys/test/test_sysmodule.py +++ b/pypy/module/sys/test/test_sysmodule.py @@ -29,29 +29,29 @@ def test_builtin_in_modules(self): import sys modules = sys.modules - assert '__builtin__' in modules, ( "An entry for __builtin__ " - "is not in sys.modules.") - import __builtin__ - builtin2 = sys.modules['__builtin__'] - assert __builtin__ is builtin2, ( "import __builtin__ " - "is not sys.modules[__builtin__].") + assert 'builtins' in modules, ( "An entry for builtins " + "is not in sys.modules.") + import builtins + builtin2 = sys.modules['builtins'] + assert builtins is builtin2, ( "import builtins " + "is not sys.modules[builtins].") def test_builtin_module_names(self): import sys names = sys.builtin_module_names assert 'sys' in names, ( - "sys is not listed as a builtin module.") - assert '__builtin__' in names, ( - "__builtin__ is not listed as a builtin module.") + "sys is not listed as a builtin module.") + assert 'builtins' in names, ( + "builtins is not listed as a builtin module.") def test_sys_exc_info(self): try: raise Exception - except Exception,e: + except Exception as e: import sys exc_type,exc_val,tb = sys.exc_info() try: raise Exception # 5 lines below the previous one - except Exception,e2: + except Exception as e2: exc_type2,exc_val2,tb2 = sys.exc_info() assert exc_type ==Exception assert exc_val ==e @@ -62,14 +62,14 @@ def test_dynamic_attributes(self): try: raise Exception - except Exception,e: + except Exception as e: import sys exc_type = sys.exc_type exc_val = sys.exc_value tb = sys.exc_traceback try: raise Exception # 7 lines below the previous one - except Exception,e2: + except Exception as e2: exc_type2 = sys.exc_type exc_val2 = sys.exc_value tb2 = sys.exc_traceback @@ -87,20 +87,20 @@ etype, val, tb = sys.exc_info() assert isinstance(val, etype) else: - raise AssertionError, "ZeroDivisionError not caught" + raise AssertionError("ZeroDivisionError not caught") def test_io(self): - import sys - assert isinstance(sys.__stdout__, file) - assert isinstance(sys.__stderr__, file) - assert isinstance(sys.__stdin__, file) + import sys, io + assert isinstance(sys.__stdout__, io.IOBase) + assert isinstance(sys.__stderr__, io.IOBase) + assert isinstance(sys.__stdin__, io.IOBase) - if self.appdirect and not isinstance(sys.stdin, file): + if self.appdirect and not isinstance(sys.stdin, io.IOBase): return - assert isinstance(sys.stdout, file) - assert isinstance(sys.stderr, file) - assert isinstance(sys.stdin, file) + assert isinstance(sys.stdout, io.IOBase) + assert isinstance(sys.stderr, io.IOBase) + assert isinstance(sys.stdin, io.IOBase) def test_getfilesystemencoding(self): import sys @@ -135,9 +135,6 @@ assert isinstance(li.inf, int) assert isinstance(li.nan, int) assert isinstance(li.imag, int) - print(li) - sys.stdout.flush() - assert 0 class AppTestSysModulePortedFromCPython: @@ -145,23 +142,23 @@ cls.w_appdirect = cls.space.wrap(option.runappdirect) def test_original_displayhook(self): - import sys, cStringIO, __builtin__ + import sys, _io, builtins savestdout = sys.stdout - out = cStringIO.StringIO() + out = _io.StringIO() sys.stdout = out dh = sys.__displayhook__ raises(TypeError, dh) - if hasattr(__builtin__, "_"): - del __builtin__._ + if hasattr(builtins, "_"): + del builtins._ dh(None) assert out.getvalue() == "" - assert not hasattr(__builtin__, "_") + assert not hasattr(builtins, "_") dh("hello") assert out.getvalue() == "'hello'\n" - assert __builtin__._ == "hello" + assert builtins._ == "hello" del sys.stdout raises(RuntimeError, dh, 42) @@ -187,9 +184,9 @@ sys.displayhook = olddisplayhook def test_original_excepthook(self): - import sys, cStringIO + import sys, _io savestderr = sys.stderr - err = cStringIO.StringIO() + err = _io.StringIO() sys.stderr = err eh = sys.__excepthook__ @@ -197,7 +194,7 @@ raises(TypeError, eh) try: raise ValueError(42) - except ValueError, exc: + except ValueError as exc: eh(*sys.exc_info()) sys.stderr = savestderr @@ -206,16 +203,16 @@ def test_excepthook_failsafe_path(self): import traceback original_print_exception = traceback.print_exception - import sys, cStringIO + import sys, _io savestderr = sys.stderr - err = cStringIO.StringIO() + err = _io.StringIO() sys.stderr = err try: traceback.print_exception = "foo" eh = sys.__excepthook__ try: raise ValueError(42) - except ValueError, exc: + except ValueError as exc: eh(*sys.exc_info()) finally: traceback.print_exception = original_print_exception @@ -247,8 +244,8 @@ def clear(): try: - raise ValueError, 42 - except ValueError, exc: + raise ValueError(42) + except ValueError as exc: clear_check(exc) # Raise an exception and check that it can be cleared @@ -257,8 +254,8 @@ # Verify that a frame currently handling an exception is # unaffected by calling exc_clear in a nested frame. try: - raise ValueError, 13 - except ValueError, exc: + raise ValueError(13) + except ValueError as exc: typ1, value1, traceback1 = sys.exc_info() clear() typ2, value2, traceback2 = sys.exc_info() @@ -278,53 +275,53 @@ # call without argument try: sys.exit(0) - except SystemExit, exc: + except SystemExit as exc: assert exc.code == 0 except: - raise AssertionError, "wrong exception" + raise AssertionError("wrong exception") else: - raise AssertionError, "no exception" + raise AssertionError("no exception") # call with tuple argument with one entry # entry will be unpacked try: sys.exit(42) - except SystemExit, exc: + except SystemExit as exc: assert exc.code == 42 except: - raise AssertionError, "wrong exception" + raise AssertionError("wrong exception") else: - raise AssertionError, "no exception" + raise AssertionError("no exception") # call with integer argument try: sys.exit((42,)) - except SystemExit, exc: + except SystemExit as exc: assert exc.code == 42 except: - raise AssertionError, "wrong exception" + raise AssertionError("wrong exception") else: - raise AssertionError, "no exception" + raise AssertionError("no exception") # call with string argument try: sys.exit("exit") - except SystemExit, exc: + except SystemExit as exc: assert exc.code == "exit" except: - raise AssertionError, "wrong exception" + raise AssertionError("wrong exception") else: - raise AssertionError, "no exception" + raise AssertionError("no exception") # call with tuple argument with two entries try: sys.exit((17, 23)) - except SystemExit, exc: + except SystemExit as exc: assert exc.code == (17, 23) except: - raise AssertionError, "wrong exception" + raise AssertionError("wrong exception") else: - raise AssertionError, "no exception" + raise AssertionError("no exception") def test_getdefaultencoding(self): import sys @@ -341,11 +338,11 @@ try: sys.setdefaultencoding("ascii") assert sys.getdefaultencoding() == 'ascii' - raises(UnicodeDecodeError, unicode, '\x80') + raises(UnicodeDecodeError, unicode, b'\x80') sys.setdefaultencoding("latin-1") assert sys.getdefaultencoding() == 'latin-1' - assert unicode('\x80') == u'\u0080' + assert unicode(b'\x80') == '\u0080' finally: sys.setdefaultencoding(encoding) @@ -459,16 +456,16 @@ assert isinstance(sys.argv, list) assert sys.byteorder in ("little", "big") assert isinstance(sys.builtin_module_names, tuple) - assert isinstance(sys.copyright, basestring) - #assert isinstance(sys.exec_prefix, basestring) -- not present! - assert isinstance(sys.executable, basestring) + assert isinstance(sys.copyright, str) + #assert isinstance(sys.exec_prefix, str) -- not present! + assert isinstance(sys.executable, str) assert isinstance(sys.hexversion, int) assert isinstance(sys.maxint, int) assert isinstance(sys.maxsize, int) assert isinstance(sys.maxunicode, int) - assert isinstance(sys.platform, basestring) - #assert isinstance(sys.prefix, basestring) -- not present! - assert isinstance(sys.version, basestring) + assert isinstance(sys.platform, str) + #assert isinstance(sys.prefix, str) -- not present! + assert isinstance(sys.version, str) assert isinstance(sys.warnoptions, list) vi = sys.version_info assert isinstance(vi, tuple) @@ -528,24 +525,6 @@ if hgid != '': assert hgid in sys.version - def test_trace_exec_execfile(self): - import sys - found = [] - def do_tracing(f, *args): - print f.f_code.co_filename, f.f_lineno, args - if f.f_code.co_filename == 'foobar': - found.append(args[0]) - return do_tracing - co = compile("execfile('this-file-does-not-exist!')", - 'foobar', 'exec') - sys.settrace(do_tracing) - try: - exec co in {} - except IOError: - pass - sys.settrace(None) - assert found == ['call', 'line', 'exception', 'return'] - def test_float_repr_style(self): import sys @@ -557,7 +536,7 @@ def test_current_frames(self): try: - import thread + import _thread except ImportError: pass else: @@ -567,7 +546,7 @@ def f(): return sys._current_frames() frames = f() - assert frames.keys() == [0] + assert list(frames) == [0] assert frames[0].f_code.co_name in ('f', '?') class AppTestCurrentFramesWithThread(AppTestCurrentFrames): @@ -577,23 +556,23 @@ def test_current_frames(self): import sys import time - import thread + import _thread # XXX workaround for now: to prevent deadlocks, call # sys._current_frames() once before starting threads. # This is an issue in non-translated versions only. sys._current_frames() - thread_id = thread.get_ident() + thread_id = _thread.get_ident() def other_thread(): - print "thread started" + print("thread started") lock2.release() lock1.acquire() - lock1 = thread.allocate_lock() - lock2 = thread.allocate_lock() + lock1 = _thread.allocate_lock() + lock2 = _thread.allocate_lock() lock1.acquire() lock2.acquire() - thread.start_new_thread(other_thread, ()) + _thread.start_new_thread(other_thread, ()) def f(): lock2.acquire() From noreply at buildbot.pypy.org Sat Jan 28 17:57:32 2012 From: noreply at buildbot.pypy.org (amauryfa) Date: Sat, 28 Jan 2012 17:57:32 +0100 (CET) Subject: [pypy-commit] pypy py3k: Remove sys.setdefaultencoding Message-ID: <20120128165732.0FFDD82114@wyvern.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc <amauryfa at gmail.com> Branch: py3k Changeset: r51921:1ff9e2433ebb Date: 2012-01-28 14:28 +0100 http://bitbucket.org/pypy/pypy/changeset/1ff9e2433ebb/ Log: Remove sys.setdefaultencoding diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py --- a/pypy/module/cpyext/unicodeobject.py +++ b/pypy/module/cpyext/unicodeobject.py @@ -258,15 +258,6 @@ i += 1 return default_encoding - at cpython_api([CONST_STRING], rffi.INT_real, error=-1) -def PyUnicode_SetDefaultEncoding(space, encoding): - """Sets the currently active default encoding. Returns 0 on - success, -1 in case of an error.""" - w_encoding = space.wrap(rffi.charp2str(encoding)) - setdefaultencoding(space, w_encoding) - default_encoding[0] = '\x00' - return 0 - @cpython_api([PyObject, CONST_STRING, CONST_STRING], PyObject) def PyUnicode_AsEncodedObject(space, w_unicode, llencoding, llerrors): """Encode a Unicode object and return the result as Python object. diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py --- a/pypy/module/sys/__init__.py +++ b/pypy/module/sys/__init__.py @@ -7,7 +7,6 @@ class Module(MixedModule): """Sys Builtin Module. """ - _immutable_fields_ = ["defaultencoding?"] def __init__(self, space, w_name): """NOT_RPYTHON""" # because parent __init__ isn't @@ -16,7 +15,7 @@ super(Module, self).__init__(space, w_name) self.recursionlimit = 100 self.w_default_encoder = None - self.defaultencoding = "ascii" + self.defaultencoding = "utf-8" self.filesystemencoding = None interpleveldefs = { @@ -74,7 +73,6 @@ 'dont_write_bytecode' : 'space.w_False', 'getdefaultencoding' : 'interp_encoding.getdefaultencoding', - 'setdefaultencoding' : 'interp_encoding.setdefaultencoding', 'getfilesystemencoding' : 'interp_encoding.getfilesystemencoding', 'float_info' : 'system.get_float_info(space)', diff --git a/pypy/module/sys/interp_encoding.py b/pypy/module/sys/interp_encoding.py --- a/pypy/module/sys/interp_encoding.py +++ b/pypy/module/sys/interp_encoding.py @@ -7,17 +7,6 @@ implementation.""" return space.wrap(space.sys.defaultencoding) -def setdefaultencoding(space, w_encoding): - """Set the current default string encoding used by the Unicode -implementation.""" - encoding = space.str_w(w_encoding) - mod = space.getbuiltinmodule("_codecs") - w_lookup = space.getattr(mod, space.wrap("lookup")) - # check whether the encoding is there - space.call_function(w_lookup, w_encoding) - space.sys.w_default_encoder = None - space.sys.defaultencoding = encoding - def get_w_default_encoder(space): assert not (space.config.translating and not we_are_translated()), \ "get_w_default_encoder() should not be called during translation" diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py --- a/pypy/module/sys/test/test_sysmodule.py +++ b/pypy/module/sys/test/test_sysmodule.py @@ -329,25 +329,6 @@ # can't check more than the type, as the user might have changed it assert isinstance(sys.getdefaultencoding(), str) - def test_setdefaultencoding(self): - import sys - if self.appdirect: - skip("not worth running appdirect") - - encoding = sys.getdefaultencoding() - try: - sys.setdefaultencoding("ascii") - assert sys.getdefaultencoding() == 'ascii' - raises(UnicodeDecodeError, unicode, b'\x80') - - sys.setdefaultencoding("latin-1") - assert sys.getdefaultencoding() == 'latin-1' - assert unicode(b'\x80') == '\u0080' - - finally: - sys.setdefaultencoding(encoding) - - # testing sys.settrace() is done in test_trace.py # testing sys.setprofile() is done in test_profile.py From noreply at buildbot.pypy.org Sat Jan 28 17:57:33 2012 From: noreply at buildbot.pypy.org (amauryfa) Date: Sat, 28 Jan 2012 17:57:33 +0100 (CET) Subject: [pypy-commit] pypy py3k: Fix all tests in the io module. Message-ID: <20120128165733.4CAFE82114@wyvern.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc <amauryfa at gmail.com> Branch: py3k Changeset: r51922:91c9b2c580d9 Date: 2012-01-28 17:56 +0100 http://bitbucket.org/pypy/pypy/changeset/91c9b2c580d9/ Log: Fix all tests in the io module. diff --git a/lib_pypy/_struct.py b/lib_pypy/_struct.py --- a/lib_pypy/_struct.py +++ b/lib_pypy/_struct.py @@ -80,14 +80,14 @@ return bytes(res) def pack_signed_int(number,size,le): - if not isinstance(number, (int,long)): + if not isinstance(number, int): raise StructError("argument for i,I,l,L,q,Q,h,H must be integer") if number > 2**(8*size-1)-1 or number < -1*2**(8*size-1): raise OverflowError("Number:%i too large to convert" % number) return pack_int(number,size,le) def pack_unsigned_int(number,size,le): - if not isinstance(number, (int,long)): + if not isinstance(number, int): raise StructError("argument for i,I,l,L,q,Q,h,H must be integer") if number < 0: raise TypeError("can't convert negative long to unsigned") diff --git a/lib_pypy/array.py b/lib_pypy/array.py --- a/lib_pypy/array.py +++ b/lib_pypy/array.py @@ -198,7 +198,7 @@ if self.typecode != "u": raise ValueError("tounicode() may only be called on type 'u' arrays") # XXX performance is not too good - return u"".join(self.tolist()) + return "".join(self.tolist()) def byteswap(self): """Byteswap all items of the array. If the items in the array are not diff --git a/pypy/module/_io/test/test_bytesio.py b/pypy/module/_io/test/test_bytesio.py --- a/pypy/module/_io/test/test_bytesio.py +++ b/pypy/module/_io/test/test_bytesio.py @@ -6,7 +6,7 @@ def test_init(self): import _io - raises(TypeError, _io.BytesIO, u"12345") + raises(TypeError, _io.BytesIO, "12345") def test_init_kwargs(self): import _io diff --git a/pypy/module/_io/test/test_fileio.py b/pypy/module/_io/test/test_fileio.py --- a/pypy/module/_io/test/test_fileio.py +++ b/pypy/module/_io/test/test_fileio.py @@ -32,7 +32,7 @@ def test_open_fd(self): import _io os = self.posix - fd = os.open(self.tmpfile, os.O_RDONLY, 0666) + fd = os.open(self.tmpfile, os.O_RDONLY, 0o666) f = _io.FileIO(fd, "rb", closefd=False) assert f.fileno() == fd assert f.closefd is False diff --git a/pypy/module/_io/test/test_io.py b/pypy/module/_io/test/test_io.py --- a/pypy/module/_io/test/test_io.py +++ b/pypy/module/_io/test/test_io.py @@ -51,7 +51,7 @@ import _io try: raise _io.BlockingIOError(42, "test blocking", 123) - except IOError, e: + except IOError as e: assert isinstance(e, _io.BlockingIOError) assert e.errno == 42 assert e.strerror == "test blocking" @@ -234,23 +234,23 @@ with _io.open(self.tmpfile, "w+", encoding="utf8") as f: p0 = f.tell() - f.write(u"\xff\n") + f.write("\xff\n") p1 = f.tell() - f.write(u"\xff\n") + f.write("\xff\n") p2 = f.tell() f.seek(0) assert f.tell() == p0 res = f.readline() - assert res == u"\xff\n" + assert res == "\xff\n" assert f.tell() == p1 res = f.readline() - assert res == u"\xff\n" + assert res == "\xff\n" assert f.tell() == p2 f.seek(0) for line in f: - assert line == u"\xff\n" + assert line == "\xff\n" raises(IOError, f.tell) assert f.tell() == p2 @@ -267,7 +267,7 @@ import _io with _io.open(self.tmpfile, "w+") as f: - f.write(u"abc") + f.write("abc") with _io.open(self.tmpfile, "w+") as f: f.truncate() @@ -290,13 +290,13 @@ # The BOM is not written again when appending to a non-empty file for charset in ["utf-8-sig", "utf-16", "utf-32"]: with _io.open(self.tmpfile, "w", encoding=charset) as f: - f.write(u"aaa") + f.write("aaa") pos = f.tell() with _io.open(self.tmpfile, "rb") as f: res = f.read() assert res == "aaa".encode(charset) with _io.open(self.tmpfile, "a", encoding=charset) as f: - f.write(u"xxx") + f.write("xxx") with _io.open(self.tmpfile, "rb") as f: res = f.read() assert res == "aaaxxx".encode(charset) diff --git a/pypy/module/_io/test/test_stringio.py b/pypy/module/_io/test/test_stringio.py --- a/pypy/module/_io/test/test_stringio.py +++ b/pypy/module/_io/test/test_stringio.py @@ -2,11 +2,11 @@ def test_stringio(self): import io sio = io.StringIO() - sio.write(u'Hello ') - sio.write(u'world') - assert sio.getvalue() == u'Hello world' + sio.write('Hello ') + sio.write('world') + assert sio.getvalue() == 'Hello world' - assert io.StringIO(u"hello").read() == u'hello' + assert io.StringIO("hello").read() == 'hello' def test_capabilities(self): import io @@ -30,22 +30,22 @@ sio = io.StringIO() sio.close() raises(ValueError, sio.read, 1) - raises(ValueError, sio.write, u"text") + raises(ValueError, sio.write, "text") def testRead(self): import io - buf = u"1234567890" + buf = "1234567890" sio = io.StringIO(buf) assert buf[:1] == sio.read(1) assert buf[1:5] == sio.read(4) assert buf[5:] == sio.read(900) - assert u"" == sio.read() + assert "" == sio.read() def test_seek(self): import io - s = u"1234567890" + s = "1234567890" sio = io.StringIO(s) sio.read(5) @@ -70,24 +70,24 @@ def test_overseek(self): import io - s = u"1234567890" + s = "1234567890" sio = io.StringIO(s) res = sio.seek(11) assert res == 11 res = sio.read() - assert res == u"" + assert res == "" assert sio.tell() == 11 assert sio.getvalue() == s - sio.write(u"") + sio.write("") assert sio.getvalue() == s sio.write(s) - assert sio.getvalue() == s + u"\0" + s + assert sio.getvalue() == s + "\0" + s def test_tell(self): import io - s = u"1234567890" + s = "1234567890" sio = io.StringIO(s) assert sio.tell() == 0 @@ -102,7 +102,7 @@ def test_truncate(self): import io - s = u"1234567890" + s = "1234567890" sio = io.StringIO(s) raises(ValueError, sio.truncate, -1) @@ -113,10 +113,6 @@ res = sio.truncate(4) assert res == 4 assert sio.getvalue() == s[:4] - # truncate() accepts long objects - res = sio.truncate(4L) - assert res == 4 - assert sio.getvalue() == s[:4] assert sio.tell() == 6 sio.seek(0, 2) sio.write(s) @@ -135,7 +131,7 @@ exc_info = raises(TypeError, io.StringIO, 3) assert "int" in exc_info.value.args[0] - sio = io.StringIO(u"") + sio = io.StringIO("") exc_info = raises(TypeError, sio.write, 3) assert "int" in exc_info.value.args[0] @@ -147,103 +143,103 @@ def test_newline_none(self): import io - sio = io.StringIO(u"a\nb\r\nc\rd", newline=None) + sio = io.StringIO("a\nb\r\nc\rd", newline=None) res = list(sio) - assert res == [u"a\n", u"b\n", u"c\n", u"d"] + assert res == ["a\n", "b\n", "c\n", "d"] sio.seek(0) res = sio.read(1) - assert res == u"a" + assert res == "a" res = sio.read(2) - assert res == u"\nb" + assert res == "\nb" res = sio.read(2) - assert res == u"\nc" + assert res == "\nc" res = sio.read(1) - assert res == u"\n" + assert res == "\n" sio = io.StringIO(newline=None) - res = sio.write(u"a\n") + res = sio.write("a\n") assert res == 2 - res = sio.write(u"b\r\n") + res = sio.write("b\r\n") assert res == 3 - res = sio.write(u"c\rd") + res = sio.write("c\rd") assert res == 3 sio.seek(0) res = sio.read() - assert res == u"a\nb\nc\nd" - sio = io.StringIO(u"a\r\nb", newline=None) + assert res == "a\nb\nc\nd" + sio = io.StringIO("a\r\nb", newline=None) res = sio.read(3) - assert res == u"a\nb" + assert res == "a\nb" def test_newline_empty(self): import io - sio = io.StringIO(u"a\nb\r\nc\rd", newline="") + sio = io.StringIO("a\nb\r\nc\rd", newline="") res = list(sio) - assert res == [u"a\n", u"b\r\n", u"c\r", u"d"] + assert res == ["a\n", "b\r\n", "c\r", "d"] sio.seek(0) res = sio.read(4) - assert res == u"a\nb\r" + assert res == "a\nb\r" res = sio.read(2) - assert res == u"\nc" + assert res == "\nc" res = sio.read(1) - assert res == u"\r" + assert res == "\r" sio = io.StringIO(newline="") - res = sio.write(u"a\n") + res = sio.write("a\n") assert res == 2 - res = sio.write(u"b\r") + res = sio.write("b\r") assert res == 2 - res = sio.write(u"\nc") + res = sio.write("\nc") assert res == 2 - res = sio.write(u"\rd") + res = sio.write("\rd") assert res == 2 sio.seek(0) res = list(sio) - assert res == [u"a\n", u"b\r\n", u"c\r", u"d"] + assert res == ["a\n", "b\r\n", "c\r", "d"] def test_newline_lf(self): import io - sio = io.StringIO(u"a\nb\r\nc\rd") + sio = io.StringIO("a\nb\r\nc\rd") res = list(sio) - assert res == [u"a\n", u"b\r\n", u"c\rd"] + assert res == ["a\n", "b\r\n", "c\rd"] def test_newline_cr(self): import io - sio = io.StringIO(u"a\nb\r\nc\rd", newline="\r") + sio = io.StringIO("a\nb\r\nc\rd", newline="\r") res = sio.read() - assert res == u"a\rb\r\rc\rd" + assert res == "a\rb\r\rc\rd" sio.seek(0) res = list(sio) - assert res == [u"a\r", u"b\r", u"\r", u"c\r", u"d"] + assert res == ["a\r", "b\r", "\r", "c\r", "d"] def test_newline_crlf(self): import io - sio = io.StringIO(u"a\nb\r\nc\rd", newline="\r\n") + sio = io.StringIO("a\nb\r\nc\rd", newline="\r\n") res = sio.read() - assert res == u"a\r\nb\r\r\nc\rd" + assert res == "a\r\nb\r\r\nc\rd" sio.seek(0) res = list(sio) - assert res == [u"a\r\n", u"b\r\r\n", u"c\rd"] + assert res == ["a\r\n", "b\r\r\n", "c\rd"] def test_newline_property(self): import io sio = io.StringIO(newline=None) assert sio.newlines is None - sio.write(u"a\n") + sio.write("a\n") assert sio.newlines == "\n" - sio.write(u"b\r\n") + sio.write("b\r\n") assert sio.newlines == ("\n", "\r\n") - sio.write(u"c\rd") + sio.write("c\rd") assert sio.newlines == ("\r", "\n", "\r\n") def test_iterator(self): import io - s = u"1234567890\n" + s = "1234567890\n" sio = io.StringIO(s * 10) assert iter(sio) is sio @@ -282,15 +278,15 @@ import io sio = io.StringIO() - sio.__setstate__((u"no error", u"\n", 0, None)) - sio.__setstate__((u"no error", u"", 0, {"spam": 3})) - raises(ValueError, sio.__setstate__, (u"", u"f", 0, None)) - raises(ValueError, sio.__setstate__, (u"", u"", -1, None)) - raises(TypeError, sio.__setstate__, (b"", u"", 0, None)) - raises(TypeError, sio.__setstate__, (u"", u"", 0.0, None)) - raises(TypeError, sio.__setstate__, (u"", u"", 0, 0)) - raises(TypeError, sio.__setstate__, (u"len-test", 0)) + sio.__setstate__(("no error", "\n", 0, None)) + sio.__setstate__(("no error", "", 0, {"spam": 3})) + raises(ValueError, sio.__setstate__, ("", "f", 0, None)) + raises(ValueError, sio.__setstate__, ("", "", -1, None)) + raises(TypeError, sio.__setstate__, (b"", "", 0, None)) + raises(TypeError, sio.__setstate__, ("", "", 0.0, None)) + raises(TypeError, sio.__setstate__, ("", "", 0, 0)) + raises(TypeError, sio.__setstate__, ("len-test", 0)) raises(TypeError, sio.__setstate__) raises(TypeError, sio.__setstate__, 0) sio.close() - raises(ValueError, sio.__setstate__, (u"closed", u"", 0, None)) + raises(ValueError, sio.__setstate__, ("closed", "", 0, None)) diff --git a/pypy/module/_io/test/test_textio.py b/pypy/module/_io/test/test_textio.py --- a/pypy/module/_io/test/test_textio.py +++ b/pypy/module/_io/test/test_textio.py @@ -15,7 +15,7 @@ t.__init__(b, encoding="utf8", line_buffering=True) assert t.encoding == "utf8" assert t.line_buffering == True - assert t.readline() == u"\xe9\n" + assert t.readline() == "\xe9\n" raises(TypeError, t.__init__, b, newline=42) raises(ValueError, t.__init__, b, newline='xyzzy') t = _io.TextIOWrapper(b) @@ -62,7 +62,7 @@ r = _io.BytesIO(b"abc\r\ndef\rg") b = _io.BufferedReader(r, 1000) t = _io.TextIOWrapper(b) - assert t.read() == u"abc\ndef\ng" + assert t.read() == "abc\ndef\ng" def test_one_by_one(self): import _io @@ -75,7 +75,7 @@ if not c: break reads.append(c) - assert u''.join(reads) == u"abc\ndef\ng" + assert ''.join(reads) == "abc\ndef\ng" def test_read_some_then_all(self): import _io @@ -83,7 +83,7 @@ t = _io.TextIOWrapper(r) reads = t.read(4) reads += t.read() - assert reads == u"abc\ndef\n" + assert reads == "abc\ndef\n" def test_read_some_then_readline(self): import _io @@ -91,11 +91,11 @@ t = _io.TextIOWrapper(r) reads = t.read(4) reads += t.readline() - assert reads == u"abc\ndef\n" + assert reads == "abc\ndef\n" def test_encoded_writes(self): import _io - data = u"1234567890" + data = "1234567890" tests = ("utf-16", "utf-16-le", "utf-16-be", @@ -131,7 +131,7 @@ _io.BytesIO.close(self) b = MyBytesIO() t = _io.TextIOWrapper(b, encoding="ascii") - t.write(u"abc") + t.write("abc") del t import gc; gc.collect() assert l == [b"abc"] @@ -203,7 +203,7 @@ assert repr(t) == "<_io.TextIOWrapper encoding='utf-8'>" t = _io.TextIOWrapper(_io.BytesIO(b""), encoding="ascii") assert repr(t) == "<_io.TextIOWrapper encoding='ascii'>" - t = _io.TextIOWrapper(_io.BytesIO(b""), encoding=u"utf-8") + t = _io.TextIOWrapper(_io.BytesIO(b""), encoding="utf-8") assert repr(t) == "<_io.TextIOWrapper encoding='utf-8'>" b = _io.BytesIO(b"") t = _io.TextIOWrapper(b, encoding="utf-8") @@ -311,15 +311,15 @@ decoder.setstate(state) assert decoder.decode(b, **kwargs) == s - _check_decode(b'\xe8\xa2\x88', u"\u8888") + _check_decode(b'\xe8\xa2\x88', "\u8888") _check_decode(b'\xe8', "") _check_decode(b'\xa2', "") - _check_decode(b'\x88', u"\u8888") + _check_decode(b'\x88', "\u8888") _check_decode(b'\xe8', "") _check_decode(b'\xa2', "") - _check_decode(b'\x88', u"\u8888") + _check_decode(b'\x88', "\u8888") _check_decode(b'\xe8', "") raises(UnicodeDecodeError, decoder.decode, b'', final=True) @@ -338,10 +338,10 @@ _check_decode(b'\r', "\n") _check_decode(b'\na', "\na") - _check_decode(b'\xe8\xa2\x88\r\n', u"\u8888\n") - _check_decode(b'\xe8\xa2\x88', u"\u8888") + _check_decode(b'\xe8\xa2\x88\r\n', "\u8888\n") + _check_decode(b'\xe8\xa2\x88', "\u8888") _check_decode(b'\n', "\n") - _check_decode(b'\xe8\xa2\x88\r', u"\u8888") + _check_decode(b'\xe8\xa2\x88\r', "\u8888") _check_decode(b'\n', "\n") def check_newline_decoding(decoder, encoding): @@ -359,18 +359,18 @@ for c in s: result.append(decoder.decode(c)) assert decoder.newlines == None - _decode_bytewise(u"abc\n\r") + _decode_bytewise("abc\n\r") assert decoder.newlines == '\n' - _decode_bytewise(u"\nabc") + _decode_bytewise("\nabc") assert decoder.newlines == ('\n', '\r\n') - _decode_bytewise(u"abc\r") + _decode_bytewise("abc\r") assert decoder.newlines == ('\n', '\r\n') - _decode_bytewise(u"abc") + _decode_bytewise("abc") assert decoder.newlines == ('\r', '\n', '\r\n') - _decode_bytewise(u"abc\r") + _decode_bytewise("abc\r") assert "".join(result) == "abc\n\nabcabc\nabcabc" decoder.reset() - input = u"abc" + input = "abc" if encoder is not None: encoder.reset() input = encoder.encode(input) @@ -398,9 +398,9 @@ # Issue 5433: Excessive optimization in IncrementalNewlineDecoder def _check(dec): assert dec.newlines is None - assert dec.decode(u"\u0D00") == u"\u0D00" + assert dec.decode("\u0D00") == "\u0D00" assert dec.newlines is None - assert dec.decode(u"\u0A00") == u"\u0A00" + assert dec.decode("\u0A00") == "\u0A00" assert dec.newlines is None dec = _io.IncrementalNewlineDecoder(None, translate=False) _check(dec) From noreply at buildbot.pypy.org Sat Jan 28 18:51:58 2012 From: noreply at buildbot.pypy.org (mattip) Date: Sat, 28 Jan 2012 18:51:58 +0100 (CET) Subject: [pypy-commit] pypy matrixmath-dot: merge from default Message-ID: <20120128175158.A419F82114@wyvern.cs.uni-duesseldorf.de> Author: mattip Branch: matrixmath-dot Changeset: r51923:57b13475b99a Date: 2012-01-28 19:32 +0200 http://bitbucket.org/pypy/pypy/changeset/57b13475b99a/ Log: merge from default diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py --- a/lib_pypy/_sqlite3.py +++ b/lib_pypy/_sqlite3.py @@ -20,6 +20,8 @@ # 2. Altered source versions must be plainly marked as such, and must not be # misrepresented as being the original software. # 3. This notice may not be removed or altered from any source distribution. +# +# Note: This software has been modified for use in PyPy. from ctypes import c_void_p, c_int, c_double, c_int64, c_char_p, cdll from ctypes import POINTER, byref, string_at, CFUNCTYPE, cast @@ -27,7 +29,6 @@ from collections import OrderedDict import datetime import sys -import time import weakref from threading import _get_ident as thread_get_ident @@ -606,7 +607,7 @@ def authorizer(userdata, action, arg1, arg2, dbname, source): try: return int(callback(action, arg1, arg2, dbname, source)) - except Exception, e: + except Exception: return SQLITE_DENY c_authorizer = AUTHORIZER(authorizer) @@ -653,7 +654,7 @@ if not aggregate_ptr[0]: try: aggregate = cls() - except Exception, e: + except Exception: msg = ("user-defined aggregate's '__init__' " "method raised error") sqlite.sqlite3_result_error(context, msg, len(msg)) @@ -667,7 +668,7 @@ params = _convert_params(context, argc, c_params) try: aggregate.step(*params) - except Exception, e: + except Exception: msg = ("user-defined aggregate's 'step' " "method raised error") sqlite.sqlite3_result_error(context, msg, len(msg)) @@ -683,7 +684,7 @@ aggregate = self.aggregate_instances[aggregate_ptr[0]] try: val = aggregate.finalize() - except Exception, e: + except Exception: msg = ("user-defined aggregate's 'finalize' " "method raised error") sqlite.sqlite3_result_error(context, msg, len(msg)) @@ -771,7 +772,7 @@ self.statement.item = None self.statement.exhausted = True - if self.statement.kind == DML or self.statement.kind == DDL: + if self.statement.kind == DML: self.statement.reset() self.rowcount = -1 @@ -791,7 +792,7 @@ if self.statement.kind == DML: self.connection._begin() else: - raise ProgrammingError, "executemany is only for DML statements" + raise ProgrammingError("executemany is only for DML statements") self.rowcount = 0 for params in many_params: @@ -861,8 +862,6 @@ except StopIteration: return None - return nextrow - def fetchmany(self, size=None): self._check_closed() self._check_reset() @@ -915,7 +914,7 @@ def __init__(self, connection, sql): self.statement = None if not isinstance(sql, str): - raise ValueError, "sql must be a string" + raise ValueError("sql must be a string") self.con = connection self.sql = sql # DEBUG ONLY first_word = self._statement_kind = sql.lstrip().split(" ")[0].upper() @@ -944,8 +943,8 @@ raise self.con._get_exception(ret) self.con._remember_statement(self) if _check_remaining_sql(next_char.value): - raise Warning, "One and only one statement required: %r" % ( - next_char.value,) + raise Warning("One and only one statement required: %r" % ( + next_char.value,)) # sql_char should remain alive until here self._build_row_cast_map() @@ -1016,7 +1015,7 @@ elif type(param) is buffer: sqlite.sqlite3_bind_blob(self.statement, idx, str(param), len(param), SQLITE_TRANSIENT) else: - raise InterfaceError, "parameter type %s is not supported" % str(type(param)) + raise InterfaceError("parameter type %s is not supported" % str(type(param))) def set_params(self, params): ret = sqlite.sqlite3_reset(self.statement) @@ -1045,11 +1044,11 @@ for idx in range(1, sqlite.sqlite3_bind_parameter_count(self.statement) + 1): param_name = sqlite.sqlite3_bind_parameter_name(self.statement, idx) if param_name is None: - raise ProgrammingError, "need named parameters" + raise ProgrammingError("need named parameters") param_name = param_name[1:] try: param = params[param_name] - except KeyError, e: + except KeyError: raise ProgrammingError("missing parameter '%s'" %param) self.set_param(idx, param) @@ -1260,7 +1259,7 @@ params = _convert_params(context, nargs, c_params) try: val = real_cb(*params) - except Exception, e: + except Exception: msg = "user-defined function raised exception" sqlite.sqlite3_result_error(context, msg, len(msg)) else: diff --git a/lib_pypy/datetime.py b/lib_pypy/datetime.py --- a/lib_pypy/datetime.py +++ b/lib_pypy/datetime.py @@ -13,7 +13,7 @@ Sources for time zone and DST data: http://www.twinsun.com/tz/tz-link.htm This was originally copied from the sandbox of the CPython CVS repository. -Thanks to Tim Peters for suggesting using it. +Thanks to Tim Peters for suggesting using it. """ import time as _time @@ -271,6 +271,8 @@ raise ValueError("%s()=%d, must be in -1439..1439" % (name, offset)) def _check_date_fields(year, month, day): + if not isinstance(year, (int, long)): + raise TypeError('int expected') if not MINYEAR <= year <= MAXYEAR: raise ValueError('year must be in %d..%d' % (MINYEAR, MAXYEAR), year) if not 1 <= month <= 12: @@ -280,6 +282,8 @@ raise ValueError('day must be in 1..%d' % dim, day) def _check_time_fields(hour, minute, second, microsecond): + if not isinstance(hour, (int, long)): + raise TypeError('int expected') if not 0 <= hour <= 23: raise ValueError('hour must be in 0..23', hour) if not 0 <= minute <= 59: @@ -543,61 +547,76 @@ self = object.__new__(cls) - self.__days = d - self.__seconds = s - self.__microseconds = us + self._days = d + self._seconds = s + self._microseconds = us if abs(d) > 999999999: raise OverflowError("timedelta # of days is too large: %d" % d) return self def __repr__(self): - if self.__microseconds: + if self._microseconds: return "%s(%d, %d, %d)" % ('datetime.' + self.__class__.__name__, - self.__days, - self.__seconds, - self.__microseconds) - if self.__seconds: + self._days, + self._seconds, + self._microseconds) + if self._seconds: return "%s(%d, %d)" % ('datetime.' + self.__class__.__name__, - self.__days, - self.__seconds) - return "%s(%d)" % ('datetime.' + self.__class__.__name__, self.__days) + self._days, + self._seconds) + return "%s(%d)" % ('datetime.' + self.__class__.__name__, self._days) def __str__(self): - mm, ss = divmod(self.__seconds, 60) + mm, ss = divmod(self._seconds, 60) hh, mm = divmod(mm, 60) s = "%d:%02d:%02d" % (hh, mm, ss) - if self.__days: + if self._days: def plural(n): return n, abs(n) != 1 and "s" or "" - s = ("%d day%s, " % plural(self.__days)) + s - if self.__microseconds: - s = s + ".%06d" % self.__microseconds + s = ("%d day%s, " % plural(self._days)) + s + if self._microseconds: + s = s + ".%06d" % self._microseconds return s - days = property(lambda self: self.__days, doc="days") - seconds = property(lambda self: self.__seconds, doc="seconds") - microseconds = property(lambda self: self.__microseconds, - doc="microseconds") - def total_seconds(self): return ((self.days * 86400 + self.seconds) * 10**6 + self.microseconds) / 1e6 + # Read-only field accessors + @property + def days(self): + """days""" + return self._days + + @property + def seconds(self): + """seconds""" + return self._seconds + + @property + def microseconds(self): + """microseconds""" + return self._microseconds + def __add__(self, other): if isinstance(other, timedelta): # for CPython compatibility, we cannot use # our __class__ here, but need a real timedelta - return timedelta(self.__days + other.__days, - self.__seconds + other.__seconds, - self.__microseconds + other.__microseconds) + return timedelta(self._days + other._days, + self._seconds + other._seconds, + self._microseconds + other._microseconds) return NotImplemented __radd__ = __add__ def __sub__(self, other): if isinstance(other, timedelta): - return self + -other + # for CPython compatibility, we cannot use + # our __class__ here, but need a real timedelta + return timedelta(self._days - other._days, + self._seconds - other._seconds, + self._microseconds - other._microseconds) return NotImplemented def __rsub__(self, other): @@ -606,17 +625,17 @@ return NotImplemented def __neg__(self): - # for CPython compatibility, we cannot use - # our __class__ here, but need a real timedelta - return timedelta(-self.__days, - -self.__seconds, - -self.__microseconds) + # for CPython compatibility, we cannot use + # our __class__ here, but need a real timedelta + return timedelta(-self._days, + -self._seconds, + -self._microseconds) def __pos__(self): return self def __abs__(self): - if self.__days < 0: + if self._days < 0: return -self else: return self @@ -625,81 +644,81 @@ if isinstance(other, (int, long)): # for CPython compatibility, we cannot use # our __class__ here, but need a real timedelta - return timedelta(self.__days * other, - self.__seconds * other, - self.__microseconds * other) + return timedelta(self._days * other, + self._seconds * other, + self._microseconds * other) return NotImplemented __rmul__ = __mul__ def __div__(self, other): if isinstance(other, (int, long)): - usec = ((self.__days * (24*3600L) + self.__seconds) * 1000000 + - self.__microseconds) + usec = ((self._days * (24*3600L) + self._seconds) * 1000000 + + self._microseconds) return timedelta(0, 0, usec // other) return NotImplemented __floordiv__ = __div__ - # Comparisons. + # Comparisons of timedelta objects with other. def __eq__(self, other): if isinstance(other, timedelta): - return self.__cmp(other) == 0 + return self._cmp(other) == 0 else: return False def __ne__(self, other): if isinstance(other, timedelta): - return self.__cmp(other) != 0 + return self._cmp(other) != 0 else: return True def __le__(self, other): if isinstance(other, timedelta): - return self.__cmp(other) <= 0 + return self._cmp(other) <= 0 else: _cmperror(self, other) def __lt__(self, other): if isinstance(other, timedelta): - return self.__cmp(other) < 0 + return self._cmp(other) < 0 else: _cmperror(self, other) def __ge__(self, other): if isinstance(other, timedelta): - return self.__cmp(other) >= 0 + return self._cmp(other) >= 0 else: _cmperror(self, other) def __gt__(self, other): if isinstance(other, timedelta): - return self.__cmp(other) > 0 + return self._cmp(other) > 0 else: _cmperror(self, other) - def __cmp(self, other): + def _cmp(self, other): assert isinstance(other, timedelta) - return cmp(self.__getstate(), other.__getstate()) + return cmp(self._getstate(), other._getstate()) def __hash__(self): - return hash(self.__getstate()) + return hash(self._getstate()) def __nonzero__(self): - return (self.__days != 0 or - self.__seconds != 0 or - self.__microseconds != 0) + return (self._days != 0 or + self._seconds != 0 or + self._microseconds != 0) # Pickle support. __safe_for_unpickling__ = True # For Python 2.2 - def __getstate(self): - return (self.__days, self.__seconds, self.__microseconds) + def _getstate(self): + return (self._days, self._seconds, self._microseconds) def __reduce__(self): - return (self.__class__, self.__getstate()) + return (self.__class__, self._getstate()) timedelta.min = timedelta(-999999999) timedelta.max = timedelta(days=999999999, hours=23, minutes=59, seconds=59, @@ -749,25 +768,26 @@ return self _check_date_fields(year, month, day) self = object.__new__(cls) - self.__year = year - self.__month = month - self.__day = day + self._year = year + self._month = month + self._day = day return self # Additional constructors + @classmethod def fromtimestamp(cls, t): "Construct a date from a POSIX timestamp (like time.time())." y, m, d, hh, mm, ss, weekday, jday, dst = _time.localtime(t) return cls(y, m, d) - fromtimestamp = classmethod(fromtimestamp) + @classmethod def today(cls): "Construct a date from time.time()." t = _time.time() return cls.fromtimestamp(t) - today = classmethod(today) + @classmethod def fromordinal(cls, n): """Contruct a date from a proleptic Gregorian ordinal. @@ -776,16 +796,24 @@ """ y, m, d = _ord2ymd(n) return cls(y, m, d) - fromordinal = classmethod(fromordinal) # Conversions to string def __repr__(self): - "Convert to formal string, for repr()." + """Convert to formal string, for repr(). + + >>> dt = datetime(2010, 1, 1) + >>> repr(dt) + 'datetime.datetime(2010, 1, 1, 0, 0)' + + >>> dt = datetime(2010, 1, 1, tzinfo=timezone.utc) + >>> repr(dt) + 'datetime.datetime(2010, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)' + """ return "%s(%d, %d, %d)" % ('datetime.' + self.__class__.__name__, - self.__year, - self.__month, - self.__day) + self._year, + self._month, + self._day) # XXX These shouldn't depend on time.localtime(), because that # clips the usable dates to [1970 .. 2038). At least ctime() is # easily done without using strftime() -- that's better too because @@ -793,12 +821,20 @@ def ctime(self): "Format a la ctime()." - return tmxxx(self.__year, self.__month, self.__day).ctime() + return tmxxx(self._year, self._month, self._day).ctime() def strftime(self, fmt): "Format using strftime()." return _wrap_strftime(self, fmt, self.timetuple()) + def __format__(self, fmt): + if not isinstance(fmt, (str, unicode)): + raise ValueError("__format__ excepts str or unicode, not %s" % + fmt.__class__.__name__) + if len(fmt) != 0: + return self.strftime(fmt) + return str(self) + def isoformat(self): """Return the date formatted according to ISO. @@ -808,29 +844,31 @@ - http://www.w3.org/TR/NOTE-datetime - http://www.cl.cam.ac.uk/~mgk25/iso-time.html """ - return "%04d-%02d-%02d" % (self.__year, self.__month, self.__day) + return "%04d-%02d-%02d" % (self._year, self._month, self._day) __str__ = isoformat - def __format__(self, format): - if not isinstance(format, (str, unicode)): - raise ValueError("__format__ excepts str or unicode, not %s" % - format.__class__.__name__) - if not format: - return str(self) - return self.strftime(format) + # Read-only field accessors + @property + def year(self): + """year (1-9999)""" + return self._year - # Read-only field accessors - year = property(lambda self: self.__year, - doc="year (%d-%d)" % (MINYEAR, MAXYEAR)) - month = property(lambda self: self.__month, doc="month (1-12)") - day = property(lambda self: self.__day, doc="day (1-31)") + @property + def month(self): + """month (1-12)""" + return self._month + + @property + def day(self): + """day (1-31)""" + return self._day # Standard conversions, __cmp__, __hash__ (and helpers) def timetuple(self): "Return local time tuple compatible with time.localtime()." - return _build_struct_time(self.__year, self.__month, self.__day, + return _build_struct_time(self._year, self._month, self._day, 0, 0, 0, -1) def toordinal(self): @@ -839,24 +877,24 @@ January 1 of year 1 is day 1. Only the year, month and day values contribute to the result. """ - return _ymd2ord(self.__year, self.__month, self.__day) + return _ymd2ord(self._year, self._month, self._day) def replace(self, year=None, month=None, day=None): """Return a new date with new values for the specified fields.""" if year is None: - year = self.__year + year = self._year if month is None: - month = self.__month + month = self._month if day is None: - day = self.__day + day = self._day _check_date_fields(year, month, day) return date(year, month, day) - # Comparisons. + # Comparisons of date objects with other. def __eq__(self, other): if isinstance(other, date): - return self.__cmp(other) == 0 + return self._cmp(other) == 0 elif hasattr(other, "timetuple"): return NotImplemented else: @@ -864,7 +902,7 @@ def __ne__(self, other): if isinstance(other, date): - return self.__cmp(other) != 0 + return self._cmp(other) != 0 elif hasattr(other, "timetuple"): return NotImplemented else: @@ -872,7 +910,7 @@ def __le__(self, other): if isinstance(other, date): - return self.__cmp(other) <= 0 + return self._cmp(other) <= 0 elif hasattr(other, "timetuple"): return NotImplemented else: @@ -880,7 +918,7 @@ def __lt__(self, other): if isinstance(other, date): - return self.__cmp(other) < 0 + return self._cmp(other) < 0 elif hasattr(other, "timetuple"): return NotImplemented else: @@ -888,7 +926,7 @@ def __ge__(self, other): if isinstance(other, date): - return self.__cmp(other) >= 0 + return self._cmp(other) >= 0 elif hasattr(other, "timetuple"): return NotImplemented else: @@ -896,21 +934,21 @@ def __gt__(self, other): if isinstance(other, date): - return self.__cmp(other) > 0 + return self._cmp(other) > 0 elif hasattr(other, "timetuple"): return NotImplemented else: _cmperror(self, other) - def __cmp(self, other): + def _cmp(self, other): assert isinstance(other, date) - y, m, d = self.__year, self.__month, self.__day - y2, m2, d2 = other.__year, other.__month, other.__day + y, m, d = self._year, self._month, self._day + y2, m2, d2 = other._year, other._month, other._day return cmp((y, m, d), (y2, m2, d2)) def __hash__(self): "Hash." - return hash(self.__getstate()) + return hash(self._getstate()) # Computations @@ -922,9 +960,9 @@ def __add__(self, other): "Add a date to a timedelta." if isinstance(other, timedelta): - t = tmxxx(self.__year, - self.__month, - self.__day + other.days) + t = tmxxx(self._year, + self._month, + self._day + other.days) self._checkOverflow(t.year) result = date(t.year, t.month, t.day) return result @@ -966,9 +1004,9 @@ ISO calendar algorithm taken from http://www.phys.uu.nl/~vgent/calendar/isocalendar.htm """ - year = self.__year + year = self._year week1monday = _isoweek1monday(year) - today = _ymd2ord(self.__year, self.__month, self.__day) + today = _ymd2ord(self._year, self._month, self._day) # Internally, week and day have origin 0 week, day = divmod(today - week1monday, 7) if week < 0: @@ -985,18 +1023,18 @@ __safe_for_unpickling__ = True # For Python 2.2 - def __getstate(self): - yhi, ylo = divmod(self.__year, 256) - return ("%c%c%c%c" % (yhi, ylo, self.__month, self.__day), ) + def _getstate(self): + yhi, ylo = divmod(self._year, 256) + return ("%c%c%c%c" % (yhi, ylo, self._month, self._day), ) def __setstate(self, string): if len(string) != 4 or not (1 <= ord(string[2]) <= 12): raise TypeError("not enough arguments") - yhi, ylo, self.__month, self.__day = map(ord, string) - self.__year = yhi * 256 + ylo + yhi, ylo, self._month, self._day = map(ord, string) + self._year = yhi * 256 + ylo def __reduce__(self): - return (self.__class__, self.__getstate()) + return (self.__class__, self._getstate()) _date_class = date # so functions w/ args named "date" can get at the class @@ -1118,62 +1156,80 @@ return self _check_tzinfo_arg(tzinfo) _check_time_fields(hour, minute, second, microsecond) - self.__hour = hour - self.__minute = minute - self.__second = second - self.__microsecond = microsecond + self._hour = hour + self._minute = minute + self._second = second + self._microsecond = microsecond self._tzinfo = tzinfo return self # Read-only field accessors - hour = property(lambda self: self.__hour, doc="hour (0-23)") - minute = property(lambda self: self.__minute, doc="minute (0-59)") - second = property(lambda self: self.__second, doc="second (0-59)") - microsecond = property(lambda self: self.__microsecond, - doc="microsecond (0-999999)") - tzinfo = property(lambda self: self._tzinfo, doc="timezone info object") + @property + def hour(self): + """hour (0-23)""" + return self._hour + + @property + def minute(self): + """minute (0-59)""" + return self._minute + + @property + def second(self): + """second (0-59)""" + return self._second + + @property + def microsecond(self): + """microsecond (0-999999)""" + return self._microsecond + + @property + def tzinfo(self): + """timezone info object""" + return self._tzinfo # Standard conversions, __hash__ (and helpers) - # Comparisons. + # Comparisons of time objects with other. def __eq__(self, other): if isinstance(other, time): - return self.__cmp(other) == 0 + return self._cmp(other) == 0 else: return False def __ne__(self, other): if isinstance(other, time): - return self.__cmp(other) != 0 + return self._cmp(other) != 0 else: return True def __le__(self, other): if isinstance(other, time): - return self.__cmp(other) <= 0 + return self._cmp(other) <= 0 else: _cmperror(self, other) def __lt__(self, other): if isinstance(other, time): - return self.__cmp(other) < 0 + return self._cmp(other) < 0 else: _cmperror(self, other) def __ge__(self, other): if isinstance(other, time): - return self.__cmp(other) >= 0 + return self._cmp(other) >= 0 else: _cmperror(self, other) def __gt__(self, other): if isinstance(other, time): - return self.__cmp(other) > 0 + return self._cmp(other) > 0 else: _cmperror(self, other) - def __cmp(self, other): + def _cmp(self, other): assert isinstance(other, time) mytz = self._tzinfo ottz = other._tzinfo @@ -1187,23 +1243,23 @@ base_compare = myoff == otoff if base_compare: - return cmp((self.__hour, self.__minute, self.__second, - self.__microsecond), - (other.__hour, other.__minute, other.__second, - other.__microsecond)) + return cmp((self._hour, self._minute, self._second, + self._microsecond), + (other._hour, other._minute, other._second, + other._microsecond)) if myoff is None or otoff is None: # XXX Buggy in 2.2.2. raise TypeError("cannot compare naive and aware times") - myhhmm = self.__hour * 60 + self.__minute - myoff - othhmm = other.__hour * 60 + other.__minute - otoff - return cmp((myhhmm, self.__second, self.__microsecond), - (othhmm, other.__second, other.__microsecond)) + myhhmm = self._hour * 60 + self._minute - myoff + othhmm = other._hour * 60 + other._minute - otoff + return cmp((myhhmm, self._second, self._microsecond), + (othhmm, other._second, other._microsecond)) def __hash__(self): """Hash.""" tzoff = self._utcoffset() if not tzoff: # zero or None - return hash(self.__getstate()[0]) + return hash(self._getstate()[0]) h, m = divmod(self.hour * 60 + self.minute - tzoff, 60) if 0 <= h < 24: return hash(time(h, m, self.second, self.microsecond)) @@ -1227,14 +1283,14 @@ def __repr__(self): """Convert to formal string, for repr().""" - if self.__microsecond != 0: - s = ", %d, %d" % (self.__second, self.__microsecond) - elif self.__second != 0: - s = ", %d" % self.__second + if self._microsecond != 0: + s = ", %d, %d" % (self._second, self._microsecond) + elif self._second != 0: + s = ", %d" % self._second else: s = "" s= "%s(%d, %d%s)" % ('datetime.' + self.__class__.__name__, - self.__hour, self.__minute, s) + self._hour, self._minute, s) if self._tzinfo is not None: assert s[-1:] == ")" s = s[:-1] + ", tzinfo=%r" % self._tzinfo + ")" @@ -1246,8 +1302,8 @@ This is 'HH:MM:SS.mmmmmm+zz:zz', or 'HH:MM:SS+zz:zz' if self.microsecond == 0. """ - s = _format_time(self.__hour, self.__minute, self.__second, - self.__microsecond) + s = _format_time(self._hour, self._minute, self._second, + self._microsecond) tz = self._tzstr() if tz: s += tz @@ -1255,14 +1311,6 @@ __str__ = isoformat - def __format__(self, format): - if not isinstance(format, (str, unicode)): - raise ValueError("__format__ excepts str or unicode, not %s" % - format.__class__.__name__) - if not format: - return str(self) - return self.strftime(format) - def strftime(self, fmt): """Format using strftime(). The date part of the timestamp passed to underlying strftime should not be used. @@ -1270,10 +1318,18 @@ # The year must be >= 1900 else Python's strftime implementation # can raise a bogus exception. timetuple = (1900, 1, 1, - self.__hour, self.__minute, self.__second, + self._hour, self._minute, self._second, 0, 1, -1) return _wrap_strftime(self, fmt, timetuple) + def __format__(self, fmt): + if not isinstance(fmt, (str, unicode)): + raise ValueError("__format__ excepts str or unicode, not %s" % + fmt.__class__.__name__) + if len(fmt) != 0: + return self.strftime(fmt) + return str(self) + # Timezone functions def utcoffset(self): @@ -1350,10 +1406,10 @@ __safe_for_unpickling__ = True # For Python 2.2 - def __getstate(self): - us2, us3 = divmod(self.__microsecond, 256) + def _getstate(self): + us2, us3 = divmod(self._microsecond, 256) us1, us2 = divmod(us2, 256) - basestate = ("%c" * 6) % (self.__hour, self.__minute, self.__second, + basestate = ("%c" * 6) % (self._hour, self._minute, self._second, us1, us2, us3) if self._tzinfo is None: return (basestate,) @@ -1363,13 +1419,13 @@ def __setstate(self, string, tzinfo): if len(string) != 6 or ord(string[0]) >= 24: raise TypeError("an integer is required") - self.__hour, self.__minute, self.__second, us1, us2, us3 = \ + self._hour, self._minute, self._second, us1, us2, us3 = \ map(ord, string) - self.__microsecond = (((us1 << 8) | us2) << 8) | us3 + self._microsecond = (((us1 << 8) | us2) << 8) | us3 self._tzinfo = tzinfo def __reduce__(self): - return (time, self.__getstate()) + return (time, self._getstate()) _time_class = time # so functions w/ args named "time" can get at the class @@ -1378,9 +1434,11 @@ time.resolution = timedelta(microseconds=1) class datetime(date): + """datetime(year, month, day[, hour[, minute[, second[, microsecond[,tzinfo]]]]]) - # XXX needs docstrings - # See http://www.zope.org/Members/fdrake/DateTimeWiki/TimeZoneInfo + The year, month and day arguments are required. tzinfo may be None, or an + instance of a tzinfo subclass. The remaining arguments may be ints or longs. + """ def __new__(cls, year, month=None, day=None, hour=0, minute=0, second=0, microsecond=0, tzinfo=None): @@ -1393,24 +1451,43 @@ _check_time_fields(hour, minute, second, microsecond) self = date.__new__(cls, year, month, day) # XXX This duplicates __year, __month, __day for convenience :-( - self.__year = year - self.__month = month - self.__day = day - self.__hour = hour - self.__minute = minute - self.__second = second - self.__microsecond = microsecond + self._year = year + self._month = month + self._day = day + self._hour = hour + self._minute = minute + self._second = second + self._microsecond = microsecond self._tzinfo = tzinfo return self # Read-only field accessors - hour = property(lambda self: self.__hour, doc="hour (0-23)") - minute = property(lambda self: self.__minute, doc="minute (0-59)") - second = property(lambda self: self.__second, doc="second (0-59)") - microsecond = property(lambda self: self.__microsecond, - doc="microsecond (0-999999)") - tzinfo = property(lambda self: self._tzinfo, doc="timezone info object") + @property + def hour(self): + """hour (0-23)""" + return self._hour + @property + def minute(self): + """minute (0-59)""" + return self._minute + + @property + def second(self): + """second (0-59)""" + return self._second + + @property + def microsecond(self): + """microsecond (0-999999)""" + return self._microsecond + + @property + def tzinfo(self): + """timezone info object""" + return self._tzinfo + + @classmethod def fromtimestamp(cls, t, tz=None): """Construct a datetime from a POSIX timestamp (like time.time()). @@ -1438,37 +1515,42 @@ if tz is not None: result = tz.fromutc(result) return result - fromtimestamp = classmethod(fromtimestamp) + @classmethod def utcfromtimestamp(cls, t): "Construct a UTC datetime from a POSIX timestamp (like time.time())." - if 1 - (t % 1.0) < 0.0000005: - t = float(int(t)) + 1 - if t < 0: - t -= 1 + t, frac = divmod(t, 1.0) + us = round(frac * 1e6) + + # If timestamp is less than one microsecond smaller than a + # full second, us can be rounded up to 1000000. In this case, + # roll over to seconds, otherwise, ValueError is raised + # by the constructor. + if us == 1000000: + t += 1 + us = 0 y, m, d, hh, mm, ss, weekday, jday, dst = _time.gmtime(t) - us = int((t % 1.0) * 1000000) ss = min(ss, 59) # clamp out leap seconds if the platform has them return cls(y, m, d, hh, mm, ss, us) - utcfromtimestamp = classmethod(utcfromtimestamp) # XXX This is supposed to do better than we *can* do by using time.time(), # XXX if the platform supports a more accurate way. The C implementation # XXX uses gettimeofday on platforms that have it, but that isn't # XXX available from Python. So now() may return different results # XXX across the implementations. + @classmethod def now(cls, tz=None): "Construct a datetime from time.time() and optional time zone info." t = _time.time() return cls.fromtimestamp(t, tz) - now = classmethod(now) + @classmethod def utcnow(cls): "Construct a UTC datetime from time.time()." t = _time.time() return cls.utcfromtimestamp(t) - utcnow = classmethod(utcnow) + @classmethod def combine(cls, date, time): "Construct a datetime from a given date and a given time." if not isinstance(date, _date_class): @@ -1478,7 +1560,6 @@ return cls(date.year, date.month, date.day, time.hour, time.minute, time.second, time.microsecond, time.tzinfo) - combine = classmethod(combine) def timetuple(self): "Return local time tuple compatible with time.localtime()." @@ -1504,7 +1585,7 @@ def date(self): "Return the date part." - return date(self.__year, self.__month, self.__day) + return date(self._year, self._month, self._day) def time(self): "Return the time part, with tzinfo None." @@ -1564,8 +1645,8 @@ def ctime(self): "Format a la ctime()." - t = tmxxx(self.__year, self.__month, self.__day, self.__hour, - self.__minute, self.__second) + t = tmxxx(self._year, self._month, self._day, self._hour, + self._minute, self._second) return t.ctime() def isoformat(self, sep='T'): @@ -1580,10 +1661,10 @@ Optional argument sep specifies the separator between date and time, default 'T'. """ - s = ("%04d-%02d-%02d%c" % (self.__year, self.__month, self.__day, + s = ("%04d-%02d-%02d%c" % (self._year, self._month, self._day, sep) + - _format_time(self.__hour, self.__minute, self.__second, - self.__microsecond)) + _format_time(self._hour, self._minute, self._second, + self._microsecond)) off = self._utcoffset() if off is not None: if off < 0: @@ -1596,13 +1677,13 @@ return s def __repr__(self): - "Convert to formal string, for repr()." - L = [self.__year, self.__month, self.__day, # These are never zero - self.__hour, self.__minute, self.__second, self.__microsecond] + """Convert to formal string, for repr().""" + L = [self._year, self._month, self._day, # These are never zero + self._hour, self._minute, self._second, self._microsecond] if L[-1] == 0: del L[-1] if L[-1] == 0: - del L[-1] + del L[-1] s = ", ".join(map(str, L)) s = "%s(%s)" % ('datetime.' + self.__class__.__name__, s) if self._tzinfo is not None: @@ -1675,7 +1756,7 @@ def __eq__(self, other): if isinstance(other, datetime): - return self.__cmp(other) == 0 + return self._cmp(other) == 0 elif hasattr(other, "timetuple") and not isinstance(other, date): return NotImplemented else: @@ -1683,7 +1764,7 @@ def __ne__(self, other): if isinstance(other, datetime): - return self.__cmp(other) != 0 + return self._cmp(other) != 0 elif hasattr(other, "timetuple") and not isinstance(other, date): return NotImplemented else: @@ -1691,7 +1772,7 @@ def __le__(self, other): if isinstance(other, datetime): - return self.__cmp(other) <= 0 + return self._cmp(other) <= 0 elif hasattr(other, "timetuple") and not isinstance(other, date): return NotImplemented else: @@ -1699,7 +1780,7 @@ def __lt__(self, other): if isinstance(other, datetime): - return self.__cmp(other) < 0 + return self._cmp(other) < 0 elif hasattr(other, "timetuple") and not isinstance(other, date): return NotImplemented else: @@ -1707,7 +1788,7 @@ def __ge__(self, other): if isinstance(other, datetime): - return self.__cmp(other) >= 0 + return self._cmp(other) >= 0 elif hasattr(other, "timetuple") and not isinstance(other, date): return NotImplemented else: @@ -1715,13 +1796,13 @@ def __gt__(self, other): if isinstance(other, datetime): - return self.__cmp(other) > 0 + return self._cmp(other) > 0 elif hasattr(other, "timetuple") and not isinstance(other, date): return NotImplemented else: _cmperror(self, other) - def __cmp(self, other): + def _cmp(self, other): assert isinstance(other, datetime) mytz = self._tzinfo ottz = other._tzinfo @@ -1737,12 +1818,12 @@ base_compare = myoff == otoff if base_compare: - return cmp((self.__year, self.__month, self.__day, - self.__hour, self.__minute, self.__second, - self.__microsecond), - (other.__year, other.__month, other.__day, - other.__hour, other.__minute, other.__second, - other.__microsecond)) + return cmp((self._year, self._month, self._day, + self._hour, self._minute, self._second, + self._microsecond), + (other._year, other._month, other._day, + other._hour, other._minute, other._second, + other._microsecond)) if myoff is None or otoff is None: # XXX Buggy in 2.2.2. raise TypeError("cannot compare naive and aware datetimes") @@ -1756,13 +1837,13 @@ "Add a datetime and a timedelta." if not isinstance(other, timedelta): return NotImplemented - t = tmxxx(self.__year, - self.__month, - self.__day + other.days, - self.__hour, - self.__minute, - self.__second + other.seconds, - self.__microsecond + other.microseconds) + t = tmxxx(self._year, + self._month, + self._day + other.days, + self._hour, + self._minute, + self._second + other.seconds, + self._microsecond + other.microseconds) self._checkOverflow(t.year) result = datetime(t.year, t.month, t.day, t.hour, t.minute, t.second, @@ -1780,11 +1861,11 @@ days1 = self.toordinal() days2 = other.toordinal() - secs1 = self.__second + self.__minute * 60 + self.__hour * 3600 - secs2 = other.__second + other.__minute * 60 + other.__hour * 3600 + secs1 = self._second + self._minute * 60 + self._hour * 3600 + secs2 = other._second + other._minute * 60 + other._hour * 3600 base = timedelta(days1 - days2, secs1 - secs2, - self.__microsecond - other.__microsecond) + self._microsecond - other._microsecond) if self._tzinfo is other._tzinfo: return base myoff = self._utcoffset() @@ -1792,13 +1873,13 @@ if myoff == otoff: return base if myoff is None or otoff is None: - raise TypeError, "cannot mix naive and timezone-aware time" + raise TypeError("cannot mix naive and timezone-aware time") return base + timedelta(minutes = otoff-myoff) def __hash__(self): tzoff = self._utcoffset() if tzoff is None: - return hash(self.__getstate()[0]) + return hash(self._getstate()[0]) days = _ymd2ord(self.year, self.month, self.day) seconds = self.hour * 3600 + (self.minute - tzoff) * 60 + self.second return hash(timedelta(days, seconds, self.microsecond)) @@ -1807,12 +1888,12 @@ __safe_for_unpickling__ = True # For Python 2.2 - def __getstate(self): - yhi, ylo = divmod(self.__year, 256) - us2, us3 = divmod(self.__microsecond, 256) + def _getstate(self): + yhi, ylo = divmod(self._year, 256) + us2, us3 = divmod(self._microsecond, 256) us1, us2 = divmod(us2, 256) - basestate = ("%c" * 10) % (yhi, ylo, self.__month, self.__day, - self.__hour, self.__minute, self.__second, + basestate = ("%c" * 10) % (yhi, ylo, self._month, self._day, + self._hour, self._minute, self._second, us1, us2, us3) if self._tzinfo is None: return (basestate,) @@ -1820,14 +1901,14 @@ return (basestate, self._tzinfo) def __setstate(self, string, tzinfo): - (yhi, ylo, self.__month, self.__day, self.__hour, - self.__minute, self.__second, us1, us2, us3) = map(ord, string) - self.__year = yhi * 256 + ylo - self.__microsecond = (((us1 << 8) | us2) << 8) | us3 + (yhi, ylo, self._month, self._day, self._hour, + self._minute, self._second, us1, us2, us3) = map(ord, string) + self._year = yhi * 256 + ylo + self._microsecond = (((us1 << 8) | us2) << 8) | us3 self._tzinfo = tzinfo def __reduce__(self): - return (self.__class__, self.__getstate()) + return (self.__class__, self._getstate()) datetime.min = datetime(1, 1, 1) @@ -2009,7 +2090,7 @@ Because we know z.d said z was in daylight time (else [5] would have held and we would have stopped then), and we know z.d != z'.d (else [8] would have held -and we we have stopped then), and there are only 2 possible values dst() can +and we have stopped then), and there are only 2 possible values dst() can return in Eastern, it follows that z'.d must be 0 (which it is in the example, but the reasoning doesn't depend on the example -- it depends on there being two possible dst() outcomes, one zero and the other non-zero). Therefore diff --git a/lib_pypy/numpypy/__init__.py b/lib_pypy/numpypy/__init__.py --- a/lib_pypy/numpypy/__init__.py +++ b/lib_pypy/numpypy/__init__.py @@ -1,2 +1,2 @@ from _numpypy import * -from .fromnumeric import * +from .core import * diff --git a/lib_pypy/numpypy/core/__init__.py b/lib_pypy/numpypy/core/__init__.py new file mode 100644 --- /dev/null +++ b/lib_pypy/numpypy/core/__init__.py @@ -0,0 +1,2 @@ +from .fromnumeric import * +from .numeric import * diff --git a/lib_pypy/numpypy/core/_methods.py b/lib_pypy/numpypy/core/_methods.py new file mode 100644 --- /dev/null +++ b/lib_pypy/numpypy/core/_methods.py @@ -0,0 +1,98 @@ +# Array methods which are called by the both the C-code for the method +# and the Python code for the NumPy-namespace function + +import _numpypy as mu +um = mu +#from numpypy.core import umath as um +from numpypy.core.numeric import asanyarray + +def _amax(a, axis=None, out=None, skipna=False, keepdims=False): + return um.maximum.reduce(a, axis=axis, + out=out, skipna=skipna, keepdims=keepdims) + +def _amin(a, axis=None, out=None, skipna=False, keepdims=False): + return um.minimum.reduce(a, axis=axis, + out=out, skipna=skipna, keepdims=keepdims) + +def _sum(a, axis=None, dtype=None, out=None, skipna=False, keepdims=False): + return um.add.reduce(a, axis=axis, dtype=dtype, + out=out, skipna=skipna, keepdims=keepdims) + +def _prod(a, axis=None, dtype=None, out=None, skipna=False, keepdims=False): + return um.multiply.reduce(a, axis=axis, dtype=dtype, + out=out, skipna=skipna, keepdims=keepdims) + +def _mean(a, axis=None, dtype=None, out=None, skipna=False, keepdims=False): + arr = asanyarray(a) + + # Upgrade bool, unsigned int, and int to float64 + if dtype is None and arr.dtype.kind in ['b','u','i']: + ret = um.add.reduce(arr, axis=axis, dtype='f8', + out=out, skipna=skipna, keepdims=keepdims) + else: + ret = um.add.reduce(arr, axis=axis, dtype=dtype, + out=out, skipna=skipna, keepdims=keepdims) + rcount = mu.count_reduce_items(arr, axis=axis, + skipna=skipna, keepdims=keepdims) + if isinstance(ret, mu.ndarray): + ret = um.true_divide(ret, rcount, + casting='unsafe', subok=False) + else: + ret = ret / float(rcount) + return ret + +def _var(a, axis=None, dtype=None, out=None, ddof=0, + skipna=False, keepdims=False): + arr = asanyarray(a) + + # First compute the mean, saving 'rcount' for reuse later + if dtype is None and arr.dtype.kind in ['b','u','i']: + arrmean = um.add.reduce(arr, axis=axis, dtype='f8', + skipna=skipna, keepdims=True) + else: + arrmean = um.add.reduce(arr, axis=axis, dtype=dtype, + skipna=skipna, keepdims=True) + rcount = mu.count_reduce_items(arr, axis=axis, + skipna=skipna, keepdims=True) + if isinstance(arrmean, mu.ndarray): + arrmean = um.true_divide(arrmean, rcount, + casting='unsafe', subok=False) + else: + arrmean = arrmean / float(rcount) + + # arr - arrmean + x = arr - arrmean + + # (arr - arrmean) ** 2 + if arr.dtype.kind == 'c': + x = um.multiply(x, um.conjugate(x)).real + else: + x = um.multiply(x, x) + + # add.reduce((arr - arrmean) ** 2, axis) + ret = um.add.reduce(x, axis=axis, dtype=dtype, out=out, + skipna=skipna, keepdims=keepdims) + + # add.reduce((arr - arrmean) ** 2, axis) / (n - ddof) + if not keepdims and isinstance(rcount, mu.ndarray): + rcount = rcount.squeeze(axis=axis) + rcount -= ddof + if isinstance(ret, mu.ndarray): + ret = um.true_divide(ret, rcount, + casting='unsafe', subok=False) + else: + ret = ret / float(rcount) + + return ret + +def _std(a, axis=None, dtype=None, out=None, ddof=0, + skipna=False, keepdims=False): + ret = _var(a, axis=axis, dtype=dtype, out=out, ddof=ddof, + skipna=skipna, keepdims=keepdims) + + if isinstance(ret, mu.ndarray): + ret = um.sqrt(ret) + else: + ret = um.sqrt(ret) + + return ret diff --git a/lib_pypy/numpypy/core/arrayprint.py b/lib_pypy/numpypy/core/arrayprint.py new file mode 100644 --- /dev/null +++ b/lib_pypy/numpypy/core/arrayprint.py @@ -0,0 +1,789 @@ +"""Array printing function + +$Id: arrayprint.py,v 1.9 2005/09/13 13:58:44 teoliphant Exp $ +""" +__all__ = ["array2string", "set_printoptions", "get_printoptions"] +__docformat__ = 'restructuredtext' + +# +# Written by Konrad Hinsen <hinsenk at ere.umontreal.ca> +# last revision: 1996-3-13 +# modified by Jim Hugunin 1997-3-3 for repr's and str's (and other details) +# and by Perry Greenfield 2000-4-1 for numarray +# and by Travis Oliphant 2005-8-22 for numpy + +import sys +import _numpypy as _nt +from _numpypy import maximum, minimum, absolute, not_equal, isinf, isnan, isna +#from _numpypy import format_longfloat, datetime_as_string, datetime_data +from .fromnumeric import ravel + + +def product(x, y): return x*y + +_summaryEdgeItems = 3 # repr N leading and trailing items of each dimension +_summaryThreshold = 1000 # total items > triggers array summarization + +_float_output_precision = 8 +_float_output_suppress_small = False +_line_width = 75 +_nan_str = 'nan' +_inf_str = 'inf' +_na_str = 'NA' +_formatter = None # formatting function for array elements + +if sys.version_info[0] >= 3: + from functools import reduce + +def set_printoptions(precision=None, threshold=None, edgeitems=None, + linewidth=None, suppress=None, + nanstr=None, infstr=None, nastr=None, + formatter=None): + """ + Set printing options. + + These options determine the way floating point numbers, arrays and + other NumPy objects are displayed. + + Parameters + ---------- + precision : int, optional + Number of digits of precision for floating point output (default 8). + threshold : int, optional + Total number of array elements which trigger summarization + rather than full repr (default 1000). + edgeitems : int, optional + Number of array items in summary at beginning and end of + each dimension (default 3). + linewidth : int, optional + The number of characters per line for the purpose of inserting + line breaks (default 75). + suppress : bool, optional + Whether or not suppress printing of small floating point values + using scientific notation (default False). + nanstr : str, optional + String representation of floating point not-a-number (default nan). + infstr : str, optional + String representation of floating point infinity (default inf). + nastr : str, optional + String representation of NA missing value (default NA). + formatter : dict of callables, optional + If not None, the keys should indicate the type(s) that the respective + formatting function applies to. Callables should return a string. + Types that are not specified (by their corresponding keys) are handled + by the default formatters. Individual types for which a formatter + can be set are:: + + - 'bool' + - 'int' + - 'timedelta' : a `numpy.timedelta64` + - 'datetime' : a `numpy.datetime64` + - 'float' + - 'longfloat' : 128-bit floats + - 'complexfloat' + - 'longcomplexfloat' : composed of two 128-bit floats + - 'numpy_str' : types `numpy.string_` and `numpy.unicode_` + - 'str' : all other strings + + Other keys that can be used to set a group of types at once are:: + + - 'all' : sets all types + - 'int_kind' : sets 'int' + - 'float_kind' : sets 'float' and 'longfloat' + - 'complex_kind' : sets 'complexfloat' and 'longcomplexfloat' + - 'str_kind' : sets 'str' and 'numpystr' + + See Also + -------- + get_printoptions, set_string_function, array2string + + Notes + ----- + `formatter` is always reset with a call to `set_printoptions`. + + Examples + -------- + Floating point precision can be set: + + >>> np.set_printoptions(precision=4) + >>> print np.array([1.123456789]) + [ 1.1235] + + Long arrays can be summarised: + + >>> np.set_printoptions(threshold=5) + >>> print np.arange(10) + [0 1 2 ..., 7 8 9] + + Small results can be suppressed: + + >>> eps = np.finfo(float).eps + >>> x = np.arange(4.) + >>> x**2 - (x + eps)**2 + array([ -4.9304e-32, -4.4409e-16, 0.0000e+00, 0.0000e+00]) + >>> np.set_printoptions(suppress=True) + >>> x**2 - (x + eps)**2 + array([-0., -0., 0., 0.]) + + A custom formatter can be used to display array elements as desired: + + >>> np.set_printoptions(formatter={'all':lambda x: 'int: '+str(-x)}) + >>> x = np.arange(3) + >>> x + array([int: 0, int: -1, int: -2]) + >>> np.set_printoptions() # formatter gets reset + >>> x + array([0, 1, 2]) + + To put back the default options, you can use: + + >>> np.set_printoptions(edgeitems=3,infstr='inf', + ... linewidth=75, nanstr='nan', precision=8, + ... suppress=False, threshold=1000, formatter=None) + """ + + global _summaryThreshold, _summaryEdgeItems, _float_output_precision, \ + _line_width, _float_output_suppress_small, _nan_str, _inf_str, \ + _na_str, _formatter + if linewidth is not None: + _line_width = linewidth + if threshold is not None: + _summaryThreshold = threshold + if edgeitems is not None: + _summaryEdgeItems = edgeitems + if precision is not None: + _float_output_precision = precision + if suppress is not None: + _float_output_suppress_small = not not suppress + if nanstr is not None: + _nan_str = nanstr + if infstr is not None: + _inf_str = infstr + if nastr is not None: + _na_str = nastr + _formatter = formatter + +def get_printoptions(): + """ + Return the current print options. + + Returns + ------- + print_opts : dict + Dictionary of current print options with keys + + - precision : int + - threshold : int + - edgeitems : int + - linewidth : int + - suppress : bool + - nanstr : str + - infstr : str + - formatter : dict of callables + + For a full description of these options, see `set_printoptions`. + + See Also + -------- + set_printoptions, set_string_function + + """ + d = dict(precision=_float_output_precision, + threshold=_summaryThreshold, + edgeitems=_summaryEdgeItems, + linewidth=_line_width, + suppress=_float_output_suppress_small, + nanstr=_nan_str, + infstr=_inf_str, + nastr=_na_str, + formatter=_formatter) + return d + +def _leading_trailing(a): + import numeric as _nc + if a.ndim == 1: + if len(a) > 2*_summaryEdgeItems: + b = _nc.concatenate((a[:_summaryEdgeItems], + a[-_summaryEdgeItems:])) + else: + b = a + else: + if len(a) > 2*_summaryEdgeItems: + l = [_leading_trailing(a[i]) for i in range( + min(len(a), _summaryEdgeItems))] + l.extend([_leading_trailing(a[-i]) for i in range( + min(len(a), _summaryEdgeItems),0,-1)]) + else: + l = [_leading_trailing(a[i]) for i in range(0, len(a))] + b = _nc.concatenate(tuple(l)) + return b + +def _boolFormatter(x): + if isna(x): + return str(x).replace('NA', _na_str, 1) + elif x: + return ' True' + else: + return 'False' + + +def repr_format(x): + if isna(x): + return str(x).replace('NA', _na_str, 1) + else: + return repr(x) + +def _array2string(a, max_line_width, precision, suppress_small, separator=' ', + prefix="", formatter=None): + + if max_line_width is None: + max_line_width = _line_width + + if precision is None: + precision = _float_output_precision + + if suppress_small is None: + suppress_small = _float_output_suppress_small + + if formatter is None: + formatter = _formatter + + if a.size > _summaryThreshold: + summary_insert = "..., " + data = _leading_trailing(a) + else: + summary_insert = "" + data = ravel(a) + + formatdict = {'bool' : _boolFormatter, + 'int' : IntegerFormat(data), + 'float' : FloatFormat(data, precision, suppress_small), + 'longfloat' : LongFloatFormat(precision), + #'complexfloat' : ComplexFormat(data, precision, + # suppress_small), + #'longcomplexfloat' : LongComplexFormat(precision), + #'datetime' : DatetimeFormat(data), + #'timedelta' : TimedeltaFormat(data), + 'numpystr' : repr_format, + 'str' : str} + + if formatter is not None: + fkeys = [k for k in formatter.keys() if formatter[k] is not None] + if 'all' in fkeys: + for key in formatdict.keys(): + formatdict[key] = formatter['all'] + if 'int_kind' in fkeys: + for key in ['int']: + formatdict[key] = formatter['int_kind'] + if 'float_kind' in fkeys: + for key in ['float', 'longfloat']: + formatdict[key] = formatter['float_kind'] + if 'complex_kind' in fkeys: + for key in ['complexfloat', 'longcomplexfloat']: + formatdict[key] = formatter['complex_kind'] + if 'str_kind' in fkeys: + for key in ['numpystr', 'str']: + formatdict[key] = formatter['str_kind'] + for key in formatdict.keys(): + if key in fkeys: + formatdict[key] = formatter[key] + + try: + format_function = a._format + msg = "The `_format` attribute is deprecated in Numpy 2.0 and " \ + "will be removed in 2.1. Use the `formatter` kw instead." + import warnings + warnings.warn(msg, DeprecationWarning) + except AttributeError: + # find the right formatting function for the array + dtypeobj = a.dtype.type + if issubclass(dtypeobj, _nt.bool_): + format_function = formatdict['bool'] + elif issubclass(dtypeobj, _nt.integer): + #if issubclass(dtypeobj, _nt.timedelta64): + # format_function = formatdict['timedelta'] + #else: + format_function = formatdict['int'] + elif issubclass(dtypeobj, _nt.floating): + #if issubclass(dtypeobj, _nt.longfloat): + # format_function = formatdict['longfloat'] + #else: + format_function = formatdict['float'] + elif issubclass(dtypeobj, _nt.complexfloating): + if issubclass(dtypeobj, _nt.clongfloat): + format_function = formatdict['longcomplexfloat'] + else: + format_function = formatdict['complexfloat'] + elif issubclass(dtypeobj, (_nt.unicode_, _nt.string_)): + format_function = formatdict['numpystr'] + elif issubclass(dtypeobj, _nt.datetime64): + format_function = formatdict['datetime'] + else: + format_function = formatdict['str'] + + # skip over "[" + next_line_prefix = " " + # skip over array( + next_line_prefix += " "*len(prefix) + + lst = _formatArray(a, format_function, len(a.shape), max_line_width, + next_line_prefix, separator, + _summaryEdgeItems, summary_insert)[:-1] + return lst + +def _convert_arrays(obj): + import numeric as _nc + newtup = [] + for k in obj: + if isinstance(k, _nc.ndarray): + k = k.tolist() + elif isinstance(k, tuple): + k = _convert_arrays(k) + newtup.append(k) + return tuple(newtup) + + +def array2string(a, max_line_width=None, precision=None, + suppress_small=None, separator=' ', prefix="", + style=repr, formatter=None): + """ + Return a string representation of an array. + + Parameters + ---------- + a : ndarray + Input array. + max_line_width : int, optional + The maximum number of columns the string should span. Newline + characters splits the string appropriately after array elements. + precision : int, optional + Floating point precision. Default is the current printing + precision (usually 8), which can be altered using `set_printoptions`. + suppress_small : bool, optional + Represent very small numbers as zero. A number is "very small" if it + is smaller than the current printing precision. + separator : str, optional + Inserted between elements. + prefix : str, optional + An array is typically printed as:: + + 'prefix(' + array2string(a) + ')' + + The length of the prefix string is used to align the + output correctly. + style : function, optional + A function that accepts an ndarray and returns a string. Used only + when the shape of `a` is equal to ``()``, i.e. for 0-D arrays. + formatter : dict of callables, optional + If not None, the keys should indicate the type(s) that the respective + formatting function applies to. Callables should return a string. + Types that are not specified (by their corresponding keys) are handled + by the default formatters. Individual types for which a formatter + can be set are:: + + - 'bool' + - 'int' + - 'timedelta' : a `numpy.timedelta64` + - 'datetime' : a `numpy.datetime64` + - 'float' + - 'longfloat' : 128-bit floats + - 'complexfloat' + - 'longcomplexfloat' : composed of two 128-bit floats + - 'numpy_str' : types `numpy.string_` and `numpy.unicode_` + - 'str' : all other strings + + Other keys that can be used to set a group of types at once are:: + + - 'all' : sets all types + - 'int_kind' : sets 'int' + - 'float_kind' : sets 'float' and 'longfloat' + - 'complex_kind' : sets 'complexfloat' and 'longcomplexfloat' + - 'str_kind' : sets 'str' and 'numpystr' + + Returns + ------- + array_str : str + String representation of the array. + + Raises + ------ + TypeError : if a callable in `formatter` does not return a string. + + See Also + -------- + array_str, array_repr, set_printoptions, get_printoptions + + Notes + ----- + If a formatter is specified for a certain type, the `precision` keyword is + ignored for that type. + + Examples + -------- + >>> x = np.array([1e-16,1,2,3]) + >>> print np.array2string(x, precision=2, separator=',', + ... suppress_small=True) + [ 0., 1., 2., 3.] + + >>> x = np.arange(3.) + >>> np.array2string(x, formatter={'float_kind':lambda x: "%.2f" % x}) + '[0.00 1.00 2.00]' + + >>> x = np.arange(3) + >>> np.array2string(x, formatter={'int':lambda x: hex(x)}) + '[0x0L 0x1L 0x2L]' + + """ + + if a.shape == (): + x = a.item() + if isna(x): + lst = str(x).replace('NA', _na_str, 1) + else: + try: + lst = a._format(x) + msg = "The `_format` attribute is deprecated in Numpy " \ + "2.0 and will be removed in 2.1. Use the " \ + "`formatter` kw instead." + import warnings + warnings.warn(msg, DeprecationWarning) + except AttributeError: + if isinstance(x, tuple): + x = _convert_arrays(x) + lst = style(x) + elif reduce(product, a.shape) == 0: + # treat as a null array if any of shape elements == 0 + lst = "[]" + else: + lst = _array2string(a, max_line_width, precision, suppress_small, + separator, prefix, formatter=formatter) + return lst + +def _extendLine(s, line, word, max_line_len, next_line_prefix): + if len(line.rstrip()) + len(word.rstrip()) >= max_line_len: + s += line.rstrip() + "\n" + line = next_line_prefix + line += word + return s, line + + +def _formatArray(a, format_function, rank, max_line_len, + next_line_prefix, separator, edge_items, summary_insert): + """formatArray is designed for two modes of operation: + + 1. Full output + + 2. Summarized output + + """ + if rank == 0: + obj = a.item() + if isinstance(obj, tuple): + obj = _convert_arrays(obj) + return str(obj) + + if summary_insert and 2*edge_items < len(a): + leading_items, trailing_items, summary_insert1 = \ + edge_items, edge_items, summary_insert + else: + leading_items, trailing_items, summary_insert1 = 0, len(a), "" + + if rank == 1: + s = "" + line = next_line_prefix + for i in xrange(leading_items): + word = format_function(a[i]) + separator + s, line = _extendLine(s, line, word, max_line_len, next_line_prefix) + + if summary_insert1: + s, line = _extendLine(s, line, summary_insert1, max_line_len, next_line_prefix) + + for i in xrange(trailing_items, 1, -1): + word = format_function(a[-i]) + separator + s, line = _extendLine(s, line, word, max_line_len, next_line_prefix) + + word = format_function(a[-1]) + s, line = _extendLine(s, line, word, max_line_len, next_line_prefix) + s += line + "]\n" + s = '[' + s[len(next_line_prefix):] + else: + s = '[' + sep = separator.rstrip() + for i in xrange(leading_items): + if i > 0: + s += next_line_prefix + s += _formatArray(a[i], format_function, rank-1, max_line_len, + " " + next_line_prefix, separator, edge_items, + summary_insert) + s = s.rstrip() + sep.rstrip() + '\n'*max(rank-1,1) + + if summary_insert1: + s += next_line_prefix + summary_insert1 + "\n" + + for i in xrange(trailing_items, 1, -1): + if leading_items or i != trailing_items: + s += next_line_prefix + s += _formatArray(a[-i], format_function, rank-1, max_line_len, + " " + next_line_prefix, separator, edge_items, + summary_insert) + s = s.rstrip() + sep.rstrip() + '\n'*max(rank-1,1) + if leading_items or trailing_items > 1: + s += next_line_prefix + s += _formatArray(a[-1], format_function, rank-1, max_line_len, + " " + next_line_prefix, separator, edge_items, + summary_insert).rstrip()+']\n' + return s + +class FloatFormat(object): + def __init__(self, data, precision, suppress_small, sign=False): + self.precision = precision + self.suppress_small = suppress_small + self.sign = sign + self.exp_format = False + self.large_exponent = False + self.max_str_len = 0 + #try: + self.fillFormat(data) + #except (TypeError, NotImplementedError): + # if reduce(data) fails, this instance will not be called, just + # instantiated in formatdict. + #pass + + def fillFormat(self, data): + import numeric as _nc + # XXX pypy unimplemented + #errstate = _nc.seterr(all='ignore') + try: + special = isnan(data) | isinf(data) | isna(data) + special[isna(data)] = False + valid = not_equal(data, 0) & ~special + valid[isna(data)] = False + non_zero = absolute(data.compress(valid)) + if len(non_zero) == 0: + max_val = 0. + min_val = 0. + else: + max_val = maximum.reduce(non_zero, skipna=True) + min_val = minimum.reduce(non_zero, skipna=True) + if max_val >= 1.e8: + self.exp_format = True + if not self.suppress_small and (min_val < 0.0001 + or max_val/min_val > 1000.): + self.exp_format = True + finally: + pass + # XXX pypy unimplemented + #_nc.seterr(**errstate) + + if self.exp_format: + self.large_exponent = 0 < min_val < 1e-99 or max_val >= 1e100 + self.max_str_len = 8 + self.precision + if self.large_exponent: + self.max_str_len += 1 + if self.sign: + format = '%+' + else: + format = '%' + format = format + '%d.%de' % (self.max_str_len, self.precision) + else: + format = '%%.%df' % (self.precision,) + if len(non_zero): + precision = max([_digits(x, self.precision, format) + for x in non_zero]) + else: + precision = 0 + precision = min(self.precision, precision) + self.max_str_len = len(str(int(max_val))) + precision + 2 + if special.any(): + self.max_str_len = max(self.max_str_len, + len(_nan_str), + len(_inf_str)+1, + len(_na_str)) + if self.sign: + format = '%#+' + else: + format = '%#' + format = format + '%d.%df' % (self.max_str_len, precision) + + self.special_fmt = '%%%ds' % (self.max_str_len,) + self.format = format + + def __call__(self, x, strip_zeros=True): + import numeric as _nc + #err = _nc.seterr(invalid='ignore') + try: + if isna(x): + return self.special_fmt % (str(x).replace('NA', _na_str, 1),) + elif isnan(x): + if self.sign: + return self.special_fmt % ('+' + _nan_str,) + else: + return self.special_fmt % (_nan_str,) + elif isinf(x): + if x > 0: + if self.sign: + return self.special_fmt % ('+' + _inf_str,) + else: + return self.special_fmt % (_inf_str,) + else: + return self.special_fmt % ('-' + _inf_str,) + finally: + pass + #_nc.seterr(**err) + + s = self.format % x + if self.large_exponent: + # 3-digit exponent + expsign = s[-3] + if expsign == '+' or expsign == '-': + s = s[1:-2] + '0' + s[-2:] + elif self.exp_format: + # 2-digit exponent + if s[-3] == '0': + s = ' ' + s[:-3] + s[-2:] + elif strip_zeros: + z = s.rstrip('0') + s = z + ' '*(len(s)-len(z)) + return s + + +def _digits(x, precision, format): + s = format % x + z = s.rstrip('0') + return precision - len(s) + len(z) + + +_MAXINT = sys.maxint +_MININT = -sys.maxint-1 +class IntegerFormat(object): + def __init__(self, data): + try: + max_str_len = max(len(str(maximum.reduce(data, skipna=True))), + len(str(minimum.reduce(data, skipna=True)))) + self.format = '%' + str(max_str_len) + 'd' + except TypeError, NotImplementedError: + # if reduce(data) fails, this instance will not be called, just + # instantiated in formatdict. + pass + except ValueError: + # this occurs when everything is NA + pass + + def __call__(self, x): + if isna(x): + return str(x).replace('NA', _na_str, 1) + elif _MININT < x < _MAXINT: + return self.format % x + else: + return "%s" % x + +class LongFloatFormat(object): + # XXX Have to add something to determine the width to use a la FloatFormat + # Right now, things won't line up properly + def __init__(self, precision, sign=False): + self.precision = precision + self.sign = sign + + def __call__(self, x): + if isna(x): + return str(x).replace('NA', _na_str, 1) + elif isnan(x): + if self.sign: + return '+' + _nan_str + else: + return ' ' + _nan_str + elif isinf(x): + if x > 0: + if self.sign: + return '+' + _inf_str + else: + return ' ' + _inf_str + else: + return '-' + _inf_str + elif x >= 0: + if self.sign: + return '+' + format_longfloat(x, self.precision) + else: + return ' ' + format_longfloat(x, self.precision) + else: + return format_longfloat(x, self.precision) + + +class LongComplexFormat(object): + def __init__(self, precision): + self.real_format = LongFloatFormat(precision) + self.imag_format = LongFloatFormat(precision, sign=True) + + def __call__(self, x): + if isna(x): + return str(x).replace('NA', _na_str, 1) + else: + r = self.real_format(x.real) + i = self.imag_format(x.imag) + return r + i + 'j' + + +class ComplexFormat(object): + def __init__(self, x, precision, suppress_small): + self.real_format = FloatFormat(x.real, precision, suppress_small) + self.imag_format = FloatFormat(x.imag, precision, suppress_small, + sign=True) + + def __call__(self, x): + if isna(x): + return str(x).replace('NA', _na_str, 1) + else: + r = self.real_format(x.real, strip_zeros=False) + i = self.imag_format(x.imag, strip_zeros=False) + if not self.imag_format.exp_format: + z = i.rstrip('0') + i = z + 'j' + ' '*(len(i)-len(z)) + else: + i = i + 'j' + return r + i + +class DatetimeFormat(object): + def __init__(self, x, unit=None, + timezone=None, casting='same_kind'): + # Get the unit from the dtype + if unit is None: + if x.dtype.kind == 'M': + unit = datetime_data(x.dtype)[0] + else: + unit = 's' + + # If timezone is default, make it 'local' or 'UTC' based on the unit + if timezone is None: + # Date units -> UTC, time units -> local + if unit in ('Y', 'M', 'W', 'D'): + self.timezone = 'UTC' + else: + self.timezone = 'local' + else: + self.timezone = timezone + self.unit = unit + self.casting = casting + + def __call__(self, x): + if isna(x): + return str(x).replace('NA', _na_str, 1) + else: + return "'%s'" % datetime_as_string(x, + unit=self.unit, + timezone=self.timezone, + casting=self.casting) + +class TimedeltaFormat(object): + def __init__(self, data): + if data.dtype.kind == 'm': + v = data.view('i8') + max_str_len = max(len(str(maximum.reduce(v))), + len(str(minimum.reduce(v)))) + self.format = '%' + str(max_str_len) + 'd' + + def __call__(self, x): + if isna(x): + return str(x).replace('NA', _na_str, 1) + else: + return self.format % x.astype('i8') + diff --git a/lib_pypy/numpypy/fromnumeric.py b/lib_pypy/numpypy/core/fromnumeric.py rename from lib_pypy/numpypy/fromnumeric.py rename to lib_pypy/numpypy/core/fromnumeric.py --- a/lib_pypy/numpypy/fromnumeric.py +++ b/lib_pypy/numpypy/core/fromnumeric.py @@ -30,7 +30,7 @@ 'rank', 'size', 'around', 'round_', 'mean', 'std', 'var', 'squeeze', 'amax', 'amin', ] - + def take(a, indices, axis=None, out=None, mode='raise'): """ Take elements from an array along an axis. @@ -85,7 +85,7 @@ array([4, 3, 6]) """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') # not deprecated --- copy if necessary, view otherwise @@ -149,6 +149,7 @@ [5, 6]]) """ + assert order == 'C' if not hasattr(a, 'reshape'): a = numpypy.array(a) return a.reshape(newshape) @@ -273,7 +274,7 @@ [-1, -2, -3, -4, -5]]]) """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def repeat(a, repeats, axis=None): @@ -315,7 +316,7 @@ [3, 4]]) """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def put(a, ind, v, mode='raise'): @@ -366,7 +367,7 @@ array([ 0, 1, 2, 3, -5]) """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def swapaxes(a, axis1, axis2): @@ -410,7 +411,7 @@ [3, 7]]]) """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def transpose(a, axes=None): @@ -451,8 +452,11 @@ (2, 1, 3) """ - raise NotImplemented('Waiting on interp level method') - + if axes is not None: + raise NotImplementedError('No "axes" arg yet.') + if not hasattr(a, 'T'): + a = numpypy.array(a) + return a.T def sort(a, axis=-1, kind='quicksort', order=None): """ @@ -553,7 +557,7 @@ dtype=[('name', '|S10'), ('height', '<f8'), ('age', '<i4')]) """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def argsort(a, axis=-1, kind='quicksort', order=None): @@ -634,7 +638,7 @@ array([0, 1]) """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def argmax(a, axis=None): @@ -687,6 +691,7 @@ 1 """ + assert axis is None if not hasattr(a, 'argmax'): a = numpypy.array(a) return a.argmax() @@ -702,6 +707,7 @@ documentation. """ + assert axis is None if not hasattr(a, 'argmin'): a = numpypy.array(a) return a.argmin() @@ -753,7 +759,7 @@ array([0, 5, 1, 2]) """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def resize(a, new_shape): @@ -794,7 +800,7 @@ [0, 1, 2, 3]]) """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def squeeze(a): @@ -821,7 +827,7 @@ (3,) """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def diagonal(a, offset=0, axis1=0, axis2=1): @@ -905,7 +911,7 @@ [5, 7]]) """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def trace(a, offset=0, axis1=0, axis2=1, dtype=None, out=None): @@ -964,7 +970,7 @@ (2, 3) """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def ravel(a, order='C'): """ @@ -1048,8 +1054,9 @@ array([ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]) """ - raise NotImplemented('Waiting on interp level method') - + if not hasattr(a, 'ravel'): + a = numpypy.array(a) + return a.ravel(order=order) def nonzero(a): """ @@ -1124,7 +1131,7 @@ (array([1, 1, 1, 2, 2, 2]), array([0, 1, 2, 0, 1, 2])) """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def shape(a): @@ -1229,7 +1236,7 @@ array([2]) """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def clip(a, a_min, a_max, out=None): @@ -1282,7 +1289,7 @@ array([3, 4, 2, 3, 4, 5, 6, 7, 8, 8]) """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def sum(a, axis=None, dtype=None, out=None): @@ -1351,9 +1358,11 @@ -128 """ + assert dtype is None + assert out is None if not hasattr(a, "sum"): a = numpypy.array(a) - return a.sum() + return a.sum(axis=axis) def product (a, axis=None, dtype=None, out=None): @@ -1365,7 +1374,7 @@ prod : equivalent function; see for details. """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def sometrue(a, axis=None, out=None): @@ -1379,6 +1388,8 @@ any : equivalent function """ + assert axis is None + assert out is None if not hasattr(a, 'any'): a = numpypy.array(a) return a.any() @@ -1393,6 +1404,8 @@ numpy.all : Equivalent function; see for details. """ + assert axis is None + assert out is None if not hasattr(a, 'all'): a = numpypy.array(a) return a.all() @@ -1461,6 +1474,8 @@ (191614240, 191614240) """ + assert axis is None + assert out is None if not hasattr(a, 'any'): a = numpypy.array(a) return a.any() @@ -1523,6 +1538,8 @@ (28293632, 28293632, array([ True], dtype=bool)) """ + assert axis is None + assert out is None if not hasattr(a, 'all'): a = numpypy.array(a) return a.all() @@ -1590,7 +1607,7 @@ [ 4, 9, 15]]) """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def cumproduct(a, axis=None, dtype=None, out=None): @@ -1603,7 +1620,7 @@ cumprod : equivalent function; see for details. """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def ptp(a, axis=None, out=None): @@ -1644,7 +1661,7 @@ array([1, 1]) """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def amax(a, axis=None, out=None): @@ -1702,6 +1719,8 @@ 4.0 """ + assert axis is None + assert out is None if not hasattr(a, "max"): a = numpypy.array(a) return a.max() @@ -1763,6 +1782,8 @@ """ # amin() is equivalent to min() + assert axis is None + assert out is None if not hasattr(a, 'min'): a = numpypy.array(a) return a.min() @@ -1873,7 +1894,7 @@ True """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def cumprod(a, axis=None, dtype=None, out=None): @@ -1936,7 +1957,7 @@ [ 4, 20, 120]]) """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def ndim(a): @@ -2054,7 +2075,7 @@ 2 """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def around(a, decimals=0, out=None): @@ -2123,7 +2144,7 @@ array([ 0, 0, 0, 10]) """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def round_(a, decimals=0, out=None): @@ -2137,7 +2158,7 @@ around : equivalent function """ - raise NotImplemented('Waiting on interp level method') + raise NotImplementedError('Waiting on interp level method') def mean(a, axis=None, dtype=None, out=None): @@ -2211,9 +2232,11 @@ 0.55000000074505806 """ + assert dtype is None + assert out is None if not hasattr(a, "mean"): a = numpypy.array(a) - return a.mean() + return a.mean(axis=axis) def std(a, axis=None, dtype=None, out=None, ddof=0): @@ -2302,9 +2325,12 @@ 0.44999999925552653 """ + assert dtype is None + assert out is None + assert ddof == 0 if not hasattr(a, "std"): a = numpypy.array(a) - return a.std() + return a.std(axis=axis) def var(a, axis=None, dtype=None, out=None, ddof=0): @@ -2395,6 +2421,9 @@ 0.20250000000000001 """ + assert dtype is None + assert out is None + assert ddof == 0 if not hasattr(a, "var"): a = numpypy.array(a) - return a.var() + return a.var(axis=axis) diff --git a/lib_pypy/numpypy/core/numeric.py b/lib_pypy/numpypy/core/numeric.py new file mode 100644 --- /dev/null +++ b/lib_pypy/numpypy/core/numeric.py @@ -0,0 +1,311 @@ + +from _numpypy import array, ndarray, int_, float_ #, complex_# , longlong +from _numpypy import concatenate +import sys +import _numpypy as multiarray # ARGH +from numpypy.core.arrayprint import array2string + + + +def asanyarray(a, dtype=None, order=None, maskna=None, ownmaskna=False): + """ + Convert the input to an ndarray, but pass ndarray subclasses through. + + Parameters + ---------- + a : array_like + Input data, in any form that can be converted to an array. This + includes scalars, lists, lists of tuples, tuples, tuples of tuples, + tuples of lists, and ndarrays. + dtype : data-type, optional + By default, the data-type is inferred from the input data. + order : {'C', 'F'}, optional + Whether to use row-major ('C') or column-major ('F') memory + representation. Defaults to 'C'. + maskna : bool or None, optional + If this is set to True, it forces the array to have an NA mask. + If this is set to False, it forces the array to not have an NA + mask. + ownmaskna : bool, optional + If this is set to True, forces the array to have a mask which + it owns. + + Returns + ------- + out : ndarray or an ndarray subclass + Array interpretation of `a`. If `a` is an ndarray or a subclass + of ndarray, it is returned as-is and no copy is performed. + + See Also + -------- + asarray : Similar function which always returns ndarrays. + ascontiguousarray : Convert input to a contiguous array. + asfarray : Convert input to a floating point ndarray. + asfortranarray : Convert input to an ndarray with column-major + memory order. + asarray_chkfinite : Similar function which checks input for NaNs and + Infs. + fromiter : Create an array from an iterator. + fromfunction : Construct an array by executing a function on grid + positions. + + Examples + -------- + Convert a list into an array: + + >>> a = [1, 2] + >>> np.asanyarray(a) + array([1, 2]) + + Instances of `ndarray` subclasses are passed through as-is: + + >>> a = np.matrix([1, 2]) + >>> np.asanyarray(a) is a + True + + """ + return array(a, dtype, copy=False, order=order, subok=True, + maskna=maskna, ownmaskna=ownmaskna) + +def base_repr(number, base=2, padding=0): + """ + Return a string representation of a number in the given base system. + + Parameters + ---------- + number : int + The value to convert. Only positive values are handled. + base : int, optional + Convert `number` to the `base` number system. The valid range is 2-36, + the default value is 2. + padding : int, optional + Number of zeros padded on the left. Default is 0 (no padding). + + Returns + ------- + out : str + String representation of `number` in `base` system. + + See Also + -------- + binary_repr : Faster version of `base_repr` for base 2. + + Examples + -------- + >>> np.base_repr(5) + '101' + >>> np.base_repr(6, 5) + '11' + >>> np.base_repr(7, base=5, padding=3) + '00012' + + >>> np.base_repr(10, base=16) + 'A' + >>> np.base_repr(32, base=16) + '20' + + """ + digits = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ' + if base > len(digits): + raise ValueError("Bases greater than 36 not handled in base_repr.") + + num = abs(number) + res = [] + while num: + res.append(digits[num % base]) + num //= base + if padding: + res.append('0' * padding) + if number < 0: + res.append('-') + return ''.join(reversed(res or '0')) + +_typelessdata = [int_, float_]#, complex_] +# XXX +#if issubclass(intc, int): +# _typelessdata.append(intc) + +#if issubclass(longlong, int): +# _typelessdata.append(longlong) + +def array_repr(arr, max_line_width=None, precision=None, suppress_small=None): + """ + Return the string representation of an array. + + Parameters + ---------- + arr : ndarray + Input array. + max_line_width : int, optional + The maximum number of columns the string should span. Newline + characters split the string appropriately after array elements. + precision : int, optional + Floating point precision. Default is the current printing precision + (usually 8), which can be altered using `set_printoptions`. + suppress_small : bool, optional + Represent very small numbers as zero, default is False. Very small + is defined by `precision`, if the precision is 8 then + numbers smaller than 5e-9 are represented as zero. + + Returns + ------- + string : str + The string representation of an array. + + See Also + -------- + array_str, array2string, set_printoptions + + Examples + -------- + >>> np.array_repr(np.array([1,2])) + 'array([1, 2])' + >>> np.array_repr(np.ma.array([0.])) + 'MaskedArray([ 0.])' + >>> np.array_repr(np.array([], np.int32)) + 'array([], dtype=int32)' + + >>> x = np.array([1e-6, 4e-7, 2, 3]) + >>> np.array_repr(x, precision=6, suppress_small=True) + 'array([ 0.000001, 0. , 2. , 3. ])' + + """ + if arr.size > 0 or arr.shape==(0,): + lst = array2string(arr, max_line_width, precision, suppress_small, + ', ', "array(") + else: # show zero-length shape unless it is (0,) + lst = "[], shape=%s" % (repr(arr.shape),) + + if arr.__class__ is not ndarray: + cName= arr.__class__.__name__ + else: + cName = "array" + + skipdtype = (arr.dtype.type in _typelessdata) and arr.size > 0 + + # XXX pypy lacks support + if 0 and arr.flags.maskna: + whichna = isna(arr) + # If nothing is NA, explicitly signal the NA-mask + if not any(whichna): + lst += ", maskna=True" + # If everything is NA, can't skip the dtype + if skipdtype and all(whichna): + skipdtype = False + + if skipdtype: + return "%s(%s)" % (cName, lst) + else: + typename = arr.dtype.name + # Quote typename in the output if it is "complex". + if typename and not (typename[0].isalpha() and typename.isalnum()): + typename = "'%s'" % typename + + lf = '' + if 0: # or issubclass(arr.dtype.type, flexible): + if arr.dtype.names: + typename = "%s" % str(arr.dtype) + else: + typename = "'%s'" % str(arr.dtype) + lf = '\n'+' '*len("array(") + return cName + "(%s, %sdtype=%s)" % (lst, lf, typename) + +def array_str(a, max_line_width=None, precision=None, suppress_small=None): + """ + Return a string representation of the data in an array. + + The data in the array is returned as a single string. This function is + similar to `array_repr`, the difference being that `array_repr` also + returns information on the kind of array and its data type. + + Parameters + ---------- + a : ndarray + Input array. + max_line_width : int, optional + Inserts newlines if text is longer than `max_line_width`. The + default is, indirectly, 75. + precision : int, optional + Floating point precision. Default is the current printing precision + (usually 8), which can be altered using `set_printoptions`. + suppress_small : bool, optional + Represent numbers "very close" to zero as zero; default is False. + Very close is defined by precision: if the precision is 8, e.g., + numbers smaller (in absolute value) than 5e-9 are represented as + zero. + + See Also + -------- + array2string, array_repr, set_printoptions + + Examples + -------- + >>> np.array_str(np.arange(3)) + '[0 1 2]' + + """ + return array2string(a, max_line_width, precision, suppress_small, ' ', "", str) + +def set_string_function(f, repr=True): + """ + Set a Python function to be used when pretty printing arrays. + + Parameters + ---------- + f : function or None + Function to be used to pretty print arrays. The function should expect + a single array argument and return a string of the representation of + the array. If None, the function is reset to the default NumPy function + to print arrays. + repr : bool, optional + If True (default), the function for pretty printing (``__repr__``) + is set, if False the function that returns the default string + representation (``__str__``) is set. + + See Also + -------- + set_printoptions, get_printoptions + + Examples + -------- + >>> def pprint(arr): + ... return 'HA! - What are you going to do now?' + ... + >>> np.set_string_function(pprint) + >>> a = np.arange(10) + >>> a + HA! - What are you going to do now? + >>> print a + [0 1 2 3 4 5 6 7 8 9] + + We can reset the function to the default: + + >>> np.set_string_function(None) + >>> a + array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) + + `repr` affects either pretty printing or normal string representation. + Note that ``__repr__`` is still affected by setting ``__str__`` + because the width of each array element in the returned string becomes + equal to the length of the result of ``__str__()``. + + >>> x = np.arange(4) + >>> np.set_string_function(lambda x:'random', repr=False) + >>> x.__str__() + 'random' + >>> x.__repr__() + 'array([ 0, 1, 2, 3])' + + """ + if f is None: + if repr: + return multiarray.set_string_function(array_repr, 1) + else: + return multiarray.set_string_function(array_str, 0) + else: + return multiarray.set_string_function(f, repr) + +set_string_function(array_str, 0) +set_string_function(array_repr, 1) + +little_endian = (sys.byteorder == 'little') diff --git a/pypy/annotation/unaryop.py b/pypy/annotation/unaryop.py --- a/pypy/annotation/unaryop.py +++ b/pypy/annotation/unaryop.py @@ -440,6 +440,12 @@ def method_popitem(dct): return dct.getanyitem('items') + def method_pop(dct, s_key, s_dfl=None): + dct.dictdef.generalize_key(s_key) + if s_dfl is not None: + dct.dictdef.generalize_value(s_dfl) + return dct.dictdef.read_value() + def _can_only_throw(dic, *ignore): if dic1.dictdef.dictkey.custom_eq_hash: return None # r_dict: can throw anything diff --git a/pypy/bin/py.py b/pypy/bin/py.py --- a/pypy/bin/py.py +++ b/pypy/bin/py.py @@ -76,6 +76,8 @@ config.objspace.suggest(allworkingmodules=False) if config.objspace.allworkingmodules: pypyoption.enable_allworkingmodules(config) + if config.objspace.usemodules._continuation: + config.translation.continuation = True if config.objspace.usemodules.thread: config.translation.thread = True diff --git a/pypy/doc/getting-started.rst b/pypy/doc/getting-started.rst --- a/pypy/doc/getting-started.rst +++ b/pypy/doc/getting-started.rst @@ -53,11 +53,11 @@ PyPy is ready to be executed as soon as you unpack the tarball or the zip file, with no need to install it in any specific location:: - $ tar xf pypy-1.6-linux.tar.bz2 + $ tar xf pypy-1.7-linux.tar.bz2 - $ ./pypy-1.6/bin/pypy + $ ./pypy-1.7/bin/pypy Python 2.7.1 (?, Apr 27 2011, 12:44:21) - [PyPy 1.6.0 with GCC 4.4.3] on linux2 + [PyPy 1.7.0 with GCC 4.4.3] on linux2 Type "help", "copyright", "credits" or "license" for more information. And now for something completely different: ``implementing LOGO in LOGO: "turtles all the way down"'' @@ -75,14 +75,14 @@ $ curl -O https://raw.github.com/pypa/pip/master/contrib/get-pip.py - $ ./pypy-1.6/bin/pypy distribute_setup.py + $ ./pypy-1.7/bin/pypy distribute_setup.py - $ ./pypy-1.6/bin/pypy get-pip.py + $ ./pypy-1.7/bin/pypy get-pip.py - $ ./pypy-1.6/bin/pip install pygments # for example + $ ./pypy-1.7/bin/pip install pygments # for example -3rd party libraries will be installed in ``pypy-1.6/site-packages``, and -the scripts in ``pypy-1.6/bin``. +3rd party libraries will be installed in ``pypy-1.7/site-packages``, and +the scripts in ``pypy-1.7/bin``. Installing using virtualenv --------------------------- diff --git a/pypy/doc/translation.rst b/pypy/doc/translation.rst --- a/pypy/doc/translation.rst +++ b/pypy/doc/translation.rst @@ -155,7 +155,7 @@ function. The two input variables are the exception class and the exception value, respectively. (No other block will actually link to the exceptblock if the function does not - explicitely raise exceptions.) + explicitly raise exceptions.) ``Block`` @@ -325,7 +325,7 @@ Mutable objects need special treatment during annotation, because the annotation of contained values needs to be possibly updated to account for mutation operations, and consequently the annotation information -reflown through the relevant parts of the flow the graphs. +reflown through the relevant parts of the flow graphs. * ``SomeList`` stands for a list of homogeneous type (i.e. all the elements of the list are represented by a single common ``SomeXxx`` @@ -503,8 +503,8 @@ Since RPython is a garbage collected language there is a lot of heap memory allocation going on all the time, which would either not occur at all in a more -traditional explicitely managed language or results in an object which dies at -a time known in advance and can thus be explicitely deallocated. For example a +traditional explicitly managed language or results in an object which dies at +a time known in advance and can thus be explicitly deallocated. For example a loop of the following form:: for i in range(n): @@ -696,7 +696,7 @@ So far it is the second most mature high level backend after GenCLI: it still can't translate the full Standard Interpreter, but after the -Leysin sprint we were able to compile and run the rpytstone and +Leysin sprint we were able to compile and run the rpystone and richards benchmarks. GenJVM is almost entirely the work of Niko Matsakis, who worked on it diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py --- a/pypy/interpreter/executioncontext.py +++ b/pypy/interpreter/executioncontext.py @@ -445,6 +445,7 @@ AsyncAction.__init__(self, space) self.dying_objects = [] self.finalizers_lock_count = 0 + self.enabled_at_app_level = True def register_callback(self, w_obj, callback, descrname): self.dying_objects.append((w_obj, callback, descrname)) diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py --- a/pypy/jit/backend/test/runner_test.py +++ b/pypy/jit/backend/test/runner_test.py @@ -551,6 +551,28 @@ res = self.execute_operation(rop.CALL, [funcbox] + map(BoxInt, args), 'int', descr=calldescr) assert res.value == func(*args) + def test_call_box_func(self): + def a(a1, a2): + return a1 + a2 + def b(b1, b2): + return b1 * b2 + + arg1 = 40 + arg2 = 2 + for f in [a, b]: + TP = lltype.Signed + FPTR = self.Ptr(self.FuncType([TP, TP], TP)) + func_ptr = llhelper(FPTR, f) + FUNC = deref(FPTR) + funcconst = self.get_funcbox(self.cpu, func_ptr) + funcbox = funcconst.clonebox() + calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, + EffectInfo.MOST_GENERAL) + res = self.execute_operation(rop.CALL, + [funcbox, BoxInt(arg1), BoxInt(arg2)], + 'int', descr=calldescr) + assert res.getint() == f(arg1, arg2) + def test_call_stack_alignment(self): # test stack alignment issues, notably for Mac OS/X. # also test the ordering of the arguments. @@ -1872,6 +1894,7 @@ values.append(descr) values.append(self.cpu.get_latest_value_int(0)) values.append(self.cpu.get_latest_value_int(1)) + values.append(token) FUNC = self.FuncType([lltype.Signed, lltype.Signed], lltype.Void) func_ptr = llhelper(lltype.Ptr(FUNC), maybe_force) @@ -1902,7 +1925,8 @@ assert fail.identifier == 1 assert self.cpu.get_latest_value_int(0) == 1 assert self.cpu.get_latest_value_int(1) == 10 - assert values == [faildescr, 1, 10] + token = self.cpu.get_latest_force_token() + assert values == [faildescr, 1, 10, token] def test_force_operations_returning_int(self): values = [] @@ -1911,6 +1935,7 @@ self.cpu.force(token) values.append(self.cpu.get_latest_value_int(0)) values.append(self.cpu.get_latest_value_int(2)) + values.append(token) return 42 FUNC = self.FuncType([lltype.Signed, lltype.Signed], lltype.Signed) @@ -1944,7 +1969,8 @@ assert self.cpu.get_latest_value_int(0) == 1 assert self.cpu.get_latest_value_int(1) == 42 assert self.cpu.get_latest_value_int(2) == 10 - assert values == [1, 10] + token = self.cpu.get_latest_force_token() + assert values == [1, 10, token] def test_force_operations_returning_float(self): values = [] @@ -1953,6 +1979,7 @@ self.cpu.force(token) values.append(self.cpu.get_latest_value_int(0)) values.append(self.cpu.get_latest_value_int(2)) + values.append(token) return 42.5 FUNC = self.FuncType([lltype.Signed, lltype.Signed], lltype.Float) @@ -1988,7 +2015,8 @@ x = self.cpu.get_latest_value_float(1) assert longlong.getrealfloat(x) == 42.5 assert self.cpu.get_latest_value_int(2) == 10 - assert values == [1, 10] + token = self.cpu.get_latest_force_token() + assert values == [1, 10, token] def test_call_to_c_function(self): from pypy.rlib.libffi import CDLL, types, ArgChain, FUNCFLAG_CDECL diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py --- a/pypy/jit/backend/x86/assembler.py +++ b/pypy/jit/backend/x86/assembler.py @@ -1118,6 +1118,12 @@ for src, dst in singlefloats: self.mc.MOVD(dst, src) # Finally remap the arguments in the main regs + # If x is a register and is in dst_locs, then oups, it needs to + # be moved away: + if x in dst_locs: + src_locs.append(x) + dst_locs.append(r10) + x = r10 remap_frame_layout(self, src_locs, dst_locs, X86_64_SCRATCH_REG) self._regalloc.reserve_param(len(pass_on_stack)) @@ -2042,10 +2048,7 @@ size = sizeloc.value signloc = arglocs[1] - if isinstance(op.getarg(0), Const): - x = imm(op.getarg(0).getint()) - else: - x = arglocs[2] + x = arglocs[2] # the function address if x is eax: tmp = ecx else: diff --git a/pypy/jit/backend/x86/runner.py b/pypy/jit/backend/x86/runner.py --- a/pypy/jit/backend/x86/runner.py +++ b/pypy/jit/backend/x86/runner.py @@ -6,7 +6,7 @@ from pypy.jit.codewriter import longlong from pypy.jit.metainterp import history, compile from pypy.jit.backend.x86.assembler import Assembler386 -from pypy.jit.backend.x86.arch import FORCE_INDEX_OFS +from pypy.jit.backend.x86.arch import FORCE_INDEX_OFS, IS_X86_32 from pypy.jit.backend.x86.profagent import ProfileAgent from pypy.jit.backend.llsupport.llmodel import AbstractLLCPU from pypy.jit.backend.x86 import regloc @@ -142,7 +142,9 @@ cast_ptr_to_int._annspecialcase_ = 'specialize:arglltype(0)' cast_ptr_to_int = staticmethod(cast_ptr_to_int) - all_null_registers = lltype.malloc(rffi.LONGP.TO, 24, + all_null_registers = lltype.malloc(rffi.LONGP.TO, + IS_X86_32 and (16+8) # 16 + 8 regs + or (16+16), # 16 + 16 regs flavor='raw', zero=True, immortal=True) diff --git a/pypy/jit/codewriter/assembler.py b/pypy/jit/codewriter/assembler.py --- a/pypy/jit/codewriter/assembler.py +++ b/pypy/jit/codewriter/assembler.py @@ -81,10 +81,15 @@ if not isinstance(value, (llmemory.AddressAsInt, ComputedIntSymbolic)): value = lltype.cast_primitive(lltype.Signed, value) - if allow_short and -128 <= value <= 127: - # emit the constant as a small integer - self.code.append(chr(value & 0xFF)) - return True + if allow_short: + try: + short_num = -128 <= value <= 127 + except TypeError: # "Symbolics cannot be compared!" + short_num = False + if short_num: + # emit the constant as a small integer + self.code.append(chr(value & 0xFF)) + return True constants = self.constants_i elif kind == 'ref': value = lltype.cast_opaque_ptr(llmemory.GCREF, value) diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py --- a/pypy/module/_codecs/interp_codecs.py +++ b/pypy/module/_codecs/interp_codecs.py @@ -108,6 +108,10 @@ w_result = state.codec_search_cache.get(normalized_encoding, None) if w_result is not None: return w_result + return _lookup_codec_loop(space, encoding, normalized_encoding) + +def _lookup_codec_loop(space, encoding, normalized_encoding): + state = space.fromcache(CodecState) if state.codec_need_encodings: w_import = space.getattr(space.builtin, space.wrap("__import__")) # registers new codecs diff --git a/pypy/module/_codecs/test/test_ztranslation.py b/pypy/module/_codecs/test/test_ztranslation.py new file mode 100644 --- /dev/null +++ b/pypy/module/_codecs/test/test_ztranslation.py @@ -0,0 +1,5 @@ +from pypy.objspace.fake.checkmodule import checkmodule + + +def test__codecs_translates(): + checkmodule('_codecs') diff --git a/pypy/module/_hashlib/interp_hashlib.py b/pypy/module/_hashlib/interp_hashlib.py --- a/pypy/module/_hashlib/interp_hashlib.py +++ b/pypy/module/_hashlib/interp_hashlib.py @@ -34,8 +34,12 @@ ctx = lltype.malloc(ropenssl.EVP_MD_CTX.TO, flavor='raw') rgc.add_memory_pressure(HASH_MALLOC_SIZE + self.digest_size) - ropenssl.EVP_DigestInit(ctx, digest_type) - self.ctx = ctx + try: + ropenssl.EVP_DigestInit(ctx, digest_type) + self.ctx = ctx + except: + lltype.free(ctx, flavor='raw') + raise def __del__(self): # self.lock.free() diff --git a/pypy/module/_io/interp_fileio.py b/pypy/module/_io/interp_fileio.py --- a/pypy/module/_io/interp_fileio.py +++ b/pypy/module/_io/interp_fileio.py @@ -349,6 +349,8 @@ try: s = os.read(self.fd, size) except OSError, e: + if e.errno == errno.EAGAIN: + return space.w_None raise wrap_oserror(space, e, exception_name='w_IOError') @@ -362,6 +364,8 @@ try: buf = os.read(self.fd, length) except OSError, e: + if e.errno == errno.EAGAIN: + return space.w_None raise wrap_oserror(space, e, exception_name='w_IOError') rwbuffer.setslice(0, buf) diff --git a/pypy/module/_io/test/test_fileio.py b/pypy/module/_io/test/test_fileio.py --- a/pypy/module/_io/test/test_fileio.py +++ b/pypy/module/_io/test/test_fileio.py @@ -133,6 +133,19 @@ f.close() assert a == 'a\nbxxxxxxx' + def test_nonblocking_read(self): + import os, fcntl + r_fd, w_fd = os.pipe() + # set nonblocking + fcntl.fcntl(r_fd, fcntl.F_SETFL, os.O_NONBLOCK) + import _io + f = _io.FileIO(r_fd, 'r') + # Read from stream sould return None + assert f.read() is None + assert f.read(10) is None + a = bytearray('x' * 10) + assert f.readinto(a) is None + def test_repr(self): import _io f = _io.FileIO(self.tmpfile, 'r') diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py --- a/pypy/module/_socket/interp_socket.py +++ b/pypy/module/_socket/interp_socket.py @@ -67,9 +67,6 @@ self.connect(self.addr_from_object(space, w_addr)) except SocketError, e: raise converted_error(space, e) - except TypeError, e: - raise OperationError(space.w_TypeError, - space.wrap(str(e))) def connect_ex_w(self, space, w_addr): """connect_ex(address) -> errno diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py --- a/pypy/module/_socket/test/test_sock_app.py +++ b/pypy/module/_socket/test/test_sock_app.py @@ -529,26 +529,31 @@ import _socket, os if not hasattr(_socket, 'AF_UNIX'): skip('AF_UNIX not supported.') - sockpath = os.path.join(self.udir, 'app_test_unix_socket_connect') + oldcwd = os.getcwd() + os.chdir(self.udir) + try: + sockpath = 'app_test_unix_socket_connect' - serversock = _socket.socket(_socket.AF_UNIX) - serversock.bind(sockpath) - serversock.listen(1) + serversock = _socket.socket(_socket.AF_UNIX) + serversock.bind(sockpath) + serversock.listen(1) - clientsock = _socket.socket(_socket.AF_UNIX) - clientsock.connect(sockpath) - s, addr = serversock.accept() - assert not addr + clientsock = _socket.socket(_socket.AF_UNIX) + clientsock.connect(sockpath) + s, addr = serversock.accept() + assert not addr - s.send('X') - data = clientsock.recv(100) - assert data == 'X' - clientsock.send('Y') - data = s.recv(100) - assert data == 'Y' + s.send('X') + data = clientsock.recv(100) + assert data == 'X' + clientsock.send('Y') + data = s.recv(100) + assert data == 'Y' - clientsock.close() - s.close() + clientsock.close() + s.close() + finally: + os.chdir(oldcwd) class AppTestSocketTCP: diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py --- a/pypy/module/_ssl/test/test_ssl.py +++ b/pypy/module/_ssl/test/test_ssl.py @@ -64,8 +64,8 @@ def test_sslwrap(self): import _ssl, _socket, sys, gc - if sys.platform == 'darwin': - skip("hangs indefinitely on OSX (also on CPython)") + if sys.platform == 'darwin' or 'freebsd' in sys.platform: + skip("hangs indefinitely on OSX & FreeBSD (also on CPython)") s = _socket.socket() ss = _ssl.sslwrap(s, 0) exc = raises(_socket.error, ss.do_handshake) diff --git a/pypy/module/cpyext/listobject.py b/pypy/module/cpyext/listobject.py --- a/pypy/module/cpyext/listobject.py +++ b/pypy/module/cpyext/listobject.py @@ -32,11 +32,10 @@ Py_DecRef(space, w_item) if not isinstance(w_list, W_ListObject): PyErr_BadInternalCall(space) - wrappeditems = w_list.getitems() - if index < 0 or index >= len(wrappeditems): + if index < 0 or index >= w_list.length(): raise OperationError(space.w_IndexError, space.wrap( "list assignment index out of range")) - wrappeditems[index] = w_item + w_list.setitem(index, w_item) return 0 @cpython_api([PyObject, Py_ssize_t], PyObject) @@ -74,7 +73,7 @@ """Macro form of PyList_Size() without error checking. """ assert isinstance(w_list, W_ListObject) - return len(w_list.getitems()) + return w_list.length() @cpython_api([PyObject], Py_ssize_t, error=-1) diff --git a/pypy/module/cpyext/test/test_listobject.py b/pypy/module/cpyext/test/test_listobject.py --- a/pypy/module/cpyext/test/test_listobject.py +++ b/pypy/module/cpyext/test/test_listobject.py @@ -128,3 +128,6 @@ module.setslice(l, None) assert l == [0, 4, 5] + l = [1, 2, 3] + module.setlistitem(l,0) + assert l == [None, 2, 3] diff --git a/pypy/module/fcntl/test/test_fcntl.py b/pypy/module/fcntl/test/test_fcntl.py --- a/pypy/module/fcntl/test/test_fcntl.py +++ b/pypy/module/fcntl/test/test_fcntl.py @@ -42,13 +42,9 @@ else: start_len = "qq" - if sys.platform in ('netbsd1', 'netbsd2', 'netbsd3', - 'Darwin1.2', 'darwin', - 'freebsd2', 'freebsd3', 'freebsd4', 'freebsd5', - 'freebsd6', 'freebsd7', 'freebsd8', 'freebsd9', - 'bsdos2', 'bsdos3', 'bsdos4', - 'openbsd', 'openbsd2', 'openbsd3', 'openbsd4', - 'openbsd5'): + if any(substring in sys.platform.lower() + for substring in ('netbsd', 'darwin', 'freebsd', 'bsdos', + 'openbsd')): if struct.calcsize('l') == 8: off_t = 'l' pid_t = 'i' @@ -182,7 +178,8 @@ def test_large_flag(self): import sys - if sys.platform == "darwin" or sys.platform.startswith("openbsd"): + if any(plat in sys.platform + for plat in ('darwin', 'openbsd', 'freebsd')): skip("Mac OS doesn't have any large flag in fcntl.h") import fcntl, sys if sys.maxint == 2147483647: diff --git a/pypy/module/gc/__init__.py b/pypy/module/gc/__init__.py --- a/pypy/module/gc/__init__.py +++ b/pypy/module/gc/__init__.py @@ -1,18 +1,18 @@ from pypy.interpreter.mixedmodule import MixedModule class Module(MixedModule): - appleveldefs = { - 'enable': 'app_gc.enable', - 'disable': 'app_gc.disable', - 'isenabled': 'app_gc.isenabled', - } interpleveldefs = { 'collect': 'interp_gc.collect', + 'enable': 'interp_gc.enable', + 'disable': 'interp_gc.disable', + 'isenabled': 'interp_gc.isenabled', 'enable_finalizers': 'interp_gc.enable_finalizers', 'disable_finalizers': 'interp_gc.disable_finalizers', 'garbage' : 'space.newlist([])', #'dump_heap_stats': 'interp_gc.dump_heap_stats', } + appleveldefs = { + } def __init__(self, space, w_name): if (not space.config.translating or diff --git a/pypy/module/gc/app_gc.py b/pypy/module/gc/app_gc.py deleted file mode 100644 --- a/pypy/module/gc/app_gc.py +++ /dev/null @@ -1,21 +0,0 @@ -# NOT_RPYTHON - -enabled = True - -def isenabled(): - global enabled - return enabled - -def enable(): - global enabled - import gc - if not enabled: - gc.enable_finalizers() - enabled = True - -def disable(): - global enabled - import gc - if enabled: - gc.disable_finalizers() - enabled = False diff --git a/pypy/module/gc/interp_gc.py b/pypy/module/gc/interp_gc.py --- a/pypy/module/gc/interp_gc.py +++ b/pypy/module/gc/interp_gc.py @@ -17,6 +17,26 @@ rgc.collect() return space.wrap(0) +def enable(space): + """Non-recursive version. Enable finalizers now. + If they were already enabled, no-op. + If they were disabled even several times, enable them anyway. + """ + if not space.user_del_action.enabled_at_app_level: + space.user_del_action.enabled_at_app_level = True + enable_finalizers(space) + +def disable(space): + """Non-recursive version. Disable finalizers now. Several calls + to this function are ignored. + """ + if space.user_del_action.enabled_at_app_level: + space.user_del_action.enabled_at_app_level = False + disable_finalizers(space) + +def isenabled(space): + return space.newbool(space.user_del_action.enabled_at_app_level) + def enable_finalizers(space): if space.user_del_action.finalizers_lock_count == 0: raise OperationError(space.w_ValueError, diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py --- a/pypy/module/micronumpy/__init__.py +++ b/pypy/module/micronumpy/__init__.py @@ -27,6 +27,12 @@ 'dot': 'interp_numarray.dot', 'fromstring': 'interp_support.fromstring', 'flatiter': 'interp_numarray.W_FlatIterator', + 'isna': 'interp_numarray.isna', + 'concatenate': 'interp_numarray.concatenate', + + 'set_string_function': 'appbridge.set_string_function', + + 'count_reduce_items': 'interp_numarray.count_reduce_items', 'True_': 'types.Bool.True', 'False_': 'types.Bool.False', @@ -66,10 +72,12 @@ ("copysign", "copysign"), ("cos", "cos"), ("divide", "divide"), + ("true_divide", "true_divide"), ("equal", "equal"), ("exp", "exp"), ("fabs", "fabs"), ("floor", "floor"), + ("ceil", "ceil"), ("greater", "greater"), ("greater_equal", "greater_equal"), ("less", "less"), @@ -85,12 +93,16 @@ ("subtract", "subtract"), ('sqrt', 'sqrt'), ("tan", "tan"), + ('bitwise_and', 'bitwise_and'), + ('bitwise_or', 'bitwise_or'), + ('bitwise_not', 'invert'), + ('isnan', 'isnan'), + ('isinf', 'isinf'), ]: interpleveldefs[exposed] = "interp_ufuncs.get(space).%s" % impl appleveldefs = { 'average': 'app_numpy.average', - 'mean': 'app_numpy.mean', 'sum': 'app_numpy.sum', 'min': 'app_numpy.min', 'identity': 'app_numpy.identity', @@ -99,5 +111,4 @@ 'e': 'app_numpy.e', 'pi': 'app_numpy.pi', 'arange': 'app_numpy.arange', - 'reshape': 'app_numpy.reshape', } diff --git a/pypy/module/micronumpy/app_numpy.py b/pypy/module/micronumpy/app_numpy.py --- a/pypy/module/micronumpy/app_numpy.py +++ b/pypy/module/micronumpy/app_numpy.py @@ -11,23 +11,20 @@ def average(a): # This implements a weighted average, for now we don't implement the # weighting, just the average part! - return mean(a) + if not hasattr(a, "mean"): + a = _numpypy.array(a) + return a.mean() def identity(n, dtype=None): - a = _numpypy.zeros((n,n), dtype=dtype) + a = _numpypy.zeros((n, n), dtype=dtype) for i in range(n): a[i][i] = 1 return a -def mean(a, axis=None): - if not hasattr(a, "mean"): - a = _numpypy.array(a) - return a.mean(axis) - def sum(a,axis=None): '''sum(a, axis=None) Sum of array elements over a given axis. - + Parameters ---------- a : array_like @@ -35,7 +32,7 @@ axis : integer, optional Axis over which the sum is taken. By default `axis` is None, and all elements are summed. - + Returns ------- sum_along_axis : ndarray @@ -43,7 +40,7 @@ axis removed. If `a` is a 0-d array, or if `axis` is None, a scalar is returned. If an output array is specified, a reference to `out` is returned. - + See Also -------- ndarray.sum : Equivalent method. @@ -62,7 +59,7 @@ if not hasattr(a, "max"): a = _numpypy.array(a) return a.max(axis) - + def arange(start, stop=None, step=1, dtype=None): '''arange([start], stop[, step], dtype=None) Generate values in the half-interval [start, stop). @@ -79,40 +76,3 @@ arr[j] = i i += step return arr - - -def reshape(a, shape): - '''reshape(a, newshape) - Gives a new shape to an array without changing its data. - - Parameters - ---------- - a : array_like - Array to be reshaped. - newshape : int or tuple of ints - The new shape should be compatible with the original shape. If - an integer, then the result will be a 1-D array of that length. - One shape dimension can be -1. In this case, the value is inferred - from the length of the array and remaining dimensions. - - Returns - ------- - reshaped_array : ndarray - This will be a new view object if possible; otherwise, it will - be a copy. - - - See Also - -------- - ndarray.reshape : Equivalent method. - - Notes - ----- - - It is not always possible to change the shape of an array without - copying the data. If you want an error to be raise if the data is copied, - you should assign the new shape to the shape attribute of the array -''' - if not hasattr(a, 'reshape'): - a = _numpypy.array(a) - return a.reshape(shape) diff --git a/pypy/module/micronumpy/appbridge.py b/pypy/module/micronumpy/appbridge.py new file mode 100644 --- /dev/null +++ b/pypy/module/micronumpy/appbridge.py @@ -0,0 +1,38 @@ + +from pypy.rlib.objectmodel import specialize + +class AppBridgeCache(object): + w__var = None + w__std = None + w_module = None + w_array_repr = None + w_array_str = None + + def __init__(self, space): + self.w_import = space.appexec([], """(): + def f(): + import sys + __import__('numpypy.core._methods') + return sys.modules['numpypy.core._methods'] + return f + """) + + @specialize.arg(2) + def call_method(self, space, name, *args): + w_meth = getattr(self, 'w_' + name) + if w_meth is None: + if self.w_module is None: + self.w_module = space.call_function(self.w_import) + w_meth = space.getattr(self.w_module, space.wrap(name)) + setattr(self, 'w_' + name, w_meth) + return space.call_function(w_meth, *args) + +def set_string_function(space, w_f, w_repr): + cache = get_appbridge_cache(space) + if space.is_true(w_repr): + cache.w_array_repr = w_f + else: + cache.w_array_str = w_f + +def get_appbridge_cache(space): + return space.fromcache(AppBridgeCache) diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py --- a/pypy/module/micronumpy/compile.py +++ b/pypy/module/micronumpy/compile.py @@ -32,14 +32,16 @@ class BadToken(Exception): pass -SINGLE_ARG_FUNCTIONS = ["sum", "prod", "max", "min", "all", "any", "unegative"] -TWO_ARG_FUNCTIONS = ["dot"] +SINGLE_ARG_FUNCTIONS = ["sum", "prod", "max", "min", "all", "any", + "unegative", "flat"] +TWO_ARG_FUNCTIONS = ["dot", 'take'] class FakeSpace(object): w_ValueError = None w_TypeError = None w_IndexError = None w_OverflowError = None + w_NotImplementedError = None w_None = None w_bool = "bool" @@ -54,6 +56,9 @@ """NOT_RPYTHON""" self.fromcache = InternalSpaceCache(self).getorbuild + def _freeze_(self): + return True + def issequence_w(self, w_obj): return isinstance(w_obj, ListObject) or isinstance(w_obj, W_NDimArray) @@ -145,6 +150,9 @@ def allocate_instance(self, klass, w_subtype): return instantiate(klass) + def newtuple(self, list_w): + raise ValueError + def len_w(self, w_obj): if isinstance(w_obj, ListObject): return len(w_obj.items) @@ -372,12 +380,12 @@ for arg in self.args])) def execute(self, interp): + arr = self.args[0].execute(interp) + if not isinstance(arr, BaseArray): + raise ArgumentNotAnArray if self.name in SINGLE_ARG_FUNCTIONS: if len(self.args) != 1 and self.name != 'sum': raise ArgumentMismatch - arr = self.args[0].execute(interp) - if not isinstance(arr, BaseArray): - raise ArgumentNotAnArray if self.name == "sum": if len(self.args)>1: w_res = arr.descr_sum(interp.space, @@ -397,19 +405,22 @@ elif self.name == "unegative": neg = interp_ufuncs.get(interp.space).negative w_res = neg.call(interp.space, [arr]) + elif self.name == "flat": + w_res = arr.descr_get_flatiter(interp.space) else: assert False # unreachable code elif self.name in TWO_ARG_FUNCTIONS: if len(self.args) != 2: raise ArgumentMismatch - arr0 = self.args[0].execute(interp) - arr1 = self.args[1].execute(interp) - if not isinstance(arr0, BaseArray): + arg = self.args[1].execute(interp) + if not isinstance(arg, BaseArray): raise ArgumentNotAnArray - if not isinstance(arr1, BaseArray): + if not isinstance(arg, BaseArray): raise ArgumentNotAnArray if self.name == "dot": - w_res = arr0.descr_dot(interp.space, arr1) + w_res = arr.descr_dot(interp.space, arg) + elif self.name == 'take': + w_res = arr.descr_take(interp.space, arg) else: assert False # unreachable code else: diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py --- a/pypy/module/micronumpy/interp_boxes.py +++ b/pypy/module/micronumpy/interp_boxes.py @@ -8,6 +8,7 @@ from pypy.tool.sourcetools import func_with_new_name +MIXIN_32 = (int_typedef,) if LONG_BIT == 32 else () MIXIN_64 = (int_typedef,) if LONG_BIT == 64 else () def new_dtype_getter(name): @@ -28,6 +29,7 @@ def convert_to(self, dtype): return dtype.box(self.value) + class W_GenericBox(Wrappable): _attrs_ = () @@ -93,7 +95,7 @@ descr_neg = _unaryop_impl("negative") descr_abs = _unaryop_impl("absolute") - def descr_tolist(self, space): + def item(self, space): return self.get_dtype(space).itemtype.to_builtin_type(space, self) @@ -104,7 +106,8 @@ _attrs_ = () class W_IntegerBox(W_NumberBox): - pass + def int_w(self, space): + return space.int_w(self.descr_int(space)) class W_SignedIntegerBox(W_IntegerBox): pass @@ -187,7 +190,7 @@ __neg__ = interp2app(W_GenericBox.descr_neg), __abs__ = interp2app(W_GenericBox.descr_abs), - tolist = interp2app(W_GenericBox.descr_tolist), + tolist = interp2app(W_GenericBox.item), ) W_BoolBox.typedef = TypeDef("bool_", W_GenericBox.typedef, @@ -231,7 +234,7 @@ __new__ = interp2app(W_UInt16Box.descr__new__.im_func), ) -W_Int32Box.typedef = TypeDef("int32", W_SignedIntegerBox.typedef, +W_Int32Box.typedef = TypeDef("int32", (W_SignedIntegerBox.typedef,) + MIXIN_32, __module__ = "numpypy", __new__ = interp2app(W_Int32Box.descr__new__.im_func), ) @@ -241,24 +244,18 @@ __new__ = interp2app(W_UInt32Box.descr__new__.im_func), ) -if LONG_BIT == 32: - long_name = "int32" -elif LONG_BIT == 64: - long_name = "int64" -W_LongBox.typedef = TypeDef(long_name, (W_SignedIntegerBox.typedef, int_typedef,), - __module__ = "numpypy", - __new__ = interp2app(W_LongBox.descr__new__.im_func), -) - -W_ULongBox.typedef = TypeDef("u" + long_name, W_UnsignedIntegerBox.typedef, - __module__ = "numpypy", -) - W_Int64Box.typedef = TypeDef("int64", (W_SignedIntegerBox.typedef,) + MIXIN_64, __module__ = "numpypy", __new__ = interp2app(W_Int64Box.descr__new__.im_func), ) +if LONG_BIT == 32: + W_LongBox = W_Int32Box + W_ULongBox = W_UInt32Box +elif LONG_BIT == 64: + W_LongBox = W_Int64Box + W_ULongBox = W_UInt64Box + W_UInt64Box.typedef = TypeDef("uint64", W_UnsignedIntegerBox.typedef, __module__ = "numpypy", __new__ = interp2app(W_UInt64Box.descr__new__.im_func), @@ -283,3 +280,4 @@ __new__ = interp2app(W_Float64Box.descr__new__.im_func), ) + diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py --- a/pypy/module/micronumpy/interp_dtype.py +++ b/pypy/module/micronumpy/interp_dtype.py @@ -20,7 +20,7 @@ class W_Dtype(Wrappable): _immutable_fields_ = ["itemtype", "num", "kind"] - def __init__(self, itemtype, num, kind, name, char, w_box_type, alternate_constructors=[]): + def __init__(self, itemtype, num, kind, name, char, w_box_type, alternate_constructors=[], aliases=[]): self.itemtype = itemtype self.num = num self.kind = kind @@ -28,6 +28,7 @@ self.char = char self.w_box_type = w_box_type self.alternate_constructors = alternate_constructors + self.aliases = aliases def malloc(self, length): # XXX find out why test_zjit explodes with tracking of allocations @@ -46,6 +47,10 @@ def getitem(self, storage, i): return self.itemtype.read(storage, self.itemtype.get_element_size(), i, 0) + def getitem_bool(self, storage, i): + isize = self.itemtype.get_element_size() + return self.itemtype.read_bool(storage, isize, i, 0) + def setitem(self, storage, i, box): self.itemtype.store(storage, self.itemtype.get_element_size(), i, 0, box) @@ -62,7 +67,7 @@ elif space.isinstance_w(w_dtype, space.w_str): name = space.str_w(w_dtype) for dtype in cache.builtin_dtypes: - if dtype.name == name or dtype.char == name: + if dtype.name == name or dtype.char == name or name in dtype.aliases: return dtype else: for dtype in cache.builtin_dtypes: @@ -84,18 +89,38 @@ def descr_get_shape(self, space): return space.newtuple([]) + def eq(self, space, w_other): + w_other = space.call_function(space.gettypefor(W_Dtype), w_other) + return space.is_w(self, w_other) + + def descr_eq(self, space, w_other): + return space.wrap(self.eq(space, w_other)) + + def descr_ne(self, space, w_other): + return space.wrap(not self.eq(space, w_other)) + + def is_int_type(self): + return (self.kind == SIGNEDLTR or self.kind == UNSIGNEDLTR or + self.kind == BOOLLTR) + + def is_bool_type(self): + return self.kind == BOOLLTR + W_Dtype.typedef = TypeDef("dtype", __module__ = "numpypy", __new__ = interp2app(W_Dtype.descr__new__.im_func), __str__= interp2app(W_Dtype.descr_str), __repr__ = interp2app(W_Dtype.descr_repr), + __eq__ = interp2app(W_Dtype.descr_eq), + __ne__ = interp2app(W_Dtype.descr_ne), num = interp_attrproperty("num", cls=W_Dtype), kind = interp_attrproperty("kind", cls=W_Dtype), type = interp_attrproperty_w("w_box_type", cls=W_Dtype), itemsize = GetSetProperty(W_Dtype.descr_get_itemsize), shape = GetSetProperty(W_Dtype.descr_get_shape), + name = interp_attrproperty('name', cls=W_Dtype), ) W_Dtype.typedef.acceptable_as_base_class = False @@ -107,7 +132,7 @@ kind=BOOLLTR, name="bool", char="?", - w_box_type = space.gettypefor(interp_boxes.W_BoolBox), + w_box_type=space.gettypefor(interp_boxes.W_BoolBox), alternate_constructors=[space.w_bool], ) self.w_int8dtype = W_Dtype( @@ -116,7 +141,7 @@ kind=SIGNEDLTR, name="int8", char="b", - w_box_type = space.gettypefor(interp_boxes.W_Int8Box) + w_box_type=space.gettypefor(interp_boxes.W_Int8Box) ) self.w_uint8dtype = W_Dtype( types.UInt8(), @@ -124,7 +149,7 @@ kind=UNSIGNEDLTR, name="uint8", char="B", - w_box_type = space.gettypefor(interp_boxes.W_UInt8Box), + w_box_type=space.gettypefor(interp_boxes.W_UInt8Box), ) self.w_int16dtype = W_Dtype( types.Int16(), @@ -132,7 +157,7 @@ kind=SIGNEDLTR, name="int16", char="h", - w_box_type = space.gettypefor(interp_boxes.W_Int16Box), + w_box_type=space.gettypefor(interp_boxes.W_Int16Box), ) self.w_uint16dtype = W_Dtype( types.UInt16(), @@ -140,7 +165,7 @@ kind=UNSIGNEDLTR, name="uint16", char="H", - w_box_type = space.gettypefor(interp_boxes.W_UInt16Box), + w_box_type=space.gettypefor(interp_boxes.W_UInt16Box), ) self.w_int32dtype = W_Dtype( types.Int32(), @@ -148,7 +173,7 @@ kind=SIGNEDLTR, name="int32", char="i", - w_box_type = space.gettypefor(interp_boxes.W_Int32Box), + w_box_type=space.gettypefor(interp_boxes.W_Int32Box), ) self.w_uint32dtype = W_Dtype( types.UInt32(), @@ -156,7 +181,7 @@ kind=UNSIGNEDLTR, name="uint32", char="I", - w_box_type = space.gettypefor(interp_boxes.W_UInt32Box), + w_box_type=space.gettypefor(interp_boxes.W_UInt32Box), ) if LONG_BIT == 32: name = "int32" @@ -168,7 +193,7 @@ kind=SIGNEDLTR, name=name, char="l", - w_box_type = space.gettypefor(interp_boxes.W_LongBox), + w_box_type=space.gettypefor(interp_boxes.W_LongBox), alternate_constructors=[space.w_int], ) self.w_ulongdtype = W_Dtype( @@ -177,7 +202,7 @@ kind=UNSIGNEDLTR, name="u" + name, char="L", - w_box_type = space.gettypefor(interp_boxes.W_ULongBox), + w_box_type=space.gettypefor(interp_boxes.W_ULongBox), ) self.w_int64dtype = W_Dtype( types.Int64(), @@ -185,7 +210,7 @@ kind=SIGNEDLTR, name="int64", char="q", - w_box_type = space.gettypefor(interp_boxes.W_Int64Box), + w_box_type=space.gettypefor(interp_boxes.W_Int64Box), alternate_constructors=[space.w_long], ) self.w_uint64dtype = W_Dtype( @@ -194,7 +219,7 @@ kind=UNSIGNEDLTR, name="uint64", char="Q", - w_box_type = space.gettypefor(interp_boxes.W_UInt64Box), + w_box_type=space.gettypefor(interp_boxes.W_UInt64Box), ) self.w_float32dtype = W_Dtype( types.Float32(), @@ -202,7 +227,7 @@ kind=FLOATINGLTR, name="float32", char="f", - w_box_type = space.gettypefor(interp_boxes.W_Float32Box), + w_box_type=space.gettypefor(interp_boxes.W_Float32Box), ) self.w_float64dtype = W_Dtype( types.Float64(), @@ -212,6 +237,7 @@ char="d", w_box_type = space.gettypefor(interp_boxes.W_Float64Box), alternate_constructors=[space.w_float], + aliases=["float"], ) self.builtin_dtypes = [ diff --git a/pypy/module/micronumpy/interp_iter.py b/pypy/module/micronumpy/interp_iter.py --- a/pypy/module/micronumpy/interp_iter.py +++ b/pypy/module/micronumpy/interp_iter.py @@ -4,6 +4,66 @@ from pypy.module.micronumpy.strides import calculate_broadcast_strides,\ calculate_slice_strides, calculate_dot_strides +""" This is a mini-tutorial on iterators, strides, and +memory layout. It assumes you are familiar with the terms, see +http://docs.scipy.org/doc/numpy/reference/arrays.ndarray.html +for a more gentle introduction. + +Given an array x: x.shape == [5,6], + +At which byte in x.data does the item x[3,4] begin? +if x.strides==[1,5]: + pData = x.pData + (x.start + 3*1 + 4*5)*sizeof(x.pData[0]) + pData = x.pData + (x.start + 24) * sizeof(x.pData[0]) +so the offset of the element is 24 elements after the first + +What is the next element in x after coordinates [3,4]? +if x.order =='C': + next == [3,5] => offset is 28 +if x.order =='F': + next == [4,4] => offset is 24 +so for the strides [1,5] x is 'F' contiguous +likewise, for the strides [6,1] x would be 'C' contiguous. + +Iterators have an internal representation of the current coordinates +(indices), the array, strides, and backstrides. A short digression to +explain backstrides: what is the coordinate and offset after [3,5] in +the example above? +if x.order == 'C': + next == [4,0] => offset is 4 +if x.order == 'F': + next == [4,5] => offset is 25 +Note that in 'C' order we stepped BACKWARDS 24 while 'overflowing' a +shape dimension + which is back 25 and forward 1, + which is x.strides[1] * (x.shape[1] - 1) + x.strides[0] +so if we precalculate the overflow backstride as +[x.strides[i] * (x.shape[i] - 1) for i in range(len(x.shape))] +we can go faster. +All the calculations happen in next() + +next_step_x() tries to do the iteration for a number of steps at once, +but then we cannot gaurentee that we only overflow one single shape +dimension, perhaps we could overflow times in one big step. +""" + +# structures to describe slicing + +class Chunk(object): + def __init__(self, start, stop, step, lgt): + self.start = start + self.stop = stop + self.step = step + self.lgt = lgt + + def extend_shape(self, shape): + if self.step != 0: + shape.append(self.lgt) + + def __repr__(self): + return 'Chunk(%d, %d, %d, %d)' % (self.start, self.stop, self.step, + self.lgt) + class BaseTransform(object): pass @@ -39,11 +99,18 @@ self.size = size def next(self, shapelen): + return self.next_skip_x(1) + + def next_skip_x(self, ofs): arr = instantiate(ArrayIterator) arr.size = self.size - arr.offset = self.offset + 1 + arr.offset = self.offset + ofs return arr + def next_no_increase(self, shapelen): + # a hack to make JIT believe this is always virtual + return self.next_skip_x(0) + def done(self): return self.offset >= self.size @@ -114,6 +181,36 @@ res._done = done return res + @jit.unroll_safe + def next_skip_x(self, shapelen, step): + shapelen = jit.promote(len(self.res_shape)) + offset = self.offset + indices = [0] * shapelen + for i in range(shapelen): + indices[i] = self.indices[i] + done = False + for i in range(shapelen - 1, -1, -1): + if indices[i] < self.res_shape[i] - step: + indices[i] += step + offset += self.strides[i] * step + break + else: + remaining_step = (indices[i] + step) // self.res_shape[i] + this_i_step = step - remaining_step * self.res_shape[i] + offset += self.strides[i] * this_i_step + indices[i] = indices[i] + this_i_step + step = remaining_step + else: + done = True + res = instantiate(ViewIterator) + res.offset = offset + res.indices = indices + res.strides = self.strides + res.backstrides = self.backstrides + res.res_shape = self.res_shape + res._done = done + return res + def apply_transformations(self, arr, transformations): v = BaseIterator.apply_transformations(self, arr, transformations) if len(arr.shape) == 1: @@ -135,8 +232,13 @@ class AxisIterator(BaseIterator): def __init__(self, start, dim, shape, strides, backstrides): self.res_shape = shape[:] - self.strides = strides[:dim] + [0] + strides[dim:] - self.backstrides = backstrides[:dim] + [0] + backstrides[dim:] + if len(shape) == len(strides): + # keepdims = True + self.strides = strides[:dim] + [0] + strides[dim + 1:] + self.backstrides = backstrides[:dim] + [0] + backstrides[dim + 1:] + else: + self.strides = strides[:dim] + [0] + strides[dim:] + self.backstrides = backstrides[:dim] + [0] + backstrides[dim:] self.first_line = True self.indices = [0] * len(shape) self._done = False diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -1,16 +1,19 @@ from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.error import OperationError, operationerrfmt -from pypy.interpreter.gateway import interp2app, NoneNotWrapped +from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.interpreter.typedef import TypeDef, GetSetProperty -from pypy.module.micronumpy import interp_ufuncs, interp_dtype, signature -from pypy.module.micronumpy.strides import calculate_slice_strides +from pypy.module.micronumpy import (interp_ufuncs, interp_dtype, interp_boxes, + signature, support) +from pypy.module.micronumpy.strides import (calculate_slice_strides, + shape_agreement, find_shape_and_elems, get_shape_from_iterable, + calc_new_strides, to_coords) from pypy.rlib import jit from pypy.rpython.lltypesystem import lltype, rffi from pypy.tool.sourcetools import func_with_new_name from pypy.rlib.rstring import StringBuilder -from pypy.module.micronumpy.interp_iter import ArrayIterator, OneDimIterator,\ - SkipLastAxisIterator -from pypy.module.micronumpy.dot import multidim_dot, match_dot_shapes +from pypy.module.micronumpy.interp_iter import (ArrayIterator, OneDimIterator, + SkipLastAxisIterator, Chunk, ViewIterator) +from pypy.module.micronumpy.appbridge import get_appbridge_cache numpy_driver = jit.JitDriver( @@ -41,177 +44,39 @@ get_printable_location=signature.new_printable_location('slice'), name='numpy_slice', ) - - -def _find_shape_and_elems(space, w_iterable): - shape = [space.len_w(w_iterable)] - batch = space.listview(w_iterable) - while True: - new_batch = [] - if not batch: - return shape, [] - if not space.issequence_w(batch[0]): - for elem in batch: - if space.issequence_w(elem): - raise OperationError(space.w_ValueError, space.wrap( - "setting an array element with a sequence")) - return shape, batch - size = space.len_w(batch[0]) - for w_elem in batch: - if not space.issequence_w(w_elem) or space.len_w(w_elem) != size: - raise OperationError(space.w_ValueError, space.wrap( - "setting an array element with a sequence")) - new_batch += space.listview(w_elem) - shape.append(size) - batch = new_batch - -def shape_agreement(space, shape1, shape2): - ret = _shape_agreement(shape1, shape2) - if len(ret) < max(len(shape1), len(shape2)): - raise OperationError(space.w_ValueError, - space.wrap("operands could not be broadcast together with shapes (%s) (%s)" % ( - ",".join([str(x) for x in shape1]), - ",".join([str(x) for x in shape2]), - )) - ) - return ret - -def _shape_agreement(shape1, shape2): - """ Checks agreement about two shapes with respect to broadcasting. Returns - the resulting shape. - """ - lshift = 0 - rshift = 0 - if len(shape1) > len(shape2): - m = len(shape1) - n = len(shape2) - rshift = len(shape2) - len(shape1) - remainder = shape1 - else: - m = len(shape2) - n = len(shape1) - lshift = len(shape1) - len(shape2) - remainder = shape2 - endshape = [0] * m - indices1 = [True] * m - indices2 = [True] * m - for i in range(m - 1, m - n - 1, -1): - left = shape1[i + lshift] - right = shape2[i + rshift] - if left == right: - endshape[i] = left - elif left == 1: - endshape[i] = right - indices1[i + lshift] = False - elif right == 1: - endshape[i] = left - indices2[i + rshift] = False - else: - return [] - #raise OperationError(space.w_ValueError, space.wrap( - # "frames are not aligned")) - for i in range(m - n): - endshape[i] = remainder[i] - return endshape - -def get_shape_from_iterable(space, old_size, w_iterable): - new_size = 0 - new_shape = [] - if space.isinstance_w(w_iterable, space.w_int): - new_size = space.int_w(w_iterable) - if new_size < 0: - new_size = old_size - new_shape = [new_size] - else: - neg_dim = -1 - batch = space.listview(w_iterable) - new_size = 1 - if len(batch) < 1: - if old_size == 1: - # Scalars can have an empty size. - new_size = 1 - else: - new_size = 0 - new_shape = [] - i = 0 - for elem in batch: - s = space.int_w(elem) - if s < 0: - if neg_dim >= 0: - raise OperationError(space.w_ValueError, space.wrap( - "can only specify one unknown dimension")) - s = 1 - neg_dim = i - new_size *= s - new_shape.append(s) - i += 1 - if neg_dim >= 0: - new_shape[neg_dim] = old_size / new_size - new_size *= new_shape[neg_dim] - if new_size != old_size: - raise OperationError(space.w_ValueError, - space.wrap("total size of new array must be unchanged")) - return new_shape - -# Recalculating strides. Find the steps that the iteration does for each -# dimension, given the stride and shape. Then try to create a new stride that -# fits the new shape, using those steps. If there is a shape/step mismatch -# (meaning that the realignment of elements crosses from one step into another) -# return None so that the caller can raise an exception. -def calc_new_strides(new_shape, old_shape, old_strides): - # Assumes that prod(old_shape) == prod(new_shape), len(old_shape) > 1, and - # len(new_shape) > 0 - steps = [] - last_step = 1 - oldI = 0 - new_strides = [] - if old_strides[0] < old_strides[-1]: - #Start at old_shape[0], old_stides[0] - for i in range(len(old_shape)): - steps.append(old_strides[i] / last_step) - last_step *= old_shape[i] - cur_step = steps[0] - n_new_elems_used = 1 - n_old_elems_to_use = old_shape[0] - for s in new_shape: - new_strides.append(cur_step * n_new_elems_used) - n_new_elems_used *= s - while n_new_elems_used > n_old_elems_to_use: - oldI += 1 - if steps[oldI] != steps[oldI - 1]: - return None - n_old_elems_to_use *= old_shape[oldI] - if n_new_elems_used == n_old_elems_to_use: - oldI += 1 - if oldI >= len(old_shape): - continue - cur_step = steps[oldI] - n_old_elems_to_use *= old_shape[oldI] - else: - #Start at old_shape[-1], old_strides[-1] - for i in range(len(old_shape) - 1, -1, -1): - steps.insert(0, old_strides[i] / last_step) - last_step *= old_shape[i] - cur_step = steps[-1] - n_new_elems_used = 1 - oldI = -1 - n_old_elems_to_use = old_shape[-1] - for i in range(len(new_shape) - 1, -1, -1): - s = new_shape[i] - new_strides.insert(0, cur_step * n_new_elems_used) - n_new_elems_used *= s - while n_new_elems_used > n_old_elems_to_use: - oldI -= 1 - if steps[oldI] != steps[oldI + 1]: - return None - n_old_elems_to_use *= old_shape[oldI] - if n_new_elems_used == n_old_elems_to_use: - oldI -= 1 - if oldI < -len(old_shape): - continue - cur_step = steps[oldI] - n_old_elems_to_use *= old_shape[oldI] - return new_strides +count_driver = jit.JitDriver( + greens=['shapelen'], + virtualizables=['frame'], + reds=['s', 'frame', 'iter', 'arr'], + name='numpy_count' +) +filter_driver = jit.JitDriver( + greens=['shapelen', 'sig'], + virtualizables=['frame'], + reds=['concr', 'argi', 'ri', 'frame', 'v', 'res', 'self'], + name='numpy_filter', +) +filter_set_driver = jit.JitDriver( + greens=['shapelen', 'sig'], + virtualizables=['frame'], + reds=['idx', 'idxi', 'frame', 'arr'], + name='numpy_filterset', +) +take_driver = jit.JitDriver( + greens=['shapelen', 'sig'], + reds=['index_i', 'res_i', 'concr', 'index', 'res'], + name='numpy_take', +) +flat_get_driver = jit.JitDriver( + greens=['shapelen', 'base'], + reds=['step', 'ri', 'basei', 'res'], + name='numpy_flatget', +) +flat_set_driver = jit.JitDriver( + greens=['shapelen', 'base'], + reds=['step', 'ai', 'lngth', 'arr', 'basei'], + name='numpy_flatset', +) class BaseArray(Wrappable): _attrs_ = ["invalidates", "shape", 'size'] @@ -252,6 +117,7 @@ descr_pos = _unaryop_impl("positive") descr_neg = _unaryop_impl("negative") descr_abs = _unaryop_impl("absolute") + descr_invert = _unaryop_impl("invert") def _binop_impl(ufunc_name): def impl(self, space, w_other): @@ -272,6 +138,9 @@ descr_gt = _binop_impl("greater") descr_ge = _binop_impl("greater_equal") + descr_and = _binop_impl("bitwise_and") + descr_or = _binop_impl("bitwise_or") + def _binop_right_impl(ufunc_name): def impl(self, space, w_other): w_other = scalar_w(space, @@ -291,9 +160,11 @@ def _reduce_ufunc_impl(ufunc_name, promote_to_largest=False): def impl(self, space, w_axis=None): if space.is_w(w_axis, space.w_None): - w_axis = space.wrap(-1) + axis = -1 + else: + axis = space.int_w(w_axis) return getattr(interp_ufuncs.get(space), ufunc_name).reduce(space, - self, True, promote_to_largest, w_axis) + self, True, promote_to_largest, axis) return func_with_new_name(impl, "reduce_%s_impl" % ufunc_name) descr_sum = _reduce_ufunc_impl("add") @@ -427,9 +298,23 @@ def descr_copy(self, space): return self.copy(space) + def descr_flatten(self, space, w_order=None): + if isinstance(self, Scalar): + # scalars have no storage + return self.descr_reshape(space, [space.wrap(1)]) + concr = self.get_concrete() + w_res = concr.descr_ravel(space, w_order) + if w_res.storage == concr.storage: + return w_res.copy(space) + return w_res + def copy(self, space): return self.get_concrete().copy(space) + def empty_copy(self, space, dtype): + shape = self.shape + return W_NDimArray(support.product(shape), shape[:], dtype, 'C') + def descr_len(self, space): if len(self.shape): return space.wrap(self.shape[0]) @@ -437,33 +322,32 @@ "len() of unsized object")) def descr_repr(self, space): - res = StringBuilder() - res.append("array(") - concrete = self.get_concrete_or_scalar() - dtype = concrete.find_dtype() - if not concrete.size: - res.append('[]') - if len(self.shape) > 1: - # An empty slice reports its shape - res.append(", shape=(") - self_shape = str(self.shape) - res.append_slice(str(self_shape), 1, len(self_shape) - 1) - res.append(')') - else: - concrete.to_str(space, 1, res, indent=' ') - if (dtype is not interp_dtype.get_dtype_cache(space).w_float64dtype and - not (dtype.kind == interp_dtype.SIGNEDLTR and - dtype.itemtype.get_element_size() == rffi.sizeof(lltype.Signed)) or - not self.size): - res.append(", dtype=" + dtype.name) - res.append(")") - return space.wrap(res.build()) + cache = get_appbridge_cache(space) + if cache.w_array_repr is None: + return space.wrap(self.dump_data()) + return space.call_function(cache.w_array_repr, self) + + def dump_data(self): + concr = self.get_concrete() + i = concr.create_iter() + first = True + s = StringBuilder() + s.append('array([') + while not i.done(): + if first: + first = False + else: + s.append(', ') + s.append(concr.dtype.itemtype.str_format(concr.getitem(i.offset))) + i = i.next(len(concr.shape)) + s.append('])') + return s.build() def descr_str(self, space): - ret = StringBuilder() - concrete = self.get_concrete_or_scalar() - concrete.to_str(space, 0, ret, ' ') - return space.wrap(ret.build()) + cache = get_appbridge_cache(space) + if cache.w_array_str is None: + return space.wrap(self.dump_data()) + return space.call_function(cache.w_array_str, self) @jit.unroll_safe def _single_item_result(self, space, w_idx): @@ -497,20 +381,86 @@ def _prepare_slice_args(self, space, w_idx): if (space.isinstance_w(w_idx, space.w_int) or space.isinstance_w(w_idx, space.w_slice)): - return [space.decode_index4(w_idx, self.shape[0])] - return [space.decode_index4(w_item, self.shape[i]) for i, w_item in + return [Chunk(*space.decode_index4(w_idx, self.shape[0]))] + return [Chunk(*space.decode_index4(w_item, self.shape[i])) for i, w_item in enumerate(space.fixedview(w_idx))] + def count_all_true(self, arr): + sig = arr.find_sig() + frame = sig.create_frame(arr) + shapelen = len(arr.shape) + s = 0 + iter = None + while not frame.done(): + count_driver.jit_merge_point(arr=arr, frame=frame, iter=iter, s=s, + shapelen=shapelen) + iter = frame.get_final_iter() + s += arr.dtype.getitem_bool(arr.storage, iter.offset) + frame.next(shapelen) + return s + + def getitem_filter(self, space, arr): + concr = arr.get_concrete() + if concr.size > self.size: + raise OperationError(space.w_IndexError, + space.wrap("index out of range for array")) + size = self.count_all_true(concr) + res = W_NDimArray(size, [size], self.find_dtype()) + ri = ArrayIterator(size) + shapelen = len(self.shape) + argi = concr.create_iter() + sig = self.find_sig() + frame = sig.create_frame(self) + v = None + while not ri.done(): + filter_driver.jit_merge_point(concr=concr, argi=argi, ri=ri, + frame=frame, v=v, res=res, sig=sig, + shapelen=shapelen, self=self) + if concr.dtype.getitem_bool(concr.storage, argi.offset): + v = sig.eval(frame, self) + res.setitem(ri.offset, v) + ri = ri.next(1) + else: + ri = ri.next_no_increase(1) + argi = argi.next(shapelen) + frame.next(shapelen) + return res + + def setitem_filter(self, space, idx, val): + size = self.count_all_true(idx) + arr = SliceArray([size], self.dtype, self, val) + sig = arr.find_sig() + shapelen = len(self.shape) + frame = sig.create_frame(arr) + idxi = idx.create_iter() + while not frame.done(): + filter_set_driver.jit_merge_point(idx=idx, idxi=idxi, sig=sig, + frame=frame, arr=arr, + shapelen=shapelen) + if idx.dtype.getitem_bool(idx.storage, idxi.offset): + sig.eval(frame, arr) + frame.next_from_second(1) + frame.next_first(shapelen) + idxi = idxi.next(shapelen) + def descr_getitem(self, space, w_idx): + if (isinstance(w_idx, BaseArray) and w_idx.shape == self.shape and + w_idx.find_dtype().is_bool_type()): + return self.getitem_filter(space, w_idx) if self._single_item_result(space, w_idx): concrete = self.get_concrete() item = concrete._index_of_single_item(space, w_idx) return concrete.getitem(item) chunks = self._prepare_slice_args(space, w_idx) - return space.wrap(self.create_slice(chunks)) + return self.create_slice(chunks) def descr_setitem(self, space, w_idx, w_value): self.invalidated() + if (isinstance(w_idx, BaseArray) and w_idx.shape == self.shape and + w_idx.find_dtype().is_bool_type()): + return self.get_concrete().setitem_filter(space, + w_idx.get_concrete(), + convert_to_array(space, w_value)) if self._single_item_result(space, w_idx): concrete = self.get_concrete() item = concrete._index_of_single_item(space, w_idx) @@ -527,9 +477,8 @@ def create_slice(self, chunks): shape = [] i = -1 - for i, (start_, stop, step, lgt) in enumerate(chunks): - if step != 0: - shape.append(lgt) + for i, chunk in enumerate(chunks): + chunk.extend_shape(shape) s = i + 1 assert s >= 0 shape += self.shape[s:] @@ -557,11 +506,14 @@ w_shape = args_w[0] else: w_shape = space.newtuple(args_w) + new_shape = get_shape_from_iterable(space, self.size, w_shape) + return self.reshape(space, new_shape) + + def reshape(self, space, new_shape): concrete = self.get_concrete() - new_shape = get_shape_from_iterable(space, concrete.size, w_shape) # Since we got to here, prod(new_shape) == self.size - new_strides = calc_new_strides(new_shape, - concrete.shape, concrete.strides) + new_strides = calc_new_strides(new_shape, concrete.shape, + concrete.strides, concrete.order) if new_strides: # We can create a view, strides somehow match up. ndims = len(new_shape) @@ -569,7 +521,7 @@ for nd in range(ndims): new_backstrides[nd] = (new_shape[nd] - 1) * new_strides[nd] arr = W_NDimSlice(concrete.start, new_strides, new_backstrides, - new_shape, self) + new_shape, concrete) else: # Create copy with contiguous data arr = concrete.copy(space) @@ -579,7 +531,7 @@ def descr_tolist(self, space): if len(self.shape) == 0: assert isinstance(self, Scalar) - return self.value.descr_tolist(space) + return self.value.item(space) w_result = space.newlist([]) for i in range(self.shape[0]): space.call_method(w_result, "append", @@ -596,17 +548,17 @@ w_denom = space.wrap(self.shape[dim]) return space.div(self.descr_sum_promote(space, w_axis), w_denom) - def descr_var(self, space): - # var = mean((values - mean(values)) ** 2) - w_res = self.descr_sub(space, self.descr_mean(space, space.w_None)) - assert isinstance(w_res, BaseArray) - w_res = w_res.descr_pow(space, space.wrap(2)) - assert isinstance(w_res, BaseArray) - return w_res.descr_mean(space, space.w_None) + def descr_var(self, space, w_axis=None): + return get_appbridge_cache(space).call_method(space, '_var', self, + w_axis) - def descr_std(self, space): - # std(v) = sqrt(var(v)) - return interp_ufuncs.get(space).sqrt.call(space, [self.descr_var(space)]) + def descr_std(self, space, w_axis=None): + return get_appbridge_cache(space).call_method(space, '_std', self, + w_axis) + + def descr_fill(self, space, w_value): + concr = self.get_concrete_or_scalar() + concr.fill(space, w_value) def descr_nonzero(self, space): if self.size > 1: @@ -635,6 +587,16 @@ return space.wrap(W_NDimSlice(concrete.start, strides, backstrides, shape, concrete)) + def descr_ravel(self, space, w_order=None): + if w_order is None or space.is_w(w_order, space.w_None): + order = 'C' + else: + order = space.str_w(w_order) + if order != 'C': + raise OperationError(space.w_NotImplementedError, space.wrap( + "order not implemented")) + return self.descr_reshape(space, [space.wrap(-1)]) + def descr_get_flatiter(self, space): return space.wrap(W_FlatIterator(self)) @@ -664,6 +626,60 @@ def supports_fast_slicing(self): return False + def descr_compress(self, space, w_obj, w_axis=None): + index = convert_to_array(space, w_obj) + return self.getitem_filter(space, index) + + def descr_take(self, space, w_obj, w_axis=None): + index = convert_to_array(space, w_obj).get_concrete() + concr = self.get_concrete() + if space.is_w(w_axis, space.w_None): + concr = concr.descr_ravel(space) + else: + raise OperationError(space.w_NotImplementedError, + space.wrap("axis unsupported for take")) + index_i = index.create_iter() + res_shape = index.shape + size = support.product(res_shape) + res = W_NDimArray(size, res_shape[:], concr.dtype, concr.order) + res_i = res.create_iter() + shapelen = len(index.shape) + sig = concr.find_sig() + while not index_i.done(): + take_driver.jit_merge_point(index_i=index_i, index=index, + res_i=res_i, concr=concr, + res=res, + shapelen=shapelen, sig=sig) + w_item = index._getitem_long(space, index_i.offset) + res.setitem(res_i.offset, concr.descr_getitem(space, w_item)) + index_i = index_i.next(shapelen) + res_i = res_i.next(shapelen) + return res + + def _getitem_long(self, space, offset): + # an obscure hack to not have longdtype inside a jitted loop + longdtype = interp_dtype.get_dtype_cache(space).w_longdtype + return self.getitem(offset).convert_to(longdtype).item( + space) + + def descr_item(self, space, w_arg=None): + if space.is_w(w_arg, space.w_None): + if not isinstance(self, Scalar): + raise OperationError(space.w_ValueError, space.wrap("index out of bounds")) + return self.value.item(space) + if space.isinstance_w(w_arg, space.w_int): + if isinstance(self, Scalar): + raise OperationError(space.w_ValueError, space.wrap("index out of bounds")) + concr = self.get_concrete() + i = to_coords(space, self.shape, concr.size, concr.order, w_arg)[0] + # XXX a bit around + item = self.descr_getitem(space, space.newtuple([space.wrap(x) + for x in i])) + assert isinstance(item, interp_boxes.W_GenericBox) + return item.item(space) + raise OperationError(space.w_NotImplementedError, space.wrap( + "non-int arg not supported")) + def convert_to_array(space, w_obj): if isinstance(w_obj, BaseArray): return w_obj @@ -689,23 +705,29 @@ self.shape = [] BaseArray.__init__(self, []) self.dtype = dtype + assert isinstance(value, interp_boxes.W_GenericBox) self.value = value def find_dtype(self): return self.dtype - def to_str(self, space, comma, builder, indent=' ', use_ellipsis=False): - builder.append(self.dtype.itemtype.str_format(self.value)) - def copy(self, space): return Scalar(self.dtype, self.value) + def fill(self, space, w_value): + self.value = self.dtype.coerce(space, w_value) + def create_sig(self): return signature.ScalarSignature(self.dtype) def get_concrete_or_scalar(self): return self + def reshape(self, space, new_shape): + size = support.product(new_shape) + res = W_NDimArray(size, new_shape, self.dtype, 'C') + res.setitem(0, self.value) + return res class VirtualArray(BaseArray): """ @@ -735,8 +757,7 @@ frame=frame, ri=ri, self=self, result=result) - result.dtype.setitem(result.storage, ri.offset, - sig.eval(frame, self)) + result.setitem(ri.offset, sig.eval(frame, self)) frame.next(shapelen) ri = ri.next(shapelen) return result @@ -763,9 +784,6 @@ class VirtualSlice(VirtualArray): def __init__(self, child, chunks, shape): - size = 1 - for sh in shape: - size *= sh self.child = child self.chunks = chunks self.size = size @@ -787,11 +805,12 @@ class Call1(VirtualArray): - def __init__(self, ufunc, name, shape, res_dtype, values): + def __init__(self, ufunc, name, shape, calc_dtype, res_dtype, values): VirtualArray.__init__(self, name, shape, res_dtype) self.values = values self.size = values.size self.ufunc = ufunc + self.calc_dtype = calc_dtype def _del_sources(self): self.values = None @@ -806,16 +825,14 @@ Intermediate class for performing binary operations. """ _immutable_fields_ = ['left', 'right'] - + def __init__(self, ufunc, name, shape, calc_dtype, res_dtype, left, right): VirtualArray.__init__(self, name, shape, res_dtype) self.ufunc = ufunc self.left = left self.right = right self.calc_dtype = calc_dtype - self.size = 1 - for s in self.shape: - self.size *= s + self.size = support.product(self.shape) def _del_sources(self): self.left = None @@ -843,14 +860,15 @@ self.left.create_sig(), self.right.create_sig()) class SliceArray(Call2): - def __init__(self, shape, dtype, left, right): + def __init__(self, shape, dtype, left, right, no_broadcast=False): + self.no_broadcast = no_broadcast Call2.__init__(self, None, 'sliceloop', shape, dtype, dtype, left, right) - + def create_sig(self): lsig = self.left.create_sig() rsig = self.right.create_sig() - if self.shape != self.right.shape: + if not self.no_broadcast and self.shape != self.right.shape: return signature.SliceloopBroadcastSignature(self.ufunc, self.name, self.calc_dtype, @@ -865,7 +883,7 @@ when we'll make AxisReduce lazy """ _immutable_fields_ = ['left', 'right'] - + def __init__(self, ufunc, name, shape, dtype, left, right, dim): Call2.__init__(self, ufunc, name, shape, dtype, dtype, left, right) @@ -924,89 +942,6 @@ self.strides = strides self.backstrides = backstrides - def to_str(self, space, comma, builder, indent=' ', use_ellipsis=False): - '''Modifies builder with a representation of the array/slice - The items will be seperated by a comma if comma is 1 - Multidimensional arrays/slices will span a number of lines, - each line will begin with indent. - ''' - size = self.size - ccomma = ',' * comma - ncomma = ',' * (1 - comma) - dtype = self.find_dtype() - if size < 1: - builder.append('[]') - return - elif size == 1: - builder.append(dtype.itemtype.str_format(self.getitem(0))) - return - if size > 1000: - # Once this goes True it does not go back to False for recursive - # calls - use_ellipsis = True - ndims = len(self.shape) - i = 0 - builder.append('[') - if ndims > 1: - if use_ellipsis: - for i in range(min(3, self.shape[0])): - if i > 0: - builder.append(ccomma + '\n') - if ndims >= 3: - builder.append('\n' + indent) - else: - builder.append(indent) - view = self.create_slice([(i, 0, 0, 1)]).get_concrete() - view.to_str(space, comma, builder, indent=indent + ' ', - use_ellipsis=use_ellipsis) - if i < self.shape[0] - 1: - builder.append(ccomma + '\n' + indent + '...' + ncomma) - i = self.shape[0] - 3 - else: - i += 1 - while i < self.shape[0]: - if i > 0: - builder.append(ccomma + '\n') - if ndims >= 3: - builder.append('\n' + indent) - else: - builder.append(indent) - # create_slice requires len(chunks) > 1 in order to reduce - # shape - view = self.create_slice([(i, 0, 0, 1)]).get_concrete() - view.to_str(space, comma, builder, indent=indent + ' ', - use_ellipsis=use_ellipsis) - i += 1 - elif ndims == 1: - spacer = ccomma + ' ' - item = self.start - # An iterator would be a nicer way to walk along the 1d array, but - # how do I reset it if printing ellipsis? iterators have no - # "set_offset()" - i = 0 - if use_ellipsis: - for i in range(min(3, self.shape[0])): - if i > 0: - builder.append(spacer) - builder.append(dtype.itemtype.str_format(self.getitem(item))) - item += self.strides[0] - if i < self.shape[0] - 1: - # Add a comma only if comma is False - this prevents adding - # two commas - builder.append(spacer + '...' + ncomma) - # Ugly, but can this be done with an iterator? - item = self.start + self.backstrides[0] - 2 * self.strides[0] - i = self.shape[0] - 3 - else: - i += 1 - while i < self.shape[0]: - if i > 0: - builder.append(spacer) - builder.append(dtype.itemtype.str_format(self.getitem(item))) - item += self.strides[0] - i += 1 - builder.append(']') - @jit.unroll_safe def _index_of_single_item(self, space, w_idx): if space.isinstance_w(w_idx, space.w_int): @@ -1079,6 +1014,9 @@ array.setslice(space, self) return array + def fill(self, space, w_value): + self.setslice(space, scalar_w(space, self.dtype, w_value)) + class ViewArray(ConcreteArray): def create_sig(self): @@ -1090,15 +1028,16 @@ assert isinstance(parent, ConcreteArray) if isinstance(parent, W_NDimSlice): parent = parent.parent - size = 1 - for sh in shape: - size *= sh self.strides = strides self.backstrides = backstrides - ViewArray.__init__(self, size, shape, parent.dtype, parent.order, - parent) + ViewArray.__init__(self, support.product(shape), shape, parent.dtype, + parent.order, parent) self.start = start + def create_iter(self): + return ViewIterator(self.start, self.strides, self.backstrides, + self.shape) + def setshape(self, space, new_shape): if len(self.shape) < 1: return @@ -1122,7 +1061,8 @@ self.backstrides = backstrides self.shape = new_shape return - new_strides = calc_new_strides(new_shape, self.shape, self.strides) + new_strides = calc_new_strides(new_shape, self.shape, self.strides, + self.order) if new_strides is None: raise OperationError(space.w_AttributeError, space.wrap( "incompatible shape for a non-contiguous array")) @@ -1145,6 +1085,9 @@ self.shape = new_shape self.calc_strides(new_shape) + def create_iter(self): + return ArrayIterator(self.size) + def create_sig(self): return signature.ArraySignature(self.dtype) @@ -1164,27 +1107,42 @@ shape.append(item) return size, shape -def array(space, w_item_or_iterable, w_dtype=None, w_order=NoneNotWrapped): + at unwrap_spec(subok=bool, copy=bool, ownmaskna=bool) +def array(space, w_item_or_iterable, w_dtype=None, w_order=None, + subok=True, copy=True, w_maskna=None, ownmaskna=False): # find scalar + if w_maskna is None: + w_maskna = space.w_None + if (not subok or not space.is_w(w_maskna, space.w_None) or + ownmaskna): + raise OperationError(space.w_NotImplementedError, space.wrap("Unsupported args")) if not space.issequence_w(w_item_or_iterable): - if space.is_w(w_dtype, space.w_None): + if w_dtype is None or space.is_w(w_dtype, space.w_None): w_dtype = interp_ufuncs.find_dtype_for_scalar(space, w_item_or_iterable) dtype = space.interp_w(interp_dtype.W_Dtype, space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype) ) return scalar_w(space, dtype, w_item_or_iterable) - if w_order is None: + if space.is_w(w_order, space.w_None) or w_order is None: order = 'C' else: order = space.str_w(w_order) if order != 'C': # or order != 'F': raise operationerrfmt(space.w_ValueError, "Unknown order: %s", order) - shape, elems_w = _find_shape_and_elems(space, w_item_or_iterable) + if isinstance(w_item_or_iterable, BaseArray): + if (not space.is_w(w_dtype, space.w_None) and + w_item_or_iterable.find_dtype() is not w_dtype): + raise OperationError(space.w_NotImplementedError, space.wrap( + "copying over different dtypes unsupported")) + if copy: + return w_item_or_iterable.copy(space) + return w_item_or_iterable + shape, elems_w = find_shape_and_elems(space, w_item_or_iterable) # they come back in C order size = len(elems_w) - if space.is_w(w_dtype, space.w_None): + if w_dtype is None or space.is_w(w_dtype, space.w_None): w_dtype = None for w_elem in elems_w: w_dtype = interp_ufuncs.find_dtype_for_scalar(space, w_elem, @@ -1199,6 +1157,7 @@ arr = W_NDimArray(size, shape[:], dtype=dtype, order=order) shapelen = len(shape) arr_iter = ArrayIterator(arr.size) + # XXX we might want to have a jitdriver here for i in range(len(elems_w)): w_elem = elems_w[i] dtype.setitem(arr.storage, arr_iter.offset, @@ -1211,6 +1170,8 @@ space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype) ) size, shape = _find_size_and_shape(space, w_size) + if not shape: + return scalar_w(space, dtype, space.wrap(0)) return space.wrap(W_NDimArray(size, shape[:], dtype=dtype)) def ones(space, w_size, w_dtype=None): @@ -1219,11 +1180,27 @@ ) size, shape = _find_size_and_shape(space, w_size) + if not shape: + return scalar_w(space, dtype, space.wrap(1)) arr = W_NDimArray(size, shape[:], dtype=dtype) one = dtype.box(1) arr.dtype.fill(arr.storage, one, 0, size) return space.wrap(arr) + at unwrap_spec(arr=BaseArray, skipna=bool, keepdims=bool) +def count_reduce_items(space, arr, w_axis=None, skipna=False, keepdims=True): + if not keepdims: + raise OperationError(space.w_NotImplementedError, space.wrap("unsupported")) + if space.is_w(w_axis, space.w_None): + return space.wrap(support.product(arr.shape)) + if space.isinstance_w(w_axis, space.w_int): + return space.wrap(arr.shape[space.int_w(w_axis)]) + s = 1 + elems = space.fixedview(w_axis) + for w_elem in elems: + s *= arr.shape[space.int_w(w_elem)] + return space.wrap(s) + def dot(space, w_obj, w_obj2): '''see numpypy.dot. Does not exist as an ndarray method in numpy. ''' @@ -1232,6 +1209,39 @@ return convert_to_array(space, w_obj2).descr_dot(space, w_arr) return w_arr.descr_dot(space, w_obj2) + at unwrap_spec(axis=int) +def concatenate(space, w_args, axis=0): + args_w = space.listview(w_args) + if len(args_w) == 0: + raise OperationError(space.w_ValueError, space.wrap("concatenation of zero-length sequences is impossible")) + args_w = [convert_to_array(space, w_arg) for w_arg in args_w] + dtype = args_w[0].find_dtype() + shape = args_w[0].shape[:] + if len(shape) <= axis: + raise OperationError(space.w_ValueError, + space.wrap("bad axis argument")) + for arr in args_w[1:]: + dtype = interp_ufuncs.find_binop_result_dtype(space, dtype, + arr.find_dtype()) + if len(arr.shape) <= axis: + raise OperationError(space.w_ValueError, + space.wrap("bad axis argument")) + for i, axis_size in enumerate(arr.shape): + if len(arr.shape) != len(shape) or (i != axis and axis_size != shape[i]): + raise OperationError(space.w_ValueError, space.wrap( + "array dimensions must agree except for axis being concatenated")) + elif i == axis: + shape[i] += axis_size + res = W_NDimArray(support.product(shape), shape, dtype, 'C') + chunks = [Chunk(0, i, 1, i) for i in shape] + axis_start = 0 + for arr in args_w: + chunks[axis] = Chunk(axis_start, axis_start + arr.shape[axis], 1, + arr.shape[axis]) + res.create_slice(chunks).setslice(space, arr) + axis_start += arr.shape[axis] + return res + BaseArray.typedef = TypeDef( 'ndarray', __module__ = "numpypy", @@ -1267,6 +1277,10 @@ __gt__ = interp2app(BaseArray.descr_gt), __ge__ = interp2app(BaseArray.descr_ge), + __and__ = interp2app(BaseArray.descr_and), + __or__ = interp2app(BaseArray.descr_or), + __invert__ = interp2app(BaseArray.descr_invert), + __repr__ = interp2app(BaseArray.descr_repr), __str__ = interp2app(BaseArray.descr_str), __array_interface__ = GetSetProperty(BaseArray.descr_array_iface), @@ -1276,9 +1290,11 @@ BaseArray.descr_set_shape), size = GetSetProperty(BaseArray.descr_get_size), ndim = GetSetProperty(BaseArray.descr_get_ndim), + item = interp2app(BaseArray.descr_item), T = GetSetProperty(BaseArray.descr_get_transpose), flat = GetSetProperty(BaseArray.descr_get_flatiter), + ravel = interp2app(BaseArray.descr_ravel), mean = interp2app(BaseArray.descr_mean), sum = interp2app(BaseArray.descr_sum), @@ -1293,9 +1309,14 @@ var = interp2app(BaseArray.descr_var), std = interp2app(BaseArray.descr_std), + fill = interp2app(BaseArray.descr_fill), + copy = interp2app(BaseArray.descr_copy), + flatten = interp2app(BaseArray.descr_flatten), reshape = interp2app(BaseArray.descr_reshape), tolist = interp2app(BaseArray.descr_tolist), + take = interp2app(BaseArray.descr_take), + compress = interp2app(BaseArray.descr_compress), ) @@ -1304,30 +1325,129 @@ @jit.unroll_safe def __init__(self, arr): arr = arr.get_concrete() - size = 1 - for sh in arr.shape: - size *= sh self.strides = [arr.strides[-1]] self.backstrides = [arr.backstrides[-1]] - ViewArray.__init__(self, size, [size], arr.dtype, arr.order, - arr) self.shapelen = len(arr.shape) - self.iter = OneDimIterator(arr.start, self.strides[0], - self.shape[0]) + sig = arr.find_sig() + self.iter = sig.create_frame(arr).get_final_iter() + self.base = arr + self.index = 0 + ViewArray.__init__(self, arr.size, [arr.size], arr.dtype, arr.order, + arr) def descr_next(self, space): if self.iter.done(): raise OperationError(space.w_StopIteration, space.w_None) - result = self.getitem(self.iter.offset) + result = self.base.getitem(self.iter.offset) self.iter = self.iter.next(self.shapelen) + self.index += 1 return result def descr_iter(self): return self + def descr_index(self, space): + return space.wrap(self.index) + + def descr_coords(self, space): + coords, step, lngth = to_coords(space, self.base.shape, + self.base.size, self.base.order, + space.wrap(self.index)) + return space.newtuple([space.wrap(c) for c in coords]) + + @jit.unroll_safe + def descr_getitem(self, space, w_idx): + if not (space.isinstance_w(w_idx, space.w_int) or + space.isinstance_w(w_idx, space.w_slice)): + raise OperationError(space.w_IndexError, + space.wrap('unsupported iterator index')) + base = self.base + start, stop, step, lngth = space.decode_index4(w_idx, base.size) + # setslice would have been better, but flat[u:v] for arbitrary + # shapes of array a cannot be represented as a[x1:x2, y1:y2] + basei = ViewIterator(base.start, base.strides, + base.backstrides,base.shape) + shapelen = len(base.shape) + basei = basei.next_skip_x(shapelen, start) + if lngth <2: + return base.getitem(basei.offset) + ri = ArrayIterator(lngth) + res = W_NDimArray(lngth, [lngth], base.dtype, + base.order) + while not ri.done(): + flat_get_driver.jit_merge_point(shapelen=shapelen, + base=base, + basei=basei, + step=step, + res=res, + ri=ri, + ) + w_val = base.getitem(basei.offset) + res.setitem(ri.offset,w_val) + basei = basei.next_skip_x(shapelen, step) + ri = ri.next(shapelen) + return res + + def descr_setitem(self, space, w_idx, w_value): + if not (space.isinstance_w(w_idx, space.w_int) or + space.isinstance_w(w_idx, space.w_slice)): + raise OperationError(space.w_IndexError, + space.wrap('unsupported iterator index')) + base = self.base + start, stop, step, lngth = space.decode_index4(w_idx, base.size) + arr = convert_to_array(space, w_value) + ai = 0 + basei = ViewIterator(base.start, base.strides, + base.backstrides,base.shape) + shapelen = len(base.shape) + basei = basei.next_skip_x(shapelen, start) + while lngth > 0: + flat_set_driver.jit_merge_point(shapelen=shapelen, + basei=basei, + base=base, + step=step, + arr=arr, + ai=ai, + lngth=lngth, + ) + v = arr.getitem(ai).convert_to(base.dtype) + base.setitem(basei.offset, v) + # need to repeat input values until all assignments are done + ai = (ai + 1) % arr.size + basei = basei.next_skip_x(shapelen, step) + lngth -= 1 + + def create_sig(self): + return signature.FlatSignature(self.base.dtype) + + def descr_base(self, space): + return space.wrap(self.base) + W_FlatIterator.typedef = TypeDef( 'flatiter', + #__array__ = #MISSING + __iter__ = interp2app(W_FlatIterator.descr_iter), + __getitem__ = interp2app(W_FlatIterator.descr_getitem), + __setitem__ = interp2app(W_FlatIterator.descr_setitem), + __eq__ = interp2app(BaseArray.descr_eq), + __ne__ = interp2app(BaseArray.descr_ne), + __lt__ = interp2app(BaseArray.descr_lt), + __le__ = interp2app(BaseArray.descr_le), + __gt__ = interp2app(BaseArray.descr_gt), + __ge__ = interp2app(BaseArray.descr_ge), + #__sizeof__ #MISSING + base = GetSetProperty(W_FlatIterator.descr_base), + index = GetSetProperty(W_FlatIterator.descr_index), + coords = GetSetProperty(W_FlatIterator.descr_coords), next = interp2app(W_FlatIterator.descr_next), - __iter__ = interp2app(W_FlatIterator.descr_iter), + ) W_FlatIterator.acceptable_as_base_class = False + +def isna(space, w_obj): + if isinstance(w_obj, BaseArray): + arr = w_obj.empty_copy(space, + interp_dtype.get_dtype_cache(space).w_booldtype) + arr.fill(space, space.wrap(False)) + return arr + return space.wrap(False) diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py --- a/pypy/module/micronumpy/interp_ufuncs.py +++ b/pypy/module/micronumpy/interp_ufuncs.py @@ -1,14 +1,15 @@ from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.error import OperationError, operationerrfmt -from pypy.interpreter.gateway import interp2app +from pypy.interpreter.gateway import interp2app, unwrap_spec, NoneNotWrapped from pypy.interpreter.typedef import TypeDef, GetSetProperty, interp_attrproperty -from pypy.module.micronumpy import interp_boxes, interp_dtype -from pypy.module.micronumpy.signature import ReduceSignature,\ - find_sig, new_printable_location, AxisReduceSignature, ScalarSignature +from pypy.module.micronumpy import interp_boxes, interp_dtype, support +from pypy.module.micronumpy.signature import (ReduceSignature, find_sig, + new_printable_location, AxisReduceSignature, ScalarSignature) from pypy.rlib import jit from pypy.rlib.rarithmetic import LONG_BIT from pypy.tool.sourcetools import func_with_new_name + reduce_driver = jit.JitDriver( greens=['shapelen', "sig"], virtualizables=["frame"], @@ -30,12 +31,14 @@ _attrs_ = ["name", "promote_to_float", "promote_bools", "identity"] _immutable_fields_ = ["promote_to_float", "promote_bools", "name"] - def __init__(self, name, promote_to_float, promote_bools, identity): + def __init__(self, name, promote_to_float, promote_bools, identity, + int_only): self.name = name self.promote_to_float = promote_to_float self.promote_bools = promote_bools self.identity = identity + self.int_only = int_only def descr_repr(self, space): return space.wrap("<ufunc '%s'>" % self.name) @@ -46,19 +49,31 @@ return self.identity def descr_call(self, space, __args__): - if __args__.keywords or len(__args__.arguments_w) < self.argcount: + args_w, kwds_w = __args__.unpack() + # it occurs to me that we don't support any datatypes that + # require casting, change it later when we do + kwds_w.pop('casting', None) + w_subok = kwds_w.pop('subok', None) + w_out = kwds_w.pop('out', space.w_None) + if ((w_subok is not None and space.is_true(w_subok)) or + not space.is_w(w_out, space.w_None)): + raise OperationError(space.w_NotImplementedError, + space.wrap("parameters unsupported")) + if kwds_w or len(args_w) < self.argcount: raise OperationError(space.w_ValueError, space.wrap("invalid number of arguments") ) - elif len(__args__.arguments_w) > self.argcount: + elif len(args_w) > self.argcount: # The extra arguments should actually be the output array, but we # don't support that yet. raise OperationError(space.w_TypeError, space.wrap("invalid number of arguments") ) - return self.call(space, __args__.arguments_w) + return self.call(space, args_w) - def descr_reduce(self, space, w_obj, w_dim=0): + @unwrap_spec(skipna=bool, keepdims=bool) + def descr_reduce(self, space, w_obj, w_axis=NoneNotWrapped, w_dtype=None, + skipna=False, keepdims=False, w_out=None): """reduce(...) reduce(a, axis=0) @@ -111,15 +126,24 @@ array([[ 1, 5], [ 9, 13]]) """ - return self.reduce(space, w_obj, False, False, w_dim) + if not space.is_w(w_out, space.w_None): + raise OperationError(space.w_NotImplementedError, space.wrap( + "out not supported")) + if w_axis is None: + axis = 0 + elif space.is_w(w_axis, space.w_None): + axis = -1 + else: + axis = space.int_w(w_axis) + return self.reduce(space, w_obj, False, False, axis, keepdims) - def reduce(self, space, w_obj, multidim, promote_to_largest, w_dim): + def reduce(self, space, w_obj, multidim, promote_to_largest, dim, + keepdims=False): from pypy.module.micronumpy.interp_numarray import convert_to_array, \ Scalar if self.argcount != 2: raise OperationError(space.w_ValueError, space.wrap("reduce only " "supported for binary functions")) - dim = space.int_w(w_dim) assert isinstance(self, W_Ufunc2) obj = convert_to_array(space, w_obj) if dim >= len(obj.shape): @@ -140,7 +164,7 @@ raise operationerrfmt(space.w_ValueError, "zero-size array to " "%s.reduce without identity", self.name) if shapelen > 1 and dim >= 0: - res = self.do_axis_reduce(obj, dtype, dim) + res = self.do_axis_reduce(obj, dtype, dim, keepdims) return space.wrap(res) scalarsig = ScalarSignature(dtype) sig = find_sig(ReduceSignature(self.func, self.name, dtype, @@ -154,15 +178,15 @@ value = self.identity.convert_to(dtype) return self.reduce_loop(shapelen, sig, frame, value, obj, dtype) - def do_axis_reduce(self, obj, dtype, dim): + def do_axis_reduce(self, obj, dtype, dim, keepdims): from pypy.module.micronumpy.interp_numarray import AxisReduce,\ W_NDimArray - - shape = obj.shape[0:dim] + obj.shape[dim + 1:len(obj.shape)] - size = 1 - for s in shape: - size *= s - result = W_NDimArray(size, shape, dtype) + + if keepdims: + shape = obj.shape[:dim] + [1] + obj.shape[dim + 1:] + else: + shape = obj.shape[:dim] + obj.shape[dim + 1:] + result = W_NDimArray(support.product(shape), shape, dtype) rightsig = obj.create_sig() # note - this is just a wrapper so signature can fetch # both left and right, nothing more, especially @@ -224,10 +248,12 @@ _immutable_fields_ = ["func", "name"] def __init__(self, func, name, promote_to_float=False, promote_bools=False, - identity=None): + identity=None, bool_result=False, int_only=False): - W_Ufunc.__init__(self, name, promote_to_float, promote_bools, identity) + W_Ufunc.__init__(self, name, promote_to_float, promote_bools, identity, + int_only) self.func = func + self.bool_result = bool_result def call(self, space, args_w): from pypy.module.micronumpy.interp_numarray import (Call1, @@ -235,27 +261,32 @@ [w_obj] = args_w w_obj = convert_to_array(space, w_obj) - res_dtype = find_unaryop_result_dtype(space, - w_obj.find_dtype(), - promote_to_float=self.promote_to_float, - promote_bools=self.promote_bools, - ) + calc_dtype = find_unaryop_result_dtype(space, + w_obj.find_dtype(), + promote_to_float=self.promote_to_float, + promote_bools=self.promote_bools) + if self.bool_result: + res_dtype = interp_dtype.get_dtype_cache(space).w_booldtype + else: + res_dtype = calc_dtype if isinstance(w_obj, Scalar): - return self.func(res_dtype, w_obj.value.convert_to(res_dtype)) + return space.wrap(self.func(calc_dtype, w_obj.value.convert_to(calc_dtype))) - w_res = Call1(self.func, self.name, w_obj.shape, res_dtype, w_obj) + w_res = Call1(self.func, self.name, w_obj.shape, calc_dtype, res_dtype, + w_obj) w_obj.add_invalidates(w_res) return w_res class W_Ufunc2(W_Ufunc): - _immutable_fields_ = ["comparison_func", "func", "name"] + _immutable_fields_ = ["comparison_func", "func", "name", "int_only"] argcount = 2 def __init__(self, func, name, promote_to_float=False, promote_bools=False, - identity=None, comparison_func=False): + identity=None, comparison_func=False, int_only=False): - W_Ufunc.__init__(self, name, promote_to_float, promote_bools, identity) + W_Ufunc.__init__(self, name, promote_to_float, promote_bools, identity, + int_only) self.func = func self.comparison_func = comparison_func @@ -268,6 +299,7 @@ w_rhs = convert_to_array(space, w_rhs) calc_dtype = find_binop_result_dtype(space, w_lhs.find_dtype(), w_rhs.find_dtype(), + int_only=self.int_only, promote_to_float=self.promote_to_float, promote_bools=self.promote_bools, ) @@ -276,10 +308,10 @@ else: res_dtype = calc_dtype if isinstance(w_lhs, Scalar) and isinstance(w_rhs, Scalar): - return self.func(calc_dtype, + return space.wrap(self.func(calc_dtype, w_lhs.value.convert_to(calc_dtype), w_rhs.value.convert_to(calc_dtype) - ) + )) new_shape = shape_agreement(space, w_lhs.shape, w_rhs.shape) w_res = Call2(self.func, self.name, @@ -304,10 +336,12 @@ def find_binop_result_dtype(space, dt1, dt2, promote_to_float=False, - promote_bools=False): + promote_bools=False, int_only=False): # dt1.num should be <= dt2.num if dt1.num > dt2.num: dt1, dt2 = dt2, dt1 + if int_only and (not dt1.is_int_type() or not dt2.is_int_type()): + raise OperationError(space.w_TypeError, space.wrap("Unsupported types")) # Some operations promote op(bool, bool) to return int8, rather than bool if promote_bools and (dt1.kind == dt2.kind == interp_dtype.BOOLLTR): return interp_dtype.get_dtype_cache(space).w_int8dtype @@ -405,12 +439,16 @@ return interp_dtype.get_dtype_cache(space).w_float64dtype -def ufunc_dtype_caller(space, ufunc_name, op_name, argcount, comparison_func): +def ufunc_dtype_caller(space, ufunc_name, op_name, argcount, comparison_func, + bool_result): + dtype_cache = interp_dtype.get_dtype_cache(space) if argcount == 1: def impl(res_dtype, value): - return getattr(res_dtype.itemtype, op_name)(value) + res = getattr(res_dtype.itemtype, op_name)(value) + if bool_result: + return dtype_cache.w_booldtype.box(res) + return res elif argcount == 2: - dtype_cache = interp_dtype.get_dtype_cache(space) def impl(res_dtype, lvalue, rvalue): res = getattr(res_dtype.itemtype, op_name)(lvalue, rvalue) if comparison_func: @@ -425,7 +463,13 @@ ("add", "add", 2, {"identity": 0}), ("subtract", "sub", 2), ("multiply", "mul", 2, {"identity": 1}), + ("bitwise_and", "bitwise_and", 2, {"identity": 1, + 'int_only': True}), + ("bitwise_or", "bitwise_or", 2, {"identity": 0, + 'int_only': True}), + ("invert", "invert", 1, {"int_only": True}), ("divide", "div", 2, {"promote_bools": True}), + ("true_divide", "div", 2, {"promote_to_float": True}), ("mod", "mod", 2, {"promote_bools": True}), ("power", "pow", 2, {"promote_bools": True}), @@ -435,6 +479,8 @@ ("less_equal", "le", 2, {"comparison_func": True}), ("greater", "gt", 2, {"comparison_func": True}), ("greater_equal", "ge", 2, {"comparison_func": True}), + ("isnan", "isnan", 1, {"bool_result": True}), + ("isinf", "isinf", 1, {"bool_result": True}), ("maximum", "max", 2), ("minimum", "min", 2), @@ -449,6 +495,7 @@ ("fabs", "fabs", 1, {"promote_to_float": True}), ("floor", "floor", 1, {"promote_to_float": True}), + ("ceil", "ceil", 1, {"promote_to_float": True}), ("exp", "exp", 1, {"promote_to_float": True}), ('sqrt', 'sqrt', 1, {'promote_to_float': True}), @@ -475,7 +522,8 @@ extra_kwargs["identity"] = identity func = ufunc_dtype_caller(space, ufunc_name, op_name, argcount, - comparison_func=extra_kwargs.get("comparison_func", False) + comparison_func=extra_kwargs.get("comparison_func", False), + bool_result=extra_kwargs.get("bool_result", False), ) if argcount == 1: ufunc = W_Ufunc1(func, ufunc_name, **extra_kwargs) @@ -485,3 +533,4 @@ def get(space): return space.fromcache(UfuncState) + diff --git a/pypy/module/micronumpy/signature.py b/pypy/module/micronumpy/signature.py --- a/pypy/module/micronumpy/signature.py +++ b/pypy/module/micronumpy/signature.py @@ -82,6 +82,16 @@ for i in range(len(self.iterators)): self.iterators[i] = self.iterators[i].next(shapelen) + @unroll_safe + def next_from_second(self, shapelen): + """ Don't increase the first iterator + """ + for i in range(1, len(self.iterators)): + self.iterators[i] = self.iterators[i].next(shapelen) + + def next_first(self, shapelen): + self.iterators[0] = self.iterators[0].next(shapelen) + def get_final_iter(self): final_iter = promote(self.final_iter) if final_iter < 0: @@ -214,6 +224,18 @@ return ViewIterator(arr.start, arr.strides, arr.backstrides, arr.shape).apply_transformations(arr, transforms) +class FlatSignature(ViewSignature): + def debug_repr(self): + return 'Flat' + + def allocate_iter(self, arr, transforms): + from pypy.module.micronumpy.interp_numarray import W_FlatIterator + assert isinstance(arr, W_FlatIterator) + return ViewIterator(arr.base.start, arr.base.strides, + arr.base.backstrides, + arr.base.shape).apply_transformations(arr.base, + transforms) + class VirtualSliceSignature(Signature): def __init__(self, child): self.child = child @@ -283,8 +305,8 @@ def eval(self, frame, arr): from pypy.module.micronumpy.interp_numarray import Call1 assert isinstance(arr, Call1) - v = self.child.eval(frame, arr.values).convert_to(arr.res_dtype) - return self.unfunc(arr.res_dtype, v) + v = self.child.eval(frame, arr.values).convert_to(arr.calc_dtype) + return self.unfunc(arr.calc_dtype, v) class Call2(Signature): _immutable_fields_ = ['binfunc', 'name', 'calc_dtype', 'left', 'right'] diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py --- a/pypy/module/micronumpy/strides.py +++ b/pypy/module/micronumpy/strides.py @@ -1,5 +1,5 @@ from pypy.rlib import jit - +from pypy.interpreter.error import OperationError @jit.look_inside_iff(lambda shape, start, strides, backstrides, chunks: jit.isconstant(len(chunks)) @@ -10,12 +10,12 @@ rstart = start rshape = [] i = -1 - for i, (start_, stop, step, lgt) in enumerate(chunks): - if step != 0: - rstrides.append(strides[i] * step) - rbackstrides.append(strides[i] * (lgt - 1) * step) - rshape.append(lgt) - rstart += strides[i] * start_ + for i, chunk in enumerate(chunks): + if chunk.step != 0: + rstrides.append(strides[i] * chunk.step) + rbackstrides.append(strides[i] * (chunk.lgt - 1) * chunk.step) + rshape.append(chunk.lgt) + rstart += strides[i] * chunk.start # add a reminder s = i + 1 assert s >= 0 @@ -38,6 +38,200 @@ rbackstrides = [0] * (len(res_shape) - len(orig_shape)) + rbackstrides return rstrides, rbackstrides +def find_shape_and_elems(space, w_iterable): + shape = [space.len_w(w_iterable)] + batch = space.listview(w_iterable) + while True: + new_batch = [] + if not batch: + return shape, [] + if not space.issequence_w(batch[0]): + for elem in batch: + if space.issequence_w(elem): + raise OperationError(space.w_ValueError, space.wrap( + "setting an array element with a sequence")) + return shape, batch + size = space.len_w(batch[0]) + for w_elem in batch: + if not space.issequence_w(w_elem) or space.len_w(w_elem) != size: + raise OperationError(space.w_ValueError, space.wrap( + "setting an array element with a sequence")) + new_batch += space.listview(w_elem) + shape.append(size) + batch = new_batch + +def to_coords(space, shape, size, order, w_item_or_slice): + '''Returns a start coord, step, and length. + ''' + start = lngth = step = 0 + if not (space.isinstance_w(w_item_or_slice, space.w_int) or + space.isinstance_w(w_item_or_slice, space.w_slice)): + raise OperationError(space.w_IndexError, + space.wrap('unsupported iterator index')) + + start, stop, step, lngth = space.decode_index4(w_item_or_slice, size) + + coords = [0] * len(shape) + i = start + if order == 'C': + for s in range(len(shape) -1, -1, -1): + coords[s] = i % shape[s] + i //= shape[s] + else: + for s in range(len(shape)): + coords[s] = i % shape[s] + i //= shape[s] + return coords, step, lngth + +def shape_agreement(space, shape1, shape2): + ret = _shape_agreement(shape1, shape2) + if len(ret) < max(len(shape1), len(shape2)): + raise OperationError(space.w_ValueError, + space.wrap("operands could not be broadcast together with shapes (%s) (%s)" % ( + ",".join([str(x) for x in shape1]), + ",".join([str(x) for x in shape2]), + )) + ) + return ret + +def _shape_agreement(shape1, shape2): + """ Checks agreement about two shapes with respect to broadcasting. Returns + the resulting shape. + """ + lshift = 0 + rshift = 0 + if len(shape1) > len(shape2): + m = len(shape1) + n = len(shape2) + rshift = len(shape2) - len(shape1) + remainder = shape1 + else: + m = len(shape2) + n = len(shape1) + lshift = len(shape1) - len(shape2) + remainder = shape2 + endshape = [0] * m + indices1 = [True] * m + indices2 = [True] * m + for i in range(m - 1, m - n - 1, -1): + left = shape1[i + lshift] + right = shape2[i + rshift] + if left == right: + endshape[i] = left + elif left == 1: + endshape[i] = right + indices1[i + lshift] = False + elif right == 1: + endshape[i] = left + indices2[i + rshift] = False + else: + return [] + #raise OperationError(space.w_ValueError, space.wrap( + # "frames are not aligned")) + for i in range(m - n): + endshape[i] = remainder[i] + return endshape + +def get_shape_from_iterable(space, old_size, w_iterable): + new_size = 0 + new_shape = [] + if space.isinstance_w(w_iterable, space.w_int): + new_size = space.int_w(w_iterable) + if new_size < 0: + new_size = old_size + new_shape = [new_size] + else: + neg_dim = -1 + batch = space.listview(w_iterable) + new_size = 1 + if len(batch) < 1: + if old_size == 1: + # Scalars can have an empty size. + new_size = 1 + else: + new_size = 0 + new_shape = [] + i = 0 + for elem in batch: + s = space.int_w(elem) + if s < 0: + if neg_dim >= 0: + raise OperationError(space.w_ValueError, space.wrap( + "can only specify one unknown dimension")) + s = 1 + neg_dim = i + new_size *= s + new_shape.append(s) + i += 1 + if neg_dim >= 0: + new_shape[neg_dim] = old_size / new_size + new_size *= new_shape[neg_dim] + if new_size != old_size: + raise OperationError(space.w_ValueError, + space.wrap("total size of new array must be unchanged")) + return new_shape + +# Recalculating strides. Find the steps that the iteration does for each +# dimension, given the stride and shape. Then try to create a new stride that +# fits the new shape, using those steps. If there is a shape/step mismatch +# (meaning that the realignment of elements crosses from one step into another) +# return None so that the caller can raise an exception. +def calc_new_strides(new_shape, old_shape, old_strides, order): + # Return the proper strides for new_shape, or None if the mapping crosses + # stepping boundaries + + # Assumes that prod(old_shape) == prod(new_shape), len(old_shape) > 1, and + # len(new_shape) > 0 + steps = [] + last_step = 1 + oldI = 0 + new_strides = [] + if order == 'F': + for i in range(len(old_shape)): + steps.append(old_strides[i] / last_step) + last_step *= old_shape[i] + cur_step = steps[0] + n_new_elems_used = 1 + n_old_elems_to_use = old_shape[0] + for s in new_shape: + new_strides.append(cur_step * n_new_elems_used) + n_new_elems_used *= s + while n_new_elems_used > n_old_elems_to_use: + oldI += 1 + if steps[oldI] != steps[oldI - 1]: + return None + n_old_elems_to_use *= old_shape[oldI] + if n_new_elems_used == n_old_elems_to_use: + oldI += 1 + if oldI < len(old_shape): + cur_step = steps[oldI] + n_old_elems_to_use *= old_shape[oldI] + elif order == 'C': + for i in range(len(old_shape) - 1, -1, -1): + steps.insert(0, old_strides[i] / last_step) + last_step *= old_shape[i] + cur_step = steps[-1] + n_new_elems_used = 1 + oldI = -1 + n_old_elems_to_use = old_shape[-1] + for i in range(len(new_shape) - 1, -1, -1): + s = new_shape[i] + new_strides.insert(0, cur_step * n_new_elems_used) + n_new_elems_used *= s + while n_new_elems_used > n_old_elems_to_use: + oldI -= 1 + if steps[oldI] != steps[oldI + 1]: + return None + n_old_elems_to_use *= old_shape[oldI] + if n_new_elems_used == n_old_elems_to_use: + oldI -= 1 + if oldI >= -len(old_shape): + cur_step = steps[oldI] + n_old_elems_to_use *= old_shape[oldI] + assert len(new_strides) == len(new_shape) + return new_strides + + def calculate_dot_strides(strides, backstrides, res_shape, skip_dims): rstrides = [] rbackstrides = [] diff --git a/pypy/module/micronumpy/support.py b/pypy/module/micronumpy/support.py new file mode 100644 --- /dev/null +++ b/pypy/module/micronumpy/support.py @@ -0,0 +1,5 @@ +def product(s): + i = 1 + for x in s: + i *= x + return i \ No newline at end of file diff --git a/pypy/module/micronumpy/test/test_base.py b/pypy/module/micronumpy/test/test_base.py --- a/pypy/module/micronumpy/test/test_base.py +++ b/pypy/module/micronumpy/test/test_base.py @@ -3,9 +3,17 @@ from pypy.module.micronumpy.interp_numarray import W_NDimArray, Scalar from pypy.module.micronumpy.interp_ufuncs import (find_binop_result_dtype, find_unaryop_result_dtype) +from pypy.module.micronumpy.interp_boxes import W_Float64Box +from pypy.conftest import option +import sys class BaseNumpyAppTest(object): def setup_class(cls): + if option.runappdirect: + if '__pypy__' not in sys.builtin_module_names: + import numpy + sys.modules['numpypy'] = numpy + sys.modules['_numpypy'] = numpy cls.space = gettestobjspace(usemodules=['micronumpy']) class TestSignature(object): @@ -16,7 +24,7 @@ ar = W_NDimArray(10, [10], dtype=float64_dtype) ar2 = W_NDimArray(10, [10], dtype=float64_dtype) v1 = ar.descr_add(space, ar) - v2 = ar.descr_add(space, Scalar(float64_dtype, 2.0)) + v2 = ar.descr_add(space, Scalar(float64_dtype, W_Float64Box(2.0))) sig1 = v1.find_sig() sig2 = v2.find_sig() assert v1 is not v2 @@ -26,7 +34,7 @@ sig1b = ar2.descr_add(space, ar).find_sig() assert sig1b.left.array_no != sig1b.right.array_no assert sig1b is not sig1 - v3 = ar.descr_add(space, Scalar(float64_dtype, 1.0)) + v3 = ar.descr_add(space, Scalar(float64_dtype, W_Float64Box(1.0))) sig3 = v3.find_sig() assert sig2 is sig3 v4 = ar.descr_add(space, ar) diff --git a/pypy/module/micronumpy/test/test_compile.py b/pypy/module/micronumpy/test/test_compile.py --- a/pypy/module/micronumpy/test/test_compile.py +++ b/pypy/module/micronumpy/test/test_compile.py @@ -244,7 +244,7 @@ a[0:30:3] = c a -> 3 """) - assert interp.results[0].value == 11 + assert interp.results[0].value == 11 def test_dot(self): interp = self.run(""" @@ -254,3 +254,19 @@ c -> 0 -> 0 """) assert interp.results[0].value == 19 + + def test_flat_iter(self): + interp = self.run(''' + a = |30| + b = flat(a) + b -> 3 + ''') + assert interp.results[0].value == 3 + + def test_take(self): + interp = self.run(""" + a = |10| + b = take(a, [1, 1, 3, 2]) + b -> 2 + """) + assert interp.results[0].value == 3 diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py --- a/pypy/module/micronumpy/test/test_dtypes.py +++ b/pypy/module/micronumpy/test/test_dtypes.py @@ -11,8 +11,17 @@ assert dtype('int8').num == 1 assert dtype(d) is d assert dtype(None) is dtype(float) + assert dtype('int8').name == 'int8' raises(TypeError, dtype, 1042) + def test_dtype_eq(self): + from _numpypy import dtype + + assert dtype("int8") == "int8" + assert "int8" == dtype("int8") + raises(TypeError, lambda: dtype("int8") == 3) + assert dtype(bool) == bool + def test_dtype_with_types(self): from _numpypy import dtype @@ -30,7 +39,7 @@ def test_repr_str(self): from _numpypy import dtype - assert repr(dtype) == "<type 'numpypy.dtype'>" + assert '.dtype' in repr(dtype) d = dtype('?') assert repr(d) == "dtype('bool')" assert str(d) == "bool" @@ -166,14 +175,11 @@ # You can't subclass dtype raises(TypeError, type, "Foo", (dtype,), {}) - def test_new(self): - import _numpypy as np - assert np.int_(4) == 4 - assert np.float_(3.4) == 3.4 + def test_aliases(self): + from _numpypy import dtype - def test_pow(self): - from _numpypy import int_ - assert int_(4) ** 2 == 16 + assert dtype("float") is dtype(float) + class AppTestTypes(BaseNumpyAppTest): def test_abstract_types(self): @@ -189,6 +195,15 @@ raises(TypeError, numpy.floating, 0) raises(TypeError, numpy.inexact, 0) + def test_new(self): + import _numpypy as np + assert np.int_(4) == 4 + assert np.float_(3.4) == 3.4 + + def test_pow(self): + from _numpypy import int_ + assert int_(4) ** 2 == 16 + def test_bool(self): import _numpypy as numpy @@ -318,7 +333,7 @@ else: raises(OverflowError, numpy.int64, 9223372036854775807) raises(OverflowError, numpy.int64, '9223372036854775807') - + raises(OverflowError, numpy.int64, 9223372036854775808) raises(OverflowError, numpy.int64, '9223372036854775808') @@ -370,3 +385,19 @@ b = X(10) assert type(b) is X assert b.m() == 12 + + def test_long_as_index(self): + skip("waiting for removal of multimethods of __index__") + from _numpypy import int_ + assert (1, 2, 3)[int_(1)] == 2 + + def test_int(self): + import sys + from _numpypy import int32, int64, int_ + assert issubclass(int_, int) + if sys.maxint == (1<<31) - 1: + assert issubclass(int32, int) + assert int_ is int32 + else: + assert issubclass(int64, int) + assert int_ is int64 diff --git a/pypy/module/micronumpy/test/test_iter.py b/pypy/module/micronumpy/test/test_iter.py new file mode 100644 --- /dev/null +++ b/pypy/module/micronumpy/test/test_iter.py @@ -0,0 +1,88 @@ +from pypy.module.micronumpy.interp_iter import ViewIterator + +class TestIterDirect(object): + def test_C_viewiterator(self): + #Let's get started, simple iteration in C order with + #contiguous layout => strides[-1] is 1 + start = 0 + shape = [3, 5] + strides = [5, 1] + backstrides = [x * (y - 1) for x,y in zip(strides, shape)] + assert backstrides == [10, 4] + i = ViewIterator(start, strides, backstrides, shape) + i = i.next(2) + i = i.next(2) + i = i.next(2) + assert i.offset == 3 + assert not i.done() + assert i.indices == [0,3] + #cause a dimension overflow + i = i.next(2) + i = i.next(2) + assert i.offset == 5 + assert i.indices == [1,0] + + #Now what happens if the array is transposed? strides[-1] != 1 + # therefore layout is non-contiguous + strides = [1, 3] + backstrides = [x * (y - 1) for x,y in zip(strides, shape)] + assert backstrides == [2, 12] + i = ViewIterator(start, strides, backstrides, shape) + i = i.next(2) + i = i.next(2) + i = i.next(2) + assert i.offset == 9 + assert not i.done() + assert i.indices == [0,3] + #cause a dimension overflow + i = i.next(2) + i = i.next(2) + assert i.offset == 1 + assert i.indices == [1,0] + + def test_C_viewiterator_step(self): + #iteration in C order with #contiguous layout => strides[-1] is 1 + #skip less than the shape + start = 0 + shape = [3, 5] + strides = [5, 1] + backstrides = [x * (y - 1) for x,y in zip(strides, shape)] + assert backstrides == [10, 4] + i = ViewIterator(start, strides, backstrides, shape) + i = i.next_skip_x(2,2) + i = i.next_skip_x(2,2) + i = i.next_skip_x(2,2) + assert i.offset == 6 + assert not i.done() + assert i.indices == [1,1] + #And for some big skips + i = i.next_skip_x(2,5) + assert i.offset == 11 + assert i.indices == [2,1] + i = i.next_skip_x(2,5) + # Note: the offset does not overflow but recycles, + # this is good for broadcast + assert i.offset == 1 + assert i.indices == [0,1] + assert i.done() + + #Now what happens if the array is transposed? strides[-1] != 1 + # therefore layout is non-contiguous + strides = [1, 3] + backstrides = [x * (y - 1) for x,y in zip(strides, shape)] + assert backstrides == [2, 12] + i = ViewIterator(start, strides, backstrides, shape) + i = i.next_skip_x(2,2) + i = i.next_skip_x(2,2) + i = i.next_skip_x(2,2) + assert i.offset == 4 + assert i.indices == [1,1] + assert not i.done() + i = i.next_skip_x(2,5) + assert i.offset == 5 + assert i.indices == [2,1] + assert not i.done() + i = i.next_skip_x(2,5) + assert i.indices == [0,1] + assert i.offset == 3 + assert i.done() diff --git a/pypy/module/micronumpy/test/test_module.py b/pypy/module/micronumpy/test/test_module.py --- a/pypy/module/micronumpy/test/test_module.py +++ b/pypy/module/micronumpy/test/test_module.py @@ -2,16 +2,11 @@ class AppTestNumPyModule(BaseNumpyAppTest): - def test_mean(self): - from _numpypy import array, mean - assert mean(array(range(5))) == 2.0 - assert mean(range(5)) == 2.0 - def test_average(self): from _numpypy import array, average assert average(range(10)) == 4.5 assert average(array(range(10))) == 4.5 - + def test_sum(self): from _numpypy import array, sum assert sum(range(10)) == 45 @@ -21,7 +16,7 @@ from _numpypy import array, min assert min(range(10)) == 0 assert min(array(range(10))) == 0 - + def test_max(self): from _numpypy import array, max assert max(range(10)) == 9 diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -2,6 +2,7 @@ import py from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest from pypy.module.micronumpy.interp_numarray import W_NDimArray, shape_agreement +from pypy.module.micronumpy.interp_iter import Chunk from pypy.module.micronumpy import signature from pypy.interpreter.error import OperationError from pypy.conftest import gettestobjspace @@ -37,53 +38,54 @@ def test_create_slice_f(self): a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'F') - s = a.create_slice([(3, 0, 0, 1)]) + s = a.create_slice([Chunk(3, 0, 0, 1)]) assert s.start == 3 assert s.strides == [10, 50] assert s.backstrides == [40, 100] - s = a.create_slice([(1, 9, 2, 4)]) + s = a.create_slice([Chunk(1, 9, 2, 4)]) assert s.start == 1 assert s.strides == [2, 10, 50] assert s.backstrides == [6, 40, 100] - s = a.create_slice([(1, 5, 3, 2), (1, 2, 1, 1), (1, 0, 0, 1)]) + s = a.create_slice([Chunk(1, 5, 3, 2), Chunk(1, 2, 1, 1), Chunk(1, 0, 0, 1)]) assert s.shape == [2, 1] assert s.strides == [3, 10] assert s.backstrides == [3, 0] - s = a.create_slice([(0, 10, 1, 10), (2, 0, 0, 1)]) + s = a.create_slice([Chunk(0, 10, 1, 10), Chunk(2, 0, 0, 1)]) assert s.start == 20 assert s.shape == [10, 3] def test_create_slice_c(self): a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'C') - s = a.create_slice([(3, 0, 0, 1)]) + s = a.create_slice([Chunk(3, 0, 0, 1)]) assert s.start == 45 assert s.strides == [3, 1] assert s.backstrides == [12, 2] - s = a.create_slice([(1, 9, 2, 4)]) + s = a.create_slice([Chunk(1, 9, 2, 4)]) assert s.start == 15 assert s.strides == [30, 3, 1] assert s.backstrides == [90, 12, 2] - s = a.create_slice([(1, 5, 3, 2), (1, 2, 1, 1), (1, 0, 0, 1)]) + s = a.create_slice([Chunk(1, 5, 3, 2), Chunk(1, 2, 1, 1), + Chunk(1, 0, 0, 1)]) assert s.start == 19 assert s.shape == [2, 1] assert s.strides == [45, 3] assert s.backstrides == [45, 0] - s = a.create_slice([(0, 10, 1, 10), (2, 0, 0, 1)]) + s = a.create_slice([Chunk(0, 10, 1, 10), Chunk(2, 0, 0, 1)]) assert s.start == 6 assert s.shape == [10, 3] def test_slice_of_slice_f(self): a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'F') - s = a.create_slice([(5, 0, 0, 1)]) + s = a.create_slice([Chunk(5, 0, 0, 1)]) assert s.start == 5 - s2 = s.create_slice([(3, 0, 0, 1)]) + s2 = s.create_slice([Chunk(3, 0, 0, 1)]) assert s2.shape == [3] assert s2.strides == [50] assert s2.parent is a assert s2.backstrides == [100] assert s2.start == 35 - s = a.create_slice([(1, 5, 3, 2)]) - s2 = s.create_slice([(0, 2, 1, 2), (2, 0, 0, 1)]) + s = a.create_slice([Chunk(1, 5, 3, 2)]) + s2 = s.create_slice([Chunk(0, 2, 1, 2), Chunk(2, 0, 0, 1)]) assert s2.shape == [2, 3] assert s2.strides == [3, 50] assert s2.backstrides == [3, 100] @@ -91,16 +93,16 @@ def test_slice_of_slice_c(self): a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), order='C') - s = a.create_slice([(5, 0, 0, 1)]) + s = a.create_slice([Chunk(5, 0, 0, 1)]) assert s.start == 15 * 5 - s2 = s.create_slice([(3, 0, 0, 1)]) + s2 = s.create_slice([Chunk(3, 0, 0, 1)]) assert s2.shape == [3] assert s2.strides == [1] assert s2.parent is a assert s2.backstrides == [2] assert s2.start == 5 * 15 + 3 * 3 - s = a.create_slice([(1, 5, 3, 2)]) - s2 = s.create_slice([(0, 2, 1, 2), (2, 0, 0, 1)]) + s = a.create_slice([Chunk(1, 5, 3, 2)]) + s2 = s.create_slice([Chunk(0, 2, 1, 2), Chunk(2, 0, 0, 1)]) assert s2.shape == [2, 3] assert s2.strides == [45, 1] assert s2.backstrides == [45, 2] @@ -108,14 +110,14 @@ def test_negative_step_f(self): a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'F') - s = a.create_slice([(9, -1, -2, 5)]) + s = a.create_slice([Chunk(9, -1, -2, 5)]) assert s.start == 9 assert s.strides == [-2, 10, 50] assert s.backstrides == [-8, 40, 100] def test_negative_step_c(self): a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), order='C') - s = a.create_slice([(9, -1, -2, 5)]) + s = a.create_slice([Chunk(9, -1, -2, 5)]) assert s.start == 135 assert s.strides == [-30, 3, 1] assert s.backstrides == [-120, 12, 2] @@ -124,7 +126,7 @@ a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'F') r = a._index_of_single_item(self.space, self.newtuple(1, 2, 2)) assert r == 1 + 2 * 10 + 2 * 50 - s = a.create_slice([(0, 10, 1, 10), (2, 0, 0, 1)]) + s = a.create_slice([Chunk(0, 10, 1, 10), Chunk(2, 0, 0, 1)]) r = s._index_of_single_item(self.space, self.newtuple(1, 0)) assert r == a._index_of_single_item(self.space, self.newtuple(1, 2, 0)) r = s._index_of_single_item(self.space, self.newtuple(1, 1)) @@ -134,7 +136,7 @@ a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'C') r = a._index_of_single_item(self.space, self.newtuple(1, 2, 2)) assert r == 1 * 3 * 5 + 2 * 3 + 2 - s = a.create_slice([(0, 10, 1, 10), (2, 0, 0, 1)]) + s = a.create_slice([Chunk(0, 10, 1, 10), Chunk(2, 0, 0, 1)]) r = s._index_of_single_item(self.space, self.newtuple(1, 0)) assert r == a._index_of_single_item(self.space, self.newtuple(1, 2, 0)) r = s._index_of_single_item(self.space, self.newtuple(1, 1)) @@ -152,14 +154,36 @@ def test_calc_new_strides(self): from pypy.module.micronumpy.interp_numarray import calc_new_strides - assert calc_new_strides([2, 4], [4, 2], [4, 2]) == [8, 2] - assert calc_new_strides([2, 4, 3], [8, 3], [1, 16]) == [1, 2, 16] - assert calc_new_strides([2, 3, 4], [8, 3], [1, 16]) is None - assert calc_new_strides([24], [2, 4, 3], [48, 6, 1]) is None - assert calc_new_strides([24], [2, 4, 3], [24, 6, 2]) == [2] - assert calc_new_strides([105, 1], [3, 5, 7], [35, 7, 1]) == [1, 1] - assert calc_new_strides([1, 105], [3, 5, 7], [35, 7, 1]) == [105, 1] + assert calc_new_strides([2, 4], [4, 2], [4, 2], "C") == [8, 2] + assert calc_new_strides([2, 4, 3], [8, 3], [1, 16], 'F') == [1, 2, 16] + assert calc_new_strides([2, 3, 4], [8, 3], [1, 16], 'F') is None + assert calc_new_strides([24], [2, 4, 3], [48, 6, 1], 'C') is None + assert calc_new_strides([24], [2, 4, 3], [24, 6, 2], 'C') == [2] + assert calc_new_strides([105, 1], [3, 5, 7], [35, 7, 1],'C') == [1, 1] + assert calc_new_strides([1, 105], [3, 5, 7], [35, 7, 1],'C') == [105, 1] + assert calc_new_strides([1, 105], [3, 5, 7], [35, 7, 1],'F') is None + assert calc_new_strides([1, 1, 1, 105, 1], [15, 7], [7, 1],'C') == \ + [105, 105, 105, 1, 1] + assert calc_new_strides([1, 1, 105, 1, 1], [7, 15], [1, 7],'F') == \ + [1, 1, 1, 105, 105] + def test_to_coords(self): + from pypy.module.micronumpy.strides import to_coords + + def _to_coords(index, order): + return to_coords(self.space, [2, 3, 4], 24, order, + self.space.wrap(index))[0] + + assert _to_coords(0, 'C') == [0, 0, 0] + assert _to_coords(1, 'C') == [0, 0, 1] + assert _to_coords(-1, 'C') == [1, 2, 3] + assert _to_coords(5, 'C') == [0, 1, 1] + assert _to_coords(13, 'C') == [1, 0, 1] + assert _to_coords(0, 'F') == [0, 0, 0] + assert _to_coords(1, 'F') == [1, 0, 0] + assert _to_coords(-1, 'F') == [1, 2, 3] + assert _to_coords(5, 'F') == [1, 2, 0] + assert _to_coords(13, 'F') == [1, 0, 2] class AppTestNumArray(BaseNumpyAppTest): def test_ndarray(self): @@ -204,6 +228,7 @@ # And check that changes stick. a[13] = 5.3 assert a[13] == 5.3 + assert zeros(()).shape == () def test_size(self): from _numpypy import array @@ -247,6 +272,7 @@ b = a[::2] c = b.copy() assert (c == b).all() + assert ((a + a).copy() == (a + a)).all() a = arange(15).reshape(5,3) b = a.copy() @@ -276,6 +302,12 @@ for i in xrange(5): assert a[i] == b[i] + def test_getitem_nd(self): + from _numpypy import arange + a = arange(15).reshape(3, 5) + assert a[1, 3] == 8 + assert a.T[1, 2] == 11 + def test_setitem(self): from _numpypy import array a = array(range(5)) @@ -363,6 +395,7 @@ assert b.shape == (5,) c = a[:3] assert c.shape == (3,) + assert array([]).shape == (0,) def test_set_shape(self): from _numpypy import array, zeros @@ -383,6 +416,8 @@ a.shape = () #numpy allows this a.shape = (1,) + a = array(range(6)).reshape(2,3).T + raises(AttributeError, 'a.shape = 6') def test_reshape(self): from _numpypy import array, zeros @@ -396,6 +431,7 @@ assert (a == [1000, 1, 2, 3, 1000, 5, 6, 7, 1000, 9, 10, 11]).all() a = zeros((4, 2, 3)) a.shape = (12, 2) + (a + a).reshape(2, 12) # assert did not explode def test_slice_reshape(self): from _numpypy import zeros, arange @@ -439,6 +475,13 @@ y = z.reshape(4, 3, 8) assert y.shape == (4, 3, 8) + def test_scalar_reshape(self): + from numpypy import array + a = array(3) + assert a.reshape([1, 1]).shape == (1, 1) + assert a.reshape([1]).shape == (1,) + raises(ValueError, "a.reshape(3)") + def test_add(self): from _numpypy import array a = array(range(5)) @@ -727,16 +770,17 @@ assert d[1] == 12 def test_mean(self): - from _numpypy import array, mean + from _numpypy import array, arange a = array(range(5)) assert a.mean() == 2.0 assert a[:4].mean() == 1.5 a = array(range(105)).reshape(3, 5, 7) - b = mean(a, axis=0) - b[0,0]==35. + b = a.mean(axis=0) + b[0, 0]==35. assert a.mean(axis=0)[0, 0] == 35 assert (b == array(range(35, 70), dtype=float).reshape(5, 7)).all() - assert (mean(a, 2) == array(range(0, 15), dtype=float).reshape(3, 5) * 7 + 3).all() + assert (a.mean(2) == array(range(0, 15), dtype=float).reshape(3, 5) * 7 + 3).all() + assert (arange(10).reshape(5, 2).mean(axis=1) == [0.5, 2.5, 4.5, 6.5, 8.5]).all() def test_sum(self): from _numpypy import array @@ -968,7 +1012,7 @@ assert debug_repr(a + a) == 'Call2(add, Array, Array)' assert debug_repr(a[::2]) == 'Slice' assert debug_repr(a + 2) == 'Call2(add, Array, Scalar)' - assert debug_repr(a + a.flat) == 'Call2(add, Array, Slice)' + assert debug_repr(a + a.flat) == 'Call2(add, Array, Flat)' assert debug_repr(sin(a)) == 'Call1(sin, Array)' b = a + a @@ -1041,11 +1085,49 @@ assert a[0].tolist() == [17.1, 27.2] def test_var(self): - from _numpypy import array + from _numpypy import array, arange a = array(range(10)) assert a.var() == 8.25 a = array([5.0]) assert a.var() == 0.0 + a = arange(10).reshape(5, 2) + assert a.var() == 8.25 + assert (a.var(0) == [8, 8]).all() + assert (a.var(1) == [.25] * 5).all() + + def test_concatenate(self): + from numpypy import array, concatenate, dtype + a1 = array([0,1,2]) + a2 = array([3,4,5]) + a = concatenate((a1, a2)) + assert len(a) == 6 + assert (a == [0,1,2,3,4,5]).all() + assert a.dtype is dtype(int) + b1 = array([[1, 2], [3, 4]]) + b2 = array([[5, 6]]) + b = concatenate((b1, b2), axis=0) + assert (b == [[1, 2],[3, 4],[5, 6]]).all() + c = concatenate((b1, b2.T), axis=1) + assert (c == [[1, 2, 5],[3, 4, 6]]).all() + d = concatenate(([0],[1])) + assert (d == [0,1]).all() + e1 = array([[0,1],[2,3]]) + e = concatenate(e1) + assert (e == [0,1,2,3]).all() + f1 = array([0,1]) + f = concatenate((f1, [2], f1, [7])) + assert (f == [0,1,2,0,1,7]).all() + + bad_axis = raises(ValueError, concatenate, (a1,a2), axis=1) + assert str(bad_axis.value) == "bad axis argument" + + concat_zero = raises(ValueError, concatenate, ()) + assert str(concat_zero.value) == \ + "concatenation of zero-length sequences is impossible" + + dims_disagree = raises(ValueError, concatenate, (a1, b1), axis=0) + assert str(dims_disagree.value) == \ + "array dimensions must agree except for axis being concatenated" def test_std(self): from _numpypy import array @@ -1054,6 +1136,28 @@ a = array([5.0]) assert a.std() == 0.0 + def test_flatten(self): + from _numpypy import array + + assert array(3).flatten().shape == (1,) + a = array([[1, 2], [3, 4]]) + b = a.flatten() + c = a.ravel() + a[0, 0] = 15 + assert b[0] == 1 + assert c[0] == 15 + a = array([[1, 2, 3], [4, 5, 6]]) + assert (a.flatten() == [1, 2, 3, 4, 5, 6]).all() + a = array([[[1, 2], [3, 4]], [[5, 6], [7, 8]]]) + assert (a.flatten() == [1, 2, 3, 4, 5, 6, 7, 8]).all() + a = array([1, 2, 3, 4, 5, 6, 7, 8]) + assert (a[::2].flatten() == [1, 3, 5, 7]).all() + a = array([1, 2, 3]) + assert ((a + a).flatten() == [2, 4, 6]).all() + a = array(2) + assert (a.flatten() == [2]).all() + a = array([[1, 2], [3, 4]]) + assert (a.T.flatten() == [1, 3, 2, 4]).all() class AppTestMultiDim(BaseNumpyAppTest): def test_init(self): @@ -1069,7 +1173,7 @@ assert _numpypy.array([[1], [2], [3]]).shape == (3, 1) assert len(_numpypy.zeros((3, 1, 2))) == 3 raises(TypeError, len, _numpypy.zeros(())) - raises(ValueError, _numpypy.array, [[1, 2], 3]) + raises(ValueError, _numpypy.array, [[1, 2], 3], dtype=float) def test_getsetitem(self): import _numpypy @@ -1284,7 +1388,7 @@ assert(b[:, 0] == a[0, :]).all() def test_flatiter(self): - from _numpypy import array, flatiter + from _numpypy import array, flatiter, arange a = array([[10, 30], [40, 60]]) f_iter = a.flat assert f_iter.next() == 10 @@ -1297,6 +1401,9 @@ for k in a.flat: s += k assert s == 140 + a = arange(10).reshape(5, 2) + raises(IndexError, 'a.flat[(1, 2)]') + assert a.flat.base is a def test_flatiter_array_conv(self): from _numpypy import array, dot @@ -1308,6 +1415,75 @@ a = ones((2, 2)) assert list(((a + a).flat)) == [2, 2, 2, 2] + def test_flatiter_getitem(self): + from _numpypy import arange + a = arange(10) + assert a.flat[3] == 3 + assert a[2:].flat[3] == 5 + assert (a + a).flat[3] == 6 + assert a[::2].flat[3] == 6 + assert a.reshape(2,5).flat[3] == 3 + b = a.reshape(2,5).flat + b.next() + b.next() + b.next() + assert b[3] == 3 + assert (b[::3] == [0, 3, 6, 9]).all() + assert (b[2::5] == [2, 7]).all() + assert b[-2] == 8 + raises(IndexError, "b[11]") + raises(IndexError, "b[-11]") + raises(IndexError, 'b[0, 1]') + assert b.index == 3 + assert b.coords == (0,3) + + def test_flatiter_setitem(self): + from _numpypy import arange, array + a = arange(12).reshape(3,4) + b = a.T.flat + b[6::2] = [-1, -2] + assert (a == [[0, 1, -1, 3], [4, 5, 6, -1], [8, 9, -2, 11]]).all() + b[0:2] = [[[100]]] + assert(a[0,0] == 100) + assert(a[1,0] == 100) + raises(IndexError, 'b[array([10, 11])] == [-20, -40]') + + def test_flatiter_ops(self): + from _numpypy import arange, array + a = arange(12).reshape(3,4) + b = a.T.flat + assert (b == [0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11]).all() + assert not (b != [0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11]).any() + assert ((b >= range(12)) == [True, True, True,False, True, True, + False, False, True, False, False, True]).all() + assert ((b < range(12)) != [True, True, True,False, True, True, + False, False, True, False, False, True]).all() + assert ((b <= range(12)) != [False, True, True,False, True, True, + False, False, True, False, False, False]).all() + assert ((b > range(12)) == [False, True, True,False, True, True, + False, False, True, False, False, False]).all() + def test_flatiter_view(self): + from _numpypy import arange + a = arange(10).reshape(5, 2) + #no == yet. + # a[::2].flat == [0, 1, 4, 5, 8, 9] + isequal = True + for y,z in zip(a[::2].flat, [0, 1, 4, 5, 8, 9]): + if y != z: + isequal = False + assert isequal == True + + def test_flatiter_transpose(self): + from _numpypy import arange + a = arange(10).reshape(2,5).T + b = a.flat + assert (b[:5] == [0, 5, 1, 6, 2]).all() + b.next() + b.next() + b.next() + assert b.index == 3 + assert b.coords == (1,1) + def test_slice_copy(self): from _numpypy import zeros a = zeros((10, 10)) @@ -1324,6 +1500,110 @@ assert isinstance(i['data'][0], int) raises(TypeError, getattr, array(3), '__array_interface__') + def test_array_indexing_one_elem(self): + skip("not yet") + from _numpypy import array, arange + raises(IndexError, 'arange(3)[array([3.5])]') + a = arange(3)[array([1])] + assert a == 1 + assert a[0] == 1 + raises(IndexError,'arange(3)[array([15])]') + assert arange(3)[array([-3])] == 0 + raises(IndexError,'arange(3)[array([-15])]') + assert arange(3)[array(1)] == 1 + + def test_fill(self): + from _numpypy import array + a = array([1, 2, 3]) + a.fill(10) + assert (a == [10, 10, 10]).all() + a.fill(False) + assert (a == [0, 0, 0]).all() + b = a[:1] + b.fill(4) + assert (b == [4]).all() + assert (a == [4, 0, 0]).all() + + c = b + b + c.fill(27) + assert (c == [27]).all() + + d = array(10) + d.fill(100) + assert d == 100 + + def test_array_indexing_bool(self): + from _numpypy import arange + a = arange(10) + assert (a[a > 3] == [4, 5, 6, 7, 8, 9]).all() + a = arange(10).reshape(5, 2) + assert (a[a > 3] == [4, 5, 6, 7, 8, 9]).all() + assert (a[a & 1 == 1] == [1, 3, 5, 7, 9]).all() + + def test_array_indexing_bool_setitem(self): + from _numpypy import arange, array + a = arange(6) + a[a > 3] = 15 + assert (a == [0, 1, 2, 3, 15, 15]).all() + a = arange(6).reshape(3, 2) + a[a & 1 == 1] = array([8, 9, 10]) + assert (a == [[0, 8], [2, 9], [4, 10]]).all() + + def test_copy_kwarg(self): + from _numpypy import array + x = array([1, 2, 3]) + assert (array(x) == x).all() + assert array(x) is not x + assert array(x, copy=False) is x + assert array(x, copy=True) is not x + + def test_isna(self): + from _numpypy import isna, array + # XXX for now + assert not isna(3) + assert (isna(array([1, 2, 3, 4])) == [False, False, False, False]).all() + + def test_ravel(self): + from _numpypy import arange + assert (arange(3).ravel() == arange(3)).all() + assert (arange(6).reshape(2, 3).ravel() == arange(6)).all() + assert (arange(6).reshape(2, 3).T.ravel() == [0, 3, 1, 4, 2, 5]).all() + + def test_take(self): + from _numpypy import arange + assert (arange(10).take([1, 2, 1, 1]) == [1, 2, 1, 1]).all() + raises(IndexError, "arange(3).take([15])") + a = arange(6).reshape(2, 3) + assert (a.take([1, 0, 3]) == [1, 0, 3]).all() + assert ((a + a).take([3]) == [6]).all() + a = arange(12).reshape(2, 6) + assert (a[:,::2].take([3, 2, 1]) == [6, 4, 2]).all() + + def test_compress(self): + from _numpypy import arange + a = arange(10) + assert (a.compress([True, False, True]) == [0, 2]).all() + assert (a.compress([1, 0, 13]) == [0, 2]).all() + assert (a.compress([1, 0, 13.5]) == [0, 2]).all() + a = arange(10).reshape(2, 5) + assert (a.compress([True, False, True]) == [0, 2]).all() + raises(IndexError, "a.compress([1] * 100)") + + def test_item(self): + from _numpypy import array + assert array(3).item() == 3 + assert type(array(3).item()) is int + assert type(array(True).item()) is bool + assert type(array(3.5).item()) is float + raises((ValueError, IndexError), "array(3).item(15)") + raises(ValueError, "array([1, 2, 3]).item()") + assert array([3]).item(0) == 3 + assert type(array([3]).item(0)) is int + assert array([1, 2, 3]).item(-1) == 3 + a = array([1, 2, 3]) + assert a[::2].item(1) == 3 + assert (a + a).item(1) == 4 + raises(ValueError, "array(5).item(1)") class AppTestSupport(BaseNumpyAppTest): def setup_class(cls): @@ -1435,123 +1715,6 @@ raises(ValueError, fromstring, "\x01\x02\x03", count=5, dtype=uint8) -class AppTestRepr(BaseNumpyAppTest): - def test_repr(self): - from _numpypy import array, zeros - int_size = array(5).dtype.itemsize - a = array(range(5), float) - assert repr(a) == "array([0.0, 1.0, 2.0, 3.0, 4.0])" - a = array([], float) - assert repr(a) == "array([], dtype=float64)" - a = zeros(1001) - assert repr(a) == "array([0.0, 0.0, 0.0, ..., 0.0, 0.0, 0.0])" - a = array(range(5), long) - if a.dtype.itemsize == int_size: - assert repr(a) == "array([0, 1, 2, 3, 4])" - else: - assert repr(a) == "array([0, 1, 2, 3, 4], dtype=int64)" - a = array(range(5), 'int32') - if a.dtype.itemsize == int_size: - assert repr(a) == "array([0, 1, 2, 3, 4])" - else: - assert repr(a) == "array([0, 1, 2, 3, 4], dtype=int32)" - a = array([], long) - assert repr(a) == "array([], dtype=int64)" - a = array([True, False, True, False], "?") - assert repr(a) == "array([True, False, True, False], dtype=bool)" - a = zeros([]) - assert repr(a) == "array(0.0)" - a = array(0.2) - assert repr(a) == "array(0.2)" - - def test_repr_multi(self): - from _numpypy import arange, zeros - a = zeros((3, 4)) - assert repr(a) == '''array([[0.0, 0.0, 0.0, 0.0], - [0.0, 0.0, 0.0, 0.0], - [0.0, 0.0, 0.0, 0.0]])''' - a = zeros((2, 3, 4)) - assert repr(a) == '''array([[[0.0, 0.0, 0.0, 0.0], - [0.0, 0.0, 0.0, 0.0], - [0.0, 0.0, 0.0, 0.0]], - - [[0.0, 0.0, 0.0, 0.0], - [0.0, 0.0, 0.0, 0.0], - [0.0, 0.0, 0.0, 0.0]]])''' - a = arange(1002).reshape((2, 501)) - assert repr(a) == '''array([[0, 1, 2, ..., 498, 499, 500], - [501, 502, 503, ..., 999, 1000, 1001]])''' - assert repr(a.T) == '''array([[0, 501], - [1, 502], - [2, 503], - ..., - [498, 999], - [499, 1000], - [500, 1001]])''' - - def test_repr_slice(self): - from _numpypy import array, zeros - a = array(range(5), float) - b = a[1::2] - assert repr(b) == "array([1.0, 3.0])" - a = zeros(2002) - b = a[::2] - assert repr(b) == "array([0.0, 0.0, 0.0, ..., 0.0, 0.0, 0.0])" - a = array((range(5), range(5, 10)), dtype="int16") - b = a[1, 2:] - assert repr(b) == "array([7, 8, 9], dtype=int16)" - # an empty slice prints its shape - b = a[2:1, ] - assert repr(b) == "array([], shape=(0, 5), dtype=int16)" - - def test_str(self): - from _numpypy import array, zeros - a = array(range(5), float) - assert str(a) == "[0.0 1.0 2.0 3.0 4.0]" - assert str((2 * a)[:]) == "[0.0 2.0 4.0 6.0 8.0]" - a = zeros(1001) - assert str(a) == "[0.0 0.0 0.0 ..., 0.0 0.0 0.0]" - - a = array(range(5), dtype=long) - assert str(a) == "[0 1 2 3 4]" - a = array([True, False, True, False], dtype="?") - assert str(a) == "[True False True False]" - - a = array(range(5), dtype="int8") - assert str(a) == "[0 1 2 3 4]" - - a = array(range(5), dtype="int16") - assert str(a) == "[0 1 2 3 4]" - - a = array((range(5), range(5, 10)), dtype="int16") - assert str(a) == "[[0 1 2 3 4]\n [5 6 7 8 9]]" - - a = array(3, dtype=int) - assert str(a) == "3" - - a = zeros((400, 400), dtype=int) - assert str(a) == "[[0 0 0 ..., 0 0 0]\n [0 0 0 ..., 0 0 0]\n" \ - " [0 0 0 ..., 0 0 0]\n ...,\n [0 0 0 ..., 0 0 0]\n" \ - " [0 0 0 ..., 0 0 0]\n [0 0 0 ..., 0 0 0]]" - a = zeros((2, 2, 2)) - r = str(a) - assert r == '[[[0.0 0.0]\n [0.0 0.0]]\n\n [[0.0 0.0]\n [0.0 0.0]]]' - - def test_str_slice(self): - from _numpypy import array, zeros - a = array(range(5), float) - b = a[1::2] - assert str(b) == "[1.0 3.0]" - a = zeros(2002) - b = a[::2] - assert str(b) == "[0.0 0.0 0.0 ..., 0.0 0.0 0.0]" - a = array((range(5), range(5, 10)), dtype="int16") - b = a[1, 2:] - assert str(b) == "[7 8 9]" - b = a[2:1, ] - assert str(b) == "[]" - - class AppTestRanges(BaseNumpyAppTest): def test_arange(self): from _numpypy import arange, array, dtype @@ -1573,17 +1736,23 @@ assert len(a) == 8 assert arange(False, True, True).dtype is dtype(int) +from pypy.module.micronumpy.appbridge import get_appbridge_cache -class AppTestRanges(BaseNumpyAppTest): - def test_app_reshape(self): - from _numpypy import arange, array, dtype, reshape - a = arange(12) - b = reshape(a, (3, 4)) - assert b.shape == (3, 4) - a = range(12) - b = reshape(a, (3, 4)) - assert b.shape == (3, 4) - a = array(range(105)).reshape(3, 5, 7) - assert a.reshape(1, -1).shape == (1, 105) - assert a.reshape(1, 1, -1).shape == (1, 1, 105) - assert a.reshape(-1, 1, 1).shape == (105, 1, 1) +class AppTestRepr(BaseNumpyAppTest): + def setup_class(cls): + BaseNumpyAppTest.setup_class.im_func(cls) + cache = get_appbridge_cache(cls.space) + cls.old_array_repr = cache.w_array_repr + cls.old_array_str = cache.w_array_str + cache.w_array_str = None + cache.w_array_repr = None + + def test_repr_str(self): + from _numpypy import array + assert repr(array([1, 2, 3])) == 'array([1, 2, 3])' + assert str(array([1, 2, 3])) == 'array([1, 2, 3])' + + def teardown_class(cls): + cache = get_appbridge_cache(cls.space) + cache.w_array_repr = cls.old_array_repr + cache.w_array_str = cls.old_array_str diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py --- a/pypy/module/micronumpy/test/test_ufuncs.py +++ b/pypy/module/micronumpy/test/test_ufuncs.py @@ -190,14 +190,24 @@ for i in range(3): assert c[i] == a[i] - b[i] - def test_floor(self): - from _numpypy import array, floor - - reference = [-2.0, -1.0, 0.0, 1.0, 1.0] - a = array([-1.4, -1.0, 0.0, 1.0, 1.4]) + def test_floorceil(self): + from _numpypy import array, floor, ceil + import math + reference = [-2.0, -2.0, -1.0, 0.0, 1.0, 1.0, 0] + a = array([-1.4, -1.5, -1.0, 0.0, 1.0, 1.4, 0.5]) b = floor(a) for i in range(5): assert b[i] == reference[i] + reference = [-1.0, -1.0, -1.0, 0.0, 1.0, 2.0, 1.0] + a = array([-1.4, -1.5, -1.0, 0.0, 1.0, 1.4, 0.5]) + b = ceil(a) + assert (reference == b).all() + inf = float("inf") + data = [1.5, 2.9999, -1.999, inf] + results = [math.floor(x) for x in data] + assert (floor(data) == results).all() + results = [math.ceil(x) for x in data] + assert (ceil(data) == results).all() def test_copysign(self): from _numpypy import array, copysign @@ -238,7 +248,7 @@ assert b[i] == math.sin(a[i]) a = sin(array([True, False], dtype=bool)) - assert abs(a[0] - sin(1)) < 1e-7 # a[0] will be less precise + assert abs(a[0] - sin(1)) < 1e-7 # a[0] will be less precise assert a[1] == 0.0 def test_cos(self): @@ -259,7 +269,6 @@ for i in range(len(a)): assert b[i] == math.tan(a[i]) - def test_arcsin(self): import math from _numpypy import array, arcsin @@ -283,7 +292,6 @@ for i in range(len(a)): assert b[i] == math.acos(a[i]) - a = array([-10, -1.5, -1.01, 1.01, 1.5, 10, float('nan'), float('inf'), float('-inf')]) b = arccos(a) for f in b: @@ -336,7 +344,7 @@ from _numpypy import sin, add raises(ValueError, sin.reduce, [1, 2, 3]) - raises(ValueError, add.reduce, 1) + raises((ValueError, TypeError), add.reduce, 1) def test_reduce_1d(self): from _numpypy import add, maximum @@ -347,11 +355,34 @@ raises(ValueError, maximum.reduce, []) def test_reduceND(self): - from numpypy import add, arange + from _numpypy import add, arange a = arange(12).reshape(3, 4) assert (add.reduce(a, 0) == [12, 15, 18, 21]).all() assert (add.reduce(a, 1) == [6.0, 22.0, 38.0]).all() + def test_reduce_keepdims(self): + from _numpypy import add, arange + a = arange(12).reshape(3, 4) + b = add.reduce(a, 0, keepdims=True) + assert b.shape == (1, 4) + assert (add.reduce(a, 0, keepdims=True) == [12, 15, 18, 21]).all() + + + def test_bitwise(self): + from _numpypy import bitwise_and, bitwise_or, arange, array + a = arange(6).reshape(2, 3) + assert (a & 1 == [[0, 1, 0], [1, 0, 1]]).all() + assert (a & 1 == bitwise_and(a, 1)).all() + assert (a | 1 == [[1, 1, 3], [3, 5, 5]]).all() + assert (a | 1 == bitwise_or(a, 1)).all() + raises(TypeError, 'array([1.0]) & 1') + + def test_unary_bitops(self): + from _numpypy import bitwise_not, array + a = array([1, 2, 3, 4]) + assert (~a == [-2, -3, -4, -5]).all() + assert (bitwise_not(a) == ~a).all() + def test_comparisons(self): import operator from _numpypy import equal, not_equal, less, less_equal, greater, greater_equal @@ -377,3 +408,28 @@ (3, 3.5), ]: assert ufunc(a, b) == func(a, b) + + def test_count_reduce_items(self): + from _numpypy import count_reduce_items, arange + a = arange(24).reshape(2, 3, 4) + assert count_reduce_items(a) == 24 + assert count_reduce_items(a, 1) == 3 + assert count_reduce_items(a, (1, 2)) == 3 * 4 + + def test_true_divide(self): + from _numpypy import arange, array, true_divide + assert (true_divide(arange(3), array([2, 2, 2])) == array([0, 0.5, 1])).all() + + def test_isnan_isinf(self): + from _numpypy import isnan, isinf, float64, array + assert isnan(float('nan')) + assert isnan(float64(float('nan'))) + assert not isnan(3) + assert isinf(float('inf')) + assert not isnan(3.5) + assert not isinf(3.5) + assert not isnan(float('inf')) + assert not isinf(float('nan')) + assert (isnan(array([0.2, float('inf'), float('nan')])) == [False, False, True]).all() + assert (isinf(array([0.2, float('inf'), float('nan')])) == [False, True, False]).all() + assert isinf(array([0.2])).dtype.kind == 'b' diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py --- a/pypy/module/micronumpy/test/test_zjit.py +++ b/pypy/module/micronumpy/test/test_zjit.py @@ -12,7 +12,7 @@ from pypy.module.micronumpy.compile import (FakeSpace, IntObject, Parser, InterpreterState) from pypy.module.micronumpy.interp_numarray import (W_NDimArray, - BaseArray) + BaseArray, W_FlatIterator) from pypy.rlib.nonconst import NonConstant @@ -217,6 +217,7 @@ # This is the sum of the ops for both loops, however if you remove the # optimization then you end up with 2 float_adds, so we can still be # sure it was optimized correctly. + py.test.skip("too fragile") self.check_resops({'setinteriorfield_raw': 4, 'getfield_gc': 22, 'getarrayitem_gc': 4, 'getarrayitem_gc_pure': 2, 'getfield_gc_pure': 8, @@ -286,6 +287,27 @@ 'jump': 1, 'arraylen_gc': 1}) + def define_take(): + return """ + a = |10| + b = take(a, [1, 1, 3, 2]) + b -> 2 + """ + + def test_take(self): + result = self.run("take") + assert result == 3 + self.check_simple_loop({'getinteriorfield_raw': 2, + 'cast_float_to_int': 1, + 'int_lt': 1, + 'int_ge': 2, + 'guard_false': 3, + 'setinteriorfield_raw': 1, + 'int_mul': 1, + 'int_add': 3, + 'jump': 1, + 'arraylen_gc': 2}) + def define_multidim(): return """ a = [[1, 2], [3, 4], [5, 6], [7, 8], [9, 10]] @@ -349,7 +371,8 @@ self.check_trace_count(1) self.check_simple_loop({'getinteriorfield_raw': 2, 'float_add': 1, 'setinteriorfield_raw': 1, 'int_add': 2, - 'int_eq': 1, 'guard_false': 1, 'jump': 1}) + 'int_eq': 1, 'guard_false': 1, 'jump': 1, + 'arraylen_gc': 1}) def define_virtual_slice(): return """ @@ -367,6 +390,71 @@ 'setinteriorfield_raw': 1, 'int_add': 2, 'int_ge': 1, 'guard_false': 1, 'jump': 1, 'arraylen_gc': 1}) + def define_flat_iter(): + return ''' + a = |30| + b = flat(a) + c = b + a + c -> 3 + ''' + + def test_flat_iter(self): + result = self.run("flat_iter") + assert result == 6 + self.check_trace_count(1) + self.check_simple_loop({'getinteriorfield_raw': 2, 'float_add': 1, + 'setinteriorfield_raw': 1, 'int_add': 3, + 'int_ge': 1, 'guard_false': 1, + 'arraylen_gc': 1, 'jump': 1}) + + def define_flat_getitem(): + return ''' + a = |30| + b = flat(a) + b -> 4: -> 6 + ''' + + def test_flat_getitem(self): + result = self.run("flat_getitem") + assert result == 10.0 + self.check_trace_count(1) + self.check_simple_loop({'getinteriorfield_raw': 1, + 'setinteriorfield_raw': 1, + 'int_lt': 1, + 'int_ge': 1, + 'int_add': 3, + 'guard_true': 1, + 'guard_false': 1, + 'arraylen_gc': 2, + 'jump': 1}) + + def define_flat_setitem(): + return ''' + a = |30| + b = flat(a) + b[4:] = a->:26 + a -> 5 + ''' + + def test_flat_setitem(self): + result = self.run("flat_setitem") + assert result == 1.0 + self.check_trace_count(1) + # XXX not ideal, but hey, let's ignore it for now + self.check_simple_loop({'getinteriorfield_raw': 1, + 'setinteriorfield_raw': 1, + 'int_lt': 1, + 'int_gt': 1, + 'int_add': 4, + 'guard_true': 2, + 'arraylen_gc': 2, + 'jump': 1, + 'int_sub': 1, + # XXX bad part + 'int_and': 1, + 'int_mod': 1, + 'int_rshift': 1, + }) def define_dot(): return """ diff --git a/pypy/module/micronumpy/tool/numready/__init__.py b/pypy/module/micronumpy/tool/numready/__init__.py new file mode 100644 diff --git a/pypy/module/micronumpy/tool/numready/__main__.py b/pypy/module/micronumpy/tool/numready/__main__.py new file mode 100644 --- /dev/null +++ b/pypy/module/micronumpy/tool/numready/__main__.py @@ -0,0 +1,6 @@ +import sys + +from .main import main + + +main(sys.argv) diff --git a/pypy/module/micronumpy/tool/numready/kinds.py b/pypy/module/micronumpy/tool/numready/kinds.py new file mode 100644 --- /dev/null +++ b/pypy/module/micronumpy/tool/numready/kinds.py @@ -0,0 +1,4 @@ +KINDS = { + "UNKNOWN": "U", + "TYPE": "T", +} diff --git a/pypy/module/micronumpy/tool/numready/main.py b/pypy/module/micronumpy/tool/numready/main.py new file mode 100644 --- /dev/null +++ b/pypy/module/micronumpy/tool/numready/main.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" +This should be run under PyPy. +""" + +import os +import platform +import subprocess +import tempfile +import webbrowser +from collections import OrderedDict + +import jinja2 + +from .kinds import KINDS + + +class SearchableSet(object): + def __init__(self, items=()): + self._items = {} + for item in items: + self.add(item) + + def __iter__(self): + return iter(self._items) + + def __contains__(self, other): + return other in self._items + + def __getitem__(self, idx): + return self._items[idx] + + def add(self, item): + self._items[item] = item + + def __len__(self): + return len(self._items) + +class Item(object): + def __init__(self, name, kind, subitems=None): + self.name = name + self.kind = kind + self.subitems = subitems + + def __hash__(self): + return hash(self.name) + + def __eq__(self, other): + if isinstance(other, str): + return self.name == other + return self.name == other.name + + +class ItemStatus(object): + def __init__(self, name, pypy_exists): + self.name = name + self.cls = 'exists' if pypy_exists else '' + self.symbol = u"✔" if pypy_exists else u'✖' + + def __lt__(self, other): + return self.name < other.name + +def find_numpy_items(python, modname="numpy", attr=None): + args = [ + python, os.path.join(os.path.dirname(__file__), "search.py"), modname + ] + if attr is not None: + args.append(attr) + lines = subprocess.check_output(args).splitlines() + items = SearchableSet() + for line in lines: + kind, name = line.split(" : ", 1) + subitems = None + if kind == KINDS["TYPE"] and name in SPECIAL_NAMES and attr is None: + subitems = find_numpy_items(python, modname, name) + items.add(Item(name, kind, subitems)) + return items + +def split(lst): + SPLIT = 5 + lgt = len(lst) // SPLIT + 1 + l = [[] for i in range(lgt)] + for i in range(lgt): + for k in range(SPLIT): + if k * lgt + i < len(lst): + l[i].append(lst[k * lgt + i]) + return l + +SPECIAL_NAMES = ["ndarray", "dtype", "generic"] + +def main(argv): + cpy_items = find_numpy_items("/usr/bin/python") + pypy_items = find_numpy_items(argv[1], "numpypy") + all_items = [] + + msg = "{:d}/{:d} names".format(len(pypy_items), len(cpy_items)) + " " + msg += ", ".join( + "{:d}/{:d} {} attributes".format( + len(pypy_items[name].subitems), len(cpy_items[name].subitems), name + ) + for name in SPECIAL_NAMES + ) + for item in cpy_items: + pypy_exists = item in pypy_items + if item.subitems: + for sub in item.subitems: + all_items.append( + ItemStatus(item.name + "." + sub.name, pypy_exists=pypy_exists and pypy_items[item].subitems and sub in pypy_items[item].subitems) + ) + all_items.append(ItemStatus(item.name, pypy_exists=item in pypy_items)) + env = jinja2.Environment( + loader=jinja2.FileSystemLoader(os.path.dirname(__file__)) + ) + html = env.get_template("page.html").render(all_items=split(sorted(all_items)), msg=msg) + if len(argv) > 2: + with open(argv[2], 'w') as f: + f.write(html.encode("utf-8")) + else: + with tempfile.NamedTemporaryFile(delete=False, suffix=".html") as f: + f.write(html.encode("utf-8")) + print "Saved in: %s" % f.name diff --git a/pypy/module/micronumpy/tool/numready/page.html b/pypy/module/micronumpy/tool/numready/page.html new file mode 100644 --- /dev/null +++ b/pypy/module/micronumpy/tool/numready/page.html @@ -0,0 +1,65 @@ +<!DOCTYPE html> +<html lang="en"> + <head> + <title>NumPyPy Status</title> + <meta http-equiv="content-type" content="text/html; charset=utf-8"> + <style type="text/css"> + body { + font-family: 'Consolas', 'Bitstream Vera Sans Mono', monospace; + } + h1 { + text-align: center; + } + h3 { + text-align: center; + } + table { + border: 8px solid #DFDECB; + margin: 30px auto; + font-size: 12px; + } + table th { + text-align: left; + } + table td { + padding: 4px 10px; + text-align: center; + } + .exists { + background-color: #337792; + color: white; + border: 1px solid #234F61; + } + </style> + </head> + <body> + <h1>NumPyPy Status</h1> + <h3>Overall: {{ msg }}</h3> + <table> + <thead> + <tr> + <th></th> + <th>PyPy</th> + <th></th> + <th>PyPy</th> + <th></th> + <th>PyPy</th> + <th></th> + <th>PyPy</th> + <th></th> + <th>PyPy</th> + </tr> + </thead> + <tbody> + {% for chunk in all_items %} + <tr> + {% for item in chunk %} + <th class='{{ item.cls }}'>{{ item.name }}</th> + <td class='{{ item.cls }}'>{{ item.symbol }}</td> + {% endfor %} + </tr> + {% endfor %} + </tbody> + </table> + </body> +</html> diff --git a/pypy/module/micronumpy/tool/numready/search.py b/pypy/module/micronumpy/tool/numready/search.py new file mode 100644 --- /dev/null +++ b/pypy/module/micronumpy/tool/numready/search.py @@ -0,0 +1,33 @@ +import sys +import types + +# Evil implicit relative import. +from kinds import KINDS + + +def main(argv): + if len(argv) == 2: + [_, modname] = argv + attr = None + elif len(argv) == 3: + [_, modname, attr] = argv + else: + sys.exit("Wrong number of args") + __import__(modname) + obj = sys.modules[modname] + + if attr is not None: + obj = getattr(obj, attr) + + for name in dir(obj): + if attr is None and name.startswith("_"): + continue + subobj = getattr(obj, name) + if isinstance(subobj, types.TypeType): + kind = KINDS["TYPE"] + else: + kind = KINDS["UNKNOWN"] + print kind, ":", name + +if __name__ == "__main__": + main(sys.argv) \ No newline at end of file diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py --- a/pypy/module/micronumpy/types.py +++ b/pypy/module/micronumpy/types.py @@ -23,6 +23,16 @@ ) return dispatcher +def raw_unary_op(func): + specialize.argtype(1)(func) + @functools.wraps(func) + def dispatcher(self, v): + return func( + self, + self.for_computation(self.unbox(v)) + ) + return dispatcher + def simple_binary_op(func): specialize.argtype(1, 2)(func) @functools.wraps(func) @@ -94,6 +104,11 @@ width, storage, i, offset )) + def read_bool(self, storage, width, i, offset): + return bool(self.for_computation( + libffi.array_getitem(clibffi.cast_type_to_ffitype(self.T), + width, storage, i, offset))) + def store(self, storage, width, i, offset, box): value = self.unbox(box) libffi.array_setitem(clibffi.cast_type_to_ffitype(self.T), @@ -134,6 +149,14 @@ def abs(self, v): return abs(v) + @raw_unary_op + def isnan(self, v): + return False + + @raw_unary_op + def isinf(self, v): + return False + @raw_binary_op def eq(self, v1, v2): return v1 == v2 @@ -169,6 +192,7 @@ def min(self, v1, v2): return min(v1, v2) + class Bool(BaseType, Primitive): T = lltype.Bool BoxType = interp_boxes.W_BoolBox @@ -205,6 +229,18 @@ def default_fromstring(self, space): return self.box(False) + @simple_binary_op + def bitwise_and(self, v1, v2): + return v1 & v2 + + @simple_binary_op + def bitwise_or(self, v1, v2): + return v1 | v2 + + @simple_unary_op + def invert(self, v): + return ~v + class Integer(Primitive): _mixin_ = True @@ -253,6 +289,18 @@ assert v == 0 return 0 + @simple_binary_op + def bitwise_and(self, v1, v2): + return v1 & v2 + + @simple_binary_op + def bitwise_or(self, v1, v2): + return v1 | v2 + + @simple_unary_op + def invert(self, v): + return ~v + class Int8(BaseType, Integer): T = rffi.SIGNEDCHAR BoxType = interp_boxes.W_Int8Box @@ -374,6 +422,10 @@ return math.floor(v) @simple_unary_op + def ceil(self, v): + return math.ceil(v) + + @simple_unary_op def exp(self, v): try: return math.exp(v) @@ -427,6 +479,14 @@ except ValueError: return rfloat.NAN + @raw_unary_op + def isnan(self, v): + return rfloat.isnan(v) + + @raw_unary_op + def isinf(self, v): + return rfloat.isinf(v) + class Float32(BaseType, Float): T = rffi.FLOAT @@ -436,4 +496,4 @@ class Float64(BaseType, Float): T = rffi.DOUBLE BoxType = interp_boxes.W_Float64Box - format_code = "d" \ No newline at end of file + format_code = "d" diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -371,6 +371,8 @@ if hasattr(__import__(os.name), "forkpty"): def test_forkpty(self): import sys + if 'freebsd' in sys.platform: + skip("hangs indifinitly on FreeBSD (also on CPython).") os = self.posix childpid, master_fd = os.forkpty() assert isinstance(childpid, int) diff --git a/pypy/module/pypyjit/__init__.py b/pypy/module/pypyjit/__init__.py --- a/pypy/module/pypyjit/__init__.py +++ b/pypy/module/pypyjit/__init__.py @@ -11,6 +11,7 @@ 'set_optimize_hook': 'interp_resop.set_optimize_hook', 'set_abort_hook': 'interp_resop.set_abort_hook', 'ResOperation': 'interp_resop.WrappedOp', + 'DebugMergePoint': 'interp_resop.DebugMergePoint', 'Box': 'interp_resop.WrappedBox', } diff --git a/pypy/module/pypyjit/interp_resop.py b/pypy/module/pypyjit/interp_resop.py --- a/pypy/module/pypyjit/interp_resop.py +++ b/pypy/module/pypyjit/interp_resop.py @@ -1,5 +1,6 @@ -from pypy.interpreter.typedef import TypeDef, GetSetProperty +from pypy.interpreter.typedef import (TypeDef, GetSetProperty, + interp_attrproperty, interp_attrproperty_w) from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.gateway import unwrap_spec, interp2app, NoneNotWrapped from pypy.interpreter.pycode import PyCode @@ -10,6 +11,7 @@ from pypy.jit.metainterp.resoperation import rop, AbstractResOp from pypy.rlib.nonconst import NonConstant from pypy.rlib import jit_hooks +from pypy.module.pypyjit.interp_jit import pypyjitdriver class Cache(object): in_recursion = False @@ -111,13 +113,24 @@ def wrap_oplist(space, logops, operations, ops_offset=None): l_w = [] + jitdrivers_sd = logops.metainterp_sd.jitdrivers_sd for op in operations: if ops_offset is None: ofs = -1 else: ofs = ops_offset.get(op, 0) - l_w.append(WrappedOp(jit_hooks._cast_to_gcref(op), ofs, - logops.repr_of_resop(op))) + if op.opnum == rop.DEBUG_MERGE_POINT: + jd_sd = jitdrivers_sd[op.getarg(0).getint()] + greenkey = op.getarglist()[2:] + repr = jd_sd.warmstate.get_location_str(greenkey) + w_greenkey = wrap_greenkey(space, jd_sd.jitdriver, greenkey, repr) + l_w.append(DebugMergePoint(space, jit_hooks._cast_to_gcref(op), + logops.repr_of_resop(op), + jd_sd.jitdriver.name, + w_greenkey)) + else: + l_w.append(WrappedOp(jit_hooks._cast_to_gcref(op), ofs, + logops.repr_of_resop(op))) return l_w class WrappedBox(Wrappable): @@ -150,6 +163,15 @@ llres = res.llbox return WrappedOp(jit_hooks.resop_new(num, args, llres), offset, repr) + at unwrap_spec(repr=str, jd_name=str) +def descr_new_dmp(space, w_tp, w_args, repr, jd_name, w_greenkey): + args = [space.interp_w(WrappedBox, w_arg).llbox for w_arg in + space.listview(w_args)] + num = rop.DEBUG_MERGE_POINT + return DebugMergePoint(space, + jit_hooks.resop_new(num, args, jit_hooks.emptyval()), + repr, jd_name, w_greenkey) + class WrappedOp(Wrappable): """ A class representing a single ResOperation, wrapped nicely """ @@ -182,6 +204,25 @@ box = space.interp_w(WrappedBox, w_box) jit_hooks.resop_setresult(self.op, box.llbox) +class DebugMergePoint(WrappedOp): + def __init__(self, space, op, repr_of_resop, jd_name, w_greenkey): + WrappedOp.__init__(self, op, -1, repr_of_resop) + self.w_greenkey = w_greenkey + self.jd_name = jd_name + + def get_pycode(self, space): + if self.jd_name == pypyjitdriver.name: + return space.getitem(self.w_greenkey, space.wrap(0)) + raise OperationError(space.w_AttributeError, space.wrap("This DebugMergePoint doesn't belong to the main Python JitDriver")) + + def get_bytecode_no(self, space): + if self.jd_name == pypyjitdriver.name: + return space.getitem(self.w_greenkey, space.wrap(1)) + raise OperationError(space.w_AttributeError, space.wrap("This DebugMergePoint doesn't belong to the main Python JitDriver")) + + def get_jitdriver_name(self, space): + return space.wrap(self.jd_name) + WrappedOp.typedef = TypeDef( 'ResOperation', __doc__ = WrappedOp.__doc__, @@ -195,3 +236,15 @@ WrappedOp.descr_setresult) ) WrappedOp.acceptable_as_base_class = False + +DebugMergePoint.typedef = TypeDef( + 'DebugMergePoint', WrappedOp.typedef, + __new__ = interp2app(descr_new_dmp), + greenkey = interp_attrproperty_w("w_greenkey", cls=DebugMergePoint), + pycode = GetSetProperty(DebugMergePoint.get_pycode), + bytecode_no = GetSetProperty(DebugMergePoint.get_bytecode_no), + jitdriver_name = GetSetProperty(DebugMergePoint.get_jitdriver_name), +) +DebugMergePoint.acceptable_as_base_class = False + + diff --git a/pypy/module/pypyjit/test/test_jit_hook.py b/pypy/module/pypyjit/test/test_jit_hook.py --- a/pypy/module/pypyjit/test/test_jit_hook.py +++ b/pypy/module/pypyjit/test/test_jit_hook.py @@ -92,6 +92,7 @@ cls.w_on_compile_bridge = space.wrap(interp2app(interp_on_compile_bridge)) cls.w_on_abort = space.wrap(interp2app(interp_on_abort)) cls.w_int_add_num = space.wrap(rop.INT_ADD) + cls.w_dmp_num = space.wrap(rop.DEBUG_MERGE_POINT) cls.w_on_optimize = space.wrap(interp2app(interp_on_optimize)) cls.orig_oplist = oplist @@ -117,6 +118,10 @@ assert elem[2][2] == False assert len(elem[3]) == 4 int_add = elem[3][0] + dmp = elem[3][1] + assert isinstance(dmp, pypyjit.DebugMergePoint) + assert dmp.pycode is self.f.func_code + assert dmp.greenkey == (self.f.func_code, 0, False) #assert int_add.name == 'int_add' assert int_add.num == self.int_add_num self.on_compile_bridge() @@ -211,3 +216,18 @@ assert op.getarg(0).getint() == 4 op.result = box assert op.result.getint() == 1 + + def test_creation_dmp(self): + from pypyjit import DebugMergePoint, Box + + def f(): + pass + + op = DebugMergePoint([Box(0)], 'repr', 'pypyjit', (f.func_code, 0, 0)) + assert op.bytecode_no == 0 + assert op.pycode is f.func_code + assert repr(op) == 'repr' + assert op.jitdriver_name == 'pypyjit' + assert op.num == self.dmp_num + op = DebugMergePoint([Box(0)], 'repr', 'notmain', ('str',)) + raises(AttributeError, 'op.pycode') diff --git a/pypy/module/pypyjit/test_pypy_c/test_math.py b/pypy/module/pypyjit/test_pypy_c/test_math.py --- a/pypy/module/pypyjit/test_pypy_c/test_math.py +++ b/pypy/module/pypyjit/test_pypy_c/test_math.py @@ -90,3 +90,19 @@ --TICK-- jump(..., descr=) """) + + def test_pow_two(self): + def main(n): + s = 0.123 + while n > 0: + s -= s ** 2 # ID: pow + n -= 1 + return s + log = self.run(main, [500]) + assert abs(log.result - main(500)) < 1e-9 + loop, = log.loops_by_filename(self.filepath) + assert loop.match_by_id("pow", """ + guard_not_invalidated(descr=...) + f38 = float_mul(f30, f30) + f39 = float_sub(f30, f38) + """) diff --git a/lib_pypy/numpypy/test/test_fromnumeric.py b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py rename from lib_pypy/numpypy/test/test_fromnumeric.py rename to pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py --- a/lib_pypy/numpypy/test/test_fromnumeric.py +++ b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py @@ -1,7 +1,7 @@ - from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest -class AppTestFromNumeric(BaseNumpyAppTest): + +class AppTestFromNumeric(BaseNumpyAppTest): def test_argmax(self): # tests taken from numpy/core/fromnumeric.py docstring from numpypy import array, arange, argmax @@ -18,12 +18,12 @@ from numpypy import array, arange, argmin a = arange(6).reshape((2,3)) assert argmin(a) == 0 - # assert (argmax(a, axis=0) == array([0, 0, 0])).all() - # assert (argmax(a, axis=1) == array([0, 0])).all() + #assert (argmin(a, axis=0) == array([0, 0, 0])).all() + #assert (argmin(a, axis=1) == array([0, 0])).all() b = arange(6) b[1] = 0 assert argmin(b) == 0 - + def test_shape(self): # tests taken from numpy/core/fromnumeric.py docstring from numpypy import array, identity, shape @@ -40,11 +40,11 @@ assert sum([0.5, 1.5])== 2.0 assert sum([[0, 1], [0, 5]]) == 6 # assert sum([0.5, 0.7, 0.2, 1.5], dtype=int32) == 1 - # assert (sum([[0, 1], [0, 5]], axis=0) == array([0, 6])).all() - # assert (sum([[0, 1], [0, 5]], axis=1) == array([1, 5])).all() + assert (sum([[0, 1], [0, 5]], axis=0) == array([0, 6])).all() + assert (sum([[0, 1], [0, 5]], axis=1) == array([1, 5])).all() # If the accumulator is too small, overflow occurs: # assert ones(128, dtype=int8).sum(dtype=int8) == -128 - + def test_amin(self): # tests taken from numpy/core/fromnumeric.py docstring from numpypy import array, arange, amin @@ -86,24 +86,54 @@ assert ndim([[1,2,3],[4,5,6]]) == 2 assert ndim(array([[1,2,3],[4,5,6]])) == 2 assert ndim(1) == 0 - + def test_rank(self): # tests taken from numpy/core/fromnumeric.py docstring from numpypy import array, rank assert rank([[1,2,3],[4,5,6]]) == 2 assert rank(array([[1,2,3],[4,5,6]])) == 2 assert rank(1) == 0 - + def test_var(self): from numpypy import array, var a = array([[1,2],[3,4]]) assert var(a) == 1.25 - # assert (np.var(a,0) == array([ 1., 1.])).all() - # assert (np.var(a,1) == array([ 0.25, 0.25])).all() + assert (var(a,0) == array([ 1., 1.])).all() + assert (var(a,1) == array([ 0.25, 0.25])).all() def test_std(self): from numpypy import array, std a = array([[1, 2], [3, 4]]) assert std(a) == 1.1180339887498949 - # assert (std(a, axis=0) == array([ 1., 1.])).all() - # assert (std(a, axis=1) == array([ 0.5, 0.5]).all() + assert (std(a, axis=0) == array([ 1., 1.])).all() + assert (std(a, axis=1) == array([ 0.5, 0.5])).all() + + def test_mean(self): + from numpypy import array, mean, arange + assert mean(array(range(5))) == 2.0 + assert mean(range(5)) == 2.0 + assert (mean(arange(10).reshape(5, 2), axis=0) == [4, 5]).all() + assert (mean(arange(10).reshape(5, 2), axis=1) == [0.5, 2.5, 4.5, 6.5, 8.5]).all() + + def test_reshape(self): + from numpypy import arange, array, dtype, reshape + a = arange(12) + b = reshape(a, (3, 4)) + assert b.shape == (3, 4) + a = range(12) + b = reshape(a, (3, 4)) + assert b.shape == (3, 4) + a = array(range(105)).reshape(3, 5, 7) + assert reshape(a, (1, -1)).shape == (1, 105) + assert reshape(a, (1, 1, -1)).shape == (1, 1, 105) + assert reshape(a, (-1, 1, 1)).shape == (105, 1, 1) + + def test_transpose(self): + from numpypy import arange, array, transpose, ones + x = arange(4).reshape((2,2)) + assert (transpose(x) == array([[0, 2],[1, 3]])).all() + # Once axes argument is implemented, add more tests + raises(NotImplementedError, "transpose(x, axes=(1, 0, 2))") + # x = ones((1, 2, 3)) + # assert transpose(x, (1, 0, 2)).shape == (2, 1, 3) + diff --git a/pypy/module/test_lib_pypy/numpypy/core/test_numeric.py b/pypy/module/test_lib_pypy/numpypy/core/test_numeric.py new file mode 100644 --- /dev/null +++ b/pypy/module/test_lib_pypy/numpypy/core/test_numeric.py @@ -0,0 +1,144 @@ + +from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest + + +class AppTestBaseRepr(BaseNumpyAppTest): + def test_base3(self): + from numpypy import base_repr + assert base_repr(3**5, 3) == '100000' + + def test_positive(self): + from numpypy import base_repr + assert base_repr(12, 10) == '12' + assert base_repr(12, 10, 4) == '000012' + assert base_repr(12, 4) == '30' + assert base_repr(3731624803700888, 36) == '10QR0ROFCEW' + + def test_negative(self): + from numpypy import base_repr + assert base_repr(-12, 10) == '-12' + assert base_repr(-12, 10, 4) == '-000012' + assert base_repr(-12, 4) == '-30' + +class AppTestRepr(BaseNumpyAppTest): + def test_repr(self): + from numpypy import array + assert repr(array([1, 2, 3, 4])) == 'array([1, 2, 3, 4])' + + def test_repr_2(self): + from numpypy import array, zeros + int_size = array(5).dtype.itemsize + a = array(range(5), float) + assert repr(a) == "array([ 0., 1., 2., 3., 4.])" + a = array([], float) + assert repr(a) == "array([], dtype=float64)" + a = zeros(1001) + assert repr(a) == "array([ 0., 0., 0., ..., 0., 0., 0.])" + a = array(range(5), long) + if a.dtype.itemsize == int_size: + assert repr(a) == "array([0, 1, 2, 3, 4])" + else: + assert repr(a) == "array([0, 1, 2, 3, 4], dtype=int64)" + a = array(range(5), 'int32') + if a.dtype.itemsize == int_size: + assert repr(a) == "array([0, 1, 2, 3, 4])" + else: + assert repr(a) == "array([0, 1, 2, 3, 4], dtype=int32)" + a = array([], long) + assert repr(a) == "array([], dtype=int64)" + a = array([True, False, True, False], "?") + assert repr(a) == "array([ True, False, True, False], dtype=bool)" + a = zeros([]) + assert repr(a) == "array(0.0)" + a = array(0.2) + assert repr(a) == "array(0.2)" + a = array([2]) + assert repr(a) == "array([2])" + + def test_repr_multi(self): + from numpypy import arange, zeros, array + a = zeros((3, 4)) + assert repr(a) == '''array([[ 0., 0., 0., 0.], + [ 0., 0., 0., 0.], + [ 0., 0., 0., 0.]])''' + a = zeros((2, 3, 4)) + assert repr(a) == '''array([[[ 0., 0., 0., 0.], + [ 0., 0., 0., 0.], + [ 0., 0., 0., 0.]], + + [[ 0., 0., 0., 0.], + [ 0., 0., 0., 0.], + [ 0., 0., 0., 0.]]])''' + a = arange(1002).reshape((2, 501)) + assert repr(a) == '''array([[ 0, 1, 2, ..., 498, 499, 500], + [ 501, 502, 503, ..., 999, 1000, 1001]])''' + assert repr(a.T) == '''array([[ 0, 501], + [ 1, 502], + [ 2, 503], + ..., + [ 498, 999], + [ 499, 1000], + [ 500, 1001]])''' + a = arange(2).reshape((2,1)) + assert repr(a) == '''array([[0], + [1]])''' + + def test_repr_slice(self): + from numpypy import array, zeros + a = array(range(5), float) + b = a[1::2] + assert repr(b) == "array([ 1., 3.])" + a = zeros(2002) + b = a[::2] + assert repr(b) == "array([ 0., 0., 0., ..., 0., 0., 0.])" + a = array((range(5), range(5, 10)), dtype="int16") + b = a[1, 2:] + assert repr(b) == "array([7, 8, 9], dtype=int16)" + # an empty slice prints its shape + b = a[2:1, ] + assert repr(b) == "array([], shape=(0, 5), dtype=int16)" + + def test_str(self): + from numpypy import array, zeros + a = array(range(5), float) + assert str(a) == "[ 0. 1. 2. 3. 4.]" + assert str((2 * a)[:]) == "[ 0. 2. 4. 6. 8.]" + a = zeros(1001) + assert str(a) == "[ 0. 0. 0. ..., 0. 0. 0.]" + + a = array(range(5), dtype=long) + assert str(a) == "[0 1 2 3 4]" + a = array([True, False, True, False], dtype="?") + assert str(a) == "[ True False True False]" + + a = array(range(5), dtype="int8") + assert str(a) == "[0 1 2 3 4]" + + a = array(range(5), dtype="int16") + assert str(a) == "[0 1 2 3 4]" + + a = array((range(5), range(5, 10)), dtype="int16") + assert str(a) == "[[0 1 2 3 4]\n [5 6 7 8 9]]" + + a = array(3, dtype=int) + assert str(a) == "3" + + a = zeros((400, 400), dtype=int) + assert str(a) == '[[0 0 0 ..., 0 0 0]\n [0 0 0 ..., 0 0 0]\n [0 0 0 ..., 0 0 0]\n ..., \n [0 0 0 ..., 0 0 0]\n [0 0 0 ..., 0 0 0]\n [0 0 0 ..., 0 0 0]]' + a = zeros((2, 2, 2)) + r = str(a) + assert r == '[[[ 0. 0.]\n [ 0. 0.]]\n\n [[ 0. 0.]\n [ 0. 0.]]]' + + def test_str_slice(self): + from numpypy import array, zeros + a = array(range(5), float) + b = a[1::2] + assert str(b) == "[ 1. 3.]" + a = zeros(2002) + b = a[::2] + assert str(b) == "[ 0. 0. 0. ..., 0. 0. 0.]" + a = array((range(5), range(5, 10)), dtype="int16") + b = a[1, 2:] + assert str(b) == "[7 8 9]" + b = a[2:1, ] + assert str(b) == "[]" diff --git a/pypy/module/test_lib_pypy/test_datetime.py b/pypy/module/test_lib_pypy/test_datetime.py new file mode 100644 --- /dev/null +++ b/pypy/module/test_lib_pypy/test_datetime.py @@ -0,0 +1,24 @@ +"""Additional tests for datetime.""" + +import time +import datetime +import os + +def test_utcfromtimestamp(): + """Confirm that utcfromtimestamp and fromtimestamp give consistent results. + + Based on danchr's test script in https://bugs.pypy.org/issue986 + """ + try: + prev_tz = os.environ.get("TZ") + os.environ["TZ"] = "GMT" + for unused in xrange(100): + now = time.time() + delta = (datetime.datetime.utcfromtimestamp(now) - + datetime.datetime.fromtimestamp(now)) + assert delta.days * 86400 + delta.seconds == 0 + finally: + if prev_tz is None: + del os.environ["TZ"] + else: + os.environ["TZ"] = prev_tz diff --git a/pypy/module/test_lib_pypy/test_sqlite3.py b/pypy/module/test_lib_pypy/test_sqlite3.py new file mode 100644 --- /dev/null +++ b/pypy/module/test_lib_pypy/test_sqlite3.py @@ -0,0 +1,10 @@ +"""Tests for _sqlite3.py""" + +def test_list_ddl(): + """From issue996. Mostly just looking for lack of exceptions.""" + from sqlite3.dbapi2 import connect + connection = connect(':memory:') + cursor = connection.cursor() + cursor.execute('CREATE TABLE foo (bar INTEGER)') + result = list(cursor) + assert result == [] diff --git a/pypy/module/thread/ll_thread.py b/pypy/module/thread/ll_thread.py --- a/pypy/module/thread/ll_thread.py +++ b/pypy/module/thread/ll_thread.py @@ -4,7 +4,7 @@ import py from pypy.rlib import jit, rgc from pypy.rlib.debug import ll_assert -from pypy.rlib.objectmodel import we_are_translated +from pypy.rlib.objectmodel import we_are_translated, specialize from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.tool import rffi_platform from pypy.tool import autopath @@ -85,6 +85,7 @@ def allocate_lock(): return Lock(allocate_ll_lock()) + at specialize.arg(0) def ll_start_new_thread(func): ident = c_thread_start(func) if ident == -1: @@ -97,6 +98,7 @@ def get_ident(): return rffi.cast(lltype.Signed, c_thread_get_ident()) + at specialize.arg(0) def start_new_thread(x, y): """In RPython, no argument can be passed. You have to use global variables to pass information to the new thread. That's not very diff --git a/pypy/module/thread/test/test_ll_thread.py b/pypy/module/thread/test/test_ll_thread.py --- a/pypy/module/thread/test/test_ll_thread.py +++ b/pypy/module/thread/test/test_ll_thread.py @@ -34,72 +34,34 @@ use_threads = True def test_start_new_thread(self): - py.test.skip("xxx ideally, investigate why it fails randomly") - # xxx but in practice start_new_thread() is also tested by the - # next test, and it's a mess to test start_new_thread() without - # the proper GIL to protect the GC import time class State: pass state = State() - class Z: - def __init__(self, value): - self.value = value - def __del__(self): - state.freed_counter += 1 - - def bootstrap(): - state.my_thread_ident = state.z.ident = get_ident() - assert state.my_thread_ident == get_ident() - assert get_ident() == state.z.ident - state.seen_value = state.z.value - state.z = None - # I think that we would need here a memory barrier in order - # to make the test pass reliably. The issue is that the - # main thread may see 'state.done = 1' before seeing the - # effect of the other assignments done above. For now let's - # emulate the write barrier by doing a system call and - # waiting a bit... - time.sleep(0.012) - state.done = 1 - - def g(i): - state.z = Z(i) - return start_new_thread(bootstrap, ()) - g._dont_inline_ = True + def bootstrap1(): + state.my_thread_ident1 = get_ident() + def bootstrap2(): + state.my_thread_ident2 = get_ident() def f(): - main_ident = get_ident() - assert main_ident == get_ident() - state.freed_counter = 0 - for i in range(50): - state.done = 0 - state.seen_value = 0 - ident = g(i) - gc.collect() - willing_to_wait_more = 1000 - while not state.done: - willing_to_wait_more -= 1 - if not willing_to_wait_more: - raise Exception("thread didn't start?") - time.sleep(0.01) - assert state.my_thread_ident != main_ident - assert state.my_thread_ident == ident - assert state.seen_value == i - # try to force Boehm to do some freeing - for i in range(3): - gc.collect() - return state.freed_counter + state.my_thread_ident1 = get_ident() + state.my_thread_ident2 = get_ident() + start_new_thread(bootstrap1, ()) + start_new_thread(bootstrap2, ()) + willing_to_wait_more = 1000 + while (state.my_thread_ident1 == get_ident() or + state.my_thread_ident2 == get_ident()): + willing_to_wait_more -= 1 + if not willing_to_wait_more: + raise Exception("thread didn't start?") + time.sleep(0.01) + return 42 fn = self.getcompiled(f, []) - freed_counter = fn() - print freed_counter - if self.gcpolicy == 'boehm': - assert freed_counter > 0 - else: - assert freed_counter == 50 + res = fn() + assert res == 42 def test_gc_locking(self): import time @@ -202,6 +164,9 @@ def getcompiled(self, f, argtypes): return f + def test_start_new_thread(self): + py.test.skip("deadlocks occasionally -- why???") + class TestUsingBoehm(AbstractThreadTests): gcpolicy = 'boehm' diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py --- a/pypy/objspace/fake/objspace.py +++ b/pypy/objspace/fake/objspace.py @@ -150,6 +150,8 @@ self._see_getsetproperty(x) if isinstance(x, r_singlefloat): self._wrap_not_rpython(x) + if isinstance(x, list): + self._wrap_not_rpython(x) return w_some_obj() wrap._annspecialcase_ = "specialize:argtype(1)" diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py --- a/pypy/objspace/std/bytearrayobject.py +++ b/pypy/objspace/std/bytearrayobject.py @@ -624,8 +624,8 @@ if step == 1: assert start >= 0 - assert slicelength >= 0 - del items[start:start+slicelength] + if slicelength > 0: + del items[start:start+slicelength] else: n = len(items) i = start @@ -662,10 +662,11 @@ while i >= lim: items[i] = items[i-delta] i -= 1 - elif start >= 0: + elif delta == 0: + pass + else: + assert start >= 0 # start<0 is only possible with slicelength==0 del items[start:start+delta] - else: - assert delta==0 # start<0 is only possible with slicelength==0 elif len2 != slicelength: # No resize for extended slices raise operationerrfmt(space.w_ValueError, "attempt to " "assign sequence of size %d to extended slice of size %d", diff --git a/pypy/objspace/std/complexobject.py b/pypy/objspace/std/complexobject.py --- a/pypy/objspace/std/complexobject.py +++ b/pypy/objspace/std/complexobject.py @@ -8,6 +8,7 @@ from pypy.rlib.rbigint import rbigint from pypy.rlib.rfloat import ( formatd, DTSF_STR_PRECISION, isinf, isnan, copysign) +from pypy.rlib import jit import math @@ -129,10 +130,10 @@ ir = len * math.sin(phase) return W_ComplexObject(rr, ir) - def pow_int(self, n): - if n > 100 or n < -100: - return self.pow(W_ComplexObject(1.0 * n, 0.0)) - elif n > 0: + def pow_small_int(self, n): + if n >= 0: + if jit.isconstant(n) and n == 2: + return self.mul(self) return self.pow_positive_int(n) else: return w_one.div(self.pow_positive_int(-n)) @@ -217,10 +218,10 @@ def pow__Complex_Complex_ANY(space, w_complex, w_exponent, thirdArg): if not space.is_w(thirdArg, space.w_None): raise OperationError(space.w_ValueError, space.wrap('complex modulo')) - int_exponent = int(w_exponent.realval) try: - if w_exponent.imagval == 0.0 and w_exponent.realval == int_exponent: - w_p = w_complex.pow_int(int_exponent) + r = w_exponent.realval + if w_exponent.imagval == 0.0 and -100.0 <= r <= 100.0 and r == int(r): + w_p = w_complex.pow_small_int(int(r)) else: w_p = w_complex.pow(w_exponent) except ZeroDivisionError: diff --git a/pypy/objspace/std/floatobject.py b/pypy/objspace/std/floatobject.py --- a/pypy/objspace/std/floatobject.py +++ b/pypy/objspace/std/floatobject.py @@ -443,6 +443,8 @@ y = w_float2.floatval # Sort out special cases here instead of relying on pow() + if y == 2.0: # special case for performance: + return W_FloatObject(x * x) # x * x is always correct if y == 0.0: # x**0 is 1, even 0**0 return W_FloatObject(1.0) diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py --- a/pypy/objspace/std/listobject.py +++ b/pypy/objspace/std/listobject.py @@ -110,7 +110,10 @@ return list(items) def switch_to_object_strategy(self): - list_w = self.getitems() + if self.strategy is self.space.fromcache(EmptyListStrategy): + list_w = [] + else: + list_w = self.getitems() self.strategy = self.space.fromcache(ObjectListStrategy) # XXX this is quite indirect self.init_from_list_w(list_w) @@ -344,8 +347,6 @@ def __init__(self, space): ListStrategy.__init__(self, space) - # cache an empty list that is used whenever getitems is called (i.e. sorting) - self.cached_emptylist_w = [] def init_from_list_w(self, w_list, list_w): assert len(list_w) == 0 @@ -373,10 +374,10 @@ def getslice(self, w_list, start, stop, step, length): # will never be called because the empty list case is already caught in # getslice__List_ANY_ANY and getitem__List_Slice - return W_ListObject(self.space, self.cached_emptylist_w) + return W_ListObject(self.space, []) def getitems(self, w_list): - return self.cached_emptylist_w + return [] def getitems_copy(self, w_list): return [] @@ -804,10 +805,11 @@ while i >= lim: items[i] = items[i-delta] i -= 1 - elif start >= 0: + elif delta == 0: + pass + else: + assert start >= 0 # start<0 is only possible with slicelength==0 del items[start:start+delta] - else: - assert delta==0 # start<0 is only possible with slicelength==0 elif len2 != slicelength: # No resize for extended slices raise operationerrfmt(self.space.w_ValueError, "attempt to " "assign sequence of size %d to extended slice of size %d", @@ -851,8 +853,8 @@ if step == 1: assert start >= 0 - assert slicelength >= 0 - del items[start:start+slicelength] + if slicelength > 0: + del items[start:start+slicelength] else: n = len(items) i = start diff --git a/pypy/objspace/std/test/test_bytearrayobject.py b/pypy/objspace/std/test/test_bytearrayobject.py --- a/pypy/objspace/std/test/test_bytearrayobject.py +++ b/pypy/objspace/std/test/test_bytearrayobject.py @@ -1,5 +1,9 @@ +from pypy import conftest class AppTestBytesArray: + def setup_class(cls): + cls.w_runappdirect = cls.space.wrap(conftest.option.runappdirect) + def test_basics(self): b = bytearray() assert type(b) is bytearray @@ -439,3 +443,15 @@ def test_reduce(self): assert bytearray('caf\xe9').__reduce__() == ( bytearray, (u'caf\xe9', 'latin-1'), None) + + def test_setitem_slice_performance(self): + # because of a complexity bug, this used to take forever on a + # translated pypy. On CPython2.6 -A, it takes around 8 seconds. + if self.runappdirect: + count = 16*1024*1024 + else: + count = 1024 + b = bytearray(count) + for i in range(count): + b[i:i+1] = 'y' + assert str(b) == 'y' * count diff --git a/pypy/objspace/std/test/test_complexobject.py b/pypy/objspace/std/test/test_complexobject.py --- a/pypy/objspace/std/test/test_complexobject.py +++ b/pypy/objspace/std/test/test_complexobject.py @@ -71,7 +71,7 @@ assert _powu((0.0,1.0),2) == (-1.0,0.0) def _powi((r1, i1), n): - w_res = W_ComplexObject(r1, i1).pow_int(n) + w_res = W_ComplexObject(r1, i1).pow_small_int(n) return w_res.realval, w_res.imagval assert _powi((0.0,2.0),0) == (1.0,0.0) assert _powi((0.0,0.0),2) == (0.0,0.0) @@ -213,6 +213,7 @@ assert a ** 105 == a ** 105 assert a ** -105 == a ** -105 assert a ** -30 == a ** -30 + assert a ** 2 == a * a assert 0.0j ** 0 == 1 diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py --- a/pypy/objspace/std/test/test_listobject.py +++ b/pypy/objspace/std/test/test_listobject.py @@ -397,6 +397,7 @@ on_cpython = (option.runappdirect and not hasattr(sys, 'pypy_translation_info')) cls.w_on_cpython = cls.space.wrap(on_cpython) + cls.w_runappdirect = cls.space.wrap(option.runappdirect) def test_getstrategyfromlist_w(self): l0 = ["a", "2", "a", True] @@ -898,6 +899,18 @@ l[::-1] = l assert l == [6,5,4,3,2,1] + def test_setitem_slice_performance(self): + # because of a complexity bug, this used to take forever on a + # translated pypy. On CPython2.6 -A, it takes around 5 seconds. + if self.runappdirect: + count = 16*1024*1024 + else: + count = 1024 + b = [None] * count + for i in range(count): + b[i:i+1] = ['y'] + assert b == ['y'] * count + def test_recursive_repr(self): l = [] assert repr(l) == '[]' @@ -1238,6 +1251,20 @@ l.reverse() assert l == [2,1,0] +class AppTestWithoutStrategies(object): + + def setup_class(cls): + cls.space = gettestobjspace(**{"objspace.std.withliststrategies" : + False}) + + def test_no_shared_empty_list(self): + l = [] + copy = l[:] + copy.append({}) + assert copy == [{}] + + notshared = l[:] + assert notshared == [] class AppTestListFastSubscr: diff --git a/pypy/rlib/jit_hooks.py b/pypy/rlib/jit_hooks.py --- a/pypy/rlib/jit_hooks.py +++ b/pypy/rlib/jit_hooks.py @@ -52,7 +52,10 @@ from pypy.jit.metainterp.history import ResOperation args = [_cast_to_box(llargs[i]) for i in range(len(llargs))] - res = _cast_to_box(llres) + if llres: + res = _cast_to_box(llres) + else: + res = None return _cast_to_gcref(ResOperation(no, args, res)) @register_helper(annmodel.SomePtr(llmemory.GCREF)) diff --git a/pypy/rlib/objectmodel.py b/pypy/rlib/objectmodel.py --- a/pypy/rlib/objectmodel.py +++ b/pypy/rlib/objectmodel.py @@ -130,19 +130,29 @@ if self is other: return 0 else: - raise TypeError("Symbolics can not be compared!") + raise TypeError("Symbolics cannot be compared! (%r, %r)" + % (self, other)) def __hash__(self): - raise TypeError("Symbolics are not hashable!") + raise TypeError("Symbolics are not hashable! %r" % (self,)) def __nonzero__(self): - raise TypeError("Symbolics are not comparable") + raise TypeError("Symbolics are not comparable! %r" % (self,)) class ComputedIntSymbolic(Symbolic): def __init__(self, compute_fn): self.compute_fn = compute_fn + def __repr__(self): + # repr(self.compute_fn) can arrive back here in an + # infinite recursion + try: + name = self.compute_fn.__name__ + except (AttributeError, TypeError): + name = hex(id(self.compute_fn)) + return '%s(%r)' % (self.__class__.__name__, name) + def annotation(self): from pypy.annotation import model return model.SomeInteger() @@ -157,6 +167,9 @@ self.expr = expr self.default = default + def __repr__(self): + return '%s(%r)' % (self.__class__.__name__, self.expr) + def annotation(self): from pypy.annotation import model return model.SomeInteger() diff --git a/pypy/rlib/rdynload.py b/pypy/rlib/rdynload.py --- a/pypy/rlib/rdynload.py +++ b/pypy/rlib/rdynload.py @@ -14,7 +14,7 @@ _MINGW = platform.name == "mingw32" _WIN32 = _MSVC or _MINGW _MAC_OS = platform.name == "darwin" -_FREEBSD = platform.name == "freebsd" +_FREEBSD = sys.platform.startswith("freebsd") if _WIN32: from pypy.rlib import rwin32 diff --git a/pypy/rlib/rsocket.py b/pypy/rlib/rsocket.py --- a/pypy/rlib/rsocket.py +++ b/pypy/rlib/rsocket.py @@ -310,10 +310,7 @@ def from_object(space, w_address): # Parse an app-level object representing an AF_INET address - try: - w_host, w_port = space.unpackiterable(w_address, 2) - except ValueError: - raise TypeError("AF_INET address must be a tuple of length 2") + w_host, w_port = space.unpackiterable(w_address, 2) host = space.str_w(w_host) port = space.int_w(w_port) port = Address.make_ushort_port(space, port) @@ -544,10 +541,7 @@ space.wrap(self.get_groups())]) def from_object(space, w_address): - try: - w_pid, w_groups = space.unpackiterable(w_address, 2) - except ValueError: - raise TypeError("AF_NETLINK address must be a tuple of length 2") + w_pid, w_groups = space.unpackiterable(w_address, 2) return NETLINKAddress(space.uint_w(w_pid), space.uint_w(w_groups)) from_object = staticmethod(from_object) diff --git a/pypy/rlib/rstacklet.py b/pypy/rlib/rstacklet.py --- a/pypy/rlib/rstacklet.py +++ b/pypy/rlib/rstacklet.py @@ -1,3 +1,4 @@ +import sys from pypy.rlib import _rffi_stacklet as _c from pypy.rlib import jit from pypy.rlib.objectmodel import we_are_translated @@ -72,6 +73,11 @@ if translated: assert config is not None, ("you have to pass a valid config, " "e.g. from 'driver.config'") + elif '__pypy__' in sys.builtin_module_names: + import py + py.test.skip("cannot run the stacklet tests on top of pypy: " + "calling directly the C function stacklet_switch() " + "will crash, depending on details of your config") if config is not None: assert config.translation.continuation, ( "stacklet: you have to translate with --continuation") diff --git a/pypy/rlib/rweakref.py b/pypy/rlib/rweakref.py --- a/pypy/rlib/rweakref.py +++ b/pypy/rlib/rweakref.py @@ -1,11 +1,14 @@ """ -Weakref support in RPython. Supports ref() without callbacks, +Weakref support in RPython. Basic regular weakrefs without callbacks +are supported. This file contains the following additions: a form of WeakKeyDictionary, and a limited version of WeakValueDictionary. LLType only for now! """ import weakref +ref = weakref.ref # basic regular weakrefs are supported in RPython + class RWeakValueDictionary(object): """A dictionary containing weak values.""" diff --git a/pypy/rpython/lltypesystem/ll2ctypes.py b/pypy/rpython/lltypesystem/ll2ctypes.py --- a/pypy/rpython/lltypesystem/ll2ctypes.py +++ b/pypy/rpython/lltypesystem/ll2ctypes.py @@ -1390,7 +1390,8 @@ def _where_is_errno(): return standard_c_lib.__errno_location() - elif sys.platform in ('darwin', 'freebsd7', 'freebsd8', 'freebsd9'): + elif any(plat in sys.platform + for plat in ('darwin', 'freebsd7', 'freebsd8', 'freebsd9')): standard_c_lib.__error.restype = ctypes.POINTER(ctypes.c_int) def _where_is_errno(): return standard_c_lib.__error() diff --git a/pypy/rpython/lltypesystem/llgroup.py b/pypy/rpython/lltypesystem/llgroup.py --- a/pypy/rpython/lltypesystem/llgroup.py +++ b/pypy/rpython/lltypesystem/llgroup.py @@ -76,6 +76,10 @@ self.index = memberindex self.member = grp.members[memberindex]._as_ptr() + def __repr__(self): + return '%s(%s, %s)' % (self.__class__.__name__, + self.grpptr, self.index) + def __nonzero__(self): return True diff --git a/pypy/rpython/lltypesystem/llmemory.py b/pypy/rpython/lltypesystem/llmemory.py --- a/pypy/rpython/lltypesystem/llmemory.py +++ b/pypy/rpython/lltypesystem/llmemory.py @@ -32,7 +32,8 @@ self.known_nonneg()): return True else: - raise TypeError("Symbolics can not be compared!") + raise TypeError("Symbolics cannot be compared! (%r, %r)" + % (self, other)) def __lt__(self, other): return not self.__ge__(other) diff --git a/pypy/rpython/lltypesystem/rdict.py b/pypy/rpython/lltypesystem/rdict.py --- a/pypy/rpython/lltypesystem/rdict.py +++ b/pypy/rpython/lltypesystem/rdict.py @@ -323,6 +323,17 @@ hop.exception_is_here() return hop.gendirectcall(ll_popitem, cTUPLE, v_dict) + def rtype_method_pop(self, hop): + if hop.nb_args == 2: + v_args = hop.inputargs(self, self.key_repr) + target = ll_pop + elif hop.nb_args == 3: + v_args = hop.inputargs(self, self.key_repr, self.value_repr) + target = ll_pop_default + hop.exception_is_here() + v_res = hop.gendirectcall(target, *v_args) + return self.recast_value(hop.llops, v_res) + class __extend__(pairtype(DictRepr, rmodel.Repr)): def rtype_getitem((r_dict, r_key), hop): @@ -874,3 +885,18 @@ r.item1 = recast(ELEM.TO.item1, entry.value) _ll_dict_del(dic, i) return r + +def ll_pop(dic, key): + i = ll_dict_lookup(dic, key, dic.keyhash(key)) + if not i & HIGHEST_BIT: + value = ll_get_value(dic, i) + _ll_dict_del(dic, i) + return value + else: + raise KeyError + +def ll_pop_default(dic, key, dfl): + try: + return ll_pop(dic, key) + except KeyError: + return dfl diff --git a/pypy/rpython/lltypesystem/rffi.py b/pypy/rpython/lltypesystem/rffi.py --- a/pypy/rpython/lltypesystem/rffi.py +++ b/pypy/rpython/lltypesystem/rffi.py @@ -27,6 +27,10 @@ self.c_name = c_name self.TP = TP + def __repr__(self): + return '%s(%r, %s)' % (self.__class__.__name__, + self.c_name, self.TP) + def annotation(self): return lltype_to_annotation(self.TP) diff --git a/pypy/rpython/lltypesystem/rtuple.py b/pypy/rpython/lltypesystem/rtuple.py --- a/pypy/rpython/lltypesystem/rtuple.py +++ b/pypy/rpython/lltypesystem/rtuple.py @@ -27,6 +27,10 @@ def newtuple(cls, llops, r_tuple, items_v): # items_v should have the lowleveltype of the internal reprs + assert len(r_tuple.items_r) == len(items_v) + for r_item, v_item in zip(r_tuple.items_r, items_v): + assert r_item.lowleveltype == v_item.concretetype + # if len(r_tuple.items_r) == 0: return inputconst(Void, ()) # a Void empty tuple c1 = inputconst(Void, r_tuple.lowleveltype.TO) diff --git a/pypy/rpython/memory/gc/env.py b/pypy/rpython/memory/gc/env.py --- a/pypy/rpython/memory/gc/env.py +++ b/pypy/rpython/memory/gc/env.py @@ -116,7 +116,7 @@ def get_total_memory(): return get_total_memory_darwin(get_darwin_sysctl_signed('hw.memsize')) -elif sys.platform.startswith('freebsd'): +elif 'freebsd' in sys.platform: def get_total_memory(): return get_total_memory_darwin(get_darwin_sysctl_signed('hw.usermem')) diff --git a/pypy/rpython/module/ll_time.py b/pypy/rpython/module/ll_time.py --- a/pypy/rpython/module/ll_time.py +++ b/pypy/rpython/module/ll_time.py @@ -41,7 +41,7 @@ RUSAGE = platform.Struct('struct rusage', [('ru_utime', TIMEVAL), ('ru_stime', TIMEVAL)]) -if "freebsd" in sys.platform: +if sys.platform.startswith('freebsd'): libraries = ['compat'] else: libraries = [] diff --git a/pypy/rpython/ootypesystem/rdict.py b/pypy/rpython/ootypesystem/rdict.py --- a/pypy/rpython/ootypesystem/rdict.py +++ b/pypy/rpython/ootypesystem/rdict.py @@ -160,6 +160,16 @@ hop.exception_is_here() return hop.gendirectcall(ll_popitem, cTUPLE, v_dict) + def rtype_method_pop(self, hop): + if hop.nb_args == 2: + v_args = hop.inputargs(self, self.key_repr) + target = ll_pop + elif hop.nb_args == 3: + v_args = hop.inputargs(self, self.key_repr, self.value_repr) + target = ll_pop_default + hop.exception_is_here() + return hop.gendirectcall(target, *v_args) + def __get_func(self, interp, r_func, fn, TYPE): if isinstance(r_func, MethodOfFrozenPBCRepr): obj = r_func.r_im_self.convert_const(fn.im_self) @@ -370,6 +380,20 @@ return res raise KeyError +def ll_pop(d, key): + if d.ll_contains(key): + value = d.ll_get(key) + d.ll_remove(key) + return value + else: + raise KeyError + +def ll_pop_default(d, key, dfl): + try: + return ll_pop(d, key) + except KeyError: + return dfl + # ____________________________________________________________ # # Iteration. diff --git a/pypy/rpython/rrange.py b/pypy/rpython/rrange.py --- a/pypy/rpython/rrange.py +++ b/pypy/rpython/rrange.py @@ -204,7 +204,10 @@ v_index = hop.gendirectcall(self.ll_getnextindex, v_enumerate) hop2 = hop.copy() hop2.args_r = [self.r_baseiter] + r_item_src = self.r_baseiter.r_list.external_item_repr + r_item_dst = hop.r_result.items_r[1] v_item = self.r_baseiter.rtype_next(hop2) + v_item = hop.llops.convertvar(v_item, r_item_src, r_item_dst) return hop.r_result.newtuple(hop.llops, hop.r_result, [v_index, v_item]) diff --git a/pypy/rpython/test/test_rdict.py b/pypy/rpython/test/test_rdict.py --- a/pypy/rpython/test/test_rdict.py +++ b/pypy/rpython/test/test_rdict.py @@ -25,37 +25,37 @@ class BaseTestRdict(BaseRtypingTest): def test_dict_creation(self): - def createdict(i): + def createdict(i): d = {'hello' : i} return d['hello'] res = self.interpret(createdict, [42]) assert res == 42 - def test_dict_getitem_setitem(self): - def func(i): + def test_dict_getitem_setitem(self): + def func(i): d = {'hello' : i} d['world'] = i + 1 - return d['hello'] * d['world'] + return d['hello'] * d['world'] res = self.interpret(func, [6]) assert res == 42 - def test_dict_getitem_keyerror(self): - def func(i): + def test_dict_getitem_keyerror(self): + def func(i): d = {'hello' : i} try: return d['world'] except KeyError: - return 0 + return 0 res = self.interpret(func, [6]) assert res == 0 def test_dict_del_simple(self): - def func(i): + def func(i): d = {'hello' : i} d['world'] = i + 1 del d['hello'] - return len(d) + return len(d) res = self.interpret(func, [6]) assert res == 1 @@ -71,7 +71,7 @@ assert res == True def test_empty_strings(self): - def func(i): + def func(i): d = {'' : i} del d[''] try: @@ -83,7 +83,7 @@ res = self.interpret(func, [6]) assert res == 1 - def func(i): + def func(i): d = {'' : i} del d[''] d[''] = i + 1 @@ -146,8 +146,8 @@ d1 = {} d1['hello'] = i + 1 d2 = {} - d2['world'] = d1 - return d2['world']['hello'] + d2['world'] = d1 + return d2['world']['hello'] res = self.interpret(func, [5]) assert res == 6 @@ -297,7 +297,7 @@ a = 0 for k, v in items: b += isinstance(k, B) - a += isinstance(v, A) + a += isinstance(v, A) return 3*b+a res = self.interpret(func, []) assert res == 8 @@ -316,7 +316,7 @@ a = 0 for k, v in dic.iteritems(): b += isinstance(k, B) - a += isinstance(v, A) + a += isinstance(v, A) return 3*b+a res = self.interpret(func, []) assert res == 8 @@ -342,11 +342,11 @@ def test_dict_contains_with_constant_dict(self): dic = {'4':1000, ' 8':200} def func(i): - return chr(i) in dic - res = self.interpret(func, [ord('4')]) + return chr(i) in dic + res = self.interpret(func, [ord('4')]) assert res is True - res = self.interpret(func, [1]) - assert res is False + res = self.interpret(func, [1]) + assert res is False def test_dict_or_none(self): class A: @@ -413,7 +413,7 @@ return g(get) res = self.interpret(f, []) - assert res == 2 + assert res == 2 def test_specific_obscure_bug(self): class A: pass @@ -622,6 +622,42 @@ res = self.interpret(func, []) assert res in [5263, 6352] + def test_dict_pop(self): + def f(n, default): + d = {} + d[2] = 3 + d[4] = 5 + if default == -1: + try: + x = d.pop(n) + except KeyError: + x = -1 + else: + x = d.pop(n, default) + return x * 10 + len(d) + res = self.interpret(f, [2, -1]) + assert res == 31 + res = self.interpret(f, [3, -1]) + assert res == -8 + res = self.interpret(f, [2, 5]) + assert res == 31 + + def test_dict_pop_instance(self): + class A(object): + pass + def f(n): + d = {} + d[2] = A() + x = d.pop(n, None) + if x is None: + return 12 + else: + return 15 + res = self.interpret(f, [2]) + assert res == 15 + res = self.interpret(f, [700]) + assert res == 12 + class TestLLtype(BaseTestRdict, LLRtypeMixin): def test_dict_but_not_with_char_keys(self): def func(i): @@ -633,19 +669,19 @@ res = self.interpret(func, [6]) assert res == 0 - def test_deleted_entry_reusage_with_colliding_hashes(self): - def lowlevelhash(value): + def test_deleted_entry_reusage_with_colliding_hashes(self): + def lowlevelhash(value): p = rstr.mallocstr(len(value)) for i in range(len(value)): p.chars[i] = value[i] - return rstr.LLHelpers.ll_strhash(p) + return rstr.LLHelpers.ll_strhash(p) - def func(c1, c2): - c1 = chr(c1) - c2 = chr(c2) + def func(c1, c2): + c1 = chr(c1) + c2 = chr(c2) d = {} d[c1] = 1 - d[c2] = 2 + d[c2] = 2 del d[c1] return d[c2] @@ -653,7 +689,7 @@ base = rdict.DICT_INITSIZE for y in range(0, 256): y = chr(y) - y_hash = lowlevelhash(y) % base + y_hash = lowlevelhash(y) % base char_by_hash.setdefault(y_hash, []).append(y) x, y = char_by_hash[0][:2] # find a collision @@ -661,18 +697,18 @@ res = self.interpret(func, [ord(x), ord(y)]) assert res == 2 - def func2(c1, c2): - c1 = chr(c1) - c2 = chr(c2) + def func2(c1, c2): + c1 = chr(c1) + c2 = chr(c2) d = {} d[c1] = 1 - d[c2] = 2 + d[c2] = 2 del d[c1] d[c1] = 3 - return d + return d res = self.interpret(func2, [ord(x), ord(y)]) - for i in range(len(res.entries)): + for i in range(len(res.entries)): assert not (res.entries.everused(i) and not res.entries.valid(i)) def func3(c0, c1, c2, c3, c4, c5, c6, c7): @@ -687,9 +723,9 @@ c7 = chr(c7) ; d[c7] = 1; del d[c7] return d - if rdict.DICT_INITSIZE != 8: + if rdict.DICT_INITSIZE != 8: py.test.skip("make dict tests more indepdent from initsize") - res = self.interpret(func3, [ord(char_by_hash[i][0]) + res = self.interpret(func3, [ord(char_by_hash[i][0]) for i in range(rdict.DICT_INITSIZE)]) count_frees = 0 for i in range(len(res.entries)): @@ -707,9 +743,9 @@ del d[chr(ord('a') + i)] return d res = self.interpret(func, [0]) - assert len(res.entries) > rdict.DICT_INITSIZE + assert len(res.entries) > rdict.DICT_INITSIZE res = self.interpret(func, [1]) - assert len(res.entries) == rdict.DICT_INITSIZE + assert len(res.entries) == rdict.DICT_INITSIZE def test_dict_valid_resize(self): # see if we find our keys after resize @@ -844,7 +880,7 @@ def test_prebuilt_list_of_addresses(self): from pypy.rpython.lltypesystem import llmemory - + TP = lltype.Struct('x', ('y', lltype.Signed)) a = lltype.malloc(TP, flavor='raw', immortal=True) b = lltype.malloc(TP, flavor='raw', immortal=True) @@ -858,7 +894,7 @@ d = {a_a: 3, a_b: 4, a_c: 5} d[a0] = 8 - + def func(i): if i == 0: ptr = a @@ -888,7 +924,7 @@ return a == b def rhash(a): return 3 - + def func(i): d = r_dict(eq, rhash, force_non_null=True) if not i: diff --git a/pypy/rpython/test/test_rrange.py b/pypy/rpython/test/test_rrange.py --- a/pypy/rpython/test/test_rrange.py +++ b/pypy/rpython/test/test_rrange.py @@ -169,6 +169,22 @@ res = self.interpret(fn, [2]) assert res == 789 + def test_enumerate_instances(self): + class A: + pass + def fn(n): + a = A() + b = A() + a.k = 10 + b.k = 20 + for i, x in enumerate([a, b]): + if i == n: + return x.k + return 5 + res = self.interpret(fn, [1]) + assert res == 20 + + class TestLLtype(BaseTestRrange, LLRtypeMixin): from pypy.rpython.lltypesystem import rrange diff --git a/pypy/tool/jitlogparser/parser.py b/pypy/tool/jitlogparser/parser.py --- a/pypy/tool/jitlogparser/parser.py +++ b/pypy/tool/jitlogparser/parser.py @@ -140,13 +140,15 @@ bytecode_name = None is_bytecode = True inline_level = None + has_dmp = False def parse_code_data(self, arg): m = re.search('<code object ([<>\w]+)[\.,] file \'(.+?)\'[\.,] line (\d+)> #(\d+) (\w+)', arg) if m is None: # a non-code loop, like StrLiteralSearch or something - self.bytecode_name = arg + if arg: + self.bytecode_name = arg else: self.name, self.filename, lineno, bytecode_no, self.bytecode_name = m.groups() self.startlineno = int(lineno) @@ -218,7 +220,7 @@ self.inputargs = inputargs self.chunks = chunks for chunk in self.chunks: - if chunk.filename is not None: + if chunk.bytecode_name is not None: self.startlineno = chunk.startlineno self.filename = chunk.filename self.name = chunk.name diff --git a/pypy/tool/jitlogparser/test/test_parser.py b/pypy/tool/jitlogparser/test/test_parser.py --- a/pypy/tool/jitlogparser/test/test_parser.py +++ b/pypy/tool/jitlogparser/test/test_parser.py @@ -283,3 +283,13 @@ assert loops[-1].count == 1234 assert loops[1].count == 123 assert loops[2].count == 12 + +def test_parse_nonpython(): + loop = parse(""" + [] + debug_merge_point(0, 'random') + debug_merge_point(0, '<code object f. file 'x.py'. line 2> #15 COMPARE_OP') + """) + f = Function.from_operations(loop.operations, LoopStorage()) + assert f.chunks[-1].filename == 'x.py' + assert f.filename is None diff --git a/pypy/tool/test/test_version.py b/pypy/tool/test/test_version.py --- a/pypy/tool/test/test_version.py +++ b/pypy/tool/test/test_version.py @@ -1,6 +1,22 @@ import os, sys import py -from pypy.tool.version import get_repo_version_info +from pypy.tool.version import get_repo_version_info, _get_hg_archive_version + +def test_hg_archival_version(tmpdir): + def version_for(name, **kw): + path = tmpdir.join(name) + path.write('\n'.join('%s: %s' % x for x in kw.items())) + return _get_hg_archive_version(str(path)) + + assert version_for('release', + tag='release-123', + node='000', + ) == ('PyPy', 'release-123', '000') + assert version_for('somebranch', + node='000', + branch='something', + ) == ('PyPy', 'something', '000') + def test_get_repo_version_info(): assert get_repo_version_info(None) diff --git a/pypy/tool/version.py b/pypy/tool/version.py --- a/pypy/tool/version.py +++ b/pypy/tool/version.py @@ -3,111 +3,139 @@ from subprocess import Popen, PIPE import pypy pypydir = os.path.dirname(os.path.abspath(pypy.__file__)) +pypyroot = os.path.dirname(pypydir) +default_retval = 'PyPy', '?', '?' + +def maywarn(err, repo_type='Mercurial'): + if not err: + return + + from pypy.tool.ansi_print import ansi_log + log = py.log.Producer("version") + py.log.setconsumer("version", ansi_log) + log.WARNING('Errors getting %s information: %s' % (repo_type, err)) def get_repo_version_info(hgexe=None): '''Obtain version information by invoking the 'hg' or 'git' commands.''' - # TODO: support extracting from .hg_archival.txt - - default_retval = 'PyPy', '?', '?' - pypyroot = os.path.abspath(os.path.join(pypydir, '..')) - - def maywarn(err, repo_type='Mercurial'): - if not err: - return - - from pypy.tool.ansi_print import ansi_log - log = py.log.Producer("version") - py.log.setconsumer("version", ansi_log) - log.WARNING('Errors getting %s information: %s' % (repo_type, err)) # Try to see if we can get info from Git if hgexe is not specified. if not hgexe: if os.path.isdir(os.path.join(pypyroot, '.git')): - gitexe = py.path.local.sysfind('git') - if gitexe: - try: - p = Popen( - [str(gitexe), 'rev-parse', 'HEAD'], - stdout=PIPE, stderr=PIPE, cwd=pypyroot - ) - except OSError, e: - maywarn(e, 'Git') - return default_retval - if p.wait() != 0: - maywarn(p.stderr.read(), 'Git') - return default_retval - revision_id = p.stdout.read().strip()[:12] - p = Popen( - [str(gitexe), 'describe', '--tags', '--exact-match'], - stdout=PIPE, stderr=PIPE, cwd=pypyroot - ) - if p.wait() != 0: - p = Popen( - [str(gitexe), 'branch'], stdout=PIPE, stderr=PIPE, - cwd=pypyroot - ) - if p.wait() != 0: - maywarn(p.stderr.read(), 'Git') - return 'PyPy', '?', revision_id - branch = '?' - for line in p.stdout.read().strip().split('\n'): - if line.startswith('* '): - branch = line[1:].strip() - if branch == '(no branch)': - branch = '?' - break - return 'PyPy', branch, revision_id - return 'PyPy', p.stdout.read().strip(), revision_id + return _get_git_version() # Fallback to trying Mercurial. if hgexe is None: hgexe = py.path.local.sysfind('hg') - if not os.path.isdir(os.path.join(pypyroot, '.hg')): + if os.path.isfile(os.path.join(pypyroot, '.hg_archival.txt')): + return _get_hg_archive_version(os.path.join(pypyroot, '.hg_archival.txt')) + elif not os.path.isdir(os.path.join(pypyroot, '.hg')): maywarn('Not running from a Mercurial repository!') return default_retval elif not hgexe: maywarn('Cannot find Mercurial command!') return default_retval else: - env = dict(os.environ) - # get Mercurial into scripting mode - env['HGPLAIN'] = '1' - # disable user configuration, extensions, etc. - env['HGRCPATH'] = os.devnull + return _get_hg_version(hgexe) - try: - p = Popen([str(hgexe), 'version', '-q'], - stdout=PIPE, stderr=PIPE, env=env) - except OSError, e: - maywarn(e) - return default_retval - if not p.stdout.read().startswith('Mercurial Distributed SCM'): - maywarn('command does not identify itself as Mercurial') - return default_retval +def _get_hg_version(hgexe): + env = dict(os.environ) + # get Mercurial into scripting mode + env['HGPLAIN'] = '1' + # disable user configuration, extensions, etc. + env['HGRCPATH'] = os.devnull - p = Popen([str(hgexe), 'id', '-i', pypyroot], + try: + p = Popen([str(hgexe), 'version', '-q'], stdout=PIPE, stderr=PIPE, env=env) - hgid = p.stdout.read().strip() + except OSError, e: + maywarn(e) + return default_retval + + if not p.stdout.read().startswith('Mercurial Distributed SCM'): + maywarn('command does not identify itself as Mercurial') + return default_retval + + p = Popen([str(hgexe), 'id', '-i', pypyroot], + stdout=PIPE, stderr=PIPE, env=env) + hgid = p.stdout.read().strip() + maywarn(p.stderr.read()) + if p.wait() != 0: + hgid = '?' + + p = Popen([str(hgexe), 'id', '-t', pypyroot], + stdout=PIPE, stderr=PIPE, env=env) + hgtags = [t for t in p.stdout.read().strip().split() if t != 'tip'] + maywarn(p.stderr.read()) + if p.wait() != 0: + hgtags = ['?'] + + if hgtags: + return 'PyPy', hgtags[0], hgid + else: + # use the branch instead + p = Popen([str(hgexe), 'id', '-b', pypyroot], + stdout=PIPE, stderr=PIPE, env=env) + hgbranch = p.stdout.read().strip() maywarn(p.stderr.read()) + + return 'PyPy', hgbranch, hgid + + +def _get_hg_archive_version(path): + fp = open(path) + try: + data = dict(x.split(': ', 1) for x in fp.read().splitlines()) + finally: + fp.close() + if 'tag' in data: + return 'PyPy', data['tag'], data['node'] + else: + return 'PyPy', data['branch'], data['node'] + + +def _get_git_version(): + #XXX: this function is a untested hack, + # so the git mirror tav made will work + gitexe = py.path.local.sysfind('git') + if not gitexe: + return default_retval + + try: + p = Popen( + [str(gitexe), 'rev-parse', 'HEAD'], + stdout=PIPE, stderr=PIPE, cwd=pypyroot + ) + except OSError, e: + maywarn(e, 'Git') + return default_retval + if p.wait() != 0: + maywarn(p.stderr.read(), 'Git') + return default_retval + revision_id = p.stdout.read().strip()[:12] + p = Popen( + [str(gitexe), 'describe', '--tags', '--exact-match'], + stdout=PIPE, stderr=PIPE, cwd=pypyroot + ) + if p.wait() != 0: + p = Popen( + [str(gitexe), 'branch'], stdout=PIPE, stderr=PIPE, + cwd=pypyroot + ) if p.wait() != 0: - hgid = '?' + maywarn(p.stderr.read(), 'Git') + return 'PyPy', '?', revision_id + branch = '?' + for line in p.stdout.read().strip().split('\n'): + if line.startswith('* '): + branch = line[1:].strip() + if branch == '(no branch)': + branch = '?' + break + return 'PyPy', branch, revision_id + return 'PyPy', p.stdout.read().strip(), revision_id - p = Popen([str(hgexe), 'id', '-t', pypyroot], - stdout=PIPE, stderr=PIPE, env=env) - hgtags = [t for t in p.stdout.read().strip().split() if t != 'tip'] - maywarn(p.stderr.read()) - if p.wait() != 0: - hgtags = ['?'] - if hgtags: - return 'PyPy', hgtags[0], hgid - else: - # use the branch instead - p = Popen([str(hgexe), 'id', '-b', pypyroot], - stdout=PIPE, stderr=PIPE, env=env) - hgbranch = p.stdout.read().strip() - maywarn(p.stderr.read()) - - return 'PyPy', hgbranch, hgid +if __name__ == '__main__': + print get_repo_version_info() diff --git a/pypy/translator/c/src/profiling.c b/pypy/translator/c/src/profiling.c --- a/pypy/translator/c/src/profiling.c +++ b/pypy/translator/c/src/profiling.c @@ -29,33 +29,33 @@ profiling_setup = 0; } } - -#elif defined(_WIN32) -#include <windows.h> - -DWORD_PTR base_affinity_mask; -int profiling_setup = 0; - -void pypy_setup_profiling() { - if (!profiling_setup) { - DWORD_PTR affinity_mask, system_affinity_mask; - GetProcessAffinityMask(GetCurrentProcess(), - &base_affinity_mask, &system_affinity_mask); - affinity_mask = 1; - /* Pick one cpu allowed by the system */ - if (system_affinity_mask) - while ((affinity_mask & system_affinity_mask) == 0) - affinity_mask <<= 1; - SetProcessAffinityMask(GetCurrentProcess(), affinity_mask); - profiling_setup = 1; - } -} - -void pypy_teardown_profiling() { - if (profiling_setup) { - SetProcessAffinityMask(GetCurrentProcess(), base_affinity_mask); - profiling_setup = 0; - } + +#elif defined(_WIN32) +#include <windows.h> + +DWORD_PTR base_affinity_mask; +int profiling_setup = 0; + +void pypy_setup_profiling() { + if (!profiling_setup) { + DWORD_PTR affinity_mask, system_affinity_mask; + GetProcessAffinityMask(GetCurrentProcess(), + &base_affinity_mask, &system_affinity_mask); + affinity_mask = 1; + /* Pick one cpu allowed by the system */ + if (system_affinity_mask) + while ((affinity_mask & system_affinity_mask) == 0) + affinity_mask <<= 1; + SetProcessAffinityMask(GetCurrentProcess(), affinity_mask); + profiling_setup = 1; + } +} + +void pypy_teardown_profiling() { + if (profiling_setup) { + SetProcessAffinityMask(GetCurrentProcess(), base_affinity_mask); + profiling_setup = 0; + } } #else diff --git a/pypy/translator/goal/targetnopstandalone.py b/pypy/translator/goal/targetnopstandalone.py --- a/pypy/translator/goal/targetnopstandalone.py +++ b/pypy/translator/goal/targetnopstandalone.py @@ -7,10 +7,8 @@ actually implementing argv of the executable. """ -import os, sys - -def debug(msg): - os.write(2, "debug: " + msg + '\n') +def debug(msg): + print "debug:", msg # __________ Entry point __________ diff --git a/pypy/translator/goal/translate.py b/pypy/translator/goal/translate.py --- a/pypy/translator/goal/translate.py +++ b/pypy/translator/goal/translate.py @@ -293,17 +293,18 @@ drv.exe_name = targetspec_dic['__name__'] + '-%(backend)s' # Double check to ensure we are not overwriting the current interpreter - try: - this_exe = py.path.local(sys.executable).new(ext='') - exe_name = drv.compute_exe_name() - samefile = this_exe.samefile(exe_name) - assert not samefile, ( - 'Output file %s is the currently running ' - 'interpreter (use --output=...)'% exe_name) - except EnvironmentError: - pass + goals = translateconfig.goals + if not goals or 'compile' in goals: + try: + this_exe = py.path.local(sys.executable).new(ext='') + exe_name = drv.compute_exe_name() + samefile = this_exe.samefile(exe_name) + assert not samefile, ( + 'Output file %s is the currently running ' + 'interpreter (use --output=...)'% exe_name) + except EnvironmentError: + pass - goals = translateconfig.goals try: drv.proceed(goals) finally: diff --git a/pypy/translator/platform/freebsd.py b/pypy/translator/platform/freebsd.py --- a/pypy/translator/platform/freebsd.py +++ b/pypy/translator/platform/freebsd.py @@ -18,8 +18,9 @@ class Freebsd(posix.BasePosix): name = "freebsd" - link_flags = get_env_vector("LDFLAGS", '-pthread') - cflags = get_env_vector("CFLAGS", "-O3 -pthread -fomit-frame-pointer") + link_flags = ['-pthread'] + get_env_vector('LDFLAGS', '') + cflags = ['-O3', '-pthread', '-fomit-frame-pointer' + ] + get_env_vector('CFLAGS', '') standalone_only = [] shared_only = [] so_ext = 'so' diff --git a/pypy/translator/platform/linux.py b/pypy/translator/platform/linux.py --- a/pypy/translator/platform/linux.py +++ b/pypy/translator/platform/linux.py @@ -1,15 +1,20 @@ """Support for Linux.""" +import os import sys from pypy.translator.platform.posix import BasePosix class BaseLinux(BasePosix): name = "linux" - link_flags = ('-pthread',) + link_flags = tuple( + ['-pthread',] + + os.environ.get('LDFLAGS', '').split()) extra_libs = ('-lrt',) - cflags = ('-O3', '-pthread', '-fomit-frame-pointer', - '-Wall', '-Wno-unused') + cflags = tuple( + ['-O3', '-pthread', '-fomit-frame-pointer', + '-Wall', '-Wno-unused'] + + os.environ.get('CFLAGS', '').split()) standalone_only = () shared_only = ('-fPIC',) so_ext = 'so' From noreply at buildbot.pypy.org Sat Jan 28 18:51:59 2012 From: noreply at buildbot.pypy.org (mattip) Date: Sat, 28 Jan 2012 18:51:59 +0100 (CET) Subject: [pypy-commit] pypy matrixmath-dot: cleanup after merge, all tests pass Message-ID: <20120128175159.F099E82114@wyvern.cs.uni-duesseldorf.de> Author: mattip Branch: matrixmath-dot Changeset: r51924:cb8369cbce6f Date: 2012-01-28 19:44 +0200 http://bitbucket.org/pypy/pypy/changeset/cb8369cbce6f/ Log: cleanup after merge, all tests pass diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -7,6 +7,7 @@ from pypy.module.micronumpy.strides import (calculate_slice_strides, shape_agreement, find_shape_and_elems, get_shape_from_iterable, calc_new_strides, to_coords) +from dot import multidim_dot, match_dot_shapes from pypy.rlib import jit from pypy.rpython.lltypesystem import lltype, rffi from pypy.tool.sourcetools import func_with_new_name @@ -786,7 +787,7 @@ def __init__(self, child, chunks, shape): self.child = child self.chunks = chunks - self.size = size + self.size = support.product(shape) VirtualArray.__init__(self, 'slice', shape, child.find_dtype()) def create_sig(self): diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py --- a/pypy/module/micronumpy/test/test_zjit.py +++ b/pypy/module/micronumpy/test/test_zjit.py @@ -467,7 +467,7 @@ def test_dot(self): result = self.run("dot") assert result == 184 - self.check_simple_loop({{'arraylen_gc': 9, + self.check_simple_loop({'arraylen_gc': 9, 'float_add': 1, 'float_mul': 1, 'getinteriorfield_raw': 3, @@ -477,7 +477,7 @@ 'int_lt': 6, 'int_sub': 3, 'jump': 1, - 'setinteriorfield_raw': 1}}) + 'setinteriorfield_raw': 1}) class TestNumpyOld(LLJitMixin): From noreply at buildbot.pypy.org Sat Jan 28 19:30:16 2012 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 28 Jan 2012 19:30:16 +0100 (CET) Subject: [pypy-commit] pypy stackroot-speedup: (fijal, arigo) Message-ID: <20120128183016.E9D1482114@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: stackroot-speedup Changeset: r51925:51157f562a0f Date: 2012-01-28 17:17 +0100 http://bitbucket.org/pypy/pypy/changeset/51157f562a0f/ Log: (fijal, arigo) Finish the test. Argh, painful :-( diff --git a/pypy/rpython/memory/gctransform/test/test_framework.py b/pypy/rpython/memory/gctransform/test/test_framework.py --- a/pypy/rpython/memory/gctransform/test/test_framework.py +++ b/pypy/rpython/memory/gctransform/test/test_framework.py @@ -7,7 +7,7 @@ from pypy.rpython.memory.gctransform.transform import GcHighLevelOp from pypy.rpython.memory.gctransform.framework import FrameworkGCTransformer, \ CollectAnalyzer, find_initializing_stores, find_clean_setarrayitems -from pypy.rpython.lltypesystem import lltype, rffi +from pypy.rpython.lltypesystem import lltype, llmemory, rffi from pypy.rpython.rtyper import LowLevelOpList from pypy.translator.c.gc import FrameworkGcPolicy from pypy.translator.translator import TranslationContext, graphof @@ -336,6 +336,7 @@ def test_contains_stack_check(): from pypy.translator.transform import insert_ll_stackcheck from pypy.translator.c.genc import CStandaloneBuilder + from pypy.rpython.memory.gctransform.shadowstack import sizeofaddr # class A: pass @@ -343,22 +344,40 @@ A() def f_recursive(n): A() - if n > 5: + if n > 2: f_recursive(n-1) - def f(n): - a = A() - f_non_recursive(n) - f_recursive(n) - return a + for f1 in [f_non_recursive, f_recursive]: - t = rtype(f, [int]) - insert_ll_stackcheck(t) - g_non_recursive = t._graphof(f_non_recursive) - assert not hasattr(g_non_recursive, 'contains_stack_check') - g_recursive = t._graphof(f_recursive) - assert g_recursive.contains_stack_check is True - # - cbuild = CStandaloneBuilder(t, f, t.config, - gcpolicy=FrameworkGcPolicy2) - db = cbuild.generate_graphs_for_llinterp() - import pdb; pdb.set_trace() + def f(n): + a = A() + f1(n) + return a + + t = rtype(f, [int]) + insert_ll_stackcheck(t) + g1 = t._graphof(f1) + if f1 is f_non_recursive: + assert not hasattr(g1, 'contains_stack_check') + else: + assert g1.contains_stack_check is True + # + cbuild = CStandaloneBuilder(t, f, t.config, + gcpolicy=FrameworkGcPolicy2) + db = cbuild.generate_graphs_for_llinterp() + # + # check that in the non-recursive case, around the call + # to f1 we save one gcptr 'a'; and in the recursive case, + # we save two, because there are 'a' and a marker for + # shadowstack. + if f1 is f_non_recursive: + expected = 1 + else: + expected = 2 + adr_adds = [] + for block in t._graphof(f).iterblocks(): + for op in block.operations: + if op.opname == 'adr_add': + adr_adds.append(op) + assert len(adr_adds) == 1 + got = adr_adds[0].args[1].value + assert got.repeat == expected From noreply at buildbot.pypy.org Sat Jan 28 19:30:23 2012 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 28 Jan 2012 19:30:23 +0100 (CET) Subject: [pypy-commit] pypy stackroot-speedup: hg merge default Message-ID: <20120128183023.19C1682119@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: stackroot-speedup Changeset: r51930:22439f4f3761 Date: 2012-01-28 19:19 +0100 http://bitbucket.org/pypy/pypy/changeset/22439f4f3761/ Log: hg merge default diff --git a/pypy/jit/backend/llsupport/gc.py b/pypy/jit/backend/llsupport/gc.py --- a/pypy/jit/backend/llsupport/gc.py +++ b/pypy/jit/backend/llsupport/gc.py @@ -394,13 +394,13 @@ This is the class supporting --gcrootfinder=shadowstack. """ is_shadow_stack = True - MARKER = 8 + MARKER_FRAME = 8 # this marker now *follows* the frame addr # The "shadowstack" is a portable way in which the GC finds the # roots that live in the stack. Normally it is just a list of # pointers to GC objects. The pointers may be moved around by a GC - # collection. But with the JIT, an entry can also be MARKER, in - # which case the next entry points to an assembler stack frame. + # collection. But with the JIT, an entry can also be MARKER_FRAME, + # in which case the previous entry points to an assembler stack frame. # During a residual CALL from the assembler (which may indirectly # call the GC), we use the force_index stored in the assembler # stack frame to identify the call: we can go from the force_index @@ -436,11 +436,13 @@ def setcontext(iself, context): iself.context = context - def next(iself, gc, next, range_highest): - # Return the "next" valid GC object' address. This usually - # means just returning "next", until we reach "range_highest", - # except that we are skipping NULLs. If "next" contains a - # MARKER instead, then we go into JIT-frame-lookup mode. + def nextleft(iself, gc, range_lowest, prev): + # Return the next valid GC object's address, in right-to-left + # order from the shadowstack array. This usually means just + # returning "prev - sizeofaddr", until we reach "range_lowest", + # except that we are skipping NULLs. If "prev - sizeofaddr" + # contains a MARKER_FRAME instead, then we go into + # JIT-frame-lookup mode. # while True: # @@ -449,20 +451,20 @@ # # Look for the next shadowstack address that # contains a valid pointer - while next != range_highest: - if next.signed[0] == self.MARKER: + while prev != range_lowest: + prev -= llmemory.sizeof(llmemory.Address) + if prev.signed[0] == self.MARKER_FRAME: break - if gc.points_to_valid_gc_object(next): - return next - next += llmemory.sizeof(llmemory.Address) + if gc.points_to_valid_gc_object(prev): + return prev else: return llmemory.NULL # done # - # It's a JIT frame. Save away 'next' for later, and + # It's a JIT frame. Save away 'prev' for later, and # go into JIT-frame-exploring mode. - next += llmemory.sizeof(llmemory.Address) - frame_addr = next.signed[0] - iself.saved_next = next + prev -= llmemory.sizeof(llmemory.Address) + frame_addr = prev.signed[0] + iself.saved_prev = prev iself.frame_addr = frame_addr addr = llmemory.cast_int_to_adr(frame_addr + self.force_index_ofs) @@ -502,8 +504,7 @@ # # Restore 'prev' and loop back to the start. iself.frame_addr = 0 - next = iself.saved_next - next += llmemory.sizeof(llmemory.Address) + prev = iself.saved_prev # --------------- # diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py --- a/pypy/jit/backend/x86/assembler.py +++ b/pypy/jit/backend/x86/assembler.py @@ -774,7 +774,7 @@ self.mc.RET() def _call_header_shadowstack(self, gcrootmap): - # we need to put two words into the shadowstack: the MARKER + # we need to put two words into the shadowstack: the MARKER_FRAME # and the address of the frame (ebp, actually) rst = gcrootmap.get_root_stack_top_addr() if rx86.fits_in_32bits(rst): @@ -783,9 +783,10 @@ self.mc.MOV_ri(r13.value, rst) # MOV r13, rootstacktop self.mc.MOV_rm(eax.value, (r13.value, 0)) # MOV eax, [r13] # - self.mc.LEA_rm(ebx.value, (eax.value, 2*WORD)) # LEA ebx, [eax+2*WORD] - self.mc.MOV_mi((eax.value, 0), gcrootmap.MARKER) # MOV [eax], MARKER - self.mc.MOV_mr((eax.value, WORD), ebp.value) # MOV [eax+WORD], ebp + MARKER = gcrootmap.MARKER_FRAME + self.mc.LEA_rm(ebx.value, (eax.value, 2*WORD)) # LEA ebx, [eax+2*WORD] + self.mc.MOV_mi((eax.value, WORD), MARKER) # MOV [eax+WORD], MARKER + self.mc.MOV_mr((eax.value, 0), ebp.value) # MOV [eax], ebp # if rx86.fits_in_32bits(rst): self.mc.MOV_jr(rst, ebx.value) # MOV [rootstacktop], ebx diff --git a/pypy/rpython/memory/gctransform/shadowstack.py b/pypy/rpython/memory/gctransform/shadowstack.py --- a/pypy/rpython/memory/gctransform/shadowstack.py +++ b/pypy/rpython/memory/gctransform/shadowstack.py @@ -33,15 +33,15 @@ self.decr_stack = decr_stack root_iterator = get_root_iterator(gctransformer) - def walk_stack_root(callback, addr, end): + def walk_stack_root(callback, start, end): root_iterator.setcontext(NonConstant(llmemory.NULL)) gc = self.gc + addr = end while True: - addr = root_iterator.next(gc, addr, end) + addr = root_iterator.nextleft(gc, start, addr) if addr == llmemory.NULL: return callback(gc, addr) - addr += sizeofaddr self.rootstackhook = walk_stack_root self.shadow_stack_pool = ShadowStackPool(gcdata) @@ -339,11 +339,11 @@ return True def setcontext(self, context): pass - def next(self, gc, addr, end): - while addr != end: + def nextleft(self, gc, start, addr): + while addr != start: + addr -= sizeofaddr if gc.points_to_valid_gc_object(addr): return addr - addr += sizeofaddr return llmemory.NULL result = RootIterator() gctransformer._root_iterator = result @@ -370,10 +370,8 @@ obj = llmemory.cast_adr_to_ptr(obj, SHADOWSTACKREFPTR) if not prev: root_iterator.setcontext(obj.context) - next = obj.base - else: - next = prev + sizeofaddr - return root_iterator.next(gc, next, obj.top) + prev = obj.top + return root_iterator.nextleft(gc, obj.base, prev) CUSTOMTRACEFUNC = lltype.FuncType([llmemory.Address, llmemory.Address], llmemory.Address) From noreply at buildbot.pypy.org Sat Jan 28 19:30:24 2012 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 28 Jan 2012 19:30:24 +0100 (CET) Subject: [pypy-commit] pypy stackroot-speedup: (fijal, arigo) add support for MARKER_TRACED that was lost during the merge Message-ID: <20120128183024.4F3E482114@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: stackroot-speedup Changeset: r51931:d160dc0c193a Date: 2012-01-28 19:29 +0100 http://bitbucket.org/pypy/pypy/changeset/d160dc0c193a/ Log: (fijal, arigo) add support for MARKER_TRACED that was lost during the merge diff --git a/pypy/rpython/memory/gctransform/framework.py b/pypy/rpython/memory/gctransform/framework.py --- a/pypy/rpython/memory/gctransform/framework.py +++ b/pypy/rpython/memory/gctransform/framework.py @@ -1233,7 +1233,7 @@ self.num_pushs += len(livevars) if insert_rec_marker: from pypy.rpython.memory.gctransform import shadowstack - MARKER = shadowstack.ShadowStackRootWalker.MARKER_NOT_TRACED + MARKER = shadowstack.MARKER_NOT_TRACED c_marker = rmodel.inputconst(lltype.Signed, MARKER) livevars.append(c_marker) if not livevars: diff --git a/pypy/rpython/memory/gctransform/shadowstack.py b/pypy/rpython/memory/gctransform/shadowstack.py --- a/pypy/rpython/memory/gctransform/shadowstack.py +++ b/pypy/rpython/memory/gctransform/shadowstack.py @@ -7,14 +7,15 @@ from pypy.annotation import model as annmodel +# special values in the shadowstack (different values than +# the jit's MARKER_FRAME, which is 8): +MARKER_NOT_TRACED = 16 +MARKER_TRACED = 24 + + class ShadowStackRootWalker(BaseRootWalker): need_root_stack = True - # special values in the shadowstack (different values than - # the jit's MARKER, which is 8): - MARKER_NOT_TRACED = 16 - MARKER_TRACED = 24 - def __init__(self, gctransformer): BaseRootWalker.__init__(self, gctransformer) # NB. 'self' is frozen, but we can use self.gcdata to store state @@ -33,12 +34,12 @@ self.decr_stack = decr_stack root_iterator = get_root_iterator(gctransformer) - def walk_stack_root(callback, start, end): + def walk_stack_root(callback, start, end, is_minor): root_iterator.setcontext(NonConstant(llmemory.NULL)) gc = self.gc addr = end while True: - addr = root_iterator.nextleft(gc, start, addr) + addr = root_iterator.nextleft(gc, start, addr, is_minor) if addr == llmemory.NULL: return callback(gc, addr) @@ -339,9 +340,29 @@ return True def setcontext(self, context): pass - def nextleft(self, gc, start, addr): + def nextleft(self, gc, start, addr, is_minor): while addr != start: addr -= sizeofaddr + value = llmemory.cast_adr_to_int(addr.address[0]) + # + # If 'addr' contains MARKER_TRACED, and if we are doing + # a minor collection, then stop here. The previous items + # have all been traced at least once already, so they + # cannot contain young pointers. + if value == MARKER_TRACED: + if is_minor: + break + continue # ignore the marker and continue + # + # If 'addr' contains MARKER_NOT_TRACED, replace it by + # MARKER_TRACED. + if value == MARKER_NOT_TRACED: + addr.address[0] = rffi.cast(llmemory.Address, + MARKER_TRACED) + continue + # + # Regular part follows: invoke 'callback' if myaddr really + # points to an object if gc.points_to_valid_gc_object(addr): return addr return llmemory.NULL From noreply at buildbot.pypy.org Sat Jan 28 19:30:18 2012 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 28 Jan 2012 19:30:18 +0100 (CET) Subject: [pypy-commit] pypy stackroot-speedup: (fijal, arigo) Message-ID: <20120128183018.2BFF982115@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: stackroot-speedup Changeset: r51926:e23446a82e4e Date: 2012-01-28 17:35 +0100 http://bitbucket.org/pypy/pypy/changeset/e23446a82e4e/ Log: (fijal, arigo) Fix the framework transformer to let it insert the MARKER_NOT_TRACED into the shadowstack, around calls to functions with the 'contains_stack_check' flag. diff --git a/pypy/rpython/memory/gctransform/framework.py b/pypy/rpython/memory/gctransform/framework.py --- a/pypy/rpython/memory/gctransform/framework.py +++ b/pypy/rpython/memory/gctransform/framework.py @@ -14,6 +14,7 @@ from pypy.translator.backendopt.support import var_needsgc from pypy.translator.backendopt.finalizer import FinalizerAnalyzer from pypy.annotation import model as annmodel +from pypy.objspace.flow.model import Constant from pypy.rpython import annlowlevel from pypy.rpython.rbuiltin import gen_cast from pypy.rpython.memory.gctypelayout import ll_weakref_deref, WEAKREF @@ -651,9 +652,17 @@ def gct_direct_call(self, hop): if self.collect_analyzer.analyze(hop.spaceop): - livevars = self.push_roots(hop) + # + v_func = hop.spaceop.args[0] + if isinstance(v_func, Constant): + graph = getattr(v_func.value._obj, 'graph', None) + rec = getattr(graph, 'contains_stack_check', False) + else: + rec = False # good enough + # + livevars = self.push_roots(hop, insert_rec_marker=rec) self.default(hop) - self.pop_roots(hop, livevars) + self.pop_roots(hop, livevars, insert_rec_marker=rec) else: self.default(hop) if hop.spaceop.opname == "direct_call": @@ -1121,7 +1130,6 @@ return None def transform_generic_set(self, hop): - from pypy.objspace.flow.model import Constant opname = hop.spaceop.opname v_struct = hop.spaceop.args[0] v_newvalue = hop.spaceop.args[-1] @@ -1217,11 +1225,17 @@ livevars = [var for var in livevars if not var_ispyobj(var)] return livevars - def push_roots(self, hop, keep_current_args=False): + def push_roots(self, hop, keep_current_args=False, + insert_rec_marker=False): if self.incr_stack_ptr is None: return livevars = self.get_livevars_for_roots(hop, keep_current_args) self.num_pushs += len(livevars) + if insert_rec_marker: + from pypy.rpython.memory.gctransform import shadowstack + MARKER = shadowstack.ShadowStackRootWalker.MARKER_NOT_TRACED + c_marker = rmodel.inputconst(lltype.Signed, MARKER) + livevars.append(c_marker) if not livevars: return [] c_len = rmodel.inputconst(lltype.Signed, len(livevars) ) @@ -1234,12 +1248,15 @@ hop.genop("raw_store", [base_addr, c_type, c_k, v_adr]) return livevars - def pop_roots(self, hop, livevars): + def pop_roots(self, hop, livevars, insert_rec_marker=False): if self.decr_stack_ptr is None: return - if not livevars: + length = len(livevars) + if insert_rec_marker: + length += 1 + if length == 0: return - c_len = rmodel.inputconst(lltype.Signed, len(livevars) ) + c_len = rmodel.inputconst(lltype.Signed, length) base_addr = hop.genop("direct_call", [self.decr_stack_ptr, c_len ], resulttype=llmemory.Address) if self.gcdata.gc.moving_gc: From noreply at buildbot.pypy.org Sat Jan 28 19:30:19 2012 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 28 Jan 2012 19:30:19 +0100 (CET) Subject: [pypy-commit] pypy default: Reduce the number of cases of shadowstack root walking: Message-ID: <20120128183019.60DC682116@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: Changeset: r51927:256d3279bde5 Date: 2012-01-28 18:20 +0100 http://bitbucket.org/pypy/pypy/changeset/256d3279bde5/ Log: Reduce the number of cases of shadowstack root walking: from 4 (jit/nojit, thread/nothread) to 2 (thread/nothread). diff --git a/pypy/jit/backend/llsupport/gc.py b/pypy/jit/backend/llsupport/gc.py --- a/pypy/jit/backend/llsupport/gc.py +++ b/pypy/jit/backend/llsupport/gc.py @@ -433,6 +433,9 @@ class RootIterator: _alloc_flavor_ = "raw" + def setcontext(iself, context): + iself.context = context + def next(iself, gc, next, range_highest): # Return the "next" valid GC object' address. This usually # means just returning "next", until we reach "range_highest", diff --git a/pypy/rpython/memory/gctransform/shadowstack.py b/pypy/rpython/memory/gctransform/shadowstack.py --- a/pypy/rpython/memory/gctransform/shadowstack.py +++ b/pypy/rpython/memory/gctransform/shadowstack.py @@ -27,28 +27,17 @@ return top self.decr_stack = decr_stack - translator = gctransformer.translator - if (hasattr(translator, '_jit2gc') and - 'root_iterator' in translator._jit2gc): - root_iterator = translator._jit2gc['root_iterator'] - def jit_walk_stack_root(callback, addr, end): - root_iterator.context = NonConstant(llmemory.NULL) - gc = self.gc - while True: - addr = root_iterator.next(gc, addr, end) - if addr == llmemory.NULL: - return - callback(gc, addr) - addr += sizeofaddr - self.rootstackhook = jit_walk_stack_root - else: - def default_walk_stack_root(callback, addr, end): - gc = self.gc - while addr != end: - if gc.points_to_valid_gc_object(addr): - callback(gc, addr) - addr += sizeofaddr - self.rootstackhook = default_walk_stack_root + root_iterator = get_root_iterator(gctransformer) + def walk_stack_root(callback, addr, end): + root_iterator.setcontext(NonConstant(llmemory.NULL)) + gc = self.gc + while True: + addr = root_iterator.next(gc, addr, end) + if addr == llmemory.NULL: + return + callback(gc, addr) + addr += sizeofaddr + self.rootstackhook = walk_stack_root self.shadow_stack_pool = ShadowStackPool(gcdata) rsd = gctransformer.root_stack_depth @@ -331,6 +320,30 @@ raise MemoryError +def get_root_iterator(gctransformer): + if hasattr(gctransformer, '_root_iterator'): + return gctransformer._root_iterator # if already built + translator = gctransformer.translator + if (hasattr(translator, '_jit2gc') and + 'root_iterator' in translator._jit2gc): + result = translator._jit2gc['root_iterator'] + else: + class RootIterator(object): + def _freeze_(self): + return True + def setcontext(self, context): + pass + def next(self, gc, addr, end): + while addr != end: + if gc.points_to_valid_gc_object(addr): + return addr + addr += sizeofaddr + return llmemory.NULL + result = RootIterator() + gctransformer._root_iterator = result + return result + + def get_shadowstackref(gctransformer): if hasattr(gctransformer, '_SHADOWSTACKREF'): return gctransformer._SHADOWSTACKREF @@ -344,28 +357,17 @@ rtti=True) SHADOWSTACKREFPTR.TO.become(SHADOWSTACKREF) - translator = gctransformer.translator - if hasattr(translator, '_jit2gc'): - gc = gctransformer.gcdata.gc - root_iterator = translator._jit2gc['root_iterator'] - def customtrace(obj, prev): - obj = llmemory.cast_adr_to_ptr(obj, SHADOWSTACKREFPTR) - if not prev: - root_iterator.context = obj.context - next = obj.base - else: - next = prev + sizeofaddr - return root_iterator.next(gc, next, obj.top) - else: - def customtrace(obj, prev): - # a simple but not JIT-ready version - if not prev: - next = llmemory.cast_adr_to_ptr(obj, SHADOWSTACKREFPTR).base - else: - next = prev + sizeofaddr - if next == llmemory.cast_adr_to_ptr(obj, SHADOWSTACKREFPTR).top: - next = llmemory.NULL - return next + gc = gctransformer.gcdata.gc + root_iterator = get_root_iterator(gctransformer) + + def customtrace(obj, prev): + obj = llmemory.cast_adr_to_ptr(obj, SHADOWSTACKREFPTR) + if not prev: + root_iterator.setcontext(obj.context) + next = obj.base + else: + next = prev + sizeofaddr + return root_iterator.next(gc, next, obj.top) CUSTOMTRACEFUNC = lltype.FuncType([llmemory.Address, llmemory.Address], llmemory.Address) From noreply at buildbot.pypy.org Sat Jan 28 19:30:20 2012 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 28 Jan 2012 19:30:20 +0100 (CET) Subject: [pypy-commit] pypy stackroot-speedup: hg merge default Message-ID: <20120128183020.9BCA782117@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: stackroot-speedup Changeset: r51928:d12ecbdb1018 Date: 2012-01-28 18:40 +0100 http://bitbucket.org/pypy/pypy/changeset/d12ecbdb1018/ Log: hg merge default (loosing the changes done so far to shadowstack; will be re-applied in the next checkin) diff --git a/pypy/jit/backend/llsupport/gc.py b/pypy/jit/backend/llsupport/gc.py --- a/pypy/jit/backend/llsupport/gc.py +++ b/pypy/jit/backend/llsupport/gc.py @@ -433,6 +433,9 @@ class RootIterator: _alloc_flavor_ = "raw" + def setcontext(iself, context): + iself.context = context + def next(iself, gc, next, range_highest): # Return the "next" valid GC object' address. This usually # means just returning "next", until we reach "range_highest", diff --git a/pypy/rpython/memory/gctransform/shadowstack.py b/pypy/rpython/memory/gctransform/shadowstack.py --- a/pypy/rpython/memory/gctransform/shadowstack.py +++ b/pypy/rpython/memory/gctransform/shadowstack.py @@ -32,50 +32,17 @@ return top self.decr_stack = decr_stack - translator = gctransformer.translator - if (hasattr(translator, '_jit2gc') and - 'root_iterator' in translator._jit2gc): - root_iterator = translator._jit2gc['root_iterator'] - def jit_walk_stack_root(callback, addr, end): - root_iterator.context = NonConstant(llmemory.NULL) - gc = self.gc - while True: - addr = root_iterator.next(gc, addr, end) - if addr == llmemory.NULL: - return - callback(gc, addr) - addr += sizeofaddr - self.rootstackhook = jit_walk_stack_root - else: - def default_walk_stack_root(callback, start, end, is_minor): - gc = self.gc - addr = end - while addr != start: - addr -= sizeofaddr - value = llmemory.cast_adr_to_int(addr.address[0]) - # - # If myaddr contains MARKER_TRACED, and if we are doing - # a minor collection, then stop here. The previous items - # have all been traced at least once already, so they - # cannot contain young pointers. - if value == self.MARKER_TRACED: - if is_minor: - break - continue # ignore the marker and continue - # - # If myaddr contains MARKER_NOT_TRACED, replace it by - # MARKER_TRACED. - if value == self.MARKER_NOT_TRACED: - addr.address[0] = rffi.cast(llmemory.Address, - self.MARKER_TRACED) - continue - # - # Regular part follows: invoke 'callback' if myaddr really - # points to an object - if gc.points_to_valid_gc_object(addr): - callback(gc, addr) - # - self.rootstackhook = default_walk_stack_root + root_iterator = get_root_iterator(gctransformer) + def walk_stack_root(callback, addr, end): + root_iterator.setcontext(NonConstant(llmemory.NULL)) + gc = self.gc + while True: + addr = root_iterator.next(gc, addr, end) + if addr == llmemory.NULL: + return + callback(gc, addr) + addr += sizeofaddr + self.rootstackhook = walk_stack_root self.shadow_stack_pool = ShadowStackPool(gcdata) rsd = gctransformer.root_stack_depth @@ -359,6 +326,30 @@ raise MemoryError +def get_root_iterator(gctransformer): + if hasattr(gctransformer, '_root_iterator'): + return gctransformer._root_iterator # if already built + translator = gctransformer.translator + if (hasattr(translator, '_jit2gc') and + 'root_iterator' in translator._jit2gc): + result = translator._jit2gc['root_iterator'] + else: + class RootIterator(object): + def _freeze_(self): + return True + def setcontext(self, context): + pass + def next(self, gc, addr, end): + while addr != end: + if gc.points_to_valid_gc_object(addr): + return addr + addr += sizeofaddr + return llmemory.NULL + result = RootIterator() + gctransformer._root_iterator = result + return result + + def get_shadowstackref(gctransformer): if hasattr(gctransformer, '_SHADOWSTACKREF'): return gctransformer._SHADOWSTACKREF @@ -372,28 +363,17 @@ rtti=True) SHADOWSTACKREFPTR.TO.become(SHADOWSTACKREF) - translator = gctransformer.translator - if hasattr(translator, '_jit2gc'): - gc = gctransformer.gcdata.gc - root_iterator = translator._jit2gc['root_iterator'] - def customtrace(obj, prev): - obj = llmemory.cast_adr_to_ptr(obj, SHADOWSTACKREFPTR) - if not prev: - root_iterator.context = obj.context - next = obj.base - else: - next = prev + sizeofaddr - return root_iterator.next(gc, next, obj.top) - else: - def customtrace(obj, prev): - # a simple but not JIT-ready version - if not prev: - next = llmemory.cast_adr_to_ptr(obj, SHADOWSTACKREFPTR).base - else: - next = prev + sizeofaddr - if next == llmemory.cast_adr_to_ptr(obj, SHADOWSTACKREFPTR).top: - next = llmemory.NULL - return next + gc = gctransformer.gcdata.gc + root_iterator = get_root_iterator(gctransformer) + + def customtrace(obj, prev): + obj = llmemory.cast_adr_to_ptr(obj, SHADOWSTACKREFPTR) + if not prev: + root_iterator.setcontext(obj.context) + next = obj.base + else: + next = prev + sizeofaddr + return root_iterator.next(gc, next, obj.top) CUSTOMTRACEFUNC = lltype.FuncType([llmemory.Address, llmemory.Address], llmemory.Address) From noreply at buildbot.pypy.org Sat Jan 28 19:30:21 2012 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 28 Jan 2012 19:30:21 +0100 (CET) Subject: [pypy-commit] pypy default: (fijal, arigo) Message-ID: <20120128183021.D4E9782118@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: Changeset: r51929:f0d095a1d379 Date: 2012-01-28 19:18 +0100 http://bitbucket.org/pypy/pypy/changeset/f0d095a1d379/ Log: (fijal, arigo) Change the order of the shadowstack processing. We now process it right-to-left instead of left-to-right (it's an array, so "left" is 0 and "right" is the end). diff --git a/pypy/jit/backend/llsupport/gc.py b/pypy/jit/backend/llsupport/gc.py --- a/pypy/jit/backend/llsupport/gc.py +++ b/pypy/jit/backend/llsupport/gc.py @@ -394,13 +394,13 @@ This is the class supporting --gcrootfinder=shadowstack. """ is_shadow_stack = True - MARKER = 8 + MARKER_FRAME = 8 # this marker now *follows* the frame addr # The "shadowstack" is a portable way in which the GC finds the # roots that live in the stack. Normally it is just a list of # pointers to GC objects. The pointers may be moved around by a GC - # collection. But with the JIT, an entry can also be MARKER, in - # which case the next entry points to an assembler stack frame. + # collection. But with the JIT, an entry can also be MARKER_FRAME, + # in which case the previous entry points to an assembler stack frame. # During a residual CALL from the assembler (which may indirectly # call the GC), we use the force_index stored in the assembler # stack frame to identify the call: we can go from the force_index @@ -436,11 +436,13 @@ def setcontext(iself, context): iself.context = context - def next(iself, gc, next, range_highest): - # Return the "next" valid GC object' address. This usually - # means just returning "next", until we reach "range_highest", - # except that we are skipping NULLs. If "next" contains a - # MARKER instead, then we go into JIT-frame-lookup mode. + def nextleft(iself, gc, range_lowest, prev): + # Return the next valid GC object's address, in right-to-left + # order from the shadowstack array. This usually means just + # returning "prev - sizeofaddr", until we reach "range_lowest", + # except that we are skipping NULLs. If "prev - sizeofaddr" + # contains a MARKER_FRAME instead, then we go into + # JIT-frame-lookup mode. # while True: # @@ -449,20 +451,20 @@ # # Look for the next shadowstack address that # contains a valid pointer - while next != range_highest: - if next.signed[0] == self.MARKER: + while prev != range_lowest: + prev -= llmemory.sizeof(llmemory.Address) + if prev.signed[0] == self.MARKER_FRAME: break - if gc.points_to_valid_gc_object(next): - return next - next += llmemory.sizeof(llmemory.Address) + if gc.points_to_valid_gc_object(prev): + return prev else: return llmemory.NULL # done # - # It's a JIT frame. Save away 'next' for later, and + # It's a JIT frame. Save away 'prev' for later, and # go into JIT-frame-exploring mode. - next += llmemory.sizeof(llmemory.Address) - frame_addr = next.signed[0] - iself.saved_next = next + prev -= llmemory.sizeof(llmemory.Address) + frame_addr = prev.signed[0] + iself.saved_prev = prev iself.frame_addr = frame_addr addr = llmemory.cast_int_to_adr(frame_addr + self.force_index_ofs) @@ -502,8 +504,7 @@ # # Restore 'prev' and loop back to the start. iself.frame_addr = 0 - next = iself.saved_next - next += llmemory.sizeof(llmemory.Address) + prev = iself.saved_prev # --------------- # diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py --- a/pypy/jit/backend/x86/assembler.py +++ b/pypy/jit/backend/x86/assembler.py @@ -774,7 +774,7 @@ self.mc.RET() def _call_header_shadowstack(self, gcrootmap): - # we need to put two words into the shadowstack: the MARKER + # we need to put two words into the shadowstack: the MARKER_FRAME # and the address of the frame (ebp, actually) rst = gcrootmap.get_root_stack_top_addr() if rx86.fits_in_32bits(rst): @@ -783,9 +783,10 @@ self.mc.MOV_ri(r13.value, rst) # MOV r13, rootstacktop self.mc.MOV_rm(eax.value, (r13.value, 0)) # MOV eax, [r13] # - self.mc.LEA_rm(ebx.value, (eax.value, 2*WORD)) # LEA ebx, [eax+2*WORD] - self.mc.MOV_mi((eax.value, 0), gcrootmap.MARKER) # MOV [eax], MARKER - self.mc.MOV_mr((eax.value, WORD), ebp.value) # MOV [eax+WORD], ebp + MARKER = gcrootmap.MARKER_FRAME + self.mc.LEA_rm(ebx.value, (eax.value, 2*WORD)) # LEA ebx, [eax+2*WORD] + self.mc.MOV_mi((eax.value, WORD), MARKER) # MOV [eax+WORD], MARKER + self.mc.MOV_mr((eax.value, 0), ebp.value) # MOV [eax], ebp # if rx86.fits_in_32bits(rst): self.mc.MOV_jr(rst, ebx.value) # MOV [rootstacktop], ebx diff --git a/pypy/rpython/memory/gctransform/shadowstack.py b/pypy/rpython/memory/gctransform/shadowstack.py --- a/pypy/rpython/memory/gctransform/shadowstack.py +++ b/pypy/rpython/memory/gctransform/shadowstack.py @@ -28,15 +28,15 @@ self.decr_stack = decr_stack root_iterator = get_root_iterator(gctransformer) - def walk_stack_root(callback, addr, end): + def walk_stack_root(callback, start, end): root_iterator.setcontext(NonConstant(llmemory.NULL)) gc = self.gc + addr = end while True: - addr = root_iterator.next(gc, addr, end) + addr = root_iterator.nextleft(gc, start, addr) if addr == llmemory.NULL: return callback(gc, addr) - addr += sizeofaddr self.rootstackhook = walk_stack_root self.shadow_stack_pool = ShadowStackPool(gcdata) @@ -333,11 +333,11 @@ return True def setcontext(self, context): pass - def next(self, gc, addr, end): - while addr != end: + def nextleft(self, gc, start, addr): + while addr != start: + addr -= sizeofaddr if gc.points_to_valid_gc_object(addr): return addr - addr += sizeofaddr return llmemory.NULL result = RootIterator() gctransformer._root_iterator = result @@ -364,10 +364,8 @@ obj = llmemory.cast_adr_to_ptr(obj, SHADOWSTACKREFPTR) if not prev: root_iterator.setcontext(obj.context) - next = obj.base - else: - next = prev + sizeofaddr - return root_iterator.next(gc, next, obj.top) + prev = obj.top + return root_iterator.nextleft(gc, obj.base, prev) CUSTOMTRACEFUNC = lltype.FuncType([llmemory.Address, llmemory.Address], llmemory.Address) From noreply at buildbot.pypy.org Sat Jan 28 20:44:08 2012 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 28 Jan 2012 20:44:08 +0100 (CET) Subject: [pypy-commit] pypy stackroot-speedup: (fijal mostly, arigo review) Message-ID: <20120128194408.C303471068F@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: stackroot-speedup Changeset: r51932:ab6b2fd5deb7 Date: 2012-01-28 20:15 +0100 http://bitbucket.org/pypy/pypy/changeset/ab6b2fd5deb7/ Log: (fijal mostly, arigo review) Adapt the logic to the JIT shadowstack code. diff --git a/pypy/jit/backend/llsupport/gc.py b/pypy/jit/backend/llsupport/gc.py --- a/pypy/jit/backend/llsupport/gc.py +++ b/pypy/jit/backend/llsupport/gc.py @@ -18,7 +18,7 @@ from pypy.jit.backend.llsupport.descr import get_array_descr from pypy.jit.backend.llsupport.descr import get_call_descr from pypy.jit.backend.llsupport.rewrite import GcRewriterAssembler -from pypy.rpython.memory.gctransform import asmgcroot +from pypy.rpython.memory.gctransform import asmgcroot, shadowstack # ____________________________________________________________ @@ -436,7 +436,7 @@ def setcontext(iself, context): iself.context = context - def nextleft(iself, gc, range_lowest, prev): + def nextleft(iself, gc, range_lowest, prev, is_minor): # Return the next valid GC object's address, in right-to-left # order from the shadowstack array. This usually means just # returning "prev - sizeofaddr", until we reach "range_lowest", @@ -453,7 +453,18 @@ # contains a valid pointer while prev != range_lowest: prev -= llmemory.sizeof(llmemory.Address) - if prev.signed[0] == self.MARKER_FRAME: + value = llmemory.cast_adr_to_int(prev.address[0]) + # this logic is directly copied from RootIterator + # in shadowstack.py, consult comments there + if value == shadowstack.MARKER_TRACED: + if is_minor: + return llmemory.NULL + continue + if value == shadowstack.MARKER_NOT_TRACED: + prev.address[0] = rffi.cast(llmemory.Address, + shadowstack.MARKER_TRACED) + continue + if value == self.MARKER_FRAME: break if gc.points_to_valid_gc_object(prev): return prev diff --git a/pypy/rpython/memory/gctransform/shadowstack.py b/pypy/rpython/memory/gctransform/shadowstack.py --- a/pypy/rpython/memory/gctransform/shadowstack.py +++ b/pypy/rpython/memory/gctransform/shadowstack.py @@ -351,7 +351,7 @@ # cannot contain young pointers. if value == MARKER_TRACED: if is_minor: - break + return llmemory.NULL continue # ignore the marker and continue # # If 'addr' contains MARKER_NOT_TRACED, replace it by From noreply at buildbot.pypy.org Sat Jan 28 20:44:10 2012 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 28 Jan 2012 20:44:10 +0100 (CET) Subject: [pypy-commit] pypy stackroot-speedup: (fijal, arigo) Message-ID: <20120128194410.05A7571068F@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: stackroot-speedup Changeset: r51933:40fbdea76ec1 Date: 2012-01-28 20:32 +0100 http://bitbucket.org/pypy/pypy/changeset/40fbdea76ec1/ Log: (fijal, arigo) Also replace MARKER_FRAME with a pair of markers: during minor collection, MARKER_FRAME is replaced by MARKER_FRAME_TRACED and a previous MARKER_FRAME_TRACED causes the tracing to stop after the corresponding jit frame. diff --git a/pypy/jit/backend/llsupport/gc.py b/pypy/jit/backend/llsupport/gc.py --- a/pypy/jit/backend/llsupport/gc.py +++ b/pypy/jit/backend/llsupport/gc.py @@ -394,7 +394,8 @@ This is the class supporting --gcrootfinder=shadowstack. """ is_shadow_stack = True - MARKER_FRAME = 8 # this marker now *follows* the frame addr + MARKER_FRAME = 8 # this marker now *follows* the frame addr + MARKER_FRAME_TRACED = 32 # The "shadowstack" is a portable way in which the GC finds the # roots that live in the stack. Normally it is just a list of @@ -407,6 +408,9 @@ # to a list of where the GC pointers are in the frame (this is the # purpose of the present class). # + # MARKER_FRAME is replaced with MARKER_FRAME_TRACED after the first + # time the GC traces the shadowstack. + # # Note that across CALL_MAY_FORCE or CALL_ASSEMBLER, we can also go # from the force_index to a ResumeGuardForcedDescr instance, which # is used if the virtualizable or the virtualrefs need to be forced @@ -464,7 +468,11 @@ prev.address[0] = rffi.cast(llmemory.Address, shadowstack.MARKER_TRACED) continue + if value == self.MARKER_FRAME_TRACED: + break if value == self.MARKER_FRAME: + prev.address[0] = rffi.cast(llmemory.Address, + self.MARKER_FRAME_TRACED) break if gc.points_to_valid_gc_object(prev): return prev @@ -475,7 +483,13 @@ # go into JIT-frame-exploring mode. prev -= llmemory.sizeof(llmemory.Address) frame_addr = prev.signed[0] - iself.saved_prev = prev + if value == self.MARKER_FRAME or not is_minor: + iself.saved_prev = prev # normal case + else: + # MARKER_FRAME_TRACED in a minor collection: + # <hack> this value in saved_prev will cause this + # function to stop after the current jit frame + iself.saved_prev = range_lowest iself.frame_addr = frame_addr addr = llmemory.cast_int_to_adr(frame_addr + self.force_index_ofs) From noreply at buildbot.pypy.org Sat Jan 28 20:44:11 2012 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 28 Jan 2012 20:44:11 +0100 (CET) Subject: [pypy-commit] pypy stackroot-speedup: (fijal, arigo) Performance tweak, maybe Message-ID: <20120128194411.3AA4571068F@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: stackroot-speedup Changeset: r51934:81fa07721a43 Date: 2012-01-28 20:43 +0100 http://bitbucket.org/pypy/pypy/changeset/81fa07721a43/ Log: (fijal, arigo) Performance tweak, maybe diff --git a/pypy/jit/backend/llsupport/gc.py b/pypy/jit/backend/llsupport/gc.py --- a/pypy/jit/backend/llsupport/gc.py +++ b/pypy/jit/backend/llsupport/gc.py @@ -2,7 +2,7 @@ from pypy.rlib import rgc from pypy.rlib.objectmodel import we_are_translated, specialize from pypy.rlib.debug import fatalerror -from pypy.rlib.rarithmetic import ovfcheck +from pypy.rlib.rarithmetic import ovfcheck, r_uint from pypy.rpython.lltypesystem import lltype, llmemory, rffi, rclass, rstr from pypy.rpython.lltypesystem import llgroup from pypy.rpython.lltypesystem.lloperation import llop @@ -396,6 +396,7 @@ is_shadow_stack = True MARKER_FRAME = 8 # this marker now *follows* the frame addr MARKER_FRAME_TRACED = 32 + MARKER_MAX = 32 # The "shadowstack" is a portable way in which the GC finds the # roots that live in the stack. Normally it is just a list of @@ -460,20 +461,25 @@ value = llmemory.cast_adr_to_int(prev.address[0]) # this logic is directly copied from RootIterator # in shadowstack.py, consult comments there - if value == shadowstack.MARKER_TRACED: - if is_minor: - return llmemory.NULL - continue - if value == shadowstack.MARKER_NOT_TRACED: - prev.address[0] = rffi.cast(llmemory.Address, - shadowstack.MARKER_TRACED) - continue - if value == self.MARKER_FRAME_TRACED: - break - if value == self.MARKER_FRAME: - prev.address[0] = rffi.cast(llmemory.Address, - self.MARKER_FRAME_TRACED) - break + if r_uint(value) <= r_uint(self.MARKER_MAX): + if value == 0: #performance only: shortcut NULLs + continue + if value == shadowstack.MARKER_TRACED: + if is_minor: + return llmemory.NULL + continue + if value == shadowstack.MARKER_NOT_TRACED: + prev.address[0] = rffi.cast( + llmemory.Address, + shadowstack.MARKER_TRACED) + continue + if value == self.MARKER_FRAME_TRACED: + break + if value == self.MARKER_FRAME: + prev.address[0] = rffi.cast( + llmemory.Address, + self.MARKER_FRAME_TRACED) + break if gc.points_to_valid_gc_object(prev): return prev else: From noreply at buildbot.pypy.org Sat Jan 28 21:08:59 2012 From: noreply at buildbot.pypy.org (amauryfa) Date: Sat, 28 Jan 2012 21:08:59 +0100 (CET) Subject: [pypy-commit] pypy default: CPython Issue #13885 (CVE-2011-3389): the _ssl module would always disable the CBC IV attack countermeasure. Message-ID: <20120128200859.5F55271068F@wyvern.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc <amauryfa at gmail.com> Branch: Changeset: r51935:cf1a8868cf4e Date: 2012-01-28 21:06 +0100 http://bitbucket.org/pypy/pypy/changeset/cf1a8868cf4e/ Log: CPython Issue #13885 (CVE-2011-3389): the _ssl module would always disable the CBC IV attack countermeasure. diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py --- a/pypy/module/_ssl/interp_ssl.py +++ b/pypy/module/_ssl/interp_ssl.py @@ -709,7 +709,8 @@ raise ssl_error(space, "SSL_CTX_use_certificate_chain_file error") # ssl compatibility - libssl_SSL_CTX_set_options(ss.ctx, SSL_OP_ALL) + libssl_SSL_CTX_set_options(ss.ctx, + SSL_OP_ALL & ~SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS) verification_mode = SSL_VERIFY_NONE if cert_mode == PY_SSL_CERT_OPTIONAL: diff --git a/pypy/rlib/ropenssl.py b/pypy/rlib/ropenssl.py --- a/pypy/rlib/ropenssl.py +++ b/pypy/rlib/ropenssl.py @@ -66,6 +66,8 @@ OPENSSL_NO_SSL2 = rffi_platform.Defined("OPENSSL_NO_SSL2") SSL_FILETYPE_PEM = rffi_platform.ConstantInteger("SSL_FILETYPE_PEM") SSL_OP_ALL = rffi_platform.ConstantInteger("SSL_OP_ALL") + SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS = rffi_platform.ConstantInteger( + "SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS") SSL_VERIFY_NONE = rffi_platform.ConstantInteger("SSL_VERIFY_NONE") SSL_VERIFY_PEER = rffi_platform.ConstantInteger("SSL_VERIFY_PEER") SSL_VERIFY_FAIL_IF_NO_PEER_CERT = rffi_platform.ConstantInteger("SSL_VERIFY_FAIL_IF_NO_PEER_CERT") From noreply at buildbot.pypy.org Sat Jan 28 21:10:47 2012 From: noreply at buildbot.pypy.org (amauryfa) Date: Sat, 28 Jan 2012 21:10:47 +0100 (CET) Subject: [pypy-commit] pypy string-NUL: A branch to try detection of NUL characters in strings Message-ID: <20120128201047.3160471068F@wyvern.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc <amauryfa at gmail.com> Branch: string-NUL Changeset: r51936:ee8a4766b942 Date: 2012-01-24 22:50 +0100 http://bitbucket.org/pypy/pypy/changeset/ee8a4766b942/ Log: A branch to try detection of NUL characters in strings From noreply at buildbot.pypy.org Sat Jan 28 21:10:48 2012 From: noreply at buildbot.pypy.org (amauryfa) Date: Sat, 28 Jan 2012 21:10:48 +0100 (CET) Subject: [pypy-commit] pypy string-NUL: In litsdef.merge(), perform checks before we start modifying the other Message-ID: <20120128201048.6766471068F@wyvern.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc <amauryfa at gmail.com> Branch: string-NUL Changeset: r51937:aa595f8bbf18 Date: 2012-01-28 20:48 +0100 http://bitbucket.org/pypy/pypy/changeset/aa595f8bbf18/ Log: In litsdef.merge(), perform checks before we start modifying the other read locations. This makes debugging much much easier. diff --git a/pypy/annotation/listdef.py b/pypy/annotation/listdef.py --- a/pypy/annotation/listdef.py +++ b/pypy/annotation/listdef.py @@ -86,18 +86,19 @@ read_locations = self.read_locations.copy() other_read_locations = other.read_locations.copy() self.read_locations.update(other.read_locations) - self.patch() # which should patch all refs to 'other' s_value = self.s_value s_other_value = other.s_value s_new_value = unionof(s_value, s_other_value) + if s_new_value != s_value: + if self.dont_change_any_more: + raise TooLateForChange if isdegenerated(s_new_value): if self.bookkeeper: self.bookkeeper.ondegenerated(self, s_new_value) elif other.bookkeeper: other.bookkeeper.ondegenerated(other, s_new_value) + self.patch() # which should patch all refs to 'other' if s_new_value != s_value: - if self.dont_change_any_more: - raise TooLateForChange self.s_value = s_new_value # reflow from reading points for position_key in read_locations: From noreply at buildbot.pypy.org Sat Jan 28 21:10:49 2012 From: noreply at buildbot.pypy.org (amauryfa) Date: Sat, 28 Jan 2012 21:10:49 +0100 (CET) Subject: [pypy-commit] pypy string-NUL: The NUL byte is now disallowed in many posix functions. Message-ID: <20120128201049.CAF2C71068F@wyvern.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc <amauryfa at gmail.com> Branch: string-NUL Changeset: r51938:56ffa1a912b1 Date: 2012-01-28 20:59 +0100 http://bitbucket.org/pypy/pypy/changeset/56ffa1a912b1/ Log: The NUL byte is now disallowed in many posix functions. We use the annotator to "prove" that a string has no NUL. diff --git a/pypy/annotation/binaryop.py b/pypy/annotation/binaryop.py --- a/pypy/annotation/binaryop.py +++ b/pypy/annotation/binaryop.py @@ -434,13 +434,18 @@ class __extend__(pairtype(SomeString, SomeString)): def union((str1, str2)): - return SomeString(can_be_None=str1.can_be_None or str2.can_be_None) + result = SomeString(can_be_None=str1.can_be_None or str2.can_be_None) + if str1.no_NUL and str2.no_NUL: + result.no_NUL = True + return result def add((str1, str2)): # propagate const-ness to help getattr(obj, 'prefix' + const_name) result = SomeString() if str1.is_immutable_constant() and str2.is_immutable_constant(): result.const = str1.const + str2.const + if str1.no_NUL and str2.no_NUL: + result.no_NUL = True return result class __extend__(pairtype(SomeChar, SomeChar)): @@ -475,7 +480,19 @@ raise NotImplementedError( "string formatting mixing strings and unicode not supported") getbookkeeper().count('strformat', str, s_tuple) - return SomeString() + result = SomeString() + no_NUL = str.no_NUL + for s_item in s_tuple.items: + if isinstance(s_item, SomeFloat): + pass + elif isinstance(s_item, SomeString) and s_item.no_NUL: + pass + else: + no_NUL = False + break + if no_NUL: + result.no_NUL = True + return result class __extend__(pairtype(SomeString, SomeObject)): @@ -806,7 +823,16 @@ # mixing Nones with other objects -def _make_none_union(classname, constructor_args='', glob=None): +def _make_none_union(classname, constructor_args='', glob=None, + copy_attributes=()): + if copy_attributes: + copy_attrs = ( + 'result.__dict__.update((name, getattr(obj, name)) ' + 'for name in %(copy_attributes)s if name in obj.__dict__)' + % locals()) + else: + copy_attrs = '' + if glob is None: glob = globals() loc = locals() @@ -814,21 +840,26 @@ class __extend__(pairtype(%(classname)s, SomePBC)): def union((obj, pbc)): if pbc.isNone(): - return %(classname)s(%(constructor_args)s) + result = %(classname)s(%(constructor_args)s) + %(copy_attrs)s + return result else: return SomeObject() class __extend__(pairtype(SomePBC, %(classname)s)): def union((pbc, obj)): if pbc.isNone(): - return %(classname)s(%(constructor_args)s) + result = %(classname)s(%(constructor_args)s) + %(copy_attrs)s + return result else: return SomeObject() """ % loc) exec source.compile() in glob _make_none_union('SomeInstance', 'classdef=obj.classdef, can_be_None=True') -_make_none_union('SomeString', 'can_be_None=True') +_make_none_union('SomeString', 'can_be_None=True', + copy_attributes=('no_NUL',)) _make_none_union('SomeUnicodeString', 'can_be_None=True') _make_none_union('SomeList', 'obj.listdef') _make_none_union('SomeDict', 'obj.dictdef') diff --git a/pypy/annotation/bookkeeper.py b/pypy/annotation/bookkeeper.py --- a/pypy/annotation/bookkeeper.py +++ b/pypy/annotation/bookkeeper.py @@ -344,8 +344,12 @@ elif issubclass(tp, str): # py.lib uses annotated str subclasses if len(x) == 1: result = SomeChar() + if not '\x00' in x: + result.no_NUL = True else: result = SomeString() + if not '\x00' in x: + result.no_NUL = True elif tp is unicode: if len(x) == 1: result = SomeUnicodeCodePoint() diff --git a/pypy/annotation/model.py b/pypy/annotation/model.py --- a/pypy/annotation/model.py +++ b/pypy/annotation/model.py @@ -229,19 +229,27 @@ "Stands for an object which is known to be a string." knowntype = str immutable = True + can_be_None=False + no_NUL = False + def __init__(self, can_be_None=False): - self.can_be_None = can_be_None + if can_be_None: + self.can_be_None = True def can_be_none(self): return self.can_be_None def nonnoneify(self): - return SomeString(can_be_None=False) + result = SomeString(can_be_None=False) + if self.no_NUL: + result.no_NUL = True + return result class SomeUnicodeString(SomeObject): "Stands for an object which is known to be an unicode string" knowntype = unicode immutable = True + no_NUL = False def __init__(self, can_be_None=False): self.can_be_None = can_be_None @@ -502,6 +510,8 @@ s_None = SomePBC([], can_be_None=True) s_Bool = SomeBool() s_ImpossibleValue = SomeImpossibleValue() +s_Str0 = SomeString() +s_Str0.no_NUL = True # ____________________________________________________________ # weakrefs @@ -716,8 +726,7 @@ def not_const(s_obj): if s_obj.is_constant(): - new_s_obj = SomeObject() - new_s_obj.__class__ = s_obj.__class__ + new_s_obj = SomeObject.__new__(s_obj.__class__) dic = new_s_obj.__dict__ = s_obj.__dict__.copy() if 'const' in dic: del new_s_obj.const diff --git a/pypy/annotation/test/test_annrpython.py b/pypy/annotation/test/test_annrpython.py --- a/pypy/annotation/test/test_annrpython.py +++ b/pypy/annotation/test/test_annrpython.py @@ -456,6 +456,20 @@ return ''.join(g(n)) s = a.build_types(f, [int]) assert s.knowntype == str + assert s.no_NUL + + def test_str_split(self): + a = self.RPythonAnnotator() + def g(n): + if n: + return "test string" + def f(n): + if n: + return g(n).split(' ') + s = a.build_types(f, [int]) + assert isinstance(s, annmodel.SomeList) + s_item = s.listdef.listitem.s_value + assert s_item.no_NUL def test_str_splitlines(self): a = self.RPythonAnnotator() @@ -1841,7 +1855,7 @@ return obj.indirect() a = self.RPythonAnnotator() s = a.build_types(f, [bool]) - assert s == annmodel.SomeString(can_be_None=True) + assert annmodel.SomeString(can_be_None=True).contains(s) def test_dont_see_AttributeError_clause(self): class Stuff: @@ -2018,6 +2032,37 @@ s = a.build_types(g, [int]) assert not s.can_be_None + def test_string_noNUL_canbeNone(self): + def f(a): + if a: + return "abc" + else: + return None + a = self.RPythonAnnotator() + s = a.build_types(f, [int]) + assert s.can_be_None + assert s.no_NUL + + def test_str_or_None(self): + def f(a): + if a: + return "abc" + else: + return None + def g(a): + x = f(a) + #assert x is not None + if x is None: + return "abcd" + return x + if isinstance(x, str): + return x + return "impossible" + a = self.RPythonAnnotator() + s = a.build_types(f, [int]) + assert s.can_be_None + assert s.no_NUL + def test_emulated_pbc_call_simple(self): def f(a,b): return a + b @@ -2071,6 +2116,19 @@ assert isinstance(s, annmodel.SomeIterator) assert s.variant == ('items',) + def test_iteritems_str0(self): + def it(d): + return d.iteritems() + def f(): + d0 = {'1a': '2a', '3': '4'} + for item in it(d0): + return "%s=%s" % item + raise ValueError + a = self.RPythonAnnotator() + s = a.build_types(f, []) + assert isinstance(s, annmodel.SomeString) + assert s.no_NUL + def test_non_none_and_none_with_isinstance(self): class A(object): pass diff --git a/pypy/annotation/unaryop.py b/pypy/annotation/unaryop.py --- a/pypy/annotation/unaryop.py +++ b/pypy/annotation/unaryop.py @@ -491,7 +491,10 @@ if isinstance(str, SomeUnicodeString): return immutablevalue(u"") return immutablevalue("") - return str.basestringclass() + result = str.basestringclass() + if str.no_NUL and s_item.no_NUL: + result.no_NUL = True + return result def iter(str): return SomeIterator(str) @@ -502,18 +505,28 @@ def method_split(str, patt, max=-1): getbookkeeper().count("str_split", str, patt) - return getbookkeeper().newlist(str.basestringclass()) + s_item = str.basestringclass() + if str.no_NUL: + s_item.no_NUL = True + result = getbookkeeper().newlist(s_item) + return result def method_rsplit(str, patt, max=-1): getbookkeeper().count("str_rsplit", str, patt) - return getbookkeeper().newlist(str.basestringclass()) + result = getbookkeeper().newlist(str.basestringclass()) + if str.no_NUL: + result.no_NUL = True + return result def method_replace(str, s1, s2): return str.basestringclass() def getslice(str, s_start, s_stop): check_negative_slice(s_start, s_stop) - return str.basestringclass() + result = str.basestringclass() + if str.no_NUL: + result.no_NUL = True + return result class __extend__(SomeUnicodeString): def method_encode(uni, s_enc): diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -1312,6 +1312,15 @@ def str_w(self, w_obj): return w_obj.str_w(self) + def str0_w(self, w_obj): + "Like str_w, but rejects strings with NUL bytes." + from pypy.rlib import rstring + result = w_obj.str_w(self) + if '\x00' in result: + raise OperationError(self.w_TypeError, self.wrap( + 'argument must be a string without NUL characters')) + return rstring.assert_str0(result) + def int_w(self, w_obj): return w_obj.int_w(self) diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py --- a/pypy/interpreter/gateway.py +++ b/pypy/interpreter/gateway.py @@ -130,6 +130,9 @@ def visit_str_or_None(self, el, app_sig): self.checked_space_method(el, app_sig) + def visit_str0(self, el, app_sig): + self.checked_space_method(el, app_sig) + def visit_nonnegint(self, el, app_sig): self.checked_space_method(el, app_sig) @@ -249,6 +252,9 @@ def visit_str_or_None(self, typ): self.run_args.append("space.str_or_None_w(%s)" % (self.scopenext(),)) + def visit_str0(self, typ): + self.run_args.append("space.str0_w(%s)" % (self.scopenext(),)) + def visit_nonnegint(self, typ): self.run_args.append("space.gateway_nonnegint_w(%s)" % ( self.scopenext(),)) @@ -383,6 +389,9 @@ def visit_str_or_None(self, typ): self.unwrap.append("space.str_or_None_w(%s)" % (self.nextarg(),)) + def visit_str0(self, typ): + self.unwrap.append("space.str0_w(%s)" % (self.nextarg(),)) + def visit_nonnegint(self, typ): self.unwrap.append("space.gateway_nonnegint_w(%s)" % (self.nextarg(),)) diff --git a/pypy/interpreter/mixedmodule.py b/pypy/interpreter/mixedmodule.py --- a/pypy/interpreter/mixedmodule.py +++ b/pypy/interpreter/mixedmodule.py @@ -50,7 +50,7 @@ space.call_method(self.w_dict, 'update', self.w_initialdict) for w_submodule in self.submodules_w: - name = space.str_w(w_submodule.w_name) + name = space.str0_w(w_submodule.w_name) space.setitem(self.w_dict, space.wrap(name.split(".")[-1]), w_submodule) space.getbuiltinmodule(name) diff --git a/pypy/interpreter/module.py b/pypy/interpreter/module.py --- a/pypy/interpreter/module.py +++ b/pypy/interpreter/module.py @@ -31,7 +31,8 @@ def install(self): """NOT_RPYTHON: installs this module into space.builtin_modules""" w_mod = self.space.wrap(self) - self.space.builtin_modules[self.space.unwrap(self.w_name)] = w_mod + modulename = self.space.str0_w(self.w_name) + self.space.builtin_modules[modulename] = w_mod def setup_after_space_initialization(self): """NOT_RPYTHON: to allow built-in modules to do some more setup diff --git a/pypy/module/bz2/interp_bz2.py b/pypy/module/bz2/interp_bz2.py --- a/pypy/module/bz2/interp_bz2.py +++ b/pypy/module/bz2/interp_bz2.py @@ -328,7 +328,7 @@ if basemode == "a": raise OperationError(space.w_ValueError, space.wrap("cannot append to bz2 file")) - stream = open_path_helper(space.str_w(w_path), os_flags, False) + stream = open_path_helper(space.str0_w(w_path), os_flags, False) if reading: bz2stream = ReadBZ2Filter(space, stream, buffering) buffering = 0 # by construction, the ReadBZ2Filter acts like diff --git a/pypy/module/gc/interp_gc.py b/pypy/module/gc/interp_gc.py --- a/pypy/module/gc/interp_gc.py +++ b/pypy/module/gc/interp_gc.py @@ -49,7 +49,7 @@ # ____________________________________________________________ - at unwrap_spec(filename=str) + at unwrap_spec(filename='str0') def dump_heap_stats(space, filename): tb = rgc._heap_stats() if not tb: diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py --- a/pypy/module/imp/importing.py +++ b/pypy/module/imp/importing.py @@ -138,7 +138,7 @@ ctxt_package = None if ctxt_w_package is not None and ctxt_w_package is not space.w_None: try: - ctxt_package = space.str_w(ctxt_w_package) + ctxt_package = space.str0_w(ctxt_w_package) except OperationError, e: if not e.match(space, space.w_TypeError): raise @@ -187,7 +187,7 @@ ctxt_name = None if ctxt_w_name is not None: try: - ctxt_name = space.str_w(ctxt_w_name) + ctxt_name = space.str0_w(ctxt_w_name) except OperationError, e: if not e.match(space, space.w_TypeError): raise @@ -230,7 +230,7 @@ return rel_modulename, rel_level - at unwrap_spec(name=str, level=int) + at unwrap_spec(name='str0', level=int) def importhook(space, name, w_globals=None, w_locals=None, w_fromlist=None, level=-1): modulename = name @@ -377,8 +377,8 @@ fromlist_w = space.fixedview(w_all) for w_name in fromlist_w: if try_getattr(space, w_mod, w_name) is None: - load_part(space, w_path, prefix, space.str_w(w_name), w_mod, - tentative=1) + load_part(space, w_path, prefix, space.str0_w(w_name), + w_mod, tentative=1) return w_mod else: return first @@ -432,7 +432,7 @@ def __init__(self, space): pass - @unwrap_spec(path=str) + @unwrap_spec(path='str0') def descr_init(self, space, path): if not path: raise OperationError(space.w_ImportError, space.wrap( @@ -513,7 +513,7 @@ if w_loader: return FindInfo.fromLoader(w_loader) - path = space.str_w(w_pathitem) + path = space.str0_w(w_pathitem) filepart = os.path.join(path, partname) if os.path.isdir(filepart) and case_ok(filepart): initfile = os.path.join(filepart, '__init__') @@ -671,7 +671,7 @@ space.wrap("reload() argument must be module")) w_modulename = space.getattr(w_module, space.wrap("__name__")) - modulename = space.str_w(w_modulename) + modulename = space.str0_w(w_modulename) if not space.is_w(check_sys_modules(space, w_modulename), w_module): raise operationerrfmt( space.w_ImportError, diff --git a/pypy/module/imp/interp_imp.py b/pypy/module/imp/interp_imp.py --- a/pypy/module/imp/interp_imp.py +++ b/pypy/module/imp/interp_imp.py @@ -44,7 +44,7 @@ return space.interp_w(W_File, w_file).stream def find_module(space, w_name, w_path=None): - name = space.str_w(w_name) + name = space.str0_w(w_name) if space.is_w(w_path, space.w_None): w_path = None @@ -75,7 +75,7 @@ def load_module(space, w_name, w_file, w_filename, w_info): w_suffix, w_filemode, w_modtype = space.unpackiterable(w_info) - filename = space.str_w(w_filename) + filename = space.str0_w(w_filename) filemode = space.str_w(w_filemode) if space.is_w(w_file, space.w_None): stream = None @@ -92,7 +92,7 @@ space, w_name, find_info, reuse=True) def load_source(space, w_modulename, w_filename, w_file=None): - filename = space.str_w(w_filename) + filename = space.str0_w(w_filename) stream = get_file(space, w_file, filename, 'U') @@ -105,7 +105,7 @@ stream.close() return w_mod - at unwrap_spec(filename=str) + at unwrap_spec(filename='str0') def _run_compiled_module(space, w_modulename, filename, w_file, w_module): # the function 'imp._run_compiled_module' is a pypy-only extension stream = get_file(space, w_file, filename, 'rb') @@ -119,7 +119,7 @@ if space.is_w(w_file, space.w_None): stream.close() - at unwrap_spec(filename=str) + at unwrap_spec(filename='str0') def load_compiled(space, w_modulename, filename, w_file=None): w_mod = space.wrap(Module(space, w_modulename)) importing._prepare_module(space, w_mod, filename, None) @@ -138,7 +138,7 @@ return space.wrap(Module(space, w_name, add_package=False)) def init_builtin(space, w_name): - name = space.str_w(w_name) + name = space.str0_w(w_name) if name not in space.builtin_modules: return if space.finditem(space.sys.get('modules'), w_name) is not None: @@ -151,7 +151,7 @@ return None def is_builtin(space, w_name): - name = space.str_w(w_name) + name = space.str0_w(w_name) if name not in space.builtin_modules: return space.wrap(0) if space.finditem(space.sys.get('modules'), w_name) is not None: diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py --- a/pypy/module/posix/interp_posix.py +++ b/pypy/module/posix/interp_posix.py @@ -37,7 +37,7 @@ if space.isinstance_w(w_obj, space.w_unicode): w_obj = space.call_method(w_obj, 'encode', getfilesystemencoding(space)) - return space.str_w(w_obj) + return space.str0_w(w_obj) class FileEncoder(object): def __init__(self, space, w_obj): @@ -56,7 +56,7 @@ self.w_obj = w_obj def as_bytes(self): - return self.space.str_w(self.w_obj) + return self.space.str0_w(self.w_obj) def as_unicode(self): space = self.space @@ -71,7 +71,7 @@ fname = FileEncoder(space, w_fname) return func(fname, *args) else: - fname = space.str_w(w_fname) + fname = space.str0_w(w_fname) return func(fname, *args) return dispatch @@ -401,7 +401,7 @@ fullpath = rposix._getfullpathname(path) w_fullpath = space.wrap(fullpath) else: - path = space.str_w(w_path) + path = space.str0_w(w_path) fullpath = rposix._getfullpathname(path) w_fullpath = space.wrap(fullpath) except OSError, e: @@ -548,7 +548,7 @@ for s in result ] else: - dirname = space.str_w(w_dirname) + dirname = space.str0_w(w_dirname) result = rposix.listdir(dirname) result_w = [space.wrap(s) for s in result] except OSError, e: @@ -635,7 +635,7 @@ import signal os.kill(os.getpid(), signal.SIGABRT) - at unwrap_spec(src=str, dst=str) + at unwrap_spec(src='str0', dst='str0') def link(space, src, dst): "Create a hard link to a file." try: @@ -650,7 +650,7 @@ except OSError, e: raise wrap_oserror(space, e) - at unwrap_spec(path=str) + at unwrap_spec(path='str0') def readlink(space, path): "Return a string representing the path to which the symbolic link points." try: @@ -765,7 +765,7 @@ w_keys = space.call_method(w_env, 'keys') for w_key in space.unpackiterable(w_keys): w_value = space.getitem(w_env, w_key) - env[space.str_w(w_key)] = space.str_w(w_value) + env[space.str0_w(w_key)] = space.str0_w(w_value) return env def execve(space, w_command, w_args, w_env): @@ -1103,7 +1103,7 @@ except OSError, e: raise wrap_oserror(space, e) - at unwrap_spec(path=str, uid=c_uid_t, gid=c_gid_t) + at unwrap_spec(path='str0', uid=c_uid_t, gid=c_gid_t) def chown(space, path, uid, gid): check_uid_range(space, uid) check_uid_range(space, gid) @@ -1113,7 +1113,7 @@ raise wrap_oserror(space, e, path) return space.w_None - at unwrap_spec(path=str, uid=c_uid_t, gid=c_gid_t) + at unwrap_spec(path='str0', uid=c_uid_t, gid=c_gid_t) def lchown(space, path, uid, gid): check_uid_range(space, uid) check_uid_range(space, gid) diff --git a/pypy/module/sys/state.py b/pypy/module/sys/state.py --- a/pypy/module/sys/state.py +++ b/pypy/module/sys/state.py @@ -74,7 +74,7 @@ # return importlist - at unwrap_spec(srcdir=str) + at unwrap_spec(srcdir='str0') def pypy_initial_path(space, srcdir): try: path = getinitialpath(get(space), srcdir) diff --git a/pypy/module/zipimport/interp_zipimport.py b/pypy/module/zipimport/interp_zipimport.py --- a/pypy/module/zipimport/interp_zipimport.py +++ b/pypy/module/zipimport/interp_zipimport.py @@ -342,7 +342,7 @@ space = self.space return space.wrap(self.filename) - at unwrap_spec(name=str) + at unwrap_spec(name='str0') def descr_new_zipimporter(space, w_type, name): w = space.wrap ok = False diff --git a/pypy/rlib/rstring.py b/pypy/rlib/rstring.py --- a/pypy/rlib/rstring.py +++ b/pypy/rlib/rstring.py @@ -205,3 +205,43 @@ assert p.const is None return SomeUnicodeBuilder(can_be_None=True) +#___________________________________________________________________ +# Support functions for SomeString.no_NUL + +def assert_str0(fname): + assert '\x00' not in fname, "NUL byte in string" + return fname + +class Entry(ExtRegistryEntry): + _about_ = assert_str0 + + def compute_result_annotation(self, s_obj): + assert isinstance(s_obj, (SomeString, SomeUnicodeString)) + if s_obj.no_NUL: + return s_obj + new_s_obj = SomeObject.__new__(s_obj.__class__) + new_s_obj.__dict__ = s_obj.__dict__.copy() + new_s_obj.no_NUL = True + return new_s_obj + + def specialize_call(self, hop): + hop.exception_cannot_occur() + return hop.inputarg(hop.args_r[0], arg=0) + +def check_str0(fname): + """A 'probe' to trigger a failure at translation time, if the + string was not proved to not contain NUL characters.""" + assert '\x00' not in fname, "NUL byte in string" + +class Entry(ExtRegistryEntry): + _about_ = check_str0 + + def compute_result_annotation(self, s_obj): + if not isinstance(s_obj, SomeString): + return s_obj + if not s_obj.no_NUL: + raise ValueError("Value is not no_NUL") + + def specialize_call(self, hop): + pass + diff --git a/pypy/rpython/extfuncregistry.py b/pypy/rpython/extfuncregistry.py --- a/pypy/rpython/extfuncregistry.py +++ b/pypy/rpython/extfuncregistry.py @@ -85,7 +85,8 @@ # llinterpreter path_functions = [ - ('join', [str, str], str), + ('join', [ll_os.str0, ll_os.str0], ll_os.str0), + ('dirname', [ll_os.str0], ll_os.str0), ] for name, args, res in path_functions: diff --git a/pypy/rpython/lltypesystem/rffi.py b/pypy/rpython/lltypesystem/rffi.py --- a/pypy/rpython/lltypesystem/rffi.py +++ b/pypy/rpython/lltypesystem/rffi.py @@ -15,7 +15,7 @@ from pypy.translator.tool.cbuild import ExternalCompilationInfo from pypy.rpython.annlowlevel import llhelper from pypy.rlib.objectmodel import we_are_translated -from pypy.rlib.rstring import StringBuilder, UnicodeBuilder +from pypy.rlib.rstring import StringBuilder, UnicodeBuilder, assert_str0 from pypy.rlib import jit from pypy.rpython.lltypesystem import llmemory import os, sys @@ -694,7 +694,7 @@ while cp[i] != lastchar: b.append(cp[i]) i += 1 - return b.build() + return assert_str0(b.build()) # str -> char* # Can't inline this because of the raw address manipulation. @@ -798,7 +798,7 @@ while i < maxlen and cp[i] != lastchar: b.append(cp[i]) i += 1 - return b.build() + return assert_str0(b.build()) # char* and size -> str (which can contain null bytes) def charpsize2str(cp, size): @@ -836,6 +836,7 @@ array[i] = str2charp(l[i]) array[len(l)] = lltype.nullptr(CCHARP.TO) return array +liststr2charpp._annenforceargs_ = [[annmodel.s_Str0]] # List of strings def free_charpp(ref): """ frees list of char**, NULL terminated diff --git a/pypy/rpython/module/ll_os.py b/pypy/rpython/module/ll_os.py --- a/pypy/rpython/module/ll_os.py +++ b/pypy/rpython/module/ll_os.py @@ -31,6 +31,12 @@ from pypy.rlib import rgc from pypy.rlib.objectmodel import specialize +str0 = SomeString() +str0.no_NUL = True + +unicode0 = SomeUnicodeString() +unicode0.no_NUL = True + def monkeypatch_rposix(posixfunc, unicodefunc, signature): func_name = posixfunc.__name__ @@ -68,6 +74,7 @@ class StringTraits: str = str + str0 = str0 CHAR = rffi.CHAR CCHARP = rffi.CCHARP charp2str = staticmethod(rffi.charp2str) @@ -85,6 +92,7 @@ class UnicodeTraits: str = unicode + str0 = unicode0 CHAR = rffi.WCHAR_T CCHARP = rffi.CWCHARP charp2str = staticmethod(rffi.wcharp2unicode) @@ -301,7 +309,7 @@ rffi.free_charpp(l_args) raise OSError(rposix.get_errno(), "execv failed") - return extdef([str, [str]], s_ImpossibleValue, llimpl=execv_llimpl, + return extdef([str0, [str0]], s_ImpossibleValue, llimpl=execv_llimpl, export_name="ll_os.ll_os_execv") @@ -319,7 +327,8 @@ # appropriate envstrs = [] for item in env.iteritems(): - envstrs.append("%s=%s" % item) + envstr = "%s=%s" % item + envstrs.append(envstr) l_args = rffi.liststr2charpp(args) l_env = rffi.liststr2charpp(envstrs) @@ -332,7 +341,7 @@ raise OSError(rposix.get_errno(), "execve failed") return extdef( - [str, [str], {str: str}], + [str0, [str0], {str0: str0}], s_ImpossibleValue, llimpl=execve_llimpl, export_name="ll_os.ll_os_execve") @@ -353,7 +362,7 @@ raise OSError(rposix.get_errno(), "os_spawnv failed") return rffi.cast(lltype.Signed, childpid) - return extdef([int, str, [str]], int, llimpl=spawnv_llimpl, + return extdef([int, str0, [str0]], int, llimpl=spawnv_llimpl, export_name="ll_os.ll_os_spawnv") @registering_if(os, 'spawnve') @@ -378,7 +387,7 @@ raise OSError(rposix.get_errno(), "os_spawnve failed") return rffi.cast(lltype.Signed, childpid) - return extdef([int, str, [str], {str: str}], int, + return extdef([int, str0, [str0], {str0: str0}], int, llimpl=spawnve_llimpl, export_name="ll_os.ll_os_spawnve") @@ -816,7 +825,7 @@ def os_open_oofakeimpl(path, flags, mode): return os.open(OOSupport.from_rstr(path), flags, mode) - return extdef([traits.str, int, int], int, traits.ll_os_name('open'), + return extdef([str0, int, int], int, traits.ll_os_name('open'), llimpl=os_open_llimpl, oofakeimpl=os_open_oofakeimpl) @registering_if(os, 'getloadavg') @@ -1174,8 +1183,8 @@ raise OSError(error, "os_readdir failed") return result - return extdef([traits.str], # a single argument which is a str - [traits.str], # returns a list of strings + return extdef([traits.str0], # a single argument which is a str + [traits.str0], # returns a list of strings traits.ll_os_name('listdir'), llimpl=os_listdir_llimpl) @@ -1241,7 +1250,7 @@ if res == -1: raise OSError(rposix.get_errno(), "os_chown failed") - return extdef([str, int, int], None, "ll_os.ll_os_chown", + return extdef([str0, int, int], None, "ll_os.ll_os_chown", llimpl=os_chown_llimpl) @registering_if(os, 'lchown') @@ -1254,7 +1263,7 @@ if res == -1: raise OSError(rposix.get_errno(), "os_lchown failed") - return extdef([str, int, int], None, "ll_os.ll_os_lchown", + return extdef([str0, int, int], None, "ll_os.ll_os_lchown", llimpl=os_lchown_llimpl) @registering_if(os, 'readlink') @@ -1283,12 +1292,11 @@ lltype.free(buf, flavor='raw') bufsize *= 4 # convert the result to a string - l = [buf[i] for i in range(res)] - result = ''.join(l) + result = rffi.charp2strn(buf, res) lltype.free(buf, flavor='raw') return result - return extdef([str], str, + return extdef([str0], str0, "ll_os.ll_os_readlink", llimpl=os_readlink_llimpl) @@ -1424,7 +1432,7 @@ if res < 0: raise OSError(rposix.get_errno(), "os_mkdir failed") - return extdef([traits.str, int], s_None, llimpl=os_mkdir_llimpl, + return extdef([traits.str0, int], s_None, llimpl=os_mkdir_llimpl, export_name=traits.ll_os_name('mkdir')) @registering_str_unicode(os.rmdir) @@ -1454,7 +1462,7 @@ from pypy.rpython.module.ll_win32file import make_chmod_impl chmod_llimpl = make_chmod_impl(traits) - return extdef([traits.str, int], s_None, llimpl=chmod_llimpl, + return extdef([traits.str0, int], s_None, llimpl=chmod_llimpl, export_name=traits.ll_os_name('chmod')) @registering_str_unicode(os.rename) @@ -1476,7 +1484,7 @@ if not win32traits.MoveFile(oldpath, newpath): raise rwin32.lastWindowsError() - return extdef([traits.str, traits.str], s_None, llimpl=rename_llimpl, + return extdef([traits.str0, traits.str0], s_None, llimpl=rename_llimpl, export_name=traits.ll_os_name('rename')) @registering_str_unicode(getattr(os, 'mkfifo', None)) @@ -1489,7 +1497,7 @@ if res < 0: raise OSError(rposix.get_errno(), "os_mkfifo failed") - return extdef([traits.str, int], s_None, llimpl=mkfifo_llimpl, + return extdef([traits.str0, int], s_None, llimpl=mkfifo_llimpl, export_name=traits.ll_os_name('mkfifo')) @registering_str_unicode(getattr(os, 'mknod', None)) @@ -1503,7 +1511,7 @@ if res < 0: raise OSError(rposix.get_errno(), "os_mknod failed") - return extdef([traits.str, int, int], s_None, llimpl=mknod_llimpl, + return extdef([traits.str0, int, int], s_None, llimpl=mknod_llimpl, export_name=traits.ll_os_name('mknod')) @registering(os.umask) @@ -1555,7 +1563,7 @@ if res < 0: raise OSError(rposix.get_errno(), "os_link failed") - return extdef([str, str], s_None, llimpl=link_llimpl, + return extdef([str0, str0], s_None, llimpl=link_llimpl, export_name="ll_os.ll_os_link") @registering_if(os, 'symlink') @@ -1568,7 +1576,7 @@ if res < 0: raise OSError(rposix.get_errno(), "os_symlink failed") - return extdef([str, str], s_None, llimpl=symlink_llimpl, + return extdef([str0, str0], s_None, llimpl=symlink_llimpl, export_name="ll_os.ll_os_symlink") @registering_if(os, 'fork') diff --git a/pypy/rpython/module/ll_os_stat.py b/pypy/rpython/module/ll_os_stat.py --- a/pypy/rpython/module/ll_os_stat.py +++ b/pypy/rpython/module/ll_os_stat.py @@ -236,7 +236,7 @@ def register_stat_variant(name, traits): if name != 'fstat': arg_is_path = True - s_arg = traits.str + s_arg = traits.str0 ARG1 = traits.CCHARP else: arg_is_path = False @@ -251,8 +251,6 @@ [s_arg], s_StatResult, traits.ll_os_name(name), llimpl=posix_stat_llimpl) - assert traits.str is str - if sys.platform.startswith('linux'): # because we always use _FILE_OFFSET_BITS 64 - this helps things work that are not a c compiler _functions = {'stat': 'stat64', From noreply at buildbot.pypy.org Sat Jan 28 21:26:26 2012 From: noreply at buildbot.pypy.org (amauryfa) Date: Sat, 28 Jan 2012 21:26:26 +0100 (CET) Subject: [pypy-commit] pypy string-NUL: Rename to SomeString.no_nul Message-ID: <20120128202626.ACA0471068F@wyvern.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc <amauryfa at gmail.com> Branch: string-NUL Changeset: r51939:ad0aa0ec7a90 Date: 2012-01-28 21:26 +0100 http://bitbucket.org/pypy/pypy/changeset/ad0aa0ec7a90/ Log: Rename to SomeString.no_nul diff --git a/pypy/annotation/binaryop.py b/pypy/annotation/binaryop.py --- a/pypy/annotation/binaryop.py +++ b/pypy/annotation/binaryop.py @@ -435,8 +435,8 @@ def union((str1, str2)): result = SomeString(can_be_None=str1.can_be_None or str2.can_be_None) - if str1.no_NUL and str2.no_NUL: - result.no_NUL = True + if str1.no_nul and str2.no_nul: + result.no_nul = True return result def add((str1, str2)): @@ -444,8 +444,8 @@ result = SomeString() if str1.is_immutable_constant() and str2.is_immutable_constant(): result.const = str1.const + str2.const - if str1.no_NUL and str2.no_NUL: - result.no_NUL = True + if str1.no_nul and str2.no_nul: + result.no_nul = True return result class __extend__(pairtype(SomeChar, SomeChar)): @@ -481,17 +481,17 @@ "string formatting mixing strings and unicode not supported") getbookkeeper().count('strformat', str, s_tuple) result = SomeString() - no_NUL = str.no_NUL + no_nul = str.no_nul for s_item in s_tuple.items: if isinstance(s_item, SomeFloat): pass - elif isinstance(s_item, SomeString) and s_item.no_NUL: + elif isinstance(s_item, SomeString) and s_item.no_nul: pass else: - no_NUL = False + no_nul = False break - if no_NUL: - result.no_NUL = True + if no_nul: + result.no_nul = True return result @@ -859,7 +859,7 @@ _make_none_union('SomeInstance', 'classdef=obj.classdef, can_be_None=True') _make_none_union('SomeString', 'can_be_None=True', - copy_attributes=('no_NUL',)) + copy_attributes=('no_nul',)) _make_none_union('SomeUnicodeString', 'can_be_None=True') _make_none_union('SomeList', 'obj.listdef') _make_none_union('SomeDict', 'obj.dictdef') diff --git a/pypy/annotation/bookkeeper.py b/pypy/annotation/bookkeeper.py --- a/pypy/annotation/bookkeeper.py +++ b/pypy/annotation/bookkeeper.py @@ -345,11 +345,11 @@ if len(x) == 1: result = SomeChar() if not '\x00' in x: - result.no_NUL = True + result.no_nul = True else: result = SomeString() if not '\x00' in x: - result.no_NUL = True + result.no_nul = True elif tp is unicode: if len(x) == 1: result = SomeUnicodeCodePoint() diff --git a/pypy/annotation/model.py b/pypy/annotation/model.py --- a/pypy/annotation/model.py +++ b/pypy/annotation/model.py @@ -230,7 +230,7 @@ knowntype = str immutable = True can_be_None=False - no_NUL = False + no_nul = False # No NUL character in the string. def __init__(self, can_be_None=False): if can_be_None: @@ -241,15 +241,15 @@ def nonnoneify(self): result = SomeString(can_be_None=False) - if self.no_NUL: - result.no_NUL = True + if self.no_nul: + result.no_nul = True return result class SomeUnicodeString(SomeObject): "Stands for an object which is known to be an unicode string" knowntype = unicode immutable = True - no_NUL = False + no_nul = False def __init__(self, can_be_None=False): self.can_be_None = can_be_None @@ -511,7 +511,7 @@ s_Bool = SomeBool() s_ImpossibleValue = SomeImpossibleValue() s_Str0 = SomeString() -s_Str0.no_NUL = True +s_Str0.no_nul = True # ____________________________________________________________ # weakrefs diff --git a/pypy/annotation/test/test_annrpython.py b/pypy/annotation/test/test_annrpython.py --- a/pypy/annotation/test/test_annrpython.py +++ b/pypy/annotation/test/test_annrpython.py @@ -456,7 +456,7 @@ return ''.join(g(n)) s = a.build_types(f, [int]) assert s.knowntype == str - assert s.no_NUL + assert s.no_nul def test_str_split(self): a = self.RPythonAnnotator() @@ -469,7 +469,7 @@ s = a.build_types(f, [int]) assert isinstance(s, annmodel.SomeList) s_item = s.listdef.listitem.s_value - assert s_item.no_NUL + assert s_item.no_nul def test_str_splitlines(self): a = self.RPythonAnnotator() @@ -2041,7 +2041,7 @@ a = self.RPythonAnnotator() s = a.build_types(f, [int]) assert s.can_be_None - assert s.no_NUL + assert s.no_nul def test_str_or_None(self): def f(a): @@ -2061,7 +2061,7 @@ a = self.RPythonAnnotator() s = a.build_types(f, [int]) assert s.can_be_None - assert s.no_NUL + assert s.no_nul def test_emulated_pbc_call_simple(self): def f(a,b): @@ -2127,7 +2127,7 @@ a = self.RPythonAnnotator() s = a.build_types(f, []) assert isinstance(s, annmodel.SomeString) - assert s.no_NUL + assert s.no_nul def test_non_none_and_none_with_isinstance(self): class A(object): diff --git a/pypy/annotation/unaryop.py b/pypy/annotation/unaryop.py --- a/pypy/annotation/unaryop.py +++ b/pypy/annotation/unaryop.py @@ -492,8 +492,8 @@ return immutablevalue(u"") return immutablevalue("") result = str.basestringclass() - if str.no_NUL and s_item.no_NUL: - result.no_NUL = True + if str.no_nul and s_item.no_nul: + result.no_nul = True return result def iter(str): @@ -506,16 +506,16 @@ def method_split(str, patt, max=-1): getbookkeeper().count("str_split", str, patt) s_item = str.basestringclass() - if str.no_NUL: - s_item.no_NUL = True + if str.no_nul: + s_item.no_nul = True result = getbookkeeper().newlist(s_item) return result def method_rsplit(str, patt, max=-1): getbookkeeper().count("str_rsplit", str, patt) result = getbookkeeper().newlist(str.basestringclass()) - if str.no_NUL: - result.no_NUL = True + if str.no_nul: + result.no_nul = True return result def method_replace(str, s1, s2): @@ -524,8 +524,8 @@ def getslice(str, s_start, s_stop): check_negative_slice(s_start, s_stop) result = str.basestringclass() - if str.no_NUL: - result.no_NUL = True + if str.no_nul: + result.no_nul = True return result class __extend__(SomeUnicodeString): diff --git a/pypy/rlib/rstring.py b/pypy/rlib/rstring.py --- a/pypy/rlib/rstring.py +++ b/pypy/rlib/rstring.py @@ -206,7 +206,7 @@ return SomeUnicodeBuilder(can_be_None=True) #___________________________________________________________________ -# Support functions for SomeString.no_NUL +# Support functions for SomeString.no_nul def assert_str0(fname): assert '\x00' not in fname, "NUL byte in string" @@ -217,11 +217,11 @@ def compute_result_annotation(self, s_obj): assert isinstance(s_obj, (SomeString, SomeUnicodeString)) - if s_obj.no_NUL: + if s_obj.no_nul: return s_obj new_s_obj = SomeObject.__new__(s_obj.__class__) new_s_obj.__dict__ = s_obj.__dict__.copy() - new_s_obj.no_NUL = True + new_s_obj.no_nul = True return new_s_obj def specialize_call(self, hop): @@ -239,8 +239,8 @@ def compute_result_annotation(self, s_obj): if not isinstance(s_obj, SomeString): return s_obj - if not s_obj.no_NUL: - raise ValueError("Value is not no_NUL") + if not s_obj.no_nul: + raise ValueError("Value is not no_nul") def specialize_call(self, hop): pass diff --git a/pypy/rpython/module/ll_os.py b/pypy/rpython/module/ll_os.py --- a/pypy/rpython/module/ll_os.py +++ b/pypy/rpython/module/ll_os.py @@ -32,10 +32,10 @@ from pypy.rlib.objectmodel import specialize str0 = SomeString() -str0.no_NUL = True +str0.no_nul = True unicode0 = SomeUnicodeString() -unicode0.no_NUL = True +unicode0.no_nul = True def monkeypatch_rposix(posixfunc, unicodefunc, signature): func_name = posixfunc.__name__ From notifications-noreply at bitbucket.org Sun Jan 29 00:09:48 2012 From: notifications-noreply at bitbucket.org (Bitbucket) Date: Sat, 28 Jan 2012 23:09:48 -0000 Subject: [pypy-commit] Notification: pypy Message-ID: <20120128230948.21269.83360@bitbucket01.managed.contegix.com> You have received a notification from David Ripton. Hi, I forked pypy. My fork is at https://bitbucket.org/dripton/pypy. -- Disable notifications at https://bitbucket.org/account/notifications/ From noreply at buildbot.pypy.org Sun Jan 29 00:33:31 2012 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 29 Jan 2012 00:33:31 +0100 (CET) Subject: [pypy-commit] pypy default: Hopefully fix the Win32 translation, which has just hit the limit of Message-ID: <20120128233331.AF12371068F@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: Changeset: r51940:c5e504c2c62b Date: 2012-01-29 00:33 +0100 http://bitbucket.org/pypy/pypy/changeset/c5e504c2c62b/ Log: Hopefully fix the Win32 translation, which has just hit the limit of generating more than 256 different instruction variants. Reduce this number by limiting instructions that have the "/..c.." variant to only the most common instructions. diff --git a/pypy/jit/codewriter/assembler.py b/pypy/jit/codewriter/assembler.py --- a/pypy/jit/codewriter/assembler.py +++ b/pypy/jit/codewriter/assembler.py @@ -131,13 +131,14 @@ self.code.append("temporary placeholder") # argcodes = [] + allow_short = (insn[0] in USE_C_FORM) for x in insn[1:]: if isinstance(x, Register): self.emit_reg(x) argcodes.append(x.kind[0]) elif isinstance(x, Constant): kind = getkind(x.concretetype) - is_short = self.emit_const(x, kind, allow_short=True) + is_short = self.emit_const(x, kind, allow_short=allow_short) if is_short: argcodes.append('c') else: @@ -257,3 +258,47 @@ for func in callinfocollection.all_function_addresses_as_int(): func = heaptracker.int2adr(func) self.see_raw_object(func.ptr) + + +# A set of instructions that use the 'c' encoding for small constants. +# Allowing it anywhere causes the number of instruction variants to +# expode, growing past 256. So we list here only the most common +# instructions where the 'c' variant might be useful. +USE_C_FORM = set([ + 'copystrcontent', + 'copyunicodecontent', + 'getarrayitem_gc_pure_i', + 'getarrayitem_gc_pure_r', + 'getarrayitem_gc_i', + 'getarrayitem_gc_r', + 'goto_if_not_int_eq', + 'goto_if_not_int_ge', + 'goto_if_not_int_gt', + 'goto_if_not_int_le', + 'goto_if_not_int_lt', + 'goto_if_not_int_ne', + 'int_add', + 'int_and', + 'int_copy', + 'int_eq', + 'int_ge', + 'int_gt', + 'int_le', + 'int_lt', + 'int_ne', + 'int_return', + 'int_sub', + 'jit_merge_point', + 'new_array', + 'newstr', + 'recursive_call_i', + 'recursive_call_r', + 'recursive_call_v', + 'setarrayitem_gc_i', + 'setarrayitem_gc_r', + 'setfield_gc_i', + 'strgetitem', + 'strsetitem', + + 'foobar', 'baz', # for tests +]) diff --git a/pypy/jit/codewriter/test/test_assembler.py b/pypy/jit/codewriter/test/test_assembler.py --- a/pypy/jit/codewriter/test/test_assembler.py +++ b/pypy/jit/codewriter/test/test_assembler.py @@ -157,15 +157,18 @@ ('foobar', ListOfKind('int', [Constant(42, lltype.Signed)])), ('foobar', ListOfKind('int', [Constant(42, lltype.Signed)])), ('baz', Constant(42, lltype.Signed)), + ('bok', Constant(41, lltype.Signed)), ] assembler = Assembler() jitcode = assembler.assemble(ssarepr) assert jitcode.code == ("\x00\x01\xFF" "\x00\x01\xFF" - "\x01\x2A") + "\x01\x2A" + "\x02\xFE") assert assembler.insns == {'foobar/I': 0, - 'baz/c': 1} - assert jitcode.constants_i == [42] + 'baz/c': 1, # in USE_C_FORM + 'bok/i': 2} # not in USE_C_FORM + assert jitcode.constants_i == [42, 41] def test_assemble_descr(): class FooDescr(AbstractDescr): From noreply at buildbot.pypy.org Sun Jan 29 00:34:40 2012 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 29 Jan 2012 00:34:40 +0100 (CET) Subject: [pypy-commit] pypy default: Unification: the other unicode operations are not in the list either. Message-ID: <20120128233440.6A27371068F@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: Changeset: r51941:b0d540cb3d25 Date: 2012-01-29 00:34 +0100 http://bitbucket.org/pypy/pypy/changeset/b0d540cb3d25/ Log: Unification: the other unicode operations are not in the list either. diff --git a/pypy/jit/codewriter/assembler.py b/pypy/jit/codewriter/assembler.py --- a/pypy/jit/codewriter/assembler.py +++ b/pypy/jit/codewriter/assembler.py @@ -266,7 +266,6 @@ # instructions where the 'c' variant might be useful. USE_C_FORM = set([ 'copystrcontent', - 'copyunicodecontent', 'getarrayitem_gc_pure_i', 'getarrayitem_gc_pure_r', 'getarrayitem_gc_i', From noreply at buildbot.pypy.org Sun Jan 29 00:38:49 2012 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 29 Jan 2012 00:38:49 +0100 (CET) Subject: [pypy-commit] pypy default: these three operations are also rare Message-ID: <20120128233849.DBB5471068F@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: Changeset: r51942:b38d41e81500 Date: 2012-01-29 00:38 +0100 http://bitbucket.org/pypy/pypy/changeset/b38d41e81500/ Log: these three operations are also rare diff --git a/pypy/jit/codewriter/assembler.py b/pypy/jit/codewriter/assembler.py --- a/pypy/jit/codewriter/assembler.py +++ b/pypy/jit/codewriter/assembler.py @@ -290,9 +290,6 @@ 'jit_merge_point', 'new_array', 'newstr', - 'recursive_call_i', - 'recursive_call_r', - 'recursive_call_v', 'setarrayitem_gc_i', 'setarrayitem_gc_r', 'setfield_gc_i', From pullrequests-noreply at bitbucket.org Sun Jan 29 01:19:48 2012 From: pullrequests-noreply at bitbucket.org (Bitbucket) Date: Sun, 29 Jan 2012 00:19:48 -0000 Subject: [pypy-commit] [OPEN] Pull request #24 for pypy/pypy: Apply hpaulj's patch to fix issue950 (startup_hook in readline / pyrepl) Message-ID: <pullreq-c8bec90e96cf@bitbucket.org> A new pull request has been opened by David Ripton. dripton/pypy has changes to be pulled into pypy/pypy. https://bitbucket.org/pypy/pypy/pull-request/24/apply-hpauljs-patch-to-fix-issue950 Title: Apply hpaulj's patch to fix issue950 (startup_hook in readline / pyrepl) I translated PyPy with this fix and verified that it works the same as CPython on the test case in issue950. Changes to be pulled: -- This is an issue notification from bitbucket.org. You are receiving this either because you are the participating in a pull request, or you are following it. From noreply at buildbot.pypy.org Sun Jan 29 02:51:38 2012 From: noreply at buildbot.pypy.org (dripton) Date: Sun, 29 Jan 2012 02:51:38 +0100 (CET) Subject: [pypy-commit] pypy default: (hpaulj) Fix startup_hook in readline to work like CPython. Fixes issue950 Message-ID: <20120129015138.6466A71068F@wyvern.cs.uni-duesseldorf.de> Author: David Ripton <dripton at ripton.net> Branch: Changeset: r51943:c8bec90e96cf Date: 2012-01-28 19:17 -0500 http://bitbucket.org/pypy/pypy/changeset/c8bec90e96cf/ Log: (hpaulj) Fix startup_hook in readline to work like CPython. Fixes issue950 diff --git a/lib_pypy/pyrepl/reader.py b/lib_pypy/pyrepl/reader.py --- a/lib_pypy/pyrepl/reader.py +++ b/lib_pypy/pyrepl/reader.py @@ -37,16 +37,16 @@ for i in range(256): c = unichr(i) if not uc_map.has_key(c): - uc_map[c] = u'\\%03o'%i + uc_map[c] = u'\\%03o'%i return uc_map # disp_str proved to be a bottleneck for large inputs, so it's been # rewritten in C; it's not required though. try: raise ImportError # currently it's borked by the unicode support - + from _pyrepl_utils import disp_str, init_unctrl_map - + init_unctrl_map(_make_unctrl_map()) del init_unctrl_map @@ -118,7 +118,7 @@ (r'\C-x\C-u', 'upcase-region'), (r'\C-y', 'yank'), (r'\C-z', 'suspend'), - + (r'\M-b', 'backward-word'), (r'\M-c', 'capitalize-word'), (r'\M-d', 'kill-word'), @@ -303,7 +303,7 @@ def process_prompt(self, prompt): """ Process the prompt. - + This means calculate the length of the prompt. The character \x01 and \x02 are used to bracket ANSI control sequences and need to be excluded from the length calculation. So also a copy of the prompt @@ -372,7 +372,7 @@ while p >= 0 and b[p] <> '\n': p -= 1 return p + 1 - + def eol(self, p=None): """Return the 0-based index of the line break following p most immediately. @@ -463,7 +463,7 @@ """This function is called to allow post command cleanup.""" if getattr(cmd, "kills_digit_arg", 1): if self.arg is not None: - self.dirty = 1 + self.dirty = 1 self.arg = None def prepare(self): @@ -575,13 +575,15 @@ def push_char(self, char): self.console.push_char(char) self.handle1(0) - - def readline(self, returns_unicode=False): + + def readline(self, returns_unicode=False, startup_hook=None): """Read a line. The implementation of this method also shows how to drive Reader if you want more control over the event loop.""" self.prepare() try: + if startup_hook is not None: + startup_hook() self.refresh() while not self.finished: self.handle1() diff --git a/lib_pypy/pyrepl/readline.py b/lib_pypy/pyrepl/readline.py --- a/lib_pypy/pyrepl/readline.py +++ b/lib_pypy/pyrepl/readline.py @@ -193,10 +193,8 @@ reader = self.get_reader() except _error: return _old_raw_input(prompt) - if self.startup_hook is not None: - self.startup_hook() reader.ps1 = prompt - return reader.readline() + return reader.readline(reader, startup_hook=self.startup_hook) def multiline_input(self, more_lines, ps1, ps2, returns_unicode=False): """Read an input on possibly multiple lines, asking for more @@ -390,7 +388,7 @@ global _old_raw_input if _old_raw_input is not None: return # don't run _setup twice - + try: f_in = sys.stdin.fileno() f_out = sys.stdout.fileno() From notifications-noreply at bitbucket.org Sun Jan 29 03:48:15 2012 From: notifications-noreply at bitbucket.org (Bitbucket) Date: Sun, 29 Jan 2012 02:48:15 -0000 Subject: [pypy-commit] [COMMENT] Pull request #24 for pypy/pypy: Apply hpaulj's patch to fix issue950 (startup_hook in readline / pyrepl) Message-ID: <20120129024815.2675.12930@bitbucket02.managed.contegix.com> New comment on pull request: https://bitbucket.org/pypy/pypy/pull-request/24/apply-hpauljs-patch-to-fix-issue950#comment-2136 David Ripton (dripton) said: fijal merged this pull request, so it can be deleted. -- This is a pull request comment notification from bitbucket.org. You are receiving this either because you are participating in a pull request, or you are following it. From noreply at buildbot.pypy.org Sun Jan 29 18:16:38 2012 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 29 Jan 2012 18:16:38 +0100 (CET) Subject: [pypy-commit] pypy.org extradoc: Mention openssl 0.9.8. Message-ID: <20120129171638.C173A8203C@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: extradoc Changeset: r308:6cb63133ae6d Date: 2012-01-29 18:16 +0100 http://bitbucket.org/pypy/pypy.org/changeset/6cb63133ae6d/ Log: Mention openssl 0.9.8. diff --git a/download.html b/download.html --- a/download.html +++ b/download.html @@ -56,7 +56,10 @@ <li><a class="reference internal" href="#other-versions">Other versions</a></li> </ul> </li> -<li><a class="reference internal" href="#installing">Installing</a> (optional)</li> +<li><a class="reference internal" href="#installing">Installing</a> (optional)<ul> +<li><a class="reference internal" href="#linux-libraries">Linux libraries</a></li> +</ul> +</li> <li><a class="reference internal" href="#building-from-source">Building from source</a></li> <li><a class="reference internal" href="#packaging">Packaging</a></li> <li><a class="reference internal" href="#checksums">Checksums</a></li> @@ -70,8 +73,8 @@ (This is the official release 1.7; for the most up-to-date version see below.)</p> <ul class="simple"> -<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy-1.7-linux.tar.bz2">Linux binary (32bit)</a></li> -<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy-1.7-linux64.tar.bz2">Linux binary (64bit)</a></li> +<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy-1.7-linux.tar.bz2">Linux binary (32bit)</a> (<a class="reference internal" href="#openssl0-9-8-notes">openssl0.9.8 notes</a>)</li> +<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy-1.7-linux64.tar.bz2">Linux binary (64bit)</a> (<a class="reference internal" href="#openssl0-9-8-notes">openssl0.9.8 notes</a>)</li> <li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy-1.7-osx64.tar.bz2">Mac OS/X binary (64bit)</a></li> <li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy-1.7-win32.zip">Windows binary (32bit)</a></li> </ul><p>If your CPU is really old, it may not have SSE2. In this case, you need @@ -105,6 +108,15 @@ not move or copy the executable <tt class="docutils literal">pypy</tt> outside the tree – put a symlink to it, otherwise it will not find its libraries.</p> </div> +<div class="section" id="linux-libraries"> +<span id="openssl0-9-8-notes"></span><h1>Linux libraries</h1> +<p>Linux binaries require a system with the correct set of libraries. +Right now one problem is OpenSSL, which some distributions distribute in +version 0.9.8 and others in version 1.0.0. The binaries require 0.9.8. +If you can't install it in your distribution, pick it the latest +<a class="reference external" href="http://www.openssl.org/source/">openssl-0.9.8*.tar.gz from sources</a> and install it with <tt class="docutils literal">./config +shared; make; make install</tt>.</p> +</div> <div class="section" id="building-from-source"> <span id="translate"></span><h1>Building from source</h1> <ol class="arabic"> diff --git a/source/download.txt b/source/download.txt --- a/source/download.txt +++ b/source/download.txt @@ -23,6 +23,9 @@ * `Other versions`_ * `Installing`_ (optional) + + * `Linux libraries`_ + * `Building from source`_ * `Packaging`_ * `Checksums`_ @@ -39,8 +42,8 @@ (This is the official release 1.7; for the most up-to-date version see below.) -* `Linux binary (32bit)`__ -* `Linux binary (64bit)`__ +* `Linux binary (32bit)`__ (`openssl0.9.8 notes`_) +* `Linux binary (64bit)`__ (`openssl0.9.8 notes`_) * `Mac OS/X binary (64bit)`__ * `Windows binary (32bit)`__ @@ -91,6 +94,21 @@ a symlink to it, otherwise it will not find its libraries. +.. _`openssl0.9.8 notes`: + +Linux libraries +--------------- + +Linux binaries require a system with the correct set of libraries. +Right now one problem is OpenSSL, which some distributions distribute in +version 0.9.8 and others in version 1.0.0. The binaries require 0.9.8. +If you can't install it in your distribution, pick it the latest +`openssl-0.9.8*.tar.gz from sources`__ and install it with ``./config +shared; make; make install``. + +.. __: http://www.openssl.org/source/ + + .. _translate: Building from source From noreply at buildbot.pypy.org Sun Jan 29 19:14:30 2012 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 29 Jan 2012 19:14:30 +0100 (CET) Subject: [pypy-commit] extradoc extradoc: fix Message-ID: <20120129181430.DBD478203C@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: extradoc Changeset: r4069:94ccf2eee106 Date: 2012-01-29 19:14 +0100 http://bitbucket.org/pypy/extradoc/changeset/94ccf2eee106/ Log: fix diff --git a/planning/stm.txt b/planning/stm.txt --- a/planning/stm.txt +++ b/planning/stm.txt @@ -191,8 +191,8 @@ The major collection could be orchestrated by either the thread that noticed one should start, or by its own thread. We first acquire all the LILs, and for every LIL, we ask the corresponding thread to do a -local marking, starting from their own stacks. Out of this, we obtain a -list of global objects. +local marking, starting from their own stacks and scanning their local +nurseries. Out of this, we obtain a list of global objects. Then we can resume running the threads while at the same time doing a mark-n-sweep collection of the global objects. There is never any @@ -215,8 +215,8 @@ like scan the heap, and at the same time another thread changes some object from the heap. -* Parallelism: there are multiple threads all going something GC-related, - like all scan the heap together. +* Parallelism: there are multiple threads all doing something GC-related, + like all scanning the heap together. When not running transactively @@ -257,9 +257,9 @@ If we know statically that both copies are local copies, then we can just compare the pointers. Otherwise we need to check their GC_GLOBAL and GC_WAS_COPIED flag, and potentially if they both have GC_WAS_COPIED -but only one of them as GC_GLOBAL, we need to check in local dictionary -if they map to each other. And we need to take care of the cases of -NULL pointers. +but only one of them has GC_GLOBAL, we need to check in the local +dictionary if they map to each other. And we need to take care of the +cases of NULL pointers. From noreply at buildbot.pypy.org Sun Jan 29 19:17:11 2012 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Sun, 29 Jan 2012 19:17:11 +0100 (CET) Subject: [pypy-commit] pypy default: Collect data on flatiter as well. Message-ID: <20120129181711.B7A198203C@wyvern.cs.uni-duesseldorf.de> Author: Alex Gaynor <alex.gaynor at gmail.com> Branch: Changeset: r51945:713d8bc8fbc7 Date: 2012-01-29 13:16 -0500 http://bitbucket.org/pypy/pypy/changeset/713d8bc8fbc7/ Log: Collect data on flatiter as well. diff --git a/pypy/module/micronumpy/tool/numready/main.py b/pypy/module/micronumpy/tool/numready/main.py --- a/pypy/module/micronumpy/tool/numready/main.py +++ b/pypy/module/micronumpy/tool/numready/main.py @@ -88,7 +88,7 @@ l[i].append(lst[k * lgt + i]) return l -SPECIAL_NAMES = ["ndarray", "dtype", "generic"] +SPECIAL_NAMES = ["ndarray", "dtype", "generic", "flatiter"] def main(argv): cpy_items = find_numpy_items("/usr/bin/python") From noreply at buildbot.pypy.org Sun Jan 29 19:20:47 2012 From: noreply at buildbot.pypy.org (amauryfa) Date: Sun, 29 Jan 2012 19:20:47 +0100 (CET) Subject: [pypy-commit] pypy string-NUL: Disallow nul bytes in more functions Message-ID: <20120129182047.D3E948203C@wyvern.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc <amauryfa at gmail.com> Branch: string-NUL Changeset: r51946:28448014baa2 Date: 2012-01-28 23:00 +0100 http://bitbucket.org/pypy/pypy/changeset/28448014baa2/ Log: Disallow nul bytes in more functions diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py --- a/pypy/module/posix/interp_posix.py +++ b/pypy/module/posix/interp_posix.py @@ -369,7 +369,7 @@ space.wrap(times[3]), space.wrap(times[4])]) - at unwrap_spec(cmd=str) + at unwrap_spec(cmd='str0') def system(space, cmd): """Execute the command (a string) in a subshell.""" try: @@ -512,7 +512,7 @@ for key, value in os.environ.items(): space.setitem(w_env, space.wrap(key), space.wrap(value)) - at unwrap_spec(name=str, value=str) + at unwrap_spec(name='str0', value='str0') def putenv(space, name, value): """Change or add an environment variable.""" try: @@ -520,7 +520,7 @@ except OSError, e: raise wrap_oserror(space, e) - at unwrap_spec(name=str) + at unwrap_spec(name='str0') def unsetenv(space, name): """Delete an environment variable.""" try: @@ -785,18 +785,18 @@ except OSError, e: raise wrap_oserror(space, e) - at unwrap_spec(mode=int, path=str) + at unwrap_spec(mode=int, path='str0') def spawnv(space, mode, path, w_args): - args = [space.str_w(w_arg) for w_arg in space.unpackiterable(w_args)] + args = [space.str0_w(w_arg) for w_arg in space.unpackiterable(w_args)] try: ret = os.spawnv(mode, path, args) except OSError, e: raise wrap_oserror(space, e) return space.wrap(ret) - at unwrap_spec(mode=int, path=str) + at unwrap_spec(mode=int, path='str0') def spawnve(space, mode, path, w_args, w_env): - args = [space.str_w(w_arg) for w_arg in space.unpackiterable(w_args)] + args = [space.str0_w(w_arg) for w_arg in space.unpackiterable(w_args)] env = _env2interp(space, w_env) try: ret = os.spawnve(mode, path, args, env) @@ -914,7 +914,7 @@ raise wrap_oserror(space, e) return space.w_None - at unwrap_spec(path=str) + at unwrap_spec(path='str0') def chroot(space, path): """ chroot(path) diff --git a/pypy/rpython/module/ll_os.py b/pypy/rpython/module/ll_os.py --- a/pypy/rpython/module/ll_os.py +++ b/pypy/rpython/module/ll_os.py @@ -526,7 +526,7 @@ else: raise Exception("os.utime() arg 2 must be None or a tuple of " "2 floats, got %s" % (s_times,)) - os_utime_normalize_args._default_signature_ = [traits.str, None] + os_utime_normalize_args._default_signature_ = [traits.str0, None] return extdef(os_utime_normalize_args, s_None, "ll_os.ll_os_utime", @@ -621,7 +621,7 @@ if result == -1: raise OSError(rposix.get_errno(), "os_chroot failed") - return extdef([str], None, export_name="ll_os.ll_os_chroot", + return extdef([str0], None, export_name="ll_os.ll_os_chroot", llimpl=chroot_llimpl) @registering_if(os, 'uname') @@ -1059,7 +1059,7 @@ def os_access_oofakeimpl(path, mode): return os.access(OOSupport.from_rstr(path), mode) - return extdef([traits.str, int], s_Bool, llimpl=access_llimpl, + return extdef([traits.str0, int], s_Bool, llimpl=access_llimpl, export_name=traits.ll_os_name("access"), oofakeimpl=os_access_oofakeimpl) @@ -1071,8 +1071,8 @@ from pypy.rpython.module.ll_win32file import make_getfullpathname_impl getfullpathname_llimpl = make_getfullpathname_impl(traits) - return extdef([traits.str], # a single argument which is a str - traits.str, # returns a string + return extdef([traits.str0], # a single argument which is a str + traits.str0, # returns a string traits.ll_os_name('_getfullpathname'), llimpl=getfullpathname_llimpl) @@ -1369,7 +1369,7 @@ res = os_system(command) return rffi.cast(lltype.Signed, res) - return extdef([str], int, llimpl=system_llimpl, + return extdef([str0], int, llimpl=system_llimpl, export_name="ll_os.ll_os_system") @registering_str_unicode(os.unlink) @@ -1391,7 +1391,7 @@ if not win32traits.DeleteFile(path): raise rwin32.lastWindowsError() - return extdef([traits.str], s_None, llimpl=unlink_llimpl, + return extdef([traits.str0], s_None, llimpl=unlink_llimpl, export_name=traits.ll_os_name('unlink')) @registering_str_unicode(os.chdir) @@ -1409,7 +1409,7 @@ from pypy.rpython.module.ll_win32file import make_chdir_impl os_chdir_llimpl = make_chdir_impl(traits) - return extdef([traits.str], s_None, llimpl=os_chdir_llimpl, + return extdef([traits.str0], s_None, llimpl=os_chdir_llimpl, export_name=traits.ll_os_name('chdir')) @registering_str_unicode(os.mkdir) @@ -1445,7 +1445,7 @@ if res < 0: raise OSError(rposix.get_errno(), "os_rmdir failed") - return extdef([traits.str], s_None, llimpl=rmdir_llimpl, + return extdef([traits.str0], s_None, llimpl=rmdir_llimpl, export_name=traits.ll_os_name('rmdir')) @registering_str_unicode(os.chmod) diff --git a/pypy/rpython/module/ll_os_environ.py b/pypy/rpython/module/ll_os_environ.py --- a/pypy/rpython/module/ll_os_environ.py +++ b/pypy/rpython/module/ll_os_environ.py @@ -5,6 +5,8 @@ from pypy.rpython.lltypesystem import rffi, lltype from pypy.rlib import rposix +str0 = annmodel.s_Str0 + # ____________________________________________________________ # # Annotation support to control access to 'os.environ' in the RPython program @@ -64,7 +66,7 @@ rffi.free_charp(l_name) return result -register_external(r_getenv, [str], annmodel.SomeString(can_be_None=True), +register_external(r_getenv, [str0], annmodel.SomeString(can_be_None=True), export_name='ll_os.ll_os_getenv', llimpl=getenv_llimpl) @@ -93,7 +95,7 @@ if l_oldstring: rffi.free_charp(l_oldstring) -register_external(r_putenv, [str, str], annmodel.s_None, +register_external(r_putenv, [str0, str0], annmodel.s_None, export_name='ll_os.ll_os_putenv', llimpl=putenv_llimpl) @@ -128,7 +130,7 @@ del envkeepalive.byname[name] rffi.free_charp(l_oldstring) - register_external(r_unsetenv, [str], annmodel.s_None, + register_external(r_unsetenv, [str0], annmodel.s_None, export_name='ll_os.ll_os_unsetenv', llimpl=unsetenv_llimpl) @@ -172,7 +174,7 @@ i += 1 return result -register_external(r_envkeys, [], [str], # returns a list of strings +register_external(r_envkeys, [], [str0], # returns a list of strings export_name='ll_os.ll_os_envkeys', llimpl=envkeys_llimpl) @@ -193,6 +195,6 @@ i += 1 return result -register_external(r_envitems, [], [(str, str)], +register_external(r_envitems, [], [(str0, str0)], export_name='ll_os.ll_os_envitems', llimpl=envitems_llimpl) diff --git a/pypy/translator/goal/nanos.py b/pypy/translator/goal/nanos.py --- a/pypy/translator/goal/nanos.py +++ b/pypy/translator/goal/nanos.py @@ -266,7 +266,7 @@ raise NotImplementedError("os.name == %r" % (os.name,)) def getenv(space, w_name): - name = space.str_w(w_name) + name = space.str0_w(w_name) return space.wrap(os.environ.get(name)) getenv_w = interp2app(getenv) From noreply at buildbot.pypy.org Sun Jan 29 19:20:49 2012 From: noreply at buildbot.pypy.org (amauryfa) Date: Sun, 29 Jan 2012 19:20:49 +0100 (CET) Subject: [pypy-commit] pypy string-NUL: Fix remaining test failures Message-ID: <20120129182049.1C9508203C@wyvern.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc <amauryfa at gmail.com> Branch: string-NUL Changeset: r51947:b591a6e4e204 Date: 2012-01-29 00:30 +0100 http://bitbucket.org/pypy/pypy/changeset/b591a6e4e204/ Log: Fix remaining test failures diff --git a/pypy/rlib/test/test_rmarshal.py b/pypy/rlib/test/test_rmarshal.py --- a/pypy/rlib/test/test_rmarshal.py +++ b/pypy/rlib/test/test_rmarshal.py @@ -169,7 +169,7 @@ assert st2.st_mode == st.st_mode assert st2[9] == st[9] return buf - fn = compile(f, [str]) + fn = compile(f, [annmodel.s_Str0]) res = fn('.') st = os.stat('.') sttuple = marshal.loads(res) diff --git a/pypy/rpython/module/ll_os_stat.py b/pypy/rpython/module/ll_os_stat.py --- a/pypy/rpython/module/ll_os_stat.py +++ b/pypy/rpython/module/ll_os_stat.py @@ -281,7 +281,7 @@ @func_renamer('os_%s_fake' % (name,)) def posix_fakeimpl(arg): - if s_arg == str: + if s_arg == traits.str0: arg = hlstr(arg) st = getattr(os, name)(arg) fields = [TYPE for fieldname, TYPE in STAT_FIELDS] diff --git a/pypy/rpython/ootypesystem/test/test_ooann.py b/pypy/rpython/ootypesystem/test/test_ooann.py --- a/pypy/rpython/ootypesystem/test/test_ooann.py +++ b/pypy/rpython/ootypesystem/test/test_ooann.py @@ -231,7 +231,7 @@ a = RPythonAnnotator() s = a.build_types(oof, [bool]) - assert s == annmodel.SomeString(can_be_None=True) + assert annmodel.SomeString(can_be_None=True).contains(s) def test_oostring(): def oof(): diff --git a/pypy/translator/c/test/test_extfunc.py b/pypy/translator/c/test/test_extfunc.py --- a/pypy/translator/c/test/test_extfunc.py +++ b/pypy/translator/c/test/test_extfunc.py @@ -3,6 +3,7 @@ import os, time, sys from pypy.tool.udir import udir from pypy.rlib.rarithmetic import r_longlong +from pypy.annotation import model as annmodel from pypy.translator.c.test.test_genc import compile from pypy.translator.c.test.test_standalone import StandaloneTests posix = __import__(os.name) @@ -325,7 +326,7 @@ os.rmdir(path) else: os.mkdir(path, 0777) - f1 = compile(does_stuff, [str, bool]) + f1 = compile(does_stuff, [annmodel.s_Str0, bool]) dirname = str(udir.join('test_mkdir_rmdir')) f1(dirname, False) assert os.path.exists(dirname) and os.path.isdir(dirname) @@ -743,7 +744,7 @@ raise AssertionError("should have failed!") result = os.listdir(s) return '/'.join(result) - func = compile(mylistdir, [str]) + func = compile(mylistdir, [annmodel.s_Str0]) for testdir in [str(udir), os.curdir]: result = func(testdir) result = result.split('/') diff --git a/pypy/translator/cli/test/runtest.py b/pypy/translator/cli/test/runtest.py --- a/pypy/translator/cli/test/runtest.py +++ b/pypy/translator/cli/test/runtest.py @@ -276,9 +276,11 @@ def get_annotation(x): if isinstance(x, basestring) and len(x) > 1: - return SomeString() + result = SomeString() else: - return lltype_to_annotation(typeOf(x)) + result = lltype_to_annotation(typeOf(x)) + if isinstance(result, SomeString) and '\x00' not in x: + result.no_nul = True class CliTest(BaseRtypingTest, OORtypeMixin): def __init__(self): diff --git a/pypy/translator/driver.py b/pypy/translator/driver.py --- a/pypy/translator/driver.py +++ b/pypy/translator/driver.py @@ -184,7 +184,8 @@ self.standalone = standalone if standalone: - inputtypes = [s_list_of_strings] + # the 'argv' parameter + inputtypes = [[annmodel.s_Str0]] self.inputtypes = inputtypes if policy is None: From noreply at buildbot.pypy.org Sun Jan 29 19:20:50 2012 From: noreply at buildbot.pypy.org (amauryfa) Date: Sun, 29 Jan 2012 19:20:50 +0100 (CET) Subject: [pypy-commit] pypy merge-2.7.2: Add a copy of test_str.py, to be modified Message-ID: <20120129182050.6CBE08203C@wyvern.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc <amauryfa at gmail.com> Branch: merge-2.7.2 Changeset: r51948:795bf94e952a Date: 2012-01-29 13:18 +0100 http://bitbucket.org/pypy/pypy/changeset/795bf94e952a/ Log: Add a copy of test_str.py, to be modified diff --git a/lib-python/2.7/test/test_str.py b/lib-python/modified-2.7/test/test_str.py copy from lib-python/2.7/test/test_str.py copy to lib-python/modified-2.7/test/test_str.py From noreply at buildbot.pypy.org Sun Jan 29 19:20:51 2012 From: noreply at buildbot.pypy.org (amauryfa) Date: Sun, 29 Jan 2012 19:20:51 +0100 (CET) Subject: [pypy-commit] pypy merge-2.7.2: Skip check on the error message: pypy's multimethods are quite different Message-ID: <20120129182051.9F8B88203C@wyvern.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc <amauryfa at gmail.com> Branch: merge-2.7.2 Changeset: r51949:dc65c138fd25 Date: 2012-01-29 13:22 +0100 http://bitbucket.org/pypy/pypy/changeset/dc65c138fd25/ Log: Skip check on the error message: pypy's multimethods are quite different diff --git a/lib-python/modified-2.7/test/test_str.py b/lib-python/modified-2.7/test/test_str.py --- a/lib-python/modified-2.7/test/test_str.py +++ b/lib-python/modified-2.7/test/test_str.py @@ -422,10 +422,11 @@ for meth in ('foo'.startswith, 'foo'.endswith): with self.assertRaises(TypeError) as cm: meth(['f']) - exc = str(cm.exception) - self.assertIn('unicode', exc) - self.assertIn('str', exc) - self.assertIn('tuple', exc) + if test_support.check_impl_detail(): + exc = str(cm.exception) + self.assertIn('unicode', exc) + self.assertIn('str', exc) + self.assertIn('tuple', exc) def test_main(): test_support.run_unittest(StrTest) diff --git a/lib-python/modified-2.7/test/test_unicode.py b/lib-python/modified-2.7/test/test_unicode.py --- a/lib-python/modified-2.7/test/test_unicode.py +++ b/lib-python/modified-2.7/test/test_unicode.py @@ -448,10 +448,11 @@ meth('\xff') with self.assertRaises(TypeError) as cm: meth(['f']) - exc = str(cm.exception) - self.assertIn('unicode', exc) - self.assertIn('str', exc) - self.assertIn('tuple', exc) + if test_support.check_impl_detail(): + exc = str(cm.exception) + self.assertIn('unicode', exc) + self.assertIn('str', exc) + self.assertIn('tuple', exc) @test_support.run_with_locale('LC_ALL', 'de_DE', 'fr_FR') def test_format_float(self): From noreply at buildbot.pypy.org Sun Jan 29 19:20:52 2012 From: noreply at buildbot.pypy.org (amauryfa) Date: Sun, 29 Jan 2012 19:20:52 +0100 (CET) Subject: [pypy-commit] pypy merge-2.7.2: tets_socket: Loosen comparison of error messages. Message-ID: <20120129182052.D2E408203C@wyvern.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc <amauryfa at gmail.com> Branch: merge-2.7.2 Changeset: r51950:674ef5a78812 Date: 2012-01-29 14:27 +0100 http://bitbucket.org/pypy/pypy/changeset/674ef5a78812/ Log: tets_socket: Loosen comparison of error messages. diff --git a/lib-python/modified-2.7/test/test_socket.py b/lib-python/modified-2.7/test/test_socket.py --- a/lib-python/modified-2.7/test/test_socket.py +++ b/lib-python/modified-2.7/test/test_socket.py @@ -286,32 +286,34 @@ s.sendto(u'\u2620', sockname) with self.assertRaises(TypeError) as cm: s.sendto(5j, sockname) - self.assertIn('not complex', str(cm.exception)) + self.assertIn('complex', str(cm.exception)) with self.assertRaises(TypeError) as cm: s.sendto('foo', None) - self.assertIn('not NoneType', str(cm.exception)) + self.assertIn('NoneType', str(cm.exception)) # 3 args with self.assertRaises(UnicodeEncodeError): s.sendto(u'\u2620', 0, sockname) with self.assertRaises(TypeError) as cm: s.sendto(5j, 0, sockname) - self.assertIn('not complex', str(cm.exception)) + self.assertIn('complex', str(cm.exception)) with self.assertRaises(TypeError) as cm: s.sendto('foo', 0, None) - self.assertIn('not NoneType', str(cm.exception)) + if test_support.check_impl_detail(): + self.assertIn('not NoneType', str(cm.exception)) with self.assertRaises(TypeError) as cm: s.sendto('foo', 'bar', sockname) - self.assertIn('an integer is required', str(cm.exception)) + self.assertIn('integer', str(cm.exception)) with self.assertRaises(TypeError) as cm: s.sendto('foo', None, None) - self.assertIn('an integer is required', str(cm.exception)) + if test_support.check_impl_detail(): + self.assertIn('an integer is required', str(cm.exception)) # wrong number of args with self.assertRaises(TypeError) as cm: s.sendto('foo') - self.assertIn('(1 given)', str(cm.exception)) + self.assertIn(' given)', str(cm.exception)) with self.assertRaises(TypeError) as cm: s.sendto('foo', 0, sockname, 4) - self.assertIn('(4 given)', str(cm.exception)) + self.assertIn(' given)', str(cm.exception)) def testCrucialConstants(self): From noreply at buildbot.pypy.org Sun Jan 29 19:56:55 2012 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 29 Jan 2012 19:56:55 +0100 (CET) Subject: [pypy-commit] pypy default: Crash earlier and more explicitly if the jit sees an unknown Message-ID: <20120129185655.7D7CC8203C@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: Changeset: r51951:161714d53cc7 Date: 2012-01-29 10:17 +0100 http://bitbucket.org/pypy/pypy/changeset/161714d53cc7/ Log: Crash earlier and more explicitly if the jit sees an unknown operation (like cast_ptr_to_adr, as win32 builds see now). diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py --- a/pypy/jit/codewriter/jtransform.py +++ b/pypy/jit/codewriter/jtransform.py @@ -7,6 +7,7 @@ from pypy.jit.metainterp import quasiimmut from pypy.jit.metainterp.history import getkind from pypy.jit.metainterp.typesystem import deref, arrayItem +from pypy.jit.metainterp.blackhole import BlackholeInterpreter from pypy.objspace.flow.model import SpaceOperation, Variable, Constant, c_last_exception from pypy.rlib import objectmodel from pypy.rlib.jit import _we_are_jitted @@ -199,9 +200,9 @@ try: rewrite = _rewrite_ops[op.opname] except KeyError: - return op # default: keep the operation unchanged - else: - return rewrite(self, op) + raise Exception("the JIT doesn't support the operation %r" + " in %r" % (op, getattr(self, 'graph', '?'))) + return rewrite(self, op) def rewrite_op_same_as(self, op): if op.args[0] in self.vable_array_vars: @@ -1720,4 +1721,18 @@ result[name[len(prefix):]] = getattr(Transformer, name) return result +def keep_operation_unchanged(jtransform, op): + return op + +def _add_default_ops(rewrite_ops): + # All operations present in the BlackholeInterpreter as bhimpl_xxx + # but not explicitly listed in this file are supposed to be just + # passed in unmodified. All other operations are forbidden. + for key, value in BlackholeInterpreter.__dict__.items(): + if key.startswith('bhimpl_'): + opname = key[len('bhimpl_'):] + rewrite_ops.setdefault(opname, keep_operation_unchanged) + rewrite_ops.setdefault('-live-', keep_operation_unchanged) + _rewrite_ops = _with_prefix('rewrite_op_') +_add_default_ops(_rewrite_ops) diff --git a/pypy/jit/codewriter/test/test_jtransform.py b/pypy/jit/codewriter/test/test_jtransform.py --- a/pypy/jit/codewriter/test/test_jtransform.py +++ b/pypy/jit/codewriter/test/test_jtransform.py @@ -1210,3 +1210,11 @@ assert op1.args == [v1] assert op1.result is None assert op2 is None + +def test_unknown_operation(): + op = SpaceOperation('foobar', [], varoftype(lltype.Void)) + tr = Transformer() + try: + tr.rewrite_operation(op) + except Exception, e: + assert 'foobar' in str(e) From noreply at buildbot.pypy.org Sun Jan 29 19:57:01 2012 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 29 Jan 2012 19:57:01 +0100 (CET) Subject: [pypy-commit] pypy default: merge heads Message-ID: <20120129185701.AB0B28203C@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: Changeset: r51956:c0a95b1a7b87 Date: 2012-01-29 19:56 +0100 http://bitbucket.org/pypy/pypy/changeset/c0a95b1a7b87/ Log: merge heads diff --git a/pypy/module/micronumpy/tool/numready/main.py b/pypy/module/micronumpy/tool/numready/main.py --- a/pypy/module/micronumpy/tool/numready/main.py +++ b/pypy/module/micronumpy/tool/numready/main.py @@ -88,7 +88,7 @@ l[i].append(lst[k * lgt + i]) return l -SPECIAL_NAMES = ["ndarray", "dtype", "generic"] +SPECIAL_NAMES = ["ndarray", "dtype", "generic", "flatiter"] def main(argv): cpy_items = find_numpy_items("/usr/bin/python") From noreply at buildbot.pypy.org Sun Jan 29 19:56:56 2012 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 29 Jan 2012 19:56:56 +0100 (CET) Subject: [pypy-commit] pypy stackroot-speedup: Fix. Message-ID: <20120129185656.B513982B1F@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: stackroot-speedup Changeset: r51952:d7ae2df81896 Date: 2012-01-28 21:24 +0100 http://bitbucket.org/pypy/pypy/changeset/d7ae2df81896/ Log: Fix. diff --git a/pypy/rpython/memory/gctransform/shadowstack.py b/pypy/rpython/memory/gctransform/shadowstack.py --- a/pypy/rpython/memory/gctransform/shadowstack.py +++ b/pypy/rpython/memory/gctransform/shadowstack.py @@ -392,7 +392,11 @@ if not prev: root_iterator.setcontext(obj.context) prev = obj.top - return root_iterator.nextleft(gc, obj.base, prev) + return root_iterator.nextleft(gc, obj.base, prev, + is_minor=False) + # note: it's probably good enough to pass is_minor=False, + # because if the 'obj' stays around for longer, it becomes + # itself an old object and is not traced any more at all CUSTOMTRACEFUNC = lltype.FuncType([llmemory.Address, llmemory.Address], llmemory.Address) From noreply at buildbot.pypy.org Sun Jan 29 19:56:57 2012 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 29 Jan 2012 19:56:57 +0100 (CET) Subject: [pypy-commit] pypy default: Don't look inside these functions. (Looking in str_from_buffer() Message-ID: <20120129185657.EC6AF82B20@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: Changeset: r51953:828709b0e0e9 Date: 2012-01-29 12:43 +0100 http://bitbucket.org/pypy/pypy/changeset/828709b0e0e9/ Log: Don't look inside these functions. (Looking in str_from_buffer() causes anyway the jit to crash on cast_ptr_to_adr.) diff --git a/pypy/rpython/lltypesystem/rffi.py b/pypy/rpython/lltypesystem/rffi.py --- a/pypy/rpython/lltypesystem/rffi.py +++ b/pypy/rpython/lltypesystem/rffi.py @@ -759,6 +759,7 @@ alloc_buffer._annenforceargs_ = [int] # (char*, str, int, int) -> None + @jit.dont_look_inside def str_from_buffer(raw_buf, gc_buf, allocated_size, needed_size): """ Converts from a pair returned by alloc_buffer to a high-level string. @@ -784,6 +785,7 @@ return hlstrtype(new_buf) # (char*, str) -> None + @jit.dont_look_inside def keep_buffer_alive_until_here(raw_buf, gc_buf): """ Keeps buffers alive or frees temporary buffers created by alloc_buffer. From noreply at buildbot.pypy.org Sun Jan 29 19:56:59 2012 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 29 Jan 2012 19:56:59 +0100 (CET) Subject: [pypy-commit] pypy stackroot-speedup: a failing test Message-ID: <20120129185659.2802D82B22@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: stackroot-speedup Changeset: r51954:fb3badd51945 Date: 2012-01-29 13:03 +0100 http://bitbucket.org/pypy/pypy/changeset/fb3badd51945/ Log: a failing test diff --git a/pypy/translator/c/test/test_newgc.py b/pypy/translator/c/test/test_newgc.py --- a/pypy/translator/c/test/test_newgc.py +++ b/pypy/translator/c/test/test_newgc.py @@ -1185,6 +1185,22 @@ assert data.startswith('member0') assert 'GcArray of * GcStruct S {' in data + def define_recursive_function(self): + def f_rec(n): + if n < 2: + rgc.collect() + return 1 + else: + return f_rec(n-1) + f_rec(n-2) + def fn(): + print f_rec(10) + return 0 + return fn + + def test_recursive_function(self): + self.run("recursive_function") + + class TestSemiSpaceGC(TestUsingFramework, snippet.SemiSpaceGCTestDefines): gcpolicy = "semispace" should_be_moving = True From noreply at buildbot.pypy.org Sun Jan 29 19:57:00 2012 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 29 Jan 2012 19:57:00 +0100 (CET) Subject: [pypy-commit] pypy stackroot-speedup: Fix. "livevars" is the list returned from the previous push_roots, Message-ID: <20120129185700.6950782B23@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: stackroot-speedup Changeset: r51955:c4b086bfb967 Date: 2012-01-29 13:12 +0100 http://bitbucket.org/pypy/pypy/changeset/c4b086bfb967/ Log: Fix. "livevars" is the list returned from the previous push_roots, so it already contains one extra item if insert_rec_marker is set. diff --git a/pypy/rpython/memory/gctransform/framework.py b/pypy/rpython/memory/gctransform/framework.py --- a/pypy/rpython/memory/gctransform/framework.py +++ b/pypy/rpython/memory/gctransform/framework.py @@ -662,7 +662,7 @@ # livevars = self.push_roots(hop, insert_rec_marker=rec) self.default(hop) - self.pop_roots(hop, livevars, insert_rec_marker=rec) + self.pop_roots(hop, livevars) else: self.default(hop) if hop.spaceop.opname == "direct_call": @@ -1248,21 +1248,20 @@ hop.genop("raw_store", [base_addr, c_type, c_k, v_adr]) return livevars - def pop_roots(self, hop, livevars, insert_rec_marker=False): + def pop_roots(self, hop, livevars): if self.decr_stack_ptr is None: return - length = len(livevars) - if insert_rec_marker: - length += 1 - if length == 0: + if not livevars: return - c_len = rmodel.inputconst(lltype.Signed, length) + c_len = rmodel.inputconst(lltype.Signed, len(livevars) ) base_addr = hop.genop("direct_call", [self.decr_stack_ptr, c_len ], resulttype=llmemory.Address) if self.gcdata.gc.moving_gc: # for moving collectors, reload the roots into the local variables c_type = rmodel.inputconst(lltype.Void, llmemory.Address) for k,var in enumerate(livevars): + if isinstance(var, Constant): # "c_marker" only + continue c_k = rmodel.inputconst(lltype.Signed, k) v_newaddr = hop.genop("raw_load", [base_addr, c_type, c_k], resulttype=llmemory.Address) diff --git a/pypy/translator/c/test/test_newgc.py b/pypy/translator/c/test/test_newgc.py --- a/pypy/translator/c/test/test_newgc.py +++ b/pypy/translator/c/test/test_newgc.py @@ -1193,7 +1193,8 @@ else: return f_rec(n-1) + f_rec(n-2) def fn(): - print f_rec(10) + n = f_rec(10) + assert n == 89 return 0 return fn From noreply at buildbot.pypy.org Sun Jan 29 21:46:52 2012 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 29 Jan 2012 21:46:52 +0100 (CET) Subject: [pypy-commit] pypy stackroot-speedup: Fix comment. Message-ID: <20120129204652.551D78203C@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: stackroot-speedup Changeset: r51957:231f3159915e Date: 2012-01-29 21:46 +0100 http://bitbucket.org/pypy/pypy/changeset/231f3159915e/ Log: Fix comment. diff --git a/pypy/rpython/memory/gctransform/shadowstack.py b/pypy/rpython/memory/gctransform/shadowstack.py --- a/pypy/rpython/memory/gctransform/shadowstack.py +++ b/pypy/rpython/memory/gctransform/shadowstack.py @@ -8,7 +8,7 @@ # special values in the shadowstack (different values than -# the jit's MARKER_FRAME, which is 8): +# the jit's MARKER_FRAME(_TRACED), which are 8 and 32): MARKER_NOT_TRACED = 16 MARKER_TRACED = 24 From noreply at buildbot.pypy.org Sun Jan 29 22:56:34 2012 From: noreply at buildbot.pypy.org (amauryfa) Date: Sun, 29 Jan 2012 22:56:34 +0100 (CET) Subject: [pypy-commit] pypy merge-2.7.2: Add large file support to mmap: 64bit offsets on 32bit platform. Message-ID: <20120129215634.AC54D8203C@wyvern.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc <amauryfa at gmail.com> Branch: merge-2.7.2 Changeset: r51958:7a82f3fbab26 Date: 2012-01-29 21:55 +0000 http://bitbucket.org/pypy/pypy/changeset/7a82f3fbab26/ Log: Add large file support to mmap: 64bit offsets on 32bit platform. problem: test_large_offset() fails when interpreted, and passes when run on a translated pypy. Maybe a ll2ctypes issue? diff --git a/pypy/module/mmap/interp_mmap.py b/pypy/module/mmap/interp_mmap.py --- a/pypy/module/mmap/interp_mmap.py +++ b/pypy/module/mmap/interp_mmap.py @@ -2,9 +2,14 @@ from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.typedef import TypeDef from pypy.interpreter.gateway import interp2app, unwrap_spec, NoneNotWrapped -from pypy.rlib import rmmap +from pypy.rlib import rmmap, rarithmetic from pypy.rlib.rmmap import RValueError, RTypeError +if rmmap.HAVE_LARGEFILE_SUPPORT: + OFF_T = rarithmetic.r_longlong +else: + OFF_T = int + class W_MMap(Wrappable): def __init__(self, space, mmap_obj): @@ -55,7 +60,7 @@ end = space.getindex_w(w_end, None) return space.wrap(self.mmap.find(tofind, start, end, True)) - @unwrap_spec(pos='index', whence=int) + @unwrap_spec(pos=OFF_T, whence=int) def seek(self, pos, whence=0): try: self.mmap.seek(pos, whence) @@ -196,8 +201,8 @@ if rmmap._POSIX: - @unwrap_spec(fileno=int, length='index', flags=int, - prot=int, access=int, offset='index') + @unwrap_spec(fileno=int, length=int, flags=int, + prot=int, access=int, offset=OFF_T) def mmap(space, w_subtype, fileno, length, flags=rmmap.MAP_SHARED, prot=rmmap.PROT_WRITE | rmmap.PROT_READ, access=rmmap._ACCESS_DEFAULT, offset=0): @@ -216,8 +221,8 @@ elif rmmap._MS_WINDOWS: - @unwrap_spec(fileno=int, length='index', tagname=str, - access=int, offset='index') + @unwrap_spec(fileno=int, length=int, tagname=str, + access=int, offset=OFF_T) def mmap(space, w_subtype, fileno, length, tagname="", access=rmmap._ACCESS_DEFAULT, offset=0): self = space.allocate_instance(W_MMap, w_subtype) diff --git a/pypy/module/mmap/test/test_mmap.py b/pypy/module/mmap/test/test_mmap.py --- a/pypy/module/mmap/test/test_mmap.py +++ b/pypy/module/mmap/test/test_mmap.py @@ -591,6 +591,45 @@ assert m.size() == halfsize + 512 m.close() + def test_large_offset(self): + import mmap + import sys + size = 0x14FFFFFFF + if sys.platform.startswith('win') or sys.platform == 'darwin': + self.skip('test requires %s bytes and a long time to run' % size) + + with open(self.tmpname, "w+b") as f: + f.seek(size) + f.write("A") + f.flush() + with open(self.tmpname, 'rb') as f2: + f2.seek(size) + c = f2.read(1) + assert c == b'A' + m = mmap.mmap(f2.fileno(), 0, offset=0x140000000, + access=mmap.ACCESS_READ) + try: + assert m[0xFFFFFFF] == b'A' + finally: + m.close() + + def test_large_filesize(self): + import mmap + import sys + size = 0x17FFFFFFF + if sys.platform.startswith('win') or sys.platform == 'darwin': + self.skip('test requires %s bytes and a long time to run' % size) + + with open(self.tmpname, "w+b") as f: + f.seek(size) + f.write(" ") + f.flush() + m = mmap.mmap(f.fileno(), 0x10000, access=mmap.ACCESS_READ) + try: + assert m.size() == 0x180000000 + finally: + m.close() + def test_all(self): # this is a global test, ported from test_mmap.py import mmap diff --git a/pypy/rlib/rmmap.py b/pypy/rlib/rmmap.py --- a/pypy/rlib/rmmap.py +++ b/pypy/rlib/rmmap.py @@ -92,6 +92,11 @@ _ACCESS_DEFAULT, ACCESS_READ, ACCESS_WRITE, ACCESS_COPY = range(4) +if rffi.sizeof(off_t) > rffi.sizeof(lltype.Signed): + HAVE_LARGEFILE_SUPPORT = True +else: + HAVE_LARGEFILE_SUPPORT = False + def external(name, args, result): unsafe = rffi.llexternal(name, args, result, compilation_info=CConfig._compilation_info_) @@ -428,10 +433,6 @@ elif _POSIX: st = os.fstat(self.fd) size = st[stat.ST_SIZE] - if size > sys.maxint: - size = sys.maxint - else: - size = int(size) return size def write(self, data): @@ -643,17 +644,15 @@ else: mode = st[stat.ST_MODE] size = st[stat.ST_SIZE] - size -= offset - if size > sys.maxint: - size = sys.maxint - else: - size = int(size) if stat.S_ISREG(mode): if map_size == 0: - if offset > st[stat.ST_SIZE]: - raise RValueError("mmap offset is greater than file size") - map_size = size - elif map_size > size: + if offset > size: + raise RValueError( + "mmap offset is greater than file size") + map_size = int(size - offset) + if map_size != size - offset: + raise RValueError("mmap length is too large") + elif offset + map_size > size: raise RValueError("mmap length is greater than file size") m = MMap(access, offset) diff --git a/pypy/rpython/lltypesystem/ll2ctypes.py b/pypy/rpython/lltypesystem/ll2ctypes.py --- a/pypy/rpython/lltypesystem/ll2ctypes.py +++ b/pypy/rpython/lltypesystem/ll2ctypes.py @@ -202,7 +202,7 @@ return cls._ptrtype # ctypes can raise OverflowError on 64-bit builds for n in [sys.maxint, 2**31]: - cls.MAX_SIZE = n/64 + cls.MAX_SIZE = n / ctypes.sizeof(ctypes_item) try: cls._ptrtype = ctypes.POINTER(cls.MAX_SIZE * ctypes_item) except OverflowError, e: From noreply at buildbot.pypy.org Sun Jan 29 23:42:33 2012 From: noreply at buildbot.pypy.org (mattip) Date: Sun, 29 Jan 2012 23:42:33 +0100 (CET) Subject: [pypy-commit] pypy win32-cleanup: fix some tests Message-ID: <20120129224233.A79108203C@wyvern.cs.uni-duesseldorf.de> Author: mattip Branch: win32-cleanup Changeset: r51959:882a62b6adfe Date: 2012-01-30 00:41 +0200 http://bitbucket.org/pypy/pypy/changeset/882a62b6adfe/ Log: fix some tests diff --git a/pypy/module/_ffi/test/test__ffi.py b/pypy/module/_ffi/test/test__ffi.py --- a/pypy/module/_ffi/test/test__ffi.py +++ b/pypy/module/_ffi/test/test__ffi.py @@ -190,6 +190,7 @@ def test_convert_strings_to_char_p(self): """ + DLLEXPORT long mystrlen(char* s) { long len = 0; @@ -215,6 +216,7 @@ def test_convert_unicode_to_unichar_p(self): """ #include <wchar.h> + DLLEXPORT long mystrlen_u(wchar_t* s) { long len = 0; @@ -241,6 +243,7 @@ def test_keepalive_temp_buffer(self): """ + DLLEXPORT char* do_nothing(char* s) { return s; @@ -525,5 +528,7 @@ from _ffi import CDLL, types libfoo = CDLL(self.libfoo_name) raises(AttributeError, "libfoo.getfunc('I_do_not_exist', [], types.void)") + if self.iswin32: + skip("unix specific") libnone = CDLL(None) raises(AttributeError, "libnone.getfunc('I_do_not_exist', [], types.void)") From noreply at buildbot.pypy.org Sun Jan 29 23:51:03 2012 From: noreply at buildbot.pypy.org (amauryfa) Date: Sun, 29 Jan 2012 23:51:03 +0100 (CET) Subject: [pypy-commit] pypy py3k: Fix syntax errors in test_codecs Message-ID: <20120129225103.743E58203C@wyvern.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc <amauryfa at gmail.com> Branch: py3k Changeset: r51960:124b3b5cc7ac Date: 2012-01-28 18:41 +0100 http://bitbucket.org/pypy/pypy/changeset/124b3b5cc7ac/ Log: Fix syntax errors in test_codecs diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py --- a/pypy/module/_codecs/test/test_codecs.py +++ b/pypy/module/_codecs/test/test_codecs.py @@ -12,7 +12,7 @@ raises(TypeError, _codecs.register, 1) def test_bigU_codecs(self): - u = u'\U00010001\U00020002\U00030003\U00040004\U00050005' + u = '\U00010001\U00020002\U00030003\U00040004\U00050005' for encoding in ('utf-8', 'utf-16', 'utf-16-le', 'utf-16-be', 'utf-32', 'utf-32-le', 'utf-32-be', 'raw_unicode_escape', @@ -20,20 +20,20 @@ assert str(u.encode(encoding),encoding) == u def test_ucs4(self): - x = u'\U00100000' + x = '\U00100000' y = x.encode("raw-unicode-escape").decode("raw-unicode-escape") assert x == y def test_named_unicode(self): - assert str(b'\\N{SPACE}','unicode-escape') == u" " + assert str(b'\\N{SPACE}','unicode-escape') == " " raises( UnicodeDecodeError, str,b'\\N{SPACE','unicode-escape') raises( UnicodeDecodeError, str,b'\\NSPACE}','unicode-escape') raises( UnicodeDecodeError, str,b'\\NSPACE','unicode-escape') raises( UnicodeDecodeError, str,b'\\N','unicode-escape') - assert str(b'\\N{SPACE}\\N{SPACE}','unicode-escape') == u" " - assert str(b'\\N{SPACE}a\\N{SPACE}','unicode-escape') == u" a " - assert b"\\N{foo}xx".decode("unicode-escape", "ignore") == u"xx" - assert 1 <= len(u"\N{CJK UNIFIED IDEOGRAPH-20000}") <= 2 + assert str(b'\\N{SPACE}\\N{SPACE}','unicode-escape') == " " + assert str(b'\\N{SPACE}a\\N{SPACE}','unicode-escape') == " a " + assert b"\\N{foo}xx".decode("unicode-escape", "ignore") == "xx" + assert 1 <= len("\N{CJK UNIFIED IDEOGRAPH-20000}") <= 2 def test_literals(self): raises(SyntaxError, eval, 'u\'\\Uffffffff\'') @@ -65,40 +65,40 @@ def test_unicodetranslateerror(self): import sys assert str(UnicodeTranslateError( - u"g\xfcrk", 1, 2, "ouch"))== "can't translate character u'\\xfc' in position 1: ouch" + "g\xfcrk", 1, 2, "ouch"))== "can't translate character '\\xfc' in position 1: ouch" assert str(UnicodeTranslateError( - u"g\u0100rk", 1, 2, "ouch"))== "can't translate character u'\\u0100' in position 1: ouch" + "g\u0100rk", 1, 2, "ouch"))== "can't translate character '\\u0100' in position 1: ouch" assert str(UnicodeTranslateError( - u"g\uffffrk", 1, 2, "ouch"))== "can't translate character u'\\uffff' in position 1: ouch" + "g\uffffrk", 1, 2, "ouch"))== "can't translate character '\\uffff' in position 1: ouch" if sys.maxunicode > 0xffff and len(chr(0x10000)) == 1: assert str(UnicodeTranslateError( - u"g\U00010000rk", 1, 2, "ouch"))== "can't translate character u'\\U00010000' in position 1: ouch" + "g\U00010000rk", 1, 2, "ouch"))== "can't translate character '\\U00010000' in position 1: ouch" assert str(UnicodeTranslateError( - u"g\xfcrk", 1, 3, "ouch"))=="can't translate characters in position 1-2: ouch" + "g\xfcrk", 1, 3, "ouch"))=="can't translate characters in position 1-2: ouch" def test_unicodeencodeerror(self): import sys assert str(UnicodeEncodeError( - "ascii", u"g\xfcrk", 1, 2, "ouch"))=="'ascii' codec can't encode character u'\\xfc' in position 1: ouch" + "ascii", "g\xfcrk", 1, 2, "ouch"))=="'ascii' codec can't encode character '\\xfc' in position 1: ouch" assert str(UnicodeEncodeError( - "ascii", u"g\xfcrk", 1, 4, "ouch"))== "'ascii' codec can't encode characters in position 1-3: ouch" + "ascii", "g\xfcrk", 1, 4, "ouch"))== "'ascii' codec can't encode characters in position 1-3: ouch" assert str(UnicodeEncodeError( - "ascii", u"\xfcx", 0, 1, "ouch"))=="'ascii' codec can't encode character u'\\xfc' in position 0: ouch" + "ascii", "\xfcx", 0, 1, "ouch"))=="'ascii' codec can't encode character '\\xfc' in position 0: ouch" assert str(UnicodeEncodeError( - "ascii", u"\u0100x", 0, 1, "ouch"))=="'ascii' codec can't encode character u'\\u0100' in position 0: ouch" + "ascii", "\u0100x", 0, 1, "ouch"))=="'ascii' codec can't encode character '\\u0100' in position 0: ouch" assert str(UnicodeEncodeError( - "ascii", u"\uffffx", 0, 1, "ouch"))=="'ascii' codec can't encode character u'\\uffff' in position 0: ouch" + "ascii", "\uffffx", 0, 1, "ouch"))=="'ascii' codec can't encode character '\\uffff' in position 0: ouch" if sys.maxunicode > 0xffff and len(chr(0x10000)) == 1: assert str(UnicodeEncodeError( - "ascii", u"\U00010000x", 0, 1, "ouch")) =="'ascii' codec can't encode character u'\\U00010000' in position 0: ouch" + "ascii", "\U00010000x", 0, 1, "ouch")) =="'ascii' codec can't encode character '\\U00010000' in position 0: ouch" def test_indexerror(self): import _codecs @@ -110,17 +110,17 @@ import sys assert charmap_decode(b'', 'strict', 'blablabla') == ('', 0) assert charmap_decode(b'xxx') == ('xxx', 3) - assert charmap_decode(b'xxx', 'strict', {ord('x'): u'XX'}) == ('XXXXXX', 3) + assert charmap_decode(b'xxx', 'strict', {ord('x'): 'XX'}) == ('XXXXXX', 3) map = tuple([chr(i) for i in range(256)]) - assert charmap_decode(b'xxx\xff', 'strict', map) == (u'xxx\xff', 4) + assert charmap_decode(b'xxx\xff', 'strict', map) == ('xxx\xff', 4) raises(TypeError, charmap_decode, '\xff', "replace", {0xff: 0x10001}) def test_unicode_escape(self): from _codecs import unicode_escape_encode, unicode_escape_decode - assert unicode_escape_encode(u'abc') == (u'abc'.encode('unicode_escape'), 3) + assert unicode_escape_encode('abc') == ('abc'.encode('unicode_escape'), 3) assert unicode_escape_decode(b'abc') == (b'abc'.decode('unicode_escape'), 3) - assert unicode_escape_decode(b'\\x61\\x62\\x63') == (u'abc', 12) + assert unicode_escape_decode(b'\\x61\\x62\\x63') == ('abc', 12) class AppTestPartialEvaluation: @@ -132,22 +132,22 @@ import _codecs encoding = 'utf-8' check_partial = [ - u"\x00", - u"\x00", - u"\x00\xff", - u"\x00\xff", - u"\x00\xff\u07ff", - u"\x00\xff\u07ff", - u"\x00\xff\u07ff", - u"\x00\xff\u07ff\u0800", - u"\x00\xff\u07ff\u0800", - u"\x00\xff\u07ff\u0800", - u"\x00\xff\u07ff\u0800\uffff", + "\x00", + "\x00", + "\x00\xff", + "\x00\xff", + "\x00\xff\u07ff", + "\x00\xff\u07ff", + "\x00\xff\u07ff", + "\x00\xff\u07ff\u0800", + "\x00\xff\u07ff\u0800", + "\x00\xff\u07ff\u0800", + "\x00\xff\u07ff\u0800\uffff", ] buffer = b'' - result = u"" - for (c, partialresult) in zip(u"\x00\xff\u07ff\u0800\uffff".encode(encoding), check_partial): + result = "" + for (c, partialresult) in zip("\x00\xff\u07ff\u0800\uffff".encode(encoding), check_partial): buffer += bytes([c]) res = _codecs.utf_8_decode(buffer,'strict',False) if res[1] >0 : @@ -159,20 +159,20 @@ import _codecs encoding = 'utf-16' check_partial = [ - u"", # first byte of BOM read - u"", # second byte of BOM read => byteorder known - u"", - u"\x00", - u"\x00", - u"\x00\xff", - u"\x00\xff", - u"\x00\xff\u0100", - u"\x00\xff\u0100", - u"\x00\xff\u0100\uffff", + "", # first byte of BOM read + "", # second byte of BOM read => byteorder known + "", + "\x00", + "\x00", + "\x00\xff", + "\x00\xff", + "\x00\xff\u0100", + "\x00\xff\u0100", + "\x00\xff\u0100\uffff", ] buffer = b'' - result = u"" - for (c, partialresult) in zip(u"\x00\xff\u0100\uffff".encode(encoding), check_partial): + result = "" + for (c, partialresult) in zip("\x00\xff\u0100\uffff".encode(encoding), check_partial): buffer += bytes([c]) res = _codecs.utf_16_decode(buffer,'strict',False) if res[1] >0 : @@ -184,9 +184,9 @@ import codecs, io self.encoding = 'utf-8' - s1 = u"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy\r\n" - s2 = u"offending line: ladfj askldfj klasdj fskla dfzaskdj fasklfj laskd fjasklfzzzzaa%whereisthis!!!\r\n" - s3 = u"next line.\r\n" + s1 = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy\r\n" + s2 = "offending line: ladfj askldfj klasdj fskla dfzaskdj fasklfj laskd fjasklfzzzzaa%whereisthis!!!\r\n" + s3 = "next line.\r\n" s = (s1+s2+s3).encode(self.encoding) stream = io.BytesIO(s) @@ -194,16 +194,16 @@ assert reader.readline() == s1 assert reader.readline() == s2 assert reader.readline() == s3 - assert reader.readline() == u"" + assert reader.readline() == "" def test_bug1098990_b(self): import codecs, io self.encoding = 'utf-8' - s1 = u"aaaaaaaaaaaaaaaaaaaaaaaa\r\n" - s2 = u"bbbbbbbbbbbbbbbbbbbbbbbb\r\n" - s3 = u"stillokay:bbbbxx\r\n" - s4 = u"broken!!!!badbad\r\n" - s5 = u"againokay.\r\n" + s1 = "aaaaaaaaaaaaaaaaaaaaaaaa\r\n" + s2 = "bbbbbbbbbbbbbbbbbbbbbbbb\r\n" + s3 = "stillokay:bbbbxx\r\n" + s4 = "broken!!!!badbad\r\n" + s5 = "againokay.\r\n" s = (s1+s2+s3+s4+s5).encode(self.encoding) stream = io.BytesIO(s) @@ -213,13 +213,13 @@ assert reader.readline() == s3 assert reader.readline() == s4 assert reader.readline() == s5 - assert reader.readline() == u"" + assert reader.readline() == "" def test_seek_utf16le(self): # all codecs should be able to encode these import codecs, io encoding = 'utf-16-le' - s = u"%s\n%s\n" % (10*u"abc123", 10*u"def456") + s = "%s\n%s\n" % (10*"abc123", 10*"def456") reader = codecs.getreader(encoding)(io.BytesIO(s.encode(encoding))) for t in range(5): # Test that calling seek resets the internal codec state and buffers @@ -232,15 +232,15 @@ import sys class U(str): pass - enc = U(u"a").encode("unicode_internal") + enc = U("a").encode("unicode_internal") if sys.maxunicode == 65535: # UCS2 build if sys.byteorder == "big": assert enc == b"\x00a" else: assert enc == b"a\x00" - elif len(u"\U00010098") == 1: + elif len("\U00010098") == 1: # UCS4 build on a UCS4 CPython - enc2 = u"\U00010098".encode("unicode_internal") + enc2 = "\U00010098".encode("unicode_internal") if sys.byteorder == "big": assert enc == b"\x00\x00\x00a" assert enc2 == b"\x00\x01\x00\x98" @@ -268,12 +268,12 @@ else: bytes = b"a\x00\x00\x00" bytes2 = b"\x98\x00\x01\x00" - assert bytes2.decode("unicode_internal") == u"\U00010098" - assert bytes.decode("unicode_internal") == u"a" + assert bytes2.decode("unicode_internal") == "\U00010098" + assert bytes.decode("unicode_internal") == "a" def test_raw_unicode_escape(self): - assert str(b"\u0663", "raw-unicode-escape") == u"\u0663" - assert u"\u0663".encode("raw-unicode-escape") == b"\u0663" + assert str(b"\u0663", "raw-unicode-escape") == "\u0663" + assert "\u0663".encode("raw-unicode-escape") == b"\u0663" def test_escape_decode(self): import _codecs @@ -281,8 +281,8 @@ assert _codecs.escape_decode(test)[0] == b'a\n\\b\x00c\td\u2045' assert _codecs.escape_decode(b'\\077')[0] == b'?' assert _codecs.escape_decode(b'\\100')[0] == b'@' - assert _codecs.escape_decode(b'\\253')[0] == bytes([0253]) - assert _codecs.escape_decode(b'\\312')[0] == bytes([0312]) + assert _codecs.escape_decode(b'\\253')[0] == bytes([0o253]) + assert _codecs.escape_decode(b'\\312')[0] == bytes([0o312]) def test_escape_decode_wrap_around(self): import _codecs @@ -301,7 +301,7 @@ assert _codecs.escape_encode(b"'")[0] == b"\\'" def test_decode_utf8_different_case(self): - constant = u"a" + constant = "a" assert constant.encode("utf-8") == constant.encode("UTF-8") def test_codec_wrong_result(self): @@ -315,13 +315,13 @@ return None _codecs.register(search_function) raises(TypeError, b"hello".decode, "test.mytestenc") - raises(TypeError, u"hello".encode, "test.mytestenc") + raises(TypeError, "hello".encode, "test.mytestenc") def test_cpytest_decode(self): import codecs - assert codecs.decode(b'\xe4\xf6\xfc', 'latin-1') == u'\xe4\xf6\xfc' + assert codecs.decode(b'\xe4\xf6\xfc', 'latin-1') == '\xe4\xf6\xfc' raises(TypeError, codecs.decode) - assert codecs.decode(b'abc') == u'abc' + assert codecs.decode(b'abc') == 'abc' raises(UnicodeDecodeError, codecs.decode, b'\xff', 'ascii') def test_bad_errorhandler_return(self): @@ -382,7 +382,7 @@ def test_utf8sig(self): import codecs d = codecs.getincrementaldecoder("utf-8-sig")() - s = u"spam" + s = "spam" assert d.decode(s.encode("utf-8-sig")) == s def test_escape_decode_escaped_newline(self): @@ -393,20 +393,20 @@ def test_charmap_decode_1(self): import codecs - assert codecs.charmap_encode(u'xxx') == (b'xxx', 3) - assert codecs.charmap_encode(u'xxx', 'strict', {ord('x'): b'XX'}) == (b'XXXXXX', 3) + assert codecs.charmap_encode('xxx') == (b'xxx', 3) + assert codecs.charmap_encode('xxx', 'strict', {ord('x'): b'XX'}) == (b'XXXXXX', 3) - res = codecs.charmap_decode(b"\x00\x01\x02", "replace", u"ab") - assert res == (u"ab\ufffd", 3) - res = codecs.charmap_decode(b"\x00\x01\x02", "replace", u"ab\ufffe") - assert res == (u'ab\ufffd', 3) + res = codecs.charmap_decode(b"\x00\x01\x02", "replace", "ab") + assert res == ("ab\ufffd", 3) + res = codecs.charmap_decode(b"\x00\x01\x02", "replace", "ab\ufffe") + assert res == ('ab\ufffd', 3) def test_decode_errors(self): import sys if sys.maxunicode > 0xffff: try: b"\x00\x00\x00\x00\x00\x11\x11\x00".decode("unicode_internal") - except UnicodeDecodeError, ex: + except UnicodeDecodeError as ex: assert "unicode_internal" == ex.encoding assert b"\x00\x00\x00\x00\x00\x11\x11\x00" == ex.object assert ex.start == 4 @@ -417,28 +417,28 @@ def test_errors(self): import codecs assert codecs.replace_errors(UnicodeEncodeError( - "ascii", u"\u3042", 0, 1, "ouch")) == (u"?", 1) + "ascii", "\u3042", 0, 1, "ouch")) == ("?", 1) assert codecs.replace_errors(UnicodeDecodeError( - "ascii", b"\xff", 0, 1, "ouch")) == (u"\ufffd", 1) + "ascii", b"\xff", 0, 1, "ouch")) == ("\ufffd", 1) assert codecs.replace_errors(UnicodeTranslateError( - u"\u3042", 0, 1, "ouch")) == (u"\ufffd", 1) + "\u3042", 0, 1, "ouch")) == ("\ufffd", 1) assert codecs.replace_errors(UnicodeEncodeError( - "ascii", u"\u3042\u3042", 0, 2, "ouch")) == (u"??", 2) + "ascii", "\u3042\u3042", 0, 2, "ouch")) == ("??", 2) assert codecs.replace_errors(UnicodeDecodeError( - "ascii", b"\xff\xff", 0, 2, "ouch")) == (u"\ufffd", 2) + "ascii", b"\xff\xff", 0, 2, "ouch")) == ("\ufffd", 2) assert codecs.replace_errors(UnicodeTranslateError( - u"\u3042\u3042", 0, 2, "ouch")) == (u"\ufffd\ufffd", 2) + "\u3042\u3042", 0, 2, "ouch")) == ("\ufffd\ufffd", 2) class BadStartUnicodeEncodeError(UnicodeEncodeError): def __init__(self): - UnicodeEncodeError.__init__(self, "ascii", u"", 0, 1, "bad") + UnicodeEncodeError.__init__(self, "ascii", "", 0, 1, "bad") self.start = [] # A UnicodeEncodeError object with a bad object attribute class BadObjectUnicodeEncodeError(UnicodeEncodeError): def __init__(self): - UnicodeEncodeError.__init__(self, "ascii", u"", 0, 1, "bad") + UnicodeEncodeError.__init__(self, "ascii", "", 0, 1, "bad") self.object = [] # A UnicodeDecodeError object without an end attribute @@ -456,19 +456,19 @@ # A UnicodeTranslateError object without a start attribute class NoStartUnicodeTranslateError(UnicodeTranslateError): def __init__(self): - UnicodeTranslateError.__init__(self, u"", 0, 1, "bad") + UnicodeTranslateError.__init__(self, "", 0, 1, "bad") del self.start # A UnicodeTranslateError object without an end attribute class NoEndUnicodeTranslateError(UnicodeTranslateError): def __init__(self): - UnicodeTranslateError.__init__(self, u"", 0, 1, "bad") + UnicodeTranslateError.__init__(self, "", 0, 1, "bad") del self.end # A UnicodeTranslateError object without an object attribute class NoObjectUnicodeTranslateError(UnicodeTranslateError): def __init__(self): - UnicodeTranslateError.__init__(self, u"", 0, 1, "bad") + UnicodeTranslateError.__init__(self, "", 0, 1, "bad") del self.object import codecs @@ -479,14 +479,14 @@ raises(TypeError, codecs.replace_errors, BadObjectUnicodeEncodeError()) raises(TypeError, codecs.replace_errors, BadObjectUnicodeDecodeError() ) - # With the correct exception, "replace" returns an "?" or u"\ufffd" replacement + # With the correct exception, "replace" returns an "?" or "\ufffd" replacement def test_decode_ignore(self): assert b'\xff'.decode('utf-7', 'ignore') == '' assert b'\x00'.decode('unicode-internal', 'ignore') == '' def test_backslahreplace(self): - assert u'a\xac\u1234\u20ac\u8000'.encode('ascii', 'backslashreplace') == b'a\\xac\u1234\u20ac\u8000' + assert 'a\xac\u1234\u20ac\u8000'.encode('ascii', 'backslashreplace') == b'a\\xac\u1234\u20ac\u8000' def test_surrogateescape(self): assert b'a\x80b'.decode('utf-8', 'surrogateescape') == 'a\udc80b' @@ -494,7 +494,7 @@ def test_badhandler(self): import codecs - results = ( 42, u"foo", (1,2,3), (u"foo", 1, 3), (u"foo", None), (u"foo",), ("foo", 1, 3), ("foo", None), ("foo",) ) + results = ( 42, "foo", (1,2,3), ("foo", 1, 3), ("foo", None), ("foo",), ("foo", 1, 3), ("foo", None), ("foo",) ) encs = ("ascii", "latin-1", "iso-8859-1", "iso-8859-15") for res in results: @@ -502,7 +502,7 @@ for enc in encs: raises( TypeError, - u"\u3042".encode, + "\u3042".encode, enc, "test.badhandler" ) @@ -531,50 +531,50 @@ res = b"\x00\x00\x00\x00\x00".decode("unicode-internal", "replace") if sys.maxunicode > 65535: - assert res == u"\u0000\ufffd" # UCS4 build + assert res == "\u0000\ufffd" # UCS4 build else: - assert res == u"\x00\x00\ufffd" # UCS2 build + assert res == "\x00\x00\ufffd" # UCS2 build res = b"\x00\x00\x00\x00\x00".decode("unicode-internal", "ignore") if sys.maxunicode > 65535: - assert res == u"\u0000" # UCS4 build + assert res == "\u0000" # UCS4 build else: - assert res == u"\x00\x00" # UCS2 build + assert res == "\x00\x00" # UCS2 build def handler_unicodeinternal(exc): if not isinstance(exc, UnicodeDecodeError): raise TypeError("don't know how to handle %r" % exc) - return (u"\x01", 1) + return ("\x01", 1) codecs.register_error("test.hui", handler_unicodeinternal) res = b"\x00\x00\x00\x00\x00".decode("unicode-internal", "test.hui") if sys.maxunicode > 65535: - assert res == u"\u0000\u0001\u0000" # UCS4 build + assert res == "\u0000\u0001\u0000" # UCS4 build else: - assert res == u"\x00\x00\x01\x00\x00" # UCS2 build + assert res == "\x00\x00\x01\x00\x00" # UCS2 build def test_encode_error_bad_handler(self): import codecs codecs.register_error("test.bad_handler", lambda e: (repl, 1)) - assert u"xyz".encode("latin-1", "test.bad_handler") == b"xyz" - repl = u"\u1234" - raises(UnicodeEncodeError, u"\u5678".encode, "latin-1", + assert "xyz".encode("latin-1", "test.bad_handler") == b"xyz" + repl = "\u1234" + raises(UnicodeEncodeError, "\u5678".encode, "latin-1", "test.bad_handler") - repl = u"\u00E9" - s = u"\u5678".encode("latin-1", "test.bad_handler") + repl = "\u00E9" + s = "\u5678".encode("latin-1", "test.bad_handler") assert s == b'\xe9' def test_charmap_encode(self): assert 'xxx'.encode('charmap') == b'xxx' import codecs - raises(TypeError, codecs.charmap_encode, u'\xff', "replace", {0xff: 300}) - raises(UnicodeError, codecs.charmap_encode, u"\xff", "replace", {0xff: None}) + raises(TypeError, codecs.charmap_encode, '\xff', "replace", {0xff: 300}) + raises(UnicodeError, codecs.charmap_encode, "\xff", "replace", {0xff: None}) def test_charmap_encode_replace(self): charmap = dict([(c, bytes([c, c]).upper()) for c in b"abcdefgh"]) charmap[ord("?")] = b"XYZ" import codecs - sin = u"abcDEF" + sin = "abcDEF" sout = codecs.charmap_encode(sin, "replace", charmap)[0] assert sout == b"AABBCCXYZXYZXYZ" @@ -583,13 +583,13 @@ def test_charmap_build(self): import codecs - assert codecs.charmap_build(u'123456') == {49: 0, 50: 1, 51: 2, + assert codecs.charmap_build('123456') == {49: 0, 50: 1, 51: 2, 52: 3, 53: 4, 54: 5} def test_utf7_start_end_in_exception(self): try: b'+IC'.decode('utf-7') - except UnicodeDecodeError, exc: + except UnicodeDecodeError as exc: assert exc.start == 0 assert exc.end == 3 @@ -598,7 +598,7 @@ def test_utf_16_encode_decode(self): import codecs, sys - x = u'123abc' + x = '123abc' if sys.byteorder == 'big': assert codecs.getencoder('utf-16')(x) == ( b'\xfe\xff\x001\x002\x003\x00a\x00b\x00c', 6) @@ -611,16 +611,16 @@ b'\xff\xfe1\x002\x003\x00a\x00b\x00c\x00') == (x, 14) def test_unicode_escape(self): - assert u'\\'.encode('unicode-escape') == b'\\\\' - assert b'\\\\'.decode('unicode-escape') == u'\\' - assert u'\ud801'.encode('unicode-escape') == b'\\ud801' - assert u'\u0013'.encode('unicode-escape') == b'\\x13' + assert '\\'.encode('unicode-escape') == b'\\\\' + assert b'\\\\'.decode('unicode-escape') == '\\' + assert '\ud801'.encode('unicode-escape') == b'\\ud801' + assert '\u0013'.encode('unicode-escape') == b'\\x13' def test_mbcs(self): import sys if sys.platform != 'win32': return - assert u'test'.encode('mbcs') == 'test' - assert u'caf\xe9'.encode('mbcs') == 'caf\xe9' - assert u'\u040a'.encode('mbcs') == '?' # some cyrillic letter - assert 'cafx\e9'.decode('mbcs') == u'cafx\e9' + assert 'test'.encode('mbcs') == 'test' + assert 'caf\xe9'.encode('mbcs') == 'caf\xe9' + assert '\u040a'.encode('mbcs') == '?' # some cyrillic letter + assert 'cafx\e9'.decode('mbcs') == 'cafx\e9' From noreply at buildbot.pypy.org Sun Jan 29 23:51:06 2012 From: noreply at buildbot.pypy.org (amauryfa) Date: Sun, 29 Jan 2012 23:51:06 +0100 (CET) Subject: [pypy-commit] pypy py3k: hg merge default Message-ID: <20120129225106.E37D38203C@wyvern.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc <amauryfa at gmail.com> Branch: py3k Changeset: r51961:a5bd7b8f5f80 Date: 2012-01-29 23:50 +0100 http://bitbucket.org/pypy/pypy/changeset/a5bd7b8f5f80/ Log: hg merge default diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py --- a/lib_pypy/_sqlite3.py +++ b/lib_pypy/_sqlite3.py @@ -20,6 +20,8 @@ # 2. Altered source versions must be plainly marked as such, and must not be # misrepresented as being the original software. # 3. This notice may not be removed or altered from any source distribution. +# +# Note: This software has been modified for use in PyPy. from ctypes import c_void_p, c_int, c_double, c_int64, c_char_p, cdll from ctypes import POINTER, byref, string_at, CFUNCTYPE, cast @@ -27,7 +29,6 @@ from collections import OrderedDict import datetime import sys -import time import weakref from threading import _get_ident as thread_get_ident @@ -606,7 +607,7 @@ def authorizer(userdata, action, arg1, arg2, dbname, source): try: return int(callback(action, arg1, arg2, dbname, source)) - except Exception as e: + except Exception: return SQLITE_DENY c_authorizer = AUTHORIZER(authorizer) @@ -653,7 +654,7 @@ if not aggregate_ptr[0]: try: aggregate = cls() - except Exception as e: + except Exception: msg = ("user-defined aggregate's '__init__' " "method raised error") sqlite.sqlite3_result_error(context, msg, len(msg)) @@ -667,7 +668,7 @@ params = _convert_params(context, argc, c_params) try: aggregate.step(*params) - except Exception as e: + except Exception: msg = ("user-defined aggregate's 'step' " "method raised error") sqlite.sqlite3_result_error(context, msg, len(msg)) @@ -683,7 +684,7 @@ aggregate = self.aggregate_instances[aggregate_ptr[0]] try: val = aggregate.finalize() - except Exception as e: + except Exception: msg = ("user-defined aggregate's 'finalize' " "method raised error") sqlite.sqlite3_result_error(context, msg, len(msg)) @@ -771,7 +772,7 @@ self.statement.item = None self.statement.exhausted = True - if self.statement.kind == DML or self.statement.kind == DDL: + if self.statement.kind == DML: self.statement.reset() self.rowcount = -1 @@ -861,8 +862,6 @@ except StopIteration: return None - return nextrow - def fetchmany(self, size=None): self._check_closed() self._check_reset() @@ -1050,7 +1049,7 @@ param_name = param_name[1:] try: param = params[param_name] - except KeyError as e: + except KeyError: raise ProgrammingError("missing parameter '%s'" %param) self.set_param(idx, param) @@ -1261,7 +1260,7 @@ params = _convert_params(context, nargs, c_params) try: val = real_cb(*params) - except Exception as e: + except Exception: msg = "user-defined function raised exception" sqlite.sqlite3_result_error(context, msg, len(msg)) else: diff --git a/lib_pypy/numpypy/core/__init__.py b/lib_pypy/numpypy/core/__init__.py --- a/lib_pypy/numpypy/core/__init__.py +++ b/lib_pypy/numpypy/core/__init__.py @@ -1,1 +1,2 @@ from .fromnumeric import * +from .numeric import * diff --git a/lib_pypy/numpypy/core/_methods.py b/lib_pypy/numpypy/core/_methods.py new file mode 100644 --- /dev/null +++ b/lib_pypy/numpypy/core/_methods.py @@ -0,0 +1,98 @@ +# Array methods which are called by the both the C-code for the method +# and the Python code for the NumPy-namespace function + +import _numpypy as mu +um = mu +#from numpypy.core import umath as um +from numpypy.core.numeric import asanyarray + +def _amax(a, axis=None, out=None, skipna=False, keepdims=False): + return um.maximum.reduce(a, axis=axis, + out=out, skipna=skipna, keepdims=keepdims) + +def _amin(a, axis=None, out=None, skipna=False, keepdims=False): + return um.minimum.reduce(a, axis=axis, + out=out, skipna=skipna, keepdims=keepdims) + +def _sum(a, axis=None, dtype=None, out=None, skipna=False, keepdims=False): + return um.add.reduce(a, axis=axis, dtype=dtype, + out=out, skipna=skipna, keepdims=keepdims) + +def _prod(a, axis=None, dtype=None, out=None, skipna=False, keepdims=False): + return um.multiply.reduce(a, axis=axis, dtype=dtype, + out=out, skipna=skipna, keepdims=keepdims) + +def _mean(a, axis=None, dtype=None, out=None, skipna=False, keepdims=False): + arr = asanyarray(a) + + # Upgrade bool, unsigned int, and int to float64 + if dtype is None and arr.dtype.kind in ['b','u','i']: + ret = um.add.reduce(arr, axis=axis, dtype='f8', + out=out, skipna=skipna, keepdims=keepdims) + else: + ret = um.add.reduce(arr, axis=axis, dtype=dtype, + out=out, skipna=skipna, keepdims=keepdims) + rcount = mu.count_reduce_items(arr, axis=axis, + skipna=skipna, keepdims=keepdims) + if isinstance(ret, mu.ndarray): + ret = um.true_divide(ret, rcount, + casting='unsafe', subok=False) + else: + ret = ret / float(rcount) + return ret + +def _var(a, axis=None, dtype=None, out=None, ddof=0, + skipna=False, keepdims=False): + arr = asanyarray(a) + + # First compute the mean, saving 'rcount' for reuse later + if dtype is None and arr.dtype.kind in ['b','u','i']: + arrmean = um.add.reduce(arr, axis=axis, dtype='f8', + skipna=skipna, keepdims=True) + else: + arrmean = um.add.reduce(arr, axis=axis, dtype=dtype, + skipna=skipna, keepdims=True) + rcount = mu.count_reduce_items(arr, axis=axis, + skipna=skipna, keepdims=True) + if isinstance(arrmean, mu.ndarray): + arrmean = um.true_divide(arrmean, rcount, + casting='unsafe', subok=False) + else: + arrmean = arrmean / float(rcount) + + # arr - arrmean + x = arr - arrmean + + # (arr - arrmean) ** 2 + if arr.dtype.kind == 'c': + x = um.multiply(x, um.conjugate(x)).real + else: + x = um.multiply(x, x) + + # add.reduce((arr - arrmean) ** 2, axis) + ret = um.add.reduce(x, axis=axis, dtype=dtype, out=out, + skipna=skipna, keepdims=keepdims) + + # add.reduce((arr - arrmean) ** 2, axis) / (n - ddof) + if not keepdims and isinstance(rcount, mu.ndarray): + rcount = rcount.squeeze(axis=axis) + rcount -= ddof + if isinstance(ret, mu.ndarray): + ret = um.true_divide(ret, rcount, + casting='unsafe', subok=False) + else: + ret = ret / float(rcount) + + return ret + +def _std(a, axis=None, dtype=None, out=None, ddof=0, + skipna=False, keepdims=False): + ret = _var(a, axis=axis, dtype=dtype, out=out, ddof=ddof, + skipna=skipna, keepdims=keepdims) + + if isinstance(ret, mu.ndarray): + ret = um.sqrt(ret) + else: + ret = um.sqrt(ret) + + return ret diff --git a/lib_pypy/numpypy/core/arrayprint.py b/lib_pypy/numpypy/core/arrayprint.py new file mode 100644 --- /dev/null +++ b/lib_pypy/numpypy/core/arrayprint.py @@ -0,0 +1,789 @@ +"""Array printing function + +$Id: arrayprint.py,v 1.9 2005/09/13 13:58:44 teoliphant Exp $ +""" +__all__ = ["array2string", "set_printoptions", "get_printoptions"] +__docformat__ = 'restructuredtext' + +# +# Written by Konrad Hinsen <hinsenk at ere.umontreal.ca> +# last revision: 1996-3-13 +# modified by Jim Hugunin 1997-3-3 for repr's and str's (and other details) +# and by Perry Greenfield 2000-4-1 for numarray +# and by Travis Oliphant 2005-8-22 for numpy + +import sys +import _numpypy as _nt +from _numpypy import maximum, minimum, absolute, not_equal, isinf, isnan, isna +#from _numpypy import format_longfloat, datetime_as_string, datetime_data +from .fromnumeric import ravel + + +def product(x, y): return x*y + +_summaryEdgeItems = 3 # repr N leading and trailing items of each dimension +_summaryThreshold = 1000 # total items > triggers array summarization + +_float_output_precision = 8 +_float_output_suppress_small = False +_line_width = 75 +_nan_str = 'nan' +_inf_str = 'inf' +_na_str = 'NA' +_formatter = None # formatting function for array elements + +if sys.version_info[0] >= 3: + from functools import reduce + +def set_printoptions(precision=None, threshold=None, edgeitems=None, + linewidth=None, suppress=None, + nanstr=None, infstr=None, nastr=None, + formatter=None): + """ + Set printing options. + + These options determine the way floating point numbers, arrays and + other NumPy objects are displayed. + + Parameters + ---------- + precision : int, optional + Number of digits of precision for floating point output (default 8). + threshold : int, optional + Total number of array elements which trigger summarization + rather than full repr (default 1000). + edgeitems : int, optional + Number of array items in summary at beginning and end of + each dimension (default 3). + linewidth : int, optional + The number of characters per line for the purpose of inserting + line breaks (default 75). + suppress : bool, optional + Whether or not suppress printing of small floating point values + using scientific notation (default False). + nanstr : str, optional + String representation of floating point not-a-number (default nan). + infstr : str, optional + String representation of floating point infinity (default inf). + nastr : str, optional + String representation of NA missing value (default NA). + formatter : dict of callables, optional + If not None, the keys should indicate the type(s) that the respective + formatting function applies to. Callables should return a string. + Types that are not specified (by their corresponding keys) are handled + by the default formatters. Individual types for which a formatter + can be set are:: + + - 'bool' + - 'int' + - 'timedelta' : a `numpy.timedelta64` + - 'datetime' : a `numpy.datetime64` + - 'float' + - 'longfloat' : 128-bit floats + - 'complexfloat' + - 'longcomplexfloat' : composed of two 128-bit floats + - 'numpy_str' : types `numpy.string_` and `numpy.unicode_` + - 'str' : all other strings + + Other keys that can be used to set a group of types at once are:: + + - 'all' : sets all types + - 'int_kind' : sets 'int' + - 'float_kind' : sets 'float' and 'longfloat' + - 'complex_kind' : sets 'complexfloat' and 'longcomplexfloat' + - 'str_kind' : sets 'str' and 'numpystr' + + See Also + -------- + get_printoptions, set_string_function, array2string + + Notes + ----- + `formatter` is always reset with a call to `set_printoptions`. + + Examples + -------- + Floating point precision can be set: + + >>> np.set_printoptions(precision=4) + >>> print np.array([1.123456789]) + [ 1.1235] + + Long arrays can be summarised: + + >>> np.set_printoptions(threshold=5) + >>> print np.arange(10) + [0 1 2 ..., 7 8 9] + + Small results can be suppressed: + + >>> eps = np.finfo(float).eps + >>> x = np.arange(4.) + >>> x**2 - (x + eps)**2 + array([ -4.9304e-32, -4.4409e-16, 0.0000e+00, 0.0000e+00]) + >>> np.set_printoptions(suppress=True) + >>> x**2 - (x + eps)**2 + array([-0., -0., 0., 0.]) + + A custom formatter can be used to display array elements as desired: + + >>> np.set_printoptions(formatter={'all':lambda x: 'int: '+str(-x)}) + >>> x = np.arange(3) + >>> x + array([int: 0, int: -1, int: -2]) + >>> np.set_printoptions() # formatter gets reset + >>> x + array([0, 1, 2]) + + To put back the default options, you can use: + + >>> np.set_printoptions(edgeitems=3,infstr='inf', + ... linewidth=75, nanstr='nan', precision=8, + ... suppress=False, threshold=1000, formatter=None) + """ + + global _summaryThreshold, _summaryEdgeItems, _float_output_precision, \ + _line_width, _float_output_suppress_small, _nan_str, _inf_str, \ + _na_str, _formatter + if linewidth is not None: + _line_width = linewidth + if threshold is not None: + _summaryThreshold = threshold + if edgeitems is not None: + _summaryEdgeItems = edgeitems + if precision is not None: + _float_output_precision = precision + if suppress is not None: + _float_output_suppress_small = not not suppress + if nanstr is not None: + _nan_str = nanstr + if infstr is not None: + _inf_str = infstr + if nastr is not None: + _na_str = nastr + _formatter = formatter + +def get_printoptions(): + """ + Return the current print options. + + Returns + ------- + print_opts : dict + Dictionary of current print options with keys + + - precision : int + - threshold : int + - edgeitems : int + - linewidth : int + - suppress : bool + - nanstr : str + - infstr : str + - formatter : dict of callables + + For a full description of these options, see `set_printoptions`. + + See Also + -------- + set_printoptions, set_string_function + + """ + d = dict(precision=_float_output_precision, + threshold=_summaryThreshold, + edgeitems=_summaryEdgeItems, + linewidth=_line_width, + suppress=_float_output_suppress_small, + nanstr=_nan_str, + infstr=_inf_str, + nastr=_na_str, + formatter=_formatter) + return d + +def _leading_trailing(a): + import numeric as _nc + if a.ndim == 1: + if len(a) > 2*_summaryEdgeItems: + b = _nc.concatenate((a[:_summaryEdgeItems], + a[-_summaryEdgeItems:])) + else: + b = a + else: + if len(a) > 2*_summaryEdgeItems: + l = [_leading_trailing(a[i]) for i in range( + min(len(a), _summaryEdgeItems))] + l.extend([_leading_trailing(a[-i]) for i in range( + min(len(a), _summaryEdgeItems),0,-1)]) + else: + l = [_leading_trailing(a[i]) for i in range(0, len(a))] + b = _nc.concatenate(tuple(l)) + return b + +def _boolFormatter(x): + if isna(x): + return str(x).replace('NA', _na_str, 1) + elif x: + return ' True' + else: + return 'False' + + +def repr_format(x): + if isna(x): + return str(x).replace('NA', _na_str, 1) + else: + return repr(x) + +def _array2string(a, max_line_width, precision, suppress_small, separator=' ', + prefix="", formatter=None): + + if max_line_width is None: + max_line_width = _line_width + + if precision is None: + precision = _float_output_precision + + if suppress_small is None: + suppress_small = _float_output_suppress_small + + if formatter is None: + formatter = _formatter + + if a.size > _summaryThreshold: + summary_insert = "..., " + data = _leading_trailing(a) + else: + summary_insert = "" + data = ravel(a) + + formatdict = {'bool' : _boolFormatter, + 'int' : IntegerFormat(data), + 'float' : FloatFormat(data, precision, suppress_small), + 'longfloat' : LongFloatFormat(precision), + #'complexfloat' : ComplexFormat(data, precision, + # suppress_small), + #'longcomplexfloat' : LongComplexFormat(precision), + #'datetime' : DatetimeFormat(data), + #'timedelta' : TimedeltaFormat(data), + 'numpystr' : repr_format, + 'str' : str} + + if formatter is not None: + fkeys = [k for k in formatter.keys() if formatter[k] is not None] + if 'all' in fkeys: + for key in formatdict.keys(): + formatdict[key] = formatter['all'] + if 'int_kind' in fkeys: + for key in ['int']: + formatdict[key] = formatter['int_kind'] + if 'float_kind' in fkeys: + for key in ['float', 'longfloat']: + formatdict[key] = formatter['float_kind'] + if 'complex_kind' in fkeys: + for key in ['complexfloat', 'longcomplexfloat']: + formatdict[key] = formatter['complex_kind'] + if 'str_kind' in fkeys: + for key in ['numpystr', 'str']: + formatdict[key] = formatter['str_kind'] + for key in formatdict.keys(): + if key in fkeys: + formatdict[key] = formatter[key] + + try: + format_function = a._format + msg = "The `_format` attribute is deprecated in Numpy 2.0 and " \ + "will be removed in 2.1. Use the `formatter` kw instead." + import warnings + warnings.warn(msg, DeprecationWarning) + except AttributeError: + # find the right formatting function for the array + dtypeobj = a.dtype.type + if issubclass(dtypeobj, _nt.bool_): + format_function = formatdict['bool'] + elif issubclass(dtypeobj, _nt.integer): + #if issubclass(dtypeobj, _nt.timedelta64): + # format_function = formatdict['timedelta'] + #else: + format_function = formatdict['int'] + elif issubclass(dtypeobj, _nt.floating): + #if issubclass(dtypeobj, _nt.longfloat): + # format_function = formatdict['longfloat'] + #else: + format_function = formatdict['float'] + elif issubclass(dtypeobj, _nt.complexfloating): + if issubclass(dtypeobj, _nt.clongfloat): + format_function = formatdict['longcomplexfloat'] + else: + format_function = formatdict['complexfloat'] + elif issubclass(dtypeobj, (_nt.unicode_, _nt.string_)): + format_function = formatdict['numpystr'] + elif issubclass(dtypeobj, _nt.datetime64): + format_function = formatdict['datetime'] + else: + format_function = formatdict['str'] + + # skip over "[" + next_line_prefix = " " + # skip over array( + next_line_prefix += " "*len(prefix) + + lst = _formatArray(a, format_function, len(a.shape), max_line_width, + next_line_prefix, separator, + _summaryEdgeItems, summary_insert)[:-1] + return lst + +def _convert_arrays(obj): + import numeric as _nc + newtup = [] + for k in obj: + if isinstance(k, _nc.ndarray): + k = k.tolist() + elif isinstance(k, tuple): + k = _convert_arrays(k) + newtup.append(k) + return tuple(newtup) + + +def array2string(a, max_line_width=None, precision=None, + suppress_small=None, separator=' ', prefix="", + style=repr, formatter=None): + """ + Return a string representation of an array. + + Parameters + ---------- + a : ndarray + Input array. + max_line_width : int, optional + The maximum number of columns the string should span. Newline + characters splits the string appropriately after array elements. + precision : int, optional + Floating point precision. Default is the current printing + precision (usually 8), which can be altered using `set_printoptions`. + suppress_small : bool, optional + Represent very small numbers as zero. A number is "very small" if it + is smaller than the current printing precision. + separator : str, optional + Inserted between elements. + prefix : str, optional + An array is typically printed as:: + + 'prefix(' + array2string(a) + ')' + + The length of the prefix string is used to align the + output correctly. + style : function, optional + A function that accepts an ndarray and returns a string. Used only + when the shape of `a` is equal to ``()``, i.e. for 0-D arrays. + formatter : dict of callables, optional + If not None, the keys should indicate the type(s) that the respective + formatting function applies to. Callables should return a string. + Types that are not specified (by their corresponding keys) are handled + by the default formatters. Individual types for which a formatter + can be set are:: + + - 'bool' + - 'int' + - 'timedelta' : a `numpy.timedelta64` + - 'datetime' : a `numpy.datetime64` + - 'float' + - 'longfloat' : 128-bit floats + - 'complexfloat' + - 'longcomplexfloat' : composed of two 128-bit floats + - 'numpy_str' : types `numpy.string_` and `numpy.unicode_` + - 'str' : all other strings + + Other keys that can be used to set a group of types at once are:: + + - 'all' : sets all types + - 'int_kind' : sets 'int' + - 'float_kind' : sets 'float' and 'longfloat' + - 'complex_kind' : sets 'complexfloat' and 'longcomplexfloat' + - 'str_kind' : sets 'str' and 'numpystr' + + Returns + ------- + array_str : str + String representation of the array. + + Raises + ------ + TypeError : if a callable in `formatter` does not return a string. + + See Also + -------- + array_str, array_repr, set_printoptions, get_printoptions + + Notes + ----- + If a formatter is specified for a certain type, the `precision` keyword is + ignored for that type. + + Examples + -------- + >>> x = np.array([1e-16,1,2,3]) + >>> print np.array2string(x, precision=2, separator=',', + ... suppress_small=True) + [ 0., 1., 2., 3.] + + >>> x = np.arange(3.) + >>> np.array2string(x, formatter={'float_kind':lambda x: "%.2f" % x}) + '[0.00 1.00 2.00]' + + >>> x = np.arange(3) + >>> np.array2string(x, formatter={'int':lambda x: hex(x)}) + '[0x0L 0x1L 0x2L]' + + """ + + if a.shape == (): + x = a.item() + if isna(x): + lst = str(x).replace('NA', _na_str, 1) + else: + try: + lst = a._format(x) + msg = "The `_format` attribute is deprecated in Numpy " \ + "2.0 and will be removed in 2.1. Use the " \ + "`formatter` kw instead." + import warnings + warnings.warn(msg, DeprecationWarning) + except AttributeError: + if isinstance(x, tuple): + x = _convert_arrays(x) + lst = style(x) + elif reduce(product, a.shape) == 0: + # treat as a null array if any of shape elements == 0 + lst = "[]" + else: + lst = _array2string(a, max_line_width, precision, suppress_small, + separator, prefix, formatter=formatter) + return lst + +def _extendLine(s, line, word, max_line_len, next_line_prefix): + if len(line.rstrip()) + len(word.rstrip()) >= max_line_len: + s += line.rstrip() + "\n" + line = next_line_prefix + line += word + return s, line + + +def _formatArray(a, format_function, rank, max_line_len, + next_line_prefix, separator, edge_items, summary_insert): + """formatArray is designed for two modes of operation: + + 1. Full output + + 2. Summarized output + + """ + if rank == 0: + obj = a.item() + if isinstance(obj, tuple): + obj = _convert_arrays(obj) + return str(obj) + + if summary_insert and 2*edge_items < len(a): + leading_items, trailing_items, summary_insert1 = \ + edge_items, edge_items, summary_insert + else: + leading_items, trailing_items, summary_insert1 = 0, len(a), "" + + if rank == 1: + s = "" + line = next_line_prefix + for i in xrange(leading_items): + word = format_function(a[i]) + separator + s, line = _extendLine(s, line, word, max_line_len, next_line_prefix) + + if summary_insert1: + s, line = _extendLine(s, line, summary_insert1, max_line_len, next_line_prefix) + + for i in xrange(trailing_items, 1, -1): + word = format_function(a[-i]) + separator + s, line = _extendLine(s, line, word, max_line_len, next_line_prefix) + + word = format_function(a[-1]) + s, line = _extendLine(s, line, word, max_line_len, next_line_prefix) + s += line + "]\n" + s = '[' + s[len(next_line_prefix):] + else: + s = '[' + sep = separator.rstrip() + for i in xrange(leading_items): + if i > 0: + s += next_line_prefix + s += _formatArray(a[i], format_function, rank-1, max_line_len, + " " + next_line_prefix, separator, edge_items, + summary_insert) + s = s.rstrip() + sep.rstrip() + '\n'*max(rank-1,1) + + if summary_insert1: + s += next_line_prefix + summary_insert1 + "\n" + + for i in xrange(trailing_items, 1, -1): + if leading_items or i != trailing_items: + s += next_line_prefix + s += _formatArray(a[-i], format_function, rank-1, max_line_len, + " " + next_line_prefix, separator, edge_items, + summary_insert) + s = s.rstrip() + sep.rstrip() + '\n'*max(rank-1,1) + if leading_items or trailing_items > 1: + s += next_line_prefix + s += _formatArray(a[-1], format_function, rank-1, max_line_len, + " " + next_line_prefix, separator, edge_items, + summary_insert).rstrip()+']\n' + return s + +class FloatFormat(object): + def __init__(self, data, precision, suppress_small, sign=False): + self.precision = precision + self.suppress_small = suppress_small + self.sign = sign + self.exp_format = False + self.large_exponent = False + self.max_str_len = 0 + #try: + self.fillFormat(data) + #except (TypeError, NotImplementedError): + # if reduce(data) fails, this instance will not be called, just + # instantiated in formatdict. + #pass + + def fillFormat(self, data): + import numeric as _nc + # XXX pypy unimplemented + #errstate = _nc.seterr(all='ignore') + try: + special = isnan(data) | isinf(data) | isna(data) + special[isna(data)] = False + valid = not_equal(data, 0) & ~special + valid[isna(data)] = False + non_zero = absolute(data.compress(valid)) + if len(non_zero) == 0: + max_val = 0. + min_val = 0. + else: + max_val = maximum.reduce(non_zero, skipna=True) + min_val = minimum.reduce(non_zero, skipna=True) + if max_val >= 1.e8: + self.exp_format = True + if not self.suppress_small and (min_val < 0.0001 + or max_val/min_val > 1000.): + self.exp_format = True + finally: + pass + # XXX pypy unimplemented + #_nc.seterr(**errstate) + + if self.exp_format: + self.large_exponent = 0 < min_val < 1e-99 or max_val >= 1e100 + self.max_str_len = 8 + self.precision + if self.large_exponent: + self.max_str_len += 1 + if self.sign: + format = '%+' + else: + format = '%' + format = format + '%d.%de' % (self.max_str_len, self.precision) + else: + format = '%%.%df' % (self.precision,) + if len(non_zero): + precision = max([_digits(x, self.precision, format) + for x in non_zero]) + else: + precision = 0 + precision = min(self.precision, precision) + self.max_str_len = len(str(int(max_val))) + precision + 2 + if special.any(): + self.max_str_len = max(self.max_str_len, + len(_nan_str), + len(_inf_str)+1, + len(_na_str)) + if self.sign: + format = '%#+' + else: + format = '%#' + format = format + '%d.%df' % (self.max_str_len, precision) + + self.special_fmt = '%%%ds' % (self.max_str_len,) + self.format = format + + def __call__(self, x, strip_zeros=True): + import numeric as _nc + #err = _nc.seterr(invalid='ignore') + try: + if isna(x): + return self.special_fmt % (str(x).replace('NA', _na_str, 1),) + elif isnan(x): + if self.sign: + return self.special_fmt % ('+' + _nan_str,) + else: + return self.special_fmt % (_nan_str,) + elif isinf(x): + if x > 0: + if self.sign: + return self.special_fmt % ('+' + _inf_str,) + else: + return self.special_fmt % (_inf_str,) + else: + return self.special_fmt % ('-' + _inf_str,) + finally: + pass + #_nc.seterr(**err) + + s = self.format % x + if self.large_exponent: + # 3-digit exponent + expsign = s[-3] + if expsign == '+' or expsign == '-': + s = s[1:-2] + '0' + s[-2:] + elif self.exp_format: + # 2-digit exponent + if s[-3] == '0': + s = ' ' + s[:-3] + s[-2:] + elif strip_zeros: + z = s.rstrip('0') + s = z + ' '*(len(s)-len(z)) + return s + + +def _digits(x, precision, format): + s = format % x + z = s.rstrip('0') + return precision - len(s) + len(z) + + +_MAXINT = sys.maxint +_MININT = -sys.maxint-1 +class IntegerFormat(object): + def __init__(self, data): + try: + max_str_len = max(len(str(maximum.reduce(data, skipna=True))), + len(str(minimum.reduce(data, skipna=True)))) + self.format = '%' + str(max_str_len) + 'd' + except TypeError, NotImplementedError: + # if reduce(data) fails, this instance will not be called, just + # instantiated in formatdict. + pass + except ValueError: + # this occurs when everything is NA + pass + + def __call__(self, x): + if isna(x): + return str(x).replace('NA', _na_str, 1) + elif _MININT < x < _MAXINT: + return self.format % x + else: + return "%s" % x + +class LongFloatFormat(object): + # XXX Have to add something to determine the width to use a la FloatFormat + # Right now, things won't line up properly + def __init__(self, precision, sign=False): + self.precision = precision + self.sign = sign + + def __call__(self, x): + if isna(x): + return str(x).replace('NA', _na_str, 1) + elif isnan(x): + if self.sign: + return '+' + _nan_str + else: + return ' ' + _nan_str + elif isinf(x): + if x > 0: + if self.sign: + return '+' + _inf_str + else: + return ' ' + _inf_str + else: + return '-' + _inf_str + elif x >= 0: + if self.sign: + return '+' + format_longfloat(x, self.precision) + else: + return ' ' + format_longfloat(x, self.precision) + else: + return format_longfloat(x, self.precision) + + +class LongComplexFormat(object): + def __init__(self, precision): + self.real_format = LongFloatFormat(precision) + self.imag_format = LongFloatFormat(precision, sign=True) + + def __call__(self, x): + if isna(x): + return str(x).replace('NA', _na_str, 1) + else: + r = self.real_format(x.real) + i = self.imag_format(x.imag) + return r + i + 'j' + + +class ComplexFormat(object): + def __init__(self, x, precision, suppress_small): + self.real_format = FloatFormat(x.real, precision, suppress_small) + self.imag_format = FloatFormat(x.imag, precision, suppress_small, + sign=True) + + def __call__(self, x): + if isna(x): + return str(x).replace('NA', _na_str, 1) + else: + r = self.real_format(x.real, strip_zeros=False) + i = self.imag_format(x.imag, strip_zeros=False) + if not self.imag_format.exp_format: + z = i.rstrip('0') + i = z + 'j' + ' '*(len(i)-len(z)) + else: + i = i + 'j' + return r + i + +class DatetimeFormat(object): + def __init__(self, x, unit=None, + timezone=None, casting='same_kind'): + # Get the unit from the dtype + if unit is None: + if x.dtype.kind == 'M': + unit = datetime_data(x.dtype)[0] + else: + unit = 's' + + # If timezone is default, make it 'local' or 'UTC' based on the unit + if timezone is None: + # Date units -> UTC, time units -> local + if unit in ('Y', 'M', 'W', 'D'): + self.timezone = 'UTC' + else: + self.timezone = 'local' + else: + self.timezone = timezone + self.unit = unit + self.casting = casting + + def __call__(self, x): + if isna(x): + return str(x).replace('NA', _na_str, 1) + else: + return "'%s'" % datetime_as_string(x, + unit=self.unit, + timezone=self.timezone, + casting=self.casting) + +class TimedeltaFormat(object): + def __init__(self, data): + if data.dtype.kind == 'm': + v = data.view('i8') + max_str_len = max(len(str(maximum.reduce(v))), + len(str(minimum.reduce(v)))) + self.format = '%' + str(max_str_len) + 'd' + + def __call__(self, x): + if isna(x): + return str(x).replace('NA', _na_str, 1) + else: + return self.format % x.astype('i8') + diff --git a/lib_pypy/numpypy/core/fromnumeric.py b/lib_pypy/numpypy/core/fromnumeric.py --- a/lib_pypy/numpypy/core/fromnumeric.py +++ b/lib_pypy/numpypy/core/fromnumeric.py @@ -30,7 +30,7 @@ 'rank', 'size', 'around', 'round_', 'mean', 'std', 'var', 'squeeze', 'amax', 'amin', ] - + def take(a, indices, axis=None, out=None, mode='raise'): """ Take elements from an array along an axis. @@ -149,6 +149,7 @@ [5, 6]]) """ + assert order == 'C' if not hasattr(a, 'reshape'): a = numpypy.array(a) return a.reshape(newshape) @@ -690,6 +691,7 @@ 1 """ + assert axis is None if not hasattr(a, 'argmax'): a = numpypy.array(a) return a.argmax() @@ -705,6 +707,7 @@ documentation. """ + assert axis is None if not hasattr(a, 'argmin'): a = numpypy.array(a) return a.argmin() @@ -1051,8 +1054,9 @@ array([ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]) """ - raise NotImplementedError('Waiting on interp level method') - + if not hasattr(a, 'ravel'): + a = numpypy.array(a) + return a.ravel(order=order) def nonzero(a): """ @@ -1354,9 +1358,11 @@ -128 """ + assert dtype is None + assert out is None if not hasattr(a, "sum"): a = numpypy.array(a) - return a.sum() + return a.sum(axis=axis) def product (a, axis=None, dtype=None, out=None): @@ -1382,6 +1388,8 @@ any : equivalent function """ + assert axis is None + assert out is None if not hasattr(a, 'any'): a = numpypy.array(a) return a.any() @@ -1396,6 +1404,8 @@ numpy.all : Equivalent function; see for details. """ + assert axis is None + assert out is None if not hasattr(a, 'all'): a = numpypy.array(a) return a.all() @@ -1464,6 +1474,8 @@ (191614240, 191614240) """ + assert axis is None + assert out is None if not hasattr(a, 'any'): a = numpypy.array(a) return a.any() @@ -1526,6 +1538,8 @@ (28293632, 28293632, array([ True], dtype=bool)) """ + assert axis is None + assert out is None if not hasattr(a, 'all'): a = numpypy.array(a) return a.all() @@ -1705,6 +1719,8 @@ 4.0 """ + assert axis is None + assert out is None if not hasattr(a, "max"): a = numpypy.array(a) return a.max() @@ -1766,6 +1782,8 @@ """ # amin() is equivalent to min() + assert axis is None + assert out is None if not hasattr(a, 'min'): a = numpypy.array(a) return a.min() @@ -2214,9 +2232,11 @@ 0.55000000074505806 """ + assert dtype is None + assert out is None if not hasattr(a, "mean"): a = numpypy.array(a) - return a.mean() + return a.mean(axis=axis) def std(a, axis=None, dtype=None, out=None, ddof=0): @@ -2305,9 +2325,12 @@ 0.44999999925552653 """ + assert dtype is None + assert out is None + assert ddof == 0 if not hasattr(a, "std"): a = numpypy.array(a) - return a.std() + return a.std(axis=axis) def var(a, axis=None, dtype=None, out=None, ddof=0): @@ -2398,6 +2421,9 @@ 0.20250000000000001 """ + assert dtype is None + assert out is None + assert ddof == 0 if not hasattr(a, "var"): a = numpypy.array(a) - return a.var() + return a.var(axis=axis) diff --git a/lib_pypy/numpypy/core/numeric.py b/lib_pypy/numpypy/core/numeric.py new file mode 100644 --- /dev/null +++ b/lib_pypy/numpypy/core/numeric.py @@ -0,0 +1,311 @@ + +from _numpypy import array, ndarray, int_, float_ #, complex_# , longlong +from _numpypy import concatenate +import sys +import _numpypy as multiarray # ARGH +from numpypy.core.arrayprint import array2string + + + +def asanyarray(a, dtype=None, order=None, maskna=None, ownmaskna=False): + """ + Convert the input to an ndarray, but pass ndarray subclasses through. + + Parameters + ---------- + a : array_like + Input data, in any form that can be converted to an array. This + includes scalars, lists, lists of tuples, tuples, tuples of tuples, + tuples of lists, and ndarrays. + dtype : data-type, optional + By default, the data-type is inferred from the input data. + order : {'C', 'F'}, optional + Whether to use row-major ('C') or column-major ('F') memory + representation. Defaults to 'C'. + maskna : bool or None, optional + If this is set to True, it forces the array to have an NA mask. + If this is set to False, it forces the array to not have an NA + mask. + ownmaskna : bool, optional + If this is set to True, forces the array to have a mask which + it owns. + + Returns + ------- + out : ndarray or an ndarray subclass + Array interpretation of `a`. If `a` is an ndarray or a subclass + of ndarray, it is returned as-is and no copy is performed. + + See Also + -------- + asarray : Similar function which always returns ndarrays. + ascontiguousarray : Convert input to a contiguous array. + asfarray : Convert input to a floating point ndarray. + asfortranarray : Convert input to an ndarray with column-major + memory order. + asarray_chkfinite : Similar function which checks input for NaNs and + Infs. + fromiter : Create an array from an iterator. + fromfunction : Construct an array by executing a function on grid + positions. + + Examples + -------- + Convert a list into an array: + + >>> a = [1, 2] + >>> np.asanyarray(a) + array([1, 2]) + + Instances of `ndarray` subclasses are passed through as-is: + + >>> a = np.matrix([1, 2]) + >>> np.asanyarray(a) is a + True + + """ + return array(a, dtype, copy=False, order=order, subok=True, + maskna=maskna, ownmaskna=ownmaskna) + +def base_repr(number, base=2, padding=0): + """ + Return a string representation of a number in the given base system. + + Parameters + ---------- + number : int + The value to convert. Only positive values are handled. + base : int, optional + Convert `number` to the `base` number system. The valid range is 2-36, + the default value is 2. + padding : int, optional + Number of zeros padded on the left. Default is 0 (no padding). + + Returns + ------- + out : str + String representation of `number` in `base` system. + + See Also + -------- + binary_repr : Faster version of `base_repr` for base 2. + + Examples + -------- + >>> np.base_repr(5) + '101' + >>> np.base_repr(6, 5) + '11' + >>> np.base_repr(7, base=5, padding=3) + '00012' + + >>> np.base_repr(10, base=16) + 'A' + >>> np.base_repr(32, base=16) + '20' + + """ + digits = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ' + if base > len(digits): + raise ValueError("Bases greater than 36 not handled in base_repr.") + + num = abs(number) + res = [] + while num: + res.append(digits[num % base]) + num //= base + if padding: + res.append('0' * padding) + if number < 0: + res.append('-') + return ''.join(reversed(res or '0')) + +_typelessdata = [int_, float_]#, complex_] +# XXX +#if issubclass(intc, int): +# _typelessdata.append(intc) + +#if issubclass(longlong, int): +# _typelessdata.append(longlong) + +def array_repr(arr, max_line_width=None, precision=None, suppress_small=None): + """ + Return the string representation of an array. + + Parameters + ---------- + arr : ndarray + Input array. + max_line_width : int, optional + The maximum number of columns the string should span. Newline + characters split the string appropriately after array elements. + precision : int, optional + Floating point precision. Default is the current printing precision + (usually 8), which can be altered using `set_printoptions`. + suppress_small : bool, optional + Represent very small numbers as zero, default is False. Very small + is defined by `precision`, if the precision is 8 then + numbers smaller than 5e-9 are represented as zero. + + Returns + ------- + string : str + The string representation of an array. + + See Also + -------- + array_str, array2string, set_printoptions + + Examples + -------- + >>> np.array_repr(np.array([1,2])) + 'array([1, 2])' + >>> np.array_repr(np.ma.array([0.])) + 'MaskedArray([ 0.])' + >>> np.array_repr(np.array([], np.int32)) + 'array([], dtype=int32)' + + >>> x = np.array([1e-6, 4e-7, 2, 3]) + >>> np.array_repr(x, precision=6, suppress_small=True) + 'array([ 0.000001, 0. , 2. , 3. ])' + + """ + if arr.size > 0 or arr.shape==(0,): + lst = array2string(arr, max_line_width, precision, suppress_small, + ', ', "array(") + else: # show zero-length shape unless it is (0,) + lst = "[], shape=%s" % (repr(arr.shape),) + + if arr.__class__ is not ndarray: + cName= arr.__class__.__name__ + else: + cName = "array" + + skipdtype = (arr.dtype.type in _typelessdata) and arr.size > 0 + + # XXX pypy lacks support + if 0 and arr.flags.maskna: + whichna = isna(arr) + # If nothing is NA, explicitly signal the NA-mask + if not any(whichna): + lst += ", maskna=True" + # If everything is NA, can't skip the dtype + if skipdtype and all(whichna): + skipdtype = False + + if skipdtype: + return "%s(%s)" % (cName, lst) + else: + typename = arr.dtype.name + # Quote typename in the output if it is "complex". + if typename and not (typename[0].isalpha() and typename.isalnum()): + typename = "'%s'" % typename + + lf = '' + if 0: # or issubclass(arr.dtype.type, flexible): + if arr.dtype.names: + typename = "%s" % str(arr.dtype) + else: + typename = "'%s'" % str(arr.dtype) + lf = '\n'+' '*len("array(") + return cName + "(%s, %sdtype=%s)" % (lst, lf, typename) + +def array_str(a, max_line_width=None, precision=None, suppress_small=None): + """ + Return a string representation of the data in an array. + + The data in the array is returned as a single string. This function is + similar to `array_repr`, the difference being that `array_repr` also + returns information on the kind of array and its data type. + + Parameters + ---------- + a : ndarray + Input array. + max_line_width : int, optional + Inserts newlines if text is longer than `max_line_width`. The + default is, indirectly, 75. + precision : int, optional + Floating point precision. Default is the current printing precision + (usually 8), which can be altered using `set_printoptions`. + suppress_small : bool, optional + Represent numbers "very close" to zero as zero; default is False. + Very close is defined by precision: if the precision is 8, e.g., + numbers smaller (in absolute value) than 5e-9 are represented as + zero. + + See Also + -------- + array2string, array_repr, set_printoptions + + Examples + -------- + >>> np.array_str(np.arange(3)) + '[0 1 2]' + + """ + return array2string(a, max_line_width, precision, suppress_small, ' ', "", str) + +def set_string_function(f, repr=True): + """ + Set a Python function to be used when pretty printing arrays. + + Parameters + ---------- + f : function or None + Function to be used to pretty print arrays. The function should expect + a single array argument and return a string of the representation of + the array. If None, the function is reset to the default NumPy function + to print arrays. + repr : bool, optional + If True (default), the function for pretty printing (``__repr__``) + is set, if False the function that returns the default string + representation (``__str__``) is set. + + See Also + -------- + set_printoptions, get_printoptions + + Examples + -------- + >>> def pprint(arr): + ... return 'HA! - What are you going to do now?' + ... + >>> np.set_string_function(pprint) + >>> a = np.arange(10) + >>> a + HA! - What are you going to do now? + >>> print a + [0 1 2 3 4 5 6 7 8 9] + + We can reset the function to the default: + + >>> np.set_string_function(None) + >>> a + array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) + + `repr` affects either pretty printing or normal string representation. + Note that ``__repr__`` is still affected by setting ``__str__`` + because the width of each array element in the returned string becomes + equal to the length of the result of ``__str__()``. + + >>> x = np.arange(4) + >>> np.set_string_function(lambda x:'random', repr=False) + >>> x.__str__() + 'random' + >>> x.__repr__() + 'array([ 0, 1, 2, 3])' + + """ + if f is None: + if repr: + return multiarray.set_string_function(array_repr, 1) + else: + return multiarray.set_string_function(array_str, 0) + else: + return multiarray.set_string_function(f, repr) + +set_string_function(array_str, 0) +set_string_function(array_repr, 1) + +little_endian = (sys.byteorder == 'little') diff --git a/lib_pypy/pyrepl/reader.py b/lib_pypy/pyrepl/reader.py --- a/lib_pypy/pyrepl/reader.py +++ b/lib_pypy/pyrepl/reader.py @@ -37,16 +37,16 @@ for i in range(256): c = chr(i) if c not in uc_map: - uc_map[c] = u'\\%03o'%i + uc_map[c] = u'\\%03o'%i return uc_map # disp_str proved to be a bottleneck for large inputs, so it's been # rewritten in C; it's not required though. try: raise ImportError # currently it's borked by the unicode support - + from _pyrepl_utils import disp_str, init_unctrl_map - + init_unctrl_map(_make_unctrl_map()) del init_unctrl_map @@ -118,7 +118,7 @@ (r'\C-x\C-u', 'upcase-region'), (r'\C-y', 'yank'), (r'\C-z', 'suspend'), - + (r'\M-b', 'backward-word'), (r'\M-c', 'capitalize-word'), (r'\M-d', 'kill-word'), @@ -303,7 +303,7 @@ def process_prompt(self, prompt): """ Process the prompt. - + This means calculate the length of the prompt. The character \x01 and \x02 are used to bracket ANSI control sequences and need to be excluded from the length calculation. So also a copy of the prompt @@ -372,7 +372,7 @@ while p >= 0 and b[p] <> '\n': p -= 1 return p + 1 - + def eol(self, p=None): """Return the 0-based index of the line break following p most immediately. @@ -463,7 +463,7 @@ """This function is called to allow post command cleanup.""" if getattr(cmd, "kills_digit_arg", 1): if self.arg is not None: - self.dirty = 1 + self.dirty = 1 self.arg = None def prepare(self): @@ -575,13 +575,15 @@ def push_char(self, char): self.console.push_char(char) self.handle1(0) - - def readline(self, returns_unicode=False): + + def readline(self, returns_unicode=False, startup_hook=None): """Read a line. The implementation of this method also shows how to drive Reader if you want more control over the event loop.""" self.prepare() try: + if startup_hook is not None: + startup_hook() self.refresh() while not self.finished: self.handle1() diff --git a/lib_pypy/pyrepl/readline.py b/lib_pypy/pyrepl/readline.py --- a/lib_pypy/pyrepl/readline.py +++ b/lib_pypy/pyrepl/readline.py @@ -193,10 +193,8 @@ reader = self.get_reader() except _error: return _old_raw_input(prompt) - if self.startup_hook is not None: - self.startup_hook() reader.ps1 = prompt - return reader.readline() + return reader.readline(reader, startup_hook=self.startup_hook) def multiline_input(self, more_lines, ps1, ps2, returns_unicode=False): """Read an input on possibly multiple lines, asking for more @@ -390,7 +388,7 @@ global _old_raw_input if _old_raw_input is not None: return # don't run _setup twice - + try: f_in = sys.stdin.fileno() f_out = sys.stdout.fileno() diff --git a/pypy/annotation/unaryop.py b/pypy/annotation/unaryop.py --- a/pypy/annotation/unaryop.py +++ b/pypy/annotation/unaryop.py @@ -441,6 +441,12 @@ def method_popitem(dct): return dct.getanyitem('items') + def method_pop(dct, s_key, s_dfl=None): + dct.dictdef.generalize_key(s_key) + if s_dfl is not None: + dct.dictdef.generalize_value(s_dfl) + return dct.dictdef.read_value() + def _can_only_throw(dic, *ignore): if dic1.dictdef.dictkey.custom_eq_hash: return None # r_dict: can throw anything diff --git a/pypy/bin/py.py b/pypy/bin/py.py --- a/pypy/bin/py.py +++ b/pypy/bin/py.py @@ -76,6 +76,8 @@ config.objspace.suggest(allworkingmodules=False) if config.objspace.allworkingmodules: pypyoption.enable_allworkingmodules(config) + if config.objspace.usemodules._continuation: + config.translation.continuation = True if config.objspace.usemodules.thread: config.translation.thread = True diff --git a/pypy/doc/getting-started.rst b/pypy/doc/getting-started.rst --- a/pypy/doc/getting-started.rst +++ b/pypy/doc/getting-started.rst @@ -53,11 +53,11 @@ PyPy is ready to be executed as soon as you unpack the tarball or the zip file, with no need to install it in any specific location:: - $ tar xf pypy-1.6-linux.tar.bz2 + $ tar xf pypy-1.7-linux.tar.bz2 - $ ./pypy-1.6/bin/pypy + $ ./pypy-1.7/bin/pypy Python 2.7.1 (?, Apr 27 2011, 12:44:21) - [PyPy 1.6.0 with GCC 4.4.3] on linux2 + [PyPy 1.7.0 with GCC 4.4.3] on linux2 Type "help", "copyright", "credits" or "license" for more information. And now for something completely different: ``implementing LOGO in LOGO: "turtles all the way down"'' @@ -75,14 +75,14 @@ $ curl -O https://raw.github.com/pypa/pip/master/contrib/get-pip.py - $ ./pypy-1.6/bin/pypy distribute_setup.py + $ ./pypy-1.7/bin/pypy distribute_setup.py - $ ./pypy-1.6/bin/pypy get-pip.py + $ ./pypy-1.7/bin/pypy get-pip.py - $ ./pypy-1.6/bin/pip install pygments # for example + $ ./pypy-1.7/bin/pip install pygments # for example -3rd party libraries will be installed in ``pypy-1.6/site-packages``, and -the scripts in ``pypy-1.6/bin``. +3rd party libraries will be installed in ``pypy-1.7/site-packages``, and +the scripts in ``pypy-1.7/bin``. Installing using virtualenv --------------------------- diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py --- a/pypy/interpreter/executioncontext.py +++ b/pypy/interpreter/executioncontext.py @@ -445,6 +445,7 @@ AsyncAction.__init__(self, space) self.dying_objects = [] self.finalizers_lock_count = 0 + self.enabled_at_app_level = True def register_callback(self, w_obj, callback, descrname): self.dying_objects.append((w_obj, callback, descrname)) diff --git a/pypy/jit/backend/llsupport/gc.py b/pypy/jit/backend/llsupport/gc.py --- a/pypy/jit/backend/llsupport/gc.py +++ b/pypy/jit/backend/llsupport/gc.py @@ -394,13 +394,13 @@ This is the class supporting --gcrootfinder=shadowstack. """ is_shadow_stack = True - MARKER = 8 + MARKER_FRAME = 8 # this marker now *follows* the frame addr # The "shadowstack" is a portable way in which the GC finds the # roots that live in the stack. Normally it is just a list of # pointers to GC objects. The pointers may be moved around by a GC - # collection. But with the JIT, an entry can also be MARKER, in - # which case the next entry points to an assembler stack frame. + # collection. But with the JIT, an entry can also be MARKER_FRAME, + # in which case the previous entry points to an assembler stack frame. # During a residual CALL from the assembler (which may indirectly # call the GC), we use the force_index stored in the assembler # stack frame to identify the call: we can go from the force_index @@ -433,11 +433,16 @@ class RootIterator: _alloc_flavor_ = "raw" - def next(iself, gc, next, range_highest): - # Return the "next" valid GC object' address. This usually - # means just returning "next", until we reach "range_highest", - # except that we are skipping NULLs. If "next" contains a - # MARKER instead, then we go into JIT-frame-lookup mode. + def setcontext(iself, context): + iself.context = context + + def nextleft(iself, gc, range_lowest, prev): + # Return the next valid GC object's address, in right-to-left + # order from the shadowstack array. This usually means just + # returning "prev - sizeofaddr", until we reach "range_lowest", + # except that we are skipping NULLs. If "prev - sizeofaddr" + # contains a MARKER_FRAME instead, then we go into + # JIT-frame-lookup mode. # while True: # @@ -446,20 +451,20 @@ # # Look for the next shadowstack address that # contains a valid pointer - while next != range_highest: - if next.signed[0] == self.MARKER: + while prev != range_lowest: + prev -= llmemory.sizeof(llmemory.Address) + if prev.signed[0] == self.MARKER_FRAME: break - if gc.points_to_valid_gc_object(next): - return next - next += llmemory.sizeof(llmemory.Address) + if gc.points_to_valid_gc_object(prev): + return prev else: return llmemory.NULL # done # - # It's a JIT frame. Save away 'next' for later, and + # It's a JIT frame. Save away 'prev' for later, and # go into JIT-frame-exploring mode. - next += llmemory.sizeof(llmemory.Address) - frame_addr = next.signed[0] - iself.saved_next = next + prev -= llmemory.sizeof(llmemory.Address) + frame_addr = prev.signed[0] + iself.saved_prev = prev iself.frame_addr = frame_addr addr = llmemory.cast_int_to_adr(frame_addr + self.force_index_ofs) @@ -499,8 +504,7 @@ # # Restore 'prev' and loop back to the start. iself.frame_addr = 0 - next = iself.saved_next - next += llmemory.sizeof(llmemory.Address) + prev = iself.saved_prev # --------------- # diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py --- a/pypy/jit/backend/x86/assembler.py +++ b/pypy/jit/backend/x86/assembler.py @@ -774,7 +774,7 @@ self.mc.RET() def _call_header_shadowstack(self, gcrootmap): - # we need to put two words into the shadowstack: the MARKER + # we need to put two words into the shadowstack: the MARKER_FRAME # and the address of the frame (ebp, actually) rst = gcrootmap.get_root_stack_top_addr() if rx86.fits_in_32bits(rst): @@ -783,9 +783,10 @@ self.mc.MOV_ri(r13.value, rst) # MOV r13, rootstacktop self.mc.MOV_rm(eax.value, (r13.value, 0)) # MOV eax, [r13] # - self.mc.LEA_rm(ebx.value, (eax.value, 2*WORD)) # LEA ebx, [eax+2*WORD] - self.mc.MOV_mi((eax.value, 0), gcrootmap.MARKER) # MOV [eax], MARKER - self.mc.MOV_mr((eax.value, WORD), ebp.value) # MOV [eax+WORD], ebp + MARKER = gcrootmap.MARKER_FRAME + self.mc.LEA_rm(ebx.value, (eax.value, 2*WORD)) # LEA ebx, [eax+2*WORD] + self.mc.MOV_mi((eax.value, WORD), MARKER) # MOV [eax+WORD], MARKER + self.mc.MOV_mr((eax.value, 0), ebp.value) # MOV [eax], ebp # if rx86.fits_in_32bits(rst): self.mc.MOV_jr(rst, ebx.value) # MOV [rootstacktop], ebx diff --git a/pypy/jit/codewriter/assembler.py b/pypy/jit/codewriter/assembler.py --- a/pypy/jit/codewriter/assembler.py +++ b/pypy/jit/codewriter/assembler.py @@ -81,10 +81,15 @@ if not isinstance(value, (llmemory.AddressAsInt, ComputedIntSymbolic)): value = lltype.cast_primitive(lltype.Signed, value) - if allow_short and -128 <= value <= 127: - # emit the constant as a small integer - self.code.append(chr(value & 0xFF)) - return True + if allow_short: + try: + short_num = -128 <= value <= 127 + except TypeError: # "Symbolics cannot be compared!" + short_num = False + if short_num: + # emit the constant as a small integer + self.code.append(chr(value & 0xFF)) + return True constants = self.constants_i elif kind == 'ref': value = lltype.cast_opaque_ptr(llmemory.GCREF, value) @@ -126,13 +131,14 @@ self.code.append("temporary placeholder") # argcodes = [] + allow_short = (insn[0] in USE_C_FORM) for x in insn[1:]: if isinstance(x, Register): self.emit_reg(x) argcodes.append(x.kind[0]) elif isinstance(x, Constant): kind = getkind(x.concretetype) - is_short = self.emit_const(x, kind, allow_short=True) + is_short = self.emit_const(x, kind, allow_short=allow_short) if is_short: argcodes.append('c') else: @@ -252,3 +258,43 @@ for func in callinfocollection.all_function_addresses_as_int(): func = heaptracker.int2adr(func) self.see_raw_object(func.ptr) + + +# A set of instructions that use the 'c' encoding for small constants. +# Allowing it anywhere causes the number of instruction variants to +# expode, growing past 256. So we list here only the most common +# instructions where the 'c' variant might be useful. +USE_C_FORM = set([ + 'copystrcontent', + 'getarrayitem_gc_pure_i', + 'getarrayitem_gc_pure_r', + 'getarrayitem_gc_i', + 'getarrayitem_gc_r', + 'goto_if_not_int_eq', + 'goto_if_not_int_ge', + 'goto_if_not_int_gt', + 'goto_if_not_int_le', + 'goto_if_not_int_lt', + 'goto_if_not_int_ne', + 'int_add', + 'int_and', + 'int_copy', + 'int_eq', + 'int_ge', + 'int_gt', + 'int_le', + 'int_lt', + 'int_ne', + 'int_return', + 'int_sub', + 'jit_merge_point', + 'new_array', + 'newstr', + 'setarrayitem_gc_i', + 'setarrayitem_gc_r', + 'setfield_gc_i', + 'strgetitem', + 'strsetitem', + + 'foobar', 'baz', # for tests +]) diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py --- a/pypy/jit/codewriter/jtransform.py +++ b/pypy/jit/codewriter/jtransform.py @@ -7,6 +7,7 @@ from pypy.jit.metainterp import quasiimmut from pypy.jit.metainterp.history import getkind from pypy.jit.metainterp.typesystem import deref, arrayItem +from pypy.jit.metainterp.blackhole import BlackholeInterpreter from pypy.objspace.flow.model import SpaceOperation, Variable, Constant, c_last_exception from pypy.rlib import objectmodel from pypy.rlib.jit import _we_are_jitted @@ -199,9 +200,9 @@ try: rewrite = _rewrite_ops[op.opname] except KeyError: - return op # default: keep the operation unchanged - else: - return rewrite(self, op) + raise Exception("the JIT doesn't support the operation %r" + " in %r" % (op, getattr(self, 'graph', '?'))) + return rewrite(self, op) def rewrite_op_same_as(self, op): if op.args[0] in self.vable_array_vars: @@ -1720,4 +1721,18 @@ result[name[len(prefix):]] = getattr(Transformer, name) return result +def keep_operation_unchanged(jtransform, op): + return op + +def _add_default_ops(rewrite_ops): + # All operations present in the BlackholeInterpreter as bhimpl_xxx + # but not explicitly listed in this file are supposed to be just + # passed in unmodified. All other operations are forbidden. + for key, value in BlackholeInterpreter.__dict__.items(): + if key.startswith('bhimpl_'): + opname = key[len('bhimpl_'):] + rewrite_ops.setdefault(opname, keep_operation_unchanged) + rewrite_ops.setdefault('-live-', keep_operation_unchanged) + _rewrite_ops = _with_prefix('rewrite_op_') +_add_default_ops(_rewrite_ops) diff --git a/pypy/jit/codewriter/test/test_assembler.py b/pypy/jit/codewriter/test/test_assembler.py --- a/pypy/jit/codewriter/test/test_assembler.py +++ b/pypy/jit/codewriter/test/test_assembler.py @@ -157,15 +157,18 @@ ('foobar', ListOfKind('int', [Constant(42, lltype.Signed)])), ('foobar', ListOfKind('int', [Constant(42, lltype.Signed)])), ('baz', Constant(42, lltype.Signed)), + ('bok', Constant(41, lltype.Signed)), ] assembler = Assembler() jitcode = assembler.assemble(ssarepr) assert jitcode.code == ("\x00\x01\xFF" "\x00\x01\xFF" - "\x01\x2A") + "\x01\x2A" + "\x02\xFE") assert assembler.insns == {'foobar/I': 0, - 'baz/c': 1} - assert jitcode.constants_i == [42] + 'baz/c': 1, # in USE_C_FORM + 'bok/i': 2} # not in USE_C_FORM + assert jitcode.constants_i == [42, 41] def test_assemble_descr(): class FooDescr(AbstractDescr): diff --git a/pypy/jit/codewriter/test/test_jtransform.py b/pypy/jit/codewriter/test/test_jtransform.py --- a/pypy/jit/codewriter/test/test_jtransform.py +++ b/pypy/jit/codewriter/test/test_jtransform.py @@ -1210,3 +1210,11 @@ assert op1.args == [v1] assert op1.result is None assert op2 is None + +def test_unknown_operation(): + op = SpaceOperation('foobar', [], varoftype(lltype.Void)) + tr = Transformer() + try: + tr.rewrite_operation(op) + except Exception, e: + assert 'foobar' in str(e) diff --git a/pypy/module/_hashlib/interp_hashlib.py b/pypy/module/_hashlib/interp_hashlib.py --- a/pypy/module/_hashlib/interp_hashlib.py +++ b/pypy/module/_hashlib/interp_hashlib.py @@ -75,8 +75,12 @@ ctx = lltype.malloc(ropenssl.EVP_MD_CTX.TO, flavor='raw') rgc.add_memory_pressure(HASH_MALLOC_SIZE + self.digest_size) - ropenssl.EVP_DigestInit(ctx, digest_type) - self.ctx = ctx + try: + ropenssl.EVP_DigestInit(ctx, digest_type) + self.ctx = ctx + except: + lltype.free(ctx, flavor='raw') + raise def __del__(self): # self.lock.free() diff --git a/pypy/module/_io/interp_fileio.py b/pypy/module/_io/interp_fileio.py --- a/pypy/module/_io/interp_fileio.py +++ b/pypy/module/_io/interp_fileio.py @@ -349,6 +349,8 @@ try: s = os.read(self.fd, size) except OSError, e: + if e.errno == errno.EAGAIN: + return space.w_None raise wrap_oserror(space, e, exception_name='w_IOError') @@ -362,6 +364,8 @@ try: buf = os.read(self.fd, length) except OSError, e: + if e.errno == errno.EAGAIN: + return space.w_None raise wrap_oserror(space, e, exception_name='w_IOError') rwbuffer.setslice(0, buf) diff --git a/pypy/module/_io/test/test_fileio.py b/pypy/module/_io/test/test_fileio.py --- a/pypy/module/_io/test/test_fileio.py +++ b/pypy/module/_io/test/test_fileio.py @@ -133,6 +133,19 @@ f.close() assert a == b'a\nbxxxxxxx' + def test_nonblocking_read(self): + import os, fcntl + r_fd, w_fd = os.pipe() + # set nonblocking + fcntl.fcntl(r_fd, fcntl.F_SETFL, os.O_NONBLOCK) + import _io + f = _io.FileIO(r_fd, 'r') + # Read from stream sould return None + assert f.read() is None + assert f.read(10) is None + a = bytearray('x' * 10) + assert f.readinto(a) is None + def test_repr(self): import _io f = _io.FileIO(self.tmpfile, 'r') diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py --- a/pypy/module/_socket/interp_socket.py +++ b/pypy/module/_socket/interp_socket.py @@ -78,9 +78,6 @@ self.connect(self.addr_from_object(space, w_addr)) except SocketError, e: raise converted_error(space, e) - except TypeError, e: - raise OperationError(space.w_TypeError, - space.wrap(str(e))) def connect_ex_w(self, space, w_addr): """connect_ex(address) -> errno diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py --- a/pypy/module/_socket/test/test_sock_app.py +++ b/pypy/module/_socket/test/test_sock_app.py @@ -530,27 +530,32 @@ import _socket, os if not hasattr(_socket, 'AF_UNIX'): skip('AF_UNIX not supported.') - sockpath = os.path.join(self.udir, 'app_test_unix_socket_connect') + oldcwd = os.getcwd() + os.chdir(self.udir) + try: + sockpath = 'app_test_unix_socket_connect' - serversock = _socket.socket(_socket.AF_UNIX) - serversock.bind(sockpath) - serversock.listen(1) + serversock = _socket.socket(_socket.AF_UNIX) + serversock.bind(sockpath) + serversock.listen(1) - clientsock = _socket.socket(_socket.AF_UNIX) - clientsock.connect(sockpath) - fileno, addr = serversock._accept() - s = _socket.socket(fileno=fileno) - assert not addr + clientsock = _socket.socket(_socket.AF_UNIX) + clientsock.connect(sockpath) + fileno, addr = serversock._accept() + s = _socket.socket(fileno=fileno) + assert not addr - s.send(b'X') - data = clientsock.recv(100) - assert data == b'X' - clientsock.send(b'Y') - data = s.recv(100) - assert data == b'Y' + s.send(b'X') + data = clientsock.recv(100) + assert data == b'X' + clientsock.send(b'Y') + data = s.recv(100) + assert data == b'Y' - clientsock.close() - s.close() + clientsock.close() + s.close() + finally: + os.chdir(oldcwd) def test_subclass(self): # Socket is not created in __new__, but in __init__. diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py --- a/pypy/module/_ssl/interp_ssl.py +++ b/pypy/module/_ssl/interp_ssl.py @@ -96,7 +96,8 @@ # Defaults libssl_SSL_CTX_set_verify(self.ctx, SSL_VERIFY_NONE, None) - libssl_SSL_CTX_set_options(self.ctx, SSL_OP_ALL) + libssl_SSL_CTX_set_options( + self.ctx, SSL_OP_ALL & ~SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS) libssl_SSL_CTX_set_session_id_context(self.ctx, "Python", len("Python")) def __del__(self): diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py --- a/pypy/module/_ssl/test/test_ssl.py +++ b/pypy/module/_ssl/test/test_ssl.py @@ -64,8 +64,8 @@ def test_sslwrap(self): import ssl, _socket, sys, gc - if sys.platform == 'darwin': - skip("hangs indefinitely on OSX (also on CPython)") + if sys.platform == 'darwin' or 'freebsd' in sys.platform: + skip("hangs indefinitely on OSX & FreeBSD (also on CPython)") s = _socket.socket() ss = ssl.wrap_socket(s) diff --git a/pypy/module/cpyext/listobject.py b/pypy/module/cpyext/listobject.py --- a/pypy/module/cpyext/listobject.py +++ b/pypy/module/cpyext/listobject.py @@ -32,11 +32,10 @@ Py_DecRef(space, w_item) if not isinstance(w_list, W_ListObject): PyErr_BadInternalCall(space) - wrappeditems = w_list.getitems() - if index < 0 or index >= len(wrappeditems): + if index < 0 or index >= w_list.length(): raise OperationError(space.w_IndexError, space.wrap( "list assignment index out of range")) - wrappeditems[index] = w_item + w_list.setitem(index, w_item) return 0 @cpython_api([PyObject, Py_ssize_t], PyObject) @@ -74,7 +73,7 @@ """Macro form of PyList_Size() without error checking. """ assert isinstance(w_list, W_ListObject) - return len(w_list.getitems()) + return w_list.length() @cpython_api([PyObject], Py_ssize_t, error=-1) diff --git a/pypy/module/cpyext/test/test_listobject.py b/pypy/module/cpyext/test/test_listobject.py --- a/pypy/module/cpyext/test/test_listobject.py +++ b/pypy/module/cpyext/test/test_listobject.py @@ -128,3 +128,6 @@ module.setslice(l, None) assert l == [0, 4, 5] + l = [1, 2, 3] + module.setlistitem(l,0) + assert l == [None, 2, 3] diff --git a/pypy/module/fcntl/test/test_fcntl.py b/pypy/module/fcntl/test/test_fcntl.py --- a/pypy/module/fcntl/test/test_fcntl.py +++ b/pypy/module/fcntl/test/test_fcntl.py @@ -42,13 +42,9 @@ else: start_len = "qq" - if sys.platform in ('netbsd1', 'netbsd2', 'netbsd3', - 'Darwin1.2', 'darwin', - 'freebsd2', 'freebsd3', 'freebsd4', 'freebsd5', - 'freebsd6', 'freebsd7', 'freebsd8', 'freebsd9', - 'bsdos2', 'bsdos3', 'bsdos4', - 'openbsd', 'openbsd2', 'openbsd3', 'openbsd4', - 'openbsd5'): + if any(substring in sys.platform.lower() + for substring in ('netbsd', 'darwin', 'freebsd', 'bsdos', + 'openbsd')): if struct.calcsize('l') == 8: off_t = 'l' pid_t = 'i' @@ -182,7 +178,8 @@ def test_large_flag(self): import sys - if sys.platform == "darwin" or sys.platform.startswith("openbsd"): + if any(plat in sys.platform + for plat in ('darwin', 'openbsd', 'freebsd')): skip("Mac OS doesn't have any large flag in fcntl.h") import fcntl, sys if sys.maxint == 2147483647: diff --git a/pypy/module/gc/__init__.py b/pypy/module/gc/__init__.py --- a/pypy/module/gc/__init__.py +++ b/pypy/module/gc/__init__.py @@ -1,18 +1,18 @@ from pypy.interpreter.mixedmodule import MixedModule class Module(MixedModule): - appleveldefs = { - 'enable': 'app_gc.enable', - 'disable': 'app_gc.disable', - 'isenabled': 'app_gc.isenabled', - } interpleveldefs = { 'collect': 'interp_gc.collect', + 'enable': 'interp_gc.enable', + 'disable': 'interp_gc.disable', + 'isenabled': 'interp_gc.isenabled', 'enable_finalizers': 'interp_gc.enable_finalizers', 'disable_finalizers': 'interp_gc.disable_finalizers', 'garbage' : 'space.newlist([])', #'dump_heap_stats': 'interp_gc.dump_heap_stats', } + appleveldefs = { + } def __init__(self, space, w_name): if (not space.config.translating or diff --git a/pypy/module/gc/app_gc.py b/pypy/module/gc/app_gc.py deleted file mode 100644 --- a/pypy/module/gc/app_gc.py +++ /dev/null @@ -1,21 +0,0 @@ -# NOT_RPYTHON - -enabled = True - -def isenabled(): - global enabled - return enabled - -def enable(): - global enabled - import gc - if not enabled: - gc.enable_finalizers() - enabled = True - -def disable(): - global enabled - import gc - if enabled: - gc.disable_finalizers() - enabled = False diff --git a/pypy/module/gc/interp_gc.py b/pypy/module/gc/interp_gc.py --- a/pypy/module/gc/interp_gc.py +++ b/pypy/module/gc/interp_gc.py @@ -17,6 +17,26 @@ rgc.collect() return space.wrap(0) +def enable(space): + """Non-recursive version. Enable finalizers now. + If they were already enabled, no-op. + If they were disabled even several times, enable them anyway. + """ + if not space.user_del_action.enabled_at_app_level: + space.user_del_action.enabled_at_app_level = True + enable_finalizers(space) + +def disable(space): + """Non-recursive version. Disable finalizers now. Several calls + to this function are ignored. + """ + if space.user_del_action.enabled_at_app_level: + space.user_del_action.enabled_at_app_level = False + disable_finalizers(space) + +def isenabled(space): + return space.newbool(space.user_del_action.enabled_at_app_level) + def enable_finalizers(space): if space.user_del_action.finalizers_lock_count == 0: raise OperationError(space.w_ValueError, diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py --- a/pypy/module/micronumpy/__init__.py +++ b/pypy/module/micronumpy/__init__.py @@ -27,6 +27,12 @@ 'dot': 'interp_numarray.dot', 'fromstring': 'interp_support.fromstring', 'flatiter': 'interp_numarray.W_FlatIterator', + 'isna': 'interp_numarray.isna', + 'concatenate': 'interp_numarray.concatenate', + + 'set_string_function': 'appbridge.set_string_function', + + 'count_reduce_items': 'interp_numarray.count_reduce_items', 'True_': 'types.Bool.True', 'False_': 'types.Bool.False', @@ -66,6 +72,7 @@ ("copysign", "copysign"), ("cos", "cos"), ("divide", "divide"), + ("true_divide", "true_divide"), ("equal", "equal"), ("exp", "exp"), ("fabs", "fabs"), @@ -88,6 +95,9 @@ ("tan", "tan"), ('bitwise_and', 'bitwise_and'), ('bitwise_or', 'bitwise_or'), + ('bitwise_not', 'invert'), + ('isnan', 'isnan'), + ('isinf', 'isinf'), ]: interpleveldefs[exposed] = "interp_ufuncs.get(space).%s" % impl diff --git a/pypy/module/micronumpy/app_numpy.py b/pypy/module/micronumpy/app_numpy.py --- a/pypy/module/micronumpy/app_numpy.py +++ b/pypy/module/micronumpy/app_numpy.py @@ -59,7 +59,7 @@ if not hasattr(a, "max"): a = _numpypy.array(a) return a.max(axis) - + def arange(start, stop=None, step=1, dtype=None): '''arange([start], stop[, step], dtype=None) Generate values in the half-interval [start, stop). diff --git a/pypy/module/micronumpy/appbridge.py b/pypy/module/micronumpy/appbridge.py new file mode 100644 --- /dev/null +++ b/pypy/module/micronumpy/appbridge.py @@ -0,0 +1,38 @@ + +from pypy.rlib.objectmodel import specialize + +class AppBridgeCache(object): + w__var = None + w__std = None + w_module = None + w_array_repr = None + w_array_str = None + + def __init__(self, space): + self.w_import = space.appexec([], """(): + def f(): + import sys + __import__('numpypy.core._methods') + return sys.modules['numpypy.core._methods'] + return f + """) + + @specialize.arg(2) + def call_method(self, space, name, *args): + w_meth = getattr(self, 'w_' + name) + if w_meth is None: + if self.w_module is None: + self.w_module = space.call_function(self.w_import) + w_meth = space.getattr(self.w_module, space.wrap(name)) + setattr(self, 'w_' + name, w_meth) + return space.call_function(w_meth, *args) + +def set_string_function(space, w_f, w_repr): + cache = get_appbridge_cache(space) + if space.is_true(w_repr): + cache.w_array_repr = w_f + else: + cache.w_array_str = w_f + +def get_appbridge_cache(space): + return space.fromcache(AppBridgeCache) diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py --- a/pypy/module/micronumpy/compile.py +++ b/pypy/module/micronumpy/compile.py @@ -32,13 +32,16 @@ class BadToken(Exception): pass -SINGLE_ARG_FUNCTIONS = ["sum", "prod", "max", "min", "all", "any", "unegative"] +SINGLE_ARG_FUNCTIONS = ["sum", "prod", "max", "min", "all", "any", + "unegative", "flat"] +TWO_ARG_FUNCTIONS = ['take'] class FakeSpace(object): w_ValueError = None w_TypeError = None w_IndexError = None w_OverflowError = None + w_NotImplementedError = None w_None = None w_bool = "bool" @@ -53,6 +56,9 @@ """NOT_RPYTHON""" self.fromcache = InternalSpaceCache(self).getorbuild + def _freeze_(self): + return True + def issequence_w(self, w_obj): return isinstance(w_obj, ListObject) or isinstance(w_obj, W_NDimArray) @@ -144,6 +150,9 @@ def allocate_instance(self, klass, w_subtype): return instantiate(klass) + def newtuple(self, list_w): + raise ValueError + def len_w(self, w_obj): if isinstance(w_obj, ListObject): return len(w_obj.items) @@ -371,12 +380,12 @@ for arg in self.args])) def execute(self, interp): + arr = self.args[0].execute(interp) + if not isinstance(arr, BaseArray): + raise ArgumentNotAnArray if self.name in SINGLE_ARG_FUNCTIONS: if len(self.args) != 1 and self.name != 'sum': raise ArgumentMismatch - arr = self.args[0].execute(interp) - if not isinstance(arr, BaseArray): - raise ArgumentNotAnArray if self.name == "sum": if len(self.args)>1: w_res = arr.descr_sum(interp.space, @@ -396,21 +405,31 @@ elif self.name == "unegative": neg = interp_ufuncs.get(interp.space).negative w_res = neg.call(interp.space, [arr]) + elif self.name == "flat": + w_res = arr.descr_get_flatiter(interp.space) else: assert False # unreachable code - if isinstance(w_res, BaseArray): - return w_res - if isinstance(w_res, FloatObject): - dtype = get_dtype_cache(interp.space).w_float64dtype - elif isinstance(w_res, BoolObject): - dtype = get_dtype_cache(interp.space).w_booldtype - elif isinstance(w_res, interp_boxes.W_GenericBox): - dtype = w_res.get_dtype(interp.space) + elif self.name in TWO_ARG_FUNCTIONS: + arg = self.args[1].execute(interp) + if not isinstance(arg, BaseArray): + raise ArgumentNotAnArray + if self.name == 'take': + w_res = arr.descr_take(interp.space, arg) else: - dtype = None - return scalar_w(interp.space, dtype, w_res) + assert False # unreachable else: raise WrongFunctionName + if isinstance(w_res, BaseArray): + return w_res + if isinstance(w_res, FloatObject): + dtype = get_dtype_cache(interp.space).w_float64dtype + elif isinstance(w_res, BoolObject): + dtype = get_dtype_cache(interp.space).w_booldtype + elif isinstance(w_res, interp_boxes.W_GenericBox): + dtype = w_res.get_dtype(interp.space) + else: + dtype = None + return scalar_w(interp.space, dtype, w_res) _REGEXES = [ ('-?[\d\.]+', 'number'), diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py --- a/pypy/module/micronumpy/interp_boxes.py +++ b/pypy/module/micronumpy/interp_boxes.py @@ -8,6 +8,7 @@ from pypy.tool.sourcetools import func_with_new_name +MIXIN_32 = (int_typedef,) if LONG_BIT == 32 else () MIXIN_64 = (int_typedef,) if LONG_BIT == 64 else () def new_dtype_getter(name): @@ -28,6 +29,7 @@ def convert_to(self, dtype): return dtype.box(self.value) + class W_GenericBox(Wrappable): _attrs_ = () @@ -93,7 +95,7 @@ descr_neg = _unaryop_impl("negative") descr_abs = _unaryop_impl("absolute") - def descr_tolist(self, space): + def item(self, space): return self.get_dtype(space).itemtype.to_builtin_type(space, self) @@ -104,7 +106,8 @@ _attrs_ = () class W_IntegerBox(W_NumberBox): - pass + def int_w(self, space): + return space.int_w(self.descr_int(space)) class W_SignedIntegerBox(W_IntegerBox): pass @@ -187,7 +190,7 @@ __neg__ = interp2app(W_GenericBox.descr_neg), __abs__ = interp2app(W_GenericBox.descr_abs), - tolist = interp2app(W_GenericBox.descr_tolist), + tolist = interp2app(W_GenericBox.item), ) W_BoolBox.typedef = TypeDef("bool_", W_GenericBox.typedef, @@ -231,7 +234,7 @@ __new__ = interp2app(W_UInt16Box.descr__new__.im_func), ) -W_Int32Box.typedef = TypeDef("int32", W_SignedIntegerBox.typedef, +W_Int32Box.typedef = TypeDef("int32", (W_SignedIntegerBox.typedef,) + MIXIN_32, __module__ = "numpypy", __new__ = interp2app(W_Int32Box.descr__new__.im_func), ) @@ -241,24 +244,18 @@ __new__ = interp2app(W_UInt32Box.descr__new__.im_func), ) -if LONG_BIT == 32: - long_name = "int32" -elif LONG_BIT == 64: - long_name = "int64" -W_LongBox.typedef = TypeDef(long_name, (W_SignedIntegerBox.typedef, int_typedef,), - __module__ = "numpypy", - __new__ = interp2app(W_LongBox.descr__new__.im_func), -) - -W_ULongBox.typedef = TypeDef("u" + long_name, W_UnsignedIntegerBox.typedef, - __module__ = "numpypy", -) - W_Int64Box.typedef = TypeDef("int64", (W_SignedIntegerBox.typedef,) + MIXIN_64, __module__ = "numpypy", __new__ = interp2app(W_Int64Box.descr__new__.im_func), ) +if LONG_BIT == 32: + W_LongBox = W_Int32Box + W_ULongBox = W_UInt32Box +elif LONG_BIT == 64: + W_LongBox = W_Int64Box + W_ULongBox = W_UInt64Box + W_UInt64Box.typedef = TypeDef("uint64", W_UnsignedIntegerBox.typedef, __module__ = "numpypy", __new__ = interp2app(W_UInt64Box.descr__new__.im_func), @@ -283,3 +280,4 @@ __new__ = interp2app(W_Float64Box.descr__new__.im_func), ) + diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py --- a/pypy/module/micronumpy/interp_dtype.py +++ b/pypy/module/micronumpy/interp_dtype.py @@ -89,8 +89,19 @@ def descr_get_shape(self, space): return space.newtuple([]) + def eq(self, space, w_other): + w_other = space.call_function(space.gettypefor(W_Dtype), w_other) + return space.is_w(self, w_other) + + def descr_eq(self, space, w_other): + return space.wrap(self.eq(space, w_other)) + + def descr_ne(self, space, w_other): + return space.wrap(not self.eq(space, w_other)) + def is_int_type(self): - return self.kind == SIGNEDLTR or self.kind == UNSIGNEDLTR + return (self.kind == SIGNEDLTR or self.kind == UNSIGNEDLTR or + self.kind == BOOLLTR) def is_bool_type(self): return self.kind == BOOLLTR @@ -101,12 +112,15 @@ __str__= interp2app(W_Dtype.descr_str), __repr__ = interp2app(W_Dtype.descr_repr), + __eq__ = interp2app(W_Dtype.descr_eq), + __ne__ = interp2app(W_Dtype.descr_ne), num = interp_attrproperty("num", cls=W_Dtype), kind = interp_attrproperty("kind", cls=W_Dtype), type = interp_attrproperty_w("w_box_type", cls=W_Dtype), itemsize = GetSetProperty(W_Dtype.descr_get_itemsize), shape = GetSetProperty(W_Dtype.descr_get_shape), + name = interp_attrproperty('name', cls=W_Dtype), ) W_Dtype.typedef.acceptable_as_base_class = False diff --git a/pypy/module/micronumpy/interp_iter.py b/pypy/module/micronumpy/interp_iter.py --- a/pypy/module/micronumpy/interp_iter.py +++ b/pypy/module/micronumpy/interp_iter.py @@ -4,6 +4,49 @@ from pypy.module.micronumpy.strides import calculate_broadcast_strides,\ calculate_slice_strides +""" This is a mini-tutorial on iterators, strides, and +memory layout. It assumes you are familiar with the terms, see +http://docs.scipy.org/doc/numpy/reference/arrays.ndarray.html +for a more gentle introduction. + +Given an array x: x.shape == [5,6], + +At which byte in x.data does the item x[3,4] begin? +if x.strides==[1,5]: + pData = x.pData + (x.start + 3*1 + 4*5)*sizeof(x.pData[0]) + pData = x.pData + (x.start + 24) * sizeof(x.pData[0]) +so the offset of the element is 24 elements after the first + +What is the next element in x after coordinates [3,4]? +if x.order =='C': + next == [3,5] => offset is 28 +if x.order =='F': + next == [4,4] => offset is 24 +so for the strides [1,5] x is 'F' contiguous +likewise, for the strides [6,1] x would be 'C' contiguous. + +Iterators have an internal representation of the current coordinates +(indices), the array, strides, and backstrides. A short digression to +explain backstrides: what is the coordinate and offset after [3,5] in +the example above? +if x.order == 'C': + next == [4,0] => offset is 4 +if x.order == 'F': + next == [4,5] => offset is 25 +Note that in 'C' order we stepped BACKWARDS 24 while 'overflowing' a +shape dimension + which is back 25 and forward 1, + which is x.strides[1] * (x.shape[1] - 1) + x.strides[0] +so if we precalculate the overflow backstride as +[x.strides[i] * (x.shape[i] - 1) for i in range(len(x.shape))] +we can go faster. +All the calculations happen in next() + +next_step_x() tries to do the iteration for a number of steps at once, +but then we cannot gaurentee that we only overflow one single shape +dimension, perhaps we could overflow times in one big step. +""" + # structures to describe slicing class Chunk(object): @@ -17,6 +60,10 @@ if self.step != 0: shape.append(self.lgt) + def __repr__(self): + return 'Chunk(%d, %d, %d, %d)' % (self.start, self.stop, self.step, + self.lgt) + class BaseTransform(object): pass @@ -51,9 +98,9 @@ self.size = size def next(self, shapelen): - return self._next(1) + return self.next_skip_x(1) - def _next(self, ofs): + def next_skip_x(self, ofs): arr = instantiate(ArrayIterator) arr.size = self.size arr.offset = self.offset + ofs @@ -61,7 +108,7 @@ def next_no_increase(self, shapelen): # a hack to make JIT believe this is always virtual - return self._next(0) + return self.next_skip_x(0) def done(self): return self.offset >= self.size @@ -133,6 +180,36 @@ res._done = done return res + @jit.unroll_safe + def next_skip_x(self, shapelen, step): + shapelen = jit.promote(len(self.res_shape)) + offset = self.offset + indices = [0] * shapelen + for i in range(shapelen): + indices[i] = self.indices[i] + done = False + for i in range(shapelen - 1, -1, -1): + if indices[i] < self.res_shape[i] - step: + indices[i] += step + offset += self.strides[i] * step + break + else: + remaining_step = (indices[i] + step) // self.res_shape[i] + this_i_step = step - remaining_step * self.res_shape[i] + offset += self.strides[i] * this_i_step + indices[i] = indices[i] + this_i_step + step = remaining_step + else: + done = True + res = instantiate(ViewIterator) + res.offset = offset + res.indices = indices + res.strides = self.strides + res.backstrides = self.backstrides + res.res_shape = self.res_shape + res._done = done + return res + def apply_transformations(self, arr, transformations): v = BaseIterator.apply_transformations(self, arr, transformations) if len(arr.shape) == 1: @@ -153,8 +230,13 @@ class AxisIterator(BaseIterator): def __init__(self, start, dim, shape, strides, backstrides): self.res_shape = shape[:] - self.strides = strides[:dim] + [0] + strides[dim:] - self.backstrides = backstrides[:dim] + [0] + backstrides[dim:] + if len(shape) == len(strides): + # keepdims = True + self.strides = strides[:dim] + [0] + strides[dim + 1:] + self.backstrides = backstrides[:dim] + [0] + backstrides[dim + 1:] + else: + self.strides = strides[:dim] + [0] + strides[dim:] + self.backstrides = backstrides[:dim] + [0] + backstrides[dim:] self.first_line = True self.indices = [0] * len(shape) self._done = False diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -1,16 +1,20 @@ from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.error import OperationError, operationerrfmt -from pypy.interpreter.gateway import interp2app, NoneNotWrapped +from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.interpreter.typedef import TypeDef, GetSetProperty -from pypy.module.micronumpy import interp_ufuncs, interp_dtype, signature,\ - interp_boxes -from pypy.module.micronumpy.strides import calculate_slice_strides +from pypy.module.micronumpy import (interp_ufuncs, interp_dtype, interp_boxes, + signature, support) +from pypy.module.micronumpy.strides import (calculate_slice_strides, + shape_agreement, find_shape_and_elems, get_shape_from_iterable, + calc_new_strides, to_coords) from pypy.rlib import jit from pypy.rpython.lltypesystem import lltype, rffi from pypy.tool.sourcetools import func_with_new_name from pypy.rlib.rstring import StringBuilder -from pypy.module.micronumpy.interp_iter import ArrayIterator, OneDimIterator,\ - SkipLastAxisIterator, Chunk, ViewIterator +from pypy.module.micronumpy.interp_iter import (ArrayIterator, OneDimIterator, + SkipLastAxisIterator, Chunk, ViewIterator) +from pypy.module.micronumpy.appbridge import get_appbridge_cache + numpy_driver = jit.JitDriver( greens=['shapelen', 'sig'], @@ -58,176 +62,21 @@ reds=['idx', 'idxi', 'frame', 'arr'], name='numpy_filterset', ) - -def _find_shape_and_elems(space, w_iterable): - shape = [space.len_w(w_iterable)] - batch = space.listview(w_iterable) - while True: - new_batch = [] - if not batch: - return shape, [] - if not space.issequence_w(batch[0]): - for elem in batch: - if space.issequence_w(elem): - raise OperationError(space.w_ValueError, space.wrap( - "setting an array element with a sequence")) - return shape, batch - size = space.len_w(batch[0]) - for w_elem in batch: - if not space.issequence_w(w_elem) or space.len_w(w_elem) != size: - raise OperationError(space.w_ValueError, space.wrap( - "setting an array element with a sequence")) - new_batch += space.listview(w_elem) - shape.append(size) - batch = new_batch - -def shape_agreement(space, shape1, shape2): - ret = _shape_agreement(shape1, shape2) - if len(ret) < max(len(shape1), len(shape2)): - raise OperationError(space.w_ValueError, - space.wrap("operands could not be broadcast together with shapes (%s) (%s)" % ( - ",".join([str(x) for x in shape1]), - ",".join([str(x) for x in shape2]), - )) - ) - return ret - -def _shape_agreement(shape1, shape2): - """ Checks agreement about two shapes with respect to broadcasting. Returns - the resulting shape. - """ - lshift = 0 - rshift = 0 - if len(shape1) > len(shape2): - m = len(shape1) - n = len(shape2) - rshift = len(shape2) - len(shape1) - remainder = shape1 - else: - m = len(shape2) - n = len(shape1) - lshift = len(shape1) - len(shape2) - remainder = shape2 - endshape = [0] * m - indices1 = [True] * m - indices2 = [True] * m - for i in range(m - 1, m - n - 1, -1): - left = shape1[i + lshift] - right = shape2[i + rshift] - if left == right: - endshape[i] = left - elif left == 1: - endshape[i] = right - indices1[i + lshift] = False - elif right == 1: - endshape[i] = left - indices2[i + rshift] = False - else: - return [] - #raise OperationError(space.w_ValueError, space.wrap( - # "frames are not aligned")) - for i in range(m - n): - endshape[i] = remainder[i] - return endshape - -def get_shape_from_iterable(space, old_size, w_iterable): - new_size = 0 - new_shape = [] - if space.isinstance_w(w_iterable, space.w_int): - new_size = space.int_w(w_iterable) - if new_size < 0: - new_size = old_size - new_shape = [new_size] - else: - neg_dim = -1 - batch = space.listview(w_iterable) - new_size = 1 - if len(batch) < 1: - if old_size == 1: - # Scalars can have an empty size. - new_size = 1 - else: - new_size = 0 - new_shape = [] - i = 0 - for elem in batch: - s = space.int_w(elem) - if s < 0: - if neg_dim >= 0: - raise OperationError(space.w_ValueError, space.wrap( - "can only specify one unknown dimension")) - s = 1 - neg_dim = i - new_size *= s - new_shape.append(s) - i += 1 - if neg_dim >= 0: - new_shape[neg_dim] = old_size / new_size - new_size *= new_shape[neg_dim] - if new_size != old_size: - raise OperationError(space.w_ValueError, - space.wrap("total size of new array must be unchanged")) - return new_shape - -# Recalculating strides. Find the steps that the iteration does for each -# dimension, given the stride and shape. Then try to create a new stride that -# fits the new shape, using those steps. If there is a shape/step mismatch -# (meaning that the realignment of elements crosses from one step into another) -# return None so that the caller can raise an exception. -def calc_new_strides(new_shape, old_shape, old_strides): - # Assumes that prod(old_shape) == prod(new_shape), len(old_shape) > 1, and - # len(new_shape) > 0 - steps = [] - last_step = 1 - oldI = 0 - new_strides = [] - if old_strides[0] < old_strides[-1]: - #Start at old_shape[0], old_stides[0] - for i in range(len(old_shape)): - steps.append(old_strides[i] / last_step) - last_step *= old_shape[i] - cur_step = steps[0] - n_new_elems_used = 1 - n_old_elems_to_use = old_shape[0] - for s in new_shape: - new_strides.append(cur_step * n_new_elems_used) - n_new_elems_used *= s - while n_new_elems_used > n_old_elems_to_use: - oldI += 1 - if steps[oldI] != steps[oldI - 1]: - return None - n_old_elems_to_use *= old_shape[oldI] - if n_new_elems_used == n_old_elems_to_use: - oldI += 1 - if oldI >= len(old_shape): - continue - cur_step = steps[oldI] - n_old_elems_to_use *= old_shape[oldI] - else: - #Start at old_shape[-1], old_strides[-1] - for i in range(len(old_shape) - 1, -1, -1): - steps.insert(0, old_strides[i] / last_step) - last_step *= old_shape[i] - cur_step = steps[-1] - n_new_elems_used = 1 - oldI = -1 - n_old_elems_to_use = old_shape[-1] - for i in range(len(new_shape) - 1, -1, -1): - s = new_shape[i] - new_strides.insert(0, cur_step * n_new_elems_used) - n_new_elems_used *= s - while n_new_elems_used > n_old_elems_to_use: - oldI -= 1 - if steps[oldI] != steps[oldI + 1]: - return None - n_old_elems_to_use *= old_shape[oldI] - if n_new_elems_used == n_old_elems_to_use: - oldI -= 1 - if oldI < -len(old_shape): - continue - cur_step = steps[oldI] - n_old_elems_to_use *= old_shape[oldI] - return new_strides +take_driver = jit.JitDriver( + greens=['shapelen', 'sig'], + reds=['index_i', 'res_i', 'concr', 'index', 'res'], + name='numpy_take', +) +flat_get_driver = jit.JitDriver( + greens=['shapelen', 'base'], + reds=['step', 'ri', 'basei', 'res'], + name='numpy_flatget', +) +flat_set_driver = jit.JitDriver( + greens=['shapelen', 'base'], + reds=['step', 'ai', 'lngth', 'arr', 'basei'], + name='numpy_flatset', +) class BaseArray(Wrappable): _attrs_ = ["invalidates", "shape", 'size'] @@ -268,6 +117,7 @@ descr_pos = _unaryop_impl("positive") descr_neg = _unaryop_impl("negative") descr_abs = _unaryop_impl("absolute") + descr_invert = _unaryop_impl("invert") def _binop_impl(ufunc_name): def impl(self, space, w_other): @@ -310,9 +160,11 @@ def _reduce_ufunc_impl(ufunc_name, promote_to_largest=False): def impl(self, space, w_axis=None): if space.is_w(w_axis, space.w_None): - w_axis = space.wrap(-1) + axis = -1 + else: + axis = space.int_w(w_axis) return getattr(interp_ufuncs.get(space), ufunc_name).reduce(space, - self, True, promote_to_largest, w_axis) + self, True, promote_to_largest, axis) return func_with_new_name(impl, "reduce_%s_impl" % ufunc_name) descr_sum = _reduce_ufunc_impl("add") @@ -430,9 +282,23 @@ def descr_copy(self, space): return self.copy(space) + def descr_flatten(self, space, w_order=None): + if isinstance(self, Scalar): + # scalars have no storage + return self.descr_reshape(space, [space.wrap(1)]) + concr = self.get_concrete() + w_res = concr.descr_ravel(space, w_order) + if w_res.storage == concr.storage: + return w_res.copy(space) + return w_res + def copy(self, space): return self.get_concrete().copy(space) + def empty_copy(self, space, dtype): + shape = self.shape + return W_NDimArray(support.product(shape), shape[:], dtype, 'C') + def descr_len(self, space): if len(self.shape): return space.wrap(self.shape[0]) @@ -440,33 +306,32 @@ "len() of unsized object")) def descr_repr(self, space): - res = StringBuilder() - res.append("array(") - concrete = self.get_concrete_or_scalar() - dtype = concrete.find_dtype() - if not concrete.size: - res.append('[]') - if len(self.shape) > 1: - # An empty slice reports its shape - res.append(", shape=(") - self_shape = str(self.shape) - res.append_slice(str(self_shape), 1, len(self_shape) - 1) - res.append(')') - else: - concrete.to_str(space, 1, res, indent=' ') - if (dtype is not interp_dtype.get_dtype_cache(space).w_float64dtype and - not (dtype.kind == interp_dtype.SIGNEDLTR and - dtype.itemtype.get_element_size() == rffi.sizeof(lltype.Signed)) or - not self.size): - res.append(", dtype=" + dtype.name) - res.append(")") - return space.wrap(res.build()) + cache = get_appbridge_cache(space) + if cache.w_array_repr is None: + return space.wrap(self.dump_data()) + return space.call_function(cache.w_array_repr, self) + + def dump_data(self): + concr = self.get_concrete() + i = concr.create_iter() + first = True + s = StringBuilder() + s.append('array([') + while not i.done(): + if first: + first = False + else: + s.append(', ') + s.append(concr.dtype.itemtype.str_format(concr.getitem(i.offset))) + i = i.next(len(concr.shape)) + s.append('])') + return s.build() def descr_str(self, space): - ret = StringBuilder() - concrete = self.get_concrete_or_scalar() - concrete.to_str(space, 0, ret, ' ') - return space.wrap(ret.build()) + cache = get_appbridge_cache(space) + if cache.w_array_str is None: + return space.wrap(self.dump_data()) + return space.call_function(cache.w_array_str, self) @jit.unroll_safe def _single_item_result(self, space, w_idx): @@ -506,7 +371,7 @@ def count_all_true(self, arr): sig = arr.find_sig() - frame = sig.create_frame(self) + frame = sig.create_frame(arr) shapelen = len(arr.shape) s = 0 iter = None @@ -520,6 +385,9 @@ def getitem_filter(self, space, arr): concr = arr.get_concrete() + if concr.size > self.size: + raise OperationError(space.w_IndexError, + space.wrap("index out of range for array")) size = self.count_all_true(concr) res = W_NDimArray(size, [size], self.find_dtype()) ri = ArrayIterator(size) @@ -528,7 +396,7 @@ sig = self.find_sig() frame = sig.create_frame(self) v = None - while not frame.done(): + while not ri.done(): filter_driver.jit_merge_point(concr=concr, argi=argi, ri=ri, frame=frame, v=v, res=res, sig=sig, shapelen=shapelen, self=self) @@ -568,7 +436,7 @@ item = concrete._index_of_single_item(space, w_idx) return concrete.getitem(item) chunks = self._prepare_slice_args(space, w_idx) - return space.wrap(self.create_slice(chunks)) + return self.create_slice(chunks) def descr_setitem(self, space, w_idx, w_value): self.invalidated() @@ -622,11 +490,14 @@ w_shape = args_w[0] else: w_shape = space.newtuple(args_w) + new_shape = get_shape_from_iterable(space, self.size, w_shape) + return self.reshape(space, new_shape) + + def reshape(self, space, new_shape): concrete = self.get_concrete() - new_shape = get_shape_from_iterable(space, concrete.size, w_shape) # Since we got to here, prod(new_shape) == self.size - new_strides = calc_new_strides(new_shape, - concrete.shape, concrete.strides) + new_strides = calc_new_strides(new_shape, concrete.shape, + concrete.strides, concrete.order) if new_strides: # We can create a view, strides somehow match up. ndims = len(new_shape) @@ -634,7 +505,7 @@ for nd in range(ndims): new_backstrides[nd] = (new_shape[nd] - 1) * new_strides[nd] arr = W_NDimSlice(concrete.start, new_strides, new_backstrides, - new_shape, self) + new_shape, concrete) else: # Create copy with contiguous data arr = concrete.copy(space) @@ -644,7 +515,7 @@ def descr_tolist(self, space): if len(self.shape) == 0: assert isinstance(self, Scalar) - return self.value.descr_tolist(space) + return self.value.item(space) w_result = space.newlist([]) for i in range(self.shape[0]): space.call_method(w_result, "append", @@ -661,17 +532,13 @@ w_denom = space.wrap(self.shape[dim]) return space.div(self.descr_sum_promote(space, w_axis), w_denom) - def descr_var(self, space): - # var = mean((values - mean(values)) ** 2) - w_res = self.descr_sub(space, self.descr_mean(space, space.w_None)) - assert isinstance(w_res, BaseArray) - w_res = w_res.descr_pow(space, space.wrap(2)) - assert isinstance(w_res, BaseArray) - return w_res.descr_mean(space, space.w_None) + def descr_var(self, space, w_axis=None): + return get_appbridge_cache(space).call_method(space, '_var', self, + w_axis) - def descr_std(self, space): - # std(v) = sqrt(var(v)) - return interp_ufuncs.get(space).sqrt.call(space, [self.descr_var(space)]) + def descr_std(self, space, w_axis=None): + return get_appbridge_cache(space).call_method(space, '_std', self, + w_axis) def descr_fill(self, space, w_value): concr = self.get_concrete_or_scalar() @@ -704,6 +571,16 @@ return space.wrap(W_NDimSlice(concrete.start, strides, backstrides, shape, concrete)) + def descr_ravel(self, space, w_order=None): + if w_order is None or space.is_w(w_order, space.w_None): + order = 'C' + else: + order = space.str_w(w_order) + if order != 'C': + raise OperationError(space.w_NotImplementedError, space.wrap( + "order not implemented")) + return self.descr_reshape(space, [space.wrap(-1)]) + def descr_get_flatiter(self, space): return space.wrap(W_FlatIterator(self)) @@ -733,6 +610,60 @@ def supports_fast_slicing(self): return False + def descr_compress(self, space, w_obj, w_axis=None): + index = convert_to_array(space, w_obj) + return self.getitem_filter(space, index) + + def descr_take(self, space, w_obj, w_axis=None): + index = convert_to_array(space, w_obj).get_concrete() + concr = self.get_concrete() + if space.is_w(w_axis, space.w_None): + concr = concr.descr_ravel(space) + else: + raise OperationError(space.w_NotImplementedError, + space.wrap("axis unsupported for take")) + index_i = index.create_iter() + res_shape = index.shape + size = support.product(res_shape) + res = W_NDimArray(size, res_shape[:], concr.dtype, concr.order) + res_i = res.create_iter() + shapelen = len(index.shape) + sig = concr.find_sig() + while not index_i.done(): + take_driver.jit_merge_point(index_i=index_i, index=index, + res_i=res_i, concr=concr, + res=res, + shapelen=shapelen, sig=sig) + w_item = index._getitem_long(space, index_i.offset) + res.setitem(res_i.offset, concr.descr_getitem(space, w_item)) + index_i = index_i.next(shapelen) + res_i = res_i.next(shapelen) + return res + + def _getitem_long(self, space, offset): + # an obscure hack to not have longdtype inside a jitted loop + longdtype = interp_dtype.get_dtype_cache(space).w_longdtype + return self.getitem(offset).convert_to(longdtype).item( + space) + + def descr_item(self, space, w_arg=None): + if space.is_w(w_arg, space.w_None): + if not isinstance(self, Scalar): + raise OperationError(space.w_ValueError, space.wrap("index out of bounds")) + return self.value.item(space) + if space.isinstance_w(w_arg, space.w_int): + if isinstance(self, Scalar): + raise OperationError(space.w_ValueError, space.wrap("index out of bounds")) + concr = self.get_concrete() + i = to_coords(space, self.shape, concr.size, concr.order, w_arg)[0] + # XXX a bit around + item = self.descr_getitem(space, space.newtuple([space.wrap(x) + for x in i])) + assert isinstance(item, interp_boxes.W_GenericBox) + return item.item(space) + raise OperationError(space.w_NotImplementedError, space.wrap( + "non-int arg not supported")) + def convert_to_array(space, w_obj): if isinstance(w_obj, BaseArray): return w_obj @@ -758,14 +689,12 @@ self.shape = [] BaseArray.__init__(self, []) self.dtype = dtype + assert isinstance(value, interp_boxes.W_GenericBox) self.value = value def find_dtype(self): return self.dtype - def to_str(self, space, comma, builder, indent=' ', use_ellipsis=False): - builder.append(self.dtype.itemtype.str_format(self.value)) - def copy(self, space): return Scalar(self.dtype, self.value) @@ -778,6 +707,11 @@ def get_concrete_or_scalar(self): return self + def reshape(self, space, new_shape): + size = support.product(new_shape) + res = W_NDimArray(size, new_shape, self.dtype, 'C') + res.setitem(0, self.value) + return res class VirtualArray(BaseArray): """ @@ -834,12 +768,9 @@ class VirtualSlice(VirtualArray): def __init__(self, child, chunks, shape): - size = 1 - for sh in shape: - size *= sh self.child = child self.chunks = chunks - self.size = size + self.size = support.product(shape) VirtualArray.__init__(self, 'slice', shape, child.find_dtype()) def create_sig(self): @@ -858,11 +789,12 @@ class Call1(VirtualArray): - def __init__(self, ufunc, name, shape, res_dtype, values): + def __init__(self, ufunc, name, shape, calc_dtype, res_dtype, values): VirtualArray.__init__(self, name, shape, res_dtype) self.values = values self.size = values.size self.ufunc = ufunc + self.calc_dtype = calc_dtype def _del_sources(self): self.values = None @@ -884,9 +816,7 @@ self.left = left self.right = right self.calc_dtype = calc_dtype - self.size = 1 - for s in self.shape: - self.size *= s + self.size = support.product(self.shape) def _del_sources(self): self.left = None @@ -914,14 +844,15 @@ self.left.create_sig(), self.right.create_sig()) class SliceArray(Call2): - def __init__(self, shape, dtype, left, right): + def __init__(self, shape, dtype, left, right, no_broadcast=False): + self.no_broadcast = no_broadcast Call2.__init__(self, None, 'sliceloop', shape, dtype, dtype, left, right) def create_sig(self): lsig = self.left.create_sig() rsig = self.right.create_sig() - if self.shape != self.right.shape: + if not self.no_broadcast and self.shape != self.right.shape: return signature.SliceloopBroadcastSignature(self.ufunc, self.name, self.calc_dtype, @@ -995,89 +926,6 @@ self.strides = strides self.backstrides = backstrides - def to_str(self, space, comma, builder, indent=' ', use_ellipsis=False): - '''Modifies builder with a representation of the array/slice - The items will be seperated by a comma if comma is 1 - Multidimensional arrays/slices will span a number of lines, - each line will begin with indent. - ''' - size = self.size - ccomma = ',' * comma - ncomma = ',' * (1 - comma) - dtype = self.find_dtype() - if size < 1: - builder.append('[]') - return - if size > 1000: - # Once this goes True it does not go back to False for recursive - # calls - use_ellipsis = True - ndims = len(self.shape) - if ndims == 0: - builder.append(dtype.itemtype.str_format(self.getitem(0))) - return - i = 0 - builder.append('[') - if ndims > 1: - if use_ellipsis: - for i in range(min(3, self.shape[0])): - if i > 0: - builder.append(ccomma + '\n') - if ndims >= 3: - builder.append('\n' + indent) - else: - builder.append(indent) - view = self.create_slice([Chunk(i, 0, 0, 1)]).get_concrete() - view.to_str(space, comma, builder, indent=indent + ' ', - use_ellipsis=use_ellipsis) - if i < self.shape[0] - 1: - builder.append(ccomma + '\n' + indent + '...' + ncomma) - i = self.shape[0] - 3 - else: - i += 1 - while i < self.shape[0]: - if i > 0: - builder.append(ccomma + '\n') - if ndims >= 3: - builder.append('\n' + indent) - else: - builder.append(indent) - # create_slice requires len(chunks) > 1 in order to reduce - # shape - view = self.create_slice([Chunk(i, 0, 0, 1)]).get_concrete() - view.to_str(space, comma, builder, indent=indent + ' ', - use_ellipsis=use_ellipsis) - i += 1 - elif ndims == 1: - spacer = ccomma + ' ' - item = self.start - # An iterator would be a nicer way to walk along the 1d array, but - # how do I reset it if printing ellipsis? iterators have no - # "set_offset()" - i = 0 - if use_ellipsis: - for i in range(min(3, self.shape[0])): - if i > 0: - builder.append(spacer) - builder.append(dtype.itemtype.str_format(self.getitem(item))) - item += self.strides[0] - if i < self.shape[0] - 1: - # Add a comma only if comma is False - this prevents adding - # two commas - builder.append(spacer + '...' + ncomma) - # Ugly, but can this be done with an iterator? - item = self.start + self.backstrides[0] - 2 * self.strides[0] - i = self.shape[0] - 3 - else: - i += 1 - while i < self.shape[0]: - if i > 0: - builder.append(spacer) - builder.append(dtype.itemtype.str_format(self.getitem(item))) - item += self.strides[0] - i += 1 - builder.append(']') - @jit.unroll_safe def _index_of_single_item(self, space, w_idx): if space.isinstance_w(w_idx, space.w_int): @@ -1164,13 +1012,10 @@ assert isinstance(parent, ConcreteArray) if isinstance(parent, W_NDimSlice): parent = parent.parent - size = 1 - for sh in shape: - size *= sh self.strides = strides self.backstrides = backstrides - ViewArray.__init__(self, size, shape, parent.dtype, parent.order, - parent) + ViewArray.__init__(self, support.product(shape), shape, parent.dtype, + parent.order, parent) self.start = start def create_iter(self): @@ -1200,7 +1045,8 @@ self.backstrides = backstrides self.shape = new_shape return - new_strides = calc_new_strides(new_shape, self.shape, self.strides) + new_strides = calc_new_strides(new_shape, self.shape, self.strides, + self.order) if new_strides is None: raise OperationError(space.w_AttributeError, space.wrap( "incompatible shape for a non-contiguous array")) @@ -1245,27 +1091,42 @@ shape.append(item) return size, shape -def array(space, w_item_or_iterable, w_dtype=None, w_order=NoneNotWrapped): + at unwrap_spec(subok=bool, copy=bool, ownmaskna=bool) +def array(space, w_item_or_iterable, w_dtype=None, w_order=None, + subok=True, copy=True, w_maskna=None, ownmaskna=False): # find scalar + if w_maskna is None: + w_maskna = space.w_None + if (not subok or not space.is_w(w_maskna, space.w_None) or + ownmaskna): + raise OperationError(space.w_NotImplementedError, space.wrap("Unsupported args")) if not space.issequence_w(w_item_or_iterable): - if space.is_w(w_dtype, space.w_None): + if w_dtype is None or space.is_w(w_dtype, space.w_None): w_dtype = interp_ufuncs.find_dtype_for_scalar(space, w_item_or_iterable) dtype = space.interp_w(interp_dtype.W_Dtype, space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype) ) return scalar_w(space, dtype, w_item_or_iterable) - if w_order is None: + if space.is_w(w_order, space.w_None) or w_order is None: order = 'C' else: order = space.str_w(w_order) if order != 'C': # or order != 'F': raise operationerrfmt(space.w_ValueError, "Unknown order: %s", order) - shape, elems_w = _find_shape_and_elems(space, w_item_or_iterable) + if isinstance(w_item_or_iterable, BaseArray): + if (not space.is_w(w_dtype, space.w_None) and + w_item_or_iterable.find_dtype() is not w_dtype): + raise OperationError(space.w_NotImplementedError, space.wrap( + "copying over different dtypes unsupported")) + if copy: + return w_item_or_iterable.copy(space) + return w_item_or_iterable + shape, elems_w = find_shape_and_elems(space, w_item_or_iterable) # they come back in C order size = len(elems_w) - if space.is_w(w_dtype, space.w_None): + if w_dtype is None or space.is_w(w_dtype, space.w_None): w_dtype = None for w_elem in elems_w: w_dtype = interp_ufuncs.find_dtype_for_scalar(space, w_elem, @@ -1293,6 +1154,8 @@ space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype) ) size, shape = _find_size_and_shape(space, w_size) + if not shape: + return scalar_w(space, dtype, space.wrap(0)) return space.wrap(W_NDimArray(size, shape[:], dtype=dtype)) def ones(space, w_size, w_dtype=None): @@ -1301,17 +1164,66 @@ ) size, shape = _find_size_and_shape(space, w_size) + if not shape: + return scalar_w(space, dtype, space.wrap(1)) arr = W_NDimArray(size, shape[:], dtype=dtype) one = dtype.box(1) arr.dtype.fill(arr.storage, one, 0, size) return space.wrap(arr) + at unwrap_spec(arr=BaseArray, skipna=bool, keepdims=bool) +def count_reduce_items(space, arr, w_axis=None, skipna=False, keepdims=True): + if not keepdims: + raise OperationError(space.w_NotImplementedError, space.wrap("unsupported")) + if space.is_w(w_axis, space.w_None): + return space.wrap(support.product(arr.shape)) + if space.isinstance_w(w_axis, space.w_int): + return space.wrap(arr.shape[space.int_w(w_axis)]) + s = 1 + elems = space.fixedview(w_axis) + for w_elem in elems: + s *= arr.shape[space.int_w(w_elem)] + return space.wrap(s) + def dot(space, w_obj, w_obj2): w_arr = convert_to_array(space, w_obj) if isinstance(w_arr, Scalar): return convert_to_array(space, w_obj2).descr_dot(space, w_arr) return w_arr.descr_dot(space, w_obj2) + at unwrap_spec(axis=int) +def concatenate(space, w_args, axis=0): + args_w = space.listview(w_args) + if len(args_w) == 0: + raise OperationError(space.w_ValueError, space.wrap("concatenation of zero-length sequences is impossible")) + args_w = [convert_to_array(space, w_arg) for w_arg in args_w] + dtype = args_w[0].find_dtype() + shape = args_w[0].shape[:] + if len(shape) <= axis: + raise OperationError(space.w_ValueError, + space.wrap("bad axis argument")) + for arr in args_w[1:]: + dtype = interp_ufuncs.find_binop_result_dtype(space, dtype, + arr.find_dtype()) + if len(arr.shape) <= axis: + raise OperationError(space.w_ValueError, + space.wrap("bad axis argument")) + for i, axis_size in enumerate(arr.shape): + if len(arr.shape) != len(shape) or (i != axis and axis_size != shape[i]): + raise OperationError(space.w_ValueError, space.wrap( + "array dimensions must agree except for axis being concatenated")) + elif i == axis: + shape[i] += axis_size + res = W_NDimArray(support.product(shape), shape, dtype, 'C') + chunks = [Chunk(0, i, 1, i) for i in shape] + axis_start = 0 + for arr in args_w: + chunks[axis] = Chunk(axis_start, axis_start + arr.shape[axis], 1, + arr.shape[axis]) + res.create_slice(chunks).setslice(space, arr) + axis_start += arr.shape[axis] + return res + BaseArray.typedef = TypeDef( 'ndarray', __module__ = "numpypy", @@ -1349,6 +1261,7 @@ __and__ = interp2app(BaseArray.descr_and), __or__ = interp2app(BaseArray.descr_or), + __invert__ = interp2app(BaseArray.descr_invert), __repr__ = interp2app(BaseArray.descr_repr), __str__ = interp2app(BaseArray.descr_str), @@ -1359,9 +1272,11 @@ BaseArray.descr_set_shape), size = GetSetProperty(BaseArray.descr_get_size), ndim = GetSetProperty(BaseArray.descr_get_ndim), + item = interp2app(BaseArray.descr_item), T = GetSetProperty(BaseArray.descr_get_transpose), flat = GetSetProperty(BaseArray.descr_get_flatiter), + ravel = interp2app(BaseArray.descr_ravel), mean = interp2app(BaseArray.descr_mean), sum = interp2app(BaseArray.descr_sum), @@ -1379,8 +1294,11 @@ fill = interp2app(BaseArray.descr_fill), copy = interp2app(BaseArray.descr_copy), + flatten = interp2app(BaseArray.descr_flatten), reshape = interp2app(BaseArray.descr_reshape), tolist = interp2app(BaseArray.descr_tolist), + take = interp2app(BaseArray.descr_take), + compress = interp2app(BaseArray.descr_compress), ) @@ -1389,30 +1307,129 @@ @jit.unroll_safe def __init__(self, arr): arr = arr.get_concrete() - size = 1 - for sh in arr.shape: - size *= sh self.strides = [arr.strides[-1]] self.backstrides = [arr.backstrides[-1]] - ViewArray.__init__(self, size, [size], arr.dtype, arr.order, - arr) self.shapelen = len(arr.shape) - self.iter = OneDimIterator(arr.start, self.strides[0], - self.shape[0]) + sig = arr.find_sig() + self.iter = sig.create_frame(arr).get_final_iter() + self.base = arr + self.index = 0 + ViewArray.__init__(self, arr.size, [arr.size], arr.dtype, arr.order, + arr) def descr_next(self, space): if self.iter.done(): raise OperationError(space.w_StopIteration, space.w_None) - result = self.getitem(self.iter.offset) + result = self.base.getitem(self.iter.offset) self.iter = self.iter.next(self.shapelen) + self.index += 1 return result def descr_iter(self): return self + def descr_index(self, space): + return space.wrap(self.index) + + def descr_coords(self, space): + coords, step, lngth = to_coords(space, self.base.shape, + self.base.size, self.base.order, + space.wrap(self.index)) + return space.newtuple([space.wrap(c) for c in coords]) + + @jit.unroll_safe + def descr_getitem(self, space, w_idx): + if not (space.isinstance_w(w_idx, space.w_int) or + space.isinstance_w(w_idx, space.w_slice)): + raise OperationError(space.w_IndexError, + space.wrap('unsupported iterator index')) + base = self.base + start, stop, step, lngth = space.decode_index4(w_idx, base.size) + # setslice would have been better, but flat[u:v] for arbitrary + # shapes of array a cannot be represented as a[x1:x2, y1:y2] + basei = ViewIterator(base.start, base.strides, + base.backstrides,base.shape) + shapelen = len(base.shape) + basei = basei.next_skip_x(shapelen, start) + if lngth <2: + return base.getitem(basei.offset) + ri = ArrayIterator(lngth) + res = W_NDimArray(lngth, [lngth], base.dtype, + base.order) + while not ri.done(): + flat_get_driver.jit_merge_point(shapelen=shapelen, + base=base, + basei=basei, + step=step, + res=res, + ri=ri, + ) + w_val = base.getitem(basei.offset) + res.setitem(ri.offset,w_val) + basei = basei.next_skip_x(shapelen, step) + ri = ri.next(shapelen) + return res + + def descr_setitem(self, space, w_idx, w_value): + if not (space.isinstance_w(w_idx, space.w_int) or + space.isinstance_w(w_idx, space.w_slice)): + raise OperationError(space.w_IndexError, + space.wrap('unsupported iterator index')) + base = self.base + start, stop, step, lngth = space.decode_index4(w_idx, base.size) + arr = convert_to_array(space, w_value) + ai = 0 + basei = ViewIterator(base.start, base.strides, + base.backstrides,base.shape) + shapelen = len(base.shape) + basei = basei.next_skip_x(shapelen, start) + while lngth > 0: + flat_set_driver.jit_merge_point(shapelen=shapelen, + basei=basei, + base=base, + step=step, + arr=arr, + ai=ai, + lngth=lngth, + ) + v = arr.getitem(ai).convert_to(base.dtype) + base.setitem(basei.offset, v) + # need to repeat input values until all assignments are done + ai = (ai + 1) % arr.size + basei = basei.next_skip_x(shapelen, step) + lngth -= 1 + + def create_sig(self): + return signature.FlatSignature(self.base.dtype) + + def descr_base(self, space): + return space.wrap(self.base) + W_FlatIterator.typedef = TypeDef( 'flatiter', + #__array__ = #MISSING + __iter__ = interp2app(W_FlatIterator.descr_iter), + __getitem__ = interp2app(W_FlatIterator.descr_getitem), + __setitem__ = interp2app(W_FlatIterator.descr_setitem), + __eq__ = interp2app(BaseArray.descr_eq), + __ne__ = interp2app(BaseArray.descr_ne), + __lt__ = interp2app(BaseArray.descr_lt), + __le__ = interp2app(BaseArray.descr_le), + __gt__ = interp2app(BaseArray.descr_gt), + __ge__ = interp2app(BaseArray.descr_ge), + #__sizeof__ #MISSING + base = GetSetProperty(W_FlatIterator.descr_base), + index = GetSetProperty(W_FlatIterator.descr_index), + coords = GetSetProperty(W_FlatIterator.descr_coords), next = interp2app(W_FlatIterator.descr_next), - __iter__ = interp2app(W_FlatIterator.descr_iter), + ) W_FlatIterator.acceptable_as_base_class = False + +def isna(space, w_obj): + if isinstance(w_obj, BaseArray): + arr = w_obj.empty_copy(space, + interp_dtype.get_dtype_cache(space).w_booldtype) + arr.fill(space, space.wrap(False)) + return arr + return space.wrap(False) diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py --- a/pypy/module/micronumpy/interp_ufuncs.py +++ b/pypy/module/micronumpy/interp_ufuncs.py @@ -1,14 +1,15 @@ from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.error import OperationError, operationerrfmt -from pypy.interpreter.gateway import interp2app +from pypy.interpreter.gateway import interp2app, unwrap_spec, NoneNotWrapped from pypy.interpreter.typedef import TypeDef, GetSetProperty, interp_attrproperty -from pypy.module.micronumpy import interp_boxes, interp_dtype -from pypy.module.micronumpy.signature import ReduceSignature,\ - find_sig, new_printable_location, AxisReduceSignature, ScalarSignature +from pypy.module.micronumpy import interp_boxes, interp_dtype, support +from pypy.module.micronumpy.signature import (ReduceSignature, find_sig, + new_printable_location, AxisReduceSignature, ScalarSignature) from pypy.rlib import jit from pypy.rlib.rarithmetic import LONG_BIT from pypy.tool.sourcetools import func_with_new_name + reduce_driver = jit.JitDriver( greens=['shapelen', "sig"], virtualizables=["frame"], @@ -30,12 +31,14 @@ _attrs_ = ["name", "promote_to_float", "promote_bools", "identity"] _immutable_fields_ = ["promote_to_float", "promote_bools", "name"] - def __init__(self, name, promote_to_float, promote_bools, identity): + def __init__(self, name, promote_to_float, promote_bools, identity, + int_only): self.name = name self.promote_to_float = promote_to_float self.promote_bools = promote_bools self.identity = identity + self.int_only = int_only def descr_repr(self, space): return space.wrap("<ufunc '%s'>" % self.name) @@ -46,19 +49,31 @@ return self.identity def descr_call(self, space, __args__): - if __args__.keywords or len(__args__.arguments_w) < self.argcount: + args_w, kwds_w = __args__.unpack() + # it occurs to me that we don't support any datatypes that + # require casting, change it later when we do + kwds_w.pop('casting', None) + w_subok = kwds_w.pop('subok', None) + w_out = kwds_w.pop('out', space.w_None) + if ((w_subok is not None and space.is_true(w_subok)) or + not space.is_w(w_out, space.w_None)): + raise OperationError(space.w_NotImplementedError, + space.wrap("parameters unsupported")) + if kwds_w or len(args_w) < self.argcount: raise OperationError(space.w_ValueError, space.wrap("invalid number of arguments") ) - elif len(__args__.arguments_w) > self.argcount: + elif len(args_w) > self.argcount: # The extra arguments should actually be the output array, but we # don't support that yet. raise OperationError(space.w_TypeError, space.wrap("invalid number of arguments") ) - return self.call(space, __args__.arguments_w) + return self.call(space, args_w) - def descr_reduce(self, space, w_obj, w_dim=0): + @unwrap_spec(skipna=bool, keepdims=bool) + def descr_reduce(self, space, w_obj, w_axis=NoneNotWrapped, w_dtype=None, + skipna=False, keepdims=False, w_out=None): """reduce(...) reduce(a, axis=0) @@ -111,15 +126,24 @@ array([[ 1, 5], [ 9, 13]]) """ - return self.reduce(space, w_obj, False, False, w_dim) + if not space.is_w(w_out, space.w_None): + raise OperationError(space.w_NotImplementedError, space.wrap( + "out not supported")) + if w_axis is None: + axis = 0 + elif space.is_w(w_axis, space.w_None): + axis = -1 + else: + axis = space.int_w(w_axis) + return self.reduce(space, w_obj, False, False, axis, keepdims) - def reduce(self, space, w_obj, multidim, promote_to_largest, w_dim): + def reduce(self, space, w_obj, multidim, promote_to_largest, dim, + keepdims=False): from pypy.module.micronumpy.interp_numarray import convert_to_array, \ Scalar if self.argcount != 2: raise OperationError(space.w_ValueError, space.wrap("reduce only " "supported for binary functions")) - dim = space.int_w(w_dim) assert isinstance(self, W_Ufunc2) obj = convert_to_array(space, w_obj) if dim >= len(obj.shape): @@ -140,7 +164,7 @@ raise operationerrfmt(space.w_ValueError, "zero-size array to " "%s.reduce without identity", self.name) if shapelen > 1 and dim >= 0: - res = self.do_axis_reduce(obj, dtype, dim) + res = self.do_axis_reduce(obj, dtype, dim, keepdims) return space.wrap(res) scalarsig = ScalarSignature(dtype) sig = find_sig(ReduceSignature(self.func, self.name, dtype, @@ -154,15 +178,15 @@ value = self.identity.convert_to(dtype) return self.reduce_loop(shapelen, sig, frame, value, obj, dtype) - def do_axis_reduce(self, obj, dtype, dim): + def do_axis_reduce(self, obj, dtype, dim, keepdims): from pypy.module.micronumpy.interp_numarray import AxisReduce,\ W_NDimArray - - shape = obj.shape[0:dim] + obj.shape[dim + 1:len(obj.shape)] - size = 1 - for s in shape: - size *= s - result = W_NDimArray(size, shape, dtype) + + if keepdims: + shape = obj.shape[:dim] + [1] + obj.shape[dim + 1:] + else: + shape = obj.shape[:dim] + obj.shape[dim + 1:] + result = W_NDimArray(support.product(shape), shape, dtype) rightsig = obj.create_sig() # note - this is just a wrapper so signature can fetch # both left and right, nothing more, especially @@ -224,10 +248,12 @@ _immutable_fields_ = ["func", "name"] def __init__(self, func, name, promote_to_float=False, promote_bools=False, - identity=None): + identity=None, bool_result=False, int_only=False): - W_Ufunc.__init__(self, name, promote_to_float, promote_bools, identity) + W_Ufunc.__init__(self, name, promote_to_float, promote_bools, identity, + int_only) self.func = func + self.bool_result = bool_result def call(self, space, args_w): from pypy.module.micronumpy.interp_numarray import (Call1, @@ -235,15 +261,19 @@ [w_obj] = args_w w_obj = convert_to_array(space, w_obj) - res_dtype = find_unaryop_result_dtype(space, - w_obj.find_dtype(), - promote_to_float=self.promote_to_float, - promote_bools=self.promote_bools, - ) + calc_dtype = find_unaryop_result_dtype(space, + w_obj.find_dtype(), + promote_to_float=self.promote_to_float, + promote_bools=self.promote_bools) + if self.bool_result: + res_dtype = interp_dtype.get_dtype_cache(space).w_booldtype + else: + res_dtype = calc_dtype if isinstance(w_obj, Scalar): - return self.func(res_dtype, w_obj.value.convert_to(res_dtype)) + return space.wrap(self.func(calc_dtype, w_obj.value.convert_to(calc_dtype))) - w_res = Call1(self.func, self.name, w_obj.shape, res_dtype, w_obj) + w_res = Call1(self.func, self.name, w_obj.shape, calc_dtype, res_dtype, + w_obj) w_obj.add_invalidates(w_res) return w_res @@ -255,10 +285,10 @@ def __init__(self, func, name, promote_to_float=False, promote_bools=False, identity=None, comparison_func=False, int_only=False): - W_Ufunc.__init__(self, name, promote_to_float, promote_bools, identity) + W_Ufunc.__init__(self, name, promote_to_float, promote_bools, identity, + int_only) self.func = func self.comparison_func = comparison_func - self.int_only = int_only def call(self, space, args_w): from pypy.module.micronumpy.interp_numarray import (Call2, @@ -278,10 +308,10 @@ else: res_dtype = calc_dtype if isinstance(w_lhs, Scalar) and isinstance(w_rhs, Scalar): - return self.func(calc_dtype, + return space.wrap(self.func(calc_dtype, w_lhs.value.convert_to(calc_dtype), w_rhs.value.convert_to(calc_dtype) - ) + )) new_shape = shape_agreement(space, w_lhs.shape, w_rhs.shape) w_res = Call2(self.func, self.name, @@ -404,12 +434,16 @@ return interp_dtype.get_dtype_cache(space).w_float64dtype -def ufunc_dtype_caller(space, ufunc_name, op_name, argcount, comparison_func): +def ufunc_dtype_caller(space, ufunc_name, op_name, argcount, comparison_func, + bool_result): + dtype_cache = interp_dtype.get_dtype_cache(space) if argcount == 1: def impl(res_dtype, value): - return getattr(res_dtype.itemtype, op_name)(value) + res = getattr(res_dtype.itemtype, op_name)(value) + if bool_result: + return dtype_cache.w_booldtype.box(res) + return res elif argcount == 2: - dtype_cache = interp_dtype.get_dtype_cache(space) def impl(res_dtype, lvalue, rvalue): res = getattr(res_dtype.itemtype, op_name)(lvalue, rvalue) if comparison_func: @@ -428,7 +462,9 @@ 'int_only': True}), ("bitwise_or", "bitwise_or", 2, {"identity": 0, 'int_only': True}), + ("invert", "invert", 1, {"int_only": True}), ("divide", "div", 2, {"promote_bools": True}), + ("true_divide", "div", 2, {"promote_to_float": True}), ("mod", "mod", 2, {"promote_bools": True}), ("power", "pow", 2, {"promote_bools": True}), @@ -438,6 +474,8 @@ ("less_equal", "le", 2, {"comparison_func": True}), ("greater", "gt", 2, {"comparison_func": True}), ("greater_equal", "ge", 2, {"comparison_func": True}), + ("isnan", "isnan", 1, {"bool_result": True}), + ("isinf", "isinf", 1, {"bool_result": True}), ("maximum", "max", 2), ("minimum", "min", 2), @@ -480,6 +518,7 @@ func = ufunc_dtype_caller(space, ufunc_name, op_name, argcount, comparison_func=extra_kwargs.get("comparison_func", False), + bool_result=extra_kwargs.get("bool_result", False), ) if argcount == 1: ufunc = W_Ufunc1(func, ufunc_name, **extra_kwargs) @@ -489,3 +528,4 @@ def get(space): return space.fromcache(UfuncState) + diff --git a/pypy/module/micronumpy/signature.py b/pypy/module/micronumpy/signature.py --- a/pypy/module/micronumpy/signature.py +++ b/pypy/module/micronumpy/signature.py @@ -224,6 +224,18 @@ return ViewIterator(arr.start, arr.strides, arr.backstrides, arr.shape).apply_transformations(arr, transforms) +class FlatSignature(ViewSignature): + def debug_repr(self): + return 'Flat' + + def allocate_iter(self, arr, transforms): + from pypy.module.micronumpy.interp_numarray import W_FlatIterator + assert isinstance(arr, W_FlatIterator) + return ViewIterator(arr.base.start, arr.base.strides, + arr.base.backstrides, + arr.base.shape).apply_transformations(arr.base, + transforms) + class VirtualSliceSignature(Signature): def __init__(self, child): self.child = child @@ -293,8 +305,8 @@ def eval(self, frame, arr): from pypy.module.micronumpy.interp_numarray import Call1 assert isinstance(arr, Call1) - v = self.child.eval(frame, arr.values).convert_to(arr.res_dtype) - return self.unfunc(arr.res_dtype, v) + v = self.child.eval(frame, arr.values).convert_to(arr.calc_dtype) + return self.unfunc(arr.calc_dtype, v) class Call2(Signature): _immutable_fields_ = ['binfunc', 'name', 'calc_dtype', 'left', 'right'] diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py --- a/pypy/module/micronumpy/strides.py +++ b/pypy/module/micronumpy/strides.py @@ -1,5 +1,5 @@ from pypy.rlib import jit - +from pypy.interpreter.error import OperationError @jit.look_inside_iff(lambda shape, start, strides, backstrides, chunks: jit.isconstant(len(chunks)) @@ -37,3 +37,196 @@ rstrides = [0] * (len(res_shape) - len(orig_shape)) + rstrides rbackstrides = [0] * (len(res_shape) - len(orig_shape)) + rbackstrides return rstrides, rbackstrides + +def find_shape_and_elems(space, w_iterable): + shape = [space.len_w(w_iterable)] + batch = space.listview(w_iterable) + while True: + new_batch = [] + if not batch: + return shape, [] + if not space.issequence_w(batch[0]): + for elem in batch: + if space.issequence_w(elem): + raise OperationError(space.w_ValueError, space.wrap( + "setting an array element with a sequence")) + return shape, batch + size = space.len_w(batch[0]) + for w_elem in batch: + if not space.issequence_w(w_elem) or space.len_w(w_elem) != size: + raise OperationError(space.w_ValueError, space.wrap( + "setting an array element with a sequence")) + new_batch += space.listview(w_elem) + shape.append(size) + batch = new_batch + +def to_coords(space, shape, size, order, w_item_or_slice): + '''Returns a start coord, step, and length. + ''' + start = lngth = step = 0 + if not (space.isinstance_w(w_item_or_slice, space.w_int) or + space.isinstance_w(w_item_or_slice, space.w_slice)): + raise OperationError(space.w_IndexError, + space.wrap('unsupported iterator index')) + + start, stop, step, lngth = space.decode_index4(w_item_or_slice, size) + + coords = [0] * len(shape) + i = start + if order == 'C': + for s in range(len(shape) -1, -1, -1): + coords[s] = i % shape[s] + i //= shape[s] + else: + for s in range(len(shape)): + coords[s] = i % shape[s] + i //= shape[s] + return coords, step, lngth + +def shape_agreement(space, shape1, shape2): + ret = _shape_agreement(shape1, shape2) + if len(ret) < max(len(shape1), len(shape2)): + raise OperationError(space.w_ValueError, + space.wrap("operands could not be broadcast together with shapes (%s) (%s)" % ( + ",".join([str(x) for x in shape1]), + ",".join([str(x) for x in shape2]), + )) + ) + return ret + +def _shape_agreement(shape1, shape2): + """ Checks agreement about two shapes with respect to broadcasting. Returns + the resulting shape. + """ + lshift = 0 + rshift = 0 + if len(shape1) > len(shape2): + m = len(shape1) + n = len(shape2) + rshift = len(shape2) - len(shape1) + remainder = shape1 + else: + m = len(shape2) + n = len(shape1) + lshift = len(shape1) - len(shape2) + remainder = shape2 + endshape = [0] * m + indices1 = [True] * m + indices2 = [True] * m + for i in range(m - 1, m - n - 1, -1): + left = shape1[i + lshift] + right = shape2[i + rshift] + if left == right: + endshape[i] = left + elif left == 1: + endshape[i] = right + indices1[i + lshift] = False + elif right == 1: + endshape[i] = left + indices2[i + rshift] = False + else: + return [] + #raise OperationError(space.w_ValueError, space.wrap( + # "frames are not aligned")) + for i in range(m - n): + endshape[i] = remainder[i] + return endshape + +def get_shape_from_iterable(space, old_size, w_iterable): + new_size = 0 + new_shape = [] + if space.isinstance_w(w_iterable, space.w_int): + new_size = space.int_w(w_iterable) + if new_size < 0: + new_size = old_size + new_shape = [new_size] + else: + neg_dim = -1 + batch = space.listview(w_iterable) + new_size = 1 + if len(batch) < 1: + if old_size == 1: + # Scalars can have an empty size. + new_size = 1 + else: + new_size = 0 + new_shape = [] + i = 0 + for elem in batch: + s = space.int_w(elem) + if s < 0: + if neg_dim >= 0: + raise OperationError(space.w_ValueError, space.wrap( + "can only specify one unknown dimension")) + s = 1 + neg_dim = i + new_size *= s + new_shape.append(s) + i += 1 + if neg_dim >= 0: + new_shape[neg_dim] = old_size / new_size + new_size *= new_shape[neg_dim] + if new_size != old_size: + raise OperationError(space.w_ValueError, + space.wrap("total size of new array must be unchanged")) + return new_shape + +# Recalculating strides. Find the steps that the iteration does for each +# dimension, given the stride and shape. Then try to create a new stride that +# fits the new shape, using those steps. If there is a shape/step mismatch +# (meaning that the realignment of elements crosses from one step into another) +# return None so that the caller can raise an exception. +def calc_new_strides(new_shape, old_shape, old_strides, order): + # Return the proper strides for new_shape, or None if the mapping crosses + # stepping boundaries + + # Assumes that prod(old_shape) == prod(new_shape), len(old_shape) > 1, and + # len(new_shape) > 0 + steps = [] + last_step = 1 + oldI = 0 + new_strides = [] + if order == 'F': + for i in range(len(old_shape)): + steps.append(old_strides[i] / last_step) + last_step *= old_shape[i] + cur_step = steps[0] + n_new_elems_used = 1 + n_old_elems_to_use = old_shape[0] + for s in new_shape: + new_strides.append(cur_step * n_new_elems_used) + n_new_elems_used *= s + while n_new_elems_used > n_old_elems_to_use: + oldI += 1 + if steps[oldI] != steps[oldI - 1]: + return None + n_old_elems_to_use *= old_shape[oldI] + if n_new_elems_used == n_old_elems_to_use: + oldI += 1 + if oldI < len(old_shape): + cur_step = steps[oldI] + n_old_elems_to_use *= old_shape[oldI] + elif order == 'C': + for i in range(len(old_shape) - 1, -1, -1): + steps.insert(0, old_strides[i] / last_step) + last_step *= old_shape[i] + cur_step = steps[-1] + n_new_elems_used = 1 + oldI = -1 + n_old_elems_to_use = old_shape[-1] + for i in range(len(new_shape) - 1, -1, -1): + s = new_shape[i] + new_strides.insert(0, cur_step * n_new_elems_used) + n_new_elems_used *= s + while n_new_elems_used > n_old_elems_to_use: + oldI -= 1 + if steps[oldI] != steps[oldI + 1]: + return None + n_old_elems_to_use *= old_shape[oldI] + if n_new_elems_used == n_old_elems_to_use: + oldI -= 1 + if oldI >= -len(old_shape): + cur_step = steps[oldI] + n_old_elems_to_use *= old_shape[oldI] + assert len(new_strides) == len(new_shape) + return new_strides diff --git a/pypy/module/micronumpy/support.py b/pypy/module/micronumpy/support.py new file mode 100644 --- /dev/null +++ b/pypy/module/micronumpy/support.py @@ -0,0 +1,5 @@ +def product(s): + i = 1 + for x in s: + i *= x + return i \ No newline at end of file diff --git a/pypy/module/micronumpy/test/test_base.py b/pypy/module/micronumpy/test/test_base.py --- a/pypy/module/micronumpy/test/test_base.py +++ b/pypy/module/micronumpy/test/test_base.py @@ -3,9 +3,17 @@ from pypy.module.micronumpy.interp_numarray import W_NDimArray, Scalar from pypy.module.micronumpy.interp_ufuncs import (find_binop_result_dtype, find_unaryop_result_dtype) +from pypy.module.micronumpy.interp_boxes import W_Float64Box +from pypy.conftest import option +import sys class BaseNumpyAppTest(object): def setup_class(cls): + if option.runappdirect: + if '__pypy__' not in sys.builtin_module_names: + import numpy + sys.modules['numpypy'] = numpy + sys.modules['_numpypy'] = numpy cls.space = gettestobjspace(usemodules=['micronumpy']) class TestSignature(object): @@ -16,7 +24,7 @@ ar = W_NDimArray(10, [10], dtype=float64_dtype) ar2 = W_NDimArray(10, [10], dtype=float64_dtype) v1 = ar.descr_add(space, ar) - v2 = ar.descr_add(space, Scalar(float64_dtype, 2.0)) + v2 = ar.descr_add(space, Scalar(float64_dtype, W_Float64Box(2.0))) sig1 = v1.find_sig() sig2 = v2.find_sig() assert v1 is not v2 @@ -26,7 +34,7 @@ sig1b = ar2.descr_add(space, ar).find_sig() assert sig1b.left.array_no != sig1b.right.array_no assert sig1b is not sig1 - v3 = ar.descr_add(space, Scalar(float64_dtype, 1.0)) + v3 = ar.descr_add(space, Scalar(float64_dtype, W_Float64Box(1.0))) sig3 = v3.find_sig() assert sig2 is sig3 v4 = ar.descr_add(space, ar) diff --git a/pypy/module/micronumpy/test/test_compile.py b/pypy/module/micronumpy/test/test_compile.py --- a/pypy/module/micronumpy/test/test_compile.py +++ b/pypy/module/micronumpy/test/test_compile.py @@ -245,3 +245,19 @@ a -> 3 """) assert interp.results[0].value == 11 + + def test_flat_iter(self): + interp = self.run(''' + a = |30| + b = flat(a) + b -> 3 + ''') + assert interp.results[0].value == 3 + + def test_take(self): + interp = self.run(""" + a = |10| + b = take(a, [1, 1, 3, 2]) + b -> 2 + """) + assert interp.results[0].value == 3 diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py --- a/pypy/module/micronumpy/test/test_dtypes.py +++ b/pypy/module/micronumpy/test/test_dtypes.py @@ -11,8 +11,17 @@ assert dtype('int8').num == 1 assert dtype(d) is d assert dtype(None) is dtype(float) + assert dtype('int8').name == 'int8' raises(TypeError, dtype, 1042) + def test_dtype_eq(self): + from _numpypy import dtype + + assert dtype("int8") == "int8" + assert "int8" == dtype("int8") + raises(TypeError, lambda: dtype("int8") == 3) + assert dtype(bool) == bool + def test_dtype_with_types(self): from _numpypy import dtype @@ -30,7 +39,7 @@ def test_repr_str(self): from _numpypy import dtype - assert repr(dtype) == "<type 'numpypy.dtype'>" + assert '.dtype' in repr(dtype) d = dtype('?') assert repr(d) == "dtype('bool')" assert str(d) == "bool" @@ -376,3 +385,19 @@ b = X(10) assert type(b) is X assert b.m() == 12 + + def test_long_as_index(self): + skip("waiting for removal of multimethods of __index__") + from _numpypy import int_ + assert (1, 2, 3)[int_(1)] == 2 + + def test_int(self): + import sys + from _numpypy import int32, int64, int_ + assert issubclass(int_, int) + if sys.maxint == (1<<31) - 1: + assert issubclass(int32, int) + assert int_ is int32 + else: + assert issubclass(int64, int) + assert int_ is int64 diff --git a/pypy/module/micronumpy/test/test_iter.py b/pypy/module/micronumpy/test/test_iter.py new file mode 100644 --- /dev/null +++ b/pypy/module/micronumpy/test/test_iter.py @@ -0,0 +1,88 @@ +from pypy.module.micronumpy.interp_iter import ViewIterator + +class TestIterDirect(object): + def test_C_viewiterator(self): + #Let's get started, simple iteration in C order with + #contiguous layout => strides[-1] is 1 + start = 0 + shape = [3, 5] + strides = [5, 1] + backstrides = [x * (y - 1) for x,y in zip(strides, shape)] + assert backstrides == [10, 4] + i = ViewIterator(start, strides, backstrides, shape) + i = i.next(2) + i = i.next(2) + i = i.next(2) + assert i.offset == 3 + assert not i.done() + assert i.indices == [0,3] + #cause a dimension overflow + i = i.next(2) + i = i.next(2) + assert i.offset == 5 + assert i.indices == [1,0] + + #Now what happens if the array is transposed? strides[-1] != 1 + # therefore layout is non-contiguous + strides = [1, 3] + backstrides = [x * (y - 1) for x,y in zip(strides, shape)] + assert backstrides == [2, 12] + i = ViewIterator(start, strides, backstrides, shape) + i = i.next(2) + i = i.next(2) + i = i.next(2) + assert i.offset == 9 + assert not i.done() + assert i.indices == [0,3] + #cause a dimension overflow + i = i.next(2) + i = i.next(2) + assert i.offset == 1 + assert i.indices == [1,0] + + def test_C_viewiterator_step(self): + #iteration in C order with #contiguous layout => strides[-1] is 1 + #skip less than the shape + start = 0 + shape = [3, 5] + strides = [5, 1] + backstrides = [x * (y - 1) for x,y in zip(strides, shape)] + assert backstrides == [10, 4] + i = ViewIterator(start, strides, backstrides, shape) + i = i.next_skip_x(2,2) + i = i.next_skip_x(2,2) + i = i.next_skip_x(2,2) + assert i.offset == 6 + assert not i.done() + assert i.indices == [1,1] + #And for some big skips + i = i.next_skip_x(2,5) + assert i.offset == 11 + assert i.indices == [2,1] + i = i.next_skip_x(2,5) + # Note: the offset does not overflow but recycles, + # this is good for broadcast + assert i.offset == 1 + assert i.indices == [0,1] + assert i.done() + + #Now what happens if the array is transposed? strides[-1] != 1 + # therefore layout is non-contiguous + strides = [1, 3] + backstrides = [x * (y - 1) for x,y in zip(strides, shape)] + assert backstrides == [2, 12] + i = ViewIterator(start, strides, backstrides, shape) + i = i.next_skip_x(2,2) + i = i.next_skip_x(2,2) + i = i.next_skip_x(2,2) + assert i.offset == 4 + assert i.indices == [1,1] + assert not i.done() + i = i.next_skip_x(2,5) + assert i.offset == 5 + assert i.indices == [2,1] + assert not i.done() + i = i.next_skip_x(2,5) + assert i.indices == [0,1] + assert i.offset == 3 + assert i.done() diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -154,14 +154,36 @@ def test_calc_new_strides(self): from pypy.module.micronumpy.interp_numarray import calc_new_strides - assert calc_new_strides([2, 4], [4, 2], [4, 2]) == [8, 2] - assert calc_new_strides([2, 4, 3], [8, 3], [1, 16]) == [1, 2, 16] - assert calc_new_strides([2, 3, 4], [8, 3], [1, 16]) is None - assert calc_new_strides([24], [2, 4, 3], [48, 6, 1]) is None - assert calc_new_strides([24], [2, 4, 3], [24, 6, 2]) == [2] - assert calc_new_strides([105, 1], [3, 5, 7], [35, 7, 1]) == [1, 1] - assert calc_new_strides([1, 105], [3, 5, 7], [35, 7, 1]) == [105, 1] + assert calc_new_strides([2, 4], [4, 2], [4, 2], "C") == [8, 2] + assert calc_new_strides([2, 4, 3], [8, 3], [1, 16], 'F') == [1, 2, 16] + assert calc_new_strides([2, 3, 4], [8, 3], [1, 16], 'F') is None + assert calc_new_strides([24], [2, 4, 3], [48, 6, 1], 'C') is None + assert calc_new_strides([24], [2, 4, 3], [24, 6, 2], 'C') == [2] + assert calc_new_strides([105, 1], [3, 5, 7], [35, 7, 1],'C') == [1, 1] + assert calc_new_strides([1, 105], [3, 5, 7], [35, 7, 1],'C') == [105, 1] + assert calc_new_strides([1, 105], [3, 5, 7], [35, 7, 1],'F') is None + assert calc_new_strides([1, 1, 1, 105, 1], [15, 7], [7, 1],'C') == \ + [105, 105, 105, 1, 1] + assert calc_new_strides([1, 1, 105, 1, 1], [7, 15], [1, 7],'F') == \ + [1, 1, 1, 105, 105] + def test_to_coords(self): + from pypy.module.micronumpy.strides import to_coords + + def _to_coords(index, order): + return to_coords(self.space, [2, 3, 4], 24, order, + self.space.wrap(index))[0] + + assert _to_coords(0, 'C') == [0, 0, 0] + assert _to_coords(1, 'C') == [0, 0, 1] + assert _to_coords(-1, 'C') == [1, 2, 3] + assert _to_coords(5, 'C') == [0, 1, 1] + assert _to_coords(13, 'C') == [1, 0, 1] + assert _to_coords(0, 'F') == [0, 0, 0] + assert _to_coords(1, 'F') == [1, 0, 0] + assert _to_coords(-1, 'F') == [1, 2, 3] + assert _to_coords(5, 'F') == [1, 2, 0] + assert _to_coords(13, 'F') == [1, 0, 2] class AppTestNumArray(BaseNumpyAppTest): def test_ndarray(self): @@ -206,6 +228,7 @@ # And check that changes stick. a[13] = 5.3 assert a[13] == 5.3 + assert zeros(()).shape == () def test_size(self): from _numpypy import array @@ -249,6 +272,7 @@ b = a[::2] c = b.copy() assert (c == b).all() + assert ((a + a).copy() == (a + a)).all() a = arange(15).reshape(5,3) b = a.copy() @@ -278,6 +302,12 @@ for i in xrange(5): assert a[i] == b[i] + def test_getitem_nd(self): + from _numpypy import arange + a = arange(15).reshape(3, 5) + assert a[1, 3] == 8 + assert a.T[1, 2] == 11 + def test_setitem(self): from _numpypy import array a = array(range(5)) @@ -365,6 +395,7 @@ assert b.shape == (5,) c = a[:3] assert c.shape == (3,) + assert array([]).shape == (0,) def test_set_shape(self): from _numpypy import array, zeros @@ -385,6 +416,8 @@ a.shape = () #numpy allows this a.shape = (1,) + a = array(range(6)).reshape(2,3).T + raises(AttributeError, 'a.shape = 6') def test_reshape(self): from _numpypy import array, zeros @@ -398,6 +431,7 @@ assert (a == [1000, 1, 2, 3, 1000, 5, 6, 7, 1000, 9, 10, 11]).all() a = zeros((4, 2, 3)) a.shape = (12, 2) + (a + a).reshape(2, 12) # assert did not explode def test_slice_reshape(self): from _numpypy import zeros, arange @@ -441,6 +475,13 @@ y = z.reshape(4, 3, 8) assert y.shape == (4, 3, 8) + def test_scalar_reshape(self): + from numpypy import array + a = array(3) + assert a.reshape([1, 1]).shape == (1, 1) + assert a.reshape([1]).shape == (1,) + raises(ValueError, "a.reshape(3)") + def test_add(self): from _numpypy import array a = array(range(5)) @@ -728,7 +769,7 @@ assert d[1] == 12 def test_mean(self): - from _numpypy import array + from _numpypy import array, arange a = array(range(5)) assert a.mean() == 2.0 assert a[:4].mean() == 1.5 @@ -738,6 +779,7 @@ assert a.mean(axis=0)[0, 0] == 35 assert (b == array(range(35, 70), dtype=float).reshape(5, 7)).all() assert (a.mean(2) == array(range(0, 15), dtype=float).reshape(3, 5) * 7 + 3).all() + assert (arange(10).reshape(5, 2).mean(axis=1) == [0.5, 2.5, 4.5, 6.5, 8.5]).all() def test_sum(self): from _numpypy import array @@ -948,7 +990,7 @@ assert debug_repr(a + a) == 'Call2(add, Array, Array)' assert debug_repr(a[::2]) == 'Slice' assert debug_repr(a + 2) == 'Call2(add, Array, Scalar)' - assert debug_repr(a + a.flat) == 'Call2(add, Array, Slice)' + assert debug_repr(a + a.flat) == 'Call2(add, Array, Flat)' assert debug_repr(sin(a)) == 'Call1(sin, Array)' b = a + a @@ -1021,11 +1063,49 @@ assert a[0].tolist() == [17.1, 27.2] def test_var(self): - from _numpypy import array + from _numpypy import array, arange a = array(range(10)) assert a.var() == 8.25 a = array([5.0]) assert a.var() == 0.0 + a = arange(10).reshape(5, 2) + assert a.var() == 8.25 + assert (a.var(0) == [8, 8]).all() + assert (a.var(1) == [.25] * 5).all() + + def test_concatenate(self): + from numpypy import array, concatenate, dtype + a1 = array([0,1,2]) + a2 = array([3,4,5]) + a = concatenate((a1, a2)) + assert len(a) == 6 + assert (a == [0,1,2,3,4,5]).all() + assert a.dtype is dtype(int) + b1 = array([[1, 2], [3, 4]]) + b2 = array([[5, 6]]) + b = concatenate((b1, b2), axis=0) + assert (b == [[1, 2],[3, 4],[5, 6]]).all() + c = concatenate((b1, b2.T), axis=1) + assert (c == [[1, 2, 5],[3, 4, 6]]).all() + d = concatenate(([0],[1])) + assert (d == [0,1]).all() + e1 = array([[0,1],[2,3]]) + e = concatenate(e1) + assert (e == [0,1,2,3]).all() + f1 = array([0,1]) + f = concatenate((f1, [2], f1, [7])) + assert (f == [0,1,2,0,1,7]).all() + + bad_axis = raises(ValueError, concatenate, (a1,a2), axis=1) + assert str(bad_axis.value) == "bad axis argument" + + concat_zero = raises(ValueError, concatenate, ()) + assert str(concat_zero.value) == \ + "concatenation of zero-length sequences is impossible" + + dims_disagree = raises(ValueError, concatenate, (a1, b1), axis=0) + assert str(dims_disagree.value) == \ + "array dimensions must agree except for axis being concatenated" def test_std(self): from _numpypy import array @@ -1034,6 +1114,28 @@ a = array([5.0]) assert a.std() == 0.0 + def test_flatten(self): + from _numpypy import array + + assert array(3).flatten().shape == (1,) + a = array([[1, 2], [3, 4]]) + b = a.flatten() + c = a.ravel() + a[0, 0] = 15 + assert b[0] == 1 + assert c[0] == 15 + a = array([[1, 2, 3], [4, 5, 6]]) + assert (a.flatten() == [1, 2, 3, 4, 5, 6]).all() + a = array([[[1, 2], [3, 4]], [[5, 6], [7, 8]]]) + assert (a.flatten() == [1, 2, 3, 4, 5, 6, 7, 8]).all() + a = array([1, 2, 3, 4, 5, 6, 7, 8]) + assert (a[::2].flatten() == [1, 3, 5, 7]).all() + a = array([1, 2, 3]) + assert ((a + a).flatten() == [2, 4, 6]).all() + a = array(2) + assert (a.flatten() == [2]).all() + a = array([[1, 2], [3, 4]]) + assert (a.T.flatten() == [1, 3, 2, 4]).all() class AppTestMultiDim(BaseNumpyAppTest): def test_init(self): @@ -1049,7 +1151,7 @@ assert _numpypy.array([[1], [2], [3]]).shape == (3, 1) assert len(_numpypy.zeros((3, 1, 2))) == 3 raises(TypeError, len, _numpypy.zeros(())) - raises(ValueError, _numpypy.array, [[1, 2], 3]) + raises(ValueError, _numpypy.array, [[1, 2], 3], dtype=float) def test_getsetitem(self): import _numpypy @@ -1264,7 +1366,7 @@ assert(b[:, 0] == a[0, :]).all() def test_flatiter(self): - from _numpypy import array, flatiter + from _numpypy import array, flatiter, arange a = array([[10, 30], [40, 60]]) f_iter = a.flat assert f_iter.next() == 10 @@ -1277,6 +1379,9 @@ for k in a.flat: s += k assert s == 140 + a = arange(10).reshape(5, 2) + raises(IndexError, 'a.flat[(1, 2)]') + assert a.flat.base is a def test_flatiter_array_conv(self): from _numpypy import array, dot @@ -1288,6 +1393,75 @@ a = ones((2, 2)) assert list(((a + a).flat)) == [2, 2, 2, 2] + def test_flatiter_getitem(self): + from _numpypy import arange + a = arange(10) + assert a.flat[3] == 3 + assert a[2:].flat[3] == 5 + assert (a + a).flat[3] == 6 + assert a[::2].flat[3] == 6 + assert a.reshape(2,5).flat[3] == 3 + b = a.reshape(2,5).flat + b.next() + b.next() + b.next() + assert b[3] == 3 + assert (b[::3] == [0, 3, 6, 9]).all() + assert (b[2::5] == [2, 7]).all() + assert b[-2] == 8 + raises(IndexError, "b[11]") + raises(IndexError, "b[-11]") + raises(IndexError, 'b[0, 1]') + assert b.index == 3 + assert b.coords == (0,3) + + def test_flatiter_setitem(self): + from _numpypy import arange, array + a = arange(12).reshape(3,4) + b = a.T.flat + b[6::2] = [-1, -2] + assert (a == [[0, 1, -1, 3], [4, 5, 6, -1], [8, 9, -2, 11]]).all() + b[0:2] = [[[100]]] + assert(a[0,0] == 100) + assert(a[1,0] == 100) + raises(IndexError, 'b[array([10, 11])] == [-20, -40]') + + def test_flatiter_ops(self): + from _numpypy import arange, array + a = arange(12).reshape(3,4) + b = a.T.flat + assert (b == [0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11]).all() + assert not (b != [0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11]).any() + assert ((b >= range(12)) == [True, True, True,False, True, True, + False, False, True, False, False, True]).all() + assert ((b < range(12)) != [True, True, True,False, True, True, + False, False, True, False, False, True]).all() + assert ((b <= range(12)) != [False, True, True,False, True, True, + False, False, True, False, False, False]).all() + assert ((b > range(12)) == [False, True, True,False, True, True, + False, False, True, False, False, False]).all() + def test_flatiter_view(self): + from _numpypy import arange + a = arange(10).reshape(5, 2) + #no == yet. + # a[::2].flat == [0, 1, 4, 5, 8, 9] + isequal = True + for y,z in zip(a[::2].flat, [0, 1, 4, 5, 8, 9]): + if y != z: + isequal = False + assert isequal == True + + def test_flatiter_transpose(self): + from _numpypy import arange + a = arange(10).reshape(2,5).T + b = a.flat + assert (b[:5] == [0, 5, 1, 6, 2]).all() + b.next() + b.next() + b.next() + assert b.index == 3 + assert b.coords == (1,1) + def test_slice_copy(self): from _numpypy import zeros a = zeros((10, 10)) @@ -1353,7 +1527,61 @@ a[a & 1 == 1] = array([8, 9, 10]) assert (a == [[0, 8], [2, 9], [4, 10]]).all() + def test_copy_kwarg(self): + from _numpypy import array + x = array([1, 2, 3]) + assert (array(x) == x).all() + assert array(x) is not x + assert array(x, copy=False) is x + assert array(x, copy=True) is not x + def test_isna(self): + from _numpypy import isna, array + # XXX for now + assert not isna(3) + assert (isna(array([1, 2, 3, 4])) == [False, False, False, False]).all() + + def test_ravel(self): + from _numpypy import arange + assert (arange(3).ravel() == arange(3)).all() + assert (arange(6).reshape(2, 3).ravel() == arange(6)).all() + assert (arange(6).reshape(2, 3).T.ravel() == [0, 3, 1, 4, 2, 5]).all() + + def test_take(self): + from _numpypy import arange + assert (arange(10).take([1, 2, 1, 1]) == [1, 2, 1, 1]).all() + raises(IndexError, "arange(3).take([15])") + a = arange(6).reshape(2, 3) + assert (a.take([1, 0, 3]) == [1, 0, 3]).all() + assert ((a + a).take([3]) == [6]).all() + a = arange(12).reshape(2, 6) + assert (a[:,::2].take([3, 2, 1]) == [6, 4, 2]).all() + + def test_compress(self): + from _numpypy import arange + a = arange(10) + assert (a.compress([True, False, True]) == [0, 2]).all() + assert (a.compress([1, 0, 13]) == [0, 2]).all() + assert (a.compress([1, 0, 13.5]) == [0, 2]).all() + a = arange(10).reshape(2, 5) + assert (a.compress([True, False, True]) == [0, 2]).all() + raises(IndexError, "a.compress([1] * 100)") + + def test_item(self): + from _numpypy import array + assert array(3).item() == 3 + assert type(array(3).item()) is int + assert type(array(True).item()) is bool + assert type(array(3.5).item()) is float + raises((ValueError, IndexError), "array(3).item(15)") + raises(ValueError, "array([1, 2, 3]).item()") + assert array([3]).item(0) == 3 + assert type(array([3]).item(0)) is int + assert array([1, 2, 3]).item(-1) == 3 + a = array([1, 2, 3]) + assert a[::2].item(1) == 3 + assert (a + a).item(1) == 4 + raises(ValueError, "array(5).item(1)") class AppTestSupport(BaseNumpyAppTest): def setup_class(cls): @@ -1465,128 +1693,6 @@ raises(ValueError, fromstring, "\x01\x02\x03", count=5, dtype=uint8) -class AppTestRepr(BaseNumpyAppTest): - def test_repr(self): - from _numpypy import array, zeros - int_size = array(5).dtype.itemsize - a = array(range(5), float) - assert repr(a) == "array([0.0, 1.0, 2.0, 3.0, 4.0])" - a = array([], float) - assert repr(a) == "array([], dtype=float64)" - a = zeros(1001) - assert repr(a) == "array([0.0, 0.0, 0.0, ..., 0.0, 0.0, 0.0])" - a = array(range(5), long) - if a.dtype.itemsize == int_size: - assert repr(a) == "array([0, 1, 2, 3, 4])" - else: - assert repr(a) == "array([0, 1, 2, 3, 4], dtype=int64)" - a = array(range(5), 'int32') - if a.dtype.itemsize == int_size: - assert repr(a) == "array([0, 1, 2, 3, 4])" - else: - assert repr(a) == "array([0, 1, 2, 3, 4], dtype=int32)" - a = array([], long) - assert repr(a) == "array([], dtype=int64)" - a = array([True, False, True, False], "?") - assert repr(a) == "array([True, False, True, False], dtype=bool)" - a = zeros([]) - assert repr(a) == "array(0.0)" - a = array(0.2) - assert repr(a) == "array(0.2)" - a = array([2]) - assert repr(a) == "array([2])" - - def test_repr_multi(self): - from _numpypy import arange, zeros, array - a = zeros((3, 4)) - assert repr(a) == '''array([[0.0, 0.0, 0.0, 0.0], - [0.0, 0.0, 0.0, 0.0], - [0.0, 0.0, 0.0, 0.0]])''' - a = zeros((2, 3, 4)) - assert repr(a) == '''array([[[0.0, 0.0, 0.0, 0.0], - [0.0, 0.0, 0.0, 0.0], - [0.0, 0.0, 0.0, 0.0]], - - [[0.0, 0.0, 0.0, 0.0], - [0.0, 0.0, 0.0, 0.0], - [0.0, 0.0, 0.0, 0.0]]])''' - a = arange(1002).reshape((2, 501)) - assert repr(a) == '''array([[0, 1, 2, ..., 498, 499, 500], - [501, 502, 503, ..., 999, 1000, 1001]])''' - assert repr(a.T) == '''array([[0, 501], - [1, 502], - [2, 503], - ..., - [498, 999], - [499, 1000], - [500, 1001]])''' - a = arange(2).reshape((2,1)) - assert repr(a) == '''array([[0], - [1]])''' - - def test_repr_slice(self): - from _numpypy import array, zeros - a = array(range(5), float) - b = a[1::2] - assert repr(b) == "array([1.0, 3.0])" - a = zeros(2002) - b = a[::2] - assert repr(b) == "array([0.0, 0.0, 0.0, ..., 0.0, 0.0, 0.0])" - a = array((range(5), range(5, 10)), dtype="int16") - b = a[1, 2:] - assert repr(b) == "array([7, 8, 9], dtype=int16)" - # an empty slice prints its shape - b = a[2:1, ] - assert repr(b) == "array([], shape=(0, 5), dtype=int16)" - - def test_str(self): - from _numpypy import array, zeros - a = array(range(5), float) - assert str(a) == "[0.0 1.0 2.0 3.0 4.0]" - assert str((2 * a)[:]) == "[0.0 2.0 4.0 6.0 8.0]" - a = zeros(1001) - assert str(a) == "[0.0 0.0 0.0 ..., 0.0 0.0 0.0]" - - a = array(range(5), dtype=long) - assert str(a) == "[0 1 2 3 4]" - a = array([True, False, True, False], dtype="?") - assert str(a) == "[True False True False]" - - a = array(range(5), dtype="int8") - assert str(a) == "[0 1 2 3 4]" - - a = array(range(5), dtype="int16") - assert str(a) == "[0 1 2 3 4]" - - a = array((range(5), range(5, 10)), dtype="int16") - assert str(a) == "[[0 1 2 3 4]\n [5 6 7 8 9]]" - - a = array(3, dtype=int) - assert str(a) == "3" - - a = zeros((400, 400), dtype=int) - assert str(a) == "[[0 0 0 ..., 0 0 0]\n [0 0 0 ..., 0 0 0]\n" \ - " [0 0 0 ..., 0 0 0]\n ...,\n [0 0 0 ..., 0 0 0]\n" \ - " [0 0 0 ..., 0 0 0]\n [0 0 0 ..., 0 0 0]]" - a = zeros((2, 2, 2)) - r = str(a) - assert r == '[[[0.0 0.0]\n [0.0 0.0]]\n\n [[0.0 0.0]\n [0.0 0.0]]]' - - def test_str_slice(self): - from _numpypy import array, zeros - a = array(range(5), float) - b = a[1::2] - assert str(b) == "[1.0 3.0]" - a = zeros(2002) - b = a[::2] - assert str(b) == "[0.0 0.0 0.0 ..., 0.0 0.0 0.0]" - a = array((range(5), range(5, 10)), dtype="int16") - b = a[1, 2:] - assert str(b) == "[7 8 9]" - b = a[2:1, ] - assert str(b) == "[]" - - class AppTestRanges(BaseNumpyAppTest): def test_arange(self): from _numpypy import arange, array, dtype @@ -1607,3 +1713,24 @@ a = arange(0, 0.8, 0.1) assert len(a) == 8 assert arange(False, True, True).dtype is dtype(int) + +from pypy.module.micronumpy.appbridge import get_appbridge_cache + +class AppTestRepr(BaseNumpyAppTest): + def setup_class(cls): + BaseNumpyAppTest.setup_class.im_func(cls) + cache = get_appbridge_cache(cls.space) + cls.old_array_repr = cache.w_array_repr + cls.old_array_str = cache.w_array_str + cache.w_array_str = None + cache.w_array_repr = None + + def test_repr_str(self): + from _numpypy import array + assert repr(array([1, 2, 3])) == 'array([1, 2, 3])' + assert str(array([1, 2, 3])) == 'array([1, 2, 3])' + + def teardown_class(cls): + cache = get_appbridge_cache(cls.space) + cache.w_array_repr = cls.old_array_repr + cache.w_array_str = cls.old_array_str diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py --- a/pypy/module/micronumpy/test/test_ufuncs.py +++ b/pypy/module/micronumpy/test/test_ufuncs.py @@ -344,7 +344,7 @@ from _numpypy import sin, add raises(ValueError, sin.reduce, [1, 2, 3]) - raises(ValueError, add.reduce, 1) + raises((ValueError, TypeError), add.reduce, 1) def test_reduce_1d(self): from _numpypy import add, maximum @@ -360,6 +360,14 @@ assert (add.reduce(a, 0) == [12, 15, 18, 21]).all() assert (add.reduce(a, 1) == [6.0, 22.0, 38.0]).all() + def test_reduce_keepdims(self): + from _numpypy import add, arange + a = arange(12).reshape(3, 4) + b = add.reduce(a, 0, keepdims=True) + assert b.shape == (1, 4) + assert (add.reduce(a, 0, keepdims=True) == [12, 15, 18, 21]).all() + + def test_bitwise(self): from _numpypy import bitwise_and, bitwise_or, arange, array a = arange(6).reshape(2, 3) @@ -369,6 +377,12 @@ assert (a | 1 == bitwise_or(a, 1)).all() raises(TypeError, 'array([1.0]) & 1') + def test_unary_bitops(self): + from _numpypy import bitwise_not, array + a = array([1, 2, 3, 4]) + assert (~a == [-2, -3, -4, -5]).all() + assert (bitwise_not(a) == ~a).all() + def test_comparisons(self): import operator from _numpypy import equal, not_equal, less, less_equal, greater, greater_equal @@ -394,3 +408,28 @@ (3, 3.5), ]: assert ufunc(a, b) == func(a, b) + + def test_count_reduce_items(self): + from _numpypy import count_reduce_items, arange + a = arange(24).reshape(2, 3, 4) + assert count_reduce_items(a) == 24 + assert count_reduce_items(a, 1) == 3 + assert count_reduce_items(a, (1, 2)) == 3 * 4 + + def test_true_divide(self): + from _numpypy import arange, array, true_divide + assert (true_divide(arange(3), array([2, 2, 2])) == array([0, 0.5, 1])).all() + + def test_isnan_isinf(self): + from _numpypy import isnan, isinf, float64, array + assert isnan(float('nan')) + assert isnan(float64(float('nan'))) + assert not isnan(3) + assert isinf(float('inf')) + assert not isnan(3.5) + assert not isinf(3.5) + assert not isnan(float('inf')) + assert not isinf(float('nan')) + assert (isnan(array([0.2, float('inf'), float('nan')])) == [False, False, True]).all() + assert (isinf(array([0.2, float('inf'), float('nan')])) == [False, True, False]).all() + assert isinf(array([0.2])).dtype.kind == 'b' diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py --- a/pypy/module/micronumpy/test/test_zjit.py +++ b/pypy/module/micronumpy/test/test_zjit.py @@ -12,7 +12,7 @@ from pypy.module.micronumpy.compile import (FakeSpace, IntObject, Parser, InterpreterState) from pypy.module.micronumpy.interp_numarray import (W_NDimArray, - BaseArray) + BaseArray, W_FlatIterator) from pypy.rlib.nonconst import NonConstant @@ -287,6 +287,27 @@ 'jump': 1, 'arraylen_gc': 1}) + def define_take(): + return """ + a = |10| + b = take(a, [1, 1, 3, 2]) + b -> 2 + """ + + def test_take(self): + result = self.run("take") + assert result == 3 + self.check_simple_loop({'getinteriorfield_raw': 2, + 'cast_float_to_int': 1, + 'int_lt': 1, + 'int_ge': 2, + 'guard_false': 3, + 'setinteriorfield_raw': 1, + 'int_mul': 1, + 'int_add': 3, + 'jump': 1, + 'arraylen_gc': 2}) + def define_multidim(): return """ a = [[1, 2], [3, 4], [5, 6], [7, 8], [9, 10]] @@ -369,7 +390,71 @@ 'setinteriorfield_raw': 1, 'int_add': 2, 'int_ge': 1, 'guard_false': 1, 'jump': 1, 'arraylen_gc': 1}) + def define_flat_iter(): + return ''' + a = |30| + b = flat(a) + c = b + a + c -> 3 + ''' + def test_flat_iter(self): + result = self.run("flat_iter") + assert result == 6 + self.check_trace_count(1) + self.check_simple_loop({'getinteriorfield_raw': 2, 'float_add': 1, + 'setinteriorfield_raw': 1, 'int_add': 3, + 'int_ge': 1, 'guard_false': 1, + 'arraylen_gc': 1, 'jump': 1}) + + def define_flat_getitem(): + return ''' + a = |30| + b = flat(a) + b -> 4: -> 6 + ''' + + def test_flat_getitem(self): + result = self.run("flat_getitem") + assert result == 10.0 + self.check_trace_count(1) + self.check_simple_loop({'getinteriorfield_raw': 1, + 'setinteriorfield_raw': 1, + 'int_lt': 1, + 'int_ge': 1, + 'int_add': 3, + 'guard_true': 1, + 'guard_false': 1, + 'arraylen_gc': 2, + 'jump': 1}) + + def define_flat_setitem(): + return ''' + a = |30| + b = flat(a) + b[4:] = a->:26 + a -> 5 + ''' + + def test_flat_setitem(self): + result = self.run("flat_setitem") + assert result == 1.0 + self.check_trace_count(1) + # XXX not ideal, but hey, let's ignore it for now + self.check_simple_loop({'getinteriorfield_raw': 1, + 'setinteriorfield_raw': 1, + 'int_lt': 1, + 'int_gt': 1, + 'int_add': 4, + 'guard_true': 2, + 'arraylen_gc': 2, + 'jump': 1, + 'int_sub': 1, + # XXX bad part + 'int_and': 1, + 'int_mod': 1, + 'int_rshift': 1, + }) class TestNumpyOld(LLJitMixin): def setup_class(cls): diff --git a/pypy/module/micronumpy/tool/numready/__init__.py b/pypy/module/micronumpy/tool/numready/__init__.py new file mode 100644 diff --git a/pypy/module/micronumpy/tool/numready/__main__.py b/pypy/module/micronumpy/tool/numready/__main__.py new file mode 100644 --- /dev/null +++ b/pypy/module/micronumpy/tool/numready/__main__.py @@ -0,0 +1,6 @@ +import sys + +from .main import main + + +main(sys.argv) diff --git a/pypy/module/micronumpy/tool/numready/kinds.py b/pypy/module/micronumpy/tool/numready/kinds.py new file mode 100644 --- /dev/null +++ b/pypy/module/micronumpy/tool/numready/kinds.py @@ -0,0 +1,4 @@ +KINDS = { + "UNKNOWN": "U", + "TYPE": "T", +} diff --git a/pypy/module/micronumpy/tool/numready/main.py b/pypy/module/micronumpy/tool/numready/main.py new file mode 100644 --- /dev/null +++ b/pypy/module/micronumpy/tool/numready/main.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" +This should be run under PyPy. +""" + +import os +import platform +import subprocess +import tempfile +import webbrowser +from collections import OrderedDict + +import jinja2 + +from .kinds import KINDS + + +class SearchableSet(object): + def __init__(self, items=()): + self._items = {} + for item in items: + self.add(item) + + def __iter__(self): + return iter(self._items) + + def __contains__(self, other): + return other in self._items + + def __getitem__(self, idx): + return self._items[idx] + + def add(self, item): + self._items[item] = item + + def __len__(self): + return len(self._items) + +class Item(object): + def __init__(self, name, kind, subitems=None): + self.name = name + self.kind = kind + self.subitems = subitems + + def __hash__(self): + return hash(self.name) + + def __eq__(self, other): + if isinstance(other, str): + return self.name == other + return self.name == other.name + + +class ItemStatus(object): + def __init__(self, name, pypy_exists): + self.name = name + self.cls = 'exists' if pypy_exists else '' + self.symbol = u"✔" if pypy_exists else u'✖' + + def __lt__(self, other): + return self.name < other.name + +def find_numpy_items(python, modname="numpy", attr=None): + args = [ + python, os.path.join(os.path.dirname(__file__), "search.py"), modname + ] + if attr is not None: + args.append(attr) + lines = subprocess.check_output(args).splitlines() + items = SearchableSet() + for line in lines: + kind, name = line.split(" : ", 1) + subitems = None + if kind == KINDS["TYPE"] and name in SPECIAL_NAMES and attr is None: + subitems = find_numpy_items(python, modname, name) + items.add(Item(name, kind, subitems)) + return items + +def split(lst): + SPLIT = 5 + lgt = len(lst) // SPLIT + 1 + l = [[] for i in range(lgt)] + for i in range(lgt): + for k in range(SPLIT): + if k * lgt + i < len(lst): + l[i].append(lst[k * lgt + i]) + return l + +SPECIAL_NAMES = ["ndarray", "dtype", "generic", "flatiter"] + +def main(argv): + cpy_items = find_numpy_items("/usr/bin/python") + pypy_items = find_numpy_items(argv[1], "numpypy") + all_items = [] + + msg = "{:d}/{:d} names".format(len(pypy_items), len(cpy_items)) + " " + msg += ", ".join( + "{:d}/{:d} {} attributes".format( + len(pypy_items[name].subitems), len(cpy_items[name].subitems), name + ) + for name in SPECIAL_NAMES + ) + for item in cpy_items: + pypy_exists = item in pypy_items + if item.subitems: + for sub in item.subitems: + all_items.append( + ItemStatus(item.name + "." + sub.name, pypy_exists=pypy_exists and pypy_items[item].subitems and sub in pypy_items[item].subitems) + ) + all_items.append(ItemStatus(item.name, pypy_exists=item in pypy_items)) + env = jinja2.Environment( + loader=jinja2.FileSystemLoader(os.path.dirname(__file__)) + ) + html = env.get_template("page.html").render(all_items=split(sorted(all_items)), msg=msg) + if len(argv) > 2: + with open(argv[2], 'w') as f: + f.write(html.encode("utf-8")) + else: + with tempfile.NamedTemporaryFile(delete=False, suffix=".html") as f: + f.write(html.encode("utf-8")) + print "Saved in: %s" % f.name diff --git a/pypy/module/micronumpy/tool/numready/page.html b/pypy/module/micronumpy/tool/numready/page.html new file mode 100644 --- /dev/null +++ b/pypy/module/micronumpy/tool/numready/page.html @@ -0,0 +1,65 @@ +<!DOCTYPE html> +<html lang="en"> + <head> + <title>NumPyPy Status</title> + <meta http-equiv="content-type" content="text/html; charset=utf-8"> + <style type="text/css"> + body { + font-family: 'Consolas', 'Bitstream Vera Sans Mono', monospace; + } + h1 { + text-align: center; + } + h3 { + text-align: center; + } + table { + border: 8px solid #DFDECB; + margin: 30px auto; + font-size: 12px; + } + table th { + text-align: left; + } + table td { + padding: 4px 10px; + text-align: center; + } + .exists { + background-color: #337792; + color: white; + border: 1px solid #234F61; + } + </style> + </head> + <body> + <h1>NumPyPy Status</h1> + <h3>Overall: {{ msg }}</h3> + <table> + <thead> + <tr> + <th></th> + <th>PyPy</th> + <th></th> + <th>PyPy</th> + <th></th> + <th>PyPy</th> + <th></th> + <th>PyPy</th> + <th></th> + <th>PyPy</th> + </tr> + </thead> + <tbody> + {% for chunk in all_items %} + <tr> + {% for item in chunk %} + <th class='{{ item.cls }}'>{{ item.name }}</th> + <td class='{{ item.cls }}'>{{ item.symbol }}</td> + {% endfor %} + </tr> + {% endfor %} + </tbody> + </table> + </body> +</html> diff --git a/pypy/module/micronumpy/tool/numready/search.py b/pypy/module/micronumpy/tool/numready/search.py new file mode 100644 --- /dev/null +++ b/pypy/module/micronumpy/tool/numready/search.py @@ -0,0 +1,33 @@ +import sys +import types + +# Evil implicit relative import. +from kinds import KINDS + + +def main(argv): + if len(argv) == 2: + [_, modname] = argv + attr = None + elif len(argv) == 3: + [_, modname, attr] = argv + else: + sys.exit("Wrong number of args") + __import__(modname) + obj = sys.modules[modname] + + if attr is not None: + obj = getattr(obj, attr) + + for name in dir(obj): + if attr is None and name.startswith("_"): + continue + subobj = getattr(obj, name) + if isinstance(subobj, types.TypeType): + kind = KINDS["TYPE"] + else: + kind = KINDS["UNKNOWN"] + print kind, ":", name + +if __name__ == "__main__": + main(sys.argv) \ No newline at end of file diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py --- a/pypy/module/micronumpy/types.py +++ b/pypy/module/micronumpy/types.py @@ -23,6 +23,16 @@ ) return dispatcher +def raw_unary_op(func): + specialize.argtype(1)(func) + @functools.wraps(func) + def dispatcher(self, v): + return func( + self, + self.for_computation(self.unbox(v)) + ) + return dispatcher + def simple_binary_op(func): specialize.argtype(1, 2)(func) @functools.wraps(func) @@ -95,7 +105,9 @@ )) def read_bool(self, storage, width, i, offset): - raise NotImplementedError + return bool(self.for_computation( + libffi.array_getitem(clibffi.cast_type_to_ffitype(self.T), + width, storage, i, offset))) def store(self, storage, width, i, offset, box): value = self.unbox(box) @@ -137,6 +149,14 @@ def abs(self, v): return abs(v) + @raw_unary_op + def isnan(self, v): + return False + + @raw_unary_op + def isinf(self, v): + return False + @raw_binary_op def eq(self, v1, v2): return v1 == v2 @@ -171,7 +191,7 @@ @simple_binary_op def min(self, v1, v2): return min(v1, v2) - + class Bool(BaseType, Primitive): T = lltype.Bool @@ -189,11 +209,6 @@ else: return self.False - - def read_bool(self, storage, width, i, offset): - return libffi.array_getitem(clibffi.cast_type_to_ffitype(self.T), - width, storage, i, offset) - def coerce_subtype(self, space, w_subtype, w_item): # Doesn't return subclasses so it can return the constants. return self._coerce(space, w_item) @@ -214,6 +229,18 @@ def default_fromstring(self, space): return self.box(False) + @simple_binary_op + def bitwise_and(self, v1, v2): + return v1 & v2 + + @simple_binary_op + def bitwise_or(self, v1, v2): + return v1 | v2 + + @simple_unary_op + def invert(self, v): + return ~v + class Integer(Primitive): _mixin_ = True @@ -270,6 +297,10 @@ def bitwise_or(self, v1, v2): return v1 | v2 + @simple_unary_op + def invert(self, v): + return ~v + class Int8(BaseType, Integer): T = rffi.SIGNEDCHAR BoxType = interp_boxes.W_Int8Box @@ -448,6 +479,14 @@ except ValueError: return rfloat.NAN + @raw_unary_op + def isnan(self, v): + return rfloat.isnan(v) + + @raw_unary_op + def isinf(self, v): + return rfloat.isinf(v) + class Float32(BaseType, Float): T = rffi.FLOAT diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -371,6 +371,8 @@ if hasattr(__import__(os.name), "forkpty"): def test_forkpty(self): import sys + if 'freebsd' in sys.platform: + skip("hangs indifinitly on FreeBSD (also on CPython).") os = self.posix childpid, master_fd = os.forkpty() assert isinstance(childpid, int) diff --git a/pypy/module/pypyjit/test_pypy_c/test_math.py b/pypy/module/pypyjit/test_pypy_c/test_math.py --- a/pypy/module/pypyjit/test_pypy_c/test_math.py +++ b/pypy/module/pypyjit/test_pypy_c/test_math.py @@ -90,3 +90,19 @@ --TICK-- jump(..., descr=) """) + + def test_pow_two(self): + def main(n): + s = 0.123 + while n > 0: + s -= s ** 2 # ID: pow + n -= 1 + return s + log = self.run(main, [500]) + assert abs(log.result - main(500)) < 1e-9 + loop, = log.loops_by_filename(self.filepath) + assert loop.match_by_id("pow", """ + guard_not_invalidated(descr=...) + f38 = float_mul(f30, f30) + f39 = float_sub(f30, f38) + """) diff --git a/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py --- a/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py +++ b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py @@ -18,8 +18,8 @@ from numpypy import array, arange, argmin a = arange(6).reshape((2,3)) assert argmin(a) == 0 - assert (argmin(a, axis=0) == array([0, 0, 0])).all() - assert (argmin(a, axis=1) == array([0, 0])).all() + #assert (argmin(a, axis=0) == array([0, 0, 0])).all() + #assert (argmin(a, axis=1) == array([0, 0])).all() b = arange(6) b[1] = 0 assert argmin(b) == 0 @@ -40,8 +40,8 @@ assert sum([0.5, 1.5])== 2.0 assert sum([[0, 1], [0, 5]]) == 6 # assert sum([0.5, 0.7, 0.2, 1.5], dtype=int32) == 1 - # assert (sum([[0, 1], [0, 5]], axis=0) == array([0, 6])).all() - # assert (sum([[0, 1], [0, 5]], axis=1) == array([1, 5])).all() + assert (sum([[0, 1], [0, 5]], axis=0) == array([0, 6])).all() + assert (sum([[0, 1], [0, 5]], axis=1) == array([1, 5])).all() # If the accumulator is too small, overflow occurs: # assert ones(128, dtype=int8).sum(dtype=int8) == -128 @@ -98,20 +98,22 @@ from numpypy import array, var a = array([[1,2],[3,4]]) assert var(a) == 1.25 - # assert (np.var(a,0) == array([ 1., 1.])).all() - # assert (np.var(a,1) == array([ 0.25, 0.25])).all() + assert (var(a,0) == array([ 1., 1.])).all() + assert (var(a,1) == array([ 0.25, 0.25])).all() def test_std(self): from numpypy import array, std a = array([[1, 2], [3, 4]]) assert std(a) == 1.1180339887498949 - # assert (std(a, axis=0) == array([ 1., 1.])).all() - # assert (std(a, axis=1) == array([ 0.5, 0.5]).all() + assert (std(a, axis=0) == array([ 1., 1.])).all() + assert (std(a, axis=1) == array([ 0.5, 0.5])).all() def test_mean(self): - from numpypy import array, mean + from numpypy import array, mean, arange assert mean(array(range(5))) == 2.0 assert mean(range(5)) == 2.0 + assert (mean(arange(10).reshape(5, 2), axis=0) == [4, 5]).all() + assert (mean(arange(10).reshape(5, 2), axis=1) == [0.5, 2.5, 4.5, 6.5, 8.5]).all() def test_reshape(self): from numpypy import arange, array, dtype, reshape @@ -134,4 +136,4 @@ raises(NotImplementedError, "transpose(x, axes=(1, 0, 2))") # x = ones((1, 2, 3)) # assert transpose(x, (1, 0, 2)).shape == (2, 1, 3) - + diff --git a/pypy/module/test_lib_pypy/numpypy/core/test_numeric.py b/pypy/module/test_lib_pypy/numpypy/core/test_numeric.py new file mode 100644 --- /dev/null +++ b/pypy/module/test_lib_pypy/numpypy/core/test_numeric.py @@ -0,0 +1,144 @@ + +from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest + + +class AppTestBaseRepr(BaseNumpyAppTest): + def test_base3(self): + from numpypy import base_repr + assert base_repr(3**5, 3) == '100000' + + def test_positive(self): + from numpypy import base_repr + assert base_repr(12, 10) == '12' + assert base_repr(12, 10, 4) == '000012' + assert base_repr(12, 4) == '30' + assert base_repr(3731624803700888, 36) == '10QR0ROFCEW' + + def test_negative(self): + from numpypy import base_repr + assert base_repr(-12, 10) == '-12' + assert base_repr(-12, 10, 4) == '-000012' + assert base_repr(-12, 4) == '-30' + +class AppTestRepr(BaseNumpyAppTest): + def test_repr(self): + from numpypy import array + assert repr(array([1, 2, 3, 4])) == 'array([1, 2, 3, 4])' + + def test_repr_2(self): + from numpypy import array, zeros + int_size = array(5).dtype.itemsize + a = array(range(5), float) + assert repr(a) == "array([ 0., 1., 2., 3., 4.])" + a = array([], float) + assert repr(a) == "array([], dtype=float64)" + a = zeros(1001) + assert repr(a) == "array([ 0., 0., 0., ..., 0., 0., 0.])" + a = array(range(5), long) + if a.dtype.itemsize == int_size: + assert repr(a) == "array([0, 1, 2, 3, 4])" + else: + assert repr(a) == "array([0, 1, 2, 3, 4], dtype=int64)" + a = array(range(5), 'int32') + if a.dtype.itemsize == int_size: + assert repr(a) == "array([0, 1, 2, 3, 4])" + else: + assert repr(a) == "array([0, 1, 2, 3, 4], dtype=int32)" + a = array([], long) + assert repr(a) == "array([], dtype=int64)" + a = array([True, False, True, False], "?") + assert repr(a) == "array([ True, False, True, False], dtype=bool)" + a = zeros([]) + assert repr(a) == "array(0.0)" + a = array(0.2) + assert repr(a) == "array(0.2)" + a = array([2]) + assert repr(a) == "array([2])" + + def test_repr_multi(self): + from numpypy import arange, zeros, array + a = zeros((3, 4)) + assert repr(a) == '''array([[ 0., 0., 0., 0.], + [ 0., 0., 0., 0.], + [ 0., 0., 0., 0.]])''' + a = zeros((2, 3, 4)) + assert repr(a) == '''array([[[ 0., 0., 0., 0.], + [ 0., 0., 0., 0.], + [ 0., 0., 0., 0.]], + + [[ 0., 0., 0., 0.], + [ 0., 0., 0., 0.], + [ 0., 0., 0., 0.]]])''' + a = arange(1002).reshape((2, 501)) + assert repr(a) == '''array([[ 0, 1, 2, ..., 498, 499, 500], + [ 501, 502, 503, ..., 999, 1000, 1001]])''' + assert repr(a.T) == '''array([[ 0, 501], + [ 1, 502], + [ 2, 503], + ..., + [ 498, 999], + [ 499, 1000], + [ 500, 1001]])''' + a = arange(2).reshape((2,1)) + assert repr(a) == '''array([[0], + [1]])''' + + def test_repr_slice(self): + from numpypy import array, zeros + a = array(range(5), float) + b = a[1::2] + assert repr(b) == "array([ 1., 3.])" + a = zeros(2002) + b = a[::2] + assert repr(b) == "array([ 0., 0., 0., ..., 0., 0., 0.])" + a = array((range(5), range(5, 10)), dtype="int16") + b = a[1, 2:] + assert repr(b) == "array([7, 8, 9], dtype=int16)" + # an empty slice prints its shape + b = a[2:1, ] + assert repr(b) == "array([], shape=(0, 5), dtype=int16)" + + def test_str(self): + from numpypy import array, zeros + a = array(range(5), float) + assert str(a) == "[ 0. 1. 2. 3. 4.]" + assert str((2 * a)[:]) == "[ 0. 2. 4. 6. 8.]" + a = zeros(1001) + assert str(a) == "[ 0. 0. 0. ..., 0. 0. 0.]" + + a = array(range(5), dtype=long) + assert str(a) == "[0 1 2 3 4]" + a = array([True, False, True, False], dtype="?") + assert str(a) == "[ True False True False]" + + a = array(range(5), dtype="int8") + assert str(a) == "[0 1 2 3 4]" + + a = array(range(5), dtype="int16") + assert str(a) == "[0 1 2 3 4]" + + a = array((range(5), range(5, 10)), dtype="int16") + assert str(a) == "[[0 1 2 3 4]\n [5 6 7 8 9]]" + + a = array(3, dtype=int) + assert str(a) == "3" + + a = zeros((400, 400), dtype=int) + assert str(a) == '[[0 0 0 ..., 0 0 0]\n [0 0 0 ..., 0 0 0]\n [0 0 0 ..., 0 0 0]\n ..., \n [0 0 0 ..., 0 0 0]\n [0 0 0 ..., 0 0 0]\n [0 0 0 ..., 0 0 0]]' + a = zeros((2, 2, 2)) + r = str(a) + assert r == '[[[ 0. 0.]\n [ 0. 0.]]\n\n [[ 0. 0.]\n [ 0. 0.]]]' + + def test_str_slice(self): + from numpypy import array, zeros + a = array(range(5), float) + b = a[1::2] + assert str(b) == "[ 1. 3.]" + a = zeros(2002) + b = a[::2] + assert str(b) == "[ 0. 0. 0. ..., 0. 0. 0.]" + a = array((range(5), range(5, 10)), dtype="int16") + b = a[1, 2:] + assert str(b) == "[7 8 9]" + b = a[2:1, ] + assert str(b) == "[]" diff --git a/pypy/module/test_lib_pypy/test_datetime.py b/pypy/module/test_lib_pypy/test_datetime.py new file mode 100644 --- /dev/null +++ b/pypy/module/test_lib_pypy/test_datetime.py @@ -0,0 +1,24 @@ +"""Additional tests for datetime.""" + +import time +import datetime +import os + +def test_utcfromtimestamp(): + """Confirm that utcfromtimestamp and fromtimestamp give consistent results. + + Based on danchr's test script in https://bugs.pypy.org/issue986 + """ + try: + prev_tz = os.environ.get("TZ") + os.environ["TZ"] = "GMT" + for unused in xrange(100): + now = time.time() + delta = (datetime.datetime.utcfromtimestamp(now) - + datetime.datetime.fromtimestamp(now)) + assert delta.days * 86400 + delta.seconds == 0 + finally: + if prev_tz is None: + del os.environ["TZ"] + else: + os.environ["TZ"] = prev_tz diff --git a/pypy/module/test_lib_pypy/test_sqlite3.py b/pypy/module/test_lib_pypy/test_sqlite3.py new file mode 100644 --- /dev/null +++ b/pypy/module/test_lib_pypy/test_sqlite3.py @@ -0,0 +1,10 @@ +"""Tests for _sqlite3.py""" + +def test_list_ddl(): + """From issue996. Mostly just looking for lack of exceptions.""" + from sqlite3.dbapi2 import connect + connection = connect(':memory:') + cursor = connection.cursor() + cursor.execute('CREATE TABLE foo (bar INTEGER)') + result = list(cursor) + assert result == [] diff --git a/pypy/module/thread/ll_thread.py b/pypy/module/thread/ll_thread.py --- a/pypy/module/thread/ll_thread.py +++ b/pypy/module/thread/ll_thread.py @@ -4,7 +4,7 @@ import py from pypy.rlib import jit, rgc from pypy.rlib.debug import ll_assert -from pypy.rlib.objectmodel import we_are_translated +from pypy.rlib.objectmodel import we_are_translated, specialize from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.tool import rffi_platform from pypy.tool import autopath @@ -89,6 +89,7 @@ def allocate_lock(): return Lock(allocate_ll_lock()) + at specialize.arg(0) def ll_start_new_thread(func): ident = c_thread_start(func) if ident == -1: @@ -101,6 +102,7 @@ def get_ident(): return rffi.cast(lltype.Signed, c_thread_get_ident()) + at specialize.arg(0) def start_new_thread(x, y): """In RPython, no argument can be passed. You have to use global variables to pass information to the new thread. That's not very diff --git a/pypy/module/thread/test/test_ll_thread.py b/pypy/module/thread/test/test_ll_thread.py --- a/pypy/module/thread/test/test_ll_thread.py +++ b/pypy/module/thread/test/test_ll_thread.py @@ -34,72 +34,34 @@ use_threads = True def test_start_new_thread(self): - py.test.skip("xxx ideally, investigate why it fails randomly") - # xxx but in practice start_new_thread() is also tested by the - # next test, and it's a mess to test start_new_thread() without - # the proper GIL to protect the GC import time class State: pass state = State() - class Z: - def __init__(self, value): - self.value = value - def __del__(self): - state.freed_counter += 1 - - def bootstrap(): - state.my_thread_ident = state.z.ident = get_ident() - assert state.my_thread_ident == get_ident() - assert get_ident() == state.z.ident - state.seen_value = state.z.value - state.z = None - # I think that we would need here a memory barrier in order - # to make the test pass reliably. The issue is that the - # main thread may see 'state.done = 1' before seeing the - # effect of the other assignments done above. For now let's - # emulate the write barrier by doing a system call and - # waiting a bit... - time.sleep(0.012) - state.done = 1 - - def g(i): - state.z = Z(i) - return start_new_thread(bootstrap, ()) - g._dont_inline_ = True + def bootstrap1(): + state.my_thread_ident1 = get_ident() + def bootstrap2(): + state.my_thread_ident2 = get_ident() def f(): - main_ident = get_ident() - assert main_ident == get_ident() - state.freed_counter = 0 - for i in range(50): - state.done = 0 - state.seen_value = 0 - ident = g(i) - gc.collect() - willing_to_wait_more = 1000 - while not state.done: - willing_to_wait_more -= 1 - if not willing_to_wait_more: - raise Exception("thread didn't start?") - time.sleep(0.01) - assert state.my_thread_ident != main_ident - assert state.my_thread_ident == ident - assert state.seen_value == i - # try to force Boehm to do some freeing - for i in range(3): - gc.collect() - return state.freed_counter + state.my_thread_ident1 = get_ident() + state.my_thread_ident2 = get_ident() + start_new_thread(bootstrap1, ()) + start_new_thread(bootstrap2, ()) + willing_to_wait_more = 1000 + while (state.my_thread_ident1 == get_ident() or + state.my_thread_ident2 == get_ident()): + willing_to_wait_more -= 1 + if not willing_to_wait_more: + raise Exception("thread didn't start?") + time.sleep(0.01) + return 42 fn = self.getcompiled(f, []) - freed_counter = fn() - print freed_counter - if self.gcpolicy == 'boehm': - assert freed_counter > 0 - else: - assert freed_counter == 50 + res = fn() + assert res == 42 def test_gc_locking(self): import time @@ -202,6 +164,9 @@ def getcompiled(self, f, argtypes): return f + def test_start_new_thread(self): + py.test.skip("deadlocks occasionally -- why???") + class TestUsingBoehm(AbstractThreadTests): gcpolicy = 'boehm' diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py --- a/pypy/objspace/fake/objspace.py +++ b/pypy/objspace/fake/objspace.py @@ -150,6 +150,8 @@ self._see_getsetproperty(x) if isinstance(x, r_singlefloat): self._wrap_not_rpython(x) + if isinstance(x, list): + self._wrap_not_rpython(x) return w_some_obj() wrap._annspecialcase_ = "specialize:argtype(1)" diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py --- a/pypy/objspace/std/bytearrayobject.py +++ b/pypy/objspace/std/bytearrayobject.py @@ -565,8 +565,8 @@ if step == 1: assert start >= 0 - assert slicelength >= 0 - del items[start:start+slicelength] + if slicelength > 0: + del items[start:start+slicelength] else: n = len(items) i = start @@ -603,10 +603,11 @@ while i >= lim: items[i] = items[i-delta] i -= 1 - elif start >= 0: + elif delta == 0: + pass + else: + assert start >= 0 # start<0 is only possible with slicelength==0 del items[start:start+delta] - else: - assert delta==0 # start<0 is only possible with slicelength==0 elif len2 != slicelength: # No resize for extended slices raise operationerrfmt(space.w_ValueError, "attempt to " "assign sequence of size %d to extended slice of size %d", diff --git a/pypy/objspace/std/complexobject.py b/pypy/objspace/std/complexobject.py --- a/pypy/objspace/std/complexobject.py +++ b/pypy/objspace/std/complexobject.py @@ -9,6 +9,7 @@ from pypy.rlib.rfloat import ( formatd, DTSF_STR_PRECISION, isinf, isnan, copysign) from pypy.rlib.objectmodel import HASH_IMAG +from pypy.rlib import jit import math @@ -119,10 +120,10 @@ ir = len * math.sin(phase) return W_ComplexObject(rr, ir) - def pow_int(self, n): - if n > 100 or n < -100: - return self.pow(W_ComplexObject(1.0 * n, 0.0)) - elif n > 0: + def pow_small_int(self, n): + if n >= 0: + if jit.isconstant(n) and n == 2: + return self.mul(self) return self.pow_positive_int(n) else: return w_one.div(self.pow_positive_int(-n)) @@ -187,10 +188,10 @@ def pow__Complex_Complex_ANY(space, w_complex, w_exponent, thirdArg): if not space.is_w(thirdArg, space.w_None): raise OperationError(space.w_ValueError, space.wrap('complex modulo')) - int_exponent = int(w_exponent.realval) try: - if w_exponent.imagval == 0.0 and w_exponent.realval == int_exponent: - w_p = w_complex.pow_int(int_exponent) + r = w_exponent.realval + if w_exponent.imagval == 0.0 and -100.0 <= r <= 100.0 and r == int(r): + w_p = w_complex.pow_small_int(int(r)) else: w_p = w_complex.pow(w_exponent) except ZeroDivisionError: diff --git a/pypy/objspace/std/floatobject.py b/pypy/objspace/std/floatobject.py --- a/pypy/objspace/std/floatobject.py +++ b/pypy/objspace/std/floatobject.py @@ -442,6 +442,8 @@ y = w_float2.floatval # Sort out special cases here instead of relying on pow() + if y == 2.0: # special case for performance: + return W_FloatObject(x * x) # x * x is always correct if y == 0.0: # x**0 is 1, even 0**0 return W_FloatObject(1.0) diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py --- a/pypy/objspace/std/listobject.py +++ b/pypy/objspace/std/listobject.py @@ -110,7 +110,10 @@ return list(items) def switch_to_object_strategy(self): - list_w = self.getitems() + if self.strategy is self.space.fromcache(EmptyListStrategy): + list_w = [] + else: + list_w = self.getitems() self.strategy = self.space.fromcache(ObjectListStrategy) # XXX this is quite indirect self.init_from_list_w(list_w) @@ -344,8 +347,6 @@ def __init__(self, space): ListStrategy.__init__(self, space) - # cache an empty list that is used whenever getitems is called (i.e. sorting) - self.cached_emptylist_w = [] def init_from_list_w(self, w_list, list_w): assert len(list_w) == 0 @@ -373,10 +374,10 @@ def getslice(self, w_list, start, stop, step, length): # will never be called because the empty list case is already caught in # getslice__List_ANY_ANY and getitem__List_Slice - return W_ListObject(self.space, self.cached_emptylist_w) + return W_ListObject(self.space, []) def getitems(self, w_list): - return self.cached_emptylist_w + return [] def getitems_copy(self, w_list): return [] @@ -804,10 +805,11 @@ while i >= lim: items[i] = items[i-delta] i -= 1 - elif start >= 0: + elif delta == 0: + pass + else: + assert start >= 0 # start<0 is only possible with slicelength==0 del items[start:start+delta] - else: - assert delta==0 # start<0 is only possible with slicelength==0 elif len2 != slicelength: # No resize for extended slices raise operationerrfmt(self.space.w_ValueError, "attempt to " "assign sequence of size %d to extended slice of size %d", @@ -851,8 +853,8 @@ if step == 1: assert start >= 0 - assert slicelength >= 0 - del items[start:start+slicelength] + if slicelength > 0: + del items[start:start+slicelength] else: n = len(items) i = start diff --git a/pypy/objspace/std/test/test_bytearrayobject.py b/pypy/objspace/std/test/test_bytearrayobject.py --- a/pypy/objspace/std/test/test_bytearrayobject.py +++ b/pypy/objspace/std/test/test_bytearrayobject.py @@ -1,5 +1,9 @@ +from pypy import conftest class AppTestBytesArray: + def setup_class(cls): + cls.w_runappdirect = cls.space.wrap(conftest.option.runappdirect) + def test_basics(self): b = bytearray() assert type(b) is bytearray @@ -434,3 +438,15 @@ def test_reduce(self): assert bytearray(b'caf\xe9').__reduce__() == ( bytearray, (u'caf\xe9', 'latin-1'), None) + + def test_setitem_slice_performance(self): + # because of a complexity bug, this used to take forever on a + # translated pypy. On CPython2.6 -A, it takes around 8 seconds. + if self.runappdirect: + count = 16*1024*1024 + else: + count = 1024 + b = bytearray(count) + for i in range(count): + b[i:i+1] = 'y' + assert str(b) == 'y' * count diff --git a/pypy/objspace/std/test/test_complexobject.py b/pypy/objspace/std/test/test_complexobject.py --- a/pypy/objspace/std/test/test_complexobject.py +++ b/pypy/objspace/std/test/test_complexobject.py @@ -74,7 +74,7 @@ assert _powu((0.0,1.0),2) == (-1.0,0.0) def _powi((r1, i1), n): - w_res = W_ComplexObject(r1, i1).pow_int(n) + w_res = W_ComplexObject(r1, i1).pow_small_int(n) return w_res.realval, w_res.imagval assert _powi((0.0,2.0),0) == (1.0,0.0) assert _powi((0.0,0.0),2) == (0.0,0.0) @@ -213,6 +213,7 @@ assert a ** 105 == a ** 105 assert a ** -105 == a ** -105 assert a ** -30 == a ** -30 + assert a ** 2 == a * a assert 0.0j ** 0 == 1 diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py --- a/pypy/objspace/std/test/test_listobject.py +++ b/pypy/objspace/std/test/test_listobject.py @@ -397,6 +397,7 @@ on_cpython = (option.runappdirect and not hasattr(sys, 'pypy_translation_info')) cls.w_on_cpython = cls.space.wrap(on_cpython) + cls.w_runappdirect = cls.space.wrap(option.runappdirect) def test_getstrategyfromlist_w(self): l0 = ["a", "2", "a", True] @@ -885,6 +886,18 @@ l[::-1] = l assert l == [6,5,4,3,2,1] + def test_setitem_slice_performance(self): + # because of a complexity bug, this used to take forever on a + # translated pypy. On CPython2.6 -A, it takes around 5 seconds. + if self.runappdirect: + count = 16*1024*1024 + else: + count = 1024 + b = [None] * count + for i in range(count): + b[i:i+1] = ['y'] + assert b == ['y'] * count + def test_recursive_repr(self): l = [] assert repr(l) == '[]' @@ -1113,6 +1126,21 @@ assert l == [] assert list(g) == [] +class AppTestWithoutStrategies(object): + + def setup_class(cls): + cls.space = gettestobjspace(**{"objspace.std.withliststrategies" : + False}) + + def test_no_shared_empty_list(self): + l = [] + copy = l[:] + copy.append({}) + assert copy == [{}] + + notshared = l[:] + assert notshared == [] + class AppTestListFastSubscr: def setup_class(cls): diff --git a/pypy/rlib/objectmodel.py b/pypy/rlib/objectmodel.py --- a/pypy/rlib/objectmodel.py +++ b/pypy/rlib/objectmodel.py @@ -130,19 +130,29 @@ if self is other: return 0 else: - raise TypeError("Symbolics can not be compared!") + raise TypeError("Symbolics cannot be compared! (%r, %r)" + % (self, other)) def __hash__(self): - raise TypeError("Symbolics are not hashable!") + raise TypeError("Symbolics are not hashable! %r" % (self,)) def __nonzero__(self): - raise TypeError("Symbolics are not comparable") + raise TypeError("Symbolics are not comparable! %r" % (self,)) class ComputedIntSymbolic(Symbolic): def __init__(self, compute_fn): self.compute_fn = compute_fn + def __repr__(self): + # repr(self.compute_fn) can arrive back here in an + # infinite recursion + try: + name = self.compute_fn.__name__ + except (AttributeError, TypeError): + name = hex(id(self.compute_fn)) + return '%s(%r)' % (self.__class__.__name__, name) + def annotation(self): from pypy.annotation import model return model.SomeInteger() @@ -157,6 +167,9 @@ self.expr = expr self.default = default + def __repr__(self): + return '%s(%r)' % (self.__class__.__name__, self.expr) + def annotation(self): from pypy.annotation import model return model.SomeInteger() diff --git a/pypy/rlib/rdynload.py b/pypy/rlib/rdynload.py --- a/pypy/rlib/rdynload.py +++ b/pypy/rlib/rdynload.py @@ -14,7 +14,7 @@ _MINGW = platform.name == "mingw32" _WIN32 = _MSVC or _MINGW _MAC_OS = platform.name == "darwin" -_FREEBSD = platform.name == "freebsd" +_FREEBSD = sys.platform.startswith("freebsd") if _WIN32: from pypy.rlib import rwin32 diff --git a/pypy/rlib/ropenssl.py b/pypy/rlib/ropenssl.py --- a/pypy/rlib/ropenssl.py +++ b/pypy/rlib/ropenssl.py @@ -70,6 +70,8 @@ SSL_OP_NO_SSLv2 = rffi_platform.ConstantInteger("SSL_OP_NO_SSLv2") SSL_OP_NO_SSLv3 = rffi_platform.ConstantInteger("SSL_OP_NO_SSLv3") SSL_OP_NO_TLSv1 = rffi_platform.ConstantInteger("SSL_OP_NO_TLSv1") + SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS = rffi_platform.ConstantInteger( + "SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS") HAS_SNI = rffi_platform.Defined("SSL_CTRL_SET_TLSEXT_HOSTNAME") SSL_VERIFY_NONE = rffi_platform.ConstantInteger("SSL_VERIFY_NONE") SSL_VERIFY_PEER = rffi_platform.ConstantInteger("SSL_VERIFY_PEER") diff --git a/pypy/rlib/rsocket.py b/pypy/rlib/rsocket.py --- a/pypy/rlib/rsocket.py +++ b/pypy/rlib/rsocket.py @@ -310,10 +310,7 @@ def from_object(space, w_address): # Parse an app-level object representing an AF_INET address - try: - w_host, w_port = space.unpackiterable(w_address, 2) - except ValueError: - raise TypeError("AF_INET address must be a tuple of length 2") + w_host, w_port = space.unpackiterable(w_address, 2) host = space.str_w(w_host) port = space.int_w(w_port) port = Address.make_ushort_port(space, port) @@ -549,10 +546,7 @@ space.wrap(self.get_groups())]) def from_object(space, w_address): - try: - w_pid, w_groups = space.unpackiterable(w_address, 2) - except ValueError: - raise TypeError("AF_NETLINK address must be a tuple of length 2") + w_pid, w_groups = space.unpackiterable(w_address, 2) return NETLINKAddress(space.uint_w(w_pid), space.uint_w(w_groups)) from_object = staticmethod(from_object) diff --git a/pypy/rlib/rstacklet.py b/pypy/rlib/rstacklet.py --- a/pypy/rlib/rstacklet.py +++ b/pypy/rlib/rstacklet.py @@ -1,3 +1,4 @@ +import sys from pypy.rlib import _rffi_stacklet as _c from pypy.rlib import jit from pypy.rlib.objectmodel import we_are_translated @@ -72,6 +73,11 @@ if translated: assert config is not None, ("you have to pass a valid config, " "e.g. from 'driver.config'") + elif '__pypy__' in sys.builtin_module_names: + import py + py.test.skip("cannot run the stacklet tests on top of pypy: " + "calling directly the C function stacklet_switch() " + "will crash, depending on details of your config") if config is not None: assert config.translation.continuation, ( "stacklet: you have to translate with --continuation") diff --git a/pypy/rlib/rweakref.py b/pypy/rlib/rweakref.py --- a/pypy/rlib/rweakref.py +++ b/pypy/rlib/rweakref.py @@ -1,11 +1,14 @@ """ -Weakref support in RPython. Supports ref() without callbacks, +Weakref support in RPython. Basic regular weakrefs without callbacks +are supported. This file contains the following additions: a form of WeakKeyDictionary, and a limited version of WeakValueDictionary. LLType only for now! """ import weakref +ref = weakref.ref # basic regular weakrefs are supported in RPython + class RWeakValueDictionary(object): """A dictionary containing weak values.""" diff --git a/pypy/rpython/lltypesystem/ll2ctypes.py b/pypy/rpython/lltypesystem/ll2ctypes.py --- a/pypy/rpython/lltypesystem/ll2ctypes.py +++ b/pypy/rpython/lltypesystem/ll2ctypes.py @@ -1390,7 +1390,8 @@ def _where_is_errno(): return standard_c_lib.__errno_location() - elif sys.platform in ('darwin', 'freebsd7', 'freebsd8', 'freebsd9'): + elif any(plat in sys.platform + for plat in ('darwin', 'freebsd7', 'freebsd8', 'freebsd9')): standard_c_lib.__error.restype = ctypes.POINTER(ctypes.c_int) def _where_is_errno(): return standard_c_lib.__error() diff --git a/pypy/rpython/lltypesystem/llgroup.py b/pypy/rpython/lltypesystem/llgroup.py --- a/pypy/rpython/lltypesystem/llgroup.py +++ b/pypy/rpython/lltypesystem/llgroup.py @@ -76,6 +76,10 @@ self.index = memberindex self.member = grp.members[memberindex]._as_ptr() + def __repr__(self): + return '%s(%s, %s)' % (self.__class__.__name__, + self.grpptr, self.index) + def __nonzero__(self): return True diff --git a/pypy/rpython/lltypesystem/llmemory.py b/pypy/rpython/lltypesystem/llmemory.py --- a/pypy/rpython/lltypesystem/llmemory.py +++ b/pypy/rpython/lltypesystem/llmemory.py @@ -32,7 +32,8 @@ self.known_nonneg()): return True else: - raise TypeError("Symbolics can not be compared!") + raise TypeError("Symbolics cannot be compared! (%r, %r)" + % (self, other)) def __lt__(self, other): return not self.__ge__(other) diff --git a/pypy/rpython/lltypesystem/rdict.py b/pypy/rpython/lltypesystem/rdict.py --- a/pypy/rpython/lltypesystem/rdict.py +++ b/pypy/rpython/lltypesystem/rdict.py @@ -330,6 +330,17 @@ hop.exception_is_here() return hop.gendirectcall(ll_popitem, cTUPLE, v_dict) + def rtype_method_pop(self, hop): + if hop.nb_args == 2: + v_args = hop.inputargs(self, self.key_repr) + target = ll_pop + elif hop.nb_args == 3: + v_args = hop.inputargs(self, self.key_repr, self.value_repr) + target = ll_pop_default + hop.exception_is_here() + v_res = hop.gendirectcall(target, *v_args) + return self.recast_value(hop.llops, v_res) + class __extend__(pairtype(DictRepr, rmodel.Repr)): def rtype_getitem((r_dict, r_key), hop): @@ -890,3 +901,18 @@ r.item1 = recast(ELEM.TO.item1, entry.value) _ll_dict_del(dic, i) return r + +def ll_pop(dic, key): + i = ll_dict_lookup(dic, key, dic.keyhash(key)) + if not i & HIGHEST_BIT: + value = ll_get_value(dic, i) + _ll_dict_del(dic, i) + return value + else: + raise KeyError + +def ll_pop_default(dic, key, dfl): + try: + return ll_pop(dic, key) + except KeyError: + return dfl diff --git a/pypy/rpython/lltypesystem/rffi.py b/pypy/rpython/lltypesystem/rffi.py --- a/pypy/rpython/lltypesystem/rffi.py +++ b/pypy/rpython/lltypesystem/rffi.py @@ -27,6 +27,10 @@ self.c_name = c_name self.TP = TP + def __repr__(self): + return '%s(%r, %s)' % (self.__class__.__name__, + self.c_name, self.TP) + def annotation(self): return lltype_to_annotation(self.TP) @@ -755,6 +759,7 @@ alloc_buffer._annenforceargs_ = [int] # (char*, str, int, int) -> None + @jit.dont_look_inside def str_from_buffer(raw_buf, gc_buf, allocated_size, needed_size): """ Converts from a pair returned by alloc_buffer to a high-level string. @@ -780,6 +785,7 @@ return hlstrtype(new_buf) # (char*, str) -> None + @jit.dont_look_inside def keep_buffer_alive_until_here(raw_buf, gc_buf): """ Keeps buffers alive or frees temporary buffers created by alloc_buffer. diff --git a/pypy/rpython/memory/gc/env.py b/pypy/rpython/memory/gc/env.py --- a/pypy/rpython/memory/gc/env.py +++ b/pypy/rpython/memory/gc/env.py @@ -116,7 +116,7 @@ def get_total_memory(): return get_total_memory_darwin(get_darwin_sysctl_signed('hw.memsize')) -elif sys.platform.startswith('freebsd'): +elif 'freebsd' in sys.platform: def get_total_memory(): return get_total_memory_darwin(get_darwin_sysctl_signed('hw.usermem')) diff --git a/pypy/rpython/memory/gctransform/shadowstack.py b/pypy/rpython/memory/gctransform/shadowstack.py --- a/pypy/rpython/memory/gctransform/shadowstack.py +++ b/pypy/rpython/memory/gctransform/shadowstack.py @@ -27,28 +27,17 @@ return top self.decr_stack = decr_stack - translator = gctransformer.translator - if (hasattr(translator, '_jit2gc') and - 'root_iterator' in translator._jit2gc): - root_iterator = translator._jit2gc['root_iterator'] - def jit_walk_stack_root(callback, addr, end): - root_iterator.context = NonConstant(llmemory.NULL) - gc = self.gc - while True: - addr = root_iterator.next(gc, addr, end) - if addr == llmemory.NULL: - return - callback(gc, addr) - addr += sizeofaddr - self.rootstackhook = jit_walk_stack_root - else: - def default_walk_stack_root(callback, addr, end): - gc = self.gc - while addr != end: - if gc.points_to_valid_gc_object(addr): - callback(gc, addr) - addr += sizeofaddr - self.rootstackhook = default_walk_stack_root + root_iterator = get_root_iterator(gctransformer) + def walk_stack_root(callback, start, end): + root_iterator.setcontext(NonConstant(llmemory.NULL)) + gc = self.gc + addr = end + while True: + addr = root_iterator.nextleft(gc, start, addr) + if addr == llmemory.NULL: + return + callback(gc, addr) + self.rootstackhook = walk_stack_root self.shadow_stack_pool = ShadowStackPool(gcdata) rsd = gctransformer.root_stack_depth @@ -331,6 +320,30 @@ raise MemoryError +def get_root_iterator(gctransformer): + if hasattr(gctransformer, '_root_iterator'): + return gctransformer._root_iterator # if already built + translator = gctransformer.translator + if (hasattr(translator, '_jit2gc') and + 'root_iterator' in translator._jit2gc): + result = translator._jit2gc['root_iterator'] + else: + class RootIterator(object): + def _freeze_(self): + return True + def setcontext(self, context): + pass + def nextleft(self, gc, start, addr): + while addr != start: + addr -= sizeofaddr + if gc.points_to_valid_gc_object(addr): + return addr + return llmemory.NULL + result = RootIterator() + gctransformer._root_iterator = result + return result + + def get_shadowstackref(gctransformer): if hasattr(gctransformer, '_SHADOWSTACKREF'): return gctransformer._SHADOWSTACKREF @@ -344,28 +357,15 @@ rtti=True) SHADOWSTACKREFPTR.TO.become(SHADOWSTACKREF) - translator = gctransformer.translator - if hasattr(translator, '_jit2gc'): - gc = gctransformer.gcdata.gc - root_iterator = translator._jit2gc['root_iterator'] - def customtrace(obj, prev): - obj = llmemory.cast_adr_to_ptr(obj, SHADOWSTACKREFPTR) - if not prev: - root_iterator.context = obj.context - next = obj.base - else: - next = prev + sizeofaddr - return root_iterator.next(gc, next, obj.top) - else: - def customtrace(obj, prev): - # a simple but not JIT-ready version - if not prev: - next = llmemory.cast_adr_to_ptr(obj, SHADOWSTACKREFPTR).base - else: - next = prev + sizeofaddr - if next == llmemory.cast_adr_to_ptr(obj, SHADOWSTACKREFPTR).top: - next = llmemory.NULL - return next + gc = gctransformer.gcdata.gc + root_iterator = get_root_iterator(gctransformer) + + def customtrace(obj, prev): + obj = llmemory.cast_adr_to_ptr(obj, SHADOWSTACKREFPTR) + if not prev: + root_iterator.setcontext(obj.context) + prev = obj.top + return root_iterator.nextleft(gc, obj.base, prev) CUSTOMTRACEFUNC = lltype.FuncType([llmemory.Address, llmemory.Address], llmemory.Address) diff --git a/pypy/rpython/module/ll_time.py b/pypy/rpython/module/ll_time.py --- a/pypy/rpython/module/ll_time.py +++ b/pypy/rpython/module/ll_time.py @@ -41,7 +41,7 @@ RUSAGE = platform.Struct('struct rusage', [('ru_utime', TIMEVAL), ('ru_stime', TIMEVAL)]) -if "freebsd" in sys.platform: +if sys.platform.startswith('freebsd'): libraries = ['compat'] else: libraries = [] diff --git a/pypy/rpython/ootypesystem/rdict.py b/pypy/rpython/ootypesystem/rdict.py --- a/pypy/rpython/ootypesystem/rdict.py +++ b/pypy/rpython/ootypesystem/rdict.py @@ -167,6 +167,16 @@ hop.exception_is_here() return hop.gendirectcall(ll_popitem, cTUPLE, v_dict) + def rtype_method_pop(self, hop): + if hop.nb_args == 2: + v_args = hop.inputargs(self, self.key_repr) + target = ll_pop + elif hop.nb_args == 3: + v_args = hop.inputargs(self, self.key_repr, self.value_repr) + target = ll_pop_default + hop.exception_is_here() + return hop.gendirectcall(target, *v_args) + def __get_func(self, interp, r_func, fn, TYPE): if isinstance(r_func, MethodOfFrozenPBCRepr): obj = r_func.r_im_self.convert_const(fn.im_self) @@ -385,6 +395,20 @@ return res raise KeyError +def ll_pop(d, key): + if d.ll_contains(key): + value = d.ll_get(key) + d.ll_remove(key) + return value + else: + raise KeyError + +def ll_pop_default(d, key, dfl): + try: + return ll_pop(d, key) + except KeyError: + return dfl + # ____________________________________________________________ # # Iteration. diff --git a/pypy/rpython/test/test_rdict.py b/pypy/rpython/test/test_rdict.py --- a/pypy/rpython/test/test_rdict.py +++ b/pypy/rpython/test/test_rdict.py @@ -25,37 +25,37 @@ class BaseTestRdict(BaseRtypingTest): def test_dict_creation(self): - def createdict(i): + def createdict(i): d = {'hello' : i} return d['hello'] res = self.interpret(createdict, [42]) assert res == 42 - def test_dict_getitem_setitem(self): - def func(i): + def test_dict_getitem_setitem(self): + def func(i): d = {'hello' : i} d['world'] = i + 1 - return d['hello'] * d['world'] + return d['hello'] * d['world'] res = self.interpret(func, [6]) assert res == 42 - def test_dict_getitem_keyerror(self): - def func(i): + def test_dict_getitem_keyerror(self): + def func(i): d = {'hello' : i} try: return d['world'] except KeyError: - return 0 + return 0 res = self.interpret(func, [6]) assert res == 0 def test_dict_del_simple(self): - def func(i): + def func(i): d = {'hello' : i} d['world'] = i + 1 del d['hello'] - return len(d) + return len(d) res = self.interpret(func, [6]) assert res == 1 @@ -71,7 +71,7 @@ assert res == True def test_empty_strings(self): - def func(i): + def func(i): d = {'' : i} del d[''] try: @@ -83,7 +83,7 @@ res = self.interpret(func, [6]) assert res == 1 - def func(i): + def func(i): d = {'' : i} del d[''] d[''] = i + 1 @@ -146,8 +146,8 @@ d1 = {} d1['hello'] = i + 1 d2 = {} - d2['world'] = d1 - return d2['world']['hello'] + d2['world'] = d1 + return d2['world']['hello'] res = self.interpret(func, [5]) assert res == 6 @@ -307,7 +307,7 @@ a = 0 for k, v in items: b += isinstance(k, B) - a += isinstance(v, A) + a += isinstance(v, A) return 3*b+a res = self.interpret(func, []) assert res == 8 @@ -326,7 +326,7 @@ a = 0 for k, v in dic.iteritems(): b += isinstance(k, B) - a += isinstance(v, A) + a += isinstance(v, A) return 3*b+a res = self.interpret(func, []) assert res == 8 @@ -352,11 +352,11 @@ def test_dict_contains_with_constant_dict(self): dic = {'4':1000, ' 8':200} def func(i): - return chr(i) in dic - res = self.interpret(func, [ord('4')]) + return chr(i) in dic + res = self.interpret(func, [ord('4')]) assert res is True - res = self.interpret(func, [1]) - assert res is False + res = self.interpret(func, [1]) + assert res is False def test_dict_or_none(self): class A: @@ -423,7 +423,7 @@ return g(get) res = self.interpret(f, []) - assert res == 2 + assert res == 2 def test_specific_obscure_bug(self): class A: pass @@ -632,6 +632,42 @@ res = self.interpret(func, []) assert res in [5263, 6352] + def test_dict_pop(self): + def f(n, default): + d = {} + d[2] = 3 + d[4] = 5 + if default == -1: + try: + x = d.pop(n) + except KeyError: + x = -1 + else: + x = d.pop(n, default) + return x * 10 + len(d) + res = self.interpret(f, [2, -1]) + assert res == 31 + res = self.interpret(f, [3, -1]) + assert res == -8 + res = self.interpret(f, [2, 5]) + assert res == 31 + + def test_dict_pop_instance(self): + class A(object): + pass + def f(n): + d = {} + d[2] = A() + x = d.pop(n, None) + if x is None: + return 12 + else: + return 15 + res = self.interpret(f, [2]) + assert res == 15 + res = self.interpret(f, [700]) + assert res == 12 + class TestLLtype(BaseTestRdict, LLRtypeMixin): def test_dict_but_not_with_char_keys(self): def func(i): @@ -643,19 +679,19 @@ res = self.interpret(func, [6]) assert res == 0 - def test_deleted_entry_reusage_with_colliding_hashes(self): - def lowlevelhash(value): + def test_deleted_entry_reusage_with_colliding_hashes(self): + def lowlevelhash(value): p = rstr.mallocstr(len(value)) for i in range(len(value)): p.chars[i] = value[i] - return rstr.LLHelpers.ll_strhash(p) + return rstr.LLHelpers.ll_strhash(p) - def func(c1, c2): - c1 = chr(c1) - c2 = chr(c2) + def func(c1, c2): + c1 = chr(c1) + c2 = chr(c2) d = {} d[c1] = 1 - d[c2] = 2 + d[c2] = 2 del d[c1] return d[c2] @@ -663,7 +699,7 @@ base = rdict.DICT_INITSIZE for y in range(0, 256): y = chr(y) - y_hash = lowlevelhash(y) % base + y_hash = lowlevelhash(y) % base char_by_hash.setdefault(y_hash, []).append(y) x, y = char_by_hash[0][:2] # find a collision @@ -671,18 +707,18 @@ res = self.interpret(func, [ord(x), ord(y)]) assert res == 2 - def func2(c1, c2): - c1 = chr(c1) - c2 = chr(c2) + def func2(c1, c2): + c1 = chr(c1) + c2 = chr(c2) d = {} d[c1] = 1 - d[c2] = 2 + d[c2] = 2 del d[c1] d[c1] = 3 - return d + return d res = self.interpret(func2, [ord(x), ord(y)]) - for i in range(len(res.entries)): + for i in range(len(res.entries)): assert not (res.entries.everused(i) and not res.entries.valid(i)) def func3(c0, c1, c2, c3, c4, c5, c6, c7): @@ -697,9 +733,9 @@ c7 = chr(c7) ; d[c7] = 1; del d[c7] return d - if rdict.DICT_INITSIZE != 8: + if rdict.DICT_INITSIZE != 8: py.test.skip("make dict tests more indepdent from initsize") - res = self.interpret(func3, [ord(char_by_hash[i][0]) + res = self.interpret(func3, [ord(char_by_hash[i][0]) for i in range(rdict.DICT_INITSIZE)]) count_frees = 0 for i in range(len(res.entries)): @@ -717,9 +753,9 @@ del d[chr(ord('a') + i)] return d res = self.interpret(func, [0]) - assert len(res.entries) > rdict.DICT_INITSIZE + assert len(res.entries) > rdict.DICT_INITSIZE res = self.interpret(func, [1]) - assert len(res.entries) == rdict.DICT_INITSIZE + assert len(res.entries) == rdict.DICT_INITSIZE def test_dict_valid_resize(self): # see if we find our keys after resize @@ -854,7 +890,7 @@ def test_prebuilt_list_of_addresses(self): from pypy.rpython.lltypesystem import llmemory - + TP = lltype.Struct('x', ('y', lltype.Signed)) a = lltype.malloc(TP, flavor='raw', immortal=True) b = lltype.malloc(TP, flavor='raw', immortal=True) @@ -868,7 +904,7 @@ d = {a_a: 3, a_b: 4, a_c: 5} d[a0] = 8 - + def func(i): if i == 0: ptr = a @@ -898,7 +934,7 @@ return a == b def rhash(a): return 3 - + def func(i): d = r_dict(eq, rhash, force_non_null=True) if not i: diff --git a/pypy/translator/c/src/profiling.c b/pypy/translator/c/src/profiling.c --- a/pypy/translator/c/src/profiling.c +++ b/pypy/translator/c/src/profiling.c @@ -29,33 +29,33 @@ profiling_setup = 0; } } - -#elif defined(_WIN32) -#include <windows.h> - -DWORD_PTR base_affinity_mask; -int profiling_setup = 0; - -void pypy_setup_profiling() { - if (!profiling_setup) { - DWORD_PTR affinity_mask, system_affinity_mask; - GetProcessAffinityMask(GetCurrentProcess(), - &base_affinity_mask, &system_affinity_mask); - affinity_mask = 1; - /* Pick one cpu allowed by the system */ - if (system_affinity_mask) - while ((affinity_mask & system_affinity_mask) == 0) - affinity_mask <<= 1; - SetProcessAffinityMask(GetCurrentProcess(), affinity_mask); - profiling_setup = 1; - } -} - -void pypy_teardown_profiling() { - if (profiling_setup) { - SetProcessAffinityMask(GetCurrentProcess(), base_affinity_mask); - profiling_setup = 0; - } + +#elif defined(_WIN32) +#include <windows.h> + +DWORD_PTR base_affinity_mask; +int profiling_setup = 0; + +void pypy_setup_profiling() { + if (!profiling_setup) { + DWORD_PTR affinity_mask, system_affinity_mask; + GetProcessAffinityMask(GetCurrentProcess(), + &base_affinity_mask, &system_affinity_mask); + affinity_mask = 1; + /* Pick one cpu allowed by the system */ + if (system_affinity_mask) + while ((affinity_mask & system_affinity_mask) == 0) + affinity_mask <<= 1; + SetProcessAffinityMask(GetCurrentProcess(), affinity_mask); + profiling_setup = 1; + } +} + +void pypy_teardown_profiling() { + if (profiling_setup) { + SetProcessAffinityMask(GetCurrentProcess(), base_affinity_mask); + profiling_setup = 0; + } } #else diff --git a/pypy/translator/goal/targetnopstandalone.py b/pypy/translator/goal/targetnopstandalone.py --- a/pypy/translator/goal/targetnopstandalone.py +++ b/pypy/translator/goal/targetnopstandalone.py @@ -7,10 +7,8 @@ actually implementing argv of the executable. """ -import os, sys - -def debug(msg): - os.write(2, "debug: " + msg + '\n') +def debug(msg): + print "debug:", msg # __________ Entry point __________ diff --git a/pypy/translator/goal/translate.py b/pypy/translator/goal/translate.py --- a/pypy/translator/goal/translate.py +++ b/pypy/translator/goal/translate.py @@ -293,17 +293,18 @@ drv.exe_name = targetspec_dic['__name__'] + '-%(backend)s' # Double check to ensure we are not overwriting the current interpreter - try: - this_exe = py.path.local(sys.executable).new(ext='') - exe_name = drv.compute_exe_name() - samefile = this_exe.samefile(exe_name) - assert not samefile, ( - 'Output file %s is the currently running ' - 'interpreter (use --output=...)'% exe_name) - except EnvironmentError: - pass + goals = translateconfig.goals + if not goals or 'compile' in goals: + try: + this_exe = py.path.local(sys.executable).new(ext='') + exe_name = drv.compute_exe_name() + samefile = this_exe.samefile(exe_name) + assert not samefile, ( + 'Output file %s is the currently running ' + 'interpreter (use --output=...)'% exe_name) + except EnvironmentError: + pass - goals = translateconfig.goals try: drv.proceed(goals) finally: diff --git a/pypy/translator/platform/freebsd.py b/pypy/translator/platform/freebsd.py --- a/pypy/translator/platform/freebsd.py +++ b/pypy/translator/platform/freebsd.py @@ -18,8 +18,9 @@ class Freebsd(posix.BasePosix): name = "freebsd" - link_flags = get_env_vector("LDFLAGS", '-pthread') - cflags = get_env_vector("CFLAGS", "-O3 -pthread -fomit-frame-pointer") + link_flags = ['-pthread'] + get_env_vector('LDFLAGS', '') + cflags = ['-O3', '-pthread', '-fomit-frame-pointer' + ] + get_env_vector('CFLAGS', '') standalone_only = [] shared_only = [] so_ext = 'so' diff --git a/pypy/translator/platform/linux.py b/pypy/translator/platform/linux.py --- a/pypy/translator/platform/linux.py +++ b/pypy/translator/platform/linux.py @@ -1,15 +1,20 @@ """Support for Linux.""" +import os import sys from pypy.translator.platform.posix import BasePosix class BaseLinux(BasePosix): name = "linux" - link_flags = ('-pthread',) + link_flags = tuple( + ['-pthread',] + + os.environ.get('LDFLAGS', '').split()) extra_libs = ('-lrt',) - cflags = ('-O3', '-pthread', '-fomit-frame-pointer', - '-Wall', '-Wno-unused') + cflags = tuple( + ['-O3', '-pthread', '-fomit-frame-pointer', + '-Wall', '-Wno-unused'] + + os.environ.get('CFLAGS', '').split()) standalone_only = () shared_only = ('-fPIC',) so_ext = 'so' From noreply at buildbot.pypy.org Sun Jan 29 23:56:21 2012 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 29 Jan 2012 23:56:21 +0100 (CET) Subject: [pypy-commit] pypy merge-2.7.2: Fix for the new large file tests in pypy.module.mmap. Message-ID: <20120129225621.4E64D8203C@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: merge-2.7.2 Changeset: r51962:ff6d67b226da Date: 2012-01-29 23:56 +0100 http://bitbucket.org/pypy/pypy/changeset/ff6d67b226da/ Log: Fix for the new large file tests in pypy.module.mmap. diff --git a/pypy/rlib/rmmap.py b/pypy/rlib/rmmap.py --- a/pypy/rlib/rmmap.py +++ b/pypy/rlib/rmmap.py @@ -97,12 +97,14 @@ else: HAVE_LARGEFILE_SUPPORT = False -def external(name, args, result): +def external(name, args, result, **kwargs): unsafe = rffi.llexternal(name, args, result, - compilation_info=CConfig._compilation_info_) + compilation_info=CConfig._compilation_info_, + **kwargs) safe = rffi.llexternal(name, args, result, compilation_info=CConfig._compilation_info_, - sandboxsafe=True, threadsafe=False) + sandboxsafe=True, threadsafe=False, + **kwargs) return unsafe, safe def winexternal(name, args, result, **kwargs): @@ -118,7 +120,8 @@ if _POSIX: has_mremap = cConfig['has_mremap'] c_mmap, c_mmap_safe = external('mmap', [PTR, size_t, rffi.INT, rffi.INT, - rffi.INT, off_t], PTR) + rffi.INT, off_t], PTR, macro=True) + # 'mmap' on linux32 is a macro that calls 'mmap64' _, c_munmap_safe = external('munmap', [PTR, size_t], rffi.INT) c_msync, _ = external('msync', [PTR, size_t, rffi.INT], rffi.INT) if has_mremap: From pullrequests-noreply at bitbucket.org Mon Jan 30 08:44:49 2012 From: pullrequests-noreply at bitbucket.org (Bitbucket) Date: Mon, 30 Jan 2012 07:44:49 -0000 Subject: [pypy-commit] [OPEN] Pull request #25 for pypy/pypy: json/decoder speed-up Message-ID: <pullreq-de5504a0f4f0@bitbucket.org> A new pull request has been opened by Stefano Parmesan. armisael/pypy has changes to be pulled into pypy/pypy. https://bitbucket.org/pypy/pypy/pull-request/25/json-decoder-speed-up Title: json/decoder speed-up Following what Fijal wrote on his blogpost of October 2011 I worked on cleanin the code of json/decoder.py Changes to be pulled: de5504a0f4f0 by Stefano Parmesa?: "removed cPython-oriented code in json and added KeyValueBuilder(s) for speeding ?" -- This is an issue notification from bitbucket.org. You are receiving this either because you are the participating in a pull request, or you are following it. From noreply at buildbot.pypy.org Mon Jan 30 11:29:33 2012 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 30 Jan 2012 11:29:33 +0100 (CET) Subject: [pypy-commit] pypy string-NUL: Tentative fix Message-ID: <20120130102933.CB6128203C@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: string-NUL Changeset: r51963:c91fe543cd74 Date: 2012-01-30 11:29 +0100 http://bitbucket.org/pypy/pypy/changeset/c91fe543cd74/ Log: Tentative fix diff --git a/pypy/translator/cli/test/runtest.py b/pypy/translator/cli/test/runtest.py --- a/pypy/translator/cli/test/runtest.py +++ b/pypy/translator/cli/test/runtest.py @@ -277,10 +277,11 @@ def get_annotation(x): if isinstance(x, basestring) and len(x) > 1: result = SomeString() + if '\x00' not in x: + result.no_nul = True + return result else: - result = lltype_to_annotation(typeOf(x)) - if isinstance(result, SomeString) and '\x00' not in x: - result.no_nul = True + return lltype_to_annotation(typeOf(x)) class CliTest(BaseRtypingTest, OORtypeMixin): def __init__(self): From noreply at buildbot.pypy.org Mon Jan 30 18:56:21 2012 From: noreply at buildbot.pypy.org (csenger) Date: Mon, 30 Jan 2012 18:56:21 +0100 (CET) Subject: [pypy-commit] benchmarks default: A little bit of pep8 Message-ID: <20120130175621.CD7F182B6C@wyvern.cs.uni-duesseldorf.de> Author: Carsten Senger <senger at rehfisch.de> Branch: Changeset: r164:5293a412c721 Date: 2012-01-22 01:00 +0100 http://bitbucket.org/pypy/benchmarks/changeset/5293a412c721/ Log: A little bit of pep8 diff --git a/runner.py b/runner.py --- a/runner.py +++ b/runner.py @@ -8,6 +8,7 @@ import benchmarks import socket + def perform_upload(pypy_c_path, args, force_host, options, res, revision, changed=True, postfix='', branch='default'): from saveresults import save @@ -24,9 +25,10 @@ host = force_host else: host = socket.gethostname() - print save(project, revision, res, options, name, host, changed=changed, branch=branch) + print save(project, revision, res, options, name, host, changed=changed, + branch=branch) - + def run_and_store(benchmark_set, result_filename, pypy_c_path, revision=0, options='', branch='default', args='', upload=False, force_host=None, fast=False, baseline=sys.executable, @@ -47,10 +49,10 @@ res = [(name, result.__class__.__name__, result.__dict__) for name, result in results] f.write(json.dumps({ - 'revision' : revision, - 'results' : res, - 'options' : options, - 'branch' : branch, + 'revision': revision, + 'results': res, + 'options': options, + 'branch': branch, })) f.close() if upload: @@ -60,17 +62,21 @@ argsbase, argschanged = args, args if 'pypy' in baseline: perform_upload(pypy_c_path, argsbase, force_host, options, res, - revision, changed=False, postfix=postfix, branch=branch) + revision, changed=False, postfix=postfix, + branch=branch) perform_upload(pypy_c_path, argschanged, force_host, options, res, revision, changed=True, postfix=postfix, branch=branch) + BENCHMARK_SET = ['richards', 'slowspitfire', 'django', 'spambayes', 'rietveld', 'html5lib', 'ai'] BENCHMARK_SET += perf._FindAllBenchmarks(benchmarks.__dict__).keys() + class WrongBenchmark(Exception): pass + def main(argv): import optparse parser = optparse.OptionParser( @@ -99,9 +105,9 @@ " If there is a comma in this option's value, the" " arguments before the comma (interpreted as a" " space-separated list) are passed to the baseline" - " python, and the arguments after are passed to the" - " changed python. If there's no comma, the same" - " options are passed to both.")) + " python, and the arguments after are passed to" + " the changed python. If there's no comma, the" + " same options are passed to both.")) parser.add_option("--upload", default=False, action="store_true", help="Upload results to speed.pypy.org") parser.add_option("--force-host", default=None, action="store", @@ -117,7 +123,7 @@ # use 'default' if the branch is empty if not options.branch: options.branch = 'default' - + benchmarks = options.benchmarks.split(',') for benchmark in benchmarks: if benchmark not in BENCHMARK_SET: diff --git a/saveresults.py b/saveresults.py --- a/saveresults.py +++ b/saveresults.py @@ -12,28 +12,33 @@ Example usage: - $ ./saveresults.py result.json -r '45757:fabe4fc0dc08' -n pypy-c-jit -H tannit - + $ ./saveresults.py result.json -r '45757:fabe4fc0dc08' -n pypy-c-jit \ + -H tannit + OR - - $ ./saveresults.py result.json -r '45757:fabe4fc0dc08' -n pypy-c-jit-64 -H tannit + $ ./saveresults.py result.json -r '45757:fabe4fc0dc08' -n pypy-c-jit-64 \ + -H tannit """ -import sys -import urllib, urllib2, time from datetime import datetime import optparse +import sys +import time +import urllib +import urllib2 + SPEEDURL = "http://speed.pypy.org/" + def save(project, revision, results, options, interpreter, host, testing=False, changed=True, branch='default'): testparams = [] #Parse data data = {} error = 0 - + for b in results: bench_name = b[0] res_type = b[1] @@ -77,20 +82,27 @@ data['std_dev'] = results['std_changed'] else: data['std_dev'] = results['std_base'] - if testing: testparams.append(data) - else: error |= send(data) + if testing: + testparams.append(data) + else: + error |= send(data) + if error: raise IOError("Saving failed. See messages above.") - if testing: return testparams - else: return 0 - + if testing: + return testparams + else: + return 0 + + def send(data): #save results params = urllib.urlencode(data) f = None response = "None" - info = str(datetime.today()) + ": Saving result for " + data['executable'] + " revision " - info += str(data['commitid']) + ", benchmark " + data['benchmark'] + info = ("%s: Saving result for %s revision %s, benchmark %s" % + (str(datetime.today()), data['executable'], + str(data['commitid']), data['benchmark'])) print(info) try: retries = [1, 2, 3, 6] @@ -136,7 +148,8 @@ if __name__ == '__main__': parser = optparse.OptionParser(usage="%prog result.json [options]") - parser.add_option('-r', '--revision', dest='revision', default=None, type=str) + parser.add_option('-r', '--revision', dest='revision', + default=None, type=str) parser.add_option('-n', '--name', dest='name', default=None, type=str) parser.add_option('-H', '--host', dest='host', default=None, type=str) parser.add_option('-b', '--baseline', dest='changed', default=True, @@ -145,8 +158,8 @@ parser.format_description = lambda fmt: __doc__ parser.description = __doc__ options, args = parser.parse_args() - if options.revision is None or options.name is None or options.host is None or \ - len(args) != 1: + if (options.revision is None or options.name is None or + options.host is None or len(args) != 1): parser.print_help() sys.exit(2) main(args[0], options) From noreply at buildbot.pypy.org Mon Jan 30 18:56:23 2012 From: noreply at buildbot.pypy.org (csenger) Date: Mon, 30 Jan 2012 18:56:23 +0100 (CET) Subject: [pypy-commit] benchmarks default: Use executable as variabel/parameter name like codespeed. Message-ID: <20120130175623.332DC82B6C@wyvern.cs.uni-duesseldorf.de> Author: Carsten Senger <senger at rehfisch.de> Branch: Changeset: r165:69cf04577df2 Date: 2012-01-22 12:57 +0100 http://bitbucket.org/pypy/benchmarks/changeset/69cf04577df2/ Log: Use executable as variabel/parameter name like codespeed. Also add the project name as a parameter to perform_upload. diff --git a/runner.py b/runner.py --- a/runner.py +++ b/runner.py @@ -3,30 +3,31 @@ """ import json +import socket import sys + +import benchmarks +from saveresults import save from unladen_swallow import perf -import benchmarks -import socket def perform_upload(pypy_c_path, args, force_host, options, res, revision, - changed=True, postfix='', branch='default'): - from saveresults import save - project = 'PyPy' - if "--jit" in args: - name = "pypy-c" + postfix - else: - name = "pypy-c-jit" + postfix + changed=True, postfix='', branch='default', project='PyPy', + executable=None): + if executable is None: + if "--jit" in args: + executable = "pypy-c" + postfix + else: + executable = "pypy-c-jit" + postfix + if "psyco.sh" in pypy_c_path: - name = "cpython psyco-profile" + executable = "cpython psyco-profile" revision = 100 project = 'cpython' - if force_host is not None: - host = force_host - else: - host = socket.gethostname() - print save(project, revision, res, options, name, host, changed=changed, - branch=branch) + + host = force_host if force_host else socket.gethostname() + print save(project, revision, res, options, executable, host, + changed=changed, branch=branch) def run_and_store(benchmark_set, result_filename, pypy_c_path, revision=0, @@ -55,6 +56,7 @@ 'branch': branch, })) f.close() + if upload: if ',' in args: argsbase, argschanged = args.split(',') diff --git a/saveresults.py b/saveresults.py --- a/saveresults.py +++ b/saveresults.py @@ -32,7 +32,7 @@ SPEEDURL = "http://speed.pypy.org/" -def save(project, revision, results, options, interpreter, host, testing=False, +def save(project, revision, results, options, executeable, host, testing=False, changed=True, branch='default'): testparams = [] #Parse data @@ -68,7 +68,7 @@ data = { 'commitid': revision, 'project': project, - 'executable': interpreter, + 'executable': executeable, 'benchmark': bench_name, 'environment': host, 'result_value': value, From noreply at buildbot.pypy.org Mon Jan 30 18:56:24 2012 From: noreply at buildbot.pypy.org (csenger) Date: Mon, 30 Jan 2012 18:56:24 +0100 (CET) Subject: [pypy-commit] benchmarks default: Refactoring, support script params executeable, project, url Message-ID: <20120130175624.83CC882B6C@wyvern.cs.uni-duesseldorf.de> Author: Carsten Senger <senger at rehfisch.de> Branch: Changeset: r166:0c6207da7e15 Date: 2012-01-22 16:30 +0100 http://bitbucket.org/pypy/benchmarks/changeset/0c6207da7e15/ Log: Refactoring, support script params executeable, project, url * Refactoring: move runner upload code from run_and_store and perform_upload to main. * factor out code to guess the executable * Remove hardcoded values for project, executable and codespeed url and add them as parameters to the runner script (--update- executable, --update-project and --upload-urls) and the saveresults script (--executable, --project and --url) * adjust parameters of saveresults.save and .send * remove unused parameter options from save() * adjust tests diff --git a/runner.py b/runner.py --- a/runner.py +++ b/runner.py @@ -10,73 +10,63 @@ from saveresults import save from unladen_swallow import perf +BENCHMARK_SET = ['richards', 'slowspitfire', 'django', 'spambayes', + 'rietveld', 'html5lib', 'ai'] +BENCHMARK_SET += perf._FindAllBenchmarks(benchmarks.__dict__).keys() -def perform_upload(pypy_c_path, args, force_host, options, res, revision, - changed=True, postfix='', branch='default', project='PyPy', - executable=None): - if executable is None: - if "--jit" in args: - executable = "pypy-c" + postfix - else: - executable = "pypy-c-jit" + postfix +# executeablenames +EX_CPY_PSYCO = "cpython psyco-profile" +EX_PYPY_C = 'pypy-c' +EX_PYPY_C_JIT = 'pypy-c-jit' + + +class WrongBenchmark(Exception): + pass + + +def guess_executeable(pypy_c_path, postfix, args, changed): + + if ',' in args: + args_baseline, args_changed = args.split(',') + args = args_changed if changed else args_baseline if "psyco.sh" in pypy_c_path: - executable = "cpython psyco-profile" - revision = 100 - project = 'cpython' + executable_prefix = EX_CPY_PSYCO + elif '--jit' in args: + executable_prefix = EX_PYPY_C + else: + executable_prefix = EX_PYPY_C_JIT - host = force_host if force_host else socket.gethostname() - print save(project, revision, res, options, executable, host, - changed=changed, branch=branch) + return executable_prefix + postfix def run_and_store(benchmark_set, result_filename, pypy_c_path, revision=0, options='', branch='default', args='', upload=False, - force_host=None, fast=False, baseline=sys.executable, - full_store=False, postfix=''): + fast=False, baseline=sys.executable, full_store=False): funcs = perf.BENCH_FUNCS.copy() funcs.update(perf._FindAllBenchmarks(benchmarks.__dict__)) - opts = ['-b', ','.join(benchmark_set), '--inherit_env=PATH', + opts = ['-b', ','.join(benchmark_set), + '--inherit_env=PATH', '--no_charts'] if fast: opts += ['--fast'] if args: opts += ['--args', args] if full_store: - opts.append('--no_statistics') + opts += ['--no_statistics'] opts += [baseline, pypy_c_path] results = perf.main(opts, funcs) f = open(str(result_filename), "w") - res = [(name, result.__class__.__name__, result.__dict__) + results = [(name, result.__class__.__name__, result.__dict__) for name, result in results] f.write(json.dumps({ 'revision': revision, - 'results': res, + 'results': results, 'options': options, 'branch': branch, })) f.close() - - if upload: - if ',' in args: - argsbase, argschanged = args.split(',') - else: - argsbase, argschanged = args, args - if 'pypy' in baseline: - perform_upload(pypy_c_path, argsbase, force_host, options, res, - revision, changed=False, postfix=postfix, - branch=branch) - perform_upload(pypy_c_path, argschanged, force_host, options, res, - revision, changed=True, postfix=postfix, branch=branch) - - -BENCHMARK_SET = ['richards', 'slowspitfire', 'django', 'spambayes', - 'rietveld', 'html5lib', 'ai'] -BENCHMARK_SET += perf._FindAllBenchmarks(benchmarks.__dict__).keys() - - -class WrongBenchmark(Exception): - pass + return results def main(argv): @@ -110,8 +100,17 @@ " python, and the arguments after are passed to" " the changed python. If there's no comma, the" " same options are passed to both.")) - parser.add_option("--upload", default=False, action="store_true", - help="Upload results to speed.pypy.org") + parser.add_option("--upload", default=None, action="store_true", + help=("Upload results to speed.pypy.org (unless " + "--upload-url is given).")) + parser.add_option("--upload-urls", default="http://speed.pypy.org/", + help=("Comma seperated urls of the codespeed instance to" + " upload to (default: http://speed.pypy.org/).")) + parser.add_option("--upload-project", default="PyPy", + help="The project name in codespeed (default: PyPy).") + parser.add_option("--upload-executable", default=None, + help=("The executable name in codespeed " + "(guessed if possible and not given).")) parser.add_option("--force-host", default=None, action="store", help="Force the hostname") parser.add_option("--fast", default=False, action="store_true", @@ -130,11 +129,42 @@ for benchmark in benchmarks: if benchmark not in BENCHMARK_SET: raise WrongBenchmark(benchmark) - run_and_store(benchmarks, options.output_filename, options.pypy_c, - options.revision, args=options.args, upload=options.upload, - force_host=options.force_host, fast=options.fast, - baseline=options.baseline, full_store=options.full_store, - postfix=options.postfix, branch=options.branch) + + pypy_c_path = options.pypy_c + baseline = options.baseline + fast = options.fast + args = options.args + full_store = options.full_store + output_filename = options.output_filename + + project = options.upload_project + executable = options.upload_executable + postfix = options.postfix + branch = options.branch + revision = options.revision + + force_host = options.force_host + + results = run_and_store(benchmarks, output_filename, pypy_c_path, revision, + args=args, fast=fast, baseline=baseline, + full_store=full_store, branch=branch) + + if options.upload: + changed = 'pypy' not in baseline + if executable is None: + executable = guess_executeable(pypy_c_path, postfix, args, changed) + + if executable == EX_CPY_PSYCO: + revision = 100 + project = 'cpython' + + changed = False + + host = force_host if force_host else socket.gethostname() + for url in options.upload_urls.split(','): + print save(project, revision, results, executable, host, url, + changed=changed, branch=branch) + if __name__ == '__main__': main(sys.argv[1:]) diff --git a/saveresults.py b/saveresults.py --- a/saveresults.py +++ b/saveresults.py @@ -29,10 +29,7 @@ import urllib2 -SPEEDURL = "http://speed.pypy.org/" - - -def save(project, revision, results, options, executeable, host, testing=False, +def save(project, revision, results, executeable, host, url, testing=False, changed=True, branch='default'): testparams = [] #Parse data @@ -85,7 +82,7 @@ if testing: testparams.append(data) else: - error |= send(data) + error |= send(data, url) if error: raise IOError("Saving failed. See messages above.") @@ -95,7 +92,7 @@ return 0 -def send(data): +def send(data, url): #save results params = urllib.urlencode(data) f = None @@ -108,7 +105,7 @@ retries = [1, 2, 3, 6] while True: try: - f = urllib2.urlopen(SPEEDURL + 'result/add/', params) + f = urllib2.urlopen(url + 'result/add/', params) response = f.read() f.close() break @@ -128,7 +125,7 @@ print response with open('error.html', 'w') as error_file: error_file.write(response) - print("Server (%s) response written to error.html" % (SPEEDURL,)) + print("Server (%s) response written to error.html" % (url,)) print(' Error code: %s\n' % (e,)) return 1 print "saved correctly!\n" @@ -141,24 +138,35 @@ data = simplejson.load(f) results = data['results'] print 'uploading results...', - save('PyPy', options.revision, results, '', options.name, options.host, - changed=options.changed) + save(options.project, options.revision, results, options.executable, + options.host, options.url, changed=options.changed) print 'done' if __name__ == '__main__': parser = optparse.OptionParser(usage="%prog result.json [options]") parser.add_option('-r', '--revision', dest='revision', - default=None, type=str) - parser.add_option('-n', '--name', dest='name', default=None, type=str) + default=None, type=str, help='VCS revision (required)') + parser.add_option('-n', '--name', dest='executable', + default=None, type=str, + help=('Name of the executable for codespeed.' + 'Deprecated. Use --e/--executable instead')) + parser.add_option('-e', '--executable', dest='executable', + default=None, type=str, + help='Name of the Executable for codespeed (required).') parser.add_option('-H', '--host', dest='host', default=None, type=str) parser.add_option('-b', '--baseline', dest='changed', default=True, action='store_false', help='upload the results as baseline instead of changed') + parser.add_option('-P', '--project', dest='project', default='PyPy') + parser.add_option('-u', '--url', dest='url', + default="http://speed.pypy.org/", + help=('Url of the codespeed instance ' + '(default: http://speed.pypy.org)')) parser.format_description = lambda fmt: __doc__ parser.description = __doc__ options, args = parser.parse_args() - if (options.revision is None or options.name is None or + if (options.revision is None or options.executable is None or options.host is None or len(args) != 1): parser.print_help() sys.exit(2) diff --git a/test/test_runner.py b/test/test_runner.py --- a/test/test_runner.py +++ b/test/test_runner.py @@ -1,9 +1,9 @@ - import py import json import sys from runner import run_and_store + def test_run_and_store(): tmpdir = py.test.ensuretemp('bench_runner') resfile = tmpdir.join('results') diff --git a/test/test_saveresults.py b/test/test_saveresults.py --- a/test/test_saveresults.py +++ b/test/test_saveresults.py @@ -22,7 +22,8 @@ def test_good_input(self): '''Given correct result data, check that every result being saved has the right parameters''' - for resultparams in saveresults.save("PyPy", 71212, self.fixture, "", "pypy-c-jit", 'host', True): + for resultparams in saveresults.save("PyPy", 71212, self.fixture, "pypy-c-jit", 'host', 'url', + testing=True): assert resultparams['project'] == "PyPy" assert resultparams['commitid'] == 71212 assert resultparams['executable'] == "pypy-c-jit" From noreply at buildbot.pypy.org Mon Jan 30 18:56:26 2012 From: noreply at buildbot.pypy.org (csenger) Date: Mon, 30 Jan 2012 18:56:26 +0100 (CET) Subject: [pypy-commit] benchmarks default: Add command line options. Don't guess codespeed parameters. Message-ID: <20120130175626.00D9D82B6C@wyvern.cs.uni-duesseldorf.de> Author: Carsten Senger <senger at rehfisch.de> Branch: Changeset: r167:d9e2730348d2 Date: 2012-01-24 21:41 +0100 http://bitbucket.org/pypy/benchmarks/changeset/d9e2730348d2/ Log: Add command line options. Don't guess codespeed parameters. * Add command line options to upload the baseline data to codespeed. * Don't guess if we want to upload the baseline result to codespeed and what's the executable and project in codespeed. diff --git a/runner.py b/runner.py --- a/runner.py +++ b/runner.py @@ -14,32 +14,14 @@ 'rietveld', 'html5lib', 'ai'] BENCHMARK_SET += perf._FindAllBenchmarks(benchmarks.__dict__).keys() -# executeablenames -EX_CPY_PSYCO = "cpython psyco-profile" -EX_PYPY_C = 'pypy-c' -EX_PYPY_C_JIT = 'pypy-c-jit' +CHANGED = 'changed' +BASELINE = 'baseline' class WrongBenchmark(Exception): pass -def guess_executeable(pypy_c_path, postfix, args, changed): - - if ',' in args: - args_baseline, args_changed = args.split(',') - args = args_changed if changed else args_baseline - - if "psyco.sh" in pypy_c_path: - executable_prefix = EX_CPY_PSYCO - elif '--jit' in args: - executable_prefix = EX_PYPY_C - else: - executable_prefix = EX_PYPY_C_JIT - - return executable_prefix + postfix - - def run_and_store(benchmark_set, result_filename, pypy_c_path, revision=0, options='', branch='default', args='', upload=False, fast=False, baseline=sys.executable, full_store=False): @@ -69,6 +51,66 @@ return results +def get_upload_options(options): + ''' + returns a dict with 2 keys: CHANGED, BASELINE. The values are + dicts with the keys + * 'upload' (boolean) + * 'project' (string) + * 'executable' (string) + * 'urls (list of strings). + * 'branch' (string) + * 'revision' (string) + + This correspondents to the the --upload* and --upload-baseline* + options. + + raises: AssertionError if upload is specified, but not the + corresponding executable or revision. + ''' + + if options.upload_baseline_revision is None: + options.upload_baseline_revision = options.upload_revision + + upload_options = {} + + for run in [CHANGED, BASELINE]: + + def get_upload_option(name): + attr_name = 'upload' + if run == BASELINE: + attr_name = '%s_baseline' % attr_name + if name: + attr_name = '%s_%s' % (attr_name, name) + return getattr(options, attr_name) + + urls = get_upload_option('urls') + urls = [url.strip() for url in urls.split(',') if url.strip()] + upload = get_upload_option(None) + project = get_upload_option('project') + executable = get_upload_option('executable') + branch = get_upload_option('branch') + revision = get_upload_option('revision') + if upload: + if executable is None: + raise AssertionError('If you want to --upload[-baseline] you ' + 'have to specify the corresponding ' + '--upload[-baseline]-executable') + if revision is None: + raise AssertionError('If you want to upload the result you ' + 'have to specify a --revision (or ' + '--upload-baseline-revision if you ' + 'want to upload the baseline result') + upload_options[run] = { + 'upload': upload, + 'project': project, + 'executable': executable, + 'urls': urls, + 'branch': branch, + 'revision': revision} + return upload_options + + def main(argv): import optparse parser = optparse.OptionParser( @@ -81,7 +123,8 @@ ", ".join(sorted(BENCHMARK_SET)))) parser.add_option('-p', '--pypy-c', default=sys.executable, help='pypy-c or other modified python to run against') - parser.add_option('-r', '--revision', default=0, action="store", + parser.add_option('-r', '--revision', action="store", + dest='upload_revision', help='specify revision of pypy-c') parser.add_option('-o', '--output-filename', default="result.json", action="store", @@ -89,7 +132,8 @@ parser.add_option('--options', default='', action='store', help='a string describing picked options, no spaces') parser.add_option('--branch', default='default', action='store', - help="pypy's branch") + dest='upload_branch', + help="pypy's branch (default: 'default'") parser.add_option('--baseline', default=sys.executable, action='store', help='baseline interpreter, defaults to host one') parser.add_option("-a", "--args", default="", @@ -104,13 +148,33 @@ help=("Upload results to speed.pypy.org (unless " "--upload-url is given).")) parser.add_option("--upload-urls", default="http://speed.pypy.org/", - help=("Comma seperated urls of the codespeed instance to" - " upload to (default: http://speed.pypy.org/).")) + help=("Comma seperated urls of the codespeed instances " + "to upload to (default: http://speed.pypy.org/).")) parser.add_option("--upload-project", default="PyPy", help="The project name in codespeed (default: PyPy).") parser.add_option("--upload-executable", default=None, - help=("The executable name in codespeed " - "(guessed if possible and not given).")) + help="The executable name in codespeed.") + parser.add_option("--upload-baseline", default=None, action="store_true", + help=("Also upload results or the baseline benchmark " + "to speed.pypy.org (unless " + "--upload-baseline-url is given).")) + parser.add_option("--upload-baseline-urls", + default="http://speed.pypy.org/", + help=("Comma seperated urls of the codespeed instances " + "to upload to (default: http://speed.pypy.org/).")) + parser.add_option("--upload-baseline-project", default="PyPy", + help="The project name in codespeed (default: PyPy).") + parser.add_option("--upload-baseline-executable", default=None, + help="The executable name in codespeed.") + parser.add_option('--upload-baseline-branch', default='default', + action='store', + help=("The name of the branch used for the baseline " + "run. (default: 'default'")) + parser.add_option('--upload-baseline-revision', action='store', + default=None, + help=("The revision of the baseline " + "run. (default: the revision given with -r")) + parser.add_option("--force-host", default=None, action="store", help="Force the hostname") parser.add_option("--fast", default=False, action="store_true", @@ -120,11 +184,8 @@ parser.add_option('--postfix', default='', action='store', help='Append a postfix to uploaded executable') options, args = parser.parse_args(argv) - # - # use 'default' if the branch is empty - if not options.branch: - options.branch = 'default' + upload_options = get_upload_options(options) benchmarks = options.benchmarks.split(',') for benchmark in benchmarks: if benchmark not in BENCHMARK_SET: @@ -137,33 +198,27 @@ full_store = options.full_store output_filename = options.output_filename - project = options.upload_project - executable = options.upload_executable - postfix = options.postfix - branch = options.branch - revision = options.revision - + branch = options.upload_branch + revision = options.upload_revision force_host = options.force_host results = run_and_store(benchmarks, output_filename, pypy_c_path, revision, args=args, fast=fast, baseline=baseline, full_store=full_store, branch=branch) - if options.upload: - changed = 'pypy' not in baseline - if executable is None: - executable = guess_executeable(pypy_c_path, postfix, args, changed) + for run in [CHANGED, BASELINE]: + upload = upload_options[run]['upload'] + urls = upload_options[run]['urls'] + project = upload_options[run]['project'] + executable = upload_options[run]['executable'] + branch = upload_options[run]['branch'] + revision = upload_options[run]['revision'] - if executable == EX_CPY_PSYCO: - revision = 100 - project = 'cpython' - - changed = False - - host = force_host if force_host else socket.gethostname() - for url in options.upload_urls.split(','): - print save(project, revision, results, executable, host, url, - changed=changed, branch=branch) + if upload: + host = force_host if force_host else socket.gethostname() + for url in urls: + print save(project, revision, results, executable, host, url, + changed=(run == CHANGED), branch=branch) if __name__ == '__main__': From noreply at buildbot.pypy.org Mon Jan 30 18:56:27 2012 From: noreply at buildbot.pypy.org (csenger) Date: Mon, 30 Jan 2012 18:56:27 +0100 (CET) Subject: [pypy-commit] benchmarks default: Add a nullpython interpreter that can be used for baseline Message-ID: <20120130175627.5106882B6C@wyvern.cs.uni-duesseldorf.de> Author: Carsten Senger <senger at rehfisch.de> Branch: Changeset: r168:a918143f6f9d Date: 2012-01-24 21:56 +0100 http://bitbucket.org/pypy/benchmarks/changeset/a918143f6f9d/ Log: Add a nullpython interpreter that can be used for baseline It will not run any benchmarks, just tell that every run of the benchmark too 0.0001 seconds. Make sure we don't upload this data to codespeed. diff --git a/nullpython.py b/nullpython.py new file mode 100755 --- /dev/null +++ b/nullpython.py @@ -0,0 +1,20 @@ +#!/usr/bin/python +'''This is a dummy that does nothing except that it returns 1 +second for every round of the benchmark. + +You can use this as the baseline interpreter if you are only +interested in the time of the changed interpreter, but not +in the difference to a baseline interpreter. +''' +from own import util +import optparse + +if __name__ == '__main__': + parser = optparse.OptionParser( + usage="%prog [options]", + description="Test the performance of the Go benchmark") + util.add_standard_options_to(parser) + options, args = parser.parse_args() + + main = lambda n: [0.0001 for x in range(options.num_runs)] + util.run_benchmark(options, options.num_runs, main) diff --git a/runner.py b/runner.py --- a/runner.py +++ b/runner.py @@ -61,7 +61,7 @@ * 'urls (list of strings). * 'branch' (string) * 'revision' (string) - + This correspondents to the the --upload* and --upload-baseline* options. @@ -101,6 +101,12 @@ 'have to specify a --revision (or ' '--upload-baseline-revision if you ' 'want to upload the baseline result') + if ((run == BASELINE and 'nullpython.py' in options.baseline) or + (run == CHANGED and 'nullpython.py' in options.pypy_c)): + raise AssertionError("Don't upload data from the nullpython " + "dummy interpreter. It won't run any " + "real benchmarks.") + upload_options[run] = { 'upload': upload, 'project': project, @@ -215,6 +221,7 @@ revision = upload_options[run]['revision'] if upload: + # prevent to upload results from the nullpython dummy host = force_host if force_host else socket.gethostname() for url in urls: print save(project, revision, results, executable, host, url, From noreply at buildbot.pypy.org Mon Jan 30 18:56:28 2012 From: noreply at buildbot.pypy.org (csenger) Date: Mon, 30 Jan 2012 18:56:28 +0100 (CET) Subject: [pypy-commit] benchmarks default: Remove obsolete --postfix option Message-ID: <20120130175628.A15D382B6C@wyvern.cs.uni-duesseldorf.de> Author: Carsten Senger <senger at rehfisch.de> Branch: Changeset: r169:fceaaf9b1f82 Date: 2012-01-24 22:08 +0100 http://bitbucket.org/pypy/benchmarks/changeset/fceaaf9b1f82/ Log: Remove obsolete --postfix option diff --git a/runner.py b/runner.py --- a/runner.py +++ b/runner.py @@ -187,8 +187,7 @@ help="Run shorter benchmark runs") parser.add_option("--full-store", default=False, action="store_true", help="") - parser.add_option('--postfix', default='', action='store', - help='Append a postfix to uploaded executable') + options, args = parser.parse_args(argv) upload_options = get_upload_options(options) From noreply at buildbot.pypy.org Mon Jan 30 18:56:29 2012 From: noreply at buildbot.pypy.org (csenger) Date: Mon, 30 Jan 2012 18:56:29 +0100 (CET) Subject: [pypy-commit] benchmarks default: Group command line options and cleanup help texts. Message-ID: <20120130175629.F01BC82B6C@wyvern.cs.uni-duesseldorf.de> Author: Carsten Senger <senger at rehfisch.de> Branch: Changeset: r170:9b8b4e17a829 Date: 2012-01-25 12:45 +0100 http://bitbucket.org/pypy/benchmarks/changeset/9b8b4e17a829/ Log: Group command line options and cleanup help texts. diff --git a/runner.py b/runner.py --- a/runner.py +++ b/runner.py @@ -122,71 +122,125 @@ parser = optparse.OptionParser( usage="%prog [options]", description="Run benchmarks and dump json") - parser.add_option("-b", "--benchmarks", metavar="BM_LIST", - default=','.join(BENCHMARK_SET), - help=("Comma-separated list of benchmarks to run" - " Valid benchmarks are: " + - ", ".join(sorted(BENCHMARK_SET)))) - parser.add_option('-p', '--pypy-c', default=sys.executable, - help='pypy-c or other modified python to run against') - parser.add_option('-r', '--revision', action="store", - dest='upload_revision', - help='specify revision of pypy-c') - parser.add_option('-o', '--output-filename', default="result.json", - action="store", - help='specify output filename to store resulting json') - parser.add_option('--options', default='', action='store', - help='a string describing picked options, no spaces') - parser.add_option('--branch', default='default', action='store', - dest='upload_branch', - help="pypy's branch (default: 'default'") - parser.add_option('--baseline', default=sys.executable, action='store', - help='baseline interpreter, defaults to host one') - parser.add_option("-a", "--args", default="", - help=("Pass extra arguments to the python binaries." - " If there is a comma in this option's value, the" - " arguments before the comma (interpreted as a" - " space-separated list) are passed to the baseline" - " python, and the arguments after are passed to" - " the changed python. If there's no comma, the" - " same options are passed to both.")) - parser.add_option("--upload", default=None, action="store_true", - help=("Upload results to speed.pypy.org (unless " - "--upload-url is given).")) - parser.add_option("--upload-urls", default="http://speed.pypy.org/", - help=("Comma seperated urls of the codespeed instances " - "to upload to (default: http://speed.pypy.org/).")) - parser.add_option("--upload-project", default="PyPy", - help="The project name in codespeed (default: PyPy).") - parser.add_option("--upload-executable", default=None, - help="The executable name in codespeed.") - parser.add_option("--upload-baseline", default=None, action="store_true", - help=("Also upload results or the baseline benchmark " - "to speed.pypy.org (unless " - "--upload-baseline-url is given).")) - parser.add_option("--upload-baseline-urls", - default="http://speed.pypy.org/", - help=("Comma seperated urls of the codespeed instances " - "to upload to (default: http://speed.pypy.org/).")) - parser.add_option("--upload-baseline-project", default="PyPy", - help="The project name in codespeed (default: PyPy).") - parser.add_option("--upload-baseline-executable", default=None, - help="The executable name in codespeed.") - parser.add_option('--upload-baseline-branch', default='default', - action='store', - help=("The name of the branch used for the baseline " - "run. (default: 'default'")) - parser.add_option('--upload-baseline-revision', action='store', - default=None, - help=("The revision of the baseline " - "run. (default: the revision given with -r")) - parser.add_option("--force-host", default=None, action="store", - help="Force the hostname") - parser.add_option("--fast", default=False, action="store_true", - help="Run shorter benchmark runs") - parser.add_option("--full-store", default=False, action="store_true", - help="") + # benchmark options + benchmark_group = optparse.OptionGroup( + parser, 'Benchmark options', + ('Options affecting the benchmark runs and the resulting output ' + 'json file.')) + benchmark_group.add_option( + "-b", "--benchmarks", metavar="BM_LIST", + default=','.join(BENCHMARK_SET), + help=("Comma-separated list of benchmarks to run" + " Valid benchmarks are: %s" + ". (default: Run all listed benchmarks)" + ) % ", ".join(sorted(BENCHMARK_SET))) + benchmark_group.add_option( + '-p', '--pypy-c', default=sys.executable, + help=('pypy-c or another modified python interpreter to run against. ' + 'Also named "changed" python. (default: the python used to ' + 'run this script)')) + benchmark_group.add_option( + '--baseline', default=sys.executable, action='store', + help=('Baseline interpreter. (default: the python used to ' + 'run this script)')) + benchmark_group.add_option( + '-o', '--output-filename', default="result.json", + action="store", + help=('Specify the output filename to store resulting json. ' + '(default: result.json)')) + benchmark_group.add_option( + '--options', default='', action='store', + help='A string describing picked options, no spaces.') + benchmark_group.add_option( + '--branch', default='default', action='store', + dest='upload_branch', + help=("The branch the 'pypy-c' interpreter was compiled from. This " + 'will be store in the result json and used for the upload. ' + "(default: 'default')")) + benchmark_group.add_option( + '-r', '--revision', action="store", + dest='upload_revision', + help=('Specify the revision of pypy-c. This will be store in the ' + 'result json and used for the upload. (default: None)')) + benchmark_group.add_option( + "-a", "--args", default="", + help=("Pass extra arguments to the python binaries." + " If there is a comma in this option's value, the" + " arguments before the comma (interpreted as a" + " space-separated list) are passed to the baseline" + " python, and the arguments after are passed to" + " the changed python. If there's no comma, the" + " same options are passed to both.")) + benchmark_group.add_option( + "--fast", default=False, action="store_true", + help="Run shorter benchmark runs.") + benchmark_group.add_option( + "--full-store", default=False, action="store_true", + help="Run the benchmarks with the --no-statistics flag.") + parser.add_option_group(benchmark_group) + + # upload changed options + upload_group = optparse.OptionGroup( + parser, 'Upload Options', + ('Options for uploading the result of the "changed" python to ' + 'codespeed. The information about revision and branch will ' + 'be taken from the options --revision and --branch.')) + upload_group.add_option( + "--upload", default=None, action="store_true", + help=("Upload results to speed.pypy.org (unless " + "--upload-url is given).")) + upload_group.add_option( + "--upload-urls", default="http://speed.pypy.org/", + help=("Comma seperated urls of the codespeed instances " + "to upload to. (default: http://speed.pypy.org/)")) + upload_group.add_option( + "--upload-project", default="PyPy", + help="The project name in codespeed. (default: PyPy)") + upload_group.add_option( + "--upload-executable", default=None, + help=("The executable name in codespeed. (required if --upload " + "is given)")) + parser.add_option_group(upload_group) + parser.add_option( + "--force-host", default=None, action="store", + help=("Force the hostname. This option will also be used when " + "uploading the baseline result.")) + + # upload baseline group + upload_baseline_group = optparse.OptionGroup( + parser, 'Upload Baseline Options', + ('Options for uploading the result of the "baseline" python to ' + 'codespeed. The hostname of the --force-host option will be used ' + 'in the baseline upload too.')) + upload_baseline_group.add_option( + "--upload-baseline", default=None, action="store_true", + help=("Also upload results or the baseline benchmark " + "to speed.pypy.org (unless " + "--upload-baseline-url is given).")) + upload_baseline_group.add_option( + "--upload-baseline-urls", + default="http://speed.pypy.org/", + help=("Comma seperated urls of the codespeed instances " + "to upload to. (default: http://speed.pypy.org/)")) + upload_baseline_group.add_option( + "--upload-baseline-project", default="PyPy", + help="The project name in codespeed (default: PyPy).") + upload_baseline_group.add_option( + "--upload-baseline-executable", default=None, + help=("The executable name in codespeed. (required if " + "--upload-baseline is given)")) + upload_baseline_group.add_option( + '--upload-baseline-branch', default='default', + action='store', + help=("The name of the branch used for the baseline " + "run. (default: 'default'")) + upload_baseline_group.add_option( + '--upload-baseline-revision', action='store', + default=None, + help=("The revision of the baseline. (required if --upload-baseline " + "is given)")) + parser.add_option_group(upload_baseline_group) options, args = parser.parse_args(argv) From noreply at buildbot.pypy.org Mon Jan 30 18:56:31 2012 From: noreply at buildbot.pypy.org (csenger) Date: Mon, 30 Jan 2012 18:56:31 +0100 (CET) Subject: [pypy-commit] benchmarks default: change --pypy-c to --changed and leave b/w compat option Message-ID: <20120130175631.4AEB282B6C@wyvern.cs.uni-duesseldorf.de> Author: Carsten Senger <senger at rehfisch.de> Branch: Changeset: r171:ab892247a1df Date: 2012-01-25 14:03 +0100 http://bitbucket.org/pypy/benchmarks/changeset/ab892247a1df/ Log: change --pypy-c to --changed and leave b/w compat option diff --git a/runner.py b/runner.py --- a/runner.py +++ b/runner.py @@ -22,9 +22,9 @@ pass -def run_and_store(benchmark_set, result_filename, pypy_c_path, revision=0, +def run_and_store(benchmark_set, result_filename, changed_path, revision=0, options='', branch='default', args='', upload=False, - fast=False, baseline=sys.executable, full_store=False): + fast=False, baseline_path=sys.executable, full_store=False): funcs = perf.BENCH_FUNCS.copy() funcs.update(perf._FindAllBenchmarks(benchmarks.__dict__)) opts = ['-b', ','.join(benchmark_set), @@ -36,7 +36,7 @@ opts += ['--args', args] if full_store: opts += ['--no_statistics'] - opts += [baseline, pypy_c_path] + opts += [baseline_path, changed_path] results = perf.main(opts, funcs) f = open(str(result_filename), "w") results = [(name, result.__class__.__name__, result.__dict__) @@ -102,7 +102,7 @@ '--upload-baseline-revision if you ' 'want to upload the baseline result') if ((run == BASELINE and 'nullpython.py' in options.baseline) or - (run == CHANGED and 'nullpython.py' in options.pypy_c)): + (run == CHANGED and 'nullpython.py' in options.changed)): raise AssertionError("Don't upload data from the nullpython " "dummy interpreter. It won't run any " "real benchmarks.") @@ -136,10 +136,10 @@ ". (default: Run all listed benchmarks)" ) % ", ".join(sorted(BENCHMARK_SET))) benchmark_group.add_option( - '-p', '--pypy-c', default=sys.executable, + '-c', '--changed', default=sys.executable, help=('pypy-c or another modified python interpreter to run against. ' - 'Also named "changed" python. (default: the python used to ' - 'run this script)')) + 'Also named the "changed" interpreter. (default: the python ' + 'used to run this script)')) benchmark_group.add_option( '--baseline', default=sys.executable, action='store', help=('Baseline interpreter. (default: the python used to ' @@ -155,13 +155,14 @@ benchmark_group.add_option( '--branch', default='default', action='store', dest='upload_branch', - help=("The branch the 'pypy-c' interpreter was compiled from. This " + help=("The branch the 'changed' interpreter was compiled from. This " 'will be store in the result json and used for the upload. ' "(default: 'default')")) benchmark_group.add_option( '-r', '--revision', action="store", dest='upload_revision', - help=('Specify the revision of pypy-c. This will be store in the ' + help=("Specify the revision of the 'changed' interpreter. " + 'This will be store in the ' 'result json and used for the upload. (default: None)')) benchmark_group.add_option( "-a", "--args", default="", @@ -242,6 +243,15 @@ "is given)")) parser.add_option_group(upload_baseline_group) + # Backward compoatibility options + deprecated_group = optparse.OptionGroup( + parser, 'Deprecated Options', + 'Still here for backward compatibility.') + deprecated_group.add_option( + '-p', '--pypy-c', default=sys.executable, + dest='changed', help='Deprecated alias for -c/--changed') + parser.add_option_group(deprecated_group) + options, args = parser.parse_args(argv) upload_options = get_upload_options(options) @@ -250,8 +260,8 @@ if benchmark not in BENCHMARK_SET: raise WrongBenchmark(benchmark) - pypy_c_path = options.pypy_c - baseline = options.baseline + changed_path = options.changed + baseline_path = options.baseline fast = options.fast args = options.args full_store = options.full_store @@ -261,8 +271,9 @@ revision = options.upload_revision force_host = options.force_host - results = run_and_store(benchmarks, output_filename, pypy_c_path, revision, - args=args, fast=fast, baseline=baseline, + results = run_and_store(benchmarks, output_filename, changed_path, + revision, args=args, fast=fast, + baseline_path=baseline_path, full_store=full_store, branch=branch) for run in [CHANGED, BASELINE]: From noreply at buildbot.pypy.org Mon Jan 30 18:56:32 2012 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 30 Jan 2012 18:56:32 +0100 (CET) Subject: [pypy-commit] benchmarks default: Merge csenger benchmark runner changes, IMO makes sense Message-ID: <20120130175632.A3C2A82B6C@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: Changeset: r172:012ff551c081 Date: 2012-01-30 19:55 +0200 http://bitbucket.org/pypy/benchmarks/changeset/012ff551c081/ Log: Merge csenger benchmark runner changes, IMO makes sense diff --git a/nullpython.py b/nullpython.py new file mode 100755 --- /dev/null +++ b/nullpython.py @@ -0,0 +1,20 @@ +#!/usr/bin/python +'''This is a dummy that does nothing except that it returns 1 +second for every round of the benchmark. + +You can use this as the baseline interpreter if you are only +interested in the time of the changed interpreter, but not +in the difference to a baseline interpreter. +''' +from own import util +import optparse + +if __name__ == '__main__': + parser = optparse.OptionParser( + usage="%prog [options]", + description="Test the performance of the Go benchmark") + util.add_standard_options_to(parser) + options, args = parser.parse_args() + + main = lambda n: [0.0001 for x in range(options.num_runs)] + util.run_benchmark(options, options.num_runs, main) diff --git a/runner.py b/runner.py --- a/runner.py +++ b/runner.py @@ -3,130 +3,294 @@ """ import json +import socket import sys + +import benchmarks +from saveresults import save from unladen_swallow import perf -import benchmarks -import socket -def perform_upload(pypy_c_path, args, force_host, options, res, revision, - changed=True, postfix='', branch='default'): - from saveresults import save - project = 'PyPy' - if "--jit" in args: - name = "pypy-c" + postfix - else: - name = "pypy-c-jit" + postfix - if "psyco.sh" in pypy_c_path: - name = "cpython psyco-profile" - revision = 100 - project = 'cpython' - if force_host is not None: - host = force_host - else: - host = socket.gethostname() - print save(project, revision, res, options, name, host, changed=changed, branch=branch) +BENCHMARK_SET = ['richards', 'slowspitfire', 'django', 'spambayes', + 'rietveld', 'html5lib', 'ai'] +BENCHMARK_SET += perf._FindAllBenchmarks(benchmarks.__dict__).keys() - -def run_and_store(benchmark_set, result_filename, pypy_c_path, revision=0, +CHANGED = 'changed' +BASELINE = 'baseline' + + +class WrongBenchmark(Exception): + pass + + +def run_and_store(benchmark_set, result_filename, changed_path, revision=0, options='', branch='default', args='', upload=False, - force_host=None, fast=False, baseline=sys.executable, - full_store=False, postfix=''): + fast=False, baseline_path=sys.executable, full_store=False): funcs = perf.BENCH_FUNCS.copy() funcs.update(perf._FindAllBenchmarks(benchmarks.__dict__)) - opts = ['-b', ','.join(benchmark_set), '--inherit_env=PATH', + opts = ['-b', ','.join(benchmark_set), + '--inherit_env=PATH', '--no_charts'] if fast: opts += ['--fast'] if args: opts += ['--args', args] if full_store: - opts.append('--no_statistics') - opts += [baseline, pypy_c_path] + opts += ['--no_statistics'] + opts += [baseline_path, changed_path] results = perf.main(opts, funcs) f = open(str(result_filename), "w") - res = [(name, result.__class__.__name__, result.__dict__) + results = [(name, result.__class__.__name__, result.__dict__) for name, result in results] f.write(json.dumps({ - 'revision' : revision, - 'results' : res, - 'options' : options, - 'branch' : branch, + 'revision': revision, + 'results': results, + 'options': options, + 'branch': branch, })) f.close() - if upload: - if ',' in args: - argsbase, argschanged = args.split(',') - else: - argsbase, argschanged = args, args - if 'pypy' in baseline: - perform_upload(pypy_c_path, argsbase, force_host, options, res, - revision, changed=False, postfix=postfix, branch=branch) - perform_upload(pypy_c_path, argschanged, force_host, options, res, - revision, changed=True, postfix=postfix, branch=branch) + return results -BENCHMARK_SET = ['richards', 'slowspitfire', 'django', 'spambayes', - 'rietveld', 'html5lib', 'ai'] -BENCHMARK_SET += perf._FindAllBenchmarks(benchmarks.__dict__).keys() -class WrongBenchmark(Exception): - pass +def get_upload_options(options): + ''' + returns a dict with 2 keys: CHANGED, BASELINE. The values are + dicts with the keys + * 'upload' (boolean) + * 'project' (string) + * 'executable' (string) + * 'urls (list of strings). + * 'branch' (string) + * 'revision' (string) + + This correspondents to the the --upload* and --upload-baseline* + options. + + raises: AssertionError if upload is specified, but not the + corresponding executable or revision. + ''' + + if options.upload_baseline_revision is None: + options.upload_baseline_revision = options.upload_revision + + upload_options = {} + + for run in [CHANGED, BASELINE]: + + def get_upload_option(name): + attr_name = 'upload' + if run == BASELINE: + attr_name = '%s_baseline' % attr_name + if name: + attr_name = '%s_%s' % (attr_name, name) + return getattr(options, attr_name) + + urls = get_upload_option('urls') + urls = [url.strip() for url in urls.split(',') if url.strip()] + upload = get_upload_option(None) + project = get_upload_option('project') + executable = get_upload_option('executable') + branch = get_upload_option('branch') + revision = get_upload_option('revision') + if upload: + if executable is None: + raise AssertionError('If you want to --upload[-baseline] you ' + 'have to specify the corresponding ' + '--upload[-baseline]-executable') + if revision is None: + raise AssertionError('If you want to upload the result you ' + 'have to specify a --revision (or ' + '--upload-baseline-revision if you ' + 'want to upload the baseline result') + if ((run == BASELINE and 'nullpython.py' in options.baseline) or + (run == CHANGED and 'nullpython.py' in options.changed)): + raise AssertionError("Don't upload data from the nullpython " + "dummy interpreter. It won't run any " + "real benchmarks.") + + upload_options[run] = { + 'upload': upload, + 'project': project, + 'executable': executable, + 'urls': urls, + 'branch': branch, + 'revision': revision} + return upload_options + def main(argv): import optparse parser = optparse.OptionParser( usage="%prog [options]", description="Run benchmarks and dump json") - parser.add_option("-b", "--benchmarks", metavar="BM_LIST", - default=','.join(BENCHMARK_SET), - help=("Comma-separated list of benchmarks to run" - " Valid benchmarks are: " + - ", ".join(sorted(BENCHMARK_SET)))) - parser.add_option('-p', '--pypy-c', default=sys.executable, - help='pypy-c or other modified python to run against') - parser.add_option('-r', '--revision', default=0, action="store", - help='specify revision of pypy-c') - parser.add_option('-o', '--output-filename', default="result.json", - action="store", - help='specify output filename to store resulting json') - parser.add_option('--options', default='', action='store', - help='a string describing picked options, no spaces') - parser.add_option('--branch', default='default', action='store', - help="pypy's branch") - parser.add_option('--baseline', default=sys.executable, action='store', - help='baseline interpreter, defaults to host one') - parser.add_option("-a", "--args", default="", - help=("Pass extra arguments to the python binaries." - " If there is a comma in this option's value, the" - " arguments before the comma (interpreted as a" - " space-separated list) are passed to the baseline" - " python, and the arguments after are passed to the" - " changed python. If there's no comma, the same" - " options are passed to both.")) - parser.add_option("--upload", default=False, action="store_true", - help="Upload results to speed.pypy.org") - parser.add_option("--force-host", default=None, action="store", - help="Force the hostname") - parser.add_option("--fast", default=False, action="store_true", - help="Run shorter benchmark runs") - parser.add_option("--full-store", default=False, action="store_true", - help="") - parser.add_option('--postfix', default='', action='store', - help='Append a postfix to uploaded executable') + + # benchmark options + benchmark_group = optparse.OptionGroup( + parser, 'Benchmark options', + ('Options affecting the benchmark runs and the resulting output ' + 'json file.')) + benchmark_group.add_option( + "-b", "--benchmarks", metavar="BM_LIST", + default=','.join(BENCHMARK_SET), + help=("Comma-separated list of benchmarks to run" + " Valid benchmarks are: %s" + ". (default: Run all listed benchmarks)" + ) % ", ".join(sorted(BENCHMARK_SET))) + benchmark_group.add_option( + '-c', '--changed', default=sys.executable, + help=('pypy-c or another modified python interpreter to run against. ' + 'Also named the "changed" interpreter. (default: the python ' + 'used to run this script)')) + benchmark_group.add_option( + '--baseline', default=sys.executable, action='store', + help=('Baseline interpreter. (default: the python used to ' + 'run this script)')) + benchmark_group.add_option( + '-o', '--output-filename', default="result.json", + action="store", + help=('Specify the output filename to store resulting json. ' + '(default: result.json)')) + benchmark_group.add_option( + '--options', default='', action='store', + help='A string describing picked options, no spaces.') + benchmark_group.add_option( + '--branch', default='default', action='store', + dest='upload_branch', + help=("The branch the 'changed' interpreter was compiled from. This " + 'will be store in the result json and used for the upload. ' + "(default: 'default')")) + benchmark_group.add_option( + '-r', '--revision', action="store", + dest='upload_revision', + help=("Specify the revision of the 'changed' interpreter. " + 'This will be store in the ' + 'result json and used for the upload. (default: None)')) + benchmark_group.add_option( + "-a", "--args", default="", + help=("Pass extra arguments to the python binaries." + " If there is a comma in this option's value, the" + " arguments before the comma (interpreted as a" + " space-separated list) are passed to the baseline" + " python, and the arguments after are passed to" + " the changed python. If there's no comma, the" + " same options are passed to both.")) + benchmark_group.add_option( + "--fast", default=False, action="store_true", + help="Run shorter benchmark runs.") + benchmark_group.add_option( + "--full-store", default=False, action="store_true", + help="Run the benchmarks with the --no-statistics flag.") + parser.add_option_group(benchmark_group) + + # upload changed options + upload_group = optparse.OptionGroup( + parser, 'Upload Options', + ('Options for uploading the result of the "changed" python to ' + 'codespeed. The information about revision and branch will ' + 'be taken from the options --revision and --branch.')) + upload_group.add_option( + "--upload", default=None, action="store_true", + help=("Upload results to speed.pypy.org (unless " + "--upload-url is given).")) + upload_group.add_option( + "--upload-urls", default="http://speed.pypy.org/", + help=("Comma seperated urls of the codespeed instances " + "to upload to. (default: http://speed.pypy.org/)")) + upload_group.add_option( + "--upload-project", default="PyPy", + help="The project name in codespeed. (default: PyPy)") + upload_group.add_option( + "--upload-executable", default=None, + help=("The executable name in codespeed. (required if --upload " + "is given)")) + parser.add_option_group(upload_group) + parser.add_option( + "--force-host", default=None, action="store", + help=("Force the hostname. This option will also be used when " + "uploading the baseline result.")) + + # upload baseline group + upload_baseline_group = optparse.OptionGroup( + parser, 'Upload Baseline Options', + ('Options for uploading the result of the "baseline" python to ' + 'codespeed. The hostname of the --force-host option will be used ' + 'in the baseline upload too.')) + upload_baseline_group.add_option( + "--upload-baseline", default=None, action="store_true", + help=("Also upload results or the baseline benchmark " + "to speed.pypy.org (unless " + "--upload-baseline-url is given).")) + upload_baseline_group.add_option( + "--upload-baseline-urls", + default="http://speed.pypy.org/", + help=("Comma seperated urls of the codespeed instances " + "to upload to. (default: http://speed.pypy.org/)")) + upload_baseline_group.add_option( + "--upload-baseline-project", default="PyPy", + help="The project name in codespeed (default: PyPy).") + upload_baseline_group.add_option( + "--upload-baseline-executable", default=None, + help=("The executable name in codespeed. (required if " + "--upload-baseline is given)")) + upload_baseline_group.add_option( + '--upload-baseline-branch', default='default', + action='store', + help=("The name of the branch used for the baseline " + "run. (default: 'default'")) + upload_baseline_group.add_option( + '--upload-baseline-revision', action='store', + default=None, + help=("The revision of the baseline. (required if --upload-baseline " + "is given)")) + parser.add_option_group(upload_baseline_group) + + # Backward compoatibility options + deprecated_group = optparse.OptionGroup( + parser, 'Deprecated Options', + 'Still here for backward compatibility.') + deprecated_group.add_option( + '-p', '--pypy-c', default=sys.executable, + dest='changed', help='Deprecated alias for -c/--changed') + parser.add_option_group(deprecated_group) + options, args = parser.parse_args(argv) - # - # use 'default' if the branch is empty - if not options.branch: - options.branch = 'default' - + + upload_options = get_upload_options(options) benchmarks = options.benchmarks.split(',') for benchmark in benchmarks: if benchmark not in BENCHMARK_SET: raise WrongBenchmark(benchmark) - run_and_store(benchmarks, options.output_filename, options.pypy_c, - options.revision, args=options.args, upload=options.upload, - force_host=options.force_host, fast=options.fast, - baseline=options.baseline, full_store=options.full_store, - postfix=options.postfix, branch=options.branch) + + changed_path = options.changed + baseline_path = options.baseline + fast = options.fast + args = options.args + full_store = options.full_store + output_filename = options.output_filename + + branch = options.upload_branch + revision = options.upload_revision + force_host = options.force_host + + results = run_and_store(benchmarks, output_filename, changed_path, + revision, args=args, fast=fast, + baseline_path=baseline_path, + full_store=full_store, branch=branch) + + for run in [CHANGED, BASELINE]: + upload = upload_options[run]['upload'] + urls = upload_options[run]['urls'] + project = upload_options[run]['project'] + executable = upload_options[run]['executable'] + branch = upload_options[run]['branch'] + revision = upload_options[run]['revision'] + + if upload: + # prevent to upload results from the nullpython dummy + host = force_host if force_host else socket.gethostname() + for url in urls: + print save(project, revision, results, executable, host, url, + changed=(run == CHANGED), branch=branch) + if __name__ == '__main__': main(sys.argv[1:]) diff --git a/saveresults.py b/saveresults.py --- a/saveresults.py +++ b/saveresults.py @@ -12,28 +12,30 @@ Example usage: - $ ./saveresults.py result.json -r '45757:fabe4fc0dc08' -n pypy-c-jit -H tannit - + $ ./saveresults.py result.json -r '45757:fabe4fc0dc08' -n pypy-c-jit \ + -H tannit + OR - - $ ./saveresults.py result.json -r '45757:fabe4fc0dc08' -n pypy-c-jit-64 -H tannit + $ ./saveresults.py result.json -r '45757:fabe4fc0dc08' -n pypy-c-jit-64 \ + -H tannit """ -import sys -import urllib, urllib2, time from datetime import datetime import optparse +import sys +import time +import urllib +import urllib2 -SPEEDURL = "http://speed.pypy.org/" -def save(project, revision, results, options, interpreter, host, testing=False, +def save(project, revision, results, executeable, host, url, testing=False, changed=True, branch='default'): testparams = [] #Parse data data = {} error = 0 - + for b in results: bench_name = b[0] res_type = b[1] @@ -63,7 +65,7 @@ data = { 'commitid': revision, 'project': project, - 'executable': interpreter, + 'executable': executeable, 'benchmark': bench_name, 'environment': host, 'result_value': value, @@ -77,26 +79,33 @@ data['std_dev'] = results['std_changed'] else: data['std_dev'] = results['std_base'] - if testing: testparams.append(data) - else: error |= send(data) + if testing: + testparams.append(data) + else: + error |= send(data, url) + if error: raise IOError("Saving failed. See messages above.") - if testing: return testparams - else: return 0 - -def send(data): + if testing: + return testparams + else: + return 0 + + +def send(data, url): #save results params = urllib.urlencode(data) f = None response = "None" - info = str(datetime.today()) + ": Saving result for " + data['executable'] + " revision " - info += str(data['commitid']) + ", benchmark " + data['benchmark'] + info = ("%s: Saving result for %s revision %s, benchmark %s" % + (str(datetime.today()), data['executable'], + str(data['commitid']), data['benchmark'])) print(info) try: retries = [1, 2, 3, 6] while True: try: - f = urllib2.urlopen(SPEEDURL + 'result/add/', params) + f = urllib2.urlopen(url + 'result/add/', params) response = f.read() f.close() break @@ -116,7 +125,7 @@ print response with open('error.html', 'w') as error_file: error_file.write(response) - print("Server (%s) response written to error.html" % (SPEEDURL,)) + print("Server (%s) response written to error.html" % (url,)) print(' Error code: %s\n' % (e,)) return 1 print "saved correctly!\n" @@ -129,24 +138,36 @@ data = simplejson.load(f) results = data['results'] print 'uploading results...', - save('PyPy', options.revision, results, '', options.name, options.host, - changed=options.changed) + save(options.project, options.revision, results, options.executable, + options.host, options.url, changed=options.changed) print 'done' if __name__ == '__main__': parser = optparse.OptionParser(usage="%prog result.json [options]") - parser.add_option('-r', '--revision', dest='revision', default=None, type=str) - parser.add_option('-n', '--name', dest='name', default=None, type=str) + parser.add_option('-r', '--revision', dest='revision', + default=None, type=str, help='VCS revision (required)') + parser.add_option('-n', '--name', dest='executable', + default=None, type=str, + help=('Name of the executable for codespeed.' + 'Deprecated. Use --e/--executable instead')) + parser.add_option('-e', '--executable', dest='executable', + default=None, type=str, + help='Name of the Executable for codespeed (required).') parser.add_option('-H', '--host', dest='host', default=None, type=str) parser.add_option('-b', '--baseline', dest='changed', default=True, action='store_false', help='upload the results as baseline instead of changed') + parser.add_option('-P', '--project', dest='project', default='PyPy') + parser.add_option('-u', '--url', dest='url', + default="http://speed.pypy.org/", + help=('Url of the codespeed instance ' + '(default: http://speed.pypy.org)')) parser.format_description = lambda fmt: __doc__ parser.description = __doc__ options, args = parser.parse_args() - if options.revision is None or options.name is None or options.host is None or \ - len(args) != 1: + if (options.revision is None or options.executable is None or + options.host is None or len(args) != 1): parser.print_help() sys.exit(2) main(args[0], options) diff --git a/test/test_runner.py b/test/test_runner.py --- a/test/test_runner.py +++ b/test/test_runner.py @@ -1,9 +1,9 @@ - import py import json import sys from runner import run_and_store + def test_run_and_store(): tmpdir = py.test.ensuretemp('bench_runner') resfile = tmpdir.join('results') diff --git a/test/test_saveresults.py b/test/test_saveresults.py --- a/test/test_saveresults.py +++ b/test/test_saveresults.py @@ -22,7 +22,8 @@ def test_good_input(self): '''Given correct result data, check that every result being saved has the right parameters''' - for resultparams in saveresults.save("PyPy", 71212, self.fixture, "", "pypy-c-jit", 'host', True): + for resultparams in saveresults.save("PyPy", 71212, self.fixture, "pypy-c-jit", 'host', 'url', + testing=True): assert resultparams['project'] == "PyPy" assert resultparams['commitid'] == 71212 assert resultparams['executable'] == "pypy-c-jit" From noreply at buildbot.pypy.org Mon Jan 30 19:35:36 2012 From: noreply at buildbot.pypy.org (csenger) Date: Mon, 30 Jan 2012 19:35:36 +0100 (CET) Subject: [pypy-commit] buildbot default: Implement cPythonBenchmark BuildFactory and rename builder Message-ID: <20120130183536.D477182B6C@wyvern.cs.uni-duesseldorf.de> Author: Carsten Senger <senger at rehfisch.de> Branch: Changeset: r625:099a467986cf Date: 2012-01-20 23:55 +0100 http://bitbucket.org/pypy/buildbot/changeset/099a467986cf/ Log: Implement cPythonBenchmark BuildFactory and rename builder * Add new class builds.cPythonBenchmark for speed.python.org related code and clean up builds.JITBenchmark * Add repourl kwarg to setup_steps() * rename builder from jit-benchmark-linux-x86-64 to cpython-2-benchmark-x86-64 diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py --- a/bot2/pypybuildbot/builds.py +++ b/bot2/pypybuildbot/builds.py @@ -192,10 +192,10 @@ command = "hg update --clean", workdir = workdir)) -def setup_steps(platform, factory, workdir=None): +def setup_steps(platform, factory, workdir=None, + repourl='https://bitbucket.org/pypy/pypy/'): # XXX: this assumes that 'hg' is in the path import getpass - repourl = 'https://bitbucket.org/pypy/pypy/' if getpass.getuser() == 'antocuni': # for debugging repourl = '/home/antocuni/pypy/default' @@ -313,21 +313,16 @@ workdir='.', blocksize=100*1024)) + class JITBenchmark(factory.BuildFactory): - def __init__(self, platform='linux', host='tannit', postfix=None): + def __init__(self, platform='linux', postfix=None): factory.BuildFactory.__init__(self) setup_steps(platform, self) # repourl = 'https://bitbucket.org/pypy/benchmarks' update_hg(platform, self, repourl, 'benchmarks', use_branch=False) - # - if host == 'tannit': - lock = TannitCPU - elif host == 'speed_python': - lock = SpeedPythonCPU - else: - assert False, 'unknown host %s' % host + lock = TannitCPU # self.addStep( Translate( @@ -363,3 +358,67 @@ self.addStep(transfer.FileUpload(slavesrc="benchmarks/result.json", masterdest=WithProperties(resfile), workdir=".")) + + +class CPythonBenchmark(factory.BuildFactory): + ''' + Check out and build CPython and run the benchmarks with it. + ''' + def __init__(self, platform='linux64'): + factory.BuildFactory.__init__(self) + + # checks out and updates the repo + setup_steps(platform, self, repourl='http://hg.python.org/cpython') + + # check out and update benchmarks + repourl = 'https://bitbucket.org/pypy/benchmarks' + update_hg(platform, self, repourl, 'benchmarks', use_branch=False) + + lock = SpeedPythonCPU + + self.addStep(ShellCmd( + description="configure cpython", + command=["./configure"], + timeout=300)) + + self.addStep(ShellCmd( + description="cleanup cpython", + command=["make", "clean"], + timeout=300)) + + self.addStep(ShellCmd( + description="make cpython", + command=["make"], + timeout=600)) + + # self.addStep(ShellCmd( + # description="test cpython", + # command=["make", "buildbottest"], + # haltOnFailure=False, + # timeout=600)) + + cpython_interpreter = '../build/python' + self.addStep(ShellCmd( + # this step needs exclusive access to the CPU + locks=[lock.access('exclusive')], + description="run benchmarks on top of cpython", + command=["python", "runner.py", '--output-filename', 'result.json', + '--revision', WithProperties('%(got_revision)s'), + '--branch', WithProperties('%(branch)s'), + '-p', cpython_interpreter, + '--baseline', cpython_interpreter, + ], + workdir='./benchmarks', + timeout=3600)) + + # a bit obscure hack to get both os.path.expand and a property + filename = '%(got_revision)s' + resultfile = os.path.expanduser("~/bench_results/%s.json" % filename) + self.addStep(transfer.FileUpload(slavesrc="benchmarks/result.json", + masterdest=WithProperties(resultfile), + workdir=".")) + + self.addStep(ShellCmd( + description="distcleanup cpython", + command=["make", "distclean"], + timeout=300)) diff --git a/bot2/pypybuildbot/master.py b/bot2/pypybuildbot/master.py --- a/bot2/pypybuildbot/master.py +++ b/bot2/pypybuildbot/master.py @@ -130,9 +130,7 @@ pypyJITBenchmarkFactory64_tannit = pypybuilds.JITBenchmark(platform='linux64', postfix='-64') -pypyJITBenchmarkFactory64_speed = pypybuilds.JITBenchmark(platform='linux64', - postfix='-64', - host='speed_python') +cPythonBenchmarkFactory64 = pypybuilds.CPythonBenchmark(platform='linux64') LINUX32 = "own-linux-x86-32" @@ -157,7 +155,7 @@ JITONLYLINUX32 = "jitonly-own-linux-x86-32" JITBENCH = "jit-benchmark-linux-x86-32" JITBENCH64 = "jit-benchmark-linux-x86-64" -JITBENCH64_2 = "jit-benchmark-linux-x86-64-2" +CPYTHON2_64 = "cpython-2-benchmark-x86-64" BuildmasterConfig = { @@ -204,7 +202,7 @@ Nightly("nightly-0-00", [ JITBENCH, # on tannit32, uses 1 core (in part exclusively) JITBENCH64, # on tannit64, uses 1 core (in part exclusively) - JITBENCH64_2, # on speed.python.org, uses 1 core (in part exclusively) + CPYTHON2_64, # on speed.python.org, uses 1 core (in part exclusively) MACOSX32, # on minime ], branch=None, hour=0, minute=0), # @@ -304,10 +302,10 @@ "category": "benchmark-run", # the locks are acquired with fine grain inside the build }, - {"name": JITBENCH64_2, + {"name": CPYTHON2_64, "slavenames": ["speed-python-64"], - "builddir": JITBENCH64_2, - "factory": pypyJITBenchmarkFactory64_speed, + "builddir": CPYTHON2_64, + "factory": cPythonBenchmarkFactory64, "category": "benchmark-run", # the locks are acquired with fine grain inside the build }, From noreply at buildbot.pypy.org Mon Jan 30 19:35:37 2012 From: noreply at buildbot.pypy.org (csenger) Date: Mon, 30 Jan 2012 19:35:37 +0100 (CET) Subject: [pypy-commit] buildbot default: Force cpython benchmark build to use the 2.7 branch. Message-ID: <20120130183537.EC58082B6C@wyvern.cs.uni-duesseldorf.de> Author: Carsten Senger <senger at rehfisch.de> Branch: Changeset: r626:dff185997530 Date: 2012-01-21 22:30 +0100 http://bitbucket.org/pypy/buildbot/changeset/dff185997530/ Log: Force cpython benchmark build to use the 2.7 branch. The default branch is python 3 with which the benchmarks don't work. * Add a 'branch' parameter to the cpython build factory. * Extend the UpdateCheckout step, setup_steps and update_hg to accept a parameter force_branch and pass that from the cpython build factory. * change the builder to use the 2.7 branch and change the name to be consistent with the other builders. diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py --- a/bot2/pypybuildbot/builds.py +++ b/bot2/pypybuildbot/builds.py @@ -124,19 +124,33 @@ d[key] = summary builder.saveYourself() -# ________________________________________________________________ + +# _______________________________________________________________ class UpdateCheckout(ShellCmd): description = 'hg update' command = 'UNKNOWN' + def __init__(self, workdir=None, haltOnFailure=True, force_branch=None, + **kwargs): + ShellCmd.__init__(self, workdir=workdir, haltOnFailure=haltOnFailure, + **kwargs) + self.force_branch = force_branch + self.addFactoryArguments(force_branch=force_branch) + def start(self): - properties = self.build.getProperties() - branch = properties['branch'] - command = ["hg", "update", "--clean", "-r", branch or 'default'] + if self.force_branch is not None: + branch = self.force_branch + # Note: We could add a warning to the output if we + # ignore the branch set by the user. + else: + properties = self.build.getProperties() + branch = properties['branch'] or 'default' + command = ["hg", "update", "--clean", "-r", branch] self.setCommand(command) ShellCmd.start(self) + class CheckGotRevision(ShellCmd): description = 'got_revision' command = ['hg', 'parents', '--template', '{rev}:{node}'] @@ -155,7 +169,8 @@ self.build.setProperty('got_revision', got_revision, 'got_revision') self.build.setProperty('final_file_name', final_file_name, 'got_revision') -def update_hg(platform, factory, repourl, workdir, use_branch): +def update_hg(platform, factory, repourl, workdir, use_branch, + force_branch=None): if platform == 'win32': command = "if not exist .hg rmdir /q /s ." else: @@ -184,23 +199,26 @@ command = "hg pull", workdir = workdir)) # - if use_branch: + if use_branch or force_branch: factory.addStep(UpdateCheckout(workdir = workdir, - haltOnFailure=True)) + haltOnFailure=True, + force_branch=force_branch)) else: factory.addStep(ShellCmd(description="hg update", command = "hg update --clean", workdir = workdir)) def setup_steps(platform, factory, workdir=None, - repourl='https://bitbucket.org/pypy/pypy/'): + repourl='https://bitbucket.org/pypy/pypy/', + force_branch=None): # XXX: this assumes that 'hg' is in the path import getpass if getpass.getuser() == 'antocuni': # for debugging repourl = '/home/antocuni/pypy/default' # - update_hg(platform, factory, repourl, workdir, use_branch=True) + update_hg(platform, factory, repourl, workdir, use_branch=True, + force_branch=force_branch) # factory.addStep(CheckGotRevision(workdir=workdir)) @@ -363,12 +381,19 @@ class CPythonBenchmark(factory.BuildFactory): ''' Check out and build CPython and run the benchmarks with it. + + This will overwrite the branch even if it was specified + in the buildbot webinterface! ''' - def __init__(self, platform='linux64'): + def __init__(self, branch, platform='linux64'): + ''' + branch: The branch of cpython that will be used. + ''' factory.BuildFactory.__init__(self) # checks out and updates the repo - setup_steps(platform, self, repourl='http://hg.python.org/cpython') + setup_steps(platform, self, repourl='http://hg.python.org/cpython', + force_branch=branch) # check out and update benchmarks repourl = 'https://bitbucket.org/pypy/benchmarks' diff --git a/bot2/pypybuildbot/master.py b/bot2/pypybuildbot/master.py --- a/bot2/pypybuildbot/master.py +++ b/bot2/pypybuildbot/master.py @@ -130,7 +130,8 @@ pypyJITBenchmarkFactory64_tannit = pypybuilds.JITBenchmark(platform='linux64', postfix='-64') -cPythonBenchmarkFactory64 = pypybuilds.CPythonBenchmark(platform='linux64') +cPython27BenchmarkFactory64 = pypybuilds.CPythonBenchmark('2.7', + platform='linux64') LINUX32 = "own-linux-x86-32" @@ -305,7 +306,7 @@ {"name": CPYTHON2_64, "slavenames": ["speed-python-64"], "builddir": CPYTHON2_64, - "factory": cPythonBenchmarkFactory64, + "factory": cPython27BenchmarkFactory64, "category": "benchmark-run", # the locks are acquired with fine grain inside the build }, From noreply at buildbot.pypy.org Mon Jan 30 19:35:39 2012 From: noreply at buildbot.pypy.org (csenger) Date: Mon, 30 Jan 2012 19:35:39 +0100 (CET) Subject: [pypy-commit] buildbot default: Halt build on failures in requred steps. Remove distclean. Message-ID: <20120130183539.0D9F282B6C@wyvern.cs.uni-duesseldorf.de> Author: Carsten Senger <senger at rehfisch.de> Branch: Changeset: r627:a9d31fb4d25c Date: 2012-01-21 22:48 +0100 http://bitbucket.org/pypy/buildbot/changeset/a9d31fb4d25c/ Log: Halt build on failures in requred steps. Remove distclean. * Halt the build on errors in configure, make or the benchmark runner. * Add the `make buildbottest` step again and mark the build as having warnings if it fails. * Remove distclean so we can inspect the build when problems occure. diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py --- a/bot2/pypybuildbot/builds.py +++ b/bot2/pypybuildbot/builds.py @@ -404,7 +404,8 @@ self.addStep(ShellCmd( description="configure cpython", command=["./configure"], - timeout=300)) + timeout=300, + haltOnFailure=True)) self.addStep(ShellCmd( description="cleanup cpython", @@ -414,14 +415,16 @@ self.addStep(ShellCmd( description="make cpython", command=["make"], + timeout=600, + haltOnFailure=True)) + + self.addStep(ShellCmd( + description="test cpython", + command=["make", "buildbottest"], + haltOnFailure=False, + warnOnFailure=True, timeout=600)) - # self.addStep(ShellCmd( - # description="test cpython", - # command=["make", "buildbottest"], - # haltOnFailure=False, - # timeout=600)) - cpython_interpreter = '../build/python' self.addStep(ShellCmd( # this step needs exclusive access to the CPU @@ -434,6 +437,7 @@ '--baseline', cpython_interpreter, ], workdir='./benchmarks', + haltOnFailure=True, timeout=3600)) # a bit obscure hack to get both os.path.expand and a property @@ -442,8 +446,3 @@ self.addStep(transfer.FileUpload(slavesrc="benchmarks/result.json", masterdest=WithProperties(resultfile), workdir=".")) - - self.addStep(ShellCmd( - description="distcleanup cpython", - command=["make", "distclean"], - timeout=300)) From noreply at buildbot.pypy.org Mon Jan 30 19:35:40 2012 From: noreply at buildbot.pypy.org (csenger) Date: Mon, 30 Jan 2012 19:35:40 +0100 (CET) Subject: [pypy-commit] buildbot default: Use the runner options --upload* to control the result upload Message-ID: <20120130183540.21DE282B6C@wyvern.cs.uni-duesseldorf.de> Author: Carsten Senger <senger at rehfisch.de> Branch: Changeset: r628:7eaaa5442817 Date: 2012-01-25 10:52 +0100 http://bitbucket.org/pypy/buildbot/changeset/7eaaa5442817/ Log: Use the runner options --upload* to control the result upload * specify all information required for the upload to codespeed as options to runner * Use the nullpython dummy as the baseline interpreter to prevent duplicate benchmark runs for cpython. We upload only the result of a benchmark, not the difference to the other interpreter to codespeed. diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py --- a/bot2/pypybuildbot/builds.py +++ b/bot2/pypybuildbot/builds.py @@ -333,7 +333,7 @@ class JITBenchmark(factory.BuildFactory): - def __init__(self, platform='linux', postfix=None): + def __init__(self, platform='linux', postfix=''): factory.BuildFactory.__init__(self) setup_steps(platform, self) @@ -351,27 +351,34 @@ locks=[lock.access('counting')], ) ) + pypy_c_rel = "../build/pypy/translator/goal/pypy-c" - if postfix: - addopts = ['--postfix', postfix] - else: - addopts = [] self.addStep(ShellCmd( # this step needs exclusive access to the CPU locks=[lock.access('exclusive')], description="run benchmarks on top of pypy-c", command=["python", "runner.py", '--output-filename', 'result.json', - '--pypy-c', pypy_c_rel, + '--pypy-c', pypy_c_rel, '--baseline', pypy_c_rel, '--args', ',--jit off', '--upload', + '--upload-executable', 'pypy-c' + postfix, + '--upload-project', 'PyPy', '--revision', WithProperties('%(got_revision)s'), '--branch', WithProperties('%(branch)s'), - ] + addopts, + '--upload-urls', 'http://speed.pypy.org/', + '--upload-baseline', + '--upload-baseline-executable', 'pypy-c-jit' + postfix, + '--upload-baseline-project', 'PyPy', + '--upload-baseline-revision', + WithProperties('%(got_revision)s'), + '--upload-baseline-branch', WithProperties('%(branch)s'), + '--upload-baseline-urls', 'http://localhost', + ], workdir='./benchmarks', timeout=3600)) # a bit obscure hack to get both os.path.expand and a property - filename = '%(got_revision)s' + (postfix or '') + filename = '%(got_revision)s' + postfix resfile = os.path.expanduser("~/bench_results/%s.json" % filename) self.addStep(transfer.FileUpload(slavesrc="benchmarks/result.json", masterdest=WithProperties(resfile), @@ -431,10 +438,14 @@ locks=[lock.access('exclusive')], description="run benchmarks on top of cpython", command=["python", "runner.py", '--output-filename', 'result.json', + '--upload', + '--upload-project', 'cpython', + '--upload-executable', 'cpython2', '--revision', WithProperties('%(got_revision)s'), '--branch', WithProperties('%(branch)s'), + '--upload-urls', 'http://localhost/', '-p', cpython_interpreter, - '--baseline', cpython_interpreter, + '--baseline', './nullpython.py', ], workdir='./benchmarks', haltOnFailure=True, From noreply at buildbot.pypy.org Mon Jan 30 19:35:41 2012 From: noreply at buildbot.pypy.org (csenger) Date: Mon, 30 Jan 2012 19:35:41 +0100 (CET) Subject: [pypy-commit] buildbot default: change --pypy-c to --changed Message-ID: <20120130183541.3E8E182B6C@wyvern.cs.uni-duesseldorf.de> Author: Carsten Senger <senger at rehfisch.de> Branch: Changeset: r629:03858640be74 Date: 2012-01-25 14:05 +0100 http://bitbucket.org/pypy/buildbot/changeset/03858640be74/ Log: change --pypy-c to --changed diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py --- a/bot2/pypybuildbot/builds.py +++ b/bot2/pypybuildbot/builds.py @@ -358,7 +358,7 @@ locks=[lock.access('exclusive')], description="run benchmarks on top of pypy-c", command=["python", "runner.py", '--output-filename', 'result.json', - '--pypy-c', pypy_c_rel, + '--changed', pypy_c_rel, '--baseline', pypy_c_rel, '--args', ',--jit off', '--upload', @@ -438,14 +438,14 @@ locks=[lock.access('exclusive')], description="run benchmarks on top of cpython", command=["python", "runner.py", '--output-filename', 'result.json', + '--changed', cpython_interpreter, + '--baseline', './nullpython.py', '--upload', '--upload-project', 'cpython', '--upload-executable', 'cpython2', '--revision', WithProperties('%(got_revision)s'), '--branch', WithProperties('%(branch)s'), '--upload-urls', 'http://localhost/', - '-p', cpython_interpreter, - '--baseline', './nullpython.py', ], workdir='./benchmarks', haltOnFailure=True, From noreply at buildbot.pypy.org Mon Jan 30 19:35:42 2012 From: noreply at buildbot.pypy.org (csenger) Date: Mon, 30 Jan 2012 19:35:42 +0100 (CET) Subject: [pypy-commit] buildbot default: A bit of pep-8 Message-ID: <20120130183542.572F782B6C@wyvern.cs.uni-duesseldorf.de> Author: Carsten Senger <senger at rehfisch.de> Branch: Changeset: r630:ad09bed12f93 Date: 2012-01-25 14:17 +0100 http://bitbucket.org/pypy/buildbot/changeset/ad09bed12f93/ Log: A bit of pep-8 diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py --- a/bot2/pypybuildbot/builds.py +++ b/bot2/pypybuildbot/builds.py @@ -1,6 +1,5 @@ from buildbot.process import factory -from buildbot.steps import source, shell, transfer, master -from buildbot.status.builder import SUCCESS +from buildbot.steps import shell, transfer from buildbot.process.properties import WithProperties from buildbot import locks from pypybuildbot.util import symlink_force @@ -30,6 +29,7 @@ return self.describe(True) + ['aborted'] return shell.ShellCommand.getText(self, cmd, results) + class PyPyUpload(transfer.FileUpload): parms = transfer.FileUpload.parms + ['basename'] @@ -68,6 +68,7 @@ except OSError: pass + class Translate(ShellCmd): name = "translate" description = ["translating"] @@ -124,7 +125,7 @@ d[key] = summary builder.saveYourself() - + # _______________________________________________________________ class UpdateCheckout(ShellCmd): @@ -162,12 +163,15 @@ # '|' in the command-line, because it doesn't work on Windows num = got_revision.find(':') if num > 0: - got_revision = got_revision[:num+13] + got_revision = got_revision[:num + 13] # final_file_name = got_revision.replace(':', '-') # ':' should not be part of filenames --- too many issues - self.build.setProperty('got_revision', got_revision, 'got_revision') - self.build.setProperty('final_file_name', final_file_name, 'got_revision') + self.build.setProperty('got_revision', got_revision, + 'got_revision') + self.build.setProperty('final_file_name', final_file_name, + 'got_revision') + def update_hg(platform, factory, repourl, workdir, use_branch, force_branch=None): @@ -176,8 +180,8 @@ else: command = "if [ ! -d .hg ]; then rm -fr * .[a-z]*; fi" factory.addStep(ShellCmd(description="rmdir?", - command = command, - workdir = workdir, + command=command, + workdir=workdir, haltOnFailure=False)) # if platform == "win32": @@ -186,27 +190,29 @@ command = "if [ ! -d .hg ]; then %s; fi" command = command % ("hg clone -U " + repourl + " .") factory.addStep(ShellCmd(description="hg clone", - command = command, - workdir = workdir, + command=command, + workdir=workdir, haltOnFailure=True)) # - factory.addStep(ShellCmd(description="hg purge", - command = "hg --config extensions.purge= purge --all", - workdir = workdir, - haltOnFailure=True)) + factory.addStep( + ShellCmd(description="hg purge", + command="hg --config extensions.purge=purge --all", + workdir=workdir, + haltOnFailure=True)) # factory.addStep(ShellCmd(description="hg pull", - command = "hg pull", - workdir = workdir)) + command="hg pull", + workdir=workdir)) # if use_branch or force_branch: - factory.addStep(UpdateCheckout(workdir = workdir, + factory.addStep(UpdateCheckout(workdir=workdir, haltOnFailure=True, force_branch=force_branch)) else: factory.addStep(ShellCmd(description="hg update", - command = "hg update --clean", - workdir = workdir)) + command="hg update --clean", + workdir=workdir)) + def setup_steps(platform, factory, workdir=None, repourl='https://bitbucket.org/pypy/pypy/', @@ -239,10 +245,11 @@ "--root=pypy", "--timeout=10800" ] + ["--config=%s" % cfg for cfg in extra_cfgs], logfiles={'pytestLog': 'testrun.log'}, - timeout = 4000, + timeout=4000, env={"PYTHONPATH": ['.'], "PYPYCHERRYPICK": cherrypick})) + class Translated(factory.BuildFactory): def __init__(self, platform='linux', @@ -270,7 +277,7 @@ "--root=pypy", "--timeout=1800" ] + ["--config=%s" % cfg for cfg in app_tests], logfiles={'pytestLog': 'pytest-A.log'}, - timeout = 4000, + timeout=4000, env={"PYTHONPATH": ['.']})) if lib_python: @@ -329,7 +336,7 @@ masterdest=WithProperties(nightly), basename=name + extension, workdir='.', - blocksize=100*1024)) + blocksize=100 * 1024)) class JITBenchmark(factory.BuildFactory): From noreply at buildbot.pypy.org Mon Jan 30 19:35:43 2012 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 30 Jan 2012 19:35:43 +0100 (CET) Subject: [pypy-commit] buildbot default: Merge csenger changes, hopefully it works. Message-ID: <20120130183543.6E94482B6C@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: Changeset: r631:62f77f09abde Date: 2012-01-30 20:35 +0200 http://bitbucket.org/pypy/buildbot/changeset/62f77f09abde/ Log: Merge csenger changes, hopefully it works. diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py --- a/bot2/pypybuildbot/builds.py +++ b/bot2/pypybuildbot/builds.py @@ -1,6 +1,5 @@ from buildbot.process import factory -from buildbot.steps import source, shell, transfer, master -from buildbot.status.builder import SUCCESS +from buildbot.steps import shell, transfer from buildbot.process.properties import WithProperties from buildbot import locks from pypybuildbot.util import symlink_force @@ -30,6 +29,7 @@ return self.describe(True) + ['aborted'] return shell.ShellCommand.getText(self, cmd, results) + class PyPyUpload(transfer.FileUpload): parms = transfer.FileUpload.parms + ['basename'] @@ -138,19 +138,33 @@ d[key] = summary builder.saveYourself() -# ________________________________________________________________ + +# _______________________________________________________________ class UpdateCheckout(ShellCmd): description = 'hg update' command = 'UNKNOWN' + def __init__(self, workdir=None, haltOnFailure=True, force_branch=None, + **kwargs): + ShellCmd.__init__(self, workdir=workdir, haltOnFailure=haltOnFailure, + **kwargs) + self.force_branch = force_branch + self.addFactoryArguments(force_branch=force_branch) + def start(self): - properties = self.build.getProperties() - branch = properties['branch'] - command = ["hg", "update", "--clean", "-r", branch or 'default'] + if self.force_branch is not None: + branch = self.force_branch + # Note: We could add a warning to the output if we + # ignore the branch set by the user. + else: + properties = self.build.getProperties() + branch = properties['branch'] or 'default' + command = ["hg", "update", "--clean", "-r", branch] self.setCommand(command) ShellCmd.start(self) + class CheckGotRevision(ShellCmd): description = 'got_revision' command = ['hg', 'parents', '--template', '{rev}:{node}'] @@ -162,21 +176,25 @@ # '|' in the command-line, because it doesn't work on Windows num = got_revision.find(':') if num > 0: - got_revision = got_revision[:num+13] + got_revision = got_revision[:num + 13] # final_file_name = got_revision.replace(':', '-') # ':' should not be part of filenames --- too many issues - self.build.setProperty('got_revision', got_revision, 'got_revision') - self.build.setProperty('final_file_name', final_file_name, 'got_revision') + self.build.setProperty('got_revision', got_revision, + 'got_revision') + self.build.setProperty('final_file_name', final_file_name, + 'got_revision') -def update_hg(platform, factory, repourl, workdir, use_branch): + +def update_hg(platform, factory, repourl, workdir, use_branch, + force_branch=None): if platform == 'win32': command = "if not exist .hg rmdir /q /s ." else: command = "if [ ! -d .hg ]; then rm -fr * .[a-z]*; fi" factory.addStep(ShellCmd(description="rmdir?", - command = command, - workdir = workdir, + command=command, + workdir=workdir, haltOnFailure=False)) # if platform == "win32": @@ -185,36 +203,41 @@ command = "if [ ! -d .hg ]; then %s; fi" command = command % ("hg clone -U " + repourl + " .") factory.addStep(ShellCmd(description="hg clone", - command = command, - workdir = workdir, + command=command, + workdir=workdir, haltOnFailure=True)) # - factory.addStep(ShellCmd(description="hg purge", - command = "hg --config extensions.purge= purge --all", - workdir = workdir, - haltOnFailure=True)) + factory.addStep( + ShellCmd(description="hg purge", + command="hg --config extensions.purge=purge --all", + workdir=workdir, + haltOnFailure=True)) # factory.addStep(ShellCmd(description="hg pull", - command = "hg pull", - workdir = workdir)) + command="hg pull", + workdir=workdir)) # - if use_branch: - factory.addStep(UpdateCheckout(workdir = workdir, - haltOnFailure=True)) + if use_branch or force_branch: + factory.addStep(UpdateCheckout(workdir=workdir, + haltOnFailure=True, + force_branch=force_branch)) else: factory.addStep(ShellCmd(description="hg update", - command = "hg update --clean", - workdir = workdir)) + command="hg update --clean", + workdir=workdir)) -def setup_steps(platform, factory, workdir=None): + +def setup_steps(platform, factory, workdir=None, + repourl='https://bitbucket.org/pypy/pypy/', + force_branch=None): # XXX: this assumes that 'hg' is in the path import getpass - repourl = 'https://bitbucket.org/pypy/pypy/' if getpass.getuser() == 'antocuni': # for debugging repourl = '/home/antocuni/pypy/default' # - update_hg(platform, factory, repourl, workdir, use_branch=True) + update_hg(platform, factory, repourl, workdir, use_branch=True, + force_branch=force_branch) # factory.addStep(CheckGotRevision(workdir=workdir)) @@ -235,10 +258,11 @@ "--root=pypy", "--timeout=10800" ] + ["--config=%s" % cfg for cfg in extra_cfgs], logfiles={'pytestLog': 'testrun.log'}, - timeout = 4000, + timeout=4000, env={"PYTHONPATH": ['.'], "PYPYCHERRYPICK": cherrypick})) + class Translated(factory.BuildFactory): def __init__(self, platform='linux', @@ -266,7 +290,7 @@ "--root=pypy", "--timeout=1800" ] + ["--config=%s" % cfg for cfg in app_tests], logfiles={'pytestLog': 'pytest-A.log'}, - timeout = 4000, + timeout=4000, env={"PYTHONPATH": ['.']})) if lib_python: @@ -325,23 +349,24 @@ masterdest=WithProperties(nightly), basename=name + extension, workdir='.', - blocksize=100*1024)) + blocksize=100 * 1024)) + class JITBenchmark(factory.BuildFactory): - def __init__(self, platform='linux', host='tannit', postfix=None): + def __init__(self, platform='linux', host='tannit', postfix=''): factory.BuildFactory.__init__(self) setup_steps(platform, self) # repourl = 'https://bitbucket.org/pypy/benchmarks' update_hg(platform, self, repourl, 'benchmarks', use_branch=False) - # if host == 'tannit': lock = TannitCPU elif host == 'speed_python': lock = SpeedPythonCPU else: assert False, 'unknown host %s' % host + # self.addStep( Translate( @@ -367,22 +392,28 @@ masterdest=WithProperties(resfile), workdir=".")) pypy_c_rel = "../build/pypy/translator/goal/pypy-c" - if postfix: - addopts = ['--postfix', postfix] - else: - addopts = [] self.addStep(ShellCmd( # this step needs exclusive access to the CPU locks=[lock.access('exclusive')], description="run benchmarks on top of pypy-c", command=["python", "runner.py", '--output-filename', 'result.json', - '--pypy-c', pypy_c_rel, + '--changed', pypy_c_rel, '--baseline', pypy_c_rel, '--args', ',--jit off', '--upload', + '--upload-executable', 'pypy-c' + postfix, + '--upload-project', 'PyPy', '--revision', WithProperties('%(got_revision)s'), '--branch', WithProperties('%(branch)s'), - ] + addopts, + '--upload-urls', 'http://speed.pypy.org/', + '--upload-baseline', + '--upload-baseline-executable', 'pypy-c-jit' + postfix, + '--upload-baseline-project', 'PyPy', + '--upload-baseline-revision', + WithProperties('%(got_revision)s'), + '--upload-baseline-branch', WithProperties('%(branch)s'), + '--upload-baseline-urls', 'http://localhost', + ], workdir='./benchmarks', timeout=3600)) # a bit obscure hack to get both os.path.expand and a property @@ -391,3 +422,77 @@ self.addStep(transfer.FileUpload(slavesrc="benchmarks/result.json", masterdest=WithProperties(resfile), workdir=".")) + + +class CPythonBenchmark(factory.BuildFactory): + ''' + Check out and build CPython and run the benchmarks with it. + + This will overwrite the branch even if it was specified + in the buildbot webinterface! + ''' + def __init__(self, branch, platform='linux64'): + ''' + branch: The branch of cpython that will be used. + ''' + factory.BuildFactory.__init__(self) + + # checks out and updates the repo + setup_steps(platform, self, repourl='http://hg.python.org/cpython', + force_branch=branch) + + # check out and update benchmarks + repourl = 'https://bitbucket.org/pypy/benchmarks' + update_hg(platform, self, repourl, 'benchmarks', use_branch=False) + + lock = SpeedPythonCPU + + self.addStep(ShellCmd( + description="configure cpython", + command=["./configure"], + timeout=300, + haltOnFailure=True)) + + self.addStep(ShellCmd( + description="cleanup cpython", + command=["make", "clean"], + timeout=300)) + + self.addStep(ShellCmd( + description="make cpython", + command=["make"], + timeout=600, + haltOnFailure=True)) + + self.addStep(ShellCmd( + description="test cpython", + command=["make", "buildbottest"], + haltOnFailure=False, + warnOnFailure=True, + timeout=600)) + + cpython_interpreter = '../build/python' + self.addStep(ShellCmd( + # this step needs exclusive access to the CPU + locks=[lock.access('exclusive')], + description="run benchmarks on top of cpython", + command=["python", "runner.py", '--output-filename', 'result.json', + '--changed', cpython_interpreter, + '--baseline', './nullpython.py', + '--upload', + '--upload-project', 'cpython', + '--upload-executable', 'cpython2', + '--revision', WithProperties('%(got_revision)s'), + '--branch', WithProperties('%(branch)s'), + '--upload-urls', 'http://localhost/', + ], + workdir='./benchmarks', + haltOnFailure=True, + timeout=3600)) + + # a bit obscure hack to get both os.path.expand and a property + filename = '%(got_revision)s' + resultfile = os.path.expanduser("~/bench_results/%s.json" % filename) + self.addStep(transfer.FileUpload(slavesrc="benchmarks/result.json", + masterdest=WithProperties(resultfile), + workdir=".")) diff --git a/bot2/pypybuildbot/master.py b/bot2/pypybuildbot/master.py --- a/bot2/pypybuildbot/master.py +++ b/bot2/pypybuildbot/master.py @@ -132,11 +132,13 @@ pypyJITBenchmarkFactory_tannit = pypybuilds.JITBenchmark() pypyJITBenchmarkFactory64_tannit = pypybuilds.JITBenchmark(platform='linux64', postfix='-64') - pypyJITBenchmarkFactory64_speed = pypybuilds.JITBenchmark(platform='linux64', postfix='-64', host='speed_python') +cPython27BenchmarkFactory64 = pypybuilds.CPythonBenchmark('2.7', + platform='linux64') + LINUX32 = "own-linux-x86-32" LINUX64 = "own-linux-x86-64" @@ -160,7 +162,8 @@ JITONLYLINUX32 = "jitonly-own-linux-x86-32" JITBENCH = "jit-benchmark-linux-x86-32" JITBENCH64 = "jit-benchmark-linux-x86-64" -JITBENCH64_2 = "jit-benchmark-linux-x86-64-2" +JITBENCH64_2 = 'jit-benchmark-linux-x86-64-2' +CPYTHON_64 = "cpython-2-benchmark-x86-64" BuildmasterConfig = { @@ -208,6 +211,7 @@ JITBENCH, # on tannit32, uses 1 core (in part exclusively) JITBENCH64, # on tannit64, uses 1 core (in part exclusively) JITBENCH64_2, # on speed.python.org, uses 1 core (in part exclusively) + CPYTHON_64, # on speed.python.org, uses 1 core (in part exclusively) MACOSX32, # on minime ], branch=None, hour=0, minute=0), # @@ -319,6 +323,13 @@ "category": "benchmark-run", # the locks are acquired with fine grain inside the build }, + {"name": CPYTHON_64, + "slavenames": ["speed-python-64"], + "builddir": CPYTHON_64, + "factory": cPython27BenchmarkFactory64, + "category": "benchmark-run", + # the locks are acquired with fine grain inside the build + }, {"name": MACOSX32, "slavenames": ["minime"], "builddir": MACOSX32, From noreply at buildbot.pypy.org Mon Jan 30 20:02:00 2012 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 30 Jan 2012 20:02:00 +0100 (CET) Subject: [pypy-commit] pypy stm: (fijal looking, arigo) Message-ID: <20120130190200.D5B1082B6A@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: stm Changeset: r51964:23ce401e071d Date: 2012-01-30 20:01 +0100 http://bitbucket.org/pypy/pypy/changeset/23ce401e071d/ Log: (fijal looking, arigo) Start add_epoll() which gives an integration between epoll objects and the transaction module. See test for a demo. diff --git a/pypy/module/transaction/__init__.py b/pypy/module/transaction/__init__.py --- a/pypy/module/transaction/__init__.py +++ b/pypy/module/transaction/__init__.py @@ -9,6 +9,7 @@ 'set_num_threads': 'interp_transaction.set_num_threads', 'add': 'interp_transaction.add', 'run': 'interp_transaction.run', + 'add_epoll': 'interp_epoll.add_epoll', # xxx linux only } appleveldefs = { diff --git a/pypy/module/transaction/interp_epoll.py b/pypy/module/transaction/interp_epoll.py new file mode 100644 --- /dev/null +++ b/pypy/module/transaction/interp_epoll.py @@ -0,0 +1,66 @@ + +# Linux-only + +import os +from select import EPOLLIN +from pypy.rpython.lltypesystem import lltype, rffi +from pypy.interpreter.gateway import unwrap_spec +from pypy.interpreter.error import OperationError +from pypy.module.select.interp_epoll import W_Epoll, FD_SETSIZE +from pypy.module.select.interp_epoll import epoll_event +from pypy.module.select.interp_epoll import epoll_wait +from pypy.module.transaction import interp_transaction +from pypy.rlib import rposix + + +class EPollPending(interp_transaction.AbstractPending): + def __init__(self, space, epoller, w_callback): + self.space = space + self.epoller = epoller + self.w_callback = w_callback + + def run(self): + # this code is run non-transactionally + state = interp_transaction.state + if state.has_exception(): + return + maxevents = FD_SETSIZE - 1 # for now + timeout = 500 # for now: half a second + with lltype.scoped_alloc(rffi.CArray(epoll_event), maxevents) as evs: + nfds = epoll_wait(self.epoller.epfd, evs, maxevents, int(timeout)) + if nfds < 0: + state.got_exception_errno = rposix.get_errno() + state.must_reraise_exception(_reraise_from_errno) + return + for i in range(nfds): + event = evs[i] + fd = rffi.cast(lltype.Signed, event.c_data.c_fd) + PendingCallback(self.w_callback, fd, event.c_events).register() + # re-register myself to epoll_wait() for more + self.register() + + +class PendingCallback(interp_transaction.AbstractPending): + def __init__(self, w_callback, fd, events): + self.w_callback = w_callback + self.fd = fd + self.events = events + + def run_in_transaction(self, space): + space.call_function(self.w_callback, space.wrap(self.fd), + space.wrap(self.events)) + + +def _reraise_from_errno(): + state = interp_transaction.state + space = state.space + errno = state.got_exception_errno + msg = os.strerror(errno) + w_type = space.w_IOError + w_error = space.call_function(w_type, space.wrap(errno), space.wrap(msg)) + raise OperationError(w_type, w_error) + + + at unwrap_spec(epoller=W_Epoll) +def add_epoll(space, epoller, w_callback): + EPollPending(space, epoller, w_callback).register() diff --git a/pypy/module/transaction/interp_transaction.py b/pypy/module/transaction/interp_transaction.py --- a/pypy/module/transaction/interp_transaction.py +++ b/pypy/module/transaction/interp_transaction.py @@ -114,6 +114,19 @@ def unlock_unfinished(self): threadintf.release(self.ll_unfinished_lock) + def init_exceptions(self): + self._reraise_exception = None + + def has_exception(self): + return self._reraise_exception is not None + + def must_reraise_exception(self, reraise_callback): + self._reraise_exception = reraise_callback + + def close_exceptions(self): + if self._reraise_exception is not None: + self._reraise_exception() + state = State() @@ -125,32 +138,45 @@ state.set_num_threads(num) -class Pending: +class AbstractPending(object): _alloc_nonmovable_ = True - def __init__(self, w_callback, args): - self.w_callback = w_callback - self.args = args - def register(self): ec = state.getvalue() ec._transaction_pending.append(self) def run(self): - rstm.perform_transaction(Pending._run_in_transaction, Pending, self) + # may also be overridden + rstm.perform_transaction(AbstractPending._run_in_transaction, + AbstractPending, self) @staticmethod def _run_in_transaction(pending, retry_counter): if retry_counter > 0: pending.register() # retrying: will be done later, try others first return - if state.got_exception is not None: - return # return early if there is already a 'got_exception' + if state.has_exception(): + return # return early if there is already an exception to reraise try: - space = state.space - space.call_args(pending.w_callback, pending.args) + pending.run_in_transaction(state.space) except Exception, e: - state.got_exception = e + state.got_exception_applevel = e + state.must_reraise_exception(_reraise_from_applevel) + + +class Pending(AbstractPending): + def __init__(self, w_callback, args): + self.w_callback = w_callback + self.args = args + + def run_in_transaction(self, space): + space.call_args(self.w_callback, self.args) + + +def _reraise_from_applevel(): + e = state.got_exception_applevel + state.got_exception_applevel = None + raise e def add(space, w_callback, __args__): @@ -217,7 +243,7 @@ state.num_waiting_threads = 0 state.finished = False state.running = True - state.got_exception = None + state.init_exceptions() # for i in range(state.num_threads): threadintf.start_new_thread(_run_thread, ()) @@ -231,7 +257,4 @@ state.running = False # # now re-raise the exception that we got in a transaction - if state.got_exception is not None: - e = state.got_exception - state.got_exception = None - raise e + state.close_exceptions() diff --git a/pypy/module/transaction/test/test_epoll.py b/pypy/module/transaction/test/test_epoll.py new file mode 100644 --- /dev/null +++ b/pypy/module/transaction/test/test_epoll.py @@ -0,0 +1,51 @@ +import py +from pypy.conftest import gettestobjspace + + +class AppTestEpoll: + def setup_class(cls): + cls.space = gettestobjspace(usemodules=['transaction', 'select']) + + def test_non_transactional(self): + import select, posix as os + fd_read, fd_write = os.pipe() + epoller = select.epoll() + epoller.register(fd_read) + os.write(fd_write, 'x') + [(fd, events)] = epoller.poll() + assert fd == fd_read + assert events & select.EPOLLIN + got = os.read(fd_read, 1) + assert got == 'x' + + def test_simple(self): + import transaction, select, posix as os + + steps = [] + + fd_read, fd_write = os.pipe() + + epoller = select.epoll() + epoller.register(fd_read) + + def write_stuff(): + os.write(fd_write, 'x') + steps.append('write_stuff') + + class Done(Exception): + pass + + def callback(fd, events): + assert fd == fd_read + assert events & select.EPOLLIN + got = os.read(fd_read, 1) + assert got == 'x' + steps.append('callback') + raise Done + + transaction.add_epoll(epoller, callback) + transaction.add(write_stuff) + + assert steps == [] + raises(Done, transaction.run) + assert steps == ['write_stuff', 'callback'] From noreply at buildbot.pypy.org Mon Jan 30 20:10:36 2012 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 30 Jan 2012 20:10:36 +0100 (CET) Subject: [pypy-commit] pypy stm: (fijal, arigo) Message-ID: <20120130191036.1594D82B69@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: stm Changeset: r51965:25643ac5bbd8 Date: 2012-01-30 20:05 +0100 http://bitbucket.org/pypy/pypy/changeset/25643ac5bbd8/ Log: (fijal, arigo) Fix for transaction/test/test_ztranslation. diff --git a/pypy/module/select/interp_epoll.py b/pypy/module/select/interp_epoll.py --- a/pypy/module/select/interp_epoll.py +++ b/pypy/module/select/interp_epoll.py @@ -57,6 +57,8 @@ class W_Epoll(Wrappable): + epfd = -1 + def __init__(self, space, epfd): self.epfd = epfd From noreply at buildbot.pypy.org Mon Jan 30 21:04:13 2012 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 30 Jan 2012 21:04:13 +0100 (CET) Subject: [pypy-commit] pypy matrixmath-dot: start of REVIEW Message-ID: <20120130200413.58A1D7106F3@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: matrixmath-dot Changeset: r51966:e37ebbf2a999 Date: 2012-01-30 22:03 +0200 http://bitbucket.org/pypy/pypy/changeset/e37ebbf2a999/ Log: start of REVIEW diff --git a/pypy/module/micronumpy/REVIEW b/pypy/module/micronumpy/REVIEW new file mode 100644 --- /dev/null +++ b/pypy/module/micronumpy/REVIEW @@ -0,0 +1,7 @@ + +Please read the basics of our jit in [1]. Essentially assembler will be +specialized on greens, that's why we do the dance of all the signature stuff, +so it's compiled for each green. 'left' in green is obviously unacceptable. + + +.. [1] http://doc.pypy.org/en/latest/jit/index.html \ No newline at end of file From noreply at buildbot.pypy.org Mon Jan 30 21:20:31 2012 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 30 Jan 2012 21:20:31 +0100 (CET) Subject: [pypy-commit] pypy matrixmath-dot: a small cleanup Message-ID: <20120130202031.37F9D7106F3@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: matrixmath-dot Changeset: r51967:c5c95be3c443 Date: 2012-01-30 22:07 +0200 http://bitbucket.org/pypy/pypy/changeset/c5c95be3c443/ Log: a small cleanup diff --git a/pypy/module/micronumpy/dot.py b/pypy/module/micronumpy/dot.py --- a/pypy/module/micronumpy/dot.py +++ b/pypy/module/micronumpy/dot.py @@ -1,8 +1,6 @@ -from pypy.module.micronumpy import interp_ufuncs from pypy.module.micronumpy.strides import calculate_dot_strides -from pypy.interpreter.error import OperationError, operationerrfmt +from pypy.interpreter.error import OperationError from pypy.module.micronumpy.interp_iter import ViewIterator -from pypy.module.micronumpy.signature import new_printable_location from pypy.rlib import jit @@ -10,8 +8,9 @@ return 'numpy dot [%d dims]' % (shapelen) dot_driver = jit.JitDriver( - greens=['shape_len', 'left'], - reds=['lefti', 'righti', 'outi', 'result', 'right','sig','dtype'], + greens=['shape_len'], + reds=['lefti', 'righti', 'outi', 'result', 'right', 'sig', 'dtype', + 'left'], get_printable_location=dot_printable_location, name='dot', ) @@ -20,12 +19,10 @@ my_critical_dim_size = left.shape[-1] right_critical_dim_size = right.shape[0] right_critical_dim = 0 - right_critical_dim_stride = right.strides[0] out_shape = [] if len(right.shape) > 1: right_critical_dim = len(right.shape) - 2 right_critical_dim_size = right.shape[right_critical_dim] - right_critical_dim_stride = right.strides[right_critical_dim] assert right_critical_dim >= 0 out_shape += left.shape[:-1] + \ right.shape[0:right_critical_dim] + \ @@ -39,7 +36,6 @@ return out_shape, right_critical_dim - at jit.unroll_safe def multidim_dot(space, left, right, result, dtype, right_critical_dim): ''' assumes left, right are concrete arrays given left.shape == [3, 5, 7], @@ -78,14 +74,12 @@ outi=outi, result=result, dtype=dtype, - sig=None, #For get_printable_location ) lval = left.getitem(lefti.offset).convert_to(dtype) rval = right.getitem(righti.offset).convert_to(dtype) outval = result.getitem(outi.offset).convert_to(dtype) v = dtype.itemtype.mul(lval, rval) - value = dtype.itemtype.add(v, outval) - #Do I need to convert it to result.dtype or does settiem do that? + value = dtype.itemtype.add(v, outval).convert_to(dtype) result.setitem(outi.offset, value) outi = outi.next(shape_len) righti = righti.next(shape_len) From noreply at buildbot.pypy.org Mon Jan 30 21:20:32 2012 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 30 Jan 2012 21:20:32 +0100 (CET) Subject: [pypy-commit] pypy matrixmath-dot: small cleanups Message-ID: <20120130202032.79B467106F3@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: matrixmath-dot Changeset: r51968:821ef0f19e33 Date: 2012-01-30 22:17 +0200 http://bitbucket.org/pypy/pypy/changeset/821ef0f19e33/ Log: small cleanups diff --git a/pypy/module/micronumpy/REVIEW b/pypy/module/micronumpy/REVIEW --- a/pypy/module/micronumpy/REVIEW +++ b/pypy/module/micronumpy/REVIEW @@ -2,6 +2,9 @@ Please read the basics of our jit in [1]. Essentially assembler will be specialized on greens, that's why we do the dance of all the signature stuff, so it's compiled for each green. 'left' in green is obviously unacceptable. +I removed this but we still need signature. +More precisely - we want different assembler for the right side being Array +or View. .. [1] http://doc.pypy.org/en/latest/jit/index.html \ No newline at end of file diff --git a/pypy/module/micronumpy/dot.py b/pypy/module/micronumpy/dot.py --- a/pypy/module/micronumpy/dot.py +++ b/pypy/module/micronumpy/dot.py @@ -3,13 +3,12 @@ from pypy.module.micronumpy.interp_iter import ViewIterator from pypy.rlib import jit - -def dot_printable_location(shapelen, sig): - return 'numpy dot [%d dims]' % (shapelen) +def dot_printable_location(shapelen): + return 'numpy dot [%d]' % shapelen dot_driver = jit.JitDriver( - greens=['shape_len'], - reds=['lefti', 'righti', 'outi', 'result', 'right', 'sig', 'dtype', + greens=['shapelen'], + reds=['lefti', 'righti', 'outi', 'result', 'right', 'dtype', 'left'], get_printable_location=dot_printable_location, name='dot', @@ -35,7 +34,6 @@ "objects are not aligned")) return out_shape, right_critical_dim - def multidim_dot(space, left, right, result, dtype, right_critical_dim): ''' assumes left, right are concrete arrays given left.shape == [3, 5, 7], @@ -51,7 +49,7 @@ right should skip 0, 1 ''' broadcast_shape = left.shape[:-1] + right.shape - shape_len = len(broadcast_shape) + shapelen = len(broadcast_shape) left_skip = [len(left.shape) - 1 + i for i in range(len(right.shape)) if i != right_critical_dim] right_skip = range(len(left.shape) - 1) @@ -68,7 +66,7 @@ while not outi.done(): dot_driver.jit_merge_point(left=left, right=right, - shape_len=shape_len, + shapelen=shapelen, lefti=lefti, righti=righti, outi=outi, @@ -81,7 +79,7 @@ v = dtype.itemtype.mul(lval, rval) value = dtype.itemtype.add(v, outval).convert_to(dtype) result.setitem(outi.offset, value) - outi = outi.next(shape_len) - righti = righti.next(shape_len) - lefti = lefti.next(shape_len) + outi = outi.next(shapelen) + righti = righti.next(shapelen) + lefti = lefti.next(shapelen) return result diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -260,13 +260,11 @@ dtype = interp_ufuncs.find_binop_result_dtype(space, self.find_dtype(), other.find_dtype()) if self.size < 1 and other.size < 1: - #numpy compatability + # numpy compatability return scalar_w(space, dtype, space.wrap(0)) - #Do the dims match? + # Do the dims match? out_shape, other_critical_dim = match_dot_shapes(space, self, other) - out_size = 1 - for o in out_shape: - out_size *= o + out_size = support.product(out_shape) result = W_NDimArray(out_size, out_shape, dtype) # This is the place to add fpypy and blas return multidim_dot(space, self.get_concrete(), From noreply at buildbot.pypy.org Mon Jan 30 21:20:33 2012 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 30 Jan 2012 21:20:33 +0100 (CET) Subject: [pypy-commit] pypy matrixmath-dot: done :) Message-ID: <20120130202033.ADEAA7106F3@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: matrixmath-dot Changeset: r51969:3c063218d9d9 Date: 2012-01-30 22:17 +0200 http://bitbucket.org/pypy/pypy/changeset/3c063218d9d9/ Log: done :) diff --git a/pypy/module/micronumpy/REVIEW b/pypy/module/micronumpy/REVIEW deleted file mode 100644 --- a/pypy/module/micronumpy/REVIEW +++ /dev/null @@ -1,10 +0,0 @@ - -Please read the basics of our jit in [1]. Essentially assembler will be -specialized on greens, that's why we do the dance of all the signature stuff, -so it's compiled for each green. 'left' in green is obviously unacceptable. -I removed this but we still need signature. - -More precisely - we want different assembler for the right side being Array -or View. - -.. [1] http://doc.pypy.org/en/latest/jit/index.html \ No newline at end of file From noreply at buildbot.pypy.org Mon Jan 30 21:20:35 2012 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 30 Jan 2012 21:20:35 +0100 (CET) Subject: [pypy-commit] pypy default: (mattip, fijal reviewing) merge matrixmath-dot, this adds 1-d and 2-d dot Message-ID: <20120130202035.6F9EF7106F3@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: Changeset: r51970:97a32a230bff Date: 2012-01-30 22:19 +0200 http://bitbucket.org/pypy/pypy/changeset/97a32a230bff/ Log: (mattip, fijal reviewing) merge matrixmath-dot, this adds 1-d and 2-d dot operations that should work. I did not check the actual numbers ;-) diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py --- a/pypy/module/micronumpy/compile.py +++ b/pypy/module/micronumpy/compile.py @@ -34,7 +34,7 @@ SINGLE_ARG_FUNCTIONS = ["sum", "prod", "max", "min", "all", "any", "unegative", "flat"] -TWO_ARG_FUNCTIONS = ['take'] +TWO_ARG_FUNCTIONS = ["dot", 'take'] class FakeSpace(object): w_ValueError = None @@ -410,13 +410,19 @@ else: assert False # unreachable code elif self.name in TWO_ARG_FUNCTIONS: + if len(self.args) != 2: + raise ArgumentMismatch arg = self.args[1].execute(interp) if not isinstance(arg, BaseArray): raise ArgumentNotAnArray - if self.name == 'take': + if not isinstance(arg, BaseArray): + raise ArgumentNotAnArray + if self.name == "dot": + w_res = arr.descr_dot(interp.space, arg) + elif self.name == 'take': w_res = arr.descr_take(interp.space, arg) else: - assert False # unreachable + assert False # unreachable code else: raise WrongFunctionName if isinstance(w_res, BaseArray): diff --git a/pypy/module/micronumpy/dot.py b/pypy/module/micronumpy/dot.py new file mode 100644 --- /dev/null +++ b/pypy/module/micronumpy/dot.py @@ -0,0 +1,85 @@ +from pypy.module.micronumpy.strides import calculate_dot_strides +from pypy.interpreter.error import OperationError +from pypy.module.micronumpy.interp_iter import ViewIterator +from pypy.rlib import jit + +def dot_printable_location(shapelen): + return 'numpy dot [%d]' % shapelen + +dot_driver = jit.JitDriver( + greens=['shapelen'], + reds=['lefti', 'righti', 'outi', 'result', 'right', 'dtype', + 'left'], + get_printable_location=dot_printable_location, + name='dot', +) + +def match_dot_shapes(space, left, right): + my_critical_dim_size = left.shape[-1] + right_critical_dim_size = right.shape[0] + right_critical_dim = 0 + out_shape = [] + if len(right.shape) > 1: + right_critical_dim = len(right.shape) - 2 + right_critical_dim_size = right.shape[right_critical_dim] + assert right_critical_dim >= 0 + out_shape += left.shape[:-1] + \ + right.shape[0:right_critical_dim] + \ + right.shape[right_critical_dim + 1:] + elif len(right.shape) > 0: + #dot does not reduce for scalars + out_shape += left.shape[:-1] + if my_critical_dim_size != right_critical_dim_size: + raise OperationError(space.w_ValueError, space.wrap( + "objects are not aligned")) + return out_shape, right_critical_dim + +def multidim_dot(space, left, right, result, dtype, right_critical_dim): + ''' assumes left, right are concrete arrays + given left.shape == [3, 5, 7], + right.shape == [2, 7, 4] + then + result.shape == [3, 5, 2, 4] + broadcast shape should be [3, 5, 2, 7, 4] + result should skip dims 3 which is len(result_shape) - 1 + (note that if right is 1d, result should + skip len(result_shape)) + left should skip 2, 4 which is a.ndims-1 + range(right.ndims) + except where it==(right.ndims-2) + right should skip 0, 1 + ''' + broadcast_shape = left.shape[:-1] + right.shape + shapelen = len(broadcast_shape) + left_skip = [len(left.shape) - 1 + i for i in range(len(right.shape)) + if i != right_critical_dim] + right_skip = range(len(left.shape) - 1) + result_skip = [len(result.shape) - (len(right.shape) > 1)] + _r = calculate_dot_strides(result.strides, result.backstrides, + broadcast_shape, result_skip) + outi = ViewIterator(result.start, _r[0], _r[1], broadcast_shape) + _r = calculate_dot_strides(left.strides, left.backstrides, + broadcast_shape, left_skip) + lefti = ViewIterator(left.start, _r[0], _r[1], broadcast_shape) + _r = calculate_dot_strides(right.strides, right.backstrides, + broadcast_shape, right_skip) + righti = ViewIterator(right.start, _r[0], _r[1], broadcast_shape) + while not outi.done(): + dot_driver.jit_merge_point(left=left, + right=right, + shapelen=shapelen, + lefti=lefti, + righti=righti, + outi=outi, + result=result, + dtype=dtype, + ) + lval = left.getitem(lefti.offset).convert_to(dtype) + rval = right.getitem(righti.offset).convert_to(dtype) + outval = result.getitem(outi.offset).convert_to(dtype) + v = dtype.itemtype.mul(lval, rval) + value = dtype.itemtype.add(v, outval).convert_to(dtype) + result.setitem(outi.offset, value) + outi = outi.next(shapelen) + righti = righti.next(shapelen) + lefti = lefti.next(shapelen) + return result diff --git a/pypy/module/micronumpy/interp_iter.py b/pypy/module/micronumpy/interp_iter.py --- a/pypy/module/micronumpy/interp_iter.py +++ b/pypy/module/micronumpy/interp_iter.py @@ -2,7 +2,7 @@ from pypy.rlib import jit from pypy.rlib.objectmodel import instantiate from pypy.module.micronumpy.strides import calculate_broadcast_strides,\ - calculate_slice_strides + calculate_slice_strides, calculate_dot_strides """ This is a mini-tutorial on iterators, strides, and memory layout. It assumes you are familiar with the terms, see @@ -76,6 +76,7 @@ def __init__(self, res_shape): self.res_shape = res_shape + class BaseIterator(object): def next(self, shapelen): raise NotImplementedError @@ -227,6 +228,7 @@ def transform(self, arr, t): pass + class AxisIterator(BaseIterator): def __init__(self, start, dim, shape, strides, backstrides): self.res_shape = shape[:] diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -7,6 +7,7 @@ from pypy.module.micronumpy.strides import (calculate_slice_strides, shape_agreement, find_shape_and_elems, get_shape_from_iterable, calc_new_strides, to_coords) +from dot import multidim_dot, match_dot_shapes from pypy.rlib import jit from pypy.rpython.lltypesystem import lltype, rffi from pypy.tool.sourcetools import func_with_new_name @@ -203,6 +204,7 @@ frame.next(shapelen) idx += 1 return result + def impl(self, space): if self.size == 0: raise OperationError(space.w_ValueError, @@ -248,13 +250,26 @@ descr_argmin = _reduce_argmax_argmin_impl("min") def descr_dot(self, space, w_other): - w_other = convert_to_array(space, w_other) - if isinstance(w_other, Scalar): - return self.descr_mul(space, w_other) - else: - w_res = self.descr_mul(space, w_other) + other = convert_to_array(space, w_other) + if isinstance(other, Scalar): + return self.descr_mul(space, other) + elif len(self.shape) < 2 and len(other.shape) < 2: + w_res = self.descr_mul(space, other) assert isinstance(w_res, BaseArray) return w_res.descr_sum(space, space.wrap(-1)) + dtype = interp_ufuncs.find_binop_result_dtype(space, + self.find_dtype(), other.find_dtype()) + if self.size < 1 and other.size < 1: + # numpy compatability + return scalar_w(space, dtype, space.wrap(0)) + # Do the dims match? + out_shape, other_critical_dim = match_dot_shapes(space, self, other) + out_size = support.product(out_shape) + result = W_NDimArray(out_size, out_shape, dtype) + # This is the place to add fpypy and blas + return multidim_dot(space, self.get_concrete(), + other.get_concrete(), result, dtype, + other_critical_dim) def get_concrete(self): raise NotImplementedError @@ -1186,6 +1201,8 @@ return space.wrap(s) def dot(space, w_obj, w_obj2): + '''see numpypy.dot. Does not exist as an ndarray method in numpy. + ''' w_arr = convert_to_array(space, w_obj) if isinstance(w_arr, Scalar): return convert_to_array(space, w_obj2).descr_dot(space, w_arr) diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py --- a/pypy/module/micronumpy/interp_ufuncs.py +++ b/pypy/module/micronumpy/interp_ufuncs.py @@ -216,17 +216,17 @@ sig=sig, identity=identity, shapelen=shapelen, arr=arr) - iter = frame.get_final_iter() + iterator = frame.get_final_iter() v = sig.eval(frame, arr).convert_to(sig.calc_dtype) - if iter.first_line: + if iterator.first_line: if identity is not None: value = self.func(sig.calc_dtype, identity, v) else: value = v else: - cur = arr.left.getitem(iter.offset) + cur = arr.left.getitem(iterator.offset) value = self.func(sig.calc_dtype, cur, v) - arr.left.setitem(iter.offset, value) + arr.left.setitem(iterator.offset, value) frame.next(shapelen) def reduce_loop(self, shapelen, sig, frame, value, obj, dtype): diff --git a/pypy/module/micronumpy/signature.py b/pypy/module/micronumpy/signature.py --- a/pypy/module/micronumpy/signature.py +++ b/pypy/module/micronumpy/signature.py @@ -353,7 +353,6 @@ assert isinstance(arr, Call2) lhs = self.left.eval(frame, arr.left).convert_to(self.calc_dtype) rhs = self.right.eval(frame, arr.right).convert_to(self.calc_dtype) - return self.binfunc(self.calc_dtype, lhs, rhs) def debug_repr(self): diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py --- a/pypy/module/micronumpy/strides.py +++ b/pypy/module/micronumpy/strides.py @@ -230,3 +230,18 @@ n_old_elems_to_use *= old_shape[oldI] assert len(new_strides) == len(new_shape) return new_strides + + +def calculate_dot_strides(strides, backstrides, res_shape, skip_dims): + rstrides = [] + rbackstrides = [] + j=0 + for i in range(len(res_shape)): + if i in skip_dims: + rstrides.append(0) + rbackstrides.append(0) + else: + rstrides.append(strides[j]) + rbackstrides.append(backstrides[j]) + j += 1 + return rstrides, rbackstrides diff --git a/pypy/module/micronumpy/test/test_compile.py b/pypy/module/micronumpy/test/test_compile.py --- a/pypy/module/micronumpy/test/test_compile.py +++ b/pypy/module/micronumpy/test/test_compile.py @@ -246,6 +246,15 @@ """) assert interp.results[0].value == 11 + def test_dot(self): + interp = self.run(""" + a = [[1, 2], [3, 4]] + b = [[5, 6], [7, 8]] + c = dot(a, b) + c -> 0 -> 0 + """) + assert interp.results[0].value == 19 + def test_flat_iter(self): interp = self.run(''' a = |30| diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -559,6 +559,7 @@ b = a * a for i in range(5): assert b[i] == i * i + assert b.dtype is a.dtype a = _numpypy.array(range(5), dtype=bool) b = a * a @@ -784,8 +785,8 @@ def test_sum(self): from _numpypy import array a = array(range(5)) - assert a.sum() == 10.0 - assert a[:4].sum() == 6.0 + assert a.sum() == 10 + assert a[:4].sum() == 6 a = array([True] * 5, bool) assert a.sum() == 5 @@ -910,15 +911,36 @@ assert c.any() == False def test_dot(self): - from _numpypy import array, dot + from _numpypy import array, dot, arange a = array(range(5)) - assert a.dot(a) == 30.0 + assert dot(a, a) == 30.0 a = array(range(5)) assert a.dot(range(5)) == 30 assert dot(range(5), range(5)) == 30 assert (dot(5, [1, 2, 3]) == [5, 10, 15]).all() + a = arange(12).reshape(3, 4) + b = arange(12).reshape(4, 3) + c = a.dot(b) + assert (c == [[ 42, 48, 54], [114, 136, 158], [186, 224, 262]]).all() + + a = arange(24).reshape(2, 3, 4) + raises(ValueError, "a.dot(a)") + b = a[0, :, :].T + #Superfluous shape test makes the intention of the test clearer + assert a.shape == (2, 3, 4) + assert b.shape == (4, 3) + c = dot(a, b) + assert (c == [[[14, 38, 62], [38, 126, 214], [62, 214, 366]], + [[86, 302, 518], [110, 390, 670], [134, 478, 822]]]).all() + c = dot(a, b[:, 2]) + assert (c == [[62, 214, 366], [518, 670, 822]]).all() + a = arange(3*4*5*6).reshape((3,4,5,6)) + b = arange(3*4*5*6)[::-1].reshape((5,4,6,3)) + assert dot(a, b)[2,3,2,1,2,2] == 499128 + assert sum(a[2,3,2,:] * b[1,2,:,2]) == 499128 + def test_dot_constant(self): from _numpypy import array a = array(range(5)) diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py --- a/pypy/module/micronumpy/test/test_zjit.py +++ b/pypy/module/micronumpy/test/test_zjit.py @@ -456,6 +456,30 @@ 'int_rshift': 1, }) + def define_dot(): + return """ + a = [[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]] + b=[[0, 1, 2], [3, 4, 5], [6, 7, 8], [9, 10, 11]] + c = dot(a, b) + c -> 1 -> 2 + """ + + def test_dot(self): + result = self.run("dot") + assert result == 184 + self.check_simple_loop({'arraylen_gc': 9, + 'float_add': 1, + 'float_mul': 1, + 'getinteriorfield_raw': 3, + 'guard_false': 3, + 'guard_true': 3, + 'int_add': 6, + 'int_lt': 6, + 'int_sub': 3, + 'jump': 1, + 'setinteriorfield_raw': 1}) + + class TestNumpyOld(LLJitMixin): def setup_class(cls): py.test.skip("old") From noreply at buildbot.pypy.org Mon Jan 30 21:20:36 2012 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 30 Jan 2012 21:20:36 +0100 (CET) Subject: [pypy-commit] pypy matrixmath-dot: close merged branch Message-ID: <20120130202036.9BEB97106F3@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: matrixmath-dot Changeset: r51971:cc88cec43893 Date: 2012-01-30 22:19 +0200 http://bitbucket.org/pypy/pypy/changeset/cc88cec43893/ Log: close merged branch From noreply at buildbot.pypy.org Mon Jan 30 21:27:53 2012 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 30 Jan 2012 21:27:53 +0100 (CET) Subject: [pypy-commit] pypy stm: Fix test. Message-ID: <20120130202753.5646F7106F3@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: stm Changeset: r51972:6ebc74fdd21c Date: 2012-01-30 21:26 +0100 http://bitbucket.org/pypy/pypy/changeset/6ebc74fdd21c/ Log: Fix test. diff --git a/pypy/rpython/lltypesystem/test/test_lltype.py b/pypy/rpython/lltypesystem/test/test_lltype.py --- a/pypy/rpython/lltypesystem/test/test_lltype.py +++ b/pypy/rpython/lltypesystem/test/test_lltype.py @@ -792,7 +792,7 @@ # class FieldListAccessor(object): def __init__(self, fields): - self.fields = fields + self._fields = fields S = GcStruct('S', ('x', lltype.Signed), hints={'immutable_fields': FieldListAccessor({'x': 1234})}) assert S._immutable_field('x') == 1234 From noreply at buildbot.pypy.org Mon Jan 30 21:27:54 2012 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 30 Jan 2012 21:27:54 +0100 (CET) Subject: [pypy-commit] pypy stm: Fix the case of lltype.FixedSizeArray. Message-ID: <20120130202754.8DBBF7106F3@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: stm Changeset: r51973:0d8ec011d011 Date: 2012-01-30 21:27 +0100 http://bitbucket.org/pypy/pypy/changeset/0d8ec011d011/ Log: Fix the case of lltype.FixedSizeArray. diff --git a/pypy/rpython/lltypesystem/lltype.py b/pypy/rpython/lltypesystem/lltype.py --- a/pypy/rpython/lltypesystem/lltype.py +++ b/pypy/rpython/lltypesystem/lltype.py @@ -355,10 +355,10 @@ n = 1 return _struct(self, n, initialization='example') - def _immutable_field(self, field): + def _immutable_field(self, field=None): if self._hints.get('immutable'): return True - if 'immutable_fields' in self._hints: + if field is not None and 'immutable_fields' in self._hints: try: return self._hints['immutable_fields']._fields[field] except KeyError: diff --git a/pypy/rpython/lltypesystem/test/test_lltype.py b/pypy/rpython/lltypesystem/test/test_lltype.py --- a/pypy/rpython/lltypesystem/test/test_lltype.py +++ b/pypy/rpython/lltypesystem/test/test_lltype.py @@ -840,6 +840,12 @@ immut = A._immutable_interiorfield([4, 'a']) assert immut == expected +def test_immutable_fixedsizearray(): + A = lltype.FixedSizeArray(lltype.Signed, 5) + assert not A._immutable_field() + A = lltype.FixedSizeArray(lltype.Signed, 5, hints={'immutable': True}) + assert A._immutable_field() + def test_typedef(): T = Typedef(Signed, 'T') From noreply at buildbot.pypy.org Mon Jan 30 22:12:17 2012 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 30 Jan 2012 22:12:17 +0100 (CET) Subject: [pypy-commit] pypy merge-2.7.2: close merged branch Message-ID: <20120130211217.7AC807106F4@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: merge-2.7.2 Changeset: r51975:fdfd5bbea0c4 Date: 2012-01-30 23:11 +0200 http://bitbucket.org/pypy/pypy/changeset/fdfd5bbea0c4/ Log: close merged branch From noreply at buildbot.pypy.org Mon Jan 30 22:12:16 2012 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 30 Jan 2012 22:12:16 +0100 (CET) Subject: [pypy-commit] pypy default: (amaury, agaynor) merge merge-2.7.2, this makes pypy up to date with cpython 2.7 branch Message-ID: <20120130211216.28CE07106F3@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: Changeset: r51974:82067a60a6cd Date: 2012-01-30 23:11 +0200 http://bitbucket.org/pypy/pypy/changeset/82067a60a6cd/ Log: (amaury, agaynor) merge merge-2.7.2, this makes pypy up to date with cpython 2.7 branch diff too long, truncating to 10000 out of 129861 lines diff --git a/lib-python/2.7/BaseHTTPServer.py b/lib-python/2.7/BaseHTTPServer.py --- a/lib-python/2.7/BaseHTTPServer.py +++ b/lib-python/2.7/BaseHTTPServer.py @@ -310,7 +310,13 @@ """ try: - self.raw_requestline = self.rfile.readline() + self.raw_requestline = self.rfile.readline(65537) + if len(self.raw_requestline) > 65536: + self.requestline = '' + self.request_version = '' + self.command = '' + self.send_error(414) + return if not self.raw_requestline: self.close_connection = 1 return diff --git a/lib-python/2.7/ConfigParser.py b/lib-python/2.7/ConfigParser.py --- a/lib-python/2.7/ConfigParser.py +++ b/lib-python/2.7/ConfigParser.py @@ -545,6 +545,38 @@ if isinstance(val, list): options[name] = '\n'.join(val) +import UserDict as _UserDict + +class _Chainmap(_UserDict.DictMixin): + """Combine multiple mappings for successive lookups. + + For example, to emulate Python's normal lookup sequence: + + import __builtin__ + pylookup = _Chainmap(locals(), globals(), vars(__builtin__)) + """ + + def __init__(self, *maps): + self._maps = maps + + def __getitem__(self, key): + for mapping in self._maps: + try: + return mapping[key] + except KeyError: + pass + raise KeyError(key) + + def keys(self): + result = [] + seen = set() + for mapping in self_maps: + for key in mapping: + if key not in seen: + result.append(key) + seen.add(key) + return result + class ConfigParser(RawConfigParser): def get(self, section, option, raw=False, vars=None): @@ -559,16 +591,18 @@ The section DEFAULT is special. """ - d = self._defaults.copy() + sectiondict = {} try: - d.update(self._sections[section]) + sectiondict = self._sections[section] except KeyError: if section != DEFAULTSECT: raise NoSectionError(section) # Update with the entry specific variables + vardict = {} if vars: for key, value in vars.items(): - d[self.optionxform(key)] = value + vardict[self.optionxform(key)] = value + d = _Chainmap(vardict, sectiondict, self._defaults) option = self.optionxform(option) try: value = d[option] diff --git a/lib-python/2.7/Cookie.py b/lib-python/2.7/Cookie.py --- a/lib-python/2.7/Cookie.py +++ b/lib-python/2.7/Cookie.py @@ -258,6 +258,11 @@ '\033' : '\\033', '\034' : '\\034', '\035' : '\\035', '\036' : '\\036', '\037' : '\\037', + # Because of the way browsers really handle cookies (as opposed + # to what the RFC says) we also encode , and ; + + ',' : '\\054', ';' : '\\073', + '"' : '\\"', '\\' : '\\\\', '\177' : '\\177', '\200' : '\\200', '\201' : '\\201', diff --git a/lib-python/2.7/HTMLParser.py b/lib-python/2.7/HTMLParser.py --- a/lib-python/2.7/HTMLParser.py +++ b/lib-python/2.7/HTMLParser.py @@ -26,7 +26,7 @@ tagfind = re.compile('[a-zA-Z][-.a-zA-Z0-9:_]*') attrfind = re.compile( r'\s*([a-zA-Z_][-.:a-zA-Z_0-9]*)(\s*=\s*' - r'(\'[^\']*\'|"[^"]*"|[-a-zA-Z0-9./,:;+*%?!&$_#=~@]*))?') + r'(\'[^\']*\'|"[^"]*"|[^\s"\'=<>`]*))?') locatestarttagend = re.compile(r""" <[a-zA-Z][-.a-zA-Z0-9:_]* # tag name @@ -99,7 +99,7 @@ markupbase.ParserBase.reset(self) def feed(self, data): - """Feed data to the parser. + r"""Feed data to the parser. Call this as often as you want, with as little or as much text as you want (may include '\n'). @@ -367,13 +367,16 @@ return s def replaceEntities(s): s = s.groups()[0] - if s[0] == "#": - s = s[1:] - if s[0] in ['x','X']: - c = int(s[1:], 16) - else: - c = int(s) - return unichr(c) + try: + if s[0] == "#": + s = s[1:] + if s[0] in ['x','X']: + c = int(s[1:], 16) + else: + c = int(s) + return unichr(c) + except ValueError: + return '&#'+s+';' else: # Cannot use name2codepoint directly, because HTMLParser supports apos, # which is not part of HTML 4 diff --git a/lib-python/2.7/SimpleHTTPServer.py b/lib-python/2.7/SimpleHTTPServer.py --- a/lib-python/2.7/SimpleHTTPServer.py +++ b/lib-python/2.7/SimpleHTTPServer.py @@ -15,6 +15,7 @@ import BaseHTTPServer import urllib import cgi +import sys import shutil import mimetypes try: @@ -131,7 +132,8 @@ length = f.tell() f.seek(0) self.send_response(200) - self.send_header("Content-type", "text/html") + encoding = sys.getfilesystemencoding() + self.send_header("Content-type", "text/html; charset=%s" % encoding) self.send_header("Content-Length", str(length)) self.end_headers() return f diff --git a/lib-python/2.7/SimpleXMLRPCServer.py b/lib-python/2.7/SimpleXMLRPCServer.py --- a/lib-python/2.7/SimpleXMLRPCServer.py +++ b/lib-python/2.7/SimpleXMLRPCServer.py @@ -246,7 +246,7 @@ marshalled data. For backwards compatibility, a dispatch function can be provided as an argument (see comment in SimpleXMLRPCRequestHandler.do_POST) but overriding the - existing method through subclassing is the prefered means + existing method through subclassing is the preferred means of changing method dispatch behavior. """ diff --git a/lib-python/2.7/SocketServer.py b/lib-python/2.7/SocketServer.py --- a/lib-python/2.7/SocketServer.py +++ b/lib-python/2.7/SocketServer.py @@ -675,7 +675,7 @@ # A timeout to apply to the request socket, if not None. timeout = None - # Disable nagle algoritm for this socket, if True. + # Disable nagle algorithm for this socket, if True. # Use only when wbufsize != 0, to avoid small packets. disable_nagle_algorithm = False diff --git a/lib-python/2.7/StringIO.py b/lib-python/2.7/StringIO.py --- a/lib-python/2.7/StringIO.py +++ b/lib-python/2.7/StringIO.py @@ -266,6 +266,7 @@ 8th bit) will cause a UnicodeError to be raised when getvalue() is called. """ + _complain_ifclosed(self.closed) if self.buflist: self.buf += ''.join(self.buflist) self.buflist = [] diff --git a/lib-python/2.7/_abcoll.py b/lib-python/2.7/_abcoll.py --- a/lib-python/2.7/_abcoll.py +++ b/lib-python/2.7/_abcoll.py @@ -82,7 +82,7 @@ @classmethod def __subclasshook__(cls, C): if cls is Iterator: - if _hasattr(C, "next"): + if _hasattr(C, "next") and _hasattr(C, "__iter__"): return True return NotImplemented diff --git a/lib-python/2.7/_pyio.py b/lib-python/2.7/_pyio.py --- a/lib-python/2.7/_pyio.py +++ b/lib-python/2.7/_pyio.py @@ -16,6 +16,7 @@ import io from io import (__all__, SEEK_SET, SEEK_CUR, SEEK_END) +from errno import EINTR __metaclass__ = type @@ -559,7 +560,11 @@ if not data: break res += data - return bytes(res) + if res: + return bytes(res) + else: + # b'' or None + return data def readinto(self, b): """Read up to len(b) bytes into b. @@ -678,7 +683,7 @@ """ def __init__(self, raw): - self.raw = raw + self._raw = raw ### Positioning ### @@ -722,8 +727,8 @@ if self.raw is None: raise ValueError("raw stream already detached") self.flush() - raw = self.raw - self.raw = None + raw = self._raw + self._raw = None return raw ### Inquiries ### @@ -738,6 +743,10 @@ return self.raw.writable() @property + def raw(self): + return self._raw + + @property def closed(self): return self.raw.closed @@ -933,7 +942,12 @@ current_size = 0 while True: # Read until EOF or until read() would block. - chunk = self.raw.read() + try: + chunk = self.raw.read() + except IOError as e: + if e.errno != EINTR: + raise + continue if chunk in empty_values: nodata_val = chunk break @@ -952,7 +966,12 @@ chunks = [buf[pos:]] wanted = max(self.buffer_size, n) while avail < n: - chunk = self.raw.read(wanted) + try: + chunk = self.raw.read(wanted) + except IOError as e: + if e.errno != EINTR: + raise + continue if chunk in empty_values: nodata_val = chunk break @@ -981,7 +1000,14 @@ have = len(self._read_buf) - self._read_pos if have < want or have <= 0: to_read = self.buffer_size - have - current = self.raw.read(to_read) + while True: + try: + current = self.raw.read(to_read) + except IOError as e: + if e.errno != EINTR: + raise + continue + break if current: self._read_buf = self._read_buf[self._read_pos:] + current self._read_pos = 0 @@ -1088,7 +1114,12 @@ written = 0 try: while self._write_buf: - n = self.raw.write(self._write_buf) + try: + n = self.raw.write(self._write_buf) + except IOError as e: + if e.errno != EINTR: + raise + continue if n > len(self._write_buf) or n < 0: raise IOError("write() returned incorrect number of bytes") del self._write_buf[:n] @@ -1456,7 +1487,7 @@ if not isinstance(errors, basestring): raise ValueError("invalid errors: %r" % errors) - self.buffer = buffer + self._buffer = buffer self._line_buffering = line_buffering self._encoding = encoding self._errors = errors @@ -1511,6 +1542,10 @@ def line_buffering(self): return self._line_buffering + @property + def buffer(self): + return self._buffer + def seekable(self): return self._seekable @@ -1724,8 +1759,8 @@ if self.buffer is None: raise ValueError("buffer is already detached") self.flush() - buffer = self.buffer - self.buffer = None + buffer = self._buffer + self._buffer = None return buffer def seek(self, cookie, whence=0): diff --git a/lib-python/2.7/_weakrefset.py b/lib-python/2.7/_weakrefset.py --- a/lib-python/2.7/_weakrefset.py +++ b/lib-python/2.7/_weakrefset.py @@ -66,7 +66,11 @@ return sum(x() is not None for x in self.data) def __contains__(self, item): - return ref(item) in self.data + try: + wr = ref(item) + except TypeError: + return False + return wr in self.data def __reduce__(self): return (self.__class__, (list(self),), diff --git a/lib-python/2.7/anydbm.py b/lib-python/2.7/anydbm.py --- a/lib-python/2.7/anydbm.py +++ b/lib-python/2.7/anydbm.py @@ -29,17 +29,8 @@ list = d.keys() # return a list of all existing keys (slow!) Future versions may change the order in which implementations are -tested for existence, add interfaces to other dbm-like +tested for existence, and add interfaces to other dbm-like implementations. - -The open function has an optional second argument. This can be 'r', -for read-only access, 'w', for read-write access of an existing -database, 'c' for read-write access to a new or existing database, and -'n' for read-write access to a new database. The default is 'r'. - -Note: 'r' and 'w' fail if the database doesn't exist; 'c' creates it -only if it doesn't exist; and 'n' always creates a new database. - """ class error(Exception): @@ -63,7 +54,18 @@ error = tuple(_errors) -def open(file, flag = 'r', mode = 0666): +def open(file, flag='r', mode=0666): + """Open or create database at path given by *file*. + + Optional argument *flag* can be 'r' (default) for read-only access, 'w' + for read-write access of an existing database, 'c' for read-write access + to a new or existing database, and 'n' for read-write access to a new + database. + + Note: 'r' and 'w' fail if the database doesn't exist; 'c' creates it + only if it doesn't exist; and 'n' always creates a new database. + """ + # guess the type of an existing database from whichdb import whichdb result=whichdb(file) diff --git a/lib-python/2.7/argparse.py b/lib-python/2.7/argparse.py --- a/lib-python/2.7/argparse.py +++ b/lib-python/2.7/argparse.py @@ -82,6 +82,7 @@ ] +import collections as _collections import copy as _copy import os as _os import re as _re @@ -1037,7 +1038,7 @@ self._prog_prefix = prog self._parser_class = parser_class - self._name_parser_map = {} + self._name_parser_map = _collections.OrderedDict() self._choices_actions = [] super(_SubParsersAction, self).__init__( @@ -1080,7 +1081,7 @@ parser = self._name_parser_map[parser_name] except KeyError: tup = parser_name, ', '.join(self._name_parser_map) - msg = _('unknown parser %r (choices: %s)' % tup) + msg = _('unknown parser %r (choices: %s)') % tup raise ArgumentError(self, msg) # parse all the remaining options into the namespace @@ -1109,7 +1110,7 @@ the builtin open() function. """ - def __init__(self, mode='r', bufsize=None): + def __init__(self, mode='r', bufsize=-1): self._mode = mode self._bufsize = bufsize @@ -1121,18 +1122,19 @@ elif 'w' in self._mode: return _sys.stdout else: - msg = _('argument "-" with mode %r' % self._mode) + msg = _('argument "-" with mode %r') % self._mode raise ValueError(msg) # all other arguments are used as file names - if self._bufsize: + try: return open(string, self._mode, self._bufsize) - else: - return open(string, self._mode) + except IOError as e: + message = _("can't open '%s': %s") + raise ArgumentTypeError(message % (string, e)) def __repr__(self): - args = [self._mode, self._bufsize] - args_str = ', '.join([repr(arg) for arg in args if arg is not None]) + args = self._mode, self._bufsize + args_str = ', '.join(repr(arg) for arg in args if arg != -1) return '%s(%s)' % (type(self).__name__, args_str) # =========================== @@ -1275,13 +1277,20 @@ # create the action object, and add it to the parser action_class = self._pop_action_class(kwargs) if not _callable(action_class): - raise ValueError('unknown action "%s"' % action_class) + raise ValueError('unknown action "%s"' % (action_class,)) action = action_class(**kwargs) # raise an error if the action type is not callable type_func = self._registry_get('type', action.type, action.type) if not _callable(type_func): - raise ValueError('%r is not callable' % type_func) + raise ValueError('%r is not callable' % (type_func,)) + + # raise an error if the metavar does not match the type + if hasattr(self, "_get_formatter"): + try: + self._get_formatter()._format_args(action, None) + except TypeError: + raise ValueError("length of metavar tuple does not match nargs") return self._add_action(action) @@ -1481,6 +1490,7 @@ self._defaults = container._defaults self._has_negative_number_optionals = \ container._has_negative_number_optionals + self._mutually_exclusive_groups = container._mutually_exclusive_groups def _add_action(self, action): action = super(_ArgumentGroup, self)._add_action(action) diff --git a/lib-python/2.7/ast.py b/lib-python/2.7/ast.py --- a/lib-python/2.7/ast.py +++ b/lib-python/2.7/ast.py @@ -29,12 +29,12 @@ from _ast import __version__ -def parse(expr, filename='<unknown>', mode='exec'): +def parse(source, filename='<unknown>', mode='exec'): """ - Parse an expression into an AST node. - Equivalent to compile(expr, filename, mode, PyCF_ONLY_AST). + Parse the source into an AST node. + Equivalent to compile(source, filename, mode, PyCF_ONLY_AST). """ - return compile(expr, filename, mode, PyCF_ONLY_AST) + return compile(source, filename, mode, PyCF_ONLY_AST) def literal_eval(node_or_string): @@ -152,8 +152,6 @@ Increment the line number of each node in the tree starting at *node* by *n*. This is useful to "move code" to a different location in a file. """ - if 'lineno' in node._attributes: - node.lineno = getattr(node, 'lineno', 0) + n for child in walk(node): if 'lineno' in child._attributes: child.lineno = getattr(child, 'lineno', 0) + n @@ -204,9 +202,9 @@ def walk(node): """ - Recursively yield all child nodes of *node*, in no specified order. This is - useful if you only want to modify nodes in place and don't care about the - context. + Recursively yield all descendant nodes in the tree starting at *node* + (including *node* itself), in no specified order. This is useful if you + only want to modify nodes in place and don't care about the context. """ from collections import deque todo = deque([node]) diff --git a/lib-python/2.7/asyncore.py b/lib-python/2.7/asyncore.py --- a/lib-python/2.7/asyncore.py +++ b/lib-python/2.7/asyncore.py @@ -54,7 +54,11 @@ import os from errno import EALREADY, EINPROGRESS, EWOULDBLOCK, ECONNRESET, EINVAL, \ - ENOTCONN, ESHUTDOWN, EINTR, EISCONN, EBADF, ECONNABORTED, errorcode + ENOTCONN, ESHUTDOWN, EINTR, EISCONN, EBADF, ECONNABORTED, EPIPE, EAGAIN, \ + errorcode + +_DISCONNECTED = frozenset((ECONNRESET, ENOTCONN, ESHUTDOWN, ECONNABORTED, EPIPE, + EBADF)) try: socket_map @@ -109,7 +113,7 @@ if flags & (select.POLLHUP | select.POLLERR | select.POLLNVAL): obj.handle_close() except socket.error, e: - if e.args[0] not in (EBADF, ECONNRESET, ENOTCONN, ESHUTDOWN, ECONNABORTED): + if e.args[0] not in _DISCONNECTED: obj.handle_error() else: obj.handle_close() @@ -353,7 +357,7 @@ except TypeError: return None except socket.error as why: - if why.args[0] in (EWOULDBLOCK, ECONNABORTED): + if why.args[0] in (EWOULDBLOCK, ECONNABORTED, EAGAIN): return None else: raise @@ -367,7 +371,7 @@ except socket.error, why: if why.args[0] == EWOULDBLOCK: return 0 - elif why.args[0] in (ECONNRESET, ENOTCONN, ESHUTDOWN, ECONNABORTED): + elif why.args[0] in _DISCONNECTED: self.handle_close() return 0 else: @@ -385,7 +389,7 @@ return data except socket.error, why: # winsock sometimes throws ENOTCONN - if why.args[0] in [ECONNRESET, ENOTCONN, ESHUTDOWN, ECONNABORTED]: + if why.args[0] in _DISCONNECTED: self.handle_close() return '' else: diff --git a/lib-python/2.7/bdb.py b/lib-python/2.7/bdb.py --- a/lib-python/2.7/bdb.py +++ b/lib-python/2.7/bdb.py @@ -250,6 +250,12 @@ list.append(lineno) bp = Breakpoint(filename, lineno, temporary, cond, funcname) + def _prune_breaks(self, filename, lineno): + if (filename, lineno) not in Breakpoint.bplist: + self.breaks[filename].remove(lineno) + if not self.breaks[filename]: + del self.breaks[filename] + def clear_break(self, filename, lineno): filename = self.canonic(filename) if not filename in self.breaks: @@ -261,10 +267,7 @@ # pair, then remove the breaks entry for bp in Breakpoint.bplist[filename, lineno][:]: bp.deleteMe() - if (filename, lineno) not in Breakpoint.bplist: - self.breaks[filename].remove(lineno) - if not self.breaks[filename]: - del self.breaks[filename] + self._prune_breaks(filename, lineno) def clear_bpbynumber(self, arg): try: @@ -277,7 +280,8 @@ return 'Breakpoint number (%d) out of range' % number if not bp: return 'Breakpoint (%d) already deleted' % number - self.clear_break(bp.file, bp.line) + bp.deleteMe() + self._prune_breaks(bp.file, bp.line) def clear_all_file_breaks(self, filename): filename = self.canonic(filename) diff --git a/lib-python/2.7/collections.py b/lib-python/2.7/collections.py --- a/lib-python/2.7/collections.py +++ b/lib-python/2.7/collections.py @@ -6,59 +6,38 @@ __all__ += _abcoll.__all__ from _collections import deque, defaultdict -from operator import itemgetter as _itemgetter, eq as _eq +from operator import itemgetter as _itemgetter from keyword import iskeyword as _iskeyword import sys as _sys import heapq as _heapq -from itertools import repeat as _repeat, chain as _chain, starmap as _starmap, \ - ifilter as _ifilter, imap as _imap +from itertools import repeat as _repeat, chain as _chain, starmap as _starmap + try: - from thread import get_ident + from thread import get_ident as _get_ident except ImportError: - from dummy_thread import get_ident - -def _recursive_repr(user_function): - 'Decorator to make a repr function return "..." for a recursive call' - repr_running = set() - - def wrapper(self): - key = id(self), get_ident() - if key in repr_running: - return '...' - repr_running.add(key) - try: - result = user_function(self) - finally: - repr_running.discard(key) - return result - - # Can't use functools.wraps() here because of bootstrap issues - wrapper.__module__ = getattr(user_function, '__module__') - wrapper.__doc__ = getattr(user_function, '__doc__') - wrapper.__name__ = getattr(user_function, '__name__') - return wrapper + from dummy_thread import get_ident as _get_ident ################################################################################ ### OrderedDict ################################################################################ -class OrderedDict(dict, MutableMapping): +class OrderedDict(dict): 'Dictionary that remembers insertion order' # An inherited dict maps keys to values. # The inherited dict provides __getitem__, __len__, __contains__, and get. # The remaining methods are order-aware. - # Big-O running times for all methods are the same as for regular dictionaries. + # Big-O running times for all methods are the same as regular dictionaries. - # The internal self.__map dictionary maps keys to links in a doubly linked list. + # The internal self.__map dict maps keys to links in a doubly linked list. # The circular doubly linked list starts and ends with a sentinel element. # The sentinel element never gets deleted (this simplifies the algorithm). # Each link is stored as a list of length three: [PREV, NEXT, KEY]. def __init__(self, *args, **kwds): - '''Initialize an ordered dictionary. Signature is the same as for - regular dictionaries, but keyword arguments are not recommended - because their insertion order is arbitrary. + '''Initialize an ordered dictionary. The signature is the same as + regular dictionaries, but keyword arguments are not recommended because + their insertion order is arbitrary. ''' if len(args) > 1: @@ -66,17 +45,15 @@ try: self.__root except AttributeError: - self.__root = root = [None, None, None] # sentinel node - PREV = 0 - NEXT = 1 - root[PREV] = root[NEXT] = root + self.__root = root = [] # sentinel node + root[:] = [root, root, None] self.__map = {} - self.update(*args, **kwds) + self.__update(*args, **kwds) def __setitem__(self, key, value, PREV=0, NEXT=1, dict_setitem=dict.__setitem__): 'od.__setitem__(i, y) <==> od[i]=y' - # Setting a new item creates a new link which goes at the end of the linked - # list, and the inherited dictionary is updated with the new key/value pair. + # Setting a new item creates a new link at the end of the linked list, + # and the inherited dictionary is updated with the new key/value pair. if key not in self: root = self.__root last = root[PREV] @@ -85,65 +62,160 @@ def __delitem__(self, key, PREV=0, NEXT=1, dict_delitem=dict.__delitem__): 'od.__delitem__(y) <==> del od[y]' - # Deleting an existing item uses self.__map to find the link which is - # then removed by updating the links in the predecessor and successor nodes. + # Deleting an existing item uses self.__map to find the link which gets + # removed by updating the links in the predecessor and successor nodes. dict_delitem(self, key) - link = self.__map.pop(key) - link_prev = link[PREV] - link_next = link[NEXT] + link_prev, link_next, key = self.__map.pop(key) link_prev[NEXT] = link_next link_next[PREV] = link_prev - def __iter__(self, NEXT=1, KEY=2): + def __iter__(self): 'od.__iter__() <==> iter(od)' # Traverse the linked list in order. + NEXT, KEY = 1, 2 root = self.__root curr = root[NEXT] while curr is not root: yield curr[KEY] curr = curr[NEXT] - def __reversed__(self, PREV=0, KEY=2): + def __reversed__(self): 'od.__reversed__() <==> reversed(od)' # Traverse the linked list in reverse order. + PREV, KEY = 0, 2 root = self.__root curr = root[PREV] while curr is not root: yield curr[KEY] curr = curr[PREV] + def clear(self): + 'od.clear() -> None. Remove all items from od.' + for node in self.__map.itervalues(): + del node[:] + root = self.__root + root[:] = [root, root, None] + self.__map.clear() + dict.clear(self) + + # -- the following methods do not depend on the internal structure -- + + def keys(self): + 'od.keys() -> list of keys in od' + return list(self) + + def values(self): + 'od.values() -> list of values in od' + return [self[key] for key in self] + + def items(self): + 'od.items() -> list of (key, value) pairs in od' + return [(key, self[key]) for key in self] + + def iterkeys(self): + 'od.iterkeys() -> an iterator over the keys in od' + return iter(self) + + def itervalues(self): + 'od.itervalues -> an iterator over the values in od' + for k in self: + yield self[k] + + def iteritems(self): + 'od.iteritems -> an iterator over the (key, value) pairs in od' + for k in self: + yield (k, self[k]) + + update = MutableMapping.update + + __update = update # let subclasses override update without breaking __init__ + + __marker = object() + + def pop(self, key, default=__marker): + '''od.pop(k[,d]) -> v, remove specified key and return the corresponding + value. If key is not found, d is returned if given, otherwise KeyError + is raised. + + ''' + if key in self: + result = self[key] + del self[key] + return result + if default is self.__marker: + raise KeyError(key) + return default + + def setdefault(self, key, default=None): + 'od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od' + if key in self: + return self[key] + self[key] = default + return default + + def popitem(self, last=True): + '''od.popitem() -> (k, v), return and remove a (key, value) pair. + Pairs are returned in LIFO order if last is true or FIFO order if false. + + ''' + if not self: + raise KeyError('dictionary is empty') + key = next(reversed(self) if last else iter(self)) + value = self.pop(key) + return key, value + + def __repr__(self, _repr_running={}): + 'od.__repr__() <==> repr(od)' + call_key = id(self), _get_ident() + if call_key in _repr_running: + return '...' + _repr_running[call_key] = 1 + try: + if not self: + return '%s()' % (self.__class__.__name__,) + return '%s(%r)' % (self.__class__.__name__, self.items()) + finally: + del _repr_running[call_key] + def __reduce__(self): 'Return state information for pickling' items = [[k, self[k]] for k in self] - tmp = self.__map, self.__root - del self.__map, self.__root inst_dict = vars(self).copy() - self.__map, self.__root = tmp + for k in vars(OrderedDict()): + inst_dict.pop(k, None) if inst_dict: return (self.__class__, (items,), inst_dict) return self.__class__, (items,) - def clear(self): - 'od.clear() -> None. Remove all items from od.' - try: - for node in self.__map.itervalues(): - del node[:] - self.__root[:] = [self.__root, self.__root, None] - self.__map.clear() - except AttributeError: - pass - dict.clear(self) + def copy(self): + 'od.copy() -> a shallow copy of od' + return self.__class__(self) - setdefault = MutableMapping.setdefault - update = MutableMapping.update - pop = MutableMapping.pop - keys = MutableMapping.keys - values = MutableMapping.values - items = MutableMapping.items - iterkeys = MutableMapping.iterkeys - itervalues = MutableMapping.itervalues - iteritems = MutableMapping.iteritems - __ne__ = MutableMapping.__ne__ + @classmethod + def fromkeys(cls, iterable, value=None): + '''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S. + If not specified, the value defaults to None. + + ''' + self = cls() + for key in iterable: + self[key] = value + return self + + def __eq__(self, other): + '''od.__eq__(y) <==> od==y. Comparison to another OD is order-sensitive + while comparison to a regular mapping is order-insensitive. + + ''' + if isinstance(other, OrderedDict): + return len(self)==len(other) and self.items() == other.items() + return dict.__eq__(self, other) + + def __ne__(self, other): + 'od.__ne__(y) <==> od!=y' + return not self == other + + # -- the following methods support python 3.x style dictionary views -- def viewkeys(self): "od.viewkeys() -> a set-like object providing a view on od's keys" @@ -157,49 +229,6 @@ "od.viewitems() -> a set-like object providing a view on od's items" return ItemsView(self) - def popitem(self, last=True): - '''od.popitem() -> (k, v), return and remove a (key, value) pair. - Pairs are returned in LIFO order if last is true or FIFO order if false. - - ''' - if not self: - raise KeyError('dictionary is empty') - key = next(reversed(self) if last else iter(self)) - value = self.pop(key) - return key, value - - @_recursive_repr - def __repr__(self): - 'od.__repr__() <==> repr(od)' - if not self: - return '%s()' % (self.__class__.__name__,) - return '%s(%r)' % (self.__class__.__name__, self.items()) - - def copy(self): - 'od.copy() -> a shallow copy of od' - return self.__class__(self) - - @classmethod - def fromkeys(cls, iterable, value=None): - '''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S - and values equal to v (which defaults to None). - - ''' - d = cls() - for key in iterable: - d[key] = value - return d - - def __eq__(self, other): - '''od.__eq__(y) <==> od==y. Comparison to another OD is order-sensitive - while comparison to a regular mapping is order-insensitive. - - ''' - if isinstance(other, OrderedDict): - return len(self)==len(other) and \ - all(_imap(_eq, self.iteritems(), other.iteritems())) - return dict.__eq__(self, other) - ################################################################################ ### namedtuple @@ -328,16 +357,16 @@ or multiset. Elements are stored as dictionary keys and their counts are stored as dictionary values. - >>> c = Counter('abracadabra') # count elements from a string + >>> c = Counter('abcdeabcdabcaba') # count elements from a string >>> c.most_common(3) # three most common elements - [('a', 5), ('r', 2), ('b', 2)] + [('a', 5), ('b', 4), ('c', 3)] >>> sorted(c) # list all unique elements - ['a', 'b', 'c', 'd', 'r'] + ['a', 'b', 'c', 'd', 'e'] >>> ''.join(sorted(c.elements())) # list elements with repetitions - 'aaaaabbcdrr' + 'aaaaabbbbcccdde' >>> sum(c.values()) # total of all counts - 11 + 15 >>> c['a'] # count of letter 'a' 5 @@ -345,8 +374,8 @@ ... c[elem] += 1 # by adding 1 to each element's count >>> c['a'] # now there are seven 'a' 7 - >>> del c['r'] # remove all 'r' - >>> c['r'] # now there are zero 'r' + >>> del c['b'] # remove all 'b' + >>> c['b'] # now there are zero 'b' 0 >>> d = Counter('simsalabim') # make another counter @@ -385,6 +414,7 @@ >>> c = Counter(a=4, b=2) # a new counter from keyword args ''' + super(Counter, self).__init__() self.update(iterable, **kwds) def __missing__(self, key): @@ -396,8 +426,8 @@ '''List the n most common elements and their counts from the most common to the least. If n is None, then list all element counts. - >>> Counter('abracadabra').most_common(3) - [('a', 5), ('r', 2), ('b', 2)] + >>> Counter('abcdeabcdabcaba').most_common(3) + [('a', 5), ('b', 4), ('c', 3)] ''' # Emulate Bag.sortedByCount from Smalltalk @@ -463,7 +493,7 @@ for elem, count in iterable.iteritems(): self[elem] = self_get(elem, 0) + count else: - dict.update(self, iterable) # fast path when counter is empty + super(Counter, self).update(iterable) # fast path when counter is empty else: self_get = self.get for elem in iterable: @@ -499,13 +529,16 @@ self.subtract(kwds) def copy(self): - 'Like dict.copy() but returns a Counter instance instead of a dict.' - return Counter(self) + 'Return a shallow copy.' + return self.__class__(self) + + def __reduce__(self): + return self.__class__, (dict(self),) def __delitem__(self, elem): 'Like dict.__delitem__() but does not raise KeyError for missing values.' if elem in self: - dict.__delitem__(self, elem) + super(Counter, self).__delitem__(elem) def __repr__(self): if not self: @@ -532,10 +565,13 @@ if not isinstance(other, Counter): return NotImplemented result = Counter() - for elem in set(self) | set(other): - newcount = self[elem] + other[elem] + for elem, count in self.items(): + newcount = count + other[elem] if newcount > 0: result[elem] = newcount + for elem, count in other.items(): + if elem not in self and count > 0: + result[elem] = count return result def __sub__(self, other): @@ -548,10 +584,13 @@ if not isinstance(other, Counter): return NotImplemented result = Counter() - for elem in set(self) | set(other): - newcount = self[elem] - other[elem] + for elem, count in self.items(): + newcount = count - other[elem] if newcount > 0: result[elem] = newcount + for elem, count in other.items(): + if elem not in self and count < 0: + result[elem] = 0 - count return result def __or__(self, other): @@ -564,11 +603,14 @@ if not isinstance(other, Counter): return NotImplemented result = Counter() - for elem in set(self) | set(other): - p, q = self[elem], other[elem] - newcount = q if p < q else p + for elem, count in self.items(): + other_count = other[elem] + newcount = other_count if count < other_count else count if newcount > 0: result[elem] = newcount + for elem, count in other.items(): + if elem not in self and count > 0: + result[elem] = count return result def __and__(self, other): @@ -581,11 +623,9 @@ if not isinstance(other, Counter): return NotImplemented result = Counter() - if len(self) < len(other): - self, other = other, self - for elem in _ifilter(self.__contains__, other): - p, q = self[elem], other[elem] - newcount = p if p < q else q + for elem, count in self.items(): + other_count = other[elem] + newcount = count if count < other_count else other_count if newcount > 0: result[elem] = newcount return result diff --git a/lib-python/2.7/compileall.py b/lib-python/2.7/compileall.py --- a/lib-python/2.7/compileall.py +++ b/lib-python/2.7/compileall.py @@ -9,7 +9,6 @@ packages -- for now, you'll have to deal with packages separately.) See module py_compile for details of the actual byte-compilation. - """ import os import sys @@ -31,7 +30,6 @@ directory name that will show up in error messages) force: if 1, force compilation, even if timestamps are up-to-date quiet: if 1, be quiet during compilation - """ if not quiet: print 'Listing', dir, '...' @@ -61,15 +59,16 @@ return success def compile_file(fullname, ddir=None, force=0, rx=None, quiet=0): - """Byte-compile file. - file: the file to byte-compile + """Byte-compile one file. + + Arguments (only fullname is required): + + fullname: the file to byte-compile ddir: if given, purported directory name (this is the directory name that will show up in error messages) force: if 1, force compilation, even if timestamps are up-to-date quiet: if 1, be quiet during compilation - """ - success = 1 name = os.path.basename(fullname) if ddir is not None: @@ -120,7 +119,6 @@ maxlevels: max recursion level (default 0) force: as for compile_dir() (default 0) quiet: as for compile_dir() (default 0) - """ success = 1 for dir in sys.path: diff --git a/lib-python/2.7/csv.py b/lib-python/2.7/csv.py --- a/lib-python/2.7/csv.py +++ b/lib-python/2.7/csv.py @@ -281,7 +281,7 @@ an all or nothing approach, so we allow for small variations in this number. 1) build a table of the frequency of each character on every line. - 2) build a table of freqencies of this frequency (meta-frequency?), + 2) build a table of frequencies of this frequency (meta-frequency?), e.g. 'x occurred 5 times in 10 rows, 6 times in 1000 rows, 7 times in 2 rows' 3) use the mode of the meta-frequency to determine the /expected/ diff --git a/lib-python/2.7/ctypes/test/test_arrays.py b/lib-python/2.7/ctypes/test/test_arrays.py --- a/lib-python/2.7/ctypes/test/test_arrays.py +++ b/lib-python/2.7/ctypes/test/test_arrays.py @@ -37,7 +37,7 @@ values = [ia[i] for i in range(len(init))] self.assertEqual(values, [0] * len(init)) - # Too many in itializers should be caught + # Too many initializers should be caught self.assertRaises(IndexError, int_array, *range(alen*2)) CharArray = ARRAY(c_char, 3) diff --git a/lib-python/2.7/ctypes/test/test_as_parameter.py b/lib-python/2.7/ctypes/test/test_as_parameter.py --- a/lib-python/2.7/ctypes/test/test_as_parameter.py +++ b/lib-python/2.7/ctypes/test/test_as_parameter.py @@ -187,6 +187,18 @@ self.assertEqual((s8i.a, s8i.b, s8i.c, s8i.d, s8i.e, s8i.f, s8i.g, s8i.h), (9*2, 8*3, 7*4, 6*5, 5*6, 4*7, 3*8, 2*9)) + def test_recursive_as_param(self): + from ctypes import c_int + + class A(object): + pass + + a = A() + a._as_parameter_ = a + with self.assertRaises(RuntimeError): + c_int.from_param(a) + + #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ class AsParamWrapper(object): diff --git a/lib-python/2.7/ctypes/test/test_callbacks.py b/lib-python/2.7/ctypes/test/test_callbacks.py --- a/lib-python/2.7/ctypes/test/test_callbacks.py +++ b/lib-python/2.7/ctypes/test/test_callbacks.py @@ -206,6 +206,42 @@ windll.user32.EnumWindows(EnumWindowsCallbackFunc, 0) + def test_callback_register_int(self): + # Issue #8275: buggy handling of callback args under Win64 + # NOTE: should be run on release builds as well + dll = CDLL(_ctypes_test.__file__) + CALLBACK = CFUNCTYPE(c_int, c_int, c_int, c_int, c_int, c_int) + # All this function does is call the callback with its args squared + func = dll._testfunc_cbk_reg_int + func.argtypes = (c_int, c_int, c_int, c_int, c_int, CALLBACK) + func.restype = c_int + + def callback(a, b, c, d, e): + return a + b + c + d + e + + result = func(2, 3, 4, 5, 6, CALLBACK(callback)) + self.assertEqual(result, callback(2*2, 3*3, 4*4, 5*5, 6*6)) + + def test_callback_register_double(self): + # Issue #8275: buggy handling of callback args under Win64 + # NOTE: should be run on release builds as well + dll = CDLL(_ctypes_test.__file__) + CALLBACK = CFUNCTYPE(c_double, c_double, c_double, c_double, + c_double, c_double) + # All this function does is call the callback with its args squared + func = dll._testfunc_cbk_reg_double + func.argtypes = (c_double, c_double, c_double, + c_double, c_double, CALLBACK) + func.restype = c_double + + def callback(a, b, c, d, e): + return a + b + c + d + e + + result = func(1.1, 2.2, 3.3, 4.4, 5.5, CALLBACK(callback)) + self.assertEqual(result, + callback(1.1*1.1, 2.2*2.2, 3.3*3.3, 4.4*4.4, 5.5*5.5)) + + ################################################################ if __name__ == '__main__': diff --git a/lib-python/2.7/ctypes/test/test_functions.py b/lib-python/2.7/ctypes/test/test_functions.py --- a/lib-python/2.7/ctypes/test/test_functions.py +++ b/lib-python/2.7/ctypes/test/test_functions.py @@ -116,7 +116,7 @@ self.assertEqual(result, 21) self.assertEqual(type(result), int) - # You cannot assing character format codes as restype any longer + # You cannot assign character format codes as restype any longer self.assertRaises(TypeError, setattr, f, "restype", "i") def test_floatresult(self): diff --git a/lib-python/2.7/ctypes/test/test_init.py b/lib-python/2.7/ctypes/test/test_init.py --- a/lib-python/2.7/ctypes/test/test_init.py +++ b/lib-python/2.7/ctypes/test/test_init.py @@ -27,7 +27,7 @@ self.assertEqual((y.x.a, y.x.b), (0, 0)) self.assertEqual(y.x.new_was_called, False) - # But explicitely creating an X structure calls __new__ and __init__, of course. + # But explicitly creating an X structure calls __new__ and __init__, of course. x = X() self.assertEqual((x.a, x.b), (9, 12)) self.assertEqual(x.new_was_called, True) diff --git a/lib-python/2.7/ctypes/test/test_numbers.py b/lib-python/2.7/ctypes/test/test_numbers.py --- a/lib-python/2.7/ctypes/test/test_numbers.py +++ b/lib-python/2.7/ctypes/test/test_numbers.py @@ -157,7 +157,7 @@ def test_int_from_address(self): from array import array for t in signed_types + unsigned_types: - # the array module doesn't suppport all format codes + # the array module doesn't support all format codes # (no 'q' or 'Q') try: array(t._type_) diff --git a/lib-python/2.7/ctypes/test/test_win32.py b/lib-python/2.7/ctypes/test/test_win32.py --- a/lib-python/2.7/ctypes/test/test_win32.py +++ b/lib-python/2.7/ctypes/test/test_win32.py @@ -17,7 +17,7 @@ # ValueError: Procedure probably called with not enough arguments (4 bytes missing) self.assertRaises(ValueError, IsWindow) - # This one should succeeed... + # This one should succeed... self.assertEqual(0, IsWindow(0)) # ValueError: Procedure probably called with too many arguments (8 bytes in excess) diff --git a/lib-python/2.7/curses/wrapper.py b/lib-python/2.7/curses/wrapper.py --- a/lib-python/2.7/curses/wrapper.py +++ b/lib-python/2.7/curses/wrapper.py @@ -43,7 +43,8 @@ return func(stdscr, *args, **kwds) finally: # Set everything back to normal - stdscr.keypad(0) - curses.echo() - curses.nocbreak() - curses.endwin() + if 'stdscr' in locals(): + stdscr.keypad(0) + curses.echo() + curses.nocbreak() + curses.endwin() diff --git a/lib-python/2.7/decimal.py b/lib-python/2.7/decimal.py --- a/lib-python/2.7/decimal.py +++ b/lib-python/2.7/decimal.py @@ -1068,14 +1068,16 @@ if ans: return ans - if not self: - # -Decimal('0') is Decimal('0'), not Decimal('-0') + if context is None: + context = getcontext() + + if not self and context.rounding != ROUND_FLOOR: + # -Decimal('0') is Decimal('0'), not Decimal('-0'), except + # in ROUND_FLOOR rounding mode. ans = self.copy_abs() else: ans = self.copy_negate() - if context is None: - context = getcontext() return ans._fix(context) def __pos__(self, context=None): @@ -1088,14 +1090,15 @@ if ans: return ans - if not self: - # + (-0) = 0 + if context is None: + context = getcontext() + + if not self and context.rounding != ROUND_FLOOR: + # + (-0) = 0, except in ROUND_FLOOR rounding mode. ans = self.copy_abs() else: ans = Decimal(self) - if context is None: - context = getcontext() return ans._fix(context) def __abs__(self, round=True, context=None): @@ -1680,7 +1683,7 @@ self = _dec_from_triple(self._sign, '1', exp_min-1) digits = 0 rounding_method = self._pick_rounding_function[context.rounding] - changed = getattr(self, rounding_method)(digits) + changed = rounding_method(self, digits) coeff = self._int[:digits] or '0' if changed > 0: coeff = str(int(coeff)+1) @@ -1720,8 +1723,6 @@ # here self was representable to begin with; return unchanged return Decimal(self) - _pick_rounding_function = {} - # for each of the rounding functions below: # self is a finite, nonzero Decimal # prec is an integer satisfying 0 <= prec < len(self._int) @@ -1788,6 +1789,17 @@ else: return -self._round_down(prec) + _pick_rounding_function = dict( + ROUND_DOWN = _round_down, + ROUND_UP = _round_up, + ROUND_HALF_UP = _round_half_up, + ROUND_HALF_DOWN = _round_half_down, + ROUND_HALF_EVEN = _round_half_even, + ROUND_CEILING = _round_ceiling, + ROUND_FLOOR = _round_floor, + ROUND_05UP = _round_05up, + ) + def fma(self, other, third, context=None): """Fused multiply-add. @@ -2492,8 +2504,8 @@ if digits < 0: self = _dec_from_triple(self._sign, '1', exp-1) digits = 0 - this_function = getattr(self, self._pick_rounding_function[rounding]) - changed = this_function(digits) + this_function = self._pick_rounding_function[rounding] + changed = this_function(self, digits) coeff = self._int[:digits] or '0' if changed == 1: coeff = str(int(coeff)+1) @@ -3705,18 +3717,6 @@ ##### Context class ####################################################### - -# get rounding method function: -rounding_functions = [name for name in Decimal.__dict__.keys() - if name.startswith('_round_')] -for name in rounding_functions: - # name is like _round_half_even, goes to the global ROUND_HALF_EVEN value. - globalname = name[1:].upper() - val = globals()[globalname] - Decimal._pick_rounding_function[val] = name - -del name, val, globalname, rounding_functions - class _ContextManager(object): """Context manager class to support localcontext(). @@ -5990,7 +5990,7 @@ def _format_align(sign, body, spec): """Given an unpadded, non-aligned numeric string 'body' and sign - string 'sign', add padding and aligment conforming to the given + string 'sign', add padding and alignment conforming to the given format specifier dictionary 'spec' (as produced by parse_format_specifier). diff --git a/lib-python/2.7/difflib.py b/lib-python/2.7/difflib.py --- a/lib-python/2.7/difflib.py +++ b/lib-python/2.7/difflib.py @@ -1140,6 +1140,21 @@ return ch in ws +######################################################################## +### Unified Diff +######################################################################## + +def _format_range_unified(start, stop): + 'Convert range to the "ed" format' + # Per the diff spec at http://www.unix.org/single_unix_specification/ + beginning = start + 1 # lines start numbering with one + length = stop - start + if length == 1: + return '{}'.format(beginning) + if not length: + beginning -= 1 # empty ranges begin at line just before the range + return '{},{}'.format(beginning, length) + def unified_diff(a, b, fromfile='', tofile='', fromfiledate='', tofiledate='', n=3, lineterm='\n'): r""" @@ -1184,25 +1199,45 @@ started = False for group in SequenceMatcher(None,a,b).get_grouped_opcodes(n): if not started: - fromdate = '\t%s' % fromfiledate if fromfiledate else '' - todate = '\t%s' % tofiledate if tofiledate else '' - yield '--- %s%s%s' % (fromfile, fromdate, lineterm) - yield '+++ %s%s%s' % (tofile, todate, lineterm) started = True - i1, i2, j1, j2 = group[0][1], group[-1][2], group[0][3], group[-1][4] - yield "@@ -%d,%d +%d,%d @@%s" % (i1+1, i2-i1, j1+1, j2-j1, lineterm) + fromdate = '\t{}'.format(fromfiledate) if fromfiledate else '' + todate = '\t{}'.format(tofiledate) if tofiledate else '' + yield '--- {}{}{}'.format(fromfile, fromdate, lineterm) + yield '+++ {}{}{}'.format(tofile, todate, lineterm) + + first, last = group[0], group[-1] + file1_range = _format_range_unified(first[1], last[2]) + file2_range = _format_range_unified(first[3], last[4]) + yield '@@ -{} +{} @@{}'.format(file1_range, file2_range, lineterm) + for tag, i1, i2, j1, j2 in group: if tag == 'equal': for line in a[i1:i2]: yield ' ' + line continue - if tag == 'replace' or tag == 'delete': + if tag in ('replace', 'delete'): for line in a[i1:i2]: yield '-' + line - if tag == 'replace' or tag == 'insert': + if tag in ('replace', 'insert'): for line in b[j1:j2]: yield '+' + line + +######################################################################## +### Context Diff +######################################################################## + +def _format_range_context(start, stop): + 'Convert range to the "ed" format' + # Per the diff spec at http://www.unix.org/single_unix_specification/ + beginning = start + 1 # lines start numbering with one + length = stop - start + if not length: + beginning -= 1 # empty ranges begin at line just before the range + if length <= 1: + return '{}'.format(beginning) + return '{},{}'.format(beginning, beginning + length - 1) + # See http://www.unix.org/single_unix_specification/ def context_diff(a, b, fromfile='', tofile='', fromfiledate='', tofiledate='', n=3, lineterm='\n'): @@ -1247,38 +1282,36 @@ four """ + prefix = dict(insert='+ ', delete='- ', replace='! ', equal=' ') started = False - prefixmap = {'insert':'+ ', 'delete':'- ', 'replace':'! ', 'equal':' '} for group in SequenceMatcher(None,a,b).get_grouped_opcodes(n): if not started: - fromdate = '\t%s' % fromfiledate if fromfiledate else '' - todate = '\t%s' % tofiledate if tofiledate else '' - yield '*** %s%s%s' % (fromfile, fromdate, lineterm) - yield '--- %s%s%s' % (tofile, todate, lineterm) started = True + fromdate = '\t{}'.format(fromfiledate) if fromfiledate else '' + todate = '\t{}'.format(tofiledate) if tofiledate else '' + yield '*** {}{}{}'.format(fromfile, fromdate, lineterm) + yield '--- {}{}{}'.format(tofile, todate, lineterm) - yield '***************%s' % (lineterm,) - if group[-1][2] - group[0][1] >= 2: - yield '*** %d,%d ****%s' % (group[0][1]+1, group[-1][2], lineterm) - else: - yield '*** %d ****%s' % (group[-1][2], lineterm) - visiblechanges = [e for e in group if e[0] in ('replace', 'delete')] - if visiblechanges: + first, last = group[0], group[-1] + yield '***************' + lineterm + + file1_range = _format_range_context(first[1], last[2]) + yield '*** {} ****{}'.format(file1_range, lineterm) + + if any(tag in ('replace', 'delete') for tag, _, _, _, _ in group): for tag, i1, i2, _, _ in group: if tag != 'insert': for line in a[i1:i2]: - yield prefixmap[tag] + line + yield prefix[tag] + line - if group[-1][4] - group[0][3] >= 2: - yield '--- %d,%d ----%s' % (group[0][3]+1, group[-1][4], lineterm) - else: - yield '--- %d ----%s' % (group[-1][4], lineterm) - visiblechanges = [e for e in group if e[0] in ('replace', 'insert')] - if visiblechanges: + file2_range = _format_range_context(first[3], last[4]) + yield '--- {} ----{}'.format(file2_range, lineterm) + + if any(tag in ('replace', 'insert') for tag, _, _, _, _ in group): for tag, _, _, j1, j2 in group: if tag != 'delete': for line in b[j1:j2]: - yield prefixmap[tag] + line + yield prefix[tag] + line def ndiff(a, b, linejunk=None, charjunk=IS_CHARACTER_JUNK): r""" @@ -1714,7 +1747,7 @@ line = line.replace(' ','\0') # expand tabs into spaces line = line.expandtabs(self._tabsize) - # relace spaces from expanded tabs back into tab characters + # replace spaces from expanded tabs back into tab characters # (we'll replace them with markup after we do differencing) line = line.replace(' ','\t') return line.replace('\0',' ').rstrip('\n') diff --git a/lib-python/2.7/distutils/__init__.py b/lib-python/2.7/distutils/__init__.py --- a/lib-python/2.7/distutils/__init__.py +++ b/lib-python/2.7/distutils/__init__.py @@ -15,5 +15,5 @@ # Updated automatically by the Python release process. # #--start constants-- -__version__ = "2.7.1" +__version__ = "2.7.2" #--end constants-- diff --git a/lib-python/2.7/distutils/archive_util.py b/lib-python/2.7/distutils/archive_util.py --- a/lib-python/2.7/distutils/archive_util.py +++ b/lib-python/2.7/distutils/archive_util.py @@ -121,7 +121,7 @@ def make_zipfile(base_name, base_dir, verbose=0, dry_run=0): """Create a zip file from all the files under 'base_dir'. - The output zip file will be named 'base_dir' + ".zip". Uses either the + The output zip file will be named 'base_name' + ".zip". Uses either the "zipfile" Python module (if available) or the InfoZIP "zip" utility (if installed and found on the default search path). If neither tool is available, raises DistutilsExecError. Returns the name of the output zip diff --git a/lib-python/2.7/distutils/cmd.py b/lib-python/2.7/distutils/cmd.py --- a/lib-python/2.7/distutils/cmd.py +++ b/lib-python/2.7/distutils/cmd.py @@ -377,7 +377,7 @@ dry_run=self.dry_run) def move_file (self, src, dst, level=1): - """Move a file respectin dry-run flag.""" + """Move a file respecting dry-run flag.""" return file_util.move_file(src, dst, dry_run = self.dry_run) def spawn (self, cmd, search_path=1, level=1): diff --git a/lib-python/2.7/distutils/command/build_ext.py b/lib-python/2.7/distutils/command/build_ext.py --- a/lib-python/2.7/distutils/command/build_ext.py +++ b/lib-python/2.7/distutils/command/build_ext.py @@ -207,7 +207,7 @@ elif MSVC_VERSION == 8: self.library_dirs.append(os.path.join(sys.exec_prefix, - 'PC', 'VS8.0', 'win32release')) + 'PC', 'VS8.0')) elif MSVC_VERSION == 7: self.library_dirs.append(os.path.join(sys.exec_prefix, 'PC', 'VS7.1')) diff --git a/lib-python/2.7/distutils/command/sdist.py b/lib-python/2.7/distutils/command/sdist.py --- a/lib-python/2.7/distutils/command/sdist.py +++ b/lib-python/2.7/distutils/command/sdist.py @@ -306,17 +306,20 @@ rstrip_ws=1, collapse_join=1) - while 1: - line = template.readline() - if line is None: # end of file - break + try: + while 1: + line = template.readline() + if line is None: # end of file + break - try: - self.filelist.process_template_line(line) - except DistutilsTemplateError, msg: - self.warn("%s, line %d: %s" % (template.filename, - template.current_line, - msg)) + try: + self.filelist.process_template_line(line) + except DistutilsTemplateError, msg: + self.warn("%s, line %d: %s" % (template.filename, + template.current_line, + msg)) + finally: + template.close() def prune_file_list(self): """Prune off branches that might slip into the file list as created diff --git a/lib-python/2.7/distutils/command/upload.py b/lib-python/2.7/distutils/command/upload.py --- a/lib-python/2.7/distutils/command/upload.py +++ b/lib-python/2.7/distutils/command/upload.py @@ -176,6 +176,9 @@ result = urlopen(request) status = result.getcode() reason = result.msg + if self.show_response: + msg = '\n'.join(('-' * 75, r.read(), '-' * 75)) + self.announce(msg, log.INFO) except socket.error, e: self.announce(str(e), log.ERROR) return @@ -189,6 +192,3 @@ else: self.announce('Upload failed (%s): %s' % (status, reason), log.ERROR) - if self.show_response: - msg = '\n'.join(('-' * 75, r.read(), '-' * 75)) - self.announce(msg, log.INFO) diff --git a/lib-python/2.7/distutils/sysconfig.py b/lib-python/2.7/distutils/sysconfig.py --- a/lib-python/2.7/distutils/sysconfig.py +++ b/lib-python/2.7/distutils/sysconfig.py @@ -389,7 +389,7 @@ cur_target = os.getenv('MACOSX_DEPLOYMENT_TARGET', '') if cur_target == '': cur_target = cfg_target - os.putenv('MACOSX_DEPLOYMENT_TARGET', cfg_target) + os.environ['MACOSX_DEPLOYMENT_TARGET'] = cfg_target elif map(int, cfg_target.split('.')) > map(int, cur_target.split('.')): my_msg = ('$MACOSX_DEPLOYMENT_TARGET mismatch: now "%s" but "%s" during configure' % (cur_target, cfg_target)) diff --git a/lib-python/2.7/distutils/tests/__init__.py b/lib-python/2.7/distutils/tests/__init__.py --- a/lib-python/2.7/distutils/tests/__init__.py +++ b/lib-python/2.7/distutils/tests/__init__.py @@ -15,9 +15,10 @@ import os import sys import unittest +from test.test_support import run_unittest -here = os.path.dirname(__file__) +here = os.path.dirname(__file__) or os.curdir def test_suite(): @@ -32,4 +33,4 @@ if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_archive_util.py b/lib-python/2.7/distutils/tests/test_archive_util.py --- a/lib-python/2.7/distutils/tests/test_archive_util.py +++ b/lib-python/2.7/distutils/tests/test_archive_util.py @@ -12,7 +12,7 @@ ARCHIVE_FORMATS) from distutils.spawn import find_executable, spawn from distutils.tests import support -from test.test_support import check_warnings +from test.test_support import check_warnings, run_unittest try: import grp @@ -281,4 +281,4 @@ return unittest.makeSuite(ArchiveUtilTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_bdist_msi.py b/lib-python/2.7/distutils/tests/test_bdist_msi.py --- a/lib-python/2.7/distutils/tests/test_bdist_msi.py +++ b/lib-python/2.7/distutils/tests/test_bdist_msi.py @@ -11,7 +11,7 @@ support.LoggingSilencer, unittest.TestCase): - def test_minial(self): + def test_minimal(self): # minimal test XXX need more tests from distutils.command.bdist_msi import bdist_msi pkg_pth, dist = self.create_dist() diff --git a/lib-python/2.7/distutils/tests/test_build.py b/lib-python/2.7/distutils/tests/test_build.py --- a/lib-python/2.7/distutils/tests/test_build.py +++ b/lib-python/2.7/distutils/tests/test_build.py @@ -2,6 +2,7 @@ import unittest import os import sys +from test.test_support import run_unittest from distutils.command.build import build from distutils.tests import support @@ -51,4 +52,4 @@ return unittest.makeSuite(BuildTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_build_clib.py b/lib-python/2.7/distutils/tests/test_build_clib.py --- a/lib-python/2.7/distutils/tests/test_build_clib.py +++ b/lib-python/2.7/distutils/tests/test_build_clib.py @@ -3,6 +3,8 @@ import os import sys +from test.test_support import run_unittest + from distutils.command.build_clib import build_clib from distutils.errors import DistutilsSetupError from distutils.tests import support @@ -140,4 +142,4 @@ return unittest.makeSuite(BuildCLibTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_build_ext.py b/lib-python/2.7/distutils/tests/test_build_ext.py --- a/lib-python/2.7/distutils/tests/test_build_ext.py +++ b/lib-python/2.7/distutils/tests/test_build_ext.py @@ -3,12 +3,13 @@ import tempfile import shutil from StringIO import StringIO +import textwrap from distutils.core import Extension, Distribution from distutils.command.build_ext import build_ext from distutils import sysconfig from distutils.tests import support -from distutils.errors import DistutilsSetupError +from distutils.errors import DistutilsSetupError, CompileError import unittest from test import test_support @@ -430,6 +431,59 @@ wanted = os.path.join(cmd.build_lib, 'UpdateManager', 'fdsend' + ext) self.assertEqual(ext_path, wanted) + @unittest.skipUnless(sys.platform == 'darwin', 'test only relevant for MacOSX') + def test_deployment_target(self): + self._try_compile_deployment_target() + + orig_environ = os.environ + os.environ = orig_environ.copy() + self.addCleanup(setattr, os, 'environ', orig_environ) + + os.environ['MACOSX_DEPLOYMENT_TARGET']='10.1' + self._try_compile_deployment_target() + + + def _try_compile_deployment_target(self): + deptarget_c = os.path.join(self.tmp_dir, 'deptargetmodule.c') + + with open(deptarget_c, 'w') as fp: + fp.write(textwrap.dedent('''\ + #include <AvailabilityMacros.h> + + int dummy; + + #if TARGET != MAC_OS_X_VERSION_MIN_REQUIRED + #error "Unexpected target" + #endif + + ''')) + + target = sysconfig.get_config_var('MACOSX_DEPLOYMENT_TARGET') + target = tuple(map(int, target.split('.'))) + target = '%02d%01d0' % target + + deptarget_ext = Extension( + 'deptarget', + [deptarget_c], + extra_compile_args=['-DTARGET=%s'%(target,)], + ) + dist = Distribution({ + 'name': 'deptarget', + 'ext_modules': [deptarget_ext] + }) + dist.package_dir = self.tmp_dir + cmd = build_ext(dist) + cmd.build_lib = self.tmp_dir + cmd.build_temp = self.tmp_dir + + try: + old_stdout = sys.stdout + cmd.ensure_finalized() + cmd.run() + + except CompileError: + self.fail("Wrong deployment target during compilation") + def test_suite(): return unittest.makeSuite(BuildExtTestCase) diff --git a/lib-python/2.7/distutils/tests/test_build_py.py b/lib-python/2.7/distutils/tests/test_build_py.py --- a/lib-python/2.7/distutils/tests/test_build_py.py +++ b/lib-python/2.7/distutils/tests/test_build_py.py @@ -10,13 +10,14 @@ from distutils.errors import DistutilsFileError from distutils.tests import support +from test.test_support import run_unittest class BuildPyTestCase(support.TempdirManager, support.LoggingSilencer, unittest.TestCase): - def _setup_package_data(self): + def test_package_data(self): sources = self.mkdtemp() f = open(os.path.join(sources, "__init__.py"), "w") try: @@ -56,20 +57,15 @@ self.assertEqual(len(cmd.get_outputs()), 3) pkgdest = os.path.join(destination, "pkg") files = os.listdir(pkgdest) - return files + self.assertIn("__init__.py", files) + self.assertIn("README.txt", files) + # XXX even with -O, distutils writes pyc, not pyo; bug? + if sys.dont_write_bytecode: + self.assertNotIn("__init__.pyc", files) + else: + self.assertIn("__init__.pyc", files) - def test_package_data(self): - files = self._setup_package_data() - self.assertTrue("__init__.py" in files) - self.assertTrue("README.txt" in files) - - @unittest.skipIf(sys.flags.optimize >= 2, - "pyc files are not written with -O2 and above") - def test_package_data_pyc(self): - files = self._setup_package_data() - self.assertTrue("__init__.pyc" in files) - - def test_empty_package_dir (self): + def test_empty_package_dir(self): # See SF 1668596/1720897. cwd = os.getcwd() @@ -117,10 +113,10 @@ finally: sys.dont_write_bytecode = old_dont_write_bytecode - self.assertTrue('byte-compiling is disabled' in self.logs[0][1]) + self.assertIn('byte-compiling is disabled', self.logs[0][1]) def test_suite(): return unittest.makeSuite(BuildPyTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_build_scripts.py b/lib-python/2.7/distutils/tests/test_build_scripts.py --- a/lib-python/2.7/distutils/tests/test_build_scripts.py +++ b/lib-python/2.7/distutils/tests/test_build_scripts.py @@ -8,6 +8,7 @@ import sysconfig from distutils.tests import support +from test.test_support import run_unittest class BuildScriptsTestCase(support.TempdirManager, @@ -108,4 +109,4 @@ return unittest.makeSuite(BuildScriptsTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_check.py b/lib-python/2.7/distutils/tests/test_check.py --- a/lib-python/2.7/distutils/tests/test_check.py +++ b/lib-python/2.7/distutils/tests/test_check.py @@ -1,5 +1,6 @@ """Tests for distutils.command.check.""" import unittest +from test.test_support import run_unittest from distutils.command.check import check, HAS_DOCUTILS from distutils.tests import support @@ -95,4 +96,4 @@ return unittest.makeSuite(CheckTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_clean.py b/lib-python/2.7/distutils/tests/test_clean.py --- a/lib-python/2.7/distutils/tests/test_clean.py +++ b/lib-python/2.7/distutils/tests/test_clean.py @@ -6,6 +6,7 @@ from distutils.command.clean import clean from distutils.tests import support +from test.test_support import run_unittest class cleanTestCase(support.TempdirManager, support.LoggingSilencer, @@ -38,7 +39,7 @@ self.assertTrue(not os.path.exists(path), '%s was not removed' % path) - # let's run the command again (should spit warnings but suceed) + # let's run the command again (should spit warnings but succeed) cmd.all = 1 cmd.ensure_finalized() cmd.run() @@ -47,4 +48,4 @@ return unittest.makeSuite(cleanTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_cmd.py b/lib-python/2.7/distutils/tests/test_cmd.py --- a/lib-python/2.7/distutils/tests/test_cmd.py +++ b/lib-python/2.7/distutils/tests/test_cmd.py @@ -99,7 +99,7 @@ def test_ensure_dirname(self): cmd = self.cmd - cmd.option1 = os.path.dirname(__file__) + cmd.option1 = os.path.dirname(__file__) or os.curdir cmd.ensure_dirname('option1') cmd.option2 = 'xxx' self.assertRaises(DistutilsOptionError, cmd.ensure_dirname, 'option2') diff --git a/lib-python/2.7/distutils/tests/test_config.py b/lib-python/2.7/distutils/tests/test_config.py --- a/lib-python/2.7/distutils/tests/test_config.py +++ b/lib-python/2.7/distutils/tests/test_config.py @@ -11,6 +11,7 @@ from distutils.log import WARN from distutils.tests import support +from test.test_support import run_unittest PYPIRC = """\ [distutils] @@ -119,4 +120,4 @@ return unittest.makeSuite(PyPIRCCommandTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_config_cmd.py b/lib-python/2.7/distutils/tests/test_config_cmd.py --- a/lib-python/2.7/distutils/tests/test_config_cmd.py +++ b/lib-python/2.7/distutils/tests/test_config_cmd.py @@ -2,6 +2,7 @@ import unittest import os import sys +from test.test_support import run_unittest from distutils.command.config import dump_file, config from distutils.tests import support @@ -86,4 +87,4 @@ return unittest.makeSuite(ConfigTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_core.py b/lib-python/2.7/distutils/tests/test_core.py --- a/lib-python/2.7/distutils/tests/test_core.py +++ b/lib-python/2.7/distutils/tests/test_core.py @@ -6,7 +6,7 @@ import shutil import sys import test.test_support -from test.test_support import captured_stdout +from test.test_support import captured_stdout, run_unittest import unittest from distutils.tests import support @@ -105,4 +105,4 @@ return unittest.makeSuite(CoreTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_dep_util.py b/lib-python/2.7/distutils/tests/test_dep_util.py --- a/lib-python/2.7/distutils/tests/test_dep_util.py +++ b/lib-python/2.7/distutils/tests/test_dep_util.py @@ -6,6 +6,7 @@ from distutils.dep_util import newer, newer_pairwise, newer_group from distutils.errors import DistutilsFileError from distutils.tests import support +from test.test_support import run_unittest class DepUtilTestCase(support.TempdirManager, unittest.TestCase): @@ -77,4 +78,4 @@ return unittest.makeSuite(DepUtilTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_dir_util.py b/lib-python/2.7/distutils/tests/test_dir_util.py --- a/lib-python/2.7/distutils/tests/test_dir_util.py +++ b/lib-python/2.7/distutils/tests/test_dir_util.py @@ -10,6 +10,7 @@ from distutils import log from distutils.tests import support +from test.test_support import run_unittest class DirUtilTestCase(support.TempdirManager, unittest.TestCase): @@ -112,4 +113,4 @@ return unittest.makeSuite(DirUtilTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_dist.py b/lib-python/2.7/distutils/tests/test_dist.py --- a/lib-python/2.7/distutils/tests/test_dist.py +++ b/lib-python/2.7/distutils/tests/test_dist.py @@ -11,7 +11,7 @@ from distutils.dist import Distribution, fix_help_options, DistributionMetadata from distutils.cmd import Command import distutils.dist -from test.test_support import TESTFN, captured_stdout +from test.test_support import TESTFN, captured_stdout, run_unittest from distutils.tests import support class test_dist(Command): @@ -433,4 +433,4 @@ return suite if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_file_util.py b/lib-python/2.7/distutils/tests/test_file_util.py --- a/lib-python/2.7/distutils/tests/test_file_util.py +++ b/lib-python/2.7/distutils/tests/test_file_util.py @@ -6,6 +6,7 @@ from distutils.file_util import move_file, write_file, copy_file from distutils import log from distutils.tests import support +from test.test_support import run_unittest class FileUtilTestCase(support.TempdirManager, unittest.TestCase): @@ -77,4 +78,4 @@ return unittest.makeSuite(FileUtilTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_filelist.py b/lib-python/2.7/distutils/tests/test_filelist.py --- a/lib-python/2.7/distutils/tests/test_filelist.py +++ b/lib-python/2.7/distutils/tests/test_filelist.py @@ -1,7 +1,7 @@ """Tests for distutils.filelist.""" from os.path import join import unittest -from test.test_support import captured_stdout +from test.test_support import captured_stdout, run_unittest from distutils.filelist import glob_to_re, FileList from distutils import debug @@ -82,4 +82,4 @@ return unittest.makeSuite(FileListTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_install.py b/lib-python/2.7/distutils/tests/test_install.py --- a/lib-python/2.7/distutils/tests/test_install.py +++ b/lib-python/2.7/distutils/tests/test_install.py @@ -3,6 +3,8 @@ import os import unittest +from test.test_support import run_unittest + from distutils.command.install import install from distutils.core import Distribution @@ -52,4 +54,4 @@ return unittest.makeSuite(InstallTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_install_data.py b/lib-python/2.7/distutils/tests/test_install_data.py --- a/lib-python/2.7/distutils/tests/test_install_data.py +++ b/lib-python/2.7/distutils/tests/test_install_data.py @@ -6,6 +6,7 @@ from distutils.command.install_data import install_data from distutils.tests import support +from test.test_support import run_unittest class InstallDataTestCase(support.TempdirManager, support.LoggingSilencer, @@ -73,4 +74,4 @@ return unittest.makeSuite(InstallDataTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_install_headers.py b/lib-python/2.7/distutils/tests/test_install_headers.py --- a/lib-python/2.7/distutils/tests/test_install_headers.py +++ b/lib-python/2.7/distutils/tests/test_install_headers.py @@ -6,6 +6,7 @@ from distutils.command.install_headers import install_headers from distutils.tests import support +from test.test_support import run_unittest class InstallHeadersTestCase(support.TempdirManager, support.LoggingSilencer, @@ -37,4 +38,4 @@ return unittest.makeSuite(InstallHeadersTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_install_lib.py b/lib-python/2.7/distutils/tests/test_install_lib.py --- a/lib-python/2.7/distutils/tests/test_install_lib.py +++ b/lib-python/2.7/distutils/tests/test_install_lib.py @@ -7,6 +7,7 @@ from distutils.extension import Extension from distutils.tests import support from distutils.errors import DistutilsOptionError +from test.test_support import run_unittest class InstallLibTestCase(support.TempdirManager, support.LoggingSilencer, @@ -103,4 +104,4 @@ return unittest.makeSuite(InstallLibTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_install_scripts.py b/lib-python/2.7/distutils/tests/test_install_scripts.py --- a/lib-python/2.7/distutils/tests/test_install_scripts.py +++ b/lib-python/2.7/distutils/tests/test_install_scripts.py @@ -7,6 +7,7 @@ from distutils.core import Distribution from distutils.tests import support +from test.test_support import run_unittest class InstallScriptsTestCase(support.TempdirManager, @@ -78,4 +79,4 @@ return unittest.makeSuite(InstallScriptsTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_msvc9compiler.py b/lib-python/2.7/distutils/tests/test_msvc9compiler.py --- a/lib-python/2.7/distutils/tests/test_msvc9compiler.py +++ b/lib-python/2.7/distutils/tests/test_msvc9compiler.py @@ -5,6 +5,7 @@ from distutils.errors import DistutilsPlatformError from distutils.tests import support +from test.test_support import run_unittest _MANIFEST = """\ <?xml version="1.0" encoding="UTF-8" standalone="yes"?> @@ -137,4 +138,4 @@ return unittest.makeSuite(msvc9compilerTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_register.py b/lib-python/2.7/distutils/tests/test_register.py --- a/lib-python/2.7/distutils/tests/test_register.py +++ b/lib-python/2.7/distutils/tests/test_register.py @@ -7,7 +7,7 @@ import urllib2 import warnings -from test.test_support import check_warnings +from test.test_support import check_warnings, run_unittest from distutils.command import register as register_module from distutils.command.register import register @@ -138,7 +138,7 @@ # let's see what the server received : we should # have 2 similar requests - self.assertTrue(self.conn.reqs, 2) + self.assertEqual(len(self.conn.reqs), 2) req1 = dict(self.conn.reqs[0].headers) req2 = dict(self.conn.reqs[1].headers) self.assertEqual(req2['Content-length'], req1['Content-length']) @@ -168,7 +168,7 @@ del register_module.raw_input # we should have send a request - self.assertTrue(self.conn.reqs, 1) + self.assertEqual(len(self.conn.reqs), 1) req = self.conn.reqs[0] headers = dict(req.headers) self.assertEqual(headers['Content-length'], '608') @@ -186,7 +186,7 @@ del register_module.raw_input # we should have send a request - self.assertTrue(self.conn.reqs, 1) + self.assertEqual(len(self.conn.reqs), 1) req = self.conn.reqs[0] headers = dict(req.headers) self.assertEqual(headers['Content-length'], '290') @@ -258,4 +258,4 @@ return unittest.makeSuite(RegisterTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_sdist.py b/lib-python/2.7/distutils/tests/test_sdist.py --- a/lib-python/2.7/distutils/tests/test_sdist.py +++ b/lib-python/2.7/distutils/tests/test_sdist.py @@ -24,11 +24,9 @@ import tempfile import warnings -from test.test_support import check_warnings -from test.test_support import captured_stdout +from test.test_support import captured_stdout, check_warnings, run_unittest -from distutils.command.sdist import sdist -from distutils.command.sdist import show_formats +from distutils.command.sdist import sdist, show_formats from distutils.core import Distribution from distutils.tests.test_config import PyPIRCCommandTestCase from distutils.errors import DistutilsExecError, DistutilsOptionError @@ -372,7 +370,7 @@ # adding a file self.write_file((self.tmp_dir, 'somecode', 'doc2.txt'), '#') - # make sure build_py is reinitinialized, like a fresh run + # make sure build_py is reinitialized, like a fresh run build_py = dist.get_command_obj('build_py') build_py.finalized = False build_py.ensure_finalized() @@ -390,6 +388,7 @@ self.assertEqual(len(manifest2), 6) self.assertIn('doc2.txt', manifest2[-1]) + @unittest.skipUnless(zlib, "requires zlib") def test_manifest_marker(self): # check that autogenerated MANIFESTs have a marker dist, cmd = self.get_cmd() @@ -406,6 +405,7 @@ self.assertEqual(manifest[0], '# file GENERATED by distutils, do NOT edit') + @unittest.skipUnless(zlib, "requires zlib") def test_manual_manifest(self): # check that a MANIFEST without a marker is left alone dist, cmd = self.get_cmd() @@ -426,4 +426,4 @@ return unittest.makeSuite(SDistTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_spawn.py b/lib-python/2.7/distutils/tests/test_spawn.py --- a/lib-python/2.7/distutils/tests/test_spawn.py +++ b/lib-python/2.7/distutils/tests/test_spawn.py @@ -2,7 +2,7 @@ import unittest import os import time -from test.test_support import captured_stdout +from test.test_support import captured_stdout, run_unittest from distutils.spawn import _nt_quote_args from distutils.spawn import spawn, find_executable @@ -57,4 +57,4 @@ return unittest.makeSuite(SpawnTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_text_file.py b/lib-python/2.7/distutils/tests/test_text_file.py --- a/lib-python/2.7/distutils/tests/test_text_file.py +++ b/lib-python/2.7/distutils/tests/test_text_file.py @@ -3,6 +3,7 @@ import unittest from distutils.text_file import TextFile from distutils.tests import support +from test.test_support import run_unittest TEST_DATA = """# test file @@ -103,4 +104,4 @@ return unittest.makeSuite(TextFileTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_unixccompiler.py b/lib-python/2.7/distutils/tests/test_unixccompiler.py --- a/lib-python/2.7/distutils/tests/test_unixccompiler.py +++ b/lib-python/2.7/distutils/tests/test_unixccompiler.py @@ -1,6 +1,7 @@ """Tests for distutils.unixccompiler.""" import sys import unittest +from test.test_support import run_unittest from distutils import sysconfig from distutils.unixccompiler import UnixCCompiler @@ -126,4 +127,4 @@ return unittest.makeSuite(UnixCCompilerTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_upload.py b/lib-python/2.7/distutils/tests/test_upload.py --- a/lib-python/2.7/distutils/tests/test_upload.py +++ b/lib-python/2.7/distutils/tests/test_upload.py @@ -1,14 +1,13 @@ +# -*- encoding: utf8 -*- """Tests for distutils.command.upload.""" -# -*- encoding: utf8 -*- -import sys import os import unittest +from test.test_support import run_unittest from distutils.command import upload as upload_mod from distutils.command.upload import upload from distutils.core import Distribution -from distutils.tests import support from distutils.tests.test_config import PYPIRC, PyPIRCCommandTestCase PYPIRC_LONG_PASSWORD = """\ @@ -129,4 +128,4 @@ return unittest.makeSuite(uploadTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_util.py b/lib-python/2.7/distutils/tests/test_util.py --- a/lib-python/2.7/distutils/tests/test_util.py +++ b/lib-python/2.7/distutils/tests/test_util.py @@ -1,6 +1,7 @@ """Tests for distutils.util.""" import sys import unittest +from test.test_support import run_unittest from distutils.errors import DistutilsPlatformError, DistutilsByteCompileError from distutils.util import byte_compile @@ -21,4 +22,4 @@ return unittest.makeSuite(UtilTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_version.py b/lib-python/2.7/distutils/tests/test_version.py --- a/lib-python/2.7/distutils/tests/test_version.py +++ b/lib-python/2.7/distutils/tests/test_version.py @@ -2,6 +2,7 @@ import unittest from distutils.version import LooseVersion from distutils.version import StrictVersion +from test.test_support import run_unittest class VersionTestCase(unittest.TestCase): @@ -67,4 +68,4 @@ return unittest.makeSuite(VersionTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_versionpredicate.py b/lib-python/2.7/distutils/tests/test_versionpredicate.py --- a/lib-python/2.7/distutils/tests/test_versionpredicate.py +++ b/lib-python/2.7/distutils/tests/test_versionpredicate.py @@ -4,6 +4,10 @@ import distutils.versionpredicate import doctest +from test.test_support import run_unittest def test_suite(): return doctest.DocTestSuite(distutils.versionpredicate) + +if __name__ == '__main__': + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/util.py b/lib-python/2.7/distutils/util.py --- a/lib-python/2.7/distutils/util.py +++ b/lib-python/2.7/distutils/util.py @@ -97,9 +97,7 @@ from distutils.sysconfig import get_config_vars cfgvars = get_config_vars() - macver = os.environ.get('MACOSX_DEPLOYMENT_TARGET') - if not macver: - macver = cfgvars.get('MACOSX_DEPLOYMENT_TARGET') + macver = cfgvars.get('MACOSX_DEPLOYMENT_TARGET') if 1: # Always calculate the release of the running machine, diff --git a/lib-python/2.7/doctest.py b/lib-python/2.7/doctest.py --- a/lib-python/2.7/doctest.py +++ b/lib-python/2.7/doctest.py @@ -1217,7 +1217,7 @@ # Process each example. for examplenum, example in enumerate(test.examples): - # If REPORT_ONLY_FIRST_FAILURE is set, then supress + # If REPORT_ONLY_FIRST_FAILURE is set, then suppress # reporting after the first failure. quiet = (self.optionflags & REPORT_ONLY_FIRST_FAILURE and failures > 0) @@ -2186,7 +2186,7 @@ caller can catch the errors and initiate post-mortem debugging. The DocTestCase provides a debug method that raises - UnexpectedException errors if there is an unexepcted + UnexpectedException errors if there is an unexpected exception: >>> test = DocTestParser().get_doctest('>>> raise KeyError\n42', diff --git a/lib-python/2.7/email/charset.py b/lib-python/2.7/email/charset.py --- a/lib-python/2.7/email/charset.py +++ b/lib-python/2.7/email/charset.py @@ -209,7 +209,7 @@ input_charset = unicode(input_charset, 'ascii') except UnicodeError: raise errors.CharsetError(input_charset) - input_charset = input_charset.lower() + input_charset = input_charset.lower().encode('ascii') # Set the input charset after filtering through the aliases and/or codecs if not (input_charset in ALIASES or input_charset in CHARSETS): try: diff --git a/lib-python/2.7/email/generator.py b/lib-python/2.7/email/generator.py --- a/lib-python/2.7/email/generator.py +++ b/lib-python/2.7/email/generator.py @@ -202,18 +202,13 @@ g = self.clone(s) g.flatten(part, unixfrom=False) msgtexts.append(s.getvalue()) - # Now make sure the boundary we've selected doesn't appear in any of - # the message texts. - alltext = NL.join(msgtexts) # BAW: What about boundaries that are wrapped in double-quotes? - boundary = msg.get_boundary(failobj=_make_boundary(alltext)) - # If we had to calculate a new boundary because the body text - # contained that string, set the new boundary. We don't do it - # unconditionally because, while set_boundary() preserves order, it - # doesn't preserve newlines/continuations in headers. This is no big - # deal in practice, but turns out to be inconvenient for the unittest - # suite. - if msg.get_boundary() != boundary: + boundary = msg.get_boundary() + if not boundary: + # Create a boundary that doesn't appear in any of the + # message texts. + alltext = NL.join(msgtexts) + boundary = _make_boundary(alltext) msg.set_boundary(boundary) # If there's a preamble, write it out, with a trailing CRLF if msg.preamble is not None: @@ -292,7 +287,7 @@ _FMT = '[Non-text (%(type)s) part of message omitted, filename %(filename)s]' class DecodedGenerator(Generator): - """Generator a text representation of a message. + """Generates a text representation of a message. Like the Generator base class, except that non-text parts are substituted with a format string representing the part. diff --git a/lib-python/2.7/email/header.py b/lib-python/2.7/email/header.py --- a/lib-python/2.7/email/header.py +++ b/lib-python/2.7/email/header.py @@ -47,6 +47,10 @@ # For use with .match() fcre = re.compile(r'[\041-\176]+:$') +# Find a header embedded in a putative header value. Used to check for +# header injection attack. +_embeded_header = re.compile(r'\n[^ \t]+:') + # Helpers @@ -403,7 +407,11 @@ newchunks += self._split(s, charset, targetlen, splitchars) lastchunk, lastcharset = newchunks[-1] lastlen = lastcharset.encoded_header_len(lastchunk) - return self._encode_chunks(newchunks, maxlinelen) + value = self._encode_chunks(newchunks, maxlinelen) + if _embeded_header.search(value): + raise HeaderParseError("header value appears to contain " + "an embedded header: {!r}".format(value)) + return value diff --git a/lib-python/2.7/email/message.py b/lib-python/2.7/email/message.py --- a/lib-python/2.7/email/message.py +++ b/lib-python/2.7/email/message.py @@ -38,7 +38,9 @@ def _formatparam(param, value=None, quote=True): """Convenience function to format and return a key=value pair. - This will quote the value if needed or if quote is true. + This will quote the value if needed or if quote is true. If value is a + three tuple (charset, language, value), it will be encoded according + to RFC2231 rules. """ if value is not None and len(value) > 0: # A tuple is used for RFC 2231 encoded parameter values where items @@ -97,7 +99,7 @@ objects, otherwise it is a string. Message objects implement part of the `mapping' interface, which assumes - there is exactly one occurrance of the header per message. Some headers + there is exactly one occurrence of the header per message. Some headers do in fact appear multiple times (e.g. Received) and for those headers, you must use the explicit API to set or get all the headers. Not all of the mapping methods are implemented. @@ -286,7 +288,7 @@ Return None if the header is missing instead of raising an exception. Note that if the header appeared multiple times, exactly which - occurrance gets returned is undefined. Use get_all() to get all + occurrence gets returned is undefined. Use get_all() to get all the values matching a header field name. """ return self.get(name) @@ -389,7 +391,10 @@ name is the header field to add. keyword arguments can be used to set additional parameters for the header field, with underscores converted to dashes. Normally the parameter will be added as key="value" unless - value is None, in which case only the key will be added. + value is None, in which case only the key will be added. If a + parameter value contains non-ASCII characters it must be specified as a + three-tuple of (charset, language, value), in which case it will be + encoded according to RFC2231 rules. Example: diff --git a/lib-python/2.7/email/mime/application.py b/lib-python/2.7/email/mime/application.py --- a/lib-python/2.7/email/mime/application.py +++ b/lib-python/2.7/email/mime/application.py @@ -17,7 +17,7 @@ _encoder=encoders.encode_base64, **_params): """Create an application/* type MIME document. - _data is a string containing the raw applicatoin data. + _data is a string containing the raw application data. _subtype is the MIME content type subtype, defaulting to 'octet-stream'. diff --git a/lib-python/2.7/email/test/data/msg_26.txt b/lib-python/2.7/email/test/data/msg_26.txt --- a/lib-python/2.7/email/test/data/msg_26.txt +++ b/lib-python/2.7/email/test/data/msg_26.txt @@ -42,4 +42,4 @@ MzMAAAAACH97tzAAAAALu3c3gAAAAAAL+7tzDABAu7f7cAAAAAAACA+3MA7EQAv/sIAA AAAAAAAIAAAAAAAAAIAAAAAA ---1618492860--2051301190--113853680-- +--1618492860--2051301190--113853680-- \ No newline at end of file diff --git a/lib-python/2.7/email/test/test_email.py b/lib-python/2.7/email/test/test_email.py --- a/lib-python/2.7/email/test/test_email.py +++ b/lib-python/2.7/email/test/test_email.py @@ -179,6 +179,17 @@ self.assertRaises(Errors.HeaderParseError, msg.set_boundary, 'BOUNDARY') + def test_make_boundary(self): + msg = MIMEMultipart('form-data') + # Note that when the boundary gets created is an implementation + # detail and might change. + self.assertEqual(msg.items()[0][1], 'multipart/form-data') + # Trigger creation of boundary + msg.as_string() + self.assertEqual(msg.items()[0][1][:33], + 'multipart/form-data; boundary="==') + # XXX: there ought to be tests of the uniqueness of the boundary, too. + def test_message_rfc822_only(self): # Issue 7970: message/rfc822 not in multipart parsed by # HeaderParser caused an exception when flattened. @@ -542,6 +553,17 @@ msg.set_charset(u'us-ascii') self.assertEqual('us-ascii', msg.get_content_charset()) + # Issue 5871: reject an attempt to embed a header inside a header value + # (header injection attack). + def test_embeded_header_via_Header_rejected(self): + msg = Message() + msg['Dummy'] = Header('dummy\nX-Injected-Header: test') + self.assertRaises(Errors.HeaderParseError, msg.as_string) + + def test_embeded_header_via_string_rejected(self): + msg = Message() + msg['Dummy'] = 'dummy\nX-Injected-Header: test' + self.assertRaises(Errors.HeaderParseError, msg.as_string) # Test the email.Encoders module @@ -3113,6 +3135,28 @@ s = 'Subject: =?EUC-KR?B?CSixpLDtKSC/7Liuvsax4iC6uLmwMcijIKHaILzSwd/H0SC8+LCjwLsgv7W/+Mj3I ?=' raises(Errors.HeaderParseError, decode_header, s) + # Issue 1078919 + def test_ascii_add_header(self): + msg = Message() + msg.add_header('Content-Disposition', 'attachment', + filename='bud.gif') + self.assertEqual('attachment; filename="bud.gif"', + msg['Content-Disposition']) + + def test_nonascii_add_header_via_triple(self): + msg = Message() + msg.add_header('Content-Disposition', 'attachment', + filename=('iso-8859-1', '', 'Fu\xdfballer.ppt')) + self.assertEqual( + 'attachment; filename*="iso-8859-1\'\'Fu%DFballer.ppt"', + msg['Content-Disposition']) + + def test_encode_unaliased_charset(self): + # Issue 1379416: when the charset has no output conversion, + # output was accidentally getting coerced to unicode. + res = Header('abc','iso-8859-2').encode() + self.assertEqual(res, '=?iso-8859-2?q?abc?=') + self.assertIsInstance(res, str) # Test RFC 2231 header parameters (en/de)coding diff --git a/lib-python/2.7/ftplib.py b/lib-python/2.7/ftplib.py --- a/lib-python/2.7/ftplib.py +++ b/lib-python/2.7/ftplib.py @@ -599,7 +599,7 @@ Usage example: >>> from ftplib import FTP_TLS >>> ftps = FTP_TLS('ftp.python.org') - >>> ftps.login() # login anonimously previously securing control channel + >>> ftps.login() # login anonymously previously securing control channel '230 Guest login ok, access restrictions apply.' >>> ftps.prot_p() # switch to secure data connection '200 Protection level set to P' diff --git a/lib-python/2.7/functools.py b/lib-python/2.7/functools.py --- a/lib-python/2.7/functools.py +++ b/lib-python/2.7/functools.py @@ -53,17 +53,17 @@ def total_ordering(cls): """Class decorator that fills in missing ordering methods""" convert = { - '__lt__': [('__gt__', lambda self, other: other < self), - ('__le__', lambda self, other: not other < self), + '__lt__': [('__gt__', lambda self, other: not (self < other or self == other)), + ('__le__', lambda self, other: self < other or self == other), ('__ge__', lambda self, other: not self < other)], - '__le__': [('__ge__', lambda self, other: other <= self), - ('__lt__', lambda self, other: not other <= self), + '__le__': [('__ge__', lambda self, other: not self <= other or self == other), + ('__lt__', lambda self, other: self <= other and not self == other), ('__gt__', lambda self, other: not self <= other)], - '__gt__': [('__lt__', lambda self, other: other > self), - ('__ge__', lambda self, other: not other > self), + '__gt__': [('__lt__', lambda self, other: not (self > other or self == other)), + ('__ge__', lambda self, other: self > other or self == other), ('__le__', lambda self, other: not self > other)], - '__ge__': [('__le__', lambda self, other: other >= self), - ('__gt__', lambda self, other: not other >= self), + '__ge__': [('__le__', lambda self, other: (not self >= other) or self == other), + ('__gt__', lambda self, other: self >= other and not self == other), ('__lt__', lambda self, other: not self >= other)] } roots = set(dir(cls)) & set(convert) @@ -80,6 +80,7 @@ def cmp_to_key(mycmp): """Convert a cmp= function into a key= function""" class K(object): + __slots__ = ['obj'] def __init__(self, obj, *args): self.obj = obj def __lt__(self, other): diff --git a/lib-python/2.7/getpass.py b/lib-python/2.7/getpass.py --- a/lib-python/2.7/getpass.py +++ b/lib-python/2.7/getpass.py @@ -62,7 +62,7 @@ try: old = termios.tcgetattr(fd) # a copy to save new = old[:] - new[3] &= ~(termios.ECHO|termios.ISIG) # 3 == 'lflags' + new[3] &= ~termios.ECHO # 3 == 'lflags' tcsetattr_flags = termios.TCSAFLUSH if hasattr(termios, 'TCSASOFT'): tcsetattr_flags |= termios.TCSASOFT diff --git a/lib-python/2.7/gettext.py b/lib-python/2.7/gettext.py --- a/lib-python/2.7/gettext.py +++ b/lib-python/2.7/gettext.py @@ -316,7 +316,7 @@ # Note: we unconditionally convert both msgids and msgstrs to # Unicode using the character encoding specified in the charset # parameter of the Content-Type header. The gettext documentation - # strongly encourages msgids to be us-ascii, but some appliations + # strongly encourages msgids to be us-ascii, but some applications # require alternative encodings (e.g. Zope's ZCML and ZPT). For # traditional gettext applications, the msgid conversion will # cause no problems since us-ascii should always be a subset of diff --git a/lib-python/2.7/hashlib.py b/lib-python/2.7/hashlib.py --- a/lib-python/2.7/hashlib.py +++ b/lib-python/2.7/hashlib.py @@ -64,26 +64,29 @@ def __get_builtin_constructor(name): - if name in ('SHA1', 'sha1'): - import _sha - return _sha.new - elif name in ('MD5', 'md5'): - import _md5 - return _md5.new - elif name in ('SHA256', 'sha256', 'SHA224', 'sha224'): - import _sha256 - bs = name[3:] - if bs == '256': - return _sha256.sha256 - elif bs == '224': - return _sha256.sha224 - elif name in ('SHA512', 'sha512', 'SHA384', 'sha384'): - import _sha512 - bs = name[3:] - if bs == '512': - return _sha512.sha512 - elif bs == '384': - return _sha512.sha384 + try: + if name in ('SHA1', 'sha1'): + import _sha + return _sha.new + elif name in ('MD5', 'md5'): + import _md5 + return _md5.new + elif name in ('SHA256', 'sha256', 'SHA224', 'sha224'): + import _sha256 + bs = name[3:] + if bs == '256': + return _sha256.sha256 + elif bs == '224': + return _sha256.sha224 + elif name in ('SHA512', 'sha512', 'SHA384', 'sha384'): + import _sha512 + bs = name[3:] + if bs == '512': + return _sha512.sha512 + elif bs == '384': + return _sha512.sha384 + except ImportError: + pass # no extension module, this hash is unsupported. raise ValueError('unsupported hash type %s' % name) diff --git a/lib-python/2.7/heapq.py b/lib-python/2.7/heapq.py --- a/lib-python/2.7/heapq.py +++ b/lib-python/2.7/heapq.py @@ -133,6 +133,11 @@ from operator import itemgetter import bisect +def cmp_lt(x, y): + # Use __lt__ if available; otherwise, try __le__. + # In Py3.x, only __lt__ will be called. + return (x < y) if hasattr(x, '__lt__') else (not y <= x) + def heappush(heap, item): """Push item onto heap, maintaining the heap invariant.""" heap.append(item) @@ -167,13 +172,13 @@ def heappushpop(heap, item): """Fast version of a heappush followed by a heappop.""" - if heap and heap[0] < item: + if heap and cmp_lt(heap[0], item): item, heap[0] = heap[0], item _siftup(heap, 0) return item def heapify(x): - """Transform list into a heap, in-place, in O(len(heap)) time.""" + """Transform list into a heap, in-place, in O(len(x)) time.""" n = len(x) # Transform bottom-up. The largest index there's any point to looking at # is the largest with a child index in-range, so must have 2*i + 1 < n, @@ -215,11 +220,10 @@ pop = result.pop los = result[-1] # los --> Largest of the nsmallest for elem in it: - if los <= elem: - continue - insort(result, elem) - pop() - los = result[-1] + if cmp_lt(elem, los): + insort(result, elem) + pop() + los = result[-1] return result # An alternative approach manifests the whole iterable in memory but # saves comparisons by heapifying all at once. Also, saves time @@ -240,7 +244,7 @@ while pos > startpos: parentpos = (pos - 1) >> 1 parent = heap[parentpos] - if newitem < parent: + if cmp_lt(newitem, parent): heap[pos] = parent pos = parentpos continue @@ -295,7 +299,7 @@ while childpos < endpos: # Set childpos to index of smaller child. rightpos = childpos + 1 - if rightpos < endpos and not heap[childpos] < heap[rightpos]: + if rightpos < endpos and not cmp_lt(heap[childpos], heap[rightpos]): childpos = rightpos # Move the smaller child up. heap[pos] = heap[childpos] @@ -364,7 +368,7 @@ return [min(chain(head, it))] return [min(chain(head, it), key=key)] - # When n>=size, it's faster to use sort() + # When n>=size, it's faster to use sorted() try: size = len(iterable) except (TypeError, AttributeError): @@ -402,7 +406,7 @@ return [max(chain(head, it))] return [max(chain(head, it), key=key)] - # When n>=size, it's faster to use sort() + # When n>=size, it's faster to use sorted() try: size = len(iterable) except (TypeError, AttributeError): diff --git a/lib-python/2.7/httplib.py b/lib-python/2.7/httplib.py --- a/lib-python/2.7/httplib.py +++ b/lib-python/2.7/httplib.py @@ -212,6 +212,9 @@ # maximal amount of data to read at one time in _safe_read MAXAMOUNT = 1048576 +# maximal line length when calling readline(). +_MAXLINE = 65536 + class HTTPMessage(mimetools.Message): def addheader(self, key, value): @@ -274,7 +277,9 @@ except IOError: startofline = tell = None self.seekable = 0 - line = self.fp.readline() + line = self.fp.readline(_MAXLINE + 1) + if len(line) > _MAXLINE: + raise LineTooLong("header line") if not line: self.status = 'EOF in headers' break @@ -404,7 +409,10 @@ break # skip the header from the 100 response while True: - skip = self.fp.readline().strip() + skip = self.fp.readline(_MAXLINE + 1) + if len(skip) > _MAXLINE: + raise LineTooLong("header line") + skip = skip.strip() if not skip: break if self.debuglevel > 0: @@ -563,7 +571,9 @@ value = [] while True: if chunk_left is None: - line = self.fp.readline() + line = self.fp.readline(_MAXLINE + 1) + if len(line) > _MAXLINE: + raise LineTooLong("chunk size") i = line.find(';') if i >= 0: line = line[:i] # strip chunk-extensions @@ -598,7 +608,9 @@ # read and discard trailer up to the CRLF terminator ### note: we shouldn't have any trailers! while True: - line = self.fp.readline() + line = self.fp.readline(_MAXLINE + 1) + if len(line) > _MAXLINE: + raise LineTooLong("trailer line") if not line: # a vanishingly small number of sites EOF without # sending the trailer @@ -730,7 +742,9 @@ raise socket.error("Tunnel connection failed: %d %s" % (code, message.strip())) while True: - line = response.fp.readline() + line = response.fp.readline(_MAXLINE + 1) + if len(line) > _MAXLINE: + raise LineTooLong("header line") if line == '\r\n': break @@ -790,7 +804,7 @@ del self._buffer[:] # If msg and message_body are sent in a single send() call, # it will avoid performance problems caused by the interaction - # between delayed ack and the Nagle algorithim. + # between delayed ack and the Nagle algorithm. if isinstance(message_body, str): msg += message_body message_body = None @@ -1233,6 +1247,11 @@ self.args = line, self.line = line +class LineTooLong(HTTPException): + def __init__(self, line_type): + HTTPException.__init__(self, "got more than %d bytes when reading %s" + % (_MAXLINE, line_type)) + # for backwards compatibility error = HTTPException diff --git a/lib-python/2.7/idlelib/Bindings.py b/lib-python/2.7/idlelib/Bindings.py --- a/lib-python/2.7/idlelib/Bindings.py +++ b/lib-python/2.7/idlelib/Bindings.py @@ -98,14 +98,6 @@ # menu del menudefs[-1][1][0:2] - menudefs.insert(0, - ('application', [ - ('About IDLE', '<<about-idle>>'), - None, - ('_Preferences....', '<<open-config-dialog>>'), - ])) - - default_keydefs = idleConf.GetCurrentKeySet() del sys diff --git a/lib-python/2.7/idlelib/EditorWindow.py b/lib-python/2.7/idlelib/EditorWindow.py --- a/lib-python/2.7/idlelib/EditorWindow.py +++ b/lib-python/2.7/idlelib/EditorWindow.py @@ -48,6 +48,21 @@ path = module.__path__ except AttributeError: raise ImportError, 'No source for module ' + module.__name__ + if descr[2] != imp.PY_SOURCE: + # If all of the above fails and didn't raise an exception,fallback + # to a straight import which can find __init__.py in a package. + m = __import__(fullname) + try: + filename = m.__file__ + except AttributeError: + pass + else: + file = None + base, ext = os.path.splitext(filename) + if ext == '.pyc': + ext = '.py' + filename = base + ext + descr = filename, None, imp.PY_SOURCE return file, filename, descr class EditorWindow(object): @@ -102,8 +117,8 @@ self.top = top = WindowList.ListedToplevel(root, menu=self.menubar) if flist: self.tkinter_vars = flist.vars - #self.top.instance_dict makes flist.inversedict avalable to - #configDialog.py so it can access all EditorWindow instaces + #self.top.instance_dict makes flist.inversedict available to + #configDialog.py so it can access all EditorWindow instances self.top.instance_dict = flist.inversedict else: self.tkinter_vars = {} # keys: Tkinter event names @@ -136,6 +151,14 @@ if macosxSupport.runningAsOSXApp(): # Command-W on editorwindows doesn't work without this. text.bind('<<close-window>>', self.close_event) + # Some OS X systems have only one mouse button, + # so use control-click for pulldown menus there. + # (Note, AquaTk defines <2> as the right button if + # present and the Tk Text widget already binds <2>.) + text.bind("<Control-Button-1>",self.right_menu_event) + else: + # Elsewhere, use right-click for pulldown menus. + text.bind("<3>",self.right_menu_event) text.bind("<<cut>>", self.cut) text.bind("<<copy>>", self.copy) text.bind("<<paste>>", self.paste) @@ -154,7 +177,6 @@ text.bind("<<find-selection>>", self.find_selection_event) text.bind("<<replace>>", self.replace_event) text.bind("<<goto-line>>", self.goto_line_event) - text.bind("<3>", self.right_menu_event) text.bind("<<smart-backspace>>",self.smart_backspace_event) text.bind("<<newline-and-indent>>",self.newline_and_indent_event) text.bind("<<smart-indent>>",self.smart_indent_event) @@ -300,13 +322,13 @@ return "break" def home_callback(self, event): - if (event.state & 12) != 0 and event.keysym == "Home": - # state&1==shift, state&4==control, state&8==alt - return # <Modifier-Home>; fall back to class binding - + if (event.state & 4) != 0 and event.keysym == "Home": + # state&4==Control. If <Control-Home>, use the Tk binding. + return if self.text.index("iomark") and \ self.text.compare("iomark", "<=", "insert lineend") and \ self.text.compare("insert linestart", "<=", "iomark"): + # In Shell on input line, go to just after prompt insertpt = int(self.text.index("iomark").split(".")[1]) else: line = self.text.get("insert linestart", "insert lineend") @@ -315,30 +337,27 @@ break else: insertpt=len(line) - lineat = int(self.text.index("insert").split('.')[1]) - if insertpt == lineat: insertpt = 0 - dest = "insert linestart+"+str(insertpt)+"c" - if (event.state&1) == 0: - # shift not pressed + # shift was not pressed self.text.tag_remove("sel", "1.0", "end") else: if not self.text.index("sel.first"): - self.text.mark_set("anchor","insert") - + self.text.mark_set("my_anchor", "insert") # there was no previous selection + else: + if self.text.compare(self.text.index("sel.first"), "<", self.text.index("insert")): + self.text.mark_set("my_anchor", "sel.first") # extend back + else: + self.text.mark_set("my_anchor", "sel.last") # extend forward first = self.text.index(dest) - last = self.text.index("anchor") - + last = self.text.index("my_anchor") if self.text.compare(first,">",last): first,last = last,first - self.text.tag_remove("sel", "1.0", "end") self.text.tag_add("sel", first, last) - self.text.mark_set("insert", dest) self.text.see("insert") return "break" @@ -385,7 +404,7 @@ menudict[name] = menu = Menu(mbar, name=name) mbar.add_cascade(label=label, menu=menu, underline=underline) - if macosxSupport.runningAsOSXApp(): + if macosxSupport.isCarbonAquaTk(self.root): # Insert the application menu menudict['application'] = menu = Menu(mbar, name='apple') mbar.add_cascade(label='IDLE', menu=menu) @@ -445,7 +464,11 @@ def python_docs(self, event=None): if sys.platform[:3] == 'win': - os.startfile(self.help_url) + try: + os.startfile(self.help_url) + except WindowsError as why: + tkMessageBox.showerror(title='Document Start Failure', + message=str(why), parent=self.text) else: webbrowser.open(self.help_url) return "break" @@ -740,9 +763,13 @@ "Create a callback with the helpfile value frozen at definition time" def display_extra_help(helpfile=helpfile): if not helpfile.startswith(('www', 'http')): - url = os.path.normpath(helpfile) + helpfile = os.path.normpath(helpfile) if sys.platform[:3] == 'win': - os.startfile(helpfile) + try: + os.startfile(helpfile) + except WindowsError as why: + tkMessageBox.showerror(title='Document Start Failure', + message=str(why), parent=self.text) else: webbrowser.open(helpfile) return display_extra_help @@ -1526,7 +1553,12 @@ def get_accelerator(keydefs, eventname): keylist = keydefs.get(eventname) - if not keylist: + # issue10940: temporary workaround to prevent hang with OS X Cocoa Tk 8.5 + # if not keylist: + if (not keylist) or (macosxSupport.runningAsOSXApp() and eventname in { + "<<open-module>>", + "<<goto-line>>", + "<<change-indentwidth>>"}): return "" s = keylist[0] s = re.sub(r"-[a-z]\b", lambda m: m.group().upper(), s) diff --git a/lib-python/2.7/idlelib/FileList.py b/lib-python/2.7/idlelib/FileList.py --- a/lib-python/2.7/idlelib/FileList.py +++ b/lib-python/2.7/idlelib/FileList.py @@ -43,7 +43,7 @@ def new(self, filename=None): return self.EditorWindow(self, filename) - def close_all_callback(self, event): + def close_all_callback(self, *args, **kwds): for edit in self.inversedict.keys(): reply = edit.close() if reply == "cancel": diff --git a/lib-python/2.7/idlelib/FormatParagraph.py b/lib-python/2.7/idlelib/FormatParagraph.py --- a/lib-python/2.7/idlelib/FormatParagraph.py +++ b/lib-python/2.7/idlelib/FormatParagraph.py @@ -54,7 +54,7 @@ # If the block ends in a \n, we dont want the comment # prefix inserted after it. (Im not sure it makes sense to # reformat a comment block that isnt made of complete - # lines, but whatever!) Can't think of a clean soltution, + # lines, but whatever!) Can't think of a clean solution, # so we hack away block_suffix = "" if not newdata[-1]: diff --git a/lib-python/2.7/idlelib/HISTORY.txt b/lib-python/2.7/idlelib/HISTORY.txt --- a/lib-python/2.7/idlelib/HISTORY.txt +++ b/lib-python/2.7/idlelib/HISTORY.txt @@ -13,7 +13,7 @@ - New tarball released as a result of the 'revitalisation' of the IDLEfork project. -- This release requires python 2.1 or better. Compatability with earlier +- This release requires python 2.1 or better. Compatibility with earlier versions of python (especially ancient ones like 1.5x) is no longer a priority in IDLEfork development. diff --git a/lib-python/2.7/idlelib/IOBinding.py b/lib-python/2.7/idlelib/IOBinding.py --- a/lib-python/2.7/idlelib/IOBinding.py +++ b/lib-python/2.7/idlelib/IOBinding.py @@ -320,17 +320,20 @@ return "yes" message = "Do you want to save %s before closing?" % ( self.filename or "this untitled document") - m = tkMessageBox.Message( - title="Save On Close", - message=message, - icon=tkMessageBox.QUESTION, - type=tkMessageBox.YESNOCANCEL, - master=self.text) - reply = m.show() - if reply == "yes": + confirm = tkMessageBox.askyesnocancel( + title="Save On Close", + message=message, + default=tkMessageBox.YES, + master=self.text) + if confirm: + reply = "yes" self.save(None) if not self.get_saved(): reply = "cancel" + elif confirm is None: + reply = "cancel" + else: + reply = "no" self.text.focus_set() return reply @@ -339,7 +342,7 @@ self.save_as(event) else: if self.writefile(self.filename): - self.set_saved(1) + self.set_saved(True) try: self.editwin.store_file_breaks() except AttributeError: # may be a PyShell @@ -465,15 +468,12 @@ self.text.insert("end-1c", "\n") def print_window(self, event): - m = tkMessageBox.Message( - title="Print", - message="Print to Default Printer", - icon=tkMessageBox.QUESTION, - type=tkMessageBox.OKCANCEL, - default=tkMessageBox.OK, - master=self.text) - reply = m.show() - if reply != tkMessageBox.OK: + confirm = tkMessageBox.askokcancel( + title="Print", + message="Print to Default Printer", + default=tkMessageBox.OK, + master=self.text) + if not confirm: self.text.focus_set() return "break" tempfilename = None @@ -488,8 +488,8 @@ if not self.writefile(tempfilename): os.unlink(tempfilename) return "break" - platform=os.name - printPlatform=1 + platform = os.name + printPlatform = True if platform == 'posix': #posix platform command = idleConf.GetOption('main','General', 'print-command-posix') @@ -497,7 +497,7 @@ elif platform == 'nt': #win32 platform command = idleConf.GetOption('main','General','print-command-win') else: #no printing for this platform - printPlatform=0 + printPlatform = False if printPlatform: #we can try to print for this platform command = command % filename pipe = os.popen(command, "r") @@ -511,7 +511,7 @@ output = "Printing command: %s\n" % repr(command) + output tkMessageBox.showerror("Print status", output, master=self.text) else: #no printing for this platform - message="Printing is not enabled for this platform: %s" % platform + message = "Printing is not enabled for this platform: %s" % platform tkMessageBox.showinfo("Print status", message, master=self.text) if tempfilename: os.unlink(tempfilename) diff --git a/lib-python/2.7/idlelib/NEWS.txt b/lib-python/2.7/idlelib/NEWS.txt --- a/lib-python/2.7/idlelib/NEWS.txt +++ b/lib-python/2.7/idlelib/NEWS.txt @@ -1,3 +1,18 @@ +What's New in IDLE 2.7.2? +======================= + +*Release date: 29-May-2011* + +- Issue #6378: Further adjust idle.bat to start associated Python + +- Issue #11896: Save on Close failed despite selecting "Yes" in dialog. + +- <Home> toggle failing on Tk 8.5, causing IDLE exits and strange selection + behavior. Issue 4676. Improve selection extension behaviour. + +- <Home> toggle non-functional when NumLock set on Windows. Issue 3851. + + What's New in IDLE 2.7? ======================= @@ -21,7 +36,7 @@ - Tk 8.5 Text widget requires 'wordprocessor' tabstyle attr to handle mixed space/tab properly. Issue 5129, patch by Guilherme Polo. - + - Issue #3549: On MacOS the preferences menu was not present diff --git a/lib-python/2.7/idlelib/PyShell.py b/lib-python/2.7/idlelib/PyShell.py --- a/lib-python/2.7/idlelib/PyShell.py +++ b/lib-python/2.7/idlelib/PyShell.py @@ -1432,6 +1432,13 @@ shell.interp.prepend_syspath(script) shell.interp.execfile(script) + # Check for problematic OS X Tk versions and print a warning message + # in the IDLE shell window; this is less intrusive than always opening + # a separate window. + tkversionwarning = macosxSupport.tkVersionWarning(root) + if tkversionwarning: + shell.interp.runcommand(''.join(("print('", tkversionwarning, "')"))) + root.mainloop() root.destroy() diff --git a/lib-python/2.7/idlelib/ScriptBinding.py b/lib-python/2.7/idlelib/ScriptBinding.py --- a/lib-python/2.7/idlelib/ScriptBinding.py +++ b/lib-python/2.7/idlelib/ScriptBinding.py @@ -26,6 +26,7 @@ from idlelib import PyShell from idlelib.configHandler import idleConf +from idlelib import macosxSupport IDENTCHARS = string.ascii_letters + string.digits + "_" @@ -53,6 +54,9 @@ self.flist = self.editwin.flist self.root = self.editwin.root + if macosxSupport.runningAsOSXApp(): + self.editwin.text_frame.bind('<<run-module-event-2>>', self._run_module_event) + def check_module_event(self, event): filename = self.getfilename() if not filename: @@ -166,6 +170,19 @@ interp.runcode(code) return 'break' + if macosxSupport.runningAsOSXApp(): + # Tk-Cocoa in MacOSX is broken until at least + # Tk 8.5.9, and without this rather + # crude workaround IDLE would hang when a user + # tries to run a module using the keyboard shortcut + # (the menu item works fine). + _run_module_event = run_module_event + + def run_module_event(self, event): + self.editwin.text_frame.after(200, + lambda: self.editwin.text_frame.event_generate('<<run-module-event-2>>')) + return 'break' + def getfilename(self): """Get source filename. If not saved, offer to save (or create) file @@ -184,9 +201,9 @@ if autosave and filename: self.editwin.io.save(None) else: - reply = self.ask_save_dialog() + confirm = self.ask_save_dialog() self.editwin.text.focus_set() - if reply == "ok": + if confirm: self.editwin.io.save(None) filename = self.editwin.io.filename else: @@ -195,13 +212,11 @@ def ask_save_dialog(self): msg = "Source Must Be Saved\n" + 5*' ' + "OK to Save?" - mb = tkMessageBox.Message(title="Save Before Run or Check", - message=msg, - icon=tkMessageBox.QUESTION, - type=tkMessageBox.OKCANCEL, - default=tkMessageBox.OK, - master=self.editwin.text) - return mb.show() + confirm = tkMessageBox.askokcancel(title="Save Before Run or Check", + message=msg, + default=tkMessageBox.OK, + master=self.editwin.text) + return confirm def errorbox(self, title, message): # XXX This should really be a function of EditorWindow... diff --git a/lib-python/2.7/idlelib/config-keys.def b/lib-python/2.7/idlelib/config-keys.def --- a/lib-python/2.7/idlelib/config-keys.def +++ b/lib-python/2.7/idlelib/config-keys.def @@ -176,7 +176,7 @@ redo = <Shift-Command-Key-Z> close-window = <Command-Key-w> restart-shell = <Control-Key-F6> -save-window-as-file = <Command-Key-S> +save-window-as-file = <Shift-Command-Key-S> close-all-windows = <Command-Key-q> view-restart = <Key-F6> tabify-region = <Control-Key-5> @@ -208,7 +208,7 @@ open-module = <Command-Key-m> find-selection = <Shift-Command-Key-F3> python-context-help = <Shift-Key-F1> -save-copy-of-window-as-file = <Shift-Command-Key-s> +save-copy-of-window-as-file = <Option-Command-Key-s> open-window-from-file = <Command-Key-o> python-docs = <Key-F1> diff --git a/lib-python/2.7/idlelib/extend.txt b/lib-python/2.7/idlelib/extend.txt --- a/lib-python/2.7/idlelib/extend.txt +++ b/lib-python/2.7/idlelib/extend.txt @@ -18,7 +18,7 @@ An IDLE extension class is instantiated with a single argument, `editwin', an EditorWindow instance. The extension cannot assume much -about this argument, but it is guarateed to have the following instance +about this argument, but it is guaranteed to have the following instance variables: text a Text instance (a widget) diff --git a/lib-python/2.7/idlelib/idle.bat b/lib-python/2.7/idlelib/idle.bat --- a/lib-python/2.7/idlelib/idle.bat +++ b/lib-python/2.7/idlelib/idle.bat @@ -1,4 +1,4 @@ @echo off rem Start IDLE using the appropriate Python interpreter set CURRDIR=%~dp0 -start "%CURRDIR%..\..\pythonw.exe" "%CURRDIR%idle.pyw" %1 %2 %3 %4 %5 %6 %7 %8 %9 +start "IDLE" "%CURRDIR%..\..\pythonw.exe" "%CURRDIR%idle.pyw" %1 %2 %3 %4 %5 %6 %7 %8 %9 diff --git a/lib-python/2.7/idlelib/idlever.py b/lib-python/2.7/idlelib/idlever.py --- a/lib-python/2.7/idlelib/idlever.py +++ b/lib-python/2.7/idlelib/idlever.py @@ -1,1 +1,1 @@ -IDLE_VERSION = "2.7.1" +IDLE_VERSION = "2.7.2" diff --git a/lib-python/2.7/idlelib/macosxSupport.py b/lib-python/2.7/idlelib/macosxSupport.py --- a/lib-python/2.7/idlelib/macosxSupport.py +++ b/lib-python/2.7/idlelib/macosxSupport.py @@ -4,6 +4,7 @@ """ import sys import Tkinter +from os import path _appbundle = None @@ -19,10 +20,41 @@ _appbundle = (sys.platform == 'darwin' and '.app' in sys.executable) return _appbundle +_carbonaquatk = None + +def isCarbonAquaTk(root): + """ + Returns True if IDLE is using a Carbon Aqua Tk (instead of the + newer Cocoa Aqua Tk). + """ + global _carbonaquatk + if _carbonaquatk is None: + _carbonaquatk = (runningAsOSXApp() and + 'aqua' in root.tk.call('tk', 'windowingsystem') and + 'AppKit' not in root.tk.call('winfo', 'server', '.')) + return _carbonaquatk + +def tkVersionWarning(root): + """ + Returns a string warning message if the Tk version in use appears to + be one known to cause problems with IDLE. The Apple Cocoa-based Tk 8.5 + that was shipped with Mac OS X 10.6. + """ + + if (runningAsOSXApp() and + ('AppKit' in root.tk.call('winfo', 'server', '.')) and + (root.tk.call('info', 'patchlevel') == '8.5.7') ): + return (r"WARNING: The version of Tcl/Tk (8.5.7) in use may" + r" be unstable.\n" + r"Visit http://www.python.org/download/mac/tcltk/" + r" for current information.") + else: + return False + def addOpenEventSupport(root, flist): """ - This ensures that the application will respont to open AppleEvents, which - makes is feaseable to use IDLE as the default application for python files. + This ensures that the application will respond to open AppleEvents, which + makes is feasible to use IDLE as the default application for python files. """ def doOpenFile(*args): for fn in args: @@ -79,9 +111,6 @@ WindowList.add_windows_to_menu(menu) WindowList.register_callback(postwindowsmenu) - menudict['application'] = menu = Menu(menubar, name='apple') - menubar.add_cascade(label='IDLE', menu=menu) - def about_dialog(event=None): from idlelib import aboutDialog aboutDialog.AboutDialog(root, 'About IDLE') @@ -91,41 +120,45 @@ root.instance_dict = flist.inversedict configDialog.ConfigDialog(root, 'Settings') + def help_dialog(event=None): + from idlelib import textView + fn = path.join(path.abspath(path.dirname(__file__)), 'help.txt') + textView.view_file(root, 'Help', fn) root.bind('<<about-idle>>', about_dialog) root.bind('<<open-config-dialog>>', config_dialog) + root.createcommand('::tk::mac::ShowPreferences', config_dialog) if flist: root.bind('<<close-all-windows>>', flist.close_all_callback) + # The binding above doesn't reliably work on all versions of Tk + # on MacOSX. Adding command definition below does seem to do the + # right thing for now. + root.createcommand('exit', flist.close_all_callback) - ###check if Tk version >= 8.4.14; if so, use hard-coded showprefs binding - tkversion = root.tk.eval('info patchlevel') - # Note: we cannot check if the string tkversion >= '8.4.14', because - # the string '8.4.7' is greater than the string '8.4.14'. - if tuple(map(int, tkversion.split('.'))) >= (8, 4, 14): - Bindings.menudefs[0] = ('application', [ + if isCarbonAquaTk(root): + # for Carbon AquaTk, replace the default Tk apple menu + menudict['application'] = menu = Menu(menubar, name='apple') + menubar.add_cascade(label='IDLE', menu=menu) + Bindings.menudefs.insert(0, + ('application', [ ('About IDLE', '<<about-idle>>'), - None, - ]) - root.createcommand('::tk::mac::ShowPreferences', config_dialog) + None, + ])) + tkversion = root.tk.eval('info patchlevel') + if tuple(map(int, tkversion.split('.'))) < (8, 4, 14): + # for earlier AquaTk versions, supply a Preferences menu item + Bindings.menudefs[0][1].append( + ('_Preferences....', '<<open-config-dialog>>'), + ) else: - for mname, entrylist in Bindings.menudefs: - menu = menudict.get(mname) - if not menu: - continue - else: - for entry in entrylist: - if not entry: - menu.add_separator() - else: - label, eventname = entry - underline, label = prepstr(label) - accelerator = get_accelerator(Bindings.default_keydefs, - eventname) - def command(text=root, eventname=eventname): - text.event_generate(eventname) - menu.add_command(label=label, underline=underline, - command=command, accelerator=accelerator) + # assume Cocoa AquaTk + # replace default About dialog with About IDLE one + root.createcommand('tkAboutDialog', about_dialog) + # replace default "Help" item in Help menu + root.createcommand('::tk::mac::ShowHelp', help_dialog) + # remove redundant "IDLE Help" from menu + del Bindings.menudefs[-1][1][0] def setupApp(root, flist): """ diff --git a/lib-python/2.7/imaplib.py b/lib-python/2.7/imaplib.py --- a/lib-python/2.7/imaplib.py +++ b/lib-python/2.7/imaplib.py @@ -1158,28 +1158,17 @@ self.port = port self.sock = socket.create_connection((host, port)) self.sslobj = ssl.wrap_socket(self.sock, self.keyfile, self.certfile) + self.file = self.sslobj.makefile('rb') def read(self, size): """Read 'size' bytes from remote.""" - # sslobj.read() sometimes returns < size bytes - chunks = [] - read = 0 - while read < size: - data = self.sslobj.read(min(size-read, 16384)) - read += len(data) - chunks.append(data) - - return ''.join(chunks) + return self.file.read(size) def readline(self): """Read line from remote.""" - line = [] - while 1: - char = self.sslobj.read(1) - line.append(char) - if char in ("\n", ""): return ''.join(line) + return self.file.readline() def send(self, data): @@ -1195,6 +1184,7 @@ def shutdown(self): """Close I/O established in "open".""" + self.file.close() self.sock.close() @@ -1321,9 +1311,10 @@ 'Jul': 7, 'Aug': 8, 'Sep': 9, 'Oct': 10, 'Nov': 11, 'Dec': 12} def Internaldate2tuple(resp): - """Convert IMAP4 INTERNALDATE to UT. + """Parse an IMAP4 INTERNALDATE string. - Returns Python time module tuple. + Return corresponding local time. The return value is a + time.struct_time instance or None if the string has wrong format. """ mo = InternalDate.match(resp) @@ -1390,9 +1381,14 @@ def Time2Internaldate(date_time): - """Convert 'date_time' to IMAP4 INTERNALDATE representation. + """Convert date_time to IMAP4 INTERNALDATE representation. - Return string in form: '"DD-Mmm-YYYY HH:MM:SS +HHMM"' + Return string in form: '"DD-Mmm-YYYY HH:MM:SS +HHMM"'. The + date_time argument can be a number (int or float) representing + seconds since epoch (as returned by time.time()), a 9-tuple + representing local time (as returned by time.localtime()), or a + double-quoted string. In the last case, it is assumed to already + be in the correct format. """ if isinstance(date_time, (int, float)): diff --git a/lib-python/2.7/inspect.py b/lib-python/2.7/inspect.py --- a/lib-python/2.7/inspect.py +++ b/lib-python/2.7/inspect.py @@ -943,8 +943,14 @@ f_name, 'at most' if defaults else 'exactly', num_args, 'arguments' if num_args > 1 else 'argument', num_total)) elif num_args == 0 and num_total: - raise TypeError('%s() takes no arguments (%d given)' % - (f_name, num_total)) + if varkw: + if num_pos: + # XXX: We should use num_pos, but Python also uses num_total: + raise TypeError('%s() takes exactly 0 arguments ' + '(%d given)' % (f_name, num_total)) + else: + raise TypeError('%s() takes no arguments (%d given)' % + (f_name, num_total)) for arg in args: if isinstance(arg, str) and arg in named: if is_assigned(arg): diff --git a/lib-python/2.7/json/decoder.py b/lib-python/2.7/json/decoder.py --- a/lib-python/2.7/json/decoder.py +++ b/lib-python/2.7/json/decoder.py @@ -4,7 +4,7 @@ import sys import struct -from json.scanner import make_scanner +from json import scanner try: from _json import scanstring as c_scanstring except ImportError: @@ -161,6 +161,12 @@ nextchar = s[end:end + 1] # Trivial empty object if nextchar == '}': + if object_pairs_hook is not None: + result = object_pairs_hook(pairs) + return result, end + pairs = {} + if object_hook is not None: + pairs = object_hook(pairs) return pairs, end + 1 elif nextchar != '"': raise ValueError(errmsg("Expecting property name", s, end)) @@ -350,7 +356,7 @@ self.parse_object = JSONObject self.parse_array = JSONArray self.parse_string = scanstring - self.scan_once = make_scanner(self) + self.scan_once = scanner.make_scanner(self) def decode(self, s, _w=WHITESPACE.match): """Return the Python representation of ``s`` (a ``str`` or ``unicode`` diff --git a/lib-python/2.7/json/encoder.py b/lib-python/2.7/json/encoder.py --- a/lib-python/2.7/json/encoder.py +++ b/lib-python/2.7/json/encoder.py @@ -251,7 +251,7 @@ if (_one_shot and c_make_encoder is not None - and not self.indent and not self.sort_keys): + and self.indent is None and not self.sort_keys): _iterencode = c_make_encoder( markers, self.default, _encoder, self.indent, self.key_separator, self.item_separator, self.sort_keys, diff --git a/lib-python/2.7/json/tests/__init__.py b/lib-python/2.7/json/tests/__init__.py --- a/lib-python/2.7/json/tests/__init__.py +++ b/lib-python/2.7/json/tests/__init__.py @@ -1,7 +1,46 @@ import os import sys +import json +import doctest import unittest -import doctest + +from test import test_support + +# import json with and without accelerations +cjson = test_support.import_fresh_module('json', fresh=['_json']) +pyjson = test_support.import_fresh_module('json', blocked=['_json']) + +# create two base classes that will be used by the other tests +class PyTest(unittest.TestCase): + json = pyjson + loads = staticmethod(pyjson.loads) + dumps = staticmethod(pyjson.dumps) + + at unittest.skipUnless(cjson, 'requires _json') +class CTest(unittest.TestCase): + if cjson is not None: + json = cjson + loads = staticmethod(cjson.loads) + dumps = staticmethod(cjson.dumps) + +# test PyTest and CTest checking if the functions come from the right module +class TestPyTest(PyTest): + def test_pyjson(self): + self.assertEqual(self.json.scanner.make_scanner.__module__, + 'json.scanner') + self.assertEqual(self.json.decoder.scanstring.__module__, + 'json.decoder') + self.assertEqual(self.json.encoder.encode_basestring_ascii.__module__, + 'json.encoder') + +class TestCTest(CTest): + def test_cjson(self): + self.assertEqual(self.json.scanner.make_scanner.__module__, '_json') + self.assertEqual(self.json.decoder.scanstring.__module__, '_json') + self.assertEqual(self.json.encoder.c_make_encoder.__module__, '_json') + self.assertEqual(self.json.encoder.encode_basestring_ascii.__module__, + '_json') + here = os.path.dirname(__file__) @@ -17,12 +56,11 @@ return suite def additional_tests(): - import json - import json.encoder - import json.decoder suite = unittest.TestSuite() for mod in (json, json.encoder, json.decoder): suite.addTest(doctest.DocTestSuite(mod)) + suite.addTest(TestPyTest('test_pyjson')) + suite.addTest(TestCTest('test_cjson')) return suite def main(): diff --git a/lib-python/2.7/json/tests/test_check_circular.py b/lib-python/2.7/json/tests/test_check_circular.py --- a/lib-python/2.7/json/tests/test_check_circular.py +++ b/lib-python/2.7/json/tests/test_check_circular.py @@ -1,30 +1,34 @@ -from unittest import TestCase -import json +from json.tests import PyTest, CTest + def default_iterable(obj): return list(obj) -class TestCheckCircular(TestCase): +class TestCheckCircular(object): def test_circular_dict(self): dct = {} dct['a'] = dct - self.assertRaises(ValueError, json.dumps, dct) + self.assertRaises(ValueError, self.dumps, dct) def test_circular_list(self): lst = [] lst.append(lst) - self.assertRaises(ValueError, json.dumps, lst) + self.assertRaises(ValueError, self.dumps, lst) def test_circular_composite(self): dct2 = {} dct2['a'] = [] dct2['a'].append(dct2) - self.assertRaises(ValueError, json.dumps, dct2) + self.assertRaises(ValueError, self.dumps, dct2) def test_circular_default(self): - json.dumps([set()], default=default_iterable) - self.assertRaises(TypeError, json.dumps, [set()]) + self.dumps([set()], default=default_iterable) + self.assertRaises(TypeError, self.dumps, [set()]) def test_circular_off_default(self): - json.dumps([set()], default=default_iterable, check_circular=False) - self.assertRaises(TypeError, json.dumps, [set()], check_circular=False) + self.dumps([set()], default=default_iterable, check_circular=False) + self.assertRaises(TypeError, self.dumps, [set()], check_circular=False) + + +class TestPyCheckCircular(TestCheckCircular, PyTest): pass +class TestCCheckCircular(TestCheckCircular, CTest): pass diff --git a/lib-python/2.7/json/tests/test_decode.py b/lib-python/2.7/json/tests/test_decode.py --- a/lib-python/2.7/json/tests/test_decode.py +++ b/lib-python/2.7/json/tests/test_decode.py @@ -1,18 +1,17 @@ import decimal -from unittest import TestCase from StringIO import StringIO +from collections import OrderedDict +from json.tests import PyTest, CTest -import json -from collections import OrderedDict -class TestDecode(TestCase): +class TestDecode(object): def test_decimal(self): - rval = json.loads('1.1', parse_float=decimal.Decimal) + rval = self.loads('1.1', parse_float=decimal.Decimal) self.assertTrue(isinstance(rval, decimal.Decimal)) self.assertEqual(rval, decimal.Decimal('1.1')) def test_float(self): - rval = json.loads('1', parse_int=float) + rval = self.loads('1', parse_int=float) self.assertTrue(isinstance(rval, float)) self.assertEqual(rval, 1.0) @@ -20,22 +19,32 @@ # Several optimizations were made that skip over calls to # the whitespace regex, so this test is designed to try and # exercise the uncommon cases. The array cases are already covered. - rval = json.loads('{ "key" : "value" , "k":"v" }') + rval = self.loads('{ "key" : "value" , "k":"v" }') self.assertEqual(rval, {"key":"value", "k":"v"}) + def test_empty_objects(self): + self.assertEqual(self.loads('{}'), {}) + self.assertEqual(self.loads('[]'), []) + self.assertEqual(self.loads('""'), u"") + self.assertIsInstance(self.loads('""'), unicode) + def test_object_pairs_hook(self): s = '{"xkd":1, "kcw":2, "art":3, "hxm":4, "qrt":5, "pad":6, "hoy":7}' p = [("xkd", 1), ("kcw", 2), ("art", 3), ("hxm", 4), ("qrt", 5), ("pad", 6), ("hoy", 7)] - self.assertEqual(json.loads(s), eval(s)) - self.assertEqual(json.loads(s, object_pairs_hook=lambda x: x), p) - self.assertEqual(json.load(StringIO(s), - object_pairs_hook=lambda x: x), p) - od = json.loads(s, object_pairs_hook=OrderedDict) + self.assertEqual(self.loads(s), eval(s)) + self.assertEqual(self.loads(s, object_pairs_hook=lambda x: x), p) + self.assertEqual(self.json.load(StringIO(s), + object_pairs_hook=lambda x: x), p) + od = self.loads(s, object_pairs_hook=OrderedDict) self.assertEqual(od, OrderedDict(p)) self.assertEqual(type(od), OrderedDict) # the object_pairs_hook takes priority over the object_hook - self.assertEqual(json.loads(s, + self.assertEqual(self.loads(s, object_pairs_hook=OrderedDict, object_hook=lambda x: None), OrderedDict(p)) + + +class TestPyDecode(TestDecode, PyTest): pass +class TestCDecode(TestDecode, CTest): pass diff --git a/lib-python/2.7/json/tests/test_default.py b/lib-python/2.7/json/tests/test_default.py --- a/lib-python/2.7/json/tests/test_default.py +++ b/lib-python/2.7/json/tests/test_default.py @@ -1,9 +1,12 @@ -from unittest import TestCase +from json.tests import PyTest, CTest -import json -class TestDefault(TestCase): +class TestDefault(object): def test_default(self): self.assertEqual( - json.dumps(type, default=repr), - json.dumps(repr(type))) + self.dumps(type, default=repr), + self.dumps(repr(type))) + + +class TestPyDefault(TestDefault, PyTest): pass +class TestCDefault(TestDefault, CTest): pass diff --git a/lib-python/2.7/json/tests/test_dump.py b/lib-python/2.7/json/tests/test_dump.py --- a/lib-python/2.7/json/tests/test_dump.py +++ b/lib-python/2.7/json/tests/test_dump.py @@ -1,21 +1,23 @@ -from unittest import TestCase from cStringIO import StringIO +from json.tests import PyTest, CTest -import json -class TestDump(TestCase): +class TestDump(object): def test_dump(self): sio = StringIO() - json.dump({}, sio) + self.json.dump({}, sio) self.assertEqual(sio.getvalue(), '{}') def test_dumps(self): - self.assertEqual(json.dumps({}), '{}') + self.assertEqual(self.dumps({}), '{}') def test_encode_truefalse(self): - self.assertEqual(json.dumps( + self.assertEqual(self.dumps( {True: False, False: True}, sort_keys=True), '{"false": true, "true": false}') - self.assertEqual(json.dumps( + self.assertEqual(self.dumps( {2: 3.0, 4.0: 5L, False: 1, 6L: True}, sort_keys=True), '{"false": 1, "2": 3.0, "4.0": 5, "6": true}') + +class TestPyDump(TestDump, PyTest): pass +class TestCDump(TestDump, CTest): pass diff --git a/lib-python/2.7/json/tests/test_encode_basestring_ascii.py b/lib-python/2.7/json/tests/test_encode_basestring_ascii.py --- a/lib-python/2.7/json/tests/test_encode_basestring_ascii.py +++ b/lib-python/2.7/json/tests/test_encode_basestring_ascii.py @@ -1,8 +1,6 @@ -from unittest import TestCase +from collections import OrderedDict +from json.tests import PyTest, CTest -import json.encoder -from json import dumps -from collections import OrderedDict CASES = [ (u'/\\"\ucafe\ubabe\uab98\ufcde\ubcda\uef4a\x08\x0c\n\r\t`1~!@#$%^&*()_+-=[]{}|;:\',./<>?', '"/\\\\\\"\\ucafe\\ubabe\\uab98\\ufcde\\ubcda\\uef4a\\b\\f\\n\\r\\t`1~!@#$%^&*()_+-=[]{}|;:\',./<>?"'), @@ -23,19 +21,11 @@ (u'\u0123\u4567\u89ab\ucdef\uabcd\uef4a', '"\\u0123\\u4567\\u89ab\\ucdef\\uabcd\\uef4a"'), ] -class TestEncodeBaseStringAscii(TestCase): - def test_py_encode_basestring_ascii(self): - self._test_encode_basestring_ascii(json.encoder.py_encode_basestring_ascii) - - def test_c_encode_basestring_ascii(self): - if not json.encoder.c_encode_basestring_ascii: - return - self._test_encode_basestring_ascii(json.encoder.c_encode_basestring_ascii) - - def _test_encode_basestring_ascii(self, encode_basestring_ascii): - fname = encode_basestring_ascii.__name__ +class TestEncodeBasestringAscii(object): + def test_encode_basestring_ascii(self): + fname = self.json.encoder.encode_basestring_ascii.__name__ for input_string, expect in CASES: - result = encode_basestring_ascii(input_string) + result = self.json.encoder.encode_basestring_ascii(input_string) self.assertEqual(result, expect, '{0!r} != {1!r} for {2}({3!r})'.format( result, expect, fname, input_string)) @@ -43,5 +33,9 @@ def test_ordered_dict(self): # See issue 6105 items = [('one', 1), ('two', 2), ('three', 3), ('four', 4), ('five', 5)] - s = json.dumps(OrderedDict(items)) + s = self.dumps(OrderedDict(items)) self.assertEqual(s, '{"one": 1, "two": 2, "three": 3, "four": 4, "five": 5}') + + +class TestPyEncodeBasestringAscii(TestEncodeBasestringAscii, PyTest): pass +class TestCEncodeBasestringAscii(TestEncodeBasestringAscii, CTest): pass diff --git a/lib-python/2.7/json/tests/test_fail.py b/lib-python/2.7/json/tests/test_fail.py --- a/lib-python/2.7/json/tests/test_fail.py +++ b/lib-python/2.7/json/tests/test_fail.py @@ -1,6 +1,4 @@ -from unittest import TestCase - -import json +from json.tests import PyTest, CTest # Fri Dec 30 18:57:26 2005 JSONDOCS = [ @@ -61,15 +59,15 @@ 18: "spec doesn't specify any nesting limitations", } -class TestFail(TestCase): +class TestFail(object): def test_failures(self): for idx, doc in enumerate(JSONDOCS): idx = idx + 1 if idx in SKIPS: - json.loads(doc) + self.loads(doc) continue try: - json.loads(doc) + self.loads(doc) except ValueError: pass else: @@ -79,7 +77,11 @@ data = {'a' : 1, (1, 2) : 2} #This is for c encoder - self.assertRaises(TypeError, json.dumps, data) + self.assertRaises(TypeError, self.dumps, data) #This is for python encoder - self.assertRaises(TypeError, json.dumps, data, indent=True) + self.assertRaises(TypeError, self.dumps, data, indent=True) + + +class TestPyFail(TestFail, PyTest): pass +class TestCFail(TestFail, CTest): pass diff --git a/lib-python/2.7/json/tests/test_float.py b/lib-python/2.7/json/tests/test_float.py --- a/lib-python/2.7/json/tests/test_float.py +++ b/lib-python/2.7/json/tests/test_float.py @@ -1,19 +1,22 @@ import math -from unittest import TestCase +from json.tests import PyTest, CTest -import json -class TestFloat(TestCase): +class TestFloat(object): def test_floats(self): for num in [1617161771.7650001, math.pi, math.pi**100, math.pi**-100, 3.1]: - self.assertEqual(float(json.dumps(num)), num) - self.assertEqual(json.loads(json.dumps(num)), num) - self.assertEqual(json.loads(unicode(json.dumps(num))), num) + self.assertEqual(float(self.dumps(num)), num) + self.assertEqual(self.loads(self.dumps(num)), num) + self.assertEqual(self.loads(unicode(self.dumps(num))), num) def test_ints(self): for num in [1, 1L, 1<<32, 1<<64]: - self.assertEqual(json.dumps(num), str(num)) - self.assertEqual(int(json.dumps(num)), num) - self.assertEqual(json.loads(json.dumps(num)), num) - self.assertEqual(json.loads(unicode(json.dumps(num))), num) + self.assertEqual(self.dumps(num), str(num)) + self.assertEqual(int(self.dumps(num)), num) + self.assertEqual(self.loads(self.dumps(num)), num) + self.assertEqual(self.loads(unicode(self.dumps(num))), num) + + +class TestPyFloat(TestFloat, PyTest): pass +class TestCFloat(TestFloat, CTest): pass diff --git a/lib-python/2.7/json/tests/test_indent.py b/lib-python/2.7/json/tests/test_indent.py --- a/lib-python/2.7/json/tests/test_indent.py +++ b/lib-python/2.7/json/tests/test_indent.py @@ -1,9 +1,9 @@ -from unittest import TestCase +import textwrap +from StringIO import StringIO +from json.tests import PyTest, CTest -import json -import textwrap -class TestIndent(TestCase): +class TestIndent(object): def test_indent(self): h = [['blorpie'], ['whoops'], [], 'd-shtaeou', 'd-nthiouh', 'i-vhbjkhnth', {'nifty': 87}, {'field': 'yes', 'morefield': False} ] @@ -30,12 +30,31 @@ ]""") - d1 = json.dumps(h) - d2 = json.dumps(h, indent=2, sort_keys=True, separators=(',', ': ')) + d1 = self.dumps(h) + d2 = self.dumps(h, indent=2, sort_keys=True, separators=(',', ': ')) - h1 = json.loads(d1) - h2 = json.loads(d2) + h1 = self.loads(d1) + h2 = self.loads(d2) self.assertEqual(h1, h) self.assertEqual(h2, h) self.assertEqual(d2, expect) + + def test_indent0(self): + h = {3: 1} + def check(indent, expected): + d1 = self.dumps(h, indent=indent) + self.assertEqual(d1, expected) + + sio = StringIO() + self.json.dump(h, sio, indent=indent) + self.assertEqual(sio.getvalue(), expected) + + # indent=0 should emit newlines + check(0, '{\n"3": 1\n}') + # indent=None is more compact + check(None, '{"3": 1}') + + +class TestPyIndent(TestIndent, PyTest): pass +class TestCIndent(TestIndent, CTest): pass diff --git a/lib-python/2.7/json/tests/test_pass1.py b/lib-python/2.7/json/tests/test_pass1.py --- a/lib-python/2.7/json/tests/test_pass1.py +++ b/lib-python/2.7/json/tests/test_pass1.py @@ -1,6 +1,5 @@ -from unittest import TestCase +from json.tests import PyTest, CTest -import json # from http://json.org/JSON_checker/test/pass1.json JSON = r''' @@ -62,15 +61,19 @@ ,"rosebud"] ''' -class TestPass1(TestCase): +class TestPass1(object): def test_parse(self): # test in/out equivalence and parsing - res = json.loads(JSON) - out = json.dumps(res) - self.assertEqual(res, json.loads(out)) + res = self.loads(JSON) + out = self.dumps(res) + self.assertEqual(res, self.loads(out)) try: - json.dumps(res, allow_nan=False) + self.dumps(res, allow_nan=False) except ValueError: pass else: self.fail("23456789012E666 should be out of range") + + +class TestPyPass1(TestPass1, PyTest): pass +class TestCPass1(TestPass1, CTest): pass diff --git a/lib-python/2.7/json/tests/test_pass2.py b/lib-python/2.7/json/tests/test_pass2.py --- a/lib-python/2.7/json/tests/test_pass2.py +++ b/lib-python/2.7/json/tests/test_pass2.py @@ -1,14 +1,18 @@ -from unittest import TestCase -import json +from json.tests import PyTest, CTest + # from http://json.org/JSON_checker/test/pass2.json JSON = r''' [[[[[[[[[[[[[[[[[[["Not too deep"]]]]]]]]]]]]]]]]]]] ''' -class TestPass2(TestCase): +class TestPass2(object): def test_parse(self): # test in/out equivalence and parsing - res = json.loads(JSON) - out = json.dumps(res) - self.assertEqual(res, json.loads(out)) + res = self.loads(JSON) + out = self.dumps(res) + self.assertEqual(res, self.loads(out)) + + +class TestPyPass2(TestPass2, PyTest): pass +class TestCPass2(TestPass2, CTest): pass diff --git a/lib-python/2.7/json/tests/test_pass3.py b/lib-python/2.7/json/tests/test_pass3.py --- a/lib-python/2.7/json/tests/test_pass3.py +++ b/lib-python/2.7/json/tests/test_pass3.py @@ -1,6 +1,5 @@ -from unittest import TestCase +from json.tests import PyTest, CTest -import json # from http://json.org/JSON_checker/test/pass3.json JSON = r''' @@ -12,9 +11,14 @@ } ''' -class TestPass3(TestCase): + +class TestPass3(object): def test_parse(self): # test in/out equivalence and parsing - res = json.loads(JSON) - out = json.dumps(res) - self.assertEqual(res, json.loads(out)) + res = self.loads(JSON) + out = self.dumps(res) + self.assertEqual(res, self.loads(out)) + + +class TestPyPass3(TestPass3, PyTest): pass +class TestCPass3(TestPass3, CTest): pass diff --git a/lib-python/2.7/json/tests/test_recursion.py b/lib-python/2.7/json/tests/test_recursion.py --- a/lib-python/2.7/json/tests/test_recursion.py +++ b/lib-python/2.7/json/tests/test_recursion.py @@ -1,28 +1,16 @@ -from unittest import TestCase +from json.tests import PyTest, CTest -import json class JSONTestObject: pass -class RecursiveJSONEncoder(json.JSONEncoder): - recurse = False - def default(self, o): - if o is JSONTestObject: - if self.recurse: - return [JSONTestObject] - else: - return 'JSONTestObject' - return json.JSONEncoder.default(o) - - -class TestRecursion(TestCase): +class TestRecursion(object): def test_listrecursion(self): x = [] x.append(x) try: - json.dumps(x) + self.dumps(x) except ValueError: pass else: @@ -31,7 +19,7 @@ y = [x] x.append(y) try: - json.dumps(x) + self.dumps(x) except ValueError: pass else: @@ -39,13 +27,13 @@ y = [] x = [y, y] # ensure that the marker is cleared - json.dumps(x) + self.dumps(x) def test_dictrecursion(self): x = {} x["test"] = x try: - json.dumps(x) + self.dumps(x) except ValueError: pass else: @@ -53,9 +41,19 @@ x = {} y = {"a": x, "b": x} # ensure that the marker is cleared - json.dumps(x) + self.dumps(x) def test_defaultrecursion(self): + class RecursiveJSONEncoder(self.json.JSONEncoder): + recurse = False + def default(self, o): + if o is JSONTestObject: + if self.recurse: + return [JSONTestObject] + else: + return 'JSONTestObject' + return pyjson.JSONEncoder.default(o) + enc = RecursiveJSONEncoder() self.assertEqual(enc.encode(JSONTestObject), '"JSONTestObject"') enc.recurse = True @@ -65,3 +63,46 @@ pass else: self.fail("didn't raise ValueError on default recursion") + + + def test_highly_nested_objects_decoding(self): + # test that loading highly-nested objects doesn't segfault when C + # accelerations are used. See #12017 + # str + with self.assertRaises(RuntimeError): + self.loads('{"a":' * 100000 + '1' + '}' * 100000) + with self.assertRaises(RuntimeError): + self.loads('{"a":' * 100000 + '[1]' + '}' * 100000) + with self.assertRaises(RuntimeError): + self.loads('[' * 100000 + '1' + ']' * 100000) + # unicode + with self.assertRaises(RuntimeError): + self.loads(u'{"a":' * 100000 + u'1' + u'}' * 100000) + with self.assertRaises(RuntimeError): + self.loads(u'{"a":' * 100000 + u'[1]' + u'}' * 100000) + with self.assertRaises(RuntimeError): + self.loads(u'[' * 100000 + u'1' + u']' * 100000) + + def test_highly_nested_objects_encoding(self): + # See #12051 + l, d = [], {} + for x in xrange(100000): + l, d = [l], {'k':d} + with self.assertRaises(RuntimeError): + self.dumps(l) + with self.assertRaises(RuntimeError): + self.dumps(d) + + def test_endless_recursion(self): + # See #12051 + class EndlessJSONEncoder(self.json.JSONEncoder): + def default(self, o): + """If check_circular is False, this will keep adding another list.""" + return [o] + + with self.assertRaises(RuntimeError): + EndlessJSONEncoder(check_circular=False).encode(5j) + + +class TestPyRecursion(TestRecursion, PyTest): pass +class TestCRecursion(TestRecursion, CTest): pass diff --git a/lib-python/2.7/json/tests/test_scanstring.py b/lib-python/2.7/json/tests/test_scanstring.py --- a/lib-python/2.7/json/tests/test_scanstring.py +++ b/lib-python/2.7/json/tests/test_scanstring.py @@ -1,18 +1,10 @@ import sys -import decimal -from unittest import TestCase +from json.tests import PyTest, CTest -import json -import json.decoder -class TestScanString(TestCase): - def test_py_scanstring(self): - self._test_scanstring(json.decoder.py_scanstring) - - def test_c_scanstring(self): - self._test_scanstring(json.decoder.c_scanstring) - - def _test_scanstring(self, scanstring): +class TestScanstring(object): + def test_scanstring(self): + scanstring = self.json.decoder.scanstring self.assertEqual( scanstring('"z\\ud834\\udd20x"', 1, None, True), (u'z\U0001d120x', 16)) @@ -103,10 +95,15 @@ (u'Bad value', 12)) def test_issue3623(self): - self.assertRaises(ValueError, json.decoder.scanstring, b"xxx", 1, + self.assertRaises(ValueError, self.json.decoder.scanstring, b"xxx", 1, "xxx") self.assertRaises(UnicodeDecodeError, - json.encoder.encode_basestring_ascii, b"xx\xff") + self.json.encoder.encode_basestring_ascii, b"xx\xff") def test_overflow(self): - self.assertRaises(OverflowError, json.decoder.scanstring, b"xxx", sys.maxsize+1) + with self.assertRaises(OverflowError): + self.json.decoder.scanstring(b"xxx", sys.maxsize+1) + + +class TestPyScanstring(TestScanstring, PyTest): pass +class TestCScanstring(TestScanstring, CTest): pass diff --git a/lib-python/2.7/json/tests/test_separators.py b/lib-python/2.7/json/tests/test_separators.py --- a/lib-python/2.7/json/tests/test_separators.py +++ b/lib-python/2.7/json/tests/test_separators.py @@ -1,10 +1,8 @@ import textwrap -from unittest import TestCase +from json.tests import PyTest, CTest -import json - -class TestSeparators(TestCase): +class TestSeparators(object): def test_separators(self): h = [['blorpie'], ['whoops'], [], 'd-shtaeou', 'd-nthiouh', 'i-vhbjkhnth', {'nifty': 87}, {'field': 'yes', 'morefield': False} ] @@ -31,12 +29,16 @@ ]""") - d1 = json.dumps(h) - d2 = json.dumps(h, indent=2, sort_keys=True, separators=(' ,', ' : ')) + d1 = self.dumps(h) + d2 = self.dumps(h, indent=2, sort_keys=True, separators=(' ,', ' : ')) - h1 = json.loads(d1) - h2 = json.loads(d2) + h1 = self.loads(d1) + h2 = self.loads(d2) self.assertEqual(h1, h) self.assertEqual(h2, h) self.assertEqual(d2, expect) + + +class TestPySeparators(TestSeparators, PyTest): pass +class TestCSeparators(TestSeparators, CTest): pass diff --git a/lib-python/2.7/json/tests/test_speedups.py b/lib-python/2.7/json/tests/test_speedups.py --- a/lib-python/2.7/json/tests/test_speedups.py +++ b/lib-python/2.7/json/tests/test_speedups.py @@ -1,24 +1,23 @@ -import decimal -from unittest import TestCase +from json.tests import CTest -from json import decoder, encoder, scanner -class TestSpeedups(TestCase): +class TestSpeedups(CTest): def test_scanstring(self): - self.assertEqual(decoder.scanstring.__module__, "_json") - self.assertTrue(decoder.scanstring is decoder.c_scanstring) + self.assertEqual(self.json.decoder.scanstring.__module__, "_json") + self.assertIs(self.json.decoder.scanstring, self.json.decoder.c_scanstring) def test_encode_basestring_ascii(self): - self.assertEqual(encoder.encode_basestring_ascii.__module__, "_json") - self.assertTrue(encoder.encode_basestring_ascii is - encoder.c_encode_basestring_ascii) + self.assertEqual(self.json.encoder.encode_basestring_ascii.__module__, + "_json") + self.assertIs(self.json.encoder.encode_basestring_ascii, + self.json.encoder.c_encode_basestring_ascii) -class TestDecode(TestCase): +class TestDecode(CTest): def test_make_scanner(self): - self.assertRaises(AttributeError, scanner.c_make_scanner, 1) + self.assertRaises(AttributeError, self.json.scanner.c_make_scanner, 1) def test_make_encoder(self): - self.assertRaises(TypeError, encoder.c_make_encoder, + self.assertRaises(TypeError, self.json.encoder.c_make_encoder, None, "\xCD\x7D\x3D\x4E\x12\x4C\xF9\x79\xD7\x52\xBA\x82\xF2\x27\x4A\x7D\xA0\xCA\x75", None) diff --git a/lib-python/2.7/json/tests/test_unicode.py b/lib-python/2.7/json/tests/test_unicode.py --- a/lib-python/2.7/json/tests/test_unicode.py +++ b/lib-python/2.7/json/tests/test_unicode.py @@ -1,11 +1,10 @@ -from unittest import TestCase +from collections import OrderedDict +from json.tests import PyTest, CTest -import json -from collections import OrderedDict -class TestUnicode(TestCase): +class TestUnicode(object): def test_encoding1(self): - encoder = json.JSONEncoder(encoding='utf-8') + encoder = self.json.JSONEncoder(encoding='utf-8') u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' s = u.encode('utf-8') ju = encoder.encode(u) @@ -15,68 +14,72 @@ def test_encoding2(self): u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' s = u.encode('utf-8') - ju = json.dumps(u, encoding='utf-8') - js = json.dumps(s, encoding='utf-8') + ju = self.dumps(u, encoding='utf-8') + js = self.dumps(s, encoding='utf-8') self.assertEqual(ju, js) def test_encoding3(self): u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' - j = json.dumps(u) + j = self.dumps(u) self.assertEqual(j, '"\\u03b1\\u03a9"') def test_encoding4(self): u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' - j = json.dumps([u]) + j = self.dumps([u]) self.assertEqual(j, '["\\u03b1\\u03a9"]') def test_encoding5(self): u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' - j = json.dumps(u, ensure_ascii=False) + j = self.dumps(u, ensure_ascii=False) self.assertEqual(j, u'"{0}"'.format(u)) def test_encoding6(self): u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' - j = json.dumps([u], ensure_ascii=False) + j = self.dumps([u], ensure_ascii=False) self.assertEqual(j, u'["{0}"]'.format(u)) def test_big_unicode_encode(self): u = u'\U0001d120' - self.assertEqual(json.dumps(u), '"\\ud834\\udd20"') - self.assertEqual(json.dumps(u, ensure_ascii=False), u'"\U0001d120"') + self.assertEqual(self.dumps(u), '"\\ud834\\udd20"') + self.assertEqual(self.dumps(u, ensure_ascii=False), u'"\U0001d120"') def test_big_unicode_decode(self): u = u'z\U0001d120x' - self.assertEqual(json.loads('"' + u + '"'), u) - self.assertEqual(json.loads('"z\\ud834\\udd20x"'), u) + self.assertEqual(self.loads('"' + u + '"'), u) + self.assertEqual(self.loads('"z\\ud834\\udd20x"'), u) def test_unicode_decode(self): for i in range(0, 0xd7ff): u = unichr(i) s = '"\\u{0:04x}"'.format(i) - self.assertEqual(json.loads(s), u) + self.assertEqual(self.loads(s), u) def test_object_pairs_hook_with_unicode(self): s = u'{"xkd":1, "kcw":2, "art":3, "hxm":4, "qrt":5, "pad":6, "hoy":7}' p = [(u"xkd", 1), (u"kcw", 2), (u"art", 3), (u"hxm", 4), (u"qrt", 5), (u"pad", 6), (u"hoy", 7)] - self.assertEqual(json.loads(s), eval(s)) - self.assertEqual(json.loads(s, object_pairs_hook = lambda x: x), p) - od = json.loads(s, object_pairs_hook = OrderedDict) + self.assertEqual(self.loads(s), eval(s)) + self.assertEqual(self.loads(s, object_pairs_hook = lambda x: x), p) + od = self.loads(s, object_pairs_hook = OrderedDict) self.assertEqual(od, OrderedDict(p)) self.assertEqual(type(od), OrderedDict) # the object_pairs_hook takes priority over the object_hook - self.assertEqual(json.loads(s, + self.assertEqual(self.loads(s, object_pairs_hook = OrderedDict, object_hook = lambda x: None), OrderedDict(p)) def test_default_encoding(self): - self.assertEqual(json.loads(u'{"a": "\xe9"}'.encode('utf-8')), + self.assertEqual(self.loads(u'{"a": "\xe9"}'.encode('utf-8')), {'a': u'\xe9'}) def test_unicode_preservation(self): - self.assertEqual(type(json.loads(u'""')), unicode) - self.assertEqual(type(json.loads(u'"a"')), unicode) - self.assertEqual(type(json.loads(u'["a"]')[0]), unicode) + self.assertEqual(type(self.loads(u'""')), unicode) + self.assertEqual(type(self.loads(u'"a"')), unicode) + self.assertEqual(type(self.loads(u'["a"]')[0]), unicode) # Issue 10038. - self.assertEqual(type(json.loads('"foo"')), unicode) + self.assertEqual(type(self.loads('"foo"')), unicode) + + +class TestPyUnicode(TestUnicode, PyTest): pass +class TestCUnicode(TestUnicode, CTest): pass diff --git a/lib-python/2.7/lib-tk/Tix.py b/lib-python/2.7/lib-tk/Tix.py --- a/lib-python/2.7/lib-tk/Tix.py +++ b/lib-python/2.7/lib-tk/Tix.py @@ -163,7 +163,7 @@ extensions) exist, then the image type is chosen according to the depth of the X display: xbm images are chosen on monochrome displays and color images are chosen on color displays. By using - tix_ getimage, you can advoid hard coding the pathnames of the + tix_ getimage, you can avoid hard coding the pathnames of the image files in your application. When successful, this command returns the name of the newly created image, which can be used to configure the -image option of the Tk and Tix widgets. @@ -171,7 +171,7 @@ return self.tk.call('tix', 'getimage', name) def tix_option_get(self, name): - """Gets the options manitained by the Tix + """Gets the options maintained by the Tix scheme mechanism. Available options include: active_bg active_fg bg @@ -576,7 +576,7 @@ class ComboBox(TixWidget): """ComboBox - an Entry field with a dropdown menu. The user can select a - choice by either typing in the entry subwdget or selecting from the + choice by either typing in the entry subwidget or selecting from the listbox subwidget. Subwidget Class @@ -869,7 +869,7 @@ """HList - Hierarchy display widget can be used to display any data that have a hierarchical structure, for example, file system directory trees. The list entries are indented and connected by branch lines - according to their places in the hierachy. + according to their places in the hierarchy. Subwidgets - None""" @@ -1520,7 +1520,7 @@ self.tk.call(self._w, 'selection', 'set', first, last) class Tree(TixWidget): - """Tree - The tixTree widget can be used to display hierachical + """Tree - The tixTree widget can be used to display hierarchical data in a tree form. The user can adjust the view of the tree by opening or closing parts of the tree.""" diff --git a/lib-python/2.7/lib-tk/Tkinter.py b/lib-python/2.7/lib-tk/Tkinter.py --- a/lib-python/2.7/lib-tk/Tkinter.py +++ b/lib-python/2.7/lib-tk/Tkinter.py @@ -1660,7 +1660,7 @@ class Tk(Misc, Wm): """Toplevel widget of Tk which represents mostly the main window - of an appliation. It has an associated Tcl interpreter.""" + of an application. It has an associated Tcl interpreter.""" _w = '.' def __init__(self, screenName=None, baseName=None, className='Tk', useTk=1, sync=0, use=None): diff --git a/lib-python/2.7/lib-tk/test/test_ttk/test_functions.py b/lib-python/2.7/lib-tk/test/test_ttk/test_functions.py --- a/lib-python/2.7/lib-tk/test/test_ttk/test_functions.py +++ b/lib-python/2.7/lib-tk/test/test_ttk/test_functions.py @@ -136,7 +136,7 @@ # minimum acceptable for image type self.assertEqual(ttk._format_elemcreate('image', False, 'test'), ("test ", ())) - # specifiyng a state spec + # specifying a state spec self.assertEqual(ttk._format_elemcreate('image', False, 'test', ('', 'a')), ("test {} a", ())) # state spec with multiple states diff --git a/lib-python/2.7/lib-tk/ttk.py b/lib-python/2.7/lib-tk/ttk.py --- a/lib-python/2.7/lib-tk/ttk.py +++ b/lib-python/2.7/lib-tk/ttk.py @@ -707,7 +707,7 @@ textvariable, values, width """ # The "values" option may need special formatting, so leave to - # _format_optdict the responsability to format it + # _format_optdict the responsibility to format it if "values" in kw: kw["values"] = _format_optdict({'v': kw["values"]})[1] @@ -993,7 +993,7 @@ pane is either an integer index or the name of a managed subwindow. If kw is not given, returns a dict of the pane option values. If option is specified then the value for that option is returned. - Otherwise, sets the options to the correspoding values.""" + Otherwise, sets the options to the corresponding values.""" if option is not None: kw[option] = None return _val_or_dict(kw, self.tk.call, self._w, "pane", pane) diff --git a/lib-python/2.7/lib-tk/turtle.py b/lib-python/2.7/lib-tk/turtle.py --- a/lib-python/2.7/lib-tk/turtle.py +++ b/lib-python/2.7/lib-tk/turtle.py @@ -1385,7 +1385,7 @@ Optional argument: picname -- a string, name of a gif-file or "nopic". - If picname is a filename, set the corresponing image as background. + If picname is a filename, set the corresponding image as background. If picname is "nopic", delete backgroundimage, if present. If picname is None, return the filename of the current backgroundimage. @@ -1409,7 +1409,7 @@ Optional arguments: canvwidth -- positive integer, new width of canvas in pixels canvheight -- positive integer, new height of canvas in pixels - bg -- colorstring or color-tupel, new backgroundcolor + bg -- colorstring or color-tuple, new backgroundcolor If no arguments are given, return current (canvaswidth, canvasheight) Do not alter the drawing window. To observe hidden parts of @@ -3079,9 +3079,9 @@ fill="", width=ps) # Turtle now at position old, self._position = old - ## if undo is done during crating a polygon, the last vertex - ## will be deleted. if the polygon is entirel deleted, - ## creatigPoly will be set to False. + ## if undo is done during creating a polygon, the last vertex + ## will be deleted. if the polygon is entirely deleted, + ## creatingPoly will be set to False. ## Polygons created before the last one will not be affected by undo() if self._creatingPoly: if len(self._poly) > 0: @@ -3221,7 +3221,7 @@ def dot(self, size=None, *color): """Draw a dot with diameter size, using color. - Optional argumentS: + Optional arguments: size -- an integer >= 1 (if given) color -- a colorstring or a numeric color tuple @@ -3691,7 +3691,7 @@ class Turtle(RawTurtle): - """RawTurtle auto-crating (scrolled) canvas. + """RawTurtle auto-creating (scrolled) canvas. When a Turtle object is created or a function derived from some Turtle method is called a TurtleScreen object is automatically created. @@ -3731,7 +3731,7 @@ filename -- a string, used as filename default value is turtle_docstringdict - Has to be called explicitely, (not used by the turtle-graphics classes) + Has to be called explicitly, (not used by the turtle-graphics classes) The docstring dictionary will be written to the Python script <filname>.py It is intended to serve as a template for translation of the docstrings into different languages. diff --git a/lib-python/2.7/lib2to3/__main__.py b/lib-python/2.7/lib2to3/__main__.py new file mode 100644 --- /dev/null +++ b/lib-python/2.7/lib2to3/__main__.py @@ -0,0 +1,4 @@ +import sys +from .main import main + +sys.exit(main("lib2to3.fixes")) diff --git a/lib-python/2.7/lib2to3/fixes/fix_itertools.py b/lib-python/2.7/lib2to3/fixes/fix_itertools.py --- a/lib-python/2.7/lib2to3/fixes/fix_itertools.py +++ b/lib-python/2.7/lib2to3/fixes/fix_itertools.py @@ -13,7 +13,7 @@ class FixItertools(fixer_base.BaseFix): BM_compatible = True - it_funcs = "('imap'|'ifilter'|'izip'|'ifilterfalse')" + it_funcs = "('imap'|'ifilter'|'izip'|'izip_longest'|'ifilterfalse')" PATTERN = """ power< it='itertools' trailer< @@ -28,7 +28,8 @@ def transform(self, node, results): prefix = None func = results['func'][0] - if 'it' in results and func.value != u'ifilterfalse': + if ('it' in results and + func.value not in (u'ifilterfalse', u'izip_longest')): dot, it = (results['dot'], results['it']) # Remove the 'itertools' prefix = it.prefix diff --git a/lib-python/2.7/lib2to3/fixes/fix_itertools_imports.py b/lib-python/2.7/lib2to3/fixes/fix_itertools_imports.py --- a/lib-python/2.7/lib2to3/fixes/fix_itertools_imports.py +++ b/lib-python/2.7/lib2to3/fixes/fix_itertools_imports.py @@ -31,9 +31,10 @@ if member_name in (u'imap', u'izip', u'ifilter'): child.value = None child.remove() - elif member_name == u'ifilterfalse': + elif member_name in (u'ifilterfalse', u'izip_longest'): node.changed() - name_node.value = u'filterfalse' + name_node.value = (u'filterfalse' if member_name[1] == u'f' + else u'zip_longest') # Make sure the import statement is still sane children = imports.children[:] or [imports] diff --git a/lib-python/2.7/lib2to3/fixes/fix_metaclass.py b/lib-python/2.7/lib2to3/fixes/fix_metaclass.py --- a/lib-python/2.7/lib2to3/fixes/fix_metaclass.py +++ b/lib-python/2.7/lib2to3/fixes/fix_metaclass.py @@ -48,7 +48,7 @@ """ for node in cls_node.children: if node.type == syms.suite: - # already in the prefered format, do nothing + # already in the preferred format, do nothing return # !%@#! oneliners have no suite node, we have to fake one up diff --git a/lib-python/2.7/lib2to3/fixes/fix_urllib.py b/lib-python/2.7/lib2to3/fixes/fix_urllib.py --- a/lib-python/2.7/lib2to3/fixes/fix_urllib.py +++ b/lib-python/2.7/lib2to3/fixes/fix_urllib.py @@ -12,7 +12,7 @@ MAPPING = {"urllib": [ ("urllib.request", - ["URLOpener", "FancyURLOpener", "urlretrieve", + ["URLopener", "FancyURLopener", "urlretrieve", "_urlopener", "urlopen", "urlcleanup", "pathname2url", "url2pathname"]), ("urllib.parse", diff --git a/lib-python/2.7/lib2to3/main.py b/lib-python/2.7/lib2to3/main.py --- a/lib-python/2.7/lib2to3/main.py +++ b/lib-python/2.7/lib2to3/main.py @@ -101,7 +101,7 @@ parser.add_option("-j", "--processes", action="store", default=1, type="int", help="Run 2to3 concurrently") parser.add_option("-x", "--nofix", action="append", default=[], - help="Prevent a fixer from being run.") + help="Prevent a transformation from being run") parser.add_option("-l", "--list-fixes", action="store_true", help="List available transformations") parser.add_option("-p", "--print-function", action="store_true", @@ -113,7 +113,7 @@ parser.add_option("-w", "--write", action="store_true", help="Write back modified files") parser.add_option("-n", "--nobackups", action="store_true", default=False, - help="Don't write backups for modified files.") + help="Don't write backups for modified files") # Parse command line arguments refactor_stdin = False diff --git a/lib-python/2.7/lib2to3/patcomp.py b/lib-python/2.7/lib2to3/patcomp.py --- a/lib-python/2.7/lib2to3/patcomp.py +++ b/lib-python/2.7/lib2to3/patcomp.py @@ -12,6 +12,7 @@ # Python imports import os +import StringIO # Fairly local imports from .pgen2 import driver, literals, token, tokenize, parse, grammar @@ -32,7 +33,7 @@ def tokenize_wrapper(input): """Tokenizes a string suppressing significant whitespace.""" skip = set((token.NEWLINE, token.INDENT, token.DEDENT)) - tokens = tokenize.generate_tokens(driver.generate_lines(input).next) + tokens = tokenize.generate_tokens(StringIO.StringIO(input).readline) for quintuple in tokens: type, value, start, end, line_text = quintuple if type not in skip: diff --git a/lib-python/2.7/lib2to3/pgen2/conv.py b/lib-python/2.7/lib2to3/pgen2/conv.py --- a/lib-python/2.7/lib2to3/pgen2/conv.py +++ b/lib-python/2.7/lib2to3/pgen2/conv.py @@ -51,7 +51,7 @@ self.finish_off() def parse_graminit_h(self, filename): - """Parse the .h file writen by pgen. (Internal) + """Parse the .h file written by pgen. (Internal) This file is a sequence of #define statements defining the nonterminals of the grammar as numbers. We build two tables @@ -82,7 +82,7 @@ return True def parse_graminit_c(self, filename): - """Parse the .c file writen by pgen. (Internal) + """Parse the .c file written by pgen. (Internal) The file looks as follows. The first two lines are always this: diff --git a/lib-python/2.7/lib2to3/pgen2/driver.py b/lib-python/2.7/lib2to3/pgen2/driver.py --- a/lib-python/2.7/lib2to3/pgen2/driver.py +++ b/lib-python/2.7/lib2to3/pgen2/driver.py @@ -19,6 +19,7 @@ import codecs import os import logging +import StringIO import sys # Pgen imports @@ -101,18 +102,10 @@ def parse_string(self, text, debug=False): """Parse a string and return the syntax tree.""" - tokens = tokenize.generate_tokens(generate_lines(text).next) + tokens = tokenize.generate_tokens(StringIO.StringIO(text).readline) return self.parse_tokens(tokens, debug) -def generate_lines(text): - """Generator that behaves like readline without using StringIO.""" - for line in text.splitlines(True): - yield line - while True: - yield "" - - def load_grammar(gt="Grammar.txt", gp=None, save=True, force=False, logger=None): """Load the grammar (maybe from a pickle).""" diff --git a/lib-python/2.7/lib2to3/pytree.py b/lib-python/2.7/lib2to3/pytree.py --- a/lib-python/2.7/lib2to3/pytree.py +++ b/lib-python/2.7/lib2to3/pytree.py @@ -658,8 +658,8 @@ content: optional sequence of subsequences of patterns; if absent, matches one node; if present, each subsequence is an alternative [*] - min: optinal minumum number of times to match, default 0 - max: optional maximum number of times tro match, default HUGE + min: optional minimum number of times to match, default 0 + max: optional maximum number of times to match, default HUGE name: optional name assigned to this match [*] Thus, if content is [[a, b, c], [d, e], [f, g, h]] this is @@ -743,9 +743,11 @@ else: # The reason for this is that hitting the recursion limit usually # results in some ugly messages about how RuntimeErrors are being - # ignored. - save_stderr = sys.stderr - sys.stderr = StringIO() + # ignored. We don't do this on non-CPython implementation because + # they don't have this problem. + if hasattr(sys, "getrefcount"): + save_stderr = sys.stderr + sys.stderr = StringIO() try: for count, r in self._recursive_matches(nodes, 0): if self.name: @@ -759,7 +761,8 @@ r[self.name] = nodes[:count] yield count, r finally: - sys.stderr = save_stderr + if hasattr(sys, "getrefcount"): + sys.stderr = save_stderr def _iterative_matches(self, nodes): """Helper to iteratively yield the matches.""" diff --git a/lib-python/2.7/lib2to3/refactor.py b/lib-python/2.7/lib2to3/refactor.py --- a/lib-python/2.7/lib2to3/refactor.py +++ b/lib-python/2.7/lib2to3/refactor.py @@ -302,13 +302,14 @@ Files and subdirectories starting with '.' are skipped. """ + py_ext = os.extsep + "py" for dirpath, dirnames, filenames in os.walk(dir_name): self.log_debug("Descending into %s", dirpath) dirnames.sort() filenames.sort() for name in filenames: - if not name.startswith(".") and \ - os.path.splitext(name)[1].endswith("py"): + if (not name.startswith(".") and + os.path.splitext(name)[1] == py_ext): fullname = os.path.join(dirpath, name) self.refactor_file(fullname, write, doctests_only) # Modify dirnames in-place to remove subdirs with leading dots diff --git a/lib-python/2.7/lib2to3/tests/data/py2_test_grammar.py b/lib-python/2.7/lib2to3/tests/data/py2_test_grammar.py --- a/lib-python/2.7/lib2to3/tests/data/py2_test_grammar.py +++ b/lib-python/2.7/lib2to3/tests/data/py2_test_grammar.py @@ -316,7 +316,7 @@ ### simple_stmt: small_stmt (';' small_stmt)* [';'] x = 1; pass; del x def foo(): - # verify statments that end with semi-colons + # verify statements that end with semi-colons x = 1; pass; del x; foo() diff --git a/lib-python/2.7/lib2to3/tests/data/py3_test_grammar.py b/lib-python/2.7/lib2to3/tests/data/py3_test_grammar.py --- a/lib-python/2.7/lib2to3/tests/data/py3_test_grammar.py +++ b/lib-python/2.7/lib2to3/tests/data/py3_test_grammar.py @@ -356,7 +356,7 @@ ### simple_stmt: small_stmt (';' small_stmt)* [';'] x = 1; pass; del x def foo(): - # verify statments that end with semi-colons + # verify statements that end with semi-colons x = 1; pass; del x; foo() diff --git a/lib-python/2.7/lib2to3/tests/test_fixers.py b/lib-python/2.7/lib2to3/tests/test_fixers.py --- a/lib-python/2.7/lib2to3/tests/test_fixers.py +++ b/lib-python/2.7/lib2to3/tests/test_fixers.py @@ -3623,16 +3623,24 @@ a = """%s(f, a)""" self.checkall(b, a) - def test_2(self): + def test_qualified(self): b = """itertools.ifilterfalse(a, b)""" a = """itertools.filterfalse(a, b)""" self.check(b, a) - def test_4(self): + b = """itertools.izip_longest(a, b)""" + a = """itertools.zip_longest(a, b)""" + self.check(b, a) + + def test_2(self): b = """ifilterfalse(a, b)""" a = """filterfalse(a, b)""" self.check(b, a) + b = """izip_longest(a, b)""" + a = """zip_longest(a, b)""" + self.check(b, a) + def test_space_1(self): b = """ %s(f, a)""" a = """ %s(f, a)""" @@ -3643,9 +3651,14 @@ a = """ itertools.filterfalse(a, b)""" self.check(b, a) + b = """ itertools.izip_longest(a, b)""" + a = """ itertools.zip_longest(a, b)""" + self.check(b, a) + def test_run_order(self): self.assert_runs_after('map', 'zip', 'filter') + class Test_itertools_imports(FixerTestCase): fixer = 'itertools_imports' @@ -3696,18 +3709,19 @@ s = "from itertools import bar as bang" self.unchanged(s) - def test_ifilter(self): - b = "from itertools import ifilterfalse" - a = "from itertools import filterfalse" - self.check(b, a) - - b = "from itertools import imap, ifilterfalse, foo" - a = "from itertools import filterfalse, foo" - self.check(b, a) - - b = "from itertools import bar, ifilterfalse, foo" - a = "from itertools import bar, filterfalse, foo" - self.check(b, a) + def test_ifilter_and_zip_longest(self): + for name in "filterfalse", "zip_longest": + b = "from itertools import i%s" % (name,) + a = "from itertools import %s" % (name,) + self.check(b, a) + + b = "from itertools import imap, i%s, foo" % (name,) + a = "from itertools import %s, foo" % (name,) + self.check(b, a) + + b = "from itertools import bar, i%s, foo" % (name,) + a = "from itertools import bar, %s, foo" % (name,) + self.check(b, a) def test_import_star(self): s = "from itertools import *" diff --git a/lib-python/2.7/lib2to3/tests/test_parser.py b/lib-python/2.7/lib2to3/tests/test_parser.py --- a/lib-python/2.7/lib2to3/tests/test_parser.py +++ b/lib-python/2.7/lib2to3/tests/test_parser.py @@ -19,6 +19,16 @@ # Local imports from lib2to3.pgen2 import tokenize from ..pgen2.parse import ParseError +from lib2to3.pygram import python_symbols as syms + + +class TestDriver(support.TestCase): + + def test_formfeed(self): + s = """print 1\n\x0Cprint 2\n""" + t = driver.parse_string(s) + self.assertEqual(t.children[0].children[0].type, syms.print_stmt) + self.assertEqual(t.children[1].children[0].type, syms.print_stmt) class GrammarTest(support.TestCase): diff --git a/lib-python/2.7/lib2to3/tests/test_refactor.py b/lib-python/2.7/lib2to3/tests/test_refactor.py --- a/lib-python/2.7/lib2to3/tests/test_refactor.py +++ b/lib-python/2.7/lib2to3/tests/test_refactor.py @@ -223,6 +223,7 @@ "hi.py", ".dumb", ".after.py", + "notpy.npy", "sappy"] expected = ["hi.py"] check(tree, expected) diff --git a/lib-python/2.7/lib2to3/tests/test_util.py b/lib-python/2.7/lib2to3/tests/test_util.py --- a/lib-python/2.7/lib2to3/tests/test_util.py +++ b/lib-python/2.7/lib2to3/tests/test_util.py @@ -568,8 +568,8 @@ def test_from_import(self): node = parse('bar()') - fixer_util.touch_import("cgi", "escape", node) - self.assertEqual(str(node), 'from cgi import escape\nbar()\n\n') + fixer_util.touch_import("html", "escape", node) + self.assertEqual(str(node), 'from html import escape\nbar()\n\n') def test_name_import(self): node = parse('bar()') diff --git a/lib-python/2.7/locale.py b/lib-python/2.7/locale.py --- a/lib-python/2.7/locale.py +++ b/lib-python/2.7/locale.py @@ -621,7 +621,7 @@ 'tactis': 'TACTIS', 'euc_jp': 'eucJP', 'euc_kr': 'eucKR', - 'utf_8': 'UTF8', + 'utf_8': 'UTF-8', 'koi8_r': 'KOI8-R', 'koi8_u': 'KOI8-U', # XXX This list is still incomplete. If you know more diff --git a/lib-python/2.7/logging/__init__.py b/lib-python/2.7/logging/__init__.py --- a/lib-python/2.7/logging/__init__.py +++ b/lib-python/2.7/logging/__init__.py @@ -1627,6 +1627,7 @@ h = wr() if h: try: + h.acquire() h.flush() h.close() except (IOError, ValueError): @@ -1635,6 +1636,8 @@ # references to them are still around at # application exit. pass + finally: + h.release() except: if raiseExceptions: raise diff --git a/lib-python/2.7/logging/config.py b/lib-python/2.7/logging/config.py --- a/lib-python/2.7/logging/config.py +++ b/lib-python/2.7/logging/config.py @@ -226,14 +226,14 @@ propagate = 1 logger = logging.getLogger(qn) if qn in existing: - i = existing.index(qn) + i = existing.index(qn) + 1 # start with the entry after qn prefixed = qn + "." pflen = len(prefixed) num_existing = len(existing) - i = i + 1 # look at the entry after qn - while (i < num_existing) and (existing[i][:pflen] == prefixed): - child_loggers.append(existing[i]) - i = i + 1 + while i < num_existing: + if existing[i][:pflen] == prefixed: + child_loggers.append(existing[i]) + i += 1 existing.remove(qn) if "level" in opts: level = cp.get(sectname, "level") diff --git a/lib-python/2.7/logging/handlers.py b/lib-python/2.7/logging/handlers.py --- a/lib-python/2.7/logging/handlers.py +++ b/lib-python/2.7/logging/handlers.py @@ -125,6 +125,7 @@ """ if self.stream: self.stream.close() + self.stream = None if self.backupCount > 0: for i in range(self.backupCount - 1, 0, -1): sfn = "%s.%d" % (self.baseFilename, i) @@ -324,6 +325,7 @@ """ if self.stream: self.stream.close() + self.stream = None # get the time that this sequence started at and make it a TimeTuple t = self.rolloverAt - self.interval if self.utc: diff --git a/lib-python/2.7/mailbox.py b/lib-python/2.7/mailbox.py --- a/lib-python/2.7/mailbox.py +++ b/lib-python/2.7/mailbox.py @@ -234,27 +234,35 @@ def __init__(self, dirname, factory=rfc822.Message, create=True): """Initialize a Maildir instance.""" Mailbox.__init__(self, dirname, factory, create) + self._paths = { + 'tmp': os.path.join(self._path, 'tmp'), + 'new': os.path.join(self._path, 'new'), + 'cur': os.path.join(self._path, 'cur'), + } if not os.path.exists(self._path): if create: os.mkdir(self._path, 0700) - os.mkdir(os.path.join(self._path, 'tmp'), 0700) - os.mkdir(os.path.join(self._path, 'new'), 0700) - os.mkdir(os.path.join(self._path, 'cur'), 0700) + for path in self._paths.values(): + os.mkdir(path, 0o700) else: raise NoSuchMailboxError(self._path) self._toc = {} - self._last_read = None # Records last time we read cur/new - # NOTE: we manually invalidate _last_read each time we do any - # modifications ourselves, otherwise we might get tripped up by - # bogus mtime behaviour on some systems (see issue #6896). + self._toc_mtimes = {} + for subdir in ('cur', 'new'): + self._toc_mtimes[subdir] = os.path.getmtime(self._paths[subdir]) + self._last_read = time.time() # Records last time we read cur/new + self._skewfactor = 0.1 # Adjust if os/fs clocks are skewing def add(self, message): """Add message and return assigned key.""" tmp_file = self._create_tmp() try: self._dump_message(message, tmp_file) - finally: - _sync_close(tmp_file) + except BaseException: + tmp_file.close() + os.remove(tmp_file.name) + raise + _sync_close(tmp_file) if isinstance(message, MaildirMessage): subdir = message.get_subdir() suffix = self.colon + message.get_info() @@ -280,15 +288,11 @@ raise if isinstance(message, MaildirMessage): os.utime(dest, (os.path.getatime(dest), message.get_date())) - # Invalidate cached toc - self._last_read = None return uniq def remove(self, key): """Remove the keyed message; raise KeyError if it doesn't exist.""" os.remove(os.path.join(self._path, self._lookup(key))) - # Invalidate cached toc (only on success) - self._last_read = None def discard(self, key): """If the keyed message exists, remove it.""" @@ -323,8 +327,6 @@ if isinstance(message, MaildirMessage): os.utime(new_path, (os.path.getatime(new_path), message.get_date())) - # Invalidate cached toc - self._last_read = None def get_message(self, key): """Return a Message representation or raise a KeyError.""" @@ -380,8 +382,8 @@ def flush(self): """Write any pending changes to disk.""" # Maildir changes are always written immediately, so there's nothing - # to do except invalidate our cached toc. - self._last_read = None + # to do. + pass def lock(self): """Lock the mailbox.""" @@ -479,36 +481,39 @@ def _refresh(self): """Update table of contents mapping.""" - if self._last_read is not None: - for subdir in ('new', 'cur'): - mtime = os.path.getmtime(os.path.join(self._path, subdir)) - if mtime > self._last_read: - break - else: + # If it has been less than two seconds since the last _refresh() call, + # we have to unconditionally re-read the mailbox just in case it has + # been modified, because os.path.mtime() has a 2 sec resolution in the + # most common worst case (FAT) and a 1 sec resolution typically. This + # results in a few unnecessary re-reads when _refresh() is called + # multiple times in that interval, but once the clock ticks over, we + # will only re-read as needed. Because the filesystem might be being + # served by an independent system with its own clock, we record and + # compare with the mtimes from the filesystem. Because the other + # system's clock might be skewing relative to our clock, we add an + # extra delta to our wait. The default is one tenth second, but is an + # instance variable and so can be adjusted if dealing with a + # particularly skewed or irregular system. + if time.time() - self._last_read > 2 + self._skewfactor: + refresh = False + for subdir in self._toc_mtimes: + mtime = os.path.getmtime(self._paths[subdir]) + if mtime > self._toc_mtimes[subdir]: + refresh = True + self._toc_mtimes[subdir] = mtime + if not refresh: return - - # We record the current time - 1sec so that, if _refresh() is called - # again in the same second, we will always re-read the mailbox - # just in case it's been modified. (os.path.mtime() only has - # 1sec resolution.) This results in a few unnecessary re-reads - # when _refresh() is called multiple times in the same second, - # but once the clock ticks over, we will only re-read as needed. - now = time.time() - 1 - + # Refresh toc self._toc = {} - def update_dir (subdir): - path = os.path.join(self._path, subdir) + for subdir in self._toc_mtimes: + path = self._paths[subdir] for entry in os.listdir(path): p = os.path.join(path, entry) if os.path.isdir(p): continue uniq = entry.split(self.colon)[0] self._toc[uniq] = os.path.join(subdir, entry) - - update_dir('new') - update_dir('cur') - - self._last_read = now + self._last_read = time.time() def _lookup(self, key): """Use TOC to return subpath for given key, or raise a KeyError.""" @@ -551,7 +556,7 @@ f = open(self._path, 'wb+') else: raise NoSuchMailboxError(self._path) - elif e.errno == errno.EACCES: + elif e.errno in (errno.EACCES, errno.EROFS): f = open(self._path, 'rb') else: raise @@ -700,9 +705,14 @@ def _append_message(self, message): """Append message to mailbox and return (start, stop) offsets.""" self._file.seek(0, 2) - self._pre_message_hook(self._file) - offsets = self._install_message(message) - self._post_message_hook(self._file) + before = self._file.tell() + try: + self._pre_message_hook(self._file) + offsets = self._install_message(message) + self._post_message_hook(self._file) + except BaseException: + self._file.truncate(before) + raise self._file.flush() self._file_length = self._file.tell() # Record current length of mailbox return offsets @@ -868,18 +878,29 @@ new_key = max(keys) + 1 new_path = os.path.join(self._path, str(new_key)) f = _create_carefully(new_path) + closed = False try: if self._locked: _lock_file(f) try: - self._dump_message(message, f) + try: + self._dump_message(message, f) + except BaseException: + # Unlock and close so it can be deleted on Windows + if self._locked: + _unlock_file(f) + _sync_close(f) + closed = True + os.remove(new_path) + raise if isinstance(message, MHMessage): self._dump_sequences(message, new_key) finally: if self._locked: _unlock_file(f) finally: - _sync_close(f) + if not closed: + _sync_close(f) return new_key def remove(self, key): @@ -1886,7 +1907,7 @@ try: fcntl.lockf(f, fcntl.LOCK_EX | fcntl.LOCK_NB) except IOError, e: - if e.errno in (errno.EAGAIN, errno.EACCES): + if e.errno in (errno.EAGAIN, errno.EACCES, errno.EROFS): raise ExternalClashError('lockf: lock unavailable: %s' % f.name) else: @@ -1896,7 +1917,7 @@ pre_lock = _create_temporary(f.name + '.lock') pre_lock.close() except IOError, e: - if e.errno == errno.EACCES: + if e.errno in (errno.EACCES, errno.EROFS): return # Without write access, just skip dotlocking. else: raise diff --git a/lib-python/2.7/msilib/__init__.py b/lib-python/2.7/msilib/__init__.py --- a/lib-python/2.7/msilib/__init__.py +++ b/lib-python/2.7/msilib/__init__.py @@ -173,11 +173,10 @@ add_data(db, table, getattr(module, table)) def make_id(str): - #str = str.replace(".", "_") # colons are allowed - str = str.replace(" ", "_") - str = str.replace("-", "_") - if str[0] in string.digits: - str = "_"+str + identifier_chars = string.ascii_letters + string.digits + "._" + str = "".join([c if c in identifier_chars else "_" for c in str]) + if str[0] in (string.digits + "."): + str = "_" + str assert re.match("^[A-Za-z_][A-Za-z0-9_.]*$", str), "FILE"+str return str @@ -285,19 +284,28 @@ [(feature.id, component)]) def make_short(self, file): + oldfile = file + file = file.replace('+', '_') + file = ''.join(c for c in file if not c in ' "/\[]:;=,') parts = file.split(".") - if len(parts)>1: + if len(parts) > 1: + prefix = "".join(parts[:-1]).upper() suffix = parts[-1].upper() + if not prefix: + prefix = suffix + suffix = None else: + prefix = file.upper() suffix = None - prefix = parts[0].upper() - if len(prefix) <= 8 and (not suffix or len(suffix)<=3): + if len(parts) < 3 and len(prefix) <= 8 and file == oldfile and ( + not suffix or len(suffix) <= 3): if suffix: file = prefix+"."+suffix else: file = prefix - assert file not in self.short_names else: + file = None + if file is None or file in self.short_names: prefix = prefix[:6] if suffix: suffix = suffix[:3] diff --git a/lib-python/2.7/multiprocessing/__init__.py b/lib-python/2.7/multiprocessing/__init__.py --- a/lib-python/2.7/multiprocessing/__init__.py +++ b/lib-python/2.7/multiprocessing/__init__.py @@ -38,6 +38,7 @@ # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. # __version__ = '0.70a1' @@ -115,8 +116,11 @@ except (ValueError, KeyError): num = 0 elif 'bsd' in sys.platform or sys.platform == 'darwin': + comm = '/sbin/sysctl -n hw.ncpu' + if sys.platform == 'darwin': + comm = '/usr' + comm try: - with os.popen('sysctl -n hw.ncpu') as p: + with os.popen(comm) as p: num = int(p.read()) except ValueError: num = 0 diff --git a/lib-python/2.7/multiprocessing/connection.py b/lib-python/2.7/multiprocessing/connection.py --- a/lib-python/2.7/multiprocessing/connection.py +++ b/lib-python/2.7/multiprocessing/connection.py @@ -3,7 +3,33 @@ # # multiprocessing/connection.py # -# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt +# Copyright (c) 2006-2008, R Oudkerk +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of author nor the names of any contributors may be +# used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. # __all__ = [ 'Client', 'Listener', 'Pipe' ] diff --git a/lib-python/2.7/multiprocessing/dummy/__init__.py b/lib-python/2.7/multiprocessing/dummy/__init__.py --- a/lib-python/2.7/multiprocessing/dummy/__init__.py +++ b/lib-python/2.7/multiprocessing/dummy/__init__.py @@ -3,7 +3,33 @@ # # multiprocessing/dummy/__init__.py # -# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt +# Copyright (c) 2006-2008, R Oudkerk +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of author nor the names of any contributors may be +# used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. # __all__ = [ diff --git a/lib-python/2.7/multiprocessing/dummy/connection.py b/lib-python/2.7/multiprocessing/dummy/connection.py --- a/lib-python/2.7/multiprocessing/dummy/connection.py +++ b/lib-python/2.7/multiprocessing/dummy/connection.py @@ -3,7 +3,33 @@ # # multiprocessing/dummy/connection.py # -# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt +# Copyright (c) 2006-2008, R Oudkerk +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of author nor the names of any contributors may be +# used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. # __all__ = [ 'Client', 'Listener', 'Pipe' ] diff --git a/lib-python/2.7/multiprocessing/forking.py b/lib-python/2.7/multiprocessing/forking.py --- a/lib-python/2.7/multiprocessing/forking.py +++ b/lib-python/2.7/multiprocessing/forking.py @@ -3,7 +3,33 @@ # # multiprocessing/forking.py # -# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt +# Copyright (c) 2006-2008, R Oudkerk +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of author nor the names of any contributors may be +# used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. # import os @@ -172,6 +198,7 @@ TERMINATE = 0x10000 WINEXE = (sys.platform == 'win32' and getattr(sys, 'frozen', False)) + WINSERVICE = sys.executable.lower().endswith("pythonservice.exe") exit = win32.ExitProcess close = win32.CloseHandle @@ -181,7 +208,7 @@ # People embedding Python want to modify it. # - if sys.executable.lower().endswith('pythonservice.exe'): + if WINSERVICE: _python_exe = os.path.join(sys.exec_prefix, 'python.exe') else: _python_exe = sys.executable @@ -371,7 +398,7 @@ if _logger is not None: d['log_level'] = _logger.getEffectiveLevel() - if not WINEXE: + if not WINEXE and not WINSERVICE: main_path = getattr(sys.modules['__main__'], '__file__', None) if not main_path and sys.argv[0] not in ('', '-c'): main_path = sys.argv[0] diff --git a/lib-python/2.7/multiprocessing/heap.py b/lib-python/2.7/multiprocessing/heap.py --- a/lib-python/2.7/multiprocessing/heap.py +++ b/lib-python/2.7/multiprocessing/heap.py @@ -3,7 +3,33 @@ # # multiprocessing/heap.py # -# Copyright (c) 2007-2008, R Oudkerk --- see COPYING.txt +# Copyright (c) 2006-2008, R Oudkerk +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of author nor the names of any contributors may be +# used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. # import bisect diff --git a/lib-python/2.7/multiprocessing/managers.py b/lib-python/2.7/multiprocessing/managers.py --- a/lib-python/2.7/multiprocessing/managers.py +++ b/lib-python/2.7/multiprocessing/managers.py @@ -4,7 +4,33 @@ # # multiprocessing/managers.py # -# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt +# Copyright (c) 2006-2008, R Oudkerk +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of author nor the names of any contributors may be +# used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. # __all__ = [ 'BaseManager', 'SyncManager', 'BaseProxy', 'Token' ] diff --git a/lib-python/2.7/multiprocessing/pool.py b/lib-python/2.7/multiprocessing/pool.py --- a/lib-python/2.7/multiprocessing/pool.py +++ b/lib-python/2.7/multiprocessing/pool.py @@ -3,7 +3,33 @@ # # multiprocessing/pool.py # -# Copyright (c) 2007-2008, R Oudkerk --- see COPYING.txt +# Copyright (c) 2006-2008, R Oudkerk +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of author nor the names of any contributors may be +# used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. # __all__ = ['Pool'] @@ -269,6 +295,8 @@ while pool._worker_handler._state == RUN and pool._state == RUN: pool._maintain_pool() time.sleep(0.1) + # send sentinel to stop workers + pool._taskqueue.put(None) debug('worker handler exiting') @staticmethod @@ -387,7 +415,6 @@ if self._state == RUN: self._state = CLOSE self._worker_handler._state = CLOSE - self._taskqueue.put(None) def terminate(self): debug('terminating pool') @@ -421,7 +448,6 @@ worker_handler._state = TERMINATE task_handler._state = TERMINATE - taskqueue.put(None) # sentinel debug('helping task handler/workers to finish') cls._help_stuff_finish(inqueue, task_handler, len(pool)) @@ -431,6 +457,11 @@ result_handler._state = TERMINATE outqueue.put(None) # sentinel + # We must wait for the worker handler to exit before terminating + # workers because we don't want workers to be restarted behind our back. + debug('joining worker handler') + worker_handler.join() + # Terminate workers which haven't already finished. if pool and hasattr(pool[0], 'terminate'): debug('terminating workers') diff --git a/lib-python/2.7/multiprocessing/process.py b/lib-python/2.7/multiprocessing/process.py --- a/lib-python/2.7/multiprocessing/process.py +++ b/lib-python/2.7/multiprocessing/process.py @@ -3,7 +3,33 @@ # # multiprocessing/process.py # -# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt +# Copyright (c) 2006-2008, R Oudkerk +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of author nor the names of any contributors may be +# used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. # __all__ = ['Process', 'current_process', 'active_children'] diff --git a/lib-python/2.7/multiprocessing/queues.py b/lib-python/2.7/multiprocessing/queues.py --- a/lib-python/2.7/multiprocessing/queues.py +++ b/lib-python/2.7/multiprocessing/queues.py @@ -3,7 +3,33 @@ # # multiprocessing/queues.py # -# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt +# Copyright (c) 2006-2008, R Oudkerk +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of author nor the names of any contributors may be +# used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. # __all__ = ['Queue', 'SimpleQueue', 'JoinableQueue'] diff --git a/lib-python/2.7/multiprocessing/reduction.py b/lib-python/2.7/multiprocessing/reduction.py --- a/lib-python/2.7/multiprocessing/reduction.py +++ b/lib-python/2.7/multiprocessing/reduction.py @@ -4,7 +4,33 @@ # # multiprocessing/reduction.py # -# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt +# Copyright (c) 2006-2008, R Oudkerk +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of author nor the names of any contributors may be +# used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. # __all__ = [] diff --git a/lib-python/2.7/multiprocessing/sharedctypes.py b/lib-python/2.7/multiprocessing/sharedctypes.py --- a/lib-python/2.7/multiprocessing/sharedctypes.py +++ b/lib-python/2.7/multiprocessing/sharedctypes.py @@ -3,7 +3,33 @@ # # multiprocessing/sharedctypes.py # -# Copyright (c) 2007-2008, R Oudkerk --- see COPYING.txt +# Copyright (c) 2006-2008, R Oudkerk +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of author nor the names of any contributors may be +# used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. # import sys @@ -52,9 +78,11 @@ Returns a ctypes array allocated from shared memory ''' type_ = typecode_to_type.get(typecode_or_type, typecode_or_type) - if isinstance(size_or_initializer, int): + if isinstance(size_or_initializer, (int, long)): type_ = type_ * size_or_initializer - return _new_value(type_) + obj = _new_value(type_) + ctypes.memset(ctypes.addressof(obj), 0, ctypes.sizeof(obj)) + return obj else: type_ = type_ * len(size_or_initializer) result = _new_value(type_) diff --git a/lib-python/2.7/multiprocessing/synchronize.py b/lib-python/2.7/multiprocessing/synchronize.py --- a/lib-python/2.7/multiprocessing/synchronize.py +++ b/lib-python/2.7/multiprocessing/synchronize.py @@ -3,7 +3,33 @@ # # multiprocessing/synchronize.py # -# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt +# Copyright (c) 2006-2008, R Oudkerk +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of author nor the names of any contributors may be +# used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. # __all__ = [ diff --git a/lib-python/2.7/multiprocessing/util.py b/lib-python/2.7/multiprocessing/util.py --- a/lib-python/2.7/multiprocessing/util.py +++ b/lib-python/2.7/multiprocessing/util.py @@ -3,7 +3,33 @@ # # multiprocessing/util.py # -# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt +# Copyright (c) 2006-2008, R Oudkerk +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of author nor the names of any contributors may be +# used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. # import itertools diff --git a/lib-python/2.7/netrc.py b/lib-python/2.7/netrc.py --- a/lib-python/2.7/netrc.py +++ b/lib-python/2.7/netrc.py @@ -34,11 +34,19 @@ def _parse(self, file, fp): lexer = shlex.shlex(fp) lexer.wordchars += r"""!"#$%&'()*+,-./:;<=>?@[\]^_`{|}~""" + lexer.commenters = lexer.commenters.replace('#', '') while 1: # Look for a machine, default, or macdef top-level keyword toplevel = tt = lexer.get_token() if not tt: break + elif tt[0] == '#': + # seek to beginning of comment, in case reading the token put + # us on a new line, and then skip the rest of the line. + pos = len(tt) + 1 + lexer.instream.seek(-pos, 1) + lexer.instream.readline() + continue elif tt == 'machine': entryname = lexer.get_token() elif tt == 'default': @@ -64,8 +72,8 @@ self.hosts[entryname] = {} while 1: tt = lexer.get_token() - if (tt=='' or tt == 'machine' or - tt == 'default' or tt =='macdef'): + if (tt.startswith('#') or + tt in {'', 'machine', 'default', 'macdef'}): if password: self.hosts[entryname] = (login, account, password) lexer.push_token(tt) diff --git a/lib-python/2.7/nntplib.py b/lib-python/2.7/nntplib.py --- a/lib-python/2.7/nntplib.py +++ b/lib-python/2.7/nntplib.py @@ -103,7 +103,7 @@ readermode is sometimes necessary if you are connecting to an NNTP server on the local machine and intend to call - reader-specific comamnds, such as `group'. If you get + reader-specific commands, such as `group'. If you get unexpected NNTPPermanentErrors, you might need to set readermode. """ diff --git a/lib-python/2.7/ntpath.py b/lib-python/2.7/ntpath.py --- a/lib-python/2.7/ntpath.py +++ b/lib-python/2.7/ntpath.py @@ -310,7 +310,7 @@ # - $varname is accepted. # - %varname% is accepted. # - varnames can be made out of letters, digits and the characters '_-' -# (though is not verifed in the ${varname} and %varname% cases) +# (though is not verified in the ${varname} and %varname% cases) # XXX With COMMAND.COM you can use any characters in a variable name, # XXX except '^|<>='. diff --git a/lib-python/2.7/nturl2path.py b/lib-python/2.7/nturl2path.py --- a/lib-python/2.7/nturl2path.py +++ b/lib-python/2.7/nturl2path.py @@ -25,11 +25,14 @@ error = 'Bad URL: ' + url raise IOError, error drive = comp[0][-1].upper() + path = drive + ':' components = comp[1].split('/') - path = drive + ':' - for comp in components: + for comp in components: if comp: path = path + '\\' + urllib.unquote(comp) + # Issue #11474: url like '/C|/' should convert into 'C:\\' + if path.endswith(':') and url.endswith('/'): + path += '\\' return path def pathname2url(p): diff --git a/lib-python/2.7/numbers.py b/lib-python/2.7/numbers.py --- a/lib-python/2.7/numbers.py +++ b/lib-python/2.7/numbers.py @@ -63,7 +63,7 @@ @abstractproperty def imag(self): - """Retrieve the real component of this number. + """Retrieve the imaginary component of this number. This should subclass Real. """ diff --git a/lib-python/2.7/optparse.py b/lib-python/2.7/optparse.py --- a/lib-python/2.7/optparse.py +++ b/lib-python/2.7/optparse.py @@ -1131,6 +1131,11 @@ prog : string the name of the current program (to override os.path.basename(sys.argv[0])). + description : string + A paragraph of text giving a brief overview of your program. + optparse reformats this paragraph to fit the current terminal + width and prints it when the user requests help (after usage, + but before the list of options). epilog : string paragraph of help text to print after option help diff --git a/lib-python/2.7/pickletools.py b/lib-python/2.7/pickletools.py --- a/lib-python/2.7/pickletools.py +++ b/lib-python/2.7/pickletools.py @@ -1370,7 +1370,7 @@ proto=0, doc="""Read an object from the memo and push it on the stack. - The index of the memo object to push is given by the newline-teriminated + The index of the memo object to push is given by the newline-terminated decimal string following. BINGET and LONG_BINGET are space-optimized versions. """), diff --git a/lib-python/2.7/pkgutil.py b/lib-python/2.7/pkgutil.py --- a/lib-python/2.7/pkgutil.py +++ b/lib-python/2.7/pkgutil.py @@ -11,7 +11,7 @@ __all__ = [ 'get_importer', 'iter_importers', 'get_loader', 'find_loader', - 'walk_packages', 'iter_modules', + 'walk_packages', 'iter_modules', 'get_data', 'ImpImporter', 'ImpLoader', 'read_code', 'extend_path', ] diff --git a/lib-python/2.7/platform.py b/lib-python/2.7/platform.py --- a/lib-python/2.7/platform.py +++ b/lib-python/2.7/platform.py @@ -503,7 +503,7 @@ info = pipe.read() if pipe.close(): raise os.error,'command failed' - # XXX How can I supress shell errors from being written + # XXX How can I suppress shell errors from being written # to stderr ? except os.error,why: #print 'Command %s failed: %s' % (cmd,why) @@ -1448,9 +1448,10 @@ """ Returns a string identifying the Python implementation. Currently, the following implementations are identified: - 'CPython' (C implementation of Python), - 'IronPython' (.NET implementation of Python), - 'Jython' (Java implementation of Python). + 'CPython' (C implementation of Python), + 'IronPython' (.NET implementation of Python), + 'Jython' (Java implementation of Python), + 'PyPy' (Python implementation of Python). """ return _sys_version()[0] diff --git a/lib-python/2.7/pydoc.py b/lib-python/2.7/pydoc.py --- a/lib-python/2.7/pydoc.py +++ b/lib-python/2.7/pydoc.py @@ -156,7 +156,7 @@ no.append(x) return yes, no -def visiblename(name, all=None): +def visiblename(name, all=None, obj=None): """Decide whether to show documentation on a variable.""" # Certain special names are redundant. _hidden_names = ('__builtins__', '__doc__', '__file__', '__path__', @@ -164,6 +164,9 @@ if name in _hidden_names: return 0 # Private names are hidden, but special names are displayed. if name.startswith('__') and name.endswith('__'): return 1 + # Namedtuples have public fields and methods with a single leading underscore + if name.startswith('_') and hasattr(obj, '_fields'): + return 1 if all is not None: # only document that which the programmer exported in __all__ return name in all @@ -475,9 +478,9 @@ def multicolumn(self, list, format, cols=4): """Format a list of items into a multi-column list.""" result = '' - rows = (len(list)+cols-1)/cols + rows = (len(list)+cols-1)//cols for col in range(cols): - result = result + '<td width="%d%%" valign=top>' % (100/cols) + result = result + '<td width="%d%%" valign=top>' % (100//cols) for i in range(rows*col, rows*col+rows): if i < len(list): result = result + format(list[i]) + '<br>\n' @@ -627,7 +630,7 @@ # if __all__ exists, believe it. Otherwise use old heuristic. if (all is not None or (inspect.getmodule(value) or object) is object): - if visiblename(key, all): + if visiblename(key, all, object): classes.append((key, value)) cdict[key] = cdict[value] = '#' + key for key, value in classes: @@ -643,13 +646,13 @@ # if __all__ exists, believe it. Otherwise use old heuristic. if (all is not None or inspect.isbuiltin(value) or inspect.getmodule(value) is object): - if visiblename(key, all): + if visiblename(key, all, object): funcs.append((key, value)) fdict[key] = '#-' + key if inspect.isfunction(value): fdict[value] = fdict[key] data = [] for key, value in inspect.getmembers(object, isdata): - if visiblename(key, all): + if visiblename(key, all, object): data.append((key, value)) doc = self.markup(getdoc(object), self.preformat, fdict, cdict) @@ -773,7 +776,7 @@ push('\n') return attrs - attrs = filter(lambda data: visiblename(data[0]), + attrs = filter(lambda data: visiblename(data[0], obj=object), classify_class_attrs(object)) mdict = {} for key, kind, homecls, value in attrs: @@ -1042,18 +1045,18 @@ # if __all__ exists, believe it. Otherwise use old heuristic. if (all is not None or (inspect.getmodule(value) or object) is object): - if visiblename(key, all): + if visiblename(key, all, object): classes.append((key, value)) funcs = [] for key, value in inspect.getmembers(object, inspect.isroutine): # if __all__ exists, believe it. Otherwise use old heuristic. if (all is not None or inspect.isbuiltin(value) or inspect.getmodule(value) is object): - if visiblename(key, all): + if visiblename(key, all, object): funcs.append((key, value)) data = [] for key, value in inspect.getmembers(object, isdata): - if visiblename(key, all): + if visiblename(key, all, object): data.append((key, value)) modpkgs = [] @@ -1113,7 +1116,7 @@ result = result + self.section('CREDITS', str(object.__credits__)) return result - def docclass(self, object, name=None, mod=None): + def docclass(self, object, name=None, mod=None, *ignored): """Produce text documentation for a given class object.""" realname = object.__name__ name = name or realname @@ -1186,7 +1189,7 @@ name, mod, maxlen=70, doc=doc) + '\n') return attrs - attrs = filter(lambda data: visiblename(data[0]), + attrs = filter(lambda data: visiblename(data[0], obj=object), classify_class_attrs(object)) while attrs: if mro: @@ -1718,8 +1721,9 @@ return '' return '<pydoc.Helper instance>' - def __call__(self, request=None): - if request is not None: + _GoInteractive = object() + def __call__(self, request=_GoInteractive): + if request is not self._GoInteractive: self.help(request) else: self.intro() diff --git a/lib-python/2.7/pydoc_data/topics.py b/lib-python/2.7/pydoc_data/topics.py --- a/lib-python/2.7/pydoc_data/topics.py +++ b/lib-python/2.7/pydoc_data/topics.py @@ -1,16 +1,16 @@ -# Autogenerated by Sphinx on Sat Jul 3 08:52:04 2010 +# Autogenerated by Sphinx on Sat Jun 11 09:49:30 2011 topics = {'assert': u'\nThe ``assert`` statement\n************************\n\nAssert statements are a convenient way to insert debugging assertions\ninto a program:\n\n assert_stmt ::= "assert" expression ["," expression]\n\nThe simple form, ``assert expression``, is equivalent to\n\n if __debug__:\n if not expression: raise AssertionError\n\nThe extended form, ``assert expression1, expression2``, is equivalent\nto\n\n if __debug__:\n if not expression1: raise AssertionError(expression2)\n\nThese equivalences assume that ``__debug__`` and ``AssertionError``\nrefer to the built-in variables with those names. In the current\nimplementation, the built-in variable ``__debug__`` is ``True`` under\nnormal circumstances, ``False`` when optimization is requested\n(command line option -O). The current code generator emits no code\nfor an assert statement when optimization is requested at compile\ntime. Note that it is unnecessary to include the source code for the\nexpression that failed in the error message; it will be displayed as\npart of the stack trace.\n\nAssignments to ``__debug__`` are illegal. The value for the built-in\nvariable is determined when the interpreter starts.\n', - 'assignment': u'\nAssignment statements\n*********************\n\nAssignment statements are used to (re)bind names to values and to\nmodify attributes or items of mutable objects:\n\n assignment_stmt ::= (target_list "=")+ (expression_list | yield_expression)\n target_list ::= target ("," target)* [","]\n target ::= identifier\n | "(" target_list ")"\n | "[" target_list "]"\n | attributeref\n | subscription\n | slicing\n\n(See section *Primaries* for the syntax definitions for the last three\nsymbols.)\n\nAn assignment statement evaluates the expression list (remember that\nthis can be a single expression or a comma-separated list, the latter\nyielding a tuple) and assigns the single resulting object to each of\nthe target lists, from left to right.\n\nAssignment is defined recursively depending on the form of the target\n(list). When a target is part of a mutable object (an attribute\nreference, subscription or slicing), the mutable object must\nultimately perform the assignment and decide about its validity, and\nmay raise an exception if the assignment is unacceptable. The rules\nobserved by various types and the exceptions raised are given with the\ndefinition of the object types (see section *The standard type\nhierarchy*).\n\nAssignment of an object to a target list is recursively defined as\nfollows.\n\n* If the target list is a single target: The object is assigned to\n that target.\n\n* If the target list is a comma-separated list of targets: The object\n must be an iterable with the same number of items as there are\n targets in the target list, and the items are assigned, from left to\n right, to the corresponding targets. (This rule is relaxed as of\n Python 1.5; in earlier versions, the object had to be a tuple.\n Since strings are sequences, an assignment like ``a, b = "xy"`` is\n now legal as long as the string has the right length.)\n\nAssignment of an object to a single target is recursively defined as\nfollows.\n\n* If the target is an identifier (name):\n\n * If the name does not occur in a ``global`` statement in the\n current code block: the name is bound to the object in the current\n local namespace.\n\n * Otherwise: the name is bound to the object in the current global\n namespace.\n\n The name is rebound if it was already bound. This may cause the\n reference count for the object previously bound to the name to reach\n zero, causing the object to be deallocated and its destructor (if it\n has one) to be called.\n\n* If the target is a target list enclosed in parentheses or in square\n brackets: The object must be an iterable with the same number of\n items as there are targets in the target list, and its items are\n assigned, from left to right, to the corresponding targets.\n\n* If the target is an attribute reference: The primary expression in\n the reference is evaluated. It should yield an object with\n assignable attributes; if this is not the case, ``TypeError`` is\n raised. That object is then asked to assign the assigned object to\n the given attribute; if it cannot perform the assignment, it raises\n an exception (usually but not necessarily ``AttributeError``).\n\n Note: If the object is a class instance and the attribute reference\n occurs on both sides of the assignment operator, the RHS expression,\n ``a.x`` can access either an instance attribute or (if no instance\n attribute exists) a class attribute. The LHS target ``a.x`` is\n always set as an instance attribute, creating it if necessary.\n Thus, the two occurrences of ``a.x`` do not necessarily refer to the\n same attribute: if the RHS expression refers to a class attribute,\n the LHS creates a new instance attribute as the target of the\n assignment:\n\n class Cls:\n x = 3 # class variable\n inst = Cls()\n inst.x = inst.x + 1 # writes inst.x as 4 leaving Cls.x as 3\n\n This description does not necessarily apply to descriptor\n attributes, such as properties created with ``property()``.\n\n* If the target is a subscription: The primary expression in the\n reference is evaluated. It should yield either a mutable sequence\n object (such as a list) or a mapping object (such as a dictionary).\n Next, the subscript expression is evaluated.\n\n If the primary is a mutable sequence object (such as a list), the\n subscript must yield a plain integer. If it is negative, the\n sequence\'s length is added to it. The resulting value must be a\n nonnegative integer less than the sequence\'s length, and the\n sequence is asked to assign the assigned object to its item with\n that index. If the index is out of range, ``IndexError`` is raised\n (assignment to a subscripted sequence cannot add new items to a\n list).\n\n If the primary is a mapping object (such as a dictionary), the\n subscript must have a type compatible with the mapping\'s key type,\n and the mapping is then asked to create a key/datum pair which maps\n the subscript to the assigned object. This can either replace an\n existing key/value pair with the same key value, or insert a new\n key/value pair (if no key with the same value existed).\n\n* If the target is a slicing: The primary expression in the reference\n is evaluated. It should yield a mutable sequence object (such as a\n list). The assigned object should be a sequence object of the same\n type. Next, the lower and upper bound expressions are evaluated,\n insofar they are present; defaults are zero and the sequence\'s\n length. The bounds should evaluate to (small) integers. If either\n bound is negative, the sequence\'s length is added to it. The\n resulting bounds are clipped to lie between zero and the sequence\'s\n length, inclusive. Finally, the sequence object is asked to replace\n the slice with the items of the assigned sequence. The length of\n the slice may be different from the length of the assigned sequence,\n thus changing the length of the target sequence, if the object\n allows it.\n\n**CPython implementation detail:** In the current implementation, the\nsyntax for targets is taken to be the same as for expressions, and\ninvalid syntax is rejected during the code generation phase, causing\nless detailed error messages.\n\nWARNING: Although the definition of assignment implies that overlaps\nbetween the left-hand side and the right-hand side are \'safe\' (for\nexample ``a, b = b, a`` swaps two variables), overlaps *within* the\ncollection of assigned-to variables are not safe! For instance, the\nfollowing program prints ``[0, 2]``:\n\n x = [0, 1]\n i = 0\n i, x[i] = 1, 2\n print x\n\n\nAugmented assignment statements\n===============================\n\nAugmented assignment is the combination, in a single statement, of a\nbinary operation and an assignment statement:\n\n augmented_assignment_stmt ::= augtarget augop (expression_list | yield_expression)\n augtarget ::= identifier | attributeref | subscription | slicing\n augop ::= "+=" | "-=" | "*=" | "/=" | "//=" | "%=" | "**="\n | ">>=" | "<<=" | "&=" | "^=" | "|="\n\n(See section *Primaries* for the syntax definitions for the last three\nsymbols.)\n\nAn augmented assignment evaluates the target (which, unlike normal\nassignment statements, cannot be an unpacking) and the expression\nlist, performs the binary operation specific to the type of assignment\non the two operands, and assigns the result to the original target.\nThe target is only evaluated once.\n\nAn augmented assignment expression like ``x += 1`` can be rewritten as\n``x = x + 1`` to achieve a similar, but not exactly equal effect. In\nthe augmented version, ``x`` is only evaluated once. Also, when\npossible, the actual operation is performed *in-place*, meaning that\nrather than creating a new object and assigning that to the target,\nthe old object is modified instead.\n\nWith the exception of assigning to tuples and multiple targets in a\nsingle statement, the assignment done by augmented assignment\nstatements is handled the same way as normal assignments. Similarly,\nwith the exception of the possible *in-place* behavior, the binary\noperation performed by augmented assignment is the same as the normal\nbinary operations.\n\nFor targets which are attribute references, the same *caveat about\nclass and instance attributes* applies as for regular assignments.\n', + 'assignment': u'\nAssignment statements\n*********************\n\nAssignment statements are used to (re)bind names to values and to\nmodify attributes or items of mutable objects:\n\n assignment_stmt ::= (target_list "=")+ (expression_list | yield_expression)\n target_list ::= target ("," target)* [","]\n target ::= identifier\n | "(" target_list ")"\n | "[" target_list "]"\n | attributeref\n | subscription\n | slicing\n\n(See section *Primaries* for the syntax definitions for the last three\nsymbols.)\n\nAn assignment statement evaluates the expression list (remember that\nthis can be a single expression or a comma-separated list, the latter\nyielding a tuple) and assigns the single resulting object to each of\nthe target lists, from left to right.\n\nAssignment is defined recursively depending on the form of the target\n(list). When a target is part of a mutable object (an attribute\nreference, subscription or slicing), the mutable object must\nultimately perform the assignment and decide about its validity, and\nmay raise an exception if the assignment is unacceptable. The rules\nobserved by various types and the exceptions raised are given with the\ndefinition of the object types (see section *The standard type\nhierarchy*).\n\nAssignment of an object to a target list is recursively defined as\nfollows.\n\n* If the target list is a single target: The object is assigned to\n that target.\n\n* If the target list is a comma-separated list of targets: The object\n must be an iterable with the same number of items as there are\n targets in the target list, and the items are assigned, from left to\n right, to the corresponding targets.\n\nAssignment of an object to a single target is recursively defined as\nfollows.\n\n* If the target is an identifier (name):\n\n * If the name does not occur in a ``global`` statement in the\n current code block: the name is bound to the object in the current\n local namespace.\n\n * Otherwise: the name is bound to the object in the current global\n namespace.\n\n The name is rebound if it was already bound. This may cause the\n reference count for the object previously bound to the name to reach\n zero, causing the object to be deallocated and its destructor (if it\n has one) to be called.\n\n* If the target is a target list enclosed in parentheses or in square\n brackets: The object must be an iterable with the same number of\n items as there are targets in the target list, and its items are\n assigned, from left to right, to the corresponding targets.\n\n* If the target is an attribute reference: The primary expression in\n the reference is evaluated. It should yield an object with\n assignable attributes; if this is not the case, ``TypeError`` is\n raised. That object is then asked to assign the assigned object to\n the given attribute; if it cannot perform the assignment, it raises\n an exception (usually but not necessarily ``AttributeError``).\n\n Note: If the object is a class instance and the attribute reference\n occurs on both sides of the assignment operator, the RHS expression,\n ``a.x`` can access either an instance attribute or (if no instance\n attribute exists) a class attribute. The LHS target ``a.x`` is\n always set as an instance attribute, creating it if necessary.\n Thus, the two occurrences of ``a.x`` do not necessarily refer to the\n same attribute: if the RHS expression refers to a class attribute,\n the LHS creates a new instance attribute as the target of the\n assignment:\n\n class Cls:\n x = 3 # class variable\n inst = Cls()\n inst.x = inst.x + 1 # writes inst.x as 4 leaving Cls.x as 3\n\n This description does not necessarily apply to descriptor\n attributes, such as properties created with ``property()``.\n\n* If the target is a subscription: The primary expression in the\n reference is evaluated. It should yield either a mutable sequence\n object (such as a list) or a mapping object (such as a dictionary).\n Next, the subscript expression is evaluated.\n\n If the primary is a mutable sequence object (such as a list), the\n subscript must yield a plain integer. If it is negative, the\n sequence\'s length is added to it. The resulting value must be a\n nonnegative integer less than the sequence\'s length, and the\n sequence is asked to assign the assigned object to its item with\n that index. If the index is out of range, ``IndexError`` is raised\n (assignment to a subscripted sequence cannot add new items to a\n list).\n\n If the primary is a mapping object (such as a dictionary), the\n subscript must have a type compatible with the mapping\'s key type,\n and the mapping is then asked to create a key/datum pair which maps\n the subscript to the assigned object. This can either replace an\n existing key/value pair with the same key value, or insert a new\n key/value pair (if no key with the same value existed).\n\n* If the target is a slicing: The primary expression in the reference\n is evaluated. It should yield a mutable sequence object (such as a\n list). The assigned object should be a sequence object of the same\n type. Next, the lower and upper bound expressions are evaluated,\n insofar they are present; defaults are zero and the sequence\'s\n length. The bounds should evaluate to (small) integers. If either\n bound is negative, the sequence\'s length is added to it. The\n resulting bounds are clipped to lie between zero and the sequence\'s\n length, inclusive. Finally, the sequence object is asked to replace\n the slice with the items of the assigned sequence. The length of\n the slice may be different from the length of the assigned sequence,\n thus changing the length of the target sequence, if the object\n allows it.\n\n**CPython implementation detail:** In the current implementation, the\nsyntax for targets is taken to be the same as for expressions, and\ninvalid syntax is rejected during the code generation phase, causing\nless detailed error messages.\n\nWARNING: Although the definition of assignment implies that overlaps\nbetween the left-hand side and the right-hand side are \'safe\' (for\nexample ``a, b = b, a`` swaps two variables), overlaps *within* the\ncollection of assigned-to variables are not safe! For instance, the\nfollowing program prints ``[0, 2]``:\n\n x = [0, 1]\n i = 0\n i, x[i] = 1, 2\n print x\n\n\nAugmented assignment statements\n===============================\n\nAugmented assignment is the combination, in a single statement, of a\nbinary operation and an assignment statement:\n\n augmented_assignment_stmt ::= augtarget augop (expression_list | yield_expression)\n augtarget ::= identifier | attributeref | subscription | slicing\n augop ::= "+=" | "-=" | "*=" | "/=" | "//=" | "%=" | "**="\n | ">>=" | "<<=" | "&=" | "^=" | "|="\n\n(See section *Primaries* for the syntax definitions for the last three\nsymbols.)\n\nAn augmented assignment evaluates the target (which, unlike normal\nassignment statements, cannot be an unpacking) and the expression\nlist, performs the binary operation specific to the type of assignment\non the two operands, and assigns the result to the original target.\nThe target is only evaluated once.\n\nAn augmented assignment expression like ``x += 1`` can be rewritten as\n``x = x + 1`` to achieve a similar, but not exactly equal effect. In\nthe augmented version, ``x`` is only evaluated once. Also, when\npossible, the actual operation is performed *in-place*, meaning that\nrather than creating a new object and assigning that to the target,\nthe old object is modified instead.\n\nWith the exception of assigning to tuples and multiple targets in a\nsingle statement, the assignment done by augmented assignment\nstatements is handled the same way as normal assignments. Similarly,\nwith the exception of the possible *in-place* behavior, the binary\noperation performed by augmented assignment is the same as the normal\nbinary operations.\n\nFor targets which are attribute references, the same *caveat about\nclass and instance attributes* applies as for regular assignments.\n', 'atom-identifiers': u'\nIdentifiers (Names)\n*******************\n\nAn identifier occurring as an atom is a name. See section\n*Identifiers and keywords* for lexical definition and section *Naming\nand binding* for documentation of naming and binding.\n\nWhen the name is bound to an object, evaluation of the atom yields\nthat object. When a name is not bound, an attempt to evaluate it\nraises a ``NameError`` exception.\n\n**Private name mangling:** When an identifier that textually occurs in\na class definition begins with two or more underscore characters and\ndoes not end in two or more underscores, it is considered a *private\nname* of that class. Private names are transformed to a longer form\nbefore code is generated for them. The transformation inserts the\nclass name in front of the name, with leading underscores removed, and\na single underscore inserted in front of the class name. For example,\nthe identifier ``__spam`` occurring in a class named ``Ham`` will be\ntransformed to ``_Ham__spam``. This transformation is independent of\nthe syntactical context in which the identifier is used. If the\ntransformed name is extremely long (longer than 255 characters),\nimplementation defined truncation may happen. If the class name\nconsists only of underscores, no transformation is done.\n', 'atom-literals': u"\nLiterals\n********\n\nPython supports string literals and various numeric literals:\n\n literal ::= stringliteral | integer | longinteger\n | floatnumber | imagnumber\n\nEvaluation of a literal yields an object of the given type (string,\ninteger, long integer, floating point number, complex number) with the\ngiven value. The value may be approximated in the case of floating\npoint and imaginary (complex) literals. See section *Literals* for\ndetails.\n\nAll literals correspond to immutable data types, and hence the\nobject's identity is less important than its value. Multiple\nevaluations of literals with the same value (either the same\noccurrence in the program text or a different occurrence) may obtain\nthe same object or a different object with the same value.\n", - 'attribute-access': u'\nCustomizing attribute access\n****************************\n\nThe following methods can be defined to customize the meaning of\nattribute access (use of, assignment to, or deletion of ``x.name``)\nfor class instances.\n\nobject.__getattr__(self, name)\n\n Called when an attribute lookup has not found the attribute in the\n usual places (i.e. it is not an instance attribute nor is it found\n in the class tree for ``self``). ``name`` is the attribute name.\n This method should return the (computed) attribute value or raise\n an ``AttributeError`` exception.\n\n Note that if the attribute is found through the normal mechanism,\n ``__getattr__()`` is not called. (This is an intentional asymmetry\n between ``__getattr__()`` and ``__setattr__()``.) This is done both\n for efficiency reasons and because otherwise ``__getattr__()``\n would have no way to access other attributes of the instance. Note\n that at least for instance variables, you can fake total control by\n not inserting any values in the instance attribute dictionary (but\n instead inserting them in another object). See the\n ``__getattribute__()`` method below for a way to actually get total\n control in new-style classes.\n\nobject.__setattr__(self, name, value)\n\n Called when an attribute assignment is attempted. This is called\n instead of the normal mechanism (i.e. store the value in the\n instance dictionary). *name* is the attribute name, *value* is the\n value to be assigned to it.\n\n If ``__setattr__()`` wants to assign to an instance attribute, it\n should not simply execute ``self.name = value`` --- this would\n cause a recursive call to itself. Instead, it should insert the\n value in the dictionary of instance attributes, e.g.,\n ``self.__dict__[name] = value``. For new-style classes, rather\n than accessing the instance dictionary, it should call the base\n class method with the same name, for example,\n ``object.__setattr__(self, name, value)``.\n\nobject.__delattr__(self, name)\n\n Like ``__setattr__()`` but for attribute deletion instead of\n assignment. This should only be implemented if ``del obj.name`` is\n meaningful for the object.\n\n\nMore attribute access for new-style classes\n===========================================\n\nThe following methods only apply to new-style classes.\n\nobject.__getattribute__(self, name)\n\n Called unconditionally to implement attribute accesses for\n instances of the class. If the class also defines\n ``__getattr__()``, the latter will not be called unless\n ``__getattribute__()`` either calls it explicitly or raises an\n ``AttributeError``. This method should return the (computed)\n attribute value or raise an ``AttributeError`` exception. In order\n to avoid infinite recursion in this method, its implementation\n should always call the base class method with the same name to\n access any attributes it needs, for example,\n ``object.__getattribute__(self, name)``.\n\n Note: This method may still be bypassed when looking up special methods\n as the result of implicit invocation via language syntax or\n built-in functions. See *Special method lookup for new-style\n classes*.\n\n\nImplementing Descriptors\n========================\n\nThe following methods only apply when an instance of the class\ncontaining the method (a so-called *descriptor* class) appears in the\nclass dictionary of another new-style class, known as the *owner*\nclass. In the examples below, "the attribute" refers to the attribute\nwhose name is the key of the property in the owner class\'\n``__dict__``. Descriptors can only be implemented as new-style\nclasses themselves.\n\nobject.__get__(self, instance, owner)\n\n Called to get the attribute of the owner class (class attribute\n access) or of an instance of that class (instance attribute\n access). *owner* is always the owner class, while *instance* is the\n instance that the attribute was accessed through, or ``None`` when\n the attribute is accessed through the *owner*. This method should\n return the (computed) attribute value or raise an\n ``AttributeError`` exception.\n\nobject.__set__(self, instance, value)\n\n Called to set the attribute on an instance *instance* of the owner\n class to a new value, *value*.\n\nobject.__delete__(self, instance)\n\n Called to delete the attribute on an instance *instance* of the\n owner class.\n\n\nInvoking Descriptors\n====================\n\nIn general, a descriptor is an object attribute with "binding\nbehavior", one whose attribute access has been overridden by methods\nin the descriptor protocol: ``__get__()``, ``__set__()``, and\n``__delete__()``. If any of those methods are defined for an object,\nit is said to be a descriptor.\n\nThe default behavior for attribute access is to get, set, or delete\nthe attribute from an object\'s dictionary. For instance, ``a.x`` has a\nlookup chain starting with ``a.__dict__[\'x\']``, then\n``type(a).__dict__[\'x\']``, and continuing through the base classes of\n``type(a)`` excluding metaclasses.\n\nHowever, if the looked-up value is an object defining one of the\ndescriptor methods, then Python may override the default behavior and\ninvoke the descriptor method instead. Where this occurs in the\nprecedence chain depends on which descriptor methods were defined and\nhow they were called. Note that descriptors are only invoked for new\nstyle objects or classes (ones that subclass ``object()`` or\n``type()``).\n\nThe starting point for descriptor invocation is a binding, ``a.x``.\nHow the arguments are assembled depends on ``a``:\n\nDirect Call\n The simplest and least common call is when user code directly\n invokes a descriptor method: ``x.__get__(a)``.\n\nInstance Binding\n If binding to a new-style object instance, ``a.x`` is transformed\n into the call: ``type(a).__dict__[\'x\'].__get__(a, type(a))``.\n\nClass Binding\n If binding to a new-style class, ``A.x`` is transformed into the\n call: ``A.__dict__[\'x\'].__get__(None, A)``.\n\nSuper Binding\n If ``a`` is an instance of ``super``, then the binding ``super(B,\n obj).m()`` searches ``obj.__class__.__mro__`` for the base class\n ``A`` immediately preceding ``B`` and then invokes the descriptor\n with the call: ``A.__dict__[\'m\'].__get__(obj, A)``.\n\nFor instance bindings, the precedence of descriptor invocation depends\non the which descriptor methods are defined. A descriptor can define\nany combination of ``__get__()``, ``__set__()`` and ``__delete__()``.\nIf it does not define ``__get__()``, then accessing the attribute will\nreturn the descriptor object itself unless there is a value in the\nobject\'s instance dictionary. If the descriptor defines ``__set__()``\nand/or ``__delete__()``, it is a data descriptor; if it defines\nneither, it is a non-data descriptor. Normally, data descriptors\ndefine both ``__get__()`` and ``__set__()``, while non-data\ndescriptors have just the ``__get__()`` method. Data descriptors with\n``__set__()`` and ``__get__()`` defined always override a redefinition\nin an instance dictionary. In contrast, non-data descriptors can be\noverridden by instances.\n\nPython methods (including ``staticmethod()`` and ``classmethod()``)\nare implemented as non-data descriptors. Accordingly, instances can\nredefine and override methods. This allows individual instances to\nacquire behaviors that differ from other instances of the same class.\n\nThe ``property()`` function is implemented as a data descriptor.\nAccordingly, instances cannot override the behavior of a property.\n\n\n__slots__\n=========\n\nBy default, instances of both old and new-style classes have a\ndictionary for attribute storage. This wastes space for objects\nhaving very few instance variables. The space consumption can become\nacute when creating large numbers of instances.\n\nThe default can be overridden by defining *__slots__* in a new-style\nclass definition. The *__slots__* declaration takes a sequence of\ninstance variables and reserves just enough space in each instance to\nhold a value for each variable. Space is saved because *__dict__* is\nnot created for each instance.\n\n__slots__\n\n This class variable can be assigned a string, iterable, or sequence\n of strings with variable names used by instances. If defined in a\n new-style class, *__slots__* reserves space for the declared\n variables and prevents the automatic creation of *__dict__* and\n *__weakref__* for each instance.\n\n New in version 2.2.\n\nNotes on using *__slots__*\n\n* When inheriting from a class without *__slots__*, the *__dict__*\n attribute of that class will always be accessible, so a *__slots__*\n definition in the subclass is meaningless.\n\n* Without a *__dict__* variable, instances cannot be assigned new\n variables not listed in the *__slots__* definition. Attempts to\n assign to an unlisted variable name raises ``AttributeError``. If\n dynamic assignment of new variables is desired, then add\n ``\'__dict__\'`` to the sequence of strings in the *__slots__*\n declaration.\n\n Changed in version 2.3: Previously, adding ``\'__dict__\'`` to the\n *__slots__* declaration would not enable the assignment of new\n attributes not specifically listed in the sequence of instance\n variable names.\n\n* Without a *__weakref__* variable for each instance, classes defining\n *__slots__* do not support weak references to its instances. If weak\n reference support is needed, then add ``\'__weakref__\'`` to the\n sequence of strings in the *__slots__* declaration.\n\n Changed in version 2.3: Previously, adding ``\'__weakref__\'`` to the\n *__slots__* declaration would not enable support for weak\n references.\n\n* *__slots__* are implemented at the class level by creating\n descriptors (*Implementing Descriptors*) for each variable name. As\n a result, class attributes cannot be used to set default values for\n instance variables defined by *__slots__*; otherwise, the class\n attribute would overwrite the descriptor assignment.\n\n* The action of a *__slots__* declaration is limited to the class\n where it is defined. As a result, subclasses will have a *__dict__*\n unless they also define *__slots__* (which must only contain names\n of any *additional* slots).\n\n* If a class defines a slot also defined in a base class, the instance\n variable defined by the base class slot is inaccessible (except by\n retrieving its descriptor directly from the base class). This\n renders the meaning of the program undefined. In the future, a\n check may be added to prevent this.\n\n* Nonempty *__slots__* does not work for classes derived from\n "variable-length" built-in types such as ``long``, ``str`` and\n ``tuple``.\n\n* Any non-string iterable may be assigned to *__slots__*. Mappings may\n also be used; however, in the future, special meaning may be\n assigned to the values corresponding to each key.\n\n* *__class__* assignment works only if both classes have the same\n *__slots__*.\n\n Changed in version 2.6: Previously, *__class__* assignment raised an\n error if either new or old class had *__slots__*.\n', + 'attribute-access': u'\nCustomizing attribute access\n****************************\n\nThe following methods can be defined to customize the meaning of\nattribute access (use of, assignment to, or deletion of ``x.name``)\nfor class instances.\n\nobject.__getattr__(self, name)\n\n Called when an attribute lookup has not found the attribute in the\n usual places (i.e. it is not an instance attribute nor is it found\n in the class tree for ``self``). ``name`` is the attribute name.\n This method should return the (computed) attribute value or raise\n an ``AttributeError`` exception.\n\n Note that if the attribute is found through the normal mechanism,\n ``__getattr__()`` is not called. (This is an intentional asymmetry\n between ``__getattr__()`` and ``__setattr__()``.) This is done both\n for efficiency reasons and because otherwise ``__getattr__()``\n would have no way to access other attributes of the instance. Note\n that at least for instance variables, you can fake total control by\n not inserting any values in the instance attribute dictionary (but\n instead inserting them in another object). See the\n ``__getattribute__()`` method below for a way to actually get total\n control in new-style classes.\n\nobject.__setattr__(self, name, value)\n\n Called when an attribute assignment is attempted. This is called\n instead of the normal mechanism (i.e. store the value in the\n instance dictionary). *name* is the attribute name, *value* is the\n value to be assigned to it.\n\n If ``__setattr__()`` wants to assign to an instance attribute, it\n should not simply execute ``self.name = value`` --- this would\n cause a recursive call to itself. Instead, it should insert the\n value in the dictionary of instance attributes, e.g.,\n ``self.__dict__[name] = value``. For new-style classes, rather\n than accessing the instance dictionary, it should call the base\n class method with the same name, for example,\n ``object.__setattr__(self, name, value)``.\n\nobject.__delattr__(self, name)\n\n Like ``__setattr__()`` but for attribute deletion instead of\n assignment. This should only be implemented if ``del obj.name`` is\n meaningful for the object.\n\n\nMore attribute access for new-style classes\n===========================================\n\nThe following methods only apply to new-style classes.\n\nobject.__getattribute__(self, name)\n\n Called unconditionally to implement attribute accesses for\n instances of the class. If the class also defines\n ``__getattr__()``, the latter will not be called unless\n ``__getattribute__()`` either calls it explicitly or raises an\n ``AttributeError``. This method should return the (computed)\n attribute value or raise an ``AttributeError`` exception. In order\n to avoid infinite recursion in this method, its implementation\n should always call the base class method with the same name to\n access any attributes it needs, for example,\n ``object.__getattribute__(self, name)``.\n\n Note: This method may still be bypassed when looking up special methods\n as the result of implicit invocation via language syntax or\n built-in functions. See *Special method lookup for new-style\n classes*.\n\n\nImplementing Descriptors\n========================\n\nThe following methods only apply when an instance of the class\ncontaining the method (a so-called *descriptor* class) appears in an\n*owner* class (the descriptor must be in either the owner\'s class\ndictionary or in the class dictionary for one of its parents). In the\nexamples below, "the attribute" refers to the attribute whose name is\nthe key of the property in the owner class\' ``__dict__``.\n\nobject.__get__(self, instance, owner)\n\n Called to get the attribute of the owner class (class attribute\n access) or of an instance of that class (instance attribute\n access). *owner* is always the owner class, while *instance* is the\n instance that the attribute was accessed through, or ``None`` when\n the attribute is accessed through the *owner*. This method should\n return the (computed) attribute value or raise an\n ``AttributeError`` exception.\n\nobject.__set__(self, instance, value)\n\n Called to set the attribute on an instance *instance* of the owner\n class to a new value, *value*.\n\nobject.__delete__(self, instance)\n\n Called to delete the attribute on an instance *instance* of the\n owner class.\n\n\nInvoking Descriptors\n====================\n\nIn general, a descriptor is an object attribute with "binding\nbehavior", one whose attribute access has been overridden by methods\nin the descriptor protocol: ``__get__()``, ``__set__()``, and\n``__delete__()``. If any of those methods are defined for an object,\nit is said to be a descriptor.\n\nThe default behavior for attribute access is to get, set, or delete\nthe attribute from an object\'s dictionary. For instance, ``a.x`` has a\nlookup chain starting with ``a.__dict__[\'x\']``, then\n``type(a).__dict__[\'x\']``, and continuing through the base classes of\n``type(a)`` excluding metaclasses.\n\nHowever, if the looked-up value is an object defining one of the\ndescriptor methods, then Python may override the default behavior and\ninvoke the descriptor method instead. Where this occurs in the\nprecedence chain depends on which descriptor methods were defined and\nhow they were called. Note that descriptors are only invoked for new\nstyle objects or classes (ones that subclass ``object()`` or\n``type()``).\n\nThe starting point for descriptor invocation is a binding, ``a.x``.\nHow the arguments are assembled depends on ``a``:\n\nDirect Call\n The simplest and least common call is when user code directly\n invokes a descriptor method: ``x.__get__(a)``.\n\nInstance Binding\n If binding to a new-style object instance, ``a.x`` is transformed\n into the call: ``type(a).__dict__[\'x\'].__get__(a, type(a))``.\n\nClass Binding\n If binding to a new-style class, ``A.x`` is transformed into the\n call: ``A.__dict__[\'x\'].__get__(None, A)``.\n\nSuper Binding\n If ``a`` is an instance of ``super``, then the binding ``super(B,\n obj).m()`` searches ``obj.__class__.__mro__`` for the base class\n ``A`` immediately preceding ``B`` and then invokes the descriptor\n with the call: ``A.__dict__[\'m\'].__get__(obj, obj.__class__)``.\n\nFor instance bindings, the precedence of descriptor invocation depends\non the which descriptor methods are defined. A descriptor can define\nany combination of ``__get__()``, ``__set__()`` and ``__delete__()``.\nIf it does not define ``__get__()``, then accessing the attribute will\nreturn the descriptor object itself unless there is a value in the\nobject\'s instance dictionary. If the descriptor defines ``__set__()``\nand/or ``__delete__()``, it is a data descriptor; if it defines\nneither, it is a non-data descriptor. Normally, data descriptors\ndefine both ``__get__()`` and ``__set__()``, while non-data\ndescriptors have just the ``__get__()`` method. Data descriptors with\n``__set__()`` and ``__get__()`` defined always override a redefinition\nin an instance dictionary. In contrast, non-data descriptors can be\noverridden by instances.\n\nPython methods (including ``staticmethod()`` and ``classmethod()``)\nare implemented as non-data descriptors. Accordingly, instances can\nredefine and override methods. This allows individual instances to\nacquire behaviors that differ from other instances of the same class.\n\nThe ``property()`` function is implemented as a data descriptor.\nAccordingly, instances cannot override the behavior of a property.\n\n\n__slots__\n=========\n\nBy default, instances of both old and new-style classes have a\ndictionary for attribute storage. This wastes space for objects\nhaving very few instance variables. The space consumption can become\nacute when creating large numbers of instances.\n\nThe default can be overridden by defining *__slots__* in a new-style\nclass definition. The *__slots__* declaration takes a sequence of\ninstance variables and reserves just enough space in each instance to\nhold a value for each variable. Space is saved because *__dict__* is\nnot created for each instance.\n\n__slots__\n\n This class variable can be assigned a string, iterable, or sequence\n of strings with variable names used by instances. If defined in a\n new-style class, *__slots__* reserves space for the declared\n variables and prevents the automatic creation of *__dict__* and\n *__weakref__* for each instance.\n\n New in version 2.2.\n\nNotes on using *__slots__*\n\n* When inheriting from a class without *__slots__*, the *__dict__*\n attribute of that class will always be accessible, so a *__slots__*\n definition in the subclass is meaningless.\n\n* Without a *__dict__* variable, instances cannot be assigned new\n variables not listed in the *__slots__* definition. Attempts to\n assign to an unlisted variable name raises ``AttributeError``. If\n dynamic assignment of new variables is desired, then add\n ``\'__dict__\'`` to the sequence of strings in the *__slots__*\n declaration.\n\n Changed in version 2.3: Previously, adding ``\'__dict__\'`` to the\n *__slots__* declaration would not enable the assignment of new\n attributes not specifically listed in the sequence of instance\n variable names.\n\n* Without a *__weakref__* variable for each instance, classes defining\n *__slots__* do not support weak references to its instances. If weak\n reference support is needed, then add ``\'__weakref__\'`` to the\n sequence of strings in the *__slots__* declaration.\n\n Changed in version 2.3: Previously, adding ``\'__weakref__\'`` to the\n *__slots__* declaration would not enable support for weak\n references.\n\n* *__slots__* are implemented at the class level by creating\n descriptors (*Implementing Descriptors*) for each variable name. As\n a result, class attributes cannot be used to set default values for\n instance variables defined by *__slots__*; otherwise, the class\n attribute would overwrite the descriptor assignment.\n\n* The action of a *__slots__* declaration is limited to the class\n where it is defined. As a result, subclasses will have a *__dict__*\n unless they also define *__slots__* (which must only contain names\n of any *additional* slots).\n\n* If a class defines a slot also defined in a base class, the instance\n variable defined by the base class slot is inaccessible (except by\n retrieving its descriptor directly from the base class). This\n renders the meaning of the program undefined. In the future, a\n check may be added to prevent this.\n\n* Nonempty *__slots__* does not work for classes derived from\n "variable-length" built-in types such as ``long``, ``str`` and\n ``tuple``.\n\n* Any non-string iterable may be assigned to *__slots__*. Mappings may\n also be used; however, in the future, special meaning may be\n assigned to the values corresponding to each key.\n\n* *__class__* assignment works only if both classes have the same\n *__slots__*.\n\n Changed in version 2.6: Previously, *__class__* assignment raised an\n error if either new or old class had *__slots__*.\n', 'attribute-references': u'\nAttribute references\n********************\n\nAn attribute reference is a primary followed by a period and a name:\n\n attributeref ::= primary "." identifier\n\nThe primary must evaluate to an object of a type that supports\nattribute references, e.g., a module, list, or an instance. This\nobject is then asked to produce the attribute whose name is the\nidentifier. If this attribute is not available, the exception\n``AttributeError`` is raised. Otherwise, the type and value of the\nobject produced is determined by the object. Multiple evaluations of\nthe same attribute reference may yield different objects.\n', 'augassign': u'\nAugmented assignment statements\n*******************************\n\nAugmented assignment is the combination, in a single statement, of a\nbinary operation and an assignment statement:\n\n augmented_assignment_stmt ::= augtarget augop (expression_list | yield_expression)\n augtarget ::= identifier | attributeref | subscription | slicing\n augop ::= "+=" | "-=" | "*=" | "/=" | "//=" | "%=" | "**="\n | ">>=" | "<<=" | "&=" | "^=" | "|="\n\n(See section *Primaries* for the syntax definitions for the last three\nsymbols.)\n\nAn augmented assignment evaluates the target (which, unlike normal\nassignment statements, cannot be an unpacking) and the expression\nlist, performs the binary operation specific to the type of assignment\non the two operands, and assigns the result to the original target.\nThe target is only evaluated once.\n\nAn augmented assignment expression like ``x += 1`` can be rewritten as\n``x = x + 1`` to achieve a similar, but not exactly equal effect. In\nthe augmented version, ``x`` is only evaluated once. Also, when\npossible, the actual operation is performed *in-place*, meaning that\nrather than creating a new object and assigning that to the target,\nthe old object is modified instead.\n\nWith the exception of assigning to tuples and multiple targets in a\nsingle statement, the assignment done by augmented assignment\nstatements is handled the same way as normal assignments. Similarly,\nwith the exception of the possible *in-place* behavior, the binary\noperation performed by augmented assignment is the same as the normal\nbinary operations.\n\nFor targets which are attribute references, the same *caveat about\nclass and instance attributes* applies as for regular assignments.\n', 'binary': u'\nBinary arithmetic operations\n****************************\n\nThe binary arithmetic operations have the conventional priority\nlevels. Note that some of these operations also apply to certain non-\nnumeric types. Apart from the power operator, there are only two\nlevels, one for multiplicative operators and one for additive\noperators:\n\n m_expr ::= u_expr | m_expr "*" u_expr | m_expr "//" u_expr | m_expr "/" u_expr\n | m_expr "%" u_expr\n a_expr ::= m_expr | a_expr "+" m_expr | a_expr "-" m_expr\n\nThe ``*`` (multiplication) operator yields the product of its\narguments. The arguments must either both be numbers, or one argument\nmust be an integer (plain or long) and the other must be a sequence.\nIn the former case, the numbers are converted to a common type and\nthen multiplied together. In the latter case, sequence repetition is\nperformed; a negative repetition factor yields an empty sequence.\n\nThe ``/`` (division) and ``//`` (floor division) operators yield the\nquotient of their arguments. The numeric arguments are first\nconverted to a common type. Plain or long integer division yields an\ninteger of the same type; the result is that of mathematical division\nwith the \'floor\' function applied to the result. Division by zero\nraises the ``ZeroDivisionError`` exception.\n\nThe ``%`` (modulo) operator yields the remainder from the division of\nthe first argument by the second. The numeric arguments are first\nconverted to a common type. A zero right argument raises the\n``ZeroDivisionError`` exception. The arguments may be floating point\nnumbers, e.g., ``3.14%0.7`` equals ``0.34`` (since ``3.14`` equals\n``4*0.7 + 0.34``.) The modulo operator always yields a result with\nthe same sign as its second operand (or zero); the absolute value of\nthe result is strictly smaller than the absolute value of the second\noperand [2].\n\nThe integer division and modulo operators are connected by the\nfollowing identity: ``x == (x/y)*y + (x%y)``. Integer division and\nmodulo are also connected with the built-in function ``divmod()``:\n``divmod(x, y) == (x/y, x%y)``. These identities don\'t hold for\nfloating point numbers; there similar identities hold approximately\nwhere ``x/y`` is replaced by ``floor(x/y)`` or ``floor(x/y) - 1`` [3].\n\nIn addition to performing the modulo operation on numbers, the ``%``\noperator is also overloaded by string and unicode objects to perform\nstring formatting (also known as interpolation). The syntax for string\nformatting is described in the Python Library Reference, section\n*String Formatting Operations*.\n\nDeprecated since version 2.3: The floor division operator, the modulo\noperator, and the ``divmod()`` function are no longer defined for\ncomplex numbers. Instead, convert to a floating point number using\nthe ``abs()`` function if appropriate.\n\nThe ``+`` (addition) operator yields the sum of its arguments. The\narguments must either both be numbers or both sequences of the same\ntype. In the former case, the numbers are converted to a common type\nand then added together. In the latter case, the sequences are\nconcatenated.\n\nThe ``-`` (subtraction) operator yields the difference of its\narguments. The numeric arguments are first converted to a common\ntype.\n', 'bitwise': u'\nBinary bitwise operations\n*************************\n\nEach of the three bitwise operations has a different priority level:\n\n and_expr ::= shift_expr | and_expr "&" shift_expr\n xor_expr ::= and_expr | xor_expr "^" and_expr\n or_expr ::= xor_expr | or_expr "|" xor_expr\n\nThe ``&`` operator yields the bitwise AND of its arguments, which must\nbe plain or long integers. The arguments are converted to a common\ntype.\n\nThe ``^`` operator yields the bitwise XOR (exclusive OR) of its\narguments, which must be plain or long integers. The arguments are\nconverted to a common type.\n\nThe ``|`` operator yields the bitwise (inclusive) OR of its arguments,\nwhich must be plain or long integers. The arguments are converted to\na common type.\n', 'bltin-code-objects': u'\nCode Objects\n************\n\nCode objects are used by the implementation to represent "pseudo-\ncompiled" executable Python code such as a function body. They differ\nfrom function objects because they don\'t contain a reference to their\nglobal execution environment. Code objects are returned by the built-\nin ``compile()`` function and can be extracted from function objects\nthrough their ``func_code`` attribute. See also the ``code`` module.\n\nA code object can be executed or evaluated by passing it (instead of a\nsource string) to the ``exec`` statement or the built-in ``eval()``\nfunction.\n\nSee *The standard type hierarchy* for more information.\n', 'bltin-ellipsis-object': u'\nThe Ellipsis Object\n*******************\n\nThis object is used by extended slice notation (see *Slicings*). It\nsupports no special operations. There is exactly one ellipsis object,\nnamed ``Ellipsis`` (a built-in name).\n\nIt is written as ``Ellipsis``.\n', - 'bltin-file-objects': u'\nFile Objects\n************\n\nFile objects are implemented using C\'s ``stdio`` package and can be\ncreated with the built-in ``open()`` function. File objects are also\nreturned by some other built-in functions and methods, such as\n``os.popen()`` and ``os.fdopen()`` and the ``makefile()`` method of\nsocket objects. Temporary files can be created using the ``tempfile``\nmodule, and high-level file operations such as copying, moving, and\ndeleting files and directories can be achieved with the ``shutil``\nmodule.\n\nWhen a file operation fails for an I/O-related reason, the exception\n``IOError`` is raised. This includes situations where the operation\nis not defined for some reason, like ``seek()`` on a tty device or\nwriting a file opened for reading.\n\nFiles have the following methods:\n\nfile.close()\n\n Close the file. A closed file cannot be read or written any more.\n Any operation which requires that the file be open will raise a\n ``ValueError`` after the file has been closed. Calling ``close()``\n more than once is allowed.\n\n As of Python 2.5, you can avoid having to call this method\n explicitly if you use the ``with`` statement. For example, the\n following code will automatically close *f* when the ``with`` block\n is exited:\n\n from __future__ import with_statement # This isn\'t required in Python 2.6\n\n with open("hello.txt") as f:\n for line in f:\n print line\n\n In older versions of Python, you would have needed to do this to\n get the same effect:\n\n f = open("hello.txt")\n try:\n for line in f:\n print line\n finally:\n f.close()\n\n Note: Not all "file-like" types in Python support use as a context\n manager for the ``with`` statement. If your code is intended to\n work with any file-like object, you can use the function\n ``contextlib.closing()`` instead of using the object directly.\n\nfile.flush()\n\n Flush the internal buffer, like ``stdio``\'s ``fflush()``. This may\n be a no-op on some file-like objects.\n\n Note: ``flush()`` does not necessarily write the file\'s data to disk.\n Use ``flush()`` followed by ``os.fsync()`` to ensure this\n behavior.\n\nfile.fileno()\n\n Return the integer "file descriptor" that is used by the underlying\n implementation to request I/O operations from the operating system.\n This can be useful for other, lower level interfaces that use file\n descriptors, such as the ``fcntl`` module or ``os.read()`` and\n friends.\n\n Note: File-like objects which do not have a real file descriptor should\n *not* provide this method!\n\nfile.isatty()\n\n Return ``True`` if the file is connected to a tty(-like) device,\n else ``False``.\n\n Note: If a file-like object is not associated with a real file, this\n method should *not* be implemented.\n\nfile.next()\n\n A file object is its own iterator, for example ``iter(f)`` returns\n *f* (unless *f* is closed). When a file is used as an iterator,\n typically in a ``for`` loop (for example, ``for line in f: print\n line``), the ``next()`` method is called repeatedly. This method\n returns the next input line, or raises ``StopIteration`` when EOF\n is hit when the file is open for reading (behavior is undefined\n when the file is open for writing). In order to make a ``for``\n loop the most efficient way of looping over the lines of a file (a\n very common operation), the ``next()`` method uses a hidden read-\n ahead buffer. As a consequence of using a read-ahead buffer,\n combining ``next()`` with other file methods (like ``readline()``)\n does not work right. However, using ``seek()`` to reposition the\n file to an absolute position will flush the read-ahead buffer.\n\n New in version 2.3.\n\nfile.read([size])\n\n Read at most *size* bytes from the file (less if the read hits EOF\n before obtaining *size* bytes). If the *size* argument is negative\n or omitted, read all data until EOF is reached. The bytes are\n returned as a string object. An empty string is returned when EOF\n is encountered immediately. (For certain files, like ttys, it\n makes sense to continue reading after an EOF is hit.) Note that\n this method may call the underlying C function ``fread()`` more\n than once in an effort to acquire as close to *size* bytes as\n possible. Also note that when in non-blocking mode, less data than\n was requested may be returned, even if no *size* parameter was\n given.\n\n Note: This function is simply a wrapper for the underlying ``fread()``\n C function, and will behave the same in corner cases, such as\n whether the EOF value is cached.\n\nfile.readline([size])\n\n Read one entire line from the file. A trailing newline character\n is kept in the string (but may be absent when a file ends with an\n incomplete line). [5] If the *size* argument is present and non-\n negative, it is a maximum byte count (including the trailing\n newline) and an incomplete line may be returned. An empty string is\n returned *only* when EOF is encountered immediately.\n\n Note: Unlike ``stdio``\'s ``fgets()``, the returned string contains null\n characters (``\'\\0\'``) if they occurred in the input.\n\nfile.readlines([sizehint])\n\n Read until EOF using ``readline()`` and return a list containing\n the lines thus read. If the optional *sizehint* argument is\n present, instead of reading up to EOF, whole lines totalling\n approximately *sizehint* bytes (possibly after rounding up to an\n internal buffer size) are read. Objects implementing a file-like\n interface may choose to ignore *sizehint* if it cannot be\n implemented, or cannot be implemented efficiently.\n\nfile.xreadlines()\n\n This method returns the same thing as ``iter(f)``.\n\n New in version 2.1.\n\n Deprecated since version 2.3: Use ``for line in file`` instead.\n\nfile.seek(offset[, whence])\n\n Set the file\'s current position, like ``stdio``\'s ``fseek()``. The\n *whence* argument is optional and defaults to ``os.SEEK_SET`` or\n ``0`` (absolute file positioning); other values are ``os.SEEK_CUR``\n or ``1`` (seek relative to the current position) and\n ``os.SEEK_END`` or ``2`` (seek relative to the file\'s end). There\n is no return value.\n\n For example, ``f.seek(2, os.SEEK_CUR)`` advances the position by\n two and ``f.seek(-3, os.SEEK_END)`` sets the position to the third\n to last.\n\n Note that if the file is opened for appending (mode ``\'a\'`` or\n ``\'a+\'``), any ``seek()`` operations will be undone at the next\n write. If the file is only opened for writing in append mode (mode\n ``\'a\'``), this method is essentially a no-op, but it remains useful\n for files opened in append mode with reading enabled (mode\n ``\'a+\'``). If the file is opened in text mode (without ``\'b\'``),\n only offsets returned by ``tell()`` are legal. Use of other\n offsets causes undefined behavior.\n\n Note that not all file objects are seekable.\n\n Changed in version 2.6: Passing float values as offset has been\n deprecated.\n\nfile.tell()\n\n Return the file\'s current position, like ``stdio``\'s ``ftell()``.\n\n Note: On Windows, ``tell()`` can return illegal values (after an\n ``fgets()``) when reading files with Unix-style line-endings. Use\n binary mode (``\'rb\'``) to circumvent this problem.\n\nfile.truncate([size])\n\n Truncate the file\'s size. If the optional *size* argument is\n present, the file is truncated to (at most) that size. The size\n defaults to the current position. The current file position is not\n changed. Note that if a specified size exceeds the file\'s current\n size, the result is platform-dependent: possibilities include that\n the file may remain unchanged, increase to the specified size as if\n zero-filled, or increase to the specified size with undefined new\n content. Availability: Windows, many Unix variants.\n\nfile.write(str)\n\n Write a string to the file. There is no return value. Due to\n buffering, the string may not actually show up in the file until\n the ``flush()`` or ``close()`` method is called.\n\nfile.writelines(sequence)\n\n Write a sequence of strings to the file. The sequence can be any\n iterable object producing strings, typically a list of strings.\n There is no return value. (The name is intended to match\n ``readlines()``; ``writelines()`` does not add line separators.)\n\nFiles support the iterator protocol. Each iteration returns the same\nresult as ``file.readline()``, and iteration ends when the\n``readline()`` method returns an empty string.\n\nFile objects also offer a number of other interesting attributes.\nThese are not required for file-like objects, but should be\nimplemented if they make sense for the particular object.\n\nfile.closed\n\n bool indicating the current state of the file object. This is a\n read-only attribute; the ``close()`` method changes the value. It\n may not be available on all file-like objects.\n\nfile.encoding\n\n The encoding that this file uses. When Unicode strings are written\n to a file, they will be converted to byte strings using this\n encoding. In addition, when the file is connected to a terminal,\n the attribute gives the encoding that the terminal is likely to use\n (that information might be incorrect if the user has misconfigured\n the terminal). The attribute is read-only and may not be present\n on all file-like objects. It may also be ``None``, in which case\n the file uses the system default encoding for converting Unicode\n strings.\n\n New in version 2.3.\n\nfile.errors\n\n The Unicode error handler used along with the encoding.\n\n New in version 2.6.\n\nfile.mode\n\n The I/O mode for the file. If the file was created using the\n ``open()`` built-in function, this will be the value of the *mode*\n parameter. This is a read-only attribute and may not be present on\n all file-like objects.\n\nfile.name\n\n If the file object was created using ``open()``, the name of the\n file. Otherwise, some string that indicates the source of the file\n object, of the form ``<...>``. This is a read-only attribute and\n may not be present on all file-like objects.\n\nfile.newlines\n\n If Python was built with the *--with-universal-newlines* option to\n **configure** (the default) this read-only attribute exists, and\n for files opened in universal newline read mode it keeps track of\n the types of newlines encountered while reading the file. The\n values it can take are ``\'\\r\'``, ``\'\\n\'``, ``\'\\r\\n\'``, ``None``\n (unknown, no newlines read yet) or a tuple containing all the\n newline types seen, to indicate that multiple newline conventions\n were encountered. For files not opened in universal newline read\n mode the value of this attribute will be ``None``.\n\nfile.softspace\n\n Boolean that indicates whether a space character needs to be\n printed before another value when using the ``print`` statement.\n Classes that are trying to simulate a file object should also have\n a writable ``softspace`` attribute, which should be initialized to\n zero. This will be automatic for most classes implemented in\n Python (care may be needed for objects that override attribute\n access); types implemented in C will have to provide a writable\n ``softspace`` attribute.\n\n Note: This attribute is not used to control the ``print`` statement,\n but to allow the implementation of ``print`` to keep track of its\n internal state.\n', + 'bltin-file-objects': u'\nFile Objects\n************\n\nFile objects are implemented using C\'s ``stdio`` package and can be\ncreated with the built-in ``open()`` function. File objects are also\nreturned by some other built-in functions and methods, such as\n``os.popen()`` and ``os.fdopen()`` and the ``makefile()`` method of\nsocket objects. Temporary files can be created using the ``tempfile``\nmodule, and high-level file operations such as copying, moving, and\ndeleting files and directories can be achieved with the ``shutil``\nmodule.\n\nWhen a file operation fails for an I/O-related reason, the exception\n``IOError`` is raised. This includes situations where the operation\nis not defined for some reason, like ``seek()`` on a tty device or\nwriting a file opened for reading.\n\nFiles have the following methods:\n\nfile.close()\n\n Close the file. A closed file cannot be read or written any more.\n Any operation which requires that the file be open will raise a\n ``ValueError`` after the file has been closed. Calling ``close()``\n more than once is allowed.\n\n As of Python 2.5, you can avoid having to call this method\n explicitly if you use the ``with`` statement. For example, the\n following code will automatically close *f* when the ``with`` block\n is exited:\n\n from __future__ import with_statement # This isn\'t required in Python 2.6\n\n with open("hello.txt") as f:\n for line in f:\n print line\n\n In older versions of Python, you would have needed to do this to\n get the same effect:\n\n f = open("hello.txt")\n try:\n for line in f:\n print line\n finally:\n f.close()\n\n Note: Not all "file-like" types in Python support use as a context\n manager for the ``with`` statement. If your code is intended to\n work with any file-like object, you can use the function\n ``contextlib.closing()`` instead of using the object directly.\n\nfile.flush()\n\n Flush the internal buffer, like ``stdio``\'s ``fflush()``. This may\n be a no-op on some file-like objects.\n\n Note: ``flush()`` does not necessarily write the file\'s data to disk.\n Use ``flush()`` followed by ``os.fsync()`` to ensure this\n behavior.\n\nfile.fileno()\n\n Return the integer "file descriptor" that is used by the underlying\n implementation to request I/O operations from the operating system.\n This can be useful for other, lower level interfaces that use file\n descriptors, such as the ``fcntl`` module or ``os.read()`` and\n friends.\n\n Note: File-like objects which do not have a real file descriptor should\n *not* provide this method!\n\nfile.isatty()\n\n Return ``True`` if the file is connected to a tty(-like) device,\n else ``False``.\n\n Note: If a file-like object is not associated with a real file, this\n method should *not* be implemented.\n\nfile.next()\n\n A file object is its own iterator, for example ``iter(f)`` returns\n *f* (unless *f* is closed). When a file is used as an iterator,\n typically in a ``for`` loop (for example, ``for line in f: print\n line``), the ``next()`` method is called repeatedly. This method\n returns the next input line, or raises ``StopIteration`` when EOF\n is hit when the file is open for reading (behavior is undefined\n when the file is open for writing). In order to make a ``for``\n loop the most efficient way of looping over the lines of a file (a\n very common operation), the ``next()`` method uses a hidden read-\n ahead buffer. As a consequence of using a read-ahead buffer,\n combining ``next()`` with other file methods (like ``readline()``)\n does not work right. However, using ``seek()`` to reposition the\n file to an absolute position will flush the read-ahead buffer.\n\n New in version 2.3.\n\nfile.read([size])\n\n Read at most *size* bytes from the file (less if the read hits EOF\n before obtaining *size* bytes). If the *size* argument is negative\n or omitted, read all data until EOF is reached. The bytes are\n returned as a string object. An empty string is returned when EOF\n is encountered immediately. (For certain files, like ttys, it\n makes sense to continue reading after an EOF is hit.) Note that\n this method may call the underlying C function ``fread()`` more\n than once in an effort to acquire as close to *size* bytes as\n possible. Also note that when in non-blocking mode, less data than\n was requested may be returned, even if no *size* parameter was\n given.\n\n Note: This function is simply a wrapper for the underlying ``fread()``\n C function, and will behave the same in corner cases, such as\n whether the EOF value is cached.\n\nfile.readline([size])\n\n Read one entire line from the file. A trailing newline character\n is kept in the string (but may be absent when a file ends with an\n incomplete line). [5] If the *size* argument is present and non-\n negative, it is a maximum byte count (including the trailing\n newline) and an incomplete line may be returned. When *size* is not\n 0, an empty string is returned *only* when EOF is encountered\n immediately.\n\n Note: Unlike ``stdio``\'s ``fgets()``, the returned string contains null\n characters (``\'\\0\'``) if they occurred in the input.\n\nfile.readlines([sizehint])\n\n Read until EOF using ``readline()`` and return a list containing\n the lines thus read. If the optional *sizehint* argument is\n present, instead of reading up to EOF, whole lines totalling\n approximately *sizehint* bytes (possibly after rounding up to an\n internal buffer size) are read. Objects implementing a file-like\n interface may choose to ignore *sizehint* if it cannot be\n implemented, or cannot be implemented efficiently.\n\nfile.xreadlines()\n\n This method returns the same thing as ``iter(f)``.\n\n New in version 2.1.\n\n Deprecated since version 2.3: Use ``for line in file`` instead.\n\nfile.seek(offset[, whence])\n\n Set the file\'s current position, like ``stdio``\'s ``fseek()``. The\n *whence* argument is optional and defaults to ``os.SEEK_SET`` or\n ``0`` (absolute file positioning); other values are ``os.SEEK_CUR``\n or ``1`` (seek relative to the current position) and\n ``os.SEEK_END`` or ``2`` (seek relative to the file\'s end). There\n is no return value.\n\n For example, ``f.seek(2, os.SEEK_CUR)`` advances the position by\n two and ``f.seek(-3, os.SEEK_END)`` sets the position to the third\n to last.\n\n Note that if the file is opened for appending (mode ``\'a\'`` or\n ``\'a+\'``), any ``seek()`` operations will be undone at the next\n write. If the file is only opened for writing in append mode (mode\n ``\'a\'``), this method is essentially a no-op, but it remains useful\n for files opened in append mode with reading enabled (mode\n ``\'a+\'``). If the file is opened in text mode (without ``\'b\'``),\n only offsets returned by ``tell()`` are legal. Use of other\n offsets causes undefined behavior.\n\n Note that not all file objects are seekable.\n\n Changed in version 2.6: Passing float values as offset has been\n deprecated.\n\nfile.tell()\n\n Return the file\'s current position, like ``stdio``\'s ``ftell()``.\n\n Note: On Windows, ``tell()`` can return illegal values (after an\n ``fgets()``) when reading files with Unix-style line-endings. Use\n binary mode (``\'rb\'``) to circumvent this problem.\n\nfile.truncate([size])\n\n Truncate the file\'s size. If the optional *size* argument is\n present, the file is truncated to (at most) that size. The size\n defaults to the current position. The current file position is not\n changed. Note that if a specified size exceeds the file\'s current\n size, the result is platform-dependent: possibilities include that\n the file may remain unchanged, increase to the specified size as if\n zero-filled, or increase to the specified size with undefined new\n content. Availability: Windows, many Unix variants.\n\nfile.write(str)\n\n Write a string to the file. There is no return value. Due to\n buffering, the string may not actually show up in the file until\n the ``flush()`` or ``close()`` method is called.\n\nfile.writelines(sequence)\n\n Write a sequence of strings to the file. The sequence can be any\n iterable object producing strings, typically a list of strings.\n There is no return value. (The name is intended to match\n ``readlines()``; ``writelines()`` does not add line separators.)\n\nFiles support the iterator protocol. Each iteration returns the same\nresult as ``file.readline()``, and iteration ends when the\n``readline()`` method returns an empty string.\n\nFile objects also offer a number of other interesting attributes.\nThese are not required for file-like objects, but should be\nimplemented if they make sense for the particular object.\n\nfile.closed\n\n bool indicating the current state of the file object. This is a\n read-only attribute; the ``close()`` method changes the value. It\n may not be available on all file-like objects.\n\nfile.encoding\n\n The encoding that this file uses. When Unicode strings are written\n to a file, they will be converted to byte strings using this\n encoding. In addition, when the file is connected to a terminal,\n the attribute gives the encoding that the terminal is likely to use\n (that information might be incorrect if the user has misconfigured\n the terminal). The attribute is read-only and may not be present\n on all file-like objects. It may also be ``None``, in which case\n the file uses the system default encoding for converting Unicode\n strings.\n\n New in version 2.3.\n\nfile.errors\n\n The Unicode error handler used along with the encoding.\n\n New in version 2.6.\n\nfile.mode\n\n The I/O mode for the file. If the file was created using the\n ``open()`` built-in function, this will be the value of the *mode*\n parameter. This is a read-only attribute and may not be present on\n all file-like objects.\n\nfile.name\n\n If the file object was created using ``open()``, the name of the\n file. Otherwise, some string that indicates the source of the file\n object, of the form ``<...>``. This is a read-only attribute and\n may not be present on all file-like objects.\n\nfile.newlines\n\n If Python was built with universal newlines enabled (the default)\n this read-only attribute exists, and for files opened in universal\n newline read mode it keeps track of the types of newlines\n encountered while reading the file. The values it can take are\n ``\'\\r\'``, ``\'\\n\'``, ``\'\\r\\n\'``, ``None`` (unknown, no newlines read\n yet) or a tuple containing all the newline types seen, to indicate\n that multiple newline conventions were encountered. For files not\n opened in universal newline read mode the value of this attribute\n will be ``None``.\n\nfile.softspace\n\n Boolean that indicates whether a space character needs to be\n printed before another value when using the ``print`` statement.\n Classes that are trying to simulate a file object should also have\n a writable ``softspace`` attribute, which should be initialized to\n zero. This will be automatic for most classes implemented in\n Python (care may be needed for objects that override attribute\n access); types implemented in C will have to provide a writable\n ``softspace`` attribute.\n\n Note: This attribute is not used to control the ``print`` statement,\n but to allow the implementation of ``print`` to keep track of its\n internal state.\n', 'bltin-null-object': u"\nThe Null Object\n***************\n\nThis object is returned by functions that don't explicitly return a\nvalue. It supports no special operations. There is exactly one null\nobject, named ``None`` (a built-in name).\n\nIt is written as ``None``.\n", 'bltin-type-objects': u"\nType Objects\n************\n\nType objects represent the various object types. An object's type is\naccessed by the built-in function ``type()``. There are no special\noperations on types. The standard module ``types`` defines names for\nall standard built-in types.\n\nTypes are written like this: ``<type 'int'>``.\n", 'booleans': u'\nBoolean operations\n******************\n\n or_test ::= and_test | or_test "or" and_test\n and_test ::= not_test | and_test "and" not_test\n not_test ::= comparison | "not" not_test\n\nIn the context of Boolean operations, and also when expressions are\nused by control flow statements, the following values are interpreted\nas false: ``False``, ``None``, numeric zero of all types, and empty\nstrings and containers (including strings, tuples, lists,\ndictionaries, sets and frozensets). All other values are interpreted\nas true. (See the ``__nonzero__()`` special method for a way to\nchange this.)\n\nThe operator ``not`` yields ``True`` if its argument is false,\n``False`` otherwise.\n\nThe expression ``x and y`` first evaluates *x*; if *x* is false, its\nvalue is returned; otherwise, *y* is evaluated and the resulting value\nis returned.\n\nThe expression ``x or y`` first evaluates *x*; if *x* is true, its\nvalue is returned; otherwise, *y* is evaluated and the resulting value\nis returned.\n\n(Note that neither ``and`` nor ``or`` restrict the value and type they\nreturn to ``False`` and ``True``, but rather return the last evaluated\nargument. This is sometimes useful, e.g., if ``s`` is a string that\nshould be replaced by a default value if it is empty, the expression\n``s or \'foo\'`` yields the desired value. Because ``not`` has to\ninvent a value anyway, it does not bother to return a value of the\nsame type as its argument, so e.g., ``not \'foo\'`` yields ``False``,\nnot ``\'\'``.)\n', @@ -20,39 +20,39 @@ 'class': u'\nClass definitions\n*****************\n\nA class definition defines a class object (see section *The standard\ntype hierarchy*):\n\n classdef ::= "class" classname [inheritance] ":" suite\n inheritance ::= "(" [expression_list] ")"\n classname ::= identifier\n\nA class definition is an executable statement. It first evaluates the\ninheritance list, if present. Each item in the inheritance list\nshould evaluate to a class object or class type which allows\nsubclassing. The class\'s suite is then executed in a new execution\nframe (see section *Naming and binding*), using a newly created local\nnamespace and the original global namespace. (Usually, the suite\ncontains only function definitions.) When the class\'s suite finishes\nexecution, its execution frame is discarded but its local namespace is\nsaved. [4] A class object is then created using the inheritance list\nfor the base classes and the saved local namespace for the attribute\ndictionary. The class name is bound to this class object in the\noriginal local namespace.\n\n**Programmer\'s note:** Variables defined in the class definition are\nclass variables; they are shared by all instances. To create instance\nvariables, they can be set in a method with ``self.name = value``.\nBoth class and instance variables are accessible through the notation\n"``self.name``", and an instance variable hides a class variable with\nthe same name when accessed in this way. Class variables can be used\nas defaults for instance variables, but using mutable values there can\nlead to unexpected results. For *new-style class*es, descriptors can\nbe used to create instance variables with different implementation\ndetails.\n\nClass definitions, like function definitions, may be wrapped by one or\nmore *decorator* expressions. The evaluation rules for the decorator\nexpressions are the same as for functions. The result must be a class\nobject, which is then bound to the class name.\n\n-[ Footnotes ]-\n\n[1] The exception is propagated to the invocation stack only if there\n is no ``finally`` clause that negates the exception.\n\n[2] Currently, control "flows off the end" except in the case of an\n exception or the execution of a ``return``, ``continue``, or\n ``break`` statement.\n\n[3] A string literal appearing as the first statement in the function\n body is transformed into the function\'s ``__doc__`` attribute and\n therefore the function\'s *docstring*.\n\n[4] A string literal appearing as the first statement in the class\n body is transformed into the namespace\'s ``__doc__`` item and\n therefore the class\'s *docstring*.\n', 'coercion-rules': u"\nCoercion rules\n**************\n\nThis section used to document the rules for coercion. As the language\nhas evolved, the coercion rules have become hard to document\nprecisely; documenting what one version of one particular\nimplementation does is undesirable. Instead, here are some informal\nguidelines regarding coercion. In Python 3.0, coercion will not be\nsupported.\n\n* If the left operand of a % operator is a string or Unicode object,\n no coercion takes place and the string formatting operation is\n invoked instead.\n\n* It is no longer recommended to define a coercion operation. Mixed-\n mode operations on types that don't define coercion pass the\n original arguments to the operation.\n\n* New-style classes (those derived from ``object``) never invoke the\n ``__coerce__()`` method in response to a binary operator; the only\n time ``__coerce__()`` is invoked is when the built-in function\n ``coerce()`` is called.\n\n* For most intents and purposes, an operator that returns\n ``NotImplemented`` is treated the same as one that is not\n implemented at all.\n\n* Below, ``__op__()`` and ``__rop__()`` are used to signify the\n generic method names corresponding to an operator; ``__iop__()`` is\n used for the corresponding in-place operator. For example, for the\n operator '``+``', ``__add__()`` and ``__radd__()`` are used for the\n left and right variant of the binary operator, and ``__iadd__()``\n for the in-place variant.\n\n* For objects *x* and *y*, first ``x.__op__(y)`` is tried. If this is\n not implemented or returns ``NotImplemented``, ``y.__rop__(x)`` is\n tried. If this is also not implemented or returns\n ``NotImplemented``, a ``TypeError`` exception is raised. But see\n the following exception:\n\n* Exception to the previous item: if the left operand is an instance\n of a built-in type or a new-style class, and the right operand is an\n instance of a proper subclass of that type or class and overrides\n the base's ``__rop__()`` method, the right operand's ``__rop__()``\n method is tried *before* the left operand's ``__op__()`` method.\n\n This is done so that a subclass can completely override binary\n operators. Otherwise, the left operand's ``__op__()`` method would\n always accept the right operand: when an instance of a given class\n is expected, an instance of a subclass of that class is always\n acceptable.\n\n* When either operand type defines a coercion, this coercion is called\n before that type's ``__op__()`` or ``__rop__()`` method is called,\n but no sooner. If the coercion returns an object of a different\n type for the operand whose coercion is invoked, part of the process\n is redone using the new object.\n\n* When an in-place operator (like '``+=``') is used, if the left\n operand implements ``__iop__()``, it is invoked without any\n coercion. When the operation falls back to ``__op__()`` and/or\n ``__rop__()``, the normal coercion rules apply.\n\n* In ``x + y``, if *x* is a sequence that implements sequence\n concatenation, sequence concatenation is invoked.\n\n* In ``x * y``, if one operator is a sequence that implements sequence\n repetition, and the other is an integer (``int`` or ``long``),\n sequence repetition is invoked.\n\n* Rich comparisons (implemented by methods ``__eq__()`` and so on)\n never use coercion. Three-way comparison (implemented by\n ``__cmp__()``) does use coercion under the same conditions as other\n binary operations use it.\n\n* In the current implementation, the built-in numeric types ``int``,\n ``long``, ``float``, and ``complex`` do not use coercion. All these\n types implement a ``__coerce__()`` method, for use by the built-in\n ``coerce()`` function.\n\n Changed in version 2.7.\n", 'comparisons': u'\nComparisons\n***********\n\nUnlike C, all comparison operations in Python have the same priority,\nwhich is lower than that of any arithmetic, shifting or bitwise\noperation. Also unlike C, expressions like ``a < b < c`` have the\ninterpretation that is conventional in mathematics:\n\n comparison ::= or_expr ( comp_operator or_expr )*\n comp_operator ::= "<" | ">" | "==" | ">=" | "<=" | "<>" | "!="\n | "is" ["not"] | ["not"] "in"\n\nComparisons yield boolean values: ``True`` or ``False``.\n\nComparisons can be chained arbitrarily, e.g., ``x < y <= z`` is\nequivalent to ``x < y and y <= z``, except that ``y`` is evaluated\nonly once (but in both cases ``z`` is not evaluated at all when ``x <\ny`` is found to be false).\n\nFormally, if *a*, *b*, *c*, ..., *y*, *z* are expressions and *op1*,\n*op2*, ..., *opN* are comparison operators, then ``a op1 b op2 c ... y\nopN z`` is equivalent to ``a op1 b and b op2 c and ... y opN z``,\nexcept that each expression is evaluated at most once.\n\nNote that ``a op1 b op2 c`` doesn\'t imply any kind of comparison\nbetween *a* and *c*, so that, e.g., ``x < y > z`` is perfectly legal\n(though perhaps not pretty).\n\nThe forms ``<>`` and ``!=`` are equivalent; for consistency with C,\n``!=`` is preferred; where ``!=`` is mentioned below ``<>`` is also\naccepted. The ``<>`` spelling is considered obsolescent.\n\nThe operators ``<``, ``>``, ``==``, ``>=``, ``<=``, and ``!=`` compare\nthe values of two objects. The objects need not have the same type.\nIf both are numbers, they are converted to a common type. Otherwise,\nobjects of different types *always* compare unequal, and are ordered\nconsistently but arbitrarily. You can control comparison behavior of\nobjects of non-built-in types by defining a ``__cmp__`` method or rich\ncomparison methods like ``__gt__``, described in section *Special\nmethod names*.\n\n(This unusual definition of comparison was used to simplify the\ndefinition of operations like sorting and the ``in`` and ``not in``\noperators. In the future, the comparison rules for objects of\ndifferent types are likely to change.)\n\nComparison of objects of the same type depends on the type:\n\n* Numbers are compared arithmetically.\n\n* Strings are compared lexicographically using the numeric equivalents\n (the result of the built-in function ``ord()``) of their characters.\n Unicode and 8-bit strings are fully interoperable in this behavior.\n [4]\n\n* Tuples and lists are compared lexicographically using comparison of\n corresponding elements. This means that to compare equal, each\n element must compare equal and the two sequences must be of the same\n type and have the same length.\n\n If not equal, the sequences are ordered the same as their first\n differing elements. For example, ``cmp([1,2,x], [1,2,y])`` returns\n the same as ``cmp(x,y)``. If the corresponding element does not\n exist, the shorter sequence is ordered first (for example, ``[1,2] <\n [1,2,3]``).\n\n* Mappings (dictionaries) compare equal if and only if their sorted\n (key, value) lists compare equal. [5] Outcomes other than equality\n are resolved consistently, but are not otherwise defined. [6]\n\n* Most other objects of built-in types compare unequal unless they are\n the same object; the choice whether one object is considered smaller\n or larger than another one is made arbitrarily but consistently\n within one execution of a program.\n\nThe operators ``in`` and ``not in`` test for collection membership.\n``x in s`` evaluates to true if *x* is a member of the collection *s*,\nand false otherwise. ``x not in s`` returns the negation of ``x in\ns``. The collection membership test has traditionally been bound to\nsequences; an object is a member of a collection if the collection is\na sequence and contains an element equal to that object. However, it\nmake sense for many other object types to support membership tests\nwithout being a sequence. In particular, dictionaries (for keys) and\nsets support membership testing.\n\nFor the list and tuple types, ``x in y`` is true if and only if there\nexists an index *i* such that ``x == y[i]`` is true.\n\nFor the Unicode and string types, ``x in y`` is true if and only if\n*x* is a substring of *y*. An equivalent test is ``y.find(x) != -1``.\nNote, *x* and *y* need not be the same type; consequently, ``u\'ab\' in\n\'abc\'`` will return ``True``. Empty strings are always considered to\nbe a substring of any other string, so ``"" in "abc"`` will return\n``True``.\n\nChanged in version 2.3: Previously, *x* was required to be a string of\nlength ``1``.\n\nFor user-defined classes which define the ``__contains__()`` method,\n``x in y`` is true if and only if ``y.__contains__(x)`` is true.\n\nFor user-defined classes which do not define ``__contains__()`` but do\ndefine ``__iter__()``, ``x in y`` is true if some value ``z`` with ``x\n== z`` is produced while iterating over ``y``. If an exception is\nraised during the iteration, it is as if ``in`` raised that exception.\n\nLastly, the old-style iteration protocol is tried: if a class defines\n``__getitem__()``, ``x in y`` is true if and only if there is a non-\nnegative integer index *i* such that ``x == y[i]``, and all lower\ninteger indices do not raise ``IndexError`` exception. (If any other\nexception is raised, it is as if ``in`` raised that exception).\n\nThe operator ``not in`` is defined to have the inverse true value of\n``in``.\n\nThe operators ``is`` and ``is not`` test for object identity: ``x is\ny`` is true if and only if *x* and *y* are the same object. ``x is\nnot y`` yields the inverse truth value. [7]\n', - 'compound': u'\nCompound statements\n*******************\n\nCompound statements contain (groups of) other statements; they affect\nor control the execution of those other statements in some way. In\ngeneral, compound statements span multiple lines, although in simple\nincarnations a whole compound statement may be contained in one line.\n\nThe ``if``, ``while`` and ``for`` statements implement traditional\ncontrol flow constructs. ``try`` specifies exception handlers and/or\ncleanup code for a group of statements. Function and class\ndefinitions are also syntactically compound statements.\n\nCompound statements consist of one or more \'clauses.\' A clause\nconsists of a header and a \'suite.\' The clause headers of a\nparticular compound statement are all at the same indentation level.\nEach clause header begins with a uniquely identifying keyword and ends\nwith a colon. A suite is a group of statements controlled by a\nclause. A suite can be one or more semicolon-separated simple\nstatements on the same line as the header, following the header\'s\ncolon, or it can be one or more indented statements on subsequent\nlines. Only the latter form of suite can contain nested compound\nstatements; the following is illegal, mostly because it wouldn\'t be\nclear to which ``if`` clause a following ``else`` clause would belong:\n\n if test1: if test2: print x\n\nAlso note that the semicolon binds tighter than the colon in this\ncontext, so that in the following example, either all or none of the\n``print`` statements are executed:\n\n if x < y < z: print x; print y; print z\n\nSummarizing:\n\n compound_stmt ::= if_stmt\n | while_stmt\n | for_stmt\n | try_stmt\n | with_stmt\n | funcdef\n | classdef\n | decorated\n suite ::= stmt_list NEWLINE | NEWLINE INDENT statement+ DEDENT\n statement ::= stmt_list NEWLINE | compound_stmt\n stmt_list ::= simple_stmt (";" simple_stmt)* [";"]\n\nNote that statements always end in a ``NEWLINE`` possibly followed by\na ``DEDENT``. Also note that optional continuation clauses always\nbegin with a keyword that cannot start a statement, thus there are no\nambiguities (the \'dangling ``else``\' problem is solved in Python by\nrequiring nested ``if`` statements to be indented).\n\nThe formatting of the grammar rules in the following sections places\neach clause on a separate line for clarity.\n\n\nThe ``if`` statement\n====================\n\nThe ``if`` statement is used for conditional execution:\n\n if_stmt ::= "if" expression ":" suite\n ( "elif" expression ":" suite )*\n ["else" ":" suite]\n\nIt selects exactly one of the suites by evaluating the expressions one\nby one until one is found to be true (see section *Boolean operations*\nfor the definition of true and false); then that suite is executed\n(and no other part of the ``if`` statement is executed or evaluated).\nIf all expressions are false, the suite of the ``else`` clause, if\npresent, is executed.\n\n\nThe ``while`` statement\n=======================\n\nThe ``while`` statement is used for repeated execution as long as an\nexpression is true:\n\n while_stmt ::= "while" expression ":" suite\n ["else" ":" suite]\n\nThis repeatedly tests the expression and, if it is true, executes the\nfirst suite; if the expression is false (which may be the first time\nit is tested) the suite of the ``else`` clause, if present, is\nexecuted and the loop terminates.\n\nA ``break`` statement executed in the first suite terminates the loop\nwithout executing the ``else`` clause\'s suite. A ``continue``\nstatement executed in the first suite skips the rest of the suite and\ngoes back to testing the expression.\n\n\nThe ``for`` statement\n=====================\n\nThe ``for`` statement is used to iterate over the elements of a\nsequence (such as a string, tuple or list) or other iterable object:\n\n for_stmt ::= "for" target_list "in" expression_list ":" suite\n ["else" ":" suite]\n\nThe expression list is evaluated once; it should yield an iterable\nobject. An iterator is created for the result of the\n``expression_list``. The suite is then executed once for each item\nprovided by the iterator, in the order of ascending indices. Each\nitem in turn is assigned to the target list using the standard rules\nfor assignments, and then the suite is executed. When the items are\nexhausted (which is immediately when the sequence is empty), the suite\nin the ``else`` clause, if present, is executed, and the loop\nterminates.\n\nA ``break`` statement executed in the first suite terminates the loop\nwithout executing the ``else`` clause\'s suite. A ``continue``\nstatement executed in the first suite skips the rest of the suite and\ncontinues with the next item, or with the ``else`` clause if there was\nno next item.\n\nThe suite may assign to the variable(s) in the target list; this does\nnot affect the next item assigned to it.\n\nThe target list is not deleted when the loop is finished, but if the\nsequence is empty, it will not have been assigned to at all by the\nloop. Hint: the built-in function ``range()`` returns a sequence of\nintegers suitable to emulate the effect of Pascal\'s ``for i := a to b\ndo``; e.g., ``range(3)`` returns the list ``[0, 1, 2]``.\n\nNote: There is a subtlety when the sequence is being modified by the loop\n (this can only occur for mutable sequences, i.e. lists). An internal\n counter is used to keep track of which item is used next, and this\n is incremented on each iteration. When this counter has reached the\n length of the sequence the loop terminates. This means that if the\n suite deletes the current (or a previous) item from the sequence,\n the next item will be skipped (since it gets the index of the\n current item which has already been treated). Likewise, if the\n suite inserts an item in the sequence before the current item, the\n current item will be treated again the next time through the loop.\n This can lead to nasty bugs that can be avoided by making a\n temporary copy using a slice of the whole sequence, e.g.,\n\n for x in a[:]:\n if x < 0: a.remove(x)\n\n\nThe ``try`` statement\n=====================\n\nThe ``try`` statement specifies exception handlers and/or cleanup code\nfor a group of statements:\n\n try_stmt ::= try1_stmt | try2_stmt\n try1_stmt ::= "try" ":" suite\n ("except" [expression [("as" | ",") target]] ":" suite)+\n ["else" ":" suite]\n ["finally" ":" suite]\n try2_stmt ::= "try" ":" suite\n "finally" ":" suite\n\nChanged in version 2.5: In previous versions of Python,\n``try``...``except``...``finally`` did not work. ``try``...``except``\nhad to be nested in ``try``...``finally``.\n\nThe ``except`` clause(s) specify one or more exception handlers. When\nno exception occurs in the ``try`` clause, no exception handler is\nexecuted. When an exception occurs in the ``try`` suite, a search for\nan exception handler is started. This search inspects the except\nclauses in turn until one is found that matches the exception. An\nexpression-less except clause, if present, must be last; it matches\nany exception. For an except clause with an expression, that\nexpression is evaluated, and the clause matches the exception if the\nresulting object is "compatible" with the exception. An object is\ncompatible with an exception if it is the class or a base class of the\nexception object, a tuple containing an item compatible with the\nexception, or, in the (deprecated) case of string exceptions, is the\nraised string itself (note that the object identities must match, i.e.\nit must be the same string object, not just a string with the same\nvalue).\n\nIf no except clause matches the exception, the search for an exception\nhandler continues in the surrounding code and on the invocation stack.\n[1]\n\nIf the evaluation of an expression in the header of an except clause\nraises an exception, the original search for a handler is canceled and\na search starts for the new exception in the surrounding code and on\nthe call stack (it is treated as if the entire ``try`` statement\nraised the exception).\n\nWhen a matching except clause is found, the exception is assigned to\nthe target specified in that except clause, if present, and the except\nclause\'s suite is executed. All except clauses must have an\nexecutable block. When the end of this block is reached, execution\ncontinues normally after the entire try statement. (This means that\nif two nested handlers exist for the same exception, and the exception\noccurs in the try clause of the inner handler, the outer handler will\nnot handle the exception.)\n\nBefore an except clause\'s suite is executed, details about the\nexception are assigned to three variables in the ``sys`` module:\n``sys.exc_type`` receives the object identifying the exception;\n``sys.exc_value`` receives the exception\'s parameter;\n``sys.exc_traceback`` receives a traceback object (see section *The\nstandard type hierarchy*) identifying the point in the program where\nthe exception occurred. These details are also available through the\n``sys.exc_info()`` function, which returns a tuple ``(exc_type,\nexc_value, exc_traceback)``. Use of the corresponding variables is\ndeprecated in favor of this function, since their use is unsafe in a\nthreaded program. As of Python 1.5, the variables are restored to\ntheir previous values (before the call) when returning from a function\nthat handled an exception.\n\nThe optional ``else`` clause is executed if and when control flows off\nthe end of the ``try`` clause. [2] Exceptions in the ``else`` clause\nare not handled by the preceding ``except`` clauses.\n\nIf ``finally`` is present, it specifies a \'cleanup\' handler. The\n``try`` clause is executed, including any ``except`` and ``else``\nclauses. If an exception occurs in any of the clauses and is not\nhandled, the exception is temporarily saved. The ``finally`` clause is\nexecuted. If there is a saved exception, it is re-raised at the end\nof the ``finally`` clause. If the ``finally`` clause raises another\nexception or executes a ``return`` or ``break`` statement, the saved\nexception is lost. The exception information is not available to the\nprogram during execution of the ``finally`` clause.\n\nWhen a ``return``, ``break`` or ``continue`` statement is executed in\nthe ``try`` suite of a ``try``...``finally`` statement, the\n``finally`` clause is also executed \'on the way out.\' A ``continue``\nstatement is illegal in the ``finally`` clause. (The reason is a\nproblem with the current implementation --- this restriction may be\nlifted in the future).\n\nAdditional information on exceptions can be found in section\n*Exceptions*, and information on using the ``raise`` statement to\ngenerate exceptions may be found in section *The raise statement*.\n\n\nThe ``with`` statement\n======================\n\nNew in version 2.5.\n\nThe ``with`` statement is used to wrap the execution of a block with\nmethods defined by a context manager (see section *With Statement\nContext Managers*). This allows common\n``try``...``except``...``finally`` usage patterns to be encapsulated\nfor convenient reuse.\n\n with_stmt ::= "with" with_item ("," with_item)* ":" suite\n with_item ::= expression ["as" target]\n\nThe execution of the ``with`` statement with one "item" proceeds as\nfollows:\n\n1. The context expression is evaluated to obtain a context manager.\n\n2. The context manager\'s ``__exit__()`` is loaded for later use.\n\n3. The context manager\'s ``__enter__()`` method is invoked.\n\n4. If a target was included in the ``with`` statement, the return\n value from ``__enter__()`` is assigned to it.\n\n Note: The ``with`` statement guarantees that if the ``__enter__()``\n method returns without an error, then ``__exit__()`` will always\n be called. Thus, if an error occurs during the assignment to the\n target list, it will be treated the same as an error occurring\n within the suite would be. See step 6 below.\n\n5. The suite is executed.\n\n6. The context manager\'s ``__exit__()`` method is invoked. If an\n exception caused the suite to be exited, its type, value, and\n traceback are passed as arguments to ``__exit__()``. Otherwise,\n three ``None`` arguments are supplied.\n\n If the suite was exited due to an exception, and the return value\n from the ``__exit__()`` method was false, the exception is\n reraised. If the return value was true, the exception is\n suppressed, and execution continues with the statement following\n the ``with`` statement.\n\n If the suite was exited for any reason other than an exception, the\n return value from ``__exit__()`` is ignored, and execution proceeds\n at the normal location for the kind of exit that was taken.\n\nWith more than one item, the context managers are processed as if\nmultiple ``with`` statements were nested:\n\n with A() as a, B() as b:\n suite\n\nis equivalent to\n\n with A() as a:\n with B() as b:\n suite\n\nNote: In Python 2.5, the ``with`` statement is only allowed when the\n ``with_statement`` feature has been enabled. It is always enabled\n in Python 2.6.\n\nChanged in version 2.7: Support for multiple context expressions.\n\nSee also:\n\n **PEP 0343** - The "with" statement\n The specification, background, and examples for the Python\n ``with`` statement.\n\n\nFunction definitions\n====================\n\nA function definition defines a user-defined function object (see\nsection *The standard type hierarchy*):\n\n decorated ::= decorators (classdef | funcdef)\n decorators ::= decorator+\n decorator ::= "@" dotted_name ["(" [argument_list [","]] ")"] NEWLINE\n funcdef ::= "def" funcname "(" [parameter_list] ")" ":" suite\n dotted_name ::= identifier ("." identifier)*\n parameter_list ::= (defparameter ",")*\n ( "*" identifier [, "**" identifier]\n | "**" identifier\n | defparameter [","] )\n defparameter ::= parameter ["=" expression]\n sublist ::= parameter ("," parameter)* [","]\n parameter ::= identifier | "(" sublist ")"\n funcname ::= identifier\n\nA function definition is an executable statement. Its execution binds\nthe function name in the current local namespace to a function object\n(a wrapper around the executable code for the function). This\nfunction object contains a reference to the current global namespace\nas the global namespace to be used when the function is called.\n\nThe function definition does not execute the function body; this gets\nexecuted only when the function is called. [3]\n\nA function definition may be wrapped by one or more *decorator*\nexpressions. Decorator expressions are evaluated when the function is\ndefined, in the scope that contains the function definition. The\nresult must be a callable, which is invoked with the function object\nas the only argument. The returned value is bound to the function name\ninstead of the function object. Multiple decorators are applied in\nnested fashion. For example, the following code:\n\n @f1(arg)\n @f2\n def func(): pass\n\nis equivalent to:\n\n def func(): pass\n func = f1(arg)(f2(func))\n\nWhen one or more top-level parameters have the form *parameter* ``=``\n*expression*, the function is said to have "default parameter values."\nFor a parameter with a default value, the corresponding argument may\nbe omitted from a call, in which case the parameter\'s default value is\nsubstituted. If a parameter has a default value, all following\nparameters must also have a default value --- this is a syntactic\nrestriction that is not expressed by the grammar.\n\n**Default parameter values are evaluated when the function definition\nis executed.** This means that the expression is evaluated once, when\nthe function is defined, and that that same "pre-computed" value is\nused for each call. This is especially important to understand when a\ndefault parameter is a mutable object, such as a list or a dictionary:\nif the function modifies the object (e.g. by appending an item to a\nlist), the default value is in effect modified. This is generally not\nwhat was intended. A way around this is to use ``None`` as the\ndefault, and explicitly test for it in the body of the function, e.g.:\n\n def whats_on_the_telly(penguin=None):\n if penguin is None:\n penguin = []\n penguin.append("property of the zoo")\n return penguin\n\nFunction call semantics are described in more detail in section\n*Calls*. A function call always assigns values to all parameters\nmentioned in the parameter list, either from position arguments, from\nkeyword arguments, or from default values. If the form\n"``*identifier``" is present, it is initialized to a tuple receiving\nany excess positional parameters, defaulting to the empty tuple. If\nthe form "``**identifier``" is present, it is initialized to a new\ndictionary receiving any excess keyword arguments, defaulting to a new\nempty dictionary.\n\nIt is also possible to create anonymous functions (functions not bound\nto a name), for immediate use in expressions. This uses lambda forms,\ndescribed in section *Lambdas*. Note that the lambda form is merely a\nshorthand for a simplified function definition; a function defined in\na "``def``" statement can be passed around or assigned to another name\njust like a function defined by a lambda form. The "``def``" form is\nactually more powerful since it allows the execution of multiple\nstatements.\n\n**Programmer\'s note:** Functions are first-class objects. A "``def``"\nform executed inside a function definition defines a local function\nthat can be returned or passed around. Free variables used in the\nnested function can access the local variables of the function\ncontaining the def. See section *Naming and binding* for details.\n\n\nClass definitions\n=================\n\nA class definition defines a class object (see section *The standard\ntype hierarchy*):\n\n classdef ::= "class" classname [inheritance] ":" suite\n inheritance ::= "(" [expression_list] ")"\n classname ::= identifier\n\nA class definition is an executable statement. It first evaluates the\ninheritance list, if present. Each item in the inheritance list\nshould evaluate to a class object or class type which allows\nsubclassing. The class\'s suite is then executed in a new execution\nframe (see section *Naming and binding*), using a newly created local\nnamespace and the original global namespace. (Usually, the suite\ncontains only function definitions.) When the class\'s suite finishes\nexecution, its execution frame is discarded but its local namespace is\nsaved. [4] A class object is then created using the inheritance list\nfor the base classes and the saved local namespace for the attribute\ndictionary. The class name is bound to this class object in the\noriginal local namespace.\n\n**Programmer\'s note:** Variables defined in the class definition are\nclass variables; they are shared by all instances. To create instance\nvariables, they can be set in a method with ``self.name = value``.\nBoth class and instance variables are accessible through the notation\n"``self.name``", and an instance variable hides a class variable with\nthe same name when accessed in this way. Class variables can be used\nas defaults for instance variables, but using mutable values there can\nlead to unexpected results. For *new-style class*es, descriptors can\nbe used to create instance variables with different implementation\ndetails.\n\nClass definitions, like function definitions, may be wrapped by one or\nmore *decorator* expressions. The evaluation rules for the decorator\nexpressions are the same as for functions. The result must be a class\nobject, which is then bound to the class name.\n\n-[ Footnotes ]-\n\n[1] The exception is propagated to the invocation stack only if there\n is no ``finally`` clause that negates the exception.\n\n[2] Currently, control "flows off the end" except in the case of an\n exception or the execution of a ``return``, ``continue``, or\n ``break`` statement.\n\n[3] A string literal appearing as the first statement in the function\n body is transformed into the function\'s ``__doc__`` attribute and\n therefore the function\'s *docstring*.\n\n[4] A string literal appearing as the first statement in the class\n body is transformed into the namespace\'s ``__doc__`` item and\n therefore the class\'s *docstring*.\n', + 'compound': u'\nCompound statements\n*******************\n\nCompound statements contain (groups of) other statements; they affect\nor control the execution of those other statements in some way. In\ngeneral, compound statements span multiple lines, although in simple\nincarnations a whole compound statement may be contained in one line.\n\nThe ``if``, ``while`` and ``for`` statements implement traditional\ncontrol flow constructs. ``try`` specifies exception handlers and/or\ncleanup code for a group of statements. Function and class\ndefinitions are also syntactically compound statements.\n\nCompound statements consist of one or more \'clauses.\' A clause\nconsists of a header and a \'suite.\' The clause headers of a\nparticular compound statement are all at the same indentation level.\nEach clause header begins with a uniquely identifying keyword and ends\nwith a colon. A suite is a group of statements controlled by a\nclause. A suite can be one or more semicolon-separated simple\nstatements on the same line as the header, following the header\'s\ncolon, or it can be one or more indented statements on subsequent\nlines. Only the latter form of suite can contain nested compound\nstatements; the following is illegal, mostly because it wouldn\'t be\nclear to which ``if`` clause a following ``else`` clause would belong:\n\n if test1: if test2: print x\n\nAlso note that the semicolon binds tighter than the colon in this\ncontext, so that in the following example, either all or none of the\n``print`` statements are executed:\n\n if x < y < z: print x; print y; print z\n\nSummarizing:\n\n compound_stmt ::= if_stmt\n | while_stmt\n | for_stmt\n | try_stmt\n | with_stmt\n | funcdef\n | classdef\n | decorated\n suite ::= stmt_list NEWLINE | NEWLINE INDENT statement+ DEDENT\n statement ::= stmt_list NEWLINE | compound_stmt\n stmt_list ::= simple_stmt (";" simple_stmt)* [";"]\n\nNote that statements always end in a ``NEWLINE`` possibly followed by\na ``DEDENT``. Also note that optional continuation clauses always\nbegin with a keyword that cannot start a statement, thus there are no\nambiguities (the \'dangling ``else``\' problem is solved in Python by\nrequiring nested ``if`` statements to be indented).\n\nThe formatting of the grammar rules in the following sections places\neach clause on a separate line for clarity.\n\n\nThe ``if`` statement\n====================\n\nThe ``if`` statement is used for conditional execution:\n\n if_stmt ::= "if" expression ":" suite\n ( "elif" expression ":" suite )*\n ["else" ":" suite]\n\nIt selects exactly one of the suites by evaluating the expressions one\nby one until one is found to be true (see section *Boolean operations*\nfor the definition of true and false); then that suite is executed\n(and no other part of the ``if`` statement is executed or evaluated).\nIf all expressions are false, the suite of the ``else`` clause, if\npresent, is executed.\n\n\nThe ``while`` statement\n=======================\n\nThe ``while`` statement is used for repeated execution as long as an\nexpression is true:\n\n while_stmt ::= "while" expression ":" suite\n ["else" ":" suite]\n\nThis repeatedly tests the expression and, if it is true, executes the\nfirst suite; if the expression is false (which may be the first time\nit is tested) the suite of the ``else`` clause, if present, is\nexecuted and the loop terminates.\n\nA ``break`` statement executed in the first suite terminates the loop\nwithout executing the ``else`` clause\'s suite. A ``continue``\nstatement executed in the first suite skips the rest of the suite and\ngoes back to testing the expression.\n\n\nThe ``for`` statement\n=====================\n\nThe ``for`` statement is used to iterate over the elements of a\nsequence (such as a string, tuple or list) or other iterable object:\n\n for_stmt ::= "for" target_list "in" expression_list ":" suite\n ["else" ":" suite]\n\nThe expression list is evaluated once; it should yield an iterable\nobject. An iterator is created for the result of the\n``expression_list``. The suite is then executed once for each item\nprovided by the iterator, in the order of ascending indices. Each\nitem in turn is assigned to the target list using the standard rules\nfor assignments, and then the suite is executed. When the items are\nexhausted (which is immediately when the sequence is empty), the suite\nin the ``else`` clause, if present, is executed, and the loop\nterminates.\n\nA ``break`` statement executed in the first suite terminates the loop\nwithout executing the ``else`` clause\'s suite. A ``continue``\nstatement executed in the first suite skips the rest of the suite and\ncontinues with the next item, or with the ``else`` clause if there was\nno next item.\n\nThe suite may assign to the variable(s) in the target list; this does\nnot affect the next item assigned to it.\n\nThe target list is not deleted when the loop is finished, but if the\nsequence is empty, it will not have been assigned to at all by the\nloop. Hint: the built-in function ``range()`` returns a sequence of\nintegers suitable to emulate the effect of Pascal\'s ``for i := a to b\ndo``; e.g., ``range(3)`` returns the list ``[0, 1, 2]``.\n\nNote: There is a subtlety when the sequence is being modified by the loop\n (this can only occur for mutable sequences, i.e. lists). An internal\n counter is used to keep track of which item is used next, and this\n is incremented on each iteration. When this counter has reached the\n length of the sequence the loop terminates. This means that if the\n suite deletes the current (or a previous) item from the sequence,\n the next item will be skipped (since it gets the index of the\n current item which has already been treated). Likewise, if the\n suite inserts an item in the sequence before the current item, the\n current item will be treated again the next time through the loop.\n This can lead to nasty bugs that can be avoided by making a\n temporary copy using a slice of the whole sequence, e.g.,\n\n for x in a[:]:\n if x < 0: a.remove(x)\n\n\nThe ``try`` statement\n=====================\n\nThe ``try`` statement specifies exception handlers and/or cleanup code\nfor a group of statements:\n\n try_stmt ::= try1_stmt | try2_stmt\n try1_stmt ::= "try" ":" suite\n ("except" [expression [("as" | ",") target]] ":" suite)+\n ["else" ":" suite]\n ["finally" ":" suite]\n try2_stmt ::= "try" ":" suite\n "finally" ":" suite\n\nChanged in version 2.5: In previous versions of Python,\n``try``...``except``...``finally`` did not work. ``try``...``except``\nhad to be nested in ``try``...``finally``.\n\nThe ``except`` clause(s) specify one or more exception handlers. When\nno exception occurs in the ``try`` clause, no exception handler is\nexecuted. When an exception occurs in the ``try`` suite, a search for\nan exception handler is started. This search inspects the except\nclauses in turn until one is found that matches the exception. An\nexpression-less except clause, if present, must be last; it matches\nany exception. For an except clause with an expression, that\nexpression is evaluated, and the clause matches the exception if the\nresulting object is "compatible" with the exception. An object is\ncompatible with an exception if it is the class or a base class of the\nexception object, a tuple containing an item compatible with the\nexception, or, in the (deprecated) case of string exceptions, is the\nraised string itself (note that the object identities must match, i.e.\nit must be the same string object, not just a string with the same\nvalue).\n\nIf no except clause matches the exception, the search for an exception\nhandler continues in the surrounding code and on the invocation stack.\n[1]\n\nIf the evaluation of an expression in the header of an except clause\nraises an exception, the original search for a handler is canceled and\na search starts for the new exception in the surrounding code and on\nthe call stack (it is treated as if the entire ``try`` statement\nraised the exception).\n\nWhen a matching except clause is found, the exception is assigned to\nthe target specified in that except clause, if present, and the except\nclause\'s suite is executed. All except clauses must have an\nexecutable block. When the end of this block is reached, execution\ncontinues normally after the entire try statement. (This means that\nif two nested handlers exist for the same exception, and the exception\noccurs in the try clause of the inner handler, the outer handler will\nnot handle the exception.)\n\nBefore an except clause\'s suite is executed, details about the\nexception are assigned to three variables in the ``sys`` module:\n``sys.exc_type`` receives the object identifying the exception;\n``sys.exc_value`` receives the exception\'s parameter;\n``sys.exc_traceback`` receives a traceback object (see section *The\nstandard type hierarchy*) identifying the point in the program where\nthe exception occurred. These details are also available through the\n``sys.exc_info()`` function, which returns a tuple ``(exc_type,\nexc_value, exc_traceback)``. Use of the corresponding variables is\ndeprecated in favor of this function, since their use is unsafe in a\nthreaded program. As of Python 1.5, the variables are restored to\ntheir previous values (before the call) when returning from a function\nthat handled an exception.\n\nThe optional ``else`` clause is executed if and when control flows off\nthe end of the ``try`` clause. [2] Exceptions in the ``else`` clause\nare not handled by the preceding ``except`` clauses.\n\nIf ``finally`` is present, it specifies a \'cleanup\' handler. The\n``try`` clause is executed, including any ``except`` and ``else``\nclauses. If an exception occurs in any of the clauses and is not\nhandled, the exception is temporarily saved. The ``finally`` clause is\nexecuted. If there is a saved exception, it is re-raised at the end\nof the ``finally`` clause. If the ``finally`` clause raises another\nexception or executes a ``return`` or ``break`` statement, the saved\nexception is lost. The exception information is not available to the\nprogram during execution of the ``finally`` clause.\n\nWhen a ``return``, ``break`` or ``continue`` statement is executed in\nthe ``try`` suite of a ``try``...``finally`` statement, the\n``finally`` clause is also executed \'on the way out.\' A ``continue``\nstatement is illegal in the ``finally`` clause. (The reason is a\nproblem with the current implementation --- this restriction may be\nlifted in the future).\n\nAdditional information on exceptions can be found in section\n*Exceptions*, and information on using the ``raise`` statement to\ngenerate exceptions may be found in section *The raise statement*.\n\n\nThe ``with`` statement\n======================\n\nNew in version 2.5.\n\nThe ``with`` statement is used to wrap the execution of a block with\nmethods defined by a context manager (see section *With Statement\nContext Managers*). This allows common\n``try``...``except``...``finally`` usage patterns to be encapsulated\nfor convenient reuse.\n\n with_stmt ::= "with" with_item ("," with_item)* ":" suite\n with_item ::= expression ["as" target]\n\nThe execution of the ``with`` statement with one "item" proceeds as\nfollows:\n\n1. The context expression (the expression given in the **with_item**)\n is evaluated to obtain a context manager.\n\n2. The context manager\'s ``__exit__()`` is loaded for later use.\n\n3. The context manager\'s ``__enter__()`` method is invoked.\n\n4. If a target was included in the ``with`` statement, the return\n value from ``__enter__()`` is assigned to it.\n\n Note: The ``with`` statement guarantees that if the ``__enter__()``\n method returns without an error, then ``__exit__()`` will always\n be called. Thus, if an error occurs during the assignment to the\n target list, it will be treated the same as an error occurring\n within the suite would be. See step 6 below.\n\n5. The suite is executed.\n\n6. The context manager\'s ``__exit__()`` method is invoked. If an\n exception caused the suite to be exited, its type, value, and\n traceback are passed as arguments to ``__exit__()``. Otherwise,\n three ``None`` arguments are supplied.\n\n If the suite was exited due to an exception, and the return value\n from the ``__exit__()`` method was false, the exception is\n reraised. If the return value was true, the exception is\n suppressed, and execution continues with the statement following\n the ``with`` statement.\n\n If the suite was exited for any reason other than an exception, the\n return value from ``__exit__()`` is ignored, and execution proceeds\n at the normal location for the kind of exit that was taken.\n\nWith more than one item, the context managers are processed as if\nmultiple ``with`` statements were nested:\n\n with A() as a, B() as b:\n suite\n\nis equivalent to\n\n with A() as a:\n with B() as b:\n suite\n\nNote: In Python 2.5, the ``with`` statement is only allowed when the\n ``with_statement`` feature has been enabled. It is always enabled\n in Python 2.6.\n\nChanged in version 2.7: Support for multiple context expressions.\n\nSee also:\n\n **PEP 0343** - The "with" statement\n The specification, background, and examples for the Python\n ``with`` statement.\n\n\nFunction definitions\n====================\n\nA function definition defines a user-defined function object (see\nsection *The standard type hierarchy*):\n\n decorated ::= decorators (classdef | funcdef)\n decorators ::= decorator+\n decorator ::= "@" dotted_name ["(" [argument_list [","]] ")"] NEWLINE\n funcdef ::= "def" funcname "(" [parameter_list] ")" ":" suite\n dotted_name ::= identifier ("." identifier)*\n parameter_list ::= (defparameter ",")*\n ( "*" identifier [, "**" identifier]\n | "**" identifier\n | defparameter [","] )\n defparameter ::= parameter ["=" expression]\n sublist ::= parameter ("," parameter)* [","]\n parameter ::= identifier | "(" sublist ")"\n funcname ::= identifier\n\nA function definition is an executable statement. Its execution binds\nthe function name in the current local namespace to a function object\n(a wrapper around the executable code for the function). This\nfunction object contains a reference to the current global namespace\nas the global namespace to be used when the function is called.\n\nThe function definition does not execute the function body; this gets\nexecuted only when the function is called. [3]\n\nA function definition may be wrapped by one or more *decorator*\nexpressions. Decorator expressions are evaluated when the function is\ndefined, in the scope that contains the function definition. The\nresult must be a callable, which is invoked with the function object\nas the only argument. The returned value is bound to the function name\ninstead of the function object. Multiple decorators are applied in\nnested fashion. For example, the following code:\n\n @f1(arg)\n @f2\n def func(): pass\n\nis equivalent to:\n\n def func(): pass\n func = f1(arg)(f2(func))\n\nWhen one or more top-level parameters have the form *parameter* ``=``\n*expression*, the function is said to have "default parameter values."\nFor a parameter with a default value, the corresponding argument may\nbe omitted from a call, in which case the parameter\'s default value is\nsubstituted. If a parameter has a default value, all following\nparameters must also have a default value --- this is a syntactic\nrestriction that is not expressed by the grammar.\n\n**Default parameter values are evaluated when the function definition\nis executed.** This means that the expression is evaluated once, when\nthe function is defined, and that that same "pre-computed" value is\nused for each call. This is especially important to understand when a\ndefault parameter is a mutable object, such as a list or a dictionary:\nif the function modifies the object (e.g. by appending an item to a\nlist), the default value is in effect modified. This is generally not\nwhat was intended. A way around this is to use ``None`` as the\ndefault, and explicitly test for it in the body of the function, e.g.:\n\n def whats_on_the_telly(penguin=None):\n if penguin is None:\n penguin = []\n penguin.append("property of the zoo")\n return penguin\n\nFunction call semantics are described in more detail in section\n*Calls*. A function call always assigns values to all parameters\nmentioned in the parameter list, either from position arguments, from\nkeyword arguments, or from default values. If the form\n"``*identifier``" is present, it is initialized to a tuple receiving\nany excess positional parameters, defaulting to the empty tuple. If\nthe form "``**identifier``" is present, it is initialized to a new\ndictionary receiving any excess keyword arguments, defaulting to a new\nempty dictionary.\n\nIt is also possible to create anonymous functions (functions not bound\nto a name), for immediate use in expressions. This uses lambda forms,\ndescribed in section *Lambdas*. Note that the lambda form is merely a\nshorthand for a simplified function definition; a function defined in\na "``def``" statement can be passed around or assigned to another name\njust like a function defined by a lambda form. The "``def``" form is\nactually more powerful since it allows the execution of multiple\nstatements.\n\n**Programmer\'s note:** Functions are first-class objects. A "``def``"\nform executed inside a function definition defines a local function\nthat can be returned or passed around. Free variables used in the\nnested function can access the local variables of the function\ncontaining the def. See section *Naming and binding* for details.\n\n\nClass definitions\n=================\n\nA class definition defines a class object (see section *The standard\ntype hierarchy*):\n\n classdef ::= "class" classname [inheritance] ":" suite\n inheritance ::= "(" [expression_list] ")"\n classname ::= identifier\n\nA class definition is an executable statement. It first evaluates the\ninheritance list, if present. Each item in the inheritance list\nshould evaluate to a class object or class type which allows\nsubclassing. The class\'s suite is then executed in a new execution\nframe (see section *Naming and binding*), using a newly created local\nnamespace and the original global namespace. (Usually, the suite\ncontains only function definitions.) When the class\'s suite finishes\nexecution, its execution frame is discarded but its local namespace is\nsaved. [4] A class object is then created using the inheritance list\nfor the base classes and the saved local namespace for the attribute\ndictionary. The class name is bound to this class object in the\noriginal local namespace.\n\n**Programmer\'s note:** Variables defined in the class definition are\nclass variables; they are shared by all instances. To create instance\nvariables, they can be set in a method with ``self.name = value``.\nBoth class and instance variables are accessible through the notation\n"``self.name``", and an instance variable hides a class variable with\nthe same name when accessed in this way. Class variables can be used\nas defaults for instance variables, but using mutable values there can\nlead to unexpected results. For *new-style class*es, descriptors can\nbe used to create instance variables with different implementation\ndetails.\n\nClass definitions, like function definitions, may be wrapped by one or\nmore *decorator* expressions. The evaluation rules for the decorator\nexpressions are the same as for functions. The result must be a class\nobject, which is then bound to the class name.\n\n-[ Footnotes ]-\n\n[1] The exception is propagated to the invocation stack only if there\n is no ``finally`` clause that negates the exception.\n\n[2] Currently, control "flows off the end" except in the case of an\n exception or the execution of a ``return``, ``continue``, or\n ``break`` statement.\n\n[3] A string literal appearing as the first statement in the function\n body is transformed into the function\'s ``__doc__`` attribute and\n therefore the function\'s *docstring*.\n\n[4] A string literal appearing as the first statement in the class\n body is transformed into the namespace\'s ``__doc__`` item and\n therefore the class\'s *docstring*.\n', 'context-managers': u'\nWith Statement Context Managers\n*******************************\n\nNew in version 2.5.\n\nA *context manager* is an object that defines the runtime context to\nbe established when executing a ``with`` statement. The context\nmanager handles the entry into, and the exit from, the desired runtime\ncontext for the execution of the block of code. Context managers are\nnormally invoked using the ``with`` statement (described in section\n*The with statement*), but can also be used by directly invoking their\nmethods.\n\nTypical uses of context managers include saving and restoring various\nkinds of global state, locking and unlocking resources, closing opened\nfiles, etc.\n\nFor more information on context managers, see *Context Manager Types*.\n\nobject.__enter__(self)\n\n Enter the runtime context related to this object. The ``with``\n statement will bind this method\'s return value to the target(s)\n specified in the ``as`` clause of the statement, if any.\n\nobject.__exit__(self, exc_type, exc_value, traceback)\n\n Exit the runtime context related to this object. The parameters\n describe the exception that caused the context to be exited. If the\n context was exited without an exception, all three arguments will\n be ``None``.\n\n If an exception is supplied, and the method wishes to suppress the\n exception (i.e., prevent it from being propagated), it should\n return a true value. Otherwise, the exception will be processed\n normally upon exit from this method.\n\n Note that ``__exit__()`` methods should not reraise the passed-in\n exception; this is the caller\'s responsibility.\n\nSee also:\n\n **PEP 0343** - The "with" statement\n The specification, background, and examples for the Python\n ``with`` statement.\n', 'continue': u'\nThe ``continue`` statement\n**************************\n\n continue_stmt ::= "continue"\n\n``continue`` may only occur syntactically nested in a ``for`` or\n``while`` loop, but not nested in a function or class definition or\n``finally`` clause within that loop. It continues with the next cycle\nof the nearest enclosing loop.\n\nWhen ``continue`` passes control out of a ``try`` statement with a\n``finally`` clause, that ``finally`` clause is executed before really\nstarting the next loop cycle.\n', 'conversions': u'\nArithmetic conversions\n**********************\n\nWhen a description of an arithmetic operator below uses the phrase\n"the numeric arguments are converted to a common type," the arguments\nare coerced using the coercion rules listed at *Coercion rules*. If\nboth arguments are standard numeric types, the following coercions are\napplied:\n\n* If either argument is a complex number, the other is converted to\n complex;\n\n* otherwise, if either argument is a floating point number, the other\n is converted to floating point;\n\n* otherwise, if either argument is a long integer, the other is\n converted to long integer;\n\n* otherwise, both must be plain integers and no conversion is\n necessary.\n\nSome additional rules apply for certain operators (e.g., a string left\nargument to the \'%\' operator). Extensions can define their own\ncoercions.\n', 'customization': u'\nBasic customization\n*******************\n\nobject.__new__(cls[, ...])\n\n Called to create a new instance of class *cls*. ``__new__()`` is a\n static method (special-cased so you need not declare it as such)\n that takes the class of which an instance was requested as its\n first argument. The remaining arguments are those passed to the\n object constructor expression (the call to the class). The return\n value of ``__new__()`` should be the new object instance (usually\n an instance of *cls*).\n\n Typical implementations create a new instance of the class by\n invoking the superclass\'s ``__new__()`` method using\n ``super(currentclass, cls).__new__(cls[, ...])`` with appropriate\n arguments and then modifying the newly-created instance as\n necessary before returning it.\n\n If ``__new__()`` returns an instance of *cls*, then the new\n instance\'s ``__init__()`` method will be invoked like\n ``__init__(self[, ...])``, where *self* is the new instance and the\n remaining arguments are the same as were passed to ``__new__()``.\n\n If ``__new__()`` does not return an instance of *cls*, then the new\n instance\'s ``__init__()`` method will not be invoked.\n\n ``__new__()`` is intended mainly to allow subclasses of immutable\n types (like int, str, or tuple) to customize instance creation. It\n is also commonly overridden in custom metaclasses in order to\n customize class creation.\n\nobject.__init__(self[, ...])\n\n Called when the instance is created. The arguments are those\n passed to the class constructor expression. If a base class has an\n ``__init__()`` method, the derived class\'s ``__init__()`` method,\n if any, must explicitly call it to ensure proper initialization of\n the base class part of the instance; for example:\n ``BaseClass.__init__(self, [args...])``. As a special constraint\n on constructors, no value may be returned; doing so will cause a\n ``TypeError`` to be raised at runtime.\n\nobject.__del__(self)\n\n Called when the instance is about to be destroyed. This is also\n called a destructor. If a base class has a ``__del__()`` method,\n the derived class\'s ``__del__()`` method, if any, must explicitly\n call it to ensure proper deletion of the base class part of the\n instance. Note that it is possible (though not recommended!) for\n the ``__del__()`` method to postpone destruction of the instance by\n creating a new reference to it. It may then be called at a later\n time when this new reference is deleted. It is not guaranteed that\n ``__del__()`` methods are called for objects that still exist when\n the interpreter exits.\n\n Note: ``del x`` doesn\'t directly call ``x.__del__()`` --- the former\n decrements the reference count for ``x`` by one, and the latter\n is only called when ``x``\'s reference count reaches zero. Some\n common situations that may prevent the reference count of an\n object from going to zero include: circular references between\n objects (e.g., a doubly-linked list or a tree data structure with\n parent and child pointers); a reference to the object on the\n stack frame of a function that caught an exception (the traceback\n stored in ``sys.exc_traceback`` keeps the stack frame alive); or\n a reference to the object on the stack frame that raised an\n unhandled exception in interactive mode (the traceback stored in\n ``sys.last_traceback`` keeps the stack frame alive). The first\n situation can only be remedied by explicitly breaking the cycles;\n the latter two situations can be resolved by storing ``None`` in\n ``sys.exc_traceback`` or ``sys.last_traceback``. Circular\n references which are garbage are detected when the option cycle\n detector is enabled (it\'s on by default), but can only be cleaned\n up if there are no Python-level ``__del__()`` methods involved.\n Refer to the documentation for the ``gc`` module for more\n information about how ``__del__()`` methods are handled by the\n cycle detector, particularly the description of the ``garbage``\n value.\n\n Warning: Due to the precarious circumstances under which ``__del__()``\n methods are invoked, exceptions that occur during their execution\n are ignored, and a warning is printed to ``sys.stderr`` instead.\n Also, when ``__del__()`` is invoked in response to a module being\n deleted (e.g., when execution of the program is done), other\n globals referenced by the ``__del__()`` method may already have\n been deleted or in the process of being torn down (e.g. the\n import machinery shutting down). For this reason, ``__del__()``\n methods should do the absolute minimum needed to maintain\n external invariants. Starting with version 1.5, Python\n guarantees that globals whose name begins with a single\n underscore are deleted from their module before other globals are\n deleted; if no other references to such globals exist, this may\n help in assuring that imported modules are still available at the\n time when the ``__del__()`` method is called.\n\nobject.__repr__(self)\n\n Called by the ``repr()`` built-in function and by string\n conversions (reverse quotes) to compute the "official" string\n representation of an object. If at all possible, this should look\n like a valid Python expression that could be used to recreate an\n object with the same value (given an appropriate environment). If\n this is not possible, a string of the form ``<...some useful\n description...>`` should be returned. The return value must be a\n string object. If a class defines ``__repr__()`` but not\n ``__str__()``, then ``__repr__()`` is also used when an "informal"\n string representation of instances of that class is required.\n\n This is typically used for debugging, so it is important that the\n representation is information-rich and unambiguous.\n\nobject.__str__(self)\n\n Called by the ``str()`` built-in function and by the ``print``\n statement to compute the "informal" string representation of an\n object. This differs from ``__repr__()`` in that it does not have\n to be a valid Python expression: a more convenient or concise\n representation may be used instead. The return value must be a\n string object.\n\nobject.__lt__(self, other)\nobject.__le__(self, other)\nobject.__eq__(self, other)\nobject.__ne__(self, other)\nobject.__gt__(self, other)\nobject.__ge__(self, other)\n\n New in version 2.1.\n\n These are the so-called "rich comparison" methods, and are called\n for comparison operators in preference to ``__cmp__()`` below. The\n correspondence between operator symbols and method names is as\n follows: ``x<y`` calls ``x.__lt__(y)``, ``x<=y`` calls\n ``x.__le__(y)``, ``x==y`` calls ``x.__eq__(y)``, ``x!=y`` and\n ``x<>y`` call ``x.__ne__(y)``, ``x>y`` calls ``x.__gt__(y)``, and\n ``x>=y`` calls ``x.__ge__(y)``.\n\n A rich comparison method may return the singleton\n ``NotImplemented`` if it does not implement the operation for a\n given pair of arguments. By convention, ``False`` and ``True`` are\n returned for a successful comparison. However, these methods can\n return any value, so if the comparison operator is used in a\n Boolean context (e.g., in the condition of an ``if`` statement),\n Python will call ``bool()`` on the value to determine if the result\n is true or false.\n\n There are no implied relationships among the comparison operators.\n The truth of ``x==y`` does not imply that ``x!=y`` is false.\n Accordingly, when defining ``__eq__()``, one should also define\n ``__ne__()`` so that the operators will behave as expected. See\n the paragraph on ``__hash__()`` for some important notes on\n creating *hashable* objects which support custom comparison\n operations and are usable as dictionary keys.\n\n There are no swapped-argument versions of these methods (to be used\n when the left argument does not support the operation but the right\n argument does); rather, ``__lt__()`` and ``__gt__()`` are each\n other\'s reflection, ``__le__()`` and ``__ge__()`` are each other\'s\n reflection, and ``__eq__()`` and ``__ne__()`` are their own\n reflection.\n\n Arguments to rich comparison methods are never coerced.\n\n To automatically generate ordering operations from a single root\n operation, see ``functools.total_ordering()``.\n\nobject.__cmp__(self, other)\n\n Called by comparison operations if rich comparison (see above) is\n not defined. Should return a negative integer if ``self < other``,\n zero if ``self == other``, a positive integer if ``self > other``.\n If no ``__cmp__()``, ``__eq__()`` or ``__ne__()`` operation is\n defined, class instances are compared by object identity\n ("address"). See also the description of ``__hash__()`` for some\n important notes on creating *hashable* objects which support custom\n comparison operations and are usable as dictionary keys. (Note: the\n restriction that exceptions are not propagated by ``__cmp__()`` has\n been removed since Python 1.5.)\n\nobject.__rcmp__(self, other)\n\n Changed in version 2.1: No longer supported.\n\nobject.__hash__(self)\n\n Called by built-in function ``hash()`` and for operations on\n members of hashed collections including ``set``, ``frozenset``, and\n ``dict``. ``__hash__()`` should return an integer. The only\n required property is that objects which compare equal have the same\n hash value; it is advised to somehow mix together (e.g. using\n exclusive or) the hash values for the components of the object that\n also play a part in comparison of objects.\n\n If a class does not define a ``__cmp__()`` or ``__eq__()`` method\n it should not define a ``__hash__()`` operation either; if it\n defines ``__cmp__()`` or ``__eq__()`` but not ``__hash__()``, its\n instances will not be usable in hashed collections. If a class\n defines mutable objects and implements a ``__cmp__()`` or\n ``__eq__()`` method, it should not implement ``__hash__()``, since\n hashable collection implementations require that a object\'s hash\n value is immutable (if the object\'s hash value changes, it will be\n in the wrong hash bucket).\n\n User-defined classes have ``__cmp__()`` and ``__hash__()`` methods\n by default; with them, all objects compare unequal (except with\n themselves) and ``x.__hash__()`` returns ``id(x)``.\n\n Classes which inherit a ``__hash__()`` method from a parent class\n but change the meaning of ``__cmp__()`` or ``__eq__()`` such that\n the hash value returned is no longer appropriate (e.g. by switching\n to a value-based concept of equality instead of the default\n identity based equality) can explicitly flag themselves as being\n unhashable by setting ``__hash__ = None`` in the class definition.\n Doing so means that not only will instances of the class raise an\n appropriate ``TypeError`` when a program attempts to retrieve their\n hash value, but they will also be correctly identified as\n unhashable when checking ``isinstance(obj, collections.Hashable)``\n (unlike classes which define their own ``__hash__()`` to explicitly\n raise ``TypeError``).\n\n Changed in version 2.5: ``__hash__()`` may now also return a long\n integer object; the 32-bit integer is then derived from the hash of\n that object.\n\n Changed in version 2.6: ``__hash__`` may now be set to ``None`` to\n explicitly flag instances of a class as unhashable.\n\nobject.__nonzero__(self)\n\n Called to implement truth value testing and the built-in operation\n ``bool()``; should return ``False`` or ``True``, or their integer\n equivalents ``0`` or ``1``. When this method is not defined,\n ``__len__()`` is called, if it is defined, and the object is\n considered true if its result is nonzero. If a class defines\n neither ``__len__()`` nor ``__nonzero__()``, all its instances are\n considered true.\n\nobject.__unicode__(self)\n\n Called to implement ``unicode()`` built-in; should return a Unicode\n object. When this method is not defined, string conversion is\n attempted, and the result of string conversion is converted to\n Unicode using the system default encoding.\n', - 'debugger': u'\n``pdb`` --- The Python Debugger\n*******************************\n\nThe module ``pdb`` defines an interactive source code debugger for\nPython programs. It supports setting (conditional) breakpoints and\nsingle stepping at the source line level, inspection of stack frames,\nsource code listing, and evaluation of arbitrary Python code in the\ncontext of any stack frame. It also supports post-mortem debugging\nand can be called under program control.\n\nThe debugger is extensible --- it is actually defined as the class\n``Pdb``. This is currently undocumented but easily understood by\nreading the source. The extension interface uses the modules ``bdb``\nand ``cmd``.\n\nThe debugger\'s prompt is ``(Pdb)``. Typical usage to run a program\nunder control of the debugger is:\n\n >>> import pdb\n >>> import mymodule\n >>> pdb.run(\'mymodule.test()\')\n > <string>(0)?()\n (Pdb) continue\n > <string>(1)?()\n (Pdb) continue\n NameError: \'spam\'\n > <string>(1)?()\n (Pdb)\n\n``pdb.py`` can also be invoked as a script to debug other scripts.\nFor example:\n\n python -m pdb myscript.py\n\nWhen invoked as a script, pdb will automatically enter post-mortem\ndebugging if the program being debugged exits abnormally. After post-\nmortem debugging (or after normal exit of the program), pdb will\nrestart the program. Automatic restarting preserves pdb\'s state (such\nas breakpoints) and in most cases is more useful than quitting the\ndebugger upon program\'s exit.\n\nNew in version 2.4: Restarting post-mortem behavior added.\n\nThe typical usage to break into the debugger from a running program is\nto insert\n\n import pdb; pdb.set_trace()\n\nat the location you want to break into the debugger. You can then\nstep through the code following this statement, and continue running\nwithout the debugger using the ``c`` command.\n\nThe typical usage to inspect a crashed program is:\n\n >>> import pdb\n >>> import mymodule\n >>> mymodule.test()\n Traceback (most recent call last):\n File "<stdin>", line 1, in ?\n File "./mymodule.py", line 4, in test\n test2()\n File "./mymodule.py", line 3, in test2\n print spam\n NameError: spam\n >>> pdb.pm()\n > ./mymodule.py(3)test2()\n -> print spam\n (Pdb)\n\nThe module defines the following functions; each enters the debugger\nin a slightly different way:\n\npdb.run(statement[, globals[, locals]])\n\n Execute the *statement* (given as a string) under debugger control.\n The debugger prompt appears before any code is executed; you can\n set breakpoints and type ``continue``, or you can step through the\n statement using ``step`` or ``next`` (all these commands are\n explained below). The optional *globals* and *locals* arguments\n specify the environment in which the code is executed; by default\n the dictionary of the module ``__main__`` is used. (See the\n explanation of the ``exec`` statement or the ``eval()`` built-in\n function.)\n\npdb.runeval(expression[, globals[, locals]])\n\n Evaluate the *expression* (given as a string) under debugger\n control. When ``runeval()`` returns, it returns the value of the\n expression. Otherwise this function is similar to ``run()``.\n\npdb.runcall(function[, argument, ...])\n\n Call the *function* (a function or method object, not a string)\n with the given arguments. When ``runcall()`` returns, it returns\n whatever the function call returned. The debugger prompt appears\n as soon as the function is entered.\n\npdb.set_trace()\n\n Enter the debugger at the calling stack frame. This is useful to\n hard-code a breakpoint at a given point in a program, even if the\n code is not otherwise being debugged (e.g. when an assertion\n fails).\n\npdb.post_mortem([traceback])\n\n Enter post-mortem debugging of the given *traceback* object. If no\n *traceback* is given, it uses the one of the exception that is\n currently being handled (an exception must be being handled if the\n default is to be used).\n\npdb.pm()\n\n Enter post-mortem debugging of the traceback found in\n ``sys.last_traceback``.\n\nThe ``run_*`` functions and ``set_trace()`` are aliases for\ninstantiating the ``Pdb`` class and calling the method of the same\nname. If you want to access further features, you have to do this\nyourself:\n\nclass class pdb.Pdb(completekey=\'tab\', stdin=None, stdout=None, skip=None)\n\n ``Pdb`` is the debugger class.\n\n The *completekey*, *stdin* and *stdout* arguments are passed to the\n underlying ``cmd.Cmd`` class; see the description there.\n\n The *skip* argument, if given, must be an iterable of glob-style\n module name patterns. The debugger will not step into frames that\n originate in a module that matches one of these patterns. [1]\n\n Example call to enable tracing with *skip*:\n\n import pdb; pdb.Pdb(skip=[\'django.*\']).set_trace()\n\n New in version 2.7: The *skip* argument.\n\n run(statement[, globals[, locals]])\n runeval(expression[, globals[, locals]])\n runcall(function[, argument, ...])\n set_trace()\n\n See the documentation for the functions explained above.\n', + 'debugger': u'\n``pdb`` --- The Python Debugger\n*******************************\n\nThe module ``pdb`` defines an interactive source code debugger for\nPython programs. It supports setting (conditional) breakpoints and\nsingle stepping at the source line level, inspection of stack frames,\nsource code listing, and evaluation of arbitrary Python code in the\ncontext of any stack frame. It also supports post-mortem debugging\nand can be called under program control.\n\nThe debugger is extensible --- it is actually defined as the class\n``Pdb``. This is currently undocumented but easily understood by\nreading the source. The extension interface uses the modules ``bdb``\nand ``cmd``.\n\nThe debugger\'s prompt is ``(Pdb)``. Typical usage to run a program\nunder control of the debugger is:\n\n >>> import pdb\n >>> import mymodule\n >>> pdb.run(\'mymodule.test()\')\n > <string>(0)?()\n (Pdb) continue\n > <string>(1)?()\n (Pdb) continue\n NameError: \'spam\'\n > <string>(1)?()\n (Pdb)\n\n``pdb.py`` can also be invoked as a script to debug other scripts.\nFor example:\n\n python -m pdb myscript.py\n\nWhen invoked as a script, pdb will automatically enter post-mortem\ndebugging if the program being debugged exits abnormally. After post-\nmortem debugging (or after normal exit of the program), pdb will\nrestart the program. Automatic restarting preserves pdb\'s state (such\nas breakpoints) and in most cases is more useful than quitting the\ndebugger upon program\'s exit.\n\nNew in version 2.4: Restarting post-mortem behavior added.\n\nThe typical usage to break into the debugger from a running program is\nto insert\n\n import pdb; pdb.set_trace()\n\nat the location you want to break into the debugger. You can then\nstep through the code following this statement, and continue running\nwithout the debugger using the ``c`` command.\n\nThe typical usage to inspect a crashed program is:\n\n >>> import pdb\n >>> import mymodule\n >>> mymodule.test()\n Traceback (most recent call last):\n File "<stdin>", line 1, in ?\n File "./mymodule.py", line 4, in test\n test2()\n File "./mymodule.py", line 3, in test2\n print spam\n NameError: spam\n >>> pdb.pm()\n > ./mymodule.py(3)test2()\n -> print spam\n (Pdb)\n\nThe module defines the following functions; each enters the debugger\nin a slightly different way:\n\npdb.run(statement[, globals[, locals]])\n\n Execute the *statement* (given as a string) under debugger control.\n The debugger prompt appears before any code is executed; you can\n set breakpoints and type ``continue``, or you can step through the\n statement using ``step`` or ``next`` (all these commands are\n explained below). The optional *globals* and *locals* arguments\n specify the environment in which the code is executed; by default\n the dictionary of the module ``__main__`` is used. (See the\n explanation of the ``exec`` statement or the ``eval()`` built-in\n function.)\n\npdb.runeval(expression[, globals[, locals]])\n\n Evaluate the *expression* (given as a string) under debugger\n control. When ``runeval()`` returns, it returns the value of the\n expression. Otherwise this function is similar to ``run()``.\n\npdb.runcall(function[, argument, ...])\n\n Call the *function* (a function or method object, not a string)\n with the given arguments. When ``runcall()`` returns, it returns\n whatever the function call returned. The debugger prompt appears\n as soon as the function is entered.\n\npdb.set_trace()\n\n Enter the debugger at the calling stack frame. This is useful to\n hard-code a breakpoint at a given point in a program, even if the\n code is not otherwise being debugged (e.g. when an assertion\n fails).\n\npdb.post_mortem([traceback])\n\n Enter post-mortem debugging of the given *traceback* object. If no\n *traceback* is given, it uses the one of the exception that is\n currently being handled (an exception must be being handled if the\n default is to be used).\n\npdb.pm()\n\n Enter post-mortem debugging of the traceback found in\n ``sys.last_traceback``.\n\nThe ``run*`` functions and ``set_trace()`` are aliases for\ninstantiating the ``Pdb`` class and calling the method of the same\nname. If you want to access further features, you have to do this\nyourself:\n\nclass class pdb.Pdb(completekey=\'tab\', stdin=None, stdout=None, skip=None)\n\n ``Pdb`` is the debugger class.\n\n The *completekey*, *stdin* and *stdout* arguments are passed to the\n underlying ``cmd.Cmd`` class; see the description there.\n\n The *skip* argument, if given, must be an iterable of glob-style\n module name patterns. The debugger will not step into frames that\n originate in a module that matches one of these patterns. [1]\n\n Example call to enable tracing with *skip*:\n\n import pdb; pdb.Pdb(skip=[\'django.*\']).set_trace()\n\n New in version 2.7: The *skip* argument.\n\n run(statement[, globals[, locals]])\n runeval(expression[, globals[, locals]])\n runcall(function[, argument, ...])\n set_trace()\n\n See the documentation for the functions explained above.\n', 'del': u'\nThe ``del`` statement\n*********************\n\n del_stmt ::= "del" target_list\n\nDeletion is recursively defined very similar to the way assignment is\ndefined. Rather that spelling it out in full details, here are some\nhints.\n\nDeletion of a target list recursively deletes each target, from left\nto right.\n\nDeletion of a name removes the binding of that name from the local or\nglobal namespace, depending on whether the name occurs in a ``global``\nstatement in the same code block. If the name is unbound, a\n``NameError`` exception will be raised.\n\nIt is illegal to delete a name from the local namespace if it occurs\nas a free variable in a nested block.\n\nDeletion of attribute references, subscriptions and slicings is passed\nto the primary object involved; deletion of a slicing is in general\nequivalent to assignment of an empty slice of the right type (but even\nthis is determined by the sliced object).\n', 'dict': u'\nDictionary displays\n*******************\n\nA dictionary display is a possibly empty series of key/datum pairs\nenclosed in curly braces:\n\n dict_display ::= "{" [key_datum_list | dict_comprehension] "}"\n key_datum_list ::= key_datum ("," key_datum)* [","]\n key_datum ::= expression ":" expression\n dict_comprehension ::= expression ":" expression comp_for\n\nA dictionary display yields a new dictionary object.\n\nIf a comma-separated sequence of key/datum pairs is given, they are\nevaluated from left to right to define the entries of the dictionary:\neach key object is used as a key into the dictionary to store the\ncorresponding datum. This means that you can specify the same key\nmultiple times in the key/datum list, and the final dictionary\'s value\nfor that key will be the last one given.\n\nA dict comprehension, in contrast to list and set comprehensions,\nneeds two expressions separated with a colon followed by the usual\n"for" and "if" clauses. When the comprehension is run, the resulting\nkey and value elements are inserted in the new dictionary in the order\nthey are produced.\n\nRestrictions on the types of the key values are listed earlier in\nsection *The standard type hierarchy*. (To summarize, the key type\nshould be *hashable*, which excludes all mutable objects.) Clashes\nbetween duplicate keys are not detected; the last datum (textually\nrightmost in the display) stored for a given key value prevails.\n', 'dynamic-features': u'\nInteraction with dynamic features\n*********************************\n\nThere are several cases where Python statements are illegal when used\nin conjunction with nested scopes that contain free variables.\n\nIf a variable is referenced in an enclosing scope, it is illegal to\ndelete the name. An error will be reported at compile time.\n\nIf the wild card form of import --- ``import *`` --- is used in a\nfunction and the function contains or is a nested block with free\nvariables, the compiler will raise a ``SyntaxError``.\n\nIf ``exec`` is used in a function and the function contains or is a\nnested block with free variables, the compiler will raise a\n``SyntaxError`` unless the exec explicitly specifies the local\nnamespace for the ``exec``. (In other words, ``exec obj`` would be\nillegal, but ``exec obj in ns`` would be legal.)\n\nThe ``eval()``, ``execfile()``, and ``input()`` functions and the\n``exec`` statement do not have access to the full environment for\nresolving names. Names may be resolved in the local and global\nnamespaces of the caller. Free variables are not resolved in the\nnearest enclosing namespace, but in the global namespace. [1] The\n``exec`` statement and the ``eval()`` and ``execfile()`` functions\nhave optional arguments to override the global and local namespace.\nIf only one namespace is specified, it is used for both.\n', 'else': u'\nThe ``if`` statement\n********************\n\nThe ``if`` statement is used for conditional execution:\n\n if_stmt ::= "if" expression ":" suite\n ( "elif" expression ":" suite )*\n ["else" ":" suite]\n\nIt selects exactly one of the suites by evaluating the expressions one\nby one until one is found to be true (see section *Boolean operations*\nfor the definition of true and false); then that suite is executed\n(and no other part of the ``if`` statement is executed or evaluated).\nIf all expressions are false, the suite of the ``else`` clause, if\npresent, is executed.\n', 'exceptions': u'\nExceptions\n**********\n\nExceptions are a means of breaking out of the normal flow of control\nof a code block in order to handle errors or other exceptional\nconditions. An exception is *raised* at the point where the error is\ndetected; it may be *handled* by the surrounding code block or by any\ncode block that directly or indirectly invoked the code block where\nthe error occurred.\n\nThe Python interpreter raises an exception when it detects a run-time\nerror (such as division by zero). A Python program can also\nexplicitly raise an exception with the ``raise`` statement. Exception\nhandlers are specified with the ``try`` ... ``except`` statement. The\n``finally`` clause of such a statement can be used to specify cleanup\ncode which does not handle the exception, but is executed whether an\nexception occurred or not in the preceding code.\n\nPython uses the "termination" model of error handling: an exception\nhandler can find out what happened and continue execution at an outer\nlevel, but it cannot repair the cause of the error and retry the\nfailing operation (except by re-entering the offending piece of code\nfrom the top).\n\nWhen an exception is not handled at all, the interpreter terminates\nexecution of the program, or returns to its interactive main loop. In\neither case, it prints a stack backtrace, except when the exception is\n``SystemExit``.\n\nExceptions are identified by class instances. The ``except`` clause\nis selected depending on the class of the instance: it must reference\nthe class of the instance or a base class thereof. The instance can\nbe received by the handler and can carry additional information about\nthe exceptional condition.\n\nExceptions can also be identified by strings, in which case the\n``except`` clause is selected by object identity. An arbitrary value\ncan be raised along with the identifying string which can be passed to\nthe handler.\n\nNote: Messages to exceptions are not part of the Python API. Their\n contents may change from one version of Python to the next without\n warning and should not be relied on by code which will run under\n multiple versions of the interpreter.\n\nSee also the description of the ``try`` statement in section *The try\nstatement* and ``raise`` statement in section *The raise statement*.\n\n-[ Footnotes ]-\n\n[1] This limitation occurs because the code that is executed by these\n operations is not available at the time the module is compiled.\n', 'exec': u'\nThe ``exec`` statement\n**********************\n\n exec_stmt ::= "exec" or_expr ["in" expression ["," expression]]\n\nThis statement supports dynamic execution of Python code. The first\nexpression should evaluate to either a string, an open file object, or\na code object. If it is a string, the string is parsed as a suite of\nPython statements which is then executed (unless a syntax error\noccurs). [1] If it is an open file, the file is parsed until EOF and\nexecuted. If it is a code object, it is simply executed. In all\ncases, the code that\'s executed is expected to be valid as file input\n(see section *File input*). Be aware that the ``return`` and\n``yield`` statements may not be used outside of function definitions\neven within the context of code passed to the ``exec`` statement.\n\nIn all cases, if the optional parts are omitted, the code is executed\nin the current scope. If only the first expression after ``in`` is\nspecified, it should be a dictionary, which will be used for both the\nglobal and the local variables. If two expressions are given, they\nare used for the global and local variables, respectively. If\nprovided, *locals* can be any mapping object.\n\nChanged in version 2.4: Formerly, *locals* was required to be a\ndictionary.\n\nAs a side effect, an implementation may insert additional keys into\nthe dictionaries given besides those corresponding to variable names\nset by the executed code. For example, the current implementation may\nadd a reference to the dictionary of the built-in module\n``__builtin__`` under the key ``__builtins__`` (!).\n\n**Programmer\'s hints:** dynamic evaluation of expressions is supported\nby the built-in function ``eval()``. The built-in functions\n``globals()`` and ``locals()`` return the current global and local\ndictionary, respectively, which may be useful to pass around for use\nby ``exec``.\n\n-[ Footnotes ]-\n\n[1] Note that the parser only accepts the Unix-style end of line\n convention. If you are reading the code from a file, make sure to\n use universal newline mode to convert Windows or Mac-style\n newlines.\n', - 'execmodel': u'\nExecution model\n***************\n\n\nNaming and binding\n==================\n\n*Names* refer to objects. Names are introduced by name binding\noperations. Each occurrence of a name in the program text refers to\nthe *binding* of that name established in the innermost function block\ncontaining the use.\n\nA *block* is a piece of Python program text that is executed as a\nunit. The following are blocks: a module, a function body, and a class\ndefinition. Each command typed interactively is a block. A script\nfile (a file given as standard input to the interpreter or specified\non the interpreter command line the first argument) is a code block.\nA script command (a command specified on the interpreter command line\nwith the \'**-c**\' option) is a code block. The file read by the\nbuilt-in function ``execfile()`` is a code block. The string argument\npassed to the built-in function ``eval()`` and to the ``exec``\nstatement is a code block. The expression read and evaluated by the\nbuilt-in function ``input()`` is a code block.\n\nA code block is executed in an *execution frame*. A frame contains\nsome administrative information (used for debugging) and determines\nwhere and how execution continues after the code block\'s execution has\ncompleted.\n\nA *scope* defines the visibility of a name within a block. If a local\nvariable is defined in a block, its scope includes that block. If the\ndefinition occurs in a function block, the scope extends to any blocks\ncontained within the defining one, unless a contained block introduces\na different binding for the name. The scope of names defined in a\nclass block is limited to the class block; it does not extend to the\ncode blocks of methods -- this includes generator expressions since\nthey are implemented using a function scope. This means that the\nfollowing will fail:\n\n class A:\n a = 42\n b = list(a + i for i in range(10))\n\nWhen a name is used in a code block, it is resolved using the nearest\nenclosing scope. The set of all such scopes visible to a code block\nis called the block\'s *environment*.\n\nIf a name is bound in a block, it is a local variable of that block.\nIf a name is bound at the module level, it is a global variable. (The\nvariables of the module code block are local and global.) If a\nvariable is used in a code block but not defined there, it is a *free\nvariable*.\n\nWhen a name is not found at all, a ``NameError`` exception is raised.\nIf the name refers to a local variable that has not been bound, a\n``UnboundLocalError`` exception is raised. ``UnboundLocalError`` is a\nsubclass of ``NameError``.\n\nThe following constructs bind names: formal parameters to functions,\n``import`` statements, class and function definitions (these bind the\nclass or function name in the defining block), and targets that are\nidentifiers if occurring in an assignment, ``for`` loop header, in the\nsecond position of an ``except`` clause header or after ``as`` in a\n``with`` statement. The ``import`` statement of the form ``from ...\nimport *`` binds all names defined in the imported module, except\nthose beginning with an underscore. This form may only be used at the\nmodule level.\n\nA target occurring in a ``del`` statement is also considered bound for\nthis purpose (though the actual semantics are to unbind the name). It\nis illegal to unbind a name that is referenced by an enclosing scope;\nthe compiler will report a ``SyntaxError``.\n\nEach assignment or import statement occurs within a block defined by a\nclass or function definition or at the module level (the top-level\ncode block).\n\nIf a name binding operation occurs anywhere within a code block, all\nuses of the name within the block are treated as references to the\ncurrent block. This can lead to errors when a name is used within a\nblock before it is bound. This rule is subtle. Python lacks\ndeclarations and allows name binding operations to occur anywhere\nwithin a code block. The local variables of a code block can be\ndetermined by scanning the entire text of the block for name binding\noperations.\n\nIf the global statement occurs within a block, all uses of the name\nspecified in the statement refer to the binding of that name in the\ntop-level namespace. Names are resolved in the top-level namespace by\nsearching the global namespace, i.e. the namespace of the module\ncontaining the code block, and the builtins namespace, the namespace\nof the module ``__builtin__``. The global namespace is searched\nfirst. If the name is not found there, the builtins namespace is\nsearched. The global statement must precede all uses of the name.\n\nThe builtins namespace associated with the execution of a code block\nis actually found by looking up the name ``__builtins__`` in its\nglobal namespace; this should be a dictionary or a module (in the\nlatter case the module\'s dictionary is used). By default, when in the\n``__main__`` module, ``__builtins__`` is the built-in module\n``__builtin__`` (note: no \'s\'); when in any other module,\n``__builtins__`` is an alias for the dictionary of the ``__builtin__``\nmodule itself. ``__builtins__`` can be set to a user-created\ndictionary to create a weak form of restricted execution.\n\n**CPython implementation detail:** Users should not touch\n``__builtins__``; it is strictly an implementation detail. Users\nwanting to override values in the builtins namespace should ``import``\nthe ``__builtin__`` (no \'s\') module and modify its attributes\nappropriately.\n\nThe namespace for a module is automatically created the first time a\nmodule is imported. The main module for a script is always called\n``__main__``.\n\nThe global statement has the same scope as a name binding operation in\nthe same block. If the nearest enclosing scope for a free variable\ncontains a global statement, the free variable is treated as a global.\n\nA class definition is an executable statement that may use and define\nnames. These references follow the normal rules for name resolution.\nThe namespace of the class definition becomes the attribute dictionary\nof the class. Names defined at the class scope are not visible in\nmethods.\n\n\nInteraction with dynamic features\n---------------------------------\n\nThere are several cases where Python statements are illegal when used\nin conjunction with nested scopes that contain free variables.\n\nIf a variable is referenced in an enclosing scope, it is illegal to\ndelete the name. An error will be reported at compile time.\n\nIf the wild card form of import --- ``import *`` --- is used in a\nfunction and the function contains or is a nested block with free\nvariables, the compiler will raise a ``SyntaxError``.\n\nIf ``exec`` is used in a function and the function contains or is a\nnested block with free variables, the compiler will raise a\n``SyntaxError`` unless the exec explicitly specifies the local\nnamespace for the ``exec``. (In other words, ``exec obj`` would be\nillegal, but ``exec obj in ns`` would be legal.)\n\nThe ``eval()``, ``execfile()``, and ``input()`` functions and the\n``exec`` statement do not have access to the full environment for\nresolving names. Names may be resolved in the local and global\nnamespaces of the caller. Free variables are not resolved in the\nnearest enclosing namespace, but in the global namespace. [1] The\n``exec`` statement and the ``eval()`` and ``execfile()`` functions\nhave optional arguments to override the global and local namespace.\nIf only one namespace is specified, it is used for both.\n\n\nExceptions\n==========\n\nExceptions are a means of breaking out of the normal flow of control\nof a code block in order to handle errors or other exceptional\nconditions. An exception is *raised* at the point where the error is\ndetected; it may be *handled* by the surrounding code block or by any\ncode block that directly or indirectly invoked the code block where\nthe error occurred.\n\nThe Python interpreter raises an exception when it detects a run-time\nerror (such as division by zero). A Python program can also\nexplicitly raise an exception with the ``raise`` statement. Exception\nhandlers are specified with the ``try`` ... ``except`` statement. The\n``finally`` clause of such a statement can be used to specify cleanup\ncode which does not handle the exception, but is executed whether an\nexception occurred or not in the preceding code.\n\nPython uses the "termination" model of error handling: an exception\nhandler can find out what happened and continue execution at an outer\nlevel, but it cannot repair the cause of the error and retry the\nfailing operation (except by re-entering the offending piece of code\nfrom the top).\n\nWhen an exception is not handled at all, the interpreter terminates\nexecution of the program, or returns to its interactive main loop. In\neither case, it prints a stack backtrace, except when the exception is\n``SystemExit``.\n\nExceptions are identified by class instances. The ``except`` clause\nis selected depending on the class of the instance: it must reference\nthe class of the instance or a base class thereof. The instance can\nbe received by the handler and can carry additional information about\nthe exceptional condition.\n\nExceptions can also be identified by strings, in which case the\n``except`` clause is selected by object identity. An arbitrary value\ncan be raised along with the identifying string which can be passed to\nthe handler.\n\nNote: Messages to exceptions are not part of the Python API. Their\n contents may change from one version of Python to the next without\n warning and should not be relied on by code which will run under\n multiple versions of the interpreter.\n\nSee also the description of the ``try`` statement in section *The try\nstatement* and ``raise`` statement in section *The raise statement*.\n\n-[ Footnotes ]-\n\n[1] This limitation occurs because the code that is executed by these\n operations is not available at the time the module is compiled.\n', + 'execmodel': u'\nExecution model\n***************\n\n\nNaming and binding\n==================\n\n*Names* refer to objects. Names are introduced by name binding\noperations. Each occurrence of a name in the program text refers to\nthe *binding* of that name established in the innermost function block\ncontaining the use.\n\nA *block* is a piece of Python program text that is executed as a\nunit. The following are blocks: a module, a function body, and a class\ndefinition. Each command typed interactively is a block. A script\nfile (a file given as standard input to the interpreter or specified\non the interpreter command line the first argument) is a code block.\nA script command (a command specified on the interpreter command line\nwith the \'**-c**\' option) is a code block. The file read by the\nbuilt-in function ``execfile()`` is a code block. The string argument\npassed to the built-in function ``eval()`` and to the ``exec``\nstatement is a code block. The expression read and evaluated by the\nbuilt-in function ``input()`` is a code block.\n\nA code block is executed in an *execution frame*. A frame contains\nsome administrative information (used for debugging) and determines\nwhere and how execution continues after the code block\'s execution has\ncompleted.\n\nA *scope* defines the visibility of a name within a block. If a local\nvariable is defined in a block, its scope includes that block. If the\ndefinition occurs in a function block, the scope extends to any blocks\ncontained within the defining one, unless a contained block introduces\na different binding for the name. The scope of names defined in a\nclass block is limited to the class block; it does not extend to the\ncode blocks of methods -- this includes generator expressions since\nthey are implemented using a function scope. This means that the\nfollowing will fail:\n\n class A:\n a = 42\n b = list(a + i for i in range(10))\n\nWhen a name is used in a code block, it is resolved using the nearest\nenclosing scope. The set of all such scopes visible to a code block\nis called the block\'s *environment*.\n\nIf a name is bound in a block, it is a local variable of that block.\nIf a name is bound at the module level, it is a global variable. (The\nvariables of the module code block are local and global.) If a\nvariable is used in a code block but not defined there, it is a *free\nvariable*.\n\nWhen a name is not found at all, a ``NameError`` exception is raised.\nIf the name refers to a local variable that has not been bound, a\n``UnboundLocalError`` exception is raised. ``UnboundLocalError`` is a\nsubclass of ``NameError``.\n\nThe following constructs bind names: formal parameters to functions,\n``import`` statements, class and function definitions (these bind the\nclass or function name in the defining block), and targets that are\nidentifiers if occurring in an assignment, ``for`` loop header, in the\nsecond position of an ``except`` clause header or after ``as`` in a\n``with`` statement. The ``import`` statement of the form ``from ...\nimport *`` binds all names defined in the imported module, except\nthose beginning with an underscore. This form may only be used at the\nmodule level.\n\nA target occurring in a ``del`` statement is also considered bound for\nthis purpose (though the actual semantics are to unbind the name). It\nis illegal to unbind a name that is referenced by an enclosing scope;\nthe compiler will report a ``SyntaxError``.\n\nEach assignment or import statement occurs within a block defined by a\nclass or function definition or at the module level (the top-level\ncode block).\n\nIf a name binding operation occurs anywhere within a code block, all\nuses of the name within the block are treated as references to the\ncurrent block. This can lead to errors when a name is used within a\nblock before it is bound. This rule is subtle. Python lacks\ndeclarations and allows name binding operations to occur anywhere\nwithin a code block. The local variables of a code block can be\ndetermined by scanning the entire text of the block for name binding\noperations.\n\nIf the global statement occurs within a block, all uses of the name\nspecified in the statement refer to the binding of that name in the\ntop-level namespace. Names are resolved in the top-level namespace by\nsearching the global namespace, i.e. the namespace of the module\ncontaining the code block, and the builtins namespace, the namespace\nof the module ``__builtin__``. The global namespace is searched\nfirst. If the name is not found there, the builtins namespace is\nsearched. The global statement must precede all uses of the name.\n\nThe builtins namespace associated with the execution of a code block\nis actually found by looking up the name ``__builtins__`` in its\nglobal namespace; this should be a dictionary or a module (in the\nlatter case the module\'s dictionary is used). By default, when in the\n``__main__`` module, ``__builtins__`` is the built-in module\n``__builtin__`` (note: no \'s\'); when in any other module,\n``__builtins__`` is an alias for the dictionary of the ``__builtin__``\nmodule itself. ``__builtins__`` can be set to a user-created\ndictionary to create a weak form of restricted execution.\n\n**CPython implementation detail:** Users should not touch\n``__builtins__``; it is strictly an implementation detail. Users\nwanting to override values in the builtins namespace should ``import``\nthe ``__builtin__`` (no \'s\') module and modify its attributes\nappropriately.\n\nThe namespace for a module is automatically created the first time a\nmodule is imported. The main module for a script is always called\n``__main__``.\n\nThe ``global`` statement has the same scope as a name binding\noperation in the same block. If the nearest enclosing scope for a\nfree variable contains a global statement, the free variable is\ntreated as a global.\n\nA class definition is an executable statement that may use and define\nnames. These references follow the normal rules for name resolution.\nThe namespace of the class definition becomes the attribute dictionary\nof the class. Names defined at the class scope are not visible in\nmethods.\n\n\nInteraction with dynamic features\n---------------------------------\n\nThere are several cases where Python statements are illegal when used\nin conjunction with nested scopes that contain free variables.\n\nIf a variable is referenced in an enclosing scope, it is illegal to\ndelete the name. An error will be reported at compile time.\n\nIf the wild card form of import --- ``import *`` --- is used in a\nfunction and the function contains or is a nested block with free\nvariables, the compiler will raise a ``SyntaxError``.\n\nIf ``exec`` is used in a function and the function contains or is a\nnested block with free variables, the compiler will raise a\n``SyntaxError`` unless the exec explicitly specifies the local\nnamespace for the ``exec``. (In other words, ``exec obj`` would be\nillegal, but ``exec obj in ns`` would be legal.)\n\nThe ``eval()``, ``execfile()``, and ``input()`` functions and the\n``exec`` statement do not have access to the full environment for\nresolving names. Names may be resolved in the local and global\nnamespaces of the caller. Free variables are not resolved in the\nnearest enclosing namespace, but in the global namespace. [1] The\n``exec`` statement and the ``eval()`` and ``execfile()`` functions\nhave optional arguments to override the global and local namespace.\nIf only one namespace is specified, it is used for both.\n\n\nExceptions\n==========\n\nExceptions are a means of breaking out of the normal flow of control\nof a code block in order to handle errors or other exceptional\nconditions. An exception is *raised* at the point where the error is\ndetected; it may be *handled* by the surrounding code block or by any\ncode block that directly or indirectly invoked the code block where\nthe error occurred.\n\nThe Python interpreter raises an exception when it detects a run-time\nerror (such as division by zero). A Python program can also\nexplicitly raise an exception with the ``raise`` statement. Exception\nhandlers are specified with the ``try`` ... ``except`` statement. The\n``finally`` clause of such a statement can be used to specify cleanup\ncode which does not handle the exception, but is executed whether an\nexception occurred or not in the preceding code.\n\nPython uses the "termination" model of error handling: an exception\nhandler can find out what happened and continue execution at an outer\nlevel, but it cannot repair the cause of the error and retry the\nfailing operation (except by re-entering the offending piece of code\nfrom the top).\n\nWhen an exception is not handled at all, the interpreter terminates\nexecution of the program, or returns to its interactive main loop. In\neither case, it prints a stack backtrace, except when the exception is\n``SystemExit``.\n\nExceptions are identified by class instances. The ``except`` clause\nis selected depending on the class of the instance: it must reference\nthe class of the instance or a base class thereof. The instance can\nbe received by the handler and can carry additional information about\nthe exceptional condition.\n\nExceptions can also be identified by strings, in which case the\n``except`` clause is selected by object identity. An arbitrary value\ncan be raised along with the identifying string which can be passed to\nthe handler.\n\nNote: Messages to exceptions are not part of the Python API. Their\n contents may change from one version of Python to the next without\n warning and should not be relied on by code which will run under\n multiple versions of the interpreter.\n\nSee also the description of the ``try`` statement in section *The try\nstatement* and ``raise`` statement in section *The raise statement*.\n\n-[ Footnotes ]-\n\n[1] This limitation occurs because the code that is executed by these\n operations is not available at the time the module is compiled.\n', 'exprlists': u'\nExpression lists\n****************\n\n expression_list ::= expression ( "," expression )* [","]\n\nAn expression list containing at least one comma yields a tuple. The\nlength of the tuple is the number of expressions in the list. The\nexpressions are evaluated from left to right.\n\nThe trailing comma is required only to create a single tuple (a.k.a. a\n*singleton*); it is optional in all other cases. A single expression\nwithout a trailing comma doesn\'t create a tuple, but rather yields the\nvalue of that expression. (To create an empty tuple, use an empty pair\nof parentheses: ``()``.)\n', 'floating': u'\nFloating point literals\n***********************\n\nFloating point literals are described by the following lexical\ndefinitions:\n\n floatnumber ::= pointfloat | exponentfloat\n pointfloat ::= [intpart] fraction | intpart "."\n exponentfloat ::= (intpart | pointfloat) exponent\n intpart ::= digit+\n fraction ::= "." digit+\n exponent ::= ("e" | "E") ["+" | "-"] digit+\n\nNote that the integer and exponent parts of floating point numbers can\nlook like octal integers, but are interpreted using radix 10. For\nexample, ``077e010`` is legal, and denotes the same number as\n``77e10``. The allowed range of floating point literals is\nimplementation-dependent. Some examples of floating point literals:\n\n 3.14 10. .001 1e100 3.14e-10 0e0\n\nNote that numeric literals do not include a sign; a phrase like ``-1``\nis actually an expression composed of the unary operator ``-`` and the\nliteral ``1``.\n', 'for': u'\nThe ``for`` statement\n*********************\n\nThe ``for`` statement is used to iterate over the elements of a\nsequence (such as a string, tuple or list) or other iterable object:\n\n for_stmt ::= "for" target_list "in" expression_list ":" suite\n ["else" ":" suite]\n\nThe expression list is evaluated once; it should yield an iterable\nobject. An iterator is created for the result of the\n``expression_list``. The suite is then executed once for each item\nprovided by the iterator, in the order of ascending indices. Each\nitem in turn is assigned to the target list using the standard rules\nfor assignments, and then the suite is executed. When the items are\nexhausted (which is immediately when the sequence is empty), the suite\nin the ``else`` clause, if present, is executed, and the loop\nterminates.\n\nA ``break`` statement executed in the first suite terminates the loop\nwithout executing the ``else`` clause\'s suite. A ``continue``\nstatement executed in the first suite skips the rest of the suite and\ncontinues with the next item, or with the ``else`` clause if there was\nno next item.\n\nThe suite may assign to the variable(s) in the target list; this does\nnot affect the next item assigned to it.\n\nThe target list is not deleted when the loop is finished, but if the\nsequence is empty, it will not have been assigned to at all by the\nloop. Hint: the built-in function ``range()`` returns a sequence of\nintegers suitable to emulate the effect of Pascal\'s ``for i := a to b\ndo``; e.g., ``range(3)`` returns the list ``[0, 1, 2]``.\n\nNote: There is a subtlety when the sequence is being modified by the loop\n (this can only occur for mutable sequences, i.e. lists). An internal\n counter is used to keep track of which item is used next, and this\n is incremented on each iteration. When this counter has reached the\n length of the sequence the loop terminates. This means that if the\n suite deletes the current (or a previous) item from the sequence,\n the next item will be skipped (since it gets the index of the\n current item which has already been treated). Likewise, if the\n suite inserts an item in the sequence before the current item, the\n current item will be treated again the next time through the loop.\n This can lead to nasty bugs that can be avoided by making a\n temporary copy using a slice of the whole sequence, e.g.,\n\n for x in a[:]:\n if x < 0: a.remove(x)\n', - 'formatstrings': u'\nFormat String Syntax\n********************\n\nThe ``str.format()`` method and the ``Formatter`` class share the same\nsyntax for format strings (although in the case of ``Formatter``,\nsubclasses can define their own format string syntax).\n\nFormat strings contain "replacement fields" surrounded by curly braces\n``{}``. Anything that is not contained in braces is considered literal\ntext, which is copied unchanged to the output. If you need to include\na brace character in the literal text, it can be escaped by doubling:\n``{{`` and ``}}``.\n\nThe grammar for a replacement field is as follows:\n\n replacement_field ::= "{" [field_name] ["!" conversion] [":" format_spec] "}"\n field_name ::= arg_name ("." attribute_name | "[" element_index "]")*\n arg_name ::= [identifier | integer]\n attribute_name ::= identifier\n element_index ::= integer | index_string\n index_string ::= <any source character except "]"> +\n conversion ::= "r" | "s"\n format_spec ::= <described in the next section>\n\nIn less formal terms, the replacement field can start with a\n*field_name* that specifies the object whose value is to be formatted\nand inserted into the output instead of the replacement field. The\n*field_name* is optionally followed by a *conversion* field, which is\npreceded by an exclamation point ``\'!\'``, and a *format_spec*, which\nis preceded by a colon ``\':\'``. These specify a non-default format\nfor the replacement value.\n\nSee also the *Format Specification Mini-Language* section.\n\nThe *field_name* itself begins with an *arg_name* that is either\neither a number or a keyword. If it\'s a number, it refers to a\npositional argument, and if it\'s a keyword, it refers to a named\nkeyword argument. If the numerical arg_names in a format string are\n0, 1, 2, ... in sequence, they can all be omitted (not just some) and\nthe numbers 0, 1, 2, ... will be automatically inserted in that order.\nThe *arg_name* can be followed by any number of index or attribute\nexpressions. An expression of the form ``\'.name\'`` selects the named\nattribute using ``getattr()``, while an expression of the form\n``\'[index]\'`` does an index lookup using ``__getitem__()``.\n\nChanged in version 2.7: The positional argument specifiers can be\nomitted, so ``\'{} {}\'`` is equivalent to ``\'{0} {1}\'``.\n\nSome simple format string examples:\n\n "First, thou shalt count to {0}" # References first positional argument\n "Bring me a {}" # Implicitly references the first positional argument\n "From {} to {}" # Same as "From {0} to {1}"\n "My quest is {name}" # References keyword argument \'name\'\n "Weight in tons {0.weight}" # \'weight\' attribute of first positional arg\n "Units destroyed: {players[0]}" # First element of keyword argument \'players\'.\n\nThe *conversion* field causes a type coercion before formatting.\nNormally, the job of formatting a value is done by the\n``__format__()`` method of the value itself. However, in some cases\nit is desirable to force a type to be formatted as a string,\noverriding its own definition of formatting. By converting the value\nto a string before calling ``__format__()``, the normal formatting\nlogic is bypassed.\n\nTwo conversion flags are currently supported: ``\'!s\'`` which calls\n``str()`` on the value, and ``\'!r\'`` which calls ``repr()``.\n\nSome examples:\n\n "Harold\'s a clever {0!s}" # Calls str() on the argument first\n "Bring out the holy {name!r}" # Calls repr() on the argument first\n\nThe *format_spec* field contains a specification of how the value\nshould be presented, including such details as field width, alignment,\npadding, decimal precision and so on. Each value type can define its\nown "formatting mini-language" or interpretation of the *format_spec*.\n\nMost built-in types support a common formatting mini-language, which\nis described in the next section.\n\nA *format_spec* field can also include nested replacement fields\nwithin it. These nested replacement fields can contain only a field\nname; conversion flags and format specifications are not allowed. The\nreplacement fields within the format_spec are substituted before the\n*format_spec* string is interpreted. This allows the formatting of a\nvalue to be dynamically specified.\n\nSee the *Format examples* section for some examples.\n\n\nFormat Specification Mini-Language\n==================================\n\n"Format specifications" are used within replacement fields contained\nwithin a format string to define how individual values are presented\n(see *Format String Syntax*). They can also be passed directly to the\nbuilt-in ``format()`` function. Each formattable type may define how\nthe format specification is to be interpreted.\n\nMost built-in types implement the following options for format\nspecifications, although some of the formatting options are only\nsupported by the numeric types.\n\nA general convention is that an empty format string (``""``) produces\nthe same result as if you had called ``str()`` on the value. A non-\nempty format string typically modifies the result.\n\nThe general form of a *standard format specifier* is:\n\n format_spec ::= [[fill]align][sign][#][0][width][,][.precision][type]\n fill ::= <a character other than \'}\'>\n align ::= "<" | ">" | "=" | "^"\n sign ::= "+" | "-" | " "\n width ::= integer\n precision ::= integer\n type ::= "b" | "c" | "d" | "e" | "E" | "f" | "F" | "g" | "G" | "n" | "o" | "s" | "x" | "X" | "%"\n\nThe *fill* character can be any character other than \'}\' (which\nsignifies the end of the field). The presence of a fill character is\nsignaled by the *next* character, which must be one of the alignment\noptions. If the second character of *format_spec* is not a valid\nalignment option, then it is assumed that both the fill character and\nthe alignment option are absent.\n\nThe meaning of the various alignment options is as follows:\n\n +-----------+------------------------------------------------------------+\n | Option | Meaning |\n +===========+============================================================+\n | ``\'<\'`` | Forces the field to be left-aligned within the available |\n | | space (this is the default). |\n +-----------+------------------------------------------------------------+\n | ``\'>\'`` | Forces the field to be right-aligned within the available |\n | | space. |\n +-----------+------------------------------------------------------------+\n | ``\'=\'`` | Forces the padding to be placed after the sign (if any) |\n | | but before the digits. This is used for printing fields |\n | | in the form \'+000000120\'. This alignment option is only |\n | | valid for numeric types. |\n +-----------+------------------------------------------------------------+\n | ``\'^\'`` | Forces the field to be centered within the available |\n | | space. |\n +-----------+------------------------------------------------------------+\n\nNote that unless a minimum field width is defined, the field width\nwill always be the same size as the data to fill it, so that the\nalignment option has no meaning in this case.\n\nThe *sign* option is only valid for number types, and can be one of\nthe following:\n\n +-----------+------------------------------------------------------------+\n | Option | Meaning |\n +===========+============================================================+\n | ``\'+\'`` | indicates that a sign should be used for both positive as |\n | | well as negative numbers. |\n +-----------+------------------------------------------------------------+\n | ``\'-\'`` | indicates that a sign should be used only for negative |\n | | numbers (this is the default behavior). |\n +-----------+------------------------------------------------------------+\n | space | indicates that a leading space should be used on positive |\n | | numbers, and a minus sign on negative numbers. |\n +-----------+------------------------------------------------------------+\n\nThe ``\'#\'`` option is only valid for integers, and only for binary,\noctal, or hexadecimal output. If present, it specifies that the\noutput will be prefixed by ``\'0b\'``, ``\'0o\'``, or ``\'0x\'``,\nrespectively.\n\nThe ``\',\'`` option signals the use of a comma for a thousands\nseparator. For a locale aware separator, use the ``\'n\'`` integer\npresentation type instead.\n\nChanged in version 2.7: Added the ``\',\'`` option (see also **PEP\n378**).\n\n*width* is a decimal integer defining the minimum field width. If not\nspecified, then the field width will be determined by the content.\n\nIf the *width* field is preceded by a zero (``\'0\'``) character, this\nenables zero-padding. This is equivalent to an *alignment* type of\n``\'=\'`` and a *fill* character of ``\'0\'``.\n\nThe *precision* is a decimal number indicating how many digits should\nbe displayed after the decimal point for a floating point value\nformatted with ``\'f\'`` and ``\'F\'``, or before and after the decimal\npoint for a floating point value formatted with ``\'g\'`` or ``\'G\'``.\nFor non-number types the field indicates the maximum field size - in\nother words, how many characters will be used from the field content.\nThe *precision* is not allowed for integer values.\n\nFinally, the *type* determines how the data should be presented.\n\nThe available string presentation types are:\n\n +-----------+------------------------------------------------------------+\n | Type | Meaning |\n +===========+============================================================+\n | ``\'s\'`` | String format. This is the default type for strings and |\n | | may be omitted. |\n +-----------+------------------------------------------------------------+\n | None | The same as ``\'s\'``. |\n +-----------+------------------------------------------------------------+\n\nThe available integer presentation types are:\n\n +-----------+------------------------------------------------------------+\n | Type | Meaning |\n +===========+============================================================+\n | ``\'b\'`` | Binary format. Outputs the number in base 2. |\n +-----------+------------------------------------------------------------+\n | ``\'c\'`` | Character. Converts the integer to the corresponding |\n | | unicode character before printing. |\n +-----------+------------------------------------------------------------+\n | ``\'d\'`` | Decimal Integer. Outputs the number in base 10. |\n +-----------+------------------------------------------------------------+\n | ``\'o\'`` | Octal format. Outputs the number in base 8. |\n +-----------+------------------------------------------------------------+\n | ``\'x\'`` | Hex format. Outputs the number in base 16, using lower- |\n | | case letters for the digits above 9. |\n +-----------+------------------------------------------------------------+\n | ``\'X\'`` | Hex format. Outputs the number in base 16, using upper- |\n | | case letters for the digits above 9. |\n +-----------+------------------------------------------------------------+\n | ``\'n\'`` | Number. This is the same as ``\'d\'``, except that it uses |\n | | the current locale setting to insert the appropriate |\n | | number separator characters. |\n +-----------+------------------------------------------------------------+\n | None | The same as ``\'d\'``. |\n +-----------+------------------------------------------------------------+\n\nIn addition to the above presentation types, integers can be formatted\nwith the floating point presentation types listed below (except\n``\'n\'`` and None). When doing so, ``float()`` is used to convert the\ninteger to a floating point number before formatting.\n\nThe available presentation types for floating point and decimal values\nare:\n\n +-----------+------------------------------------------------------------+\n | Type | Meaning |\n +===========+============================================================+\n | ``\'e\'`` | Exponent notation. Prints the number in scientific |\n | | notation using the letter \'e\' to indicate the exponent. |\n +-----------+------------------------------------------------------------+\n | ``\'E\'`` | Exponent notation. Same as ``\'e\'`` except it uses an upper |\n | | case \'E\' as the separator character. |\n +-----------+------------------------------------------------------------+\n | ``\'f\'`` | Fixed point. Displays the number as a fixed-point number. |\n +-----------+------------------------------------------------------------+\n | ``\'F\'`` | Fixed point. Same as ``\'f\'``. |\n +-----------+------------------------------------------------------------+\n | ``\'g\'`` | General format. For a given precision ``p >= 1``, this |\n | | rounds the number to ``p`` significant digits and then |\n | | formats the result in either fixed-point format or in |\n | | scientific notation, depending on its magnitude. The |\n | | precise rules are as follows: suppose that the result |\n | | formatted with presentation type ``\'e\'`` and precision |\n | | ``p-1`` would have exponent ``exp``. Then if ``-4 <= exp |\n | | < p``, the number is formatted with presentation type |\n | | ``\'f\'`` and precision ``p-1-exp``. Otherwise, the number |\n | | is formatted with presentation type ``\'e\'`` and precision |\n | | ``p-1``. In both cases insignificant trailing zeros are |\n | | removed from the significand, and the decimal point is |\n | | also removed if there are no remaining digits following |\n | | it. Postive and negative infinity, positive and negative |\n | | zero, and nans, are formatted as ``inf``, ``-inf``, ``0``, |\n | | ``-0`` and ``nan`` respectively, regardless of the |\n | | precision. A precision of ``0`` is treated as equivalent |\n | | to a precision of ``1``. |\n +-----------+------------------------------------------------------------+\n | ``\'G\'`` | General format. Same as ``\'g\'`` except switches to ``\'E\'`` |\n | | if the number gets too large. The representations of |\n | | infinity and NaN are uppercased, too. |\n +-----------+------------------------------------------------------------+\n | ``\'n\'`` | Number. This is the same as ``\'g\'``, except that it uses |\n | | the current locale setting to insert the appropriate |\n | | number separator characters. |\n +-----------+------------------------------------------------------------+\n | ``\'%\'`` | Percentage. Multiplies the number by 100 and displays in |\n | | fixed (``\'f\'``) format, followed by a percent sign. |\n +-----------+------------------------------------------------------------+\n | None | The same as ``\'g\'``. |\n +-----------+------------------------------------------------------------+\n\n\nFormat examples\n===============\n\nThis section contains examples of the new format syntax and comparison\nwith the old ``%``-formatting.\n\nIn most of the cases the syntax is similar to the old\n``%``-formatting, with the addition of the ``{}`` and with ``:`` used\ninstead of ``%``. For example, ``\'%03.2f\'`` can be translated to\n``\'{:03.2f}\'``.\n\nThe new format syntax also supports new and different options, shown\nin the follow examples.\n\nAccessing arguments by position:\n\n >>> \'{0}, {1}, {2}\'.format(\'a\', \'b\', \'c\')\n \'a, b, c\'\n >>> \'{}, {}, {}\'.format(\'a\', \'b\', \'c\') # 2.7+ only\n \'a, b, c\'\n >>> \'{2}, {1}, {0}\'.format(\'a\', \'b\', \'c\')\n \'c, b, a\'\n >>> \'{2}, {1}, {0}\'.format(*\'abc\') # unpacking argument sequence\n \'c, b, a\'\n >>> \'{0}{1}{0}\'.format(\'abra\', \'cad\') # arguments\' indices can be repeated\n \'abracadabra\'\n\nAccessing arguments by name:\n\n >>> \'Coordinates: {latitude}, {longitude}\'.format(latitude=\'37.24N\', longitude=\'-115.81W\')\n \'Coordinates: 37.24N, -115.81W\'\n >>> coord = {\'latitude\': \'37.24N\', \'longitude\': \'-115.81W\'}\n >>> \'Coordinates: {latitude}, {longitude}\'.format(**coord)\n \'Coordinates: 37.24N, -115.81W\'\n\nAccessing arguments\' attributes:\n\n >>> c = 3-5j\n >>> (\'The complex number {0} is formed from the real part {0.real} \'\n ... \'and the imaginary part {0.imag}.\').format(c)\n \'The complex number (3-5j) is formed from the real part 3.0 and the imaginary part -5.0.\'\n >>> class Point(object):\n ... def __init__(self, x, y):\n ... self.x, self.y = x, y\n ... def __str__(self):\n ... return \'Point({self.x}, {self.y})\'.format(self=self)\n ...\n >>> str(Point(4, 2))\n \'Point(4, 2)\'\n\nAccessing arguments\' items:\n\n >>> coord = (3, 5)\n >>> \'X: {0[0]}; Y: {0[1]}\'.format(coord)\n \'X: 3; Y: 5\'\n\nReplacing ``%s`` and ``%r``:\n\n >>> "repr() shows quotes: {!r}; str() doesn\'t: {!s}".format(\'test1\', \'test2\')\n "repr() shows quotes: \'test1\'; str() doesn\'t: test2"\n\nAligning the text and specifying a width:\n\n >>> \'{:<30}\'.format(\'left aligned\')\n \'left aligned \'\n >>> \'{:>30}\'.format(\'right aligned\')\n \' right aligned\'\n >>> \'{:^30}\'.format(\'centered\')\n \' centered \'\n >>> \'{:*^30}\'.format(\'centered\') # use \'*\' as a fill char\n \'***********centered***********\'\n\nReplacing ``%+f``, ``%-f``, and ``% f`` and specifying a sign:\n\n >>> \'{:+f}; {:+f}\'.format(3.14, -3.14) # show it always\n \'+3.140000; -3.140000\'\n >>> \'{: f}; {: f}\'.format(3.14, -3.14) # show a space for positive numbers\n \' 3.140000; -3.140000\'\n >>> \'{:-f}; {:-f}\'.format(3.14, -3.14) # show only the minus -- same as \'{:f}; {:f}\'\n \'3.140000; -3.140000\'\n\nReplacing ``%x`` and ``%o`` and converting the value to different\nbases:\n\n >>> # format also supports binary numbers\n >>> "int: {0:d}; hex: {0:x}; oct: {0:o}; bin: {0:b}".format(42)\n \'int: 42; hex: 2a; oct: 52; bin: 101010\'\n >>> # with 0x, 0o, or 0b as prefix:\n >>> "int: {0:d}; hex: {0:#x}; oct: {0:#o}; bin: {0:#b}".format(42)\n \'int: 42; hex: 0x2a; oct: 0o52; bin: 0b101010\'\n\nUsing the comma as a thousands separator:\n\n >>> \'{:,}\'.format(1234567890)\n \'1,234,567,890\'\n\nExpressing a percentage:\n\n >>> points = 19.5\n >>> total = 22\n >>> \'Correct answers: {:.2%}.\'.format(points/total)\n \'Correct answers: 88.64%\'\n\nUsing type-specific formatting:\n\n >>> import datetime\n >>> d = datetime.datetime(2010, 7, 4, 12, 15, 58)\n >>> \'{:%Y-%m-%d %H:%M:%S}\'.format(d)\n \'2010-07-04 12:15:58\'\n\nNesting arguments and more complex examples:\n\n >>> for align, text in zip(\'<^>\', [\'left\', \'center\', \'right\']):\n ... \'{0:{align}{fill}16}\'.format(text, fill=align, align=align)\n ...\n \'left<<<<<<<<<<<<\'\n \'^^^^^center^^^^^\'\n \'>>>>>>>>>>>right\'\n >>>\n >>> octets = [192, 168, 0, 1]\n >>> \'{:02X}{:02X}{:02X}{:02X}\'.format(*octets)\n \'C0A80001\'\n >>> int(_, 16)\n 3232235521\n >>>\n >>> width = 5\n >>> for num in range(5,12):\n ... for base in \'dXob\':\n ... print \'{0:{width}{base}}\'.format(num, base=base, width=width),\n ... print\n ...\n 5 5 5 101\n 6 6 6 110\n 7 7 7 111\n 8 8 10 1000\n 9 9 11 1001\n 10 A 12 1010\n 11 B 13 1011\n', + 'formatstrings': u'\nFormat String Syntax\n********************\n\nThe ``str.format()`` method and the ``Formatter`` class share the same\nsyntax for format strings (although in the case of ``Formatter``,\nsubclasses can define their own format string syntax).\n\nFormat strings contain "replacement fields" surrounded by curly braces\n``{}``. Anything that is not contained in braces is considered literal\ntext, which is copied unchanged to the output. If you need to include\na brace character in the literal text, it can be escaped by doubling:\n``{{`` and ``}}``.\n\nThe grammar for a replacement field is as follows:\n\n replacement_field ::= "{" [field_name] ["!" conversion] [":" format_spec] "}"\n field_name ::= arg_name ("." attribute_name | "[" element_index "]")*\n arg_name ::= [identifier | integer]\n attribute_name ::= identifier\n element_index ::= integer | index_string\n index_string ::= <any source character except "]"> +\n conversion ::= "r" | "s"\n format_spec ::= <described in the next section>\n\nIn less formal terms, the replacement field can start with a\n*field_name* that specifies the object whose value is to be formatted\nand inserted into the output instead of the replacement field. The\n*field_name* is optionally followed by a *conversion* field, which is\npreceded by an exclamation point ``\'!\'``, and a *format_spec*, which\nis preceded by a colon ``\':\'``. These specify a non-default format\nfor the replacement value.\n\nSee also the *Format Specification Mini-Language* section.\n\nThe *field_name* itself begins with an *arg_name* that is either\neither a number or a keyword. If it\'s a number, it refers to a\npositional argument, and if it\'s a keyword, it refers to a named\nkeyword argument. If the numerical arg_names in a format string are\n0, 1, 2, ... in sequence, they can all be omitted (not just some) and\nthe numbers 0, 1, 2, ... will be automatically inserted in that order.\nThe *arg_name* can be followed by any number of index or attribute\nexpressions. An expression of the form ``\'.name\'`` selects the named\nattribute using ``getattr()``, while an expression of the form\n``\'[index]\'`` does an index lookup using ``__getitem__()``.\n\nChanged in version 2.7: The positional argument specifiers can be\nomitted, so ``\'{} {}\'`` is equivalent to ``\'{0} {1}\'``.\n\nSome simple format string examples:\n\n "First, thou shalt count to {0}" # References first positional argument\n "Bring me a {}" # Implicitly references the first positional argument\n "From {} to {}" # Same as "From {0} to {1}"\n "My quest is {name}" # References keyword argument \'name\'\n "Weight in tons {0.weight}" # \'weight\' attribute of first positional arg\n "Units destroyed: {players[0]}" # First element of keyword argument \'players\'.\n\nThe *conversion* field causes a type coercion before formatting.\nNormally, the job of formatting a value is done by the\n``__format__()`` method of the value itself. However, in some cases\nit is desirable to force a type to be formatted as a string,\noverriding its own definition of formatting. By converting the value\nto a string before calling ``__format__()``, the normal formatting\nlogic is bypassed.\n\nTwo conversion flags are currently supported: ``\'!s\'`` which calls\n``str()`` on the value, and ``\'!r\'`` which calls ``repr()``.\n\nSome examples:\n\n "Harold\'s a clever {0!s}" # Calls str() on the argument first\n "Bring out the holy {name!r}" # Calls repr() on the argument first\n\nThe *format_spec* field contains a specification of how the value\nshould be presented, including such details as field width, alignment,\npadding, decimal precision and so on. Each value type can define its\nown "formatting mini-language" or interpretation of the *format_spec*.\n\nMost built-in types support a common formatting mini-language, which\nis described in the next section.\n\nA *format_spec* field can also include nested replacement fields\nwithin it. These nested replacement fields can contain only a field\nname; conversion flags and format specifications are not allowed. The\nreplacement fields within the format_spec are substituted before the\n*format_spec* string is interpreted. This allows the formatting of a\nvalue to be dynamically specified.\n\nSee the *Format examples* section for some examples.\n\n\nFormat Specification Mini-Language\n==================================\n\n"Format specifications" are used within replacement fields contained\nwithin a format string to define how individual values are presented\n(see *Format String Syntax*). They can also be passed directly to the\nbuilt-in ``format()`` function. Each formattable type may define how\nthe format specification is to be interpreted.\n\nMost built-in types implement the following options for format\nspecifications, although some of the formatting options are only\nsupported by the numeric types.\n\nA general convention is that an empty format string (``""``) produces\nthe same result as if you had called ``str()`` on the value. A non-\nempty format string typically modifies the result.\n\nThe general form of a *standard format specifier* is:\n\n format_spec ::= [[fill]align][sign][#][0][width][,][.precision][type]\n fill ::= <a character other than \'}\'>\n align ::= "<" | ">" | "=" | "^"\n sign ::= "+" | "-" | " "\n width ::= integer\n precision ::= integer\n type ::= "b" | "c" | "d" | "e" | "E" | "f" | "F" | "g" | "G" | "n" | "o" | "s" | "x" | "X" | "%"\n\nThe *fill* character can be any character other than \'{\' or \'}\'. The\npresence of a fill character is signaled by the character following\nit, which must be one of the alignment options. If the second\ncharacter of *format_spec* is not a valid alignment option, then it is\nassumed that both the fill character and the alignment option are\nabsent.\n\nThe meaning of the various alignment options is as follows:\n\n +-----------+------------------------------------------------------------+\n | Option | Meaning |\n +===========+============================================================+\n | ``\'<\'`` | Forces the field to be left-aligned within the available |\n | | space (this is the default for most objects). |\n +-----------+------------------------------------------------------------+\n | ``\'>\'`` | Forces the field to be right-aligned within the available |\n | | space (this is the default for numbers). |\n +-----------+------------------------------------------------------------+\n | ``\'=\'`` | Forces the padding to be placed after the sign (if any) |\n | | but before the digits. This is used for printing fields |\n | | in the form \'+000000120\'. This alignment option is only |\n | | valid for numeric types. |\n +-----------+------------------------------------------------------------+\n | ``\'^\'`` | Forces the field to be centered within the available |\n | | space. |\n +-----------+------------------------------------------------------------+\n\nNote that unless a minimum field width is defined, the field width\nwill always be the same size as the data to fill it, so that the\nalignment option has no meaning in this case.\n\nThe *sign* option is only valid for number types, and can be one of\nthe following:\n\n +-----------+------------------------------------------------------------+\n | Option | Meaning |\n +===========+============================================================+\n | ``\'+\'`` | indicates that a sign should be used for both positive as |\n | | well as negative numbers. |\n +-----------+------------------------------------------------------------+\n | ``\'-\'`` | indicates that a sign should be used only for negative |\n | | numbers (this is the default behavior). |\n +-----------+------------------------------------------------------------+\n | space | indicates that a leading space should be used on positive |\n | | numbers, and a minus sign on negative numbers. |\n +-----------+------------------------------------------------------------+\n\nThe ``\'#\'`` option is only valid for integers, and only for binary,\noctal, or hexadecimal output. If present, it specifies that the\noutput will be prefixed by ``\'0b\'``, ``\'0o\'``, or ``\'0x\'``,\nrespectively.\n\nThe ``\',\'`` option signals the use of a comma for a thousands\nseparator. For a locale aware separator, use the ``\'n\'`` integer\npresentation type instead.\n\nChanged in version 2.7: Added the ``\',\'`` option (see also **PEP\n378**).\n\n*width* is a decimal integer defining the minimum field width. If not\nspecified, then the field width will be determined by the content.\n\nIf the *width* field is preceded by a zero (``\'0\'``) character, this\nenables zero-padding. This is equivalent to an *alignment* type of\n``\'=\'`` and a *fill* character of ``\'0\'``.\n\nThe *precision* is a decimal number indicating how many digits should\nbe displayed after the decimal point for a floating point value\nformatted with ``\'f\'`` and ``\'F\'``, or before and after the decimal\npoint for a floating point value formatted with ``\'g\'`` or ``\'G\'``.\nFor non-number types the field indicates the maximum field size - in\nother words, how many characters will be used from the field content.\nThe *precision* is not allowed for integer values.\n\nFinally, the *type* determines how the data should be presented.\n\nThe available string presentation types are:\n\n +-----------+------------------------------------------------------------+\n | Type | Meaning |\n +===========+============================================================+\n | ``\'s\'`` | String format. This is the default type for strings and |\n | | may be omitted. |\n +-----------+------------------------------------------------------------+\n | None | The same as ``\'s\'``. |\n +-----------+------------------------------------------------------------+\n\nThe available integer presentation types are:\n\n +-----------+------------------------------------------------------------+\n | Type | Meaning |\n +===========+============================================================+\n | ``\'b\'`` | Binary format. Outputs the number in base 2. |\n +-----------+------------------------------------------------------------+\n | ``\'c\'`` | Character. Converts the integer to the corresponding |\n | | unicode character before printing. |\n +-----------+------------------------------------------------------------+\n | ``\'d\'`` | Decimal Integer. Outputs the number in base 10. |\n +-----------+------------------------------------------------------------+\n | ``\'o\'`` | Octal format. Outputs the number in base 8. |\n +-----------+------------------------------------------------------------+\n | ``\'x\'`` | Hex format. Outputs the number in base 16, using lower- |\n | | case letters for the digits above 9. |\n +-----------+------------------------------------------------------------+\n | ``\'X\'`` | Hex format. Outputs the number in base 16, using upper- |\n | | case letters for the digits above 9. |\n +-----------+------------------------------------------------------------+\n | ``\'n\'`` | Number. This is the same as ``\'d\'``, except that it uses |\n | | the current locale setting to insert the appropriate |\n | | number separator characters. |\n +-----------+------------------------------------------------------------+\n | None | The same as ``\'d\'``. |\n +-----------+------------------------------------------------------------+\n\nIn addition to the above presentation types, integers can be formatted\nwith the floating point presentation types listed below (except\n``\'n\'`` and None). When doing so, ``float()`` is used to convert the\ninteger to a floating point number before formatting.\n\nThe available presentation types for floating point and decimal values\nare:\n\n +-----------+------------------------------------------------------------+\n | Type | Meaning |\n +===========+============================================================+\n | ``\'e\'`` | Exponent notation. Prints the number in scientific |\n | | notation using the letter \'e\' to indicate the exponent. |\n +-----------+------------------------------------------------------------+\n | ``\'E\'`` | Exponent notation. Same as ``\'e\'`` except it uses an upper |\n | | case \'E\' as the separator character. |\n +-----------+------------------------------------------------------------+\n | ``\'f\'`` | Fixed point. Displays the number as a fixed-point number. |\n +-----------+------------------------------------------------------------+\n | ``\'F\'`` | Fixed point. Same as ``\'f\'``. |\n +-----------+------------------------------------------------------------+\n | ``\'g\'`` | General format. For a given precision ``p >= 1``, this |\n | | rounds the number to ``p`` significant digits and then |\n | | formats the result in either fixed-point format or in |\n | | scientific notation, depending on its magnitude. The |\n | | precise rules are as follows: suppose that the result |\n | | formatted with presentation type ``\'e\'`` and precision |\n | | ``p-1`` would have exponent ``exp``. Then if ``-4 <= exp |\n | | < p``, the number is formatted with presentation type |\n | | ``\'f\'`` and precision ``p-1-exp``. Otherwise, the number |\n | | is formatted with presentation type ``\'e\'`` and precision |\n | | ``p-1``. In both cases insignificant trailing zeros are |\n | | removed from the significand, and the decimal point is |\n | | also removed if there are no remaining digits following |\n | | it. Positive and negative infinity, positive and negative |\n | | zero, and nans, are formatted as ``inf``, ``-inf``, ``0``, |\n | | ``-0`` and ``nan`` respectively, regardless of the |\n | | precision. A precision of ``0`` is treated as equivalent |\n | | to a precision of ``1``. |\n +-----------+------------------------------------------------------------+\n | ``\'G\'`` | General format. Same as ``\'g\'`` except switches to ``\'E\'`` |\n | | if the number gets too large. The representations of |\n | | infinity and NaN are uppercased, too. |\n +-----------+------------------------------------------------------------+\n | ``\'n\'`` | Number. This is the same as ``\'g\'``, except that it uses |\n | | the current locale setting to insert the appropriate |\n | | number separator characters. |\n +-----------+------------------------------------------------------------+\n | ``\'%\'`` | Percentage. Multiplies the number by 100 and displays in |\n | | fixed (``\'f\'``) format, followed by a percent sign. |\n +-----------+------------------------------------------------------------+\n | None | The same as ``\'g\'``. |\n +-----------+------------------------------------------------------------+\n\n\nFormat examples\n===============\n\nThis section contains examples of the new format syntax and comparison\nwith the old ``%``-formatting.\n\nIn most of the cases the syntax is similar to the old\n``%``-formatting, with the addition of the ``{}`` and with ``:`` used\ninstead of ``%``. For example, ``\'%03.2f\'`` can be translated to\n``\'{:03.2f}\'``.\n\nThe new format syntax also supports new and different options, shown\nin the follow examples.\n\nAccessing arguments by position:\n\n >>> \'{0}, {1}, {2}\'.format(\'a\', \'b\', \'c\')\n \'a, b, c\'\n >>> \'{}, {}, {}\'.format(\'a\', \'b\', \'c\') # 2.7+ only\n \'a, b, c\'\n >>> \'{2}, {1}, {0}\'.format(\'a\', \'b\', \'c\')\n \'c, b, a\'\n >>> \'{2}, {1}, {0}\'.format(*\'abc\') # unpacking argument sequence\n \'c, b, a\'\n >>> \'{0}{1}{0}\'.format(\'abra\', \'cad\') # arguments\' indices can be repeated\n \'abracadabra\'\n\nAccessing arguments by name:\n\n >>> \'Coordinates: {latitude}, {longitude}\'.format(latitude=\'37.24N\', longitude=\'-115.81W\')\n \'Coordinates: 37.24N, -115.81W\'\n >>> coord = {\'latitude\': \'37.24N\', \'longitude\': \'-115.81W\'}\n >>> \'Coordinates: {latitude}, {longitude}\'.format(**coord)\n \'Coordinates: 37.24N, -115.81W\'\n\nAccessing arguments\' attributes:\n\n >>> c = 3-5j\n >>> (\'The complex number {0} is formed from the real part {0.real} \'\n ... \'and the imaginary part {0.imag}.\').format(c)\n \'The complex number (3-5j) is formed from the real part 3.0 and the imaginary part -5.0.\'\n >>> class Point(object):\n ... def __init__(self, x, y):\n ... self.x, self.y = x, y\n ... def __str__(self):\n ... return \'Point({self.x}, {self.y})\'.format(self=self)\n ...\n >>> str(Point(4, 2))\n \'Point(4, 2)\'\n\nAccessing arguments\' items:\n\n >>> coord = (3, 5)\n >>> \'X: {0[0]}; Y: {0[1]}\'.format(coord)\n \'X: 3; Y: 5\'\n\nReplacing ``%s`` and ``%r``:\n\n >>> "repr() shows quotes: {!r}; str() doesn\'t: {!s}".format(\'test1\', \'test2\')\n "repr() shows quotes: \'test1\'; str() doesn\'t: test2"\n\nAligning the text and specifying a width:\n\n >>> \'{:<30}\'.format(\'left aligned\')\n \'left aligned \'\n >>> \'{:>30}\'.format(\'right aligned\')\n \' right aligned\'\n >>> \'{:^30}\'.format(\'centered\')\n \' centered \'\n >>> \'{:*^30}\'.format(\'centered\') # use \'*\' as a fill char\n \'***********centered***********\'\n\nReplacing ``%+f``, ``%-f``, and ``% f`` and specifying a sign:\n\n >>> \'{:+f}; {:+f}\'.format(3.14, -3.14) # show it always\n \'+3.140000; -3.140000\'\n >>> \'{: f}; {: f}\'.format(3.14, -3.14) # show a space for positive numbers\n \' 3.140000; -3.140000\'\n >>> \'{:-f}; {:-f}\'.format(3.14, -3.14) # show only the minus -- same as \'{:f}; {:f}\'\n \'3.140000; -3.140000\'\n\nReplacing ``%x`` and ``%o`` and converting the value to different\nbases:\n\n >>> # format also supports binary numbers\n >>> "int: {0:d}; hex: {0:x}; oct: {0:o}; bin: {0:b}".format(42)\n \'int: 42; hex: 2a; oct: 52; bin: 101010\'\n >>> # with 0x, 0o, or 0b as prefix:\n >>> "int: {0:d}; hex: {0:#x}; oct: {0:#o}; bin: {0:#b}".format(42)\n \'int: 42; hex: 0x2a; oct: 0o52; bin: 0b101010\'\n\nUsing the comma as a thousands separator:\n\n >>> \'{:,}\'.format(1234567890)\n \'1,234,567,890\'\n\nExpressing a percentage:\n\n >>> points = 19.5\n >>> total = 22\n >>> \'Correct answers: {:.2%}.\'.format(points/total)\n \'Correct answers: 88.64%\'\n\nUsing type-specific formatting:\n\n >>> import datetime\n >>> d = datetime.datetime(2010, 7, 4, 12, 15, 58)\n >>> \'{:%Y-%m-%d %H:%M:%S}\'.format(d)\n \'2010-07-04 12:15:58\'\n\nNesting arguments and more complex examples:\n\n >>> for align, text in zip(\'<^>\', [\'left\', \'center\', \'right\']):\n ... \'{0:{fill}{align}16}\'.format(text, fill=align, align=align)\n ...\n \'left<<<<<<<<<<<<\'\n \'^^^^^center^^^^^\'\n \'>>>>>>>>>>>right\'\n >>>\n >>> octets = [192, 168, 0, 1]\n >>> \'{:02X}{:02X}{:02X}{:02X}\'.format(*octets)\n \'C0A80001\'\n >>> int(_, 16)\n 3232235521\n >>>\n >>> width = 5\n >>> for num in range(5,12):\n ... for base in \'dXob\':\n ... print \'{0:{width}{base}}\'.format(num, base=base, width=width),\n ... print\n ...\n 5 5 5 101\n 6 6 6 110\n 7 7 7 111\n 8 8 10 1000\n 9 9 11 1001\n 10 A 12 1010\n 11 B 13 1011\n', 'function': u'\nFunction definitions\n********************\n\nA function definition defines a user-defined function object (see\nsection *The standard type hierarchy*):\n\n decorated ::= decorators (classdef | funcdef)\n decorators ::= decorator+\n decorator ::= "@" dotted_name ["(" [argument_list [","]] ")"] NEWLINE\n funcdef ::= "def" funcname "(" [parameter_list] ")" ":" suite\n dotted_name ::= identifier ("." identifier)*\n parameter_list ::= (defparameter ",")*\n ( "*" identifier [, "**" identifier]\n | "**" identifier\n | defparameter [","] )\n defparameter ::= parameter ["=" expression]\n sublist ::= parameter ("," parameter)* [","]\n parameter ::= identifier | "(" sublist ")"\n funcname ::= identifier\n\nA function definition is an executable statement. Its execution binds\nthe function name in the current local namespace to a function object\n(a wrapper around the executable code for the function). This\nfunction object contains a reference to the current global namespace\nas the global namespace to be used when the function is called.\n\nThe function definition does not execute the function body; this gets\nexecuted only when the function is called. [3]\n\nA function definition may be wrapped by one or more *decorator*\nexpressions. Decorator expressions are evaluated when the function is\ndefined, in the scope that contains the function definition. The\nresult must be a callable, which is invoked with the function object\nas the only argument. The returned value is bound to the function name\ninstead of the function object. Multiple decorators are applied in\nnested fashion. For example, the following code:\n\n @f1(arg)\n @f2\n def func(): pass\n\nis equivalent to:\n\n def func(): pass\n func = f1(arg)(f2(func))\n\nWhen one or more top-level parameters have the form *parameter* ``=``\n*expression*, the function is said to have "default parameter values."\nFor a parameter with a default value, the corresponding argument may\nbe omitted from a call, in which case the parameter\'s default value is\nsubstituted. If a parameter has a default value, all following\nparameters must also have a default value --- this is a syntactic\nrestriction that is not expressed by the grammar.\n\n**Default parameter values are evaluated when the function definition\nis executed.** This means that the expression is evaluated once, when\nthe function is defined, and that that same "pre-computed" value is\nused for each call. This is especially important to understand when a\ndefault parameter is a mutable object, such as a list or a dictionary:\nif the function modifies the object (e.g. by appending an item to a\nlist), the default value is in effect modified. This is generally not\nwhat was intended. A way around this is to use ``None`` as the\ndefault, and explicitly test for it in the body of the function, e.g.:\n\n def whats_on_the_telly(penguin=None):\n if penguin is None:\n penguin = []\n penguin.append("property of the zoo")\n return penguin\n\nFunction call semantics are described in more detail in section\n*Calls*. A function call always assigns values to all parameters\nmentioned in the parameter list, either from position arguments, from\nkeyword arguments, or from default values. If the form\n"``*identifier``" is present, it is initialized to a tuple receiving\nany excess positional parameters, defaulting to the empty tuple. If\nthe form "``**identifier``" is present, it is initialized to a new\ndictionary receiving any excess keyword arguments, defaulting to a new\nempty dictionary.\n\nIt is also possible to create anonymous functions (functions not bound\nto a name), for immediate use in expressions. This uses lambda forms,\ndescribed in section *Lambdas*. Note that the lambda form is merely a\nshorthand for a simplified function definition; a function defined in\na "``def``" statement can be passed around or assigned to another name\njust like a function defined by a lambda form. The "``def``" form is\nactually more powerful since it allows the execution of multiple\nstatements.\n\n**Programmer\'s note:** Functions are first-class objects. A "``def``"\nform executed inside a function definition defines a local function\nthat can be returned or passed around. Free variables used in the\nnested function can access the local variables of the function\ncontaining the def. See section *Naming and binding* for details.\n', 'global': u'\nThe ``global`` statement\n************************\n\n global_stmt ::= "global" identifier ("," identifier)*\n\nThe ``global`` statement is a declaration which holds for the entire\ncurrent code block. It means that the listed identifiers are to be\ninterpreted as globals. It would be impossible to assign to a global\nvariable without ``global``, although free variables may refer to\nglobals without being declared global.\n\nNames listed in a ``global`` statement must not be used in the same\ncode block textually preceding that ``global`` statement.\n\nNames listed in a ``global`` statement must not be defined as formal\nparameters or in a ``for`` loop control target, ``class`` definition,\nfunction definition, or ``import`` statement.\n\n**CPython implementation detail:** The current implementation does not\nenforce the latter two restrictions, but programs should not abuse\nthis freedom, as future implementations may enforce them or silently\nchange the meaning of the program.\n\n**Programmer\'s note:** the ``global`` is a directive to the parser.\nIt applies only to code parsed at the same time as the ``global``\nstatement. In particular, a ``global`` statement contained in an\n``exec`` statement does not affect the code block *containing* the\n``exec`` statement, and code contained in an ``exec`` statement is\nunaffected by ``global`` statements in the code containing the\n``exec`` statement. The same applies to the ``eval()``,\n``execfile()`` and ``compile()`` functions.\n', - 'id-classes': u'\nReserved classes of identifiers\n*******************************\n\nCertain classes of identifiers (besides keywords) have special\nmeanings. These classes are identified by the patterns of leading and\ntrailing underscore characters:\n\n``_*``\n Not imported by ``from module import *``. The special identifier\n ``_`` is used in the interactive interpreter to store the result of\n the last evaluation; it is stored in the ``__builtin__`` module.\n When not in interactive mode, ``_`` has no special meaning and is\n not defined. See section *The import statement*.\n\n Note: The name ``_`` is often used in conjunction with\n internationalization; refer to the documentation for the\n ``gettext`` module for more information on this convention.\n\n``__*__``\n System-defined names. These names are defined by the interpreter\n and its implementation (including the standard library);\n applications should not expect to define additional names using\n this convention. The set of names of this class defined by Python\n may be extended in future versions. See section *Special method\n names*.\n\n``__*``\n Class-private names. Names in this category, when used within the\n context of a class definition, are re-written to use a mangled form\n to help avoid name clashes between "private" attributes of base and\n derived classes. See section *Identifiers (Names)*.\n', - 'identifiers': u'\nIdentifiers and keywords\n************************\n\nIdentifiers (also referred to as *names*) are described by the\nfollowing lexical definitions:\n\n identifier ::= (letter|"_") (letter | digit | "_")*\n letter ::= lowercase | uppercase\n lowercase ::= "a"..."z"\n uppercase ::= "A"..."Z"\n digit ::= "0"..."9"\n\nIdentifiers are unlimited in length. Case is significant.\n\n\nKeywords\n========\n\nThe following identifiers are used as reserved words, or *keywords* of\nthe language, and cannot be used as ordinary identifiers. They must\nbe spelled exactly as written here:\n\n and del from not while\n as elif global or with\n assert else if pass yield\n break except import print\n class exec in raise\n continue finally is return\n def for lambda try\n\nChanged in version 2.4: ``None`` became a constant and is now\nrecognized by the compiler as a name for the built-in object ``None``.\nAlthough it is not a keyword, you cannot assign a different object to\nit.\n\nChanged in version 2.5: Both ``as`` and ``with`` are only recognized\nwhen the ``with_statement`` future feature has been enabled. It will\nalways be enabled in Python 2.6. See section *The with statement* for\ndetails. Note that using ``as`` and ``with`` as identifiers will\nalways issue a warning, even when the ``with_statement`` future\ndirective is not in effect.\n\n\nReserved classes of identifiers\n===============================\n\nCertain classes of identifiers (besides keywords) have special\nmeanings. These classes are identified by the patterns of leading and\ntrailing underscore characters:\n\n``_*``\n Not imported by ``from module import *``. The special identifier\n ``_`` is used in the interactive interpreter to store the result of\n the last evaluation; it is stored in the ``__builtin__`` module.\n When not in interactive mode, ``_`` has no special meaning and is\n not defined. See section *The import statement*.\n\n Note: The name ``_`` is often used in conjunction with\n internationalization; refer to the documentation for the\n ``gettext`` module for more information on this convention.\n\n``__*__``\n System-defined names. These names are defined by the interpreter\n and its implementation (including the standard library);\n applications should not expect to define additional names using\n this convention. The set of names of this class defined by Python\n may be extended in future versions. See section *Special method\n names*.\n\n``__*``\n Class-private names. Names in this category, when used within the\n context of a class definition, are re-written to use a mangled form\n to help avoid name clashes between "private" attributes of base and\n derived classes. See section *Identifiers (Names)*.\n', + 'id-classes': u'\nReserved classes of identifiers\n*******************************\n\nCertain classes of identifiers (besides keywords) have special\nmeanings. These classes are identified by the patterns of leading and\ntrailing underscore characters:\n\n``_*``\n Not imported by ``from module import *``. The special identifier\n ``_`` is used in the interactive interpreter to store the result of\n the last evaluation; it is stored in the ``__builtin__`` module.\n When not in interactive mode, ``_`` has no special meaning and is\n not defined. See section *The import statement*.\n\n Note: The name ``_`` is often used in conjunction with\n internationalization; refer to the documentation for the\n ``gettext`` module for more information on this convention.\n\n``__*__``\n System-defined names. These names are defined by the interpreter\n and its implementation (including the standard library). Current\n system names are discussed in the *Special method names* section\n and elsewhere. More will likely be defined in future versions of\n Python. *Any* use of ``__*__`` names, in any context, that does\n not follow explicitly documented use, is subject to breakage\n without warning.\n\n``__*``\n Class-private names. Names in this category, when used within the\n context of a class definition, are re-written to use a mangled form\n to help avoid name clashes between "private" attributes of base and\n derived classes. See section *Identifiers (Names)*.\n', + 'identifiers': u'\nIdentifiers and keywords\n************************\n\nIdentifiers (also referred to as *names*) are described by the\nfollowing lexical definitions:\n\n identifier ::= (letter|"_") (letter | digit | "_")*\n letter ::= lowercase | uppercase\n lowercase ::= "a"..."z"\n uppercase ::= "A"..."Z"\n digit ::= "0"..."9"\n\nIdentifiers are unlimited in length. Case is significant.\n\n\nKeywords\n========\n\nThe following identifiers are used as reserved words, or *keywords* of\nthe language, and cannot be used as ordinary identifiers. They must\nbe spelled exactly as written here:\n\n and del from not while\n as elif global or with\n assert else if pass yield\n break except import print\n class exec in raise\n continue finally is return\n def for lambda try\n\nChanged in version 2.4: ``None`` became a constant and is now\nrecognized by the compiler as a name for the built-in object ``None``.\nAlthough it is not a keyword, you cannot assign a different object to\nit.\n\nChanged in version 2.5: Both ``as`` and ``with`` are only recognized\nwhen the ``with_statement`` future feature has been enabled. It will\nalways be enabled in Python 2.6. See section *The with statement* for\ndetails. Note that using ``as`` and ``with`` as identifiers will\nalways issue a warning, even when the ``with_statement`` future\ndirective is not in effect.\n\n\nReserved classes of identifiers\n===============================\n\nCertain classes of identifiers (besides keywords) have special\nmeanings. These classes are identified by the patterns of leading and\ntrailing underscore characters:\n\n``_*``\n Not imported by ``from module import *``. The special identifier\n ``_`` is used in the interactive interpreter to store the result of\n the last evaluation; it is stored in the ``__builtin__`` module.\n When not in interactive mode, ``_`` has no special meaning and is\n not defined. See section *The import statement*.\n\n Note: The name ``_`` is often used in conjunction with\n internationalization; refer to the documentation for the\n ``gettext`` module for more information on this convention.\n\n``__*__``\n System-defined names. These names are defined by the interpreter\n and its implementation (including the standard library). Current\n system names are discussed in the *Special method names* section\n and elsewhere. More will likely be defined in future versions of\n Python. *Any* use of ``__*__`` names, in any context, that does\n not follow explicitly documented use, is subject to breakage\n without warning.\n\n``__*``\n Class-private names. Names in this category, when used within the\n context of a class definition, are re-written to use a mangled form\n to help avoid name clashes between "private" attributes of base and\n derived classes. See section *Identifiers (Names)*.\n', 'if': u'\nThe ``if`` statement\n********************\n\nThe ``if`` statement is used for conditional execution:\n\n if_stmt ::= "if" expression ":" suite\n ( "elif" expression ":" suite )*\n ["else" ":" suite]\n\nIt selects exactly one of the suites by evaluating the expressions one\nby one until one is found to be true (see section *Boolean operations*\nfor the definition of true and false); then that suite is executed\n(and no other part of the ``if`` statement is executed or evaluated).\nIf all expressions are false, the suite of the ``else`` clause, if\npresent, is executed.\n', 'imaginary': u'\nImaginary literals\n******************\n\nImaginary literals are described by the following lexical definitions:\n\n imagnumber ::= (floatnumber | intpart) ("j" | "J")\n\nAn imaginary literal yields a complex number with a real part of 0.0.\nComplex numbers are represented as a pair of floating point numbers\nand have the same restrictions on their range. To create a complex\nnumber with a nonzero real part, add a floating point number to it,\ne.g., ``(3+4j)``. Some examples of imaginary literals:\n\n 3.14j 10.j 10j .001j 1e100j 3.14e-10j\n', - 'import': u'\nThe ``import`` statement\n************************\n\n import_stmt ::= "import" module ["as" name] ( "," module ["as" name] )*\n | "from" relative_module "import" identifier ["as" name]\n ( "," identifier ["as" name] )*\n | "from" relative_module "import" "(" identifier ["as" name]\n ( "," identifier ["as" name] )* [","] ")"\n | "from" module "import" "*"\n module ::= (identifier ".")* identifier\n relative_module ::= "."* module | "."+\n name ::= identifier\n\nImport statements are executed in two steps: (1) find a module, and\ninitialize it if necessary; (2) define a name or names in the local\nnamespace (of the scope where the ``import`` statement occurs). The\nstatement comes in two forms differing on whether it uses the ``from``\nkeyword. The first form (without ``from``) repeats these steps for\neach identifier in the list. The form with ``from`` performs step (1)\nonce, and then performs step (2) repeatedly.\n\nTo understand how step (1) occurs, one must first understand how\nPython handles hierarchical naming of modules. To help organize\nmodules and provide a hierarchy in naming, Python has a concept of\npackages. A package can contain other packages and modules while\nmodules cannot contain other modules or packages. From a file system\nperspective, packages are directories and modules are files. The\noriginal specification for packages is still available to read,\nalthough minor details have changed since the writing of that\ndocument.\n\nOnce the name of the module is known (unless otherwise specified, the\nterm "module" will refer to both packages and modules), searching for\nthe module or package can begin. The first place checked is\n``sys.modules``, the cache of all modules that have been imported\npreviously. If the module is found there then it is used in step (2)\nof import.\n\nIf the module is not found in the cache, then ``sys.meta_path`` is\nsearched (the specification for ``sys.meta_path`` can be found in\n**PEP 302**). The object is a list of *finder* objects which are\nqueried in order as to whether they know how to load the module by\ncalling their ``find_module()`` method with the name of the module. If\nthe module happens to be contained within a package (as denoted by the\nexistence of a dot in the name), then a second argument to\n``find_module()`` is given as the value of the ``__path__`` attribute\nfrom the parent package (everything up to the last dot in the name of\nthe module being imported). If a finder can find the module it returns\na *loader* (discussed later) or returns ``None``.\n\nIf none of the finders on ``sys.meta_path`` are able to find the\nmodule then some implicitly defined finders are queried.\nImplementations of Python vary in what implicit meta path finders are\ndefined. The one they all do define, though, is one that handles\n``sys.path_hooks``, ``sys.path_importer_cache``, and ``sys.path``.\n\nThe implicit finder searches for the requested module in the "paths"\nspecified in one of two places ("paths" do not have to be file system\npaths). If the module being imported is supposed to be contained\nwithin a package then the second argument passed to ``find_module()``,\n``__path__`` on the parent package, is used as the source of paths. If\nthe module is not contained in a package then ``sys.path`` is used as\nthe source of paths.\n\nOnce the source of paths is chosen it is iterated over to find a\nfinder that can handle that path. The dict at\n``sys.path_importer_cache`` caches finders for paths and is checked\nfor a finder. If the path does not have a finder cached then\n``sys.path_hooks`` is searched by calling each object in the list with\na single argument of the path, returning a finder or raises\n``ImportError``. If a finder is returned then it is cached in\n``sys.path_importer_cache`` and then used for that path entry. If no\nfinder can be found but the path exists then a value of ``None`` is\nstored in ``sys.path_importer_cache`` to signify that an implicit,\nfile-based finder that handles modules stored as individual files\nshould be used for that path. If the path does not exist then a finder\nwhich always returns ``None`` is placed in the cache for the path.\n\nIf no finder can find the module then ``ImportError`` is raised.\nOtherwise some finder returned a loader whose ``load_module()`` method\nis called with the name of the module to load (see **PEP 302** for the\noriginal definition of loaders). A loader has several responsibilities\nto perform on a module it loads. First, if the module already exists\nin ``sys.modules`` (a possibility if the loader is called outside of\nthe import machinery) then it is to use that module for initialization\nand not a new module. But if the module does not exist in\n``sys.modules`` then it is to be added to that dict before\ninitialization begins. If an error occurs during loading of the module\nand it was added to ``sys.modules`` it is to be removed from the dict.\nIf an error occurs but the module was already in ``sys.modules`` it is\nleft in the dict.\n\nThe loader must set several attributes on the module. ``__name__`` is\nto be set to the name of the module. ``__file__`` is to be the "path"\nto the file unless the module is built-in (and thus listed in\n``sys.builtin_module_names``) in which case the attribute is not set.\nIf what is being imported is a package then ``__path__`` is to be set\nto a list of paths to be searched when looking for modules and\npackages contained within the package being imported. ``__package__``\nis optional but should be set to the name of package that contains the\nmodule or package (the empty string is used for module not contained\nin a package). ``__loader__`` is also optional but should be set to\nthe loader object that is loading the module.\n\nIf an error occurs during loading then the loader raises\n``ImportError`` if some other exception is not already being\npropagated. Otherwise the loader returns the module that was loaded\nand initialized.\n\nWhen step (1) finishes without raising an exception, step (2) can\nbegin.\n\nThe first form of ``import`` statement binds the module name in the\nlocal namespace to the module object, and then goes on to import the\nnext identifier, if any. If the module name is followed by ``as``,\nthe name following ``as`` is used as the local name for the module.\n\nThe ``from`` form does not bind the module name: it goes through the\nlist of identifiers, looks each one of them up in the module found in\nstep (1), and binds the name in the local namespace to the object thus\nfound. As with the first form of ``import``, an alternate local name\ncan be supplied by specifying "``as`` localname". If a name is not\nfound, ``ImportError`` is raised. If the list of identifiers is\nreplaced by a star (``\'*\'``), all public names defined in the module\nare bound in the local namespace of the ``import`` statement..\n\nThe *public names* defined by a module are determined by checking the\nmodule\'s namespace for a variable named ``__all__``; if defined, it\nmust be a sequence of strings which are names defined or imported by\nthat module. The names given in ``__all__`` are all considered public\nand are required to exist. If ``__all__`` is not defined, the set of\npublic names includes all names found in the module\'s namespace which\ndo not begin with an underscore character (``\'_\'``). ``__all__``\nshould contain the entire public API. It is intended to avoid\naccidentally exporting items that are not part of the API (such as\nlibrary modules which were imported and used within the module).\n\nThe ``from`` form with ``*`` may only occur in a module scope. If the\nwild card form of import --- ``import *`` --- is used in a function\nand the function contains or is a nested block with free variables,\nthe compiler will raise a ``SyntaxError``.\n\nWhen specifying what module to import you do not have to specify the\nabsolute name of the module. When a module or package is contained\nwithin another package it is possible to make a relative import within\nthe same top package without having to mention the package name. By\nusing leading dots in the specified module or package after ``from``\nyou can specify how high to traverse up the current package hierarchy\nwithout specifying exact names. One leading dot means the current\npackage where the module making the import exists. Two dots means up\none package level. Three dots is up two levels, etc. So if you execute\n``from . import mod`` from a module in the ``pkg`` package then you\nwill end up importing ``pkg.mod``. If you execute ``from ..subpkg2\nimprt mod`` from within ``pkg.subpkg1`` you will import\n``pkg.subpkg2.mod``. The specification for relative imports is\ncontained within **PEP 328**.\n\n``importlib.import_module()`` is provided to support applications that\ndetermine which modules need to be loaded dynamically.\n\n\nFuture statements\n=================\n\nA *future statement* is a directive to the compiler that a particular\nmodule should be compiled using syntax or semantics that will be\navailable in a specified future release of Python. The future\nstatement is intended to ease migration to future versions of Python\nthat introduce incompatible changes to the language. It allows use of\nthe new features on a per-module basis before the release in which the\nfeature becomes standard.\n\n future_statement ::= "from" "__future__" "import" feature ["as" name]\n ("," feature ["as" name])*\n | "from" "__future__" "import" "(" feature ["as" name]\n ("," feature ["as" name])* [","] ")"\n feature ::= identifier\n name ::= identifier\n\nA future statement must appear near the top of the module. The only\nlines that can appear before a future statement are:\n\n* the module docstring (if any),\n\n* comments,\n\n* blank lines, and\n\n* other future statements.\n\nThe features recognized by Python 2.6 are ``unicode_literals``,\n``print_function``, ``absolute_import``, ``division``, ``generators``,\n``nested_scopes`` and ``with_statement``. ``generators``,\n``with_statement``, ``nested_scopes`` are redundant in Python version\n2.6 and above because they are always enabled.\n\nA future statement is recognized and treated specially at compile\ntime: Changes to the semantics of core constructs are often\nimplemented by generating different code. It may even be the case\nthat a new feature introduces new incompatible syntax (such as a new\nreserved word), in which case the compiler may need to parse the\nmodule differently. Such decisions cannot be pushed off until\nruntime.\n\nFor any given release, the compiler knows which feature names have\nbeen defined, and raises a compile-time error if a future statement\ncontains a feature not known to it.\n\nThe direct runtime semantics are the same as for any import statement:\nthere is a standard module ``__future__``, described later, and it\nwill be imported in the usual way at the time the future statement is\nexecuted.\n\nThe interesting runtime semantics depend on the specific feature\nenabled by the future statement.\n\nNote that there is nothing special about the statement:\n\n import __future__ [as name]\n\nThat is not a future statement; it\'s an ordinary import statement with\nno special semantics or syntax restrictions.\n\nCode compiled by an ``exec`` statement or calls to the built-in\nfunctions ``compile()`` and ``execfile()`` that occur in a module\n``M`` containing a future statement will, by default, use the new\nsyntax or semantics associated with the future statement. This can,\nstarting with Python 2.2 be controlled by optional arguments to\n``compile()`` --- see the documentation of that function for details.\n\nA future statement typed at an interactive interpreter prompt will\ntake effect for the rest of the interpreter session. If an\ninterpreter is started with the *-i* option, is passed a script name\nto execute, and the script includes a future statement, it will be in\neffect in the interactive session started after the script is\nexecuted.\n\nSee also:\n\n **PEP 236** - Back to the __future__\n The original proposal for the __future__ mechanism.\n', + 'import': u'\nThe ``import`` statement\n************************\n\n import_stmt ::= "import" module ["as" name] ( "," module ["as" name] )*\n | "from" relative_module "import" identifier ["as" name]\n ( "," identifier ["as" name] )*\n | "from" relative_module "import" "(" identifier ["as" name]\n ( "," identifier ["as" name] )* [","] ")"\n | "from" module "import" "*"\n module ::= (identifier ".")* identifier\n relative_module ::= "."* module | "."+\n name ::= identifier\n\nImport statements are executed in two steps: (1) find a module, and\ninitialize it if necessary; (2) define a name or names in the local\nnamespace (of the scope where the ``import`` statement occurs). The\nstatement comes in two forms differing on whether it uses the ``from``\nkeyword. The first form (without ``from``) repeats these steps for\neach identifier in the list. The form with ``from`` performs step (1)\nonce, and then performs step (2) repeatedly.\n\nTo understand how step (1) occurs, one must first understand how\nPython handles hierarchical naming of modules. To help organize\nmodules and provide a hierarchy in naming, Python has a concept of\npackages. A package can contain other packages and modules while\nmodules cannot contain other modules or packages. From a file system\nperspective, packages are directories and modules are files. The\noriginal specification for packages is still available to read,\nalthough minor details have changed since the writing of that\ndocument.\n\nOnce the name of the module is known (unless otherwise specified, the\nterm "module" will refer to both packages and modules), searching for\nthe module or package can begin. The first place checked is\n``sys.modules``, the cache of all modules that have been imported\npreviously. If the module is found there then it is used in step (2)\nof import.\n\nIf the module is not found in the cache, then ``sys.meta_path`` is\nsearched (the specification for ``sys.meta_path`` can be found in\n**PEP 302**). The object is a list of *finder* objects which are\nqueried in order as to whether they know how to load the module by\ncalling their ``find_module()`` method with the name of the module. If\nthe module happens to be contained within a package (as denoted by the\nexistence of a dot in the name), then a second argument to\n``find_module()`` is given as the value of the ``__path__`` attribute\nfrom the parent package (everything up to the last dot in the name of\nthe module being imported). If a finder can find the module it returns\na *loader* (discussed later) or returns ``None``.\n\nIf none of the finders on ``sys.meta_path`` are able to find the\nmodule then some implicitly defined finders are queried.\nImplementations of Python vary in what implicit meta path finders are\ndefined. The one they all do define, though, is one that handles\n``sys.path_hooks``, ``sys.path_importer_cache``, and ``sys.path``.\n\nThe implicit finder searches for the requested module in the "paths"\nspecified in one of two places ("paths" do not have to be file system\npaths). If the module being imported is supposed to be contained\nwithin a package then the second argument passed to ``find_module()``,\n``__path__`` on the parent package, is used as the source of paths. If\nthe module is not contained in a package then ``sys.path`` is used as\nthe source of paths.\n\nOnce the source of paths is chosen it is iterated over to find a\nfinder that can handle that path. The dict at\n``sys.path_importer_cache`` caches finders for paths and is checked\nfor a finder. If the path does not have a finder cached then\n``sys.path_hooks`` is searched by calling each object in the list with\na single argument of the path, returning a finder or raises\n``ImportError``. If a finder is returned then it is cached in\n``sys.path_importer_cache`` and then used for that path entry. If no\nfinder can be found but the path exists then a value of ``None`` is\nstored in ``sys.path_importer_cache`` to signify that an implicit,\nfile-based finder that handles modules stored as individual files\nshould be used for that path. If the path does not exist then a finder\nwhich always returns ``None`` is placed in the cache for the path.\n\nIf no finder can find the module then ``ImportError`` is raised.\nOtherwise some finder returned a loader whose ``load_module()`` method\nis called with the name of the module to load (see **PEP 302** for the\noriginal definition of loaders). A loader has several responsibilities\nto perform on a module it loads. First, if the module already exists\nin ``sys.modules`` (a possibility if the loader is called outside of\nthe import machinery) then it is to use that module for initialization\nand not a new module. But if the module does not exist in\n``sys.modules`` then it is to be added to that dict before\ninitialization begins. If an error occurs during loading of the module\nand it was added to ``sys.modules`` it is to be removed from the dict.\nIf an error occurs but the module was already in ``sys.modules`` it is\nleft in the dict.\n\nThe loader must set several attributes on the module. ``__name__`` is\nto be set to the name of the module. ``__file__`` is to be the "path"\nto the file unless the module is built-in (and thus listed in\n``sys.builtin_module_names``) in which case the attribute is not set.\nIf what is being imported is a package then ``__path__`` is to be set\nto a list of paths to be searched when looking for modules and\npackages contained within the package being imported. ``__package__``\nis optional but should be set to the name of package that contains the\nmodule or package (the empty string is used for module not contained\nin a package). ``__loader__`` is also optional but should be set to\nthe loader object that is loading the module.\n\nIf an error occurs during loading then the loader raises\n``ImportError`` if some other exception is not already being\npropagated. Otherwise the loader returns the module that was loaded\nand initialized.\n\nWhen step (1) finishes without raising an exception, step (2) can\nbegin.\n\nThe first form of ``import`` statement binds the module name in the\nlocal namespace to the module object, and then goes on to import the\nnext identifier, if any. If the module name is followed by ``as``,\nthe name following ``as`` is used as the local name for the module.\n\nThe ``from`` form does not bind the module name: it goes through the\nlist of identifiers, looks each one of them up in the module found in\nstep (1), and binds the name in the local namespace to the object thus\nfound. As with the first form of ``import``, an alternate local name\ncan be supplied by specifying "``as`` localname". If a name is not\nfound, ``ImportError`` is raised. If the list of identifiers is\nreplaced by a star (``\'*\'``), all public names defined in the module\nare bound in the local namespace of the ``import`` statement..\n\nThe *public names* defined by a module are determined by checking the\nmodule\'s namespace for a variable named ``__all__``; if defined, it\nmust be a sequence of strings which are names defined or imported by\nthat module. The names given in ``__all__`` are all considered public\nand are required to exist. If ``__all__`` is not defined, the set of\npublic names includes all names found in the module\'s namespace which\ndo not begin with an underscore character (``\'_\'``). ``__all__``\nshould contain the entire public API. It is intended to avoid\naccidentally exporting items that are not part of the API (such as\nlibrary modules which were imported and used within the module).\n\nThe ``from`` form with ``*`` may only occur in a module scope. If the\nwild card form of import --- ``import *`` --- is used in a function\nand the function contains or is a nested block with free variables,\nthe compiler will raise a ``SyntaxError``.\n\nWhen specifying what module to import you do not have to specify the\nabsolute name of the module. When a module or package is contained\nwithin another package it is possible to make a relative import within\nthe same top package without having to mention the package name. By\nusing leading dots in the specified module or package after ``from``\nyou can specify how high to traverse up the current package hierarchy\nwithout specifying exact names. One leading dot means the current\npackage where the module making the import exists. Two dots means up\none package level. Three dots is up two levels, etc. So if you execute\n``from . import mod`` from a module in the ``pkg`` package then you\nwill end up importing ``pkg.mod``. If you execute ``from ..subpkg2\nimport mod`` from within ``pkg.subpkg1`` you will import\n``pkg.subpkg2.mod``. The specification for relative imports is\ncontained within **PEP 328**.\n\n``importlib.import_module()`` is provided to support applications that\ndetermine which modules need to be loaded dynamically.\n\n\nFuture statements\n=================\n\nA *future statement* is a directive to the compiler that a particular\nmodule should be compiled using syntax or semantics that will be\navailable in a specified future release of Python. The future\nstatement is intended to ease migration to future versions of Python\nthat introduce incompatible changes to the language. It allows use of\nthe new features on a per-module basis before the release in which the\nfeature becomes standard.\n\n future_statement ::= "from" "__future__" "import" feature ["as" name]\n ("," feature ["as" name])*\n | "from" "__future__" "import" "(" feature ["as" name]\n ("," feature ["as" name])* [","] ")"\n feature ::= identifier\n name ::= identifier\n\nA future statement must appear near the top of the module. The only\nlines that can appear before a future statement are:\n\n* the module docstring (if any),\n\n* comments,\n\n* blank lines, and\n\n* other future statements.\n\nThe features recognized by Python 2.6 are ``unicode_literals``,\n``print_function``, ``absolute_import``, ``division``, ``generators``,\n``nested_scopes`` and ``with_statement``. ``generators``,\n``with_statement``, ``nested_scopes`` are redundant in Python version\n2.6 and above because they are always enabled.\n\nA future statement is recognized and treated specially at compile\ntime: Changes to the semantics of core constructs are often\nimplemented by generating different code. It may even be the case\nthat a new feature introduces new incompatible syntax (such as a new\nreserved word), in which case the compiler may need to parse the\nmodule differently. Such decisions cannot be pushed off until\nruntime.\n\nFor any given release, the compiler knows which feature names have\nbeen defined, and raises a compile-time error if a future statement\ncontains a feature not known to it.\n\nThe direct runtime semantics are the same as for any import statement:\nthere is a standard module ``__future__``, described later, and it\nwill be imported in the usual way at the time the future statement is\nexecuted.\n\nThe interesting runtime semantics depend on the specific feature\nenabled by the future statement.\n\nNote that there is nothing special about the statement:\n\n import __future__ [as name]\n\nThat is not a future statement; it\'s an ordinary import statement with\nno special semantics or syntax restrictions.\n\nCode compiled by an ``exec`` statement or calls to the built-in\nfunctions ``compile()`` and ``execfile()`` that occur in a module\n``M`` containing a future statement will, by default, use the new\nsyntax or semantics associated with the future statement. This can,\nstarting with Python 2.2 be controlled by optional arguments to\n``compile()`` --- see the documentation of that function for details.\n\nA future statement typed at an interactive interpreter prompt will\ntake effect for the rest of the interpreter session. If an\ninterpreter is started with the *-i* option, is passed a script name\nto execute, and the script includes a future statement, it will be in\neffect in the interactive session started after the script is\nexecuted.\n\nSee also:\n\n **PEP 236** - Back to the __future__\n The original proposal for the __future__ mechanism.\n', 'in': u'\nComparisons\n***********\n\nUnlike C, all comparison operations in Python have the same priority,\nwhich is lower than that of any arithmetic, shifting or bitwise\noperation. Also unlike C, expressions like ``a < b < c`` have the\ninterpretation that is conventional in mathematics:\n\n comparison ::= or_expr ( comp_operator or_expr )*\n comp_operator ::= "<" | ">" | "==" | ">=" | "<=" | "<>" | "!="\n | "is" ["not"] | ["not"] "in"\n\nComparisons yield boolean values: ``True`` or ``False``.\n\nComparisons can be chained arbitrarily, e.g., ``x < y <= z`` is\nequivalent to ``x < y and y <= z``, except that ``y`` is evaluated\nonly once (but in both cases ``z`` is not evaluated at all when ``x <\ny`` is found to be false).\n\nFormally, if *a*, *b*, *c*, ..., *y*, *z* are expressions and *op1*,\n*op2*, ..., *opN* are comparison operators, then ``a op1 b op2 c ... y\nopN z`` is equivalent to ``a op1 b and b op2 c and ... y opN z``,\nexcept that each expression is evaluated at most once.\n\nNote that ``a op1 b op2 c`` doesn\'t imply any kind of comparison\nbetween *a* and *c*, so that, e.g., ``x < y > z`` is perfectly legal\n(though perhaps not pretty).\n\nThe forms ``<>`` and ``!=`` are equivalent; for consistency with C,\n``!=`` is preferred; where ``!=`` is mentioned below ``<>`` is also\naccepted. The ``<>`` spelling is considered obsolescent.\n\nThe operators ``<``, ``>``, ``==``, ``>=``, ``<=``, and ``!=`` compare\nthe values of two objects. The objects need not have the same type.\nIf both are numbers, they are converted to a common type. Otherwise,\nobjects of different types *always* compare unequal, and are ordered\nconsistently but arbitrarily. You can control comparison behavior of\nobjects of non-built-in types by defining a ``__cmp__`` method or rich\ncomparison methods like ``__gt__``, described in section *Special\nmethod names*.\n\n(This unusual definition of comparison was used to simplify the\ndefinition of operations like sorting and the ``in`` and ``not in``\noperators. In the future, the comparison rules for objects of\ndifferent types are likely to change.)\n\nComparison of objects of the same type depends on the type:\n\n* Numbers are compared arithmetically.\n\n* Strings are compared lexicographically using the numeric equivalents\n (the result of the built-in function ``ord()``) of their characters.\n Unicode and 8-bit strings are fully interoperable in this behavior.\n [4]\n\n* Tuples and lists are compared lexicographically using comparison of\n corresponding elements. This means that to compare equal, each\n element must compare equal and the two sequences must be of the same\n type and have the same length.\n\n If not equal, the sequences are ordered the same as their first\n differing elements. For example, ``cmp([1,2,x], [1,2,y])`` returns\n the same as ``cmp(x,y)``. If the corresponding element does not\n exist, the shorter sequence is ordered first (for example, ``[1,2] <\n [1,2,3]``).\n\n* Mappings (dictionaries) compare equal if and only if their sorted\n (key, value) lists compare equal. [5] Outcomes other than equality\n are resolved consistently, but are not otherwise defined. [6]\n\n* Most other objects of built-in types compare unequal unless they are\n the same object; the choice whether one object is considered smaller\n or larger than another one is made arbitrarily but consistently\n within one execution of a program.\n\nThe operators ``in`` and ``not in`` test for collection membership.\n``x in s`` evaluates to true if *x* is a member of the collection *s*,\nand false otherwise. ``x not in s`` returns the negation of ``x in\ns``. The collection membership test has traditionally been bound to\nsequences; an object is a member of a collection if the collection is\na sequence and contains an element equal to that object. However, it\nmake sense for many other object types to support membership tests\nwithout being a sequence. In particular, dictionaries (for keys) and\nsets support membership testing.\n\nFor the list and tuple types, ``x in y`` is true if and only if there\nexists an index *i* such that ``x == y[i]`` is true.\n\nFor the Unicode and string types, ``x in y`` is true if and only if\n*x* is a substring of *y*. An equivalent test is ``y.find(x) != -1``.\nNote, *x* and *y* need not be the same type; consequently, ``u\'ab\' in\n\'abc\'`` will return ``True``. Empty strings are always considered to\nbe a substring of any other string, so ``"" in "abc"`` will return\n``True``.\n\nChanged in version 2.3: Previously, *x* was required to be a string of\nlength ``1``.\n\nFor user-defined classes which define the ``__contains__()`` method,\n``x in y`` is true if and only if ``y.__contains__(x)`` is true.\n\nFor user-defined classes which do not define ``__contains__()`` but do\ndefine ``__iter__()``, ``x in y`` is true if some value ``z`` with ``x\n== z`` is produced while iterating over ``y``. If an exception is\nraised during the iteration, it is as if ``in`` raised that exception.\n\nLastly, the old-style iteration protocol is tried: if a class defines\n``__getitem__()``, ``x in y`` is true if and only if there is a non-\nnegative integer index *i* such that ``x == y[i]``, and all lower\ninteger indices do not raise ``IndexError`` exception. (If any other\nexception is raised, it is as if ``in`` raised that exception).\n\nThe operator ``not in`` is defined to have the inverse true value of\n``in``.\n\nThe operators ``is`` and ``is not`` test for object identity: ``x is\ny`` is true if and only if *x* and *y* are the same object. ``x is\nnot y`` yields the inverse truth value. [7]\n', 'integers': u'\nInteger and long integer literals\n*********************************\n\nInteger and long integer literals are described by the following\nlexical definitions:\n\n longinteger ::= integer ("l" | "L")\n integer ::= decimalinteger | octinteger | hexinteger | bininteger\n decimalinteger ::= nonzerodigit digit* | "0"\n octinteger ::= "0" ("o" | "O") octdigit+ | "0" octdigit+\n hexinteger ::= "0" ("x" | "X") hexdigit+\n bininteger ::= "0" ("b" | "B") bindigit+\n nonzerodigit ::= "1"..."9"\n octdigit ::= "0"..."7"\n bindigit ::= "0" | "1"\n hexdigit ::= digit | "a"..."f" | "A"..."F"\n\nAlthough both lower case ``\'l\'`` and upper case ``\'L\'`` are allowed as\nsuffix for long integers, it is strongly recommended to always use\n``\'L\'``, since the letter ``\'l\'`` looks too much like the digit\n``\'1\'``.\n\nPlain integer literals that are above the largest representable plain\ninteger (e.g., 2147483647 when using 32-bit arithmetic) are accepted\nas if they were long integers instead. [1] There is no limit for long\ninteger literals apart from what can be stored in available memory.\n\nSome examples of plain integer literals (first row) and long integer\nliterals (second and third rows):\n\n 7 2147483647 0177\n 3L 79228162514264337593543950336L 0377L 0x100000000L\n 79228162514264337593543950336 0xdeadbeef\n', 'lambda': u'\nLambdas\n*******\n\n lambda_form ::= "lambda" [parameter_list]: expression\n old_lambda_form ::= "lambda" [parameter_list]: old_expression\n\nLambda forms (lambda expressions) have the same syntactic position as\nexpressions. They are a shorthand to create anonymous functions; the\nexpression ``lambda arguments: expression`` yields a function object.\nThe unnamed object behaves like a function object defined with\n\n def name(arguments):\n return expression\n\nSee section *Function definitions* for the syntax of parameter lists.\nNote that functions created with lambda forms cannot contain\nstatements.\n', 'lists': u'\nList displays\n*************\n\nA list display is a possibly empty series of expressions enclosed in\nsquare brackets:\n\n list_display ::= "[" [expression_list | list_comprehension] "]"\n list_comprehension ::= expression list_for\n list_for ::= "for" target_list "in" old_expression_list [list_iter]\n old_expression_list ::= old_expression [("," old_expression)+ [","]]\n old_expression ::= or_test | old_lambda_form\n list_iter ::= list_for | list_if\n list_if ::= "if" old_expression [list_iter]\n\nA list display yields a new list object. Its contents are specified\nby providing either a list of expressions or a list comprehension.\nWhen a comma-separated list of expressions is supplied, its elements\nare evaluated from left to right and placed into the list object in\nthat order. When a list comprehension is supplied, it consists of a\nsingle expression followed by at least one ``for`` clause and zero or\nmore ``for`` or ``if`` clauses. In this case, the elements of the new\nlist are those that would be produced by considering each of the\n``for`` or ``if`` clauses a block, nesting from left to right, and\nevaluating the expression to produce a list element each time the\ninnermost block is reached [1].\n', - 'naming': u"\nNaming and binding\n******************\n\n*Names* refer to objects. Names are introduced by name binding\noperations. Each occurrence of a name in the program text refers to\nthe *binding* of that name established in the innermost function block\ncontaining the use.\n\nA *block* is a piece of Python program text that is executed as a\nunit. The following are blocks: a module, a function body, and a class\ndefinition. Each command typed interactively is a block. A script\nfile (a file given as standard input to the interpreter or specified\non the interpreter command line the first argument) is a code block.\nA script command (a command specified on the interpreter command line\nwith the '**-c**' option) is a code block. The file read by the\nbuilt-in function ``execfile()`` is a code block. The string argument\npassed to the built-in function ``eval()`` and to the ``exec``\nstatement is a code block. The expression read and evaluated by the\nbuilt-in function ``input()`` is a code block.\n\nA code block is executed in an *execution frame*. A frame contains\nsome administrative information (used for debugging) and determines\nwhere and how execution continues after the code block's execution has\ncompleted.\n\nA *scope* defines the visibility of a name within a block. If a local\nvariable is defined in a block, its scope includes that block. If the\ndefinition occurs in a function block, the scope extends to any blocks\ncontained within the defining one, unless a contained block introduces\na different binding for the name. The scope of names defined in a\nclass block is limited to the class block; it does not extend to the\ncode blocks of methods -- this includes generator expressions since\nthey are implemented using a function scope. This means that the\nfollowing will fail:\n\n class A:\n a = 42\n b = list(a + i for i in range(10))\n\nWhen a name is used in a code block, it is resolved using the nearest\nenclosing scope. The set of all such scopes visible to a code block\nis called the block's *environment*.\n\nIf a name is bound in a block, it is a local variable of that block.\nIf a name is bound at the module level, it is a global variable. (The\nvariables of the module code block are local and global.) If a\nvariable is used in a code block but not defined there, it is a *free\nvariable*.\n\nWhen a name is not found at all, a ``NameError`` exception is raised.\nIf the name refers to a local variable that has not been bound, a\n``UnboundLocalError`` exception is raised. ``UnboundLocalError`` is a\nsubclass of ``NameError``.\n\nThe following constructs bind names: formal parameters to functions,\n``import`` statements, class and function definitions (these bind the\nclass or function name in the defining block), and targets that are\nidentifiers if occurring in an assignment, ``for`` loop header, in the\nsecond position of an ``except`` clause header or after ``as`` in a\n``with`` statement. The ``import`` statement of the form ``from ...\nimport *`` binds all names defined in the imported module, except\nthose beginning with an underscore. This form may only be used at the\nmodule level.\n\nA target occurring in a ``del`` statement is also considered bound for\nthis purpose (though the actual semantics are to unbind the name). It\nis illegal to unbind a name that is referenced by an enclosing scope;\nthe compiler will report a ``SyntaxError``.\n\nEach assignment or import statement occurs within a block defined by a\nclass or function definition or at the module level (the top-level\ncode block).\n\nIf a name binding operation occurs anywhere within a code block, all\nuses of the name within the block are treated as references to the\ncurrent block. This can lead to errors when a name is used within a\nblock before it is bound. This rule is subtle. Python lacks\ndeclarations and allows name binding operations to occur anywhere\nwithin a code block. The local variables of a code block can be\ndetermined by scanning the entire text of the block for name binding\noperations.\n\nIf the global statement occurs within a block, all uses of the name\nspecified in the statement refer to the binding of that name in the\ntop-level namespace. Names are resolved in the top-level namespace by\nsearching the global namespace, i.e. the namespace of the module\ncontaining the code block, and the builtins namespace, the namespace\nof the module ``__builtin__``. The global namespace is searched\nfirst. If the name is not found there, the builtins namespace is\nsearched. The global statement must precede all uses of the name.\n\nThe builtins namespace associated with the execution of a code block\nis actually found by looking up the name ``__builtins__`` in its\nglobal namespace; this should be a dictionary or a module (in the\nlatter case the module's dictionary is used). By default, when in the\n``__main__`` module, ``__builtins__`` is the built-in module\n``__builtin__`` (note: no 's'); when in any other module,\n``__builtins__`` is an alias for the dictionary of the ``__builtin__``\nmodule itself. ``__builtins__`` can be set to a user-created\ndictionary to create a weak form of restricted execution.\n\n**CPython implementation detail:** Users should not touch\n``__builtins__``; it is strictly an implementation detail. Users\nwanting to override values in the builtins namespace should ``import``\nthe ``__builtin__`` (no 's') module and modify its attributes\nappropriately.\n\nThe namespace for a module is automatically created the first time a\nmodule is imported. The main module for a script is always called\n``__main__``.\n\nThe global statement has the same scope as a name binding operation in\nthe same block. If the nearest enclosing scope for a free variable\ncontains a global statement, the free variable is treated as a global.\n\nA class definition is an executable statement that may use and define\nnames. These references follow the normal rules for name resolution.\nThe namespace of the class definition becomes the attribute dictionary\nof the class. Names defined at the class scope are not visible in\nmethods.\n\n\nInteraction with dynamic features\n=================================\n\nThere are several cases where Python statements are illegal when used\nin conjunction with nested scopes that contain free variables.\n\nIf a variable is referenced in an enclosing scope, it is illegal to\ndelete the name. An error will be reported at compile time.\n\nIf the wild card form of import --- ``import *`` --- is used in a\nfunction and the function contains or is a nested block with free\nvariables, the compiler will raise a ``SyntaxError``.\n\nIf ``exec`` is used in a function and the function contains or is a\nnested block with free variables, the compiler will raise a\n``SyntaxError`` unless the exec explicitly specifies the local\nnamespace for the ``exec``. (In other words, ``exec obj`` would be\nillegal, but ``exec obj in ns`` would be legal.)\n\nThe ``eval()``, ``execfile()``, and ``input()`` functions and the\n``exec`` statement do not have access to the full environment for\nresolving names. Names may be resolved in the local and global\nnamespaces of the caller. Free variables are not resolved in the\nnearest enclosing namespace, but in the global namespace. [1] The\n``exec`` statement and the ``eval()`` and ``execfile()`` functions\nhave optional arguments to override the global and local namespace.\nIf only one namespace is specified, it is used for both.\n", + 'naming': u"\nNaming and binding\n******************\n\n*Names* refer to objects. Names are introduced by name binding\noperations. Each occurrence of a name in the program text refers to\nthe *binding* of that name established in the innermost function block\ncontaining the use.\n\nA *block* is a piece of Python program text that is executed as a\nunit. The following are blocks: a module, a function body, and a class\ndefinition. Each command typed interactively is a block. A script\nfile (a file given as standard input to the interpreter or specified\non the interpreter command line the first argument) is a code block.\nA script command (a command specified on the interpreter command line\nwith the '**-c**' option) is a code block. The file read by the\nbuilt-in function ``execfile()`` is a code block. The string argument\npassed to the built-in function ``eval()`` and to the ``exec``\nstatement is a code block. The expression read and evaluated by the\nbuilt-in function ``input()`` is a code block.\n\nA code block is executed in an *execution frame*. A frame contains\nsome administrative information (used for debugging) and determines\nwhere and how execution continues after the code block's execution has\ncompleted.\n\nA *scope* defines the visibility of a name within a block. If a local\nvariable is defined in a block, its scope includes that block. If the\ndefinition occurs in a function block, the scope extends to any blocks\ncontained within the defining one, unless a contained block introduces\na different binding for the name. The scope of names defined in a\nclass block is limited to the class block; it does not extend to the\ncode blocks of methods -- this includes generator expressions since\nthey are implemented using a function scope. This means that the\nfollowing will fail:\n\n class A:\n a = 42\n b = list(a + i for i in range(10))\n\nWhen a name is used in a code block, it is resolved using the nearest\nenclosing scope. The set of all such scopes visible to a code block\nis called the block's *environment*.\n\nIf a name is bound in a block, it is a local variable of that block.\nIf a name is bound at the module level, it is a global variable. (The\nvariables of the module code block are local and global.) If a\nvariable is used in a code block but not defined there, it is a *free\nvariable*.\n\nWhen a name is not found at all, a ``NameError`` exception is raised.\nIf the name refers to a local variable that has not been bound, a\n``UnboundLocalError`` exception is raised. ``UnboundLocalError`` is a\nsubclass of ``NameError``.\n\nThe following constructs bind names: formal parameters to functions,\n``import`` statements, class and function definitions (these bind the\nclass or function name in the defining block), and targets that are\nidentifiers if occurring in an assignment, ``for`` loop header, in the\nsecond position of an ``except`` clause header or after ``as`` in a\n``with`` statement. The ``import`` statement of the form ``from ...\nimport *`` binds all names defined in the imported module, except\nthose beginning with an underscore. This form may only be used at the\nmodule level.\n\nA target occurring in a ``del`` statement is also considered bound for\nthis purpose (though the actual semantics are to unbind the name). It\nis illegal to unbind a name that is referenced by an enclosing scope;\nthe compiler will report a ``SyntaxError``.\n\nEach assignment or import statement occurs within a block defined by a\nclass or function definition or at the module level (the top-level\ncode block).\n\nIf a name binding operation occurs anywhere within a code block, all\nuses of the name within the block are treated as references to the\ncurrent block. This can lead to errors when a name is used within a\nblock before it is bound. This rule is subtle. Python lacks\ndeclarations and allows name binding operations to occur anywhere\nwithin a code block. The local variables of a code block can be\ndetermined by scanning the entire text of the block for name binding\noperations.\n\nIf the global statement occurs within a block, all uses of the name\nspecified in the statement refer to the binding of that name in the\ntop-level namespace. Names are resolved in the top-level namespace by\nsearching the global namespace, i.e. the namespace of the module\ncontaining the code block, and the builtins namespace, the namespace\nof the module ``__builtin__``. The global namespace is searched\nfirst. If the name is not found there, the builtins namespace is\nsearched. The global statement must precede all uses of the name.\n\nThe builtins namespace associated with the execution of a code block\nis actually found by looking up the name ``__builtins__`` in its\nglobal namespace; this should be a dictionary or a module (in the\nlatter case the module's dictionary is used). By default, when in the\n``__main__`` module, ``__builtins__`` is the built-in module\n``__builtin__`` (note: no 's'); when in any other module,\n``__builtins__`` is an alias for the dictionary of the ``__builtin__``\nmodule itself. ``__builtins__`` can be set to a user-created\ndictionary to create a weak form of restricted execution.\n\n**CPython implementation detail:** Users should not touch\n``__builtins__``; it is strictly an implementation detail. Users\nwanting to override values in the builtins namespace should ``import``\nthe ``__builtin__`` (no 's') module and modify its attributes\nappropriately.\n\nThe namespace for a module is automatically created the first time a\nmodule is imported. The main module for a script is always called\n``__main__``.\n\nThe ``global`` statement has the same scope as a name binding\noperation in the same block. If the nearest enclosing scope for a\nfree variable contains a global statement, the free variable is\ntreated as a global.\n\nA class definition is an executable statement that may use and define\nnames. These references follow the normal rules for name resolution.\nThe namespace of the class definition becomes the attribute dictionary\nof the class. Names defined at the class scope are not visible in\nmethods.\n\n\nInteraction with dynamic features\n=================================\n\nThere are several cases where Python statements are illegal when used\nin conjunction with nested scopes that contain free variables.\n\nIf a variable is referenced in an enclosing scope, it is illegal to\ndelete the name. An error will be reported at compile time.\n\nIf the wild card form of import --- ``import *`` --- is used in a\nfunction and the function contains or is a nested block with free\nvariables, the compiler will raise a ``SyntaxError``.\n\nIf ``exec`` is used in a function and the function contains or is a\nnested block with free variables, the compiler will raise a\n``SyntaxError`` unless the exec explicitly specifies the local\nnamespace for the ``exec``. (In other words, ``exec obj`` would be\nillegal, but ``exec obj in ns`` would be legal.)\n\nThe ``eval()``, ``execfile()``, and ``input()`` functions and the\n``exec`` statement do not have access to the full environment for\nresolving names. Names may be resolved in the local and global\nnamespaces of the caller. Free variables are not resolved in the\nnearest enclosing namespace, but in the global namespace. [1] The\n``exec`` statement and the ``eval()`` and ``execfile()`` functions\nhave optional arguments to override the global and local namespace.\nIf only one namespace is specified, it is used for both.\n", 'numbers': u"\nNumeric literals\n****************\n\nThere are four types of numeric literals: plain integers, long\nintegers, floating point numbers, and imaginary numbers. There are no\ncomplex literals (complex numbers can be formed by adding a real\nnumber and an imaginary number).\n\nNote that numeric literals do not include a sign; a phrase like ``-1``\nis actually an expression composed of the unary operator '``-``' and\nthe literal ``1``.\n", 'numeric-types': u'\nEmulating numeric types\n***********************\n\nThe following methods can be defined to emulate numeric objects.\nMethods corresponding to operations that are not supported by the\nparticular kind of number implemented (e.g., bitwise operations for\nnon-integral numbers) should be left undefined.\n\nobject.__add__(self, other)\nobject.__sub__(self, other)\nobject.__mul__(self, other)\nobject.__floordiv__(self, other)\nobject.__mod__(self, other)\nobject.__divmod__(self, other)\nobject.__pow__(self, other[, modulo])\nobject.__lshift__(self, other)\nobject.__rshift__(self, other)\nobject.__and__(self, other)\nobject.__xor__(self, other)\nobject.__or__(self, other)\n\n These methods are called to implement the binary arithmetic\n operations (``+``, ``-``, ``*``, ``//``, ``%``, ``divmod()``,\n ``pow()``, ``**``, ``<<``, ``>>``, ``&``, ``^``, ``|``). For\n instance, to evaluate the expression ``x + y``, where *x* is an\n instance of a class that has an ``__add__()`` method,\n ``x.__add__(y)`` is called. The ``__divmod__()`` method should be\n the equivalent to using ``__floordiv__()`` and ``__mod__()``; it\n should not be related to ``__truediv__()`` (described below). Note\n that ``__pow__()`` should be defined to accept an optional third\n argument if the ternary version of the built-in ``pow()`` function\n is to be supported.\n\n If one of those methods does not support the operation with the\n supplied arguments, it should return ``NotImplemented``.\n\nobject.__div__(self, other)\nobject.__truediv__(self, other)\n\n The division operator (``/``) is implemented by these methods. The\n ``__truediv__()`` method is used when ``__future__.division`` is in\n effect, otherwise ``__div__()`` is used. If only one of these two\n methods is defined, the object will not support division in the\n alternate context; ``TypeError`` will be raised instead.\n\nobject.__radd__(self, other)\nobject.__rsub__(self, other)\nobject.__rmul__(self, other)\nobject.__rdiv__(self, other)\nobject.__rtruediv__(self, other)\nobject.__rfloordiv__(self, other)\nobject.__rmod__(self, other)\nobject.__rdivmod__(self, other)\nobject.__rpow__(self, other)\nobject.__rlshift__(self, other)\nobject.__rrshift__(self, other)\nobject.__rand__(self, other)\nobject.__rxor__(self, other)\nobject.__ror__(self, other)\n\n These methods are called to implement the binary arithmetic\n operations (``+``, ``-``, ``*``, ``/``, ``%``, ``divmod()``,\n ``pow()``, ``**``, ``<<``, ``>>``, ``&``, ``^``, ``|``) with\n reflected (swapped) operands. These functions are only called if\n the left operand does not support the corresponding operation and\n the operands are of different types. [2] For instance, to evaluate\n the expression ``x - y``, where *y* is an instance of a class that\n has an ``__rsub__()`` method, ``y.__rsub__(x)`` is called if\n ``x.__sub__(y)`` returns *NotImplemented*.\n\n Note that ternary ``pow()`` will not try calling ``__rpow__()``\n (the coercion rules would become too complicated).\n\n Note: If the right operand\'s type is a subclass of the left operand\'s\n type and that subclass provides the reflected method for the\n operation, this method will be called before the left operand\'s\n non-reflected method. This behavior allows subclasses to\n override their ancestors\' operations.\n\nobject.__iadd__(self, other)\nobject.__isub__(self, other)\nobject.__imul__(self, other)\nobject.__idiv__(self, other)\nobject.__itruediv__(self, other)\nobject.__ifloordiv__(self, other)\nobject.__imod__(self, other)\nobject.__ipow__(self, other[, modulo])\nobject.__ilshift__(self, other)\nobject.__irshift__(self, other)\nobject.__iand__(self, other)\nobject.__ixor__(self, other)\nobject.__ior__(self, other)\n\n These methods are called to implement the augmented arithmetic\n assignments (``+=``, ``-=``, ``*=``, ``/=``, ``//=``, ``%=``,\n ``**=``, ``<<=``, ``>>=``, ``&=``, ``^=``, ``|=``). These methods\n should attempt to do the operation in-place (modifying *self*) and\n return the result (which could be, but does not have to be,\n *self*). If a specific method is not defined, the augmented\n assignment falls back to the normal methods. For instance, to\n execute the statement ``x += y``, where *x* is an instance of a\n class that has an ``__iadd__()`` method, ``x.__iadd__(y)`` is\n called. If *x* is an instance of a class that does not define a\n ``__iadd__()`` method, ``x.__add__(y)`` and ``y.__radd__(x)`` are\n considered, as with the evaluation of ``x + y``.\n\nobject.__neg__(self)\nobject.__pos__(self)\nobject.__abs__(self)\nobject.__invert__(self)\n\n Called to implement the unary arithmetic operations (``-``, ``+``,\n ``abs()`` and ``~``).\n\nobject.__complex__(self)\nobject.__int__(self)\nobject.__long__(self)\nobject.__float__(self)\n\n Called to implement the built-in functions ``complex()``,\n ``int()``, ``long()``, and ``float()``. Should return a value of\n the appropriate type.\n\nobject.__oct__(self)\nobject.__hex__(self)\n\n Called to implement the built-in functions ``oct()`` and ``hex()``.\n Should return a string value.\n\nobject.__index__(self)\n\n Called to implement ``operator.index()``. Also called whenever\n Python needs an integer object (such as in slicing). Must return\n an integer (int or long).\n\n New in version 2.5.\n\nobject.__coerce__(self, other)\n\n Called to implement "mixed-mode" numeric arithmetic. Should either\n return a 2-tuple containing *self* and *other* converted to a\n common numeric type, or ``None`` if conversion is impossible. When\n the common type would be the type of ``other``, it is sufficient to\n return ``None``, since the interpreter will also ask the other\n object to attempt a coercion (but sometimes, if the implementation\n of the other type cannot be changed, it is useful to do the\n conversion to the other type here). A return value of\n ``NotImplemented`` is equivalent to returning ``None``.\n', - 'objects': u'\nObjects, values and types\n*************************\n\n*Objects* are Python\'s abstraction for data. All data in a Python\nprogram is represented by objects or by relations between objects. (In\na sense, and in conformance to Von Neumann\'s model of a "stored\nprogram computer," code is also represented by objects.)\n\nEvery object has an identity, a type and a value. An object\'s\n*identity* never changes once it has been created; you may think of it\nas the object\'s address in memory. The \'``is``\' operator compares the\nidentity of two objects; the ``id()`` function returns an integer\nrepresenting its identity (currently implemented as its address). An\nobject\'s *type* is also unchangeable. [1] An object\'s type determines\nthe operations that the object supports (e.g., "does it have a\nlength?") and also defines the possible values for objects of that\ntype. The ``type()`` function returns an object\'s type (which is an\nobject itself). The *value* of some objects can change. Objects\nwhose value can change are said to be *mutable*; objects whose value\nis unchangeable once they are created are called *immutable*. (The\nvalue of an immutable container object that contains a reference to a\nmutable object can change when the latter\'s value is changed; however\nthe container is still considered immutable, because the collection of\nobjects it contains cannot be changed. So, immutability is not\nstrictly the same as having an unchangeable value, it is more subtle.)\nAn object\'s mutability is determined by its type; for instance,\nnumbers, strings and tuples are immutable, while dictionaries and\nlists are mutable.\n\nObjects are never explicitly destroyed; however, when they become\nunreachable they may be garbage-collected. An implementation is\nallowed to postpone garbage collection or omit it altogether --- it is\na matter of implementation quality how garbage collection is\nimplemented, as long as no objects are collected that are still\nreachable.\n\n**CPython implementation detail:** CPython currently uses a reference-\ncounting scheme with (optional) delayed detection of cyclically linked\ngarbage, which collects most objects as soon as they become\nunreachable, but is not guaranteed to collect garbage containing\ncircular references. See the documentation of the ``gc`` module for\ninformation on controlling the collection of cyclic garbage. Other\nimplementations act differently and CPython may change.\n\nNote that the use of the implementation\'s tracing or debugging\nfacilities may keep objects alive that would normally be collectable.\nAlso note that catching an exception with a \'``try``...``except``\'\nstatement may keep objects alive.\n\nSome objects contain references to "external" resources such as open\nfiles or windows. It is understood that these resources are freed\nwhen the object is garbage-collected, but since garbage collection is\nnot guaranteed to happen, such objects also provide an explicit way to\nrelease the external resource, usually a ``close()`` method. Programs\nare strongly recommended to explicitly close such objects. The\n\'``try``...``finally``\' statement provides a convenient way to do\nthis.\n\nSome objects contain references to other objects; these are called\n*containers*. Examples of containers are tuples, lists and\ndictionaries. The references are part of a container\'s value. In\nmost cases, when we talk about the value of a container, we imply the\nvalues, not the identities of the contained objects; however, when we\ntalk about the mutability of a container, only the identities of the\nimmediately contained objects are implied. So, if an immutable\ncontainer (like a tuple) contains a reference to a mutable object, its\nvalue changes if that mutable object is changed.\n\nTypes affect almost all aspects of object behavior. Even the\nimportance of object identity is affected in some sense: for immutable\ntypes, operations that compute new values may actually return a\nreference to any existing object with the same type and value, while\nfor mutable objects this is not allowed. E.g., after ``a = 1; b =\n1``, ``a`` and ``b`` may or may not refer to the same object with the\nvalue one, depending on the implementation, but after ``c = []; d =\n[]``, ``c`` and ``d`` are guaranteed to refer to two different,\nunique, newly created empty lists. (Note that ``c = d = []`` assigns\nthe same object to both ``c`` and ``d``.)\n', - 'operator-summary': u'\nSummary\n*******\n\nThe following table summarizes the operator precedences in Python,\nfrom lowest precedence (least binding) to highest precedence (most\nbinding). Operators in the same box have the same precedence. Unless\nthe syntax is explicitly given, operators are binary. Operators in\nthe same box group left to right (except for comparisons, including\ntests, which all have the same precedence and chain from left to right\n--- see section *Comparisons* --- and exponentiation, which groups\nfrom right to left).\n\n+-------------------------------------------------+---------------------------------------+\n| Operator | Description |\n+=================================================+=======================================+\n| ``lambda`` | Lambda expression |\n+-------------------------------------------------+---------------------------------------+\n| ``if`` -- ``else`` | Conditional expression |\n+-------------------------------------------------+---------------------------------------+\n| ``or`` | Boolean OR |\n+-------------------------------------------------+---------------------------------------+\n| ``and`` | Boolean AND |\n+-------------------------------------------------+---------------------------------------+\n| ``not`` *x* | Boolean NOT |\n+-------------------------------------------------+---------------------------------------+\n| ``in``, ``not`` ``in``, ``is``, ``is not``, | Comparisons, including membership |\n| ``<``, ``<=``, ``>``, ``>=``, ``<>``, ``!=``, | tests and identity tests, |\n| ``==`` | |\n+-------------------------------------------------+---------------------------------------+\n| ``|`` | Bitwise OR |\n+-------------------------------------------------+---------------------------------------+\n| ``^`` | Bitwise XOR |\n+-------------------------------------------------+---------------------------------------+\n| ``&`` | Bitwise AND |\n+-------------------------------------------------+---------------------------------------+\n| ``<<``, ``>>`` | Shifts |\n+-------------------------------------------------+---------------------------------------+\n| ``+``, ``-`` | Addition and subtraction |\n+-------------------------------------------------+---------------------------------------+\n| ``*``, ``/``, ``//``, ``%`` | Multiplication, division, remainder |\n+-------------------------------------------------+---------------------------------------+\n| ``+x``, ``-x``, ``~x`` | Positive, negative, bitwise NOT |\n+-------------------------------------------------+---------------------------------------+\n| ``**`` | Exponentiation [8] |\n+-------------------------------------------------+---------------------------------------+\n| ``x[index]``, ``x[index:index]``, | Subscription, slicing, call, |\n| ``x(arguments...)``, ``x.attribute`` | attribute reference |\n+-------------------------------------------------+---------------------------------------+\n| ``(expressions...)``, ``[expressions...]``, | Binding or tuple display, list |\n| ``{key:datum...}``, ```expressions...``` | display, dictionary display, string |\n| | conversion |\n+-------------------------------------------------+---------------------------------------+\n\n-[ Footnotes ]-\n\n[1] In Python 2.3 and later releases, a list comprehension "leaks" the\n control variables of each ``for`` it contains into the containing\n scope. However, this behavior is deprecated, and relying on it\n will not work in Python 3.0\n\n[2] While ``abs(x%y) < abs(y)`` is true mathematically, for floats it\n may not be true numerically due to roundoff. For example, and\n assuming a platform on which a Python float is an IEEE 754 double-\n precision number, in order that ``-1e-100 % 1e100`` have the same\n sign as ``1e100``, the computed result is ``-1e-100 + 1e100``,\n which is numerically exactly equal to ``1e100``. Function\n ``fmod()`` in the ``math`` module returns a result whose sign\n matches the sign of the first argument instead, and so returns\n ``-1e-100`` in this case. Which approach is more appropriate\n depends on the application.\n\n[3] If x is very close to an exact integer multiple of y, it\'s\n possible for ``floor(x/y)`` to be one larger than ``(x-x%y)/y``\n due to rounding. In such cases, Python returns the latter result,\n in order to preserve that ``divmod(x,y)[0] * y + x % y`` be very\n close to ``x``.\n\n[4] While comparisons between unicode strings make sense at the byte\n level, they may be counter-intuitive to users. For example, the\n strings ``u"\\u00C7"`` and ``u"\\u0043\\u0327"`` compare differently,\n even though they both represent the same unicode character (LATIN\n CAPITAL LETTER C WITH CEDILLA). To compare strings in a human\n recognizable way, compare using ``unicodedata.normalize()``.\n\n[5] The implementation computes this efficiently, without constructing\n lists or sorting.\n\n[6] Earlier versions of Python used lexicographic comparison of the\n sorted (key, value) lists, but this was very expensive for the\n common case of comparing for equality. An even earlier version of\n Python compared dictionaries by identity only, but this caused\n surprises because people expected to be able to test a dictionary\n for emptiness by comparing it to ``{}``.\n\n[7] Due to automatic garbage-collection, free lists, and the dynamic\n nature of descriptors, you may notice seemingly unusual behaviour\n in certain uses of the ``is`` operator, like those involving\n comparisons between instance methods, or constants. Check their\n documentation for more info.\n\n[8] The power operator ``**`` binds less tightly than an arithmetic or\n bitwise unary operator on its right, that is, ``2**-1`` is\n ``0.5``.\n', + 'objects': u'\nObjects, values and types\n*************************\n\n*Objects* are Python\'s abstraction for data. All data in a Python\nprogram is represented by objects or by relations between objects. (In\na sense, and in conformance to Von Neumann\'s model of a "stored\nprogram computer," code is also represented by objects.)\n\nEvery object has an identity, a type and a value. An object\'s\n*identity* never changes once it has been created; you may think of it\nas the object\'s address in memory. The \'``is``\' operator compares the\nidentity of two objects; the ``id()`` function returns an integer\nrepresenting its identity (currently implemented as its address). An\nobject\'s *type* is also unchangeable. [1] An object\'s type determines\nthe operations that the object supports (e.g., "does it have a\nlength?") and also defines the possible values for objects of that\ntype. The ``type()`` function returns an object\'s type (which is an\nobject itself). The *value* of some objects can change. Objects\nwhose value can change are said to be *mutable*; objects whose value\nis unchangeable once they are created are called *immutable*. (The\nvalue of an immutable container object that contains a reference to a\nmutable object can change when the latter\'s value is changed; however\nthe container is still considered immutable, because the collection of\nobjects it contains cannot be changed. So, immutability is not\nstrictly the same as having an unchangeable value, it is more subtle.)\nAn object\'s mutability is determined by its type; for instance,\nnumbers, strings and tuples are immutable, while dictionaries and\nlists are mutable.\n\nObjects are never explicitly destroyed; however, when they become\nunreachable they may be garbage-collected. An implementation is\nallowed to postpone garbage collection or omit it altogether --- it is\na matter of implementation quality how garbage collection is\nimplemented, as long as no objects are collected that are still\nreachable.\n\n**CPython implementation detail:** CPython currently uses a reference-\ncounting scheme with (optional) delayed detection of cyclically linked\ngarbage, which collects most objects as soon as they become\nunreachable, but is not guaranteed to collect garbage containing\ncircular references. See the documentation of the ``gc`` module for\ninformation on controlling the collection of cyclic garbage. Other\nimplementations act differently and CPython may change. Do not depend\non immediate finalization of objects when they become unreachable (ex:\nalways close files).\n\nNote that the use of the implementation\'s tracing or debugging\nfacilities may keep objects alive that would normally be collectable.\nAlso note that catching an exception with a \'``try``...``except``\'\nstatement may keep objects alive.\n\nSome objects contain references to "external" resources such as open\nfiles or windows. It is understood that these resources are freed\nwhen the object is garbage-collected, but since garbage collection is\nnot guaranteed to happen, such objects also provide an explicit way to\nrelease the external resource, usually a ``close()`` method. Programs\nare strongly recommended to explicitly close such objects. The\n\'``try``...``finally``\' statement provides a convenient way to do\nthis.\n\nSome objects contain references to other objects; these are called\n*containers*. Examples of containers are tuples, lists and\ndictionaries. The references are part of a container\'s value. In\nmost cases, when we talk about the value of a container, we imply the\nvalues, not the identities of the contained objects; however, when we\ntalk about the mutability of a container, only the identities of the\nimmediately contained objects are implied. So, if an immutable\ncontainer (like a tuple) contains a reference to a mutable object, its\nvalue changes if that mutable object is changed.\n\nTypes affect almost all aspects of object behavior. Even the\nimportance of object identity is affected in some sense: for immutable\ntypes, operations that compute new values may actually return a\nreference to any existing object with the same type and value, while\nfor mutable objects this is not allowed. E.g., after ``a = 1; b =\n1``, ``a`` and ``b`` may or may not refer to the same object with the\nvalue one, depending on the implementation, but after ``c = []; d =\n[]``, ``c`` and ``d`` are guaranteed to refer to two different,\nunique, newly created empty lists. (Note that ``c = d = []`` assigns\nthe same object to both ``c`` and ``d``.)\n', + 'operator-summary': u'\nSummary\n*******\n\nThe following table summarizes the operator precedences in Python,\nfrom lowest precedence (least binding) to highest precedence (most\nbinding). Operators in the same box have the same precedence. Unless\nthe syntax is explicitly given, operators are binary. Operators in\nthe same box group left to right (except for comparisons, including\ntests, which all have the same precedence and chain from left to right\n--- see section *Comparisons* --- and exponentiation, which groups\nfrom right to left).\n\n+-------------------------------------------------+---------------------------------------+\n| Operator | Description |\n+=================================================+=======================================+\n| ``lambda`` | Lambda expression |\n+-------------------------------------------------+---------------------------------------+\n| ``if`` -- ``else`` | Conditional expression |\n+-------------------------------------------------+---------------------------------------+\n| ``or`` | Boolean OR |\n+-------------------------------------------------+---------------------------------------+\n| ``and`` | Boolean AND |\n+-------------------------------------------------+---------------------------------------+\n| ``not`` *x* | Boolean NOT |\n+-------------------------------------------------+---------------------------------------+\n| ``in``, ``not`` ``in``, ``is``, ``is not``, | Comparisons, including membership |\n| ``<``, ``<=``, ``>``, ``>=``, ``<>``, ``!=``, | tests and identity tests, |\n| ``==`` | |\n+-------------------------------------------------+---------------------------------------+\n| ``|`` | Bitwise OR |\n+-------------------------------------------------+---------------------------------------+\n| ``^`` | Bitwise XOR |\n+-------------------------------------------------+---------------------------------------+\n| ``&`` | Bitwise AND |\n+-------------------------------------------------+---------------------------------------+\n| ``<<``, ``>>`` | Shifts |\n+-------------------------------------------------+---------------------------------------+\n| ``+``, ``-`` | Addition and subtraction |\n+-------------------------------------------------+---------------------------------------+\n| ``*``, ``/``, ``//``, ``%`` | Multiplication, division, remainder |\n| | [8] |\n+-------------------------------------------------+---------------------------------------+\n| ``+x``, ``-x``, ``~x`` | Positive, negative, bitwise NOT |\n+-------------------------------------------------+---------------------------------------+\n| ``**`` | Exponentiation [9] |\n+-------------------------------------------------+---------------------------------------+\n| ``x[index]``, ``x[index:index]``, | Subscription, slicing, call, |\n| ``x(arguments...)``, ``x.attribute`` | attribute reference |\n+-------------------------------------------------+---------------------------------------+\n| ``(expressions...)``, ``[expressions...]``, | Binding or tuple display, list |\n| ``{key:datum...}``, ```expressions...``` | display, dictionary display, string |\n| | conversion |\n+-------------------------------------------------+---------------------------------------+\n\n-[ Footnotes ]-\n\n[1] In Python 2.3 and later releases, a list comprehension "leaks" the\n control variables of each ``for`` it contains into the containing\n scope. However, this behavior is deprecated, and relying on it\n will not work in Python 3.0\n\n[2] While ``abs(x%y) < abs(y)`` is true mathematically, for floats it\n may not be true numerically due to roundoff. For example, and\n assuming a platform on which a Python float is an IEEE 754 double-\n precision number, in order that ``-1e-100 % 1e100`` have the same\n sign as ``1e100``, the computed result is ``-1e-100 + 1e100``,\n which is numerically exactly equal to ``1e100``. The function\n ``math.fmod()`` returns a result whose sign matches the sign of\n the first argument instead, and so returns ``-1e-100`` in this\n case. Which approach is more appropriate depends on the\n application.\n\n[3] If x is very close to an exact integer multiple of y, it\'s\n possible for ``floor(x/y)`` to be one larger than ``(x-x%y)/y``\n due to rounding. In such cases, Python returns the latter result,\n in order to preserve that ``divmod(x,y)[0] * y + x % y`` be very\n close to ``x``.\n\n[4] While comparisons between unicode strings make sense at the byte\n level, they may be counter-intuitive to users. For example, the\n strings ``u"\\u00C7"`` and ``u"\\u0043\\u0327"`` compare differently,\n even though they both represent the same unicode character (LATIN\n CAPITAL LETTER C WITH CEDILLA). To compare strings in a human\n recognizable way, compare using ``unicodedata.normalize()``.\n\n[5] The implementation computes this efficiently, without constructing\n lists or sorting.\n\n[6] Earlier versions of Python used lexicographic comparison of the\n sorted (key, value) lists, but this was very expensive for the\n common case of comparing for equality. An even earlier version of\n Python compared dictionaries by identity only, but this caused\n surprises because people expected to be able to test a dictionary\n for emptiness by comparing it to ``{}``.\n\n[7] Due to automatic garbage-collection, free lists, and the dynamic\n nature of descriptors, you may notice seemingly unusual behaviour\n in certain uses of the ``is`` operator, like those involving\n comparisons between instance methods, or constants. Check their\n documentation for more info.\n\n[8] The ``%`` operator is also used for string formatting; the same\n precedence applies.\n\n[9] The power operator ``**`` binds less tightly than an arithmetic or\n bitwise unary operator on its right, that is, ``2**-1`` is\n ``0.5``.\n', 'pass': u'\nThe ``pass`` statement\n**********************\n\n pass_stmt ::= "pass"\n\n``pass`` is a null operation --- when it is executed, nothing happens.\nIt is useful as a placeholder when a statement is required\nsyntactically, but no code needs to be executed, for example:\n\n def f(arg): pass # a function that does nothing (yet)\n\n class C: pass # a class with no methods (yet)\n', 'power': u'\nThe power operator\n******************\n\nThe power operator binds more tightly than unary operators on its\nleft; it binds less tightly than unary operators on its right. The\nsyntax is:\n\n power ::= primary ["**" u_expr]\n\nThus, in an unparenthesized sequence of power and unary operators, the\noperators are evaluated from right to left (this does not constrain\nthe evaluation order for the operands): ``-1**2`` results in ``-1``.\n\nThe power operator has the same semantics as the built-in ``pow()``\nfunction, when called with two arguments: it yields its left argument\nraised to the power of its right argument. The numeric arguments are\nfirst converted to a common type. The result type is that of the\narguments after coercion.\n\nWith mixed operand types, the coercion rules for binary arithmetic\noperators apply. For int and long int operands, the result has the\nsame type as the operands (after coercion) unless the second argument\nis negative; in that case, all arguments are converted to float and a\nfloat result is delivered. For example, ``10**2`` returns ``100``, but\n``10**-2`` returns ``0.01``. (This last feature was added in Python\n2.2. In Python 2.1 and before, if both arguments were of integer types\nand the second argument was negative, an exception was raised).\n\nRaising ``0.0`` to a negative power results in a\n``ZeroDivisionError``. Raising a negative number to a fractional power\nresults in a ``ValueError``.\n', 'print': u'\nThe ``print`` statement\n***********************\n\n print_stmt ::= "print" ([expression ("," expression)* [","]]\n | ">>" expression [("," expression)+ [","]])\n\n``print`` evaluates each expression in turn and writes the resulting\nobject to standard output (see below). If an object is not a string,\nit is first converted to a string using the rules for string\nconversions. The (resulting or original) string is then written. A\nspace is written before each object is (converted and) written, unless\nthe output system believes it is positioned at the beginning of a\nline. This is the case (1) when no characters have yet been written\nto standard output, (2) when the last character written to standard\noutput is a whitespace character except ``\' \'``, or (3) when the last\nwrite operation on standard output was not a ``print`` statement. (In\nsome cases it may be functional to write an empty string to standard\noutput for this reason.)\n\nNote: Objects which act like file objects but which are not the built-in\n file objects often do not properly emulate this aspect of the file\n object\'s behavior, so it is best not to rely on this.\n\nA ``\'\\n\'`` character is written at the end, unless the ``print``\nstatement ends with a comma. This is the only action if the statement\ncontains just the keyword ``print``.\n\nStandard output is defined as the file object named ``stdout`` in the\nbuilt-in module ``sys``. If no such object exists, or if it does not\nhave a ``write()`` method, a ``RuntimeError`` exception is raised.\n\n``print`` also has an extended form, defined by the second portion of\nthe syntax described above. This form is sometimes referred to as\n"``print`` chevron." In this form, the first expression after the\n``>>`` must evaluate to a "file-like" object, specifically an object\nthat has a ``write()`` method as described above. With this extended\nform, the subsequent expressions are printed to this file object. If\nthe first expression evaluates to ``None``, then ``sys.stdout`` is\nused as the file for output.\n', @@ -63,21 +63,21 @@ 'shifting': u'\nShifting operations\n*******************\n\nThe shifting operations have lower priority than the arithmetic\noperations:\n\n shift_expr ::= a_expr | shift_expr ( "<<" | ">>" ) a_expr\n\nThese operators accept plain or long integers as arguments. The\narguments are converted to a common type. They shift the first\nargument to the left or right by the number of bits given by the\nsecond argument.\n\nA right shift by *n* bits is defined as division by ``pow(2, n)``. A\nleft shift by *n* bits is defined as multiplication with ``pow(2,\nn)``. Negative shift counts raise a ``ValueError`` exception.\n\nNote: In the current implementation, the right-hand operand is required to\n be at most ``sys.maxsize``. If the right-hand operand is larger\n than ``sys.maxsize`` an ``OverflowError`` exception is raised.\n', 'slicings': u'\nSlicings\n********\n\nA slicing selects a range of items in a sequence object (e.g., a\nstring, tuple or list). Slicings may be used as expressions or as\ntargets in assignment or ``del`` statements. The syntax for a\nslicing:\n\n slicing ::= simple_slicing | extended_slicing\n simple_slicing ::= primary "[" short_slice "]"\n extended_slicing ::= primary "[" slice_list "]"\n slice_list ::= slice_item ("," slice_item)* [","]\n slice_item ::= expression | proper_slice | ellipsis\n proper_slice ::= short_slice | long_slice\n short_slice ::= [lower_bound] ":" [upper_bound]\n long_slice ::= short_slice ":" [stride]\n lower_bound ::= expression\n upper_bound ::= expression\n stride ::= expression\n ellipsis ::= "..."\n\nThere is ambiguity in the formal syntax here: anything that looks like\nan expression list also looks like a slice list, so any subscription\ncan be interpreted as a slicing. Rather than further complicating the\nsyntax, this is disambiguated by defining that in this case the\ninterpretation as a subscription takes priority over the\ninterpretation as a slicing (this is the case if the slice list\ncontains no proper slice nor ellipses). Similarly, when the slice\nlist has exactly one short slice and no trailing comma, the\ninterpretation as a simple slicing takes priority over that as an\nextended slicing.\n\nThe semantics for a simple slicing are as follows. The primary must\nevaluate to a sequence object. The lower and upper bound expressions,\nif present, must evaluate to plain integers; defaults are zero and the\n``sys.maxint``, respectively. If either bound is negative, the\nsequence\'s length is added to it. The slicing now selects all items\nwith index *k* such that ``i <= k < j`` where *i* and *j* are the\nspecified lower and upper bounds. This may be an empty sequence. It\nis not an error if *i* or *j* lie outside the range of valid indexes\n(such items don\'t exist so they aren\'t selected).\n\nThe semantics for an extended slicing are as follows. The primary\nmust evaluate to a mapping object, and it is indexed with a key that\nis constructed from the slice list, as follows. If the slice list\ncontains at least one comma, the key is a tuple containing the\nconversion of the slice items; otherwise, the conversion of the lone\nslice item is the key. The conversion of a slice item that is an\nexpression is that expression. The conversion of an ellipsis slice\nitem is the built-in ``Ellipsis`` object. The conversion of a proper\nslice is a slice object (see section *The standard type hierarchy*)\nwhose ``start``, ``stop`` and ``step`` attributes are the values of\nthe expressions given as lower bound, upper bound and stride,\nrespectively, substituting ``None`` for missing expressions.\n', 'specialattrs': u"\nSpecial Attributes\n******************\n\nThe implementation adds a few special read-only attributes to several\nobject types, where they are relevant. Some of these are not reported\nby the ``dir()`` built-in function.\n\nobject.__dict__\n\n A dictionary or other mapping object used to store an object's\n (writable) attributes.\n\nobject.__methods__\n\n Deprecated since version 2.2: Use the built-in function ``dir()``\n to get a list of an object's attributes. This attribute is no\n longer available.\n\nobject.__members__\n\n Deprecated since version 2.2: Use the built-in function ``dir()``\n to get a list of an object's attributes. This attribute is no\n longer available.\n\ninstance.__class__\n\n The class to which a class instance belongs.\n\nclass.__bases__\n\n The tuple of base classes of a class object.\n\nclass.__name__\n\n The name of the class or type.\n\nThe following attributes are only supported by *new-style class*es.\n\nclass.__mro__\n\n This attribute is a tuple of classes that are considered when\n looking for base classes during method resolution.\n\nclass.mro()\n\n This method can be overridden by a metaclass to customize the\n method resolution order for its instances. It is called at class\n instantiation, and its result is stored in ``__mro__``.\n\nclass.__subclasses__()\n\n Each new-style class keeps a list of weak references to its\n immediate subclasses. This method returns a list of all those\n references still alive. Example:\n\n >>> int.__subclasses__()\n [<type 'bool'>]\n\n-[ Footnotes ]-\n\n[1] Additional information on these special methods may be found in\n the Python Reference Manual (*Basic customization*).\n\n[2] As a consequence, the list ``[1, 2]`` is considered equal to\n ``[1.0, 2.0]``, and similarly for tuples.\n\n[3] They must have since the parser can't tell the type of the\n operands.\n\n[4] To format only a tuple you should therefore provide a singleton\n tuple whose only element is the tuple to be formatted.\n\n[5] The advantage of leaving the newline on is that returning an empty\n string is then an unambiguous EOF indication. It is also possible\n (in cases where it might matter, for example, if you want to make\n an exact copy of a file while scanning its lines) to tell whether\n the last line of a file ended in a newline or not (yes this\n happens!).\n", - 'specialnames': u'\nSpecial method names\n********************\n\nA class can implement certain operations that are invoked by special\nsyntax (such as arithmetic operations or subscripting and slicing) by\ndefining methods with special names. This is Python\'s approach to\n*operator overloading*, allowing classes to define their own behavior\nwith respect to language operators. For instance, if a class defines\na method named ``__getitem__()``, and ``x`` is an instance of this\nclass, then ``x[i]`` is roughly equivalent to ``x.__getitem__(i)`` for\nold-style classes and ``type(x).__getitem__(x, i)`` for new-style\nclasses. Except where mentioned, attempts to execute an operation\nraise an exception when no appropriate method is defined (typically\n``AttributeError`` or ``TypeError``).\n\nWhen implementing a class that emulates any built-in type, it is\nimportant that the emulation only be implemented to the degree that it\nmakes sense for the object being modelled. For example, some\nsequences may work well with retrieval of individual elements, but\nextracting a slice may not make sense. (One example of this is the\n``NodeList`` interface in the W3C\'s Document Object Model.)\n\n\nBasic customization\n===================\n\nobject.__new__(cls[, ...])\n\n Called to create a new instance of class *cls*. ``__new__()`` is a\n static method (special-cased so you need not declare it as such)\n that takes the class of which an instance was requested as its\n first argument. The remaining arguments are those passed to the\n object constructor expression (the call to the class). The return\n value of ``__new__()`` should be the new object instance (usually\n an instance of *cls*).\n\n Typical implementations create a new instance of the class by\n invoking the superclass\'s ``__new__()`` method using\n ``super(currentclass, cls).__new__(cls[, ...])`` with appropriate\n arguments and then modifying the newly-created instance as\n necessary before returning it.\n\n If ``__new__()`` returns an instance of *cls*, then the new\n instance\'s ``__init__()`` method will be invoked like\n ``__init__(self[, ...])``, where *self* is the new instance and the\n remaining arguments are the same as were passed to ``__new__()``.\n\n If ``__new__()`` does not return an instance of *cls*, then the new\n instance\'s ``__init__()`` method will not be invoked.\n\n ``__new__()`` is intended mainly to allow subclasses of immutable\n types (like int, str, or tuple) to customize instance creation. It\n is also commonly overridden in custom metaclasses in order to\n customize class creation.\n\nobject.__init__(self[, ...])\n\n Called when the instance is created. The arguments are those\n passed to the class constructor expression. If a base class has an\n ``__init__()`` method, the derived class\'s ``__init__()`` method,\n if any, must explicitly call it to ensure proper initialization of\n the base class part of the instance; for example:\n ``BaseClass.__init__(self, [args...])``. As a special constraint\n on constructors, no value may be returned; doing so will cause a\n ``TypeError`` to be raised at runtime.\n\nobject.__del__(self)\n\n Called when the instance is about to be destroyed. This is also\n called a destructor. If a base class has a ``__del__()`` method,\n the derived class\'s ``__del__()`` method, if any, must explicitly\n call it to ensure proper deletion of the base class part of the\n instance. Note that it is possible (though not recommended!) for\n the ``__del__()`` method to postpone destruction of the instance by\n creating a new reference to it. It may then be called at a later\n time when this new reference is deleted. It is not guaranteed that\n ``__del__()`` methods are called for objects that still exist when\n the interpreter exits.\n\n Note: ``del x`` doesn\'t directly call ``x.__del__()`` --- the former\n decrements the reference count for ``x`` by one, and the latter\n is only called when ``x``\'s reference count reaches zero. Some\n common situations that may prevent the reference count of an\n object from going to zero include: circular references between\n objects (e.g., a doubly-linked list or a tree data structure with\n parent and child pointers); a reference to the object on the\n stack frame of a function that caught an exception (the traceback\n stored in ``sys.exc_traceback`` keeps the stack frame alive); or\n a reference to the object on the stack frame that raised an\n unhandled exception in interactive mode (the traceback stored in\n ``sys.last_traceback`` keeps the stack frame alive). The first\n situation can only be remedied by explicitly breaking the cycles;\n the latter two situations can be resolved by storing ``None`` in\n ``sys.exc_traceback`` or ``sys.last_traceback``. Circular\n references which are garbage are detected when the option cycle\n detector is enabled (it\'s on by default), but can only be cleaned\n up if there are no Python-level ``__del__()`` methods involved.\n Refer to the documentation for the ``gc`` module for more\n information about how ``__del__()`` methods are handled by the\n cycle detector, particularly the description of the ``garbage``\n value.\n\n Warning: Due to the precarious circumstances under which ``__del__()``\n methods are invoked, exceptions that occur during their execution\n are ignored, and a warning is printed to ``sys.stderr`` instead.\n Also, when ``__del__()`` is invoked in response to a module being\n deleted (e.g., when execution of the program is done), other\n globals referenced by the ``__del__()`` method may already have\n been deleted or in the process of being torn down (e.g. the\n import machinery shutting down). For this reason, ``__del__()``\n methods should do the absolute minimum needed to maintain\n external invariants. Starting with version 1.5, Python\n guarantees that globals whose name begins with a single\n underscore are deleted from their module before other globals are\n deleted; if no other references to such globals exist, this may\n help in assuring that imported modules are still available at the\n time when the ``__del__()`` method is called.\n\nobject.__repr__(self)\n\n Called by the ``repr()`` built-in function and by string\n conversions (reverse quotes) to compute the "official" string\n representation of an object. If at all possible, this should look\n like a valid Python expression that could be used to recreate an\n object with the same value (given an appropriate environment). If\n this is not possible, a string of the form ``<...some useful\n description...>`` should be returned. The return value must be a\n string object. If a class defines ``__repr__()`` but not\n ``__str__()``, then ``__repr__()`` is also used when an "informal"\n string representation of instances of that class is required.\n\n This is typically used for debugging, so it is important that the\n representation is information-rich and unambiguous.\n\nobject.__str__(self)\n\n Called by the ``str()`` built-in function and by the ``print``\n statement to compute the "informal" string representation of an\n object. This differs from ``__repr__()`` in that it does not have\n to be a valid Python expression: a more convenient or concise\n representation may be used instead. The return value must be a\n string object.\n\nobject.__lt__(self, other)\nobject.__le__(self, other)\nobject.__eq__(self, other)\nobject.__ne__(self, other)\nobject.__gt__(self, other)\nobject.__ge__(self, other)\n\n New in version 2.1.\n\n These are the so-called "rich comparison" methods, and are called\n for comparison operators in preference to ``__cmp__()`` below. The\n correspondence between operator symbols and method names is as\n follows: ``x<y`` calls ``x.__lt__(y)``, ``x<=y`` calls\n ``x.__le__(y)``, ``x==y`` calls ``x.__eq__(y)``, ``x!=y`` and\n ``x<>y`` call ``x.__ne__(y)``, ``x>y`` calls ``x.__gt__(y)``, and\n ``x>=y`` calls ``x.__ge__(y)``.\n\n A rich comparison method may return the singleton\n ``NotImplemented`` if it does not implement the operation for a\n given pair of arguments. By convention, ``False`` and ``True`` are\n returned for a successful comparison. However, these methods can\n return any value, so if the comparison operator is used in a\n Boolean context (e.g., in the condition of an ``if`` statement),\n Python will call ``bool()`` on the value to determine if the result\n is true or false.\n\n There are no implied relationships among the comparison operators.\n The truth of ``x==y`` does not imply that ``x!=y`` is false.\n Accordingly, when defining ``__eq__()``, one should also define\n ``__ne__()`` so that the operators will behave as expected. See\n the paragraph on ``__hash__()`` for some important notes on\n creating *hashable* objects which support custom comparison\n operations and are usable as dictionary keys.\n\n There are no swapped-argument versions of these methods (to be used\n when the left argument does not support the operation but the right\n argument does); rather, ``__lt__()`` and ``__gt__()`` are each\n other\'s reflection, ``__le__()`` and ``__ge__()`` are each other\'s\n reflection, and ``__eq__()`` and ``__ne__()`` are their own\n reflection.\n\n Arguments to rich comparison methods are never coerced.\n\n To automatically generate ordering operations from a single root\n operation, see ``functools.total_ordering()``.\n\nobject.__cmp__(self, other)\n\n Called by comparison operations if rich comparison (see above) is\n not defined. Should return a negative integer if ``self < other``,\n zero if ``self == other``, a positive integer if ``self > other``.\n If no ``__cmp__()``, ``__eq__()`` or ``__ne__()`` operation is\n defined, class instances are compared by object identity\n ("address"). See also the description of ``__hash__()`` for some\n important notes on creating *hashable* objects which support custom\n comparison operations and are usable as dictionary keys. (Note: the\n restriction that exceptions are not propagated by ``__cmp__()`` has\n been removed since Python 1.5.)\n\nobject.__rcmp__(self, other)\n\n Changed in version 2.1: No longer supported.\n\nobject.__hash__(self)\n\n Called by built-in function ``hash()`` and for operations on\n members of hashed collections including ``set``, ``frozenset``, and\n ``dict``. ``__hash__()`` should return an integer. The only\n required property is that objects which compare equal have the same\n hash value; it is advised to somehow mix together (e.g. using\n exclusive or) the hash values for the components of the object that\n also play a part in comparison of objects.\n\n If a class does not define a ``__cmp__()`` or ``__eq__()`` method\n it should not define a ``__hash__()`` operation either; if it\n defines ``__cmp__()`` or ``__eq__()`` but not ``__hash__()``, its\n instances will not be usable in hashed collections. If a class\n defines mutable objects and implements a ``__cmp__()`` or\n ``__eq__()`` method, it should not implement ``__hash__()``, since\n hashable collection implementations require that a object\'s hash\n value is immutable (if the object\'s hash value changes, it will be\n in the wrong hash bucket).\n\n User-defined classes have ``__cmp__()`` and ``__hash__()`` methods\n by default; with them, all objects compare unequal (except with\n themselves) and ``x.__hash__()`` returns ``id(x)``.\n\n Classes which inherit a ``__hash__()`` method from a parent class\n but change the meaning of ``__cmp__()`` or ``__eq__()`` such that\n the hash value returned is no longer appropriate (e.g. by switching\n to a value-based concept of equality instead of the default\n identity based equality) can explicitly flag themselves as being\n unhashable by setting ``__hash__ = None`` in the class definition.\n Doing so means that not only will instances of the class raise an\n appropriate ``TypeError`` when a program attempts to retrieve their\n hash value, but they will also be correctly identified as\n unhashable when checking ``isinstance(obj, collections.Hashable)``\n (unlike classes which define their own ``__hash__()`` to explicitly\n raise ``TypeError``).\n\n Changed in version 2.5: ``__hash__()`` may now also return a long\n integer object; the 32-bit integer is then derived from the hash of\n that object.\n\n Changed in version 2.6: ``__hash__`` may now be set to ``None`` to\n explicitly flag instances of a class as unhashable.\n\nobject.__nonzero__(self)\n\n Called to implement truth value testing and the built-in operation\n ``bool()``; should return ``False`` or ``True``, or their integer\n equivalents ``0`` or ``1``. When this method is not defined,\n ``__len__()`` is called, if it is defined, and the object is\n considered true if its result is nonzero. If a class defines\n neither ``__len__()`` nor ``__nonzero__()``, all its instances are\n considered true.\n\nobject.__unicode__(self)\n\n Called to implement ``unicode()`` built-in; should return a Unicode\n object. When this method is not defined, string conversion is\n attempted, and the result of string conversion is converted to\n Unicode using the system default encoding.\n\n\nCustomizing attribute access\n============================\n\nThe following methods can be defined to customize the meaning of\nattribute access (use of, assignment to, or deletion of ``x.name``)\nfor class instances.\n\nobject.__getattr__(self, name)\n\n Called when an attribute lookup has not found the attribute in the\n usual places (i.e. it is not an instance attribute nor is it found\n in the class tree for ``self``). ``name`` is the attribute name.\n This method should return the (computed) attribute value or raise\n an ``AttributeError`` exception.\n\n Note that if the attribute is found through the normal mechanism,\n ``__getattr__()`` is not called. (This is an intentional asymmetry\n between ``__getattr__()`` and ``__setattr__()``.) This is done both\n for efficiency reasons and because otherwise ``__getattr__()``\n would have no way to access other attributes of the instance. Note\n that at least for instance variables, you can fake total control by\n not inserting any values in the instance attribute dictionary (but\n instead inserting them in another object). See the\n ``__getattribute__()`` method below for a way to actually get total\n control in new-style classes.\n\nobject.__setattr__(self, name, value)\n\n Called when an attribute assignment is attempted. This is called\n instead of the normal mechanism (i.e. store the value in the\n instance dictionary). *name* is the attribute name, *value* is the\n value to be assigned to it.\n\n If ``__setattr__()`` wants to assign to an instance attribute, it\n should not simply execute ``self.name = value`` --- this would\n cause a recursive call to itself. Instead, it should insert the\n value in the dictionary of instance attributes, e.g.,\n ``self.__dict__[name] = value``. For new-style classes, rather\n than accessing the instance dictionary, it should call the base\n class method with the same name, for example,\n ``object.__setattr__(self, name, value)``.\n\nobject.__delattr__(self, name)\n\n Like ``__setattr__()`` but for attribute deletion instead of\n assignment. This should only be implemented if ``del obj.name`` is\n meaningful for the object.\n\n\nMore attribute access for new-style classes\n-------------------------------------------\n\nThe following methods only apply to new-style classes.\n\nobject.__getattribute__(self, name)\n\n Called unconditionally to implement attribute accesses for\n instances of the class. If the class also defines\n ``__getattr__()``, the latter will not be called unless\n ``__getattribute__()`` either calls it explicitly or raises an\n ``AttributeError``. This method should return the (computed)\n attribute value or raise an ``AttributeError`` exception. In order\n to avoid infinite recursion in this method, its implementation\n should always call the base class method with the same name to\n access any attributes it needs, for example,\n ``object.__getattribute__(self, name)``.\n\n Note: This method may still be bypassed when looking up special methods\n as the result of implicit invocation via language syntax or\n built-in functions. See *Special method lookup for new-style\n classes*.\n\n\nImplementing Descriptors\n------------------------\n\nThe following methods only apply when an instance of the class\ncontaining the method (a so-called *descriptor* class) appears in the\nclass dictionary of another new-style class, known as the *owner*\nclass. In the examples below, "the attribute" refers to the attribute\nwhose name is the key of the property in the owner class\'\n``__dict__``. Descriptors can only be implemented as new-style\nclasses themselves.\n\nobject.__get__(self, instance, owner)\n\n Called to get the attribute of the owner class (class attribute\n access) or of an instance of that class (instance attribute\n access). *owner* is always the owner class, while *instance* is the\n instance that the attribute was accessed through, or ``None`` when\n the attribute is accessed through the *owner*. This method should\n return the (computed) attribute value or raise an\n ``AttributeError`` exception.\n\nobject.__set__(self, instance, value)\n\n Called to set the attribute on an instance *instance* of the owner\n class to a new value, *value*.\n\nobject.__delete__(self, instance)\n\n Called to delete the attribute on an instance *instance* of the\n owner class.\n\n\nInvoking Descriptors\n--------------------\n\nIn general, a descriptor is an object attribute with "binding\nbehavior", one whose attribute access has been overridden by methods\nin the descriptor protocol: ``__get__()``, ``__set__()``, and\n``__delete__()``. If any of those methods are defined for an object,\nit is said to be a descriptor.\n\nThe default behavior for attribute access is to get, set, or delete\nthe attribute from an object\'s dictionary. For instance, ``a.x`` has a\nlookup chain starting with ``a.__dict__[\'x\']``, then\n``type(a).__dict__[\'x\']``, and continuing through the base classes of\n``type(a)`` excluding metaclasses.\n\nHowever, if the looked-up value is an object defining one of the\ndescriptor methods, then Python may override the default behavior and\ninvoke the descriptor method instead. Where this occurs in the\nprecedence chain depends on which descriptor methods were defined and\nhow they were called. Note that descriptors are only invoked for new\nstyle objects or classes (ones that subclass ``object()`` or\n``type()``).\n\nThe starting point for descriptor invocation is a binding, ``a.x``.\nHow the arguments are assembled depends on ``a``:\n\nDirect Call\n The simplest and least common call is when user code directly\n invokes a descriptor method: ``x.__get__(a)``.\n\nInstance Binding\n If binding to a new-style object instance, ``a.x`` is transformed\n into the call: ``type(a).__dict__[\'x\'].__get__(a, type(a))``.\n\nClass Binding\n If binding to a new-style class, ``A.x`` is transformed into the\n call: ``A.__dict__[\'x\'].__get__(None, A)``.\n\nSuper Binding\n If ``a`` is an instance of ``super``, then the binding ``super(B,\n obj).m()`` searches ``obj.__class__.__mro__`` for the base class\n ``A`` immediately preceding ``B`` and then invokes the descriptor\n with the call: ``A.__dict__[\'m\'].__get__(obj, A)``.\n\nFor instance bindings, the precedence of descriptor invocation depends\non the which descriptor methods are defined. A descriptor can define\nany combination of ``__get__()``, ``__set__()`` and ``__delete__()``.\nIf it does not define ``__get__()``, then accessing the attribute will\nreturn the descriptor object itself unless there is a value in the\nobject\'s instance dictionary. If the descriptor defines ``__set__()``\nand/or ``__delete__()``, it is a data descriptor; if it defines\nneither, it is a non-data descriptor. Normally, data descriptors\ndefine both ``__get__()`` and ``__set__()``, while non-data\ndescriptors have just the ``__get__()`` method. Data descriptors with\n``__set__()`` and ``__get__()`` defined always override a redefinition\nin an instance dictionary. In contrast, non-data descriptors can be\noverridden by instances.\n\nPython methods (including ``staticmethod()`` and ``classmethod()``)\nare implemented as non-data descriptors. Accordingly, instances can\nredefine and override methods. This allows individual instances to\nacquire behaviors that differ from other instances of the same class.\n\nThe ``property()`` function is implemented as a data descriptor.\nAccordingly, instances cannot override the behavior of a property.\n\n\n__slots__\n---------\n\nBy default, instances of both old and new-style classes have a\ndictionary for attribute storage. This wastes space for objects\nhaving very few instance variables. The space consumption can become\nacute when creating large numbers of instances.\n\nThe default can be overridden by defining *__slots__* in a new-style\nclass definition. The *__slots__* declaration takes a sequence of\ninstance variables and reserves just enough space in each instance to\nhold a value for each variable. Space is saved because *__dict__* is\nnot created for each instance.\n\n__slots__\n\n This class variable can be assigned a string, iterable, or sequence\n of strings with variable names used by instances. If defined in a\n new-style class, *__slots__* reserves space for the declared\n variables and prevents the automatic creation of *__dict__* and\n *__weakref__* for each instance.\n\n New in version 2.2.\n\nNotes on using *__slots__*\n\n* When inheriting from a class without *__slots__*, the *__dict__*\n attribute of that class will always be accessible, so a *__slots__*\n definition in the subclass is meaningless.\n\n* Without a *__dict__* variable, instances cannot be assigned new\n variables not listed in the *__slots__* definition. Attempts to\n assign to an unlisted variable name raises ``AttributeError``. If\n dynamic assignment of new variables is desired, then add\n ``\'__dict__\'`` to the sequence of strings in the *__slots__*\n declaration.\n\n Changed in version 2.3: Previously, adding ``\'__dict__\'`` to the\n *__slots__* declaration would not enable the assignment of new\n attributes not specifically listed in the sequence of instance\n variable names.\n\n* Without a *__weakref__* variable for each instance, classes defining\n *__slots__* do not support weak references to its instances. If weak\n reference support is needed, then add ``\'__weakref__\'`` to the\n sequence of strings in the *__slots__* declaration.\n\n Changed in version 2.3: Previously, adding ``\'__weakref__\'`` to the\n *__slots__* declaration would not enable support for weak\n references.\n\n* *__slots__* are implemented at the class level by creating\n descriptors (*Implementing Descriptors*) for each variable name. As\n a result, class attributes cannot be used to set default values for\n instance variables defined by *__slots__*; otherwise, the class\n attribute would overwrite the descriptor assignment.\n\n* The action of a *__slots__* declaration is limited to the class\n where it is defined. As a result, subclasses will have a *__dict__*\n unless they also define *__slots__* (which must only contain names\n of any *additional* slots).\n\n* If a class defines a slot also defined in a base class, the instance\n variable defined by the base class slot is inaccessible (except by\n retrieving its descriptor directly from the base class). This\n renders the meaning of the program undefined. In the future, a\n check may be added to prevent this.\n\n* Nonempty *__slots__* does not work for classes derived from\n "variable-length" built-in types such as ``long``, ``str`` and\n ``tuple``.\n\n* Any non-string iterable may be assigned to *__slots__*. Mappings may\n also be used; however, in the future, special meaning may be\n assigned to the values corresponding to each key.\n\n* *__class__* assignment works only if both classes have the same\n *__slots__*.\n\n Changed in version 2.6: Previously, *__class__* assignment raised an\n error if either new or old class had *__slots__*.\n\n\nCustomizing class creation\n==========================\n\nBy default, new-style classes are constructed using ``type()``. A\nclass definition is read into a separate namespace and the value of\nclass name is bound to the result of ``type(name, bases, dict)``.\n\nWhen the class definition is read, if *__metaclass__* is defined then\nthe callable assigned to it will be called instead of ``type()``. This\nallows classes or functions to be written which monitor or alter the\nclass creation process:\n\n* Modifying the class dictionary prior to the class being created.\n\n* Returning an instance of another class -- essentially performing the\n role of a factory function.\n\nThese steps will have to be performed in the metaclass\'s ``__new__()``\nmethod -- ``type.__new__()`` can then be called from this method to\ncreate a class with different properties. This example adds a new\nelement to the class dictionary before creating the class:\n\n class metacls(type):\n def __new__(mcs, name, bases, dict):\n dict[\'foo\'] = \'metacls was here\'\n return type.__new__(mcs, name, bases, dict)\n\nYou can of course also override other class methods (or add new\nmethods); for example defining a custom ``__call__()`` method in the\nmetaclass allows custom behavior when the class is called, e.g. not\nalways creating a new instance.\n\n__metaclass__\n\n This variable can be any callable accepting arguments for ``name``,\n ``bases``, and ``dict``. Upon class creation, the callable is used\n instead of the built-in ``type()``.\n\n New in version 2.2.\n\nThe appropriate metaclass is determined by the following precedence\nrules:\n\n* If ``dict[\'__metaclass__\']`` exists, it is used.\n\n* Otherwise, if there is at least one base class, its metaclass is\n used (this looks for a *__class__* attribute first and if not found,\n uses its type).\n\n* Otherwise, if a global variable named __metaclass__ exists, it is\n used.\n\n* Otherwise, the old-style, classic metaclass (types.ClassType) is\n used.\n\nThe potential uses for metaclasses are boundless. Some ideas that have\nbeen explored including logging, interface checking, automatic\ndelegation, automatic property creation, proxies, frameworks, and\nautomatic resource locking/synchronization.\n\n\nCustomizing instance and subclass checks\n========================================\n\nNew in version 2.6.\n\nThe following methods are used to override the default behavior of the\n``isinstance()`` and ``issubclass()`` built-in functions.\n\nIn particular, the metaclass ``abc.ABCMeta`` implements these methods\nin order to allow the addition of Abstract Base Classes (ABCs) as\n"virtual base classes" to any class or type (including built-in\ntypes), including other ABCs.\n\nclass.__instancecheck__(self, instance)\n\n Return true if *instance* should be considered a (direct or\n indirect) instance of *class*. If defined, called to implement\n ``isinstance(instance, class)``.\n\nclass.__subclasscheck__(self, subclass)\n\n Return true if *subclass* should be considered a (direct or\n indirect) subclass of *class*. If defined, called to implement\n ``issubclass(subclass, class)``.\n\nNote that these methods are looked up on the type (metaclass) of a\nclass. They cannot be defined as class methods in the actual class.\nThis is consistent with the lookup of special methods that are called\non instances, only in this case the instance is itself a class.\n\nSee also:\n\n **PEP 3119** - Introducing Abstract Base Classes\n Includes the specification for customizing ``isinstance()`` and\n ``issubclass()`` behavior through ``__instancecheck__()`` and\n ``__subclasscheck__()``, with motivation for this functionality\n in the context of adding Abstract Base Classes (see the ``abc``\n module) to the language.\n\n\nEmulating callable objects\n==========================\n\nobject.__call__(self[, args...])\n\n Called when the instance is "called" as a function; if this method\n is defined, ``x(arg1, arg2, ...)`` is a shorthand for\n ``x.__call__(arg1, arg2, ...)``.\n\n\nEmulating container types\n=========================\n\nThe following methods can be defined to implement container objects.\nContainers usually are sequences (such as lists or tuples) or mappings\n(like dictionaries), but can represent other containers as well. The\nfirst set of methods is used either to emulate a sequence or to\nemulate a mapping; the difference is that for a sequence, the\nallowable keys should be the integers *k* for which ``0 <= k < N``\nwhere *N* is the length of the sequence, or slice objects, which\ndefine a range of items. (For backwards compatibility, the method\n``__getslice__()`` (see below) can also be defined to handle simple,\nbut not extended slices.) It is also recommended that mappings provide\nthe methods ``keys()``, ``values()``, ``items()``, ``has_key()``,\n``get()``, ``clear()``, ``setdefault()``, ``iterkeys()``,\n``itervalues()``, ``iteritems()``, ``pop()``, ``popitem()``,\n``copy()``, and ``update()`` behaving similar to those for Python\'s\nstandard dictionary objects. The ``UserDict`` module provides a\n``DictMixin`` class to help create those methods from a base set of\n``__getitem__()``, ``__setitem__()``, ``__delitem__()``, and\n``keys()``. Mutable sequences should provide methods ``append()``,\n``count()``, ``index()``, ``extend()``, ``insert()``, ``pop()``,\n``remove()``, ``reverse()`` and ``sort()``, like Python standard list\nobjects. Finally, sequence types should implement addition (meaning\nconcatenation) and multiplication (meaning repetition) by defining the\nmethods ``__add__()``, ``__radd__()``, ``__iadd__()``, ``__mul__()``,\n``__rmul__()`` and ``__imul__()`` described below; they should not\ndefine ``__coerce__()`` or other numerical operators. It is\nrecommended that both mappings and sequences implement the\n``__contains__()`` method to allow efficient use of the ``in``\noperator; for mappings, ``in`` should be equivalent of ``has_key()``;\nfor sequences, it should search through the values. It is further\nrecommended that both mappings and sequences implement the\n``__iter__()`` method to allow efficient iteration through the\ncontainer; for mappings, ``__iter__()`` should be the same as\n``iterkeys()``; for sequences, it should iterate through the values.\n\nobject.__len__(self)\n\n Called to implement the built-in function ``len()``. Should return\n the length of the object, an integer ``>=`` 0. Also, an object\n that doesn\'t define a ``__nonzero__()`` method and whose\n ``__len__()`` method returns zero is considered to be false in a\n Boolean context.\n\nobject.__getitem__(self, key)\n\n Called to implement evaluation of ``self[key]``. For sequence\n types, the accepted keys should be integers and slice objects.\n Note that the special interpretation of negative indexes (if the\n class wishes to emulate a sequence type) is up to the\n ``__getitem__()`` method. If *key* is of an inappropriate type,\n ``TypeError`` may be raised; if of a value outside the set of\n indexes for the sequence (after any special interpretation of\n negative values), ``IndexError`` should be raised. For mapping\n types, if *key* is missing (not in the container), ``KeyError``\n should be raised.\n\n Note: ``for`` loops expect that an ``IndexError`` will be raised for\n illegal indexes to allow proper detection of the end of the\n sequence.\n\nobject.__setitem__(self, key, value)\n\n Called to implement assignment to ``self[key]``. Same note as for\n ``__getitem__()``. This should only be implemented for mappings if\n the objects support changes to the values for keys, or if new keys\n can be added, or for sequences if elements can be replaced. The\n same exceptions should be raised for improper *key* values as for\n the ``__getitem__()`` method.\n\nobject.__delitem__(self, key)\n\n Called to implement deletion of ``self[key]``. Same note as for\n ``__getitem__()``. This should only be implemented for mappings if\n the objects support removal of keys, or for sequences if elements\n can be removed from the sequence. The same exceptions should be\n raised for improper *key* values as for the ``__getitem__()``\n method.\n\nobject.__iter__(self)\n\n This method is called when an iterator is required for a container.\n This method should return a new iterator object that can iterate\n over all the objects in the container. For mappings, it should\n iterate over the keys of the container, and should also be made\n available as the method ``iterkeys()``.\n\n Iterator objects also need to implement this method; they are\n required to return themselves. For more information on iterator\n objects, see *Iterator Types*.\n\nobject.__reversed__(self)\n\n Called (if present) by the ``reversed()`` built-in to implement\n reverse iteration. It should return a new iterator object that\n iterates over all the objects in the container in reverse order.\n\n If the ``__reversed__()`` method is not provided, the\n ``reversed()`` built-in will fall back to using the sequence\n protocol (``__len__()`` and ``__getitem__()``). Objects that\n support the sequence protocol should only provide\n ``__reversed__()`` if they can provide an implementation that is\n more efficient than the one provided by ``reversed()``.\n\n New in version 2.6.\n\nThe membership test operators (``in`` and ``not in``) are normally\nimplemented as an iteration through a sequence. However, container\nobjects can supply the following special method with a more efficient\nimplementation, which also does not require the object be a sequence.\n\nobject.__contains__(self, item)\n\n Called to implement membership test operators. Should return true\n if *item* is in *self*, false otherwise. For mapping objects, this\n should consider the keys of the mapping rather than the values or\n the key-item pairs.\n\n For objects that don\'t define ``__contains__()``, the membership\n test first tries iteration via ``__iter__()``, then the old\n sequence iteration protocol via ``__getitem__()``, see *this\n section in the language reference*.\n\n\nAdditional methods for emulation of sequence types\n==================================================\n\nThe following optional methods can be defined to further emulate\nsequence objects. Immutable sequences methods should at most only\ndefine ``__getslice__()``; mutable sequences might define all three\nmethods.\n\nobject.__getslice__(self, i, j)\n\n Deprecated since version 2.0: Support slice objects as parameters\n to the ``__getitem__()`` method. (However, built-in types in\n CPython currently still implement ``__getslice__()``. Therefore,\n you have to override it in derived classes when implementing\n slicing.)\n\n Called to implement evaluation of ``self[i:j]``. The returned\n object should be of the same type as *self*. Note that missing *i*\n or *j* in the slice expression are replaced by zero or\n ``sys.maxint``, respectively. If negative indexes are used in the\n slice, the length of the sequence is added to that index. If the\n instance does not implement the ``__len__()`` method, an\n ``AttributeError`` is raised. No guarantee is made that indexes\n adjusted this way are not still negative. Indexes which are\n greater than the length of the sequence are not modified. If no\n ``__getslice__()`` is found, a slice object is created instead, and\n passed to ``__getitem__()`` instead.\n\nobject.__setslice__(self, i, j, sequence)\n\n Called to implement assignment to ``self[i:j]``. Same notes for *i*\n and *j* as for ``__getslice__()``.\n\n This method is deprecated. If no ``__setslice__()`` is found, or\n for extended slicing of the form ``self[i:j:k]``, a slice object is\n created, and passed to ``__setitem__()``, instead of\n ``__setslice__()`` being called.\n\nobject.__delslice__(self, i, j)\n\n Called to implement deletion of ``self[i:j]``. Same notes for *i*\n and *j* as for ``__getslice__()``. This method is deprecated. If no\n ``__delslice__()`` is found, or for extended slicing of the form\n ``self[i:j:k]``, a slice object is created, and passed to\n ``__delitem__()``, instead of ``__delslice__()`` being called.\n\nNotice that these methods are only invoked when a single slice with a\nsingle colon is used, and the slice method is available. For slice\noperations involving extended slice notation, or in absence of the\nslice methods, ``__getitem__()``, ``__setitem__()`` or\n``__delitem__()`` is called with a slice object as argument.\n\nThe following example demonstrate how to make your program or module\ncompatible with earlier versions of Python (assuming that methods\n``__getitem__()``, ``__setitem__()`` and ``__delitem__()`` support\nslice objects as arguments):\n\n class MyClass:\n ...\n def __getitem__(self, index):\n ...\n def __setitem__(self, index, value):\n ...\n def __delitem__(self, index):\n ...\n\n if sys.version_info < (2, 0):\n # They won\'t be defined if version is at least 2.0 final\n\n def __getslice__(self, i, j):\n return self[max(0, i):max(0, j):]\n def __setslice__(self, i, j, seq):\n self[max(0, i):max(0, j):] = seq\n def __delslice__(self, i, j):\n del self[max(0, i):max(0, j):]\n ...\n\nNote the calls to ``max()``; these are necessary because of the\nhandling of negative indices before the ``__*slice__()`` methods are\ncalled. When negative indexes are used, the ``__*item__()`` methods\nreceive them as provided, but the ``__*slice__()`` methods get a\n"cooked" form of the index values. For each negative index value, the\nlength of the sequence is added to the index before calling the method\n(which may still result in a negative index); this is the customary\nhandling of negative indexes by the built-in sequence types, and the\n``__*item__()`` methods are expected to do this as well. However,\nsince they should already be doing that, negative indexes cannot be\npassed in; they must be constrained to the bounds of the sequence\nbefore being passed to the ``__*item__()`` methods. Calling ``max(0,\ni)`` conveniently returns the proper value.\n\n\nEmulating numeric types\n=======================\n\nThe following methods can be defined to emulate numeric objects.\nMethods corresponding to operations that are not supported by the\nparticular kind of number implemented (e.g., bitwise operations for\nnon-integral numbers) should be left undefined.\n\nobject.__add__(self, other)\nobject.__sub__(self, other)\nobject.__mul__(self, other)\nobject.__floordiv__(self, other)\nobject.__mod__(self, other)\nobject.__divmod__(self, other)\nobject.__pow__(self, other[, modulo])\nobject.__lshift__(self, other)\nobject.__rshift__(self, other)\nobject.__and__(self, other)\nobject.__xor__(self, other)\nobject.__or__(self, other)\n\n These methods are called to implement the binary arithmetic\n operations (``+``, ``-``, ``*``, ``//``, ``%``, ``divmod()``,\n ``pow()``, ``**``, ``<<``, ``>>``, ``&``, ``^``, ``|``). For\n instance, to evaluate the expression ``x + y``, where *x* is an\n instance of a class that has an ``__add__()`` method,\n ``x.__add__(y)`` is called. The ``__divmod__()`` method should be\n the equivalent to using ``__floordiv__()`` and ``__mod__()``; it\n should not be related to ``__truediv__()`` (described below). Note\n that ``__pow__()`` should be defined to accept an optional third\n argument if the ternary version of the built-in ``pow()`` function\n is to be supported.\n\n If one of those methods does not support the operation with the\n supplied arguments, it should return ``NotImplemented``.\n\nobject.__div__(self, other)\nobject.__truediv__(self, other)\n\n The division operator (``/``) is implemented by these methods. The\n ``__truediv__()`` method is used when ``__future__.division`` is in\n effect, otherwise ``__div__()`` is used. If only one of these two\n methods is defined, the object will not support division in the\n alternate context; ``TypeError`` will be raised instead.\n\nobject.__radd__(self, other)\nobject.__rsub__(self, other)\nobject.__rmul__(self, other)\nobject.__rdiv__(self, other)\nobject.__rtruediv__(self, other)\nobject.__rfloordiv__(self, other)\nobject.__rmod__(self, other)\nobject.__rdivmod__(self, other)\nobject.__rpow__(self, other)\nobject.__rlshift__(self, other)\nobject.__rrshift__(self, other)\nobject.__rand__(self, other)\nobject.__rxor__(self, other)\nobject.__ror__(self, other)\n\n These methods are called to implement the binary arithmetic\n operations (``+``, ``-``, ``*``, ``/``, ``%``, ``divmod()``,\n ``pow()``, ``**``, ``<<``, ``>>``, ``&``, ``^``, ``|``) with\n reflected (swapped) operands. These functions are only called if\n the left operand does not support the corresponding operation and\n the operands are of different types. [2] For instance, to evaluate\n the expression ``x - y``, where *y* is an instance of a class that\n has an ``__rsub__()`` method, ``y.__rsub__(x)`` is called if\n ``x.__sub__(y)`` returns *NotImplemented*.\n\n Note that ternary ``pow()`` will not try calling ``__rpow__()``\n (the coercion rules would become too complicated).\n\n Note: If the right operand\'s type is a subclass of the left operand\'s\n type and that subclass provides the reflected method for the\n operation, this method will be called before the left operand\'s\n non-reflected method. This behavior allows subclasses to\n override their ancestors\' operations.\n\nobject.__iadd__(self, other)\nobject.__isub__(self, other)\nobject.__imul__(self, other)\nobject.__idiv__(self, other)\nobject.__itruediv__(self, other)\nobject.__ifloordiv__(self, other)\nobject.__imod__(self, other)\nobject.__ipow__(self, other[, modulo])\nobject.__ilshift__(self, other)\nobject.__irshift__(self, other)\nobject.__iand__(self, other)\nobject.__ixor__(self, other)\nobject.__ior__(self, other)\n\n These methods are called to implement the augmented arithmetic\n assignments (``+=``, ``-=``, ``*=``, ``/=``, ``//=``, ``%=``,\n ``**=``, ``<<=``, ``>>=``, ``&=``, ``^=``, ``|=``). These methods\n should attempt to do the operation in-place (modifying *self*) and\n return the result (which could be, but does not have to be,\n *self*). If a specific method is not defined, the augmented\n assignment falls back to the normal methods. For instance, to\n execute the statement ``x += y``, where *x* is an instance of a\n class that has an ``__iadd__()`` method, ``x.__iadd__(y)`` is\n called. If *x* is an instance of a class that does not define a\n ``__iadd__()`` method, ``x.__add__(y)`` and ``y.__radd__(x)`` are\n considered, as with the evaluation of ``x + y``.\n\nobject.__neg__(self)\nobject.__pos__(self)\nobject.__abs__(self)\nobject.__invert__(self)\n\n Called to implement the unary arithmetic operations (``-``, ``+``,\n ``abs()`` and ``~``).\n\nobject.__complex__(self)\nobject.__int__(self)\nobject.__long__(self)\nobject.__float__(self)\n\n Called to implement the built-in functions ``complex()``,\n ``int()``, ``long()``, and ``float()``. Should return a value of\n the appropriate type.\n\nobject.__oct__(self)\nobject.__hex__(self)\n\n Called to implement the built-in functions ``oct()`` and ``hex()``.\n Should return a string value.\n\nobject.__index__(self)\n\n Called to implement ``operator.index()``. Also called whenever\n Python needs an integer object (such as in slicing). Must return\n an integer (int or long).\n\n New in version 2.5.\n\nobject.__coerce__(self, other)\n\n Called to implement "mixed-mode" numeric arithmetic. Should either\n return a 2-tuple containing *self* and *other* converted to a\n common numeric type, or ``None`` if conversion is impossible. When\n the common type would be the type of ``other``, it is sufficient to\n return ``None``, since the interpreter will also ask the other\n object to attempt a coercion (but sometimes, if the implementation\n of the other type cannot be changed, it is useful to do the\n conversion to the other type here). A return value of\n ``NotImplemented`` is equivalent to returning ``None``.\n\n\nCoercion rules\n==============\n\nThis section used to document the rules for coercion. As the language\nhas evolved, the coercion rules have become hard to document\nprecisely; documenting what one version of one particular\nimplementation does is undesirable. Instead, here are some informal\nguidelines regarding coercion. In Python 3.0, coercion will not be\nsupported.\n\n* If the left operand of a % operator is a string or Unicode object,\n no coercion takes place and the string formatting operation is\n invoked instead.\n\n* It is no longer recommended to define a coercion operation. Mixed-\n mode operations on types that don\'t define coercion pass the\n original arguments to the operation.\n\n* New-style classes (those derived from ``object``) never invoke the\n ``__coerce__()`` method in response to a binary operator; the only\n time ``__coerce__()`` is invoked is when the built-in function\n ``coerce()`` is called.\n\n* For most intents and purposes, an operator that returns\n ``NotImplemented`` is treated the same as one that is not\n implemented at all.\n\n* Below, ``__op__()`` and ``__rop__()`` are used to signify the\n generic method names corresponding to an operator; ``__iop__()`` is\n used for the corresponding in-place operator. For example, for the\n operator \'``+``\', ``__add__()`` and ``__radd__()`` are used for the\n left and right variant of the binary operator, and ``__iadd__()``\n for the in-place variant.\n\n* For objects *x* and *y*, first ``x.__op__(y)`` is tried. If this is\n not implemented or returns ``NotImplemented``, ``y.__rop__(x)`` is\n tried. If this is also not implemented or returns\n ``NotImplemented``, a ``TypeError`` exception is raised. But see\n the following exception:\n\n* Exception to the previous item: if the left operand is an instance\n of a built-in type or a new-style class, and the right operand is an\n instance of a proper subclass of that type or class and overrides\n the base\'s ``__rop__()`` method, the right operand\'s ``__rop__()``\n method is tried *before* the left operand\'s ``__op__()`` method.\n\n This is done so that a subclass can completely override binary\n operators. Otherwise, the left operand\'s ``__op__()`` method would\n always accept the right operand: when an instance of a given class\n is expected, an instance of a subclass of that class is always\n acceptable.\n\n* When either operand type defines a coercion, this coercion is called\n before that type\'s ``__op__()`` or ``__rop__()`` method is called,\n but no sooner. If the coercion returns an object of a different\n type for the operand whose coercion is invoked, part of the process\n is redone using the new object.\n\n* When an in-place operator (like \'``+=``\') is used, if the left\n operand implements ``__iop__()``, it is invoked without any\n coercion. When the operation falls back to ``__op__()`` and/or\n ``__rop__()``, the normal coercion rules apply.\n\n* In ``x + y``, if *x* is a sequence that implements sequence\n concatenation, sequence concatenation is invoked.\n\n* In ``x * y``, if one operator is a sequence that implements sequence\n repetition, and the other is an integer (``int`` or ``long``),\n sequence repetition is invoked.\n\n* Rich comparisons (implemented by methods ``__eq__()`` and so on)\n never use coercion. Three-way comparison (implemented by\n ``__cmp__()``) does use coercion under the same conditions as other\n binary operations use it.\n\n* In the current implementation, the built-in numeric types ``int``,\n ``long``, ``float``, and ``complex`` do not use coercion. All these\n types implement a ``__coerce__()`` method, for use by the built-in\n ``coerce()`` function.\n\n Changed in version 2.7.\n\n\nWith Statement Context Managers\n===============================\n\nNew in version 2.5.\n\nA *context manager* is an object that defines the runtime context to\nbe established when executing a ``with`` statement. The context\nmanager handles the entry into, and the exit from, the desired runtime\ncontext for the execution of the block of code. Context managers are\nnormally invoked using the ``with`` statement (described in section\n*The with statement*), but can also be used by directly invoking their\nmethods.\n\nTypical uses of context managers include saving and restoring various\nkinds of global state, locking and unlocking resources, closing opened\nfiles, etc.\n\nFor more information on context managers, see *Context Manager Types*.\n\nobject.__enter__(self)\n\n Enter the runtime context related to this object. The ``with``\n statement will bind this method\'s return value to the target(s)\n specified in the ``as`` clause of the statement, if any.\n\nobject.__exit__(self, exc_type, exc_value, traceback)\n\n Exit the runtime context related to this object. The parameters\n describe the exception that caused the context to be exited. If the\n context was exited without an exception, all three arguments will\n be ``None``.\n\n If an exception is supplied, and the method wishes to suppress the\n exception (i.e., prevent it from being propagated), it should\n return a true value. Otherwise, the exception will be processed\n normally upon exit from this method.\n\n Note that ``__exit__()`` methods should not reraise the passed-in\n exception; this is the caller\'s responsibility.\n\nSee also:\n\n **PEP 0343** - The "with" statement\n The specification, background, and examples for the Python\n ``with`` statement.\n\n\nSpecial method lookup for old-style classes\n===========================================\n\nFor old-style classes, special methods are always looked up in exactly\nthe same way as any other method or attribute. This is the case\nregardless of whether the method is being looked up explicitly as in\n``x.__getitem__(i)`` or implicitly as in ``x[i]``.\n\nThis behaviour means that special methods may exhibit different\nbehaviour for different instances of a single old-style class if the\nappropriate special attributes are set differently:\n\n >>> class C:\n ... pass\n ...\n >>> c1 = C()\n >>> c2 = C()\n >>> c1.__len__ = lambda: 5\n >>> c2.__len__ = lambda: 9\n >>> len(c1)\n 5\n >>> len(c2)\n 9\n\n\nSpecial method lookup for new-style classes\n===========================================\n\nFor new-style classes, implicit invocations of special methods are\nonly guaranteed to work correctly if defined on an object\'s type, not\nin the object\'s instance dictionary. That behaviour is the reason why\nthe following code raises an exception (unlike the equivalent example\nwith old-style classes):\n\n >>> class C(object):\n ... pass\n ...\n >>> c = C()\n >>> c.__len__ = lambda: 5\n >>> len(c)\n Traceback (most recent call last):\n File "<stdin>", line 1, in <module>\n TypeError: object of type \'C\' has no len()\n\nThe rationale behind this behaviour lies with a number of special\nmethods such as ``__hash__()`` and ``__repr__()`` that are implemented\nby all objects, including type objects. If the implicit lookup of\nthese methods used the conventional lookup process, they would fail\nwhen invoked on the type object itself:\n\n >>> 1 .__hash__() == hash(1)\n True\n >>> int.__hash__() == hash(int)\n Traceback (most recent call last):\n File "<stdin>", line 1, in <module>\n TypeError: descriptor \'__hash__\' of \'int\' object needs an argument\n\nIncorrectly attempting to invoke an unbound method of a class in this\nway is sometimes referred to as \'metaclass confusion\', and is avoided\nby bypassing the instance when looking up special methods:\n\n >>> type(1).__hash__(1) == hash(1)\n True\n >>> type(int).__hash__(int) == hash(int)\n True\n\nIn addition to bypassing any instance attributes in the interest of\ncorrectness, implicit special method lookup generally also bypasses\nthe ``__getattribute__()`` method even of the object\'s metaclass:\n\n >>> class Meta(type):\n ... def __getattribute__(*args):\n ... print "Metaclass getattribute invoked"\n ... return type.__getattribute__(*args)\n ...\n >>> class C(object):\n ... __metaclass__ = Meta\n ... def __len__(self):\n ... return 10\n ... def __getattribute__(*args):\n ... print "Class getattribute invoked"\n ... return object.__getattribute__(*args)\n ...\n >>> c = C()\n >>> c.__len__() # Explicit lookup via instance\n Class getattribute invoked\n 10\n >>> type(c).__len__(c) # Explicit lookup via type\n Metaclass getattribute invoked\n 10\n >>> len(c) # Implicit lookup\n 10\n\nBypassing the ``__getattribute__()`` machinery in this fashion\nprovides significant scope for speed optimisations within the\ninterpreter, at the cost of some flexibility in the handling of\nspecial methods (the special method *must* be set on the class object\nitself in order to be consistently invoked by the interpreter).\n\n-[ Footnotes ]-\n\n[1] It *is* possible in some cases to change an object\'s type, under\n certain controlled conditions. It generally isn\'t a good idea\n though, since it can lead to some very strange behaviour if it is\n handled incorrectly.\n\n[2] For operands of the same type, it is assumed that if the non-\n reflected method (such as ``__add__()``) fails the operation is\n not supported, which is why the reflected method is not called.\n', + 'specialnames': u'\nSpecial method names\n********************\n\nA class can implement certain operations that are invoked by special\nsyntax (such as arithmetic operations or subscripting and slicing) by\ndefining methods with special names. This is Python\'s approach to\n*operator overloading*, allowing classes to define their own behavior\nwith respect to language operators. For instance, if a class defines\na method named ``__getitem__()``, and ``x`` is an instance of this\nclass, then ``x[i]`` is roughly equivalent to ``x.__getitem__(i)`` for\nold-style classes and ``type(x).__getitem__(x, i)`` for new-style\nclasses. Except where mentioned, attempts to execute an operation\nraise an exception when no appropriate method is defined (typically\n``AttributeError`` or ``TypeError``).\n\nWhen implementing a class that emulates any built-in type, it is\nimportant that the emulation only be implemented to the degree that it\nmakes sense for the object being modelled. For example, some\nsequences may work well with retrieval of individual elements, but\nextracting a slice may not make sense. (One example of this is the\n``NodeList`` interface in the W3C\'s Document Object Model.)\n\n\nBasic customization\n===================\n\nobject.__new__(cls[, ...])\n\n Called to create a new instance of class *cls*. ``__new__()`` is a\n static method (special-cased so you need not declare it as such)\n that takes the class of which an instance was requested as its\n first argument. The remaining arguments are those passed to the\n object constructor expression (the call to the class). The return\n value of ``__new__()`` should be the new object instance (usually\n an instance of *cls*).\n\n Typical implementations create a new instance of the class by\n invoking the superclass\'s ``__new__()`` method using\n ``super(currentclass, cls).__new__(cls[, ...])`` with appropriate\n arguments and then modifying the newly-created instance as\n necessary before returning it.\n\n If ``__new__()`` returns an instance of *cls*, then the new\n instance\'s ``__init__()`` method will be invoked like\n ``__init__(self[, ...])``, where *self* is the new instance and the\n remaining arguments are the same as were passed to ``__new__()``.\n\n If ``__new__()`` does not return an instance of *cls*, then the new\n instance\'s ``__init__()`` method will not be invoked.\n\n ``__new__()`` is intended mainly to allow subclasses of immutable\n types (like int, str, or tuple) to customize instance creation. It\n is also commonly overridden in custom metaclasses in order to\n customize class creation.\n\nobject.__init__(self[, ...])\n\n Called when the instance is created. The arguments are those\n passed to the class constructor expression. If a base class has an\n ``__init__()`` method, the derived class\'s ``__init__()`` method,\n if any, must explicitly call it to ensure proper initialization of\n the base class part of the instance; for example:\n ``BaseClass.__init__(self, [args...])``. As a special constraint\n on constructors, no value may be returned; doing so will cause a\n ``TypeError`` to be raised at runtime.\n\nobject.__del__(self)\n\n Called when the instance is about to be destroyed. This is also\n called a destructor. If a base class has a ``__del__()`` method,\n the derived class\'s ``__del__()`` method, if any, must explicitly\n call it to ensure proper deletion of the base class part of the\n instance. Note that it is possible (though not recommended!) for\n the ``__del__()`` method to postpone destruction of the instance by\n creating a new reference to it. It may then be called at a later\n time when this new reference is deleted. It is not guaranteed that\n ``__del__()`` methods are called for objects that still exist when\n the interpreter exits.\n\n Note: ``del x`` doesn\'t directly call ``x.__del__()`` --- the former\n decrements the reference count for ``x`` by one, and the latter\n is only called when ``x``\'s reference count reaches zero. Some\n common situations that may prevent the reference count of an\n object from going to zero include: circular references between\n objects (e.g., a doubly-linked list or a tree data structure with\n parent and child pointers); a reference to the object on the\n stack frame of a function that caught an exception (the traceback\n stored in ``sys.exc_traceback`` keeps the stack frame alive); or\n a reference to the object on the stack frame that raised an\n unhandled exception in interactive mode (the traceback stored in\n ``sys.last_traceback`` keeps the stack frame alive). The first\n situation can only be remedied by explicitly breaking the cycles;\n the latter two situations can be resolved by storing ``None`` in\n ``sys.exc_traceback`` or ``sys.last_traceback``. Circular\n references which are garbage are detected when the option cycle\n detector is enabled (it\'s on by default), but can only be cleaned\n up if there are no Python-level ``__del__()`` methods involved.\n Refer to the documentation for the ``gc`` module for more\n information about how ``__del__()`` methods are handled by the\n cycle detector, particularly the description of the ``garbage``\n value.\n\n Warning: Due to the precarious circumstances under which ``__del__()``\n methods are invoked, exceptions that occur during their execution\n are ignored, and a warning is printed to ``sys.stderr`` instead.\n Also, when ``__del__()`` is invoked in response to a module being\n deleted (e.g., when execution of the program is done), other\n globals referenced by the ``__del__()`` method may already have\n been deleted or in the process of being torn down (e.g. the\n import machinery shutting down). For this reason, ``__del__()``\n methods should do the absolute minimum needed to maintain\n external invariants. Starting with version 1.5, Python\n guarantees that globals whose name begins with a single\n underscore are deleted from their module before other globals are\n deleted; if no other references to such globals exist, this may\n help in assuring that imported modules are still available at the\n time when the ``__del__()`` method is called.\n\nobject.__repr__(self)\n\n Called by the ``repr()`` built-in function and by string\n conversions (reverse quotes) to compute the "official" string\n representation of an object. If at all possible, this should look\n like a valid Python expression that could be used to recreate an\n object with the same value (given an appropriate environment). If\n this is not possible, a string of the form ``<...some useful\n description...>`` should be returned. The return value must be a\n string object. If a class defines ``__repr__()`` but not\n ``__str__()``, then ``__repr__()`` is also used when an "informal"\n string representation of instances of that class is required.\n\n This is typically used for debugging, so it is important that the\n representation is information-rich and unambiguous.\n\nobject.__str__(self)\n\n Called by the ``str()`` built-in function and by the ``print``\n statement to compute the "informal" string representation of an\n object. This differs from ``__repr__()`` in that it does not have\n to be a valid Python expression: a more convenient or concise\n representation may be used instead. The return value must be a\n string object.\n\nobject.__lt__(self, other)\nobject.__le__(self, other)\nobject.__eq__(self, other)\nobject.__ne__(self, other)\nobject.__gt__(self, other)\nobject.__ge__(self, other)\n\n New in version 2.1.\n\n These are the so-called "rich comparison" methods, and are called\n for comparison operators in preference to ``__cmp__()`` below. The\n correspondence between operator symbols and method names is as\n follows: ``x<y`` calls ``x.__lt__(y)``, ``x<=y`` calls\n ``x.__le__(y)``, ``x==y`` calls ``x.__eq__(y)``, ``x!=y`` and\n ``x<>y`` call ``x.__ne__(y)``, ``x>y`` calls ``x.__gt__(y)``, and\n ``x>=y`` calls ``x.__ge__(y)``.\n\n A rich comparison method may return the singleton\n ``NotImplemented`` if it does not implement the operation for a\n given pair of arguments. By convention, ``False`` and ``True`` are\n returned for a successful comparison. However, these methods can\n return any value, so if the comparison operator is used in a\n Boolean context (e.g., in the condition of an ``if`` statement),\n Python will call ``bool()`` on the value to determine if the result\n is true or false.\n\n There are no implied relationships among the comparison operators.\n The truth of ``x==y`` does not imply that ``x!=y`` is false.\n Accordingly, when defining ``__eq__()``, one should also define\n ``__ne__()`` so that the operators will behave as expected. See\n the paragraph on ``__hash__()`` for some important notes on\n creating *hashable* objects which support custom comparison\n operations and are usable as dictionary keys.\n\n There are no swapped-argument versions of these methods (to be used\n when the left argument does not support the operation but the right\n argument does); rather, ``__lt__()`` and ``__gt__()`` are each\n other\'s reflection, ``__le__()`` and ``__ge__()`` are each other\'s\n reflection, and ``__eq__()`` and ``__ne__()`` are their own\n reflection.\n\n Arguments to rich comparison methods are never coerced.\n\n To automatically generate ordering operations from a single root\n operation, see ``functools.total_ordering()``.\n\nobject.__cmp__(self, other)\n\n Called by comparison operations if rich comparison (see above) is\n not defined. Should return a negative integer if ``self < other``,\n zero if ``self == other``, a positive integer if ``self > other``.\n If no ``__cmp__()``, ``__eq__()`` or ``__ne__()`` operation is\n defined, class instances are compared by object identity\n ("address"). See also the description of ``__hash__()`` for some\n important notes on creating *hashable* objects which support custom\n comparison operations and are usable as dictionary keys. (Note: the\n restriction that exceptions are not propagated by ``__cmp__()`` has\n been removed since Python 1.5.)\n\nobject.__rcmp__(self, other)\n\n Changed in version 2.1: No longer supported.\n\nobject.__hash__(self)\n\n Called by built-in function ``hash()`` and for operations on\n members of hashed collections including ``set``, ``frozenset``, and\n ``dict``. ``__hash__()`` should return an integer. The only\n required property is that objects which compare equal have the same\n hash value; it is advised to somehow mix together (e.g. using\n exclusive or) the hash values for the components of the object that\n also play a part in comparison of objects.\n\n If a class does not define a ``__cmp__()`` or ``__eq__()`` method\n it should not define a ``__hash__()`` operation either; if it\n defines ``__cmp__()`` or ``__eq__()`` but not ``__hash__()``, its\n instances will not be usable in hashed collections. If a class\n defines mutable objects and implements a ``__cmp__()`` or\n ``__eq__()`` method, it should not implement ``__hash__()``, since\n hashable collection implementations require that a object\'s hash\n value is immutable (if the object\'s hash value changes, it will be\n in the wrong hash bucket).\n\n User-defined classes have ``__cmp__()`` and ``__hash__()`` methods\n by default; with them, all objects compare unequal (except with\n themselves) and ``x.__hash__()`` returns ``id(x)``.\n\n Classes which inherit a ``__hash__()`` method from a parent class\n but change the meaning of ``__cmp__()`` or ``__eq__()`` such that\n the hash value returned is no longer appropriate (e.g. by switching\n to a value-based concept of equality instead of the default\n identity based equality) can explicitly flag themselves as being\n unhashable by setting ``__hash__ = None`` in the class definition.\n Doing so means that not only will instances of the class raise an\n appropriate ``TypeError`` when a program attempts to retrieve their\n hash value, but they will also be correctly identified as\n unhashable when checking ``isinstance(obj, collections.Hashable)``\n (unlike classes which define their own ``__hash__()`` to explicitly\n raise ``TypeError``).\n\n Changed in version 2.5: ``__hash__()`` may now also return a long\n integer object; the 32-bit integer is then derived from the hash of\n that object.\n\n Changed in version 2.6: ``__hash__`` may now be set to ``None`` to\n explicitly flag instances of a class as unhashable.\n\nobject.__nonzero__(self)\n\n Called to implement truth value testing and the built-in operation\n ``bool()``; should return ``False`` or ``True``, or their integer\n equivalents ``0`` or ``1``. When this method is not defined,\n ``__len__()`` is called, if it is defined, and the object is\n considered true if its result is nonzero. If a class defines\n neither ``__len__()`` nor ``__nonzero__()``, all its instances are\n considered true.\n\nobject.__unicode__(self)\n\n Called to implement ``unicode()`` built-in; should return a Unicode\n object. When this method is not defined, string conversion is\n attempted, and the result of string conversion is converted to\n Unicode using the system default encoding.\n\n\nCustomizing attribute access\n============================\n\nThe following methods can be defined to customize the meaning of\nattribute access (use of, assignment to, or deletion of ``x.name``)\nfor class instances.\n\nobject.__getattr__(self, name)\n\n Called when an attribute lookup has not found the attribute in the\n usual places (i.e. it is not an instance attribute nor is it found\n in the class tree for ``self``). ``name`` is the attribute name.\n This method should return the (computed) attribute value or raise\n an ``AttributeError`` exception.\n\n Note that if the attribute is found through the normal mechanism,\n ``__getattr__()`` is not called. (This is an intentional asymmetry\n between ``__getattr__()`` and ``__setattr__()``.) This is done both\n for efficiency reasons and because otherwise ``__getattr__()``\n would have no way to access other attributes of the instance. Note\n that at least for instance variables, you can fake total control by\n not inserting any values in the instance attribute dictionary (but\n instead inserting them in another object). See the\n ``__getattribute__()`` method below for a way to actually get total\n control in new-style classes.\n\nobject.__setattr__(self, name, value)\n\n Called when an attribute assignment is attempted. This is called\n instead of the normal mechanism (i.e. store the value in the\n instance dictionary). *name* is the attribute name, *value* is the\n value to be assigned to it.\n\n If ``__setattr__()`` wants to assign to an instance attribute, it\n should not simply execute ``self.name = value`` --- this would\n cause a recursive call to itself. Instead, it should insert the\n value in the dictionary of instance attributes, e.g.,\n ``self.__dict__[name] = value``. For new-style classes, rather\n than accessing the instance dictionary, it should call the base\n class method with the same name, for example,\n ``object.__setattr__(self, name, value)``.\n\nobject.__delattr__(self, name)\n\n Like ``__setattr__()`` but for attribute deletion instead of\n assignment. This should only be implemented if ``del obj.name`` is\n meaningful for the object.\n\n\nMore attribute access for new-style classes\n-------------------------------------------\n\nThe following methods only apply to new-style classes.\n\nobject.__getattribute__(self, name)\n\n Called unconditionally to implement attribute accesses for\n instances of the class. If the class also defines\n ``__getattr__()``, the latter will not be called unless\n ``__getattribute__()`` either calls it explicitly or raises an\n ``AttributeError``. This method should return the (computed)\n attribute value or raise an ``AttributeError`` exception. In order\n to avoid infinite recursion in this method, its implementation\n should always call the base class method with the same name to\n access any attributes it needs, for example,\n ``object.__getattribute__(self, name)``.\n\n Note: This method may still be bypassed when looking up special methods\n as the result of implicit invocation via language syntax or\n built-in functions. See *Special method lookup for new-style\n classes*.\n\n\nImplementing Descriptors\n------------------------\n\nThe following methods only apply when an instance of the class\ncontaining the method (a so-called *descriptor* class) appears in an\n*owner* class (the descriptor must be in either the owner\'s class\ndictionary or in the class dictionary for one of its parents). In the\nexamples below, "the attribute" refers to the attribute whose name is\nthe key of the property in the owner class\' ``__dict__``.\n\nobject.__get__(self, instance, owner)\n\n Called to get the attribute of the owner class (class attribute\n access) or of an instance of that class (instance attribute\n access). *owner* is always the owner class, while *instance* is the\n instance that the attribute was accessed through, or ``None`` when\n the attribute is accessed through the *owner*. This method should\n return the (computed) attribute value or raise an\n ``AttributeError`` exception.\n\nobject.__set__(self, instance, value)\n\n Called to set the attribute on an instance *instance* of the owner\n class to a new value, *value*.\n\nobject.__delete__(self, instance)\n\n Called to delete the attribute on an instance *instance* of the\n owner class.\n\n\nInvoking Descriptors\n--------------------\n\nIn general, a descriptor is an object attribute with "binding\nbehavior", one whose attribute access has been overridden by methods\nin the descriptor protocol: ``__get__()``, ``__set__()``, and\n``__delete__()``. If any of those methods are defined for an object,\nit is said to be a descriptor.\n\nThe default behavior for attribute access is to get, set, or delete\nthe attribute from an object\'s dictionary. For instance, ``a.x`` has a\nlookup chain starting with ``a.__dict__[\'x\']``, then\n``type(a).__dict__[\'x\']``, and continuing through the base classes of\n``type(a)`` excluding metaclasses.\n\nHowever, if the looked-up value is an object defining one of the\ndescriptor methods, then Python may override the default behavior and\ninvoke the descriptor method instead. Where this occurs in the\nprecedence chain depends on which descriptor methods were defined and\nhow they were called. Note that descriptors are only invoked for new\nstyle objects or classes (ones that subclass ``object()`` or\n``type()``).\n\nThe starting point for descriptor invocation is a binding, ``a.x``.\nHow the arguments are assembled depends on ``a``:\n\nDirect Call\n The simplest and least common call is when user code directly\n invokes a descriptor method: ``x.__get__(a)``.\n\nInstance Binding\n If binding to a new-style object instance, ``a.x`` is transformed\n into the call: ``type(a).__dict__[\'x\'].__get__(a, type(a))``.\n\nClass Binding\n If binding to a new-style class, ``A.x`` is transformed into the\n call: ``A.__dict__[\'x\'].__get__(None, A)``.\n\nSuper Binding\n If ``a`` is an instance of ``super``, then the binding ``super(B,\n obj).m()`` searches ``obj.__class__.__mro__`` for the base class\n ``A`` immediately preceding ``B`` and then invokes the descriptor\n with the call: ``A.__dict__[\'m\'].__get__(obj, obj.__class__)``.\n\nFor instance bindings, the precedence of descriptor invocation depends\non the which descriptor methods are defined. A descriptor can define\nany combination of ``__get__()``, ``__set__()`` and ``__delete__()``.\nIf it does not define ``__get__()``, then accessing the attribute will\nreturn the descriptor object itself unless there is a value in the\nobject\'s instance dictionary. If the descriptor defines ``__set__()``\nand/or ``__delete__()``, it is a data descriptor; if it defines\nneither, it is a non-data descriptor. Normally, data descriptors\ndefine both ``__get__()`` and ``__set__()``, while non-data\ndescriptors have just the ``__get__()`` method. Data descriptors with\n``__set__()`` and ``__get__()`` defined always override a redefinition\nin an instance dictionary. In contrast, non-data descriptors can be\noverridden by instances.\n\nPython methods (including ``staticmethod()`` and ``classmethod()``)\nare implemented as non-data descriptors. Accordingly, instances can\nredefine and override methods. This allows individual instances to\nacquire behaviors that differ from other instances of the same class.\n\nThe ``property()`` function is implemented as a data descriptor.\nAccordingly, instances cannot override the behavior of a property.\n\n\n__slots__\n---------\n\nBy default, instances of both old and new-style classes have a\ndictionary for attribute storage. This wastes space for objects\nhaving very few instance variables. The space consumption can become\nacute when creating large numbers of instances.\n\nThe default can be overridden by defining *__slots__* in a new-style\nclass definition. The *__slots__* declaration takes a sequence of\ninstance variables and reserves just enough space in each instance to\nhold a value for each variable. Space is saved because *__dict__* is\nnot created for each instance.\n\n__slots__\n\n This class variable can be assigned a string, iterable, or sequence\n of strings with variable names used by instances. If defined in a\n new-style class, *__slots__* reserves space for the declared\n variables and prevents the automatic creation of *__dict__* and\n *__weakref__* for each instance.\n\n New in version 2.2.\n\nNotes on using *__slots__*\n\n* When inheriting from a class without *__slots__*, the *__dict__*\n attribute of that class will always be accessible, so a *__slots__*\n definition in the subclass is meaningless.\n\n* Without a *__dict__* variable, instances cannot be assigned new\n variables not listed in the *__slots__* definition. Attempts to\n assign to an unlisted variable name raises ``AttributeError``. If\n dynamic assignment of new variables is desired, then add\n ``\'__dict__\'`` to the sequence of strings in the *__slots__*\n declaration.\n\n Changed in version 2.3: Previously, adding ``\'__dict__\'`` to the\n *__slots__* declaration would not enable the assignment of new\n attributes not specifically listed in the sequence of instance\n variable names.\n\n* Without a *__weakref__* variable for each instance, classes defining\n *__slots__* do not support weak references to its instances. If weak\n reference support is needed, then add ``\'__weakref__\'`` to the\n sequence of strings in the *__slots__* declaration.\n\n Changed in version 2.3: Previously, adding ``\'__weakref__\'`` to the\n *__slots__* declaration would not enable support for weak\n references.\n\n* *__slots__* are implemented at the class level by creating\n descriptors (*Implementing Descriptors*) for each variable name. As\n a result, class attributes cannot be used to set default values for\n instance variables defined by *__slots__*; otherwise, the class\n attribute would overwrite the descriptor assignment.\n\n* The action of a *__slots__* declaration is limited to the class\n where it is defined. As a result, subclasses will have a *__dict__*\n unless they also define *__slots__* (which must only contain names\n of any *additional* slots).\n\n* If a class defines a slot also defined in a base class, the instance\n variable defined by the base class slot is inaccessible (except by\n retrieving its descriptor directly from the base class). This\n renders the meaning of the program undefined. In the future, a\n check may be added to prevent this.\n\n* Nonempty *__slots__* does not work for classes derived from\n "variable-length" built-in types such as ``long``, ``str`` and\n ``tuple``.\n\n* Any non-string iterable may be assigned to *__slots__*. Mappings may\n also be used; however, in the future, special meaning may be\n assigned to the values corresponding to each key.\n\n* *__class__* assignment works only if both classes have the same\n *__slots__*.\n\n Changed in version 2.6: Previously, *__class__* assignment raised an\n error if either new or old class had *__slots__*.\n\n\nCustomizing class creation\n==========================\n\nBy default, new-style classes are constructed using ``type()``. A\nclass definition is read into a separate namespace and the value of\nclass name is bound to the result of ``type(name, bases, dict)``.\n\nWhen the class definition is read, if *__metaclass__* is defined then\nthe callable assigned to it will be called instead of ``type()``. This\nallows classes or functions to be written which monitor or alter the\nclass creation process:\n\n* Modifying the class dictionary prior to the class being created.\n\n* Returning an instance of another class -- essentially performing the\n role of a factory function.\n\nThese steps will have to be performed in the metaclass\'s ``__new__()``\nmethod -- ``type.__new__()`` can then be called from this method to\ncreate a class with different properties. This example adds a new\nelement to the class dictionary before creating the class:\n\n class metacls(type):\n def __new__(mcs, name, bases, dict):\n dict[\'foo\'] = \'metacls was here\'\n return type.__new__(mcs, name, bases, dict)\n\nYou can of course also override other class methods (or add new\nmethods); for example defining a custom ``__call__()`` method in the\nmetaclass allows custom behavior when the class is called, e.g. not\nalways creating a new instance.\n\n__metaclass__\n\n This variable can be any callable accepting arguments for ``name``,\n ``bases``, and ``dict``. Upon class creation, the callable is used\n instead of the built-in ``type()``.\n\n New in version 2.2.\n\nThe appropriate metaclass is determined by the following precedence\nrules:\n\n* If ``dict[\'__metaclass__\']`` exists, it is used.\n\n* Otherwise, if there is at least one base class, its metaclass is\n used (this looks for a *__class__* attribute first and if not found,\n uses its type).\n\n* Otherwise, if a global variable named __metaclass__ exists, it is\n used.\n\n* Otherwise, the old-style, classic metaclass (types.ClassType) is\n used.\n\nThe potential uses for metaclasses are boundless. Some ideas that have\nbeen explored including logging, interface checking, automatic\ndelegation, automatic property creation, proxies, frameworks, and\nautomatic resource locking/synchronization.\n\n\nCustomizing instance and subclass checks\n========================================\n\nNew in version 2.6.\n\nThe following methods are used to override the default behavior of the\n``isinstance()`` and ``issubclass()`` built-in functions.\n\nIn particular, the metaclass ``abc.ABCMeta`` implements these methods\nin order to allow the addition of Abstract Base Classes (ABCs) as\n"virtual base classes" to any class or type (including built-in\ntypes), including other ABCs.\n\nclass.__instancecheck__(self, instance)\n\n Return true if *instance* should be considered a (direct or\n indirect) instance of *class*. If defined, called to implement\n ``isinstance(instance, class)``.\n\nclass.__subclasscheck__(self, subclass)\n\n Return true if *subclass* should be considered a (direct or\n indirect) subclass of *class*. If defined, called to implement\n ``issubclass(subclass, class)``.\n\nNote that these methods are looked up on the type (metaclass) of a\nclass. They cannot be defined as class methods in the actual class.\nThis is consistent with the lookup of special methods that are called\non instances, only in this case the instance is itself a class.\n\nSee also:\n\n **PEP 3119** - Introducing Abstract Base Classes\n Includes the specification for customizing ``isinstance()`` and\n ``issubclass()`` behavior through ``__instancecheck__()`` and\n ``__subclasscheck__()``, with motivation for this functionality\n in the context of adding Abstract Base Classes (see the ``abc``\n module) to the language.\n\n\nEmulating callable objects\n==========================\n\nobject.__call__(self[, args...])\n\n Called when the instance is "called" as a function; if this method\n is defined, ``x(arg1, arg2, ...)`` is a shorthand for\n ``x.__call__(arg1, arg2, ...)``.\n\n\nEmulating container types\n=========================\n\nThe following methods can be defined to implement container objects.\nContainers usually are sequences (such as lists or tuples) or mappings\n(like dictionaries), but can represent other containers as well. The\nfirst set of methods is used either to emulate a sequence or to\nemulate a mapping; the difference is that for a sequence, the\nallowable keys should be the integers *k* for which ``0 <= k < N``\nwhere *N* is the length of the sequence, or slice objects, which\ndefine a range of items. (For backwards compatibility, the method\n``__getslice__()`` (see below) can also be defined to handle simple,\nbut not extended slices.) It is also recommended that mappings provide\nthe methods ``keys()``, ``values()``, ``items()``, ``has_key()``,\n``get()``, ``clear()``, ``setdefault()``, ``iterkeys()``,\n``itervalues()``, ``iteritems()``, ``pop()``, ``popitem()``,\n``copy()``, and ``update()`` behaving similar to those for Python\'s\nstandard dictionary objects. The ``UserDict`` module provides a\n``DictMixin`` class to help create those methods from a base set of\n``__getitem__()``, ``__setitem__()``, ``__delitem__()``, and\n``keys()``. Mutable sequences should provide methods ``append()``,\n``count()``, ``index()``, ``extend()``, ``insert()``, ``pop()``,\n``remove()``, ``reverse()`` and ``sort()``, like Python standard list\nobjects. Finally, sequence types should implement addition (meaning\nconcatenation) and multiplication (meaning repetition) by defining the\nmethods ``__add__()``, ``__radd__()``, ``__iadd__()``, ``__mul__()``,\n``__rmul__()`` and ``__imul__()`` described below; they should not\ndefine ``__coerce__()`` or other numerical operators. It is\nrecommended that both mappings and sequences implement the\n``__contains__()`` method to allow efficient use of the ``in``\noperator; for mappings, ``in`` should be equivalent of ``has_key()``;\nfor sequences, it should search through the values. It is further\nrecommended that both mappings and sequences implement the\n``__iter__()`` method to allow efficient iteration through the\ncontainer; for mappings, ``__iter__()`` should be the same as\n``iterkeys()``; for sequences, it should iterate through the values.\n\nobject.__len__(self)\n\n Called to implement the built-in function ``len()``. Should return\n the length of the object, an integer ``>=`` 0. Also, an object\n that doesn\'t define a ``__nonzero__()`` method and whose\n ``__len__()`` method returns zero is considered to be false in a\n Boolean context.\n\nobject.__getitem__(self, key)\n\n Called to implement evaluation of ``self[key]``. For sequence\n types, the accepted keys should be integers and slice objects.\n Note that the special interpretation of negative indexes (if the\n class wishes to emulate a sequence type) is up to the\n ``__getitem__()`` method. If *key* is of an inappropriate type,\n ``TypeError`` may be raised; if of a value outside the set of\n indexes for the sequence (after any special interpretation of\n negative values), ``IndexError`` should be raised. For mapping\n types, if *key* is missing (not in the container), ``KeyError``\n should be raised.\n\n Note: ``for`` loops expect that an ``IndexError`` will be raised for\n illegal indexes to allow proper detection of the end of the\n sequence.\n\nobject.__setitem__(self, key, value)\n\n Called to implement assignment to ``self[key]``. Same note as for\n ``__getitem__()``. This should only be implemented for mappings if\n the objects support changes to the values for keys, or if new keys\n can be added, or for sequences if elements can be replaced. The\n same exceptions should be raised for improper *key* values as for\n the ``__getitem__()`` method.\n\nobject.__delitem__(self, key)\n\n Called to implement deletion of ``self[key]``. Same note as for\n ``__getitem__()``. This should only be implemented for mappings if\n the objects support removal of keys, or for sequences if elements\n can be removed from the sequence. The same exceptions should be\n raised for improper *key* values as for the ``__getitem__()``\n method.\n\nobject.__iter__(self)\n\n This method is called when an iterator is required for a container.\n This method should return a new iterator object that can iterate\n over all the objects in the container. For mappings, it should\n iterate over the keys of the container, and should also be made\n available as the method ``iterkeys()``.\n\n Iterator objects also need to implement this method; they are\n required to return themselves. For more information on iterator\n objects, see *Iterator Types*.\n\nobject.__reversed__(self)\n\n Called (if present) by the ``reversed()`` built-in to implement\n reverse iteration. It should return a new iterator object that\n iterates over all the objects in the container in reverse order.\n\n If the ``__reversed__()`` method is not provided, the\n ``reversed()`` built-in will fall back to using the sequence\n protocol (``__len__()`` and ``__getitem__()``). Objects that\n support the sequence protocol should only provide\n ``__reversed__()`` if they can provide an implementation that is\n more efficient than the one provided by ``reversed()``.\n\n New in version 2.6.\n\nThe membership test operators (``in`` and ``not in``) are normally\nimplemented as an iteration through a sequence. However, container\nobjects can supply the following special method with a more efficient\nimplementation, which also does not require the object be a sequence.\n\nobject.__contains__(self, item)\n\n Called to implement membership test operators. Should return true\n if *item* is in *self*, false otherwise. For mapping objects, this\n should consider the keys of the mapping rather than the values or\n the key-item pairs.\n\n For objects that don\'t define ``__contains__()``, the membership\n test first tries iteration via ``__iter__()``, then the old\n sequence iteration protocol via ``__getitem__()``, see *this\n section in the language reference*.\n\n\nAdditional methods for emulation of sequence types\n==================================================\n\nThe following optional methods can be defined to further emulate\nsequence objects. Immutable sequences methods should at most only\ndefine ``__getslice__()``; mutable sequences might define all three\nmethods.\n\nobject.__getslice__(self, i, j)\n\n Deprecated since version 2.0: Support slice objects as parameters\n to the ``__getitem__()`` method. (However, built-in types in\n CPython currently still implement ``__getslice__()``. Therefore,\n you have to override it in derived classes when implementing\n slicing.)\n\n Called to implement evaluation of ``self[i:j]``. The returned\n object should be of the same type as *self*. Note that missing *i*\n or *j* in the slice expression are replaced by zero or\n ``sys.maxint``, respectively. If negative indexes are used in the\n slice, the length of the sequence is added to that index. If the\n instance does not implement the ``__len__()`` method, an\n ``AttributeError`` is raised. No guarantee is made that indexes\n adjusted this way are not still negative. Indexes which are\n greater than the length of the sequence are not modified. If no\n ``__getslice__()`` is found, a slice object is created instead, and\n passed to ``__getitem__()`` instead.\n\nobject.__setslice__(self, i, j, sequence)\n\n Called to implement assignment to ``self[i:j]``. Same notes for *i*\n and *j* as for ``__getslice__()``.\n\n This method is deprecated. If no ``__setslice__()`` is found, or\n for extended slicing of the form ``self[i:j:k]``, a slice object is\n created, and passed to ``__setitem__()``, instead of\n ``__setslice__()`` being called.\n\nobject.__delslice__(self, i, j)\n\n Called to implement deletion of ``self[i:j]``. Same notes for *i*\n and *j* as for ``__getslice__()``. This method is deprecated. If no\n ``__delslice__()`` is found, or for extended slicing of the form\n ``self[i:j:k]``, a slice object is created, and passed to\n ``__delitem__()``, instead of ``__delslice__()`` being called.\n\nNotice that these methods are only invoked when a single slice with a\nsingle colon is used, and the slice method is available. For slice\noperations involving extended slice notation, or in absence of the\nslice methods, ``__getitem__()``, ``__setitem__()`` or\n``__delitem__()`` is called with a slice object as argument.\n\nThe following example demonstrate how to make your program or module\ncompatible with earlier versions of Python (assuming that methods\n``__getitem__()``, ``__setitem__()`` and ``__delitem__()`` support\nslice objects as arguments):\n\n class MyClass:\n ...\n def __getitem__(self, index):\n ...\n def __setitem__(self, index, value):\n ...\n def __delitem__(self, index):\n ...\n\n if sys.version_info < (2, 0):\n # They won\'t be defined if version is at least 2.0 final\n\n def __getslice__(self, i, j):\n return self[max(0, i):max(0, j):]\n def __setslice__(self, i, j, seq):\n self[max(0, i):max(0, j):] = seq\n def __delslice__(self, i, j):\n del self[max(0, i):max(0, j):]\n ...\n\nNote the calls to ``max()``; these are necessary because of the\nhandling of negative indices before the ``__*slice__()`` methods are\ncalled. When negative indexes are used, the ``__*item__()`` methods\nreceive them as provided, but the ``__*slice__()`` methods get a\n"cooked" form of the index values. For each negative index value, the\nlength of the sequence is added to the index before calling the method\n(which may still result in a negative index); this is the customary\nhandling of negative indexes by the built-in sequence types, and the\n``__*item__()`` methods are expected to do this as well. However,\nsince they should already be doing that, negative indexes cannot be\npassed in; they must be constrained to the bounds of the sequence\nbefore being passed to the ``__*item__()`` methods. Calling ``max(0,\ni)`` conveniently returns the proper value.\n\n\nEmulating numeric types\n=======================\n\nThe following methods can be defined to emulate numeric objects.\nMethods corresponding to operations that are not supported by the\nparticular kind of number implemented (e.g., bitwise operations for\nnon-integral numbers) should be left undefined.\n\nobject.__add__(self, other)\nobject.__sub__(self, other)\nobject.__mul__(self, other)\nobject.__floordiv__(self, other)\nobject.__mod__(self, other)\nobject.__divmod__(self, other)\nobject.__pow__(self, other[, modulo])\nobject.__lshift__(self, other)\nobject.__rshift__(self, other)\nobject.__and__(self, other)\nobject.__xor__(self, other)\nobject.__or__(self, other)\n\n These methods are called to implement the binary arithmetic\n operations (``+``, ``-``, ``*``, ``//``, ``%``, ``divmod()``,\n ``pow()``, ``**``, ``<<``, ``>>``, ``&``, ``^``, ``|``). For\n instance, to evaluate the expression ``x + y``, where *x* is an\n instance of a class that has an ``__add__()`` method,\n ``x.__add__(y)`` is called. The ``__divmod__()`` method should be\n the equivalent to using ``__floordiv__()`` and ``__mod__()``; it\n should not be related to ``__truediv__()`` (described below). Note\n that ``__pow__()`` should be defined to accept an optional third\n argument if the ternary version of the built-in ``pow()`` function\n is to be supported.\n\n If one of those methods does not support the operation with the\n supplied arguments, it should return ``NotImplemented``.\n\nobject.__div__(self, other)\nobject.__truediv__(self, other)\n\n The division operator (``/``) is implemented by these methods. The\n ``__truediv__()`` method is used when ``__future__.division`` is in\n effect, otherwise ``__div__()`` is used. If only one of these two\n methods is defined, the object will not support division in the\n alternate context; ``TypeError`` will be raised instead.\n\nobject.__radd__(self, other)\nobject.__rsub__(self, other)\nobject.__rmul__(self, other)\nobject.__rdiv__(self, other)\nobject.__rtruediv__(self, other)\nobject.__rfloordiv__(self, other)\nobject.__rmod__(self, other)\nobject.__rdivmod__(self, other)\nobject.__rpow__(self, other)\nobject.__rlshift__(self, other)\nobject.__rrshift__(self, other)\nobject.__rand__(self, other)\nobject.__rxor__(self, other)\nobject.__ror__(self, other)\n\n These methods are called to implement the binary arithmetic\n operations (``+``, ``-``, ``*``, ``/``, ``%``, ``divmod()``,\n ``pow()``, ``**``, ``<<``, ``>>``, ``&``, ``^``, ``|``) with\n reflected (swapped) operands. These functions are only called if\n the left operand does not support the corresponding operation and\n the operands are of different types. [2] For instance, to evaluate\n the expression ``x - y``, where *y* is an instance of a class that\n has an ``__rsub__()`` method, ``y.__rsub__(x)`` is called if\n ``x.__sub__(y)`` returns *NotImplemented*.\n\n Note that ternary ``pow()`` will not try calling ``__rpow__()``\n (the coercion rules would become too complicated).\n\n Note: If the right operand\'s type is a subclass of the left operand\'s\n type and that subclass provides the reflected method for the\n operation, this method will be called before the left operand\'s\n non-reflected method. This behavior allows subclasses to\n override their ancestors\' operations.\n\nobject.__iadd__(self, other)\nobject.__isub__(self, other)\nobject.__imul__(self, other)\nobject.__idiv__(self, other)\nobject.__itruediv__(self, other)\nobject.__ifloordiv__(self, other)\nobject.__imod__(self, other)\nobject.__ipow__(self, other[, modulo])\nobject.__ilshift__(self, other)\nobject.__irshift__(self, other)\nobject.__iand__(self, other)\nobject.__ixor__(self, other)\nobject.__ior__(self, other)\n\n These methods are called to implement the augmented arithmetic\n assignments (``+=``, ``-=``, ``*=``, ``/=``, ``//=``, ``%=``,\n ``**=``, ``<<=``, ``>>=``, ``&=``, ``^=``, ``|=``). These methods\n should attempt to do the operation in-place (modifying *self*) and\n return the result (which could be, but does not have to be,\n *self*). If a specific method is not defined, the augmented\n assignment falls back to the normal methods. For instance, to\n execute the statement ``x += y``, where *x* is an instance of a\n class that has an ``__iadd__()`` method, ``x.__iadd__(y)`` is\n called. If *x* is an instance of a class that does not define a\n ``__iadd__()`` method, ``x.__add__(y)`` and ``y.__radd__(x)`` are\n considered, as with the evaluation of ``x + y``.\n\nobject.__neg__(self)\nobject.__pos__(self)\nobject.__abs__(self)\nobject.__invert__(self)\n\n Called to implement the unary arithmetic operations (``-``, ``+``,\n ``abs()`` and ``~``).\n\nobject.__complex__(self)\nobject.__int__(self)\nobject.__long__(self)\nobject.__float__(self)\n\n Called to implement the built-in functions ``complex()``,\n ``int()``, ``long()``, and ``float()``. Should return a value of\n the appropriate type.\n\nobject.__oct__(self)\nobject.__hex__(self)\n\n Called to implement the built-in functions ``oct()`` and ``hex()``.\n Should return a string value.\n\nobject.__index__(self)\n\n Called to implement ``operator.index()``. Also called whenever\n Python needs an integer object (such as in slicing). Must return\n an integer (int or long).\n\n New in version 2.5.\n\nobject.__coerce__(self, other)\n\n Called to implement "mixed-mode" numeric arithmetic. Should either\n return a 2-tuple containing *self* and *other* converted to a\n common numeric type, or ``None`` if conversion is impossible. When\n the common type would be the type of ``other``, it is sufficient to\n return ``None``, since the interpreter will also ask the other\n object to attempt a coercion (but sometimes, if the implementation\n of the other type cannot be changed, it is useful to do the\n conversion to the other type here). A return value of\n ``NotImplemented`` is equivalent to returning ``None``.\n\n\nCoercion rules\n==============\n\nThis section used to document the rules for coercion. As the language\nhas evolved, the coercion rules have become hard to document\nprecisely; documenting what one version of one particular\nimplementation does is undesirable. Instead, here are some informal\nguidelines regarding coercion. In Python 3.0, coercion will not be\nsupported.\n\n* If the left operand of a % operator is a string or Unicode object,\n no coercion takes place and the string formatting operation is\n invoked instead.\n\n* It is no longer recommended to define a coercion operation. Mixed-\n mode operations on types that don\'t define coercion pass the\n original arguments to the operation.\n\n* New-style classes (those derived from ``object``) never invoke the\n ``__coerce__()`` method in response to a binary operator; the only\n time ``__coerce__()`` is invoked is when the built-in function\n ``coerce()`` is called.\n\n* For most intents and purposes, an operator that returns\n ``NotImplemented`` is treated the same as one that is not\n implemented at all.\n\n* Below, ``__op__()`` and ``__rop__()`` are used to signify the\n generic method names corresponding to an operator; ``__iop__()`` is\n used for the corresponding in-place operator. For example, for the\n operator \'``+``\', ``__add__()`` and ``__radd__()`` are used for the\n left and right variant of the binary operator, and ``__iadd__()``\n for the in-place variant.\n\n* For objects *x* and *y*, first ``x.__op__(y)`` is tried. If this is\n not implemented or returns ``NotImplemented``, ``y.__rop__(x)`` is\n tried. If this is also not implemented or returns\n ``NotImplemented``, a ``TypeError`` exception is raised. But see\n the following exception:\n\n* Exception to the previous item: if the left operand is an instance\n of a built-in type or a new-style class, and the right operand is an\n instance of a proper subclass of that type or class and overrides\n the base\'s ``__rop__()`` method, the right operand\'s ``__rop__()``\n method is tried *before* the left operand\'s ``__op__()`` method.\n\n This is done so that a subclass can completely override binary\n operators. Otherwise, the left operand\'s ``__op__()`` method would\n always accept the right operand: when an instance of a given class\n is expected, an instance of a subclass of that class is always\n acceptable.\n\n* When either operand type defines a coercion, this coercion is called\n before that type\'s ``__op__()`` or ``__rop__()`` method is called,\n but no sooner. If the coercion returns an object of a different\n type for the operand whose coercion is invoked, part of the process\n is redone using the new object.\n\n* When an in-place operator (like \'``+=``\') is used, if the left\n operand implements ``__iop__()``, it is invoked without any\n coercion. When the operation falls back to ``__op__()`` and/or\n ``__rop__()``, the normal coercion rules apply.\n\n* In ``x + y``, if *x* is a sequence that implements sequence\n concatenation, sequence concatenation is invoked.\n\n* In ``x * y``, if one operator is a sequence that implements sequence\n repetition, and the other is an integer (``int`` or ``long``),\n sequence repetition is invoked.\n\n* Rich comparisons (implemented by methods ``__eq__()`` and so on)\n never use coercion. Three-way comparison (implemented by\n ``__cmp__()``) does use coercion under the same conditions as other\n binary operations use it.\n\n* In the current implementation, the built-in numeric types ``int``,\n ``long``, ``float``, and ``complex`` do not use coercion. All these\n types implement a ``__coerce__()`` method, for use by the built-in\n ``coerce()`` function.\n\n Changed in version 2.7.\n\n\nWith Statement Context Managers\n===============================\n\nNew in version 2.5.\n\nA *context manager* is an object that defines the runtime context to\nbe established when executing a ``with`` statement. The context\nmanager handles the entry into, and the exit from, the desired runtime\ncontext for the execution of the block of code. Context managers are\nnormally invoked using the ``with`` statement (described in section\n*The with statement*), but can also be used by directly invoking their\nmethods.\n\nTypical uses of context managers include saving and restoring various\nkinds of global state, locking and unlocking resources, closing opened\nfiles, etc.\n\nFor more information on context managers, see *Context Manager Types*.\n\nobject.__enter__(self)\n\n Enter the runtime context related to this object. The ``with``\n statement will bind this method\'s return value to the target(s)\n specified in the ``as`` clause of the statement, if any.\n\nobject.__exit__(self, exc_type, exc_value, traceback)\n\n Exit the runtime context related to this object. The parameters\n describe the exception that caused the context to be exited. If the\n context was exited without an exception, all three arguments will\n be ``None``.\n\n If an exception is supplied, and the method wishes to suppress the\n exception (i.e., prevent it from being propagated), it should\n return a true value. Otherwise, the exception will be processed\n normally upon exit from this method.\n\n Note that ``__exit__()`` methods should not reraise the passed-in\n exception; this is the caller\'s responsibility.\n\nSee also:\n\n **PEP 0343** - The "with" statement\n The specification, background, and examples for the Python\n ``with`` statement.\n\n\nSpecial method lookup for old-style classes\n===========================================\n\nFor old-style classes, special methods are always looked up in exactly\nthe same way as any other method or attribute. This is the case\nregardless of whether the method is being looked up explicitly as in\n``x.__getitem__(i)`` or implicitly as in ``x[i]``.\n\nThis behaviour means that special methods may exhibit different\nbehaviour for different instances of a single old-style class if the\nappropriate special attributes are set differently:\n\n >>> class C:\n ... pass\n ...\n >>> c1 = C()\n >>> c2 = C()\n >>> c1.__len__ = lambda: 5\n >>> c2.__len__ = lambda: 9\n >>> len(c1)\n 5\n >>> len(c2)\n 9\n\n\nSpecial method lookup for new-style classes\n===========================================\n\nFor new-style classes, implicit invocations of special methods are\nonly guaranteed to work correctly if defined on an object\'s type, not\nin the object\'s instance dictionary. That behaviour is the reason why\nthe following code raises an exception (unlike the equivalent example\nwith old-style classes):\n\n >>> class C(object):\n ... pass\n ...\n >>> c = C()\n >>> c.__len__ = lambda: 5\n >>> len(c)\n Traceback (most recent call last):\n File "<stdin>", line 1, in <module>\n TypeError: object of type \'C\' has no len()\n\nThe rationale behind this behaviour lies with a number of special\nmethods such as ``__hash__()`` and ``__repr__()`` that are implemented\nby all objects, including type objects. If the implicit lookup of\nthese methods used the conventional lookup process, they would fail\nwhen invoked on the type object itself:\n\n >>> 1 .__hash__() == hash(1)\n True\n >>> int.__hash__() == hash(int)\n Traceback (most recent call last):\n File "<stdin>", line 1, in <module>\n TypeError: descriptor \'__hash__\' of \'int\' object needs an argument\n\nIncorrectly attempting to invoke an unbound method of a class in this\nway is sometimes referred to as \'metaclass confusion\', and is avoided\nby bypassing the instance when looking up special methods:\n\n >>> type(1).__hash__(1) == hash(1)\n True\n >>> type(int).__hash__(int) == hash(int)\n True\n\nIn addition to bypassing any instance attributes in the interest of\ncorrectness, implicit special method lookup generally also bypasses\nthe ``__getattribute__()`` method even of the object\'s metaclass:\n\n >>> class Meta(type):\n ... def __getattribute__(*args):\n ... print "Metaclass getattribute invoked"\n ... return type.__getattribute__(*args)\n ...\n >>> class C(object):\n ... __metaclass__ = Meta\n ... def __len__(self):\n ... return 10\n ... def __getattribute__(*args):\n ... print "Class getattribute invoked"\n ... return object.__getattribute__(*args)\n ...\n >>> c = C()\n >>> c.__len__() # Explicit lookup via instance\n Class getattribute invoked\n 10\n >>> type(c).__len__(c) # Explicit lookup via type\n Metaclass getattribute invoked\n 10\n >>> len(c) # Implicit lookup\n 10\n\nBypassing the ``__getattribute__()`` machinery in this fashion\nprovides significant scope for speed optimisations within the\ninterpreter, at the cost of some flexibility in the handling of\nspecial methods (the special method *must* be set on the class object\nitself in order to be consistently invoked by the interpreter).\n\n-[ Footnotes ]-\n\n[1] It *is* possible in some cases to change an object\'s type, under\n certain controlled conditions. It generally isn\'t a good idea\n though, since it can lead to some very strange behaviour if it is\n handled incorrectly.\n\n[2] For operands of the same type, it is assumed that if the non-\n reflected method (such as ``__add__()``) fails the operation is\n not supported, which is why the reflected method is not called.\n', 'string-conversions': u'\nString conversions\n******************\n\nA string conversion is an expression list enclosed in reverse (a.k.a.\nbackward) quotes:\n\n string_conversion ::= "\'" expression_list "\'"\n\nA string conversion evaluates the contained expression list and\nconverts the resulting object into a string according to rules\nspecific to its type.\n\nIf the object is a string, a number, ``None``, or a tuple, list or\ndictionary containing only objects whose type is one of these, the\nresulting string is a valid Python expression which can be passed to\nthe built-in function ``eval()`` to yield an expression with the same\nvalue (or an approximation, if floating point numbers are involved).\n\n(In particular, converting a string adds quotes around it and converts\n"funny" characters to escape sequences that are safe to print.)\n\nRecursive objects (for example, lists or dictionaries that contain a\nreference to themselves, directly or indirectly) use ``...`` to\nindicate a recursive reference, and the result cannot be passed to\n``eval()`` to get an equal value (``SyntaxError`` will be raised\ninstead).\n\nThe built-in function ``repr()`` performs exactly the same conversion\nin its argument as enclosing it in parentheses and reverse quotes\ndoes. The built-in function ``str()`` performs a similar but more\nuser-friendly conversion.\n', - 'string-methods': u'\nString Methods\n**************\n\nBelow are listed the string methods which both 8-bit strings and\nUnicode objects support.\n\nIn addition, Python\'s strings support the sequence type methods\ndescribed in the *Sequence Types --- str, unicode, list, tuple,\nbuffer, xrange* section. To output formatted strings use template\nstrings or the ``%`` operator described in the *String Formatting\nOperations* section. Also, see the ``re`` module for string functions\nbased on regular expressions.\n\nstr.capitalize()\n\n Return a copy of the string with only its first character\n capitalized.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.center(width[, fillchar])\n\n Return centered in a string of length *width*. Padding is done\n using the specified *fillchar* (default is a space).\n\n Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.count(sub[, start[, end]])\n\n Return the number of non-overlapping occurrences of substring *sub*\n in the range [*start*, *end*]. Optional arguments *start* and\n *end* are interpreted as in slice notation.\n\nstr.decode([encoding[, errors]])\n\n Decodes the string using the codec registered for *encoding*.\n *encoding* defaults to the default string encoding. *errors* may\n be given to set a different error handling scheme. The default is\n ``\'strict\'``, meaning that encoding errors raise ``UnicodeError``.\n Other possible values are ``\'ignore\'``, ``\'replace\'`` and any other\n name registered via ``codecs.register_error()``, see section *Codec\n Base Classes*.\n\n New in version 2.2.\n\n Changed in version 2.3: Support for other error handling schemes\n added.\n\n Changed in version 2.7: Support for keyword arguments added.\n\nstr.encode([encoding[, errors]])\n\n Return an encoded version of the string. Default encoding is the\n current default string encoding. *errors* may be given to set a\n different error handling scheme. The default for *errors* is\n ``\'strict\'``, meaning that encoding errors raise a\n ``UnicodeError``. Other possible values are ``\'ignore\'``,\n ``\'replace\'``, ``\'xmlcharrefreplace\'``, ``\'backslashreplace\'`` and\n any other name registered via ``codecs.register_error()``, see\n section *Codec Base Classes*. For a list of possible encodings, see\n section *Standard Encodings*.\n\n New in version 2.0.\n\n Changed in version 2.3: Support for ``\'xmlcharrefreplace\'`` and\n ``\'backslashreplace\'`` and other error handling schemes added.\n\n Changed in version 2.7: Support for keyword arguments added.\n\nstr.endswith(suffix[, start[, end]])\n\n Return ``True`` if the string ends with the specified *suffix*,\n otherwise return ``False``. *suffix* can also be a tuple of\n suffixes to look for. With optional *start*, test beginning at\n that position. With optional *end*, stop comparing at that\n position.\n\n Changed in version 2.5: Accept tuples as *suffix*.\n\nstr.expandtabs([tabsize])\n\n Return a copy of the string where all tab characters are replaced\n by one or more spaces, depending on the current column and the\n given tab size. The column number is reset to zero after each\n newline occurring in the string. If *tabsize* is not given, a tab\n size of ``8`` characters is assumed. This doesn\'t understand other\n non-printing characters or escape sequences.\n\nstr.find(sub[, start[, end]])\n\n Return the lowest index in the string where substring *sub* is\n found, such that *sub* is contained in the slice ``s[start:end]``.\n Optional arguments *start* and *end* are interpreted as in slice\n notation. Return ``-1`` if *sub* is not found.\n\nstr.format(*args, **kwargs)\n\n Perform a string formatting operation. The string on which this\n method is called can contain literal text or replacement fields\n delimited by braces ``{}``. Each replacement field contains either\n the numeric index of a positional argument, or the name of a\n keyword argument. Returns a copy of the string where each\n replacement field is replaced with the string value of the\n corresponding argument.\n\n >>> "The sum of 1 + 2 is {0}".format(1+2)\n \'The sum of 1 + 2 is 3\'\n\n See *Format String Syntax* for a description of the various\n formatting options that can be specified in format strings.\n\n This method of string formatting is the new standard in Python 3.0,\n and should be preferred to the ``%`` formatting described in\n *String Formatting Operations* in new code.\n\n New in version 2.6.\n\nstr.index(sub[, start[, end]])\n\n Like ``find()``, but raise ``ValueError`` when the substring is not\n found.\n\nstr.isalnum()\n\n Return true if all characters in the string are alphanumeric and\n there is at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isalpha()\n\n Return true if all characters in the string are alphabetic and\n there is at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isdigit()\n\n Return true if all characters in the string are digits and there is\n at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.islower()\n\n Return true if all cased characters in the string are lowercase and\n there is at least one cased character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isspace()\n\n Return true if there are only whitespace characters in the string\n and there is at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.istitle()\n\n Return true if the string is a titlecased string and there is at\n least one character, for example uppercase characters may only\n follow uncased characters and lowercase characters only cased ones.\n Return false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isupper()\n\n Return true if all cased characters in the string are uppercase and\n there is at least one cased character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.join(iterable)\n\n Return a string which is the concatenation of the strings in the\n *iterable* *iterable*. The separator between elements is the\n string providing this method.\n\nstr.ljust(width[, fillchar])\n\n Return the string left justified in a string of length *width*.\n Padding is done using the specified *fillchar* (default is a\n space). The original string is returned if *width* is less than\n ``len(s)``.\n\n Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.lower()\n\n Return a copy of the string converted to lowercase.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.lstrip([chars])\n\n Return a copy of the string with leading characters removed. The\n *chars* argument is a string specifying the set of characters to be\n removed. If omitted or ``None``, the *chars* argument defaults to\n removing whitespace. The *chars* argument is not a prefix; rather,\n all combinations of its values are stripped:\n\n >>> \' spacious \'.lstrip()\n \'spacious \'\n >>> \'www.example.com\'.lstrip(\'cmowz.\')\n \'example.com\'\n\n Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.partition(sep)\n\n Split the string at the first occurrence of *sep*, and return a\n 3-tuple containing the part before the separator, the separator\n itself, and the part after the separator. If the separator is not\n found, return a 3-tuple containing the string itself, followed by\n two empty strings.\n\n New in version 2.5.\n\nstr.replace(old, new[, count])\n\n Return a copy of the string with all occurrences of substring *old*\n replaced by *new*. If the optional argument *count* is given, only\n the first *count* occurrences are replaced.\n\nstr.rfind(sub[, start[, end]])\n\n Return the highest index in the string where substring *sub* is\n found, such that *sub* is contained within ``s[start:end]``.\n Optional arguments *start* and *end* are interpreted as in slice\n notation. Return ``-1`` on failure.\n\nstr.rindex(sub[, start[, end]])\n\n Like ``rfind()`` but raises ``ValueError`` when the substring *sub*\n is not found.\n\nstr.rjust(width[, fillchar])\n\n Return the string right justified in a string of length *width*.\n Padding is done using the specified *fillchar* (default is a\n space). The original string is returned if *width* is less than\n ``len(s)``.\n\n Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.rpartition(sep)\n\n Split the string at the last occurrence of *sep*, and return a\n 3-tuple containing the part before the separator, the separator\n itself, and the part after the separator. If the separator is not\n found, return a 3-tuple containing two empty strings, followed by\n the string itself.\n\n New in version 2.5.\n\nstr.rsplit([sep[, maxsplit]])\n\n Return a list of the words in the string, using *sep* as the\n delimiter string. If *maxsplit* is given, at most *maxsplit* splits\n are done, the *rightmost* ones. If *sep* is not specified or\n ``None``, any whitespace string is a separator. Except for\n splitting from the right, ``rsplit()`` behaves like ``split()``\n which is described in detail below.\n\n New in version 2.4.\n\nstr.rstrip([chars])\n\n Return a copy of the string with trailing characters removed. The\n *chars* argument is a string specifying the set of characters to be\n removed. If omitted or ``None``, the *chars* argument defaults to\n removing whitespace. The *chars* argument is not a suffix; rather,\n all combinations of its values are stripped:\n\n >>> \' spacious \'.rstrip()\n \' spacious\'\n >>> \'mississippi\'.rstrip(\'ipz\')\n \'mississ\'\n\n Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.split([sep[, maxsplit]])\n\n Return a list of the words in the string, using *sep* as the\n delimiter string. If *maxsplit* is given, at most *maxsplit*\n splits are done (thus, the list will have at most ``maxsplit+1``\n elements). If *maxsplit* is not specified, then there is no limit\n on the number of splits (all possible splits are made).\n\n If *sep* is given, consecutive delimiters are not grouped together\n and are deemed to delimit empty strings (for example,\n ``\'1,,2\'.split(\',\')`` returns ``[\'1\', \'\', \'2\']``). The *sep*\n argument may consist of multiple characters (for example,\n ``\'1<>2<>3\'.split(\'<>\')`` returns ``[\'1\', \'2\', \'3\']``). Splitting\n an empty string with a specified separator returns ``[\'\']``.\n\n If *sep* is not specified or is ``None``, a different splitting\n algorithm is applied: runs of consecutive whitespace are regarded\n as a single separator, and the result will contain no empty strings\n at the start or end if the string has leading or trailing\n whitespace. Consequently, splitting an empty string or a string\n consisting of just whitespace with a ``None`` separator returns\n ``[]``.\n\n For example, ``\' 1 2 3 \'.split()`` returns ``[\'1\', \'2\', \'3\']``,\n and ``\' 1 2 3 \'.split(None, 1)`` returns ``[\'1\', \'2 3 \']``.\n\nstr.splitlines([keepends])\n\n Return a list of the lines in the string, breaking at line\n boundaries. Line breaks are not included in the resulting list\n unless *keepends* is given and true.\n\nstr.startswith(prefix[, start[, end]])\n\n Return ``True`` if string starts with the *prefix*, otherwise\n return ``False``. *prefix* can also be a tuple of prefixes to look\n for. With optional *start*, test string beginning at that\n position. With optional *end*, stop comparing string at that\n position.\n\n Changed in version 2.5: Accept tuples as *prefix*.\n\nstr.strip([chars])\n\n Return a copy of the string with the leading and trailing\n characters removed. The *chars* argument is a string specifying the\n set of characters to be removed. If omitted or ``None``, the\n *chars* argument defaults to removing whitespace. The *chars*\n argument is not a prefix or suffix; rather, all combinations of its\n values are stripped:\n\n >>> \' spacious \'.strip()\n \'spacious\'\n >>> \'www.example.com\'.strip(\'cmowz.\')\n \'example\'\n\n Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.swapcase()\n\n Return a copy of the string with uppercase characters converted to\n lowercase and vice versa.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.title()\n\n Return a titlecased version of the string where words start with an\n uppercase character and the remaining characters are lowercase.\n\n The algorithm uses a simple language-independent definition of a\n word as groups of consecutive letters. The definition works in\n many contexts but it means that apostrophes in contractions and\n possessives form word boundaries, which may not be the desired\n result:\n\n >>> "they\'re bill\'s friends from the UK".title()\n "They\'Re Bill\'S Friends From The Uk"\n\n A workaround for apostrophes can be constructed using regular\n expressions:\n\n >>> import re\n >>> def titlecase(s):\n return re.sub(r"[A-Za-z]+(\'[A-Za-z]+)?",\n lambda mo: mo.group(0)[0].upper() +\n mo.group(0)[1:].lower(),\n s)\n\n >>> titlecase("they\'re bill\'s friends.")\n "They\'re Bill\'s Friends."\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.translate(table[, deletechars])\n\n Return a copy of the string where all characters occurring in the\n optional argument *deletechars* are removed, and the remaining\n characters have been mapped through the given translation table,\n which must be a string of length 256.\n\n You can use the ``maketrans()`` helper function in the ``string``\n module to create a translation table. For string objects, set the\n *table* argument to ``None`` for translations that only delete\n characters:\n\n >>> \'read this short text\'.translate(None, \'aeiou\')\n \'rd ths shrt txt\'\n\n New in version 2.6: Support for a ``None`` *table* argument.\n\n For Unicode objects, the ``translate()`` method does not accept the\n optional *deletechars* argument. Instead, it returns a copy of the\n *s* where all characters have been mapped through the given\n translation table which must be a mapping of Unicode ordinals to\n Unicode ordinals, Unicode strings or ``None``. Unmapped characters\n are left untouched. Characters mapped to ``None`` are deleted.\n Note, a more flexible approach is to create a custom character\n mapping codec using the ``codecs`` module (see ``encodings.cp1251``\n for an example).\n\nstr.upper()\n\n Return a copy of the string converted to uppercase.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.zfill(width)\n\n Return the numeric string left filled with zeros in a string of\n length *width*. A sign prefix is handled correctly. The original\n string is returned if *width* is less than ``len(s)``.\n\n New in version 2.2.2.\n\nThe following methods are present only on unicode objects:\n\nunicode.isnumeric()\n\n Return ``True`` if there are only numeric characters in S,\n ``False`` otherwise. Numeric characters include digit characters,\n and all characters that have the Unicode numeric value property,\n e.g. U+2155, VULGAR FRACTION ONE FIFTH.\n\nunicode.isdecimal()\n\n Return ``True`` if there are only decimal characters in S,\n ``False`` otherwise. Decimal characters include digit characters,\n and all characters that that can be used to form decimal-radix\n numbers, e.g. U+0660, ARABIC-INDIC DIGIT ZERO.\n', - 'strings': u'\nString literals\n***************\n\nString literals are described by the following lexical definitions:\n\n stringliteral ::= [stringprefix](shortstring | longstring)\n stringprefix ::= "r" | "u" | "ur" | "R" | "U" | "UR" | "Ur" | "uR"\n shortstring ::= "\'" shortstringitem* "\'" | \'"\' shortstringitem* \'"\'\n longstring ::= "\'\'\'" longstringitem* "\'\'\'"\n | \'"""\' longstringitem* \'"""\'\n shortstringitem ::= shortstringchar | escapeseq\n longstringitem ::= longstringchar | escapeseq\n shortstringchar ::= <any source character except "\\" or newline or the quote>\n longstringchar ::= <any source character except "\\">\n escapeseq ::= "\\" <any ASCII character>\n\nOne syntactic restriction not indicated by these productions is that\nwhitespace is not allowed between the **stringprefix** and the rest of\nthe string literal. The source character set is defined by the\nencoding declaration; it is ASCII if no encoding declaration is given\nin the source file; see section *Encoding declarations*.\n\nIn plain English: String literals can be enclosed in matching single\nquotes (``\'``) or double quotes (``"``). They can also be enclosed in\nmatching groups of three single or double quotes (these are generally\nreferred to as *triple-quoted strings*). The backslash (``\\``)\ncharacter is used to escape characters that otherwise have a special\nmeaning, such as newline, backslash itself, or the quote character.\nString literals may optionally be prefixed with a letter ``\'r\'`` or\n``\'R\'``; such strings are called *raw strings* and use different rules\nfor interpreting backslash escape sequences. A prefix of ``\'u\'`` or\n``\'U\'`` makes the string a Unicode string. Unicode strings use the\nUnicode character set as defined by the Unicode Consortium and ISO\n10646. Some additional escape sequences, described below, are\navailable in Unicode strings. The two prefix characters may be\ncombined; in this case, ``\'u\'`` must appear before ``\'r\'``.\n\nIn triple-quoted strings, unescaped newlines and quotes are allowed\n(and are retained), except that three unescaped quotes in a row\nterminate the string. (A "quote" is the character used to open the\nstring, i.e. either ``\'`` or ``"``.)\n\nUnless an ``\'r\'`` or ``\'R\'`` prefix is present, escape sequences in\nstrings are interpreted according to rules similar to those used by\nStandard C. The recognized escape sequences are:\n\n+-------------------+-----------------------------------+---------+\n| Escape Sequence | Meaning | Notes |\n+===================+===================================+=========+\n| ``\\newline`` | Ignored | |\n+-------------------+-----------------------------------+---------+\n| ``\\\\`` | Backslash (``\\``) | |\n+-------------------+-----------------------------------+---------+\n| ``\\\'`` | Single quote (``\'``) | |\n+-------------------+-----------------------------------+---------+\n| ``\\"`` | Double quote (``"``) | |\n+-------------------+-----------------------------------+---------+\n| ``\\a`` | ASCII Bell (BEL) | |\n+-------------------+-----------------------------------+---------+\n| ``\\b`` | ASCII Backspace (BS) | |\n+-------------------+-----------------------------------+---------+\n| ``\\f`` | ASCII Formfeed (FF) | |\n+-------------------+-----------------------------------+---------+\n| ``\\n`` | ASCII Linefeed (LF) | |\n+-------------------+-----------------------------------+---------+\n| ``\\N{name}`` | Character named *name* in the | |\n| | Unicode database (Unicode only) | |\n+-------------------+-----------------------------------+---------+\n| ``\\r`` | ASCII Carriage Return (CR) | |\n+-------------------+-----------------------------------+---------+\n| ``\\t`` | ASCII Horizontal Tab (TAB) | |\n+-------------------+-----------------------------------+---------+\n| ``\\uxxxx`` | Character with 16-bit hex value | (1) |\n| | *xxxx* (Unicode only) | |\n+-------------------+-----------------------------------+---------+\n| ``\\Uxxxxxxxx`` | Character with 32-bit hex value | (2) |\n| | *xxxxxxxx* (Unicode only) | |\n+-------------------+-----------------------------------+---------+\n| ``\\v`` | ASCII Vertical Tab (VT) | |\n+-------------------+-----------------------------------+---------+\n| ``\\ooo`` | Character with octal value *ooo* | (3,5) |\n+-------------------+-----------------------------------+---------+\n| ``\\xhh`` | Character with hex value *hh* | (4,5) |\n+-------------------+-----------------------------------+---------+\n\nNotes:\n\n1. Individual code units which form parts of a surrogate pair can be\n encoded using this escape sequence.\n\n2. Any Unicode character can be encoded this way, but characters\n outside the Basic Multilingual Plane (BMP) will be encoded using a\n surrogate pair if Python is compiled to use 16-bit code units (the\n default). Individual code units which form parts of a surrogate\n pair can be encoded using this escape sequence.\n\n3. As in Standard C, up to three octal digits are accepted.\n\n4. Unlike in Standard C, exactly two hex digits are required.\n\n5. In a string literal, hexadecimal and octal escapes denote the byte\n with the given value; it is not necessary that the byte encodes a\n character in the source character set. In a Unicode literal, these\n escapes denote a Unicode character with the given value.\n\nUnlike Standard C, all unrecognized escape sequences are left in the\nstring unchanged, i.e., *the backslash is left in the string*. (This\nbehavior is useful when debugging: if an escape sequence is mistyped,\nthe resulting output is more easily recognized as broken.) It is also\nimportant to note that the escape sequences marked as "(Unicode only)"\nin the table above fall into the category of unrecognized escapes for\nnon-Unicode string literals.\n\nWhen an ``\'r\'`` or ``\'R\'`` prefix is present, a character following a\nbackslash is included in the string without change, and *all\nbackslashes are left in the string*. For example, the string literal\n``r"\\n"`` consists of two characters: a backslash and a lowercase\n``\'n\'``. String quotes can be escaped with a backslash, but the\nbackslash remains in the string; for example, ``r"\\""`` is a valid\nstring literal consisting of two characters: a backslash and a double\nquote; ``r"\\"`` is not a valid string literal (even a raw string\ncannot end in an odd number of backslashes). Specifically, *a raw\nstring cannot end in a single backslash* (since the backslash would\nescape the following quote character). Note also that a single\nbackslash followed by a newline is interpreted as those two characters\nas part of the string, *not* as a line continuation.\n\nWhen an ``\'r\'`` or ``\'R\'`` prefix is used in conjunction with a\n``\'u\'`` or ``\'U\'`` prefix, then the ``\\uXXXX`` and ``\\UXXXXXXXX``\nescape sequences are processed while *all other backslashes are left\nin the string*. For example, the string literal ``ur"\\u0062\\n"``\nconsists of three Unicode characters: \'LATIN SMALL LETTER B\', \'REVERSE\nSOLIDUS\', and \'LATIN SMALL LETTER N\'. Backslashes can be escaped with\na preceding backslash; however, both remain in the string. As a\nresult, ``\\uXXXX`` escape sequences are only recognized when there are\nan odd number of backslashes.\n', + 'string-methods': u'\nString Methods\n**************\n\nBelow are listed the string methods which both 8-bit strings and\nUnicode objects support. Some of them are also available on\n``bytearray`` objects.\n\nIn addition, Python\'s strings support the sequence type methods\ndescribed in the *Sequence Types --- str, unicode, list, tuple,\nbytearray, buffer, xrange* section. To output formatted strings use\ntemplate strings or the ``%`` operator described in the *String\nFormatting Operations* section. Also, see the ``re`` module for string\nfunctions based on regular expressions.\n\nstr.capitalize()\n\n Return a copy of the string with its first character capitalized\n and the rest lowercased.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.center(width[, fillchar])\n\n Return centered in a string of length *width*. Padding is done\n using the specified *fillchar* (default is a space).\n\n Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.count(sub[, start[, end]])\n\n Return the number of non-overlapping occurrences of substring *sub*\n in the range [*start*, *end*]. Optional arguments *start* and\n *end* are interpreted as in slice notation.\n\nstr.decode([encoding[, errors]])\n\n Decodes the string using the codec registered for *encoding*.\n *encoding* defaults to the default string encoding. *errors* may\n be given to set a different error handling scheme. The default is\n ``\'strict\'``, meaning that encoding errors raise ``UnicodeError``.\n Other possible values are ``\'ignore\'``, ``\'replace\'`` and any other\n name registered via ``codecs.register_error()``, see section *Codec\n Base Classes*.\n\n New in version 2.2.\n\n Changed in version 2.3: Support for other error handling schemes\n added.\n\n Changed in version 2.7: Support for keyword arguments added.\n\nstr.encode([encoding[, errors]])\n\n Return an encoded version of the string. Default encoding is the\n current default string encoding. *errors* may be given to set a\n different error handling scheme. The default for *errors* is\n ``\'strict\'``, meaning that encoding errors raise a\n ``UnicodeError``. Other possible values are ``\'ignore\'``,\n ``\'replace\'``, ``\'xmlcharrefreplace\'``, ``\'backslashreplace\'`` and\n any other name registered via ``codecs.register_error()``, see\n section *Codec Base Classes*. For a list of possible encodings, see\n section *Standard Encodings*.\n\n New in version 2.0.\n\n Changed in version 2.3: Support for ``\'xmlcharrefreplace\'`` and\n ``\'backslashreplace\'`` and other error handling schemes added.\n\n Changed in version 2.7: Support for keyword arguments added.\n\nstr.endswith(suffix[, start[, end]])\n\n Return ``True`` if the string ends with the specified *suffix*,\n otherwise return ``False``. *suffix* can also be a tuple of\n suffixes to look for. With optional *start*, test beginning at\n that position. With optional *end*, stop comparing at that\n position.\n\n Changed in version 2.5: Accept tuples as *suffix*.\n\nstr.expandtabs([tabsize])\n\n Return a copy of the string where all tab characters are replaced\n by one or more spaces, depending on the current column and the\n given tab size. The column number is reset to zero after each\n newline occurring in the string. If *tabsize* is not given, a tab\n size of ``8`` characters is assumed. This doesn\'t understand other\n non-printing characters or escape sequences.\n\nstr.find(sub[, start[, end]])\n\n Return the lowest index in the string where substring *sub* is\n found, such that *sub* is contained in the slice ``s[start:end]``.\n Optional arguments *start* and *end* are interpreted as in slice\n notation. Return ``-1`` if *sub* is not found.\n\n Note: The ``find()`` method should be used only if you need to know the\n position of *sub*. To check if *sub* is a substring or not, use\n the ``in`` operator:\n\n >>> \'Py\' in \'Python\'\n True\n\nstr.format(*args, **kwargs)\n\n Perform a string formatting operation. The string on which this\n method is called can contain literal text or replacement fields\n delimited by braces ``{}``. Each replacement field contains either\n the numeric index of a positional argument, or the name of a\n keyword argument. Returns a copy of the string where each\n replacement field is replaced with the string value of the\n corresponding argument.\n\n >>> "The sum of 1 + 2 is {0}".format(1+2)\n \'The sum of 1 + 2 is 3\'\n\n See *Format String Syntax* for a description of the various\n formatting options that can be specified in format strings.\n\n This method of string formatting is the new standard in Python 3.0,\n and should be preferred to the ``%`` formatting described in\n *String Formatting Operations* in new code.\n\n New in version 2.6.\n\nstr.index(sub[, start[, end]])\n\n Like ``find()``, but raise ``ValueError`` when the substring is not\n found.\n\nstr.isalnum()\n\n Return true if all characters in the string are alphanumeric and\n there is at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isalpha()\n\n Return true if all characters in the string are alphabetic and\n there is at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isdigit()\n\n Return true if all characters in the string are digits and there is\n at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.islower()\n\n Return true if all cased characters in the string are lowercase and\n there is at least one cased character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isspace()\n\n Return true if there are only whitespace characters in the string\n and there is at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.istitle()\n\n Return true if the string is a titlecased string and there is at\n least one character, for example uppercase characters may only\n follow uncased characters and lowercase characters only cased ones.\n Return false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isupper()\n\n Return true if all cased characters in the string are uppercase and\n there is at least one cased character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.join(iterable)\n\n Return a string which is the concatenation of the strings in the\n *iterable* *iterable*. The separator between elements is the\n string providing this method.\n\nstr.ljust(width[, fillchar])\n\n Return the string left justified in a string of length *width*.\n Padding is done using the specified *fillchar* (default is a\n space). The original string is returned if *width* is less than\n ``len(s)``.\n\n Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.lower()\n\n Return a copy of the string converted to lowercase.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.lstrip([chars])\n\n Return a copy of the string with leading characters removed. The\n *chars* argument is a string specifying the set of characters to be\n removed. If omitted or ``None``, the *chars* argument defaults to\n removing whitespace. The *chars* argument is not a prefix; rather,\n all combinations of its values are stripped:\n\n >>> \' spacious \'.lstrip()\n \'spacious \'\n >>> \'www.example.com\'.lstrip(\'cmowz.\')\n \'example.com\'\n\n Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.partition(sep)\n\n Split the string at the first occurrence of *sep*, and return a\n 3-tuple containing the part before the separator, the separator\n itself, and the part after the separator. If the separator is not\n found, return a 3-tuple containing the string itself, followed by\n two empty strings.\n\n New in version 2.5.\n\nstr.replace(old, new[, count])\n\n Return a copy of the string with all occurrences of substring *old*\n replaced by *new*. If the optional argument *count* is given, only\n the first *count* occurrences are replaced.\n\nstr.rfind(sub[, start[, end]])\n\n Return the highest index in the string where substring *sub* is\n found, such that *sub* is contained within ``s[start:end]``.\n Optional arguments *start* and *end* are interpreted as in slice\n notation. Return ``-1`` on failure.\n\nstr.rindex(sub[, start[, end]])\n\n Like ``rfind()`` but raises ``ValueError`` when the substring *sub*\n is not found.\n\nstr.rjust(width[, fillchar])\n\n Return the string right justified in a string of length *width*.\n Padding is done using the specified *fillchar* (default is a\n space). The original string is returned if *width* is less than\n ``len(s)``.\n\n Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.rpartition(sep)\n\n Split the string at the last occurrence of *sep*, and return a\n 3-tuple containing the part before the separator, the separator\n itself, and the part after the separator. If the separator is not\n found, return a 3-tuple containing two empty strings, followed by\n the string itself.\n\n New in version 2.5.\n\nstr.rsplit([sep[, maxsplit]])\n\n Return a list of the words in the string, using *sep* as the\n delimiter string. If *maxsplit* is given, at most *maxsplit* splits\n are done, the *rightmost* ones. If *sep* is not specified or\n ``None``, any whitespace string is a separator. Except for\n splitting from the right, ``rsplit()`` behaves like ``split()``\n which is described in detail below.\n\n New in version 2.4.\n\nstr.rstrip([chars])\n\n Return a copy of the string with trailing characters removed. The\n *chars* argument is a string specifying the set of characters to be\n removed. If omitted or ``None``, the *chars* argument defaults to\n removing whitespace. The *chars* argument is not a suffix; rather,\n all combinations of its values are stripped:\n\n >>> \' spacious \'.rstrip()\n \' spacious\'\n >>> \'mississippi\'.rstrip(\'ipz\')\n \'mississ\'\n\n Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.split([sep[, maxsplit]])\n\n Return a list of the words in the string, using *sep* as the\n delimiter string. If *maxsplit* is given, at most *maxsplit*\n splits are done (thus, the list will have at most ``maxsplit+1``\n elements). If *maxsplit* is not specified, then there is no limit\n on the number of splits (all possible splits are made).\n\n If *sep* is given, consecutive delimiters are not grouped together\n and are deemed to delimit empty strings (for example,\n ``\'1,,2\'.split(\',\')`` returns ``[\'1\', \'\', \'2\']``). The *sep*\n argument may consist of multiple characters (for example,\n ``\'1<>2<>3\'.split(\'<>\')`` returns ``[\'1\', \'2\', \'3\']``). Splitting\n an empty string with a specified separator returns ``[\'\']``.\n\n If *sep* is not specified or is ``None``, a different splitting\n algorithm is applied: runs of consecutive whitespace are regarded\n as a single separator, and the result will contain no empty strings\n at the start or end if the string has leading or trailing\n whitespace. Consequently, splitting an empty string or a string\n consisting of just whitespace with a ``None`` separator returns\n ``[]``.\n\n For example, ``\' 1 2 3 \'.split()`` returns ``[\'1\', \'2\', \'3\']``,\n and ``\' 1 2 3 \'.split(None, 1)`` returns ``[\'1\', \'2 3 \']``.\n\nstr.splitlines([keepends])\n\n Return a list of the lines in the string, breaking at line\n boundaries. Line breaks are not included in the resulting list\n unless *keepends* is given and true.\n\nstr.startswith(prefix[, start[, end]])\n\n Return ``True`` if string starts with the *prefix*, otherwise\n return ``False``. *prefix* can also be a tuple of prefixes to look\n for. With optional *start*, test string beginning at that\n position. With optional *end*, stop comparing string at that\n position.\n\n Changed in version 2.5: Accept tuples as *prefix*.\n\nstr.strip([chars])\n\n Return a copy of the string with the leading and trailing\n characters removed. The *chars* argument is a string specifying the\n set of characters to be removed. If omitted or ``None``, the\n *chars* argument defaults to removing whitespace. The *chars*\n argument is not a prefix or suffix; rather, all combinations of its\n values are stripped:\n\n >>> \' spacious \'.strip()\n \'spacious\'\n >>> \'www.example.com\'.strip(\'cmowz.\')\n \'example\'\n\n Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.swapcase()\n\n Return a copy of the string with uppercase characters converted to\n lowercase and vice versa.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.title()\n\n Return a titlecased version of the string where words start with an\n uppercase character and the remaining characters are lowercase.\n\n The algorithm uses a simple language-independent definition of a\n word as groups of consecutive letters. The definition works in\n many contexts but it means that apostrophes in contractions and\n possessives form word boundaries, which may not be the desired\n result:\n\n >>> "they\'re bill\'s friends from the UK".title()\n "They\'Re Bill\'S Friends From The Uk"\n\n A workaround for apostrophes can be constructed using regular\n expressions:\n\n >>> import re\n >>> def titlecase(s):\n return re.sub(r"[A-Za-z]+(\'[A-Za-z]+)?",\n lambda mo: mo.group(0)[0].upper() +\n mo.group(0)[1:].lower(),\n s)\n\n >>> titlecase("they\'re bill\'s friends.")\n "They\'re Bill\'s Friends."\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.translate(table[, deletechars])\n\n Return a copy of the string where all characters occurring in the\n optional argument *deletechars* are removed, and the remaining\n characters have been mapped through the given translation table,\n which must be a string of length 256.\n\n You can use the ``maketrans()`` helper function in the ``string``\n module to create a translation table. For string objects, set the\n *table* argument to ``None`` for translations that only delete\n characters:\n\n >>> \'read this short text\'.translate(None, \'aeiou\')\n \'rd ths shrt txt\'\n\n New in version 2.6: Support for a ``None`` *table* argument.\n\n For Unicode objects, the ``translate()`` method does not accept the\n optional *deletechars* argument. Instead, it returns a copy of the\n *s* where all characters have been mapped through the given\n translation table which must be a mapping of Unicode ordinals to\n Unicode ordinals, Unicode strings or ``None``. Unmapped characters\n are left untouched. Characters mapped to ``None`` are deleted.\n Note, a more flexible approach is to create a custom character\n mapping codec using the ``codecs`` module (see ``encodings.cp1251``\n for an example).\n\nstr.upper()\n\n Return a copy of the string converted to uppercase.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.zfill(width)\n\n Return the numeric string left filled with zeros in a string of\n length *width*. A sign prefix is handled correctly. The original\n string is returned if *width* is less than ``len(s)``.\n\n New in version 2.2.2.\n\nThe following methods are present only on unicode objects:\n\nunicode.isnumeric()\n\n Return ``True`` if there are only numeric characters in S,\n ``False`` otherwise. Numeric characters include digit characters,\n and all characters that have the Unicode numeric value property,\n e.g. U+2155, VULGAR FRACTION ONE FIFTH.\n\nunicode.isdecimal()\n\n Return ``True`` if there are only decimal characters in S,\n ``False`` otherwise. Decimal characters include digit characters,\n and all characters that that can be used to form decimal-radix\n numbers, e.g. U+0660, ARABIC-INDIC DIGIT ZERO.\n', + 'strings': u'\nString literals\n***************\n\nString literals are described by the following lexical definitions:\n\n stringliteral ::= [stringprefix](shortstring | longstring)\n stringprefix ::= "r" | "u" | "ur" | "R" | "U" | "UR" | "Ur" | "uR"\n | "b" | "B" | "br" | "Br" | "bR" | "BR"\n shortstring ::= "\'" shortstringitem* "\'" | \'"\' shortstringitem* \'"\'\n longstring ::= "\'\'\'" longstringitem* "\'\'\'"\n | \'"""\' longstringitem* \'"""\'\n shortstringitem ::= shortstringchar | escapeseq\n longstringitem ::= longstringchar | escapeseq\n shortstringchar ::= <any source character except "\\" or newline or the quote>\n longstringchar ::= <any source character except "\\">\n escapeseq ::= "\\" <any ASCII character>\n\nOne syntactic restriction not indicated by these productions is that\nwhitespace is not allowed between the **stringprefix** and the rest of\nthe string literal. The source character set is defined by the\nencoding declaration; it is ASCII if no encoding declaration is given\nin the source file; see section *Encoding declarations*.\n\nIn plain English: String literals can be enclosed in matching single\nquotes (``\'``) or double quotes (``"``). They can also be enclosed in\nmatching groups of three single or double quotes (these are generally\nreferred to as *triple-quoted strings*). The backslash (``\\``)\ncharacter is used to escape characters that otherwise have a special\nmeaning, such as newline, backslash itself, or the quote character.\nString literals may optionally be prefixed with a letter ``\'r\'`` or\n``\'R\'``; such strings are called *raw strings* and use different rules\nfor interpreting backslash escape sequences. A prefix of ``\'u\'`` or\n``\'U\'`` makes the string a Unicode string. Unicode strings use the\nUnicode character set as defined by the Unicode Consortium and ISO\n10646. Some additional escape sequences, described below, are\navailable in Unicode strings. A prefix of ``\'b\'`` or ``\'B\'`` is\nignored in Python 2; it indicates that the literal should become a\nbytes literal in Python 3 (e.g. when code is automatically converted\nwith 2to3). A ``\'u\'`` or ``\'b\'`` prefix may be followed by an ``\'r\'``\nprefix.\n\nIn triple-quoted strings, unescaped newlines and quotes are allowed\n(and are retained), except that three unescaped quotes in a row\nterminate the string. (A "quote" is the character used to open the\nstring, i.e. either ``\'`` or ``"``.)\n\nUnless an ``\'r\'`` or ``\'R\'`` prefix is present, escape sequences in\nstrings are interpreted according to rules similar to those used by\nStandard C. The recognized escape sequences are:\n\n+-------------------+-----------------------------------+---------+\n| Escape Sequence | Meaning | Notes |\n+===================+===================================+=========+\n| ``\\newline`` | Ignored | |\n+-------------------+-----------------------------------+---------+\n| ``\\\\`` | Backslash (``\\``) | |\n+-------------------+-----------------------------------+---------+\n| ``\\\'`` | Single quote (``\'``) | |\n+-------------------+-----------------------------------+---------+\n| ``\\"`` | Double quote (``"``) | |\n+-------------------+-----------------------------------+---------+\n| ``\\a`` | ASCII Bell (BEL) | |\n+-------------------+-----------------------------------+---------+\n| ``\\b`` | ASCII Backspace (BS) | |\n+-------------------+-----------------------------------+---------+\n| ``\\f`` | ASCII Formfeed (FF) | |\n+-------------------+-----------------------------------+---------+\n| ``\\n`` | ASCII Linefeed (LF) | |\n+-------------------+-----------------------------------+---------+\n| ``\\N{name}`` | Character named *name* in the | |\n| | Unicode database (Unicode only) | |\n+-------------------+-----------------------------------+---------+\n| ``\\r`` | ASCII Carriage Return (CR) | |\n+-------------------+-----------------------------------+---------+\n| ``\\t`` | ASCII Horizontal Tab (TAB) | |\n+-------------------+-----------------------------------+---------+\n| ``\\uxxxx`` | Character with 16-bit hex value | (1) |\n| | *xxxx* (Unicode only) | |\n+-------------------+-----------------------------------+---------+\n| ``\\Uxxxxxxxx`` | Character with 32-bit hex value | (2) |\n| | *xxxxxxxx* (Unicode only) | |\n+-------------------+-----------------------------------+---------+\n| ``\\v`` | ASCII Vertical Tab (VT) | |\n+-------------------+-----------------------------------+---------+\n| ``\\ooo`` | Character with octal value *ooo* | (3,5) |\n+-------------------+-----------------------------------+---------+\n| ``\\xhh`` | Character with hex value *hh* | (4,5) |\n+-------------------+-----------------------------------+---------+\n\nNotes:\n\n1. Individual code units which form parts of a surrogate pair can be\n encoded using this escape sequence.\n\n2. Any Unicode character can be encoded this way, but characters\n outside the Basic Multilingual Plane (BMP) will be encoded using a\n surrogate pair if Python is compiled to use 16-bit code units (the\n default). Individual code units which form parts of a surrogate\n pair can be encoded using this escape sequence.\n\n3. As in Standard C, up to three octal digits are accepted.\n\n4. Unlike in Standard C, exactly two hex digits are required.\n\n5. In a string literal, hexadecimal and octal escapes denote the byte\n with the given value; it is not necessary that the byte encodes a\n character in the source character set. In a Unicode literal, these\n escapes denote a Unicode character with the given value.\n\nUnlike Standard C, all unrecognized escape sequences are left in the\nstring unchanged, i.e., *the backslash is left in the string*. (This\nbehavior is useful when debugging: if an escape sequence is mistyped,\nthe resulting output is more easily recognized as broken.) It is also\nimportant to note that the escape sequences marked as "(Unicode only)"\nin the table above fall into the category of unrecognized escapes for\nnon-Unicode string literals.\n\nWhen an ``\'r\'`` or ``\'R\'`` prefix is present, a character following a\nbackslash is included in the string without change, and *all\nbackslashes are left in the string*. For example, the string literal\n``r"\\n"`` consists of two characters: a backslash and a lowercase\n``\'n\'``. String quotes can be escaped with a backslash, but the\nbackslash remains in the string; for example, ``r"\\""`` is a valid\nstring literal consisting of two characters: a backslash and a double\nquote; ``r"\\"`` is not a valid string literal (even a raw string\ncannot end in an odd number of backslashes). Specifically, *a raw\nstring cannot end in a single backslash* (since the backslash would\nescape the following quote character). Note also that a single\nbackslash followed by a newline is interpreted as those two characters\nas part of the string, *not* as a line continuation.\n\nWhen an ``\'r\'`` or ``\'R\'`` prefix is used in conjunction with a\n``\'u\'`` or ``\'U\'`` prefix, then the ``\\uXXXX`` and ``\\UXXXXXXXX``\nescape sequences are processed while *all other backslashes are left\nin the string*. For example, the string literal ``ur"\\u0062\\n"``\nconsists of three Unicode characters: \'LATIN SMALL LETTER B\', \'REVERSE\nSOLIDUS\', and \'LATIN SMALL LETTER N\'. Backslashes can be escaped with\na preceding backslash; however, both remain in the string. As a\nresult, ``\\uXXXX`` escape sequences are only recognized when there are\nan odd number of backslashes.\n', 'subscriptions': u'\nSubscriptions\n*************\n\nA subscription selects an item of a sequence (string, tuple or list)\nor mapping (dictionary) object:\n\n subscription ::= primary "[" expression_list "]"\n\nThe primary must evaluate to an object of a sequence or mapping type.\n\nIf the primary is a mapping, the expression list must evaluate to an\nobject whose value is one of the keys of the mapping, and the\nsubscription selects the value in the mapping that corresponds to that\nkey. (The expression list is a tuple except if it has exactly one\nitem.)\n\nIf the primary is a sequence, the expression (list) must evaluate to a\nplain integer. If this value is negative, the length of the sequence\nis added to it (so that, e.g., ``x[-1]`` selects the last item of\n``x``.) The resulting value must be a nonnegative integer less than\nthe number of items in the sequence, and the subscription selects the\nitem whose index is that value (counting from zero).\n\nA string\'s items are characters. A character is not a separate data\ntype but a string of exactly one character.\n', 'truth': u"\nTruth Value Testing\n*******************\n\nAny object can be tested for truth value, for use in an ``if`` or\n``while`` condition or as operand of the Boolean operations below. The\nfollowing values are considered false:\n\n* ``None``\n\n* ``False``\n\n* zero of any numeric type, for example, ``0``, ``0L``, ``0.0``,\n ``0j``.\n\n* any empty sequence, for example, ``''``, ``()``, ``[]``.\n\n* any empty mapping, for example, ``{}``.\n\n* instances of user-defined classes, if the class defines a\n ``__nonzero__()`` or ``__len__()`` method, when that method returns\n the integer zero or ``bool`` value ``False``. [1]\n\nAll other values are considered true --- so objects of many types are\nalways true.\n\nOperations and built-in functions that have a Boolean result always\nreturn ``0`` or ``False`` for false and ``1`` or ``True`` for true,\nunless otherwise stated. (Important exception: the Boolean operations\n``or`` and ``and`` always return one of their operands.)\n", 'try': u'\nThe ``try`` statement\n*********************\n\nThe ``try`` statement specifies exception handlers and/or cleanup code\nfor a group of statements:\n\n try_stmt ::= try1_stmt | try2_stmt\n try1_stmt ::= "try" ":" suite\n ("except" [expression [("as" | ",") target]] ":" suite)+\n ["else" ":" suite]\n ["finally" ":" suite]\n try2_stmt ::= "try" ":" suite\n "finally" ":" suite\n\nChanged in version 2.5: In previous versions of Python,\n``try``...``except``...``finally`` did not work. ``try``...``except``\nhad to be nested in ``try``...``finally``.\n\nThe ``except`` clause(s) specify one or more exception handlers. When\nno exception occurs in the ``try`` clause, no exception handler is\nexecuted. When an exception occurs in the ``try`` suite, a search for\nan exception handler is started. This search inspects the except\nclauses in turn until one is found that matches the exception. An\nexpression-less except clause, if present, must be last; it matches\nany exception. For an except clause with an expression, that\nexpression is evaluated, and the clause matches the exception if the\nresulting object is "compatible" with the exception. An object is\ncompatible with an exception if it is the class or a base class of the\nexception object, a tuple containing an item compatible with the\nexception, or, in the (deprecated) case of string exceptions, is the\nraised string itself (note that the object identities must match, i.e.\nit must be the same string object, not just a string with the same\nvalue).\n\nIf no except clause matches the exception, the search for an exception\nhandler continues in the surrounding code and on the invocation stack.\n[1]\n\nIf the evaluation of an expression in the header of an except clause\nraises an exception, the original search for a handler is canceled and\na search starts for the new exception in the surrounding code and on\nthe call stack (it is treated as if the entire ``try`` statement\nraised the exception).\n\nWhen a matching except clause is found, the exception is assigned to\nthe target specified in that except clause, if present, and the except\nclause\'s suite is executed. All except clauses must have an\nexecutable block. When the end of this block is reached, execution\ncontinues normally after the entire try statement. (This means that\nif two nested handlers exist for the same exception, and the exception\noccurs in the try clause of the inner handler, the outer handler will\nnot handle the exception.)\n\nBefore an except clause\'s suite is executed, details about the\nexception are assigned to three variables in the ``sys`` module:\n``sys.exc_type`` receives the object identifying the exception;\n``sys.exc_value`` receives the exception\'s parameter;\n``sys.exc_traceback`` receives a traceback object (see section *The\nstandard type hierarchy*) identifying the point in the program where\nthe exception occurred. These details are also available through the\n``sys.exc_info()`` function, which returns a tuple ``(exc_type,\nexc_value, exc_traceback)``. Use of the corresponding variables is\ndeprecated in favor of this function, since their use is unsafe in a\nthreaded program. As of Python 1.5, the variables are restored to\ntheir previous values (before the call) when returning from a function\nthat handled an exception.\n\nThe optional ``else`` clause is executed if and when control flows off\nthe end of the ``try`` clause. [2] Exceptions in the ``else`` clause\nare not handled by the preceding ``except`` clauses.\n\nIf ``finally`` is present, it specifies a \'cleanup\' handler. The\n``try`` clause is executed, including any ``except`` and ``else``\nclauses. If an exception occurs in any of the clauses and is not\nhandled, the exception is temporarily saved. The ``finally`` clause is\nexecuted. If there is a saved exception, it is re-raised at the end\nof the ``finally`` clause. If the ``finally`` clause raises another\nexception or executes a ``return`` or ``break`` statement, the saved\nexception is lost. The exception information is not available to the\nprogram during execution of the ``finally`` clause.\n\nWhen a ``return``, ``break`` or ``continue`` statement is executed in\nthe ``try`` suite of a ``try``...``finally`` statement, the\n``finally`` clause is also executed \'on the way out.\' A ``continue``\nstatement is illegal in the ``finally`` clause. (The reason is a\nproblem with the current implementation --- this restriction may be\nlifted in the future).\n\nAdditional information on exceptions can be found in section\n*Exceptions*, and information on using the ``raise`` statement to\ngenerate exceptions may be found in section *The raise statement*.\n', - 'types': u'\nThe standard type hierarchy\n***************************\n\nBelow is a list of the types that are built into Python. Extension\nmodules (written in C, Java, or other languages, depending on the\nimplementation) can define additional types. Future versions of\nPython may add types to the type hierarchy (e.g., rational numbers,\nefficiently stored arrays of integers, etc.).\n\nSome of the type descriptions below contain a paragraph listing\n\'special attributes.\' These are attributes that provide access to the\nimplementation and are not intended for general use. Their definition\nmay change in the future.\n\nNone\n This type has a single value. There is a single object with this\n value. This object is accessed through the built-in name ``None``.\n It is used to signify the absence of a value in many situations,\n e.g., it is returned from functions that don\'t explicitly return\n anything. Its truth value is false.\n\nNotImplemented\n This type has a single value. There is a single object with this\n value. This object is accessed through the built-in name\n ``NotImplemented``. Numeric methods and rich comparison methods may\n return this value if they do not implement the operation for the\n operands provided. (The interpreter will then try the reflected\n operation, or some other fallback, depending on the operator.) Its\n truth value is true.\n\nEllipsis\n This type has a single value. There is a single object with this\n value. This object is accessed through the built-in name\n ``Ellipsis``. It is used to indicate the presence of the ``...``\n syntax in a slice. Its truth value is true.\n\n``numbers.Number``\n These are created by numeric literals and returned as results by\n arithmetic operators and arithmetic built-in functions. Numeric\n objects are immutable; once created their value never changes.\n Python numbers are of course strongly related to mathematical\n numbers, but subject to the limitations of numerical representation\n in computers.\n\n Python distinguishes between integers, floating point numbers, and\n complex numbers:\n\n ``numbers.Integral``\n These represent elements from the mathematical set of integers\n (positive and negative).\n\n There are three types of integers:\n\n Plain integers\n These represent numbers in the range -2147483648 through\n 2147483647. (The range may be larger on machines with a\n larger natural word size, but not smaller.) When the result\n of an operation would fall outside this range, the result is\n normally returned as a long integer (in some cases, the\n exception ``OverflowError`` is raised instead). For the\n purpose of shift and mask operations, integers are assumed to\n have a binary, 2\'s complement notation using 32 or more bits,\n and hiding no bits from the user (i.e., all 4294967296\n different bit patterns correspond to different values).\n\n Long integers\n These represent numbers in an unlimited range, subject to\n available (virtual) memory only. For the purpose of shift\n and mask operations, a binary representation is assumed, and\n negative numbers are represented in a variant of 2\'s\n complement which gives the illusion of an infinite string of\n sign bits extending to the left.\n\n Booleans\n These represent the truth values False and True. The two\n objects representing the values False and True are the only\n Boolean objects. The Boolean type is a subtype of plain\n integers, and Boolean values behave like the values 0 and 1,\n respectively, in almost all contexts, the exception being\n that when converted to a string, the strings ``"False"`` or\n ``"True"`` are returned, respectively.\n\n The rules for integer representation are intended to give the\n most meaningful interpretation of shift and mask operations\n involving negative integers and the least surprises when\n switching between the plain and long integer domains. Any\n operation, if it yields a result in the plain integer domain,\n will yield the same result in the long integer domain or when\n using mixed operands. The switch between domains is transparent\n to the programmer.\n\n ``numbers.Real`` (``float``)\n These represent machine-level double precision floating point\n numbers. You are at the mercy of the underlying machine\n architecture (and C or Java implementation) for the accepted\n range and handling of overflow. Python does not support single-\n precision floating point numbers; the savings in processor and\n memory usage that are usually the reason for using these is\n dwarfed by the overhead of using objects in Python, so there is\n no reason to complicate the language with two kinds of floating\n point numbers.\n\n ``numbers.Complex``\n These represent complex numbers as a pair of machine-level\n double precision floating point numbers. The same caveats apply\n as for floating point numbers. The real and imaginary parts of a\n complex number ``z`` can be retrieved through the read-only\n attributes ``z.real`` and ``z.imag``.\n\nSequences\n These represent finite ordered sets indexed by non-negative\n numbers. The built-in function ``len()`` returns the number of\n items of a sequence. When the length of a sequence is *n*, the\n index set contains the numbers 0, 1, ..., *n*-1. Item *i* of\n sequence *a* is selected by ``a[i]``.\n\n Sequences also support slicing: ``a[i:j]`` selects all items with\n index *k* such that *i* ``<=`` *k* ``<`` *j*. When used as an\n expression, a slice is a sequence of the same type. This implies\n that the index set is renumbered so that it starts at 0.\n\n Some sequences also support "extended slicing" with a third "step"\n parameter: ``a[i:j:k]`` selects all items of *a* with index *x*\n where ``x = i + n*k``, *n* ``>=`` ``0`` and *i* ``<=`` *x* ``<``\n *j*.\n\n Sequences are distinguished according to their mutability:\n\n Immutable sequences\n An object of an immutable sequence type cannot change once it is\n created. (If the object contains references to other objects,\n these other objects may be mutable and may be changed; however,\n the collection of objects directly referenced by an immutable\n object cannot change.)\n\n The following types are immutable sequences:\n\n Strings\n The items of a string are characters. There is no separate\n character type; a character is represented by a string of one\n item. Characters represent (at least) 8-bit bytes. The\n built-in functions ``chr()`` and ``ord()`` convert between\n characters and nonnegative integers representing the byte\n values. Bytes with the values 0-127 usually represent the\n corresponding ASCII values, but the interpretation of values\n is up to the program. The string data type is also used to\n represent arrays of bytes, e.g., to hold data read from a\n file.\n\n (On systems whose native character set is not ASCII, strings\n may use EBCDIC in their internal representation, provided the\n functions ``chr()`` and ``ord()`` implement a mapping between\n ASCII and EBCDIC, and string comparison preserves the ASCII\n order. Or perhaps someone can propose a better rule?)\n\n Unicode\n The items of a Unicode object are Unicode code units. A\n Unicode code unit is represented by a Unicode object of one\n item and can hold either a 16-bit or 32-bit value\n representing a Unicode ordinal (the maximum value for the\n ordinal is given in ``sys.maxunicode``, and depends on how\n Python is configured at compile time). Surrogate pairs may\n be present in the Unicode object, and will be reported as two\n separate items. The built-in functions ``unichr()`` and\n ``ord()`` convert between code units and nonnegative integers\n representing the Unicode ordinals as defined in the Unicode\n Standard 3.0. Conversion from and to other encodings are\n possible through the Unicode method ``encode()`` and the\n built-in function ``unicode()``.\n\n Tuples\n The items of a tuple are arbitrary Python objects. Tuples of\n two or more items are formed by comma-separated lists of\n expressions. A tuple of one item (a \'singleton\') can be\n formed by affixing a comma to an expression (an expression by\n itself does not create a tuple, since parentheses must be\n usable for grouping of expressions). An empty tuple can be\n formed by an empty pair of parentheses.\n\n Mutable sequences\n Mutable sequences can be changed after they are created. The\n subscription and slicing notations can be used as the target of\n assignment and ``del`` (delete) statements.\n\n There are currently two intrinsic mutable sequence types:\n\n Lists\n The items of a list are arbitrary Python objects. Lists are\n formed by placing a comma-separated list of expressions in\n square brackets. (Note that there are no special cases needed\n to form lists of length 0 or 1.)\n\n Byte Arrays\n A bytearray object is a mutable array. They are created by\n the built-in ``bytearray()`` constructor. Aside from being\n mutable (and hence unhashable), byte arrays otherwise provide\n the same interface and functionality as immutable bytes\n objects.\n\n The extension module ``array`` provides an additional example of\n a mutable sequence type.\n\nSet types\n These represent unordered, finite sets of unique, immutable\n objects. As such, they cannot be indexed by any subscript. However,\n they can be iterated over, and the built-in function ``len()``\n returns the number of items in a set. Common uses for sets are fast\n membership testing, removing duplicates from a sequence, and\n computing mathematical operations such as intersection, union,\n difference, and symmetric difference.\n\n For set elements, the same immutability rules apply as for\n dictionary keys. Note that numeric types obey the normal rules for\n numeric comparison: if two numbers compare equal (e.g., ``1`` and\n ``1.0``), only one of them can be contained in a set.\n\n There are currently two intrinsic set types:\n\n Sets\n These represent a mutable set. They are created by the built-in\n ``set()`` constructor and can be modified afterwards by several\n methods, such as ``add()``.\n\n Frozen sets\n These represent an immutable set. They are created by the\n built-in ``frozenset()`` constructor. As a frozenset is\n immutable and *hashable*, it can be used again as an element of\n another set, or as a dictionary key.\n\nMappings\n These represent finite sets of objects indexed by arbitrary index\n sets. The subscript notation ``a[k]`` selects the item indexed by\n ``k`` from the mapping ``a``; this can be used in expressions and\n as the target of assignments or ``del`` statements. The built-in\n function ``len()`` returns the number of items in a mapping.\n\n There is currently a single intrinsic mapping type:\n\n Dictionaries\n These represent finite sets of objects indexed by nearly\n arbitrary values. The only types of values not acceptable as\n keys are values containing lists or dictionaries or other\n mutable types that are compared by value rather than by object\n identity, the reason being that the efficient implementation of\n dictionaries requires a key\'s hash value to remain constant.\n Numeric types used for keys obey the normal rules for numeric\n comparison: if two numbers compare equal (e.g., ``1`` and\n ``1.0``) then they can be used interchangeably to index the same\n dictionary entry.\n\n Dictionaries are mutable; they can be created by the ``{...}``\n notation (see section *Dictionary displays*).\n\n The extension modules ``dbm``, ``gdbm``, and ``bsddb`` provide\n additional examples of mapping types.\n\nCallable types\n These are the types to which the function call operation (see\n section *Calls*) can be applied:\n\n User-defined functions\n A user-defined function object is created by a function\n definition (see section *Function definitions*). It should be\n called with an argument list containing the same number of items\n as the function\'s formal parameter list.\n\n Special attributes:\n\n +-------------------------+---------------------------------+-------------+\n | Attribute | Meaning | |\n +=========================+=================================+=============+\n | ``func_doc`` | The function\'s documentation | Writable |\n | | string, or ``None`` if | |\n | | unavailable | |\n +-------------------------+---------------------------------+-------------+\n | ``__doc__`` | Another way of spelling | Writable |\n | | ``func_doc`` | |\n +-------------------------+---------------------------------+-------------+\n | ``func_name`` | The function\'s name | Writable |\n +-------------------------+---------------------------------+-------------+\n | ``__name__`` | Another way of spelling | Writable |\n | | ``func_name`` | |\n +-------------------------+---------------------------------+-------------+\n | ``__module__`` | The name of the module the | Writable |\n | | function was defined in, or | |\n | | ``None`` if unavailable. | |\n +-------------------------+---------------------------------+-------------+\n | ``func_defaults`` | A tuple containing default | Writable |\n | | argument values for those | |\n | | arguments that have defaults, | |\n | | or ``None`` if no arguments | |\n | | have a default value | |\n +-------------------------+---------------------------------+-------------+\n | ``func_code`` | The code object representing | Writable |\n | | the compiled function body. | |\n +-------------------------+---------------------------------+-------------+\n | ``func_globals`` | A reference to the dictionary | Read-only |\n | | that holds the function\'s | |\n | | global variables --- the global | |\n | | namespace of the module in | |\n | | which the function was defined. | |\n +-------------------------+---------------------------------+-------------+\n | ``func_dict`` | The namespace supporting | Writable |\n | | arbitrary function attributes. | |\n +-------------------------+---------------------------------+-------------+\n | ``func_closure`` | ``None`` or a tuple of cells | Read-only |\n | | that contain bindings for the | |\n | | function\'s free variables. | |\n +-------------------------+---------------------------------+-------------+\n\n Most of the attributes labelled "Writable" check the type of the\n assigned value.\n\n Changed in version 2.4: ``func_name`` is now writable.\n\n Function objects also support getting and setting arbitrary\n attributes, which can be used, for example, to attach metadata\n to functions. Regular attribute dot-notation is used to get and\n set such attributes. *Note that the current implementation only\n supports function attributes on user-defined functions. Function\n attributes on built-in functions may be supported in the\n future.*\n\n Additional information about a function\'s definition can be\n retrieved from its code object; see the description of internal\n types below.\n\n User-defined methods\n A user-defined method object combines a class, a class instance\n (or ``None``) and any callable object (normally a user-defined\n function).\n\n Special read-only attributes: ``im_self`` is the class instance\n object, ``im_func`` is the function object; ``im_class`` is the\n class of ``im_self`` for bound methods or the class that asked\n for the method for unbound methods; ``__doc__`` is the method\'s\n documentation (same as ``im_func.__doc__``); ``__name__`` is the\n method name (same as ``im_func.__name__``); ``__module__`` is\n the name of the module the method was defined in, or ``None`` if\n unavailable.\n\n Changed in version 2.2: ``im_self`` used to refer to the class\n that defined the method.\n\n Changed in version 2.6: For 3.0 forward-compatibility,\n ``im_func`` is also available as ``__func__``, and ``im_self``\n as ``__self__``.\n\n Methods also support accessing (but not setting) the arbitrary\n function attributes on the underlying function object.\n\n User-defined method objects may be created when getting an\n attribute of a class (perhaps via an instance of that class), if\n that attribute is a user-defined function object, an unbound\n user-defined method object, or a class method object. When the\n attribute is a user-defined method object, a new method object\n is only created if the class from which it is being retrieved is\n the same as, or a derived class of, the class stored in the\n original method object; otherwise, the original method object is\n used as it is.\n\n When a user-defined method object is created by retrieving a\n user-defined function object from a class, its ``im_self``\n attribute is ``None`` and the method object is said to be\n unbound. When one is created by retrieving a user-defined\n function object from a class via one of its instances, its\n ``im_self`` attribute is the instance, and the method object is\n said to be bound. In either case, the new method\'s ``im_class``\n attribute is the class from which the retrieval takes place, and\n its ``im_func`` attribute is the original function object.\n\n When a user-defined method object is created by retrieving\n another method object from a class or instance, the behaviour is\n the same as for a function object, except that the ``im_func``\n attribute of the new instance is not the original method object\n but its ``im_func`` attribute.\n\n When a user-defined method object is created by retrieving a\n class method object from a class or instance, its ``im_self``\n attribute is the class itself (the same as the ``im_class``\n attribute), and its ``im_func`` attribute is the function object\n underlying the class method.\n\n When an unbound user-defined method object is called, the\n underlying function (``im_func``) is called, with the\n restriction that the first argument must be an instance of the\n proper class (``im_class``) or of a derived class thereof.\n\n When a bound user-defined method object is called, the\n underlying function (``im_func``) is called, inserting the class\n instance (``im_self``) in front of the argument list. For\n instance, when ``C`` is a class which contains a definition for\n a function ``f()``, and ``x`` is an instance of ``C``, calling\n ``x.f(1)`` is equivalent to calling ``C.f(x, 1)``.\n\n When a user-defined method object is derived from a class method\n object, the "class instance" stored in ``im_self`` will actually\n be the class itself, so that calling either ``x.f(1)`` or\n ``C.f(1)`` is equivalent to calling ``f(C,1)`` where ``f`` is\n the underlying function.\n\n Note that the transformation from function object to (unbound or\n bound) method object happens each time the attribute is\n retrieved from the class or instance. In some cases, a fruitful\n optimization is to assign the attribute to a local variable and\n call that local variable. Also notice that this transformation\n only happens for user-defined functions; other callable objects\n (and all non-callable objects) are retrieved without\n transformation. It is also important to note that user-defined\n functions which are attributes of a class instance are not\n converted to bound methods; this *only* happens when the\n function is an attribute of the class.\n\n Generator functions\n A function or method which uses the ``yield`` statement (see\n section *The yield statement*) is called a *generator function*.\n Such a function, when called, always returns an iterator object\n which can be used to execute the body of the function: calling\n the iterator\'s ``next()`` method will cause the function to\n execute until it provides a value using the ``yield`` statement.\n When the function executes a ``return`` statement or falls off\n the end, a ``StopIteration`` exception is raised and the\n iterator will have reached the end of the set of values to be\n returned.\n\n Built-in functions\n A built-in function object is a wrapper around a C function.\n Examples of built-in functions are ``len()`` and ``math.sin()``\n (``math`` is a standard built-in module). The number and type of\n the arguments are determined by the C function. Special read-\n only attributes: ``__doc__`` is the function\'s documentation\n string, or ``None`` if unavailable; ``__name__`` is the\n function\'s name; ``__self__`` is set to ``None`` (but see the\n next item); ``__module__`` is the name of the module the\n function was defined in or ``None`` if unavailable.\n\n Built-in methods\n This is really a different disguise of a built-in function, this\n time containing an object passed to the C function as an\n implicit extra argument. An example of a built-in method is\n ``alist.append()``, assuming *alist* is a list object. In this\n case, the special read-only attribute ``__self__`` is set to the\n object denoted by *list*.\n\n Class Types\n Class types, or "new-style classes," are callable. These\n objects normally act as factories for new instances of\n themselves, but variations are possible for class types that\n override ``__new__()``. The arguments of the call are passed to\n ``__new__()`` and, in the typical case, to ``__init__()`` to\n initialize the new instance.\n\n Classic Classes\n Class objects are described below. When a class object is\n called, a new class instance (also described below) is created\n and returned. This implies a call to the class\'s ``__init__()``\n method if it has one. Any arguments are passed on to the\n ``__init__()`` method. If there is no ``__init__()`` method,\n the class must be called without arguments.\n\n Class instances\n Class instances are described below. Class instances are\n callable only when the class has a ``__call__()`` method;\n ``x(arguments)`` is a shorthand for ``x.__call__(arguments)``.\n\nModules\n Modules are imported by the ``import`` statement (see section *The\n import statement*). A module object has a namespace implemented by\n a dictionary object (this is the dictionary referenced by the\n func_globals attribute of functions defined in the module).\n Attribute references are translated to lookups in this dictionary,\n e.g., ``m.x`` is equivalent to ``m.__dict__["x"]``. A module object\n does not contain the code object used to initialize the module\n (since it isn\'t needed once the initialization is done).\n\n Attribute assignment updates the module\'s namespace dictionary,\n e.g., ``m.x = 1`` is equivalent to ``m.__dict__["x"] = 1``.\n\n Special read-only attribute: ``__dict__`` is the module\'s namespace\n as a dictionary object.\n\n Predefined (writable) attributes: ``__name__`` is the module\'s\n name; ``__doc__`` is the module\'s documentation string, or ``None``\n if unavailable; ``__file__`` is the pathname of the file from which\n the module was loaded, if it was loaded from a file. The\n ``__file__`` attribute is not present for C modules that are\n statically linked into the interpreter; for extension modules\n loaded dynamically from a shared library, it is the pathname of the\n shared library file.\n\nClasses\n Both class types (new-style classes) and class objects (old-\n style/classic classes) are typically created by class definitions\n (see section *Class definitions*). A class has a namespace\n implemented by a dictionary object. Class attribute references are\n translated to lookups in this dictionary, e.g., ``C.x`` is\n translated to ``C.__dict__["x"]`` (although for new-style classes\n in particular there are a number of hooks which allow for other\n means of locating attributes). When the attribute name is not found\n there, the attribute search continues in the base classes. For\n old-style classes, the search is depth-first, left-to-right in the\n order of occurrence in the base class list. New-style classes use\n the more complex C3 method resolution order which behaves correctly\n even in the presence of \'diamond\' inheritance structures where\n there are multiple inheritance paths leading back to a common\n ancestor. Additional details on the C3 MRO used by new-style\n classes can be found in the documentation accompanying the 2.3\n release at http://www.python.org/download/releases/2.3/mro/.\n\n When a class attribute reference (for class ``C``, say) would yield\n a user-defined function object or an unbound user-defined method\n object whose associated class is either ``C`` or one of its base\n classes, it is transformed into an unbound user-defined method\n object whose ``im_class`` attribute is ``C``. When it would yield a\n class method object, it is transformed into a bound user-defined\n method object whose ``im_class`` and ``im_self`` attributes are\n both ``C``. When it would yield a static method object, it is\n transformed into the object wrapped by the static method object.\n See section *Implementing Descriptors* for another way in which\n attributes retrieved from a class may differ from those actually\n contained in its ``__dict__`` (note that only new-style classes\n support descriptors).\n\n Class attribute assignments update the class\'s dictionary, never\n the dictionary of a base class.\n\n A class object can be called (see above) to yield a class instance\n (see below).\n\n Special attributes: ``__name__`` is the class name; ``__module__``\n is the module name in which the class was defined; ``__dict__`` is\n the dictionary containing the class\'s namespace; ``__bases__`` is a\n tuple (possibly empty or a singleton) containing the base classes,\n in the order of their occurrence in the base class list;\n ``__doc__`` is the class\'s documentation string, or None if\n undefined.\n\nClass instances\n A class instance is created by calling a class object (see above).\n A class instance has a namespace implemented as a dictionary which\n is the first place in which attribute references are searched.\n When an attribute is not found there, and the instance\'s class has\n an attribute by that name, the search continues with the class\n attributes. If a class attribute is found that is a user-defined\n function object or an unbound user-defined method object whose\n associated class is the class (call it ``C``) of the instance for\n which the attribute reference was initiated or one of its bases, it\n is transformed into a bound user-defined method object whose\n ``im_class`` attribute is ``C`` and whose ``im_self`` attribute is\n the instance. Static method and class method objects are also\n transformed, as if they had been retrieved from class ``C``; see\n above under "Classes". See section *Implementing Descriptors* for\n another way in which attributes of a class retrieved via its\n instances may differ from the objects actually stored in the\n class\'s ``__dict__``. If no class attribute is found, and the\n object\'s class has a ``__getattr__()`` method, that is called to\n satisfy the lookup.\n\n Attribute assignments and deletions update the instance\'s\n dictionary, never a class\'s dictionary. If the class has a\n ``__setattr__()`` or ``__delattr__()`` method, this is called\n instead of updating the instance dictionary directly.\n\n Class instances can pretend to be numbers, sequences, or mappings\n if they have methods with certain special names. See section\n *Special method names*.\n\n Special attributes: ``__dict__`` is the attribute dictionary;\n ``__class__`` is the instance\'s class.\n\nFiles\n A file object represents an open file. File objects are created by\n the ``open()`` built-in function, and also by ``os.popen()``,\n ``os.fdopen()``, and the ``makefile()`` method of socket objects\n (and perhaps by other functions or methods provided by extension\n modules). The objects ``sys.stdin``, ``sys.stdout`` and\n ``sys.stderr`` are initialized to file objects corresponding to the\n interpreter\'s standard input, output and error streams. See *File\n Objects* for complete documentation of file objects.\n\nInternal types\n A few types used internally by the interpreter are exposed to the\n user. Their definitions may change with future versions of the\n interpreter, but they are mentioned here for completeness.\n\n Code objects\n Code objects represent *byte-compiled* executable Python code,\n or *bytecode*. The difference between a code object and a\n function object is that the function object contains an explicit\n reference to the function\'s globals (the module in which it was\n defined), while a code object contains no context; also the\n default argument values are stored in the function object, not\n in the code object (because they represent values calculated at\n run-time). Unlike function objects, code objects are immutable\n and contain no references (directly or indirectly) to mutable\n objects.\n\n Special read-only attributes: ``co_name`` gives the function\n name; ``co_argcount`` is the number of positional arguments\n (including arguments with default values); ``co_nlocals`` is the\n number of local variables used by the function (including\n arguments); ``co_varnames`` is a tuple containing the names of\n the local variables (starting with the argument names);\n ``co_cellvars`` is a tuple containing the names of local\n variables that are referenced by nested functions;\n ``co_freevars`` is a tuple containing the names of free\n variables; ``co_code`` is a string representing the sequence of\n bytecode instructions; ``co_consts`` is a tuple containing the\n literals used by the bytecode; ``co_names`` is a tuple\n containing the names used by the bytecode; ``co_filename`` is\n the filename from which the code was compiled;\n ``co_firstlineno`` is the first line number of the function;\n ``co_lnotab`` is a string encoding the mapping from bytecode\n offsets to line numbers (for details see the source code of the\n interpreter); ``co_stacksize`` is the required stack size\n (including local variables); ``co_flags`` is an integer encoding\n a number of flags for the interpreter.\n\n The following flag bits are defined for ``co_flags``: bit\n ``0x04`` is set if the function uses the ``*arguments`` syntax\n to accept an arbitrary number of positional arguments; bit\n ``0x08`` is set if the function uses the ``**keywords`` syntax\n to accept arbitrary keyword arguments; bit ``0x20`` is set if\n the function is a generator.\n\n Future feature declarations (``from __future__ import\n division``) also use bits in ``co_flags`` to indicate whether a\n code object was compiled with a particular feature enabled: bit\n ``0x2000`` is set if the function was compiled with future\n division enabled; bits ``0x10`` and ``0x1000`` were used in\n earlier versions of Python.\n\n Other bits in ``co_flags`` are reserved for internal use.\n\n If a code object represents a function, the first item in\n ``co_consts`` is the documentation string of the function, or\n ``None`` if undefined.\n\n Frame objects\n Frame objects represent execution frames. They may occur in\n traceback objects (see below).\n\n Special read-only attributes: ``f_back`` is to the previous\n stack frame (towards the caller), or ``None`` if this is the\n bottom stack frame; ``f_code`` is the code object being executed\n in this frame; ``f_locals`` is the dictionary used to look up\n local variables; ``f_globals`` is used for global variables;\n ``f_builtins`` is used for built-in (intrinsic) names;\n ``f_restricted`` is a flag indicating whether the function is\n executing in restricted execution mode; ``f_lasti`` gives the\n precise instruction (this is an index into the bytecode string\n of the code object).\n\n Special writable attributes: ``f_trace``, if not ``None``, is a\n function called at the start of each source code line (this is\n used by the debugger); ``f_exc_type``, ``f_exc_value``,\n ``f_exc_traceback`` represent the last exception raised in the\n parent frame provided another exception was ever raised in the\n current frame (in all other cases they are None); ``f_lineno``\n is the current line number of the frame --- writing to this from\n within a trace function jumps to the given line (only for the\n bottom-most frame). A debugger can implement a Jump command\n (aka Set Next Statement) by writing to f_lineno.\n\n Traceback objects\n Traceback objects represent a stack trace of an exception. A\n traceback object is created when an exception occurs. When the\n search for an exception handler unwinds the execution stack, at\n each unwound level a traceback object is inserted in front of\n the current traceback. When an exception handler is entered,\n the stack trace is made available to the program. (See section\n *The try statement*.) It is accessible as ``sys.exc_traceback``,\n and also as the third item of the tuple returned by\n ``sys.exc_info()``. The latter is the preferred interface,\n since it works correctly when the program is using multiple\n threads. When the program contains no suitable handler, the\n stack trace is written (nicely formatted) to the standard error\n stream; if the interpreter is interactive, it is also made\n available to the user as ``sys.last_traceback``.\n\n Special read-only attributes: ``tb_next`` is the next level in\n the stack trace (towards the frame where the exception\n occurred), or ``None`` if there is no next level; ``tb_frame``\n points to the execution frame of the current level;\n ``tb_lineno`` gives the line number where the exception\n occurred; ``tb_lasti`` indicates the precise instruction. The\n line number and last instruction in the traceback may differ\n from the line number of its frame object if the exception\n occurred in a ``try`` statement with no matching except clause\n or with a finally clause.\n\n Slice objects\n Slice objects are used to represent slices when *extended slice\n syntax* is used. This is a slice using two colons, or multiple\n slices or ellipses separated by commas, e.g., ``a[i:j:step]``,\n ``a[i:j, k:l]``, or ``a[..., i:j]``. They are also created by\n the built-in ``slice()`` function.\n\n Special read-only attributes: ``start`` is the lower bound;\n ``stop`` is the upper bound; ``step`` is the step value; each is\n ``None`` if omitted. These attributes can have any type.\n\n Slice objects support one method:\n\n slice.indices(self, length)\n\n This method takes a single integer argument *length* and\n computes information about the extended slice that the slice\n object would describe if applied to a sequence of *length*\n items. It returns a tuple of three integers; respectively\n these are the *start* and *stop* indices and the *step* or\n stride length of the slice. Missing or out-of-bounds indices\n are handled in a manner consistent with regular slices.\n\n New in version 2.3.\n\n Static method objects\n Static method objects provide a way of defeating the\n transformation of function objects to method objects described\n above. A static method object is a wrapper around any other\n object, usually a user-defined method object. When a static\n method object is retrieved from a class or a class instance, the\n object actually returned is the wrapped object, which is not\n subject to any further transformation. Static method objects are\n not themselves callable, although the objects they wrap usually\n are. Static method objects are created by the built-in\n ``staticmethod()`` constructor.\n\n Class method objects\n A class method object, like a static method object, is a wrapper\n around another object that alters the way in which that object\n is retrieved from classes and class instances. The behaviour of\n class method objects upon such retrieval is described above,\n under "User-defined methods". Class method objects are created\n by the built-in ``classmethod()`` constructor.\n', + 'types': u'\nThe standard type hierarchy\n***************************\n\nBelow is a list of the types that are built into Python. Extension\nmodules (written in C, Java, or other languages, depending on the\nimplementation) can define additional types. Future versions of\nPython may add types to the type hierarchy (e.g., rational numbers,\nefficiently stored arrays of integers, etc.).\n\nSome of the type descriptions below contain a paragraph listing\n\'special attributes.\' These are attributes that provide access to the\nimplementation and are not intended for general use. Their definition\nmay change in the future.\n\nNone\n This type has a single value. There is a single object with this\n value. This object is accessed through the built-in name ``None``.\n It is used to signify the absence of a value in many situations,\n e.g., it is returned from functions that don\'t explicitly return\n anything. Its truth value is false.\n\nNotImplemented\n This type has a single value. There is a single object with this\n value. This object is accessed through the built-in name\n ``NotImplemented``. Numeric methods and rich comparison methods may\n return this value if they do not implement the operation for the\n operands provided. (The interpreter will then try the reflected\n operation, or some other fallback, depending on the operator.) Its\n truth value is true.\n\nEllipsis\n This type has a single value. There is a single object with this\n value. This object is accessed through the built-in name\n ``Ellipsis``. It is used to indicate the presence of the ``...``\n syntax in a slice. Its truth value is true.\n\n``numbers.Number``\n These are created by numeric literals and returned as results by\n arithmetic operators and arithmetic built-in functions. Numeric\n objects are immutable; once created their value never changes.\n Python numbers are of course strongly related to mathematical\n numbers, but subject to the limitations of numerical representation\n in computers.\n\n Python distinguishes between integers, floating point numbers, and\n complex numbers:\n\n ``numbers.Integral``\n These represent elements from the mathematical set of integers\n (positive and negative).\n\n There are three types of integers:\n\n Plain integers\n These represent numbers in the range -2147483648 through\n 2147483647. (The range may be larger on machines with a\n larger natural word size, but not smaller.) When the result\n of an operation would fall outside this range, the result is\n normally returned as a long integer (in some cases, the\n exception ``OverflowError`` is raised instead). For the\n purpose of shift and mask operations, integers are assumed to\n have a binary, 2\'s complement notation using 32 or more bits,\n and hiding no bits from the user (i.e., all 4294967296\n different bit patterns correspond to different values).\n\n Long integers\n These represent numbers in an unlimited range, subject to\n available (virtual) memory only. For the purpose of shift\n and mask operations, a binary representation is assumed, and\n negative numbers are represented in a variant of 2\'s\n complement which gives the illusion of an infinite string of\n sign bits extending to the left.\n\n Booleans\n These represent the truth values False and True. The two\n objects representing the values False and True are the only\n Boolean objects. The Boolean type is a subtype of plain\n integers, and Boolean values behave like the values 0 and 1,\n respectively, in almost all contexts, the exception being\n that when converted to a string, the strings ``"False"`` or\n ``"True"`` are returned, respectively.\n\n The rules for integer representation are intended to give the\n most meaningful interpretation of shift and mask operations\n involving negative integers and the least surprises when\n switching between the plain and long integer domains. Any\n operation, if it yields a result in the plain integer domain,\n will yield the same result in the long integer domain or when\n using mixed operands. The switch between domains is transparent\n to the programmer.\n\n ``numbers.Real`` (``float``)\n These represent machine-level double precision floating point\n numbers. You are at the mercy of the underlying machine\n architecture (and C or Java implementation) for the accepted\n range and handling of overflow. Python does not support single-\n precision floating point numbers; the savings in processor and\n memory usage that are usually the reason for using these is\n dwarfed by the overhead of using objects in Python, so there is\n no reason to complicate the language with two kinds of floating\n point numbers.\n\n ``numbers.Complex``\n These represent complex numbers as a pair of machine-level\n double precision floating point numbers. The same caveats apply\n as for floating point numbers. The real and imaginary parts of a\n complex number ``z`` can be retrieved through the read-only\n attributes ``z.real`` and ``z.imag``.\n\nSequences\n These represent finite ordered sets indexed by non-negative\n numbers. The built-in function ``len()`` returns the number of\n items of a sequence. When the length of a sequence is *n*, the\n index set contains the numbers 0, 1, ..., *n*-1. Item *i* of\n sequence *a* is selected by ``a[i]``.\n\n Sequences also support slicing: ``a[i:j]`` selects all items with\n index *k* such that *i* ``<=`` *k* ``<`` *j*. When used as an\n expression, a slice is a sequence of the same type. This implies\n that the index set is renumbered so that it starts at 0.\n\n Some sequences also support "extended slicing" with a third "step"\n parameter: ``a[i:j:k]`` selects all items of *a* with index *x*\n where ``x = i + n*k``, *n* ``>=`` ``0`` and *i* ``<=`` *x* ``<``\n *j*.\n\n Sequences are distinguished according to their mutability:\n\n Immutable sequences\n An object of an immutable sequence type cannot change once it is\n created. (If the object contains references to other objects,\n these other objects may be mutable and may be changed; however,\n the collection of objects directly referenced by an immutable\n object cannot change.)\n\n The following types are immutable sequences:\n\n Strings\n The items of a string are characters. There is no separate\n character type; a character is represented by a string of one\n item. Characters represent (at least) 8-bit bytes. The\n built-in functions ``chr()`` and ``ord()`` convert between\n characters and nonnegative integers representing the byte\n values. Bytes with the values 0-127 usually represent the\n corresponding ASCII values, but the interpretation of values\n is up to the program. The string data type is also used to\n represent arrays of bytes, e.g., to hold data read from a\n file.\n\n (On systems whose native character set is not ASCII, strings\n may use EBCDIC in their internal representation, provided the\n functions ``chr()`` and ``ord()`` implement a mapping between\n ASCII and EBCDIC, and string comparison preserves the ASCII\n order. Or perhaps someone can propose a better rule?)\n\n Unicode\n The items of a Unicode object are Unicode code units. A\n Unicode code unit is represented by a Unicode object of one\n item and can hold either a 16-bit or 32-bit value\n representing a Unicode ordinal (the maximum value for the\n ordinal is given in ``sys.maxunicode``, and depends on how\n Python is configured at compile time). Surrogate pairs may\n be present in the Unicode object, and will be reported as two\n separate items. The built-in functions ``unichr()`` and\n ``ord()`` convert between code units and nonnegative integers\n representing the Unicode ordinals as defined in the Unicode\n Standard 3.0. Conversion from and to other encodings are\n possible through the Unicode method ``encode()`` and the\n built-in function ``unicode()``.\n\n Tuples\n The items of a tuple are arbitrary Python objects. Tuples of\n two or more items are formed by comma-separated lists of\n expressions. A tuple of one item (a \'singleton\') can be\n formed by affixing a comma to an expression (an expression by\n itself does not create a tuple, since parentheses must be\n usable for grouping of expressions). An empty tuple can be\n formed by an empty pair of parentheses.\n\n Mutable sequences\n Mutable sequences can be changed after they are created. The\n subscription and slicing notations can be used as the target of\n assignment and ``del`` (delete) statements.\n\n There are currently two intrinsic mutable sequence types:\n\n Lists\n The items of a list are arbitrary Python objects. Lists are\n formed by placing a comma-separated list of expressions in\n square brackets. (Note that there are no special cases needed\n to form lists of length 0 or 1.)\n\n Byte Arrays\n A bytearray object is a mutable array. They are created by\n the built-in ``bytearray()`` constructor. Aside from being\n mutable (and hence unhashable), byte arrays otherwise provide\n the same interface and functionality as immutable bytes\n objects.\n\n The extension module ``array`` provides an additional example of\n a mutable sequence type.\n\nSet types\n These represent unordered, finite sets of unique, immutable\n objects. As such, they cannot be indexed by any subscript. However,\n they can be iterated over, and the built-in function ``len()``\n returns the number of items in a set. Common uses for sets are fast\n membership testing, removing duplicates from a sequence, and\n computing mathematical operations such as intersection, union,\n difference, and symmetric difference.\n\n For set elements, the same immutability rules apply as for\n dictionary keys. Note that numeric types obey the normal rules for\n numeric comparison: if two numbers compare equal (e.g., ``1`` and\n ``1.0``), only one of them can be contained in a set.\n\n There are currently two intrinsic set types:\n\n Sets\n These represent a mutable set. They are created by the built-in\n ``set()`` constructor and can be modified afterwards by several\n methods, such as ``add()``.\n\n Frozen sets\n These represent an immutable set. They are created by the\n built-in ``frozenset()`` constructor. As a frozenset is\n immutable and *hashable*, it can be used again as an element of\n another set, or as a dictionary key.\n\nMappings\n These represent finite sets of objects indexed by arbitrary index\n sets. The subscript notation ``a[k]`` selects the item indexed by\n ``k`` from the mapping ``a``; this can be used in expressions and\n as the target of assignments or ``del`` statements. The built-in\n function ``len()`` returns the number of items in a mapping.\n\n There is currently a single intrinsic mapping type:\n\n Dictionaries\n These represent finite sets of objects indexed by nearly\n arbitrary values. The only types of values not acceptable as\n keys are values containing lists or dictionaries or other\n mutable types that are compared by value rather than by object\n identity, the reason being that the efficient implementation of\n dictionaries requires a key\'s hash value to remain constant.\n Numeric types used for keys obey the normal rules for numeric\n comparison: if two numbers compare equal (e.g., ``1`` and\n ``1.0``) then they can be used interchangeably to index the same\n dictionary entry.\n\n Dictionaries are mutable; they can be created by the ``{...}``\n notation (see section *Dictionary displays*).\n\n The extension modules ``dbm``, ``gdbm``, and ``bsddb`` provide\n additional examples of mapping types.\n\nCallable types\n These are the types to which the function call operation (see\n section *Calls*) can be applied:\n\n User-defined functions\n A user-defined function object is created by a function\n definition (see section *Function definitions*). It should be\n called with an argument list containing the same number of items\n as the function\'s formal parameter list.\n\n Special attributes:\n\n +-------------------------+---------------------------------+-------------+\n | Attribute | Meaning | |\n +=========================+=================================+=============+\n | ``func_doc`` | The function\'s documentation | Writable |\n | | string, or ``None`` if | |\n | | unavailable | |\n +-------------------------+---------------------------------+-------------+\n | ``__doc__`` | Another way of spelling | Writable |\n | | ``func_doc`` | |\n +-------------------------+---------------------------------+-------------+\n | ``func_name`` | The function\'s name | Writable |\n +-------------------------+---------------------------------+-------------+\n | ``__name__`` | Another way of spelling | Writable |\n | | ``func_name`` | |\n +-------------------------+---------------------------------+-------------+\n | ``__module__`` | The name of the module the | Writable |\n | | function was defined in, or | |\n | | ``None`` if unavailable. | |\n +-------------------------+---------------------------------+-------------+\n | ``func_defaults`` | A tuple containing default | Writable |\n | | argument values for those | |\n | | arguments that have defaults, | |\n | | or ``None`` if no arguments | |\n | | have a default value | |\n +-------------------------+---------------------------------+-------------+\n | ``func_code`` | The code object representing | Writable |\n | | the compiled function body. | |\n +-------------------------+---------------------------------+-------------+\n | ``func_globals`` | A reference to the dictionary | Read-only |\n | | that holds the function\'s | |\n | | global variables --- the global | |\n | | namespace of the module in | |\n | | which the function was defined. | |\n +-------------------------+---------------------------------+-------------+\n | ``func_dict`` | The namespace supporting | Writable |\n | | arbitrary function attributes. | |\n +-------------------------+---------------------------------+-------------+\n | ``func_closure`` | ``None`` or a tuple of cells | Read-only |\n | | that contain bindings for the | |\n | | function\'s free variables. | |\n +-------------------------+---------------------------------+-------------+\n\n Most of the attributes labelled "Writable" check the type of the\n assigned value.\n\n Changed in version 2.4: ``func_name`` is now writable.\n\n Function objects also support getting and setting arbitrary\n attributes, which can be used, for example, to attach metadata\n to functions. Regular attribute dot-notation is used to get and\n set such attributes. *Note that the current implementation only\n supports function attributes on user-defined functions. Function\n attributes on built-in functions may be supported in the\n future.*\n\n Additional information about a function\'s definition can be\n retrieved from its code object; see the description of internal\n types below.\n\n User-defined methods\n A user-defined method object combines a class, a class instance\n (or ``None``) and any callable object (normally a user-defined\n function).\n\n Special read-only attributes: ``im_self`` is the class instance\n object, ``im_func`` is the function object; ``im_class`` is the\n class of ``im_self`` for bound methods or the class that asked\n for the method for unbound methods; ``__doc__`` is the method\'s\n documentation (same as ``im_func.__doc__``); ``__name__`` is the\n method name (same as ``im_func.__name__``); ``__module__`` is\n the name of the module the method was defined in, or ``None`` if\n unavailable.\n\n Changed in version 2.2: ``im_self`` used to refer to the class\n that defined the method.\n\n Changed in version 2.6: For 3.0 forward-compatibility,\n ``im_func`` is also available as ``__func__``, and ``im_self``\n as ``__self__``.\n\n Methods also support accessing (but not setting) the arbitrary\n function attributes on the underlying function object.\n\n User-defined method objects may be created when getting an\n attribute of a class (perhaps via an instance of that class), if\n that attribute is a user-defined function object, an unbound\n user-defined method object, or a class method object. When the\n attribute is a user-defined method object, a new method object\n is only created if the class from which it is being retrieved is\n the same as, or a derived class of, the class stored in the\n original method object; otherwise, the original method object is\n used as it is.\n\n When a user-defined method object is created by retrieving a\n user-defined function object from a class, its ``im_self``\n attribute is ``None`` and the method object is said to be\n unbound. When one is created by retrieving a user-defined\n function object from a class via one of its instances, its\n ``im_self`` attribute is the instance, and the method object is\n said to be bound. In either case, the new method\'s ``im_class``\n attribute is the class from which the retrieval takes place, and\n its ``im_func`` attribute is the original function object.\n\n When a user-defined method object is created by retrieving\n another method object from a class or instance, the behaviour is\n the same as for a function object, except that the ``im_func``\n attribute of the new instance is not the original method object\n but its ``im_func`` attribute.\n\n When a user-defined method object is created by retrieving a\n class method object from a class or instance, its ``im_self``\n attribute is the class itself (the same as the ``im_class``\n attribute), and its ``im_func`` attribute is the function object\n underlying the class method.\n\n When an unbound user-defined method object is called, the\n underlying function (``im_func``) is called, with the\n restriction that the first argument must be an instance of the\n proper class (``im_class``) or of a derived class thereof.\n\n When a bound user-defined method object is called, the\n underlying function (``im_func``) is called, inserting the class\n instance (``im_self``) in front of the argument list. For\n instance, when ``C`` is a class which contains a definition for\n a function ``f()``, and ``x`` is an instance of ``C``, calling\n ``x.f(1)`` is equivalent to calling ``C.f(x, 1)``.\n\n When a user-defined method object is derived from a class method\n object, the "class instance" stored in ``im_self`` will actually\n be the class itself, so that calling either ``x.f(1)`` or\n ``C.f(1)`` is equivalent to calling ``f(C,1)`` where ``f`` is\n the underlying function.\n\n Note that the transformation from function object to (unbound or\n bound) method object happens each time the attribute is\n retrieved from the class or instance. In some cases, a fruitful\n optimization is to assign the attribute to a local variable and\n call that local variable. Also notice that this transformation\n only happens for user-defined functions; other callable objects\n (and all non-callable objects) are retrieved without\n transformation. It is also important to note that user-defined\n functions which are attributes of a class instance are not\n converted to bound methods; this *only* happens when the\n function is an attribute of the class.\n\n Generator functions\n A function or method which uses the ``yield`` statement (see\n section *The yield statement*) is called a *generator function*.\n Such a function, when called, always returns an iterator object\n which can be used to execute the body of the function: calling\n the iterator\'s ``next()`` method will cause the function to\n execute until it provides a value using the ``yield`` statement.\n When the function executes a ``return`` statement or falls off\n the end, a ``StopIteration`` exception is raised and the\n iterator will have reached the end of the set of values to be\n returned.\n\n Built-in functions\n A built-in function object is a wrapper around a C function.\n Examples of built-in functions are ``len()`` and ``math.sin()``\n (``math`` is a standard built-in module). The number and type of\n the arguments are determined by the C function. Special read-\n only attributes: ``__doc__`` is the function\'s documentation\n string, or ``None`` if unavailable; ``__name__`` is the\n function\'s name; ``__self__`` is set to ``None`` (but see the\n next item); ``__module__`` is the name of the module the\n function was defined in or ``None`` if unavailable.\n\n Built-in methods\n This is really a different disguise of a built-in function, this\n time containing an object passed to the C function as an\n implicit extra argument. An example of a built-in method is\n ``alist.append()``, assuming *alist* is a list object. In this\n case, the special read-only attribute ``__self__`` is set to the\n object denoted by *alist*.\n\n Class Types\n Class types, or "new-style classes," are callable. These\n objects normally act as factories for new instances of\n themselves, but variations are possible for class types that\n override ``__new__()``. The arguments of the call are passed to\n ``__new__()`` and, in the typical case, to ``__init__()`` to\n initialize the new instance.\n\n Classic Classes\n Class objects are described below. When a class object is\n called, a new class instance (also described below) is created\n and returned. This implies a call to the class\'s ``__init__()``\n method if it has one. Any arguments are passed on to the\n ``__init__()`` method. If there is no ``__init__()`` method,\n the class must be called without arguments.\n\n Class instances\n Class instances are described below. Class instances are\n callable only when the class has a ``__call__()`` method;\n ``x(arguments)`` is a shorthand for ``x.__call__(arguments)``.\n\nModules\n Modules are imported by the ``import`` statement (see section *The\n import statement*). A module object has a namespace implemented by\n a dictionary object (this is the dictionary referenced by the\n func_globals attribute of functions defined in the module).\n Attribute references are translated to lookups in this dictionary,\n e.g., ``m.x`` is equivalent to ``m.__dict__["x"]``. A module object\n does not contain the code object used to initialize the module\n (since it isn\'t needed once the initialization is done).\n\n Attribute assignment updates the module\'s namespace dictionary,\n e.g., ``m.x = 1`` is equivalent to ``m.__dict__["x"] = 1``.\n\n Special read-only attribute: ``__dict__`` is the module\'s namespace\n as a dictionary object.\n\n **CPython implementation detail:** Because of the way CPython\n clears module dictionaries, the module dictionary will be cleared\n when the module falls out of scope even if the dictionary still has\n live references. To avoid this, copy the dictionary or keep the\n module around while using its dictionary directly.\n\n Predefined (writable) attributes: ``__name__`` is the module\'s\n name; ``__doc__`` is the module\'s documentation string, or ``None``\n if unavailable; ``__file__`` is the pathname of the file from which\n the module was loaded, if it was loaded from a file. The\n ``__file__`` attribute is not present for C modules that are\n statically linked into the interpreter; for extension modules\n loaded dynamically from a shared library, it is the pathname of the\n shared library file.\n\nClasses\n Both class types (new-style classes) and class objects (old-\n style/classic classes) are typically created by class definitions\n (see section *Class definitions*). A class has a namespace\n implemented by a dictionary object. Class attribute references are\n translated to lookups in this dictionary, e.g., ``C.x`` is\n translated to ``C.__dict__["x"]`` (although for new-style classes\n in particular there are a number of hooks which allow for other\n means of locating attributes). When the attribute name is not found\n there, the attribute search continues in the base classes. For\n old-style classes, the search is depth-first, left-to-right in the\n order of occurrence in the base class list. New-style classes use\n the more complex C3 method resolution order which behaves correctly\n even in the presence of \'diamond\' inheritance structures where\n there are multiple inheritance paths leading back to a common\n ancestor. Additional details on the C3 MRO used by new-style\n classes can be found in the documentation accompanying the 2.3\n release at http://www.python.org/download/releases/2.3/mro/.\n\n When a class attribute reference (for class ``C``, say) would yield\n a user-defined function object or an unbound user-defined method\n object whose associated class is either ``C`` or one of its base\n classes, it is transformed into an unbound user-defined method\n object whose ``im_class`` attribute is ``C``. When it would yield a\n class method object, it is transformed into a bound user-defined\n method object whose ``im_class`` and ``im_self`` attributes are\n both ``C``. When it would yield a static method object, it is\n transformed into the object wrapped by the static method object.\n See section *Implementing Descriptors* for another way in which\n attributes retrieved from a class may differ from those actually\n contained in its ``__dict__`` (note that only new-style classes\n support descriptors).\n\n Class attribute assignments update the class\'s dictionary, never\n the dictionary of a base class.\n\n A class object can be called (see above) to yield a class instance\n (see below).\n\n Special attributes: ``__name__`` is the class name; ``__module__``\n is the module name in which the class was defined; ``__dict__`` is\n the dictionary containing the class\'s namespace; ``__bases__`` is a\n tuple (possibly empty or a singleton) containing the base classes,\n in the order of their occurrence in the base class list;\n ``__doc__`` is the class\'s documentation string, or None if\n undefined.\n\nClass instances\n A class instance is created by calling a class object (see above).\n A class instance has a namespace implemented as a dictionary which\n is the first place in which attribute references are searched.\n When an attribute is not found there, and the instance\'s class has\n an attribute by that name, the search continues with the class\n attributes. If a class attribute is found that is a user-defined\n function object or an unbound user-defined method object whose\n associated class is the class (call it ``C``) of the instance for\n which the attribute reference was initiated or one of its bases, it\n is transformed into a bound user-defined method object whose\n ``im_class`` attribute is ``C`` and whose ``im_self`` attribute is\n the instance. Static method and class method objects are also\n transformed, as if they had been retrieved from class ``C``; see\n above under "Classes". See section *Implementing Descriptors* for\n another way in which attributes of a class retrieved via its\n instances may differ from the objects actually stored in the\n class\'s ``__dict__``. If no class attribute is found, and the\n object\'s class has a ``__getattr__()`` method, that is called to\n satisfy the lookup.\n\n Attribute assignments and deletions update the instance\'s\n dictionary, never a class\'s dictionary. If the class has a\n ``__setattr__()`` or ``__delattr__()`` method, this is called\n instead of updating the instance dictionary directly.\n\n Class instances can pretend to be numbers, sequences, or mappings\n if they have methods with certain special names. See section\n *Special method names*.\n\n Special attributes: ``__dict__`` is the attribute dictionary;\n ``__class__`` is the instance\'s class.\n\nFiles\n A file object represents an open file. File objects are created by\n the ``open()`` built-in function, and also by ``os.popen()``,\n ``os.fdopen()``, and the ``makefile()`` method of socket objects\n (and perhaps by other functions or methods provided by extension\n modules). The objects ``sys.stdin``, ``sys.stdout`` and\n ``sys.stderr`` are initialized to file objects corresponding to the\n interpreter\'s standard input, output and error streams. See *File\n Objects* for complete documentation of file objects.\n\nInternal types\n A few types used internally by the interpreter are exposed to the\n user. Their definitions may change with future versions of the\n interpreter, but they are mentioned here for completeness.\n\n Code objects\n Code objects represent *byte-compiled* executable Python code,\n or *bytecode*. The difference between a code object and a\n function object is that the function object contains an explicit\n reference to the function\'s globals (the module in which it was\n defined), while a code object contains no context; also the\n default argument values are stored in the function object, not\n in the code object (because they represent values calculated at\n run-time). Unlike function objects, code objects are immutable\n and contain no references (directly or indirectly) to mutable\n objects.\n\n Special read-only attributes: ``co_name`` gives the function\n name; ``co_argcount`` is the number of positional arguments\n (including arguments with default values); ``co_nlocals`` is the\n number of local variables used by the function (including\n arguments); ``co_varnames`` is a tuple containing the names of\n the local variables (starting with the argument names);\n ``co_cellvars`` is a tuple containing the names of local\n variables that are referenced by nested functions;\n ``co_freevars`` is a tuple containing the names of free\n variables; ``co_code`` is a string representing the sequence of\n bytecode instructions; ``co_consts`` is a tuple containing the\n literals used by the bytecode; ``co_names`` is a tuple\n containing the names used by the bytecode; ``co_filename`` is\n the filename from which the code was compiled;\n ``co_firstlineno`` is the first line number of the function;\n ``co_lnotab`` is a string encoding the mapping from bytecode\n offsets to line numbers (for details see the source code of the\n interpreter); ``co_stacksize`` is the required stack size\n (including local variables); ``co_flags`` is an integer encoding\n a number of flags for the interpreter.\n\n The following flag bits are defined for ``co_flags``: bit\n ``0x04`` is set if the function uses the ``*arguments`` syntax\n to accept an arbitrary number of positional arguments; bit\n ``0x08`` is set if the function uses the ``**keywords`` syntax\n to accept arbitrary keyword arguments; bit ``0x20`` is set if\n the function is a generator.\n\n Future feature declarations (``from __future__ import\n division``) also use bits in ``co_flags`` to indicate whether a\n code object was compiled with a particular feature enabled: bit\n ``0x2000`` is set if the function was compiled with future\n division enabled; bits ``0x10`` and ``0x1000`` were used in\n earlier versions of Python.\n\n Other bits in ``co_flags`` are reserved for internal use.\n\n If a code object represents a function, the first item in\n ``co_consts`` is the documentation string of the function, or\n ``None`` if undefined.\n\n Frame objects\n Frame objects represent execution frames. They may occur in\n traceback objects (see below).\n\n Special read-only attributes: ``f_back`` is to the previous\n stack frame (towards the caller), or ``None`` if this is the\n bottom stack frame; ``f_code`` is the code object being executed\n in this frame; ``f_locals`` is the dictionary used to look up\n local variables; ``f_globals`` is used for global variables;\n ``f_builtins`` is used for built-in (intrinsic) names;\n ``f_restricted`` is a flag indicating whether the function is\n executing in restricted execution mode; ``f_lasti`` gives the\n precise instruction (this is an index into the bytecode string\n of the code object).\n\n Special writable attributes: ``f_trace``, if not ``None``, is a\n function called at the start of each source code line (this is\n used by the debugger); ``f_exc_type``, ``f_exc_value``,\n ``f_exc_traceback`` represent the last exception raised in the\n parent frame provided another exception was ever raised in the\n current frame (in all other cases they are None); ``f_lineno``\n is the current line number of the frame --- writing to this from\n within a trace function jumps to the given line (only for the\n bottom-most frame). A debugger can implement a Jump command\n (aka Set Next Statement) by writing to f_lineno.\n\n Traceback objects\n Traceback objects represent a stack trace of an exception. A\n traceback object is created when an exception occurs. When the\n search for an exception handler unwinds the execution stack, at\n each unwound level a traceback object is inserted in front of\n the current traceback. When an exception handler is entered,\n the stack trace is made available to the program. (See section\n *The try statement*.) It is accessible as ``sys.exc_traceback``,\n and also as the third item of the tuple returned by\n ``sys.exc_info()``. The latter is the preferred interface,\n since it works correctly when the program is using multiple\n threads. When the program contains no suitable handler, the\n stack trace is written (nicely formatted) to the standard error\n stream; if the interpreter is interactive, it is also made\n available to the user as ``sys.last_traceback``.\n\n Special read-only attributes: ``tb_next`` is the next level in\n the stack trace (towards the frame where the exception\n occurred), or ``None`` if there is no next level; ``tb_frame``\n points to the execution frame of the current level;\n ``tb_lineno`` gives the line number where the exception\n occurred; ``tb_lasti`` indicates the precise instruction. The\n line number and last instruction in the traceback may differ\n from the line number of its frame object if the exception\n occurred in a ``try`` statement with no matching except clause\n or with a finally clause.\n\n Slice objects\n Slice objects are used to represent slices when *extended slice\n syntax* is used. This is a slice using two colons, or multiple\n slices or ellipses separated by commas, e.g., ``a[i:j:step]``,\n ``a[i:j, k:l]``, or ``a[..., i:j]``. They are also created by\n the built-in ``slice()`` function.\n\n Special read-only attributes: ``start`` is the lower bound;\n ``stop`` is the upper bound; ``step`` is the step value; each is\n ``None`` if omitted. These attributes can have any type.\n\n Slice objects support one method:\n\n slice.indices(self, length)\n\n This method takes a single integer argument *length* and\n computes information about the extended slice that the slice\n object would describe if applied to a sequence of *length*\n items. It returns a tuple of three integers; respectively\n these are the *start* and *stop* indices and the *step* or\n stride length of the slice. Missing or out-of-bounds indices\n are handled in a manner consistent with regular slices.\n\n New in version 2.3.\n\n Static method objects\n Static method objects provide a way of defeating the\n transformation of function objects to method objects described\n above. A static method object is a wrapper around any other\n object, usually a user-defined method object. When a static\n method object is retrieved from a class or a class instance, the\n object actually returned is the wrapped object, which is not\n subject to any further transformation. Static method objects are\n not themselves callable, although the objects they wrap usually\n are. Static method objects are created by the built-in\n ``staticmethod()`` constructor.\n\n Class method objects\n A class method object, like a static method object, is a wrapper\n around another object that alters the way in which that object\n is retrieved from classes and class instances. The behaviour of\n class method objects upon such retrieval is described above,\n under "User-defined methods". Class method objects are created\n by the built-in ``classmethod()`` constructor.\n', 'typesfunctions': u'\nFunctions\n*********\n\nFunction objects are created by function definitions. The only\noperation on a function object is to call it: ``func(argument-list)``.\n\nThere are really two flavors of function objects: built-in functions\nand user-defined functions. Both support the same operation (to call\nthe function), but the implementation is different, hence the\ndifferent object types.\n\nSee *Function definitions* for more information.\n', - 'typesmapping': u'\nMapping Types --- ``dict``\n**************************\n\nA *mapping* object maps *hashable* values to arbitrary objects.\nMappings are mutable objects. There is currently only one standard\nmapping type, the *dictionary*. (For other containers see the built\nin ``list``, ``set``, and ``tuple`` classes, and the ``collections``\nmodule.)\n\nA dictionary\'s keys are *almost* arbitrary values. Values that are\nnot *hashable*, that is, values containing lists, dictionaries or\nother mutable types (that are compared by value rather than by object\nidentity) may not be used as keys. Numeric types used for keys obey\nthe normal rules for numeric comparison: if two numbers compare equal\n(such as ``1`` and ``1.0``) then they can be used interchangeably to\nindex the same dictionary entry. (Note however, that since computers\nstore floating-point numbers as approximations it is usually unwise to\nuse them as dictionary keys.)\n\nDictionaries can be created by placing a comma-separated list of\n``key: value`` pairs within braces, for example: ``{\'jack\': 4098,\n\'sjoerd\': 4127}`` or ``{4098: \'jack\', 4127: \'sjoerd\'}``, or by the\n``dict`` constructor.\n\nclass class dict([arg])\n\n Return a new dictionary initialized from an optional positional\n argument or from a set of keyword arguments. If no arguments are\n given, return a new empty dictionary. If the positional argument\n *arg* is a mapping object, return a dictionary mapping the same\n keys to the same values as does the mapping object. Otherwise the\n positional argument must be a sequence, a container that supports\n iteration, or an iterator object. The elements of the argument\n must each also be of one of those kinds, and each must in turn\n contain exactly two objects. The first is used as a key in the new\n dictionary, and the second as the key\'s value. If a given key is\n seen more than once, the last value associated with it is retained\n in the new dictionary.\n\n If keyword arguments are given, the keywords themselves with their\n associated values are added as items to the dictionary. If a key is\n specified both in the positional argument and as a keyword\n argument, the value associated with the keyword is retained in the\n dictionary. For example, these all return a dictionary equal to\n ``{"one": 2, "two": 3}``:\n\n * ``dict(one=2, two=3)``\n\n * ``dict({\'one\': 2, \'two\': 3})``\n\n * ``dict(zip((\'one\', \'two\'), (2, 3)))``\n\n * ``dict([[\'two\', 3], [\'one\', 2]])``\n\n The first example only works for keys that are valid Python\n identifiers; the others work with any valid keys.\n\n New in version 2.2.\n\n Changed in version 2.3: Support for building a dictionary from\n keyword arguments added.\n\n These are the operations that dictionaries support (and therefore,\n custom mapping types should support too):\n\n len(d)\n\n Return the number of items in the dictionary *d*.\n\n d[key]\n\n Return the item of *d* with key *key*. Raises a ``KeyError`` if\n *key* is not in the map.\n\n New in version 2.5: If a subclass of dict defines a method\n ``__missing__()``, if the key *key* is not present, the\n ``d[key]`` operation calls that method with the key *key* as\n argument. The ``d[key]`` operation then returns or raises\n whatever is returned or raised by the ``__missing__(key)`` call\n if the key is not present. No other operations or methods invoke\n ``__missing__()``. If ``__missing__()`` is not defined,\n ``KeyError`` is raised. ``__missing__()`` must be a method; it\n cannot be an instance variable. For an example, see\n ``collections.defaultdict``.\n\n d[key] = value\n\n Set ``d[key]`` to *value*.\n\n del d[key]\n\n Remove ``d[key]`` from *d*. Raises a ``KeyError`` if *key* is\n not in the map.\n\n key in d\n\n Return ``True`` if *d* has a key *key*, else ``False``.\n\n New in version 2.2.\n\n key not in d\n\n Equivalent to ``not key in d``.\n\n New in version 2.2.\n\n iter(d)\n\n Return an iterator over the keys of the dictionary. This is a\n shortcut for ``iterkeys()``.\n\n clear()\n\n Remove all items from the dictionary.\n\n copy()\n\n Return a shallow copy of the dictionary.\n\n fromkeys(seq[, value])\n\n Create a new dictionary with keys from *seq* and values set to\n *value*.\n\n ``fromkeys()`` is a class method that returns a new dictionary.\n *value* defaults to ``None``.\n\n New in version 2.3.\n\n get(key[, default])\n\n Return the value for *key* if *key* is in the dictionary, else\n *default*. If *default* is not given, it defaults to ``None``,\n so that this method never raises a ``KeyError``.\n\n has_key(key)\n\n Test for the presence of *key* in the dictionary. ``has_key()``\n is deprecated in favor of ``key in d``.\n\n items()\n\n Return a copy of the dictionary\'s list of ``(key, value)``\n pairs.\n\n **CPython implementation detail:** Keys and values are listed in\n an arbitrary order which is non-random, varies across Python\n implementations, and depends on the dictionary\'s history of\n insertions and deletions.\n\n If ``items()``, ``keys()``, ``values()``, ``iteritems()``,\n ``iterkeys()``, and ``itervalues()`` are called with no\n intervening modifications to the dictionary, the lists will\n directly correspond. This allows the creation of ``(value,\n key)`` pairs using ``zip()``: ``pairs = zip(d.values(),\n d.keys())``. The same relationship holds for the ``iterkeys()``\n and ``itervalues()`` methods: ``pairs = zip(d.itervalues(),\n d.iterkeys())`` provides the same value for ``pairs``. Another\n way to create the same list is ``pairs = [(v, k) for (k, v) in\n d.iteritems()]``.\n\n iteritems()\n\n Return an iterator over the dictionary\'s ``(key, value)`` pairs.\n See the note for ``dict.items()``.\n\n Using ``iteritems()`` while adding or deleting entries in the\n dictionary may raise a ``RuntimeError`` or fail to iterate over\n all entries.\n\n New in version 2.2.\n\n iterkeys()\n\n Return an iterator over the dictionary\'s keys. See the note for\n ``dict.items()``.\n\n Using ``iterkeys()`` while adding or deleting entries in the\n dictionary may raise a ``RuntimeError`` or fail to iterate over\n all entries.\n\n New in version 2.2.\n\n itervalues()\n\n Return an iterator over the dictionary\'s values. See the note\n for ``dict.items()``.\n\n Using ``itervalues()`` while adding or deleting entries in the\n dictionary may raise a ``RuntimeError`` or fail to iterate over\n all entries.\n\n New in version 2.2.\n\n keys()\n\n Return a copy of the dictionary\'s list of keys. See the note\n for ``dict.items()``.\n\n pop(key[, default])\n\n If *key* is in the dictionary, remove it and return its value,\n else return *default*. If *default* is not given and *key* is\n not in the dictionary, a ``KeyError`` is raised.\n\n New in version 2.3.\n\n popitem()\n\n Remove and return an arbitrary ``(key, value)`` pair from the\n dictionary.\n\n ``popitem()`` is useful to destructively iterate over a\n dictionary, as often used in set algorithms. If the dictionary\n is empty, calling ``popitem()`` raises a ``KeyError``.\n\n setdefault(key[, default])\n\n If *key* is in the dictionary, return its value. If not, insert\n *key* with a value of *default* and return *default*. *default*\n defaults to ``None``.\n\n update([other])\n\n Update the dictionary with the key/value pairs from *other*,\n overwriting existing keys. Return ``None``.\n\n ``update()`` accepts either another dictionary object or an\n iterable of key/value pairs (as a tuple or other iterable of\n length two). If keyword arguments are specified, the dictionary\n is then updated with those key/value pairs: ``d.update(red=1,\n blue=2)``.\n\n Changed in version 2.4: Allowed the argument to be an iterable\n of key/value pairs and allowed keyword arguments.\n\n values()\n\n Return a copy of the dictionary\'s list of values. See the note\n for ``dict.items()``.\n\n viewitems()\n\n Return a new view of the dictionary\'s items (``(key, value)``\n pairs). See below for documentation of view objects.\n\n New in version 2.7.\n\n viewkeys()\n\n Return a new view of the dictionary\'s keys. See below for\n documentation of view objects.\n\n New in version 2.7.\n\n viewvalues()\n\n Return a new view of the dictionary\'s values. See below for\n documentation of view objects.\n\n New in version 2.7.\n\n\nDictionary view objects\n=======================\n\nThe objects returned by ``dict.viewkeys()``, ``dict.viewvalues()`` and\n``dict.viewitems()`` are *view objects*. They provide a dynamic view\non the dictionary\'s entries, which means that when the dictionary\nchanges, the view reflects these changes.\n\nDictionary views can be iterated over to yield their respective data,\nand support membership tests:\n\nlen(dictview)\n\n Return the number of entries in the dictionary.\n\niter(dictview)\n\n Return an iterator over the keys, values or items (represented as\n tuples of ``(key, value)``) in the dictionary.\n\n Keys and values are iterated over in an arbitrary order which is\n non-random, varies across Python implementations, and depends on\n the dictionary\'s history of insertions and deletions. If keys,\n values and items views are iterated over with no intervening\n modifications to the dictionary, the order of items will directly\n correspond. This allows the creation of ``(value, key)`` pairs\n using ``zip()``: ``pairs = zip(d.values(), d.keys())``. Another\n way to create the same list is ``pairs = [(v, k) for (k, v) in\n d.items()]``.\n\n Iterating views while adding or deleting entries in the dictionary\n may raise a ``RuntimeError`` or fail to iterate over all entries.\n\nx in dictview\n\n Return ``True`` if *x* is in the underlying dictionary\'s keys,\n values or items (in the latter case, *x* should be a ``(key,\n value)`` tuple).\n\nKeys views are set-like since their entries are unique and hashable.\nIf all values are hashable, so that (key, value) pairs are unique and\nhashable, then the items view is also set-like. (Values views are not\ntreated as set-like since the entries are generally not unique.) Then\nthese set operations are available ("other" refers either to another\nview or a set):\n\ndictview & other\n\n Return the intersection of the dictview and the other object as a\n new set.\n\ndictview | other\n\n Return the union of the dictview and the other object as a new set.\n\ndictview - other\n\n Return the difference between the dictview and the other object\n (all elements in *dictview* that aren\'t in *other*) as a new set.\n\ndictview ^ other\n\n Return the symmetric difference (all elements either in *dictview*\n or *other*, but not in both) of the dictview and the other object\n as a new set.\n\nAn example of dictionary view usage:\n\n >>> dishes = {\'eggs\': 2, \'sausage\': 1, \'bacon\': 1, \'spam\': 500}\n >>> keys = dishes.viewkeys()\n >>> values = dishes.viewvalues()\n\n >>> # iteration\n >>> n = 0\n >>> for val in values:\n ... n += val\n >>> print(n)\n 504\n\n >>> # keys and values are iterated over in the same order\n >>> list(keys)\n [\'eggs\', \'bacon\', \'sausage\', \'spam\']\n >>> list(values)\n [2, 1, 1, 500]\n\n >>> # view objects are dynamic and reflect dict changes\n >>> del dishes[\'eggs\']\n >>> del dishes[\'sausage\']\n >>> list(keys)\n [\'spam\', \'bacon\']\n\n >>> # set operations\n >>> keys & {\'eggs\', \'bacon\', \'salad\'}\n {\'bacon\'}\n', + 'typesmapping': u'\nMapping Types --- ``dict``\n**************************\n\nA *mapping* object maps *hashable* values to arbitrary objects.\nMappings are mutable objects. There is currently only one standard\nmapping type, the *dictionary*. (For other containers see the built\nin ``list``, ``set``, and ``tuple`` classes, and the ``collections``\nmodule.)\n\nA dictionary\'s keys are *almost* arbitrary values. Values that are\nnot *hashable*, that is, values containing lists, dictionaries or\nother mutable types (that are compared by value rather than by object\nidentity) may not be used as keys. Numeric types used for keys obey\nthe normal rules for numeric comparison: if two numbers compare equal\n(such as ``1`` and ``1.0``) then they can be used interchangeably to\nindex the same dictionary entry. (Note however, that since computers\nstore floating-point numbers as approximations it is usually unwise to\nuse them as dictionary keys.)\n\nDictionaries can be created by placing a comma-separated list of\n``key: value`` pairs within braces, for example: ``{\'jack\': 4098,\n\'sjoerd\': 4127}`` or ``{4098: \'jack\', 4127: \'sjoerd\'}``, or by the\n``dict`` constructor.\n\nclass class dict([arg])\n\n Return a new dictionary initialized from an optional positional\n argument or from a set of keyword arguments. If no arguments are\n given, return a new empty dictionary. If the positional argument\n *arg* is a mapping object, return a dictionary mapping the same\n keys to the same values as does the mapping object. Otherwise the\n positional argument must be a sequence, a container that supports\n iteration, or an iterator object. The elements of the argument\n must each also be of one of those kinds, and each must in turn\n contain exactly two objects. The first is used as a key in the new\n dictionary, and the second as the key\'s value. If a given key is\n seen more than once, the last value associated with it is retained\n in the new dictionary.\n\n If keyword arguments are given, the keywords themselves with their\n associated values are added as items to the dictionary. If a key is\n specified both in the positional argument and as a keyword\n argument, the value associated with the keyword is retained in the\n dictionary. For example, these all return a dictionary equal to\n ``{"one": 1, "two": 2}``:\n\n * ``dict(one=1, two=2)``\n\n * ``dict({\'one\': 1, \'two\': 2})``\n\n * ``dict(zip((\'one\', \'two\'), (1, 2)))``\n\n * ``dict([[\'two\', 2], [\'one\', 1]])``\n\n The first example only works for keys that are valid Python\n identifiers; the others work with any valid keys.\n\n New in version 2.2.\n\n Changed in version 2.3: Support for building a dictionary from\n keyword arguments added.\n\n These are the operations that dictionaries support (and therefore,\n custom mapping types should support too):\n\n len(d)\n\n Return the number of items in the dictionary *d*.\n\n d[key]\n\n Return the item of *d* with key *key*. Raises a ``KeyError`` if\n *key* is not in the map.\n\n New in version 2.5: If a subclass of dict defines a method\n ``__missing__()``, if the key *key* is not present, the\n ``d[key]`` operation calls that method with the key *key* as\n argument. The ``d[key]`` operation then returns or raises\n whatever is returned or raised by the ``__missing__(key)`` call\n if the key is not present. No other operations or methods invoke\n ``__missing__()``. If ``__missing__()`` is not defined,\n ``KeyError`` is raised. ``__missing__()`` must be a method; it\n cannot be an instance variable. For an example, see\n ``collections.defaultdict``.\n\n d[key] = value\n\n Set ``d[key]`` to *value*.\n\n del d[key]\n\n Remove ``d[key]`` from *d*. Raises a ``KeyError`` if *key* is\n not in the map.\n\n key in d\n\n Return ``True`` if *d* has a key *key*, else ``False``.\n\n New in version 2.2.\n\n key not in d\n\n Equivalent to ``not key in d``.\n\n New in version 2.2.\n\n iter(d)\n\n Return an iterator over the keys of the dictionary. This is a\n shortcut for ``iterkeys()``.\n\n clear()\n\n Remove all items from the dictionary.\n\n copy()\n\n Return a shallow copy of the dictionary.\n\n fromkeys(seq[, value])\n\n Create a new dictionary with keys from *seq* and values set to\n *value*.\n\n ``fromkeys()`` is a class method that returns a new dictionary.\n *value* defaults to ``None``.\n\n New in version 2.3.\n\n get(key[, default])\n\n Return the value for *key* if *key* is in the dictionary, else\n *default*. If *default* is not given, it defaults to ``None``,\n so that this method never raises a ``KeyError``.\n\n has_key(key)\n\n Test for the presence of *key* in the dictionary. ``has_key()``\n is deprecated in favor of ``key in d``.\n\n items()\n\n Return a copy of the dictionary\'s list of ``(key, value)``\n pairs.\n\n **CPython implementation detail:** Keys and values are listed in\n an arbitrary order which is non-random, varies across Python\n implementations, and depends on the dictionary\'s history of\n insertions and deletions.\n\n If ``items()``, ``keys()``, ``values()``, ``iteritems()``,\n ``iterkeys()``, and ``itervalues()`` are called with no\n intervening modifications to the dictionary, the lists will\n directly correspond. This allows the creation of ``(value,\n key)`` pairs using ``zip()``: ``pairs = zip(d.values(),\n d.keys())``. The same relationship holds for the ``iterkeys()``\n and ``itervalues()`` methods: ``pairs = zip(d.itervalues(),\n d.iterkeys())`` provides the same value for ``pairs``. Another\n way to create the same list is ``pairs = [(v, k) for (k, v) in\n d.iteritems()]``.\n\n iteritems()\n\n Return an iterator over the dictionary\'s ``(key, value)`` pairs.\n See the note for ``dict.items()``.\n\n Using ``iteritems()`` while adding or deleting entries in the\n dictionary may raise a ``RuntimeError`` or fail to iterate over\n all entries.\n\n New in version 2.2.\n\n iterkeys()\n\n Return an iterator over the dictionary\'s keys. See the note for\n ``dict.items()``.\n\n Using ``iterkeys()`` while adding or deleting entries in the\n dictionary may raise a ``RuntimeError`` or fail to iterate over\n all entries.\n\n New in version 2.2.\n\n itervalues()\n\n Return an iterator over the dictionary\'s values. See the note\n for ``dict.items()``.\n\n Using ``itervalues()`` while adding or deleting entries in the\n dictionary may raise a ``RuntimeError`` or fail to iterate over\n all entries.\n\n New in version 2.2.\n\n keys()\n\n Return a copy of the dictionary\'s list of keys. See the note\n for ``dict.items()``.\n\n pop(key[, default])\n\n If *key* is in the dictionary, remove it and return its value,\n else return *default*. If *default* is not given and *key* is\n not in the dictionary, a ``KeyError`` is raised.\n\n New in version 2.3.\n\n popitem()\n\n Remove and return an arbitrary ``(key, value)`` pair from the\n dictionary.\n\n ``popitem()`` is useful to destructively iterate over a\n dictionary, as often used in set algorithms. If the dictionary\n is empty, calling ``popitem()`` raises a ``KeyError``.\n\n setdefault(key[, default])\n\n If *key* is in the dictionary, return its value. If not, insert\n *key* with a value of *default* and return *default*. *default*\n defaults to ``None``.\n\n update([other])\n\n Update the dictionary with the key/value pairs from *other*,\n overwriting existing keys. Return ``None``.\n\n ``update()`` accepts either another dictionary object or an\n iterable of key/value pairs (as tuples or other iterables of\n length two). If keyword arguments are specified, the dictionary\n is then updated with those key/value pairs: ``d.update(red=1,\n blue=2)``.\n\n Changed in version 2.4: Allowed the argument to be an iterable\n of key/value pairs and allowed keyword arguments.\n\n values()\n\n Return a copy of the dictionary\'s list of values. See the note\n for ``dict.items()``.\n\n viewitems()\n\n Return a new view of the dictionary\'s items (``(key, value)``\n pairs). See below for documentation of view objects.\n\n New in version 2.7.\n\n viewkeys()\n\n Return a new view of the dictionary\'s keys. See below for\n documentation of view objects.\n\n New in version 2.7.\n\n viewvalues()\n\n Return a new view of the dictionary\'s values. See below for\n documentation of view objects.\n\n New in version 2.7.\n\n\nDictionary view objects\n=======================\n\nThe objects returned by ``dict.viewkeys()``, ``dict.viewvalues()`` and\n``dict.viewitems()`` are *view objects*. They provide a dynamic view\non the dictionary\'s entries, which means that when the dictionary\nchanges, the view reflects these changes.\n\nDictionary views can be iterated over to yield their respective data,\nand support membership tests:\n\nlen(dictview)\n\n Return the number of entries in the dictionary.\n\niter(dictview)\n\n Return an iterator over the keys, values or items (represented as\n tuples of ``(key, value)``) in the dictionary.\n\n Keys and values are iterated over in an arbitrary order which is\n non-random, varies across Python implementations, and depends on\n the dictionary\'s history of insertions and deletions. If keys,\n values and items views are iterated over with no intervening\n modifications to the dictionary, the order of items will directly\n correspond. This allows the creation of ``(value, key)`` pairs\n using ``zip()``: ``pairs = zip(d.values(), d.keys())``. Another\n way to create the same list is ``pairs = [(v, k) for (k, v) in\n d.items()]``.\n\n Iterating views while adding or deleting entries in the dictionary\n may raise a ``RuntimeError`` or fail to iterate over all entries.\n\nx in dictview\n\n Return ``True`` if *x* is in the underlying dictionary\'s keys,\n values or items (in the latter case, *x* should be a ``(key,\n value)`` tuple).\n\nKeys views are set-like since their entries are unique and hashable.\nIf all values are hashable, so that (key, value) pairs are unique and\nhashable, then the items view is also set-like. (Values views are not\ntreated as set-like since the entries are generally not unique.) Then\nthese set operations are available ("other" refers either to another\nview or a set):\n\ndictview & other\n\n Return the intersection of the dictview and the other object as a\n new set.\n\ndictview | other\n\n Return the union of the dictview and the other object as a new set.\n\ndictview - other\n\n Return the difference between the dictview and the other object\n (all elements in *dictview* that aren\'t in *other*) as a new set.\n\ndictview ^ other\n\n Return the symmetric difference (all elements either in *dictview*\n or *other*, but not in both) of the dictview and the other object\n as a new set.\n\nAn example of dictionary view usage:\n\n >>> dishes = {\'eggs\': 2, \'sausage\': 1, \'bacon\': 1, \'spam\': 500}\n >>> keys = dishes.viewkeys()\n >>> values = dishes.viewvalues()\n\n >>> # iteration\n >>> n = 0\n >>> for val in values:\n ... n += val\n >>> print(n)\n 504\n\n >>> # keys and values are iterated over in the same order\n >>> list(keys)\n [\'eggs\', \'bacon\', \'sausage\', \'spam\']\n >>> list(values)\n [2, 1, 1, 500]\n\n >>> # view objects are dynamic and reflect dict changes\n >>> del dishes[\'eggs\']\n >>> del dishes[\'sausage\']\n >>> list(keys)\n [\'spam\', \'bacon\']\n\n >>> # set operations\n >>> keys & {\'eggs\', \'bacon\', \'salad\'}\n {\'bacon\'}\n', 'typesmethods': u"\nMethods\n*******\n\nMethods are functions that are called using the attribute notation.\nThere are two flavors: built-in methods (such as ``append()`` on\nlists) and class instance methods. Built-in methods are described\nwith the types that support them.\n\nThe implementation adds two special read-only attributes to class\ninstance methods: ``m.im_self`` is the object on which the method\noperates, and ``m.im_func`` is the function implementing the method.\nCalling ``m(arg-1, arg-2, ..., arg-n)`` is completely equivalent to\ncalling ``m.im_func(m.im_self, arg-1, arg-2, ..., arg-n)``.\n\nClass instance methods are either *bound* or *unbound*, referring to\nwhether the method was accessed through an instance or a class,\nrespectively. When a method is unbound, its ``im_self`` attribute\nwill be ``None`` and if called, an explicit ``self`` object must be\npassed as the first argument. In this case, ``self`` must be an\ninstance of the unbound method's class (or a subclass of that class),\notherwise a ``TypeError`` is raised.\n\nLike function objects, methods objects support getting arbitrary\nattributes. However, since method attributes are actually stored on\nthe underlying function object (``meth.im_func``), setting method\nattributes on either bound or unbound methods is disallowed.\nAttempting to set a method attribute results in a ``TypeError`` being\nraised. In order to set a method attribute, you need to explicitly\nset it on the underlying function object:\n\n class C:\n def method(self):\n pass\n\n c = C()\n c.method.im_func.whoami = 'my name is c'\n\nSee *The standard type hierarchy* for more information.\n", 'typesmodules': u"\nModules\n*******\n\nThe only special operation on a module is attribute access:\n``m.name``, where *m* is a module and *name* accesses a name defined\nin *m*'s symbol table. Module attributes can be assigned to. (Note\nthat the ``import`` statement is not, strictly speaking, an operation\non a module object; ``import foo`` does not require a module object\nnamed *foo* to exist, rather it requires an (external) *definition*\nfor a module named *foo* somewhere.)\n\nA special member of every module is ``__dict__``. This is the\ndictionary containing the module's symbol table. Modifying this\ndictionary will actually change the module's symbol table, but direct\nassignment to the ``__dict__`` attribute is not possible (you can\nwrite ``m.__dict__['a'] = 1``, which defines ``m.a`` to be ``1``, but\nyou can't write ``m.__dict__ = {}``). Modifying ``__dict__`` directly\nis not recommended.\n\nModules built into the interpreter are written like this: ``<module\n'sys' (built-in)>``. If loaded from a file, they are written as\n``<module 'os' from '/usr/local/lib/pythonX.Y/os.pyc'>``.\n", - 'typesseq': u'\nSequence Types --- ``str``, ``unicode``, ``list``, ``tuple``, ``buffer``, ``xrange``\n************************************************************************************\n\nThere are six sequence types: strings, Unicode strings, lists, tuples,\nbuffers, and xrange objects.\n\nFor other containers see the built in ``dict`` and ``set`` classes,\nand the ``collections`` module.\n\nString literals are written in single or double quotes: ``\'xyzzy\'``,\n``"frobozz"``. See *String literals* for more about string literals.\nUnicode strings are much like strings, but are specified in the syntax\nusing a preceding ``\'u\'`` character: ``u\'abc\'``, ``u"def"``. In\naddition to the functionality described here, there are also string-\nspecific methods described in the *String Methods* section. Lists are\nconstructed with square brackets, separating items with commas: ``[a,\nb, c]``. Tuples are constructed by the comma operator (not within\nsquare brackets), with or without enclosing parentheses, but an empty\ntuple must have the enclosing parentheses, such as ``a, b, c`` or\n``()``. A single item tuple must have a trailing comma, such as\n``(d,)``.\n\nBuffer objects are not directly supported by Python syntax, but can be\ncreated by calling the built-in function ``buffer()``. They don\'t\nsupport concatenation or repetition.\n\nObjects of type xrange are similar to buffers in that there is no\nspecific syntax to create them, but they are created using the\n``xrange()`` function. They don\'t support slicing, concatenation or\nrepetition, and using ``in``, ``not in``, ``min()`` or ``max()`` on\nthem is inefficient.\n\nMost sequence types support the following operations. The ``in`` and\n``not in`` operations have the same priorities as the comparison\noperations. The ``+`` and ``*`` operations have the same priority as\nthe corresponding numeric operations. [3] Additional methods are\nprovided for *Mutable Sequence Types*.\n\nThis table lists the sequence operations sorted in ascending priority\n(operations in the same box have the same priority). In the table,\n*s* and *t* are sequences of the same type; *n*, *i* and *j* are\nintegers:\n\n+--------------------+----------------------------------+------------+\n| Operation | Result | Notes |\n+====================+==================================+============+\n| ``x in s`` | ``True`` if an item of *s* is | (1) |\n| | equal to *x*, else ``False`` | |\n+--------------------+----------------------------------+------------+\n| ``x not in s`` | ``False`` if an item of *s* is | (1) |\n| | equal to *x*, else ``True`` | |\n+--------------------+----------------------------------+------------+\n| ``s + t`` | the concatenation of *s* and *t* | (6) |\n+--------------------+----------------------------------+------------+\n| ``s * n, n * s`` | *n* shallow copies of *s* | (2) |\n| | concatenated | |\n+--------------------+----------------------------------+------------+\n| ``s[i]`` | *i*\'th item of *s*, origin 0 | (3) |\n+--------------------+----------------------------------+------------+\n| ``s[i:j]`` | slice of *s* from *i* to *j* | (3)(4) |\n+--------------------+----------------------------------+------------+\n| ``s[i:j:k]`` | slice of *s* from *i* to *j* | (3)(5) |\n| | with step *k* | |\n+--------------------+----------------------------------+------------+\n| ``len(s)`` | length of *s* | |\n+--------------------+----------------------------------+------------+\n| ``min(s)`` | smallest item of *s* | |\n+--------------------+----------------------------------+------------+\n| ``max(s)`` | largest item of *s* | |\n+--------------------+----------------------------------+------------+\n\nSequence types also support comparisons. In particular, tuples and\nlists are compared lexicographically by comparing corresponding\nelements. This means that to compare equal, every element must compare\nequal and the two sequences must be of the same type and have the same\nlength. (For full details see *Comparisons* in the language\nreference.)\n\nNotes:\n\n1. When *s* is a string or Unicode string object the ``in`` and ``not\n in`` operations act like a substring test. In Python versions\n before 2.3, *x* had to be a string of length 1. In Python 2.3 and\n beyond, *x* may be a string of any length.\n\n2. Values of *n* less than ``0`` are treated as ``0`` (which yields an\n empty sequence of the same type as *s*). Note also that the copies\n are shallow; nested structures are not copied. This often haunts\n new Python programmers; consider:\n\n >>> lists = [[]] * 3\n >>> lists\n [[], [], []]\n >>> lists[0].append(3)\n >>> lists\n [[3], [3], [3]]\n\n What has happened is that ``[[]]`` is a one-element list containing\n an empty list, so all three elements of ``[[]] * 3`` are (pointers\n to) this single empty list. Modifying any of the elements of\n ``lists`` modifies this single list. You can create a list of\n different lists this way:\n\n >>> lists = [[] for i in range(3)]\n >>> lists[0].append(3)\n >>> lists[1].append(5)\n >>> lists[2].append(7)\n >>> lists\n [[3], [5], [7]]\n\n3. If *i* or *j* is negative, the index is relative to the end of the\n string: ``len(s) + i`` or ``len(s) + j`` is substituted. But note\n that ``-0`` is still ``0``.\n\n4. The slice of *s* from *i* to *j* is defined as the sequence of\n items with index *k* such that ``i <= k < j``. If *i* or *j* is\n greater than ``len(s)``, use ``len(s)``. If *i* is omitted or\n ``None``, use ``0``. If *j* is omitted or ``None``, use\n ``len(s)``. If *i* is greater than or equal to *j*, the slice is\n empty.\n\n5. The slice of *s* from *i* to *j* with step *k* is defined as the\n sequence of items with index ``x = i + n*k`` such that ``0 <= n <\n (j-i)/k``. In other words, the indices are ``i``, ``i+k``,\n ``i+2*k``, ``i+3*k`` and so on, stopping when *j* is reached (but\n never including *j*). If *i* or *j* is greater than ``len(s)``,\n use ``len(s)``. If *i* or *j* are omitted or ``None``, they become\n "end" values (which end depends on the sign of *k*). Note, *k*\n cannot be zero. If *k* is ``None``, it is treated like ``1``.\n\n6. **CPython implementation detail:** If *s* and *t* are both strings,\n some Python implementations such as CPython can usually perform an\n in-place optimization for assignments of the form ``s = s + t`` or\n ``s += t``. When applicable, this optimization makes quadratic\n run-time much less likely. This optimization is both version and\n implementation dependent. For performance sensitive code, it is\n preferable to use the ``str.join()`` method which assures\n consistent linear concatenation performance across versions and\n implementations.\n\n Changed in version 2.4: Formerly, string concatenation never\n occurred in-place.\n\n\nString Methods\n==============\n\nBelow are listed the string methods which both 8-bit strings and\nUnicode objects support.\n\nIn addition, Python\'s strings support the sequence type methods\ndescribed in the *Sequence Types --- str, unicode, list, tuple,\nbuffer, xrange* section. To output formatted strings use template\nstrings or the ``%`` operator described in the *String Formatting\nOperations* section. Also, see the ``re`` module for string functions\nbased on regular expressions.\n\nstr.capitalize()\n\n Return a copy of the string with only its first character\n capitalized.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.center(width[, fillchar])\n\n Return centered in a string of length *width*. Padding is done\n using the specified *fillchar* (default is a space).\n\n Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.count(sub[, start[, end]])\n\n Return the number of non-overlapping occurrences of substring *sub*\n in the range [*start*, *end*]. Optional arguments *start* and\n *end* are interpreted as in slice notation.\n\nstr.decode([encoding[, errors]])\n\n Decodes the string using the codec registered for *encoding*.\n *encoding* defaults to the default string encoding. *errors* may\n be given to set a different error handling scheme. The default is\n ``\'strict\'``, meaning that encoding errors raise ``UnicodeError``.\n Other possible values are ``\'ignore\'``, ``\'replace\'`` and any other\n name registered via ``codecs.register_error()``, see section *Codec\n Base Classes*.\n\n New in version 2.2.\n\n Changed in version 2.3: Support for other error handling schemes\n added.\n\n Changed in version 2.7: Support for keyword arguments added.\n\nstr.encode([encoding[, errors]])\n\n Return an encoded version of the string. Default encoding is the\n current default string encoding. *errors* may be given to set a\n different error handling scheme. The default for *errors* is\n ``\'strict\'``, meaning that encoding errors raise a\n ``UnicodeError``. Other possible values are ``\'ignore\'``,\n ``\'replace\'``, ``\'xmlcharrefreplace\'``, ``\'backslashreplace\'`` and\n any other name registered via ``codecs.register_error()``, see\n section *Codec Base Classes*. For a list of possible encodings, see\n section *Standard Encodings*.\n\n New in version 2.0.\n\n Changed in version 2.3: Support for ``\'xmlcharrefreplace\'`` and\n ``\'backslashreplace\'`` and other error handling schemes added.\n\n Changed in version 2.7: Support for keyword arguments added.\n\nstr.endswith(suffix[, start[, end]])\n\n Return ``True`` if the string ends with the specified *suffix*,\n otherwise return ``False``. *suffix* can also be a tuple of\n suffixes to look for. With optional *start*, test beginning at\n that position. With optional *end*, stop comparing at that\n position.\n\n Changed in version 2.5: Accept tuples as *suffix*.\n\nstr.expandtabs([tabsize])\n\n Return a copy of the string where all tab characters are replaced\n by one or more spaces, depending on the current column and the\n given tab size. The column number is reset to zero after each\n newline occurring in the string. If *tabsize* is not given, a tab\n size of ``8`` characters is assumed. This doesn\'t understand other\n non-printing characters or escape sequences.\n\nstr.find(sub[, start[, end]])\n\n Return the lowest index in the string where substring *sub* is\n found, such that *sub* is contained in the slice ``s[start:end]``.\n Optional arguments *start* and *end* are interpreted as in slice\n notation. Return ``-1`` if *sub* is not found.\n\nstr.format(*args, **kwargs)\n\n Perform a string formatting operation. The string on which this\n method is called can contain literal text or replacement fields\n delimited by braces ``{}``. Each replacement field contains either\n the numeric index of a positional argument, or the name of a\n keyword argument. Returns a copy of the string where each\n replacement field is replaced with the string value of the\n corresponding argument.\n\n >>> "The sum of 1 + 2 is {0}".format(1+2)\n \'The sum of 1 + 2 is 3\'\n\n See *Format String Syntax* for a description of the various\n formatting options that can be specified in format strings.\n\n This method of string formatting is the new standard in Python 3.0,\n and should be preferred to the ``%`` formatting described in\n *String Formatting Operations* in new code.\n\n New in version 2.6.\n\nstr.index(sub[, start[, end]])\n\n Like ``find()``, but raise ``ValueError`` when the substring is not\n found.\n\nstr.isalnum()\n\n Return true if all characters in the string are alphanumeric and\n there is at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isalpha()\n\n Return true if all characters in the string are alphabetic and\n there is at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isdigit()\n\n Return true if all characters in the string are digits and there is\n at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.islower()\n\n Return true if all cased characters in the string are lowercase and\n there is at least one cased character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isspace()\n\n Return true if there are only whitespace characters in the string\n and there is at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.istitle()\n\n Return true if the string is a titlecased string and there is at\n least one character, for example uppercase characters may only\n follow uncased characters and lowercase characters only cased ones.\n Return false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isupper()\n\n Return true if all cased characters in the string are uppercase and\n there is at least one cased character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.join(iterable)\n\n Return a string which is the concatenation of the strings in the\n *iterable* *iterable*. The separator between elements is the\n string providing this method.\n\nstr.ljust(width[, fillchar])\n\n Return the string left justified in a string of length *width*.\n Padding is done using the specified *fillchar* (default is a\n space). The original string is returned if *width* is less than\n ``len(s)``.\n\n Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.lower()\n\n Return a copy of the string converted to lowercase.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.lstrip([chars])\n\n Return a copy of the string with leading characters removed. The\n *chars* argument is a string specifying the set of characters to be\n removed. If omitted or ``None``, the *chars* argument defaults to\n removing whitespace. The *chars* argument is not a prefix; rather,\n all combinations of its values are stripped:\n\n >>> \' spacious \'.lstrip()\n \'spacious \'\n >>> \'www.example.com\'.lstrip(\'cmowz.\')\n \'example.com\'\n\n Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.partition(sep)\n\n Split the string at the first occurrence of *sep*, and return a\n 3-tuple containing the part before the separator, the separator\n itself, and the part after the separator. If the separator is not\n found, return a 3-tuple containing the string itself, followed by\n two empty strings.\n\n New in version 2.5.\n\nstr.replace(old, new[, count])\n\n Return a copy of the string with all occurrences of substring *old*\n replaced by *new*. If the optional argument *count* is given, only\n the first *count* occurrences are replaced.\n\nstr.rfind(sub[, start[, end]])\n\n Return the highest index in the string where substring *sub* is\n found, such that *sub* is contained within ``s[start:end]``.\n Optional arguments *start* and *end* are interpreted as in slice\n notation. Return ``-1`` on failure.\n\nstr.rindex(sub[, start[, end]])\n\n Like ``rfind()`` but raises ``ValueError`` when the substring *sub*\n is not found.\n\nstr.rjust(width[, fillchar])\n\n Return the string right justified in a string of length *width*.\n Padding is done using the specified *fillchar* (default is a\n space). The original string is returned if *width* is less than\n ``len(s)``.\n\n Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.rpartition(sep)\n\n Split the string at the last occurrence of *sep*, and return a\n 3-tuple containing the part before the separator, the separator\n itself, and the part after the separator. If the separator is not\n found, return a 3-tuple containing two empty strings, followed by\n the string itself.\n\n New in version 2.5.\n\nstr.rsplit([sep[, maxsplit]])\n\n Return a list of the words in the string, using *sep* as the\n delimiter string. If *maxsplit* is given, at most *maxsplit* splits\n are done, the *rightmost* ones. If *sep* is not specified or\n ``None``, any whitespace string is a separator. Except for\n splitting from the right, ``rsplit()`` behaves like ``split()``\n which is described in detail below.\n\n New in version 2.4.\n\nstr.rstrip([chars])\n\n Return a copy of the string with trailing characters removed. The\n *chars* argument is a string specifying the set of characters to be\n removed. If omitted or ``None``, the *chars* argument defaults to\n removing whitespace. The *chars* argument is not a suffix; rather,\n all combinations of its values are stripped:\n\n >>> \' spacious \'.rstrip()\n \' spacious\'\n >>> \'mississippi\'.rstrip(\'ipz\')\n \'mississ\'\n\n Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.split([sep[, maxsplit]])\n\n Return a list of the words in the string, using *sep* as the\n delimiter string. If *maxsplit* is given, at most *maxsplit*\n splits are done (thus, the list will have at most ``maxsplit+1``\n elements). If *maxsplit* is not specified, then there is no limit\n on the number of splits (all possible splits are made).\n\n If *sep* is given, consecutive delimiters are not grouped together\n and are deemed to delimit empty strings (for example,\n ``\'1,,2\'.split(\',\')`` returns ``[\'1\', \'\', \'2\']``). The *sep*\n argument may consist of multiple characters (for example,\n ``\'1<>2<>3\'.split(\'<>\')`` returns ``[\'1\', \'2\', \'3\']``). Splitting\n an empty string with a specified separator returns ``[\'\']``.\n\n If *sep* is not specified or is ``None``, a different splitting\n algorithm is applied: runs of consecutive whitespace are regarded\n as a single separator, and the result will contain no empty strings\n at the start or end if the string has leading or trailing\n whitespace. Consequently, splitting an empty string or a string\n consisting of just whitespace with a ``None`` separator returns\n ``[]``.\n\n For example, ``\' 1 2 3 \'.split()`` returns ``[\'1\', \'2\', \'3\']``,\n and ``\' 1 2 3 \'.split(None, 1)`` returns ``[\'1\', \'2 3 \']``.\n\nstr.splitlines([keepends])\n\n Return a list of the lines in the string, breaking at line\n boundaries. Line breaks are not included in the resulting list\n unless *keepends* is given and true.\n\nstr.startswith(prefix[, start[, end]])\n\n Return ``True`` if string starts with the *prefix*, otherwise\n return ``False``. *prefix* can also be a tuple of prefixes to look\n for. With optional *start*, test string beginning at that\n position. With optional *end*, stop comparing string at that\n position.\n\n Changed in version 2.5: Accept tuples as *prefix*.\n\nstr.strip([chars])\n\n Return a copy of the string with the leading and trailing\n characters removed. The *chars* argument is a string specifying the\n set of characters to be removed. If omitted or ``None``, the\n *chars* argument defaults to removing whitespace. The *chars*\n argument is not a prefix or suffix; rather, all combinations of its\n values are stripped:\n\n >>> \' spacious \'.strip()\n \'spacious\'\n >>> \'www.example.com\'.strip(\'cmowz.\')\n \'example\'\n\n Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.swapcase()\n\n Return a copy of the string with uppercase characters converted to\n lowercase and vice versa.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.title()\n\n Return a titlecased version of the string where words start with an\n uppercase character and the remaining characters are lowercase.\n\n The algorithm uses a simple language-independent definition of a\n word as groups of consecutive letters. The definition works in\n many contexts but it means that apostrophes in contractions and\n possessives form word boundaries, which may not be the desired\n result:\n\n >>> "they\'re bill\'s friends from the UK".title()\n "They\'Re Bill\'S Friends From The Uk"\n\n A workaround for apostrophes can be constructed using regular\n expressions:\n\n >>> import re\n >>> def titlecase(s):\n return re.sub(r"[A-Za-z]+(\'[A-Za-z]+)?",\n lambda mo: mo.group(0)[0].upper() +\n mo.group(0)[1:].lower(),\n s)\n\n >>> titlecase("they\'re bill\'s friends.")\n "They\'re Bill\'s Friends."\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.translate(table[, deletechars])\n\n Return a copy of the string where all characters occurring in the\n optional argument *deletechars* are removed, and the remaining\n characters have been mapped through the given translation table,\n which must be a string of length 256.\n\n You can use the ``maketrans()`` helper function in the ``string``\n module to create a translation table. For string objects, set the\n *table* argument to ``None`` for translations that only delete\n characters:\n\n >>> \'read this short text\'.translate(None, \'aeiou\')\n \'rd ths shrt txt\'\n\n New in version 2.6: Support for a ``None`` *table* argument.\n\n For Unicode objects, the ``translate()`` method does not accept the\n optional *deletechars* argument. Instead, it returns a copy of the\n *s* where all characters have been mapped through the given\n translation table which must be a mapping of Unicode ordinals to\n Unicode ordinals, Unicode strings or ``None``. Unmapped characters\n are left untouched. Characters mapped to ``None`` are deleted.\n Note, a more flexible approach is to create a custom character\n mapping codec using the ``codecs`` module (see ``encodings.cp1251``\n for an example).\n\nstr.upper()\n\n Return a copy of the string converted to uppercase.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.zfill(width)\n\n Return the numeric string left filled with zeros in a string of\n length *width*. A sign prefix is handled correctly. The original\n string is returned if *width* is less than ``len(s)``.\n\n New in version 2.2.2.\n\nThe following methods are present only on unicode objects:\n\nunicode.isnumeric()\n\n Return ``True`` if there are only numeric characters in S,\n ``False`` otherwise. Numeric characters include digit characters,\n and all characters that have the Unicode numeric value property,\n e.g. U+2155, VULGAR FRACTION ONE FIFTH.\n\nunicode.isdecimal()\n\n Return ``True`` if there are only decimal characters in S,\n ``False`` otherwise. Decimal characters include digit characters,\n and all characters that that can be used to form decimal-radix\n numbers, e.g. U+0660, ARABIC-INDIC DIGIT ZERO.\n\n\nString Formatting Operations\n============================\n\nString and Unicode objects have one unique built-in operation: the\n``%`` operator (modulo). This is also known as the string\n*formatting* or *interpolation* operator. Given ``format % values``\n(where *format* is a string or Unicode object), ``%`` conversion\nspecifications in *format* are replaced with zero or more elements of\n*values*. The effect is similar to the using ``sprintf()`` in the C\nlanguage. If *format* is a Unicode object, or if any of the objects\nbeing converted using the ``%s`` conversion are Unicode objects, the\nresult will also be a Unicode object.\n\nIf *format* requires a single argument, *values* may be a single non-\ntuple object. [4] Otherwise, *values* must be a tuple with exactly\nthe number of items specified by the format string, or a single\nmapping object (for example, a dictionary).\n\nA conversion specifier contains two or more characters and has the\nfollowing components, which must occur in this order:\n\n1. The ``\'%\'`` character, which marks the start of the specifier.\n\n2. Mapping key (optional), consisting of a parenthesised sequence of\n characters (for example, ``(somename)``).\n\n3. Conversion flags (optional), which affect the result of some\n conversion types.\n\n4. Minimum field width (optional). If specified as an ``\'*\'``\n (asterisk), the actual width is read from the next element of the\n tuple in *values*, and the object to convert comes after the\n minimum field width and optional precision.\n\n5. Precision (optional), given as a ``\'.\'`` (dot) followed by the\n precision. If specified as ``\'*\'`` (an asterisk), the actual width\n is read from the next element of the tuple in *values*, and the\n value to convert comes after the precision.\n\n6. Length modifier (optional).\n\n7. Conversion type.\n\nWhen the right argument is a dictionary (or other mapping type), then\nthe formats in the string *must* include a parenthesised mapping key\ninto that dictionary inserted immediately after the ``\'%\'`` character.\nThe mapping key selects the value to be formatted from the mapping.\nFor example:\n\n>>> print \'%(language)s has %(#)03d quote types.\' % \\\n... {\'language\': "Python", "#": 2}\nPython has 002 quote types.\n\nIn this case no ``*`` specifiers may occur in a format (since they\nrequire a sequential parameter list).\n\nThe conversion flag characters are:\n\n+-----------+-----------------------------------------------------------------------+\n| Flag | Meaning |\n+===========+=======================================================================+\n| ``\'#\'`` | The value conversion will use the "alternate form" (where defined |\n| | below). |\n+-----------+-----------------------------------------------------------------------+\n| ``\'0\'`` | The conversion will be zero padded for numeric values. |\n+-----------+-----------------------------------------------------------------------+\n| ``\'-\'`` | The converted value is left adjusted (overrides the ``\'0\'`` |\n| | conversion if both are given). |\n+-----------+-----------------------------------------------------------------------+\n| ``\' \'`` | (a space) A blank should be left before a positive number (or empty |\n| | string) produced by a signed conversion. |\n+-----------+-----------------------------------------------------------------------+\n| ``\'+\'`` | A sign character (``\'+\'`` or ``\'-\'``) will precede the conversion |\n| | (overrides a "space" flag). |\n+-----------+-----------------------------------------------------------------------+\n\nA length modifier (``h``, ``l``, or ``L``) may be present, but is\nignored as it is not necessary for Python -- so e.g. ``%ld`` is\nidentical to ``%d``.\n\nThe conversion types are:\n\n+--------------+-------------------------------------------------------+---------+\n| Conversion | Meaning | Notes |\n+==============+=======================================================+=========+\n| ``\'d\'`` | Signed integer decimal. | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'i\'`` | Signed integer decimal. | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'o\'`` | Signed octal value. | (1) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'u\'`` | Obsolete type -- it is identical to ``\'d\'``. | (7) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'x\'`` | Signed hexadecimal (lowercase). | (2) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'X\'`` | Signed hexadecimal (uppercase). | (2) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'e\'`` | Floating point exponential format (lowercase). | (3) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'E\'`` | Floating point exponential format (uppercase). | (3) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'f\'`` | Floating point decimal format. | (3) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'F\'`` | Floating point decimal format. | (3) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'g\'`` | Floating point format. Uses lowercase exponential | (4) |\n| | format if exponent is less than -4 or not less than | |\n| | precision, decimal format otherwise. | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'G\'`` | Floating point format. Uses uppercase exponential | (4) |\n| | format if exponent is less than -4 or not less than | |\n| | precision, decimal format otherwise. | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'c\'`` | Single character (accepts integer or single character | |\n| | string). | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'r\'`` | String (converts any Python object using ``repr()``). | (5) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'s\'`` | String (converts any Python object using ``str()``). | (6) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'%\'`` | No argument is converted, results in a ``\'%\'`` | |\n| | character in the result. | |\n+--------------+-------------------------------------------------------+---------+\n\nNotes:\n\n1. The alternate form causes a leading zero (``\'0\'``) to be inserted\n between left-hand padding and the formatting of the number if the\n leading character of the result is not already a zero.\n\n2. The alternate form causes a leading ``\'0x\'`` or ``\'0X\'`` (depending\n on whether the ``\'x\'`` or ``\'X\'`` format was used) to be inserted\n between left-hand padding and the formatting of the number if the\n leading character of the result is not already a zero.\n\n3. The alternate form causes the result to always contain a decimal\n point, even if no digits follow it.\n\n The precision determines the number of digits after the decimal\n point and defaults to 6.\n\n4. The alternate form causes the result to always contain a decimal\n point, and trailing zeroes are not removed as they would otherwise\n be.\n\n The precision determines the number of significant digits before\n and after the decimal point and defaults to 6.\n\n5. The ``%r`` conversion was added in Python 2.0.\n\n The precision determines the maximal number of characters used.\n\n6. If the object or format provided is a ``unicode`` string, the\n resulting string will also be ``unicode``.\n\n The precision determines the maximal number of characters used.\n\n7. See **PEP 237**.\n\nSince Python strings have an explicit length, ``%s`` conversions do\nnot assume that ``\'\\0\'`` is the end of the string.\n\nChanged in version 2.7: ``%f`` conversions for numbers whose absolute\nvalue is over 1e50 are no longer replaced by ``%g`` conversions.\n\nAdditional string operations are defined in standard modules\n``string`` and ``re``.\n\n\nXRange Type\n===========\n\nThe ``xrange`` type is an immutable sequence which is commonly used\nfor looping. The advantage of the ``xrange`` type is that an\n``xrange`` object will always take the same amount of memory, no\nmatter the size of the range it represents. There are no consistent\nperformance advantages.\n\nXRange objects have very little behavior: they only support indexing,\niteration, and the ``len()`` function.\n\n\nMutable Sequence Types\n======================\n\nList objects support additional operations that allow in-place\nmodification of the object. Other mutable sequence types (when added\nto the language) should also support these operations. Strings and\ntuples are immutable sequence types: such objects cannot be modified\nonce created. The following operations are defined on mutable sequence\ntypes (where *x* is an arbitrary object):\n\n+--------------------------------+----------------------------------+-----------------------+\n| Operation | Result | Notes |\n+================================+==================================+=======================+\n| ``s[i] = x`` | item *i* of *s* is replaced by | |\n| | *x* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s[i:j] = t`` | slice of *s* from *i* to *j* is | |\n| | replaced by the contents of the | |\n| | iterable *t* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``del s[i:j]`` | same as ``s[i:j] = []`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s[i:j:k] = t`` | the elements of ``s[i:j:k]`` are | (1) |\n| | replaced by those of *t* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``del s[i:j:k]`` | removes the elements of | |\n| | ``s[i:j:k]`` from the list | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.append(x)`` | same as ``s[len(s):len(s)] = | (2) |\n| | [x]`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.extend(x)`` | same as ``s[len(s):len(s)] = x`` | (3) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.count(x)`` | return number of *i*\'s for which | |\n| | ``s[i] == x`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.index(x[, i[, j]])`` | return smallest *k* such that | (4) |\n| | ``s[k] == x`` and ``i <= k < j`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.insert(i, x)`` | same as ``s[i:i] = [x]`` | (5) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.pop([i])`` | same as ``x = s[i]; del s[i]; | (6) |\n| | return x`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.remove(x)`` | same as ``del s[s.index(x)]`` | (4) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.reverse()`` | reverses the items of *s* in | (7) |\n| | place | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.sort([cmp[, key[, | sort the items of *s* in place | (7)(8)(9)(10) |\n| reverse]]])`` | | |\n+--------------------------------+----------------------------------+-----------------------+\n\nNotes:\n\n1. *t* must have the same length as the slice it is replacing.\n\n2. The C implementation of Python has historically accepted multiple\n parameters and implicitly joined them into a tuple; this no longer\n works in Python 2.0. Use of this misfeature has been deprecated\n since Python 1.4.\n\n3. *x* can be any iterable object.\n\n4. Raises ``ValueError`` when *x* is not found in *s*. When a negative\n index is passed as the second or third parameter to the ``index()``\n method, the list length is added, as for slice indices. If it is\n still negative, it is truncated to zero, as for slice indices.\n\n Changed in version 2.3: Previously, ``index()`` didn\'t have\n arguments for specifying start and stop positions.\n\n5. When a negative index is passed as the first parameter to the\n ``insert()`` method, the list length is added, as for slice\n indices. If it is still negative, it is truncated to zero, as for\n slice indices.\n\n Changed in version 2.3: Previously, all negative indices were\n truncated to zero.\n\n6. The ``pop()`` method is only supported by the list and array types.\n The optional argument *i* defaults to ``-1``, so that by default\n the last item is removed and returned.\n\n7. The ``sort()`` and ``reverse()`` methods modify the list in place\n for economy of space when sorting or reversing a large list. To\n remind you that they operate by side effect, they don\'t return the\n sorted or reversed list.\n\n8. The ``sort()`` method takes optional arguments for controlling the\n comparisons.\n\n *cmp* specifies a custom comparison function of two arguments (list\n items) which should return a negative, zero or positive number\n depending on whether the first argument is considered smaller than,\n equal to, or larger than the second argument: ``cmp=lambda x,y:\n cmp(x.lower(), y.lower())``. The default value is ``None``.\n\n *key* specifies a function of one argument that is used to extract\n a comparison key from each list element: ``key=str.lower``. The\n default value is ``None``.\n\n *reverse* is a boolean value. If set to ``True``, then the list\n elements are sorted as if each comparison were reversed.\n\n In general, the *key* and *reverse* conversion processes are much\n faster than specifying an equivalent *cmp* function. This is\n because *cmp* is called multiple times for each list element while\n *key* and *reverse* touch each element only once. Use\n ``functools.cmp_to_key()`` to convert an old-style *cmp* function\n to a *key* function.\n\n Changed in version 2.3: Support for ``None`` as an equivalent to\n omitting *cmp* was added.\n\n Changed in version 2.4: Support for *key* and *reverse* was added.\n\n9. Starting with Python 2.3, the ``sort()`` method is guaranteed to be\n stable. A sort is stable if it guarantees not to change the\n relative order of elements that compare equal --- this is helpful\n for sorting in multiple passes (for example, sort by department,\n then by salary grade).\n\n10. **CPython implementation detail:** While a list is being sorted,\n the effect of attempting to mutate, or even inspect, the list is\n undefined. The C implementation of Python 2.3 and newer makes the\n list appear empty for the duration, and raises ``ValueError`` if\n it can detect that the list has been mutated during a sort.\n', - 'typesseq-mutable': u"\nMutable Sequence Types\n**********************\n\nList objects support additional operations that allow in-place\nmodification of the object. Other mutable sequence types (when added\nto the language) should also support these operations. Strings and\ntuples are immutable sequence types: such objects cannot be modified\nonce created. The following operations are defined on mutable sequence\ntypes (where *x* is an arbitrary object):\n\n+--------------------------------+----------------------------------+-----------------------+\n| Operation | Result | Notes |\n+================================+==================================+=======================+\n| ``s[i] = x`` | item *i* of *s* is replaced by | |\n| | *x* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s[i:j] = t`` | slice of *s* from *i* to *j* is | |\n| | replaced by the contents of the | |\n| | iterable *t* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``del s[i:j]`` | same as ``s[i:j] = []`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s[i:j:k] = t`` | the elements of ``s[i:j:k]`` are | (1) |\n| | replaced by those of *t* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``del s[i:j:k]`` | removes the elements of | |\n| | ``s[i:j:k]`` from the list | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.append(x)`` | same as ``s[len(s):len(s)] = | (2) |\n| | [x]`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.extend(x)`` | same as ``s[len(s):len(s)] = x`` | (3) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.count(x)`` | return number of *i*'s for which | |\n| | ``s[i] == x`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.index(x[, i[, j]])`` | return smallest *k* such that | (4) |\n| | ``s[k] == x`` and ``i <= k < j`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.insert(i, x)`` | same as ``s[i:i] = [x]`` | (5) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.pop([i])`` | same as ``x = s[i]; del s[i]; | (6) |\n| | return x`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.remove(x)`` | same as ``del s[s.index(x)]`` | (4) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.reverse()`` | reverses the items of *s* in | (7) |\n| | place | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.sort([cmp[, key[, | sort the items of *s* in place | (7)(8)(9)(10) |\n| reverse]]])`` | | |\n+--------------------------------+----------------------------------+-----------------------+\n\nNotes:\n\n1. *t* must have the same length as the slice it is replacing.\n\n2. The C implementation of Python has historically accepted multiple\n parameters and implicitly joined them into a tuple; this no longer\n works in Python 2.0. Use of this misfeature has been deprecated\n since Python 1.4.\n\n3. *x* can be any iterable object.\n\n4. Raises ``ValueError`` when *x* is not found in *s*. When a negative\n index is passed as the second or third parameter to the ``index()``\n method, the list length is added, as for slice indices. If it is\n still negative, it is truncated to zero, as for slice indices.\n\n Changed in version 2.3: Previously, ``index()`` didn't have\n arguments for specifying start and stop positions.\n\n5. When a negative index is passed as the first parameter to the\n ``insert()`` method, the list length is added, as for slice\n indices. If it is still negative, it is truncated to zero, as for\n slice indices.\n\n Changed in version 2.3: Previously, all negative indices were\n truncated to zero.\n\n6. The ``pop()`` method is only supported by the list and array types.\n The optional argument *i* defaults to ``-1``, so that by default\n the last item is removed and returned.\n\n7. The ``sort()`` and ``reverse()`` methods modify the list in place\n for economy of space when sorting or reversing a large list. To\n remind you that they operate by side effect, they don't return the\n sorted or reversed list.\n\n8. The ``sort()`` method takes optional arguments for controlling the\n comparisons.\n\n *cmp* specifies a custom comparison function of two arguments (list\n items) which should return a negative, zero or positive number\n depending on whether the first argument is considered smaller than,\n equal to, or larger than the second argument: ``cmp=lambda x,y:\n cmp(x.lower(), y.lower())``. The default value is ``None``.\n\n *key* specifies a function of one argument that is used to extract\n a comparison key from each list element: ``key=str.lower``. The\n default value is ``None``.\n\n *reverse* is a boolean value. If set to ``True``, then the list\n elements are sorted as if each comparison were reversed.\n\n In general, the *key* and *reverse* conversion processes are much\n faster than specifying an equivalent *cmp* function. This is\n because *cmp* is called multiple times for each list element while\n *key* and *reverse* touch each element only once. Use\n ``functools.cmp_to_key()`` to convert an old-style *cmp* function\n to a *key* function.\n\n Changed in version 2.3: Support for ``None`` as an equivalent to\n omitting *cmp* was added.\n\n Changed in version 2.4: Support for *key* and *reverse* was added.\n\n9. Starting with Python 2.3, the ``sort()`` method is guaranteed to be\n stable. A sort is stable if it guarantees not to change the\n relative order of elements that compare equal --- this is helpful\n for sorting in multiple passes (for example, sort by department,\n then by salary grade).\n\n10. **CPython implementation detail:** While a list is being sorted,\n the effect of attempting to mutate, or even inspect, the list is\n undefined. The C implementation of Python 2.3 and newer makes the\n list appear empty for the duration, and raises ``ValueError`` if\n it can detect that the list has been mutated during a sort.\n", + 'typesseq': u'\nSequence Types --- ``str``, ``unicode``, ``list``, ``tuple``, ``bytearray``, ``buffer``, ``xrange``\n***************************************************************************************************\n\nThere are seven sequence types: strings, Unicode strings, lists,\ntuples, bytearrays, buffers, and xrange objects.\n\nFor other containers see the built in ``dict`` and ``set`` classes,\nand the ``collections`` module.\n\nString literals are written in single or double quotes: ``\'xyzzy\'``,\n``"frobozz"``. See *String literals* for more about string literals.\nUnicode strings are much like strings, but are specified in the syntax\nusing a preceding ``\'u\'`` character: ``u\'abc\'``, ``u"def"``. In\naddition to the functionality described here, there are also string-\nspecific methods described in the *String Methods* section. Lists are\nconstructed with square brackets, separating items with commas: ``[a,\nb, c]``. Tuples are constructed by the comma operator (not within\nsquare brackets), with or without enclosing parentheses, but an empty\ntuple must have the enclosing parentheses, such as ``a, b, c`` or\n``()``. A single item tuple must have a trailing comma, such as\n``(d,)``.\n\nBytearray objects are created with the built-in function\n``bytearray()``.\n\nBuffer objects are not directly supported by Python syntax, but can be\ncreated by calling the built-in function ``buffer()``. They don\'t\nsupport concatenation or repetition.\n\nObjects of type xrange are similar to buffers in that there is no\nspecific syntax to create them, but they are created using the\n``xrange()`` function. They don\'t support slicing, concatenation or\nrepetition, and using ``in``, ``not in``, ``min()`` or ``max()`` on\nthem is inefficient.\n\nMost sequence types support the following operations. The ``in`` and\n``not in`` operations have the same priorities as the comparison\noperations. The ``+`` and ``*`` operations have the same priority as\nthe corresponding numeric operations. [3] Additional methods are\nprovided for *Mutable Sequence Types*.\n\nThis table lists the sequence operations sorted in ascending priority\n(operations in the same box have the same priority). In the table,\n*s* and *t* are sequences of the same type; *n*, *i* and *j* are\nintegers:\n\n+--------------------+----------------------------------+------------+\n| Operation | Result | Notes |\n+====================+==================================+============+\n| ``x in s`` | ``True`` if an item of *s* is | (1) |\n| | equal to *x*, else ``False`` | |\n+--------------------+----------------------------------+------------+\n| ``x not in s`` | ``False`` if an item of *s* is | (1) |\n| | equal to *x*, else ``True`` | |\n+--------------------+----------------------------------+------------+\n| ``s + t`` | the concatenation of *s* and *t* | (6) |\n+--------------------+----------------------------------+------------+\n| ``s * n, n * s`` | *n* shallow copies of *s* | (2) |\n| | concatenated | |\n+--------------------+----------------------------------+------------+\n| ``s[i]`` | *i*\'th item of *s*, origin 0 | (3) |\n+--------------------+----------------------------------+------------+\n| ``s[i:j]`` | slice of *s* from *i* to *j* | (3)(4) |\n+--------------------+----------------------------------+------------+\n| ``s[i:j:k]`` | slice of *s* from *i* to *j* | (3)(5) |\n| | with step *k* | |\n+--------------------+----------------------------------+------------+\n| ``len(s)`` | length of *s* | |\n+--------------------+----------------------------------+------------+\n| ``min(s)`` | smallest item of *s* | |\n+--------------------+----------------------------------+------------+\n| ``max(s)`` | largest item of *s* | |\n+--------------------+----------------------------------+------------+\n| ``s.index(i)`` | index of the first occurence of | |\n| | *i* in *s* | |\n+--------------------+----------------------------------+------------+\n| ``s.count(i)`` | total number of occurences of | |\n| | *i* in *s* | |\n+--------------------+----------------------------------+------------+\n\nSequence types also support comparisons. In particular, tuples and\nlists are compared lexicographically by comparing corresponding\nelements. This means that to compare equal, every element must compare\nequal and the two sequences must be of the same type and have the same\nlength. (For full details see *Comparisons* in the language\nreference.)\n\nNotes:\n\n1. When *s* is a string or Unicode string object the ``in`` and ``not\n in`` operations act like a substring test. In Python versions\n before 2.3, *x* had to be a string of length 1. In Python 2.3 and\n beyond, *x* may be a string of any length.\n\n2. Values of *n* less than ``0`` are treated as ``0`` (which yields an\n empty sequence of the same type as *s*). Note also that the copies\n are shallow; nested structures are not copied. This often haunts\n new Python programmers; consider:\n\n >>> lists = [[]] * 3\n >>> lists\n [[], [], []]\n >>> lists[0].append(3)\n >>> lists\n [[3], [3], [3]]\n\n What has happened is that ``[[]]`` is a one-element list containing\n an empty list, so all three elements of ``[[]] * 3`` are (pointers\n to) this single empty list. Modifying any of the elements of\n ``lists`` modifies this single list. You can create a list of\n different lists this way:\n\n >>> lists = [[] for i in range(3)]\n >>> lists[0].append(3)\n >>> lists[1].append(5)\n >>> lists[2].append(7)\n >>> lists\n [[3], [5], [7]]\n\n3. If *i* or *j* is negative, the index is relative to the end of the\n string: ``len(s) + i`` or ``len(s) + j`` is substituted. But note\n that ``-0`` is still ``0``.\n\n4. The slice of *s* from *i* to *j* is defined as the sequence of\n items with index *k* such that ``i <= k < j``. If *i* or *j* is\n greater than ``len(s)``, use ``len(s)``. If *i* is omitted or\n ``None``, use ``0``. If *j* is omitted or ``None``, use\n ``len(s)``. If *i* is greater than or equal to *j*, the slice is\n empty.\n\n5. The slice of *s* from *i* to *j* with step *k* is defined as the\n sequence of items with index ``x = i + n*k`` such that ``0 <= n <\n (j-i)/k``. In other words, the indices are ``i``, ``i+k``,\n ``i+2*k``, ``i+3*k`` and so on, stopping when *j* is reached (but\n never including *j*). If *i* or *j* is greater than ``len(s)``,\n use ``len(s)``. If *i* or *j* are omitted or ``None``, they become\n "end" values (which end depends on the sign of *k*). Note, *k*\n cannot be zero. If *k* is ``None``, it is treated like ``1``.\n\n6. **CPython implementation detail:** If *s* and *t* are both strings,\n some Python implementations such as CPython can usually perform an\n in-place optimization for assignments of the form ``s = s + t`` or\n ``s += t``. When applicable, this optimization makes quadratic\n run-time much less likely. This optimization is both version and\n implementation dependent. For performance sensitive code, it is\n preferable to use the ``str.join()`` method which assures\n consistent linear concatenation performance across versions and\n implementations.\n\n Changed in version 2.4: Formerly, string concatenation never\n occurred in-place.\n\n\nString Methods\n==============\n\nBelow are listed the string methods which both 8-bit strings and\nUnicode objects support. Some of them are also available on\n``bytearray`` objects.\n\nIn addition, Python\'s strings support the sequence type methods\ndescribed in the *Sequence Types --- str, unicode, list, tuple,\nbytearray, buffer, xrange* section. To output formatted strings use\ntemplate strings or the ``%`` operator described in the *String\nFormatting Operations* section. Also, see the ``re`` module for string\nfunctions based on regular expressions.\n\nstr.capitalize()\n\n Return a copy of the string with its first character capitalized\n and the rest lowercased.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.center(width[, fillchar])\n\n Return centered in a string of length *width*. Padding is done\n using the specified *fillchar* (default is a space).\n\n Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.count(sub[, start[, end]])\n\n Return the number of non-overlapping occurrences of substring *sub*\n in the range [*start*, *end*]. Optional arguments *start* and\n *end* are interpreted as in slice notation.\n\nstr.decode([encoding[, errors]])\n\n Decodes the string using the codec registered for *encoding*.\n *encoding* defaults to the default string encoding. *errors* may\n be given to set a different error handling scheme. The default is\n ``\'strict\'``, meaning that encoding errors raise ``UnicodeError``.\n Other possible values are ``\'ignore\'``, ``\'replace\'`` and any other\n name registered via ``codecs.register_error()``, see section *Codec\n Base Classes*.\n\n New in version 2.2.\n\n Changed in version 2.3: Support for other error handling schemes\n added.\n\n Changed in version 2.7: Support for keyword arguments added.\n\nstr.encode([encoding[, errors]])\n\n Return an encoded version of the string. Default encoding is the\n current default string encoding. *errors* may be given to set a\n different error handling scheme. The default for *errors* is\n ``\'strict\'``, meaning that encoding errors raise a\n ``UnicodeError``. Other possible values are ``\'ignore\'``,\n ``\'replace\'``, ``\'xmlcharrefreplace\'``, ``\'backslashreplace\'`` and\n any other name registered via ``codecs.register_error()``, see\n section *Codec Base Classes*. For a list of possible encodings, see\n section *Standard Encodings*.\n\n New in version 2.0.\n\n Changed in version 2.3: Support for ``\'xmlcharrefreplace\'`` and\n ``\'backslashreplace\'`` and other error handling schemes added.\n\n Changed in version 2.7: Support for keyword arguments added.\n\nstr.endswith(suffix[, start[, end]])\n\n Return ``True`` if the string ends with the specified *suffix*,\n otherwise return ``False``. *suffix* can also be a tuple of\n suffixes to look for. With optional *start*, test beginning at\n that position. With optional *end*, stop comparing at that\n position.\n\n Changed in version 2.5: Accept tuples as *suffix*.\n\nstr.expandtabs([tabsize])\n\n Return a copy of the string where all tab characters are replaced\n by one or more spaces, depending on the current column and the\n given tab size. The column number is reset to zero after each\n newline occurring in the string. If *tabsize* is not given, a tab\n size of ``8`` characters is assumed. This doesn\'t understand other\n non-printing characters or escape sequences.\n\nstr.find(sub[, start[, end]])\n\n Return the lowest index in the string where substring *sub* is\n found, such that *sub* is contained in the slice ``s[start:end]``.\n Optional arguments *start* and *end* are interpreted as in slice\n notation. Return ``-1`` if *sub* is not found.\n\n Note: The ``find()`` method should be used only if you need to know the\n position of *sub*. To check if *sub* is a substring or not, use\n the ``in`` operator:\n\n >>> \'Py\' in \'Python\'\n True\n\nstr.format(*args, **kwargs)\n\n Perform a string formatting operation. The string on which this\n method is called can contain literal text or replacement fields\n delimited by braces ``{}``. Each replacement field contains either\n the numeric index of a positional argument, or the name of a\n keyword argument. Returns a copy of the string where each\n replacement field is replaced with the string value of the\n corresponding argument.\n\n >>> "The sum of 1 + 2 is {0}".format(1+2)\n \'The sum of 1 + 2 is 3\'\n\n See *Format String Syntax* for a description of the various\n formatting options that can be specified in format strings.\n\n This method of string formatting is the new standard in Python 3.0,\n and should be preferred to the ``%`` formatting described in\n *String Formatting Operations* in new code.\n\n New in version 2.6.\n\nstr.index(sub[, start[, end]])\n\n Like ``find()``, but raise ``ValueError`` when the substring is not\n found.\n\nstr.isalnum()\n\n Return true if all characters in the string are alphanumeric and\n there is at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isalpha()\n\n Return true if all characters in the string are alphabetic and\n there is at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isdigit()\n\n Return true if all characters in the string are digits and there is\n at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.islower()\n\n Return true if all cased characters in the string are lowercase and\n there is at least one cased character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isspace()\n\n Return true if there are only whitespace characters in the string\n and there is at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.istitle()\n\n Return true if the string is a titlecased string and there is at\n least one character, for example uppercase characters may only\n follow uncased characters and lowercase characters only cased ones.\n Return false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isupper()\n\n Return true if all cased characters in the string are uppercase and\n there is at least one cased character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.join(iterable)\n\n Return a string which is the concatenation of the strings in the\n *iterable* *iterable*. The separator between elements is the\n string providing this method.\n\nstr.ljust(width[, fillchar])\n\n Return the string left justified in a string of length *width*.\n Padding is done using the specified *fillchar* (default is a\n space). The original string is returned if *width* is less than\n ``len(s)``.\n\n Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.lower()\n\n Return a copy of the string converted to lowercase.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.lstrip([chars])\n\n Return a copy of the string with leading characters removed. The\n *chars* argument is a string specifying the set of characters to be\n removed. If omitted or ``None``, the *chars* argument defaults to\n removing whitespace. The *chars* argument is not a prefix; rather,\n all combinations of its values are stripped:\n\n >>> \' spacious \'.lstrip()\n \'spacious \'\n >>> \'www.example.com\'.lstrip(\'cmowz.\')\n \'example.com\'\n\n Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.partition(sep)\n\n Split the string at the first occurrence of *sep*, and return a\n 3-tuple containing the part before the separator, the separator\n itself, and the part after the separator. If the separator is not\n found, return a 3-tuple containing the string itself, followed by\n two empty strings.\n\n New in version 2.5.\n\nstr.replace(old, new[, count])\n\n Return a copy of the string with all occurrences of substring *old*\n replaced by *new*. If the optional argument *count* is given, only\n the first *count* occurrences are replaced.\n\nstr.rfind(sub[, start[, end]])\n\n Return the highest index in the string where substring *sub* is\n found, such that *sub* is contained within ``s[start:end]``.\n Optional arguments *start* and *end* are interpreted as in slice\n notation. Return ``-1`` on failure.\n\nstr.rindex(sub[, start[, end]])\n\n Like ``rfind()`` but raises ``ValueError`` when the substring *sub*\n is not found.\n\nstr.rjust(width[, fillchar])\n\n Return the string right justified in a string of length *width*.\n Padding is done using the specified *fillchar* (default is a\n space). The original string is returned if *width* is less than\n ``len(s)``.\n\n Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.rpartition(sep)\n\n Split the string at the last occurrence of *sep*, and return a\n 3-tuple containing the part before the separator, the separator\n itself, and the part after the separator. If the separator is not\n found, return a 3-tuple containing two empty strings, followed by\n the string itself.\n\n New in version 2.5.\n\nstr.rsplit([sep[, maxsplit]])\n\n Return a list of the words in the string, using *sep* as the\n delimiter string. If *maxsplit* is given, at most *maxsplit* splits\n are done, the *rightmost* ones. If *sep* is not specified or\n ``None``, any whitespace string is a separator. Except for\n splitting from the right, ``rsplit()`` behaves like ``split()``\n which is described in detail below.\n\n New in version 2.4.\n\nstr.rstrip([chars])\n\n Return a copy of the string with trailing characters removed. The\n *chars* argument is a string specifying the set of characters to be\n removed. If omitted or ``None``, the *chars* argument defaults to\n removing whitespace. The *chars* argument is not a suffix; rather,\n all combinations of its values are stripped:\n\n >>> \' spacious \'.rstrip()\n \' spacious\'\n >>> \'mississippi\'.rstrip(\'ipz\')\n \'mississ\'\n\n Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.split([sep[, maxsplit]])\n\n Return a list of the words in the string, using *sep* as the\n delimiter string. If *maxsplit* is given, at most *maxsplit*\n splits are done (thus, the list will have at most ``maxsplit+1``\n elements). If *maxsplit* is not specified, then there is no limit\n on the number of splits (all possible splits are made).\n\n If *sep* is given, consecutive delimiters are not grouped together\n and are deemed to delimit empty strings (for example,\n ``\'1,,2\'.split(\',\')`` returns ``[\'1\', \'\', \'2\']``). The *sep*\n argument may consist of multiple characters (for example,\n ``\'1<>2<>3\'.split(\'<>\')`` returns ``[\'1\', \'2\', \'3\']``). Splitting\n an empty string with a specified separator returns ``[\'\']``.\n\n If *sep* is not specified or is ``None``, a different splitting\n algorithm is applied: runs of consecutive whitespace are regarded\n as a single separator, and the result will contain no empty strings\n at the start or end if the string has leading or trailing\n whitespace. Consequently, splitting an empty string or a string\n consisting of just whitespace with a ``None`` separator returns\n ``[]``.\n\n For example, ``\' 1 2 3 \'.split()`` returns ``[\'1\', \'2\', \'3\']``,\n and ``\' 1 2 3 \'.split(None, 1)`` returns ``[\'1\', \'2 3 \']``.\n\nstr.splitlines([keepends])\n\n Return a list of the lines in the string, breaking at line\n boundaries. Line breaks are not included in the resulting list\n unless *keepends* is given and true.\n\nstr.startswith(prefix[, start[, end]])\n\n Return ``True`` if string starts with the *prefix*, otherwise\n return ``False``. *prefix* can also be a tuple of prefixes to look\n for. With optional *start*, test string beginning at that\n position. With optional *end*, stop comparing string at that\n position.\n\n Changed in version 2.5: Accept tuples as *prefix*.\n\nstr.strip([chars])\n\n Return a copy of the string with the leading and trailing\n characters removed. The *chars* argument is a string specifying the\n set of characters to be removed. If omitted or ``None``, the\n *chars* argument defaults to removing whitespace. The *chars*\n argument is not a prefix or suffix; rather, all combinations of its\n values are stripped:\n\n >>> \' spacious \'.strip()\n \'spacious\'\n >>> \'www.example.com\'.strip(\'cmowz.\')\n \'example\'\n\n Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.swapcase()\n\n Return a copy of the string with uppercase characters converted to\n lowercase and vice versa.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.title()\n\n Return a titlecased version of the string where words start with an\n uppercase character and the remaining characters are lowercase.\n\n The algorithm uses a simple language-independent definition of a\n word as groups of consecutive letters. The definition works in\n many contexts but it means that apostrophes in contractions and\n possessives form word boundaries, which may not be the desired\n result:\n\n >>> "they\'re bill\'s friends from the UK".title()\n "They\'Re Bill\'S Friends From The Uk"\n\n A workaround for apostrophes can be constructed using regular\n expressions:\n\n >>> import re\n >>> def titlecase(s):\n return re.sub(r"[A-Za-z]+(\'[A-Za-z]+)?",\n lambda mo: mo.group(0)[0].upper() +\n mo.group(0)[1:].lower(),\n s)\n\n >>> titlecase("they\'re bill\'s friends.")\n "They\'re Bill\'s Friends."\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.translate(table[, deletechars])\n\n Return a copy of the string where all characters occurring in the\n optional argument *deletechars* are removed, and the remaining\n characters have been mapped through the given translation table,\n which must be a string of length 256.\n\n You can use the ``maketrans()`` helper function in the ``string``\n module to create a translation table. For string objects, set the\n *table* argument to ``None`` for translations that only delete\n characters:\n\n >>> \'read this short text\'.translate(None, \'aeiou\')\n \'rd ths shrt txt\'\n\n New in version 2.6: Support for a ``None`` *table* argument.\n\n For Unicode objects, the ``translate()`` method does not accept the\n optional *deletechars* argument. Instead, it returns a copy of the\n *s* where all characters have been mapped through the given\n translation table which must be a mapping of Unicode ordinals to\n Unicode ordinals, Unicode strings or ``None``. Unmapped characters\n are left untouched. Characters mapped to ``None`` are deleted.\n Note, a more flexible approach is to create a custom character\n mapping codec using the ``codecs`` module (see ``encodings.cp1251``\n for an example).\n\nstr.upper()\n\n Return a copy of the string converted to uppercase.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.zfill(width)\n\n Return the numeric string left filled with zeros in a string of\n length *width*. A sign prefix is handled correctly. The original\n string is returned if *width* is less than ``len(s)``.\n\n New in version 2.2.2.\n\nThe following methods are present only on unicode objects:\n\nunicode.isnumeric()\n\n Return ``True`` if there are only numeric characters in S,\n ``False`` otherwise. Numeric characters include digit characters,\n and all characters that have the Unicode numeric value property,\n e.g. U+2155, VULGAR FRACTION ONE FIFTH.\n\nunicode.isdecimal()\n\n Return ``True`` if there are only decimal characters in S,\n ``False`` otherwise. Decimal characters include digit characters,\n and all characters that that can be used to form decimal-radix\n numbers, e.g. U+0660, ARABIC-INDIC DIGIT ZERO.\n\n\nString Formatting Operations\n============================\n\nString and Unicode objects have one unique built-in operation: the\n``%`` operator (modulo). This is also known as the string\n*formatting* or *interpolation* operator. Given ``format % values``\n(where *format* is a string or Unicode object), ``%`` conversion\nspecifications in *format* are replaced with zero or more elements of\n*values*. The effect is similar to the using ``sprintf()`` in the C\nlanguage. If *format* is a Unicode object, or if any of the objects\nbeing converted using the ``%s`` conversion are Unicode objects, the\nresult will also be a Unicode object.\n\nIf *format* requires a single argument, *values* may be a single non-\ntuple object. [4] Otherwise, *values* must be a tuple with exactly\nthe number of items specified by the format string, or a single\nmapping object (for example, a dictionary).\n\nA conversion specifier contains two or more characters and has the\nfollowing components, which must occur in this order:\n\n1. The ``\'%\'`` character, which marks the start of the specifier.\n\n2. Mapping key (optional), consisting of a parenthesised sequence of\n characters (for example, ``(somename)``).\n\n3. Conversion flags (optional), which affect the result of some\n conversion types.\n\n4. Minimum field width (optional). If specified as an ``\'*\'``\n (asterisk), the actual width is read from the next element of the\n tuple in *values*, and the object to convert comes after the\n minimum field width and optional precision.\n\n5. Precision (optional), given as a ``\'.\'`` (dot) followed by the\n precision. If specified as ``\'*\'`` (an asterisk), the actual width\n is read from the next element of the tuple in *values*, and the\n value to convert comes after the precision.\n\n6. Length modifier (optional).\n\n7. Conversion type.\n\nWhen the right argument is a dictionary (or other mapping type), then\nthe formats in the string *must* include a parenthesised mapping key\ninto that dictionary inserted immediately after the ``\'%\'`` character.\nThe mapping key selects the value to be formatted from the mapping.\nFor example:\n\n>>> print \'%(language)s has %(number)03d quote types.\' % \\\n... {"language": "Python", "number": 2}\nPython has 002 quote types.\n\nIn this case no ``*`` specifiers may occur in a format (since they\nrequire a sequential parameter list).\n\nThe conversion flag characters are:\n\n+-----------+-----------------------------------------------------------------------+\n| Flag | Meaning |\n+===========+=======================================================================+\n| ``\'#\'`` | The value conversion will use the "alternate form" (where defined |\n| | below). |\n+-----------+-----------------------------------------------------------------------+\n| ``\'0\'`` | The conversion will be zero padded for numeric values. |\n+-----------+-----------------------------------------------------------------------+\n| ``\'-\'`` | The converted value is left adjusted (overrides the ``\'0\'`` |\n| | conversion if both are given). |\n+-----------+-----------------------------------------------------------------------+\n| ``\' \'`` | (a space) A blank should be left before a positive number (or empty |\n| | string) produced by a signed conversion. |\n+-----------+-----------------------------------------------------------------------+\n| ``\'+\'`` | A sign character (``\'+\'`` or ``\'-\'``) will precede the conversion |\n| | (overrides a "space" flag). |\n+-----------+-----------------------------------------------------------------------+\n\nA length modifier (``h``, ``l``, or ``L``) may be present, but is\nignored as it is not necessary for Python -- so e.g. ``%ld`` is\nidentical to ``%d``.\n\nThe conversion types are:\n\n+--------------+-------------------------------------------------------+---------+\n| Conversion | Meaning | Notes |\n+==============+=======================================================+=========+\n| ``\'d\'`` | Signed integer decimal. | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'i\'`` | Signed integer decimal. | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'o\'`` | Signed octal value. | (1) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'u\'`` | Obsolete type -- it is identical to ``\'d\'``. | (7) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'x\'`` | Signed hexadecimal (lowercase). | (2) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'X\'`` | Signed hexadecimal (uppercase). | (2) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'e\'`` | Floating point exponential format (lowercase). | (3) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'E\'`` | Floating point exponential format (uppercase). | (3) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'f\'`` | Floating point decimal format. | (3) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'F\'`` | Floating point decimal format. | (3) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'g\'`` | Floating point format. Uses lowercase exponential | (4) |\n| | format if exponent is less than -4 or not less than | |\n| | precision, decimal format otherwise. | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'G\'`` | Floating point format. Uses uppercase exponential | (4) |\n| | format if exponent is less than -4 or not less than | |\n| | precision, decimal format otherwise. | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'c\'`` | Single character (accepts integer or single character | |\n| | string). | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'r\'`` | String (converts any Python object using ``repr()``). | (5) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'s\'`` | String (converts any Python object using ``str()``). | (6) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'%\'`` | No argument is converted, results in a ``\'%\'`` | |\n| | character in the result. | |\n+--------------+-------------------------------------------------------+---------+\n\nNotes:\n\n1. The alternate form causes a leading zero (``\'0\'``) to be inserted\n between left-hand padding and the formatting of the number if the\n leading character of the result is not already a zero.\n\n2. The alternate form causes a leading ``\'0x\'`` or ``\'0X\'`` (depending\n on whether the ``\'x\'`` or ``\'X\'`` format was used) to be inserted\n between left-hand padding and the formatting of the number if the\n leading character of the result is not already a zero.\n\n3. The alternate form causes the result to always contain a decimal\n point, even if no digits follow it.\n\n The precision determines the number of digits after the decimal\n point and defaults to 6.\n\n4. The alternate form causes the result to always contain a decimal\n point, and trailing zeroes are not removed as they would otherwise\n be.\n\n The precision determines the number of significant digits before\n and after the decimal point and defaults to 6.\n\n5. The ``%r`` conversion was added in Python 2.0.\n\n The precision determines the maximal number of characters used.\n\n6. If the object or format provided is a ``unicode`` string, the\n resulting string will also be ``unicode``.\n\n The precision determines the maximal number of characters used.\n\n7. See **PEP 237**.\n\nSince Python strings have an explicit length, ``%s`` conversions do\nnot assume that ``\'\\0\'`` is the end of the string.\n\nChanged in version 2.7: ``%f`` conversions for numbers whose absolute\nvalue is over 1e50 are no longer replaced by ``%g`` conversions.\n\nAdditional string operations are defined in standard modules\n``string`` and ``re``.\n\n\nXRange Type\n===========\n\nThe ``xrange`` type is an immutable sequence which is commonly used\nfor looping. The advantage of the ``xrange`` type is that an\n``xrange`` object will always take the same amount of memory, no\nmatter the size of the range it represents. There are no consistent\nperformance advantages.\n\nXRange objects have very little behavior: they only support indexing,\niteration, and the ``len()`` function.\n\n\nMutable Sequence Types\n======================\n\nList and ``bytearray`` objects support additional operations that\nallow in-place modification of the object. Other mutable sequence\ntypes (when added to the language) should also support these\noperations. Strings and tuples are immutable sequence types: such\nobjects cannot be modified once created. The following operations are\ndefined on mutable sequence types (where *x* is an arbitrary object):\n\n+--------------------------------+----------------------------------+-----------------------+\n| Operation | Result | Notes |\n+================================+==================================+=======================+\n| ``s[i] = x`` | item *i* of *s* is replaced by | |\n| | *x* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s[i:j] = t`` | slice of *s* from *i* to *j* is | |\n| | replaced by the contents of the | |\n| | iterable *t* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``del s[i:j]`` | same as ``s[i:j] = []`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s[i:j:k] = t`` | the elements of ``s[i:j:k]`` are | (1) |\n| | replaced by those of *t* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``del s[i:j:k]`` | removes the elements of | |\n| | ``s[i:j:k]`` from the list | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.append(x)`` | same as ``s[len(s):len(s)] = | (2) |\n| | [x]`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.extend(x)`` | same as ``s[len(s):len(s)] = x`` | (3) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.count(x)`` | return number of *i*\'s for which | |\n| | ``s[i] == x`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.index(x[, i[, j]])`` | return smallest *k* such that | (4) |\n| | ``s[k] == x`` and ``i <= k < j`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.insert(i, x)`` | same as ``s[i:i] = [x]`` | (5) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.pop([i])`` | same as ``x = s[i]; del s[i]; | (6) |\n| | return x`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.remove(x)`` | same as ``del s[s.index(x)]`` | (4) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.reverse()`` | reverses the items of *s* in | (7) |\n| | place | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.sort([cmp[, key[, | sort the items of *s* in place | (7)(8)(9)(10) |\n| reverse]]])`` | | |\n+--------------------------------+----------------------------------+-----------------------+\n\nNotes:\n\n1. *t* must have the same length as the slice it is replacing.\n\n2. The C implementation of Python has historically accepted multiple\n parameters and implicitly joined them into a tuple; this no longer\n works in Python 2.0. Use of this misfeature has been deprecated\n since Python 1.4.\n\n3. *x* can be any iterable object.\n\n4. Raises ``ValueError`` when *x* is not found in *s*. When a negative\n index is passed as the second or third parameter to the ``index()``\n method, the list length is added, as for slice indices. If it is\n still negative, it is truncated to zero, as for slice indices.\n\n Changed in version 2.3: Previously, ``index()`` didn\'t have\n arguments for specifying start and stop positions.\n\n5. When a negative index is passed as the first parameter to the\n ``insert()`` method, the list length is added, as for slice\n indices. If it is still negative, it is truncated to zero, as for\n slice indices.\n\n Changed in version 2.3: Previously, all negative indices were\n truncated to zero.\n\n6. The ``pop()`` method is only supported by the list and array types.\n The optional argument *i* defaults to ``-1``, so that by default\n the last item is removed and returned.\n\n7. The ``sort()`` and ``reverse()`` methods modify the list in place\n for economy of space when sorting or reversing a large list. To\n remind you that they operate by side effect, they don\'t return the\n sorted or reversed list.\n\n8. The ``sort()`` method takes optional arguments for controlling the\n comparisons.\n\n *cmp* specifies a custom comparison function of two arguments (list\n items) which should return a negative, zero or positive number\n depending on whether the first argument is considered smaller than,\n equal to, or larger than the second argument: ``cmp=lambda x,y:\n cmp(x.lower(), y.lower())``. The default value is ``None``.\n\n *key* specifies a function of one argument that is used to extract\n a comparison key from each list element: ``key=str.lower``. The\n default value is ``None``.\n\n *reverse* is a boolean value. If set to ``True``, then the list\n elements are sorted as if each comparison were reversed.\n\n In general, the *key* and *reverse* conversion processes are much\n faster than specifying an equivalent *cmp* function. This is\n because *cmp* is called multiple times for each list element while\n *key* and *reverse* touch each element only once. Use\n ``functools.cmp_to_key()`` to convert an old-style *cmp* function\n to a *key* function.\n\n Changed in version 2.3: Support for ``None`` as an equivalent to\n omitting *cmp* was added.\n\n Changed in version 2.4: Support for *key* and *reverse* was added.\n\n9. Starting with Python 2.3, the ``sort()`` method is guaranteed to be\n stable. A sort is stable if it guarantees not to change the\n relative order of elements that compare equal --- this is helpful\n for sorting in multiple passes (for example, sort by department,\n then by salary grade).\n\n10. **CPython implementation detail:** While a list is being sorted,\n the effect of attempting to mutate, or even inspect, the list is\n undefined. The C implementation of Python 2.3 and newer makes the\n list appear empty for the duration, and raises ``ValueError`` if\n it can detect that the list has been mutated during a sort.\n', + 'typesseq-mutable': u"\nMutable Sequence Types\n**********************\n\nList and ``bytearray`` objects support additional operations that\nallow in-place modification of the object. Other mutable sequence\ntypes (when added to the language) should also support these\noperations. Strings and tuples are immutable sequence types: such\nobjects cannot be modified once created. The following operations are\ndefined on mutable sequence types (where *x* is an arbitrary object):\n\n+--------------------------------+----------------------------------+-----------------------+\n| Operation | Result | Notes |\n+================================+==================================+=======================+\n| ``s[i] = x`` | item *i* of *s* is replaced by | |\n| | *x* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s[i:j] = t`` | slice of *s* from *i* to *j* is | |\n| | replaced by the contents of the | |\n| | iterable *t* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``del s[i:j]`` | same as ``s[i:j] = []`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s[i:j:k] = t`` | the elements of ``s[i:j:k]`` are | (1) |\n| | replaced by those of *t* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``del s[i:j:k]`` | removes the elements of | |\n| | ``s[i:j:k]`` from the list | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.append(x)`` | same as ``s[len(s):len(s)] = | (2) |\n| | [x]`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.extend(x)`` | same as ``s[len(s):len(s)] = x`` | (3) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.count(x)`` | return number of *i*'s for which | |\n| | ``s[i] == x`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.index(x[, i[, j]])`` | return smallest *k* such that | (4) |\n| | ``s[k] == x`` and ``i <= k < j`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.insert(i, x)`` | same as ``s[i:i] = [x]`` | (5) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.pop([i])`` | same as ``x = s[i]; del s[i]; | (6) |\n| | return x`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.remove(x)`` | same as ``del s[s.index(x)]`` | (4) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.reverse()`` | reverses the items of *s* in | (7) |\n| | place | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.sort([cmp[, key[, | sort the items of *s* in place | (7)(8)(9)(10) |\n| reverse]]])`` | | |\n+--------------------------------+----------------------------------+-----------------------+\n\nNotes:\n\n1. *t* must have the same length as the slice it is replacing.\n\n2. The C implementation of Python has historically accepted multiple\n parameters and implicitly joined them into a tuple; this no longer\n works in Python 2.0. Use of this misfeature has been deprecated\n since Python 1.4.\n\n3. *x* can be any iterable object.\n\n4. Raises ``ValueError`` when *x* is not found in *s*. When a negative\n index is passed as the second or third parameter to the ``index()``\n method, the list length is added, as for slice indices. If it is\n still negative, it is truncated to zero, as for slice indices.\n\n Changed in version 2.3: Previously, ``index()`` didn't have\n arguments for specifying start and stop positions.\n\n5. When a negative index is passed as the first parameter to the\n ``insert()`` method, the list length is added, as for slice\n indices. If it is still negative, it is truncated to zero, as for\n slice indices.\n\n Changed in version 2.3: Previously, all negative indices were\n truncated to zero.\n\n6. The ``pop()`` method is only supported by the list and array types.\n The optional argument *i* defaults to ``-1``, so that by default\n the last item is removed and returned.\n\n7. The ``sort()`` and ``reverse()`` methods modify the list in place\n for economy of space when sorting or reversing a large list. To\n remind you that they operate by side effect, they don't return the\n sorted or reversed list.\n\n8. The ``sort()`` method takes optional arguments for controlling the\n comparisons.\n\n *cmp* specifies a custom comparison function of two arguments (list\n items) which should return a negative, zero or positive number\n depending on whether the first argument is considered smaller than,\n equal to, or larger than the second argument: ``cmp=lambda x,y:\n cmp(x.lower(), y.lower())``. The default value is ``None``.\n\n *key* specifies a function of one argument that is used to extract\n a comparison key from each list element: ``key=str.lower``. The\n default value is ``None``.\n\n *reverse* is a boolean value. If set to ``True``, then the list\n elements are sorted as if each comparison were reversed.\n\n In general, the *key* and *reverse* conversion processes are much\n faster than specifying an equivalent *cmp* function. This is\n because *cmp* is called multiple times for each list element while\n *key* and *reverse* touch each element only once. Use\n ``functools.cmp_to_key()`` to convert an old-style *cmp* function\n to a *key* function.\n\n Changed in version 2.3: Support for ``None`` as an equivalent to\n omitting *cmp* was added.\n\n Changed in version 2.4: Support for *key* and *reverse* was added.\n\n9. Starting with Python 2.3, the ``sort()`` method is guaranteed to be\n stable. A sort is stable if it guarantees not to change the\n relative order of elements that compare equal --- this is helpful\n for sorting in multiple passes (for example, sort by department,\n then by salary grade).\n\n10. **CPython implementation detail:** While a list is being sorted,\n the effect of attempting to mutate, or even inspect, the list is\n undefined. The C implementation of Python 2.3 and newer makes the\n list appear empty for the duration, and raises ``ValueError`` if\n it can detect that the list has been mutated during a sort.\n", 'unary': u'\nUnary arithmetic and bitwise operations\n***************************************\n\nAll unary arithmetic and bitwise operations have the same priority:\n\n u_expr ::= power | "-" u_expr | "+" u_expr | "~" u_expr\n\nThe unary ``-`` (minus) operator yields the negation of its numeric\nargument.\n\nThe unary ``+`` (plus) operator yields its numeric argument unchanged.\n\nThe unary ``~`` (invert) operator yields the bitwise inversion of its\nplain or long integer argument. The bitwise inversion of ``x`` is\ndefined as ``-(x+1)``. It only applies to integral numbers.\n\nIn all three cases, if the argument does not have the proper type, a\n``TypeError`` exception is raised.\n', 'while': u'\nThe ``while`` statement\n***********************\n\nThe ``while`` statement is used for repeated execution as long as an\nexpression is true:\n\n while_stmt ::= "while" expression ":" suite\n ["else" ":" suite]\n\nThis repeatedly tests the expression and, if it is true, executes the\nfirst suite; if the expression is false (which may be the first time\nit is tested) the suite of the ``else`` clause, if present, is\nexecuted and the loop terminates.\n\nA ``break`` statement executed in the first suite terminates the loop\nwithout executing the ``else`` clause\'s suite. A ``continue``\nstatement executed in the first suite skips the rest of the suite and\ngoes back to testing the expression.\n', - 'with': u'\nThe ``with`` statement\n**********************\n\nNew in version 2.5.\n\nThe ``with`` statement is used to wrap the execution of a block with\nmethods defined by a context manager (see section *With Statement\nContext Managers*). This allows common\n``try``...``except``...``finally`` usage patterns to be encapsulated\nfor convenient reuse.\n\n with_stmt ::= "with" with_item ("," with_item)* ":" suite\n with_item ::= expression ["as" target]\n\nThe execution of the ``with`` statement with one "item" proceeds as\nfollows:\n\n1. The context expression is evaluated to obtain a context manager.\n\n2. The context manager\'s ``__exit__()`` is loaded for later use.\n\n3. The context manager\'s ``__enter__()`` method is invoked.\n\n4. If a target was included in the ``with`` statement, the return\n value from ``__enter__()`` is assigned to it.\n\n Note: The ``with`` statement guarantees that if the ``__enter__()``\n method returns without an error, then ``__exit__()`` will always\n be called. Thus, if an error occurs during the assignment to the\n target list, it will be treated the same as an error occurring\n within the suite would be. See step 6 below.\n\n5. The suite is executed.\n\n6. The context manager\'s ``__exit__()`` method is invoked. If an\n exception caused the suite to be exited, its type, value, and\n traceback are passed as arguments to ``__exit__()``. Otherwise,\n three ``None`` arguments are supplied.\n\n If the suite was exited due to an exception, and the return value\n from the ``__exit__()`` method was false, the exception is\n reraised. If the return value was true, the exception is\n suppressed, and execution continues with the statement following\n the ``with`` statement.\n\n If the suite was exited for any reason other than an exception, the\n return value from ``__exit__()`` is ignored, and execution proceeds\n at the normal location for the kind of exit that was taken.\n\nWith more than one item, the context managers are processed as if\nmultiple ``with`` statements were nested:\n\n with A() as a, B() as b:\n suite\n\nis equivalent to\n\n with A() as a:\n with B() as b:\n suite\n\nNote: In Python 2.5, the ``with`` statement is only allowed when the\n ``with_statement`` feature has been enabled. It is always enabled\n in Python 2.6.\n\nChanged in version 2.7: Support for multiple context expressions.\n\nSee also:\n\n **PEP 0343** - The "with" statement\n The specification, background, and examples for the Python\n ``with`` statement.\n', + 'with': u'\nThe ``with`` statement\n**********************\n\nNew in version 2.5.\n\nThe ``with`` statement is used to wrap the execution of a block with\nmethods defined by a context manager (see section *With Statement\nContext Managers*). This allows common\n``try``...``except``...``finally`` usage patterns to be encapsulated\nfor convenient reuse.\n\n with_stmt ::= "with" with_item ("," with_item)* ":" suite\n with_item ::= expression ["as" target]\n\nThe execution of the ``with`` statement with one "item" proceeds as\nfollows:\n\n1. The context expression (the expression given in the **with_item**)\n is evaluated to obtain a context manager.\n\n2. The context manager\'s ``__exit__()`` is loaded for later use.\n\n3. The context manager\'s ``__enter__()`` method is invoked.\n\n4. If a target was included in the ``with`` statement, the return\n value from ``__enter__()`` is assigned to it.\n\n Note: The ``with`` statement guarantees that if the ``__enter__()``\n method returns without an error, then ``__exit__()`` will always\n be called. Thus, if an error occurs during the assignment to the\n target list, it will be treated the same as an error occurring\n within the suite would be. See step 6 below.\n\n5. The suite is executed.\n\n6. The context manager\'s ``__exit__()`` method is invoked. If an\n exception caused the suite to be exited, its type, value, and\n traceback are passed as arguments to ``__exit__()``. Otherwise,\n three ``None`` arguments are supplied.\n\n If the suite was exited due to an exception, and the return value\n from the ``__exit__()`` method was false, the exception is\n reraised. If the return value was true, the exception is\n suppressed, and execution continues with the statement following\n the ``with`` statement.\n\n If the suite was exited for any reason other than an exception, the\n return value from ``__exit__()`` is ignored, and execution proceeds\n at the normal location for the kind of exit that was taken.\n\nWith more than one item, the context managers are processed as if\nmultiple ``with`` statements were nested:\n\n with A() as a, B() as b:\n suite\n\nis equivalent to\n\n with A() as a:\n with B() as b:\n suite\n\nNote: In Python 2.5, the ``with`` statement is only allowed when the\n ``with_statement`` feature has been enabled. It is always enabled\n in Python 2.6.\n\nChanged in version 2.7: Support for multiple context expressions.\n\nSee also:\n\n **PEP 0343** - The "with" statement\n The specification, background, and examples for the Python\n ``with`` statement.\n', 'yield': u'\nThe ``yield`` statement\n***********************\n\n yield_stmt ::= yield_expression\n\nThe ``yield`` statement is only used when defining a generator\nfunction, and is only used in the body of the generator function.\nUsing a ``yield`` statement in a function definition is sufficient to\ncause that definition to create a generator function instead of a\nnormal function.\n\nWhen a generator function is called, it returns an iterator known as a\ngenerator iterator, or more commonly, a generator. The body of the\ngenerator function is executed by calling the generator\'s ``next()``\nmethod repeatedly until it raises an exception.\n\nWhen a ``yield`` statement is executed, the state of the generator is\nfrozen and the value of **expression_list** is returned to\n``next()``\'s caller. By "frozen" we mean that all local state is\nretained, including the current bindings of local variables, the\ninstruction pointer, and the internal evaluation stack: enough\ninformation is saved so that the next time ``next()`` is invoked, the\nfunction can proceed exactly as if the ``yield`` statement were just\nanother external call.\n\nAs of Python version 2.5, the ``yield`` statement is now allowed in\nthe ``try`` clause of a ``try`` ... ``finally`` construct. If the\ngenerator is not resumed before it is finalized (by reaching a zero\nreference count or by being garbage collected), the generator-\niterator\'s ``close()`` method will be called, allowing any pending\n``finally`` clauses to execute.\n\nNote: In Python 2.2, the ``yield`` statement was only allowed when the\n ``generators`` feature has been enabled. This ``__future__`` import\n statement was used to enable the feature:\n\n from __future__ import generators\n\nSee also:\n\n **PEP 0255** - Simple Generators\n The proposal for adding generators and the ``yield`` statement\n to Python.\n\n **PEP 0342** - Coroutines via Enhanced Generators\n The proposal that, among other generator enhancements, proposed\n allowing ``yield`` to appear inside a ``try`` ... ``finally``\n block.\n'} diff --git a/lib-python/2.7/random.py b/lib-python/2.7/random.py --- a/lib-python/2.7/random.py +++ b/lib-python/2.7/random.py @@ -317,7 +317,7 @@ n = len(population) if not 0 <= k <= n: - raise ValueError, "sample larger than population" + raise ValueError("sample larger than population") random = self.random _int = int result = [None] * k @@ -490,6 +490,12 @@ Conditions on the parameters are alpha > 0 and beta > 0. + The probability distribution function is: + + x ** (alpha - 1) * math.exp(-x / beta) + pdf(x) = -------------------------------------- + math.gamma(alpha) * beta ** alpha + """ # alpha > 0, beta > 0, mean is alpha*beta, variance is alpha*beta**2 @@ -592,7 +598,7 @@ ## -------------------- beta -------------------- ## See -## http://sourceforge.net/bugs/?func=detailbug&bug_id=130030&group_id=5470 +## http://mail.python.org/pipermail/python-bugs-list/2001-January/003752.html ## for Ivan Frohne's insightful analysis of why the original implementation: ## ## def betavariate(self, alpha, beta): diff --git a/lib-python/2.7/re.py b/lib-python/2.7/re.py --- a/lib-python/2.7/re.py +++ b/lib-python/2.7/re.py @@ -207,8 +207,7 @@ "Escape all non-alphanumeric characters in pattern." s = list(pattern) alphanum = _alphanum - for i in range(len(pattern)): - c = pattern[i] + for i, c in enumerate(pattern): if c not in alphanum: if c == "\000": s[i] = "\\000" diff --git a/lib-python/2.7/shutil.py b/lib-python/2.7/shutil.py --- a/lib-python/2.7/shutil.py +++ b/lib-python/2.7/shutil.py @@ -277,6 +277,12 @@ """ real_dst = dst if os.path.isdir(dst): + if _samefile(src, dst): + # We might be on a case insensitive filesystem, + # perform the rename anyway. + os.rename(src, dst) + return + real_dst = os.path.join(dst, _basename(src)) if os.path.exists(real_dst): raise Error, "Destination path '%s' already exists" % real_dst @@ -336,7 +342,7 @@ archive that is being built. If not provided, the current owner and group will be used. - The output tar file will be named 'base_dir' + ".tar", possibly plus + The output tar file will be named 'base_name' + ".tar", possibly plus the appropriate compression extension (".gz", or ".bz2"). Returns the output filename. @@ -406,7 +412,7 @@ def _make_zipfile(base_name, base_dir, verbose=0, dry_run=0, logger=None): """Create a zip file from all the files under 'base_dir'. - The output zip file will be named 'base_dir' + ".zip". Uses either the + The output zip file will be named 'base_name' + ".zip". Uses either the "zipfile" Python module (if available) or the InfoZIP "zip" utility (if installed and found on the default search path). If neither tool is available, raises ExecError. Returns the name of the output zip diff --git a/lib-python/2.7/site.py b/lib-python/2.7/site.py --- a/lib-python/2.7/site.py +++ b/lib-python/2.7/site.py @@ -61,6 +61,7 @@ import sys import os import __builtin__ +import traceback # Prefixes for site-packages; add additional prefixes like /usr/local here PREFIXES = [sys.prefix, sys.exec_prefix] @@ -155,17 +156,26 @@ except IOError: return with f: - for line in f: + for n, line in enumerate(f): if line.startswith("#"): continue - if line.startswith(("import ", "import\t")): - exec line - continue - line = line.rstrip() - dir, dircase = makepath(sitedir, line) - if not dircase in known_paths and os.path.exists(dir): - sys.path.append(dir) - known_paths.add(dircase) + try: + if line.startswith(("import ", "import\t")): + exec line + continue + line = line.rstrip() + dir, dircase = makepath(sitedir, line) + if not dircase in known_paths and os.path.exists(dir): + sys.path.append(dir) + known_paths.add(dircase) + except Exception as err: + print >>sys.stderr, "Error processing line {:d} of {}:\n".format( + n+1, fullname) + for record in traceback.format_exception(*sys.exc_info()): + for line in record.splitlines(): + print >>sys.stderr, ' '+line + print >>sys.stderr, "\nRemainder of file ignored" + break if reset: known_paths = None return known_paths diff --git a/lib-python/2.7/smtplib.py b/lib-python/2.7/smtplib.py --- a/lib-python/2.7/smtplib.py +++ b/lib-python/2.7/smtplib.py @@ -49,17 +49,18 @@ from email.base64mime import encode as encode_base64 from sys import stderr -__all__ = ["SMTPException","SMTPServerDisconnected","SMTPResponseException", - "SMTPSenderRefused","SMTPRecipientsRefused","SMTPDataError", - "SMTPConnectError","SMTPHeloError","SMTPAuthenticationError", - "quoteaddr","quotedata","SMTP"] +__all__ = ["SMTPException", "SMTPServerDisconnected", "SMTPResponseException", + "SMTPSenderRefused", "SMTPRecipientsRefused", "SMTPDataError", + "SMTPConnectError", "SMTPHeloError", "SMTPAuthenticationError", + "quoteaddr", "quotedata", "SMTP"] SMTP_PORT = 25 SMTP_SSL_PORT = 465 -CRLF="\r\n" +CRLF = "\r\n" OLDSTYLE_AUTH = re.compile(r"auth=(.*)", re.I) + # Exception classes used by this module. class SMTPException(Exception): """Base class for all exceptions raised by this module.""" @@ -109,7 +110,7 @@ def __init__(self, recipients): self.recipients = recipients - self.args = ( recipients,) + self.args = (recipients,) class SMTPDataError(SMTPResponseException): @@ -128,6 +129,7 @@ combination provided. """ + def quoteaddr(addr): """Quote a subset of the email addresses defined by RFC 821. @@ -138,7 +140,7 @@ m = email.utils.parseaddr(addr)[1] except AttributeError: pass - if m == (None, None): # Indicates parse failure or AttributeError + if m == (None, None): # Indicates parse failure or AttributeError # something weird here.. punt -ddm return "<%s>" % addr elif m is None: @@ -175,7 +177,8 @@ chr = None while chr != "\n": chr = self.sslobj.read(1) - if not chr: break + if not chr: + break str += chr return str @@ -219,6 +222,7 @@ ehlo_msg = "ehlo" ehlo_resp = None does_esmtp = 0 + default_port = SMTP_PORT def __init__(self, host='', port=0, local_hostname=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT): @@ -234,7 +238,6 @@ """ self.timeout = timeout self.esmtp_features = {} - self.default_port = SMTP_PORT if host: (code, msg) = self.connect(host, port) if code != 220: @@ -269,10 +272,11 @@ def _get_socket(self, port, host, timeout): # This makes it simpler for SMTP_SSL to use the SMTP connect code # and just alter the socket connection bit. - if self.debuglevel > 0: print>>stderr, 'connect:', (host, port) + if self.debuglevel > 0: + print>>stderr, 'connect:', (host, port) return socket.create_connection((port, host), timeout) - def connect(self, host='localhost', port = 0): + def connect(self, host='localhost', port=0): """Connect to a host on a given port. If the hostname ends with a colon (`:') followed by a number, and @@ -286,20 +290,25 @@ if not port and (host.find(':') == host.rfind(':')): i = host.rfind(':') if i >= 0: - host, port = host[:i], host[i+1:] - try: port = int(port) + host, port = host[:i], host[i + 1:] + try: + port = int(port) except ValueError: raise socket.error, "nonnumeric port" - if not port: port = self.default_port - if self.debuglevel > 0: print>>stderr, 'connect:', (host, port) + if not port: + port = self.default_port + if self.debuglevel > 0: + print>>stderr, 'connect:', (host, port) self.sock = self._get_socket(host, port, self.timeout) (code, msg) = self.getreply() - if self.debuglevel > 0: print>>stderr, "connect:", msg + if self.debuglevel > 0: + print>>stderr, "connect:", msg return (code, msg) def send(self, str): """Send `str' to the server.""" - if self.debuglevel > 0: print>>stderr, 'send:', repr(str) + if self.debuglevel > 0: + print>>stderr, 'send:', repr(str) if hasattr(self, 'sock') and self.sock: try: self.sock.sendall(str) @@ -330,7 +339,7 @@ Raises SMTPServerDisconnected if end-of-file is reached. """ - resp=[] + resp = [] if self.file is None: self.file = self.sock.makefile('rb') while 1: @@ -341,9 +350,10 @@ if line == '': self.close() raise SMTPServerDisconnected("Connection unexpectedly closed") - if self.debuglevel > 0: print>>stderr, 'reply:', repr(line) + if self.debuglevel > 0: + print>>stderr, 'reply:', repr(line) resp.append(line[4:].strip()) - code=line[:3] + code = line[:3] # Check that the error code is syntactically correct. # Don't attempt to read a continuation line if it is broken. try: @@ -352,17 +362,17 @@ errcode = -1 break # Check if multiline response. - if line[3:4]!="-": + if line[3:4] != "-": break errmsg = "\n".join(resp) if self.debuglevel > 0: - print>>stderr, 'reply: retcode (%s); Msg: %s' % (errcode,errmsg) + print>>stderr, 'reply: retcode (%s); Msg: %s' % (errcode, errmsg) return errcode, errmsg def docmd(self, cmd, args=""): """Send a command, and return its response code.""" - self.putcmd(cmd,args) + self.putcmd(cmd, args) return self.getreply() # std smtp commands @@ -372,9 +382,9 @@ host. """ self.putcmd("helo", name or self.local_hostname) - (code,msg)=self.getreply() - self.helo_resp=msg - return (code,msg) + (code, msg) = self.getreply() + self.helo_resp = msg + return (code, msg) def ehlo(self, name=''): """ SMTP 'ehlo' command. @@ -383,19 +393,19 @@ """ self.esmtp_features = {} self.putcmd(self.ehlo_msg, name or self.local_hostname) - (code,msg)=self.getreply() + (code, msg) = self.getreply() # According to RFC1869 some (badly written) # MTA's will disconnect on an ehlo. Toss an exception if # that happens -ddm if code == -1 and len(msg) == 0: self.close() raise SMTPServerDisconnected("Server not connected") - self.ehlo_resp=msg + self.ehlo_resp = msg if code != 250: - return (code,msg) - self.does_esmtp=1 + return (code, msg) + self.does_esmtp = 1 #parse the ehlo response -ddm - resp=self.ehlo_resp.split('\n') + resp = self.ehlo_resp.split('\n') del resp[0] for each in resp: # To be able to communicate with as many SMTP servers as possible, @@ -415,16 +425,16 @@ # It's actually stricter, in that only spaces are allowed between # parameters, but were not going to check for that here. Note # that the space isn't present if there are no parameters. - m=re.match(r'(?P<feature>[A-Za-z0-9][A-Za-z0-9\-]*) ?',each) + m = re.match(r'(?P<feature>[A-Za-z0-9][A-Za-z0-9\-]*) ?', each) if m: - feature=m.group("feature").lower() - params=m.string[m.end("feature"):].strip() + feature = m.group("feature").lower() + params = m.string[m.end("feature"):].strip() if feature == "auth": self.esmtp_features[feature] = self.esmtp_features.get(feature, "") \ + " " + params else: - self.esmtp_features[feature]=params - return (code,msg) + self.esmtp_features[feature] = params + return (code, msg) def has_extn(self, opt): """Does the server support a given SMTP service extension?""" @@ -444,23 +454,23 @@ """SMTP 'noop' command -- doesn't do anything :>""" return self.docmd("noop") - def mail(self,sender,options=[]): + def mail(self, sender, options=[]): """SMTP 'mail' command -- begins mail xfer session.""" optionlist = '' if options and self.does_esmtp: optionlist = ' ' + ' '.join(options) - self.putcmd("mail", "FROM:%s%s" % (quoteaddr(sender) ,optionlist)) + self.putcmd("mail", "FROM:%s%s" % (quoteaddr(sender), optionlist)) return self.getreply() - def rcpt(self,recip,options=[]): + def rcpt(self, recip, options=[]): """SMTP 'rcpt' command -- indicates 1 recipient for this mail.""" optionlist = '' if options and self.does_esmtp: optionlist = ' ' + ' '.join(options) - self.putcmd("rcpt","TO:%s%s" % (quoteaddr(recip),optionlist)) + self.putcmd("rcpt", "TO:%s%s" % (quoteaddr(recip), optionlist)) return self.getreply() - def data(self,msg): + def data(self, msg): """SMTP 'DATA' command -- sends message data to server. Automatically quotes lines beginning with a period per rfc821. @@ -469,26 +479,28 @@ response code received when the all data is sent. """ self.putcmd("data") - (code,repl)=self.getreply() - if self.debuglevel >0 : print>>stderr, "data:", (code,repl) + (code, repl) = self.getreply() + if self.debuglevel > 0: + print>>stderr, "data:", (code, repl) if code != 354: - raise SMTPDataError(code,repl) + raise SMTPDataError(code, repl) else: q = quotedata(msg) if q[-2:] != CRLF: q = q + CRLF q = q + "." + CRLF self.send(q) - (code,msg)=self.getreply() - if self.debuglevel >0 : print>>stderr, "data:", (code,msg) - return (code,msg) + (code, msg) = self.getreply() + if self.debuglevel > 0: + print>>stderr, "data:", (code, msg) + return (code, msg) def verify(self, address): """SMTP 'verify' command -- checks for address validity.""" self.putcmd("vrfy", quoteaddr(address)) return self.getreply() # a.k.a. - vrfy=verify + vrfy = verify def expn(self, address): """SMTP 'expn' command -- expands a mailing list.""" @@ -592,7 +604,7 @@ raise SMTPAuthenticationError(code, resp) return (code, resp) - def starttls(self, keyfile = None, certfile = None): + def starttls(self, keyfile=None, certfile=None): """Puts the connection to the SMTP server into TLS mode. If there has been no previous EHLO or HELO command this session, this @@ -695,22 +707,22 @@ for option in mail_options: esmtp_opts.append(option) - (code,resp) = self.mail(from_addr, esmtp_opts) + (code, resp) = self.mail(from_addr, esmtp_opts) if code != 250: self.rset() raise SMTPSenderRefused(code, resp, from_addr) - senderrs={} + senderrs = {} if isinstance(to_addrs, basestring): to_addrs = [to_addrs] for each in to_addrs: - (code,resp)=self.rcpt(each, rcpt_options) + (code, resp) = self.rcpt(each, rcpt_options) if (code != 250) and (code != 251): - senderrs[each]=(code,resp) - if len(senderrs)==len(to_addrs): + senderrs[each] = (code, resp) + if len(senderrs) == len(to_addrs): # the server refused all our recipients self.rset() raise SMTPRecipientsRefused(senderrs) - (code,resp) = self.data(msg) + (code, resp) = self.data(msg) if code != 250: self.rset() raise SMTPDataError(code, resp) @@ -744,16 +756,19 @@ are also optional - they can contain a PEM formatted private key and certificate chain file for the SSL connection. """ + + default_port = SMTP_SSL_PORT + def __init__(self, host='', port=0, local_hostname=None, keyfile=None, certfile=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT): self.keyfile = keyfile self.certfile = certfile SMTP.__init__(self, host, port, local_hostname, timeout) - self.default_port = SMTP_SSL_PORT def _get_socket(self, host, port, timeout): - if self.debuglevel > 0: print>>stderr, 'connect:', (host, port) + if self.debuglevel > 0: + print>>stderr, 'connect:', (host, port) new_socket = socket.create_connection((host, port), timeout) new_socket = ssl.wrap_socket(new_socket, self.keyfile, self.certfile) self.file = SSLFakeFile(new_socket) @@ -781,11 +796,11 @@ ehlo_msg = "lhlo" - def __init__(self, host = '', port = LMTP_PORT, local_hostname = None): + def __init__(self, host='', port=LMTP_PORT, local_hostname=None): """Initialize a new instance.""" SMTP.__init__(self, host, port, local_hostname) - def connect(self, host = 'localhost', port = 0): + def connect(self, host='localhost', port=0): """Connect to the LMTP daemon, on either a Unix or a TCP socket.""" if host[0] != '/': return SMTP.connect(self, host, port) @@ -795,13 +810,15 @@ self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) self.sock.connect(host) except socket.error, msg: - if self.debuglevel > 0: print>>stderr, 'connect fail:', host + if self.debuglevel > 0: + print>>stderr, 'connect fail:', host if self.sock: self.sock.close() self.sock = None raise socket.error, msg (code, msg) = self.getreply() - if self.debuglevel > 0: print>>stderr, "connect:", msg + if self.debuglevel > 0: + print>>stderr, "connect:", msg return (code, msg) @@ -815,7 +832,7 @@ return sys.stdin.readline().strip() fromaddr = prompt("From") - toaddrs = prompt("To").split(',') + toaddrs = prompt("To").split(',') print "Enter message, end with ^D:" msg = '' while 1: diff --git a/lib-python/2.7/ssl.py b/lib-python/2.7/ssl.py --- a/lib-python/2.7/ssl.py +++ b/lib-python/2.7/ssl.py @@ -121,9 +121,11 @@ if e.errno != errno.ENOTCONN: raise # no, no connection yet + self._connected = False self._sslobj = None else: # yes, create the SSL object + self._connected = True self._sslobj = _ssl.sslwrap(self._sock, server_side, keyfile, certfile, cert_reqs, ssl_version, ca_certs, @@ -293,21 +295,36 @@ self._sslobj.do_handshake() - def connect(self, addr): - - """Connects to remote ADDR, and then wraps the connection in - an SSL channel.""" - + def _real_connect(self, addr, return_errno): # Here we assume that the socket is client-side, and not # connected at the time of the call. We connect it, then wrap it. - if self._sslobj: + if self._connected: raise ValueError("attempt to connect already-connected SSLSocket!") - socket.connect(self, addr) self._sslobj = _ssl.sslwrap(self._sock, False, self.keyfile, self.certfile, self.cert_reqs, self.ssl_version, self.ca_certs, self.ciphers) - if self.do_handshake_on_connect: - self.do_handshake() + try: + socket.connect(self, addr) + if self.do_handshake_on_connect: + self.do_handshake() + except socket_error as e: + if return_errno: + return e.errno + else: + self._sslobj = None + raise e + self._connected = True + return 0 + + def connect(self, addr): + """Connects to remote ADDR, and then wraps the connection in + an SSL channel.""" + self._real_connect(addr, False) + + def connect_ex(self, addr): + """Connects to remote ADDR, and then wraps the connection in + an SSL channel.""" + return self._real_connect(addr, True) def accept(self): diff --git a/lib-python/2.7/subprocess.py b/lib-python/2.7/subprocess.py --- a/lib-python/2.7/subprocess.py +++ b/lib-python/2.7/subprocess.py @@ -396,6 +396,7 @@ import traceback import gc import signal +import errno # Exception classes used by this module. class CalledProcessError(Exception): @@ -427,7 +428,6 @@ else: import select _has_poll = hasattr(select, 'poll') - import errno import fcntl import pickle @@ -441,8 +441,15 @@ "check_output", "CalledProcessError"] if mswindows: - from _subprocess import CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP - __all__.extend(["CREATE_NEW_CONSOLE", "CREATE_NEW_PROCESS_GROUP"]) + from _subprocess import (CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP, + STD_INPUT_HANDLE, STD_OUTPUT_HANDLE, + STD_ERROR_HANDLE, SW_HIDE, + STARTF_USESTDHANDLES, STARTF_USESHOWWINDOW) + + __all__.extend(["CREATE_NEW_CONSOLE", "CREATE_NEW_PROCESS_GROUP", + "STD_INPUT_HANDLE", "STD_OUTPUT_HANDLE", + "STD_ERROR_HANDLE", "SW_HIDE", + "STARTF_USESTDHANDLES", "STARTF_USESHOWWINDOW"]) try: MAXFD = os.sysconf("SC_OPEN_MAX") except: @@ -726,7 +733,11 @@ stderr = None if self.stdin: if input: - self.stdin.write(input) + try: + self.stdin.write(input) + except IOError as e: + if e.errno != errno.EPIPE and e.errno != errno.EINVAL: + raise self.stdin.close() elif self.stdout: stdout = self.stdout.read() @@ -883,7 +894,7 @@ except pywintypes.error, e: # Translate pywintypes.error to WindowsError, which is # a subclass of OSError. FIXME: We should really - # translate errno using _sys_errlist (or simliar), but + # translate errno using _sys_errlist (or similar), but # how can this be done from Python? raise WindowsError(*e.args) finally: @@ -956,7 +967,11 @@ if self.stdin: if input is not None: - self.stdin.write(input) + try: + self.stdin.write(input) + except IOError as e: + if e.errno != errno.EPIPE: + raise self.stdin.close() if self.stdout: @@ -1051,14 +1066,17 @@ errread, errwrite) - def _set_cloexec_flag(self, fd): + def _set_cloexec_flag(self, fd, cloexec=True): try: cloexec_flag = fcntl.FD_CLOEXEC except AttributeError: cloexec_flag = 1 old = fcntl.fcntl(fd, fcntl.F_GETFD) - fcntl.fcntl(fd, fcntl.F_SETFD, old | cloexec_flag) + if cloexec: + fcntl.fcntl(fd, fcntl.F_SETFD, old | cloexec_flag) + else: + fcntl.fcntl(fd, fcntl.F_SETFD, old & ~cloexec_flag) def _close_fds(self, but): @@ -1128,21 +1146,25 @@ os.close(errpipe_read) # Dup fds for child - if p2cread is not None: - os.dup2(p2cread, 0) - if c2pwrite is not None: - os.dup2(c2pwrite, 1) - if errwrite is not None: - os.dup2(errwrite, 2) + def _dup2(a, b): + # dup2() removes the CLOEXEC flag but + # we must do it ourselves if dup2() + # would be a no-op (issue #10806). + if a == b: + self._set_cloexec_flag(a, False) + elif a is not None: + os.dup2(a, b) + _dup2(p2cread, 0) + _dup2(c2pwrite, 1) + _dup2(errwrite, 2) - # Close pipe fds. Make sure we don't close the same - # fd more than once, or standard fds. - if p2cread is not None and p2cread not in (0,): - os.close(p2cread) - if c2pwrite is not None and c2pwrite not in (p2cread, 1): - os.close(c2pwrite) - if errwrite is not None and errwrite not in (p2cread, c2pwrite, 2): - os.close(errwrite) + # Close pipe fds. Make sure we don't close the + # same fd more than once, or standard fds. + closed = { None } + for fd in [p2cread, c2pwrite, errwrite]: + if fd not in closed and fd > 2: + os.close(fd) + closed.add(fd) # Close all other fds, if asked for if close_fds: @@ -1194,7 +1216,11 @@ os.close(errpipe_read) if data != "": - _eintr_retry_call(os.waitpid, self.pid, 0) + try: + _eintr_retry_call(os.waitpid, self.pid, 0) + except OSError as e: + if e.errno != errno.ECHILD: + raise child_exception = pickle.loads(data) for fd in (p2cwrite, c2pread, errread): if fd is not None: @@ -1240,7 +1266,15 @@ """Wait for child process to terminate. Returns returncode attribute.""" if self.returncode is None: - pid, sts = _eintr_retry_call(os.waitpid, self.pid, 0) + try: + pid, sts = _eintr_retry_call(os.waitpid, self.pid, 0) + except OSError as e: + if e.errno != errno.ECHILD: + raise + # This happens if SIGCLD is set to be ignored or waiting + # for child processes has otherwise been disabled for our + # process. This child is dead, we can't get the status. + sts = 0 self._handle_exitstatus(sts) return self.returncode @@ -1317,9 +1351,16 @@ for fd, mode in ready: if mode & select.POLLOUT: chunk = input[input_offset : input_offset + _PIPE_BUF] - input_offset += os.write(fd, chunk) - if input_offset >= len(input): - close_unregister_and_remove(fd) + try: + input_offset += os.write(fd, chunk) + except OSError as e: + if e.errno == errno.EPIPE: + close_unregister_and_remove(fd) + else: + raise + else: + if input_offset >= len(input): + close_unregister_and_remove(fd) elif mode & select_POLLIN_POLLPRI: data = os.read(fd, 4096) if not data: @@ -1358,11 +1399,19 @@ if self.stdin in wlist: chunk = input[input_offset : input_offset + _PIPE_BUF] - bytes_written = os.write(self.stdin.fileno(), chunk) - input_offset += bytes_written - if input_offset >= len(input): - self.stdin.close() - write_set.remove(self.stdin) + try: + bytes_written = os.write(self.stdin.fileno(), chunk) + except OSError as e: + if e.errno == errno.EPIPE: + self.stdin.close() + write_set.remove(self.stdin) + else: + raise + else: + input_offset += bytes_written + if input_offset >= len(input): + self.stdin.close() + write_set.remove(self.stdin) if self.stdout in rlist: data = os.read(self.stdout.fileno(), 1024) diff --git a/lib-python/2.7/symbol.py b/lib-python/2.7/symbol.py --- a/lib-python/2.7/symbol.py +++ b/lib-python/2.7/symbol.py @@ -82,20 +82,19 @@ sliceop = 325 exprlist = 326 testlist = 327 -dictmaker = 328 -dictorsetmaker = 329 -classdef = 330 -arglist = 331 -argument = 332 -list_iter = 333 -list_for = 334 -list_if = 335 -comp_iter = 336 -comp_for = 337 -comp_if = 338 -testlist1 = 339 -encoding_decl = 340 -yield_expr = 341 +dictorsetmaker = 328 +classdef = 329 +arglist = 330 +argument = 331 +list_iter = 332 +list_for = 333 +list_if = 334 +comp_iter = 335 +comp_for = 336 +comp_if = 337 +testlist1 = 338 +encoding_decl = 339 +yield_expr = 340 #--end constants-- sym_name = {} diff --git a/lib-python/2.7/sysconfig.py b/lib-python/2.7/sysconfig.py --- a/lib-python/2.7/sysconfig.py +++ b/lib-python/2.7/sysconfig.py @@ -271,7 +271,7 @@ def _get_makefile_filename(): if _PYTHON_BUILD: return os.path.join(_PROJECT_BASE, "Makefile") - return os.path.join(get_path('stdlib'), "config", "Makefile") + return os.path.join(get_path('platstdlib'), "config", "Makefile") def _init_posix(vars): @@ -297,21 +297,6 @@ msg = msg + " (%s)" % e.strerror raise IOError(msg) - # On MacOSX we need to check the setting of the environment variable - # MACOSX_DEPLOYMENT_TARGET: configure bases some choices on it so - # it needs to be compatible. - # If it isn't set we set it to the configure-time value - if sys.platform == 'darwin' and 'MACOSX_DEPLOYMENT_TARGET' in vars: - cfg_target = vars['MACOSX_DEPLOYMENT_TARGET'] - cur_target = os.getenv('MACOSX_DEPLOYMENT_TARGET', '') - if cur_target == '': - cur_target = cfg_target - os.putenv('MACOSX_DEPLOYMENT_TARGET', cfg_target) - elif map(int, cfg_target.split('.')) > map(int, cur_target.split('.')): - msg = ('$MACOSX_DEPLOYMENT_TARGET mismatch: now "%s" but "%s" ' - 'during configure' % (cur_target, cfg_target)) - raise IOError(msg) - # On AIX, there are wrong paths to the linker scripts in the Makefile # -- these paths are relative to the Python source, but when installed # the scripts are in another directory. @@ -616,9 +601,7 @@ # machine is going to compile and link as if it were # MACOSX_DEPLOYMENT_TARGET. cfgvars = get_config_vars() - macver = os.environ.get('MACOSX_DEPLOYMENT_TARGET') - if not macver: - macver = cfgvars.get('MACOSX_DEPLOYMENT_TARGET') + macver = cfgvars.get('MACOSX_DEPLOYMENT_TARGET') if 1: # Always calculate the release of the running machine, @@ -639,7 +622,6 @@ m = re.search( r'<key>ProductUserVisibleVersion</key>\s*' + r'<string>(.*?)</string>', f.read()) - f.close() if m is not None: macrelease = '.'.join(m.group(1).split('.')[:2]) # else: fall back to the default behaviour diff --git a/lib-python/2.7/tarfile.py b/lib-python/2.7/tarfile.py --- a/lib-python/2.7/tarfile.py +++ b/lib-python/2.7/tarfile.py @@ -2239,10 +2239,14 @@ if hasattr(os, "symlink") and hasattr(os, "link"): # For systems that support symbolic and hard links. if tarinfo.issym(): + if os.path.lexists(targetpath): + os.unlink(targetpath) os.symlink(tarinfo.linkname, targetpath) else: # See extract(). if os.path.exists(tarinfo._link_target): + if os.path.lexists(targetpath): + os.unlink(targetpath) os.link(tarinfo._link_target, targetpath) else: self._extract_member(self._find_link_target(tarinfo), targetpath) diff --git a/lib-python/2.7/telnetlib.py b/lib-python/2.7/telnetlib.py --- a/lib-python/2.7/telnetlib.py +++ b/lib-python/2.7/telnetlib.py @@ -236,7 +236,7 @@ """ if self.debuglevel > 0: - print 'Telnet(%s,%d):' % (self.host, self.port), + print 'Telnet(%s,%s):' % (self.host, self.port), if args: print msg % args else: diff --git a/lib-python/2.7/test/cjkencodings/big5-utf8.txt b/lib-python/2.7/test/cjkencodings/big5-utf8.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/big5-utf8.txt @@ -0,0 +1,9 @@ +如何在 Python 中使用既有的 C library? +　在資訊科技快速發展的今天, 開發及測試軟體的速度是不容忽視的 +課題. 為加快開發及測試的速度, 我們便常希望能利用一些已開發好的 +library, 並有一個 fast prototyping 的 programming language 可 +供使用. 目前有許許多多的 library 是以 C 寫成, 而 Python 是一個 +fast prototyping 的 programming language. 故我們希望能將既有的 +C library 拿到 Python 的環境中測試及整合. 其中最主要也是我們所 +要討論的問題就是: + diff --git a/lib-python/2.7/test/cjkencodings/big5.txt b/lib-python/2.7/test/cjkencodings/big5.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/big5.txt @@ -0,0 +1,9 @@ +�p��b Python ��ϥάJ�� C library? +�@�b��T��ާֳt�o�i��, �}�o�δ��ճn�骺�t�׬O��e�� +��D. ��[�ֶ}�o�δ��ժ��t��, �ڭ̫K�`�Ʊ��Q�Τ@�Ǥw�}�o�n�� +library, �æ��@�� fast prototyping �� programming language �i +�Ѩϥ�. �ثe��\�\�h�h�� library �O�H C �g��, �� Python �O�@�� +fast prototyping �� programming language. �G�ڭ̧Ʊ��N�J�� +C library �� Python ��Ҥ��դξ�X. �䤤�̥D�n�]�O�ڭ̩� +�n�Q�ת��D�N�O: + diff --git a/lib-python/2.7/test/cjkencodings/big5hkscs-utf8.txt b/lib-python/2.7/test/cjkencodings/big5hkscs-utf8.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/big5hkscs-utf8.txt @@ -0,0 +1,2 @@ +𠄌Ě鵮罓洆 +ÊÊ̄ê êê̄ diff --git a/lib-python/2.7/test/cjkencodings/big5hkscs.txt b/lib-python/2.7/test/cjkencodings/big5hkscs.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/big5hkscs.txt @@ -0,0 +1,2 @@ +�E�\�s�ڍ� +�f�b�� diff --git a/lib-python/2.7/test/cjkencodings/cp949-utf8.txt b/lib-python/2.7/test/cjkencodings/cp949-utf8.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/cp949-utf8.txt @@ -0,0 +1,9 @@ +똠방각하 펲시콜라 + +㉯㉯납!! 因九月패믤릔궈 ⓡⓖ훀¿¿¿ 긍뒙 ⓔ뎨 ㉯. . +亞영ⓔ능횹 . . . . 서울뤄 뎐학乙家훀 ! ! !ㅠ.ㅠ +흐흐흐 ㄱㄱㄱ☆ㅠ_ㅠ 어릨 탸콰긐 뎌응 칑九들乙㉯드긐 +설릌 家훀 . . . . 굴애쉌 ⓔ궈 ⓡ릘㉱긐 因仁川女中까즼 +와쒀훀 ! ! 亞영ⓔ 家능궈 ☆上관 없능궈능 亞능뒈훀 글애듴 +ⓡ려듀九 싀풔숴훀 어릨 因仁川女中싁⑨들앜!! ㉯㉯납♡ ⌒⌒* + diff --git a/lib-python/2.7/test/cjkencodings/cp949.txt b/lib-python/2.7/test/cjkencodings/cp949.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/cp949.txt @@ -0,0 +1,9 @@ +�c�氢�� ݶ� + +��!! ��В�p�� ި��R�� ѵ� ��. . +䬿��Ѵ�� . . . . �� ʫ�R ! ! !��.�� +�� ٤�_��  ��O �� h�� O +��j ʫ�R . . . . ��֚f �ѱ� �ސt�ƒO �� +�;��R ! ! 䬿�� ʫ�ɱ� ��߾�� ɱŴ� 䬴ɵ��R �۾֊� +�޷�� Ǵ��R � ��Ĩ��!! �� ҡ�* + diff --git a/lib-python/2.7/test/cjkencodings/euc_jisx0213-utf8.txt b/lib-python/2.7/test/cjkencodings/euc_jisx0213-utf8.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/euc_jisx0213-utf8.txt @@ -0,0 +1,8 @@ +Python の開発は、1990 年ごろから開始されています。 +開発者の Guido van Rossum は教育用のプログラミング言語「ABC」の開発に参加していましたが、ABC は実用上の目的にはあまり適していませんでした。 +このため、Guido はより実用的なプログラミング言語の開発を開始し、英国 BBS 放送のコメディ番組「モンティパイソン」のファンである Guido はこの言語を「Python」と名づけました。 +このような背景から生まれた Python の言語設計は、「シンプル」で「習得が容易」という目標に重点が置かれています。 +多くのスクリプト系言語ではユーザの目先の利便性を優先して色々な機能を言語要素として取り入れる場合が多いのですが、Python ではそういった小細工が追加されることはあまりありません。 +言語自体の機能は最小限に押さえ、必要な機能は拡張モジュールとして追加する、というのが Python のポリシーです。 + +ノか゚ト゚トキ喝塀𡚴𪎌麀齁𩛰 diff --git a/lib-python/2.7/test/cjkencodings/euc_jisx0213.txt b/lib-python/2.7/test/cjkencodings/euc_jisx0213.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/euc_jisx0213.txt @@ -0,0 +1,8 @@ +Python �γ�ȯ�ϡ�1990 ǯ��鳫�Ϥ��Ƥ��ޤ�� +��ȯ�Ԥ� Guido van Rossum �϶��ѤΥץ��ߥ󥰸��ABC�פγ�ȯ�˻��ä��Ƥ��ޤ��ABC �ϼ��Ѿ��Ū�ˤϤ��ޤ�Ŭ��Ƥ��ޤ��Ǥ�� +��Τ��ᡢGuido �Ϥ��Ū�ʥץ��ߥ󥰸��γ�ȯ�򳫻Ϥ��ѹ� BBS ��Υ��ǥ��ȡ֥��ƥ� �ѥ��פΥե��Ǥ�� Guido �Ϥ��θ��Python�פ�̾�Ť��ޤ�� +��Τ褦��طʤ��ޤ줿 Python �θ��߷פϡ��֥��ץ�פǡֽ��ưספȤ��ɸ�˽��֤��Ƥ��ޤ�� +¿��Υ��ץȷϸ��Ǥϥ桼��ͥ�褷�ƿ��ʵ�ǽ��ǤȤ��Ƽ��礬¿��ΤǤ��Python �ǤϤ��ä��ٹ��ɲä��뤳�ȤϤ��ޤꤢ��ޤ�� +��켫�Τε�ǽ�ϺǾ��¤˲��ɬ�פʵ�ǽ�ϳ�ĥ�⥸�塼��Ȥ��ɲä��롢�Ȥ��Τ� Python �Υݥꥷ��Ǥ�� + +�Τ� �� ȥ�� ԏ�� diff --git a/lib-python/2.7/test/cjkencodings/euc_jp-utf8.txt b/lib-python/2.7/test/cjkencodings/euc_jp-utf8.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/euc_jp-utf8.txt @@ -0,0 +1,7 @@ +Python の開発は、1990 年ごろから開始されています。 +開発者の Guido van Rossum は教育用のプログラミング言語「ABC」の開発に参加していましたが、ABC は実用上の目的にはあまり適していませんでした。 +このため、Guido はより実用的なプログラミング言語の開発を開始し、英国 BBS 放送のコメディ番組「モンティパイソン」のファンである Guido はこの言語を「Python」と名づけました。 +このような背景から生まれた Python の言語設計は、「シンプル」で「習得が容易」という目標に重点が置かれています。 +多くのスクリプト系言語ではユーザの目先の利便性を優先して色々な機能を言語要素として取り入れる場合が多いのですが、Python ではそういった小細工が追加されることはあまりありません。 +言語自体の機能は最小限に押さえ、必要な機能は拡張モジュールとして追加する、というのが Python のポリシーです。 + diff --git a/lib-python/2.7/test/cjkencodings/euc_jp.txt b/lib-python/2.7/test/cjkencodings/euc_jp.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/euc_jp.txt @@ -0,0 +1,7 @@ +Python �γ�ȯ�ϡ�1990 ǯ��鳫�Ϥ��Ƥ��ޤ�� +��ȯ�Ԥ� Guido van Rossum �϶��ѤΥץ��ߥ󥰸��ABC�פγ�ȯ�˻��ä��Ƥ��ޤ��ABC �ϼ��Ѿ��Ū�ˤϤ��ޤ�Ŭ��Ƥ��ޤ��Ǥ�� +��Τ��ᡢGuido �Ϥ��Ū�ʥץ��ߥ󥰸��γ�ȯ�򳫻Ϥ��ѹ� BBS ��Υ��ǥ��ȡ֥��ƥ� �ѥ��פΥե��Ǥ�� Guido �Ϥ��θ��Python�פ�̾�Ť��ޤ�� +��Τ褦��طʤ��ޤ줿 Python �θ��߷פϡ��֥��ץ�פǡֽ��ưספȤ��ɸ�˽��֤��Ƥ��ޤ�� +¿��Υ��ץȷϸ��Ǥϥ桼��ͥ�褷�ƿ��ʵ�ǽ��ǤȤ��Ƽ��礬¿��ΤǤ��Python �ǤϤ��ä��ٹ��ɲä��뤳�ȤϤ��ޤꤢ��ޤ�� +��켫�Τε�ǽ�ϺǾ��¤˲��ɬ�פʵ�ǽ�ϳ�ĥ�⥸�塼��Ȥ��ɲä��롢�Ȥ��Τ� Python �Υݥꥷ��Ǥ�� + diff --git a/lib-python/2.7/test/cjkencodings/euc_kr-utf8.txt b/lib-python/2.7/test/cjkencodings/euc_kr-utf8.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/euc_kr-utf8.txt @@ -0,0 +1,7 @@ +◎ 파이썬(Python)은 배우기 쉽고, 강력한 프로그래밍 언어입니다. 파이썬은 +효율적인 고수준 데이터 구조와 간단하지만 효율적인 객체지향프로그래밍을 +지원합니다. 파이썬의 우아(優雅)한 문법과 동적 타이핑, 그리고 인터프리팅 +환경은 파이썬을 스크립팅과 여러 분야에서와 대부분의 플랫폼에서의 빠른 +애플리케이션 개발을 할 수 있는 이상적인 언어로 만들어줍니다. + +☆첫가끝: 날아라 쓔쓔쓩~ 닁큼! 뜽금없이 전홥니다. 뷁. 그런거 읎다. diff --git a/lib-python/2.7/test/cjkencodings/euc_kr.txt b/lib-python/2.7/test/cjkencodings/euc_kr.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/euc_kr.txt @@ -0,0 +1,7 @@ +�� ̽�(Python)�� , �� α׷�� Դϴ�. ��̽�� +ȿ�� ȿ�� ü��α׷�� +��մϴ�. ��̽�� (��)�� Ÿ��, �׸�� +ȯ�� ̽�� ũ��ð� �� о߿�� κ�� ÷�� +��ø��̼� �� ִ� �̻�� ݴϴ�. + +��ù��: ��ƶ� �Ԥ��ФԤԤ��ФԾ�~ �Ԥ��Ҥ�ŭ! �Ԥ��Ѥ��ݾ�� Ԥ��Ȥ��ϴ�. �Ԥ��Τ�. �׷�� Ԥ��Ѥ��. diff --git a/lib-python/2.7/test/cjkencodings/gb18030-utf8.txt b/lib-python/2.7/test/cjkencodings/gb18030-utf8.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/gb18030-utf8.txt @@ -0,0 +1,15 @@ +Python（派森）语言是一种功能强大而完善的通用型计算机程序设计语言， +已经具有十多年的发展历史，成熟且稳定。这种语言具有非常简捷而清晰 +的语法特点，适合完成各种高层任务，几乎可以在所有的操作系统中 +运行。这种语言简单而强大，适合各种人士学习使用。目前，基于这 +种语言的相关技术正在飞速的发展，用户数量急剧扩大，相关的资源非常多。 +如何在 Python 中使用既有的 C library? +　在資訊科技快速發展的今天, 開發及測試軟體的速度是不容忽視的 +課題. 為加快開發及測試的速度, 我們便常希望能利用一些已開發好的 +library, 並有一個 fast prototyping 的 programming language 可 +供使用. 目前有許許多多的 library 是以 C 寫成, 而 Python 是一個 +fast prototyping 的 programming language. 故我們希望能將既有的 +C library 拿到 Python 的環境中測試及整合. 其中最主要也是我們所 +要討論的問題就是: +파이썬은 강력한 기능을 지닌 범용 컴퓨터 프로그래밍 언어다. + diff --git a/lib-python/2.7/test/cjkencodings/gb18030.txt b/lib-python/2.7/test/cjkencodings/gb18030.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/gb18030.txt @@ -0,0 +1,15 @@ +Python��ɭ��һ�ֹ��ǿ��Ƶ�ͨ��ͼ��ԣ� +�Ѿ��ʮ��ķ�չ��ʷ��ȶ��Ծ��зǳ��ݶ�� +��﷨�ص㣬�ʺ��ɸ��ָ߲��񣬼��еĲ��ϵͳ�� +��С��Լ򵥶�ǿ��ʺϸ��ʿѧϰʹ�á�Ŀǰ�� +��Ե��ؼ��ڷ��ٵķ�չ��û��ص��Դ�ǳ��ࡣ +�� Python ��ʹ�ü��е� C library? +��YӍ�Ƽ��ٰlչ�Ľ��, �_�l��yԇܛ�w��ٶ��ǲ��ݺ�ҕ�� +�n�}. ��ӿ��_�l��yԇ��ٶ�, �҂��㳣ϣ��һЩ��_�l�õ� +library, �K��һ�� fast prototyping �� programming language �� +��ʹ��. Ŀǰ��S�S�� library �� C ��, �� Python ��һ�� +fast prototyping �� programming language. ��҂�ϣ��܌��е� +C library �õ� Python �ĭh��Мyԇ��. ��ҪҲ��҂�� +ҪӑՓ�Ć��}��: +�5�1�3�3�2�1�3�1 �7�6�0�4�6�3 �8�5�8�6�3�5 �3�1�9�5 �0�9�3�0 �4�3�5�7�5�5 �5�5�0�9�8�9�9�3�0�4 �2�9�2�5�9�9. + diff --git a/lib-python/2.7/test/cjkencodings/gb2312-utf8.txt b/lib-python/2.7/test/cjkencodings/gb2312-utf8.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/gb2312-utf8.txt @@ -0,0 +1,6 @@ +Python（派森）语言是一种功能强大而完善的通用型计算机程序设计语言， +已经具有十多年的发展历史，成熟且稳定。这种语言具有非常简捷而清晰 +的语法特点，适合完成各种高层任务，几乎可以在所有的操作系统中 +运行。这种语言简单而强大，适合各种人士学习使用。目前，基于这 +种语言的相关技术正在飞速的发展，用户数量急剧扩大，相关的资源非常多。 + diff --git a/lib-python/2.7/test/cjkencodings/gb2312.txt b/lib-python/2.7/test/cjkencodings/gb2312.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/gb2312.txt @@ -0,0 +1,6 @@ +Python��ɭ��һ�ֹ��ǿ��Ƶ�ͨ��ͼ��ԣ� +�Ѿ��ʮ��ķ�չ��ʷ��ȶ��Ծ��зǳ��ݶ�� +��﷨�ص㣬�ʺ��ɸ��ָ߲��񣬼��еĲ��ϵͳ�� +��С��Լ򵥶�ǿ��ʺϸ��ʿѧϰʹ�á�Ŀǰ�� +��Ե��ؼ��ڷ��ٵķ�չ��û��ص��Դ�ǳ��ࡣ + diff --git a/lib-python/2.7/test/cjkencodings/gbk-utf8.txt b/lib-python/2.7/test/cjkencodings/gbk-utf8.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/gbk-utf8.txt @@ -0,0 +1,14 @@ +Python（派森）语言是一种功能强大而完善的通用型计算机程序设计语言， +已经具有十多年的发展历史，成熟且稳定。这种语言具有非常简捷而清晰 +的语法特点，适合完成各种高层任务，几乎可以在所有的操作系统中 +运行。这种语言简单而强大，适合各种人士学习使用。目前，基于这 +种语言的相关技术正在飞速的发展，用户数量急剧扩大，相关的资源非常多。 +如何在 Python 中使用既有的 C library? +　在資訊科技快速發展的今天, 開發及測試軟體的速度是不容忽視的 +課題. 為加快開發及測試的速度, 我們便常希望能利用一些已開發好的 +library, 並有一個 fast prototyping 的 programming language 可 +供使用. 目前有許許多多的 library 是以 C 寫成, 而 Python 是一個 +fast prototyping 的 programming language. 故我們希望能將既有的 +C library 拿到 Python 的環境中測試及整合. 其中最主要也是我們所 +要討論的問題就是: + diff --git a/lib-python/2.7/test/cjkencodings/gbk.txt b/lib-python/2.7/test/cjkencodings/gbk.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/gbk.txt @@ -0,0 +1,14 @@ +Python��ɭ��һ�ֹ��ǿ��Ƶ�ͨ��ͼ��ԣ� +�Ѿ��ʮ��ķ�չ��ʷ��ȶ��Ծ��зǳ��ݶ�� +��﷨�ص㣬�ʺ��ɸ��ָ߲��񣬼��еĲ��ϵͳ�� +��С��Լ򵥶�ǿ��ʺϸ��ʿѧϰʹ�á�Ŀǰ�� +��Ե��ؼ��ڷ��ٵķ�չ��û��ص��Դ�ǳ��ࡣ +�� Python ��ʹ�ü��е� C library? +��YӍ�Ƽ��ٰlչ�Ľ��, �_�l��yԇܛ�w��ٶ��ǲ��ݺ�ҕ�� +�n�}. ��ӿ��_�l��yԇ��ٶ�, �҂��㳣ϣ��һЩ��_�l�õ� +library, �K��һ�� fast prototyping �� programming language �� +��ʹ��. Ŀǰ��S�S�� library �� C ��, �� Python ��һ�� +fast prototyping �� programming language. ��҂�ϣ��܌��е� +C library �õ� Python �ĭh��Мyԇ��. ��ҪҲ��҂�� +ҪӑՓ�Ć��}��: + diff --git a/lib-python/2.7/test/cjkencodings/hz-utf8.txt b/lib-python/2.7/test/cjkencodings/hz-utf8.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/hz-utf8.txt @@ -0,0 +1,2 @@ +This sentence is in ASCII. +The next sentence is in GB.己所不欲，勿施於人。Bye. diff --git a/lib-python/2.7/test/cjkencodings/hz.txt b/lib-python/2.7/test/cjkencodings/hz.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/hz.txt @@ -0,0 +1,2 @@ +This sentence is in ASCII. +The next sentence is in GB.~{<:Ky2;S{#,NpJ)l6HK!#~}Bye. diff --git a/lib-python/2.7/test/cjkencodings/johab-utf8.txt b/lib-python/2.7/test/cjkencodings/johab-utf8.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/johab-utf8.txt @@ -0,0 +1,9 @@ +똠방각하 펲시콜라 + +㉯㉯납!! 因九月패믤릔궈 ⓡⓖ훀¿¿¿ 긍뒙 ⓔ뎨 ㉯. . +亞영ⓔ능횹 . . . . 서울뤄 뎐학乙家훀 ! ! !ㅠ.ㅠ +흐흐흐 ㄱㄱㄱ☆ㅠ_ㅠ 어릨 탸콰긐 뎌응 칑九들乙㉯드긐 +설릌 家훀 . . . . 굴애쉌 ⓔ궈 ⓡ릘㉱긐 因仁川女中까즼 +와쒀훀 ! ! 亞영ⓔ 家능궈 ☆上관 없능궈능 亞능뒈훀 글애듴 +ⓡ려듀九 싀풔숴훀 어릨 因仁川女中싁⑨들앜!! ㉯㉯납♡ ⌒⌒* + diff --git a/lib-python/2.7/test/cjkencodings/johab.txt b/lib-python/2.7/test/cjkencodings/johab.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/johab.txt @@ -0,0 +1,9 @@ +��w�b�a �\��ũ�a + +��s!! �g��Ú�� zٯٯٯ �w�� ѕ� ��. . +�<�w�ѓw�s . . . . �ᶉ�� e�b�� ;�z ! ! !�A.�A +�a�a�a �A�A�A�i�A_�A �៚ ȡ��z �a�w ×✗i�� a�z +��z �;�z . . . . �� ъ� �ޟ��‹z �g�b�I��a�� +��z ! ! �<�w�� ;�w�� i꾉� ��w��w �<�w��z �i��z +�ޝa�A� ��Ρ��z �៚ �g�b�I��鯂��i�z!! ��sٽ �b�b* + diff --git a/lib-python/2.7/test/cjkencodings/shift_jis-utf8.txt b/lib-python/2.7/test/cjkencodings/shift_jis-utf8.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/shift_jis-utf8.txt @@ -0,0 +1,7 @@ +Python の開発は、1990 年ごろから開始されています。 +開発者の Guido van Rossum は教育用のプログラミング言語「ABC」の開発に参加していましたが、ABC は実用上の目的にはあまり適していませんでした。 +このため、Guido はより実用的なプログラミング言語の開発を開始し、英国 BBS 放送のコメディ番組「モンティパイソン」のファンである Guido はこの言語を「Python」と名づけました。 +このような背景から生まれた Python の言語設計は、「シンプル」で「習得が容易」という目標に重点が置かれています。 +多くのスクリプト系言語ではユーザの目先の利便性を優先して色々な機能を言語要素として取り入れる場合が多いのですが、Python ではそういった小細工が追加されることはあまりありません。 +言語自体の機能は最小限に押さえ、必要な機能は拡張モジュールとして追加する、というのが Python のポリシーです。 + diff --git a/lib-python/2.7/test/cjkencodings/shift_jis.txt b/lib-python/2.7/test/cjkencodings/shift_jis.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/shift_jis.txt @@ -0,0 +1,7 @@ +Python �̊J��́A1990 �N��납��J�n��Ă��܂��B +�J��҂� Guido van Rossum �͋��p�̃v��O��~��O��uABC�v�̊J��ɎQ��Ă��܂��AABC �͎��p��̖ړI�ɂ͂��܂�K��Ă��܂��ł��B +��̂��߁AGuido �͂��p�I�ȃv��O��~��O��̊J��J�n��A�p�� BBS ��̃R��f�B�ԑg�u��e�B �p�C�\��v�̃t�@��ł�� Guido �͂��̌��uPython�v�Ɩ��Â��܂��B +��̂悤�Ȕw�i��琶�܂ꂽ Python �̌��݌v�́A�u�V��v��v�Łu�K��e�Ձv�Ƃ��ڕW�ɏd�_��u��Ă��܂��B +��̃X�N��v�g�n��ł̓��[�U�̖ڐ�̗��֐��D�悵�ĐF�X�ȋ@�\��v�f�Ƃ��Ď��ꍇ��̂ł��APython �ł͂��׍H��ǉ��邱�Ƃ͂��܂肠��܂��B +��ꎩ�̂̋@�\�͍ŏ��ɉ��A�K�v�ȋ@�\�͊g��W��[��Ƃ��Ēǉ��A�Ƃ��̂� Python �̃|��V�[�ł��B + diff --git a/lib-python/2.7/test/cjkencodings/shift_jisx0213-utf8.txt b/lib-python/2.7/test/cjkencodings/shift_jisx0213-utf8.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/shift_jisx0213-utf8.txt @@ -0,0 +1,8 @@ +Python の開発は、1990 年ごろから開始されています。 +開発者の Guido van Rossum は教育用のプログラミング言語「ABC」の開発に参加していましたが、ABC は実用上の目的にはあまり適していませんでした。 +このため、Guido はより実用的なプログラミング言語の開発を開始し、英国 BBS 放送のコメディ番組「モンティパイソン」のファンである Guido はこの言語を「Python」と名づけました。 +このような背景から生まれた Python の言語設計は、「シンプル」で「習得が容易」という目標に重点が置かれています。 +多くのスクリプト系言語ではユーザの目先の利便性を優先して色々な機能を言語要素として取り入れる場合が多いのですが、Python ではそういった小細工が追加されることはあまりありません。 +言語自体の機能は最小限に押さえ、必要な機能は拡張モジュールとして追加する、というのが Python のポリシーです。 + +ノか゚ト゚トキ喝塀𡚴𪎌麀齁𩛰 diff --git a/lib-python/2.7/test/cjkencodings/shift_jisx0213.txt b/lib-python/2.7/test/cjkencodings/shift_jisx0213.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/shift_jisx0213.txt @@ -0,0 +1,8 @@ +Python �̊J��́A1990 �N��납��J�n��Ă��܂��B +�J��҂� Guido van Rossum �͋��p�̃v��O��~��O��uABC�v�̊J��ɎQ��Ă��܂��AABC �͎��p��̖ړI�ɂ͂��܂�K��Ă��܂��ł��B +��̂��߁AGuido �͂��p�I�ȃv��O��~��O��̊J��J�n��A�p�� BBS ��̃R��f�B�ԑg�u��e�B �p�C�\��v�̃t�@��ł�� Guido �͂��̌��uPython�v�Ɩ��Â��܂��B +��̂悤�Ȕw�i��琶�܂ꂽ Python �̌��݌v�́A�u�V��v��v�Łu�K��e�Ձv�Ƃ��ڕW�ɏd�_��u��Ă��܂��B +��̃X�N��v�g�n��ł̓��[�U�̖ڐ�̗��֐��D�悵�ĐF�X�ȋ@�\��v�f�Ƃ��Ď��ꍇ��̂ł��APython �ł͂��׍H��ǉ��邱�Ƃ͂��܂肠��܂��B +��ꎩ�̂̋@�\�͍ŏ��ɉ��A�K�v�ȋ@�\�͊g��W��[��Ƃ��Ēǉ��A�Ƃ��̂� Python �̃|��V�[�ł��B + +�m�� g�L�K�y �� diff --git a/lib-python/2.7/test/cjkencodings_test.py b/lib-python/2.7/test/cjkencodings_test.py deleted file mode 100644 --- a/lib-python/2.7/test/cjkencodings_test.py +++ /dev/null @@ -1,1019 +0,0 @@ -teststring = { -'big5': ( -"\xa6\x70\xa6\xf3\xa6\x62\x20\x50\x79\x74\x68\x6f\x6e\x20\xa4\xa4" -"\xa8\xcf\xa5\xce\xac\x4a\xa6\xb3\xaa\xba\x20\x43\x20\x6c\x69\x62" -"\x72\x61\x72\x79\x3f\x0a\xa1\x40\xa6\x62\xb8\xea\xb0\x54\xac\xec" -"\xa7\xde\xa7\xd6\xb3\x74\xb5\x6f\xae\x69\xaa\xba\xa4\xb5\xa4\xd1" -"\x2c\x20\xb6\x7d\xb5\x6f\xa4\xce\xb4\xfa\xb8\xd5\xb3\x6e\xc5\xe9" -"\xaa\xba\xb3\x74\xab\xd7\xac\x4f\xa4\xa3\xae\x65\xa9\xbf\xb5\xf8" -"\xaa\xba\x0a\xbd\xd2\xc3\x44\x2e\x20\xac\xb0\xa5\x5b\xa7\xd6\xb6" -"\x7d\xb5\x6f\xa4\xce\xb4\xfa\xb8\xd5\xaa\xba\xb3\x74\xab\xd7\x2c" -"\x20\xa7\xda\xad\xcc\xab\x4b\xb1\x60\xa7\xc6\xb1\xe6\xaf\xe0\xa7" -"\x51\xa5\xce\xa4\x40\xa8\xc7\xa4\x77\xb6\x7d\xb5\x6f\xa6\x6e\xaa" -"\xba\x0a\x6c\x69\x62\x72\x61\x72\x79\x2c\x20\xa8\xc3\xa6\xb3\xa4" -"\x40\xad\xd3\x20\x66\x61\x73\x74\x20\x70\x72\x6f\x74\x6f\x74\x79" -"\x70\x69\x6e\x67\x20\xaa\xba\x20\x70\x72\x6f\x67\x72\x61\x6d\x6d" -"\x69\x6e\x67\x20\x6c\x61\x6e\x67\x75\x61\x67\x65\x20\xa5\x69\x0a" -"\xa8\xd1\xa8\xcf\xa5\xce\x2e\x20\xa5\xd8\xab\x65\xa6\xb3\xb3\x5c" -"\xb3\x5c\xa6\x68\xa6\x68\xaa\xba\x20\x6c\x69\x62\x72\x61\x72\x79" -"\x20\xac\x4f\xa5\x48\x20\x43\x20\xbc\x67\xa6\xa8\x2c\x20\xa6\xd3" -"\x20\x50\x79\x74\x68\x6f\x6e\x20\xac\x4f\xa4\x40\xad\xd3\x0a\x66" -"\x61\x73\x74\x20\x70\x72\x6f\x74\x6f\x74\x79\x70\x69\x6e\x67\x20" -"\xaa\xba\x20\x70\x72\x6f\x67\x72\x61\x6d\x6d\x69\x6e\x67\x20\x6c" -"\x61\x6e\x67\x75\x61\x67\x65\x2e\x20\xac\x47\xa7\xda\xad\xcc\xa7" -"\xc6\xb1\xe6\xaf\xe0\xb1\x4e\xac\x4a\xa6\xb3\xaa\xba\x0a\x43\x20" -"\x6c\x69\x62\x72\x61\x72\x79\x20\xae\xb3\xa8\xec\x20\x50\x79\x74" -"\x68\x6f\x6e\x20\xaa\xba\xc0\xf4\xb9\xd2\xa4\xa4\xb4\xfa\xb8\xd5" -"\xa4\xce\xbe\xe3\xa6\x58\x2e\x20\xa8\xe4\xa4\xa4\xb3\xcc\xa5\x44" -"\xad\x6e\xa4\x5d\xac\x4f\xa7\xda\xad\xcc\xa9\xd2\x0a\xad\x6e\xb0" -"\x51\xbd\xd7\xaa\xba\xb0\xdd\xc3\x44\xb4\x4e\xac\x4f\x3a\x0a\x0a", -"\xe5\xa6\x82\xe4\xbd\x95\xe5\x9c\xa8\x20\x50\x79\x74\x68\x6f\x6e" -"\x20\xe4\xb8\xad\xe4\xbd\xbf\xe7\x94\xa8\xe6\x97\xa2\xe6\x9c\x89" -"\xe7\x9a\x84\x20\x43\x20\x6c\x69\x62\x72\x61\x72\x79\x3f\x0a\xe3" -"\x80\x80\xe5\x9c\xa8\xe8\xb3\x87\xe8\xa8\x8a\xe7\xa7\x91\xe6\x8a" -"\x80\xe5\xbf\xab\xe9\x80\x9f\xe7\x99\xbc\xe5\xb1\x95\xe7\x9a\x84" -"\xe4\xbb\x8a\xe5\xa4\xa9\x2c\x20\xe9\x96\x8b\xe7\x99\xbc\xe5\x8f" -"\x8a\xe6\xb8\xac\xe8\xa9\xa6\xe8\xbb\x9f\xe9\xab\x94\xe7\x9a\x84" -"\xe9\x80\x9f\xe5\xba\xa6\xe6\x98\xaf\xe4\xb8\x8d\xe5\xae\xb9\xe5" -"\xbf\xbd\xe8\xa6\x96\xe7\x9a\x84\x0a\xe8\xaa\xb2\xe9\xa1\x8c\x2e" -"\x20\xe7\x82\xba\xe5\x8a\xa0\xe5\xbf\xab\xe9\x96\x8b\xe7\x99\xbc" -"\xe5\x8f\x8a\xe6\xb8\xac\xe8\xa9\xa6\xe7\x9a\x84\xe9\x80\x9f\xe5" -"\xba\xa6\x2c\x20\xe6\x88\x91\xe5\x80\x91\xe4\xbe\xbf\xe5\xb8\xb8" -"\xe5\xb8\x8c\xe6\x9c\x9b\xe8\x83\xbd\xe5\x88\xa9\xe7\x94\xa8\xe4" -"\xb8\x80\xe4\xba\x9b\xe5\xb7\xb2\xe9\x96\x8b\xe7\x99\xbc\xe5\xa5" -"\xbd\xe7\x9a\x84\x0a\x6c\x69\x62\x72\x61\x72\x79\x2c\x20\xe4\xb8" -"\xa6\xe6\x9c\x89\xe4\xb8\x80\xe5\x80\x8b\x20\x66\x61\x73\x74\x20" -"\x70\x72\x6f\x74\x6f\x74\x79\x70\x69\x6e\x67\x20\xe7\x9a\x84\x20" -"\x70\x72\x6f\x67\x72\x61\x6d\x6d\x69\x6e\x67\x20\x6c\x61\x6e\x67" -"\x75\x61\x67\x65\x20\xe5\x8f\xaf\x0a\xe4\xbe\x9b\xe4\xbd\xbf\xe7" -"\x94\xa8\x2e\x20\xe7\x9b\xae\xe5\x89\x8d\xe6\x9c\x89\xe8\xa8\xb1" -"\xe8\xa8\xb1\xe5\xa4\x9a\xe5\xa4\x9a\xe7\x9a\x84\x20\x6c\x69\x62" -"\x72\x61\x72\x79\x20\xe6\x98\xaf\xe4\xbb\xa5\x20\x43\x20\xe5\xaf" -"\xab\xe6\x88\x90\x2c\x20\xe8\x80\x8c\x20\x50\x79\x74\x68\x6f\x6e" -"\x20\xe6\x98\xaf\xe4\xb8\x80\xe5\x80\x8b\x0a\x66\x61\x73\x74\x20" -"\x70\x72\x6f\x74\x6f\x74\x79\x70\x69\x6e\x67\x20\xe7\x9a\x84\x20" -"\x70\x72\x6f\x67\x72\x61\x6d\x6d\x69\x6e\x67\x20\x6c\x61\x6e\x67" -"\x75\x61\x67\x65\x2e\x20\xe6\x95\x85\xe6\x88\x91\xe5\x80\x91\xe5" -"\xb8\x8c\xe6\x9c\x9b\xe8\x83\xbd\xe5\xb0\x87\xe6\x97\xa2\xe6\x9c" -"\x89\xe7\x9a\x84\x0a\x43\x20\x6c\x69\x62\x72\x61\x72\x79\x20\xe6" -"\x8b\xbf\xe5\x88\xb0\x20\x50\x79\x74\x68\x6f\x6e\x20\xe7\x9a\x84" -"\xe7\x92\xb0\xe5\xa2\x83\xe4\xb8\xad\xe6\xb8\xac\xe8\xa9\xa6\xe5" -"\x8f\x8a\xe6\x95\xb4\xe5\x90\x88\x2e\x20\xe5\x85\xb6\xe4\xb8\xad" -"\xe6\x9c\x80\xe4\xb8\xbb\xe8\xa6\x81\xe4\xb9\x9f\xe6\x98\xaf\xe6" -"\x88\x91\xe5\x80\x91\xe6\x89\x80\x0a\xe8\xa6\x81\xe8\xa8\x8e\xe8" -"\xab\x96\xe7\x9a\x84\xe5\x95\x8f\xe9\xa1\x8c\xe5\xb0\xb1\xe6\x98" -"\xaf\x3a\x0a\x0a"), -'big5hkscs': ( -"\x88\x45\x88\x5c\x8a\x73\x8b\xda\x8d\xd8\x0a\x88\x66\x88\x62\x88" -"\xa7\x20\x88\xa7\x88\xa3\x0a", -"\xf0\xa0\x84\x8c\xc4\x9a\xe9\xb5\xae\xe7\xbd\x93\xe6\xb4\x86\x0a" -"\xc3\x8a\xc3\x8a\xcc\x84\xc3\xaa\x20\xc3\xaa\xc3\xaa\xcc\x84\x0a"), -'cp949': ( -"\x8c\x63\xb9\xe6\xb0\xa2\xc7\xcf\x20\xbc\x84\xbd\xc3\xc4\xdd\xb6" -"\xf3\x0a\x0a\xa8\xc0\xa8\xc0\xb3\xb3\x21\x21\x20\xec\xd7\xce\xfa" -"\xea\xc5\xc6\xd0\x92\xe6\x90\x70\xb1\xc5\x20\xa8\xde\xa8\xd3\xc4" -"\x52\xa2\xaf\xa2\xaf\xa2\xaf\x20\xb1\xe0\x8a\x96\x20\xa8\xd1\xb5" -"\xb3\x20\xa8\xc0\x2e\x20\x2e\x0a\xe4\xac\xbf\xb5\xa8\xd1\xb4\xc9" -"\xc8\xc2\x20\x2e\x20\x2e\x20\x2e\x20\x2e\x20\xbc\xad\xbf\xef\xb7" -"\xef\x20\xb5\xaf\xc7\xd0\xeb\xe0\x20\xca\xab\xc4\x52\x20\x21\x20" -"\x21\x20\x21\xa4\xd0\x2e\xa4\xd0\x0a\xc8\xe5\xc8\xe5\xc8\xe5\x20" -"\xa4\xa1\xa4\xa1\xa4\xa1\xa1\xd9\xa4\xd0\x5f\xa4\xd0\x20\xbe\xee" -"\x90\x8a\x20\xc5\xcb\xc4\xe2\x83\x4f\x20\xb5\xae\xc0\xc0\x20\xaf" -"\x68\xce\xfa\xb5\xe9\xeb\xe0\x20\xa8\xc0\xb5\xe5\x83\x4f\x0a\xbc" -"\xb3\x90\x6a\x20\xca\xab\xc4\x52\x20\x2e\x20\x2e\x20\x2e\x20\x2e" -"\x20\xb1\xbc\xbe\xd6\x9a\x66\x20\xa8\xd1\xb1\xc5\x20\xa8\xde\x90" -"\x74\xa8\xc2\x83\x4f\x20\xec\xd7\xec\xd2\xf4\xb9\xe5\xfc\xf1\xe9" -"\xb1\xee\xa3\x8e\x0a\xbf\xcd\xbe\xac\xc4\x52\x20\x21\x20\x21\x20" -"\xe4\xac\xbf\xb5\xa8\xd1\x20\xca\xab\xb4\xc9\xb1\xc5\x20\xa1\xd9" -"\xdf\xbe\xb0\xfc\x20\xbe\xf8\xb4\xc9\xb1\xc5\xb4\xc9\x20\xe4\xac" -"\xb4\xc9\xb5\xd8\xc4\x52\x20\xb1\xdb\xbe\xd6\x8a\xdb\x0a\xa8\xde" -"\xb7\xc1\xb5\xe0\xce\xfa\x20\x9a\xc3\xc7\xb4\xbd\xa4\xc4\x52\x20" -"\xbe\xee\x90\x8a\x20\xec\xd7\xec\xd2\xf4\xb9\xe5\xfc\xf1\xe9\x9a" -"\xc4\xa8\xef\xb5\xe9\x9d\xda\x21\x21\x20\xa8\xc0\xa8\xc0\xb3\xb3" -"\xa2\xbd\x20\xa1\xd2\xa1\xd2\x2a\x0a\x0a", -"\xeb\x98\xa0\xeb\xb0\xa9\xea\xb0\x81\xed\x95\x98\x20\xed\x8e\xb2" -"\xec\x8b\x9c\xec\xbd\x9c\xeb\x9d\xbc\x0a\x0a\xe3\x89\xaf\xe3\x89" -"\xaf\xeb\x82\xa9\x21\x21\x20\xe5\x9b\xa0\xe4\xb9\x9d\xe6\x9c\x88" -"\xed\x8c\xa8\xeb\xaf\xa4\xeb\xa6\x94\xea\xb6\x88\x20\xe2\x93\xa1" -"\xe2\x93\x96\xed\x9b\x80\xc2\xbf\xc2\xbf\xc2\xbf\x20\xea\xb8\x8d" -"\xeb\x92\x99\x20\xe2\x93\x94\xeb\x8e\xa8\x20\xe3\x89\xaf\x2e\x20" -"\x2e\x0a\xe4\xba\x9e\xec\x98\x81\xe2\x93\x94\xeb\x8a\xa5\xed\x9a" -"\xb9\x20\x2e\x20\x2e\x20\x2e\x20\x2e\x20\xec\x84\x9c\xec\x9a\xb8" -"\xeb\xa4\x84\x20\xeb\x8e\x90\xed\x95\x99\xe4\xb9\x99\x20\xe5\xae" -"\xb6\xed\x9b\x80\x20\x21\x20\x21\x20\x21\xe3\x85\xa0\x2e\xe3\x85" -"\xa0\x0a\xed\x9d\x90\xed\x9d\x90\xed\x9d\x90\x20\xe3\x84\xb1\xe3" -"\x84\xb1\xe3\x84\xb1\xe2\x98\x86\xe3\x85\xa0\x5f\xe3\x85\xa0\x20" -"\xec\x96\xb4\xeb\xa6\xa8\x20\xed\x83\xb8\xec\xbd\xb0\xea\xb8\x90" -"\x20\xeb\x8e\x8c\xec\x9d\x91\x20\xec\xb9\x91\xe4\xb9\x9d\xeb\x93" -"\xa4\xe4\xb9\x99\x20\xe3\x89\xaf\xeb\x93\x9c\xea\xb8\x90\x0a\xec" -"\x84\xa4\xeb\xa6\x8c\x20\xe5\xae\xb6\xed\x9b\x80\x20\x2e\x20\x2e" -"\x20\x2e\x20\x2e\x20\xea\xb5\xb4\xec\x95\xa0\xec\x89\x8c\x20\xe2" -"\x93\x94\xea\xb6\x88\x20\xe2\x93\xa1\xeb\xa6\x98\xe3\x89\xb1\xea" -"\xb8\x90\x20\xe5\x9b\xa0\xe4\xbb\x81\xe5\xb7\x9d\xef\xa6\x81\xe4" -"\xb8\xad\xea\xb9\x8c\xec\xa6\xbc\x0a\xec\x99\x80\xec\x92\x80\xed" -"\x9b\x80\x20\x21\x20\x21\x20\xe4\xba\x9e\xec\x98\x81\xe2\x93\x94" -"\x20\xe5\xae\xb6\xeb\x8a\xa5\xea\xb6\x88\x20\xe2\x98\x86\xe4\xb8" -"\x8a\xea\xb4\x80\x20\xec\x97\x86\xeb\x8a\xa5\xea\xb6\x88\xeb\x8a" -"\xa5\x20\xe4\xba\x9e\xeb\x8a\xa5\xeb\x92\x88\xed\x9b\x80\x20\xea" -"\xb8\x80\xec\x95\xa0\xeb\x93\xb4\x0a\xe2\x93\xa1\xeb\xa0\xa4\xeb" -"\x93\x80\xe4\xb9\x9d\x20\xec\x8b\x80\xed\x92\x94\xec\x88\xb4\xed" -"\x9b\x80\x20\xec\x96\xb4\xeb\xa6\xa8\x20\xe5\x9b\xa0\xe4\xbb\x81" -"\xe5\xb7\x9d\xef\xa6\x81\xe4\xb8\xad\xec\x8b\x81\xe2\x91\xa8\xeb" -"\x93\xa4\xec\x95\x9c\x21\x21\x20\xe3\x89\xaf\xe3\x89\xaf\xeb\x82" -"\xa9\xe2\x99\xa1\x20\xe2\x8c\x92\xe2\x8c\x92\x2a\x0a\x0a"), -'euc_jisx0213': ( -"\x50\x79\x74\x68\x6f\x6e\x20\xa4\xce\xb3\xab\xc8\xaf\xa4\xcf\xa1" -"\xa2\x31\x39\x39\x30\x20\xc7\xaf\xa4\xb4\xa4\xed\xa4\xab\xa4\xe9" -"\xb3\xab\xbb\xcf\xa4\xb5\xa4\xec\xa4\xc6\xa4\xa4\xa4\xde\xa4\xb9" -"\xa1\xa3\x0a\xb3\xab\xc8\xaf\xbc\xd4\xa4\xce\x20\x47\x75\x69\x64" -"\x6f\x20\x76\x61\x6e\x20\x52\x6f\x73\x73\x75\x6d\x20\xa4\xcf\xb6" -"\xb5\xb0\xe9\xcd\xd1\xa4\xce\xa5\xd7\xa5\xed\xa5\xb0\xa5\xe9\xa5" -"\xdf\xa5\xf3\xa5\xb0\xb8\xc0\xb8\xec\xa1\xd6\x41\x42\x43\xa1\xd7" -"\xa4\xce\xb3\xab\xc8\xaf\xa4\xcb\xbb\xb2\xb2\xc3\xa4\xb7\xa4\xc6" -"\xa4\xa4\xa4\xde\xa4\xb7\xa4\xbf\xa4\xac\xa1\xa2\x41\x42\x43\x20" -"\xa4\xcf\xbc\xc2\xcd\xd1\xbe\xe5\xa4\xce\xcc\xdc\xc5\xaa\xa4\xcb" -"\xa4\xcf\xa4\xa2\xa4\xde\xa4\xea\xc5\xac\xa4\xb7\xa4\xc6\xa4\xa4" -"\xa4\xde\xa4\xbb\xa4\xf3\xa4\xc7\xa4\xb7\xa4\xbf\xa1\xa3\x0a\xa4" -"\xb3\xa4\xce\xa4\xbf\xa4\xe1\xa1\xa2\x47\x75\x69\x64\x6f\x20\xa4" -"\xcf\xa4\xe8\xa4\xea\xbc\xc2\xcd\xd1\xc5\xaa\xa4\xca\xa5\xd7\xa5" -"\xed\xa5\xb0\xa5\xe9\xa5\xdf\xa5\xf3\xa5\xb0\xb8\xc0\xb8\xec\xa4" -"\xce\xb3\xab\xc8\xaf\xa4\xf2\xb3\xab\xbb\xcf\xa4\xb7\xa1\xa2\xb1" -"\xd1\xb9\xf1\x20\x42\x42\x53\x20\xca\xfc\xc1\xf7\xa4\xce\xa5\xb3" -"\xa5\xe1\xa5\xc7\xa5\xa3\xc8\xd6\xc1\xc8\xa1\xd6\xa5\xe2\xa5\xf3" -"\xa5\xc6\xa5\xa3\x20\xa5\xd1\xa5\xa4\xa5\xbd\xa5\xf3\xa1\xd7\xa4" -"\xce\xa5\xd5\xa5\xa1\xa5\xf3\xa4\xc7\xa4\xa2\xa4\xeb\x20\x47\x75" -"\x69\x64\x6f\x20\xa4\xcf\xa4\xb3\xa4\xce\xb8\xc0\xb8\xec\xa4\xf2" -"\xa1\xd6\x50\x79\x74\x68\x6f\x6e\xa1\xd7\xa4\xc8\xcc\xbe\xa4\xc5" -"\xa4\xb1\xa4\xde\xa4\xb7\xa4\xbf\xa1\xa3\x0a\xa4\xb3\xa4\xce\xa4" -"\xe8\xa4\xa6\xa4\xca\xc7\xd8\xb7\xca\xa4\xab\xa4\xe9\xc0\xb8\xa4" -"\xde\xa4\xec\xa4\xbf\x20\x50\x79\x74\x68\x6f\x6e\x20\xa4\xce\xb8" -"\xc0\xb8\xec\xc0\xdf\xb7\xd7\xa4\xcf\xa1\xa2\xa1\xd6\xa5\xb7\xa5" -"\xf3\xa5\xd7\xa5\xeb\xa1\xd7\xa4\xc7\xa1\xd6\xbd\xac\xc6\xc0\xa4" -"\xac\xcd\xc6\xb0\xd7\xa1\xd7\xa4\xc8\xa4\xa4\xa4\xa6\xcc\xdc\xc9" -"\xb8\xa4\xcb\xbd\xc5\xc5\xc0\xa4\xac\xc3\xd6\xa4\xab\xa4\xec\xa4" -"\xc6\xa4\xa4\xa4\xde\xa4\xb9\xa1\xa3\x0a\xc2\xbf\xa4\xaf\xa4\xce" -"\xa5\xb9\xa5\xaf\xa5\xea\xa5\xd7\xa5\xc8\xb7\xcf\xb8\xc0\xb8\xec" -"\xa4\xc7\xa4\xcf\xa5\xe6\xa1\xbc\xa5\xb6\xa4\xce\xcc\xdc\xc0\xe8" -"\xa4\xce\xcd\xf8\xca\xd8\xc0\xad\xa4\xf2\xcd\xa5\xc0\xe8\xa4\xb7" -"\xa4\xc6\xbf\xa7\xa1\xb9\xa4\xca\xb5\xa1\xc7\xbd\xa4\xf2\xb8\xc0" -"\xb8\xec\xcd\xd7\xc1\xc7\xa4\xc8\xa4\xb7\xa4\xc6\xbc\xe8\xa4\xea" -"\xc6\xfe\xa4\xec\xa4\xeb\xbe\xec\xb9\xe7\xa4\xac\xc2\xbf\xa4\xa4" -"\xa4\xce\xa4\xc7\xa4\xb9\xa4\xac\xa1\xa2\x50\x79\x74\x68\x6f\x6e" -"\x20\xa4\xc7\xa4\xcf\xa4\xbd\xa4\xa6\xa4\xa4\xa4\xc3\xa4\xbf\xbe" -"\xae\xba\xd9\xb9\xa9\xa4\xac\xc4\xc9\xb2\xc3\xa4\xb5\xa4\xec\xa4" -"\xeb\xa4\xb3\xa4\xc8\xa4\xcf\xa4\xa2\xa4\xde\xa4\xea\xa4\xa2\xa4" -"\xea\xa4\xde\xa4\xbb\xa4\xf3\xa1\xa3\x0a\xb8\xc0\xb8\xec\xbc\xab" -"\xc2\xce\xa4\xce\xb5\xa1\xc7\xbd\xa4\xcf\xba\xc7\xbe\xae\xb8\xc2" -"\xa4\xcb\xb2\xa1\xa4\xb5\xa4\xa8\xa1\xa2\xc9\xac\xcd\xd7\xa4\xca" -"\xb5\xa1\xc7\xbd\xa4\xcf\xb3\xc8\xc4\xa5\xa5\xe2\xa5\xb8\xa5\xe5" -"\xa1\xbc\xa5\xeb\xa4\xc8\xa4\xb7\xa4\xc6\xc4\xc9\xb2\xc3\xa4\xb9" -"\xa4\xeb\xa1\xa2\xa4\xc8\xa4\xa4\xa4\xa6\xa4\xce\xa4\xac\x20\x50" -"\x79\x74\x68\x6f\x6e\x20\xa4\xce\xa5\xdd\xa5\xea\xa5\xb7\xa1\xbc" -"\xa4\xc7\xa4\xb9\xa1\xa3\x0a\x0a\xa5\xce\xa4\xf7\x20\xa5\xfe\x20" -"\xa5\xc8\xa5\xad\xaf\xac\xaf\xda\x20\xcf\xe3\x8f\xfe\xd8\x20\x8f" -"\xfe\xd4\x8f\xfe\xe8\x8f\xfc\xd6\x0a", -"\x50\x79\x74\x68\x6f\x6e\x20\xe3\x81\xae\xe9\x96\x8b\xe7\x99\xba" -"\xe3\x81\xaf\xe3\x80\x81\x31\x39\x39\x30\x20\xe5\xb9\xb4\xe3\x81" -"\x94\xe3\x82\x8d\xe3\x81\x8b\xe3\x82\x89\xe9\x96\x8b\xe5\xa7\x8b" -"\xe3\x81\x95\xe3\x82\x8c\xe3\x81\xa6\xe3\x81\x84\xe3\x81\xbe\xe3" -"\x81\x99\xe3\x80\x82\x0a\xe9\x96\x8b\xe7\x99\xba\xe8\x80\x85\xe3" -"\x81\xae\x20\x47\x75\x69\x64\x6f\x20\x76\x61\x6e\x20\x52\x6f\x73" -"\x73\x75\x6d\x20\xe3\x81\xaf\xe6\x95\x99\xe8\x82\xb2\xe7\x94\xa8" -"\xe3\x81\xae\xe3\x83\x97\xe3\x83\xad\xe3\x82\xb0\xe3\x83\xa9\xe3" -"\x83\x9f\xe3\x83\xb3\xe3\x82\xb0\xe8\xa8\x80\xe8\xaa\x9e\xe3\x80" -"\x8c\x41\x42\x43\xe3\x80\x8d\xe3\x81\xae\xe9\x96\x8b\xe7\x99\xba" -"\xe3\x81\xab\xe5\x8f\x82\xe5\x8a\xa0\xe3\x81\x97\xe3\x81\xa6\xe3" -"\x81\x84\xe3\x81\xbe\xe3\x81\x97\xe3\x81\x9f\xe3\x81\x8c\xe3\x80" -"\x81\x41\x42\x43\x20\xe3\x81\xaf\xe5\xae\x9f\xe7\x94\xa8\xe4\xb8" -"\x8a\xe3\x81\xae\xe7\x9b\xae\xe7\x9a\x84\xe3\x81\xab\xe3\x81\xaf" -"\xe3\x81\x82\xe3\x81\xbe\xe3\x82\x8a\xe9\x81\xa9\xe3\x81\x97\xe3" -"\x81\xa6\xe3\x81\x84\xe3\x81\xbe\xe3\x81\x9b\xe3\x82\x93\xe3\x81" -"\xa7\xe3\x81\x97\xe3\x81\x9f\xe3\x80\x82\x0a\xe3\x81\x93\xe3\x81" -"\xae\xe3\x81\x9f\xe3\x82\x81\xe3\x80\x81\x47\x75\x69\x64\x6f\x20" -"\xe3\x81\xaf\xe3\x82\x88\xe3\x82\x8a\xe5\xae\x9f\xe7\x94\xa8\xe7" -"\x9a\x84\xe3\x81\xaa\xe3\x83\x97\xe3\x83\xad\xe3\x82\xb0\xe3\x83" -"\xa9\xe3\x83\x9f\xe3\x83\xb3\xe3\x82\xb0\xe8\xa8\x80\xe8\xaa\x9e" -"\xe3\x81\xae\xe9\x96\x8b\xe7\x99\xba\xe3\x82\x92\xe9\x96\x8b\xe5" -"\xa7\x8b\xe3\x81\x97\xe3\x80\x81\xe8\x8b\xb1\xe5\x9b\xbd\x20\x42" -"\x42\x53\x20\xe6\x94\xbe\xe9\x80\x81\xe3\x81\xae\xe3\x82\xb3\xe3" -"\x83\xa1\xe3\x83\x87\xe3\x82\xa3\xe7\x95\xaa\xe7\xb5\x84\xe3\x80" -"\x8c\xe3\x83\xa2\xe3\x83\xb3\xe3\x83\x86\xe3\x82\xa3\x20\xe3\x83" -"\x91\xe3\x82\xa4\xe3\x82\xbd\xe3\x83\xb3\xe3\x80\x8d\xe3\x81\xae" -"\xe3\x83\x95\xe3\x82\xa1\xe3\x83\xb3\xe3\x81\xa7\xe3\x81\x82\xe3" -"\x82\x8b\x20\x47\x75\x69\x64\x6f\x20\xe3\x81\xaf\xe3\x81\x93\xe3" -"\x81\xae\xe8\xa8\x80\xe8\xaa\x9e\xe3\x82\x92\xe3\x80\x8c\x50\x79" -"\x74\x68\x6f\x6e\xe3\x80\x8d\xe3\x81\xa8\xe5\x90\x8d\xe3\x81\xa5" -"\xe3\x81\x91\xe3\x81\xbe\xe3\x81\x97\xe3\x81\x9f\xe3\x80\x82\x0a" -"\xe3\x81\x93\xe3\x81\xae\xe3\x82\x88\xe3\x81\x86\xe3\x81\xaa\xe8" -"\x83\x8c\xe6\x99\xaf\xe3\x81\x8b\xe3\x82\x89\xe7\x94\x9f\xe3\x81" -"\xbe\xe3\x82\x8c\xe3\x81\x9f\x20\x50\x79\x74\x68\x6f\x6e\x20\xe3" -"\x81\xae\xe8\xa8\x80\xe8\xaa\x9e\xe8\xa8\xad\xe8\xa8\x88\xe3\x81" -"\xaf\xe3\x80\x81\xe3\x80\x8c\xe3\x82\xb7\xe3\x83\xb3\xe3\x83\x97" -"\xe3\x83\xab\xe3\x80\x8d\xe3\x81\xa7\xe3\x80\x8c\xe7\xbf\x92\xe5" -"\xbe\x97\xe3\x81\x8c\xe5\xae\xb9\xe6\x98\x93\xe3\x80\x8d\xe3\x81" -"\xa8\xe3\x81\x84\xe3\x81\x86\xe7\x9b\xae\xe6\xa8\x99\xe3\x81\xab" -"\xe9\x87\x8d\xe7\x82\xb9\xe3\x81\x8c\xe7\xbd\xae\xe3\x81\x8b\xe3" -"\x82\x8c\xe3\x81\xa6\xe3\x81\x84\xe3\x81\xbe\xe3\x81\x99\xe3\x80" -"\x82\x0a\xe5\xa4\x9a\xe3\x81\x8f\xe3\x81\xae\xe3\x82\xb9\xe3\x82" -"\xaf\xe3\x83\xaa\xe3\x83\x97\xe3\x83\x88\xe7\xb3\xbb\xe8\xa8\x80" -"\xe8\xaa\x9e\xe3\x81\xa7\xe3\x81\xaf\xe3\x83\xa6\xe3\x83\xbc\xe3" -"\x82\xb6\xe3\x81\xae\xe7\x9b\xae\xe5\x85\x88\xe3\x81\xae\xe5\x88" -"\xa9\xe4\xbe\xbf\xe6\x80\xa7\xe3\x82\x92\xe5\x84\xaa\xe5\x85\x88" -"\xe3\x81\x97\xe3\x81\xa6\xe8\x89\xb2\xe3\x80\x85\xe3\x81\xaa\xe6" -"\xa9\x9f\xe8\x83\xbd\xe3\x82\x92\xe8\xa8\x80\xe8\xaa\x9e\xe8\xa6" -"\x81\xe7\xb4\xa0\xe3\x81\xa8\xe3\x81\x97\xe3\x81\xa6\xe5\x8f\x96" -"\xe3\x82\x8a\xe5\x85\xa5\xe3\x82\x8c\xe3\x82\x8b\xe5\xa0\xb4\xe5" -"\x90\x88\xe3\x81\x8c\xe5\xa4\x9a\xe3\x81\x84\xe3\x81\xae\xe3\x81" -"\xa7\xe3\x81\x99\xe3\x81\x8c\xe3\x80\x81\x50\x79\x74\x68\x6f\x6e" -"\x20\xe3\x81\xa7\xe3\x81\xaf\xe3\x81\x9d\xe3\x81\x86\xe3\x81\x84" -"\xe3\x81\xa3\xe3\x81\x9f\xe5\xb0\x8f\xe7\xb4\xb0\xe5\xb7\xa5\xe3" -"\x81\x8c\xe8\xbf\xbd\xe5\x8a\xa0\xe3\x81\x95\xe3\x82\x8c\xe3\x82" -"\x8b\xe3\x81\x93\xe3\x81\xa8\xe3\x81\xaf\xe3\x81\x82\xe3\x81\xbe" -"\xe3\x82\x8a\xe3\x81\x82\xe3\x82\x8a\xe3\x81\xbe\xe3\x81\x9b\xe3" -"\x82\x93\xe3\x80\x82\x0a\xe8\xa8\x80\xe8\xaa\x9e\xe8\x87\xaa\xe4" -"\xbd\x93\xe3\x81\xae\xe6\xa9\x9f\xe8\x83\xbd\xe3\x81\xaf\xe6\x9c" -"\x80\xe5\xb0\x8f\xe9\x99\x90\xe3\x81\xab\xe6\x8a\xbc\xe3\x81\x95" -"\xe3\x81\x88\xe3\x80\x81\xe5\xbf\x85\xe8\xa6\x81\xe3\x81\xaa\xe6" -"\xa9\x9f\xe8\x83\xbd\xe3\x81\xaf\xe6\x8b\xa1\xe5\xbc\xb5\xe3\x83" -"\xa2\xe3\x82\xb8\xe3\x83\xa5\xe3\x83\xbc\xe3\x83\xab\xe3\x81\xa8" -"\xe3\x81\x97\xe3\x81\xa6\xe8\xbf\xbd\xe5\x8a\xa0\xe3\x81\x99\xe3" -"\x82\x8b\xe3\x80\x81\xe3\x81\xa8\xe3\x81\x84\xe3\x81\x86\xe3\x81" -"\xae\xe3\x81\x8c\x20\x50\x79\x74\x68\x6f\x6e\x20\xe3\x81\xae\xe3" -"\x83\x9d\xe3\x83\xaa\xe3\x82\xb7\xe3\x83\xbc\xe3\x81\xa7\xe3\x81" -"\x99\xe3\x80\x82\x0a\x0a\xe3\x83\x8e\xe3\x81\x8b\xe3\x82\x9a\x20" -"\xe3\x83\x88\xe3\x82\x9a\x20\xe3\x83\x88\xe3\x82\xad\xef\xa8\xb6" -"\xef\xa8\xb9\x20\xf0\xa1\x9a\xb4\xf0\xaa\x8e\x8c\x20\xe9\xba\x80" -"\xe9\xbd\x81\xf0\xa9\x9b\xb0\x0a"), -'euc_jp': ( -"\x50\x79\x74\x68\x6f\x6e\x20\xa4\xce\xb3\xab\xc8\xaf\xa4\xcf\xa1" -"\xa2\x31\x39\x39\x30\x20\xc7\xaf\xa4\xb4\xa4\xed\xa4\xab\xa4\xe9" -"\xb3\xab\xbb\xcf\xa4\xb5\xa4\xec\xa4\xc6\xa4\xa4\xa4\xde\xa4\xb9" -"\xa1\xa3\x0a\xb3\xab\xc8\xaf\xbc\xd4\xa4\xce\x20\x47\x75\x69\x64" -"\x6f\x20\x76\x61\x6e\x20\x52\x6f\x73\x73\x75\x6d\x20\xa4\xcf\xb6" -"\xb5\xb0\xe9\xcd\xd1\xa4\xce\xa5\xd7\xa5\xed\xa5\xb0\xa5\xe9\xa5" -"\xdf\xa5\xf3\xa5\xb0\xb8\xc0\xb8\xec\xa1\xd6\x41\x42\x43\xa1\xd7" -"\xa4\xce\xb3\xab\xc8\xaf\xa4\xcb\xbb\xb2\xb2\xc3\xa4\xb7\xa4\xc6" -"\xa4\xa4\xa4\xde\xa4\xb7\xa4\xbf\xa4\xac\xa1\xa2\x41\x42\x43\x20" -"\xa4\xcf\xbc\xc2\xcd\xd1\xbe\xe5\xa4\xce\xcc\xdc\xc5\xaa\xa4\xcb" -"\xa4\xcf\xa4\xa2\xa4\xde\xa4\xea\xc5\xac\xa4\xb7\xa4\xc6\xa4\xa4" -"\xa4\xde\xa4\xbb\xa4\xf3\xa4\xc7\xa4\xb7\xa4\xbf\xa1\xa3\x0a\xa4" -"\xb3\xa4\xce\xa4\xbf\xa4\xe1\xa1\xa2\x47\x75\x69\x64\x6f\x20\xa4" -"\xcf\xa4\xe8\xa4\xea\xbc\xc2\xcd\xd1\xc5\xaa\xa4\xca\xa5\xd7\xa5" -"\xed\xa5\xb0\xa5\xe9\xa5\xdf\xa5\xf3\xa5\xb0\xb8\xc0\xb8\xec\xa4" -"\xce\xb3\xab\xc8\xaf\xa4\xf2\xb3\xab\xbb\xcf\xa4\xb7\xa1\xa2\xb1" -"\xd1\xb9\xf1\x20\x42\x42\x53\x20\xca\xfc\xc1\xf7\xa4\xce\xa5\xb3" -"\xa5\xe1\xa5\xc7\xa5\xa3\xc8\xd6\xc1\xc8\xa1\xd6\xa5\xe2\xa5\xf3" -"\xa5\xc6\xa5\xa3\x20\xa5\xd1\xa5\xa4\xa5\xbd\xa5\xf3\xa1\xd7\xa4" -"\xce\xa5\xd5\xa5\xa1\xa5\xf3\xa4\xc7\xa4\xa2\xa4\xeb\x20\x47\x75" -"\x69\x64\x6f\x20\xa4\xcf\xa4\xb3\xa4\xce\xb8\xc0\xb8\xec\xa4\xf2" -"\xa1\xd6\x50\x79\x74\x68\x6f\x6e\xa1\xd7\xa4\xc8\xcc\xbe\xa4\xc5" -"\xa4\xb1\xa4\xde\xa4\xb7\xa4\xbf\xa1\xa3\x0a\xa4\xb3\xa4\xce\xa4" -"\xe8\xa4\xa6\xa4\xca\xc7\xd8\xb7\xca\xa4\xab\xa4\xe9\xc0\xb8\xa4" -"\xde\xa4\xec\xa4\xbf\x20\x50\x79\x74\x68\x6f\x6e\x20\xa4\xce\xb8" -"\xc0\xb8\xec\xc0\xdf\xb7\xd7\xa4\xcf\xa1\xa2\xa1\xd6\xa5\xb7\xa5" -"\xf3\xa5\xd7\xa5\xeb\xa1\xd7\xa4\xc7\xa1\xd6\xbd\xac\xc6\xc0\xa4" -"\xac\xcd\xc6\xb0\xd7\xa1\xd7\xa4\xc8\xa4\xa4\xa4\xa6\xcc\xdc\xc9" -"\xb8\xa4\xcb\xbd\xc5\xc5\xc0\xa4\xac\xc3\xd6\xa4\xab\xa4\xec\xa4" -"\xc6\xa4\xa4\xa4\xde\xa4\xb9\xa1\xa3\x0a\xc2\xbf\xa4\xaf\xa4\xce" -"\xa5\xb9\xa5\xaf\xa5\xea\xa5\xd7\xa5\xc8\xb7\xcf\xb8\xc0\xb8\xec" -"\xa4\xc7\xa4\xcf\xa5\xe6\xa1\xbc\xa5\xb6\xa4\xce\xcc\xdc\xc0\xe8" -"\xa4\xce\xcd\xf8\xca\xd8\xc0\xad\xa4\xf2\xcd\xa5\xc0\xe8\xa4\xb7" -"\xa4\xc6\xbf\xa7\xa1\xb9\xa4\xca\xb5\xa1\xc7\xbd\xa4\xf2\xb8\xc0" -"\xb8\xec\xcd\xd7\xc1\xc7\xa4\xc8\xa4\xb7\xa4\xc6\xbc\xe8\xa4\xea" -"\xc6\xfe\xa4\xec\xa4\xeb\xbe\xec\xb9\xe7\xa4\xac\xc2\xbf\xa4\xa4" -"\xa4\xce\xa4\xc7\xa4\xb9\xa4\xac\xa1\xa2\x50\x79\x74\x68\x6f\x6e" -"\x20\xa4\xc7\xa4\xcf\xa4\xbd\xa4\xa6\xa4\xa4\xa4\xc3\xa4\xbf\xbe" -"\xae\xba\xd9\xb9\xa9\xa4\xac\xc4\xc9\xb2\xc3\xa4\xb5\xa4\xec\xa4" -"\xeb\xa4\xb3\xa4\xc8\xa4\xcf\xa4\xa2\xa4\xde\xa4\xea\xa4\xa2\xa4" -"\xea\xa4\xde\xa4\xbb\xa4\xf3\xa1\xa3\x0a\xb8\xc0\xb8\xec\xbc\xab" -"\xc2\xce\xa4\xce\xb5\xa1\xc7\xbd\xa4\xcf\xba\xc7\xbe\xae\xb8\xc2" -"\xa4\xcb\xb2\xa1\xa4\xb5\xa4\xa8\xa1\xa2\xc9\xac\xcd\xd7\xa4\xca" -"\xb5\xa1\xc7\xbd\xa4\xcf\xb3\xc8\xc4\xa5\xa5\xe2\xa5\xb8\xa5\xe5" -"\xa1\xbc\xa5\xeb\xa4\xc8\xa4\xb7\xa4\xc6\xc4\xc9\xb2\xc3\xa4\xb9" -"\xa4\xeb\xa1\xa2\xa4\xc8\xa4\xa4\xa4\xa6\xa4\xce\xa4\xac\x20\x50" -"\x79\x74\x68\x6f\x6e\x20\xa4\xce\xa5\xdd\xa5\xea\xa5\xb7\xa1\xbc" -"\xa4\xc7\xa4\xb9\xa1\xa3\x0a\x0a", -"\x50\x79\x74\x68\x6f\x6e\x20\xe3\x81\xae\xe9\x96\x8b\xe7\x99\xba" -"\xe3\x81\xaf\xe3\x80\x81\x31\x39\x39\x30\x20\xe5\xb9\xb4\xe3\x81" -"\x94\xe3\x82\x8d\xe3\x81\x8b\xe3\x82\x89\xe9\x96\x8b\xe5\xa7\x8b" -"\xe3\x81\x95\xe3\x82\x8c\xe3\x81\xa6\xe3\x81\x84\xe3\x81\xbe\xe3" -"\x81\x99\xe3\x80\x82\x0a\xe9\x96\x8b\xe7\x99\xba\xe8\x80\x85\xe3" -"\x81\xae\x20\x47\x75\x69\x64\x6f\x20\x76\x61\x6e\x20\x52\x6f\x73" -"\x73\x75\x6d\x20\xe3\x81\xaf\xe6\x95\x99\xe8\x82\xb2\xe7\x94\xa8" -"\xe3\x81\xae\xe3\x83\x97\xe3\x83\xad\xe3\x82\xb0\xe3\x83\xa9\xe3" -"\x83\x9f\xe3\x83\xb3\xe3\x82\xb0\xe8\xa8\x80\xe8\xaa\x9e\xe3\x80" -"\x8c\x41\x42\x43\xe3\x80\x8d\xe3\x81\xae\xe9\x96\x8b\xe7\x99\xba" -"\xe3\x81\xab\xe5\x8f\x82\xe5\x8a\xa0\xe3\x81\x97\xe3\x81\xa6\xe3" -"\x81\x84\xe3\x81\xbe\xe3\x81\x97\xe3\x81\x9f\xe3\x81\x8c\xe3\x80" -"\x81\x41\x42\x43\x20\xe3\x81\xaf\xe5\xae\x9f\xe7\x94\xa8\xe4\xb8" -"\x8a\xe3\x81\xae\xe7\x9b\xae\xe7\x9a\x84\xe3\x81\xab\xe3\x81\xaf" -"\xe3\x81\x82\xe3\x81\xbe\xe3\x82\x8a\xe9\x81\xa9\xe3\x81\x97\xe3" -"\x81\xa6\xe3\x81\x84\xe3\x81\xbe\xe3\x81\x9b\xe3\x82\x93\xe3\x81" -"\xa7\xe3\x81\x97\xe3\x81\x9f\xe3\x80\x82\x0a\xe3\x81\x93\xe3\x81" -"\xae\xe3\x81\x9f\xe3\x82\x81\xe3\x80\x81\x47\x75\x69\x64\x6f\x20" -"\xe3\x81\xaf\xe3\x82\x88\xe3\x82\x8a\xe5\xae\x9f\xe7\x94\xa8\xe7" -"\x9a\x84\xe3\x81\xaa\xe3\x83\x97\xe3\x83\xad\xe3\x82\xb0\xe3\x83" -"\xa9\xe3\x83\x9f\xe3\x83\xb3\xe3\x82\xb0\xe8\xa8\x80\xe8\xaa\x9e" -"\xe3\x81\xae\xe9\x96\x8b\xe7\x99\xba\xe3\x82\x92\xe9\x96\x8b\xe5" -"\xa7\x8b\xe3\x81\x97\xe3\x80\x81\xe8\x8b\xb1\xe5\x9b\xbd\x20\x42" -"\x42\x53\x20\xe6\x94\xbe\xe9\x80\x81\xe3\x81\xae\xe3\x82\xb3\xe3" -"\x83\xa1\xe3\x83\x87\xe3\x82\xa3\xe7\x95\xaa\xe7\xb5\x84\xe3\x80" -"\x8c\xe3\x83\xa2\xe3\x83\xb3\xe3\x83\x86\xe3\x82\xa3\x20\xe3\x83" -"\x91\xe3\x82\xa4\xe3\x82\xbd\xe3\x83\xb3\xe3\x80\x8d\xe3\x81\xae" -"\xe3\x83\x95\xe3\x82\xa1\xe3\x83\xb3\xe3\x81\xa7\xe3\x81\x82\xe3" -"\x82\x8b\x20\x47\x75\x69\x64\x6f\x20\xe3\x81\xaf\xe3\x81\x93\xe3" -"\x81\xae\xe8\xa8\x80\xe8\xaa\x9e\xe3\x82\x92\xe3\x80\x8c\x50\x79" -"\x74\x68\x6f\x6e\xe3\x80\x8d\xe3\x81\xa8\xe5\x90\x8d\xe3\x81\xa5" -"\xe3\x81\x91\xe3\x81\xbe\xe3\x81\x97\xe3\x81\x9f\xe3\x80\x82\x0a" -"\xe3\x81\x93\xe3\x81\xae\xe3\x82\x88\xe3\x81\x86\xe3\x81\xaa\xe8" -"\x83\x8c\xe6\x99\xaf\xe3\x81\x8b\xe3\x82\x89\xe7\x94\x9f\xe3\x81" -"\xbe\xe3\x82\x8c\xe3\x81\x9f\x20\x50\x79\x74\x68\x6f\x6e\x20\xe3" -"\x81\xae\xe8\xa8\x80\xe8\xaa\x9e\xe8\xa8\xad\xe8\xa8\x88\xe3\x81" -"\xaf\xe3\x80\x81\xe3\x80\x8c\xe3\x82\xb7\xe3\x83\xb3\xe3\x83\x97" -"\xe3\x83\xab\xe3\x80\x8d\xe3\x81\xa7\xe3\x80\x8c\xe7\xbf\x92\xe5" -"\xbe\x97\xe3\x81\x8c\xe5\xae\xb9\xe6\x98\x93\xe3\x80\x8d\xe3\x81" -"\xa8\xe3\x81\x84\xe3\x81\x86\xe7\x9b\xae\xe6\xa8\x99\xe3\x81\xab" -"\xe9\x87\x8d\xe7\x82\xb9\xe3\x81\x8c\xe7\xbd\xae\xe3\x81\x8b\xe3" -"\x82\x8c\xe3\x81\xa6\xe3\x81\x84\xe3\x81\xbe\xe3\x81\x99\xe3\x80" -"\x82\x0a\xe5\xa4\x9a\xe3\x81\x8f\xe3\x81\xae\xe3\x82\xb9\xe3\x82" -"\xaf\xe3\x83\xaa\xe3\x83\x97\xe3\x83\x88\xe7\xb3\xbb\xe8\xa8\x80" -"\xe8\xaa\x9e\xe3\x81\xa7\xe3\x81\xaf\xe3\x83\xa6\xe3\x83\xbc\xe3" -"\x82\xb6\xe3\x81\xae\xe7\x9b\xae\xe5\x85\x88\xe3\x81\xae\xe5\x88" -"\xa9\xe4\xbe\xbf\xe6\x80\xa7\xe3\x82\x92\xe5\x84\xaa\xe5\x85\x88" -"\xe3\x81\x97\xe3\x81\xa6\xe8\x89\xb2\xe3\x80\x85\xe3\x81\xaa\xe6" -"\xa9\x9f\xe8\x83\xbd\xe3\x82\x92\xe8\xa8\x80\xe8\xaa\x9e\xe8\xa6" -"\x81\xe7\xb4\xa0\xe3\x81\xa8\xe3\x81\x97\xe3\x81\xa6\xe5\x8f\x96" -"\xe3\x82\x8a\xe5\x85\xa5\xe3\x82\x8c\xe3\x82\x8b\xe5\xa0\xb4\xe5" -"\x90\x88\xe3\x81\x8c\xe5\xa4\x9a\xe3\x81\x84\xe3\x81\xae\xe3\x81" -"\xa7\xe3\x81\x99\xe3\x81\x8c\xe3\x80\x81\x50\x79\x74\x68\x6f\x6e" -"\x20\xe3\x81\xa7\xe3\x81\xaf\xe3\x81\x9d\xe3\x81\x86\xe3\x81\x84" -"\xe3\x81\xa3\xe3\x81\x9f\xe5\xb0\x8f\xe7\xb4\xb0\xe5\xb7\xa5\xe3" -"\x81\x8c\xe8\xbf\xbd\xe5\x8a\xa0\xe3\x81\x95\xe3\x82\x8c\xe3\x82" -"\x8b\xe3\x81\x93\xe3\x81\xa8\xe3\x81\xaf\xe3\x81\x82\xe3\x81\xbe" -"\xe3\x82\x8a\xe3\x81\x82\xe3\x82\x8a\xe3\x81\xbe\xe3\x81\x9b\xe3" -"\x82\x93\xe3\x80\x82\x0a\xe8\xa8\x80\xe8\xaa\x9e\xe8\x87\xaa\xe4" -"\xbd\x93\xe3\x81\xae\xe6\xa9\x9f\xe8\x83\xbd\xe3\x81\xaf\xe6\x9c" -"\x80\xe5\xb0\x8f\xe9\x99\x90\xe3\x81\xab\xe6\x8a\xbc\xe3\x81\x95" -"\xe3\x81\x88\xe3\x80\x81\xe5\xbf\x85\xe8\xa6\x81\xe3\x81\xaa\xe6" -"\xa9\x9f\xe8\x83\xbd\xe3\x81\xaf\xe6\x8b\xa1\xe5\xbc\xb5\xe3\x83" -"\xa2\xe3\x82\xb8\xe3\x83\xa5\xe3\x83\xbc\xe3\x83\xab\xe3\x81\xa8" -"\xe3\x81\x97\xe3\x81\xa6\xe8\xbf\xbd\xe5\x8a\xa0\xe3\x81\x99\xe3" -"\x82\x8b\xe3\x80\x81\xe3\x81\xa8\xe3\x81\x84\xe3\x81\x86\xe3\x81" -"\xae\xe3\x81\x8c\x20\x50\x79\x74\x68\x6f\x6e\x20\xe3\x81\xae\xe3" -"\x83\x9d\xe3\x83\xaa\xe3\x82\xb7\xe3\x83\xbc\xe3\x81\xa7\xe3\x81" -"\x99\xe3\x80\x82\x0a\x0a"), -'euc_kr': ( -"\xa1\xdd\x20\xc6\xc4\xc0\xcc\xbd\xe3\x28\x50\x79\x74\x68\x6f\x6e" -"\x29\xc0\xba\x20\xb9\xe8\xbf\xec\xb1\xe2\x20\xbd\xb1\xb0\xed\x2c" -"\x20\xb0\xad\xb7\xc2\xc7\xd1\x20\xc7\xc1\xb7\xce\xb1\xd7\xb7\xa1" -"\xb9\xd6\x20\xbe\xf0\xbe\xee\xc0\xd4\xb4\xcf\xb4\xd9\x2e\x20\xc6" -"\xc4\xc0\xcc\xbd\xe3\xc0\xba\x0a\xc8\xbf\xc0\xb2\xc0\xfb\xc0\xce" -"\x20\xb0\xed\xbc\xf6\xc1\xd8\x20\xb5\xa5\xc0\xcc\xc5\xcd\x20\xb1" -"\xb8\xc1\xb6\xbf\xcd\x20\xb0\xa3\xb4\xdc\xc7\xcf\xc1\xf6\xb8\xb8" -"\x20\xc8\xbf\xc0\xb2\xc0\xfb\xc0\xce\x20\xb0\xb4\xc3\xbc\xc1\xf6" -"\xc7\xe2\xc7\xc1\xb7\xce\xb1\xd7\xb7\xa1\xb9\xd6\xc0\xbb\x0a\xc1" -"\xf6\xbf\xf8\xc7\xd5\xb4\xcf\xb4\xd9\x2e\x20\xc6\xc4\xc0\xcc\xbd" -"\xe3\xc0\xc7\x20\xbf\xec\xbe\xc6\x28\xe9\xd0\xe4\xba\x29\xc7\xd1" -"\x20\xb9\xae\xb9\xfd\xb0\xfa\x20\xb5\xbf\xc0\xfb\x20\xc5\xb8\xc0" -"\xcc\xc7\xce\x2c\x20\xb1\xd7\xb8\xae\xb0\xed\x20\xc0\xce\xc5\xcd" -"\xc7\xc1\xb8\xae\xc6\xc3\x0a\xc8\xaf\xb0\xe6\xc0\xba\x20\xc6\xc4" -"\xc0\xcc\xbd\xe3\xc0\xbb\x20\xbd\xba\xc5\xa9\xb8\xb3\xc6\xc3\xb0" -"\xfa\x20\xbf\xa9\xb7\xaf\x20\xba\xd0\xbe\xdf\xbf\xa1\xbc\xad\xbf" -"\xcd\x20\xb4\xeb\xba\xce\xba\xd0\xc0\xc7\x20\xc7\xc3\xb7\xa7\xc6" -"\xfb\xbf\xa1\xbc\xad\xc0\xc7\x20\xba\xfc\xb8\xa5\x0a\xbe\xd6\xc7" -"\xc3\xb8\xae\xc4\xc9\xc0\xcc\xbc\xc7\x20\xb0\xb3\xb9\xdf\xc0\xbb" -"\x20\xc7\xd2\x20\xbc\xf6\x20\xc0\xd6\xb4\xc2\x20\xc0\xcc\xbb\xf3" -"\xc0\xfb\xc0\xce\x20\xbe\xf0\xbe\xee\xb7\xce\x20\xb8\xb8\xb5\xe9" -"\xbe\xee\xc1\xdd\xb4\xcf\xb4\xd9\x2e\x0a\x0a\xa1\xd9\xc3\xb9\xb0" -"\xa1\xb3\xa1\x3a\x20\xb3\xaf\xbe\xc6\xb6\xf3\x20\xa4\xd4\xa4\xb6" -"\xa4\xd0\xa4\xd4\xa4\xd4\xa4\xb6\xa4\xd0\xa4\xd4\xbe\xb1\x7e\x20" -"\xa4\xd4\xa4\xa4\xa4\xd2\xa4\xb7\xc5\xad\x21\x20\xa4\xd4\xa4\xa8" -"\xa4\xd1\xa4\xb7\xb1\xdd\xbe\xf8\xc0\xcc\x20\xc0\xfc\xa4\xd4\xa4" -"\xbe\xa4\xc8\xa4\xb2\xb4\xcf\xb4\xd9\x2e\x20\xa4\xd4\xa4\xb2\xa4" -"\xce\xa4\xaa\x2e\x20\xb1\xd7\xb7\xb1\xb0\xc5\x20\xa4\xd4\xa4\xb7" -"\xa4\xd1\xa4\xb4\xb4\xd9\x2e\x0a", -"\xe2\x97\x8e\x20\xed\x8c\x8c\xec\x9d\xb4\xec\x8d\xac\x28\x50\x79" -"\x74\x68\x6f\x6e\x29\xec\x9d\x80\x20\xeb\xb0\xb0\xec\x9a\xb0\xea" -"\xb8\xb0\x20\xec\x89\xbd\xea\xb3\xa0\x2c\x20\xea\xb0\x95\xeb\xa0" -"\xa5\xed\x95\x9c\x20\xed\x94\x84\xeb\xa1\x9c\xea\xb7\xb8\xeb\x9e" -"\x98\xeb\xb0\x8d\x20\xec\x96\xb8\xec\x96\xb4\xec\x9e\x85\xeb\x8b" -"\x88\xeb\x8b\xa4\x2e\x20\xed\x8c\x8c\xec\x9d\xb4\xec\x8d\xac\xec" -"\x9d\x80\x0a\xed\x9a\xa8\xec\x9c\xa8\xec\xa0\x81\xec\x9d\xb8\x20" -"\xea\xb3\xa0\xec\x88\x98\xec\xa4\x80\x20\xeb\x8d\xb0\xec\x9d\xb4" -"\xed\x84\xb0\x20\xea\xb5\xac\xec\xa1\xb0\xec\x99\x80\x20\xea\xb0" -"\x84\xeb\x8b\xa8\xed\x95\x98\xec\xa7\x80\xeb\xa7\x8c\x20\xed\x9a" -"\xa8\xec\x9c\xa8\xec\xa0\x81\xec\x9d\xb8\x20\xea\xb0\x9d\xec\xb2" -"\xb4\xec\xa7\x80\xed\x96\xa5\xed\x94\x84\xeb\xa1\x9c\xea\xb7\xb8" -"\xeb\x9e\x98\xeb\xb0\x8d\xec\x9d\x84\x0a\xec\xa7\x80\xec\x9b\x90" -"\xed\x95\xa9\xeb\x8b\x88\xeb\x8b\xa4\x2e\x20\xed\x8c\x8c\xec\x9d" -"\xb4\xec\x8d\xac\xec\x9d\x98\x20\xec\x9a\xb0\xec\x95\x84\x28\xe5" -"\x84\xaa\xe9\x9b\x85\x29\xed\x95\x9c\x20\xeb\xac\xb8\xeb\xb2\x95" -"\xea\xb3\xbc\x20\xeb\x8f\x99\xec\xa0\x81\x20\xed\x83\x80\xec\x9d" -"\xb4\xed\x95\x91\x2c\x20\xea\xb7\xb8\xeb\xa6\xac\xea\xb3\xa0\x20" -"\xec\x9d\xb8\xed\x84\xb0\xed\x94\x84\xeb\xa6\xac\xed\x8c\x85\x0a" -"\xed\x99\x98\xea\xb2\xbd\xec\x9d\x80\x20\xed\x8c\x8c\xec\x9d\xb4" -"\xec\x8d\xac\xec\x9d\x84\x20\xec\x8a\xa4\xed\x81\xac\xeb\xa6\xbd" -"\xed\x8c\x85\xea\xb3\xbc\x20\xec\x97\xac\xeb\x9f\xac\x20\xeb\xb6" -"\x84\xec\x95\xbc\xec\x97\x90\xec\x84\x9c\xec\x99\x80\x20\xeb\x8c" -"\x80\xeb\xb6\x80\xeb\xb6\x84\xec\x9d\x98\x20\xed\x94\x8c\xeb\x9e" -"\xab\xed\x8f\xbc\xec\x97\x90\xec\x84\x9c\xec\x9d\x98\x20\xeb\xb9" -"\xa0\xeb\xa5\xb8\x0a\xec\x95\xa0\xed\x94\x8c\xeb\xa6\xac\xec\xbc" -"\x80\xec\x9d\xb4\xec\x85\x98\x20\xea\xb0\x9c\xeb\xb0\x9c\xec\x9d" -"\x84\x20\xed\x95\xa0\x20\xec\x88\x98\x20\xec\x9e\x88\xeb\x8a\x94" -"\x20\xec\x9d\xb4\xec\x83\x81\xec\xa0\x81\xec\x9d\xb8\x20\xec\x96" -"\xb8\xec\x96\xb4\xeb\xa1\x9c\x20\xeb\xa7\x8c\xeb\x93\xa4\xec\x96" -"\xb4\xec\xa4\x8d\xeb\x8b\x88\xeb\x8b\xa4\x2e\x0a\x0a\xe2\x98\x86" -"\xec\xb2\xab\xea\xb0\x80\xeb\x81\x9d\x3a\x20\xeb\x82\xa0\xec\x95" -"\x84\xeb\x9d\xbc\x20\xec\x93\x94\xec\x93\x94\xec\x93\xa9\x7e\x20" -"\xeb\x8b\x81\xed\x81\xbc\x21\x20\xeb\x9c\xbd\xea\xb8\x88\xec\x97" -"\x86\xec\x9d\xb4\x20\xec\xa0\x84\xed\x99\xa5\xeb\x8b\x88\xeb\x8b" -"\xa4\x2e\x20\xeb\xb7\x81\x2e\x20\xea\xb7\xb8\xeb\x9f\xb0\xea\xb1" -"\xb0\x20\xec\x9d\x8e\xeb\x8b\xa4\x2e\x0a"), -'gb18030': ( -"\x50\x79\x74\x68\x6f\x6e\xa3\xa8\xc5\xc9\xc9\xad\xa3\xa9\xd3\xef" -"\xd1\xd4\xca\xc7\xd2\xbb\xd6\xd6\xb9\xa6\xc4\xdc\xc7\xbf\xb4\xf3" -"\xb6\xf8\xcd\xea\xc9\xc6\xb5\xc4\xcd\xa8\xd3\xc3\xd0\xcd\xbc\xc6" -"\xcb\xe3\xbb\xfa\xb3\xcc\xd0\xf2\xc9\xe8\xbc\xc6\xd3\xef\xd1\xd4" -"\xa3\xac\x0a\xd2\xd1\xbe\xad\xbe\xdf\xd3\xd0\xca\xae\xb6\xe0\xc4" -"\xea\xb5\xc4\xb7\xa2\xd5\xb9\xc0\xfa\xca\xb7\xa3\xac\xb3\xc9\xca" -"\xec\xc7\xd2\xce\xc8\xb6\xa8\xa1\xa3\xd5\xe2\xd6\xd6\xd3\xef\xd1" -"\xd4\xbe\xdf\xd3\xd0\xb7\xc7\xb3\xa3\xbc\xf2\xbd\xdd\xb6\xf8\xc7" -"\xe5\xce\xfa\x0a\xb5\xc4\xd3\xef\xb7\xa8\xcc\xd8\xb5\xe3\xa3\xac" -"\xca\xca\xba\xcf\xcd\xea\xb3\xc9\xb8\xf7\xd6\xd6\xb8\xdf\xb2\xe3" -"\xc8\xce\xce\xf1\xa3\xac\xbc\xb8\xba\xf5\xbf\xc9\xd2\xd4\xd4\xda" -"\xcb\xf9\xd3\xd0\xb5\xc4\xb2\xd9\xd7\xf7\xcf\xb5\xcd\xb3\xd6\xd0" -"\x0a\xd4\xcb\xd0\xd0\xa1\xa3\xd5\xe2\xd6\xd6\xd3\xef\xd1\xd4\xbc" -"\xf2\xb5\xa5\xb6\xf8\xc7\xbf\xb4\xf3\xa3\xac\xca\xca\xba\xcf\xb8" -"\xf7\xd6\xd6\xc8\xcb\xca\xbf\xd1\xa7\xcf\xb0\xca\xb9\xd3\xc3\xa1" -"\xa3\xc4\xbf\xc7\xb0\xa3\xac\xbb\xf9\xd3\xda\xd5\xe2\x0a\xd6\xd6" -"\xd3\xef\xd1\xd4\xb5\xc4\xcf\xe0\xb9\xd8\xbc\xbc\xca\xf5\xd5\xfd" -"\xd4\xda\xb7\xc9\xcb\xd9\xb5\xc4\xb7\xa2\xd5\xb9\xa3\xac\xd3\xc3" -"\xbb\xa7\xca\xfd\xc1\xbf\xbc\xb1\xbe\xe7\xc0\xa9\xb4\xf3\xa3\xac" -"\xcf\xe0\xb9\xd8\xb5\xc4\xd7\xca\xd4\xb4\xb7\xc7\xb3\xa3\xb6\xe0" -"\xa1\xa3\x0a\xc8\xe7\xba\xce\xd4\xda\x20\x50\x79\x74\x68\x6f\x6e" -"\x20\xd6\xd0\xca\xb9\xd3\xc3\xbc\xc8\xd3\xd0\xb5\xc4\x20\x43\x20" -"\x6c\x69\x62\x72\x61\x72\x79\x3f\x0a\xa1\xa1\xd4\xda\xd9\x59\xd3" -"\x8d\xbf\xc6\xbc\xbc\xbf\xec\xcb\xd9\xb0\x6c\xd5\xb9\xb5\xc4\xbd" -"\xf1\xcc\xec\x2c\x20\xe9\x5f\xb0\x6c\xbc\xb0\x9c\x79\xd4\x87\xdc" -"\x9b\xf3\x77\xb5\xc4\xcb\xd9\xb6\xc8\xca\xc7\xb2\xbb\xc8\xdd\xba" -"\xf6\xd2\x95\xb5\xc4\x0a\xd5\x6e\xee\x7d\x2e\x20\x9e\xe9\xbc\xd3" -"\xbf\xec\xe9\x5f\xb0\x6c\xbc\xb0\x9c\x79\xd4\x87\xb5\xc4\xcb\xd9" -"\xb6\xc8\x2c\x20\xce\xd2\x82\x83\xb1\xe3\xb3\xa3\xcf\xa3\xcd\xfb" -"\xc4\xdc\xc0\xfb\xd3\xc3\xd2\xbb\xd0\xa9\xd2\xd1\xe9\x5f\xb0\x6c" -"\xba\xc3\xb5\xc4\x0a\x6c\x69\x62\x72\x61\x72\x79\x2c\x20\x81\x4b" -"\xd3\xd0\xd2\xbb\x82\x80\x20\x66\x61\x73\x74\x20\x70\x72\x6f\x74" -"\x6f\x74\x79\x70\x69\x6e\x67\x20\xb5\xc4\x20\x70\x72\x6f\x67\x72" -"\x61\x6d\x6d\x69\x6e\x67\x20\x6c\x61\x6e\x67\x75\x61\x67\x65\x20" -"\xbf\xc9\x0a\xb9\xa9\xca\xb9\xd3\xc3\x2e\x20\xc4\xbf\xc7\xb0\xd3" -"\xd0\xd4\x53\xd4\x53\xb6\xe0\xb6\xe0\xb5\xc4\x20\x6c\x69\x62\x72" -"\x61\x72\x79\x20\xca\xc7\xd2\xd4\x20\x43\x20\x8c\x91\xb3\xc9\x2c" -"\x20\xb6\xf8\x20\x50\x79\x74\x68\x6f\x6e\x20\xca\xc7\xd2\xbb\x82" -"\x80\x0a\x66\x61\x73\x74\x20\x70\x72\x6f\x74\x6f\x74\x79\x70\x69" -"\x6e\x67\x20\xb5\xc4\x20\x70\x72\x6f\x67\x72\x61\x6d\x6d\x69\x6e" -"\x67\x20\x6c\x61\x6e\x67\x75\x61\x67\x65\x2e\x20\xb9\xca\xce\xd2" -"\x82\x83\xcf\xa3\xcd\xfb\xc4\xdc\x8c\xa2\xbc\xc8\xd3\xd0\xb5\xc4" -"\x0a\x43\x20\x6c\x69\x62\x72\x61\x72\x79\x20\xc4\xc3\xb5\xbd\x20" -"\x50\x79\x74\x68\x6f\x6e\x20\xb5\xc4\xad\x68\xbe\xb3\xd6\xd0\x9c" -"\x79\xd4\x87\xbc\xb0\xd5\xfb\xba\xcf\x2e\x20\xc6\xe4\xd6\xd0\xd7" -"\xee\xd6\xf7\xd2\xaa\xd2\xb2\xca\xc7\xce\xd2\x82\x83\xcb\xf9\x0a" -"\xd2\xaa\xd3\x91\xd5\x93\xb5\xc4\x86\x96\xee\x7d\xbe\xcd\xca\xc7" -"\x3a\x0a\x83\x35\xc7\x31\x83\x33\x9a\x33\x83\x32\xb1\x31\x83\x33" -"\x95\x31\x20\x82\x37\xd1\x36\x83\x30\x8c\x34\x83\x36\x84\x33\x20" -"\x82\x38\x89\x35\x82\x38\xfb\x36\x83\x33\x95\x35\x20\x83\x33\xd5" -"\x31\x82\x39\x81\x35\x20\x83\x30\xfd\x39\x83\x33\x86\x30\x20\x83" -"\x34\xdc\x33\x83\x35\xf6\x37\x83\x35\x97\x35\x20\x83\x35\xf9\x35" -"\x83\x30\x91\x39\x82\x38\x83\x39\x82\x39\xfc\x33\x83\x30\xf0\x34" -"\x20\x83\x32\xeb\x39\x83\x32\xeb\x35\x82\x39\x83\x39\x2e\x0a\x0a", -"\x50\x79\x74\x68\x6f\x6e\xef\xbc\x88\xe6\xb4\xbe\xe6\xa3\xae\xef" -"\xbc\x89\xe8\xaf\xad\xe8\xa8\x80\xe6\x98\xaf\xe4\xb8\x80\xe7\xa7" -"\x8d\xe5\x8a\x9f\xe8\x83\xbd\xe5\xbc\xba\xe5\xa4\xa7\xe8\x80\x8c" -"\xe5\xae\x8c\xe5\x96\x84\xe7\x9a\x84\xe9\x80\x9a\xe7\x94\xa8\xe5" -"\x9e\x8b\xe8\xae\xa1\xe7\xae\x97\xe6\x9c\xba\xe7\xa8\x8b\xe5\xba" -"\x8f\xe8\xae\xbe\xe8\xae\xa1\xe8\xaf\xad\xe8\xa8\x80\xef\xbc\x8c" -"\x0a\xe5\xb7\xb2\xe7\xbb\x8f\xe5\x85\xb7\xe6\x9c\x89\xe5\x8d\x81" -"\xe5\xa4\x9a\xe5\xb9\xb4\xe7\x9a\x84\xe5\x8f\x91\xe5\xb1\x95\xe5" -"\x8e\x86\xe5\x8f\xb2\xef\xbc\x8c\xe6\x88\x90\xe7\x86\x9f\xe4\xb8" -"\x94\xe7\xa8\xb3\xe5\xae\x9a\xe3\x80\x82\xe8\xbf\x99\xe7\xa7\x8d" -"\xe8\xaf\xad\xe8\xa8\x80\xe5\x85\xb7\xe6\x9c\x89\xe9\x9d\x9e\xe5" -"\xb8\xb8\xe7\xae\x80\xe6\x8d\xb7\xe8\x80\x8c\xe6\xb8\x85\xe6\x99" -"\xb0\x0a\xe7\x9a\x84\xe8\xaf\xad\xe6\xb3\x95\xe7\x89\xb9\xe7\x82" -"\xb9\xef\xbc\x8c\xe9\x80\x82\xe5\x90\x88\xe5\xae\x8c\xe6\x88\x90" -"\xe5\x90\x84\xe7\xa7\x8d\xe9\xab\x98\xe5\xb1\x82\xe4\xbb\xbb\xe5" -"\x8a\xa1\xef\xbc\x8c\xe5\x87\xa0\xe4\xb9\x8e\xe5\x8f\xaf\xe4\xbb" -"\xa5\xe5\x9c\xa8\xe6\x89\x80\xe6\x9c\x89\xe7\x9a\x84\xe6\x93\x8d" -"\xe4\xbd\x9c\xe7\xb3\xbb\xe7\xbb\x9f\xe4\xb8\xad\x0a\xe8\xbf\x90" -"\xe8\xa1\x8c\xe3\x80\x82\xe8\xbf\x99\xe7\xa7\x8d\xe8\xaf\xad\xe8" -"\xa8\x80\xe7\xae\x80\xe5\x8d\x95\xe8\x80\x8c\xe5\xbc\xba\xe5\xa4" -"\xa7\xef\xbc\x8c\xe9\x80\x82\xe5\x90\x88\xe5\x90\x84\xe7\xa7\x8d" -"\xe4\xba\xba\xe5\xa3\xab\xe5\xad\xa6\xe4\xb9\xa0\xe4\xbd\xbf\xe7" -"\x94\xa8\xe3\x80\x82\xe7\x9b\xae\xe5\x89\x8d\xef\xbc\x8c\xe5\x9f" -"\xba\xe4\xba\x8e\xe8\xbf\x99\x0a\xe7\xa7\x8d\xe8\xaf\xad\xe8\xa8" -"\x80\xe7\x9a\x84\xe7\x9b\xb8\xe5\x85\xb3\xe6\x8a\x80\xe6\x9c\xaf" -"\xe6\xad\xa3\xe5\x9c\xa8\xe9\xa3\x9e\xe9\x80\x9f\xe7\x9a\x84\xe5" -"\x8f\x91\xe5\xb1\x95\xef\xbc\x8c\xe7\x94\xa8\xe6\x88\xb7\xe6\x95" -"\xb0\xe9\x87\x8f\xe6\x80\xa5\xe5\x89\xa7\xe6\x89\xa9\xe5\xa4\xa7" -"\xef\xbc\x8c\xe7\x9b\xb8\xe5\x85\xb3\xe7\x9a\x84\xe8\xb5\x84\xe6" -"\xba\x90\xe9\x9d\x9e\xe5\xb8\xb8\xe5\xa4\x9a\xe3\x80\x82\x0a\xe5" -"\xa6\x82\xe4\xbd\x95\xe5\x9c\xa8\x20\x50\x79\x74\x68\x6f\x6e\x20" -"\xe4\xb8\xad\xe4\xbd\xbf\xe7\x94\xa8\xe6\x97\xa2\xe6\x9c\x89\xe7" -"\x9a\x84\x20\x43\x20\x6c\x69\x62\x72\x61\x72\x79\x3f\x0a\xe3\x80" -"\x80\xe5\x9c\xa8\xe8\xb3\x87\xe8\xa8\x8a\xe7\xa7\x91\xe6\x8a\x80" -"\xe5\xbf\xab\xe9\x80\x9f\xe7\x99\xbc\xe5\xb1\x95\xe7\x9a\x84\xe4" -"\xbb\x8a\xe5\xa4\xa9\x2c\x20\xe9\x96\x8b\xe7\x99\xbc\xe5\x8f\x8a" -"\xe6\xb8\xac\xe8\xa9\xa6\xe8\xbb\x9f\xe9\xab\x94\xe7\x9a\x84\xe9" -"\x80\x9f\xe5\xba\xa6\xe6\x98\xaf\xe4\xb8\x8d\xe5\xae\xb9\xe5\xbf" -"\xbd\xe8\xa6\x96\xe7\x9a\x84\x0a\xe8\xaa\xb2\xe9\xa1\x8c\x2e\x20" -"\xe7\x82\xba\xe5\x8a\xa0\xe5\xbf\xab\xe9\x96\x8b\xe7\x99\xbc\xe5" -"\x8f\x8a\xe6\xb8\xac\xe8\xa9\xa6\xe7\x9a\x84\xe9\x80\x9f\xe5\xba" -"\xa6\x2c\x20\xe6\x88\x91\xe5\x80\x91\xe4\xbe\xbf\xe5\xb8\xb8\xe5" -"\xb8\x8c\xe6\x9c\x9b\xe8\x83\xbd\xe5\x88\xa9\xe7\x94\xa8\xe4\xb8" -"\x80\xe4\xba\x9b\xe5\xb7\xb2\xe9\x96\x8b\xe7\x99\xbc\xe5\xa5\xbd" -"\xe7\x9a\x84\x0a\x6c\x69\x62\x72\x61\x72\x79\x2c\x20\xe4\xb8\xa6" -"\xe6\x9c\x89\xe4\xb8\x80\xe5\x80\x8b\x20\x66\x61\x73\x74\x20\x70" -"\x72\x6f\x74\x6f\x74\x79\x70\x69\x6e\x67\x20\xe7\x9a\x84\x20\x70" -"\x72\x6f\x67\x72\x61\x6d\x6d\x69\x6e\x67\x20\x6c\x61\x6e\x67\x75" -"\x61\x67\x65\x20\xe5\x8f\xaf\x0a\xe4\xbe\x9b\xe4\xbd\xbf\xe7\x94" -"\xa8\x2e\x20\xe7\x9b\xae\xe5\x89\x8d\xe6\x9c\x89\xe8\xa8\xb1\xe8" -"\xa8\xb1\xe5\xa4\x9a\xe5\xa4\x9a\xe7\x9a\x84\x20\x6c\x69\x62\x72" -"\x61\x72\x79\x20\xe6\x98\xaf\xe4\xbb\xa5\x20\x43\x20\xe5\xaf\xab" -"\xe6\x88\x90\x2c\x20\xe8\x80\x8c\x20\x50\x79\x74\x68\x6f\x6e\x20" -"\xe6\x98\xaf\xe4\xb8\x80\xe5\x80\x8b\x0a\x66\x61\x73\x74\x20\x70" -"\x72\x6f\x74\x6f\x74\x79\x70\x69\x6e\x67\x20\xe7\x9a\x84\x20\x70" -"\x72\x6f\x67\x72\x61\x6d\x6d\x69\x6e\x67\x20\x6c\x61\x6e\x67\x75" -"\x61\x67\x65\x2e\x20\xe6\x95\x85\xe6\x88\x91\xe5\x80\x91\xe5\xb8" -"\x8c\xe6\x9c\x9b\xe8\x83\xbd\xe5\xb0\x87\xe6\x97\xa2\xe6\x9c\x89" -"\xe7\x9a\x84\x0a\x43\x20\x6c\x69\x62\x72\x61\x72\x79\x20\xe6\x8b" -"\xbf\xe5\x88\xb0\x20\x50\x79\x74\x68\x6f\x6e\x20\xe7\x9a\x84\xe7" -"\x92\xb0\xe5\xa2\x83\xe4\xb8\xad\xe6\xb8\xac\xe8\xa9\xa6\xe5\x8f" -"\x8a\xe6\x95\xb4\xe5\x90\x88\x2e\x20\xe5\x85\xb6\xe4\xb8\xad\xe6" -"\x9c\x80\xe4\xb8\xbb\xe8\xa6\x81\xe4\xb9\x9f\xe6\x98\xaf\xe6\x88" -"\x91\xe5\x80\x91\xe6\x89\x80\x0a\xe8\xa6\x81\xe8\xa8\x8e\xe8\xab" -"\x96\xe7\x9a\x84\xe5\x95\x8f\xe9\xa1\x8c\xe5\xb0\xb1\xe6\x98\xaf" -"\x3a\x0a\xed\x8c\x8c\xec\x9d\xb4\xec\x8d\xac\xec\x9d\x80\x20\xea" -"\xb0\x95\xeb\xa0\xa5\xed\x95\x9c\x20\xea\xb8\xb0\xeb\x8a\xa5\xec" -"\x9d\x84\x20\xec\xa7\x80\xeb\x8b\x8c\x20\xeb\xb2\x94\xec\x9a\xa9" -"\x20\xec\xbb\xb4\xed\x93\xa8\xed\x84\xb0\x20\xed\x94\x84\xeb\xa1" -"\x9c\xea\xb7\xb8\xeb\x9e\x98\xeb\xb0\x8d\x20\xec\x96\xb8\xec\x96" -"\xb4\xeb\x8b\xa4\x2e\x0a\x0a"), -'gb2312': ( -"\x50\x79\x74\x68\x6f\x6e\xa3\xa8\xc5\xc9\xc9\xad\xa3\xa9\xd3\xef" -"\xd1\xd4\xca\xc7\xd2\xbb\xd6\xd6\xb9\xa6\xc4\xdc\xc7\xbf\xb4\xf3" -"\xb6\xf8\xcd\xea\xc9\xc6\xb5\xc4\xcd\xa8\xd3\xc3\xd0\xcd\xbc\xc6" -"\xcb\xe3\xbb\xfa\xb3\xcc\xd0\xf2\xc9\xe8\xbc\xc6\xd3\xef\xd1\xd4" -"\xa3\xac\x0a\xd2\xd1\xbe\xad\xbe\xdf\xd3\xd0\xca\xae\xb6\xe0\xc4" -"\xea\xb5\xc4\xb7\xa2\xd5\xb9\xc0\xfa\xca\xb7\xa3\xac\xb3\xc9\xca" -"\xec\xc7\xd2\xce\xc8\xb6\xa8\xa1\xa3\xd5\xe2\xd6\xd6\xd3\xef\xd1" -"\xd4\xbe\xdf\xd3\xd0\xb7\xc7\xb3\xa3\xbc\xf2\xbd\xdd\xb6\xf8\xc7" -"\xe5\xce\xfa\x0a\xb5\xc4\xd3\xef\xb7\xa8\xcc\xd8\xb5\xe3\xa3\xac" -"\xca\xca\xba\xcf\xcd\xea\xb3\xc9\xb8\xf7\xd6\xd6\xb8\xdf\xb2\xe3" -"\xc8\xce\xce\xf1\xa3\xac\xbc\xb8\xba\xf5\xbf\xc9\xd2\xd4\xd4\xda" -"\xcb\xf9\xd3\xd0\xb5\xc4\xb2\xd9\xd7\xf7\xcf\xb5\xcd\xb3\xd6\xd0" -"\x0a\xd4\xcb\xd0\xd0\xa1\xa3\xd5\xe2\xd6\xd6\xd3\xef\xd1\xd4\xbc" -"\xf2\xb5\xa5\xb6\xf8\xc7\xbf\xb4\xf3\xa3\xac\xca\xca\xba\xcf\xb8" -"\xf7\xd6\xd6\xc8\xcb\xca\xbf\xd1\xa7\xcf\xb0\xca\xb9\xd3\xc3\xa1" -"\xa3\xc4\xbf\xc7\xb0\xa3\xac\xbb\xf9\xd3\xda\xd5\xe2\x0a\xd6\xd6" -"\xd3\xef\xd1\xd4\xb5\xc4\xcf\xe0\xb9\xd8\xbc\xbc\xca\xf5\xd5\xfd" -"\xd4\xda\xb7\xc9\xcb\xd9\xb5\xc4\xb7\xa2\xd5\xb9\xa3\xac\xd3\xc3" -"\xbb\xa7\xca\xfd\xc1\xbf\xbc\xb1\xbe\xe7\xc0\xa9\xb4\xf3\xa3\xac" -"\xcf\xe0\xb9\xd8\xb5\xc4\xd7\xca\xd4\xb4\xb7\xc7\xb3\xa3\xb6\xe0" -"\xa1\xa3\x0a\x0a", -"\x50\x79\x74\x68\x6f\x6e\xef\xbc\x88\xe6\xb4\xbe\xe6\xa3\xae\xef" -"\xbc\x89\xe8\xaf\xad\xe8\xa8\x80\xe6\x98\xaf\xe4\xb8\x80\xe7\xa7" -"\x8d\xe5\x8a\x9f\xe8\x83\xbd\xe5\xbc\xba\xe5\xa4\xa7\xe8\x80\x8c" -"\xe5\xae\x8c\xe5\x96\x84\xe7\x9a\x84\xe9\x80\x9a\xe7\x94\xa8\xe5" -"\x9e\x8b\xe8\xae\xa1\xe7\xae\x97\xe6\x9c\xba\xe7\xa8\x8b\xe5\xba" -"\x8f\xe8\xae\xbe\xe8\xae\xa1\xe8\xaf\xad\xe8\xa8\x80\xef\xbc\x8c" -"\x0a\xe5\xb7\xb2\xe7\xbb\x8f\xe5\x85\xb7\xe6\x9c\x89\xe5\x8d\x81" -"\xe5\xa4\x9a\xe5\xb9\xb4\xe7\x9a\x84\xe5\x8f\x91\xe5\xb1\x95\xe5" -"\x8e\x86\xe5\x8f\xb2\xef\xbc\x8c\xe6\x88\x90\xe7\x86\x9f\xe4\xb8" -"\x94\xe7\xa8\xb3\xe5\xae\x9a\xe3\x80\x82\xe8\xbf\x99\xe7\xa7\x8d" -"\xe8\xaf\xad\xe8\xa8\x80\xe5\x85\xb7\xe6\x9c\x89\xe9\x9d\x9e\xe5" -"\xb8\xb8\xe7\xae\x80\xe6\x8d\xb7\xe8\x80\x8c\xe6\xb8\x85\xe6\x99" -"\xb0\x0a\xe7\x9a\x84\xe8\xaf\xad\xe6\xb3\x95\xe7\x89\xb9\xe7\x82" -"\xb9\xef\xbc\x8c\xe9\x80\x82\xe5\x90\x88\xe5\xae\x8c\xe6\x88\x90" -"\xe5\x90\x84\xe7\xa7\x8d\xe9\xab\x98\xe5\xb1\x82\xe4\xbb\xbb\xe5" -"\x8a\xa1\xef\xbc\x8c\xe5\x87\xa0\xe4\xb9\x8e\xe5\x8f\xaf\xe4\xbb" -"\xa5\xe5\x9c\xa8\xe6\x89\x80\xe6\x9c\x89\xe7\x9a\x84\xe6\x93\x8d" -"\xe4\xbd\x9c\xe7\xb3\xbb\xe7\xbb\x9f\xe4\xb8\xad\x0a\xe8\xbf\x90" -"\xe8\xa1\x8c\xe3\x80\x82\xe8\xbf\x99\xe7\xa7\x8d\xe8\xaf\xad\xe8" -"\xa8\x80\xe7\xae\x80\xe5\x8d\x95\xe8\x80\x8c\xe5\xbc\xba\xe5\xa4" -"\xa7\xef\xbc\x8c\xe9\x80\x82\xe5\x90\x88\xe5\x90\x84\xe7\xa7\x8d" -"\xe4\xba\xba\xe5\xa3\xab\xe5\xad\xa6\xe4\xb9\xa0\xe4\xbd\xbf\xe7" -"\x94\xa8\xe3\x80\x82\xe7\x9b\xae\xe5\x89\x8d\xef\xbc\x8c\xe5\x9f" -"\xba\xe4\xba\x8e\xe8\xbf\x99\x0a\xe7\xa7\x8d\xe8\xaf\xad\xe8\xa8" -"\x80\xe7\x9a\x84\xe7\x9b\xb8\xe5\x85\xb3\xe6\x8a\x80\xe6\x9c\xaf" -"\xe6\xad\xa3\xe5\x9c\xa8\xe9\xa3\x9e\xe9\x80\x9f\xe7\x9a\x84\xe5" -"\x8f\x91\xe5\xb1\x95\xef\xbc\x8c\xe7\x94\xa8\xe6\x88\xb7\xe6\x95" -"\xb0\xe9\x87\x8f\xe6\x80\xa5\xe5\x89\xa7\xe6\x89\xa9\xe5\xa4\xa7" -"\xef\xbc\x8c\xe7\x9b\xb8\xe5\x85\xb3\xe7\x9a\x84\xe8\xb5\x84\xe6" -"\xba\x90\xe9\x9d\x9e\xe5\xb8\xb8\xe5\xa4\x9a\xe3\x80\x82\x0a\x0a"), -'gbk': ( -"\x50\x79\x74\x68\x6f\x6e\xa3\xa8\xc5\xc9\xc9\xad\xa3\xa9\xd3\xef" -"\xd1\xd4\xca\xc7\xd2\xbb\xd6\xd6\xb9\xa6\xc4\xdc\xc7\xbf\xb4\xf3" -"\xb6\xf8\xcd\xea\xc9\xc6\xb5\xc4\xcd\xa8\xd3\xc3\xd0\xcd\xbc\xc6" -"\xcb\xe3\xbb\xfa\xb3\xcc\xd0\xf2\xc9\xe8\xbc\xc6\xd3\xef\xd1\xd4" -"\xa3\xac\x0a\xd2\xd1\xbe\xad\xbe\xdf\xd3\xd0\xca\xae\xb6\xe0\xc4" -"\xea\xb5\xc4\xb7\xa2\xd5\xb9\xc0\xfa\xca\xb7\xa3\xac\xb3\xc9\xca" -"\xec\xc7\xd2\xce\xc8\xb6\xa8\xa1\xa3\xd5\xe2\xd6\xd6\xd3\xef\xd1" -"\xd4\xbe\xdf\xd3\xd0\xb7\xc7\xb3\xa3\xbc\xf2\xbd\xdd\xb6\xf8\xc7" -"\xe5\xce\xfa\x0a\xb5\xc4\xd3\xef\xb7\xa8\xcc\xd8\xb5\xe3\xa3\xac" -"\xca\xca\xba\xcf\xcd\xea\xb3\xc9\xb8\xf7\xd6\xd6\xb8\xdf\xb2\xe3" -"\xc8\xce\xce\xf1\xa3\xac\xbc\xb8\xba\xf5\xbf\xc9\xd2\xd4\xd4\xda" -"\xcb\xf9\xd3\xd0\xb5\xc4\xb2\xd9\xd7\xf7\xcf\xb5\xcd\xb3\xd6\xd0" -"\x0a\xd4\xcb\xd0\xd0\xa1\xa3\xd5\xe2\xd6\xd6\xd3\xef\xd1\xd4\xbc" -"\xf2\xb5\xa5\xb6\xf8\xc7\xbf\xb4\xf3\xa3\xac\xca\xca\xba\xcf\xb8" -"\xf7\xd6\xd6\xc8\xcb\xca\xbf\xd1\xa7\xcf\xb0\xca\xb9\xd3\xc3\xa1" -"\xa3\xc4\xbf\xc7\xb0\xa3\xac\xbb\xf9\xd3\xda\xd5\xe2\x0a\xd6\xd6" -"\xd3\xef\xd1\xd4\xb5\xc4\xcf\xe0\xb9\xd8\xbc\xbc\xca\xf5\xd5\xfd" -"\xd4\xda\xb7\xc9\xcb\xd9\xb5\xc4\xb7\xa2\xd5\xb9\xa3\xac\xd3\xc3" -"\xbb\xa7\xca\xfd\xc1\xbf\xbc\xb1\xbe\xe7\xc0\xa9\xb4\xf3\xa3\xac" -"\xcf\xe0\xb9\xd8\xb5\xc4\xd7\xca\xd4\xb4\xb7\xc7\xb3\xa3\xb6\xe0" -"\xa1\xa3\x0a\xc8\xe7\xba\xce\xd4\xda\x20\x50\x79\x74\x68\x6f\x6e" -"\x20\xd6\xd0\xca\xb9\xd3\xc3\xbc\xc8\xd3\xd0\xb5\xc4\x20\x43\x20" -"\x6c\x69\x62\x72\x61\x72\x79\x3f\x0a\xa1\xa1\xd4\xda\xd9\x59\xd3" -"\x8d\xbf\xc6\xbc\xbc\xbf\xec\xcb\xd9\xb0\x6c\xd5\xb9\xb5\xc4\xbd" -"\xf1\xcc\xec\x2c\x20\xe9\x5f\xb0\x6c\xbc\xb0\x9c\x79\xd4\x87\xdc" -"\x9b\xf3\x77\xb5\xc4\xcb\xd9\xb6\xc8\xca\xc7\xb2\xbb\xc8\xdd\xba" -"\xf6\xd2\x95\xb5\xc4\x0a\xd5\x6e\xee\x7d\x2e\x20\x9e\xe9\xbc\xd3" -"\xbf\xec\xe9\x5f\xb0\x6c\xbc\xb0\x9c\x79\xd4\x87\xb5\xc4\xcb\xd9" -"\xb6\xc8\x2c\x20\xce\xd2\x82\x83\xb1\xe3\xb3\xa3\xcf\xa3\xcd\xfb" -"\xc4\xdc\xc0\xfb\xd3\xc3\xd2\xbb\xd0\xa9\xd2\xd1\xe9\x5f\xb0\x6c" -"\xba\xc3\xb5\xc4\x0a\x6c\x69\x62\x72\x61\x72\x79\x2c\x20\x81\x4b" -"\xd3\xd0\xd2\xbb\x82\x80\x20\x66\x61\x73\x74\x20\x70\x72\x6f\x74" -"\x6f\x74\x79\x70\x69\x6e\x67\x20\xb5\xc4\x20\x70\x72\x6f\x67\x72" -"\x61\x6d\x6d\x69\x6e\x67\x20\x6c\x61\x6e\x67\x75\x61\x67\x65\x20" -"\xbf\xc9\x0a\xb9\xa9\xca\xb9\xd3\xc3\x2e\x20\xc4\xbf\xc7\xb0\xd3" -"\xd0\xd4\x53\xd4\x53\xb6\xe0\xb6\xe0\xb5\xc4\x20\x6c\x69\x62\x72" -"\x61\x72\x79\x20\xca\xc7\xd2\xd4\x20\x43\x20\x8c\x91\xb3\xc9\x2c" -"\x20\xb6\xf8\x20\x50\x79\x74\x68\x6f\x6e\x20\xca\xc7\xd2\xbb\x82" -"\x80\x0a\x66\x61\x73\x74\x20\x70\x72\x6f\x74\x6f\x74\x79\x70\x69" -"\x6e\x67\x20\xb5\xc4\x20\x70\x72\x6f\x67\x72\x61\x6d\x6d\x69\x6e" -"\x67\x20\x6c\x61\x6e\x67\x75\x61\x67\x65\x2e\x20\xb9\xca\xce\xd2" -"\x82\x83\xcf\xa3\xcd\xfb\xc4\xdc\x8c\xa2\xbc\xc8\xd3\xd0\xb5\xc4" -"\x0a\x43\x20\x6c\x69\x62\x72\x61\x72\x79\x20\xc4\xc3\xb5\xbd\x20" -"\x50\x79\x74\x68\x6f\x6e\x20\xb5\xc4\xad\x68\xbe\xb3\xd6\xd0\x9c" -"\x79\xd4\x87\xbc\xb0\xd5\xfb\xba\xcf\x2e\x20\xc6\xe4\xd6\xd0\xd7" -"\xee\xd6\xf7\xd2\xaa\xd2\xb2\xca\xc7\xce\xd2\x82\x83\xcb\xf9\x0a" -"\xd2\xaa\xd3\x91\xd5\x93\xb5\xc4\x86\x96\xee\x7d\xbe\xcd\xca\xc7" -"\x3a\x0a\x0a", -"\x50\x79\x74\x68\x6f\x6e\xef\xbc\x88\xe6\xb4\xbe\xe6\xa3\xae\xef" -"\xbc\x89\xe8\xaf\xad\xe8\xa8\x80\xe6\x98\xaf\xe4\xb8\x80\xe7\xa7" -"\x8d\xe5\x8a\x9f\xe8\x83\xbd\xe5\xbc\xba\xe5\xa4\xa7\xe8\x80\x8c" -"\xe5\xae\x8c\xe5\x96\x84\xe7\x9a\x84\xe9\x80\x9a\xe7\x94\xa8\xe5" -"\x9e\x8b\xe8\xae\xa1\xe7\xae\x97\xe6\x9c\xba\xe7\xa8\x8b\xe5\xba" -"\x8f\xe8\xae\xbe\xe8\xae\xa1\xe8\xaf\xad\xe8\xa8\x80\xef\xbc\x8c" -"\x0a\xe5\xb7\xb2\xe7\xbb\x8f\xe5\x85\xb7\xe6\x9c\x89\xe5\x8d\x81" -"\xe5\xa4\x9a\xe5\xb9\xb4\xe7\x9a\x84\xe5\x8f\x91\xe5\xb1\x95\xe5" -"\x8e\x86\xe5\x8f\xb2\xef\xbc\x8c\xe6\x88\x90\xe7\x86\x9f\xe4\xb8" -"\x94\xe7\xa8\xb3\xe5\xae\x9a\xe3\x80\x82\xe8\xbf\x99\xe7\xa7\x8d" -"\xe8\xaf\xad\xe8\xa8\x80\xe5\x85\xb7\xe6\x9c\x89\xe9\x9d\x9e\xe5" -"\xb8\xb8\xe7\xae\x80\xe6\x8d\xb7\xe8\x80\x8c\xe6\xb8\x85\xe6\x99" -"\xb0\x0a\xe7\x9a\x84\xe8\xaf\xad\xe6\xb3\x95\xe7\x89\xb9\xe7\x82" -"\xb9\xef\xbc\x8c\xe9\x80\x82\xe5\x90\x88\xe5\xae\x8c\xe6\x88\x90" -"\xe5\x90\x84\xe7\xa7\x8d\xe9\xab\x98\xe5\xb1\x82\xe4\xbb\xbb\xe5" -"\x8a\xa1\xef\xbc\x8c\xe5\x87\xa0\xe4\xb9\x8e\xe5\x8f\xaf\xe4\xbb" -"\xa5\xe5\x9c\xa8\xe6\x89\x80\xe6\x9c\x89\xe7\x9a\x84\xe6\x93\x8d" -"\xe4\xbd\x9c\xe7\xb3\xbb\xe7\xbb\x9f\xe4\xb8\xad\x0a\xe8\xbf\x90" -"\xe8\xa1\x8c\xe3\x80\x82\xe8\xbf\x99\xe7\xa7\x8d\xe8\xaf\xad\xe8" -"\xa8\x80\xe7\xae\x80\xe5\x8d\x95\xe8\x80\x8c\xe5\xbc\xba\xe5\xa4" -"\xa7\xef\xbc\x8c\xe9\x80\x82\xe5\x90\x88\xe5\x90\x84\xe7\xa7\x8d" -"\xe4\xba\xba\xe5\xa3\xab\xe5\xad\xa6\xe4\xb9\xa0\xe4\xbd\xbf\xe7" -"\x94\xa8\xe3\x80\x82\xe7\x9b\xae\xe5\x89\x8d\xef\xbc\x8c\xe5\x9f" -"\xba\xe4\xba\x8e\xe8\xbf\x99\x0a\xe7\xa7\x8d\xe8\xaf\xad\xe8\xa8" -"\x80\xe7\x9a\x84\xe7\x9b\xb8\xe5\x85\xb3\xe6\x8a\x80\xe6\x9c\xaf" -"\xe6\xad\xa3\xe5\x9c\xa8\xe9\xa3\x9e\xe9\x80\x9f\xe7\x9a\x84\xe5" -"\x8f\x91\xe5\xb1\x95\xef\xbc\x8c\xe7\x94\xa8\xe6\x88\xb7\xe6\x95" -"\xb0\xe9\x87\x8f\xe6\x80\xa5\xe5\x89\xa7\xe6\x89\xa9\xe5\xa4\xa7" -"\xef\xbc\x8c\xe7\x9b\xb8\xe5\x85\xb3\xe7\x9a\x84\xe8\xb5\x84\xe6" -"\xba\x90\xe9\x9d\x9e\xe5\xb8\xb8\xe5\xa4\x9a\xe3\x80\x82\x0a\xe5" -"\xa6\x82\xe4\xbd\x95\xe5\x9c\xa8\x20\x50\x79\x74\x68\x6f\x6e\x20" -"\xe4\xb8\xad\xe4\xbd\xbf\xe7\x94\xa8\xe6\x97\xa2\xe6\x9c\x89\xe7" -"\x9a\x84\x20\x43\x20\x6c\x69\x62\x72\x61\x72\x79\x3f\x0a\xe3\x80" -"\x80\xe5\x9c\xa8\xe8\xb3\x87\xe8\xa8\x8a\xe7\xa7\x91\xe6\x8a\x80" -"\xe5\xbf\xab\xe9\x80\x9f\xe7\x99\xbc\xe5\xb1\x95\xe7\x9a\x84\xe4" -"\xbb\x8a\xe5\xa4\xa9\x2c\x20\xe9\x96\x8b\xe7\x99\xbc\xe5\x8f\x8a" -"\xe6\xb8\xac\xe8\xa9\xa6\xe8\xbb\x9f\xe9\xab\x94\xe7\x9a\x84\xe9" -"\x80\x9f\xe5\xba\xa6\xe6\x98\xaf\xe4\xb8\x8d\xe5\xae\xb9\xe5\xbf" -"\xbd\xe8\xa6\x96\xe7\x9a\x84\x0a\xe8\xaa\xb2\xe9\xa1\x8c\x2e\x20" -"\xe7\x82\xba\xe5\x8a\xa0\xe5\xbf\xab\xe9\x96\x8b\xe7\x99\xbc\xe5" -"\x8f\x8a\xe6\xb8\xac\xe8\xa9\xa6\xe7\x9a\x84\xe9\x80\x9f\xe5\xba" -"\xa6\x2c\x20\xe6\x88\x91\xe5\x80\x91\xe4\xbe\xbf\xe5\xb8\xb8\xe5" -"\xb8\x8c\xe6\x9c\x9b\xe8\x83\xbd\xe5\x88\xa9\xe7\x94\xa8\xe4\xb8" -"\x80\xe4\xba\x9b\xe5\xb7\xb2\xe9\x96\x8b\xe7\x99\xbc\xe5\xa5\xbd" -"\xe7\x9a\x84\x0a\x6c\x69\x62\x72\x61\x72\x79\x2c\x20\xe4\xb8\xa6" -"\xe6\x9c\x89\xe4\xb8\x80\xe5\x80\x8b\x20\x66\x61\x73\x74\x20\x70" -"\x72\x6f\x74\x6f\x74\x79\x70\x69\x6e\x67\x20\xe7\x9a\x84\x20\x70" -"\x72\x6f\x67\x72\x61\x6d\x6d\x69\x6e\x67\x20\x6c\x61\x6e\x67\x75" -"\x61\x67\x65\x20\xe5\x8f\xaf\x0a\xe4\xbe\x9b\xe4\xbd\xbf\xe7\x94" -"\xa8\x2e\x20\xe7\x9b\xae\xe5\x89\x8d\xe6\x9c\x89\xe8\xa8\xb1\xe8" -"\xa8\xb1\xe5\xa4\x9a\xe5\xa4\x9a\xe7\x9a\x84\x20\x6c\x69\x62\x72" -"\x61\x72\x79\x20\xe6\x98\xaf\xe4\xbb\xa5\x20\x43\x20\xe5\xaf\xab" -"\xe6\x88\x90\x2c\x20\xe8\x80\x8c\x20\x50\x79\x74\x68\x6f\x6e\x20" -"\xe6\x98\xaf\xe4\xb8\x80\xe5\x80\x8b\x0a\x66\x61\x73\x74\x20\x70" -"\x72\x6f\x74\x6f\x74\x79\x70\x69\x6e\x67\x20\xe7\x9a\x84\x20\x70" -"\x72\x6f\x67\x72\x61\x6d\x6d\x69\x6e\x67\x20\x6c\x61\x6e\x67\x75" -"\x61\x67\x65\x2e\x20\xe6\x95\x85\xe6\x88\x91\xe5\x80\x91\xe5\xb8" -"\x8c\xe6\x9c\x9b\xe8\x83\xbd\xe5\xb0\x87\xe6\x97\xa2\xe6\x9c\x89" -"\xe7\x9a\x84\x0a\x43\x20\x6c\x69\x62\x72\x61\x72\x79\x20\xe6\x8b" -"\xbf\xe5\x88\xb0\x20\x50\x79\x74\x68\x6f\x6e\x20\xe7\x9a\x84\xe7" -"\x92\xb0\xe5\xa2\x83\xe4\xb8\xad\xe6\xb8\xac\xe8\xa9\xa6\xe5\x8f" -"\x8a\xe6\x95\xb4\xe5\x90\x88\x2e\x20\xe5\x85\xb6\xe4\xb8\xad\xe6" -"\x9c\x80\xe4\xb8\xbb\xe8\xa6\x81\xe4\xb9\x9f\xe6\x98\xaf\xe6\x88" -"\x91\xe5\x80\x91\xe6\x89\x80\x0a\xe8\xa6\x81\xe8\xa8\x8e\xe8\xab" -"\x96\xe7\x9a\x84\xe5\x95\x8f\xe9\xa1\x8c\xe5\xb0\xb1\xe6\x98\xaf" -"\x3a\x0a\x0a"), -'johab': ( -"\x99\xb1\xa4\x77\x88\x62\xd0\x61\x20\xcd\x5c\xaf\xa1\xc5\xa9\x9c" -"\x61\x0a\x0a\xdc\xc0\xdc\xc0\x90\x73\x21\x21\x20\xf1\x67\xe2\x9c" -"\xf0\x55\xcc\x81\xa3\x89\x9f\x85\x8a\xa1\x20\xdc\xde\xdc\xd3\xd2" -"\x7a\xd9\xaf\xd9\xaf\xd9\xaf\x20\x8b\x77\x96\xd3\x20\xdc\xd1\x95" -"\x81\x20\xdc\xc0\x2e\x20\x2e\x0a\xed\x3c\xb5\x77\xdc\xd1\x93\x77" -"\xd2\x73\x20\x2e\x20\x2e\x20\x2e\x20\x2e\x20\xac\xe1\xb6\x89\x9e" -"\xa1\x20\x95\x65\xd0\x62\xf0\xe0\x20\xe0\x3b\xd2\x7a\x20\x21\x20" -"\x21\x20\x21\x87\x41\x2e\x87\x41\x0a\xd3\x61\xd3\x61\xd3\x61\x20" -"\x88\x41\x88\x41\x88\x41\xd9\x69\x87\x41\x5f\x87\x41\x20\xb4\xe1" -"\x9f\x9a\x20\xc8\xa1\xc5\xc1\x8b\x7a\x20\x95\x61\xb7\x77\x20\xc3" -"\x97\xe2\x9c\x97\x69\xf0\xe0\x20\xdc\xc0\x97\x61\x8b\x7a\x0a\xac" -"\xe9\x9f\x7a\x20\xe0\x3b\xd2\x7a\x20\x2e\x20\x2e\x20\x2e\x20\x2e" -"\x20\x8a\x89\xb4\x81\xae\xba\x20\xdc\xd1\x8a\xa1\x20\xdc\xde\x9f" -"\x89\xdc\xc2\x8b\x7a\x20\xf1\x67\xf1\x62\xf5\x49\xed\xfc\xf3\xe9" -"\x8c\x61\xbb\x9a\x0a\xb5\xc1\xb2\xa1\xd2\x7a\x20\x21\x20\x21\x20" -"\xed\x3c\xb5\x77\xdc\xd1\x20\xe0\x3b\x93\x77\x8a\xa1\x20\xd9\x69" -"\xea\xbe\x89\xc5\x20\xb4\xf4\x93\x77\x8a\xa1\x93\x77\x20\xed\x3c" -"\x93\x77\x96\xc1\xd2\x7a\x20\x8b\x69\xb4\x81\x97\x7a\x0a\xdc\xde" -"\x9d\x61\x97\x41\xe2\x9c\x20\xaf\x81\xce\xa1\xae\xa1\xd2\x7a\x20" -"\xb4\xe1\x9f\x9a\x20\xf1\x67\xf1\x62\xf5\x49\xed\xfc\xf3\xe9\xaf" -"\x82\xdc\xef\x97\x69\xb4\x7a\x21\x21\x20\xdc\xc0\xdc\xc0\x90\x73" -"\xd9\xbd\x20\xd9\x62\xd9\x62\x2a\x0a\x0a", -"\xeb\x98\xa0\xeb\xb0\xa9\xea\xb0\x81\xed\x95\x98\x20\xed\x8e\xb2" -"\xec\x8b\x9c\xec\xbd\x9c\xeb\x9d\xbc\x0a\x0a\xe3\x89\xaf\xe3\x89" -"\xaf\xeb\x82\xa9\x21\x21\x20\xe5\x9b\xa0\xe4\xb9\x9d\xe6\x9c\x88" -"\xed\x8c\xa8\xeb\xaf\xa4\xeb\xa6\x94\xea\xb6\x88\x20\xe2\x93\xa1" -"\xe2\x93\x96\xed\x9b\x80\xc2\xbf\xc2\xbf\xc2\xbf\x20\xea\xb8\x8d" -"\xeb\x92\x99\x20\xe2\x93\x94\xeb\x8e\xa8\x20\xe3\x89\xaf\x2e\x20" -"\x2e\x0a\xe4\xba\x9e\xec\x98\x81\xe2\x93\x94\xeb\x8a\xa5\xed\x9a" -"\xb9\x20\x2e\x20\x2e\x20\x2e\x20\x2e\x20\xec\x84\x9c\xec\x9a\xb8" -"\xeb\xa4\x84\x20\xeb\x8e\x90\xed\x95\x99\xe4\xb9\x99\x20\xe5\xae" -"\xb6\xed\x9b\x80\x20\x21\x20\x21\x20\x21\xe3\x85\xa0\x2e\xe3\x85" -"\xa0\x0a\xed\x9d\x90\xed\x9d\x90\xed\x9d\x90\x20\xe3\x84\xb1\xe3" -"\x84\xb1\xe3\x84\xb1\xe2\x98\x86\xe3\x85\xa0\x5f\xe3\x85\xa0\x20" -"\xec\x96\xb4\xeb\xa6\xa8\x20\xed\x83\xb8\xec\xbd\xb0\xea\xb8\x90" -"\x20\xeb\x8e\x8c\xec\x9d\x91\x20\xec\xb9\x91\xe4\xb9\x9d\xeb\x93" -"\xa4\xe4\xb9\x99\x20\xe3\x89\xaf\xeb\x93\x9c\xea\xb8\x90\x0a\xec" -"\x84\xa4\xeb\xa6\x8c\x20\xe5\xae\xb6\xed\x9b\x80\x20\x2e\x20\x2e" -"\x20\x2e\x20\x2e\x20\xea\xb5\xb4\xec\x95\xa0\xec\x89\x8c\x20\xe2" -"\x93\x94\xea\xb6\x88\x20\xe2\x93\xa1\xeb\xa6\x98\xe3\x89\xb1\xea" -"\xb8\x90\x20\xe5\x9b\xa0\xe4\xbb\x81\xe5\xb7\x9d\xef\xa6\x81\xe4" -"\xb8\xad\xea\xb9\x8c\xec\xa6\xbc\x0a\xec\x99\x80\xec\x92\x80\xed" -"\x9b\x80\x20\x21\x20\x21\x20\xe4\xba\x9e\xec\x98\x81\xe2\x93\x94" -"\x20\xe5\xae\xb6\xeb\x8a\xa5\xea\xb6\x88\x20\xe2\x98\x86\xe4\xb8" -"\x8a\xea\xb4\x80\x20\xec\x97\x86\xeb\x8a\xa5\xea\xb6\x88\xeb\x8a" -"\xa5\x20\xe4\xba\x9e\xeb\x8a\xa5\xeb\x92\x88\xed\x9b\x80\x20\xea" -"\xb8\x80\xec\x95\xa0\xeb\x93\xb4\x0a\xe2\x93\xa1\xeb\xa0\xa4\xeb" -"\x93\x80\xe4\xb9\x9d\x20\xec\x8b\x80\xed\x92\x94\xec\x88\xb4\xed" -"\x9b\x80\x20\xec\x96\xb4\xeb\xa6\xa8\x20\xe5\x9b\xa0\xe4\xbb\x81" -"\xe5\xb7\x9d\xef\xa6\x81\xe4\xb8\xad\xec\x8b\x81\xe2\x91\xa8\xeb" -"\x93\xa4\xec\x95\x9c\x21\x21\x20\xe3\x89\xaf\xe3\x89\xaf\xeb\x82" -"\xa9\xe2\x99\xa1\x20\xe2\x8c\x92\xe2\x8c\x92\x2a\x0a\x0a"), -'shift_jis': ( -"\x50\x79\x74\x68\x6f\x6e\x20\x82\xcc\x8a\x4a\x94\xad\x82\xcd\x81" -"\x41\x31\x39\x39\x30\x20\x94\x4e\x82\xb2\x82\xeb\x82\xa9\x82\xe7" -"\x8a\x4a\x8e\x6e\x82\xb3\x82\xea\x82\xc4\x82\xa2\x82\xdc\x82\xb7" -"\x81\x42\x0a\x8a\x4a\x94\xad\x8e\xd2\x82\xcc\x20\x47\x75\x69\x64" -"\x6f\x20\x76\x61\x6e\x20\x52\x6f\x73\x73\x75\x6d\x20\x82\xcd\x8b" -"\xb3\x88\xe7\x97\x70\x82\xcc\x83\x76\x83\x8d\x83\x4f\x83\x89\x83" -"\x7e\x83\x93\x83\x4f\x8c\xbe\x8c\xea\x81\x75\x41\x42\x43\x81\x76" -"\x82\xcc\x8a\x4a\x94\xad\x82\xc9\x8e\x51\x89\xc1\x82\xb5\x82\xc4" -"\x82\xa2\x82\xdc\x82\xb5\x82\xbd\x82\xaa\x81\x41\x41\x42\x43\x20" -"\x82\xcd\x8e\xc0\x97\x70\x8f\xe3\x82\xcc\x96\xda\x93\x49\x82\xc9" -"\x82\xcd\x82\xa0\x82\xdc\x82\xe8\x93\x4b\x82\xb5\x82\xc4\x82\xa2" -"\x82\xdc\x82\xb9\x82\xf1\x82\xc5\x82\xb5\x82\xbd\x81\x42\x0a\x82" -"\xb1\x82\xcc\x82\xbd\x82\xdf\x81\x41\x47\x75\x69\x64\x6f\x20\x82" -"\xcd\x82\xe6\x82\xe8\x8e\xc0\x97\x70\x93\x49\x82\xc8\x83\x76\x83" -"\x8d\x83\x4f\x83\x89\x83\x7e\x83\x93\x83\x4f\x8c\xbe\x8c\xea\x82" -"\xcc\x8a\x4a\x94\xad\x82\xf0\x8a\x4a\x8e\x6e\x82\xb5\x81\x41\x89" -"\x70\x8d\x91\x20\x42\x42\x53\x20\x95\xfa\x91\x97\x82\xcc\x83\x52" -"\x83\x81\x83\x66\x83\x42\x94\xd4\x91\x67\x81\x75\x83\x82\x83\x93" -"\x83\x65\x83\x42\x20\x83\x70\x83\x43\x83\x5c\x83\x93\x81\x76\x82" -"\xcc\x83\x74\x83\x40\x83\x93\x82\xc5\x82\xa0\x82\xe9\x20\x47\x75" -"\x69\x64\x6f\x20\x82\xcd\x82\xb1\x82\xcc\x8c\xbe\x8c\xea\x82\xf0" -"\x81\x75\x50\x79\x74\x68\x6f\x6e\x81\x76\x82\xc6\x96\xbc\x82\xc3" -"\x82\xaf\x82\xdc\x82\xb5\x82\xbd\x81\x42\x0a\x82\xb1\x82\xcc\x82" -"\xe6\x82\xa4\x82\xc8\x94\x77\x8c\x69\x82\xa9\x82\xe7\x90\xb6\x82" -"\xdc\x82\xea\x82\xbd\x20\x50\x79\x74\x68\x6f\x6e\x20\x82\xcc\x8c" -"\xbe\x8c\xea\x90\xdd\x8c\x76\x82\xcd\x81\x41\x81\x75\x83\x56\x83" -"\x93\x83\x76\x83\x8b\x81\x76\x82\xc5\x81\x75\x8f\x4b\x93\xbe\x82" -"\xaa\x97\x65\x88\xd5\x81\x76\x82\xc6\x82\xa2\x82\xa4\x96\xda\x95" -"\x57\x82\xc9\x8f\x64\x93\x5f\x82\xaa\x92\x75\x82\xa9\x82\xea\x82" -"\xc4\x82\xa2\x82\xdc\x82\xb7\x81\x42\x0a\x91\xbd\x82\xad\x82\xcc" -"\x83\x58\x83\x4e\x83\x8a\x83\x76\x83\x67\x8c\x6e\x8c\xbe\x8c\xea" -"\x82\xc5\x82\xcd\x83\x86\x81\x5b\x83\x55\x82\xcc\x96\xda\x90\xe6" -"\x82\xcc\x97\x98\x95\xd6\x90\xab\x82\xf0\x97\x44\x90\xe6\x82\xb5" -"\x82\xc4\x90\x46\x81\x58\x82\xc8\x8b\x40\x94\x5c\x82\xf0\x8c\xbe" -"\x8c\xea\x97\x76\x91\x66\x82\xc6\x82\xb5\x82\xc4\x8e\xe6\x82\xe8" -"\x93\xfc\x82\xea\x82\xe9\x8f\xea\x8d\x87\x82\xaa\x91\xbd\x82\xa2" -"\x82\xcc\x82\xc5\x82\xb7\x82\xaa\x81\x41\x50\x79\x74\x68\x6f\x6e" -"\x20\x82\xc5\x82\xcd\x82\xbb\x82\xa4\x82\xa2\x82\xc1\x82\xbd\x8f" -"\xac\x8d\xd7\x8d\x48\x82\xaa\x92\xc7\x89\xc1\x82\xb3\x82\xea\x82" -"\xe9\x82\xb1\x82\xc6\x82\xcd\x82\xa0\x82\xdc\x82\xe8\x82\xa0\x82" -"\xe8\x82\xdc\x82\xb9\x82\xf1\x81\x42\x0a\x8c\xbe\x8c\xea\x8e\xa9" -"\x91\xcc\x82\xcc\x8b\x40\x94\x5c\x82\xcd\x8d\xc5\x8f\xac\x8c\xc0" -"\x82\xc9\x89\x9f\x82\xb3\x82\xa6\x81\x41\x95\x4b\x97\x76\x82\xc8" -"\x8b\x40\x94\x5c\x82\xcd\x8a\x67\x92\xa3\x83\x82\x83\x57\x83\x85" -"\x81\x5b\x83\x8b\x82\xc6\x82\xb5\x82\xc4\x92\xc7\x89\xc1\x82\xb7" -"\x82\xe9\x81\x41\x82\xc6\x82\xa2\x82\xa4\x82\xcc\x82\xaa\x20\x50" -"\x79\x74\x68\x6f\x6e\x20\x82\xcc\x83\x7c\x83\x8a\x83\x56\x81\x5b" -"\x82\xc5\x82\xb7\x81\x42\x0a\x0a", -"\x50\x79\x74\x68\x6f\x6e\x20\xe3\x81\xae\xe9\x96\x8b\xe7\x99\xba" -"\xe3\x81\xaf\xe3\x80\x81\x31\x39\x39\x30\x20\xe5\xb9\xb4\xe3\x81" -"\x94\xe3\x82\x8d\xe3\x81\x8b\xe3\x82\x89\xe9\x96\x8b\xe5\xa7\x8b" -"\xe3\x81\x95\xe3\x82\x8c\xe3\x81\xa6\xe3\x81\x84\xe3\x81\xbe\xe3" -"\x81\x99\xe3\x80\x82\x0a\xe9\x96\x8b\xe7\x99\xba\xe8\x80\x85\xe3" -"\x81\xae\x20\x47\x75\x69\x64\x6f\x20\x76\x61\x6e\x20\x52\x6f\x73" -"\x73\x75\x6d\x20\xe3\x81\xaf\xe6\x95\x99\xe8\x82\xb2\xe7\x94\xa8" -"\xe3\x81\xae\xe3\x83\x97\xe3\x83\xad\xe3\x82\xb0\xe3\x83\xa9\xe3" -"\x83\x9f\xe3\x83\xb3\xe3\x82\xb0\xe8\xa8\x80\xe8\xaa\x9e\xe3\x80" -"\x8c\x41\x42\x43\xe3\x80\x8d\xe3\x81\xae\xe9\x96\x8b\xe7\x99\xba" -"\xe3\x81\xab\xe5\x8f\x82\xe5\x8a\xa0\xe3\x81\x97\xe3\x81\xa6\xe3" -"\x81\x84\xe3\x81\xbe\xe3\x81\x97\xe3\x81\x9f\xe3\x81\x8c\xe3\x80" -"\x81\x41\x42\x43\x20\xe3\x81\xaf\xe5\xae\x9f\xe7\x94\xa8\xe4\xb8" -"\x8a\xe3\x81\xae\xe7\x9b\xae\xe7\x9a\x84\xe3\x81\xab\xe3\x81\xaf" -"\xe3\x81\x82\xe3\x81\xbe\xe3\x82\x8a\xe9\x81\xa9\xe3\x81\x97\xe3" -"\x81\xa6\xe3\x81\x84\xe3\x81\xbe\xe3\x81\x9b\xe3\x82\x93\xe3\x81" -"\xa7\xe3\x81\x97\xe3\x81\x9f\xe3\x80\x82\x0a\xe3\x81\x93\xe3\x81" -"\xae\xe3\x81\x9f\xe3\x82\x81\xe3\x80\x81\x47\x75\x69\x64\x6f\x20" -"\xe3\x81\xaf\xe3\x82\x88\xe3\x82\x8a\xe5\xae\x9f\xe7\x94\xa8\xe7" -"\x9a\x84\xe3\x81\xaa\xe3\x83\x97\xe3\x83\xad\xe3\x82\xb0\xe3\x83" -"\xa9\xe3\x83\x9f\xe3\x83\xb3\xe3\x82\xb0\xe8\xa8\x80\xe8\xaa\x9e" -"\xe3\x81\xae\xe9\x96\x8b\xe7\x99\xba\xe3\x82\x92\xe9\x96\x8b\xe5" -"\xa7\x8b\xe3\x81\x97\xe3\x80\x81\xe8\x8b\xb1\xe5\x9b\xbd\x20\x42" -"\x42\x53\x20\xe6\x94\xbe\xe9\x80\x81\xe3\x81\xae\xe3\x82\xb3\xe3" -"\x83\xa1\xe3\x83\x87\xe3\x82\xa3\xe7\x95\xaa\xe7\xb5\x84\xe3\x80" -"\x8c\xe3\x83\xa2\xe3\x83\xb3\xe3\x83\x86\xe3\x82\xa3\x20\xe3\x83" -"\x91\xe3\x82\xa4\xe3\x82\xbd\xe3\x83\xb3\xe3\x80\x8d\xe3\x81\xae" -"\xe3\x83\x95\xe3\x82\xa1\xe3\x83\xb3\xe3\x81\xa7\xe3\x81\x82\xe3" -"\x82\x8b\x20\x47\x75\x69\x64\x6f\x20\xe3\x81\xaf\xe3\x81\x93\xe3" -"\x81\xae\xe8\xa8\x80\xe8\xaa\x9e\xe3\x82\x92\xe3\x80\x8c\x50\x79" -"\x74\x68\x6f\x6e\xe3\x80\x8d\xe3\x81\xa8\xe5\x90\x8d\xe3\x81\xa5" -"\xe3\x81\x91\xe3\x81\xbe\xe3\x81\x97\xe3\x81\x9f\xe3\x80\x82\x0a" -"\xe3\x81\x93\xe3\x81\xae\xe3\x82\x88\xe3\x81\x86\xe3\x81\xaa\xe8" -"\x83\x8c\xe6\x99\xaf\xe3\x81\x8b\xe3\x82\x89\xe7\x94\x9f\xe3\x81" -"\xbe\xe3\x82\x8c\xe3\x81\x9f\x20\x50\x79\x74\x68\x6f\x6e\x20\xe3" -"\x81\xae\xe8\xa8\x80\xe8\xaa\x9e\xe8\xa8\xad\xe8\xa8\x88\xe3\x81" -"\xaf\xe3\x80\x81\xe3\x80\x8c\xe3\x82\xb7\xe3\x83\xb3\xe3\x83\x97" -"\xe3\x83\xab\xe3\x80\x8d\xe3\x81\xa7\xe3\x80\x8c\xe7\xbf\x92\xe5" -"\xbe\x97\xe3\x81\x8c\xe5\xae\xb9\xe6\x98\x93\xe3\x80\x8d\xe3\x81" -"\xa8\xe3\x81\x84\xe3\x81\x86\xe7\x9b\xae\xe6\xa8\x99\xe3\x81\xab" -"\xe9\x87\x8d\xe7\x82\xb9\xe3\x81\x8c\xe7\xbd\xae\xe3\x81\x8b\xe3" -"\x82\x8c\xe3\x81\xa6\xe3\x81\x84\xe3\x81\xbe\xe3\x81\x99\xe3\x80" -"\x82\x0a\xe5\xa4\x9a\xe3\x81\x8f\xe3\x81\xae\xe3\x82\xb9\xe3\x82" -"\xaf\xe3\x83\xaa\xe3\x83\x97\xe3\x83\x88\xe7\xb3\xbb\xe8\xa8\x80" -"\xe8\xaa\x9e\xe3\x81\xa7\xe3\x81\xaf\xe3\x83\xa6\xe3\x83\xbc\xe3" -"\x82\xb6\xe3\x81\xae\xe7\x9b\xae\xe5\x85\x88\xe3\x81\xae\xe5\x88" -"\xa9\xe4\xbe\xbf\xe6\x80\xa7\xe3\x82\x92\xe5\x84\xaa\xe5\x85\x88" -"\xe3\x81\x97\xe3\x81\xa6\xe8\x89\xb2\xe3\x80\x85\xe3\x81\xaa\xe6" -"\xa9\x9f\xe8\x83\xbd\xe3\x82\x92\xe8\xa8\x80\xe8\xaa\x9e\xe8\xa6" -"\x81\xe7\xb4\xa0\xe3\x81\xa8\xe3\x81\x97\xe3\x81\xa6\xe5\x8f\x96" -"\xe3\x82\x8a\xe5\x85\xa5\xe3\x82\x8c\xe3\x82\x8b\xe5\xa0\xb4\xe5" -"\x90\x88\xe3\x81\x8c\xe5\xa4\x9a\xe3\x81\x84\xe3\x81\xae\xe3\x81" -"\xa7\xe3\x81\x99\xe3\x81\x8c\xe3\x80\x81\x50\x79\x74\x68\x6f\x6e" -"\x20\xe3\x81\xa7\xe3\x81\xaf\xe3\x81\x9d\xe3\x81\x86\xe3\x81\x84" -"\xe3\x81\xa3\xe3\x81\x9f\xe5\xb0\x8f\xe7\xb4\xb0\xe5\xb7\xa5\xe3" -"\x81\x8c\xe8\xbf\xbd\xe5\x8a\xa0\xe3\x81\x95\xe3\x82\x8c\xe3\x82" -"\x8b\xe3\x81\x93\xe3\x81\xa8\xe3\x81\xaf\xe3\x81\x82\xe3\x81\xbe" -"\xe3\x82\x8a\xe3\x81\x82\xe3\x82\x8a\xe3\x81\xbe\xe3\x81\x9b\xe3" -"\x82\x93\xe3\x80\x82\x0a\xe8\xa8\x80\xe8\xaa\x9e\xe8\x87\xaa\xe4" -"\xbd\x93\xe3\x81\xae\xe6\xa9\x9f\xe8\x83\xbd\xe3\x81\xaf\xe6\x9c" -"\x80\xe5\xb0\x8f\xe9\x99\x90\xe3\x81\xab\xe6\x8a\xbc\xe3\x81\x95" -"\xe3\x81\x88\xe3\x80\x81\xe5\xbf\x85\xe8\xa6\x81\xe3\x81\xaa\xe6" -"\xa9\x9f\xe8\x83\xbd\xe3\x81\xaf\xe6\x8b\xa1\xe5\xbc\xb5\xe3\x83" -"\xa2\xe3\x82\xb8\xe3\x83\xa5\xe3\x83\xbc\xe3\x83\xab\xe3\x81\xa8" -"\xe3\x81\x97\xe3\x81\xa6\xe8\xbf\xbd\xe5\x8a\xa0\xe3\x81\x99\xe3" -"\x82\x8b\xe3\x80\x81\xe3\x81\xa8\xe3\x81\x84\xe3\x81\x86\xe3\x81" -"\xae\xe3\x81\x8c\x20\x50\x79\x74\x68\x6f\x6e\x20\xe3\x81\xae\xe3" -"\x83\x9d\xe3\x83\xaa\xe3\x82\xb7\xe3\x83\xbc\xe3\x81\xa7\xe3\x81" -"\x99\xe3\x80\x82\x0a\x0a"), -'shift_jisx0213': ( -"\x50\x79\x74\x68\x6f\x6e\x20\x82\xcc\x8a\x4a\x94\xad\x82\xcd\x81" -"\x41\x31\x39\x39\x30\x20\x94\x4e\x82\xb2\x82\xeb\x82\xa9\x82\xe7" -"\x8a\x4a\x8e\x6e\x82\xb3\x82\xea\x82\xc4\x82\xa2\x82\xdc\x82\xb7" -"\x81\x42\x0a\x8a\x4a\x94\xad\x8e\xd2\x82\xcc\x20\x47\x75\x69\x64" -"\x6f\x20\x76\x61\x6e\x20\x52\x6f\x73\x73\x75\x6d\x20\x82\xcd\x8b" -"\xb3\x88\xe7\x97\x70\x82\xcc\x83\x76\x83\x8d\x83\x4f\x83\x89\x83" -"\x7e\x83\x93\x83\x4f\x8c\xbe\x8c\xea\x81\x75\x41\x42\x43\x81\x76" -"\x82\xcc\x8a\x4a\x94\xad\x82\xc9\x8e\x51\x89\xc1\x82\xb5\x82\xc4" -"\x82\xa2\x82\xdc\x82\xb5\x82\xbd\x82\xaa\x81\x41\x41\x42\x43\x20" -"\x82\xcd\x8e\xc0\x97\x70\x8f\xe3\x82\xcc\x96\xda\x93\x49\x82\xc9" -"\x82\xcd\x82\xa0\x82\xdc\x82\xe8\x93\x4b\x82\xb5\x82\xc4\x82\xa2" -"\x82\xdc\x82\xb9\x82\xf1\x82\xc5\x82\xb5\x82\xbd\x81\x42\x0a\x82" -"\xb1\x82\xcc\x82\xbd\x82\xdf\x81\x41\x47\x75\x69\x64\x6f\x20\x82" -"\xcd\x82\xe6\x82\xe8\x8e\xc0\x97\x70\x93\x49\x82\xc8\x83\x76\x83" -"\x8d\x83\x4f\x83\x89\x83\x7e\x83\x93\x83\x4f\x8c\xbe\x8c\xea\x82" -"\xcc\x8a\x4a\x94\xad\x82\xf0\x8a\x4a\x8e\x6e\x82\xb5\x81\x41\x89" -"\x70\x8d\x91\x20\x42\x42\x53\x20\x95\xfa\x91\x97\x82\xcc\x83\x52" -"\x83\x81\x83\x66\x83\x42\x94\xd4\x91\x67\x81\x75\x83\x82\x83\x93" -"\x83\x65\x83\x42\x20\x83\x70\x83\x43\x83\x5c\x83\x93\x81\x76\x82" -"\xcc\x83\x74\x83\x40\x83\x93\x82\xc5\x82\xa0\x82\xe9\x20\x47\x75" -"\x69\x64\x6f\x20\x82\xcd\x82\xb1\x82\xcc\x8c\xbe\x8c\xea\x82\xf0" -"\x81\x75\x50\x79\x74\x68\x6f\x6e\x81\x76\x82\xc6\x96\xbc\x82\xc3" -"\x82\xaf\x82\xdc\x82\xb5\x82\xbd\x81\x42\x0a\x82\xb1\x82\xcc\x82" -"\xe6\x82\xa4\x82\xc8\x94\x77\x8c\x69\x82\xa9\x82\xe7\x90\xb6\x82" -"\xdc\x82\xea\x82\xbd\x20\x50\x79\x74\x68\x6f\x6e\x20\x82\xcc\x8c" -"\xbe\x8c\xea\x90\xdd\x8c\x76\x82\xcd\x81\x41\x81\x75\x83\x56\x83" -"\x93\x83\x76\x83\x8b\x81\x76\x82\xc5\x81\x75\x8f\x4b\x93\xbe\x82" -"\xaa\x97\x65\x88\xd5\x81\x76\x82\xc6\x82\xa2\x82\xa4\x96\xda\x95" -"\x57\x82\xc9\x8f\x64\x93\x5f\x82\xaa\x92\x75\x82\xa9\x82\xea\x82" -"\xc4\x82\xa2\x82\xdc\x82\xb7\x81\x42\x0a\x91\xbd\x82\xad\x82\xcc" -"\x83\x58\x83\x4e\x83\x8a\x83\x76\x83\x67\x8c\x6e\x8c\xbe\x8c\xea" -"\x82\xc5\x82\xcd\x83\x86\x81\x5b\x83\x55\x82\xcc\x96\xda\x90\xe6" -"\x82\xcc\x97\x98\x95\xd6\x90\xab\x82\xf0\x97\x44\x90\xe6\x82\xb5" -"\x82\xc4\x90\x46\x81\x58\x82\xc8\x8b\x40\x94\x5c\x82\xf0\x8c\xbe" -"\x8c\xea\x97\x76\x91\x66\x82\xc6\x82\xb5\x82\xc4\x8e\xe6\x82\xe8" -"\x93\xfc\x82\xea\x82\xe9\x8f\xea\x8d\x87\x82\xaa\x91\xbd\x82\xa2" -"\x82\xcc\x82\xc5\x82\xb7\x82\xaa\x81\x41\x50\x79\x74\x68\x6f\x6e" -"\x20\x82\xc5\x82\xcd\x82\xbb\x82\xa4\x82\xa2\x82\xc1\x82\xbd\x8f" -"\xac\x8d\xd7\x8d\x48\x82\xaa\x92\xc7\x89\xc1\x82\xb3\x82\xea\x82" -"\xe9\x82\xb1\x82\xc6\x82\xcd\x82\xa0\x82\xdc\x82\xe8\x82\xa0\x82" -"\xe8\x82\xdc\x82\xb9\x82\xf1\x81\x42\x0a\x8c\xbe\x8c\xea\x8e\xa9" -"\x91\xcc\x82\xcc\x8b\x40\x94\x5c\x82\xcd\x8d\xc5\x8f\xac\x8c\xc0" -"\x82\xc9\x89\x9f\x82\xb3\x82\xa6\x81\x41\x95\x4b\x97\x76\x82\xc8" -"\x8b\x40\x94\x5c\x82\xcd\x8a\x67\x92\xa3\x83\x82\x83\x57\x83\x85" -"\x81\x5b\x83\x8b\x82\xc6\x82\xb5\x82\xc4\x92\xc7\x89\xc1\x82\xb7" -"\x82\xe9\x81\x41\x82\xc6\x82\xa2\x82\xa4\x82\xcc\x82\xaa\x20\x50" -"\x79\x74\x68\x6f\x6e\x20\x82\xcc\x83\x7c\x83\x8a\x83\x56\x81\x5b" -"\x82\xc5\x82\xb7\x81\x42\x0a\x0a\x83\x6d\x82\xf5\x20\x83\x9e\x20" -"\x83\x67\x83\x4c\x88\x4b\x88\x79\x20\x98\x83\xfc\xd6\x20\xfc\xd2" -"\xfc\xe6\xfb\xd4\x0a", -"\x50\x79\x74\x68\x6f\x6e\x20\xe3\x81\xae\xe9\x96\x8b\xe7\x99\xba" -"\xe3\x81\xaf\xe3\x80\x81\x31\x39\x39\x30\x20\xe5\xb9\xb4\xe3\x81" -"\x94\xe3\x82\x8d\xe3\x81\x8b\xe3\x82\x89\xe9\x96\x8b\xe5\xa7\x8b" -"\xe3\x81\x95\xe3\x82\x8c\xe3\x81\xa6\xe3\x81\x84\xe3\x81\xbe\xe3" -"\x81\x99\xe3\x80\x82\x0a\xe9\x96\x8b\xe7\x99\xba\xe8\x80\x85\xe3" -"\x81\xae\x20\x47\x75\x69\x64\x6f\x20\x76\x61\x6e\x20\x52\x6f\x73" -"\x73\x75\x6d\x20\xe3\x81\xaf\xe6\x95\x99\xe8\x82\xb2\xe7\x94\xa8" -"\xe3\x81\xae\xe3\x83\x97\xe3\x83\xad\xe3\x82\xb0\xe3\x83\xa9\xe3" -"\x83\x9f\xe3\x83\xb3\xe3\x82\xb0\xe8\xa8\x80\xe8\xaa\x9e\xe3\x80" -"\x8c\x41\x42\x43\xe3\x80\x8d\xe3\x81\xae\xe9\x96\x8b\xe7\x99\xba" -"\xe3\x81\xab\xe5\x8f\x82\xe5\x8a\xa0\xe3\x81\x97\xe3\x81\xa6\xe3" -"\x81\x84\xe3\x81\xbe\xe3\x81\x97\xe3\x81\x9f\xe3\x81\x8c\xe3\x80" -"\x81\x41\x42\x43\x20\xe3\x81\xaf\xe5\xae\x9f\xe7\x94\xa8\xe4\xb8" -"\x8a\xe3\x81\xae\xe7\x9b\xae\xe7\x9a\x84\xe3\x81\xab\xe3\x81\xaf" -"\xe3\x81\x82\xe3\x81\xbe\xe3\x82\x8a\xe9\x81\xa9\xe3\x81\x97\xe3" -"\x81\xa6\xe3\x81\x84\xe3\x81\xbe\xe3\x81\x9b\xe3\x82\x93\xe3\x81" -"\xa7\xe3\x81\x97\xe3\x81\x9f\xe3\x80\x82\x0a\xe3\x81\x93\xe3\x81" -"\xae\xe3\x81\x9f\xe3\x82\x81\xe3\x80\x81\x47\x75\x69\x64\x6f\x20" -"\xe3\x81\xaf\xe3\x82\x88\xe3\x82\x8a\xe5\xae\x9f\xe7\x94\xa8\xe7" -"\x9a\x84\xe3\x81\xaa\xe3\x83\x97\xe3\x83\xad\xe3\x82\xb0\xe3\x83" -"\xa9\xe3\x83\x9f\xe3\x83\xb3\xe3\x82\xb0\xe8\xa8\x80\xe8\xaa\x9e" -"\xe3\x81\xae\xe9\x96\x8b\xe7\x99\xba\xe3\x82\x92\xe9\x96\x8b\xe5" -"\xa7\x8b\xe3\x81\x97\xe3\x80\x81\xe8\x8b\xb1\xe5\x9b\xbd\x20\x42" -"\x42\x53\x20\xe6\x94\xbe\xe9\x80\x81\xe3\x81\xae\xe3\x82\xb3\xe3" -"\x83\xa1\xe3\x83\x87\xe3\x82\xa3\xe7\x95\xaa\xe7\xb5\x84\xe3\x80" -"\x8c\xe3\x83\xa2\xe3\x83\xb3\xe3\x83\x86\xe3\x82\xa3\x20\xe3\x83" -"\x91\xe3\x82\xa4\xe3\x82\xbd\xe3\x83\xb3\xe3\x80\x8d\xe3\x81\xae" -"\xe3\x83\x95\xe3\x82\xa1\xe3\x83\xb3\xe3\x81\xa7\xe3\x81\x82\xe3" -"\x82\x8b\x20\x47\x75\x69\x64\x6f\x20\xe3\x81\xaf\xe3\x81\x93\xe3" -"\x81\xae\xe8\xa8\x80\xe8\xaa\x9e\xe3\x82\x92\xe3\x80\x8c\x50\x79" -"\x74\x68\x6f\x6e\xe3\x80\x8d\xe3\x81\xa8\xe5\x90\x8d\xe3\x81\xa5" -"\xe3\x81\x91\xe3\x81\xbe\xe3\x81\x97\xe3\x81\x9f\xe3\x80\x82\x0a" -"\xe3\x81\x93\xe3\x81\xae\xe3\x82\x88\xe3\x81\x86\xe3\x81\xaa\xe8" -"\x83\x8c\xe6\x99\xaf\xe3\x81\x8b\xe3\x82\x89\xe7\x94\x9f\xe3\x81" -"\xbe\xe3\x82\x8c\xe3\x81\x9f\x20\x50\x79\x74\x68\x6f\x6e\x20\xe3" -"\x81\xae\xe8\xa8\x80\xe8\xaa\x9e\xe8\xa8\xad\xe8\xa8\x88\xe3\x81" -"\xaf\xe3\x80\x81\xe3\x80\x8c\xe3\x82\xb7\xe3\x83\xb3\xe3\x83\x97" -"\xe3\x83\xab\xe3\x80\x8d\xe3\x81\xa7\xe3\x80\x8c\xe7\xbf\x92\xe5" -"\xbe\x97\xe3\x81\x8c\xe5\xae\xb9\xe6\x98\x93\xe3\x80\x8d\xe3\x81" -"\xa8\xe3\x81\x84\xe3\x81\x86\xe7\x9b\xae\xe6\xa8\x99\xe3\x81\xab" -"\xe9\x87\x8d\xe7\x82\xb9\xe3\x81\x8c\xe7\xbd\xae\xe3\x81\x8b\xe3" -"\x82\x8c\xe3\x81\xa6\xe3\x81\x84\xe3\x81\xbe\xe3\x81\x99\xe3\x80" -"\x82\x0a\xe5\xa4\x9a\xe3\x81\x8f\xe3\x81\xae\xe3\x82\xb9\xe3\x82" -"\xaf\xe3\x83\xaa\xe3\x83\x97\xe3\x83\x88\xe7\xb3\xbb\xe8\xa8\x80" -"\xe8\xaa\x9e\xe3\x81\xa7\xe3\x81\xaf\xe3\x83\xa6\xe3\x83\xbc\xe3" -"\x82\xb6\xe3\x81\xae\xe7\x9b\xae\xe5\x85\x88\xe3\x81\xae\xe5\x88" -"\xa9\xe4\xbe\xbf\xe6\x80\xa7\xe3\x82\x92\xe5\x84\xaa\xe5\x85\x88" -"\xe3\x81\x97\xe3\x81\xa6\xe8\x89\xb2\xe3\x80\x85\xe3\x81\xaa\xe6" -"\xa9\x9f\xe8\x83\xbd\xe3\x82\x92\xe8\xa8\x80\xe8\xaa\x9e\xe8\xa6" -"\x81\xe7\xb4\xa0\xe3\x81\xa8\xe3\x81\x97\xe3\x81\xa6\xe5\x8f\x96" -"\xe3\x82\x8a\xe5\x85\xa5\xe3\x82\x8c\xe3\x82\x8b\xe5\xa0\xb4\xe5" -"\x90\x88\xe3\x81\x8c\xe5\xa4\x9a\xe3\x81\x84\xe3\x81\xae\xe3\x81" -"\xa7\xe3\x81\x99\xe3\x81\x8c\xe3\x80\x81\x50\x79\x74\x68\x6f\x6e" -"\x20\xe3\x81\xa7\xe3\x81\xaf\xe3\x81\x9d\xe3\x81\x86\xe3\x81\x84" -"\xe3\x81\xa3\xe3\x81\x9f\xe5\xb0\x8f\xe7\xb4\xb0\xe5\xb7\xa5\xe3" -"\x81\x8c\xe8\xbf\xbd\xe5\x8a\xa0\xe3\x81\x95\xe3\x82\x8c\xe3\x82" -"\x8b\xe3\x81\x93\xe3\x81\xa8\xe3\x81\xaf\xe3\x81\x82\xe3\x81\xbe" -"\xe3\x82\x8a\xe3\x81\x82\xe3\x82\x8a\xe3\x81\xbe\xe3\x81\x9b\xe3" -"\x82\x93\xe3\x80\x82\x0a\xe8\xa8\x80\xe8\xaa\x9e\xe8\x87\xaa\xe4" -"\xbd\x93\xe3\x81\xae\xe6\xa9\x9f\xe8\x83\xbd\xe3\x81\xaf\xe6\x9c" -"\x80\xe5\xb0\x8f\xe9\x99\x90\xe3\x81\xab\xe6\x8a\xbc\xe3\x81\x95" -"\xe3\x81\x88\xe3\x80\x81\xe5\xbf\x85\xe8\xa6\x81\xe3\x81\xaa\xe6" -"\xa9\x9f\xe8\x83\xbd\xe3\x81\xaf\xe6\x8b\xa1\xe5\xbc\xb5\xe3\x83" -"\xa2\xe3\x82\xb8\xe3\x83\xa5\xe3\x83\xbc\xe3\x83\xab\xe3\x81\xa8" -"\xe3\x81\x97\xe3\x81\xa6\xe8\xbf\xbd\xe5\x8a\xa0\xe3\x81\x99\xe3" -"\x82\x8b\xe3\x80\x81\xe3\x81\xa8\xe3\x81\x84\xe3\x81\x86\xe3\x81" -"\xae\xe3\x81\x8c\x20\x50\x79\x74\x68\x6f\x6e\x20\xe3\x81\xae\xe3" -"\x83\x9d\xe3\x83\xaa\xe3\x82\xb7\xe3\x83\xbc\xe3\x81\xa7\xe3\x81" -"\x99\xe3\x80\x82\x0a\x0a\xe3\x83\x8e\xe3\x81\x8b\xe3\x82\x9a\x20" -"\xe3\x83\x88\xe3\x82\x9a\x20\xe3\x83\x88\xe3\x82\xad\xef\xa8\xb6" -"\xef\xa8\xb9\x20\xf0\xa1\x9a\xb4\xf0\xaa\x8e\x8c\x20\xe9\xba\x80" -"\xe9\xbd\x81\xf0\xa9\x9b\xb0\x0a"), -} diff --git a/lib-python/2.7/test/crashers/README b/lib-python/2.7/test/crashers/README --- a/lib-python/2.7/test/crashers/README +++ b/lib-python/2.7/test/crashers/README @@ -1,20 +1,16 @@ -This directory only contains tests for outstanding bugs that cause -the interpreter to segfault. Ideally this directory should always -be empty. Sometimes it may not be easy to fix the underlying cause. +This directory only contains tests for outstanding bugs that cause the +interpreter to segfault. Ideally this directory should always be empty, but +sometimes it may not be easy to fix the underlying cause and the bug is deemed +too obscure to invest the effort. Each test should fail when run from the command line: ./python Lib/test/crashers/weakref_in_del.py -Each test should have a link to the bug report: +Put as much info into a docstring or comments to help determine the cause of the +failure, as well as a bugs.python.org issue number if it exists. Particularly +note if the cause is system or environment dependent and what the variables are. - # http://python.org/sf/BUG# - -Put as much info into a docstring or comments to help determine -the cause of the failure. Particularly note if the cause is -system or environment dependent and what the variables are. - -Once the crash is fixed, the test case should be moved into an appropriate -test (even if it was originally from the test suite). This ensures the -regression doesn't happen again. And if it does, it should be easier -to track down. +Once the crash is fixed, the test case should be moved into an appropriate test +(even if it was originally from the test suite). This ensures the regression +doesn't happen again. And if it does, it should be easier to track down. diff --git a/lib-python/2.7/test/crashers/recursion_limit_too_high.py b/lib-python/2.7/test/crashers/recursion_limit_too_high.py --- a/lib-python/2.7/test/crashers/recursion_limit_too_high.py +++ b/lib-python/2.7/test/crashers/recursion_limit_too_high.py @@ -5,7 +5,7 @@ # file handles. # The point of this example is to show that sys.setrecursionlimit() is a -# hack, and not a robust solution. This example simply exercices a path +# hack, and not a robust solution. This example simply exercises a path # where it takes many C-level recursions, consuming a lot of stack # space, for each Python-level recursion. So 1000 times this amount of # stack space may be too much for standard platforms already. diff --git a/lib-python/2.7/test/decimaltestdata/and.decTest b/lib-python/2.7/test/decimaltestdata/and.decTest --- a/lib-python/2.7/test/decimaltestdata/and.decTest +++ b/lib-python/2.7/test/decimaltestdata/and.decTest @@ -1,338 +1,338 @@ ------------------------------------------------------------------------- --- and.decTest -- digitwise logical AND -- --- Copyright (c) IBM Corporation, 1981, 2008. All rights reserved. -- ------------------------------------------------------------------------- --- Please see the document "General Decimal Arithmetic Testcases" -- --- at http://www2.hursley.ibm.com/decimal for the description of -- --- these testcases. -- --- -- --- These testcases are experimental ('beta' versions), and they -- --- may contain errors. They are offered on an as-is basis. In -- --- particular, achieving the same results as the tests here is not -- --- a guarantee that an implementation complies with any Standard -- --- or specification. The tests are not exhaustive. -- --- -- --- Please send comments, suggestions, and corrections to the author: -- --- Mike Cowlishaw, IBM Fellow -- --- IBM UK, PO Box 31, Birmingham Road, Warwick CV34 5JL, UK -- --- mfc at uk.ibm.com -- ------------------------------------------------------------------------- -version: 2.59 - -extended: 1 -precision: 9 -rounding: half_up -maxExponent: 999 -minExponent: -999 - --- Sanity check (truth table) -andx001 and 0 0 -> 0 -andx002 and 0 1 -> 0 -andx003 and 1 0 -> 0 -andx004 and 1 1 -> 1 -andx005 and 1100 1010 -> 1000 -andx006 and 1111 10 -> 10 -andx007 and 1111 1010 -> 1010 - --- and at msd and msd-1 -andx010 and 000000000 000000000 -> 0 -andx011 and 000000000 100000000 -> 0 -andx012 and 100000000 000000000 -> 0 -andx013 and 100000000 100000000 -> 100000000 -andx014 and 000000000 000000000 -> 0 -andx015 and 000000000 010000000 -> 0 -andx016 and 010000000 000000000 -> 0 -andx017 and 010000000 010000000 -> 10000000 - --- Various lengths --- 123456789 123456789 123456789 -andx021 and 111111111 111111111 -> 111111111 -andx022 and 111111111111 111111111 -> 111111111 -andx023 and 111111111111 11111111 -> 11111111 -andx024 and 111111111 11111111 -> 11111111 -andx025 and 111111111 1111111 -> 1111111 -andx026 and 111111111111 111111 -> 111111 -andx027 and 111111111111 11111 -> 11111 -andx028 and 111111111111 1111 -> 1111 -andx029 and 111111111111 111 -> 111 -andx031 and 111111111111 11 -> 11 -andx032 and 111111111111 1 -> 1 -andx033 and 111111111111 1111111111 -> 111111111 -andx034 and 11111111111 11111111111 -> 111111111 -andx035 and 1111111111 111111111111 -> 111111111 -andx036 and 111111111 1111111111111 -> 111111111 - -andx040 and 111111111 111111111111 -> 111111111 -andx041 and 11111111 111111111111 -> 11111111 -andx042 and 11111111 111111111 -> 11111111 -andx043 and 1111111 111111111 -> 1111111 -andx044 and 111111 111111111 -> 111111 -andx045 and 11111 111111111 -> 11111 -andx046 and 1111 111111111 -> 1111 -andx047 and 111 111111111 -> 111 -andx048 and 11 111111111 -> 11 -andx049 and 1 111111111 -> 1 - -andx050 and 1111111111 1 -> 1 -andx051 and 111111111 1 -> 1 -andx052 and 11111111 1 -> 1 -andx053 and 1111111 1 -> 1 -andx054 and 111111 1 -> 1 -andx055 and 11111 1 -> 1 -andx056 and 1111 1 -> 1 -andx057 and 111 1 -> 1 -andx058 and 11 1 -> 1 -andx059 and 1 1 -> 1 - -andx060 and 1111111111 0 -> 0 -andx061 and 111111111 0 -> 0 -andx062 and 11111111 0 -> 0 -andx063 and 1111111 0 -> 0 -andx064 and 111111 0 -> 0 -andx065 and 11111 0 -> 0 -andx066 and 1111 0 -> 0 -andx067 and 111 0 -> 0 -andx068 and 11 0 -> 0 -andx069 and 1 0 -> 0 - -andx070 and 1 1111111111 -> 1 -andx071 and 1 111111111 -> 1 -andx072 and 1 11111111 -> 1 -andx073 and 1 1111111 -> 1 -andx074 and 1 111111 -> 1 -andx075 and 1 11111 -> 1 -andx076 and 1 1111 -> 1 -andx077 and 1 111 -> 1 -andx078 and 1 11 -> 1 -andx079 and 1 1 -> 1 - -andx080 and 0 1111111111 -> 0 -andx081 and 0 111111111 -> 0 -andx082 and 0 11111111 -> 0 -andx083 and 0 1111111 -> 0 -andx084 and 0 111111 -> 0 -andx085 and 0 11111 -> 0 -andx086 and 0 1111 -> 0 -andx087 and 0 111 -> 0 -andx088 and 0 11 -> 0 -andx089 and 0 1 -> 0 - -andx090 and 011111111 111111111 -> 11111111 -andx091 and 101111111 111111111 -> 101111111 -andx092 and 110111111 111111111 -> 110111111 -andx093 and 111011111 111111111 -> 111011111 -andx094 and 111101111 111111111 -> 111101111 -andx095 and 111110111 111111111 -> 111110111 -andx096 and 111111011 111111111 -> 111111011 -andx097 and 111111101 111111111 -> 111111101 -andx098 and 111111110 111111111 -> 111111110 - -andx100 and 111111111 011111111 -> 11111111 -andx101 and 111111111 101111111 -> 101111111 -andx102 and 111111111 110111111 -> 110111111 -andx103 and 111111111 111011111 -> 111011111 -andx104 and 111111111 111101111 -> 111101111 -andx105 and 111111111 111110111 -> 111110111 -andx106 and 111111111 111111011 -> 111111011 -andx107 and 111111111 111111101 -> 111111101 -andx108 and 111111111 111111110 -> 111111110 - --- non-0/1 should not be accepted, nor should signs -andx220 and 111111112 111111111 -> NaN Invalid_operation -andx221 and 333333333 333333333 -> NaN Invalid_operation -andx222 and 555555555 555555555 -> NaN Invalid_operation -andx223 and 777777777 777777777 -> NaN Invalid_operation -andx224 and 999999999 999999999 -> NaN Invalid_operation -andx225 and 222222222 999999999 -> NaN Invalid_operation -andx226 and 444444444 999999999 -> NaN Invalid_operation -andx227 and 666666666 999999999 -> NaN Invalid_operation -andx228 and 888888888 999999999 -> NaN Invalid_operation -andx229 and 999999999 222222222 -> NaN Invalid_operation -andx230 and 999999999 444444444 -> NaN Invalid_operation -andx231 and 999999999 666666666 -> NaN Invalid_operation -andx232 and 999999999 888888888 -> NaN Invalid_operation --- a few randoms -andx240 and 567468689 -934981942 -> NaN Invalid_operation -andx241 and 567367689 934981942 -> NaN Invalid_operation -andx242 and -631917772 -706014634 -> NaN Invalid_operation -andx243 and -756253257 138579234 -> NaN Invalid_operation -andx244 and 835590149 567435400 -> NaN Invalid_operation --- test MSD -andx250 and 200000000 100000000 -> NaN Invalid_operation -andx251 and 700000000 100000000 -> NaN Invalid_operation -andx252 and 800000000 100000000 -> NaN Invalid_operation -andx253 and 900000000 100000000 -> NaN Invalid_operation -andx254 and 200000000 000000000 -> NaN Invalid_operation -andx255 and 700000000 000000000 -> NaN Invalid_operation -andx256 and 800000000 000000000 -> NaN Invalid_operation -andx257 and 900000000 000000000 -> NaN Invalid_operation -andx258 and 100000000 200000000 -> NaN Invalid_operation -andx259 and 100000000 700000000 -> NaN Invalid_operation -andx260 and 100000000 800000000 -> NaN Invalid_operation -andx261 and 100000000 900000000 -> NaN Invalid_operation -andx262 and 000000000 200000000 -> NaN Invalid_operation -andx263 and 000000000 700000000 -> NaN Invalid_operation -andx264 and 000000000 800000000 -> NaN Invalid_operation -andx265 and 000000000 900000000 -> NaN Invalid_operation --- test MSD-1 -andx270 and 020000000 100000000 -> NaN Invalid_operation -andx271 and 070100000 100000000 -> NaN Invalid_operation -andx272 and 080010000 100000001 -> NaN Invalid_operation -andx273 and 090001000 100000010 -> NaN Invalid_operation -andx274 and 100000100 020010100 -> NaN Invalid_operation -andx275 and 100000000 070001000 -> NaN Invalid_operation -andx276 and 100000010 080010100 -> NaN Invalid_operation -andx277 and 100000000 090000010 -> NaN Invalid_operation --- test LSD -andx280 and 001000002 100000000 -> NaN Invalid_operation -andx281 and 000000007 100000000 -> NaN Invalid_operation -andx282 and 000000008 100000000 -> NaN Invalid_operation -andx283 and 000000009 100000000 -> NaN Invalid_operation -andx284 and 100000000 000100002 -> NaN Invalid_operation -andx285 and 100100000 001000007 -> NaN Invalid_operation -andx286 and 100010000 010000008 -> NaN Invalid_operation -andx287 and 100001000 100000009 -> NaN Invalid_operation --- test Middie -andx288 and 001020000 100000000 -> NaN Invalid_operation -andx289 and 000070001 100000000 -> NaN Invalid_operation -andx290 and 000080000 100010000 -> NaN Invalid_operation -andx291 and 000090000 100001000 -> NaN Invalid_operation -andx292 and 100000010 000020100 -> NaN Invalid_operation -andx293 and 100100000 000070010 -> NaN Invalid_operation -andx294 and 100010100 000080001 -> NaN Invalid_operation -andx295 and 100001000 000090000 -> NaN Invalid_operation --- signs -andx296 and -100001000 -000000000 -> NaN Invalid_operation -andx297 and -100001000 000010000 -> NaN Invalid_operation -andx298 and 100001000 -000000000 -> NaN Invalid_operation -andx299 and 100001000 000011000 -> 1000 - --- Nmax, Nmin, Ntiny -andx331 and 2 9.99999999E+999 -> NaN Invalid_operation -andx332 and 3 1E-999 -> NaN Invalid_operation -andx333 and 4 1.00000000E-999 -> NaN Invalid_operation -andx334 and 5 1E-1007 -> NaN Invalid_operation -andx335 and 6 -1E-1007 -> NaN Invalid_operation -andx336 and 7 -1.00000000E-999 -> NaN Invalid_operation -andx337 and 8 -1E-999 -> NaN Invalid_operation -andx338 and 9 -9.99999999E+999 -> NaN Invalid_operation -andx341 and 9.99999999E+999 -18 -> NaN Invalid_operation -andx342 and 1E-999 01 -> NaN Invalid_operation -andx343 and 1.00000000E-999 -18 -> NaN Invalid_operation -andx344 and 1E-1007 18 -> NaN Invalid_operation -andx345 and -1E-1007 -10 -> NaN Invalid_operation -andx346 and -1.00000000E-999 18 -> NaN Invalid_operation -andx347 and -1E-999 10 -> NaN Invalid_operation -andx348 and -9.99999999E+999 -18 -> NaN Invalid_operation - --- A few other non-integers -andx361 and 1.0 1 -> NaN Invalid_operation -andx362 and 1E+1 1 -> NaN Invalid_operation -andx363 and 0.0 1 -> NaN Invalid_operation -andx364 and 0E+1 1 -> NaN Invalid_operation -andx365 and 9.9 1 -> NaN Invalid_operation -andx366 and 9E+1 1 -> NaN Invalid_operation -andx371 and 0 1.0 -> NaN Invalid_operation -andx372 and 0 1E+1 -> NaN Invalid_operation -andx373 and 0 0.0 -> NaN Invalid_operation -andx374 and 0 0E+1 -> NaN Invalid_operation -andx375 and 0 9.9 -> NaN Invalid_operation -andx376 and 0 9E+1 -> NaN Invalid_operation - --- All Specials are in error -andx780 and -Inf -Inf -> NaN Invalid_operation -andx781 and -Inf -1000 -> NaN Invalid_operation -andx782 and -Inf -1 -> NaN Invalid_operation -andx783 and -Inf -0 -> NaN Invalid_operation -andx784 and -Inf 0 -> NaN Invalid_operation -andx785 and -Inf 1 -> NaN Invalid_operation -andx786 and -Inf 1000 -> NaN Invalid_operation -andx787 and -1000 -Inf -> NaN Invalid_operation -andx788 and -Inf -Inf -> NaN Invalid_operation -andx789 and -1 -Inf -> NaN Invalid_operation -andx790 and -0 -Inf -> NaN Invalid_operation -andx791 and 0 -Inf -> NaN Invalid_operation -andx792 and 1 -Inf -> NaN Invalid_operation -andx793 and 1000 -Inf -> NaN Invalid_operation -andx794 and Inf -Inf -> NaN Invalid_operation - -andx800 and Inf -Inf -> NaN Invalid_operation -andx801 and Inf -1000 -> NaN Invalid_operation -andx802 and Inf -1 -> NaN Invalid_operation -andx803 and Inf -0 -> NaN Invalid_operation -andx804 and Inf 0 -> NaN Invalid_operation -andx805 and Inf 1 -> NaN Invalid_operation -andx806 and Inf 1000 -> NaN Invalid_operation -andx807 and Inf Inf -> NaN Invalid_operation -andx808 and -1000 Inf -> NaN Invalid_operation -andx809 and -Inf Inf -> NaN Invalid_operation -andx810 and -1 Inf -> NaN Invalid_operation -andx811 and -0 Inf -> NaN Invalid_operation -andx812 and 0 Inf -> NaN Invalid_operation -andx813 and 1 Inf -> NaN Invalid_operation -andx814 and 1000 Inf -> NaN Invalid_operation -andx815 and Inf Inf -> NaN Invalid_operation - -andx821 and NaN -Inf -> NaN Invalid_operation -andx822 and NaN -1000 -> NaN Invalid_operation -andx823 and NaN -1 -> NaN Invalid_operation -andx824 and NaN -0 -> NaN Invalid_operation -andx825 and NaN 0 -> NaN Invalid_operation -andx826 and NaN 1 -> NaN Invalid_operation -andx827 and NaN 1000 -> NaN Invalid_operation -andx828 and NaN Inf -> NaN Invalid_operation -andx829 and NaN NaN -> NaN Invalid_operation -andx830 and -Inf NaN -> NaN Invalid_operation -andx831 and -1000 NaN -> NaN Invalid_operation -andx832 and -1 NaN -> NaN Invalid_operation -andx833 and -0 NaN -> NaN Invalid_operation -andx834 and 0 NaN -> NaN Invalid_operation -andx835 and 1 NaN -> NaN Invalid_operation -andx836 and 1000 NaN -> NaN Invalid_operation -andx837 and Inf NaN -> NaN Invalid_operation - -andx841 and sNaN -Inf -> NaN Invalid_operation -andx842 and sNaN -1000 -> NaN Invalid_operation -andx843 and sNaN -1 -> NaN Invalid_operation -andx844 and sNaN -0 -> NaN Invalid_operation -andx845 and sNaN 0 -> NaN Invalid_operation -andx846 and sNaN 1 -> NaN Invalid_operation -andx847 and sNaN 1000 -> NaN Invalid_operation -andx848 and sNaN NaN -> NaN Invalid_operation -andx849 and sNaN sNaN -> NaN Invalid_operation -andx850 and NaN sNaN -> NaN Invalid_operation -andx851 and -Inf sNaN -> NaN Invalid_operation -andx852 and -1000 sNaN -> NaN Invalid_operation -andx853 and -1 sNaN -> NaN Invalid_operation -andx854 and -0 sNaN -> NaN Invalid_operation -andx855 and 0 sNaN -> NaN Invalid_operation -andx856 and 1 sNaN -> NaN Invalid_operation -andx857 and 1000 sNaN -> NaN Invalid_operation -andx858 and Inf sNaN -> NaN Invalid_operation -andx859 and NaN sNaN -> NaN Invalid_operation - --- propagating NaNs -andx861 and NaN1 -Inf -> NaN Invalid_operation -andx862 and +NaN2 -1000 -> NaN Invalid_operation -andx863 and NaN3 1000 -> NaN Invalid_operation -andx864 and NaN4 Inf -> NaN Invalid_operation -andx865 and NaN5 +NaN6 -> NaN Invalid_operation -andx866 and -Inf NaN7 -> NaN Invalid_operation -andx867 and -1000 NaN8 -> NaN Invalid_operation -andx868 and 1000 NaN9 -> NaN Invalid_operation -andx869 and Inf +NaN10 -> NaN Invalid_operation -andx871 and sNaN11 -Inf -> NaN Invalid_operation -andx872 and sNaN12 -1000 -> NaN Invalid_operation -andx873 and sNaN13 1000 -> NaN Invalid_operation -andx874 and sNaN14 NaN17 -> NaN Invalid_operation -andx875 and sNaN15 sNaN18 -> NaN Invalid_operation -andx876 and NaN16 sNaN19 -> NaN Invalid_operation -andx877 and -Inf +sNaN20 -> NaN Invalid_operation -andx878 and -1000 sNaN21 -> NaN Invalid_operation -andx879 and 1000 sNaN22 -> NaN Invalid_operation -andx880 and Inf sNaN23 -> NaN Invalid_operation -andx881 and +NaN25 +sNaN24 -> NaN Invalid_operation -andx882 and -NaN26 NaN28 -> NaN Invalid_operation -andx883 and -sNaN27 sNaN29 -> NaN Invalid_operation -andx884 and 1000 -NaN30 -> NaN Invalid_operation -andx885 and 1000 -sNaN31 -> NaN Invalid_operation +------------------------------------------------------------------------ +-- and.decTest -- digitwise logical AND -- +-- Copyright (c) IBM Corporation, 1981, 2008. All rights reserved. -- +------------------------------------------------------------------------ +-- Please see the document "General Decimal Arithmetic Testcases" -- +-- at http://www2.hursley.ibm.com/decimal for the description of -- +-- these testcases. -- +-- -- +-- These testcases are experimental ('beta' versions), and they -- +-- may contain errors. They are offered on an as-is basis. In -- +-- particular, achieving the same results as the tests here is not -- +-- a guarantee that an implementation complies with any Standard -- +-- or specification. The tests are not exhaustive. -- +-- -- +-- Please send comments, suggestions, and corrections to the author: -- +-- Mike Cowlishaw, IBM Fellow -- +-- IBM UK, PO Box 31, Birmingham Road, Warwick CV34 5JL, UK -- +-- mfc at uk.ibm.com -- +------------------------------------------------------------------------ +version: 2.59 + +extended: 1 +precision: 9 +rounding: half_up +maxExponent: 999 +minExponent: -999 + +-- Sanity check (truth table) +andx001 and 0 0 -> 0 +andx002 and 0 1 -> 0 +andx003 and 1 0 -> 0 +andx004 and 1 1 -> 1 +andx005 and 1100 1010 -> 1000 +andx006 and 1111 10 -> 10 +andx007 and 1111 1010 -> 1010 + +-- and at msd and msd-1 +andx010 and 000000000 000000000 -> 0 +andx011 and 000000000 100000000 -> 0 +andx012 and 100000000 000000000 -> 0 +andx013 and 100000000 100000000 -> 100000000 +andx014 and 000000000 000000000 -> 0 +andx015 and 000000000 010000000 -> 0 +andx016 and 010000000 000000000 -> 0 +andx017 and 010000000 010000000 -> 10000000 + +-- Various lengths +-- 123456789 123456789 123456789 +andx021 and 111111111 111111111 -> 111111111 +andx022 and 111111111111 111111111 -> 111111111 +andx023 and 111111111111 11111111 -> 11111111 +andx024 and 111111111 11111111 -> 11111111 +andx025 and 111111111 1111111 -> 1111111 +andx026 and 111111111111 111111 -> 111111 +andx027 and 111111111111 11111 -> 11111 +andx028 and 111111111111 1111 -> 1111 +andx029 and 111111111111 111 -> 111 +andx031 and 111111111111 11 -> 11 +andx032 and 111111111111 1 -> 1 +andx033 and 111111111111 1111111111 -> 111111111 +andx034 and 11111111111 11111111111 -> 111111111 +andx035 and 1111111111 111111111111 -> 111111111 +andx036 and 111111111 1111111111111 -> 111111111 + +andx040 and 111111111 111111111111 -> 111111111 +andx041 and 11111111 111111111111 -> 11111111 +andx042 and 11111111 111111111 -> 11111111 +andx043 and 1111111 111111111 -> 1111111 +andx044 and 111111 111111111 -> 111111 +andx045 and 11111 111111111 -> 11111 +andx046 and 1111 111111111 -> 1111 +andx047 and 111 111111111 -> 111 +andx048 and 11 111111111 -> 11 +andx049 and 1 111111111 -> 1 + +andx050 and 1111111111 1 -> 1 +andx051 and 111111111 1 -> 1 +andx052 and 11111111 1 -> 1 +andx053 and 1111111 1 -> 1 +andx054 and 111111 1 -> 1 +andx055 and 11111 1 -> 1 +andx056 and 1111 1 -> 1 +andx057 and 111 1 -> 1 +andx058 and 11 1 -> 1 +andx059 and 1 1 -> 1 + +andx060 and 1111111111 0 -> 0 +andx061 and 111111111 0 -> 0 +andx062 and 11111111 0 -> 0 +andx063 and 1111111 0 -> 0 +andx064 and 111111 0 -> 0 +andx065 and 11111 0 -> 0 +andx066 and 1111 0 -> 0 +andx067 and 111 0 -> 0 +andx068 and 11 0 -> 0 +andx069 and 1 0 -> 0 + +andx070 and 1 1111111111 -> 1 +andx071 and 1 111111111 -> 1 +andx072 and 1 11111111 -> 1 +andx073 and 1 1111111 -> 1 +andx074 and 1 111111 -> 1 +andx075 and 1 11111 -> 1 +andx076 and 1 1111 -> 1 +andx077 and 1 111 -> 1 +andx078 and 1 11 -> 1 +andx079 and 1 1 -> 1 + +andx080 and 0 1111111111 -> 0 +andx081 and 0 111111111 -> 0 +andx082 and 0 11111111 -> 0 +andx083 and 0 1111111 -> 0 +andx084 and 0 111111 -> 0 +andx085 and 0 11111 -> 0 +andx086 and 0 1111 -> 0 +andx087 and 0 111 -> 0 +andx088 and 0 11 -> 0 +andx089 and 0 1 -> 0 + +andx090 and 011111111 111111111 -> 11111111 +andx091 and 101111111 111111111 -> 101111111 +andx092 and 110111111 111111111 -> 110111111 +andx093 and 111011111 111111111 -> 111011111 +andx094 and 111101111 111111111 -> 111101111 +andx095 and 111110111 111111111 -> 111110111 +andx096 and 111111011 111111111 -> 111111011 +andx097 and 111111101 111111111 -> 111111101 +andx098 and 111111110 111111111 -> 111111110 + +andx100 and 111111111 011111111 -> 11111111 +andx101 and 111111111 101111111 -> 101111111 +andx102 and 111111111 110111111 -> 110111111 +andx103 and 111111111 111011111 -> 111011111 +andx104 and 111111111 111101111 -> 111101111 +andx105 and 111111111 111110111 -> 111110111 +andx106 and 111111111 111111011 -> 111111011 +andx107 and 111111111 111111101 -> 111111101 +andx108 and 111111111 111111110 -> 111111110 + +-- non-0/1 should not be accepted, nor should signs +andx220 and 111111112 111111111 -> NaN Invalid_operation +andx221 and 333333333 333333333 -> NaN Invalid_operation +andx222 and 555555555 555555555 -> NaN Invalid_operation +andx223 and 777777777 777777777 -> NaN Invalid_operation +andx224 and 999999999 999999999 -> NaN Invalid_operation +andx225 and 222222222 999999999 -> NaN Invalid_operation +andx226 and 444444444 999999999 -> NaN Invalid_operation +andx227 and 666666666 999999999 -> NaN Invalid_operation +andx228 and 888888888 999999999 -> NaN Invalid_operation +andx229 and 999999999 222222222 -> NaN Invalid_operation +andx230 and 999999999 444444444 -> NaN Invalid_operation +andx231 and 999999999 666666666 -> NaN Invalid_operation +andx232 and 999999999 888888888 -> NaN Invalid_operation +-- a few randoms +andx240 and 567468689 -934981942 -> NaN Invalid_operation +andx241 and 567367689 934981942 -> NaN Invalid_operation +andx242 and -631917772 -706014634 -> NaN Invalid_operation +andx243 and -756253257 138579234 -> NaN Invalid_operation +andx244 and 835590149 567435400 -> NaN Invalid_operation +-- test MSD +andx250 and 200000000 100000000 -> NaN Invalid_operation +andx251 and 700000000 100000000 -> NaN Invalid_operation +andx252 and 800000000 100000000 -> NaN Invalid_operation +andx253 and 900000000 100000000 -> NaN Invalid_operation +andx254 and 200000000 000000000 -> NaN Invalid_operation +andx255 and 700000000 000000000 -> NaN Invalid_operation +andx256 and 800000000 000000000 -> NaN Invalid_operation +andx257 and 900000000 000000000 -> NaN Invalid_operation +andx258 and 100000000 200000000 -> NaN Invalid_operation +andx259 and 100000000 700000000 -> NaN Invalid_operation +andx260 and 100000000 800000000 -> NaN Invalid_operation +andx261 and 100000000 900000000 -> NaN Invalid_operation +andx262 and 000000000 200000000 -> NaN Invalid_operation +andx263 and 000000000 700000000 -> NaN Invalid_operation +andx264 and 000000000 800000000 -> NaN Invalid_operation +andx265 and 000000000 900000000 -> NaN Invalid_operation +-- test MSD-1 +andx270 and 020000000 100000000 -> NaN Invalid_operation +andx271 and 070100000 100000000 -> NaN Invalid_operation +andx272 and 080010000 100000001 -> NaN Invalid_operation +andx273 and 090001000 100000010 -> NaN Invalid_operation +andx274 and 100000100 020010100 -> NaN Invalid_operation +andx275 and 100000000 070001000 -> NaN Invalid_operation +andx276 and 100000010 080010100 -> NaN Invalid_operation +andx277 and 100000000 090000010 -> NaN Invalid_operation +-- test LSD +andx280 and 001000002 100000000 -> NaN Invalid_operation +andx281 and 000000007 100000000 -> NaN Invalid_operation +andx282 and 000000008 100000000 -> NaN Invalid_operation +andx283 and 000000009 100000000 -> NaN Invalid_operation +andx284 and 100000000 000100002 -> NaN Invalid_operation +andx285 and 100100000 001000007 -> NaN Invalid_operation +andx286 and 100010000 010000008 -> NaN Invalid_operation +andx287 and 100001000 100000009 -> NaN Invalid_operation +-- test Middie +andx288 and 001020000 100000000 -> NaN Invalid_operation +andx289 and 000070001 100000000 -> NaN Invalid_operation +andx290 and 000080000 100010000 -> NaN Invalid_operation +andx291 and 000090000 100001000 -> NaN Invalid_operation +andx292 and 100000010 000020100 -> NaN Invalid_operation +andx293 and 100100000 000070010 -> NaN Invalid_operation +andx294 and 100010100 000080001 -> NaN Invalid_operation +andx295 and 100001000 000090000 -> NaN Invalid_operation +-- signs +andx296 and -100001000 -000000000 -> NaN Invalid_operation +andx297 and -100001000 000010000 -> NaN Invalid_operation +andx298 and 100001000 -000000000 -> NaN Invalid_operation +andx299 and 100001000 000011000 -> 1000 + +-- Nmax, Nmin, Ntiny +andx331 and 2 9.99999999E+999 -> NaN Invalid_operation +andx332 and 3 1E-999 -> NaN Invalid_operation +andx333 and 4 1.00000000E-999 -> NaN Invalid_operation +andx334 and 5 1E-1007 -> NaN Invalid_operation +andx335 and 6 -1E-1007 -> NaN Invalid_operation +andx336 and 7 -1.00000000E-999 -> NaN Invalid_operation +andx337 and 8 -1E-999 -> NaN Invalid_operation +andx338 and 9 -9.99999999E+999 -> NaN Invalid_operation +andx341 and 9.99999999E+999 -18 -> NaN Invalid_operation +andx342 and 1E-999 01 -> NaN Invalid_operation +andx343 and 1.00000000E-999 -18 -> NaN Invalid_operation +andx344 and 1E-1007 18 -> NaN Invalid_operation +andx345 and -1E-1007 -10 -> NaN Invalid_operation +andx346 and -1.00000000E-999 18 -> NaN Invalid_operation +andx347 and -1E-999 10 -> NaN Invalid_operation +andx348 and -9.99999999E+999 -18 -> NaN Invalid_operation + +-- A few other non-integers +andx361 and 1.0 1 -> NaN Invalid_operation +andx362 and 1E+1 1 -> NaN Invalid_operation +andx363 and 0.0 1 -> NaN Invalid_operation +andx364 and 0E+1 1 -> NaN Invalid_operation +andx365 and 9.9 1 -> NaN Invalid_operation +andx366 and 9E+1 1 -> NaN Invalid_operation +andx371 and 0 1.0 -> NaN Invalid_operation +andx372 and 0 1E+1 -> NaN Invalid_operation +andx373 and 0 0.0 -> NaN Invalid_operation +andx374 and 0 0E+1 -> NaN Invalid_operation +andx375 and 0 9.9 -> NaN Invalid_operation +andx376 and 0 9E+1 -> NaN Invalid_operation + +-- All Specials are in error +andx780 and -Inf -Inf -> NaN Invalid_operation +andx781 and -Inf -1000 -> NaN Invalid_operation +andx782 and -Inf -1 -> NaN Invalid_operation +andx783 and -Inf -0 -> NaN Invalid_operation +andx784 and -Inf 0 -> NaN Invalid_operation +andx785 and -Inf 1 -> NaN Invalid_operation +andx786 and -Inf 1000 -> NaN Invalid_operation +andx787 and -1000 -Inf -> NaN Invalid_operation +andx788 and -Inf -Inf -> NaN Invalid_operation +andx789 and -1 -Inf -> NaN Invalid_operation +andx790 and -0 -Inf -> NaN Invalid_operation +andx791 and 0 -Inf -> NaN Invalid_operation +andx792 and 1 -Inf -> NaN Invalid_operation +andx793 and 1000 -Inf -> NaN Invalid_operation +andx794 and Inf -Inf -> NaN Invalid_operation + +andx800 and Inf -Inf -> NaN Invalid_operation +andx801 and Inf -1000 -> NaN Invalid_operation +andx802 and Inf -1 -> NaN Invalid_operation +andx803 and Inf -0 -> NaN Invalid_operation +andx804 and Inf 0 -> NaN Invalid_operation +andx805 and Inf 1 -> NaN Invalid_operation +andx806 and Inf 1000 -> NaN Invalid_operation +andx807 and Inf Inf -> NaN Invalid_operation +andx808 and -1000 Inf -> NaN Invalid_operation +andx809 and -Inf Inf -> NaN Invalid_operation +andx810 and -1 Inf -> NaN Invalid_operation +andx811 and -0 Inf -> NaN Invalid_operation +andx812 and 0 Inf -> NaN Invalid_operation +andx813 and 1 Inf -> NaN Invalid_operation +andx814 and 1000 Inf -> NaN Invalid_operation +andx815 and Inf Inf -> NaN Invalid_operation + +andx821 and NaN -Inf -> NaN Invalid_operation +andx822 and NaN -1000 -> NaN Invalid_operation +andx823 and NaN -1 -> NaN Invalid_operation +andx824 and NaN -0 -> NaN Invalid_operation +andx825 and NaN 0 -> NaN Invalid_operation +andx826 and NaN 1 -> NaN Invalid_operation +andx827 and NaN 1000 -> NaN Invalid_operation +andx828 and NaN Inf -> NaN Invalid_operation +andx829 and NaN NaN -> NaN Invalid_operation +andx830 and -Inf NaN -> NaN Invalid_operation +andx831 and -1000 NaN -> NaN Invalid_operation +andx832 and -1 NaN -> NaN Invalid_operation +andx833 and -0 NaN -> NaN Invalid_operation +andx834 and 0 NaN -> NaN Invalid_operation +andx835 and 1 NaN -> NaN Invalid_operation +andx836 and 1000 NaN -> NaN Invalid_operation +andx837 and Inf NaN -> NaN Invalid_operation + +andx841 and sNaN -Inf -> NaN Invalid_operation +andx842 and sNaN -1000 -> NaN Invalid_operation +andx843 and sNaN -1 -> NaN Invalid_operation +andx844 and sNaN -0 -> NaN Invalid_operation +andx845 and sNaN 0 -> NaN Invalid_operation +andx846 and sNaN 1 -> NaN Invalid_operation +andx847 and sNaN 1000 -> NaN Invalid_operation +andx848 and sNaN NaN -> NaN Invalid_operation +andx849 and sNaN sNaN -> NaN Invalid_operation +andx850 and NaN sNaN -> NaN Invalid_operation +andx851 and -Inf sNaN -> NaN Invalid_operation +andx852 and -1000 sNaN -> NaN Invalid_operation +andx853 and -1 sNaN -> NaN Invalid_operation +andx854 and -0 sNaN -> NaN Invalid_operation +andx855 and 0 sNaN -> NaN Invalid_operation +andx856 and 1 sNaN -> NaN Invalid_operation +andx857 and 1000 sNaN -> NaN Invalid_operation +andx858 and Inf sNaN -> NaN Invalid_operation +andx859 and NaN sNaN -> NaN Invalid_operation + +-- propagating NaNs +andx861 and NaN1 -Inf -> NaN Invalid_operation +andx862 and +NaN2 -1000 -> NaN Invalid_operation +andx863 and NaN3 1000 -> NaN Invalid_operation +andx864 and NaN4 Inf -> NaN Invalid_operation +andx865 and NaN5 +NaN6 -> NaN Invalid_operation +andx866 and -Inf NaN7 -> NaN Invalid_operation +andx867 and -1000 NaN8 -> NaN Invalid_operation +andx868 and 1000 NaN9 -> NaN Invalid_operation +andx869 and Inf +NaN10 -> NaN Invalid_operation +andx871 and sNaN11 -Inf -> NaN Invalid_operation +andx872 and sNaN12 -1000 -> NaN Invalid_operation +andx873 and sNaN13 1000 -> NaN Invalid_operation +andx874 and sNaN14 NaN17 -> NaN Invalid_operation +andx875 and sNaN15 sNaN18 -> NaN Invalid_operation +andx876 and NaN16 sNaN19 -> NaN Invalid_operation +andx877 and -Inf +sNaN20 -> NaN Invalid_operation +andx878 and -1000 sNaN21 -> NaN Invalid_operation +andx879 and 1000 sNaN22 -> NaN Invalid_operation +andx880 and Inf sNaN23 -> NaN Invalid_operation +andx881 and +NaN25 +sNaN24 -> NaN Invalid_operation +andx882 and -NaN26 NaN28 -> NaN Invalid_operation +andx883 and -sNaN27 sNaN29 -> NaN Invalid_operation +andx884 and 1000 -NaN30 -> NaN Invalid_operation +andx885 and 1000 -sNaN31 -> NaN Invalid_operation diff --git a/lib-python/2.7/test/decimaltestdata/class.decTest b/lib-python/2.7/test/decimaltestdata/class.decTest --- a/lib-python/2.7/test/decimaltestdata/class.decTest +++ b/lib-python/2.7/test/decimaltestdata/class.decTest @@ -1,131 +1,131 @@ ------------------------------------------------------------------------- --- class.decTest -- Class operations -- --- Copyright (c) IBM Corporation, 1981, 2008. All rights reserved. -- ------------------------------------------------------------------------- --- Please see the document "General Decimal Arithmetic Testcases" -- --- at http://www2.hursley.ibm.com/decimal for the description of -- --- these testcases. -- --- -- --- These testcases are experimental ('beta' versions), and they -- --- may contain errors. They are offered on an as-is basis. In -- --- particular, achieving the same results as the tests here is not -- --- a guarantee that an implementation complies with any Standard -- --- or specification. The tests are not exhaustive. -- --- -- --- Please send comments, suggestions, and corrections to the author: -- --- Mike Cowlishaw, IBM Fellow -- --- IBM UK, PO Box 31, Birmingham Road, Warwick CV34 5JL, UK -- --- mfc at uk.ibm.com -- ------------------------------------------------------------------------- -version: 2.59 - --- [New 2006.11.27] - -precision: 9 -maxExponent: 999 -minExponent: -999 -extended: 1 -clamp: 1 -rounding: half_even - -clasx001 class 0 -> +Zero -clasx002 class 0.00 -> +Zero -clasx003 class 0E+5 -> +Zero -clasx004 class 1E-1007 -> +Subnormal -clasx005 class 0.1E-999 -> +Subnormal -clasx006 class 0.99999999E-999 -> +Subnormal -clasx007 class 1.00000000E-999 -> +Normal -clasx008 class 1E-999 -> +Normal -clasx009 class 1E-100 -> +Normal -clasx010 class 1E-10 -> +Normal -clasx012 class 1E-1 -> +Normal -clasx013 class 1 -> +Normal -clasx014 class 2.50 -> +Normal -clasx015 class 100.100 -> +Normal -clasx016 class 1E+30 -> +Normal -clasx017 class 1E+999 -> +Normal -clasx018 class 9.99999999E+999 -> +Normal -clasx019 class Inf -> +Infinity - -clasx021 class -0 -> -Zero -clasx022 class -0.00 -> -Zero -clasx023 class -0E+5 -> -Zero -clasx024 class -1E-1007 -> -Subnormal -clasx025 class -0.1E-999 -> -Subnormal -clasx026 class -0.99999999E-999 -> -Subnormal -clasx027 class -1.00000000E-999 -> -Normal -clasx028 class -1E-999 -> -Normal -clasx029 class -1E-100 -> -Normal -clasx030 class -1E-10 -> -Normal -clasx032 class -1E-1 -> -Normal -clasx033 class -1 -> -Normal -clasx034 class -2.50 -> -Normal -clasx035 class -100.100 -> -Normal -clasx036 class -1E+30 -> -Normal -clasx037 class -1E+999 -> -Normal -clasx038 class -9.99999999E+999 -> -Normal -clasx039 class -Inf -> -Infinity - -clasx041 class NaN -> NaN -clasx042 class -NaN -> NaN -clasx043 class +NaN12345 -> NaN -clasx044 class sNaN -> sNaN -clasx045 class -sNaN -> sNaN -clasx046 class +sNaN12345 -> sNaN - - --- decimal64 bounds - -precision: 16 -maxExponent: 384 -minExponent: -383 -clamp: 1 -rounding: half_even - -clasx201 class 0 -> +Zero -clasx202 class 0.00 -> +Zero -clasx203 class 0E+5 -> +Zero -clasx204 class 1E-396 -> +Subnormal -clasx205 class 0.1E-383 -> +Subnormal -clasx206 class 0.999999999999999E-383 -> +Subnormal -clasx207 class 1.000000000000000E-383 -> +Normal -clasx208 class 1E-383 -> +Normal -clasx209 class 1E-100 -> +Normal -clasx210 class 1E-10 -> +Normal -clasx212 class 1E-1 -> +Normal -clasx213 class 1 -> +Normal -clasx214 class 2.50 -> +Normal -clasx215 class 100.100 -> +Normal -clasx216 class 1E+30 -> +Normal -clasx217 class 1E+384 -> +Normal -clasx218 class 9.999999999999999E+384 -> +Normal -clasx219 class Inf -> +Infinity - -clasx221 class -0 -> -Zero -clasx222 class -0.00 -> -Zero -clasx223 class -0E+5 -> -Zero -clasx224 class -1E-396 -> -Subnormal -clasx225 class -0.1E-383 -> -Subnormal -clasx226 class -0.999999999999999E-383 -> -Subnormal -clasx227 class -1.000000000000000E-383 -> -Normal -clasx228 class -1E-383 -> -Normal -clasx229 class -1E-100 -> -Normal -clasx230 class -1E-10 -> -Normal -clasx232 class -1E-1 -> -Normal -clasx233 class -1 -> -Normal -clasx234 class -2.50 -> -Normal -clasx235 class -100.100 -> -Normal -clasx236 class -1E+30 -> -Normal -clasx237 class -1E+384 -> -Normal -clasx238 class -9.999999999999999E+384 -> -Normal -clasx239 class -Inf -> -Infinity - -clasx241 class NaN -> NaN -clasx242 class -NaN -> NaN -clasx243 class +NaN12345 -> NaN -clasx244 class sNaN -> sNaN -clasx245 class -sNaN -> sNaN -clasx246 class +sNaN12345 -> sNaN - - - +------------------------------------------------------------------------ +-- class.decTest -- Class operations -- +-- Copyright (c) IBM Corporation, 1981, 2008. All rights reserved. -- +------------------------------------------------------------------------ +-- Please see the document "General Decimal Arithmetic Testcases" -- +-- at http://www2.hursley.ibm.com/decimal for the description of -- +-- these testcases. -- +-- -- +-- These testcases are experimental ('beta' versions), and they -- +-- may contain errors. They are offered on an as-is basis. In -- +-- particular, achieving the same results as the tests here is not -- +-- a guarantee that an implementation complies with any Standard -- +-- or specification. The tests are not exhaustive. -- +-- -- +-- Please send comments, suggestions, and corrections to the author: -- +-- Mike Cowlishaw, IBM Fellow -- +-- IBM UK, PO Box 31, Birmingham Road, Warwick CV34 5JL, UK -- +-- mfc at uk.ibm.com -- +------------------------------------------------------------------------ +version: 2.59 + +-- [New 2006.11.27] + +precision: 9 +maxExponent: 999 +minExponent: -999 +extended: 1 +clamp: 1 +rounding: half_even + +clasx001 class 0 -> +Zero +clasx002 class 0.00 -> +Zero +clasx003 class 0E+5 -> +Zero +clasx004 class 1E-1007 -> +Subnormal +clasx005 class 0.1E-999 -> +Subnormal +clasx006 class 0.99999999E-999 -> +Subnormal +clasx007 class 1.00000000E-999 -> +Normal +clasx008 class 1E-999 -> +Normal +clasx009 class 1E-100 -> +Normal +clasx010 class 1E-10 -> +Normal +clasx012 class 1E-1 -> +Normal +clasx013 class 1 -> +Normal +clasx014 class 2.50 -> +Normal +clasx015 class 100.100 -> +Normal +clasx016 class 1E+30 -> +Normal +clasx017 class 1E+999 -> +Normal +clasx018 class 9.99999999E+999 -> +Normal +clasx019 class Inf -> +Infinity + +clasx021 class -0 -> -Zero +clasx022 class -0.00 -> -Zero +clasx023 class -0E+5 -> -Zero +clasx024 class -1E-1007 -> -Subnormal +clasx025 class -0.1E-999 -> -Subnormal +clasx026 class -0.99999999E-999 -> -Subnormal +clasx027 class -1.00000000E-999 -> -Normal +clasx028 class -1E-999 -> -Normal +clasx029 class -1E-100 -> -Normal +clasx030 class -1E-10 -> -Normal +clasx032 class -1E-1 -> -Normal +clasx033 class -1 -> -Normal +clasx034 class -2.50 -> -Normal +clasx035 class -100.100 -> -Normal +clasx036 class -1E+30 -> -Normal +clasx037 class -1E+999 -> -Normal +clasx038 class -9.99999999E+999 -> -Normal +clasx039 class -Inf -> -Infinity + +clasx041 class NaN -> NaN +clasx042 class -NaN -> NaN +clasx043 class +NaN12345 -> NaN +clasx044 class sNaN -> sNaN +clasx045 class -sNaN -> sNaN +clasx046 class +sNaN12345 -> sNaN + + +-- decimal64 bounds + +precision: 16 +maxExponent: 384 +minExponent: -383 +clamp: 1 +rounding: half_even + +clasx201 class 0 -> +Zero +clasx202 class 0.00 -> +Zero +clasx203 class 0E+5 -> +Zero +clasx204 class 1E-396 -> +Subnormal +clasx205 class 0.1E-383 -> +Subnormal +clasx206 class 0.999999999999999E-383 -> +Subnormal +clasx207 class 1.000000000000000E-383 -> +Normal +clasx208 class 1E-383 -> +Normal +clasx209 class 1E-100 -> +Normal +clasx210 class 1E-10 -> +Normal +clasx212 class 1E-1 -> +Normal +clasx213 class 1 -> +Normal +clasx214 class 2.50 -> +Normal +clasx215 class 100.100 -> +Normal +clasx216 class 1E+30 -> +Normal +clasx217 class 1E+384 -> +Normal +clasx218 class 9.999999999999999E+384 -> +Normal +clasx219 class Inf -> +Infinity + +clasx221 class -0 -> -Zero +clasx222 class -0.00 -> -Zero +clasx223 class -0E+5 -> -Zero +clasx224 class -1E-396 -> -Subnormal +clasx225 class -0.1E-383 -> -Subnormal +clasx226 class -0.999999999999999E-383 -> -Subnormal +clasx227 class -1.000000000000000E-383 -> -Normal +clasx228 class -1E-383 -> -Normal +clasx229 class -1E-100 -> -Normal +clasx230 class -1E-10 -> -Normal +clasx232 class -1E-1 -> -Normal +clasx233 class -1 -> -Normal +clasx234 class -2.50 -> -Normal +clasx235 class -100.100 -> -Normal +clasx236 class -1E+30 -> -Normal +clasx237 class -1E+384 -> -Normal +clasx238 class -9.999999999999999E+384 -> -Normal +clasx239 class -Inf -> -Infinity + +clasx241 class NaN -> NaN +clasx242 class -NaN -> NaN +clasx243 class +NaN12345 -> NaN +clasx244 class sNaN -> sNaN +clasx245 class -sNaN -> sNaN +clasx246 class +sNaN12345 -> sNaN + + + diff --git a/lib-python/2.7/test/decimaltestdata/comparetotal.decTest b/lib-python/2.7/test/decimaltestdata/comparetotal.decTest --- a/lib-python/2.7/test/decimaltestdata/comparetotal.decTest +++ b/lib-python/2.7/test/decimaltestdata/comparetotal.decTest @@ -1,798 +1,798 @@ ------------------------------------------------------------------------- --- comparetotal.decTest -- decimal comparison using total ordering -- --- Copyright (c) IBM Corporation, 1981, 2008. All rights reserved. -- ------------------------------------------------------------------------- --- Please see the document "General Decimal Arithmetic Testcases" -- --- at http://www2.hursley.ibm.com/decimal for the description of -- --- these testcases. -- --- -- --- These testcases are experimental ('beta' versions), and they -- --- may contain errors. They are offered on an as-is basis. In -- --- particular, achieving the same results as the tests here is not -- --- a guarantee that an implementation complies with any Standard -- --- or specification. The tests are not exhaustive. -- --- -- --- Please send comments, suggestions, and corrections to the author: -- --- Mike Cowlishaw, IBM Fellow -- --- IBM UK, PO Box 31, Birmingham Road, Warwick CV34 5JL, UK -- --- mfc at uk.ibm.com -- ------------------------------------------------------------------------- -version: 2.59 - --- Note that we cannot assume add/subtract tests cover paths adequately, --- here, because the code might be quite different (comparison cannot --- overflow or underflow, so actual subtractions are not necessary). --- Similarly, comparetotal will have some radically different paths --- than compare. - -extended: 1 -precision: 16 -rounding: half_up -maxExponent: 384 -minExponent: -383 - --- sanity checks -cotx001 comparetotal -2 -2 -> 0 -cotx002 comparetotal -2 -1 -> -1 -cotx003 comparetotal -2 0 -> -1 -cotx004 comparetotal -2 1 -> -1 -cotx005 comparetotal -2 2 -> -1 -cotx006 comparetotal -1 -2 -> 1 -cotx007 comparetotal -1 -1 -> 0 -cotx008 comparetotal -1 0 -> -1 -cotx009 comparetotal -1 1 -> -1 -cotx010 comparetotal -1 2 -> -1 -cotx011 comparetotal 0 -2 -> 1 -cotx012 comparetotal 0 -1 -> 1 -cotx013 comparetotal 0 0 -> 0 -cotx014 comparetotal 0 1 -> -1 -cotx015 comparetotal 0 2 -> -1 -cotx016 comparetotal 1 -2 -> 1 -cotx017 comparetotal 1 -1 -> 1 -cotx018 comparetotal 1 0 -> 1 -cotx019 comparetotal 1 1 -> 0 -cotx020 comparetotal 1 2 -> -1 -cotx021 comparetotal 2 -2 -> 1 -cotx022 comparetotal 2 -1 -> 1 -cotx023 comparetotal 2 0 -> 1 -cotx025 comparetotal 2 1 -> 1 -cotx026 comparetotal 2 2 -> 0 - -cotx031 comparetotal -20 -20 -> 0 -cotx032 comparetotal -20 -10 -> -1 -cotx033 comparetotal -20 00 -> -1 -cotx034 comparetotal -20 10 -> -1 -cotx035 comparetotal -20 20 -> -1 -cotx036 comparetotal -10 -20 -> 1 -cotx037 comparetotal -10 -10 -> 0 -cotx038 comparetotal -10 00 -> -1 -cotx039 comparetotal -10 10 -> -1 -cotx040 comparetotal -10 20 -> -1 -cotx041 comparetotal 00 -20 -> 1 -cotx042 comparetotal 00 -10 -> 1 -cotx043 comparetotal 00 00 -> 0 -cotx044 comparetotal 00 10 -> -1 -cotx045 comparetotal 00 20 -> -1 -cotx046 comparetotal 10 -20 -> 1 -cotx047 comparetotal 10 -10 -> 1 -cotx048 comparetotal 10 00 -> 1 -cotx049 comparetotal 10 10 -> 0 -cotx050 comparetotal 10 20 -> -1 -cotx051 comparetotal 20 -20 -> 1 -cotx052 comparetotal 20 -10 -> 1 -cotx053 comparetotal 20 00 -> 1 -cotx055 comparetotal 20 10 -> 1 -cotx056 comparetotal 20 20 -> 0 - -cotx061 comparetotal -2.0 -2.0 -> 0 -cotx062 comparetotal -2.0 -1.0 -> -1 -cotx063 comparetotal -2.0 0.0 -> -1 -cotx064 comparetotal -2.0 1.0 -> -1 -cotx065 comparetotal -2.0 2.0 -> -1 -cotx066 comparetotal -1.0 -2.0 -> 1 -cotx067 comparetotal -1.0 -1.0 -> 0 -cotx068 comparetotal -1.0 0.0 -> -1 -cotx069 comparetotal -1.0 1.0 -> -1 -cotx070 comparetotal -1.0 2.0 -> -1 -cotx071 comparetotal 0.0 -2.0 -> 1 -cotx072 comparetotal 0.0 -1.0 -> 1 -cotx073 comparetotal 0.0 0.0 -> 0 -cotx074 comparetotal 0.0 1.0 -> -1 -cotx075 comparetotal 0.0 2.0 -> -1 -cotx076 comparetotal 1.0 -2.0 -> 1 -cotx077 comparetotal 1.0 -1.0 -> 1 -cotx078 comparetotal 1.0 0.0 -> 1 -cotx079 comparetotal 1.0 1.0 -> 0 -cotx080 comparetotal 1.0 2.0 -> -1 -cotx081 comparetotal 2.0 -2.0 -> 1 -cotx082 comparetotal 2.0 -1.0 -> 1 -cotx083 comparetotal 2.0 0.0 -> 1 -cotx085 comparetotal 2.0 1.0 -> 1 -cotx086 comparetotal 2.0 2.0 -> 0 - --- now some cases which might overflow if subtract were used -maxexponent: 999999999 -minexponent: -999999999 -cotx090 comparetotal 9.99999999E+999999999 9.99999999E+999999999 -> 0 -cotx091 comparetotal -9.99999999E+999999999 9.99999999E+999999999 -> -1 -cotx092 comparetotal 9.99999999E+999999999 -9.99999999E+999999999 -> 1 -cotx093 comparetotal -9.99999999E+999999999 -9.99999999E+999999999 -> 0 - --- Examples -cotx094 comparetotal 12.73 127.9 -> -1 -cotx095 comparetotal -127 12 -> -1 -cotx096 comparetotal 12.30 12.3 -> -1 -cotx097 comparetotal 12.30 12.30 -> 0 -cotx098 comparetotal 12.3 12.300 -> 1 -cotx099 comparetotal 12.3 NaN -> -1 - --- some differing length/exponent cases --- in this first group, compare would compare all equal -cotx100 comparetotal 7.0 7.0 -> 0 -cotx101 comparetotal 7.0 7 -> -1 -cotx102 comparetotal 7 7.0 -> 1 -cotx103 comparetotal 7E+0 7.0 -> 1 -cotx104 comparetotal 70E-1 7.0 -> 0 -cotx105 comparetotal 0.7E+1 7 -> 0 -cotx106 comparetotal 70E-1 7 -> -1 -cotx107 comparetotal 7.0 7E+0 -> -1 -cotx108 comparetotal 7.0 70E-1 -> 0 -cotx109 comparetotal 7 0.7E+1 -> 0 -cotx110 comparetotal 7 70E-1 -> 1 - -cotx120 comparetotal 8.0 7.0 -> 1 -cotx121 comparetotal 8.0 7 -> 1 -cotx122 comparetotal 8 7.0 -> 1 -cotx123 comparetotal 8E+0 7.0 -> 1 -cotx124 comparetotal 80E-1 7.0 -> 1 -cotx125 comparetotal 0.8E+1 7 -> 1 -cotx126 comparetotal 80E-1 7 -> 1 -cotx127 comparetotal 8.0 7E+0 -> 1 -cotx128 comparetotal 8.0 70E-1 -> 1 -cotx129 comparetotal 8 0.7E+1 -> 1 -cotx130 comparetotal 8 70E-1 -> 1 - -cotx140 comparetotal 8.0 9.0 -> -1 -cotx141 comparetotal 8.0 9 -> -1 -cotx142 comparetotal 8 9.0 -> -1 -cotx143 comparetotal 8E+0 9.0 -> -1 -cotx144 comparetotal 80E-1 9.0 -> -1 -cotx145 comparetotal 0.8E+1 9 -> -1 -cotx146 comparetotal 80E-1 9 -> -1 -cotx147 comparetotal 8.0 9E+0 -> -1 -cotx148 comparetotal 8.0 90E-1 -> -1 -cotx149 comparetotal 8 0.9E+1 -> -1 -cotx150 comparetotal 8 90E-1 -> -1 - --- and again, with sign changes -+ .. -cotx200 comparetotal -7.0 7.0 -> -1 -cotx201 comparetotal -7.0 7 -> -1 -cotx202 comparetotal -7 7.0 -> -1 -cotx203 comparetotal -7E+0 7.0 -> -1 -cotx204 comparetotal -70E-1 7.0 -> -1 -cotx205 comparetotal -0.7E+1 7 -> -1 -cotx206 comparetotal -70E-1 7 -> -1 -cotx207 comparetotal -7.0 7E+0 -> -1 -cotx208 comparetotal -7.0 70E-1 -> -1 -cotx209 comparetotal -7 0.7E+1 -> -1 -cotx210 comparetotal -7 70E-1 -> -1 - -cotx220 comparetotal -8.0 7.0 -> -1 -cotx221 comparetotal -8.0 7 -> -1 -cotx222 comparetotal -8 7.0 -> -1 -cotx223 comparetotal -8E+0 7.0 -> -1 -cotx224 comparetotal -80E-1 7.0 -> -1 -cotx225 comparetotal -0.8E+1 7 -> -1 -cotx226 comparetotal -80E-1 7 -> -1 -cotx227 comparetotal -8.0 7E+0 -> -1 -cotx228 comparetotal -8.0 70E-1 -> -1 -cotx229 comparetotal -8 0.7E+1 -> -1 -cotx230 comparetotal -8 70E-1 -> -1 - -cotx240 comparetotal -8.0 9.0 -> -1 -cotx241 comparetotal -8.0 9 -> -1 -cotx242 comparetotal -8 9.0 -> -1 -cotx243 comparetotal -8E+0 9.0 -> -1 -cotx244 comparetotal -80E-1 9.0 -> -1 -cotx245 comparetotal -0.8E+1 9 -> -1 -cotx246 comparetotal -80E-1 9 -> -1 -cotx247 comparetotal -8.0 9E+0 -> -1 -cotx248 comparetotal -8.0 90E-1 -> -1 -cotx249 comparetotal -8 0.9E+1 -> -1 -cotx250 comparetotal -8 90E-1 -> -1 - --- and again, with sign changes +- .. -cotx300 comparetotal 7.0 -7.0 -> 1 -cotx301 comparetotal 7.0 -7 -> 1 -cotx302 comparetotal 7 -7.0 -> 1 -cotx303 comparetotal 7E+0 -7.0 -> 1 -cotx304 comparetotal 70E-1 -7.0 -> 1 -cotx305 comparetotal .7E+1 -7 -> 1 -cotx306 comparetotal 70E-1 -7 -> 1 -cotx307 comparetotal 7.0 -7E+0 -> 1 -cotx308 comparetotal 7.0 -70E-1 -> 1 -cotx309 comparetotal 7 -.7E+1 -> 1 -cotx310 comparetotal 7 -70E-1 -> 1 - -cotx320 comparetotal 8.0 -7.0 -> 1 -cotx321 comparetotal 8.0 -7 -> 1 -cotx322 comparetotal 8 -7.0 -> 1 -cotx323 comparetotal 8E+0 -7.0 -> 1 -cotx324 comparetotal 80E-1 -7.0 -> 1 -cotx325 comparetotal .8E+1 -7 -> 1 -cotx326 comparetotal 80E-1 -7 -> 1 -cotx327 comparetotal 8.0 -7E+0 -> 1 -cotx328 comparetotal 8.0 -70E-1 -> 1 -cotx329 comparetotal 8 -.7E+1 -> 1 -cotx330 comparetotal 8 -70E-1 -> 1 - -cotx340 comparetotal 8.0 -9.0 -> 1 -cotx341 comparetotal 8.0 -9 -> 1 -cotx342 comparetotal 8 -9.0 -> 1 -cotx343 comparetotal 8E+0 -9.0 -> 1 -cotx344 comparetotal 80E-1 -9.0 -> 1 -cotx345 comparetotal .8E+1 -9 -> 1 -cotx346 comparetotal 80E-1 -9 -> 1 -cotx347 comparetotal 8.0 -9E+0 -> 1 -cotx348 comparetotal 8.0 -90E-1 -> 1 -cotx349 comparetotal 8 -.9E+1 -> 1 -cotx350 comparetotal 8 -90E-1 -> 1 - --- and again, with sign changes -- .. -cotx400 comparetotal -7.0 -7.0 -> 0 -cotx401 comparetotal -7.0 -7 -> 1 -cotx402 comparetotal -7 -7.0 -> -1 -cotx403 comparetotal -7E+0 -7.0 -> -1 -cotx404 comparetotal -70E-1 -7.0 -> 0 -cotx405 comparetotal -.7E+1 -7 -> 0 -cotx406 comparetotal -70E-1 -7 -> 1 -cotx407 comparetotal -7.0 -7E+0 -> 1 -cotx408 comparetotal -7.0 -70E-1 -> 0 -cotx409 comparetotal -7 -.7E+1 -> 0 From noreply at buildbot.pypy.org Mon Jan 30 22:39:04 2012 From: noreply at buildbot.pypy.org (mattip) Date: Mon, 30 Jan 2012 22:39:04 +0100 (CET) Subject: [pypy-commit] pypy default: add scalar test to dot Message-ID: <20120130213904.64B7D7106F3@wyvern.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r51976:0e3677233cc5 Date: 2012-01-29 05:34 +0200 http://bitbucket.org/pypy/pypy/changeset/0e3677233cc5/ Log: add scalar test to dot diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -942,11 +942,13 @@ assert sum(a[2,3,2,:] * b[1,2,:,2]) == 499128 def test_dot_constant(self): - from _numpypy import array + from _numpypy import array, dot a = array(range(5)) b = a.dot(2.5) for i in xrange(5): assert b[i] == 2.5 * a[i] + c = dot(4, 3.0) + assert c == 12.0 def test_dtype_guessing(self): from _numpypy import array, dtype, float64, int8, bool_ From noreply at buildbot.pypy.org Mon Jan 30 22:43:27 2012 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Mon, 30 Jan 2012 22:43:27 +0100 (CET) Subject: [pypy-commit] pypy default: merged upstream Message-ID: <20120130214327.1DB957106F4@wyvern.cs.uni-duesseldorf.de> Author: Alex Gaynor <alex.gaynor at gmail.com> Branch: Changeset: r51978:2c9b35e6eb65 Date: 2012-01-30 16:43 -0500 http://bitbucket.org/pypy/pypy/changeset/2c9b35e6eb65/ Log: merged upstream diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -942,11 +942,13 @@ assert sum(a[2,3,2,:] * b[1,2,:,2]) == 499128 def test_dot_constant(self): - from _numpypy import array + from _numpypy import array, dot a = array(range(5)) b = a.dot(2.5) for i in xrange(5): assert b[i] == 2.5 * a[i] + c = dot(4, 3.0) + assert c == 12.0 def test_dtype_guessing(self): from _numpypy import array, dtype, float64, int8, bool_ From noreply at buildbot.pypy.org Mon Jan 30 22:43:25 2012 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Mon, 30 Jan 2012 22:43:25 +0100 (CET) Subject: [pypy-commit] pypy default: Remove the modified email package, we have cjk codecs now. Message-ID: <20120130214325.CE45E7106F3@wyvern.cs.uni-duesseldorf.de> Author: Alex Gaynor <alex.gaynor at gmail.com> Branch: Changeset: r51977:955920bb9326 Date: 2012-01-30 16:42 -0500 http://bitbucket.org/pypy/pypy/changeset/955920bb9326/ Log: Remove the modified email package, we have cjk codecs now. diff too long, truncating to 10000 out of 14097 lines diff --git a/lib-python/modified-2.7/email/__init__.py b/lib-python/modified-2.7/email/__init__.py deleted file mode 100644 --- a/lib-python/modified-2.7/email/__init__.py +++ /dev/null @@ -1,123 +0,0 @@ -# Copyright (C) 2001-2006 Python Software Foundation -# Author: Barry Warsaw -# Contact: email-sig at python.org - -"""A package for parsing, handling, and generating email messages.""" - -__version__ = '4.0.3' - -__all__ = [ - # Old names - 'base64MIME', - 'Charset', - 'Encoders', - 'Errors', - 'Generator', - 'Header', - 'Iterators', - 'Message', - 'MIMEAudio', - 'MIMEBase', - 'MIMEImage', - 'MIMEMessage', - 'MIMEMultipart', - 'MIMENonMultipart', - 'MIMEText', - 'Parser', - 'quopriMIME', - 'Utils', - 'message_from_string', - 'message_from_file', - # new names - 'base64mime', - 'charset', - 'encoders', - 'errors', - 'generator', - 'header', - 'iterators', - 'message', - 'mime', - 'parser', - 'quoprimime', - 'utils', - ] - - - -# Some convenience routines. Don't import Parser and Message as side-effects -# of importing email since those cascadingly import most of the rest of the -# email package. -def message_from_string(s, *args, **kws): - """Parse a string into a Message object model. - - Optional _class and strict are passed to the Parser constructor. - """ - from email.parser import Parser - return Parser(*args, **kws).parsestr(s) - - -def message_from_file(fp, *args, **kws): - """Read a file and parse its contents into a Message object model. - - Optional _class and strict are passed to the Parser constructor. - """ - from email.parser import Parser - return Parser(*args, **kws).parse(fp) - - - -# Lazy loading to provide name mapping from new-style names (PEP 8 compatible -# email 4.0 module names), to old-style names (email 3.0 module names). -import sys - -class LazyImporter(object): - def __init__(self, module_name): - self.__name__ = 'email.' + module_name - - def __getattr__(self, name): - __import__(self.__name__) - mod = sys.modules[self.__name__] - self.__dict__.update(mod.__dict__) - return getattr(mod, name) - - -_LOWERNAMES = [ - # email.<old name> -> email.<new name is lowercased old name> - 'Charset', - 'Encoders', - 'Errors', - 'FeedParser', - 'Generator', - 'Header', - 'Iterators', - 'Message', - 'Parser', - 'Utils', - 'base64MIME', - 'quopriMIME', - ] - -_MIMENAMES = [ - # email.MIME<old name> -> email.mime.<new name is lowercased old name> - 'Audio', - 'Base', - 'Image', - 'Message', - 'Multipart', - 'NonMultipart', - 'Text', - ] - -for _name in _LOWERNAMES: - importer = LazyImporter(_name.lower()) - sys.modules['email.' + _name] = importer - setattr(sys.modules['email'], _name, importer) - - -import email.mime -for _name in _MIMENAMES: - importer = LazyImporter('mime.' + _name.lower()) - sys.modules['email.MIME' + _name] = importer - setattr(sys.modules['email'], 'MIME' + _name, importer) - setattr(sys.modules['email.mime'], _name, importer) diff --git a/lib-python/modified-2.7/email/_parseaddr.py b/lib-python/modified-2.7/email/_parseaddr.py deleted file mode 100644 --- a/lib-python/modified-2.7/email/_parseaddr.py +++ /dev/null @@ -1,497 +0,0 @@ -# Copyright (C) 2002-2007 Python Software Foundation -# Contact: email-sig at python.org - -"""Email address parsing code. - -Lifted directly from rfc822.py. This should eventually be rewritten. -""" - -__all__ = [ - 'mktime_tz', - 'parsedate', - 'parsedate_tz', - 'quote', - ] - -import time - -SPACE = ' ' -EMPTYSTRING = '' -COMMASPACE = ', ' - -# Parse a date field -_monthnames = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', - 'aug', 'sep', 'oct', 'nov', 'dec', - 'january', 'february', 'march', 'april', 'may', 'june', 'july', - 'august', 'september', 'october', 'november', 'december'] - -_daynames = ['mon', 'tue', 'wed', 'thu', 'fri', 'sat', 'sun'] - -# The timezone table does not include the military time zones defined -# in RFC822, other than Z. According to RFC1123, the description in -# RFC822 gets the signs wrong, so we can't rely on any such time -# zones. RFC1123 recommends that numeric timezone indicators be used -# instead of timezone names. - -_timezones = {'UT':0, 'UTC':0, 'GMT':0, 'Z':0, - 'AST': -400, 'ADT': -300, # Atlantic (used in Canada) - 'EST': -500, 'EDT': -400, # Eastern - 'CST': -600, 'CDT': -500, # Central - 'MST': -700, 'MDT': -600, # Mountain - 'PST': -800, 'PDT': -700 # Pacific - } - - -def parsedate_tz(data): - """Convert a date string to a time tuple. - - Accounts for military timezones. - """ - data = data.split() - # The FWS after the comma after the day-of-week is optional, so search and - # adjust for this. - if data[0].endswith(',') or data[0].lower() in _daynames: - # There's a dayname here. Skip it - del data[0] - else: - i = data[0].rfind(',') - if i >= 0: - data[0] = data[0][i+1:] - if len(data) == 3: # RFC 850 date, deprecated - stuff = data[0].split('-') - if len(stuff) == 3: - data = stuff + data[1:] - if len(data) == 4: - s = data[3] - i = s.find('+') - if i > 0: - data[3:] = [s[:i], s[i+1:]] - else: - data.append('') # Dummy tz - if len(data) < 5: - return None - data = data[:5] - [dd, mm, yy, tm, tz] = data - mm = mm.lower() - if mm not in _monthnames: - dd, mm = mm, dd.lower() - if mm not in _monthnames: - return None - mm = _monthnames.index(mm) + 1 - if mm > 12: - mm -= 12 - if dd[-1] == ',': - dd = dd[:-1] - i = yy.find(':') - if i > 0: - yy, tm = tm, yy - if yy[-1] == ',': - yy = yy[:-1] - if not yy[0].isdigit(): - yy, tz = tz, yy - if tm[-1] == ',': - tm = tm[:-1] - tm = tm.split(':') - if len(tm) == 2: - [thh, tmm] = tm - tss = '0' - elif len(tm) == 3: - [thh, tmm, tss] = tm - else: - return None - try: - yy = int(yy) - dd = int(dd) - thh = int(thh) - tmm = int(tmm) - tss = int(tss) - except ValueError: - return None - # Check for a yy specified in two-digit format, then convert it to the - # appropriate four-digit format, according to the POSIX standard. RFC 822 - # calls for a two-digit yy, but RFC 2822 (which obsoletes RFC 822) - # mandates a 4-digit yy. For more information, see the documentation for - # the time module. - if yy < 100: - # The year is between 1969 and 1999 (inclusive). - if yy > 68: - yy += 1900 - # The year is between 2000 and 2068 (inclusive). - else: - yy += 2000 - tzoffset = None - tz = tz.upper() - if tz in _timezones: - tzoffset = _timezones[tz] - else: - try: - tzoffset = int(tz) - except ValueError: - pass - # Convert a timezone offset into seconds ; -0500 -> -18000 - if tzoffset: - if tzoffset < 0: - tzsign = -1 - tzoffset = -tzoffset - else: - tzsign = 1 - tzoffset = tzsign * ( (tzoffset//100)*3600 + (tzoffset % 100)*60) - # Daylight Saving Time flag is set to -1, since DST is unknown. - return yy, mm, dd, thh, tmm, tss, 0, 1, -1, tzoffset - - -def parsedate(data): - """Convert a time string to a time tuple.""" - t = parsedate_tz(data) - if isinstance(t, tuple): - return t[:9] - else: - return t - - -def mktime_tz(data): - """Turn a 10-tuple as returned by parsedate_tz() into a UTC timestamp.""" - if data[9] is None: - # No zone info, so localtime is better assumption than GMT - return time.mktime(data[:8] + (-1,)) - else: - t = time.mktime(data[:8] + (0,)) - return t - data[9] - time.timezone - - -def quote(str): - """Prepare string to be used in a quoted string. - - Turns backslash and double quote characters into quoted pairs. These - are the only characters that need to be quoted inside a quoted string. - Does not add the surrounding double quotes. - """ - return str.replace('\\', '\\\\').replace('"', '\\"') - - -class AddrlistClass: - """Address parser class by Ben Escoto. - - To understand what this class does, it helps to have a copy of RFC 2822 in - front of you. - - Note: this class interface is deprecated and may be removed in the future. - Use rfc822.AddressList instead. - """ - - def __init__(self, field): - """Initialize a new instance. - - `field' is an unparsed address header field, containing - one or more addresses. - """ - self.specials = '()<>@,:;.\"[]' - self.pos = 0 - self.LWS = ' \t' - self.CR = '\r\n' - self.FWS = self.LWS + self.CR - self.atomends = self.specials + self.LWS + self.CR - # Note that RFC 2822 now specifies `.' as obs-phrase, meaning that it - # is obsolete syntax. RFC 2822 requires that we recognize obsolete - # syntax, so allow dots in phrases. - self.phraseends = self.atomends.replace('.', '') - self.field = field - self.commentlist = [] - - def gotonext(self): - """Parse up to the start of the next address.""" - while self.pos < len(self.field): - if self.field[self.pos] in self.LWS + '\n\r': - self.pos += 1 - elif self.field[self.pos] == '(': - self.commentlist.append(self.getcomment()) - else: - break - - def getaddrlist(self): - """Parse all addresses. - - Returns a list containing all of the addresses. - """ - result = [] - while self.pos < len(self.field): - ad = self.getaddress() - if ad: - result += ad - else: - result.append(('', '')) - return result - - def getaddress(self): - """Parse the next address.""" - self.commentlist = [] - self.gotonext() - - oldpos = self.pos - oldcl = self.commentlist - plist = self.getphraselist() - - self.gotonext() - returnlist = [] - - if self.pos >= len(self.field): - # Bad email address technically, no domain. - if plist: - returnlist = [(SPACE.join(self.commentlist), plist[0])] - - elif self.field[self.pos] in '.@': - # email address is just an addrspec - # this isn't very efficient since we start over - self.pos = oldpos - self.commentlist = oldcl - addrspec = self.getaddrspec() - returnlist = [(SPACE.join(self.commentlist), addrspec)] - - elif self.field[self.pos] == ':': - # address is a group - returnlist = [] - - fieldlen = len(self.field) - self.pos += 1 - while self.pos < len(self.field): - self.gotonext() - if self.pos < fieldlen and self.field[self.pos] == ';': - self.pos += 1 - break - returnlist = returnlist + self.getaddress() - - elif self.field[self.pos] == '<': - # Address is a phrase then a route addr - routeaddr = self.getrouteaddr() - - if self.commentlist: - returnlist = [(SPACE.join(plist) + ' (' + - ' '.join(self.commentlist) + ')', routeaddr)] - else: - returnlist = [(SPACE.join(plist), routeaddr)] - - else: - if plist: - returnlist = [(SPACE.join(self.commentlist), plist[0])] - elif self.field[self.pos] in self.specials: - self.pos += 1 - - self.gotonext() - if self.pos < len(self.field) and self.field[self.pos] == ',': - self.pos += 1 - return returnlist - - def getrouteaddr(self): - """Parse a route address (Return-path value). - - This method just skips all the route stuff and returns the addrspec. - """ - if self.field[self.pos] != '<': - return - - expectroute = False - self.pos += 1 - self.gotonext() - adlist = '' - while self.pos < len(self.field): - if expectroute: - self.getdomain() - expectroute = False - elif self.field[self.pos] == '>': - self.pos += 1 - break - elif self.field[self.pos] == '@': - self.pos += 1 - expectroute = True - elif self.field[self.pos] == ':': - self.pos += 1 - else: - adlist = self.getaddrspec() - self.pos += 1 - break - self.gotonext() - - return adlist - - def getaddrspec(self): - """Parse an RFC 2822 addr-spec.""" - aslist = [] - - self.gotonext() - while self.pos < len(self.field): - if self.field[self.pos] == '.': - aslist.append('.') - self.pos += 1 - elif self.field[self.pos] == '"': - aslist.append('"%s"' % quote(self.getquote())) - elif self.field[self.pos] in self.atomends: - break - else: - aslist.append(self.getatom()) - self.gotonext() - - if self.pos >= len(self.field) or self.field[self.pos] != '@': - return EMPTYSTRING.join(aslist) - - aslist.append('@') - self.pos += 1 - self.gotonext() - return EMPTYSTRING.join(aslist) + self.getdomain() - - def getdomain(self): - """Get the complete domain name from an address.""" - sdlist = [] - while self.pos < len(self.field): - if self.field[self.pos] in self.LWS: - self.pos += 1 - elif self.field[self.pos] == '(': - self.commentlist.append(self.getcomment()) - elif self.field[self.pos] == '[': - sdlist.append(self.getdomainliteral()) - elif self.field[self.pos] == '.': - self.pos += 1 - sdlist.append('.') - elif self.field[self.pos] in self.atomends: - break - else: - sdlist.append(self.getatom()) - return EMPTYSTRING.join(sdlist) - - def getdelimited(self, beginchar, endchars, allowcomments=True): - """Parse a header fragment delimited by special characters. - - `beginchar' is the start character for the fragment. - If self is not looking at an instance of `beginchar' then - getdelimited returns the empty string. - - `endchars' is a sequence of allowable end-delimiting characters. - Parsing stops when one of these is encountered. - - If `allowcomments' is non-zero, embedded RFC 2822 comments are allowed - within the parsed fragment. - """ - if self.field[self.pos] != beginchar: - return '' - - slist = [''] - quote = False - self.pos += 1 - while self.pos < len(self.field): - if quote: - slist.append(self.field[self.pos]) - quote = False - elif self.field[self.pos] in endchars: - self.pos += 1 - break - elif allowcomments and self.field[self.pos] == '(': - slist.append(self.getcomment()) - continue # have already advanced pos from getcomment - elif self.field[self.pos] == '\\': - quote = True - else: - slist.append(self.field[self.pos]) - self.pos += 1 - - return EMPTYSTRING.join(slist) - - def getquote(self): - """Get a quote-delimited fragment from self's field.""" - return self.getdelimited('"', '"\r', False) - - def getcomment(self): - """Get a parenthesis-delimited fragment from self's field.""" - return self.getdelimited('(', ')\r', True) - - def getdomainliteral(self): - """Parse an RFC 2822 domain-literal.""" - return '[%s]' % self.getdelimited('[', ']\r', False) - - def getatom(self, atomends=None): - """Parse an RFC 2822 atom. - - Optional atomends specifies a different set of end token delimiters - (the default is to use self.atomends). This is used e.g. in - getphraselist() since phrase endings must not include the `.' (which - is legal in phrases).""" - atomlist = [''] - if atomends is None: - atomends = self.atomends - - while self.pos < len(self.field): - if self.field[self.pos] in atomends: - break - else: - atomlist.append(self.field[self.pos]) - self.pos += 1 - - return EMPTYSTRING.join(atomlist) - - def getphraselist(self): - """Parse a sequence of RFC 2822 phrases. - - A phrase is a sequence of words, which are in turn either RFC 2822 - atoms or quoted-strings. Phrases are canonicalized by squeezing all - runs of continuous whitespace into one space. - """ - plist = [] - - while self.pos < len(self.field): - if self.field[self.pos] in self.FWS: - self.pos += 1 - elif self.field[self.pos] == '"': - plist.append(self.getquote()) - elif self.field[self.pos] == '(': - self.commentlist.append(self.getcomment()) - elif self.field[self.pos] in self.phraseends: - break - else: - plist.append(self.getatom(self.phraseends)) - - return plist - -class AddressList(AddrlistClass): - """An AddressList encapsulates a list of parsed RFC 2822 addresses.""" - def __init__(self, field): - AddrlistClass.__init__(self, field) - if field: - self.addresslist = self.getaddrlist() - else: - self.addresslist = [] - - def __len__(self): - return len(self.addresslist) - - def __add__(self, other): - # Set union - newaddr = AddressList(None) - newaddr.addresslist = self.addresslist[:] - for x in other.addresslist: - if not x in self.addresslist: - newaddr.addresslist.append(x) - return newaddr - - def __iadd__(self, other): - # Set union, in-place - for x in other.addresslist: - if not x in self.addresslist: - self.addresslist.append(x) - return self - - def __sub__(self, other): - # Set difference - newaddr = AddressList(None) - for x in self.addresslist: - if not x in other.addresslist: - newaddr.addresslist.append(x) - return newaddr - - def __isub__(self, other): - # Set difference, in-place - for x in other.addresslist: - if x in self.addresslist: - self.addresslist.remove(x) - return self - - def __getitem__(self, index): - # Make indexing, slices, and 'in' work - return self.addresslist[index] diff --git a/lib-python/modified-2.7/email/base64mime.py b/lib-python/modified-2.7/email/base64mime.py deleted file mode 100644 --- a/lib-python/modified-2.7/email/base64mime.py +++ /dev/null @@ -1,183 +0,0 @@ -# Copyright (C) 2002-2006 Python Software Foundation -# Author: Ben Gertzfield -# Contact: email-sig at python.org - -"""Base64 content transfer encoding per RFCs 2045-2047. - -This module handles the content transfer encoding method defined in RFC 2045 -to encode arbitrary 8-bit data using the three 8-bit bytes in four 7-bit -characters encoding known as Base64. - -It is used in the MIME standards for email to attach images, audio, and text -using some 8-bit character sets to messages. - -This module provides an interface to encode and decode both headers and bodies -with Base64 encoding. - -RFC 2045 defines a method for including character set information in an -`encoded-word' in a header. This method is commonly used for 8-bit real names -in To:, From:, Cc:, etc. fields, as well as Subject: lines. - -This module does not do the line wrapping or end-of-line character conversion -necessary for proper internationalized headers; it only does dumb encoding and -decoding. To deal with the various line wrapping issues, use the email.header -module. -""" - -__all__ = [ - 'base64_len', - 'body_decode', - 'body_encode', - 'decode', - 'decodestring', - 'encode', - 'encodestring', - 'header_encode', - ] - - -from binascii import b2a_base64, a2b_base64 -from email.utils import fix_eols - -CRLF = '\r\n' -NL = '\n' -EMPTYSTRING = '' - -# See also Charset.py -MISC_LEN = 7 - - - -# Helpers -def base64_len(s): - """Return the length of s when it is encoded with base64.""" - groups_of_3, leftover = divmod(len(s), 3) - # 4 bytes out for each 3 bytes (or nonzero fraction thereof) in. - # Thanks, Tim! - n = groups_of_3 * 4 - if leftover: - n += 4 - return n - - - -def header_encode(header, charset='iso-8859-1', keep_eols=False, - maxlinelen=76, eol=NL): - """Encode a single header line with Base64 encoding in a given charset. - - Defined in RFC 2045, this Base64 encoding is identical to normal Base64 - encoding, except that each line must be intelligently wrapped (respecting - the Base64 encoding), and subsequent lines must start with a space. - - charset names the character set to use to encode the header. It defaults - to iso-8859-1. - - End-of-line characters (\\r, \\n, \\r\\n) will be automatically converted - to the canonical email line separator \\r\\n unless the keep_eols - parameter is True (the default is False). - - Each line of the header will be terminated in the value of eol, which - defaults to "\\n". Set this to "\\r\\n" if you are using the result of - this function directly in email. - - The resulting string will be in the form: - - "=?charset?b?WW/5ciBtYXp66XLrIHf8eiBhIGhhbXBzdGHuciBBIFlv+XIgbWF6euly?=\\n - =?charset?b?6yB3/HogYSBoYW1wc3Rh7nIgQkMgWW/5ciBtYXp66XLrIHf8eiBhIGhh?=" - - with each line wrapped at, at most, maxlinelen characters (defaults to 76 - characters). - """ - # Return empty headers unchanged - if not header: - return header - - if not keep_eols: - header = fix_eols(header) - - # Base64 encode each line, in encoded chunks no greater than maxlinelen in - # length, after the RFC chrome is added in. - base64ed = [] - max_encoded = maxlinelen - len(charset) - MISC_LEN - max_unencoded = max_encoded * 3 // 4 - - for i in range(0, len(header), max_unencoded): - base64ed.append(b2a_base64(header[i:i+max_unencoded])) - - # Now add the RFC chrome to each encoded chunk - lines = [] - for line in base64ed: - # Ignore the last character of each line if it is a newline - if line.endswith(NL): - line = line[:-1] - # Add the chrome - lines.append('=?%s?b?%s?=' % (charset, line)) - # Glue the lines together and return it. BAW: should we be able to - # specify the leading whitespace in the joiner? - joiner = eol + ' ' - return joiner.join(lines) - - - -def encode(s, binary=True, maxlinelen=76, eol=NL): - """Encode a string with base64. - - Each line will be wrapped at, at most, maxlinelen characters (defaults to - 76 characters). - - If binary is False, end-of-line characters will be converted to the - canonical email end-of-line sequence \\r\\n. Otherwise they will be left - verbatim (this is the default). - - Each line of encoded text will end with eol, which defaults to "\\n". Set - this to "\r\n" if you will be using the result of this function directly - in an email. - """ - if not s: - return s - - if not binary: - s = fix_eols(s) - - encvec = [] - max_unencoded = maxlinelen * 3 // 4 - for i in range(0, len(s), max_unencoded): - # BAW: should encode() inherit b2a_base64()'s dubious behavior in - # adding a newline to the encoded string? - enc = b2a_base64(s[i:i + max_unencoded]) - if enc.endswith(NL) and eol != NL: - enc = enc[:-1] + eol - encvec.append(enc) - return EMPTYSTRING.join(encvec) - - -# For convenience and backwards compatibility w/ standard base64 module -body_encode = encode -encodestring = encode - - - -def decode(s, convert_eols=None): - """Decode a raw base64 string. - - If convert_eols is set to a string value, all canonical email linefeeds, - e.g. "\\r\\n", in the decoded text will be converted to the value of - convert_eols. os.linesep is a good choice for convert_eols if you are - decoding a text attachment. - - This function does not parse a full MIME header value encoded with - base64 (like =?iso-8895-1?b?bmloISBuaWgh?=) -- please use the high - level email.header class for that functionality. - """ - if not s: - return s - - dec = a2b_base64(s) - if convert_eols: - return dec.replace(CRLF, convert_eols) - return dec - - -# For convenience and backwards compatibility w/ standard base64 module -body_decode = decode -decodestring = decode diff --git a/lib-python/modified-2.7/email/charset.py b/lib-python/modified-2.7/email/charset.py deleted file mode 100644 --- a/lib-python/modified-2.7/email/charset.py +++ /dev/null @@ -1,397 +0,0 @@ -# Copyright (C) 2001-2006 Python Software Foundation -# Author: Ben Gertzfield, Barry Warsaw -# Contact: email-sig at python.org - -__all__ = [ - 'Charset', - 'add_alias', - 'add_charset', - 'add_codec', - ] - -import codecs -import email.base64mime -import email.quoprimime - -from email import errors -from email.encoders import encode_7or8bit - - - -# Flags for types of header encodings -QP = 1 # Quoted-Printable -BASE64 = 2 # Base64 -SHORTEST = 3 # the shorter of QP and base64, but only for headers - -# In "=?charset?q?hello_world?=", the =?, ?q?, and ?= add up to 7 -MISC_LEN = 7 - -DEFAULT_CHARSET = 'us-ascii' - - - -# Defaults -CHARSETS = { - # input header enc body enc output conv - 'iso-8859-1': (QP, QP, None), - 'iso-8859-2': (QP, QP, None), - 'iso-8859-3': (QP, QP, None), - 'iso-8859-4': (QP, QP, None), - # iso-8859-5 is Cyrillic, and not especially used - # iso-8859-6 is Arabic, also not particularly used - # iso-8859-7 is Greek, QP will not make it readable - # iso-8859-8 is Hebrew, QP will not make it readable - 'iso-8859-9': (QP, QP, None), - 'iso-8859-10': (QP, QP, None), - # iso-8859-11 is Thai, QP will not make it readable - 'iso-8859-13': (QP, QP, None), - 'iso-8859-14': (QP, QP, None), - 'iso-8859-15': (QP, QP, None), - 'iso-8859-16': (QP, QP, None), - 'windows-1252':(QP, QP, None), - 'viscii': (QP, QP, None), - 'us-ascii': (None, None, None), - 'big5': (BASE64, BASE64, None), - 'gb2312': (BASE64, BASE64, None), - 'euc-jp': (BASE64, None, 'iso-2022-jp'), - 'shift_jis': (BASE64, None, 'iso-2022-jp'), - 'iso-2022-jp': (BASE64, None, None), - 'koi8-r': (BASE64, BASE64, None), - 'utf-8': (SHORTEST, BASE64, 'utf-8'), - # We're making this one up to represent raw unencoded 8-bit - '8bit': (None, BASE64, 'utf-8'), - } - -# Aliases for other commonly-used names for character sets. Map -# them to the real ones used in email. -ALIASES = { - 'latin_1': 'iso-8859-1', - 'latin-1': 'iso-8859-1', - 'latin_2': 'iso-8859-2', - 'latin-2': 'iso-8859-2', - 'latin_3': 'iso-8859-3', - 'latin-3': 'iso-8859-3', - 'latin_4': 'iso-8859-4', - 'latin-4': 'iso-8859-4', - 'latin_5': 'iso-8859-9', - 'latin-5': 'iso-8859-9', - 'latin_6': 'iso-8859-10', - 'latin-6': 'iso-8859-10', - 'latin_7': 'iso-8859-13', - 'latin-7': 'iso-8859-13', - 'latin_8': 'iso-8859-14', - 'latin-8': 'iso-8859-14', - 'latin_9': 'iso-8859-15', - 'latin-9': 'iso-8859-15', - 'latin_10':'iso-8859-16', - 'latin-10':'iso-8859-16', - 'cp949': 'ks_c_5601-1987', - 'euc_jp': 'euc-jp', - 'euc_kr': 'euc-kr', - 'ascii': 'us-ascii', - } - - -# Map charsets to their Unicode codec strings. -CODEC_MAP = { - 'gb2312': 'eucgb2312_cn', - 'big5': 'big5_tw', - # Hack: We don't want *any* conversion for stuff marked us-ascii, as all - # sorts of garbage might be sent to us in the guise of 7-bit us-ascii. - # Let that stuff pass through without conversion to/from Unicode. - 'us-ascii': None, - } - - - -# Convenience functions for extending the above mappings -def add_charset(charset, header_enc=None, body_enc=None, output_charset=None): - """Add character set properties to the global registry. - - charset is the input character set, and must be the canonical name of a - character set. - - Optional header_enc and body_enc is either Charset.QP for - quoted-printable, Charset.BASE64 for base64 encoding, Charset.SHORTEST for - the shortest of qp or base64 encoding, or None for no encoding. SHORTEST - is only valid for header_enc. It describes how message headers and - message bodies in the input charset are to be encoded. Default is no - encoding. - - Optional output_charset is the character set that the output should be - in. Conversions will proceed from input charset, to Unicode, to the - output charset when the method Charset.convert() is called. The default - is to output in the same character set as the input. - - Both input_charset and output_charset must have Unicode codec entries in - the module's charset-to-codec mapping; use add_codec(charset, codecname) - to add codecs the module does not know about. See the codecs module's - documentation for more information. - """ - if body_enc == SHORTEST: - raise ValueError('SHORTEST not allowed for body_enc') - CHARSETS[charset] = (header_enc, body_enc, output_charset) - - -def add_alias(alias, canonical): - """Add a character set alias. - - alias is the alias name, e.g. latin-1 - canonical is the character set's canonical name, e.g. iso-8859-1 - """ - ALIASES[alias] = canonical - - -def add_codec(charset, codecname): - """Add a codec that map characters in the given charset to/from Unicode. - - charset is the canonical name of a character set. codecname is the name - of a Python codec, as appropriate for the second argument to the unicode() - built-in, or to the encode() method of a Unicode string. - """ - CODEC_MAP[charset] = codecname - - - -class Charset: - """Map character sets to their email properties. - - This class provides information about the requirements imposed on email - for a specific character set. It also provides convenience routines for - converting between character sets, given the availability of the - applicable codecs. Given a character set, it will do its best to provide - information on how to use that character set in an email in an - RFC-compliant way. - - Certain character sets must be encoded with quoted-printable or base64 - when used in email headers or bodies. Certain character sets must be - converted outright, and are not allowed in email. Instances of this - module expose the following information about a character set: - - input_charset: The initial character set specified. Common aliases - are converted to their `official' email names (e.g. latin_1 - is converted to iso-8859-1). Defaults to 7-bit us-ascii. - - header_encoding: If the character set must be encoded before it can be - used in an email header, this attribute will be set to - Charset.QP (for quoted-printable), Charset.BASE64 (for - base64 encoding), or Charset.SHORTEST for the shortest of - QP or BASE64 encoding. Otherwise, it will be None. - - body_encoding: Same as header_encoding, but describes the encoding for the - mail message's body, which indeed may be different than the - header encoding. Charset.SHORTEST is not allowed for - body_encoding. - - output_charset: Some character sets must be converted before the can be - used in email headers or bodies. If the input_charset is - one of them, this attribute will contain the name of the - charset output will be converted to. Otherwise, it will - be None. - - input_codec: The name of the Python codec used to convert the - input_charset to Unicode. If no conversion codec is - necessary, this attribute will be None. - - output_codec: The name of the Python codec used to convert Unicode - to the output_charset. If no conversion codec is necessary, - this attribute will have the same value as the input_codec. - """ - def __init__(self, input_charset=DEFAULT_CHARSET): - # RFC 2046, $4.1.2 says charsets are not case sensitive. We coerce to - # unicode because its .lower() is locale insensitive. If the argument - # is already a unicode, we leave it at that, but ensure that the - # charset is ASCII, as the standard (RFC XXX) requires. - try: - if isinstance(input_charset, unicode): - input_charset.encode('ascii') - else: - input_charset = unicode(input_charset, 'ascii') - except UnicodeError: - raise errors.CharsetError(input_charset) - input_charset = input_charset.lower().encode('ascii') - # Set the input charset after filtering through the aliases and/or codecs - if not (input_charset in ALIASES or input_charset in CHARSETS): - try: - input_charset = codecs.lookup(input_charset).name - except LookupError: - pass - self.input_charset = ALIASES.get(input_charset, input_charset) - # We can try to guess which encoding and conversion to use by the - # charset_map dictionary. Try that first, but let the user override - # it. - henc, benc, conv = CHARSETS.get(self.input_charset, - (SHORTEST, BASE64, None)) - if not conv: - conv = self.input_charset - # Set the attributes, allowing the arguments to override the default. - self.header_encoding = henc - self.body_encoding = benc - self.output_charset = ALIASES.get(conv, conv) - # Now set the codecs. If one isn't defined for input_charset, - # guess and try a Unicode codec with the same name as input_codec. - self.input_codec = CODEC_MAP.get(self.input_charset, - self.input_charset) - self.output_codec = CODEC_MAP.get(self.output_charset, - self.output_charset) - - def __str__(self): - return self.input_charset.lower() - - __repr__ = __str__ - - def __eq__(self, other): - return str(self) == str(other).lower() - - def __ne__(self, other): - return not self.__eq__(other) - - def get_body_encoding(self): - """Return the content-transfer-encoding used for body encoding. - - This is either the string `quoted-printable' or `base64' depending on - the encoding used, or it is a function in which case you should call - the function with a single argument, the Message object being - encoded. The function should then set the Content-Transfer-Encoding - header itself to whatever is appropriate. - - Returns "quoted-printable" if self.body_encoding is QP. - Returns "base64" if self.body_encoding is BASE64. - Returns "7bit" otherwise. - """ - assert self.body_encoding != SHORTEST - if self.body_encoding == QP: - return 'quoted-printable' - elif self.body_encoding == BASE64: - return 'base64' - else: - return encode_7or8bit - - def convert(self, s): - """Convert a string from the input_codec to the output_codec.""" - if self.input_codec != self.output_codec: - return unicode(s, self.input_codec).encode(self.output_codec) - else: - return s - - def to_splittable(self, s): - """Convert a possibly multibyte string to a safely splittable format. - - Uses the input_codec to try and convert the string to Unicode, so it - can be safely split on character boundaries (even for multibyte - characters). - - Returns the string as-is if it isn't known how to convert it to - Unicode with the input_charset. - - Characters that could not be converted to Unicode will be replaced - with the Unicode replacement character U+FFFD. - """ - if isinstance(s, unicode) or self.input_codec is None: - return s - try: - return unicode(s, self.input_codec, 'replace') - except LookupError: - # Input codec not installed on system, so return the original - # string unchanged. - return s - - def from_splittable(self, ustr, to_output=True): - """Convert a splittable string back into an encoded string. - - Uses the proper codec to try and convert the string from Unicode back - into an encoded format. Return the string as-is if it is not Unicode, - or if it could not be converted from Unicode. - - Characters that could not be converted from Unicode will be replaced - with an appropriate character (usually '?'). - - If to_output is True (the default), uses output_codec to convert to an - encoded format. If to_output is False, uses input_codec. - """ - if to_output: - codec = self.output_codec - else: - codec = self.input_codec - if not isinstance(ustr, unicode) or codec is None: - return ustr - try: - return ustr.encode(codec, 'replace') - except LookupError: - # Output codec not installed - return ustr - - def get_output_charset(self): - """Return the output character set. - - This is self.output_charset if that is not None, otherwise it is - self.input_charset. - """ - return self.output_charset or self.input_charset - - def encoded_header_len(self, s): - """Return the length of the encoded header string.""" - cset = self.get_output_charset() - # The len(s) of a 7bit encoding is len(s) - if self.header_encoding == BASE64: - return email.base64mime.base64_len(s) + len(cset) + MISC_LEN - elif self.header_encoding == QP: - return email.quoprimime.header_quopri_len(s) + len(cset) + MISC_LEN - elif self.header_encoding == SHORTEST: - lenb64 = email.base64mime.base64_len(s) - lenqp = email.quoprimime.header_quopri_len(s) - return min(lenb64, lenqp) + len(cset) + MISC_LEN - else: - return len(s) - - def header_encode(self, s, convert=False): - """Header-encode a string, optionally converting it to output_charset. - - If convert is True, the string will be converted from the input - charset to the output charset automatically. This is not useful for - multibyte character sets, which have line length issues (multibyte - characters must be split on a character, not a byte boundary); use the - high-level Header class to deal with these issues. convert defaults - to False. - - The type of encoding (base64 or quoted-printable) will be based on - self.header_encoding. - """ - cset = self.get_output_charset() - if convert: - s = self.convert(s) - # 7bit/8bit encodings return the string unchanged (modulo conversions) - if self.header_encoding == BASE64: - return email.base64mime.header_encode(s, cset) - elif self.header_encoding == QP: - return email.quoprimime.header_encode(s, cset, maxlinelen=None) - elif self.header_encoding == SHORTEST: - lenb64 = email.base64mime.base64_len(s) - lenqp = email.quoprimime.header_quopri_len(s) - if lenb64 < lenqp: - return email.base64mime.header_encode(s, cset) - else: - return email.quoprimime.header_encode(s, cset, maxlinelen=None) - else: - return s - - def body_encode(self, s, convert=True): - """Body-encode a string and convert it to output_charset. - - If convert is True (the default), the string will be converted from - the input charset to output charset automatically. Unlike - header_encode(), there are no issues with byte boundaries and - multibyte charsets in email bodies, so this is usually pretty safe. - - The type of encoding (base64 or quoted-printable) will be based on - self.body_encoding. - """ - if convert: - s = self.convert(s) - # 7bit/8bit encodings return the string unchanged (module conversions) - if self.body_encoding is BASE64: - return email.base64mime.body_encode(s) - elif self.body_encoding is QP: - return email.quoprimime.body_encode(s) - else: - return s diff --git a/lib-python/modified-2.7/email/encoders.py b/lib-python/modified-2.7/email/encoders.py deleted file mode 100644 --- a/lib-python/modified-2.7/email/encoders.py +++ /dev/null @@ -1,82 +0,0 @@ -# Copyright (C) 2001-2006 Python Software Foundation -# Author: Barry Warsaw -# Contact: email-sig at python.org - -"""Encodings and related functions.""" - -__all__ = [ - 'encode_7or8bit', - 'encode_base64', - 'encode_noop', - 'encode_quopri', - ] - -import base64 - -from quopri import encodestring as _encodestring - - - -def _qencode(s): - enc = _encodestring(s, quotetabs=True) - # Must encode spaces, which quopri.encodestring() doesn't do - return enc.replace(' ', '=20') - - -def _bencode(s): - # We can't quite use base64.encodestring() since it tacks on a "courtesy - # newline". Blech! - if not s: - return s - hasnewline = (s[-1] == '\n') - value = base64.encodestring(s) - if not hasnewline and value[-1] == '\n': - return value[:-1] - return value - - - -def encode_base64(msg): - """Encode the message's payload in Base64. - - Also, add an appropriate Content-Transfer-Encoding header. - """ - orig = msg.get_payload() - encdata = _bencode(orig) - msg.set_payload(encdata) - msg['Content-Transfer-Encoding'] = 'base64' - - - -def encode_quopri(msg): - """Encode the message's payload in quoted-printable. - - Also, add an appropriate Content-Transfer-Encoding header. - """ - orig = msg.get_payload() - encdata = _qencode(orig) - msg.set_payload(encdata) - msg['Content-Transfer-Encoding'] = 'quoted-printable' - - - -def encode_7or8bit(msg): - """Set the Content-Transfer-Encoding header to 7bit or 8bit.""" - orig = msg.get_payload() - if orig is None: - # There's no payload. For backwards compatibility we use 7bit - msg['Content-Transfer-Encoding'] = '7bit' - return - # We play a trick to make this go fast. If encoding to ASCII succeeds, we - # know the data must be 7bit, otherwise treat it as 8bit. - try: - orig.encode('ascii') - except UnicodeError: - msg['Content-Transfer-Encoding'] = '8bit' - else: - msg['Content-Transfer-Encoding'] = '7bit' - - - -def encode_noop(msg): - """Do nothing.""" diff --git a/lib-python/modified-2.7/email/errors.py b/lib-python/modified-2.7/email/errors.py deleted file mode 100644 --- a/lib-python/modified-2.7/email/errors.py +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright (C) 2001-2006 Python Software Foundation -# Author: Barry Warsaw -# Contact: email-sig at python.org - -"""email package exception classes.""" - - - -class MessageError(Exception): - """Base class for errors in the email package.""" - - -class MessageParseError(MessageError): - """Base class for message parsing errors.""" - - -class HeaderParseError(MessageParseError): - """Error while parsing headers.""" - - -class BoundaryError(MessageParseError): - """Couldn't find terminating boundary.""" - - -class MultipartConversionError(MessageError, TypeError): - """Conversion to a multipart is prohibited.""" - - -class CharsetError(MessageError): - """An illegal charset was given.""" - - - -# These are parsing defects which the parser was able to work around. -class MessageDefect: - """Base class for a message defect.""" - - def __init__(self, line=None): - self.line = line - -class NoBoundaryInMultipartDefect(MessageDefect): - """A message claimed to be a multipart but had no boundary parameter.""" - -class StartBoundaryNotFoundDefect(MessageDefect): - """The claimed start boundary was never found.""" - -class FirstHeaderLineIsContinuationDefect(MessageDefect): - """A message had a continuation line as its first header line.""" - -class MisplacedEnvelopeHeaderDefect(MessageDefect): - """A 'Unix-from' header was found in the middle of a header block.""" - -class MalformedHeaderDefect(MessageDefect): - """Found a header that was missing a colon, or was otherwise malformed.""" - -class MultipartInvariantViolationDefect(MessageDefect): - """A message claimed to be a multipart but no subparts were found.""" diff --git a/lib-python/modified-2.7/email/feedparser.py b/lib-python/modified-2.7/email/feedparser.py deleted file mode 100644 --- a/lib-python/modified-2.7/email/feedparser.py +++ /dev/null @@ -1,484 +0,0 @@ -# Copyright (C) 2004-2006 Python Software Foundation -# Authors: Baxter, Wouters and Warsaw -# Contact: email-sig at python.org - -"""FeedParser - An email feed parser. - -The feed parser implements an interface for incrementally parsing an email -message, line by line. This has advantages for certain applications, such as -those reading email messages off a socket. - -FeedParser.feed() is the primary interface for pushing new data into the -parser. It returns when there's nothing more it can do with the available -data. When you have no more data to push into the parser, call .close(). -This completes the parsing and returns the root message object. - -The other advantage of this parser is that it will never throw a parsing -exception. Instead, when it finds something unexpected, it adds a 'defect' to -the current message. Defects are just instances that live on the message -object's .defects attribute. -""" - -__all__ = ['FeedParser'] - -import re - -from email import errors -from email import message - -NLCRE = re.compile('\r\n|\r|\n') -NLCRE_bol = re.compile('(\r\n|\r|\n)') -NLCRE_eol = re.compile('(\r\n|\r|\n)\Z') -NLCRE_crack = re.compile('(\r\n|\r|\n)') -# RFC 2822 $3.6.8 Optional fields. ftext is %d33-57 / %d59-126, Any character -# except controls, SP, and ":". -headerRE = re.compile(r'^(From |[\041-\071\073-\176]{1,}:|[\t ])') -EMPTYSTRING = '' -NL = '\n' - -NeedMoreData = object() - - - -class BufferedSubFile(object): - """A file-ish object that can have new data loaded into it. - - You can also push and pop line-matching predicates onto a stack. When the - current predicate matches the current line, a false EOF response - (i.e. empty string) is returned instead. This lets the parser adhere to a - simple abstraction -- it parses until EOF closes the current message. - """ - def __init__(self): - # The last partial line pushed into this object. - self._partial = '' - # The list of full, pushed lines, in reverse order - self._lines = [] - # The stack of false-EOF checking predicates. - self._eofstack = [] - # A flag indicating whether the file has been closed or not. - self._closed = False - - def push_eof_matcher(self, pred): - self._eofstack.append(pred) - - def pop_eof_matcher(self): - return self._eofstack.pop() - - def close(self): - # Don't forget any trailing partial line. - self._lines.append(self._partial) - self._partial = '' - self._closed = True - - def readline(self): - if not self._lines: - if self._closed: - return '' - return NeedMoreData - # Pop the line off the stack and see if it matches the current - # false-EOF predicate. - line = self._lines.pop() - # RFC 2046, section 5.1.2 requires us to recognize outer level - # boundaries at any level of inner nesting. Do this, but be sure it's - # in the order of most to least nested. - for ateof in self._eofstack[::-1]: - if ateof(line): - # We're at the false EOF. But push the last line back first. - self._lines.append(line) - return '' - return line - - def unreadline(self, line): - # Let the consumer push a line back into the buffer. - assert line is not NeedMoreData - self._lines.append(line) - - def push(self, data): - """Push some new data into this object.""" - # Handle any previous leftovers - data, self._partial = self._partial + data, '' - # Crack into lines, but preserve the newlines on the end of each - parts = NLCRE_crack.split(data) - # The *ahem* interesting behaviour of re.split when supplied grouping - # parentheses is that the last element of the resulting list is the - # data after the final RE. In the case of a NL/CR terminated string, - # this is the empty string. - self._partial = parts.pop() - #GAN 29Mar09 bugs 1555570, 1721862 Confusion at 8K boundary ending with \r: - # is there a \n to follow later? - if not self._partial and parts and parts[-1].endswith('\r'): - self._partial = parts.pop(-2)+parts.pop() - # parts is a list of strings, alternating between the line contents - # and the eol character(s). Gather up a list of lines after - # re-attaching the newlines. - lines = [] - for i in range(len(parts) // 2): - lines.append(parts[i*2] + parts[i*2+1]) - self.pushlines(lines) - - def pushlines(self, lines): - # Reverse and insert at the front of the lines. - self._lines[:0] = lines[::-1] - - def is_closed(self): - return self._closed - - def __iter__(self): - return self - - def next(self): - line = self.readline() - if line == '': - raise StopIteration - return line - - - -class FeedParser: - """A feed-style parser of email.""" - - def __init__(self, _factory=message.Message): - """_factory is called with no arguments to create a new message obj""" - self._factory = _factory - self._input = BufferedSubFile() - self._msgstack = [] - self._parse = self._parsegen().next - self._cur = None - self._last = None - self._headersonly = False - - # Non-public interface for supporting Parser's headersonly flag - def _set_headersonly(self): - self._headersonly = True - - def feed(self, data): - """Push more data into the parser.""" - self._input.push(data) - self._call_parse() - - def _call_parse(self): - try: - self._parse() - except StopIteration: - pass - - def close(self): - """Parse all remaining data and return the root message object.""" - self._input.close() - self._call_parse() - root = self._pop_message() - assert not self._msgstack - # Look for final set of defects - if root.get_content_maintype() == 'multipart' \ - and not root.is_multipart(): - root.defects.append(errors.MultipartInvariantViolationDefect()) - return root - - def _new_message(self): - msg = self._factory() - if self._cur and self._cur.get_content_type() == 'multipart/digest': - msg.set_default_type('message/rfc822') - if self._msgstack: - self._msgstack[-1].attach(msg) - self._msgstack.append(msg) - self._cur = msg - self._last = msg - - def _pop_message(self): - retval = self._msgstack.pop() - if self._msgstack: - self._cur = self._msgstack[-1] - else: - self._cur = None - return retval - - def _parsegen(self): - # Create a new message and start by parsing headers. - self._new_message() - headers = [] - # Collect the headers, searching for a line that doesn't match the RFC - # 2822 header or continuation pattern (including an empty line). - for line in self._input: - if line is NeedMoreData: - yield NeedMoreData - continue - if not headerRE.match(line): - # If we saw the RFC defined header/body separator - # (i.e. newline), just throw it away. Otherwise the line is - # part of the body so push it back. - if not NLCRE.match(line): - self._input.unreadline(line) - break - headers.append(line) - # Done with the headers, so parse them and figure out what we're - # supposed to see in the body of the message. - self._parse_headers(headers) - # Headers-only parsing is a backwards compatibility hack, which was - # necessary in the older parser, which could throw errors. All - # remaining lines in the input are thrown into the message body. - if self._headersonly: - lines = [] - while True: - line = self._input.readline() - if line is NeedMoreData: - yield NeedMoreData - continue - if line == '': - break - lines.append(line) - self._cur.set_payload(EMPTYSTRING.join(lines)) - return - if self._cur.get_content_type() == 'message/delivery-status': - # message/delivery-status contains blocks of headers separated by - # a blank line. We'll represent each header block as a separate - # nested message object, but the processing is a bit different - # than standard message/* types because there is no body for the - # nested messages. A blank line separates the subparts. - while True: - self._input.push_eof_matcher(NLCRE.match) - for retval in self._parsegen(): - if retval is NeedMoreData: - yield NeedMoreData - continue - break - msg = self._pop_message() - # We need to pop the EOF matcher in order to tell if we're at - # the end of the current file, not the end of the last block - # of message headers. - self._input.pop_eof_matcher() - # The input stream must be sitting at the newline or at the - # EOF. We want to see if we're at the end of this subpart, so - # first consume the blank line, then test the next line to see - # if we're at this subpart's EOF. - while True: - line = self._input.readline() - if line is NeedMoreData: - yield NeedMoreData - continue - break - while True: - line = self._input.readline() - if line is NeedMoreData: - yield NeedMoreData - continue - break - if line == '': - break - # Not at EOF so this is a line we're going to need. - self._input.unreadline(line) - return - if self._cur.get_content_maintype() == 'message': - # The message claims to be a message/* type, then what follows is - # another RFC 2822 message. - for retval in self._parsegen(): - if retval is NeedMoreData: - yield NeedMoreData - continue - break - self._pop_message() - return - if self._cur.get_content_maintype() == 'multipart': - boundary = self._cur.get_boundary() - if boundary is None: - # The message /claims/ to be a multipart but it has not - # defined a boundary. That's a problem which we'll handle by - # reading everything until the EOF and marking the message as - # defective. - self._cur.defects.append(errors.NoBoundaryInMultipartDefect()) - lines = [] - for line in self._input: - if line is NeedMoreData: - yield NeedMoreData - continue - lines.append(line) - self._cur.set_payload(EMPTYSTRING.join(lines)) - return - # Create a line match predicate which matches the inter-part - # boundary as well as the end-of-multipart boundary. Don't push - # this onto the input stream until we've scanned past the - # preamble. - separator = '--' + boundary - boundaryre = re.compile( - '(?P<sep>' + re.escape(separator) + - r')(?P<end>--)?(?P<ws>[ \t]*)(?P<linesep>\r\n|\r|\n)?$') - capturing_preamble = True - preamble = [] - linesep = False - while True: - line = self._input.readline() - if line is NeedMoreData: - yield NeedMoreData - continue - if line == '': - break - mo = boundaryre.match(line) - if mo: - # If we're looking at the end boundary, we're done with - # this multipart. If there was a newline at the end of - # the closing boundary, then we need to initialize the - # epilogue with the empty string (see below). - if mo.group('end'): - linesep = mo.group('linesep') - break - # We saw an inter-part boundary. Were we in the preamble? - if capturing_preamble: - if preamble: - # According to RFC 2046, the last newline belongs - # to the boundary. - lastline = preamble[-1] - eolmo = NLCRE_eol.search(lastline) - if eolmo: - preamble[-1] = lastline[:-len(eolmo.group(0))] - self._cur.preamble = EMPTYSTRING.join(preamble) - capturing_preamble = False - self._input.unreadline(line) - continue - # We saw a boundary separating two parts. Consume any - # multiple boundary lines that may be following. Our - # interpretation of RFC 2046 BNF grammar does not produce - # body parts within such double boundaries. - while True: - line = self._input.readline() - if line is NeedMoreData: - yield NeedMoreData - continue - mo = boundaryre.match(line) - if not mo: - self._input.unreadline(line) - break - # Recurse to parse this subpart; the input stream points - # at the subpart's first line. - self._input.push_eof_matcher(boundaryre.match) - for retval in self._parsegen(): - if retval is NeedMoreData: - yield NeedMoreData - continue - break - # Because of RFC 2046, the newline preceding the boundary - # separator actually belongs to the boundary, not the - # previous subpart's payload (or epilogue if the previous - # part is a multipart). - if self._last.get_content_maintype() == 'multipart': - epilogue = self._last.epilogue - if epilogue == '': - self._last.epilogue = None - elif epilogue is not None: - mo = NLCRE_eol.search(epilogue) - if mo: - end = len(mo.group(0)) - self._last.epilogue = epilogue[:-end] - else: - payload = self._last.get_payload() - if isinstance(payload, basestring): - mo = NLCRE_eol.search(payload) - if mo: - payload = payload[:-len(mo.group(0))] - self._last.set_payload(payload) - self._input.pop_eof_matcher() - self._pop_message() - # Set the multipart up for newline cleansing, which will - # happen if we're in a nested multipart. - self._last = self._cur - else: - # I think we must be in the preamble - assert capturing_preamble - preamble.append(line) - # We've seen either the EOF or the end boundary. If we're still - # capturing the preamble, we never saw the start boundary. Note - # that as a defect and store the captured text as the payload. - # Everything from here to the EOF is epilogue. - if capturing_preamble: - self._cur.defects.append(errors.StartBoundaryNotFoundDefect()) - self._cur.set_payload(EMPTYSTRING.join(preamble)) - epilogue = [] - for line in self._input: - if line is NeedMoreData: - yield NeedMoreData - continue - self._cur.epilogue = EMPTYSTRING.join(epilogue) - return - # If the end boundary ended in a newline, we'll need to make sure - # the epilogue isn't None - if linesep: - epilogue = [''] - else: - epilogue = [] - for line in self._input: - if line is NeedMoreData: - yield NeedMoreData - continue - epilogue.append(line) - # Any CRLF at the front of the epilogue is not technically part of - # the epilogue. Also, watch out for an empty string epilogue, - # which means a single newline. - if epilogue: - firstline = epilogue[0] - bolmo = NLCRE_bol.match(firstline) - if bolmo: - epilogue[0] = firstline[len(bolmo.group(0)):] - self._cur.epilogue = EMPTYSTRING.join(epilogue) - return - # Otherwise, it's some non-multipart type, so the entire rest of the - # file contents becomes the payload. - lines = [] - for line in self._input: - if line is NeedMoreData: - yield NeedMoreData - continue - lines.append(line) - self._cur.set_payload(EMPTYSTRING.join(lines)) - - def _parse_headers(self, lines): - # Passed a list of lines that make up the headers for the current msg - lastheader = '' - lastvalue = [] - for lineno, line in enumerate(lines): - # Check for continuation - if line[0] in ' \t': - if not lastheader: - # The first line of the headers was a continuation. This - # is illegal, so let's note the defect, store the illegal - # line, and ignore it for purposes of headers. - defect = errors.FirstHeaderLineIsContinuationDefect(line) - self._cur.defects.append(defect) - continue - lastvalue.append(line) - continue - if lastheader: - # XXX reconsider the joining of folded lines - lhdr = EMPTYSTRING.join(lastvalue)[:-1].rstrip('\r\n') - self._cur[lastheader] = lhdr - lastheader, lastvalue = '', [] - # Check for envelope header, i.e. unix-from - if line.startswith('From '): - if lineno == 0: - # Strip off the trailing newline - mo = NLCRE_eol.search(line) - if mo: - line = line[:-len(mo.group(0))] - self._cur.set_unixfrom(line) - continue - elif lineno == len(lines) - 1: - # Something looking like a unix-from at the end - it's - # probably the first line of the body, so push back the - # line and stop. - self._input.unreadline(line) - return - else: - # Weirdly placed unix-from line. Note this as a defect - # and ignore it. - defect = errors.MisplacedEnvelopeHeaderDefect(line) - self._cur.defects.append(defect) - continue - # Split the line on the colon separating field name from value. - i = line.find(':') - if i < 0: - defect = errors.MalformedHeaderDefect(line) - self._cur.defects.append(defect) - continue - lastheader = line[:i] - lastvalue = [line[i+1:].lstrip()] - # Done with all the lines, so handle the last header. - if lastheader: - # XXX reconsider the joining of folded lines - self._cur[lastheader] = EMPTYSTRING.join(lastvalue).rstrip('\r\n') diff --git a/lib-python/modified-2.7/email/generator.py b/lib-python/modified-2.7/email/generator.py deleted file mode 100644 --- a/lib-python/modified-2.7/email/generator.py +++ /dev/null @@ -1,364 +0,0 @@ -# Copyright (C) 2001-2010 Python Software Foundation -# Contact: email-sig at python.org - -"""Classes to generate plain text from a message object tree.""" - -__all__ = ['Generator', 'DecodedGenerator'] - -import re -import sys -import time -import random -import warnings - -from cStringIO import StringIO -from email.header import Header - -UNDERSCORE = '_' -NL = '\n' - -fcre = re.compile(r'^From ', re.MULTILINE) - -def _is8bitstring(s): - if isinstance(s, str): - try: - unicode(s, 'us-ascii') - except UnicodeError: - return True - return False - - - -class Generator: - """Generates output from a Message object tree. - - This basic generator writes the message to the given file object as plain - text. - """ - # - # Public interface - # - - def __init__(self, outfp, mangle_from_=True, maxheaderlen=78): - """Create the generator for message flattening. - - outfp is the output file-like object for writing the message to. It - must have a write() method. - - Optional mangle_from_ is a flag that, when True (the default), escapes - From_ lines in the body of the message by putting a `>' in front of - them. - - Optional maxheaderlen specifies the longest length for a non-continued - header. When a header line is longer (in characters, with tabs - expanded to 8 spaces) than maxheaderlen, the header will split as - defined in the Header class. Set maxheaderlen to zero to disable - header wrapping. The default is 78, as recommended (but not required) - by RFC 2822. - """ - self._fp = outfp - self._mangle_from_ = mangle_from_ - self._maxheaderlen = maxheaderlen - - def write(self, s): - # Just delegate to the file object - self._fp.write(s) - - def flatten(self, msg, unixfrom=False): - """Print the message object tree rooted at msg to the output file - specified when the Generator instance was created. - - unixfrom is a flag that forces the printing of a Unix From_ delimiter - before the first object in the message tree. If the original message - has no From_ delimiter, a `standard' one is crafted. By default, this - is False to inhibit the printing of any From_ delimiter. - - Note that for subobjects, no From_ line is printed. - """ - if unixfrom: - ufrom = msg.get_unixfrom() - if not ufrom: - ufrom = 'From nobody ' + time.ctime(time.time()) - print >> self._fp, ufrom - self._write(msg) - - def clone(self, fp): - """Clone this generator with the exact same options.""" - return self.__class__(fp, self._mangle_from_, self._maxheaderlen) - - # - # Protected interface - undocumented ;/ - # - - def _write(self, msg): - # We can't write the headers yet because of the following scenario: - # say a multipart message includes the boundary string somewhere in - # its body. We'd have to calculate the new boundary /before/ we write - # the headers so that we can write the correct Content-Type: - # parameter. - # - # The way we do this, so as to make the _handle_*() methods simpler, - # is to cache any subpart writes into a StringIO. The we write the - # headers and the StringIO contents. That way, subpart handlers can - # Do The Right Thing, and can still modify the Content-Type: header if - # necessary. - oldfp = self._fp - try: - self._fp = sfp = StringIO() - self._dispatch(msg) - finally: - self._fp = oldfp - # Write the headers. First we see if the message object wants to - # handle that itself. If not, we'll do it generically. - meth = getattr(msg, '_write_headers', None) - if meth is None: - self._write_headers(msg) - else: - meth(self) - self._fp.write(sfp.getvalue()) - - def _dispatch(self, msg): - # Get the Content-Type: for the message, then try to dispatch to - # self._handle_<maintype>_<subtype>(). If there's no handler for the - # full MIME type, then dispatch to self._handle_<maintype>(). If - # that's missing too, then dispatch to self._writeBody(). - main = msg.get_content_maintype() - sub = msg.get_content_subtype() - specific = UNDERSCORE.join((main, sub)).replace('-', '_') - meth = getattr(self, '_handle_' + specific, None) - if meth is None: - generic = main.replace('-', '_') - meth = getattr(self, '_handle_' + generic, None) - if meth is None: - meth = self._writeBody - meth(msg) - - # - # Default handlers - # - - def _write_headers(self, msg): - for h, v in msg.items(): - print >> self._fp, '%s:' % h, - if self._maxheaderlen == 0: - # Explicit no-wrapping - print >> self._fp, v - elif isinstance(v, Header): - # Header instances know what to do - print >> self._fp, v.encode() - elif _is8bitstring(v): - # If we have raw 8bit data in a byte string, we have no idea - # what the encoding is. There is no safe way to split this - # string. If it's ascii-subset, then we could do a normal - # ascii split, but if it's multibyte then we could break the - # string. There's no way to know so the least harm seems to - # be to not split the string and risk it being too long. - print >> self._fp, v - else: - # Header's got lots of smarts, so use it. Note that this is - # fundamentally broken though because we lose idempotency when - # the header string is continued with tabs. It will now be - # continued with spaces. This was reversedly broken before we - # fixed bug 1974. Either way, we lose. - print >> self._fp, Header( - v, maxlinelen=self._maxheaderlen, header_name=h).encode() - # A blank line always separates headers from body - print >> self._fp - - # - # Handlers for writing types and subtypes - # - - def _handle_text(self, msg): - payload = msg.get_payload() - if payload is None: - return - if not isinstance(payload, basestring): - raise TypeError('string payload expected: %s' % type(payload)) - if self._mangle_from_: - payload = fcre.sub('>From ', payload) - self._fp.write(payload) - - # Default body handler - _writeBody = _handle_text - - def _handle_multipart(self, msg): - # The trick here is to write out each part separately, merge them all - # together, and then make sure that the boundary we've chosen isn't - # present in the payload. - msgtexts = [] - subparts = msg.get_payload() - if subparts is None: - subparts = [] - elif isinstance(subparts, basestring): - # e.g. a non-strict parse of a message with no starting boundary. - self._fp.write(subparts) - return - elif not isinstance(subparts, list): - # Scalar payload - subparts = [subparts] - for part in subparts: - s = StringIO() - g = self.clone(s) - g.flatten(part, unixfrom=False) - msgtexts.append(s.getvalue()) - # BAW: What about boundaries that are wrapped in double-quotes? - boundary = msg.get_boundary() - if not boundary: - # Create a boundary that doesn't appear in any of the - # message texts. - alltext = NL.join(msgtexts) - boundary = _make_boundary(alltext) - msg.set_boundary(boundary) - # If there's a preamble, write it out, with a trailing CRLF - if msg.preamble is not None: - print >> self._fp, msg.preamble - # dash-boundary transport-padding CRLF - print >> self._fp, '--' + boundary - # body-part - if msgtexts: - self._fp.write(msgtexts.pop(0)) - # *encapsulation - # --> delimiter transport-padding - # --> CRLF body-part - for body_part in msgtexts: - # delimiter transport-padding CRLF - print >> self._fp, '\n--' + boundary - # body-part - self._fp.write(body_part) - # close-delimiter transport-padding - self._fp.write('\n--' + boundary + '--') - if msg.epilogue is not None: - print >> self._fp - self._fp.write(msg.epilogue) - - def _handle_multipart_signed(self, msg): - # The contents of signed parts has to stay unmodified in order to keep - # the signature intact per RFC1847 2.1, so we disable header wrapping. - # RDM: This isn't enough to completely preserve the part, but it helps. - old_maxheaderlen = self._maxheaderlen - try: - self._maxheaderlen = 0 - self._handle_multipart(msg) - finally: - self._maxheaderlen = old_maxheaderlen - - def _handle_message_delivery_status(self, msg): - # We can't just write the headers directly to self's file object - # because this will leave an extra newline between the last header - # block and the boundary. Sigh. - blocks = [] - for part in msg.get_payload(): - s = StringIO() - g = self.clone(s) - g.flatten(part, unixfrom=False) - text = s.getvalue() - lines = text.split('\n') - # Strip off the unnecessary trailing empty line - if lines and lines[-1] == '': - blocks.append(NL.join(lines[:-1])) - else: - blocks.append(text) - # Now join all the blocks with an empty line. This has the lovely - # effect of separating each block with an empty line, but not adding - # an extra one after the last one. - self._fp.write(NL.join(blocks)) - - def _handle_message(self, msg): - s = StringIO() - g = self.clone(s) - # The payload of a message/rfc822 part should be a multipart sequence - # of length 1. The zeroth element of the list should be the Message - # object for the subpart. Extract that object, stringify it, and - # write it out. - # Except, it turns out, when it's a string instead, which happens when - # and only when HeaderParser is used on a message of mime type - # message/rfc822. Such messages are generated by, for example, - # Groupwise when forwarding unadorned messages. (Issue 7970.) So - # in that case we just emit the string body. - payload = msg.get_payload() - if isinstance(payload, list): - g.flatten(msg.get_payload(0), unixfrom=False) - payload = s.getvalue() - self._fp.write(payload) - - - -_FMT = '[Non-text (%(type)s) part of message omitted, filename %(filename)s]' - -class DecodedGenerator(Generator): - """Generates a text representation of a message. - - Like the Generator base class, except that non-text parts are substituted - with a format string representing the part. - """ - def __init__(self, outfp, mangle_from_=True, maxheaderlen=78, fmt=None): - """Like Generator.__init__() except that an additional optional - argument is allowed. - - Walks through all subparts of a message. If the subpart is of main - type `text', then it prints the decoded payload of the subpart. - - Otherwise, fmt is a format string that is used instead of the message - payload. fmt is expanded with the following keywords (in - %(keyword)s format): - - type : Full MIME type of the non-text part - maintype : Main MIME type of the non-text part - subtype : Sub-MIME type of the non-text part - filename : Filename of the non-text part - description: Description associated with the non-text part - encoding : Content transfer encoding of the non-text part - - The default value for fmt is None, meaning - - [Non-text (%(type)s) part of message omitted, filename %(filename)s] - """ - Generator.__init__(self, outfp, mangle_from_, maxheaderlen) - if fmt is None: - self._fmt = _FMT - else: - self._fmt = fmt - - def _dispatch(self, msg): - for part in msg.walk(): - maintype = part.get_content_maintype() - if maintype == 'text': - print >> self, part.get_payload(decode=True) - elif maintype == 'multipart': - # Just skip this - pass - else: - print >> self, self._fmt % { - 'type' : part.get_content_type(), - 'maintype' : part.get_content_maintype(), - 'subtype' : part.get_content_subtype(), - 'filename' : part.get_filename('[no filename]'), - 'description': part.get('Content-Description', - '[no description]'), - 'encoding' : part.get('Content-Transfer-Encoding', - '[no encoding]'), - } - - - -# Helper -_width = len(repr(sys.maxint-1)) -_fmt = '%%0%dd' % _width - -def _make_boundary(text=None): - # Craft a random boundary. If text is given, ensure that the chosen - # boundary doesn't appear in the text. - token = random.randrange(sys.maxint) - boundary = ('=' * 15) + (_fmt % token) + '==' - if text is None: - return boundary - b = boundary - counter = 0 - while True: - cre = re.compile('^--' + re.escape(b) + '(--)?$', re.MULTILINE) - if not cre.search(text): - break - b = boundary + '.' + str(counter) - counter += 1 - return b diff --git a/lib-python/modified-2.7/email/header.py b/lib-python/modified-2.7/email/header.py deleted file mode 100644 --- a/lib-python/modified-2.7/email/header.py +++ /dev/null @@ -1,514 +0,0 @@ -# Copyright (C) 2002-2006 Python Software Foundation -# Author: Ben Gertzfield, Barry Warsaw -# Contact: email-sig at python.org - -"""Header encoding and decoding functionality.""" - -__all__ = [ - 'Header', - 'decode_header', - 'make_header', - ] - -import re -import binascii - -import email.quoprimime -import email.base64mime - -from email.errors import HeaderParseError -from email.charset import Charset - -NL = '\n' -SPACE = ' ' -USPACE = u' ' -SPACE8 = ' ' * 8 -UEMPTYSTRING = u'' - -MAXLINELEN = 76 - -USASCII = Charset('us-ascii') -UTF8 = Charset('utf-8') - -# Match encoded-word strings in the form =?charset?q?Hello_World?= -ecre = re.compile(r''' - =\? # literal =? - (?P<charset>[^?]*?) # non-greedy up to the next ? is the charset - \? # literal ? - (?P<encoding>[qb]) # either a "q" or a "b", case insensitive - \? # literal ? - (?P<encoded>.*?) # non-greedy up to the next ?= is the encoded string - \?= # literal ?= - (?=[ \t]|$) # whitespace or the end of the string - ''', re.VERBOSE | re.IGNORECASE | re.MULTILINE) - -# Field name regexp, including trailing colon, but not separating whitespace, -# according to RFC 2822. Character range is from tilde to exclamation mark. -# For use with .match() -fcre = re.compile(r'[\041-\176]+:$') - -# Find a header embedded in a putative header value. Used to check for -# header injection attack. -_embeded_header = re.compile(r'\n[^ \t]+:') - - - -# Helpers -_max_append = email.quoprimime._max_append - - - -def decode_header(header): - """Decode a message header value without converting charset. - - Returns a list of (decoded_string, charset) pairs containing each of the - decoded parts of the header. Charset is None for non-encoded parts of the - header, otherwise a lower-case string containing the name of the character - set specified in the encoded string. - - An email.errors.HeaderParseError may be raised when certain decoding error - occurs (e.g. a base64 decoding exception). - """ - # If no encoding, just return the header - header = str(header) - if not ecre.search(header): - return [(header, None)] - decoded = [] - dec = '' - for line in header.splitlines(): - # This line might not have an encoding in it - if not ecre.search(line): - decoded.append((line, None)) - continue - parts = ecre.split(line) - while parts: - unenc = parts.pop(0).strip() - if unenc: - # Should we continue a long line? - if decoded and decoded[-1][1] is None: - decoded[-1] = (decoded[-1][0] + SPACE + unenc, None) - else: - decoded.append((unenc, None)) - if parts: - charset, encoding = [s.lower() for s in parts[0:2]] - encoded = parts[2] - dec = None - if encoding == 'q': - dec = email.quoprimime.header_decode(encoded) - elif encoding == 'b': - paderr = len(encoded) % 4 # Postel's law: add missing padding - if paderr: - encoded += '==='[:4 - paderr] - try: - dec = email.base64mime.decode(encoded) - except binascii.Error: - # Turn this into a higher level exception. BAW: Right - # now we throw the lower level exception away but - # when/if we get exception chaining, we'll preserve it. - raise HeaderParseError - if dec is None: - dec = encoded - - if decoded and decoded[-1][1] == charset: - decoded[-1] = (decoded[-1][0] + dec, decoded[-1][1]) - else: - decoded.append((dec, charset)) - del parts[0:3] - return decoded - - - -def make_header(decoded_seq, maxlinelen=None, header_name=None, - continuation_ws=' '): - """Create a Header from a sequence of pairs as returned by decode_header() - - decode_header() takes a header value string and returns a sequence of - pairs of the format (decoded_string, charset) where charset is the string - name of the character set. - - This function takes one of those sequence of pairs and returns a Header - instance. Optional maxlinelen, header_name, and continuation_ws are as in - the Header constructor. - """ - h = Header(maxlinelen=maxlinelen, header_name=header_name, - continuation_ws=continuation_ws) - for s, charset in decoded_seq: - # None means us-ascii but we can simply pass it on to h.append() - if charset is not None and not isinstance(charset, Charset): - charset = Charset(charset) - h.append(s, charset) - return h - - - -class Header: - def __init__(self, s=None, charset=None, - maxlinelen=None, header_name=None, - continuation_ws=' ', errors='strict'): - """Create a MIME-compliant header that can contain many character sets. - - Optional s is the initial header value. If None, the initial header - value is not set. You can later append to the header with .append() - method calls. s may be a byte string or a Unicode string, but see the - .append() documentation for semantics. - - Optional charset serves two purposes: it has the same meaning as the - charset argument to the .append() method. It also sets the default - character set for all subsequent .append() calls that omit the charset - argument. If charset is not provided in the constructor, the us-ascii - charset is used both as s's initial charset and as the default for - subsequent .append() calls. - - The maximum line length can be specified explicit via maxlinelen. For - splitting the first line to a shorter value (to account for the field - header which isn't included in s, e.g. `Subject') pass in the name of - the field in header_name. The default maxlinelen is 76. - - continuation_ws must be RFC 2822 compliant folding whitespace (usually - either a space or a hard tab) which will be prepended to continuation - lines. - - errors is passed through to the .append() call. - """ - if charset is None: - charset = USASCII - if not isinstance(charset, Charset): - charset = Charset(charset) - self._charset = charset - self._continuation_ws = continuation_ws - cws_expanded_len = len(continuation_ws.replace('\t', SPACE8)) - # BAW: I believe `chunks' and `maxlinelen' should be non-public. - self._chunks = [] - if s is not None: - self.append(s, charset, errors) - if maxlinelen is None: - maxlinelen = MAXLINELEN - if header_name is None: - # We don't know anything about the field header so the first line - # is the same length as subsequent lines. - self._firstlinelen = maxlinelen - else: - # The first line should be shorter to take into account the field - # header. Also subtract off 2 extra for the colon and space. - self._firstlinelen = maxlinelen - len(header_name) - 2 - # Second and subsequent lines should subtract off the length in - # columns of the continuation whitespace prefix. - self._maxlinelen = maxlinelen - cws_expanded_len - - def __str__(self): - """A synonym for self.encode().""" - return self.encode() - - def __unicode__(self): - """Helper for the built-in unicode function.""" - uchunks = [] - lastcs = None - for s, charset in self._chunks: - # We must preserve spaces between encoded and non-encoded word - # boundaries, which means for us we need to add a space when we go - # from a charset to None/us-ascii, or from None/us-ascii to a - # charset. Only do this for the second and subsequent chunks. - nextcs = charset - if uchunks: - if lastcs not in (None, 'us-ascii'): - if nextcs in (None, 'us-ascii'): - uchunks.append(USPACE) - nextcs = None - elif nextcs not in (None, 'us-ascii'): - uchunks.append(USPACE) - lastcs = nextcs - uchunks.append(unicode(s, str(charset))) - return UEMPTYSTRING.join(uchunks) - - # Rich comparison operators for equality only. BAW: does it make sense to - # have or explicitly disable <, <=, >, >= operators? - def __eq__(self, other): - # other may be a Header or a string. Both are fine so coerce - # ourselves to a string, swap the args and do another comparison. - return other == self.encode() - - def __ne__(self, other): - return not self == other - - def append(self, s, charset=None, errors='strict'): - """Append a string to the MIME header. - - Optional charset, if given, should be a Charset instance or the name - of a character set (which will be converted to a Charset instance). A - value of None (the default) means that the charset given in the - constructor is used. - - s may be a byte string or a Unicode string. If it is a byte string - (i.e. isinstance(s, str) is true), then charset is the encoding of - that byte string, and a UnicodeError will be raised if the string - cannot be decoded with that charset. If s is a Unicode string, then - charset is a hint specifying the character set of the characters in - the string. In this case, when producing an RFC 2822 compliant header - using RFC 2047 rules, the Unicode string will be encoded using the - following charsets in order: us-ascii, the charset hint, utf-8. The - first character set not to provoke a UnicodeError is used. - - Optional `errors' is passed as the third argument to any unicode() or - ustr.encode() call. - """ - if charset is None: - charset = self._charset - elif not isinstance(charset, Charset): - charset = Charset(charset) - # If the charset is our faux 8bit charset, leave the string unchanged - if charset != '8bit': - # We need to test that the string can be converted to unicode and - # back to a byte string, given the input and output codecs of the - # charset. - if isinstance(s, str): - # Possibly raise UnicodeError if the byte string can't be - # converted to a unicode with the input codec of the charset. - incodec = charset.input_codec or 'us-ascii' - ustr = unicode(s, incodec, errors) - # Now make sure that the unicode could be converted back to a - # byte string with the output codec, which may be different - # than the iput coded. Still, use the original byte string. - outcodec = charset.output_codec or 'us-ascii' - ustr.encode(outcodec, errors) - elif isinstance(s, unicode): - # Now we have to be sure the unicode string can be converted - # to a byte string with a reasonable output codec. We want to - # use the byte string in the chunk. - for charset in USASCII, charset, UTF8: - try: - outcodec = charset.output_codec or 'us-ascii' - s = s.encode(outcodec, errors) - break - except UnicodeError: - pass - else: - assert False, 'utf-8 conversion failed' - self._chunks.append((s, charset)) - - def _split(self, s, charset, maxlinelen, splitchars): - # Split up a header safely for use with encode_chunks. - splittable = charset.to_splittable(s) - encoded = charset.from_splittable(splittable, True) - elen = charset.encoded_header_len(encoded) - # If the line's encoded length first, just return it - if elen <= maxlinelen: - return [(encoded, charset)] - # If we have undetermined raw 8bit characters sitting in a byte - # string, we really don't know what the right thing to do is. We - # can't really split it because it might be multibyte data which we - # could break if we split it between pairs. The least harm seems to - # be to not split the header at all, but that means they could go out - # longer than maxlinelen. - if charset == '8bit': - return [(s, charset)] - # BAW: I'm not sure what the right test here is. What we're trying to - # do is be faithful to RFC 2822's recommendation that ($2.2.3): - # - # "Note: Though structured field bodies are defined in such a way that - # folding can take place between many of the lexical tokens (and even - # within some of the lexical tokens), folding SHOULD be limited to - # placing the CRLF at higher-level syntactic breaks." - # - # For now, I can only imagine doing this when the charset is us-ascii, - # although it's possible that other charsets may also benefit from the - # higher-level syntactic breaks. - elif charset == 'us-ascii': - return self._split_ascii(s, charset, maxlinelen, splitchars) - # BAW: should we use encoded? - elif elen == len(s): - # We can split on _maxlinelen boundaries because we know that the - # encoding won't change the size of the string - splitpnt = maxlinelen - first = charset.from_splittable(splittable[:splitpnt], False) - last = charset.from_splittable(splittable[splitpnt:], False) - else: - # Binary search for split point - first, last = _binsplit(splittable, charset, maxlinelen) - # first is of the proper length so just wrap it in the appropriate - # chrome. last must be recursively split. - fsplittable = charset.to_splittable(first) - fencoded = charset.from_splittable(fsplittable, True) - chunk = [(fencoded, charset)] - return chunk + self._split(last, charset, self._maxlinelen, splitchars) - - def _split_ascii(self, s, charset, firstlen, splitchars): - chunks = _split_ascii(s, firstlen, self._maxlinelen, - self._continuation_ws, splitchars) - return zip(chunks, [charset]*len(chunks)) - - def _encode_chunks(self, newchunks, maxlinelen): - # MIME-encode a header with many different charsets and/or encodings. - # - # Given a list of pairs (string, charset), return a MIME-encoded - # string suitable for use in a header field. Each pair may have - # different charsets and/or encodings, and the resulting header will - # accurately reflect each setting. - # - # Each encoding can be email.utils.QP (quoted-printable, for - # ASCII-like character sets like iso-8859-1), email.utils.BASE64 - # (Base64, for non-ASCII like character sets like KOI8-R and - # iso-2022-jp), or None (no encoding). - # - # Each pair will be represented on a separate line; the resulting - # string will be in the format: - # - # =?charset1?q?Mar=EDa_Gonz=E1lez_Alonso?=\n - # =?charset2?b?SvxyZ2VuIEL2aW5n?=" - chunks = [] - for header, charset in newchunks: - if not header: - continue - if charset is None or charset.header_encoding is None: - s = header - else: - s = charset.header_encode(header) - # Don't add more folding whitespace than necessary - if chunks and chunks[-1].endswith(' '): - extra = '' - else: - extra = ' ' - _max_append(chunks, s, maxlinelen, extra) - joiner = NL + self._continuation_ws - return joiner.join(chunks) - - def encode(self, splitchars=';, '): - """Encode a message header into an RFC-compliant format. - - There are many issues involved in converting a given string for use in - an email header. Only certain character sets are readable in most - email clients, and as header strings can only contain a subset of - 7-bit ASCII, care must be taken to properly convert and encode (with - Base64 or quoted-printable) header strings. In addition, there is a - 75-character length limit on any given encoded header field, so - line-wrapping must be performed, even with double-byte character sets. - - This method will do its best to convert the string to the correct - character set used in email, and encode and line wrap it safely with - the appropriate scheme for that character set. - - If the given charset is not known or an error occurs during - conversion, this function will return the header untouched. - - Optional splitchars is a string containing characters to split long - ASCII lines on, in rough support of RFC 2822's `highest level - syntactic breaks'. This doesn't affect RFC 2047 encoded lines. - """ - newchunks = [] - maxlinelen = self._firstlinelen - lastlen = 0 - for s, charset in self._chunks: - # The first bit of the next chunk should be just long enough to - # fill the next line. Don't forget the space separating the - # encoded words. - targetlen = maxlinelen - lastlen - 1 - if targetlen < charset.encoded_header_len(''): - # Stick it on the next line - targetlen = maxlinelen - newchunks += self._split(s, charset, targetlen, splitchars) - lastchunk, lastcharset = newchunks[-1] - lastlen = lastcharset.encoded_header_len(lastchunk) - value = self._encode_chunks(newchunks, maxlinelen) - if _embeded_header.search(value): - raise HeaderParseError("header value appears to contain " - "an embedded header: {!r}".format(value)) - return value - - - -def _split_ascii(s, firstlen, restlen, continuation_ws, splitchars): - lines = [] - maxlen = firstlen - for line in s.splitlines(): - # Ignore any leading whitespace (i.e. continuation whitespace) already - # on the line, since we'll be adding our own. - line = line.lstrip() - if len(line) < maxlen: - lines.append(line) - maxlen = restlen - continue - # Attempt to split the line at the highest-level syntactic break - # possible. Note that we don't have a lot of smarts about field - # syntax; we just try to break on semi-colons, then commas, then - # whitespace. - for ch in splitchars: - if ch in line: - break - else: - # There's nothing useful to split the line on, not even spaces, so - # just append this line unchanged - lines.append(line) - maxlen = restlen - continue - # Now split the line on the character plus trailing whitespace - cre = re.compile(r'%s\s*' % ch) - if ch in ';,': - eol = ch - else: - eol = '' - joiner = eol + ' ' - joinlen = len(joiner) - wslen = len(continuation_ws.replace('\t', SPACE8)) - this = [] - linelen = 0 - for part in cre.split(line): - curlen = linelen + max(0, len(this)-1) * joinlen - partlen = len(part) - onfirstline = not lines - # We don't want to split after the field name, if we're on the - # first line and the field name is present in the header string. - if ch == ' ' and onfirstline and \ - len(this) == 1 and fcre.match(this[0]): - this.append(part) - linelen += partlen - elif curlen + partlen > maxlen: - if this: - lines.append(joiner.join(this) + eol) - # If this part is longer than maxlen and we aren't already - # splitting on whitespace, try to recursively split this line - # on whitespace. - if partlen > maxlen and ch != ' ': - subl = _split_ascii(part, maxlen, restlen, - continuation_ws, ' ') - lines.extend(subl[:-1]) - this = [subl[-1]] - else: - this = [part] - linelen = wslen + len(this[-1]) - maxlen = restlen - else: - this.append(part) - linelen += partlen - # Put any left over parts on a line by themselves - if this: - lines.append(joiner.join(this)) - return lines - - - -def _binsplit(splittable, charset, maxlinelen): - i = 0 - j = len(splittable) - while i < j: - # Invariants: - # 1. splittable[:k] fits for all k <= i (note that we *assume*, - # at the start, that splittable[:0] fits). - # 2. splittable[:k] does not fit for any k > j (at the start, - # this means we shouldn't look at any k > len(splittable)). - # 3. We don't know about splittable[:k] for k in i+1..j. - # 4. We want to set i to the largest k that fits, with i <= k <= j. - # - m = (i+j+1) >> 1 # ceiling((i+j)/2); i < m <= j - chunk = charset.from_splittable(splittable[:m], True) - chunklen = charset.encoded_header_len(chunk) - if chunklen <= maxlinelen: - # m is acceptable, so is a new lower bound. - i = m - else: - # m is not acceptable, so final i must be < m. - j = m - 1 - # i == j. Invariant #1 implies that splittable[:i] fits, and - # invariant #2 implies that splittable[:i+1] does not fit, so i - # is what we're looking for. - first = charset.from_splittable(splittable[:i], False) - last = charset.from_splittable(splittable[i:], False) - return first, last diff --git a/lib-python/modified-2.7/email/iterators.py b/lib-python/modified-2.7/email/iterators.py deleted file mode 100644 --- a/lib-python/modified-2.7/email/iterators.py +++ /dev/null @@ -1,73 +0,0 @@ -# Copyright (C) 2001-2006 Python Software Foundation -# Author: Barry Warsaw -# Contact: email-sig at python.org - -"""Various types of useful iterators and generators.""" - -__all__ = [ - 'body_line_iterator', - 'typed_subpart_iterator', - 'walk', - # Do not include _structure() since it's part of the debugging API. - ] - -import sys -from cStringIO import StringIO - - - -# This function will become a method of the Message class -def walk(self): - """Walk over the message tree, yielding each subpart. - - The walk is performed in depth-first order. This method is a - generator. - """ - yield self - if self.is_multipart(): - for subpart in self.get_payload(): - for subsubpart in subpart.walk(): - yield subsubpart - - - -# These two functions are imported into the Iterators.py interface module. -def body_line_iterator(msg, decode=False): - """Iterate over the parts, returning string payloads line-by-line. - - Optional decode (default False) is passed through to .get_payload(). - """ - for subpart in msg.walk(): - payload = subpart.get_payload(decode=decode) - if isinstance(payload, basestring): - for line in StringIO(payload): - yield line - - -def typed_subpart_iterator(msg, maintype='text', subtype=None): - """Iterate over the subparts with a given MIME type. - - Use `maintype' as the main MIME type to match against; this defaults to - "text". Optional `subtype' is the MIME subtype to match against; if - omitted, only the main type is matched. - """ - for subpart in msg.walk(): - if subpart.get_content_maintype() == maintype: - if subtype is None or subpart.get_content_subtype() == subtype: - yield subpart - - - -def _structure(msg, fp=None, level=0, include_default=False): - """A handy debugging aid""" - if fp is None: - fp = sys.stdout - tab = ' ' * (level * 4) - print >> fp, tab + msg.get_content_type(), - if include_default: - print >> fp, '[%s]' % msg.get_default_type() - else: - print >> fp - if msg.is_multipart(): - for subpart in msg.get_payload(): - _structure(subpart, fp, level+1, include_default) diff --git a/lib-python/modified-2.7/email/message.py b/lib-python/modified-2.7/email/message.py deleted file mode 100644 --- a/lib-python/modified-2.7/email/message.py +++ /dev/null @@ -1,797 +0,0 @@ -# Copyright (C) 2001-2006 Python Software Foundation -# Author: Barry Warsaw -# Contact: email-sig at python.org - -"""Basic message object for the email package object model.""" - -__all__ = ['Message'] - -import re -import uu -import binascii -import warnings -from cStringIO import StringIO - -# Intrapackage imports -import email.charset -from email import utils -from email import errors - -SEMISPACE = '; ' - -# Regular expression that matches `special' characters in parameters, the -# existence of which force quoting of the parameter value. -tspecials = re.compile(r'[ <>@,;:\\"/\[\]\?=]') - - -# Helper functions -def _splitparam(param): - # Split header parameters. BAW: this may be too simple. It isn't - # strictly RFC 2045 (section 5.1) compliant, but it catches most headers - # found in the wild. We may eventually need a full fledged parser - # eventually. - a, sep, b = param.partition(';') - if not sep: - return a.strip(), None - return a.strip(), b.strip() - -def _formatparam(param, value=None, quote=True): - """Convenience function to format and return a key=value pair. - - This will quote the value if needed or if quote is true. If value is a - three tuple (charset, language, value), it will be encoded according - to RFC2231 rules. - """ - if value is not None and len(value) > 0: - # A tuple is used for RFC 2231 encoded parameter values where items - # are (charset, language, value). charset is a string, not a Charset - # instance. - if isinstance(value, tuple): - # Encode as per RFC 2231 - param += '*' - value = utils.encode_rfc2231(value[2], value[0], value[1]) - # BAW: Please check this. I think that if quote is set it should - # force quoting even if not necessary. - if quote or tspecials.search(value): - return '%s="%s"' % (param, utils.quote(value)) - else: - return '%s=%s' % (param, value) - else: - return param - -def _parseparam(s): - plist = [] - while s[:1] == ';': - s = s[1:] - end = s.find(';') - while end > 0 and (s.count('"', 0, end) - s.count('\\"', 0, end)) % 2: - end = s.find(';', end + 1) - if end < 0: - end = len(s) - f = s[:end] - if '=' in f: - i = f.index('=') - f = f[:i].strip().lower() + '=' + f[i+1:].strip() - plist.append(f.strip()) - s = s[end:] - return plist - - -def _unquotevalue(value): - # This is different than utils.collapse_rfc2231_value() because it doesn't - # try to convert the value to a unicode. Message.get_param() and - # Message.get_params() are both currently defined to return the tuple in - # the face of RFC 2231 parameters. - if isinstance(value, tuple): - return value[0], value[1], utils.unquote(value[2]) - else: - return utils.unquote(value) - - - -class Message: - """Basic message object. - - A message object is defined as something that has a bunch of RFC 2822 - headers and a payload. It may optionally have an envelope header - (a.k.a. Unix-From or From_ header). If the message is a container (i.e. a - multipart or a message/rfc822), then the payload is a list of Message - objects, otherwise it is a string. - - Message objects implement part of the `mapping' interface, which assumes - there is exactly one occurrence of the header per message. Some headers - do in fact appear multiple times (e.g. Received) and for those headers, - you must use the explicit API to set or get all the headers. Not all of - the mapping methods are implemented. - """ - def __init__(self): - self._headers = [] - self._unixfrom = None - self._payload = None - self._charset = None - # Defaults for multipart messages - self.preamble = self.epilogue = None - self.defects = [] - # Default content type - self._default_type = 'text/plain' - - def __str__(self): - """Return the entire formatted message as a string. - This includes the headers, body, and envelope header. - """ - return self.as_string(unixfrom=True) - - def as_string(self, unixfrom=False): - """Return the entire formatted message as a string. - Optional `unixfrom' when True, means include the Unix From_ envelope - header. - - This is a convenience method and may not generate the message exactly - as you intend because by default it mangles lines that begin with - "From ". For more flexibility, use the flatten() method of a - Generator instance. - """ - from email.generator import Generator - fp = StringIO() - g = Generator(fp) - g.flatten(self, unixfrom=unixfrom) - return fp.getvalue() - - def is_multipart(self): - """Return True if the message consists of multiple parts.""" - return isinstance(self._payload, list) - - # - # Unix From_ line - # - def set_unixfrom(self, unixfrom): - self._unixfrom = unixfrom - - def get_unixfrom(self): - return self._unixfrom - - # - # Payload manipulation. - # - def attach(self, payload): - """Add the given payload to the current payload. - - The current payload will always be a list of objects after this method - is called. If you want to set the payload to a scalar object, use - set_payload() instead. - """ - if self._payload is None: - self._payload = [payload] - else: - self._payload.append(payload) - - def get_payload(self, i=None, decode=False): - """Return a reference to the payload. - - The payload will either be a list object or a string. If you mutate - the list object, you modify the message's payload in place. Optional - i returns that index into the payload. - - Optional decode is a flag indicating whether the payload should be - decoded or not, according to the Content-Transfer-Encoding header - (default is False). - - When True and the message is not a multipart, the payload will be - decoded if this header's value is `quoted-printable' or `base64'. If - some other encoding is used, or the header is missing, or if the - payload has bogus data (i.e. bogus base64 or uuencoded data), the - payload is returned as-is. - - If the message is a multipart and the decode flag is True, then None - is returned. - """ - if i is None: - payload = self._payload - elif not isinstance(self._payload, list): - raise TypeError('Expected list, got %s' % type(self._payload)) - else: - payload = self._payload[i] - if decode: - if self.is_multipart(): - return None - cte = self.get('content-transfer-encoding', '').lower() - if cte == 'quoted-printable': - return utils._qdecode(payload) - elif cte == 'base64': - try: - return utils._bdecode(payload) - except binascii.Error: - # Incorrect padding - return payload - elif cte in ('x-uuencode', 'uuencode', 'uue', 'x-uue'): - sfp = StringIO() - try: - uu.decode(StringIO(payload+'\n'), sfp, quiet=True) - payload = sfp.getvalue() - except uu.Error: - # Some decoding problem - return payload - # Everything else, including encodings with 8bit or 7bit are returned - # unchanged. - return payload - - def set_payload(self, payload, charset=None): - """Set the payload to the given value. - - Optional charset sets the message's default character set. See - set_charset() for details. - """ - self._payload = payload - if charset is not None: - self.set_charset(charset) - - def set_charset(self, charset): - """Set the charset of the payload to a given character set. - - charset can be a Charset instance, a string naming a character set, or - None. If it is a string it will be converted to a Charset instance. - If charset is None, the charset parameter will be removed from the - Content-Type field. Anything else will generate a TypeError. - - The message will be assumed to be of type text/* encoded with - charset.input_charset. It will be converted to charset.output_charset - and encoded properly, if needed, when generating the plain text - representation of the message. MIME headers (MIME-Version, - Content-Type, Content-Transfer-Encoding) will be added as needed. - - """ - if charset is None: - self.del_param('charset') - self._charset = None - return - if isinstance(charset, basestring): - charset = email.charset.Charset(charset) - if not isinstance(charset, email.charset.Charset): - raise TypeError(charset) - # BAW: should we accept strings that can serve as arguments to the - # Charset constructor? - self._charset = charset - if 'MIME-Version' not in self: - self.add_header('MIME-Version', '1.0') - if 'Content-Type' not in self: - self.add_header('Content-Type', 'text/plain', - charset=charset.get_output_charset()) - else: - self.set_param('charset', charset.get_output_charset()) - if isinstance(self._payload, unicode): - self._payload = self._payload.encode(charset.output_charset) - if str(charset) != charset.get_output_charset(): - self._payload = charset.body_encode(self._payload) - if 'Content-Transfer-Encoding' not in self: - cte = charset.get_body_encoding() - try: - cte(self) - except TypeError: - self._payload = charset.body_encode(self._payload) - self.add_header('Content-Transfer-Encoding', cte) - - def get_charset(self): - """Return the Charset instance associated with the message's payload. - """ - return self._charset - - # - # MAPPING INTERFACE (partial) - # - def __len__(self): - """Return the total number of headers, including duplicates.""" - return len(self._headers) - - def __getitem__(self, name): - """Get a header value. - - Return None if the header is missing instead of raising an exception. - - Note that if the header appeared multiple times, exactly which - occurrence gets returned is undefined. Use get_all() to get all - the values matching a header field name. - """ - return self.get(name) - - def __setitem__(self, name, val): - """Set the value of a header. - - Note: this does not overwrite an existing header with the same field - name. Use __delitem__() first to delete any existing headers. - """ - self._headers.append((name, val)) - - def __delitem__(self, name): - """Delete all occurrences of a header, if present. - - Does not raise an exception if the header is missing. - """ - name = name.lower() - newheaders = [] - for k, v in self._headers: - if k.lower() != name: - newheaders.append((k, v)) - self._headers = newheaders - - def __contains__(self, name): - return name.lower() in [k.lower() for k, v in self._headers] - - def has_key(self, name): - """Return true if the message contains the header.""" - missing = object() - return self.get(name, missing) is not missing - - def keys(self): - """Return a list of all the message's header field names. - - These will be sorted in the order they appeared in the original - message, or were added to the message, and may contain duplicates. - Any fields deleted and re-inserted are always appended to the header - list. - """ - return [k for k, v in self._headers] - - def values(self): - """Return a list of all the message's header values. - - These will be sorted in the order they appeared in the original - message, or were added to the message, and may contain duplicates. - Any fields deleted and re-inserted are always appended to the header - list. - """ - return [v for k, v in self._headers] - - def items(self): - """Get all the message's header fields and values. - - These will be sorted in the order they appeared in the original - message, or were added to the message, and may contain duplicates. - Any fields deleted and re-inserted are always appended to the header - list. - """ - return self._headers[:] - - def get(self, name, failobj=None): - """Get a header value. - - Like __getitem__() but return failobj instead of None when the field - is missing. - """ - name = name.lower() - for k, v in self._headers: - if k.lower() == name: - return v - return failobj - - # - # Additional useful stuff - # - - def get_all(self, name, failobj=None): - """Return a list of all the values for the named field. - - These will be sorted in the order they appeared in the original - message, and may contain duplicates. Any fields deleted and - re-inserted are always appended to the header list. - - If no such fields exist, failobj is returned (defaults to None). - """ - values = [] - name = name.lower() - for k, v in self._headers: - if k.lower() == name: - values.append(v) - if not values: - return failobj - return values - - def add_header(self, _name, _value, **_params): - """Extended header setting. - - name is the header field to add. keyword arguments can be used to set - additional parameters for the header field, with underscores converted - to dashes. Normally the parameter will be added as key="value" unless - value is None, in which case only the key will be added. If a - parameter value contains non-ASCII characters it must be specified as a - three-tuple of (charset, language, value), in which case it will be - encoded according to RFC2231 rules. - - Example: - - msg.add_header('content-disposition', 'attachment', filename='bud.gif') - """ - parts = [] - for k, v in _params.items(): - if v is None: - parts.append(k.replace('_', '-')) - else: - parts.append(_formatparam(k.replace('_', '-'), v)) - if _value is not None: - parts.insert(0, _value) - self._headers.append((_name, SEMISPACE.join(parts))) - - def replace_header(self, _name, _value): - """Replace a header. - - Replace the first matching header found in the message, retaining - header order and case. If no matching header was found, a KeyError is - raised. - """ - _name = _name.lower() - for i, (k, v) in zip(range(len(self._headers)), self._headers): - if k.lower() == _name: - self._headers[i] = (k, _value) - break - else: - raise KeyError(_name) - - # - # Use these three methods instead of the three above. - # - - def get_content_type(self): - """Return the message's content type. - - The returned string is coerced to lower case of the form - `maintype/subtype'. If there was no Content-Type header in the - message, the default type as given by get_default_type() will be - returned. Since according to RFC 2045, messages always have a default - type this will always return a value. - - RFC 2045 defines a message's default type to be text/plain unless it - appears inside a multipart/digest container, in which case it would be - message/rfc822. - """ - missing = object() - value = self.get('content-type', missing) - if value is missing: - # This should have no parameters - return self.get_default_type() - ctype = _splitparam(value)[0].lower() - # RFC 2045, section 5.2 says if its invalid, use text/plain - if ctype.count('/') != 1: - return 'text/plain' - return ctype - - def get_content_maintype(self): - """Return the message's main content type. - - This is the `maintype' part of the string returned by - get_content_type(). - """ - ctype = self.get_content_type() - return ctype.split('/')[0] - - def get_content_subtype(self): - """Returns the message's sub-content type. - - This is the `subtype' part of the string returned by - get_content_type(). - """ - ctype = self.get_content_type() - return ctype.split('/')[1] - - def get_default_type(self): - """Return the `default' content type. - - Most messages have a default content type of text/plain, except for - messages that are subparts of multipart/digest containers. Such - subparts have a default content type of message/rfc822. - """ - return self._default_type - - def set_default_type(self, ctype): - """Set the `default' content type. - - ctype should be either "text/plain" or "message/rfc822", although this - is not enforced. The default content type is not stored in the - Content-Type header. - """ - self._default_type = ctype - - def _get_params_preserve(self, failobj, header): - # Like get_params() but preserves the quoting of values. BAW: - # should this be part of the public interface? - missing = object() - value = self.get(header, missing) - if value is missing: - return failobj - params = [] - for p in _parseparam(';' + value): - try: - name, val = p.split('=', 1) - name = name.strip() - val = val.strip() - except ValueError: - # Must have been a bare attribute - name = p.strip() - val = '' - params.append((name, val)) - params = utils.decode_params(params) - return params - - def get_params(self, failobj=None, header='content-type', unquote=True): - """Return the message's Content-Type parameters, as a list. - - The elements of the returned list are 2-tuples of key/value pairs, as - split on the `=' sign. The left hand side of the `=' is the key, - while the right hand side is the value. If there is no `=' sign in - the parameter the value is the empty string. The value is as - described in the get_param() method. - - Optional failobj is the object to return if there is no Content-Type - header. Optional header is the header to search instead of - Content-Type. If unquote is True, the value is unquoted. - """ - missing = object() - params = self._get_params_preserve(missing, header) - if params is missing: - return failobj - if unquote: - return [(k, _unquotevalue(v)) for k, v in params] - else: - return params - - def get_param(self, param, failobj=None, header='content-type', - unquote=True): - """Return the parameter value if found in the Content-Type header. - - Optional failobj is the object to return if there is no Content-Type - header, or the Content-Type header has no such parameter. Optional - header is the header to search instead of Content-Type. - - Parameter keys are always compared case insensitively. The return - value can either be a string, or a 3-tuple if the parameter was RFC - 2231 encoded. When it's a 3-tuple, the elements of the value are of - the form (CHARSET, LANGUAGE, VALUE). Note that both CHARSET and - LANGUAGE can be None, in which case you should consider VALUE to be - encoded in the us-ascii charset. You can usually ignore LANGUAGE. - - Your application should be prepared to deal with 3-tuple return - values, and can convert the parameter to a Unicode string like so: - - param = msg.get_param('foo') - if isinstance(param, tuple): - param = unicode(param[2], param[0] or 'us-ascii') - - In any case, the parameter value (either the returned string, or the - VALUE item in the 3-tuple) is always unquoted, unless unquote is set - to False. - """ - if header not in self: - return failobj - for k, v in self._get_params_preserve(failobj, header): - if k.lower() == param.lower(): - if unquote: - return _unquotevalue(v) - else: - return v - return failobj - - def set_param(self, param, value, header='Content-Type', requote=True, - charset=None, language=''): - """Set a parameter in the Content-Type header. - - If the parameter already exists in the header, its value will be - replaced with the new value. - - If header is Content-Type and has not yet been defined for this - message, it will be set to "text/plain" and the new parameter and - value will be appended as per RFC 2045. - - An alternate header can specified in the header argument, and all - parameters will be quoted as necessary unless requote is False. - - If charset is specified, the parameter will be encoded according to RFC - 2231. Optional language specifies the RFC 2231 language, defaulting - to the empty string. Both charset and language should be strings. - """ - if not isinstance(value, tuple) and charset: - value = (charset, language, value) - - if header not in self and header.lower() == 'content-type': - ctype = 'text/plain' - else: - ctype = self.get(header) - if not self.get_param(param, header=header): - if not ctype: - ctype = _formatparam(param, value, requote) - else: - ctype = SEMISPACE.join( - [ctype, _formatparam(param, value, requote)]) - else: - ctype = '' - for old_param, old_value in self.get_params(header=header, - unquote=requote): - append_param = '' - if old_param.lower() == param.lower(): - append_param = _formatparam(param, value, requote) - else: - append_param = _formatparam(old_param, old_value, requote) - if not ctype: - ctype = append_param - else: - ctype = SEMISPACE.join([ctype, append_param]) - if ctype != self.get(header): - del self[header] - self[header] = ctype - - def del_param(self, param, header='content-type', requote=True): - """Remove the given parameter completely from the Content-Type header. - - The header will be re-written in place without the parameter or its - value. All values will be quoted as necessary unless requote is - False. Optional header specifies an alternative to the Content-Type - header. - """ - if header not in self: - return - new_ctype = '' - for p, v in self.get_params(header=header, unquote=requote): - if p.lower() != param.lower(): - if not new_ctype: - new_ctype = _formatparam(p, v, requote) - else: - new_ctype = SEMISPACE.join([new_ctype, - _formatparam(p, v, requote)]) - if new_ctype != self.get(header): - del self[header] - self[header] = new_ctype - - def set_type(self, type, header='Content-Type', requote=True): - """Set the main type and subtype for the Content-Type header. - - type must be a string in the form "maintype/subtype", otherwise a - ValueError is raised. - - This method replaces the Content-Type header, keeping all the - parameters in place. If requote is False, this leaves the existing - header's quoting as is. Otherwise, the parameters will be quoted (the - default). - - An alternative header can be specified in the header argument. When - the Content-Type header is set, we'll always also add a MIME-Version - header. - """ - # BAW: should we be strict? - if not type.count('/') == 1: - raise ValueError - # Set the Content-Type, you get a MIME-Version - if header.lower() == 'content-type': - del self['mime-version'] - self['MIME-Version'] = '1.0' - if header not in self: - self[header] = type - return - params = self.get_params(header=header, unquote=requote) - del self[header] - self[header] = type - # Skip the first param; it's the old type. - for p, v in params[1:]: - self.set_param(p, v, header, requote) - - def get_filename(self, failobj=None): - """Return the filename associated with the payload if present. - - The filename is extracted from the Content-Disposition header's - `filename' parameter, and it is unquoted. If that header is missing - the `filename' parameter, this method falls back to looking for the - `name' parameter. - """ - missing = object() - filename = self.get_param('filename', missing, 'content-disposition') - if filename is missing: - filename = self.get_param('name', missing, 'content-type') - if filename is missing: - return failobj - return utils.collapse_rfc2231_value(filename).strip() - - def get_boundary(self, failobj=None): - """Return the boundary associated with the payload if present. - - The boundary is extracted from the Content-Type header's `boundary' - parameter, and it is unquoted. - """ - missing = object() - boundary = self.get_param('boundary', missing) - if boundary is missing: - return failobj - # RFC 2046 says that boundaries may begin but not end in w/s - return utils.collapse_rfc2231_value(boundary).rstrip() - - def set_boundary(self, boundary): - """Set the boundary parameter in Content-Type to 'boundary'. - - This is subtly different than deleting the Content-Type header and - adding a new one with a new boundary parameter via add_header(). The - main difference is that using the set_boundary() method preserves the - order of the Content-Type header in the original message. - - HeaderParseError is raised if the message has no Content-Type header. - """ - missing = object() - params = self._get_params_preserve(missing, 'content-type') - if params is missing: - # There was no Content-Type header, and we don't know what type - # to set it to, so raise an exception. - raise errors.HeaderParseError('No Content-Type header found') - newparams = [] - foundp = False - for pk, pv in params: - if pk.lower() == 'boundary': - newparams.append(('boundary', '"%s"' % boundary)) - foundp = True - else: - newparams.append((pk, pv)) - if not foundp: - # The original Content-Type header had no boundary attribute. - # Tack one on the end. BAW: should we raise an exception - # instead??? - newparams.append(('boundary', '"%s"' % boundary)) - # Replace the existing Content-Type header with the new value - newheaders = [] - for h, v in self._headers: - if h.lower() == 'content-type': - parts = [] - for k, v in newparams: - if v == '': - parts.append(k) - else: - parts.append('%s=%s' % (k, v)) - newheaders.append((h, SEMISPACE.join(parts))) - - else: - newheaders.append((h, v)) - self._headers = newheaders - - def get_content_charset(self, failobj=None): - """Return the charset parameter of the Content-Type header. - - The returned string is always coerced to lower case. If there is no - Content-Type header, or if that header has no charset parameter, - failobj is returned. - """ - missing = object() - charset = self.get_param('charset', missing) - if charset is missing: - return failobj - if isinstance(charset, tuple): - # RFC 2231 encoded, so decode it, and it better end up as ascii. - pcharset = charset[0] or 'us-ascii' - try: - # LookupError will be raised if the charset isn't known to - # Python. UnicodeError will be raised if the encoded text - # contains a character not in the charset. - charset = unicode(charset[2], pcharset).encode('us-ascii') - except (LookupError, UnicodeError): - charset = charset[2] - # charset character must be in us-ascii range - try: - if isinstance(charset, str): - charset = unicode(charset, 'us-ascii') - charset = charset.encode('us-ascii') - except UnicodeError: - return failobj - # RFC 2046, $4.1.2 says charsets are not case sensitive - return charset.lower() - - def get_charsets(self, failobj=None): - """Return a list containing the charset(s) used in this message. - - The returned list of items describes the Content-Type headers' - charset parameter for this message and all the subparts in its - payload. - - Each item will either be a string (the value of the charset parameter - in the Content-Type header of that part) or the value of the - 'failobj' parameter (defaults to None), if the part does not have a - main MIME type of "text", or the charset is not defined. - - The list will contain one string for each part of the message, plus - one for the container message (i.e. self), so that a non-multipart - message will still return a list of length 1. - """ - return [part.get_content_charset(failobj) for part in self.walk()] - - # I.e. def walk(self): ... - from email.iterators import walk diff --git a/lib-python/modified-2.7/email/mime/__init__.py b/lib-python/modified-2.7/email/mime/__init__.py deleted file mode 100644 diff --git a/lib-python/modified-2.7/email/mime/application.py b/lib-python/modified-2.7/email/mime/application.py deleted file mode 100644 --- a/lib-python/modified-2.7/email/mime/application.py +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright (C) 2001-2006 Python Software Foundation -# Author: Keith Dart -# Contact: email-sig at python.org - -"""Class representing application/* type MIME documents.""" - -__all__ = ["MIMEApplication"] - -from email import encoders -from email.mime.nonmultipart import MIMENonMultipart - - -class MIMEApplication(MIMENonMultipart): - """Class for generating application/* MIME documents.""" - - def __init__(self, _data, _subtype='octet-stream', - _encoder=encoders.encode_base64, **_params): - """Create an application/* type MIME document. - - _data is a string containing the raw application data. - - _subtype is the MIME content type subtype, defaulting to - 'octet-stream'. - - _encoder is a function which will perform the actual encoding for - transport of the application data, defaulting to base64 encoding. - - Any additional keyword arguments are passed to the base class - constructor, which turns them into parameters on the Content-Type - header. - """ - if _subtype is None: - raise TypeError('Invalid application MIME subtype') - MIMENonMultipart.__init__(self, 'application', _subtype, **_params) - self.set_payload(_data) - _encoder(self) diff --git a/lib-python/modified-2.7/email/mime/audio.py b/lib-python/modified-2.7/email/mime/audio.py deleted file mode 100644 --- a/lib-python/modified-2.7/email/mime/audio.py +++ /dev/null @@ -1,73 +0,0 @@ -# Copyright (C) 2001-2006 Python Software Foundation -# Author: Anthony Baxter -# Contact: email-sig at python.org - -"""Class representing audio/* type MIME documents.""" - -__all__ = ['MIMEAudio'] - -import sndhdr - -from cStringIO import StringIO -from email import encoders -from email.mime.nonmultipart import MIMENonMultipart - - - -_sndhdr_MIMEmap = {'au' : 'basic', - 'wav' :'x-wav', - 'aiff':'x-aiff', - 'aifc':'x-aiff', - } - -# There are others in sndhdr that don't have MIME types. :( -# Additional ones to be added to sndhdr? midi, mp3, realaudio, wma?? -def _whatsnd(data): - """Try to identify a sound file type. - - sndhdr.what() has a pretty cruddy interface, unfortunately. This is why - we re-do it here. It would be easier to reverse engineer the Unix 'file' - command and use the standard 'magic' file, as shipped with a modern Unix. - """ - hdr = data[:512] - fakefile = StringIO(hdr) - for testfn in sndhdr.tests: - res = testfn(hdr, fakefile) - if res is not None: - return _sndhdr_MIMEmap.get(res[0]) - return None - - - -class MIMEAudio(MIMENonMultipart): - """Class for generating audio/* MIME documents.""" - - def __init__(self, _audiodata, _subtype=None, - _encoder=encoders.encode_base64, **_params): - """Create an audio/* type MIME document. - - _audiodata is a string containing the raw audio data. If this data - can be decoded by the standard Python `sndhdr' module, then the - subtype will be automatically included in the Content-Type header. - Otherwise, you can specify the specific audio subtype via the - _subtype parameter. If _subtype is not given, and no subtype can be - guessed, a TypeError is raised. - - _encoder is a function which will perform the actual encoding for - transport of the image data. It takes one argument, which is this - Image instance. It should use get_payload() and set_payload() to - change the payload to the encoded form. It should also add any - Content-Transfer-Encoding or other headers to the message as - necessary. The default encoding is Base64. - - Any additional keyword arguments are passed to the base class - constructor, which turns them into parameters on the Content-Type - header. - """ - if _subtype is None: - _subtype = _whatsnd(_audiodata) - if _subtype is None: - raise TypeError('Could not find audio MIME subtype') - MIMENonMultipart.__init__(self, 'audio', _subtype, **_params) - self.set_payload(_audiodata) - _encoder(self) diff --git a/lib-python/modified-2.7/email/mime/base.py b/lib-python/modified-2.7/email/mime/base.py deleted file mode 100644 --- a/lib-python/modified-2.7/email/mime/base.py +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright (C) 2001-2006 Python Software Foundation -# Author: Barry Warsaw -# Contact: email-sig at python.org - -"""Base class for MIME specializations.""" - -__all__ = ['MIMEBase'] - -from email import message - - - -class MIMEBase(message.Message): - """Base class for MIME specializations.""" - - def __init__(self, _maintype, _subtype, **_params): - """This constructor adds a Content-Type: and a MIME-Version: header. - - The Content-Type: header is taken from the _maintype and _subtype - arguments. Additional parameters for this header are taken from the - keyword arguments. - """ - message.Message.__init__(self) - ctype = '%s/%s' % (_maintype, _subtype) - self.add_header('Content-Type', ctype, **_params) - self['MIME-Version'] = '1.0' diff --git a/lib-python/modified-2.7/email/mime/image.py b/lib-python/modified-2.7/email/mime/image.py deleted file mode 100644 --- a/lib-python/modified-2.7/email/mime/image.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright (C) 2001-2006 Python Software Foundation -# Author: Barry Warsaw -# Contact: email-sig at python.org - -"""Class representing image/* type MIME documents.""" - -__all__ = ['MIMEImage'] - -import imghdr - -from email import encoders -from email.mime.nonmultipart import MIMENonMultipart - - - -class MIMEImage(MIMENonMultipart): - """Class for generating image/* type MIME documents.""" - - def __init__(self, _imagedata, _subtype=None, - _encoder=encoders.encode_base64, **_params): - """Create an image/* type MIME document. - - _imagedata is a string containing the raw image data. If this data - can be decoded by the standard Python `imghdr' module, then the - subtype will be automatically included in the Content-Type header. - Otherwise, you can specify the specific image subtype via the _subtype - parameter. - - _encoder is a function which will perform the actual encoding for - transport of the image data. It takes one argument, which is this - Image instance. It should use get_payload() and set_payload() to - change the payload to the encoded form. It should also add any - Content-Transfer-Encoding or other headers to the message as - necessary. The default encoding is Base64. - - Any additional keyword arguments are passed to the base class - constructor, which turns them into parameters on the Content-Type - header. - """ - if _subtype is None: - _subtype = imghdr.what(None, _imagedata) - if _subtype is None: - raise TypeError('Could not guess image MIME subtype') - MIMENonMultipart.__init__(self, 'image', _subtype, **_params) - self.set_payload(_imagedata) - _encoder(self) diff --git a/lib-python/modified-2.7/email/mime/message.py b/lib-python/modified-2.7/email/mime/message.py deleted file mode 100644 --- a/lib-python/modified-2.7/email/mime/message.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright (C) 2001-2006 Python Software Foundation -# Author: Barry Warsaw -# Contact: email-sig at python.org - -"""Class representing message/* MIME documents.""" - -__all__ = ['MIMEMessage'] - -from email import message -from email.mime.nonmultipart import MIMENonMultipart - - - -class MIMEMessage(MIMENonMultipart): - """Class representing message/* MIME documents.""" - - def __init__(self, _msg, _subtype='rfc822'): - """Create a message/* type MIME document. - - _msg is a message object and must be an instance of Message, or a - derived class of Message, otherwise a TypeError is raised. - - Optional _subtype defines the subtype of the contained message. The - default is "rfc822" (this is defined by the MIME standard, even though - the term "rfc822" is technically outdated by RFC 2822). - """ - MIMENonMultipart.__init__(self, 'message', _subtype) - if not isinstance(_msg, message.Message): - raise TypeError('Argument is not an instance of Message') - # It's convenient to use this base class method. We need to do it - # this way or we'll get an exception - message.Message.attach(self, _msg) - # And be sure our default type is set correctly - self.set_default_type('message/rfc822') diff --git a/lib-python/modified-2.7/email/mime/multipart.py b/lib-python/modified-2.7/email/mime/multipart.py deleted file mode 100644 --- a/lib-python/modified-2.7/email/mime/multipart.py +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright (C) 2002-2006 Python Software Foundation -# Author: Barry Warsaw -# Contact: email-sig at python.org - -"""Base class for MIME multipart/* type messages.""" - -__all__ = ['MIMEMultipart'] - -from email.mime.base import MIMEBase - - - -class MIMEMultipart(MIMEBase): - """Base class for MIME multipart/* type messages.""" - - def __init__(self, _subtype='mixed', boundary=None, _subparts=None, - **_params): - """Creates a multipart/* type message. - - By default, creates a multipart/mixed message, with proper - Content-Type and MIME-Version headers. - - _subtype is the subtype of the multipart content type, defaulting to - `mixed'. - - boundary is the multipart boundary string. By default it is - calculated as needed. - - _subparts is a sequence of initial subparts for the payload. It - must be an iterable object, such as a list. You can always - attach new subparts to the message by using the attach() method. - - Additional parameters for the Content-Type header are taken from the - keyword arguments (or passed into the _params argument). - """ - MIMEBase.__init__(self, 'multipart', _subtype, **_params) - - # Initialise _payload to an empty list as the Message superclass's - # implementation of is_multipart assumes that _payload is a list for - # multipart messages. - self._payload = [] - - if _subparts: - for p in _subparts: - self.attach(p) - if boundary: - self.set_boundary(boundary) diff --git a/lib-python/modified-2.7/email/mime/nonmultipart.py b/lib-python/modified-2.7/email/mime/nonmultipart.py deleted file mode 100644 --- a/lib-python/modified-2.7/email/mime/nonmultipart.py +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright (C) 2002-2006 Python Software Foundation -# Author: Barry Warsaw -# Contact: email-sig at python.org - -"""Base class for MIME type messages that are not multipart.""" - -__all__ = ['MIMENonMultipart'] - -from email import errors -from email.mime.base import MIMEBase - - - -class MIMENonMultipart(MIMEBase): - """Base class for MIME multipart/* type messages.""" - - def attach(self, payload): - # The public API prohibits attaching multiple subparts to MIMEBase - # derived subtypes since none of them are, by definition, of content - # type multipart/* - raise errors.MultipartConversionError( - 'Cannot attach additional subparts to non-multipart/*') diff --git a/lib-python/modified-2.7/email/mime/text.py b/lib-python/modified-2.7/email/mime/text.py deleted file mode 100644 --- a/lib-python/modified-2.7/email/mime/text.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright (C) 2001-2006 Python Software Foundation -# Author: Barry Warsaw -# Contact: email-sig at python.org - -"""Class representing text/* type MIME documents.""" - -__all__ = ['MIMEText'] - -from email.encoders import encode_7or8bit -from email.mime.nonmultipart import MIMENonMultipart - - - -class MIMEText(MIMENonMultipart): - """Class for generating text/* type MIME documents.""" - - def __init__(self, _text, _subtype='plain', _charset='us-ascii'): - """Create a text/* type MIME document. - - _text is the string for this message object. - - _subtype is the MIME sub content type, defaulting to "plain". - - _charset is the character set parameter added to the Content-Type - header. This defaults to "us-ascii". Note that as a side-effect, the - Content-Transfer-Encoding header will also be set. - """ - MIMENonMultipart.__init__(self, 'text', _subtype, - **{'charset': _charset}) - self.set_payload(_text, _charset) diff --git a/lib-python/modified-2.7/email/parser.py b/lib-python/modified-2.7/email/parser.py deleted file mode 100644 --- a/lib-python/modified-2.7/email/parser.py +++ /dev/null @@ -1,91 +0,0 @@ -# Copyright (C) 2001-2006 Python Software Foundation -# Author: Barry Warsaw, Thomas Wouters, Anthony Baxter -# Contact: email-sig at python.org - -"""A parser of RFC 2822 and MIME email messages.""" - -__all__ = ['Parser', 'HeaderParser'] - -import warnings -from cStringIO import StringIO - -from email.feedparser import FeedParser -from email.message import Message - - - -class Parser: - def __init__(self, *args, **kws): - """Parser of RFC 2822 and MIME email messages. - - Creates an in-memory object tree representing the email message, which - can then be manipulated and turned over to a Generator to return the - textual representation of the message. - - The string must be formatted as a block of RFC 2822 headers and header - continuation lines, optionally preceeded by a `Unix-from' header. The - header block is terminated either by the end of the string or by a - blank line. - - _class is the class to instantiate for new message objects when they - must be created. This class must have a constructor that can take - zero arguments. Default is Message.Message. - """ - if len(args) >= 1: - if '_class' in kws: - raise TypeError("Multiple values for keyword arg '_class'") - kws['_class'] = args[0] - if len(args) == 2: - if 'strict' in kws: - raise TypeError("Multiple values for keyword arg 'strict'") - kws['strict'] = args[1] - if len(args) > 2: - raise TypeError('Too many arguments') - if '_class' in kws: - self._class = kws['_class'] - del kws['_class'] - else: - self._class = Message - if 'strict' in kws: - warnings.warn("'strict' argument is deprecated (and ignored)", - DeprecationWarning, 2) - del kws['strict'] - if kws: - raise TypeError('Unexpected keyword arguments') - - def parse(self, fp, headersonly=False): - """Create a message structure from the data in a file. - - Reads all the data from the file and returns the root of the message - structure. Optional headersonly is a flag specifying whether to stop - parsing after reading the headers or not. The default is False, - meaning it parses the entire contents of the file. - """ - feedparser = FeedParser(self._class) - if headersonly: - feedparser._set_headersonly() - while True: - data = fp.read(8192) - if not data: - break - feedparser.feed(data) - return feedparser.close() - - def parsestr(self, text, headersonly=False): - """Create a message structure from a string. - - Returns the root of the message structure. Optional headersonly is a - flag specifying whether to stop parsing after reading the headers or - not. The default is False, meaning it parses the entire contents of - the file. - """ - return self.parse(StringIO(text), headersonly=headersonly) - - - -class HeaderParser(Parser): - def parse(self, fp, headersonly=True): - return Parser.parse(self, fp, True) - - def parsestr(self, text, headersonly=True): - return Parser.parsestr(self, text, True) diff --git a/lib-python/modified-2.7/email/quoprimime.py b/lib-python/modified-2.7/email/quoprimime.py deleted file mode 100644 --- a/lib-python/modified-2.7/email/quoprimime.py +++ /dev/null @@ -1,336 +0,0 @@ -# Copyright (C) 2001-2006 Python Software Foundation -# Author: Ben Gertzfield -# Contact: email-sig at python.org - -"""Quoted-printable content transfer encoding per RFCs 2045-2047. - -This module handles the content transfer encoding method defined in RFC 2045 -to encode US ASCII-like 8-bit data called `quoted-printable'. It is used to -safely encode text that is in a character set similar to the 7-bit US ASCII -character set, but that includes some 8-bit characters that are normally not -allowed in email bodies or headers. - -Quoted-printable is very space-inefficient for encoding binary files; use the -email.base64mime module for that instead. - -This module provides an interface to encode and decode both headers and bodies -with quoted-printable encoding. - -RFC 2045 defines a method for including character set information in an -`encoded-word' in a header. This method is commonly used for 8-bit real names -in To:/From:/Cc: etc. fields, as well as Subject: lines. - -This module does not do the line wrapping or end-of-line character -conversion necessary for proper internationalized headers; it only -does dumb encoding and decoding. To deal with the various line -wrapping issues, use the email.header module. -""" - -__all__ = [ - 'body_decode', - 'body_encode', - 'body_quopri_check', - 'body_quopri_len', - 'decode', - 'decodestring', - 'encode', - 'encodestring', - 'header_decode', - 'header_encode', - 'header_quopri_check', - 'header_quopri_len', - 'quote', - 'unquote', - ] - -import re - -from string import hexdigits -from email.utils import fix_eols - -CRLF = '\r\n' -NL = '\n' - -# See also Charset.py -MISC_LEN = 7 - -hqre = re.compile(r'[^-a-zA-Z0-9!*+/ ]') -bqre = re.compile(r'[^ !-<>-~\t]') - - - -# Helpers -def header_quopri_check(c): - """Return True if the character should be escaped with header quopri.""" - return bool(hqre.match(c)) - - -def body_quopri_check(c): - """Return True if the character should be escaped with body quopri.""" - return bool(bqre.match(c)) - - -def header_quopri_len(s): - """Return the length of str when it is encoded with header quopri.""" - count = 0 - for c in s: - if hqre.match(c): - count += 3 - else: - count += 1 - return count - - -def body_quopri_len(str): - """Return the length of str when it is encoded with body quopri.""" - count = 0 - for c in str: - if bqre.match(c): - count += 3 - else: - count += 1 - return count - - -def _max_append(L, s, maxlen, extra=''): - if not L: - L.append(s.lstrip()) - elif len(L[-1]) + len(s) <= maxlen: - L[-1] += extra + s - else: - L.append(s.lstrip()) - - -def unquote(s): - """Turn a string in the form =AB to the ASCII character with value 0xab""" - return chr(int(s[1:3], 16)) - - -def quote(c): - return "=%02X" % ord(c) - - - -def header_encode(header, charset="iso-8859-1", keep_eols=False, - maxlinelen=76, eol=NL): - """Encode a single header line with quoted-printable (like) encoding. - - Defined in RFC 2045, this `Q' encoding is similar to quoted-printable, but - used specifically for email header fields to allow charsets with mostly 7 - bit characters (and some 8 bit) to remain more or less readable in non-RFC - 2045 aware mail clients. - - charset names the character set to use to encode the header. It defaults - to iso-8859-1. - - The resulting string will be in the form: - - "=?charset?q?I_f=E2rt_in_your_g=E8n=E8ral_dire=E7tion?\\n - =?charset?q?Silly_=C8nglish_Kn=EEghts?=" - - with each line wrapped safely at, at most, maxlinelen characters (defaults - to 76 characters). If maxlinelen is None, the entire string is encoded in - one chunk with no splitting. - - End-of-line characters (\\r, \\n, \\r\\n) will be automatically converted - to the canonical email line separator \\r\\n unless the keep_eols - parameter is True (the default is False). - - Each line of the header will be terminated in the value of eol, which - defaults to "\\n". Set this to "\\r\\n" if you are using the result of - this function directly in email. - """ - # Return empty headers unchanged - if not header: - return header - - if not keep_eols: - header = fix_eols(header) - - # Quopri encode each line, in encoded chunks no greater than maxlinelen in - # length, after the RFC chrome is added in. - quoted = [] - if maxlinelen is None: - # An obnoxiously large number that's good enough - max_encoded = 100000 - else: - max_encoded = maxlinelen - len(charset) - MISC_LEN - 1 - - for c in header: - # Space may be represented as _ instead of =20 for readability - if c == ' ': - _max_append(quoted, '_', max_encoded) - # These characters can be included verbatim - elif not hqre.match(c): - _max_append(quoted, c, max_encoded) - # Otherwise, replace with hex value like =E2 - else: - _max_append(quoted, "=%02X" % ord(c), max_encoded) - - # Now add the RFC chrome to each encoded chunk and glue the chunks - # together. BAW: should we be able to specify the leading whitespace in - # the joiner? - joiner = eol + ' ' - return joiner.join(['=?%s?q?%s?=' % (charset, line) for line in quoted]) - - - -def encode(body, binary=False, maxlinelen=76, eol=NL): - """Encode with quoted-printable, wrapping at maxlinelen characters. - - If binary is False (the default), end-of-line characters will be converted - to the canonical email end-of-line sequence \\r\\n. Otherwise they will - be left verbatim. - - Each line of encoded text will end with eol, which defaults to "\\n". Set - this to "\\r\\n" if you will be using the result of this function directly - in an email. - - Each line will be wrapped at, at most, maxlinelen characters (defaults to - 76 characters). Long lines will have the `soft linefeed' quoted-printable - character "=" appended to them, so the decoded text will be identical to - the original text. - """ - if not body: - return body - - if not binary: - body = fix_eols(body) - - # BAW: We're accumulating the body text by string concatenation. That - # can't be very efficient, but I don't have time now to rewrite it. It - # just feels like this algorithm could be more efficient. - encoded_body = '' - lineno = -1 - # Preserve line endings here so we can check later to see an eol needs to - # be added to the output later. - lines = body.splitlines(1) - for line in lines: - # But strip off line-endings for processing this line. - if line.endswith(CRLF): - line = line[:-2] - elif line[-1] in CRLF: - line = line[:-1] - - lineno += 1 - encoded_line = '' - prev = None - linelen = len(line) - # Now we need to examine every character to see if it needs to be - # quopri encoded. BAW: again, string concatenation is inefficient. - for j in range(linelen): - c = line[j] - prev = c - if bqre.match(c): - c = quote(c) - elif j+1 == linelen: - # Check for whitespace at end of line; special case - if c not in ' \t': - encoded_line += c - prev = c - continue - # Check to see to see if the line has reached its maximum length - if len(encoded_line) + len(c) >= maxlinelen: - encoded_body += encoded_line + '=' + eol - encoded_line = '' - encoded_line += c - # Now at end of line.. - if prev and prev in ' \t': - # Special case for whitespace at end of file - if lineno + 1 == len(lines): - prev = quote(prev) - if len(encoded_line) + len(prev) > maxlinelen: - encoded_body += encoded_line + '=' + eol + prev - else: - encoded_body += encoded_line + prev - # Just normal whitespace at end of line - else: - encoded_body += encoded_line + prev + '=' + eol - encoded_line = '' - # Now look at the line we just finished and it has a line ending, we - # need to add eol to the end of the line. - if lines[lineno].endswith(CRLF) or lines[lineno][-1] in CRLF: - encoded_body += encoded_line + eol - else: - encoded_body += encoded_line - encoded_line = '' - return encoded_body - - -# For convenience and backwards compatibility w/ standard base64 module -body_encode = encode -encodestring = encode - - - -# BAW: I'm not sure if the intent was for the signature of this function to be -# the same as base64MIME.decode() or not... -def decode(encoded, eol=NL): - """Decode a quoted-printable string. - - Lines are separated with eol, which defaults to \\n. - """ - if not encoded: - return encoded - # BAW: see comment in encode() above. Again, we're building up the - # decoded string with string concatenation, which could be done much more - # efficiently. - decoded = '' - - for line in encoded.splitlines(): - line = line.rstrip() - if not line: - decoded += eol - continue - - i = 0 - n = len(line) - while i < n: - c = line[i] - if c != '=': - decoded += c - i += 1 - # Otherwise, c == "=". Are we at the end of the line? If so, add - # a soft line break. - elif i+1 == n: - i += 1 - continue - # Decode if in form =AB - elif i+2 < n and line[i+1] in hexdigits and line[i+2] in hexdigits: - decoded += unquote(line[i:i+3]) - i += 3 - # Otherwise, not in form =AB, pass literally - else: - decoded += c - i += 1 - - if i == n: - decoded += eol - # Special case if original string did not end with eol - if not encoded.endswith(eol) and decoded.endswith(eol): - decoded = decoded[:-1] - return decoded - - -# For convenience and backwards compatibility w/ standard base64 module -body_decode = decode -decodestring = decode - - - -def _unquote_match(match): - """Turn a match in the form =AB to the ASCII character with value 0xab""" - s = match.group(0) - return unquote(s) - - -# Header decoding is done a bit differently -def header_decode(s): - """Decode a string encoded with RFC 2045 MIME header `Q' encoding. - - This function does not parse a full MIME header value encoded with - quoted-printable (like =?iso-8895-1?q?Hello_World?=) -- please use - the high level email.header class for that functionality. - """ - s = s.replace('_', ' ') - return re.sub(r'=[a-fA-F0-9]{2}', _unquote_match, s) diff --git a/lib-python/modified-2.7/email/test/__init__.py b/lib-python/modified-2.7/email/test/__init__.py deleted file mode 100644 diff --git a/lib-python/modified-2.7/email/test/data/PyBanner048.gif b/lib-python/modified-2.7/email/test/data/PyBanner048.gif deleted file mode 100644 Binary file lib-python/modified-2.7/email/test/data/PyBanner048.gif has changed diff --git a/lib-python/modified-2.7/email/test/data/audiotest.au b/lib-python/modified-2.7/email/test/data/audiotest.au deleted file mode 100644 Binary file lib-python/modified-2.7/email/test/data/audiotest.au has changed diff --git a/lib-python/modified-2.7/email/test/data/msg_01.txt b/lib-python/modified-2.7/email/test/data/msg_01.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_01.txt +++ /dev/null @@ -1,19 +0,0 @@ -Return-Path: <bbb at zzz.org> -Delivered-To: bbb at zzz.org -Received: by mail.zzz.org (Postfix, from userid 889) - id 27CEAD38CC; Fri, 4 May 2001 14:05:44 -0400 (EDT) -MIME-Version: 1.0 -Content-Type: text/plain; charset=us-ascii -Content-Transfer-Encoding: 7bit -Message-ID: <15090.61304.110929.45684 at aaa.zzz.org> -From: bbb at ddd.com (John X. Doe) -To: bbb at zzz.org -Subject: This is a test message -Date: Fri, 4 May 2001 14:05:44 -0400 - - -Hi, - -Do you like this message? - --Me diff --git a/lib-python/modified-2.7/email/test/data/msg_02.txt b/lib-python/modified-2.7/email/test/data/msg_02.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_02.txt +++ /dev/null @@ -1,135 +0,0 @@ -MIME-version: 1.0 -From: ppp-request at zzz.org -Sender: ppp-admin at zzz.org -To: ppp at zzz.org -Subject: Ppp digest, Vol 1 #2 - 5 msgs -Date: Fri, 20 Apr 2001 20:18:00 -0400 (EDT) -X-Mailer: Mailman v2.0.4 -X-Mailman-Version: 2.0.4 -Content-Type: multipart/mixed; boundary="192.168.1.2.889.32614.987812255.500.21814" - ---192.168.1.2.889.32614.987812255.500.21814 -Content-type: text/plain; charset=us-ascii -Content-description: Masthead (Ppp digest, Vol 1 #2) - -Send Ppp mailing list submissions to - ppp at zzz.org - -To subscribe or unsubscribe via the World Wide Web, visit - http://www.zzz.org/mailman/listinfo/ppp -or, via email, send a message with subject or body 'help' to - ppp-request at zzz.org - -You can reach the person managing the list at - ppp-admin at zzz.org - -When replying, please edit your Subject line so it is more specific -than "Re: Contents of Ppp digest..." - - ---192.168.1.2.889.32614.987812255.500.21814 -Content-type: text/plain; charset=us-ascii -Content-description: Today's Topics (5 msgs) - -Today's Topics: - - 1. testing #1 (Barry A. Warsaw) - 2. testing #2 (Barry A. Warsaw) - 3. testing #3 (Barry A. Warsaw) - 4. testing #4 (Barry A. Warsaw) - 5. testing #5 (Barry A. Warsaw) - ---192.168.1.2.889.32614.987812255.500.21814 -Content-Type: multipart/digest; boundary="__--__--" - ---__--__-- - -Message: 1 -Content-Type: text/plain; charset=us-ascii -Content-Transfer-Encoding: 7bit -Date: Fri, 20 Apr 2001 20:16:13 -0400 -To: ppp at zzz.org -From: barry at digicool.com (Barry A. Warsaw) -Subject: [Ppp] testing #1 -Precedence: bulk - - -hello - - ---__--__-- - -Message: 2 -Date: Fri, 20 Apr 2001 20:16:21 -0400 -Content-Type: text/plain; charset=us-ascii -Content-Transfer-Encoding: 7bit -To: ppp at zzz.org -From: barry at digicool.com (Barry A. Warsaw) -Precedence: bulk - - -hello - - ---__--__-- - -Message: 3 -Date: Fri, 20 Apr 2001 20:16:25 -0400 -Content-Type: text/plain; charset=us-ascii -Content-Transfer-Encoding: 7bit -To: ppp at zzz.org -From: barry at digicool.com (Barry A. Warsaw) -Subject: [Ppp] testing #3 -Precedence: bulk - - -hello - - ---__--__-- - -Message: 4 -Date: Fri, 20 Apr 2001 20:16:28 -0400 -Content-Type: text/plain; charset=us-ascii -Content-Transfer-Encoding: 7bit -To: ppp at zzz.org -From: barry at digicool.com (Barry A. Warsaw) -Subject: [Ppp] testing #4 -Precedence: bulk - - -hello - - ---__--__-- - -Message: 5 -Date: Fri, 20 Apr 2001 20:16:32 -0400 -Content-Type: text/plain; charset=us-ascii -Content-Transfer-Encoding: 7bit -To: ppp at zzz.org -From: barry at digicool.com (Barry A. Warsaw) -Subject: [Ppp] testing #5 -Precedence: bulk - - -hello - - - - ---__--__---- ---192.168.1.2.889.32614.987812255.500.21814 -Content-type: text/plain; charset=us-ascii -Content-description: Digest Footer - -_______________________________________________ -Ppp mailing list -Ppp at zzz.org -http://www.zzz.org/mailman/listinfo/ppp - - ---192.168.1.2.889.32614.987812255.500.21814-- - -End of Ppp Digest - diff --git a/lib-python/modified-2.7/email/test/data/msg_03.txt b/lib-python/modified-2.7/email/test/data/msg_03.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_03.txt +++ /dev/null @@ -1,16 +0,0 @@ -Return-Path: <bbb at zzz.org> -Delivered-To: bbb at zzz.org -Received: by mail.zzz.org (Postfix, from userid 889) - id 27CEAD38CC; Fri, 4 May 2001 14:05:44 -0400 (EDT) -Message-ID: <15090.61304.110929.45684 at aaa.zzz.org> -From: bbb at ddd.com (John X. Doe) -To: bbb at zzz.org -Subject: This is a test message -Date: Fri, 4 May 2001 14:05:44 -0400 - - -Hi, - -Do you like this message? - --Me diff --git a/lib-python/modified-2.7/email/test/data/msg_04.txt b/lib-python/modified-2.7/email/test/data/msg_04.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_04.txt +++ /dev/null @@ -1,37 +0,0 @@ -Return-Path: <barry at python.org> -Delivered-To: barry at python.org -Received: by mail.python.org (Postfix, from userid 889) - id C2BF0D37C6; Tue, 11 Sep 2001 00:05:05 -0400 (EDT) -MIME-Version: 1.0 -Content-Type: multipart/mixed; boundary="h90VIIIKmx" -Content-Transfer-Encoding: 7bit -Message-ID: <15261.36209.358846.118674 at anthem.python.org> -From: barry at python.org (Barry A. Warsaw) -To: barry at python.org -Subject: a simple multipart -Date: Tue, 11 Sep 2001 00:05:05 -0400 -X-Mailer: VM 6.95 under 21.4 (patch 4) "Artificial Intelligence" XEmacs Lucid -X-Attribution: BAW -X-Oblique-Strategy: Make a door into a window - - ---h90VIIIKmx -Content-Type: text/plain -Content-Disposition: inline; - filename="msg.txt" -Content-Transfer-Encoding: 7bit - -a simple kind of mirror -to reflect upon our own - ---h90VIIIKmx -Content-Type: text/plain -Content-Disposition: inline; - filename="msg.txt" -Content-Transfer-Encoding: 7bit - -a simple kind of mirror -to reflect upon our own - ---h90VIIIKmx-- - diff --git a/lib-python/modified-2.7/email/test/data/msg_05.txt b/lib-python/modified-2.7/email/test/data/msg_05.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_05.txt +++ /dev/null @@ -1,28 +0,0 @@ -From: foo -Subject: bar -To: baz -MIME-Version: 1.0 -Content-Type: multipart/report; report-type=delivery-status; - boundary="D1690A7AC1.996856090/mail.example.com" -Message-Id: <20010803162810.0CA8AA7ACC at mail.example.com> - -This is a MIME-encapsulated message. - ---D1690A7AC1.996856090/mail.example.com -Content-Type: text/plain - -Yadda yadda yadda - ---D1690A7AC1.996856090/mail.example.com - -Yadda yadda yadda - ---D1690A7AC1.996856090/mail.example.com -Content-Type: message/rfc822 - -From: nobody at python.org - -Yadda yadda yadda - ---D1690A7AC1.996856090/mail.example.com-- - diff --git a/lib-python/modified-2.7/email/test/data/msg_06.txt b/lib-python/modified-2.7/email/test/data/msg_06.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_06.txt +++ /dev/null @@ -1,33 +0,0 @@ -Return-Path: <barry at python.org> -Delivered-To: barry at python.org -MIME-Version: 1.0 -Content-Type: message/rfc822 -Content-Description: forwarded message -Content-Transfer-Encoding: 7bit -Message-ID: <15265.9482.641338.555352 at python.org> -From: barry at zope.com (Barry A. Warsaw) -Sender: barry at python.org -To: barry at python.org -Subject: forwarded message from Barry A. Warsaw -Date: Thu, 13 Sep 2001 17:28:42 -0400 -X-Mailer: VM 6.95 under 21.4 (patch 4) "Artificial Intelligence" XEmacs Lucid -X-Attribution: BAW -X-Oblique-Strategy: Be dirty -X-Url: http://barry.wooz.org - -MIME-Version: 1.0 -Content-Type: text/plain; charset=us-ascii -Return-Path: <barry at python.org> -Delivered-To: barry at python.org -Message-ID: <15265.9468.713530.98441 at python.org> -From: barry at zope.com (Barry A. Warsaw) -Sender: barry at python.org -To: barry at python.org -Subject: testing -Date: Thu, 13 Sep 2001 17:28:28 -0400 -X-Mailer: VM 6.95 under 21.4 (patch 4) "Artificial Intelligence" XEmacs Lucid -X-Attribution: BAW -X-Oblique-Strategy: Spectrum analysis -X-Url: http://barry.wooz.org - - diff --git a/lib-python/modified-2.7/email/test/data/msg_07.txt b/lib-python/modified-2.7/email/test/data/msg_07.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_07.txt +++ /dev/null @@ -1,83 +0,0 @@ -MIME-Version: 1.0 -From: Barry <barry at digicool.com> -To: Dingus Lovers <cravindogs at cravindogs.com> -Subject: Here is your dingus fish -Date: Fri, 20 Apr 2001 19:35:02 -0400 -Content-Type: multipart/mixed; boundary="BOUNDARY" - ---BOUNDARY -Content-Type: text/plain; charset="us-ascii" - -Hi there, - -This is the dingus fish. - ---BOUNDARY -Content-Type: image/gif; name="dingusfish.gif" -Content-Transfer-Encoding: base64 -content-disposition: attachment; filename="dingusfish.gif" - -R0lGODdhAAEAAfAAAP///wAAACwAAAAAAAEAAQAC/oSPqcvtD6OctNqLs968+w+G4kiW5omm6sq2 -7gvH8kzX9o3n+s73/g8MCofEovGITGICTKbyCV0FDNOo9SqpQqpOrJfXzTQj2vD3TGtqL+NtGQ2f -qTXmxzuOd7WXdcc9DyjU53ewFni4s0fGhdiYaEhGBelICTNoV1j5NUnFcrmUqemjNifJVWpaOqaI -oFq3SspZsSraE7sHq3jr1MZqWvi662vxV4tD+pvKW6aLDOCLyur8PDwbanyDeq0N3DctbQYeLDvR -RY6t95m6UB0d3mwIrV7e2VGNvjjffukeJp4w7F65KecGFsTHQGAygOrgrWs1jt28Rc88KESYcGLA -/obvTkH6p+CinWJiJmIMqXGQwH/y4qk0SYjgQTczT3ajKZGfuI0uJ4kkVI/DT5s3/ejkxI0aT4Y+ -YTYgWbImUaXk9nlLmnSh1qJiJFl0OpUqRK4oOy7NyRQtHWofhoYVxkwWXKUSn0YsS+fUV6lhqfYb -6ayd3Z5qQdG1B7bvQzaJjwUV2lixMUZ7JVsOlfjWVr/3NB/uFvnySBN6Dcb6rGwaRM3wsormw5cC -M9NxWy/bWdufudCvy8bOAjXjVVwta/uO21sE5RHBCzNFXtgq9ORtH4eYjVP4Yryo026nvkFmCeyA -B29efV6ravCMK5JwWd5897Qrx7ll38o6iHDZ/rXPR//feevhF4l7wjUGX3xq1eeRfM4RSJGBIV1D -z1gKPkfWag3mVBVvva1RlX5bAJTPR/2YqNtw/FkIYYEi/pIZiAdpcxpoHtmnYYoZtvhUftzdx5ZX -JSKDW405zkGcZzzGZ6KEv4FI224oDmijlEf+xp6MJK5ojY/ASeVUR+wsKRuJ+XFZ5o7ZeEime8t1 -ouUsU6YjF5ZtUihhkGfCdFQLWQFJ3UXxmElfhQnR+eCdcDbkFZp6vTRmj56ApCihn5QGpaToNZmR -n3NVSpZcQpZ2KEONusaiCsKAug0wkQbJSFO+PTSjneGxOuFjPlUk3ovWvdIerjUg9ZGIOtGq/qeX -eCYrrCX+1UPsgTKGGRSbzd5q156d/gpfbJxe66eD5iQKrXj7RGgruGxs62qebBHUKS32CKluCiqZ -qh+pmehmEb71noAUoe5e9Zm17S7773V10pjrtG4CmuurCV/n6zLK5turWNhqOvFXbjhZrMD0YhKe -wR0zOyuvsh6MWrGoIuzvyWu5y1WIFAqmJselypxXh6dKLNOKEB98L88bS2rkNqqlKzCNJp9c0G0j -Gzh0iRrCbHSXmPR643QS+4rWhgFmnSbSuXCjS0xAOWkU2UdLqyuUNfHSFdUouy3bm5i5GnDM3tG8 -doJ4r5tqu3pPbRSVfvs8uJzeNXhp3n4j/tZ42SwH7eaWUUOjc3qFV9453UHTXZfcLH+OeNs5g36x -lBnHvTm7EbMbLeuaLncao8vWCXimfo1o+843Ak6y4ChNeGntvAYvfLK4ezmoyNIbNCLTCXO9ZV3A -E8/s88RczPzDwI4Ob7XZyl7+9Miban29h+tJZPrE21wgvBphDfrrfPdCTPKJD/y98L1rZwHcV6Jq -Zab0metpuNIX/qAFPoz171WUaUb4HAhBSzHuHfjzHb3kha/2Cctis/ORArVHNYfFyYRH2pYIRzic -isVOfPWD1b6mRTqpCRBozzof6UZVvFXRxWIr3GGrEviGYgyPMfahheiSaLs/9QeFu7oZ/ndSY8DD -ya9x+uPed+7mxN2IzIISBOMLFYWVqC3Pew1T2nFuuCiwZS5/v6II10i4t1OJcUH2U9zxKodHsGGv -Oa+zkvNUYUOa/TCCRutF9MzDwdlUMJADTCGSbDQ5OV4PTamDoPEi6Ecc/RF5RWwkcdSXvSOaDWSn -I9LlvubFTQpuc6JKXLcKeb+xdbKRBnwREemXyjg6ME65aJiOuBgrktzykfPLJBKR9ClMavJ62/Ff -BlNIyod9yX9wcSXexnXFpvkrbXk64xsx5Db7wXKP5fSgsvwIMM/9631VLBfkmtbHRXpqmtei52hG -pUwSlo+BASQoeILDOBgREECxBBh5/iYmNsQ9dIv5+OI++QkqdsJPc3uykz5fkM+OraeekcQF7X4n -B5S67za5U967PmooGQhUXfF7afXyCD7ONdRe17QogYjVx38uLwtrS6nhTnm15LQUnu9E2uK6CNI/ -1HOABj0ESwOjut4FEpFQpdNAm4K2LHnDWHNcmKB2ioKBogysVZtMO2nSxUdZ8Yk2kJc7URioLVI0 -YgmtIwZj4LoeKemgnOnbUdGnzZ4Oa6scqiolBGqS6RgWNLu0RMhcaE6rhhU4hiuqFXPAG8fGwTPW -FKeLMtdVmXLSs5YJGF/YeVm7rREMlY3UYE+yCxbaMXX8y15m5zVHq6GOKDMynzII/jdUHdyVqIy0 -ifX2+r/EgtZcvRzSb72gU9ui87M2VecjKildW/aFqaYhKoryUjfB/g4qtyVuc60xFDGmCxwjW+qu -zjuwl2GkOWn66+3QiiEctvd04OVvcCVzjgT7lrkvjVGKKHmmlDUKowSeikb5kK/mJReuWOxONx+s -ULsl+Lqb0CVn0SrVyJ6wt4t6yTeSCafhPhAf0OXn6L60UMxiLolFAtmN35S2Ob1lZpQ1r/n0Qb5D -oQ1zJiRVDgF8N3Q8TYfbi3DyWCy3lT1nxyBs6FT3S2GOzWRlxwKvlRP0RPJA9SjxEy0UoEnkA+M4 -cnzLMJrBGWLFEaaUb5lvpqbq/loOaU5+DFuHPxo82/OZuM8FXG3oVNZhtWpMpb/0Xu5m/LfLhHZQ -7yuVI0MqZ7NE43imC8jH3IwGZlbPm0xkJYs7+2U48hXTsFSMqgGDvai0kLxyynKNT/waj+q1c1tz -GjOpPBgdCSq3UKZxCSsqFIY+O6JbAWGWcV1pwqLyj5sGqCF1xb1F3varUWqrJv6cN3PrUXzijtfZ -FshpBL3Xwr4GIPvU2N8EjrJgS1zl21rbXQMXeXc5jjFyrhpCzijSv/RQtyPSzHCFMhlME95fHglt -pRsX+dfSQjUeHAlpWzJ5iOo79Ldnaxai6bXTcGO3fp07ri7HLEmXXPlYi8bv/qVxvNcdra6m7Rlb -6JBTb5fd66VhFRjGArh2n7R1rDW4P5NOT9K0I183T2scYkeZ3q/VFyLb09U9ajzXBS8Kgkhc4mBS -kYY9cy3Vy9lUnuNJH8HGIclUilwnBtjUOH0gteGOZ4c/XNrhXLSYDyxfnD8z1pDy7rYRvDolhnbe -UMzxCZUs40s6s7UIvBnLgc0+vKuOkIXeOrDymlp+Zxra4MZLBbVrqD/jTJ597pDmnw5c4+DbyB88 -9Cg9DodYcSuMZT/114pptqc/EuTjRPvH/z5slzI3tluOEBBLqOXLOX+0I5929tO97wkvl/atCz+y -xJrdwteW2FNW/NSmBP+f/maYtVs/bYyBC7Ox3jsYZHL05CIrBa/nS+b3bHfiYm4Ueil1YZZSgAUI -fFZ1dxUmeA2oQRQ3RuGXNGLFV9/XbGFGPV6kfzk1TBBCd+izc7q1H+OHMJwmaBX2IQNYVAKHYepV -SSGCe6CnbYHHETKGNe43EDvFgZr0gB/nVHPHZ80VV1ojOiI3XDvYIkl4ayo4bxQIgrFXWTvBI0nH -VElWMuw2aLUWCRHHf8ymVCHjFlJnOSojfevCYyyyZDH0IcvHhrsnQ5O1OsWzONuVVKIxSxiFZ/tR -fKDAf6xFTnw4O9Qig2VCfW2hJQrmMOuHW0W3dLQmCMO2ccdUd/xyfflH/olTiHZVdGwb8nIwRzSE -J15jFlOJuBZBZ4CiyHyd2IFylFlB+HgHhYabhWOGwYO1ZH/Og1dtQlFMk352CGRSIFTapnWQEUtN -l4zv8S0aaCFDyGCBqDUxZYpxGHX01y/JuH1xhn7TOCnNCI4eKDs5WGX4R425F4vF1o3BJ4vO0otq -I3rimI7jJY1jISqnBxknCIvruF83mF5wN4X7qGLIhR8A2Vg0yFERSIXn9Vv3GHy3Vj/WIkKddlYi -yIMv2I/VMjTLpW7pt05SWIZR0RPyxpB4SIUM9lBPGBl0GC7oSEEwRYLe4pJpZY2P0zbI1n+Oc44w -qY3PUnmF0ixjVpDD/mJ9wpOBGTVgXlaCaZiPcIWK5NiKBIiPdGaQ0TWGvAiG7nMchdZb7Vgf8zNi -MuMyzRdy/lePe9iC4TRx7WhhOQI/QiSVNAmAa2lT/piFbuh7ofJoYSZzrSZ1bvmWw3eN2nKUPVky -uPN5/VRfohRd0VYZoqhKIlU6TXYhJxmPUIloAwc1bPmHEpaZYZORHNlXUJM07hATwHR8MJYqkwWR -WaIezFhxSFlc8/Fq82hEnpeRozg3ULhhr9lAGtVEkCg5ZNRuuVleBPaZadhG0ZgkyPmDOTOKzViM -YgOcpukKqQcbjAWS0IleQ2ROjdh6A+md1qWdBRSX7iSYgFRTtRmBpJioieXJiHfJiMGIR9fJOn8I -MSfXYhspn4ooSa2mSAj4n+8Bmg03fBJZoPOJgsVZRxu1oOMRPXYYjdqjihFaEoZpXBREanuJoRI6 -cibFinq4ngUKh/wQd/H5ofYCZ0HJXR62opZFaAT0iFIZo4DIiUojkjeqKiuoZirKo5Y1a7AWckGa -BkuYoD5lpDK6eUs6CkDqpETwl1EqpfhJpVeKpVl6EgUAADs= - ---BOUNDARY-- diff --git a/lib-python/modified-2.7/email/test/data/msg_08.txt b/lib-python/modified-2.7/email/test/data/msg_08.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_08.txt +++ /dev/null @@ -1,24 +0,0 @@ -MIME-Version: 1.0 -From: Barry Warsaw <barry at zope.com> -To: Dingus Lovers <cravindogs at cravindogs.com> -Subject: Lyrics -Date: Fri, 20 Apr 2001 19:35:02 -0400 -Content-Type: multipart/mixed; boundary="BOUNDARY" - ---BOUNDARY -Content-Type: text/plain; charset="us-ascii" - - ---BOUNDARY -Content-Type: text/html; charset="iso-8859-1" - - ---BOUNDARY -Content-Type: text/plain; charset="iso-8859-2" - - ---BOUNDARY -Content-Type: text/plain; charset="koi8-r" - - ---BOUNDARY-- diff --git a/lib-python/modified-2.7/email/test/data/msg_09.txt b/lib-python/modified-2.7/email/test/data/msg_09.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_09.txt +++ /dev/null @@ -1,24 +0,0 @@ -MIME-Version: 1.0 -From: Barry Warsaw <barry at zope.com> -To: Dingus Lovers <cravindogs at cravindogs.com> -Subject: Lyrics -Date: Fri, 20 Apr 2001 19:35:02 -0400 -Content-Type: multipart/mixed; boundary="BOUNDARY" - ---BOUNDARY -Content-Type: text/plain; charset="us-ascii" - - ---BOUNDARY -Content-Type: text/html; charset="iso-8859-1" - - ---BOUNDARY -Content-Type: text/plain - - ---BOUNDARY -Content-Type: text/plain; charset="koi8-r" - - ---BOUNDARY-- diff --git a/lib-python/modified-2.7/email/test/data/msg_10.txt b/lib-python/modified-2.7/email/test/data/msg_10.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_10.txt +++ /dev/null @@ -1,39 +0,0 @@ -MIME-Version: 1.0 -From: Barry Warsaw <barry at zope.com> -To: Dingus Lovers <cravindogs at cravindogs.com> -Subject: Lyrics -Date: Fri, 20 Apr 2001 19:35:02 -0400 -Content-Type: multipart/mixed; boundary="BOUNDARY" - ---BOUNDARY -Content-Type: text/plain; charset="us-ascii" -Content-Transfer-Encoding: 7bit - -This is a 7bit encoded message. - ---BOUNDARY -Content-Type: text/html; charset="iso-8859-1" -Content-Transfer-Encoding: Quoted-Printable - -=A1This is a Quoted Printable encoded message! - ---BOUNDARY -Content-Type: text/plain; charset="iso-8859-1" -Content-Transfer-Encoding: Base64 - -VGhpcyBpcyBhIEJhc2U2NCBlbmNvZGVkIG1lc3NhZ2Uu - - ---BOUNDARY -Content-Type: text/plain; charset="iso-8859-1" -Content-Transfer-Encoding: Base64 - -VGhpcyBpcyBhIEJhc2U2NCBlbmNvZGVkIG1lc3NhZ2UuCg== - - ---BOUNDARY -Content-Type: text/plain; charset="iso-8859-1" - -This has no Content-Transfer-Encoding: header. - ---BOUNDARY-- diff --git a/lib-python/modified-2.7/email/test/data/msg_11.txt b/lib-python/modified-2.7/email/test/data/msg_11.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_11.txt +++ /dev/null @@ -1,7 +0,0 @@ -Content-Type: message/rfc822 -MIME-Version: 1.0 -Subject: The enclosing message - -Subject: An enclosed message - -Here is the body of the message. diff --git a/lib-python/modified-2.7/email/test/data/msg_12.txt b/lib-python/modified-2.7/email/test/data/msg_12.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_12.txt +++ /dev/null @@ -1,36 +0,0 @@ -MIME-Version: 1.0 -From: Barry Warsaw <barry at zope.com> -To: Dingus Lovers <cravindogs at cravindogs.com> -Subject: Lyrics -Date: Fri, 20 Apr 2001 19:35:02 -0400 -Content-Type: multipart/mixed; boundary="BOUNDARY" - ---BOUNDARY -Content-Type: text/plain; charset="us-ascii" - - ---BOUNDARY -Content-Type: text/html; charset="iso-8859-1" - - ---BOUNDARY -Content-Type: multipart/mixed; boundary="ANOTHER" - ---ANOTHER -Content-Type: text/plain; charset="iso-8859-2" - - ---ANOTHER -Content-Type: text/plain; charset="iso-8859-3" - ---ANOTHER-- - ---BOUNDARY -Content-Type: text/plain; charset="us-ascii" - - ---BOUNDARY -Content-Type: text/plain; charset="koi8-r" - - ---BOUNDARY-- diff --git a/lib-python/modified-2.7/email/test/data/msg_12a.txt b/lib-python/modified-2.7/email/test/data/msg_12a.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_12a.txt +++ /dev/null @@ -1,38 +0,0 @@ -MIME-Version: 1.0 -From: Barry Warsaw <barry at zope.com> -To: Dingus Lovers <cravindogs at cravindogs.com> -Subject: Lyrics -Date: Fri, 20 Apr 2001 19:35:02 -0400 -Content-Type: multipart/mixed; boundary="BOUNDARY" - ---BOUNDARY -Content-Type: text/plain; charset="us-ascii" - - ---BOUNDARY -Content-Type: text/html; charset="iso-8859-1" - - ---BOUNDARY -Content-Type: multipart/mixed; boundary="ANOTHER" - ---ANOTHER -Content-Type: text/plain; charset="iso-8859-2" - - ---ANOTHER -Content-Type: text/plain; charset="iso-8859-3" - - ---ANOTHER-- - - ---BOUNDARY -Content-Type: text/plain; charset="us-ascii" - - ---BOUNDARY -Content-Type: text/plain; charset="koi8-r" - - ---BOUNDARY-- diff --git a/lib-python/modified-2.7/email/test/data/msg_13.txt b/lib-python/modified-2.7/email/test/data/msg_13.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_13.txt +++ /dev/null @@ -1,94 +0,0 @@ -MIME-Version: 1.0 -From: Barry <barry at digicool.com> -To: Dingus Lovers <cravindogs at cravindogs.com> -Subject: Here is your dingus fish -Date: Fri, 20 Apr 2001 19:35:02 -0400 -Content-Type: multipart/mixed; boundary="OUTER" - ---OUTER -Content-Type: text/plain; charset="us-ascii" - -A text/plain part - ---OUTER -Content-Type: multipart/mixed; boundary=BOUNDARY - - ---BOUNDARY -Content-Type: text/plain; charset="us-ascii" - -Hi there, - -This is the dingus fish. - ---BOUNDARY -Content-Type: image/gif; name="dingusfish.gif" -Content-Transfer-Encoding: base64 -content-disposition: attachment; filename="dingusfish.gif" - -R0lGODdhAAEAAfAAAP///wAAACwAAAAAAAEAAQAC/oSPqcvtD6OctNqLs968+w+G4kiW5omm6sq2 -7gvH8kzX9o3n+s73/g8MCofEovGITGICTKbyCV0FDNOo9SqpQqpOrJfXzTQj2vD3TGtqL+NtGQ2f -qTXmxzuOd7WXdcc9DyjU53ewFni4s0fGhdiYaEhGBelICTNoV1j5NUnFcrmUqemjNifJVWpaOqaI -oFq3SspZsSraE7sHq3jr1MZqWvi662vxV4tD+pvKW6aLDOCLyur8PDwbanyDeq0N3DctbQYeLDvR -RY6t95m6UB0d3mwIrV7e2VGNvjjffukeJp4w7F65KecGFsTHQGAygOrgrWs1jt28Rc88KESYcGLA -/obvTkH6p+CinWJiJmIMqXGQwH/y4qk0SYjgQTczT3ajKZGfuI0uJ4kkVI/DT5s3/ejkxI0aT4Y+ -YTYgWbImUaXk9nlLmnSh1qJiJFl0OpUqRK4oOy7NyRQtHWofhoYVxkwWXKUSn0YsS+fUV6lhqfYb -6ayd3Z5qQdG1B7bvQzaJjwUV2lixMUZ7JVsOlfjWVr/3NB/uFvnySBN6Dcb6rGwaRM3wsormw5cC -M9NxWy/bWdufudCvy8bOAjXjVVwta/uO21sE5RHBCzNFXtgq9ORtH4eYjVP4Yryo026nvkFmCeyA -B29efV6ravCMK5JwWd5897Qrx7ll38o6iHDZ/rXPR//feevhF4l7wjUGX3xq1eeRfM4RSJGBIV1D -z1gKPkfWag3mVBVvva1RlX5bAJTPR/2YqNtw/FkIYYEi/pIZiAdpcxpoHtmnYYoZtvhUftzdx5ZX -JSKDW405zkGcZzzGZ6KEv4FI224oDmijlEf+xp6MJK5ojY/ASeVUR+wsKRuJ+XFZ5o7ZeEime8t1 -ouUsU6YjF5ZtUihhkGfCdFQLWQFJ3UXxmElfhQnR+eCdcDbkFZp6vTRmj56ApCihn5QGpaToNZmR -n3NVSpZcQpZ2KEONusaiCsKAug0wkQbJSFO+PTSjneGxOuFjPlUk3ovWvdIerjUg9ZGIOtGq/qeX -eCYrrCX+1UPsgTKGGRSbzd5q156d/gpfbJxe66eD5iQKrXj7RGgruGxs62qebBHUKS32CKluCiqZ -qh+pmehmEb71noAUoe5e9Zm17S7773V10pjrtG4CmuurCV/n6zLK5turWNhqOvFXbjhZrMD0YhKe -wR0zOyuvsh6MWrGoIuzvyWu5y1WIFAqmJselypxXh6dKLNOKEB98L88bS2rkNqqlKzCNJp9c0G0j -Gzh0iRrCbHSXmPR643QS+4rWhgFmnSbSuXCjS0xAOWkU2UdLqyuUNfHSFdUouy3bm5i5GnDM3tG8 -doJ4r5tqu3pPbRSVfvs8uJzeNXhp3n4j/tZ42SwH7eaWUUOjc3qFV9453UHTXZfcLH+OeNs5g36x -lBnHvTm7EbMbLeuaLncao8vWCXimfo1o+843Ak6y4ChNeGntvAYvfLK4ezmoyNIbNCLTCXO9ZV3A -E8/s88RczPzDwI4Ob7XZyl7+9Miban29h+tJZPrE21wgvBphDfrrfPdCTPKJD/y98L1rZwHcV6Jq -Zab0metpuNIX/qAFPoz171WUaUb4HAhBSzHuHfjzHb3kha/2Cctis/ORArVHNYfFyYRH2pYIRzic -isVOfPWD1b6mRTqpCRBozzof6UZVvFXRxWIr3GGrEviGYgyPMfahheiSaLs/9QeFu7oZ/ndSY8DD -ya9x+uPed+7mxN2IzIISBOMLFYWVqC3Pew1T2nFuuCiwZS5/v6II10i4t1OJcUH2U9zxKodHsGGv -Oa+zkvNUYUOa/TCCRutF9MzDwdlUMJADTCGSbDQ5OV4PTamDoPEi6Ecc/RF5RWwkcdSXvSOaDWSn -I9LlvubFTQpuc6JKXLcKeb+xdbKRBnwREemXyjg6ME65aJiOuBgrktzykfPLJBKR9ClMavJ62/Ff -BlNIyod9yX9wcSXexnXFpvkrbXk64xsx5Db7wXKP5fSgsvwIMM/9631VLBfkmtbHRXpqmtei52hG -pUwSlo+BASQoeILDOBgREECxBBh5/iYmNsQ9dIv5+OI++QkqdsJPc3uykz5fkM+OraeekcQF7X4n -B5S67za5U967PmooGQhUXfF7afXyCD7ONdRe17QogYjVx38uLwtrS6nhTnm15LQUnu9E2uK6CNI/ -1HOABj0ESwOjut4FEpFQpdNAm4K2LHnDWHNcmKB2ioKBogysVZtMO2nSxUdZ8Yk2kJc7URioLVI0 -YgmtIwZj4LoeKemgnOnbUdGnzZ4Oa6scqiolBGqS6RgWNLu0RMhcaE6rhhU4hiuqFXPAG8fGwTPW -FKeLMtdVmXLSs5YJGF/YeVm7rREMlY3UYE+yCxbaMXX8y15m5zVHq6GOKDMynzII/jdUHdyVqIy0 -ifX2+r/EgtZcvRzSb72gU9ui87M2VecjKildW/aFqaYhKoryUjfB/g4qtyVuc60xFDGmCxwjW+qu -zjuwl2GkOWn66+3QiiEctvd04OVvcCVzjgT7lrkvjVGKKHmmlDUKowSeikb5kK/mJReuWOxONx+s -ULsl+Lqb0CVn0SrVyJ6wt4t6yTeSCafhPhAf0OXn6L60UMxiLolFAtmN35S2Ob1lZpQ1r/n0Qb5D -oQ1zJiRVDgF8N3Q8TYfbi3DyWCy3lT1nxyBs6FT3S2GOzWRlxwKvlRP0RPJA9SjxEy0UoEnkA+M4 -cnzLMJrBGWLFEaaUb5lvpqbq/loOaU5+DFuHPxo82/OZuM8FXG3oVNZhtWpMpb/0Xu5m/LfLhHZQ -7yuVI0MqZ7NE43imC8jH3IwGZlbPm0xkJYs7+2U48hXTsFSMqgGDvai0kLxyynKNT/waj+q1c1tz -GjOpPBgdCSq3UKZxCSsqFIY+O6JbAWGWcV1pwqLyj5sGqCF1xb1F3varUWqrJv6cN3PrUXzijtfZ -FshpBL3Xwr4GIPvU2N8EjrJgS1zl21rbXQMXeXc5jjFyrhpCzijSv/RQtyPSzHCFMhlME95fHglt -pRsX+dfSQjUeHAlpWzJ5iOo79Ldnaxai6bXTcGO3fp07ri7HLEmXXPlYi8bv/qVxvNcdra6m7Rlb -6JBTb5fd66VhFRjGArh2n7R1rDW4P5NOT9K0I183T2scYkeZ3q/VFyLb09U9ajzXBS8Kgkhc4mBS -kYY9cy3Vy9lUnuNJH8HGIclUilwnBtjUOH0gteGOZ4c/XNrhXLSYDyxfnD8z1pDy7rYRvDolhnbe -UMzxCZUs40s6s7UIvBnLgc0+vKuOkIXeOrDymlp+Zxra4MZLBbVrqD/jTJ597pDmnw5c4+DbyB88 -9Cg9DodYcSuMZT/114pptqc/EuTjRPvH/z5slzI3tluOEBBLqOXLOX+0I5929tO97wkvl/atCz+y -xJrdwteW2FNW/NSmBP+f/maYtVs/bYyBC7Ox3jsYZHL05CIrBa/nS+b3bHfiYm4Ueil1YZZSgAUI -fFZ1dxUmeA2oQRQ3RuGXNGLFV9/XbGFGPV6kfzk1TBBCd+izc7q1H+OHMJwmaBX2IQNYVAKHYepV -SSGCe6CnbYHHETKGNe43EDvFgZr0gB/nVHPHZ80VV1ojOiI3XDvYIkl4ayo4bxQIgrFXWTvBI0nH -VElWMuw2aLUWCRHHf8ymVCHjFlJnOSojfevCYyyyZDH0IcvHhrsnQ5O1OsWzONuVVKIxSxiFZ/tR -fKDAf6xFTnw4O9Qig2VCfW2hJQrmMOuHW0W3dLQmCMO2ccdUd/xyfflH/olTiHZVdGwb8nIwRzSE -J15jFlOJuBZBZ4CiyHyd2IFylFlB+HgHhYabhWOGwYO1ZH/Og1dtQlFMk352CGRSIFTapnWQEUtN -l4zv8S0aaCFDyGCBqDUxZYpxGHX01y/JuH1xhn7TOCnNCI4eKDs5WGX4R425F4vF1o3BJ4vO0otq -I3rimI7jJY1jISqnBxknCIvruF83mF5wN4X7qGLIhR8A2Vg0yFERSIXn9Vv3GHy3Vj/WIkKddlYi -yIMv2I/VMjTLpW7pt05SWIZR0RPyxpB4SIUM9lBPGBl0GC7oSEEwRYLe4pJpZY2P0zbI1n+Oc44w -qY3PUnmF0ixjVpDD/mJ9wpOBGTVgXlaCaZiPcIWK5NiKBIiPdGaQ0TWGvAiG7nMchdZb7Vgf8zNi -MuMyzRdy/lePe9iC4TRx7WhhOQI/QiSVNAmAa2lT/piFbuh7ofJoYSZzrSZ1bvmWw3eN2nKUPVky -uPN5/VRfohRd0VYZoqhKIlU6TXYhJxmPUIloAwc1bPmHEpaZYZORHNlXUJM07hATwHR8MJYqkwWR -WaIezFhxSFlc8/Fq82hEnpeRozg3ULhhr9lAGtVEkCg5ZNRuuVleBPaZadhG0ZgkyPmDOTOKzViM -YgOcpukKqQcbjAWS0IleQ2ROjdh6A+md1qWdBRSX7iSYgFRTtRmBpJioieXJiHfJiMGIR9fJOn8I -MSfXYhspn4ooSa2mSAj4n+8Bmg03fBJZoPOJgsVZRxu1oOMRPXYYjdqjihFaEoZpXBREanuJoRI6 -cibFinq4ngUKh/wQd/H5ofYCZ0HJXR62opZFaAT0iFIZo4DIiUojkjeqKiuoZirKo5Y1a7AWckGa -BkuYoD5lpDK6eUs6CkDqpETwl1EqpfhJpVeKpVl6EgUAADs= - ---BOUNDARY-- - ---OUTER-- diff --git a/lib-python/modified-2.7/email/test/data/msg_14.txt b/lib-python/modified-2.7/email/test/data/msg_14.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_14.txt +++ /dev/null @@ -1,23 +0,0 @@ -Return-Path: <bbb at zzz.org> -Delivered-To: bbb at zzz.org -Received: by mail.zzz.org (Postfix, from userid 889) - id 27CEAD38CC; Fri, 4 May 2001 14:05:44 -0400 (EDT) -MIME-Version: 1.0 -Content-Type: text; charset=us-ascii -Content-Transfer-Encoding: 7bit -Message-ID: <15090.61304.110929.45684 at aaa.zzz.org> -From: bbb at ddd.com (John X. Doe) -To: bbb at zzz.org -Subject: This is a test message -Date: Fri, 4 May 2001 14:05:44 -0400 - - -Hi, - -I'm sorry but I'm using a drainbread ISP, which although big and -wealthy can't seem to generate standard compliant email. :( - -This message has a Content-Type: header with no subtype. I hope you -can still read it. - --Me diff --git a/lib-python/modified-2.7/email/test/data/msg_15.txt b/lib-python/modified-2.7/email/test/data/msg_15.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_15.txt +++ /dev/null @@ -1,52 +0,0 @@ -Return-Path: <xx at xx.dk> -Received: from fepD.post.tele.dk (195.41.46.149) by mail.groupcare.dk (LSMTP for Windows NT v1.1b) with SMTP id <0.0014F8A2 at mail.groupcare.dk>; Mon, 30 Apr 2001 12:17:50 +0200 -User-Agent: Microsoft-Outlook-Express-Macintosh-Edition/5.02.2106 -Subject: XX -From: xx at xx.dk -To: XX -Message-ID: <xxxx> -Mime-version: 1.0 -Content-type: multipart/mixed; - boundary="MS_Mac_OE_3071477847_720252_MIME_Part" - -> Denne meddelelse er i MIME-format. Da dit postl�sningsprogram ikke forst�r dette format, kan del af eller hele meddelelsen v�re ul�selig. - ---MS_Mac_OE_3071477847_720252_MIME_Part -Content-type: multipart/alternative; - boundary="MS_Mac_OE_3071477847_720252_MIME_Part" - - ---MS_Mac_OE_3071477847_720252_MIME_Part -Content-type: text/plain; charset="ISO-8859-1" -Content-transfer-encoding: quoted-printable - -Some removed test. - ---MS_Mac_OE_3071477847_720252_MIME_Part -Content-type: text/html; charset="ISO-8859-1" -Content-transfer-encoding: quoted-printable - -<HTML> -<HEAD> -<TITLE>Some removed HTML</TITLE> -</HEAD> -<BODY> -Some removed text. -</BODY> -</HTML> - - ---MS_Mac_OE_3071477847_720252_MIME_Part-- - - ---MS_Mac_OE_3071477847_720252_MIME_Part -Content-type: image/gif; name="xx.gif"; - x-mac-creator="6F676C65"; - x-mac-type="47494666" -Content-disposition: attachment -Content-transfer-encoding: base64 - -Some removed base64 encoded chars. - ---MS_Mac_OE_3071477847_720252_MIME_Part-- - diff --git a/lib-python/modified-2.7/email/test/data/msg_16.txt b/lib-python/modified-2.7/email/test/data/msg_16.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_16.txt +++ /dev/null @@ -1,123 +0,0 @@ -Return-Path: <> -Delivered-To: scr-admin at socal-raves.org -Received: from cougar.noc.ucla.edu (cougar.noc.ucla.edu [169.232.10.18]) - by babylon.socal-raves.org (Postfix) with ESMTP id CCC2C51B84 - for <scr-admin at socal-raves.org>; Sun, 23 Sep 2001 20:13:54 -0700 (PDT) -Received: from sims-ms-daemon by cougar.noc.ucla.edu - (Sun Internet Mail Server sims.3.5.2000.03.23.18.03.p10) - id <0GK500B01D0B8Y at cougar.noc.ucla.edu> for scr-admin at socal-raves.org; Sun, - 23 Sep 2001 20:14:35 -0700 (PDT) -Received: from cougar.noc.ucla.edu - (Sun Internet Mail Server sims.3.5.2000.03.23.18.03.p10) - id <0GK500B01D0B8X at cougar.noc.ucla.edu>; Sun, 23 Sep 2001 20:14:35 -0700 (PDT) -Date: Sun, 23 Sep 2001 20:14:35 -0700 (PDT) -From: Internet Mail Delivery <postmaster at ucla.edu> -Subject: Delivery Notification: Delivery has failed -To: scr-admin at socal-raves.org -Message-id: <0GK500B04D0B8X at cougar.noc.ucla.edu> -MIME-version: 1.0 -Sender: scr-owner at socal-raves.org -Errors-To: scr-owner at socal-raves.org -X-BeenThere: scr at socal-raves.org -X-Mailman-Version: 2.1a3 -Precedence: bulk -List-Help: <mailto:scr-request at socal-raves.org?subject=help> -List-Post: <mailto:scr at socal-raves.org> -List-Subscribe: <http://socal-raves.org/mailman/listinfo/scr>, - <mailto:scr-request at socal-raves.org?subject=subscribe> -List-Id: SoCal-Raves <scr.socal-raves.org> -List-Unsubscribe: <http://socal-raves.org/mailman/listinfo/scr>, - <mailto:scr-request at socal-raves.org?subject=unsubscribe> -List-Archive: <http://socal-raves.org/mailman/private/scr/> -Content-Type: multipart/report; boundary="Boundary_(ID_PGS2F2a+z+/jL7hupKgRhA)" - - ---Boundary_(ID_PGS2F2a+z+/jL7hupKgRhA) -Content-type: text/plain; charset=ISO-8859-1 - -This report relates to a message you sent with the following header fields: - - Message-id: <002001c144a6$8752e060$56104586 at oxy.edu> - Date: Sun, 23 Sep 2001 20:10:55 -0700 - From: "Ian T. Henry" <henryi at oxy.edu> - To: SoCal Raves <scr at socal-raves.org> - Subject: [scr] yeah for Ians!! - -Your message cannot be delivered to the following recipients: - - Recipient address: jangel1 at cougar.noc.ucla.edu - Reason: recipient reached disk quota - - ---Boundary_(ID_PGS2F2a+z+/jL7hupKgRhA) -Content-type: message/DELIVERY-STATUS - -Original-envelope-id: 0GK500B4HD0888 at cougar.noc.ucla.edu -Reporting-MTA: dns; cougar.noc.ucla.edu - -Action: failed -Status: 5.0.0 (recipient reached disk quota) -Original-recipient: rfc822;jangel1 at cougar.noc.ucla.edu -Final-recipient: rfc822;jangel1 at cougar.noc.ucla.edu - ---Boundary_(ID_PGS2F2a+z+/jL7hupKgRhA) -Content-type: MESSAGE/RFC822 - -Return-path: scr-admin at socal-raves.org -Received: from sims-ms-daemon by cougar.noc.ucla.edu - (Sun Internet Mail Server sims.3.5.2000.03.23.18.03.p10) - id <0GK500B01D0B8X at cougar.noc.ucla.edu>; Sun, 23 Sep 2001 20:14:35 -0700 (PDT) -Received: from panther.noc.ucla.edu by cougar.noc.ucla.edu - (Sun Internet Mail Server sims.3.5.2000.03.23.18.03.p10) - with ESMTP id <0GK500B4GD0888 at cougar.noc.ucla.edu> for jangel1 at sims-ms-daemon; - Sun, 23 Sep 2001 20:14:33 -0700 (PDT) -Received: from babylon.socal-raves.org - (ip-209-85-222-117.dreamhost.com [209.85.222.117]) - by panther.noc.ucla.edu (8.9.1a/8.9.1) with ESMTP id UAA09793 for - <jangel1 at ucla.edu>; Sun, 23 Sep 2001 20:14:32 -0700 (PDT) -Received: from babylon (localhost [127.0.0.1]) by babylon.socal-raves.org - (Postfix) with ESMTP id D3B2951B70; Sun, 23 Sep 2001 20:13:47 -0700 (PDT) -Received: by babylon.socal-raves.org (Postfix, from userid 60001) - id A611F51B82; Sun, 23 Sep 2001 20:13:46 -0700 (PDT) -Received: from tiger.cc.oxy.edu (tiger.cc.oxy.edu [134.69.3.112]) - by babylon.socal-raves.org (Postfix) with ESMTP id ADA7351B70 for - <scr at socal-raves.org>; Sun, 23 Sep 2001 20:13:44 -0700 (PDT) -Received: from ent (n16h86.dhcp.oxy.edu [134.69.16.86]) - by tiger.cc.oxy.edu (8.8.8/8.8.8) with SMTP id UAA08100 for - <scr at socal-raves.org>; Sun, 23 Sep 2001 20:14:24 -0700 (PDT) -Date: Sun, 23 Sep 2001 20:10:55 -0700 -From: "Ian T. Henry" <henryi at oxy.edu> -Subject: [scr] yeah for Ians!! -Sender: scr-admin at socal-raves.org -To: SoCal Raves <scr at socal-raves.org> -Errors-to: scr-admin at socal-raves.org -Message-id: <002001c144a6$8752e060$56104586 at oxy.edu> -MIME-version: 1.0 -X-Mailer: Microsoft Outlook Express 5.50.4522.1200 -Content-type: text/plain; charset=us-ascii -Precedence: bulk -Delivered-to: scr-post at babylon.socal-raves.org -Delivered-to: scr at socal-raves.org -X-Converted-To-Plain-Text: from multipart/alternative by demime 0.98e -X-Converted-To-Plain-Text: Alternative section used was text/plain -X-BeenThere: scr at socal-raves.org -X-Mailman-Version: 2.1a3 -List-Help: <mailto:scr-request at socal-raves.org?subject=help> -List-Post: <mailto:scr at socal-raves.org> -List-Subscribe: <http://socal-raves.org/mailman/listinfo/scr>, - <mailto:scr-request at socal-raves.org?subject=subscribe> -List-Id: SoCal-Raves <scr.socal-raves.org> -List-Unsubscribe: <http://socal-raves.org/mailman/listinfo/scr>, - <mailto:scr-request at socal-raves.org?subject=unsubscribe> -List-Archive: <http://socal-raves.org/mailman/private/scr/> - -I always love to find more Ian's that are over 3 years old!! - -Ian -_______________________________________________ -For event info, list questions, or to unsubscribe, see http://www.socal-raves.org/ - - - ---Boundary_(ID_PGS2F2a+z+/jL7hupKgRhA)-- - diff --git a/lib-python/modified-2.7/email/test/data/msg_17.txt b/lib-python/modified-2.7/email/test/data/msg_17.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_17.txt +++ /dev/null @@ -1,12 +0,0 @@ -MIME-Version: 1.0 -From: Barry <barry at digicool.com> -To: Dingus Lovers <cravindogs at cravindogs.com> -Subject: Here is your dingus fish -Date: Fri, 20 Apr 2001 19:35:02 -0400 -Content-Type: multipart/mixed; boundary="BOUNDARY" - -Hi there, - -This is the dingus fish. - -[Non-text (image/gif) part of message omitted, filename dingusfish.gif] diff --git a/lib-python/modified-2.7/email/test/data/msg_18.txt b/lib-python/modified-2.7/email/test/data/msg_18.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_18.txt +++ /dev/null @@ -1,6 +0,0 @@ -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit -X-Foobar-Spoink-Defrobnit: wasnipoop; giraffes="very-long-necked-animals"; - spooge="yummy"; hippos="gargantuan"; marshmallows="gooey" - diff --git a/lib-python/modified-2.7/email/test/data/msg_19.txt b/lib-python/modified-2.7/email/test/data/msg_19.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_19.txt +++ /dev/null @@ -1,43 +0,0 @@ -Send Ppp mailing list submissions to - ppp at zzz.org - -To subscribe or unsubscribe via the World Wide Web, visit - http://www.zzz.org/mailman/listinfo/ppp -or, via email, send a message with subject or body 'help' to - ppp-request at zzz.org - -You can reach the person managing the list at - ppp-admin at zzz.org - -When replying, please edit your Subject line so it is more specific -than "Re: Contents of Ppp digest..." - -Today's Topics: - - 1. testing #1 (Barry A. Warsaw) - 2. testing #2 (Barry A. Warsaw) - 3. testing #3 (Barry A. Warsaw) - 4. testing #4 (Barry A. Warsaw) - 5. testing #5 (Barry A. Warsaw) - -hello - - -hello - - -hello - - -hello - - -hello - - - -_______________________________________________ -Ppp mailing list -Ppp at zzz.org -http://www.zzz.org/mailman/listinfo/ppp - diff --git a/lib-python/modified-2.7/email/test/data/msg_20.txt b/lib-python/modified-2.7/email/test/data/msg_20.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_20.txt +++ /dev/null @@ -1,22 +0,0 @@ -Return-Path: <bbb at zzz.org> -Delivered-To: bbb at zzz.org -Received: by mail.zzz.org (Postfix, from userid 889) - id 27CEAD38CC; Fri, 4 May 2001 14:05:44 -0400 (EDT) -MIME-Version: 1.0 -Content-Type: text/plain; charset=us-ascii -Content-Transfer-Encoding: 7bit -Message-ID: <15090.61304.110929.45684 at aaa.zzz.org> -From: bbb at ddd.com (John X. Doe) -To: bbb at zzz.org -Cc: ccc at zzz.org -CC: ddd at zzz.org -cc: eee at zzz.org -Subject: This is a test message -Date: Fri, 4 May 2001 14:05:44 -0400 - - -Hi, - -Do you like this message? - --Me diff --git a/lib-python/modified-2.7/email/test/data/msg_21.txt b/lib-python/modified-2.7/email/test/data/msg_21.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_21.txt +++ /dev/null @@ -1,20 +0,0 @@ -From: aperson at dom.ain -To: bperson at dom.ain -Subject: Test -Content-Type: multipart/mixed; boundary="BOUNDARY" - -MIME message ---BOUNDARY -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit - -One ---BOUNDARY -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit - -Two ---BOUNDARY-- -End of MIME message diff --git a/lib-python/modified-2.7/email/test/data/msg_22.txt b/lib-python/modified-2.7/email/test/data/msg_22.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_22.txt +++ /dev/null @@ -1,46 +0,0 @@ -Mime-Version: 1.0 -Message-Id: <a05001902b7f1c33773e9@[134.84.183.138]> -Date: Tue, 16 Oct 2001 13:59:25 +0300 -To: a at example.com -From: b at example.com -Content-Type: multipart/mixed; boundary="============_-1208892523==_============" - ---============_-1208892523==_============ -Content-Type: text/plain; charset="us-ascii" ; format="flowed" - -Text text text. ---============_-1208892523==_============ -Content-Id: <a05001902b7f1c33773e9@[134.84.183.138].0.0> -Content-Type: image/jpeg; name="wibble.JPG" - ; x-mac-type="4A504547" - ; x-mac-creator="474B4F4E" -Content-Disposition: attachment; filename="wibble.JPG" -Content-Transfer-Encoding: base64 - -/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEB -AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/wAALCAXABIEBAREA -g6bCjjw/pIZSjO6FWFpldjySOmCNrO7DBZibUXhTwtCixw+GtAijVdqxxaPp0aKvmGXa -qrbBQvms0mAMeYS/3iTV1dG0hHaRNK01XblnWxtVdjkHLMIgTyqnk9VB7CrP2KzIINpa -4O7I+zxYO9WV8jZg71Zlb+8rMDkEirAVQFAUAKAFAAAUAYAUDgADgY6DjpRtXj5RxjHA -4wQRj0wQCMdCAewpaKKK/9k= ---============_-1208892523==_============ -Content-Id: <a05001902b7f1c33773e9@[134.84.183.138].0.1> -Content-Type: image/jpeg; name="wibble2.JPG" - ; x-mac-type="4A504547" - ; x-mac-creator="474B4F4E" -Content-Disposition: attachment; filename="wibble2.JPG" -Content-Transfer-Encoding: base64 - -/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEB -AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/wAALCAXABJ0BAREA -/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQA -W6NFJJBEkU10kKGTcWMDwxuU+0JHvk8qAtOpNwqSR0n8c3BlDyXHlqsUltHEiTvdXLxR -7vMiGDNJAJWkAMk8ZkCFp5G2oo5W++INrbQtNfTQxJAuXlupz9oS4d5Y1W+E2XlWZJJE -Y7LWYQxTLE1zuMbfBPxw8X2fibVdIbSbI6nLZxX635t9TjtYreWR7WGKJTLJFFKSlozO -0ShxIXM43uC3/9k= ---============_-1208892523==_============ -Content-Type: text/plain; charset="us-ascii" ; format="flowed" - -Text text text. ---============_-1208892523==_============-- - diff --git a/lib-python/modified-2.7/email/test/data/msg_23.txt b/lib-python/modified-2.7/email/test/data/msg_23.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_23.txt +++ /dev/null @@ -1,8 +0,0 @@ -From: aperson at dom.ain -Content-Type: multipart/mixed; boundary="BOUNDARY" - ---BOUNDARY -Content-Type: text/plain - -A message part ---BOUNDARY-- diff --git a/lib-python/modified-2.7/email/test/data/msg_24.txt b/lib-python/modified-2.7/email/test/data/msg_24.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_24.txt +++ /dev/null @@ -1,10 +0,0 @@ -Content-Type: multipart/mixed; boundary="BOUNDARY" -MIME-Version: 1.0 -Subject: A subject -To: aperson at dom.ain -From: bperson at dom.ain - ---BOUNDARY - - ---BOUNDARY-- diff --git a/lib-python/modified-2.7/email/test/data/msg_25.txt b/lib-python/modified-2.7/email/test/data/msg_25.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_25.txt +++ /dev/null @@ -1,117 +0,0 @@ -From MAILER-DAEMON Fri Apr 06 16:46:09 2001 -Received: from [204.245.199.98] (helo=zinfandel.lacita.com) - by www.linux.org.uk with esmtp (Exim 3.13 #1) - id 14lYR6-0008Iv-00 - for linuxuser-admin at www.linux.org.uk; Fri, 06 Apr 2001 16:46:09 +0100 -Received: from localhost (localhost) by zinfandel.lacita.com (8.7.3/8.6.10-MT4.00) with internal id JAB03225; Fri, 6 Apr 2001 09:23:06 -0800 (GMT-0800) -Date: Fri, 6 Apr 2001 09:23:06 -0800 (GMT-0800) -From: Mail Delivery Subsystem <MAILER-DAEMON at zinfandel.lacita.com> -Subject: Returned mail: Too many hops 19 (17 max): from <linuxuser-admin at www.linux.org.uk> via [199.164.235.226], to <scoffman at wellpartner.com> -Message-Id: <200104061723.JAB03225 at zinfandel.lacita.com> -To: <linuxuser-admin at www.linux.org.uk> -To: postmaster at zinfandel.lacita.com -MIME-Version: 1.0 -Content-Type: multipart/report; report-type=delivery-status; - bo -Auto-Submitted: auto-generated (failure) - -This is a MIME-encapsulated message - ---JAB03225.986577786/zinfandel.lacita.com - -The original message was received at Fri, 6 Apr 2001 09:23:03 -0800 (GMT-0800) -from [199.164.235.226] - - ----- The following addresses have delivery notifications ----- -<scoffman at wellpartner.com> (unrecoverable error) - - ----- Transcript of session follows ----- -554 Too many hops 19 (17 max): from <linuxuser-admin at www.linux.org.uk> via [199.164.235.226], to <scoffman at wellpartner.com> - ---JAB03225.986577786/zinfandel.lacita.com -Content-Type: message/delivery-status - -Reporting-MTA: dns; zinfandel.lacita.com -Received-From-MTA: dns; [199.164.235.226] -Arrival-Date: Fri, 6 Apr 2001 09:23:03 -0800 (GMT-0800) - -Final-Recipient: rfc822; scoffman at wellpartner.com -Action: failed -Status: 5.4.6 -Last-Attempt-Date: Fri, 6 Apr 2001 09:23:06 -0800 (GMT-0800) - ---JAB03225.986577786/zinfandel.lacita.com -Content-Type: text/rfc822-headers - -Return-Path: linuxuser-admin at www.linux.org.uk -Received: from ns1.wellpartner.net ([199.164.235.226]) by zinfandel.lacita.com (8.7.3/8.6.10-MT4.00) with ESMTP id JAA03225 for <scoffman at wellpartner.com>; Fri, 6 Apr 2001 09:23:03 -0800 (GMT-0800) -Received: from zinfandel.lacita.com ([204.245.199.98]) - by - fo -Received: from ns1.wellpartner.net ([199.164.235.226]) by zinfandel.lacita.com (8.7.3/8.6.10-MT4.00) with ESMTP id JAA03221 for <scoffman at wellpartner.com>; Fri, 6 Apr 2001 09:22:18 -0800 (GMT-0800) -Received: from zinfandel.lacita.com ([204.245.199.98]) - by - fo -Received: from ns1.wellpartner.net ([199.164.235.226]) by zinfandel.lacita.com (8.7.3/8.6.10-MT4.00) with ESMTP id JAA03217 for <scoffman at wellpartner.com>; Fri, 6 Apr 2001 09:21:37 -0800 (GMT-0800) -Received: from zinfandel.lacita.com ([204.245.199.98]) - by - fo -Received: from ns1.wellpartner.net ([199.164.235.226]) by zinfandel.lacita.com (8.7.3/8.6.10-MT4.00) with ESMTP id JAA03213 for <scoffman at wellpartner.com>; Fri, 6 Apr 2001 09:20:56 -0800 (GMT-0800) -Received: from zinfandel.lacita.com ([204.245.199.98]) - by - fo -Received: from ns1.wellpartner.net ([199.164.235.226]) by zinfandel.lacita.com (8.7.3/8.6.10-MT4.00) with ESMTP id JAA03209 for <scoffman at wellpartner.com>; Fri, 6 Apr 2001 09:20:15 -0800 (GMT-0800) -Received: from zinfandel.lacita.com ([204.245.199.98]) - by - fo -Received: from ns1.wellpartner.net ([199.164.235.226]) by zinfandel.lacita.com (8.7.3/8.6.10-MT4.00) with ESMTP id JAA03205 for <scoffman at wellpartner.com>; Fri, 6 Apr 2001 09:19:33 -0800 (GMT-0800) -Received: from zinfandel.lacita.com ([204.245.199.98]) - by - fo -Received: from ns1.wellpartner.net ([199.164.235.226]) by zinfandel.lacita.com (8.7.3/8.6.10-MT4.00) with ESMTP id JAA03201 for <scoffman at wellpartner.com>; Fri, 6 Apr 2001 09:18:52 -0800 (GMT-0800) -Received: from zinfandel.lacita.com ([204.245.199.98]) - by - fo -Received: from ns1.wellpartner.net ([199.164.235.226]) by zinfandel.lacita.com (8.7.3/8.6.10-MT4.00) with ESMTP id JAA03197 for <scoffman at wellpartner.com>; Fri, 6 Apr 2001 09:17:54 -0800 (GMT-0800) -Received: from www.linux.org.uk (parcelfarce.linux.theplanet.co.uk [195.92.249.252]) - by - fo -Received: from localhost.localdomain - ([ - by - id -Received: from [212.1.130.11] (helo=s1.uklinux.net ident=root) - by - id - fo -Received: from server (ppp-2-22.cvx4.telinco.net [212.1.149.22]) - by - fo -From: Daniel James <daniel at linuxuser.co.uk> -Organization: LinuxUser -To: linuxuser at www.linux.org.uk -X-Mailer: KMail [version 1.1.99] -Content-Type: text/plain; - c -MIME-Version: 1.0 -Message-Id: <01040616033903.00962 at server> -Content-Transfer-Encoding: 8bit -Subject: [LinuxUser] bulletin no. 45 -Sender: linuxuser-admin at www.linux.org.uk -Errors-To: linuxuser-admin at www.linux.org.uk -X-BeenThere: linuxuser at www.linux.org.uk -X-Mailman-Version: 2.0.3 -Precedence: bulk -List-Help: <mailto:linuxuser-request at www.linux.org.uk?subject=help> -List-Post: <mailto:linuxuser at www.linux.org.uk> -List-Subscribe: <http://www.linux.org.uk/mailman/listinfo/linuxuser>, - <m -List-Id: bulletins from LinuxUser magazine <linuxuser.www.linux.org.uk> -List-Unsubscribe: <http://www.linux.org.uk/mailman/listinfo/linuxuser>, - <m -List-Archive: <http://www.linux.org.uk/pipermail/linuxuser/> -Date: Fri, 6 Apr 2001 16:03:39 +0100 - ---JAB03225.986577786/zinfandel.lacita.com-- - - diff --git a/lib-python/modified-2.7/email/test/data/msg_26.txt b/lib-python/modified-2.7/email/test/data/msg_26.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_26.txt +++ /dev/null @@ -1,45 +0,0 @@ -Received: from xcar [192.168.0.2] by jeeves.wooster.local - (SMTPD32-7.07 EVAL) id AFF92F0214; Sun, 12 May 2002 08:55:37 +0100 -Date: Sun, 12 May 2002 08:56:15 +0100 -From: Father Time <father.time at xcar.wooster.local> -To: timbo at jeeves.wooster.local -Subject: IMAP file test -Message-ID: <6df65d354b.father.time at rpc.wooster.local> -X-Organization: Home -User-Agent: Messenger-Pro/2.50a (MsgServe/1.50) (RISC-OS/4.02) POPstar/2.03 -MIME-Version: 1.0 -Content-Type: multipart/mixed; boundary="1618492860--2051301190--113853680" -Status: R -X-UIDL: 319998302 - -This message is in MIME format which your mailer apparently does not support. -You either require a newer version of your software which supports MIME, or -a separate MIME decoding utility. Alternatively, ask the sender of this -message to resend it in a different format. - ---1618492860--2051301190--113853680 -Content-Type: text/plain; charset=us-ascii - -Simple email with attachment. - - ---1618492860--2051301190--113853680 -Content-Type: application/riscos; name="clock.bmp,69c"; type=BMP; load=&fff69c4b; exec=&355dd4d1; access=&03 -Content-Disposition: attachment; filename="clock.bmp" -Content-Transfer-Encoding: base64 - -Qk12AgAAAAAAAHYAAAAoAAAAIAAAACAAAAABAAQAAAAAAAAAAADXDQAA1w0AAAAAAAAA -AAAAAAAAAAAAiAAAiAAAAIiIAIgAAACIAIgAiIgAALu7uwCIiIgAERHdACLuIgAz//8A -zAAAAN0R3QDu7iIA////AAAAAAAAAAAAAAAAAAAAAAAAAAi3AAAAAAAAADeAAAAAAAAA -C3ADMzMzMANwAAAAAAAAAAAHMAAAAANwAAAAAAAAAACAMAd3zPfwAwgAAAAAAAAIAwd/ -f8x/f3AwgAAAAAAAgDB0x/f3//zPAwgAAAAAAAcHfM9////8z/AwAAAAAAiwd/f3//// -////A4AAAAAAcEx/f///////zAMAAAAAiwfM9////3///8zwOAAAAAcHf3////B///// -8DAAAAALB/f3///wd3d3//AwAAAABwTPf//wCQAAD/zAMAAAAAsEx/f///B////8wDAA -AAAHB39////wf/////AwAAAACwf39///8H/////wMAAAAIcHfM9///B////M8DgAAAAA -sHTH///wf///xAMAAAAACHB3f3//8H////cDgAAAAAALB3zH//D//M9wMAAAAAAAgLB0 -z39///xHAwgAAAAAAAgLB3d3RHd3cDCAAAAAAAAAgLAHd0R3cAMIAAAAAAAAgAgLcAAA -AAMwgAgAAAAACDAAAAu7t7cwAAgDgAAAAABzcIAAAAAAAAgDMwAAAAAAN7uwgAAAAAgH -MzMAAAAACH97tzAAAAALu3c3gAAAAAAL+7tzDABAu7f7cAAAAAAACA+3MA7EQAv/sIAA -AAAAAAAIAAAAAAAAAIAAAAAA - ---1618492860--2051301190--113853680-- \ No newline at end of file diff --git a/lib-python/modified-2.7/email/test/data/msg_27.txt b/lib-python/modified-2.7/email/test/data/msg_27.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_27.txt +++ /dev/null @@ -1,15 +0,0 @@ -Return-Path: <aperson at dom.ain> -Received: by mail.dom.ain (Postfix, from userid 889) - id B9D0AD35DB; Tue, 4 Jun 2002 21:46:59 -0400 (EDT) -Message-ID: <15613.28051.707126.569693 at dom.ain> -Date: Tue, 4 Jun 2002 21:46:59 -0400 -MIME-Version: 1.0 -Content-Type: text/plain; charset=us-ascii -Content-Transfer-Encoding: 7bit -Subject: bug demonstration - 12345678911234567892123456789312345678941234567895123456789612345678971234567898112345678911234567892123456789112345678911234567892123456789 - more text -From: aperson at dom.ain (Anne P. Erson) -To: bperson at dom.ain (Barney P. Erson) - -test diff --git a/lib-python/modified-2.7/email/test/data/msg_28.txt b/lib-python/modified-2.7/email/test/data/msg_28.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_28.txt +++ /dev/null @@ -1,25 +0,0 @@ -From: aperson at dom.ain -MIME-Version: 1.0 -Content-Type: multipart/digest; boundary=BOUNDARY - ---BOUNDARY -Content-Type: message/rfc822 - -Content-Type: text/plain; charset=us-ascii -To: aa at bb.org -From: cc at dd.org -Subject: ee - -message 1 - ---BOUNDARY -Content-Type: message/rfc822 - -Content-Type: text/plain; charset=us-ascii -To: aa at bb.org -From: cc at dd.org -Subject: ee - -message 2 - ---BOUNDARY-- diff --git a/lib-python/modified-2.7/email/test/data/msg_29.txt b/lib-python/modified-2.7/email/test/data/msg_29.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_29.txt +++ /dev/null @@ -1,22 +0,0 @@ -Return-Path: <bbb at zzz.org> -Delivered-To: bbb at zzz.org -Received: by mail.zzz.org (Postfix, from userid 889) - id 27CEAD38CC; Fri, 4 May 2001 14:05:44 -0400 (EDT) -MIME-Version: 1.0 -Content-Type: text/plain; charset=us-ascii; - title*0*="us-ascii'en'This%20is%20even%20more%20"; - title*1*="%2A%2A%2Afun%2A%2A%2A%20"; - title*2="isn't it!" -Content-Transfer-Encoding: 7bit -Message-ID: <15090.61304.110929.45684 at aaa.zzz.org> -From: bbb at ddd.com (John X. Doe) -To: bbb at zzz.org -Subject: This is a test message -Date: Fri, 4 May 2001 14:05:44 -0400 - - -Hi, - -Do you like this message? - --Me diff --git a/lib-python/modified-2.7/email/test/data/msg_30.txt b/lib-python/modified-2.7/email/test/data/msg_30.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_30.txt +++ /dev/null @@ -1,23 +0,0 @@ -From: aperson at dom.ain -MIME-Version: 1.0 -Content-Type: multipart/digest; boundary=BOUNDARY - ---BOUNDARY - -Content-Type: text/plain; charset=us-ascii -To: aa at bb.org -From: cc at dd.org -Subject: ee - -message 1 - ---BOUNDARY - -Content-Type: text/plain; charset=us-ascii -To: aa at bb.org -From: cc at dd.org -Subject: ee - -message 2 - ---BOUNDARY-- diff --git a/lib-python/modified-2.7/email/test/data/msg_31.txt b/lib-python/modified-2.7/email/test/data/msg_31.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_31.txt +++ /dev/null @@ -1,15 +0,0 @@ -From: aperson at dom.ain -MIME-Version: 1.0 -Content-Type: multipart/mixed; boundary=BOUNDARY_ - ---BOUNDARY -Content-Type: text/plain - -message 1 - ---BOUNDARY -Content-Type: text/plain - -message 2 - ---BOUNDARY-- diff --git a/lib-python/modified-2.7/email/test/data/msg_32.txt b/lib-python/modified-2.7/email/test/data/msg_32.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_32.txt +++ /dev/null @@ -1,14 +0,0 @@ -Delivered-To: freebsd-isp at freebsd.org -Date: Tue, 26 Sep 2000 12:23:03 -0500 -From: Anne Person <aperson at example.com> -To: Barney Dude <bdude at example.com> -Subject: Re: Limiting Perl CPU Utilization... -Mime-Version: 1.0 -Content-Type: text/plain; charset*=ansi-x3.4-1968''us-ascii -Content-Disposition: inline -User-Agent: Mutt/1.3.8i -Sender: owner-freebsd-isp at FreeBSD.ORG -Precedence: bulk -X-Loop: FreeBSD.org - -Some message. diff --git a/lib-python/modified-2.7/email/test/data/msg_33.txt b/lib-python/modified-2.7/email/test/data/msg_33.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_33.txt +++ /dev/null @@ -1,29 +0,0 @@ -Delivered-To: freebsd-isp at freebsd.org -Date: Wed, 27 Sep 2000 11:11:09 -0500 -From: Anne Person <aperson at example.com> -To: Barney Dude <bdude at example.com> -Subject: Re: Limiting Perl CPU Utilization... -Mime-Version: 1.0 -Content-Type: multipart/signed; micalg*=ansi-x3.4-1968''pgp-md5; - protocol*=ansi-x3.4-1968''application%2Fpgp-signature; - boundary*="ansi-x3.4-1968''EeQfGwPcQSOJBaQU" -Content-Disposition: inline -Sender: owner-freebsd-isp at FreeBSD.ORG -Precedence: bulk -X-Loop: FreeBSD.org - - ---EeQfGwPcQSOJBaQU -Content-Type: text/plain; charset*=ansi-x3.4-1968''us-ascii -Content-Disposition: inline -Content-Transfer-Encoding: quoted-printable - -part 1 - ---EeQfGwPcQSOJBaQU -Content-Type: text/plain -Content-Disposition: inline - -part 2 - ---EeQfGwPcQSOJBaQU-- diff --git a/lib-python/modified-2.7/email/test/data/msg_34.txt b/lib-python/modified-2.7/email/test/data/msg_34.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_34.txt +++ /dev/null @@ -1,19 +0,0 @@ -From: aperson at dom.ain -To: bperson at dom.ain -Content-Type: multipart/digest; boundary=XYZ - ---XYZ -Content-Type: text/plain - - -This is a text plain part that is counter to recommended practice in -RFC 2046, $5.1.5, but is not illegal - ---XYZ - -From: cperson at dom.ain -To: dperson at dom.ain - -A submessage - ---XYZ-- diff --git a/lib-python/modified-2.7/email/test/data/msg_35.txt b/lib-python/modified-2.7/email/test/data/msg_35.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_35.txt +++ /dev/null @@ -1,4 +0,0 @@ -From: aperson at dom.ain -To: bperson at dom.ain -Subject: here's something interesting -counter to RFC 2822, there's no separating newline here diff --git a/lib-python/modified-2.7/email/test/data/msg_36.txt b/lib-python/modified-2.7/email/test/data/msg_36.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_36.txt +++ /dev/null @@ -1,40 +0,0 @@ -Mime-Version: 1.0 -Content-Type: Multipart/Mixed; Boundary="NextPart" -To: IETF-Announce:; -From: Internet-Drafts at ietf.org -Subject: I-D ACTION:draft-ietf-mboned-mix-00.txt -Date: Tue, 22 Dec 1998 16:55:06 -0500 - ---NextPart - -Blah blah blah - ---NextPart -Content-Type: Multipart/Alternative; Boundary="OtherAccess" - ---OtherAccess -Content-Type: Message/External-body; - access-type="mail-server"; - server="mailserv at ietf.org" - -Content-Type: text/plain -Content-ID: <19981222151406.I-D at ietf.org> - -ENCODING mime -FILE /internet-drafts/draft-ietf-mboned-mix-00.txt - ---OtherAccess -Content-Type: Message/External-body; - name="draft-ietf-mboned-mix-00.txt"; - site="ftp.ietf.org"; - access-type="anon-ftp"; - directory="internet-drafts" - -Content-Type: text/plain -Content-ID: <19981222151406.I-D at ietf.org> - - ---OtherAccess-- - ---NextPart-- - diff --git a/lib-python/modified-2.7/email/test/data/msg_37.txt b/lib-python/modified-2.7/email/test/data/msg_37.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_37.txt +++ /dev/null @@ -1,22 +0,0 @@ -Content-Type: multipart/mixed; boundary=ABCDE - ---ABCDE -Content-Type: text/x-one - -Blah - ---ABCDE ---ABCDE -Content-Type: text/x-two - -Blah - ---ABCDE ---ABCDE ---ABCDE ---ABCDE -Content-Type: text/x-two - -Blah - ---ABCDE-- diff --git a/lib-python/modified-2.7/email/test/data/msg_38.txt b/lib-python/modified-2.7/email/test/data/msg_38.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_38.txt +++ /dev/null @@ -1,101 +0,0 @@ -MIME-Version: 1.0 -Content-Type: multipart/mixed; boundary="----- =_aaaaaaaaaa0" - -------- =_aaaaaaaaaa0 -Content-Type: multipart/mixed; boundary="----- =_aaaaaaaaaa1" -Content-ID: <20592.1022586929.1 at example.com> - -------- =_aaaaaaaaaa1 -Content-Type: multipart/alternative; boundary="----- =_aaaaaaaaaa2" -Content-ID: <20592.1022586929.2 at example.com> - -------- =_aaaaaaaaaa2 -Content-Type: text/plain -Content-ID: <20592.1022586929.3 at example.com> -Content-Description: very tricky -Content-Transfer-Encoding: 7bit - - -Unlike the test test_nested-multiples-with-internal-boundary, this -piece of text not only contains the outer boundary tags -------- =_aaaaaaaaaa1 -and -------- =_aaaaaaaaaa0 -but puts them at the start of a line! And, to be even nastier, it -even includes a couple of end tags, such as this one: - -------- =_aaaaaaaaaa1-- - -and this one, which is from a multipart we haven't even seen yet! - -------- =_aaaaaaaaaa4-- - -This will, I'm sure, cause much breakage of MIME parsers. But, as -far as I can tell, it's perfectly legal. I have not yet ever seen -a case of this in the wild, but I've seen *similar* things. - - -------- =_aaaaaaaaaa2 -Content-Type: application/octet-stream -Content-ID: <20592.1022586929.4 at example.com> -Content-Description: patch2 -Content-Transfer-Encoding: base64 - -XXX - -------- =_aaaaaaaaaa2-- - -------- =_aaaaaaaaaa1 -Content-Type: multipart/alternative; boundary="----- =_aaaaaaaaaa3" -Content-ID: <20592.1022586929.6 at example.com> - -------- =_aaaaaaaaaa3 -Content-Type: application/octet-stream -Content-ID: <20592.1022586929.7 at example.com> -Content-Description: patch3 -Content-Transfer-Encoding: base64 - -XXX - -------- =_aaaaaaaaaa3 -Content-Type: application/octet-stream -Content-ID: <20592.1022586929.8 at example.com> -Content-Description: patch4 -Content-Transfer-Encoding: base64 - -XXX - -------- =_aaaaaaaaaa3-- - -------- =_aaaaaaaaaa1 -Content-Type: multipart/alternative; boundary="----- =_aaaaaaaaaa4" -Content-ID: <20592.1022586929.10 at example.com> - -------- =_aaaaaaaaaa4 -Content-Type: application/octet-stream -Content-ID: <20592.1022586929.11 at example.com> -Content-Description: patch5 -Content-Transfer-Encoding: base64 - -XXX - -------- =_aaaaaaaaaa4 -Content-Type: application/octet-stream -Content-ID: <20592.1022586929.12 at example.com> -Content-Description: patch6 -Content-Transfer-Encoding: base64 - -XXX - -------- =_aaaaaaaaaa4-- - -------- =_aaaaaaaaaa1-- - -------- =_aaaaaaaaaa0 -Content-Type: text/plain; charset="us-ascii" -Content-ID: <20592.1022586929.15 at example.com> - --- -It's never too late to have a happy childhood. - -------- =_aaaaaaaaaa0-- diff --git a/lib-python/modified-2.7/email/test/data/msg_39.txt b/lib-python/modified-2.7/email/test/data/msg_39.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_39.txt +++ /dev/null @@ -1,83 +0,0 @@ -MIME-Version: 1.0 -Content-Type: multipart/mixed; boundary="----- =_aaaaaaaaaa0" - -------- =_aaaaaaaaaa0 -Content-Type: multipart/mixed; boundary="----- =_aaaaaaaaaa1" -Content-ID: <20592.1022586929.1 at example.com> - -------- =_aaaaaaaaaa1 -Content-Type: multipart/alternative; boundary="----- =_aaaaaaaaaa1" -Content-ID: <20592.1022586929.2 at example.com> - -------- =_aaaaaaaaaa1 -Content-Type: application/octet-stream -Content-ID: <20592.1022586929.3 at example.com> -Content-Description: patch1 -Content-Transfer-Encoding: base64 - -XXX - -------- =_aaaaaaaaaa1 -Content-Type: application/octet-stream -Content-ID: <20592.1022586929.4 at example.com> -Content-Description: patch2 -Content-Transfer-Encoding: base64 - -XXX - -------- =_aaaaaaaaaa1-- - -------- =_aaaaaaaaaa1 -Content-Type: multipart/alternative; boundary="----- =_aaaaaaaaaa1" -Content-ID: <20592.1022586929.6 at example.com> - -------- =_aaaaaaaaaa1 -Content-Type: application/octet-stream -Content-ID: <20592.1022586929.7 at example.com> -Content-Description: patch3 -Content-Transfer-Encoding: base64 - -XXX - -------- =_aaaaaaaaaa1 -Content-Type: application/octet-stream -Content-ID: <20592.1022586929.8 at example.com> -Content-Description: patch4 -Content-Transfer-Encoding: base64 - -XXX - -------- =_aaaaaaaaaa1-- - -------- =_aaaaaaaaaa1 -Content-Type: multipart/alternative; boundary="----- =_aaaaaaaaaa1" -Content-ID: <20592.1022586929.10 at example.com> - -------- =_aaaaaaaaaa1 -Content-Type: application/octet-stream -Content-ID: <20592.1022586929.11 at example.com> -Content-Description: patch5 -Content-Transfer-Encoding: base64 - -XXX - -------- =_aaaaaaaaaa1 -Content-Type: application/octet-stream -Content-ID: <20592.1022586929.12 at example.com> -Content-Description: patch6 -Content-Transfer-Encoding: base64 - -XXX - -------- =_aaaaaaaaaa1-- - -------- =_aaaaaaaaaa1-- - -------- =_aaaaaaaaaa0 -Content-Type: text/plain; charset="us-ascii" -Content-ID: <20592.1022586929.15 at example.com> - --- -It's never too late to have a happy childhood. - -------- =_aaaaaaaaaa0-- diff --git a/lib-python/modified-2.7/email/test/data/msg_40.txt b/lib-python/modified-2.7/email/test/data/msg_40.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_40.txt +++ /dev/null @@ -1,10 +0,0 @@ -MIME-Version: 1.0 -Content-Type: text/html; boundary="--961284236552522269" - -----961284236552522269 -Content-Type: text/html; -Content-Transfer-Encoding: 7Bit - -<html></html> - -----961284236552522269-- diff --git a/lib-python/modified-2.7/email/test/data/msg_41.txt b/lib-python/modified-2.7/email/test/data/msg_41.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_41.txt +++ /dev/null @@ -1,8 +0,0 @@ -From: "Allison Dunlap" <xxx at example.com> -To: yyy at example.com -Subject: 64423 -Date: Sun, 11 Jul 2004 16:09:27 -0300 -MIME-Version: 1.0 -Content-Type: multipart/alternative; - -Blah blah blah diff --git a/lib-python/modified-2.7/email/test/data/msg_42.txt b/lib-python/modified-2.7/email/test/data/msg_42.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_42.txt +++ /dev/null @@ -1,20 +0,0 @@ -Content-Type: multipart/mixed; boundary="AAA" -From: Mail Delivery Subsystem <xxx at example.com> -To: yyy at example.com - -This is a MIME-encapsulated message - ---AAA - -Stuff - ---AAA -Content-Type: message/rfc822 - -From: webmaster at python.org -To: zzz at example.com -Content-Type: multipart/mixed; boundary="BBB" - ---BBB-- - ---AAA-- diff --git a/lib-python/modified-2.7/email/test/data/msg_43.txt b/lib-python/modified-2.7/email/test/data/msg_43.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_43.txt +++ /dev/null @@ -1,217 +0,0 @@ -From SRS0=aO/p=ON=bag.python.org=None at bounce2.pobox.com Fri Nov 26 21:40:36 2004 -X-VM-v5-Data: ([nil nil nil nil nil nil nil nil nil] - [nil nil nil nil nil nil nil "MAILER DAEMON <>" "MAILER DAEMON <>" nil nil "Banned file: auto__mail.python.bat in mail from you" "^From:" nil nil nil nil "Banned file: auto__mail.python.bat in mail from you" nil nil nil nil nil nil nil] - nil) -MIME-Version: 1.0 -Message-Id: <edab.7804f5cb8070 at python.org> -Content-Type: multipart/report; report-type=delivery-status; - charset=utf-8; - boundary="----------=_1101526904-1956-5" -X-Virus-Scanned: by XS4ALL Virus Scanner -X-UIDL: 4\G!!!<c"!UV["!M7C!! -From: MAILER DAEMON <> -To: <webmaster at python.org> -Subject: Banned file: auto__mail.python.bat in mail from you -Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -This is a multi-part message in MIME format... - -------------=_1101526904-1956-5 -Content-Type: text/plain; charset="utf-8" -Content-Disposition: inline -Content-Transfer-Encoding: 7bit - -BANNED FILENAME ALERT - -Your message to: xxxxxxx at dot.ca.gov, xxxxxxxxxxxxx at dot.ca.gov, xxxxxxxxxx at dot.ca.gov, xxxxxxxx at dot.ca.gov, xxxxxxxxxx at dot.ca.gov, xxxxxx at dot.ca.gov, xxxxxxxxxx at dot.ca.gov, xxxxxx at dot.ca.gov, xxxxxx at dot.ca.gov, xxxxxxxxxxxxxxxx at dot.ca.gov, xxxxxxxxxxx at dot.ca.gov, xxxxxxxxxx at dot.ca.gov, xxxxxxxxxx at dot.ca.gov, xxxxxxxxxxxx at dot.ca.gov, xxxxxxxxxxxx at dot.ca.gov, xxxxxxx at dot.ca.gov, xxxxxxxxx at dot.ca.gov, xxxxxxxxxx at dot.ca.gov, xxxxxx at dot.ca.gov, xxx at dot.ca.gov, xxxxxxx at dot.ca.gov, xxxxxxx at dot.ca.gov, xxxxxxxxxxxxxxx at dot.ca.gov, xxxxxxxxxx at dot.ca.gov, xxxxxxx at dot.ca.gov, xxx at dot.ca.gov, xxxxxxxx at dot.ca.gov, xxxxxxxxxxxxx at dot.ca.gov, xxxxxxxxxxxxx at dot.ca.gov, xxxxxxxxxxx at dot.ca.gov, xxxxxxxxx at dot.ca.gov, xxxxxxxxxx at dot.ca.gov, xxxxxxxxxxxx at dot.ca.gov, xxxxxxx at dot.ca.gov, xxxxxxxxxxxxxxx at dot.ca.gov, xxxxxxxxxxxxx at dot.ca.gov, xxxx at dot.ca.gov, xxxxxxxx at dot.ca.gov, xxxxxxxxxx at dot.ca.gov, xxxxxxxxxxxxxxxxxx at dot.ca.gov -was blocked by our Spam Firewall. The email you sent with the following subject has NOT BEEN DELIVERED: - -Subject: Delivery_failure_notice - -An attachment in that mail was of a file type that the Spam Firewall is set to block. - - - -------------=_1101526904-1956-5 -Content-Type: message/delivery-status -Content-Disposition: inline -Content-Transfer-Encoding: 7bit -Content-Description: Delivery error report - -Reporting-MTA: dns; sacspam01.dot.ca.gov -Received-From-MTA: smtp; sacspam01.dot.ca.gov ([127.0.0.1]) -Arrival-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxxxxxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxxxxxxxxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxxxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxxxxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxxxxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxxxxxxxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -------------=_1101526904-1956-5 -Content-Type: text/rfc822-headers -Content-Disposition: inline -Content-Transfer-Encoding: 7bit -Content-Description: Undelivered-message headers - -Received: from kgsav.org (ppp-70-242-162-63.dsl.spfdmo.swbell.net [70.242.162.63]) - by sacspam01.dot.ca.gov (Spam Firewall) with SMTP - id A232AD03DE3A; Fri, 26 Nov 2004 19:41:35 -0800 (PST) -From: webmaster at python.org -To: xxxxx at dot.ca.gov -Date: Sat, 27 Nov 2004 03:35:30 UTC -Subject: Delivery_failure_notice -Importance: Normal -X-Priority: 3 (Normal) -X-MSMail-Priority: Normal -Message-ID: <edab.7804f5cb8070 at python.org> -MIME-Version: 1.0 -Content-Type: multipart/mixed; boundary="====67bd2b7a5.f99f7" -Content-Transfer-Encoding: 7bit - -------------=_1101526904-1956-5-- - diff --git a/lib-python/modified-2.7/email/test/data/msg_44.txt b/lib-python/modified-2.7/email/test/data/msg_44.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_44.txt +++ /dev/null @@ -1,33 +0,0 @@ -Return-Path: <barry at python.org> -Delivered-To: barry at python.org -Received: by mail.python.org (Postfix, from userid 889) - id C2BF0D37C6; Tue, 11 Sep 2001 00:05:05 -0400 (EDT) -MIME-Version: 1.0 -Content-Type: multipart/mixed; boundary="h90VIIIKmx" -Content-Transfer-Encoding: 7bit -Message-ID: <15261.36209.358846.118674 at anthem.python.org> -From: barry at python.org (Barry A. Warsaw) -To: barry at python.org -Subject: a simple multipart -Date: Tue, 11 Sep 2001 00:05:05 -0400 -X-Mailer: VM 6.95 under 21.4 (patch 4) "Artificial Intelligence" XEmacs Lucid -X-Attribution: BAW -X-Oblique-Strategy: Make a door into a window - - ---h90VIIIKmx -Content-Type: text/plain; name="msg.txt" -Content-Transfer-Encoding: 7bit - -a simple kind of mirror -to reflect upon our own - ---h90VIIIKmx -Content-Type: text/plain; name="msg.txt" -Content-Transfer-Encoding: 7bit - -a simple kind of mirror -to reflect upon our own - ---h90VIIIKmx-- - diff --git a/lib-python/modified-2.7/email/test/data/msg_45.txt b/lib-python/modified-2.7/email/test/data/msg_45.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_45.txt +++ /dev/null @@ -1,33 +0,0 @@ -From: <foo at bar.baz> -To: <baz at bar.foo> -Subject: test -X-Long-Line: Some really long line contains a lot of text and thus has to be rewrapped because it is some - really long - line -MIME-Version: 1.0 -Content-Type: multipart/signed; boundary="borderline"; - protocol="application/pgp-signature"; micalg=pgp-sha1 - -This is an OpenPGP/MIME signed message (RFC 2440 and 3156) ---borderline -Content-Type: text/plain -X-Long-Line: Another really long line contains a lot of text and thus has to be rewrapped because it is another - really long - line - -This is the signed contents. - ---borderline -Content-Type: application/pgp-signature; name="signature.asc" -Content-Description: OpenPGP digital signature -Content-Disposition: attachment; filename="signature.asc" - ------BEGIN PGP SIGNATURE----- -Version: GnuPG v2.0.6 (GNU/Linux) - -iD8DBQFG03voRhp6o4m9dFsRApSZAKCCAN3IkJlVRg6NvAiMHlvvIuMGPQCeLZtj -FGwfnRHFBFO/S4/DKysm0lI= -=t7+s ------END PGP SIGNATURE----- - ---borderline-- diff --git a/lib-python/modified-2.7/email/test/data/msg_46.txt b/lib-python/modified-2.7/email/test/data/msg_46.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_46.txt +++ /dev/null @@ -1,23 +0,0 @@ -Return-Path: <sender at example.net> -Delivery-Date: Mon, 08 Feb 2010 14:05:16 +0100 -Received: from example.org (example.org [64.5.53.58]) - by example.net (node=mxbap2) with ESMTP (Nemesis) - id UNIQUE for someone at example.com; Mon, 08 Feb 2010 14:05:16 +0100 -Date: Mon, 01 Feb 2010 12:21:16 +0100 -From: "Sender" <sender at example.net> -To: <someone at example.com> -Subject: GroupwiseForwardingTest -Mime-Version: 1.0 -Content-Type: message/rfc822 - -Return-path: <sender at example.net> -Message-ID: <4B66B890.4070408 at teconcept.de> -Date: Mon, 01 Feb 2010 12:18:40 +0100 -From: "Dr. Sender" <sender at example.net> -MIME-Version: 1.0 -To: "Recipient" <recipient at example.com> -Subject: GroupwiseForwardingTest -Content-Type: text/plain; charset=ISO-8859-15 -Content-Transfer-Encoding: 7bit - -Testing email forwarding with Groupwise 1.2.2010 diff --git a/lib-python/modified-2.7/email/test/test_email.py b/lib-python/modified-2.7/email/test/test_email.py deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/test_email.py +++ /dev/null @@ -1,3530 +0,0 @@ -# Copyright (C) 2001-2010 Python Software Foundation -# Contact: email-sig at python.org -# email package unit tests - -import os -import sys -import time -import base64 -import difflib -import unittest -import warnings -from cStringIO import StringIO - -import email - -from email.Charset import Charset -from email.Header import Header, decode_header, make_header -from email.Parser import Parser, HeaderParser -from email.Generator import Generator, DecodedGenerator -from email.Message import Message -from email.MIMEAudio import MIMEAudio -from email.MIMEText import MIMEText -from email.MIMEImage import MIMEImage -from email.MIMEBase import MIMEBase -from email.MIMEMessage import MIMEMessage -from email.MIMEMultipart import MIMEMultipart -from email import Utils -from email import Errors -from email import Encoders -from email import Iterators -from email import base64MIME -from email import quopriMIME - -from test.test_support import findfile, run_unittest, impl_detail -from email.test import __file__ as landmark - - -NL = '\n' -EMPTYSTRING = '' -SPACE = ' ' - - - -def openfile(filename, mode='r'): - path = os.path.join(os.path.dirname(landmark), 'data', filename) - return open(path, mode) - - - -# Base test class -class TestEmailBase(unittest.TestCase): - def ndiffAssertEqual(self, first, second): - """Like assertEqual except use ndiff for readable output.""" - if first != second: - sfirst = str(first) - ssecond = str(second) - diff = difflib.ndiff(sfirst.splitlines(), ssecond.splitlines()) - fp = StringIO() - print >> fp, NL, NL.join(diff) - raise self.failureException, fp.getvalue() - - def _msgobj(self, filename): - fp = openfile(findfile(filename)) - try: - msg = email.message_from_file(fp) - finally: - fp.close() - return msg - - - -# Test various aspects of the Message class's API -class TestMessageAPI(TestEmailBase): - def test_get_all(self): - eq = self.assertEqual - msg = self._msgobj('msg_20.txt') - eq(msg.get_all('cc'), ['ccc at zzz.org', 'ddd at zzz.org', 'eee at zzz.org']) - eq(msg.get_all('xx', 'n/a'), 'n/a') - - def test_getset_charset(self): - eq = self.assertEqual - msg = Message() - eq(msg.get_charset(), None) - charset = Charset('iso-8859-1') - msg.set_charset(charset) - eq(msg['mime-version'], '1.0') - eq(msg.get_content_type(), 'text/plain') - eq(msg['content-type'], 'text/plain; charset="iso-8859-1"') - eq(msg.get_param('charset'), 'iso-8859-1') - eq(msg['content-transfer-encoding'], 'quoted-printable') - eq(msg.get_charset().input_charset, 'iso-8859-1') - # Remove the charset - msg.set_charset(None) - eq(msg.get_charset(), None) - eq(msg['content-type'], 'text/plain') - # Try adding a charset when there's already MIME headers present - msg = Message() - msg['MIME-Version'] = '2.0' - msg['Content-Type'] = 'text/x-weird' - msg['Content-Transfer-Encoding'] = 'quinted-puntable' - msg.set_charset(charset) - eq(msg['mime-version'], '2.0') - eq(msg['content-type'], 'text/x-weird; charset="iso-8859-1"') - eq(msg['content-transfer-encoding'], 'quinted-puntable') - - def test_set_charset_from_string(self): - eq = self.assertEqual - msg = Message() - msg.set_charset('us-ascii') - eq(msg.get_charset().input_charset, 'us-ascii') - eq(msg['content-type'], 'text/plain; charset="us-ascii"') - - def test_set_payload_with_charset(self): - msg = Message() - charset = Charset('iso-8859-1') - msg.set_payload('This is a string payload', charset) - self.assertEqual(msg.get_charset().input_charset, 'iso-8859-1') - - def test_get_charsets(self): - eq = self.assertEqual - - msg = self._msgobj('msg_08.txt') - charsets = msg.get_charsets() - eq(charsets, [None, 'us-ascii', 'iso-8859-1', 'iso-8859-2', 'koi8-r']) - - msg = self._msgobj('msg_09.txt') - charsets = msg.get_charsets('dingbat') - eq(charsets, ['dingbat', 'us-ascii', 'iso-8859-1', 'dingbat', - 'koi8-r']) - - msg = self._msgobj('msg_12.txt') - charsets = msg.get_charsets() - eq(charsets, [None, 'us-ascii', 'iso-8859-1', None, 'iso-8859-2', - 'iso-8859-3', 'us-ascii', 'koi8-r']) - - def test_get_filename(self): - eq = self.assertEqual - - msg = self._msgobj('msg_04.txt') - filenames = [p.get_filename() for p in msg.get_payload()] - eq(filenames, ['msg.txt', 'msg.txt']) - - msg = self._msgobj('msg_07.txt') - subpart = msg.get_payload(1) - eq(subpart.get_filename(), 'dingusfish.gif') - - def test_get_filename_with_name_parameter(self): - eq = self.assertEqual - - msg = self._msgobj('msg_44.txt') - filenames = [p.get_filename() for p in msg.get_payload()] - eq(filenames, ['msg.txt', 'msg.txt']) - - def test_get_boundary(self): - eq = self.assertEqual - msg = self._msgobj('msg_07.txt') - # No quotes! - eq(msg.get_boundary(), 'BOUNDARY') - - def test_set_boundary(self): - eq = self.assertEqual - # This one has no existing boundary parameter, but the Content-Type: - # header appears fifth. - msg = self._msgobj('msg_01.txt') - msg.set_boundary('BOUNDARY') - header, value = msg.items()[4] - eq(header.lower(), 'content-type') - eq(value, 'text/plain; charset="us-ascii"; boundary="BOUNDARY"') - # This one has a Content-Type: header, with a boundary, stuck in the - # middle of its headers. Make sure the order is preserved; it should - # be fifth. - msg = self._msgobj('msg_04.txt') - msg.set_boundary('BOUNDARY') - header, value = msg.items()[4] - eq(header.lower(), 'content-type') - eq(value, 'multipart/mixed; boundary="BOUNDARY"') - # And this one has no Content-Type: header at all. - msg = self._msgobj('msg_03.txt') - self.assertRaises(Errors.HeaderParseError, - msg.set_boundary, 'BOUNDARY') - - def test_make_boundary(self): - msg = MIMEMultipart('form-data') - # Note that when the boundary gets created is an implementation - # detail and might change. - self.assertEqual(msg.items()[0][1], 'multipart/form-data') - # Trigger creation of boundary - msg.as_string() - self.assertEqual(msg.items()[0][1][:33], - 'multipart/form-data; boundary="==') - # XXX: there ought to be tests of the uniqueness of the boundary, too. - - def test_message_rfc822_only(self): - # Issue 7970: message/rfc822 not in multipart parsed by - # HeaderParser caused an exception when flattened. - fp = openfile(findfile('msg_46.txt')) - msgdata = fp.read() - parser = email.Parser.HeaderParser() - msg = parser.parsestr(msgdata) - out = StringIO() - gen = email.Generator.Generator(out, True, 0) - gen.flatten(msg, False) - self.assertEqual(out.getvalue(), msgdata) - - def test_get_decoded_payload(self): - eq = self.assertEqual - msg = self._msgobj('msg_10.txt') - # The outer message is a multipart - eq(msg.get_payload(decode=True), None) - # Subpart 1 is 7bit encoded - eq(msg.get_payload(0).get_payload(decode=True), - 'This is a 7bit encoded message.\n') - # Subpart 2 is quopri - eq(msg.get_payload(1).get_payload(decode=True), - '\xa1This is a Quoted Printable encoded message!\n') - # Subpart 3 is base64 - eq(msg.get_payload(2).get_payload(decode=True), - 'This is a Base64 encoded message.') - # Subpart 4 is base64 with a trailing newline, which - # used to be stripped (issue 7143). - eq(msg.get_payload(3).get_payload(decode=True), - 'This is a Base64 encoded message.\n') - # Subpart 5 has no Content-Transfer-Encoding: header. - eq(msg.get_payload(4).get_payload(decode=True), - 'This has no Content-Transfer-Encoding: header.\n') - - def test_get_decoded_uu_payload(self): - eq = self.assertEqual - msg = Message() - msg.set_payload('begin 666 -\n+:&5L;&\\@=V]R;&0 \n \nend\n') - for cte in ('x-uuencode', 'uuencode', 'uue', 'x-uue'): - msg['content-transfer-encoding'] = cte - eq(msg.get_payload(decode=True), 'hello world') - # Now try some bogus data - msg.set_payload('foo') - eq(msg.get_payload(decode=True), 'foo') - - def test_decode_bogus_uu_payload_quietly(self): - msg = Message() - msg.set_payload('begin 664 foo.txt\n%<W1F=0000H \n \nend\n') - msg['Content-Transfer-Encoding'] = 'x-uuencode' - old_stderr = sys.stderr - try: - sys.stderr = sfp = StringIO() - # We don't care about the payload - msg.get_payload(decode=True) - finally: - sys.stderr = old_stderr - self.assertEqual(sfp.getvalue(), '') - - def test_decoded_generator(self): - eq = self.assertEqual - msg = self._msgobj('msg_07.txt') - fp = openfile('msg_17.txt') - try: - text = fp.read() - finally: - fp.close() - s = StringIO() - g = DecodedGenerator(s) - g.flatten(msg) - eq(s.getvalue(), text) - - def test__contains__(self): - msg = Message() - msg['From'] = 'Me' - msg['to'] = 'You' - # Check for case insensitivity - self.assertTrue('from' in msg) - self.assertTrue('From' in msg) - self.assertTrue('FROM' in msg) - self.assertTrue('to' in msg) - self.assertTrue('To' in msg) - self.assertTrue('TO' in msg) - - def test_as_string(self): - eq = self.assertEqual - msg = self._msgobj('msg_01.txt') - fp = openfile('msg_01.txt') - try: - # BAW 30-Mar-2009 Evil be here. So, the generator is broken with - # respect to long line breaking. It's also not idempotent when a - # header from a parsed message is continued with tabs rather than - # spaces. Before we fixed bug 1974 it was reversedly broken, - # i.e. headers that were continued with spaces got continued with - # tabs. For Python 2.x there's really no good fix and in Python - # 3.x all this stuff is re-written to be right(er). Chris Withers - # convinced me that using space as the default continuation - # character is less bad for more applications. - text = fp.read().replace('\t', ' ') - finally: - fp.close() - eq(text, msg.as_string()) - fullrepr = str(msg) - lines = fullrepr.split('\n') - self.assertTrue(lines[0].startswith('From ')) - eq(text, NL.join(lines[1:])) - - def test_bad_param(self): - msg = email.message_from_string("Content-Type: blarg; baz; boo\n") - self.assertEqual(msg.get_param('baz'), '') - - def test_missing_filename(self): - msg = email.message_from_string("From: foo\n") - self.assertEqual(msg.get_filename(), None) - - def test_bogus_filename(self): - msg = email.message_from_string( - "Content-Disposition: blarg; filename\n") - self.assertEqual(msg.get_filename(), '') - - def test_missing_boundary(self): - msg = email.message_from_string("From: foo\n") - self.assertEqual(msg.get_boundary(), None) - - def test_get_params(self): - eq = self.assertEqual - msg = email.message_from_string( - 'X-Header: foo=one; bar=two; baz=three\n') - eq(msg.get_params(header='x-header'), - [('foo', 'one'), ('bar', 'two'), ('baz', 'three')]) - msg = email.message_from_string( - 'X-Header: foo; bar=one; baz=two\n') - eq(msg.get_params(header='x-header'), - [('foo', ''), ('bar', 'one'), ('baz', 'two')]) - eq(msg.get_params(), None) - msg = email.message_from_string( - 'X-Header: foo; bar="one"; baz=two\n') - eq(msg.get_params(header='x-header'), - [('foo', ''), ('bar', 'one'), ('baz', 'two')]) - - def test_get_param_liberal(self): - msg = Message() - msg['Content-Type'] = 'Content-Type: Multipart/mixed; boundary = "CPIMSSMTPC06p5f3tG"' - self.assertEqual(msg.get_param('boundary'), 'CPIMSSMTPC06p5f3tG') - - def test_get_param(self): - eq = self.assertEqual - msg = email.message_from_string( - "X-Header: foo=one; bar=two; baz=three\n") - eq(msg.get_param('bar', header='x-header'), 'two') - eq(msg.get_param('quuz', header='x-header'), None) - eq(msg.get_param('quuz'), None) - msg = email.message_from_string( - 'X-Header: foo; bar="one"; baz=two\n') - eq(msg.get_param('foo', header='x-header'), '') - eq(msg.get_param('bar', header='x-header'), 'one') - eq(msg.get_param('baz', header='x-header'), 'two') - # XXX: We are not RFC-2045 compliant! We cannot parse: - # msg["Content-Type"] = 'text/plain; weird="hey; dolly? [you] @ <\\"home\\">?"' - # msg.get_param("weird") - # yet. - - def test_get_param_funky_continuation_lines(self): - msg = self._msgobj('msg_22.txt') - self.assertEqual(msg.get_payload(1).get_param('name'), 'wibble.JPG') - - def test_get_param_with_semis_in_quotes(self): - msg = email.message_from_string( - 'Content-Type: image/pjpeg; name="Jim&&Jill"\n') - self.assertEqual(msg.get_param('name'), 'Jim&&Jill') - self.assertEqual(msg.get_param('name', unquote=False), - '"Jim&&Jill"') - - def test_get_param_with_quotes(self): - msg = email.message_from_string( - 'Content-Type: foo; bar*0="baz\\"foobar"; bar*1="\\"baz"') - self.assertEqual(msg.get_param('bar'), 'baz"foobar"baz') - msg = email.message_from_string( - "Content-Type: foo; bar*0=\"baz\\\"foobar\"; bar*1=\"\\\"baz\"") - self.assertEqual(msg.get_param('bar'), 'baz"foobar"baz') - - def test_has_key(self): - msg = email.message_from_string('Header: exists') - self.assertTrue(msg.has_key('header')) - self.assertTrue(msg.has_key('Header')) - self.assertTrue(msg.has_key('HEADER')) - self.assertFalse(msg.has_key('headeri')) - - def test_set_param(self): - eq = self.assertEqual - msg = Message() - msg.set_param('charset', 'iso-2022-jp') - eq(msg.get_param('charset'), 'iso-2022-jp') - msg.set_param('importance', 'high value') - eq(msg.get_param('importance'), 'high value') - eq(msg.get_param('importance', unquote=False), '"high value"') - eq(msg.get_params(), [('text/plain', ''), - ('charset', 'iso-2022-jp'), - ('importance', 'high value')]) - eq(msg.get_params(unquote=False), [('text/plain', ''), - ('charset', '"iso-2022-jp"'), - ('importance', '"high value"')]) - msg.set_param('charset', 'iso-9999-xx', header='X-Jimmy') - eq(msg.get_param('charset', header='X-Jimmy'), 'iso-9999-xx') - - def test_del_param(self): - eq = self.assertEqual - msg = self._msgobj('msg_05.txt') - eq(msg.get_params(), - [('multipart/report', ''), ('report-type', 'delivery-status'), - ('boundary', 'D1690A7AC1.996856090/mail.example.com')]) - old_val = msg.get_param("report-type") - msg.del_param("report-type") - eq(msg.get_params(), - [('multipart/report', ''), - ('boundary', 'D1690A7AC1.996856090/mail.example.com')]) - msg.set_param("report-type", old_val) - eq(msg.get_params(), - [('multipart/report', ''), - ('boundary', 'D1690A7AC1.996856090/mail.example.com'), - ('report-type', old_val)]) - - def test_del_param_on_other_header(self): - msg = Message() - msg.add_header('Content-Disposition', 'attachment', filename='bud.gif') - msg.del_param('filename', 'content-disposition') - self.assertEqual(msg['content-disposition'], 'attachment') - - def test_set_type(self): - eq = self.assertEqual - msg = Message() - self.assertRaises(ValueError, msg.set_type, 'text') - msg.set_type('text/plain') - eq(msg['content-type'], 'text/plain') - msg.set_param('charset', 'us-ascii') - eq(msg['content-type'], 'text/plain; charset="us-ascii"') - msg.set_type('text/html') - eq(msg['content-type'], 'text/html; charset="us-ascii"') - - def test_set_type_on_other_header(self): - msg = Message() - msg['X-Content-Type'] = 'text/plain' - msg.set_type('application/octet-stream', 'X-Content-Type') - self.assertEqual(msg['x-content-type'], 'application/octet-stream') - - def test_get_content_type_missing(self): - msg = Message() - self.assertEqual(msg.get_content_type(), 'text/plain') - - def test_get_content_type_missing_with_default_type(self): - msg = Message() - msg.set_default_type('message/rfc822') - self.assertEqual(msg.get_content_type(), 'message/rfc822') - - def test_get_content_type_from_message_implicit(self): - msg = self._msgobj('msg_30.txt') - self.assertEqual(msg.get_payload(0).get_content_type(), - 'message/rfc822') - - def test_get_content_type_from_message_explicit(self): - msg = self._msgobj('msg_28.txt') - self.assertEqual(msg.get_payload(0).get_content_type(), - 'message/rfc822') - - def test_get_content_type_from_message_text_plain_implicit(self): - msg = self._msgobj('msg_03.txt') - self.assertEqual(msg.get_content_type(), 'text/plain') - - def test_get_content_type_from_message_text_plain_explicit(self): - msg = self._msgobj('msg_01.txt') - self.assertEqual(msg.get_content_type(), 'text/plain') - - def test_get_content_maintype_missing(self): - msg = Message() - self.assertEqual(msg.get_content_maintype(), 'text') - - def test_get_content_maintype_missing_with_default_type(self): - msg = Message() - msg.set_default_type('message/rfc822') - self.assertEqual(msg.get_content_maintype(), 'message') - - def test_get_content_maintype_from_message_implicit(self): - msg = self._msgobj('msg_30.txt') - self.assertEqual(msg.get_payload(0).get_content_maintype(), 'message') - - def test_get_content_maintype_from_message_explicit(self): - msg = self._msgobj('msg_28.txt') - self.assertEqual(msg.get_payload(0).get_content_maintype(), 'message') - - def test_get_content_maintype_from_message_text_plain_implicit(self): - msg = self._msgobj('msg_03.txt') - self.assertEqual(msg.get_content_maintype(), 'text') - - def test_get_content_maintype_from_message_text_plain_explicit(self): - msg = self._msgobj('msg_01.txt') - self.assertEqual(msg.get_content_maintype(), 'text') - - def test_get_content_subtype_missing(self): - msg = Message() - self.assertEqual(msg.get_content_subtype(), 'plain') - - def test_get_content_subtype_missing_with_default_type(self): - msg = Message() - msg.set_default_type('message/rfc822') - self.assertEqual(msg.get_content_subtype(), 'rfc822') - - def test_get_content_subtype_from_message_implicit(self): - msg = self._msgobj('msg_30.txt') - self.assertEqual(msg.get_payload(0).get_content_subtype(), 'rfc822') - - def test_get_content_subtype_from_message_explicit(self): - msg = self._msgobj('msg_28.txt') - self.assertEqual(msg.get_payload(0).get_content_subtype(), 'rfc822') - - def test_get_content_subtype_from_message_text_plain_implicit(self): - msg = self._msgobj('msg_03.txt') - self.assertEqual(msg.get_content_subtype(), 'plain') - - def test_get_content_subtype_from_message_text_plain_explicit(self): - msg = self._msgobj('msg_01.txt') - self.assertEqual(msg.get_content_subtype(), 'plain') - - def test_get_content_maintype_error(self): - msg = Message() - msg['Content-Type'] = 'no-slash-in-this-string' - self.assertEqual(msg.get_content_maintype(), 'text') - - def test_get_content_subtype_error(self): - msg = Message() - msg['Content-Type'] = 'no-slash-in-this-string' - self.assertEqual(msg.get_content_subtype(), 'plain') - - def test_replace_header(self): - eq = self.assertEqual - msg = Message() - msg.add_header('First', 'One') - msg.add_header('Second', 'Two') - msg.add_header('Third', 'Three') - eq(msg.keys(), ['First', 'Second', 'Third']) - eq(msg.values(), ['One', 'Two', 'Three']) - msg.replace_header('Second', 'Twenty') - eq(msg.keys(), ['First', 'Second', 'Third']) - eq(msg.values(), ['One', 'Twenty', 'Three']) - msg.add_header('First', 'Eleven') - msg.replace_header('First', 'One Hundred') - eq(msg.keys(), ['First', 'Second', 'Third', 'First']) - eq(msg.values(), ['One Hundred', 'Twenty', 'Three', 'Eleven']) - self.assertRaises(KeyError, msg.replace_header, 'Fourth', 'Missing') - - def test_broken_base64_payload(self): - x = 'AwDp0P7//y6LwKEAcPa/6Q=9' - msg = Message() - msg['content-type'] = 'audio/x-midi' - msg['content-transfer-encoding'] = 'base64' - msg.set_payload(x) - self.assertEqual(msg.get_payload(decode=True), x) - - def test_get_content_charset(self): - msg = Message() - msg.set_charset('us-ascii') - self.assertEqual('us-ascii', msg.get_content_charset()) - msg.set_charset(u'us-ascii') - self.assertEqual('us-ascii', msg.get_content_charset()) - - # Issue 5871: reject an attempt to embed a header inside a header value - # (header injection attack). - def test_embeded_header_via_Header_rejected(self): - msg = Message() - msg['Dummy'] = Header('dummy\nX-Injected-Header: test') - self.assertRaises(Errors.HeaderParseError, msg.as_string) - - def test_embeded_header_via_string_rejected(self): - msg = Message() - msg['Dummy'] = 'dummy\nX-Injected-Header: test' - self.assertRaises(Errors.HeaderParseError, msg.as_string) - - -# Test the email.Encoders module -class TestEncoders(unittest.TestCase): - def test_encode_empty_payload(self): - eq = self.assertEqual - msg = Message() - msg.set_charset('us-ascii') - eq(msg['content-transfer-encoding'], '7bit') - - def test_default_cte(self): - eq = self.assertEqual - # 7bit data and the default us-ascii _charset - msg = MIMEText('hello world') - eq(msg['content-transfer-encoding'], '7bit') - # Similar, but with 8bit data - msg = MIMEText('hello \xf8 world') - eq(msg['content-transfer-encoding'], '8bit') - # And now with a different charset - msg = MIMEText('hello \xf8 world', _charset='iso-8859-1') - eq(msg['content-transfer-encoding'], 'quoted-printable') - - @impl_detail("PyPy has no cjkc codec yet", pypy=False) - def test_encode7or8bit(self): - # Make sure a charset whose input character set is 8bit but - # whose output character set is 7bit gets a transfer-encoding - # of 7bit. - eq = self.assertEqual - msg = email.MIMEText.MIMEText('\xca\xb8', _charset='euc-jp') - eq(msg['content-transfer-encoding'], '7bit') - - -# Test long header wrapping -class TestLongHeaders(TestEmailBase): - def test_split_long_continuation(self): - eq = self.ndiffAssertEqual - msg = email.message_from_string("""\ -Subject: bug demonstration -\t12345678911234567892123456789312345678941234567895123456789612345678971234567898112345678911234567892123456789112345678911234567892123456789 -\tmore text - -test -""") - sfp = StringIO() - g = Generator(sfp) - g.flatten(msg) - eq(sfp.getvalue(), """\ -Subject: bug demonstration - 12345678911234567892123456789312345678941234567895123456789612345678971234567898112345678911234567892123456789112345678911234567892123456789 - more text - -test -""") - - def test_another_long_almost_unsplittable_header(self): - eq = self.ndiffAssertEqual - hstr = """\ -bug demonstration -\t12345678911234567892123456789312345678941234567895123456789612345678971234567898112345678911234567892123456789112345678911234567892123456789 -\tmore text""" - h = Header(hstr, continuation_ws='\t') - eq(h.encode(), """\ -bug demonstration -\t12345678911234567892123456789312345678941234567895123456789612345678971234567898112345678911234567892123456789112345678911234567892123456789 -\tmore text""") - h = Header(hstr) - eq(h.encode(), """\ -bug demonstration - 12345678911234567892123456789312345678941234567895123456789612345678971234567898112345678911234567892123456789112345678911234567892123456789 - more text""") - - def test_long_nonstring(self): - eq = self.ndiffAssertEqual - g = Charset("iso-8859-1") - cz = Charset("iso-8859-2") - utf8 = Charset("utf-8") - g_head = "Die Mieter treten hier ein werden mit einem Foerderband komfortabel den Korridor entlang, an s\xfcdl\xfcndischen Wandgem\xe4lden vorbei, gegen die rotierenden Klingen bef\xf6rdert. " - cz_head = "Finan\xe8ni metropole se hroutily pod tlakem jejich d\xf9vtipu.. " - utf8_head = u"\u6b63\u78ba\u306b\u8a00\u3046\u3068\u7ffb\u8a33\u306f\u3055\u308c\u3066\u3044\u307e\u305b\u3093\u3002\u4e00\u90e8\u306f\u30c9\u30a4\u30c4\u8a9e\u3067\u3059\u304c\u3001\u3042\u3068\u306f\u3067\u305f\u3089\u3081\u3067\u3059\u3002\u5b9f\u969b\u306b\u306f\u300cWenn ist das Nunstuck git und Slotermeyer? Ja! Beiherhund das Oder die Flipperwaldt gersput.\u300d\u3068\u8a00\u3063\u3066\u3044\u307e\u3059\u3002".encode("utf-8") - h = Header(g_head, g, header_name='Subject') - h.append(cz_head, cz) - h.append(utf8_head, utf8) - msg = Message() - msg['Subject'] = h - sfp = StringIO() - g = Generator(sfp) - g.flatten(msg) - eq(sfp.getvalue(), """\ -Subject: =?iso-8859-1?q?Die_Mieter_treten_hier_ein_werden_mit_einem_Foerd?= - =?iso-8859-1?q?erband_komfortabel_den_Korridor_entlang=2C_an_s=FCdl=FCndi?= - =?iso-8859-1?q?schen_Wandgem=E4lden_vorbei=2C_gegen_die_rotierenden_Kling?= - =?iso-8859-1?q?en_bef=F6rdert=2E_?= =?iso-8859-2?q?Finan=E8ni_met?= - =?iso-8859-2?q?ropole_se_hroutily_pod_tlakem_jejich_d=F9vtipu=2E=2E_?= - =?utf-8?b?5q2j56K644Gr6KiA44GG44Go57+76Kiz44Gv44GV44KM44Gm44GE?= - =?utf-8?b?44G+44Gb44KT44CC5LiA6YOo44Gv44OJ44Kk44OE6Kqe44Gn44GZ44GM44CB?= - =?utf-8?b?44GC44Go44Gv44Gn44Gf44KJ44KB44Gn44GZ44CC5a6f6Zqb44Gr44Gv44CM?= - =?utf-8?q?Wenn_ist_das_Nunstuck_git_und_Slotermeyer=3F_Ja!_Beiherhund_das?= - =?utf-8?b?IE9kZXIgZGllIEZsaXBwZXJ3YWxkdCBnZXJzcHV0LuOAjeOBqOiogOOBow==?= - =?utf-8?b?44Gm44GE44G+44GZ44CC?= - -""") - eq(h.encode(), """\ -=?iso-8859-1?q?Die_Mieter_treten_hier_ein_werden_mit_einem_Foerd?= - =?iso-8859-1?q?erband_komfortabel_den_Korridor_entlang=2C_an_s=FCdl=FCndi?= - =?iso-8859-1?q?schen_Wandgem=E4lden_vorbei=2C_gegen_die_rotierenden_Kling?= - =?iso-8859-1?q?en_bef=F6rdert=2E_?= =?iso-8859-2?q?Finan=E8ni_met?= - =?iso-8859-2?q?ropole_se_hroutily_pod_tlakem_jejich_d=F9vtipu=2E=2E_?= - =?utf-8?b?5q2j56K644Gr6KiA44GG44Go57+76Kiz44Gv44GV44KM44Gm44GE?= - =?utf-8?b?44G+44Gb44KT44CC5LiA6YOo44Gv44OJ44Kk44OE6Kqe44Gn44GZ44GM44CB?= - =?utf-8?b?44GC44Go44Gv44Gn44Gf44KJ44KB44Gn44GZ44CC5a6f6Zqb44Gr44Gv44CM?= - =?utf-8?q?Wenn_ist_das_Nunstuck_git_und_Slotermeyer=3F_Ja!_Beiherhund_das?= - =?utf-8?b?IE9kZXIgZGllIEZsaXBwZXJ3YWxkdCBnZXJzcHV0LuOAjeOBqOiogOOBow==?= - =?utf-8?b?44Gm44GE44G+44GZ44CC?=""") - - def test_long_header_encode(self): - eq = self.ndiffAssertEqual - h = Header('wasnipoop; giraffes="very-long-necked-animals"; ' - 'spooge="yummy"; hippos="gargantuan"; marshmallows="gooey"', - header_name='X-Foobar-Spoink-Defrobnit') - eq(h.encode(), '''\ -wasnipoop; giraffes="very-long-necked-animals"; - spooge="yummy"; hippos="gargantuan"; marshmallows="gooey"''') - - def test_long_header_encode_with_tab_continuation(self): - eq = self.ndiffAssertEqual - h = Header('wasnipoop; giraffes="very-long-necked-animals"; ' - 'spooge="yummy"; hippos="gargantuan"; marshmallows="gooey"', - header_name='X-Foobar-Spoink-Defrobnit', - continuation_ws='\t') - eq(h.encode(), '''\ -wasnipoop; giraffes="very-long-necked-animals"; -\tspooge="yummy"; hippos="gargantuan"; marshmallows="gooey"''') - - def test_header_splitter(self): - eq = self.ndiffAssertEqual - msg = MIMEText('') - # It'd be great if we could use add_header() here, but that doesn't - # guarantee an order of the parameters. - msg['X-Foobar-Spoink-Defrobnit'] = ( - 'wasnipoop; giraffes="very-long-necked-animals"; ' - 'spooge="yummy"; hippos="gargantuan"; marshmallows="gooey"') - sfp = StringIO() - g = Generator(sfp) - g.flatten(msg) - eq(sfp.getvalue(), '''\ -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit -X-Foobar-Spoink-Defrobnit: wasnipoop; giraffes="very-long-necked-animals"; - spooge="yummy"; hippos="gargantuan"; marshmallows="gooey" - -''') - - def test_no_semis_header_splitter(self): - eq = self.ndiffAssertEqual - msg = Message() - msg['From'] = 'test at dom.ain' - msg['References'] = SPACE.join(['<%d at dom.ain>' % i for i in range(10)]) - msg.set_payload('Test') - sfp = StringIO() - g = Generator(sfp) - g.flatten(msg) - eq(sfp.getvalue(), """\ -From: test at dom.ain -References: <0 at dom.ain> <1 at dom.ain> <2 at dom.ain> <3 at dom.ain> <4 at dom.ain> - <5 at dom.ain> <6 at dom.ain> <7 at dom.ain> <8 at dom.ain> <9 at dom.ain> - -Test""") - - def test_no_split_long_header(self): - eq = self.ndiffAssertEqual - hstr = 'References: ' + 'x' * 80 - h = Header(hstr, continuation_ws='\t') - eq(h.encode(), """\ -References: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx""") - - def test_splitting_multiple_long_lines(self): - eq = self.ndiffAssertEqual - hstr = """\ -from babylon.socal-raves.org (localhost [127.0.0.1]); by babylon.socal-raves.org (Postfix) with ESMTP id B570E51B81; for <mailman-admin at babylon.socal-raves.org>; Sat, 2 Feb 2002 17:00:06 -0800 (PST) -\tfrom babylon.socal-raves.org (localhost [127.0.0.1]); by babylon.socal-raves.org (Postfix) with ESMTP id B570E51B81; for <mailman-admin at babylon.socal-raves.org>; Sat, 2 Feb 2002 17:00:06 -0800 (PST) -\tfrom babylon.socal-raves.org (localhost [127.0.0.1]); by babylon.socal-raves.org (Postfix) with ESMTP id B570E51B81; for <mailman-admin at babylon.socal-raves.org>; Sat, 2 Feb 2002 17:00:06 -0800 (PST) -""" - h = Header(hstr, continuation_ws='\t') - eq(h.encode(), """\ -from babylon.socal-raves.org (localhost [127.0.0.1]); -\tby babylon.socal-raves.org (Postfix) with ESMTP id B570E51B81; -\tfor <mailman-admin at babylon.socal-raves.org>; -\tSat, 2 Feb 2002 17:00:06 -0800 (PST) -\tfrom babylon.socal-raves.org (localhost [127.0.0.1]); -\tby babylon.socal-raves.org (Postfix) with ESMTP id B570E51B81; -\tfor <mailman-admin at babylon.socal-raves.org>; -\tSat, 2 Feb 2002 17:00:06 -0800 (PST) -\tfrom babylon.socal-raves.org (localhost [127.0.0.1]); -\tby babylon.socal-raves.org (Postfix) with ESMTP id B570E51B81; -\tfor <mailman-admin at babylon.socal-raves.org>; -\tSat, 2 Feb 2002 17:00:06 -0800 (PST)""") - - def test_splitting_first_line_only_is_long(self): - eq = self.ndiffAssertEqual - hstr = """\ -from modemcable093.139-201-24.que.mc.videotron.ca ([24.201.139.93] helo=cthulhu.gerg.ca) -\tby kronos.mems-exchange.org with esmtp (Exim 4.05) -\tid 17k4h5-00034i-00 -\tfor test at mems-exchange.org; Wed, 28 Aug 2002 11:25:20 -0400""" - h = Header(hstr, maxlinelen=78, header_name='Received', - continuation_ws='\t') - eq(h.encode(), """\ -from modemcable093.139-201-24.que.mc.videotron.ca ([24.201.139.93] -\thelo=cthulhu.gerg.ca) -\tby kronos.mems-exchange.org with esmtp (Exim 4.05) -\tid 17k4h5-00034i-00 -\tfor test at mems-exchange.org; Wed, 28 Aug 2002 11:25:20 -0400""") - - def test_long_8bit_header(self): - eq = self.ndiffAssertEqual - msg = Message() - h = Header('Britische Regierung gibt', 'iso-8859-1', - header_name='Subject') - h.append('gr\xfcnes Licht f\xfcr Offshore-Windkraftprojekte') - msg['Subject'] = h - eq(msg.as_string(), """\ -Subject: =?iso-8859-1?q?Britische_Regierung_gibt?= =?iso-8859-1?q?gr=FCnes?= - =?iso-8859-1?q?_Licht_f=FCr_Offshore-Windkraftprojekte?= - -""") - - def test_long_8bit_header_no_charset(self): - eq = self.ndiffAssertEqual - msg = Message() - msg['Reply-To'] = 'Britische Regierung gibt gr\xfcnes Licht f\xfcr Offshore-Windkraftprojekte <a-very-long-address at example.com>' - eq(msg.as_string(), """\ -Reply-To: Britische Regierung gibt gr\xfcnes Licht f\xfcr Offshore-Windkraftprojekte <a-very-long-address at example.com> - -""") - - def test_long_to_header(self): - eq = self.ndiffAssertEqual - to = '"Someone Test #A" <someone at eecs.umich.edu>,<someone at eecs.umich.edu>,"Someone Test #B" <someone at umich.edu>, "Someone Test #C" <someone at eecs.umich.edu>, "Someone Test #D" <someone at eecs.umich.edu>' - msg = Message() - msg['To'] = to - eq(msg.as_string(0), '''\ -To: "Someone Test #A" <someone at eecs.umich.edu>, <someone at eecs.umich.edu>, - "Someone Test #B" <someone at umich.edu>, - "Someone Test #C" <someone at eecs.umich.edu>, - "Someone Test #D" <someone at eecs.umich.edu> - -''') - - def test_long_line_after_append(self): - eq = self.ndiffAssertEqual - s = 'This is an example of string which has almost the limit of header length.' - h = Header(s) - h.append('Add another line.') - eq(h.encode(), """\ -This is an example of string which has almost the limit of header length. - Add another line.""") - - def test_shorter_line_with_append(self): - eq = self.ndiffAssertEqual - s = 'This is a shorter line.' - h = Header(s) - h.append('Add another sentence. (Surprise?)') - eq(h.encode(), - 'This is a shorter line. Add another sentence. (Surprise?)') - - def test_long_field_name(self): - eq = self.ndiffAssertEqual - fn = 'X-Very-Very-Very-Long-Header-Name' - gs = "Die Mieter treten hier ein werden mit einem Foerderband komfortabel den Korridor entlang, an s\xfcdl\xfcndischen Wandgem\xe4lden vorbei, gegen die rotierenden Klingen bef\xf6rdert. " - h = Header(gs, 'iso-8859-1', header_name=fn) - # BAW: this seems broken because the first line is too long - eq(h.encode(), """\ -=?iso-8859-1?q?Die_Mieter_treten_hier_?= - =?iso-8859-1?q?ein_werden_mit_einem_Foerderband_komfortabel_den_Korridor_?= - =?iso-8859-1?q?entlang=2C_an_s=FCdl=FCndischen_Wandgem=E4lden_vorbei=2C_g?= - =?iso-8859-1?q?egen_die_rotierenden_Klingen_bef=F6rdert=2E_?=""") - - def test_long_received_header(self): - h = 'from FOO.TLD (vizworld.acl.foo.tld [123.452.678.9]) by hrothgar.la.mastaler.com (tmda-ofmipd) with ESMTP; Wed, 05 Mar 2003 18:10:18 -0700' - msg = Message() - msg['Received-1'] = Header(h, continuation_ws='\t') - msg['Received-2'] = h - self.assertEqual(msg.as_string(), """\ -Received-1: from FOO.TLD (vizworld.acl.foo.tld [123.452.678.9]) by -\throthgar.la.mastaler.com (tmda-ofmipd) with ESMTP; -\tWed, 05 Mar 2003 18:10:18 -0700 -Received-2: from FOO.TLD (vizworld.acl.foo.tld [123.452.678.9]) by - hrothgar.la.mastaler.com (tmda-ofmipd) with ESMTP; - Wed, 05 Mar 2003 18:10:18 -0700 - -""") - - def test_string_headerinst_eq(self): - h = '<15975.17901.207240.414604 at sgigritzmann1.mathematik.tu-muenchen.de> (David Bremner\'s message of "Thu, 6 Mar 2003 13:58:21 +0100")' - msg = Message() - msg['Received'] = Header(h, header_name='Received', - continuation_ws='\t') - msg['Received'] = h - self.ndiffAssertEqual(msg.as_string(), """\ -Received: <15975.17901.207240.414604 at sgigritzmann1.mathematik.tu-muenchen.de> -\t(David Bremner's message of "Thu, 6 Mar 2003 13:58:21 +0100") -Received: <15975.17901.207240.414604 at sgigritzmann1.mathematik.tu-muenchen.de> - (David Bremner's message of "Thu, 6 Mar 2003 13:58:21 +0100") - -""") - - def test_long_unbreakable_lines_with_continuation(self): - eq = self.ndiffAssertEqual - msg = Message() - t = """\ - iVBORw0KGgoAAAANSUhEUgAAADAAAAAwBAMAAAClLOS0AAAAGFBMVEUAAAAkHiJeRUIcGBi9 - locQDQ4zJykFBAXJfWDjAAACYUlEQVR4nF2TQY/jIAyFc6lydlG5x8Nyp1Y69wj1PN2I5gzp""" - msg['Face-1'] = t - msg['Face-2'] = Header(t, header_name='Face-2') - eq(msg.as_string(), """\ -Face-1: iVBORw0KGgoAAAANSUhEUgAAADAAAAAwBAMAAAClLOS0AAAAGFBMVEUAAAAkHiJeRUIcGBi9 - locQDQ4zJykFBAXJfWDjAAACYUlEQVR4nF2TQY/jIAyFc6lydlG5x8Nyp1Y69wj1PN2I5gzp -Face-2: iVBORw0KGgoAAAANSUhEUgAAADAAAAAwBAMAAAClLOS0AAAAGFBMVEUAAAAkHiJeRUIcGBi9 - locQDQ4zJykFBAXJfWDjAAACYUlEQVR4nF2TQY/jIAyFc6lydlG5x8Nyp1Y69wj1PN2I5gzp - -""") - - def test_another_long_multiline_header(self): - eq = self.ndiffAssertEqual - m = '''\ -Received: from siimage.com ([172.25.1.3]) by zima.siliconimage.com with Microsoft SMTPSVC(5.0.2195.4905); - Wed, 16 Oct 2002 07:41:11 -0700''' - msg = email.message_from_string(m) - eq(msg.as_string(), '''\ -Received: from siimage.com ([172.25.1.3]) by zima.siliconimage.com with - Microsoft SMTPSVC(5.0.2195.4905); Wed, 16 Oct 2002 07:41:11 -0700 - -''') - - def test_long_lines_with_different_header(self): - eq = self.ndiffAssertEqual - h = """\ -List-Unsubscribe: <https://lists.sourceforge.net/lists/listinfo/spamassassin-talk>, - <mailto:spamassassin-talk-request at lists.sourceforge.net?subject=unsubscribe>""" - msg = Message() - msg['List'] = h - msg['List'] = Header(h, header_name='List') - eq(msg.as_string(), """\ -List: List-Unsubscribe: <https://lists.sourceforge.net/lists/listinfo/spamassassin-talk>, - <mailto:spamassassin-talk-request at lists.sourceforge.net?subject=unsubscribe> -List: List-Unsubscribe: <https://lists.sourceforge.net/lists/listinfo/spamassassin-talk>, - <mailto:spamassassin-talk-request at lists.sourceforge.net?subject=unsubscribe> - -""") - - - -# Test mangling of "From " lines in the body of a message -class TestFromMangling(unittest.TestCase): - def setUp(self): - self.msg = Message() - self.msg['From'] = 'aaa at bbb.org' - self.msg.set_payload("""\ -From the desk of A.A.A.: -Blah blah blah -""") - - def test_mangled_from(self): - s = StringIO() - g = Generator(s, mangle_from_=True) - g.flatten(self.msg) - self.assertEqual(s.getvalue(), """\ -From: aaa at bbb.org - ->From the desk of A.A.A.: -Blah blah blah -""") - - def test_dont_mangle_from(self): - s = StringIO() - g = Generator(s, mangle_from_=False) - g.flatten(self.msg) - self.assertEqual(s.getvalue(), """\ -From: aaa at bbb.org - -From the desk of A.A.A.: -Blah blah blah -""") - - - -# Test the basic MIMEAudio class -class TestMIMEAudio(unittest.TestCase): - def setUp(self): - # Make sure we pick up the audiotest.au that lives in email/test/data. - # In Python, there's an audiotest.au living in Lib/test but that isn't - # included in some binary distros that don't include the test - # package. The trailing empty string on the .join() is significant - # since findfile() will do a dirname(). - datadir = os.path.join(os.path.dirname(landmark), 'data', '') - fp = open(findfile('audiotest.au', datadir), 'rb') - try: - self._audiodata = fp.read() - finally: - fp.close() - self._au = MIMEAudio(self._audiodata) - - def test_guess_minor_type(self): - self.assertEqual(self._au.get_content_type(), 'audio/basic') - - def test_encoding(self): - payload = self._au.get_payload() - self.assertEqual(base64.decodestring(payload), self._audiodata) - - def test_checkSetMinor(self): - au = MIMEAudio(self._audiodata, 'fish') - self.assertEqual(au.get_content_type(), 'audio/fish') - - def test_add_header(self): - eq = self.assertEqual - unless = self.assertTrue - self._au.add_header('Content-Disposition', 'attachment', - filename='audiotest.au') - eq(self._au['content-disposition'], - 'attachment; filename="audiotest.au"') - eq(self._au.get_params(header='content-disposition'), - [('attachment', ''), ('filename', 'audiotest.au')]) - eq(self._au.get_param('filename', header='content-disposition'), - 'audiotest.au') - missing = [] - eq(self._au.get_param('attachment', header='content-disposition'), '') - unless(self._au.get_param('foo', failobj=missing, - header='content-disposition') is missing) - # Try some missing stuff - unless(self._au.get_param('foobar', missing) is missing) - unless(self._au.get_param('attachment', missing, - header='foobar') is missing) - - - -# Test the basic MIMEImage class -class TestMIMEImage(unittest.TestCase): - def setUp(self): - fp = openfile('PyBanner048.gif') - try: - self._imgdata = fp.read() - finally: - fp.close() - self._im = MIMEImage(self._imgdata) - - def test_guess_minor_type(self): - self.assertEqual(self._im.get_content_type(), 'image/gif') - - def test_encoding(self): - payload = self._im.get_payload() - self.assertEqual(base64.decodestring(payload), self._imgdata) - - def test_checkSetMinor(self): - im = MIMEImage(self._imgdata, 'fish') - self.assertEqual(im.get_content_type(), 'image/fish') - - def test_add_header(self): - eq = self.assertEqual - unless = self.assertTrue - self._im.add_header('Content-Disposition', 'attachment', - filename='dingusfish.gif') - eq(self._im['content-disposition'], - 'attachment; filename="dingusfish.gif"') - eq(self._im.get_params(header='content-disposition'), - [('attachment', ''), ('filename', 'dingusfish.gif')]) - eq(self._im.get_param('filename', header='content-disposition'), - 'dingusfish.gif') - missing = [] - eq(self._im.get_param('attachment', header='content-disposition'), '') - unless(self._im.get_param('foo', failobj=missing, - header='content-disposition') is missing) - # Try some missing stuff - unless(self._im.get_param('foobar', missing) is missing) - unless(self._im.get_param('attachment', missing, - header='foobar') is missing) - - - -# Test the basic MIMEText class -class TestMIMEText(unittest.TestCase): - def setUp(self): - self._msg = MIMEText('hello there') - - def test_types(self): - eq = self.assertEqual - unless = self.assertTrue - eq(self._msg.get_content_type(), 'text/plain') - eq(self._msg.get_param('charset'), 'us-ascii') - missing = [] - unless(self._msg.get_param('foobar', missing) is missing) - unless(self._msg.get_param('charset', missing, header='foobar') - is missing) - - def test_payload(self): - self.assertEqual(self._msg.get_payload(), 'hello there') - self.assertTrue(not self._msg.is_multipart()) - - def test_charset(self): - eq = self.assertEqual - msg = MIMEText('hello there', _charset='us-ascii') - eq(msg.get_charset().input_charset, 'us-ascii') - eq(msg['content-type'], 'text/plain; charset="us-ascii"') - - def test_7bit_unicode_input(self): - eq = self.assertEqual - msg = MIMEText(u'hello there', _charset='us-ascii') - eq(msg.get_charset().input_charset, 'us-ascii') - eq(msg['content-type'], 'text/plain; charset="us-ascii"') - - def test_7bit_unicode_input_no_charset(self): - eq = self.assertEqual - msg = MIMEText(u'hello there') - eq(msg.get_charset(), 'us-ascii') - eq(msg['content-type'], 'text/plain; charset="us-ascii"') - self.assertTrue('hello there' in msg.as_string()) - - def test_8bit_unicode_input(self): - teststr = u'\u043a\u0438\u0440\u0438\u043b\u0438\u0446\u0430' - eq = self.assertEqual - msg = MIMEText(teststr, _charset='utf-8') - eq(msg.get_charset().output_charset, 'utf-8') - eq(msg['content-type'], 'text/plain; charset="utf-8"') - eq(msg.get_payload(decode=True), teststr.encode('utf-8')) - - def test_8bit_unicode_input_no_charset(self): - teststr = u'\u043a\u0438\u0440\u0438\u043b\u0438\u0446\u0430' - self.assertRaises(UnicodeEncodeError, MIMEText, teststr) - - - -# Test complicated multipart/* messages -class TestMultipart(TestEmailBase): - def setUp(self): - fp = openfile('PyBanner048.gif') - try: - data = fp.read() - finally: - fp.close() - - container = MIMEBase('multipart', 'mixed', boundary='BOUNDARY') - image = MIMEImage(data, name='dingusfish.gif') - image.add_header('content-disposition', 'attachment', - filename='dingusfish.gif') - intro = MIMEText('''\ -Hi there, - -This is the dingus fish. -''') - container.attach(intro) - container.attach(image) - container['From'] = 'Barry <barry at digicool.com>' - container['To'] = 'Dingus Lovers <cravindogs at cravindogs.com>' - container['Subject'] = 'Here is your dingus fish' - - now = 987809702.54848599 - timetuple = time.localtime(now) - if timetuple[-1] == 0: - tzsecs = time.timezone - else: - tzsecs = time.altzone - if tzsecs > 0: - sign = '-' - else: - sign = '+' - tzoffset = ' %s%04d' % (sign, tzsecs // 36) - container['Date'] = time.strftime( - '%a, %d %b %Y %H:%M:%S', - time.localtime(now)) + tzoffset - self._msg = container - self._im = image - self._txt = intro - - def test_hierarchy(self): - # convenience - eq = self.assertEqual - unless = self.assertTrue - raises = self.assertRaises - # tests - m = self._msg - unless(m.is_multipart()) - eq(m.get_content_type(), 'multipart/mixed') - eq(len(m.get_payload()), 2) - raises(IndexError, m.get_payload, 2) - m0 = m.get_payload(0) - m1 = m.get_payload(1) - unless(m0 is self._txt) - unless(m1 is self._im) - eq(m.get_payload(), [m0, m1]) - unless(not m0.is_multipart()) - unless(not m1.is_multipart()) - - def test_empty_multipart_idempotent(self): - text = """\ -Content-Type: multipart/mixed; boundary="BOUNDARY" -MIME-Version: 1.0 -Subject: A subject -To: aperson at dom.ain -From: bperson at dom.ain - - ---BOUNDARY - - ---BOUNDARY-- -""" - msg = Parser().parsestr(text) - self.ndiffAssertEqual(text, msg.as_string()) - - def test_no_parts_in_a_multipart_with_none_epilogue(self): - outer = MIMEBase('multipart', 'mixed') - outer['Subject'] = 'A subject' - outer['To'] = 'aperson at dom.ain' - outer['From'] = 'bperson at dom.ain' - outer.set_boundary('BOUNDARY') - self.ndiffAssertEqual(outer.as_string(), '''\ -Content-Type: multipart/mixed; boundary="BOUNDARY" -MIME-Version: 1.0 -Subject: A subject -To: aperson at dom.ain -From: bperson at dom.ain - ---BOUNDARY - ---BOUNDARY--''') - - def test_no_parts_in_a_multipart_with_empty_epilogue(self): - outer = MIMEBase('multipart', 'mixed') - outer['Subject'] = 'A subject' - outer['To'] = 'aperson at dom.ain' - outer['From'] = 'bperson at dom.ain' - outer.preamble = '' - outer.epilogue = '' - outer.set_boundary('BOUNDARY') - self.ndiffAssertEqual(outer.as_string(), '''\ -Content-Type: multipart/mixed; boundary="BOUNDARY" -MIME-Version: 1.0 -Subject: A subject -To: aperson at dom.ain -From: bperson at dom.ain - - ---BOUNDARY - ---BOUNDARY-- -''') - - def test_one_part_in_a_multipart(self): - eq = self.ndiffAssertEqual - outer = MIMEBase('multipart', 'mixed') - outer['Subject'] = 'A subject' - outer['To'] = 'aperson at dom.ain' - outer['From'] = 'bperson at dom.ain' - outer.set_boundary('BOUNDARY') - msg = MIMEText('hello world') - outer.attach(msg) - eq(outer.as_string(), '''\ -Content-Type: multipart/mixed; boundary="BOUNDARY" -MIME-Version: 1.0 -Subject: A subject -To: aperson at dom.ain -From: bperson at dom.ain - ---BOUNDARY -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit - -hello world ---BOUNDARY--''') - - def test_seq_parts_in_a_multipart_with_empty_preamble(self): - eq = self.ndiffAssertEqual - outer = MIMEBase('multipart', 'mixed') - outer['Subject'] = 'A subject' - outer['To'] = 'aperson at dom.ain' - outer['From'] = 'bperson at dom.ain' - outer.preamble = '' - msg = MIMEText('hello world') - outer.attach(msg) - outer.set_boundary('BOUNDARY') - eq(outer.as_string(), '''\ -Content-Type: multipart/mixed; boundary="BOUNDARY" -MIME-Version: 1.0 -Subject: A subject -To: aperson at dom.ain -From: bperson at dom.ain - - ---BOUNDARY -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit - -hello world ---BOUNDARY--''') - - - def test_seq_parts_in_a_multipart_with_none_preamble(self): - eq = self.ndiffAssertEqual - outer = MIMEBase('multipart', 'mixed') - outer['Subject'] = 'A subject' - outer['To'] = 'aperson at dom.ain' - outer['From'] = 'bperson at dom.ain' - outer.preamble = None - msg = MIMEText('hello world') - outer.attach(msg) - outer.set_boundary('BOUNDARY') - eq(outer.as_string(), '''\ -Content-Type: multipart/mixed; boundary="BOUNDARY" -MIME-Version: 1.0 -Subject: A subject -To: aperson at dom.ain -From: bperson at dom.ain - ---BOUNDARY -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit - -hello world ---BOUNDARY--''') - - - def test_seq_parts_in_a_multipart_with_none_epilogue(self): - eq = self.ndiffAssertEqual - outer = MIMEBase('multipart', 'mixed') - outer['Subject'] = 'A subject' - outer['To'] = 'aperson at dom.ain' - outer['From'] = 'bperson at dom.ain' - outer.epilogue = None - msg = MIMEText('hello world') - outer.attach(msg) - outer.set_boundary('BOUNDARY') - eq(outer.as_string(), '''\ -Content-Type: multipart/mixed; boundary="BOUNDARY" -MIME-Version: 1.0 -Subject: A subject -To: aperson at dom.ain -From: bperson at dom.ain - ---BOUNDARY -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit - -hello world ---BOUNDARY--''') - - - def test_seq_parts_in_a_multipart_with_empty_epilogue(self): - eq = self.ndiffAssertEqual - outer = MIMEBase('multipart', 'mixed') - outer['Subject'] = 'A subject' - outer['To'] = 'aperson at dom.ain' - outer['From'] = 'bperson at dom.ain' - outer.epilogue = '' - msg = MIMEText('hello world') - outer.attach(msg) - outer.set_boundary('BOUNDARY') - eq(outer.as_string(), '''\ -Content-Type: multipart/mixed; boundary="BOUNDARY" -MIME-Version: 1.0 -Subject: A subject -To: aperson at dom.ain -From: bperson at dom.ain - ---BOUNDARY -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit - -hello world ---BOUNDARY-- -''') - - - def test_seq_parts_in_a_multipart_with_nl_epilogue(self): - eq = self.ndiffAssertEqual - outer = MIMEBase('multipart', 'mixed') - outer['Subject'] = 'A subject' - outer['To'] = 'aperson at dom.ain' - outer['From'] = 'bperson at dom.ain' - outer.epilogue = '\n' - msg = MIMEText('hello world') - outer.attach(msg) - outer.set_boundary('BOUNDARY') - eq(outer.as_string(), '''\ -Content-Type: multipart/mixed; boundary="BOUNDARY" -MIME-Version: 1.0 -Subject: A subject -To: aperson at dom.ain -From: bperson at dom.ain - ---BOUNDARY -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit - -hello world ---BOUNDARY-- - -''') - - def test_message_external_body(self): - eq = self.assertEqual - msg = self._msgobj('msg_36.txt') - eq(len(msg.get_payload()), 2) - msg1 = msg.get_payload(1) - eq(msg1.get_content_type(), 'multipart/alternative') - eq(len(msg1.get_payload()), 2) - for subpart in msg1.get_payload(): - eq(subpart.get_content_type(), 'message/external-body') - eq(len(subpart.get_payload()), 1) - subsubpart = subpart.get_payload(0) - eq(subsubpart.get_content_type(), 'text/plain') - - def test_double_boundary(self): - # msg_37.txt is a multipart that contains two dash-boundary's in a - # row. Our interpretation of RFC 2046 calls for ignoring the second - # and subsequent boundaries. - msg = self._msgobj('msg_37.txt') - self.assertEqual(len(msg.get_payload()), 3) - - def test_nested_inner_contains_outer_boundary(self): - eq = self.ndiffAssertEqual - # msg_38.txt has an inner part that contains outer boundaries. My - # interpretation of RFC 2046 (based on sections 5.1 and 5.1.2) say - # these are illegal and should be interpreted as unterminated inner - # parts. - msg = self._msgobj('msg_38.txt') - sfp = StringIO() - Iterators._structure(msg, sfp) - eq(sfp.getvalue(), """\ -multipart/mixed - multipart/mixed - multipart/alternative - text/plain - text/plain - text/plain - text/plain -""") - - def test_nested_with_same_boundary(self): - eq = self.ndiffAssertEqual - # msg 39.txt is similarly evil in that it's got inner parts that use - # the same boundary as outer parts. Again, I believe the way this is - # parsed is closest to the spirit of RFC 2046 - msg = self._msgobj('msg_39.txt') - sfp = StringIO() - Iterators._structure(msg, sfp) - eq(sfp.getvalue(), """\ -multipart/mixed - multipart/mixed - multipart/alternative - application/octet-stream - application/octet-stream - text/plain -""") - - def test_boundary_in_non_multipart(self): - msg = self._msgobj('msg_40.txt') - self.assertEqual(msg.as_string(), '''\ -MIME-Version: 1.0 -Content-Type: text/html; boundary="--961284236552522269" - -----961284236552522269 -Content-Type: text/html; -Content-Transfer-Encoding: 7Bit - -<html></html> - -----961284236552522269-- -''') - - def test_boundary_with_leading_space(self): - eq = self.assertEqual - msg = email.message_from_string('''\ -MIME-Version: 1.0 -Content-Type: multipart/mixed; boundary=" XXXX" - --- XXXX -Content-Type: text/plain - - --- XXXX -Content-Type: text/plain - --- XXXX-- -''') - self.assertTrue(msg.is_multipart()) - eq(msg.get_boundary(), ' XXXX') - eq(len(msg.get_payload()), 2) - - def test_boundary_without_trailing_newline(self): - m = Parser().parsestr("""\ -Content-Type: multipart/mixed; boundary="===============0012394164==" -MIME-Version: 1.0 - ---===============0012394164== -Content-Type: image/file1.jpg -MIME-Version: 1.0 -Content-Transfer-Encoding: base64 - -YXNkZg== ---===============0012394164==--""") - self.assertEqual(m.get_payload(0).get_payload(), 'YXNkZg==') - - - -# Test some badly formatted messages -class TestNonConformant(TestEmailBase): - def test_parse_missing_minor_type(self): - eq = self.assertEqual - msg = self._msgobj('msg_14.txt') - eq(msg.get_content_type(), 'text/plain') - eq(msg.get_content_maintype(), 'text') - eq(msg.get_content_subtype(), 'plain') - - def test_same_boundary_inner_outer(self): - unless = self.assertTrue - msg = self._msgobj('msg_15.txt') - # XXX We can probably eventually do better - inner = msg.get_payload(0) - unless(hasattr(inner, 'defects')) - self.assertEqual(len(inner.defects), 1) - unless(isinstance(inner.defects[0], - Errors.StartBoundaryNotFoundDefect)) - - def test_multipart_no_boundary(self): - unless = self.assertTrue - msg = self._msgobj('msg_25.txt') - unless(isinstance(msg.get_payload(), str)) - self.assertEqual(len(msg.defects), 2) - unless(isinstance(msg.defects[0], Errors.NoBoundaryInMultipartDefect)) - unless(isinstance(msg.defects[1], - Errors.MultipartInvariantViolationDefect)) - - def test_invalid_content_type(self): - eq = self.assertEqual - neq = self.ndiffAssertEqual - msg = Message() - # RFC 2045, $5.2 says invalid yields text/plain - msg['Content-Type'] = 'text' - eq(msg.get_content_maintype(), 'text') - eq(msg.get_content_subtype(), 'plain') - eq(msg.get_content_type(), 'text/plain') - # Clear the old value and try something /really/ invalid - del msg['content-type'] - msg['Content-Type'] = 'foo' - eq(msg.get_content_maintype(), 'text') - eq(msg.get_content_subtype(), 'plain') - eq(msg.get_content_type(), 'text/plain') - # Still, make sure that the message is idempotently generated - s = StringIO() - g = Generator(s) - g.flatten(msg) - neq(s.getvalue(), 'Content-Type: foo\n\n') - - def test_no_start_boundary(self): - eq = self.ndiffAssertEqual - msg = self._msgobj('msg_31.txt') - eq(msg.get_payload(), """\ ---BOUNDARY -Content-Type: text/plain - -message 1 - ---BOUNDARY -Content-Type: text/plain - -message 2 - ---BOUNDARY-- -""") - - def test_no_separating_blank_line(self): - eq = self.ndiffAssertEqual - msg = self._msgobj('msg_35.txt') - eq(msg.as_string(), """\ -From: aperson at dom.ain -To: bperson at dom.ain -Subject: here's something interesting - -counter to RFC 2822, there's no separating newline here -""") - - def test_lying_multipart(self): - unless = self.assertTrue - msg = self._msgobj('msg_41.txt') - unless(hasattr(msg, 'defects')) - self.assertEqual(len(msg.defects), 2) - unless(isinstance(msg.defects[0], Errors.NoBoundaryInMultipartDefect)) - unless(isinstance(msg.defects[1], - Errors.MultipartInvariantViolationDefect)) - - def test_missing_start_boundary(self): - outer = self._msgobj('msg_42.txt') - # The message structure is: - # - # multipart/mixed - # text/plain - # message/rfc822 - # multipart/mixed [*] - # - # [*] This message is missing its start boundary - bad = outer.get_payload(1).get_payload(0) - self.assertEqual(len(bad.defects), 1) - self.assertTrue(isinstance(bad.defects[0], - Errors.StartBoundaryNotFoundDefect)) - - def test_first_line_is_continuation_header(self): - eq = self.assertEqual - m = ' Line 1\nLine 2\nLine 3' - msg = email.message_from_string(m) - eq(msg.keys(), []) - eq(msg.get_payload(), 'Line 2\nLine 3') - eq(len(msg.defects), 1) - self.assertTrue(isinstance(msg.defects[0], - Errors.FirstHeaderLineIsContinuationDefect)) - eq(msg.defects[0].line, ' Line 1\n') - - - - -# Test RFC 2047 header encoding and decoding -class TestRFC2047(unittest.TestCase): - def test_rfc2047_multiline(self): - eq = self.assertEqual - s = """Re: =?mac-iceland?q?r=8Aksm=9Arg=8Cs?= baz - foo bar =?mac-iceland?q?r=8Aksm=9Arg=8Cs?=""" - dh = decode_header(s) - eq(dh, [ - ('Re:', None), - ('r\x8aksm\x9arg\x8cs', 'mac-iceland'), - ('baz foo bar', None), - ('r\x8aksm\x9arg\x8cs', 'mac-iceland')]) - eq(str(make_header(dh)), - """Re: =?mac-iceland?q?r=8Aksm=9Arg=8Cs?= baz foo bar - =?mac-iceland?q?r=8Aksm=9Arg=8Cs?=""") - - def test_whitespace_eater_unicode(self): - eq = self.assertEqual - s = '=?ISO-8859-1?Q?Andr=E9?= Pirard <pirard at dom.ain>' - dh = decode_header(s) - eq(dh, [('Andr\xe9', 'iso-8859-1'), ('Pirard <pirard at dom.ain>', None)]) - hu = unicode(make_header(dh)).encode('latin-1') - eq(hu, 'Andr\xe9 Pirard <pirard at dom.ain>') - - def test_whitespace_eater_unicode_2(self): - eq = self.assertEqual - s = 'The =?iso-8859-1?b?cXVpY2sgYnJvd24gZm94?= jumped over the =?iso-8859-1?b?bGF6eSBkb2c=?=' - dh = decode_header(s) - eq(dh, [('The', None), ('quick brown fox', 'iso-8859-1'), - ('jumped over the', None), ('lazy dog', 'iso-8859-1')]) - hu = make_header(dh).__unicode__() - eq(hu, u'The quick brown fox jumped over the lazy dog') - - def test_rfc2047_without_whitespace(self): - s = 'Sm=?ISO-8859-1?B?9g==?=rg=?ISO-8859-1?B?5Q==?=sbord' - dh = decode_header(s) - self.assertEqual(dh, [(s, None)]) - - def test_rfc2047_with_whitespace(self): - s = 'Sm =?ISO-8859-1?B?9g==?= rg =?ISO-8859-1?B?5Q==?= sbord' - dh = decode_header(s) - self.assertEqual(dh, [('Sm', None), ('\xf6', 'iso-8859-1'), - ('rg', None), ('\xe5', 'iso-8859-1'), - ('sbord', None)]) - - def test_rfc2047_B_bad_padding(self): - s = '=?iso-8859-1?B?%s?=' - data = [ # only test complete bytes - ('dm==', 'v'), ('dm=', 'v'), ('dm', 'v'), - ('dmk=', 'vi'), ('dmk', 'vi') - ] - for q, a in data: - dh = decode_header(s % q) - self.assertEqual(dh, [(a, 'iso-8859-1')]) - - def test_rfc2047_Q_invalid_digits(self): - # issue 10004. - s = '=?iso-8659-1?Q?andr=e9=zz?=' - self.assertEqual(decode_header(s), - [(b'andr\xe9=zz', 'iso-8659-1')]) - - -# Test the MIMEMessage class -class TestMIMEMessage(TestEmailBase): - def setUp(self): - fp = openfile('msg_11.txt') - try: - self._text = fp.read() - finally: - fp.close() - - def test_type_error(self): - self.assertRaises(TypeError, MIMEMessage, 'a plain string') - - def test_valid_argument(self): - eq = self.assertEqual - unless = self.assertTrue - subject = 'A sub-message' - m = Message() - m['Subject'] = subject - r = MIMEMessage(m) - eq(r.get_content_type(), 'message/rfc822') - payload = r.get_payload() - unless(isinstance(payload, list)) - eq(len(payload), 1) - subpart = payload[0] - unless(subpart is m) - eq(subpart['subject'], subject) - - def test_bad_multipart(self): - eq = self.assertEqual - msg1 = Message() - msg1['Subject'] = 'subpart 1' - msg2 = Message() - msg2['Subject'] = 'subpart 2' - r = MIMEMessage(msg1) - self.assertRaises(Errors.MultipartConversionError, r.attach, msg2) - - def test_generate(self): - # First craft the message to be encapsulated - m = Message() - m['Subject'] = 'An enclosed message' - m.set_payload('Here is the body of the message.\n') - r = MIMEMessage(m) - r['Subject'] = 'The enclosing message' - s = StringIO() - g = Generator(s) - g.flatten(r) - self.assertEqual(s.getvalue(), """\ -Content-Type: message/rfc822 -MIME-Version: 1.0 -Subject: The enclosing message - -Subject: An enclosed message - -Here is the body of the message. -""") - - def test_parse_message_rfc822(self): - eq = self.assertEqual - unless = self.assertTrue - msg = self._msgobj('msg_11.txt') - eq(msg.get_content_type(), 'message/rfc822') - payload = msg.get_payload() - unless(isinstance(payload, list)) - eq(len(payload), 1) - submsg = payload[0] - self.assertTrue(isinstance(submsg, Message)) - eq(submsg['subject'], 'An enclosed message') - eq(submsg.get_payload(), 'Here is the body of the message.\n') - - def test_dsn(self): - eq = self.assertEqual - unless = self.assertTrue - # msg 16 is a Delivery Status Notification, see RFC 1894 - msg = self._msgobj('msg_16.txt') - eq(msg.get_content_type(), 'multipart/report') - unless(msg.is_multipart()) - eq(len(msg.get_payload()), 3) - # Subpart 1 is a text/plain, human readable section - subpart = msg.get_payload(0) - eq(subpart.get_content_type(), 'text/plain') - eq(subpart.get_payload(), """\ -This report relates to a message you sent with the following header fields: - - Message-id: <002001c144a6$8752e060$56104586 at oxy.edu> - Date: Sun, 23 Sep 2001 20:10:55 -0700 - From: "Ian T. Henry" <henryi at oxy.edu> - To: SoCal Raves <scr at socal-raves.org> - Subject: [scr] yeah for Ians!! - -Your message cannot be delivered to the following recipients: - - Recipient address: jangel1 at cougar.noc.ucla.edu - Reason: recipient reached disk quota - -""") - # Subpart 2 contains the machine parsable DSN information. It - # consists of two blocks of headers, represented by two nested Message - # objects. - subpart = msg.get_payload(1) - eq(subpart.get_content_type(), 'message/delivery-status') - eq(len(subpart.get_payload()), 2) - # message/delivery-status should treat each block as a bunch of - # headers, i.e. a bunch of Message objects. - dsn1 = subpart.get_payload(0) - unless(isinstance(dsn1, Message)) - eq(dsn1['original-envelope-id'], '0GK500B4HD0888 at cougar.noc.ucla.edu') - eq(dsn1.get_param('dns', header='reporting-mta'), '') - # Try a missing one <wink> - eq(dsn1.get_param('nsd', header='reporting-mta'), None) - dsn2 = subpart.get_payload(1) - unless(isinstance(dsn2, Message)) - eq(dsn2['action'], 'failed') - eq(dsn2.get_params(header='original-recipient'), - [('rfc822', ''), ('jangel1 at cougar.noc.ucla.edu', '')]) - eq(dsn2.get_param('rfc822', header='final-recipient'), '') - # Subpart 3 is the original message - subpart = msg.get_payload(2) - eq(subpart.get_content_type(), 'message/rfc822') - payload = subpart.get_payload() - unless(isinstance(payload, list)) - eq(len(payload), 1) - subsubpart = payload[0] - unless(isinstance(subsubpart, Message)) - eq(subsubpart.get_content_type(), 'text/plain') - eq(subsubpart['message-id'], - '<002001c144a6$8752e060$56104586 at oxy.edu>') - - def test_epilogue(self): - eq = self.ndiffAssertEqual - fp = openfile('msg_21.txt') - try: - text = fp.read() - finally: - fp.close() - msg = Message() - msg['From'] = 'aperson at dom.ain' - msg['To'] = 'bperson at dom.ain' - msg['Subject'] = 'Test' - msg.preamble = 'MIME message' - msg.epilogue = 'End of MIME message\n' - msg1 = MIMEText('One') - msg2 = MIMEText('Two') - msg.add_header('Content-Type', 'multipart/mixed', boundary='BOUNDARY') - msg.attach(msg1) - msg.attach(msg2) - sfp = StringIO() - g = Generator(sfp) - g.flatten(msg) - eq(sfp.getvalue(), text) - - def test_no_nl_preamble(self): - eq = self.ndiffAssertEqual - msg = Message() - msg['From'] = 'aperson at dom.ain' - msg['To'] = 'bperson at dom.ain' - msg['Subject'] = 'Test' - msg.preamble = 'MIME message' - msg.epilogue = '' - msg1 = MIMEText('One') - msg2 = MIMEText('Two') - msg.add_header('Content-Type', 'multipart/mixed', boundary='BOUNDARY') - msg.attach(msg1) - msg.attach(msg2) - eq(msg.as_string(), """\ -From: aperson at dom.ain -To: bperson at dom.ain -Subject: Test -Content-Type: multipart/mixed; boundary="BOUNDARY" - -MIME message ---BOUNDARY -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit - -One ---BOUNDARY -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit - -Two ---BOUNDARY-- -""") - - def test_default_type(self): - eq = self.assertEqual - fp = openfile('msg_30.txt') - try: - msg = email.message_from_file(fp) - finally: - fp.close() - container1 = msg.get_payload(0) - eq(container1.get_default_type(), 'message/rfc822') - eq(container1.get_content_type(), 'message/rfc822') - container2 = msg.get_payload(1) - eq(container2.get_default_type(), 'message/rfc822') - eq(container2.get_content_type(), 'message/rfc822') - container1a = container1.get_payload(0) - eq(container1a.get_default_type(), 'text/plain') - eq(container1a.get_content_type(), 'text/plain') - container2a = container2.get_payload(0) - eq(container2a.get_default_type(), 'text/plain') - eq(container2a.get_content_type(), 'text/plain') - - def test_default_type_with_explicit_container_type(self): - eq = self.assertEqual - fp = openfile('msg_28.txt') - try: - msg = email.message_from_file(fp) - finally: - fp.close() - container1 = msg.get_payload(0) - eq(container1.get_default_type(), 'message/rfc822') - eq(container1.get_content_type(), 'message/rfc822') - container2 = msg.get_payload(1) - eq(container2.get_default_type(), 'message/rfc822') - eq(container2.get_content_type(), 'message/rfc822') - container1a = container1.get_payload(0) - eq(container1a.get_default_type(), 'text/plain') - eq(container1a.get_content_type(), 'text/plain') - container2a = container2.get_payload(0) - eq(container2a.get_default_type(), 'text/plain') - eq(container2a.get_content_type(), 'text/plain') - - def test_default_type_non_parsed(self): - eq = self.assertEqual - neq = self.ndiffAssertEqual - # Set up container - container = MIMEMultipart('digest', 'BOUNDARY') - container.epilogue = '' - # Set up subparts - subpart1a = MIMEText('message 1\n') - subpart2a = MIMEText('message 2\n') - subpart1 = MIMEMessage(subpart1a) - subpart2 = MIMEMessage(subpart2a) - container.attach(subpart1) - container.attach(subpart2) - eq(subpart1.get_content_type(), 'message/rfc822') - eq(subpart1.get_default_type(), 'message/rfc822') - eq(subpart2.get_content_type(), 'message/rfc822') - eq(subpart2.get_default_type(), 'message/rfc822') - neq(container.as_string(0), '''\ -Content-Type: multipart/digest; boundary="BOUNDARY" -MIME-Version: 1.0 - ---BOUNDARY -Content-Type: message/rfc822 -MIME-Version: 1.0 - -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit - -message 1 - ---BOUNDARY -Content-Type: message/rfc822 -MIME-Version: 1.0 - -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit - -message 2 - ---BOUNDARY-- -''') - del subpart1['content-type'] - del subpart1['mime-version'] - del subpart2['content-type'] - del subpart2['mime-version'] - eq(subpart1.get_content_type(), 'message/rfc822') - eq(subpart1.get_default_type(), 'message/rfc822') - eq(subpart2.get_content_type(), 'message/rfc822') - eq(subpart2.get_default_type(), 'message/rfc822') - neq(container.as_string(0), '''\ -Content-Type: multipart/digest; boundary="BOUNDARY" -MIME-Version: 1.0 - ---BOUNDARY - -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit - -message 1 - ---BOUNDARY - -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit - -message 2 - ---BOUNDARY-- -''') - - def test_mime_attachments_in_constructor(self): - eq = self.assertEqual - text1 = MIMEText('') - text2 = MIMEText('') - msg = MIMEMultipart(_subparts=(text1, text2)) - eq(len(msg.get_payload()), 2) - eq(msg.get_payload(0), text1) - eq(msg.get_payload(1), text2) - - def test_default_multipart_constructor(self): - msg = MIMEMultipart() - self.assertTrue(msg.is_multipart()) - - -# A general test of parser->model->generator idempotency. IOW, read a message -# in, parse it into a message object tree, then without touching the tree, -# regenerate the plain text. The original text and the transformed text -# should be identical. Note: that we ignore the Unix-From since that may -# contain a changed date. -class TestIdempotent(TestEmailBase): - def _msgobj(self, filename): - fp = openfile(filename) - try: - data = fp.read() - finally: - fp.close() - msg = email.message_from_string(data) - return msg, data - - def _idempotent(self, msg, text): - eq = self.ndiffAssertEqual - s = StringIO() - g = Generator(s, maxheaderlen=0) - g.flatten(msg) - eq(text, s.getvalue()) - - def test_parse_text_message(self): - eq = self.assertEqual - msg, text = self._msgobj('msg_01.txt') - eq(msg.get_content_type(), 'text/plain') - eq(msg.get_content_maintype(), 'text') - eq(msg.get_content_subtype(), 'plain') - eq(msg.get_params()[1], ('charset', 'us-ascii')) - eq(msg.get_param('charset'), 'us-ascii') - eq(msg.preamble, None) - eq(msg.epilogue, None) - self._idempotent(msg, text) - - def test_parse_untyped_message(self): - eq = self.assertEqual - msg, text = self._msgobj('msg_03.txt') - eq(msg.get_content_type(), 'text/plain') - eq(msg.get_params(), None) - eq(msg.get_param('charset'), None) - self._idempotent(msg, text) - - def test_simple_multipart(self): - msg, text = self._msgobj('msg_04.txt') - self._idempotent(msg, text) - - def test_MIME_digest(self): - msg, text = self._msgobj('msg_02.txt') - self._idempotent(msg, text) - - def test_long_header(self): - msg, text = self._msgobj('msg_27.txt') - self._idempotent(msg, text) - - def test_MIME_digest_with_part_headers(self): - msg, text = self._msgobj('msg_28.txt') - self._idempotent(msg, text) - - def test_mixed_with_image(self): - msg, text = self._msgobj('msg_06.txt') - self._idempotent(msg, text) - - def test_multipart_report(self): - msg, text = self._msgobj('msg_05.txt') - self._idempotent(msg, text) - - def test_dsn(self): - msg, text = self._msgobj('msg_16.txt') - self._idempotent(msg, text) - - def test_preamble_epilogue(self): - msg, text = self._msgobj('msg_21.txt') - self._idempotent(msg, text) - - def test_multipart_one_part(self): - msg, text = self._msgobj('msg_23.txt') - self._idempotent(msg, text) - - def test_multipart_no_parts(self): - msg, text = self._msgobj('msg_24.txt') - self._idempotent(msg, text) - - def test_no_start_boundary(self): - msg, text = self._msgobj('msg_31.txt') - self._idempotent(msg, text) - - def test_rfc2231_charset(self): - msg, text = self._msgobj('msg_32.txt') - self._idempotent(msg, text) - - def test_more_rfc2231_parameters(self): - msg, text = self._msgobj('msg_33.txt') - self._idempotent(msg, text) - - def test_text_plain_in_a_multipart_digest(self): - msg, text = self._msgobj('msg_34.txt') - self._idempotent(msg, text) - - def test_nested_multipart_mixeds(self): - msg, text = self._msgobj('msg_12a.txt') - self._idempotent(msg, text) - - def test_message_external_body_idempotent(self): - msg, text = self._msgobj('msg_36.txt') - self._idempotent(msg, text) - - def test_content_type(self): - eq = self.assertEqual - unless = self.assertTrue - # Get a message object and reset the seek pointer for other tests - msg, text = self._msgobj('msg_05.txt') - eq(msg.get_content_type(), 'multipart/report') - # Test the Content-Type: parameters - params = {} - for pk, pv in msg.get_params(): - params[pk] = pv - eq(params['report-type'], 'delivery-status') - eq(params['boundary'], 'D1690A7AC1.996856090/mail.example.com') - eq(msg.preamble, 'This is a MIME-encapsulated message.\n') - eq(msg.epilogue, '\n') - eq(len(msg.get_payload()), 3) - # Make sure the subparts are what we expect - msg1 = msg.get_payload(0) - eq(msg1.get_content_type(), 'text/plain') - eq(msg1.get_payload(), 'Yadda yadda yadda\n') - msg2 = msg.get_payload(1) - eq(msg2.get_content_type(), 'text/plain') - eq(msg2.get_payload(), 'Yadda yadda yadda\n') - msg3 = msg.get_payload(2) - eq(msg3.get_content_type(), 'message/rfc822') - self.assertTrue(isinstance(msg3, Message)) - payload = msg3.get_payload() - unless(isinstance(payload, list)) - eq(len(payload), 1) - msg4 = payload[0] - unless(isinstance(msg4, Message)) - eq(msg4.get_payload(), 'Yadda yadda yadda\n') - - def test_parser(self): - eq = self.assertEqual - unless = self.assertTrue - msg, text = self._msgobj('msg_06.txt') - # Check some of the outer headers - eq(msg.get_content_type(), 'message/rfc822') - # Make sure the payload is a list of exactly one sub-Message, and that - # that submessage has a type of text/plain - payload = msg.get_payload() - unless(isinstance(payload, list)) - eq(len(payload), 1) - msg1 = payload[0] - self.assertTrue(isinstance(msg1, Message)) - eq(msg1.get_content_type(), 'text/plain') - self.assertTrue(isinstance(msg1.get_payload(), str)) - eq(msg1.get_payload(), '\n') - - - -# Test various other bits of the package's functionality -class TestMiscellaneous(TestEmailBase): - def test_message_from_string(self): - fp = openfile('msg_01.txt') - try: - text = fp.read() - finally: - fp.close() - msg = email.message_from_string(text) - s = StringIO() - # Don't wrap/continue long headers since we're trying to test - # idempotency. - g = Generator(s, maxheaderlen=0) - g.flatten(msg) - self.assertEqual(text, s.getvalue()) - - def test_message_from_file(self): - fp = openfile('msg_01.txt') - try: - text = fp.read() - fp.seek(0) - msg = email.message_from_file(fp) - s = StringIO() - # Don't wrap/continue long headers since we're trying to test - # idempotency. - g = Generator(s, maxheaderlen=0) - g.flatten(msg) - self.assertEqual(text, s.getvalue()) - finally: - fp.close() - - def test_message_from_string_with_class(self): - unless = self.assertTrue - fp = openfile('msg_01.txt') - try: - text = fp.read() - finally: - fp.close() - # Create a subclass - class MyMessage(Message): - pass - - msg = email.message_from_string(text, MyMessage) - unless(isinstance(msg, MyMessage)) - # Try something more complicated - fp = openfile('msg_02.txt') - try: - text = fp.read() - finally: - fp.close() - msg = email.message_from_string(text, MyMessage) - for subpart in msg.walk(): - unless(isinstance(subpart, MyMessage)) - - def test_message_from_file_with_class(self): - unless = self.assertTrue - # Create a subclass - class MyMessage(Message): - pass - - fp = openfile('msg_01.txt') - try: - msg = email.message_from_file(fp, MyMessage) - finally: - fp.close() - unless(isinstance(msg, MyMessage)) - # Try something more complicated - fp = openfile('msg_02.txt') - try: - msg = email.message_from_file(fp, MyMessage) - finally: - fp.close() - for subpart in msg.walk(): - unless(isinstance(subpart, MyMessage)) - - def test__all__(self): - module = __import__('email') - all = module.__all__ - all.sort() - self.assertEqual(all, [ - # Old names - 'Charset', 'Encoders', 'Errors', 'Generator', - 'Header', 'Iterators', 'MIMEAudio', 'MIMEBase', - 'MIMEImage', 'MIMEMessage', 'MIMEMultipart', - 'MIMENonMultipart', 'MIMEText', 'Message', - 'Parser', 'Utils', 'base64MIME', - # new names - 'base64mime', 'charset', 'encoders', 'errors', 'generator', - 'header', 'iterators', 'message', 'message_from_file', - 'message_from_string', 'mime', 'parser', - 'quopriMIME', 'quoprimime', 'utils', - ]) - - def test_formatdate(self): - now = time.time() - self.assertEqual(Utils.parsedate(Utils.formatdate(now))[:6], - time.gmtime(now)[:6]) - - def test_formatdate_localtime(self): - now = time.time() - self.assertEqual( - Utils.parsedate(Utils.formatdate(now, localtime=True))[:6], - time.localtime(now)[:6]) - - def test_formatdate_usegmt(self): - now = time.time() - self.assertEqual( - Utils.formatdate(now, localtime=False), - time.strftime('%a, %d %b %Y %H:%M:%S -0000', time.gmtime(now))) - self.assertEqual( - Utils.formatdate(now, localtime=False, usegmt=True), - time.strftime('%a, %d %b %Y %H:%M:%S GMT', time.gmtime(now))) - - def test_parsedate_none(self): - self.assertEqual(Utils.parsedate(''), None) - - def test_parsedate_compact(self): - # The FWS after the comma is optional - self.assertEqual(Utils.parsedate('Wed,3 Apr 2002 14:58:26 +0800'), - Utils.parsedate('Wed, 3 Apr 2002 14:58:26 +0800')) - - def test_parsedate_no_dayofweek(self): - eq = self.assertEqual - eq(Utils.parsedate_tz('25 Feb 2003 13:47:26 -0800'), - (2003, 2, 25, 13, 47, 26, 0, 1, -1, -28800)) - - def test_parsedate_compact_no_dayofweek(self): - eq = self.assertEqual - eq(Utils.parsedate_tz('5 Feb 2003 13:47:26 -0800'), - (2003, 2, 5, 13, 47, 26, 0, 1, -1, -28800)) - - def test_parsedate_acceptable_to_time_functions(self): - eq = self.assertEqual - timetup = Utils.parsedate('5 Feb 2003 13:47:26 -0800') - t = int(time.mktime(timetup)) - eq(time.localtime(t)[:6], timetup[:6]) - eq(int(time.strftime('%Y', timetup)), 2003) - timetup = Utils.parsedate_tz('5 Feb 2003 13:47:26 -0800') - t = int(time.mktime(timetup[:9])) - eq(time.localtime(t)[:6], timetup[:6]) - eq(int(time.strftime('%Y', timetup[:9])), 2003) - - def test_parsedate_y2k(self): - """Test for parsing a date with a two-digit year. - - Parsing a date with a two-digit year should return the correct - four-digit year. RFC822 allows two-digit years, but RFC2822 (which - obsoletes RFC822) requires four-digit years. - - """ - self.assertEqual(Utils.parsedate_tz('25 Feb 03 13:47:26 -0800'), - Utils.parsedate_tz('25 Feb 2003 13:47:26 -0800')) - self.assertEqual(Utils.parsedate_tz('25 Feb 71 13:47:26 -0800'), - Utils.parsedate_tz('25 Feb 1971 13:47:26 -0800')) - - def test_parseaddr_empty(self): - self.assertEqual(Utils.parseaddr('<>'), ('', '')) - self.assertEqual(Utils.formataddr(Utils.parseaddr('<>')), '') - - def test_noquote_dump(self): - self.assertEqual( - Utils.formataddr(('A Silly Person', 'person at dom.ain')), - 'A Silly Person <person at dom.ain>') - - def test_escape_dump(self): - self.assertEqual( - Utils.formataddr(('A (Very) Silly Person', 'person at dom.ain')), - r'"A $Very$ Silly Person" <person at dom.ain>') - a = r'A $Special$ Person' - b = 'person at dom.ain' - self.assertEqual(Utils.parseaddr(Utils.formataddr((a, b))), (a, b)) - - def test_escape_backslashes(self): - self.assertEqual( - Utils.formataddr(('Arthur \Backslash\ Foobar', 'person at dom.ain')), - r'"Arthur \\Backslash\\ Foobar" <person at dom.ain>') - a = r'Arthur \Backslash\ Foobar' - b = 'person at dom.ain' - self.assertEqual(Utils.parseaddr(Utils.formataddr((a, b))), (a, b)) - - def test_name_with_dot(self): - x = 'John X. Doe <jxd at example.com>' - y = '"John X. Doe" <jxd at example.com>' - a, b = ('John X. Doe', 'jxd at example.com') - self.assertEqual(Utils.parseaddr(x), (a, b)) - self.assertEqual(Utils.parseaddr(y), (a, b)) - # formataddr() quotes the name if there's a dot in it - self.assertEqual(Utils.formataddr((a, b)), y) - - def test_parseaddr_preserves_quoted_pairs_in_addresses(self): - # issue 10005. Note that in the third test the second pair of - # backslashes is not actually a quoted pair because it is not inside a - # comment or quoted string: the address being parsed has a quoted - # string containing a quoted backslash, followed by 'example' and two - # backslashes, followed by another quoted string containing a space and - # the word 'example'. parseaddr copies those two backslashes - # literally. Per rfc5322 this is not technically correct since a \ may - # not appear in an address outside of a quoted string. It is probably - # a sensible Postel interpretation, though. - eq = self.assertEqual - eq(Utils.parseaddr('""example" example"@example.com'), - ('', '""example" example"@example.com')) - eq(Utils.parseaddr('"\\"example\\" example"@example.com'), - ('', '"\\"example\\" example"@example.com')) - eq(Utils.parseaddr('"\\\\"example\\\\" example"@example.com'), - ('', '"\\\\"example\\\\" example"@example.com')) - - def test_multiline_from_comment(self): - x = """\ -Foo -\tBar <foo at example.com>""" - self.assertEqual(Utils.parseaddr(x), ('Foo Bar', 'foo at example.com')) - - def test_quote_dump(self): - self.assertEqual( - Utils.formataddr(('A Silly; Person', 'person at dom.ain')), - r'"A Silly; Person" <person at dom.ain>') - - def test_fix_eols(self): - eq = self.assertEqual - eq(Utils.fix_eols('hello'), 'hello') - eq(Utils.fix_eols('hello\n'), 'hello\r\n') - eq(Utils.fix_eols('hello\r'), 'hello\r\n') - eq(Utils.fix_eols('hello\r\n'), 'hello\r\n') - eq(Utils.fix_eols('hello\n\r'), 'hello\r\n\r\n') - - def test_charset_richcomparisons(self): - eq = self.assertEqual - ne = self.assertNotEqual - cset1 = Charset() - cset2 = Charset() - eq(cset1, 'us-ascii') - eq(cset1, 'US-ASCII') - eq(cset1, 'Us-AsCiI') - eq('us-ascii', cset1) - eq('US-ASCII', cset1) - eq('Us-AsCiI', cset1) - ne(cset1, 'usascii') - ne(cset1, 'USASCII') - ne(cset1, 'UsAsCiI') - ne('usascii', cset1) - ne('USASCII', cset1) - ne('UsAsCiI', cset1) - eq(cset1, cset2) - eq(cset2, cset1) - - def test_getaddresses(self): - eq = self.assertEqual - eq(Utils.getaddresses(['aperson at dom.ain (Al Person)', - 'Bud Person <bperson at dom.ain>']), - [('Al Person', 'aperson at dom.ain'), - ('Bud Person', 'bperson at dom.ain')]) - - def test_getaddresses_nasty(self): - eq = self.assertEqual - eq(Utils.getaddresses(['foo: ;']), [('', '')]) - eq(Utils.getaddresses( - ['[]*-- =~$']), - [('', ''), ('', ''), ('', '*--')]) - eq(Utils.getaddresses( - ['foo: ;', '"Jason R. Mastaler" <jason at dom.ain>']), - [('', ''), ('Jason R. Mastaler', 'jason at dom.ain')]) - - def test_getaddresses_embedded_comment(self): - """Test proper handling of a nested comment""" - eq = self.assertEqual - addrs = Utils.getaddresses(['User ((nested comment)) <foo at bar.com>']) - eq(addrs[0][1], 'foo at bar.com') - - def test_utils_quote_unquote(self): - eq = self.assertEqual - msg = Message() - msg.add_header('content-disposition', 'attachment', - filename='foo\\wacky"name') - eq(msg.get_filename(), 'foo\\wacky"name') - - def test_get_body_encoding_with_bogus_charset(self): - charset = Charset('not a charset') - self.assertEqual(charset.get_body_encoding(), 'base64') - - def test_get_body_encoding_with_uppercase_charset(self): - eq = self.assertEqual - msg = Message() - msg['Content-Type'] = 'text/plain; charset=UTF-8' - eq(msg['content-type'], 'text/plain; charset=UTF-8') - charsets = msg.get_charsets() - eq(len(charsets), 1) - eq(charsets[0], 'utf-8') - charset = Charset(charsets[0]) - eq(charset.get_body_encoding(), 'base64') - msg.set_payload('hello world', charset=charset) - eq(msg.get_payload(), 'aGVsbG8gd29ybGQ=\n') - eq(msg.get_payload(decode=True), 'hello world') - eq(msg['content-transfer-encoding'], 'base64') - # Try another one - msg = Message() - msg['Content-Type'] = 'text/plain; charset="US-ASCII"' - charsets = msg.get_charsets() - eq(len(charsets), 1) - eq(charsets[0], 'us-ascii') - charset = Charset(charsets[0]) - eq(charset.get_body_encoding(), Encoders.encode_7or8bit) - msg.set_payload('hello world', charset=charset) - eq(msg.get_payload(), 'hello world') - eq(msg['content-transfer-encoding'], '7bit') - - def test_charsets_case_insensitive(self): - lc = Charset('us-ascii') - uc = Charset('US-ASCII') - self.assertEqual(lc.get_body_encoding(), uc.get_body_encoding()) - - def test_partial_falls_inside_message_delivery_status(self): - eq = self.ndiffAssertEqual - # The Parser interface provides chunks of data to FeedParser in 8192 - # byte gulps. SF bug #1076485 found one of those chunks inside - # message/delivery-status header block, which triggered an - # unreadline() of NeedMoreData. - msg = self._msgobj('msg_43.txt') - sfp = StringIO() - Iterators._structure(msg, sfp) - eq(sfp.getvalue(), """\ -multipart/report - text/plain - message/delivery-status - text/plain - text/plain - text/plain - text/plain - text/plain - text/plain - text/plain - text/plain - text/plain - text/plain - text/plain - text/plain - text/plain - text/plain - text/plain - text/plain - text/plain - text/plain - text/plain - text/plain - text/plain - text/plain - text/plain - text/plain - text/plain - text/plain - text/rfc822-headers -""") - - - -# Test the iterator/generators -class TestIterators(TestEmailBase): - def test_body_line_iterator(self): - eq = self.assertEqual - neq = self.ndiffAssertEqual - # First a simple non-multipart message - msg = self._msgobj('msg_01.txt') - it = Iterators.body_line_iterator(msg) - lines = list(it) - eq(len(lines), 6) - neq(EMPTYSTRING.join(lines), msg.get_payload()) - # Now a more complicated multipart - msg = self._msgobj('msg_02.txt') - it = Iterators.body_line_iterator(msg) - lines = list(it) - eq(len(lines), 43) - fp = openfile('msg_19.txt') - try: - neq(EMPTYSTRING.join(lines), fp.read()) - finally: - fp.close() - - def test_typed_subpart_iterator(self): - eq = self.assertEqual - msg = self._msgobj('msg_04.txt') - it = Iterators.typed_subpart_iterator(msg, 'text') - lines = [] - subparts = 0 - for subpart in it: - subparts += 1 - lines.append(subpart.get_payload()) - eq(subparts, 2) - eq(EMPTYSTRING.join(lines), """\ -a simple kind of mirror -to reflect upon our own -a simple kind of mirror -to reflect upon our own -""") - - def test_typed_subpart_iterator_default_type(self): - eq = self.assertEqual - msg = self._msgobj('msg_03.txt') - it = Iterators.typed_subpart_iterator(msg, 'text', 'plain') - lines = [] - subparts = 0 - for subpart in it: - subparts += 1 - lines.append(subpart.get_payload()) - eq(subparts, 1) - eq(EMPTYSTRING.join(lines), """\ - -Hi, - -Do you like this message? - --Me -""") - - def test_pushCR_LF(self): - '''FeedParser BufferedSubFile.push() assumed it received complete - line endings. A CR ending one push() followed by a LF starting - the next push() added an empty line. - ''' - imt = [ - ("a\r \n", 2), - ("b", 0), - ("c\n", 1), - ("", 0), - ("d\r\n", 1), - ("e\r", 0), - ("\nf", 1), - ("\r\n", 1), - ] - from email.feedparser import BufferedSubFile, NeedMoreData - bsf = BufferedSubFile() - om = [] - nt = 0 - for il, n in imt: - bsf.push(il) - nt += n - n1 = 0 - while True: - ol = bsf.readline() - if ol == NeedMoreData: - break - om.append(ol) - n1 += 1 - self.assertTrue(n == n1) - self.assertTrue(len(om) == nt) - self.assertTrue(''.join([il for il, n in imt]) == ''.join(om)) - - - -class TestParsers(TestEmailBase): - def test_header_parser(self): - eq = self.assertEqual - # Parse only the headers of a complex multipart MIME document - fp = openfile('msg_02.txt') - try: - msg = HeaderParser().parse(fp) - finally: - fp.close() - eq(msg['from'], 'ppp-request at zzz.org') - eq(msg['to'], 'ppp at zzz.org') - eq(msg.get_content_type(), 'multipart/mixed') - self.assertFalse(msg.is_multipart()) - self.assertTrue(isinstance(msg.get_payload(), str)) - - def test_whitespace_continuation(self): - eq = self.assertEqual - # This message contains a line after the Subject: header that has only - # whitespace, but it is not empty! - msg = email.message_from_string("""\ -From: aperson at dom.ain -To: bperson at dom.ain -Subject: the next line has a space on it -\x20 -Date: Mon, 8 Apr 2002 15:09:19 -0400 -Message-ID: spam - -Here's the message body -""") - eq(msg['subject'], 'the next line has a space on it\n ') - eq(msg['message-id'], 'spam') - eq(msg.get_payload(), "Here's the message body\n") - - def test_whitespace_continuation_last_header(self): - eq = self.assertEqual - # Like the previous test, but the subject line is the last - # header. - msg = email.message_from_string("""\ -From: aperson at dom.ain -To: bperson at dom.ain -Date: Mon, 8 Apr 2002 15:09:19 -0400 -Message-ID: spam -Subject: the next line has a space on it -\x20 - -Here's the message body -""") - eq(msg['subject'], 'the next line has a space on it\n ') - eq(msg['message-id'], 'spam') - eq(msg.get_payload(), "Here's the message body\n") - - def test_crlf_separation(self): - eq = self.assertEqual - fp = openfile('msg_26.txt', mode='rb') - try: - msg = Parser().parse(fp) - finally: - fp.close() - eq(len(msg.get_payload()), 2) - part1 = msg.get_payload(0) - eq(part1.get_content_type(), 'text/plain') - eq(part1.get_payload(), 'Simple email with attachment.\r\n\r\n') - part2 = msg.get_payload(1) - eq(part2.get_content_type(), 'application/riscos') - - def test_multipart_digest_with_extra_mime_headers(self): - eq = self.assertEqual - neq = self.ndiffAssertEqual - fp = openfile('msg_28.txt') - try: - msg = email.message_from_file(fp) - finally: - fp.close() - # Structure is: - # multipart/digest - # message/rfc822 - # text/plain - # message/rfc822 - # text/plain - eq(msg.is_multipart(), 1) - eq(len(msg.get_payload()), 2) - part1 = msg.get_payload(0) - eq(part1.get_content_type(), 'message/rfc822') - eq(part1.is_multipart(), 1) - eq(len(part1.get_payload()), 1) - part1a = part1.get_payload(0) - eq(part1a.is_multipart(), 0) - eq(part1a.get_content_type(), 'text/plain') - neq(part1a.get_payload(), 'message 1\n') - # next message/rfc822 - part2 = msg.get_payload(1) - eq(part2.get_content_type(), 'message/rfc822') - eq(part2.is_multipart(), 1) - eq(len(part2.get_payload()), 1) - part2a = part2.get_payload(0) - eq(part2a.is_multipart(), 0) - eq(part2a.get_content_type(), 'text/plain') - neq(part2a.get_payload(), 'message 2\n') - - def test_three_lines(self): - # A bug report by Andrew McNamara - lines = ['From: Andrew Person <aperson at dom.ain', - 'Subject: Test', - 'Date: Tue, 20 Aug 2002 16:43:45 +1000'] - msg = email.message_from_string(NL.join(lines)) - self.assertEqual(msg['date'], 'Tue, 20 Aug 2002 16:43:45 +1000') - - def test_strip_line_feed_and_carriage_return_in_headers(self): - eq = self.assertEqual - # For [ 1002475 ] email message parser doesn't handle \r\n correctly - value1 = 'text' - value2 = 'more text' - m = 'Header: %s\r\nNext-Header: %s\r\n\r\nBody\r\n\r\n' % ( - value1, value2) - msg = email.message_from_string(m) - eq(msg.get('Header'), value1) - eq(msg.get('Next-Header'), value2) - - def test_rfc2822_header_syntax(self): - eq = self.assertEqual - m = '>From: foo\nFrom: bar\n!"#QUX;~: zoo\n\nbody' - msg = email.message_from_string(m) - eq(len(msg.keys()), 3) - keys = msg.keys() - keys.sort() - eq(keys, ['!"#QUX;~', '>From', 'From']) - eq(msg.get_payload(), 'body') - - def test_rfc2822_space_not_allowed_in_header(self): - eq = self.assertEqual - m = '>From foo at example.com 11:25:53\nFrom: bar\n!"#QUX;~: zoo\n\nbody' - msg = email.message_from_string(m) - eq(len(msg.keys()), 0) - - def test_rfc2822_one_character_header(self): - eq = self.assertEqual - m = 'A: first header\nB: second header\nCC: third header\n\nbody' - msg = email.message_from_string(m) - headers = msg.keys() - headers.sort() - eq(headers, ['A', 'B', 'CC']) - eq(msg.get_payload(), 'body') - - def test_CRLFLF_at_end_of_part(self): - # issue 5610: feedparser should not eat two chars from body part ending - # with "\r\n\n". - m = ( - "From: foo at bar.com\n" - "To: baz\n" - "Mime-Version: 1.0\n" - "Content-Type: multipart/mixed; boundary=BOUNDARY\n" - "\n" - "--BOUNDARY\n" - "Content-Type: text/plain\n" - "\n" - "body ending with CRLF newline\r\n" - "\n" - "--BOUNDARY--\n" - ) - msg = email.message_from_string(m) - self.assertTrue(msg.get_payload(0).get_payload().endswith('\r\n')) - - -class TestBase64(unittest.TestCase): - def test_len(self): - eq = self.assertEqual - eq(base64MIME.base64_len('hello'), - len(base64MIME.encode('hello', eol=''))) - for size in range(15): - if size == 0 : bsize = 0 - elif size <= 3 : bsize = 4 - elif size <= 6 : bsize = 8 - elif size <= 9 : bsize = 12 - elif size <= 12: bsize = 16 - else : bsize = 20 - eq(base64MIME.base64_len('x'*size), bsize) - - def test_decode(self): - eq = self.assertEqual - eq(base64MIME.decode(''), '') - eq(base64MIME.decode('aGVsbG8='), 'hello') - eq(base64MIME.decode('aGVsbG8=', 'X'), 'hello') - eq(base64MIME.decode('aGVsbG8NCndvcmxk\n', 'X'), 'helloXworld') - - def test_encode(self): - eq = self.assertEqual - eq(base64MIME.encode(''), '') - eq(base64MIME.encode('hello'), 'aGVsbG8=\n') - # Test the binary flag - eq(base64MIME.encode('hello\n'), 'aGVsbG8K\n') - eq(base64MIME.encode('hello\n', 0), 'aGVsbG8NCg==\n') - # Test the maxlinelen arg - eq(base64MIME.encode('xxxx ' * 20, maxlinelen=40), """\ -eHh4eCB4eHh4IHh4eHggeHh4eCB4eHh4IHh4eHgg -eHh4eCB4eHh4IHh4eHggeHh4eCB4eHh4IHh4eHgg -eHh4eCB4eHh4IHh4eHggeHh4eCB4eHh4IHh4eHgg -eHh4eCB4eHh4IA== -""") - # Test the eol argument - eq(base64MIME.encode('xxxx ' * 20, maxlinelen=40, eol='\r\n'), """\ -eHh4eCB4eHh4IHh4eHggeHh4eCB4eHh4IHh4eHgg\r -eHh4eCB4eHh4IHh4eHggeHh4eCB4eHh4IHh4eHgg\r -eHh4eCB4eHh4IHh4eHggeHh4eCB4eHh4IHh4eHgg\r -eHh4eCB4eHh4IA==\r -""") - - def test_header_encode(self): - eq = self.assertEqual - he = base64MIME.header_encode - eq(he('hello'), '=?iso-8859-1?b?aGVsbG8=?=') - eq(he('hello\nworld'), '=?iso-8859-1?b?aGVsbG8NCndvcmxk?=') - # Test the charset option - eq(he('hello', charset='iso-8859-2'), '=?iso-8859-2?b?aGVsbG8=?=') - # Test the keep_eols flag - eq(he('hello\nworld', keep_eols=True), - '=?iso-8859-1?b?aGVsbG8Kd29ybGQ=?=') - # Test the maxlinelen argument - eq(he('xxxx ' * 20, maxlinelen=40), """\ -=?iso-8859-1?b?eHh4eCB4eHh4IHh4eHggeHg=?= - =?iso-8859-1?b?eHggeHh4eCB4eHh4IHh4eHg=?= - =?iso-8859-1?b?IHh4eHggeHh4eCB4eHh4IHg=?= - =?iso-8859-1?b?eHh4IHh4eHggeHh4eCB4eHg=?= - =?iso-8859-1?b?eCB4eHh4IHh4eHggeHh4eCA=?= - =?iso-8859-1?b?eHh4eCB4eHh4IHh4eHgg?=""") - # Test the eol argument - eq(he('xxxx ' * 20, maxlinelen=40, eol='\r\n'), """\ -=?iso-8859-1?b?eHh4eCB4eHh4IHh4eHggeHg=?=\r - =?iso-8859-1?b?eHggeHh4eCB4eHh4IHh4eHg=?=\r - =?iso-8859-1?b?IHh4eHggeHh4eCB4eHh4IHg=?=\r - =?iso-8859-1?b?eHh4IHh4eHggeHh4eCB4eHg=?=\r - =?iso-8859-1?b?eCB4eHh4IHh4eHggeHh4eCA=?=\r - =?iso-8859-1?b?eHh4eCB4eHh4IHh4eHgg?=""") - - - -class TestQuopri(unittest.TestCase): - def setUp(self): - self.hlit = [chr(x) for x in range(ord('a'), ord('z')+1)] + \ - [chr(x) for x in range(ord('A'), ord('Z')+1)] + \ - [chr(x) for x in range(ord('0'), ord('9')+1)] + \ - ['!', '*', '+', '-', '/', ' '] - self.hnon = [chr(x) for x in range(256) if chr(x) not in self.hlit] - assert len(self.hlit) + len(self.hnon) == 256 - self.blit = [chr(x) for x in range(ord(' '), ord('~')+1)] + ['\t'] - self.blit.remove('=') - self.bnon = [chr(x) for x in range(256) if chr(x) not in self.blit] - assert len(self.blit) + len(self.bnon) == 256 - - def test_header_quopri_check(self): - for c in self.hlit: - self.assertFalse(quopriMIME.header_quopri_check(c)) - for c in self.hnon: - self.assertTrue(quopriMIME.header_quopri_check(c)) - - def test_body_quopri_check(self): - for c in self.blit: - self.assertFalse(quopriMIME.body_quopri_check(c)) - for c in self.bnon: - self.assertTrue(quopriMIME.body_quopri_check(c)) - - def test_header_quopri_len(self): - eq = self.assertEqual - hql = quopriMIME.header_quopri_len - enc = quopriMIME.header_encode - for s in ('hello', 'h at e@l at l@o@'): - # Empty charset and no line-endings. 7 == RFC chrome - eq(hql(s), len(enc(s, charset='', eol=''))-7) - for c in self.hlit: - eq(hql(c), 1) - for c in self.hnon: - eq(hql(c), 3) - - def test_body_quopri_len(self): - eq = self.assertEqual - bql = quopriMIME.body_quopri_len - for c in self.blit: - eq(bql(c), 1) - for c in self.bnon: - eq(bql(c), 3) - - def test_quote_unquote_idempotent(self): - for x in range(256): - c = chr(x) - self.assertEqual(quopriMIME.unquote(quopriMIME.quote(c)), c) - - def test_header_encode(self): - eq = self.assertEqual - he = quopriMIME.header_encode - eq(he('hello'), '=?iso-8859-1?q?hello?=') - eq(he('hello\nworld'), '=?iso-8859-1?q?hello=0D=0Aworld?=') - # Test the charset option - eq(he('hello', charset='iso-8859-2'), '=?iso-8859-2?q?hello?=') - # Test the keep_eols flag - eq(he('hello\nworld', keep_eols=True), '=?iso-8859-1?q?hello=0Aworld?=') - # Test a non-ASCII character - eq(he('hello\xc7there'), '=?iso-8859-1?q?hello=C7there?=') - # Test the maxlinelen argument - eq(he('xxxx ' * 20, maxlinelen=40), """\ -=?iso-8859-1?q?xxxx_xxxx_xxxx_xxxx_xx?= - =?iso-8859-1?q?xx_xxxx_xxxx_xxxx_xxxx?= - =?iso-8859-1?q?_xxxx_xxxx_xxxx_xxxx_x?= - =?iso-8859-1?q?xxx_xxxx_xxxx_xxxx_xxx?= - =?iso-8859-1?q?x_xxxx_xxxx_?=""") - # Test the eol argument - eq(he('xxxx ' * 20, maxlinelen=40, eol='\r\n'), """\ -=?iso-8859-1?q?xxxx_xxxx_xxxx_xxxx_xx?=\r - =?iso-8859-1?q?xx_xxxx_xxxx_xxxx_xxxx?=\r - =?iso-8859-1?q?_xxxx_xxxx_xxxx_xxxx_x?=\r - =?iso-8859-1?q?xxx_xxxx_xxxx_xxxx_xxx?=\r - =?iso-8859-1?q?x_xxxx_xxxx_?=""") - - def test_decode(self): - eq = self.assertEqual - eq(quopriMIME.decode(''), '') - eq(quopriMIME.decode('hello'), 'hello') - eq(quopriMIME.decode('hello', 'X'), 'hello') - eq(quopriMIME.decode('hello\nworld', 'X'), 'helloXworld') - - def test_encode(self): - eq = self.assertEqual - eq(quopriMIME.encode(''), '') - eq(quopriMIME.encode('hello'), 'hello') - # Test the binary flag - eq(quopriMIME.encode('hello\r\nworld'), 'hello\nworld') - eq(quopriMIME.encode('hello\r\nworld', 0), 'hello\nworld') - # Test the maxlinelen arg - eq(quopriMIME.encode('xxxx ' * 20, maxlinelen=40), """\ -xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxxx= - xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxx= -x xxxx xxxx xxxx xxxx=20""") - # Test the eol argument - eq(quopriMIME.encode('xxxx ' * 20, maxlinelen=40, eol='\r\n'), """\ -xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxxx=\r - xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxx=\r -x xxxx xxxx xxxx xxxx=20""") - eq(quopriMIME.encode("""\ -one line - -two line"""), """\ -one line - -two line""") - - - -# Test the Charset class -class TestCharset(unittest.TestCase): - def tearDown(self): - from email import Charset as CharsetModule - try: - del CharsetModule.CHARSETS['fake'] - except KeyError: - pass - - def test_idempotent(self): - eq = self.assertEqual - # Make sure us-ascii = no Unicode conversion - c = Charset('us-ascii') - s = 'Hello World!' - sp = c.to_splittable(s) - eq(s, c.from_splittable(sp)) - # test 8-bit idempotency with us-ascii - s = '\xa4\xa2\xa4\xa4\xa4\xa6\xa4\xa8\xa4\xaa' - sp = c.to_splittable(s) - eq(s, c.from_splittable(sp)) - - def test_body_encode(self): - eq = self.assertEqual - # Try a charset with QP body encoding - c = Charset('iso-8859-1') - eq('hello w=F6rld', c.body_encode('hello w\xf6rld')) - # Try a charset with Base64 body encoding - c = Charset('utf-8') - eq('aGVsbG8gd29ybGQ=\n', c.body_encode('hello world')) - # Try a charset with None body encoding - c = Charset('us-ascii') - eq('hello world', c.body_encode('hello world')) - # Try the convert argument, where input codec != output codec - c = Charset('euc-jp') - # With apologies to Tokio Kikuchi ;) - try: - eq('\x1b$B5FCO;~IW\x1b(B', - c.body_encode('\xb5\xc6\xc3\xcf\xbb\xfe\xc9\xd7')) - eq('\xb5\xc6\xc3\xcf\xbb\xfe\xc9\xd7', - c.body_encode('\xb5\xc6\xc3\xcf\xbb\xfe\xc9\xd7', False)) - except LookupError: - # We probably don't have the Japanese codecs installed - pass - # Testing SF bug #625509, which we have to fake, since there are no - # built-in encodings where the header encoding is QP but the body - # encoding is not. - from email import Charset as CharsetModule - CharsetModule.add_charset('fake', CharsetModule.QP, None) - c = Charset('fake') - eq('hello w\xf6rld', c.body_encode('hello w\xf6rld')) - - def test_unicode_charset_name(self): - charset = Charset(u'us-ascii') - self.assertEqual(str(charset), 'us-ascii') - self.assertRaises(Errors.CharsetError, Charset, 'asc\xffii') - - def test_codecs_aliases_accepted(self): - charset = Charset('utf8') - self.assertEqual(str(charset), 'utf-8') - - -# Test multilingual MIME headers. -class TestHeader(TestEmailBase): - def test_simple(self): - eq = self.ndiffAssertEqual - h = Header('Hello World!') - eq(h.encode(), 'Hello World!') - h.append(' Goodbye World!') - eq(h.encode(), 'Hello World! Goodbye World!') - - def test_simple_surprise(self): - eq = self.ndiffAssertEqual - h = Header('Hello World!') - eq(h.encode(), 'Hello World!') - h.append('Goodbye World!') - eq(h.encode(), 'Hello World! Goodbye World!') - - def test_header_needs_no_decoding(self): - h = 'no decoding needed' - self.assertEqual(decode_header(h), [(h, None)]) - - def test_long(self): - h = Header("I am the very model of a modern Major-General; I've information vegetable, animal, and mineral; I know the kings of England, and I quote the fights historical from Marathon to Waterloo, in order categorical; I'm very well acquainted, too, with matters mathematical; I understand equations, both the simple and quadratical; about binomial theorem I'm teeming with a lot o' news, with many cheerful facts about the square of the hypotenuse.", - maxlinelen=76) - for l in h.encode(splitchars=' ').split('\n '): - self.assertTrue(len(l) <= 76) - - def test_multilingual(self): - eq = self.ndiffAssertEqual - g = Charset("iso-8859-1") - cz = Charset("iso-8859-2") - utf8 = Charset("utf-8") - g_head = "Die Mieter treten hier ein werden mit einem Foerderband komfortabel den Korridor entlang, an s\xfcdl\xfcndischen Wandgem\xe4lden vorbei, gegen die rotierenden Klingen bef\xf6rdert. " - cz_head = "Finan\xe8ni metropole se hroutily pod tlakem jejich d\xf9vtipu.. " - utf8_head = u"\u6b63\u78ba\u306b\u8a00\u3046\u3068\u7ffb\u8a33\u306f\u3055\u308c\u3066\u3044\u307e\u305b\u3093\u3002\u4e00\u90e8\u306f\u30c9\u30a4\u30c4\u8a9e\u3067\u3059\u304c\u3001\u3042\u3068\u306f\u3067\u305f\u3089\u3081\u3067\u3059\u3002\u5b9f\u969b\u306b\u306f\u300cWenn ist das Nunstuck git und Slotermeyer? Ja! Beiherhund das Oder die Flipperwaldt gersput.\u300d\u3068\u8a00\u3063\u3066\u3044\u307e\u3059\u3002".encode("utf-8") - h = Header(g_head, g) - h.append(cz_head, cz) - h.append(utf8_head, utf8) - enc = h.encode() - eq(enc, """\ -=?iso-8859-1?q?Die_Mieter_treten_hier_ein_werden_mit_einem_Foerderband_ko?= - =?iso-8859-1?q?mfortabel_den_Korridor_entlang=2C_an_s=FCdl=FCndischen_Wan?= - =?iso-8859-1?q?dgem=E4lden_vorbei=2C_gegen_die_rotierenden_Klingen_bef=F6?= - =?iso-8859-1?q?rdert=2E_?= =?iso-8859-2?q?Finan=E8ni_metropole_se_hroutily?= - =?iso-8859-2?q?_pod_tlakem_jejich_d=F9vtipu=2E=2E_?= =?utf-8?b?5q2j56K6?= - =?utf-8?b?44Gr6KiA44GG44Go57+76Kiz44Gv44GV44KM44Gm44GE44G+44Gb44KT44CC?= - =?utf-8?b?5LiA6YOo44Gv44OJ44Kk44OE6Kqe44Gn44GZ44GM44CB44GC44Go44Gv44Gn?= - =?utf-8?b?44Gf44KJ44KB44Gn44GZ44CC5a6f6Zqb44Gr44Gv44CMV2VubiBpc3QgZGFz?= - =?utf-8?q?_Nunstuck_git_und_Slotermeyer=3F_Ja!_Beiherhund_das_Oder_die_Fl?= - =?utf-8?b?aXBwZXJ3YWxkdCBnZXJzcHV0LuOAjeOBqOiogOOBo+OBpuOBhOOBvuOBmQ==?= - =?utf-8?b?44CC?=""") - eq(decode_header(enc), - [(g_head, "iso-8859-1"), (cz_head, "iso-8859-2"), - (utf8_head, "utf-8")]) - ustr = unicode(h) - eq(ustr.encode('utf-8'), - 'Die Mieter treten hier ein werden mit einem Foerderband ' - 'komfortabel den Korridor entlang, an s\xc3\xbcdl\xc3\xbcndischen ' - 'Wandgem\xc3\xa4lden vorbei, gegen die rotierenden Klingen ' - 'bef\xc3\xb6rdert. Finan\xc4\x8dni metropole se hroutily pod ' - 'tlakem jejich d\xc5\xafvtipu.. \xe6\xad\xa3\xe7\xa2\xba\xe3\x81' - '\xab\xe8\xa8\x80\xe3\x81\x86\xe3\x81\xa8\xe7\xbf\xbb\xe8\xa8\xb3' - '\xe3\x81\xaf\xe3\x81\x95\xe3\x82\x8c\xe3\x81\xa6\xe3\x81\x84\xe3' - '\x81\xbe\xe3\x81\x9b\xe3\x82\x93\xe3\x80\x82\xe4\xb8\x80\xe9\x83' - '\xa8\xe3\x81\xaf\xe3\x83\x89\xe3\x82\xa4\xe3\x83\x84\xe8\xaa\x9e' - '\xe3\x81\xa7\xe3\x81\x99\xe3\x81\x8c\xe3\x80\x81\xe3\x81\x82\xe3' - '\x81\xa8\xe3\x81\xaf\xe3\x81\xa7\xe3\x81\x9f\xe3\x82\x89\xe3\x82' - '\x81\xe3\x81\xa7\xe3\x81\x99\xe3\x80\x82\xe5\xae\x9f\xe9\x9a\x9b' - '\xe3\x81\xab\xe3\x81\xaf\xe3\x80\x8cWenn ist das Nunstuck git ' - 'und Slotermeyer? Ja! Beiherhund das Oder die Flipperwaldt ' - 'gersput.\xe3\x80\x8d\xe3\x81\xa8\xe8\xa8\x80\xe3\x81\xa3\xe3\x81' - '\xa6\xe3\x81\x84\xe3\x81\xbe\xe3\x81\x99\xe3\x80\x82') - # Test make_header() - newh = make_header(decode_header(enc)) - eq(newh, enc) - - def test_header_ctor_default_args(self): - eq = self.ndiffAssertEqual - h = Header() - eq(h, '') - h.append('foo', Charset('iso-8859-1')) - eq(h, '=?iso-8859-1?q?foo?=') - - def test_explicit_maxlinelen(self): - eq = self.ndiffAssertEqual - hstr = 'A very long line that must get split to something other than at the 76th character boundary to test the non-default behavior' - h = Header(hstr) - eq(h.encode(), '''\ -A very long line that must get split to something other than at the 76th - character boundary to test the non-default behavior''') - h = Header(hstr, header_name='Subject') - eq(h.encode(), '''\ -A very long line that must get split to something other than at the - 76th character boundary to test the non-default behavior''') - h = Header(hstr, maxlinelen=1024, header_name='Subject') - eq(h.encode(), hstr) - - def test_us_ascii_header(self): - eq = self.assertEqual - s = 'hello' - x = decode_header(s) - eq(x, [('hello', None)]) - h = make_header(x) - eq(s, h.encode()) - - def test_string_charset(self): - eq = self.assertEqual - h = Header() - h.append('hello', 'iso-8859-1') - eq(h, '=?iso-8859-1?q?hello?=') - -## def test_unicode_error(self): -## raises = self.assertRaises -## raises(UnicodeError, Header, u'[P\xf6stal]', 'us-ascii') -## raises(UnicodeError, Header, '[P\xf6stal]', 'us-ascii') -## h = Header() -## raises(UnicodeError, h.append, u'[P\xf6stal]', 'us-ascii') -## raises(UnicodeError, h.append, '[P\xf6stal]', 'us-ascii') -## raises(UnicodeError, Header, u'\u83ca\u5730\u6642\u592b', 'iso-8859-1') - - def test_utf8_shortest(self): - eq = self.assertEqual - h = Header(u'p\xf6stal', 'utf-8') - eq(h.encode(), '=?utf-8?q?p=C3=B6stal?=') - h = Header(u'\u83ca\u5730\u6642\u592b', 'utf-8') - eq(h.encode(), '=?utf-8?b?6I+K5Zyw5pmC5aSr?=') - - def test_bad_8bit_header(self): - raises = self.assertRaises - eq = self.assertEqual - x = 'Ynwp4dUEbay Auction Semiar- No Charge \x96 Earn Big' - raises(UnicodeError, Header, x) - h = Header() - raises(UnicodeError, h.append, x) - eq(str(Header(x, errors='replace')), x) - h.append(x, errors='replace') - eq(str(h), x) - - def test_encoded_adjacent_nonencoded(self): - eq = self.assertEqual - h = Header() - h.append('hello', 'iso-8859-1') - h.append('world') - s = h.encode() - eq(s, '=?iso-8859-1?q?hello?= world') - h = make_header(decode_header(s)) - eq(h.encode(), s) - - def test_whitespace_eater(self): - eq = self.assertEqual - s = 'Subject: =?koi8-r?b?8NLP18XSy8EgzsEgxsnOwczYztk=?= =?koi8-r?q?=CA?= zz.' - parts = decode_header(s) - eq(parts, [('Subject:', None), ('\xf0\xd2\xcf\xd7\xc5\xd2\xcb\xc1 \xce\xc1 \xc6\xc9\xce\xc1\xcc\xd8\xce\xd9\xca', 'koi8-r'), ('zz.', None)]) - hdr = make_header(parts) - eq(hdr.encode(), - 'Subject: =?koi8-r?b?8NLP18XSy8EgzsEgxsnOwczYztnK?= zz.') - - def test_broken_base64_header(self): - raises = self.assertRaises - s = 'Subject: =?EUC-KR?B?CSixpLDtKSC/7Liuvsax4iC6uLmwMcijIKHaILzSwd/H0SC8+LCjwLsgv7W/+Mj3I ?=' - raises(Errors.HeaderParseError, decode_header, s) - - # Issue 1078919 - def test_ascii_add_header(self): - msg = Message() - msg.add_header('Content-Disposition', 'attachment', - filename='bud.gif') - self.assertEqual('attachment; filename="bud.gif"', - msg['Content-Disposition']) - - def test_nonascii_add_header_via_triple(self): - msg = Message() - msg.add_header('Content-Disposition', 'attachment', - filename=('iso-8859-1', '', 'Fu\xdfballer.ppt')) - self.assertEqual( - 'attachment; filename*="iso-8859-1\'\'Fu%DFballer.ppt"', - msg['Content-Disposition']) - - def test_encode_unaliased_charset(self): - # Issue 1379416: when the charset has no output conversion, - # output was accidentally getting coerced to unicode. - res = Header('abc','iso-8859-2').encode() - self.assertEqual(res, '=?iso-8859-2?q?abc?=') - self.assertIsInstance(res, str) - - -# Test RFC 2231 header parameters (en/de)coding -class TestRFC2231(TestEmailBase): - def test_get_param(self): - eq = self.assertEqual - msg = self._msgobj('msg_29.txt') - eq(msg.get_param('title'), - ('us-ascii', 'en', 'This is even more ***fun*** isn\'t it!')) - eq(msg.get_param('title', unquote=False), - ('us-ascii', 'en', '"This is even more ***fun*** isn\'t it!"')) - - def test_set_param(self): - eq = self.assertEqual - msg = Message() - msg.set_param('title', 'This is even more ***fun*** isn\'t it!', - charset='us-ascii') - eq(msg.get_param('title'), - ('us-ascii', '', 'This is even more ***fun*** isn\'t it!')) - msg.set_param('title', 'This is even more ***fun*** isn\'t it!', - charset='us-ascii', language='en') - eq(msg.get_param('title'), - ('us-ascii', 'en', 'This is even more ***fun*** isn\'t it!')) - msg = self._msgobj('msg_01.txt') - msg.set_param('title', 'This is even more ***fun*** isn\'t it!', - charset='us-ascii', language='en') - self.ndiffAssertEqual(msg.as_string(), """\ -Return-Path: <bbb at zzz.org> -Delivered-To: bbb at zzz.org -Received: by mail.zzz.org (Postfix, from userid 889) - id 27CEAD38CC; Fri, 4 May 2001 14:05:44 -0400 (EDT) -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit -Message-ID: <15090.61304.110929.45684 at aaa.zzz.org> -From: bbb at ddd.com (John X. Doe) -To: bbb at zzz.org -Subject: This is a test message -Date: Fri, 4 May 2001 14:05:44 -0400 -Content-Type: text/plain; charset=us-ascii; - title*="us-ascii'en'This%20is%20even%20more%20%2A%2A%2Afun%2A%2A%2A%20isn%27t%20it%21" - - -Hi, - -Do you like this message? - --Me -""") - - def test_del_param(self): - eq = self.ndiffAssertEqual - msg = self._msgobj('msg_01.txt') - msg.set_param('foo', 'bar', charset='us-ascii', language='en') - msg.set_param('title', 'This is even more ***fun*** isn\'t it!', - charset='us-ascii', language='en') - msg.del_param('foo', header='Content-Type') - eq(msg.as_string(), """\ -Return-Path: <bbb at zzz.org> -Delivered-To: bbb at zzz.org -Received: by mail.zzz.org (Postfix, from userid 889) - id 27CEAD38CC; Fri, 4 May 2001 14:05:44 -0400 (EDT) -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit -Message-ID: <15090.61304.110929.45684 at aaa.zzz.org> -From: bbb at ddd.com (John X. Doe) -To: bbb at zzz.org -Subject: This is a test message -Date: Fri, 4 May 2001 14:05:44 -0400 -Content-Type: text/plain; charset="us-ascii"; - title*="us-ascii'en'This%20is%20even%20more%20%2A%2A%2Afun%2A%2A%2A%20isn%27t%20it%21" - - -Hi, - -Do you like this message? - --Me -""") - - def test_rfc2231_get_content_charset(self): - eq = self.assertEqual - msg = self._msgobj('msg_32.txt') - eq(msg.get_content_charset(), 'us-ascii') - - def test_rfc2231_no_language_or_charset(self): - m = '''\ -Content-Transfer-Encoding: 8bit -Content-Disposition: inline; filename="file____C__DOCUMENTS_20AND_20SETTINGS_FABIEN_LOCAL_20SETTINGS_TEMP_nsmail.htm" -Content-Type: text/html; NAME*0=file____C__DOCUMENTS_20AND_20SETTINGS_FABIEN_LOCAL_20SETTINGS_TEM; NAME*1=P_nsmail.htm - -''' - msg = email.message_from_string(m) - param = msg.get_param('NAME') - self.assertFalse(isinstance(param, tuple)) - self.assertEqual( - param, - 'file____C__DOCUMENTS_20AND_20SETTINGS_FABIEN_LOCAL_20SETTINGS_TEMP_nsmail.htm') - - def test_rfc2231_no_language_or_charset_in_filename(self): - m = '''\ -Content-Disposition: inline; -\tfilename*0*="''This%20is%20even%20more%20"; -\tfilename*1*="%2A%2A%2Afun%2A%2A%2A%20"; -\tfilename*2="is it not.pdf" - -''' - msg = email.message_from_string(m) - self.assertEqual(msg.get_filename(), - 'This is even more ***fun*** is it not.pdf') - - def test_rfc2231_no_language_or_charset_in_filename_encoded(self): - m = '''\ -Content-Disposition: inline; -\tfilename*0*="''This%20is%20even%20more%20"; -\tfilename*1*="%2A%2A%2Afun%2A%2A%2A%20"; -\tfilename*2="is it not.pdf" - -''' - msg = email.message_from_string(m) - self.assertEqual(msg.get_filename(), - 'This is even more ***fun*** is it not.pdf') - - def test_rfc2231_partly_encoded(self): - m = '''\ -Content-Disposition: inline; -\tfilename*0="''This%20is%20even%20more%20"; -\tfilename*1*="%2A%2A%2Afun%2A%2A%2A%20"; -\tfilename*2="is it not.pdf" - -''' - msg = email.message_from_string(m) - self.assertEqual( - msg.get_filename(), - 'This%20is%20even%20more%20***fun*** is it not.pdf') - - def test_rfc2231_partly_nonencoded(self): - m = '''\ -Content-Disposition: inline; -\tfilename*0="This%20is%20even%20more%20"; -\tfilename*1="%2A%2A%2Afun%2A%2A%2A%20"; -\tfilename*2="is it not.pdf" - -''' - msg = email.message_from_string(m) - self.assertEqual( - msg.get_filename(), - 'This%20is%20even%20more%20%2A%2A%2Afun%2A%2A%2A%20is it not.pdf') - - def test_rfc2231_no_language_or_charset_in_boundary(self): - m = '''\ -Content-Type: multipart/alternative; -\tboundary*0*="''This%20is%20even%20more%20"; -\tboundary*1*="%2A%2A%2Afun%2A%2A%2A%20"; -\tboundary*2="is it not.pdf" - -''' - msg = email.message_from_string(m) - self.assertEqual(msg.get_boundary(), - 'This is even more ***fun*** is it not.pdf') - - def test_rfc2231_no_language_or_charset_in_charset(self): - # This is a nonsensical charset value, but tests the code anyway - m = '''\ -Content-Type: text/plain; -\tcharset*0*="This%20is%20even%20more%20"; -\tcharset*1*="%2A%2A%2Afun%2A%2A%2A%20"; -\tcharset*2="is it not.pdf" - -''' - msg = email.message_from_string(m) - self.assertEqual(msg.get_content_charset(), - 'this is even more ***fun*** is it not.pdf') - - def test_rfc2231_bad_encoding_in_filename(self): - m = '''\ -Content-Disposition: inline; -\tfilename*0*="bogus'xx'This%20is%20even%20more%20"; -\tfilename*1*="%2A%2A%2Afun%2A%2A%2A%20"; -\tfilename*2="is it not.pdf" - -''' - msg = email.message_from_string(m) - self.assertEqual(msg.get_filename(), - 'This is even more ***fun*** is it not.pdf') - - def test_rfc2231_bad_encoding_in_charset(self): - m = """\ -Content-Type: text/plain; charset*=bogus''utf-8%E2%80%9D - -""" - msg = email.message_from_string(m) - # This should return None because non-ascii characters in the charset - # are not allowed. - self.assertEqual(msg.get_content_charset(), None) - - def test_rfc2231_bad_character_in_charset(self): - m = """\ -Content-Type: text/plain; charset*=ascii''utf-8%E2%80%9D - -""" - msg = email.message_from_string(m) - # This should return None because non-ascii characters in the charset - # are not allowed. - self.assertEqual(msg.get_content_charset(), None) - - def test_rfc2231_bad_character_in_filename(self): - m = '''\ -Content-Disposition: inline; -\tfilename*0*="ascii'xx'This%20is%20even%20more%20"; -\tfilename*1*="%2A%2A%2Afun%2A%2A%2A%20"; -\tfilename*2*="is it not.pdf%E2" - -''' - msg = email.message_from_string(m) - self.assertEqual(msg.get_filename(), - u'This is even more ***fun*** is it not.pdf\ufffd') - - def test_rfc2231_unknown_encoding(self): - m = """\ -Content-Transfer-Encoding: 8bit -Content-Disposition: inline; filename*=X-UNKNOWN''myfile.txt - -""" - msg = email.message_from_string(m) - self.assertEqual(msg.get_filename(), 'myfile.txt') - - def test_rfc2231_single_tick_in_filename_extended(self): - eq = self.assertEqual From noreply at buildbot.pypy.org Mon Jan 30 22:51:53 2012 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 30 Jan 2012 22:51:53 +0100 (CET) Subject: [pypy-commit] pypy default: remove unused impot Message-ID: <20120130215153.E54B67106F3@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: Changeset: r51979:55410cd3cef5 Date: 2012-01-30 23:49 +0200 http://bitbucket.org/pypy/pypy/changeset/55410cd3cef5/ Log: remove unused impot diff --git a/pypy/jit/metainterp/history.py b/pypy/jit/metainterp/history.py --- a/pypy/jit/metainterp/history.py +++ b/pypy/jit/metainterp/history.py @@ -2,7 +2,7 @@ from pypy.rpython.extregistry import ExtRegistryEntry from pypy.rpython.lltypesystem import lltype, llmemory, rffi from pypy.rpython.ootypesystem import ootype -from pypy.rlib.objectmodel import we_are_translated, r_dict, Symbolic +from pypy.rlib.objectmodel import we_are_translated, Symbolic from pypy.rlib.objectmodel import compute_unique_id from pypy.rlib.rarithmetic import r_int64 from pypy.conftest import option From noreply at buildbot.pypy.org Mon Jan 30 22:51:55 2012 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 30 Jan 2012 22:51:55 +0100 (CET) Subject: [pypy-commit] pypy default: merge Message-ID: <20120130215155.9BFA47106F3@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: Changeset: r51980:43588eb36c3d Date: 2012-01-30 23:51 +0200 http://bitbucket.org/pypy/pypy/changeset/43588eb36c3d/ Log: merge diff too long, truncating to 10000 out of 14115 lines diff --git a/lib-python/modified-2.7/email/__init__.py b/lib-python/modified-2.7/email/__init__.py deleted file mode 100644 --- a/lib-python/modified-2.7/email/__init__.py +++ /dev/null @@ -1,123 +0,0 @@ -# Copyright (C) 2001-2006 Python Software Foundation -# Author: Barry Warsaw -# Contact: email-sig at python.org - -"""A package for parsing, handling, and generating email messages.""" - -__version__ = '4.0.3' - -__all__ = [ - # Old names - 'base64MIME', - 'Charset', - 'Encoders', - 'Errors', - 'Generator', - 'Header', - 'Iterators', - 'Message', - 'MIMEAudio', - 'MIMEBase', - 'MIMEImage', - 'MIMEMessage', - 'MIMEMultipart', - 'MIMENonMultipart', - 'MIMEText', - 'Parser', - 'quopriMIME', - 'Utils', - 'message_from_string', - 'message_from_file', - # new names - 'base64mime', - 'charset', - 'encoders', - 'errors', - 'generator', - 'header', - 'iterators', - 'message', - 'mime', - 'parser', - 'quoprimime', - 'utils', - ] - - - -# Some convenience routines. Don't import Parser and Message as side-effects -# of importing email since those cascadingly import most of the rest of the -# email package. -def message_from_string(s, *args, **kws): - """Parse a string into a Message object model. - - Optional _class and strict are passed to the Parser constructor. - """ - from email.parser import Parser - return Parser(*args, **kws).parsestr(s) - - -def message_from_file(fp, *args, **kws): - """Read a file and parse its contents into a Message object model. - - Optional _class and strict are passed to the Parser constructor. - """ - from email.parser import Parser - return Parser(*args, **kws).parse(fp) - - - -# Lazy loading to provide name mapping from new-style names (PEP 8 compatible -# email 4.0 module names), to old-style names (email 3.0 module names). -import sys - -class LazyImporter(object): - def __init__(self, module_name): - self.__name__ = 'email.' + module_name - - def __getattr__(self, name): - __import__(self.__name__) - mod = sys.modules[self.__name__] - self.__dict__.update(mod.__dict__) - return getattr(mod, name) - - -_LOWERNAMES = [ - # email.<old name> -> email.<new name is lowercased old name> - 'Charset', - 'Encoders', - 'Errors', - 'FeedParser', - 'Generator', - 'Header', - 'Iterators', - 'Message', - 'Parser', - 'Utils', - 'base64MIME', - 'quopriMIME', - ] - -_MIMENAMES = [ - # email.MIME<old name> -> email.mime.<new name is lowercased old name> - 'Audio', - 'Base', - 'Image', - 'Message', - 'Multipart', - 'NonMultipart', - 'Text', - ] - -for _name in _LOWERNAMES: - importer = LazyImporter(_name.lower()) - sys.modules['email.' + _name] = importer - setattr(sys.modules['email'], _name, importer) - - -import email.mime -for _name in _MIMENAMES: - importer = LazyImporter('mime.' + _name.lower()) - sys.modules['email.MIME' + _name] = importer - setattr(sys.modules['email'], 'MIME' + _name, importer) - setattr(sys.modules['email.mime'], _name, importer) diff --git a/lib-python/modified-2.7/email/_parseaddr.py b/lib-python/modified-2.7/email/_parseaddr.py deleted file mode 100644 --- a/lib-python/modified-2.7/email/_parseaddr.py +++ /dev/null @@ -1,497 +0,0 @@ -# Copyright (C) 2002-2007 Python Software Foundation -# Contact: email-sig at python.org - -"""Email address parsing code. - -Lifted directly from rfc822.py. This should eventually be rewritten. -""" - -__all__ = [ - 'mktime_tz', - 'parsedate', - 'parsedate_tz', - 'quote', - ] - -import time - -SPACE = ' ' -EMPTYSTRING = '' -COMMASPACE = ', ' - -# Parse a date field -_monthnames = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', - 'aug', 'sep', 'oct', 'nov', 'dec', - 'january', 'february', 'march', 'april', 'may', 'june', 'july', - 'august', 'september', 'october', 'november', 'december'] - -_daynames = ['mon', 'tue', 'wed', 'thu', 'fri', 'sat', 'sun'] - -# The timezone table does not include the military time zones defined -# in RFC822, other than Z. According to RFC1123, the description in -# RFC822 gets the signs wrong, so we can't rely on any such time -# zones. RFC1123 recommends that numeric timezone indicators be used -# instead of timezone names. - -_timezones = {'UT':0, 'UTC':0, 'GMT':0, 'Z':0, - 'AST': -400, 'ADT': -300, # Atlantic (used in Canada) - 'EST': -500, 'EDT': -400, # Eastern - 'CST': -600, 'CDT': -500, # Central - 'MST': -700, 'MDT': -600, # Mountain - 'PST': -800, 'PDT': -700 # Pacific - } - - -def parsedate_tz(data): - """Convert a date string to a time tuple. - - Accounts for military timezones. - """ - data = data.split() - # The FWS after the comma after the day-of-week is optional, so search and - # adjust for this. - if data[0].endswith(',') or data[0].lower() in _daynames: - # There's a dayname here. Skip it - del data[0] - else: - i = data[0].rfind(',') - if i >= 0: - data[0] = data[0][i+1:] - if len(data) == 3: # RFC 850 date, deprecated - stuff = data[0].split('-') - if len(stuff) == 3: - data = stuff + data[1:] - if len(data) == 4: - s = data[3] - i = s.find('+') - if i > 0: - data[3:] = [s[:i], s[i+1:]] - else: - data.append('') # Dummy tz - if len(data) < 5: - return None - data = data[:5] - [dd, mm, yy, tm, tz] = data - mm = mm.lower() - if mm not in _monthnames: - dd, mm = mm, dd.lower() - if mm not in _monthnames: - return None - mm = _monthnames.index(mm) + 1 - if mm > 12: - mm -= 12 - if dd[-1] == ',': - dd = dd[:-1] - i = yy.find(':') - if i > 0: - yy, tm = tm, yy - if yy[-1] == ',': - yy = yy[:-1] - if not yy[0].isdigit(): - yy, tz = tz, yy - if tm[-1] == ',': - tm = tm[:-1] - tm = tm.split(':') - if len(tm) == 2: - [thh, tmm] = tm - tss = '0' - elif len(tm) == 3: - [thh, tmm, tss] = tm - else: - return None - try: - yy = int(yy) - dd = int(dd) - thh = int(thh) - tmm = int(tmm) - tss = int(tss) - except ValueError: - return None - # Check for a yy specified in two-digit format, then convert it to the - # appropriate four-digit format, according to the POSIX standard. RFC 822 - # calls for a two-digit yy, but RFC 2822 (which obsoletes RFC 822) - # mandates a 4-digit yy. For more information, see the documentation for - # the time module. - if yy < 100: - # The year is between 1969 and 1999 (inclusive). - if yy > 68: - yy += 1900 - # The year is between 2000 and 2068 (inclusive). - else: - yy += 2000 - tzoffset = None - tz = tz.upper() - if tz in _timezones: - tzoffset = _timezones[tz] - else: - try: - tzoffset = int(tz) - except ValueError: - pass - # Convert a timezone offset into seconds ; -0500 -> -18000 - if tzoffset: - if tzoffset < 0: - tzsign = -1 - tzoffset = -tzoffset - else: - tzsign = 1 - tzoffset = tzsign * ( (tzoffset//100)*3600 + (tzoffset % 100)*60) - # Daylight Saving Time flag is set to -1, since DST is unknown. - return yy, mm, dd, thh, tmm, tss, 0, 1, -1, tzoffset - - -def parsedate(data): - """Convert a time string to a time tuple.""" - t = parsedate_tz(data) - if isinstance(t, tuple): - return t[:9] - else: - return t - - -def mktime_tz(data): - """Turn a 10-tuple as returned by parsedate_tz() into a UTC timestamp.""" - if data[9] is None: - # No zone info, so localtime is better assumption than GMT - return time.mktime(data[:8] + (-1,)) - else: - t = time.mktime(data[:8] + (0,)) - return t - data[9] - time.timezone - - -def quote(str): - """Prepare string to be used in a quoted string. - - Turns backslash and double quote characters into quoted pairs. These - are the only characters that need to be quoted inside a quoted string. - Does not add the surrounding double quotes. - """ - return str.replace('\\', '\\\\').replace('"', '\\"') - - -class AddrlistClass: - """Address parser class by Ben Escoto. - - To understand what this class does, it helps to have a copy of RFC 2822 in - front of you. - - Note: this class interface is deprecated and may be removed in the future. - Use rfc822.AddressList instead. - """ - - def __init__(self, field): - """Initialize a new instance. - - `field' is an unparsed address header field, containing - one or more addresses. - """ - self.specials = '()<>@,:;.\"[]' - self.pos = 0 - self.LWS = ' \t' - self.CR = '\r\n' - self.FWS = self.LWS + self.CR - self.atomends = self.specials + self.LWS + self.CR - # Note that RFC 2822 now specifies `.' as obs-phrase, meaning that it - # is obsolete syntax. RFC 2822 requires that we recognize obsolete - # syntax, so allow dots in phrases. - self.phraseends = self.atomends.replace('.', '') - self.field = field - self.commentlist = [] - - def gotonext(self): - """Parse up to the start of the next address.""" - while self.pos < len(self.field): - if self.field[self.pos] in self.LWS + '\n\r': - self.pos += 1 - elif self.field[self.pos] == '(': - self.commentlist.append(self.getcomment()) - else: - break - - def getaddrlist(self): - """Parse all addresses. - - Returns a list containing all of the addresses. - """ - result = [] - while self.pos < len(self.field): - ad = self.getaddress() - if ad: - result += ad - else: - result.append(('', '')) - return result - - def getaddress(self): - """Parse the next address.""" - self.commentlist = [] - self.gotonext() - - oldpos = self.pos - oldcl = self.commentlist - plist = self.getphraselist() - - self.gotonext() - returnlist = [] - - if self.pos >= len(self.field): - # Bad email address technically, no domain. - if plist: - returnlist = [(SPACE.join(self.commentlist), plist[0])] - - elif self.field[self.pos] in '.@': - # email address is just an addrspec - # this isn't very efficient since we start over - self.pos = oldpos - self.commentlist = oldcl - addrspec = self.getaddrspec() - returnlist = [(SPACE.join(self.commentlist), addrspec)] - - elif self.field[self.pos] == ':': - # address is a group - returnlist = [] - - fieldlen = len(self.field) - self.pos += 1 - while self.pos < len(self.field): - self.gotonext() - if self.pos < fieldlen and self.field[self.pos] == ';': - self.pos += 1 - break - returnlist = returnlist + self.getaddress() - - elif self.field[self.pos] == '<': - # Address is a phrase then a route addr - routeaddr = self.getrouteaddr() - - if self.commentlist: - returnlist = [(SPACE.join(plist) + ' (' + - ' '.join(self.commentlist) + ')', routeaddr)] - else: - returnlist = [(SPACE.join(plist), routeaddr)] - - else: - if plist: - returnlist = [(SPACE.join(self.commentlist), plist[0])] - elif self.field[self.pos] in self.specials: - self.pos += 1 - - self.gotonext() - if self.pos < len(self.field) and self.field[self.pos] == ',': - self.pos += 1 - return returnlist - - def getrouteaddr(self): - """Parse a route address (Return-path value). - - This method just skips all the route stuff and returns the addrspec. - """ - if self.field[self.pos] != '<': - return - - expectroute = False - self.pos += 1 - self.gotonext() - adlist = '' - while self.pos < len(self.field): - if expectroute: - self.getdomain() - expectroute = False - elif self.field[self.pos] == '>': - self.pos += 1 - break - elif self.field[self.pos] == '@': - self.pos += 1 - expectroute = True - elif self.field[self.pos] == ':': - self.pos += 1 - else: - adlist = self.getaddrspec() - self.pos += 1 - break - self.gotonext() - - return adlist - - def getaddrspec(self): - """Parse an RFC 2822 addr-spec.""" - aslist = [] - - self.gotonext() - while self.pos < len(self.field): - if self.field[self.pos] == '.': - aslist.append('.') - self.pos += 1 - elif self.field[self.pos] == '"': - aslist.append('"%s"' % quote(self.getquote())) - elif self.field[self.pos] in self.atomends: - break - else: - aslist.append(self.getatom()) - self.gotonext() - - if self.pos >= len(self.field) or self.field[self.pos] != '@': - return EMPTYSTRING.join(aslist) - - aslist.append('@') - self.pos += 1 - self.gotonext() - return EMPTYSTRING.join(aslist) + self.getdomain() - - def getdomain(self): - """Get the complete domain name from an address.""" - sdlist = [] - while self.pos < len(self.field): - if self.field[self.pos] in self.LWS: - self.pos += 1 - elif self.field[self.pos] == '(': - self.commentlist.append(self.getcomment()) - elif self.field[self.pos] == '[': - sdlist.append(self.getdomainliteral()) - elif self.field[self.pos] == '.': - self.pos += 1 - sdlist.append('.') - elif self.field[self.pos] in self.atomends: - break - else: - sdlist.append(self.getatom()) - return EMPTYSTRING.join(sdlist) - - def getdelimited(self, beginchar, endchars, allowcomments=True): - """Parse a header fragment delimited by special characters. - - `beginchar' is the start character for the fragment. - If self is not looking at an instance of `beginchar' then - getdelimited returns the empty string. - - `endchars' is a sequence of allowable end-delimiting characters. - Parsing stops when one of these is encountered. - - If `allowcomments' is non-zero, embedded RFC 2822 comments are allowed - within the parsed fragment. - """ - if self.field[self.pos] != beginchar: - return '' - - slist = [''] - quote = False - self.pos += 1 - while self.pos < len(self.field): - if quote: - slist.append(self.field[self.pos]) - quote = False - elif self.field[self.pos] in endchars: - self.pos += 1 - break - elif allowcomments and self.field[self.pos] == '(': - slist.append(self.getcomment()) - continue # have already advanced pos from getcomment - elif self.field[self.pos] == '\\': - quote = True - else: - slist.append(self.field[self.pos]) - self.pos += 1 - - return EMPTYSTRING.join(slist) - - def getquote(self): - """Get a quote-delimited fragment from self's field.""" - return self.getdelimited('"', '"\r', False) - - def getcomment(self): - """Get a parenthesis-delimited fragment from self's field.""" - return self.getdelimited('(', ')\r', True) - - def getdomainliteral(self): - """Parse an RFC 2822 domain-literal.""" - return '[%s]' % self.getdelimited('[', ']\r', False) - - def getatom(self, atomends=None): - """Parse an RFC 2822 atom. - - Optional atomends specifies a different set of end token delimiters - (the default is to use self.atomends). This is used e.g. in - getphraselist() since phrase endings must not include the `.' (which - is legal in phrases).""" - atomlist = [''] - if atomends is None: - atomends = self.atomends - - while self.pos < len(self.field): - if self.field[self.pos] in atomends: - break - else: - atomlist.append(self.field[self.pos]) - self.pos += 1 - - return EMPTYSTRING.join(atomlist) - - def getphraselist(self): - """Parse a sequence of RFC 2822 phrases. - - A phrase is a sequence of words, which are in turn either RFC 2822 - atoms or quoted-strings. Phrases are canonicalized by squeezing all - runs of continuous whitespace into one space. - """ - plist = [] - - while self.pos < len(self.field): - if self.field[self.pos] in self.FWS: - self.pos += 1 - elif self.field[self.pos] == '"': - plist.append(self.getquote()) - elif self.field[self.pos] == '(': - self.commentlist.append(self.getcomment()) - elif self.field[self.pos] in self.phraseends: - break - else: - plist.append(self.getatom(self.phraseends)) - - return plist - -class AddressList(AddrlistClass): - """An AddressList encapsulates a list of parsed RFC 2822 addresses.""" - def __init__(self, field): - AddrlistClass.__init__(self, field) - if field: - self.addresslist = self.getaddrlist() - else: - self.addresslist = [] - - def __len__(self): - return len(self.addresslist) - - def __add__(self, other): - # Set union - newaddr = AddressList(None) - newaddr.addresslist = self.addresslist[:] - for x in other.addresslist: - if not x in self.addresslist: - newaddr.addresslist.append(x) - return newaddr - - def __iadd__(self, other): - # Set union, in-place - for x in other.addresslist: - if not x in self.addresslist: - self.addresslist.append(x) - return self - - def __sub__(self, other): - # Set difference - newaddr = AddressList(None) - for x in self.addresslist: - if not x in other.addresslist: - newaddr.addresslist.append(x) - return newaddr - - def __isub__(self, other): - # Set difference, in-place - for x in other.addresslist: - if x in self.addresslist: - self.addresslist.remove(x) - return self - - def __getitem__(self, index): - # Make indexing, slices, and 'in' work - return self.addresslist[index] diff --git a/lib-python/modified-2.7/email/base64mime.py b/lib-python/modified-2.7/email/base64mime.py deleted file mode 100644 --- a/lib-python/modified-2.7/email/base64mime.py +++ /dev/null @@ -1,183 +0,0 @@ -# Copyright (C) 2002-2006 Python Software Foundation -# Author: Ben Gertzfield -# Contact: email-sig at python.org - -"""Base64 content transfer encoding per RFCs 2045-2047. - -This module handles the content transfer encoding method defined in RFC 2045 -to encode arbitrary 8-bit data using the three 8-bit bytes in four 7-bit -characters encoding known as Base64. - -It is used in the MIME standards for email to attach images, audio, and text -using some 8-bit character sets to messages. - -This module provides an interface to encode and decode both headers and bodies -with Base64 encoding. - -RFC 2045 defines a method for including character set information in an -`encoded-word' in a header. This method is commonly used for 8-bit real names -in To:, From:, Cc:, etc. fields, as well as Subject: lines. - -This module does not do the line wrapping or end-of-line character conversion -necessary for proper internationalized headers; it only does dumb encoding and -decoding. To deal with the various line wrapping issues, use the email.header -module. -""" - -__all__ = [ - 'base64_len', - 'body_decode', - 'body_encode', - 'decode', - 'decodestring', - 'encode', - 'encodestring', - 'header_encode', - ] - - -from binascii import b2a_base64, a2b_base64 -from email.utils import fix_eols - -CRLF = '\r\n' -NL = '\n' -EMPTYSTRING = '' - -# See also Charset.py -MISC_LEN = 7 - - - -# Helpers -def base64_len(s): - """Return the length of s when it is encoded with base64.""" - groups_of_3, leftover = divmod(len(s), 3) - # 4 bytes out for each 3 bytes (or nonzero fraction thereof) in. - # Thanks, Tim! - n = groups_of_3 * 4 - if leftover: - n += 4 - return n - - - -def header_encode(header, charset='iso-8859-1', keep_eols=False, - maxlinelen=76, eol=NL): - """Encode a single header line with Base64 encoding in a given charset. - - Defined in RFC 2045, this Base64 encoding is identical to normal Base64 - encoding, except that each line must be intelligently wrapped (respecting - the Base64 encoding), and subsequent lines must start with a space. - - charset names the character set to use to encode the header. It defaults - to iso-8859-1. - - End-of-line characters (\\r, \\n, \\r\\n) will be automatically converted - to the canonical email line separator \\r\\n unless the keep_eols - parameter is True (the default is False). - - Each line of the header will be terminated in the value of eol, which - defaults to "\\n". Set this to "\\r\\n" if you are using the result of - this function directly in email. - - The resulting string will be in the form: - - "=?charset?b?WW/5ciBtYXp66XLrIHf8eiBhIGhhbXBzdGHuciBBIFlv+XIgbWF6euly?=\\n - =?charset?b?6yB3/HogYSBoYW1wc3Rh7nIgQkMgWW/5ciBtYXp66XLrIHf8eiBhIGhh?=" - - with each line wrapped at, at most, maxlinelen characters (defaults to 76 - characters). - """ - # Return empty headers unchanged - if not header: - return header - - if not keep_eols: - header = fix_eols(header) - - # Base64 encode each line, in encoded chunks no greater than maxlinelen in - # length, after the RFC chrome is added in. - base64ed = [] - max_encoded = maxlinelen - len(charset) - MISC_LEN - max_unencoded = max_encoded * 3 // 4 - - for i in range(0, len(header), max_unencoded): - base64ed.append(b2a_base64(header[i:i+max_unencoded])) - - # Now add the RFC chrome to each encoded chunk - lines = [] - for line in base64ed: - # Ignore the last character of each line if it is a newline - if line.endswith(NL): - line = line[:-1] - # Add the chrome - lines.append('=?%s?b?%s?=' % (charset, line)) - # Glue the lines together and return it. BAW: should we be able to - # specify the leading whitespace in the joiner? - joiner = eol + ' ' - return joiner.join(lines) - - - -def encode(s, binary=True, maxlinelen=76, eol=NL): - """Encode a string with base64. - - Each line will be wrapped at, at most, maxlinelen characters (defaults to - 76 characters). - - If binary is False, end-of-line characters will be converted to the - canonical email end-of-line sequence \\r\\n. Otherwise they will be left - verbatim (this is the default). - - Each line of encoded text will end with eol, which defaults to "\\n". Set - this to "\r\n" if you will be using the result of this function directly - in an email. - """ - if not s: - return s - - if not binary: - s = fix_eols(s) - - encvec = [] - max_unencoded = maxlinelen * 3 // 4 - for i in range(0, len(s), max_unencoded): - # BAW: should encode() inherit b2a_base64()'s dubious behavior in - # adding a newline to the encoded string? - enc = b2a_base64(s[i:i + max_unencoded]) - if enc.endswith(NL) and eol != NL: - enc = enc[:-1] + eol - encvec.append(enc) - return EMPTYSTRING.join(encvec) - - -# For convenience and backwards compatibility w/ standard base64 module -body_encode = encode -encodestring = encode - - - -def decode(s, convert_eols=None): - """Decode a raw base64 string. - - If convert_eols is set to a string value, all canonical email linefeeds, - e.g. "\\r\\n", in the decoded text will be converted to the value of - convert_eols. os.linesep is a good choice for convert_eols if you are - decoding a text attachment. - - This function does not parse a full MIME header value encoded with - base64 (like =?iso-8895-1?b?bmloISBuaWgh?=) -- please use the high - level email.header class for that functionality. - """ - if not s: - return s - - dec = a2b_base64(s) - if convert_eols: - return dec.replace(CRLF, convert_eols) - return dec - - -# For convenience and backwards compatibility w/ standard base64 module -body_decode = decode -decodestring = decode diff --git a/lib-python/modified-2.7/email/charset.py b/lib-python/modified-2.7/email/charset.py deleted file mode 100644 --- a/lib-python/modified-2.7/email/charset.py +++ /dev/null @@ -1,397 +0,0 @@ -# Copyright (C) 2001-2006 Python Software Foundation -# Author: Ben Gertzfield, Barry Warsaw -# Contact: email-sig at python.org - -__all__ = [ - 'Charset', - 'add_alias', - 'add_charset', - 'add_codec', - ] - -import codecs -import email.base64mime -import email.quoprimime - -from email import errors -from email.encoders import encode_7or8bit - - - -# Flags for types of header encodings -QP = 1 # Quoted-Printable -BASE64 = 2 # Base64 -SHORTEST = 3 # the shorter of QP and base64, but only for headers - -# In "=?charset?q?hello_world?=", the =?, ?q?, and ?= add up to 7 -MISC_LEN = 7 - -DEFAULT_CHARSET = 'us-ascii' - - - -# Defaults -CHARSETS = { - # input header enc body enc output conv - 'iso-8859-1': (QP, QP, None), - 'iso-8859-2': (QP, QP, None), - 'iso-8859-3': (QP, QP, None), - 'iso-8859-4': (QP, QP, None), - # iso-8859-5 is Cyrillic, and not especially used - # iso-8859-6 is Arabic, also not particularly used - # iso-8859-7 is Greek, QP will not make it readable - # iso-8859-8 is Hebrew, QP will not make it readable - 'iso-8859-9': (QP, QP, None), - 'iso-8859-10': (QP, QP, None), - # iso-8859-11 is Thai, QP will not make it readable - 'iso-8859-13': (QP, QP, None), - 'iso-8859-14': (QP, QP, None), - 'iso-8859-15': (QP, QP, None), - 'iso-8859-16': (QP, QP, None), - 'windows-1252':(QP, QP, None), - 'viscii': (QP, QP, None), - 'us-ascii': (None, None, None), - 'big5': (BASE64, BASE64, None), - 'gb2312': (BASE64, BASE64, None), - 'euc-jp': (BASE64, None, 'iso-2022-jp'), - 'shift_jis': (BASE64, None, 'iso-2022-jp'), - 'iso-2022-jp': (BASE64, None, None), - 'koi8-r': (BASE64, BASE64, None), - 'utf-8': (SHORTEST, BASE64, 'utf-8'), - # We're making this one up to represent raw unencoded 8-bit - '8bit': (None, BASE64, 'utf-8'), - } - -# Aliases for other commonly-used names for character sets. Map -# them to the real ones used in email. -ALIASES = { - 'latin_1': 'iso-8859-1', - 'latin-1': 'iso-8859-1', - 'latin_2': 'iso-8859-2', - 'latin-2': 'iso-8859-2', - 'latin_3': 'iso-8859-3', - 'latin-3': 'iso-8859-3', - 'latin_4': 'iso-8859-4', - 'latin-4': 'iso-8859-4', - 'latin_5': 'iso-8859-9', - 'latin-5': 'iso-8859-9', - 'latin_6': 'iso-8859-10', - 'latin-6': 'iso-8859-10', - 'latin_7': 'iso-8859-13', - 'latin-7': 'iso-8859-13', - 'latin_8': 'iso-8859-14', - 'latin-8': 'iso-8859-14', - 'latin_9': 'iso-8859-15', - 'latin-9': 'iso-8859-15', - 'latin_10':'iso-8859-16', - 'latin-10':'iso-8859-16', - 'cp949': 'ks_c_5601-1987', - 'euc_jp': 'euc-jp', - 'euc_kr': 'euc-kr', - 'ascii': 'us-ascii', - } - - -# Map charsets to their Unicode codec strings. -CODEC_MAP = { - 'gb2312': 'eucgb2312_cn', - 'big5': 'big5_tw', - # Hack: We don't want *any* conversion for stuff marked us-ascii, as all - # sorts of garbage might be sent to us in the guise of 7-bit us-ascii. - # Let that stuff pass through without conversion to/from Unicode. - 'us-ascii': None, - } - - - -# Convenience functions for extending the above mappings -def add_charset(charset, header_enc=None, body_enc=None, output_charset=None): - """Add character set properties to the global registry. - - charset is the input character set, and must be the canonical name of a - character set. - - Optional header_enc and body_enc is either Charset.QP for - quoted-printable, Charset.BASE64 for base64 encoding, Charset.SHORTEST for - the shortest of qp or base64 encoding, or None for no encoding. SHORTEST - is only valid for header_enc. It describes how message headers and - message bodies in the input charset are to be encoded. Default is no - encoding. - - Optional output_charset is the character set that the output should be - in. Conversions will proceed from input charset, to Unicode, to the - output charset when the method Charset.convert() is called. The default - is to output in the same character set as the input. - - Both input_charset and output_charset must have Unicode codec entries in - the module's charset-to-codec mapping; use add_codec(charset, codecname) - to add codecs the module does not know about. See the codecs module's - documentation for more information. - """ - if body_enc == SHORTEST: - raise ValueError('SHORTEST not allowed for body_enc') - CHARSETS[charset] = (header_enc, body_enc, output_charset) - - -def add_alias(alias, canonical): - """Add a character set alias. - - alias is the alias name, e.g. latin-1 - canonical is the character set's canonical name, e.g. iso-8859-1 - """ - ALIASES[alias] = canonical - - -def add_codec(charset, codecname): - """Add a codec that map characters in the given charset to/from Unicode. - - charset is the canonical name of a character set. codecname is the name - of a Python codec, as appropriate for the second argument to the unicode() - built-in, or to the encode() method of a Unicode string. - """ - CODEC_MAP[charset] = codecname - - - -class Charset: - """Map character sets to their email properties. - - This class provides information about the requirements imposed on email - for a specific character set. It also provides convenience routines for - converting between character sets, given the availability of the - applicable codecs. Given a character set, it will do its best to provide - information on how to use that character set in an email in an - RFC-compliant way. - - Certain character sets must be encoded with quoted-printable or base64 - when used in email headers or bodies. Certain character sets must be - converted outright, and are not allowed in email. Instances of this - module expose the following information about a character set: - - input_charset: The initial character set specified. Common aliases - are converted to their `official' email names (e.g. latin_1 - is converted to iso-8859-1). Defaults to 7-bit us-ascii. - - header_encoding: If the character set must be encoded before it can be - used in an email header, this attribute will be set to - Charset.QP (for quoted-printable), Charset.BASE64 (for - base64 encoding), or Charset.SHORTEST for the shortest of - QP or BASE64 encoding. Otherwise, it will be None. - - body_encoding: Same as header_encoding, but describes the encoding for the - mail message's body, which indeed may be different than the - header encoding. Charset.SHORTEST is not allowed for - body_encoding. - - output_charset: Some character sets must be converted before the can be - used in email headers or bodies. If the input_charset is - one of them, this attribute will contain the name of the - charset output will be converted to. Otherwise, it will - be None. - - input_codec: The name of the Python codec used to convert the - input_charset to Unicode. If no conversion codec is - necessary, this attribute will be None. - - output_codec: The name of the Python codec used to convert Unicode - to the output_charset. If no conversion codec is necessary, - this attribute will have the same value as the input_codec. - """ - def __init__(self, input_charset=DEFAULT_CHARSET): - # RFC 2046, $4.1.2 says charsets are not case sensitive. We coerce to - # unicode because its .lower() is locale insensitive. If the argument - # is already a unicode, we leave it at that, but ensure that the - # charset is ASCII, as the standard (RFC XXX) requires. - try: - if isinstance(input_charset, unicode): - input_charset.encode('ascii') - else: - input_charset = unicode(input_charset, 'ascii') - except UnicodeError: - raise errors.CharsetError(input_charset) - input_charset = input_charset.lower().encode('ascii') - # Set the input charset after filtering through the aliases and/or codecs - if not (input_charset in ALIASES or input_charset in CHARSETS): - try: - input_charset = codecs.lookup(input_charset).name - except LookupError: - pass - self.input_charset = ALIASES.get(input_charset, input_charset) - # We can try to guess which encoding and conversion to use by the - # charset_map dictionary. Try that first, but let the user override - # it. - henc, benc, conv = CHARSETS.get(self.input_charset, - (SHORTEST, BASE64, None)) - if not conv: - conv = self.input_charset - # Set the attributes, allowing the arguments to override the default. - self.header_encoding = henc - self.body_encoding = benc - self.output_charset = ALIASES.get(conv, conv) - # Now set the codecs. If one isn't defined for input_charset, - # guess and try a Unicode codec with the same name as input_codec. - self.input_codec = CODEC_MAP.get(self.input_charset, - self.input_charset) - self.output_codec = CODEC_MAP.get(self.output_charset, - self.output_charset) - - def __str__(self): - return self.input_charset.lower() - - __repr__ = __str__ - - def __eq__(self, other): - return str(self) == str(other).lower() - - def __ne__(self, other): - return not self.__eq__(other) - - def get_body_encoding(self): - """Return the content-transfer-encoding used for body encoding. - - This is either the string `quoted-printable' or `base64' depending on - the encoding used, or it is a function in which case you should call - the function with a single argument, the Message object being - encoded. The function should then set the Content-Transfer-Encoding - header itself to whatever is appropriate. - - Returns "quoted-printable" if self.body_encoding is QP. - Returns "base64" if self.body_encoding is BASE64. - Returns "7bit" otherwise. - """ - assert self.body_encoding != SHORTEST - if self.body_encoding == QP: - return 'quoted-printable' - elif self.body_encoding == BASE64: - return 'base64' - else: - return encode_7or8bit - - def convert(self, s): - """Convert a string from the input_codec to the output_codec.""" - if self.input_codec != self.output_codec: - return unicode(s, self.input_codec).encode(self.output_codec) - else: - return s - - def to_splittable(self, s): - """Convert a possibly multibyte string to a safely splittable format. - - Uses the input_codec to try and convert the string to Unicode, so it - can be safely split on character boundaries (even for multibyte - characters). - - Returns the string as-is if it isn't known how to convert it to - Unicode with the input_charset. - - Characters that could not be converted to Unicode will be replaced - with the Unicode replacement character U+FFFD. - """ - if isinstance(s, unicode) or self.input_codec is None: - return s - try: - return unicode(s, self.input_codec, 'replace') - except LookupError: - # Input codec not installed on system, so return the original - # string unchanged. - return s - - def from_splittable(self, ustr, to_output=True): - """Convert a splittable string back into an encoded string. - - Uses the proper codec to try and convert the string from Unicode back - into an encoded format. Return the string as-is if it is not Unicode, - or if it could not be converted from Unicode. - - Characters that could not be converted from Unicode will be replaced - with an appropriate character (usually '?'). - - If to_output is True (the default), uses output_codec to convert to an - encoded format. If to_output is False, uses input_codec. - """ - if to_output: - codec = self.output_codec - else: - codec = self.input_codec - if not isinstance(ustr, unicode) or codec is None: - return ustr - try: - return ustr.encode(codec, 'replace') - except LookupError: - # Output codec not installed - return ustr - - def get_output_charset(self): - """Return the output character set. - - This is self.output_charset if that is not None, otherwise it is - self.input_charset. - """ - return self.output_charset or self.input_charset - - def encoded_header_len(self, s): - """Return the length of the encoded header string.""" - cset = self.get_output_charset() - # The len(s) of a 7bit encoding is len(s) - if self.header_encoding == BASE64: - return email.base64mime.base64_len(s) + len(cset) + MISC_LEN - elif self.header_encoding == QP: - return email.quoprimime.header_quopri_len(s) + len(cset) + MISC_LEN - elif self.header_encoding == SHORTEST: - lenb64 = email.base64mime.base64_len(s) - lenqp = email.quoprimime.header_quopri_len(s) - return min(lenb64, lenqp) + len(cset) + MISC_LEN - else: - return len(s) - - def header_encode(self, s, convert=False): - """Header-encode a string, optionally converting it to output_charset. - - If convert is True, the string will be converted from the input - charset to the output charset automatically. This is not useful for - multibyte character sets, which have line length issues (multibyte - characters must be split on a character, not a byte boundary); use the - high-level Header class to deal with these issues. convert defaults - to False. - - The type of encoding (base64 or quoted-printable) will be based on - self.header_encoding. - """ - cset = self.get_output_charset() - if convert: - s = self.convert(s) - # 7bit/8bit encodings return the string unchanged (modulo conversions) - if self.header_encoding == BASE64: - return email.base64mime.header_encode(s, cset) - elif self.header_encoding == QP: - return email.quoprimime.header_encode(s, cset, maxlinelen=None) - elif self.header_encoding == SHORTEST: - lenb64 = email.base64mime.base64_len(s) - lenqp = email.quoprimime.header_quopri_len(s) - if lenb64 < lenqp: - return email.base64mime.header_encode(s, cset) - else: - return email.quoprimime.header_encode(s, cset, maxlinelen=None) - else: - return s - - def body_encode(self, s, convert=True): - """Body-encode a string and convert it to output_charset. - - If convert is True (the default), the string will be converted from - the input charset to output charset automatically. Unlike - header_encode(), there are no issues with byte boundaries and - multibyte charsets in email bodies, so this is usually pretty safe. - - The type of encoding (base64 or quoted-printable) will be based on - self.body_encoding. - """ - if convert: - s = self.convert(s) - # 7bit/8bit encodings return the string unchanged (module conversions) - if self.body_encoding is BASE64: - return email.base64mime.body_encode(s) - elif self.body_encoding is QP: - return email.quoprimime.body_encode(s) - else: - return s diff --git a/lib-python/modified-2.7/email/encoders.py b/lib-python/modified-2.7/email/encoders.py deleted file mode 100644 --- a/lib-python/modified-2.7/email/encoders.py +++ /dev/null @@ -1,82 +0,0 @@ -# Copyright (C) 2001-2006 Python Software Foundation -# Author: Barry Warsaw -# Contact: email-sig at python.org - -"""Encodings and related functions.""" - -__all__ = [ - 'encode_7or8bit', - 'encode_base64', - 'encode_noop', - 'encode_quopri', - ] - -import base64 - -from quopri import encodestring as _encodestring - - - -def _qencode(s): - enc = _encodestring(s, quotetabs=True) - # Must encode spaces, which quopri.encodestring() doesn't do - return enc.replace(' ', '=20') - - -def _bencode(s): - # We can't quite use base64.encodestring() since it tacks on a "courtesy - # newline". Blech! - if not s: - return s - hasnewline = (s[-1] == '\n') - value = base64.encodestring(s) - if not hasnewline and value[-1] == '\n': - return value[:-1] - return value - - - -def encode_base64(msg): - """Encode the message's payload in Base64. - - Also, add an appropriate Content-Transfer-Encoding header. - """ - orig = msg.get_payload() - encdata = _bencode(orig) - msg.set_payload(encdata) - msg['Content-Transfer-Encoding'] = 'base64' - - - -def encode_quopri(msg): - """Encode the message's payload in quoted-printable. - - Also, add an appropriate Content-Transfer-Encoding header. - """ - orig = msg.get_payload() - encdata = _qencode(orig) - msg.set_payload(encdata) - msg['Content-Transfer-Encoding'] = 'quoted-printable' - - - -def encode_7or8bit(msg): - """Set the Content-Transfer-Encoding header to 7bit or 8bit.""" - orig = msg.get_payload() - if orig is None: - # There's no payload. For backwards compatibility we use 7bit - msg['Content-Transfer-Encoding'] = '7bit' - return - # We play a trick to make this go fast. If encoding to ASCII succeeds, we - # know the data must be 7bit, otherwise treat it as 8bit. - try: - orig.encode('ascii') - except UnicodeError: - msg['Content-Transfer-Encoding'] = '8bit' - else: - msg['Content-Transfer-Encoding'] = '7bit' - - - -def encode_noop(msg): - """Do nothing.""" diff --git a/lib-python/modified-2.7/email/errors.py b/lib-python/modified-2.7/email/errors.py deleted file mode 100644 --- a/lib-python/modified-2.7/email/errors.py +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright (C) 2001-2006 Python Software Foundation -# Author: Barry Warsaw -# Contact: email-sig at python.org - -"""email package exception classes.""" - - - -class MessageError(Exception): - """Base class for errors in the email package.""" - - -class MessageParseError(MessageError): - """Base class for message parsing errors.""" - - -class HeaderParseError(MessageParseError): - """Error while parsing headers.""" - - -class BoundaryError(MessageParseError): - """Couldn't find terminating boundary.""" - - -class MultipartConversionError(MessageError, TypeError): - """Conversion to a multipart is prohibited.""" - - -class CharsetError(MessageError): - """An illegal charset was given.""" - - - -# These are parsing defects which the parser was able to work around. -class MessageDefect: - """Base class for a message defect.""" - - def __init__(self, line=None): - self.line = line - -class NoBoundaryInMultipartDefect(MessageDefect): - """A message claimed to be a multipart but had no boundary parameter.""" - -class StartBoundaryNotFoundDefect(MessageDefect): - """The claimed start boundary was never found.""" - -class FirstHeaderLineIsContinuationDefect(MessageDefect): - """A message had a continuation line as its first header line.""" - -class MisplacedEnvelopeHeaderDefect(MessageDefect): - """A 'Unix-from' header was found in the middle of a header block.""" - -class MalformedHeaderDefect(MessageDefect): - """Found a header that was missing a colon, or was otherwise malformed.""" - -class MultipartInvariantViolationDefect(MessageDefect): - """A message claimed to be a multipart but no subparts were found.""" diff --git a/lib-python/modified-2.7/email/feedparser.py b/lib-python/modified-2.7/email/feedparser.py deleted file mode 100644 --- a/lib-python/modified-2.7/email/feedparser.py +++ /dev/null @@ -1,484 +0,0 @@ -# Copyright (C) 2004-2006 Python Software Foundation -# Authors: Baxter, Wouters and Warsaw -# Contact: email-sig at python.org - -"""FeedParser - An email feed parser. - -The feed parser implements an interface for incrementally parsing an email -message, line by line. This has advantages for certain applications, such as -those reading email messages off a socket. - -FeedParser.feed() is the primary interface for pushing new data into the -parser. It returns when there's nothing more it can do with the available -data. When you have no more data to push into the parser, call .close(). -This completes the parsing and returns the root message object. - -The other advantage of this parser is that it will never throw a parsing -exception. Instead, when it finds something unexpected, it adds a 'defect' to -the current message. Defects are just instances that live on the message -object's .defects attribute. -""" - -__all__ = ['FeedParser'] - -import re - -from email import errors -from email import message - -NLCRE = re.compile('\r\n|\r|\n') -NLCRE_bol = re.compile('(\r\n|\r|\n)') -NLCRE_eol = re.compile('(\r\n|\r|\n)\Z') -NLCRE_crack = re.compile('(\r\n|\r|\n)') -# RFC 2822 $3.6.8 Optional fields. ftext is %d33-57 / %d59-126, Any character -# except controls, SP, and ":". -headerRE = re.compile(r'^(From |[\041-\071\073-\176]{1,}:|[\t ])') -EMPTYSTRING = '' -NL = '\n' - -NeedMoreData = object() - - - -class BufferedSubFile(object): - """A file-ish object that can have new data loaded into it. - - You can also push and pop line-matching predicates onto a stack. When the - current predicate matches the current line, a false EOF response - (i.e. empty string) is returned instead. This lets the parser adhere to a - simple abstraction -- it parses until EOF closes the current message. - """ - def __init__(self): - # The last partial line pushed into this object. - self._partial = '' - # The list of full, pushed lines, in reverse order - self._lines = [] - # The stack of false-EOF checking predicates. - self._eofstack = [] - # A flag indicating whether the file has been closed or not. - self._closed = False - - def push_eof_matcher(self, pred): - self._eofstack.append(pred) - - def pop_eof_matcher(self): - return self._eofstack.pop() - - def close(self): - # Don't forget any trailing partial line. - self._lines.append(self._partial) - self._partial = '' - self._closed = True - - def readline(self): - if not self._lines: - if self._closed: - return '' - return NeedMoreData - # Pop the line off the stack and see if it matches the current - # false-EOF predicate. - line = self._lines.pop() - # RFC 2046, section 5.1.2 requires us to recognize outer level - # boundaries at any level of inner nesting. Do this, but be sure it's - # in the order of most to least nested. - for ateof in self._eofstack[::-1]: - if ateof(line): - # We're at the false EOF. But push the last line back first. - self._lines.append(line) - return '' - return line - - def unreadline(self, line): - # Let the consumer push a line back into the buffer. - assert line is not NeedMoreData - self._lines.append(line) - - def push(self, data): - """Push some new data into this object.""" - # Handle any previous leftovers - data, self._partial = self._partial + data, '' - # Crack into lines, but preserve the newlines on the end of each - parts = NLCRE_crack.split(data) - # The *ahem* interesting behaviour of re.split when supplied grouping - # parentheses is that the last element of the resulting list is the - # data after the final RE. In the case of a NL/CR terminated string, - # this is the empty string. - self._partial = parts.pop() - #GAN 29Mar09 bugs 1555570, 1721862 Confusion at 8K boundary ending with \r: - # is there a \n to follow later? - if not self._partial and parts and parts[-1].endswith('\r'): - self._partial = parts.pop(-2)+parts.pop() - # parts is a list of strings, alternating between the line contents - # and the eol character(s). Gather up a list of lines after - # re-attaching the newlines. - lines = [] - for i in range(len(parts) // 2): - lines.append(parts[i*2] + parts[i*2+1]) - self.pushlines(lines) - - def pushlines(self, lines): - # Reverse and insert at the front of the lines. - self._lines[:0] = lines[::-1] - - def is_closed(self): - return self._closed - - def __iter__(self): - return self - - def next(self): - line = self.readline() - if line == '': - raise StopIteration - return line - - - -class FeedParser: - """A feed-style parser of email.""" - - def __init__(self, _factory=message.Message): - """_factory is called with no arguments to create a new message obj""" - self._factory = _factory - self._input = BufferedSubFile() - self._msgstack = [] - self._parse = self._parsegen().next - self._cur = None - self._last = None - self._headersonly = False - - # Non-public interface for supporting Parser's headersonly flag - def _set_headersonly(self): - self._headersonly = True - - def feed(self, data): - """Push more data into the parser.""" - self._input.push(data) - self._call_parse() - - def _call_parse(self): - try: - self._parse() - except StopIteration: - pass - - def close(self): - """Parse all remaining data and return the root message object.""" - self._input.close() - self._call_parse() - root = self._pop_message() - assert not self._msgstack - # Look for final set of defects - if root.get_content_maintype() == 'multipart' \ - and not root.is_multipart(): - root.defects.append(errors.MultipartInvariantViolationDefect()) - return root - - def _new_message(self): - msg = self._factory() - if self._cur and self._cur.get_content_type() == 'multipart/digest': - msg.set_default_type('message/rfc822') - if self._msgstack: - self._msgstack[-1].attach(msg) - self._msgstack.append(msg) - self._cur = msg - self._last = msg - - def _pop_message(self): - retval = self._msgstack.pop() - if self._msgstack: - self._cur = self._msgstack[-1] - else: - self._cur = None - return retval - - def _parsegen(self): - # Create a new message and start by parsing headers. - self._new_message() - headers = [] - # Collect the headers, searching for a line that doesn't match the RFC - # 2822 header or continuation pattern (including an empty line). - for line in self._input: - if line is NeedMoreData: - yield NeedMoreData - continue - if not headerRE.match(line): - # If we saw the RFC defined header/body separator - # (i.e. newline), just throw it away. Otherwise the line is - # part of the body so push it back. - if not NLCRE.match(line): - self._input.unreadline(line) - break - headers.append(line) - # Done with the headers, so parse them and figure out what we're - # supposed to see in the body of the message. - self._parse_headers(headers) - # Headers-only parsing is a backwards compatibility hack, which was - # necessary in the older parser, which could throw errors. All - # remaining lines in the input are thrown into the message body. - if self._headersonly: - lines = [] - while True: - line = self._input.readline() - if line is NeedMoreData: - yield NeedMoreData - continue - if line == '': - break - lines.append(line) - self._cur.set_payload(EMPTYSTRING.join(lines)) - return - if self._cur.get_content_type() == 'message/delivery-status': - # message/delivery-status contains blocks of headers separated by - # a blank line. We'll represent each header block as a separate - # nested message object, but the processing is a bit different - # than standard message/* types because there is no body for the - # nested messages. A blank line separates the subparts. - while True: - self._input.push_eof_matcher(NLCRE.match) - for retval in self._parsegen(): - if retval is NeedMoreData: - yield NeedMoreData - continue - break - msg = self._pop_message() - # We need to pop the EOF matcher in order to tell if we're at - # the end of the current file, not the end of the last block - # of message headers. - self._input.pop_eof_matcher() - # The input stream must be sitting at the newline or at the - # EOF. We want to see if we're at the end of this subpart, so - # first consume the blank line, then test the next line to see - # if we're at this subpart's EOF. - while True: - line = self._input.readline() - if line is NeedMoreData: - yield NeedMoreData - continue - break - while True: - line = self._input.readline() - if line is NeedMoreData: - yield NeedMoreData - continue - break - if line == '': - break - # Not at EOF so this is a line we're going to need. - self._input.unreadline(line) - return - if self._cur.get_content_maintype() == 'message': - # The message claims to be a message/* type, then what follows is - # another RFC 2822 message. - for retval in self._parsegen(): - if retval is NeedMoreData: - yield NeedMoreData - continue - break - self._pop_message() - return - if self._cur.get_content_maintype() == 'multipart': - boundary = self._cur.get_boundary() - if boundary is None: - # The message /claims/ to be a multipart but it has not - # defined a boundary. That's a problem which we'll handle by - # reading everything until the EOF and marking the message as - # defective. - self._cur.defects.append(errors.NoBoundaryInMultipartDefect()) - lines = [] - for line in self._input: - if line is NeedMoreData: - yield NeedMoreData - continue - lines.append(line) - self._cur.set_payload(EMPTYSTRING.join(lines)) - return - # Create a line match predicate which matches the inter-part - # boundary as well as the end-of-multipart boundary. Don't push - # this onto the input stream until we've scanned past the - # preamble. - separator = '--' + boundary - boundaryre = re.compile( - '(?P<sep>' + re.escape(separator) + - r')(?P<end>--)?(?P<ws>[ \t]*)(?P<linesep>\r\n|\r|\n)?$') - capturing_preamble = True - preamble = [] - linesep = False - while True: - line = self._input.readline() - if line is NeedMoreData: - yield NeedMoreData - continue - if line == '': - break - mo = boundaryre.match(line) - if mo: - # If we're looking at the end boundary, we're done with - # this multipart. If there was a newline at the end of - # the closing boundary, then we need to initialize the - # epilogue with the empty string (see below). - if mo.group('end'): - linesep = mo.group('linesep') - break - # We saw an inter-part boundary. Were we in the preamble? - if capturing_preamble: - if preamble: - # According to RFC 2046, the last newline belongs - # to the boundary. - lastline = preamble[-1] - eolmo = NLCRE_eol.search(lastline) - if eolmo: - preamble[-1] = lastline[:-len(eolmo.group(0))] - self._cur.preamble = EMPTYSTRING.join(preamble) - capturing_preamble = False - self._input.unreadline(line) - continue - # We saw a boundary separating two parts. Consume any - # multiple boundary lines that may be following. Our - # interpretation of RFC 2046 BNF grammar does not produce - # body parts within such double boundaries. - while True: - line = self._input.readline() - if line is NeedMoreData: - yield NeedMoreData - continue - mo = boundaryre.match(line) - if not mo: - self._input.unreadline(line) - break - # Recurse to parse this subpart; the input stream points - # at the subpart's first line. - self._input.push_eof_matcher(boundaryre.match) - for retval in self._parsegen(): - if retval is NeedMoreData: - yield NeedMoreData - continue - break - # Because of RFC 2046, the newline preceding the boundary - # separator actually belongs to the boundary, not the - # previous subpart's payload (or epilogue if the previous - # part is a multipart). - if self._last.get_content_maintype() == 'multipart': - epilogue = self._last.epilogue - if epilogue == '': - self._last.epilogue = None - elif epilogue is not None: - mo = NLCRE_eol.search(epilogue) - if mo: - end = len(mo.group(0)) - self._last.epilogue = epilogue[:-end] - else: - payload = self._last.get_payload() - if isinstance(payload, basestring): - mo = NLCRE_eol.search(payload) - if mo: - payload = payload[:-len(mo.group(0))] - self._last.set_payload(payload) - self._input.pop_eof_matcher() - self._pop_message() - # Set the multipart up for newline cleansing, which will - # happen if we're in a nested multipart. - self._last = self._cur - else: - # I think we must be in the preamble - assert capturing_preamble - preamble.append(line) - # We've seen either the EOF or the end boundary. If we're still - # capturing the preamble, we never saw the start boundary. Note - # that as a defect and store the captured text as the payload. - # Everything from here to the EOF is epilogue. - if capturing_preamble: - self._cur.defects.append(errors.StartBoundaryNotFoundDefect()) - self._cur.set_payload(EMPTYSTRING.join(preamble)) - epilogue = [] - for line in self._input: - if line is NeedMoreData: - yield NeedMoreData - continue - self._cur.epilogue = EMPTYSTRING.join(epilogue) - return - # If the end boundary ended in a newline, we'll need to make sure - # the epilogue isn't None - if linesep: - epilogue = [''] - else: - epilogue = [] - for line in self._input: - if line is NeedMoreData: - yield NeedMoreData - continue - epilogue.append(line) - # Any CRLF at the front of the epilogue is not technically part of - # the epilogue. Also, watch out for an empty string epilogue, - # which means a single newline. - if epilogue: - firstline = epilogue[0] - bolmo = NLCRE_bol.match(firstline) - if bolmo: - epilogue[0] = firstline[len(bolmo.group(0)):] - self._cur.epilogue = EMPTYSTRING.join(epilogue) - return - # Otherwise, it's some non-multipart type, so the entire rest of the - # file contents becomes the payload. - lines = [] - for line in self._input: - if line is NeedMoreData: - yield NeedMoreData - continue - lines.append(line) - self._cur.set_payload(EMPTYSTRING.join(lines)) - - def _parse_headers(self, lines): - # Passed a list of lines that make up the headers for the current msg - lastheader = '' - lastvalue = [] - for lineno, line in enumerate(lines): - # Check for continuation - if line[0] in ' \t': - if not lastheader: - # The first line of the headers was a continuation. This - # is illegal, so let's note the defect, store the illegal - # line, and ignore it for purposes of headers. - defect = errors.FirstHeaderLineIsContinuationDefect(line) - self._cur.defects.append(defect) - continue - lastvalue.append(line) - continue - if lastheader: - # XXX reconsider the joining of folded lines - lhdr = EMPTYSTRING.join(lastvalue)[:-1].rstrip('\r\n') - self._cur[lastheader] = lhdr - lastheader, lastvalue = '', [] - # Check for envelope header, i.e. unix-from - if line.startswith('From '): - if lineno == 0: - # Strip off the trailing newline - mo = NLCRE_eol.search(line) - if mo: - line = line[:-len(mo.group(0))] - self._cur.set_unixfrom(line) - continue - elif lineno == len(lines) - 1: - # Something looking like a unix-from at the end - it's - # probably the first line of the body, so push back the - # line and stop. - self._input.unreadline(line) - return - else: - # Weirdly placed unix-from line. Note this as a defect - # and ignore it. - defect = errors.MisplacedEnvelopeHeaderDefect(line) - self._cur.defects.append(defect) - continue - # Split the line on the colon separating field name from value. - i = line.find(':') - if i < 0: - defect = errors.MalformedHeaderDefect(line) - self._cur.defects.append(defect) - continue - lastheader = line[:i] - lastvalue = [line[i+1:].lstrip()] - # Done with all the lines, so handle the last header. - if lastheader: - # XXX reconsider the joining of folded lines - self._cur[lastheader] = EMPTYSTRING.join(lastvalue).rstrip('\r\n') diff --git a/lib-python/modified-2.7/email/generator.py b/lib-python/modified-2.7/email/generator.py deleted file mode 100644 --- a/lib-python/modified-2.7/email/generator.py +++ /dev/null @@ -1,364 +0,0 @@ -# Copyright (C) 2001-2010 Python Software Foundation -# Contact: email-sig at python.org - -"""Classes to generate plain text from a message object tree.""" - -__all__ = ['Generator', 'DecodedGenerator'] - -import re -import sys -import time -import random -import warnings - -from cStringIO import StringIO -from email.header import Header - -UNDERSCORE = '_' -NL = '\n' - -fcre = re.compile(r'^From ', re.MULTILINE) - -def _is8bitstring(s): - if isinstance(s, str): - try: - unicode(s, 'us-ascii') - except UnicodeError: - return True - return False - - - -class Generator: - """Generates output from a Message object tree. - - This basic generator writes the message to the given file object as plain - text. - """ - # - # Public interface - # - - def __init__(self, outfp, mangle_from_=True, maxheaderlen=78): - """Create the generator for message flattening. - - outfp is the output file-like object for writing the message to. It - must have a write() method. - - Optional mangle_from_ is a flag that, when True (the default), escapes - From_ lines in the body of the message by putting a `>' in front of - them. - - Optional maxheaderlen specifies the longest length for a non-continued - header. When a header line is longer (in characters, with tabs - expanded to 8 spaces) than maxheaderlen, the header will split as - defined in the Header class. Set maxheaderlen to zero to disable - header wrapping. The default is 78, as recommended (but not required) - by RFC 2822. - """ - self._fp = outfp - self._mangle_from_ = mangle_from_ - self._maxheaderlen = maxheaderlen - - def write(self, s): - # Just delegate to the file object - self._fp.write(s) - - def flatten(self, msg, unixfrom=False): - """Print the message object tree rooted at msg to the output file - specified when the Generator instance was created. - - unixfrom is a flag that forces the printing of a Unix From_ delimiter - before the first object in the message tree. If the original message - has no From_ delimiter, a `standard' one is crafted. By default, this - is False to inhibit the printing of any From_ delimiter. - - Note that for subobjects, no From_ line is printed. - """ - if unixfrom: - ufrom = msg.get_unixfrom() - if not ufrom: - ufrom = 'From nobody ' + time.ctime(time.time()) - print >> self._fp, ufrom - self._write(msg) - - def clone(self, fp): - """Clone this generator with the exact same options.""" - return self.__class__(fp, self._mangle_from_, self._maxheaderlen) - - # - # Protected interface - undocumented ;/ - # - - def _write(self, msg): - # We can't write the headers yet because of the following scenario: - # say a multipart message includes the boundary string somewhere in - # its body. We'd have to calculate the new boundary /before/ we write - # the headers so that we can write the correct Content-Type: - # parameter. - # - # The way we do this, so as to make the _handle_*() methods simpler, - # is to cache any subpart writes into a StringIO. The we write the - # headers and the StringIO contents. That way, subpart handlers can - # Do The Right Thing, and can still modify the Content-Type: header if - # necessary. - oldfp = self._fp - try: - self._fp = sfp = StringIO() - self._dispatch(msg) - finally: - self._fp = oldfp - # Write the headers. First we see if the message object wants to - # handle that itself. If not, we'll do it generically. - meth = getattr(msg, '_write_headers', None) - if meth is None: - self._write_headers(msg) - else: - meth(self) - self._fp.write(sfp.getvalue()) - - def _dispatch(self, msg): - # Get the Content-Type: for the message, then try to dispatch to - # self._handle_<maintype>_<subtype>(). If there's no handler for the - # full MIME type, then dispatch to self._handle_<maintype>(). If - # that's missing too, then dispatch to self._writeBody(). - main = msg.get_content_maintype() - sub = msg.get_content_subtype() - specific = UNDERSCORE.join((main, sub)).replace('-', '_') - meth = getattr(self, '_handle_' + specific, None) - if meth is None: - generic = main.replace('-', '_') - meth = getattr(self, '_handle_' + generic, None) - if meth is None: - meth = self._writeBody - meth(msg) - - # - # Default handlers - # - - def _write_headers(self, msg): - for h, v in msg.items(): - print >> self._fp, '%s:' % h, - if self._maxheaderlen == 0: - # Explicit no-wrapping - print >> self._fp, v - elif isinstance(v, Header): - # Header instances know what to do - print >> self._fp, v.encode() - elif _is8bitstring(v): - # If we have raw 8bit data in a byte string, we have no idea - # what the encoding is. There is no safe way to split this - # string. If it's ascii-subset, then we could do a normal - # ascii split, but if it's multibyte then we could break the - # string. There's no way to know so the least harm seems to - # be to not split the string and risk it being too long. - print >> self._fp, v - else: - # Header's got lots of smarts, so use it. Note that this is - # fundamentally broken though because we lose idempotency when - # the header string is continued with tabs. It will now be - # continued with spaces. This was reversedly broken before we - # fixed bug 1974. Either way, we lose. - print >> self._fp, Header( - v, maxlinelen=self._maxheaderlen, header_name=h).encode() - # A blank line always separates headers from body - print >> self._fp - - # - # Handlers for writing types and subtypes - # - - def _handle_text(self, msg): - payload = msg.get_payload() - if payload is None: - return - if not isinstance(payload, basestring): - raise TypeError('string payload expected: %s' % type(payload)) - if self._mangle_from_: - payload = fcre.sub('>From ', payload) - self._fp.write(payload) - - # Default body handler - _writeBody = _handle_text - - def _handle_multipart(self, msg): - # The trick here is to write out each part separately, merge them all - # together, and then make sure that the boundary we've chosen isn't - # present in the payload. - msgtexts = [] - subparts = msg.get_payload() - if subparts is None: - subparts = [] - elif isinstance(subparts, basestring): - # e.g. a non-strict parse of a message with no starting boundary. - self._fp.write(subparts) - return - elif not isinstance(subparts, list): - # Scalar payload - subparts = [subparts] - for part in subparts: - s = StringIO() - g = self.clone(s) - g.flatten(part, unixfrom=False) - msgtexts.append(s.getvalue()) - # BAW: What about boundaries that are wrapped in double-quotes? - boundary = msg.get_boundary() - if not boundary: - # Create a boundary that doesn't appear in any of the - # message texts. - alltext = NL.join(msgtexts) - boundary = _make_boundary(alltext) - msg.set_boundary(boundary) - # If there's a preamble, write it out, with a trailing CRLF - if msg.preamble is not None: - print >> self._fp, msg.preamble - # dash-boundary transport-padding CRLF - print >> self._fp, '--' + boundary - # body-part - if msgtexts: - self._fp.write(msgtexts.pop(0)) - # *encapsulation - # --> delimiter transport-padding - # --> CRLF body-part - for body_part in msgtexts: - # delimiter transport-padding CRLF - print >> self._fp, '\n--' + boundary - # body-part - self._fp.write(body_part) - # close-delimiter transport-padding - self._fp.write('\n--' + boundary + '--') - if msg.epilogue is not None: - print >> self._fp - self._fp.write(msg.epilogue) - - def _handle_multipart_signed(self, msg): - # The contents of signed parts has to stay unmodified in order to keep - # the signature intact per RFC1847 2.1, so we disable header wrapping. - # RDM: This isn't enough to completely preserve the part, but it helps. - old_maxheaderlen = self._maxheaderlen - try: - self._maxheaderlen = 0 - self._handle_multipart(msg) - finally: - self._maxheaderlen = old_maxheaderlen - - def _handle_message_delivery_status(self, msg): - # We can't just write the headers directly to self's file object - # because this will leave an extra newline between the last header - # block and the boundary. Sigh. - blocks = [] - for part in msg.get_payload(): - s = StringIO() - g = self.clone(s) - g.flatten(part, unixfrom=False) - text = s.getvalue() - lines = text.split('\n') - # Strip off the unnecessary trailing empty line - if lines and lines[-1] == '': - blocks.append(NL.join(lines[:-1])) - else: - blocks.append(text) - # Now join all the blocks with an empty line. This has the lovely - # effect of separating each block with an empty line, but not adding - # an extra one after the last one. - self._fp.write(NL.join(blocks)) - - def _handle_message(self, msg): - s = StringIO() - g = self.clone(s) - # The payload of a message/rfc822 part should be a multipart sequence - # of length 1. The zeroth element of the list should be the Message - # object for the subpart. Extract that object, stringify it, and - # write it out. - # Except, it turns out, when it's a string instead, which happens when - # and only when HeaderParser is used on a message of mime type - # message/rfc822. Such messages are generated by, for example, - # Groupwise when forwarding unadorned messages. (Issue 7970.) So - # in that case we just emit the string body. - payload = msg.get_payload() - if isinstance(payload, list): - g.flatten(msg.get_payload(0), unixfrom=False) - payload = s.getvalue() - self._fp.write(payload) - - - -_FMT = '[Non-text (%(type)s) part of message omitted, filename %(filename)s]' - -class DecodedGenerator(Generator): - """Generates a text representation of a message. - - Like the Generator base class, except that non-text parts are substituted - with a format string representing the part. - """ - def __init__(self, outfp, mangle_from_=True, maxheaderlen=78, fmt=None): - """Like Generator.__init__() except that an additional optional - argument is allowed. - - Walks through all subparts of a message. If the subpart is of main - type `text', then it prints the decoded payload of the subpart. - - Otherwise, fmt is a format string that is used instead of the message - payload. fmt is expanded with the following keywords (in - %(keyword)s format): - - type : Full MIME type of the non-text part - maintype : Main MIME type of the non-text part - subtype : Sub-MIME type of the non-text part - filename : Filename of the non-text part - description: Description associated with the non-text part - encoding : Content transfer encoding of the non-text part - - The default value for fmt is None, meaning - - [Non-text (%(type)s) part of message omitted, filename %(filename)s] - """ - Generator.__init__(self, outfp, mangle_from_, maxheaderlen) - if fmt is None: - self._fmt = _FMT - else: - self._fmt = fmt - - def _dispatch(self, msg): - for part in msg.walk(): - maintype = part.get_content_maintype() - if maintype == 'text': - print >> self, part.get_payload(decode=True) - elif maintype == 'multipart': - # Just skip this - pass - else: - print >> self, self._fmt % { - 'type' : part.get_content_type(), - 'maintype' : part.get_content_maintype(), - 'subtype' : part.get_content_subtype(), - 'filename' : part.get_filename('[no filename]'), - 'description': part.get('Content-Description', - '[no description]'), - 'encoding' : part.get('Content-Transfer-Encoding', - '[no encoding]'), - } - - - -# Helper -_width = len(repr(sys.maxint-1)) -_fmt = '%%0%dd' % _width - -def _make_boundary(text=None): - # Craft a random boundary. If text is given, ensure that the chosen - # boundary doesn't appear in the text. - token = random.randrange(sys.maxint) - boundary = ('=' * 15) + (_fmt % token) + '==' - if text is None: - return boundary - b = boundary - counter = 0 - while True: - cre = re.compile('^--' + re.escape(b) + '(--)?$', re.MULTILINE) - if not cre.search(text): - break - b = boundary + '.' + str(counter) - counter += 1 - return b diff --git a/lib-python/modified-2.7/email/header.py b/lib-python/modified-2.7/email/header.py deleted file mode 100644 --- a/lib-python/modified-2.7/email/header.py +++ /dev/null @@ -1,514 +0,0 @@ -# Copyright (C) 2002-2006 Python Software Foundation -# Author: Ben Gertzfield, Barry Warsaw -# Contact: email-sig at python.org - -"""Header encoding and decoding functionality.""" - -__all__ = [ - 'Header', - 'decode_header', - 'make_header', - ] - -import re -import binascii - -import email.quoprimime -import email.base64mime - -from email.errors import HeaderParseError -from email.charset import Charset - -NL = '\n' -SPACE = ' ' -USPACE = u' ' -SPACE8 = ' ' * 8 -UEMPTYSTRING = u'' - -MAXLINELEN = 76 - -USASCII = Charset('us-ascii') -UTF8 = Charset('utf-8') - -# Match encoded-word strings in the form =?charset?q?Hello_World?= -ecre = re.compile(r''' - =\? # literal =? - (?P<charset>[^?]*?) # non-greedy up to the next ? is the charset - \? # literal ? - (?P<encoding>[qb]) # either a "q" or a "b", case insensitive - \? # literal ? - (?P<encoded>.*?) # non-greedy up to the next ?= is the encoded string - \?= # literal ?= - (?=[ \t]|$) # whitespace or the end of the string - ''', re.VERBOSE | re.IGNORECASE | re.MULTILINE) - -# Field name regexp, including trailing colon, but not separating whitespace, -# according to RFC 2822. Character range is from tilde to exclamation mark. -# For use with .match() -fcre = re.compile(r'[\041-\176]+:$') - -# Find a header embedded in a putative header value. Used to check for -# header injection attack. -_embeded_header = re.compile(r'\n[^ \t]+:') - - - -# Helpers -_max_append = email.quoprimime._max_append - - - -def decode_header(header): - """Decode a message header value without converting charset. - - Returns a list of (decoded_string, charset) pairs containing each of the - decoded parts of the header. Charset is None for non-encoded parts of the - header, otherwise a lower-case string containing the name of the character - set specified in the encoded string. - - An email.errors.HeaderParseError may be raised when certain decoding error - occurs (e.g. a base64 decoding exception). - """ - # If no encoding, just return the header - header = str(header) - if not ecre.search(header): - return [(header, None)] - decoded = [] - dec = '' - for line in header.splitlines(): - # This line might not have an encoding in it - if not ecre.search(line): - decoded.append((line, None)) - continue - parts = ecre.split(line) - while parts: - unenc = parts.pop(0).strip() - if unenc: - # Should we continue a long line? - if decoded and decoded[-1][1] is None: - decoded[-1] = (decoded[-1][0] + SPACE + unenc, None) - else: - decoded.append((unenc, None)) - if parts: - charset, encoding = [s.lower() for s in parts[0:2]] - encoded = parts[2] - dec = None - if encoding == 'q': - dec = email.quoprimime.header_decode(encoded) - elif encoding == 'b': - paderr = len(encoded) % 4 # Postel's law: add missing padding - if paderr: - encoded += '==='[:4 - paderr] - try: - dec = email.base64mime.decode(encoded) - except binascii.Error: - # Turn this into a higher level exception. BAW: Right - # now we throw the lower level exception away but - # when/if we get exception chaining, we'll preserve it. - raise HeaderParseError - if dec is None: - dec = encoded - - if decoded and decoded[-1][1] == charset: - decoded[-1] = (decoded[-1][0] + dec, decoded[-1][1]) - else: - decoded.append((dec, charset)) - del parts[0:3] - return decoded - - - -def make_header(decoded_seq, maxlinelen=None, header_name=None, - continuation_ws=' '): - """Create a Header from a sequence of pairs as returned by decode_header() - - decode_header() takes a header value string and returns a sequence of - pairs of the format (decoded_string, charset) where charset is the string - name of the character set. - - This function takes one of those sequence of pairs and returns a Header - instance. Optional maxlinelen, header_name, and continuation_ws are as in - the Header constructor. - """ - h = Header(maxlinelen=maxlinelen, header_name=header_name, - continuation_ws=continuation_ws) - for s, charset in decoded_seq: - # None means us-ascii but we can simply pass it on to h.append() - if charset is not None and not isinstance(charset, Charset): - charset = Charset(charset) - h.append(s, charset) - return h - - - -class Header: - def __init__(self, s=None, charset=None, - maxlinelen=None, header_name=None, - continuation_ws=' ', errors='strict'): - """Create a MIME-compliant header that can contain many character sets. - - Optional s is the initial header value. If None, the initial header - value is not set. You can later append to the header with .append() - method calls. s may be a byte string or a Unicode string, but see the - .append() documentation for semantics. - - Optional charset serves two purposes: it has the same meaning as the - charset argument to the .append() method. It also sets the default - character set for all subsequent .append() calls that omit the charset - argument. If charset is not provided in the constructor, the us-ascii - charset is used both as s's initial charset and as the default for - subsequent .append() calls. - - The maximum line length can be specified explicit via maxlinelen. For - splitting the first line to a shorter value (to account for the field - header which isn't included in s, e.g. `Subject') pass in the name of - the field in header_name. The default maxlinelen is 76. - - continuation_ws must be RFC 2822 compliant folding whitespace (usually - either a space or a hard tab) which will be prepended to continuation - lines. - - errors is passed through to the .append() call. - """ - if charset is None: - charset = USASCII - if not isinstance(charset, Charset): - charset = Charset(charset) - self._charset = charset - self._continuation_ws = continuation_ws - cws_expanded_len = len(continuation_ws.replace('\t', SPACE8)) - # BAW: I believe `chunks' and `maxlinelen' should be non-public. - self._chunks = [] - if s is not None: - self.append(s, charset, errors) - if maxlinelen is None: - maxlinelen = MAXLINELEN - if header_name is None: - # We don't know anything about the field header so the first line - # is the same length as subsequent lines. - self._firstlinelen = maxlinelen - else: - # The first line should be shorter to take into account the field - # header. Also subtract off 2 extra for the colon and space. - self._firstlinelen = maxlinelen - len(header_name) - 2 - # Second and subsequent lines should subtract off the length in - # columns of the continuation whitespace prefix. - self._maxlinelen = maxlinelen - cws_expanded_len - - def __str__(self): - """A synonym for self.encode().""" - return self.encode() - - def __unicode__(self): - """Helper for the built-in unicode function.""" - uchunks = [] - lastcs = None - for s, charset in self._chunks: - # We must preserve spaces between encoded and non-encoded word - # boundaries, which means for us we need to add a space when we go - # from a charset to None/us-ascii, or from None/us-ascii to a - # charset. Only do this for the second and subsequent chunks. - nextcs = charset - if uchunks: - if lastcs not in (None, 'us-ascii'): - if nextcs in (None, 'us-ascii'): - uchunks.append(USPACE) - nextcs = None - elif nextcs not in (None, 'us-ascii'): - uchunks.append(USPACE) - lastcs = nextcs - uchunks.append(unicode(s, str(charset))) - return UEMPTYSTRING.join(uchunks) - - # Rich comparison operators for equality only. BAW: does it make sense to - # have or explicitly disable <, <=, >, >= operators? - def __eq__(self, other): - # other may be a Header or a string. Both are fine so coerce - # ourselves to a string, swap the args and do another comparison. - return other == self.encode() - - def __ne__(self, other): - return not self == other - - def append(self, s, charset=None, errors='strict'): - """Append a string to the MIME header. - - Optional charset, if given, should be a Charset instance or the name - of a character set (which will be converted to a Charset instance). A - value of None (the default) means that the charset given in the - constructor is used. - - s may be a byte string or a Unicode string. If it is a byte string - (i.e. isinstance(s, str) is true), then charset is the encoding of - that byte string, and a UnicodeError will be raised if the string - cannot be decoded with that charset. If s is a Unicode string, then - charset is a hint specifying the character set of the characters in - the string. In this case, when producing an RFC 2822 compliant header - using RFC 2047 rules, the Unicode string will be encoded using the - following charsets in order: us-ascii, the charset hint, utf-8. The - first character set not to provoke a UnicodeError is used. - - Optional `errors' is passed as the third argument to any unicode() or - ustr.encode() call. - """ - if charset is None: - charset = self._charset - elif not isinstance(charset, Charset): - charset = Charset(charset) - # If the charset is our faux 8bit charset, leave the string unchanged - if charset != '8bit': - # We need to test that the string can be converted to unicode and - # back to a byte string, given the input and output codecs of the - # charset. - if isinstance(s, str): - # Possibly raise UnicodeError if the byte string can't be - # converted to a unicode with the input codec of the charset. - incodec = charset.input_codec or 'us-ascii' - ustr = unicode(s, incodec, errors) - # Now make sure that the unicode could be converted back to a - # byte string with the output codec, which may be different - # than the iput coded. Still, use the original byte string. - outcodec = charset.output_codec or 'us-ascii' - ustr.encode(outcodec, errors) - elif isinstance(s, unicode): - # Now we have to be sure the unicode string can be converted - # to a byte string with a reasonable output codec. We want to - # use the byte string in the chunk. - for charset in USASCII, charset, UTF8: - try: - outcodec = charset.output_codec or 'us-ascii' - s = s.encode(outcodec, errors) - break - except UnicodeError: - pass - else: - assert False, 'utf-8 conversion failed' - self._chunks.append((s, charset)) - - def _split(self, s, charset, maxlinelen, splitchars): - # Split up a header safely for use with encode_chunks. - splittable = charset.to_splittable(s) - encoded = charset.from_splittable(splittable, True) - elen = charset.encoded_header_len(encoded) - # If the line's encoded length first, just return it - if elen <= maxlinelen: - return [(encoded, charset)] - # If we have undetermined raw 8bit characters sitting in a byte - # string, we really don't know what the right thing to do is. We - # can't really split it because it might be multibyte data which we - # could break if we split it between pairs. The least harm seems to - # be to not split the header at all, but that means they could go out - # longer than maxlinelen. - if charset == '8bit': - return [(s, charset)] - # BAW: I'm not sure what the right test here is. What we're trying to - # do is be faithful to RFC 2822's recommendation that ($2.2.3): - # - # "Note: Though structured field bodies are defined in such a way that - # folding can take place between many of the lexical tokens (and even - # within some of the lexical tokens), folding SHOULD be limited to - # placing the CRLF at higher-level syntactic breaks." - # - # For now, I can only imagine doing this when the charset is us-ascii, - # although it's possible that other charsets may also benefit from the - # higher-level syntactic breaks. - elif charset == 'us-ascii': - return self._split_ascii(s, charset, maxlinelen, splitchars) - # BAW: should we use encoded? - elif elen == len(s): - # We can split on _maxlinelen boundaries because we know that the - # encoding won't change the size of the string - splitpnt = maxlinelen - first = charset.from_splittable(splittable[:splitpnt], False) - last = charset.from_splittable(splittable[splitpnt:], False) - else: - # Binary search for split point - first, last = _binsplit(splittable, charset, maxlinelen) - # first is of the proper length so just wrap it in the appropriate - # chrome. last must be recursively split. - fsplittable = charset.to_splittable(first) - fencoded = charset.from_splittable(fsplittable, True) - chunk = [(fencoded, charset)] - return chunk + self._split(last, charset, self._maxlinelen, splitchars) - - def _split_ascii(self, s, charset, firstlen, splitchars): - chunks = _split_ascii(s, firstlen, self._maxlinelen, - self._continuation_ws, splitchars) - return zip(chunks, [charset]*len(chunks)) - - def _encode_chunks(self, newchunks, maxlinelen): - # MIME-encode a header with many different charsets and/or encodings. - # - # Given a list of pairs (string, charset), return a MIME-encoded - # string suitable for use in a header field. Each pair may have - # different charsets and/or encodings, and the resulting header will - # accurately reflect each setting. - # - # Each encoding can be email.utils.QP (quoted-printable, for - # ASCII-like character sets like iso-8859-1), email.utils.BASE64 - # (Base64, for non-ASCII like character sets like KOI8-R and - # iso-2022-jp), or None (no encoding). - # - # Each pair will be represented on a separate line; the resulting - # string will be in the format: - # - # =?charset1?q?Mar=EDa_Gonz=E1lez_Alonso?=\n - # =?charset2?b?SvxyZ2VuIEL2aW5n?=" - chunks = [] - for header, charset in newchunks: - if not header: - continue - if charset is None or charset.header_encoding is None: - s = header - else: - s = charset.header_encode(header) - # Don't add more folding whitespace than necessary - if chunks and chunks[-1].endswith(' '): - extra = '' - else: - extra = ' ' - _max_append(chunks, s, maxlinelen, extra) - joiner = NL + self._continuation_ws - return joiner.join(chunks) - - def encode(self, splitchars=';, '): - """Encode a message header into an RFC-compliant format. - - There are many issues involved in converting a given string for use in - an email header. Only certain character sets are readable in most - email clients, and as header strings can only contain a subset of - 7-bit ASCII, care must be taken to properly convert and encode (with - Base64 or quoted-printable) header strings. In addition, there is a - 75-character length limit on any given encoded header field, so - line-wrapping must be performed, even with double-byte character sets. - - This method will do its best to convert the string to the correct - character set used in email, and encode and line wrap it safely with - the appropriate scheme for that character set. - - If the given charset is not known or an error occurs during - conversion, this function will return the header untouched. - - Optional splitchars is a string containing characters to split long - ASCII lines on, in rough support of RFC 2822's `highest level - syntactic breaks'. This doesn't affect RFC 2047 encoded lines. - """ - newchunks = [] - maxlinelen = self._firstlinelen - lastlen = 0 - for s, charset in self._chunks: - # The first bit of the next chunk should be just long enough to - # fill the next line. Don't forget the space separating the - # encoded words. - targetlen = maxlinelen - lastlen - 1 - if targetlen < charset.encoded_header_len(''): - # Stick it on the next line - targetlen = maxlinelen - newchunks += self._split(s, charset, targetlen, splitchars) - lastchunk, lastcharset = newchunks[-1] - lastlen = lastcharset.encoded_header_len(lastchunk) - value = self._encode_chunks(newchunks, maxlinelen) - if _embeded_header.search(value): - raise HeaderParseError("header value appears to contain " - "an embedded header: {!r}".format(value)) - return value - - - -def _split_ascii(s, firstlen, restlen, continuation_ws, splitchars): - lines = [] - maxlen = firstlen - for line in s.splitlines(): - # Ignore any leading whitespace (i.e. continuation whitespace) already - # on the line, since we'll be adding our own. - line = line.lstrip() - if len(line) < maxlen: - lines.append(line) - maxlen = restlen - continue - # Attempt to split the line at the highest-level syntactic break - # possible. Note that we don't have a lot of smarts about field - # syntax; we just try to break on semi-colons, then commas, then - # whitespace. - for ch in splitchars: - if ch in line: - break - else: - # There's nothing useful to split the line on, not even spaces, so - # just append this line unchanged - lines.append(line) - maxlen = restlen - continue - # Now split the line on the character plus trailing whitespace - cre = re.compile(r'%s\s*' % ch) - if ch in ';,': - eol = ch - else: - eol = '' - joiner = eol + ' ' - joinlen = len(joiner) - wslen = len(continuation_ws.replace('\t', SPACE8)) - this = [] - linelen = 0 - for part in cre.split(line): - curlen = linelen + max(0, len(this)-1) * joinlen - partlen = len(part) - onfirstline = not lines - # We don't want to split after the field name, if we're on the - # first line and the field name is present in the header string. - if ch == ' ' and onfirstline and \ - len(this) == 1 and fcre.match(this[0]): - this.append(part) - linelen += partlen - elif curlen + partlen > maxlen: - if this: - lines.append(joiner.join(this) + eol) - # If this part is longer than maxlen and we aren't already - # splitting on whitespace, try to recursively split this line - # on whitespace. - if partlen > maxlen and ch != ' ': - subl = _split_ascii(part, maxlen, restlen, - continuation_ws, ' ') - lines.extend(subl[:-1]) - this = [subl[-1]] - else: - this = [part] - linelen = wslen + len(this[-1]) - maxlen = restlen - else: - this.append(part) - linelen += partlen - # Put any left over parts on a line by themselves - if this: - lines.append(joiner.join(this)) - return lines - - - -def _binsplit(splittable, charset, maxlinelen): - i = 0 - j = len(splittable) - while i < j: - # Invariants: - # 1. splittable[:k] fits for all k <= i (note that we *assume*, - # at the start, that splittable[:0] fits). - # 2. splittable[:k] does not fit for any k > j (at the start, - # this means we shouldn't look at any k > len(splittable)). - # 3. We don't know about splittable[:k] for k in i+1..j. - # 4. We want to set i to the largest k that fits, with i <= k <= j. - # - m = (i+j+1) >> 1 # ceiling((i+j)/2); i < m <= j - chunk = charset.from_splittable(splittable[:m], True) - chunklen = charset.encoded_header_len(chunk) - if chunklen <= maxlinelen: - # m is acceptable, so is a new lower bound. - i = m - else: - # m is not acceptable, so final i must be < m. - j = m - 1 - # i == j. Invariant #1 implies that splittable[:i] fits, and - # invariant #2 implies that splittable[:i+1] does not fit, so i - # is what we're looking for. - first = charset.from_splittable(splittable[:i], False) - last = charset.from_splittable(splittable[i:], False) - return first, last diff --git a/lib-python/modified-2.7/email/iterators.py b/lib-python/modified-2.7/email/iterators.py deleted file mode 100644 --- a/lib-python/modified-2.7/email/iterators.py +++ /dev/null @@ -1,73 +0,0 @@ -# Copyright (C) 2001-2006 Python Software Foundation -# Author: Barry Warsaw -# Contact: email-sig at python.org - -"""Various types of useful iterators and generators.""" - -__all__ = [ - 'body_line_iterator', - 'typed_subpart_iterator', - 'walk', - # Do not include _structure() since it's part of the debugging API. - ] - -import sys -from cStringIO import StringIO - - - -# This function will become a method of the Message class -def walk(self): - """Walk over the message tree, yielding each subpart. - - The walk is performed in depth-first order. This method is a - generator. - """ - yield self - if self.is_multipart(): - for subpart in self.get_payload(): - for subsubpart in subpart.walk(): - yield subsubpart - - - -# These two functions are imported into the Iterators.py interface module. -def body_line_iterator(msg, decode=False): - """Iterate over the parts, returning string payloads line-by-line. - - Optional decode (default False) is passed through to .get_payload(). - """ - for subpart in msg.walk(): - payload = subpart.get_payload(decode=decode) - if isinstance(payload, basestring): - for line in StringIO(payload): - yield line - - -def typed_subpart_iterator(msg, maintype='text', subtype=None): - """Iterate over the subparts with a given MIME type. - - Use `maintype' as the main MIME type to match against; this defaults to - "text". Optional `subtype' is the MIME subtype to match against; if - omitted, only the main type is matched. - """ - for subpart in msg.walk(): - if subpart.get_content_maintype() == maintype: - if subtype is None or subpart.get_content_subtype() == subtype: - yield subpart - - - -def _structure(msg, fp=None, level=0, include_default=False): - """A handy debugging aid""" - if fp is None: - fp = sys.stdout - tab = ' ' * (level * 4) - print >> fp, tab + msg.get_content_type(), - if include_default: - print >> fp, '[%s]' % msg.get_default_type() - else: - print >> fp - if msg.is_multipart(): - for subpart in msg.get_payload(): - _structure(subpart, fp, level+1, include_default) diff --git a/lib-python/modified-2.7/email/message.py b/lib-python/modified-2.7/email/message.py deleted file mode 100644 --- a/lib-python/modified-2.7/email/message.py +++ /dev/null @@ -1,797 +0,0 @@ -# Copyright (C) 2001-2006 Python Software Foundation -# Author: Barry Warsaw -# Contact: email-sig at python.org - -"""Basic message object for the email package object model.""" - -__all__ = ['Message'] - -import re -import uu -import binascii -import warnings -from cStringIO import StringIO - -# Intrapackage imports -import email.charset -from email import utils -from email import errors - -SEMISPACE = '; ' - -# Regular expression that matches `special' characters in parameters, the -# existence of which force quoting of the parameter value. -tspecials = re.compile(r'[ <>@,;:\\"/\[\]\?=]') - - -# Helper functions -def _splitparam(param): - # Split header parameters. BAW: this may be too simple. It isn't - # strictly RFC 2045 (section 5.1) compliant, but it catches most headers - # found in the wild. We may eventually need a full fledged parser - # eventually. - a, sep, b = param.partition(';') - if not sep: - return a.strip(), None - return a.strip(), b.strip() - -def _formatparam(param, value=None, quote=True): - """Convenience function to format and return a key=value pair. - - This will quote the value if needed or if quote is true. If value is a - three tuple (charset, language, value), it will be encoded according - to RFC2231 rules. - """ - if value is not None and len(value) > 0: - # A tuple is used for RFC 2231 encoded parameter values where items - # are (charset, language, value). charset is a string, not a Charset - # instance. - if isinstance(value, tuple): - # Encode as per RFC 2231 - param += '*' - value = utils.encode_rfc2231(value[2], value[0], value[1]) - # BAW: Please check this. I think that if quote is set it should - # force quoting even if not necessary. - if quote or tspecials.search(value): - return '%s="%s"' % (param, utils.quote(value)) - else: - return '%s=%s' % (param, value) - else: - return param - -def _parseparam(s): - plist = [] - while s[:1] == ';': - s = s[1:] - end = s.find(';') - while end > 0 and (s.count('"', 0, end) - s.count('\\"', 0, end)) % 2: - end = s.find(';', end + 1) - if end < 0: - end = len(s) - f = s[:end] - if '=' in f: - i = f.index('=') - f = f[:i].strip().lower() + '=' + f[i+1:].strip() - plist.append(f.strip()) - s = s[end:] - return plist - - -def _unquotevalue(value): - # This is different than utils.collapse_rfc2231_value() because it doesn't - # try to convert the value to a unicode. Message.get_param() and - # Message.get_params() are both currently defined to return the tuple in - # the face of RFC 2231 parameters. - if isinstance(value, tuple): - return value[0], value[1], utils.unquote(value[2]) - else: - return utils.unquote(value) - - - -class Message: - """Basic message object. - - A message object is defined as something that has a bunch of RFC 2822 - headers and a payload. It may optionally have an envelope header - (a.k.a. Unix-From or From_ header). If the message is a container (i.e. a - multipart or a message/rfc822), then the payload is a list of Message - objects, otherwise it is a string. - - Message objects implement part of the `mapping' interface, which assumes - there is exactly one occurrence of the header per message. Some headers - do in fact appear multiple times (e.g. Received) and for those headers, - you must use the explicit API to set or get all the headers. Not all of - the mapping methods are implemented. - """ - def __init__(self): - self._headers = [] - self._unixfrom = None - self._payload = None - self._charset = None - # Defaults for multipart messages - self.preamble = self.epilogue = None - self.defects = [] - # Default content type - self._default_type = 'text/plain' - - def __str__(self): - """Return the entire formatted message as a string. - This includes the headers, body, and envelope header. - """ - return self.as_string(unixfrom=True) - - def as_string(self, unixfrom=False): - """Return the entire formatted message as a string. - Optional `unixfrom' when True, means include the Unix From_ envelope - header. - - This is a convenience method and may not generate the message exactly - as you intend because by default it mangles lines that begin with - "From ". For more flexibility, use the flatten() method of a - Generator instance. - """ - from email.generator import Generator - fp = StringIO() - g = Generator(fp) - g.flatten(self, unixfrom=unixfrom) - return fp.getvalue() - - def is_multipart(self): - """Return True if the message consists of multiple parts.""" - return isinstance(self._payload, list) - - # - # Unix From_ line - # - def set_unixfrom(self, unixfrom): - self._unixfrom = unixfrom - - def get_unixfrom(self): - return self._unixfrom - - # - # Payload manipulation. - # - def attach(self, payload): - """Add the given payload to the current payload. - - The current payload will always be a list of objects after this method - is called. If you want to set the payload to a scalar object, use - set_payload() instead. - """ - if self._payload is None: - self._payload = [payload] - else: - self._payload.append(payload) - - def get_payload(self, i=None, decode=False): - """Return a reference to the payload. - - The payload will either be a list object or a string. If you mutate - the list object, you modify the message's payload in place. Optional - i returns that index into the payload. - - Optional decode is a flag indicating whether the payload should be - decoded or not, according to the Content-Transfer-Encoding header - (default is False). - - When True and the message is not a multipart, the payload will be - decoded if this header's value is `quoted-printable' or `base64'. If - some other encoding is used, or the header is missing, or if the - payload has bogus data (i.e. bogus base64 or uuencoded data), the - payload is returned as-is. - - If the message is a multipart and the decode flag is True, then None - is returned. - """ - if i is None: - payload = self._payload - elif not isinstance(self._payload, list): - raise TypeError('Expected list, got %s' % type(self._payload)) - else: - payload = self._payload[i] - if decode: - if self.is_multipart(): - return None - cte = self.get('content-transfer-encoding', '').lower() - if cte == 'quoted-printable': - return utils._qdecode(payload) - elif cte == 'base64': - try: - return utils._bdecode(payload) - except binascii.Error: - # Incorrect padding - return payload - elif cte in ('x-uuencode', 'uuencode', 'uue', 'x-uue'): - sfp = StringIO() - try: - uu.decode(StringIO(payload+'\n'), sfp, quiet=True) - payload = sfp.getvalue() - except uu.Error: - # Some decoding problem - return payload - # Everything else, including encodings with 8bit or 7bit are returned - # unchanged. - return payload - - def set_payload(self, payload, charset=None): - """Set the payload to the given value. - - Optional charset sets the message's default character set. See - set_charset() for details. - """ - self._payload = payload - if charset is not None: - self.set_charset(charset) - - def set_charset(self, charset): - """Set the charset of the payload to a given character set. - - charset can be a Charset instance, a string naming a character set, or - None. If it is a string it will be converted to a Charset instance. - If charset is None, the charset parameter will be removed from the - Content-Type field. Anything else will generate a TypeError. - - The message will be assumed to be of type text/* encoded with - charset.input_charset. It will be converted to charset.output_charset - and encoded properly, if needed, when generating the plain text - representation of the message. MIME headers (MIME-Version, - Content-Type, Content-Transfer-Encoding) will be added as needed. - - """ - if charset is None: - self.del_param('charset') - self._charset = None - return - if isinstance(charset, basestring): - charset = email.charset.Charset(charset) - if not isinstance(charset, email.charset.Charset): - raise TypeError(charset) - # BAW: should we accept strings that can serve as arguments to the - # Charset constructor? - self._charset = charset - if 'MIME-Version' not in self: - self.add_header('MIME-Version', '1.0') - if 'Content-Type' not in self: - self.add_header('Content-Type', 'text/plain', - charset=charset.get_output_charset()) - else: - self.set_param('charset', charset.get_output_charset()) - if isinstance(self._payload, unicode): - self._payload = self._payload.encode(charset.output_charset) - if str(charset) != charset.get_output_charset(): - self._payload = charset.body_encode(self._payload) - if 'Content-Transfer-Encoding' not in self: - cte = charset.get_body_encoding() - try: - cte(self) - except TypeError: - self._payload = charset.body_encode(self._payload) - self.add_header('Content-Transfer-Encoding', cte) - - def get_charset(self): - """Return the Charset instance associated with the message's payload. - """ - return self._charset - - # - # MAPPING INTERFACE (partial) - # - def __len__(self): - """Return the total number of headers, including duplicates.""" - return len(self._headers) - - def __getitem__(self, name): - """Get a header value. - - Return None if the header is missing instead of raising an exception. - - Note that if the header appeared multiple times, exactly which - occurrence gets returned is undefined. Use get_all() to get all - the values matching a header field name. - """ - return self.get(name) - - def __setitem__(self, name, val): - """Set the value of a header. - - Note: this does not overwrite an existing header with the same field - name. Use __delitem__() first to delete any existing headers. - """ - self._headers.append((name, val)) - - def __delitem__(self, name): - """Delete all occurrences of a header, if present. - - Does not raise an exception if the header is missing. - """ - name = name.lower() - newheaders = [] - for k, v in self._headers: - if k.lower() != name: - newheaders.append((k, v)) - self._headers = newheaders - - def __contains__(self, name): - return name.lower() in [k.lower() for k, v in self._headers] - - def has_key(self, name): - """Return true if the message contains the header.""" - missing = object() - return self.get(name, missing) is not missing - - def keys(self): - """Return a list of all the message's header field names. - - These will be sorted in the order they appeared in the original - message, or were added to the message, and may contain duplicates. - Any fields deleted and re-inserted are always appended to the header - list. - """ - return [k for k, v in self._headers] - - def values(self): - """Return a list of all the message's header values. - - These will be sorted in the order they appeared in the original - message, or were added to the message, and may contain duplicates. - Any fields deleted and re-inserted are always appended to the header - list. - """ - return [v for k, v in self._headers] - - def items(self): - """Get all the message's header fields and values. - - These will be sorted in the order they appeared in the original - message, or were added to the message, and may contain duplicates. - Any fields deleted and re-inserted are always appended to the header - list. - """ - return self._headers[:] - - def get(self, name, failobj=None): - """Get a header value. - - Like __getitem__() but return failobj instead of None when the field - is missing. - """ - name = name.lower() - for k, v in self._headers: - if k.lower() == name: - return v - return failobj - - # - # Additional useful stuff - # - - def get_all(self, name, failobj=None): - """Return a list of all the values for the named field. - - These will be sorted in the order they appeared in the original - message, and may contain duplicates. Any fields deleted and - re-inserted are always appended to the header list. - - If no such fields exist, failobj is returned (defaults to None). - """ - values = [] - name = name.lower() - for k, v in self._headers: - if k.lower() == name: - values.append(v) - if not values: - return failobj - return values - - def add_header(self, _name, _value, **_params): - """Extended header setting. - - name is the header field to add. keyword arguments can be used to set - additional parameters for the header field, with underscores converted - to dashes. Normally the parameter will be added as key="value" unless - value is None, in which case only the key will be added. If a - parameter value contains non-ASCII characters it must be specified as a - three-tuple of (charset, language, value), in which case it will be - encoded according to RFC2231 rules. - - Example: - - msg.add_header('content-disposition', 'attachment', filename='bud.gif') - """ - parts = [] - for k, v in _params.items(): - if v is None: - parts.append(k.replace('_', '-')) - else: - parts.append(_formatparam(k.replace('_', '-'), v)) - if _value is not None: - parts.insert(0, _value) - self._headers.append((_name, SEMISPACE.join(parts))) - - def replace_header(self, _name, _value): - """Replace a header. - - Replace the first matching header found in the message, retaining - header order and case. If no matching header was found, a KeyError is - raised. - """ - _name = _name.lower() - for i, (k, v) in zip(range(len(self._headers)), self._headers): - if k.lower() == _name: - self._headers[i] = (k, _value) - break - else: - raise KeyError(_name) - - # - # Use these three methods instead of the three above. - # - - def get_content_type(self): - """Return the message's content type. - - The returned string is coerced to lower case of the form - `maintype/subtype'. If there was no Content-Type header in the - message, the default type as given by get_default_type() will be - returned. Since according to RFC 2045, messages always have a default - type this will always return a value. - - RFC 2045 defines a message's default type to be text/plain unless it - appears inside a multipart/digest container, in which case it would be - message/rfc822. - """ - missing = object() - value = self.get('content-type', missing) - if value is missing: - # This should have no parameters - return self.get_default_type() - ctype = _splitparam(value)[0].lower() - # RFC 2045, section 5.2 says if its invalid, use text/plain - if ctype.count('/') != 1: - return 'text/plain' - return ctype - - def get_content_maintype(self): - """Return the message's main content type. - - This is the `maintype' part of the string returned by - get_content_type(). - """ - ctype = self.get_content_type() - return ctype.split('/')[0] - - def get_content_subtype(self): - """Returns the message's sub-content type. - - This is the `subtype' part of the string returned by - get_content_type(). - """ - ctype = self.get_content_type() - return ctype.split('/')[1] - - def get_default_type(self): - """Return the `default' content type. - - Most messages have a default content type of text/plain, except for - messages that are subparts of multipart/digest containers. Such - subparts have a default content type of message/rfc822. - """ - return self._default_type - - def set_default_type(self, ctype): - """Set the `default' content type. - - ctype should be either "text/plain" or "message/rfc822", although this - is not enforced. The default content type is not stored in the - Content-Type header. - """ - self._default_type = ctype - - def _get_params_preserve(self, failobj, header): - # Like get_params() but preserves the quoting of values. BAW: - # should this be part of the public interface? - missing = object() - value = self.get(header, missing) - if value is missing: - return failobj - params = [] - for p in _parseparam(';' + value): - try: - name, val = p.split('=', 1) - name = name.strip() - val = val.strip() - except ValueError: - # Must have been a bare attribute - name = p.strip() - val = '' - params.append((name, val)) - params = utils.decode_params(params) - return params - - def get_params(self, failobj=None, header='content-type', unquote=True): - """Return the message's Content-Type parameters, as a list. - - The elements of the returned list are 2-tuples of key/value pairs, as - split on the `=' sign. The left hand side of the `=' is the key, - while the right hand side is the value. If there is no `=' sign in - the parameter the value is the empty string. The value is as - described in the get_param() method. - - Optional failobj is the object to return if there is no Content-Type - header. Optional header is the header to search instead of - Content-Type. If unquote is True, the value is unquoted. - """ - missing = object() - params = self._get_params_preserve(missing, header) - if params is missing: - return failobj - if unquote: - return [(k, _unquotevalue(v)) for k, v in params] - else: - return params - - def get_param(self, param, failobj=None, header='content-type', - unquote=True): - """Return the parameter value if found in the Content-Type header. - - Optional failobj is the object to return if there is no Content-Type - header, or the Content-Type header has no such parameter. Optional - header is the header to search instead of Content-Type. - - Parameter keys are always compared case insensitively. The return - value can either be a string, or a 3-tuple if the parameter was RFC - 2231 encoded. When it's a 3-tuple, the elements of the value are of - the form (CHARSET, LANGUAGE, VALUE). Note that both CHARSET and - LANGUAGE can be None, in which case you should consider VALUE to be - encoded in the us-ascii charset. You can usually ignore LANGUAGE. - - Your application should be prepared to deal with 3-tuple return - values, and can convert the parameter to a Unicode string like so: - - param = msg.get_param('foo') - if isinstance(param, tuple): - param = unicode(param[2], param[0] or 'us-ascii') - - In any case, the parameter value (either the returned string, or the - VALUE item in the 3-tuple) is always unquoted, unless unquote is set - to False. - """ - if header not in self: - return failobj - for k, v in self._get_params_preserve(failobj, header): - if k.lower() == param.lower(): - if unquote: - return _unquotevalue(v) - else: - return v - return failobj - - def set_param(self, param, value, header='Content-Type', requote=True, - charset=None, language=''): - """Set a parameter in the Content-Type header. - - If the parameter already exists in the header, its value will be - replaced with the new value. - - If header is Content-Type and has not yet been defined for this - message, it will be set to "text/plain" and the new parameter and - value will be appended as per RFC 2045. - - An alternate header can specified in the header argument, and all - parameters will be quoted as necessary unless requote is False. - - If charset is specified, the parameter will be encoded according to RFC - 2231. Optional language specifies the RFC 2231 language, defaulting - to the empty string. Both charset and language should be strings. - """ - if not isinstance(value, tuple) and charset: - value = (charset, language, value) - - if header not in self and header.lower() == 'content-type': - ctype = 'text/plain' - else: - ctype = self.get(header) - if not self.get_param(param, header=header): - if not ctype: - ctype = _formatparam(param, value, requote) - else: - ctype = SEMISPACE.join( - [ctype, _formatparam(param, value, requote)]) - else: - ctype = '' - for old_param, old_value in self.get_params(header=header, - unquote=requote): - append_param = '' - if old_param.lower() == param.lower(): - append_param = _formatparam(param, value, requote) - else: - append_param = _formatparam(old_param, old_value, requote) - if not ctype: - ctype = append_param - else: - ctype = SEMISPACE.join([ctype, append_param]) - if ctype != self.get(header): - del self[header] - self[header] = ctype - - def del_param(self, param, header='content-type', requote=True): - """Remove the given parameter completely from the Content-Type header. - - The header will be re-written in place without the parameter or its - value. All values will be quoted as necessary unless requote is - False. Optional header specifies an alternative to the Content-Type - header. - """ - if header not in self: - return - new_ctype = '' - for p, v in self.get_params(header=header, unquote=requote): - if p.lower() != param.lower(): - if not new_ctype: - new_ctype = _formatparam(p, v, requote) - else: - new_ctype = SEMISPACE.join([new_ctype, - _formatparam(p, v, requote)]) - if new_ctype != self.get(header): - del self[header] - self[header] = new_ctype - - def set_type(self, type, header='Content-Type', requote=True): - """Set the main type and subtype for the Content-Type header. - - type must be a string in the form "maintype/subtype", otherwise a - ValueError is raised. - - This method replaces the Content-Type header, keeping all the - parameters in place. If requote is False, this leaves the existing - header's quoting as is. Otherwise, the parameters will be quoted (the - default). - - An alternative header can be specified in the header argument. When - the Content-Type header is set, we'll always also add a MIME-Version - header. - """ - # BAW: should we be strict? - if not type.count('/') == 1: - raise ValueError - # Set the Content-Type, you get a MIME-Version - if header.lower() == 'content-type': - del self['mime-version'] - self['MIME-Version'] = '1.0' - if header not in self: - self[header] = type - return - params = self.get_params(header=header, unquote=requote) - del self[header] - self[header] = type - # Skip the first param; it's the old type. - for p, v in params[1:]: - self.set_param(p, v, header, requote) - - def get_filename(self, failobj=None): - """Return the filename associated with the payload if present. - - The filename is extracted from the Content-Disposition header's - `filename' parameter, and it is unquoted. If that header is missing - the `filename' parameter, this method falls back to looking for the - `name' parameter. - """ - missing = object() - filename = self.get_param('filename', missing, 'content-disposition') - if filename is missing: - filename = self.get_param('name', missing, 'content-type') - if filename is missing: - return failobj - return utils.collapse_rfc2231_value(filename).strip() - - def get_boundary(self, failobj=None): - """Return the boundary associated with the payload if present. - - The boundary is extracted from the Content-Type header's `boundary' - parameter, and it is unquoted. - """ - missing = object() - boundary = self.get_param('boundary', missing) - if boundary is missing: - return failobj - # RFC 2046 says that boundaries may begin but not end in w/s - return utils.collapse_rfc2231_value(boundary).rstrip() - - def set_boundary(self, boundary): - """Set the boundary parameter in Content-Type to 'boundary'. - - This is subtly different than deleting the Content-Type header and - adding a new one with a new boundary parameter via add_header(). The - main difference is that using the set_boundary() method preserves the - order of the Content-Type header in the original message. - - HeaderParseError is raised if the message has no Content-Type header. - """ - missing = object() - params = self._get_params_preserve(missing, 'content-type') - if params is missing: - # There was no Content-Type header, and we don't know what type - # to set it to, so raise an exception. - raise errors.HeaderParseError('No Content-Type header found') - newparams = [] - foundp = False - for pk, pv in params: - if pk.lower() == 'boundary': - newparams.append(('boundary', '"%s"' % boundary)) - foundp = True - else: - newparams.append((pk, pv)) - if not foundp: - # The original Content-Type header had no boundary attribute. - # Tack one on the end. BAW: should we raise an exception - # instead??? - newparams.append(('boundary', '"%s"' % boundary)) - # Replace the existing Content-Type header with the new value - newheaders = [] - for h, v in self._headers: - if h.lower() == 'content-type': - parts = [] - for k, v in newparams: - if v == '': - parts.append(k) - else: - parts.append('%s=%s' % (k, v)) - newheaders.append((h, SEMISPACE.join(parts))) - - else: - newheaders.append((h, v)) - self._headers = newheaders - - def get_content_charset(self, failobj=None): - """Return the charset parameter of the Content-Type header. - - The returned string is always coerced to lower case. If there is no - Content-Type header, or if that header has no charset parameter, - failobj is returned. - """ - missing = object() - charset = self.get_param('charset', missing) - if charset is missing: - return failobj - if isinstance(charset, tuple): - # RFC 2231 encoded, so decode it, and it better end up as ascii. - pcharset = charset[0] or 'us-ascii' - try: - # LookupError will be raised if the charset isn't known to - # Python. UnicodeError will be raised if the encoded text - # contains a character not in the charset. - charset = unicode(charset[2], pcharset).encode('us-ascii') - except (LookupError, UnicodeError): - charset = charset[2] - # charset character must be in us-ascii range - try: - if isinstance(charset, str): - charset = unicode(charset, 'us-ascii') - charset = charset.encode('us-ascii') - except UnicodeError: - return failobj - # RFC 2046, $4.1.2 says charsets are not case sensitive - return charset.lower() - - def get_charsets(self, failobj=None): - """Return a list containing the charset(s) used in this message. - - The returned list of items describes the Content-Type headers' - charset parameter for this message and all the subparts in its - payload. - - Each item will either be a string (the value of the charset parameter - in the Content-Type header of that part) or the value of the - 'failobj' parameter (defaults to None), if the part does not have a - main MIME type of "text", or the charset is not defined. - - The list will contain one string for each part of the message, plus - one for the container message (i.e. self), so that a non-multipart - message will still return a list of length 1. - """ - return [part.get_content_charset(failobj) for part in self.walk()] - - # I.e. def walk(self): ... - from email.iterators import walk diff --git a/lib-python/modified-2.7/email/mime/__init__.py b/lib-python/modified-2.7/email/mime/__init__.py deleted file mode 100644 diff --git a/lib-python/modified-2.7/email/mime/application.py b/lib-python/modified-2.7/email/mime/application.py deleted file mode 100644 --- a/lib-python/modified-2.7/email/mime/application.py +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright (C) 2001-2006 Python Software Foundation -# Author: Keith Dart -# Contact: email-sig at python.org - -"""Class representing application/* type MIME documents.""" - -__all__ = ["MIMEApplication"] - -from email import encoders -from email.mime.nonmultipart import MIMENonMultipart - - -class MIMEApplication(MIMENonMultipart): - """Class for generating application/* MIME documents.""" - - def __init__(self, _data, _subtype='octet-stream', - _encoder=encoders.encode_base64, **_params): - """Create an application/* type MIME document. - - _data is a string containing the raw application data. - - _subtype is the MIME content type subtype, defaulting to - 'octet-stream'. - - _encoder is a function which will perform the actual encoding for - transport of the application data, defaulting to base64 encoding. - - Any additional keyword arguments are passed to the base class - constructor, which turns them into parameters on the Content-Type - header. - """ - if _subtype is None: - raise TypeError('Invalid application MIME subtype') - MIMENonMultipart.__init__(self, 'application', _subtype, **_params) - self.set_payload(_data) - _encoder(self) diff --git a/lib-python/modified-2.7/email/mime/audio.py b/lib-python/modified-2.7/email/mime/audio.py deleted file mode 100644 --- a/lib-python/modified-2.7/email/mime/audio.py +++ /dev/null @@ -1,73 +0,0 @@ -# Copyright (C) 2001-2006 Python Software Foundation -# Author: Anthony Baxter -# Contact: email-sig at python.org - -"""Class representing audio/* type MIME documents.""" - -__all__ = ['MIMEAudio'] - -import sndhdr - -from cStringIO import StringIO -from email import encoders -from email.mime.nonmultipart import MIMENonMultipart - - - -_sndhdr_MIMEmap = {'au' : 'basic', - 'wav' :'x-wav', - 'aiff':'x-aiff', - 'aifc':'x-aiff', - } - -# There are others in sndhdr that don't have MIME types. :( -# Additional ones to be added to sndhdr? midi, mp3, realaudio, wma?? -def _whatsnd(data): - """Try to identify a sound file type. - - sndhdr.what() has a pretty cruddy interface, unfortunately. This is why - we re-do it here. It would be easier to reverse engineer the Unix 'file' - command and use the standard 'magic' file, as shipped with a modern Unix. - """ - hdr = data[:512] - fakefile = StringIO(hdr) - for testfn in sndhdr.tests: - res = testfn(hdr, fakefile) - if res is not None: - return _sndhdr_MIMEmap.get(res[0]) - return None - - - -class MIMEAudio(MIMENonMultipart): - """Class for generating audio/* MIME documents.""" - - def __init__(self, _audiodata, _subtype=None, - _encoder=encoders.encode_base64, **_params): - """Create an audio/* type MIME document. - - _audiodata is a string containing the raw audio data. If this data - can be decoded by the standard Python `sndhdr' module, then the - subtype will be automatically included in the Content-Type header. - Otherwise, you can specify the specific audio subtype via the - _subtype parameter. If _subtype is not given, and no subtype can be - guessed, a TypeError is raised. - - _encoder is a function which will perform the actual encoding for - transport of the image data. It takes one argument, which is this - Image instance. It should use get_payload() and set_payload() to - change the payload to the encoded form. It should also add any - Content-Transfer-Encoding or other headers to the message as - necessary. The default encoding is Base64. - - Any additional keyword arguments are passed to the base class - constructor, which turns them into parameters on the Content-Type - header. - """ - if _subtype is None: - _subtype = _whatsnd(_audiodata) - if _subtype is None: - raise TypeError('Could not find audio MIME subtype') - MIMENonMultipart.__init__(self, 'audio', _subtype, **_params) - self.set_payload(_audiodata) - _encoder(self) diff --git a/lib-python/modified-2.7/email/mime/base.py b/lib-python/modified-2.7/email/mime/base.py deleted file mode 100644 --- a/lib-python/modified-2.7/email/mime/base.py +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright (C) 2001-2006 Python Software Foundation -# Author: Barry Warsaw -# Contact: email-sig at python.org - -"""Base class for MIME specializations.""" - -__all__ = ['MIMEBase'] - -from email import message - - - -class MIMEBase(message.Message): - """Base class for MIME specializations.""" - - def __init__(self, _maintype, _subtype, **_params): - """This constructor adds a Content-Type: and a MIME-Version: header. - - The Content-Type: header is taken from the _maintype and _subtype - arguments. Additional parameters for this header are taken from the - keyword arguments. - """ - message.Message.__init__(self) - ctype = '%s/%s' % (_maintype, _subtype) - self.add_header('Content-Type', ctype, **_params) - self['MIME-Version'] = '1.0' diff --git a/lib-python/modified-2.7/email/mime/image.py b/lib-python/modified-2.7/email/mime/image.py deleted file mode 100644 --- a/lib-python/modified-2.7/email/mime/image.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright (C) 2001-2006 Python Software Foundation -# Author: Barry Warsaw -# Contact: email-sig at python.org - -"""Class representing image/* type MIME documents.""" - -__all__ = ['MIMEImage'] - -import imghdr - -from email import encoders -from email.mime.nonmultipart import MIMENonMultipart - - - -class MIMEImage(MIMENonMultipart): - """Class for generating image/* type MIME documents.""" - - def __init__(self, _imagedata, _subtype=None, - _encoder=encoders.encode_base64, **_params): - """Create an image/* type MIME document. - - _imagedata is a string containing the raw image data. If this data - can be decoded by the standard Python `imghdr' module, then the - subtype will be automatically included in the Content-Type header. - Otherwise, you can specify the specific image subtype via the _subtype - parameter. - - _encoder is a function which will perform the actual encoding for - transport of the image data. It takes one argument, which is this - Image instance. It should use get_payload() and set_payload() to - change the payload to the encoded form. It should also add any - Content-Transfer-Encoding or other headers to the message as - necessary. The default encoding is Base64. - - Any additional keyword arguments are passed to the base class - constructor, which turns them into parameters on the Content-Type - header. - """ - if _subtype is None: - _subtype = imghdr.what(None, _imagedata) - if _subtype is None: - raise TypeError('Could not guess image MIME subtype') - MIMENonMultipart.__init__(self, 'image', _subtype, **_params) - self.set_payload(_imagedata) - _encoder(self) diff --git a/lib-python/modified-2.7/email/mime/message.py b/lib-python/modified-2.7/email/mime/message.py deleted file mode 100644 --- a/lib-python/modified-2.7/email/mime/message.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright (C) 2001-2006 Python Software Foundation -# Author: Barry Warsaw -# Contact: email-sig at python.org - -"""Class representing message/* MIME documents.""" - -__all__ = ['MIMEMessage'] - -from email import message -from email.mime.nonmultipart import MIMENonMultipart - - - -class MIMEMessage(MIMENonMultipart): - """Class representing message/* MIME documents.""" - - def __init__(self, _msg, _subtype='rfc822'): - """Create a message/* type MIME document. - - _msg is a message object and must be an instance of Message, or a - derived class of Message, otherwise a TypeError is raised. - - Optional _subtype defines the subtype of the contained message. The - default is "rfc822" (this is defined by the MIME standard, even though - the term "rfc822" is technically outdated by RFC 2822). - """ - MIMENonMultipart.__init__(self, 'message', _subtype) - if not isinstance(_msg, message.Message): - raise TypeError('Argument is not an instance of Message') - # It's convenient to use this base class method. We need to do it - # this way or we'll get an exception - message.Message.attach(self, _msg) - # And be sure our default type is set correctly - self.set_default_type('message/rfc822') diff --git a/lib-python/modified-2.7/email/mime/multipart.py b/lib-python/modified-2.7/email/mime/multipart.py deleted file mode 100644 --- a/lib-python/modified-2.7/email/mime/multipart.py +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright (C) 2002-2006 Python Software Foundation -# Author: Barry Warsaw -# Contact: email-sig at python.org - -"""Base class for MIME multipart/* type messages.""" - -__all__ = ['MIMEMultipart'] - -from email.mime.base import MIMEBase - - - -class MIMEMultipart(MIMEBase): - """Base class for MIME multipart/* type messages.""" - - def __init__(self, _subtype='mixed', boundary=None, _subparts=None, - **_params): - """Creates a multipart/* type message. - - By default, creates a multipart/mixed message, with proper - Content-Type and MIME-Version headers. - - _subtype is the subtype of the multipart content type, defaulting to - `mixed'. - - boundary is the multipart boundary string. By default it is - calculated as needed. - - _subparts is a sequence of initial subparts for the payload. It - must be an iterable object, such as a list. You can always - attach new subparts to the message by using the attach() method. - - Additional parameters for the Content-Type header are taken from the - keyword arguments (or passed into the _params argument). - """ - MIMEBase.__init__(self, 'multipart', _subtype, **_params) - - # Initialise _payload to an empty list as the Message superclass's - # implementation of is_multipart assumes that _payload is a list for - # multipart messages. - self._payload = [] - - if _subparts: - for p in _subparts: - self.attach(p) - if boundary: - self.set_boundary(boundary) diff --git a/lib-python/modified-2.7/email/mime/nonmultipart.py b/lib-python/modified-2.7/email/mime/nonmultipart.py deleted file mode 100644 --- a/lib-python/modified-2.7/email/mime/nonmultipart.py +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright (C) 2002-2006 Python Software Foundation -# Author: Barry Warsaw -# Contact: email-sig at python.org - -"""Base class for MIME type messages that are not multipart.""" - -__all__ = ['MIMENonMultipart'] - -from email import errors -from email.mime.base import MIMEBase - - - -class MIMENonMultipart(MIMEBase): - """Base class for MIME multipart/* type messages.""" - - def attach(self, payload): - # The public API prohibits attaching multiple subparts to MIMEBase - # derived subtypes since none of them are, by definition, of content - # type multipart/* - raise errors.MultipartConversionError( - 'Cannot attach additional subparts to non-multipart/*') diff --git a/lib-python/modified-2.7/email/mime/text.py b/lib-python/modified-2.7/email/mime/text.py deleted file mode 100644 --- a/lib-python/modified-2.7/email/mime/text.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright (C) 2001-2006 Python Software Foundation -# Author: Barry Warsaw -# Contact: email-sig at python.org - -"""Class representing text/* type MIME documents.""" - -__all__ = ['MIMEText'] - -from email.encoders import encode_7or8bit -from email.mime.nonmultipart import MIMENonMultipart - - - -class MIMEText(MIMENonMultipart): - """Class for generating text/* type MIME documents.""" - - def __init__(self, _text, _subtype='plain', _charset='us-ascii'): - """Create a text/* type MIME document. - - _text is the string for this message object. - - _subtype is the MIME sub content type, defaulting to "plain". - - _charset is the character set parameter added to the Content-Type - header. This defaults to "us-ascii". Note that as a side-effect, the - Content-Transfer-Encoding header will also be set. - """ - MIMENonMultipart.__init__(self, 'text', _subtype, - **{'charset': _charset}) - self.set_payload(_text, _charset) diff --git a/lib-python/modified-2.7/email/parser.py b/lib-python/modified-2.7/email/parser.py deleted file mode 100644 --- a/lib-python/modified-2.7/email/parser.py +++ /dev/null @@ -1,91 +0,0 @@ -# Copyright (C) 2001-2006 Python Software Foundation -# Author: Barry Warsaw, Thomas Wouters, Anthony Baxter -# Contact: email-sig at python.org - -"""A parser of RFC 2822 and MIME email messages.""" - -__all__ = ['Parser', 'HeaderParser'] - -import warnings -from cStringIO import StringIO - -from email.feedparser import FeedParser -from email.message import Message - - - -class Parser: - def __init__(self, *args, **kws): - """Parser of RFC 2822 and MIME email messages. - - Creates an in-memory object tree representing the email message, which - can then be manipulated and turned over to a Generator to return the - textual representation of the message. - - The string must be formatted as a block of RFC 2822 headers and header - continuation lines, optionally preceeded by a `Unix-from' header. The - header block is terminated either by the end of the string or by a - blank line. - - _class is the class to instantiate for new message objects when they - must be created. This class must have a constructor that can take - zero arguments. Default is Message.Message. - """ - if len(args) >= 1: - if '_class' in kws: - raise TypeError("Multiple values for keyword arg '_class'") - kws['_class'] = args[0] - if len(args) == 2: - if 'strict' in kws: - raise TypeError("Multiple values for keyword arg 'strict'") - kws['strict'] = args[1] - if len(args) > 2: - raise TypeError('Too many arguments') - if '_class' in kws: - self._class = kws['_class'] - del kws['_class'] - else: - self._class = Message - if 'strict' in kws: - warnings.warn("'strict' argument is deprecated (and ignored)", - DeprecationWarning, 2) - del kws['strict'] - if kws: - raise TypeError('Unexpected keyword arguments') - - def parse(self, fp, headersonly=False): - """Create a message structure from the data in a file. - - Reads all the data from the file and returns the root of the message - structure. Optional headersonly is a flag specifying whether to stop - parsing after reading the headers or not. The default is False, - meaning it parses the entire contents of the file. - """ - feedparser = FeedParser(self._class) - if headersonly: - feedparser._set_headersonly() - while True: - data = fp.read(8192) - if not data: - break - feedparser.feed(data) - return feedparser.close() - - def parsestr(self, text, headersonly=False): - """Create a message structure from a string. - - Returns the root of the message structure. Optional headersonly is a - flag specifying whether to stop parsing after reading the headers or - not. The default is False, meaning it parses the entire contents of - the file. - """ - return self.parse(StringIO(text), headersonly=headersonly) - - - -class HeaderParser(Parser): - def parse(self, fp, headersonly=True): - return Parser.parse(self, fp, True) - - def parsestr(self, text, headersonly=True): - return Parser.parsestr(self, text, True) diff --git a/lib-python/modified-2.7/email/quoprimime.py b/lib-python/modified-2.7/email/quoprimime.py deleted file mode 100644 --- a/lib-python/modified-2.7/email/quoprimime.py +++ /dev/null @@ -1,336 +0,0 @@ -# Copyright (C) 2001-2006 Python Software Foundation -# Author: Ben Gertzfield -# Contact: email-sig at python.org - -"""Quoted-printable content transfer encoding per RFCs 2045-2047. - -This module handles the content transfer encoding method defined in RFC 2045 -to encode US ASCII-like 8-bit data called `quoted-printable'. It is used to -safely encode text that is in a character set similar to the 7-bit US ASCII -character set, but that includes some 8-bit characters that are normally not -allowed in email bodies or headers. - -Quoted-printable is very space-inefficient for encoding binary files; use the -email.base64mime module for that instead. - -This module provides an interface to encode and decode both headers and bodies -with quoted-printable encoding. - -RFC 2045 defines a method for including character set information in an -`encoded-word' in a header. This method is commonly used for 8-bit real names -in To:/From:/Cc: etc. fields, as well as Subject: lines. - -This module does not do the line wrapping or end-of-line character -conversion necessary for proper internationalized headers; it only -does dumb encoding and decoding. To deal with the various line -wrapping issues, use the email.header module. -""" - -__all__ = [ - 'body_decode', - 'body_encode', - 'body_quopri_check', - 'body_quopri_len', - 'decode', - 'decodestring', - 'encode', - 'encodestring', - 'header_decode', - 'header_encode', - 'header_quopri_check', - 'header_quopri_len', - 'quote', - 'unquote', - ] - -import re - -from string import hexdigits -from email.utils import fix_eols - -CRLF = '\r\n' -NL = '\n' - -# See also Charset.py -MISC_LEN = 7 - -hqre = re.compile(r'[^-a-zA-Z0-9!*+/ ]') -bqre = re.compile(r'[^ !-<>-~\t]') - - - -# Helpers -def header_quopri_check(c): - """Return True if the character should be escaped with header quopri.""" - return bool(hqre.match(c)) - - -def body_quopri_check(c): - """Return True if the character should be escaped with body quopri.""" - return bool(bqre.match(c)) - - -def header_quopri_len(s): - """Return the length of str when it is encoded with header quopri.""" - count = 0 - for c in s: - if hqre.match(c): - count += 3 - else: - count += 1 - return count - - -def body_quopri_len(str): - """Return the length of str when it is encoded with body quopri.""" - count = 0 - for c in str: - if bqre.match(c): - count += 3 - else: - count += 1 - return count - - -def _max_append(L, s, maxlen, extra=''): - if not L: - L.append(s.lstrip()) - elif len(L[-1]) + len(s) <= maxlen: - L[-1] += extra + s - else: - L.append(s.lstrip()) - - -def unquote(s): - """Turn a string in the form =AB to the ASCII character with value 0xab""" - return chr(int(s[1:3], 16)) - - -def quote(c): - return "=%02X" % ord(c) - - - -def header_encode(header, charset="iso-8859-1", keep_eols=False, - maxlinelen=76, eol=NL): - """Encode a single header line with quoted-printable (like) encoding. - - Defined in RFC 2045, this `Q' encoding is similar to quoted-printable, but - used specifically for email header fields to allow charsets with mostly 7 - bit characters (and some 8 bit) to remain more or less readable in non-RFC - 2045 aware mail clients. - - charset names the character set to use to encode the header. It defaults - to iso-8859-1. - - The resulting string will be in the form: - - "=?charset?q?I_f=E2rt_in_your_g=E8n=E8ral_dire=E7tion?\\n - =?charset?q?Silly_=C8nglish_Kn=EEghts?=" - - with each line wrapped safely at, at most, maxlinelen characters (defaults - to 76 characters). If maxlinelen is None, the entire string is encoded in - one chunk with no splitting. - - End-of-line characters (\\r, \\n, \\r\\n) will be automatically converted - to the canonical email line separator \\r\\n unless the keep_eols - parameter is True (the default is False). - - Each line of the header will be terminated in the value of eol, which - defaults to "\\n". Set this to "\\r\\n" if you are using the result of - this function directly in email. - """ - # Return empty headers unchanged - if not header: - return header - - if not keep_eols: - header = fix_eols(header) - - # Quopri encode each line, in encoded chunks no greater than maxlinelen in - # length, after the RFC chrome is added in. - quoted = [] - if maxlinelen is None: - # An obnoxiously large number that's good enough - max_encoded = 100000 - else: - max_encoded = maxlinelen - len(charset) - MISC_LEN - 1 - - for c in header: - # Space may be represented as _ instead of =20 for readability - if c == ' ': - _max_append(quoted, '_', max_encoded) - # These characters can be included verbatim - elif not hqre.match(c): - _max_append(quoted, c, max_encoded) - # Otherwise, replace with hex value like =E2 - else: - _max_append(quoted, "=%02X" % ord(c), max_encoded) - - # Now add the RFC chrome to each encoded chunk and glue the chunks - # together. BAW: should we be able to specify the leading whitespace in - # the joiner? - joiner = eol + ' ' - return joiner.join(['=?%s?q?%s?=' % (charset, line) for line in quoted]) - - - -def encode(body, binary=False, maxlinelen=76, eol=NL): - """Encode with quoted-printable, wrapping at maxlinelen characters. - - If binary is False (the default), end-of-line characters will be converted - to the canonical email end-of-line sequence \\r\\n. Otherwise they will - be left verbatim. - - Each line of encoded text will end with eol, which defaults to "\\n". Set - this to "\\r\\n" if you will be using the result of this function directly - in an email. - - Each line will be wrapped at, at most, maxlinelen characters (defaults to - 76 characters). Long lines will have the `soft linefeed' quoted-printable - character "=" appended to them, so the decoded text will be identical to - the original text. - """ - if not body: - return body - - if not binary: - body = fix_eols(body) - - # BAW: We're accumulating the body text by string concatenation. That - # can't be very efficient, but I don't have time now to rewrite it. It - # just feels like this algorithm could be more efficient. - encoded_body = '' - lineno = -1 - # Preserve line endings here so we can check later to see an eol needs to - # be added to the output later. - lines = body.splitlines(1) - for line in lines: - # But strip off line-endings for processing this line. - if line.endswith(CRLF): - line = line[:-2] - elif line[-1] in CRLF: - line = line[:-1] - - lineno += 1 - encoded_line = '' - prev = None - linelen = len(line) - # Now we need to examine every character to see if it needs to be - # quopri encoded. BAW: again, string concatenation is inefficient. - for j in range(linelen): - c = line[j] - prev = c - if bqre.match(c): - c = quote(c) - elif j+1 == linelen: - # Check for whitespace at end of line; special case - if c not in ' \t': - encoded_line += c - prev = c - continue - # Check to see to see if the line has reached its maximum length - if len(encoded_line) + len(c) >= maxlinelen: - encoded_body += encoded_line + '=' + eol - encoded_line = '' - encoded_line += c - # Now at end of line.. - if prev and prev in ' \t': - # Special case for whitespace at end of file - if lineno + 1 == len(lines): - prev = quote(prev) - if len(encoded_line) + len(prev) > maxlinelen: - encoded_body += encoded_line + '=' + eol + prev - else: - encoded_body += encoded_line + prev - # Just normal whitespace at end of line - else: - encoded_body += encoded_line + prev + '=' + eol - encoded_line = '' - # Now look at the line we just finished and it has a line ending, we - # need to add eol to the end of the line. - if lines[lineno].endswith(CRLF) or lines[lineno][-1] in CRLF: - encoded_body += encoded_line + eol - else: - encoded_body += encoded_line - encoded_line = '' - return encoded_body - - -# For convenience and backwards compatibility w/ standard base64 module -body_encode = encode -encodestring = encode - - - -# BAW: I'm not sure if the intent was for the signature of this function to be -# the same as base64MIME.decode() or not... -def decode(encoded, eol=NL): - """Decode a quoted-printable string. - - Lines are separated with eol, which defaults to \\n. - """ - if not encoded: - return encoded - # BAW: see comment in encode() above. Again, we're building up the - # decoded string with string concatenation, which could be done much more - # efficiently. - decoded = '' - - for line in encoded.splitlines(): - line = line.rstrip() - if not line: - decoded += eol - continue - - i = 0 - n = len(line) - while i < n: - c = line[i] - if c != '=': - decoded += c - i += 1 - # Otherwise, c == "=". Are we at the end of the line? If so, add - # a soft line break. - elif i+1 == n: - i += 1 - continue - # Decode if in form =AB - elif i+2 < n and line[i+1] in hexdigits and line[i+2] in hexdigits: - decoded += unquote(line[i:i+3]) - i += 3 - # Otherwise, not in form =AB, pass literally - else: - decoded += c - i += 1 - - if i == n: - decoded += eol - # Special case if original string did not end with eol - if not encoded.endswith(eol) and decoded.endswith(eol): - decoded = decoded[:-1] - return decoded - - -# For convenience and backwards compatibility w/ standard base64 module -body_decode = decode -decodestring = decode - - - -def _unquote_match(match): - """Turn a match in the form =AB to the ASCII character with value 0xab""" - s = match.group(0) - return unquote(s) - - -# Header decoding is done a bit differently -def header_decode(s): - """Decode a string encoded with RFC 2045 MIME header `Q' encoding. - - This function does not parse a full MIME header value encoded with - quoted-printable (like =?iso-8895-1?q?Hello_World?=) -- please use - the high level email.header class for that functionality. - """ - s = s.replace('_', ' ') - return re.sub(r'=[a-fA-F0-9]{2}', _unquote_match, s) diff --git a/lib-python/modified-2.7/email/test/__init__.py b/lib-python/modified-2.7/email/test/__init__.py deleted file mode 100644 diff --git a/lib-python/modified-2.7/email/test/data/PyBanner048.gif b/lib-python/modified-2.7/email/test/data/PyBanner048.gif deleted file mode 100644 Binary file lib-python/modified-2.7/email/test/data/PyBanner048.gif has changed diff --git a/lib-python/modified-2.7/email/test/data/audiotest.au b/lib-python/modified-2.7/email/test/data/audiotest.au deleted file mode 100644 Binary file lib-python/modified-2.7/email/test/data/audiotest.au has changed diff --git a/lib-python/modified-2.7/email/test/data/msg_01.txt b/lib-python/modified-2.7/email/test/data/msg_01.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_01.txt +++ /dev/null @@ -1,19 +0,0 @@ -Return-Path: <bbb at zzz.org> -Delivered-To: bbb at zzz.org -Received: by mail.zzz.org (Postfix, from userid 889) - id 27CEAD38CC; Fri, 4 May 2001 14:05:44 -0400 (EDT) -MIME-Version: 1.0 -Content-Type: text/plain; charset=us-ascii -Content-Transfer-Encoding: 7bit -Message-ID: <15090.61304.110929.45684 at aaa.zzz.org> -From: bbb at ddd.com (John X. Doe) -To: bbb at zzz.org -Subject: This is a test message -Date: Fri, 4 May 2001 14:05:44 -0400 - - -Hi, - -Do you like this message? - --Me diff --git a/lib-python/modified-2.7/email/test/data/msg_02.txt b/lib-python/modified-2.7/email/test/data/msg_02.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_02.txt +++ /dev/null @@ -1,135 +0,0 @@ -MIME-version: 1.0 -From: ppp-request at zzz.org -Sender: ppp-admin at zzz.org -To: ppp at zzz.org -Subject: Ppp digest, Vol 1 #2 - 5 msgs -Date: Fri, 20 Apr 2001 20:18:00 -0400 (EDT) -X-Mailer: Mailman v2.0.4 -X-Mailman-Version: 2.0.4 -Content-Type: multipart/mixed; boundary="192.168.1.2.889.32614.987812255.500.21814" - ---192.168.1.2.889.32614.987812255.500.21814 -Content-type: text/plain; charset=us-ascii -Content-description: Masthead (Ppp digest, Vol 1 #2) - -Send Ppp mailing list submissions to - ppp at zzz.org - -To subscribe or unsubscribe via the World Wide Web, visit - http://www.zzz.org/mailman/listinfo/ppp -or, via email, send a message with subject or body 'help' to - ppp-request at zzz.org - -You can reach the person managing the list at - ppp-admin at zzz.org - -When replying, please edit your Subject line so it is more specific -than "Re: Contents of Ppp digest..." - - ---192.168.1.2.889.32614.987812255.500.21814 -Content-type: text/plain; charset=us-ascii -Content-description: Today's Topics (5 msgs) - -Today's Topics: - - 1. testing #1 (Barry A. Warsaw) - 2. testing #2 (Barry A. Warsaw) - 3. testing #3 (Barry A. Warsaw) - 4. testing #4 (Barry A. Warsaw) - 5. testing #5 (Barry A. Warsaw) - ---192.168.1.2.889.32614.987812255.500.21814 -Content-Type: multipart/digest; boundary="__--__--" - ---__--__-- - -Message: 1 -Content-Type: text/plain; charset=us-ascii -Content-Transfer-Encoding: 7bit -Date: Fri, 20 Apr 2001 20:16:13 -0400 -To: ppp at zzz.org -From: barry at digicool.com (Barry A. Warsaw) -Subject: [Ppp] testing #1 -Precedence: bulk - - -hello - - ---__--__-- - -Message: 2 -Date: Fri, 20 Apr 2001 20:16:21 -0400 -Content-Type: text/plain; charset=us-ascii -Content-Transfer-Encoding: 7bit -To: ppp at zzz.org -From: barry at digicool.com (Barry A. Warsaw) -Precedence: bulk - - -hello - - ---__--__-- - -Message: 3 -Date: Fri, 20 Apr 2001 20:16:25 -0400 -Content-Type: text/plain; charset=us-ascii -Content-Transfer-Encoding: 7bit -To: ppp at zzz.org -From: barry at digicool.com (Barry A. Warsaw) -Subject: [Ppp] testing #3 -Precedence: bulk - - -hello - - ---__--__-- - -Message: 4 -Date: Fri, 20 Apr 2001 20:16:28 -0400 -Content-Type: text/plain; charset=us-ascii -Content-Transfer-Encoding: 7bit -To: ppp at zzz.org -From: barry at digicool.com (Barry A. Warsaw) -Subject: [Ppp] testing #4 -Precedence: bulk - - -hello - - ---__--__-- - -Message: 5 -Date: Fri, 20 Apr 2001 20:16:32 -0400 -Content-Type: text/plain; charset=us-ascii -Content-Transfer-Encoding: 7bit -To: ppp at zzz.org -From: barry at digicool.com (Barry A. Warsaw) -Subject: [Ppp] testing #5 -Precedence: bulk - - -hello - - - - ---__--__---- ---192.168.1.2.889.32614.987812255.500.21814 -Content-type: text/plain; charset=us-ascii -Content-description: Digest Footer - -_______________________________________________ -Ppp mailing list -Ppp at zzz.org -http://www.zzz.org/mailman/listinfo/ppp - - ---192.168.1.2.889.32614.987812255.500.21814-- - -End of Ppp Digest - diff --git a/lib-python/modified-2.7/email/test/data/msg_03.txt b/lib-python/modified-2.7/email/test/data/msg_03.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_03.txt +++ /dev/null @@ -1,16 +0,0 @@ -Return-Path: <bbb at zzz.org> -Delivered-To: bbb at zzz.org -Received: by mail.zzz.org (Postfix, from userid 889) - id 27CEAD38CC; Fri, 4 May 2001 14:05:44 -0400 (EDT) -Message-ID: <15090.61304.110929.45684 at aaa.zzz.org> -From: bbb at ddd.com (John X. Doe) -To: bbb at zzz.org -Subject: This is a test message -Date: Fri, 4 May 2001 14:05:44 -0400 - - -Hi, - -Do you like this message? - --Me diff --git a/lib-python/modified-2.7/email/test/data/msg_04.txt b/lib-python/modified-2.7/email/test/data/msg_04.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_04.txt +++ /dev/null @@ -1,37 +0,0 @@ -Return-Path: <barry at python.org> -Delivered-To: barry at python.org -Received: by mail.python.org (Postfix, from userid 889) - id C2BF0D37C6; Tue, 11 Sep 2001 00:05:05 -0400 (EDT) -MIME-Version: 1.0 -Content-Type: multipart/mixed; boundary="h90VIIIKmx" -Content-Transfer-Encoding: 7bit -Message-ID: <15261.36209.358846.118674 at anthem.python.org> -From: barry at python.org (Barry A. Warsaw) -To: barry at python.org -Subject: a simple multipart -Date: Tue, 11 Sep 2001 00:05:05 -0400 -X-Mailer: VM 6.95 under 21.4 (patch 4) "Artificial Intelligence" XEmacs Lucid -X-Attribution: BAW -X-Oblique-Strategy: Make a door into a window - - ---h90VIIIKmx -Content-Type: text/plain -Content-Disposition: inline; - filename="msg.txt" -Content-Transfer-Encoding: 7bit - -a simple kind of mirror -to reflect upon our own - ---h90VIIIKmx -Content-Type: text/plain -Content-Disposition: inline; - filename="msg.txt" -Content-Transfer-Encoding: 7bit - -a simple kind of mirror -to reflect upon our own - ---h90VIIIKmx-- - diff --git a/lib-python/modified-2.7/email/test/data/msg_05.txt b/lib-python/modified-2.7/email/test/data/msg_05.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_05.txt +++ /dev/null @@ -1,28 +0,0 @@ -From: foo -Subject: bar -To: baz -MIME-Version: 1.0 -Content-Type: multipart/report; report-type=delivery-status; - boundary="D1690A7AC1.996856090/mail.example.com" -Message-Id: <20010803162810.0CA8AA7ACC at mail.example.com> - -This is a MIME-encapsulated message. - ---D1690A7AC1.996856090/mail.example.com -Content-Type: text/plain - -Yadda yadda yadda - ---D1690A7AC1.996856090/mail.example.com - -Yadda yadda yadda - ---D1690A7AC1.996856090/mail.example.com -Content-Type: message/rfc822 - -From: nobody at python.org - -Yadda yadda yadda - ---D1690A7AC1.996856090/mail.example.com-- - diff --git a/lib-python/modified-2.7/email/test/data/msg_06.txt b/lib-python/modified-2.7/email/test/data/msg_06.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_06.txt +++ /dev/null @@ -1,33 +0,0 @@ -Return-Path: <barry at python.org> -Delivered-To: barry at python.org -MIME-Version: 1.0 -Content-Type: message/rfc822 -Content-Description: forwarded message -Content-Transfer-Encoding: 7bit -Message-ID: <15265.9482.641338.555352 at python.org> -From: barry at zope.com (Barry A. Warsaw) -Sender: barry at python.org -To: barry at python.org -Subject: forwarded message from Barry A. Warsaw -Date: Thu, 13 Sep 2001 17:28:42 -0400 -X-Mailer: VM 6.95 under 21.4 (patch 4) "Artificial Intelligence" XEmacs Lucid -X-Attribution: BAW -X-Oblique-Strategy: Be dirty -X-Url: http://barry.wooz.org - -MIME-Version: 1.0 -Content-Type: text/plain; charset=us-ascii -Return-Path: <barry at python.org> -Delivered-To: barry at python.org -Message-ID: <15265.9468.713530.98441 at python.org> -From: barry at zope.com (Barry A. Warsaw) -Sender: barry at python.org -To: barry at python.org -Subject: testing -Date: Thu, 13 Sep 2001 17:28:28 -0400 -X-Mailer: VM 6.95 under 21.4 (patch 4) "Artificial Intelligence" XEmacs Lucid -X-Attribution: BAW -X-Oblique-Strategy: Spectrum analysis -X-Url: http://barry.wooz.org - - diff --git a/lib-python/modified-2.7/email/test/data/msg_07.txt b/lib-python/modified-2.7/email/test/data/msg_07.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_07.txt +++ /dev/null @@ -1,83 +0,0 @@ -MIME-Version: 1.0 -From: Barry <barry at digicool.com> -To: Dingus Lovers <cravindogs at cravindogs.com> -Subject: Here is your dingus fish -Date: Fri, 20 Apr 2001 19:35:02 -0400 -Content-Type: multipart/mixed; boundary="BOUNDARY" - ---BOUNDARY -Content-Type: text/plain; charset="us-ascii" - -Hi there, - -This is the dingus fish. - ---BOUNDARY -Content-Type: image/gif; name="dingusfish.gif" -Content-Transfer-Encoding: base64 -content-disposition: attachment; filename="dingusfish.gif" - -R0lGODdhAAEAAfAAAP///wAAACwAAAAAAAEAAQAC/oSPqcvtD6OctNqLs968+w+G4kiW5omm6sq2 -7gvH8kzX9o3n+s73/g8MCofEovGITGICTKbyCV0FDNOo9SqpQqpOrJfXzTQj2vD3TGtqL+NtGQ2f -qTXmxzuOd7WXdcc9DyjU53ewFni4s0fGhdiYaEhGBelICTNoV1j5NUnFcrmUqemjNifJVWpaOqaI -oFq3SspZsSraE7sHq3jr1MZqWvi662vxV4tD+pvKW6aLDOCLyur8PDwbanyDeq0N3DctbQYeLDvR -RY6t95m6UB0d3mwIrV7e2VGNvjjffukeJp4w7F65KecGFsTHQGAygOrgrWs1jt28Rc88KESYcGLA -/obvTkH6p+CinWJiJmIMqXGQwH/y4qk0SYjgQTczT3ajKZGfuI0uJ4kkVI/DT5s3/ejkxI0aT4Y+ -YTYgWbImUaXk9nlLmnSh1qJiJFl0OpUqRK4oOy7NyRQtHWofhoYVxkwWXKUSn0YsS+fUV6lhqfYb -6ayd3Z5qQdG1B7bvQzaJjwUV2lixMUZ7JVsOlfjWVr/3NB/uFvnySBN6Dcb6rGwaRM3wsormw5cC -M9NxWy/bWdufudCvy8bOAjXjVVwta/uO21sE5RHBCzNFXtgq9ORtH4eYjVP4Yryo026nvkFmCeyA -B29efV6ravCMK5JwWd5897Qrx7ll38o6iHDZ/rXPR//feevhF4l7wjUGX3xq1eeRfM4RSJGBIV1D -z1gKPkfWag3mVBVvva1RlX5bAJTPR/2YqNtw/FkIYYEi/pIZiAdpcxpoHtmnYYoZtvhUftzdx5ZX -JSKDW405zkGcZzzGZ6KEv4FI224oDmijlEf+xp6MJK5ojY/ASeVUR+wsKRuJ+XFZ5o7ZeEime8t1 -ouUsU6YjF5ZtUihhkGfCdFQLWQFJ3UXxmElfhQnR+eCdcDbkFZp6vTRmj56ApCihn5QGpaToNZmR -n3NVSpZcQpZ2KEONusaiCsKAug0wkQbJSFO+PTSjneGxOuFjPlUk3ovWvdIerjUg9ZGIOtGq/qeX -eCYrrCX+1UPsgTKGGRSbzd5q156d/gpfbJxe66eD5iQKrXj7RGgruGxs62qebBHUKS32CKluCiqZ -qh+pmehmEb71noAUoe5e9Zm17S7773V10pjrtG4CmuurCV/n6zLK5turWNhqOvFXbjhZrMD0YhKe -wR0zOyuvsh6MWrGoIuzvyWu5y1WIFAqmJselypxXh6dKLNOKEB98L88bS2rkNqqlKzCNJp9c0G0j -Gzh0iRrCbHSXmPR643QS+4rWhgFmnSbSuXCjS0xAOWkU2UdLqyuUNfHSFdUouy3bm5i5GnDM3tG8 -doJ4r5tqu3pPbRSVfvs8uJzeNXhp3n4j/tZ42SwH7eaWUUOjc3qFV9453UHTXZfcLH+OeNs5g36x -lBnHvTm7EbMbLeuaLncao8vWCXimfo1o+843Ak6y4ChNeGntvAYvfLK4ezmoyNIbNCLTCXO9ZV3A -E8/s88RczPzDwI4Ob7XZyl7+9Miban29h+tJZPrE21wgvBphDfrrfPdCTPKJD/y98L1rZwHcV6Jq -Zab0metpuNIX/qAFPoz171WUaUb4HAhBSzHuHfjzHb3kha/2Cctis/ORArVHNYfFyYRH2pYIRzic -isVOfPWD1b6mRTqpCRBozzof6UZVvFXRxWIr3GGrEviGYgyPMfahheiSaLs/9QeFu7oZ/ndSY8DD -ya9x+uPed+7mxN2IzIISBOMLFYWVqC3Pew1T2nFuuCiwZS5/v6II10i4t1OJcUH2U9zxKodHsGGv -Oa+zkvNUYUOa/TCCRutF9MzDwdlUMJADTCGSbDQ5OV4PTamDoPEi6Ecc/RF5RWwkcdSXvSOaDWSn -I9LlvubFTQpuc6JKXLcKeb+xdbKRBnwREemXyjg6ME65aJiOuBgrktzykfPLJBKR9ClMavJ62/Ff -BlNIyod9yX9wcSXexnXFpvkrbXk64xsx5Db7wXKP5fSgsvwIMM/9631VLBfkmtbHRXpqmtei52hG -pUwSlo+BASQoeILDOBgREECxBBh5/iYmNsQ9dIv5+OI++QkqdsJPc3uykz5fkM+OraeekcQF7X4n -B5S67za5U967PmooGQhUXfF7afXyCD7ONdRe17QogYjVx38uLwtrS6nhTnm15LQUnu9E2uK6CNI/ -1HOABj0ESwOjut4FEpFQpdNAm4K2LHnDWHNcmKB2ioKBogysVZtMO2nSxUdZ8Yk2kJc7URioLVI0 -YgmtIwZj4LoeKemgnOnbUdGnzZ4Oa6scqiolBGqS6RgWNLu0RMhcaE6rhhU4hiuqFXPAG8fGwTPW -FKeLMtdVmXLSs5YJGF/YeVm7rREMlY3UYE+yCxbaMXX8y15m5zVHq6GOKDMynzII/jdUHdyVqIy0 -ifX2+r/EgtZcvRzSb72gU9ui87M2VecjKildW/aFqaYhKoryUjfB/g4qtyVuc60xFDGmCxwjW+qu -zjuwl2GkOWn66+3QiiEctvd04OVvcCVzjgT7lrkvjVGKKHmmlDUKowSeikb5kK/mJReuWOxONx+s -ULsl+Lqb0CVn0SrVyJ6wt4t6yTeSCafhPhAf0OXn6L60UMxiLolFAtmN35S2Ob1lZpQ1r/n0Qb5D -oQ1zJiRVDgF8N3Q8TYfbi3DyWCy3lT1nxyBs6FT3S2GOzWRlxwKvlRP0RPJA9SjxEy0UoEnkA+M4 -cnzLMJrBGWLFEaaUb5lvpqbq/loOaU5+DFuHPxo82/OZuM8FXG3oVNZhtWpMpb/0Xu5m/LfLhHZQ -7yuVI0MqZ7NE43imC8jH3IwGZlbPm0xkJYs7+2U48hXTsFSMqgGDvai0kLxyynKNT/waj+q1c1tz -GjOpPBgdCSq3UKZxCSsqFIY+O6JbAWGWcV1pwqLyj5sGqCF1xb1F3varUWqrJv6cN3PrUXzijtfZ -FshpBL3Xwr4GIPvU2N8EjrJgS1zl21rbXQMXeXc5jjFyrhpCzijSv/RQtyPSzHCFMhlME95fHglt -pRsX+dfSQjUeHAlpWzJ5iOo79Ldnaxai6bXTcGO3fp07ri7HLEmXXPlYi8bv/qVxvNcdra6m7Rlb -6JBTb5fd66VhFRjGArh2n7R1rDW4P5NOT9K0I183T2scYkeZ3q/VFyLb09U9ajzXBS8Kgkhc4mBS -kYY9cy3Vy9lUnuNJH8HGIclUilwnBtjUOH0gteGOZ4c/XNrhXLSYDyxfnD8z1pDy7rYRvDolhnbe -UMzxCZUs40s6s7UIvBnLgc0+vKuOkIXeOrDymlp+Zxra4MZLBbVrqD/jTJ597pDmnw5c4+DbyB88 -9Cg9DodYcSuMZT/114pptqc/EuTjRPvH/z5slzI3tluOEBBLqOXLOX+0I5929tO97wkvl/atCz+y -xJrdwteW2FNW/NSmBP+f/maYtVs/bYyBC7Ox3jsYZHL05CIrBa/nS+b3bHfiYm4Ueil1YZZSgAUI -fFZ1dxUmeA2oQRQ3RuGXNGLFV9/XbGFGPV6kfzk1TBBCd+izc7q1H+OHMJwmaBX2IQNYVAKHYepV -SSGCe6CnbYHHETKGNe43EDvFgZr0gB/nVHPHZ80VV1ojOiI3XDvYIkl4ayo4bxQIgrFXWTvBI0nH -VElWMuw2aLUWCRHHf8ymVCHjFlJnOSojfevCYyyyZDH0IcvHhrsnQ5O1OsWzONuVVKIxSxiFZ/tR -fKDAf6xFTnw4O9Qig2VCfW2hJQrmMOuHW0W3dLQmCMO2ccdUd/xyfflH/olTiHZVdGwb8nIwRzSE -J15jFlOJuBZBZ4CiyHyd2IFylFlB+HgHhYabhWOGwYO1ZH/Og1dtQlFMk352CGRSIFTapnWQEUtN -l4zv8S0aaCFDyGCBqDUxZYpxGHX01y/JuH1xhn7TOCnNCI4eKDs5WGX4R425F4vF1o3BJ4vO0otq -I3rimI7jJY1jISqnBxknCIvruF83mF5wN4X7qGLIhR8A2Vg0yFERSIXn9Vv3GHy3Vj/WIkKddlYi -yIMv2I/VMjTLpW7pt05SWIZR0RPyxpB4SIUM9lBPGBl0GC7oSEEwRYLe4pJpZY2P0zbI1n+Oc44w -qY3PUnmF0ixjVpDD/mJ9wpOBGTVgXlaCaZiPcIWK5NiKBIiPdGaQ0TWGvAiG7nMchdZb7Vgf8zNi -MuMyzRdy/lePe9iC4TRx7WhhOQI/QiSVNAmAa2lT/piFbuh7ofJoYSZzrSZ1bvmWw3eN2nKUPVky -uPN5/VRfohRd0VYZoqhKIlU6TXYhJxmPUIloAwc1bPmHEpaZYZORHNlXUJM07hATwHR8MJYqkwWR -WaIezFhxSFlc8/Fq82hEnpeRozg3ULhhr9lAGtVEkCg5ZNRuuVleBPaZadhG0ZgkyPmDOTOKzViM -YgOcpukKqQcbjAWS0IleQ2ROjdh6A+md1qWdBRSX7iSYgFRTtRmBpJioieXJiHfJiMGIR9fJOn8I -MSfXYhspn4ooSa2mSAj4n+8Bmg03fBJZoPOJgsVZRxu1oOMRPXYYjdqjihFaEoZpXBREanuJoRI6 -cibFinq4ngUKh/wQd/H5ofYCZ0HJXR62opZFaAT0iFIZo4DIiUojkjeqKiuoZirKo5Y1a7AWckGa -BkuYoD5lpDK6eUs6CkDqpETwl1EqpfhJpVeKpVl6EgUAADs= - ---BOUNDARY-- diff --git a/lib-python/modified-2.7/email/test/data/msg_08.txt b/lib-python/modified-2.7/email/test/data/msg_08.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_08.txt +++ /dev/null @@ -1,24 +0,0 @@ -MIME-Version: 1.0 -From: Barry Warsaw <barry at zope.com> -To: Dingus Lovers <cravindogs at cravindogs.com> -Subject: Lyrics -Date: Fri, 20 Apr 2001 19:35:02 -0400 -Content-Type: multipart/mixed; boundary="BOUNDARY" - ---BOUNDARY -Content-Type: text/plain; charset="us-ascii" - - ---BOUNDARY -Content-Type: text/html; charset="iso-8859-1" - - ---BOUNDARY -Content-Type: text/plain; charset="iso-8859-2" - - ---BOUNDARY -Content-Type: text/plain; charset="koi8-r" - - ---BOUNDARY-- diff --git a/lib-python/modified-2.7/email/test/data/msg_09.txt b/lib-python/modified-2.7/email/test/data/msg_09.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_09.txt +++ /dev/null @@ -1,24 +0,0 @@ -MIME-Version: 1.0 -From: Barry Warsaw <barry at zope.com> -To: Dingus Lovers <cravindogs at cravindogs.com> -Subject: Lyrics -Date: Fri, 20 Apr 2001 19:35:02 -0400 -Content-Type: multipart/mixed; boundary="BOUNDARY" - ---BOUNDARY -Content-Type: text/plain; charset="us-ascii" - - ---BOUNDARY -Content-Type: text/html; charset="iso-8859-1" - - ---BOUNDARY -Content-Type: text/plain - - ---BOUNDARY -Content-Type: text/plain; charset="koi8-r" - - ---BOUNDARY-- diff --git a/lib-python/modified-2.7/email/test/data/msg_10.txt b/lib-python/modified-2.7/email/test/data/msg_10.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_10.txt +++ /dev/null @@ -1,39 +0,0 @@ -MIME-Version: 1.0 -From: Barry Warsaw <barry at zope.com> -To: Dingus Lovers <cravindogs at cravindogs.com> -Subject: Lyrics -Date: Fri, 20 Apr 2001 19:35:02 -0400 -Content-Type: multipart/mixed; boundary="BOUNDARY" - ---BOUNDARY -Content-Type: text/plain; charset="us-ascii" -Content-Transfer-Encoding: 7bit - -This is a 7bit encoded message. - ---BOUNDARY -Content-Type: text/html; charset="iso-8859-1" -Content-Transfer-Encoding: Quoted-Printable - -=A1This is a Quoted Printable encoded message! - ---BOUNDARY -Content-Type: text/plain; charset="iso-8859-1" -Content-Transfer-Encoding: Base64 - -VGhpcyBpcyBhIEJhc2U2NCBlbmNvZGVkIG1lc3NhZ2Uu - - ---BOUNDARY -Content-Type: text/plain; charset="iso-8859-1" -Content-Transfer-Encoding: Base64 - -VGhpcyBpcyBhIEJhc2U2NCBlbmNvZGVkIG1lc3NhZ2UuCg== - - ---BOUNDARY -Content-Type: text/plain; charset="iso-8859-1" - -This has no Content-Transfer-Encoding: header. - ---BOUNDARY-- diff --git a/lib-python/modified-2.7/email/test/data/msg_11.txt b/lib-python/modified-2.7/email/test/data/msg_11.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_11.txt +++ /dev/null @@ -1,7 +0,0 @@ -Content-Type: message/rfc822 -MIME-Version: 1.0 -Subject: The enclosing message - -Subject: An enclosed message - -Here is the body of the message. diff --git a/lib-python/modified-2.7/email/test/data/msg_12.txt b/lib-python/modified-2.7/email/test/data/msg_12.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_12.txt +++ /dev/null @@ -1,36 +0,0 @@ -MIME-Version: 1.0 -From: Barry Warsaw <barry at zope.com> -To: Dingus Lovers <cravindogs at cravindogs.com> -Subject: Lyrics -Date: Fri, 20 Apr 2001 19:35:02 -0400 -Content-Type: multipart/mixed; boundary="BOUNDARY" - ---BOUNDARY -Content-Type: text/plain; charset="us-ascii" - - ---BOUNDARY -Content-Type: text/html; charset="iso-8859-1" - - ---BOUNDARY -Content-Type: multipart/mixed; boundary="ANOTHER" - ---ANOTHER -Content-Type: text/plain; charset="iso-8859-2" - - ---ANOTHER -Content-Type: text/plain; charset="iso-8859-3" - ---ANOTHER-- - ---BOUNDARY -Content-Type: text/plain; charset="us-ascii" - - ---BOUNDARY -Content-Type: text/plain; charset="koi8-r" - - ---BOUNDARY-- diff --git a/lib-python/modified-2.7/email/test/data/msg_12a.txt b/lib-python/modified-2.7/email/test/data/msg_12a.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_12a.txt +++ /dev/null @@ -1,38 +0,0 @@ -MIME-Version: 1.0 -From: Barry Warsaw <barry at zope.com> -To: Dingus Lovers <cravindogs at cravindogs.com> -Subject: Lyrics -Date: Fri, 20 Apr 2001 19:35:02 -0400 -Content-Type: multipart/mixed; boundary="BOUNDARY" - ---BOUNDARY -Content-Type: text/plain; charset="us-ascii" - - ---BOUNDARY -Content-Type: text/html; charset="iso-8859-1" - - ---BOUNDARY -Content-Type: multipart/mixed; boundary="ANOTHER" - ---ANOTHER -Content-Type: text/plain; charset="iso-8859-2" - - ---ANOTHER -Content-Type: text/plain; charset="iso-8859-3" - - ---ANOTHER-- - - ---BOUNDARY -Content-Type: text/plain; charset="us-ascii" - - ---BOUNDARY -Content-Type: text/plain; charset="koi8-r" - - ---BOUNDARY-- diff --git a/lib-python/modified-2.7/email/test/data/msg_13.txt b/lib-python/modified-2.7/email/test/data/msg_13.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_13.txt +++ /dev/null @@ -1,94 +0,0 @@ -MIME-Version: 1.0 -From: Barry <barry at digicool.com> -To: Dingus Lovers <cravindogs at cravindogs.com> -Subject: Here is your dingus fish -Date: Fri, 20 Apr 2001 19:35:02 -0400 -Content-Type: multipart/mixed; boundary="OUTER" - ---OUTER -Content-Type: text/plain; charset="us-ascii" - -A text/plain part - ---OUTER -Content-Type: multipart/mixed; boundary=BOUNDARY - - ---BOUNDARY -Content-Type: text/plain; charset="us-ascii" - -Hi there, - -This is the dingus fish. - ---BOUNDARY -Content-Type: image/gif; name="dingusfish.gif" -Content-Transfer-Encoding: base64 -content-disposition: attachment; filename="dingusfish.gif" - -R0lGODdhAAEAAfAAAP///wAAACwAAAAAAAEAAQAC/oSPqcvtD6OctNqLs968+w+G4kiW5omm6sq2 -7gvH8kzX9o3n+s73/g8MCofEovGITGICTKbyCV0FDNOo9SqpQqpOrJfXzTQj2vD3TGtqL+NtGQ2f -qTXmxzuOd7WXdcc9DyjU53ewFni4s0fGhdiYaEhGBelICTNoV1j5NUnFcrmUqemjNifJVWpaOqaI -oFq3SspZsSraE7sHq3jr1MZqWvi662vxV4tD+pvKW6aLDOCLyur8PDwbanyDeq0N3DctbQYeLDvR -RY6t95m6UB0d3mwIrV7e2VGNvjjffukeJp4w7F65KecGFsTHQGAygOrgrWs1jt28Rc88KESYcGLA -/obvTkH6p+CinWJiJmIMqXGQwH/y4qk0SYjgQTczT3ajKZGfuI0uJ4kkVI/DT5s3/ejkxI0aT4Y+ -YTYgWbImUaXk9nlLmnSh1qJiJFl0OpUqRK4oOy7NyRQtHWofhoYVxkwWXKUSn0YsS+fUV6lhqfYb -6ayd3Z5qQdG1B7bvQzaJjwUV2lixMUZ7JVsOlfjWVr/3NB/uFvnySBN6Dcb6rGwaRM3wsormw5cC -M9NxWy/bWdufudCvy8bOAjXjVVwta/uO21sE5RHBCzNFXtgq9ORtH4eYjVP4Yryo026nvkFmCeyA -B29efV6ravCMK5JwWd5897Qrx7ll38o6iHDZ/rXPR//feevhF4l7wjUGX3xq1eeRfM4RSJGBIV1D -z1gKPkfWag3mVBVvva1RlX5bAJTPR/2YqNtw/FkIYYEi/pIZiAdpcxpoHtmnYYoZtvhUftzdx5ZX -JSKDW405zkGcZzzGZ6KEv4FI224oDmijlEf+xp6MJK5ojY/ASeVUR+wsKRuJ+XFZ5o7ZeEime8t1 -ouUsU6YjF5ZtUihhkGfCdFQLWQFJ3UXxmElfhQnR+eCdcDbkFZp6vTRmj56ApCihn5QGpaToNZmR -n3NVSpZcQpZ2KEONusaiCsKAug0wkQbJSFO+PTSjneGxOuFjPlUk3ovWvdIerjUg9ZGIOtGq/qeX -eCYrrCX+1UPsgTKGGRSbzd5q156d/gpfbJxe66eD5iQKrXj7RGgruGxs62qebBHUKS32CKluCiqZ -qh+pmehmEb71noAUoe5e9Zm17S7773V10pjrtG4CmuurCV/n6zLK5turWNhqOvFXbjhZrMD0YhKe -wR0zOyuvsh6MWrGoIuzvyWu5y1WIFAqmJselypxXh6dKLNOKEB98L88bS2rkNqqlKzCNJp9c0G0j -Gzh0iRrCbHSXmPR643QS+4rWhgFmnSbSuXCjS0xAOWkU2UdLqyuUNfHSFdUouy3bm5i5GnDM3tG8 -doJ4r5tqu3pPbRSVfvs8uJzeNXhp3n4j/tZ42SwH7eaWUUOjc3qFV9453UHTXZfcLH+OeNs5g36x -lBnHvTm7EbMbLeuaLncao8vWCXimfo1o+843Ak6y4ChNeGntvAYvfLK4ezmoyNIbNCLTCXO9ZV3A -E8/s88RczPzDwI4Ob7XZyl7+9Miban29h+tJZPrE21wgvBphDfrrfPdCTPKJD/y98L1rZwHcV6Jq -Zab0metpuNIX/qAFPoz171WUaUb4HAhBSzHuHfjzHb3kha/2Cctis/ORArVHNYfFyYRH2pYIRzic -isVOfPWD1b6mRTqpCRBozzof6UZVvFXRxWIr3GGrEviGYgyPMfahheiSaLs/9QeFu7oZ/ndSY8DD -ya9x+uPed+7mxN2IzIISBOMLFYWVqC3Pew1T2nFuuCiwZS5/v6II10i4t1OJcUH2U9zxKodHsGGv -Oa+zkvNUYUOa/TCCRutF9MzDwdlUMJADTCGSbDQ5OV4PTamDoPEi6Ecc/RF5RWwkcdSXvSOaDWSn -I9LlvubFTQpuc6JKXLcKeb+xdbKRBnwREemXyjg6ME65aJiOuBgrktzykfPLJBKR9ClMavJ62/Ff -BlNIyod9yX9wcSXexnXFpvkrbXk64xsx5Db7wXKP5fSgsvwIMM/9631VLBfkmtbHRXpqmtei52hG -pUwSlo+BASQoeILDOBgREECxBBh5/iYmNsQ9dIv5+OI++QkqdsJPc3uykz5fkM+OraeekcQF7X4n -B5S67za5U967PmooGQhUXfF7afXyCD7ONdRe17QogYjVx38uLwtrS6nhTnm15LQUnu9E2uK6CNI/ -1HOABj0ESwOjut4FEpFQpdNAm4K2LHnDWHNcmKB2ioKBogysVZtMO2nSxUdZ8Yk2kJc7URioLVI0 -YgmtIwZj4LoeKemgnOnbUdGnzZ4Oa6scqiolBGqS6RgWNLu0RMhcaE6rhhU4hiuqFXPAG8fGwTPW -FKeLMtdVmXLSs5YJGF/YeVm7rREMlY3UYE+yCxbaMXX8y15m5zVHq6GOKDMynzII/jdUHdyVqIy0 -ifX2+r/EgtZcvRzSb72gU9ui87M2VecjKildW/aFqaYhKoryUjfB/g4qtyVuc60xFDGmCxwjW+qu -zjuwl2GkOWn66+3QiiEctvd04OVvcCVzjgT7lrkvjVGKKHmmlDUKowSeikb5kK/mJReuWOxONx+s -ULsl+Lqb0CVn0SrVyJ6wt4t6yTeSCafhPhAf0OXn6L60UMxiLolFAtmN35S2Ob1lZpQ1r/n0Qb5D -oQ1zJiRVDgF8N3Q8TYfbi3DyWCy3lT1nxyBs6FT3S2GOzWRlxwKvlRP0RPJA9SjxEy0UoEnkA+M4 -cnzLMJrBGWLFEaaUb5lvpqbq/loOaU5+DFuHPxo82/OZuM8FXG3oVNZhtWpMpb/0Xu5m/LfLhHZQ -7yuVI0MqZ7NE43imC8jH3IwGZlbPm0xkJYs7+2U48hXTsFSMqgGDvai0kLxyynKNT/waj+q1c1tz -GjOpPBgdCSq3UKZxCSsqFIY+O6JbAWGWcV1pwqLyj5sGqCF1xb1F3varUWqrJv6cN3PrUXzijtfZ -FshpBL3Xwr4GIPvU2N8EjrJgS1zl21rbXQMXeXc5jjFyrhpCzijSv/RQtyPSzHCFMhlME95fHglt -pRsX+dfSQjUeHAlpWzJ5iOo79Ldnaxai6bXTcGO3fp07ri7HLEmXXPlYi8bv/qVxvNcdra6m7Rlb -6JBTb5fd66VhFRjGArh2n7R1rDW4P5NOT9K0I183T2scYkeZ3q/VFyLb09U9ajzXBS8Kgkhc4mBS -kYY9cy3Vy9lUnuNJH8HGIclUilwnBtjUOH0gteGOZ4c/XNrhXLSYDyxfnD8z1pDy7rYRvDolhnbe -UMzxCZUs40s6s7UIvBnLgc0+vKuOkIXeOrDymlp+Zxra4MZLBbVrqD/jTJ597pDmnw5c4+DbyB88 -9Cg9DodYcSuMZT/114pptqc/EuTjRPvH/z5slzI3tluOEBBLqOXLOX+0I5929tO97wkvl/atCz+y -xJrdwteW2FNW/NSmBP+f/maYtVs/bYyBC7Ox3jsYZHL05CIrBa/nS+b3bHfiYm4Ueil1YZZSgAUI -fFZ1dxUmeA2oQRQ3RuGXNGLFV9/XbGFGPV6kfzk1TBBCd+izc7q1H+OHMJwmaBX2IQNYVAKHYepV -SSGCe6CnbYHHETKGNe43EDvFgZr0gB/nVHPHZ80VV1ojOiI3XDvYIkl4ayo4bxQIgrFXWTvBI0nH -VElWMuw2aLUWCRHHf8ymVCHjFlJnOSojfevCYyyyZDH0IcvHhrsnQ5O1OsWzONuVVKIxSxiFZ/tR -fKDAf6xFTnw4O9Qig2VCfW2hJQrmMOuHW0W3dLQmCMO2ccdUd/xyfflH/olTiHZVdGwb8nIwRzSE -J15jFlOJuBZBZ4CiyHyd2IFylFlB+HgHhYabhWOGwYO1ZH/Og1dtQlFMk352CGRSIFTapnWQEUtN -l4zv8S0aaCFDyGCBqDUxZYpxGHX01y/JuH1xhn7TOCnNCI4eKDs5WGX4R425F4vF1o3BJ4vO0otq -I3rimI7jJY1jISqnBxknCIvruF83mF5wN4X7qGLIhR8A2Vg0yFERSIXn9Vv3GHy3Vj/WIkKddlYi -yIMv2I/VMjTLpW7pt05SWIZR0RPyxpB4SIUM9lBPGBl0GC7oSEEwRYLe4pJpZY2P0zbI1n+Oc44w -qY3PUnmF0ixjVpDD/mJ9wpOBGTVgXlaCaZiPcIWK5NiKBIiPdGaQ0TWGvAiG7nMchdZb7Vgf8zNi -MuMyzRdy/lePe9iC4TRx7WhhOQI/QiSVNAmAa2lT/piFbuh7ofJoYSZzrSZ1bvmWw3eN2nKUPVky -uPN5/VRfohRd0VYZoqhKIlU6TXYhJxmPUIloAwc1bPmHEpaZYZORHNlXUJM07hATwHR8MJYqkwWR -WaIezFhxSFlc8/Fq82hEnpeRozg3ULhhr9lAGtVEkCg5ZNRuuVleBPaZadhG0ZgkyPmDOTOKzViM -YgOcpukKqQcbjAWS0IleQ2ROjdh6A+md1qWdBRSX7iSYgFRTtRmBpJioieXJiHfJiMGIR9fJOn8I -MSfXYhspn4ooSa2mSAj4n+8Bmg03fBJZoPOJgsVZRxu1oOMRPXYYjdqjihFaEoZpXBREanuJoRI6 -cibFinq4ngUKh/wQd/H5ofYCZ0HJXR62opZFaAT0iFIZo4DIiUojkjeqKiuoZirKo5Y1a7AWckGa -BkuYoD5lpDK6eUs6CkDqpETwl1EqpfhJpVeKpVl6EgUAADs= - ---BOUNDARY-- - ---OUTER-- diff --git a/lib-python/modified-2.7/email/test/data/msg_14.txt b/lib-python/modified-2.7/email/test/data/msg_14.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_14.txt +++ /dev/null @@ -1,23 +0,0 @@ -Return-Path: <bbb at zzz.org> -Delivered-To: bbb at zzz.org -Received: by mail.zzz.org (Postfix, from userid 889) - id 27CEAD38CC; Fri, 4 May 2001 14:05:44 -0400 (EDT) -MIME-Version: 1.0 -Content-Type: text; charset=us-ascii -Content-Transfer-Encoding: 7bit -Message-ID: <15090.61304.110929.45684 at aaa.zzz.org> -From: bbb at ddd.com (John X. Doe) -To: bbb at zzz.org -Subject: This is a test message -Date: Fri, 4 May 2001 14:05:44 -0400 - - -Hi, - -I'm sorry but I'm using a drainbread ISP, which although big and -wealthy can't seem to generate standard compliant email. :( - -This message has a Content-Type: header with no subtype. I hope you -can still read it. - --Me diff --git a/lib-python/modified-2.7/email/test/data/msg_15.txt b/lib-python/modified-2.7/email/test/data/msg_15.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_15.txt +++ /dev/null @@ -1,52 +0,0 @@ -Return-Path: <xx at xx.dk> -Received: from fepD.post.tele.dk (195.41.46.149) by mail.groupcare.dk (LSMTP for Windows NT v1.1b) with SMTP id <0.0014F8A2 at mail.groupcare.dk>; Mon, 30 Apr 2001 12:17:50 +0200 -User-Agent: Microsoft-Outlook-Express-Macintosh-Edition/5.02.2106 -Subject: XX -From: xx at xx.dk -To: XX -Message-ID: <xxxx> -Mime-version: 1.0 -Content-type: multipart/mixed; - boundary="MS_Mac_OE_3071477847_720252_MIME_Part" - -> Denne meddelelse er i MIME-format. Da dit postl�sningsprogram ikke forst�r dette format, kan del af eller hele meddelelsen v�re ul�selig. - ---MS_Mac_OE_3071477847_720252_MIME_Part -Content-type: multipart/alternative; - boundary="MS_Mac_OE_3071477847_720252_MIME_Part" - - ---MS_Mac_OE_3071477847_720252_MIME_Part -Content-type: text/plain; charset="ISO-8859-1" -Content-transfer-encoding: quoted-printable - -Some removed test. - ---MS_Mac_OE_3071477847_720252_MIME_Part -Content-type: text/html; charset="ISO-8859-1" -Content-transfer-encoding: quoted-printable - -<HTML> -<HEAD> -<TITLE>Some removed HTML</TITLE> -</HEAD> -<BODY> -Some removed text. -</BODY> -</HTML> - - ---MS_Mac_OE_3071477847_720252_MIME_Part-- - - ---MS_Mac_OE_3071477847_720252_MIME_Part -Content-type: image/gif; name="xx.gif"; - x-mac-creator="6F676C65"; - x-mac-type="47494666" -Content-disposition: attachment -Content-transfer-encoding: base64 - -Some removed base64 encoded chars. - ---MS_Mac_OE_3071477847_720252_MIME_Part-- - diff --git a/lib-python/modified-2.7/email/test/data/msg_16.txt b/lib-python/modified-2.7/email/test/data/msg_16.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_16.txt +++ /dev/null @@ -1,123 +0,0 @@ -Return-Path: <> -Delivered-To: scr-admin at socal-raves.org -Received: from cougar.noc.ucla.edu (cougar.noc.ucla.edu [169.232.10.18]) - by babylon.socal-raves.org (Postfix) with ESMTP id CCC2C51B84 - for <scr-admin at socal-raves.org>; Sun, 23 Sep 2001 20:13:54 -0700 (PDT) -Received: from sims-ms-daemon by cougar.noc.ucla.edu - (Sun Internet Mail Server sims.3.5.2000.03.23.18.03.p10) - id <0GK500B01D0B8Y at cougar.noc.ucla.edu> for scr-admin at socal-raves.org; Sun, - 23 Sep 2001 20:14:35 -0700 (PDT) -Received: from cougar.noc.ucla.edu - (Sun Internet Mail Server sims.3.5.2000.03.23.18.03.p10) - id <0GK500B01D0B8X at cougar.noc.ucla.edu>; Sun, 23 Sep 2001 20:14:35 -0700 (PDT) -Date: Sun, 23 Sep 2001 20:14:35 -0700 (PDT) -From: Internet Mail Delivery <postmaster at ucla.edu> -Subject: Delivery Notification: Delivery has failed -To: scr-admin at socal-raves.org -Message-id: <0GK500B04D0B8X at cougar.noc.ucla.edu> -MIME-version: 1.0 -Sender: scr-owner at socal-raves.org -Errors-To: scr-owner at socal-raves.org -X-BeenThere: scr at socal-raves.org -X-Mailman-Version: 2.1a3 -Precedence: bulk -List-Help: <mailto:scr-request at socal-raves.org?subject=help> -List-Post: <mailto:scr at socal-raves.org> -List-Subscribe: <http://socal-raves.org/mailman/listinfo/scr>, - <mailto:scr-request at socal-raves.org?subject=subscribe> -List-Id: SoCal-Raves <scr.socal-raves.org> -List-Unsubscribe: <http://socal-raves.org/mailman/listinfo/scr>, - <mailto:scr-request at socal-raves.org?subject=unsubscribe> -List-Archive: <http://socal-raves.org/mailman/private/scr/> -Content-Type: multipart/report; boundary="Boundary_(ID_PGS2F2a+z+/jL7hupKgRhA)" - - ---Boundary_(ID_PGS2F2a+z+/jL7hupKgRhA) -Content-type: text/plain; charset=ISO-8859-1 - -This report relates to a message you sent with the following header fields: - - Message-id: <002001c144a6$8752e060$56104586 at oxy.edu> - Date: Sun, 23 Sep 2001 20:10:55 -0700 - From: "Ian T. Henry" <henryi at oxy.edu> - To: SoCal Raves <scr at socal-raves.org> - Subject: [scr] yeah for Ians!! - -Your message cannot be delivered to the following recipients: - - Recipient address: jangel1 at cougar.noc.ucla.edu - Reason: recipient reached disk quota - - ---Boundary_(ID_PGS2F2a+z+/jL7hupKgRhA) -Content-type: message/DELIVERY-STATUS - -Original-envelope-id: 0GK500B4HD0888 at cougar.noc.ucla.edu -Reporting-MTA: dns; cougar.noc.ucla.edu - -Action: failed -Status: 5.0.0 (recipient reached disk quota) -Original-recipient: rfc822;jangel1 at cougar.noc.ucla.edu -Final-recipient: rfc822;jangel1 at cougar.noc.ucla.edu - ---Boundary_(ID_PGS2F2a+z+/jL7hupKgRhA) -Content-type: MESSAGE/RFC822 - -Return-path: scr-admin at socal-raves.org -Received: from sims-ms-daemon by cougar.noc.ucla.edu - (Sun Internet Mail Server sims.3.5.2000.03.23.18.03.p10) - id <0GK500B01D0B8X at cougar.noc.ucla.edu>; Sun, 23 Sep 2001 20:14:35 -0700 (PDT) -Received: from panther.noc.ucla.edu by cougar.noc.ucla.edu - (Sun Internet Mail Server sims.3.5.2000.03.23.18.03.p10) - with ESMTP id <0GK500B4GD0888 at cougar.noc.ucla.edu> for jangel1 at sims-ms-daemon; - Sun, 23 Sep 2001 20:14:33 -0700 (PDT) -Received: from babylon.socal-raves.org - (ip-209-85-222-117.dreamhost.com [209.85.222.117]) - by panther.noc.ucla.edu (8.9.1a/8.9.1) with ESMTP id UAA09793 for - <jangel1 at ucla.edu>; Sun, 23 Sep 2001 20:14:32 -0700 (PDT) -Received: from babylon (localhost [127.0.0.1]) by babylon.socal-raves.org - (Postfix) with ESMTP id D3B2951B70; Sun, 23 Sep 2001 20:13:47 -0700 (PDT) -Received: by babylon.socal-raves.org (Postfix, from userid 60001) - id A611F51B82; Sun, 23 Sep 2001 20:13:46 -0700 (PDT) -Received: from tiger.cc.oxy.edu (tiger.cc.oxy.edu [134.69.3.112]) - by babylon.socal-raves.org (Postfix) with ESMTP id ADA7351B70 for - <scr at socal-raves.org>; Sun, 23 Sep 2001 20:13:44 -0700 (PDT) -Received: from ent (n16h86.dhcp.oxy.edu [134.69.16.86]) - by tiger.cc.oxy.edu (8.8.8/8.8.8) with SMTP id UAA08100 for - <scr at socal-raves.org>; Sun, 23 Sep 2001 20:14:24 -0700 (PDT) -Date: Sun, 23 Sep 2001 20:10:55 -0700 -From: "Ian T. Henry" <henryi at oxy.edu> -Subject: [scr] yeah for Ians!! -Sender: scr-admin at socal-raves.org -To: SoCal Raves <scr at socal-raves.org> -Errors-to: scr-admin at socal-raves.org -Message-id: <002001c144a6$8752e060$56104586 at oxy.edu> -MIME-version: 1.0 -X-Mailer: Microsoft Outlook Express 5.50.4522.1200 -Content-type: text/plain; charset=us-ascii -Precedence: bulk -Delivered-to: scr-post at babylon.socal-raves.org -Delivered-to: scr at socal-raves.org -X-Converted-To-Plain-Text: from multipart/alternative by demime 0.98e -X-Converted-To-Plain-Text: Alternative section used was text/plain -X-BeenThere: scr at socal-raves.org -X-Mailman-Version: 2.1a3 -List-Help: <mailto:scr-request at socal-raves.org?subject=help> -List-Post: <mailto:scr at socal-raves.org> -List-Subscribe: <http://socal-raves.org/mailman/listinfo/scr>, - <mailto:scr-request at socal-raves.org?subject=subscribe> -List-Id: SoCal-Raves <scr.socal-raves.org> -List-Unsubscribe: <http://socal-raves.org/mailman/listinfo/scr>, - <mailto:scr-request at socal-raves.org?subject=unsubscribe> -List-Archive: <http://socal-raves.org/mailman/private/scr/> - -I always love to find more Ian's that are over 3 years old!! - -Ian -_______________________________________________ -For event info, list questions, or to unsubscribe, see http://www.socal-raves.org/ - - - ---Boundary_(ID_PGS2F2a+z+/jL7hupKgRhA)-- - diff --git a/lib-python/modified-2.7/email/test/data/msg_17.txt b/lib-python/modified-2.7/email/test/data/msg_17.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_17.txt +++ /dev/null @@ -1,12 +0,0 @@ -MIME-Version: 1.0 -From: Barry <barry at digicool.com> -To: Dingus Lovers <cravindogs at cravindogs.com> -Subject: Here is your dingus fish -Date: Fri, 20 Apr 2001 19:35:02 -0400 -Content-Type: multipart/mixed; boundary="BOUNDARY" - -Hi there, - -This is the dingus fish. - -[Non-text (image/gif) part of message omitted, filename dingusfish.gif] diff --git a/lib-python/modified-2.7/email/test/data/msg_18.txt b/lib-python/modified-2.7/email/test/data/msg_18.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_18.txt +++ /dev/null @@ -1,6 +0,0 @@ -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit -X-Foobar-Spoink-Defrobnit: wasnipoop; giraffes="very-long-necked-animals"; - spooge="yummy"; hippos="gargantuan"; marshmallows="gooey" - diff --git a/lib-python/modified-2.7/email/test/data/msg_19.txt b/lib-python/modified-2.7/email/test/data/msg_19.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_19.txt +++ /dev/null @@ -1,43 +0,0 @@ -Send Ppp mailing list submissions to - ppp at zzz.org - -To subscribe or unsubscribe via the World Wide Web, visit - http://www.zzz.org/mailman/listinfo/ppp -or, via email, send a message with subject or body 'help' to - ppp-request at zzz.org - -You can reach the person managing the list at - ppp-admin at zzz.org - -When replying, please edit your Subject line so it is more specific -than "Re: Contents of Ppp digest..." - -Today's Topics: - - 1. testing #1 (Barry A. Warsaw) - 2. testing #2 (Barry A. Warsaw) - 3. testing #3 (Barry A. Warsaw) - 4. testing #4 (Barry A. Warsaw) - 5. testing #5 (Barry A. Warsaw) - -hello - - -hello - - -hello - - -hello - - -hello - - - -_______________________________________________ -Ppp mailing list -Ppp at zzz.org -http://www.zzz.org/mailman/listinfo/ppp - diff --git a/lib-python/modified-2.7/email/test/data/msg_20.txt b/lib-python/modified-2.7/email/test/data/msg_20.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_20.txt +++ /dev/null @@ -1,22 +0,0 @@ -Return-Path: <bbb at zzz.org> -Delivered-To: bbb at zzz.org -Received: by mail.zzz.org (Postfix, from userid 889) - id 27CEAD38CC; Fri, 4 May 2001 14:05:44 -0400 (EDT) -MIME-Version: 1.0 -Content-Type: text/plain; charset=us-ascii -Content-Transfer-Encoding: 7bit -Message-ID: <15090.61304.110929.45684 at aaa.zzz.org> -From: bbb at ddd.com (John X. Doe) -To: bbb at zzz.org -Cc: ccc at zzz.org -CC: ddd at zzz.org -cc: eee at zzz.org -Subject: This is a test message -Date: Fri, 4 May 2001 14:05:44 -0400 - - -Hi, - -Do you like this message? - --Me diff --git a/lib-python/modified-2.7/email/test/data/msg_21.txt b/lib-python/modified-2.7/email/test/data/msg_21.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_21.txt +++ /dev/null @@ -1,20 +0,0 @@ -From: aperson at dom.ain -To: bperson at dom.ain -Subject: Test -Content-Type: multipart/mixed; boundary="BOUNDARY" - -MIME message ---BOUNDARY -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit - -One ---BOUNDARY -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit - -Two ---BOUNDARY-- -End of MIME message diff --git a/lib-python/modified-2.7/email/test/data/msg_22.txt b/lib-python/modified-2.7/email/test/data/msg_22.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_22.txt +++ /dev/null @@ -1,46 +0,0 @@ -Mime-Version: 1.0 -Message-Id: <a05001902b7f1c33773e9@[134.84.183.138]> -Date: Tue, 16 Oct 2001 13:59:25 +0300 -To: a at example.com -From: b at example.com -Content-Type: multipart/mixed; boundary="============_-1208892523==_============" - ---============_-1208892523==_============ -Content-Type: text/plain; charset="us-ascii" ; format="flowed" - -Text text text. ---============_-1208892523==_============ -Content-Id: <a05001902b7f1c33773e9@[134.84.183.138].0.0> -Content-Type: image/jpeg; name="wibble.JPG" - ; x-mac-type="4A504547" - ; x-mac-creator="474B4F4E" -Content-Disposition: attachment; filename="wibble.JPG" -Content-Transfer-Encoding: base64 - -/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEB -AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/wAALCAXABIEBAREA -g6bCjjw/pIZSjO6FWFpldjySOmCNrO7DBZibUXhTwtCixw+GtAijVdqxxaPp0aKvmGXa -qrbBQvms0mAMeYS/3iTV1dG0hHaRNK01XblnWxtVdjkHLMIgTyqnk9VB7CrP2KzIINpa -4O7I+zxYO9WV8jZg71Zlb+8rMDkEirAVQFAUAKAFAAAUAYAUDgADgY6DjpRtXj5RxjHA -4wQRj0wQCMdCAewpaKKK/9k= ---============_-1208892523==_============ -Content-Id: <a05001902b7f1c33773e9@[134.84.183.138].0.1> -Content-Type: image/jpeg; name="wibble2.JPG" - ; x-mac-type="4A504547" - ; x-mac-creator="474B4F4E" -Content-Disposition: attachment; filename="wibble2.JPG" -Content-Transfer-Encoding: base64 - -/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEB -AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/wAALCAXABJ0BAREA -/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQA -W6NFJJBEkU10kKGTcWMDwxuU+0JHvk8qAtOpNwqSR0n8c3BlDyXHlqsUltHEiTvdXLxR -7vMiGDNJAJWkAMk8ZkCFp5G2oo5W++INrbQtNfTQxJAuXlupz9oS4d5Y1W+E2XlWZJJE -Y7LWYQxTLE1zuMbfBPxw8X2fibVdIbSbI6nLZxX635t9TjtYreWR7WGKJTLJFFKSlozO -0ShxIXM43uC3/9k= ---============_-1208892523==_============ -Content-Type: text/plain; charset="us-ascii" ; format="flowed" - -Text text text. ---============_-1208892523==_============-- - diff --git a/lib-python/modified-2.7/email/test/data/msg_23.txt b/lib-python/modified-2.7/email/test/data/msg_23.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_23.txt +++ /dev/null @@ -1,8 +0,0 @@ -From: aperson at dom.ain -Content-Type: multipart/mixed; boundary="BOUNDARY" - ---BOUNDARY -Content-Type: text/plain - -A message part ---BOUNDARY-- diff --git a/lib-python/modified-2.7/email/test/data/msg_24.txt b/lib-python/modified-2.7/email/test/data/msg_24.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_24.txt +++ /dev/null @@ -1,10 +0,0 @@ -Content-Type: multipart/mixed; boundary="BOUNDARY" -MIME-Version: 1.0 -Subject: A subject -To: aperson at dom.ain -From: bperson at dom.ain - ---BOUNDARY - - ---BOUNDARY-- diff --git a/lib-python/modified-2.7/email/test/data/msg_25.txt b/lib-python/modified-2.7/email/test/data/msg_25.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_25.txt +++ /dev/null @@ -1,117 +0,0 @@ -From MAILER-DAEMON Fri Apr 06 16:46:09 2001 -Received: from [204.245.199.98] (helo=zinfandel.lacita.com) - by www.linux.org.uk with esmtp (Exim 3.13 #1) - id 14lYR6-0008Iv-00 - for linuxuser-admin at www.linux.org.uk; Fri, 06 Apr 2001 16:46:09 +0100 -Received: from localhost (localhost) by zinfandel.lacita.com (8.7.3/8.6.10-MT4.00) with internal id JAB03225; Fri, 6 Apr 2001 09:23:06 -0800 (GMT-0800) -Date: Fri, 6 Apr 2001 09:23:06 -0800 (GMT-0800) -From: Mail Delivery Subsystem <MAILER-DAEMON at zinfandel.lacita.com> -Subject: Returned mail: Too many hops 19 (17 max): from <linuxuser-admin at www.linux.org.uk> via [199.164.235.226], to <scoffman at wellpartner.com> -Message-Id: <200104061723.JAB03225 at zinfandel.lacita.com> -To: <linuxuser-admin at www.linux.org.uk> -To: postmaster at zinfandel.lacita.com -MIME-Version: 1.0 -Content-Type: multipart/report; report-type=delivery-status; - bo -Auto-Submitted: auto-generated (failure) - -This is a MIME-encapsulated message - ---JAB03225.986577786/zinfandel.lacita.com - -The original message was received at Fri, 6 Apr 2001 09:23:03 -0800 (GMT-0800) -from [199.164.235.226] - - ----- The following addresses have delivery notifications ----- -<scoffman at wellpartner.com> (unrecoverable error) - - ----- Transcript of session follows ----- -554 Too many hops 19 (17 max): from <linuxuser-admin at www.linux.org.uk> via [199.164.235.226], to <scoffman at wellpartner.com> - ---JAB03225.986577786/zinfandel.lacita.com -Content-Type: message/delivery-status - -Reporting-MTA: dns; zinfandel.lacita.com -Received-From-MTA: dns; [199.164.235.226] -Arrival-Date: Fri, 6 Apr 2001 09:23:03 -0800 (GMT-0800) - -Final-Recipient: rfc822; scoffman at wellpartner.com -Action: failed -Status: 5.4.6 -Last-Attempt-Date: Fri, 6 Apr 2001 09:23:06 -0800 (GMT-0800) - ---JAB03225.986577786/zinfandel.lacita.com -Content-Type: text/rfc822-headers - -Return-Path: linuxuser-admin at www.linux.org.uk -Received: from ns1.wellpartner.net ([199.164.235.226]) by zinfandel.lacita.com (8.7.3/8.6.10-MT4.00) with ESMTP id JAA03225 for <scoffman at wellpartner.com>; Fri, 6 Apr 2001 09:23:03 -0800 (GMT-0800) -Received: from zinfandel.lacita.com ([204.245.199.98]) - by - fo -Received: from ns1.wellpartner.net ([199.164.235.226]) by zinfandel.lacita.com (8.7.3/8.6.10-MT4.00) with ESMTP id JAA03221 for <scoffman at wellpartner.com>; Fri, 6 Apr 2001 09:22:18 -0800 (GMT-0800) -Received: from zinfandel.lacita.com ([204.245.199.98]) - by - fo -Received: from ns1.wellpartner.net ([199.164.235.226]) by zinfandel.lacita.com (8.7.3/8.6.10-MT4.00) with ESMTP id JAA03217 for <scoffman at wellpartner.com>; Fri, 6 Apr 2001 09:21:37 -0800 (GMT-0800) -Received: from zinfandel.lacita.com ([204.245.199.98]) - by - fo -Received: from ns1.wellpartner.net ([199.164.235.226]) by zinfandel.lacita.com (8.7.3/8.6.10-MT4.00) with ESMTP id JAA03213 for <scoffman at wellpartner.com>; Fri, 6 Apr 2001 09:20:56 -0800 (GMT-0800) -Received: from zinfandel.lacita.com ([204.245.199.98]) - by - fo -Received: from ns1.wellpartner.net ([199.164.235.226]) by zinfandel.lacita.com (8.7.3/8.6.10-MT4.00) with ESMTP id JAA03209 for <scoffman at wellpartner.com>; Fri, 6 Apr 2001 09:20:15 -0800 (GMT-0800) -Received: from zinfandel.lacita.com ([204.245.199.98]) - by - fo -Received: from ns1.wellpartner.net ([199.164.235.226]) by zinfandel.lacita.com (8.7.3/8.6.10-MT4.00) with ESMTP id JAA03205 for <scoffman at wellpartner.com>; Fri, 6 Apr 2001 09:19:33 -0800 (GMT-0800) -Received: from zinfandel.lacita.com ([204.245.199.98]) - by - fo -Received: from ns1.wellpartner.net ([199.164.235.226]) by zinfandel.lacita.com (8.7.3/8.6.10-MT4.00) with ESMTP id JAA03201 for <scoffman at wellpartner.com>; Fri, 6 Apr 2001 09:18:52 -0800 (GMT-0800) -Received: from zinfandel.lacita.com ([204.245.199.98]) - by - fo -Received: from ns1.wellpartner.net ([199.164.235.226]) by zinfandel.lacita.com (8.7.3/8.6.10-MT4.00) with ESMTP id JAA03197 for <scoffman at wellpartner.com>; Fri, 6 Apr 2001 09:17:54 -0800 (GMT-0800) -Received: from www.linux.org.uk (parcelfarce.linux.theplanet.co.uk [195.92.249.252]) - by - fo -Received: from localhost.localdomain - ([ - by - id -Received: from [212.1.130.11] (helo=s1.uklinux.net ident=root) - by - id - fo -Received: from server (ppp-2-22.cvx4.telinco.net [212.1.149.22]) - by - fo -From: Daniel James <daniel at linuxuser.co.uk> -Organization: LinuxUser -To: linuxuser at www.linux.org.uk -X-Mailer: KMail [version 1.1.99] -Content-Type: text/plain; - c -MIME-Version: 1.0 -Message-Id: <01040616033903.00962 at server> -Content-Transfer-Encoding: 8bit -Subject: [LinuxUser] bulletin no. 45 -Sender: linuxuser-admin at www.linux.org.uk -Errors-To: linuxuser-admin at www.linux.org.uk -X-BeenThere: linuxuser at www.linux.org.uk -X-Mailman-Version: 2.0.3 -Precedence: bulk -List-Help: <mailto:linuxuser-request at www.linux.org.uk?subject=help> -List-Post: <mailto:linuxuser at www.linux.org.uk> -List-Subscribe: <http://www.linux.org.uk/mailman/listinfo/linuxuser>, - <m -List-Id: bulletins from LinuxUser magazine <linuxuser.www.linux.org.uk> -List-Unsubscribe: <http://www.linux.org.uk/mailman/listinfo/linuxuser>, - <m -List-Archive: <http://www.linux.org.uk/pipermail/linuxuser/> -Date: Fri, 6 Apr 2001 16:03:39 +0100 - ---JAB03225.986577786/zinfandel.lacita.com-- - - diff --git a/lib-python/modified-2.7/email/test/data/msg_26.txt b/lib-python/modified-2.7/email/test/data/msg_26.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_26.txt +++ /dev/null @@ -1,45 +0,0 @@ -Received: from xcar [192.168.0.2] by jeeves.wooster.local - (SMTPD32-7.07 EVAL) id AFF92F0214; Sun, 12 May 2002 08:55:37 +0100 -Date: Sun, 12 May 2002 08:56:15 +0100 -From: Father Time <father.time at xcar.wooster.local> -To: timbo at jeeves.wooster.local -Subject: IMAP file test -Message-ID: <6df65d354b.father.time at rpc.wooster.local> -X-Organization: Home -User-Agent: Messenger-Pro/2.50a (MsgServe/1.50) (RISC-OS/4.02) POPstar/2.03 -MIME-Version: 1.0 -Content-Type: multipart/mixed; boundary="1618492860--2051301190--113853680" -Status: R -X-UIDL: 319998302 - -This message is in MIME format which your mailer apparently does not support. -You either require a newer version of your software which supports MIME, or -a separate MIME decoding utility. Alternatively, ask the sender of this -message to resend it in a different format. - ---1618492860--2051301190--113853680 -Content-Type: text/plain; charset=us-ascii - -Simple email with attachment. - - ---1618492860--2051301190--113853680 -Content-Type: application/riscos; name="clock.bmp,69c"; type=BMP; load=&fff69c4b; exec=&355dd4d1; access=&03 -Content-Disposition: attachment; filename="clock.bmp" -Content-Transfer-Encoding: base64 - -Qk12AgAAAAAAAHYAAAAoAAAAIAAAACAAAAABAAQAAAAAAAAAAADXDQAA1w0AAAAAAAAA -AAAAAAAAAAAAiAAAiAAAAIiIAIgAAACIAIgAiIgAALu7uwCIiIgAERHdACLuIgAz//8A -zAAAAN0R3QDu7iIA////AAAAAAAAAAAAAAAAAAAAAAAAAAi3AAAAAAAAADeAAAAAAAAA -C3ADMzMzMANwAAAAAAAAAAAHMAAAAANwAAAAAAAAAACAMAd3zPfwAwgAAAAAAAAIAwd/ -f8x/f3AwgAAAAAAAgDB0x/f3//zPAwgAAAAAAAcHfM9////8z/AwAAAAAAiwd/f3//// -////A4AAAAAAcEx/f///////zAMAAAAAiwfM9////3///8zwOAAAAAcHf3////B///// -8DAAAAALB/f3///wd3d3//AwAAAABwTPf//wCQAAD/zAMAAAAAsEx/f///B////8wDAA -AAAHB39////wf/////AwAAAACwf39///8H/////wMAAAAIcHfM9///B////M8DgAAAAA -sHTH///wf///xAMAAAAACHB3f3//8H////cDgAAAAAALB3zH//D//M9wMAAAAAAAgLB0 -z39///xHAwgAAAAAAAgLB3d3RHd3cDCAAAAAAAAAgLAHd0R3cAMIAAAAAAAAgAgLcAAA -AAMwgAgAAAAACDAAAAu7t7cwAAgDgAAAAABzcIAAAAAAAAgDMwAAAAAAN7uwgAAAAAgH -MzMAAAAACH97tzAAAAALu3c3gAAAAAAL+7tzDABAu7f7cAAAAAAACA+3MA7EQAv/sIAA -AAAAAAAIAAAAAAAAAIAAAAAA - ---1618492860--2051301190--113853680-- \ No newline at end of file diff --git a/lib-python/modified-2.7/email/test/data/msg_27.txt b/lib-python/modified-2.7/email/test/data/msg_27.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_27.txt +++ /dev/null @@ -1,15 +0,0 @@ -Return-Path: <aperson at dom.ain> -Received: by mail.dom.ain (Postfix, from userid 889) - id B9D0AD35DB; Tue, 4 Jun 2002 21:46:59 -0400 (EDT) -Message-ID: <15613.28051.707126.569693 at dom.ain> -Date: Tue, 4 Jun 2002 21:46:59 -0400 -MIME-Version: 1.0 -Content-Type: text/plain; charset=us-ascii -Content-Transfer-Encoding: 7bit -Subject: bug demonstration - 12345678911234567892123456789312345678941234567895123456789612345678971234567898112345678911234567892123456789112345678911234567892123456789 - more text -From: aperson at dom.ain (Anne P. Erson) -To: bperson at dom.ain (Barney P. Erson) - -test diff --git a/lib-python/modified-2.7/email/test/data/msg_28.txt b/lib-python/modified-2.7/email/test/data/msg_28.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_28.txt +++ /dev/null @@ -1,25 +0,0 @@ -From: aperson at dom.ain -MIME-Version: 1.0 -Content-Type: multipart/digest; boundary=BOUNDARY - ---BOUNDARY -Content-Type: message/rfc822 - -Content-Type: text/plain; charset=us-ascii -To: aa at bb.org -From: cc at dd.org -Subject: ee - -message 1 - ---BOUNDARY -Content-Type: message/rfc822 - -Content-Type: text/plain; charset=us-ascii -To: aa at bb.org -From: cc at dd.org -Subject: ee - -message 2 - ---BOUNDARY-- diff --git a/lib-python/modified-2.7/email/test/data/msg_29.txt b/lib-python/modified-2.7/email/test/data/msg_29.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_29.txt +++ /dev/null @@ -1,22 +0,0 @@ -Return-Path: <bbb at zzz.org> -Delivered-To: bbb at zzz.org -Received: by mail.zzz.org (Postfix, from userid 889) - id 27CEAD38CC; Fri, 4 May 2001 14:05:44 -0400 (EDT) -MIME-Version: 1.0 -Content-Type: text/plain; charset=us-ascii; - title*0*="us-ascii'en'This%20is%20even%20more%20"; - title*1*="%2A%2A%2Afun%2A%2A%2A%20"; - title*2="isn't it!" -Content-Transfer-Encoding: 7bit -Message-ID: <15090.61304.110929.45684 at aaa.zzz.org> -From: bbb at ddd.com (John X. Doe) -To: bbb at zzz.org -Subject: This is a test message -Date: Fri, 4 May 2001 14:05:44 -0400 - - -Hi, - -Do you like this message? - --Me diff --git a/lib-python/modified-2.7/email/test/data/msg_30.txt b/lib-python/modified-2.7/email/test/data/msg_30.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_30.txt +++ /dev/null @@ -1,23 +0,0 @@ -From: aperson at dom.ain -MIME-Version: 1.0 -Content-Type: multipart/digest; boundary=BOUNDARY - ---BOUNDARY - -Content-Type: text/plain; charset=us-ascii -To: aa at bb.org -From: cc at dd.org -Subject: ee - -message 1 - ---BOUNDARY - -Content-Type: text/plain; charset=us-ascii -To: aa at bb.org -From: cc at dd.org -Subject: ee - -message 2 - ---BOUNDARY-- diff --git a/lib-python/modified-2.7/email/test/data/msg_31.txt b/lib-python/modified-2.7/email/test/data/msg_31.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_31.txt +++ /dev/null @@ -1,15 +0,0 @@ -From: aperson at dom.ain -MIME-Version: 1.0 -Content-Type: multipart/mixed; boundary=BOUNDARY_ - ---BOUNDARY -Content-Type: text/plain - -message 1 - ---BOUNDARY -Content-Type: text/plain - -message 2 - ---BOUNDARY-- diff --git a/lib-python/modified-2.7/email/test/data/msg_32.txt b/lib-python/modified-2.7/email/test/data/msg_32.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_32.txt +++ /dev/null @@ -1,14 +0,0 @@ -Delivered-To: freebsd-isp at freebsd.org -Date: Tue, 26 Sep 2000 12:23:03 -0500 -From: Anne Person <aperson at example.com> -To: Barney Dude <bdude at example.com> -Subject: Re: Limiting Perl CPU Utilization... -Mime-Version: 1.0 -Content-Type: text/plain; charset*=ansi-x3.4-1968''us-ascii -Content-Disposition: inline -User-Agent: Mutt/1.3.8i -Sender: owner-freebsd-isp at FreeBSD.ORG -Precedence: bulk -X-Loop: FreeBSD.org - -Some message. diff --git a/lib-python/modified-2.7/email/test/data/msg_33.txt b/lib-python/modified-2.7/email/test/data/msg_33.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_33.txt +++ /dev/null @@ -1,29 +0,0 @@ -Delivered-To: freebsd-isp at freebsd.org -Date: Wed, 27 Sep 2000 11:11:09 -0500 -From: Anne Person <aperson at example.com> -To: Barney Dude <bdude at example.com> -Subject: Re: Limiting Perl CPU Utilization... -Mime-Version: 1.0 -Content-Type: multipart/signed; micalg*=ansi-x3.4-1968''pgp-md5; - protocol*=ansi-x3.4-1968''application%2Fpgp-signature; - boundary*="ansi-x3.4-1968''EeQfGwPcQSOJBaQU" -Content-Disposition: inline -Sender: owner-freebsd-isp at FreeBSD.ORG -Precedence: bulk -X-Loop: FreeBSD.org - - ---EeQfGwPcQSOJBaQU -Content-Type: text/plain; charset*=ansi-x3.4-1968''us-ascii -Content-Disposition: inline -Content-Transfer-Encoding: quoted-printable - -part 1 - ---EeQfGwPcQSOJBaQU -Content-Type: text/plain -Content-Disposition: inline - -part 2 - ---EeQfGwPcQSOJBaQU-- diff --git a/lib-python/modified-2.7/email/test/data/msg_34.txt b/lib-python/modified-2.7/email/test/data/msg_34.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_34.txt +++ /dev/null @@ -1,19 +0,0 @@ -From: aperson at dom.ain -To: bperson at dom.ain -Content-Type: multipart/digest; boundary=XYZ - ---XYZ -Content-Type: text/plain - - -This is a text plain part that is counter to recommended practice in -RFC 2046, $5.1.5, but is not illegal - ---XYZ - -From: cperson at dom.ain -To: dperson at dom.ain - -A submessage - ---XYZ-- diff --git a/lib-python/modified-2.7/email/test/data/msg_35.txt b/lib-python/modified-2.7/email/test/data/msg_35.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_35.txt +++ /dev/null @@ -1,4 +0,0 @@ -From: aperson at dom.ain -To: bperson at dom.ain -Subject: here's something interesting -counter to RFC 2822, there's no separating newline here diff --git a/lib-python/modified-2.7/email/test/data/msg_36.txt b/lib-python/modified-2.7/email/test/data/msg_36.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_36.txt +++ /dev/null @@ -1,40 +0,0 @@ -Mime-Version: 1.0 -Content-Type: Multipart/Mixed; Boundary="NextPart" -To: IETF-Announce:; -From: Internet-Drafts at ietf.org -Subject: I-D ACTION:draft-ietf-mboned-mix-00.txt -Date: Tue, 22 Dec 1998 16:55:06 -0500 - ---NextPart - -Blah blah blah - ---NextPart -Content-Type: Multipart/Alternative; Boundary="OtherAccess" - ---OtherAccess -Content-Type: Message/External-body; - access-type="mail-server"; - server="mailserv at ietf.org" - -Content-Type: text/plain -Content-ID: <19981222151406.I-D at ietf.org> - -ENCODING mime -FILE /internet-drafts/draft-ietf-mboned-mix-00.txt - ---OtherAccess -Content-Type: Message/External-body; - name="draft-ietf-mboned-mix-00.txt"; - site="ftp.ietf.org"; - access-type="anon-ftp"; - directory="internet-drafts" - -Content-Type: text/plain -Content-ID: <19981222151406.I-D at ietf.org> - - ---OtherAccess-- - ---NextPart-- - diff --git a/lib-python/modified-2.7/email/test/data/msg_37.txt b/lib-python/modified-2.7/email/test/data/msg_37.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_37.txt +++ /dev/null @@ -1,22 +0,0 @@ -Content-Type: multipart/mixed; boundary=ABCDE - ---ABCDE -Content-Type: text/x-one - -Blah - ---ABCDE ---ABCDE -Content-Type: text/x-two - -Blah - ---ABCDE ---ABCDE ---ABCDE ---ABCDE -Content-Type: text/x-two - -Blah - ---ABCDE-- diff --git a/lib-python/modified-2.7/email/test/data/msg_38.txt b/lib-python/modified-2.7/email/test/data/msg_38.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_38.txt +++ /dev/null @@ -1,101 +0,0 @@ -MIME-Version: 1.0 -Content-Type: multipart/mixed; boundary="----- =_aaaaaaaaaa0" - -------- =_aaaaaaaaaa0 -Content-Type: multipart/mixed; boundary="----- =_aaaaaaaaaa1" -Content-ID: <20592.1022586929.1 at example.com> - -------- =_aaaaaaaaaa1 -Content-Type: multipart/alternative; boundary="----- =_aaaaaaaaaa2" -Content-ID: <20592.1022586929.2 at example.com> - -------- =_aaaaaaaaaa2 -Content-Type: text/plain -Content-ID: <20592.1022586929.3 at example.com> -Content-Description: very tricky -Content-Transfer-Encoding: 7bit - - -Unlike the test test_nested-multiples-with-internal-boundary, this -piece of text not only contains the outer boundary tags -------- =_aaaaaaaaaa1 -and -------- =_aaaaaaaaaa0 -but puts them at the start of a line! And, to be even nastier, it -even includes a couple of end tags, such as this one: - -------- =_aaaaaaaaaa1-- - -and this one, which is from a multipart we haven't even seen yet! - -------- =_aaaaaaaaaa4-- - -This will, I'm sure, cause much breakage of MIME parsers. But, as -far as I can tell, it's perfectly legal. I have not yet ever seen -a case of this in the wild, but I've seen *similar* things. - - -------- =_aaaaaaaaaa2 -Content-Type: application/octet-stream -Content-ID: <20592.1022586929.4 at example.com> -Content-Description: patch2 -Content-Transfer-Encoding: base64 - -XXX - -------- =_aaaaaaaaaa2-- - -------- =_aaaaaaaaaa1 -Content-Type: multipart/alternative; boundary="----- =_aaaaaaaaaa3" -Content-ID: <20592.1022586929.6 at example.com> - -------- =_aaaaaaaaaa3 -Content-Type: application/octet-stream -Content-ID: <20592.1022586929.7 at example.com> -Content-Description: patch3 -Content-Transfer-Encoding: base64 - -XXX - -------- =_aaaaaaaaaa3 -Content-Type: application/octet-stream -Content-ID: <20592.1022586929.8 at example.com> -Content-Description: patch4 -Content-Transfer-Encoding: base64 - -XXX - -------- =_aaaaaaaaaa3-- - -------- =_aaaaaaaaaa1 -Content-Type: multipart/alternative; boundary="----- =_aaaaaaaaaa4" -Content-ID: <20592.1022586929.10 at example.com> - -------- =_aaaaaaaaaa4 -Content-Type: application/octet-stream -Content-ID: <20592.1022586929.11 at example.com> -Content-Description: patch5 -Content-Transfer-Encoding: base64 - -XXX - -------- =_aaaaaaaaaa4 -Content-Type: application/octet-stream -Content-ID: <20592.1022586929.12 at example.com> -Content-Description: patch6 -Content-Transfer-Encoding: base64 - -XXX - -------- =_aaaaaaaaaa4-- - -------- =_aaaaaaaaaa1-- - -------- =_aaaaaaaaaa0 -Content-Type: text/plain; charset="us-ascii" -Content-ID: <20592.1022586929.15 at example.com> - --- -It's never too late to have a happy childhood. - -------- =_aaaaaaaaaa0-- diff --git a/lib-python/modified-2.7/email/test/data/msg_39.txt b/lib-python/modified-2.7/email/test/data/msg_39.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_39.txt +++ /dev/null @@ -1,83 +0,0 @@ -MIME-Version: 1.0 -Content-Type: multipart/mixed; boundary="----- =_aaaaaaaaaa0" - -------- =_aaaaaaaaaa0 -Content-Type: multipart/mixed; boundary="----- =_aaaaaaaaaa1" -Content-ID: <20592.1022586929.1 at example.com> - -------- =_aaaaaaaaaa1 -Content-Type: multipart/alternative; boundary="----- =_aaaaaaaaaa1" -Content-ID: <20592.1022586929.2 at example.com> - -------- =_aaaaaaaaaa1 -Content-Type: application/octet-stream -Content-ID: <20592.1022586929.3 at example.com> -Content-Description: patch1 -Content-Transfer-Encoding: base64 - -XXX - -------- =_aaaaaaaaaa1 -Content-Type: application/octet-stream -Content-ID: <20592.1022586929.4 at example.com> -Content-Description: patch2 -Content-Transfer-Encoding: base64 - -XXX - -------- =_aaaaaaaaaa1-- - -------- =_aaaaaaaaaa1 -Content-Type: multipart/alternative; boundary="----- =_aaaaaaaaaa1" -Content-ID: <20592.1022586929.6 at example.com> - -------- =_aaaaaaaaaa1 -Content-Type: application/octet-stream -Content-ID: <20592.1022586929.7 at example.com> -Content-Description: patch3 -Content-Transfer-Encoding: base64 - -XXX - -------- =_aaaaaaaaaa1 -Content-Type: application/octet-stream -Content-ID: <20592.1022586929.8 at example.com> -Content-Description: patch4 -Content-Transfer-Encoding: base64 - -XXX - -------- =_aaaaaaaaaa1-- - -------- =_aaaaaaaaaa1 -Content-Type: multipart/alternative; boundary="----- =_aaaaaaaaaa1" -Content-ID: <20592.1022586929.10 at example.com> - -------- =_aaaaaaaaaa1 -Content-Type: application/octet-stream -Content-ID: <20592.1022586929.11 at example.com> -Content-Description: patch5 -Content-Transfer-Encoding: base64 - -XXX - -------- =_aaaaaaaaaa1 -Content-Type: application/octet-stream -Content-ID: <20592.1022586929.12 at example.com> -Content-Description: patch6 -Content-Transfer-Encoding: base64 - -XXX - -------- =_aaaaaaaaaa1-- - -------- =_aaaaaaaaaa1-- - -------- =_aaaaaaaaaa0 -Content-Type: text/plain; charset="us-ascii" -Content-ID: <20592.1022586929.15 at example.com> - --- -It's never too late to have a happy childhood. - -------- =_aaaaaaaaaa0-- diff --git a/lib-python/modified-2.7/email/test/data/msg_40.txt b/lib-python/modified-2.7/email/test/data/msg_40.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_40.txt +++ /dev/null @@ -1,10 +0,0 @@ -MIME-Version: 1.0 -Content-Type: text/html; boundary="--961284236552522269" - -----961284236552522269 -Content-Type: text/html; -Content-Transfer-Encoding: 7Bit - -<html></html> - -----961284236552522269-- diff --git a/lib-python/modified-2.7/email/test/data/msg_41.txt b/lib-python/modified-2.7/email/test/data/msg_41.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_41.txt +++ /dev/null @@ -1,8 +0,0 @@ -From: "Allison Dunlap" <xxx at example.com> -To: yyy at example.com -Subject: 64423 -Date: Sun, 11 Jul 2004 16:09:27 -0300 -MIME-Version: 1.0 -Content-Type: multipart/alternative; - -Blah blah blah diff --git a/lib-python/modified-2.7/email/test/data/msg_42.txt b/lib-python/modified-2.7/email/test/data/msg_42.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_42.txt +++ /dev/null @@ -1,20 +0,0 @@ -Content-Type: multipart/mixed; boundary="AAA" -From: Mail Delivery Subsystem <xxx at example.com> -To: yyy at example.com - -This is a MIME-encapsulated message - ---AAA - -Stuff - ---AAA -Content-Type: message/rfc822 - -From: webmaster at python.org -To: zzz at example.com -Content-Type: multipart/mixed; boundary="BBB" - ---BBB-- - ---AAA-- diff --git a/lib-python/modified-2.7/email/test/data/msg_43.txt b/lib-python/modified-2.7/email/test/data/msg_43.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_43.txt +++ /dev/null @@ -1,217 +0,0 @@ -From SRS0=aO/p=ON=bag.python.org=None at bounce2.pobox.com Fri Nov 26 21:40:36 2004 -X-VM-v5-Data: ([nil nil nil nil nil nil nil nil nil] - [nil nil nil nil nil nil nil "MAILER DAEMON <>" "MAILER DAEMON <>" nil nil "Banned file: auto__mail.python.bat in mail from you" "^From:" nil nil nil nil "Banned file: auto__mail.python.bat in mail from you" nil nil nil nil nil nil nil] - nil) -MIME-Version: 1.0 -Message-Id: <edab.7804f5cb8070 at python.org> -Content-Type: multipart/report; report-type=delivery-status; - charset=utf-8; - boundary="----------=_1101526904-1956-5" -X-Virus-Scanned: by XS4ALL Virus Scanner -X-UIDL: 4\G!!!<c"!UV["!M7C!! -From: MAILER DAEMON <> -To: <webmaster at python.org> -Subject: Banned file: auto__mail.python.bat in mail from you -Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -This is a multi-part message in MIME format... - -------------=_1101526904-1956-5 -Content-Type: text/plain; charset="utf-8" -Content-Disposition: inline -Content-Transfer-Encoding: 7bit - -BANNED FILENAME ALERT - -Your message to: xxxxxxx at dot.ca.gov, xxxxxxxxxxxxx at dot.ca.gov, xxxxxxxxxx at dot.ca.gov, xxxxxxxx at dot.ca.gov, xxxxxxxxxx at dot.ca.gov, xxxxxx at dot.ca.gov, xxxxxxxxxx at dot.ca.gov, xxxxxx at dot.ca.gov, xxxxxx at dot.ca.gov, xxxxxxxxxxxxxxxx at dot.ca.gov, xxxxxxxxxxx at dot.ca.gov, xxxxxxxxxx at dot.ca.gov, xxxxxxxxxx at dot.ca.gov, xxxxxxxxxxxx at dot.ca.gov, xxxxxxxxxxxx at dot.ca.gov, xxxxxxx at dot.ca.gov, xxxxxxxxx at dot.ca.gov, xxxxxxxxxx at dot.ca.gov, xxxxxx at dot.ca.gov, xxx at dot.ca.gov, xxxxxxx at dot.ca.gov, xxxxxxx at dot.ca.gov, xxxxxxxxxxxxxxx at dot.ca.gov, xxxxxxxxxx at dot.ca.gov, xxxxxxx at dot.ca.gov, xxx at dot.ca.gov, xxxxxxxx at dot.ca.gov, xxxxxxxxxxxxx at dot.ca.gov, xxxxxxxxxxxxx at dot.ca.gov, xxxxxxxxxxx at dot.ca.gov, xxxxxxxxx at dot.ca.gov, xxxxxxxxxx at dot.ca.gov, xxxxxxxxxxxx at dot.ca.gov, xxxxxxx at dot.ca.gov, xxxxxxxxxxxxxxx at dot.ca.gov, xxxxxxxxxxxxx at dot.ca.gov, xxxx at dot.ca.gov, xxxxxxxx at dot.ca.gov, xxxxxxxxxx at dot.ca.gov, xxxxxxxxxxxxxxxxxx at dot.ca.gov -was blocked by our Spam Firewall. The email you sent with the following subject has NOT BEEN DELIVERED: - -Subject: Delivery_failure_notice - -An attachment in that mail was of a file type that the Spam Firewall is set to block. - - - -------------=_1101526904-1956-5 -Content-Type: message/delivery-status -Content-Disposition: inline -Content-Transfer-Encoding: 7bit -Content-Description: Delivery error report - -Reporting-MTA: dns; sacspam01.dot.ca.gov -Received-From-MTA: smtp; sacspam01.dot.ca.gov ([127.0.0.1]) -Arrival-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxxxxxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxxxxxxxxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxxxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxxxxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxxxxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxxxxxxxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -------------=_1101526904-1956-5 -Content-Type: text/rfc822-headers -Content-Disposition: inline -Content-Transfer-Encoding: 7bit -Content-Description: Undelivered-message headers - -Received: from kgsav.org (ppp-70-242-162-63.dsl.spfdmo.swbell.net [70.242.162.63]) - by sacspam01.dot.ca.gov (Spam Firewall) with SMTP - id A232AD03DE3A; Fri, 26 Nov 2004 19:41:35 -0800 (PST) -From: webmaster at python.org -To: xxxxx at dot.ca.gov -Date: Sat, 27 Nov 2004 03:35:30 UTC -Subject: Delivery_failure_notice -Importance: Normal -X-Priority: 3 (Normal) -X-MSMail-Priority: Normal -Message-ID: <edab.7804f5cb8070 at python.org> -MIME-Version: 1.0 -Content-Type: multipart/mixed; boundary="====67bd2b7a5.f99f7" -Content-Transfer-Encoding: 7bit - -------------=_1101526904-1956-5-- - diff --git a/lib-python/modified-2.7/email/test/data/msg_44.txt b/lib-python/modified-2.7/email/test/data/msg_44.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_44.txt +++ /dev/null @@ -1,33 +0,0 @@ -Return-Path: <barry at python.org> -Delivered-To: barry at python.org -Received: by mail.python.org (Postfix, from userid 889) - id C2BF0D37C6; Tue, 11 Sep 2001 00:05:05 -0400 (EDT) -MIME-Version: 1.0 -Content-Type: multipart/mixed; boundary="h90VIIIKmx" -Content-Transfer-Encoding: 7bit -Message-ID: <15261.36209.358846.118674 at anthem.python.org> -From: barry at python.org (Barry A. Warsaw) -To: barry at python.org -Subject: a simple multipart -Date: Tue, 11 Sep 2001 00:05:05 -0400 -X-Mailer: VM 6.95 under 21.4 (patch 4) "Artificial Intelligence" XEmacs Lucid -X-Attribution: BAW -X-Oblique-Strategy: Make a door into a window - - ---h90VIIIKmx -Content-Type: text/plain; name="msg.txt" -Content-Transfer-Encoding: 7bit - -a simple kind of mirror -to reflect upon our own - ---h90VIIIKmx -Content-Type: text/plain; name="msg.txt" -Content-Transfer-Encoding: 7bit - -a simple kind of mirror -to reflect upon our own - ---h90VIIIKmx-- - diff --git a/lib-python/modified-2.7/email/test/data/msg_45.txt b/lib-python/modified-2.7/email/test/data/msg_45.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_45.txt +++ /dev/null @@ -1,33 +0,0 @@ -From: <foo at bar.baz> -To: <baz at bar.foo> -Subject: test -X-Long-Line: Some really long line contains a lot of text and thus has to be rewrapped because it is some - really long - line -MIME-Version: 1.0 -Content-Type: multipart/signed; boundary="borderline"; - protocol="application/pgp-signature"; micalg=pgp-sha1 - -This is an OpenPGP/MIME signed message (RFC 2440 and 3156) ---borderline -Content-Type: text/plain -X-Long-Line: Another really long line contains a lot of text and thus has to be rewrapped because it is another - really long - line - -This is the signed contents. - ---borderline -Content-Type: application/pgp-signature; name="signature.asc" -Content-Description: OpenPGP digital signature -Content-Disposition: attachment; filename="signature.asc" - ------BEGIN PGP SIGNATURE----- -Version: GnuPG v2.0.6 (GNU/Linux) - -iD8DBQFG03voRhp6o4m9dFsRApSZAKCCAN3IkJlVRg6NvAiMHlvvIuMGPQCeLZtj -FGwfnRHFBFO/S4/DKysm0lI= -=t7+s ------END PGP SIGNATURE----- - ---borderline-- diff --git a/lib-python/modified-2.7/email/test/data/msg_46.txt b/lib-python/modified-2.7/email/test/data/msg_46.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_46.txt +++ /dev/null @@ -1,23 +0,0 @@ -Return-Path: <sender at example.net> -Delivery-Date: Mon, 08 Feb 2010 14:05:16 +0100 -Received: from example.org (example.org [64.5.53.58]) - by example.net (node=mxbap2) with ESMTP (Nemesis) - id UNIQUE for someone at example.com; Mon, 08 Feb 2010 14:05:16 +0100 -Date: Mon, 01 Feb 2010 12:21:16 +0100 -From: "Sender" <sender at example.net> -To: <someone at example.com> -Subject: GroupwiseForwardingTest -Mime-Version: 1.0 -Content-Type: message/rfc822 - -Return-path: <sender at example.net> -Message-ID: <4B66B890.4070408 at teconcept.de> -Date: Mon, 01 Feb 2010 12:18:40 +0100 -From: "Dr. Sender" <sender at example.net> -MIME-Version: 1.0 -To: "Recipient" <recipient at example.com> -Subject: GroupwiseForwardingTest -Content-Type: text/plain; charset=ISO-8859-15 -Content-Transfer-Encoding: 7bit - -Testing email forwarding with Groupwise 1.2.2010 diff --git a/lib-python/modified-2.7/email/test/test_email.py b/lib-python/modified-2.7/email/test/test_email.py deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/test_email.py +++ /dev/null @@ -1,3530 +0,0 @@ -# Copyright (C) 2001-2010 Python Software Foundation -# Contact: email-sig at python.org -# email package unit tests - -import os -import sys -import time -import base64 -import difflib -import unittest -import warnings -from cStringIO import StringIO - -import email - -from email.Charset import Charset -from email.Header import Header, decode_header, make_header -from email.Parser import Parser, HeaderParser -from email.Generator import Generator, DecodedGenerator -from email.Message import Message -from email.MIMEAudio import MIMEAudio -from email.MIMEText import MIMEText -from email.MIMEImage import MIMEImage -from email.MIMEBase import MIMEBase -from email.MIMEMessage import MIMEMessage -from email.MIMEMultipart import MIMEMultipart -from email import Utils -from email import Errors -from email import Encoders -from email import Iterators -from email import base64MIME -from email import quopriMIME - -from test.test_support import findfile, run_unittest, impl_detail -from email.test import __file__ as landmark - - -NL = '\n' -EMPTYSTRING = '' -SPACE = ' ' - - - -def openfile(filename, mode='r'): - path = os.path.join(os.path.dirname(landmark), 'data', filename) - return open(path, mode) - - - -# Base test class -class TestEmailBase(unittest.TestCase): - def ndiffAssertEqual(self, first, second): - """Like assertEqual except use ndiff for readable output.""" - if first != second: - sfirst = str(first) - ssecond = str(second) - diff = difflib.ndiff(sfirst.splitlines(), ssecond.splitlines()) - fp = StringIO() - print >> fp, NL, NL.join(diff) - raise self.failureException, fp.getvalue() - - def _msgobj(self, filename): - fp = openfile(findfile(filename)) - try: - msg = email.message_from_file(fp) - finally: - fp.close() - return msg - - - -# Test various aspects of the Message class's API -class TestMessageAPI(TestEmailBase): - def test_get_all(self): - eq = self.assertEqual - msg = self._msgobj('msg_20.txt') - eq(msg.get_all('cc'), ['ccc at zzz.org', 'ddd at zzz.org', 'eee at zzz.org']) - eq(msg.get_all('xx', 'n/a'), 'n/a') - - def test_getset_charset(self): - eq = self.assertEqual - msg = Message() - eq(msg.get_charset(), None) - charset = Charset('iso-8859-1') - msg.set_charset(charset) - eq(msg['mime-version'], '1.0') - eq(msg.get_content_type(), 'text/plain') - eq(msg['content-type'], 'text/plain; charset="iso-8859-1"') - eq(msg.get_param('charset'), 'iso-8859-1') - eq(msg['content-transfer-encoding'], 'quoted-printable') - eq(msg.get_charset().input_charset, 'iso-8859-1') - # Remove the charset - msg.set_charset(None) - eq(msg.get_charset(), None) - eq(msg['content-type'], 'text/plain') - # Try adding a charset when there's already MIME headers present - msg = Message() - msg['MIME-Version'] = '2.0' - msg['Content-Type'] = 'text/x-weird' - msg['Content-Transfer-Encoding'] = 'quinted-puntable' - msg.set_charset(charset) - eq(msg['mime-version'], '2.0') - eq(msg['content-type'], 'text/x-weird; charset="iso-8859-1"') - eq(msg['content-transfer-encoding'], 'quinted-puntable') - - def test_set_charset_from_string(self): - eq = self.assertEqual - msg = Message() - msg.set_charset('us-ascii') - eq(msg.get_charset().input_charset, 'us-ascii') - eq(msg['content-type'], 'text/plain; charset="us-ascii"') - - def test_set_payload_with_charset(self): - msg = Message() - charset = Charset('iso-8859-1') - msg.set_payload('This is a string payload', charset) - self.assertEqual(msg.get_charset().input_charset, 'iso-8859-1') - - def test_get_charsets(self): - eq = self.assertEqual - - msg = self._msgobj('msg_08.txt') - charsets = msg.get_charsets() - eq(charsets, [None, 'us-ascii', 'iso-8859-1', 'iso-8859-2', 'koi8-r']) - - msg = self._msgobj('msg_09.txt') - charsets = msg.get_charsets('dingbat') - eq(charsets, ['dingbat', 'us-ascii', 'iso-8859-1', 'dingbat', - 'koi8-r']) - - msg = self._msgobj('msg_12.txt') - charsets = msg.get_charsets() - eq(charsets, [None, 'us-ascii', 'iso-8859-1', None, 'iso-8859-2', - 'iso-8859-3', 'us-ascii', 'koi8-r']) - - def test_get_filename(self): - eq = self.assertEqual - - msg = self._msgobj('msg_04.txt') - filenames = [p.get_filename() for p in msg.get_payload()] - eq(filenames, ['msg.txt', 'msg.txt']) - - msg = self._msgobj('msg_07.txt') - subpart = msg.get_payload(1) - eq(subpart.get_filename(), 'dingusfish.gif') - - def test_get_filename_with_name_parameter(self): - eq = self.assertEqual - - msg = self._msgobj('msg_44.txt') - filenames = [p.get_filename() for p in msg.get_payload()] - eq(filenames, ['msg.txt', 'msg.txt']) - - def test_get_boundary(self): - eq = self.assertEqual - msg = self._msgobj('msg_07.txt') - # No quotes! - eq(msg.get_boundary(), 'BOUNDARY') - - def test_set_boundary(self): - eq = self.assertEqual - # This one has no existing boundary parameter, but the Content-Type: - # header appears fifth. - msg = self._msgobj('msg_01.txt') - msg.set_boundary('BOUNDARY') - header, value = msg.items()[4] - eq(header.lower(), 'content-type') - eq(value, 'text/plain; charset="us-ascii"; boundary="BOUNDARY"') - # This one has a Content-Type: header, with a boundary, stuck in the - # middle of its headers. Make sure the order is preserved; it should - # be fifth. - msg = self._msgobj('msg_04.txt') - msg.set_boundary('BOUNDARY') - header, value = msg.items()[4] - eq(header.lower(), 'content-type') - eq(value, 'multipart/mixed; boundary="BOUNDARY"') - # And this one has no Content-Type: header at all. - msg = self._msgobj('msg_03.txt') - self.assertRaises(Errors.HeaderParseError, - msg.set_boundary, 'BOUNDARY') - - def test_make_boundary(self): - msg = MIMEMultipart('form-data') - # Note that when the boundary gets created is an implementation - # detail and might change. - self.assertEqual(msg.items()[0][1], 'multipart/form-data') - # Trigger creation of boundary - msg.as_string() - self.assertEqual(msg.items()[0][1][:33], - 'multipart/form-data; boundary="==') - # XXX: there ought to be tests of the uniqueness of the boundary, too. - - def test_message_rfc822_only(self): - # Issue 7970: message/rfc822 not in multipart parsed by - # HeaderParser caused an exception when flattened. - fp = openfile(findfile('msg_46.txt')) - msgdata = fp.read() - parser = email.Parser.HeaderParser() - msg = parser.parsestr(msgdata) - out = StringIO() - gen = email.Generator.Generator(out, True, 0) - gen.flatten(msg, False) - self.assertEqual(out.getvalue(), msgdata) - - def test_get_decoded_payload(self): - eq = self.assertEqual - msg = self._msgobj('msg_10.txt') - # The outer message is a multipart - eq(msg.get_payload(decode=True), None) - # Subpart 1 is 7bit encoded - eq(msg.get_payload(0).get_payload(decode=True), - 'This is a 7bit encoded message.\n') - # Subpart 2 is quopri - eq(msg.get_payload(1).get_payload(decode=True), - '\xa1This is a Quoted Printable encoded message!\n') - # Subpart 3 is base64 - eq(msg.get_payload(2).get_payload(decode=True), - 'This is a Base64 encoded message.') - # Subpart 4 is base64 with a trailing newline, which - # used to be stripped (issue 7143). - eq(msg.get_payload(3).get_payload(decode=True), - 'This is a Base64 encoded message.\n') - # Subpart 5 has no Content-Transfer-Encoding: header. - eq(msg.get_payload(4).get_payload(decode=True), - 'This has no Content-Transfer-Encoding: header.\n') - - def test_get_decoded_uu_payload(self): - eq = self.assertEqual - msg = Message() - msg.set_payload('begin 666 -\n+:&5L;&\\@=V]R;&0 \n \nend\n') - for cte in ('x-uuencode', 'uuencode', 'uue', 'x-uue'): - msg['content-transfer-encoding'] = cte - eq(msg.get_payload(decode=True), 'hello world') - # Now try some bogus data - msg.set_payload('foo') - eq(msg.get_payload(decode=True), 'foo') - - def test_decode_bogus_uu_payload_quietly(self): - msg = Message() - msg.set_payload('begin 664 foo.txt\n%<W1F=0000H \n \nend\n') - msg['Content-Transfer-Encoding'] = 'x-uuencode' - old_stderr = sys.stderr - try: - sys.stderr = sfp = StringIO() - # We don't care about the payload - msg.get_payload(decode=True) - finally: - sys.stderr = old_stderr - self.assertEqual(sfp.getvalue(), '') - - def test_decoded_generator(self): - eq = self.assertEqual - msg = self._msgobj('msg_07.txt') - fp = openfile('msg_17.txt') - try: - text = fp.read() - finally: - fp.close() - s = StringIO() - g = DecodedGenerator(s) - g.flatten(msg) - eq(s.getvalue(), text) - - def test__contains__(self): - msg = Message() - msg['From'] = 'Me' - msg['to'] = 'You' - # Check for case insensitivity - self.assertTrue('from' in msg) - self.assertTrue('From' in msg) - self.assertTrue('FROM' in msg) - self.assertTrue('to' in msg) - self.assertTrue('To' in msg) - self.assertTrue('TO' in msg) - - def test_as_string(self): - eq = self.assertEqual - msg = self._msgobj('msg_01.txt') - fp = openfile('msg_01.txt') - try: - # BAW 30-Mar-2009 Evil be here. So, the generator is broken with - # respect to long line breaking. It's also not idempotent when a - # header from a parsed message is continued with tabs rather than - # spaces. Before we fixed bug 1974 it was reversedly broken, - # i.e. headers that were continued with spaces got continued with - # tabs. For Python 2.x there's really no good fix and in Python - # 3.x all this stuff is re-written to be right(er). Chris Withers - # convinced me that using space as the default continuation - # character is less bad for more applications. - text = fp.read().replace('\t', ' ') - finally: - fp.close() - eq(text, msg.as_string()) - fullrepr = str(msg) - lines = fullrepr.split('\n') - self.assertTrue(lines[0].startswith('From ')) - eq(text, NL.join(lines[1:])) - - def test_bad_param(self): - msg = email.message_from_string("Content-Type: blarg; baz; boo\n") - self.assertEqual(msg.get_param('baz'), '') - - def test_missing_filename(self): - msg = email.message_from_string("From: foo\n") - self.assertEqual(msg.get_filename(), None) - - def test_bogus_filename(self): - msg = email.message_from_string( - "Content-Disposition: blarg; filename\n") - self.assertEqual(msg.get_filename(), '') - - def test_missing_boundary(self): - msg = email.message_from_string("From: foo\n") - self.assertEqual(msg.get_boundary(), None) - - def test_get_params(self): - eq = self.assertEqual - msg = email.message_from_string( - 'X-Header: foo=one; bar=two; baz=three\n') - eq(msg.get_params(header='x-header'), - [('foo', 'one'), ('bar', 'two'), ('baz', 'three')]) - msg = email.message_from_string( - 'X-Header: foo; bar=one; baz=two\n') - eq(msg.get_params(header='x-header'), - [('foo', ''), ('bar', 'one'), ('baz', 'two')]) - eq(msg.get_params(), None) - msg = email.message_from_string( - 'X-Header: foo; bar="one"; baz=two\n') - eq(msg.get_params(header='x-header'), - [('foo', ''), ('bar', 'one'), ('baz', 'two')]) - - def test_get_param_liberal(self): - msg = Message() - msg['Content-Type'] = 'Content-Type: Multipart/mixed; boundary = "CPIMSSMTPC06p5f3tG"' - self.assertEqual(msg.get_param('boundary'), 'CPIMSSMTPC06p5f3tG') - - def test_get_param(self): - eq = self.assertEqual - msg = email.message_from_string( - "X-Header: foo=one; bar=two; baz=three\n") - eq(msg.get_param('bar', header='x-header'), 'two') - eq(msg.get_param('quuz', header='x-header'), None) - eq(msg.get_param('quuz'), None) - msg = email.message_from_string( - 'X-Header: foo; bar="one"; baz=two\n') - eq(msg.get_param('foo', header='x-header'), '') - eq(msg.get_param('bar', header='x-header'), 'one') - eq(msg.get_param('baz', header='x-header'), 'two') - # XXX: We are not RFC-2045 compliant! We cannot parse: - # msg["Content-Type"] = 'text/plain; weird="hey; dolly? [you] @ <\\"home\\">?"' - # msg.get_param("weird") - # yet. - - def test_get_param_funky_continuation_lines(self): - msg = self._msgobj('msg_22.txt') - self.assertEqual(msg.get_payload(1).get_param('name'), 'wibble.JPG') - - def test_get_param_with_semis_in_quotes(self): - msg = email.message_from_string( - 'Content-Type: image/pjpeg; name="Jim&&Jill"\n') - self.assertEqual(msg.get_param('name'), 'Jim&&Jill') - self.assertEqual(msg.get_param('name', unquote=False), - '"Jim&&Jill"') - - def test_get_param_with_quotes(self): - msg = email.message_from_string( - 'Content-Type: foo; bar*0="baz\\"foobar"; bar*1="\\"baz"') - self.assertEqual(msg.get_param('bar'), 'baz"foobar"baz') - msg = email.message_from_string( - "Content-Type: foo; bar*0=\"baz\\\"foobar\"; bar*1=\"\\\"baz\"") - self.assertEqual(msg.get_param('bar'), 'baz"foobar"baz') - - def test_has_key(self): - msg = email.message_from_string('Header: exists') - self.assertTrue(msg.has_key('header')) - self.assertTrue(msg.has_key('Header')) - self.assertTrue(msg.has_key('HEADER')) - self.assertFalse(msg.has_key('headeri')) - - def test_set_param(self): - eq = self.assertEqual - msg = Message() - msg.set_param('charset', 'iso-2022-jp') - eq(msg.get_param('charset'), 'iso-2022-jp') - msg.set_param('importance', 'high value') - eq(msg.get_param('importance'), 'high value') - eq(msg.get_param('importance', unquote=False), '"high value"') - eq(msg.get_params(), [('text/plain', ''), - ('charset', 'iso-2022-jp'), - ('importance', 'high value')]) - eq(msg.get_params(unquote=False), [('text/plain', ''), - ('charset', '"iso-2022-jp"'), - ('importance', '"high value"')]) - msg.set_param('charset', 'iso-9999-xx', header='X-Jimmy') - eq(msg.get_param('charset', header='X-Jimmy'), 'iso-9999-xx') - - def test_del_param(self): - eq = self.assertEqual - msg = self._msgobj('msg_05.txt') - eq(msg.get_params(), - [('multipart/report', ''), ('report-type', 'delivery-status'), - ('boundary', 'D1690A7AC1.996856090/mail.example.com')]) - old_val = msg.get_param("report-type") - msg.del_param("report-type") - eq(msg.get_params(), - [('multipart/report', ''), - ('boundary', 'D1690A7AC1.996856090/mail.example.com')]) - msg.set_param("report-type", old_val) - eq(msg.get_params(), - [('multipart/report', ''), - ('boundary', 'D1690A7AC1.996856090/mail.example.com'), - ('report-type', old_val)]) - - def test_del_param_on_other_header(self): - msg = Message() - msg.add_header('Content-Disposition', 'attachment', filename='bud.gif') - msg.del_param('filename', 'content-disposition') - self.assertEqual(msg['content-disposition'], 'attachment') - - def test_set_type(self): - eq = self.assertEqual - msg = Message() - self.assertRaises(ValueError, msg.set_type, 'text') - msg.set_type('text/plain') - eq(msg['content-type'], 'text/plain') - msg.set_param('charset', 'us-ascii') - eq(msg['content-type'], 'text/plain; charset="us-ascii"') - msg.set_type('text/html') - eq(msg['content-type'], 'text/html; charset="us-ascii"') - - def test_set_type_on_other_header(self): - msg = Message() - msg['X-Content-Type'] = 'text/plain' - msg.set_type('application/octet-stream', 'X-Content-Type') - self.assertEqual(msg['x-content-type'], 'application/octet-stream') - - def test_get_content_type_missing(self): - msg = Message() - self.assertEqual(msg.get_content_type(), 'text/plain') - - def test_get_content_type_missing_with_default_type(self): - msg = Message() - msg.set_default_type('message/rfc822') - self.assertEqual(msg.get_content_type(), 'message/rfc822') - - def test_get_content_type_from_message_implicit(self): - msg = self._msgobj('msg_30.txt') - self.assertEqual(msg.get_payload(0).get_content_type(), - 'message/rfc822') - - def test_get_content_type_from_message_explicit(self): - msg = self._msgobj('msg_28.txt') - self.assertEqual(msg.get_payload(0).get_content_type(), - 'message/rfc822') - - def test_get_content_type_from_message_text_plain_implicit(self): - msg = self._msgobj('msg_03.txt') - self.assertEqual(msg.get_content_type(), 'text/plain') - - def test_get_content_type_from_message_text_plain_explicit(self): - msg = self._msgobj('msg_01.txt') - self.assertEqual(msg.get_content_type(), 'text/plain') - - def test_get_content_maintype_missing(self): - msg = Message() - self.assertEqual(msg.get_content_maintype(), 'text') - - def test_get_content_maintype_missing_with_default_type(self): - msg = Message() - msg.set_default_type('message/rfc822') - self.assertEqual(msg.get_content_maintype(), 'message') - - def test_get_content_maintype_from_message_implicit(self): - msg = self._msgobj('msg_30.txt') - self.assertEqual(msg.get_payload(0).get_content_maintype(), 'message') - - def test_get_content_maintype_from_message_explicit(self): - msg = self._msgobj('msg_28.txt') - self.assertEqual(msg.get_payload(0).get_content_maintype(), 'message') - - def test_get_content_maintype_from_message_text_plain_implicit(self): - msg = self._msgobj('msg_03.txt') - self.assertEqual(msg.get_content_maintype(), 'text') - - def test_get_content_maintype_from_message_text_plain_explicit(self): - msg = self._msgobj('msg_01.txt') - self.assertEqual(msg.get_content_maintype(), 'text') - - def test_get_content_subtype_missing(self): - msg = Message() - self.assertEqual(msg.get_content_subtype(), 'plain') - - def test_get_content_subtype_missing_with_default_type(self): - msg = Message() - msg.set_default_type('message/rfc822') - self.assertEqual(msg.get_content_subtype(), 'rfc822') - - def test_get_content_subtype_from_message_implicit(self): - msg = self._msgobj('msg_30.txt') - self.assertEqual(msg.get_payload(0).get_content_subtype(), 'rfc822') - - def test_get_content_subtype_from_message_explicit(self): - msg = self._msgobj('msg_28.txt') - self.assertEqual(msg.get_payload(0).get_content_subtype(), 'rfc822') - - def test_get_content_subtype_from_message_text_plain_implicit(self): - msg = self._msgobj('msg_03.txt') - self.assertEqual(msg.get_content_subtype(), 'plain') - - def test_get_content_subtype_from_message_text_plain_explicit(self): - msg = self._msgobj('msg_01.txt') - self.assertEqual(msg.get_content_subtype(), 'plain') - - def test_get_content_maintype_error(self): - msg = Message() - msg['Content-Type'] = 'no-slash-in-this-string' - self.assertEqual(msg.get_content_maintype(), 'text') - - def test_get_content_subtype_error(self): - msg = Message() - msg['Content-Type'] = 'no-slash-in-this-string' - self.assertEqual(msg.get_content_subtype(), 'plain') - - def test_replace_header(self): - eq = self.assertEqual - msg = Message() - msg.add_header('First', 'One') - msg.add_header('Second', 'Two') - msg.add_header('Third', 'Three') - eq(msg.keys(), ['First', 'Second', 'Third']) - eq(msg.values(), ['One', 'Two', 'Three']) - msg.replace_header('Second', 'Twenty') - eq(msg.keys(), ['First', 'Second', 'Third']) - eq(msg.values(), ['One', 'Twenty', 'Three']) - msg.add_header('First', 'Eleven') - msg.replace_header('First', 'One Hundred') - eq(msg.keys(), ['First', 'Second', 'Third', 'First']) - eq(msg.values(), ['One Hundred', 'Twenty', 'Three', 'Eleven']) - self.assertRaises(KeyError, msg.replace_header, 'Fourth', 'Missing') - - def test_broken_base64_payload(self): - x = 'AwDp0P7//y6LwKEAcPa/6Q=9' - msg = Message() - msg['content-type'] = 'audio/x-midi' - msg['content-transfer-encoding'] = 'base64' - msg.set_payload(x) - self.assertEqual(msg.get_payload(decode=True), x) - - def test_get_content_charset(self): - msg = Message() - msg.set_charset('us-ascii') - self.assertEqual('us-ascii', msg.get_content_charset()) - msg.set_charset(u'us-ascii') - self.assertEqual('us-ascii', msg.get_content_charset()) - - # Issue 5871: reject an attempt to embed a header inside a header value - # (header injection attack). - def test_embeded_header_via_Header_rejected(self): - msg = Message() - msg['Dummy'] = Header('dummy\nX-Injected-Header: test') - self.assertRaises(Errors.HeaderParseError, msg.as_string) - - def test_embeded_header_via_string_rejected(self): - msg = Message() - msg['Dummy'] = 'dummy\nX-Injected-Header: test' - self.assertRaises(Errors.HeaderParseError, msg.as_string) - - -# Test the email.Encoders module -class TestEncoders(unittest.TestCase): - def test_encode_empty_payload(self): - eq = self.assertEqual - msg = Message() - msg.set_charset('us-ascii') - eq(msg['content-transfer-encoding'], '7bit') - - def test_default_cte(self): - eq = self.assertEqual - # 7bit data and the default us-ascii _charset - msg = MIMEText('hello world') - eq(msg['content-transfer-encoding'], '7bit') - # Similar, but with 8bit data - msg = MIMEText('hello \xf8 world') - eq(msg['content-transfer-encoding'], '8bit') - # And now with a different charset - msg = MIMEText('hello \xf8 world', _charset='iso-8859-1') - eq(msg['content-transfer-encoding'], 'quoted-printable') - - @impl_detail("PyPy has no cjkc codec yet", pypy=False) - def test_encode7or8bit(self): - # Make sure a charset whose input character set is 8bit but - # whose output character set is 7bit gets a transfer-encoding - # of 7bit. - eq = self.assertEqual - msg = email.MIMEText.MIMEText('\xca\xb8', _charset='euc-jp') - eq(msg['content-transfer-encoding'], '7bit') - - -# Test long header wrapping -class TestLongHeaders(TestEmailBase): - def test_split_long_continuation(self): - eq = self.ndiffAssertEqual - msg = email.message_from_string("""\ -Subject: bug demonstration -\t12345678911234567892123456789312345678941234567895123456789612345678971234567898112345678911234567892123456789112345678911234567892123456789 -\tmore text - -test -""") - sfp = StringIO() - g = Generator(sfp) - g.flatten(msg) - eq(sfp.getvalue(), """\ -Subject: bug demonstration - 12345678911234567892123456789312345678941234567895123456789612345678971234567898112345678911234567892123456789112345678911234567892123456789 - more text - -test -""") - - def test_another_long_almost_unsplittable_header(self): - eq = self.ndiffAssertEqual - hstr = """\ -bug demonstration -\t12345678911234567892123456789312345678941234567895123456789612345678971234567898112345678911234567892123456789112345678911234567892123456789 -\tmore text""" - h = Header(hstr, continuation_ws='\t') - eq(h.encode(), """\ -bug demonstration -\t12345678911234567892123456789312345678941234567895123456789612345678971234567898112345678911234567892123456789112345678911234567892123456789 -\tmore text""") - h = Header(hstr) - eq(h.encode(), """\ -bug demonstration - 12345678911234567892123456789312345678941234567895123456789612345678971234567898112345678911234567892123456789112345678911234567892123456789 - more text""") - - def test_long_nonstring(self): - eq = self.ndiffAssertEqual - g = Charset("iso-8859-1") - cz = Charset("iso-8859-2") - utf8 = Charset("utf-8") - g_head = "Die Mieter treten hier ein werden mit einem Foerderband komfortabel den Korridor entlang, an s\xfcdl\xfcndischen Wandgem\xe4lden vorbei, gegen die rotierenden Klingen bef\xf6rdert. " - cz_head = "Finan\xe8ni metropole se hroutily pod tlakem jejich d\xf9vtipu.. " - utf8_head = u"\u6b63\u78ba\u306b\u8a00\u3046\u3068\u7ffb\u8a33\u306f\u3055\u308c\u3066\u3044\u307e\u305b\u3093\u3002\u4e00\u90e8\u306f\u30c9\u30a4\u30c4\u8a9e\u3067\u3059\u304c\u3001\u3042\u3068\u306f\u3067\u305f\u3089\u3081\u3067\u3059\u3002\u5b9f\u969b\u306b\u306f\u300cWenn ist das Nunstuck git und Slotermeyer? Ja! Beiherhund das Oder die Flipperwaldt gersput.\u300d\u3068\u8a00\u3063\u3066\u3044\u307e\u3059\u3002".encode("utf-8") - h = Header(g_head, g, header_name='Subject') - h.append(cz_head, cz) - h.append(utf8_head, utf8) - msg = Message() - msg['Subject'] = h - sfp = StringIO() - g = Generator(sfp) - g.flatten(msg) - eq(sfp.getvalue(), """\ -Subject: =?iso-8859-1?q?Die_Mieter_treten_hier_ein_werden_mit_einem_Foerd?= - =?iso-8859-1?q?erband_komfortabel_den_Korridor_entlang=2C_an_s=FCdl=FCndi?= - =?iso-8859-1?q?schen_Wandgem=E4lden_vorbei=2C_gegen_die_rotierenden_Kling?= - =?iso-8859-1?q?en_bef=F6rdert=2E_?= =?iso-8859-2?q?Finan=E8ni_met?= - =?iso-8859-2?q?ropole_se_hroutily_pod_tlakem_jejich_d=F9vtipu=2E=2E_?= - =?utf-8?b?5q2j56K644Gr6KiA44GG44Go57+76Kiz44Gv44GV44KM44Gm44GE?= - =?utf-8?b?44G+44Gb44KT44CC5LiA6YOo44Gv44OJ44Kk44OE6Kqe44Gn44GZ44GM44CB?= - =?utf-8?b?44GC44Go44Gv44Gn44Gf44KJ44KB44Gn44GZ44CC5a6f6Zqb44Gr44Gv44CM?= - =?utf-8?q?Wenn_ist_das_Nunstuck_git_und_Slotermeyer=3F_Ja!_Beiherhund_das?= - =?utf-8?b?IE9kZXIgZGllIEZsaXBwZXJ3YWxkdCBnZXJzcHV0LuOAjeOBqOiogOOBow==?= - =?utf-8?b?44Gm44GE44G+44GZ44CC?= - -""") - eq(h.encode(), """\ -=?iso-8859-1?q?Die_Mieter_treten_hier_ein_werden_mit_einem_Foerd?= - =?iso-8859-1?q?erband_komfortabel_den_Korridor_entlang=2C_an_s=FCdl=FCndi?= - =?iso-8859-1?q?schen_Wandgem=E4lden_vorbei=2C_gegen_die_rotierenden_Kling?= - =?iso-8859-1?q?en_bef=F6rdert=2E_?= =?iso-8859-2?q?Finan=E8ni_met?= - =?iso-8859-2?q?ropole_se_hroutily_pod_tlakem_jejich_d=F9vtipu=2E=2E_?= - =?utf-8?b?5q2j56K644Gr6KiA44GG44Go57+76Kiz44Gv44GV44KM44Gm44GE?= - =?utf-8?b?44G+44Gb44KT44CC5LiA6YOo44Gv44OJ44Kk44OE6Kqe44Gn44GZ44GM44CB?= - =?utf-8?b?44GC44Go44Gv44Gn44Gf44KJ44KB44Gn44GZ44CC5a6f6Zqb44Gr44Gv44CM?= - =?utf-8?q?Wenn_ist_das_Nunstuck_git_und_Slotermeyer=3F_Ja!_Beiherhund_das?= - =?utf-8?b?IE9kZXIgZGllIEZsaXBwZXJ3YWxkdCBnZXJzcHV0LuOAjeOBqOiogOOBow==?= - =?utf-8?b?44Gm44GE44G+44GZ44CC?=""") - - def test_long_header_encode(self): - eq = self.ndiffAssertEqual - h = Header('wasnipoop; giraffes="very-long-necked-animals"; ' - 'spooge="yummy"; hippos="gargantuan"; marshmallows="gooey"', - header_name='X-Foobar-Spoink-Defrobnit') - eq(h.encode(), '''\ -wasnipoop; giraffes="very-long-necked-animals"; - spooge="yummy"; hippos="gargantuan"; marshmallows="gooey"''') - - def test_long_header_encode_with_tab_continuation(self): - eq = self.ndiffAssertEqual - h = Header('wasnipoop; giraffes="very-long-necked-animals"; ' - 'spooge="yummy"; hippos="gargantuan"; marshmallows="gooey"', - header_name='X-Foobar-Spoink-Defrobnit', - continuation_ws='\t') - eq(h.encode(), '''\ -wasnipoop; giraffes="very-long-necked-animals"; -\tspooge="yummy"; hippos="gargantuan"; marshmallows="gooey"''') - - def test_header_splitter(self): - eq = self.ndiffAssertEqual - msg = MIMEText('') - # It'd be great if we could use add_header() here, but that doesn't - # guarantee an order of the parameters. - msg['X-Foobar-Spoink-Defrobnit'] = ( - 'wasnipoop; giraffes="very-long-necked-animals"; ' - 'spooge="yummy"; hippos="gargantuan"; marshmallows="gooey"') - sfp = StringIO() - g = Generator(sfp) - g.flatten(msg) - eq(sfp.getvalue(), '''\ -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit -X-Foobar-Spoink-Defrobnit: wasnipoop; giraffes="very-long-necked-animals"; - spooge="yummy"; hippos="gargantuan"; marshmallows="gooey" - -''') - - def test_no_semis_header_splitter(self): - eq = self.ndiffAssertEqual - msg = Message() - msg['From'] = 'test at dom.ain' - msg['References'] = SPACE.join(['<%d at dom.ain>' % i for i in range(10)]) - msg.set_payload('Test') - sfp = StringIO() - g = Generator(sfp) - g.flatten(msg) - eq(sfp.getvalue(), """\ -From: test at dom.ain -References: <0 at dom.ain> <1 at dom.ain> <2 at dom.ain> <3 at dom.ain> <4 at dom.ain> - <5 at dom.ain> <6 at dom.ain> <7 at dom.ain> <8 at dom.ain> <9 at dom.ain> - -Test""") - - def test_no_split_long_header(self): - eq = self.ndiffAssertEqual - hstr = 'References: ' + 'x' * 80 - h = Header(hstr, continuation_ws='\t') - eq(h.encode(), """\ -References: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx""") - - def test_splitting_multiple_long_lines(self): - eq = self.ndiffAssertEqual - hstr = """\ -from babylon.socal-raves.org (localhost [127.0.0.1]); by babylon.socal-raves.org (Postfix) with ESMTP id B570E51B81; for <mailman-admin at babylon.socal-raves.org>; Sat, 2 Feb 2002 17:00:06 -0800 (PST) -\tfrom babylon.socal-raves.org (localhost [127.0.0.1]); by babylon.socal-raves.org (Postfix) with ESMTP id B570E51B81; for <mailman-admin at babylon.socal-raves.org>; Sat, 2 Feb 2002 17:00:06 -0800 (PST) -\tfrom babylon.socal-raves.org (localhost [127.0.0.1]); by babylon.socal-raves.org (Postfix) with ESMTP id B570E51B81; for <mailman-admin at babylon.socal-raves.org>; Sat, 2 Feb 2002 17:00:06 -0800 (PST) -""" - h = Header(hstr, continuation_ws='\t') - eq(h.encode(), """\ -from babylon.socal-raves.org (localhost [127.0.0.1]); -\tby babylon.socal-raves.org (Postfix) with ESMTP id B570E51B81; -\tfor <mailman-admin at babylon.socal-raves.org>; -\tSat, 2 Feb 2002 17:00:06 -0800 (PST) -\tfrom babylon.socal-raves.org (localhost [127.0.0.1]); -\tby babylon.socal-raves.org (Postfix) with ESMTP id B570E51B81; -\tfor <mailman-admin at babylon.socal-raves.org>; -\tSat, 2 Feb 2002 17:00:06 -0800 (PST) -\tfrom babylon.socal-raves.org (localhost [127.0.0.1]); -\tby babylon.socal-raves.org (Postfix) with ESMTP id B570E51B81; -\tfor <mailman-admin at babylon.socal-raves.org>; -\tSat, 2 Feb 2002 17:00:06 -0800 (PST)""") - - def test_splitting_first_line_only_is_long(self): - eq = self.ndiffAssertEqual - hstr = """\ -from modemcable093.139-201-24.que.mc.videotron.ca ([24.201.139.93] helo=cthulhu.gerg.ca) -\tby kronos.mems-exchange.org with esmtp (Exim 4.05) -\tid 17k4h5-00034i-00 -\tfor test at mems-exchange.org; Wed, 28 Aug 2002 11:25:20 -0400""" - h = Header(hstr, maxlinelen=78, header_name='Received', - continuation_ws='\t') - eq(h.encode(), """\ -from modemcable093.139-201-24.que.mc.videotron.ca ([24.201.139.93] -\thelo=cthulhu.gerg.ca) -\tby kronos.mems-exchange.org with esmtp (Exim 4.05) -\tid 17k4h5-00034i-00 -\tfor test at mems-exchange.org; Wed, 28 Aug 2002 11:25:20 -0400""") - - def test_long_8bit_header(self): - eq = self.ndiffAssertEqual - msg = Message() - h = Header('Britische Regierung gibt', 'iso-8859-1', - header_name='Subject') - h.append('gr\xfcnes Licht f\xfcr Offshore-Windkraftprojekte') - msg['Subject'] = h - eq(msg.as_string(), """\ -Subject: =?iso-8859-1?q?Britische_Regierung_gibt?= =?iso-8859-1?q?gr=FCnes?= - =?iso-8859-1?q?_Licht_f=FCr_Offshore-Windkraftprojekte?= - -""") - - def test_long_8bit_header_no_charset(self): - eq = self.ndiffAssertEqual - msg = Message() - msg['Reply-To'] = 'Britische Regierung gibt gr\xfcnes Licht f\xfcr Offshore-Windkraftprojekte <a-very-long-address at example.com>' - eq(msg.as_string(), """\ -Reply-To: Britische Regierung gibt gr\xfcnes Licht f\xfcr Offshore-Windkraftprojekte <a-very-long-address at example.com> - -""") - - def test_long_to_header(self): - eq = self.ndiffAssertEqual - to = '"Someone Test #A" <someone at eecs.umich.edu>,<someone at eecs.umich.edu>,"Someone Test #B" <someone at umich.edu>, "Someone Test #C" <someone at eecs.umich.edu>, "Someone Test #D" <someone at eecs.umich.edu>' - msg = Message() - msg['To'] = to - eq(msg.as_string(0), '''\ -To: "Someone Test #A" <someone at eecs.umich.edu>, <someone at eecs.umich.edu>, - "Someone Test #B" <someone at umich.edu>, - "Someone Test #C" <someone at eecs.umich.edu>, - "Someone Test #D" <someone at eecs.umich.edu> - -''') - - def test_long_line_after_append(self): - eq = self.ndiffAssertEqual - s = 'This is an example of string which has almost the limit of header length.' - h = Header(s) - h.append('Add another line.') - eq(h.encode(), """\ -This is an example of string which has almost the limit of header length. - Add another line.""") - - def test_shorter_line_with_append(self): - eq = self.ndiffAssertEqual - s = 'This is a shorter line.' - h = Header(s) - h.append('Add another sentence. (Surprise?)') - eq(h.encode(), - 'This is a shorter line. Add another sentence. (Surprise?)') - - def test_long_field_name(self): - eq = self.ndiffAssertEqual - fn = 'X-Very-Very-Very-Long-Header-Name' - gs = "Die Mieter treten hier ein werden mit einem Foerderband komfortabel den Korridor entlang, an s\xfcdl\xfcndischen Wandgem\xe4lden vorbei, gegen die rotierenden Klingen bef\xf6rdert. " - h = Header(gs, 'iso-8859-1', header_name=fn) - # BAW: this seems broken because the first line is too long - eq(h.encode(), """\ -=?iso-8859-1?q?Die_Mieter_treten_hier_?= - =?iso-8859-1?q?ein_werden_mit_einem_Foerderband_komfortabel_den_Korridor_?= - =?iso-8859-1?q?entlang=2C_an_s=FCdl=FCndischen_Wandgem=E4lden_vorbei=2C_g?= - =?iso-8859-1?q?egen_die_rotierenden_Klingen_bef=F6rdert=2E_?=""") - - def test_long_received_header(self): - h = 'from FOO.TLD (vizworld.acl.foo.tld [123.452.678.9]) by hrothgar.la.mastaler.com (tmda-ofmipd) with ESMTP; Wed, 05 Mar 2003 18:10:18 -0700' - msg = Message() - msg['Received-1'] = Header(h, continuation_ws='\t') - msg['Received-2'] = h - self.assertEqual(msg.as_string(), """\ -Received-1: from FOO.TLD (vizworld.acl.foo.tld [123.452.678.9]) by -\throthgar.la.mastaler.com (tmda-ofmipd) with ESMTP; -\tWed, 05 Mar 2003 18:10:18 -0700 -Received-2: from FOO.TLD (vizworld.acl.foo.tld [123.452.678.9]) by - hrothgar.la.mastaler.com (tmda-ofmipd) with ESMTP; - Wed, 05 Mar 2003 18:10:18 -0700 - -""") - - def test_string_headerinst_eq(self): - h = '<15975.17901.207240.414604 at sgigritzmann1.mathematik.tu-muenchen.de> (David Bremner\'s message of "Thu, 6 Mar 2003 13:58:21 +0100")' - msg = Message() - msg['Received'] = Header(h, header_name='Received', - continuation_ws='\t') - msg['Received'] = h - self.ndiffAssertEqual(msg.as_string(), """\ -Received: <15975.17901.207240.414604 at sgigritzmann1.mathematik.tu-muenchen.de> -\t(David Bremner's message of "Thu, 6 Mar 2003 13:58:21 +0100") -Received: <15975.17901.207240.414604 at sgigritzmann1.mathematik.tu-muenchen.de> - (David Bremner's message of "Thu, 6 Mar 2003 13:58:21 +0100") - -""") - - def test_long_unbreakable_lines_with_continuation(self): - eq = self.ndiffAssertEqual - msg = Message() - t = """\ - iVBORw0KGgoAAAANSUhEUgAAADAAAAAwBAMAAAClLOS0AAAAGFBMVEUAAAAkHiJeRUIcGBi9 - locQDQ4zJykFBAXJfWDjAAACYUlEQVR4nF2TQY/jIAyFc6lydlG5x8Nyp1Y69wj1PN2I5gzp""" - msg['Face-1'] = t - msg['Face-2'] = Header(t, header_name='Face-2') - eq(msg.as_string(), """\ -Face-1: iVBORw0KGgoAAAANSUhEUgAAADAAAAAwBAMAAAClLOS0AAAAGFBMVEUAAAAkHiJeRUIcGBi9 - locQDQ4zJykFBAXJfWDjAAACYUlEQVR4nF2TQY/jIAyFc6lydlG5x8Nyp1Y69wj1PN2I5gzp -Face-2: iVBORw0KGgoAAAANSUhEUgAAADAAAAAwBAMAAAClLOS0AAAAGFBMVEUAAAAkHiJeRUIcGBi9 - locQDQ4zJykFBAXJfWDjAAACYUlEQVR4nF2TQY/jIAyFc6lydlG5x8Nyp1Y69wj1PN2I5gzp - -""") - - def test_another_long_multiline_header(self): - eq = self.ndiffAssertEqual - m = '''\ -Received: from siimage.com ([172.25.1.3]) by zima.siliconimage.com with Microsoft SMTPSVC(5.0.2195.4905); - Wed, 16 Oct 2002 07:41:11 -0700''' - msg = email.message_from_string(m) - eq(msg.as_string(), '''\ -Received: from siimage.com ([172.25.1.3]) by zima.siliconimage.com with - Microsoft SMTPSVC(5.0.2195.4905); Wed, 16 Oct 2002 07:41:11 -0700 - -''') - - def test_long_lines_with_different_header(self): - eq = self.ndiffAssertEqual - h = """\ -List-Unsubscribe: <https://lists.sourceforge.net/lists/listinfo/spamassassin-talk>, - <mailto:spamassassin-talk-request at lists.sourceforge.net?subject=unsubscribe>""" - msg = Message() - msg['List'] = h - msg['List'] = Header(h, header_name='List') - eq(msg.as_string(), """\ -List: List-Unsubscribe: <https://lists.sourceforge.net/lists/listinfo/spamassassin-talk>, - <mailto:spamassassin-talk-request at lists.sourceforge.net?subject=unsubscribe> -List: List-Unsubscribe: <https://lists.sourceforge.net/lists/listinfo/spamassassin-talk>, - <mailto:spamassassin-talk-request at lists.sourceforge.net?subject=unsubscribe> - -""") - - - -# Test mangling of "From " lines in the body of a message -class TestFromMangling(unittest.TestCase): - def setUp(self): - self.msg = Message() - self.msg['From'] = 'aaa at bbb.org' - self.msg.set_payload("""\ -From the desk of A.A.A.: -Blah blah blah -""") - - def test_mangled_from(self): - s = StringIO() - g = Generator(s, mangle_from_=True) - g.flatten(self.msg) - self.assertEqual(s.getvalue(), """\ -From: aaa at bbb.org - ->From the desk of A.A.A.: -Blah blah blah -""") - - def test_dont_mangle_from(self): - s = StringIO() - g = Generator(s, mangle_from_=False) - g.flatten(self.msg) - self.assertEqual(s.getvalue(), """\ -From: aaa at bbb.org - -From the desk of A.A.A.: -Blah blah blah -""") - - - -# Test the basic MIMEAudio class -class TestMIMEAudio(unittest.TestCase): - def setUp(self): - # Make sure we pick up the audiotest.au that lives in email/test/data. - # In Python, there's an audiotest.au living in Lib/test but that isn't - # included in some binary distros that don't include the test - # package. The trailing empty string on the .join() is significant - # since findfile() will do a dirname(). - datadir = os.path.join(os.path.dirname(landmark), 'data', '') - fp = open(findfile('audiotest.au', datadir), 'rb') - try: - self._audiodata = fp.read() - finally: - fp.close() - self._au = MIMEAudio(self._audiodata) - - def test_guess_minor_type(self): - self.assertEqual(self._au.get_content_type(), 'audio/basic') - - def test_encoding(self): - payload = self._au.get_payload() - self.assertEqual(base64.decodestring(payload), self._audiodata) - - def test_checkSetMinor(self): - au = MIMEAudio(self._audiodata, 'fish') - self.assertEqual(au.get_content_type(), 'audio/fish') - - def test_add_header(self): - eq = self.assertEqual - unless = self.assertTrue - self._au.add_header('Content-Disposition', 'attachment', - filename='audiotest.au') - eq(self._au['content-disposition'], - 'attachment; filename="audiotest.au"') - eq(self._au.get_params(header='content-disposition'), - [('attachment', ''), ('filename', 'audiotest.au')]) - eq(self._au.get_param('filename', header='content-disposition'), - 'audiotest.au') - missing = [] - eq(self._au.get_param('attachment', header='content-disposition'), '') - unless(self._au.get_param('foo', failobj=missing, - header='content-disposition') is missing) - # Try some missing stuff - unless(self._au.get_param('foobar', missing) is missing) - unless(self._au.get_param('attachment', missing, - header='foobar') is missing) - - - -# Test the basic MIMEImage class -class TestMIMEImage(unittest.TestCase): - def setUp(self): - fp = openfile('PyBanner048.gif') - try: - self._imgdata = fp.read() - finally: - fp.close() - self._im = MIMEImage(self._imgdata) - - def test_guess_minor_type(self): - self.assertEqual(self._im.get_content_type(), 'image/gif') - - def test_encoding(self): - payload = self._im.get_payload() - self.assertEqual(base64.decodestring(payload), self._imgdata) - - def test_checkSetMinor(self): - im = MIMEImage(self._imgdata, 'fish') - self.assertEqual(im.get_content_type(), 'image/fish') - - def test_add_header(self): - eq = self.assertEqual - unless = self.assertTrue - self._im.add_header('Content-Disposition', 'attachment', - filename='dingusfish.gif') - eq(self._im['content-disposition'], - 'attachment; filename="dingusfish.gif"') - eq(self._im.get_params(header='content-disposition'), - [('attachment', ''), ('filename', 'dingusfish.gif')]) - eq(self._im.get_param('filename', header='content-disposition'), - 'dingusfish.gif') - missing = [] - eq(self._im.get_param('attachment', header='content-disposition'), '') - unless(self._im.get_param('foo', failobj=missing, - header='content-disposition') is missing) - # Try some missing stuff - unless(self._im.get_param('foobar', missing) is missing) - unless(self._im.get_param('attachment', missing, - header='foobar') is missing) - - - -# Test the basic MIMEText class -class TestMIMEText(unittest.TestCase): - def setUp(self): - self._msg = MIMEText('hello there') - - def test_types(self): - eq = self.assertEqual - unless = self.assertTrue - eq(self._msg.get_content_type(), 'text/plain') - eq(self._msg.get_param('charset'), 'us-ascii') - missing = [] - unless(self._msg.get_param('foobar', missing) is missing) - unless(self._msg.get_param('charset', missing, header='foobar') - is missing) - - def test_payload(self): - self.assertEqual(self._msg.get_payload(), 'hello there') - self.assertTrue(not self._msg.is_multipart()) - - def test_charset(self): - eq = self.assertEqual - msg = MIMEText('hello there', _charset='us-ascii') - eq(msg.get_charset().input_charset, 'us-ascii') - eq(msg['content-type'], 'text/plain; charset="us-ascii"') - - def test_7bit_unicode_input(self): - eq = self.assertEqual - msg = MIMEText(u'hello there', _charset='us-ascii') - eq(msg.get_charset().input_charset, 'us-ascii') - eq(msg['content-type'], 'text/plain; charset="us-ascii"') - - def test_7bit_unicode_input_no_charset(self): - eq = self.assertEqual - msg = MIMEText(u'hello there') - eq(msg.get_charset(), 'us-ascii') - eq(msg['content-type'], 'text/plain; charset="us-ascii"') - self.assertTrue('hello there' in msg.as_string()) - - def test_8bit_unicode_input(self): - teststr = u'\u043a\u0438\u0440\u0438\u043b\u0438\u0446\u0430' - eq = self.assertEqual - msg = MIMEText(teststr, _charset='utf-8') - eq(msg.get_charset().output_charset, 'utf-8') - eq(msg['content-type'], 'text/plain; charset="utf-8"') - eq(msg.get_payload(decode=True), teststr.encode('utf-8')) - - def test_8bit_unicode_input_no_charset(self): - teststr = u'\u043a\u0438\u0440\u0438\u043b\u0438\u0446\u0430' - self.assertRaises(UnicodeEncodeError, MIMEText, teststr) - - - -# Test complicated multipart/* messages -class TestMultipart(TestEmailBase): - def setUp(self): - fp = openfile('PyBanner048.gif') - try: - data = fp.read() - finally: - fp.close() - - container = MIMEBase('multipart', 'mixed', boundary='BOUNDARY') - image = MIMEImage(data, name='dingusfish.gif') - image.add_header('content-disposition', 'attachment', - filename='dingusfish.gif') - intro = MIMEText('''\ -Hi there, - -This is the dingus fish. -''') - container.attach(intro) - container.attach(image) - container['From'] = 'Barry <barry at digicool.com>' - container['To'] = 'Dingus Lovers <cravindogs at cravindogs.com>' - container['Subject'] = 'Here is your dingus fish' - - now = 987809702.54848599 - timetuple = time.localtime(now) - if timetuple[-1] == 0: - tzsecs = time.timezone - else: - tzsecs = time.altzone - if tzsecs > 0: - sign = '-' - else: - sign = '+' - tzoffset = ' %s%04d' % (sign, tzsecs // 36) - container['Date'] = time.strftime( - '%a, %d %b %Y %H:%M:%S', - time.localtime(now)) + tzoffset - self._msg = container - self._im = image - self._txt = intro - - def test_hierarchy(self): - # convenience - eq = self.assertEqual - unless = self.assertTrue - raises = self.assertRaises - # tests - m = self._msg - unless(m.is_multipart()) - eq(m.get_content_type(), 'multipart/mixed') - eq(len(m.get_payload()), 2) - raises(IndexError, m.get_payload, 2) - m0 = m.get_payload(0) - m1 = m.get_payload(1) - unless(m0 is self._txt) - unless(m1 is self._im) - eq(m.get_payload(), [m0, m1]) - unless(not m0.is_multipart()) - unless(not m1.is_multipart()) - - def test_empty_multipart_idempotent(self): - text = """\ -Content-Type: multipart/mixed; boundary="BOUNDARY" -MIME-Version: 1.0 -Subject: A subject -To: aperson at dom.ain -From: bperson at dom.ain - - ---BOUNDARY - - ---BOUNDARY-- -""" - msg = Parser().parsestr(text) - self.ndiffAssertEqual(text, msg.as_string()) - - def test_no_parts_in_a_multipart_with_none_epilogue(self): - outer = MIMEBase('multipart', 'mixed') - outer['Subject'] = 'A subject' - outer['To'] = 'aperson at dom.ain' - outer['From'] = 'bperson at dom.ain' - outer.set_boundary('BOUNDARY') - self.ndiffAssertEqual(outer.as_string(), '''\ -Content-Type: multipart/mixed; boundary="BOUNDARY" -MIME-Version: 1.0 -Subject: A subject -To: aperson at dom.ain -From: bperson at dom.ain - ---BOUNDARY - ---BOUNDARY--''') - - def test_no_parts_in_a_multipart_with_empty_epilogue(self): - outer = MIMEBase('multipart', 'mixed') - outer['Subject'] = 'A subject' - outer['To'] = 'aperson at dom.ain' - outer['From'] = 'bperson at dom.ain' - outer.preamble = '' - outer.epilogue = '' - outer.set_boundary('BOUNDARY') - self.ndiffAssertEqual(outer.as_string(), '''\ -Content-Type: multipart/mixed; boundary="BOUNDARY" -MIME-Version: 1.0 -Subject: A subject -To: aperson at dom.ain -From: bperson at dom.ain - - ---BOUNDARY - ---BOUNDARY-- -''') - - def test_one_part_in_a_multipart(self): - eq = self.ndiffAssertEqual - outer = MIMEBase('multipart', 'mixed') - outer['Subject'] = 'A subject' - outer['To'] = 'aperson at dom.ain' - outer['From'] = 'bperson at dom.ain' - outer.set_boundary('BOUNDARY') - msg = MIMEText('hello world') - outer.attach(msg) - eq(outer.as_string(), '''\ -Content-Type: multipart/mixed; boundary="BOUNDARY" -MIME-Version: 1.0 -Subject: A subject -To: aperson at dom.ain -From: bperson at dom.ain - ---BOUNDARY -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit - -hello world ---BOUNDARY--''') - - def test_seq_parts_in_a_multipart_with_empty_preamble(self): - eq = self.ndiffAssertEqual - outer = MIMEBase('multipart', 'mixed') - outer['Subject'] = 'A subject' - outer['To'] = 'aperson at dom.ain' - outer['From'] = 'bperson at dom.ain' - outer.preamble = '' - msg = MIMEText('hello world') - outer.attach(msg) - outer.set_boundary('BOUNDARY') - eq(outer.as_string(), '''\ -Content-Type: multipart/mixed; boundary="BOUNDARY" -MIME-Version: 1.0 -Subject: A subject -To: aperson at dom.ain -From: bperson at dom.ain - - ---BOUNDARY -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit - -hello world ---BOUNDARY--''') - - - def test_seq_parts_in_a_multipart_with_none_preamble(self): - eq = self.ndiffAssertEqual - outer = MIMEBase('multipart', 'mixed') - outer['Subject'] = 'A subject' - outer['To'] = 'aperson at dom.ain' - outer['From'] = 'bperson at dom.ain' - outer.preamble = None - msg = MIMEText('hello world') - outer.attach(msg) - outer.set_boundary('BOUNDARY') - eq(outer.as_string(), '''\ -Content-Type: multipart/mixed; boundary="BOUNDARY" -MIME-Version: 1.0 -Subject: A subject -To: aperson at dom.ain -From: bperson at dom.ain - ---BOUNDARY -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit - -hello world ---BOUNDARY--''') - - - def test_seq_parts_in_a_multipart_with_none_epilogue(self): - eq = self.ndiffAssertEqual - outer = MIMEBase('multipart', 'mixed') - outer['Subject'] = 'A subject' - outer['To'] = 'aperson at dom.ain' - outer['From'] = 'bperson at dom.ain' - outer.epilogue = None - msg = MIMEText('hello world') - outer.attach(msg) - outer.set_boundary('BOUNDARY') - eq(outer.as_string(), '''\ -Content-Type: multipart/mixed; boundary="BOUNDARY" -MIME-Version: 1.0 -Subject: A subject -To: aperson at dom.ain -From: bperson at dom.ain - ---BOUNDARY -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit - -hello world ---BOUNDARY--''') - - - def test_seq_parts_in_a_multipart_with_empty_epilogue(self): - eq = self.ndiffAssertEqual - outer = MIMEBase('multipart', 'mixed') - outer['Subject'] = 'A subject' - outer['To'] = 'aperson at dom.ain' - outer['From'] = 'bperson at dom.ain' - outer.epilogue = '' - msg = MIMEText('hello world') - outer.attach(msg) - outer.set_boundary('BOUNDARY') - eq(outer.as_string(), '''\ -Content-Type: multipart/mixed; boundary="BOUNDARY" -MIME-Version: 1.0 -Subject: A subject -To: aperson at dom.ain -From: bperson at dom.ain - ---BOUNDARY -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit - -hello world ---BOUNDARY-- -''') - - - def test_seq_parts_in_a_multipart_with_nl_epilogue(self): - eq = self.ndiffAssertEqual - outer = MIMEBase('multipart', 'mixed') - outer['Subject'] = 'A subject' - outer['To'] = 'aperson at dom.ain' - outer['From'] = 'bperson at dom.ain' - outer.epilogue = '\n' - msg = MIMEText('hello world') - outer.attach(msg) - outer.set_boundary('BOUNDARY') - eq(outer.as_string(), '''\ -Content-Type: multipart/mixed; boundary="BOUNDARY" -MIME-Version: 1.0 -Subject: A subject -To: aperson at dom.ain -From: bperson at dom.ain - ---BOUNDARY -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit - -hello world ---BOUNDARY-- - -''') - - def test_message_external_body(self): - eq = self.assertEqual - msg = self._msgobj('msg_36.txt') - eq(len(msg.get_payload()), 2) - msg1 = msg.get_payload(1) - eq(msg1.get_content_type(), 'multipart/alternative') - eq(len(msg1.get_payload()), 2) - for subpart in msg1.get_payload(): - eq(subpart.get_content_type(), 'message/external-body') - eq(len(subpart.get_payload()), 1) - subsubpart = subpart.get_payload(0) - eq(subsubpart.get_content_type(), 'text/plain') - - def test_double_boundary(self): - # msg_37.txt is a multipart that contains two dash-boundary's in a - # row. Our interpretation of RFC 2046 calls for ignoring the second - # and subsequent boundaries. - msg = self._msgobj('msg_37.txt') - self.assertEqual(len(msg.get_payload()), 3) - - def test_nested_inner_contains_outer_boundary(self): - eq = self.ndiffAssertEqual - # msg_38.txt has an inner part that contains outer boundaries. My - # interpretation of RFC 2046 (based on sections 5.1 and 5.1.2) say - # these are illegal and should be interpreted as unterminated inner - # parts. - msg = self._msgobj('msg_38.txt') - sfp = StringIO() - Iterators._structure(msg, sfp) - eq(sfp.getvalue(), """\ -multipart/mixed - multipart/mixed - multipart/alternative - text/plain - text/plain - text/plain - text/plain -""") - - def test_nested_with_same_boundary(self): - eq = self.ndiffAssertEqual - # msg 39.txt is similarly evil in that it's got inner parts that use - # the same boundary as outer parts. Again, I believe the way this is - # parsed is closest to the spirit of RFC 2046 - msg = self._msgobj('msg_39.txt') - sfp = StringIO() - Iterators._structure(msg, sfp) - eq(sfp.getvalue(), """\ -multipart/mixed - multipart/mixed - multipart/alternative - application/octet-stream - application/octet-stream - text/plain -""") - - def test_boundary_in_non_multipart(self): - msg = self._msgobj('msg_40.txt') - self.assertEqual(msg.as_string(), '''\ -MIME-Version: 1.0 -Content-Type: text/html; boundary="--961284236552522269" - -----961284236552522269 -Content-Type: text/html; -Content-Transfer-Encoding: 7Bit - -<html></html> - -----961284236552522269-- -''') - - def test_boundary_with_leading_space(self): - eq = self.assertEqual - msg = email.message_from_string('''\ -MIME-Version: 1.0 -Content-Type: multipart/mixed; boundary=" XXXX" - --- XXXX -Content-Type: text/plain - - --- XXXX -Content-Type: text/plain - --- XXXX-- -''') - self.assertTrue(msg.is_multipart()) - eq(msg.get_boundary(), ' XXXX') - eq(len(msg.get_payload()), 2) - - def test_boundary_without_trailing_newline(self): - m = Parser().parsestr("""\ -Content-Type: multipart/mixed; boundary="===============0012394164==" -MIME-Version: 1.0 - ---===============0012394164== -Content-Type: image/file1.jpg -MIME-Version: 1.0 -Content-Transfer-Encoding: base64 - -YXNkZg== ---===============0012394164==--""") - self.assertEqual(m.get_payload(0).get_payload(), 'YXNkZg==') - - - -# Test some badly formatted messages -class TestNonConformant(TestEmailBase): - def test_parse_missing_minor_type(self): - eq = self.assertEqual - msg = self._msgobj('msg_14.txt') - eq(msg.get_content_type(), 'text/plain') - eq(msg.get_content_maintype(), 'text') - eq(msg.get_content_subtype(), 'plain') - - def test_same_boundary_inner_outer(self): - unless = self.assertTrue - msg = self._msgobj('msg_15.txt') - # XXX We can probably eventually do better - inner = msg.get_payload(0) - unless(hasattr(inner, 'defects')) - self.assertEqual(len(inner.defects), 1) - unless(isinstance(inner.defects[0], - Errors.StartBoundaryNotFoundDefect)) - - def test_multipart_no_boundary(self): - unless = self.assertTrue - msg = self._msgobj('msg_25.txt') - unless(isinstance(msg.get_payload(), str)) - self.assertEqual(len(msg.defects), 2) - unless(isinstance(msg.defects[0], Errors.NoBoundaryInMultipartDefect)) - unless(isinstance(msg.defects[1], - Errors.MultipartInvariantViolationDefect)) - - def test_invalid_content_type(self): - eq = self.assertEqual - neq = self.ndiffAssertEqual - msg = Message() - # RFC 2045, $5.2 says invalid yields text/plain - msg['Content-Type'] = 'text' - eq(msg.get_content_maintype(), 'text') - eq(msg.get_content_subtype(), 'plain') - eq(msg.get_content_type(), 'text/plain') - # Clear the old value and try something /really/ invalid - del msg['content-type'] - msg['Content-Type'] = 'foo' - eq(msg.get_content_maintype(), 'text') - eq(msg.get_content_subtype(), 'plain') - eq(msg.get_content_type(), 'text/plain') - # Still, make sure that the message is idempotently generated - s = StringIO() - g = Generator(s) - g.flatten(msg) - neq(s.getvalue(), 'Content-Type: foo\n\n') - - def test_no_start_boundary(self): - eq = self.ndiffAssertEqual - msg = self._msgobj('msg_31.txt') - eq(msg.get_payload(), """\ ---BOUNDARY -Content-Type: text/plain - -message 1 - ---BOUNDARY -Content-Type: text/plain - -message 2 - ---BOUNDARY-- -""") - - def test_no_separating_blank_line(self): - eq = self.ndiffAssertEqual - msg = self._msgobj('msg_35.txt') - eq(msg.as_string(), """\ -From: aperson at dom.ain -To: bperson at dom.ain -Subject: here's something interesting - -counter to RFC 2822, there's no separating newline here -""") - - def test_lying_multipart(self): - unless = self.assertTrue - msg = self._msgobj('msg_41.txt') - unless(hasattr(msg, 'defects')) - self.assertEqual(len(msg.defects), 2) - unless(isinstance(msg.defects[0], Errors.NoBoundaryInMultipartDefect)) - unless(isinstance(msg.defects[1], - Errors.MultipartInvariantViolationDefect)) - - def test_missing_start_boundary(self): - outer = self._msgobj('msg_42.txt') - # The message structure is: - # - # multipart/mixed - # text/plain - # message/rfc822 - # multipart/mixed [*] - # - # [*] This message is missing its start boundary - bad = outer.get_payload(1).get_payload(0) - self.assertEqual(len(bad.defects), 1) - self.assertTrue(isinstance(bad.defects[0], - Errors.StartBoundaryNotFoundDefect)) - - def test_first_line_is_continuation_header(self): - eq = self.assertEqual - m = ' Line 1\nLine 2\nLine 3' - msg = email.message_from_string(m) - eq(msg.keys(), []) - eq(msg.get_payload(), 'Line 2\nLine 3') - eq(len(msg.defects), 1) - self.assertTrue(isinstance(msg.defects[0], - Errors.FirstHeaderLineIsContinuationDefect)) - eq(msg.defects[0].line, ' Line 1\n') - - - - -# Test RFC 2047 header encoding and decoding -class TestRFC2047(unittest.TestCase): - def test_rfc2047_multiline(self): - eq = self.assertEqual - s = """Re: =?mac-iceland?q?r=8Aksm=9Arg=8Cs?= baz - foo bar =?mac-iceland?q?r=8Aksm=9Arg=8Cs?=""" - dh = decode_header(s) - eq(dh, [ - ('Re:', None), - ('r\x8aksm\x9arg\x8cs', 'mac-iceland'), - ('baz foo bar', None), - ('r\x8aksm\x9arg\x8cs', 'mac-iceland')]) - eq(str(make_header(dh)), - """Re: =?mac-iceland?q?r=8Aksm=9Arg=8Cs?= baz foo bar - =?mac-iceland?q?r=8Aksm=9Arg=8Cs?=""") - - def test_whitespace_eater_unicode(self): - eq = self.assertEqual - s = '=?ISO-8859-1?Q?Andr=E9?= Pirard <pirard at dom.ain>' - dh = decode_header(s) - eq(dh, [('Andr\xe9', 'iso-8859-1'), ('Pirard <pirard at dom.ain>', None)]) - hu = unicode(make_header(dh)).encode('latin-1') - eq(hu, 'Andr\xe9 Pirard <pirard at dom.ain>') - - def test_whitespace_eater_unicode_2(self): - eq = self.assertEqual - s = 'The =?iso-8859-1?b?cXVpY2sgYnJvd24gZm94?= jumped over the =?iso-8859-1?b?bGF6eSBkb2c=?=' - dh = decode_header(s) - eq(dh, [('The', None), ('quick brown fox', 'iso-8859-1'), - ('jumped over the', None), ('lazy dog', 'iso-8859-1')]) - hu = make_header(dh).__unicode__() - eq(hu, u'The quick brown fox jumped over the lazy dog') - - def test_rfc2047_without_whitespace(self): - s = 'Sm=?ISO-8859-1?B?9g==?=rg=?ISO-8859-1?B?5Q==?=sbord' - dh = decode_header(s) - self.assertEqual(dh, [(s, None)]) - - def test_rfc2047_with_whitespace(self): - s = 'Sm =?ISO-8859-1?B?9g==?= rg =?ISO-8859-1?B?5Q==?= sbord' - dh = decode_header(s) - self.assertEqual(dh, [('Sm', None), ('\xf6', 'iso-8859-1'), - ('rg', None), ('\xe5', 'iso-8859-1'), - ('sbord', None)]) - - def test_rfc2047_B_bad_padding(self): - s = '=?iso-8859-1?B?%s?=' - data = [ # only test complete bytes - ('dm==', 'v'), ('dm=', 'v'), ('dm', 'v'), - ('dmk=', 'vi'), ('dmk', 'vi') - ] - for q, a in data: - dh = decode_header(s % q) - self.assertEqual(dh, [(a, 'iso-8859-1')]) - - def test_rfc2047_Q_invalid_digits(self): - # issue 10004. - s = '=?iso-8659-1?Q?andr=e9=zz?=' - self.assertEqual(decode_header(s), - [(b'andr\xe9=zz', 'iso-8659-1')]) - - -# Test the MIMEMessage class -class TestMIMEMessage(TestEmailBase): - def setUp(self): - fp = openfile('msg_11.txt') - try: - self._text = fp.read() - finally: - fp.close() - - def test_type_error(self): - self.assertRaises(TypeError, MIMEMessage, 'a plain string') - - def test_valid_argument(self): - eq = self.assertEqual - unless = self.assertTrue - subject = 'A sub-message' - m = Message() - m['Subject'] = subject - r = MIMEMessage(m) - eq(r.get_content_type(), 'message/rfc822') - payload = r.get_payload() - unless(isinstance(payload, list)) - eq(len(payload), 1) - subpart = payload[0] - unless(subpart is m) - eq(subpart['subject'], subject) - - def test_bad_multipart(self): - eq = self.assertEqual - msg1 = Message() - msg1['Subject'] = 'subpart 1' - msg2 = Message() - msg2['Subject'] = 'subpart 2' - r = MIMEMessage(msg1) - self.assertRaises(Errors.MultipartConversionError, r.attach, msg2) - - def test_generate(self): - # First craft the message to be encapsulated - m = Message() - m['Subject'] = 'An enclosed message' - m.set_payload('Here is the body of the message.\n') - r = MIMEMessage(m) - r['Subject'] = 'The enclosing message' - s = StringIO() - g = Generator(s) - g.flatten(r) - self.assertEqual(s.getvalue(), """\ -Content-Type: message/rfc822 -MIME-Version: 1.0 -Subject: The enclosing message - -Subject: An enclosed message - -Here is the body of the message. -""") - - def test_parse_message_rfc822(self): - eq = self.assertEqual - unless = self.assertTrue - msg = self._msgobj('msg_11.txt') - eq(msg.get_content_type(), 'message/rfc822') - payload = msg.get_payload() - unless(isinstance(payload, list)) - eq(len(payload), 1) - submsg = payload[0] - self.assertTrue(isinstance(submsg, Message)) - eq(submsg['subject'], 'An enclosed message') - eq(submsg.get_payload(), 'Here is the body of the message.\n') - - def test_dsn(self): - eq = self.assertEqual - unless = self.assertTrue - # msg 16 is a Delivery Status Notification, see RFC 1894 - msg = self._msgobj('msg_16.txt') - eq(msg.get_content_type(), 'multipart/report') - unless(msg.is_multipart()) - eq(len(msg.get_payload()), 3) - # Subpart 1 is a text/plain, human readable section - subpart = msg.get_payload(0) - eq(subpart.get_content_type(), 'text/plain') - eq(subpart.get_payload(), """\ -This report relates to a message you sent with the following header fields: - - Message-id: <002001c144a6$8752e060$56104586 at oxy.edu> - Date: Sun, 23 Sep 2001 20:10:55 -0700 - From: "Ian T. Henry" <henryi at oxy.edu> - To: SoCal Raves <scr at socal-raves.org> - Subject: [scr] yeah for Ians!! - -Your message cannot be delivered to the following recipients: - - Recipient address: jangel1 at cougar.noc.ucla.edu - Reason: recipient reached disk quota - -""") - # Subpart 2 contains the machine parsable DSN information. It - # consists of two blocks of headers, represented by two nested Message - # objects. - subpart = msg.get_payload(1) - eq(subpart.get_content_type(), 'message/delivery-status') - eq(len(subpart.get_payload()), 2) - # message/delivery-status should treat each block as a bunch of - # headers, i.e. a bunch of Message objects. - dsn1 = subpart.get_payload(0) - unless(isinstance(dsn1, Message)) - eq(dsn1['original-envelope-id'], '0GK500B4HD0888 at cougar.noc.ucla.edu') - eq(dsn1.get_param('dns', header='reporting-mta'), '') - # Try a missing one <wink> - eq(dsn1.get_param('nsd', header='reporting-mta'), None) - dsn2 = subpart.get_payload(1) - unless(isinstance(dsn2, Message)) - eq(dsn2['action'], 'failed') - eq(dsn2.get_params(header='original-recipient'), - [('rfc822', ''), ('jangel1 at cougar.noc.ucla.edu', '')]) - eq(dsn2.get_param('rfc822', header='final-recipient'), '') - # Subpart 3 is the original message - subpart = msg.get_payload(2) - eq(subpart.get_content_type(), 'message/rfc822') - payload = subpart.get_payload() - unless(isinstance(payload, list)) - eq(len(payload), 1) - subsubpart = payload[0] - unless(isinstance(subsubpart, Message)) - eq(subsubpart.get_content_type(), 'text/plain') - eq(subsubpart['message-id'], - '<002001c144a6$8752e060$56104586 at oxy.edu>') - - def test_epilogue(self): - eq = self.ndiffAssertEqual - fp = openfile('msg_21.txt') - try: - text = fp.read() - finally: - fp.close() - msg = Message() - msg['From'] = 'aperson at dom.ain' - msg['To'] = 'bperson at dom.ain' - msg['Subject'] = 'Test' - msg.preamble = 'MIME message' - msg.epilogue = 'End of MIME message\n' - msg1 = MIMEText('One') - msg2 = MIMEText('Two') - msg.add_header('Content-Type', 'multipart/mixed', boundary='BOUNDARY') - msg.attach(msg1) - msg.attach(msg2) - sfp = StringIO() - g = Generator(sfp) - g.flatten(msg) - eq(sfp.getvalue(), text) - - def test_no_nl_preamble(self): - eq = self.ndiffAssertEqual - msg = Message() - msg['From'] = 'aperson at dom.ain' - msg['To'] = 'bperson at dom.ain' - msg['Subject'] = 'Test' - msg.preamble = 'MIME message' - msg.epilogue = '' - msg1 = MIMEText('One') - msg2 = MIMEText('Two') - msg.add_header('Content-Type', 'multipart/mixed', boundary='BOUNDARY') - msg.attach(msg1) - msg.attach(msg2) - eq(msg.as_string(), """\ -From: aperson at dom.ain -To: bperson at dom.ain -Subject: Test -Content-Type: multipart/mixed; boundary="BOUNDARY" - -MIME message ---BOUNDARY -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit - -One ---BOUNDARY -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit - -Two ---BOUNDARY-- -""") - - def test_default_type(self): - eq = self.assertEqual - fp = openfile('msg_30.txt') - try: - msg = email.message_from_file(fp) - finally: - fp.close() - container1 = msg.get_payload(0) - eq(container1.get_default_type(), 'message/rfc822') - eq(container1.get_content_type(), 'message/rfc822') - container2 = msg.get_payload(1) - eq(container2.get_default_type(), 'message/rfc822') - eq(container2.get_content_type(), 'message/rfc822') - container1a = container1.get_payload(0) - eq(container1a.get_default_type(), 'text/plain') - eq(container1a.get_content_type(), 'text/plain') - container2a = container2.get_payload(0) - eq(container2a.get_default_type(), 'text/plain') - eq(container2a.get_content_type(), 'text/plain') - - def test_default_type_with_explicit_container_type(self): - eq = self.assertEqual - fp = openfile('msg_28.txt') - try: - msg = email.message_from_file(fp) - finally: - fp.close() - container1 = msg.get_payload(0) - eq(container1.get_default_type(), 'message/rfc822') - eq(container1.get_content_type(), 'message/rfc822') - container2 = msg.get_payload(1) - eq(container2.get_default_type(), 'message/rfc822') - eq(container2.get_content_type(), 'message/rfc822') - container1a = container1.get_payload(0) - eq(container1a.get_default_type(), 'text/plain') - eq(container1a.get_content_type(), 'text/plain') - container2a = container2.get_payload(0) - eq(container2a.get_default_type(), 'text/plain') - eq(container2a.get_content_type(), 'text/plain') - - def test_default_type_non_parsed(self): - eq = self.assertEqual - neq = self.ndiffAssertEqual - # Set up container - container = MIMEMultipart('digest', 'BOUNDARY') - container.epilogue = '' - # Set up subparts - subpart1a = MIMEText('message 1\n') - subpart2a = MIMEText('message 2\n') - subpart1 = MIMEMessage(subpart1a) - subpart2 = MIMEMessage(subpart2a) - container.attach(subpart1) - container.attach(subpart2) - eq(subpart1.get_content_type(), 'message/rfc822') - eq(subpart1.get_default_type(), 'message/rfc822') - eq(subpart2.get_content_type(), 'message/rfc822') - eq(subpart2.get_default_type(), 'message/rfc822') - neq(container.as_string(0), '''\ -Content-Type: multipart/digest; boundary="BOUNDARY" -MIME-Version: 1.0 - ---BOUNDARY -Content-Type: message/rfc822 -MIME-Version: 1.0 - -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit - -message 1 - ---BOUNDARY -Content-Type: message/rfc822 -MIME-Version: 1.0 - -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit - -message 2 - ---BOUNDARY-- -''') - del subpart1['content-type'] - del subpart1['mime-version'] - del subpart2['content-type'] - del subpart2['mime-version'] - eq(subpart1.get_content_type(), 'message/rfc822') - eq(subpart1.get_default_type(), 'message/rfc822') - eq(subpart2.get_content_type(), 'message/rfc822') - eq(subpart2.get_default_type(), 'message/rfc822') - neq(container.as_string(0), '''\ -Content-Type: multipart/digest; boundary="BOUNDARY" -MIME-Version: 1.0 - ---BOUNDARY - -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit - -message 1 - ---BOUNDARY - -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit - -message 2 - ---BOUNDARY-- -''') - - def test_mime_attachments_in_constructor(self): - eq = self.assertEqual - text1 = MIMEText('') - text2 = MIMEText('') - msg = MIMEMultipart(_subparts=(text1, text2)) - eq(len(msg.get_payload()), 2) - eq(msg.get_payload(0), text1) - eq(msg.get_payload(1), text2) - - def test_default_multipart_constructor(self): - msg = MIMEMultipart() - self.assertTrue(msg.is_multipart()) - - -# A general test of parser->model->generator idempotency. IOW, read a message -# in, parse it into a message object tree, then without touching the tree, -# regenerate the plain text. The original text and the transformed text -# should be identical. Note: that we ignore the Unix-From since that may -# contain a changed date. -class TestIdempotent(TestEmailBase): - def _msgobj(self, filename): - fp = openfile(filename) - try: - data = fp.read() - finally: - fp.close() - msg = email.message_from_string(data) - return msg, data - - def _idempotent(self, msg, text): - eq = self.ndiffAssertEqual - s = StringIO() - g = Generator(s, maxheaderlen=0) - g.flatten(msg) - eq(text, s.getvalue()) - - def test_parse_text_message(self): - eq = self.assertEqual - msg, text = self._msgobj('msg_01.txt') - eq(msg.get_content_type(), 'text/plain') - eq(msg.get_content_maintype(), 'text') - eq(msg.get_content_subtype(), 'plain') - eq(msg.get_params()[1], ('charset', 'us-ascii')) - eq(msg.get_param('charset'), 'us-ascii') - eq(msg.preamble, None) - eq(msg.epilogue, None) - self._idempotent(msg, text) - - def test_parse_untyped_message(self): - eq = self.assertEqual - msg, text = self._msgobj('msg_03.txt') - eq(msg.get_content_type(), 'text/plain') - eq(msg.get_params(), None) - eq(msg.get_param('charset'), None) - self._idempotent(msg, text) - - def test_simple_multipart(self): - msg, text = self._msgobj('msg_04.txt') - self._idempotent(msg, text) - - def test_MIME_digest(self): - msg, text = self._msgobj('msg_02.txt') - self._idempotent(msg, text) - - def test_long_header(self): - msg, text = self._msgobj('msg_27.txt') - self._idempotent(msg, text) - - def test_MIME_digest_with_part_headers(self): - msg, text = self._msgobj('msg_28.txt') - self._idempotent(msg, text) - - def test_mixed_with_image(self): - msg, text = self._msgobj('msg_06.txt') - self._idempotent(msg, text) - - def test_multipart_report(self): - msg, text = self._msgobj('msg_05.txt') - self._idempotent(msg, text) - - def test_dsn(self): - msg, text = self._msgobj('msg_16.txt') - self._idempotent(msg, text) - - def test_preamble_epilogue(self): - msg, text = self._msgobj('msg_21.txt') - self._idempotent(msg, text) - - def test_multipart_one_part(self): - msg, text = self._msgobj('msg_23.txt') - self._idempotent(msg, text) - - def test_multipart_no_parts(self): - msg, text = self._msgobj('msg_24.txt') - self._idempotent(msg, text) - - def test_no_start_boundary(self): - msg, text = self._msgobj('msg_31.txt') - self._idempotent(msg, text) - - def test_rfc2231_charset(self): - msg, text = self._msgobj('msg_32.txt') - self._idempotent(msg, text) - - def test_more_rfc2231_parameters(self): - msg, text = self._msgobj('msg_33.txt') - self._idempotent(msg, text) - - def test_text_plain_in_a_multipart_digest(self): - msg, text = self._msgobj('msg_34.txt') - self._idempotent(msg, text) - - def test_nested_multipart_mixeds(self): - msg, text = self._msgobj('msg_12a.txt') - self._idempotent(msg, text) - - def test_message_external_body_idempotent(self): - msg, text = self._msgobj('msg_36.txt') - self._idempotent(msg, text) - - def test_content_type(self): - eq = self.assertEqual - unless = self.assertTrue - # Get a message object and reset the seek pointer for other tests - msg, text = self._msgobj('msg_05.txt') - eq(msg.get_content_type(), 'multipart/report') - # Test the Content-Type: parameters - params = {} - for pk, pv in msg.get_params(): - params[pk] = pv - eq(params['report-type'], 'delivery-status') - eq(params['boundary'], 'D1690A7AC1.996856090/mail.example.com') - eq(msg.preamble, 'This is a MIME-encapsulated message.\n') - eq(msg.epilogue, '\n') - eq(len(msg.get_payload()), 3) - # Make sure the subparts are what we expect - msg1 = msg.get_payload(0) - eq(msg1.get_content_type(), 'text/plain') - eq(msg1.get_payload(), 'Yadda yadda yadda\n') - msg2 = msg.get_payload(1) - eq(msg2.get_content_type(), 'text/plain') - eq(msg2.get_payload(), 'Yadda yadda yadda\n') - msg3 = msg.get_payload(2) - eq(msg3.get_content_type(), 'message/rfc822') - self.assertTrue(isinstance(msg3, Message)) - payload = msg3.get_payload() - unless(isinstance(payload, list)) - eq(len(payload), 1) - msg4 = payload[0] - unless(isinstance(msg4, Message)) - eq(msg4.get_payload(), 'Yadda yadda yadda\n') - - def test_parser(self): - eq = self.assertEqual - unless = self.assertTrue - msg, text = self._msgobj('msg_06.txt') - # Check some of the outer headers - eq(msg.get_content_type(), 'message/rfc822') - # Make sure the payload is a list of exactly one sub-Message, and that - # that submessage has a type of text/plain - payload = msg.get_payload() - unless(isinstance(payload, list)) - eq(len(payload), 1) - msg1 = payload[0] - self.assertTrue(isinstance(msg1, Message)) - eq(msg1.get_content_type(), 'text/plain') - self.assertTrue(isinstance(msg1.get_payload(), str)) - eq(msg1.get_payload(), '\n') - - - -# Test various other bits of the package's functionality -class TestMiscellaneous(TestEmailBase): - def test_message_from_string(self): - fp = openfile('msg_01.txt') - try: - text = fp.read() - finally: - fp.close() - msg = email.message_from_string(text) - s = StringIO() - # Don't wrap/continue long headers since we're trying to test - # idempotency. - g = Generator(s, maxheaderlen=0) - g.flatten(msg) - self.assertEqual(text, s.getvalue()) - - def test_message_from_file(self): - fp = openfile('msg_01.txt') - try: - text = fp.read() - fp.seek(0) - msg = email.message_from_file(fp) - s = StringIO() - # Don't wrap/continue long headers since we're trying to test - # idempotency. - g = Generator(s, maxheaderlen=0) - g.flatten(msg) - self.assertEqual(text, s.getvalue()) - finally: - fp.close() - - def test_message_from_string_with_class(self): - unless = self.assertTrue - fp = openfile('msg_01.txt') - try: - text = fp.read() - finally: - fp.close() - # Create a subclass - class MyMessage(Message): - pass - - msg = email.message_from_string(text, MyMessage) - unless(isinstance(msg, MyMessage)) - # Try something more complicated - fp = openfile('msg_02.txt') - try: - text = fp.read() - finally: - fp.close() - msg = email.message_from_string(text, MyMessage) - for subpart in msg.walk(): - unless(isinstance(subpart, MyMessage)) - - def test_message_from_file_with_class(self): - unless = self.assertTrue - # Create a subclass - class MyMessage(Message): - pass - - fp = openfile('msg_01.txt') - try: - msg = email.message_from_file(fp, MyMessage) - finally: - fp.close() - unless(isinstance(msg, MyMessage)) - # Try something more complicated - fp = openfile('msg_02.txt') - try: - msg = email.message_from_file(fp, MyMessage) - finally: - fp.close() - for subpart in msg.walk(): - unless(isinstance(subpart, MyMessage)) - - def test__all__(self): - module = __import__('email') - all = module.__all__ - all.sort() - self.assertEqual(all, [ - # Old names - 'Charset', 'Encoders', 'Errors', 'Generator', - 'Header', 'Iterators', 'MIMEAudio', 'MIMEBase', - 'MIMEImage', 'MIMEMessage', 'MIMEMultipart', - 'MIMENonMultipart', 'MIMEText', 'Message', - 'Parser', 'Utils', 'base64MIME', - # new names - 'base64mime', 'charset', 'encoders', 'errors', 'generator', - 'header', 'iterators', 'message', 'message_from_file', - 'message_from_string', 'mime', 'parser', - 'quopriMIME', 'quoprimime', 'utils', - ]) - - def test_formatdate(self): - now = time.time() - self.assertEqual(Utils.parsedate(Utils.formatdate(now))[:6], - time.gmtime(now)[:6]) - - def test_formatdate_localtime(self): - now = time.time() - self.assertEqual( - Utils.parsedate(Utils.formatdate(now, localtime=True))[:6], - time.localtime(now)[:6]) - - def test_formatdate_usegmt(self): - now = time.time() - self.assertEqual( - Utils.formatdate(now, localtime=False), - time.strftime('%a, %d %b %Y %H:%M:%S -0000', time.gmtime(now))) - self.assertEqual( - Utils.formatdate(now, localtime=False, usegmt=True), - time.strftime('%a, %d %b %Y %H:%M:%S GMT', time.gmtime(now))) - - def test_parsedate_none(self): - self.assertEqual(Utils.parsedate(''), None) - - def test_parsedate_compact(self): - # The FWS after the comma is optional - self.assertEqual(Utils.parsedate('Wed,3 Apr 2002 14:58:26 +0800'), - Utils.parsedate('Wed, 3 Apr 2002 14:58:26 +0800')) - - def test_parsedate_no_dayofweek(self): - eq = self.assertEqual - eq(Utils.parsedate_tz('25 Feb 2003 13:47:26 -0800'), - (2003, 2, 25, 13, 47, 26, 0, 1, -1, -28800)) - - def test_parsedate_compact_no_dayofweek(self): - eq = self.assertEqual - eq(Utils.parsedate_tz('5 Feb 2003 13:47:26 -0800'), - (2003, 2, 5, 13, 47, 26, 0, 1, -1, -28800)) - - def test_parsedate_acceptable_to_time_functions(self): - eq = self.assertEqual - timetup = Utils.parsedate('5 Feb 2003 13:47:26 -0800') - t = int(time.mktime(timetup)) - eq(time.localtime(t)[:6], timetup[:6]) - eq(int(time.strftime('%Y', timetup)), 2003) - timetup = Utils.parsedate_tz('5 Feb 2003 13:47:26 -0800') - t = int(time.mktime(timetup[:9])) - eq(time.localtime(t)[:6], timetup[:6]) - eq(int(time.strftime('%Y', timetup[:9])), 2003) - - def test_parsedate_y2k(self): - """Test for parsing a date with a two-digit year. - - Parsing a date with a two-digit year should return the correct - four-digit year. RFC822 allows two-digit years, but RFC2822 (which - obsoletes RFC822) requires four-digit years. - - """ - self.assertEqual(Utils.parsedate_tz('25 Feb 03 13:47:26 -0800'), - Utils.parsedate_tz('25 Feb 2003 13:47:26 -0800')) - self.assertEqual(Utils.parsedate_tz('25 Feb 71 13:47:26 -0800'), - Utils.parsedate_tz('25 Feb 1971 13:47:26 -0800')) - - def test_parseaddr_empty(self): - self.assertEqual(Utils.parseaddr('<>'), ('', '')) - self.assertEqual(Utils.formataddr(Utils.parseaddr('<>')), '') - - def test_noquote_dump(self): - self.assertEqual( - Utils.formataddr(('A Silly Person', 'person at dom.ain')), - 'A Silly Person <person at dom.ain>') - - def test_escape_dump(self): - self.assertEqual( - Utils.formataddr(('A (Very) Silly Person', 'person at dom.ain')), - r'"A $Very$ Silly Person" <person at dom.ain>') - a = r'A $Special$ Person' - b = 'person at dom.ain' - self.assertEqual(Utils.parseaddr(Utils.formataddr((a, b))), (a, b)) - - def test_escape_backslashes(self): - self.assertEqual( - Utils.formataddr(('Arthur \Backslash\ Foobar', 'person at dom.ain')), - r'"Arthur \\Backslash\\ Foobar" <person at dom.ain>') - a = r'Arthur \Backslash\ Foobar' - b = 'person at dom.ain' - self.assertEqual(Utils.parseaddr(Utils.formataddr((a, b))), (a, b)) - - def test_name_with_dot(self): - x = 'John X. Doe <jxd at example.com>' - y = '"John X. Doe" <jxd at example.com>' - a, b = ('John X. Doe', 'jxd at example.com') - self.assertEqual(Utils.parseaddr(x), (a, b)) - self.assertEqual(Utils.parseaddr(y), (a, b)) - # formataddr() quotes the name if there's a dot in it - self.assertEqual(Utils.formataddr((a, b)), y) - - def test_parseaddr_preserves_quoted_pairs_in_addresses(self): - # issue 10005. Note that in the third test the second pair of - # backslashes is not actually a quoted pair because it is not inside a - # comment or quoted string: the address being parsed has a quoted - # string containing a quoted backslash, followed by 'example' and two - # backslashes, followed by another quoted string containing a space and - # the word 'example'. parseaddr copies those two backslashes - # literally. Per rfc5322 this is not technically correct since a \ may - # not appear in an address outside of a quoted string. It is probably - # a sensible Postel interpretation, though. - eq = self.assertEqual - eq(Utils.parseaddr('""example" example"@example.com'), - ('', '""example" example"@example.com')) - eq(Utils.parseaddr('"\\"example\\" example"@example.com'), - ('', '"\\"example\\" example"@example.com')) - eq(Utils.parseaddr('"\\\\"example\\\\" example"@example.com'), - ('', '"\\\\"example\\\\" example"@example.com')) - - def test_multiline_from_comment(self): - x = """\ -Foo -\tBar <foo at example.com>""" - self.assertEqual(Utils.parseaddr(x), ('Foo Bar', 'foo at example.com')) - - def test_quote_dump(self): - self.assertEqual( - Utils.formataddr(('A Silly; Person', 'person at dom.ain')), - r'"A Silly; Person" <person at dom.ain>') - - def test_fix_eols(self): - eq = self.assertEqual - eq(Utils.fix_eols('hello'), 'hello') - eq(Utils.fix_eols('hello\n'), 'hello\r\n') - eq(Utils.fix_eols('hello\r'), 'hello\r\n') - eq(Utils.fix_eols('hello\r\n'), 'hello\r\n') - eq(Utils.fix_eols('hello\n\r'), 'hello\r\n\r\n') - - def test_charset_richcomparisons(self): - eq = self.assertEqual - ne = self.assertNotEqual - cset1 = Charset() - cset2 = Charset() - eq(cset1, 'us-ascii') - eq(cset1, 'US-ASCII') - eq(cset1, 'Us-AsCiI') - eq('us-ascii', cset1) - eq('US-ASCII', cset1) - eq('Us-AsCiI', cset1) - ne(cset1, 'usascii') - ne(cset1, 'USASCII') - ne(cset1, 'UsAsCiI') - ne('usascii', cset1) - ne('USASCII', cset1) - ne('UsAsCiI', cset1) - eq(cset1, cset2) - eq(cset2, cset1) - - def test_getaddresses(self): - eq = self.assertEqual - eq(Utils.getaddresses(['aperson at dom.ain (Al Person)', - 'Bud Person <bperson at dom.ain>']), - [('Al Person', 'aperson at dom.ain'), - ('Bud Person', 'bperson at dom.ain')]) - - def test_getaddresses_nasty(self): - eq = self.assertEqual - eq(Utils.getaddresses(['foo: ;']), [('', '')]) - eq(Utils.getaddresses( - ['[]*-- =~$']), - [('', ''), ('', ''), ('', '*--')]) - eq(Utils.getaddresses( - ['foo: ;', '"Jason R. Mastaler" <jason at dom.ain>']), - [('', ''), ('Jason R. Mastaler', 'jason at dom.ain')]) - - def test_getaddresses_embedded_comment(self): - """Test proper handling of a nested comment""" - eq = self.assertEqual - addrs = Utils.getaddresses(['User ((nested comment)) <foo at bar.com>']) - eq(addrs[0][1], 'foo at bar.com') - - def test_utils_quote_unquote(self): - eq = self.assertEqual - msg = Message() - msg.add_header('content-disposition', 'attachment', - filename='foo\\wacky"name') - eq(msg.get_filename(), 'foo\\wacky"name') - - def test_get_body_encoding_with_bogus_charset(self): - charset = Charset('not a charset') - self.assertEqual(charset.get_body_encoding(), 'base64') - - def test_get_body_encoding_with_uppercase_charset(self): - eq = self.assertEqual - msg = Message() - msg['Content-Type'] = 'text/plain; charset=UTF-8' - eq(msg['content-type'], 'text/plain; charset=UTF-8') - charsets = msg.get_charsets() - eq(len(charsets), 1) - eq(charsets[0], 'utf-8') - charset = Charset(charsets[0]) - eq(charset.get_body_encoding(), 'base64') - msg.set_payload('hello world', charset=charset) - eq(msg.get_payload(), 'aGVsbG8gd29ybGQ=\n') - eq(msg.get_payload(decode=True), 'hello world') - eq(msg['content-transfer-encoding'], 'base64') - # Try another one - msg = Message() - msg['Content-Type'] = 'text/plain; charset="US-ASCII"' - charsets = msg.get_charsets() - eq(len(charsets), 1) - eq(charsets[0], 'us-ascii') - charset = Charset(charsets[0]) - eq(charset.get_body_encoding(), Encoders.encode_7or8bit) - msg.set_payload('hello world', charset=charset) - eq(msg.get_payload(), 'hello world') - eq(msg['content-transfer-encoding'], '7bit') - - def test_charsets_case_insensitive(self): - lc = Charset('us-ascii') - uc = Charset('US-ASCII') - self.assertEqual(lc.get_body_encoding(), uc.get_body_encoding()) - - def test_partial_falls_inside_message_delivery_status(self): - eq = self.ndiffAssertEqual - # The Parser interface provides chunks of data to FeedParser in 8192 - # byte gulps. SF bug #1076485 found one of those chunks inside - # message/delivery-status header block, which triggered an - # unreadline() of NeedMoreData. - msg = self._msgobj('msg_43.txt') - sfp = StringIO() - Iterators._structure(msg, sfp) - eq(sfp.getvalue(), """\ -multipart/report - text/plain - message/delivery-status - text/plain - text/plain - text/plain - text/plain - text/plain - text/plain - text/plain - text/plain - text/plain - text/plain - text/plain - text/plain - text/plain - text/plain - text/plain - text/plain - text/plain - text/plain - text/plain - text/plain - text/plain - text/plain - text/plain - text/plain - text/plain - text/plain - text/rfc822-headers -""") - - - -# Test the iterator/generators -class TestIterators(TestEmailBase): - def test_body_line_iterator(self): - eq = self.assertEqual - neq = self.ndiffAssertEqual - # First a simple non-multipart message - msg = self._msgobj('msg_01.txt') - it = Iterators.body_line_iterator(msg) - lines = list(it) - eq(len(lines), 6) - neq(EMPTYSTRING.join(lines), msg.get_payload()) - # Now a more complicated multipart - msg = self._msgobj('msg_02.txt') - it = Iterators.body_line_iterator(msg) - lines = list(it) - eq(len(lines), 43) - fp = openfile('msg_19.txt') - try: - neq(EMPTYSTRING.join(lines), fp.read()) - finally: - fp.close() - - def test_typed_subpart_iterator(self): - eq = self.assertEqual - msg = self._msgobj('msg_04.txt') - it = Iterators.typed_subpart_iterator(msg, 'text') - lines = [] - subparts = 0 - for subpart in it: - subparts += 1 - lines.append(subpart.get_payload()) - eq(subparts, 2) - eq(EMPTYSTRING.join(lines), """\ -a simple kind of mirror -to reflect upon our own -a simple kind of mirror -to reflect upon our own -""") - - def test_typed_subpart_iterator_default_type(self): - eq = self.assertEqual - msg = self._msgobj('msg_03.txt') - it = Iterators.typed_subpart_iterator(msg, 'text', 'plain') - lines = [] - subparts = 0 - for subpart in it: - subparts += 1 - lines.append(subpart.get_payload()) - eq(subparts, 1) - eq(EMPTYSTRING.join(lines), """\ - -Hi, - -Do you like this message? - --Me -""") - - def test_pushCR_LF(self): - '''FeedParser BufferedSubFile.push() assumed it received complete - line endings. A CR ending one push() followed by a LF starting - the next push() added an empty line. - ''' - imt = [ - ("a\r \n", 2), - ("b", 0), - ("c\n", 1), - ("", 0), - ("d\r\n", 1), - ("e\r", 0), - ("\nf", 1), - ("\r\n", 1), - ] - from email.feedparser import BufferedSubFile, NeedMoreData - bsf = BufferedSubFile() - om = [] - nt = 0 - for il, n in imt: - bsf.push(il) - nt += n - n1 = 0 - while True: - ol = bsf.readline() - if ol == NeedMoreData: - break - om.append(ol) - n1 += 1 - self.assertTrue(n == n1) - self.assertTrue(len(om) == nt) - self.assertTrue(''.join([il for il, n in imt]) == ''.join(om)) - - - -class TestParsers(TestEmailBase): - def test_header_parser(self): - eq = self.assertEqual - # Parse only the headers of a complex multipart MIME document - fp = openfile('msg_02.txt') - try: - msg = HeaderParser().parse(fp) - finally: - fp.close() - eq(msg['from'], 'ppp-request at zzz.org') - eq(msg['to'], 'ppp at zzz.org') - eq(msg.get_content_type(), 'multipart/mixed') - self.assertFalse(msg.is_multipart()) - self.assertTrue(isinstance(msg.get_payload(), str)) - - def test_whitespace_continuation(self): - eq = self.assertEqual - # This message contains a line after the Subject: header that has only - # whitespace, but it is not empty! - msg = email.message_from_string("""\ -From: aperson at dom.ain -To: bperson at dom.ain -Subject: the next line has a space on it -\x20 -Date: Mon, 8 Apr 2002 15:09:19 -0400 -Message-ID: spam - -Here's the message body -""") - eq(msg['subject'], 'the next line has a space on it\n ') - eq(msg['message-id'], 'spam') - eq(msg.get_payload(), "Here's the message body\n") - - def test_whitespace_continuation_last_header(self): - eq = self.assertEqual - # Like the previous test, but the subject line is the last - # header. - msg = email.message_from_string("""\ -From: aperson at dom.ain -To: bperson at dom.ain -Date: Mon, 8 Apr 2002 15:09:19 -0400 -Message-ID: spam -Subject: the next line has a space on it -\x20 - -Here's the message body -""") - eq(msg['subject'], 'the next line has a space on it\n ') - eq(msg['message-id'], 'spam') - eq(msg.get_payload(), "Here's the message body\n") - - def test_crlf_separation(self): - eq = self.assertEqual - fp = openfile('msg_26.txt', mode='rb') - try: - msg = Parser().parse(fp) - finally: - fp.close() - eq(len(msg.get_payload()), 2) - part1 = msg.get_payload(0) - eq(part1.get_content_type(), 'text/plain') - eq(part1.get_payload(), 'Simple email with attachment.\r\n\r\n') - part2 = msg.get_payload(1) - eq(part2.get_content_type(), 'application/riscos') - - def test_multipart_digest_with_extra_mime_headers(self): - eq = self.assertEqual - neq = self.ndiffAssertEqual - fp = openfile('msg_28.txt') - try: - msg = email.message_from_file(fp) - finally: - fp.close() - # Structure is: - # multipart/digest - # message/rfc822 - # text/plain - # message/rfc822 - # text/plain - eq(msg.is_multipart(), 1) - eq(len(msg.get_payload()), 2) - part1 = msg.get_payload(0) - eq(part1.get_content_type(), 'message/rfc822') - eq(part1.is_multipart(), 1) - eq(len(part1.get_payload()), 1) - part1a = part1.get_payload(0) - eq(part1a.is_multipart(), 0) - eq(part1a.get_content_type(), 'text/plain') - neq(part1a.get_payload(), 'message 1\n') - # next message/rfc822 - part2 = msg.get_payload(1) - eq(part2.get_content_type(), 'message/rfc822') - eq(part2.is_multipart(), 1) - eq(len(part2.get_payload()), 1) - part2a = part2.get_payload(0) - eq(part2a.is_multipart(), 0) - eq(part2a.get_content_type(), 'text/plain') - neq(part2a.get_payload(), 'message 2\n') - - def test_three_lines(self): - # A bug report by Andrew McNamara - lines = ['From: Andrew Person <aperson at dom.ain', - 'Subject: Test', - 'Date: Tue, 20 Aug 2002 16:43:45 +1000'] - msg = email.message_from_string(NL.join(lines)) - self.assertEqual(msg['date'], 'Tue, 20 Aug 2002 16:43:45 +1000') - - def test_strip_line_feed_and_carriage_return_in_headers(self): - eq = self.assertEqual - # For [ 1002475 ] email message parser doesn't handle \r\n correctly - value1 = 'text' - value2 = 'more text' - m = 'Header: %s\r\nNext-Header: %s\r\n\r\nBody\r\n\r\n' % ( - value1, value2) - msg = email.message_from_string(m) - eq(msg.get('Header'), value1) - eq(msg.get('Next-Header'), value2) - - def test_rfc2822_header_syntax(self): - eq = self.assertEqual - m = '>From: foo\nFrom: bar\n!"#QUX;~: zoo\n\nbody' - msg = email.message_from_string(m) - eq(len(msg.keys()), 3) - keys = msg.keys() - keys.sort() - eq(keys, ['!"#QUX;~', '>From', 'From']) - eq(msg.get_payload(), 'body') - - def test_rfc2822_space_not_allowed_in_header(self): - eq = self.assertEqual - m = '>From foo at example.com 11:25:53\nFrom: bar\n!"#QUX;~: zoo\n\nbody' - msg = email.message_from_string(m) - eq(len(msg.keys()), 0) - - def test_rfc2822_one_character_header(self): - eq = self.assertEqual - m = 'A: first header\nB: second header\nCC: third header\n\nbody' - msg = email.message_from_string(m) - headers = msg.keys() - headers.sort() - eq(headers, ['A', 'B', 'CC']) - eq(msg.get_payload(), 'body') - - def test_CRLFLF_at_end_of_part(self): - # issue 5610: feedparser should not eat two chars from body part ending - # with "\r\n\n". - m = ( - "From: foo at bar.com\n" - "To: baz\n" - "Mime-Version: 1.0\n" - "Content-Type: multipart/mixed; boundary=BOUNDARY\n" - "\n" - "--BOUNDARY\n" - "Content-Type: text/plain\n" - "\n" - "body ending with CRLF newline\r\n" - "\n" - "--BOUNDARY--\n" - ) - msg = email.message_from_string(m) - self.assertTrue(msg.get_payload(0).get_payload().endswith('\r\n')) - - -class TestBase64(unittest.TestCase): - def test_len(self): - eq = self.assertEqual - eq(base64MIME.base64_len('hello'), - len(base64MIME.encode('hello', eol=''))) - for size in range(15): - if size == 0 : bsize = 0 - elif size <= 3 : bsize = 4 - elif size <= 6 : bsize = 8 - elif size <= 9 : bsize = 12 - elif size <= 12: bsize = 16 - else : bsize = 20 - eq(base64MIME.base64_len('x'*size), bsize) - - def test_decode(self): - eq = self.assertEqual - eq(base64MIME.decode(''), '') - eq(base64MIME.decode('aGVsbG8='), 'hello') - eq(base64MIME.decode('aGVsbG8=', 'X'), 'hello') - eq(base64MIME.decode('aGVsbG8NCndvcmxk\n', 'X'), 'helloXworld') - - def test_encode(self): - eq = self.assertEqual - eq(base64MIME.encode(''), '') - eq(base64MIME.encode('hello'), 'aGVsbG8=\n') - # Test the binary flag - eq(base64MIME.encode('hello\n'), 'aGVsbG8K\n') - eq(base64MIME.encode('hello\n', 0), 'aGVsbG8NCg==\n') - # Test the maxlinelen arg - eq(base64MIME.encode('xxxx ' * 20, maxlinelen=40), """\ -eHh4eCB4eHh4IHh4eHggeHh4eCB4eHh4IHh4eHgg -eHh4eCB4eHh4IHh4eHggeHh4eCB4eHh4IHh4eHgg -eHh4eCB4eHh4IHh4eHggeHh4eCB4eHh4IHh4eHgg -eHh4eCB4eHh4IA== -""") - # Test the eol argument - eq(base64MIME.encode('xxxx ' * 20, maxlinelen=40, eol='\r\n'), """\ -eHh4eCB4eHh4IHh4eHggeHh4eCB4eHh4IHh4eHgg\r -eHh4eCB4eHh4IHh4eHggeHh4eCB4eHh4IHh4eHgg\r -eHh4eCB4eHh4IHh4eHggeHh4eCB4eHh4IHh4eHgg\r -eHh4eCB4eHh4IA==\r -""") - - def test_header_encode(self): - eq = self.assertEqual - he = base64MIME.header_encode - eq(he('hello'), '=?iso-8859-1?b?aGVsbG8=?=') - eq(he('hello\nworld'), '=?iso-8859-1?b?aGVsbG8NCndvcmxk?=') - # Test the charset option - eq(he('hello', charset='iso-8859-2'), '=?iso-8859-2?b?aGVsbG8=?=') - # Test the keep_eols flag - eq(he('hello\nworld', keep_eols=True), - '=?iso-8859-1?b?aGVsbG8Kd29ybGQ=?=') - # Test the maxlinelen argument - eq(he('xxxx ' * 20, maxlinelen=40), """\ -=?iso-8859-1?b?eHh4eCB4eHh4IHh4eHggeHg=?= - =?iso-8859-1?b?eHggeHh4eCB4eHh4IHh4eHg=?= - =?iso-8859-1?b?IHh4eHggeHh4eCB4eHh4IHg=?= - =?iso-8859-1?b?eHh4IHh4eHggeHh4eCB4eHg=?= - =?iso-8859-1?b?eCB4eHh4IHh4eHggeHh4eCA=?= - =?iso-8859-1?b?eHh4eCB4eHh4IHh4eHgg?=""") - # Test the eol argument - eq(he('xxxx ' * 20, maxlinelen=40, eol='\r\n'), """\ -=?iso-8859-1?b?eHh4eCB4eHh4IHh4eHggeHg=?=\r - =?iso-8859-1?b?eHggeHh4eCB4eHh4IHh4eHg=?=\r - =?iso-8859-1?b?IHh4eHggeHh4eCB4eHh4IHg=?=\r - =?iso-8859-1?b?eHh4IHh4eHggeHh4eCB4eHg=?=\r - =?iso-8859-1?b?eCB4eHh4IHh4eHggeHh4eCA=?=\r - =?iso-8859-1?b?eHh4eCB4eHh4IHh4eHgg?=""") - - - -class TestQuopri(unittest.TestCase): - def setUp(self): - self.hlit = [chr(x) for x in range(ord('a'), ord('z')+1)] + \ - [chr(x) for x in range(ord('A'), ord('Z')+1)] + \ - [chr(x) for x in range(ord('0'), ord('9')+1)] + \ - ['!', '*', '+', '-', '/', ' '] - self.hnon = [chr(x) for x in range(256) if chr(x) not in self.hlit] - assert len(self.hlit) + len(self.hnon) == 256 - self.blit = [chr(x) for x in range(ord(' '), ord('~')+1)] + ['\t'] - self.blit.remove('=') - self.bnon = [chr(x) for x in range(256) if chr(x) not in self.blit] - assert len(self.blit) + len(self.bnon) == 256 - - def test_header_quopri_check(self): - for c in self.hlit: - self.assertFalse(quopriMIME.header_quopri_check(c)) - for c in self.hnon: - self.assertTrue(quopriMIME.header_quopri_check(c)) - - def test_body_quopri_check(self): - for c in self.blit: - self.assertFalse(quopriMIME.body_quopri_check(c)) - for c in self.bnon: - self.assertTrue(quopriMIME.body_quopri_check(c)) - - def test_header_quopri_len(self): - eq = self.assertEqual - hql = quopriMIME.header_quopri_len - enc = quopriMIME.header_encode - for s in ('hello', 'h at e@l at l@o@'): - # Empty charset and no line-endings. 7 == RFC chrome - eq(hql(s), len(enc(s, charset='', eol=''))-7) - for c in self.hlit: - eq(hql(c), 1) - for c in self.hnon: - eq(hql(c), 3) - - def test_body_quopri_len(self): - eq = self.assertEqual - bql = quopriMIME.body_quopri_len - for c in self.blit: - eq(bql(c), 1) - for c in self.bnon: - eq(bql(c), 3) - - def test_quote_unquote_idempotent(self): - for x in range(256): - c = chr(x) - self.assertEqual(quopriMIME.unquote(quopriMIME.quote(c)), c) - - def test_header_encode(self): - eq = self.assertEqual - he = quopriMIME.header_encode - eq(he('hello'), '=?iso-8859-1?q?hello?=') - eq(he('hello\nworld'), '=?iso-8859-1?q?hello=0D=0Aworld?=') - # Test the charset option - eq(he('hello', charset='iso-8859-2'), '=?iso-8859-2?q?hello?=') - # Test the keep_eols flag - eq(he('hello\nworld', keep_eols=True), '=?iso-8859-1?q?hello=0Aworld?=') - # Test a non-ASCII character - eq(he('hello\xc7there'), '=?iso-8859-1?q?hello=C7there?=') - # Test the maxlinelen argument - eq(he('xxxx ' * 20, maxlinelen=40), """\ -=?iso-8859-1?q?xxxx_xxxx_xxxx_xxxx_xx?= - =?iso-8859-1?q?xx_xxxx_xxxx_xxxx_xxxx?= - =?iso-8859-1?q?_xxxx_xxxx_xxxx_xxxx_x?= - =?iso-8859-1?q?xxx_xxxx_xxxx_xxxx_xxx?= - =?iso-8859-1?q?x_xxxx_xxxx_?=""") - # Test the eol argument - eq(he('xxxx ' * 20, maxlinelen=40, eol='\r\n'), """\ -=?iso-8859-1?q?xxxx_xxxx_xxxx_xxxx_xx?=\r - =?iso-8859-1?q?xx_xxxx_xxxx_xxxx_xxxx?=\r - =?iso-8859-1?q?_xxxx_xxxx_xxxx_xxxx_x?=\r - =?iso-8859-1?q?xxx_xxxx_xxxx_xxxx_xxx?=\r - =?iso-8859-1?q?x_xxxx_xxxx_?=""") - - def test_decode(self): - eq = self.assertEqual - eq(quopriMIME.decode(''), '') - eq(quopriMIME.decode('hello'), 'hello') - eq(quopriMIME.decode('hello', 'X'), 'hello') - eq(quopriMIME.decode('hello\nworld', 'X'), 'helloXworld') - - def test_encode(self): - eq = self.assertEqual - eq(quopriMIME.encode(''), '') - eq(quopriMIME.encode('hello'), 'hello') - # Test the binary flag - eq(quopriMIME.encode('hello\r\nworld'), 'hello\nworld') - eq(quopriMIME.encode('hello\r\nworld', 0), 'hello\nworld') - # Test the maxlinelen arg - eq(quopriMIME.encode('xxxx ' * 20, maxlinelen=40), """\ -xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxxx= - xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxx= -x xxxx xxxx xxxx xxxx=20""") - # Test the eol argument - eq(quopriMIME.encode('xxxx ' * 20, maxlinelen=40, eol='\r\n'), """\ -xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxxx=\r - xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxx=\r -x xxxx xxxx xxxx xxxx=20""") - eq(quopriMIME.encode("""\ -one line - -two line"""), """\ -one line - -two line""") - - - -# Test the Charset class -class TestCharset(unittest.TestCase): - def tearDown(self): - from email import Charset as CharsetModule - try: - del CharsetModule.CHARSETS['fake'] - except KeyError: - pass - - def test_idempotent(self): - eq = self.assertEqual - # Make sure us-ascii = no Unicode conversion - c = Charset('us-ascii') - s = 'Hello World!' - sp = c.to_splittable(s) - eq(s, c.from_splittable(sp)) - # test 8-bit idempotency with us-ascii - s = '\xa4\xa2\xa4\xa4\xa4\xa6\xa4\xa8\xa4\xaa' - sp = c.to_splittable(s) - eq(s, c.from_splittable(sp)) - - def test_body_encode(self): - eq = self.assertEqual - # Try a charset with QP body encoding - c = Charset('iso-8859-1') - eq('hello w=F6rld', c.body_encode('hello w\xf6rld')) - # Try a charset with Base64 body encoding - c = Charset('utf-8') - eq('aGVsbG8gd29ybGQ=\n', c.body_encode('hello world')) - # Try a charset with None body encoding - c = Charset('us-ascii') - eq('hello world', c.body_encode('hello world')) - # Try the convert argument, where input codec != output codec - c = Charset('euc-jp') - # With apologies to Tokio Kikuchi ;) - try: - eq('\x1b$B5FCO;~IW\x1b(B', - c.body_encode('\xb5\xc6\xc3\xcf\xbb\xfe\xc9\xd7')) - eq('\xb5\xc6\xc3\xcf\xbb\xfe\xc9\xd7', - c.body_encode('\xb5\xc6\xc3\xcf\xbb\xfe\xc9\xd7', False)) - except LookupError: - # We probably don't have the Japanese codecs installed - pass - # Testing SF bug #625509, which we have to fake, since there are no - # built-in encodings where the header encoding is QP but the body - # encoding is not. - from email import Charset as CharsetModule - CharsetModule.add_charset('fake', CharsetModule.QP, None) - c = Charset('fake') - eq('hello w\xf6rld', c.body_encode('hello w\xf6rld')) - - def test_unicode_charset_name(self): - charset = Charset(u'us-ascii') - self.assertEqual(str(charset), 'us-ascii') - self.assertRaises(Errors.CharsetError, Charset, 'asc\xffii') - - def test_codecs_aliases_accepted(self): - charset = Charset('utf8') - self.assertEqual(str(charset), 'utf-8') - - -# Test multilingual MIME headers. -class TestHeader(TestEmailBase): - def test_simple(self): - eq = self.ndiffAssertEqual - h = Header('Hello World!') - eq(h.encode(), 'Hello World!') - h.append(' Goodbye World!') - eq(h.encode(), 'Hello World! Goodbye World!') - - def test_simple_surprise(self): - eq = self.ndiffAssertEqual - h = Header('Hello World!') - eq(h.encode(), 'Hello World!') - h.append('Goodbye World!') - eq(h.encode(), 'Hello World! Goodbye World!') - - def test_header_needs_no_decoding(self): - h = 'no decoding needed' - self.assertEqual(decode_header(h), [(h, None)]) - - def test_long(self): - h = Header("I am the very model of a modern Major-General; I've information vegetable, animal, and mineral; I know the kings of England, and I quote the fights historical from Marathon to Waterloo, in order categorical; I'm very well acquainted, too, with matters mathematical; I understand equations, both the simple and quadratical; about binomial theorem I'm teeming with a lot o' news, with many cheerful facts about the square of the hypotenuse.", - maxlinelen=76) - for l in h.encode(splitchars=' ').split('\n '): - self.assertTrue(len(l) <= 76) - - def test_multilingual(self): - eq = self.ndiffAssertEqual - g = Charset("iso-8859-1") - cz = Charset("iso-8859-2") - utf8 = Charset("utf-8") - g_head = "Die Mieter treten hier ein werden mit einem Foerderband komfortabel den Korridor entlang, an s\xfcdl\xfcndischen Wandgem\xe4lden vorbei, gegen die rotierenden Klingen bef\xf6rdert. " - cz_head = "Finan\xe8ni metropole se hroutily pod tlakem jejich d\xf9vtipu.. " - utf8_head = u"\u6b63\u78ba\u306b\u8a00\u3046\u3068\u7ffb\u8a33\u306f\u3055\u308c\u3066\u3044\u307e\u305b\u3093\u3002\u4e00\u90e8\u306f\u30c9\u30a4\u30c4\u8a9e\u3067\u3059\u304c\u3001\u3042\u3068\u306f\u3067\u305f\u3089\u3081\u3067\u3059\u3002\u5b9f\u969b\u306b\u306f\u300cWenn ist das Nunstuck git und Slotermeyer? Ja! Beiherhund das Oder die Flipperwaldt gersput.\u300d\u3068\u8a00\u3063\u3066\u3044\u307e\u3059\u3002".encode("utf-8") - h = Header(g_head, g) - h.append(cz_head, cz) - h.append(utf8_head, utf8) - enc = h.encode() - eq(enc, """\ -=?iso-8859-1?q?Die_Mieter_treten_hier_ein_werden_mit_einem_Foerderband_ko?= - =?iso-8859-1?q?mfortabel_den_Korridor_entlang=2C_an_s=FCdl=FCndischen_Wan?= - =?iso-8859-1?q?dgem=E4lden_vorbei=2C_gegen_die_rotierenden_Klingen_bef=F6?= - =?iso-8859-1?q?rdert=2E_?= =?iso-8859-2?q?Finan=E8ni_metropole_se_hroutily?= - =?iso-8859-2?q?_pod_tlakem_jejich_d=F9vtipu=2E=2E_?= =?utf-8?b?5q2j56K6?= - =?utf-8?b?44Gr6KiA44GG44Go57+76Kiz44Gv44GV44KM44Gm44GE44G+44Gb44KT44CC?= - =?utf-8?b?5LiA6YOo44Gv44OJ44Kk44OE6Kqe44Gn44GZ44GM44CB44GC44Go44Gv44Gn?= - =?utf-8?b?44Gf44KJ44KB44Gn44GZ44CC5a6f6Zqb44Gr44Gv44CMV2VubiBpc3QgZGFz?= - =?utf-8?q?_Nunstuck_git_und_Slotermeyer=3F_Ja!_Beiherhund_das_Oder_die_Fl?= - =?utf-8?b?aXBwZXJ3YWxkdCBnZXJzcHV0LuOAjeOBqOiogOOBo+OBpuOBhOOBvuOBmQ==?= - =?utf-8?b?44CC?=""") - eq(decode_header(enc), - [(g_head, "iso-8859-1"), (cz_head, "iso-8859-2"), - (utf8_head, "utf-8")]) - ustr = unicode(h) - eq(ustr.encode('utf-8'), - 'Die Mieter treten hier ein werden mit einem Foerderband ' - 'komfortabel den Korridor entlang, an s\xc3\xbcdl\xc3\xbcndischen ' - 'Wandgem\xc3\xa4lden vorbei, gegen die rotierenden Klingen ' - 'bef\xc3\xb6rdert. Finan\xc4\x8dni metropole se hroutily pod ' - 'tlakem jejich d\xc5\xafvtipu.. \xe6\xad\xa3\xe7\xa2\xba\xe3\x81' - '\xab\xe8\xa8\x80\xe3\x81\x86\xe3\x81\xa8\xe7\xbf\xbb\xe8\xa8\xb3' - '\xe3\x81\xaf\xe3\x81\x95\xe3\x82\x8c\xe3\x81\xa6\xe3\x81\x84\xe3' - '\x81\xbe\xe3\x81\x9b\xe3\x82\x93\xe3\x80\x82\xe4\xb8\x80\xe9\x83' - '\xa8\xe3\x81\xaf\xe3\x83\x89\xe3\x82\xa4\xe3\x83\x84\xe8\xaa\x9e' - '\xe3\x81\xa7\xe3\x81\x99\xe3\x81\x8c\xe3\x80\x81\xe3\x81\x82\xe3' - '\x81\xa8\xe3\x81\xaf\xe3\x81\xa7\xe3\x81\x9f\xe3\x82\x89\xe3\x82' - '\x81\xe3\x81\xa7\xe3\x81\x99\xe3\x80\x82\xe5\xae\x9f\xe9\x9a\x9b' - '\xe3\x81\xab\xe3\x81\xaf\xe3\x80\x8cWenn ist das Nunstuck git ' - 'und Slotermeyer? Ja! Beiherhund das Oder die Flipperwaldt ' - 'gersput.\xe3\x80\x8d\xe3\x81\xa8\xe8\xa8\x80\xe3\x81\xa3\xe3\x81' - '\xa6\xe3\x81\x84\xe3\x81\xbe\xe3\x81\x99\xe3\x80\x82') - # Test make_header() - newh = make_header(decode_header(enc)) - eq(newh, enc) - - def test_header_ctor_default_args(self): - eq = self.ndiffAssertEqual - h = Header() - eq(h, '') - h.append('foo', Charset('iso-8859-1')) - eq(h, '=?iso-8859-1?q?foo?=') - - def test_explicit_maxlinelen(self): - eq = self.ndiffAssertEqual - hstr = 'A very long line that must get split to something other than at the 76th character boundary to test the non-default behavior' - h = Header(hstr) - eq(h.encode(), '''\ -A very long line that must get split to something other than at the 76th - character boundary to test the non-default behavior''') - h = Header(hstr, header_name='Subject') - eq(h.encode(), '''\ -A very long line that must get split to something other than at the - 76th character boundary to test the non-default behavior''') - h = Header(hstr, maxlinelen=1024, header_name='Subject') - eq(h.encode(), hstr) - - def test_us_ascii_header(self): - eq = self.assertEqual - s = 'hello' - x = decode_header(s) - eq(x, [('hello', None)]) - h = make_header(x) - eq(s, h.encode()) - - def test_string_charset(self): - eq = self.assertEqual - h = Header() - h.append('hello', 'iso-8859-1') - eq(h, '=?iso-8859-1?q?hello?=') - -## def test_unicode_error(self): -## raises = self.assertRaises -## raises(UnicodeError, Header, u'[P\xf6stal]', 'us-ascii') -## raises(UnicodeError, Header, '[P\xf6stal]', 'us-ascii') -## h = Header() -## raises(UnicodeError, h.append, u'[P\xf6stal]', 'us-ascii') -## raises(UnicodeError, h.append, '[P\xf6stal]', 'us-ascii') -## raises(UnicodeError, Header, u'\u83ca\u5730\u6642\u592b', 'iso-8859-1') - - def test_utf8_shortest(self): - eq = self.assertEqual - h = Header(u'p\xf6stal', 'utf-8') - eq(h.encode(), '=?utf-8?q?p=C3=B6stal?=') - h = Header(u'\u83ca\u5730\u6642\u592b', 'utf-8') - eq(h.encode(), '=?utf-8?b?6I+K5Zyw5pmC5aSr?=') - - def test_bad_8bit_header(self): - raises = self.assertRaises - eq = self.assertEqual - x = 'Ynwp4dUEbay Auction Semiar- No Charge \x96 Earn Big' - raises(UnicodeError, Header, x) - h = Header() - raises(UnicodeError, h.append, x) - eq(str(Header(x, errors='replace')), x) - h.append(x, errors='replace') - eq(str(h), x) - - def test_encoded_adjacent_nonencoded(self): - eq = self.assertEqual - h = Header() - h.append('hello', 'iso-8859-1') - h.append('world') - s = h.encode() - eq(s, '=?iso-8859-1?q?hello?= world') - h = make_header(decode_header(s)) - eq(h.encode(), s) - - def test_whitespace_eater(self): - eq = self.assertEqual - s = 'Subject: =?koi8-r?b?8NLP18XSy8EgzsEgxsnOwczYztk=?= =?koi8-r?q?=CA?= zz.' - parts = decode_header(s) - eq(parts, [('Subject:', None), ('\xf0\xd2\xcf\xd7\xc5\xd2\xcb\xc1 \xce\xc1 \xc6\xc9\xce\xc1\xcc\xd8\xce\xd9\xca', 'koi8-r'), ('zz.', None)]) - hdr = make_header(parts) - eq(hdr.encode(), - 'Subject: =?koi8-r?b?8NLP18XSy8EgzsEgxsnOwczYztnK?= zz.') - - def test_broken_base64_header(self): - raises = self.assertRaises - s = 'Subject: =?EUC-KR?B?CSixpLDtKSC/7Liuvsax4iC6uLmwMcijIKHaILzSwd/H0SC8+LCjwLsgv7W/+Mj3I ?=' - raises(Errors.HeaderParseError, decode_header, s) - - # Issue 1078919 - def test_ascii_add_header(self): - msg = Message() - msg.add_header('Content-Disposition', 'attachment', - filename='bud.gif') - self.assertEqual('attachment; filename="bud.gif"', - msg['Content-Disposition']) - - def test_nonascii_add_header_via_triple(self): - msg = Message() - msg.add_header('Content-Disposition', 'attachment', - filename=('iso-8859-1', '', 'Fu\xdfballer.ppt')) - self.assertEqual( - 'attachment; filename*="iso-8859-1\'\'Fu%DFballer.ppt"', - msg['Content-Disposition']) - - def test_encode_unaliased_charset(self): - # Issue 1379416: when the charset has no output conversion, - # output was accidentally getting coerced to unicode. - res = Header('abc','iso-8859-2').encode() - self.assertEqual(res, '=?iso-8859-2?q?abc?=') - self.assertIsInstance(res, str) - - -# Test RFC 2231 header parameters (en/de)coding -class TestRFC2231(TestEmailBase): - def test_get_param(self): - eq = self.assertEqual - msg = self._msgobj('msg_29.txt') - eq(msg.get_param('title'), - ('us-ascii', 'en', 'This is even more ***fun*** isn\'t it!')) - eq(msg.get_param('title', unquote=False), - ('us-ascii', 'en', '"This is even more ***fun*** isn\'t it!"')) - - def test_set_param(self): - eq = self.assertEqual - msg = Message() - msg.set_param('title', 'This is even more ***fun*** isn\'t it!', - charset='us-ascii') - eq(msg.get_param('title'), - ('us-ascii', '', 'This is even more ***fun*** isn\'t it!')) - msg.set_param('title', 'This is even more ***fun*** isn\'t it!', - charset='us-ascii', language='en') - eq(msg.get_param('title'), - ('us-ascii', 'en', 'This is even more ***fun*** isn\'t it!')) - msg = self._msgobj('msg_01.txt') - msg.set_param('title', 'This is even more ***fun*** isn\'t it!', - charset='us-ascii', language='en') - self.ndiffAssertEqual(msg.as_string(), """\ -Return-Path: <bbb at zzz.org> -Delivered-To: bbb at zzz.org -Received: by mail.zzz.org (Postfix, from userid 889) - id 27CEAD38CC; Fri, 4 May 2001 14:05:44 -0400 (EDT) -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit -Message-ID: <15090.61304.110929.45684 at aaa.zzz.org> -From: bbb at ddd.com (John X. Doe) -To: bbb at zzz.org -Subject: This is a test message -Date: Fri, 4 May 2001 14:05:44 -0400 -Content-Type: text/plain; charset=us-ascii; - title*="us-ascii'en'This%20is%20even%20more%20%2A%2A%2Afun%2A%2A%2A%20isn%27t%20it%21" - - -Hi, - -Do you like this message? - --Me -""") - - def test_del_param(self): - eq = self.ndiffAssertEqual - msg = self._msgobj('msg_01.txt') - msg.set_param('foo', 'bar', charset='us-ascii', language='en') - msg.set_param('title', 'This is even more ***fun*** isn\'t it!', - charset='us-ascii', language='en') - msg.del_param('foo', header='Content-Type') - eq(msg.as_string(), """\ -Return-Path: <bbb at zzz.org> -Delivered-To: bbb at zzz.org -Received: by mail.zzz.org (Postfix, from userid 889) - id 27CEAD38CC; Fri, 4 May 2001 14:05:44 -0400 (EDT) -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit -Message-ID: <15090.61304.110929.45684 at aaa.zzz.org> -From: bbb at ddd.com (John X. Doe) -To: bbb at zzz.org -Subject: This is a test message -Date: Fri, 4 May 2001 14:05:44 -0400 -Content-Type: text/plain; charset="us-ascii"; - title*="us-ascii'en'This%20is%20even%20more%20%2A%2A%2Afun%2A%2A%2A%20isn%27t%20it%21" - - -Hi, - -Do you like this message? - --Me -""") - - def test_rfc2231_get_content_charset(self): - eq = self.assertEqual - msg = self._msgobj('msg_32.txt') - eq(msg.get_content_charset(), 'us-ascii') - - def test_rfc2231_no_language_or_charset(self): - m = '''\ -Content-Transfer-Encoding: 8bit -Content-Disposition: inline; filename="file____C__DOCUMENTS_20AND_20SETTINGS_FABIEN_LOCAL_20SETTINGS_TEMP_nsmail.htm" -Content-Type: text/html; NAME*0=file____C__DOCUMENTS_20AND_20SETTINGS_FABIEN_LOCAL_20SETTINGS_TEM; NAME*1=P_nsmail.htm - -''' - msg = email.message_from_string(m) - param = msg.get_param('NAME') - self.assertFalse(isinstance(param, tuple)) - self.assertEqual( - param, - 'file____C__DOCUMENTS_20AND_20SETTINGS_FABIEN_LOCAL_20SETTINGS_TEMP_nsmail.htm') - - def test_rfc2231_no_language_or_charset_in_filename(self): - m = '''\ -Content-Disposition: inline; -\tfilename*0*="''This%20is%20even%20more%20"; -\tfilename*1*="%2A%2A%2Afun%2A%2A%2A%20"; -\tfilename*2="is it not.pdf" - -''' - msg = email.message_from_string(m) - self.assertEqual(msg.get_filename(), - 'This is even more ***fun*** is it not.pdf') - - def test_rfc2231_no_language_or_charset_in_filename_encoded(self): - m = '''\ -Content-Disposition: inline; -\tfilename*0*="''This%20is%20even%20more%20"; -\tfilename*1*="%2A%2A%2Afun%2A%2A%2A%20"; -\tfilename*2="is it not.pdf" - -''' - msg = email.message_from_string(m) - self.assertEqual(msg.get_filename(), - 'This is even more ***fun*** is it not.pdf') - - def test_rfc2231_partly_encoded(self): - m = '''\ -Content-Disposition: inline; -\tfilename*0="''This%20is%20even%20more%20"; -\tfilename*1*="%2A%2A%2Afun%2A%2A%2A%20"; -\tfilename*2="is it not.pdf" - -''' - msg = email.message_from_string(m) - self.assertEqual( - msg.get_filename(), - 'This%20is%20even%20more%20***fun*** is it not.pdf') - - def test_rfc2231_partly_nonencoded(self): - m = '''\ -Content-Disposition: inline; -\tfilename*0="This%20is%20even%20more%20"; -\tfilename*1="%2A%2A%2Afun%2A%2A%2A%20"; -\tfilename*2="is it not.pdf" - -''' - msg = email.message_from_string(m) - self.assertEqual( - msg.get_filename(), - 'This%20is%20even%20more%20%2A%2A%2Afun%2A%2A%2A%20is it not.pdf') - - def test_rfc2231_no_language_or_charset_in_boundary(self): - m = '''\ -Content-Type: multipart/alternative; -\tboundary*0*="''This%20is%20even%20more%20"; -\tboundary*1*="%2A%2A%2Afun%2A%2A%2A%20"; -\tboundary*2="is it not.pdf" - -''' - msg = email.message_from_string(m) - self.assertEqual(msg.get_boundary(), - 'This is even more ***fun*** is it not.pdf') - - def test_rfc2231_no_language_or_charset_in_charset(self): - # This is a nonsensical charset value, but tests the code anyway - m = '''\ -Content-Type: text/plain; -\tcharset*0*="This%20is%20even%20more%20"; -\tcharset*1*="%2A%2A%2Afun%2A%2A%2A%20"; -\tcharset*2="is it not.pdf" - -''' - msg = email.message_from_string(m) - self.assertEqual(msg.get_content_charset(), - 'this is even more ***fun*** is it not.pdf') - - def test_rfc2231_bad_encoding_in_filename(self): - m = '''\ -Content-Disposition: inline; -\tfilename*0*="bogus'xx'This%20is%20even%20more%20"; -\tfilename*1*="%2A%2A%2Afun%2A%2A%2A%20"; -\tfilename*2="is it not.pdf" - -''' - msg = email.message_from_string(m) - self.assertEqual(msg.get_filename(), - 'This is even more ***fun*** is it not.pdf') - - def test_rfc2231_bad_encoding_in_charset(self): - m = """\ -Content-Type: text/plain; charset*=bogus''utf-8%E2%80%9D - -""" - msg = email.message_from_string(m) - # This should return None because non-ascii characters in the charset - # are not allowed. - self.assertEqual(msg.get_content_charset(), None) - - def test_rfc2231_bad_character_in_charset(self): - m = """\ -Content-Type: text/plain; charset*=ascii''utf-8%E2%80%9D - -""" - msg = email.message_from_string(m) - # This should return None because non-ascii characters in the charset - # are not allowed. - self.assertEqual(msg.get_content_charset(), None) - - def test_rfc2231_bad_character_in_filename(self): - m = '''\ -Content-Disposition: inline; -\tfilename*0*="ascii'xx'This%20is%20even%20more%20"; -\tfilename*1*="%2A%2A%2Afun%2A%2A%2A%20"; -\tfilename*2*="is it not.pdf%E2" - -''' - msg = email.message_from_string(m) - self.assertEqual(msg.get_filename(), - u'This is even more ***fun*** is it not.pdf\ufffd') - - def test_rfc2231_unknown_encoding(self): - m = """\ -Content-Transfer-Encoding: 8bit -Content-Disposition: inline; filename*=X-UNKNOWN''myfile.txt - -""" - msg = email.message_from_string(m) - self.assertEqual(msg.get_filename(), 'myfile.txt') - - def test_rfc2231_single_tick_in_filename_extended(self): - eq = self.assertEqual From noreply at buildbot.pypy.org Mon Jan 30 22:55:57 2012 From: noreply at buildbot.pypy.org (mattip) Date: Mon, 30 Jan 2012 22:55:57 +0100 (CET) Subject: [pypy-commit] pypy default: more scalar dot testing Message-ID: <20120130215557.8E29D7106F3@wyvern.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r51981:73b88e7136fe Date: 2012-01-29 05:58 +0200 http://bitbucket.org/pypy/pypy/changeset/73b88e7136fe/ Log: more scalar dot testing diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -949,6 +949,8 @@ assert b[i] == 2.5 * a[i] c = dot(4, 3.0) assert c == 12.0 + c = array(3.0).dot(array(4)) + assert c == 12.0 def test_dtype_guessing(self): from _numpypy import array, dtype, float64, int8, bool_ From noreply at buildbot.pypy.org Mon Jan 30 22:55:59 2012 From: noreply at buildbot.pypy.org (mattip) Date: Mon, 30 Jan 2012 22:55:59 +0100 (CET) Subject: [pypy-commit] pypy default: merge Message-ID: <20120130215559.540897106F3@wyvern.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r51982:7f2c4e746139 Date: 2012-01-29 05:59 +0200 http://bitbucket.org/pypy/pypy/changeset/7f2c4e746139/ Log: merge diff too long, truncating to 10000 out of 14109 lines diff --git a/lib-python/modified-2.7/email/__init__.py b/lib-python/modified-2.7/email/__init__.py deleted file mode 100644 --- a/lib-python/modified-2.7/email/__init__.py +++ /dev/null @@ -1,123 +0,0 @@ -# Copyright (C) 2001-2006 Python Software Foundation -# Author: Barry Warsaw -# Contact: email-sig at python.org - -"""A package for parsing, handling, and generating email messages.""" - -__version__ = '4.0.3' - -__all__ = [ - # Old names - 'base64MIME', - 'Charset', - 'Encoders', - 'Errors', - 'Generator', - 'Header', - 'Iterators', - 'Message', - 'MIMEAudio', - 'MIMEBase', - 'MIMEImage', - 'MIMEMessage', - 'MIMEMultipart', - 'MIMENonMultipart', - 'MIMEText', - 'Parser', - 'quopriMIME', - 'Utils', - 'message_from_string', - 'message_from_file', - # new names - 'base64mime', - 'charset', - 'encoders', - 'errors', - 'generator', - 'header', - 'iterators', - 'message', - 'mime', - 'parser', - 'quoprimime', - 'utils', - ] - - - -# Some convenience routines. Don't import Parser and Message as side-effects -# of importing email since those cascadingly import most of the rest of the -# email package. -def message_from_string(s, *args, **kws): - """Parse a string into a Message object model. - - Optional _class and strict are passed to the Parser constructor. - """ - from email.parser import Parser - return Parser(*args, **kws).parsestr(s) - - -def message_from_file(fp, *args, **kws): - """Read a file and parse its contents into a Message object model. - - Optional _class and strict are passed to the Parser constructor. - """ - from email.parser import Parser - return Parser(*args, **kws).parse(fp) - - - -# Lazy loading to provide name mapping from new-style names (PEP 8 compatible -# email 4.0 module names), to old-style names (email 3.0 module names). -import sys - -class LazyImporter(object): - def __init__(self, module_name): - self.__name__ = 'email.' + module_name - - def __getattr__(self, name): - __import__(self.__name__) - mod = sys.modules[self.__name__] - self.__dict__.update(mod.__dict__) - return getattr(mod, name) - - -_LOWERNAMES = [ - # email.<old name> -> email.<new name is lowercased old name> - 'Charset', - 'Encoders', - 'Errors', - 'FeedParser', - 'Generator', - 'Header', - 'Iterators', - 'Message', - 'Parser', - 'Utils', - 'base64MIME', - 'quopriMIME', - ] - -_MIMENAMES = [ - # email.MIME<old name> -> email.mime.<new name is lowercased old name> - 'Audio', - 'Base', - 'Image', - 'Message', - 'Multipart', - 'NonMultipart', - 'Text', - ] - -for _name in _LOWERNAMES: - importer = LazyImporter(_name.lower()) - sys.modules['email.' + _name] = importer - setattr(sys.modules['email'], _name, importer) - - -import email.mime -for _name in _MIMENAMES: - importer = LazyImporter('mime.' + _name.lower()) - sys.modules['email.MIME' + _name] = importer - setattr(sys.modules['email'], 'MIME' + _name, importer) - setattr(sys.modules['email.mime'], _name, importer) diff --git a/lib-python/modified-2.7/email/_parseaddr.py b/lib-python/modified-2.7/email/_parseaddr.py deleted file mode 100644 --- a/lib-python/modified-2.7/email/_parseaddr.py +++ /dev/null @@ -1,497 +0,0 @@ -# Copyright (C) 2002-2007 Python Software Foundation -# Contact: email-sig at python.org - -"""Email address parsing code. - -Lifted directly from rfc822.py. This should eventually be rewritten. -""" - -__all__ = [ - 'mktime_tz', - 'parsedate', - 'parsedate_tz', - 'quote', - ] - -import time - -SPACE = ' ' -EMPTYSTRING = '' -COMMASPACE = ', ' - -# Parse a date field -_monthnames = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', - 'aug', 'sep', 'oct', 'nov', 'dec', - 'january', 'february', 'march', 'april', 'may', 'june', 'july', - 'august', 'september', 'october', 'november', 'december'] - -_daynames = ['mon', 'tue', 'wed', 'thu', 'fri', 'sat', 'sun'] - -# The timezone table does not include the military time zones defined -# in RFC822, other than Z. According to RFC1123, the description in -# RFC822 gets the signs wrong, so we can't rely on any such time -# zones. RFC1123 recommends that numeric timezone indicators be used -# instead of timezone names. - -_timezones = {'UT':0, 'UTC':0, 'GMT':0, 'Z':0, - 'AST': -400, 'ADT': -300, # Atlantic (used in Canada) - 'EST': -500, 'EDT': -400, # Eastern - 'CST': -600, 'CDT': -500, # Central - 'MST': -700, 'MDT': -600, # Mountain - 'PST': -800, 'PDT': -700 # Pacific - } - - -def parsedate_tz(data): - """Convert a date string to a time tuple. - - Accounts for military timezones. - """ - data = data.split() - # The FWS after the comma after the day-of-week is optional, so search and - # adjust for this. - if data[0].endswith(',') or data[0].lower() in _daynames: - # There's a dayname here. Skip it - del data[0] - else: - i = data[0].rfind(',') - if i >= 0: - data[0] = data[0][i+1:] - if len(data) == 3: # RFC 850 date, deprecated - stuff = data[0].split('-') - if len(stuff) == 3: - data = stuff + data[1:] - if len(data) == 4: - s = data[3] - i = s.find('+') - if i > 0: - data[3:] = [s[:i], s[i+1:]] - else: - data.append('') # Dummy tz - if len(data) < 5: - return None - data = data[:5] - [dd, mm, yy, tm, tz] = data - mm = mm.lower() - if mm not in _monthnames: - dd, mm = mm, dd.lower() - if mm not in _monthnames: - return None - mm = _monthnames.index(mm) + 1 - if mm > 12: - mm -= 12 - if dd[-1] == ',': - dd = dd[:-1] - i = yy.find(':') - if i > 0: - yy, tm = tm, yy - if yy[-1] == ',': - yy = yy[:-1] - if not yy[0].isdigit(): - yy, tz = tz, yy - if tm[-1] == ',': - tm = tm[:-1] - tm = tm.split(':') - if len(tm) == 2: - [thh, tmm] = tm - tss = '0' - elif len(tm) == 3: - [thh, tmm, tss] = tm - else: - return None - try: - yy = int(yy) - dd = int(dd) - thh = int(thh) - tmm = int(tmm) - tss = int(tss) - except ValueError: - return None - # Check for a yy specified in two-digit format, then convert it to the - # appropriate four-digit format, according to the POSIX standard. RFC 822 - # calls for a two-digit yy, but RFC 2822 (which obsoletes RFC 822) - # mandates a 4-digit yy. For more information, see the documentation for - # the time module. - if yy < 100: - # The year is between 1969 and 1999 (inclusive). - if yy > 68: - yy += 1900 - # The year is between 2000 and 2068 (inclusive). - else: - yy += 2000 - tzoffset = None - tz = tz.upper() - if tz in _timezones: - tzoffset = _timezones[tz] - else: - try: - tzoffset = int(tz) - except ValueError: - pass - # Convert a timezone offset into seconds ; -0500 -> -18000 - if tzoffset: - if tzoffset < 0: - tzsign = -1 - tzoffset = -tzoffset - else: - tzsign = 1 - tzoffset = tzsign * ( (tzoffset//100)*3600 + (tzoffset % 100)*60) - # Daylight Saving Time flag is set to -1, since DST is unknown. - return yy, mm, dd, thh, tmm, tss, 0, 1, -1, tzoffset - - -def parsedate(data): - """Convert a time string to a time tuple.""" - t = parsedate_tz(data) - if isinstance(t, tuple): - return t[:9] - else: - return t - - -def mktime_tz(data): - """Turn a 10-tuple as returned by parsedate_tz() into a UTC timestamp.""" - if data[9] is None: - # No zone info, so localtime is better assumption than GMT - return time.mktime(data[:8] + (-1,)) - else: - t = time.mktime(data[:8] + (0,)) - return t - data[9] - time.timezone - - -def quote(str): - """Prepare string to be used in a quoted string. - - Turns backslash and double quote characters into quoted pairs. These - are the only characters that need to be quoted inside a quoted string. - Does not add the surrounding double quotes. - """ - return str.replace('\\', '\\\\').replace('"', '\\"') - - -class AddrlistClass: - """Address parser class by Ben Escoto. - - To understand what this class does, it helps to have a copy of RFC 2822 in - front of you. - - Note: this class interface is deprecated and may be removed in the future. - Use rfc822.AddressList instead. - """ - - def __init__(self, field): - """Initialize a new instance. - - `field' is an unparsed address header field, containing - one or more addresses. - """ - self.specials = '()<>@,:;.\"[]' - self.pos = 0 - self.LWS = ' \t' - self.CR = '\r\n' - self.FWS = self.LWS + self.CR - self.atomends = self.specials + self.LWS + self.CR - # Note that RFC 2822 now specifies `.' as obs-phrase, meaning that it - # is obsolete syntax. RFC 2822 requires that we recognize obsolete - # syntax, so allow dots in phrases. - self.phraseends = self.atomends.replace('.', '') - self.field = field - self.commentlist = [] - - def gotonext(self): - """Parse up to the start of the next address.""" - while self.pos < len(self.field): - if self.field[self.pos] in self.LWS + '\n\r': - self.pos += 1 - elif self.field[self.pos] == '(': - self.commentlist.append(self.getcomment()) - else: - break - - def getaddrlist(self): - """Parse all addresses. - - Returns a list containing all of the addresses. - """ - result = [] - while self.pos < len(self.field): - ad = self.getaddress() - if ad: - result += ad - else: - result.append(('', '')) - return result - - def getaddress(self): - """Parse the next address.""" - self.commentlist = [] - self.gotonext() - - oldpos = self.pos - oldcl = self.commentlist - plist = self.getphraselist() - - self.gotonext() - returnlist = [] - - if self.pos >= len(self.field): - # Bad email address technically, no domain. - if plist: - returnlist = [(SPACE.join(self.commentlist), plist[0])] - - elif self.field[self.pos] in '.@': - # email address is just an addrspec - # this isn't very efficient since we start over - self.pos = oldpos - self.commentlist = oldcl - addrspec = self.getaddrspec() - returnlist = [(SPACE.join(self.commentlist), addrspec)] - - elif self.field[self.pos] == ':': - # address is a group - returnlist = [] - - fieldlen = len(self.field) - self.pos += 1 - while self.pos < len(self.field): - self.gotonext() - if self.pos < fieldlen and self.field[self.pos] == ';': - self.pos += 1 - break - returnlist = returnlist + self.getaddress() - - elif self.field[self.pos] == '<': - # Address is a phrase then a route addr - routeaddr = self.getrouteaddr() - - if self.commentlist: - returnlist = [(SPACE.join(plist) + ' (' + - ' '.join(self.commentlist) + ')', routeaddr)] - else: - returnlist = [(SPACE.join(plist), routeaddr)] - - else: - if plist: - returnlist = [(SPACE.join(self.commentlist), plist[0])] - elif self.field[self.pos] in self.specials: - self.pos += 1 - - self.gotonext() - if self.pos < len(self.field) and self.field[self.pos] == ',': - self.pos += 1 - return returnlist - - def getrouteaddr(self): - """Parse a route address (Return-path value). - - This method just skips all the route stuff and returns the addrspec. - """ - if self.field[self.pos] != '<': - return - - expectroute = False - self.pos += 1 - self.gotonext() - adlist = '' - while self.pos < len(self.field): - if expectroute: - self.getdomain() - expectroute = False - elif self.field[self.pos] == '>': - self.pos += 1 - break - elif self.field[self.pos] == '@': - self.pos += 1 - expectroute = True - elif self.field[self.pos] == ':': - self.pos += 1 - else: - adlist = self.getaddrspec() - self.pos += 1 - break - self.gotonext() - - return adlist - - def getaddrspec(self): - """Parse an RFC 2822 addr-spec.""" - aslist = [] - - self.gotonext() - while self.pos < len(self.field): - if self.field[self.pos] == '.': - aslist.append('.') - self.pos += 1 - elif self.field[self.pos] == '"': - aslist.append('"%s"' % quote(self.getquote())) - elif self.field[self.pos] in self.atomends: - break - else: - aslist.append(self.getatom()) - self.gotonext() - - if self.pos >= len(self.field) or self.field[self.pos] != '@': - return EMPTYSTRING.join(aslist) - - aslist.append('@') - self.pos += 1 - self.gotonext() - return EMPTYSTRING.join(aslist) + self.getdomain() - - def getdomain(self): - """Get the complete domain name from an address.""" - sdlist = [] - while self.pos < len(self.field): - if self.field[self.pos] in self.LWS: - self.pos += 1 - elif self.field[self.pos] == '(': - self.commentlist.append(self.getcomment()) - elif self.field[self.pos] == '[': - sdlist.append(self.getdomainliteral()) - elif self.field[self.pos] == '.': - self.pos += 1 - sdlist.append('.') - elif self.field[self.pos] in self.atomends: - break - else: - sdlist.append(self.getatom()) - return EMPTYSTRING.join(sdlist) - - def getdelimited(self, beginchar, endchars, allowcomments=True): - """Parse a header fragment delimited by special characters. - - `beginchar' is the start character for the fragment. - If self is not looking at an instance of `beginchar' then - getdelimited returns the empty string. - - `endchars' is a sequence of allowable end-delimiting characters. - Parsing stops when one of these is encountered. - - If `allowcomments' is non-zero, embedded RFC 2822 comments are allowed - within the parsed fragment. - """ - if self.field[self.pos] != beginchar: - return '' - - slist = [''] - quote = False - self.pos += 1 - while self.pos < len(self.field): - if quote: - slist.append(self.field[self.pos]) - quote = False - elif self.field[self.pos] in endchars: - self.pos += 1 - break - elif allowcomments and self.field[self.pos] == '(': - slist.append(self.getcomment()) - continue # have already advanced pos from getcomment - elif self.field[self.pos] == '\\': - quote = True - else: - slist.append(self.field[self.pos]) - self.pos += 1 - - return EMPTYSTRING.join(slist) - - def getquote(self): - """Get a quote-delimited fragment from self's field.""" - return self.getdelimited('"', '"\r', False) - - def getcomment(self): - """Get a parenthesis-delimited fragment from self's field.""" - return self.getdelimited('(', ')\r', True) - - def getdomainliteral(self): - """Parse an RFC 2822 domain-literal.""" - return '[%s]' % self.getdelimited('[', ']\r', False) - - def getatom(self, atomends=None): - """Parse an RFC 2822 atom. - - Optional atomends specifies a different set of end token delimiters - (the default is to use self.atomends). This is used e.g. in - getphraselist() since phrase endings must not include the `.' (which - is legal in phrases).""" - atomlist = [''] - if atomends is None: - atomends = self.atomends - - while self.pos < len(self.field): - if self.field[self.pos] in atomends: - break - else: - atomlist.append(self.field[self.pos]) - self.pos += 1 - - return EMPTYSTRING.join(atomlist) - - def getphraselist(self): - """Parse a sequence of RFC 2822 phrases. - - A phrase is a sequence of words, which are in turn either RFC 2822 - atoms or quoted-strings. Phrases are canonicalized by squeezing all - runs of continuous whitespace into one space. - """ - plist = [] - - while self.pos < len(self.field): - if self.field[self.pos] in self.FWS: - self.pos += 1 - elif self.field[self.pos] == '"': - plist.append(self.getquote()) - elif self.field[self.pos] == '(': - self.commentlist.append(self.getcomment()) - elif self.field[self.pos] in self.phraseends: - break - else: - plist.append(self.getatom(self.phraseends)) - - return plist - -class AddressList(AddrlistClass): - """An AddressList encapsulates a list of parsed RFC 2822 addresses.""" - def __init__(self, field): - AddrlistClass.__init__(self, field) - if field: - self.addresslist = self.getaddrlist() - else: - self.addresslist = [] - - def __len__(self): - return len(self.addresslist) - - def __add__(self, other): - # Set union - newaddr = AddressList(None) - newaddr.addresslist = self.addresslist[:] - for x in other.addresslist: - if not x in self.addresslist: - newaddr.addresslist.append(x) - return newaddr - - def __iadd__(self, other): - # Set union, in-place - for x in other.addresslist: - if not x in self.addresslist: - self.addresslist.append(x) - return self - - def __sub__(self, other): - # Set difference - newaddr = AddressList(None) - for x in self.addresslist: - if not x in other.addresslist: - newaddr.addresslist.append(x) - return newaddr - - def __isub__(self, other): - # Set difference, in-place - for x in other.addresslist: - if x in self.addresslist: - self.addresslist.remove(x) - return self - - def __getitem__(self, index): - # Make indexing, slices, and 'in' work - return self.addresslist[index] diff --git a/lib-python/modified-2.7/email/base64mime.py b/lib-python/modified-2.7/email/base64mime.py deleted file mode 100644 --- a/lib-python/modified-2.7/email/base64mime.py +++ /dev/null @@ -1,183 +0,0 @@ -# Copyright (C) 2002-2006 Python Software Foundation -# Author: Ben Gertzfield -# Contact: email-sig at python.org - -"""Base64 content transfer encoding per RFCs 2045-2047. - -This module handles the content transfer encoding method defined in RFC 2045 -to encode arbitrary 8-bit data using the three 8-bit bytes in four 7-bit -characters encoding known as Base64. - -It is used in the MIME standards for email to attach images, audio, and text -using some 8-bit character sets to messages. - -This module provides an interface to encode and decode both headers and bodies -with Base64 encoding. - -RFC 2045 defines a method for including character set information in an -`encoded-word' in a header. This method is commonly used for 8-bit real names -in To:, From:, Cc:, etc. fields, as well as Subject: lines. - -This module does not do the line wrapping or end-of-line character conversion -necessary for proper internationalized headers; it only does dumb encoding and -decoding. To deal with the various line wrapping issues, use the email.header -module. -""" - -__all__ = [ - 'base64_len', - 'body_decode', - 'body_encode', - 'decode', - 'decodestring', - 'encode', - 'encodestring', - 'header_encode', - ] - - -from binascii import b2a_base64, a2b_base64 -from email.utils import fix_eols - -CRLF = '\r\n' -NL = '\n' -EMPTYSTRING = '' - -# See also Charset.py -MISC_LEN = 7 - - - -# Helpers -def base64_len(s): - """Return the length of s when it is encoded with base64.""" - groups_of_3, leftover = divmod(len(s), 3) - # 4 bytes out for each 3 bytes (or nonzero fraction thereof) in. - # Thanks, Tim! - n = groups_of_3 * 4 - if leftover: - n += 4 - return n - - - -def header_encode(header, charset='iso-8859-1', keep_eols=False, - maxlinelen=76, eol=NL): - """Encode a single header line with Base64 encoding in a given charset. - - Defined in RFC 2045, this Base64 encoding is identical to normal Base64 - encoding, except that each line must be intelligently wrapped (respecting - the Base64 encoding), and subsequent lines must start with a space. - - charset names the character set to use to encode the header. It defaults - to iso-8859-1. - - End-of-line characters (\\r, \\n, \\r\\n) will be automatically converted - to the canonical email line separator \\r\\n unless the keep_eols - parameter is True (the default is False). - - Each line of the header will be terminated in the value of eol, which - defaults to "\\n". Set this to "\\r\\n" if you are using the result of - this function directly in email. - - The resulting string will be in the form: - - "=?charset?b?WW/5ciBtYXp66XLrIHf8eiBhIGhhbXBzdGHuciBBIFlv+XIgbWF6euly?=\\n - =?charset?b?6yB3/HogYSBoYW1wc3Rh7nIgQkMgWW/5ciBtYXp66XLrIHf8eiBhIGhh?=" - - with each line wrapped at, at most, maxlinelen characters (defaults to 76 - characters). - """ - # Return empty headers unchanged - if not header: - return header - - if not keep_eols: - header = fix_eols(header) - - # Base64 encode each line, in encoded chunks no greater than maxlinelen in - # length, after the RFC chrome is added in. - base64ed = [] - max_encoded = maxlinelen - len(charset) - MISC_LEN - max_unencoded = max_encoded * 3 // 4 - - for i in range(0, len(header), max_unencoded): - base64ed.append(b2a_base64(header[i:i+max_unencoded])) - - # Now add the RFC chrome to each encoded chunk - lines = [] - for line in base64ed: - # Ignore the last character of each line if it is a newline - if line.endswith(NL): - line = line[:-1] - # Add the chrome - lines.append('=?%s?b?%s?=' % (charset, line)) - # Glue the lines together and return it. BAW: should we be able to - # specify the leading whitespace in the joiner? - joiner = eol + ' ' - return joiner.join(lines) - - - -def encode(s, binary=True, maxlinelen=76, eol=NL): - """Encode a string with base64. - - Each line will be wrapped at, at most, maxlinelen characters (defaults to - 76 characters). - - If binary is False, end-of-line characters will be converted to the - canonical email end-of-line sequence \\r\\n. Otherwise they will be left - verbatim (this is the default). - - Each line of encoded text will end with eol, which defaults to "\\n". Set - this to "\r\n" if you will be using the result of this function directly - in an email. - """ - if not s: - return s - - if not binary: - s = fix_eols(s) - - encvec = [] - max_unencoded = maxlinelen * 3 // 4 - for i in range(0, len(s), max_unencoded): - # BAW: should encode() inherit b2a_base64()'s dubious behavior in - # adding a newline to the encoded string? - enc = b2a_base64(s[i:i + max_unencoded]) - if enc.endswith(NL) and eol != NL: - enc = enc[:-1] + eol - encvec.append(enc) - return EMPTYSTRING.join(encvec) - - -# For convenience and backwards compatibility w/ standard base64 module -body_encode = encode -encodestring = encode - - - -def decode(s, convert_eols=None): - """Decode a raw base64 string. - - If convert_eols is set to a string value, all canonical email linefeeds, - e.g. "\\r\\n", in the decoded text will be converted to the value of - convert_eols. os.linesep is a good choice for convert_eols if you are - decoding a text attachment. - - This function does not parse a full MIME header value encoded with - base64 (like =?iso-8895-1?b?bmloISBuaWgh?=) -- please use the high - level email.header class for that functionality. - """ - if not s: - return s - - dec = a2b_base64(s) - if convert_eols: - return dec.replace(CRLF, convert_eols) - return dec - - -# For convenience and backwards compatibility w/ standard base64 module -body_decode = decode -decodestring = decode diff --git a/lib-python/modified-2.7/email/charset.py b/lib-python/modified-2.7/email/charset.py deleted file mode 100644 --- a/lib-python/modified-2.7/email/charset.py +++ /dev/null @@ -1,397 +0,0 @@ -# Copyright (C) 2001-2006 Python Software Foundation -# Author: Ben Gertzfield, Barry Warsaw -# Contact: email-sig at python.org - -__all__ = [ - 'Charset', - 'add_alias', - 'add_charset', - 'add_codec', - ] - -import codecs -import email.base64mime -import email.quoprimime - -from email import errors -from email.encoders import encode_7or8bit - - - -# Flags for types of header encodings -QP = 1 # Quoted-Printable -BASE64 = 2 # Base64 -SHORTEST = 3 # the shorter of QP and base64, but only for headers - -# In "=?charset?q?hello_world?=", the =?, ?q?, and ?= add up to 7 -MISC_LEN = 7 - -DEFAULT_CHARSET = 'us-ascii' - - - -# Defaults -CHARSETS = { - # input header enc body enc output conv - 'iso-8859-1': (QP, QP, None), - 'iso-8859-2': (QP, QP, None), - 'iso-8859-3': (QP, QP, None), - 'iso-8859-4': (QP, QP, None), - # iso-8859-5 is Cyrillic, and not especially used - # iso-8859-6 is Arabic, also not particularly used - # iso-8859-7 is Greek, QP will not make it readable - # iso-8859-8 is Hebrew, QP will not make it readable - 'iso-8859-9': (QP, QP, None), - 'iso-8859-10': (QP, QP, None), - # iso-8859-11 is Thai, QP will not make it readable - 'iso-8859-13': (QP, QP, None), - 'iso-8859-14': (QP, QP, None), - 'iso-8859-15': (QP, QP, None), - 'iso-8859-16': (QP, QP, None), - 'windows-1252':(QP, QP, None), - 'viscii': (QP, QP, None), - 'us-ascii': (None, None, None), - 'big5': (BASE64, BASE64, None), - 'gb2312': (BASE64, BASE64, None), - 'euc-jp': (BASE64, None, 'iso-2022-jp'), - 'shift_jis': (BASE64, None, 'iso-2022-jp'), - 'iso-2022-jp': (BASE64, None, None), - 'koi8-r': (BASE64, BASE64, None), - 'utf-8': (SHORTEST, BASE64, 'utf-8'), - # We're making this one up to represent raw unencoded 8-bit - '8bit': (None, BASE64, 'utf-8'), - } - -# Aliases for other commonly-used names for character sets. Map -# them to the real ones used in email. -ALIASES = { - 'latin_1': 'iso-8859-1', - 'latin-1': 'iso-8859-1', - 'latin_2': 'iso-8859-2', - 'latin-2': 'iso-8859-2', - 'latin_3': 'iso-8859-3', - 'latin-3': 'iso-8859-3', - 'latin_4': 'iso-8859-4', - 'latin-4': 'iso-8859-4', - 'latin_5': 'iso-8859-9', - 'latin-5': 'iso-8859-9', - 'latin_6': 'iso-8859-10', - 'latin-6': 'iso-8859-10', - 'latin_7': 'iso-8859-13', - 'latin-7': 'iso-8859-13', - 'latin_8': 'iso-8859-14', - 'latin-8': 'iso-8859-14', - 'latin_9': 'iso-8859-15', - 'latin-9': 'iso-8859-15', - 'latin_10':'iso-8859-16', - 'latin-10':'iso-8859-16', - 'cp949': 'ks_c_5601-1987', - 'euc_jp': 'euc-jp', - 'euc_kr': 'euc-kr', - 'ascii': 'us-ascii', - } - - -# Map charsets to their Unicode codec strings. -CODEC_MAP = { - 'gb2312': 'eucgb2312_cn', - 'big5': 'big5_tw', - # Hack: We don't want *any* conversion for stuff marked us-ascii, as all - # sorts of garbage might be sent to us in the guise of 7-bit us-ascii. - # Let that stuff pass through without conversion to/from Unicode. - 'us-ascii': None, - } - - - -# Convenience functions for extending the above mappings -def add_charset(charset, header_enc=None, body_enc=None, output_charset=None): - """Add character set properties to the global registry. - - charset is the input character set, and must be the canonical name of a - character set. - - Optional header_enc and body_enc is either Charset.QP for - quoted-printable, Charset.BASE64 for base64 encoding, Charset.SHORTEST for - the shortest of qp or base64 encoding, or None for no encoding. SHORTEST - is only valid for header_enc. It describes how message headers and - message bodies in the input charset are to be encoded. Default is no - encoding. - - Optional output_charset is the character set that the output should be - in. Conversions will proceed from input charset, to Unicode, to the - output charset when the method Charset.convert() is called. The default - is to output in the same character set as the input. - - Both input_charset and output_charset must have Unicode codec entries in - the module's charset-to-codec mapping; use add_codec(charset, codecname) - to add codecs the module does not know about. See the codecs module's - documentation for more information. - """ - if body_enc == SHORTEST: - raise ValueError('SHORTEST not allowed for body_enc') - CHARSETS[charset] = (header_enc, body_enc, output_charset) - - -def add_alias(alias, canonical): - """Add a character set alias. - - alias is the alias name, e.g. latin-1 - canonical is the character set's canonical name, e.g. iso-8859-1 - """ - ALIASES[alias] = canonical - - -def add_codec(charset, codecname): - """Add a codec that map characters in the given charset to/from Unicode. - - charset is the canonical name of a character set. codecname is the name - of a Python codec, as appropriate for the second argument to the unicode() - built-in, or to the encode() method of a Unicode string. - """ - CODEC_MAP[charset] = codecname - - - -class Charset: - """Map character sets to their email properties. - - This class provides information about the requirements imposed on email - for a specific character set. It also provides convenience routines for - converting between character sets, given the availability of the - applicable codecs. Given a character set, it will do its best to provide - information on how to use that character set in an email in an - RFC-compliant way. - - Certain character sets must be encoded with quoted-printable or base64 - when used in email headers or bodies. Certain character sets must be - converted outright, and are not allowed in email. Instances of this - module expose the following information about a character set: - - input_charset: The initial character set specified. Common aliases - are converted to their `official' email names (e.g. latin_1 - is converted to iso-8859-1). Defaults to 7-bit us-ascii. - - header_encoding: If the character set must be encoded before it can be - used in an email header, this attribute will be set to - Charset.QP (for quoted-printable), Charset.BASE64 (for - base64 encoding), or Charset.SHORTEST for the shortest of - QP or BASE64 encoding. Otherwise, it will be None. - - body_encoding: Same as header_encoding, but describes the encoding for the - mail message's body, which indeed may be different than the - header encoding. Charset.SHORTEST is not allowed for - body_encoding. - - output_charset: Some character sets must be converted before the can be - used in email headers or bodies. If the input_charset is - one of them, this attribute will contain the name of the - charset output will be converted to. Otherwise, it will - be None. - - input_codec: The name of the Python codec used to convert the - input_charset to Unicode. If no conversion codec is - necessary, this attribute will be None. - - output_codec: The name of the Python codec used to convert Unicode - to the output_charset. If no conversion codec is necessary, - this attribute will have the same value as the input_codec. - """ - def __init__(self, input_charset=DEFAULT_CHARSET): - # RFC 2046, $4.1.2 says charsets are not case sensitive. We coerce to - # unicode because its .lower() is locale insensitive. If the argument - # is already a unicode, we leave it at that, but ensure that the - # charset is ASCII, as the standard (RFC XXX) requires. - try: - if isinstance(input_charset, unicode): - input_charset.encode('ascii') - else: - input_charset = unicode(input_charset, 'ascii') - except UnicodeError: - raise errors.CharsetError(input_charset) - input_charset = input_charset.lower().encode('ascii') - # Set the input charset after filtering through the aliases and/or codecs - if not (input_charset in ALIASES or input_charset in CHARSETS): - try: - input_charset = codecs.lookup(input_charset).name - except LookupError: - pass - self.input_charset = ALIASES.get(input_charset, input_charset) - # We can try to guess which encoding and conversion to use by the - # charset_map dictionary. Try that first, but let the user override - # it. - henc, benc, conv = CHARSETS.get(self.input_charset, - (SHORTEST, BASE64, None)) - if not conv: - conv = self.input_charset - # Set the attributes, allowing the arguments to override the default. - self.header_encoding = henc - self.body_encoding = benc - self.output_charset = ALIASES.get(conv, conv) - # Now set the codecs. If one isn't defined for input_charset, - # guess and try a Unicode codec with the same name as input_codec. - self.input_codec = CODEC_MAP.get(self.input_charset, - self.input_charset) - self.output_codec = CODEC_MAP.get(self.output_charset, - self.output_charset) - - def __str__(self): - return self.input_charset.lower() - - __repr__ = __str__ - - def __eq__(self, other): - return str(self) == str(other).lower() - - def __ne__(self, other): - return not self.__eq__(other) - - def get_body_encoding(self): - """Return the content-transfer-encoding used for body encoding. - - This is either the string `quoted-printable' or `base64' depending on - the encoding used, or it is a function in which case you should call - the function with a single argument, the Message object being - encoded. The function should then set the Content-Transfer-Encoding - header itself to whatever is appropriate. - - Returns "quoted-printable" if self.body_encoding is QP. - Returns "base64" if self.body_encoding is BASE64. - Returns "7bit" otherwise. - """ - assert self.body_encoding != SHORTEST - if self.body_encoding == QP: - return 'quoted-printable' - elif self.body_encoding == BASE64: - return 'base64' - else: - return encode_7or8bit - - def convert(self, s): - """Convert a string from the input_codec to the output_codec.""" - if self.input_codec != self.output_codec: - return unicode(s, self.input_codec).encode(self.output_codec) - else: - return s - - def to_splittable(self, s): - """Convert a possibly multibyte string to a safely splittable format. - - Uses the input_codec to try and convert the string to Unicode, so it - can be safely split on character boundaries (even for multibyte - characters). - - Returns the string as-is if it isn't known how to convert it to - Unicode with the input_charset. - - Characters that could not be converted to Unicode will be replaced - with the Unicode replacement character U+FFFD. - """ - if isinstance(s, unicode) or self.input_codec is None: - return s - try: - return unicode(s, self.input_codec, 'replace') - except LookupError: - # Input codec not installed on system, so return the original - # string unchanged. - return s - - def from_splittable(self, ustr, to_output=True): - """Convert a splittable string back into an encoded string. - - Uses the proper codec to try and convert the string from Unicode back - into an encoded format. Return the string as-is if it is not Unicode, - or if it could not be converted from Unicode. - - Characters that could not be converted from Unicode will be replaced - with an appropriate character (usually '?'). - - If to_output is True (the default), uses output_codec to convert to an - encoded format. If to_output is False, uses input_codec. - """ - if to_output: - codec = self.output_codec - else: - codec = self.input_codec - if not isinstance(ustr, unicode) or codec is None: - return ustr - try: - return ustr.encode(codec, 'replace') - except LookupError: - # Output codec not installed - return ustr - - def get_output_charset(self): - """Return the output character set. - - This is self.output_charset if that is not None, otherwise it is - self.input_charset. - """ - return self.output_charset or self.input_charset - - def encoded_header_len(self, s): - """Return the length of the encoded header string.""" - cset = self.get_output_charset() - # The len(s) of a 7bit encoding is len(s) - if self.header_encoding == BASE64: - return email.base64mime.base64_len(s) + len(cset) + MISC_LEN - elif self.header_encoding == QP: - return email.quoprimime.header_quopri_len(s) + len(cset) + MISC_LEN - elif self.header_encoding == SHORTEST: - lenb64 = email.base64mime.base64_len(s) - lenqp = email.quoprimime.header_quopri_len(s) - return min(lenb64, lenqp) + len(cset) + MISC_LEN - else: - return len(s) - - def header_encode(self, s, convert=False): - """Header-encode a string, optionally converting it to output_charset. - - If convert is True, the string will be converted from the input - charset to the output charset automatically. This is not useful for - multibyte character sets, which have line length issues (multibyte - characters must be split on a character, not a byte boundary); use the - high-level Header class to deal with these issues. convert defaults - to False. - - The type of encoding (base64 or quoted-printable) will be based on - self.header_encoding. - """ - cset = self.get_output_charset() - if convert: - s = self.convert(s) - # 7bit/8bit encodings return the string unchanged (modulo conversions) - if self.header_encoding == BASE64: - return email.base64mime.header_encode(s, cset) - elif self.header_encoding == QP: - return email.quoprimime.header_encode(s, cset, maxlinelen=None) - elif self.header_encoding == SHORTEST: - lenb64 = email.base64mime.base64_len(s) - lenqp = email.quoprimime.header_quopri_len(s) - if lenb64 < lenqp: - return email.base64mime.header_encode(s, cset) - else: - return email.quoprimime.header_encode(s, cset, maxlinelen=None) - else: - return s - - def body_encode(self, s, convert=True): - """Body-encode a string and convert it to output_charset. - - If convert is True (the default), the string will be converted from - the input charset to output charset automatically. Unlike - header_encode(), there are no issues with byte boundaries and - multibyte charsets in email bodies, so this is usually pretty safe. - - The type of encoding (base64 or quoted-printable) will be based on - self.body_encoding. - """ - if convert: - s = self.convert(s) - # 7bit/8bit encodings return the string unchanged (module conversions) - if self.body_encoding is BASE64: - return email.base64mime.body_encode(s) - elif self.body_encoding is QP: - return email.quoprimime.body_encode(s) - else: - return s diff --git a/lib-python/modified-2.7/email/encoders.py b/lib-python/modified-2.7/email/encoders.py deleted file mode 100644 --- a/lib-python/modified-2.7/email/encoders.py +++ /dev/null @@ -1,82 +0,0 @@ -# Copyright (C) 2001-2006 Python Software Foundation -# Author: Barry Warsaw -# Contact: email-sig at python.org - -"""Encodings and related functions.""" - -__all__ = [ - 'encode_7or8bit', - 'encode_base64', - 'encode_noop', - 'encode_quopri', - ] - -import base64 - -from quopri import encodestring as _encodestring - - - -def _qencode(s): - enc = _encodestring(s, quotetabs=True) - # Must encode spaces, which quopri.encodestring() doesn't do - return enc.replace(' ', '=20') - - -def _bencode(s): - # We can't quite use base64.encodestring() since it tacks on a "courtesy - # newline". Blech! - if not s: - return s - hasnewline = (s[-1] == '\n') - value = base64.encodestring(s) - if not hasnewline and value[-1] == '\n': - return value[:-1] - return value - - - -def encode_base64(msg): - """Encode the message's payload in Base64. - - Also, add an appropriate Content-Transfer-Encoding header. - """ - orig = msg.get_payload() - encdata = _bencode(orig) - msg.set_payload(encdata) - msg['Content-Transfer-Encoding'] = 'base64' - - - -def encode_quopri(msg): - """Encode the message's payload in quoted-printable. - - Also, add an appropriate Content-Transfer-Encoding header. - """ - orig = msg.get_payload() - encdata = _qencode(orig) - msg.set_payload(encdata) - msg['Content-Transfer-Encoding'] = 'quoted-printable' - - - -def encode_7or8bit(msg): - """Set the Content-Transfer-Encoding header to 7bit or 8bit.""" - orig = msg.get_payload() - if orig is None: - # There's no payload. For backwards compatibility we use 7bit - msg['Content-Transfer-Encoding'] = '7bit' - return - # We play a trick to make this go fast. If encoding to ASCII succeeds, we - # know the data must be 7bit, otherwise treat it as 8bit. - try: - orig.encode('ascii') - except UnicodeError: - msg['Content-Transfer-Encoding'] = '8bit' - else: - msg['Content-Transfer-Encoding'] = '7bit' - - - -def encode_noop(msg): - """Do nothing.""" diff --git a/lib-python/modified-2.7/email/errors.py b/lib-python/modified-2.7/email/errors.py deleted file mode 100644 --- a/lib-python/modified-2.7/email/errors.py +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright (C) 2001-2006 Python Software Foundation -# Author: Barry Warsaw -# Contact: email-sig at python.org - -"""email package exception classes.""" - - - -class MessageError(Exception): - """Base class for errors in the email package.""" - - -class MessageParseError(MessageError): - """Base class for message parsing errors.""" - - -class HeaderParseError(MessageParseError): - """Error while parsing headers.""" - - -class BoundaryError(MessageParseError): - """Couldn't find terminating boundary.""" - - -class MultipartConversionError(MessageError, TypeError): - """Conversion to a multipart is prohibited.""" - - -class CharsetError(MessageError): - """An illegal charset was given.""" - - - -# These are parsing defects which the parser was able to work around. -class MessageDefect: - """Base class for a message defect.""" - - def __init__(self, line=None): - self.line = line - -class NoBoundaryInMultipartDefect(MessageDefect): - """A message claimed to be a multipart but had no boundary parameter.""" - -class StartBoundaryNotFoundDefect(MessageDefect): - """The claimed start boundary was never found.""" - -class FirstHeaderLineIsContinuationDefect(MessageDefect): - """A message had a continuation line as its first header line.""" - -class MisplacedEnvelopeHeaderDefect(MessageDefect): - """A 'Unix-from' header was found in the middle of a header block.""" - -class MalformedHeaderDefect(MessageDefect): - """Found a header that was missing a colon, or was otherwise malformed.""" - -class MultipartInvariantViolationDefect(MessageDefect): - """A message claimed to be a multipart but no subparts were found.""" diff --git a/lib-python/modified-2.7/email/feedparser.py b/lib-python/modified-2.7/email/feedparser.py deleted file mode 100644 --- a/lib-python/modified-2.7/email/feedparser.py +++ /dev/null @@ -1,484 +0,0 @@ -# Copyright (C) 2004-2006 Python Software Foundation -# Authors: Baxter, Wouters and Warsaw -# Contact: email-sig at python.org - -"""FeedParser - An email feed parser. - -The feed parser implements an interface for incrementally parsing an email -message, line by line. This has advantages for certain applications, such as -those reading email messages off a socket. - -FeedParser.feed() is the primary interface for pushing new data into the -parser. It returns when there's nothing more it can do with the available -data. When you have no more data to push into the parser, call .close(). -This completes the parsing and returns the root message object. - -The other advantage of this parser is that it will never throw a parsing -exception. Instead, when it finds something unexpected, it adds a 'defect' to -the current message. Defects are just instances that live on the message -object's .defects attribute. -""" - -__all__ = ['FeedParser'] - -import re - -from email import errors -from email import message - -NLCRE = re.compile('\r\n|\r|\n') -NLCRE_bol = re.compile('(\r\n|\r|\n)') -NLCRE_eol = re.compile('(\r\n|\r|\n)\Z') -NLCRE_crack = re.compile('(\r\n|\r|\n)') -# RFC 2822 $3.6.8 Optional fields. ftext is %d33-57 / %d59-126, Any character -# except controls, SP, and ":". -headerRE = re.compile(r'^(From |[\041-\071\073-\176]{1,}:|[\t ])') -EMPTYSTRING = '' -NL = '\n' - -NeedMoreData = object() - - - -class BufferedSubFile(object): - """A file-ish object that can have new data loaded into it. - - You can also push and pop line-matching predicates onto a stack. When the - current predicate matches the current line, a false EOF response - (i.e. empty string) is returned instead. This lets the parser adhere to a - simple abstraction -- it parses until EOF closes the current message. - """ - def __init__(self): - # The last partial line pushed into this object. - self._partial = '' - # The list of full, pushed lines, in reverse order - self._lines = [] - # The stack of false-EOF checking predicates. - self._eofstack = [] - # A flag indicating whether the file has been closed or not. - self._closed = False - - def push_eof_matcher(self, pred): - self._eofstack.append(pred) - - def pop_eof_matcher(self): - return self._eofstack.pop() - - def close(self): - # Don't forget any trailing partial line. - self._lines.append(self._partial) - self._partial = '' - self._closed = True - - def readline(self): - if not self._lines: - if self._closed: - return '' - return NeedMoreData - # Pop the line off the stack and see if it matches the current - # false-EOF predicate. - line = self._lines.pop() - # RFC 2046, section 5.1.2 requires us to recognize outer level - # boundaries at any level of inner nesting. Do this, but be sure it's - # in the order of most to least nested. - for ateof in self._eofstack[::-1]: - if ateof(line): - # We're at the false EOF. But push the last line back first. - self._lines.append(line) - return '' - return line - - def unreadline(self, line): - # Let the consumer push a line back into the buffer. - assert line is not NeedMoreData - self._lines.append(line) - - def push(self, data): - """Push some new data into this object.""" - # Handle any previous leftovers - data, self._partial = self._partial + data, '' - # Crack into lines, but preserve the newlines on the end of each - parts = NLCRE_crack.split(data) - # The *ahem* interesting behaviour of re.split when supplied grouping - # parentheses is that the last element of the resulting list is the - # data after the final RE. In the case of a NL/CR terminated string, - # this is the empty string. - self._partial = parts.pop() - #GAN 29Mar09 bugs 1555570, 1721862 Confusion at 8K boundary ending with \r: - # is there a \n to follow later? - if not self._partial and parts and parts[-1].endswith('\r'): - self._partial = parts.pop(-2)+parts.pop() - # parts is a list of strings, alternating between the line contents - # and the eol character(s). Gather up a list of lines after - # re-attaching the newlines. - lines = [] - for i in range(len(parts) // 2): - lines.append(parts[i*2] + parts[i*2+1]) - self.pushlines(lines) - - def pushlines(self, lines): - # Reverse and insert at the front of the lines. - self._lines[:0] = lines[::-1] - - def is_closed(self): - return self._closed - - def __iter__(self): - return self - - def next(self): - line = self.readline() - if line == '': - raise StopIteration - return line - - - -class FeedParser: - """A feed-style parser of email.""" - - def __init__(self, _factory=message.Message): - """_factory is called with no arguments to create a new message obj""" - self._factory = _factory - self._input = BufferedSubFile() - self._msgstack = [] - self._parse = self._parsegen().next - self._cur = None - self._last = None - self._headersonly = False - - # Non-public interface for supporting Parser's headersonly flag - def _set_headersonly(self): - self._headersonly = True - - def feed(self, data): - """Push more data into the parser.""" - self._input.push(data) - self._call_parse() - - def _call_parse(self): - try: - self._parse() - except StopIteration: - pass - - def close(self): - """Parse all remaining data and return the root message object.""" - self._input.close() - self._call_parse() - root = self._pop_message() - assert not self._msgstack - # Look for final set of defects - if root.get_content_maintype() == 'multipart' \ - and not root.is_multipart(): - root.defects.append(errors.MultipartInvariantViolationDefect()) - return root - - def _new_message(self): - msg = self._factory() - if self._cur and self._cur.get_content_type() == 'multipart/digest': - msg.set_default_type('message/rfc822') - if self._msgstack: - self._msgstack[-1].attach(msg) - self._msgstack.append(msg) - self._cur = msg - self._last = msg - - def _pop_message(self): - retval = self._msgstack.pop() - if self._msgstack: - self._cur = self._msgstack[-1] - else: - self._cur = None - return retval - - def _parsegen(self): - # Create a new message and start by parsing headers. - self._new_message() - headers = [] - # Collect the headers, searching for a line that doesn't match the RFC - # 2822 header or continuation pattern (including an empty line). - for line in self._input: - if line is NeedMoreData: - yield NeedMoreData - continue - if not headerRE.match(line): - # If we saw the RFC defined header/body separator - # (i.e. newline), just throw it away. Otherwise the line is - # part of the body so push it back. - if not NLCRE.match(line): - self._input.unreadline(line) - break - headers.append(line) - # Done with the headers, so parse them and figure out what we're - # supposed to see in the body of the message. - self._parse_headers(headers) - # Headers-only parsing is a backwards compatibility hack, which was - # necessary in the older parser, which could throw errors. All - # remaining lines in the input are thrown into the message body. - if self._headersonly: - lines = [] - while True: - line = self._input.readline() - if line is NeedMoreData: - yield NeedMoreData - continue - if line == '': - break - lines.append(line) - self._cur.set_payload(EMPTYSTRING.join(lines)) - return - if self._cur.get_content_type() == 'message/delivery-status': - # message/delivery-status contains blocks of headers separated by - # a blank line. We'll represent each header block as a separate - # nested message object, but the processing is a bit different - # than standard message/* types because there is no body for the - # nested messages. A blank line separates the subparts. - while True: - self._input.push_eof_matcher(NLCRE.match) - for retval in self._parsegen(): - if retval is NeedMoreData: - yield NeedMoreData - continue - break - msg = self._pop_message() - # We need to pop the EOF matcher in order to tell if we're at - # the end of the current file, not the end of the last block - # of message headers. - self._input.pop_eof_matcher() - # The input stream must be sitting at the newline or at the - # EOF. We want to see if we're at the end of this subpart, so - # first consume the blank line, then test the next line to see - # if we're at this subpart's EOF. - while True: - line = self._input.readline() - if line is NeedMoreData: - yield NeedMoreData - continue - break - while True: - line = self._input.readline() - if line is NeedMoreData: - yield NeedMoreData - continue - break - if line == '': - break - # Not at EOF so this is a line we're going to need. - self._input.unreadline(line) - return - if self._cur.get_content_maintype() == 'message': - # The message claims to be a message/* type, then what follows is - # another RFC 2822 message. - for retval in self._parsegen(): - if retval is NeedMoreData: - yield NeedMoreData - continue - break - self._pop_message() - return - if self._cur.get_content_maintype() == 'multipart': - boundary = self._cur.get_boundary() - if boundary is None: - # The message /claims/ to be a multipart but it has not - # defined a boundary. That's a problem which we'll handle by - # reading everything until the EOF and marking the message as - # defective. - self._cur.defects.append(errors.NoBoundaryInMultipartDefect()) - lines = [] - for line in self._input: - if line is NeedMoreData: - yield NeedMoreData - continue - lines.append(line) - self._cur.set_payload(EMPTYSTRING.join(lines)) - return - # Create a line match predicate which matches the inter-part - # boundary as well as the end-of-multipart boundary. Don't push - # this onto the input stream until we've scanned past the - # preamble. - separator = '--' + boundary - boundaryre = re.compile( - '(?P<sep>' + re.escape(separator) + - r')(?P<end>--)?(?P<ws>[ \t]*)(?P<linesep>\r\n|\r|\n)?$') - capturing_preamble = True - preamble = [] - linesep = False - while True: - line = self._input.readline() - if line is NeedMoreData: - yield NeedMoreData - continue - if line == '': - break - mo = boundaryre.match(line) - if mo: - # If we're looking at the end boundary, we're done with - # this multipart. If there was a newline at the end of - # the closing boundary, then we need to initialize the - # epilogue with the empty string (see below). - if mo.group('end'): - linesep = mo.group('linesep') - break - # We saw an inter-part boundary. Were we in the preamble? - if capturing_preamble: - if preamble: - # According to RFC 2046, the last newline belongs - # to the boundary. - lastline = preamble[-1] - eolmo = NLCRE_eol.search(lastline) - if eolmo: - preamble[-1] = lastline[:-len(eolmo.group(0))] - self._cur.preamble = EMPTYSTRING.join(preamble) - capturing_preamble = False - self._input.unreadline(line) - continue - # We saw a boundary separating two parts. Consume any - # multiple boundary lines that may be following. Our - # interpretation of RFC 2046 BNF grammar does not produce - # body parts within such double boundaries. - while True: - line = self._input.readline() - if line is NeedMoreData: - yield NeedMoreData - continue - mo = boundaryre.match(line) - if not mo: - self._input.unreadline(line) - break - # Recurse to parse this subpart; the input stream points - # at the subpart's first line. - self._input.push_eof_matcher(boundaryre.match) - for retval in self._parsegen(): - if retval is NeedMoreData: - yield NeedMoreData - continue - break - # Because of RFC 2046, the newline preceding the boundary - # separator actually belongs to the boundary, not the - # previous subpart's payload (or epilogue if the previous - # part is a multipart). - if self._last.get_content_maintype() == 'multipart': - epilogue = self._last.epilogue - if epilogue == '': - self._last.epilogue = None - elif epilogue is not None: - mo = NLCRE_eol.search(epilogue) - if mo: - end = len(mo.group(0)) - self._last.epilogue = epilogue[:-end] - else: - payload = self._last.get_payload() - if isinstance(payload, basestring): - mo = NLCRE_eol.search(payload) - if mo: - payload = payload[:-len(mo.group(0))] - self._last.set_payload(payload) - self._input.pop_eof_matcher() - self._pop_message() - # Set the multipart up for newline cleansing, which will - # happen if we're in a nested multipart. - self._last = self._cur - else: - # I think we must be in the preamble - assert capturing_preamble - preamble.append(line) - # We've seen either the EOF or the end boundary. If we're still - # capturing the preamble, we never saw the start boundary. Note - # that as a defect and store the captured text as the payload. - # Everything from here to the EOF is epilogue. - if capturing_preamble: - self._cur.defects.append(errors.StartBoundaryNotFoundDefect()) - self._cur.set_payload(EMPTYSTRING.join(preamble)) - epilogue = [] - for line in self._input: - if line is NeedMoreData: - yield NeedMoreData - continue - self._cur.epilogue = EMPTYSTRING.join(epilogue) - return - # If the end boundary ended in a newline, we'll need to make sure - # the epilogue isn't None - if linesep: - epilogue = [''] - else: - epilogue = [] - for line in self._input: - if line is NeedMoreData: - yield NeedMoreData - continue - epilogue.append(line) - # Any CRLF at the front of the epilogue is not technically part of - # the epilogue. Also, watch out for an empty string epilogue, - # which means a single newline. - if epilogue: - firstline = epilogue[0] - bolmo = NLCRE_bol.match(firstline) - if bolmo: - epilogue[0] = firstline[len(bolmo.group(0)):] - self._cur.epilogue = EMPTYSTRING.join(epilogue) - return - # Otherwise, it's some non-multipart type, so the entire rest of the - # file contents becomes the payload. - lines = [] - for line in self._input: - if line is NeedMoreData: - yield NeedMoreData - continue - lines.append(line) - self._cur.set_payload(EMPTYSTRING.join(lines)) - - def _parse_headers(self, lines): - # Passed a list of lines that make up the headers for the current msg - lastheader = '' - lastvalue = [] - for lineno, line in enumerate(lines): - # Check for continuation - if line[0] in ' \t': - if not lastheader: - # The first line of the headers was a continuation. This - # is illegal, so let's note the defect, store the illegal - # line, and ignore it for purposes of headers. - defect = errors.FirstHeaderLineIsContinuationDefect(line) - self._cur.defects.append(defect) - continue - lastvalue.append(line) - continue - if lastheader: - # XXX reconsider the joining of folded lines - lhdr = EMPTYSTRING.join(lastvalue)[:-1].rstrip('\r\n') - self._cur[lastheader] = lhdr - lastheader, lastvalue = '', [] - # Check for envelope header, i.e. unix-from - if line.startswith('From '): - if lineno == 0: - # Strip off the trailing newline - mo = NLCRE_eol.search(line) - if mo: - line = line[:-len(mo.group(0))] - self._cur.set_unixfrom(line) - continue - elif lineno == len(lines) - 1: - # Something looking like a unix-from at the end - it's - # probably the first line of the body, so push back the - # line and stop. - self._input.unreadline(line) - return - else: - # Weirdly placed unix-from line. Note this as a defect - # and ignore it. - defect = errors.MisplacedEnvelopeHeaderDefect(line) - self._cur.defects.append(defect) - continue - # Split the line on the colon separating field name from value. - i = line.find(':') - if i < 0: - defect = errors.MalformedHeaderDefect(line) - self._cur.defects.append(defect) - continue - lastheader = line[:i] - lastvalue = [line[i+1:].lstrip()] - # Done with all the lines, so handle the last header. - if lastheader: - # XXX reconsider the joining of folded lines - self._cur[lastheader] = EMPTYSTRING.join(lastvalue).rstrip('\r\n') diff --git a/lib-python/modified-2.7/email/generator.py b/lib-python/modified-2.7/email/generator.py deleted file mode 100644 --- a/lib-python/modified-2.7/email/generator.py +++ /dev/null @@ -1,364 +0,0 @@ -# Copyright (C) 2001-2010 Python Software Foundation -# Contact: email-sig at python.org - -"""Classes to generate plain text from a message object tree.""" - -__all__ = ['Generator', 'DecodedGenerator'] - -import re -import sys -import time -import random -import warnings - -from cStringIO import StringIO -from email.header import Header - -UNDERSCORE = '_' -NL = '\n' - -fcre = re.compile(r'^From ', re.MULTILINE) - -def _is8bitstring(s): - if isinstance(s, str): - try: - unicode(s, 'us-ascii') - except UnicodeError: - return True - return False - - - -class Generator: - """Generates output from a Message object tree. - - This basic generator writes the message to the given file object as plain - text. - """ - # - # Public interface - # - - def __init__(self, outfp, mangle_from_=True, maxheaderlen=78): - """Create the generator for message flattening. - - outfp is the output file-like object for writing the message to. It - must have a write() method. - - Optional mangle_from_ is a flag that, when True (the default), escapes - From_ lines in the body of the message by putting a `>' in front of - them. - - Optional maxheaderlen specifies the longest length for a non-continued - header. When a header line is longer (in characters, with tabs - expanded to 8 spaces) than maxheaderlen, the header will split as - defined in the Header class. Set maxheaderlen to zero to disable - header wrapping. The default is 78, as recommended (but not required) - by RFC 2822. - """ - self._fp = outfp - self._mangle_from_ = mangle_from_ - self._maxheaderlen = maxheaderlen - - def write(self, s): - # Just delegate to the file object - self._fp.write(s) - - def flatten(self, msg, unixfrom=False): - """Print the message object tree rooted at msg to the output file - specified when the Generator instance was created. - - unixfrom is a flag that forces the printing of a Unix From_ delimiter - before the first object in the message tree. If the original message - has no From_ delimiter, a `standard' one is crafted. By default, this - is False to inhibit the printing of any From_ delimiter. - - Note that for subobjects, no From_ line is printed. - """ - if unixfrom: - ufrom = msg.get_unixfrom() - if not ufrom: - ufrom = 'From nobody ' + time.ctime(time.time()) - print >> self._fp, ufrom - self._write(msg) - - def clone(self, fp): - """Clone this generator with the exact same options.""" - return self.__class__(fp, self._mangle_from_, self._maxheaderlen) - - # - # Protected interface - undocumented ;/ - # - - def _write(self, msg): - # We can't write the headers yet because of the following scenario: - # say a multipart message includes the boundary string somewhere in - # its body. We'd have to calculate the new boundary /before/ we write - # the headers so that we can write the correct Content-Type: - # parameter. - # - # The way we do this, so as to make the _handle_*() methods simpler, - # is to cache any subpart writes into a StringIO. The we write the - # headers and the StringIO contents. That way, subpart handlers can - # Do The Right Thing, and can still modify the Content-Type: header if - # necessary. - oldfp = self._fp - try: - self._fp = sfp = StringIO() - self._dispatch(msg) - finally: - self._fp = oldfp - # Write the headers. First we see if the message object wants to - # handle that itself. If not, we'll do it generically. - meth = getattr(msg, '_write_headers', None) - if meth is None: - self._write_headers(msg) - else: - meth(self) - self._fp.write(sfp.getvalue()) - - def _dispatch(self, msg): - # Get the Content-Type: for the message, then try to dispatch to - # self._handle_<maintype>_<subtype>(). If there's no handler for the - # full MIME type, then dispatch to self._handle_<maintype>(). If - # that's missing too, then dispatch to self._writeBody(). - main = msg.get_content_maintype() - sub = msg.get_content_subtype() - specific = UNDERSCORE.join((main, sub)).replace('-', '_') - meth = getattr(self, '_handle_' + specific, None) - if meth is None: - generic = main.replace('-', '_') - meth = getattr(self, '_handle_' + generic, None) - if meth is None: - meth = self._writeBody - meth(msg) - - # - # Default handlers - # - - def _write_headers(self, msg): - for h, v in msg.items(): - print >> self._fp, '%s:' % h, - if self._maxheaderlen == 0: - # Explicit no-wrapping - print >> self._fp, v - elif isinstance(v, Header): - # Header instances know what to do - print >> self._fp, v.encode() - elif _is8bitstring(v): - # If we have raw 8bit data in a byte string, we have no idea - # what the encoding is. There is no safe way to split this - # string. If it's ascii-subset, then we could do a normal - # ascii split, but if it's multibyte then we could break the - # string. There's no way to know so the least harm seems to - # be to not split the string and risk it being too long. - print >> self._fp, v - else: - # Header's got lots of smarts, so use it. Note that this is - # fundamentally broken though because we lose idempotency when - # the header string is continued with tabs. It will now be - # continued with spaces. This was reversedly broken before we - # fixed bug 1974. Either way, we lose. - print >> self._fp, Header( - v, maxlinelen=self._maxheaderlen, header_name=h).encode() - # A blank line always separates headers from body - print >> self._fp - - # - # Handlers for writing types and subtypes - # - - def _handle_text(self, msg): - payload = msg.get_payload() - if payload is None: - return - if not isinstance(payload, basestring): - raise TypeError('string payload expected: %s' % type(payload)) - if self._mangle_from_: - payload = fcre.sub('>From ', payload) - self._fp.write(payload) - - # Default body handler - _writeBody = _handle_text - - def _handle_multipart(self, msg): - # The trick here is to write out each part separately, merge them all - # together, and then make sure that the boundary we've chosen isn't - # present in the payload. - msgtexts = [] - subparts = msg.get_payload() - if subparts is None: - subparts = [] - elif isinstance(subparts, basestring): - # e.g. a non-strict parse of a message with no starting boundary. - self._fp.write(subparts) - return - elif not isinstance(subparts, list): - # Scalar payload - subparts = [subparts] - for part in subparts: - s = StringIO() - g = self.clone(s) - g.flatten(part, unixfrom=False) - msgtexts.append(s.getvalue()) - # BAW: What about boundaries that are wrapped in double-quotes? - boundary = msg.get_boundary() - if not boundary: - # Create a boundary that doesn't appear in any of the - # message texts. - alltext = NL.join(msgtexts) - boundary = _make_boundary(alltext) - msg.set_boundary(boundary) - # If there's a preamble, write it out, with a trailing CRLF - if msg.preamble is not None: - print >> self._fp, msg.preamble - # dash-boundary transport-padding CRLF - print >> self._fp, '--' + boundary - # body-part - if msgtexts: - self._fp.write(msgtexts.pop(0)) - # *encapsulation - # --> delimiter transport-padding - # --> CRLF body-part - for body_part in msgtexts: - # delimiter transport-padding CRLF - print >> self._fp, '\n--' + boundary - # body-part - self._fp.write(body_part) - # close-delimiter transport-padding - self._fp.write('\n--' + boundary + '--') - if msg.epilogue is not None: - print >> self._fp - self._fp.write(msg.epilogue) - - def _handle_multipart_signed(self, msg): - # The contents of signed parts has to stay unmodified in order to keep - # the signature intact per RFC1847 2.1, so we disable header wrapping. - # RDM: This isn't enough to completely preserve the part, but it helps. - old_maxheaderlen = self._maxheaderlen - try: - self._maxheaderlen = 0 - self._handle_multipart(msg) - finally: - self._maxheaderlen = old_maxheaderlen - - def _handle_message_delivery_status(self, msg): - # We can't just write the headers directly to self's file object - # because this will leave an extra newline between the last header - # block and the boundary. Sigh. - blocks = [] - for part in msg.get_payload(): - s = StringIO() - g = self.clone(s) - g.flatten(part, unixfrom=False) - text = s.getvalue() - lines = text.split('\n') - # Strip off the unnecessary trailing empty line - if lines and lines[-1] == '': - blocks.append(NL.join(lines[:-1])) - else: - blocks.append(text) - # Now join all the blocks with an empty line. This has the lovely - # effect of separating each block with an empty line, but not adding - # an extra one after the last one. - self._fp.write(NL.join(blocks)) - - def _handle_message(self, msg): - s = StringIO() - g = self.clone(s) - # The payload of a message/rfc822 part should be a multipart sequence - # of length 1. The zeroth element of the list should be the Message - # object for the subpart. Extract that object, stringify it, and - # write it out. - # Except, it turns out, when it's a string instead, which happens when - # and only when HeaderParser is used on a message of mime type - # message/rfc822. Such messages are generated by, for example, - # Groupwise when forwarding unadorned messages. (Issue 7970.) So - # in that case we just emit the string body. - payload = msg.get_payload() - if isinstance(payload, list): - g.flatten(msg.get_payload(0), unixfrom=False) - payload = s.getvalue() - self._fp.write(payload) - - - -_FMT = '[Non-text (%(type)s) part of message omitted, filename %(filename)s]' - -class DecodedGenerator(Generator): - """Generates a text representation of a message. - - Like the Generator base class, except that non-text parts are substituted - with a format string representing the part. - """ - def __init__(self, outfp, mangle_from_=True, maxheaderlen=78, fmt=None): - """Like Generator.__init__() except that an additional optional - argument is allowed. - - Walks through all subparts of a message. If the subpart is of main - type `text', then it prints the decoded payload of the subpart. - - Otherwise, fmt is a format string that is used instead of the message - payload. fmt is expanded with the following keywords (in - %(keyword)s format): - - type : Full MIME type of the non-text part - maintype : Main MIME type of the non-text part - subtype : Sub-MIME type of the non-text part - filename : Filename of the non-text part - description: Description associated with the non-text part - encoding : Content transfer encoding of the non-text part - - The default value for fmt is None, meaning - - [Non-text (%(type)s) part of message omitted, filename %(filename)s] - """ - Generator.__init__(self, outfp, mangle_from_, maxheaderlen) - if fmt is None: - self._fmt = _FMT - else: - self._fmt = fmt - - def _dispatch(self, msg): - for part in msg.walk(): - maintype = part.get_content_maintype() - if maintype == 'text': - print >> self, part.get_payload(decode=True) - elif maintype == 'multipart': - # Just skip this - pass - else: - print >> self, self._fmt % { - 'type' : part.get_content_type(), - 'maintype' : part.get_content_maintype(), - 'subtype' : part.get_content_subtype(), - 'filename' : part.get_filename('[no filename]'), - 'description': part.get('Content-Description', - '[no description]'), - 'encoding' : part.get('Content-Transfer-Encoding', - '[no encoding]'), - } - - - -# Helper -_width = len(repr(sys.maxint-1)) -_fmt = '%%0%dd' % _width - -def _make_boundary(text=None): - # Craft a random boundary. If text is given, ensure that the chosen - # boundary doesn't appear in the text. - token = random.randrange(sys.maxint) - boundary = ('=' * 15) + (_fmt % token) + '==' - if text is None: - return boundary - b = boundary - counter = 0 - while True: - cre = re.compile('^--' + re.escape(b) + '(--)?$', re.MULTILINE) - if not cre.search(text): - break - b = boundary + '.' + str(counter) - counter += 1 - return b diff --git a/lib-python/modified-2.7/email/header.py b/lib-python/modified-2.7/email/header.py deleted file mode 100644 --- a/lib-python/modified-2.7/email/header.py +++ /dev/null @@ -1,514 +0,0 @@ -# Copyright (C) 2002-2006 Python Software Foundation -# Author: Ben Gertzfield, Barry Warsaw -# Contact: email-sig at python.org - -"""Header encoding and decoding functionality.""" - -__all__ = [ - 'Header', - 'decode_header', - 'make_header', - ] - -import re -import binascii - -import email.quoprimime -import email.base64mime - -from email.errors import HeaderParseError -from email.charset import Charset - -NL = '\n' -SPACE = ' ' -USPACE = u' ' -SPACE8 = ' ' * 8 -UEMPTYSTRING = u'' - -MAXLINELEN = 76 - -USASCII = Charset('us-ascii') -UTF8 = Charset('utf-8') - -# Match encoded-word strings in the form =?charset?q?Hello_World?= -ecre = re.compile(r''' - =\? # literal =? - (?P<charset>[^?]*?) # non-greedy up to the next ? is the charset - \? # literal ? - (?P<encoding>[qb]) # either a "q" or a "b", case insensitive - \? # literal ? - (?P<encoded>.*?) # non-greedy up to the next ?= is the encoded string - \?= # literal ?= - (?=[ \t]|$) # whitespace or the end of the string - ''', re.VERBOSE | re.IGNORECASE | re.MULTILINE) - -# Field name regexp, including trailing colon, but not separating whitespace, -# according to RFC 2822. Character range is from tilde to exclamation mark. -# For use with .match() -fcre = re.compile(r'[\041-\176]+:$') - -# Find a header embedded in a putative header value. Used to check for -# header injection attack. -_embeded_header = re.compile(r'\n[^ \t]+:') - - - -# Helpers -_max_append = email.quoprimime._max_append - - - -def decode_header(header): - """Decode a message header value without converting charset. - - Returns a list of (decoded_string, charset) pairs containing each of the - decoded parts of the header. Charset is None for non-encoded parts of the - header, otherwise a lower-case string containing the name of the character - set specified in the encoded string. - - An email.errors.HeaderParseError may be raised when certain decoding error - occurs (e.g. a base64 decoding exception). - """ - # If no encoding, just return the header - header = str(header) - if not ecre.search(header): - return [(header, None)] - decoded = [] - dec = '' - for line in header.splitlines(): - # This line might not have an encoding in it - if not ecre.search(line): - decoded.append((line, None)) - continue - parts = ecre.split(line) - while parts: - unenc = parts.pop(0).strip() - if unenc: - # Should we continue a long line? - if decoded and decoded[-1][1] is None: - decoded[-1] = (decoded[-1][0] + SPACE + unenc, None) - else: - decoded.append((unenc, None)) - if parts: - charset, encoding = [s.lower() for s in parts[0:2]] - encoded = parts[2] - dec = None - if encoding == 'q': - dec = email.quoprimime.header_decode(encoded) - elif encoding == 'b': - paderr = len(encoded) % 4 # Postel's law: add missing padding - if paderr: - encoded += '==='[:4 - paderr] - try: - dec = email.base64mime.decode(encoded) - except binascii.Error: - # Turn this into a higher level exception. BAW: Right - # now we throw the lower level exception away but - # when/if we get exception chaining, we'll preserve it. - raise HeaderParseError - if dec is None: - dec = encoded - - if decoded and decoded[-1][1] == charset: - decoded[-1] = (decoded[-1][0] + dec, decoded[-1][1]) - else: - decoded.append((dec, charset)) - del parts[0:3] - return decoded - - - -def make_header(decoded_seq, maxlinelen=None, header_name=None, - continuation_ws=' '): - """Create a Header from a sequence of pairs as returned by decode_header() - - decode_header() takes a header value string and returns a sequence of - pairs of the format (decoded_string, charset) where charset is the string - name of the character set. - - This function takes one of those sequence of pairs and returns a Header - instance. Optional maxlinelen, header_name, and continuation_ws are as in - the Header constructor. - """ - h = Header(maxlinelen=maxlinelen, header_name=header_name, - continuation_ws=continuation_ws) - for s, charset in decoded_seq: - # None means us-ascii but we can simply pass it on to h.append() - if charset is not None and not isinstance(charset, Charset): - charset = Charset(charset) - h.append(s, charset) - return h - - - -class Header: - def __init__(self, s=None, charset=None, - maxlinelen=None, header_name=None, - continuation_ws=' ', errors='strict'): - """Create a MIME-compliant header that can contain many character sets. - - Optional s is the initial header value. If None, the initial header - value is not set. You can later append to the header with .append() - method calls. s may be a byte string or a Unicode string, but see the - .append() documentation for semantics. - - Optional charset serves two purposes: it has the same meaning as the - charset argument to the .append() method. It also sets the default - character set for all subsequent .append() calls that omit the charset - argument. If charset is not provided in the constructor, the us-ascii - charset is used both as s's initial charset and as the default for - subsequent .append() calls. - - The maximum line length can be specified explicit via maxlinelen. For - splitting the first line to a shorter value (to account for the field - header which isn't included in s, e.g. `Subject') pass in the name of - the field in header_name. The default maxlinelen is 76. - - continuation_ws must be RFC 2822 compliant folding whitespace (usually - either a space or a hard tab) which will be prepended to continuation - lines. - - errors is passed through to the .append() call. - """ - if charset is None: - charset = USASCII - if not isinstance(charset, Charset): - charset = Charset(charset) - self._charset = charset - self._continuation_ws = continuation_ws - cws_expanded_len = len(continuation_ws.replace('\t', SPACE8)) - # BAW: I believe `chunks' and `maxlinelen' should be non-public. - self._chunks = [] - if s is not None: - self.append(s, charset, errors) - if maxlinelen is None: - maxlinelen = MAXLINELEN - if header_name is None: - # We don't know anything about the field header so the first line - # is the same length as subsequent lines. - self._firstlinelen = maxlinelen - else: - # The first line should be shorter to take into account the field - # header. Also subtract off 2 extra for the colon and space. - self._firstlinelen = maxlinelen - len(header_name) - 2 - # Second and subsequent lines should subtract off the length in - # columns of the continuation whitespace prefix. - self._maxlinelen = maxlinelen - cws_expanded_len - - def __str__(self): - """A synonym for self.encode().""" - return self.encode() - - def __unicode__(self): - """Helper for the built-in unicode function.""" - uchunks = [] - lastcs = None - for s, charset in self._chunks: - # We must preserve spaces between encoded and non-encoded word - # boundaries, which means for us we need to add a space when we go - # from a charset to None/us-ascii, or from None/us-ascii to a - # charset. Only do this for the second and subsequent chunks. - nextcs = charset - if uchunks: - if lastcs not in (None, 'us-ascii'): - if nextcs in (None, 'us-ascii'): - uchunks.append(USPACE) - nextcs = None - elif nextcs not in (None, 'us-ascii'): - uchunks.append(USPACE) - lastcs = nextcs - uchunks.append(unicode(s, str(charset))) - return UEMPTYSTRING.join(uchunks) - - # Rich comparison operators for equality only. BAW: does it make sense to - # have or explicitly disable <, <=, >, >= operators? - def __eq__(self, other): - # other may be a Header or a string. Both are fine so coerce - # ourselves to a string, swap the args and do another comparison. - return other == self.encode() - - def __ne__(self, other): - return not self == other - - def append(self, s, charset=None, errors='strict'): - """Append a string to the MIME header. - - Optional charset, if given, should be a Charset instance or the name - of a character set (which will be converted to a Charset instance). A - value of None (the default) means that the charset given in the - constructor is used. - - s may be a byte string or a Unicode string. If it is a byte string - (i.e. isinstance(s, str) is true), then charset is the encoding of - that byte string, and a UnicodeError will be raised if the string - cannot be decoded with that charset. If s is a Unicode string, then - charset is a hint specifying the character set of the characters in - the string. In this case, when producing an RFC 2822 compliant header - using RFC 2047 rules, the Unicode string will be encoded using the - following charsets in order: us-ascii, the charset hint, utf-8. The - first character set not to provoke a UnicodeError is used. - - Optional `errors' is passed as the third argument to any unicode() or - ustr.encode() call. - """ - if charset is None: - charset = self._charset - elif not isinstance(charset, Charset): - charset = Charset(charset) - # If the charset is our faux 8bit charset, leave the string unchanged - if charset != '8bit': - # We need to test that the string can be converted to unicode and - # back to a byte string, given the input and output codecs of the - # charset. - if isinstance(s, str): - # Possibly raise UnicodeError if the byte string can't be - # converted to a unicode with the input codec of the charset. - incodec = charset.input_codec or 'us-ascii' - ustr = unicode(s, incodec, errors) - # Now make sure that the unicode could be converted back to a - # byte string with the output codec, which may be different - # than the iput coded. Still, use the original byte string. - outcodec = charset.output_codec or 'us-ascii' - ustr.encode(outcodec, errors) - elif isinstance(s, unicode): - # Now we have to be sure the unicode string can be converted - # to a byte string with a reasonable output codec. We want to - # use the byte string in the chunk. - for charset in USASCII, charset, UTF8: - try: - outcodec = charset.output_codec or 'us-ascii' - s = s.encode(outcodec, errors) - break - except UnicodeError: - pass - else: - assert False, 'utf-8 conversion failed' - self._chunks.append((s, charset)) - - def _split(self, s, charset, maxlinelen, splitchars): - # Split up a header safely for use with encode_chunks. - splittable = charset.to_splittable(s) - encoded = charset.from_splittable(splittable, True) - elen = charset.encoded_header_len(encoded) - # If the line's encoded length first, just return it - if elen <= maxlinelen: - return [(encoded, charset)] - # If we have undetermined raw 8bit characters sitting in a byte - # string, we really don't know what the right thing to do is. We - # can't really split it because it might be multibyte data which we - # could break if we split it between pairs. The least harm seems to - # be to not split the header at all, but that means they could go out - # longer than maxlinelen. - if charset == '8bit': - return [(s, charset)] - # BAW: I'm not sure what the right test here is. What we're trying to - # do is be faithful to RFC 2822's recommendation that ($2.2.3): - # - # "Note: Though structured field bodies are defined in such a way that - # folding can take place between many of the lexical tokens (and even - # within some of the lexical tokens), folding SHOULD be limited to - # placing the CRLF at higher-level syntactic breaks." - # - # For now, I can only imagine doing this when the charset is us-ascii, - # although it's possible that other charsets may also benefit from the - # higher-level syntactic breaks. - elif charset == 'us-ascii': - return self._split_ascii(s, charset, maxlinelen, splitchars) - # BAW: should we use encoded? - elif elen == len(s): - # We can split on _maxlinelen boundaries because we know that the - # encoding won't change the size of the string - splitpnt = maxlinelen - first = charset.from_splittable(splittable[:splitpnt], False) - last = charset.from_splittable(splittable[splitpnt:], False) - else: - # Binary search for split point - first, last = _binsplit(splittable, charset, maxlinelen) - # first is of the proper length so just wrap it in the appropriate - # chrome. last must be recursively split. - fsplittable = charset.to_splittable(first) - fencoded = charset.from_splittable(fsplittable, True) - chunk = [(fencoded, charset)] - return chunk + self._split(last, charset, self._maxlinelen, splitchars) - - def _split_ascii(self, s, charset, firstlen, splitchars): - chunks = _split_ascii(s, firstlen, self._maxlinelen, - self._continuation_ws, splitchars) - return zip(chunks, [charset]*len(chunks)) - - def _encode_chunks(self, newchunks, maxlinelen): - # MIME-encode a header with many different charsets and/or encodings. - # - # Given a list of pairs (string, charset), return a MIME-encoded - # string suitable for use in a header field. Each pair may have - # different charsets and/or encodings, and the resulting header will - # accurately reflect each setting. - # - # Each encoding can be email.utils.QP (quoted-printable, for - # ASCII-like character sets like iso-8859-1), email.utils.BASE64 - # (Base64, for non-ASCII like character sets like KOI8-R and - # iso-2022-jp), or None (no encoding). - # - # Each pair will be represented on a separate line; the resulting - # string will be in the format: - # - # =?charset1?q?Mar=EDa_Gonz=E1lez_Alonso?=\n - # =?charset2?b?SvxyZ2VuIEL2aW5n?=" - chunks = [] - for header, charset in newchunks: - if not header: - continue - if charset is None or charset.header_encoding is None: - s = header - else: - s = charset.header_encode(header) - # Don't add more folding whitespace than necessary - if chunks and chunks[-1].endswith(' '): - extra = '' - else: - extra = ' ' - _max_append(chunks, s, maxlinelen, extra) - joiner = NL + self._continuation_ws - return joiner.join(chunks) - - def encode(self, splitchars=';, '): - """Encode a message header into an RFC-compliant format. - - There are many issues involved in converting a given string for use in - an email header. Only certain character sets are readable in most - email clients, and as header strings can only contain a subset of - 7-bit ASCII, care must be taken to properly convert and encode (with - Base64 or quoted-printable) header strings. In addition, there is a - 75-character length limit on any given encoded header field, so - line-wrapping must be performed, even with double-byte character sets. - - This method will do its best to convert the string to the correct - character set used in email, and encode and line wrap it safely with - the appropriate scheme for that character set. - - If the given charset is not known or an error occurs during - conversion, this function will return the header untouched. - - Optional splitchars is a string containing characters to split long - ASCII lines on, in rough support of RFC 2822's `highest level - syntactic breaks'. This doesn't affect RFC 2047 encoded lines. - """ - newchunks = [] - maxlinelen = self._firstlinelen - lastlen = 0 - for s, charset in self._chunks: - # The first bit of the next chunk should be just long enough to - # fill the next line. Don't forget the space separating the - # encoded words. - targetlen = maxlinelen - lastlen - 1 - if targetlen < charset.encoded_header_len(''): - # Stick it on the next line - targetlen = maxlinelen - newchunks += self._split(s, charset, targetlen, splitchars) - lastchunk, lastcharset = newchunks[-1] - lastlen = lastcharset.encoded_header_len(lastchunk) - value = self._encode_chunks(newchunks, maxlinelen) - if _embeded_header.search(value): - raise HeaderParseError("header value appears to contain " - "an embedded header: {!r}".format(value)) - return value - - - -def _split_ascii(s, firstlen, restlen, continuation_ws, splitchars): - lines = [] - maxlen = firstlen - for line in s.splitlines(): - # Ignore any leading whitespace (i.e. continuation whitespace) already - # on the line, since we'll be adding our own. - line = line.lstrip() - if len(line) < maxlen: - lines.append(line) - maxlen = restlen - continue - # Attempt to split the line at the highest-level syntactic break - # possible. Note that we don't have a lot of smarts about field - # syntax; we just try to break on semi-colons, then commas, then - # whitespace. - for ch in splitchars: - if ch in line: - break - else: - # There's nothing useful to split the line on, not even spaces, so - # just append this line unchanged - lines.append(line) - maxlen = restlen - continue - # Now split the line on the character plus trailing whitespace - cre = re.compile(r'%s\s*' % ch) - if ch in ';,': - eol = ch - else: - eol = '' - joiner = eol + ' ' - joinlen = len(joiner) - wslen = len(continuation_ws.replace('\t', SPACE8)) - this = [] - linelen = 0 - for part in cre.split(line): - curlen = linelen + max(0, len(this)-1) * joinlen - partlen = len(part) - onfirstline = not lines - # We don't want to split after the field name, if we're on the - # first line and the field name is present in the header string. - if ch == ' ' and onfirstline and \ - len(this) == 1 and fcre.match(this[0]): - this.append(part) - linelen += partlen - elif curlen + partlen > maxlen: - if this: - lines.append(joiner.join(this) + eol) - # If this part is longer than maxlen and we aren't already - # splitting on whitespace, try to recursively split this line - # on whitespace. - if partlen > maxlen and ch != ' ': - subl = _split_ascii(part, maxlen, restlen, - continuation_ws, ' ') - lines.extend(subl[:-1]) - this = [subl[-1]] - else: - this = [part] - linelen = wslen + len(this[-1]) - maxlen = restlen - else: - this.append(part) - linelen += partlen - # Put any left over parts on a line by themselves - if this: - lines.append(joiner.join(this)) - return lines - - - -def _binsplit(splittable, charset, maxlinelen): - i = 0 - j = len(splittable) - while i < j: - # Invariants: - # 1. splittable[:k] fits for all k <= i (note that we *assume*, - # at the start, that splittable[:0] fits). - # 2. splittable[:k] does not fit for any k > j (at the start, - # this means we shouldn't look at any k > len(splittable)). - # 3. We don't know about splittable[:k] for k in i+1..j. - # 4. We want to set i to the largest k that fits, with i <= k <= j. - # - m = (i+j+1) >> 1 # ceiling((i+j)/2); i < m <= j - chunk = charset.from_splittable(splittable[:m], True) - chunklen = charset.encoded_header_len(chunk) - if chunklen <= maxlinelen: - # m is acceptable, so is a new lower bound. - i = m - else: - # m is not acceptable, so final i must be < m. - j = m - 1 - # i == j. Invariant #1 implies that splittable[:i] fits, and - # invariant #2 implies that splittable[:i+1] does not fit, so i - # is what we're looking for. - first = charset.from_splittable(splittable[:i], False) - last = charset.from_splittable(splittable[i:], False) - return first, last diff --git a/lib-python/modified-2.7/email/iterators.py b/lib-python/modified-2.7/email/iterators.py deleted file mode 100644 --- a/lib-python/modified-2.7/email/iterators.py +++ /dev/null @@ -1,73 +0,0 @@ -# Copyright (C) 2001-2006 Python Software Foundation -# Author: Barry Warsaw -# Contact: email-sig at python.org - -"""Various types of useful iterators and generators.""" - -__all__ = [ - 'body_line_iterator', - 'typed_subpart_iterator', - 'walk', - # Do not include _structure() since it's part of the debugging API. - ] - -import sys -from cStringIO import StringIO - - - -# This function will become a method of the Message class -def walk(self): - """Walk over the message tree, yielding each subpart. - - The walk is performed in depth-first order. This method is a - generator. - """ - yield self - if self.is_multipart(): - for subpart in self.get_payload(): - for subsubpart in subpart.walk(): - yield subsubpart - - - -# These two functions are imported into the Iterators.py interface module. -def body_line_iterator(msg, decode=False): - """Iterate over the parts, returning string payloads line-by-line. - - Optional decode (default False) is passed through to .get_payload(). - """ - for subpart in msg.walk(): - payload = subpart.get_payload(decode=decode) - if isinstance(payload, basestring): - for line in StringIO(payload): - yield line - - -def typed_subpart_iterator(msg, maintype='text', subtype=None): - """Iterate over the subparts with a given MIME type. - - Use `maintype' as the main MIME type to match against; this defaults to - "text". Optional `subtype' is the MIME subtype to match against; if - omitted, only the main type is matched. - """ - for subpart in msg.walk(): - if subpart.get_content_maintype() == maintype: - if subtype is None or subpart.get_content_subtype() == subtype: - yield subpart - - - -def _structure(msg, fp=None, level=0, include_default=False): - """A handy debugging aid""" - if fp is None: - fp = sys.stdout - tab = ' ' * (level * 4) - print >> fp, tab + msg.get_content_type(), - if include_default: - print >> fp, '[%s]' % msg.get_default_type() - else: - print >> fp - if msg.is_multipart(): - for subpart in msg.get_payload(): - _structure(subpart, fp, level+1, include_default) diff --git a/lib-python/modified-2.7/email/message.py b/lib-python/modified-2.7/email/message.py deleted file mode 100644 --- a/lib-python/modified-2.7/email/message.py +++ /dev/null @@ -1,797 +0,0 @@ -# Copyright (C) 2001-2006 Python Software Foundation -# Author: Barry Warsaw -# Contact: email-sig at python.org - -"""Basic message object for the email package object model.""" - -__all__ = ['Message'] - -import re -import uu -import binascii -import warnings -from cStringIO import StringIO - -# Intrapackage imports -import email.charset -from email import utils -from email import errors - -SEMISPACE = '; ' - -# Regular expression that matches `special' characters in parameters, the -# existence of which force quoting of the parameter value. -tspecials = re.compile(r'[ <>@,;:\\"/\[\]\?=]') - - -# Helper functions -def _splitparam(param): - # Split header parameters. BAW: this may be too simple. It isn't - # strictly RFC 2045 (section 5.1) compliant, but it catches most headers - # found in the wild. We may eventually need a full fledged parser - # eventually. - a, sep, b = param.partition(';') - if not sep: - return a.strip(), None - return a.strip(), b.strip() - -def _formatparam(param, value=None, quote=True): - """Convenience function to format and return a key=value pair. - - This will quote the value if needed or if quote is true. If value is a - three tuple (charset, language, value), it will be encoded according - to RFC2231 rules. - """ - if value is not None and len(value) > 0: - # A tuple is used for RFC 2231 encoded parameter values where items - # are (charset, language, value). charset is a string, not a Charset - # instance. - if isinstance(value, tuple): - # Encode as per RFC 2231 - param += '*' - value = utils.encode_rfc2231(value[2], value[0], value[1]) - # BAW: Please check this. I think that if quote is set it should - # force quoting even if not necessary. - if quote or tspecials.search(value): - return '%s="%s"' % (param, utils.quote(value)) - else: - return '%s=%s' % (param, value) - else: - return param - -def _parseparam(s): - plist = [] - while s[:1] == ';': - s = s[1:] - end = s.find(';') - while end > 0 and (s.count('"', 0, end) - s.count('\\"', 0, end)) % 2: - end = s.find(';', end + 1) - if end < 0: - end = len(s) - f = s[:end] - if '=' in f: - i = f.index('=') - f = f[:i].strip().lower() + '=' + f[i+1:].strip() - plist.append(f.strip()) - s = s[end:] - return plist - - -def _unquotevalue(value): - # This is different than utils.collapse_rfc2231_value() because it doesn't - # try to convert the value to a unicode. Message.get_param() and - # Message.get_params() are both currently defined to return the tuple in - # the face of RFC 2231 parameters. - if isinstance(value, tuple): - return value[0], value[1], utils.unquote(value[2]) - else: - return utils.unquote(value) - - - -class Message: - """Basic message object. - - A message object is defined as something that has a bunch of RFC 2822 - headers and a payload. It may optionally have an envelope header - (a.k.a. Unix-From or From_ header). If the message is a container (i.e. a - multipart or a message/rfc822), then the payload is a list of Message - objects, otherwise it is a string. - - Message objects implement part of the `mapping' interface, which assumes - there is exactly one occurrence of the header per message. Some headers - do in fact appear multiple times (e.g. Received) and for those headers, - you must use the explicit API to set or get all the headers. Not all of - the mapping methods are implemented. - """ - def __init__(self): - self._headers = [] - self._unixfrom = None - self._payload = None - self._charset = None - # Defaults for multipart messages - self.preamble = self.epilogue = None - self.defects = [] - # Default content type - self._default_type = 'text/plain' - - def __str__(self): - """Return the entire formatted message as a string. - This includes the headers, body, and envelope header. - """ - return self.as_string(unixfrom=True) - - def as_string(self, unixfrom=False): - """Return the entire formatted message as a string. - Optional `unixfrom' when True, means include the Unix From_ envelope - header. - - This is a convenience method and may not generate the message exactly - as you intend because by default it mangles lines that begin with - "From ". For more flexibility, use the flatten() method of a - Generator instance. - """ - from email.generator import Generator - fp = StringIO() - g = Generator(fp) - g.flatten(self, unixfrom=unixfrom) - return fp.getvalue() - - def is_multipart(self): - """Return True if the message consists of multiple parts.""" - return isinstance(self._payload, list) - - # - # Unix From_ line - # - def set_unixfrom(self, unixfrom): - self._unixfrom = unixfrom - - def get_unixfrom(self): - return self._unixfrom - - # - # Payload manipulation. - # - def attach(self, payload): - """Add the given payload to the current payload. - - The current payload will always be a list of objects after this method - is called. If you want to set the payload to a scalar object, use - set_payload() instead. - """ - if self._payload is None: - self._payload = [payload] - else: - self._payload.append(payload) - - def get_payload(self, i=None, decode=False): - """Return a reference to the payload. - - The payload will either be a list object or a string. If you mutate - the list object, you modify the message's payload in place. Optional - i returns that index into the payload. - - Optional decode is a flag indicating whether the payload should be - decoded or not, according to the Content-Transfer-Encoding header - (default is False). - - When True and the message is not a multipart, the payload will be - decoded if this header's value is `quoted-printable' or `base64'. If - some other encoding is used, or the header is missing, or if the - payload has bogus data (i.e. bogus base64 or uuencoded data), the - payload is returned as-is. - - If the message is a multipart and the decode flag is True, then None - is returned. - """ - if i is None: - payload = self._payload - elif not isinstance(self._payload, list): - raise TypeError('Expected list, got %s' % type(self._payload)) - else: - payload = self._payload[i] - if decode: - if self.is_multipart(): - return None - cte = self.get('content-transfer-encoding', '').lower() - if cte == 'quoted-printable': - return utils._qdecode(payload) - elif cte == 'base64': - try: - return utils._bdecode(payload) - except binascii.Error: - # Incorrect padding - return payload - elif cte in ('x-uuencode', 'uuencode', 'uue', 'x-uue'): - sfp = StringIO() - try: - uu.decode(StringIO(payload+'\n'), sfp, quiet=True) - payload = sfp.getvalue() - except uu.Error: - # Some decoding problem - return payload - # Everything else, including encodings with 8bit or 7bit are returned - # unchanged. - return payload - - def set_payload(self, payload, charset=None): - """Set the payload to the given value. - - Optional charset sets the message's default character set. See - set_charset() for details. - """ - self._payload = payload - if charset is not None: - self.set_charset(charset) - - def set_charset(self, charset): - """Set the charset of the payload to a given character set. - - charset can be a Charset instance, a string naming a character set, or - None. If it is a string it will be converted to a Charset instance. - If charset is None, the charset parameter will be removed from the - Content-Type field. Anything else will generate a TypeError. - - The message will be assumed to be of type text/* encoded with - charset.input_charset. It will be converted to charset.output_charset - and encoded properly, if needed, when generating the plain text - representation of the message. MIME headers (MIME-Version, - Content-Type, Content-Transfer-Encoding) will be added as needed. - - """ - if charset is None: - self.del_param('charset') - self._charset = None - return - if isinstance(charset, basestring): - charset = email.charset.Charset(charset) - if not isinstance(charset, email.charset.Charset): - raise TypeError(charset) - # BAW: should we accept strings that can serve as arguments to the - # Charset constructor? - self._charset = charset - if 'MIME-Version' not in self: - self.add_header('MIME-Version', '1.0') - if 'Content-Type' not in self: - self.add_header('Content-Type', 'text/plain', - charset=charset.get_output_charset()) - else: - self.set_param('charset', charset.get_output_charset()) - if isinstance(self._payload, unicode): - self._payload = self._payload.encode(charset.output_charset) - if str(charset) != charset.get_output_charset(): - self._payload = charset.body_encode(self._payload) - if 'Content-Transfer-Encoding' not in self: - cte = charset.get_body_encoding() - try: - cte(self) - except TypeError: - self._payload = charset.body_encode(self._payload) - self.add_header('Content-Transfer-Encoding', cte) - - def get_charset(self): - """Return the Charset instance associated with the message's payload. - """ - return self._charset - - # - # MAPPING INTERFACE (partial) - # - def __len__(self): - """Return the total number of headers, including duplicates.""" - return len(self._headers) - - def __getitem__(self, name): - """Get a header value. - - Return None if the header is missing instead of raising an exception. - - Note that if the header appeared multiple times, exactly which - occurrence gets returned is undefined. Use get_all() to get all - the values matching a header field name. - """ - return self.get(name) - - def __setitem__(self, name, val): - """Set the value of a header. - - Note: this does not overwrite an existing header with the same field - name. Use __delitem__() first to delete any existing headers. - """ - self._headers.append((name, val)) - - def __delitem__(self, name): - """Delete all occurrences of a header, if present. - - Does not raise an exception if the header is missing. - """ - name = name.lower() - newheaders = [] - for k, v in self._headers: - if k.lower() != name: - newheaders.append((k, v)) - self._headers = newheaders - - def __contains__(self, name): - return name.lower() in [k.lower() for k, v in self._headers] - - def has_key(self, name): - """Return true if the message contains the header.""" - missing = object() - return self.get(name, missing) is not missing - - def keys(self): - """Return a list of all the message's header field names. - - These will be sorted in the order they appeared in the original - message, or were added to the message, and may contain duplicates. - Any fields deleted and re-inserted are always appended to the header - list. - """ - return [k for k, v in self._headers] - - def values(self): - """Return a list of all the message's header values. - - These will be sorted in the order they appeared in the original - message, or were added to the message, and may contain duplicates. - Any fields deleted and re-inserted are always appended to the header - list. - """ - return [v for k, v in self._headers] - - def items(self): - """Get all the message's header fields and values. - - These will be sorted in the order they appeared in the original - message, or were added to the message, and may contain duplicates. - Any fields deleted and re-inserted are always appended to the header - list. - """ - return self._headers[:] - - def get(self, name, failobj=None): - """Get a header value. - - Like __getitem__() but return failobj instead of None when the field - is missing. - """ - name = name.lower() - for k, v in self._headers: - if k.lower() == name: - return v - return failobj - - # - # Additional useful stuff - # - - def get_all(self, name, failobj=None): - """Return a list of all the values for the named field. - - These will be sorted in the order they appeared in the original - message, and may contain duplicates. Any fields deleted and - re-inserted are always appended to the header list. - - If no such fields exist, failobj is returned (defaults to None). - """ - values = [] - name = name.lower() - for k, v in self._headers: - if k.lower() == name: - values.append(v) - if not values: - return failobj - return values - - def add_header(self, _name, _value, **_params): - """Extended header setting. - - name is the header field to add. keyword arguments can be used to set - additional parameters for the header field, with underscores converted - to dashes. Normally the parameter will be added as key="value" unless - value is None, in which case only the key will be added. If a - parameter value contains non-ASCII characters it must be specified as a - three-tuple of (charset, language, value), in which case it will be - encoded according to RFC2231 rules. - - Example: - - msg.add_header('content-disposition', 'attachment', filename='bud.gif') - """ - parts = [] - for k, v in _params.items(): - if v is None: - parts.append(k.replace('_', '-')) - else: - parts.append(_formatparam(k.replace('_', '-'), v)) - if _value is not None: - parts.insert(0, _value) - self._headers.append((_name, SEMISPACE.join(parts))) - - def replace_header(self, _name, _value): - """Replace a header. - - Replace the first matching header found in the message, retaining - header order and case. If no matching header was found, a KeyError is - raised. - """ - _name = _name.lower() - for i, (k, v) in zip(range(len(self._headers)), self._headers): - if k.lower() == _name: - self._headers[i] = (k, _value) - break - else: - raise KeyError(_name) - - # - # Use these three methods instead of the three above. - # - - def get_content_type(self): - """Return the message's content type. - - The returned string is coerced to lower case of the form - `maintype/subtype'. If there was no Content-Type header in the - message, the default type as given by get_default_type() will be - returned. Since according to RFC 2045, messages always have a default - type this will always return a value. - - RFC 2045 defines a message's default type to be text/plain unless it - appears inside a multipart/digest container, in which case it would be - message/rfc822. - """ - missing = object() - value = self.get('content-type', missing) - if value is missing: - # This should have no parameters - return self.get_default_type() - ctype = _splitparam(value)[0].lower() - # RFC 2045, section 5.2 says if its invalid, use text/plain - if ctype.count('/') != 1: - return 'text/plain' - return ctype - - def get_content_maintype(self): - """Return the message's main content type. - - This is the `maintype' part of the string returned by - get_content_type(). - """ - ctype = self.get_content_type() - return ctype.split('/')[0] - - def get_content_subtype(self): - """Returns the message's sub-content type. - - This is the `subtype' part of the string returned by - get_content_type(). - """ - ctype = self.get_content_type() - return ctype.split('/')[1] - - def get_default_type(self): - """Return the `default' content type. - - Most messages have a default content type of text/plain, except for - messages that are subparts of multipart/digest containers. Such - subparts have a default content type of message/rfc822. - """ - return self._default_type - - def set_default_type(self, ctype): - """Set the `default' content type. - - ctype should be either "text/plain" or "message/rfc822", although this - is not enforced. The default content type is not stored in the - Content-Type header. - """ - self._default_type = ctype - - def _get_params_preserve(self, failobj, header): - # Like get_params() but preserves the quoting of values. BAW: - # should this be part of the public interface? - missing = object() - value = self.get(header, missing) - if value is missing: - return failobj - params = [] - for p in _parseparam(';' + value): - try: - name, val = p.split('=', 1) - name = name.strip() - val = val.strip() - except ValueError: - # Must have been a bare attribute - name = p.strip() - val = '' - params.append((name, val)) - params = utils.decode_params(params) - return params - - def get_params(self, failobj=None, header='content-type', unquote=True): - """Return the message's Content-Type parameters, as a list. - - The elements of the returned list are 2-tuples of key/value pairs, as - split on the `=' sign. The left hand side of the `=' is the key, - while the right hand side is the value. If there is no `=' sign in - the parameter the value is the empty string. The value is as - described in the get_param() method. - - Optional failobj is the object to return if there is no Content-Type - header. Optional header is the header to search instead of - Content-Type. If unquote is True, the value is unquoted. - """ - missing = object() - params = self._get_params_preserve(missing, header) - if params is missing: - return failobj - if unquote: - return [(k, _unquotevalue(v)) for k, v in params] - else: - return params - - def get_param(self, param, failobj=None, header='content-type', - unquote=True): - """Return the parameter value if found in the Content-Type header. - - Optional failobj is the object to return if there is no Content-Type - header, or the Content-Type header has no such parameter. Optional - header is the header to search instead of Content-Type. - - Parameter keys are always compared case insensitively. The return - value can either be a string, or a 3-tuple if the parameter was RFC - 2231 encoded. When it's a 3-tuple, the elements of the value are of - the form (CHARSET, LANGUAGE, VALUE). Note that both CHARSET and - LANGUAGE can be None, in which case you should consider VALUE to be - encoded in the us-ascii charset. You can usually ignore LANGUAGE. - - Your application should be prepared to deal with 3-tuple return - values, and can convert the parameter to a Unicode string like so: - - param = msg.get_param('foo') - if isinstance(param, tuple): - param = unicode(param[2], param[0] or 'us-ascii') - - In any case, the parameter value (either the returned string, or the - VALUE item in the 3-tuple) is always unquoted, unless unquote is set - to False. - """ - if header not in self: - return failobj - for k, v in self._get_params_preserve(failobj, header): - if k.lower() == param.lower(): - if unquote: - return _unquotevalue(v) - else: - return v - return failobj - - def set_param(self, param, value, header='Content-Type', requote=True, - charset=None, language=''): - """Set a parameter in the Content-Type header. - - If the parameter already exists in the header, its value will be - replaced with the new value. - - If header is Content-Type and has not yet been defined for this - message, it will be set to "text/plain" and the new parameter and - value will be appended as per RFC 2045. - - An alternate header can specified in the header argument, and all - parameters will be quoted as necessary unless requote is False. - - If charset is specified, the parameter will be encoded according to RFC - 2231. Optional language specifies the RFC 2231 language, defaulting - to the empty string. Both charset and language should be strings. - """ - if not isinstance(value, tuple) and charset: - value = (charset, language, value) - - if header not in self and header.lower() == 'content-type': - ctype = 'text/plain' - else: - ctype = self.get(header) - if not self.get_param(param, header=header): - if not ctype: - ctype = _formatparam(param, value, requote) - else: - ctype = SEMISPACE.join( - [ctype, _formatparam(param, value, requote)]) - else: - ctype = '' - for old_param, old_value in self.get_params(header=header, - unquote=requote): - append_param = '' - if old_param.lower() == param.lower(): - append_param = _formatparam(param, value, requote) - else: - append_param = _formatparam(old_param, old_value, requote) - if not ctype: - ctype = append_param - else: - ctype = SEMISPACE.join([ctype, append_param]) - if ctype != self.get(header): - del self[header] - self[header] = ctype - - def del_param(self, param, header='content-type', requote=True): - """Remove the given parameter completely from the Content-Type header. - - The header will be re-written in place without the parameter or its - value. All values will be quoted as necessary unless requote is - False. Optional header specifies an alternative to the Content-Type - header. - """ - if header not in self: - return - new_ctype = '' - for p, v in self.get_params(header=header, unquote=requote): - if p.lower() != param.lower(): - if not new_ctype: - new_ctype = _formatparam(p, v, requote) - else: - new_ctype = SEMISPACE.join([new_ctype, - _formatparam(p, v, requote)]) - if new_ctype != self.get(header): - del self[header] - self[header] = new_ctype - - def set_type(self, type, header='Content-Type', requote=True): - """Set the main type and subtype for the Content-Type header. - - type must be a string in the form "maintype/subtype", otherwise a - ValueError is raised. - - This method replaces the Content-Type header, keeping all the - parameters in place. If requote is False, this leaves the existing - header's quoting as is. Otherwise, the parameters will be quoted (the - default). - - An alternative header can be specified in the header argument. When - the Content-Type header is set, we'll always also add a MIME-Version - header. - """ - # BAW: should we be strict? - if not type.count('/') == 1: - raise ValueError - # Set the Content-Type, you get a MIME-Version - if header.lower() == 'content-type': - del self['mime-version'] - self['MIME-Version'] = '1.0' - if header not in self: - self[header] = type - return - params = self.get_params(header=header, unquote=requote) - del self[header] - self[header] = type - # Skip the first param; it's the old type. - for p, v in params[1:]: - self.set_param(p, v, header, requote) - - def get_filename(self, failobj=None): - """Return the filename associated with the payload if present. - - The filename is extracted from the Content-Disposition header's - `filename' parameter, and it is unquoted. If that header is missing - the `filename' parameter, this method falls back to looking for the - `name' parameter. - """ - missing = object() - filename = self.get_param('filename', missing, 'content-disposition') - if filename is missing: - filename = self.get_param('name', missing, 'content-type') - if filename is missing: - return failobj - return utils.collapse_rfc2231_value(filename).strip() - - def get_boundary(self, failobj=None): - """Return the boundary associated with the payload if present. - - The boundary is extracted from the Content-Type header's `boundary' - parameter, and it is unquoted. - """ - missing = object() - boundary = self.get_param('boundary', missing) - if boundary is missing: - return failobj - # RFC 2046 says that boundaries may begin but not end in w/s - return utils.collapse_rfc2231_value(boundary).rstrip() - - def set_boundary(self, boundary): - """Set the boundary parameter in Content-Type to 'boundary'. - - This is subtly different than deleting the Content-Type header and - adding a new one with a new boundary parameter via add_header(). The - main difference is that using the set_boundary() method preserves the - order of the Content-Type header in the original message. - - HeaderParseError is raised if the message has no Content-Type header. - """ - missing = object() - params = self._get_params_preserve(missing, 'content-type') - if params is missing: - # There was no Content-Type header, and we don't know what type - # to set it to, so raise an exception. - raise errors.HeaderParseError('No Content-Type header found') - newparams = [] - foundp = False - for pk, pv in params: - if pk.lower() == 'boundary': - newparams.append(('boundary', '"%s"' % boundary)) - foundp = True - else: - newparams.append((pk, pv)) - if not foundp: - # The original Content-Type header had no boundary attribute. - # Tack one on the end. BAW: should we raise an exception - # instead??? - newparams.append(('boundary', '"%s"' % boundary)) - # Replace the existing Content-Type header with the new value - newheaders = [] - for h, v in self._headers: - if h.lower() == 'content-type': - parts = [] - for k, v in newparams: - if v == '': - parts.append(k) - else: - parts.append('%s=%s' % (k, v)) - newheaders.append((h, SEMISPACE.join(parts))) - - else: - newheaders.append((h, v)) - self._headers = newheaders - - def get_content_charset(self, failobj=None): - """Return the charset parameter of the Content-Type header. - - The returned string is always coerced to lower case. If there is no - Content-Type header, or if that header has no charset parameter, - failobj is returned. - """ - missing = object() - charset = self.get_param('charset', missing) - if charset is missing: - return failobj - if isinstance(charset, tuple): - # RFC 2231 encoded, so decode it, and it better end up as ascii. - pcharset = charset[0] or 'us-ascii' - try: - # LookupError will be raised if the charset isn't known to - # Python. UnicodeError will be raised if the encoded text - # contains a character not in the charset. - charset = unicode(charset[2], pcharset).encode('us-ascii') - except (LookupError, UnicodeError): - charset = charset[2] - # charset character must be in us-ascii range - try: - if isinstance(charset, str): - charset = unicode(charset, 'us-ascii') - charset = charset.encode('us-ascii') - except UnicodeError: - return failobj - # RFC 2046, $4.1.2 says charsets are not case sensitive - return charset.lower() - - def get_charsets(self, failobj=None): - """Return a list containing the charset(s) used in this message. - - The returned list of items describes the Content-Type headers' - charset parameter for this message and all the subparts in its - payload. - - Each item will either be a string (the value of the charset parameter - in the Content-Type header of that part) or the value of the - 'failobj' parameter (defaults to None), if the part does not have a - main MIME type of "text", or the charset is not defined. - - The list will contain one string for each part of the message, plus - one for the container message (i.e. self), so that a non-multipart - message will still return a list of length 1. - """ - return [part.get_content_charset(failobj) for part in self.walk()] - - # I.e. def walk(self): ... - from email.iterators import walk diff --git a/lib-python/modified-2.7/email/mime/__init__.py b/lib-python/modified-2.7/email/mime/__init__.py deleted file mode 100644 diff --git a/lib-python/modified-2.7/email/mime/application.py b/lib-python/modified-2.7/email/mime/application.py deleted file mode 100644 --- a/lib-python/modified-2.7/email/mime/application.py +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright (C) 2001-2006 Python Software Foundation -# Author: Keith Dart -# Contact: email-sig at python.org - -"""Class representing application/* type MIME documents.""" - -__all__ = ["MIMEApplication"] - -from email import encoders -from email.mime.nonmultipart import MIMENonMultipart - - -class MIMEApplication(MIMENonMultipart): - """Class for generating application/* MIME documents.""" - - def __init__(self, _data, _subtype='octet-stream', - _encoder=encoders.encode_base64, **_params): - """Create an application/* type MIME document. - - _data is a string containing the raw application data. - - _subtype is the MIME content type subtype, defaulting to - 'octet-stream'. - - _encoder is a function which will perform the actual encoding for - transport of the application data, defaulting to base64 encoding. - - Any additional keyword arguments are passed to the base class - constructor, which turns them into parameters on the Content-Type - header. - """ - if _subtype is None: - raise TypeError('Invalid application MIME subtype') - MIMENonMultipart.__init__(self, 'application', _subtype, **_params) - self.set_payload(_data) - _encoder(self) diff --git a/lib-python/modified-2.7/email/mime/audio.py b/lib-python/modified-2.7/email/mime/audio.py deleted file mode 100644 --- a/lib-python/modified-2.7/email/mime/audio.py +++ /dev/null @@ -1,73 +0,0 @@ -# Copyright (C) 2001-2006 Python Software Foundation -# Author: Anthony Baxter -# Contact: email-sig at python.org - -"""Class representing audio/* type MIME documents.""" - -__all__ = ['MIMEAudio'] - -import sndhdr - -from cStringIO import StringIO -from email import encoders -from email.mime.nonmultipart import MIMENonMultipart - - - -_sndhdr_MIMEmap = {'au' : 'basic', - 'wav' :'x-wav', - 'aiff':'x-aiff', - 'aifc':'x-aiff', - } - -# There are others in sndhdr that don't have MIME types. :( -# Additional ones to be added to sndhdr? midi, mp3, realaudio, wma?? -def _whatsnd(data): - """Try to identify a sound file type. - - sndhdr.what() has a pretty cruddy interface, unfortunately. This is why - we re-do it here. It would be easier to reverse engineer the Unix 'file' - command and use the standard 'magic' file, as shipped with a modern Unix. - """ - hdr = data[:512] - fakefile = StringIO(hdr) - for testfn in sndhdr.tests: - res = testfn(hdr, fakefile) - if res is not None: - return _sndhdr_MIMEmap.get(res[0]) - return None - - - -class MIMEAudio(MIMENonMultipart): - """Class for generating audio/* MIME documents.""" - - def __init__(self, _audiodata, _subtype=None, - _encoder=encoders.encode_base64, **_params): - """Create an audio/* type MIME document. - - _audiodata is a string containing the raw audio data. If this data - can be decoded by the standard Python `sndhdr' module, then the - subtype will be automatically included in the Content-Type header. - Otherwise, you can specify the specific audio subtype via the - _subtype parameter. If _subtype is not given, and no subtype can be - guessed, a TypeError is raised. - - _encoder is a function which will perform the actual encoding for - transport of the image data. It takes one argument, which is this - Image instance. It should use get_payload() and set_payload() to - change the payload to the encoded form. It should also add any - Content-Transfer-Encoding or other headers to the message as - necessary. The default encoding is Base64. - - Any additional keyword arguments are passed to the base class - constructor, which turns them into parameters on the Content-Type - header. - """ - if _subtype is None: - _subtype = _whatsnd(_audiodata) - if _subtype is None: - raise TypeError('Could not find audio MIME subtype') - MIMENonMultipart.__init__(self, 'audio', _subtype, **_params) - self.set_payload(_audiodata) - _encoder(self) diff --git a/lib-python/modified-2.7/email/mime/base.py b/lib-python/modified-2.7/email/mime/base.py deleted file mode 100644 --- a/lib-python/modified-2.7/email/mime/base.py +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright (C) 2001-2006 Python Software Foundation -# Author: Barry Warsaw -# Contact: email-sig at python.org - -"""Base class for MIME specializations.""" - -__all__ = ['MIMEBase'] - -from email import message - - - -class MIMEBase(message.Message): - """Base class for MIME specializations.""" - - def __init__(self, _maintype, _subtype, **_params): - """This constructor adds a Content-Type: and a MIME-Version: header. - - The Content-Type: header is taken from the _maintype and _subtype - arguments. Additional parameters for this header are taken from the - keyword arguments. - """ - message.Message.__init__(self) - ctype = '%s/%s' % (_maintype, _subtype) - self.add_header('Content-Type', ctype, **_params) - self['MIME-Version'] = '1.0' diff --git a/lib-python/modified-2.7/email/mime/image.py b/lib-python/modified-2.7/email/mime/image.py deleted file mode 100644 --- a/lib-python/modified-2.7/email/mime/image.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright (C) 2001-2006 Python Software Foundation -# Author: Barry Warsaw -# Contact: email-sig at python.org - -"""Class representing image/* type MIME documents.""" - -__all__ = ['MIMEImage'] - -import imghdr - -from email import encoders -from email.mime.nonmultipart import MIMENonMultipart - - - -class MIMEImage(MIMENonMultipart): - """Class for generating image/* type MIME documents.""" - - def __init__(self, _imagedata, _subtype=None, - _encoder=encoders.encode_base64, **_params): - """Create an image/* type MIME document. - - _imagedata is a string containing the raw image data. If this data - can be decoded by the standard Python `imghdr' module, then the - subtype will be automatically included in the Content-Type header. - Otherwise, you can specify the specific image subtype via the _subtype - parameter. - - _encoder is a function which will perform the actual encoding for - transport of the image data. It takes one argument, which is this - Image instance. It should use get_payload() and set_payload() to - change the payload to the encoded form. It should also add any - Content-Transfer-Encoding or other headers to the message as - necessary. The default encoding is Base64. - - Any additional keyword arguments are passed to the base class - constructor, which turns them into parameters on the Content-Type - header. - """ - if _subtype is None: - _subtype = imghdr.what(None, _imagedata) - if _subtype is None: - raise TypeError('Could not guess image MIME subtype') - MIMENonMultipart.__init__(self, 'image', _subtype, **_params) - self.set_payload(_imagedata) - _encoder(self) diff --git a/lib-python/modified-2.7/email/mime/message.py b/lib-python/modified-2.7/email/mime/message.py deleted file mode 100644 --- a/lib-python/modified-2.7/email/mime/message.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright (C) 2001-2006 Python Software Foundation -# Author: Barry Warsaw -# Contact: email-sig at python.org - -"""Class representing message/* MIME documents.""" - -__all__ = ['MIMEMessage'] - -from email import message -from email.mime.nonmultipart import MIMENonMultipart - - - -class MIMEMessage(MIMENonMultipart): - """Class representing message/* MIME documents.""" - - def __init__(self, _msg, _subtype='rfc822'): - """Create a message/* type MIME document. - - _msg is a message object and must be an instance of Message, or a - derived class of Message, otherwise a TypeError is raised. - - Optional _subtype defines the subtype of the contained message. The - default is "rfc822" (this is defined by the MIME standard, even though - the term "rfc822" is technically outdated by RFC 2822). - """ - MIMENonMultipart.__init__(self, 'message', _subtype) - if not isinstance(_msg, message.Message): - raise TypeError('Argument is not an instance of Message') - # It's convenient to use this base class method. We need to do it - # this way or we'll get an exception - message.Message.attach(self, _msg) - # And be sure our default type is set correctly - self.set_default_type('message/rfc822') diff --git a/lib-python/modified-2.7/email/mime/multipart.py b/lib-python/modified-2.7/email/mime/multipart.py deleted file mode 100644 --- a/lib-python/modified-2.7/email/mime/multipart.py +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright (C) 2002-2006 Python Software Foundation -# Author: Barry Warsaw -# Contact: email-sig at python.org - -"""Base class for MIME multipart/* type messages.""" - -__all__ = ['MIMEMultipart'] - -from email.mime.base import MIMEBase - - - -class MIMEMultipart(MIMEBase): - """Base class for MIME multipart/* type messages.""" - - def __init__(self, _subtype='mixed', boundary=None, _subparts=None, - **_params): - """Creates a multipart/* type message. - - By default, creates a multipart/mixed message, with proper - Content-Type and MIME-Version headers. - - _subtype is the subtype of the multipart content type, defaulting to - `mixed'. - - boundary is the multipart boundary string. By default it is - calculated as needed. - - _subparts is a sequence of initial subparts for the payload. It - must be an iterable object, such as a list. You can always - attach new subparts to the message by using the attach() method. - - Additional parameters for the Content-Type header are taken from the - keyword arguments (or passed into the _params argument). - """ - MIMEBase.__init__(self, 'multipart', _subtype, **_params) - - # Initialise _payload to an empty list as the Message superclass's - # implementation of is_multipart assumes that _payload is a list for - # multipart messages. - self._payload = [] - - if _subparts: - for p in _subparts: - self.attach(p) - if boundary: - self.set_boundary(boundary) diff --git a/lib-python/modified-2.7/email/mime/nonmultipart.py b/lib-python/modified-2.7/email/mime/nonmultipart.py deleted file mode 100644 --- a/lib-python/modified-2.7/email/mime/nonmultipart.py +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright (C) 2002-2006 Python Software Foundation -# Author: Barry Warsaw -# Contact: email-sig at python.org - -"""Base class for MIME type messages that are not multipart.""" - -__all__ = ['MIMENonMultipart'] - -from email import errors -from email.mime.base import MIMEBase - - - -class MIMENonMultipart(MIMEBase): - """Base class for MIME multipart/* type messages.""" - - def attach(self, payload): - # The public API prohibits attaching multiple subparts to MIMEBase - # derived subtypes since none of them are, by definition, of content - # type multipart/* - raise errors.MultipartConversionError( - 'Cannot attach additional subparts to non-multipart/*') diff --git a/lib-python/modified-2.7/email/mime/text.py b/lib-python/modified-2.7/email/mime/text.py deleted file mode 100644 --- a/lib-python/modified-2.7/email/mime/text.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright (C) 2001-2006 Python Software Foundation -# Author: Barry Warsaw -# Contact: email-sig at python.org - -"""Class representing text/* type MIME documents.""" - -__all__ = ['MIMEText'] - -from email.encoders import encode_7or8bit -from email.mime.nonmultipart import MIMENonMultipart - - - -class MIMEText(MIMENonMultipart): - """Class for generating text/* type MIME documents.""" - - def __init__(self, _text, _subtype='plain', _charset='us-ascii'): - """Create a text/* type MIME document. - - _text is the string for this message object. - - _subtype is the MIME sub content type, defaulting to "plain". - - _charset is the character set parameter added to the Content-Type - header. This defaults to "us-ascii". Note that as a side-effect, the - Content-Transfer-Encoding header will also be set. - """ - MIMENonMultipart.__init__(self, 'text', _subtype, - **{'charset': _charset}) - self.set_payload(_text, _charset) diff --git a/lib-python/modified-2.7/email/parser.py b/lib-python/modified-2.7/email/parser.py deleted file mode 100644 --- a/lib-python/modified-2.7/email/parser.py +++ /dev/null @@ -1,91 +0,0 @@ -# Copyright (C) 2001-2006 Python Software Foundation -# Author: Barry Warsaw, Thomas Wouters, Anthony Baxter -# Contact: email-sig at python.org - -"""A parser of RFC 2822 and MIME email messages.""" - -__all__ = ['Parser', 'HeaderParser'] - -import warnings -from cStringIO import StringIO - -from email.feedparser import FeedParser -from email.message import Message - - - -class Parser: - def __init__(self, *args, **kws): - """Parser of RFC 2822 and MIME email messages. - - Creates an in-memory object tree representing the email message, which - can then be manipulated and turned over to a Generator to return the - textual representation of the message. - - The string must be formatted as a block of RFC 2822 headers and header - continuation lines, optionally preceeded by a `Unix-from' header. The - header block is terminated either by the end of the string or by a - blank line. - - _class is the class to instantiate for new message objects when they - must be created. This class must have a constructor that can take - zero arguments. Default is Message.Message. - """ - if len(args) >= 1: - if '_class' in kws: - raise TypeError("Multiple values for keyword arg '_class'") - kws['_class'] = args[0] - if len(args) == 2: - if 'strict' in kws: - raise TypeError("Multiple values for keyword arg 'strict'") - kws['strict'] = args[1] - if len(args) > 2: - raise TypeError('Too many arguments') - if '_class' in kws: - self._class = kws['_class'] - del kws['_class'] - else: - self._class = Message - if 'strict' in kws: - warnings.warn("'strict' argument is deprecated (and ignored)", - DeprecationWarning, 2) - del kws['strict'] - if kws: - raise TypeError('Unexpected keyword arguments') - - def parse(self, fp, headersonly=False): - """Create a message structure from the data in a file. - - Reads all the data from the file and returns the root of the message - structure. Optional headersonly is a flag specifying whether to stop - parsing after reading the headers or not. The default is False, - meaning it parses the entire contents of the file. - """ - feedparser = FeedParser(self._class) - if headersonly: - feedparser._set_headersonly() - while True: - data = fp.read(8192) - if not data: - break - feedparser.feed(data) - return feedparser.close() - - def parsestr(self, text, headersonly=False): - """Create a message structure from a string. - - Returns the root of the message structure. Optional headersonly is a - flag specifying whether to stop parsing after reading the headers or - not. The default is False, meaning it parses the entire contents of - the file. - """ - return self.parse(StringIO(text), headersonly=headersonly) - - - -class HeaderParser(Parser): - def parse(self, fp, headersonly=True): - return Parser.parse(self, fp, True) - - def parsestr(self, text, headersonly=True): - return Parser.parsestr(self, text, True) diff --git a/lib-python/modified-2.7/email/quoprimime.py b/lib-python/modified-2.7/email/quoprimime.py deleted file mode 100644 --- a/lib-python/modified-2.7/email/quoprimime.py +++ /dev/null @@ -1,336 +0,0 @@ -# Copyright (C) 2001-2006 Python Software Foundation -# Author: Ben Gertzfield -# Contact: email-sig at python.org - -"""Quoted-printable content transfer encoding per RFCs 2045-2047. - -This module handles the content transfer encoding method defined in RFC 2045 -to encode US ASCII-like 8-bit data called `quoted-printable'. It is used to -safely encode text that is in a character set similar to the 7-bit US ASCII -character set, but that includes some 8-bit characters that are normally not -allowed in email bodies or headers. - -Quoted-printable is very space-inefficient for encoding binary files; use the -email.base64mime module for that instead. - -This module provides an interface to encode and decode both headers and bodies -with quoted-printable encoding. - -RFC 2045 defines a method for including character set information in an -`encoded-word' in a header. This method is commonly used for 8-bit real names -in To:/From:/Cc: etc. fields, as well as Subject: lines. - -This module does not do the line wrapping or end-of-line character -conversion necessary for proper internationalized headers; it only -does dumb encoding and decoding. To deal with the various line -wrapping issues, use the email.header module. -""" - -__all__ = [ - 'body_decode', - 'body_encode', - 'body_quopri_check', - 'body_quopri_len', - 'decode', - 'decodestring', - 'encode', - 'encodestring', - 'header_decode', - 'header_encode', - 'header_quopri_check', - 'header_quopri_len', - 'quote', - 'unquote', - ] - -import re - -from string import hexdigits -from email.utils import fix_eols - -CRLF = '\r\n' -NL = '\n' - -# See also Charset.py -MISC_LEN = 7 - -hqre = re.compile(r'[^-a-zA-Z0-9!*+/ ]') -bqre = re.compile(r'[^ !-<>-~\t]') - - - -# Helpers -def header_quopri_check(c): - """Return True if the character should be escaped with header quopri.""" - return bool(hqre.match(c)) - - -def body_quopri_check(c): - """Return True if the character should be escaped with body quopri.""" - return bool(bqre.match(c)) - - -def header_quopri_len(s): - """Return the length of str when it is encoded with header quopri.""" - count = 0 - for c in s: - if hqre.match(c): - count += 3 - else: - count += 1 - return count - - -def body_quopri_len(str): - """Return the length of str when it is encoded with body quopri.""" - count = 0 - for c in str: - if bqre.match(c): - count += 3 - else: - count += 1 - return count - - -def _max_append(L, s, maxlen, extra=''): - if not L: - L.append(s.lstrip()) - elif len(L[-1]) + len(s) <= maxlen: - L[-1] += extra + s - else: - L.append(s.lstrip()) - - -def unquote(s): - """Turn a string in the form =AB to the ASCII character with value 0xab""" - return chr(int(s[1:3], 16)) - - -def quote(c): - return "=%02X" % ord(c) - - - -def header_encode(header, charset="iso-8859-1", keep_eols=False, - maxlinelen=76, eol=NL): - """Encode a single header line with quoted-printable (like) encoding. - - Defined in RFC 2045, this `Q' encoding is similar to quoted-printable, but - used specifically for email header fields to allow charsets with mostly 7 - bit characters (and some 8 bit) to remain more or less readable in non-RFC - 2045 aware mail clients. - - charset names the character set to use to encode the header. It defaults - to iso-8859-1. - - The resulting string will be in the form: - - "=?charset?q?I_f=E2rt_in_your_g=E8n=E8ral_dire=E7tion?\\n - =?charset?q?Silly_=C8nglish_Kn=EEghts?=" - - with each line wrapped safely at, at most, maxlinelen characters (defaults - to 76 characters). If maxlinelen is None, the entire string is encoded in - one chunk with no splitting. - - End-of-line characters (\\r, \\n, \\r\\n) will be automatically converted - to the canonical email line separator \\r\\n unless the keep_eols - parameter is True (the default is False). - - Each line of the header will be terminated in the value of eol, which - defaults to "\\n". Set this to "\\r\\n" if you are using the result of - this function directly in email. - """ - # Return empty headers unchanged - if not header: - return header - - if not keep_eols: - header = fix_eols(header) - - # Quopri encode each line, in encoded chunks no greater than maxlinelen in - # length, after the RFC chrome is added in. - quoted = [] - if maxlinelen is None: - # An obnoxiously large number that's good enough - max_encoded = 100000 - else: - max_encoded = maxlinelen - len(charset) - MISC_LEN - 1 - - for c in header: - # Space may be represented as _ instead of =20 for readability - if c == ' ': - _max_append(quoted, '_', max_encoded) - # These characters can be included verbatim - elif not hqre.match(c): - _max_append(quoted, c, max_encoded) - # Otherwise, replace with hex value like =E2 - else: - _max_append(quoted, "=%02X" % ord(c), max_encoded) - - # Now add the RFC chrome to each encoded chunk and glue the chunks - # together. BAW: should we be able to specify the leading whitespace in - # the joiner? - joiner = eol + ' ' - return joiner.join(['=?%s?q?%s?=' % (charset, line) for line in quoted]) - - - -def encode(body, binary=False, maxlinelen=76, eol=NL): - """Encode with quoted-printable, wrapping at maxlinelen characters. - - If binary is False (the default), end-of-line characters will be converted - to the canonical email end-of-line sequence \\r\\n. Otherwise they will - be left verbatim. - - Each line of encoded text will end with eol, which defaults to "\\n". Set - this to "\\r\\n" if you will be using the result of this function directly - in an email. - - Each line will be wrapped at, at most, maxlinelen characters (defaults to - 76 characters). Long lines will have the `soft linefeed' quoted-printable - character "=" appended to them, so the decoded text will be identical to - the original text. - """ - if not body: - return body - - if not binary: - body = fix_eols(body) - - # BAW: We're accumulating the body text by string concatenation. That - # can't be very efficient, but I don't have time now to rewrite it. It - # just feels like this algorithm could be more efficient. - encoded_body = '' - lineno = -1 - # Preserve line endings here so we can check later to see an eol needs to - # be added to the output later. - lines = body.splitlines(1) - for line in lines: - # But strip off line-endings for processing this line. - if line.endswith(CRLF): - line = line[:-2] - elif line[-1] in CRLF: - line = line[:-1] - - lineno += 1 - encoded_line = '' - prev = None - linelen = len(line) - # Now we need to examine every character to see if it needs to be - # quopri encoded. BAW: again, string concatenation is inefficient. - for j in range(linelen): - c = line[j] - prev = c - if bqre.match(c): - c = quote(c) - elif j+1 == linelen: - # Check for whitespace at end of line; special case - if c not in ' \t': - encoded_line += c - prev = c - continue - # Check to see to see if the line has reached its maximum length - if len(encoded_line) + len(c) >= maxlinelen: - encoded_body += encoded_line + '=' + eol - encoded_line = '' - encoded_line += c - # Now at end of line.. - if prev and prev in ' \t': - # Special case for whitespace at end of file - if lineno + 1 == len(lines): - prev = quote(prev) - if len(encoded_line) + len(prev) > maxlinelen: - encoded_body += encoded_line + '=' + eol + prev - else: - encoded_body += encoded_line + prev - # Just normal whitespace at end of line - else: - encoded_body += encoded_line + prev + '=' + eol - encoded_line = '' - # Now look at the line we just finished and it has a line ending, we - # need to add eol to the end of the line. - if lines[lineno].endswith(CRLF) or lines[lineno][-1] in CRLF: - encoded_body += encoded_line + eol - else: - encoded_body += encoded_line - encoded_line = '' - return encoded_body - - -# For convenience and backwards compatibility w/ standard base64 module -body_encode = encode -encodestring = encode - - - -# BAW: I'm not sure if the intent was for the signature of this function to be -# the same as base64MIME.decode() or not... -def decode(encoded, eol=NL): - """Decode a quoted-printable string. - - Lines are separated with eol, which defaults to \\n. - """ - if not encoded: - return encoded - # BAW: see comment in encode() above. Again, we're building up the - # decoded string with string concatenation, which could be done much more - # efficiently. - decoded = '' - - for line in encoded.splitlines(): - line = line.rstrip() - if not line: - decoded += eol - continue - - i = 0 - n = len(line) - while i < n: - c = line[i] - if c != '=': - decoded += c - i += 1 - # Otherwise, c == "=". Are we at the end of the line? If so, add - # a soft line break. - elif i+1 == n: - i += 1 - continue - # Decode if in form =AB - elif i+2 < n and line[i+1] in hexdigits and line[i+2] in hexdigits: - decoded += unquote(line[i:i+3]) - i += 3 - # Otherwise, not in form =AB, pass literally - else: - decoded += c - i += 1 - - if i == n: - decoded += eol - # Special case if original string did not end with eol - if not encoded.endswith(eol) and decoded.endswith(eol): - decoded = decoded[:-1] - return decoded - - -# For convenience and backwards compatibility w/ standard base64 module -body_decode = decode -decodestring = decode - - - -def _unquote_match(match): - """Turn a match in the form =AB to the ASCII character with value 0xab""" - s = match.group(0) - return unquote(s) - - -# Header decoding is done a bit differently -def header_decode(s): - """Decode a string encoded with RFC 2045 MIME header `Q' encoding. - - This function does not parse a full MIME header value encoded with - quoted-printable (like =?iso-8895-1?q?Hello_World?=) -- please use - the high level email.header class for that functionality. - """ - s = s.replace('_', ' ') - return re.sub(r'=[a-fA-F0-9]{2}', _unquote_match, s) diff --git a/lib-python/modified-2.7/email/test/__init__.py b/lib-python/modified-2.7/email/test/__init__.py deleted file mode 100644 diff --git a/lib-python/modified-2.7/email/test/data/PyBanner048.gif b/lib-python/modified-2.7/email/test/data/PyBanner048.gif deleted file mode 100644 Binary file lib-python/modified-2.7/email/test/data/PyBanner048.gif has changed diff --git a/lib-python/modified-2.7/email/test/data/audiotest.au b/lib-python/modified-2.7/email/test/data/audiotest.au deleted file mode 100644 Binary file lib-python/modified-2.7/email/test/data/audiotest.au has changed diff --git a/lib-python/modified-2.7/email/test/data/msg_01.txt b/lib-python/modified-2.7/email/test/data/msg_01.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_01.txt +++ /dev/null @@ -1,19 +0,0 @@ -Return-Path: <bbb at zzz.org> -Delivered-To: bbb at zzz.org -Received: by mail.zzz.org (Postfix, from userid 889) - id 27CEAD38CC; Fri, 4 May 2001 14:05:44 -0400 (EDT) -MIME-Version: 1.0 -Content-Type: text/plain; charset=us-ascii -Content-Transfer-Encoding: 7bit -Message-ID: <15090.61304.110929.45684 at aaa.zzz.org> -From: bbb at ddd.com (John X. Doe) -To: bbb at zzz.org -Subject: This is a test message -Date: Fri, 4 May 2001 14:05:44 -0400 - - -Hi, - -Do you like this message? - --Me diff --git a/lib-python/modified-2.7/email/test/data/msg_02.txt b/lib-python/modified-2.7/email/test/data/msg_02.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_02.txt +++ /dev/null @@ -1,135 +0,0 @@ -MIME-version: 1.0 -From: ppp-request at zzz.org -Sender: ppp-admin at zzz.org -To: ppp at zzz.org -Subject: Ppp digest, Vol 1 #2 - 5 msgs -Date: Fri, 20 Apr 2001 20:18:00 -0400 (EDT) -X-Mailer: Mailman v2.0.4 -X-Mailman-Version: 2.0.4 -Content-Type: multipart/mixed; boundary="192.168.1.2.889.32614.987812255.500.21814" - ---192.168.1.2.889.32614.987812255.500.21814 -Content-type: text/plain; charset=us-ascii -Content-description: Masthead (Ppp digest, Vol 1 #2) - -Send Ppp mailing list submissions to - ppp at zzz.org - -To subscribe or unsubscribe via the World Wide Web, visit - http://www.zzz.org/mailman/listinfo/ppp -or, via email, send a message with subject or body 'help' to - ppp-request at zzz.org - -You can reach the person managing the list at - ppp-admin at zzz.org - -When replying, please edit your Subject line so it is more specific -than "Re: Contents of Ppp digest..." - - ---192.168.1.2.889.32614.987812255.500.21814 -Content-type: text/plain; charset=us-ascii -Content-description: Today's Topics (5 msgs) - -Today's Topics: - - 1. testing #1 (Barry A. Warsaw) - 2. testing #2 (Barry A. Warsaw) - 3. testing #3 (Barry A. Warsaw) - 4. testing #4 (Barry A. Warsaw) - 5. testing #5 (Barry A. Warsaw) - ---192.168.1.2.889.32614.987812255.500.21814 -Content-Type: multipart/digest; boundary="__--__--" - ---__--__-- - -Message: 1 -Content-Type: text/plain; charset=us-ascii -Content-Transfer-Encoding: 7bit -Date: Fri, 20 Apr 2001 20:16:13 -0400 -To: ppp at zzz.org -From: barry at digicool.com (Barry A. Warsaw) -Subject: [Ppp] testing #1 -Precedence: bulk - - -hello - - ---__--__-- - -Message: 2 -Date: Fri, 20 Apr 2001 20:16:21 -0400 -Content-Type: text/plain; charset=us-ascii -Content-Transfer-Encoding: 7bit -To: ppp at zzz.org -From: barry at digicool.com (Barry A. Warsaw) -Precedence: bulk - - -hello - - ---__--__-- - -Message: 3 -Date: Fri, 20 Apr 2001 20:16:25 -0400 -Content-Type: text/plain; charset=us-ascii -Content-Transfer-Encoding: 7bit -To: ppp at zzz.org -From: barry at digicool.com (Barry A. Warsaw) -Subject: [Ppp] testing #3 -Precedence: bulk - - -hello - - ---__--__-- - -Message: 4 -Date: Fri, 20 Apr 2001 20:16:28 -0400 -Content-Type: text/plain; charset=us-ascii -Content-Transfer-Encoding: 7bit -To: ppp at zzz.org -From: barry at digicool.com (Barry A. Warsaw) -Subject: [Ppp] testing #4 -Precedence: bulk - - -hello - - ---__--__-- - -Message: 5 -Date: Fri, 20 Apr 2001 20:16:32 -0400 -Content-Type: text/plain; charset=us-ascii -Content-Transfer-Encoding: 7bit -To: ppp at zzz.org -From: barry at digicool.com (Barry A. Warsaw) -Subject: [Ppp] testing #5 -Precedence: bulk - - -hello - - - - ---__--__---- ---192.168.1.2.889.32614.987812255.500.21814 -Content-type: text/plain; charset=us-ascii -Content-description: Digest Footer - -_______________________________________________ -Ppp mailing list -Ppp at zzz.org -http://www.zzz.org/mailman/listinfo/ppp - - ---192.168.1.2.889.32614.987812255.500.21814-- - -End of Ppp Digest - diff --git a/lib-python/modified-2.7/email/test/data/msg_03.txt b/lib-python/modified-2.7/email/test/data/msg_03.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_03.txt +++ /dev/null @@ -1,16 +0,0 @@ -Return-Path: <bbb at zzz.org> -Delivered-To: bbb at zzz.org -Received: by mail.zzz.org (Postfix, from userid 889) - id 27CEAD38CC; Fri, 4 May 2001 14:05:44 -0400 (EDT) -Message-ID: <15090.61304.110929.45684 at aaa.zzz.org> -From: bbb at ddd.com (John X. Doe) -To: bbb at zzz.org -Subject: This is a test message -Date: Fri, 4 May 2001 14:05:44 -0400 - - -Hi, - -Do you like this message? - --Me diff --git a/lib-python/modified-2.7/email/test/data/msg_04.txt b/lib-python/modified-2.7/email/test/data/msg_04.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_04.txt +++ /dev/null @@ -1,37 +0,0 @@ -Return-Path: <barry at python.org> -Delivered-To: barry at python.org -Received: by mail.python.org (Postfix, from userid 889) - id C2BF0D37C6; Tue, 11 Sep 2001 00:05:05 -0400 (EDT) -MIME-Version: 1.0 -Content-Type: multipart/mixed; boundary="h90VIIIKmx" -Content-Transfer-Encoding: 7bit -Message-ID: <15261.36209.358846.118674 at anthem.python.org> -From: barry at python.org (Barry A. Warsaw) -To: barry at python.org -Subject: a simple multipart -Date: Tue, 11 Sep 2001 00:05:05 -0400 -X-Mailer: VM 6.95 under 21.4 (patch 4) "Artificial Intelligence" XEmacs Lucid -X-Attribution: BAW -X-Oblique-Strategy: Make a door into a window - - ---h90VIIIKmx -Content-Type: text/plain -Content-Disposition: inline; - filename="msg.txt" -Content-Transfer-Encoding: 7bit - -a simple kind of mirror -to reflect upon our own - ---h90VIIIKmx -Content-Type: text/plain -Content-Disposition: inline; - filename="msg.txt" -Content-Transfer-Encoding: 7bit - -a simple kind of mirror -to reflect upon our own - ---h90VIIIKmx-- - diff --git a/lib-python/modified-2.7/email/test/data/msg_05.txt b/lib-python/modified-2.7/email/test/data/msg_05.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_05.txt +++ /dev/null @@ -1,28 +0,0 @@ -From: foo -Subject: bar -To: baz -MIME-Version: 1.0 -Content-Type: multipart/report; report-type=delivery-status; - boundary="D1690A7AC1.996856090/mail.example.com" -Message-Id: <20010803162810.0CA8AA7ACC at mail.example.com> - -This is a MIME-encapsulated message. - ---D1690A7AC1.996856090/mail.example.com -Content-Type: text/plain - -Yadda yadda yadda - ---D1690A7AC1.996856090/mail.example.com - -Yadda yadda yadda - ---D1690A7AC1.996856090/mail.example.com -Content-Type: message/rfc822 - -From: nobody at python.org - -Yadda yadda yadda - ---D1690A7AC1.996856090/mail.example.com-- - diff --git a/lib-python/modified-2.7/email/test/data/msg_06.txt b/lib-python/modified-2.7/email/test/data/msg_06.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_06.txt +++ /dev/null @@ -1,33 +0,0 @@ -Return-Path: <barry at python.org> -Delivered-To: barry at python.org -MIME-Version: 1.0 -Content-Type: message/rfc822 -Content-Description: forwarded message -Content-Transfer-Encoding: 7bit -Message-ID: <15265.9482.641338.555352 at python.org> -From: barry at zope.com (Barry A. Warsaw) -Sender: barry at python.org -To: barry at python.org -Subject: forwarded message from Barry A. Warsaw -Date: Thu, 13 Sep 2001 17:28:42 -0400 -X-Mailer: VM 6.95 under 21.4 (patch 4) "Artificial Intelligence" XEmacs Lucid -X-Attribution: BAW -X-Oblique-Strategy: Be dirty -X-Url: http://barry.wooz.org - -MIME-Version: 1.0 -Content-Type: text/plain; charset=us-ascii -Return-Path: <barry at python.org> -Delivered-To: barry at python.org -Message-ID: <15265.9468.713530.98441 at python.org> -From: barry at zope.com (Barry A. Warsaw) -Sender: barry at python.org -To: barry at python.org -Subject: testing -Date: Thu, 13 Sep 2001 17:28:28 -0400 -X-Mailer: VM 6.95 under 21.4 (patch 4) "Artificial Intelligence" XEmacs Lucid -X-Attribution: BAW -X-Oblique-Strategy: Spectrum analysis -X-Url: http://barry.wooz.org - - diff --git a/lib-python/modified-2.7/email/test/data/msg_07.txt b/lib-python/modified-2.7/email/test/data/msg_07.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_07.txt +++ /dev/null @@ -1,83 +0,0 @@ -MIME-Version: 1.0 -From: Barry <barry at digicool.com> -To: Dingus Lovers <cravindogs at cravindogs.com> -Subject: Here is your dingus fish -Date: Fri, 20 Apr 2001 19:35:02 -0400 -Content-Type: multipart/mixed; boundary="BOUNDARY" - ---BOUNDARY -Content-Type: text/plain; charset="us-ascii" - -Hi there, - -This is the dingus fish. - ---BOUNDARY -Content-Type: image/gif; name="dingusfish.gif" -Content-Transfer-Encoding: base64 -content-disposition: attachment; filename="dingusfish.gif" - -R0lGODdhAAEAAfAAAP///wAAACwAAAAAAAEAAQAC/oSPqcvtD6OctNqLs968+w+G4kiW5omm6sq2 -7gvH8kzX9o3n+s73/g8MCofEovGITGICTKbyCV0FDNOo9SqpQqpOrJfXzTQj2vD3TGtqL+NtGQ2f -qTXmxzuOd7WXdcc9DyjU53ewFni4s0fGhdiYaEhGBelICTNoV1j5NUnFcrmUqemjNifJVWpaOqaI -oFq3SspZsSraE7sHq3jr1MZqWvi662vxV4tD+pvKW6aLDOCLyur8PDwbanyDeq0N3DctbQYeLDvR -RY6t95m6UB0d3mwIrV7e2VGNvjjffukeJp4w7F65KecGFsTHQGAygOrgrWs1jt28Rc88KESYcGLA -/obvTkH6p+CinWJiJmIMqXGQwH/y4qk0SYjgQTczT3ajKZGfuI0uJ4kkVI/DT5s3/ejkxI0aT4Y+ -YTYgWbImUaXk9nlLmnSh1qJiJFl0OpUqRK4oOy7NyRQtHWofhoYVxkwWXKUSn0YsS+fUV6lhqfYb -6ayd3Z5qQdG1B7bvQzaJjwUV2lixMUZ7JVsOlfjWVr/3NB/uFvnySBN6Dcb6rGwaRM3wsormw5cC -M9NxWy/bWdufudCvy8bOAjXjVVwta/uO21sE5RHBCzNFXtgq9ORtH4eYjVP4Yryo026nvkFmCeyA -B29efV6ravCMK5JwWd5897Qrx7ll38o6iHDZ/rXPR//feevhF4l7wjUGX3xq1eeRfM4RSJGBIV1D -z1gKPkfWag3mVBVvva1RlX5bAJTPR/2YqNtw/FkIYYEi/pIZiAdpcxpoHtmnYYoZtvhUftzdx5ZX -JSKDW405zkGcZzzGZ6KEv4FI224oDmijlEf+xp6MJK5ojY/ASeVUR+wsKRuJ+XFZ5o7ZeEime8t1 -ouUsU6YjF5ZtUihhkGfCdFQLWQFJ3UXxmElfhQnR+eCdcDbkFZp6vTRmj56ApCihn5QGpaToNZmR -n3NVSpZcQpZ2KEONusaiCsKAug0wkQbJSFO+PTSjneGxOuFjPlUk3ovWvdIerjUg9ZGIOtGq/qeX -eCYrrCX+1UPsgTKGGRSbzd5q156d/gpfbJxe66eD5iQKrXj7RGgruGxs62qebBHUKS32CKluCiqZ -qh+pmehmEb71noAUoe5e9Zm17S7773V10pjrtG4CmuurCV/n6zLK5turWNhqOvFXbjhZrMD0YhKe -wR0zOyuvsh6MWrGoIuzvyWu5y1WIFAqmJselypxXh6dKLNOKEB98L88bS2rkNqqlKzCNJp9c0G0j -Gzh0iRrCbHSXmPR643QS+4rWhgFmnSbSuXCjS0xAOWkU2UdLqyuUNfHSFdUouy3bm5i5GnDM3tG8 -doJ4r5tqu3pPbRSVfvs8uJzeNXhp3n4j/tZ42SwH7eaWUUOjc3qFV9453UHTXZfcLH+OeNs5g36x -lBnHvTm7EbMbLeuaLncao8vWCXimfo1o+843Ak6y4ChNeGntvAYvfLK4ezmoyNIbNCLTCXO9ZV3A -E8/s88RczPzDwI4Ob7XZyl7+9Miban29h+tJZPrE21wgvBphDfrrfPdCTPKJD/y98L1rZwHcV6Jq -Zab0metpuNIX/qAFPoz171WUaUb4HAhBSzHuHfjzHb3kha/2Cctis/ORArVHNYfFyYRH2pYIRzic -isVOfPWD1b6mRTqpCRBozzof6UZVvFXRxWIr3GGrEviGYgyPMfahheiSaLs/9QeFu7oZ/ndSY8DD -ya9x+uPed+7mxN2IzIISBOMLFYWVqC3Pew1T2nFuuCiwZS5/v6II10i4t1OJcUH2U9zxKodHsGGv -Oa+zkvNUYUOa/TCCRutF9MzDwdlUMJADTCGSbDQ5OV4PTamDoPEi6Ecc/RF5RWwkcdSXvSOaDWSn -I9LlvubFTQpuc6JKXLcKeb+xdbKRBnwREemXyjg6ME65aJiOuBgrktzykfPLJBKR9ClMavJ62/Ff -BlNIyod9yX9wcSXexnXFpvkrbXk64xsx5Db7wXKP5fSgsvwIMM/9631VLBfkmtbHRXpqmtei52hG -pUwSlo+BASQoeILDOBgREECxBBh5/iYmNsQ9dIv5+OI++QkqdsJPc3uykz5fkM+OraeekcQF7X4n -B5S67za5U967PmooGQhUXfF7afXyCD7ONdRe17QogYjVx38uLwtrS6nhTnm15LQUnu9E2uK6CNI/ -1HOABj0ESwOjut4FEpFQpdNAm4K2LHnDWHNcmKB2ioKBogysVZtMO2nSxUdZ8Yk2kJc7URioLVI0 -YgmtIwZj4LoeKemgnOnbUdGnzZ4Oa6scqiolBGqS6RgWNLu0RMhcaE6rhhU4hiuqFXPAG8fGwTPW -FKeLMtdVmXLSs5YJGF/YeVm7rREMlY3UYE+yCxbaMXX8y15m5zVHq6GOKDMynzII/jdUHdyVqIy0 -ifX2+r/EgtZcvRzSb72gU9ui87M2VecjKildW/aFqaYhKoryUjfB/g4qtyVuc60xFDGmCxwjW+qu -zjuwl2GkOWn66+3QiiEctvd04OVvcCVzjgT7lrkvjVGKKHmmlDUKowSeikb5kK/mJReuWOxONx+s -ULsl+Lqb0CVn0SrVyJ6wt4t6yTeSCafhPhAf0OXn6L60UMxiLolFAtmN35S2Ob1lZpQ1r/n0Qb5D -oQ1zJiRVDgF8N3Q8TYfbi3DyWCy3lT1nxyBs6FT3S2GOzWRlxwKvlRP0RPJA9SjxEy0UoEnkA+M4 -cnzLMJrBGWLFEaaUb5lvpqbq/loOaU5+DFuHPxo82/OZuM8FXG3oVNZhtWpMpb/0Xu5m/LfLhHZQ -7yuVI0MqZ7NE43imC8jH3IwGZlbPm0xkJYs7+2U48hXTsFSMqgGDvai0kLxyynKNT/waj+q1c1tz -GjOpPBgdCSq3UKZxCSsqFIY+O6JbAWGWcV1pwqLyj5sGqCF1xb1F3varUWqrJv6cN3PrUXzijtfZ -FshpBL3Xwr4GIPvU2N8EjrJgS1zl21rbXQMXeXc5jjFyrhpCzijSv/RQtyPSzHCFMhlME95fHglt -pRsX+dfSQjUeHAlpWzJ5iOo79Ldnaxai6bXTcGO3fp07ri7HLEmXXPlYi8bv/qVxvNcdra6m7Rlb -6JBTb5fd66VhFRjGArh2n7R1rDW4P5NOT9K0I183T2scYkeZ3q/VFyLb09U9ajzXBS8Kgkhc4mBS -kYY9cy3Vy9lUnuNJH8HGIclUilwnBtjUOH0gteGOZ4c/XNrhXLSYDyxfnD8z1pDy7rYRvDolhnbe -UMzxCZUs40s6s7UIvBnLgc0+vKuOkIXeOrDymlp+Zxra4MZLBbVrqD/jTJ597pDmnw5c4+DbyB88 -9Cg9DodYcSuMZT/114pptqc/EuTjRPvH/z5slzI3tluOEBBLqOXLOX+0I5929tO97wkvl/atCz+y -xJrdwteW2FNW/NSmBP+f/maYtVs/bYyBC7Ox3jsYZHL05CIrBa/nS+b3bHfiYm4Ueil1YZZSgAUI -fFZ1dxUmeA2oQRQ3RuGXNGLFV9/XbGFGPV6kfzk1TBBCd+izc7q1H+OHMJwmaBX2IQNYVAKHYepV -SSGCe6CnbYHHETKGNe43EDvFgZr0gB/nVHPHZ80VV1ojOiI3XDvYIkl4ayo4bxQIgrFXWTvBI0nH -VElWMuw2aLUWCRHHf8ymVCHjFlJnOSojfevCYyyyZDH0IcvHhrsnQ5O1OsWzONuVVKIxSxiFZ/tR -fKDAf6xFTnw4O9Qig2VCfW2hJQrmMOuHW0W3dLQmCMO2ccdUd/xyfflH/olTiHZVdGwb8nIwRzSE -J15jFlOJuBZBZ4CiyHyd2IFylFlB+HgHhYabhWOGwYO1ZH/Og1dtQlFMk352CGRSIFTapnWQEUtN -l4zv8S0aaCFDyGCBqDUxZYpxGHX01y/JuH1xhn7TOCnNCI4eKDs5WGX4R425F4vF1o3BJ4vO0otq -I3rimI7jJY1jISqnBxknCIvruF83mF5wN4X7qGLIhR8A2Vg0yFERSIXn9Vv3GHy3Vj/WIkKddlYi -yIMv2I/VMjTLpW7pt05SWIZR0RPyxpB4SIUM9lBPGBl0GC7oSEEwRYLe4pJpZY2P0zbI1n+Oc44w -qY3PUnmF0ixjVpDD/mJ9wpOBGTVgXlaCaZiPcIWK5NiKBIiPdGaQ0TWGvAiG7nMchdZb7Vgf8zNi -MuMyzRdy/lePe9iC4TRx7WhhOQI/QiSVNAmAa2lT/piFbuh7ofJoYSZzrSZ1bvmWw3eN2nKUPVky -uPN5/VRfohRd0VYZoqhKIlU6TXYhJxmPUIloAwc1bPmHEpaZYZORHNlXUJM07hATwHR8MJYqkwWR -WaIezFhxSFlc8/Fq82hEnpeRozg3ULhhr9lAGtVEkCg5ZNRuuVleBPaZadhG0ZgkyPmDOTOKzViM -YgOcpukKqQcbjAWS0IleQ2ROjdh6A+md1qWdBRSX7iSYgFRTtRmBpJioieXJiHfJiMGIR9fJOn8I -MSfXYhspn4ooSa2mSAj4n+8Bmg03fBJZoPOJgsVZRxu1oOMRPXYYjdqjihFaEoZpXBREanuJoRI6 -cibFinq4ngUKh/wQd/H5ofYCZ0HJXR62opZFaAT0iFIZo4DIiUojkjeqKiuoZirKo5Y1a7AWckGa -BkuYoD5lpDK6eUs6CkDqpETwl1EqpfhJpVeKpVl6EgUAADs= - ---BOUNDARY-- diff --git a/lib-python/modified-2.7/email/test/data/msg_08.txt b/lib-python/modified-2.7/email/test/data/msg_08.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_08.txt +++ /dev/null @@ -1,24 +0,0 @@ -MIME-Version: 1.0 -From: Barry Warsaw <barry at zope.com> -To: Dingus Lovers <cravindogs at cravindogs.com> -Subject: Lyrics -Date: Fri, 20 Apr 2001 19:35:02 -0400 -Content-Type: multipart/mixed; boundary="BOUNDARY" - ---BOUNDARY -Content-Type: text/plain; charset="us-ascii" - - ---BOUNDARY -Content-Type: text/html; charset="iso-8859-1" - - ---BOUNDARY -Content-Type: text/plain; charset="iso-8859-2" - - ---BOUNDARY -Content-Type: text/plain; charset="koi8-r" - - ---BOUNDARY-- diff --git a/lib-python/modified-2.7/email/test/data/msg_09.txt b/lib-python/modified-2.7/email/test/data/msg_09.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_09.txt +++ /dev/null @@ -1,24 +0,0 @@ -MIME-Version: 1.0 -From: Barry Warsaw <barry at zope.com> -To: Dingus Lovers <cravindogs at cravindogs.com> -Subject: Lyrics -Date: Fri, 20 Apr 2001 19:35:02 -0400 -Content-Type: multipart/mixed; boundary="BOUNDARY" - ---BOUNDARY -Content-Type: text/plain; charset="us-ascii" - - ---BOUNDARY -Content-Type: text/html; charset="iso-8859-1" - - ---BOUNDARY -Content-Type: text/plain - - ---BOUNDARY -Content-Type: text/plain; charset="koi8-r" - - ---BOUNDARY-- diff --git a/lib-python/modified-2.7/email/test/data/msg_10.txt b/lib-python/modified-2.7/email/test/data/msg_10.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_10.txt +++ /dev/null @@ -1,39 +0,0 @@ -MIME-Version: 1.0 -From: Barry Warsaw <barry at zope.com> -To: Dingus Lovers <cravindogs at cravindogs.com> -Subject: Lyrics -Date: Fri, 20 Apr 2001 19:35:02 -0400 -Content-Type: multipart/mixed; boundary="BOUNDARY" - ---BOUNDARY -Content-Type: text/plain; charset="us-ascii" -Content-Transfer-Encoding: 7bit - -This is a 7bit encoded message. - ---BOUNDARY -Content-Type: text/html; charset="iso-8859-1" -Content-Transfer-Encoding: Quoted-Printable - -=A1This is a Quoted Printable encoded message! - ---BOUNDARY -Content-Type: text/plain; charset="iso-8859-1" -Content-Transfer-Encoding: Base64 - -VGhpcyBpcyBhIEJhc2U2NCBlbmNvZGVkIG1lc3NhZ2Uu - - ---BOUNDARY -Content-Type: text/plain; charset="iso-8859-1" -Content-Transfer-Encoding: Base64 - -VGhpcyBpcyBhIEJhc2U2NCBlbmNvZGVkIG1lc3NhZ2UuCg== - - ---BOUNDARY -Content-Type: text/plain; charset="iso-8859-1" - -This has no Content-Transfer-Encoding: header. - ---BOUNDARY-- diff --git a/lib-python/modified-2.7/email/test/data/msg_11.txt b/lib-python/modified-2.7/email/test/data/msg_11.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_11.txt +++ /dev/null @@ -1,7 +0,0 @@ -Content-Type: message/rfc822 -MIME-Version: 1.0 -Subject: The enclosing message - -Subject: An enclosed message - -Here is the body of the message. diff --git a/lib-python/modified-2.7/email/test/data/msg_12.txt b/lib-python/modified-2.7/email/test/data/msg_12.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_12.txt +++ /dev/null @@ -1,36 +0,0 @@ -MIME-Version: 1.0 -From: Barry Warsaw <barry at zope.com> -To: Dingus Lovers <cravindogs at cravindogs.com> -Subject: Lyrics -Date: Fri, 20 Apr 2001 19:35:02 -0400 -Content-Type: multipart/mixed; boundary="BOUNDARY" - ---BOUNDARY -Content-Type: text/plain; charset="us-ascii" - - ---BOUNDARY -Content-Type: text/html; charset="iso-8859-1" - - ---BOUNDARY -Content-Type: multipart/mixed; boundary="ANOTHER" - ---ANOTHER -Content-Type: text/plain; charset="iso-8859-2" - - ---ANOTHER -Content-Type: text/plain; charset="iso-8859-3" - ---ANOTHER-- - ---BOUNDARY -Content-Type: text/plain; charset="us-ascii" - - ---BOUNDARY -Content-Type: text/plain; charset="koi8-r" - - ---BOUNDARY-- diff --git a/lib-python/modified-2.7/email/test/data/msg_12a.txt b/lib-python/modified-2.7/email/test/data/msg_12a.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_12a.txt +++ /dev/null @@ -1,38 +0,0 @@ -MIME-Version: 1.0 -From: Barry Warsaw <barry at zope.com> -To: Dingus Lovers <cravindogs at cravindogs.com> -Subject: Lyrics -Date: Fri, 20 Apr 2001 19:35:02 -0400 -Content-Type: multipart/mixed; boundary="BOUNDARY" - ---BOUNDARY -Content-Type: text/plain; charset="us-ascii" - - ---BOUNDARY -Content-Type: text/html; charset="iso-8859-1" - - ---BOUNDARY -Content-Type: multipart/mixed; boundary="ANOTHER" - ---ANOTHER -Content-Type: text/plain; charset="iso-8859-2" - - ---ANOTHER -Content-Type: text/plain; charset="iso-8859-3" - - ---ANOTHER-- - - ---BOUNDARY -Content-Type: text/plain; charset="us-ascii" - - ---BOUNDARY -Content-Type: text/plain; charset="koi8-r" - - ---BOUNDARY-- diff --git a/lib-python/modified-2.7/email/test/data/msg_13.txt b/lib-python/modified-2.7/email/test/data/msg_13.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_13.txt +++ /dev/null @@ -1,94 +0,0 @@ -MIME-Version: 1.0 -From: Barry <barry at digicool.com> -To: Dingus Lovers <cravindogs at cravindogs.com> -Subject: Here is your dingus fish -Date: Fri, 20 Apr 2001 19:35:02 -0400 -Content-Type: multipart/mixed; boundary="OUTER" - ---OUTER -Content-Type: text/plain; charset="us-ascii" - -A text/plain part - ---OUTER -Content-Type: multipart/mixed; boundary=BOUNDARY - - ---BOUNDARY -Content-Type: text/plain; charset="us-ascii" - -Hi there, - -This is the dingus fish. - ---BOUNDARY -Content-Type: image/gif; name="dingusfish.gif" -Content-Transfer-Encoding: base64 -content-disposition: attachment; filename="dingusfish.gif" - -R0lGODdhAAEAAfAAAP///wAAACwAAAAAAAEAAQAC/oSPqcvtD6OctNqLs968+w+G4kiW5omm6sq2 -7gvH8kzX9o3n+s73/g8MCofEovGITGICTKbyCV0FDNOo9SqpQqpOrJfXzTQj2vD3TGtqL+NtGQ2f -qTXmxzuOd7WXdcc9DyjU53ewFni4s0fGhdiYaEhGBelICTNoV1j5NUnFcrmUqemjNifJVWpaOqaI -oFq3SspZsSraE7sHq3jr1MZqWvi662vxV4tD+pvKW6aLDOCLyur8PDwbanyDeq0N3DctbQYeLDvR -RY6t95m6UB0d3mwIrV7e2VGNvjjffukeJp4w7F65KecGFsTHQGAygOrgrWs1jt28Rc88KESYcGLA -/obvTkH6p+CinWJiJmIMqXGQwH/y4qk0SYjgQTczT3ajKZGfuI0uJ4kkVI/DT5s3/ejkxI0aT4Y+ -YTYgWbImUaXk9nlLmnSh1qJiJFl0OpUqRK4oOy7NyRQtHWofhoYVxkwWXKUSn0YsS+fUV6lhqfYb -6ayd3Z5qQdG1B7bvQzaJjwUV2lixMUZ7JVsOlfjWVr/3NB/uFvnySBN6Dcb6rGwaRM3wsormw5cC -M9NxWy/bWdufudCvy8bOAjXjVVwta/uO21sE5RHBCzNFXtgq9ORtH4eYjVP4Yryo026nvkFmCeyA -B29efV6ravCMK5JwWd5897Qrx7ll38o6iHDZ/rXPR//feevhF4l7wjUGX3xq1eeRfM4RSJGBIV1D -z1gKPkfWag3mVBVvva1RlX5bAJTPR/2YqNtw/FkIYYEi/pIZiAdpcxpoHtmnYYoZtvhUftzdx5ZX -JSKDW405zkGcZzzGZ6KEv4FI224oDmijlEf+xp6MJK5ojY/ASeVUR+wsKRuJ+XFZ5o7ZeEime8t1 -ouUsU6YjF5ZtUihhkGfCdFQLWQFJ3UXxmElfhQnR+eCdcDbkFZp6vTRmj56ApCihn5QGpaToNZmR -n3NVSpZcQpZ2KEONusaiCsKAug0wkQbJSFO+PTSjneGxOuFjPlUk3ovWvdIerjUg9ZGIOtGq/qeX -eCYrrCX+1UPsgTKGGRSbzd5q156d/gpfbJxe66eD5iQKrXj7RGgruGxs62qebBHUKS32CKluCiqZ -qh+pmehmEb71noAUoe5e9Zm17S7773V10pjrtG4CmuurCV/n6zLK5turWNhqOvFXbjhZrMD0YhKe -wR0zOyuvsh6MWrGoIuzvyWu5y1WIFAqmJselypxXh6dKLNOKEB98L88bS2rkNqqlKzCNJp9c0G0j -Gzh0iRrCbHSXmPR643QS+4rWhgFmnSbSuXCjS0xAOWkU2UdLqyuUNfHSFdUouy3bm5i5GnDM3tG8 -doJ4r5tqu3pPbRSVfvs8uJzeNXhp3n4j/tZ42SwH7eaWUUOjc3qFV9453UHTXZfcLH+OeNs5g36x -lBnHvTm7EbMbLeuaLncao8vWCXimfo1o+843Ak6y4ChNeGntvAYvfLK4ezmoyNIbNCLTCXO9ZV3A -E8/s88RczPzDwI4Ob7XZyl7+9Miban29h+tJZPrE21wgvBphDfrrfPdCTPKJD/y98L1rZwHcV6Jq -Zab0metpuNIX/qAFPoz171WUaUb4HAhBSzHuHfjzHb3kha/2Cctis/ORArVHNYfFyYRH2pYIRzic -isVOfPWD1b6mRTqpCRBozzof6UZVvFXRxWIr3GGrEviGYgyPMfahheiSaLs/9QeFu7oZ/ndSY8DD -ya9x+uPed+7mxN2IzIISBOMLFYWVqC3Pew1T2nFuuCiwZS5/v6II10i4t1OJcUH2U9zxKodHsGGv -Oa+zkvNUYUOa/TCCRutF9MzDwdlUMJADTCGSbDQ5OV4PTamDoPEi6Ecc/RF5RWwkcdSXvSOaDWSn -I9LlvubFTQpuc6JKXLcKeb+xdbKRBnwREemXyjg6ME65aJiOuBgrktzykfPLJBKR9ClMavJ62/Ff -BlNIyod9yX9wcSXexnXFpvkrbXk64xsx5Db7wXKP5fSgsvwIMM/9631VLBfkmtbHRXpqmtei52hG -pUwSlo+BASQoeILDOBgREECxBBh5/iYmNsQ9dIv5+OI++QkqdsJPc3uykz5fkM+OraeekcQF7X4n -B5S67za5U967PmooGQhUXfF7afXyCD7ONdRe17QogYjVx38uLwtrS6nhTnm15LQUnu9E2uK6CNI/ -1HOABj0ESwOjut4FEpFQpdNAm4K2LHnDWHNcmKB2ioKBogysVZtMO2nSxUdZ8Yk2kJc7URioLVI0 -YgmtIwZj4LoeKemgnOnbUdGnzZ4Oa6scqiolBGqS6RgWNLu0RMhcaE6rhhU4hiuqFXPAG8fGwTPW -FKeLMtdVmXLSs5YJGF/YeVm7rREMlY3UYE+yCxbaMXX8y15m5zVHq6GOKDMynzII/jdUHdyVqIy0 -ifX2+r/EgtZcvRzSb72gU9ui87M2VecjKildW/aFqaYhKoryUjfB/g4qtyVuc60xFDGmCxwjW+qu -zjuwl2GkOWn66+3QiiEctvd04OVvcCVzjgT7lrkvjVGKKHmmlDUKowSeikb5kK/mJReuWOxONx+s -ULsl+Lqb0CVn0SrVyJ6wt4t6yTeSCafhPhAf0OXn6L60UMxiLolFAtmN35S2Ob1lZpQ1r/n0Qb5D -oQ1zJiRVDgF8N3Q8TYfbi3DyWCy3lT1nxyBs6FT3S2GOzWRlxwKvlRP0RPJA9SjxEy0UoEnkA+M4 -cnzLMJrBGWLFEaaUb5lvpqbq/loOaU5+DFuHPxo82/OZuM8FXG3oVNZhtWpMpb/0Xu5m/LfLhHZQ -7yuVI0MqZ7NE43imC8jH3IwGZlbPm0xkJYs7+2U48hXTsFSMqgGDvai0kLxyynKNT/waj+q1c1tz -GjOpPBgdCSq3UKZxCSsqFIY+O6JbAWGWcV1pwqLyj5sGqCF1xb1F3varUWqrJv6cN3PrUXzijtfZ -FshpBL3Xwr4GIPvU2N8EjrJgS1zl21rbXQMXeXc5jjFyrhpCzijSv/RQtyPSzHCFMhlME95fHglt -pRsX+dfSQjUeHAlpWzJ5iOo79Ldnaxai6bXTcGO3fp07ri7HLEmXXPlYi8bv/qVxvNcdra6m7Rlb -6JBTb5fd66VhFRjGArh2n7R1rDW4P5NOT9K0I183T2scYkeZ3q/VFyLb09U9ajzXBS8Kgkhc4mBS -kYY9cy3Vy9lUnuNJH8HGIclUilwnBtjUOH0gteGOZ4c/XNrhXLSYDyxfnD8z1pDy7rYRvDolhnbe -UMzxCZUs40s6s7UIvBnLgc0+vKuOkIXeOrDymlp+Zxra4MZLBbVrqD/jTJ597pDmnw5c4+DbyB88 -9Cg9DodYcSuMZT/114pptqc/EuTjRPvH/z5slzI3tluOEBBLqOXLOX+0I5929tO97wkvl/atCz+y -xJrdwteW2FNW/NSmBP+f/maYtVs/bYyBC7Ox3jsYZHL05CIrBa/nS+b3bHfiYm4Ueil1YZZSgAUI -fFZ1dxUmeA2oQRQ3RuGXNGLFV9/XbGFGPV6kfzk1TBBCd+izc7q1H+OHMJwmaBX2IQNYVAKHYepV -SSGCe6CnbYHHETKGNe43EDvFgZr0gB/nVHPHZ80VV1ojOiI3XDvYIkl4ayo4bxQIgrFXWTvBI0nH -VElWMuw2aLUWCRHHf8ymVCHjFlJnOSojfevCYyyyZDH0IcvHhrsnQ5O1OsWzONuVVKIxSxiFZ/tR -fKDAf6xFTnw4O9Qig2VCfW2hJQrmMOuHW0W3dLQmCMO2ccdUd/xyfflH/olTiHZVdGwb8nIwRzSE -J15jFlOJuBZBZ4CiyHyd2IFylFlB+HgHhYabhWOGwYO1ZH/Og1dtQlFMk352CGRSIFTapnWQEUtN -l4zv8S0aaCFDyGCBqDUxZYpxGHX01y/JuH1xhn7TOCnNCI4eKDs5WGX4R425F4vF1o3BJ4vO0otq -I3rimI7jJY1jISqnBxknCIvruF83mF5wN4X7qGLIhR8A2Vg0yFERSIXn9Vv3GHy3Vj/WIkKddlYi -yIMv2I/VMjTLpW7pt05SWIZR0RPyxpB4SIUM9lBPGBl0GC7oSEEwRYLe4pJpZY2P0zbI1n+Oc44w -qY3PUnmF0ixjVpDD/mJ9wpOBGTVgXlaCaZiPcIWK5NiKBIiPdGaQ0TWGvAiG7nMchdZb7Vgf8zNi -MuMyzRdy/lePe9iC4TRx7WhhOQI/QiSVNAmAa2lT/piFbuh7ofJoYSZzrSZ1bvmWw3eN2nKUPVky -uPN5/VRfohRd0VYZoqhKIlU6TXYhJxmPUIloAwc1bPmHEpaZYZORHNlXUJM07hATwHR8MJYqkwWR -WaIezFhxSFlc8/Fq82hEnpeRozg3ULhhr9lAGtVEkCg5ZNRuuVleBPaZadhG0ZgkyPmDOTOKzViM -YgOcpukKqQcbjAWS0IleQ2ROjdh6A+md1qWdBRSX7iSYgFRTtRmBpJioieXJiHfJiMGIR9fJOn8I -MSfXYhspn4ooSa2mSAj4n+8Bmg03fBJZoPOJgsVZRxu1oOMRPXYYjdqjihFaEoZpXBREanuJoRI6 -cibFinq4ngUKh/wQd/H5ofYCZ0HJXR62opZFaAT0iFIZo4DIiUojkjeqKiuoZirKo5Y1a7AWckGa -BkuYoD5lpDK6eUs6CkDqpETwl1EqpfhJpVeKpVl6EgUAADs= - ---BOUNDARY-- - ---OUTER-- diff --git a/lib-python/modified-2.7/email/test/data/msg_14.txt b/lib-python/modified-2.7/email/test/data/msg_14.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_14.txt +++ /dev/null @@ -1,23 +0,0 @@ -Return-Path: <bbb at zzz.org> -Delivered-To: bbb at zzz.org -Received: by mail.zzz.org (Postfix, from userid 889) - id 27CEAD38CC; Fri, 4 May 2001 14:05:44 -0400 (EDT) -MIME-Version: 1.0 -Content-Type: text; charset=us-ascii -Content-Transfer-Encoding: 7bit -Message-ID: <15090.61304.110929.45684 at aaa.zzz.org> -From: bbb at ddd.com (John X. Doe) -To: bbb at zzz.org -Subject: This is a test message -Date: Fri, 4 May 2001 14:05:44 -0400 - - -Hi, - -I'm sorry but I'm using a drainbread ISP, which although big and -wealthy can't seem to generate standard compliant email. :( - -This message has a Content-Type: header with no subtype. I hope you -can still read it. - --Me diff --git a/lib-python/modified-2.7/email/test/data/msg_15.txt b/lib-python/modified-2.7/email/test/data/msg_15.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_15.txt +++ /dev/null @@ -1,52 +0,0 @@ -Return-Path: <xx at xx.dk> -Received: from fepD.post.tele.dk (195.41.46.149) by mail.groupcare.dk (LSMTP for Windows NT v1.1b) with SMTP id <0.0014F8A2 at mail.groupcare.dk>; Mon, 30 Apr 2001 12:17:50 +0200 -User-Agent: Microsoft-Outlook-Express-Macintosh-Edition/5.02.2106 -Subject: XX -From: xx at xx.dk -To: XX -Message-ID: <xxxx> -Mime-version: 1.0 -Content-type: multipart/mixed; - boundary="MS_Mac_OE_3071477847_720252_MIME_Part" - -> Denne meddelelse er i MIME-format. Da dit postl�sningsprogram ikke forst�r dette format, kan del af eller hele meddelelsen v�re ul�selig. - ---MS_Mac_OE_3071477847_720252_MIME_Part -Content-type: multipart/alternative; - boundary="MS_Mac_OE_3071477847_720252_MIME_Part" - - ---MS_Mac_OE_3071477847_720252_MIME_Part -Content-type: text/plain; charset="ISO-8859-1" -Content-transfer-encoding: quoted-printable - -Some removed test. - ---MS_Mac_OE_3071477847_720252_MIME_Part -Content-type: text/html; charset="ISO-8859-1" -Content-transfer-encoding: quoted-printable - -<HTML> -<HEAD> -<TITLE>Some removed HTML</TITLE> -</HEAD> -<BODY> -Some removed text. -</BODY> -</HTML> - - ---MS_Mac_OE_3071477847_720252_MIME_Part-- - - ---MS_Mac_OE_3071477847_720252_MIME_Part -Content-type: image/gif; name="xx.gif"; - x-mac-creator="6F676C65"; - x-mac-type="47494666" -Content-disposition: attachment -Content-transfer-encoding: base64 - -Some removed base64 encoded chars. - ---MS_Mac_OE_3071477847_720252_MIME_Part-- - diff --git a/lib-python/modified-2.7/email/test/data/msg_16.txt b/lib-python/modified-2.7/email/test/data/msg_16.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_16.txt +++ /dev/null @@ -1,123 +0,0 @@ -Return-Path: <> -Delivered-To: scr-admin at socal-raves.org -Received: from cougar.noc.ucla.edu (cougar.noc.ucla.edu [169.232.10.18]) - by babylon.socal-raves.org (Postfix) with ESMTP id CCC2C51B84 - for <scr-admin at socal-raves.org>; Sun, 23 Sep 2001 20:13:54 -0700 (PDT) -Received: from sims-ms-daemon by cougar.noc.ucla.edu - (Sun Internet Mail Server sims.3.5.2000.03.23.18.03.p10) - id <0GK500B01D0B8Y at cougar.noc.ucla.edu> for scr-admin at socal-raves.org; Sun, - 23 Sep 2001 20:14:35 -0700 (PDT) -Received: from cougar.noc.ucla.edu - (Sun Internet Mail Server sims.3.5.2000.03.23.18.03.p10) - id <0GK500B01D0B8X at cougar.noc.ucla.edu>; Sun, 23 Sep 2001 20:14:35 -0700 (PDT) -Date: Sun, 23 Sep 2001 20:14:35 -0700 (PDT) -From: Internet Mail Delivery <postmaster at ucla.edu> -Subject: Delivery Notification: Delivery has failed -To: scr-admin at socal-raves.org -Message-id: <0GK500B04D0B8X at cougar.noc.ucla.edu> -MIME-version: 1.0 -Sender: scr-owner at socal-raves.org -Errors-To: scr-owner at socal-raves.org -X-BeenThere: scr at socal-raves.org -X-Mailman-Version: 2.1a3 -Precedence: bulk -List-Help: <mailto:scr-request at socal-raves.org?subject=help> -List-Post: <mailto:scr at socal-raves.org> -List-Subscribe: <http://socal-raves.org/mailman/listinfo/scr>, - <mailto:scr-request at socal-raves.org?subject=subscribe> -List-Id: SoCal-Raves <scr.socal-raves.org> -List-Unsubscribe: <http://socal-raves.org/mailman/listinfo/scr>, - <mailto:scr-request at socal-raves.org?subject=unsubscribe> -List-Archive: <http://socal-raves.org/mailman/private/scr/> -Content-Type: multipart/report; boundary="Boundary_(ID_PGS2F2a+z+/jL7hupKgRhA)" - - ---Boundary_(ID_PGS2F2a+z+/jL7hupKgRhA) -Content-type: text/plain; charset=ISO-8859-1 - -This report relates to a message you sent with the following header fields: - - Message-id: <002001c144a6$8752e060$56104586 at oxy.edu> - Date: Sun, 23 Sep 2001 20:10:55 -0700 - From: "Ian T. Henry" <henryi at oxy.edu> - To: SoCal Raves <scr at socal-raves.org> - Subject: [scr] yeah for Ians!! - -Your message cannot be delivered to the following recipients: - - Recipient address: jangel1 at cougar.noc.ucla.edu - Reason: recipient reached disk quota - - ---Boundary_(ID_PGS2F2a+z+/jL7hupKgRhA) -Content-type: message/DELIVERY-STATUS - -Original-envelope-id: 0GK500B4HD0888 at cougar.noc.ucla.edu -Reporting-MTA: dns; cougar.noc.ucla.edu - -Action: failed -Status: 5.0.0 (recipient reached disk quota) -Original-recipient: rfc822;jangel1 at cougar.noc.ucla.edu -Final-recipient: rfc822;jangel1 at cougar.noc.ucla.edu - ---Boundary_(ID_PGS2F2a+z+/jL7hupKgRhA) -Content-type: MESSAGE/RFC822 - -Return-path: scr-admin at socal-raves.org -Received: from sims-ms-daemon by cougar.noc.ucla.edu - (Sun Internet Mail Server sims.3.5.2000.03.23.18.03.p10) - id <0GK500B01D0B8X at cougar.noc.ucla.edu>; Sun, 23 Sep 2001 20:14:35 -0700 (PDT) -Received: from panther.noc.ucla.edu by cougar.noc.ucla.edu - (Sun Internet Mail Server sims.3.5.2000.03.23.18.03.p10) - with ESMTP id <0GK500B4GD0888 at cougar.noc.ucla.edu> for jangel1 at sims-ms-daemon; - Sun, 23 Sep 2001 20:14:33 -0700 (PDT) -Received: from babylon.socal-raves.org - (ip-209-85-222-117.dreamhost.com [209.85.222.117]) - by panther.noc.ucla.edu (8.9.1a/8.9.1) with ESMTP id UAA09793 for - <jangel1 at ucla.edu>; Sun, 23 Sep 2001 20:14:32 -0700 (PDT) -Received: from babylon (localhost [127.0.0.1]) by babylon.socal-raves.org - (Postfix) with ESMTP id D3B2951B70; Sun, 23 Sep 2001 20:13:47 -0700 (PDT) -Received: by babylon.socal-raves.org (Postfix, from userid 60001) - id A611F51B82; Sun, 23 Sep 2001 20:13:46 -0700 (PDT) -Received: from tiger.cc.oxy.edu (tiger.cc.oxy.edu [134.69.3.112]) - by babylon.socal-raves.org (Postfix) with ESMTP id ADA7351B70 for - <scr at socal-raves.org>; Sun, 23 Sep 2001 20:13:44 -0700 (PDT) -Received: from ent (n16h86.dhcp.oxy.edu [134.69.16.86]) - by tiger.cc.oxy.edu (8.8.8/8.8.8) with SMTP id UAA08100 for - <scr at socal-raves.org>; Sun, 23 Sep 2001 20:14:24 -0700 (PDT) -Date: Sun, 23 Sep 2001 20:10:55 -0700 -From: "Ian T. Henry" <henryi at oxy.edu> -Subject: [scr] yeah for Ians!! -Sender: scr-admin at socal-raves.org -To: SoCal Raves <scr at socal-raves.org> -Errors-to: scr-admin at socal-raves.org -Message-id: <002001c144a6$8752e060$56104586 at oxy.edu> -MIME-version: 1.0 -X-Mailer: Microsoft Outlook Express 5.50.4522.1200 -Content-type: text/plain; charset=us-ascii -Precedence: bulk -Delivered-to: scr-post at babylon.socal-raves.org -Delivered-to: scr at socal-raves.org -X-Converted-To-Plain-Text: from multipart/alternative by demime 0.98e -X-Converted-To-Plain-Text: Alternative section used was text/plain -X-BeenThere: scr at socal-raves.org -X-Mailman-Version: 2.1a3 -List-Help: <mailto:scr-request at socal-raves.org?subject=help> -List-Post: <mailto:scr at socal-raves.org> -List-Subscribe: <http://socal-raves.org/mailman/listinfo/scr>, - <mailto:scr-request at socal-raves.org?subject=subscribe> -List-Id: SoCal-Raves <scr.socal-raves.org> -List-Unsubscribe: <http://socal-raves.org/mailman/listinfo/scr>, - <mailto:scr-request at socal-raves.org?subject=unsubscribe> -List-Archive: <http://socal-raves.org/mailman/private/scr/> - -I always love to find more Ian's that are over 3 years old!! - -Ian -_______________________________________________ -For event info, list questions, or to unsubscribe, see http://www.socal-raves.org/ - - - ---Boundary_(ID_PGS2F2a+z+/jL7hupKgRhA)-- - diff --git a/lib-python/modified-2.7/email/test/data/msg_17.txt b/lib-python/modified-2.7/email/test/data/msg_17.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_17.txt +++ /dev/null @@ -1,12 +0,0 @@ -MIME-Version: 1.0 -From: Barry <barry at digicool.com> -To: Dingus Lovers <cravindogs at cravindogs.com> -Subject: Here is your dingus fish -Date: Fri, 20 Apr 2001 19:35:02 -0400 -Content-Type: multipart/mixed; boundary="BOUNDARY" - -Hi there, - -This is the dingus fish. - -[Non-text (image/gif) part of message omitted, filename dingusfish.gif] diff --git a/lib-python/modified-2.7/email/test/data/msg_18.txt b/lib-python/modified-2.7/email/test/data/msg_18.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_18.txt +++ /dev/null @@ -1,6 +0,0 @@ -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit -X-Foobar-Spoink-Defrobnit: wasnipoop; giraffes="very-long-necked-animals"; - spooge="yummy"; hippos="gargantuan"; marshmallows="gooey" - diff --git a/lib-python/modified-2.7/email/test/data/msg_19.txt b/lib-python/modified-2.7/email/test/data/msg_19.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_19.txt +++ /dev/null @@ -1,43 +0,0 @@ -Send Ppp mailing list submissions to - ppp at zzz.org - -To subscribe or unsubscribe via the World Wide Web, visit - http://www.zzz.org/mailman/listinfo/ppp -or, via email, send a message with subject or body 'help' to - ppp-request at zzz.org - -You can reach the person managing the list at - ppp-admin at zzz.org - -When replying, please edit your Subject line so it is more specific -than "Re: Contents of Ppp digest..." - -Today's Topics: - - 1. testing #1 (Barry A. Warsaw) - 2. testing #2 (Barry A. Warsaw) - 3. testing #3 (Barry A. Warsaw) - 4. testing #4 (Barry A. Warsaw) - 5. testing #5 (Barry A. Warsaw) - -hello - - -hello - - -hello - - -hello - - -hello - - - -_______________________________________________ -Ppp mailing list -Ppp at zzz.org -http://www.zzz.org/mailman/listinfo/ppp - diff --git a/lib-python/modified-2.7/email/test/data/msg_20.txt b/lib-python/modified-2.7/email/test/data/msg_20.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_20.txt +++ /dev/null @@ -1,22 +0,0 @@ -Return-Path: <bbb at zzz.org> -Delivered-To: bbb at zzz.org -Received: by mail.zzz.org (Postfix, from userid 889) - id 27CEAD38CC; Fri, 4 May 2001 14:05:44 -0400 (EDT) -MIME-Version: 1.0 -Content-Type: text/plain; charset=us-ascii -Content-Transfer-Encoding: 7bit -Message-ID: <15090.61304.110929.45684 at aaa.zzz.org> -From: bbb at ddd.com (John X. Doe) -To: bbb at zzz.org -Cc: ccc at zzz.org -CC: ddd at zzz.org -cc: eee at zzz.org -Subject: This is a test message -Date: Fri, 4 May 2001 14:05:44 -0400 - - -Hi, - -Do you like this message? - --Me diff --git a/lib-python/modified-2.7/email/test/data/msg_21.txt b/lib-python/modified-2.7/email/test/data/msg_21.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_21.txt +++ /dev/null @@ -1,20 +0,0 @@ -From: aperson at dom.ain -To: bperson at dom.ain -Subject: Test -Content-Type: multipart/mixed; boundary="BOUNDARY" - -MIME message ---BOUNDARY -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit - -One ---BOUNDARY -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit - -Two ---BOUNDARY-- -End of MIME message diff --git a/lib-python/modified-2.7/email/test/data/msg_22.txt b/lib-python/modified-2.7/email/test/data/msg_22.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_22.txt +++ /dev/null @@ -1,46 +0,0 @@ -Mime-Version: 1.0 -Message-Id: <a05001902b7f1c33773e9@[134.84.183.138]> -Date: Tue, 16 Oct 2001 13:59:25 +0300 -To: a at example.com -From: b at example.com -Content-Type: multipart/mixed; boundary="============_-1208892523==_============" - ---============_-1208892523==_============ -Content-Type: text/plain; charset="us-ascii" ; format="flowed" - -Text text text. ---============_-1208892523==_============ -Content-Id: <a05001902b7f1c33773e9@[134.84.183.138].0.0> -Content-Type: image/jpeg; name="wibble.JPG" - ; x-mac-type="4A504547" - ; x-mac-creator="474B4F4E" -Content-Disposition: attachment; filename="wibble.JPG" -Content-Transfer-Encoding: base64 - -/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEB -AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/wAALCAXABIEBAREA -g6bCjjw/pIZSjO6FWFpldjySOmCNrO7DBZibUXhTwtCixw+GtAijVdqxxaPp0aKvmGXa -qrbBQvms0mAMeYS/3iTV1dG0hHaRNK01XblnWxtVdjkHLMIgTyqnk9VB7CrP2KzIINpa -4O7I+zxYO9WV8jZg71Zlb+8rMDkEirAVQFAUAKAFAAAUAYAUDgADgY6DjpRtXj5RxjHA -4wQRj0wQCMdCAewpaKKK/9k= ---============_-1208892523==_============ -Content-Id: <a05001902b7f1c33773e9@[134.84.183.138].0.1> -Content-Type: image/jpeg; name="wibble2.JPG" - ; x-mac-type="4A504547" - ; x-mac-creator="474B4F4E" -Content-Disposition: attachment; filename="wibble2.JPG" -Content-Transfer-Encoding: base64 - -/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEB -AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/wAALCAXABJ0BAREA -/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQA -W6NFJJBEkU10kKGTcWMDwxuU+0JHvk8qAtOpNwqSR0n8c3BlDyXHlqsUltHEiTvdXLxR -7vMiGDNJAJWkAMk8ZkCFp5G2oo5W++INrbQtNfTQxJAuXlupz9oS4d5Y1W+E2XlWZJJE -Y7LWYQxTLE1zuMbfBPxw8X2fibVdIbSbI6nLZxX635t9TjtYreWR7WGKJTLJFFKSlozO -0ShxIXM43uC3/9k= ---============_-1208892523==_============ -Content-Type: text/plain; charset="us-ascii" ; format="flowed" - -Text text text. ---============_-1208892523==_============-- - diff --git a/lib-python/modified-2.7/email/test/data/msg_23.txt b/lib-python/modified-2.7/email/test/data/msg_23.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_23.txt +++ /dev/null @@ -1,8 +0,0 @@ -From: aperson at dom.ain -Content-Type: multipart/mixed; boundary="BOUNDARY" - ---BOUNDARY -Content-Type: text/plain - -A message part ---BOUNDARY-- diff --git a/lib-python/modified-2.7/email/test/data/msg_24.txt b/lib-python/modified-2.7/email/test/data/msg_24.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_24.txt +++ /dev/null @@ -1,10 +0,0 @@ -Content-Type: multipart/mixed; boundary="BOUNDARY" -MIME-Version: 1.0 -Subject: A subject -To: aperson at dom.ain -From: bperson at dom.ain - ---BOUNDARY - - ---BOUNDARY-- diff --git a/lib-python/modified-2.7/email/test/data/msg_25.txt b/lib-python/modified-2.7/email/test/data/msg_25.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_25.txt +++ /dev/null @@ -1,117 +0,0 @@ -From MAILER-DAEMON Fri Apr 06 16:46:09 2001 -Received: from [204.245.199.98] (helo=zinfandel.lacita.com) - by www.linux.org.uk with esmtp (Exim 3.13 #1) - id 14lYR6-0008Iv-00 - for linuxuser-admin at www.linux.org.uk; Fri, 06 Apr 2001 16:46:09 +0100 -Received: from localhost (localhost) by zinfandel.lacita.com (8.7.3/8.6.10-MT4.00) with internal id JAB03225; Fri, 6 Apr 2001 09:23:06 -0800 (GMT-0800) -Date: Fri, 6 Apr 2001 09:23:06 -0800 (GMT-0800) -From: Mail Delivery Subsystem <MAILER-DAEMON at zinfandel.lacita.com> -Subject: Returned mail: Too many hops 19 (17 max): from <linuxuser-admin at www.linux.org.uk> via [199.164.235.226], to <scoffman at wellpartner.com> -Message-Id: <200104061723.JAB03225 at zinfandel.lacita.com> -To: <linuxuser-admin at www.linux.org.uk> -To: postmaster at zinfandel.lacita.com -MIME-Version: 1.0 -Content-Type: multipart/report; report-type=delivery-status; - bo -Auto-Submitted: auto-generated (failure) - -This is a MIME-encapsulated message - ---JAB03225.986577786/zinfandel.lacita.com - -The original message was received at Fri, 6 Apr 2001 09:23:03 -0800 (GMT-0800) -from [199.164.235.226] - - ----- The following addresses have delivery notifications ----- -<scoffman at wellpartner.com> (unrecoverable error) - - ----- Transcript of session follows ----- -554 Too many hops 19 (17 max): from <linuxuser-admin at www.linux.org.uk> via [199.164.235.226], to <scoffman at wellpartner.com> - ---JAB03225.986577786/zinfandel.lacita.com -Content-Type: message/delivery-status - -Reporting-MTA: dns; zinfandel.lacita.com -Received-From-MTA: dns; [199.164.235.226] -Arrival-Date: Fri, 6 Apr 2001 09:23:03 -0800 (GMT-0800) - -Final-Recipient: rfc822; scoffman at wellpartner.com -Action: failed -Status: 5.4.6 -Last-Attempt-Date: Fri, 6 Apr 2001 09:23:06 -0800 (GMT-0800) - ---JAB03225.986577786/zinfandel.lacita.com -Content-Type: text/rfc822-headers - -Return-Path: linuxuser-admin at www.linux.org.uk -Received: from ns1.wellpartner.net ([199.164.235.226]) by zinfandel.lacita.com (8.7.3/8.6.10-MT4.00) with ESMTP id JAA03225 for <scoffman at wellpartner.com>; Fri, 6 Apr 2001 09:23:03 -0800 (GMT-0800) -Received: from zinfandel.lacita.com ([204.245.199.98]) - by - fo -Received: from ns1.wellpartner.net ([199.164.235.226]) by zinfandel.lacita.com (8.7.3/8.6.10-MT4.00) with ESMTP id JAA03221 for <scoffman at wellpartner.com>; Fri, 6 Apr 2001 09:22:18 -0800 (GMT-0800) -Received: from zinfandel.lacita.com ([204.245.199.98]) - by - fo -Received: from ns1.wellpartner.net ([199.164.235.226]) by zinfandel.lacita.com (8.7.3/8.6.10-MT4.00) with ESMTP id JAA03217 for <scoffman at wellpartner.com>; Fri, 6 Apr 2001 09:21:37 -0800 (GMT-0800) -Received: from zinfandel.lacita.com ([204.245.199.98]) - by - fo -Received: from ns1.wellpartner.net ([199.164.235.226]) by zinfandel.lacita.com (8.7.3/8.6.10-MT4.00) with ESMTP id JAA03213 for <scoffman at wellpartner.com>; Fri, 6 Apr 2001 09:20:56 -0800 (GMT-0800) -Received: from zinfandel.lacita.com ([204.245.199.98]) - by - fo -Received: from ns1.wellpartner.net ([199.164.235.226]) by zinfandel.lacita.com (8.7.3/8.6.10-MT4.00) with ESMTP id JAA03209 for <scoffman at wellpartner.com>; Fri, 6 Apr 2001 09:20:15 -0800 (GMT-0800) -Received: from zinfandel.lacita.com ([204.245.199.98]) - by - fo -Received: from ns1.wellpartner.net ([199.164.235.226]) by zinfandel.lacita.com (8.7.3/8.6.10-MT4.00) with ESMTP id JAA03205 for <scoffman at wellpartner.com>; Fri, 6 Apr 2001 09:19:33 -0800 (GMT-0800) -Received: from zinfandel.lacita.com ([204.245.199.98]) - by - fo -Received: from ns1.wellpartner.net ([199.164.235.226]) by zinfandel.lacita.com (8.7.3/8.6.10-MT4.00) with ESMTP id JAA03201 for <scoffman at wellpartner.com>; Fri, 6 Apr 2001 09:18:52 -0800 (GMT-0800) -Received: from zinfandel.lacita.com ([204.245.199.98]) - by - fo -Received: from ns1.wellpartner.net ([199.164.235.226]) by zinfandel.lacita.com (8.7.3/8.6.10-MT4.00) with ESMTP id JAA03197 for <scoffman at wellpartner.com>; Fri, 6 Apr 2001 09:17:54 -0800 (GMT-0800) -Received: from www.linux.org.uk (parcelfarce.linux.theplanet.co.uk [195.92.249.252]) - by - fo -Received: from localhost.localdomain - ([ - by - id -Received: from [212.1.130.11] (helo=s1.uklinux.net ident=root) - by - id - fo -Received: from server (ppp-2-22.cvx4.telinco.net [212.1.149.22]) - by - fo -From: Daniel James <daniel at linuxuser.co.uk> -Organization: LinuxUser -To: linuxuser at www.linux.org.uk -X-Mailer: KMail [version 1.1.99] -Content-Type: text/plain; - c -MIME-Version: 1.0 -Message-Id: <01040616033903.00962 at server> -Content-Transfer-Encoding: 8bit -Subject: [LinuxUser] bulletin no. 45 -Sender: linuxuser-admin at www.linux.org.uk -Errors-To: linuxuser-admin at www.linux.org.uk -X-BeenThere: linuxuser at www.linux.org.uk -X-Mailman-Version: 2.0.3 -Precedence: bulk -List-Help: <mailto:linuxuser-request at www.linux.org.uk?subject=help> -List-Post: <mailto:linuxuser at www.linux.org.uk> -List-Subscribe: <http://www.linux.org.uk/mailman/listinfo/linuxuser>, - <m -List-Id: bulletins from LinuxUser magazine <linuxuser.www.linux.org.uk> -List-Unsubscribe: <http://www.linux.org.uk/mailman/listinfo/linuxuser>, - <m -List-Archive: <http://www.linux.org.uk/pipermail/linuxuser/> -Date: Fri, 6 Apr 2001 16:03:39 +0100 - ---JAB03225.986577786/zinfandel.lacita.com-- - - diff --git a/lib-python/modified-2.7/email/test/data/msg_26.txt b/lib-python/modified-2.7/email/test/data/msg_26.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_26.txt +++ /dev/null @@ -1,45 +0,0 @@ -Received: from xcar [192.168.0.2] by jeeves.wooster.local - (SMTPD32-7.07 EVAL) id AFF92F0214; Sun, 12 May 2002 08:55:37 +0100 -Date: Sun, 12 May 2002 08:56:15 +0100 -From: Father Time <father.time at xcar.wooster.local> -To: timbo at jeeves.wooster.local -Subject: IMAP file test -Message-ID: <6df65d354b.father.time at rpc.wooster.local> -X-Organization: Home -User-Agent: Messenger-Pro/2.50a (MsgServe/1.50) (RISC-OS/4.02) POPstar/2.03 -MIME-Version: 1.0 -Content-Type: multipart/mixed; boundary="1618492860--2051301190--113853680" -Status: R -X-UIDL: 319998302 - -This message is in MIME format which your mailer apparently does not support. -You either require a newer version of your software which supports MIME, or -a separate MIME decoding utility. Alternatively, ask the sender of this -message to resend it in a different format. - ---1618492860--2051301190--113853680 -Content-Type: text/plain; charset=us-ascii - -Simple email with attachment. - - ---1618492860--2051301190--113853680 -Content-Type: application/riscos; name="clock.bmp,69c"; type=BMP; load=&fff69c4b; exec=&355dd4d1; access=&03 -Content-Disposition: attachment; filename="clock.bmp" -Content-Transfer-Encoding: base64 - -Qk12AgAAAAAAAHYAAAAoAAAAIAAAACAAAAABAAQAAAAAAAAAAADXDQAA1w0AAAAAAAAA -AAAAAAAAAAAAiAAAiAAAAIiIAIgAAACIAIgAiIgAALu7uwCIiIgAERHdACLuIgAz//8A -zAAAAN0R3QDu7iIA////AAAAAAAAAAAAAAAAAAAAAAAAAAi3AAAAAAAAADeAAAAAAAAA -C3ADMzMzMANwAAAAAAAAAAAHMAAAAANwAAAAAAAAAACAMAd3zPfwAwgAAAAAAAAIAwd/ -f8x/f3AwgAAAAAAAgDB0x/f3//zPAwgAAAAAAAcHfM9////8z/AwAAAAAAiwd/f3//// -////A4AAAAAAcEx/f///////zAMAAAAAiwfM9////3///8zwOAAAAAcHf3////B///// -8DAAAAALB/f3///wd3d3//AwAAAABwTPf//wCQAAD/zAMAAAAAsEx/f///B////8wDAA -AAAHB39////wf/////AwAAAACwf39///8H/////wMAAAAIcHfM9///B////M8DgAAAAA -sHTH///wf///xAMAAAAACHB3f3//8H////cDgAAAAAALB3zH//D//M9wMAAAAAAAgLB0 -z39///xHAwgAAAAAAAgLB3d3RHd3cDCAAAAAAAAAgLAHd0R3cAMIAAAAAAAAgAgLcAAA -AAMwgAgAAAAACDAAAAu7t7cwAAgDgAAAAABzcIAAAAAAAAgDMwAAAAAAN7uwgAAAAAgH -MzMAAAAACH97tzAAAAALu3c3gAAAAAAL+7tzDABAu7f7cAAAAAAACA+3MA7EQAv/sIAA -AAAAAAAIAAAAAAAAAIAAAAAA - ---1618492860--2051301190--113853680-- \ No newline at end of file diff --git a/lib-python/modified-2.7/email/test/data/msg_27.txt b/lib-python/modified-2.7/email/test/data/msg_27.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_27.txt +++ /dev/null @@ -1,15 +0,0 @@ -Return-Path: <aperson at dom.ain> -Received: by mail.dom.ain (Postfix, from userid 889) - id B9D0AD35DB; Tue, 4 Jun 2002 21:46:59 -0400 (EDT) -Message-ID: <15613.28051.707126.569693 at dom.ain> -Date: Tue, 4 Jun 2002 21:46:59 -0400 -MIME-Version: 1.0 -Content-Type: text/plain; charset=us-ascii -Content-Transfer-Encoding: 7bit -Subject: bug demonstration - 12345678911234567892123456789312345678941234567895123456789612345678971234567898112345678911234567892123456789112345678911234567892123456789 - more text -From: aperson at dom.ain (Anne P. Erson) -To: bperson at dom.ain (Barney P. Erson) - -test diff --git a/lib-python/modified-2.7/email/test/data/msg_28.txt b/lib-python/modified-2.7/email/test/data/msg_28.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_28.txt +++ /dev/null @@ -1,25 +0,0 @@ -From: aperson at dom.ain -MIME-Version: 1.0 -Content-Type: multipart/digest; boundary=BOUNDARY - ---BOUNDARY -Content-Type: message/rfc822 - -Content-Type: text/plain; charset=us-ascii -To: aa at bb.org -From: cc at dd.org -Subject: ee - -message 1 - ---BOUNDARY -Content-Type: message/rfc822 - -Content-Type: text/plain; charset=us-ascii -To: aa at bb.org -From: cc at dd.org -Subject: ee - -message 2 - ---BOUNDARY-- diff --git a/lib-python/modified-2.7/email/test/data/msg_29.txt b/lib-python/modified-2.7/email/test/data/msg_29.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_29.txt +++ /dev/null @@ -1,22 +0,0 @@ -Return-Path: <bbb at zzz.org> -Delivered-To: bbb at zzz.org -Received: by mail.zzz.org (Postfix, from userid 889) - id 27CEAD38CC; Fri, 4 May 2001 14:05:44 -0400 (EDT) -MIME-Version: 1.0 -Content-Type: text/plain; charset=us-ascii; - title*0*="us-ascii'en'This%20is%20even%20more%20"; - title*1*="%2A%2A%2Afun%2A%2A%2A%20"; - title*2="isn't it!" -Content-Transfer-Encoding: 7bit -Message-ID: <15090.61304.110929.45684 at aaa.zzz.org> -From: bbb at ddd.com (John X. Doe) -To: bbb at zzz.org -Subject: This is a test message -Date: Fri, 4 May 2001 14:05:44 -0400 - - -Hi, - -Do you like this message? - --Me diff --git a/lib-python/modified-2.7/email/test/data/msg_30.txt b/lib-python/modified-2.7/email/test/data/msg_30.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_30.txt +++ /dev/null @@ -1,23 +0,0 @@ -From: aperson at dom.ain -MIME-Version: 1.0 -Content-Type: multipart/digest; boundary=BOUNDARY - ---BOUNDARY - -Content-Type: text/plain; charset=us-ascii -To: aa at bb.org -From: cc at dd.org -Subject: ee - -message 1 - ---BOUNDARY - -Content-Type: text/plain; charset=us-ascii -To: aa at bb.org -From: cc at dd.org -Subject: ee - -message 2 - ---BOUNDARY-- diff --git a/lib-python/modified-2.7/email/test/data/msg_31.txt b/lib-python/modified-2.7/email/test/data/msg_31.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_31.txt +++ /dev/null @@ -1,15 +0,0 @@ -From: aperson at dom.ain -MIME-Version: 1.0 -Content-Type: multipart/mixed; boundary=BOUNDARY_ - ---BOUNDARY -Content-Type: text/plain - -message 1 - ---BOUNDARY -Content-Type: text/plain - -message 2 - ---BOUNDARY-- diff --git a/lib-python/modified-2.7/email/test/data/msg_32.txt b/lib-python/modified-2.7/email/test/data/msg_32.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_32.txt +++ /dev/null @@ -1,14 +0,0 @@ -Delivered-To: freebsd-isp at freebsd.org -Date: Tue, 26 Sep 2000 12:23:03 -0500 -From: Anne Person <aperson at example.com> -To: Barney Dude <bdude at example.com> -Subject: Re: Limiting Perl CPU Utilization... -Mime-Version: 1.0 -Content-Type: text/plain; charset*=ansi-x3.4-1968''us-ascii -Content-Disposition: inline -User-Agent: Mutt/1.3.8i -Sender: owner-freebsd-isp at FreeBSD.ORG -Precedence: bulk -X-Loop: FreeBSD.org - -Some message. diff --git a/lib-python/modified-2.7/email/test/data/msg_33.txt b/lib-python/modified-2.7/email/test/data/msg_33.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_33.txt +++ /dev/null @@ -1,29 +0,0 @@ -Delivered-To: freebsd-isp at freebsd.org -Date: Wed, 27 Sep 2000 11:11:09 -0500 -From: Anne Person <aperson at example.com> -To: Barney Dude <bdude at example.com> -Subject: Re: Limiting Perl CPU Utilization... -Mime-Version: 1.0 -Content-Type: multipart/signed; micalg*=ansi-x3.4-1968''pgp-md5; - protocol*=ansi-x3.4-1968''application%2Fpgp-signature; - boundary*="ansi-x3.4-1968''EeQfGwPcQSOJBaQU" -Content-Disposition: inline -Sender: owner-freebsd-isp at FreeBSD.ORG -Precedence: bulk -X-Loop: FreeBSD.org - - ---EeQfGwPcQSOJBaQU -Content-Type: text/plain; charset*=ansi-x3.4-1968''us-ascii -Content-Disposition: inline -Content-Transfer-Encoding: quoted-printable - -part 1 - ---EeQfGwPcQSOJBaQU -Content-Type: text/plain -Content-Disposition: inline - -part 2 - ---EeQfGwPcQSOJBaQU-- diff --git a/lib-python/modified-2.7/email/test/data/msg_34.txt b/lib-python/modified-2.7/email/test/data/msg_34.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_34.txt +++ /dev/null @@ -1,19 +0,0 @@ -From: aperson at dom.ain -To: bperson at dom.ain -Content-Type: multipart/digest; boundary=XYZ - ---XYZ -Content-Type: text/plain - - -This is a text plain part that is counter to recommended practice in -RFC 2046, $5.1.5, but is not illegal - ---XYZ - -From: cperson at dom.ain -To: dperson at dom.ain - -A submessage - ---XYZ-- diff --git a/lib-python/modified-2.7/email/test/data/msg_35.txt b/lib-python/modified-2.7/email/test/data/msg_35.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_35.txt +++ /dev/null @@ -1,4 +0,0 @@ -From: aperson at dom.ain -To: bperson at dom.ain -Subject: here's something interesting -counter to RFC 2822, there's no separating newline here diff --git a/lib-python/modified-2.7/email/test/data/msg_36.txt b/lib-python/modified-2.7/email/test/data/msg_36.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_36.txt +++ /dev/null @@ -1,40 +0,0 @@ -Mime-Version: 1.0 -Content-Type: Multipart/Mixed; Boundary="NextPart" -To: IETF-Announce:; -From: Internet-Drafts at ietf.org -Subject: I-D ACTION:draft-ietf-mboned-mix-00.txt -Date: Tue, 22 Dec 1998 16:55:06 -0500 - ---NextPart - -Blah blah blah - ---NextPart -Content-Type: Multipart/Alternative; Boundary="OtherAccess" - ---OtherAccess -Content-Type: Message/External-body; - access-type="mail-server"; - server="mailserv at ietf.org" - -Content-Type: text/plain -Content-ID: <19981222151406.I-D at ietf.org> - -ENCODING mime -FILE /internet-drafts/draft-ietf-mboned-mix-00.txt - ---OtherAccess -Content-Type: Message/External-body; - name="draft-ietf-mboned-mix-00.txt"; - site="ftp.ietf.org"; - access-type="anon-ftp"; - directory="internet-drafts" - -Content-Type: text/plain -Content-ID: <19981222151406.I-D at ietf.org> - - ---OtherAccess-- - ---NextPart-- - diff --git a/lib-python/modified-2.7/email/test/data/msg_37.txt b/lib-python/modified-2.7/email/test/data/msg_37.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_37.txt +++ /dev/null @@ -1,22 +0,0 @@ -Content-Type: multipart/mixed; boundary=ABCDE - ---ABCDE -Content-Type: text/x-one - -Blah - ---ABCDE ---ABCDE -Content-Type: text/x-two - -Blah - ---ABCDE ---ABCDE ---ABCDE ---ABCDE -Content-Type: text/x-two - -Blah - ---ABCDE-- diff --git a/lib-python/modified-2.7/email/test/data/msg_38.txt b/lib-python/modified-2.7/email/test/data/msg_38.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_38.txt +++ /dev/null @@ -1,101 +0,0 @@ -MIME-Version: 1.0 -Content-Type: multipart/mixed; boundary="----- =_aaaaaaaaaa0" - -------- =_aaaaaaaaaa0 -Content-Type: multipart/mixed; boundary="----- =_aaaaaaaaaa1" -Content-ID: <20592.1022586929.1 at example.com> - -------- =_aaaaaaaaaa1 -Content-Type: multipart/alternative; boundary="----- =_aaaaaaaaaa2" -Content-ID: <20592.1022586929.2 at example.com> - -------- =_aaaaaaaaaa2 -Content-Type: text/plain -Content-ID: <20592.1022586929.3 at example.com> -Content-Description: very tricky -Content-Transfer-Encoding: 7bit - - -Unlike the test test_nested-multiples-with-internal-boundary, this -piece of text not only contains the outer boundary tags -------- =_aaaaaaaaaa1 -and -------- =_aaaaaaaaaa0 -but puts them at the start of a line! And, to be even nastier, it -even includes a couple of end tags, such as this one: - -------- =_aaaaaaaaaa1-- - -and this one, which is from a multipart we haven't even seen yet! - -------- =_aaaaaaaaaa4-- - -This will, I'm sure, cause much breakage of MIME parsers. But, as -far as I can tell, it's perfectly legal. I have not yet ever seen -a case of this in the wild, but I've seen *similar* things. - - -------- =_aaaaaaaaaa2 -Content-Type: application/octet-stream -Content-ID: <20592.1022586929.4 at example.com> -Content-Description: patch2 -Content-Transfer-Encoding: base64 - -XXX - -------- =_aaaaaaaaaa2-- - -------- =_aaaaaaaaaa1 -Content-Type: multipart/alternative; boundary="----- =_aaaaaaaaaa3" -Content-ID: <20592.1022586929.6 at example.com> - -------- =_aaaaaaaaaa3 -Content-Type: application/octet-stream -Content-ID: <20592.1022586929.7 at example.com> -Content-Description: patch3 -Content-Transfer-Encoding: base64 - -XXX - -------- =_aaaaaaaaaa3 -Content-Type: application/octet-stream -Content-ID: <20592.1022586929.8 at example.com> -Content-Description: patch4 -Content-Transfer-Encoding: base64 - -XXX - -------- =_aaaaaaaaaa3-- - -------- =_aaaaaaaaaa1 -Content-Type: multipart/alternative; boundary="----- =_aaaaaaaaaa4" -Content-ID: <20592.1022586929.10 at example.com> - -------- =_aaaaaaaaaa4 -Content-Type: application/octet-stream -Content-ID: <20592.1022586929.11 at example.com> -Content-Description: patch5 -Content-Transfer-Encoding: base64 - -XXX - -------- =_aaaaaaaaaa4 -Content-Type: application/octet-stream -Content-ID: <20592.1022586929.12 at example.com> -Content-Description: patch6 -Content-Transfer-Encoding: base64 - -XXX - -------- =_aaaaaaaaaa4-- - -------- =_aaaaaaaaaa1-- - -------- =_aaaaaaaaaa0 -Content-Type: text/plain; charset="us-ascii" -Content-ID: <20592.1022586929.15 at example.com> - --- -It's never too late to have a happy childhood. - -------- =_aaaaaaaaaa0-- diff --git a/lib-python/modified-2.7/email/test/data/msg_39.txt b/lib-python/modified-2.7/email/test/data/msg_39.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_39.txt +++ /dev/null @@ -1,83 +0,0 @@ -MIME-Version: 1.0 -Content-Type: multipart/mixed; boundary="----- =_aaaaaaaaaa0" - -------- =_aaaaaaaaaa0 -Content-Type: multipart/mixed; boundary="----- =_aaaaaaaaaa1" -Content-ID: <20592.1022586929.1 at example.com> - -------- =_aaaaaaaaaa1 -Content-Type: multipart/alternative; boundary="----- =_aaaaaaaaaa1" -Content-ID: <20592.1022586929.2 at example.com> - -------- =_aaaaaaaaaa1 -Content-Type: application/octet-stream -Content-ID: <20592.1022586929.3 at example.com> -Content-Description: patch1 -Content-Transfer-Encoding: base64 - -XXX - -------- =_aaaaaaaaaa1 -Content-Type: application/octet-stream -Content-ID: <20592.1022586929.4 at example.com> -Content-Description: patch2 -Content-Transfer-Encoding: base64 - -XXX - -------- =_aaaaaaaaaa1-- - -------- =_aaaaaaaaaa1 -Content-Type: multipart/alternative; boundary="----- =_aaaaaaaaaa1" -Content-ID: <20592.1022586929.6 at example.com> - -------- =_aaaaaaaaaa1 -Content-Type: application/octet-stream -Content-ID: <20592.1022586929.7 at example.com> -Content-Description: patch3 -Content-Transfer-Encoding: base64 - -XXX - -------- =_aaaaaaaaaa1 -Content-Type: application/octet-stream -Content-ID: <20592.1022586929.8 at example.com> -Content-Description: patch4 -Content-Transfer-Encoding: base64 - -XXX - -------- =_aaaaaaaaaa1-- - -------- =_aaaaaaaaaa1 -Content-Type: multipart/alternative; boundary="----- =_aaaaaaaaaa1" -Content-ID: <20592.1022586929.10 at example.com> - -------- =_aaaaaaaaaa1 -Content-Type: application/octet-stream -Content-ID: <20592.1022586929.11 at example.com> -Content-Description: patch5 -Content-Transfer-Encoding: base64 - -XXX - -------- =_aaaaaaaaaa1 -Content-Type: application/octet-stream -Content-ID: <20592.1022586929.12 at example.com> -Content-Description: patch6 -Content-Transfer-Encoding: base64 - -XXX - -------- =_aaaaaaaaaa1-- - -------- =_aaaaaaaaaa1-- - -------- =_aaaaaaaaaa0 -Content-Type: text/plain; charset="us-ascii" -Content-ID: <20592.1022586929.15 at example.com> - --- -It's never too late to have a happy childhood. - -------- =_aaaaaaaaaa0-- diff --git a/lib-python/modified-2.7/email/test/data/msg_40.txt b/lib-python/modified-2.7/email/test/data/msg_40.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_40.txt +++ /dev/null @@ -1,10 +0,0 @@ -MIME-Version: 1.0 -Content-Type: text/html; boundary="--961284236552522269" - -----961284236552522269 -Content-Type: text/html; -Content-Transfer-Encoding: 7Bit - -<html></html> - -----961284236552522269-- diff --git a/lib-python/modified-2.7/email/test/data/msg_41.txt b/lib-python/modified-2.7/email/test/data/msg_41.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_41.txt +++ /dev/null @@ -1,8 +0,0 @@ -From: "Allison Dunlap" <xxx at example.com> -To: yyy at example.com -Subject: 64423 -Date: Sun, 11 Jul 2004 16:09:27 -0300 -MIME-Version: 1.0 -Content-Type: multipart/alternative; - -Blah blah blah diff --git a/lib-python/modified-2.7/email/test/data/msg_42.txt b/lib-python/modified-2.7/email/test/data/msg_42.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_42.txt +++ /dev/null @@ -1,20 +0,0 @@ -Content-Type: multipart/mixed; boundary="AAA" -From: Mail Delivery Subsystem <xxx at example.com> -To: yyy at example.com - -This is a MIME-encapsulated message - ---AAA - -Stuff - ---AAA -Content-Type: message/rfc822 - -From: webmaster at python.org -To: zzz at example.com -Content-Type: multipart/mixed; boundary="BBB" - ---BBB-- - ---AAA-- diff --git a/lib-python/modified-2.7/email/test/data/msg_43.txt b/lib-python/modified-2.7/email/test/data/msg_43.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_43.txt +++ /dev/null @@ -1,217 +0,0 @@ -From SRS0=aO/p=ON=bag.python.org=None at bounce2.pobox.com Fri Nov 26 21:40:36 2004 -X-VM-v5-Data: ([nil nil nil nil nil nil nil nil nil] - [nil nil nil nil nil nil nil "MAILER DAEMON <>" "MAILER DAEMON <>" nil nil "Banned file: auto__mail.python.bat in mail from you" "^From:" nil nil nil nil "Banned file: auto__mail.python.bat in mail from you" nil nil nil nil nil nil nil] - nil) -MIME-Version: 1.0 -Message-Id: <edab.7804f5cb8070 at python.org> -Content-Type: multipart/report; report-type=delivery-status; - charset=utf-8; - boundary="----------=_1101526904-1956-5" -X-Virus-Scanned: by XS4ALL Virus Scanner -X-UIDL: 4\G!!!<c"!UV["!M7C!! -From: MAILER DAEMON <> -To: <webmaster at python.org> -Subject: Banned file: auto__mail.python.bat in mail from you -Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -This is a multi-part message in MIME format... - -------------=_1101526904-1956-5 -Content-Type: text/plain; charset="utf-8" -Content-Disposition: inline -Content-Transfer-Encoding: 7bit - -BANNED FILENAME ALERT - -Your message to: xxxxxxx at dot.ca.gov, xxxxxxxxxxxxx at dot.ca.gov, xxxxxxxxxx at dot.ca.gov, xxxxxxxx at dot.ca.gov, xxxxxxxxxx at dot.ca.gov, xxxxxx at dot.ca.gov, xxxxxxxxxx at dot.ca.gov, xxxxxx at dot.ca.gov, xxxxxx at dot.ca.gov, xxxxxxxxxxxxxxxx at dot.ca.gov, xxxxxxxxxxx at dot.ca.gov, xxxxxxxxxx at dot.ca.gov, xxxxxxxxxx at dot.ca.gov, xxxxxxxxxxxx at dot.ca.gov, xxxxxxxxxxxx at dot.ca.gov, xxxxxxx at dot.ca.gov, xxxxxxxxx at dot.ca.gov, xxxxxxxxxx at dot.ca.gov, xxxxxx at dot.ca.gov, xxx at dot.ca.gov, xxxxxxx at dot.ca.gov, xxxxxxx at dot.ca.gov, xxxxxxxxxxxxxxx at dot.ca.gov, xxxxxxxxxx at dot.ca.gov, xxxxxxx at dot.ca.gov, xxx at dot.ca.gov, xxxxxxxx at dot.ca.gov, xxxxxxxxxxxxx at dot.ca.gov, xxxxxxxxxxxxx at dot.ca.gov, xxxxxxxxxxx at dot.ca.gov, xxxxxxxxx at dot.ca.gov, xxxxxxxxxx at dot.ca.gov, xxxxxxxxxxxx at dot.ca.gov, xxxxxxx at dot.ca.gov, xxxxxxxxxxxxxxx at dot.ca.gov, xxxxxxxxxxxxx at dot.ca.gov, xxxx at dot.ca.gov, xxxxxxxx at dot.ca.gov, xxxxxxxxxx at dot.ca.gov, xxxxxxxxxxxxxxxxxx at dot.ca.gov -was blocked by our Spam Firewall. The email you sent with the following subject has NOT BEEN DELIVERED: - -Subject: Delivery_failure_notice - -An attachment in that mail was of a file type that the Spam Firewall is set to block. - - - -------------=_1101526904-1956-5 -Content-Type: message/delivery-status -Content-Disposition: inline -Content-Transfer-Encoding: 7bit -Content-Description: Delivery error report - -Reporting-MTA: dns; sacspam01.dot.ca.gov -Received-From-MTA: smtp; sacspam01.dot.ca.gov ([127.0.0.1]) -Arrival-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxxxxxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxxxxxxxxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxxxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxxxxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxxxxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxxxxxxxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -Final-Recipient: rfc822; xxxxxxx at dot.ca.gov -Action: failed -Status: 5.7.1 -Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat -Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) - -------------=_1101526904-1956-5 -Content-Type: text/rfc822-headers -Content-Disposition: inline -Content-Transfer-Encoding: 7bit -Content-Description: Undelivered-message headers - -Received: from kgsav.org (ppp-70-242-162-63.dsl.spfdmo.swbell.net [70.242.162.63]) - by sacspam01.dot.ca.gov (Spam Firewall) with SMTP - id A232AD03DE3A; Fri, 26 Nov 2004 19:41:35 -0800 (PST) -From: webmaster at python.org -To: xxxxx at dot.ca.gov -Date: Sat, 27 Nov 2004 03:35:30 UTC -Subject: Delivery_failure_notice -Importance: Normal -X-Priority: 3 (Normal) -X-MSMail-Priority: Normal -Message-ID: <edab.7804f5cb8070 at python.org> -MIME-Version: 1.0 -Content-Type: multipart/mixed; boundary="====67bd2b7a5.f99f7" -Content-Transfer-Encoding: 7bit - -------------=_1101526904-1956-5-- - diff --git a/lib-python/modified-2.7/email/test/data/msg_44.txt b/lib-python/modified-2.7/email/test/data/msg_44.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_44.txt +++ /dev/null @@ -1,33 +0,0 @@ -Return-Path: <barry at python.org> -Delivered-To: barry at python.org -Received: by mail.python.org (Postfix, from userid 889) - id C2BF0D37C6; Tue, 11 Sep 2001 00:05:05 -0400 (EDT) -MIME-Version: 1.0 -Content-Type: multipart/mixed; boundary="h90VIIIKmx" -Content-Transfer-Encoding: 7bit -Message-ID: <15261.36209.358846.118674 at anthem.python.org> -From: barry at python.org (Barry A. Warsaw) -To: barry at python.org -Subject: a simple multipart -Date: Tue, 11 Sep 2001 00:05:05 -0400 -X-Mailer: VM 6.95 under 21.4 (patch 4) "Artificial Intelligence" XEmacs Lucid -X-Attribution: BAW -X-Oblique-Strategy: Make a door into a window - - ---h90VIIIKmx -Content-Type: text/plain; name="msg.txt" -Content-Transfer-Encoding: 7bit - -a simple kind of mirror -to reflect upon our own - ---h90VIIIKmx -Content-Type: text/plain; name="msg.txt" -Content-Transfer-Encoding: 7bit - -a simple kind of mirror -to reflect upon our own - ---h90VIIIKmx-- - diff --git a/lib-python/modified-2.7/email/test/data/msg_45.txt b/lib-python/modified-2.7/email/test/data/msg_45.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_45.txt +++ /dev/null @@ -1,33 +0,0 @@ -From: <foo at bar.baz> -To: <baz at bar.foo> -Subject: test -X-Long-Line: Some really long line contains a lot of text and thus has to be rewrapped because it is some - really long - line -MIME-Version: 1.0 -Content-Type: multipart/signed; boundary="borderline"; - protocol="application/pgp-signature"; micalg=pgp-sha1 - -This is an OpenPGP/MIME signed message (RFC 2440 and 3156) ---borderline -Content-Type: text/plain -X-Long-Line: Another really long line contains a lot of text and thus has to be rewrapped because it is another - really long - line - -This is the signed contents. - ---borderline -Content-Type: application/pgp-signature; name="signature.asc" -Content-Description: OpenPGP digital signature -Content-Disposition: attachment; filename="signature.asc" - ------BEGIN PGP SIGNATURE----- -Version: GnuPG v2.0.6 (GNU/Linux) - -iD8DBQFG03voRhp6o4m9dFsRApSZAKCCAN3IkJlVRg6NvAiMHlvvIuMGPQCeLZtj -FGwfnRHFBFO/S4/DKysm0lI= -=t7+s ------END PGP SIGNATURE----- - ---borderline-- diff --git a/lib-python/modified-2.7/email/test/data/msg_46.txt b/lib-python/modified-2.7/email/test/data/msg_46.txt deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/data/msg_46.txt +++ /dev/null @@ -1,23 +0,0 @@ -Return-Path: <sender at example.net> -Delivery-Date: Mon, 08 Feb 2010 14:05:16 +0100 -Received: from example.org (example.org [64.5.53.58]) - by example.net (node=mxbap2) with ESMTP (Nemesis) - id UNIQUE for someone at example.com; Mon, 08 Feb 2010 14:05:16 +0100 -Date: Mon, 01 Feb 2010 12:21:16 +0100 -From: "Sender" <sender at example.net> -To: <someone at example.com> -Subject: GroupwiseForwardingTest -Mime-Version: 1.0 -Content-Type: message/rfc822 - -Return-path: <sender at example.net> -Message-ID: <4B66B890.4070408 at teconcept.de> -Date: Mon, 01 Feb 2010 12:18:40 +0100 -From: "Dr. Sender" <sender at example.net> -MIME-Version: 1.0 -To: "Recipient" <recipient at example.com> -Subject: GroupwiseForwardingTest -Content-Type: text/plain; charset=ISO-8859-15 -Content-Transfer-Encoding: 7bit - -Testing email forwarding with Groupwise 1.2.2010 diff --git a/lib-python/modified-2.7/email/test/test_email.py b/lib-python/modified-2.7/email/test/test_email.py deleted file mode 100644 --- a/lib-python/modified-2.7/email/test/test_email.py +++ /dev/null @@ -1,3530 +0,0 @@ -# Copyright (C) 2001-2010 Python Software Foundation -# Contact: email-sig at python.org -# email package unit tests - -import os -import sys -import time -import base64 -import difflib -import unittest -import warnings -from cStringIO import StringIO - -import email - -from email.Charset import Charset -from email.Header import Header, decode_header, make_header -from email.Parser import Parser, HeaderParser -from email.Generator import Generator, DecodedGenerator -from email.Message import Message -from email.MIMEAudio import MIMEAudio -from email.MIMEText import MIMEText -from email.MIMEImage import MIMEImage -from email.MIMEBase import MIMEBase -from email.MIMEMessage import MIMEMessage -from email.MIMEMultipart import MIMEMultipart -from email import Utils -from email import Errors -from email import Encoders -from email import Iterators -from email import base64MIME -from email import quopriMIME - -from test.test_support import findfile, run_unittest, impl_detail -from email.test import __file__ as landmark - - -NL = '\n' -EMPTYSTRING = '' -SPACE = ' ' - - - -def openfile(filename, mode='r'): - path = os.path.join(os.path.dirname(landmark), 'data', filename) - return open(path, mode) - - - -# Base test class -class TestEmailBase(unittest.TestCase): - def ndiffAssertEqual(self, first, second): - """Like assertEqual except use ndiff for readable output.""" - if first != second: - sfirst = str(first) - ssecond = str(second) - diff = difflib.ndiff(sfirst.splitlines(), ssecond.splitlines()) - fp = StringIO() - print >> fp, NL, NL.join(diff) - raise self.failureException, fp.getvalue() - - def _msgobj(self, filename): - fp = openfile(findfile(filename)) - try: - msg = email.message_from_file(fp) - finally: - fp.close() - return msg - - - -# Test various aspects of the Message class's API -class TestMessageAPI(TestEmailBase): - def test_get_all(self): - eq = self.assertEqual - msg = self._msgobj('msg_20.txt') - eq(msg.get_all('cc'), ['ccc at zzz.org', 'ddd at zzz.org', 'eee at zzz.org']) - eq(msg.get_all('xx', 'n/a'), 'n/a') - - def test_getset_charset(self): - eq = self.assertEqual - msg = Message() - eq(msg.get_charset(), None) - charset = Charset('iso-8859-1') - msg.set_charset(charset) - eq(msg['mime-version'], '1.0') - eq(msg.get_content_type(), 'text/plain') - eq(msg['content-type'], 'text/plain; charset="iso-8859-1"') - eq(msg.get_param('charset'), 'iso-8859-1') - eq(msg['content-transfer-encoding'], 'quoted-printable') - eq(msg.get_charset().input_charset, 'iso-8859-1') - # Remove the charset - msg.set_charset(None) - eq(msg.get_charset(), None) - eq(msg['content-type'], 'text/plain') - # Try adding a charset when there's already MIME headers present - msg = Message() - msg['MIME-Version'] = '2.0' - msg['Content-Type'] = 'text/x-weird' - msg['Content-Transfer-Encoding'] = 'quinted-puntable' - msg.set_charset(charset) - eq(msg['mime-version'], '2.0') - eq(msg['content-type'], 'text/x-weird; charset="iso-8859-1"') - eq(msg['content-transfer-encoding'], 'quinted-puntable') - - def test_set_charset_from_string(self): - eq = self.assertEqual - msg = Message() - msg.set_charset('us-ascii') - eq(msg.get_charset().input_charset, 'us-ascii') - eq(msg['content-type'], 'text/plain; charset="us-ascii"') - - def test_set_payload_with_charset(self): - msg = Message() - charset = Charset('iso-8859-1') - msg.set_payload('This is a string payload', charset) - self.assertEqual(msg.get_charset().input_charset, 'iso-8859-1') - - def test_get_charsets(self): - eq = self.assertEqual - - msg = self._msgobj('msg_08.txt') - charsets = msg.get_charsets() - eq(charsets, [None, 'us-ascii', 'iso-8859-1', 'iso-8859-2', 'koi8-r']) - - msg = self._msgobj('msg_09.txt') - charsets = msg.get_charsets('dingbat') - eq(charsets, ['dingbat', 'us-ascii', 'iso-8859-1', 'dingbat', - 'koi8-r']) - - msg = self._msgobj('msg_12.txt') - charsets = msg.get_charsets() - eq(charsets, [None, 'us-ascii', 'iso-8859-1', None, 'iso-8859-2', - 'iso-8859-3', 'us-ascii', 'koi8-r']) - - def test_get_filename(self): - eq = self.assertEqual - - msg = self._msgobj('msg_04.txt') - filenames = [p.get_filename() for p in msg.get_payload()] - eq(filenames, ['msg.txt', 'msg.txt']) - - msg = self._msgobj('msg_07.txt') - subpart = msg.get_payload(1) - eq(subpart.get_filename(), 'dingusfish.gif') - - def test_get_filename_with_name_parameter(self): - eq = self.assertEqual - - msg = self._msgobj('msg_44.txt') - filenames = [p.get_filename() for p in msg.get_payload()] - eq(filenames, ['msg.txt', 'msg.txt']) - - def test_get_boundary(self): - eq = self.assertEqual - msg = self._msgobj('msg_07.txt') - # No quotes! - eq(msg.get_boundary(), 'BOUNDARY') - - def test_set_boundary(self): - eq = self.assertEqual - # This one has no existing boundary parameter, but the Content-Type: - # header appears fifth. - msg = self._msgobj('msg_01.txt') - msg.set_boundary('BOUNDARY') - header, value = msg.items()[4] - eq(header.lower(), 'content-type') - eq(value, 'text/plain; charset="us-ascii"; boundary="BOUNDARY"') - # This one has a Content-Type: header, with a boundary, stuck in the - # middle of its headers. Make sure the order is preserved; it should - # be fifth. - msg = self._msgobj('msg_04.txt') - msg.set_boundary('BOUNDARY') - header, value = msg.items()[4] - eq(header.lower(), 'content-type') - eq(value, 'multipart/mixed; boundary="BOUNDARY"') - # And this one has no Content-Type: header at all. - msg = self._msgobj('msg_03.txt') - self.assertRaises(Errors.HeaderParseError, - msg.set_boundary, 'BOUNDARY') - - def test_make_boundary(self): - msg = MIMEMultipart('form-data') - # Note that when the boundary gets created is an implementation - # detail and might change. - self.assertEqual(msg.items()[0][1], 'multipart/form-data') - # Trigger creation of boundary - msg.as_string() - self.assertEqual(msg.items()[0][1][:33], - 'multipart/form-data; boundary="==') - # XXX: there ought to be tests of the uniqueness of the boundary, too. - - def test_message_rfc822_only(self): - # Issue 7970: message/rfc822 not in multipart parsed by - # HeaderParser caused an exception when flattened. - fp = openfile(findfile('msg_46.txt')) - msgdata = fp.read() - parser = email.Parser.HeaderParser() - msg = parser.parsestr(msgdata) - out = StringIO() - gen = email.Generator.Generator(out, True, 0) - gen.flatten(msg, False) - self.assertEqual(out.getvalue(), msgdata) - - def test_get_decoded_payload(self): - eq = self.assertEqual - msg = self._msgobj('msg_10.txt') - # The outer message is a multipart - eq(msg.get_payload(decode=True), None) - # Subpart 1 is 7bit encoded - eq(msg.get_payload(0).get_payload(decode=True), - 'This is a 7bit encoded message.\n') - # Subpart 2 is quopri - eq(msg.get_payload(1).get_payload(decode=True), - '\xa1This is a Quoted Printable encoded message!\n') - # Subpart 3 is base64 - eq(msg.get_payload(2).get_payload(decode=True), - 'This is a Base64 encoded message.') - # Subpart 4 is base64 with a trailing newline, which - # used to be stripped (issue 7143). - eq(msg.get_payload(3).get_payload(decode=True), - 'This is a Base64 encoded message.\n') - # Subpart 5 has no Content-Transfer-Encoding: header. - eq(msg.get_payload(4).get_payload(decode=True), - 'This has no Content-Transfer-Encoding: header.\n') - - def test_get_decoded_uu_payload(self): - eq = self.assertEqual - msg = Message() - msg.set_payload('begin 666 -\n+:&5L;&\\@=V]R;&0 \n \nend\n') - for cte in ('x-uuencode', 'uuencode', 'uue', 'x-uue'): - msg['content-transfer-encoding'] = cte - eq(msg.get_payload(decode=True), 'hello world') - # Now try some bogus data - msg.set_payload('foo') - eq(msg.get_payload(decode=True), 'foo') - - def test_decode_bogus_uu_payload_quietly(self): - msg = Message() - msg.set_payload('begin 664 foo.txt\n%<W1F=0000H \n \nend\n') - msg['Content-Transfer-Encoding'] = 'x-uuencode' - old_stderr = sys.stderr - try: - sys.stderr = sfp = StringIO() - # We don't care about the payload - msg.get_payload(decode=True) - finally: - sys.stderr = old_stderr - self.assertEqual(sfp.getvalue(), '') - - def test_decoded_generator(self): - eq = self.assertEqual - msg = self._msgobj('msg_07.txt') - fp = openfile('msg_17.txt') - try: - text = fp.read() - finally: - fp.close() - s = StringIO() - g = DecodedGenerator(s) - g.flatten(msg) - eq(s.getvalue(), text) - - def test__contains__(self): - msg = Message() - msg['From'] = 'Me' - msg['to'] = 'You' - # Check for case insensitivity - self.assertTrue('from' in msg) - self.assertTrue('From' in msg) - self.assertTrue('FROM' in msg) - self.assertTrue('to' in msg) - self.assertTrue('To' in msg) - self.assertTrue('TO' in msg) - - def test_as_string(self): - eq = self.assertEqual - msg = self._msgobj('msg_01.txt') - fp = openfile('msg_01.txt') - try: - # BAW 30-Mar-2009 Evil be here. So, the generator is broken with - # respect to long line breaking. It's also not idempotent when a - # header from a parsed message is continued with tabs rather than - # spaces. Before we fixed bug 1974 it was reversedly broken, - # i.e. headers that were continued with spaces got continued with - # tabs. For Python 2.x there's really no good fix and in Python - # 3.x all this stuff is re-written to be right(er). Chris Withers - # convinced me that using space as the default continuation - # character is less bad for more applications. - text = fp.read().replace('\t', ' ') - finally: - fp.close() - eq(text, msg.as_string()) - fullrepr = str(msg) - lines = fullrepr.split('\n') - self.assertTrue(lines[0].startswith('From ')) - eq(text, NL.join(lines[1:])) - - def test_bad_param(self): - msg = email.message_from_string("Content-Type: blarg; baz; boo\n") - self.assertEqual(msg.get_param('baz'), '') - - def test_missing_filename(self): - msg = email.message_from_string("From: foo\n") - self.assertEqual(msg.get_filename(), None) - - def test_bogus_filename(self): - msg = email.message_from_string( - "Content-Disposition: blarg; filename\n") - self.assertEqual(msg.get_filename(), '') - - def test_missing_boundary(self): - msg = email.message_from_string("From: foo\n") - self.assertEqual(msg.get_boundary(), None) - - def test_get_params(self): - eq = self.assertEqual - msg = email.message_from_string( - 'X-Header: foo=one; bar=two; baz=three\n') - eq(msg.get_params(header='x-header'), - [('foo', 'one'), ('bar', 'two'), ('baz', 'three')]) - msg = email.message_from_string( - 'X-Header: foo; bar=one; baz=two\n') - eq(msg.get_params(header='x-header'), - [('foo', ''), ('bar', 'one'), ('baz', 'two')]) - eq(msg.get_params(), None) - msg = email.message_from_string( - 'X-Header: foo; bar="one"; baz=two\n') - eq(msg.get_params(header='x-header'), - [('foo', ''), ('bar', 'one'), ('baz', 'two')]) - - def test_get_param_liberal(self): - msg = Message() - msg['Content-Type'] = 'Content-Type: Multipart/mixed; boundary = "CPIMSSMTPC06p5f3tG"' - self.assertEqual(msg.get_param('boundary'), 'CPIMSSMTPC06p5f3tG') - - def test_get_param(self): - eq = self.assertEqual - msg = email.message_from_string( - "X-Header: foo=one; bar=two; baz=three\n") - eq(msg.get_param('bar', header='x-header'), 'two') - eq(msg.get_param('quuz', header='x-header'), None) - eq(msg.get_param('quuz'), None) - msg = email.message_from_string( - 'X-Header: foo; bar="one"; baz=two\n') - eq(msg.get_param('foo', header='x-header'), '') - eq(msg.get_param('bar', header='x-header'), 'one') - eq(msg.get_param('baz', header='x-header'), 'two') - # XXX: We are not RFC-2045 compliant! We cannot parse: - # msg["Content-Type"] = 'text/plain; weird="hey; dolly? [you] @ <\\"home\\">?"' - # msg.get_param("weird") - # yet. - - def test_get_param_funky_continuation_lines(self): - msg = self._msgobj('msg_22.txt') - self.assertEqual(msg.get_payload(1).get_param('name'), 'wibble.JPG') - - def test_get_param_with_semis_in_quotes(self): - msg = email.message_from_string( - 'Content-Type: image/pjpeg; name="Jim&&Jill"\n') - self.assertEqual(msg.get_param('name'), 'Jim&&Jill') - self.assertEqual(msg.get_param('name', unquote=False), - '"Jim&&Jill"') - - def test_get_param_with_quotes(self): - msg = email.message_from_string( - 'Content-Type: foo; bar*0="baz\\"foobar"; bar*1="\\"baz"') - self.assertEqual(msg.get_param('bar'), 'baz"foobar"baz') - msg = email.message_from_string( - "Content-Type: foo; bar*0=\"baz\\\"foobar\"; bar*1=\"\\\"baz\"") - self.assertEqual(msg.get_param('bar'), 'baz"foobar"baz') - - def test_has_key(self): - msg = email.message_from_string('Header: exists') - self.assertTrue(msg.has_key('header')) - self.assertTrue(msg.has_key('Header')) - self.assertTrue(msg.has_key('HEADER')) - self.assertFalse(msg.has_key('headeri')) - - def test_set_param(self): - eq = self.assertEqual - msg = Message() - msg.set_param('charset', 'iso-2022-jp') - eq(msg.get_param('charset'), 'iso-2022-jp') - msg.set_param('importance', 'high value') - eq(msg.get_param('importance'), 'high value') - eq(msg.get_param('importance', unquote=False), '"high value"') - eq(msg.get_params(), [('text/plain', ''), - ('charset', 'iso-2022-jp'), - ('importance', 'high value')]) - eq(msg.get_params(unquote=False), [('text/plain', ''), - ('charset', '"iso-2022-jp"'), - ('importance', '"high value"')]) - msg.set_param('charset', 'iso-9999-xx', header='X-Jimmy') - eq(msg.get_param('charset', header='X-Jimmy'), 'iso-9999-xx') - - def test_del_param(self): - eq = self.assertEqual - msg = self._msgobj('msg_05.txt') - eq(msg.get_params(), - [('multipart/report', ''), ('report-type', 'delivery-status'), - ('boundary', 'D1690A7AC1.996856090/mail.example.com')]) - old_val = msg.get_param("report-type") - msg.del_param("report-type") - eq(msg.get_params(), - [('multipart/report', ''), - ('boundary', 'D1690A7AC1.996856090/mail.example.com')]) - msg.set_param("report-type", old_val) - eq(msg.get_params(), - [('multipart/report', ''), - ('boundary', 'D1690A7AC1.996856090/mail.example.com'), - ('report-type', old_val)]) - - def test_del_param_on_other_header(self): - msg = Message() - msg.add_header('Content-Disposition', 'attachment', filename='bud.gif') - msg.del_param('filename', 'content-disposition') - self.assertEqual(msg['content-disposition'], 'attachment') - - def test_set_type(self): - eq = self.assertEqual - msg = Message() - self.assertRaises(ValueError, msg.set_type, 'text') - msg.set_type('text/plain') - eq(msg['content-type'], 'text/plain') - msg.set_param('charset', 'us-ascii') - eq(msg['content-type'], 'text/plain; charset="us-ascii"') - msg.set_type('text/html') - eq(msg['content-type'], 'text/html; charset="us-ascii"') - - def test_set_type_on_other_header(self): - msg = Message() - msg['X-Content-Type'] = 'text/plain' - msg.set_type('application/octet-stream', 'X-Content-Type') - self.assertEqual(msg['x-content-type'], 'application/octet-stream') - - def test_get_content_type_missing(self): - msg = Message() - self.assertEqual(msg.get_content_type(), 'text/plain') - - def test_get_content_type_missing_with_default_type(self): - msg = Message() - msg.set_default_type('message/rfc822') - self.assertEqual(msg.get_content_type(), 'message/rfc822') - - def test_get_content_type_from_message_implicit(self): - msg = self._msgobj('msg_30.txt') - self.assertEqual(msg.get_payload(0).get_content_type(), - 'message/rfc822') - - def test_get_content_type_from_message_explicit(self): - msg = self._msgobj('msg_28.txt') - self.assertEqual(msg.get_payload(0).get_content_type(), - 'message/rfc822') - - def test_get_content_type_from_message_text_plain_implicit(self): - msg = self._msgobj('msg_03.txt') - self.assertEqual(msg.get_content_type(), 'text/plain') - - def test_get_content_type_from_message_text_plain_explicit(self): - msg = self._msgobj('msg_01.txt') - self.assertEqual(msg.get_content_type(), 'text/plain') - - def test_get_content_maintype_missing(self): - msg = Message() - self.assertEqual(msg.get_content_maintype(), 'text') - - def test_get_content_maintype_missing_with_default_type(self): - msg = Message() - msg.set_default_type('message/rfc822') - self.assertEqual(msg.get_content_maintype(), 'message') - - def test_get_content_maintype_from_message_implicit(self): - msg = self._msgobj('msg_30.txt') - self.assertEqual(msg.get_payload(0).get_content_maintype(), 'message') - - def test_get_content_maintype_from_message_explicit(self): - msg = self._msgobj('msg_28.txt') - self.assertEqual(msg.get_payload(0).get_content_maintype(), 'message') - - def test_get_content_maintype_from_message_text_plain_implicit(self): - msg = self._msgobj('msg_03.txt') - self.assertEqual(msg.get_content_maintype(), 'text') - - def test_get_content_maintype_from_message_text_plain_explicit(self): - msg = self._msgobj('msg_01.txt') - self.assertEqual(msg.get_content_maintype(), 'text') - - def test_get_content_subtype_missing(self): - msg = Message() - self.assertEqual(msg.get_content_subtype(), 'plain') - - def test_get_content_subtype_missing_with_default_type(self): - msg = Message() - msg.set_default_type('message/rfc822') - self.assertEqual(msg.get_content_subtype(), 'rfc822') - - def test_get_content_subtype_from_message_implicit(self): - msg = self._msgobj('msg_30.txt') - self.assertEqual(msg.get_payload(0).get_content_subtype(), 'rfc822') - - def test_get_content_subtype_from_message_explicit(self): - msg = self._msgobj('msg_28.txt') - self.assertEqual(msg.get_payload(0).get_content_subtype(), 'rfc822') - - def test_get_content_subtype_from_message_text_plain_implicit(self): - msg = self._msgobj('msg_03.txt') - self.assertEqual(msg.get_content_subtype(), 'plain') - - def test_get_content_subtype_from_message_text_plain_explicit(self): - msg = self._msgobj('msg_01.txt') - self.assertEqual(msg.get_content_subtype(), 'plain') - - def test_get_content_maintype_error(self): - msg = Message() - msg['Content-Type'] = 'no-slash-in-this-string' - self.assertEqual(msg.get_content_maintype(), 'text') - - def test_get_content_subtype_error(self): - msg = Message() - msg['Content-Type'] = 'no-slash-in-this-string' - self.assertEqual(msg.get_content_subtype(), 'plain') - - def test_replace_header(self): - eq = self.assertEqual - msg = Message() - msg.add_header('First', 'One') - msg.add_header('Second', 'Two') - msg.add_header('Third', 'Three') - eq(msg.keys(), ['First', 'Second', 'Third']) - eq(msg.values(), ['One', 'Two', 'Three']) - msg.replace_header('Second', 'Twenty') - eq(msg.keys(), ['First', 'Second', 'Third']) - eq(msg.values(), ['One', 'Twenty', 'Three']) - msg.add_header('First', 'Eleven') - msg.replace_header('First', 'One Hundred') - eq(msg.keys(), ['First', 'Second', 'Third', 'First']) - eq(msg.values(), ['One Hundred', 'Twenty', 'Three', 'Eleven']) - self.assertRaises(KeyError, msg.replace_header, 'Fourth', 'Missing') - - def test_broken_base64_payload(self): - x = 'AwDp0P7//y6LwKEAcPa/6Q=9' - msg = Message() - msg['content-type'] = 'audio/x-midi' - msg['content-transfer-encoding'] = 'base64' - msg.set_payload(x) - self.assertEqual(msg.get_payload(decode=True), x) - - def test_get_content_charset(self): - msg = Message() - msg.set_charset('us-ascii') - self.assertEqual('us-ascii', msg.get_content_charset()) - msg.set_charset(u'us-ascii') - self.assertEqual('us-ascii', msg.get_content_charset()) - - # Issue 5871: reject an attempt to embed a header inside a header value - # (header injection attack). - def test_embeded_header_via_Header_rejected(self): - msg = Message() - msg['Dummy'] = Header('dummy\nX-Injected-Header: test') - self.assertRaises(Errors.HeaderParseError, msg.as_string) - - def test_embeded_header_via_string_rejected(self): - msg = Message() - msg['Dummy'] = 'dummy\nX-Injected-Header: test' - self.assertRaises(Errors.HeaderParseError, msg.as_string) - - -# Test the email.Encoders module -class TestEncoders(unittest.TestCase): - def test_encode_empty_payload(self): - eq = self.assertEqual - msg = Message() - msg.set_charset('us-ascii') - eq(msg['content-transfer-encoding'], '7bit') - - def test_default_cte(self): - eq = self.assertEqual - # 7bit data and the default us-ascii _charset - msg = MIMEText('hello world') - eq(msg['content-transfer-encoding'], '7bit') - # Similar, but with 8bit data - msg = MIMEText('hello \xf8 world') - eq(msg['content-transfer-encoding'], '8bit') - # And now with a different charset - msg = MIMEText('hello \xf8 world', _charset='iso-8859-1') - eq(msg['content-transfer-encoding'], 'quoted-printable') - - @impl_detail("PyPy has no cjkc codec yet", pypy=False) - def test_encode7or8bit(self): - # Make sure a charset whose input character set is 8bit but - # whose output character set is 7bit gets a transfer-encoding - # of 7bit. - eq = self.assertEqual - msg = email.MIMEText.MIMEText('\xca\xb8', _charset='euc-jp') - eq(msg['content-transfer-encoding'], '7bit') - - -# Test long header wrapping -class TestLongHeaders(TestEmailBase): - def test_split_long_continuation(self): - eq = self.ndiffAssertEqual - msg = email.message_from_string("""\ -Subject: bug demonstration -\t12345678911234567892123456789312345678941234567895123456789612345678971234567898112345678911234567892123456789112345678911234567892123456789 -\tmore text - -test -""") - sfp = StringIO() - g = Generator(sfp) - g.flatten(msg) - eq(sfp.getvalue(), """\ -Subject: bug demonstration - 12345678911234567892123456789312345678941234567895123456789612345678971234567898112345678911234567892123456789112345678911234567892123456789 - more text - -test -""") - - def test_another_long_almost_unsplittable_header(self): - eq = self.ndiffAssertEqual - hstr = """\ -bug demonstration -\t12345678911234567892123456789312345678941234567895123456789612345678971234567898112345678911234567892123456789112345678911234567892123456789 -\tmore text""" - h = Header(hstr, continuation_ws='\t') - eq(h.encode(), """\ -bug demonstration -\t12345678911234567892123456789312345678941234567895123456789612345678971234567898112345678911234567892123456789112345678911234567892123456789 -\tmore text""") - h = Header(hstr) - eq(h.encode(), """\ -bug demonstration - 12345678911234567892123456789312345678941234567895123456789612345678971234567898112345678911234567892123456789112345678911234567892123456789 - more text""") - - def test_long_nonstring(self): - eq = self.ndiffAssertEqual - g = Charset("iso-8859-1") - cz = Charset("iso-8859-2") - utf8 = Charset("utf-8") - g_head = "Die Mieter treten hier ein werden mit einem Foerderband komfortabel den Korridor entlang, an s\xfcdl\xfcndischen Wandgem\xe4lden vorbei, gegen die rotierenden Klingen bef\xf6rdert. " - cz_head = "Finan\xe8ni metropole se hroutily pod tlakem jejich d\xf9vtipu.. " - utf8_head = u"\u6b63\u78ba\u306b\u8a00\u3046\u3068\u7ffb\u8a33\u306f\u3055\u308c\u3066\u3044\u307e\u305b\u3093\u3002\u4e00\u90e8\u306f\u30c9\u30a4\u30c4\u8a9e\u3067\u3059\u304c\u3001\u3042\u3068\u306f\u3067\u305f\u3089\u3081\u3067\u3059\u3002\u5b9f\u969b\u306b\u306f\u300cWenn ist das Nunstuck git und Slotermeyer? Ja! Beiherhund das Oder die Flipperwaldt gersput.\u300d\u3068\u8a00\u3063\u3066\u3044\u307e\u3059\u3002".encode("utf-8") - h = Header(g_head, g, header_name='Subject') - h.append(cz_head, cz) - h.append(utf8_head, utf8) - msg = Message() - msg['Subject'] = h - sfp = StringIO() - g = Generator(sfp) - g.flatten(msg) - eq(sfp.getvalue(), """\ -Subject: =?iso-8859-1?q?Die_Mieter_treten_hier_ein_werden_mit_einem_Foerd?= - =?iso-8859-1?q?erband_komfortabel_den_Korridor_entlang=2C_an_s=FCdl=FCndi?= - =?iso-8859-1?q?schen_Wandgem=E4lden_vorbei=2C_gegen_die_rotierenden_Kling?= - =?iso-8859-1?q?en_bef=F6rdert=2E_?= =?iso-8859-2?q?Finan=E8ni_met?= - =?iso-8859-2?q?ropole_se_hroutily_pod_tlakem_jejich_d=F9vtipu=2E=2E_?= - =?utf-8?b?5q2j56K644Gr6KiA44GG44Go57+76Kiz44Gv44GV44KM44Gm44GE?= - =?utf-8?b?44G+44Gb44KT44CC5LiA6YOo44Gv44OJ44Kk44OE6Kqe44Gn44GZ44GM44CB?= - =?utf-8?b?44GC44Go44Gv44Gn44Gf44KJ44KB44Gn44GZ44CC5a6f6Zqb44Gr44Gv44CM?= - =?utf-8?q?Wenn_ist_das_Nunstuck_git_und_Slotermeyer=3F_Ja!_Beiherhund_das?= - =?utf-8?b?IE9kZXIgZGllIEZsaXBwZXJ3YWxkdCBnZXJzcHV0LuOAjeOBqOiogOOBow==?= - =?utf-8?b?44Gm44GE44G+44GZ44CC?= - -""") - eq(h.encode(), """\ -=?iso-8859-1?q?Die_Mieter_treten_hier_ein_werden_mit_einem_Foerd?= - =?iso-8859-1?q?erband_komfortabel_den_Korridor_entlang=2C_an_s=FCdl=FCndi?= - =?iso-8859-1?q?schen_Wandgem=E4lden_vorbei=2C_gegen_die_rotierenden_Kling?= - =?iso-8859-1?q?en_bef=F6rdert=2E_?= =?iso-8859-2?q?Finan=E8ni_met?= - =?iso-8859-2?q?ropole_se_hroutily_pod_tlakem_jejich_d=F9vtipu=2E=2E_?= - =?utf-8?b?5q2j56K644Gr6KiA44GG44Go57+76Kiz44Gv44GV44KM44Gm44GE?= - =?utf-8?b?44G+44Gb44KT44CC5LiA6YOo44Gv44OJ44Kk44OE6Kqe44Gn44GZ44GM44CB?= - =?utf-8?b?44GC44Go44Gv44Gn44Gf44KJ44KB44Gn44GZ44CC5a6f6Zqb44Gr44Gv44CM?= - =?utf-8?q?Wenn_ist_das_Nunstuck_git_und_Slotermeyer=3F_Ja!_Beiherhund_das?= - =?utf-8?b?IE9kZXIgZGllIEZsaXBwZXJ3YWxkdCBnZXJzcHV0LuOAjeOBqOiogOOBow==?= - =?utf-8?b?44Gm44GE44G+44GZ44CC?=""") - - def test_long_header_encode(self): - eq = self.ndiffAssertEqual - h = Header('wasnipoop; giraffes="very-long-necked-animals"; ' - 'spooge="yummy"; hippos="gargantuan"; marshmallows="gooey"', - header_name='X-Foobar-Spoink-Defrobnit') - eq(h.encode(), '''\ -wasnipoop; giraffes="very-long-necked-animals"; - spooge="yummy"; hippos="gargantuan"; marshmallows="gooey"''') - - def test_long_header_encode_with_tab_continuation(self): - eq = self.ndiffAssertEqual - h = Header('wasnipoop; giraffes="very-long-necked-animals"; ' - 'spooge="yummy"; hippos="gargantuan"; marshmallows="gooey"', - header_name='X-Foobar-Spoink-Defrobnit', - continuation_ws='\t') - eq(h.encode(), '''\ -wasnipoop; giraffes="very-long-necked-animals"; -\tspooge="yummy"; hippos="gargantuan"; marshmallows="gooey"''') - - def test_header_splitter(self): - eq = self.ndiffAssertEqual - msg = MIMEText('') - # It'd be great if we could use add_header() here, but that doesn't - # guarantee an order of the parameters. - msg['X-Foobar-Spoink-Defrobnit'] = ( - 'wasnipoop; giraffes="very-long-necked-animals"; ' - 'spooge="yummy"; hippos="gargantuan"; marshmallows="gooey"') - sfp = StringIO() - g = Generator(sfp) - g.flatten(msg) - eq(sfp.getvalue(), '''\ -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit -X-Foobar-Spoink-Defrobnit: wasnipoop; giraffes="very-long-necked-animals"; - spooge="yummy"; hippos="gargantuan"; marshmallows="gooey" - -''') - - def test_no_semis_header_splitter(self): - eq = self.ndiffAssertEqual - msg = Message() - msg['From'] = 'test at dom.ain' - msg['References'] = SPACE.join(['<%d at dom.ain>' % i for i in range(10)]) - msg.set_payload('Test') - sfp = StringIO() - g = Generator(sfp) - g.flatten(msg) - eq(sfp.getvalue(), """\ -From: test at dom.ain -References: <0 at dom.ain> <1 at dom.ain> <2 at dom.ain> <3 at dom.ain> <4 at dom.ain> - <5 at dom.ain> <6 at dom.ain> <7 at dom.ain> <8 at dom.ain> <9 at dom.ain> - -Test""") - - def test_no_split_long_header(self): - eq = self.ndiffAssertEqual - hstr = 'References: ' + 'x' * 80 - h = Header(hstr, continuation_ws='\t') - eq(h.encode(), """\ -References: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx""") - - def test_splitting_multiple_long_lines(self): - eq = self.ndiffAssertEqual - hstr = """\ -from babylon.socal-raves.org (localhost [127.0.0.1]); by babylon.socal-raves.org (Postfix) with ESMTP id B570E51B81; for <mailman-admin at babylon.socal-raves.org>; Sat, 2 Feb 2002 17:00:06 -0800 (PST) -\tfrom babylon.socal-raves.org (localhost [127.0.0.1]); by babylon.socal-raves.org (Postfix) with ESMTP id B570E51B81; for <mailman-admin at babylon.socal-raves.org>; Sat, 2 Feb 2002 17:00:06 -0800 (PST) -\tfrom babylon.socal-raves.org (localhost [127.0.0.1]); by babylon.socal-raves.org (Postfix) with ESMTP id B570E51B81; for <mailman-admin at babylon.socal-raves.org>; Sat, 2 Feb 2002 17:00:06 -0800 (PST) -""" - h = Header(hstr, continuation_ws='\t') - eq(h.encode(), """\ -from babylon.socal-raves.org (localhost [127.0.0.1]); -\tby babylon.socal-raves.org (Postfix) with ESMTP id B570E51B81; -\tfor <mailman-admin at babylon.socal-raves.org>; -\tSat, 2 Feb 2002 17:00:06 -0800 (PST) -\tfrom babylon.socal-raves.org (localhost [127.0.0.1]); -\tby babylon.socal-raves.org (Postfix) with ESMTP id B570E51B81; -\tfor <mailman-admin at babylon.socal-raves.org>; -\tSat, 2 Feb 2002 17:00:06 -0800 (PST) -\tfrom babylon.socal-raves.org (localhost [127.0.0.1]); -\tby babylon.socal-raves.org (Postfix) with ESMTP id B570E51B81; -\tfor <mailman-admin at babylon.socal-raves.org>; -\tSat, 2 Feb 2002 17:00:06 -0800 (PST)""") - - def test_splitting_first_line_only_is_long(self): - eq = self.ndiffAssertEqual - hstr = """\ -from modemcable093.139-201-24.que.mc.videotron.ca ([24.201.139.93] helo=cthulhu.gerg.ca) -\tby kronos.mems-exchange.org with esmtp (Exim 4.05) -\tid 17k4h5-00034i-00 -\tfor test at mems-exchange.org; Wed, 28 Aug 2002 11:25:20 -0400""" - h = Header(hstr, maxlinelen=78, header_name='Received', - continuation_ws='\t') - eq(h.encode(), """\ -from modemcable093.139-201-24.que.mc.videotron.ca ([24.201.139.93] -\thelo=cthulhu.gerg.ca) -\tby kronos.mems-exchange.org with esmtp (Exim 4.05) -\tid 17k4h5-00034i-00 -\tfor test at mems-exchange.org; Wed, 28 Aug 2002 11:25:20 -0400""") - - def test_long_8bit_header(self): - eq = self.ndiffAssertEqual - msg = Message() - h = Header('Britische Regierung gibt', 'iso-8859-1', - header_name='Subject') - h.append('gr\xfcnes Licht f\xfcr Offshore-Windkraftprojekte') - msg['Subject'] = h - eq(msg.as_string(), """\ -Subject: =?iso-8859-1?q?Britische_Regierung_gibt?= =?iso-8859-1?q?gr=FCnes?= - =?iso-8859-1?q?_Licht_f=FCr_Offshore-Windkraftprojekte?= - -""") - - def test_long_8bit_header_no_charset(self): - eq = self.ndiffAssertEqual - msg = Message() - msg['Reply-To'] = 'Britische Regierung gibt gr\xfcnes Licht f\xfcr Offshore-Windkraftprojekte <a-very-long-address at example.com>' - eq(msg.as_string(), """\ -Reply-To: Britische Regierung gibt gr\xfcnes Licht f\xfcr Offshore-Windkraftprojekte <a-very-long-address at example.com> - -""") - - def test_long_to_header(self): - eq = self.ndiffAssertEqual - to = '"Someone Test #A" <someone at eecs.umich.edu>,<someone at eecs.umich.edu>,"Someone Test #B" <someone at umich.edu>, "Someone Test #C" <someone at eecs.umich.edu>, "Someone Test #D" <someone at eecs.umich.edu>' - msg = Message() - msg['To'] = to - eq(msg.as_string(0), '''\ -To: "Someone Test #A" <someone at eecs.umich.edu>, <someone at eecs.umich.edu>, - "Someone Test #B" <someone at umich.edu>, - "Someone Test #C" <someone at eecs.umich.edu>, - "Someone Test #D" <someone at eecs.umich.edu> - -''') - - def test_long_line_after_append(self): - eq = self.ndiffAssertEqual - s = 'This is an example of string which has almost the limit of header length.' - h = Header(s) - h.append('Add another line.') - eq(h.encode(), """\ -This is an example of string which has almost the limit of header length. - Add another line.""") - - def test_shorter_line_with_append(self): - eq = self.ndiffAssertEqual - s = 'This is a shorter line.' - h = Header(s) - h.append('Add another sentence. (Surprise?)') - eq(h.encode(), - 'This is a shorter line. Add another sentence. (Surprise?)') - - def test_long_field_name(self): - eq = self.ndiffAssertEqual - fn = 'X-Very-Very-Very-Long-Header-Name' - gs = "Die Mieter treten hier ein werden mit einem Foerderband komfortabel den Korridor entlang, an s\xfcdl\xfcndischen Wandgem\xe4lden vorbei, gegen die rotierenden Klingen bef\xf6rdert. " - h = Header(gs, 'iso-8859-1', header_name=fn) - # BAW: this seems broken because the first line is too long - eq(h.encode(), """\ -=?iso-8859-1?q?Die_Mieter_treten_hier_?= - =?iso-8859-1?q?ein_werden_mit_einem_Foerderband_komfortabel_den_Korridor_?= - =?iso-8859-1?q?entlang=2C_an_s=FCdl=FCndischen_Wandgem=E4lden_vorbei=2C_g?= - =?iso-8859-1?q?egen_die_rotierenden_Klingen_bef=F6rdert=2E_?=""") - - def test_long_received_header(self): - h = 'from FOO.TLD (vizworld.acl.foo.tld [123.452.678.9]) by hrothgar.la.mastaler.com (tmda-ofmipd) with ESMTP; Wed, 05 Mar 2003 18:10:18 -0700' - msg = Message() - msg['Received-1'] = Header(h, continuation_ws='\t') - msg['Received-2'] = h - self.assertEqual(msg.as_string(), """\ -Received-1: from FOO.TLD (vizworld.acl.foo.tld [123.452.678.9]) by -\throthgar.la.mastaler.com (tmda-ofmipd) with ESMTP; -\tWed, 05 Mar 2003 18:10:18 -0700 -Received-2: from FOO.TLD (vizworld.acl.foo.tld [123.452.678.9]) by - hrothgar.la.mastaler.com (tmda-ofmipd) with ESMTP; - Wed, 05 Mar 2003 18:10:18 -0700 - -""") - - def test_string_headerinst_eq(self): - h = '<15975.17901.207240.414604 at sgigritzmann1.mathematik.tu-muenchen.de> (David Bremner\'s message of "Thu, 6 Mar 2003 13:58:21 +0100")' - msg = Message() - msg['Received'] = Header(h, header_name='Received', - continuation_ws='\t') - msg['Received'] = h - self.ndiffAssertEqual(msg.as_string(), """\ -Received: <15975.17901.207240.414604 at sgigritzmann1.mathematik.tu-muenchen.de> -\t(David Bremner's message of "Thu, 6 Mar 2003 13:58:21 +0100") -Received: <15975.17901.207240.414604 at sgigritzmann1.mathematik.tu-muenchen.de> - (David Bremner's message of "Thu, 6 Mar 2003 13:58:21 +0100") - -""") - - def test_long_unbreakable_lines_with_continuation(self): - eq = self.ndiffAssertEqual - msg = Message() - t = """\ - iVBORw0KGgoAAAANSUhEUgAAADAAAAAwBAMAAAClLOS0AAAAGFBMVEUAAAAkHiJeRUIcGBi9 - locQDQ4zJykFBAXJfWDjAAACYUlEQVR4nF2TQY/jIAyFc6lydlG5x8Nyp1Y69wj1PN2I5gzp""" - msg['Face-1'] = t - msg['Face-2'] = Header(t, header_name='Face-2') - eq(msg.as_string(), """\ -Face-1: iVBORw0KGgoAAAANSUhEUgAAADAAAAAwBAMAAAClLOS0AAAAGFBMVEUAAAAkHiJeRUIcGBi9 - locQDQ4zJykFBAXJfWDjAAACYUlEQVR4nF2TQY/jIAyFc6lydlG5x8Nyp1Y69wj1PN2I5gzp -Face-2: iVBORw0KGgoAAAANSUhEUgAAADAAAAAwBAMAAAClLOS0AAAAGFBMVEUAAAAkHiJeRUIcGBi9 - locQDQ4zJykFBAXJfWDjAAACYUlEQVR4nF2TQY/jIAyFc6lydlG5x8Nyp1Y69wj1PN2I5gzp - -""") - - def test_another_long_multiline_header(self): - eq = self.ndiffAssertEqual - m = '''\ -Received: from siimage.com ([172.25.1.3]) by zima.siliconimage.com with Microsoft SMTPSVC(5.0.2195.4905); - Wed, 16 Oct 2002 07:41:11 -0700''' - msg = email.message_from_string(m) - eq(msg.as_string(), '''\ -Received: from siimage.com ([172.25.1.3]) by zima.siliconimage.com with - Microsoft SMTPSVC(5.0.2195.4905); Wed, 16 Oct 2002 07:41:11 -0700 - -''') - - def test_long_lines_with_different_header(self): - eq = self.ndiffAssertEqual - h = """\ -List-Unsubscribe: <https://lists.sourceforge.net/lists/listinfo/spamassassin-talk>, - <mailto:spamassassin-talk-request at lists.sourceforge.net?subject=unsubscribe>""" - msg = Message() - msg['List'] = h - msg['List'] = Header(h, header_name='List') - eq(msg.as_string(), """\ -List: List-Unsubscribe: <https://lists.sourceforge.net/lists/listinfo/spamassassin-talk>, - <mailto:spamassassin-talk-request at lists.sourceforge.net?subject=unsubscribe> -List: List-Unsubscribe: <https://lists.sourceforge.net/lists/listinfo/spamassassin-talk>, - <mailto:spamassassin-talk-request at lists.sourceforge.net?subject=unsubscribe> - -""") - - - -# Test mangling of "From " lines in the body of a message -class TestFromMangling(unittest.TestCase): - def setUp(self): - self.msg = Message() - self.msg['From'] = 'aaa at bbb.org' - self.msg.set_payload("""\ -From the desk of A.A.A.: -Blah blah blah -""") - - def test_mangled_from(self): - s = StringIO() - g = Generator(s, mangle_from_=True) - g.flatten(self.msg) - self.assertEqual(s.getvalue(), """\ -From: aaa at bbb.org - ->From the desk of A.A.A.: -Blah blah blah -""") - - def test_dont_mangle_from(self): - s = StringIO() - g = Generator(s, mangle_from_=False) - g.flatten(self.msg) - self.assertEqual(s.getvalue(), """\ -From: aaa at bbb.org - -From the desk of A.A.A.: -Blah blah blah -""") - - - -# Test the basic MIMEAudio class -class TestMIMEAudio(unittest.TestCase): - def setUp(self): - # Make sure we pick up the audiotest.au that lives in email/test/data. - # In Python, there's an audiotest.au living in Lib/test but that isn't - # included in some binary distros that don't include the test - # package. The trailing empty string on the .join() is significant - # since findfile() will do a dirname(). - datadir = os.path.join(os.path.dirname(landmark), 'data', '') - fp = open(findfile('audiotest.au', datadir), 'rb') - try: - self._audiodata = fp.read() - finally: - fp.close() - self._au = MIMEAudio(self._audiodata) - - def test_guess_minor_type(self): - self.assertEqual(self._au.get_content_type(), 'audio/basic') - - def test_encoding(self): - payload = self._au.get_payload() - self.assertEqual(base64.decodestring(payload), self._audiodata) - - def test_checkSetMinor(self): - au = MIMEAudio(self._audiodata, 'fish') - self.assertEqual(au.get_content_type(), 'audio/fish') - - def test_add_header(self): - eq = self.assertEqual - unless = self.assertTrue - self._au.add_header('Content-Disposition', 'attachment', - filename='audiotest.au') - eq(self._au['content-disposition'], - 'attachment; filename="audiotest.au"') - eq(self._au.get_params(header='content-disposition'), - [('attachment', ''), ('filename', 'audiotest.au')]) - eq(self._au.get_param('filename', header='content-disposition'), - 'audiotest.au') - missing = [] - eq(self._au.get_param('attachment', header='content-disposition'), '') - unless(self._au.get_param('foo', failobj=missing, - header='content-disposition') is missing) - # Try some missing stuff - unless(self._au.get_param('foobar', missing) is missing) - unless(self._au.get_param('attachment', missing, - header='foobar') is missing) - - - -# Test the basic MIMEImage class -class TestMIMEImage(unittest.TestCase): - def setUp(self): - fp = openfile('PyBanner048.gif') - try: - self._imgdata = fp.read() - finally: - fp.close() - self._im = MIMEImage(self._imgdata) - - def test_guess_minor_type(self): - self.assertEqual(self._im.get_content_type(), 'image/gif') - - def test_encoding(self): - payload = self._im.get_payload() - self.assertEqual(base64.decodestring(payload), self._imgdata) - - def test_checkSetMinor(self): - im = MIMEImage(self._imgdata, 'fish') - self.assertEqual(im.get_content_type(), 'image/fish') - - def test_add_header(self): - eq = self.assertEqual - unless = self.assertTrue - self._im.add_header('Content-Disposition', 'attachment', - filename='dingusfish.gif') - eq(self._im['content-disposition'], - 'attachment; filename="dingusfish.gif"') - eq(self._im.get_params(header='content-disposition'), - [('attachment', ''), ('filename', 'dingusfish.gif')]) - eq(self._im.get_param('filename', header='content-disposition'), - 'dingusfish.gif') - missing = [] - eq(self._im.get_param('attachment', header='content-disposition'), '') - unless(self._im.get_param('foo', failobj=missing, - header='content-disposition') is missing) - # Try some missing stuff - unless(self._im.get_param('foobar', missing) is missing) - unless(self._im.get_param('attachment', missing, - header='foobar') is missing) - - - -# Test the basic MIMEText class -class TestMIMEText(unittest.TestCase): - def setUp(self): - self._msg = MIMEText('hello there') - - def test_types(self): - eq = self.assertEqual - unless = self.assertTrue - eq(self._msg.get_content_type(), 'text/plain') - eq(self._msg.get_param('charset'), 'us-ascii') - missing = [] - unless(self._msg.get_param('foobar', missing) is missing) - unless(self._msg.get_param('charset', missing, header='foobar') - is missing) - - def test_payload(self): - self.assertEqual(self._msg.get_payload(), 'hello there') - self.assertTrue(not self._msg.is_multipart()) - - def test_charset(self): - eq = self.assertEqual - msg = MIMEText('hello there', _charset='us-ascii') - eq(msg.get_charset().input_charset, 'us-ascii') - eq(msg['content-type'], 'text/plain; charset="us-ascii"') - - def test_7bit_unicode_input(self): - eq = self.assertEqual - msg = MIMEText(u'hello there', _charset='us-ascii') - eq(msg.get_charset().input_charset, 'us-ascii') - eq(msg['content-type'], 'text/plain; charset="us-ascii"') - - def test_7bit_unicode_input_no_charset(self): - eq = self.assertEqual - msg = MIMEText(u'hello there') - eq(msg.get_charset(), 'us-ascii') - eq(msg['content-type'], 'text/plain; charset="us-ascii"') - self.assertTrue('hello there' in msg.as_string()) - - def test_8bit_unicode_input(self): - teststr = u'\u043a\u0438\u0440\u0438\u043b\u0438\u0446\u0430' - eq = self.assertEqual - msg = MIMEText(teststr, _charset='utf-8') - eq(msg.get_charset().output_charset, 'utf-8') - eq(msg['content-type'], 'text/plain; charset="utf-8"') - eq(msg.get_payload(decode=True), teststr.encode('utf-8')) - - def test_8bit_unicode_input_no_charset(self): - teststr = u'\u043a\u0438\u0440\u0438\u043b\u0438\u0446\u0430' - self.assertRaises(UnicodeEncodeError, MIMEText, teststr) - - - -# Test complicated multipart/* messages -class TestMultipart(TestEmailBase): - def setUp(self): - fp = openfile('PyBanner048.gif') - try: - data = fp.read() - finally: - fp.close() - - container = MIMEBase('multipart', 'mixed', boundary='BOUNDARY') - image = MIMEImage(data, name='dingusfish.gif') - image.add_header('content-disposition', 'attachment', - filename='dingusfish.gif') - intro = MIMEText('''\ -Hi there, - -This is the dingus fish. -''') - container.attach(intro) - container.attach(image) - container['From'] = 'Barry <barry at digicool.com>' - container['To'] = 'Dingus Lovers <cravindogs at cravindogs.com>' - container['Subject'] = 'Here is your dingus fish' - - now = 987809702.54848599 - timetuple = time.localtime(now) - if timetuple[-1] == 0: - tzsecs = time.timezone - else: - tzsecs = time.altzone - if tzsecs > 0: - sign = '-' - else: - sign = '+' - tzoffset = ' %s%04d' % (sign, tzsecs // 36) - container['Date'] = time.strftime( - '%a, %d %b %Y %H:%M:%S', - time.localtime(now)) + tzoffset - self._msg = container - self._im = image - self._txt = intro - - def test_hierarchy(self): - # convenience - eq = self.assertEqual - unless = self.assertTrue - raises = self.assertRaises - # tests - m = self._msg - unless(m.is_multipart()) - eq(m.get_content_type(), 'multipart/mixed') - eq(len(m.get_payload()), 2) - raises(IndexError, m.get_payload, 2) - m0 = m.get_payload(0) - m1 = m.get_payload(1) - unless(m0 is self._txt) - unless(m1 is self._im) - eq(m.get_payload(), [m0, m1]) - unless(not m0.is_multipart()) - unless(not m1.is_multipart()) - - def test_empty_multipart_idempotent(self): - text = """\ -Content-Type: multipart/mixed; boundary="BOUNDARY" -MIME-Version: 1.0 -Subject: A subject -To: aperson at dom.ain -From: bperson at dom.ain - - ---BOUNDARY - - ---BOUNDARY-- -""" - msg = Parser().parsestr(text) - self.ndiffAssertEqual(text, msg.as_string()) - - def test_no_parts_in_a_multipart_with_none_epilogue(self): - outer = MIMEBase('multipart', 'mixed') - outer['Subject'] = 'A subject' - outer['To'] = 'aperson at dom.ain' - outer['From'] = 'bperson at dom.ain' - outer.set_boundary('BOUNDARY') - self.ndiffAssertEqual(outer.as_string(), '''\ -Content-Type: multipart/mixed; boundary="BOUNDARY" -MIME-Version: 1.0 -Subject: A subject -To: aperson at dom.ain -From: bperson at dom.ain - ---BOUNDARY - ---BOUNDARY--''') - - def test_no_parts_in_a_multipart_with_empty_epilogue(self): - outer = MIMEBase('multipart', 'mixed') - outer['Subject'] = 'A subject' - outer['To'] = 'aperson at dom.ain' - outer['From'] = 'bperson at dom.ain' - outer.preamble = '' - outer.epilogue = '' - outer.set_boundary('BOUNDARY') - self.ndiffAssertEqual(outer.as_string(), '''\ -Content-Type: multipart/mixed; boundary="BOUNDARY" -MIME-Version: 1.0 -Subject: A subject -To: aperson at dom.ain -From: bperson at dom.ain - - ---BOUNDARY - ---BOUNDARY-- -''') - - def test_one_part_in_a_multipart(self): - eq = self.ndiffAssertEqual - outer = MIMEBase('multipart', 'mixed') - outer['Subject'] = 'A subject' - outer['To'] = 'aperson at dom.ain' - outer['From'] = 'bperson at dom.ain' - outer.set_boundary('BOUNDARY') - msg = MIMEText('hello world') - outer.attach(msg) - eq(outer.as_string(), '''\ -Content-Type: multipart/mixed; boundary="BOUNDARY" -MIME-Version: 1.0 -Subject: A subject -To: aperson at dom.ain -From: bperson at dom.ain - ---BOUNDARY -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit - -hello world ---BOUNDARY--''') - - def test_seq_parts_in_a_multipart_with_empty_preamble(self): - eq = self.ndiffAssertEqual - outer = MIMEBase('multipart', 'mixed') - outer['Subject'] = 'A subject' - outer['To'] = 'aperson at dom.ain' - outer['From'] = 'bperson at dom.ain' - outer.preamble = '' - msg = MIMEText('hello world') - outer.attach(msg) - outer.set_boundary('BOUNDARY') - eq(outer.as_string(), '''\ -Content-Type: multipart/mixed; boundary="BOUNDARY" -MIME-Version: 1.0 -Subject: A subject -To: aperson at dom.ain -From: bperson at dom.ain - - ---BOUNDARY -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit - -hello world ---BOUNDARY--''') - - - def test_seq_parts_in_a_multipart_with_none_preamble(self): - eq = self.ndiffAssertEqual - outer = MIMEBase('multipart', 'mixed') - outer['Subject'] = 'A subject' - outer['To'] = 'aperson at dom.ain' - outer['From'] = 'bperson at dom.ain' - outer.preamble = None - msg = MIMEText('hello world') - outer.attach(msg) - outer.set_boundary('BOUNDARY') - eq(outer.as_string(), '''\ -Content-Type: multipart/mixed; boundary="BOUNDARY" -MIME-Version: 1.0 -Subject: A subject -To: aperson at dom.ain -From: bperson at dom.ain - ---BOUNDARY -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit - -hello world ---BOUNDARY--''') - - - def test_seq_parts_in_a_multipart_with_none_epilogue(self): - eq = self.ndiffAssertEqual - outer = MIMEBase('multipart', 'mixed') - outer['Subject'] = 'A subject' - outer['To'] = 'aperson at dom.ain' - outer['From'] = 'bperson at dom.ain' - outer.epilogue = None - msg = MIMEText('hello world') - outer.attach(msg) - outer.set_boundary('BOUNDARY') - eq(outer.as_string(), '''\ -Content-Type: multipart/mixed; boundary="BOUNDARY" -MIME-Version: 1.0 -Subject: A subject -To: aperson at dom.ain -From: bperson at dom.ain - ---BOUNDARY -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit - -hello world ---BOUNDARY--''') - - - def test_seq_parts_in_a_multipart_with_empty_epilogue(self): - eq = self.ndiffAssertEqual - outer = MIMEBase('multipart', 'mixed') - outer['Subject'] = 'A subject' - outer['To'] = 'aperson at dom.ain' - outer['From'] = 'bperson at dom.ain' - outer.epilogue = '' - msg = MIMEText('hello world') - outer.attach(msg) - outer.set_boundary('BOUNDARY') - eq(outer.as_string(), '''\ -Content-Type: multipart/mixed; boundary="BOUNDARY" -MIME-Version: 1.0 -Subject: A subject -To: aperson at dom.ain -From: bperson at dom.ain - ---BOUNDARY -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit - -hello world ---BOUNDARY-- -''') - - - def test_seq_parts_in_a_multipart_with_nl_epilogue(self): - eq = self.ndiffAssertEqual - outer = MIMEBase('multipart', 'mixed') - outer['Subject'] = 'A subject' - outer['To'] = 'aperson at dom.ain' - outer['From'] = 'bperson at dom.ain' - outer.epilogue = '\n' - msg = MIMEText('hello world') - outer.attach(msg) - outer.set_boundary('BOUNDARY') - eq(outer.as_string(), '''\ -Content-Type: multipart/mixed; boundary="BOUNDARY" -MIME-Version: 1.0 -Subject: A subject -To: aperson at dom.ain -From: bperson at dom.ain - ---BOUNDARY -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit - -hello world ---BOUNDARY-- - -''') - - def test_message_external_body(self): - eq = self.assertEqual - msg = self._msgobj('msg_36.txt') - eq(len(msg.get_payload()), 2) - msg1 = msg.get_payload(1) - eq(msg1.get_content_type(), 'multipart/alternative') - eq(len(msg1.get_payload()), 2) - for subpart in msg1.get_payload(): - eq(subpart.get_content_type(), 'message/external-body') - eq(len(subpart.get_payload()), 1) - subsubpart = subpart.get_payload(0) - eq(subsubpart.get_content_type(), 'text/plain') - - def test_double_boundary(self): - # msg_37.txt is a multipart that contains two dash-boundary's in a - # row. Our interpretation of RFC 2046 calls for ignoring the second - # and subsequent boundaries. - msg = self._msgobj('msg_37.txt') - self.assertEqual(len(msg.get_payload()), 3) - - def test_nested_inner_contains_outer_boundary(self): - eq = self.ndiffAssertEqual - # msg_38.txt has an inner part that contains outer boundaries. My - # interpretation of RFC 2046 (based on sections 5.1 and 5.1.2) say - # these are illegal and should be interpreted as unterminated inner - # parts. - msg = self._msgobj('msg_38.txt') - sfp = StringIO() - Iterators._structure(msg, sfp) - eq(sfp.getvalue(), """\ -multipart/mixed - multipart/mixed - multipart/alternative - text/plain - text/plain - text/plain - text/plain -""") - - def test_nested_with_same_boundary(self): - eq = self.ndiffAssertEqual - # msg 39.txt is similarly evil in that it's got inner parts that use - # the same boundary as outer parts. Again, I believe the way this is - # parsed is closest to the spirit of RFC 2046 - msg = self._msgobj('msg_39.txt') - sfp = StringIO() - Iterators._structure(msg, sfp) - eq(sfp.getvalue(), """\ -multipart/mixed - multipart/mixed - multipart/alternative - application/octet-stream - application/octet-stream - text/plain -""") - - def test_boundary_in_non_multipart(self): - msg = self._msgobj('msg_40.txt') - self.assertEqual(msg.as_string(), '''\ -MIME-Version: 1.0 -Content-Type: text/html; boundary="--961284236552522269" - -----961284236552522269 -Content-Type: text/html; -Content-Transfer-Encoding: 7Bit - -<html></html> - -----961284236552522269-- -''') - - def test_boundary_with_leading_space(self): - eq = self.assertEqual - msg = email.message_from_string('''\ -MIME-Version: 1.0 -Content-Type: multipart/mixed; boundary=" XXXX" - --- XXXX -Content-Type: text/plain - - --- XXXX -Content-Type: text/plain - --- XXXX-- -''') - self.assertTrue(msg.is_multipart()) - eq(msg.get_boundary(), ' XXXX') - eq(len(msg.get_payload()), 2) - - def test_boundary_without_trailing_newline(self): - m = Parser().parsestr("""\ -Content-Type: multipart/mixed; boundary="===============0012394164==" -MIME-Version: 1.0 - ---===============0012394164== -Content-Type: image/file1.jpg -MIME-Version: 1.0 -Content-Transfer-Encoding: base64 - -YXNkZg== ---===============0012394164==--""") - self.assertEqual(m.get_payload(0).get_payload(), 'YXNkZg==') - - - -# Test some badly formatted messages -class TestNonConformant(TestEmailBase): - def test_parse_missing_minor_type(self): - eq = self.assertEqual - msg = self._msgobj('msg_14.txt') - eq(msg.get_content_type(), 'text/plain') - eq(msg.get_content_maintype(), 'text') - eq(msg.get_content_subtype(), 'plain') - - def test_same_boundary_inner_outer(self): - unless = self.assertTrue - msg = self._msgobj('msg_15.txt') - # XXX We can probably eventually do better - inner = msg.get_payload(0) - unless(hasattr(inner, 'defects')) - self.assertEqual(len(inner.defects), 1) - unless(isinstance(inner.defects[0], - Errors.StartBoundaryNotFoundDefect)) - - def test_multipart_no_boundary(self): - unless = self.assertTrue - msg = self._msgobj('msg_25.txt') - unless(isinstance(msg.get_payload(), str)) - self.assertEqual(len(msg.defects), 2) - unless(isinstance(msg.defects[0], Errors.NoBoundaryInMultipartDefect)) - unless(isinstance(msg.defects[1], - Errors.MultipartInvariantViolationDefect)) - - def test_invalid_content_type(self): - eq = self.assertEqual - neq = self.ndiffAssertEqual - msg = Message() - # RFC 2045, $5.2 says invalid yields text/plain - msg['Content-Type'] = 'text' - eq(msg.get_content_maintype(), 'text') - eq(msg.get_content_subtype(), 'plain') - eq(msg.get_content_type(), 'text/plain') - # Clear the old value and try something /really/ invalid - del msg['content-type'] - msg['Content-Type'] = 'foo' - eq(msg.get_content_maintype(), 'text') - eq(msg.get_content_subtype(), 'plain') - eq(msg.get_content_type(), 'text/plain') - # Still, make sure that the message is idempotently generated - s = StringIO() - g = Generator(s) - g.flatten(msg) - neq(s.getvalue(), 'Content-Type: foo\n\n') - - def test_no_start_boundary(self): - eq = self.ndiffAssertEqual - msg = self._msgobj('msg_31.txt') - eq(msg.get_payload(), """\ ---BOUNDARY -Content-Type: text/plain - -message 1 - ---BOUNDARY -Content-Type: text/plain - -message 2 - ---BOUNDARY-- -""") - - def test_no_separating_blank_line(self): - eq = self.ndiffAssertEqual - msg = self._msgobj('msg_35.txt') - eq(msg.as_string(), """\ -From: aperson at dom.ain -To: bperson at dom.ain -Subject: here's something interesting - -counter to RFC 2822, there's no separating newline here -""") - - def test_lying_multipart(self): - unless = self.assertTrue - msg = self._msgobj('msg_41.txt') - unless(hasattr(msg, 'defects')) - self.assertEqual(len(msg.defects), 2) - unless(isinstance(msg.defects[0], Errors.NoBoundaryInMultipartDefect)) - unless(isinstance(msg.defects[1], - Errors.MultipartInvariantViolationDefect)) - - def test_missing_start_boundary(self): - outer = self._msgobj('msg_42.txt') - # The message structure is: - # - # multipart/mixed - # text/plain - # message/rfc822 - # multipart/mixed [*] - # - # [*] This message is missing its start boundary - bad = outer.get_payload(1).get_payload(0) - self.assertEqual(len(bad.defects), 1) - self.assertTrue(isinstance(bad.defects[0], - Errors.StartBoundaryNotFoundDefect)) - - def test_first_line_is_continuation_header(self): - eq = self.assertEqual - m = ' Line 1\nLine 2\nLine 3' - msg = email.message_from_string(m) - eq(msg.keys(), []) - eq(msg.get_payload(), 'Line 2\nLine 3') - eq(len(msg.defects), 1) - self.assertTrue(isinstance(msg.defects[0], - Errors.FirstHeaderLineIsContinuationDefect)) - eq(msg.defects[0].line, ' Line 1\n') - - - - -# Test RFC 2047 header encoding and decoding -class TestRFC2047(unittest.TestCase): - def test_rfc2047_multiline(self): - eq = self.assertEqual - s = """Re: =?mac-iceland?q?r=8Aksm=9Arg=8Cs?= baz - foo bar =?mac-iceland?q?r=8Aksm=9Arg=8Cs?=""" - dh = decode_header(s) - eq(dh, [ - ('Re:', None), - ('r\x8aksm\x9arg\x8cs', 'mac-iceland'), - ('baz foo bar', None), - ('r\x8aksm\x9arg\x8cs', 'mac-iceland')]) - eq(str(make_header(dh)), - """Re: =?mac-iceland?q?r=8Aksm=9Arg=8Cs?= baz foo bar - =?mac-iceland?q?r=8Aksm=9Arg=8Cs?=""") - - def test_whitespace_eater_unicode(self): - eq = self.assertEqual - s = '=?ISO-8859-1?Q?Andr=E9?= Pirard <pirard at dom.ain>' - dh = decode_header(s) - eq(dh, [('Andr\xe9', 'iso-8859-1'), ('Pirard <pirard at dom.ain>', None)]) - hu = unicode(make_header(dh)).encode('latin-1') - eq(hu, 'Andr\xe9 Pirard <pirard at dom.ain>') - - def test_whitespace_eater_unicode_2(self): - eq = self.assertEqual - s = 'The =?iso-8859-1?b?cXVpY2sgYnJvd24gZm94?= jumped over the =?iso-8859-1?b?bGF6eSBkb2c=?=' - dh = decode_header(s) - eq(dh, [('The', None), ('quick brown fox', 'iso-8859-1'), - ('jumped over the', None), ('lazy dog', 'iso-8859-1')]) - hu = make_header(dh).__unicode__() - eq(hu, u'The quick brown fox jumped over the lazy dog') - - def test_rfc2047_without_whitespace(self): - s = 'Sm=?ISO-8859-1?B?9g==?=rg=?ISO-8859-1?B?5Q==?=sbord' - dh = decode_header(s) - self.assertEqual(dh, [(s, None)]) - - def test_rfc2047_with_whitespace(self): - s = 'Sm =?ISO-8859-1?B?9g==?= rg =?ISO-8859-1?B?5Q==?= sbord' - dh = decode_header(s) - self.assertEqual(dh, [('Sm', None), ('\xf6', 'iso-8859-1'), - ('rg', None), ('\xe5', 'iso-8859-1'), - ('sbord', None)]) - - def test_rfc2047_B_bad_padding(self): - s = '=?iso-8859-1?B?%s?=' - data = [ # only test complete bytes - ('dm==', 'v'), ('dm=', 'v'), ('dm', 'v'), - ('dmk=', 'vi'), ('dmk', 'vi') - ] - for q, a in data: - dh = decode_header(s % q) - self.assertEqual(dh, [(a, 'iso-8859-1')]) - - def test_rfc2047_Q_invalid_digits(self): - # issue 10004. - s = '=?iso-8659-1?Q?andr=e9=zz?=' - self.assertEqual(decode_header(s), - [(b'andr\xe9=zz', 'iso-8659-1')]) - - -# Test the MIMEMessage class -class TestMIMEMessage(TestEmailBase): - def setUp(self): - fp = openfile('msg_11.txt') - try: - self._text = fp.read() - finally: - fp.close() - - def test_type_error(self): - self.assertRaises(TypeError, MIMEMessage, 'a plain string') - - def test_valid_argument(self): - eq = self.assertEqual - unless = self.assertTrue - subject = 'A sub-message' - m = Message() - m['Subject'] = subject - r = MIMEMessage(m) - eq(r.get_content_type(), 'message/rfc822') - payload = r.get_payload() - unless(isinstance(payload, list)) - eq(len(payload), 1) - subpart = payload[0] - unless(subpart is m) - eq(subpart['subject'], subject) - - def test_bad_multipart(self): - eq = self.assertEqual - msg1 = Message() - msg1['Subject'] = 'subpart 1' - msg2 = Message() - msg2['Subject'] = 'subpart 2' - r = MIMEMessage(msg1) - self.assertRaises(Errors.MultipartConversionError, r.attach, msg2) - - def test_generate(self): - # First craft the message to be encapsulated - m = Message() - m['Subject'] = 'An enclosed message' - m.set_payload('Here is the body of the message.\n') - r = MIMEMessage(m) - r['Subject'] = 'The enclosing message' - s = StringIO() - g = Generator(s) - g.flatten(r) - self.assertEqual(s.getvalue(), """\ -Content-Type: message/rfc822 -MIME-Version: 1.0 -Subject: The enclosing message - -Subject: An enclosed message - -Here is the body of the message. -""") - - def test_parse_message_rfc822(self): - eq = self.assertEqual - unless = self.assertTrue - msg = self._msgobj('msg_11.txt') - eq(msg.get_content_type(), 'message/rfc822') - payload = msg.get_payload() - unless(isinstance(payload, list)) - eq(len(payload), 1) - submsg = payload[0] - self.assertTrue(isinstance(submsg, Message)) - eq(submsg['subject'], 'An enclosed message') - eq(submsg.get_payload(), 'Here is the body of the message.\n') - - def test_dsn(self): - eq = self.assertEqual - unless = self.assertTrue - # msg 16 is a Delivery Status Notification, see RFC 1894 - msg = self._msgobj('msg_16.txt') - eq(msg.get_content_type(), 'multipart/report') - unless(msg.is_multipart()) - eq(len(msg.get_payload()), 3) - # Subpart 1 is a text/plain, human readable section - subpart = msg.get_payload(0) - eq(subpart.get_content_type(), 'text/plain') - eq(subpart.get_payload(), """\ -This report relates to a message you sent with the following header fields: - - Message-id: <002001c144a6$8752e060$56104586 at oxy.edu> - Date: Sun, 23 Sep 2001 20:10:55 -0700 - From: "Ian T. Henry" <henryi at oxy.edu> - To: SoCal Raves <scr at socal-raves.org> - Subject: [scr] yeah for Ians!! - -Your message cannot be delivered to the following recipients: - - Recipient address: jangel1 at cougar.noc.ucla.edu - Reason: recipient reached disk quota - -""") - # Subpart 2 contains the machine parsable DSN information. It - # consists of two blocks of headers, represented by two nested Message - # objects. - subpart = msg.get_payload(1) - eq(subpart.get_content_type(), 'message/delivery-status') - eq(len(subpart.get_payload()), 2) - # message/delivery-status should treat each block as a bunch of - # headers, i.e. a bunch of Message objects. - dsn1 = subpart.get_payload(0) - unless(isinstance(dsn1, Message)) - eq(dsn1['original-envelope-id'], '0GK500B4HD0888 at cougar.noc.ucla.edu') - eq(dsn1.get_param('dns', header='reporting-mta'), '') - # Try a missing one <wink> - eq(dsn1.get_param('nsd', header='reporting-mta'), None) - dsn2 = subpart.get_payload(1) - unless(isinstance(dsn2, Message)) - eq(dsn2['action'], 'failed') - eq(dsn2.get_params(header='original-recipient'), - [('rfc822', ''), ('jangel1 at cougar.noc.ucla.edu', '')]) - eq(dsn2.get_param('rfc822', header='final-recipient'), '') - # Subpart 3 is the original message - subpart = msg.get_payload(2) - eq(subpart.get_content_type(), 'message/rfc822') - payload = subpart.get_payload() - unless(isinstance(payload, list)) - eq(len(payload), 1) - subsubpart = payload[0] - unless(isinstance(subsubpart, Message)) - eq(subsubpart.get_content_type(), 'text/plain') - eq(subsubpart['message-id'], - '<002001c144a6$8752e060$56104586 at oxy.edu>') - - def test_epilogue(self): - eq = self.ndiffAssertEqual - fp = openfile('msg_21.txt') - try: - text = fp.read() - finally: - fp.close() - msg = Message() - msg['From'] = 'aperson at dom.ain' - msg['To'] = 'bperson at dom.ain' - msg['Subject'] = 'Test' - msg.preamble = 'MIME message' - msg.epilogue = 'End of MIME message\n' - msg1 = MIMEText('One') - msg2 = MIMEText('Two') - msg.add_header('Content-Type', 'multipart/mixed', boundary='BOUNDARY') - msg.attach(msg1) - msg.attach(msg2) - sfp = StringIO() - g = Generator(sfp) - g.flatten(msg) - eq(sfp.getvalue(), text) - - def test_no_nl_preamble(self): - eq = self.ndiffAssertEqual - msg = Message() - msg['From'] = 'aperson at dom.ain' - msg['To'] = 'bperson at dom.ain' - msg['Subject'] = 'Test' - msg.preamble = 'MIME message' - msg.epilogue = '' - msg1 = MIMEText('One') - msg2 = MIMEText('Two') - msg.add_header('Content-Type', 'multipart/mixed', boundary='BOUNDARY') - msg.attach(msg1) - msg.attach(msg2) - eq(msg.as_string(), """\ -From: aperson at dom.ain -To: bperson at dom.ain -Subject: Test -Content-Type: multipart/mixed; boundary="BOUNDARY" - -MIME message ---BOUNDARY -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit - -One ---BOUNDARY -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit - -Two ---BOUNDARY-- -""") - - def test_default_type(self): - eq = self.assertEqual - fp = openfile('msg_30.txt') - try: - msg = email.message_from_file(fp) - finally: - fp.close() - container1 = msg.get_payload(0) - eq(container1.get_default_type(), 'message/rfc822') - eq(container1.get_content_type(), 'message/rfc822') - container2 = msg.get_payload(1) - eq(container2.get_default_type(), 'message/rfc822') - eq(container2.get_content_type(), 'message/rfc822') - container1a = container1.get_payload(0) - eq(container1a.get_default_type(), 'text/plain') - eq(container1a.get_content_type(), 'text/plain') - container2a = container2.get_payload(0) - eq(container2a.get_default_type(), 'text/plain') - eq(container2a.get_content_type(), 'text/plain') - - def test_default_type_with_explicit_container_type(self): - eq = self.assertEqual - fp = openfile('msg_28.txt') - try: - msg = email.message_from_file(fp) - finally: - fp.close() - container1 = msg.get_payload(0) - eq(container1.get_default_type(), 'message/rfc822') - eq(container1.get_content_type(), 'message/rfc822') - container2 = msg.get_payload(1) - eq(container2.get_default_type(), 'message/rfc822') - eq(container2.get_content_type(), 'message/rfc822') - container1a = container1.get_payload(0) - eq(container1a.get_default_type(), 'text/plain') - eq(container1a.get_content_type(), 'text/plain') - container2a = container2.get_payload(0) - eq(container2a.get_default_type(), 'text/plain') - eq(container2a.get_content_type(), 'text/plain') - - def test_default_type_non_parsed(self): - eq = self.assertEqual - neq = self.ndiffAssertEqual - # Set up container - container = MIMEMultipart('digest', 'BOUNDARY') - container.epilogue = '' - # Set up subparts - subpart1a = MIMEText('message 1\n') - subpart2a = MIMEText('message 2\n') - subpart1 = MIMEMessage(subpart1a) - subpart2 = MIMEMessage(subpart2a) - container.attach(subpart1) - container.attach(subpart2) - eq(subpart1.get_content_type(), 'message/rfc822') - eq(subpart1.get_default_type(), 'message/rfc822') - eq(subpart2.get_content_type(), 'message/rfc822') - eq(subpart2.get_default_type(), 'message/rfc822') - neq(container.as_string(0), '''\ -Content-Type: multipart/digest; boundary="BOUNDARY" -MIME-Version: 1.0 - ---BOUNDARY -Content-Type: message/rfc822 -MIME-Version: 1.0 - -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit - -message 1 - ---BOUNDARY -Content-Type: message/rfc822 -MIME-Version: 1.0 - -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit - -message 2 - ---BOUNDARY-- -''') - del subpart1['content-type'] - del subpart1['mime-version'] - del subpart2['content-type'] - del subpart2['mime-version'] - eq(subpart1.get_content_type(), 'message/rfc822') - eq(subpart1.get_default_type(), 'message/rfc822') - eq(subpart2.get_content_type(), 'message/rfc822') - eq(subpart2.get_default_type(), 'message/rfc822') - neq(container.as_string(0), '''\ -Content-Type: multipart/digest; boundary="BOUNDARY" -MIME-Version: 1.0 - ---BOUNDARY - -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit - -message 1 - ---BOUNDARY - -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit - -message 2 - ---BOUNDARY-- -''') - - def test_mime_attachments_in_constructor(self): - eq = self.assertEqual - text1 = MIMEText('') - text2 = MIMEText('') - msg = MIMEMultipart(_subparts=(text1, text2)) - eq(len(msg.get_payload()), 2) - eq(msg.get_payload(0), text1) - eq(msg.get_payload(1), text2) - - def test_default_multipart_constructor(self): - msg = MIMEMultipart() - self.assertTrue(msg.is_multipart()) - - -# A general test of parser->model->generator idempotency. IOW, read a message -# in, parse it into a message object tree, then without touching the tree, -# regenerate the plain text. The original text and the transformed text -# should be identical. Note: that we ignore the Unix-From since that may -# contain a changed date. -class TestIdempotent(TestEmailBase): - def _msgobj(self, filename): - fp = openfile(filename) - try: - data = fp.read() - finally: - fp.close() - msg = email.message_from_string(data) - return msg, data - - def _idempotent(self, msg, text): - eq = self.ndiffAssertEqual - s = StringIO() - g = Generator(s, maxheaderlen=0) - g.flatten(msg) - eq(text, s.getvalue()) - - def test_parse_text_message(self): - eq = self.assertEqual - msg, text = self._msgobj('msg_01.txt') - eq(msg.get_content_type(), 'text/plain') - eq(msg.get_content_maintype(), 'text') - eq(msg.get_content_subtype(), 'plain') - eq(msg.get_params()[1], ('charset', 'us-ascii')) - eq(msg.get_param('charset'), 'us-ascii') - eq(msg.preamble, None) - eq(msg.epilogue, None) - self._idempotent(msg, text) - - def test_parse_untyped_message(self): - eq = self.assertEqual - msg, text = self._msgobj('msg_03.txt') - eq(msg.get_content_type(), 'text/plain') - eq(msg.get_params(), None) - eq(msg.get_param('charset'), None) - self._idempotent(msg, text) - - def test_simple_multipart(self): - msg, text = self._msgobj('msg_04.txt') - self._idempotent(msg, text) - - def test_MIME_digest(self): - msg, text = self._msgobj('msg_02.txt') - self._idempotent(msg, text) - - def test_long_header(self): - msg, text = self._msgobj('msg_27.txt') - self._idempotent(msg, text) - - def test_MIME_digest_with_part_headers(self): - msg, text = self._msgobj('msg_28.txt') - self._idempotent(msg, text) - - def test_mixed_with_image(self): - msg, text = self._msgobj('msg_06.txt') - self._idempotent(msg, text) - - def test_multipart_report(self): - msg, text = self._msgobj('msg_05.txt') - self._idempotent(msg, text) - - def test_dsn(self): - msg, text = self._msgobj('msg_16.txt') - self._idempotent(msg, text) - - def test_preamble_epilogue(self): - msg, text = self._msgobj('msg_21.txt') - self._idempotent(msg, text) - - def test_multipart_one_part(self): - msg, text = self._msgobj('msg_23.txt') - self._idempotent(msg, text) - - def test_multipart_no_parts(self): - msg, text = self._msgobj('msg_24.txt') - self._idempotent(msg, text) - - def test_no_start_boundary(self): - msg, text = self._msgobj('msg_31.txt') - self._idempotent(msg, text) - - def test_rfc2231_charset(self): - msg, text = self._msgobj('msg_32.txt') - self._idempotent(msg, text) - - def test_more_rfc2231_parameters(self): - msg, text = self._msgobj('msg_33.txt') - self._idempotent(msg, text) - - def test_text_plain_in_a_multipart_digest(self): - msg, text = self._msgobj('msg_34.txt') - self._idempotent(msg, text) - - def test_nested_multipart_mixeds(self): - msg, text = self._msgobj('msg_12a.txt') - self._idempotent(msg, text) - - def test_message_external_body_idempotent(self): - msg, text = self._msgobj('msg_36.txt') - self._idempotent(msg, text) - - def test_content_type(self): - eq = self.assertEqual - unless = self.assertTrue - # Get a message object and reset the seek pointer for other tests - msg, text = self._msgobj('msg_05.txt') - eq(msg.get_content_type(), 'multipart/report') - # Test the Content-Type: parameters - params = {} - for pk, pv in msg.get_params(): - params[pk] = pv - eq(params['report-type'], 'delivery-status') - eq(params['boundary'], 'D1690A7AC1.996856090/mail.example.com') - eq(msg.preamble, 'This is a MIME-encapsulated message.\n') - eq(msg.epilogue, '\n') - eq(len(msg.get_payload()), 3) - # Make sure the subparts are what we expect - msg1 = msg.get_payload(0) - eq(msg1.get_content_type(), 'text/plain') - eq(msg1.get_payload(), 'Yadda yadda yadda\n') - msg2 = msg.get_payload(1) - eq(msg2.get_content_type(), 'text/plain') - eq(msg2.get_payload(), 'Yadda yadda yadda\n') - msg3 = msg.get_payload(2) - eq(msg3.get_content_type(), 'message/rfc822') - self.assertTrue(isinstance(msg3, Message)) - payload = msg3.get_payload() - unless(isinstance(payload, list)) - eq(len(payload), 1) - msg4 = payload[0] - unless(isinstance(msg4, Message)) - eq(msg4.get_payload(), 'Yadda yadda yadda\n') - - def test_parser(self): - eq = self.assertEqual - unless = self.assertTrue - msg, text = self._msgobj('msg_06.txt') - # Check some of the outer headers - eq(msg.get_content_type(), 'message/rfc822') - # Make sure the payload is a list of exactly one sub-Message, and that - # that submessage has a type of text/plain - payload = msg.get_payload() - unless(isinstance(payload, list)) - eq(len(payload), 1) - msg1 = payload[0] - self.assertTrue(isinstance(msg1, Message)) - eq(msg1.get_content_type(), 'text/plain') - self.assertTrue(isinstance(msg1.get_payload(), str)) - eq(msg1.get_payload(), '\n') - - - -# Test various other bits of the package's functionality -class TestMiscellaneous(TestEmailBase): - def test_message_from_string(self): - fp = openfile('msg_01.txt') - try: - text = fp.read() - finally: - fp.close() - msg = email.message_from_string(text) - s = StringIO() - # Don't wrap/continue long headers since we're trying to test - # idempotency. - g = Generator(s, maxheaderlen=0) - g.flatten(msg) - self.assertEqual(text, s.getvalue()) - - def test_message_from_file(self): - fp = openfile('msg_01.txt') - try: - text = fp.read() - fp.seek(0) - msg = email.message_from_file(fp) - s = StringIO() - # Don't wrap/continue long headers since we're trying to test - # idempotency. - g = Generator(s, maxheaderlen=0) - g.flatten(msg) - self.assertEqual(text, s.getvalue()) - finally: - fp.close() - - def test_message_from_string_with_class(self): - unless = self.assertTrue - fp = openfile('msg_01.txt') - try: - text = fp.read() - finally: - fp.close() - # Create a subclass - class MyMessage(Message): - pass - - msg = email.message_from_string(text, MyMessage) - unless(isinstance(msg, MyMessage)) - # Try something more complicated - fp = openfile('msg_02.txt') - try: - text = fp.read() - finally: - fp.close() - msg = email.message_from_string(text, MyMessage) - for subpart in msg.walk(): - unless(isinstance(subpart, MyMessage)) - - def test_message_from_file_with_class(self): - unless = self.assertTrue - # Create a subclass - class MyMessage(Message): - pass - - fp = openfile('msg_01.txt') - try: - msg = email.message_from_file(fp, MyMessage) - finally: - fp.close() - unless(isinstance(msg, MyMessage)) - # Try something more complicated - fp = openfile('msg_02.txt') - try: - msg = email.message_from_file(fp, MyMessage) - finally: - fp.close() - for subpart in msg.walk(): - unless(isinstance(subpart, MyMessage)) - - def test__all__(self): - module = __import__('email') - all = module.__all__ - all.sort() - self.assertEqual(all, [ - # Old names - 'Charset', 'Encoders', 'Errors', 'Generator', - 'Header', 'Iterators', 'MIMEAudio', 'MIMEBase', - 'MIMEImage', 'MIMEMessage', 'MIMEMultipart', - 'MIMENonMultipart', 'MIMEText', 'Message', - 'Parser', 'Utils', 'base64MIME', - # new names - 'base64mime', 'charset', 'encoders', 'errors', 'generator', - 'header', 'iterators', 'message', 'message_from_file', - 'message_from_string', 'mime', 'parser', - 'quopriMIME', 'quoprimime', 'utils', - ]) - - def test_formatdate(self): - now = time.time() - self.assertEqual(Utils.parsedate(Utils.formatdate(now))[:6], - time.gmtime(now)[:6]) - - def test_formatdate_localtime(self): - now = time.time() - self.assertEqual( - Utils.parsedate(Utils.formatdate(now, localtime=True))[:6], - time.localtime(now)[:6]) - - def test_formatdate_usegmt(self): - now = time.time() - self.assertEqual( - Utils.formatdate(now, localtime=False), - time.strftime('%a, %d %b %Y %H:%M:%S -0000', time.gmtime(now))) - self.assertEqual( - Utils.formatdate(now, localtime=False, usegmt=True), - time.strftime('%a, %d %b %Y %H:%M:%S GMT', time.gmtime(now))) - - def test_parsedate_none(self): - self.assertEqual(Utils.parsedate(''), None) - - def test_parsedate_compact(self): - # The FWS after the comma is optional - self.assertEqual(Utils.parsedate('Wed,3 Apr 2002 14:58:26 +0800'), - Utils.parsedate('Wed, 3 Apr 2002 14:58:26 +0800')) - - def test_parsedate_no_dayofweek(self): - eq = self.assertEqual - eq(Utils.parsedate_tz('25 Feb 2003 13:47:26 -0800'), - (2003, 2, 25, 13, 47, 26, 0, 1, -1, -28800)) - - def test_parsedate_compact_no_dayofweek(self): - eq = self.assertEqual - eq(Utils.parsedate_tz('5 Feb 2003 13:47:26 -0800'), - (2003, 2, 5, 13, 47, 26, 0, 1, -1, -28800)) - - def test_parsedate_acceptable_to_time_functions(self): - eq = self.assertEqual - timetup = Utils.parsedate('5 Feb 2003 13:47:26 -0800') - t = int(time.mktime(timetup)) - eq(time.localtime(t)[:6], timetup[:6]) - eq(int(time.strftime('%Y', timetup)), 2003) - timetup = Utils.parsedate_tz('5 Feb 2003 13:47:26 -0800') - t = int(time.mktime(timetup[:9])) - eq(time.localtime(t)[:6], timetup[:6]) - eq(int(time.strftime('%Y', timetup[:9])), 2003) - - def test_parsedate_y2k(self): - """Test for parsing a date with a two-digit year. - - Parsing a date with a two-digit year should return the correct - four-digit year. RFC822 allows two-digit years, but RFC2822 (which - obsoletes RFC822) requires four-digit years. - - """ - self.assertEqual(Utils.parsedate_tz('25 Feb 03 13:47:26 -0800'), - Utils.parsedate_tz('25 Feb 2003 13:47:26 -0800')) - self.assertEqual(Utils.parsedate_tz('25 Feb 71 13:47:26 -0800'), - Utils.parsedate_tz('25 Feb 1971 13:47:26 -0800')) - - def test_parseaddr_empty(self): - self.assertEqual(Utils.parseaddr('<>'), ('', '')) - self.assertEqual(Utils.formataddr(Utils.parseaddr('<>')), '') - - def test_noquote_dump(self): - self.assertEqual( - Utils.formataddr(('A Silly Person', 'person at dom.ain')), - 'A Silly Person <person at dom.ain>') - - def test_escape_dump(self): - self.assertEqual( - Utils.formataddr(('A (Very) Silly Person', 'person at dom.ain')), - r'"A $Very$ Silly Person" <person at dom.ain>') - a = r'A $Special$ Person' - b = 'person at dom.ain' - self.assertEqual(Utils.parseaddr(Utils.formataddr((a, b))), (a, b)) - - def test_escape_backslashes(self): - self.assertEqual( - Utils.formataddr(('Arthur \Backslash\ Foobar', 'person at dom.ain')), - r'"Arthur \\Backslash\\ Foobar" <person at dom.ain>') - a = r'Arthur \Backslash\ Foobar' - b = 'person at dom.ain' - self.assertEqual(Utils.parseaddr(Utils.formataddr((a, b))), (a, b)) - - def test_name_with_dot(self): - x = 'John X. Doe <jxd at example.com>' - y = '"John X. Doe" <jxd at example.com>' - a, b = ('John X. Doe', 'jxd at example.com') - self.assertEqual(Utils.parseaddr(x), (a, b)) - self.assertEqual(Utils.parseaddr(y), (a, b)) - # formataddr() quotes the name if there's a dot in it - self.assertEqual(Utils.formataddr((a, b)), y) - - def test_parseaddr_preserves_quoted_pairs_in_addresses(self): - # issue 10005. Note that in the third test the second pair of - # backslashes is not actually a quoted pair because it is not inside a - # comment or quoted string: the address being parsed has a quoted - # string containing a quoted backslash, followed by 'example' and two - # backslashes, followed by another quoted string containing a space and - # the word 'example'. parseaddr copies those two backslashes - # literally. Per rfc5322 this is not technically correct since a \ may - # not appear in an address outside of a quoted string. It is probably - # a sensible Postel interpretation, though. - eq = self.assertEqual - eq(Utils.parseaddr('""example" example"@example.com'), - ('', '""example" example"@example.com')) - eq(Utils.parseaddr('"\\"example\\" example"@example.com'), - ('', '"\\"example\\" example"@example.com')) - eq(Utils.parseaddr('"\\\\"example\\\\" example"@example.com'), - ('', '"\\\\"example\\\\" example"@example.com')) - - def test_multiline_from_comment(self): - x = """\ -Foo -\tBar <foo at example.com>""" - self.assertEqual(Utils.parseaddr(x), ('Foo Bar', 'foo at example.com')) - - def test_quote_dump(self): - self.assertEqual( - Utils.formataddr(('A Silly; Person', 'person at dom.ain')), - r'"A Silly; Person" <person at dom.ain>') - - def test_fix_eols(self): - eq = self.assertEqual - eq(Utils.fix_eols('hello'), 'hello') - eq(Utils.fix_eols('hello\n'), 'hello\r\n') - eq(Utils.fix_eols('hello\r'), 'hello\r\n') - eq(Utils.fix_eols('hello\r\n'), 'hello\r\n') - eq(Utils.fix_eols('hello\n\r'), 'hello\r\n\r\n') - - def test_charset_richcomparisons(self): - eq = self.assertEqual - ne = self.assertNotEqual - cset1 = Charset() - cset2 = Charset() - eq(cset1, 'us-ascii') - eq(cset1, 'US-ASCII') - eq(cset1, 'Us-AsCiI') - eq('us-ascii', cset1) - eq('US-ASCII', cset1) - eq('Us-AsCiI', cset1) - ne(cset1, 'usascii') - ne(cset1, 'USASCII') - ne(cset1, 'UsAsCiI') - ne('usascii', cset1) - ne('USASCII', cset1) - ne('UsAsCiI', cset1) - eq(cset1, cset2) - eq(cset2, cset1) - - def test_getaddresses(self): - eq = self.assertEqual - eq(Utils.getaddresses(['aperson at dom.ain (Al Person)', - 'Bud Person <bperson at dom.ain>']), - [('Al Person', 'aperson at dom.ain'), - ('Bud Person', 'bperson at dom.ain')]) - - def test_getaddresses_nasty(self): - eq = self.assertEqual - eq(Utils.getaddresses(['foo: ;']), [('', '')]) - eq(Utils.getaddresses( - ['[]*-- =~$']), - [('', ''), ('', ''), ('', '*--')]) - eq(Utils.getaddresses( - ['foo: ;', '"Jason R. Mastaler" <jason at dom.ain>']), - [('', ''), ('Jason R. Mastaler', 'jason at dom.ain')]) - - def test_getaddresses_embedded_comment(self): - """Test proper handling of a nested comment""" - eq = self.assertEqual - addrs = Utils.getaddresses(['User ((nested comment)) <foo at bar.com>']) - eq(addrs[0][1], 'foo at bar.com') - - def test_utils_quote_unquote(self): - eq = self.assertEqual - msg = Message() - msg.add_header('content-disposition', 'attachment', - filename='foo\\wacky"name') - eq(msg.get_filename(), 'foo\\wacky"name') - - def test_get_body_encoding_with_bogus_charset(self): - charset = Charset('not a charset') - self.assertEqual(charset.get_body_encoding(), 'base64') - - def test_get_body_encoding_with_uppercase_charset(self): - eq = self.assertEqual - msg = Message() - msg['Content-Type'] = 'text/plain; charset=UTF-8' - eq(msg['content-type'], 'text/plain; charset=UTF-8') - charsets = msg.get_charsets() - eq(len(charsets), 1) - eq(charsets[0], 'utf-8') - charset = Charset(charsets[0]) - eq(charset.get_body_encoding(), 'base64') - msg.set_payload('hello world', charset=charset) - eq(msg.get_payload(), 'aGVsbG8gd29ybGQ=\n') - eq(msg.get_payload(decode=True), 'hello world') - eq(msg['content-transfer-encoding'], 'base64') - # Try another one - msg = Message() - msg['Content-Type'] = 'text/plain; charset="US-ASCII"' - charsets = msg.get_charsets() - eq(len(charsets), 1) - eq(charsets[0], 'us-ascii') - charset = Charset(charsets[0]) - eq(charset.get_body_encoding(), Encoders.encode_7or8bit) - msg.set_payload('hello world', charset=charset) - eq(msg.get_payload(), 'hello world') - eq(msg['content-transfer-encoding'], '7bit') - - def test_charsets_case_insensitive(self): - lc = Charset('us-ascii') - uc = Charset('US-ASCII') - self.assertEqual(lc.get_body_encoding(), uc.get_body_encoding()) - - def test_partial_falls_inside_message_delivery_status(self): - eq = self.ndiffAssertEqual - # The Parser interface provides chunks of data to FeedParser in 8192 - # byte gulps. SF bug #1076485 found one of those chunks inside - # message/delivery-status header block, which triggered an - # unreadline() of NeedMoreData. - msg = self._msgobj('msg_43.txt') - sfp = StringIO() - Iterators._structure(msg, sfp) - eq(sfp.getvalue(), """\ -multipart/report - text/plain - message/delivery-status - text/plain - text/plain - text/plain - text/plain - text/plain - text/plain - text/plain - text/plain - text/plain - text/plain - text/plain - text/plain - text/plain - text/plain - text/plain - text/plain - text/plain - text/plain - text/plain - text/plain - text/plain - text/plain - text/plain - text/plain - text/plain - text/plain - text/rfc822-headers -""") - - - -# Test the iterator/generators -class TestIterators(TestEmailBase): - def test_body_line_iterator(self): - eq = self.assertEqual - neq = self.ndiffAssertEqual - # First a simple non-multipart message - msg = self._msgobj('msg_01.txt') - it = Iterators.body_line_iterator(msg) - lines = list(it) - eq(len(lines), 6) - neq(EMPTYSTRING.join(lines), msg.get_payload()) - # Now a more complicated multipart - msg = self._msgobj('msg_02.txt') - it = Iterators.body_line_iterator(msg) - lines = list(it) - eq(len(lines), 43) - fp = openfile('msg_19.txt') - try: - neq(EMPTYSTRING.join(lines), fp.read()) - finally: - fp.close() - - def test_typed_subpart_iterator(self): - eq = self.assertEqual - msg = self._msgobj('msg_04.txt') - it = Iterators.typed_subpart_iterator(msg, 'text') - lines = [] - subparts = 0 - for subpart in it: - subparts += 1 - lines.append(subpart.get_payload()) - eq(subparts, 2) - eq(EMPTYSTRING.join(lines), """\ -a simple kind of mirror -to reflect upon our own -a simple kind of mirror -to reflect upon our own -""") - - def test_typed_subpart_iterator_default_type(self): - eq = self.assertEqual - msg = self._msgobj('msg_03.txt') - it = Iterators.typed_subpart_iterator(msg, 'text', 'plain') - lines = [] - subparts = 0 - for subpart in it: - subparts += 1 - lines.append(subpart.get_payload()) - eq(subparts, 1) - eq(EMPTYSTRING.join(lines), """\ - -Hi, - -Do you like this message? - --Me -""") - - def test_pushCR_LF(self): - '''FeedParser BufferedSubFile.push() assumed it received complete - line endings. A CR ending one push() followed by a LF starting - the next push() added an empty line. - ''' - imt = [ - ("a\r \n", 2), - ("b", 0), - ("c\n", 1), - ("", 0), - ("d\r\n", 1), - ("e\r", 0), - ("\nf", 1), - ("\r\n", 1), - ] - from email.feedparser import BufferedSubFile, NeedMoreData - bsf = BufferedSubFile() - om = [] - nt = 0 - for il, n in imt: - bsf.push(il) - nt += n - n1 = 0 - while True: - ol = bsf.readline() - if ol == NeedMoreData: - break - om.append(ol) - n1 += 1 - self.assertTrue(n == n1) - self.assertTrue(len(om) == nt) - self.assertTrue(''.join([il for il, n in imt]) == ''.join(om)) - - - -class TestParsers(TestEmailBase): - def test_header_parser(self): - eq = self.assertEqual - # Parse only the headers of a complex multipart MIME document - fp = openfile('msg_02.txt') - try: - msg = HeaderParser().parse(fp) - finally: - fp.close() - eq(msg['from'], 'ppp-request at zzz.org') - eq(msg['to'], 'ppp at zzz.org') - eq(msg.get_content_type(), 'multipart/mixed') - self.assertFalse(msg.is_multipart()) - self.assertTrue(isinstance(msg.get_payload(), str)) - - def test_whitespace_continuation(self): - eq = self.assertEqual - # This message contains a line after the Subject: header that has only - # whitespace, but it is not empty! - msg = email.message_from_string("""\ -From: aperson at dom.ain -To: bperson at dom.ain -Subject: the next line has a space on it -\x20 -Date: Mon, 8 Apr 2002 15:09:19 -0400 -Message-ID: spam - -Here's the message body -""") - eq(msg['subject'], 'the next line has a space on it\n ') - eq(msg['message-id'], 'spam') - eq(msg.get_payload(), "Here's the message body\n") - - def test_whitespace_continuation_last_header(self): - eq = self.assertEqual - # Like the previous test, but the subject line is the last - # header. - msg = email.message_from_string("""\ -From: aperson at dom.ain -To: bperson at dom.ain -Date: Mon, 8 Apr 2002 15:09:19 -0400 -Message-ID: spam -Subject: the next line has a space on it -\x20 - -Here's the message body -""") - eq(msg['subject'], 'the next line has a space on it\n ') - eq(msg['message-id'], 'spam') - eq(msg.get_payload(), "Here's the message body\n") - - def test_crlf_separation(self): - eq = self.assertEqual - fp = openfile('msg_26.txt', mode='rb') - try: - msg = Parser().parse(fp) - finally: - fp.close() - eq(len(msg.get_payload()), 2) - part1 = msg.get_payload(0) - eq(part1.get_content_type(), 'text/plain') - eq(part1.get_payload(), 'Simple email with attachment.\r\n\r\n') - part2 = msg.get_payload(1) - eq(part2.get_content_type(), 'application/riscos') - - def test_multipart_digest_with_extra_mime_headers(self): - eq = self.assertEqual - neq = self.ndiffAssertEqual - fp = openfile('msg_28.txt') - try: - msg = email.message_from_file(fp) - finally: - fp.close() - # Structure is: - # multipart/digest - # message/rfc822 - # text/plain - # message/rfc822 - # text/plain - eq(msg.is_multipart(), 1) - eq(len(msg.get_payload()), 2) - part1 = msg.get_payload(0) - eq(part1.get_content_type(), 'message/rfc822') - eq(part1.is_multipart(), 1) - eq(len(part1.get_payload()), 1) - part1a = part1.get_payload(0) - eq(part1a.is_multipart(), 0) - eq(part1a.get_content_type(), 'text/plain') - neq(part1a.get_payload(), 'message 1\n') - # next message/rfc822 - part2 = msg.get_payload(1) - eq(part2.get_content_type(), 'message/rfc822') - eq(part2.is_multipart(), 1) - eq(len(part2.get_payload()), 1) - part2a = part2.get_payload(0) - eq(part2a.is_multipart(), 0) - eq(part2a.get_content_type(), 'text/plain') - neq(part2a.get_payload(), 'message 2\n') - - def test_three_lines(self): - # A bug report by Andrew McNamara - lines = ['From: Andrew Person <aperson at dom.ain', - 'Subject: Test', - 'Date: Tue, 20 Aug 2002 16:43:45 +1000'] - msg = email.message_from_string(NL.join(lines)) - self.assertEqual(msg['date'], 'Tue, 20 Aug 2002 16:43:45 +1000') - - def test_strip_line_feed_and_carriage_return_in_headers(self): - eq = self.assertEqual - # For [ 1002475 ] email message parser doesn't handle \r\n correctly - value1 = 'text' - value2 = 'more text' - m = 'Header: %s\r\nNext-Header: %s\r\n\r\nBody\r\n\r\n' % ( - value1, value2) - msg = email.message_from_string(m) - eq(msg.get('Header'), value1) - eq(msg.get('Next-Header'), value2) - - def test_rfc2822_header_syntax(self): - eq = self.assertEqual - m = '>From: foo\nFrom: bar\n!"#QUX;~: zoo\n\nbody' - msg = email.message_from_string(m) - eq(len(msg.keys()), 3) - keys = msg.keys() - keys.sort() - eq(keys, ['!"#QUX;~', '>From', 'From']) - eq(msg.get_payload(), 'body') - - def test_rfc2822_space_not_allowed_in_header(self): - eq = self.assertEqual - m = '>From foo at example.com 11:25:53\nFrom: bar\n!"#QUX;~: zoo\n\nbody' - msg = email.message_from_string(m) - eq(len(msg.keys()), 0) - - def test_rfc2822_one_character_header(self): - eq = self.assertEqual - m = 'A: first header\nB: second header\nCC: third header\n\nbody' - msg = email.message_from_string(m) - headers = msg.keys() - headers.sort() - eq(headers, ['A', 'B', 'CC']) - eq(msg.get_payload(), 'body') - - def test_CRLFLF_at_end_of_part(self): - # issue 5610: feedparser should not eat two chars from body part ending - # with "\r\n\n". - m = ( - "From: foo at bar.com\n" - "To: baz\n" - "Mime-Version: 1.0\n" - "Content-Type: multipart/mixed; boundary=BOUNDARY\n" - "\n" - "--BOUNDARY\n" - "Content-Type: text/plain\n" - "\n" - "body ending with CRLF newline\r\n" - "\n" - "--BOUNDARY--\n" - ) - msg = email.message_from_string(m) - self.assertTrue(msg.get_payload(0).get_payload().endswith('\r\n')) - - -class TestBase64(unittest.TestCase): - def test_len(self): - eq = self.assertEqual - eq(base64MIME.base64_len('hello'), - len(base64MIME.encode('hello', eol=''))) - for size in range(15): - if size == 0 : bsize = 0 - elif size <= 3 : bsize = 4 - elif size <= 6 : bsize = 8 - elif size <= 9 : bsize = 12 - elif size <= 12: bsize = 16 - else : bsize = 20 - eq(base64MIME.base64_len('x'*size), bsize) - - def test_decode(self): - eq = self.assertEqual - eq(base64MIME.decode(''), '') - eq(base64MIME.decode('aGVsbG8='), 'hello') - eq(base64MIME.decode('aGVsbG8=', 'X'), 'hello') - eq(base64MIME.decode('aGVsbG8NCndvcmxk\n', 'X'), 'helloXworld') - - def test_encode(self): - eq = self.assertEqual - eq(base64MIME.encode(''), '') - eq(base64MIME.encode('hello'), 'aGVsbG8=\n') - # Test the binary flag - eq(base64MIME.encode('hello\n'), 'aGVsbG8K\n') - eq(base64MIME.encode('hello\n', 0), 'aGVsbG8NCg==\n') - # Test the maxlinelen arg - eq(base64MIME.encode('xxxx ' * 20, maxlinelen=40), """\ -eHh4eCB4eHh4IHh4eHggeHh4eCB4eHh4IHh4eHgg -eHh4eCB4eHh4IHh4eHggeHh4eCB4eHh4IHh4eHgg -eHh4eCB4eHh4IHh4eHggeHh4eCB4eHh4IHh4eHgg -eHh4eCB4eHh4IA== -""") - # Test the eol argument - eq(base64MIME.encode('xxxx ' * 20, maxlinelen=40, eol='\r\n'), """\ -eHh4eCB4eHh4IHh4eHggeHh4eCB4eHh4IHh4eHgg\r -eHh4eCB4eHh4IHh4eHggeHh4eCB4eHh4IHh4eHgg\r -eHh4eCB4eHh4IHh4eHggeHh4eCB4eHh4IHh4eHgg\r -eHh4eCB4eHh4IA==\r -""") - - def test_header_encode(self): - eq = self.assertEqual - he = base64MIME.header_encode - eq(he('hello'), '=?iso-8859-1?b?aGVsbG8=?=') - eq(he('hello\nworld'), '=?iso-8859-1?b?aGVsbG8NCndvcmxk?=') - # Test the charset option - eq(he('hello', charset='iso-8859-2'), '=?iso-8859-2?b?aGVsbG8=?=') - # Test the keep_eols flag - eq(he('hello\nworld', keep_eols=True), - '=?iso-8859-1?b?aGVsbG8Kd29ybGQ=?=') - # Test the maxlinelen argument - eq(he('xxxx ' * 20, maxlinelen=40), """\ -=?iso-8859-1?b?eHh4eCB4eHh4IHh4eHggeHg=?= - =?iso-8859-1?b?eHggeHh4eCB4eHh4IHh4eHg=?= - =?iso-8859-1?b?IHh4eHggeHh4eCB4eHh4IHg=?= - =?iso-8859-1?b?eHh4IHh4eHggeHh4eCB4eHg=?= - =?iso-8859-1?b?eCB4eHh4IHh4eHggeHh4eCA=?= - =?iso-8859-1?b?eHh4eCB4eHh4IHh4eHgg?=""") - # Test the eol argument - eq(he('xxxx ' * 20, maxlinelen=40, eol='\r\n'), """\ -=?iso-8859-1?b?eHh4eCB4eHh4IHh4eHggeHg=?=\r - =?iso-8859-1?b?eHggeHh4eCB4eHh4IHh4eHg=?=\r - =?iso-8859-1?b?IHh4eHggeHh4eCB4eHh4IHg=?=\r - =?iso-8859-1?b?eHh4IHh4eHggeHh4eCB4eHg=?=\r - =?iso-8859-1?b?eCB4eHh4IHh4eHggeHh4eCA=?=\r - =?iso-8859-1?b?eHh4eCB4eHh4IHh4eHgg?=""") - - - -class TestQuopri(unittest.TestCase): - def setUp(self): - self.hlit = [chr(x) for x in range(ord('a'), ord('z')+1)] + \ - [chr(x) for x in range(ord('A'), ord('Z')+1)] + \ - [chr(x) for x in range(ord('0'), ord('9')+1)] + \ - ['!', '*', '+', '-', '/', ' '] - self.hnon = [chr(x) for x in range(256) if chr(x) not in self.hlit] - assert len(self.hlit) + len(self.hnon) == 256 - self.blit = [chr(x) for x in range(ord(' '), ord('~')+1)] + ['\t'] - self.blit.remove('=') - self.bnon = [chr(x) for x in range(256) if chr(x) not in self.blit] - assert len(self.blit) + len(self.bnon) == 256 - - def test_header_quopri_check(self): - for c in self.hlit: - self.assertFalse(quopriMIME.header_quopri_check(c)) - for c in self.hnon: - self.assertTrue(quopriMIME.header_quopri_check(c)) - - def test_body_quopri_check(self): - for c in self.blit: - self.assertFalse(quopriMIME.body_quopri_check(c)) - for c in self.bnon: - self.assertTrue(quopriMIME.body_quopri_check(c)) - - def test_header_quopri_len(self): - eq = self.assertEqual - hql = quopriMIME.header_quopri_len - enc = quopriMIME.header_encode - for s in ('hello', 'h at e@l at l@o@'): - # Empty charset and no line-endings. 7 == RFC chrome - eq(hql(s), len(enc(s, charset='', eol=''))-7) - for c in self.hlit: - eq(hql(c), 1) - for c in self.hnon: - eq(hql(c), 3) - - def test_body_quopri_len(self): - eq = self.assertEqual - bql = quopriMIME.body_quopri_len - for c in self.blit: - eq(bql(c), 1) - for c in self.bnon: - eq(bql(c), 3) - - def test_quote_unquote_idempotent(self): - for x in range(256): - c = chr(x) - self.assertEqual(quopriMIME.unquote(quopriMIME.quote(c)), c) - - def test_header_encode(self): - eq = self.assertEqual - he = quopriMIME.header_encode - eq(he('hello'), '=?iso-8859-1?q?hello?=') - eq(he('hello\nworld'), '=?iso-8859-1?q?hello=0D=0Aworld?=') - # Test the charset option - eq(he('hello', charset='iso-8859-2'), '=?iso-8859-2?q?hello?=') - # Test the keep_eols flag - eq(he('hello\nworld', keep_eols=True), '=?iso-8859-1?q?hello=0Aworld?=') - # Test a non-ASCII character - eq(he('hello\xc7there'), '=?iso-8859-1?q?hello=C7there?=') - # Test the maxlinelen argument - eq(he('xxxx ' * 20, maxlinelen=40), """\ -=?iso-8859-1?q?xxxx_xxxx_xxxx_xxxx_xx?= - =?iso-8859-1?q?xx_xxxx_xxxx_xxxx_xxxx?= - =?iso-8859-1?q?_xxxx_xxxx_xxxx_xxxx_x?= - =?iso-8859-1?q?xxx_xxxx_xxxx_xxxx_xxx?= - =?iso-8859-1?q?x_xxxx_xxxx_?=""") - # Test the eol argument - eq(he('xxxx ' * 20, maxlinelen=40, eol='\r\n'), """\ -=?iso-8859-1?q?xxxx_xxxx_xxxx_xxxx_xx?=\r - =?iso-8859-1?q?xx_xxxx_xxxx_xxxx_xxxx?=\r - =?iso-8859-1?q?_xxxx_xxxx_xxxx_xxxx_x?=\r - =?iso-8859-1?q?xxx_xxxx_xxxx_xxxx_xxx?=\r - =?iso-8859-1?q?x_xxxx_xxxx_?=""") - - def test_decode(self): - eq = self.assertEqual - eq(quopriMIME.decode(''), '') - eq(quopriMIME.decode('hello'), 'hello') - eq(quopriMIME.decode('hello', 'X'), 'hello') - eq(quopriMIME.decode('hello\nworld', 'X'), 'helloXworld') - - def test_encode(self): - eq = self.assertEqual - eq(quopriMIME.encode(''), '') - eq(quopriMIME.encode('hello'), 'hello') - # Test the binary flag - eq(quopriMIME.encode('hello\r\nworld'), 'hello\nworld') - eq(quopriMIME.encode('hello\r\nworld', 0), 'hello\nworld') - # Test the maxlinelen arg - eq(quopriMIME.encode('xxxx ' * 20, maxlinelen=40), """\ -xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxxx= - xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxx= -x xxxx xxxx xxxx xxxx=20""") - # Test the eol argument - eq(quopriMIME.encode('xxxx ' * 20, maxlinelen=40, eol='\r\n'), """\ -xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxxx=\r - xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxx=\r -x xxxx xxxx xxxx xxxx=20""") - eq(quopriMIME.encode("""\ -one line - -two line"""), """\ -one line - -two line""") - - - -# Test the Charset class -class TestCharset(unittest.TestCase): - def tearDown(self): - from email import Charset as CharsetModule - try: - del CharsetModule.CHARSETS['fake'] - except KeyError: - pass - - def test_idempotent(self): - eq = self.assertEqual - # Make sure us-ascii = no Unicode conversion - c = Charset('us-ascii') - s = 'Hello World!' - sp = c.to_splittable(s) - eq(s, c.from_splittable(sp)) - # test 8-bit idempotency with us-ascii - s = '\xa4\xa2\xa4\xa4\xa4\xa6\xa4\xa8\xa4\xaa' - sp = c.to_splittable(s) - eq(s, c.from_splittable(sp)) - - def test_body_encode(self): - eq = self.assertEqual - # Try a charset with QP body encoding - c = Charset('iso-8859-1') - eq('hello w=F6rld', c.body_encode('hello w\xf6rld')) - # Try a charset with Base64 body encoding - c = Charset('utf-8') - eq('aGVsbG8gd29ybGQ=\n', c.body_encode('hello world')) - # Try a charset with None body encoding - c = Charset('us-ascii') - eq('hello world', c.body_encode('hello world')) - # Try the convert argument, where input codec != output codec - c = Charset('euc-jp') - # With apologies to Tokio Kikuchi ;) - try: - eq('\x1b$B5FCO;~IW\x1b(B', - c.body_encode('\xb5\xc6\xc3\xcf\xbb\xfe\xc9\xd7')) - eq('\xb5\xc6\xc3\xcf\xbb\xfe\xc9\xd7', - c.body_encode('\xb5\xc6\xc3\xcf\xbb\xfe\xc9\xd7', False)) - except LookupError: - # We probably don't have the Japanese codecs installed - pass - # Testing SF bug #625509, which we have to fake, since there are no - # built-in encodings where the header encoding is QP but the body - # encoding is not. - from email import Charset as CharsetModule - CharsetModule.add_charset('fake', CharsetModule.QP, None) - c = Charset('fake') - eq('hello w\xf6rld', c.body_encode('hello w\xf6rld')) - - def test_unicode_charset_name(self): - charset = Charset(u'us-ascii') - self.assertEqual(str(charset), 'us-ascii') - self.assertRaises(Errors.CharsetError, Charset, 'asc\xffii') - - def test_codecs_aliases_accepted(self): - charset = Charset('utf8') - self.assertEqual(str(charset), 'utf-8') - - -# Test multilingual MIME headers. -class TestHeader(TestEmailBase): - def test_simple(self): - eq = self.ndiffAssertEqual - h = Header('Hello World!') - eq(h.encode(), 'Hello World!') - h.append(' Goodbye World!') - eq(h.encode(), 'Hello World! Goodbye World!') - - def test_simple_surprise(self): - eq = self.ndiffAssertEqual - h = Header('Hello World!') - eq(h.encode(), 'Hello World!') - h.append('Goodbye World!') - eq(h.encode(), 'Hello World! Goodbye World!') - - def test_header_needs_no_decoding(self): - h = 'no decoding needed' - self.assertEqual(decode_header(h), [(h, None)]) - - def test_long(self): - h = Header("I am the very model of a modern Major-General; I've information vegetable, animal, and mineral; I know the kings of England, and I quote the fights historical from Marathon to Waterloo, in order categorical; I'm very well acquainted, too, with matters mathematical; I understand equations, both the simple and quadratical; about binomial theorem I'm teeming with a lot o' news, with many cheerful facts about the square of the hypotenuse.", - maxlinelen=76) - for l in h.encode(splitchars=' ').split('\n '): - self.assertTrue(len(l) <= 76) - - def test_multilingual(self): - eq = self.ndiffAssertEqual - g = Charset("iso-8859-1") - cz = Charset("iso-8859-2") - utf8 = Charset("utf-8") - g_head = "Die Mieter treten hier ein werden mit einem Foerderband komfortabel den Korridor entlang, an s\xfcdl\xfcndischen Wandgem\xe4lden vorbei, gegen die rotierenden Klingen bef\xf6rdert. " - cz_head = "Finan\xe8ni metropole se hroutily pod tlakem jejich d\xf9vtipu.. " - utf8_head = u"\u6b63\u78ba\u306b\u8a00\u3046\u3068\u7ffb\u8a33\u306f\u3055\u308c\u3066\u3044\u307e\u305b\u3093\u3002\u4e00\u90e8\u306f\u30c9\u30a4\u30c4\u8a9e\u3067\u3059\u304c\u3001\u3042\u3068\u306f\u3067\u305f\u3089\u3081\u3067\u3059\u3002\u5b9f\u969b\u306b\u306f\u300cWenn ist das Nunstuck git und Slotermeyer? Ja! Beiherhund das Oder die Flipperwaldt gersput.\u300d\u3068\u8a00\u3063\u3066\u3044\u307e\u3059\u3002".encode("utf-8") - h = Header(g_head, g) - h.append(cz_head, cz) - h.append(utf8_head, utf8) - enc = h.encode() - eq(enc, """\ -=?iso-8859-1?q?Die_Mieter_treten_hier_ein_werden_mit_einem_Foerderband_ko?= - =?iso-8859-1?q?mfortabel_den_Korridor_entlang=2C_an_s=FCdl=FCndischen_Wan?= - =?iso-8859-1?q?dgem=E4lden_vorbei=2C_gegen_die_rotierenden_Klingen_bef=F6?= - =?iso-8859-1?q?rdert=2E_?= =?iso-8859-2?q?Finan=E8ni_metropole_se_hroutily?= - =?iso-8859-2?q?_pod_tlakem_jejich_d=F9vtipu=2E=2E_?= =?utf-8?b?5q2j56K6?= - =?utf-8?b?44Gr6KiA44GG44Go57+76Kiz44Gv44GV44KM44Gm44GE44G+44Gb44KT44CC?= - =?utf-8?b?5LiA6YOo44Gv44OJ44Kk44OE6Kqe44Gn44GZ44GM44CB44GC44Go44Gv44Gn?= - =?utf-8?b?44Gf44KJ44KB44Gn44GZ44CC5a6f6Zqb44Gr44Gv44CMV2VubiBpc3QgZGFz?= - =?utf-8?q?_Nunstuck_git_und_Slotermeyer=3F_Ja!_Beiherhund_das_Oder_die_Fl?= - =?utf-8?b?aXBwZXJ3YWxkdCBnZXJzcHV0LuOAjeOBqOiogOOBo+OBpuOBhOOBvuOBmQ==?= - =?utf-8?b?44CC?=""") - eq(decode_header(enc), - [(g_head, "iso-8859-1"), (cz_head, "iso-8859-2"), - (utf8_head, "utf-8")]) - ustr = unicode(h) - eq(ustr.encode('utf-8'), - 'Die Mieter treten hier ein werden mit einem Foerderband ' - 'komfortabel den Korridor entlang, an s\xc3\xbcdl\xc3\xbcndischen ' - 'Wandgem\xc3\xa4lden vorbei, gegen die rotierenden Klingen ' - 'bef\xc3\xb6rdert. Finan\xc4\x8dni metropole se hroutily pod ' - 'tlakem jejich d\xc5\xafvtipu.. \xe6\xad\xa3\xe7\xa2\xba\xe3\x81' - '\xab\xe8\xa8\x80\xe3\x81\x86\xe3\x81\xa8\xe7\xbf\xbb\xe8\xa8\xb3' - '\xe3\x81\xaf\xe3\x81\x95\xe3\x82\x8c\xe3\x81\xa6\xe3\x81\x84\xe3' - '\x81\xbe\xe3\x81\x9b\xe3\x82\x93\xe3\x80\x82\xe4\xb8\x80\xe9\x83' - '\xa8\xe3\x81\xaf\xe3\x83\x89\xe3\x82\xa4\xe3\x83\x84\xe8\xaa\x9e' - '\xe3\x81\xa7\xe3\x81\x99\xe3\x81\x8c\xe3\x80\x81\xe3\x81\x82\xe3' - '\x81\xa8\xe3\x81\xaf\xe3\x81\xa7\xe3\x81\x9f\xe3\x82\x89\xe3\x82' - '\x81\xe3\x81\xa7\xe3\x81\x99\xe3\x80\x82\xe5\xae\x9f\xe9\x9a\x9b' - '\xe3\x81\xab\xe3\x81\xaf\xe3\x80\x8cWenn ist das Nunstuck git ' - 'und Slotermeyer? Ja! Beiherhund das Oder die Flipperwaldt ' - 'gersput.\xe3\x80\x8d\xe3\x81\xa8\xe8\xa8\x80\xe3\x81\xa3\xe3\x81' - '\xa6\xe3\x81\x84\xe3\x81\xbe\xe3\x81\x99\xe3\x80\x82') - # Test make_header() - newh = make_header(decode_header(enc)) - eq(newh, enc) - - def test_header_ctor_default_args(self): - eq = self.ndiffAssertEqual - h = Header() - eq(h, '') - h.append('foo', Charset('iso-8859-1')) - eq(h, '=?iso-8859-1?q?foo?=') - - def test_explicit_maxlinelen(self): - eq = self.ndiffAssertEqual - hstr = 'A very long line that must get split to something other than at the 76th character boundary to test the non-default behavior' - h = Header(hstr) - eq(h.encode(), '''\ -A very long line that must get split to something other than at the 76th - character boundary to test the non-default behavior''') - h = Header(hstr, header_name='Subject') - eq(h.encode(), '''\ -A very long line that must get split to something other than at the - 76th character boundary to test the non-default behavior''') - h = Header(hstr, maxlinelen=1024, header_name='Subject') - eq(h.encode(), hstr) - - def test_us_ascii_header(self): - eq = self.assertEqual - s = 'hello' - x = decode_header(s) - eq(x, [('hello', None)]) - h = make_header(x) - eq(s, h.encode()) - - def test_string_charset(self): - eq = self.assertEqual - h = Header() - h.append('hello', 'iso-8859-1') - eq(h, '=?iso-8859-1?q?hello?=') - -## def test_unicode_error(self): -## raises = self.assertRaises -## raises(UnicodeError, Header, u'[P\xf6stal]', 'us-ascii') -## raises(UnicodeError, Header, '[P\xf6stal]', 'us-ascii') -## h = Header() -## raises(UnicodeError, h.append, u'[P\xf6stal]', 'us-ascii') -## raises(UnicodeError, h.append, '[P\xf6stal]', 'us-ascii') -## raises(UnicodeError, Header, u'\u83ca\u5730\u6642\u592b', 'iso-8859-1') - - def test_utf8_shortest(self): - eq = self.assertEqual - h = Header(u'p\xf6stal', 'utf-8') - eq(h.encode(), '=?utf-8?q?p=C3=B6stal?=') - h = Header(u'\u83ca\u5730\u6642\u592b', 'utf-8') - eq(h.encode(), '=?utf-8?b?6I+K5Zyw5pmC5aSr?=') - - def test_bad_8bit_header(self): - raises = self.assertRaises - eq = self.assertEqual - x = 'Ynwp4dUEbay Auction Semiar- No Charge \x96 Earn Big' - raises(UnicodeError, Header, x) - h = Header() - raises(UnicodeError, h.append, x) - eq(str(Header(x, errors='replace')), x) - h.append(x, errors='replace') - eq(str(h), x) - - def test_encoded_adjacent_nonencoded(self): - eq = self.assertEqual - h = Header() - h.append('hello', 'iso-8859-1') - h.append('world') - s = h.encode() - eq(s, '=?iso-8859-1?q?hello?= world') - h = make_header(decode_header(s)) - eq(h.encode(), s) - - def test_whitespace_eater(self): - eq = self.assertEqual - s = 'Subject: =?koi8-r?b?8NLP18XSy8EgzsEgxsnOwczYztk=?= =?koi8-r?q?=CA?= zz.' - parts = decode_header(s) - eq(parts, [('Subject:', None), ('\xf0\xd2\xcf\xd7\xc5\xd2\xcb\xc1 \xce\xc1 \xc6\xc9\xce\xc1\xcc\xd8\xce\xd9\xca', 'koi8-r'), ('zz.', None)]) - hdr = make_header(parts) - eq(hdr.encode(), - 'Subject: =?koi8-r?b?8NLP18XSy8EgzsEgxsnOwczYztnK?= zz.') - - def test_broken_base64_header(self): - raises = self.assertRaises - s = 'Subject: =?EUC-KR?B?CSixpLDtKSC/7Liuvsax4iC6uLmwMcijIKHaILzSwd/H0SC8+LCjwLsgv7W/+Mj3I ?=' - raises(Errors.HeaderParseError, decode_header, s) - - # Issue 1078919 - def test_ascii_add_header(self): - msg = Message() - msg.add_header('Content-Disposition', 'attachment', - filename='bud.gif') - self.assertEqual('attachment; filename="bud.gif"', - msg['Content-Disposition']) - - def test_nonascii_add_header_via_triple(self): - msg = Message() - msg.add_header('Content-Disposition', 'attachment', - filename=('iso-8859-1', '', 'Fu\xdfballer.ppt')) - self.assertEqual( - 'attachment; filename*="iso-8859-1\'\'Fu%DFballer.ppt"', - msg['Content-Disposition']) - - def test_encode_unaliased_charset(self): - # Issue 1379416: when the charset has no output conversion, - # output was accidentally getting coerced to unicode. - res = Header('abc','iso-8859-2').encode() - self.assertEqual(res, '=?iso-8859-2?q?abc?=') - self.assertIsInstance(res, str) - - -# Test RFC 2231 header parameters (en/de)coding -class TestRFC2231(TestEmailBase): - def test_get_param(self): - eq = self.assertEqual - msg = self._msgobj('msg_29.txt') - eq(msg.get_param('title'), - ('us-ascii', 'en', 'This is even more ***fun*** isn\'t it!')) - eq(msg.get_param('title', unquote=False), - ('us-ascii', 'en', '"This is even more ***fun*** isn\'t it!"')) - - def test_set_param(self): - eq = self.assertEqual - msg = Message() - msg.set_param('title', 'This is even more ***fun*** isn\'t it!', - charset='us-ascii') - eq(msg.get_param('title'), - ('us-ascii', '', 'This is even more ***fun*** isn\'t it!')) - msg.set_param('title', 'This is even more ***fun*** isn\'t it!', - charset='us-ascii', language='en') - eq(msg.get_param('title'), - ('us-ascii', 'en', 'This is even more ***fun*** isn\'t it!')) - msg = self._msgobj('msg_01.txt') - msg.set_param('title', 'This is even more ***fun*** isn\'t it!', - charset='us-ascii', language='en') - self.ndiffAssertEqual(msg.as_string(), """\ -Return-Path: <bbb at zzz.org> -Delivered-To: bbb at zzz.org -Received: by mail.zzz.org (Postfix, from userid 889) - id 27CEAD38CC; Fri, 4 May 2001 14:05:44 -0400 (EDT) -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit -Message-ID: <15090.61304.110929.45684 at aaa.zzz.org> -From: bbb at ddd.com (John X. Doe) -To: bbb at zzz.org -Subject: This is a test message -Date: Fri, 4 May 2001 14:05:44 -0400 -Content-Type: text/plain; charset=us-ascii; - title*="us-ascii'en'This%20is%20even%20more%20%2A%2A%2Afun%2A%2A%2A%20isn%27t%20it%21" - - -Hi, - -Do you like this message? - --Me -""") - - def test_del_param(self): - eq = self.ndiffAssertEqual - msg = self._msgobj('msg_01.txt') - msg.set_param('foo', 'bar', charset='us-ascii', language='en') - msg.set_param('title', 'This is even more ***fun*** isn\'t it!', - charset='us-ascii', language='en') - msg.del_param('foo', header='Content-Type') - eq(msg.as_string(), """\ -Return-Path: <bbb at zzz.org> -Delivered-To: bbb at zzz.org -Received: by mail.zzz.org (Postfix, from userid 889) - id 27CEAD38CC; Fri, 4 May 2001 14:05:44 -0400 (EDT) -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit -Message-ID: <15090.61304.110929.45684 at aaa.zzz.org> -From: bbb at ddd.com (John X. Doe) -To: bbb at zzz.org -Subject: This is a test message -Date: Fri, 4 May 2001 14:05:44 -0400 -Content-Type: text/plain; charset="us-ascii"; - title*="us-ascii'en'This%20is%20even%20more%20%2A%2A%2Afun%2A%2A%2A%20isn%27t%20it%21" - - -Hi, - -Do you like this message? - --Me -""") - - def test_rfc2231_get_content_charset(self): - eq = self.assertEqual - msg = self._msgobj('msg_32.txt') - eq(msg.get_content_charset(), 'us-ascii') - - def test_rfc2231_no_language_or_charset(self): - m = '''\ -Content-Transfer-Encoding: 8bit -Content-Disposition: inline; filename="file____C__DOCUMENTS_20AND_20SETTINGS_FABIEN_LOCAL_20SETTINGS_TEMP_nsmail.htm" -Content-Type: text/html; NAME*0=file____C__DOCUMENTS_20AND_20SETTINGS_FABIEN_LOCAL_20SETTINGS_TEM; NAME*1=P_nsmail.htm - -''' - msg = email.message_from_string(m) - param = msg.get_param('NAME') - self.assertFalse(isinstance(param, tuple)) - self.assertEqual( - param, - 'file____C__DOCUMENTS_20AND_20SETTINGS_FABIEN_LOCAL_20SETTINGS_TEMP_nsmail.htm') - - def test_rfc2231_no_language_or_charset_in_filename(self): - m = '''\ -Content-Disposition: inline; -\tfilename*0*="''This%20is%20even%20more%20"; -\tfilename*1*="%2A%2A%2Afun%2A%2A%2A%20"; -\tfilename*2="is it not.pdf" - -''' - msg = email.message_from_string(m) - self.assertEqual(msg.get_filename(), - 'This is even more ***fun*** is it not.pdf') - - def test_rfc2231_no_language_or_charset_in_filename_encoded(self): - m = '''\ -Content-Disposition: inline; -\tfilename*0*="''This%20is%20even%20more%20"; -\tfilename*1*="%2A%2A%2Afun%2A%2A%2A%20"; -\tfilename*2="is it not.pdf" - -''' - msg = email.message_from_string(m) - self.assertEqual(msg.get_filename(), - 'This is even more ***fun*** is it not.pdf') - - def test_rfc2231_partly_encoded(self): - m = '''\ -Content-Disposition: inline; -\tfilename*0="''This%20is%20even%20more%20"; -\tfilename*1*="%2A%2A%2Afun%2A%2A%2A%20"; -\tfilename*2="is it not.pdf" - -''' - msg = email.message_from_string(m) - self.assertEqual( - msg.get_filename(), - 'This%20is%20even%20more%20***fun*** is it not.pdf') - - def test_rfc2231_partly_nonencoded(self): - m = '''\ -Content-Disposition: inline; -\tfilename*0="This%20is%20even%20more%20"; -\tfilename*1="%2A%2A%2Afun%2A%2A%2A%20"; -\tfilename*2="is it not.pdf" - -''' - msg = email.message_from_string(m) - self.assertEqual( - msg.get_filename(), - 'This%20is%20even%20more%20%2A%2A%2Afun%2A%2A%2A%20is it not.pdf') - - def test_rfc2231_no_language_or_charset_in_boundary(self): - m = '''\ -Content-Type: multipart/alternative; -\tboundary*0*="''This%20is%20even%20more%20"; -\tboundary*1*="%2A%2A%2Afun%2A%2A%2A%20"; -\tboundary*2="is it not.pdf" - -''' - msg = email.message_from_string(m) - self.assertEqual(msg.get_boundary(), - 'This is even more ***fun*** is it not.pdf') - - def test_rfc2231_no_language_or_charset_in_charset(self): - # This is a nonsensical charset value, but tests the code anyway - m = '''\ -Content-Type: text/plain; -\tcharset*0*="This%20is%20even%20more%20"; -\tcharset*1*="%2A%2A%2Afun%2A%2A%2A%20"; -\tcharset*2="is it not.pdf" - -''' - msg = email.message_from_string(m) - self.assertEqual(msg.get_content_charset(), - 'this is even more ***fun*** is it not.pdf') - - def test_rfc2231_bad_encoding_in_filename(self): - m = '''\ -Content-Disposition: inline; -\tfilename*0*="bogus'xx'This%20is%20even%20more%20"; -\tfilename*1*="%2A%2A%2Afun%2A%2A%2A%20"; -\tfilename*2="is it not.pdf" - -''' - msg = email.message_from_string(m) - self.assertEqual(msg.get_filename(), - 'This is even more ***fun*** is it not.pdf') - - def test_rfc2231_bad_encoding_in_charset(self): - m = """\ -Content-Type: text/plain; charset*=bogus''utf-8%E2%80%9D - -""" - msg = email.message_from_string(m) - # This should return None because non-ascii characters in the charset - # are not allowed. - self.assertEqual(msg.get_content_charset(), None) - - def test_rfc2231_bad_character_in_charset(self): - m = """\ -Content-Type: text/plain; charset*=ascii''utf-8%E2%80%9D - -""" - msg = email.message_from_string(m) - # This should return None because non-ascii characters in the charset - # are not allowed. - self.assertEqual(msg.get_content_charset(), None) - - def test_rfc2231_bad_character_in_filename(self): - m = '''\ -Content-Disposition: inline; -\tfilename*0*="ascii'xx'This%20is%20even%20more%20"; -\tfilename*1*="%2A%2A%2Afun%2A%2A%2A%20"; -\tfilename*2*="is it not.pdf%E2" - -''' - msg = email.message_from_string(m) - self.assertEqual(msg.get_filename(), - u'This is even more ***fun*** is it not.pdf\ufffd') - - def test_rfc2231_unknown_encoding(self): - m = """\ -Content-Transfer-Encoding: 8bit -Content-Disposition: inline; filename*=X-UNKNOWN''myfile.txt - -""" - msg = email.message_from_string(m) - self.assertEqual(msg.get_filename(), 'myfile.txt') - - def test_rfc2231_single_tick_in_filename_extended(self): - eq = self.assertEqual From noreply at buildbot.pypy.org Mon Jan 30 22:58:29 2012 From: noreply at buildbot.pypy.org (amauryfa) Date: Mon, 30 Jan 2012 22:58:29 +0100 (CET) Subject: [pypy-commit] pypy string-NUL: (arigo, amaury) pass no_nul to SomeString constructor instead of changing Message-ID: <20120130215829.E53CD7106F3@wyvern.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc <amauryfa at gmail.com> Branch: string-NUL Changeset: r51983:8fcec5fcbb57 Date: 2012-01-30 22:56 +0100 http://bitbucket.org/pypy/pypy/changeset/8fcec5fcbb57/ Log: (arigo, amaury) pass no_nul to SomeString constructor instead of changing the property afterwards. + s_list_of_string is now a list of strings without NULs. diff --git a/pypy/annotation/binaryop.py b/pypy/annotation/binaryop.py --- a/pypy/annotation/binaryop.py +++ b/pypy/annotation/binaryop.py @@ -434,18 +434,15 @@ class __extend__(pairtype(SomeString, SomeString)): def union((str1, str2)): - result = SomeString(can_be_None=str1.can_be_None or str2.can_be_None) - if str1.no_nul and str2.no_nul: - result.no_nul = True - return result + can_be_None = str1.can_be_None or str2.can_be_None + no_nul = str1.no_nul and str2.no_nul + return SomeString(can_be_None=can_be_None, no_nul=no_nul) def add((str1, str2)): # propagate const-ness to help getattr(obj, 'prefix' + const_name) - result = SomeString() + result = SomeString(no_nul=str1.no_nul and str2.no_nul) if str1.is_immutable_constant() and str2.is_immutable_constant(): result.const = str1.const + str2.const - if str1.no_nul and str2.no_nul: - result.no_nul = True return result class __extend__(pairtype(SomeChar, SomeChar)): @@ -480,7 +477,6 @@ raise NotImplementedError( "string formatting mixing strings and unicode not supported") getbookkeeper().count('strformat', str, s_tuple) - result = SomeString() no_nul = str.no_nul for s_item in s_tuple.items: if isinstance(s_item, SomeFloat): @@ -490,9 +486,7 @@ else: no_nul = False break - if no_nul: - result.no_nul = True - return result + return SomeString(no_nul=no_nul) class __extend__(pairtype(SomeString, SomeObject)): diff --git a/pypy/annotation/bookkeeper.py b/pypy/annotation/bookkeeper.py --- a/pypy/annotation/bookkeeper.py +++ b/pypy/annotation/bookkeeper.py @@ -342,14 +342,11 @@ else: raise Exception("seeing a prebuilt long (value %s)" % hex(x)) elif issubclass(tp, str): # py.lib uses annotated str subclasses + no_nul = not '\x00' in x if len(x) == 1: - result = SomeChar() - if not '\x00' in x: - result.no_nul = True + result = SomeChar(no_nul=no_nul) else: - result = SomeString() - if not '\x00' in x: - result.no_nul = True + result = SomeString(no_nul=no_nul) elif tp is unicode: if len(x) == 1: result = SomeUnicodeCodePoint() diff --git a/pypy/annotation/listdef.py b/pypy/annotation/listdef.py --- a/pypy/annotation/listdef.py +++ b/pypy/annotation/listdef.py @@ -223,4 +223,5 @@ MOST_GENERAL_LISTDEF = ListDef(None, SomeObject()) -s_list_of_strings = SomeList(ListDef(None, SomeString(), resized = True)) +s_list_of_strings = SomeList(ListDef(None, SomeString(no_nul=True), + resized = True)) diff --git a/pypy/annotation/model.py b/pypy/annotation/model.py --- a/pypy/annotation/model.py +++ b/pypy/annotation/model.py @@ -232,26 +232,30 @@ can_be_None=False no_nul = False # No NUL character in the string. - def __init__(self, can_be_None=False): + def __init__(self, can_be_None=False, no_nul=False): if can_be_None: self.can_be_None = True + if no_nul: + self.no_nul = True def can_be_none(self): return self.can_be_None def nonnoneify(self): - result = SomeString(can_be_None=False) - if self.no_nul: - result.no_nul = True - return result + return SomeString(can_be_None=False, no_nul=self.no_nul) class SomeUnicodeString(SomeObject): "Stands for an object which is known to be an unicode string" knowntype = unicode immutable = True + can_be_None=False no_nul = False - def __init__(self, can_be_None=False): - self.can_be_None = can_be_None + + def __init__(self, can_be_None=False, no_nul=False): + if can_be_None: + self.can_be_None = True + if no_nul: + self.no_nul = True def can_be_none(self): return self.can_be_None @@ -262,14 +266,16 @@ class SomeChar(SomeString): "Stands for an object known to be a string of length 1." can_be_None = False - def __init__(self): # no 'can_be_None' argument here - pass + def __init__(self, no_nul=False): # no 'can_be_None' argument here + if no_nul: + self.no_nul = True class SomeUnicodeCodePoint(SomeUnicodeString): "Stands for an object known to be a unicode codepoint." can_be_None = False - def __init__(self): # no 'can_be_None' argument here - pass + def __init__(self, no_nul=False): # no 'can_be_None' argument here + if no_nul: + self.no_nul = True SomeString.basestringclass = SomeString SomeString.basecharclass = SomeChar @@ -510,8 +516,7 @@ s_None = SomePBC([], can_be_None=True) s_Bool = SomeBool() s_ImpossibleValue = SomeImpossibleValue() -s_Str0 = SomeString() -s_Str0.no_nul = True +s_Str0 = SomeString(no_nul=True) # ____________________________________________________________ # weakrefs diff --git a/pypy/annotation/unaryop.py b/pypy/annotation/unaryop.py --- a/pypy/annotation/unaryop.py +++ b/pypy/annotation/unaryop.py @@ -491,10 +491,8 @@ if isinstance(str, SomeUnicodeString): return immutablevalue(u"") return immutablevalue("") - result = str.basestringclass() - if str.no_nul and s_item.no_nul: - result.no_nul = True - return result + no_nul = str.no_nul and s_item.no_nul + return str.basestringclass(no_nul=no_nul) def iter(str): return SomeIterator(str) @@ -505,27 +503,20 @@ def method_split(str, patt, max=-1): getbookkeeper().count("str_split", str, patt) - s_item = str.basestringclass() - if str.no_nul: - s_item.no_nul = True - result = getbookkeeper().newlist(s_item) - return result + s_item = str.basestringclass(no_nul=str.no_nul) + return getbookkeeper().newlist(s_item) def method_rsplit(str, patt, max=-1): getbookkeeper().count("str_rsplit", str, patt) - result = getbookkeeper().newlist(str.basestringclass()) - if str.no_nul: - result.no_nul = True - return result + s_item = str.basestringclass(no_nul=str.no_nul) + return getbookkeeper().newlist(s_item) def method_replace(str, s1, s2): return str.basestringclass() def getslice(str, s_start, s_stop): check_negative_slice(s_start, s_stop) - result = str.basestringclass() - if str.no_nul: - result.no_nul = True + result = str.basestringclass(no_nul=str.no_nul) return result class __extend__(SomeUnicodeString): diff --git a/pypy/rpython/module/ll_os.py b/pypy/rpython/module/ll_os.py --- a/pypy/rpython/module/ll_os.py +++ b/pypy/rpython/module/ll_os.py @@ -31,11 +31,9 @@ from pypy.rlib import rgc from pypy.rlib.objectmodel import specialize -str0 = SomeString() -str0.no_nul = True +str0 = SomeString(no_nul=True) +unicode0 = SomeUnicodeString(no_nul=True) -unicode0 = SomeUnicodeString() -unicode0.no_nul = True def monkeypatch_rposix(posixfunc, unicodefunc, signature): func_name = posixfunc.__name__ diff --git a/pypy/translator/cli/test/runtest.py b/pypy/translator/cli/test/runtest.py --- a/pypy/translator/cli/test/runtest.py +++ b/pypy/translator/cli/test/runtest.py @@ -276,10 +276,7 @@ def get_annotation(x): if isinstance(x, basestring) and len(x) > 1: - result = SomeString() - if '\x00' not in x: - result.no_nul = True - return result + return SomeString(no_nul='\x00' not in x) else: return lltype_to_annotation(typeOf(x)) diff --git a/pypy/translator/driver.py b/pypy/translator/driver.py --- a/pypy/translator/driver.py +++ b/pypy/translator/driver.py @@ -185,7 +185,7 @@ if standalone: # the 'argv' parameter - inputtypes = [[annmodel.s_Str0]] + inputtypes = [s_list_of_strings] self.inputtypes = inputtypes if policy is None: From noreply at buildbot.pypy.org Mon Jan 30 23:09:15 2012 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 30 Jan 2012 23:09:15 +0100 (CET) Subject: [pypy-commit] pypy.org extradoc: update the bar Message-ID: <20120130220915.F21C47106F3@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: extradoc Changeset: r309:1476faec89fd Date: 2012-01-30 23:51 +0200 http://bitbucket.org/pypy/pypy.org/changeset/1476faec89fd/ Log: update the bar diff --git a/don3.html b/don3.html --- a/don3.html +++ b/don3.html @@ -8,12 +8,12 @@ <script> $(function() { $("#progressbar").progressbar({ - value: 68.0 + value: 68.9 }); }); </script> - $40820 of $60000 (68.0%) + $41329 of $60000 (68.0%) <div id="progressbar"> </div> From noreply at buildbot.pypy.org Mon Jan 30 23:09:17 2012 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 30 Jan 2012 23:09:17 +0100 (CET) Subject: [pypy-commit] pypy.org extradoc: merge Message-ID: <20120130220917.1E4337106F3@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: extradoc Changeset: r310:7e272a0d59d0 Date: 2012-01-31 00:08 +0200 http://bitbucket.org/pypy/pypy.org/changeset/7e272a0d59d0/ Log: merge diff --git a/download.html b/download.html --- a/download.html +++ b/download.html @@ -56,7 +56,10 @@ <li><a class="reference internal" href="#other-versions">Other versions</a></li> </ul> </li> -<li><a class="reference internal" href="#installing">Installing</a> (optional)</li> +<li><a class="reference internal" href="#installing">Installing</a> (optional)<ul> +<li><a class="reference internal" href="#linux-libraries">Linux libraries</a></li> +</ul> +</li> <li><a class="reference internal" href="#building-from-source">Building from source</a></li> <li><a class="reference internal" href="#packaging">Packaging</a></li> <li><a class="reference internal" href="#checksums">Checksums</a></li> @@ -70,8 +73,8 @@ (This is the official release 1.7; for the most up-to-date version see below.)</p> <ul class="simple"> -<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy-1.7-linux.tar.bz2">Linux binary (32bit)</a></li> -<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy-1.7-linux64.tar.bz2">Linux binary (64bit)</a></li> +<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy-1.7-linux.tar.bz2">Linux binary (32bit)</a> (<a class="reference internal" href="#openssl0-9-8-notes">openssl0.9.8 notes</a>)</li> +<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy-1.7-linux64.tar.bz2">Linux binary (64bit)</a> (<a class="reference internal" href="#openssl0-9-8-notes">openssl0.9.8 notes</a>)</li> <li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy-1.7-osx64.tar.bz2">Mac OS/X binary (64bit)</a></li> <li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy-1.7-win32.zip">Windows binary (32bit)</a></li> </ul><p>If your CPU is really old, it may not have SSE2. In this case, you need @@ -105,6 +108,15 @@ not move or copy the executable <tt class="docutils literal">pypy</tt> outside the tree – put a symlink to it, otherwise it will not find its libraries.</p> </div> +<div class="section" id="linux-libraries"> +<span id="openssl0-9-8-notes"></span><h1>Linux libraries</h1> +<p>Linux binaries require a system with the correct set of libraries. +Right now one problem is OpenSSL, which some distributions distribute in +version 0.9.8 and others in version 1.0.0. The binaries require 0.9.8. +If you can't install it in your distribution, pick it the latest +<a class="reference external" href="http://www.openssl.org/source/">openssl-0.9.8*.tar.gz from sources</a> and install it with <tt class="docutils literal">./config +shared; make; make install</tt>.</p> +</div> <div class="section" id="building-from-source"> <span id="translate"></span><h1>Building from source</h1> <ol class="arabic"> diff --git a/source/download.txt b/source/download.txt --- a/source/download.txt +++ b/source/download.txt @@ -23,6 +23,9 @@ * `Other versions`_ * `Installing`_ (optional) + + * `Linux libraries`_ + * `Building from source`_ * `Packaging`_ * `Checksums`_ @@ -39,8 +42,8 @@ (This is the official release 1.7; for the most up-to-date version see below.) -* `Linux binary (32bit)`__ -* `Linux binary (64bit)`__ +* `Linux binary (32bit)`__ (`openssl0.9.8 notes`_) +* `Linux binary (64bit)`__ (`openssl0.9.8 notes`_) * `Mac OS/X binary (64bit)`__ * `Windows binary (32bit)`__ @@ -91,6 +94,21 @@ a symlink to it, otherwise it will not find its libraries. +.. _`openssl0.9.8 notes`: + +Linux libraries +--------------- + +Linux binaries require a system with the correct set of libraries. +Right now one problem is OpenSSL, which some distributions distribute in +version 0.9.8 and others in version 1.0.0. The binaries require 0.9.8. +If you can't install it in your distribution, pick it the latest +`openssl-0.9.8*.tar.gz from sources`__ and install it with ``./config +shared; make; make install``. + +.. __: http://www.openssl.org/source/ + + .. _translate: Building from source From noreply at buildbot.pypy.org Mon Jan 30 23:23:42 2012 From: noreply at buildbot.pypy.org (mattip) Date: Mon, 30 Jan 2012 23:23:42 +0100 (CET) Subject: [pypy-commit] pypy win32-cleanup: openssl now passes some tests Message-ID: <20120130222342.7D6067106F3@wyvern.cs.uni-duesseldorf.de> Author: mattip Branch: win32-cleanup Changeset: r51984:60f69529bcc0 Date: 2012-01-31 00:23 +0200 http://bitbucket.org/pypy/pypy/changeset/60f69529bcc0/ Log: openssl now passes some tests diff --git a/pypy/rlib/ropenssl.py b/pypy/rlib/ropenssl.py --- a/pypy/rlib/ropenssl.py +++ b/pypy/rlib/ropenssl.py @@ -7,7 +7,7 @@ if sys.platform == 'win32' and platform.name != 'mingw32': libraries = ['libeay32', 'ssleay32', - 'user32', 'advapi32', 'gdi32', 'msvcrt', 'ws2_32'] + 'user32', 'advapi32', 'gdi32', 'msvcrt', 'ws2_32', 'zlib'] includes = [ # ssl.h includes winsock.h, which will conflict with our own # need of winsock2. Remove this when separate compilation is @@ -54,6 +54,7 @@ ASN1_STRING = lltype.Ptr(lltype.ForwardReference()) ASN1_ITEM = rffi.COpaquePtr('ASN1_ITEM') +ASN1_ITEM_EXP = lltype.FuncType([], ASN1_ITEM) X509_NAME = rffi.COpaquePtr('X509_NAME') class CConfig: @@ -101,7 +102,6 @@ X509_extension_st = rffi_platform.Struct( 'struct X509_extension_st', [('value', ASN1_STRING)]) - ASN1_ITEM_EXP = lltype.FuncType([], ASN1_ITEM) X509V3_EXT_D2I = lltype.FuncType([rffi.VOIDP, rffi.CCHARPP, rffi.LONG], rffi.VOIDP) v3_ext_method = rffi_platform.Struct( @@ -224,7 +224,10 @@ ssl_external('i2a_ASN1_INTEGER', [BIO, ASN1_INTEGER], rffi.INT) ssl_external('ASN1_item_d2i', [rffi.VOIDP, rffi.CCHARPP, rffi.LONG, ASN1_ITEM], rffi.VOIDP) -ssl_external('ASN1_ITEM_ptr', [rffi.VOIDP], ASN1_ITEM, macro=True) +if rffi_platform.Defined("OPENSSL_EXPORT_VAR_AS_FUNCTION"): + ssl_external('ASN1_ITEM_ptr', [ASN1_ITEM_EXP], ASN1_ITEM, macro=True) +else: + ssl_external('ASN1_ITEM_ptr', [rffi.VOIDP], ASN1_ITEM, macro=True) ssl_external('sk_GENERAL_NAME_num', [GENERAL_NAMES], rffi.INT, macro=True) From noreply at buildbot.pypy.org Tue Jan 31 00:19:05 2012 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 31 Jan 2012 00:19:05 +0100 (CET) Subject: [pypy-commit] buildbot default: style fix commits are bad Message-ID: <20120130231905.159727106F3@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: Changeset: r632:20f86228d582 Date: 2012-01-31 01:18 +0200 http://bitbucket.org/pypy/buildbot/changeset/20f86228d582/ Log: style fix commits are bad diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py --- a/bot2/pypybuildbot/builds.py +++ b/bot2/pypybuildbot/builds.py @@ -209,7 +209,7 @@ # factory.addStep( ShellCmd(description="hg purge", - command="hg --config extensions.purge=purge --all", + command="hg --config extensions.purge= purge --all", workdir=workdir, haltOnFailure=True)) # From noreply at buildbot.pypy.org Tue Jan 31 01:08:45 2012 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 31 Jan 2012 01:08:45 +0100 (CET) Subject: [pypy-commit] pypy default: typo, USE FLYMAKE AND RUN TESTS IT WILL GLOW RED Message-ID: <20120131000845.7E98882B67@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski <fijall at gmail.com> Branch: Changeset: r51985:2c2944d51e51 Date: 2012-01-31 02:08 +0200 http://bitbucket.org/pypy/pypy/changeset/2c2944d51e51/ Log: typo, USE FLYMAKE AND RUN TESTS IT WILL GLOW RED diff --git a/pypy/interpreter/astcompiler/optimize.py b/pypy/interpreter/astcompiler/optimize.py --- a/pypy/interpreter/astcompiler/optimize.py +++ b/pypy/interpreter/astcompiler/optimize.py @@ -311,7 +311,7 @@ else: ch = 0 if (ch > 0xFFFF or - (MAXUNICODE == 0xFFFF and 0xD800 <= ch <= OxDFFFF)): + (MAXUNICODE == 0xFFFF and 0xD800 <= ch <= 0xDFFFF)): return subs return ast.Const(w_const, subs.lineno, subs.col_offset) From noreply at buildbot.pypy.org Tue Jan 31 12:20:16 2012 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 31 Jan 2012 12:20:16 +0100 (CET) Subject: [pypy-commit] pypy string-NUL: Simplify the code again, now that no_nul is a constructor argument. Message-ID: <20120131112016.D4CCA82B67@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: string-NUL Changeset: r51986:7669c39900d1 Date: 2012-01-31 12:05 +0100 http://bitbucket.org/pypy/pypy/changeset/7669c39900d1/ Log: Simplify the code again, now that no_nul is a constructor argument. diff --git a/pypy/annotation/binaryop.py b/pypy/annotation/binaryop.py --- a/pypy/annotation/binaryop.py +++ b/pypy/annotation/binaryop.py @@ -480,7 +480,7 @@ no_nul = str.no_nul for s_item in s_tuple.items: if isinstance(s_item, SomeFloat): - pass + pass # or s_item is a subclass, like SomeInteger elif isinstance(s_item, SomeString) and s_item.no_nul: pass else: @@ -817,16 +817,7 @@ # mixing Nones with other objects -def _make_none_union(classname, constructor_args='', glob=None, - copy_attributes=()): - if copy_attributes: - copy_attrs = ( - 'result.__dict__.update((name, getattr(obj, name)) ' - 'for name in %(copy_attributes)s if name in obj.__dict__)' - % locals()) - else: - copy_attrs = '' - +def _make_none_union(classname, constructor_args='', glob=None): if glob is None: glob = globals() loc = locals() @@ -834,26 +825,21 @@ class __extend__(pairtype(%(classname)s, SomePBC)): def union((obj, pbc)): if pbc.isNone(): - result = %(classname)s(%(constructor_args)s) - %(copy_attrs)s - return result + return %(classname)s(%(constructor_args)s) else: return SomeObject() class __extend__(pairtype(SomePBC, %(classname)s)): def union((pbc, obj)): if pbc.isNone(): - result = %(classname)s(%(constructor_args)s) - %(copy_attrs)s - return result + return %(classname)s(%(constructor_args)s) else: return SomeObject() """ % loc) exec source.compile() in glob _make_none_union('SomeInstance', 'classdef=obj.classdef, can_be_None=True') -_make_none_union('SomeString', 'can_be_None=True', - copy_attributes=('no_nul',)) +_make_none_union('SomeString', 'no_nul=obj.no_nul, can_be_None=True') _make_none_union('SomeUnicodeString', 'can_be_None=True') _make_none_union('SomeList', 'obj.listdef') _make_none_union('SomeDict', 'obj.dictdef') From noreply at buildbot.pypy.org Tue Jan 31 12:29:42 2012 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 31 Jan 2012 12:29:42 +0100 (CET) Subject: [pypy-commit] pypy string-NUL: Add a test checking that the "'\x00' in s" done by space.str0() Message-ID: <20120131112942.D8FFC82B67@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: string-NUL Changeset: r51987:300213fb2de0 Date: 2012-01-31 12:28 +0100 http://bitbucket.org/pypy/pypy/changeset/300213fb2de0/ Log: Add a test checking that the "'\x00' in s" done by space.str0() can be constant-folded if s is a constant. diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py --- a/pypy/jit/metainterp/test/test_ajit.py +++ b/pypy/jit/metainterp/test/test_ajit.py @@ -322,6 +322,17 @@ res = self.interp_operations(f, [42]) assert res == ord(u"?") + def test_char_in_constant_string(self): + def g(string): + return '\x00' in string + def f(): + if g('abcdef'): return -60 + if not g('abc\x00ef'): return -61 + return 42 + res = self.interp_operations(f, []) + assert res == 42 + self.check_operations_history({'finish': 1}) # nothing else + def test_residual_call(self): @dont_look_inside def externfn(x, y): From noreply at buildbot.pypy.org Tue Jan 31 15:33:51 2012 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 31 Jan 2012 15:33:51 +0100 (CET) Subject: [pypy-commit] pypy default: Expose select.epoll() on Linux, even if we are building pypy or running Message-ID: <20120131143351.375687106F3@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: Changeset: r51988:dab1ba579e09 Date: 2012-01-31 15:33 +0100 http://bitbucket.org/pypy/pypy/changeset/dab1ba579e09/ Log: Expose select.epoll() on Linux, even if we are building pypy or running tests with Python 2.5, where the 'select' module doesn't have epoll yet. diff --git a/pypy/module/select/__init__.py b/pypy/module/select/__init__.py --- a/pypy/module/select/__init__.py +++ b/pypy/module/select/__init__.py @@ -1,7 +1,6 @@ # Package initialisation from pypy.interpreter.mixedmodule import MixedModule -import select import sys @@ -15,18 +14,13 @@ 'error' : 'space.fromcache(interp_select.Cache).w_error' } - # TODO: this doesn't feel right... - if hasattr(select, "epoll"): + if sys.platform.startswith('linux'): interpleveldefs['epoll'] = 'interp_epoll.W_Epoll' - symbols = [ - "EPOLLIN", "EPOLLOUT", "EPOLLPRI", "EPOLLERR", "EPOLLHUP", - "EPOLLET", "EPOLLONESHOT", "EPOLLRDNORM", "EPOLLRDBAND", - "EPOLLWRNORM", "EPOLLWRBAND", "EPOLLMSG" - ] - for symbol in symbols: - if hasattr(select, symbol): - interpleveldefs[symbol] = "space.wrap(%s)" % getattr(select, symbol) - + from pypy.module.select.interp_epoll import cconfig, public_symbols + for symbol in public_symbols: + value = cconfig[symbol] + if value is not None: + interpleveldefs[symbol] = "space.wrap(%r)" % value def buildloaders(cls): from pypy.rlib import rpoll diff --git a/pypy/module/select/interp_epoll.py b/pypy/module/select/interp_epoll.py --- a/pypy/module/select/interp_epoll.py +++ b/pypy/module/select/interp_epoll.py @@ -29,8 +29,16 @@ ("data", CConfig.epoll_data) ]) +public_symbols = [ + "EPOLLIN", "EPOLLOUT", "EPOLLPRI", "EPOLLERR", "EPOLLHUP", + "EPOLLET", "EPOLLONESHOT", "EPOLLRDNORM", "EPOLLRDBAND", + "EPOLLWRNORM", "EPOLLWRBAND", "EPOLLMSG" + ] +for symbol in public_symbols: + setattr(CConfig, symbol, rffi_platform.DefinedConstantInteger(symbol)) + for symbol in ["EPOLL_CTL_ADD", "EPOLL_CTL_MOD", "EPOLL_CTL_DEL"]: - setattr(CConfig, symbol, rffi_platform.DefinedConstantInteger(symbol)) + setattr(CConfig, symbol, rffi_platform.ConstantInteger(symbol)) cconfig = rffi_platform.configure(CConfig) diff --git a/pypy/module/select/test/test_epoll.py b/pypy/module/select/test/test_epoll.py --- a/pypy/module/select/test/test_epoll.py +++ b/pypy/module/select/test/test_epoll.py @@ -1,23 +1,17 @@ import py +import sys from pypy.conftest import gettestobjspace class AppTestEpoll(object): def setup_class(cls): + # NB. we should ideally py.test.skip() if running on an old linux + # where the kernel doesn't support epoll() + if not sys.platform.startswith('linux'): + py.test.skip("test requires linux (assumed >= 2.6)") cls.space = gettestobjspace(usemodules=["select", "_socket", "posix"]) - import errno - import select - - if not hasattr(select, "epoll"): - py.test.skip("test requires linux 2.6") - try: - select.epoll() - except IOError, e: - if e.errno == errno.ENOSYS: - py.test.skip("kernel doesn't support epoll()") - def setup_method(self, meth): self.w_sockets = self.space.wrap([]) From noreply at buildbot.pypy.org Tue Jan 31 15:59:47 2012 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 31 Jan 2012 15:59:47 +0100 (CET) Subject: [pypy-commit] pypy stm: Add an emulator for "transaction", running serially and Message-ID: <20120131145947.E230B82B67@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: stm Changeset: r51989:d30f3b2ae5fc Date: 2012-01-31 15:57 +0100 http://bitbucket.org/pypy/pypy/changeset/d30f3b2ae5fc/ Log: Add an emulator for "transaction", running serially and dispatching events in random order. diff --git a/lib_pypy/transaction.py b/lib_pypy/transaction.py new file mode 100644 --- /dev/null +++ b/lib_pypy/transaction.py @@ -0,0 +1,35 @@ +import sys +import random + + +print >> sys.stderr, "warning: using lib_pypy/transaction.py, the emulator" + +_pending = {} + +def set_num_threads(num): + pass + +def add(f, *args): + r = random.random() + assert r not in _pending # very bad luck if it is + _pending[r] = (f, args) + +def add_epoll(ep, callback): + def poll_reader(): + # assume only one epoll is added. If the _pending list is + # now empty, wait. If not, then just poll non-blockingly. + if len(_pending) == 0: + timeout = -1 + else: + timeout = 0 + got = ep.poll(timeout=timeout) + for fd, events in got: + add(callback, fd, events) + add(poll_reader) + add(poll_reader) + +def run(): + pending = _pending + while pending: + _, (f, args) = pending.popitem() + f(*args) diff --git a/pypy/module/transaction/test/test_epoll.py b/pypy/module/transaction/test/test_epoll.py --- a/pypy/module/transaction/test/test_epoll.py +++ b/pypy/module/transaction/test/test_epoll.py @@ -49,3 +49,9 @@ assert steps == [] raises(Done, transaction.run) assert steps == ['write_stuff', 'callback'] + + +class AppTestEpollEmulator(AppTestEpoll): + def setup_class(cls): + # test for lib_pypy/transaction.py + cls.space = gettestobjspace(usemodules=['select']) diff --git a/pypy/module/transaction/test/test_transaction.py b/pypy/module/transaction/test/test_transaction.py --- a/pypy/module/transaction/test/test_transaction.py +++ b/pypy/module/transaction/test/test_transaction.py @@ -59,3 +59,9 @@ pass assert len(lst) == 1 assert lst[0] == e.args[0] + + +class AppTestTransactionEmulator(AppTestTransaction): + def setup_class(cls): + # test for lib_pypy/transaction.py + cls.space = gettestobjspace(usemodules=[]) From noreply at buildbot.pypy.org Tue Jan 31 16:00:01 2012 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 31 Jan 2012 16:00:01 +0100 (CET) Subject: [pypy-commit] pypy stm: Python 2.5 compat. Message-ID: <20120131150001.9375082B68@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: stm Changeset: r51991:cbf4aa8ac66e Date: 2012-01-31 15:59 +0100 http://bitbucket.org/pypy/pypy/changeset/cbf4aa8ac66e/ Log: Python 2.5 compat. diff --git a/pypy/module/transaction/interp_epoll.py b/pypy/module/transaction/interp_epoll.py --- a/pypy/module/transaction/interp_epoll.py +++ b/pypy/module/transaction/interp_epoll.py @@ -1,8 +1,8 @@ # Linux-only +from __future__ import with_statement import os -from select import EPOLLIN from pypy.rpython.lltypesystem import lltype, rffi from pypy.interpreter.gateway import unwrap_spec from pypy.interpreter.error import OperationError From noreply at buildbot.pypy.org Tue Jan 31 16:00:00 2012 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 31 Jan 2012 16:00:00 +0100 (CET) Subject: [pypy-commit] pypy stm: hg merge default Message-ID: <20120131150000.4BC6F82B67@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: stm Changeset: r51990:ed64896a3f19 Date: 2012-01-31 15:58 +0100 http://bitbucket.org/pypy/pypy/changeset/ed64896a3f19/ Log: hg merge default diff too long, truncating to 10000 out of 149665 lines diff --git a/lib-python/2.7/BaseHTTPServer.py b/lib-python/2.7/BaseHTTPServer.py --- a/lib-python/2.7/BaseHTTPServer.py +++ b/lib-python/2.7/BaseHTTPServer.py @@ -310,7 +310,13 @@ """ try: - self.raw_requestline = self.rfile.readline() + self.raw_requestline = self.rfile.readline(65537) + if len(self.raw_requestline) > 65536: + self.requestline = '' + self.request_version = '' + self.command = '' + self.send_error(414) + return if not self.raw_requestline: self.close_connection = 1 return diff --git a/lib-python/2.7/ConfigParser.py b/lib-python/2.7/ConfigParser.py --- a/lib-python/2.7/ConfigParser.py +++ b/lib-python/2.7/ConfigParser.py @@ -545,6 +545,38 @@ if isinstance(val, list): options[name] = '\n'.join(val) +import UserDict as _UserDict + +class _Chainmap(_UserDict.DictMixin): + """Combine multiple mappings for successive lookups. + + For example, to emulate Python's normal lookup sequence: + + import __builtin__ + pylookup = _Chainmap(locals(), globals(), vars(__builtin__)) + """ + + def __init__(self, *maps): + self._maps = maps + + def __getitem__(self, key): + for mapping in self._maps: + try: + return mapping[key] + except KeyError: + pass + raise KeyError(key) + + def keys(self): + result = [] + seen = set() + for mapping in self_maps: + for key in mapping: + if key not in seen: + result.append(key) + seen.add(key) + return result + class ConfigParser(RawConfigParser): def get(self, section, option, raw=False, vars=None): @@ -559,16 +591,18 @@ The section DEFAULT is special. """ - d = self._defaults.copy() + sectiondict = {} try: - d.update(self._sections[section]) + sectiondict = self._sections[section] except KeyError: if section != DEFAULTSECT: raise NoSectionError(section) # Update with the entry specific variables + vardict = {} if vars: for key, value in vars.items(): - d[self.optionxform(key)] = value + vardict[self.optionxform(key)] = value + d = _Chainmap(vardict, sectiondict, self._defaults) option = self.optionxform(option) try: value = d[option] diff --git a/lib-python/2.7/Cookie.py b/lib-python/2.7/Cookie.py --- a/lib-python/2.7/Cookie.py +++ b/lib-python/2.7/Cookie.py @@ -258,6 +258,11 @@ '\033' : '\\033', '\034' : '\\034', '\035' : '\\035', '\036' : '\\036', '\037' : '\\037', + # Because of the way browsers really handle cookies (as opposed + # to what the RFC says) we also encode , and ; + + ',' : '\\054', ';' : '\\073', + '"' : '\\"', '\\' : '\\\\', '\177' : '\\177', '\200' : '\\200', '\201' : '\\201', diff --git a/lib-python/2.7/HTMLParser.py b/lib-python/2.7/HTMLParser.py --- a/lib-python/2.7/HTMLParser.py +++ b/lib-python/2.7/HTMLParser.py @@ -26,7 +26,7 @@ tagfind = re.compile('[a-zA-Z][-.a-zA-Z0-9:_]*') attrfind = re.compile( r'\s*([a-zA-Z_][-.:a-zA-Z_0-9]*)(\s*=\s*' - r'(\'[^\']*\'|"[^"]*"|[-a-zA-Z0-9./,:;+*%?!&$_#=~@]*))?') + r'(\'[^\']*\'|"[^"]*"|[^\s"\'=<>`]*))?') locatestarttagend = re.compile(r""" <[a-zA-Z][-.a-zA-Z0-9:_]* # tag name @@ -99,7 +99,7 @@ markupbase.ParserBase.reset(self) def feed(self, data): - """Feed data to the parser. + r"""Feed data to the parser. Call this as often as you want, with as little or as much text as you want (may include '\n'). @@ -367,13 +367,16 @@ return s def replaceEntities(s): s = s.groups()[0] - if s[0] == "#": - s = s[1:] - if s[0] in ['x','X']: - c = int(s[1:], 16) - else: - c = int(s) - return unichr(c) + try: + if s[0] == "#": + s = s[1:] + if s[0] in ['x','X']: + c = int(s[1:], 16) + else: + c = int(s) + return unichr(c) + except ValueError: + return '&#'+s+';' else: # Cannot use name2codepoint directly, because HTMLParser supports apos, # which is not part of HTML 4 diff --git a/lib-python/2.7/SimpleHTTPServer.py b/lib-python/2.7/SimpleHTTPServer.py --- a/lib-python/2.7/SimpleHTTPServer.py +++ b/lib-python/2.7/SimpleHTTPServer.py @@ -15,6 +15,7 @@ import BaseHTTPServer import urllib import cgi +import sys import shutil import mimetypes try: @@ -131,7 +132,8 @@ length = f.tell() f.seek(0) self.send_response(200) - self.send_header("Content-type", "text/html") + encoding = sys.getfilesystemencoding() + self.send_header("Content-type", "text/html; charset=%s" % encoding) self.send_header("Content-Length", str(length)) self.end_headers() return f diff --git a/lib-python/2.7/SimpleXMLRPCServer.py b/lib-python/2.7/SimpleXMLRPCServer.py --- a/lib-python/2.7/SimpleXMLRPCServer.py +++ b/lib-python/2.7/SimpleXMLRPCServer.py @@ -246,7 +246,7 @@ marshalled data. For backwards compatibility, a dispatch function can be provided as an argument (see comment in SimpleXMLRPCRequestHandler.do_POST) but overriding the - existing method through subclassing is the prefered means + existing method through subclassing is the preferred means of changing method dispatch behavior. """ diff --git a/lib-python/2.7/SocketServer.py b/lib-python/2.7/SocketServer.py --- a/lib-python/2.7/SocketServer.py +++ b/lib-python/2.7/SocketServer.py @@ -675,7 +675,7 @@ # A timeout to apply to the request socket, if not None. timeout = None - # Disable nagle algoritm for this socket, if True. + # Disable nagle algorithm for this socket, if True. # Use only when wbufsize != 0, to avoid small packets. disable_nagle_algorithm = False diff --git a/lib-python/2.7/StringIO.py b/lib-python/2.7/StringIO.py --- a/lib-python/2.7/StringIO.py +++ b/lib-python/2.7/StringIO.py @@ -266,6 +266,7 @@ 8th bit) will cause a UnicodeError to be raised when getvalue() is called. """ + _complain_ifclosed(self.closed) if self.buflist: self.buf += ''.join(self.buflist) self.buflist = [] diff --git a/lib-python/2.7/_abcoll.py b/lib-python/2.7/_abcoll.py --- a/lib-python/2.7/_abcoll.py +++ b/lib-python/2.7/_abcoll.py @@ -82,7 +82,7 @@ @classmethod def __subclasshook__(cls, C): if cls is Iterator: - if _hasattr(C, "next"): + if _hasattr(C, "next") and _hasattr(C, "__iter__"): return True return NotImplemented diff --git a/lib-python/2.7/_pyio.py b/lib-python/2.7/_pyio.py --- a/lib-python/2.7/_pyio.py +++ b/lib-python/2.7/_pyio.py @@ -16,6 +16,7 @@ import io from io import (__all__, SEEK_SET, SEEK_CUR, SEEK_END) +from errno import EINTR __metaclass__ = type @@ -559,7 +560,11 @@ if not data: break res += data - return bytes(res) + if res: + return bytes(res) + else: + # b'' or None + return data def readinto(self, b): """Read up to len(b) bytes into b. @@ -678,7 +683,7 @@ """ def __init__(self, raw): - self.raw = raw + self._raw = raw ### Positioning ### @@ -722,8 +727,8 @@ if self.raw is None: raise ValueError("raw stream already detached") self.flush() - raw = self.raw - self.raw = None + raw = self._raw + self._raw = None return raw ### Inquiries ### @@ -738,6 +743,10 @@ return self.raw.writable() @property + def raw(self): + return self._raw + + @property def closed(self): return self.raw.closed @@ -933,7 +942,12 @@ current_size = 0 while True: # Read until EOF or until read() would block. - chunk = self.raw.read() + try: + chunk = self.raw.read() + except IOError as e: + if e.errno != EINTR: + raise + continue if chunk in empty_values: nodata_val = chunk break @@ -952,7 +966,12 @@ chunks = [buf[pos:]] wanted = max(self.buffer_size, n) while avail < n: - chunk = self.raw.read(wanted) + try: + chunk = self.raw.read(wanted) + except IOError as e: + if e.errno != EINTR: + raise + continue if chunk in empty_values: nodata_val = chunk break @@ -981,7 +1000,14 @@ have = len(self._read_buf) - self._read_pos if have < want or have <= 0: to_read = self.buffer_size - have - current = self.raw.read(to_read) + while True: + try: + current = self.raw.read(to_read) + except IOError as e: + if e.errno != EINTR: + raise + continue + break if current: self._read_buf = self._read_buf[self._read_pos:] + current self._read_pos = 0 @@ -1088,7 +1114,12 @@ written = 0 try: while self._write_buf: - n = self.raw.write(self._write_buf) + try: + n = self.raw.write(self._write_buf) + except IOError as e: + if e.errno != EINTR: + raise + continue if n > len(self._write_buf) or n < 0: raise IOError("write() returned incorrect number of bytes") del self._write_buf[:n] @@ -1456,7 +1487,7 @@ if not isinstance(errors, basestring): raise ValueError("invalid errors: %r" % errors) - self.buffer = buffer + self._buffer = buffer self._line_buffering = line_buffering self._encoding = encoding self._errors = errors @@ -1511,6 +1542,10 @@ def line_buffering(self): return self._line_buffering + @property + def buffer(self): + return self._buffer + def seekable(self): return self._seekable @@ -1724,8 +1759,8 @@ if self.buffer is None: raise ValueError("buffer is already detached") self.flush() - buffer = self.buffer - self.buffer = None + buffer = self._buffer + self._buffer = None return buffer def seek(self, cookie, whence=0): diff --git a/lib-python/2.7/_weakrefset.py b/lib-python/2.7/_weakrefset.py --- a/lib-python/2.7/_weakrefset.py +++ b/lib-python/2.7/_weakrefset.py @@ -66,7 +66,11 @@ return sum(x() is not None for x in self.data) def __contains__(self, item): - return ref(item) in self.data + try: + wr = ref(item) + except TypeError: + return False + return wr in self.data def __reduce__(self): return (self.__class__, (list(self),), diff --git a/lib-python/2.7/anydbm.py b/lib-python/2.7/anydbm.py --- a/lib-python/2.7/anydbm.py +++ b/lib-python/2.7/anydbm.py @@ -29,17 +29,8 @@ list = d.keys() # return a list of all existing keys (slow!) Future versions may change the order in which implementations are -tested for existence, add interfaces to other dbm-like +tested for existence, and add interfaces to other dbm-like implementations. - -The open function has an optional second argument. This can be 'r', -for read-only access, 'w', for read-write access of an existing -database, 'c' for read-write access to a new or existing database, and -'n' for read-write access to a new database. The default is 'r'. - -Note: 'r' and 'w' fail if the database doesn't exist; 'c' creates it -only if it doesn't exist; and 'n' always creates a new database. - """ class error(Exception): @@ -63,7 +54,18 @@ error = tuple(_errors) -def open(file, flag = 'r', mode = 0666): +def open(file, flag='r', mode=0666): + """Open or create database at path given by *file*. + + Optional argument *flag* can be 'r' (default) for read-only access, 'w' + for read-write access of an existing database, 'c' for read-write access + to a new or existing database, and 'n' for read-write access to a new + database. + + Note: 'r' and 'w' fail if the database doesn't exist; 'c' creates it + only if it doesn't exist; and 'n' always creates a new database. + """ + # guess the type of an existing database from whichdb import whichdb result=whichdb(file) diff --git a/lib-python/2.7/argparse.py b/lib-python/2.7/argparse.py --- a/lib-python/2.7/argparse.py +++ b/lib-python/2.7/argparse.py @@ -82,6 +82,7 @@ ] +import collections as _collections import copy as _copy import os as _os import re as _re @@ -1037,7 +1038,7 @@ self._prog_prefix = prog self._parser_class = parser_class - self._name_parser_map = {} + self._name_parser_map = _collections.OrderedDict() self._choices_actions = [] super(_SubParsersAction, self).__init__( @@ -1080,7 +1081,7 @@ parser = self._name_parser_map[parser_name] except KeyError: tup = parser_name, ', '.join(self._name_parser_map) - msg = _('unknown parser %r (choices: %s)' % tup) + msg = _('unknown parser %r (choices: %s)') % tup raise ArgumentError(self, msg) # parse all the remaining options into the namespace @@ -1109,7 +1110,7 @@ the builtin open() function. """ - def __init__(self, mode='r', bufsize=None): + def __init__(self, mode='r', bufsize=-1): self._mode = mode self._bufsize = bufsize @@ -1121,18 +1122,19 @@ elif 'w' in self._mode: return _sys.stdout else: - msg = _('argument "-" with mode %r' % self._mode) + msg = _('argument "-" with mode %r') % self._mode raise ValueError(msg) # all other arguments are used as file names - if self._bufsize: + try: return open(string, self._mode, self._bufsize) - else: - return open(string, self._mode) + except IOError as e: + message = _("can't open '%s': %s") + raise ArgumentTypeError(message % (string, e)) def __repr__(self): - args = [self._mode, self._bufsize] - args_str = ', '.join([repr(arg) for arg in args if arg is not None]) + args = self._mode, self._bufsize + args_str = ', '.join(repr(arg) for arg in args if arg != -1) return '%s(%s)' % (type(self).__name__, args_str) # =========================== @@ -1275,13 +1277,20 @@ # create the action object, and add it to the parser action_class = self._pop_action_class(kwargs) if not _callable(action_class): - raise ValueError('unknown action "%s"' % action_class) + raise ValueError('unknown action "%s"' % (action_class,)) action = action_class(**kwargs) # raise an error if the action type is not callable type_func = self._registry_get('type', action.type, action.type) if not _callable(type_func): - raise ValueError('%r is not callable' % type_func) + raise ValueError('%r is not callable' % (type_func,)) + + # raise an error if the metavar does not match the type + if hasattr(self, "_get_formatter"): + try: + self._get_formatter()._format_args(action, None) + except TypeError: + raise ValueError("length of metavar tuple does not match nargs") return self._add_action(action) @@ -1481,6 +1490,7 @@ self._defaults = container._defaults self._has_negative_number_optionals = \ container._has_negative_number_optionals + self._mutually_exclusive_groups = container._mutually_exclusive_groups def _add_action(self, action): action = super(_ArgumentGroup, self)._add_action(action) diff --git a/lib-python/2.7/ast.py b/lib-python/2.7/ast.py --- a/lib-python/2.7/ast.py +++ b/lib-python/2.7/ast.py @@ -29,12 +29,12 @@ from _ast import __version__ -def parse(expr, filename='<unknown>', mode='exec'): +def parse(source, filename='<unknown>', mode='exec'): """ - Parse an expression into an AST node. - Equivalent to compile(expr, filename, mode, PyCF_ONLY_AST). + Parse the source into an AST node. + Equivalent to compile(source, filename, mode, PyCF_ONLY_AST). """ - return compile(expr, filename, mode, PyCF_ONLY_AST) + return compile(source, filename, mode, PyCF_ONLY_AST) def literal_eval(node_or_string): @@ -152,8 +152,6 @@ Increment the line number of each node in the tree starting at *node* by *n*. This is useful to "move code" to a different location in a file. """ - if 'lineno' in node._attributes: - node.lineno = getattr(node, 'lineno', 0) + n for child in walk(node): if 'lineno' in child._attributes: child.lineno = getattr(child, 'lineno', 0) + n @@ -204,9 +202,9 @@ def walk(node): """ - Recursively yield all child nodes of *node*, in no specified order. This is - useful if you only want to modify nodes in place and don't care about the - context. + Recursively yield all descendant nodes in the tree starting at *node* + (including *node* itself), in no specified order. This is useful if you + only want to modify nodes in place and don't care about the context. """ from collections import deque todo = deque([node]) diff --git a/lib-python/2.7/asyncore.py b/lib-python/2.7/asyncore.py --- a/lib-python/2.7/asyncore.py +++ b/lib-python/2.7/asyncore.py @@ -54,7 +54,11 @@ import os from errno import EALREADY, EINPROGRESS, EWOULDBLOCK, ECONNRESET, EINVAL, \ - ENOTCONN, ESHUTDOWN, EINTR, EISCONN, EBADF, ECONNABORTED, errorcode + ENOTCONN, ESHUTDOWN, EINTR, EISCONN, EBADF, ECONNABORTED, EPIPE, EAGAIN, \ + errorcode + +_DISCONNECTED = frozenset((ECONNRESET, ENOTCONN, ESHUTDOWN, ECONNABORTED, EPIPE, + EBADF)) try: socket_map @@ -109,7 +113,7 @@ if flags & (select.POLLHUP | select.POLLERR | select.POLLNVAL): obj.handle_close() except socket.error, e: - if e.args[0] not in (EBADF, ECONNRESET, ENOTCONN, ESHUTDOWN, ECONNABORTED): + if e.args[0] not in _DISCONNECTED: obj.handle_error() else: obj.handle_close() @@ -353,7 +357,7 @@ except TypeError: return None except socket.error as why: - if why.args[0] in (EWOULDBLOCK, ECONNABORTED): + if why.args[0] in (EWOULDBLOCK, ECONNABORTED, EAGAIN): return None else: raise @@ -367,7 +371,7 @@ except socket.error, why: if why.args[0] == EWOULDBLOCK: return 0 - elif why.args[0] in (ECONNRESET, ENOTCONN, ESHUTDOWN, ECONNABORTED): + elif why.args[0] in _DISCONNECTED: self.handle_close() return 0 else: @@ -385,7 +389,7 @@ return data except socket.error, why: # winsock sometimes throws ENOTCONN - if why.args[0] in [ECONNRESET, ENOTCONN, ESHUTDOWN, ECONNABORTED]: + if why.args[0] in _DISCONNECTED: self.handle_close() return '' else: diff --git a/lib-python/2.7/bdb.py b/lib-python/2.7/bdb.py --- a/lib-python/2.7/bdb.py +++ b/lib-python/2.7/bdb.py @@ -250,6 +250,12 @@ list.append(lineno) bp = Breakpoint(filename, lineno, temporary, cond, funcname) + def _prune_breaks(self, filename, lineno): + if (filename, lineno) not in Breakpoint.bplist: + self.breaks[filename].remove(lineno) + if not self.breaks[filename]: + del self.breaks[filename] + def clear_break(self, filename, lineno): filename = self.canonic(filename) if not filename in self.breaks: @@ -261,10 +267,7 @@ # pair, then remove the breaks entry for bp in Breakpoint.bplist[filename, lineno][:]: bp.deleteMe() - if (filename, lineno) not in Breakpoint.bplist: - self.breaks[filename].remove(lineno) - if not self.breaks[filename]: - del self.breaks[filename] + self._prune_breaks(filename, lineno) def clear_bpbynumber(self, arg): try: @@ -277,7 +280,8 @@ return 'Breakpoint number (%d) out of range' % number if not bp: return 'Breakpoint (%d) already deleted' % number - self.clear_break(bp.file, bp.line) + bp.deleteMe() + self._prune_breaks(bp.file, bp.line) def clear_all_file_breaks(self, filename): filename = self.canonic(filename) diff --git a/lib-python/2.7/collections.py b/lib-python/2.7/collections.py --- a/lib-python/2.7/collections.py +++ b/lib-python/2.7/collections.py @@ -6,59 +6,38 @@ __all__ += _abcoll.__all__ from _collections import deque, defaultdict -from operator import itemgetter as _itemgetter, eq as _eq +from operator import itemgetter as _itemgetter from keyword import iskeyword as _iskeyword import sys as _sys import heapq as _heapq -from itertools import repeat as _repeat, chain as _chain, starmap as _starmap, \ - ifilter as _ifilter, imap as _imap +from itertools import repeat as _repeat, chain as _chain, starmap as _starmap + try: - from thread import get_ident + from thread import get_ident as _get_ident except ImportError: - from dummy_thread import get_ident - -def _recursive_repr(user_function): - 'Decorator to make a repr function return "..." for a recursive call' - repr_running = set() - - def wrapper(self): - key = id(self), get_ident() - if key in repr_running: - return '...' - repr_running.add(key) - try: - result = user_function(self) - finally: - repr_running.discard(key) - return result - - # Can't use functools.wraps() here because of bootstrap issues - wrapper.__module__ = getattr(user_function, '__module__') - wrapper.__doc__ = getattr(user_function, '__doc__') - wrapper.__name__ = getattr(user_function, '__name__') - return wrapper + from dummy_thread import get_ident as _get_ident ################################################################################ ### OrderedDict ################################################################################ -class OrderedDict(dict, MutableMapping): +class OrderedDict(dict): 'Dictionary that remembers insertion order' # An inherited dict maps keys to values. # The inherited dict provides __getitem__, __len__, __contains__, and get. # The remaining methods are order-aware. - # Big-O running times for all methods are the same as for regular dictionaries. + # Big-O running times for all methods are the same as regular dictionaries. - # The internal self.__map dictionary maps keys to links in a doubly linked list. + # The internal self.__map dict maps keys to links in a doubly linked list. # The circular doubly linked list starts and ends with a sentinel element. # The sentinel element never gets deleted (this simplifies the algorithm). # Each link is stored as a list of length three: [PREV, NEXT, KEY]. def __init__(self, *args, **kwds): - '''Initialize an ordered dictionary. Signature is the same as for - regular dictionaries, but keyword arguments are not recommended - because their insertion order is arbitrary. + '''Initialize an ordered dictionary. The signature is the same as + regular dictionaries, but keyword arguments are not recommended because + their insertion order is arbitrary. ''' if len(args) > 1: @@ -66,17 +45,15 @@ try: self.__root except AttributeError: - self.__root = root = [None, None, None] # sentinel node - PREV = 0 - NEXT = 1 - root[PREV] = root[NEXT] = root + self.__root = root = [] # sentinel node + root[:] = [root, root, None] self.__map = {} - self.update(*args, **kwds) + self.__update(*args, **kwds) def __setitem__(self, key, value, PREV=0, NEXT=1, dict_setitem=dict.__setitem__): 'od.__setitem__(i, y) <==> od[i]=y' - # Setting a new item creates a new link which goes at the end of the linked - # list, and the inherited dictionary is updated with the new key/value pair. + # Setting a new item creates a new link at the end of the linked list, + # and the inherited dictionary is updated with the new key/value pair. if key not in self: root = self.__root last = root[PREV] @@ -85,65 +62,160 @@ def __delitem__(self, key, PREV=0, NEXT=1, dict_delitem=dict.__delitem__): 'od.__delitem__(y) <==> del od[y]' - # Deleting an existing item uses self.__map to find the link which is - # then removed by updating the links in the predecessor and successor nodes. + # Deleting an existing item uses self.__map to find the link which gets + # removed by updating the links in the predecessor and successor nodes. dict_delitem(self, key) - link = self.__map.pop(key) - link_prev = link[PREV] - link_next = link[NEXT] + link_prev, link_next, key = self.__map.pop(key) link_prev[NEXT] = link_next link_next[PREV] = link_prev - def __iter__(self, NEXT=1, KEY=2): + def __iter__(self): 'od.__iter__() <==> iter(od)' # Traverse the linked list in order. + NEXT, KEY = 1, 2 root = self.__root curr = root[NEXT] while curr is not root: yield curr[KEY] curr = curr[NEXT] - def __reversed__(self, PREV=0, KEY=2): + def __reversed__(self): 'od.__reversed__() <==> reversed(od)' # Traverse the linked list in reverse order. + PREV, KEY = 0, 2 root = self.__root curr = root[PREV] while curr is not root: yield curr[KEY] curr = curr[PREV] + def clear(self): + 'od.clear() -> None. Remove all items from od.' + for node in self.__map.itervalues(): + del node[:] + root = self.__root + root[:] = [root, root, None] + self.__map.clear() + dict.clear(self) + + # -- the following methods do not depend on the internal structure -- + + def keys(self): + 'od.keys() -> list of keys in od' + return list(self) + + def values(self): + 'od.values() -> list of values in od' + return [self[key] for key in self] + + def items(self): + 'od.items() -> list of (key, value) pairs in od' + return [(key, self[key]) for key in self] + + def iterkeys(self): + 'od.iterkeys() -> an iterator over the keys in od' + return iter(self) + + def itervalues(self): + 'od.itervalues -> an iterator over the values in od' + for k in self: + yield self[k] + + def iteritems(self): + 'od.iteritems -> an iterator over the (key, value) pairs in od' + for k in self: + yield (k, self[k]) + + update = MutableMapping.update + + __update = update # let subclasses override update without breaking __init__ + + __marker = object() + + def pop(self, key, default=__marker): + '''od.pop(k[,d]) -> v, remove specified key and return the corresponding + value. If key is not found, d is returned if given, otherwise KeyError + is raised. + + ''' + if key in self: + result = self[key] + del self[key] + return result + if default is self.__marker: + raise KeyError(key) + return default + + def setdefault(self, key, default=None): + 'od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od' + if key in self: + return self[key] + self[key] = default + return default + + def popitem(self, last=True): + '''od.popitem() -> (k, v), return and remove a (key, value) pair. + Pairs are returned in LIFO order if last is true or FIFO order if false. + + ''' + if not self: + raise KeyError('dictionary is empty') + key = next(reversed(self) if last else iter(self)) + value = self.pop(key) + return key, value + + def __repr__(self, _repr_running={}): + 'od.__repr__() <==> repr(od)' + call_key = id(self), _get_ident() + if call_key in _repr_running: + return '...' + _repr_running[call_key] = 1 + try: + if not self: + return '%s()' % (self.__class__.__name__,) + return '%s(%r)' % (self.__class__.__name__, self.items()) + finally: + del _repr_running[call_key] + def __reduce__(self): 'Return state information for pickling' items = [[k, self[k]] for k in self] - tmp = self.__map, self.__root - del self.__map, self.__root inst_dict = vars(self).copy() - self.__map, self.__root = tmp + for k in vars(OrderedDict()): + inst_dict.pop(k, None) if inst_dict: return (self.__class__, (items,), inst_dict) return self.__class__, (items,) - def clear(self): - 'od.clear() -> None. Remove all items from od.' - try: - for node in self.__map.itervalues(): - del node[:] - self.__root[:] = [self.__root, self.__root, None] - self.__map.clear() - except AttributeError: - pass - dict.clear(self) + def copy(self): + 'od.copy() -> a shallow copy of od' + return self.__class__(self) - setdefault = MutableMapping.setdefault - update = MutableMapping.update - pop = MutableMapping.pop - keys = MutableMapping.keys - values = MutableMapping.values - items = MutableMapping.items - iterkeys = MutableMapping.iterkeys - itervalues = MutableMapping.itervalues - iteritems = MutableMapping.iteritems - __ne__ = MutableMapping.__ne__ + @classmethod + def fromkeys(cls, iterable, value=None): + '''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S. + If not specified, the value defaults to None. + + ''' + self = cls() + for key in iterable: + self[key] = value + return self + + def __eq__(self, other): + '''od.__eq__(y) <==> od==y. Comparison to another OD is order-sensitive + while comparison to a regular mapping is order-insensitive. + + ''' + if isinstance(other, OrderedDict): + return len(self)==len(other) and self.items() == other.items() + return dict.__eq__(self, other) + + def __ne__(self, other): + 'od.__ne__(y) <==> od!=y' + return not self == other + + # -- the following methods support python 3.x style dictionary views -- def viewkeys(self): "od.viewkeys() -> a set-like object providing a view on od's keys" @@ -157,49 +229,6 @@ "od.viewitems() -> a set-like object providing a view on od's items" return ItemsView(self) - def popitem(self, last=True): - '''od.popitem() -> (k, v), return and remove a (key, value) pair. - Pairs are returned in LIFO order if last is true or FIFO order if false. - - ''' - if not self: - raise KeyError('dictionary is empty') - key = next(reversed(self) if last else iter(self)) - value = self.pop(key) - return key, value - - @_recursive_repr - def __repr__(self): - 'od.__repr__() <==> repr(od)' - if not self: - return '%s()' % (self.__class__.__name__,) - return '%s(%r)' % (self.__class__.__name__, self.items()) - - def copy(self): - 'od.copy() -> a shallow copy of od' - return self.__class__(self) - - @classmethod - def fromkeys(cls, iterable, value=None): - '''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S - and values equal to v (which defaults to None). - - ''' - d = cls() - for key in iterable: - d[key] = value - return d - - def __eq__(self, other): - '''od.__eq__(y) <==> od==y. Comparison to another OD is order-sensitive - while comparison to a regular mapping is order-insensitive. - - ''' - if isinstance(other, OrderedDict): - return len(self)==len(other) and \ - all(_imap(_eq, self.iteritems(), other.iteritems())) - return dict.__eq__(self, other) - ################################################################################ ### namedtuple @@ -328,16 +357,16 @@ or multiset. Elements are stored as dictionary keys and their counts are stored as dictionary values. - >>> c = Counter('abracadabra') # count elements from a string + >>> c = Counter('abcdeabcdabcaba') # count elements from a string >>> c.most_common(3) # three most common elements - [('a', 5), ('r', 2), ('b', 2)] + [('a', 5), ('b', 4), ('c', 3)] >>> sorted(c) # list all unique elements - ['a', 'b', 'c', 'd', 'r'] + ['a', 'b', 'c', 'd', 'e'] >>> ''.join(sorted(c.elements())) # list elements with repetitions - 'aaaaabbcdrr' + 'aaaaabbbbcccdde' >>> sum(c.values()) # total of all counts - 11 + 15 >>> c['a'] # count of letter 'a' 5 @@ -345,8 +374,8 @@ ... c[elem] += 1 # by adding 1 to each element's count >>> c['a'] # now there are seven 'a' 7 - >>> del c['r'] # remove all 'r' - >>> c['r'] # now there are zero 'r' + >>> del c['b'] # remove all 'b' + >>> c['b'] # now there are zero 'b' 0 >>> d = Counter('simsalabim') # make another counter @@ -385,6 +414,7 @@ >>> c = Counter(a=4, b=2) # a new counter from keyword args ''' + super(Counter, self).__init__() self.update(iterable, **kwds) def __missing__(self, key): @@ -396,8 +426,8 @@ '''List the n most common elements and their counts from the most common to the least. If n is None, then list all element counts. - >>> Counter('abracadabra').most_common(3) - [('a', 5), ('r', 2), ('b', 2)] + >>> Counter('abcdeabcdabcaba').most_common(3) + [('a', 5), ('b', 4), ('c', 3)] ''' # Emulate Bag.sortedByCount from Smalltalk @@ -463,7 +493,7 @@ for elem, count in iterable.iteritems(): self[elem] = self_get(elem, 0) + count else: - dict.update(self, iterable) # fast path when counter is empty + super(Counter, self).update(iterable) # fast path when counter is empty else: self_get = self.get for elem in iterable: @@ -499,13 +529,16 @@ self.subtract(kwds) def copy(self): - 'Like dict.copy() but returns a Counter instance instead of a dict.' - return Counter(self) + 'Return a shallow copy.' + return self.__class__(self) + + def __reduce__(self): + return self.__class__, (dict(self),) def __delitem__(self, elem): 'Like dict.__delitem__() but does not raise KeyError for missing values.' if elem in self: - dict.__delitem__(self, elem) + super(Counter, self).__delitem__(elem) def __repr__(self): if not self: @@ -532,10 +565,13 @@ if not isinstance(other, Counter): return NotImplemented result = Counter() - for elem in set(self) | set(other): - newcount = self[elem] + other[elem] + for elem, count in self.items(): + newcount = count + other[elem] if newcount > 0: result[elem] = newcount + for elem, count in other.items(): + if elem not in self and count > 0: + result[elem] = count return result def __sub__(self, other): @@ -548,10 +584,13 @@ if not isinstance(other, Counter): return NotImplemented result = Counter() - for elem in set(self) | set(other): - newcount = self[elem] - other[elem] + for elem, count in self.items(): + newcount = count - other[elem] if newcount > 0: result[elem] = newcount + for elem, count in other.items(): + if elem not in self and count < 0: + result[elem] = 0 - count return result def __or__(self, other): @@ -564,11 +603,14 @@ if not isinstance(other, Counter): return NotImplemented result = Counter() - for elem in set(self) | set(other): - p, q = self[elem], other[elem] - newcount = q if p < q else p + for elem, count in self.items(): + other_count = other[elem] + newcount = other_count if count < other_count else count if newcount > 0: result[elem] = newcount + for elem, count in other.items(): + if elem not in self and count > 0: + result[elem] = count return result def __and__(self, other): @@ -581,11 +623,9 @@ if not isinstance(other, Counter): return NotImplemented result = Counter() - if len(self) < len(other): - self, other = other, self - for elem in _ifilter(self.__contains__, other): - p, q = self[elem], other[elem] - newcount = p if p < q else q + for elem, count in self.items(): + other_count = other[elem] + newcount = count if count < other_count else other_count if newcount > 0: result[elem] = newcount return result diff --git a/lib-python/2.7/compileall.py b/lib-python/2.7/compileall.py --- a/lib-python/2.7/compileall.py +++ b/lib-python/2.7/compileall.py @@ -9,7 +9,6 @@ packages -- for now, you'll have to deal with packages separately.) See module py_compile for details of the actual byte-compilation. - """ import os import sys @@ -31,7 +30,6 @@ directory name that will show up in error messages) force: if 1, force compilation, even if timestamps are up-to-date quiet: if 1, be quiet during compilation - """ if not quiet: print 'Listing', dir, '...' @@ -61,15 +59,16 @@ return success def compile_file(fullname, ddir=None, force=0, rx=None, quiet=0): - """Byte-compile file. - file: the file to byte-compile + """Byte-compile one file. + + Arguments (only fullname is required): + + fullname: the file to byte-compile ddir: if given, purported directory name (this is the directory name that will show up in error messages) force: if 1, force compilation, even if timestamps are up-to-date quiet: if 1, be quiet during compilation - """ - success = 1 name = os.path.basename(fullname) if ddir is not None: @@ -120,7 +119,6 @@ maxlevels: max recursion level (default 0) force: as for compile_dir() (default 0) quiet: as for compile_dir() (default 0) - """ success = 1 for dir in sys.path: diff --git a/lib-python/2.7/csv.py b/lib-python/2.7/csv.py --- a/lib-python/2.7/csv.py +++ b/lib-python/2.7/csv.py @@ -281,7 +281,7 @@ an all or nothing approach, so we allow for small variations in this number. 1) build a table of the frequency of each character on every line. - 2) build a table of freqencies of this frequency (meta-frequency?), + 2) build a table of frequencies of this frequency (meta-frequency?), e.g. 'x occurred 5 times in 10 rows, 6 times in 1000 rows, 7 times in 2 rows' 3) use the mode of the meta-frequency to determine the /expected/ diff --git a/lib-python/2.7/ctypes/test/test_arrays.py b/lib-python/2.7/ctypes/test/test_arrays.py --- a/lib-python/2.7/ctypes/test/test_arrays.py +++ b/lib-python/2.7/ctypes/test/test_arrays.py @@ -37,7 +37,7 @@ values = [ia[i] for i in range(len(init))] self.assertEqual(values, [0] * len(init)) - # Too many in itializers should be caught + # Too many initializers should be caught self.assertRaises(IndexError, int_array, *range(alen*2)) CharArray = ARRAY(c_char, 3) diff --git a/lib-python/2.7/ctypes/test/test_as_parameter.py b/lib-python/2.7/ctypes/test/test_as_parameter.py --- a/lib-python/2.7/ctypes/test/test_as_parameter.py +++ b/lib-python/2.7/ctypes/test/test_as_parameter.py @@ -187,6 +187,18 @@ self.assertEqual((s8i.a, s8i.b, s8i.c, s8i.d, s8i.e, s8i.f, s8i.g, s8i.h), (9*2, 8*3, 7*4, 6*5, 5*6, 4*7, 3*8, 2*9)) + def test_recursive_as_param(self): + from ctypes import c_int + + class A(object): + pass + + a = A() + a._as_parameter_ = a + with self.assertRaises(RuntimeError): + c_int.from_param(a) + + #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ class AsParamWrapper(object): diff --git a/lib-python/2.7/ctypes/test/test_callbacks.py b/lib-python/2.7/ctypes/test/test_callbacks.py --- a/lib-python/2.7/ctypes/test/test_callbacks.py +++ b/lib-python/2.7/ctypes/test/test_callbacks.py @@ -206,6 +206,42 @@ windll.user32.EnumWindows(EnumWindowsCallbackFunc, 0) + def test_callback_register_int(self): + # Issue #8275: buggy handling of callback args under Win64 + # NOTE: should be run on release builds as well + dll = CDLL(_ctypes_test.__file__) + CALLBACK = CFUNCTYPE(c_int, c_int, c_int, c_int, c_int, c_int) + # All this function does is call the callback with its args squared + func = dll._testfunc_cbk_reg_int + func.argtypes = (c_int, c_int, c_int, c_int, c_int, CALLBACK) + func.restype = c_int + + def callback(a, b, c, d, e): + return a + b + c + d + e + + result = func(2, 3, 4, 5, 6, CALLBACK(callback)) + self.assertEqual(result, callback(2*2, 3*3, 4*4, 5*5, 6*6)) + + def test_callback_register_double(self): + # Issue #8275: buggy handling of callback args under Win64 + # NOTE: should be run on release builds as well + dll = CDLL(_ctypes_test.__file__) + CALLBACK = CFUNCTYPE(c_double, c_double, c_double, c_double, + c_double, c_double) + # All this function does is call the callback with its args squared + func = dll._testfunc_cbk_reg_double + func.argtypes = (c_double, c_double, c_double, + c_double, c_double, CALLBACK) + func.restype = c_double + + def callback(a, b, c, d, e): + return a + b + c + d + e + + result = func(1.1, 2.2, 3.3, 4.4, 5.5, CALLBACK(callback)) + self.assertEqual(result, + callback(1.1*1.1, 2.2*2.2, 3.3*3.3, 4.4*4.4, 5.5*5.5)) + + ################################################################ if __name__ == '__main__': diff --git a/lib-python/2.7/ctypes/test/test_functions.py b/lib-python/2.7/ctypes/test/test_functions.py --- a/lib-python/2.7/ctypes/test/test_functions.py +++ b/lib-python/2.7/ctypes/test/test_functions.py @@ -116,7 +116,7 @@ self.assertEqual(result, 21) self.assertEqual(type(result), int) - # You cannot assing character format codes as restype any longer + # You cannot assign character format codes as restype any longer self.assertRaises(TypeError, setattr, f, "restype", "i") def test_floatresult(self): diff --git a/lib-python/2.7/ctypes/test/test_init.py b/lib-python/2.7/ctypes/test/test_init.py --- a/lib-python/2.7/ctypes/test/test_init.py +++ b/lib-python/2.7/ctypes/test/test_init.py @@ -27,7 +27,7 @@ self.assertEqual((y.x.a, y.x.b), (0, 0)) self.assertEqual(y.x.new_was_called, False) - # But explicitely creating an X structure calls __new__ and __init__, of course. + # But explicitly creating an X structure calls __new__ and __init__, of course. x = X() self.assertEqual((x.a, x.b), (9, 12)) self.assertEqual(x.new_was_called, True) diff --git a/lib-python/2.7/ctypes/test/test_numbers.py b/lib-python/2.7/ctypes/test/test_numbers.py --- a/lib-python/2.7/ctypes/test/test_numbers.py +++ b/lib-python/2.7/ctypes/test/test_numbers.py @@ -157,7 +157,7 @@ def test_int_from_address(self): from array import array for t in signed_types + unsigned_types: - # the array module doesn't suppport all format codes + # the array module doesn't support all format codes # (no 'q' or 'Q') try: array(t._type_) diff --git a/lib-python/2.7/ctypes/test/test_win32.py b/lib-python/2.7/ctypes/test/test_win32.py --- a/lib-python/2.7/ctypes/test/test_win32.py +++ b/lib-python/2.7/ctypes/test/test_win32.py @@ -17,7 +17,7 @@ # ValueError: Procedure probably called with not enough arguments (4 bytes missing) self.assertRaises(ValueError, IsWindow) - # This one should succeeed... + # This one should succeed... self.assertEqual(0, IsWindow(0)) # ValueError: Procedure probably called with too many arguments (8 bytes in excess) diff --git a/lib-python/2.7/curses/wrapper.py b/lib-python/2.7/curses/wrapper.py --- a/lib-python/2.7/curses/wrapper.py +++ b/lib-python/2.7/curses/wrapper.py @@ -43,7 +43,8 @@ return func(stdscr, *args, **kwds) finally: # Set everything back to normal - stdscr.keypad(0) - curses.echo() - curses.nocbreak() - curses.endwin() + if 'stdscr' in locals(): + stdscr.keypad(0) + curses.echo() + curses.nocbreak() + curses.endwin() diff --git a/lib-python/2.7/decimal.py b/lib-python/2.7/decimal.py --- a/lib-python/2.7/decimal.py +++ b/lib-python/2.7/decimal.py @@ -1068,14 +1068,16 @@ if ans: return ans - if not self: - # -Decimal('0') is Decimal('0'), not Decimal('-0') + if context is None: + context = getcontext() + + if not self and context.rounding != ROUND_FLOOR: + # -Decimal('0') is Decimal('0'), not Decimal('-0'), except + # in ROUND_FLOOR rounding mode. ans = self.copy_abs() else: ans = self.copy_negate() - if context is None: - context = getcontext() return ans._fix(context) def __pos__(self, context=None): @@ -1088,14 +1090,15 @@ if ans: return ans - if not self: - # + (-0) = 0 + if context is None: + context = getcontext() + + if not self and context.rounding != ROUND_FLOOR: + # + (-0) = 0, except in ROUND_FLOOR rounding mode. ans = self.copy_abs() else: ans = Decimal(self) - if context is None: - context = getcontext() return ans._fix(context) def __abs__(self, round=True, context=None): @@ -1680,7 +1683,7 @@ self = _dec_from_triple(self._sign, '1', exp_min-1) digits = 0 rounding_method = self._pick_rounding_function[context.rounding] - changed = getattr(self, rounding_method)(digits) + changed = rounding_method(self, digits) coeff = self._int[:digits] or '0' if changed > 0: coeff = str(int(coeff)+1) @@ -1720,8 +1723,6 @@ # here self was representable to begin with; return unchanged return Decimal(self) - _pick_rounding_function = {} - # for each of the rounding functions below: # self is a finite, nonzero Decimal # prec is an integer satisfying 0 <= prec < len(self._int) @@ -1788,6 +1789,17 @@ else: return -self._round_down(prec) + _pick_rounding_function = dict( + ROUND_DOWN = _round_down, + ROUND_UP = _round_up, + ROUND_HALF_UP = _round_half_up, + ROUND_HALF_DOWN = _round_half_down, + ROUND_HALF_EVEN = _round_half_even, + ROUND_CEILING = _round_ceiling, + ROUND_FLOOR = _round_floor, + ROUND_05UP = _round_05up, + ) + def fma(self, other, third, context=None): """Fused multiply-add. @@ -2492,8 +2504,8 @@ if digits < 0: self = _dec_from_triple(self._sign, '1', exp-1) digits = 0 - this_function = getattr(self, self._pick_rounding_function[rounding]) - changed = this_function(digits) + this_function = self._pick_rounding_function[rounding] + changed = this_function(self, digits) coeff = self._int[:digits] or '0' if changed == 1: coeff = str(int(coeff)+1) @@ -3705,18 +3717,6 @@ ##### Context class ####################################################### - -# get rounding method function: -rounding_functions = [name for name in Decimal.__dict__.keys() - if name.startswith('_round_')] -for name in rounding_functions: - # name is like _round_half_even, goes to the global ROUND_HALF_EVEN value. - globalname = name[1:].upper() - val = globals()[globalname] - Decimal._pick_rounding_function[val] = name - -del name, val, globalname, rounding_functions - class _ContextManager(object): """Context manager class to support localcontext(). @@ -5990,7 +5990,7 @@ def _format_align(sign, body, spec): """Given an unpadded, non-aligned numeric string 'body' and sign - string 'sign', add padding and aligment conforming to the given + string 'sign', add padding and alignment conforming to the given format specifier dictionary 'spec' (as produced by parse_format_specifier). diff --git a/lib-python/2.7/difflib.py b/lib-python/2.7/difflib.py --- a/lib-python/2.7/difflib.py +++ b/lib-python/2.7/difflib.py @@ -1140,6 +1140,21 @@ return ch in ws +######################################################################## +### Unified Diff +######################################################################## + +def _format_range_unified(start, stop): + 'Convert range to the "ed" format' + # Per the diff spec at http://www.unix.org/single_unix_specification/ + beginning = start + 1 # lines start numbering with one + length = stop - start + if length == 1: + return '{}'.format(beginning) + if not length: + beginning -= 1 # empty ranges begin at line just before the range + return '{},{}'.format(beginning, length) + def unified_diff(a, b, fromfile='', tofile='', fromfiledate='', tofiledate='', n=3, lineterm='\n'): r""" @@ -1184,25 +1199,45 @@ started = False for group in SequenceMatcher(None,a,b).get_grouped_opcodes(n): if not started: - fromdate = '\t%s' % fromfiledate if fromfiledate else '' - todate = '\t%s' % tofiledate if tofiledate else '' - yield '--- %s%s%s' % (fromfile, fromdate, lineterm) - yield '+++ %s%s%s' % (tofile, todate, lineterm) started = True - i1, i2, j1, j2 = group[0][1], group[-1][2], group[0][3], group[-1][4] - yield "@@ -%d,%d +%d,%d @@%s" % (i1+1, i2-i1, j1+1, j2-j1, lineterm) + fromdate = '\t{}'.format(fromfiledate) if fromfiledate else '' + todate = '\t{}'.format(tofiledate) if tofiledate else '' + yield '--- {}{}{}'.format(fromfile, fromdate, lineterm) + yield '+++ {}{}{}'.format(tofile, todate, lineterm) + + first, last = group[0], group[-1] + file1_range = _format_range_unified(first[1], last[2]) + file2_range = _format_range_unified(first[3], last[4]) + yield '@@ -{} +{} @@{}'.format(file1_range, file2_range, lineterm) + for tag, i1, i2, j1, j2 in group: if tag == 'equal': for line in a[i1:i2]: yield ' ' + line continue - if tag == 'replace' or tag == 'delete': + if tag in ('replace', 'delete'): for line in a[i1:i2]: yield '-' + line - if tag == 'replace' or tag == 'insert': + if tag in ('replace', 'insert'): for line in b[j1:j2]: yield '+' + line + +######################################################################## +### Context Diff +######################################################################## + +def _format_range_context(start, stop): + 'Convert range to the "ed" format' + # Per the diff spec at http://www.unix.org/single_unix_specification/ + beginning = start + 1 # lines start numbering with one + length = stop - start + if not length: + beginning -= 1 # empty ranges begin at line just before the range + if length <= 1: + return '{}'.format(beginning) + return '{},{}'.format(beginning, beginning + length - 1) + # See http://www.unix.org/single_unix_specification/ def context_diff(a, b, fromfile='', tofile='', fromfiledate='', tofiledate='', n=3, lineterm='\n'): @@ -1247,38 +1282,36 @@ four """ + prefix = dict(insert='+ ', delete='- ', replace='! ', equal=' ') started = False - prefixmap = {'insert':'+ ', 'delete':'- ', 'replace':'! ', 'equal':' '} for group in SequenceMatcher(None,a,b).get_grouped_opcodes(n): if not started: - fromdate = '\t%s' % fromfiledate if fromfiledate else '' - todate = '\t%s' % tofiledate if tofiledate else '' - yield '*** %s%s%s' % (fromfile, fromdate, lineterm) - yield '--- %s%s%s' % (tofile, todate, lineterm) started = True + fromdate = '\t{}'.format(fromfiledate) if fromfiledate else '' + todate = '\t{}'.format(tofiledate) if tofiledate else '' + yield '*** {}{}{}'.format(fromfile, fromdate, lineterm) + yield '--- {}{}{}'.format(tofile, todate, lineterm) - yield '***************%s' % (lineterm,) - if group[-1][2] - group[0][1] >= 2: - yield '*** %d,%d ****%s' % (group[0][1]+1, group[-1][2], lineterm) - else: - yield '*** %d ****%s' % (group[-1][2], lineterm) - visiblechanges = [e for e in group if e[0] in ('replace', 'delete')] - if visiblechanges: + first, last = group[0], group[-1] + yield '***************' + lineterm + + file1_range = _format_range_context(first[1], last[2]) + yield '*** {} ****{}'.format(file1_range, lineterm) + + if any(tag in ('replace', 'delete') for tag, _, _, _, _ in group): for tag, i1, i2, _, _ in group: if tag != 'insert': for line in a[i1:i2]: - yield prefixmap[tag] + line + yield prefix[tag] + line - if group[-1][4] - group[0][3] >= 2: - yield '--- %d,%d ----%s' % (group[0][3]+1, group[-1][4], lineterm) - else: - yield '--- %d ----%s' % (group[-1][4], lineterm) - visiblechanges = [e for e in group if e[0] in ('replace', 'insert')] - if visiblechanges: + file2_range = _format_range_context(first[3], last[4]) + yield '--- {} ----{}'.format(file2_range, lineterm) + + if any(tag in ('replace', 'insert') for tag, _, _, _, _ in group): for tag, _, _, j1, j2 in group: if tag != 'delete': for line in b[j1:j2]: - yield prefixmap[tag] + line + yield prefix[tag] + line def ndiff(a, b, linejunk=None, charjunk=IS_CHARACTER_JUNK): r""" @@ -1714,7 +1747,7 @@ line = line.replace(' ','\0') # expand tabs into spaces line = line.expandtabs(self._tabsize) - # relace spaces from expanded tabs back into tab characters + # replace spaces from expanded tabs back into tab characters # (we'll replace them with markup after we do differencing) line = line.replace(' ','\t') return line.replace('\0',' ').rstrip('\n') diff --git a/lib-python/2.7/distutils/__init__.py b/lib-python/2.7/distutils/__init__.py --- a/lib-python/2.7/distutils/__init__.py +++ b/lib-python/2.7/distutils/__init__.py @@ -15,5 +15,5 @@ # Updated automatically by the Python release process. # #--start constants-- -__version__ = "2.7.1" +__version__ = "2.7.2" #--end constants-- diff --git a/lib-python/2.7/distutils/archive_util.py b/lib-python/2.7/distutils/archive_util.py --- a/lib-python/2.7/distutils/archive_util.py +++ b/lib-python/2.7/distutils/archive_util.py @@ -121,7 +121,7 @@ def make_zipfile(base_name, base_dir, verbose=0, dry_run=0): """Create a zip file from all the files under 'base_dir'. - The output zip file will be named 'base_dir' + ".zip". Uses either the + The output zip file will be named 'base_name' + ".zip". Uses either the "zipfile" Python module (if available) or the InfoZIP "zip" utility (if installed and found on the default search path). If neither tool is available, raises DistutilsExecError. Returns the name of the output zip diff --git a/lib-python/2.7/distutils/cmd.py b/lib-python/2.7/distutils/cmd.py --- a/lib-python/2.7/distutils/cmd.py +++ b/lib-python/2.7/distutils/cmd.py @@ -377,7 +377,7 @@ dry_run=self.dry_run) def move_file (self, src, dst, level=1): - """Move a file respectin dry-run flag.""" + """Move a file respecting dry-run flag.""" return file_util.move_file(src, dst, dry_run = self.dry_run) def spawn (self, cmd, search_path=1, level=1): diff --git a/lib-python/2.7/distutils/command/build_ext.py b/lib-python/2.7/distutils/command/build_ext.py --- a/lib-python/2.7/distutils/command/build_ext.py +++ b/lib-python/2.7/distutils/command/build_ext.py @@ -207,7 +207,7 @@ elif MSVC_VERSION == 8: self.library_dirs.append(os.path.join(sys.exec_prefix, - 'PC', 'VS8.0', 'win32release')) + 'PC', 'VS8.0')) elif MSVC_VERSION == 7: self.library_dirs.append(os.path.join(sys.exec_prefix, 'PC', 'VS7.1')) diff --git a/lib-python/2.7/distutils/command/sdist.py b/lib-python/2.7/distutils/command/sdist.py --- a/lib-python/2.7/distutils/command/sdist.py +++ b/lib-python/2.7/distutils/command/sdist.py @@ -306,17 +306,20 @@ rstrip_ws=1, collapse_join=1) - while 1: - line = template.readline() - if line is None: # end of file - break + try: + while 1: + line = template.readline() + if line is None: # end of file + break - try: - self.filelist.process_template_line(line) - except DistutilsTemplateError, msg: - self.warn("%s, line %d: %s" % (template.filename, - template.current_line, - msg)) + try: + self.filelist.process_template_line(line) + except DistutilsTemplateError, msg: + self.warn("%s, line %d: %s" % (template.filename, + template.current_line, + msg)) + finally: + template.close() def prune_file_list(self): """Prune off branches that might slip into the file list as created diff --git a/lib-python/2.7/distutils/command/upload.py b/lib-python/2.7/distutils/command/upload.py --- a/lib-python/2.7/distutils/command/upload.py +++ b/lib-python/2.7/distutils/command/upload.py @@ -176,6 +176,9 @@ result = urlopen(request) status = result.getcode() reason = result.msg + if self.show_response: + msg = '\n'.join(('-' * 75, r.read(), '-' * 75)) + self.announce(msg, log.INFO) except socket.error, e: self.announce(str(e), log.ERROR) return @@ -189,6 +192,3 @@ else: self.announce('Upload failed (%s): %s' % (status, reason), log.ERROR) - if self.show_response: - msg = '\n'.join(('-' * 75, r.read(), '-' * 75)) - self.announce(msg, log.INFO) diff --git a/lib-python/2.7/distutils/sysconfig.py b/lib-python/2.7/distutils/sysconfig.py --- a/lib-python/2.7/distutils/sysconfig.py +++ b/lib-python/2.7/distutils/sysconfig.py @@ -389,7 +389,7 @@ cur_target = os.getenv('MACOSX_DEPLOYMENT_TARGET', '') if cur_target == '': cur_target = cfg_target - os.putenv('MACOSX_DEPLOYMENT_TARGET', cfg_target) + os.environ['MACOSX_DEPLOYMENT_TARGET'] = cfg_target elif map(int, cfg_target.split('.')) > map(int, cur_target.split('.')): my_msg = ('$MACOSX_DEPLOYMENT_TARGET mismatch: now "%s" but "%s" during configure' % (cur_target, cfg_target)) diff --git a/lib-python/2.7/distutils/tests/__init__.py b/lib-python/2.7/distutils/tests/__init__.py --- a/lib-python/2.7/distutils/tests/__init__.py +++ b/lib-python/2.7/distutils/tests/__init__.py @@ -15,9 +15,10 @@ import os import sys import unittest +from test.test_support import run_unittest -here = os.path.dirname(__file__) +here = os.path.dirname(__file__) or os.curdir def test_suite(): @@ -32,4 +33,4 @@ if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_archive_util.py b/lib-python/2.7/distutils/tests/test_archive_util.py --- a/lib-python/2.7/distutils/tests/test_archive_util.py +++ b/lib-python/2.7/distutils/tests/test_archive_util.py @@ -12,7 +12,7 @@ ARCHIVE_FORMATS) from distutils.spawn import find_executable, spawn from distutils.tests import support -from test.test_support import check_warnings +from test.test_support import check_warnings, run_unittest try: import grp @@ -281,4 +281,4 @@ return unittest.makeSuite(ArchiveUtilTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_bdist_msi.py b/lib-python/2.7/distutils/tests/test_bdist_msi.py --- a/lib-python/2.7/distutils/tests/test_bdist_msi.py +++ b/lib-python/2.7/distutils/tests/test_bdist_msi.py @@ -11,7 +11,7 @@ support.LoggingSilencer, unittest.TestCase): - def test_minial(self): + def test_minimal(self): # minimal test XXX need more tests from distutils.command.bdist_msi import bdist_msi pkg_pth, dist = self.create_dist() diff --git a/lib-python/2.7/distutils/tests/test_build.py b/lib-python/2.7/distutils/tests/test_build.py --- a/lib-python/2.7/distutils/tests/test_build.py +++ b/lib-python/2.7/distutils/tests/test_build.py @@ -2,6 +2,7 @@ import unittest import os import sys +from test.test_support import run_unittest from distutils.command.build import build from distutils.tests import support @@ -51,4 +52,4 @@ return unittest.makeSuite(BuildTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_build_clib.py b/lib-python/2.7/distutils/tests/test_build_clib.py --- a/lib-python/2.7/distutils/tests/test_build_clib.py +++ b/lib-python/2.7/distutils/tests/test_build_clib.py @@ -3,6 +3,8 @@ import os import sys +from test.test_support import run_unittest + from distutils.command.build_clib import build_clib from distutils.errors import DistutilsSetupError from distutils.tests import support @@ -140,4 +142,4 @@ return unittest.makeSuite(BuildCLibTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_build_ext.py b/lib-python/2.7/distutils/tests/test_build_ext.py --- a/lib-python/2.7/distutils/tests/test_build_ext.py +++ b/lib-python/2.7/distutils/tests/test_build_ext.py @@ -3,12 +3,13 @@ import tempfile import shutil from StringIO import StringIO +import textwrap from distutils.core import Extension, Distribution from distutils.command.build_ext import build_ext from distutils import sysconfig from distutils.tests import support -from distutils.errors import DistutilsSetupError +from distutils.errors import DistutilsSetupError, CompileError import unittest from test import test_support @@ -430,6 +431,59 @@ wanted = os.path.join(cmd.build_lib, 'UpdateManager', 'fdsend' + ext) self.assertEqual(ext_path, wanted) + @unittest.skipUnless(sys.platform == 'darwin', 'test only relevant for MacOSX') + def test_deployment_target(self): + self._try_compile_deployment_target() + + orig_environ = os.environ + os.environ = orig_environ.copy() + self.addCleanup(setattr, os, 'environ', orig_environ) + + os.environ['MACOSX_DEPLOYMENT_TARGET']='10.1' + self._try_compile_deployment_target() + + + def _try_compile_deployment_target(self): + deptarget_c = os.path.join(self.tmp_dir, 'deptargetmodule.c') + + with open(deptarget_c, 'w') as fp: + fp.write(textwrap.dedent('''\ + #include <AvailabilityMacros.h> + + int dummy; + + #if TARGET != MAC_OS_X_VERSION_MIN_REQUIRED + #error "Unexpected target" + #endif + + ''')) + + target = sysconfig.get_config_var('MACOSX_DEPLOYMENT_TARGET') + target = tuple(map(int, target.split('.'))) + target = '%02d%01d0' % target + + deptarget_ext = Extension( + 'deptarget', + [deptarget_c], + extra_compile_args=['-DTARGET=%s'%(target,)], + ) + dist = Distribution({ + 'name': 'deptarget', + 'ext_modules': [deptarget_ext] + }) + dist.package_dir = self.tmp_dir + cmd = build_ext(dist) + cmd.build_lib = self.tmp_dir + cmd.build_temp = self.tmp_dir + + try: + old_stdout = sys.stdout + cmd.ensure_finalized() + cmd.run() + + except CompileError: + self.fail("Wrong deployment target during compilation") + def test_suite(): return unittest.makeSuite(BuildExtTestCase) diff --git a/lib-python/2.7/distutils/tests/test_build_py.py b/lib-python/2.7/distutils/tests/test_build_py.py --- a/lib-python/2.7/distutils/tests/test_build_py.py +++ b/lib-python/2.7/distutils/tests/test_build_py.py @@ -10,13 +10,14 @@ from distutils.errors import DistutilsFileError from distutils.tests import support +from test.test_support import run_unittest class BuildPyTestCase(support.TempdirManager, support.LoggingSilencer, unittest.TestCase): - def _setup_package_data(self): + def test_package_data(self): sources = self.mkdtemp() f = open(os.path.join(sources, "__init__.py"), "w") try: @@ -56,20 +57,15 @@ self.assertEqual(len(cmd.get_outputs()), 3) pkgdest = os.path.join(destination, "pkg") files = os.listdir(pkgdest) - return files + self.assertIn("__init__.py", files) + self.assertIn("README.txt", files) + # XXX even with -O, distutils writes pyc, not pyo; bug? + if sys.dont_write_bytecode: + self.assertNotIn("__init__.pyc", files) + else: + self.assertIn("__init__.pyc", files) - def test_package_data(self): - files = self._setup_package_data() - self.assertTrue("__init__.py" in files) - self.assertTrue("README.txt" in files) - - @unittest.skipIf(sys.flags.optimize >= 2, - "pyc files are not written with -O2 and above") - def test_package_data_pyc(self): - files = self._setup_package_data() - self.assertTrue("__init__.pyc" in files) - - def test_empty_package_dir (self): + def test_empty_package_dir(self): # See SF 1668596/1720897. cwd = os.getcwd() @@ -117,10 +113,10 @@ finally: sys.dont_write_bytecode = old_dont_write_bytecode - self.assertTrue('byte-compiling is disabled' in self.logs[0][1]) + self.assertIn('byte-compiling is disabled', self.logs[0][1]) def test_suite(): return unittest.makeSuite(BuildPyTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_build_scripts.py b/lib-python/2.7/distutils/tests/test_build_scripts.py --- a/lib-python/2.7/distutils/tests/test_build_scripts.py +++ b/lib-python/2.7/distutils/tests/test_build_scripts.py @@ -8,6 +8,7 @@ import sysconfig from distutils.tests import support +from test.test_support import run_unittest class BuildScriptsTestCase(support.TempdirManager, @@ -108,4 +109,4 @@ return unittest.makeSuite(BuildScriptsTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_check.py b/lib-python/2.7/distutils/tests/test_check.py --- a/lib-python/2.7/distutils/tests/test_check.py +++ b/lib-python/2.7/distutils/tests/test_check.py @@ -1,5 +1,6 @@ """Tests for distutils.command.check.""" import unittest +from test.test_support import run_unittest from distutils.command.check import check, HAS_DOCUTILS from distutils.tests import support @@ -95,4 +96,4 @@ return unittest.makeSuite(CheckTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_clean.py b/lib-python/2.7/distutils/tests/test_clean.py --- a/lib-python/2.7/distutils/tests/test_clean.py +++ b/lib-python/2.7/distutils/tests/test_clean.py @@ -6,6 +6,7 @@ from distutils.command.clean import clean from distutils.tests import support +from test.test_support import run_unittest class cleanTestCase(support.TempdirManager, support.LoggingSilencer, @@ -38,7 +39,7 @@ self.assertTrue(not os.path.exists(path), '%s was not removed' % path) - # let's run the command again (should spit warnings but suceed) + # let's run the command again (should spit warnings but succeed) cmd.all = 1 cmd.ensure_finalized() cmd.run() @@ -47,4 +48,4 @@ return unittest.makeSuite(cleanTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_cmd.py b/lib-python/2.7/distutils/tests/test_cmd.py --- a/lib-python/2.7/distutils/tests/test_cmd.py +++ b/lib-python/2.7/distutils/tests/test_cmd.py @@ -99,7 +99,7 @@ def test_ensure_dirname(self): cmd = self.cmd - cmd.option1 = os.path.dirname(__file__) + cmd.option1 = os.path.dirname(__file__) or os.curdir cmd.ensure_dirname('option1') cmd.option2 = 'xxx' self.assertRaises(DistutilsOptionError, cmd.ensure_dirname, 'option2') diff --git a/lib-python/2.7/distutils/tests/test_config.py b/lib-python/2.7/distutils/tests/test_config.py --- a/lib-python/2.7/distutils/tests/test_config.py +++ b/lib-python/2.7/distutils/tests/test_config.py @@ -11,6 +11,7 @@ from distutils.log import WARN from distutils.tests import support +from test.test_support import run_unittest PYPIRC = """\ [distutils] @@ -119,4 +120,4 @@ return unittest.makeSuite(PyPIRCCommandTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_config_cmd.py b/lib-python/2.7/distutils/tests/test_config_cmd.py --- a/lib-python/2.7/distutils/tests/test_config_cmd.py +++ b/lib-python/2.7/distutils/tests/test_config_cmd.py @@ -2,6 +2,7 @@ import unittest import os import sys +from test.test_support import run_unittest from distutils.command.config import dump_file, config from distutils.tests import support @@ -86,4 +87,4 @@ return unittest.makeSuite(ConfigTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_core.py b/lib-python/2.7/distutils/tests/test_core.py --- a/lib-python/2.7/distutils/tests/test_core.py +++ b/lib-python/2.7/distutils/tests/test_core.py @@ -6,7 +6,7 @@ import shutil import sys import test.test_support -from test.test_support import captured_stdout +from test.test_support import captured_stdout, run_unittest import unittest from distutils.tests import support @@ -105,4 +105,4 @@ return unittest.makeSuite(CoreTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_dep_util.py b/lib-python/2.7/distutils/tests/test_dep_util.py --- a/lib-python/2.7/distutils/tests/test_dep_util.py +++ b/lib-python/2.7/distutils/tests/test_dep_util.py @@ -6,6 +6,7 @@ from distutils.dep_util import newer, newer_pairwise, newer_group from distutils.errors import DistutilsFileError from distutils.tests import support +from test.test_support import run_unittest class DepUtilTestCase(support.TempdirManager, unittest.TestCase): @@ -77,4 +78,4 @@ return unittest.makeSuite(DepUtilTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_dir_util.py b/lib-python/2.7/distutils/tests/test_dir_util.py --- a/lib-python/2.7/distutils/tests/test_dir_util.py +++ b/lib-python/2.7/distutils/tests/test_dir_util.py @@ -10,6 +10,7 @@ from distutils import log from distutils.tests import support +from test.test_support import run_unittest class DirUtilTestCase(support.TempdirManager, unittest.TestCase): @@ -112,4 +113,4 @@ return unittest.makeSuite(DirUtilTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_dist.py b/lib-python/2.7/distutils/tests/test_dist.py --- a/lib-python/2.7/distutils/tests/test_dist.py +++ b/lib-python/2.7/distutils/tests/test_dist.py @@ -11,7 +11,7 @@ from distutils.dist import Distribution, fix_help_options, DistributionMetadata from distutils.cmd import Command import distutils.dist -from test.test_support import TESTFN, captured_stdout +from test.test_support import TESTFN, captured_stdout, run_unittest from distutils.tests import support class test_dist(Command): @@ -433,4 +433,4 @@ return suite if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_file_util.py b/lib-python/2.7/distutils/tests/test_file_util.py --- a/lib-python/2.7/distutils/tests/test_file_util.py +++ b/lib-python/2.7/distutils/tests/test_file_util.py @@ -6,6 +6,7 @@ from distutils.file_util import move_file, write_file, copy_file from distutils import log from distutils.tests import support +from test.test_support import run_unittest class FileUtilTestCase(support.TempdirManager, unittest.TestCase): @@ -77,4 +78,4 @@ return unittest.makeSuite(FileUtilTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_filelist.py b/lib-python/2.7/distutils/tests/test_filelist.py --- a/lib-python/2.7/distutils/tests/test_filelist.py +++ b/lib-python/2.7/distutils/tests/test_filelist.py @@ -1,7 +1,7 @@ """Tests for distutils.filelist.""" from os.path import join import unittest -from test.test_support import captured_stdout +from test.test_support import captured_stdout, run_unittest from distutils.filelist import glob_to_re, FileList from distutils import debug @@ -82,4 +82,4 @@ return unittest.makeSuite(FileListTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_install.py b/lib-python/2.7/distutils/tests/test_install.py --- a/lib-python/2.7/distutils/tests/test_install.py +++ b/lib-python/2.7/distutils/tests/test_install.py @@ -3,6 +3,8 @@ import os import unittest +from test.test_support import run_unittest + from distutils.command.install import install from distutils.core import Distribution @@ -52,4 +54,4 @@ return unittest.makeSuite(InstallTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_install_data.py b/lib-python/2.7/distutils/tests/test_install_data.py --- a/lib-python/2.7/distutils/tests/test_install_data.py +++ b/lib-python/2.7/distutils/tests/test_install_data.py @@ -6,6 +6,7 @@ from distutils.command.install_data import install_data from distutils.tests import support +from test.test_support import run_unittest class InstallDataTestCase(support.TempdirManager, support.LoggingSilencer, @@ -73,4 +74,4 @@ return unittest.makeSuite(InstallDataTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_install_headers.py b/lib-python/2.7/distutils/tests/test_install_headers.py --- a/lib-python/2.7/distutils/tests/test_install_headers.py +++ b/lib-python/2.7/distutils/tests/test_install_headers.py @@ -6,6 +6,7 @@ from distutils.command.install_headers import install_headers from distutils.tests import support +from test.test_support import run_unittest class InstallHeadersTestCase(support.TempdirManager, support.LoggingSilencer, @@ -37,4 +38,4 @@ return unittest.makeSuite(InstallHeadersTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_install_lib.py b/lib-python/2.7/distutils/tests/test_install_lib.py --- a/lib-python/2.7/distutils/tests/test_install_lib.py +++ b/lib-python/2.7/distutils/tests/test_install_lib.py @@ -7,6 +7,7 @@ from distutils.extension import Extension from distutils.tests import support from distutils.errors import DistutilsOptionError +from test.test_support import run_unittest class InstallLibTestCase(support.TempdirManager, support.LoggingSilencer, @@ -103,4 +104,4 @@ return unittest.makeSuite(InstallLibTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_install_scripts.py b/lib-python/2.7/distutils/tests/test_install_scripts.py --- a/lib-python/2.7/distutils/tests/test_install_scripts.py +++ b/lib-python/2.7/distutils/tests/test_install_scripts.py @@ -7,6 +7,7 @@ from distutils.core import Distribution from distutils.tests import support +from test.test_support import run_unittest class InstallScriptsTestCase(support.TempdirManager, @@ -78,4 +79,4 @@ return unittest.makeSuite(InstallScriptsTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_msvc9compiler.py b/lib-python/2.7/distutils/tests/test_msvc9compiler.py --- a/lib-python/2.7/distutils/tests/test_msvc9compiler.py +++ b/lib-python/2.7/distutils/tests/test_msvc9compiler.py @@ -5,6 +5,7 @@ from distutils.errors import DistutilsPlatformError from distutils.tests import support +from test.test_support import run_unittest _MANIFEST = """\ <?xml version="1.0" encoding="UTF-8" standalone="yes"?> @@ -137,4 +138,4 @@ return unittest.makeSuite(msvc9compilerTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_register.py b/lib-python/2.7/distutils/tests/test_register.py --- a/lib-python/2.7/distutils/tests/test_register.py +++ b/lib-python/2.7/distutils/tests/test_register.py @@ -7,7 +7,7 @@ import urllib2 import warnings -from test.test_support import check_warnings +from test.test_support import check_warnings, run_unittest from distutils.command import register as register_module from distutils.command.register import register @@ -138,7 +138,7 @@ # let's see what the server received : we should # have 2 similar requests - self.assertTrue(self.conn.reqs, 2) + self.assertEqual(len(self.conn.reqs), 2) req1 = dict(self.conn.reqs[0].headers) req2 = dict(self.conn.reqs[1].headers) self.assertEqual(req2['Content-length'], req1['Content-length']) @@ -168,7 +168,7 @@ del register_module.raw_input # we should have send a request - self.assertTrue(self.conn.reqs, 1) + self.assertEqual(len(self.conn.reqs), 1) req = self.conn.reqs[0] headers = dict(req.headers) self.assertEqual(headers['Content-length'], '608') @@ -186,7 +186,7 @@ del register_module.raw_input # we should have send a request - self.assertTrue(self.conn.reqs, 1) + self.assertEqual(len(self.conn.reqs), 1) req = self.conn.reqs[0] headers = dict(req.headers) self.assertEqual(headers['Content-length'], '290') @@ -258,4 +258,4 @@ return unittest.makeSuite(RegisterTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_sdist.py b/lib-python/2.7/distutils/tests/test_sdist.py --- a/lib-python/2.7/distutils/tests/test_sdist.py +++ b/lib-python/2.7/distutils/tests/test_sdist.py @@ -24,11 +24,9 @@ import tempfile import warnings -from test.test_support import check_warnings -from test.test_support import captured_stdout +from test.test_support import captured_stdout, check_warnings, run_unittest -from distutils.command.sdist import sdist -from distutils.command.sdist import show_formats +from distutils.command.sdist import sdist, show_formats from distutils.core import Distribution from distutils.tests.test_config import PyPIRCCommandTestCase from distutils.errors import DistutilsExecError, DistutilsOptionError @@ -372,7 +370,7 @@ # adding a file self.write_file((self.tmp_dir, 'somecode', 'doc2.txt'), '#') - # make sure build_py is reinitinialized, like a fresh run + # make sure build_py is reinitialized, like a fresh run build_py = dist.get_command_obj('build_py') build_py.finalized = False build_py.ensure_finalized() @@ -390,6 +388,7 @@ self.assertEqual(len(manifest2), 6) self.assertIn('doc2.txt', manifest2[-1]) + @unittest.skipUnless(zlib, "requires zlib") def test_manifest_marker(self): # check that autogenerated MANIFESTs have a marker dist, cmd = self.get_cmd() @@ -406,6 +405,7 @@ self.assertEqual(manifest[0], '# file GENERATED by distutils, do NOT edit') + @unittest.skipUnless(zlib, "requires zlib") def test_manual_manifest(self): # check that a MANIFEST without a marker is left alone dist, cmd = self.get_cmd() @@ -426,4 +426,4 @@ return unittest.makeSuite(SDistTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_spawn.py b/lib-python/2.7/distutils/tests/test_spawn.py --- a/lib-python/2.7/distutils/tests/test_spawn.py +++ b/lib-python/2.7/distutils/tests/test_spawn.py @@ -2,7 +2,7 @@ import unittest import os import time -from test.test_support import captured_stdout +from test.test_support import captured_stdout, run_unittest from distutils.spawn import _nt_quote_args from distutils.spawn import spawn, find_executable @@ -57,4 +57,4 @@ return unittest.makeSuite(SpawnTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_text_file.py b/lib-python/2.7/distutils/tests/test_text_file.py --- a/lib-python/2.7/distutils/tests/test_text_file.py +++ b/lib-python/2.7/distutils/tests/test_text_file.py @@ -3,6 +3,7 @@ import unittest from distutils.text_file import TextFile from distutils.tests import support +from test.test_support import run_unittest TEST_DATA = """# test file @@ -103,4 +104,4 @@ return unittest.makeSuite(TextFileTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_unixccompiler.py b/lib-python/2.7/distutils/tests/test_unixccompiler.py --- a/lib-python/2.7/distutils/tests/test_unixccompiler.py +++ b/lib-python/2.7/distutils/tests/test_unixccompiler.py @@ -1,6 +1,7 @@ """Tests for distutils.unixccompiler.""" import sys import unittest +from test.test_support import run_unittest from distutils import sysconfig from distutils.unixccompiler import UnixCCompiler @@ -126,4 +127,4 @@ return unittest.makeSuite(UnixCCompilerTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_upload.py b/lib-python/2.7/distutils/tests/test_upload.py --- a/lib-python/2.7/distutils/tests/test_upload.py +++ b/lib-python/2.7/distutils/tests/test_upload.py @@ -1,14 +1,13 @@ +# -*- encoding: utf8 -*- """Tests for distutils.command.upload.""" -# -*- encoding: utf8 -*- -import sys import os import unittest +from test.test_support import run_unittest from distutils.command import upload as upload_mod from distutils.command.upload import upload from distutils.core import Distribution -from distutils.tests import support from distutils.tests.test_config import PYPIRC, PyPIRCCommandTestCase PYPIRC_LONG_PASSWORD = """\ @@ -129,4 +128,4 @@ return unittest.makeSuite(uploadTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_util.py b/lib-python/2.7/distutils/tests/test_util.py --- a/lib-python/2.7/distutils/tests/test_util.py +++ b/lib-python/2.7/distutils/tests/test_util.py @@ -1,6 +1,7 @@ """Tests for distutils.util.""" import sys import unittest +from test.test_support import run_unittest from distutils.errors import DistutilsPlatformError, DistutilsByteCompileError from distutils.util import byte_compile @@ -21,4 +22,4 @@ return unittest.makeSuite(UtilTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_version.py b/lib-python/2.7/distutils/tests/test_version.py --- a/lib-python/2.7/distutils/tests/test_version.py +++ b/lib-python/2.7/distutils/tests/test_version.py @@ -2,6 +2,7 @@ import unittest from distutils.version import LooseVersion from distutils.version import StrictVersion +from test.test_support import run_unittest class VersionTestCase(unittest.TestCase): @@ -67,4 +68,4 @@ return unittest.makeSuite(VersionTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_versionpredicate.py b/lib-python/2.7/distutils/tests/test_versionpredicate.py --- a/lib-python/2.7/distutils/tests/test_versionpredicate.py +++ b/lib-python/2.7/distutils/tests/test_versionpredicate.py @@ -4,6 +4,10 @@ import distutils.versionpredicate import doctest +from test.test_support import run_unittest def test_suite(): return doctest.DocTestSuite(distutils.versionpredicate) + +if __name__ == '__main__': + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/util.py b/lib-python/2.7/distutils/util.py --- a/lib-python/2.7/distutils/util.py +++ b/lib-python/2.7/distutils/util.py @@ -97,9 +97,7 @@ from distutils.sysconfig import get_config_vars cfgvars = get_config_vars() - macver = os.environ.get('MACOSX_DEPLOYMENT_TARGET') - if not macver: - macver = cfgvars.get('MACOSX_DEPLOYMENT_TARGET') + macver = cfgvars.get('MACOSX_DEPLOYMENT_TARGET') if 1: # Always calculate the release of the running machine, diff --git a/lib-python/2.7/doctest.py b/lib-python/2.7/doctest.py --- a/lib-python/2.7/doctest.py +++ b/lib-python/2.7/doctest.py @@ -1217,7 +1217,7 @@ # Process each example. for examplenum, example in enumerate(test.examples): - # If REPORT_ONLY_FIRST_FAILURE is set, then supress + # If REPORT_ONLY_FIRST_FAILURE is set, then suppress # reporting after the first failure. quiet = (self.optionflags & REPORT_ONLY_FIRST_FAILURE and failures > 0) @@ -2186,7 +2186,7 @@ caller can catch the errors and initiate post-mortem debugging. The DocTestCase provides a debug method that raises - UnexpectedException errors if there is an unexepcted + UnexpectedException errors if there is an unexpected exception: >>> test = DocTestParser().get_doctest('>>> raise KeyError\n42', diff --git a/lib-python/2.7/email/charset.py b/lib-python/2.7/email/charset.py --- a/lib-python/2.7/email/charset.py +++ b/lib-python/2.7/email/charset.py @@ -209,7 +209,7 @@ input_charset = unicode(input_charset, 'ascii') except UnicodeError: raise errors.CharsetError(input_charset) - input_charset = input_charset.lower() + input_charset = input_charset.lower().encode('ascii') # Set the input charset after filtering through the aliases and/or codecs if not (input_charset in ALIASES or input_charset in CHARSETS): try: diff --git a/lib-python/2.7/email/generator.py b/lib-python/2.7/email/generator.py --- a/lib-python/2.7/email/generator.py +++ b/lib-python/2.7/email/generator.py @@ -202,18 +202,13 @@ g = self.clone(s) g.flatten(part, unixfrom=False) msgtexts.append(s.getvalue()) - # Now make sure the boundary we've selected doesn't appear in any of - # the message texts. - alltext = NL.join(msgtexts) # BAW: What about boundaries that are wrapped in double-quotes? - boundary = msg.get_boundary(failobj=_make_boundary(alltext)) - # If we had to calculate a new boundary because the body text - # contained that string, set the new boundary. We don't do it - # unconditionally because, while set_boundary() preserves order, it - # doesn't preserve newlines/continuations in headers. This is no big - # deal in practice, but turns out to be inconvenient for the unittest - # suite. - if msg.get_boundary() != boundary: + boundary = msg.get_boundary() + if not boundary: + # Create a boundary that doesn't appear in any of the + # message texts. + alltext = NL.join(msgtexts) + boundary = _make_boundary(alltext) msg.set_boundary(boundary) # If there's a preamble, write it out, with a trailing CRLF if msg.preamble is not None: @@ -292,7 +287,7 @@ _FMT = '[Non-text (%(type)s) part of message omitted, filename %(filename)s]' class DecodedGenerator(Generator): - """Generator a text representation of a message. + """Generates a text representation of a message. Like the Generator base class, except that non-text parts are substituted with a format string representing the part. diff --git a/lib-python/2.7/email/header.py b/lib-python/2.7/email/header.py --- a/lib-python/2.7/email/header.py +++ b/lib-python/2.7/email/header.py @@ -47,6 +47,10 @@ # For use with .match() fcre = re.compile(r'[\041-\176]+:$') +# Find a header embedded in a putative header value. Used to check for +# header injection attack. +_embeded_header = re.compile(r'\n[^ \t]+:') + # Helpers @@ -403,7 +407,11 @@ newchunks += self._split(s, charset, targetlen, splitchars) lastchunk, lastcharset = newchunks[-1] lastlen = lastcharset.encoded_header_len(lastchunk) - return self._encode_chunks(newchunks, maxlinelen) + value = self._encode_chunks(newchunks, maxlinelen) + if _embeded_header.search(value): + raise HeaderParseError("header value appears to contain " + "an embedded header: {!r}".format(value)) + return value diff --git a/lib-python/2.7/email/message.py b/lib-python/2.7/email/message.py --- a/lib-python/2.7/email/message.py +++ b/lib-python/2.7/email/message.py @@ -38,7 +38,9 @@ def _formatparam(param, value=None, quote=True): """Convenience function to format and return a key=value pair. - This will quote the value if needed or if quote is true. + This will quote the value if needed or if quote is true. If value is a + three tuple (charset, language, value), it will be encoded according + to RFC2231 rules. """ if value is not None and len(value) > 0: # A tuple is used for RFC 2231 encoded parameter values where items @@ -97,7 +99,7 @@ objects, otherwise it is a string. Message objects implement part of the `mapping' interface, which assumes - there is exactly one occurrance of the header per message. Some headers + there is exactly one occurrence of the header per message. Some headers do in fact appear multiple times (e.g. Received) and for those headers, you must use the explicit API to set or get all the headers. Not all of the mapping methods are implemented. @@ -286,7 +288,7 @@ Return None if the header is missing instead of raising an exception. Note that if the header appeared multiple times, exactly which - occurrance gets returned is undefined. Use get_all() to get all + occurrence gets returned is undefined. Use get_all() to get all the values matching a header field name. """ return self.get(name) @@ -389,7 +391,10 @@ name is the header field to add. keyword arguments can be used to set additional parameters for the header field, with underscores converted to dashes. Normally the parameter will be added as key="value" unless - value is None, in which case only the key will be added. + value is None, in which case only the key will be added. If a + parameter value contains non-ASCII characters it must be specified as a + three-tuple of (charset, language, value), in which case it will be + encoded according to RFC2231 rules. Example: diff --git a/lib-python/2.7/email/mime/application.py b/lib-python/2.7/email/mime/application.py --- a/lib-python/2.7/email/mime/application.py +++ b/lib-python/2.7/email/mime/application.py @@ -17,7 +17,7 @@ _encoder=encoders.encode_base64, **_params): """Create an application/* type MIME document. - _data is a string containing the raw applicatoin data. + _data is a string containing the raw application data. _subtype is the MIME content type subtype, defaulting to 'octet-stream'. diff --git a/lib-python/2.7/email/test/data/msg_26.txt b/lib-python/2.7/email/test/data/msg_26.txt --- a/lib-python/2.7/email/test/data/msg_26.txt +++ b/lib-python/2.7/email/test/data/msg_26.txt @@ -42,4 +42,4 @@ MzMAAAAACH97tzAAAAALu3c3gAAAAAAL+7tzDABAu7f7cAAAAAAACA+3MA7EQAv/sIAA AAAAAAAIAAAAAAAAAIAAAAAA ---1618492860--2051301190--113853680-- +--1618492860--2051301190--113853680-- \ No newline at end of file diff --git a/lib-python/2.7/email/test/test_email.py b/lib-python/2.7/email/test/test_email.py --- a/lib-python/2.7/email/test/test_email.py +++ b/lib-python/2.7/email/test/test_email.py @@ -179,6 +179,17 @@ self.assertRaises(Errors.HeaderParseError, msg.set_boundary, 'BOUNDARY') + def test_make_boundary(self): + msg = MIMEMultipart('form-data') + # Note that when the boundary gets created is an implementation + # detail and might change. + self.assertEqual(msg.items()[0][1], 'multipart/form-data') + # Trigger creation of boundary + msg.as_string() + self.assertEqual(msg.items()[0][1][:33], + 'multipart/form-data; boundary="==') + # XXX: there ought to be tests of the uniqueness of the boundary, too. + def test_message_rfc822_only(self): # Issue 7970: message/rfc822 not in multipart parsed by # HeaderParser caused an exception when flattened. @@ -542,6 +553,17 @@ msg.set_charset(u'us-ascii') self.assertEqual('us-ascii', msg.get_content_charset()) + # Issue 5871: reject an attempt to embed a header inside a header value + # (header injection attack). + def test_embeded_header_via_Header_rejected(self): + msg = Message() + msg['Dummy'] = Header('dummy\nX-Injected-Header: test') + self.assertRaises(Errors.HeaderParseError, msg.as_string) + + def test_embeded_header_via_string_rejected(self): + msg = Message() + msg['Dummy'] = 'dummy\nX-Injected-Header: test' + self.assertRaises(Errors.HeaderParseError, msg.as_string) # Test the email.Encoders module @@ -3113,6 +3135,28 @@ s = 'Subject: =?EUC-KR?B?CSixpLDtKSC/7Liuvsax4iC6uLmwMcijIKHaILzSwd/H0SC8+LCjwLsgv7W/+Mj3I ?=' raises(Errors.HeaderParseError, decode_header, s) + # Issue 1078919 + def test_ascii_add_header(self): + msg = Message() + msg.add_header('Content-Disposition', 'attachment', + filename='bud.gif') + self.assertEqual('attachment; filename="bud.gif"', + msg['Content-Disposition']) + + def test_nonascii_add_header_via_triple(self): + msg = Message() + msg.add_header('Content-Disposition', 'attachment', + filename=('iso-8859-1', '', 'Fu\xdfballer.ppt')) + self.assertEqual( + 'attachment; filename*="iso-8859-1\'\'Fu%DFballer.ppt"', + msg['Content-Disposition']) + + def test_encode_unaliased_charset(self): + # Issue 1379416: when the charset has no output conversion, + # output was accidentally getting coerced to unicode. + res = Header('abc','iso-8859-2').encode() + self.assertEqual(res, '=?iso-8859-2?q?abc?=') + self.assertIsInstance(res, str) # Test RFC 2231 header parameters (en/de)coding diff --git a/lib-python/2.7/ftplib.py b/lib-python/2.7/ftplib.py --- a/lib-python/2.7/ftplib.py +++ b/lib-python/2.7/ftplib.py @@ -599,7 +599,7 @@ Usage example: >>> from ftplib import FTP_TLS >>> ftps = FTP_TLS('ftp.python.org') - >>> ftps.login() # login anonimously previously securing control channel + >>> ftps.login() # login anonymously previously securing control channel '230 Guest login ok, access restrictions apply.' >>> ftps.prot_p() # switch to secure data connection '200 Protection level set to P' diff --git a/lib-python/2.7/functools.py b/lib-python/2.7/functools.py --- a/lib-python/2.7/functools.py +++ b/lib-python/2.7/functools.py @@ -53,17 +53,17 @@ def total_ordering(cls): """Class decorator that fills in missing ordering methods""" convert = { - '__lt__': [('__gt__', lambda self, other: other < self), - ('__le__', lambda self, other: not other < self), + '__lt__': [('__gt__', lambda self, other: not (self < other or self == other)), + ('__le__', lambda self, other: self < other or self == other), ('__ge__', lambda self, other: not self < other)], - '__le__': [('__ge__', lambda self, other: other <= self), - ('__lt__', lambda self, other: not other <= self), + '__le__': [('__ge__', lambda self, other: not self <= other or self == other), + ('__lt__', lambda self, other: self <= other and not self == other), ('__gt__', lambda self, other: not self <= other)], - '__gt__': [('__lt__', lambda self, other: other > self), - ('__ge__', lambda self, other: not other > self), + '__gt__': [('__lt__', lambda self, other: not (self > other or self == other)), + ('__ge__', lambda self, other: self > other or self == other), ('__le__', lambda self, other: not self > other)], - '__ge__': [('__le__', lambda self, other: other >= self), - ('__gt__', lambda self, other: not other >= self), + '__ge__': [('__le__', lambda self, other: (not self >= other) or self == other), + ('__gt__', lambda self, other: self >= other and not self == other), ('__lt__', lambda self, other: not self >= other)] } roots = set(dir(cls)) & set(convert) @@ -80,6 +80,7 @@ def cmp_to_key(mycmp): """Convert a cmp= function into a key= function""" class K(object): + __slots__ = ['obj'] def __init__(self, obj, *args): self.obj = obj def __lt__(self, other): diff --git a/lib-python/2.7/getpass.py b/lib-python/2.7/getpass.py --- a/lib-python/2.7/getpass.py +++ b/lib-python/2.7/getpass.py @@ -62,7 +62,7 @@ try: old = termios.tcgetattr(fd) # a copy to save new = old[:] - new[3] &= ~(termios.ECHO|termios.ISIG) # 3 == 'lflags' + new[3] &= ~termios.ECHO # 3 == 'lflags' tcsetattr_flags = termios.TCSAFLUSH if hasattr(termios, 'TCSASOFT'): tcsetattr_flags |= termios.TCSASOFT diff --git a/lib-python/2.7/gettext.py b/lib-python/2.7/gettext.py --- a/lib-python/2.7/gettext.py +++ b/lib-python/2.7/gettext.py @@ -316,7 +316,7 @@ # Note: we unconditionally convert both msgids and msgstrs to # Unicode using the character encoding specified in the charset # parameter of the Content-Type header. The gettext documentation - # strongly encourages msgids to be us-ascii, but some appliations + # strongly encourages msgids to be us-ascii, but some applications # require alternative encodings (e.g. Zope's ZCML and ZPT). For # traditional gettext applications, the msgid conversion will # cause no problems since us-ascii should always be a subset of diff --git a/lib-python/2.7/hashlib.py b/lib-python/2.7/hashlib.py --- a/lib-python/2.7/hashlib.py +++ b/lib-python/2.7/hashlib.py @@ -64,26 +64,29 @@ def __get_builtin_constructor(name): - if name in ('SHA1', 'sha1'): - import _sha - return _sha.new - elif name in ('MD5', 'md5'): - import _md5 - return _md5.new - elif name in ('SHA256', 'sha256', 'SHA224', 'sha224'): - import _sha256 - bs = name[3:] - if bs == '256': - return _sha256.sha256 - elif bs == '224': - return _sha256.sha224 - elif name in ('SHA512', 'sha512', 'SHA384', 'sha384'): - import _sha512 - bs = name[3:] - if bs == '512': - return _sha512.sha512 - elif bs == '384': - return _sha512.sha384 + try: + if name in ('SHA1', 'sha1'): + import _sha + return _sha.new + elif name in ('MD5', 'md5'): + import _md5 + return _md5.new + elif name in ('SHA256', 'sha256', 'SHA224', 'sha224'): + import _sha256 + bs = name[3:] + if bs == '256': + return _sha256.sha256 + elif bs == '224': + return _sha256.sha224 + elif name in ('SHA512', 'sha512', 'SHA384', 'sha384'): + import _sha512 + bs = name[3:] + if bs == '512': + return _sha512.sha512 + elif bs == '384': + return _sha512.sha384 + except ImportError: + pass # no extension module, this hash is unsupported. raise ValueError('unsupported hash type %s' % name) diff --git a/lib-python/2.7/heapq.py b/lib-python/2.7/heapq.py --- a/lib-python/2.7/heapq.py +++ b/lib-python/2.7/heapq.py @@ -133,6 +133,11 @@ from operator import itemgetter import bisect +def cmp_lt(x, y): + # Use __lt__ if available; otherwise, try __le__. + # In Py3.x, only __lt__ will be called. + return (x < y) if hasattr(x, '__lt__') else (not y <= x) + def heappush(heap, item): """Push item onto heap, maintaining the heap invariant.""" heap.append(item) @@ -167,13 +172,13 @@ def heappushpop(heap, item): """Fast version of a heappush followed by a heappop.""" - if heap and heap[0] < item: + if heap and cmp_lt(heap[0], item): item, heap[0] = heap[0], item _siftup(heap, 0) return item def heapify(x): - """Transform list into a heap, in-place, in O(len(heap)) time.""" + """Transform list into a heap, in-place, in O(len(x)) time.""" n = len(x) # Transform bottom-up. The largest index there's any point to looking at # is the largest with a child index in-range, so must have 2*i + 1 < n, @@ -215,11 +220,10 @@ pop = result.pop los = result[-1] # los --> Largest of the nsmallest for elem in it: - if los <= elem: - continue - insort(result, elem) - pop() - los = result[-1] + if cmp_lt(elem, los): + insort(result, elem) + pop() + los = result[-1] return result # An alternative approach manifests the whole iterable in memory but # saves comparisons by heapifying all at once. Also, saves time @@ -240,7 +244,7 @@ while pos > startpos: parentpos = (pos - 1) >> 1 parent = heap[parentpos] - if newitem < parent: + if cmp_lt(newitem, parent): heap[pos] = parent pos = parentpos continue @@ -295,7 +299,7 @@ while childpos < endpos: # Set childpos to index of smaller child. rightpos = childpos + 1 - if rightpos < endpos and not heap[childpos] < heap[rightpos]: + if rightpos < endpos and not cmp_lt(heap[childpos], heap[rightpos]): childpos = rightpos # Move the smaller child up. heap[pos] = heap[childpos] @@ -364,7 +368,7 @@ return [min(chain(head, it))] return [min(chain(head, it), key=key)] - # When n>=size, it's faster to use sort() + # When n>=size, it's faster to use sorted() try: size = len(iterable) except (TypeError, AttributeError): @@ -402,7 +406,7 @@ return [max(chain(head, it))] return [max(chain(head, it), key=key)] - # When n>=size, it's faster to use sort() + # When n>=size, it's faster to use sorted() try: size = len(iterable) except (TypeError, AttributeError): diff --git a/lib-python/2.7/httplib.py b/lib-python/2.7/httplib.py --- a/lib-python/2.7/httplib.py +++ b/lib-python/2.7/httplib.py @@ -212,6 +212,9 @@ # maximal amount of data to read at one time in _safe_read MAXAMOUNT = 1048576 +# maximal line length when calling readline(). +_MAXLINE = 65536 + class HTTPMessage(mimetools.Message): def addheader(self, key, value): @@ -274,7 +277,9 @@ except IOError: startofline = tell = None self.seekable = 0 - line = self.fp.readline() + line = self.fp.readline(_MAXLINE + 1) + if len(line) > _MAXLINE: + raise LineTooLong("header line") if not line: self.status = 'EOF in headers' break @@ -404,7 +409,10 @@ break # skip the header from the 100 response while True: - skip = self.fp.readline().strip() + skip = self.fp.readline(_MAXLINE + 1) + if len(skip) > _MAXLINE: + raise LineTooLong("header line") + skip = skip.strip() if not skip: break if self.debuglevel > 0: @@ -563,7 +571,9 @@ value = [] while True: if chunk_left is None: - line = self.fp.readline() + line = self.fp.readline(_MAXLINE + 1) + if len(line) > _MAXLINE: + raise LineTooLong("chunk size") i = line.find(';') if i >= 0: line = line[:i] # strip chunk-extensions @@ -598,7 +608,9 @@ # read and discard trailer up to the CRLF terminator ### note: we shouldn't have any trailers! while True: - line = self.fp.readline() + line = self.fp.readline(_MAXLINE + 1) + if len(line) > _MAXLINE: + raise LineTooLong("trailer line") if not line: # a vanishingly small number of sites EOF without # sending the trailer @@ -730,7 +742,9 @@ raise socket.error("Tunnel connection failed: %d %s" % (code, message.strip())) while True: - line = response.fp.readline() + line = response.fp.readline(_MAXLINE + 1) + if len(line) > _MAXLINE: + raise LineTooLong("header line") if line == '\r\n': break @@ -790,7 +804,7 @@ del self._buffer[:] # If msg and message_body are sent in a single send() call, # it will avoid performance problems caused by the interaction - # between delayed ack and the Nagle algorithim. + # between delayed ack and the Nagle algorithm. if isinstance(message_body, str): msg += message_body message_body = None @@ -1233,6 +1247,11 @@ self.args = line, self.line = line +class LineTooLong(HTTPException): + def __init__(self, line_type): + HTTPException.__init__(self, "got more than %d bytes when reading %s" + % (_MAXLINE, line_type)) + # for backwards compatibility error = HTTPException diff --git a/lib-python/2.7/idlelib/Bindings.py b/lib-python/2.7/idlelib/Bindings.py --- a/lib-python/2.7/idlelib/Bindings.py +++ b/lib-python/2.7/idlelib/Bindings.py @@ -98,14 +98,6 @@ # menu del menudefs[-1][1][0:2] - menudefs.insert(0, - ('application', [ - ('About IDLE', '<<about-idle>>'), - None, - ('_Preferences....', '<<open-config-dialog>>'), - ])) - - default_keydefs = idleConf.GetCurrentKeySet() del sys diff --git a/lib-python/2.7/idlelib/EditorWindow.py b/lib-python/2.7/idlelib/EditorWindow.py --- a/lib-python/2.7/idlelib/EditorWindow.py +++ b/lib-python/2.7/idlelib/EditorWindow.py @@ -48,6 +48,21 @@ path = module.__path__ except AttributeError: raise ImportError, 'No source for module ' + module.__name__ + if descr[2] != imp.PY_SOURCE: + # If all of the above fails and didn't raise an exception,fallback + # to a straight import which can find __init__.py in a package. + m = __import__(fullname) + try: + filename = m.__file__ + except AttributeError: + pass + else: + file = None + base, ext = os.path.splitext(filename) + if ext == '.pyc': + ext = '.py' + filename = base + ext + descr = filename, None, imp.PY_SOURCE return file, filename, descr class EditorWindow(object): @@ -102,8 +117,8 @@ self.top = top = WindowList.ListedToplevel(root, menu=self.menubar) if flist: self.tkinter_vars = flist.vars - #self.top.instance_dict makes flist.inversedict avalable to - #configDialog.py so it can access all EditorWindow instaces + #self.top.instance_dict makes flist.inversedict available to + #configDialog.py so it can access all EditorWindow instances self.top.instance_dict = flist.inversedict else: self.tkinter_vars = {} # keys: Tkinter event names @@ -136,6 +151,14 @@ if macosxSupport.runningAsOSXApp(): # Command-W on editorwindows doesn't work without this. text.bind('<<close-window>>', self.close_event) + # Some OS X systems have only one mouse button, + # so use control-click for pulldown menus there. + # (Note, AquaTk defines <2> as the right button if + # present and the Tk Text widget already binds <2>.) + text.bind("<Control-Button-1>",self.right_menu_event) + else: + # Elsewhere, use right-click for pulldown menus. + text.bind("<3>",self.right_menu_event) text.bind("<<cut>>", self.cut) text.bind("<<copy>>", self.copy) text.bind("<<paste>>", self.paste) @@ -154,7 +177,6 @@ text.bind("<<find-selection>>", self.find_selection_event) text.bind("<<replace>>", self.replace_event) text.bind("<<goto-line>>", self.goto_line_event) - text.bind("<3>", self.right_menu_event) text.bind("<<smart-backspace>>",self.smart_backspace_event) text.bind("<<newline-and-indent>>",self.newline_and_indent_event) text.bind("<<smart-indent>>",self.smart_indent_event) @@ -300,13 +322,13 @@ return "break" def home_callback(self, event): - if (event.state & 12) != 0 and event.keysym == "Home": - # state&1==shift, state&4==control, state&8==alt - return # <Modifier-Home>; fall back to class binding - + if (event.state & 4) != 0 and event.keysym == "Home": + # state&4==Control. If <Control-Home>, use the Tk binding. + return if self.text.index("iomark") and \ self.text.compare("iomark", "<=", "insert lineend") and \ self.text.compare("insert linestart", "<=", "iomark"): + # In Shell on input line, go to just after prompt insertpt = int(self.text.index("iomark").split(".")[1]) else: line = self.text.get("insert linestart", "insert lineend") @@ -315,30 +337,27 @@ break else: insertpt=len(line) - lineat = int(self.text.index("insert").split('.')[1]) - if insertpt == lineat: insertpt = 0 - dest = "insert linestart+"+str(insertpt)+"c" - if (event.state&1) == 0: - # shift not pressed + # shift was not pressed self.text.tag_remove("sel", "1.0", "end") else: if not self.text.index("sel.first"): - self.text.mark_set("anchor","insert") - + self.text.mark_set("my_anchor", "insert") # there was no previous selection + else: + if self.text.compare(self.text.index("sel.first"), "<", self.text.index("insert")): + self.text.mark_set("my_anchor", "sel.first") # extend back + else: + self.text.mark_set("my_anchor", "sel.last") # extend forward first = self.text.index(dest) - last = self.text.index("anchor") - + last = self.text.index("my_anchor") if self.text.compare(first,">",last): first,last = last,first - self.text.tag_remove("sel", "1.0", "end") self.text.tag_add("sel", first, last) - self.text.mark_set("insert", dest) self.text.see("insert") return "break" @@ -385,7 +404,7 @@ menudict[name] = menu = Menu(mbar, name=name) mbar.add_cascade(label=label, menu=menu, underline=underline) - if macosxSupport.runningAsOSXApp(): + if macosxSupport.isCarbonAquaTk(self.root): # Insert the application menu menudict['application'] = menu = Menu(mbar, name='apple') mbar.add_cascade(label='IDLE', menu=menu) @@ -445,7 +464,11 @@ def python_docs(self, event=None): if sys.platform[:3] == 'win': - os.startfile(self.help_url) + try: + os.startfile(self.help_url) + except WindowsError as why: + tkMessageBox.showerror(title='Document Start Failure', + message=str(why), parent=self.text) else: webbrowser.open(self.help_url) return "break" @@ -740,9 +763,13 @@ "Create a callback with the helpfile value frozen at definition time" def display_extra_help(helpfile=helpfile): if not helpfile.startswith(('www', 'http')): - url = os.path.normpath(helpfile) + helpfile = os.path.normpath(helpfile) if sys.platform[:3] == 'win': - os.startfile(helpfile) + try: + os.startfile(helpfile) + except WindowsError as why: + tkMessageBox.showerror(title='Document Start Failure', + message=str(why), parent=self.text) else: webbrowser.open(helpfile) return display_extra_help @@ -1526,7 +1553,12 @@ def get_accelerator(keydefs, eventname): keylist = keydefs.get(eventname) - if not keylist: + # issue10940: temporary workaround to prevent hang with OS X Cocoa Tk 8.5 + # if not keylist: + if (not keylist) or (macosxSupport.runningAsOSXApp() and eventname in { + "<<open-module>>", + "<<goto-line>>", + "<<change-indentwidth>>"}): return "" s = keylist[0] s = re.sub(r"-[a-z]\b", lambda m: m.group().upper(), s) diff --git a/lib-python/2.7/idlelib/FileList.py b/lib-python/2.7/idlelib/FileList.py --- a/lib-python/2.7/idlelib/FileList.py +++ b/lib-python/2.7/idlelib/FileList.py @@ -43,7 +43,7 @@ def new(self, filename=None): return self.EditorWindow(self, filename) - def close_all_callback(self, event): + def close_all_callback(self, *args, **kwds): for edit in self.inversedict.keys(): reply = edit.close() if reply == "cancel": diff --git a/lib-python/2.7/idlelib/FormatParagraph.py b/lib-python/2.7/idlelib/FormatParagraph.py --- a/lib-python/2.7/idlelib/FormatParagraph.py +++ b/lib-python/2.7/idlelib/FormatParagraph.py @@ -54,7 +54,7 @@ # If the block ends in a \n, we dont want the comment # prefix inserted after it. (Im not sure it makes sense to # reformat a comment block that isnt made of complete - # lines, but whatever!) Can't think of a clean soltution, + # lines, but whatever!) Can't think of a clean solution, # so we hack away block_suffix = "" if not newdata[-1]: diff --git a/lib-python/2.7/idlelib/HISTORY.txt b/lib-python/2.7/idlelib/HISTORY.txt --- a/lib-python/2.7/idlelib/HISTORY.txt +++ b/lib-python/2.7/idlelib/HISTORY.txt @@ -13,7 +13,7 @@ - New tarball released as a result of the 'revitalisation' of the IDLEfork project. -- This release requires python 2.1 or better. Compatability with earlier +- This release requires python 2.1 or better. Compatibility with earlier versions of python (especially ancient ones like 1.5x) is no longer a priority in IDLEfork development. diff --git a/lib-python/2.7/idlelib/IOBinding.py b/lib-python/2.7/idlelib/IOBinding.py --- a/lib-python/2.7/idlelib/IOBinding.py +++ b/lib-python/2.7/idlelib/IOBinding.py @@ -320,17 +320,20 @@ return "yes" message = "Do you want to save %s before closing?" % ( self.filename or "this untitled document") - m = tkMessageBox.Message( - title="Save On Close", - message=message, - icon=tkMessageBox.QUESTION, - type=tkMessageBox.YESNOCANCEL, - master=self.text) - reply = m.show() - if reply == "yes": + confirm = tkMessageBox.askyesnocancel( + title="Save On Close", + message=message, + default=tkMessageBox.YES, + master=self.text) + if confirm: + reply = "yes" self.save(None) if not self.get_saved(): reply = "cancel" + elif confirm is None: + reply = "cancel" + else: + reply = "no" self.text.focus_set() return reply @@ -339,7 +342,7 @@ self.save_as(event) else: if self.writefile(self.filename): - self.set_saved(1) + self.set_saved(True) try: self.editwin.store_file_breaks() except AttributeError: # may be a PyShell @@ -465,15 +468,12 @@ self.text.insert("end-1c", "\n") def print_window(self, event): - m = tkMessageBox.Message( - title="Print", - message="Print to Default Printer", - icon=tkMessageBox.QUESTION, - type=tkMessageBox.OKCANCEL, - default=tkMessageBox.OK, - master=self.text) - reply = m.show() - if reply != tkMessageBox.OK: + confirm = tkMessageBox.askokcancel( + title="Print", + message="Print to Default Printer", + default=tkMessageBox.OK, + master=self.text) + if not confirm: self.text.focus_set() return "break" tempfilename = None @@ -488,8 +488,8 @@ if not self.writefile(tempfilename): os.unlink(tempfilename) return "break" - platform=os.name - printPlatform=1 + platform = os.name + printPlatform = True if platform == 'posix': #posix platform command = idleConf.GetOption('main','General', 'print-command-posix') @@ -497,7 +497,7 @@ elif platform == 'nt': #win32 platform command = idleConf.GetOption('main','General','print-command-win') else: #no printing for this platform - printPlatform=0 + printPlatform = False if printPlatform: #we can try to print for this platform command = command % filename pipe = os.popen(command, "r") @@ -511,7 +511,7 @@ output = "Printing command: %s\n" % repr(command) + output tkMessageBox.showerror("Print status", output, master=self.text) else: #no printing for this platform - message="Printing is not enabled for this platform: %s" % platform + message = "Printing is not enabled for this platform: %s" % platform tkMessageBox.showinfo("Print status", message, master=self.text) if tempfilename: os.unlink(tempfilename) diff --git a/lib-python/2.7/idlelib/NEWS.txt b/lib-python/2.7/idlelib/NEWS.txt --- a/lib-python/2.7/idlelib/NEWS.txt +++ b/lib-python/2.7/idlelib/NEWS.txt @@ -1,3 +1,18 @@ +What's New in IDLE 2.7.2? +======================= + +*Release date: 29-May-2011* + +- Issue #6378: Further adjust idle.bat to start associated Python + +- Issue #11896: Save on Close failed despite selecting "Yes" in dialog. + +- <Home> toggle failing on Tk 8.5, causing IDLE exits and strange selection + behavior. Issue 4676. Improve selection extension behaviour. + +- <Home> toggle non-functional when NumLock set on Windows. Issue 3851. + + What's New in IDLE 2.7? ======================= @@ -21,7 +36,7 @@ - Tk 8.5 Text widget requires 'wordprocessor' tabstyle attr to handle mixed space/tab properly. Issue 5129, patch by Guilherme Polo. - + - Issue #3549: On MacOS the preferences menu was not present diff --git a/lib-python/2.7/idlelib/PyShell.py b/lib-python/2.7/idlelib/PyShell.py --- a/lib-python/2.7/idlelib/PyShell.py +++ b/lib-python/2.7/idlelib/PyShell.py @@ -1432,6 +1432,13 @@ shell.interp.prepend_syspath(script) shell.interp.execfile(script) + # Check for problematic OS X Tk versions and print a warning message + # in the IDLE shell window; this is less intrusive than always opening + # a separate window. + tkversionwarning = macosxSupport.tkVersionWarning(root) + if tkversionwarning: + shell.interp.runcommand(''.join(("print('", tkversionwarning, "')"))) + root.mainloop() root.destroy() diff --git a/lib-python/2.7/idlelib/ScriptBinding.py b/lib-python/2.7/idlelib/ScriptBinding.py --- a/lib-python/2.7/idlelib/ScriptBinding.py +++ b/lib-python/2.7/idlelib/ScriptBinding.py @@ -26,6 +26,7 @@ from idlelib import PyShell from idlelib.configHandler import idleConf +from idlelib import macosxSupport IDENTCHARS = string.ascii_letters + string.digits + "_" @@ -53,6 +54,9 @@ self.flist = self.editwin.flist self.root = self.editwin.root + if macosxSupport.runningAsOSXApp(): + self.editwin.text_frame.bind('<<run-module-event-2>>', self._run_module_event) + def check_module_event(self, event): filename = self.getfilename() if not filename: @@ -166,6 +170,19 @@ interp.runcode(code) return 'break' + if macosxSupport.runningAsOSXApp(): + # Tk-Cocoa in MacOSX is broken until at least + # Tk 8.5.9, and without this rather + # crude workaround IDLE would hang when a user + # tries to run a module using the keyboard shortcut + # (the menu item works fine). + _run_module_event = run_module_event + + def run_module_event(self, event): + self.editwin.text_frame.after(200, + lambda: self.editwin.text_frame.event_generate('<<run-module-event-2>>')) + return 'break' + def getfilename(self): """Get source filename. If not saved, offer to save (or create) file @@ -184,9 +201,9 @@ if autosave and filename: self.editwin.io.save(None) else: - reply = self.ask_save_dialog() + confirm = self.ask_save_dialog() self.editwin.text.focus_set() - if reply == "ok": + if confirm: self.editwin.io.save(None) filename = self.editwin.io.filename else: @@ -195,13 +212,11 @@ def ask_save_dialog(self): msg = "Source Must Be Saved\n" + 5*' ' + "OK to Save?" - mb = tkMessageBox.Message(title="Save Before Run or Check", - message=msg, - icon=tkMessageBox.QUESTION, - type=tkMessageBox.OKCANCEL, - default=tkMessageBox.OK, - master=self.editwin.text) - return mb.show() + confirm = tkMessageBox.askokcancel(title="Save Before Run or Check", + message=msg, + default=tkMessageBox.OK, + master=self.editwin.text) + return confirm def errorbox(self, title, message): # XXX This should really be a function of EditorWindow... diff --git a/lib-python/2.7/idlelib/config-keys.def b/lib-python/2.7/idlelib/config-keys.def --- a/lib-python/2.7/idlelib/config-keys.def +++ b/lib-python/2.7/idlelib/config-keys.def @@ -176,7 +176,7 @@ redo = <Shift-Command-Key-Z> close-window = <Command-Key-w> restart-shell = <Control-Key-F6> -save-window-as-file = <Command-Key-S> +save-window-as-file = <Shift-Command-Key-S> close-all-windows = <Command-Key-q> view-restart = <Key-F6> tabify-region = <Control-Key-5> @@ -208,7 +208,7 @@ open-module = <Command-Key-m> find-selection = <Shift-Command-Key-F3> python-context-help = <Shift-Key-F1> -save-copy-of-window-as-file = <Shift-Command-Key-s> +save-copy-of-window-as-file = <Option-Command-Key-s> open-window-from-file = <Command-Key-o> python-docs = <Key-F1> diff --git a/lib-python/2.7/idlelib/extend.txt b/lib-python/2.7/idlelib/extend.txt --- a/lib-python/2.7/idlelib/extend.txt +++ b/lib-python/2.7/idlelib/extend.txt @@ -18,7 +18,7 @@ An IDLE extension class is instantiated with a single argument, `editwin', an EditorWindow instance. The extension cannot assume much -about this argument, but it is guarateed to have the following instance +about this argument, but it is guaranteed to have the following instance variables: text a Text instance (a widget) diff --git a/lib-python/2.7/idlelib/idle.bat b/lib-python/2.7/idlelib/idle.bat --- a/lib-python/2.7/idlelib/idle.bat +++ b/lib-python/2.7/idlelib/idle.bat @@ -1,4 +1,4 @@ @echo off rem Start IDLE using the appropriate Python interpreter set CURRDIR=%~dp0 -start "%CURRDIR%..\..\pythonw.exe" "%CURRDIR%idle.pyw" %1 %2 %3 %4 %5 %6 %7 %8 %9 +start "IDLE" "%CURRDIR%..\..\pythonw.exe" "%CURRDIR%idle.pyw" %1 %2 %3 %4 %5 %6 %7 %8 %9 diff --git a/lib-python/2.7/idlelib/idlever.py b/lib-python/2.7/idlelib/idlever.py --- a/lib-python/2.7/idlelib/idlever.py +++ b/lib-python/2.7/idlelib/idlever.py @@ -1,1 +1,1 @@ -IDLE_VERSION = "2.7.1" +IDLE_VERSION = "2.7.2" diff --git a/lib-python/2.7/idlelib/macosxSupport.py b/lib-python/2.7/idlelib/macosxSupport.py --- a/lib-python/2.7/idlelib/macosxSupport.py +++ b/lib-python/2.7/idlelib/macosxSupport.py @@ -4,6 +4,7 @@ """ import sys import Tkinter +from os import path _appbundle = None @@ -19,10 +20,41 @@ _appbundle = (sys.platform == 'darwin' and '.app' in sys.executable) return _appbundle +_carbonaquatk = None + +def isCarbonAquaTk(root): + """ + Returns True if IDLE is using a Carbon Aqua Tk (instead of the + newer Cocoa Aqua Tk). + """ + global _carbonaquatk + if _carbonaquatk is None: + _carbonaquatk = (runningAsOSXApp() and + 'aqua' in root.tk.call('tk', 'windowingsystem') and + 'AppKit' not in root.tk.call('winfo', 'server', '.')) + return _carbonaquatk + +def tkVersionWarning(root): + """ + Returns a string warning message if the Tk version in use appears to + be one known to cause problems with IDLE. The Apple Cocoa-based Tk 8.5 + that was shipped with Mac OS X 10.6. + """ + + if (runningAsOSXApp() and + ('AppKit' in root.tk.call('winfo', 'server', '.')) and + (root.tk.call('info', 'patchlevel') == '8.5.7') ): + return (r"WARNING: The version of Tcl/Tk (8.5.7) in use may" + r" be unstable.\n" + r"Visit http://www.python.org/download/mac/tcltk/" + r" for current information.") + else: + return False + def addOpenEventSupport(root, flist): """ - This ensures that the application will respont to open AppleEvents, which - makes is feaseable to use IDLE as the default application for python files. + This ensures that the application will respond to open AppleEvents, which + makes is feasible to use IDLE as the default application for python files. """ def doOpenFile(*args): for fn in args: @@ -79,9 +111,6 @@ WindowList.add_windows_to_menu(menu) WindowList.register_callback(postwindowsmenu) - menudict['application'] = menu = Menu(menubar, name='apple') - menubar.add_cascade(label='IDLE', menu=menu) - def about_dialog(event=None): from idlelib import aboutDialog aboutDialog.AboutDialog(root, 'About IDLE') @@ -91,41 +120,45 @@ root.instance_dict = flist.inversedict configDialog.ConfigDialog(root, 'Settings') + def help_dialog(event=None): + from idlelib import textView + fn = path.join(path.abspath(path.dirname(__file__)), 'help.txt') + textView.view_file(root, 'Help', fn) root.bind('<<about-idle>>', about_dialog) root.bind('<<open-config-dialog>>', config_dialog) + root.createcommand('::tk::mac::ShowPreferences', config_dialog) if flist: root.bind('<<close-all-windows>>', flist.close_all_callback) + # The binding above doesn't reliably work on all versions of Tk + # on MacOSX. Adding command definition below does seem to do the + # right thing for now. + root.createcommand('exit', flist.close_all_callback) - ###check if Tk version >= 8.4.14; if so, use hard-coded showprefs binding - tkversion = root.tk.eval('info patchlevel') - # Note: we cannot check if the string tkversion >= '8.4.14', because - # the string '8.4.7' is greater than the string '8.4.14'. - if tuple(map(int, tkversion.split('.'))) >= (8, 4, 14): - Bindings.menudefs[0] = ('application', [ + if isCarbonAquaTk(root): + # for Carbon AquaTk, replace the default Tk apple menu + menudict['application'] = menu = Menu(menubar, name='apple') + menubar.add_cascade(label='IDLE', menu=menu) + Bindings.menudefs.insert(0, + ('application', [ ('About IDLE', '<<about-idle>>'), - None, - ]) - root.createcommand('::tk::mac::ShowPreferences', config_dialog) + None, + ])) + tkversion = root.tk.eval('info patchlevel') + if tuple(map(int, tkversion.split('.'))) < (8, 4, 14): + # for earlier AquaTk versions, supply a Preferences menu item + Bindings.menudefs[0][1].append( + ('_Preferences....', '<<open-config-dialog>>'), + ) else: - for mname, entrylist in Bindings.menudefs: - menu = menudict.get(mname) - if not menu: - continue - else: - for entry in entrylist: - if not entry: - menu.add_separator() - else: - label, eventname = entry - underline, label = prepstr(label) - accelerator = get_accelerator(Bindings.default_keydefs, - eventname) - def command(text=root, eventname=eventname): - text.event_generate(eventname) - menu.add_command(label=label, underline=underline, - command=command, accelerator=accelerator) + # assume Cocoa AquaTk + # replace default About dialog with About IDLE one + root.createcommand('tkAboutDialog', about_dialog) + # replace default "Help" item in Help menu + root.createcommand('::tk::mac::ShowHelp', help_dialog) + # remove redundant "IDLE Help" from menu + del Bindings.menudefs[-1][1][0] def setupApp(root, flist): """ diff --git a/lib-python/2.7/imaplib.py b/lib-python/2.7/imaplib.py --- a/lib-python/2.7/imaplib.py +++ b/lib-python/2.7/imaplib.py @@ -1158,28 +1158,17 @@ self.port = port self.sock = socket.create_connection((host, port)) self.sslobj = ssl.wrap_socket(self.sock, self.keyfile, self.certfile) + self.file = self.sslobj.makefile('rb') def read(self, size): """Read 'size' bytes from remote.""" - # sslobj.read() sometimes returns < size bytes - chunks = [] - read = 0 - while read < size: - data = self.sslobj.read(min(size-read, 16384)) - read += len(data) - chunks.append(data) - - return ''.join(chunks) + return self.file.read(size) def readline(self): """Read line from remote.""" - line = [] - while 1: - char = self.sslobj.read(1) - line.append(char) - if char in ("\n", ""): return ''.join(line) + return self.file.readline() def send(self, data): @@ -1195,6 +1184,7 @@ def shutdown(self): """Close I/O established in "open".""" + self.file.close() self.sock.close() @@ -1321,9 +1311,10 @@ 'Jul': 7, 'Aug': 8, 'Sep': 9, 'Oct': 10, 'Nov': 11, 'Dec': 12} def Internaldate2tuple(resp): - """Convert IMAP4 INTERNALDATE to UT. + """Parse an IMAP4 INTERNALDATE string. - Returns Python time module tuple. + Return corresponding local time. The return value is a + time.struct_time instance or None if the string has wrong format. """ mo = InternalDate.match(resp) @@ -1390,9 +1381,14 @@ def Time2Internaldate(date_time): - """Convert 'date_time' to IMAP4 INTERNALDATE representation. + """Convert date_time to IMAP4 INTERNALDATE representation. - Return string in form: '"DD-Mmm-YYYY HH:MM:SS +HHMM"' + Return string in form: '"DD-Mmm-YYYY HH:MM:SS +HHMM"'. The + date_time argument can be a number (int or float) representing + seconds since epoch (as returned by time.time()), a 9-tuple + representing local time (as returned by time.localtime()), or a + double-quoted string. In the last case, it is assumed to already + be in the correct format. """ if isinstance(date_time, (int, float)): diff --git a/lib-python/2.7/inspect.py b/lib-python/2.7/inspect.py --- a/lib-python/2.7/inspect.py +++ b/lib-python/2.7/inspect.py @@ -943,8 +943,14 @@ f_name, 'at most' if defaults else 'exactly', num_args, 'arguments' if num_args > 1 else 'argument', num_total)) elif num_args == 0 and num_total: - raise TypeError('%s() takes no arguments (%d given)' % - (f_name, num_total)) + if varkw: + if num_pos: + # XXX: We should use num_pos, but Python also uses num_total: + raise TypeError('%s() takes exactly 0 arguments ' + '(%d given)' % (f_name, num_total)) + else: + raise TypeError('%s() takes no arguments (%d given)' % + (f_name, num_total)) for arg in args: if isinstance(arg, str) and arg in named: if is_assigned(arg): diff --git a/lib-python/2.7/json/decoder.py b/lib-python/2.7/json/decoder.py --- a/lib-python/2.7/json/decoder.py +++ b/lib-python/2.7/json/decoder.py @@ -4,7 +4,7 @@ import sys import struct -from json.scanner import make_scanner +from json import scanner try: from _json import scanstring as c_scanstring except ImportError: @@ -161,6 +161,12 @@ nextchar = s[end:end + 1] # Trivial empty object if nextchar == '}': + if object_pairs_hook is not None: + result = object_pairs_hook(pairs) + return result, end + pairs = {} + if object_hook is not None: + pairs = object_hook(pairs) return pairs, end + 1 elif nextchar != '"': raise ValueError(errmsg("Expecting property name", s, end)) @@ -350,7 +356,7 @@ self.parse_object = JSONObject self.parse_array = JSONArray self.parse_string = scanstring - self.scan_once = make_scanner(self) + self.scan_once = scanner.make_scanner(self) def decode(self, s, _w=WHITESPACE.match): """Return the Python representation of ``s`` (a ``str`` or ``unicode`` diff --git a/lib-python/2.7/json/encoder.py b/lib-python/2.7/json/encoder.py --- a/lib-python/2.7/json/encoder.py +++ b/lib-python/2.7/json/encoder.py @@ -251,7 +251,7 @@ if (_one_shot and c_make_encoder is not None - and not self.indent and not self.sort_keys): + and self.indent is None and not self.sort_keys): _iterencode = c_make_encoder( markers, self.default, _encoder, self.indent, self.key_separator, self.item_separator, self.sort_keys, diff --git a/lib-python/2.7/json/tests/__init__.py b/lib-python/2.7/json/tests/__init__.py --- a/lib-python/2.7/json/tests/__init__.py +++ b/lib-python/2.7/json/tests/__init__.py @@ -1,7 +1,46 @@ import os import sys +import json +import doctest import unittest -import doctest + +from test import test_support + +# import json with and without accelerations +cjson = test_support.import_fresh_module('json', fresh=['_json']) +pyjson = test_support.import_fresh_module('json', blocked=['_json']) + +# create two base classes that will be used by the other tests +class PyTest(unittest.TestCase): + json = pyjson + loads = staticmethod(pyjson.loads) + dumps = staticmethod(pyjson.dumps) + + at unittest.skipUnless(cjson, 'requires _json') +class CTest(unittest.TestCase): + if cjson is not None: + json = cjson + loads = staticmethod(cjson.loads) + dumps = staticmethod(cjson.dumps) + +# test PyTest and CTest checking if the functions come from the right module +class TestPyTest(PyTest): + def test_pyjson(self): + self.assertEqual(self.json.scanner.make_scanner.__module__, + 'json.scanner') + self.assertEqual(self.json.decoder.scanstring.__module__, + 'json.decoder') + self.assertEqual(self.json.encoder.encode_basestring_ascii.__module__, + 'json.encoder') + +class TestCTest(CTest): + def test_cjson(self): + self.assertEqual(self.json.scanner.make_scanner.__module__, '_json') + self.assertEqual(self.json.decoder.scanstring.__module__, '_json') + self.assertEqual(self.json.encoder.c_make_encoder.__module__, '_json') + self.assertEqual(self.json.encoder.encode_basestring_ascii.__module__, + '_json') + here = os.path.dirname(__file__) @@ -17,12 +56,11 @@ return suite def additional_tests(): - import json - import json.encoder - import json.decoder suite = unittest.TestSuite() for mod in (json, json.encoder, json.decoder): suite.addTest(doctest.DocTestSuite(mod)) + suite.addTest(TestPyTest('test_pyjson')) + suite.addTest(TestCTest('test_cjson')) return suite def main(): diff --git a/lib-python/2.7/json/tests/test_check_circular.py b/lib-python/2.7/json/tests/test_check_circular.py --- a/lib-python/2.7/json/tests/test_check_circular.py +++ b/lib-python/2.7/json/tests/test_check_circular.py @@ -1,30 +1,34 @@ -from unittest import TestCase -import json +from json.tests import PyTest, CTest + def default_iterable(obj): return list(obj) -class TestCheckCircular(TestCase): +class TestCheckCircular(object): def test_circular_dict(self): dct = {} dct['a'] = dct - self.assertRaises(ValueError, json.dumps, dct) + self.assertRaises(ValueError, self.dumps, dct) def test_circular_list(self): lst = [] lst.append(lst) - self.assertRaises(ValueError, json.dumps, lst) + self.assertRaises(ValueError, self.dumps, lst) def test_circular_composite(self): dct2 = {} dct2['a'] = [] dct2['a'].append(dct2) - self.assertRaises(ValueError, json.dumps, dct2) + self.assertRaises(ValueError, self.dumps, dct2) def test_circular_default(self): - json.dumps([set()], default=default_iterable) - self.assertRaises(TypeError, json.dumps, [set()]) + self.dumps([set()], default=default_iterable) + self.assertRaises(TypeError, self.dumps, [set()]) def test_circular_off_default(self): - json.dumps([set()], default=default_iterable, check_circular=False) - self.assertRaises(TypeError, json.dumps, [set()], check_circular=False) + self.dumps([set()], default=default_iterable, check_circular=False) + self.assertRaises(TypeError, self.dumps, [set()], check_circular=False) + + +class TestPyCheckCircular(TestCheckCircular, PyTest): pass +class TestCCheckCircular(TestCheckCircular, CTest): pass diff --git a/lib-python/2.7/json/tests/test_decode.py b/lib-python/2.7/json/tests/test_decode.py --- a/lib-python/2.7/json/tests/test_decode.py +++ b/lib-python/2.7/json/tests/test_decode.py @@ -1,18 +1,17 @@ import decimal -from unittest import TestCase from StringIO import StringIO +from collections import OrderedDict +from json.tests import PyTest, CTest -import json -from collections import OrderedDict -class TestDecode(TestCase): +class TestDecode(object): def test_decimal(self): - rval = json.loads('1.1', parse_float=decimal.Decimal) + rval = self.loads('1.1', parse_float=decimal.Decimal) self.assertTrue(isinstance(rval, decimal.Decimal)) self.assertEqual(rval, decimal.Decimal('1.1')) def test_float(self): - rval = json.loads('1', parse_int=float) + rval = self.loads('1', parse_int=float) self.assertTrue(isinstance(rval, float)) self.assertEqual(rval, 1.0) @@ -20,22 +19,32 @@ # Several optimizations were made that skip over calls to # the whitespace regex, so this test is designed to try and # exercise the uncommon cases. The array cases are already covered. - rval = json.loads('{ "key" : "value" , "k":"v" }') + rval = self.loads('{ "key" : "value" , "k":"v" }') self.assertEqual(rval, {"key":"value", "k":"v"}) + def test_empty_objects(self): + self.assertEqual(self.loads('{}'), {}) + self.assertEqual(self.loads('[]'), []) + self.assertEqual(self.loads('""'), u"") + self.assertIsInstance(self.loads('""'), unicode) + def test_object_pairs_hook(self): s = '{"xkd":1, "kcw":2, "art":3, "hxm":4, "qrt":5, "pad":6, "hoy":7}' p = [("xkd", 1), ("kcw", 2), ("art", 3), ("hxm", 4), ("qrt", 5), ("pad", 6), ("hoy", 7)] - self.assertEqual(json.loads(s), eval(s)) - self.assertEqual(json.loads(s, object_pairs_hook=lambda x: x), p) - self.assertEqual(json.load(StringIO(s), - object_pairs_hook=lambda x: x), p) - od = json.loads(s, object_pairs_hook=OrderedDict) + self.assertEqual(self.loads(s), eval(s)) + self.assertEqual(self.loads(s, object_pairs_hook=lambda x: x), p) + self.assertEqual(self.json.load(StringIO(s), + object_pairs_hook=lambda x: x), p) + od = self.loads(s, object_pairs_hook=OrderedDict) self.assertEqual(od, OrderedDict(p)) self.assertEqual(type(od), OrderedDict) # the object_pairs_hook takes priority over the object_hook - self.assertEqual(json.loads(s, + self.assertEqual(self.loads(s, object_pairs_hook=OrderedDict, object_hook=lambda x: None), OrderedDict(p)) + + +class TestPyDecode(TestDecode, PyTest): pass +class TestCDecode(TestDecode, CTest): pass diff --git a/lib-python/2.7/json/tests/test_default.py b/lib-python/2.7/json/tests/test_default.py --- a/lib-python/2.7/json/tests/test_default.py +++ b/lib-python/2.7/json/tests/test_default.py @@ -1,9 +1,12 @@ -from unittest import TestCase +from json.tests import PyTest, CTest -import json -class TestDefault(TestCase): +class TestDefault(object): def test_default(self): self.assertEqual( - json.dumps(type, default=repr), - json.dumps(repr(type))) + self.dumps(type, default=repr), + self.dumps(repr(type))) + + +class TestPyDefault(TestDefault, PyTest): pass +class TestCDefault(TestDefault, CTest): pass diff --git a/lib-python/2.7/json/tests/test_dump.py b/lib-python/2.7/json/tests/test_dump.py --- a/lib-python/2.7/json/tests/test_dump.py +++ b/lib-python/2.7/json/tests/test_dump.py @@ -1,21 +1,23 @@ -from unittest import TestCase from cStringIO import StringIO +from json.tests import PyTest, CTest -import json -class TestDump(TestCase): +class TestDump(object): def test_dump(self): sio = StringIO() - json.dump({}, sio) + self.json.dump({}, sio) self.assertEqual(sio.getvalue(), '{}') def test_dumps(self): - self.assertEqual(json.dumps({}), '{}') + self.assertEqual(self.dumps({}), '{}') def test_encode_truefalse(self): - self.assertEqual(json.dumps( + self.assertEqual(self.dumps( {True: False, False: True}, sort_keys=True), '{"false": true, "true": false}') - self.assertEqual(json.dumps( + self.assertEqual(self.dumps( {2: 3.0, 4.0: 5L, False: 1, 6L: True}, sort_keys=True), '{"false": 1, "2": 3.0, "4.0": 5, "6": true}') + +class TestPyDump(TestDump, PyTest): pass +class TestCDump(TestDump, CTest): pass diff --git a/lib-python/2.7/json/tests/test_encode_basestring_ascii.py b/lib-python/2.7/json/tests/test_encode_basestring_ascii.py --- a/lib-python/2.7/json/tests/test_encode_basestring_ascii.py +++ b/lib-python/2.7/json/tests/test_encode_basestring_ascii.py @@ -1,8 +1,6 @@ -from unittest import TestCase +from collections import OrderedDict +from json.tests import PyTest, CTest -import json.encoder -from json import dumps -from collections import OrderedDict CASES = [ (u'/\\"\ucafe\ubabe\uab98\ufcde\ubcda\uef4a\x08\x0c\n\r\t`1~!@#$%^&*()_+-=[]{}|;:\',./<>?', '"/\\\\\\"\\ucafe\\ubabe\\uab98\\ufcde\\ubcda\\uef4a\\b\\f\\n\\r\\t`1~!@#$%^&*()_+-=[]{}|;:\',./<>?"'), @@ -23,19 +21,11 @@ (u'\u0123\u4567\u89ab\ucdef\uabcd\uef4a', '"\\u0123\\u4567\\u89ab\\ucdef\\uabcd\\uef4a"'), ] -class TestEncodeBaseStringAscii(TestCase): - def test_py_encode_basestring_ascii(self): - self._test_encode_basestring_ascii(json.encoder.py_encode_basestring_ascii) - - def test_c_encode_basestring_ascii(self): - if not json.encoder.c_encode_basestring_ascii: - return - self._test_encode_basestring_ascii(json.encoder.c_encode_basestring_ascii) - - def _test_encode_basestring_ascii(self, encode_basestring_ascii): - fname = encode_basestring_ascii.__name__ +class TestEncodeBasestringAscii(object): + def test_encode_basestring_ascii(self): + fname = self.json.encoder.encode_basestring_ascii.__name__ for input_string, expect in CASES: - result = encode_basestring_ascii(input_string) + result = self.json.encoder.encode_basestring_ascii(input_string) self.assertEqual(result, expect, '{0!r} != {1!r} for {2}({3!r})'.format( result, expect, fname, input_string)) @@ -43,5 +33,9 @@ def test_ordered_dict(self): # See issue 6105 items = [('one', 1), ('two', 2), ('three', 3), ('four', 4), ('five', 5)] - s = json.dumps(OrderedDict(items)) + s = self.dumps(OrderedDict(items)) self.assertEqual(s, '{"one": 1, "two": 2, "three": 3, "four": 4, "five": 5}') + + +class TestPyEncodeBasestringAscii(TestEncodeBasestringAscii, PyTest): pass +class TestCEncodeBasestringAscii(TestEncodeBasestringAscii, CTest): pass diff --git a/lib-python/2.7/json/tests/test_fail.py b/lib-python/2.7/json/tests/test_fail.py --- a/lib-python/2.7/json/tests/test_fail.py +++ b/lib-python/2.7/json/tests/test_fail.py @@ -1,6 +1,4 @@ -from unittest import TestCase - -import json +from json.tests import PyTest, CTest # Fri Dec 30 18:57:26 2005 JSONDOCS = [ @@ -61,15 +59,15 @@ 18: "spec doesn't specify any nesting limitations", } -class TestFail(TestCase): +class TestFail(object): def test_failures(self): for idx, doc in enumerate(JSONDOCS): idx = idx + 1 if idx in SKIPS: - json.loads(doc) + self.loads(doc) continue try: - json.loads(doc) + self.loads(doc) except ValueError: pass else: @@ -79,7 +77,11 @@ data = {'a' : 1, (1, 2) : 2} #This is for c encoder - self.assertRaises(TypeError, json.dumps, data) + self.assertRaises(TypeError, self.dumps, data) #This is for python encoder - self.assertRaises(TypeError, json.dumps, data, indent=True) + self.assertRaises(TypeError, self.dumps, data, indent=True) + + +class TestPyFail(TestFail, PyTest): pass +class TestCFail(TestFail, CTest): pass diff --git a/lib-python/2.7/json/tests/test_float.py b/lib-python/2.7/json/tests/test_float.py --- a/lib-python/2.7/json/tests/test_float.py +++ b/lib-python/2.7/json/tests/test_float.py @@ -1,19 +1,22 @@ import math -from unittest import TestCase +from json.tests import PyTest, CTest -import json -class TestFloat(TestCase): +class TestFloat(object): def test_floats(self): for num in [1617161771.7650001, math.pi, math.pi**100, math.pi**-100, 3.1]: - self.assertEqual(float(json.dumps(num)), num) - self.assertEqual(json.loads(json.dumps(num)), num) - self.assertEqual(json.loads(unicode(json.dumps(num))), num) + self.assertEqual(float(self.dumps(num)), num) + self.assertEqual(self.loads(self.dumps(num)), num) + self.assertEqual(self.loads(unicode(self.dumps(num))), num) def test_ints(self): for num in [1, 1L, 1<<32, 1<<64]: - self.assertEqual(json.dumps(num), str(num)) - self.assertEqual(int(json.dumps(num)), num) - self.assertEqual(json.loads(json.dumps(num)), num) - self.assertEqual(json.loads(unicode(json.dumps(num))), num) + self.assertEqual(self.dumps(num), str(num)) + self.assertEqual(int(self.dumps(num)), num) + self.assertEqual(self.loads(self.dumps(num)), num) + self.assertEqual(self.loads(unicode(self.dumps(num))), num) + + +class TestPyFloat(TestFloat, PyTest): pass +class TestCFloat(TestFloat, CTest): pass diff --git a/lib-python/2.7/json/tests/test_indent.py b/lib-python/2.7/json/tests/test_indent.py --- a/lib-python/2.7/json/tests/test_indent.py +++ b/lib-python/2.7/json/tests/test_indent.py @@ -1,9 +1,9 @@ -from unittest import TestCase +import textwrap +from StringIO import StringIO +from json.tests import PyTest, CTest -import json -import textwrap -class TestIndent(TestCase): +class TestIndent(object): def test_indent(self): h = [['blorpie'], ['whoops'], [], 'd-shtaeou', 'd-nthiouh', 'i-vhbjkhnth', {'nifty': 87}, {'field': 'yes', 'morefield': False} ] @@ -30,12 +30,31 @@ ]""") - d1 = json.dumps(h) - d2 = json.dumps(h, indent=2, sort_keys=True, separators=(',', ': ')) + d1 = self.dumps(h) + d2 = self.dumps(h, indent=2, sort_keys=True, separators=(',', ': ')) - h1 = json.loads(d1) - h2 = json.loads(d2) + h1 = self.loads(d1) + h2 = self.loads(d2) self.assertEqual(h1, h) self.assertEqual(h2, h) self.assertEqual(d2, expect) + + def test_indent0(self): + h = {3: 1} + def check(indent, expected): + d1 = self.dumps(h, indent=indent) + self.assertEqual(d1, expected) + + sio = StringIO() + self.json.dump(h, sio, indent=indent) + self.assertEqual(sio.getvalue(), expected) + + # indent=0 should emit newlines + check(0, '{\n"3": 1\n}') + # indent=None is more compact + check(None, '{"3": 1}') + + +class TestPyIndent(TestIndent, PyTest): pass +class TestCIndent(TestIndent, CTest): pass diff --git a/lib-python/2.7/json/tests/test_pass1.py b/lib-python/2.7/json/tests/test_pass1.py --- a/lib-python/2.7/json/tests/test_pass1.py +++ b/lib-python/2.7/json/tests/test_pass1.py @@ -1,6 +1,5 @@ -from unittest import TestCase +from json.tests import PyTest, CTest -import json # from http://json.org/JSON_checker/test/pass1.json JSON = r''' @@ -62,15 +61,19 @@ ,"rosebud"] ''' -class TestPass1(TestCase): +class TestPass1(object): def test_parse(self): # test in/out equivalence and parsing - res = json.loads(JSON) - out = json.dumps(res) - self.assertEqual(res, json.loads(out)) + res = self.loads(JSON) + out = self.dumps(res) + self.assertEqual(res, self.loads(out)) try: - json.dumps(res, allow_nan=False) + self.dumps(res, allow_nan=False) except ValueError: pass else: self.fail("23456789012E666 should be out of range") + + +class TestPyPass1(TestPass1, PyTest): pass +class TestCPass1(TestPass1, CTest): pass diff --git a/lib-python/2.7/json/tests/test_pass2.py b/lib-python/2.7/json/tests/test_pass2.py --- a/lib-python/2.7/json/tests/test_pass2.py +++ b/lib-python/2.7/json/tests/test_pass2.py @@ -1,14 +1,18 @@ -from unittest import TestCase -import json +from json.tests import PyTest, CTest + # from http://json.org/JSON_checker/test/pass2.json JSON = r''' [[[[[[[[[[[[[[[[[[["Not too deep"]]]]]]]]]]]]]]]]]]] ''' -class TestPass2(TestCase): +class TestPass2(object): def test_parse(self): # test in/out equivalence and parsing - res = json.loads(JSON) - out = json.dumps(res) - self.assertEqual(res, json.loads(out)) + res = self.loads(JSON) + out = self.dumps(res) + self.assertEqual(res, self.loads(out)) + + +class TestPyPass2(TestPass2, PyTest): pass +class TestCPass2(TestPass2, CTest): pass diff --git a/lib-python/2.7/json/tests/test_pass3.py b/lib-python/2.7/json/tests/test_pass3.py --- a/lib-python/2.7/json/tests/test_pass3.py +++ b/lib-python/2.7/json/tests/test_pass3.py @@ -1,6 +1,5 @@ -from unittest import TestCase +from json.tests import PyTest, CTest -import json # from http://json.org/JSON_checker/test/pass3.json JSON = r''' @@ -12,9 +11,14 @@ } ''' -class TestPass3(TestCase): + +class TestPass3(object): def test_parse(self): # test in/out equivalence and parsing - res = json.loads(JSON) - out = json.dumps(res) - self.assertEqual(res, json.loads(out)) + res = self.loads(JSON) + out = self.dumps(res) + self.assertEqual(res, self.loads(out)) + + +class TestPyPass3(TestPass3, PyTest): pass +class TestCPass3(TestPass3, CTest): pass diff --git a/lib-python/2.7/json/tests/test_recursion.py b/lib-python/2.7/json/tests/test_recursion.py --- a/lib-python/2.7/json/tests/test_recursion.py +++ b/lib-python/2.7/json/tests/test_recursion.py @@ -1,28 +1,16 @@ -from unittest import TestCase +from json.tests import PyTest, CTest -import json class JSONTestObject: pass -class RecursiveJSONEncoder(json.JSONEncoder): - recurse = False - def default(self, o): - if o is JSONTestObject: - if self.recurse: - return [JSONTestObject] - else: - return 'JSONTestObject' - return json.JSONEncoder.default(o) - - -class TestRecursion(TestCase): +class TestRecursion(object): def test_listrecursion(self): x = [] x.append(x) try: - json.dumps(x) + self.dumps(x) except ValueError: pass else: @@ -31,7 +19,7 @@ y = [x] x.append(y) try: - json.dumps(x) + self.dumps(x) except ValueError: pass else: @@ -39,13 +27,13 @@ y = [] x = [y, y] # ensure that the marker is cleared - json.dumps(x) + self.dumps(x) def test_dictrecursion(self): x = {} x["test"] = x try: - json.dumps(x) + self.dumps(x) except ValueError: pass else: @@ -53,9 +41,19 @@ x = {} y = {"a": x, "b": x} # ensure that the marker is cleared - json.dumps(x) + self.dumps(x) def test_defaultrecursion(self): + class RecursiveJSONEncoder(self.json.JSONEncoder): + recurse = False + def default(self, o): + if o is JSONTestObject: + if self.recurse: + return [JSONTestObject] + else: + return 'JSONTestObject' + return pyjson.JSONEncoder.default(o) + enc = RecursiveJSONEncoder() self.assertEqual(enc.encode(JSONTestObject), '"JSONTestObject"') enc.recurse = True @@ -65,3 +63,46 @@ pass else: self.fail("didn't raise ValueError on default recursion") + + + def test_highly_nested_objects_decoding(self): + # test that loading highly-nested objects doesn't segfault when C + # accelerations are used. See #12017 + # str + with self.assertRaises(RuntimeError): + self.loads('{"a":' * 100000 + '1' + '}' * 100000) + with self.assertRaises(RuntimeError): + self.loads('{"a":' * 100000 + '[1]' + '}' * 100000) + with self.assertRaises(RuntimeError): + self.loads('[' * 100000 + '1' + ']' * 100000) + # unicode + with self.assertRaises(RuntimeError): + self.loads(u'{"a":' * 100000 + u'1' + u'}' * 100000) + with self.assertRaises(RuntimeError): + self.loads(u'{"a":' * 100000 + u'[1]' + u'}' * 100000) + with self.assertRaises(RuntimeError): + self.loads(u'[' * 100000 + u'1' + u']' * 100000) + + def test_highly_nested_objects_encoding(self): + # See #12051 + l, d = [], {} + for x in xrange(100000): + l, d = [l], {'k':d} + with self.assertRaises(RuntimeError): + self.dumps(l) + with self.assertRaises(RuntimeError): + self.dumps(d) + + def test_endless_recursion(self): + # See #12051 + class EndlessJSONEncoder(self.json.JSONEncoder): + def default(self, o): + """If check_circular is False, this will keep adding another list.""" + return [o] + + with self.assertRaises(RuntimeError): + EndlessJSONEncoder(check_circular=False).encode(5j) + + +class TestPyRecursion(TestRecursion, PyTest): pass +class TestCRecursion(TestRecursion, CTest): pass diff --git a/lib-python/2.7/json/tests/test_scanstring.py b/lib-python/2.7/json/tests/test_scanstring.py --- a/lib-python/2.7/json/tests/test_scanstring.py +++ b/lib-python/2.7/json/tests/test_scanstring.py @@ -1,18 +1,10 @@ import sys -import decimal -from unittest import TestCase +from json.tests import PyTest, CTest -import json -import json.decoder -class TestScanString(TestCase): - def test_py_scanstring(self): - self._test_scanstring(json.decoder.py_scanstring) - - def test_c_scanstring(self): - self._test_scanstring(json.decoder.c_scanstring) - - def _test_scanstring(self, scanstring): +class TestScanstring(object): + def test_scanstring(self): + scanstring = self.json.decoder.scanstring self.assertEqual( scanstring('"z\\ud834\\udd20x"', 1, None, True), (u'z\U0001d120x', 16)) @@ -103,10 +95,15 @@ (u'Bad value', 12)) def test_issue3623(self): - self.assertRaises(ValueError, json.decoder.scanstring, b"xxx", 1, + self.assertRaises(ValueError, self.json.decoder.scanstring, b"xxx", 1, "xxx") self.assertRaises(UnicodeDecodeError, - json.encoder.encode_basestring_ascii, b"xx\xff") + self.json.encoder.encode_basestring_ascii, b"xx\xff") def test_overflow(self): - self.assertRaises(OverflowError, json.decoder.scanstring, b"xxx", sys.maxsize+1) + with self.assertRaises(OverflowError): + self.json.decoder.scanstring(b"xxx", sys.maxsize+1) + + +class TestPyScanstring(TestScanstring, PyTest): pass +class TestCScanstring(TestScanstring, CTest): pass diff --git a/lib-python/2.7/json/tests/test_separators.py b/lib-python/2.7/json/tests/test_separators.py --- a/lib-python/2.7/json/tests/test_separators.py +++ b/lib-python/2.7/json/tests/test_separators.py @@ -1,10 +1,8 @@ import textwrap -from unittest import TestCase +from json.tests import PyTest, CTest -import json - -class TestSeparators(TestCase): +class TestSeparators(object): def test_separators(self): h = [['blorpie'], ['whoops'], [], 'd-shtaeou', 'd-nthiouh', 'i-vhbjkhnth', {'nifty': 87}, {'field': 'yes', 'morefield': False} ] @@ -31,12 +29,16 @@ ]""") - d1 = json.dumps(h) - d2 = json.dumps(h, indent=2, sort_keys=True, separators=(' ,', ' : ')) + d1 = self.dumps(h) + d2 = self.dumps(h, indent=2, sort_keys=True, separators=(' ,', ' : ')) - h1 = json.loads(d1) - h2 = json.loads(d2) + h1 = self.loads(d1) + h2 = self.loads(d2) self.assertEqual(h1, h) self.assertEqual(h2, h) self.assertEqual(d2, expect) + + +class TestPySeparators(TestSeparators, PyTest): pass +class TestCSeparators(TestSeparators, CTest): pass diff --git a/lib-python/2.7/json/tests/test_speedups.py b/lib-python/2.7/json/tests/test_speedups.py --- a/lib-python/2.7/json/tests/test_speedups.py +++ b/lib-python/2.7/json/tests/test_speedups.py @@ -1,24 +1,23 @@ -import decimal -from unittest import TestCase +from json.tests import CTest -from json import decoder, encoder, scanner -class TestSpeedups(TestCase): +class TestSpeedups(CTest): def test_scanstring(self): - self.assertEqual(decoder.scanstring.__module__, "_json") - self.assertTrue(decoder.scanstring is decoder.c_scanstring) + self.assertEqual(self.json.decoder.scanstring.__module__, "_json") + self.assertIs(self.json.decoder.scanstring, self.json.decoder.c_scanstring) def test_encode_basestring_ascii(self): - self.assertEqual(encoder.encode_basestring_ascii.__module__, "_json") - self.assertTrue(encoder.encode_basestring_ascii is - encoder.c_encode_basestring_ascii) + self.assertEqual(self.json.encoder.encode_basestring_ascii.__module__, + "_json") + self.assertIs(self.json.encoder.encode_basestring_ascii, + self.json.encoder.c_encode_basestring_ascii) -class TestDecode(TestCase): +class TestDecode(CTest): def test_make_scanner(self): - self.assertRaises(AttributeError, scanner.c_make_scanner, 1) + self.assertRaises(AttributeError, self.json.scanner.c_make_scanner, 1) def test_make_encoder(self): - self.assertRaises(TypeError, encoder.c_make_encoder, + self.assertRaises(TypeError, self.json.encoder.c_make_encoder, None, "\xCD\x7D\x3D\x4E\x12\x4C\xF9\x79\xD7\x52\xBA\x82\xF2\x27\x4A\x7D\xA0\xCA\x75", None) diff --git a/lib-python/2.7/json/tests/test_unicode.py b/lib-python/2.7/json/tests/test_unicode.py --- a/lib-python/2.7/json/tests/test_unicode.py +++ b/lib-python/2.7/json/tests/test_unicode.py @@ -1,11 +1,10 @@ -from unittest import TestCase +from collections import OrderedDict +from json.tests import PyTest, CTest -import json -from collections import OrderedDict -class TestUnicode(TestCase): +class TestUnicode(object): def test_encoding1(self): - encoder = json.JSONEncoder(encoding='utf-8') + encoder = self.json.JSONEncoder(encoding='utf-8') u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' s = u.encode('utf-8') ju = encoder.encode(u) @@ -15,68 +14,72 @@ def test_encoding2(self): u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' s = u.encode('utf-8') - ju = json.dumps(u, encoding='utf-8') - js = json.dumps(s, encoding='utf-8') + ju = self.dumps(u, encoding='utf-8') + js = self.dumps(s, encoding='utf-8') self.assertEqual(ju, js) def test_encoding3(self): u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' - j = json.dumps(u) + j = self.dumps(u) self.assertEqual(j, '"\\u03b1\\u03a9"') def test_encoding4(self): u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' - j = json.dumps([u]) + j = self.dumps([u]) self.assertEqual(j, '["\\u03b1\\u03a9"]') def test_encoding5(self): u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' - j = json.dumps(u, ensure_ascii=False) + j = self.dumps(u, ensure_ascii=False) self.assertEqual(j, u'"{0}"'.format(u)) def test_encoding6(self): u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' - j = json.dumps([u], ensure_ascii=False) + j = self.dumps([u], ensure_ascii=False) self.assertEqual(j, u'["{0}"]'.format(u)) def test_big_unicode_encode(self): u = u'\U0001d120' - self.assertEqual(json.dumps(u), '"\\ud834\\udd20"') - self.assertEqual(json.dumps(u, ensure_ascii=False), u'"\U0001d120"') + self.assertEqual(self.dumps(u), '"\\ud834\\udd20"') + self.assertEqual(self.dumps(u, ensure_ascii=False), u'"\U0001d120"') def test_big_unicode_decode(self): u = u'z\U0001d120x' - self.assertEqual(json.loads('"' + u + '"'), u) - self.assertEqual(json.loads('"z\\ud834\\udd20x"'), u) + self.assertEqual(self.loads('"' + u + '"'), u) + self.assertEqual(self.loads('"z\\ud834\\udd20x"'), u) def test_unicode_decode(self): for i in range(0, 0xd7ff): u = unichr(i) s = '"\\u{0:04x}"'.format(i) - self.assertEqual(json.loads(s), u) + self.assertEqual(self.loads(s), u) def test_object_pairs_hook_with_unicode(self): s = u'{"xkd":1, "kcw":2, "art":3, "hxm":4, "qrt":5, "pad":6, "hoy":7}' p = [(u"xkd", 1), (u"kcw", 2), (u"art", 3), (u"hxm", 4), (u"qrt", 5), (u"pad", 6), (u"hoy", 7)] - self.assertEqual(json.loads(s), eval(s)) - self.assertEqual(json.loads(s, object_pairs_hook = lambda x: x), p) - od = json.loads(s, object_pairs_hook = OrderedDict) + self.assertEqual(self.loads(s), eval(s)) + self.assertEqual(self.loads(s, object_pairs_hook = lambda x: x), p) + od = self.loads(s, object_pairs_hook = OrderedDict) self.assertEqual(od, OrderedDict(p)) self.assertEqual(type(od), OrderedDict) # the object_pairs_hook takes priority over the object_hook - self.assertEqual(json.loads(s, + self.assertEqual(self.loads(s, object_pairs_hook = OrderedDict, object_hook = lambda x: None), OrderedDict(p)) def test_default_encoding(self): - self.assertEqual(json.loads(u'{"a": "\xe9"}'.encode('utf-8')), + self.assertEqual(self.loads(u'{"a": "\xe9"}'.encode('utf-8')), {'a': u'\xe9'}) def test_unicode_preservation(self): - self.assertEqual(type(json.loads(u'""')), unicode) - self.assertEqual(type(json.loads(u'"a"')), unicode) - self.assertEqual(type(json.loads(u'["a"]')[0]), unicode) + self.assertEqual(type(self.loads(u'""')), unicode) + self.assertEqual(type(self.loads(u'"a"')), unicode) + self.assertEqual(type(self.loads(u'["a"]')[0]), unicode) # Issue 10038. - self.assertEqual(type(json.loads('"foo"')), unicode) + self.assertEqual(type(self.loads('"foo"')), unicode) + + +class TestPyUnicode(TestUnicode, PyTest): pass +class TestCUnicode(TestUnicode, CTest): pass diff --git a/lib-python/2.7/lib-tk/Tix.py b/lib-python/2.7/lib-tk/Tix.py --- a/lib-python/2.7/lib-tk/Tix.py +++ b/lib-python/2.7/lib-tk/Tix.py @@ -163,7 +163,7 @@ extensions) exist, then the image type is chosen according to the depth of the X display: xbm images are chosen on monochrome displays and color images are chosen on color displays. By using - tix_ getimage, you can advoid hard coding the pathnames of the + tix_ getimage, you can avoid hard coding the pathnames of the image files in your application. When successful, this command returns the name of the newly created image, which can be used to configure the -image option of the Tk and Tix widgets. @@ -171,7 +171,7 @@ return self.tk.call('tix', 'getimage', name) def tix_option_get(self, name): - """Gets the options manitained by the Tix + """Gets the options maintained by the Tix scheme mechanism. Available options include: active_bg active_fg bg @@ -576,7 +576,7 @@ class ComboBox(TixWidget): """ComboBox - an Entry field with a dropdown menu. The user can select a - choice by either typing in the entry subwdget or selecting from the + choice by either typing in the entry subwidget or selecting from the listbox subwidget. Subwidget Class @@ -869,7 +869,7 @@ """HList - Hierarchy display widget can be used to display any data that have a hierarchical structure, for example, file system directory trees. The list entries are indented and connected by branch lines - according to their places in the hierachy. + according to their places in the hierarchy. Subwidgets - None""" @@ -1520,7 +1520,7 @@ self.tk.call(self._w, 'selection', 'set', first, last) class Tree(TixWidget): - """Tree - The tixTree widget can be used to display hierachical + """Tree - The tixTree widget can be used to display hierarchical data in a tree form. The user can adjust the view of the tree by opening or closing parts of the tree.""" diff --git a/lib-python/2.7/lib-tk/Tkinter.py b/lib-python/2.7/lib-tk/Tkinter.py --- a/lib-python/2.7/lib-tk/Tkinter.py +++ b/lib-python/2.7/lib-tk/Tkinter.py @@ -1660,7 +1660,7 @@ class Tk(Misc, Wm): """Toplevel widget of Tk which represents mostly the main window - of an appliation. It has an associated Tcl interpreter.""" + of an application. It has an associated Tcl interpreter.""" _w = '.' def __init__(self, screenName=None, baseName=None, className='Tk', useTk=1, sync=0, use=None): diff --git a/lib-python/2.7/lib-tk/test/test_ttk/test_functions.py b/lib-python/2.7/lib-tk/test/test_ttk/test_functions.py --- a/lib-python/2.7/lib-tk/test/test_ttk/test_functions.py +++ b/lib-python/2.7/lib-tk/test/test_ttk/test_functions.py @@ -136,7 +136,7 @@ # minimum acceptable for image type self.assertEqual(ttk._format_elemcreate('image', False, 'test'), ("test ", ())) - # specifiyng a state spec + # specifying a state spec self.assertEqual(ttk._format_elemcreate('image', False, 'test', ('', 'a')), ("test {} a", ())) # state spec with multiple states diff --git a/lib-python/2.7/lib-tk/ttk.py b/lib-python/2.7/lib-tk/ttk.py --- a/lib-python/2.7/lib-tk/ttk.py +++ b/lib-python/2.7/lib-tk/ttk.py @@ -707,7 +707,7 @@ textvariable, values, width """ # The "values" option may need special formatting, so leave to - # _format_optdict the responsability to format it + # _format_optdict the responsibility to format it if "values" in kw: kw["values"] = _format_optdict({'v': kw["values"]})[1] @@ -993,7 +993,7 @@ pane is either an integer index or the name of a managed subwindow. If kw is not given, returns a dict of the pane option values. If option is specified then the value for that option is returned. - Otherwise, sets the options to the correspoding values.""" + Otherwise, sets the options to the corresponding values.""" if option is not None: kw[option] = None return _val_or_dict(kw, self.tk.call, self._w, "pane", pane) diff --git a/lib-python/2.7/lib-tk/turtle.py b/lib-python/2.7/lib-tk/turtle.py --- a/lib-python/2.7/lib-tk/turtle.py +++ b/lib-python/2.7/lib-tk/turtle.py @@ -1385,7 +1385,7 @@ Optional argument: picname -- a string, name of a gif-file or "nopic". - If picname is a filename, set the corresponing image as background. + If picname is a filename, set the corresponding image as background. If picname is "nopic", delete backgroundimage, if present. If picname is None, return the filename of the current backgroundimage. @@ -1409,7 +1409,7 @@ Optional arguments: canvwidth -- positive integer, new width of canvas in pixels canvheight -- positive integer, new height of canvas in pixels - bg -- colorstring or color-tupel, new backgroundcolor + bg -- colorstring or color-tuple, new backgroundcolor If no arguments are given, return current (canvaswidth, canvasheight) Do not alter the drawing window. To observe hidden parts of @@ -3079,9 +3079,9 @@ fill="", width=ps) # Turtle now at position old, self._position = old - ## if undo is done during crating a polygon, the last vertex - ## will be deleted. if the polygon is entirel deleted, - ## creatigPoly will be set to False. + ## if undo is done during creating a polygon, the last vertex + ## will be deleted. if the polygon is entirely deleted, + ## creatingPoly will be set to False. ## Polygons created before the last one will not be affected by undo() if self._creatingPoly: if len(self._poly) > 0: @@ -3221,7 +3221,7 @@ def dot(self, size=None, *color): """Draw a dot with diameter size, using color. - Optional argumentS: + Optional arguments: size -- an integer >= 1 (if given) color -- a colorstring or a numeric color tuple @@ -3691,7 +3691,7 @@ class Turtle(RawTurtle): - """RawTurtle auto-crating (scrolled) canvas. + """RawTurtle auto-creating (scrolled) canvas. When a Turtle object is created or a function derived from some Turtle method is called a TurtleScreen object is automatically created. @@ -3731,7 +3731,7 @@ filename -- a string, used as filename default value is turtle_docstringdict - Has to be called explicitely, (not used by the turtle-graphics classes) + Has to be called explicitly, (not used by the turtle-graphics classes) The docstring dictionary will be written to the Python script <filname>.py It is intended to serve as a template for translation of the docstrings into different languages. diff --git a/lib-python/2.7/lib2to3/__main__.py b/lib-python/2.7/lib2to3/__main__.py new file mode 100644 --- /dev/null +++ b/lib-python/2.7/lib2to3/__main__.py @@ -0,0 +1,4 @@ +import sys +from .main import main + +sys.exit(main("lib2to3.fixes")) diff --git a/lib-python/2.7/lib2to3/fixes/fix_itertools.py b/lib-python/2.7/lib2to3/fixes/fix_itertools.py --- a/lib-python/2.7/lib2to3/fixes/fix_itertools.py +++ b/lib-python/2.7/lib2to3/fixes/fix_itertools.py @@ -13,7 +13,7 @@ class FixItertools(fixer_base.BaseFix): BM_compatible = True - it_funcs = "('imap'|'ifilter'|'izip'|'ifilterfalse')" + it_funcs = "('imap'|'ifilter'|'izip'|'izip_longest'|'ifilterfalse')" PATTERN = """ power< it='itertools' trailer< @@ -28,7 +28,8 @@ def transform(self, node, results): prefix = None func = results['func'][0] - if 'it' in results and func.value != u'ifilterfalse': + if ('it' in results and + func.value not in (u'ifilterfalse', u'izip_longest')): dot, it = (results['dot'], results['it']) # Remove the 'itertools' prefix = it.prefix diff --git a/lib-python/2.7/lib2to3/fixes/fix_itertools_imports.py b/lib-python/2.7/lib2to3/fixes/fix_itertools_imports.py --- a/lib-python/2.7/lib2to3/fixes/fix_itertools_imports.py +++ b/lib-python/2.7/lib2to3/fixes/fix_itertools_imports.py @@ -31,9 +31,10 @@ if member_name in (u'imap', u'izip', u'ifilter'): child.value = None child.remove() - elif member_name == u'ifilterfalse': + elif member_name in (u'ifilterfalse', u'izip_longest'): node.changed() - name_node.value = u'filterfalse' + name_node.value = (u'filterfalse' if member_name[1] == u'f' + else u'zip_longest') # Make sure the import statement is still sane children = imports.children[:] or [imports] diff --git a/lib-python/2.7/lib2to3/fixes/fix_metaclass.py b/lib-python/2.7/lib2to3/fixes/fix_metaclass.py --- a/lib-python/2.7/lib2to3/fixes/fix_metaclass.py +++ b/lib-python/2.7/lib2to3/fixes/fix_metaclass.py @@ -48,7 +48,7 @@ """ for node in cls_node.children: if node.type == syms.suite: - # already in the prefered format, do nothing + # already in the preferred format, do nothing return # !%@#! oneliners have no suite node, we have to fake one up diff --git a/lib-python/2.7/lib2to3/fixes/fix_urllib.py b/lib-python/2.7/lib2to3/fixes/fix_urllib.py --- a/lib-python/2.7/lib2to3/fixes/fix_urllib.py +++ b/lib-python/2.7/lib2to3/fixes/fix_urllib.py @@ -12,7 +12,7 @@ MAPPING = {"urllib": [ ("urllib.request", - ["URLOpener", "FancyURLOpener", "urlretrieve", + ["URLopener", "FancyURLopener", "urlretrieve", "_urlopener", "urlopen", "urlcleanup", "pathname2url", "url2pathname"]), ("urllib.parse", diff --git a/lib-python/2.7/lib2to3/main.py b/lib-python/2.7/lib2to3/main.py --- a/lib-python/2.7/lib2to3/main.py +++ b/lib-python/2.7/lib2to3/main.py @@ -101,7 +101,7 @@ parser.add_option("-j", "--processes", action="store", default=1, type="int", help="Run 2to3 concurrently") parser.add_option("-x", "--nofix", action="append", default=[], - help="Prevent a fixer from being run.") + help="Prevent a transformation from being run") parser.add_option("-l", "--list-fixes", action="store_true", help="List available transformations") parser.add_option("-p", "--print-function", action="store_true", @@ -113,7 +113,7 @@ parser.add_option("-w", "--write", action="store_true", help="Write back modified files") parser.add_option("-n", "--nobackups", action="store_true", default=False, - help="Don't write backups for modified files.") + help="Don't write backups for modified files") # Parse command line arguments refactor_stdin = False diff --git a/lib-python/2.7/lib2to3/patcomp.py b/lib-python/2.7/lib2to3/patcomp.py --- a/lib-python/2.7/lib2to3/patcomp.py +++ b/lib-python/2.7/lib2to3/patcomp.py @@ -12,6 +12,7 @@ # Python imports import os +import StringIO # Fairly local imports from .pgen2 import driver, literals, token, tokenize, parse, grammar @@ -32,7 +33,7 @@ def tokenize_wrapper(input): """Tokenizes a string suppressing significant whitespace.""" skip = set((token.NEWLINE, token.INDENT, token.DEDENT)) - tokens = tokenize.generate_tokens(driver.generate_lines(input).next) + tokens = tokenize.generate_tokens(StringIO.StringIO(input).readline) for quintuple in tokens: type, value, start, end, line_text = quintuple if type not in skip: diff --git a/lib-python/2.7/lib2to3/pgen2/conv.py b/lib-python/2.7/lib2to3/pgen2/conv.py --- a/lib-python/2.7/lib2to3/pgen2/conv.py +++ b/lib-python/2.7/lib2to3/pgen2/conv.py @@ -51,7 +51,7 @@ self.finish_off() def parse_graminit_h(self, filename): - """Parse the .h file writen by pgen. (Internal) + """Parse the .h file written by pgen. (Internal) This file is a sequence of #define statements defining the nonterminals of the grammar as numbers. We build two tables @@ -82,7 +82,7 @@ return True def parse_graminit_c(self, filename): - """Parse the .c file writen by pgen. (Internal) + """Parse the .c file written by pgen. (Internal) The file looks as follows. The first two lines are always this: diff --git a/lib-python/2.7/lib2to3/pgen2/driver.py b/lib-python/2.7/lib2to3/pgen2/driver.py --- a/lib-python/2.7/lib2to3/pgen2/driver.py +++ b/lib-python/2.7/lib2to3/pgen2/driver.py @@ -19,6 +19,7 @@ import codecs import os import logging +import StringIO import sys # Pgen imports @@ -101,18 +102,10 @@ def parse_string(self, text, debug=False): """Parse a string and return the syntax tree.""" - tokens = tokenize.generate_tokens(generate_lines(text).next) + tokens = tokenize.generate_tokens(StringIO.StringIO(text).readline) return self.parse_tokens(tokens, debug) -def generate_lines(text): - """Generator that behaves like readline without using StringIO.""" - for line in text.splitlines(True): - yield line - while True: - yield "" - - def load_grammar(gt="Grammar.txt", gp=None, save=True, force=False, logger=None): """Load the grammar (maybe from a pickle).""" diff --git a/lib-python/2.7/lib2to3/pytree.py b/lib-python/2.7/lib2to3/pytree.py --- a/lib-python/2.7/lib2to3/pytree.py +++ b/lib-python/2.7/lib2to3/pytree.py @@ -658,8 +658,8 @@ content: optional sequence of subsequences of patterns; if absent, matches one node; if present, each subsequence is an alternative [*] - min: optinal minumum number of times to match, default 0 - max: optional maximum number of times tro match, default HUGE + min: optional minimum number of times to match, default 0 + max: optional maximum number of times to match, default HUGE name: optional name assigned to this match [*] Thus, if content is [[a, b, c], [d, e], [f, g, h]] this is @@ -743,9 +743,11 @@ else: # The reason for this is that hitting the recursion limit usually # results in some ugly messages about how RuntimeErrors are being - # ignored. - save_stderr = sys.stderr - sys.stderr = StringIO() + # ignored. We don't do this on non-CPython implementation because + # they don't have this problem. + if hasattr(sys, "getrefcount"): + save_stderr = sys.stderr + sys.stderr = StringIO() try: for count, r in self._recursive_matches(nodes, 0): if self.name: @@ -759,7 +761,8 @@ r[self.name] = nodes[:count] yield count, r finally: - sys.stderr = save_stderr + if hasattr(sys, "getrefcount"): + sys.stderr = save_stderr def _iterative_matches(self, nodes): """Helper to iteratively yield the matches.""" diff --git a/lib-python/2.7/lib2to3/refactor.py b/lib-python/2.7/lib2to3/refactor.py --- a/lib-python/2.7/lib2to3/refactor.py +++ b/lib-python/2.7/lib2to3/refactor.py @@ -302,13 +302,14 @@ Files and subdirectories starting with '.' are skipped. """ + py_ext = os.extsep + "py" for dirpath, dirnames, filenames in os.walk(dir_name): self.log_debug("Descending into %s", dirpath) dirnames.sort() filenames.sort() for name in filenames: - if not name.startswith(".") and \ - os.path.splitext(name)[1].endswith("py"): + if (not name.startswith(".") and + os.path.splitext(name)[1] == py_ext): fullname = os.path.join(dirpath, name) self.refactor_file(fullname, write, doctests_only) # Modify dirnames in-place to remove subdirs with leading dots diff --git a/lib-python/2.7/lib2to3/tests/data/py2_test_grammar.py b/lib-python/2.7/lib2to3/tests/data/py2_test_grammar.py --- a/lib-python/2.7/lib2to3/tests/data/py2_test_grammar.py +++ b/lib-python/2.7/lib2to3/tests/data/py2_test_grammar.py @@ -316,7 +316,7 @@ ### simple_stmt: small_stmt (';' small_stmt)* [';'] x = 1; pass; del x def foo(): - # verify statments that end with semi-colons + # verify statements that end with semi-colons x = 1; pass; del x; foo() diff --git a/lib-python/2.7/lib2to3/tests/data/py3_test_grammar.py b/lib-python/2.7/lib2to3/tests/data/py3_test_grammar.py --- a/lib-python/2.7/lib2to3/tests/data/py3_test_grammar.py +++ b/lib-python/2.7/lib2to3/tests/data/py3_test_grammar.py @@ -356,7 +356,7 @@ ### simple_stmt: small_stmt (';' small_stmt)* [';'] x = 1; pass; del x def foo(): - # verify statments that end with semi-colons + # verify statements that end with semi-colons x = 1; pass; del x; foo() diff --git a/lib-python/2.7/lib2to3/tests/test_fixers.py b/lib-python/2.7/lib2to3/tests/test_fixers.py --- a/lib-python/2.7/lib2to3/tests/test_fixers.py +++ b/lib-python/2.7/lib2to3/tests/test_fixers.py @@ -3623,16 +3623,24 @@ a = """%s(f, a)""" self.checkall(b, a) - def test_2(self): + def test_qualified(self): b = """itertools.ifilterfalse(a, b)""" a = """itertools.filterfalse(a, b)""" self.check(b, a) - def test_4(self): + b = """itertools.izip_longest(a, b)""" + a = """itertools.zip_longest(a, b)""" + self.check(b, a) + + def test_2(self): b = """ifilterfalse(a, b)""" a = """filterfalse(a, b)""" self.check(b, a) + b = """izip_longest(a, b)""" + a = """zip_longest(a, b)""" + self.check(b, a) + def test_space_1(self): b = """ %s(f, a)""" a = """ %s(f, a)""" @@ -3643,9 +3651,14 @@ a = """ itertools.filterfalse(a, b)""" self.check(b, a) + b = """ itertools.izip_longest(a, b)""" + a = """ itertools.zip_longest(a, b)""" + self.check(b, a) + def test_run_order(self): self.assert_runs_after('map', 'zip', 'filter') + class Test_itertools_imports(FixerTestCase): fixer = 'itertools_imports' @@ -3696,18 +3709,19 @@ s = "from itertools import bar as bang" self.unchanged(s) - def test_ifilter(self): - b = "from itertools import ifilterfalse" - a = "from itertools import filterfalse" - self.check(b, a) - - b = "from itertools import imap, ifilterfalse, foo" - a = "from itertools import filterfalse, foo" - self.check(b, a) - - b = "from itertools import bar, ifilterfalse, foo" - a = "from itertools import bar, filterfalse, foo" - self.check(b, a) + def test_ifilter_and_zip_longest(self): + for name in "filterfalse", "zip_longest": + b = "from itertools import i%s" % (name,) + a = "from itertools import %s" % (name,) + self.check(b, a) + + b = "from itertools import imap, i%s, foo" % (name,) + a = "from itertools import %s, foo" % (name,) + self.check(b, a) + + b = "from itertools import bar, i%s, foo" % (name,) + a = "from itertools import bar, %s, foo" % (name,) + self.check(b, a) def test_import_star(self): s = "from itertools import *" diff --git a/lib-python/2.7/lib2to3/tests/test_parser.py b/lib-python/2.7/lib2to3/tests/test_parser.py --- a/lib-python/2.7/lib2to3/tests/test_parser.py +++ b/lib-python/2.7/lib2to3/tests/test_parser.py @@ -19,6 +19,16 @@ # Local imports from lib2to3.pgen2 import tokenize from ..pgen2.parse import ParseError +from lib2to3.pygram import python_symbols as syms + + +class TestDriver(support.TestCase): + + def test_formfeed(self): + s = """print 1\n\x0Cprint 2\n""" + t = driver.parse_string(s) + self.assertEqual(t.children[0].children[0].type, syms.print_stmt) + self.assertEqual(t.children[1].children[0].type, syms.print_stmt) class GrammarTest(support.TestCase): diff --git a/lib-python/2.7/lib2to3/tests/test_refactor.py b/lib-python/2.7/lib2to3/tests/test_refactor.py --- a/lib-python/2.7/lib2to3/tests/test_refactor.py +++ b/lib-python/2.7/lib2to3/tests/test_refactor.py @@ -223,6 +223,7 @@ "hi.py", ".dumb", ".after.py", + "notpy.npy", "sappy"] expected = ["hi.py"] check(tree, expected) diff --git a/lib-python/2.7/lib2to3/tests/test_util.py b/lib-python/2.7/lib2to3/tests/test_util.py --- a/lib-python/2.7/lib2to3/tests/test_util.py +++ b/lib-python/2.7/lib2to3/tests/test_util.py @@ -568,8 +568,8 @@ def test_from_import(self): node = parse('bar()') - fixer_util.touch_import("cgi", "escape", node) - self.assertEqual(str(node), 'from cgi import escape\nbar()\n\n') + fixer_util.touch_import("html", "escape", node) + self.assertEqual(str(node), 'from html import escape\nbar()\n\n') def test_name_import(self): node = parse('bar()') diff --git a/lib-python/2.7/locale.py b/lib-python/2.7/locale.py --- a/lib-python/2.7/locale.py +++ b/lib-python/2.7/locale.py @@ -621,7 +621,7 @@ 'tactis': 'TACTIS', 'euc_jp': 'eucJP', 'euc_kr': 'eucKR', - 'utf_8': 'UTF8', + 'utf_8': 'UTF-8', 'koi8_r': 'KOI8-R', 'koi8_u': 'KOI8-U', # XXX This list is still incomplete. If you know more diff --git a/lib-python/2.7/logging/__init__.py b/lib-python/2.7/logging/__init__.py --- a/lib-python/2.7/logging/__init__.py +++ b/lib-python/2.7/logging/__init__.py @@ -1627,6 +1627,7 @@ h = wr() if h: try: + h.acquire() h.flush() h.close() except (IOError, ValueError): @@ -1635,6 +1636,8 @@ # references to them are still around at # application exit. pass + finally: + h.release() except: if raiseExceptions: raise diff --git a/lib-python/2.7/logging/config.py b/lib-python/2.7/logging/config.py --- a/lib-python/2.7/logging/config.py +++ b/lib-python/2.7/logging/config.py @@ -226,14 +226,14 @@ propagate = 1 logger = logging.getLogger(qn) if qn in existing: - i = existing.index(qn) + i = existing.index(qn) + 1 # start with the entry after qn prefixed = qn + "." pflen = len(prefixed) num_existing = len(existing) - i = i + 1 # look at the entry after qn - while (i < num_existing) and (existing[i][:pflen] == prefixed): - child_loggers.append(existing[i]) - i = i + 1 + while i < num_existing: + if existing[i][:pflen] == prefixed: + child_loggers.append(existing[i]) + i += 1 existing.remove(qn) if "level" in opts: level = cp.get(sectname, "level") diff --git a/lib-python/2.7/logging/handlers.py b/lib-python/2.7/logging/handlers.py --- a/lib-python/2.7/logging/handlers.py +++ b/lib-python/2.7/logging/handlers.py @@ -125,6 +125,7 @@ """ if self.stream: self.stream.close() + self.stream = None if self.backupCount > 0: for i in range(self.backupCount - 1, 0, -1): sfn = "%s.%d" % (self.baseFilename, i) @@ -324,6 +325,7 @@ """ if self.stream: self.stream.close() + self.stream = None # get the time that this sequence started at and make it a TimeTuple t = self.rolloverAt - self.interval if self.utc: diff --git a/lib-python/2.7/mailbox.py b/lib-python/2.7/mailbox.py --- a/lib-python/2.7/mailbox.py +++ b/lib-python/2.7/mailbox.py @@ -234,27 +234,35 @@ def __init__(self, dirname, factory=rfc822.Message, create=True): """Initialize a Maildir instance.""" Mailbox.__init__(self, dirname, factory, create) + self._paths = { + 'tmp': os.path.join(self._path, 'tmp'), + 'new': os.path.join(self._path, 'new'), + 'cur': os.path.join(self._path, 'cur'), + } if not os.path.exists(self._path): if create: os.mkdir(self._path, 0700) - os.mkdir(os.path.join(self._path, 'tmp'), 0700) - os.mkdir(os.path.join(self._path, 'new'), 0700) - os.mkdir(os.path.join(self._path, 'cur'), 0700) + for path in self._paths.values(): + os.mkdir(path, 0o700) else: raise NoSuchMailboxError(self._path) self._toc = {} - self._last_read = None # Records last time we read cur/new - # NOTE: we manually invalidate _last_read each time we do any - # modifications ourselves, otherwise we might get tripped up by - # bogus mtime behaviour on some systems (see issue #6896). + self._toc_mtimes = {} + for subdir in ('cur', 'new'): + self._toc_mtimes[subdir] = os.path.getmtime(self._paths[subdir]) + self._last_read = time.time() # Records last time we read cur/new + self._skewfactor = 0.1 # Adjust if os/fs clocks are skewing def add(self, message): """Add message and return assigned key.""" tmp_file = self._create_tmp() try: self._dump_message(message, tmp_file) - finally: - _sync_close(tmp_file) + except BaseException: + tmp_file.close() + os.remove(tmp_file.name) + raise + _sync_close(tmp_file) if isinstance(message, MaildirMessage): subdir = message.get_subdir() suffix = self.colon + message.get_info() @@ -280,15 +288,11 @@ raise if isinstance(message, MaildirMessage): os.utime(dest, (os.path.getatime(dest), message.get_date())) - # Invalidate cached toc - self._last_read = None return uniq def remove(self, key): """Remove the keyed message; raise KeyError if it doesn't exist.""" os.remove(os.path.join(self._path, self._lookup(key))) - # Invalidate cached toc (only on success) - self._last_read = None def discard(self, key): """If the keyed message exists, remove it.""" @@ -323,8 +327,6 @@ if isinstance(message, MaildirMessage): os.utime(new_path, (os.path.getatime(new_path), message.get_date())) - # Invalidate cached toc - self._last_read = None def get_message(self, key): """Return a Message representation or raise a KeyError.""" @@ -380,8 +382,8 @@ def flush(self): """Write any pending changes to disk.""" # Maildir changes are always written immediately, so there's nothing - # to do except invalidate our cached toc. - self._last_read = None + # to do. + pass def lock(self): """Lock the mailbox.""" @@ -479,36 +481,39 @@ def _refresh(self): """Update table of contents mapping.""" - if self._last_read is not None: - for subdir in ('new', 'cur'): - mtime = os.path.getmtime(os.path.join(self._path, subdir)) - if mtime > self._last_read: - break - else: + # If it has been less than two seconds since the last _refresh() call, + # we have to unconditionally re-read the mailbox just in case it has + # been modified, because os.path.mtime() has a 2 sec resolution in the + # most common worst case (FAT) and a 1 sec resolution typically. This + # results in a few unnecessary re-reads when _refresh() is called + # multiple times in that interval, but once the clock ticks over, we + # will only re-read as needed. Because the filesystem might be being + # served by an independent system with its own clock, we record and + # compare with the mtimes from the filesystem. Because the other + # system's clock might be skewing relative to our clock, we add an + # extra delta to our wait. The default is one tenth second, but is an + # instance variable and so can be adjusted if dealing with a + # particularly skewed or irregular system. + if time.time() - self._last_read > 2 + self._skewfactor: + refresh = False + for subdir in self._toc_mtimes: + mtime = os.path.getmtime(self._paths[subdir]) + if mtime > self._toc_mtimes[subdir]: + refresh = True + self._toc_mtimes[subdir] = mtime + if not refresh: return - - # We record the current time - 1sec so that, if _refresh() is called - # again in the same second, we will always re-read the mailbox - # just in case it's been modified. (os.path.mtime() only has - # 1sec resolution.) This results in a few unnecessary re-reads - # when _refresh() is called multiple times in the same second, - # but once the clock ticks over, we will only re-read as needed. - now = time.time() - 1 - + # Refresh toc self._toc = {} - def update_dir (subdir): - path = os.path.join(self._path, subdir) + for subdir in self._toc_mtimes: + path = self._paths[subdir] for entry in os.listdir(path): p = os.path.join(path, entry) if os.path.isdir(p): continue uniq = entry.split(self.colon)[0] self._toc[uniq] = os.path.join(subdir, entry) - - update_dir('new') - update_dir('cur') - - self._last_read = now + self._last_read = time.time() def _lookup(self, key): """Use TOC to return subpath for given key, or raise a KeyError.""" @@ -551,7 +556,7 @@ f = open(self._path, 'wb+') else: raise NoSuchMailboxError(self._path) - elif e.errno == errno.EACCES: + elif e.errno in (errno.EACCES, errno.EROFS): f = open(self._path, 'rb') else: raise @@ -700,9 +705,14 @@ def _append_message(self, message): """Append message to mailbox and return (start, stop) offsets.""" self._file.seek(0, 2) - self._pre_message_hook(self._file) - offsets = self._install_message(message) - self._post_message_hook(self._file) + before = self._file.tell() + try: + self._pre_message_hook(self._file) + offsets = self._install_message(message) + self._post_message_hook(self._file) + except BaseException: + self._file.truncate(before) + raise self._file.flush() self._file_length = self._file.tell() # Record current length of mailbox return offsets @@ -868,18 +878,29 @@ new_key = max(keys) + 1 new_path = os.path.join(self._path, str(new_key)) f = _create_carefully(new_path) + closed = False try: if self._locked: _lock_file(f) try: - self._dump_message(message, f) + try: + self._dump_message(message, f) + except BaseException: + # Unlock and close so it can be deleted on Windows + if self._locked: + _unlock_file(f) + _sync_close(f) + closed = True + os.remove(new_path) + raise if isinstance(message, MHMessage): self._dump_sequences(message, new_key) finally: if self._locked: _unlock_file(f) finally: - _sync_close(f) + if not closed: + _sync_close(f) return new_key def remove(self, key): @@ -1886,7 +1907,7 @@ try: fcntl.lockf(f, fcntl.LOCK_EX | fcntl.LOCK_NB) except IOError, e: - if e.errno in (errno.EAGAIN, errno.EACCES): + if e.errno in (errno.EAGAIN, errno.EACCES, errno.EROFS): raise ExternalClashError('lockf: lock unavailable: %s' % f.name) else: @@ -1896,7 +1917,7 @@ pre_lock = _create_temporary(f.name + '.lock') pre_lock.close() except IOError, e: - if e.errno == errno.EACCES: + if e.errno in (errno.EACCES, errno.EROFS): return # Without write access, just skip dotlocking. else: raise diff --git a/lib-python/2.7/msilib/__init__.py b/lib-python/2.7/msilib/__init__.py --- a/lib-python/2.7/msilib/__init__.py +++ b/lib-python/2.7/msilib/__init__.py @@ -173,11 +173,10 @@ add_data(db, table, getattr(module, table)) def make_id(str): - #str = str.replace(".", "_") # colons are allowed - str = str.replace(" ", "_") - str = str.replace("-", "_") - if str[0] in string.digits: - str = "_"+str + identifier_chars = string.ascii_letters + string.digits + "._" + str = "".join([c if c in identifier_chars else "_" for c in str]) + if str[0] in (string.digits + "."): + str = "_" + str assert re.match("^[A-Za-z_][A-Za-z0-9_.]*$", str), "FILE"+str return str @@ -285,19 +284,28 @@ [(feature.id, component)]) def make_short(self, file): + oldfile = file + file = file.replace('+', '_') + file = ''.join(c for c in file if not c in ' "/\[]:;=,') parts = file.split(".") - if len(parts)>1: + if len(parts) > 1: + prefix = "".join(parts[:-1]).upper() suffix = parts[-1].upper() + if not prefix: + prefix = suffix + suffix = None else: + prefix = file.upper() suffix = None - prefix = parts[0].upper() - if len(prefix) <= 8 and (not suffix or len(suffix)<=3): + if len(parts) < 3 and len(prefix) <= 8 and file == oldfile and ( + not suffix or len(suffix) <= 3): if suffix: file = prefix+"."+suffix else: file = prefix - assert file not in self.short_names else: + file = None + if file is None or file in self.short_names: prefix = prefix[:6] if suffix: suffix = suffix[:3] diff --git a/lib-python/2.7/multiprocessing/__init__.py b/lib-python/2.7/multiprocessing/__init__.py --- a/lib-python/2.7/multiprocessing/__init__.py +++ b/lib-python/2.7/multiprocessing/__init__.py @@ -38,6 +38,7 @@ # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. # __version__ = '0.70a1' @@ -115,8 +116,11 @@ except (ValueError, KeyError): num = 0 elif 'bsd' in sys.platform or sys.platform == 'darwin': + comm = '/sbin/sysctl -n hw.ncpu' + if sys.platform == 'darwin': + comm = '/usr' + comm try: - with os.popen('sysctl -n hw.ncpu') as p: + with os.popen(comm) as p: num = int(p.read()) except ValueError: num = 0 diff --git a/lib-python/2.7/multiprocessing/connection.py b/lib-python/2.7/multiprocessing/connection.py --- a/lib-python/2.7/multiprocessing/connection.py +++ b/lib-python/2.7/multiprocessing/connection.py @@ -3,7 +3,33 @@ # # multiprocessing/connection.py # -# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt +# Copyright (c) 2006-2008, R Oudkerk +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of author nor the names of any contributors may be +# used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. # __all__ = [ 'Client', 'Listener', 'Pipe' ] diff --git a/lib-python/2.7/multiprocessing/dummy/__init__.py b/lib-python/2.7/multiprocessing/dummy/__init__.py --- a/lib-python/2.7/multiprocessing/dummy/__init__.py +++ b/lib-python/2.7/multiprocessing/dummy/__init__.py @@ -3,7 +3,33 @@ # # multiprocessing/dummy/__init__.py # -# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt +# Copyright (c) 2006-2008, R Oudkerk +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of author nor the names of any contributors may be +# used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. # __all__ = [ diff --git a/lib-python/2.7/multiprocessing/dummy/connection.py b/lib-python/2.7/multiprocessing/dummy/connection.py --- a/lib-python/2.7/multiprocessing/dummy/connection.py +++ b/lib-python/2.7/multiprocessing/dummy/connection.py @@ -3,7 +3,33 @@ # # multiprocessing/dummy/connection.py # -# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt +# Copyright (c) 2006-2008, R Oudkerk +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of author nor the names of any contributors may be +# used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. # __all__ = [ 'Client', 'Listener', 'Pipe' ] diff --git a/lib-python/2.7/multiprocessing/forking.py b/lib-python/2.7/multiprocessing/forking.py --- a/lib-python/2.7/multiprocessing/forking.py +++ b/lib-python/2.7/multiprocessing/forking.py @@ -3,7 +3,33 @@ # # multiprocessing/forking.py # -# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt +# Copyright (c) 2006-2008, R Oudkerk +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of author nor the names of any contributors may be +# used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. # import os @@ -172,6 +198,7 @@ TERMINATE = 0x10000 WINEXE = (sys.platform == 'win32' and getattr(sys, 'frozen', False)) + WINSERVICE = sys.executable.lower().endswith("pythonservice.exe") exit = win32.ExitProcess close = win32.CloseHandle @@ -181,7 +208,7 @@ # People embedding Python want to modify it. # - if sys.executable.lower().endswith('pythonservice.exe'): + if WINSERVICE: _python_exe = os.path.join(sys.exec_prefix, 'python.exe') else: _python_exe = sys.executable @@ -371,7 +398,7 @@ if _logger is not None: d['log_level'] = _logger.getEffectiveLevel() - if not WINEXE: + if not WINEXE and not WINSERVICE: main_path = getattr(sys.modules['__main__'], '__file__', None) if not main_path and sys.argv[0] not in ('', '-c'): main_path = sys.argv[0] diff --git a/lib-python/2.7/multiprocessing/heap.py b/lib-python/2.7/multiprocessing/heap.py --- a/lib-python/2.7/multiprocessing/heap.py +++ b/lib-python/2.7/multiprocessing/heap.py @@ -3,7 +3,33 @@ # # multiprocessing/heap.py # -# Copyright (c) 2007-2008, R Oudkerk --- see COPYING.txt +# Copyright (c) 2006-2008, R Oudkerk +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of author nor the names of any contributors may be +# used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. # import bisect diff --git a/lib-python/2.7/multiprocessing/managers.py b/lib-python/2.7/multiprocessing/managers.py --- a/lib-python/2.7/multiprocessing/managers.py +++ b/lib-python/2.7/multiprocessing/managers.py @@ -4,7 +4,33 @@ # # multiprocessing/managers.py # -# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt +# Copyright (c) 2006-2008, R Oudkerk +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of author nor the names of any contributors may be +# used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. # __all__ = [ 'BaseManager', 'SyncManager', 'BaseProxy', 'Token' ] diff --git a/lib-python/2.7/multiprocessing/pool.py b/lib-python/2.7/multiprocessing/pool.py --- a/lib-python/2.7/multiprocessing/pool.py +++ b/lib-python/2.7/multiprocessing/pool.py @@ -3,7 +3,33 @@ # # multiprocessing/pool.py # -# Copyright (c) 2007-2008, R Oudkerk --- see COPYING.txt +# Copyright (c) 2006-2008, R Oudkerk +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of author nor the names of any contributors may be +# used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. # __all__ = ['Pool'] @@ -269,6 +295,8 @@ while pool._worker_handler._state == RUN and pool._state == RUN: pool._maintain_pool() time.sleep(0.1) + # send sentinel to stop workers + pool._taskqueue.put(None) debug('worker handler exiting') @staticmethod @@ -387,7 +415,6 @@ if self._state == RUN: self._state = CLOSE self._worker_handler._state = CLOSE - self._taskqueue.put(None) def terminate(self): debug('terminating pool') @@ -421,7 +448,6 @@ worker_handler._state = TERMINATE task_handler._state = TERMINATE - taskqueue.put(None) # sentinel debug('helping task handler/workers to finish') cls._help_stuff_finish(inqueue, task_handler, len(pool)) @@ -431,6 +457,11 @@ result_handler._state = TERMINATE outqueue.put(None) # sentinel + # We must wait for the worker handler to exit before terminating + # workers because we don't want workers to be restarted behind our back. + debug('joining worker handler') + worker_handler.join() + # Terminate workers which haven't already finished. if pool and hasattr(pool[0], 'terminate'): debug('terminating workers') diff --git a/lib-python/2.7/multiprocessing/process.py b/lib-python/2.7/multiprocessing/process.py --- a/lib-python/2.7/multiprocessing/process.py +++ b/lib-python/2.7/multiprocessing/process.py @@ -3,7 +3,33 @@ # # multiprocessing/process.py # -# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt +# Copyright (c) 2006-2008, R Oudkerk +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of author nor the names of any contributors may be +# used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. # __all__ = ['Process', 'current_process', 'active_children'] diff --git a/lib-python/2.7/multiprocessing/queues.py b/lib-python/2.7/multiprocessing/queues.py --- a/lib-python/2.7/multiprocessing/queues.py +++ b/lib-python/2.7/multiprocessing/queues.py @@ -3,7 +3,33 @@ # # multiprocessing/queues.py # -# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt +# Copyright (c) 2006-2008, R Oudkerk +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of author nor the names of any contributors may be +# used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. # __all__ = ['Queue', 'SimpleQueue', 'JoinableQueue'] diff --git a/lib-python/2.7/multiprocessing/reduction.py b/lib-python/2.7/multiprocessing/reduction.py --- a/lib-python/2.7/multiprocessing/reduction.py +++ b/lib-python/2.7/multiprocessing/reduction.py @@ -4,7 +4,33 @@ # # multiprocessing/reduction.py # -# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt +# Copyright (c) 2006-2008, R Oudkerk +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of author nor the names of any contributors may be +# used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. # __all__ = [] diff --git a/lib-python/2.7/multiprocessing/sharedctypes.py b/lib-python/2.7/multiprocessing/sharedctypes.py --- a/lib-python/2.7/multiprocessing/sharedctypes.py +++ b/lib-python/2.7/multiprocessing/sharedctypes.py @@ -3,7 +3,33 @@ # # multiprocessing/sharedctypes.py # -# Copyright (c) 2007-2008, R Oudkerk --- see COPYING.txt +# Copyright (c) 2006-2008, R Oudkerk +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of author nor the names of any contributors may be +# used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. # import sys @@ -52,9 +78,11 @@ Returns a ctypes array allocated from shared memory ''' type_ = typecode_to_type.get(typecode_or_type, typecode_or_type) - if isinstance(size_or_initializer, int): + if isinstance(size_or_initializer, (int, long)): type_ = type_ * size_or_initializer - return _new_value(type_) + obj = _new_value(type_) + ctypes.memset(ctypes.addressof(obj), 0, ctypes.sizeof(obj)) + return obj else: type_ = type_ * len(size_or_initializer) result = _new_value(type_) diff --git a/lib-python/2.7/multiprocessing/synchronize.py b/lib-python/2.7/multiprocessing/synchronize.py --- a/lib-python/2.7/multiprocessing/synchronize.py +++ b/lib-python/2.7/multiprocessing/synchronize.py @@ -3,7 +3,33 @@ # # multiprocessing/synchronize.py # -# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt +# Copyright (c) 2006-2008, R Oudkerk +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of author nor the names of any contributors may be +# used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. # __all__ = [ diff --git a/lib-python/2.7/multiprocessing/util.py b/lib-python/2.7/multiprocessing/util.py --- a/lib-python/2.7/multiprocessing/util.py +++ b/lib-python/2.7/multiprocessing/util.py @@ -3,7 +3,33 @@ # # multiprocessing/util.py # -# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt +# Copyright (c) 2006-2008, R Oudkerk +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of author nor the names of any contributors may be +# used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. # import itertools diff --git a/lib-python/2.7/netrc.py b/lib-python/2.7/netrc.py --- a/lib-python/2.7/netrc.py +++ b/lib-python/2.7/netrc.py @@ -34,11 +34,19 @@ def _parse(self, file, fp): lexer = shlex.shlex(fp) lexer.wordchars += r"""!"#$%&'()*+,-./:;<=>?@[\]^_`{|}~""" + lexer.commenters = lexer.commenters.replace('#', '') while 1: # Look for a machine, default, or macdef top-level keyword toplevel = tt = lexer.get_token() if not tt: break + elif tt[0] == '#': + # seek to beginning of comment, in case reading the token put + # us on a new line, and then skip the rest of the line. + pos = len(tt) + 1 + lexer.instream.seek(-pos, 1) + lexer.instream.readline() + continue elif tt == 'machine': entryname = lexer.get_token() elif tt == 'default': @@ -64,8 +72,8 @@ self.hosts[entryname] = {} while 1: tt = lexer.get_token() - if (tt=='' or tt == 'machine' or - tt == 'default' or tt =='macdef'): + if (tt.startswith('#') or + tt in {'', 'machine', 'default', 'macdef'}): if password: self.hosts[entryname] = (login, account, password) lexer.push_token(tt) diff --git a/lib-python/2.7/nntplib.py b/lib-python/2.7/nntplib.py --- a/lib-python/2.7/nntplib.py +++ b/lib-python/2.7/nntplib.py @@ -103,7 +103,7 @@ readermode is sometimes necessary if you are connecting to an NNTP server on the local machine and intend to call - reader-specific comamnds, such as `group'. If you get + reader-specific commands, such as `group'. If you get unexpected NNTPPermanentErrors, you might need to set readermode. """ diff --git a/lib-python/2.7/ntpath.py b/lib-python/2.7/ntpath.py --- a/lib-python/2.7/ntpath.py +++ b/lib-python/2.7/ntpath.py @@ -310,7 +310,7 @@ # - $varname is accepted. # - %varname% is accepted. # - varnames can be made out of letters, digits and the characters '_-' -# (though is not verifed in the ${varname} and %varname% cases) +# (though is not verified in the ${varname} and %varname% cases) # XXX With COMMAND.COM you can use any characters in a variable name, # XXX except '^|<>='. diff --git a/lib-python/2.7/nturl2path.py b/lib-python/2.7/nturl2path.py --- a/lib-python/2.7/nturl2path.py +++ b/lib-python/2.7/nturl2path.py @@ -25,11 +25,14 @@ error = 'Bad URL: ' + url raise IOError, error drive = comp[0][-1].upper() + path = drive + ':' components = comp[1].split('/') - path = drive + ':' - for comp in components: + for comp in components: if comp: path = path + '\\' + urllib.unquote(comp) + # Issue #11474: url like '/C|/' should convert into 'C:\\' + if path.endswith(':') and url.endswith('/'): + path += '\\' return path def pathname2url(p): diff --git a/lib-python/2.7/numbers.py b/lib-python/2.7/numbers.py --- a/lib-python/2.7/numbers.py +++ b/lib-python/2.7/numbers.py @@ -63,7 +63,7 @@ @abstractproperty def imag(self): - """Retrieve the real component of this number. + """Retrieve the imaginary component of this number. This should subclass Real. """ diff --git a/lib-python/2.7/optparse.py b/lib-python/2.7/optparse.py --- a/lib-python/2.7/optparse.py +++ b/lib-python/2.7/optparse.py @@ -1131,6 +1131,11 @@ prog : string the name of the current program (to override os.path.basename(sys.argv[0])). + description : string + A paragraph of text giving a brief overview of your program. + optparse reformats this paragraph to fit the current terminal + width and prints it when the user requests help (after usage, + but before the list of options). epilog : string paragraph of help text to print after option help diff --git a/lib-python/2.7/pickletools.py b/lib-python/2.7/pickletools.py --- a/lib-python/2.7/pickletools.py +++ b/lib-python/2.7/pickletools.py @@ -1370,7 +1370,7 @@ proto=0, doc="""Read an object from the memo and push it on the stack. - The index of the memo object to push is given by the newline-teriminated + The index of the memo object to push is given by the newline-terminated decimal string following. BINGET and LONG_BINGET are space-optimized versions. """), diff --git a/lib-python/2.7/pkgutil.py b/lib-python/2.7/pkgutil.py --- a/lib-python/2.7/pkgutil.py +++ b/lib-python/2.7/pkgutil.py @@ -11,7 +11,7 @@ __all__ = [ 'get_importer', 'iter_importers', 'get_loader', 'find_loader', - 'walk_packages', 'iter_modules', + 'walk_packages', 'iter_modules', 'get_data', 'ImpImporter', 'ImpLoader', 'read_code', 'extend_path', ] diff --git a/lib-python/2.7/platform.py b/lib-python/2.7/platform.py --- a/lib-python/2.7/platform.py +++ b/lib-python/2.7/platform.py @@ -503,7 +503,7 @@ info = pipe.read() if pipe.close(): raise os.error,'command failed' - # XXX How can I supress shell errors from being written + # XXX How can I suppress shell errors from being written # to stderr ? except os.error,why: #print 'Command %s failed: %s' % (cmd,why) @@ -1448,9 +1448,10 @@ """ Returns a string identifying the Python implementation. Currently, the following implementations are identified: - 'CPython' (C implementation of Python), - 'IronPython' (.NET implementation of Python), - 'Jython' (Java implementation of Python). + 'CPython' (C implementation of Python), + 'IronPython' (.NET implementation of Python), + 'Jython' (Java implementation of Python), + 'PyPy' (Python implementation of Python). """ return _sys_version()[0] diff --git a/lib-python/2.7/pydoc.py b/lib-python/2.7/pydoc.py --- a/lib-python/2.7/pydoc.py +++ b/lib-python/2.7/pydoc.py @@ -156,7 +156,7 @@ no.append(x) return yes, no -def visiblename(name, all=None): +def visiblename(name, all=None, obj=None): """Decide whether to show documentation on a variable.""" # Certain special names are redundant. _hidden_names = ('__builtins__', '__doc__', '__file__', '__path__', @@ -164,6 +164,9 @@ if name in _hidden_names: return 0 # Private names are hidden, but special names are displayed. if name.startswith('__') and name.endswith('__'): return 1 + # Namedtuples have public fields and methods with a single leading underscore + if name.startswith('_') and hasattr(obj, '_fields'): + return 1 if all is not None: # only document that which the programmer exported in __all__ return name in all @@ -475,9 +478,9 @@ def multicolumn(self, list, format, cols=4): """Format a list of items into a multi-column list.""" result = '' - rows = (len(list)+cols-1)/cols + rows = (len(list)+cols-1)//cols for col in range(cols): - result = result + '<td width="%d%%" valign=top>' % (100/cols) + result = result + '<td width="%d%%" valign=top>' % (100//cols) for i in range(rows*col, rows*col+rows): if i < len(list): result = result + format(list[i]) + '<br>\n' @@ -627,7 +630,7 @@ # if __all__ exists, believe it. Otherwise use old heuristic. if (all is not None or (inspect.getmodule(value) or object) is object): - if visiblename(key, all): + if visiblename(key, all, object): classes.append((key, value)) cdict[key] = cdict[value] = '#' + key for key, value in classes: @@ -643,13 +646,13 @@ # if __all__ exists, believe it. Otherwise use old heuristic. if (all is not None or inspect.isbuiltin(value) or inspect.getmodule(value) is object): - if visiblename(key, all): + if visiblename(key, all, object): funcs.append((key, value)) fdict[key] = '#-' + key if inspect.isfunction(value): fdict[value] = fdict[key] data = [] for key, value in inspect.getmembers(object, isdata): - if visiblename(key, all): + if visiblename(key, all, object): data.append((key, value)) doc = self.markup(getdoc(object), self.preformat, fdict, cdict) @@ -773,7 +776,7 @@ push('\n') return attrs - attrs = filter(lambda data: visiblename(data[0]), + attrs = filter(lambda data: visiblename(data[0], obj=object), classify_class_attrs(object)) mdict = {} for key, kind, homecls, value in attrs: @@ -1042,18 +1045,18 @@ # if __all__ exists, believe it. Otherwise use old heuristic. if (all is not None or (inspect.getmodule(value) or object) is object): - if visiblename(key, all): + if visiblename(key, all, object): classes.append((key, value)) funcs = [] for key, value in inspect.getmembers(object, inspect.isroutine): # if __all__ exists, believe it. Otherwise use old heuristic. if (all is not None or inspect.isbuiltin(value) or inspect.getmodule(value) is object): - if visiblename(key, all): + if visiblename(key, all, object): funcs.append((key, value)) data = [] for key, value in inspect.getmembers(object, isdata): - if visiblename(key, all): + if visiblename(key, all, object): data.append((key, value)) modpkgs = [] @@ -1113,7 +1116,7 @@ result = result + self.section('CREDITS', str(object.__credits__)) return result - def docclass(self, object, name=None, mod=None): + def docclass(self, object, name=None, mod=None, *ignored): """Produce text documentation for a given class object.""" realname = object.__name__ name = name or realname @@ -1186,7 +1189,7 @@ name, mod, maxlen=70, doc=doc) + '\n') return attrs - attrs = filter(lambda data: visiblename(data[0]), + attrs = filter(lambda data: visiblename(data[0], obj=object), classify_class_attrs(object)) while attrs: if mro: @@ -1718,8 +1721,9 @@ return '' return '<pydoc.Helper instance>' - def __call__(self, request=None): - if request is not None: + _GoInteractive = object() + def __call__(self, request=_GoInteractive): + if request is not self._GoInteractive: self.help(request) else: self.intro() diff --git a/lib-python/2.7/pydoc_data/topics.py b/lib-python/2.7/pydoc_data/topics.py --- a/lib-python/2.7/pydoc_data/topics.py +++ b/lib-python/2.7/pydoc_data/topics.py @@ -1,16 +1,16 @@ -# Autogenerated by Sphinx on Sat Jul 3 08:52:04 2010 +# Autogenerated by Sphinx on Sat Jun 11 09:49:30 2011 topics = {'assert': u'\nThe ``assert`` statement\n************************\n\nAssert statements are a convenient way to insert debugging assertions\ninto a program:\n\n assert_stmt ::= "assert" expression ["," expression]\n\nThe simple form, ``assert expression``, is equivalent to\n\n if __debug__:\n if not expression: raise AssertionError\n\nThe extended form, ``assert expression1, expression2``, is equivalent\nto\n\n if __debug__:\n if not expression1: raise AssertionError(expression2)\n\nThese equivalences assume that ``__debug__`` and ``AssertionError``\nrefer to the built-in variables with those names. In the current\nimplementation, the built-in variable ``__debug__`` is ``True`` under\nnormal circumstances, ``False`` when optimization is requested\n(command line option -O). The current code generator emits no code\nfor an assert statement when optimization is requested at compile\ntime. Note that it is unnecessary to include the source code for the\nexpression that failed in the error message; it will be displayed as\npart of the stack trace.\n\nAssignments to ``__debug__`` are illegal. The value for the built-in\nvariable is determined when the interpreter starts.\n', - 'assignment': u'\nAssignment statements\n*********************\n\nAssignment statements are used to (re)bind names to values and to\nmodify attributes or items of mutable objects:\n\n assignment_stmt ::= (target_list "=")+ (expression_list | yield_expression)\n target_list ::= target ("," target)* [","]\n target ::= identifier\n | "(" target_list ")"\n | "[" target_list "]"\n | attributeref\n | subscription\n | slicing\n\n(See section *Primaries* for the syntax definitions for the last three\nsymbols.)\n\nAn assignment statement evaluates the expression list (remember that\nthis can be a single expression or a comma-separated list, the latter\nyielding a tuple) and assigns the single resulting object to each of\nthe target lists, from left to right.\n\nAssignment is defined recursively depending on the form of the target\n(list). When a target is part of a mutable object (an attribute\nreference, subscription or slicing), the mutable object must\nultimately perform the assignment and decide about its validity, and\nmay raise an exception if the assignment is unacceptable. The rules\nobserved by various types and the exceptions raised are given with the\ndefinition of the object types (see section *The standard type\nhierarchy*).\n\nAssignment of an object to a target list is recursively defined as\nfollows.\n\n* If the target list is a single target: The object is assigned to\n that target.\n\n* If the target list is a comma-separated list of targets: The object\n must be an iterable with the same number of items as there are\n targets in the target list, and the items are assigned, from left to\n right, to the corresponding targets. (This rule is relaxed as of\n Python 1.5; in earlier versions, the object had to be a tuple.\n Since strings are sequences, an assignment like ``a, b = "xy"`` is\n now legal as long as the string has the right length.)\n\nAssignment of an object to a single target is recursively defined as\nfollows.\n\n* If the target is an identifier (name):\n\n * If the name does not occur in a ``global`` statement in the\n current code block: the name is bound to the object in the current\n local namespace.\n\n * Otherwise: the name is bound to the object in the current global\n namespace.\n\n The name is rebound if it was already bound. This may cause the\n reference count for the object previously bound to the name to reach\n zero, causing the object to be deallocated and its destructor (if it\n has one) to be called.\n\n* If the target is a target list enclosed in parentheses or in square\n brackets: The object must be an iterable with the same number of\n items as there are targets in the target list, and its items are\n assigned, from left to right, to the corresponding targets.\n\n* If the target is an attribute reference: The primary expression in\n the reference is evaluated. It should yield an object with\n assignable attributes; if this is not the case, ``TypeError`` is\n raised. That object is then asked to assign the assigned object to\n the given attribute; if it cannot perform the assignment, it raises\n an exception (usually but not necessarily ``AttributeError``).\n\n Note: If the object is a class instance and the attribute reference\n occurs on both sides of the assignment operator, the RHS expression,\n ``a.x`` can access either an instance attribute or (if no instance\n attribute exists) a class attribute. The LHS target ``a.x`` is\n always set as an instance attribute, creating it if necessary.\n Thus, the two occurrences of ``a.x`` do not necessarily refer to the\n same attribute: if the RHS expression refers to a class attribute,\n the LHS creates a new instance attribute as the target of the\n assignment:\n\n class Cls:\n x = 3 # class variable\n inst = Cls()\n inst.x = inst.x + 1 # writes inst.x as 4 leaving Cls.x as 3\n\n This description does not necessarily apply to descriptor\n attributes, such as properties created with ``property()``.\n\n* If the target is a subscription: The primary expression in the\n reference is evaluated. It should yield either a mutable sequence\n object (such as a list) or a mapping object (such as a dictionary).\n Next, the subscript expression is evaluated.\n\n If the primary is a mutable sequence object (such as a list), the\n subscript must yield a plain integer. If it is negative, the\n sequence\'s length is added to it. The resulting value must be a\n nonnegative integer less than the sequence\'s length, and the\n sequence is asked to assign the assigned object to its item with\n that index. If the index is out of range, ``IndexError`` is raised\n (assignment to a subscripted sequence cannot add new items to a\n list).\n\n If the primary is a mapping object (such as a dictionary), the\n subscript must have a type compatible with the mapping\'s key type,\n and the mapping is then asked to create a key/datum pair which maps\n the subscript to the assigned object. This can either replace an\n existing key/value pair with the same key value, or insert a new\n key/value pair (if no key with the same value existed).\n\n* If the target is a slicing: The primary expression in the reference\n is evaluated. It should yield a mutable sequence object (such as a\n list). The assigned object should be a sequence object of the same\n type. Next, the lower and upper bound expressions are evaluated,\n insofar they are present; defaults are zero and the sequence\'s\n length. The bounds should evaluate to (small) integers. If either\n bound is negative, the sequence\'s length is added to it. The\n resulting bounds are clipped to lie between zero and the sequence\'s\n length, inclusive. Finally, the sequence object is asked to replace\n the slice with the items of the assigned sequence. The length of\n the slice may be different from the length of the assigned sequence,\n thus changing the length of the target sequence, if the object\n allows it.\n\n**CPython implementation detail:** In the current implementation, the\nsyntax for targets is taken to be the same as for expressions, and\ninvalid syntax is rejected during the code generation phase, causing\nless detailed error messages.\n\nWARNING: Although the definition of assignment implies that overlaps\nbetween the left-hand side and the right-hand side are \'safe\' (for\nexample ``a, b = b, a`` swaps two variables), overlaps *within* the\ncollection of assigned-to variables are not safe! For instance, the\nfollowing program prints ``[0, 2]``:\n\n x = [0, 1]\n i = 0\n i, x[i] = 1, 2\n print x\n\n\nAugmented assignment statements\n===============================\n\nAugmented assignment is the combination, in a single statement, of a\nbinary operation and an assignment statement:\n\n augmented_assignment_stmt ::= augtarget augop (expression_list | yield_expression)\n augtarget ::= identifier | attributeref | subscription | slicing\n augop ::= "+=" | "-=" | "*=" | "/=" | "//=" | "%=" | "**="\n | ">>=" | "<<=" | "&=" | "^=" | "|="\n\n(See section *Primaries* for the syntax definitions for the last three\nsymbols.)\n\nAn augmented assignment evaluates the target (which, unlike normal\nassignment statements, cannot be an unpacking) and the expression\nlist, performs the binary operation specific to the type of assignment\non the two operands, and assigns the result to the original target.\nThe target is only evaluated once.\n\nAn augmented assignment expression like ``x += 1`` can be rewritten as\n``x = x + 1`` to achieve a similar, but not exactly equal effect. In\nthe augmented version, ``x`` is only evaluated once. Also, when\npossible, the actual operation is performed *in-place*, meaning that\nrather than creating a new object and assigning that to the target,\nthe old object is modified instead.\n\nWith the exception of assigning to tuples and multiple targets in a\nsingle statement, the assignment done by augmented assignment\nstatements is handled the same way as normal assignments. Similarly,\nwith the exception of the possible *in-place* behavior, the binary\noperation performed by augmented assignment is the same as the normal\nbinary operations.\n\nFor targets which are attribute references, the same *caveat about\nclass and instance attributes* applies as for regular assignments.\n', + 'assignment': u'\nAssignment statements\n*********************\n\nAssignment statements are used to (re)bind names to values and to\nmodify attributes or items of mutable objects:\n\n assignment_stmt ::= (target_list "=")+ (expression_list | yield_expression)\n target_list ::= target ("," target)* [","]\n target ::= identifier\n | "(" target_list ")"\n | "[" target_list "]"\n | attributeref\n | subscription\n | slicing\n\n(See section *Primaries* for the syntax definitions for the last three\nsymbols.)\n\nAn assignment statement evaluates the expression list (remember that\nthis can be a single expression or a comma-separated list, the latter\nyielding a tuple) and assigns the single resulting object to each of\nthe target lists, from left to right.\n\nAssignment is defined recursively depending on the form of the target\n(list). When a target is part of a mutable object (an attribute\nreference, subscription or slicing), the mutable object must\nultimately perform the assignment and decide about its validity, and\nmay raise an exception if the assignment is unacceptable. The rules\nobserved by various types and the exceptions raised are given with the\ndefinition of the object types (see section *The standard type\nhierarchy*).\n\nAssignment of an object to a target list is recursively defined as\nfollows.\n\n* If the target list is a single target: The object is assigned to\n that target.\n\n* If the target list is a comma-separated list of targets: The object\n must be an iterable with the same number of items as there are\n targets in the target list, and the items are assigned, from left to\n right, to the corresponding targets.\n\nAssignment of an object to a single target is recursively defined as\nfollows.\n\n* If the target is an identifier (name):\n\n * If the name does not occur in a ``global`` statement in the\n current code block: the name is bound to the object in the current\n local namespace.\n\n * Otherwise: the name is bound to the object in the current global\n namespace.\n\n The name is rebound if it was already bound. This may cause the\n reference count for the object previously bound to the name to reach\n zero, causing the object to be deallocated and its destructor (if it\n has one) to be called.\n\n* If the target is a target list enclosed in parentheses or in square\n brackets: The object must be an iterable with the same number of\n items as there are targets in the target list, and its items are\n assigned, from left to right, to the corresponding targets.\n\n* If the target is an attribute reference: The primary expression in\n the reference is evaluated. It should yield an object with\n assignable attributes; if this is not the case, ``TypeError`` is\n raised. That object is then asked to assign the assigned object to\n the given attribute; if it cannot perform the assignment, it raises\n an exception (usually but not necessarily ``AttributeError``).\n\n Note: If the object is a class instance and the attribute reference\n occurs on both sides of the assignment operator, the RHS expression,\n ``a.x`` can access either an instance attribute or (if no instance\n attribute exists) a class attribute. The LHS target ``a.x`` is\n always set as an instance attribute, creating it if necessary.\n Thus, the two occurrences of ``a.x`` do not necessarily refer to the\n same attribute: if the RHS expression refers to a class attribute,\n the LHS creates a new instance attribute as the target of the\n assignment:\n\n class Cls:\n x = 3 # class variable\n inst = Cls()\n inst.x = inst.x + 1 # writes inst.x as 4 leaving Cls.x as 3\n\n This description does not necessarily apply to descriptor\n attributes, such as properties created with ``property()``.\n\n* If the target is a subscription: The primary expression in the\n reference is evaluated. It should yield either a mutable sequence\n object (such as a list) or a mapping object (such as a dictionary).\n Next, the subscript expression is evaluated.\n\n If the primary is a mutable sequence object (such as a list), the\n subscript must yield a plain integer. If it is negative, the\n sequence\'s length is added to it. The resulting value must be a\n nonnegative integer less than the sequence\'s length, and the\n sequence is asked to assign the assigned object to its item with\n that index. If the index is out of range, ``IndexError`` is raised\n (assignment to a subscripted sequence cannot add new items to a\n list).\n\n If the primary is a mapping object (such as a dictionary), the\n subscript must have a type compatible with the mapping\'s key type,\n and the mapping is then asked to create a key/datum pair which maps\n the subscript to the assigned object. This can either replace an\n existing key/value pair with the same key value, or insert a new\n key/value pair (if no key with the same value existed).\n\n* If the target is a slicing: The primary expression in the reference\n is evaluated. It should yield a mutable sequence object (such as a\n list). The assigned object should be a sequence object of the same\n type. Next, the lower and upper bound expressions are evaluated,\n insofar they are present; defaults are zero and the sequence\'s\n length. The bounds should evaluate to (small) integers. If either\n bound is negative, the sequence\'s length is added to it. The\n resulting bounds are clipped to lie between zero and the sequence\'s\n length, inclusive. Finally, the sequence object is asked to replace\n the slice with the items of the assigned sequence. The length of\n the slice may be different from the length of the assigned sequence,\n thus changing the length of the target sequence, if the object\n allows it.\n\n**CPython implementation detail:** In the current implementation, the\nsyntax for targets is taken to be the same as for expressions, and\ninvalid syntax is rejected during the code generation phase, causing\nless detailed error messages.\n\nWARNING: Although the definition of assignment implies that overlaps\nbetween the left-hand side and the right-hand side are \'safe\' (for\nexample ``a, b = b, a`` swaps two variables), overlaps *within* the\ncollection of assigned-to variables are not safe! For instance, the\nfollowing program prints ``[0, 2]``:\n\n x = [0, 1]\n i = 0\n i, x[i] = 1, 2\n print x\n\n\nAugmented assignment statements\n===============================\n\nAugmented assignment is the combination, in a single statement, of a\nbinary operation and an assignment statement:\n\n augmented_assignment_stmt ::= augtarget augop (expression_list | yield_expression)\n augtarget ::= identifier | attributeref | subscription | slicing\n augop ::= "+=" | "-=" | "*=" | "/=" | "//=" | "%=" | "**="\n | ">>=" | "<<=" | "&=" | "^=" | "|="\n\n(See section *Primaries* for the syntax definitions for the last three\nsymbols.)\n\nAn augmented assignment evaluates the target (which, unlike normal\nassignment statements, cannot be an unpacking) and the expression\nlist, performs the binary operation specific to the type of assignment\non the two operands, and assigns the result to the original target.\nThe target is only evaluated once.\n\nAn augmented assignment expression like ``x += 1`` can be rewritten as\n``x = x + 1`` to achieve a similar, but not exactly equal effect. In\nthe augmented version, ``x`` is only evaluated once. Also, when\npossible, the actual operation is performed *in-place*, meaning that\nrather than creating a new object and assigning that to the target,\nthe old object is modified instead.\n\nWith the exception of assigning to tuples and multiple targets in a\nsingle statement, the assignment done by augmented assignment\nstatements is handled the same way as normal assignments. Similarly,\nwith the exception of the possible *in-place* behavior, the binary\noperation performed by augmented assignment is the same as the normal\nbinary operations.\n\nFor targets which are attribute references, the same *caveat about\nclass and instance attributes* applies as for regular assignments.\n', 'atom-identifiers': u'\nIdentifiers (Names)\n*******************\n\nAn identifier occurring as an atom is a name. See section\n*Identifiers and keywords* for lexical definition and section *Naming\nand binding* for documentation of naming and binding.\n\nWhen the name is bound to an object, evaluation of the atom yields\nthat object. When a name is not bound, an attempt to evaluate it\nraises a ``NameError`` exception.\n\n**Private name mangling:** When an identifier that textually occurs in\na class definition begins with two or more underscore characters and\ndoes not end in two or more underscores, it is considered a *private\nname* of that class. Private names are transformed to a longer form\nbefore code is generated for them. The transformation inserts the\nclass name in front of the name, with leading underscores removed, and\na single underscore inserted in front of the class name. For example,\nthe identifier ``__spam`` occurring in a class named ``Ham`` will be\ntransformed to ``_Ham__spam``. This transformation is independent of\nthe syntactical context in which the identifier is used. If the\ntransformed name is extremely long (longer than 255 characters),\nimplementation defined truncation may happen. If the class name\nconsists only of underscores, no transformation is done.\n', 'atom-literals': u"\nLiterals\n********\n\nPython supports string literals and various numeric literals:\n\n literal ::= stringliteral | integer | longinteger\n | floatnumber | imagnumber\n\nEvaluation of a literal yields an object of the given type (string,\ninteger, long integer, floating point number, complex number) with the\ngiven value. The value may be approximated in the case of floating\npoint and imaginary (complex) literals. See section *Literals* for\ndetails.\n\nAll literals correspond to immutable data types, and hence the\nobject's identity is less important than its value. Multiple\nevaluations of literals with the same value (either the same\noccurrence in the program text or a different occurrence) may obtain\nthe same object or a different object with the same value.\n", - 'attribute-access': u'\nCustomizing attribute access\n****************************\n\nThe following methods can be defined to customize the meaning of\nattribute access (use of, assignment to, or deletion of ``x.name``)\nfor class instances.\n\nobject.__getattr__(self, name)\n\n Called when an attribute lookup has not found the attribute in the\n usual places (i.e. it is not an instance attribute nor is it found\n in the class tree for ``self``). ``name`` is the attribute name.\n This method should return the (computed) attribute value or raise\n an ``AttributeError`` exception.\n\n Note that if the attribute is found through the normal mechanism,\n ``__getattr__()`` is not called. (This is an intentional asymmetry\n between ``__getattr__()`` and ``__setattr__()``.) This is done both\n for efficiency reasons and because otherwise ``__getattr__()``\n would have no way to access other attributes of the instance. Note\n that at least for instance variables, you can fake total control by\n not inserting any values in the instance attribute dictionary (but\n instead inserting them in another object). See the\n ``__getattribute__()`` method below for a way to actually get total\n control in new-style classes.\n\nobject.__setattr__(self, name, value)\n\n Called when an attribute assignment is attempted. This is called\n instead of the normal mechanism (i.e. store the value in the\n instance dictionary). *name* is the attribute name, *value* is the\n value to be assigned to it.\n\n If ``__setattr__()`` wants to assign to an instance attribute, it\n should not simply execute ``self.name = value`` --- this would\n cause a recursive call to itself. Instead, it should insert the\n value in the dictionary of instance attributes, e.g.,\n ``self.__dict__[name] = value``. For new-style classes, rather\n than accessing the instance dictionary, it should call the base\n class method with the same name, for example,\n ``object.__setattr__(self, name, value)``.\n\nobject.__delattr__(self, name)\n\n Like ``__setattr__()`` but for attribute deletion instead of\n assignment. This should only be implemented if ``del obj.name`` is\n meaningful for the object.\n\n\nMore attribute access for new-style classes\n===========================================\n\nThe following methods only apply to new-style classes.\n\nobject.__getattribute__(self, name)\n\n Called unconditionally to implement attribute accesses for\n instances of the class. If the class also defines\n ``__getattr__()``, the latter will not be called unless\n ``__getattribute__()`` either calls it explicitly or raises an\n ``AttributeError``. This method should return the (computed)\n attribute value or raise an ``AttributeError`` exception. In order\n to avoid infinite recursion in this method, its implementation\n should always call the base class method with the same name to\n access any attributes it needs, for example,\n ``object.__getattribute__(self, name)``.\n\n Note: This method may still be bypassed when looking up special methods\n as the result of implicit invocation via language syntax or\n built-in functions. See *Special method lookup for new-style\n classes*.\n\n\nImplementing Descriptors\n========================\n\nThe following methods only apply when an instance of the class\ncontaining the method (a so-called *descriptor* class) appears in the\nclass dictionary of another new-style class, known as the *owner*\nclass. In the examples below, "the attribute" refers to the attribute\nwhose name is the key of the property in the owner class\'\n``__dict__``. Descriptors can only be implemented as new-style\nclasses themselves.\n\nobject.__get__(self, instance, owner)\n\n Called to get the attribute of the owner class (class attribute\n access) or of an instance of that class (instance attribute\n access). *owner* is always the owner class, while *instance* is the\n instance that the attribute was accessed through, or ``None`` when\n the attribute is accessed through the *owner*. This method should\n return the (computed) attribute value or raise an\n ``AttributeError`` exception.\n\nobject.__set__(self, instance, value)\n\n Called to set the attribute on an instance *instance* of the owner\n class to a new value, *value*.\n\nobject.__delete__(self, instance)\n\n Called to delete the attribute on an instance *instance* of the\n owner class.\n\n\nInvoking Descriptors\n====================\n\nIn general, a descriptor is an object attribute with "binding\nbehavior", one whose attribute access has been overridden by methods\nin the descriptor protocol: ``__get__()``, ``__set__()``, and\n``__delete__()``. If any of those methods are defined for an object,\nit is said to be a descriptor.\n\nThe default behavior for attribute access is to get, set, or delete\nthe attribute from an object\'s dictionary. For instance, ``a.x`` has a\nlookup chain starting with ``a.__dict__[\'x\']``, then\n``type(a).__dict__[\'x\']``, and continuing through the base classes of\n``type(a)`` excluding metaclasses.\n\nHowever, if the looked-up value is an object defining one of the\ndescriptor methods, then Python may override the default behavior and\ninvoke the descriptor method instead. Where this occurs in the\nprecedence chain depends on which descriptor methods were defined and\nhow they were called. Note that descriptors are only invoked for new\nstyle objects or classes (ones that subclass ``object()`` or\n``type()``).\n\nThe starting point for descriptor invocation is a binding, ``a.x``.\nHow the arguments are assembled depends on ``a``:\n\nDirect Call\n The simplest and least common call is when user code directly\n invokes a descriptor method: ``x.__get__(a)``.\n\nInstance Binding\n If binding to a new-style object instance, ``a.x`` is transformed\n into the call: ``type(a).__dict__[\'x\'].__get__(a, type(a))``.\n\nClass Binding\n If binding to a new-style class, ``A.x`` is transformed into the\n call: ``A.__dict__[\'x\'].__get__(None, A)``.\n\nSuper Binding\n If ``a`` is an instance of ``super``, then the binding ``super(B,\n obj).m()`` searches ``obj.__class__.__mro__`` for the base class\n ``A`` immediately preceding ``B`` and then invokes the descriptor\n with the call: ``A.__dict__[\'m\'].__get__(obj, A)``.\n\nFor instance bindings, the precedence of descriptor invocation depends\non the which descriptor methods are defined. A descriptor can define\nany combination of ``__get__()``, ``__set__()`` and ``__delete__()``.\nIf it does not define ``__get__()``, then accessing the attribute will\nreturn the descriptor object itself unless there is a value in the\nobject\'s instance dictionary. If the descriptor defines ``__set__()``\nand/or ``__delete__()``, it is a data descriptor; if it defines\nneither, it is a non-data descriptor. Normally, data descriptors\ndefine both ``__get__()`` and ``__set__()``, while non-data\ndescriptors have just the ``__get__()`` method. Data descriptors with\n``__set__()`` and ``__get__()`` defined always override a redefinition\nin an instance dictionary. In contrast, non-data descriptors can be\noverridden by instances.\n\nPython methods (including ``staticmethod()`` and ``classmethod()``)\nare implemented as non-data descriptors. Accordingly, instances can\nredefine and override methods. This allows individual instances to\nacquire behaviors that differ from other instances of the same class.\n\nThe ``property()`` function is implemented as a data descriptor.\nAccordingly, instances cannot override the behavior of a property.\n\n\n__slots__\n=========\n\nBy default, instances of both old and new-style classes have a\ndictionary for attribute storage. This wastes space for objects\nhaving very few instance variables. The space consumption can become\nacute when creating large numbers of instances.\n\nThe default can be overridden by defining *__slots__* in a new-style\nclass definition. The *__slots__* declaration takes a sequence of\ninstance variables and reserves just enough space in each instance to\nhold a value for each variable. Space is saved because *__dict__* is\nnot created for each instance.\n\n__slots__\n\n This class variable can be assigned a string, iterable, or sequence\n of strings with variable names used by instances. If defined in a\n new-style class, *__slots__* reserves space for the declared\n variables and prevents the automatic creation of *__dict__* and\n *__weakref__* for each instance.\n\n New in version 2.2.\n\nNotes on using *__slots__*\n\n* When inheriting from a class without *__slots__*, the *__dict__*\n attribute of that class will always be accessible, so a *__slots__*\n definition in the subclass is meaningless.\n\n* Without a *__dict__* variable, instances cannot be assigned new\n variables not listed in the *__slots__* definition. Attempts to\n assign to an unlisted variable name raises ``AttributeError``. If\n dynamic assignment of new variables is desired, then add\n ``\'__dict__\'`` to the sequence of strings in the *__slots__*\n declaration.\n\n Changed in version 2.3: Previously, adding ``\'__dict__\'`` to the\n *__slots__* declaration would not enable the assignment of new\n attributes not specifically listed in the sequence of instance\n variable names.\n\n* Without a *__weakref__* variable for each instance, classes defining\n *__slots__* do not support weak references to its instances. If weak\n reference support is needed, then add ``\'__weakref__\'`` to the\n sequence of strings in the *__slots__* declaration.\n\n Changed in version 2.3: Previously, adding ``\'__weakref__\'`` to the\n *__slots__* declaration would not enable support for weak\n references.\n\n* *__slots__* are implemented at the class level by creating\n descriptors (*Implementing Descriptors*) for each variable name. As\n a result, class attributes cannot be used to set default values for\n instance variables defined by *__slots__*; otherwise, the class\n attribute would overwrite the descriptor assignment.\n\n* The action of a *__slots__* declaration is limited to the class\n where it is defined. As a result, subclasses will have a *__dict__*\n unless they also define *__slots__* (which must only contain names\n of any *additional* slots).\n\n* If a class defines a slot also defined in a base class, the instance\n variable defined by the base class slot is inaccessible (except by\n retrieving its descriptor directly from the base class). This\n renders the meaning of the program undefined. In the future, a\n check may be added to prevent this.\n\n* Nonempty *__slots__* does not work for classes derived from\n "variable-length" built-in types such as ``long``, ``str`` and\n ``tuple``.\n\n* Any non-string iterable may be assigned to *__slots__*. Mappings may\n also be used; however, in the future, special meaning may be\n assigned to the values corresponding to each key.\n\n* *__class__* assignment works only if both classes have the same\n *__slots__*.\n\n Changed in version 2.6: Previously, *__class__* assignment raised an\n error if either new or old class had *__slots__*.\n', + 'attribute-access': u'\nCustomizing attribute access\n****************************\n\nThe following methods can be defined to customize the meaning of\nattribute access (use of, assignment to, or deletion of ``x.name``)\nfor class instances.\n\nobject.__getattr__(self, name)\n\n Called when an attribute lookup has not found the attribute in the\n usual places (i.e. it is not an instance attribute nor is it found\n in the class tree for ``self``). ``name`` is the attribute name.\n This method should return the (computed) attribute value or raise\n an ``AttributeError`` exception.\n\n Note that if the attribute is found through the normal mechanism,\n ``__getattr__()`` is not called. (This is an intentional asymmetry\n between ``__getattr__()`` and ``__setattr__()``.) This is done both\n for efficiency reasons and because otherwise ``__getattr__()``\n would have no way to access other attributes of the instance. Note\n that at least for instance variables, you can fake total control by\n not inserting any values in the instance attribute dictionary (but\n instead inserting them in another object). See the\n ``__getattribute__()`` method below for a way to actually get total\n control in new-style classes.\n\nobject.__setattr__(self, name, value)\n\n Called when an attribute assignment is attempted. This is called\n instead of the normal mechanism (i.e. store the value in the\n instance dictionary). *name* is the attribute name, *value* is the\n value to be assigned to it.\n\n If ``__setattr__()`` wants to assign to an instance attribute, it\n should not simply execute ``self.name = value`` --- this would\n cause a recursive call to itself. Instead, it should insert the\n value in the dictionary of instance attributes, e.g.,\n ``self.__dict__[name] = value``. For new-style classes, rather\n than accessing the instance dictionary, it should call the base\n class method with the same name, for example,\n ``object.__setattr__(self, name, value)``.\n\nobject.__delattr__(self, name)\n\n Like ``__setattr__()`` but for attribute deletion instead of\n assignment. This should only be implemented if ``del obj.name`` is\n meaningful for the object.\n\n\nMore attribute access for new-style classes\n===========================================\n\nThe following methods only apply to new-style classes.\n\nobject.__getattribute__(self, name)\n\n Called unconditionally to implement attribute accesses for\n instances of the class. If the class also defines\n ``__getattr__()``, the latter will not be called unless\n ``__getattribute__()`` either calls it explicitly or raises an\n ``AttributeError``. This method should return the (computed)\n attribute value or raise an ``AttributeError`` exception. In order\n to avoid infinite recursion in this method, its implementation\n should always call the base class method with the same name to\n access any attributes it needs, for example,\n ``object.__getattribute__(self, name)``.\n\n Note: This method may still be bypassed when looking up special methods\n as the result of implicit invocation via language syntax or\n built-in functions. See *Special method lookup for new-style\n classes*.\n\n\nImplementing Descriptors\n========================\n\nThe following methods only apply when an instance of the class\ncontaining the method (a so-called *descriptor* class) appears in an\n*owner* class (the descriptor must be in either the owner\'s class\ndictionary or in the class dictionary for one of its parents). In the\nexamples below, "the attribute" refers to the attribute whose name is\nthe key of the property in the owner class\' ``__dict__``.\n\nobject.__get__(self, instance, owner)\n\n Called to get the attribute of the owner class (class attribute\n access) or of an instance of that class (instance attribute\n access). *owner* is always the owner class, while *instance* is the\n instance that the attribute was accessed through, or ``None`` when\n the attribute is accessed through the *owner*. This method should\n return the (computed) attribute value or raise an\n ``AttributeError`` exception.\n\nobject.__set__(self, instance, value)\n\n Called to set the attribute on an instance *instance* of the owner\n class to a new value, *value*.\n\nobject.__delete__(self, instance)\n\n Called to delete the attribute on an instance *instance* of the\n owner class.\n\n\nInvoking Descriptors\n====================\n\nIn general, a descriptor is an object attribute with "binding\nbehavior", one whose attribute access has been overridden by methods\nin the descriptor protocol: ``__get__()``, ``__set__()``, and\n``__delete__()``. If any of those methods are defined for an object,\nit is said to be a descriptor.\n\nThe default behavior for attribute access is to get, set, or delete\nthe attribute from an object\'s dictionary. For instance, ``a.x`` has a\nlookup chain starting with ``a.__dict__[\'x\']``, then\n``type(a).__dict__[\'x\']``, and continuing through the base classes of\n``type(a)`` excluding metaclasses.\n\nHowever, if the looked-up value is an object defining one of the\ndescriptor methods, then Python may override the default behavior and\ninvoke the descriptor method instead. Where this occurs in the\nprecedence chain depends on which descriptor methods were defined and\nhow they were called. Note that descriptors are only invoked for new\nstyle objects or classes (ones that subclass ``object()`` or\n``type()``).\n\nThe starting point for descriptor invocation is a binding, ``a.x``.\nHow the arguments are assembled depends on ``a``:\n\nDirect Call\n The simplest and least common call is when user code directly\n invokes a descriptor method: ``x.__get__(a)``.\n\nInstance Binding\n If binding to a new-style object instance, ``a.x`` is transformed\n into the call: ``type(a).__dict__[\'x\'].__get__(a, type(a))``.\n\nClass Binding\n If binding to a new-style class, ``A.x`` is transformed into the\n call: ``A.__dict__[\'x\'].__get__(None, A)``.\n\nSuper Binding\n If ``a`` is an instance of ``super``, then the binding ``super(B,\n obj).m()`` searches ``obj.__class__.__mro__`` for the base class\n ``A`` immediately preceding ``B`` and then invokes the descriptor\n with the call: ``A.__dict__[\'m\'].__get__(obj, obj.__class__)``.\n\nFor instance bindings, the precedence of descriptor invocation depends\non the which descriptor methods are defined. A descriptor can define\nany combination of ``__get__()``, ``__set__()`` and ``__delete__()``.\nIf it does not define ``__get__()``, then accessing the attribute will\nreturn the descriptor object itself unless there is a value in the\nobject\'s instance dictionary. If the descriptor defines ``__set__()``\nand/or ``__delete__()``, it is a data descriptor; if it defines\nneither, it is a non-data descriptor. Normally, data descriptors\ndefine both ``__get__()`` and ``__set__()``, while non-data\ndescriptors have just the ``__get__()`` method. Data descriptors with\n``__set__()`` and ``__get__()`` defined always override a redefinition\nin an instance dictionary. In contrast, non-data descriptors can be\noverridden by instances.\n\nPython methods (including ``staticmethod()`` and ``classmethod()``)\nare implemented as non-data descriptors. Accordingly, instances can\nredefine and override methods. This allows individual instances to\nacquire behaviors that differ from other instances of the same class.\n\nThe ``property()`` function is implemented as a data descriptor.\nAccordingly, instances cannot override the behavior of a property.\n\n\n__slots__\n=========\n\nBy default, instances of both old and new-style classes have a\ndictionary for attribute storage. This wastes space for objects\nhaving very few instance variables. The space consumption can become\nacute when creating large numbers of instances.\n\nThe default can be overridden by defining *__slots__* in a new-style\nclass definition. The *__slots__* declaration takes a sequence of\ninstance variables and reserves just enough space in each instance to\nhold a value for each variable. Space is saved because *__dict__* is\nnot created for each instance.\n\n__slots__\n\n This class variable can be assigned a string, iterable, or sequence\n of strings with variable names used by instances. If defined in a\n new-style class, *__slots__* reserves space for the declared\n variables and prevents the automatic creation of *__dict__* and\n *__weakref__* for each instance.\n\n New in version 2.2.\n\nNotes on using *__slots__*\n\n* When inheriting from a class without *__slots__*, the *__dict__*\n attribute of that class will always be accessible, so a *__slots__*\n definition in the subclass is meaningless.\n\n* Without a *__dict__* variable, instances cannot be assigned new\n variables not listed in the *__slots__* definition. Attempts to\n assign to an unlisted variable name raises ``AttributeError``. If\n dynamic assignment of new variables is desired, then add\n ``\'__dict__\'`` to the sequence of strings in the *__slots__*\n declaration.\n\n Changed in version 2.3: Previously, adding ``\'__dict__\'`` to the\n *__slots__* declaration would not enable the assignment of new\n attributes not specifically listed in the sequence of instance\n variable names.\n\n* Without a *__weakref__* variable for each instance, classes defining\n *__slots__* do not support weak references to its instances. If weak\n reference support is needed, then add ``\'__weakref__\'`` to the\n sequence of strings in the *__slots__* declaration.\n\n Changed in version 2.3: Previously, adding ``\'__weakref__\'`` to the\n *__slots__* declaration would not enable support for weak\n references.\n\n* *__slots__* are implemented at the class level by creating\n descriptors (*Implementing Descriptors*) for each variable name. As\n a result, class attributes cannot be used to set default values for\n instance variables defined by *__slots__*; otherwise, the class\n attribute would overwrite the descriptor assignment.\n\n* The action of a *__slots__* declaration is limited to the class\n where it is defined. As a result, subclasses will have a *__dict__*\n unless they also define *__slots__* (which must only contain names\n of any *additional* slots).\n\n* If a class defines a slot also defined in a base class, the instance\n variable defined by the base class slot is inaccessible (except by\n retrieving its descriptor directly from the base class). This\n renders the meaning of the program undefined. In the future, a\n check may be added to prevent this.\n\n* Nonempty *__slots__* does not work for classes derived from\n "variable-length" built-in types such as ``long``, ``str`` and\n ``tuple``.\n\n* Any non-string iterable may be assigned to *__slots__*. Mappings may\n also be used; however, in the future, special meaning may be\n assigned to the values corresponding to each key.\n\n* *__class__* assignment works only if both classes have the same\n *__slots__*.\n\n Changed in version 2.6: Previously, *__class__* assignment raised an\n error if either new or old class had *__slots__*.\n', 'attribute-references': u'\nAttribute references\n********************\n\nAn attribute reference is a primary followed by a period and a name:\n\n attributeref ::= primary "." identifier\n\nThe primary must evaluate to an object of a type that supports\nattribute references, e.g., a module, list, or an instance. This\nobject is then asked to produce the attribute whose name is the\nidentifier. If this attribute is not available, the exception\n``AttributeError`` is raised. Otherwise, the type and value of the\nobject produced is determined by the object. Multiple evaluations of\nthe same attribute reference may yield different objects.\n', 'augassign': u'\nAugmented assignment statements\n*******************************\n\nAugmented assignment is the combination, in a single statement, of a\nbinary operation and an assignment statement:\n\n augmented_assignment_stmt ::= augtarget augop (expression_list | yield_expression)\n augtarget ::= identifier | attributeref | subscription | slicing\n augop ::= "+=" | "-=" | "*=" | "/=" | "//=" | "%=" | "**="\n | ">>=" | "<<=" | "&=" | "^=" | "|="\n\n(See section *Primaries* for the syntax definitions for the last three\nsymbols.)\n\nAn augmented assignment evaluates the target (which, unlike normal\nassignment statements, cannot be an unpacking) and the expression\nlist, performs the binary operation specific to the type of assignment\non the two operands, and assigns the result to the original target.\nThe target is only evaluated once.\n\nAn augmented assignment expression like ``x += 1`` can be rewritten as\n``x = x + 1`` to achieve a similar, but not exactly equal effect. In\nthe augmented version, ``x`` is only evaluated once. Also, when\npossible, the actual operation is performed *in-place*, meaning that\nrather than creating a new object and assigning that to the target,\nthe old object is modified instead.\n\nWith the exception of assigning to tuples and multiple targets in a\nsingle statement, the assignment done by augmented assignment\nstatements is handled the same way as normal assignments. Similarly,\nwith the exception of the possible *in-place* behavior, the binary\noperation performed by augmented assignment is the same as the normal\nbinary operations.\n\nFor targets which are attribute references, the same *caveat about\nclass and instance attributes* applies as for regular assignments.\n', 'binary': u'\nBinary arithmetic operations\n****************************\n\nThe binary arithmetic operations have the conventional priority\nlevels. Note that some of these operations also apply to certain non-\nnumeric types. Apart from the power operator, there are only two\nlevels, one for multiplicative operators and one for additive\noperators:\n\n m_expr ::= u_expr | m_expr "*" u_expr | m_expr "//" u_expr | m_expr "/" u_expr\n | m_expr "%" u_expr\n a_expr ::= m_expr | a_expr "+" m_expr | a_expr "-" m_expr\n\nThe ``*`` (multiplication) operator yields the product of its\narguments. The arguments must either both be numbers, or one argument\nmust be an integer (plain or long) and the other must be a sequence.\nIn the former case, the numbers are converted to a common type and\nthen multiplied together. In the latter case, sequence repetition is\nperformed; a negative repetition factor yields an empty sequence.\n\nThe ``/`` (division) and ``//`` (floor division) operators yield the\nquotient of their arguments. The numeric arguments are first\nconverted to a common type. Plain or long integer division yields an\ninteger of the same type; the result is that of mathematical division\nwith the \'floor\' function applied to the result. Division by zero\nraises the ``ZeroDivisionError`` exception.\n\nThe ``%`` (modulo) operator yields the remainder from the division of\nthe first argument by the second. The numeric arguments are first\nconverted to a common type. A zero right argument raises the\n``ZeroDivisionError`` exception. The arguments may be floating point\nnumbers, e.g., ``3.14%0.7`` equals ``0.34`` (since ``3.14`` equals\n``4*0.7 + 0.34``.) The modulo operator always yields a result with\nthe same sign as its second operand (or zero); the absolute value of\nthe result is strictly smaller than the absolute value of the second\noperand [2].\n\nThe integer division and modulo operators are connected by the\nfollowing identity: ``x == (x/y)*y + (x%y)``. Integer division and\nmodulo are also connected with the built-in function ``divmod()``:\n``divmod(x, y) == (x/y, x%y)``. These identities don\'t hold for\nfloating point numbers; there similar identities hold approximately\nwhere ``x/y`` is replaced by ``floor(x/y)`` or ``floor(x/y) - 1`` [3].\n\nIn addition to performing the modulo operation on numbers, the ``%``\noperator is also overloaded by string and unicode objects to perform\nstring formatting (also known as interpolation). The syntax for string\nformatting is described in the Python Library Reference, section\n*String Formatting Operations*.\n\nDeprecated since version 2.3: The floor division operator, the modulo\noperator, and the ``divmod()`` function are no longer defined for\ncomplex numbers. Instead, convert to a floating point number using\nthe ``abs()`` function if appropriate.\n\nThe ``+`` (addition) operator yields the sum of its arguments. The\narguments must either both be numbers or both sequences of the same\ntype. In the former case, the numbers are converted to a common type\nand then added together. In the latter case, the sequences are\nconcatenated.\n\nThe ``-`` (subtraction) operator yields the difference of its\narguments. The numeric arguments are first converted to a common\ntype.\n', 'bitwise': u'\nBinary bitwise operations\n*************************\n\nEach of the three bitwise operations has a different priority level:\n\n and_expr ::= shift_expr | and_expr "&" shift_expr\n xor_expr ::= and_expr | xor_expr "^" and_expr\n or_expr ::= xor_expr | or_expr "|" xor_expr\n\nThe ``&`` operator yields the bitwise AND of its arguments, which must\nbe plain or long integers. The arguments are converted to a common\ntype.\n\nThe ``^`` operator yields the bitwise XOR (exclusive OR) of its\narguments, which must be plain or long integers. The arguments are\nconverted to a common type.\n\nThe ``|`` operator yields the bitwise (inclusive) OR of its arguments,\nwhich must be plain or long integers. The arguments are converted to\na common type.\n', 'bltin-code-objects': u'\nCode Objects\n************\n\nCode objects are used by the implementation to represent "pseudo-\ncompiled" executable Python code such as a function body. They differ\nfrom function objects because they don\'t contain a reference to their\nglobal execution environment. Code objects are returned by the built-\nin ``compile()`` function and can be extracted from function objects\nthrough their ``func_code`` attribute. See also the ``code`` module.\n\nA code object can be executed or evaluated by passing it (instead of a\nsource string) to the ``exec`` statement or the built-in ``eval()``\nfunction.\n\nSee *The standard type hierarchy* for more information.\n', 'bltin-ellipsis-object': u'\nThe Ellipsis Object\n*******************\n\nThis object is used by extended slice notation (see *Slicings*). It\nsupports no special operations. There is exactly one ellipsis object,\nnamed ``Ellipsis`` (a built-in name).\n\nIt is written as ``Ellipsis``.\n', - 'bltin-file-objects': u'\nFile Objects\n************\n\nFile objects are implemented using C\'s ``stdio`` package and can be\ncreated with the built-in ``open()`` function. File objects are also\nreturned by some other built-in functions and methods, such as\n``os.popen()`` and ``os.fdopen()`` and the ``makefile()`` method of\nsocket objects. Temporary files can be created using the ``tempfile``\nmodule, and high-level file operations such as copying, moving, and\ndeleting files and directories can be achieved with the ``shutil``\nmodule.\n\nWhen a file operation fails for an I/O-related reason, the exception\n``IOError`` is raised. This includes situations where the operation\nis not defined for some reason, like ``seek()`` on a tty device or\nwriting a file opened for reading.\n\nFiles have the following methods:\n\nfile.close()\n\n Close the file. A closed file cannot be read or written any more.\n Any operation which requires that the file be open will raise a\n ``ValueError`` after the file has been closed. Calling ``close()``\n more than once is allowed.\n\n As of Python 2.5, you can avoid having to call this method\n explicitly if you use the ``with`` statement. For example, the\n following code will automatically close *f* when the ``with`` block\n is exited:\n\n from __future__ import with_statement # This isn\'t required in Python 2.6\n\n with open("hello.txt") as f:\n for line in f:\n print line\n\n In older versions of Python, you would have needed to do this to\n get the same effect:\n\n f = open("hello.txt")\n try:\n for line in f:\n print line\n finally:\n f.close()\n\n Note: Not all "file-like" types in Python support use as a context\n manager for the ``with`` statement. If your code is intended to\n work with any file-like object, you can use the function\n ``contextlib.closing()`` instead of using the object directly.\n\nfile.flush()\n\n Flush the internal buffer, like ``stdio``\'s ``fflush()``. This may\n be a no-op on some file-like objects.\n\n Note: ``flush()`` does not necessarily write the file\'s data to disk.\n Use ``flush()`` followed by ``os.fsync()`` to ensure this\n behavior.\n\nfile.fileno()\n\n Return the integer "file descriptor" that is used by the underlying\n implementation to request I/O operations from the operating system.\n This can be useful for other, lower level interfaces that use file\n descriptors, such as the ``fcntl`` module or ``os.read()`` and\n friends.\n\n Note: File-like objects which do not have a real file descriptor should\n *not* provide this method!\n\nfile.isatty()\n\n Return ``True`` if the file is connected to a tty(-like) device,\n else ``False``.\n\n Note: If a file-like object is not associated with a real file, this\n method should *not* be implemented.\n\nfile.next()\n\n A file object is its own iterator, for example ``iter(f)`` returns\n *f* (unless *f* is closed). When a file is used as an iterator,\n typically in a ``for`` loop (for example, ``for line in f: print\n line``), the ``next()`` method is called repeatedly. This method\n returns the next input line, or raises ``StopIteration`` when EOF\n is hit when the file is open for reading (behavior is undefined\n when the file is open for writing). In order to make a ``for``\n loop the most efficient way of looping over the lines of a file (a\n very common operation), the ``next()`` method uses a hidden read-\n ahead buffer. As a consequence of using a read-ahead buffer,\n combining ``next()`` with other file methods (like ``readline()``)\n does not work right. However, using ``seek()`` to reposition the\n file to an absolute position will flush the read-ahead buffer.\n\n New in version 2.3.\n\nfile.read([size])\n\n Read at most *size* bytes from the file (less if the read hits EOF\n before obtaining *size* bytes). If the *size* argument is negative\n or omitted, read all data until EOF is reached. The bytes are\n returned as a string object. An empty string is returned when EOF\n is encountered immediately. (For certain files, like ttys, it\n makes sense to continue reading after an EOF is hit.) Note that\n this method may call the underlying C function ``fread()`` more\n than once in an effort to acquire as close to *size* bytes as\n possible. Also note that when in non-blocking mode, less data than\n was requested may be returned, even if no *size* parameter was\n given.\n\n Note: This function is simply a wrapper for the underlying ``fread()``\n C function, and will behave the same in corner cases, such as\n whether the EOF value is cached.\n\nfile.readline([size])\n\n Read one entire line from the file. A trailing newline character\n is kept in the string (but may be absent when a file ends with an\n incomplete line). [5] If the *size* argument is present and non-\n negative, it is a maximum byte count (including the trailing\n newline) and an incomplete line may be returned. An empty string is\n returned *only* when EOF is encountered immediately.\n\n Note: Unlike ``stdio``\'s ``fgets()``, the returned string contains null\n characters (``\'\\0\'``) if they occurred in the input.\n\nfile.readlines([sizehint])\n\n Read until EOF using ``readline()`` and return a list containing\n the lines thus read. If the optional *sizehint* argument is\n present, instead of reading up to EOF, whole lines totalling\n approximately *sizehint* bytes (possibly after rounding up to an\n internal buffer size) are read. Objects implementing a file-like\n interface may choose to ignore *sizehint* if it cannot be\n implemented, or cannot be implemented efficiently.\n\nfile.xreadlines()\n\n This method returns the same thing as ``iter(f)``.\n\n New in version 2.1.\n\n Deprecated since version 2.3: Use ``for line in file`` instead.\n\nfile.seek(offset[, whence])\n\n Set the file\'s current position, like ``stdio``\'s ``fseek()``. The\n *whence* argument is optional and defaults to ``os.SEEK_SET`` or\n ``0`` (absolute file positioning); other values are ``os.SEEK_CUR``\n or ``1`` (seek relative to the current position) and\n ``os.SEEK_END`` or ``2`` (seek relative to the file\'s end). There\n is no return value.\n\n For example, ``f.seek(2, os.SEEK_CUR)`` advances the position by\n two and ``f.seek(-3, os.SEEK_END)`` sets the position to the third\n to last.\n\n Note that if the file is opened for appending (mode ``\'a\'`` or\n ``\'a+\'``), any ``seek()`` operations will be undone at the next\n write. If the file is only opened for writing in append mode (mode\n ``\'a\'``), this method is essentially a no-op, but it remains useful\n for files opened in append mode with reading enabled (mode\n ``\'a+\'``). If the file is opened in text mode (without ``\'b\'``),\n only offsets returned by ``tell()`` are legal. Use of other\n offsets causes undefined behavior.\n\n Note that not all file objects are seekable.\n\n Changed in version 2.6: Passing float values as offset has been\n deprecated.\n\nfile.tell()\n\n Return the file\'s current position, like ``stdio``\'s ``ftell()``.\n\n Note: On Windows, ``tell()`` can return illegal values (after an\n ``fgets()``) when reading files with Unix-style line-endings. Use\n binary mode (``\'rb\'``) to circumvent this problem.\n\nfile.truncate([size])\n\n Truncate the file\'s size. If the optional *size* argument is\n present, the file is truncated to (at most) that size. The size\n defaults to the current position. The current file position is not\n changed. Note that if a specified size exceeds the file\'s current\n size, the result is platform-dependent: possibilities include that\n the file may remain unchanged, increase to the specified size as if\n zero-filled, or increase to the specified size with undefined new\n content. Availability: Windows, many Unix variants.\n\nfile.write(str)\n\n Write a string to the file. There is no return value. Due to\n buffering, the string may not actually show up in the file until\n the ``flush()`` or ``close()`` method is called.\n\nfile.writelines(sequence)\n\n Write a sequence of strings to the file. The sequence can be any\n iterable object producing strings, typically a list of strings.\n There is no return value. (The name is intended to match\n ``readlines()``; ``writelines()`` does not add line separators.)\n\nFiles support the iterator protocol. Each iteration returns the same\nresult as ``file.readline()``, and iteration ends when the\n``readline()`` method returns an empty string.\n\nFile objects also offer a number of other interesting attributes.\nThese are not required for file-like objects, but should be\nimplemented if they make sense for the particular object.\n\nfile.closed\n\n bool indicating the current state of the file object. This is a\n read-only attribute; the ``close()`` method changes the value. It\n may not be available on all file-like objects.\n\nfile.encoding\n\n The encoding that this file uses. When Unicode strings are written\n to a file, they will be converted to byte strings using this\n encoding. In addition, when the file is connected to a terminal,\n the attribute gives the encoding that the terminal is likely to use\n (that information might be incorrect if the user has misconfigured\n the terminal). The attribute is read-only and may not be present\n on all file-like objects. It may also be ``None``, in which case\n the file uses the system default encoding for converting Unicode\n strings.\n\n New in version 2.3.\n\nfile.errors\n\n The Unicode error handler used along with the encoding.\n\n New in version 2.6.\n\nfile.mode\n\n The I/O mode for the file. If the file was created using the\n ``open()`` built-in function, this will be the value of the *mode*\n parameter. This is a read-only attribute and may not be present on\n all file-like objects.\n\nfile.name\n\n If the file object was created using ``open()``, the name of the\n file. Otherwise, some string that indicates the source of the file\n object, of the form ``<...>``. This is a read-only attribute and\n may not be present on all file-like objects.\n\nfile.newlines\n\n If Python was built with the *--with-universal-newlines* option to\n **configure** (the default) this read-only attribute exists, and\n for files opened in universal newline read mode it keeps track of\n the types of newlines encountered while reading the file. The\n values it can take are ``\'\\r\'``, ``\'\\n\'``, ``\'\\r\\n\'``, ``None``\n (unknown, no newlines read yet) or a tuple containing all the\n newline types seen, to indicate that multiple newline conventions\n were encountered. For files not opened in universal newline read\n mode the value of this attribute will be ``None``.\n\nfile.softspace\n\n Boolean that indicates whether a space character needs to be\n printed before another value when using the ``print`` statement.\n Classes that are trying to simulate a file object should also have\n a writable ``softspace`` attribute, which should be initialized to\n zero. This will be automatic for most classes implemented in\n Python (care may be needed for objects that override attribute\n access); types implemented in C will have to provide a writable\n ``softspace`` attribute.\n\n Note: This attribute is not used to control the ``print`` statement,\n but to allow the implementation of ``print`` to keep track of its\n internal state.\n', + 'bltin-file-objects': u'\nFile Objects\n************\n\nFile objects are implemented using C\'s ``stdio`` package and can be\ncreated with the built-in ``open()`` function. File objects are also\nreturned by some other built-in functions and methods, such as\n``os.popen()`` and ``os.fdopen()`` and the ``makefile()`` method of\nsocket objects. Temporary files can be created using the ``tempfile``\nmodule, and high-level file operations such as copying, moving, and\ndeleting files and directories can be achieved with the ``shutil``\nmodule.\n\nWhen a file operation fails for an I/O-related reason, the exception\n``IOError`` is raised. This includes situations where the operation\nis not defined for some reason, like ``seek()`` on a tty device or\nwriting a file opened for reading.\n\nFiles have the following methods:\n\nfile.close()\n\n Close the file. A closed file cannot be read or written any more.\n Any operation which requires that the file be open will raise a\n ``ValueError`` after the file has been closed. Calling ``close()``\n more than once is allowed.\n\n As of Python 2.5, you can avoid having to call this method\n explicitly if you use the ``with`` statement. For example, the\n following code will automatically close *f* when the ``with`` block\n is exited:\n\n from __future__ import with_statement # This isn\'t required in Python 2.6\n\n with open("hello.txt") as f:\n for line in f:\n print line\n\n In older versions of Python, you would have needed to do this to\n get the same effect:\n\n f = open("hello.txt")\n try:\n for line in f:\n print line\n finally:\n f.close()\n\n Note: Not all "file-like" types in Python support use as a context\n manager for the ``with`` statement. If your code is intended to\n work with any file-like object, you can use the function\n ``contextlib.closing()`` instead of using the object directly.\n\nfile.flush()\n\n Flush the internal buffer, like ``stdio``\'s ``fflush()``. This may\n be a no-op on some file-like objects.\n\n Note: ``flush()`` does not necessarily write the file\'s data to disk.\n Use ``flush()`` followed by ``os.fsync()`` to ensure this\n behavior.\n\nfile.fileno()\n\n Return the integer "file descriptor" that is used by the underlying\n implementation to request I/O operations from the operating system.\n This can be useful for other, lower level interfaces that use file\n descriptors, such as the ``fcntl`` module or ``os.read()`` and\n friends.\n\n Note: File-like objects which do not have a real file descriptor should\n *not* provide this method!\n\nfile.isatty()\n\n Return ``True`` if the file is connected to a tty(-like) device,\n else ``False``.\n\n Note: If a file-like object is not associated with a real file, this\n method should *not* be implemented.\n\nfile.next()\n\n A file object is its own iterator, for example ``iter(f)`` returns\n *f* (unless *f* is closed). When a file is used as an iterator,\n typically in a ``for`` loop (for example, ``for line in f: print\n line``), the ``next()`` method is called repeatedly. This method\n returns the next input line, or raises ``StopIteration`` when EOF\n is hit when the file is open for reading (behavior is undefined\n when the file is open for writing). In order to make a ``for``\n loop the most efficient way of looping over the lines of a file (a\n very common operation), the ``next()`` method uses a hidden read-\n ahead buffer. As a consequence of using a read-ahead buffer,\n combining ``next()`` with other file methods (like ``readline()``)\n does not work right. However, using ``seek()`` to reposition the\n file to an absolute position will flush the read-ahead buffer.\n\n New in version 2.3.\n\nfile.read([size])\n\n Read at most *size* bytes from the file (less if the read hits EOF\n before obtaining *size* bytes). If the *size* argument is negative\n or omitted, read all data until EOF is reached. The bytes are\n returned as a string object. An empty string is returned when EOF\n is encountered immediately. (For certain files, like ttys, it\n makes sense to continue reading after an EOF is hit.) Note that\n this method may call the underlying C function ``fread()`` more\n than once in an effort to acquire as close to *size* bytes as\n possible. Also note that when in non-blocking mode, less data than\n was requested may be returned, even if no *size* parameter was\n given.\n\n Note: This function is simply a wrapper for the underlying ``fread()``\n C function, and will behave the same in corner cases, such as\n whether the EOF value is cached.\n\nfile.readline([size])\n\n Read one entire line from the file. A trailing newline character\n is kept in the string (but may be absent when a file ends with an\n incomplete line). [5] If the *size* argument is present and non-\n negative, it is a maximum byte count (including the trailing\n newline) and an incomplete line may be returned. When *size* is not\n 0, an empty string is returned *only* when EOF is encountered\n immediately.\n\n Note: Unlike ``stdio``\'s ``fgets()``, the returned string contains null\n characters (``\'\\0\'``) if they occurred in the input.\n\nfile.readlines([sizehint])\n\n Read until EOF using ``readline()`` and return a list containing\n the lines thus read. If the optional *sizehint* argument is\n present, instead of reading up to EOF, whole lines totalling\n approximately *sizehint* bytes (possibly after rounding up to an\n internal buffer size) are read. Objects implementing a file-like\n interface may choose to ignore *sizehint* if it cannot be\n implemented, or cannot be implemented efficiently.\n\nfile.xreadlines()\n\n This method returns the same thing as ``iter(f)``.\n\n New in version 2.1.\n\n Deprecated since version 2.3: Use ``for line in file`` instead.\n\nfile.seek(offset[, whence])\n\n Set the file\'s current position, like ``stdio``\'s ``fseek()``. The\n *whence* argument is optional and defaults to ``os.SEEK_SET`` or\n ``0`` (absolute file positioning); other values are ``os.SEEK_CUR``\n or ``1`` (seek relative to the current position) and\n ``os.SEEK_END`` or ``2`` (seek relative to the file\'s end). There\n is no return value.\n\n For example, ``f.seek(2, os.SEEK_CUR)`` advances the position by\n two and ``f.seek(-3, os.SEEK_END)`` sets the position to the third\n to last.\n\n Note that if the file is opened for appending (mode ``\'a\'`` or\n ``\'a+\'``), any ``seek()`` operations will be undone at the next\n write. If the file is only opened for writing in append mode (mode\n ``\'a\'``), this method is essentially a no-op, but it remains useful\n for files opened in append mode with reading enabled (mode\n ``\'a+\'``). If the file is opened in text mode (without ``\'b\'``),\n only offsets returned by ``tell()`` are legal. Use of other\n offsets causes undefined behavior.\n\n Note that not all file objects are seekable.\n\n Changed in version 2.6: Passing float values as offset has been\n deprecated.\n\nfile.tell()\n\n Return the file\'s current position, like ``stdio``\'s ``ftell()``.\n\n Note: On Windows, ``tell()`` can return illegal values (after an\n ``fgets()``) when reading files with Unix-style line-endings. Use\n binary mode (``\'rb\'``) to circumvent this problem.\n\nfile.truncate([size])\n\n Truncate the file\'s size. If the optional *size* argument is\n present, the file is truncated to (at most) that size. The size\n defaults to the current position. The current file position is not\n changed. Note that if a specified size exceeds the file\'s current\n size, the result is platform-dependent: possibilities include that\n the file may remain unchanged, increase to the specified size as if\n zero-filled, or increase to the specified size with undefined new\n content. Availability: Windows, many Unix variants.\n\nfile.write(str)\n\n Write a string to the file. There is no return value. Due to\n buffering, the string may not actually show up in the file until\n the ``flush()`` or ``close()`` method is called.\n\nfile.writelines(sequence)\n\n Write a sequence of strings to the file. The sequence can be any\n iterable object producing strings, typically a list of strings.\n There is no return value. (The name is intended to match\n ``readlines()``; ``writelines()`` does not add line separators.)\n\nFiles support the iterator protocol. Each iteration returns the same\nresult as ``file.readline()``, and iteration ends when the\n``readline()`` method returns an empty string.\n\nFile objects also offer a number of other interesting attributes.\nThese are not required for file-like objects, but should be\nimplemented if they make sense for the particular object.\n\nfile.closed\n\n bool indicating the current state of the file object. This is a\n read-only attribute; the ``close()`` method changes the value. It\n may not be available on all file-like objects.\n\nfile.encoding\n\n The encoding that this file uses. When Unicode strings are written\n to a file, they will be converted to byte strings using this\n encoding. In addition, when the file is connected to a terminal,\n the attribute gives the encoding that the terminal is likely to use\n (that information might be incorrect if the user has misconfigured\n the terminal). The attribute is read-only and may not be present\n on all file-like objects. It may also be ``None``, in which case\n the file uses the system default encoding for converting Unicode\n strings.\n\n New in version 2.3.\n\nfile.errors\n\n The Unicode error handler used along with the encoding.\n\n New in version 2.6.\n\nfile.mode\n\n The I/O mode for the file. If the file was created using the\n ``open()`` built-in function, this will be the value of the *mode*\n parameter. This is a read-only attribute and may not be present on\n all file-like objects.\n\nfile.name\n\n If the file object was created using ``open()``, the name of the\n file. Otherwise, some string that indicates the source of the file\n object, of the form ``<...>``. This is a read-only attribute and\n may not be present on all file-like objects.\n\nfile.newlines\n\n If Python was built with universal newlines enabled (the default)\n this read-only attribute exists, and for files opened in universal\n newline read mode it keeps track of the types of newlines\n encountered while reading the file. The values it can take are\n ``\'\\r\'``, ``\'\\n\'``, ``\'\\r\\n\'``, ``None`` (unknown, no newlines read\n yet) or a tuple containing all the newline types seen, to indicate\n that multiple newline conventions were encountered. For files not\n opened in universal newline read mode the value of this attribute\n will be ``None``.\n\nfile.softspace\n\n Boolean that indicates whether a space character needs to be\n printed before another value when using the ``print`` statement.\n Classes that are trying to simulate a file object should also have\n a writable ``softspace`` attribute, which should be initialized to\n zero. This will be automatic for most classes implemented in\n Python (care may be needed for objects that override attribute\n access); types implemented in C will have to provide a writable\n ``softspace`` attribute.\n\n Note: This attribute is not used to control the ``print`` statement,\n but to allow the implementation of ``print`` to keep track of its\n internal state.\n', 'bltin-null-object': u"\nThe Null Object\n***************\n\nThis object is returned by functions that don't explicitly return a\nvalue. It supports no special operations. There is exactly one null\nobject, named ``None`` (a built-in name).\n\nIt is written as ``None``.\n", 'bltin-type-objects': u"\nType Objects\n************\n\nType objects represent the various object types. An object's type is\naccessed by the built-in function ``type()``. There are no special\noperations on types. The standard module ``types`` defines names for\nall standard built-in types.\n\nTypes are written like this: ``<type 'int'>``.\n", 'booleans': u'\nBoolean operations\n******************\n\n or_test ::= and_test | or_test "or" and_test\n and_test ::= not_test | and_test "and" not_test\n not_test ::= comparison | "not" not_test\n\nIn the context of Boolean operations, and also when expressions are\nused by control flow statements, the following values are interpreted\nas false: ``False``, ``None``, numeric zero of all types, and empty\nstrings and containers (including strings, tuples, lists,\ndictionaries, sets and frozensets). All other values are interpreted\nas true. (See the ``__nonzero__()`` special method for a way to\nchange this.)\n\nThe operator ``not`` yields ``True`` if its argument is false,\n``False`` otherwise.\n\nThe expression ``x and y`` first evaluates *x*; if *x* is false, its\nvalue is returned; otherwise, *y* is evaluated and the resulting value\nis returned.\n\nThe expression ``x or y`` first evaluates *x*; if *x* is true, its\nvalue is returned; otherwise, *y* is evaluated and the resulting value\nis returned.\n\n(Note that neither ``and`` nor ``or`` restrict the value and type they\nreturn to ``False`` and ``True``, but rather return the last evaluated\nargument. This is sometimes useful, e.g., if ``s`` is a string that\nshould be replaced by a default value if it is empty, the expression\n``s or \'foo\'`` yields the desired value. Because ``not`` has to\ninvent a value anyway, it does not bother to return a value of the\nsame type as its argument, so e.g., ``not \'foo\'`` yields ``False``,\nnot ``\'\'``.)\n', @@ -20,39 +20,39 @@ 'class': u'\nClass definitions\n*****************\n\nA class definition defines a class object (see section *The standard\ntype hierarchy*):\n\n classdef ::= "class" classname [inheritance] ":" suite\n inheritance ::= "(" [expression_list] ")"\n classname ::= identifier\n\nA class definition is an executable statement. It first evaluates the\ninheritance list, if present. Each item in the inheritance list\nshould evaluate to a class object or class type which allows\nsubclassing. The class\'s suite is then executed in a new execution\nframe (see section *Naming and binding*), using a newly created local\nnamespace and the original global namespace. (Usually, the suite\ncontains only function definitions.) When the class\'s suite finishes\nexecution, its execution frame is discarded but its local namespace is\nsaved. [4] A class object is then created using the inheritance list\nfor the base classes and the saved local namespace for the attribute\ndictionary. The class name is bound to this class object in the\noriginal local namespace.\n\n**Programmer\'s note:** Variables defined in the class definition are\nclass variables; they are shared by all instances. To create instance\nvariables, they can be set in a method with ``self.name = value``.\nBoth class and instance variables are accessible through the notation\n"``self.name``", and an instance variable hides a class variable with\nthe same name when accessed in this way. Class variables can be used\nas defaults for instance variables, but using mutable values there can\nlead to unexpected results. For *new-style class*es, descriptors can\nbe used to create instance variables with different implementation\ndetails.\n\nClass definitions, like function definitions, may be wrapped by one or\nmore *decorator* expressions. The evaluation rules for the decorator\nexpressions are the same as for functions. The result must be a class\nobject, which is then bound to the class name.\n\n-[ Footnotes ]-\n\n[1] The exception is propagated to the invocation stack only if there\n is no ``finally`` clause that negates the exception.\n\n[2] Currently, control "flows off the end" except in the case of an\n exception or the execution of a ``return``, ``continue``, or\n ``break`` statement.\n\n[3] A string literal appearing as the first statement in the function\n body is transformed into the function\'s ``__doc__`` attribute and\n therefore the function\'s *docstring*.\n\n[4] A string literal appearing as the first statement in the class\n body is transformed into the namespace\'s ``__doc__`` item and\n therefore the class\'s *docstring*.\n', 'coercion-rules': u"\nCoercion rules\n**************\n\nThis section used to document the rules for coercion. As the language\nhas evolved, the coercion rules have become hard to document\nprecisely; documenting what one version of one particular\nimplementation does is undesirable. Instead, here are some informal\nguidelines regarding coercion. In Python 3.0, coercion will not be\nsupported.\n\n* If the left operand of a % operator is a string or Unicode object,\n no coercion takes place and the string formatting operation is\n invoked instead.\n\n* It is no longer recommended to define a coercion operation. Mixed-\n mode operations on types that don't define coercion pass the\n original arguments to the operation.\n\n* New-style classes (those derived from ``object``) never invoke the\n ``__coerce__()`` method in response to a binary operator; the only\n time ``__coerce__()`` is invoked is when the built-in function\n ``coerce()`` is called.\n\n* For most intents and purposes, an operator that returns\n ``NotImplemented`` is treated the same as one that is not\n implemented at all.\n\n* Below, ``__op__()`` and ``__rop__()`` are used to signify the\n generic method names corresponding to an operator; ``__iop__()`` is\n used for the corresponding in-place operator. For example, for the\n operator '``+``', ``__add__()`` and ``__radd__()`` are used for the\n left and right variant of the binary operator, and ``__iadd__()``\n for the in-place variant.\n\n* For objects *x* and *y*, first ``x.__op__(y)`` is tried. If this is\n not implemented or returns ``NotImplemented``, ``y.__rop__(x)`` is\n tried. If this is also not implemented or returns\n ``NotImplemented``, a ``TypeError`` exception is raised. But see\n the following exception:\n\n* Exception to the previous item: if the left operand is an instance\n of a built-in type or a new-style class, and the right operand is an\n instance of a proper subclass of that type or class and overrides\n the base's ``__rop__()`` method, the right operand's ``__rop__()``\n method is tried *before* the left operand's ``__op__()`` method.\n\n This is done so that a subclass can completely override binary\n operators. Otherwise, the left operand's ``__op__()`` method would\n always accept the right operand: when an instance of a given class\n is expected, an instance of a subclass of that class is always\n acceptable.\n\n* When either operand type defines a coercion, this coercion is called\n before that type's ``__op__()`` or ``__rop__()`` method is called,\n but no sooner. If the coercion returns an object of a different\n type for the operand whose coercion is invoked, part of the process\n is redone using the new object.\n\n* When an in-place operator (like '``+=``') is used, if the left\n operand implements ``__iop__()``, it is invoked without any\n coercion. When the operation falls back to ``__op__()`` and/or\n ``__rop__()``, the normal coercion rules apply.\n\n* In ``x + y``, if *x* is a sequence that implements sequence\n concatenation, sequence concatenation is invoked.\n\n* In ``x * y``, if one operator is a sequence that implements sequence\n repetition, and the other is an integer (``int`` or ``long``),\n sequence repetition is invoked.\n\n* Rich comparisons (implemented by methods ``__eq__()`` and so on)\n never use coercion. Three-way comparison (implemented by\n ``__cmp__()``) does use coercion under the same conditions as other\n binary operations use it.\n\n* In the current implementation, the built-in numeric types ``int``,\n ``long``, ``float``, and ``complex`` do not use coercion. All these\n types implement a ``__coerce__()`` method, for use by the built-in\n ``coerce()`` function.\n\n Changed in version 2.7.\n", 'comparisons': u'\nComparisons\n***********\n\nUnlike C, all comparison operations in Python have the same priority,\nwhich is lower than that of any arithmetic, shifting or bitwise\noperation. Also unlike C, expressions like ``a < b < c`` have the\ninterpretation that is conventional in mathematics:\n\n comparison ::= or_expr ( comp_operator or_expr )*\n comp_operator ::= "<" | ">" | "==" | ">=" | "<=" | "<>" | "!="\n | "is" ["not"] | ["not"] "in"\n\nComparisons yield boolean values: ``True`` or ``False``.\n\nComparisons can be chained arbitrarily, e.g., ``x < y <= z`` is\nequivalent to ``x < y and y <= z``, except that ``y`` is evaluated\nonly once (but in both cases ``z`` is not evaluated at all when ``x <\ny`` is found to be false).\n\nFormally, if *a*, *b*, *c*, ..., *y*, *z* are expressions and *op1*,\n*op2*, ..., *opN* are comparison operators, then ``a op1 b op2 c ... y\nopN z`` is equivalent to ``a op1 b and b op2 c and ... y opN z``,\nexcept that each expression is evaluated at most once.\n\nNote that ``a op1 b op2 c`` doesn\'t imply any kind of comparison\nbetween *a* and *c*, so that, e.g., ``x < y > z`` is perfectly legal\n(though perhaps not pretty).\n\nThe forms ``<>`` and ``!=`` are equivalent; for consistency with C,\n``!=`` is preferred; where ``!=`` is mentioned below ``<>`` is also\naccepted. The ``<>`` spelling is considered obsolescent.\n\nThe operators ``<``, ``>``, ``==``, ``>=``, ``<=``, and ``!=`` compare\nthe values of two objects. The objects need not have the same type.\nIf both are numbers, they are converted to a common type. Otherwise,\nobjects of different types *always* compare unequal, and are ordered\nconsistently but arbitrarily. You can control comparison behavior of\nobjects of non-built-in types by defining a ``__cmp__`` method or rich\ncomparison methods like ``__gt__``, described in section *Special\nmethod names*.\n\n(This unusual definition of comparison was used to simplify the\ndefinition of operations like sorting and the ``in`` and ``not in``\noperators. In the future, the comparison rules for objects of\ndifferent types are likely to change.)\n\nComparison of objects of the same type depends on the type:\n\n* Numbers are compared arithmetically.\n\n* Strings are compared lexicographically using the numeric equivalents\n (the result of the built-in function ``ord()``) of their characters.\n Unicode and 8-bit strings are fully interoperable in this behavior.\n [4]\n\n* Tuples and lists are compared lexicographically using comparison of\n corresponding elements. This means that to compare equal, each\n element must compare equal and the two sequences must be of the same\n type and have the same length.\n\n If not equal, the sequences are ordered the same as their first\n differing elements. For example, ``cmp([1,2,x], [1,2,y])`` returns\n the same as ``cmp(x,y)``. If the corresponding element does not\n exist, the shorter sequence is ordered first (for example, ``[1,2] <\n [1,2,3]``).\n\n* Mappings (dictionaries) compare equal if and only if their sorted\n (key, value) lists compare equal. [5] Outcomes other than equality\n are resolved consistently, but are not otherwise defined. [6]\n\n* Most other objects of built-in types compare unequal unless they are\n the same object; the choice whether one object is considered smaller\n or larger than another one is made arbitrarily but consistently\n within one execution of a program.\n\nThe operators ``in`` and ``not in`` test for collection membership.\n``x in s`` evaluates to true if *x* is a member of the collection *s*,\nand false otherwise. ``x not in s`` returns the negation of ``x in\ns``. The collection membership test has traditionally been bound to\nsequences; an object is a member of a collection if the collection is\na sequence and contains an element equal to that object. However, it\nmake sense for many other object types to support membership tests\nwithout being a sequence. In particular, dictionaries (for keys) and\nsets support membership testing.\n\nFor the list and tuple types, ``x in y`` is true if and only if there\nexists an index *i* such that ``x == y[i]`` is true.\n\nFor the Unicode and string types, ``x in y`` is true if and only if\n*x* is a substring of *y*. An equivalent test is ``y.find(x) != -1``.\nNote, *x* and *y* need not be the same type; consequently, ``u\'ab\' in\n\'abc\'`` will return ``True``. Empty strings are always considered to\nbe a substring of any other string, so ``"" in "abc"`` will return\n``True``.\n\nChanged in version 2.3: Previously, *x* was required to be a string of\nlength ``1``.\n\nFor user-defined classes which define the ``__contains__()`` method,\n``x in y`` is true if and only if ``y.__contains__(x)`` is true.\n\nFor user-defined classes which do not define ``__contains__()`` but do\ndefine ``__iter__()``, ``x in y`` is true if some value ``z`` with ``x\n== z`` is produced while iterating over ``y``. If an exception is\nraised during the iteration, it is as if ``in`` raised that exception.\n\nLastly, the old-style iteration protocol is tried: if a class defines\n``__getitem__()``, ``x in y`` is true if and only if there is a non-\nnegative integer index *i* such that ``x == y[i]``, and all lower\ninteger indices do not raise ``IndexError`` exception. (If any other\nexception is raised, it is as if ``in`` raised that exception).\n\nThe operator ``not in`` is defined to have the inverse true value of\n``in``.\n\nThe operators ``is`` and ``is not`` test for object identity: ``x is\ny`` is true if and only if *x* and *y* are the same object. ``x is\nnot y`` yields the inverse truth value. [7]\n', - 'compound': u'\nCompound statements\n*******************\n\nCompound statements contain (groups of) other statements; they affect\nor control the execution of those other statements in some way. In\ngeneral, compound statements span multiple lines, although in simple\nincarnations a whole compound statement may be contained in one line.\n\nThe ``if``, ``while`` and ``for`` statements implement traditional\ncontrol flow constructs. ``try`` specifies exception handlers and/or\ncleanup code for a group of statements. Function and class\ndefinitions are also syntactically compound statements.\n\nCompound statements consist of one or more \'clauses.\' A clause\nconsists of a header and a \'suite.\' The clause headers of a\nparticular compound statement are all at the same indentation level.\nEach clause header begins with a uniquely identifying keyword and ends\nwith a colon. A suite is a group of statements controlled by a\nclause. A suite can be one or more semicolon-separated simple\nstatements on the same line as the header, following the header\'s\ncolon, or it can be one or more indented statements on subsequent\nlines. Only the latter form of suite can contain nested compound\nstatements; the following is illegal, mostly because it wouldn\'t be\nclear to which ``if`` clause a following ``else`` clause would belong:\n\n if test1: if test2: print x\n\nAlso note that the semicolon binds tighter than the colon in this\ncontext, so that in the following example, either all or none of the\n``print`` statements are executed:\n\n if x < y < z: print x; print y; print z\n\nSummarizing:\n\n compound_stmt ::= if_stmt\n | while_stmt\n | for_stmt\n | try_stmt\n | with_stmt\n | funcdef\n | classdef\n | decorated\n suite ::= stmt_list NEWLINE | NEWLINE INDENT statement+ DEDENT\n statement ::= stmt_list NEWLINE | compound_stmt\n stmt_list ::= simple_stmt (";" simple_stmt)* [";"]\n\nNote that statements always end in a ``NEWLINE`` possibly followed by\na ``DEDENT``. Also note that optional continuation clauses always\nbegin with a keyword that cannot start a statement, thus there are no\nambiguities (the \'dangling ``else``\' problem is solved in Python by\nrequiring nested ``if`` statements to be indented).\n\nThe formatting of the grammar rules in the following sections places\neach clause on a separate line for clarity.\n\n\nThe ``if`` statement\n====================\n\nThe ``if`` statement is used for conditional execution:\n\n if_stmt ::= "if" expression ":" suite\n ( "elif" expression ":" suite )*\n ["else" ":" suite]\n\nIt selects exactly one of the suites by evaluating the expressions one\nby one until one is found to be true (see section *Boolean operations*\nfor the definition of true and false); then that suite is executed\n(and no other part of the ``if`` statement is executed or evaluated).\nIf all expressions are false, the suite of the ``else`` clause, if\npresent, is executed.\n\n\nThe ``while`` statement\n=======================\n\nThe ``while`` statement is used for repeated execution as long as an\nexpression is true:\n\n while_stmt ::= "while" expression ":" suite\n ["else" ":" suite]\n\nThis repeatedly tests the expression and, if it is true, executes the\nfirst suite; if the expression is false (which may be the first time\nit is tested) the suite of the ``else`` clause, if present, is\nexecuted and the loop terminates.\n\nA ``break`` statement executed in the first suite terminates the loop\nwithout executing the ``else`` clause\'s suite. A ``continue``\nstatement executed in the first suite skips the rest of the suite and\ngoes back to testing the expression.\n\n\nThe ``for`` statement\n=====================\n\nThe ``for`` statement is used to iterate over the elements of a\nsequence (such as a string, tuple or list) or other iterable object:\n\n for_stmt ::= "for" target_list "in" expression_list ":" suite\n ["else" ":" suite]\n\nThe expression list is evaluated once; it should yield an iterable\nobject. An iterator is created for the result of the\n``expression_list``. The suite is then executed once for each item\nprovided by the iterator, in the order of ascending indices. Each\nitem in turn is assigned to the target list using the standard rules\nfor assignments, and then the suite is executed. When the items are\nexhausted (which is immediately when the sequence is empty), the suite\nin the ``else`` clause, if present, is executed, and the loop\nterminates.\n\nA ``break`` statement executed in the first suite terminates the loop\nwithout executing the ``else`` clause\'s suite. A ``continue``\nstatement executed in the first suite skips the rest of the suite and\ncontinues with the next item, or with the ``else`` clause if there was\nno next item.\n\nThe suite may assign to the variable(s) in the target list; this does\nnot affect the next item assigned to it.\n\nThe target list is not deleted when the loop is finished, but if the\nsequence is empty, it will not have been assigned to at all by the\nloop. Hint: the built-in function ``range()`` returns a sequence of\nintegers suitable to emulate the effect of Pascal\'s ``for i := a to b\ndo``; e.g., ``range(3)`` returns the list ``[0, 1, 2]``.\n\nNote: There is a subtlety when the sequence is being modified by the loop\n (this can only occur for mutable sequences, i.e. lists). An internal\n counter is used to keep track of which item is used next, and this\n is incremented on each iteration. When this counter has reached the\n length of the sequence the loop terminates. This means that if the\n suite deletes the current (or a previous) item from the sequence,\n the next item will be skipped (since it gets the index of the\n current item which has already been treated). Likewise, if the\n suite inserts an item in the sequence before the current item, the\n current item will be treated again the next time through the loop.\n This can lead to nasty bugs that can be avoided by making a\n temporary copy using a slice of the whole sequence, e.g.,\n\n for x in a[:]:\n if x < 0: a.remove(x)\n\n\nThe ``try`` statement\n=====================\n\nThe ``try`` statement specifies exception handlers and/or cleanup code\nfor a group of statements:\n\n try_stmt ::= try1_stmt | try2_stmt\n try1_stmt ::= "try" ":" suite\n ("except" [expression [("as" | ",") target]] ":" suite)+\n ["else" ":" suite]\n ["finally" ":" suite]\n try2_stmt ::= "try" ":" suite\n "finally" ":" suite\n\nChanged in version 2.5: In previous versions of Python,\n``try``...``except``...``finally`` did not work. ``try``...``except``\nhad to be nested in ``try``...``finally``.\n\nThe ``except`` clause(s) specify one or more exception handlers. When\nno exception occurs in the ``try`` clause, no exception handler is\nexecuted. When an exception occurs in the ``try`` suite, a search for\nan exception handler is started. This search inspects the except\nclauses in turn until one is found that matches the exception. An\nexpression-less except clause, if present, must be last; it matches\nany exception. For an except clause with an expression, that\nexpression is evaluated, and the clause matches the exception if the\nresulting object is "compatible" with the exception. An object is\ncompatible with an exception if it is the class or a base class of the\nexception object, a tuple containing an item compatible with the\nexception, or, in the (deprecated) case of string exceptions, is the\nraised string itself (note that the object identities must match, i.e.\nit must be the same string object, not just a string with the same\nvalue).\n\nIf no except clause matches the exception, the search for an exception\nhandler continues in the surrounding code and on the invocation stack.\n[1]\n\nIf the evaluation of an expression in the header of an except clause\nraises an exception, the original search for a handler is canceled and\na search starts for the new exception in the surrounding code and on\nthe call stack (it is treated as if the entire ``try`` statement\nraised the exception).\n\nWhen a matching except clause is found, the exception is assigned to\nthe target specified in that except clause, if present, and the except\nclause\'s suite is executed. All except clauses must have an\nexecutable block. When the end of this block is reached, execution\ncontinues normally after the entire try statement. (This means that\nif two nested handlers exist for the same exception, and the exception\noccurs in the try clause of the inner handler, the outer handler will\nnot handle the exception.)\n\nBefore an except clause\'s suite is executed, details about the\nexception are assigned to three variables in the ``sys`` module:\n``sys.exc_type`` receives the object identifying the exception;\n``sys.exc_value`` receives the exception\'s parameter;\n``sys.exc_traceback`` receives a traceback object (see section *The\nstandard type hierarchy*) identifying the point in the program where\nthe exception occurred. These details are also available through the\n``sys.exc_info()`` function, which returns a tuple ``(exc_type,\nexc_value, exc_traceback)``. Use of the corresponding variables is\ndeprecated in favor of this function, since their use is unsafe in a\nthreaded program. As of Python 1.5, the variables are restored to\ntheir previous values (before the call) when returning from a function\nthat handled an exception.\n\nThe optional ``else`` clause is executed if and when control flows off\nthe end of the ``try`` clause. [2] Exceptions in the ``else`` clause\nare not handled by the preceding ``except`` clauses.\n\nIf ``finally`` is present, it specifies a \'cleanup\' handler. The\n``try`` clause is executed, including any ``except`` and ``else``\nclauses. If an exception occurs in any of the clauses and is not\nhandled, the exception is temporarily saved. The ``finally`` clause is\nexecuted. If there is a saved exception, it is re-raised at the end\nof the ``finally`` clause. If the ``finally`` clause raises another\nexception or executes a ``return`` or ``break`` statement, the saved\nexception is lost. The exception information is not available to the\nprogram during execution of the ``finally`` clause.\n\nWhen a ``return``, ``break`` or ``continue`` statement is executed in\nthe ``try`` suite of a ``try``...``finally`` statement, the\n``finally`` clause is also executed \'on the way out.\' A ``continue``\nstatement is illegal in the ``finally`` clause. (The reason is a\nproblem with the current implementation --- this restriction may be\nlifted in the future).\n\nAdditional information on exceptions can be found in section\n*Exceptions*, and information on using the ``raise`` statement to\ngenerate exceptions may be found in section *The raise statement*.\n\n\nThe ``with`` statement\n======================\n\nNew in version 2.5.\n\nThe ``with`` statement is used to wrap the execution of a block with\nmethods defined by a context manager (see section *With Statement\nContext Managers*). This allows common\n``try``...``except``...``finally`` usage patterns to be encapsulated\nfor convenient reuse.\n\n with_stmt ::= "with" with_item ("," with_item)* ":" suite\n with_item ::= expression ["as" target]\n\nThe execution of the ``with`` statement with one "item" proceeds as\nfollows:\n\n1. The context expression is evaluated to obtain a context manager.\n\n2. The context manager\'s ``__exit__()`` is loaded for later use.\n\n3. The context manager\'s ``__enter__()`` method is invoked.\n\n4. If a target was included in the ``with`` statement, the return\n value from ``__enter__()`` is assigned to it.\n\n Note: The ``with`` statement guarantees that if the ``__enter__()``\n method returns without an error, then ``__exit__()`` will always\n be called. Thus, if an error occurs during the assignment to the\n target list, it will be treated the same as an error occurring\n within the suite would be. See step 6 below.\n\n5. The suite is executed.\n\n6. The context manager\'s ``__exit__()`` method is invoked. If an\n exception caused the suite to be exited, its type, value, and\n traceback are passed as arguments to ``__exit__()``. Otherwise,\n three ``None`` arguments are supplied.\n\n If the suite was exited due to an exception, and the return value\n from the ``__exit__()`` method was false, the exception is\n reraised. If the return value was true, the exception is\n suppressed, and execution continues with the statement following\n the ``with`` statement.\n\n If the suite was exited for any reason other than an exception, the\n return value from ``__exit__()`` is ignored, and execution proceeds\n at the normal location for the kind of exit that was taken.\n\nWith more than one item, the context managers are processed as if\nmultiple ``with`` statements were nested:\n\n with A() as a, B() as b:\n suite\n\nis equivalent to\n\n with A() as a:\n with B() as b:\n suite\n\nNote: In Python 2.5, the ``with`` statement is only allowed when the\n ``with_statement`` feature has been enabled. It is always enabled\n in Python 2.6.\n\nChanged in version 2.7: Support for multiple context expressions.\n\nSee also:\n\n **PEP 0343** - The "with" statement\n The specification, background, and examples for the Python\n ``with`` statement.\n\n\nFunction definitions\n====================\n\nA function definition defines a user-defined function object (see\nsection *The standard type hierarchy*):\n\n decorated ::= decorators (classdef | funcdef)\n decorators ::= decorator+\n decorator ::= "@" dotted_name ["(" [argument_list [","]] ")"] NEWLINE\n funcdef ::= "def" funcname "(" [parameter_list] ")" ":" suite\n dotted_name ::= identifier ("." identifier)*\n parameter_list ::= (defparameter ",")*\n ( "*" identifier [, "**" identifier]\n | "**" identifier\n | defparameter [","] )\n defparameter ::= parameter ["=" expression]\n sublist ::= parameter ("," parameter)* [","]\n parameter ::= identifier | "(" sublist ")"\n funcname ::= identifier\n\nA function definition is an executable statement. Its execution binds\nthe function name in the current local namespace to a function object\n(a wrapper around the executable code for the function). This\nfunction object contains a reference to the current global namespace\nas the global namespace to be used when the function is called.\n\nThe function definition does not execute the function body; this gets\nexecuted only when the function is called. [3]\n\nA function definition may be wrapped by one or more *decorator*\nexpressions. Decorator expressions are evaluated when the function is\ndefined, in the scope that contains the function definition. The\nresult must be a callable, which is invoked with the function object\nas the only argument. The returned value is bound to the function name\ninstead of the function object. Multiple decorators are applied in\nnested fashion. For example, the following code:\n\n @f1(arg)\n @f2\n def func(): pass\n\nis equivalent to:\n\n def func(): pass\n func = f1(arg)(f2(func))\n\nWhen one or more top-level parameters have the form *parameter* ``=``\n*expression*, the function is said to have "default parameter values."\nFor a parameter with a default value, the corresponding argument may\nbe omitted from a call, in which case the parameter\'s default value is\nsubstituted. If a parameter has a default value, all following\nparameters must also have a default value --- this is a syntactic\nrestriction that is not expressed by the grammar.\n\n**Default parameter values are evaluated when the function definition\nis executed.** This means that the expression is evaluated once, when\nthe function is defined, and that that same "pre-computed" value is\nused for each call. This is especially important to understand when a\ndefault parameter is a mutable object, such as a list or a dictionary:\nif the function modifies the object (e.g. by appending an item to a\nlist), the default value is in effect modified. This is generally not\nwhat was intended. A way around this is to use ``None`` as the\ndefault, and explicitly test for it in the body of the function, e.g.:\n\n def whats_on_the_telly(penguin=None):\n if penguin is None:\n penguin = []\n penguin.append("property of the zoo")\n return penguin\n\nFunction call semantics are described in more detail in section\n*Calls*. A function call always assigns values to all parameters\nmentioned in the parameter list, either from position arguments, from\nkeyword arguments, or from default values. If the form\n"``*identifier``" is present, it is initialized to a tuple receiving\nany excess positional parameters, defaulting to the empty tuple. If\nthe form "``**identifier``" is present, it is initialized to a new\ndictionary receiving any excess keyword arguments, defaulting to a new\nempty dictionary.\n\nIt is also possible to create anonymous functions (functions not bound\nto a name), for immediate use in expressions. This uses lambda forms,\ndescribed in section *Lambdas*. Note that the lambda form is merely a\nshorthand for a simplified function definition; a function defined in\na "``def``" statement can be passed around or assigned to another name\njust like a function defined by a lambda form. The "``def``" form is\nactually more powerful since it allows the execution of multiple\nstatements.\n\n**Programmer\'s note:** Functions are first-class objects. A "``def``"\nform executed inside a function definition defines a local function\nthat can be returned or passed around. Free variables used in the\nnested function can access the local variables of the function\ncontaining the def. See section *Naming and binding* for details.\n\n\nClass definitions\n=================\n\nA class definition defines a class object (see section *The standard\ntype hierarchy*):\n\n classdef ::= "class" classname [inheritance] ":" suite\n inheritance ::= "(" [expression_list] ")"\n classname ::= identifier\n\nA class definition is an executable statement. It first evaluates the\ninheritance list, if present. Each item in the inheritance list\nshould evaluate to a class object or class type which allows\nsubclassing. The class\'s suite is then executed in a new execution\nframe (see section *Naming and binding*), using a newly created local\nnamespace and the original global namespace. (Usually, the suite\ncontains only function definitions.) When the class\'s suite finishes\nexecution, its execution frame is discarded but its local namespace is\nsaved. [4] A class object is then created using the inheritance list\nfor the base classes and the saved local namespace for the attribute\ndictionary. The class name is bound to this class object in the\noriginal local namespace.\n\n**Programmer\'s note:** Variables defined in the class definition are\nclass variables; they are shared by all instances. To create instance\nvariables, they can be set in a method with ``self.name = value``.\nBoth class and instance variables are accessible through the notation\n"``self.name``", and an instance variable hides a class variable with\nthe same name when accessed in this way. Class variables can be used\nas defaults for instance variables, but using mutable values there can\nlead to unexpected results. For *new-style class*es, descriptors can\nbe used to create instance variables with different implementation\ndetails.\n\nClass definitions, like function definitions, may be wrapped by one or\nmore *decorator* expressions. The evaluation rules for the decorator\nexpressions are the same as for functions. The result must be a class\nobject, which is then bound to the class name.\n\n-[ Footnotes ]-\n\n[1] The exception is propagated to the invocation stack only if there\n is no ``finally`` clause that negates the exception.\n\n[2] Currently, control "flows off the end" except in the case of an\n exception or the execution of a ``return``, ``continue``, or\n ``break`` statement.\n\n[3] A string literal appearing as the first statement in the function\n body is transformed into the function\'s ``__doc__`` attribute and\n therefore the function\'s *docstring*.\n\n[4] A string literal appearing as the first statement in the class\n body is transformed into the namespace\'s ``__doc__`` item and\n therefore the class\'s *docstring*.\n', + 'compound': u'\nCompound statements\n*******************\n\nCompound statements contain (groups of) other statements; they affect\nor control the execution of those other statements in some way. In\ngeneral, compound statements span multiple lines, although in simple\nincarnations a whole compound statement may be contained in one line.\n\nThe ``if``, ``while`` and ``for`` statements implement traditional\ncontrol flow constructs. ``try`` specifies exception handlers and/or\ncleanup code for a group of statements. Function and class\ndefinitions are also syntactically compound statements.\n\nCompound statements consist of one or more \'clauses.\' A clause\nconsists of a header and a \'suite.\' The clause headers of a\nparticular compound statement are all at the same indentation level.\nEach clause header begins with a uniquely identifying keyword and ends\nwith a colon. A suite is a group of statements controlled by a\nclause. A suite can be one or more semicolon-separated simple\nstatements on the same line as the header, following the header\'s\ncolon, or it can be one or more indented statements on subsequent\nlines. Only the latter form of suite can contain nested compound\nstatements; the following is illegal, mostly because it wouldn\'t be\nclear to which ``if`` clause a following ``else`` clause would belong:\n\n if test1: if test2: print x\n\nAlso note that the semicolon binds tighter than the colon in this\ncontext, so that in the following example, either all or none of the\n``print`` statements are executed:\n\n if x < y < z: print x; print y; print z\n\nSummarizing:\n\n compound_stmt ::= if_stmt\n | while_stmt\n | for_stmt\n | try_stmt\n | with_stmt\n | funcdef\n | classdef\n | decorated\n suite ::= stmt_list NEWLINE | NEWLINE INDENT statement+ DEDENT\n statement ::= stmt_list NEWLINE | compound_stmt\n stmt_list ::= simple_stmt (";" simple_stmt)* [";"]\n\nNote that statements always end in a ``NEWLINE`` possibly followed by\na ``DEDENT``. Also note that optional continuation clauses always\nbegin with a keyword that cannot start a statement, thus there are no\nambiguities (the \'dangling ``else``\' problem is solved in Python by\nrequiring nested ``if`` statements to be indented).\n\nThe formatting of the grammar rules in the following sections places\neach clause on a separate line for clarity.\n\n\nThe ``if`` statement\n====================\n\nThe ``if`` statement is used for conditional execution:\n\n if_stmt ::= "if" expression ":" suite\n ( "elif" expression ":" suite )*\n ["else" ":" suite]\n\nIt selects exactly one of the suites by evaluating the expressions one\nby one until one is found to be true (see section *Boolean operations*\nfor the definition of true and false); then that suite is executed\n(and no other part of the ``if`` statement is executed or evaluated).\nIf all expressions are false, the suite of the ``else`` clause, if\npresent, is executed.\n\n\nThe ``while`` statement\n=======================\n\nThe ``while`` statement is used for repeated execution as long as an\nexpression is true:\n\n while_stmt ::= "while" expression ":" suite\n ["else" ":" suite]\n\nThis repeatedly tests the expression and, if it is true, executes the\nfirst suite; if the expression is false (which may be the first time\nit is tested) the suite of the ``else`` clause, if present, is\nexecuted and the loop terminates.\n\nA ``break`` statement executed in the first suite terminates the loop\nwithout executing the ``else`` clause\'s suite. A ``continue``\nstatement executed in the first suite skips the rest of the suite and\ngoes back to testing the expression.\n\n\nThe ``for`` statement\n=====================\n\nThe ``for`` statement is used to iterate over the elements of a\nsequence (such as a string, tuple or list) or other iterable object:\n\n for_stmt ::= "for" target_list "in" expression_list ":" suite\n ["else" ":" suite]\n\nThe expression list is evaluated once; it should yield an iterable\nobject. An iterator is created for the result of the\n``expression_list``. The suite is then executed once for each item\nprovided by the iterator, in the order of ascending indices. Each\nitem in turn is assigned to the target list using the standard rules\nfor assignments, and then the suite is executed. When the items are\nexhausted (which is immediately when the sequence is empty), the suite\nin the ``else`` clause, if present, is executed, and the loop\nterminates.\n\nA ``break`` statement executed in the first suite terminates the loop\nwithout executing the ``else`` clause\'s suite. A ``continue``\nstatement executed in the first suite skips the rest of the suite and\ncontinues with the next item, or with the ``else`` clause if there was\nno next item.\n\nThe suite may assign to the variable(s) in the target list; this does\nnot affect the next item assigned to it.\n\nThe target list is not deleted when the loop is finished, but if the\nsequence is empty, it will not have been assigned to at all by the\nloop. Hint: the built-in function ``range()`` returns a sequence of\nintegers suitable to emulate the effect of Pascal\'s ``for i := a to b\ndo``; e.g., ``range(3)`` returns the list ``[0, 1, 2]``.\n\nNote: There is a subtlety when the sequence is being modified by the loop\n (this can only occur for mutable sequences, i.e. lists). An internal\n counter is used to keep track of which item is used next, and this\n is incremented on each iteration. When this counter has reached the\n length of the sequence the loop terminates. This means that if the\n suite deletes the current (or a previous) item from the sequence,\n the next item will be skipped (since it gets the index of the\n current item which has already been treated). Likewise, if the\n suite inserts an item in the sequence before the current item, the\n current item will be treated again the next time through the loop.\n This can lead to nasty bugs that can be avoided by making a\n temporary copy using a slice of the whole sequence, e.g.,\n\n for x in a[:]:\n if x < 0: a.remove(x)\n\n\nThe ``try`` statement\n=====================\n\nThe ``try`` statement specifies exception handlers and/or cleanup code\nfor a group of statements:\n\n try_stmt ::= try1_stmt | try2_stmt\n try1_stmt ::= "try" ":" suite\n ("except" [expression [("as" | ",") target]] ":" suite)+\n ["else" ":" suite]\n ["finally" ":" suite]\n try2_stmt ::= "try" ":" suite\n "finally" ":" suite\n\nChanged in version 2.5: In previous versions of Python,\n``try``...``except``...``finally`` did not work. ``try``...``except``\nhad to be nested in ``try``...``finally``.\n\nThe ``except`` clause(s) specify one or more exception handlers. When\nno exception occurs in the ``try`` clause, no exception handler is\nexecuted. When an exception occurs in the ``try`` suite, a search for\nan exception handler is started. This search inspects the except\nclauses in turn until one is found that matches the exception. An\nexpression-less except clause, if present, must be last; it matches\nany exception. For an except clause with an expression, that\nexpression is evaluated, and the clause matches the exception if the\nresulting object is "compatible" with the exception. An object is\ncompatible with an exception if it is the class or a base class of the\nexception object, a tuple containing an item compatible with the\nexception, or, in the (deprecated) case of string exceptions, is the\nraised string itself (note that the object identities must match, i.e.\nit must be the same string object, not just a string with the same\nvalue).\n\nIf no except clause matches the exception, the search for an exception\nhandler continues in the surrounding code and on the invocation stack.\n[1]\n\nIf the evaluation of an expression in the header of an except clause\nraises an exception, the original search for a handler is canceled and\na search starts for the new exception in the surrounding code and on\nthe call stack (it is treated as if the entire ``try`` statement\nraised the exception).\n\nWhen a matching except clause is found, the exception is assigned to\nthe target specified in that except clause, if present, and the except\nclause\'s suite is executed. All except clauses must have an\nexecutable block. When the end of this block is reached, execution\ncontinues normally after the entire try statement. (This means that\nif two nested handlers exist for the same exception, and the exception\noccurs in the try clause of the inner handler, the outer handler will\nnot handle the exception.)\n\nBefore an except clause\'s suite is executed, details about the\nexception are assigned to three variables in the ``sys`` module:\n``sys.exc_type`` receives the object identifying the exception;\n``sys.exc_value`` receives the exception\'s parameter;\n``sys.exc_traceback`` receives a traceback object (see section *The\nstandard type hierarchy*) identifying the point in the program where\nthe exception occurred. These details are also available through the\n``sys.exc_info()`` function, which returns a tuple ``(exc_type,\nexc_value, exc_traceback)``. Use of the corresponding variables is\ndeprecated in favor of this function, since their use is unsafe in a\nthreaded program. As of Python 1.5, the variables are restored to\ntheir previous values (before the call) when returning from a function\nthat handled an exception.\n\nThe optional ``else`` clause is executed if and when control flows off\nthe end of the ``try`` clause. [2] Exceptions in the ``else`` clause\nare not handled by the preceding ``except`` clauses.\n\nIf ``finally`` is present, it specifies a \'cleanup\' handler. The\n``try`` clause is executed, including any ``except`` and ``else``\nclauses. If an exception occurs in any of the clauses and is not\nhandled, the exception is temporarily saved. The ``finally`` clause is\nexecuted. If there is a saved exception, it is re-raised at the end\nof the ``finally`` clause. If the ``finally`` clause raises another\nexception or executes a ``return`` or ``break`` statement, the saved\nexception is lost. The exception information is not available to the\nprogram during execution of the ``finally`` clause.\n\nWhen a ``return``, ``break`` or ``continue`` statement is executed in\nthe ``try`` suite of a ``try``...``finally`` statement, the\n``finally`` clause is also executed \'on the way out.\' A ``continue``\nstatement is illegal in the ``finally`` clause. (The reason is a\nproblem with the current implementation --- this restriction may be\nlifted in the future).\n\nAdditional information on exceptions can be found in section\n*Exceptions*, and information on using the ``raise`` statement to\ngenerate exceptions may be found in section *The raise statement*.\n\n\nThe ``with`` statement\n======================\n\nNew in version 2.5.\n\nThe ``with`` statement is used to wrap the execution of a block with\nmethods defined by a context manager (see section *With Statement\nContext Managers*). This allows common\n``try``...``except``...``finally`` usage patterns to be encapsulated\nfor convenient reuse.\n\n with_stmt ::= "with" with_item ("," with_item)* ":" suite\n with_item ::= expression ["as" target]\n\nThe execution of the ``with`` statement with one "item" proceeds as\nfollows:\n\n1. The context expression (the expression given in the **with_item**)\n is evaluated to obtain a context manager.\n\n2. The context manager\'s ``__exit__()`` is loaded for later use.\n\n3. The context manager\'s ``__enter__()`` method is invoked.\n\n4. If a target was included in the ``with`` statement, the return\n value from ``__enter__()`` is assigned to it.\n\n Note: The ``with`` statement guarantees that if the ``__enter__()``\n method returns without an error, then ``__exit__()`` will always\n be called. Thus, if an error occurs during the assignment to the\n target list, it will be treated the same as an error occurring\n within the suite would be. See step 6 below.\n\n5. The suite is executed.\n\n6. The context manager\'s ``__exit__()`` method is invoked. If an\n exception caused the suite to be exited, its type, value, and\n traceback are passed as arguments to ``__exit__()``. Otherwise,\n three ``None`` arguments are supplied.\n\n If the suite was exited due to an exception, and the return value\n from the ``__exit__()`` method was false, the exception is\n reraised. If the return value was true, the exception is\n suppressed, and execution continues with the statement following\n the ``with`` statement.\n\n If the suite was exited for any reason other than an exception, the\n return value from ``__exit__()`` is ignored, and execution proceeds\n at the normal location for the kind of exit that was taken.\n\nWith more than one item, the context managers are processed as if\nmultiple ``with`` statements were nested:\n\n with A() as a, B() as b:\n suite\n\nis equivalent to\n\n with A() as a:\n with B() as b:\n suite\n\nNote: In Python 2.5, the ``with`` statement is only allowed when the\n ``with_statement`` feature has been enabled. It is always enabled\n in Python 2.6.\n\nChanged in version 2.7: Support for multiple context expressions.\n\nSee also:\n\n **PEP 0343** - The "with" statement\n The specification, background, and examples for the Python\n ``with`` statement.\n\n\nFunction definitions\n====================\n\nA function definition defines a user-defined function object (see\nsection *The standard type hierarchy*):\n\n decorated ::= decorators (classdef | funcdef)\n decorators ::= decorator+\n decorator ::= "@" dotted_name ["(" [argument_list [","]] ")"] NEWLINE\n funcdef ::= "def" funcname "(" [parameter_list] ")" ":" suite\n dotted_name ::= identifier ("." identifier)*\n parameter_list ::= (defparameter ",")*\n ( "*" identifier [, "**" identifier]\n | "**" identifier\n | defparameter [","] )\n defparameter ::= parameter ["=" expression]\n sublist ::= parameter ("," parameter)* [","]\n parameter ::= identifier | "(" sublist ")"\n funcname ::= identifier\n\nA function definition is an executable statement. Its execution binds\nthe function name in the current local namespace to a function object\n(a wrapper around the executable code for the function). This\nfunction object contains a reference to the current global namespace\nas the global namespace to be used when the function is called.\n\nThe function definition does not execute the function body; this gets\nexecuted only when the function is called. [3]\n\nA function definition may be wrapped by one or more *decorator*\nexpressions. Decorator expressions are evaluated when the function is\ndefined, in the scope that contains the function definition. The\nresult must be a callable, which is invoked with the function object\nas the only argument. The returned value is bound to the function name\ninstead of the function object. Multiple decorators are applied in\nnested fashion. For example, the following code:\n\n @f1(arg)\n @f2\n def func(): pass\n\nis equivalent to:\n\n def func(): pass\n func = f1(arg)(f2(func))\n\nWhen one or more top-level parameters have the form *parameter* ``=``\n*expression*, the function is said to have "default parameter values."\nFor a parameter with a default value, the corresponding argument may\nbe omitted from a call, in which case the parameter\'s default value is\nsubstituted. If a parameter has a default value, all following\nparameters must also have a default value --- this is a syntactic\nrestriction that is not expressed by the grammar.\n\n**Default parameter values are evaluated when the function definition\nis executed.** This means that the expression is evaluated once, when\nthe function is defined, and that that same "pre-computed" value is\nused for each call. This is especially important to understand when a\ndefault parameter is a mutable object, such as a list or a dictionary:\nif the function modifies the object (e.g. by appending an item to a\nlist), the default value is in effect modified. This is generally not\nwhat was intended. A way around this is to use ``None`` as the\ndefault, and explicitly test for it in the body of the function, e.g.:\n\n def whats_on_the_telly(penguin=None):\n if penguin is None:\n penguin = []\n penguin.append("property of the zoo")\n return penguin\n\nFunction call semantics are described in more detail in section\n*Calls*. A function call always assigns values to all parameters\nmentioned in the parameter list, either from position arguments, from\nkeyword arguments, or from default values. If the form\n"``*identifier``" is present, it is initialized to a tuple receiving\nany excess positional parameters, defaulting to the empty tuple. If\nthe form "``**identifier``" is present, it is initialized to a new\ndictionary receiving any excess keyword arguments, defaulting to a new\nempty dictionary.\n\nIt is also possible to create anonymous functions (functions not bound\nto a name), for immediate use in expressions. This uses lambda forms,\ndescribed in section *Lambdas*. Note that the lambda form is merely a\nshorthand for a simplified function definition; a function defined in\na "``def``" statement can be passed around or assigned to another name\njust like a function defined by a lambda form. The "``def``" form is\nactually more powerful since it allows the execution of multiple\nstatements.\n\n**Programmer\'s note:** Functions are first-class objects. A "``def``"\nform executed inside a function definition defines a local function\nthat can be returned or passed around. Free variables used in the\nnested function can access the local variables of the function\ncontaining the def. See section *Naming and binding* for details.\n\n\nClass definitions\n=================\n\nA class definition defines a class object (see section *The standard\ntype hierarchy*):\n\n classdef ::= "class" classname [inheritance] ":" suite\n inheritance ::= "(" [expression_list] ")"\n classname ::= identifier\n\nA class definition is an executable statement. It first evaluates the\ninheritance list, if present. Each item in the inheritance list\nshould evaluate to a class object or class type which allows\nsubclassing. The class\'s suite is then executed in a new execution\nframe (see section *Naming and binding*), using a newly created local\nnamespace and the original global namespace. (Usually, the suite\ncontains only function definitions.) When the class\'s suite finishes\nexecution, its execution frame is discarded but its local namespace is\nsaved. [4] A class object is then created using the inheritance list\nfor the base classes and the saved local namespace for the attribute\ndictionary. The class name is bound to this class object in the\noriginal local namespace.\n\n**Programmer\'s note:** Variables defined in the class definition are\nclass variables; they are shared by all instances. To create instance\nvariables, they can be set in a method with ``self.name = value``.\nBoth class and instance variables are accessible through the notation\n"``self.name``", and an instance variable hides a class variable with\nthe same name when accessed in this way. Class variables can be used\nas defaults for instance variables, but using mutable values there can\nlead to unexpected results. For *new-style class*es, descriptors can\nbe used to create instance variables with different implementation\ndetails.\n\nClass definitions, like function definitions, may be wrapped by one or\nmore *decorator* expressions. The evaluation rules for the decorator\nexpressions are the same as for functions. The result must be a class\nobject, which is then bound to the class name.\n\n-[ Footnotes ]-\n\n[1] The exception is propagated to the invocation stack only if there\n is no ``finally`` clause that negates the exception.\n\n[2] Currently, control "flows off the end" except in the case of an\n exception or the execution of a ``return``, ``continue``, or\n ``break`` statement.\n\n[3] A string literal appearing as the first statement in the function\n body is transformed into the function\'s ``__doc__`` attribute and\n therefore the function\'s *docstring*.\n\n[4] A string literal appearing as the first statement in the class\n body is transformed into the namespace\'s ``__doc__`` item and\n therefore the class\'s *docstring*.\n', 'context-managers': u'\nWith Statement Context Managers\n*******************************\n\nNew in version 2.5.\n\nA *context manager* is an object that defines the runtime context to\nbe established when executing a ``with`` statement. The context\nmanager handles the entry into, and the exit from, the desired runtime\ncontext for the execution of the block of code. Context managers are\nnormally invoked using the ``with`` statement (described in section\n*The with statement*), but can also be used by directly invoking their\nmethods.\n\nTypical uses of context managers include saving and restoring various\nkinds of global state, locking and unlocking resources, closing opened\nfiles, etc.\n\nFor more information on context managers, see *Context Manager Types*.\n\nobject.__enter__(self)\n\n Enter the runtime context related to this object. The ``with``\n statement will bind this method\'s return value to the target(s)\n specified in the ``as`` clause of the statement, if any.\n\nobject.__exit__(self, exc_type, exc_value, traceback)\n\n Exit the runtime context related to this object. The parameters\n describe the exception that caused the context to be exited. If the\n context was exited without an exception, all three arguments will\n be ``None``.\n\n If an exception is supplied, and the method wishes to suppress the\n exception (i.e., prevent it from being propagated), it should\n return a true value. Otherwise, the exception will be processed\n normally upon exit from this method.\n\n Note that ``__exit__()`` methods should not reraise the passed-in\n exception; this is the caller\'s responsibility.\n\nSee also:\n\n **PEP 0343** - The "with" statement\n The specification, background, and examples for the Python\n ``with`` statement.\n', 'continue': u'\nThe ``continue`` statement\n**************************\n\n continue_stmt ::= "continue"\n\n``continue`` may only occur syntactically nested in a ``for`` or\n``while`` loop, but not nested in a function or class definition or\n``finally`` clause within that loop. It continues with the next cycle\nof the nearest enclosing loop.\n\nWhen ``continue`` passes control out of a ``try`` statement with a\n``finally`` clause, that ``finally`` clause is executed before really\nstarting the next loop cycle.\n', 'conversions': u'\nArithmetic conversions\n**********************\n\nWhen a description of an arithmetic operator below uses the phrase\n"the numeric arguments are converted to a common type," the arguments\nare coerced using the coercion rules listed at *Coercion rules*. If\nboth arguments are standard numeric types, the following coercions are\napplied:\n\n* If either argument is a complex number, the other is converted to\n complex;\n\n* otherwise, if either argument is a floating point number, the other\n is converted to floating point;\n\n* otherwise, if either argument is a long integer, the other is\n converted to long integer;\n\n* otherwise, both must be plain integers and no conversion is\n necessary.\n\nSome additional rules apply for certain operators (e.g., a string left\nargument to the \'%\' operator). Extensions can define their own\ncoercions.\n', 'customization': u'\nBasic customization\n*******************\n\nobject.__new__(cls[, ...])\n\n Called to create a new instance of class *cls*. ``__new__()`` is a\n static method (special-cased so you need not declare it as such)\n that takes the class of which an instance was requested as its\n first argument. The remaining arguments are those passed to the\n object constructor expression (the call to the class). The return\n value of ``__new__()`` should be the new object instance (usually\n an instance of *cls*).\n\n Typical implementations create a new instance of the class by\n invoking the superclass\'s ``__new__()`` method using\n ``super(currentclass, cls).__new__(cls[, ...])`` with appropriate\n arguments and then modifying the newly-created instance as\n necessary before returning it.\n\n If ``__new__()`` returns an instance of *cls*, then the new\n instance\'s ``__init__()`` method will be invoked like\n ``__init__(self[, ...])``, where *self* is the new instance and the\n remaining arguments are the same as were passed to ``__new__()``.\n\n If ``__new__()`` does not return an instance of *cls*, then the new\n instance\'s ``__init__()`` method will not be invoked.\n\n ``__new__()`` is intended mainly to allow subclasses of immutable\n types (like int, str, or tuple) to customize instance creation. It\n is also commonly overridden in custom metaclasses in order to\n customize class creation.\n\nobject.__init__(self[, ...])\n\n Called when the instance is created. The arguments are those\n passed to the class constructor expression. If a base class has an\n ``__init__()`` method, the derived class\'s ``__init__()`` method,\n if any, must explicitly call it to ensure proper initialization of\n the base class part of the instance; for example:\n ``BaseClass.__init__(self, [args...])``. As a special constraint\n on constructors, no value may be returned; doing so will cause a\n ``TypeError`` to be raised at runtime.\n\nobject.__del__(self)\n\n Called when the instance is about to be destroyed. This is also\n called a destructor. If a base class has a ``__del__()`` method,\n the derived class\'s ``__del__()`` method, if any, must explicitly\n call it to ensure proper deletion of the base class part of the\n instance. Note that it is possible (though not recommended!) for\n the ``__del__()`` method to postpone destruction of the instance by\n creating a new reference to it. It may then be called at a later\n time when this new reference is deleted. It is not guaranteed that\n ``__del__()`` methods are called for objects that still exist when\n the interpreter exits.\n\n Note: ``del x`` doesn\'t directly call ``x.__del__()`` --- the former\n decrements the reference count for ``x`` by one, and the latter\n is only called when ``x``\'s reference count reaches zero. Some\n common situations that may prevent the reference count of an\n object from going to zero include: circular references between\n objects (e.g., a doubly-linked list or a tree data structure with\n parent and child pointers); a reference to the object on the\n stack frame of a function that caught an exception (the traceback\n stored in ``sys.exc_traceback`` keeps the stack frame alive); or\n a reference to the object on the stack frame that raised an\n unhandled exception in interactive mode (the traceback stored in\n ``sys.last_traceback`` keeps the stack frame alive). The first\n situation can only be remedied by explicitly breaking the cycles;\n the latter two situations can be resolved by storing ``None`` in\n ``sys.exc_traceback`` or ``sys.last_traceback``. Circular\n references which are garbage are detected when the option cycle\n detector is enabled (it\'s on by default), but can only be cleaned\n up if there are no Python-level ``__del__()`` methods involved.\n Refer to the documentation for the ``gc`` module for more\n information about how ``__del__()`` methods are handled by the\n cycle detector, particularly the description of the ``garbage``\n value.\n\n Warning: Due to the precarious circumstances under which ``__del__()``\n methods are invoked, exceptions that occur during their execution\n are ignored, and a warning is printed to ``sys.stderr`` instead.\n Also, when ``__del__()`` is invoked in response to a module being\n deleted (e.g., when execution of the program is done), other\n globals referenced by the ``__del__()`` method may already have\n been deleted or in the process of being torn down (e.g. the\n import machinery shutting down). For this reason, ``__del__()``\n methods should do the absolute minimum needed to maintain\n external invariants. Starting with version 1.5, Python\n guarantees that globals whose name begins with a single\n underscore are deleted from their module before other globals are\n deleted; if no other references to such globals exist, this may\n help in assuring that imported modules are still available at the\n time when the ``__del__()`` method is called.\n\nobject.__repr__(self)\n\n Called by the ``repr()`` built-in function and by string\n conversions (reverse quotes) to compute the "official" string\n representation of an object. If at all possible, this should look\n like a valid Python expression that could be used to recreate an\n object with the same value (given an appropriate environment). If\n this is not possible, a string of the form ``<...some useful\n description...>`` should be returned. The return value must be a\n string object. If a class defines ``__repr__()`` but not\n ``__str__()``, then ``__repr__()`` is also used when an "informal"\n string representation of instances of that class is required.\n\n This is typically used for debugging, so it is important that the\n representation is information-rich and unambiguous.\n\nobject.__str__(self)\n\n Called by the ``str()`` built-in function and by the ``print``\n statement to compute the "informal" string representation of an\n object. This differs from ``__repr__()`` in that it does not have\n to be a valid Python expression: a more convenient or concise\n representation may be used instead. The return value must be a\n string object.\n\nobject.__lt__(self, other)\nobject.__le__(self, other)\nobject.__eq__(self, other)\nobject.__ne__(self, other)\nobject.__gt__(self, other)\nobject.__ge__(self, other)\n\n New in version 2.1.\n\n These are the so-called "rich comparison" methods, and are called\n for comparison operators in preference to ``__cmp__()`` below. The\n correspondence between operator symbols and method names is as\n follows: ``x<y`` calls ``x.__lt__(y)``, ``x<=y`` calls\n ``x.__le__(y)``, ``x==y`` calls ``x.__eq__(y)``, ``x!=y`` and\n ``x<>y`` call ``x.__ne__(y)``, ``x>y`` calls ``x.__gt__(y)``, and\n ``x>=y`` calls ``x.__ge__(y)``.\n\n A rich comparison method may return the singleton\n ``NotImplemented`` if it does not implement the operation for a\n given pair of arguments. By convention, ``False`` and ``True`` are\n returned for a successful comparison. However, these methods can\n return any value, so if the comparison operator is used in a\n Boolean context (e.g., in the condition of an ``if`` statement),\n Python will call ``bool()`` on the value to determine if the result\n is true or false.\n\n There are no implied relationships among the comparison operators.\n The truth of ``x==y`` does not imply that ``x!=y`` is false.\n Accordingly, when defining ``__eq__()``, one should also define\n ``__ne__()`` so that the operators will behave as expected. See\n the paragraph on ``__hash__()`` for some important notes on\n creating *hashable* objects which support custom comparison\n operations and are usable as dictionary keys.\n\n There are no swapped-argument versions of these methods (to be used\n when the left argument does not support the operation but the right\n argument does); rather, ``__lt__()`` and ``__gt__()`` are each\n other\'s reflection, ``__le__()`` and ``__ge__()`` are each other\'s\n reflection, and ``__eq__()`` and ``__ne__()`` are their own\n reflection.\n\n Arguments to rich comparison methods are never coerced.\n\n To automatically generate ordering operations from a single root\n operation, see ``functools.total_ordering()``.\n\nobject.__cmp__(self, other)\n\n Called by comparison operations if rich comparison (see above) is\n not defined. Should return a negative integer if ``self < other``,\n zero if ``self == other``, a positive integer if ``self > other``.\n If no ``__cmp__()``, ``__eq__()`` or ``__ne__()`` operation is\n defined, class instances are compared by object identity\n ("address"). See also the description of ``__hash__()`` for some\n important notes on creating *hashable* objects which support custom\n comparison operations and are usable as dictionary keys. (Note: the\n restriction that exceptions are not propagated by ``__cmp__()`` has\n been removed since Python 1.5.)\n\nobject.__rcmp__(self, other)\n\n Changed in version 2.1: No longer supported.\n\nobject.__hash__(self)\n\n Called by built-in function ``hash()`` and for operations on\n members of hashed collections including ``set``, ``frozenset``, and\n ``dict``. ``__hash__()`` should return an integer. The only\n required property is that objects which compare equal have the same\n hash value; it is advised to somehow mix together (e.g. using\n exclusive or) the hash values for the components of the object that\n also play a part in comparison of objects.\n\n If a class does not define a ``__cmp__()`` or ``__eq__()`` method\n it should not define a ``__hash__()`` operation either; if it\n defines ``__cmp__()`` or ``__eq__()`` but not ``__hash__()``, its\n instances will not be usable in hashed collections. If a class\n defines mutable objects and implements a ``__cmp__()`` or\n ``__eq__()`` method, it should not implement ``__hash__()``, since\n hashable collection implementations require that a object\'s hash\n value is immutable (if the object\'s hash value changes, it will be\n in the wrong hash bucket).\n\n User-defined classes have ``__cmp__()`` and ``__hash__()`` methods\n by default; with them, all objects compare unequal (except with\n themselves) and ``x.__hash__()`` returns ``id(x)``.\n\n Classes which inherit a ``__hash__()`` method from a parent class\n but change the meaning of ``__cmp__()`` or ``__eq__()`` such that\n the hash value returned is no longer appropriate (e.g. by switching\n to a value-based concept of equality instead of the default\n identity based equality) can explicitly flag themselves as being\n unhashable by setting ``__hash__ = None`` in the class definition.\n Doing so means that not only will instances of the class raise an\n appropriate ``TypeError`` when a program attempts to retrieve their\n hash value, but they will also be correctly identified as\n unhashable when checking ``isinstance(obj, collections.Hashable)``\n (unlike classes which define their own ``__hash__()`` to explicitly\n raise ``TypeError``).\n\n Changed in version 2.5: ``__hash__()`` may now also return a long\n integer object; the 32-bit integer is then derived from the hash of\n that object.\n\n Changed in version 2.6: ``__hash__`` may now be set to ``None`` to\n explicitly flag instances of a class as unhashable.\n\nobject.__nonzero__(self)\n\n Called to implement truth value testing and the built-in operation\n ``bool()``; should return ``False`` or ``True``, or their integer\n equivalents ``0`` or ``1``. When this method is not defined,\n ``__len__()`` is called, if it is defined, and the object is\n considered true if its result is nonzero. If a class defines\n neither ``__len__()`` nor ``__nonzero__()``, all its instances are\n considered true.\n\nobject.__unicode__(self)\n\n Called to implement ``unicode()`` built-in; should return a Unicode\n object. When this method is not defined, string conversion is\n attempted, and the result of string conversion is converted to\n Unicode using the system default encoding.\n', - 'debugger': u'\n``pdb`` --- The Python Debugger\n*******************************\n\nThe module ``pdb`` defines an interactive source code debugger for\nPython programs. It supports setting (conditional) breakpoints and\nsingle stepping at the source line level, inspection of stack frames,\nsource code listing, and evaluation of arbitrary Python code in the\ncontext of any stack frame. It also supports post-mortem debugging\nand can be called under program control.\n\nThe debugger is extensible --- it is actually defined as the class\n``Pdb``. This is currently undocumented but easily understood by\nreading the source. The extension interface uses the modules ``bdb``\nand ``cmd``.\n\nThe debugger\'s prompt is ``(Pdb)``. Typical usage to run a program\nunder control of the debugger is:\n\n >>> import pdb\n >>> import mymodule\n >>> pdb.run(\'mymodule.test()\')\n > <string>(0)?()\n (Pdb) continue\n > <string>(1)?()\n (Pdb) continue\n NameError: \'spam\'\n > <string>(1)?()\n (Pdb)\n\n``pdb.py`` can also be invoked as a script to debug other scripts.\nFor example:\n\n python -m pdb myscript.py\n\nWhen invoked as a script, pdb will automatically enter post-mortem\ndebugging if the program being debugged exits abnormally. After post-\nmortem debugging (or after normal exit of the program), pdb will\nrestart the program. Automatic restarting preserves pdb\'s state (such\nas breakpoints) and in most cases is more useful than quitting the\ndebugger upon program\'s exit.\n\nNew in version 2.4: Restarting post-mortem behavior added.\n\nThe typical usage to break into the debugger from a running program is\nto insert\n\n import pdb; pdb.set_trace()\n\nat the location you want to break into the debugger. You can then\nstep through the code following this statement, and continue running\nwithout the debugger using the ``c`` command.\n\nThe typical usage to inspect a crashed program is:\n\n >>> import pdb\n >>> import mymodule\n >>> mymodule.test()\n Traceback (most recent call last):\n File "<stdin>", line 1, in ?\n File "./mymodule.py", line 4, in test\n test2()\n File "./mymodule.py", line 3, in test2\n print spam\n NameError: spam\n >>> pdb.pm()\n > ./mymodule.py(3)test2()\n -> print spam\n (Pdb)\n\nThe module defines the following functions; each enters the debugger\nin a slightly different way:\n\npdb.run(statement[, globals[, locals]])\n\n Execute the *statement* (given as a string) under debugger control.\n The debugger prompt appears before any code is executed; you can\n set breakpoints and type ``continue``, or you can step through the\n statement using ``step`` or ``next`` (all these commands are\n explained below). The optional *globals* and *locals* arguments\n specify the environment in which the code is executed; by default\n the dictionary of the module ``__main__`` is used. (See the\n explanation of the ``exec`` statement or the ``eval()`` built-in\n function.)\n\npdb.runeval(expression[, globals[, locals]])\n\n Evaluate the *expression* (given as a string) under debugger\n control. When ``runeval()`` returns, it returns the value of the\n expression. Otherwise this function is similar to ``run()``.\n\npdb.runcall(function[, argument, ...])\n\n Call the *function* (a function or method object, not a string)\n with the given arguments. When ``runcall()`` returns, it returns\n whatever the function call returned. The debugger prompt appears\n as soon as the function is entered.\n\npdb.set_trace()\n\n Enter the debugger at the calling stack frame. This is useful to\n hard-code a breakpoint at a given point in a program, even if the\n code is not otherwise being debugged (e.g. when an assertion\n fails).\n\npdb.post_mortem([traceback])\n\n Enter post-mortem debugging of the given *traceback* object. If no\n *traceback* is given, it uses the one of the exception that is\n currently being handled (an exception must be being handled if the\n default is to be used).\n\npdb.pm()\n\n Enter post-mortem debugging of the traceback found in\n ``sys.last_traceback``.\n\nThe ``run_*`` functions and ``set_trace()`` are aliases for\ninstantiating the ``Pdb`` class and calling the method of the same\nname. If you want to access further features, you have to do this\nyourself:\n\nclass class pdb.Pdb(completekey=\'tab\', stdin=None, stdout=None, skip=None)\n\n ``Pdb`` is the debugger class.\n\n The *completekey*, *stdin* and *stdout* arguments are passed to the\n underlying ``cmd.Cmd`` class; see the description there.\n\n The *skip* argument, if given, must be an iterable of glob-style\n module name patterns. The debugger will not step into frames that\n originate in a module that matches one of these patterns. [1]\n\n Example call to enable tracing with *skip*:\n\n import pdb; pdb.Pdb(skip=[\'django.*\']).set_trace()\n\n New in version 2.7: The *skip* argument.\n\n run(statement[, globals[, locals]])\n runeval(expression[, globals[, locals]])\n runcall(function[, argument, ...])\n set_trace()\n\n See the documentation for the functions explained above.\n', + 'debugger': u'\n``pdb`` --- The Python Debugger\n*******************************\n\nThe module ``pdb`` defines an interactive source code debugger for\nPython programs. It supports setting (conditional) breakpoints and\nsingle stepping at the source line level, inspection of stack frames,\nsource code listing, and evaluation of arbitrary Python code in the\ncontext of any stack frame. It also supports post-mortem debugging\nand can be called under program control.\n\nThe debugger is extensible --- it is actually defined as the class\n``Pdb``. This is currently undocumented but easily understood by\nreading the source. The extension interface uses the modules ``bdb``\nand ``cmd``.\n\nThe debugger\'s prompt is ``(Pdb)``. Typical usage to run a program\nunder control of the debugger is:\n\n >>> import pdb\n >>> import mymodule\n >>> pdb.run(\'mymodule.test()\')\n > <string>(0)?()\n (Pdb) continue\n > <string>(1)?()\n (Pdb) continue\n NameError: \'spam\'\n > <string>(1)?()\n (Pdb)\n\n``pdb.py`` can also be invoked as a script to debug other scripts.\nFor example:\n\n python -m pdb myscript.py\n\nWhen invoked as a script, pdb will automatically enter post-mortem\ndebugging if the program being debugged exits abnormally. After post-\nmortem debugging (or after normal exit of the program), pdb will\nrestart the program. Automatic restarting preserves pdb\'s state (such\nas breakpoints) and in most cases is more useful than quitting the\ndebugger upon program\'s exit.\n\nNew in version 2.4: Restarting post-mortem behavior added.\n\nThe typical usage to break into the debugger from a running program is\nto insert\n\n import pdb; pdb.set_trace()\n\nat the location you want to break into the debugger. You can then\nstep through the code following this statement, and continue running\nwithout the debugger using the ``c`` command.\n\nThe typical usage to inspect a crashed program is:\n\n >>> import pdb\n >>> import mymodule\n >>> mymodule.test()\n Traceback (most recent call last):\n File "<stdin>", line 1, in ?\n File "./mymodule.py", line 4, in test\n test2()\n File "./mymodule.py", line 3, in test2\n print spam\n NameError: spam\n >>> pdb.pm()\n > ./mymodule.py(3)test2()\n -> print spam\n (Pdb)\n\nThe module defines the following functions; each enters the debugger\nin a slightly different way:\n\npdb.run(statement[, globals[, locals]])\n\n Execute the *statement* (given as a string) under debugger control.\n The debugger prompt appears before any code is executed; you can\n set breakpoints and type ``continue``, or you can step through the\n statement using ``step`` or ``next`` (all these commands are\n explained below). The optional *globals* and *locals* arguments\n specify the environment in which the code is executed; by default\n the dictionary of the module ``__main__`` is used. (See the\n explanation of the ``exec`` statement or the ``eval()`` built-in\n function.)\n\npdb.runeval(expression[, globals[, locals]])\n\n Evaluate the *expression* (given as a string) under debugger\n control. When ``runeval()`` returns, it returns the value of the\n expression. Otherwise this function is similar to ``run()``.\n\npdb.runcall(function[, argument, ...])\n\n Call the *function* (a function or method object, not a string)\n with the given arguments. When ``runcall()`` returns, it returns\n whatever the function call returned. The debugger prompt appears\n as soon as the function is entered.\n\npdb.set_trace()\n\n Enter the debugger at the calling stack frame. This is useful to\n hard-code a breakpoint at a given point in a program, even if the\n code is not otherwise being debugged (e.g. when an assertion\n fails).\n\npdb.post_mortem([traceback])\n\n Enter post-mortem debugging of the given *traceback* object. If no\n *traceback* is given, it uses the one of the exception that is\n currently being handled (an exception must be being handled if the\n default is to be used).\n\npdb.pm()\n\n Enter post-mortem debugging of the traceback found in\n ``sys.last_traceback``.\n\nThe ``run*`` functions and ``set_trace()`` are aliases for\ninstantiating the ``Pdb`` class and calling the method of the same\nname. If you want to access further features, you have to do this\nyourself:\n\nclass class pdb.Pdb(completekey=\'tab\', stdin=None, stdout=None, skip=None)\n\n ``Pdb`` is the debugger class.\n\n The *completekey*, *stdin* and *stdout* arguments are passed to the\n underlying ``cmd.Cmd`` class; see the description there.\n\n The *skip* argument, if given, must be an iterable of glob-style\n module name patterns. The debugger will not step into frames that\n originate in a module that matches one of these patterns. [1]\n\n Example call to enable tracing with *skip*:\n\n import pdb; pdb.Pdb(skip=[\'django.*\']).set_trace()\n\n New in version 2.7: The *skip* argument.\n\n run(statement[, globals[, locals]])\n runeval(expression[, globals[, locals]])\n runcall(function[, argument, ...])\n set_trace()\n\n See the documentation for the functions explained above.\n', 'del': u'\nThe ``del`` statement\n*********************\n\n del_stmt ::= "del" target_list\n\nDeletion is recursively defined very similar to the way assignment is\ndefined. Rather that spelling it out in full details, here are some\nhints.\n\nDeletion of a target list recursively deletes each target, from left\nto right.\n\nDeletion of a name removes the binding of that name from the local or\nglobal namespace, depending on whether the name occurs in a ``global``\nstatement in the same code block. If the name is unbound, a\n``NameError`` exception will be raised.\n\nIt is illegal to delete a name from the local namespace if it occurs\nas a free variable in a nested block.\n\nDeletion of attribute references, subscriptions and slicings is passed\nto the primary object involved; deletion of a slicing is in general\nequivalent to assignment of an empty slice of the right type (but even\nthis is determined by the sliced object).\n', 'dict': u'\nDictionary displays\n*******************\n\nA dictionary display is a possibly empty series of key/datum pairs\nenclosed in curly braces:\n\n dict_display ::= "{" [key_datum_list | dict_comprehension] "}"\n key_datum_list ::= key_datum ("," key_datum)* [","]\n key_datum ::= expression ":" expression\n dict_comprehension ::= expression ":" expression comp_for\n\nA dictionary display yields a new dictionary object.\n\nIf a comma-separated sequence of key/datum pairs is given, they are\nevaluated from left to right to define the entries of the dictionary:\neach key object is used as a key into the dictionary to store the\ncorresponding datum. This means that you can specify the same key\nmultiple times in the key/datum list, and the final dictionary\'s value\nfor that key will be the last one given.\n\nA dict comprehension, in contrast to list and set comprehensions,\nneeds two expressions separated with a colon followed by the usual\n"for" and "if" clauses. When the comprehension is run, the resulting\nkey and value elements are inserted in the new dictionary in the order\nthey are produced.\n\nRestrictions on the types of the key values are listed earlier in\nsection *The standard type hierarchy*. (To summarize, the key type\nshould be *hashable*, which excludes all mutable objects.) Clashes\nbetween duplicate keys are not detected; the last datum (textually\nrightmost in the display) stored for a given key value prevails.\n', 'dynamic-features': u'\nInteraction with dynamic features\n*********************************\n\nThere are several cases where Python statements are illegal when used\nin conjunction with nested scopes that contain free variables.\n\nIf a variable is referenced in an enclosing scope, it is illegal to\ndelete the name. An error will be reported at compile time.\n\nIf the wild card form of import --- ``import *`` --- is used in a\nfunction and the function contains or is a nested block with free\nvariables, the compiler will raise a ``SyntaxError``.\n\nIf ``exec`` is used in a function and the function contains or is a\nnested block with free variables, the compiler will raise a\n``SyntaxError`` unless the exec explicitly specifies the local\nnamespace for the ``exec``. (In other words, ``exec obj`` would be\nillegal, but ``exec obj in ns`` would be legal.)\n\nThe ``eval()``, ``execfile()``, and ``input()`` functions and the\n``exec`` statement do not have access to the full environment for\nresolving names. Names may be resolved in the local and global\nnamespaces of the caller. Free variables are not resolved in the\nnearest enclosing namespace, but in the global namespace. [1] The\n``exec`` statement and the ``eval()`` and ``execfile()`` functions\nhave optional arguments to override the global and local namespace.\nIf only one namespace is specified, it is used for both.\n', 'else': u'\nThe ``if`` statement\n********************\n\nThe ``if`` statement is used for conditional execution:\n\n if_stmt ::= "if" expression ":" suite\n ( "elif" expression ":" suite )*\n ["else" ":" suite]\n\nIt selects exactly one of the suites by evaluating the expressions one\nby one until one is found to be true (see section *Boolean operations*\nfor the definition of true and false); then that suite is executed\n(and no other part of the ``if`` statement is executed or evaluated).\nIf all expressions are false, the suite of the ``else`` clause, if\npresent, is executed.\n', 'exceptions': u'\nExceptions\n**********\n\nExceptions are a means of breaking out of the normal flow of control\nof a code block in order to handle errors or other exceptional\nconditions. An exception is *raised* at the point where the error is\ndetected; it may be *handled* by the surrounding code block or by any\ncode block that directly or indirectly invoked the code block where\nthe error occurred.\n\nThe Python interpreter raises an exception when it detects a run-time\nerror (such as division by zero). A Python program can also\nexplicitly raise an exception with the ``raise`` statement. Exception\nhandlers are specified with the ``try`` ... ``except`` statement. The\n``finally`` clause of such a statement can be used to specify cleanup\ncode which does not handle the exception, but is executed whether an\nexception occurred or not in the preceding code.\n\nPython uses the "termination" model of error handling: an exception\nhandler can find out what happened and continue execution at an outer\nlevel, but it cannot repair the cause of the error and retry the\nfailing operation (except by re-entering the offending piece of code\nfrom the top).\n\nWhen an exception is not handled at all, the interpreter terminates\nexecution of the program, or returns to its interactive main loop. In\neither case, it prints a stack backtrace, except when the exception is\n``SystemExit``.\n\nExceptions are identified by class instances. The ``except`` clause\nis selected depending on the class of the instance: it must reference\nthe class of the instance or a base class thereof. The instance can\nbe received by the handler and can carry additional information about\nthe exceptional condition.\n\nExceptions can also be identified by strings, in which case the\n``except`` clause is selected by object identity. An arbitrary value\ncan be raised along with the identifying string which can be passed to\nthe handler.\n\nNote: Messages to exceptions are not part of the Python API. Their\n contents may change from one version of Python to the next without\n warning and should not be relied on by code which will run under\n multiple versions of the interpreter.\n\nSee also the description of the ``try`` statement in section *The try\nstatement* and ``raise`` statement in section *The raise statement*.\n\n-[ Footnotes ]-\n\n[1] This limitation occurs because the code that is executed by these\n operations is not available at the time the module is compiled.\n', 'exec': u'\nThe ``exec`` statement\n**********************\n\n exec_stmt ::= "exec" or_expr ["in" expression ["," expression]]\n\nThis statement supports dynamic execution of Python code. The first\nexpression should evaluate to either a string, an open file object, or\na code object. If it is a string, the string is parsed as a suite of\nPython statements which is then executed (unless a syntax error\noccurs). [1] If it is an open file, the file is parsed until EOF and\nexecuted. If it is a code object, it is simply executed. In all\ncases, the code that\'s executed is expected to be valid as file input\n(see section *File input*). Be aware that the ``return`` and\n``yield`` statements may not be used outside of function definitions\neven within the context of code passed to the ``exec`` statement.\n\nIn all cases, if the optional parts are omitted, the code is executed\nin the current scope. If only the first expression after ``in`` is\nspecified, it should be a dictionary, which will be used for both the\nglobal and the local variables. If two expressions are given, they\nare used for the global and local variables, respectively. If\nprovided, *locals* can be any mapping object.\n\nChanged in version 2.4: Formerly, *locals* was required to be a\ndictionary.\n\nAs a side effect, an implementation may insert additional keys into\nthe dictionaries given besides those corresponding to variable names\nset by the executed code. For example, the current implementation may\nadd a reference to the dictionary of the built-in module\n``__builtin__`` under the key ``__builtins__`` (!).\n\n**Programmer\'s hints:** dynamic evaluation of expressions is supported\nby the built-in function ``eval()``. The built-in functions\n``globals()`` and ``locals()`` return the current global and local\ndictionary, respectively, which may be useful to pass around for use\nby ``exec``.\n\n-[ Footnotes ]-\n\n[1] Note that the parser only accepts the Unix-style end of line\n convention. If you are reading the code from a file, make sure to\n use universal newline mode to convert Windows or Mac-style\n newlines.\n', - 'execmodel': u'\nExecution model\n***************\n\n\nNaming and binding\n==================\n\n*Names* refer to objects. Names are introduced by name binding\noperations. Each occurrence of a name in the program text refers to\nthe *binding* of that name established in the innermost function block\ncontaining the use.\n\nA *block* is a piece of Python program text that is executed as a\nunit. The following are blocks: a module, a function body, and a class\ndefinition. Each command typed interactively is a block. A script\nfile (a file given as standard input to the interpreter or specified\non the interpreter command line the first argument) is a code block.\nA script command (a command specified on the interpreter command line\nwith the \'**-c**\' option) is a code block. The file read by the\nbuilt-in function ``execfile()`` is a code block. The string argument\npassed to the built-in function ``eval()`` and to the ``exec``\nstatement is a code block. The expression read and evaluated by the\nbuilt-in function ``input()`` is a code block.\n\nA code block is executed in an *execution frame*. A frame contains\nsome administrative information (used for debugging) and determines\nwhere and how execution continues after the code block\'s execution has\ncompleted.\n\nA *scope* defines the visibility of a name within a block. If a local\nvariable is defined in a block, its scope includes that block. If the\ndefinition occurs in a function block, the scope extends to any blocks\ncontained within the defining one, unless a contained block introduces\na different binding for the name. The scope of names defined in a\nclass block is limited to the class block; it does not extend to the\ncode blocks of methods -- this includes generator expressions since\nthey are implemented using a function scope. This means that the\nfollowing will fail:\n\n class A:\n a = 42\n b = list(a + i for i in range(10))\n\nWhen a name is used in a code block, it is resolved using the nearest\nenclosing scope. The set of all such scopes visible to a code block\nis called the block\'s *environment*.\n\nIf a name is bound in a block, it is a local variable of that block.\nIf a name is bound at the module level, it is a global variable. (The\nvariables of the module code block are local and global.) If a\nvariable is used in a code block but not defined there, it is a *free\nvariable*.\n\nWhen a name is not found at all, a ``NameError`` exception is raised.\nIf the name refers to a local variable that has not been bound, a\n``UnboundLocalError`` exception is raised. ``UnboundLocalError`` is a\nsubclass of ``NameError``.\n\nThe following constructs bind names: formal parameters to functions,\n``import`` statements, class and function definitions (these bind the\nclass or function name in the defining block), and targets that are\nidentifiers if occurring in an assignment, ``for`` loop header, in the\nsecond position of an ``except`` clause header or after ``as`` in a\n``with`` statement. The ``import`` statement of the form ``from ...\nimport *`` binds all names defined in the imported module, except\nthose beginning with an underscore. This form may only be used at the\nmodule level.\n\nA target occurring in a ``del`` statement is also considered bound for\nthis purpose (though the actual semantics are to unbind the name). It\nis illegal to unbind a name that is referenced by an enclosing scope;\nthe compiler will report a ``SyntaxError``.\n\nEach assignment or import statement occurs within a block defined by a\nclass or function definition or at the module level (the top-level\ncode block).\n\nIf a name binding operation occurs anywhere within a code block, all\nuses of the name within the block are treated as references to the\ncurrent block. This can lead to errors when a name is used within a\nblock before it is bound. This rule is subtle. Python lacks\ndeclarations and allows name binding operations to occur anywhere\nwithin a code block. The local variables of a code block can be\ndetermined by scanning the entire text of the block for name binding\noperations.\n\nIf the global statement occurs within a block, all uses of the name\nspecified in the statement refer to the binding of that name in the\ntop-level namespace. Names are resolved in the top-level namespace by\nsearching the global namespace, i.e. the namespace of the module\ncontaining the code block, and the builtins namespace, the namespace\nof the module ``__builtin__``. The global namespace is searched\nfirst. If the name is not found there, the builtins namespace is\nsearched. The global statement must precede all uses of the name.\n\nThe builtins namespace associated with the execution of a code block\nis actually found by looking up the name ``__builtins__`` in its\nglobal namespace; this should be a dictionary or a module (in the\nlatter case the module\'s dictionary is used). By default, when in the\n``__main__`` module, ``__builtins__`` is the built-in module\n``__builtin__`` (note: no \'s\'); when in any other module,\n``__builtins__`` is an alias for the dictionary of the ``__builtin__``\nmodule itself. ``__builtins__`` can be set to a user-created\ndictionary to create a weak form of restricted execution.\n\n**CPython implementation detail:** Users should not touch\n``__builtins__``; it is strictly an implementation detail. Users\nwanting to override values in the builtins namespace should ``import``\nthe ``__builtin__`` (no \'s\') module and modify its attributes\nappropriately.\n\nThe namespace for a module is automatically created the first time a\nmodule is imported. The main module for a script is always called\n``__main__``.\n\nThe global statement has the same scope as a name binding operation in\nthe same block. If the nearest enclosing scope for a free variable\ncontains a global statement, the free variable is treated as a global.\n\nA class definition is an executable statement that may use and define\nnames. These references follow the normal rules for name resolution.\nThe namespace of the class definition becomes the attribute dictionary\nof the class. Names defined at the class scope are not visible in\nmethods.\n\n\nInteraction with dynamic features\n---------------------------------\n\nThere are several cases where Python statements are illegal when used\nin conjunction with nested scopes that contain free variables.\n\nIf a variable is referenced in an enclosing scope, it is illegal to\ndelete the name. An error will be reported at compile time.\n\nIf the wild card form of import --- ``import *`` --- is used in a\nfunction and the function contains or is a nested block with free\nvariables, the compiler will raise a ``SyntaxError``.\n\nIf ``exec`` is used in a function and the function contains or is a\nnested block with free variables, the compiler will raise a\n``SyntaxError`` unless the exec explicitly specifies the local\nnamespace for the ``exec``. (In other words, ``exec obj`` would be\nillegal, but ``exec obj in ns`` would be legal.)\n\nThe ``eval()``, ``execfile()``, and ``input()`` functions and the\n``exec`` statement do not have access to the full environment for\nresolving names. Names may be resolved in the local and global\nnamespaces of the caller. Free variables are not resolved in the\nnearest enclosing namespace, but in the global namespace. [1] The\n``exec`` statement and the ``eval()`` and ``execfile()`` functions\nhave optional arguments to override the global and local namespace.\nIf only one namespace is specified, it is used for both.\n\n\nExceptions\n==========\n\nExceptions are a means of breaking out of the normal flow of control\nof a code block in order to handle errors or other exceptional\nconditions. An exception is *raised* at the point where the error is\ndetected; it may be *handled* by the surrounding code block or by any\ncode block that directly or indirectly invoked the code block where\nthe error occurred.\n\nThe Python interpreter raises an exception when it detects a run-time\nerror (such as division by zero). A Python program can also\nexplicitly raise an exception with the ``raise`` statement. Exception\nhandlers are specified with the ``try`` ... ``except`` statement. The\n``finally`` clause of such a statement can be used to specify cleanup\ncode which does not handle the exception, but is executed whether an\nexception occurred or not in the preceding code.\n\nPython uses the "termination" model of error handling: an exception\nhandler can find out what happened and continue execution at an outer\nlevel, but it cannot repair the cause of the error and retry the\nfailing operation (except by re-entering the offending piece of code\nfrom the top).\n\nWhen an exception is not handled at all, the interpreter terminates\nexecution of the program, or returns to its interactive main loop. In\neither case, it prints a stack backtrace, except when the exception is\n``SystemExit``.\n\nExceptions are identified by class instances. The ``except`` clause\nis selected depending on the class of the instance: it must reference\nthe class of the instance or a base class thereof. The instance can\nbe received by the handler and can carry additional information about\nthe exceptional condition.\n\nExceptions can also be identified by strings, in which case the\n``except`` clause is selected by object identity. An arbitrary value\ncan be raised along with the identifying string which can be passed to\nthe handler.\n\nNote: Messages to exceptions are not part of the Python API. Their\n contents may change from one version of Python to the next without\n warning and should not be relied on by code which will run under\n multiple versions of the interpreter.\n\nSee also the description of the ``try`` statement in section *The try\nstatement* and ``raise`` statement in section *The raise statement*.\n\n-[ Footnotes ]-\n\n[1] This limitation occurs because the code that is executed by these\n operations is not available at the time the module is compiled.\n', + 'execmodel': u'\nExecution model\n***************\n\n\nNaming and binding\n==================\n\n*Names* refer to objects. Names are introduced by name binding\noperations. Each occurrence of a name in the program text refers to\nthe *binding* of that name established in the innermost function block\ncontaining the use.\n\nA *block* is a piece of Python program text that is executed as a\nunit. The following are blocks: a module, a function body, and a class\ndefinition. Each command typed interactively is a block. A script\nfile (a file given as standard input to the interpreter or specified\non the interpreter command line the first argument) is a code block.\nA script command (a command specified on the interpreter command line\nwith the \'**-c**\' option) is a code block. The file read by the\nbuilt-in function ``execfile()`` is a code block. The string argument\npassed to the built-in function ``eval()`` and to the ``exec``\nstatement is a code block. The expression read and evaluated by the\nbuilt-in function ``input()`` is a code block.\n\nA code block is executed in an *execution frame*. A frame contains\nsome administrative information (used for debugging) and determines\nwhere and how execution continues after the code block\'s execution has\ncompleted.\n\nA *scope* defines the visibility of a name within a block. If a local\nvariable is defined in a block, its scope includes that block. If the\ndefinition occurs in a function block, the scope extends to any blocks\ncontained within the defining one, unless a contained block introduces\na different binding for the name. The scope of names defined in a\nclass block is limited to the class block; it does not extend to the\ncode blocks of methods -- this includes generator expressions since\nthey are implemented using a function scope. This means that the\nfollowing will fail:\n\n class A:\n a = 42\n b = list(a + i for i in range(10))\n\nWhen a name is used in a code block, it is resolved using the nearest\nenclosing scope. The set of all such scopes visible to a code block\nis called the block\'s *environment*.\n\nIf a name is bound in a block, it is a local variable of that block.\nIf a name is bound at the module level, it is a global variable. (The\nvariables of the module code block are local and global.) If a\nvariable is used in a code block but not defined there, it is a *free\nvariable*.\n\nWhen a name is not found at all, a ``NameError`` exception is raised.\nIf the name refers to a local variable that has not been bound, a\n``UnboundLocalError`` exception is raised. ``UnboundLocalError`` is a\nsubclass of ``NameError``.\n\nThe following constructs bind names: formal parameters to functions,\n``import`` statements, class and function definitions (these bind the\nclass or function name in the defining block), and targets that are\nidentifiers if occurring in an assignment, ``for`` loop header, in the\nsecond position of an ``except`` clause header or after ``as`` in a\n``with`` statement. The ``import`` statement of the form ``from ...\nimport *`` binds all names defined in the imported module, except\nthose beginning with an underscore. This form may only be used at the\nmodule level.\n\nA target occurring in a ``del`` statement is also considered bound for\nthis purpose (though the actual semantics are to unbind the name). It\nis illegal to unbind a name that is referenced by an enclosing scope;\nthe compiler will report a ``SyntaxError``.\n\nEach assignment or import statement occurs within a block defined by a\nclass or function definition or at the module level (the top-level\ncode block).\n\nIf a name binding operation occurs anywhere within a code block, all\nuses of the name within the block are treated as references to the\ncurrent block. This can lead to errors when a name is used within a\nblock before it is bound. This rule is subtle. Python lacks\ndeclarations and allows name binding operations to occur anywhere\nwithin a code block. The local variables of a code block can be\ndetermined by scanning the entire text of the block for name binding\noperations.\n\nIf the global statement occurs within a block, all uses of the name\nspecified in the statement refer to the binding of that name in the\ntop-level namespace. Names are resolved in the top-level namespace by\nsearching the global namespace, i.e. the namespace of the module\ncontaining the code block, and the builtins namespace, the namespace\nof the module ``__builtin__``. The global namespace is searched\nfirst. If the name is not found there, the builtins namespace is\nsearched. The global statement must precede all uses of the name.\n\nThe builtins namespace associated with the execution of a code block\nis actually found by looking up the name ``__builtins__`` in its\nglobal namespace; this should be a dictionary or a module (in the\nlatter case the module\'s dictionary is used). By default, when in the\n``__main__`` module, ``__builtins__`` is the built-in module\n``__builtin__`` (note: no \'s\'); when in any other module,\n``__builtins__`` is an alias for the dictionary of the ``__builtin__``\nmodule itself. ``__builtins__`` can be set to a user-created\ndictionary to create a weak form of restricted execution.\n\n**CPython implementation detail:** Users should not touch\n``__builtins__``; it is strictly an implementation detail. Users\nwanting to override values in the builtins namespace should ``import``\nthe ``__builtin__`` (no \'s\') module and modify its attributes\nappropriately.\n\nThe namespace for a module is automatically created the first time a\nmodule is imported. The main module for a script is always called\n``__main__``.\n\nThe ``global`` statement has the same scope as a name binding\noperation in the same block. If the nearest enclosing scope for a\nfree variable contains a global statement, the free variable is\ntreated as a global.\n\nA class definition is an executable statement that may use and define\nnames. These references follow the normal rules for name resolution.\nThe namespace of the class definition becomes the attribute dictionary\nof the class. Names defined at the class scope are not visible in\nmethods.\n\n\nInteraction with dynamic features\n---------------------------------\n\nThere are several cases where Python statements are illegal when used\nin conjunction with nested scopes that contain free variables.\n\nIf a variable is referenced in an enclosing scope, it is illegal to\ndelete the name. An error will be reported at compile time.\n\nIf the wild card form of import --- ``import *`` --- is used in a\nfunction and the function contains or is a nested block with free\nvariables, the compiler will raise a ``SyntaxError``.\n\nIf ``exec`` is used in a function and the function contains or is a\nnested block with free variables, the compiler will raise a\n``SyntaxError`` unless the exec explicitly specifies the local\nnamespace for the ``exec``. (In other words, ``exec obj`` would be\nillegal, but ``exec obj in ns`` would be legal.)\n\nThe ``eval()``, ``execfile()``, and ``input()`` functions and the\n``exec`` statement do not have access to the full environment for\nresolving names. Names may be resolved in the local and global\nnamespaces of the caller. Free variables are not resolved in the\nnearest enclosing namespace, but in the global namespace. [1] The\n``exec`` statement and the ``eval()`` and ``execfile()`` functions\nhave optional arguments to override the global and local namespace.\nIf only one namespace is specified, it is used for both.\n\n\nExceptions\n==========\n\nExceptions are a means of breaking out of the normal flow of control\nof a code block in order to handle errors or other exceptional\nconditions. An exception is *raised* at the point where the error is\ndetected; it may be *handled* by the surrounding code block or by any\ncode block that directly or indirectly invoked the code block where\nthe error occurred.\n\nThe Python interpreter raises an exception when it detects a run-time\nerror (such as division by zero). A Python program can also\nexplicitly raise an exception with the ``raise`` statement. Exception\nhandlers are specified with the ``try`` ... ``except`` statement. The\n``finally`` clause of such a statement can be used to specify cleanup\ncode which does not handle the exception, but is executed whether an\nexception occurred or not in the preceding code.\n\nPython uses the "termination" model of error handling: an exception\nhandler can find out what happened and continue execution at an outer\nlevel, but it cannot repair the cause of the error and retry the\nfailing operation (except by re-entering the offending piece of code\nfrom the top).\n\nWhen an exception is not handled at all, the interpreter terminates\nexecution of the program, or returns to its interactive main loop. In\neither case, it prints a stack backtrace, except when the exception is\n``SystemExit``.\n\nExceptions are identified by class instances. The ``except`` clause\nis selected depending on the class of the instance: it must reference\nthe class of the instance or a base class thereof. The instance can\nbe received by the handler and can carry additional information about\nthe exceptional condition.\n\nExceptions can also be identified by strings, in which case the\n``except`` clause is selected by object identity. An arbitrary value\ncan be raised along with the identifying string which can be passed to\nthe handler.\n\nNote: Messages to exceptions are not part of the Python API. Their\n contents may change from one version of Python to the next without\n warning and should not be relied on by code which will run under\n multiple versions of the interpreter.\n\nSee also the description of the ``try`` statement in section *The try\nstatement* and ``raise`` statement in section *The raise statement*.\n\n-[ Footnotes ]-\n\n[1] This limitation occurs because the code that is executed by these\n operations is not available at the time the module is compiled.\n', 'exprlists': u'\nExpression lists\n****************\n\n expression_list ::= expression ( "," expression )* [","]\n\nAn expression list containing at least one comma yields a tuple. The\nlength of the tuple is the number of expressions in the list. The\nexpressions are evaluated from left to right.\n\nThe trailing comma is required only to create a single tuple (a.k.a. a\n*singleton*); it is optional in all other cases. A single expression\nwithout a trailing comma doesn\'t create a tuple, but rather yields the\nvalue of that expression. (To create an empty tuple, use an empty pair\nof parentheses: ``()``.)\n', 'floating': u'\nFloating point literals\n***********************\n\nFloating point literals are described by the following lexical\ndefinitions:\n\n floatnumber ::= pointfloat | exponentfloat\n pointfloat ::= [intpart] fraction | intpart "."\n exponentfloat ::= (intpart | pointfloat) exponent\n intpart ::= digit+\n fraction ::= "." digit+\n exponent ::= ("e" | "E") ["+" | "-"] digit+\n\nNote that the integer and exponent parts of floating point numbers can\nlook like octal integers, but are interpreted using radix 10. For\nexample, ``077e010`` is legal, and denotes the same number as\n``77e10``. The allowed range of floating point literals is\nimplementation-dependent. Some examples of floating point literals:\n\n 3.14 10. .001 1e100 3.14e-10 0e0\n\nNote that numeric literals do not include a sign; a phrase like ``-1``\nis actually an expression composed of the unary operator ``-`` and the\nliteral ``1``.\n', 'for': u'\nThe ``for`` statement\n*********************\n\nThe ``for`` statement is used to iterate over the elements of a\nsequence (such as a string, tuple or list) or other iterable object:\n\n for_stmt ::= "for" target_list "in" expression_list ":" suite\n ["else" ":" suite]\n\nThe expression list is evaluated once; it should yield an iterable\nobject. An iterator is created for the result of the\n``expression_list``. The suite is then executed once for each item\nprovided by the iterator, in the order of ascending indices. Each\nitem in turn is assigned to the target list using the standard rules\nfor assignments, and then the suite is executed. When the items are\nexhausted (which is immediately when the sequence is empty), the suite\nin the ``else`` clause, if present, is executed, and the loop\nterminates.\n\nA ``break`` statement executed in the first suite terminates the loop\nwithout executing the ``else`` clause\'s suite. A ``continue``\nstatement executed in the first suite skips the rest of the suite and\ncontinues with the next item, or with the ``else`` clause if there was\nno next item.\n\nThe suite may assign to the variable(s) in the target list; this does\nnot affect the next item assigned to it.\n\nThe target list is not deleted when the loop is finished, but if the\nsequence is empty, it will not have been assigned to at all by the\nloop. Hint: the built-in function ``range()`` returns a sequence of\nintegers suitable to emulate the effect of Pascal\'s ``for i := a to b\ndo``; e.g., ``range(3)`` returns the list ``[0, 1, 2]``.\n\nNote: There is a subtlety when the sequence is being modified by the loop\n (this can only occur for mutable sequences, i.e. lists). An internal\n counter is used to keep track of which item is used next, and this\n is incremented on each iteration. When this counter has reached the\n length of the sequence the loop terminates. This means that if the\n suite deletes the current (or a previous) item from the sequence,\n the next item will be skipped (since it gets the index of the\n current item which has already been treated). Likewise, if the\n suite inserts an item in the sequence before the current item, the\n current item will be treated again the next time through the loop.\n This can lead to nasty bugs that can be avoided by making a\n temporary copy using a slice of the whole sequence, e.g.,\n\n for x in a[:]:\n if x < 0: a.remove(x)\n', - 'formatstrings': u'\nFormat String Syntax\n********************\n\nThe ``str.format()`` method and the ``Formatter`` class share the same\nsyntax for format strings (although in the case of ``Formatter``,\nsubclasses can define their own format string syntax).\n\nFormat strings contain "replacement fields" surrounded by curly braces\n``{}``. Anything that is not contained in braces is considered literal\ntext, which is copied unchanged to the output. If you need to include\na brace character in the literal text, it can be escaped by doubling:\n``{{`` and ``}}``.\n\nThe grammar for a replacement field is as follows:\n\n replacement_field ::= "{" [field_name] ["!" conversion] [":" format_spec] "}"\n field_name ::= arg_name ("." attribute_name | "[" element_index "]")*\n arg_name ::= [identifier | integer]\n attribute_name ::= identifier\n element_index ::= integer | index_string\n index_string ::= <any source character except "]"> +\n conversion ::= "r" | "s"\n format_spec ::= <described in the next section>\n\nIn less formal terms, the replacement field can start with a\n*field_name* that specifies the object whose value is to be formatted\nand inserted into the output instead of the replacement field. The\n*field_name* is optionally followed by a *conversion* field, which is\npreceded by an exclamation point ``\'!\'``, and a *format_spec*, which\nis preceded by a colon ``\':\'``. These specify a non-default format\nfor the replacement value.\n\nSee also the *Format Specification Mini-Language* section.\n\nThe *field_name* itself begins with an *arg_name* that is either\neither a number or a keyword. If it\'s a number, it refers to a\npositional argument, and if it\'s a keyword, it refers to a named\nkeyword argument. If the numerical arg_names in a format string are\n0, 1, 2, ... in sequence, they can all be omitted (not just some) and\nthe numbers 0, 1, 2, ... will be automatically inserted in that order.\nThe *arg_name* can be followed by any number of index or attribute\nexpressions. An expression of the form ``\'.name\'`` selects the named\nattribute using ``getattr()``, while an expression of the form\n``\'[index]\'`` does an index lookup using ``__getitem__()``.\n\nChanged in version 2.7: The positional argument specifiers can be\nomitted, so ``\'{} {}\'`` is equivalent to ``\'{0} {1}\'``.\n\nSome simple format string examples:\n\n "First, thou shalt count to {0}" # References first positional argument\n "Bring me a {}" # Implicitly references the first positional argument\n "From {} to {}" # Same as "From {0} to {1}"\n "My quest is {name}" # References keyword argument \'name\'\n "Weight in tons {0.weight}" # \'weight\' attribute of first positional arg\n "Units destroyed: {players[0]}" # First element of keyword argument \'players\'.\n\nThe *conversion* field causes a type coercion before formatting.\nNormally, the job of formatting a value is done by the\n``__format__()`` method of the value itself. However, in some cases\nit is desirable to force a type to be formatted as a string,\noverriding its own definition of formatting. By converting the value\nto a string before calling ``__format__()``, the normal formatting\nlogic is bypassed.\n\nTwo conversion flags are currently supported: ``\'!s\'`` which calls\n``str()`` on the value, and ``\'!r\'`` which calls ``repr()``.\n\nSome examples:\n\n "Harold\'s a clever {0!s}" # Calls str() on the argument first\n "Bring out the holy {name!r}" # Calls repr() on the argument first\n\nThe *format_spec* field contains a specification of how the value\nshould be presented, including such details as field width, alignment,\npadding, decimal precision and so on. Each value type can define its\nown "formatting mini-language" or interpretation of the *format_spec*.\n\nMost built-in types support a common formatting mini-language, which\nis described in the next section.\n\nA *format_spec* field can also include nested replacement fields\nwithin it. These nested replacement fields can contain only a field\nname; conversion flags and format specifications are not allowed. The\nreplacement fields within the format_spec are substituted before the\n*format_spec* string is interpreted. This allows the formatting of a\nvalue to be dynamically specified.\n\nSee the *Format examples* section for some examples.\n\n\nFormat Specification Mini-Language\n==================================\n\n"Format specifications" are used within replacement fields contained\nwithin a format string to define how individual values are presented\n(see *Format String Syntax*). They can also be passed directly to the\nbuilt-in ``format()`` function. Each formattable type may define how\nthe format specification is to be interpreted.\n\nMost built-in types implement the following options for format\nspecifications, although some of the formatting options are only\nsupported by the numeric types.\n\nA general convention is that an empty format string (``""``) produces\nthe same result as if you had called ``str()`` on the value. A non-\nempty format string typically modifies the result.\n\nThe general form of a *standard format specifier* is:\n\n format_spec ::= [[fill]align][sign][#][0][width][,][.precision][type]\n fill ::= <a character other than \'}\'>\n align ::= "<" | ">" | "=" | "^"\n sign ::= "+" | "-" | " "\n width ::= integer\n precision ::= integer\n type ::= "b" | "c" | "d" | "e" | "E" | "f" | "F" | "g" | "G" | "n" | "o" | "s" | "x" | "X" | "%"\n\nThe *fill* character can be any character other than \'}\' (which\nsignifies the end of the field). The presence of a fill character is\nsignaled by the *next* character, which must be one of the alignment\noptions. If the second character of *format_spec* is not a valid\nalignment option, then it is assumed that both the fill character and\nthe alignment option are absent.\n\nThe meaning of the various alignment options is as follows:\n\n +-----------+------------------------------------------------------------+\n | Option | Meaning |\n +===========+============================================================+\n | ``\'<\'`` | Forces the field to be left-aligned within the available |\n | | space (this is the default). |\n +-----------+------------------------------------------------------------+\n | ``\'>\'`` | Forces the field to be right-aligned within the available |\n | | space. |\n +-----------+------------------------------------------------------------+\n | ``\'=\'`` | Forces the padding to be placed after the sign (if any) |\n | | but before the digits. This is used for printing fields |\n | | in the form \'+000000120\'. This alignment option is only |\n | | valid for numeric types. |\n +-----------+------------------------------------------------------------+\n | ``\'^\'`` | Forces the field to be centered within the available |\n | | space. |\n +-----------+------------------------------------------------------------+\n\nNote that unless a minimum field width is defined, the field width\nwill always be the same size as the data to fill it, so that the\nalignment option has no meaning in this case.\n\nThe *sign* option is only valid for number types, and can be one of\nthe following:\n\n +-----------+------------------------------------------------------------+\n | Option | Meaning |\n +===========+============================================================+\n | ``\'+\'`` | indicates that a sign should be used for both positive as |\n | | well as negative numbers. |\n +-----------+------------------------------------------------------------+\n | ``\'-\'`` | indicates that a sign should be used only for negative |\n | | numbers (this is the default behavior). |\n +-----------+------------------------------------------------------------+\n | space | indicates that a leading space should be used on positive |\n | | numbers, and a minus sign on negative numbers. |\n +-----------+------------------------------------------------------------+\n\nThe ``\'#\'`` option is only valid for integers, and only for binary,\noctal, or hexadecimal output. If present, it specifies that the\noutput will be prefixed by ``\'0b\'``, ``\'0o\'``, or ``\'0x\'``,\nrespectively.\n\nThe ``\',\'`` option signals the use of a comma for a thousands\nseparator. For a locale aware separator, use the ``\'n\'`` integer\npresentation type instead.\n\nChanged in version 2.7: Added the ``\',\'`` option (see also **PEP\n378**).\n\n*width* is a decimal integer defining the minimum field width. If not\nspecified, then the field width will be determined by the content.\n\nIf the *width* field is preceded by a zero (``\'0\'``) character, this\nenables zero-padding. This is equivalent to an *alignment* type of\n``\'=\'`` and a *fill* character of ``\'0\'``.\n\nThe *precision* is a decimal number indicating how many digits should\nbe displayed after the decimal point for a floating point value\nformatted with ``\'f\'`` and ``\'F\'``, or before and after the decimal\npoint for a floating point value formatted with ``\'g\'`` or ``\'G\'``.\nFor non-number types the field indicates the maximum field size - in\nother words, how many characters will be used from the field content.\nThe *precision* is not allowed for integer values.\n\nFinally, the *type* determines how the data should be presented.\n\nThe available string presentation types are:\n\n +-----------+------------------------------------------------------------+\n | Type | Meaning |\n +===========+============================================================+\n | ``\'s\'`` | String format. This is the default type for strings and |\n | | may be omitted. |\n +-----------+------------------------------------------------------------+\n | None | The same as ``\'s\'``. |\n +-----------+------------------------------------------------------------+\n\nThe available integer presentation types are:\n\n +-----------+------------------------------------------------------------+\n | Type | Meaning |\n +===========+============================================================+\n | ``\'b\'`` | Binary format. Outputs the number in base 2. |\n +-----------+------------------------------------------------------------+\n | ``\'c\'`` | Character. Converts the integer to the corresponding |\n | | unicode character before printing. |\n +-----------+------------------------------------------------------------+\n | ``\'d\'`` | Decimal Integer. Outputs the number in base 10. |\n +-----------+------------------------------------------------------------+\n | ``\'o\'`` | Octal format. Outputs the number in base 8. |\n +-----------+------------------------------------------------------------+\n | ``\'x\'`` | Hex format. Outputs the number in base 16, using lower- |\n | | case letters for the digits above 9. |\n +-----------+------------------------------------------------------------+\n | ``\'X\'`` | Hex format. Outputs the number in base 16, using upper- |\n | | case letters for the digits above 9. |\n +-----------+------------------------------------------------------------+\n | ``\'n\'`` | Number. This is the same as ``\'d\'``, except that it uses |\n | | the current locale setting to insert the appropriate |\n | | number separator characters. |\n +-----------+------------------------------------------------------------+\n | None | The same as ``\'d\'``. |\n +-----------+------------------------------------------------------------+\n\nIn addition to the above presentation types, integers can be formatted\nwith the floating point presentation types listed below (except\n``\'n\'`` and None). When doing so, ``float()`` is used to convert the\ninteger to a floating point number before formatting.\n\nThe available presentation types for floating point and decimal values\nare:\n\n +-----------+------------------------------------------------------------+\n | Type | Meaning |\n +===========+============================================================+\n | ``\'e\'`` | Exponent notation. Prints the number in scientific |\n | | notation using the letter \'e\' to indicate the exponent. |\n +-----------+------------------------------------------------------------+\n | ``\'E\'`` | Exponent notation. Same as ``\'e\'`` except it uses an upper |\n | | case \'E\' as the separator character. |\n +-----------+------------------------------------------------------------+\n | ``\'f\'`` | Fixed point. Displays the number as a fixed-point number. |\n +-----------+------------------------------------------------------------+\n | ``\'F\'`` | Fixed point. Same as ``\'f\'``. |\n +-----------+------------------------------------------------------------+\n | ``\'g\'`` | General format. For a given precision ``p >= 1``, this |\n | | rounds the number to ``p`` significant digits and then |\n | | formats the result in either fixed-point format or in |\n | | scientific notation, depending on its magnitude. The |\n | | precise rules are as follows: suppose that the result |\n | | formatted with presentation type ``\'e\'`` and precision |\n | | ``p-1`` would have exponent ``exp``. Then if ``-4 <= exp |\n | | < p``, the number is formatted with presentation type |\n | | ``\'f\'`` and precision ``p-1-exp``. Otherwise, the number |\n | | is formatted with presentation type ``\'e\'`` and precision |\n | | ``p-1``. In both cases insignificant trailing zeros are |\n | | removed from the significand, and the decimal point is |\n | | also removed if there are no remaining digits following |\n | | it. Postive and negative infinity, positive and negative |\n | | zero, and nans, are formatted as ``inf``, ``-inf``, ``0``, |\n | | ``-0`` and ``nan`` respectively, regardless of the |\n | | precision. A precision of ``0`` is treated as equivalent |\n | | to a precision of ``1``. |\n +-----------+------------------------------------------------------------+\n | ``\'G\'`` | General format. Same as ``\'g\'`` except switches to ``\'E\'`` |\n | | if the number gets too large. The representations of |\n | | infinity and NaN are uppercased, too. |\n +-----------+------------------------------------------------------------+\n | ``\'n\'`` | Number. This is the same as ``\'g\'``, except that it uses |\n | | the current locale setting to insert the appropriate |\n | | number separator characters. |\n +-----------+------------------------------------------------------------+\n | ``\'%\'`` | Percentage. Multiplies the number by 100 and displays in |\n | | fixed (``\'f\'``) format, followed by a percent sign. |\n +-----------+------------------------------------------------------------+\n | None | The same as ``\'g\'``. |\n +-----------+------------------------------------------------------------+\n\n\nFormat examples\n===============\n\nThis section contains examples of the new format syntax and comparison\nwith the old ``%``-formatting.\n\nIn most of the cases the syntax is similar to the old\n``%``-formatting, with the addition of the ``{}`` and with ``:`` used\ninstead of ``%``. For example, ``\'%03.2f\'`` can be translated to\n``\'{:03.2f}\'``.\n\nThe new format syntax also supports new and different options, shown\nin the follow examples.\n\nAccessing arguments by position:\n\n >>> \'{0}, {1}, {2}\'.format(\'a\', \'b\', \'c\')\n \'a, b, c\'\n >>> \'{}, {}, {}\'.format(\'a\', \'b\', \'c\') # 2.7+ only\n \'a, b, c\'\n >>> \'{2}, {1}, {0}\'.format(\'a\', \'b\', \'c\')\n \'c, b, a\'\n >>> \'{2}, {1}, {0}\'.format(*\'abc\') # unpacking argument sequence\n \'c, b, a\'\n >>> \'{0}{1}{0}\'.format(\'abra\', \'cad\') # arguments\' indices can be repeated\n \'abracadabra\'\n\nAccessing arguments by name:\n\n >>> \'Coordinates: {latitude}, {longitude}\'.format(latitude=\'37.24N\', longitude=\'-115.81W\')\n \'Coordinates: 37.24N, -115.81W\'\n >>> coord = {\'latitude\': \'37.24N\', \'longitude\': \'-115.81W\'}\n >>> \'Coordinates: {latitude}, {longitude}\'.format(**coord)\n \'Coordinates: 37.24N, -115.81W\'\n\nAccessing arguments\' attributes:\n\n >>> c = 3-5j\n >>> (\'The complex number {0} is formed from the real part {0.real} \'\n ... \'and the imaginary part {0.imag}.\').format(c)\n \'The complex number (3-5j) is formed from the real part 3.0 and the imaginary part -5.0.\'\n >>> class Point(object):\n ... def __init__(self, x, y):\n ... self.x, self.y = x, y\n ... def __str__(self):\n ... return \'Point({self.x}, {self.y})\'.format(self=self)\n ...\n >>> str(Point(4, 2))\n \'Point(4, 2)\'\n\nAccessing arguments\' items:\n\n >>> coord = (3, 5)\n >>> \'X: {0[0]}; Y: {0[1]}\'.format(coord)\n \'X: 3; Y: 5\'\n\nReplacing ``%s`` and ``%r``:\n\n >>> "repr() shows quotes: {!r}; str() doesn\'t: {!s}".format(\'test1\', \'test2\')\n "repr() shows quotes: \'test1\'; str() doesn\'t: test2"\n\nAligning the text and specifying a width:\n\n >>> \'{:<30}\'.format(\'left aligned\')\n \'left aligned \'\n >>> \'{:>30}\'.format(\'right aligned\')\n \' right aligned\'\n >>> \'{:^30}\'.format(\'centered\')\n \' centered \'\n >>> \'{:*^30}\'.format(\'centered\') # use \'*\' as a fill char\n \'***********centered***********\'\n\nReplacing ``%+f``, ``%-f``, and ``% f`` and specifying a sign:\n\n >>> \'{:+f}; {:+f}\'.format(3.14, -3.14) # show it always\n \'+3.140000; -3.140000\'\n >>> \'{: f}; {: f}\'.format(3.14, -3.14) # show a space for positive numbers\n \' 3.140000; -3.140000\'\n >>> \'{:-f}; {:-f}\'.format(3.14, -3.14) # show only the minus -- same as \'{:f}; {:f}\'\n \'3.140000; -3.140000\'\n\nReplacing ``%x`` and ``%o`` and converting the value to different\nbases:\n\n >>> # format also supports binary numbers\n >>> "int: {0:d}; hex: {0:x}; oct: {0:o}; bin: {0:b}".format(42)\n \'int: 42; hex: 2a; oct: 52; bin: 101010\'\n >>> # with 0x, 0o, or 0b as prefix:\n >>> "int: {0:d}; hex: {0:#x}; oct: {0:#o}; bin: {0:#b}".format(42)\n \'int: 42; hex: 0x2a; oct: 0o52; bin: 0b101010\'\n\nUsing the comma as a thousands separator:\n\n >>> \'{:,}\'.format(1234567890)\n \'1,234,567,890\'\n\nExpressing a percentage:\n\n >>> points = 19.5\n >>> total = 22\n >>> \'Correct answers: {:.2%}.\'.format(points/total)\n \'Correct answers: 88.64%\'\n\nUsing type-specific formatting:\n\n >>> import datetime\n >>> d = datetime.datetime(2010, 7, 4, 12, 15, 58)\n >>> \'{:%Y-%m-%d %H:%M:%S}\'.format(d)\n \'2010-07-04 12:15:58\'\n\nNesting arguments and more complex examples:\n\n >>> for align, text in zip(\'<^>\', [\'left\', \'center\', \'right\']):\n ... \'{0:{align}{fill}16}\'.format(text, fill=align, align=align)\n ...\n \'left<<<<<<<<<<<<\'\n \'^^^^^center^^^^^\'\n \'>>>>>>>>>>>right\'\n >>>\n >>> octets = [192, 168, 0, 1]\n >>> \'{:02X}{:02X}{:02X}{:02X}\'.format(*octets)\n \'C0A80001\'\n >>> int(_, 16)\n 3232235521\n >>>\n >>> width = 5\n >>> for num in range(5,12):\n ... for base in \'dXob\':\n ... print \'{0:{width}{base}}\'.format(num, base=base, width=width),\n ... print\n ...\n 5 5 5 101\n 6 6 6 110\n 7 7 7 111\n 8 8 10 1000\n 9 9 11 1001\n 10 A 12 1010\n 11 B 13 1011\n', + 'formatstrings': u'\nFormat String Syntax\n********************\n\nThe ``str.format()`` method and the ``Formatter`` class share the same\nsyntax for format strings (although in the case of ``Formatter``,\nsubclasses can define their own format string syntax).\n\nFormat strings contain "replacement fields" surrounded by curly braces\n``{}``. Anything that is not contained in braces is considered literal\ntext, which is copied unchanged to the output. If you need to include\na brace character in the literal text, it can be escaped by doubling:\n``{{`` and ``}}``.\n\nThe grammar for a replacement field is as follows:\n\n replacement_field ::= "{" [field_name] ["!" conversion] [":" format_spec] "}"\n field_name ::= arg_name ("." attribute_name | "[" element_index "]")*\n arg_name ::= [identifier | integer]\n attribute_name ::= identifier\n element_index ::= integer | index_string\n index_string ::= <any source character except "]"> +\n conversion ::= "r" | "s"\n format_spec ::= <described in the next section>\n\nIn less formal terms, the replacement field can start with a\n*field_name* that specifies the object whose value is to be formatted\nand inserted into the output instead of the replacement field. The\n*field_name* is optionally followed by a *conversion* field, which is\npreceded by an exclamation point ``\'!\'``, and a *format_spec*, which\nis preceded by a colon ``\':\'``. These specify a non-default format\nfor the replacement value.\n\nSee also the *Format Specification Mini-Language* section.\n\nThe *field_name* itself begins with an *arg_name* that is either\neither a number or a keyword. If it\'s a number, it refers to a\npositional argument, and if it\'s a keyword, it refers to a named\nkeyword argument. If the numerical arg_names in a format string are\n0, 1, 2, ... in sequence, they can all be omitted (not just some) and\nthe numbers 0, 1, 2, ... will be automatically inserted in that order.\nThe *arg_name* can be followed by any number of index or attribute\nexpressions. An expression of the form ``\'.name\'`` selects the named\nattribute using ``getattr()``, while an expression of the form\n``\'[index]\'`` does an index lookup using ``__getitem__()``.\n\nChanged in version 2.7: The positional argument specifiers can be\nomitted, so ``\'{} {}\'`` is equivalent to ``\'{0} {1}\'``.\n\nSome simple format string examples:\n\n "First, thou shalt count to {0}" # References first positional argument\n "Bring me a {}" # Implicitly references the first positional argument\n "From {} to {}" # Same as "From {0} to {1}"\n "My quest is {name}" # References keyword argument \'name\'\n "Weight in tons {0.weight}" # \'weight\' attribute of first positional arg\n "Units destroyed: {players[0]}" # First element of keyword argument \'players\'.\n\nThe *conversion* field causes a type coercion before formatting.\nNormally, the job of formatting a value is done by the\n``__format__()`` method of the value itself. However, in some cases\nit is desirable to force a type to be formatted as a string,\noverriding its own definition of formatting. By converting the value\nto a string before calling ``__format__()``, the normal formatting\nlogic is bypassed.\n\nTwo conversion flags are currently supported: ``\'!s\'`` which calls\n``str()`` on the value, and ``\'!r\'`` which calls ``repr()``.\n\nSome examples:\n\n "Harold\'s a clever {0!s}" # Calls str() on the argument first\n "Bring out the holy {name!r}" # Calls repr() on the argument first\n\nThe *format_spec* field contains a specification of how the value\nshould be presented, including such details as field width, alignment,\npadding, decimal precision and so on. Each value type can define its\nown "formatting mini-language" or interpretation of the *format_spec*.\n\nMost built-in types support a common formatting mini-language, which\nis described in the next section.\n\nA *format_spec* field can also include nested replacement fields\nwithin it. These nested replacement fields can contain only a field\nname; conversion flags and format specifications are not allowed. The\nreplacement fields within the format_spec are substituted before the\n*format_spec* string is interpreted. This allows the formatting of a\nvalue to be dynamically specified.\n\nSee the *Format examples* section for some examples.\n\n\nFormat Specification Mini-Language\n==================================\n\n"Format specifications" are used within replacement fields contained\nwithin a format string to define how individual values are presented\n(see *Format String Syntax*). They can also be passed directly to the\nbuilt-in ``format()`` function. Each formattable type may define how\nthe format specification is to be interpreted.\n\nMost built-in types implement the following options for format\nspecifications, although some of the formatting options are only\nsupported by the numeric types.\n\nA general convention is that an empty format string (``""``) produces\nthe same result as if you had called ``str()`` on the value. A non-\nempty format string typically modifies the result.\n\nThe general form of a *standard format specifier* is:\n\n format_spec ::= [[fill]align][sign][#][0][width][,][.precision][type]\n fill ::= <a character other than \'}\'>\n align ::= "<" | ">" | "=" | "^"\n sign ::= "+" | "-" | " "\n width ::= integer\n precision ::= integer\n type ::= "b" | "c" | "d" | "e" | "E" | "f" | "F" | "g" | "G" | "n" | "o" | "s" | "x" | "X" | "%"\n\nThe *fill* character can be any character other than \'{\' or \'}\'. The\npresence of a fill character is signaled by the character following\nit, which must be one of the alignment options. If the second\ncharacter of *format_spec* is not a valid alignment option, then it is\nassumed that both the fill character and the alignment option are\nabsent.\n\nThe meaning of the various alignment options is as follows:\n\n +-----------+------------------------------------------------------------+\n | Option | Meaning |\n +===========+============================================================+\n | ``\'<\'`` | Forces the field to be left-aligned within the available |\n | | space (this is the default for most objects). |\n +-----------+------------------------------------------------------------+\n | ``\'>\'`` | Forces the field to be right-aligned within the available |\n | | space (this is the default for numbers). |\n +-----------+------------------------------------------------------------+\n | ``\'=\'`` | Forces the padding to be placed after the sign (if any) |\n | | but before the digits. This is used for printing fields |\n | | in the form \'+000000120\'. This alignment option is only |\n | | valid for numeric types. |\n +-----------+------------------------------------------------------------+\n | ``\'^\'`` | Forces the field to be centered within the available |\n | | space. |\n +-----------+------------------------------------------------------------+\n\nNote that unless a minimum field width is defined, the field width\nwill always be the same size as the data to fill it, so that the\nalignment option has no meaning in this case.\n\nThe *sign* option is only valid for number types, and can be one of\nthe following:\n\n +-----------+------------------------------------------------------------+\n | Option | Meaning |\n +===========+============================================================+\n | ``\'+\'`` | indicates that a sign should be used for both positive as |\n | | well as negative numbers. |\n +-----------+------------------------------------------------------------+\n | ``\'-\'`` | indicates that a sign should be used only for negative |\n | | numbers (this is the default behavior). |\n +-----------+------------------------------------------------------------+\n | space | indicates that a leading space should be used on positive |\n | | numbers, and a minus sign on negative numbers. |\n +-----------+------------------------------------------------------------+\n\nThe ``\'#\'`` option is only valid for integers, and only for binary,\noctal, or hexadecimal output. If present, it specifies that the\noutput will be prefixed by ``\'0b\'``, ``\'0o\'``, or ``\'0x\'``,\nrespectively.\n\nThe ``\',\'`` option signals the use of a comma for a thousands\nseparator. For a locale aware separator, use the ``\'n\'`` integer\npresentation type instead.\n\nChanged in version 2.7: Added the ``\',\'`` option (see also **PEP\n378**).\n\n*width* is a decimal integer defining the minimum field width. If not\nspecified, then the field width will be determined by the content.\n\nIf the *width* field is preceded by a zero (``\'0\'``) character, this\nenables zero-padding. This is equivalent to an *alignment* type of\n``\'=\'`` and a *fill* character of ``\'0\'``.\n\nThe *precision* is a decimal number indicating how many digits should\nbe displayed after the decimal point for a floating point value\nformatted with ``\'f\'`` and ``\'F\'``, or before and after the decimal\npoint for a floating point value formatted with ``\'g\'`` or ``\'G\'``.\nFor non-number types the field indicates the maximum field size - in\nother words, how many characters will be used from the field content.\nThe *precision* is not allowed for integer values.\n\nFinally, the *type* determines how the data should be presented.\n\nThe available string presentation types are:\n\n +-----------+------------------------------------------------------------+\n | Type | Meaning |\n +===========+============================================================+\n | ``\'s\'`` | String format. This is the default type for strings and |\n | | may be omitted. |\n +-----------+------------------------------------------------------------+\n | None | The same as ``\'s\'``. |\n +-----------+------------------------------------------------------------+\n\nThe available integer presentation types are:\n\n +-----------+------------------------------------------------------------+\n | Type | Meaning |\n +===========+============================================================+\n | ``\'b\'`` | Binary format. Outputs the number in base 2. |\n +-----------+------------------------------------------------------------+\n | ``\'c\'`` | Character. Converts the integer to the corresponding |\n | | unicode character before printing. |\n +-----------+------------------------------------------------------------+\n | ``\'d\'`` | Decimal Integer. Outputs the number in base 10. |\n +-----------+------------------------------------------------------------+\n | ``\'o\'`` | Octal format. Outputs the number in base 8. |\n +-----------+------------------------------------------------------------+\n | ``\'x\'`` | Hex format. Outputs the number in base 16, using lower- |\n | | case letters for the digits above 9. |\n +-----------+------------------------------------------------------------+\n | ``\'X\'`` | Hex format. Outputs the number in base 16, using upper- |\n | | case letters for the digits above 9. |\n +-----------+------------------------------------------------------------+\n | ``\'n\'`` | Number. This is the same as ``\'d\'``, except that it uses |\n | | the current locale setting to insert the appropriate |\n | | number separator characters. |\n +-----------+------------------------------------------------------------+\n | None | The same as ``\'d\'``. |\n +-----------+------------------------------------------------------------+\n\nIn addition to the above presentation types, integers can be formatted\nwith the floating point presentation types listed below (except\n``\'n\'`` and None). When doing so, ``float()`` is used to convert the\ninteger to a floating point number before formatting.\n\nThe available presentation types for floating point and decimal values\nare:\n\n +-----------+------------------------------------------------------------+\n | Type | Meaning |\n +===========+============================================================+\n | ``\'e\'`` | Exponent notation. Prints the number in scientific |\n | | notation using the letter \'e\' to indicate the exponent. |\n +-----------+------------------------------------------------------------+\n | ``\'E\'`` | Exponent notation. Same as ``\'e\'`` except it uses an upper |\n | | case \'E\' as the separator character. |\n +-----------+------------------------------------------------------------+\n | ``\'f\'`` | Fixed point. Displays the number as a fixed-point number. |\n +-----------+------------------------------------------------------------+\n | ``\'F\'`` | Fixed point. Same as ``\'f\'``. |\n +-----------+------------------------------------------------------------+\n | ``\'g\'`` | General format. For a given precision ``p >= 1``, this |\n | | rounds the number to ``p`` significant digits and then |\n | | formats the result in either fixed-point format or in |\n | | scientific notation, depending on its magnitude. The |\n | | precise rules are as follows: suppose that the result |\n | | formatted with presentation type ``\'e\'`` and precision |\n | | ``p-1`` would have exponent ``exp``. Then if ``-4 <= exp |\n | | < p``, the number is formatted with presentation type |\n | | ``\'f\'`` and precision ``p-1-exp``. Otherwise, the number |\n | | is formatted with presentation type ``\'e\'`` and precision |\n | | ``p-1``. In both cases insignificant trailing zeros are |\n | | removed from the significand, and the decimal point is |\n | | also removed if there are no remaining digits following |\n | | it. Positive and negative infinity, positive and negative |\n | | zero, and nans, are formatted as ``inf``, ``-inf``, ``0``, |\n | | ``-0`` and ``nan`` respectively, regardless of the |\n | | precision. A precision of ``0`` is treated as equivalent |\n | | to a precision of ``1``. |\n +-----------+------------------------------------------------------------+\n | ``\'G\'`` | General format. Same as ``\'g\'`` except switches to ``\'E\'`` |\n | | if the number gets too large. The representations of |\n | | infinity and NaN are uppercased, too. |\n +-----------+------------------------------------------------------------+\n | ``\'n\'`` | Number. This is the same as ``\'g\'``, except that it uses |\n | | the current locale setting to insert the appropriate |\n | | number separator characters. |\n +-----------+------------------------------------------------------------+\n | ``\'%\'`` | Percentage. Multiplies the number by 100 and displays in |\n | | fixed (``\'f\'``) format, followed by a percent sign. |\n +-----------+------------------------------------------------------------+\n | None | The same as ``\'g\'``. |\n +-----------+------------------------------------------------------------+\n\n\nFormat examples\n===============\n\nThis section contains examples of the new format syntax and comparison\nwith the old ``%``-formatting.\n\nIn most of the cases the syntax is similar to the old\n``%``-formatting, with the addition of the ``{}`` and with ``:`` used\ninstead of ``%``. For example, ``\'%03.2f\'`` can be translated to\n``\'{:03.2f}\'``.\n\nThe new format syntax also supports new and different options, shown\nin the follow examples.\n\nAccessing arguments by position:\n\n >>> \'{0}, {1}, {2}\'.format(\'a\', \'b\', \'c\')\n \'a, b, c\'\n >>> \'{}, {}, {}\'.format(\'a\', \'b\', \'c\') # 2.7+ only\n \'a, b, c\'\n >>> \'{2}, {1}, {0}\'.format(\'a\', \'b\', \'c\')\n \'c, b, a\'\n >>> \'{2}, {1}, {0}\'.format(*\'abc\') # unpacking argument sequence\n \'c, b, a\'\n >>> \'{0}{1}{0}\'.format(\'abra\', \'cad\') # arguments\' indices can be repeated\n \'abracadabra\'\n\nAccessing arguments by name:\n\n >>> \'Coordinates: {latitude}, {longitude}\'.format(latitude=\'37.24N\', longitude=\'-115.81W\')\n \'Coordinates: 37.24N, -115.81W\'\n >>> coord = {\'latitude\': \'37.24N\', \'longitude\': \'-115.81W\'}\n >>> \'Coordinates: {latitude}, {longitude}\'.format(**coord)\n \'Coordinates: 37.24N, -115.81W\'\n\nAccessing arguments\' attributes:\n\n >>> c = 3-5j\n >>> (\'The complex number {0} is formed from the real part {0.real} \'\n ... \'and the imaginary part {0.imag}.\').format(c)\n \'The complex number (3-5j) is formed from the real part 3.0 and the imaginary part -5.0.\'\n >>> class Point(object):\n ... def __init__(self, x, y):\n ... self.x, self.y = x, y\n ... def __str__(self):\n ... return \'Point({self.x}, {self.y})\'.format(self=self)\n ...\n >>> str(Point(4, 2))\n \'Point(4, 2)\'\n\nAccessing arguments\' items:\n\n >>> coord = (3, 5)\n >>> \'X: {0[0]}; Y: {0[1]}\'.format(coord)\n \'X: 3; Y: 5\'\n\nReplacing ``%s`` and ``%r``:\n\n >>> "repr() shows quotes: {!r}; str() doesn\'t: {!s}".format(\'test1\', \'test2\')\n "repr() shows quotes: \'test1\'; str() doesn\'t: test2"\n\nAligning the text and specifying a width:\n\n >>> \'{:<30}\'.format(\'left aligned\')\n \'left aligned \'\n >>> \'{:>30}\'.format(\'right aligned\')\n \' right aligned\'\n >>> \'{:^30}\'.format(\'centered\')\n \' centered \'\n >>> \'{:*^30}\'.format(\'centered\') # use \'*\' as a fill char\n \'***********centered***********\'\n\nReplacing ``%+f``, ``%-f``, and ``% f`` and specifying a sign:\n\n >>> \'{:+f}; {:+f}\'.format(3.14, -3.14) # show it always\n \'+3.140000; -3.140000\'\n >>> \'{: f}; {: f}\'.format(3.14, -3.14) # show a space for positive numbers\n \' 3.140000; -3.140000\'\n >>> \'{:-f}; {:-f}\'.format(3.14, -3.14) # show only the minus -- same as \'{:f}; {:f}\'\n \'3.140000; -3.140000\'\n\nReplacing ``%x`` and ``%o`` and converting the value to different\nbases:\n\n >>> # format also supports binary numbers\n >>> "int: {0:d}; hex: {0:x}; oct: {0:o}; bin: {0:b}".format(42)\n \'int: 42; hex: 2a; oct: 52; bin: 101010\'\n >>> # with 0x, 0o, or 0b as prefix:\n >>> "int: {0:d}; hex: {0:#x}; oct: {0:#o}; bin: {0:#b}".format(42)\n \'int: 42; hex: 0x2a; oct: 0o52; bin: 0b101010\'\n\nUsing the comma as a thousands separator:\n\n >>> \'{:,}\'.format(1234567890)\n \'1,234,567,890\'\n\nExpressing a percentage:\n\n >>> points = 19.5\n >>> total = 22\n >>> \'Correct answers: {:.2%}.\'.format(points/total)\n \'Correct answers: 88.64%\'\n\nUsing type-specific formatting:\n\n >>> import datetime\n >>> d = datetime.datetime(2010, 7, 4, 12, 15, 58)\n >>> \'{:%Y-%m-%d %H:%M:%S}\'.format(d)\n \'2010-07-04 12:15:58\'\n\nNesting arguments and more complex examples:\n\n >>> for align, text in zip(\'<^>\', [\'left\', \'center\', \'right\']):\n ... \'{0:{fill}{align}16}\'.format(text, fill=align, align=align)\n ...\n \'left<<<<<<<<<<<<\'\n \'^^^^^center^^^^^\'\n \'>>>>>>>>>>>right\'\n >>>\n >>> octets = [192, 168, 0, 1]\n >>> \'{:02X}{:02X}{:02X}{:02X}\'.format(*octets)\n \'C0A80001\'\n >>> int(_, 16)\n 3232235521\n >>>\n >>> width = 5\n >>> for num in range(5,12):\n ... for base in \'dXob\':\n ... print \'{0:{width}{base}}\'.format(num, base=base, width=width),\n ... print\n ...\n 5 5 5 101\n 6 6 6 110\n 7 7 7 111\n 8 8 10 1000\n 9 9 11 1001\n 10 A 12 1010\n 11 B 13 1011\n', 'function': u'\nFunction definitions\n********************\n\nA function definition defines a user-defined function object (see\nsection *The standard type hierarchy*):\n\n decorated ::= decorators (classdef | funcdef)\n decorators ::= decorator+\n decorator ::= "@" dotted_name ["(" [argument_list [","]] ")"] NEWLINE\n funcdef ::= "def" funcname "(" [parameter_list] ")" ":" suite\n dotted_name ::= identifier ("." identifier)*\n parameter_list ::= (defparameter ",")*\n ( "*" identifier [, "**" identifier]\n | "**" identifier\n | defparameter [","] )\n defparameter ::= parameter ["=" expression]\n sublist ::= parameter ("," parameter)* [","]\n parameter ::= identifier | "(" sublist ")"\n funcname ::= identifier\n\nA function definition is an executable statement. Its execution binds\nthe function name in the current local namespace to a function object\n(a wrapper around the executable code for the function). This\nfunction object contains a reference to the current global namespace\nas the global namespace to be used when the function is called.\n\nThe function definition does not execute the function body; this gets\nexecuted only when the function is called. [3]\n\nA function definition may be wrapped by one or more *decorator*\nexpressions. Decorator expressions are evaluated when the function is\ndefined, in the scope that contains the function definition. The\nresult must be a callable, which is invoked with the function object\nas the only argument. The returned value is bound to the function name\ninstead of the function object. Multiple decorators are applied in\nnested fashion. For example, the following code:\n\n @f1(arg)\n @f2\n def func(): pass\n\nis equivalent to:\n\n def func(): pass\n func = f1(arg)(f2(func))\n\nWhen one or more top-level parameters have the form *parameter* ``=``\n*expression*, the function is said to have "default parameter values."\nFor a parameter with a default value, the corresponding argument may\nbe omitted from a call, in which case the parameter\'s default value is\nsubstituted. If a parameter has a default value, all following\nparameters must also have a default value --- this is a syntactic\nrestriction that is not expressed by the grammar.\n\n**Default parameter values are evaluated when the function definition\nis executed.** This means that the expression is evaluated once, when\nthe function is defined, and that that same "pre-computed" value is\nused for each call. This is especially important to understand when a\ndefault parameter is a mutable object, such as a list or a dictionary:\nif the function modifies the object (e.g. by appending an item to a\nlist), the default value is in effect modified. This is generally not\nwhat was intended. A way around this is to use ``None`` as the\ndefault, and explicitly test for it in the body of the function, e.g.:\n\n def whats_on_the_telly(penguin=None):\n if penguin is None:\n penguin = []\n penguin.append("property of the zoo")\n return penguin\n\nFunction call semantics are described in more detail in section\n*Calls*. A function call always assigns values to all parameters\nmentioned in the parameter list, either from position arguments, from\nkeyword arguments, or from default values. If the form\n"``*identifier``" is present, it is initialized to a tuple receiving\nany excess positional parameters, defaulting to the empty tuple. If\nthe form "``**identifier``" is present, it is initialized to a new\ndictionary receiving any excess keyword arguments, defaulting to a new\nempty dictionary.\n\nIt is also possible to create anonymous functions (functions not bound\nto a name), for immediate use in expressions. This uses lambda forms,\ndescribed in section *Lambdas*. Note that the lambda form is merely a\nshorthand for a simplified function definition; a function defined in\na "``def``" statement can be passed around or assigned to another name\njust like a function defined by a lambda form. The "``def``" form is\nactually more powerful since it allows the execution of multiple\nstatements.\n\n**Programmer\'s note:** Functions are first-class objects. A "``def``"\nform executed inside a function definition defines a local function\nthat can be returned or passed around. Free variables used in the\nnested function can access the local variables of the function\ncontaining the def. See section *Naming and binding* for details.\n', 'global': u'\nThe ``global`` statement\n************************\n\n global_stmt ::= "global" identifier ("," identifier)*\n\nThe ``global`` statement is a declaration which holds for the entire\ncurrent code block. It means that the listed identifiers are to be\ninterpreted as globals. It would be impossible to assign to a global\nvariable without ``global``, although free variables may refer to\nglobals without being declared global.\n\nNames listed in a ``global`` statement must not be used in the same\ncode block textually preceding that ``global`` statement.\n\nNames listed in a ``global`` statement must not be defined as formal\nparameters or in a ``for`` loop control target, ``class`` definition,\nfunction definition, or ``import`` statement.\n\n**CPython implementation detail:** The current implementation does not\nenforce the latter two restrictions, but programs should not abuse\nthis freedom, as future implementations may enforce them or silently\nchange the meaning of the program.\n\n**Programmer\'s note:** the ``global`` is a directive to the parser.\nIt applies only to code parsed at the same time as the ``global``\nstatement. In particular, a ``global`` statement contained in an\n``exec`` statement does not affect the code block *containing* the\n``exec`` statement, and code contained in an ``exec`` statement is\nunaffected by ``global`` statements in the code containing the\n``exec`` statement. The same applies to the ``eval()``,\n``execfile()`` and ``compile()`` functions.\n', - 'id-classes': u'\nReserved classes of identifiers\n*******************************\n\nCertain classes of identifiers (besides keywords) have special\nmeanings. These classes are identified by the patterns of leading and\ntrailing underscore characters:\n\n``_*``\n Not imported by ``from module import *``. The special identifier\n ``_`` is used in the interactive interpreter to store the result of\n the last evaluation; it is stored in the ``__builtin__`` module.\n When not in interactive mode, ``_`` has no special meaning and is\n not defined. See section *The import statement*.\n\n Note: The name ``_`` is often used in conjunction with\n internationalization; refer to the documentation for the\n ``gettext`` module for more information on this convention.\n\n``__*__``\n System-defined names. These names are defined by the interpreter\n and its implementation (including the standard library);\n applications should not expect to define additional names using\n this convention. The set of names of this class defined by Python\n may be extended in future versions. See section *Special method\n names*.\n\n``__*``\n Class-private names. Names in this category, when used within the\n context of a class definition, are re-written to use a mangled form\n to help avoid name clashes between "private" attributes of base and\n derived classes. See section *Identifiers (Names)*.\n', - 'identifiers': u'\nIdentifiers and keywords\n************************\n\nIdentifiers (also referred to as *names*) are described by the\nfollowing lexical definitions:\n\n identifier ::= (letter|"_") (letter | digit | "_")*\n letter ::= lowercase | uppercase\n lowercase ::= "a"..."z"\n uppercase ::= "A"..."Z"\n digit ::= "0"..."9"\n\nIdentifiers are unlimited in length. Case is significant.\n\n\nKeywords\n========\n\nThe following identifiers are used as reserved words, or *keywords* of\nthe language, and cannot be used as ordinary identifiers. They must\nbe spelled exactly as written here:\n\n and del from not while\n as elif global or with\n assert else if pass yield\n break except import print\n class exec in raise\n continue finally is return\n def for lambda try\n\nChanged in version 2.4: ``None`` became a constant and is now\nrecognized by the compiler as a name for the built-in object ``None``.\nAlthough it is not a keyword, you cannot assign a different object to\nit.\n\nChanged in version 2.5: Both ``as`` and ``with`` are only recognized\nwhen the ``with_statement`` future feature has been enabled. It will\nalways be enabled in Python 2.6. See section *The with statement* for\ndetails. Note that using ``as`` and ``with`` as identifiers will\nalways issue a warning, even when the ``with_statement`` future\ndirective is not in effect.\n\n\nReserved classes of identifiers\n===============================\n\nCertain classes of identifiers (besides keywords) have special\nmeanings. These classes are identified by the patterns of leading and\ntrailing underscore characters:\n\n``_*``\n Not imported by ``from module import *``. The special identifier\n ``_`` is used in the interactive interpreter to store the result of\n the last evaluation; it is stored in the ``__builtin__`` module.\n When not in interactive mode, ``_`` has no special meaning and is\n not defined. See section *The import statement*.\n\n Note: The name ``_`` is often used in conjunction with\n internationalization; refer to the documentation for the\n ``gettext`` module for more information on this convention.\n\n``__*__``\n System-defined names. These names are defined by the interpreter\n and its implementation (including the standard library);\n applications should not expect to define additional names using\n this convention. The set of names of this class defined by Python\n may be extended in future versions. See section *Special method\n names*.\n\n``__*``\n Class-private names. Names in this category, when used within the\n context of a class definition, are re-written to use a mangled form\n to help avoid name clashes between "private" attributes of base and\n derived classes. See section *Identifiers (Names)*.\n', + 'id-classes': u'\nReserved classes of identifiers\n*******************************\n\nCertain classes of identifiers (besides keywords) have special\nmeanings. These classes are identified by the patterns of leading and\ntrailing underscore characters:\n\n``_*``\n Not imported by ``from module import *``. The special identifier\n ``_`` is used in the interactive interpreter to store the result of\n the last evaluation; it is stored in the ``__builtin__`` module.\n When not in interactive mode, ``_`` has no special meaning and is\n not defined. See section *The import statement*.\n\n Note: The name ``_`` is often used in conjunction with\n internationalization; refer to the documentation for the\n ``gettext`` module for more information on this convention.\n\n``__*__``\n System-defined names. These names are defined by the interpreter\n and its implementation (including the standard library). Current\n system names are discussed in the *Special method names* section\n and elsewhere. More will likely be defined in future versions of\n Python. *Any* use of ``__*__`` names, in any context, that does\n not follow explicitly documented use, is subject to breakage\n without warning.\n\n``__*``\n Class-private names. Names in this category, when used within the\n context of a class definition, are re-written to use a mangled form\n to help avoid name clashes between "private" attributes of base and\n derived classes. See section *Identifiers (Names)*.\n', + 'identifiers': u'\nIdentifiers and keywords\n************************\n\nIdentifiers (also referred to as *names*) are described by the\nfollowing lexical definitions:\n\n identifier ::= (letter|"_") (letter | digit | "_")*\n letter ::= lowercase | uppercase\n lowercase ::= "a"..."z"\n uppercase ::= "A"..."Z"\n digit ::= "0"..."9"\n\nIdentifiers are unlimited in length. Case is significant.\n\n\nKeywords\n========\n\nThe following identifiers are used as reserved words, or *keywords* of\nthe language, and cannot be used as ordinary identifiers. They must\nbe spelled exactly as written here:\n\n and del from not while\n as elif global or with\n assert else if pass yield\n break except import print\n class exec in raise\n continue finally is return\n def for lambda try\n\nChanged in version 2.4: ``None`` became a constant and is now\nrecognized by the compiler as a name for the built-in object ``None``.\nAlthough it is not a keyword, you cannot assign a different object to\nit.\n\nChanged in version 2.5: Both ``as`` and ``with`` are only recognized\nwhen the ``with_statement`` future feature has been enabled. It will\nalways be enabled in Python 2.6. See section *The with statement* for\ndetails. Note that using ``as`` and ``with`` as identifiers will\nalways issue a warning, even when the ``with_statement`` future\ndirective is not in effect.\n\n\nReserved classes of identifiers\n===============================\n\nCertain classes of identifiers (besides keywords) have special\nmeanings. These classes are identified by the patterns of leading and\ntrailing underscore characters:\n\n``_*``\n Not imported by ``from module import *``. The special identifier\n ``_`` is used in the interactive interpreter to store the result of\n the last evaluation; it is stored in the ``__builtin__`` module.\n When not in interactive mode, ``_`` has no special meaning and is\n not defined. See section *The import statement*.\n\n Note: The name ``_`` is often used in conjunction with\n internationalization; refer to the documentation for the\n ``gettext`` module for more information on this convention.\n\n``__*__``\n System-defined names. These names are defined by the interpreter\n and its implementation (including the standard library). Current\n system names are discussed in the *Special method names* section\n and elsewhere. More will likely be defined in future versions of\n Python. *Any* use of ``__*__`` names, in any context, that does\n not follow explicitly documented use, is subject to breakage\n without warning.\n\n``__*``\n Class-private names. Names in this category, when used within the\n context of a class definition, are re-written to use a mangled form\n to help avoid name clashes between "private" attributes of base and\n derived classes. See section *Identifiers (Names)*.\n', 'if': u'\nThe ``if`` statement\n********************\n\nThe ``if`` statement is used for conditional execution:\n\n if_stmt ::= "if" expression ":" suite\n ( "elif" expression ":" suite )*\n ["else" ":" suite]\n\nIt selects exactly one of the suites by evaluating the expressions one\nby one until one is found to be true (see section *Boolean operations*\nfor the definition of true and false); then that suite is executed\n(and no other part of the ``if`` statement is executed or evaluated).\nIf all expressions are false, the suite of the ``else`` clause, if\npresent, is executed.\n', 'imaginary': u'\nImaginary literals\n******************\n\nImaginary literals are described by the following lexical definitions:\n\n imagnumber ::= (floatnumber | intpart) ("j" | "J")\n\nAn imaginary literal yields a complex number with a real part of 0.0.\nComplex numbers are represented as a pair of floating point numbers\nand have the same restrictions on their range. To create a complex\nnumber with a nonzero real part, add a floating point number to it,\ne.g., ``(3+4j)``. Some examples of imaginary literals:\n\n 3.14j 10.j 10j .001j 1e100j 3.14e-10j\n', - 'import': u'\nThe ``import`` statement\n************************\n\n import_stmt ::= "import" module ["as" name] ( "," module ["as" name] )*\n | "from" relative_module "import" identifier ["as" name]\n ( "," identifier ["as" name] )*\n | "from" relative_module "import" "(" identifier ["as" name]\n ( "," identifier ["as" name] )* [","] ")"\n | "from" module "import" "*"\n module ::= (identifier ".")* identifier\n relative_module ::= "."* module | "."+\n name ::= identifier\n\nImport statements are executed in two steps: (1) find a module, and\ninitialize it if necessary; (2) define a name or names in the local\nnamespace (of the scope where the ``import`` statement occurs). The\nstatement comes in two forms differing on whether it uses the ``from``\nkeyword. The first form (without ``from``) repeats these steps for\neach identifier in the list. The form with ``from`` performs step (1)\nonce, and then performs step (2) repeatedly.\n\nTo understand how step (1) occurs, one must first understand how\nPython handles hierarchical naming of modules. To help organize\nmodules and provide a hierarchy in naming, Python has a concept of\npackages. A package can contain other packages and modules while\nmodules cannot contain other modules or packages. From a file system\nperspective, packages are directories and modules are files. The\noriginal specification for packages is still available to read,\nalthough minor details have changed since the writing of that\ndocument.\n\nOnce the name of the module is known (unless otherwise specified, the\nterm "module" will refer to both packages and modules), searching for\nthe module or package can begin. The first place checked is\n``sys.modules``, the cache of all modules that have been imported\npreviously. If the module is found there then it is used in step (2)\nof import.\n\nIf the module is not found in the cache, then ``sys.meta_path`` is\nsearched (the specification for ``sys.meta_path`` can be found in\n**PEP 302**). The object is a list of *finder* objects which are\nqueried in order as to whether they know how to load the module by\ncalling their ``find_module()`` method with the name of the module. If\nthe module happens to be contained within a package (as denoted by the\nexistence of a dot in the name), then a second argument to\n``find_module()`` is given as the value of the ``__path__`` attribute\nfrom the parent package (everything up to the last dot in the name of\nthe module being imported). If a finder can find the module it returns\na *loader* (discussed later) or returns ``None``.\n\nIf none of the finders on ``sys.meta_path`` are able to find the\nmodule then some implicitly defined finders are queried.\nImplementations of Python vary in what implicit meta path finders are\ndefined. The one they all do define, though, is one that handles\n``sys.path_hooks``, ``sys.path_importer_cache``, and ``sys.path``.\n\nThe implicit finder searches for the requested module in the "paths"\nspecified in one of two places ("paths" do not have to be file system\npaths). If the module being imported is supposed to be contained\nwithin a package then the second argument passed to ``find_module()``,\n``__path__`` on the parent package, is used as the source of paths. If\nthe module is not contained in a package then ``sys.path`` is used as\nthe source of paths.\n\nOnce the source of paths is chosen it is iterated over to find a\nfinder that can handle that path. The dict at\n``sys.path_importer_cache`` caches finders for paths and is checked\nfor a finder. If the path does not have a finder cached then\n``sys.path_hooks`` is searched by calling each object in the list with\na single argument of the path, returning a finder or raises\n``ImportError``. If a finder is returned then it is cached in\n``sys.path_importer_cache`` and then used for that path entry. If no\nfinder can be found but the path exists then a value of ``None`` is\nstored in ``sys.path_importer_cache`` to signify that an implicit,\nfile-based finder that handles modules stored as individual files\nshould be used for that path. If the path does not exist then a finder\nwhich always returns ``None`` is placed in the cache for the path.\n\nIf no finder can find the module then ``ImportError`` is raised.\nOtherwise some finder returned a loader whose ``load_module()`` method\nis called with the name of the module to load (see **PEP 302** for the\noriginal definition of loaders). A loader has several responsibilities\nto perform on a module it loads. First, if the module already exists\nin ``sys.modules`` (a possibility if the loader is called outside of\nthe import machinery) then it is to use that module for initialization\nand not a new module. But if the module does not exist in\n``sys.modules`` then it is to be added to that dict before\ninitialization begins. If an error occurs during loading of the module\nand it was added to ``sys.modules`` it is to be removed from the dict.\nIf an error occurs but the module was already in ``sys.modules`` it is\nleft in the dict.\n\nThe loader must set several attributes on the module. ``__name__`` is\nto be set to the name of the module. ``__file__`` is to be the "path"\nto the file unless the module is built-in (and thus listed in\n``sys.builtin_module_names``) in which case the attribute is not set.\nIf what is being imported is a package then ``__path__`` is to be set\nto a list of paths to be searched when looking for modules and\npackages contained within the package being imported. ``__package__``\nis optional but should be set to the name of package that contains the\nmodule or package (the empty string is used for module not contained\nin a package). ``__loader__`` is also optional but should be set to\nthe loader object that is loading the module.\n\nIf an error occurs during loading then the loader raises\n``ImportError`` if some other exception is not already being\npropagated. Otherwise the loader returns the module that was loaded\nand initialized.\n\nWhen step (1) finishes without raising an exception, step (2) can\nbegin.\n\nThe first form of ``import`` statement binds the module name in the\nlocal namespace to the module object, and then goes on to import the\nnext identifier, if any. If the module name is followed by ``as``,\nthe name following ``as`` is used as the local name for the module.\n\nThe ``from`` form does not bind the module name: it goes through the\nlist of identifiers, looks each one of them up in the module found in\nstep (1), and binds the name in the local namespace to the object thus\nfound. As with the first form of ``import``, an alternate local name\ncan be supplied by specifying "``as`` localname". If a name is not\nfound, ``ImportError`` is raised. If the list of identifiers is\nreplaced by a star (``\'*\'``), all public names defined in the module\nare bound in the local namespace of the ``import`` statement..\n\nThe *public names* defined by a module are determined by checking the\nmodule\'s namespace for a variable named ``__all__``; if defined, it\nmust be a sequence of strings which are names defined or imported by\nthat module. The names given in ``__all__`` are all considered public\nand are required to exist. If ``__all__`` is not defined, the set of\npublic names includes all names found in the module\'s namespace which\ndo not begin with an underscore character (``\'_\'``). ``__all__``\nshould contain the entire public API. It is intended to avoid\naccidentally exporting items that are not part of the API (such as\nlibrary modules which were imported and used within the module).\n\nThe ``from`` form with ``*`` may only occur in a module scope. If the\nwild card form of import --- ``import *`` --- is used in a function\nand the function contains or is a nested block with free variables,\nthe compiler will raise a ``SyntaxError``.\n\nWhen specifying what module to import you do not have to specify the\nabsolute name of the module. When a module or package is contained\nwithin another package it is possible to make a relative import within\nthe same top package without having to mention the package name. By\nusing leading dots in the specified module or package after ``from``\nyou can specify how high to traverse up the current package hierarchy\nwithout specifying exact names. One leading dot means the current\npackage where the module making the import exists. Two dots means up\none package level. Three dots is up two levels, etc. So if you execute\n``from . import mod`` from a module in the ``pkg`` package then you\nwill end up importing ``pkg.mod``. If you execute ``from ..subpkg2\nimprt mod`` from within ``pkg.subpkg1`` you will import\n``pkg.subpkg2.mod``. The specification for relative imports is\ncontained within **PEP 328**.\n\n``importlib.import_module()`` is provided to support applications that\ndetermine which modules need to be loaded dynamically.\n\n\nFuture statements\n=================\n\nA *future statement* is a directive to the compiler that a particular\nmodule should be compiled using syntax or semantics that will be\navailable in a specified future release of Python. The future\nstatement is intended to ease migration to future versions of Python\nthat introduce incompatible changes to the language. It allows use of\nthe new features on a per-module basis before the release in which the\nfeature becomes standard.\n\n future_statement ::= "from" "__future__" "import" feature ["as" name]\n ("," feature ["as" name])*\n | "from" "__future__" "import" "(" feature ["as" name]\n ("," feature ["as" name])* [","] ")"\n feature ::= identifier\n name ::= identifier\n\nA future statement must appear near the top of the module. The only\nlines that can appear before a future statement are:\n\n* the module docstring (if any),\n\n* comments,\n\n* blank lines, and\n\n* other future statements.\n\nThe features recognized by Python 2.6 are ``unicode_literals``,\n``print_function``, ``absolute_import``, ``division``, ``generators``,\n``nested_scopes`` and ``with_statement``. ``generators``,\n``with_statement``, ``nested_scopes`` are redundant in Python version\n2.6 and above because they are always enabled.\n\nA future statement is recognized and treated specially at compile\ntime: Changes to the semantics of core constructs are often\nimplemented by generating different code. It may even be the case\nthat a new feature introduces new incompatible syntax (such as a new\nreserved word), in which case the compiler may need to parse the\nmodule differently. Such decisions cannot be pushed off until\nruntime.\n\nFor any given release, the compiler knows which feature names have\nbeen defined, and raises a compile-time error if a future statement\ncontains a feature not known to it.\n\nThe direct runtime semantics are the same as for any import statement:\nthere is a standard module ``__future__``, described later, and it\nwill be imported in the usual way at the time the future statement is\nexecuted.\n\nThe interesting runtime semantics depend on the specific feature\nenabled by the future statement.\n\nNote that there is nothing special about the statement:\n\n import __future__ [as name]\n\nThat is not a future statement; it\'s an ordinary import statement with\nno special semantics or syntax restrictions.\n\nCode compiled by an ``exec`` statement or calls to the built-in\nfunctions ``compile()`` and ``execfile()`` that occur in a module\n``M`` containing a future statement will, by default, use the new\nsyntax or semantics associated with the future statement. This can,\nstarting with Python 2.2 be controlled by optional arguments to\n``compile()`` --- see the documentation of that function for details.\n\nA future statement typed at an interactive interpreter prompt will\ntake effect for the rest of the interpreter session. If an\ninterpreter is started with the *-i* option, is passed a script name\nto execute, and the script includes a future statement, it will be in\neffect in the interactive session started after the script is\nexecuted.\n\nSee also:\n\n **PEP 236** - Back to the __future__\n The original proposal for the __future__ mechanism.\n', + 'import': u'\nThe ``import`` statement\n************************\n\n import_stmt ::= "import" module ["as" name] ( "," module ["as" name] )*\n | "from" relative_module "import" identifier ["as" name]\n ( "," identifier ["as" name] )*\n | "from" relative_module "import" "(" identifier ["as" name]\n ( "," identifier ["as" name] )* [","] ")"\n | "from" module "import" "*"\n module ::= (identifier ".")* identifier\n relative_module ::= "."* module | "."+\n name ::= identifier\n\nImport statements are executed in two steps: (1) find a module, and\ninitialize it if necessary; (2) define a name or names in the local\nnamespace (of the scope where the ``import`` statement occurs). The\nstatement comes in two forms differing on whether it uses the ``from``\nkeyword. The first form (without ``from``) repeats these steps for\neach identifier in the list. The form with ``from`` performs step (1)\nonce, and then performs step (2) repeatedly.\n\nTo understand how step (1) occurs, one must first understand how\nPython handles hierarchical naming of modules. To help organize\nmodules and provide a hierarchy in naming, Python has a concept of\npackages. A package can contain other packages and modules while\nmodules cannot contain other modules or packages. From a file system\nperspective, packages are directories and modules are files. The\noriginal specification for packages is still available to read,\nalthough minor details have changed since the writing of that\ndocument.\n\nOnce the name of the module is known (unless otherwise specified, the\nterm "module" will refer to both packages and modules), searching for\nthe module or package can begin. The first place checked is\n``sys.modules``, the cache of all modules that have been imported\npreviously. If the module is found there then it is used in step (2)\nof import.\n\nIf the module is not found in the cache, then ``sys.meta_path`` is\nsearched (the specification for ``sys.meta_path`` can be found in\n**PEP 302**). The object is a list of *finder* objects which are\nqueried in order as to whether they know how to load the module by\ncalling their ``find_module()`` method with the name of the module. If\nthe module happens to be contained within a package (as denoted by the\nexistence of a dot in the name), then a second argument to\n``find_module()`` is given as the value of the ``__path__`` attribute\nfrom the parent package (everything up to the last dot in the name of\nthe module being imported). If a finder can find the module it returns\na *loader* (discussed later) or returns ``None``.\n\nIf none of the finders on ``sys.meta_path`` are able to find the\nmodule then some implicitly defined finders are queried.\nImplementations of Python vary in what implicit meta path finders are\ndefined. The one they all do define, though, is one that handles\n``sys.path_hooks``, ``sys.path_importer_cache``, and ``sys.path``.\n\nThe implicit finder searches for the requested module in the "paths"\nspecified in one of two places ("paths" do not have to be file system\npaths). If the module being imported is supposed to be contained\nwithin a package then the second argument passed to ``find_module()``,\n``__path__`` on the parent package, is used as the source of paths. If\nthe module is not contained in a package then ``sys.path`` is used as\nthe source of paths.\n\nOnce the source of paths is chosen it is iterated over to find a\nfinder that can handle that path. The dict at\n``sys.path_importer_cache`` caches finders for paths and is checked\nfor a finder. If the path does not have a finder cached then\n``sys.path_hooks`` is searched by calling each object in the list with\na single argument of the path, returning a finder or raises\n``ImportError``. If a finder is returned then it is cached in\n``sys.path_importer_cache`` and then used for that path entry. If no\nfinder can be found but the path exists then a value of ``None`` is\nstored in ``sys.path_importer_cache`` to signify that an implicit,\nfile-based finder that handles modules stored as individual files\nshould be used for that path. If the path does not exist then a finder\nwhich always returns ``None`` is placed in the cache for the path.\n\nIf no finder can find the module then ``ImportError`` is raised.\nOtherwise some finder returned a loader whose ``load_module()`` method\nis called with the name of the module to load (see **PEP 302** for the\noriginal definition of loaders). A loader has several responsibilities\nto perform on a module it loads. First, if the module already exists\nin ``sys.modules`` (a possibility if the loader is called outside of\nthe import machinery) then it is to use that module for initialization\nand not a new module. But if the module does not exist in\n``sys.modules`` then it is to be added to that dict before\ninitialization begins. If an error occurs during loading of the module\nand it was added to ``sys.modules`` it is to be removed from the dict.\nIf an error occurs but the module was already in ``sys.modules`` it is\nleft in the dict.\n\nThe loader must set several attributes on the module. ``__name__`` is\nto be set to the name of the module. ``__file__`` is to be the "path"\nto the file unless the module is built-in (and thus listed in\n``sys.builtin_module_names``) in which case the attribute is not set.\nIf what is being imported is a package then ``__path__`` is to be set\nto a list of paths to be searched when looking for modules and\npackages contained within the package being imported. ``__package__``\nis optional but should be set to the name of package that contains the\nmodule or package (the empty string is used for module not contained\nin a package). ``__loader__`` is also optional but should be set to\nthe loader object that is loading the module.\n\nIf an error occurs during loading then the loader raises\n``ImportError`` if some other exception is not already being\npropagated. Otherwise the loader returns the module that was loaded\nand initialized.\n\nWhen step (1) finishes without raising an exception, step (2) can\nbegin.\n\nThe first form of ``import`` statement binds the module name in the\nlocal namespace to the module object, and then goes on to import the\nnext identifier, if any. If the module name is followed by ``as``,\nthe name following ``as`` is used as the local name for the module.\n\nThe ``from`` form does not bind the module name: it goes through the\nlist of identifiers, looks each one of them up in the module found in\nstep (1), and binds the name in the local namespace to the object thus\nfound. As with the first form of ``import``, an alternate local name\ncan be supplied by specifying "``as`` localname". If a name is not\nfound, ``ImportError`` is raised. If the list of identifiers is\nreplaced by a star (``\'*\'``), all public names defined in the module\nare bound in the local namespace of the ``import`` statement..\n\nThe *public names* defined by a module are determined by checking the\nmodule\'s namespace for a variable named ``__all__``; if defined, it\nmust be a sequence of strings which are names defined or imported by\nthat module. The names given in ``__all__`` are all considered public\nand are required to exist. If ``__all__`` is not defined, the set of\npublic names includes all names found in the module\'s namespace which\ndo not begin with an underscore character (``\'_\'``). ``__all__``\nshould contain the entire public API. It is intended to avoid\naccidentally exporting items that are not part of the API (such as\nlibrary modules which were imported and used within the module).\n\nThe ``from`` form with ``*`` may only occur in a module scope. If the\nwild card form of import --- ``import *`` --- is used in a function\nand the function contains or is a nested block with free variables,\nthe compiler will raise a ``SyntaxError``.\n\nWhen specifying what module to import you do not have to specify the\nabsolute name of the module. When a module or package is contained\nwithin another package it is possible to make a relative import within\nthe same top package without having to mention the package name. By\nusing leading dots in the specified module or package after ``from``\nyou can specify how high to traverse up the current package hierarchy\nwithout specifying exact names. One leading dot means the current\npackage where the module making the import exists. Two dots means up\none package level. Three dots is up two levels, etc. So if you execute\n``from . import mod`` from a module in the ``pkg`` package then you\nwill end up importing ``pkg.mod``. If you execute ``from ..subpkg2\nimport mod`` from within ``pkg.subpkg1`` you will import\n``pkg.subpkg2.mod``. The specification for relative imports is\ncontained within **PEP 328**.\n\n``importlib.import_module()`` is provided to support applications that\ndetermine which modules need to be loaded dynamically.\n\n\nFuture statements\n=================\n\nA *future statement* is a directive to the compiler that a particular\nmodule should be compiled using syntax or semantics that will be\navailable in a specified future release of Python. The future\nstatement is intended to ease migration to future versions of Python\nthat introduce incompatible changes to the language. It allows use of\nthe new features on a per-module basis before the release in which the\nfeature becomes standard.\n\n future_statement ::= "from" "__future__" "import" feature ["as" name]\n ("," feature ["as" name])*\n | "from" "__future__" "import" "(" feature ["as" name]\n ("," feature ["as" name])* [","] ")"\n feature ::= identifier\n name ::= identifier\n\nA future statement must appear near the top of the module. The only\nlines that can appear before a future statement are:\n\n* the module docstring (if any),\n\n* comments,\n\n* blank lines, and\n\n* other future statements.\n\nThe features recognized by Python 2.6 are ``unicode_literals``,\n``print_function``, ``absolute_import``, ``division``, ``generators``,\n``nested_scopes`` and ``with_statement``. ``generators``,\n``with_statement``, ``nested_scopes`` are redundant in Python version\n2.6 and above because they are always enabled.\n\nA future statement is recognized and treated specially at compile\ntime: Changes to the semantics of core constructs are often\nimplemented by generating different code. It may even be the case\nthat a new feature introduces new incompatible syntax (such as a new\nreserved word), in which case the compiler may need to parse the\nmodule differently. Such decisions cannot be pushed off until\nruntime.\n\nFor any given release, the compiler knows which feature names have\nbeen defined, and raises a compile-time error if a future statement\ncontains a feature not known to it.\n\nThe direct runtime semantics are the same as for any import statement:\nthere is a standard module ``__future__``, described later, and it\nwill be imported in the usual way at the time the future statement is\nexecuted.\n\nThe interesting runtime semantics depend on the specific feature\nenabled by the future statement.\n\nNote that there is nothing special about the statement:\n\n import __future__ [as name]\n\nThat is not a future statement; it\'s an ordinary import statement with\nno special semantics or syntax restrictions.\n\nCode compiled by an ``exec`` statement or calls to the built-in\nfunctions ``compile()`` and ``execfile()`` that occur in a module\n``M`` containing a future statement will, by default, use the new\nsyntax or semantics associated with the future statement. This can,\nstarting with Python 2.2 be controlled by optional arguments to\n``compile()`` --- see the documentation of that function for details.\n\nA future statement typed at an interactive interpreter prompt will\ntake effect for the rest of the interpreter session. If an\ninterpreter is started with the *-i* option, is passed a script name\nto execute, and the script includes a future statement, it will be in\neffect in the interactive session started after the script is\nexecuted.\n\nSee also:\n\n **PEP 236** - Back to the __future__\n The original proposal for the __future__ mechanism.\n', 'in': u'\nComparisons\n***********\n\nUnlike C, all comparison operations in Python have the same priority,\nwhich is lower than that of any arithmetic, shifting or bitwise\noperation. Also unlike C, expressions like ``a < b < c`` have the\ninterpretation that is conventional in mathematics:\n\n comparison ::= or_expr ( comp_operator or_expr )*\n comp_operator ::= "<" | ">" | "==" | ">=" | "<=" | "<>" | "!="\n | "is" ["not"] | ["not"] "in"\n\nComparisons yield boolean values: ``True`` or ``False``.\n\nComparisons can be chained arbitrarily, e.g., ``x < y <= z`` is\nequivalent to ``x < y and y <= z``, except that ``y`` is evaluated\nonly once (but in both cases ``z`` is not evaluated at all when ``x <\ny`` is found to be false).\n\nFormally, if *a*, *b*, *c*, ..., *y*, *z* are expressions and *op1*,\n*op2*, ..., *opN* are comparison operators, then ``a op1 b op2 c ... y\nopN z`` is equivalent to ``a op1 b and b op2 c and ... y opN z``,\nexcept that each expression is evaluated at most once.\n\nNote that ``a op1 b op2 c`` doesn\'t imply any kind of comparison\nbetween *a* and *c*, so that, e.g., ``x < y > z`` is perfectly legal\n(though perhaps not pretty).\n\nThe forms ``<>`` and ``!=`` are equivalent; for consistency with C,\n``!=`` is preferred; where ``!=`` is mentioned below ``<>`` is also\naccepted. The ``<>`` spelling is considered obsolescent.\n\nThe operators ``<``, ``>``, ``==``, ``>=``, ``<=``, and ``!=`` compare\nthe values of two objects. The objects need not have the same type.\nIf both are numbers, they are converted to a common type. Otherwise,\nobjects of different types *always* compare unequal, and are ordered\nconsistently but arbitrarily. You can control comparison behavior of\nobjects of non-built-in types by defining a ``__cmp__`` method or rich\ncomparison methods like ``__gt__``, described in section *Special\nmethod names*.\n\n(This unusual definition of comparison was used to simplify the\ndefinition of operations like sorting and the ``in`` and ``not in``\noperators. In the future, the comparison rules for objects of\ndifferent types are likely to change.)\n\nComparison of objects of the same type depends on the type:\n\n* Numbers are compared arithmetically.\n\n* Strings are compared lexicographically using the numeric equivalents\n (the result of the built-in function ``ord()``) of their characters.\n Unicode and 8-bit strings are fully interoperable in this behavior.\n [4]\n\n* Tuples and lists are compared lexicographically using comparison of\n corresponding elements. This means that to compare equal, each\n element must compare equal and the two sequences must be of the same\n type and have the same length.\n\n If not equal, the sequences are ordered the same as their first\n differing elements. For example, ``cmp([1,2,x], [1,2,y])`` returns\n the same as ``cmp(x,y)``. If the corresponding element does not\n exist, the shorter sequence is ordered first (for example, ``[1,2] <\n [1,2,3]``).\n\n* Mappings (dictionaries) compare equal if and only if their sorted\n (key, value) lists compare equal. [5] Outcomes other than equality\n are resolved consistently, but are not otherwise defined. [6]\n\n* Most other objects of built-in types compare unequal unless they are\n the same object; the choice whether one object is considered smaller\n or larger than another one is made arbitrarily but consistently\n within one execution of a program.\n\nThe operators ``in`` and ``not in`` test for collection membership.\n``x in s`` evaluates to true if *x* is a member of the collection *s*,\nand false otherwise. ``x not in s`` returns the negation of ``x in\ns``. The collection membership test has traditionally been bound to\nsequences; an object is a member of a collection if the collection is\na sequence and contains an element equal to that object. However, it\nmake sense for many other object types to support membership tests\nwithout being a sequence. In particular, dictionaries (for keys) and\nsets support membership testing.\n\nFor the list and tuple types, ``x in y`` is true if and only if there\nexists an index *i* such that ``x == y[i]`` is true.\n\nFor the Unicode and string types, ``x in y`` is true if and only if\n*x* is a substring of *y*. An equivalent test is ``y.find(x) != -1``.\nNote, *x* and *y* need not be the same type; consequently, ``u\'ab\' in\n\'abc\'`` will return ``True``. Empty strings are always considered to\nbe a substring of any other string, so ``"" in "abc"`` will return\n``True``.\n\nChanged in version 2.3: Previously, *x* was required to be a string of\nlength ``1``.\n\nFor user-defined classes which define the ``__contains__()`` method,\n``x in y`` is true if and only if ``y.__contains__(x)`` is true.\n\nFor user-defined classes which do not define ``__contains__()`` but do\ndefine ``__iter__()``, ``x in y`` is true if some value ``z`` with ``x\n== z`` is produced while iterating over ``y``. If an exception is\nraised during the iteration, it is as if ``in`` raised that exception.\n\nLastly, the old-style iteration protocol is tried: if a class defines\n``__getitem__()``, ``x in y`` is true if and only if there is a non-\nnegative integer index *i* such that ``x == y[i]``, and all lower\ninteger indices do not raise ``IndexError`` exception. (If any other\nexception is raised, it is as if ``in`` raised that exception).\n\nThe operator ``not in`` is defined to have the inverse true value of\n``in``.\n\nThe operators ``is`` and ``is not`` test for object identity: ``x is\ny`` is true if and only if *x* and *y* are the same object. ``x is\nnot y`` yields the inverse truth value. [7]\n', 'integers': u'\nInteger and long integer literals\n*********************************\n\nInteger and long integer literals are described by the following\nlexical definitions:\n\n longinteger ::= integer ("l" | "L")\n integer ::= decimalinteger | octinteger | hexinteger | bininteger\n decimalinteger ::= nonzerodigit digit* | "0"\n octinteger ::= "0" ("o" | "O") octdigit+ | "0" octdigit+\n hexinteger ::= "0" ("x" | "X") hexdigit+\n bininteger ::= "0" ("b" | "B") bindigit+\n nonzerodigit ::= "1"..."9"\n octdigit ::= "0"..."7"\n bindigit ::= "0" | "1"\n hexdigit ::= digit | "a"..."f" | "A"..."F"\n\nAlthough both lower case ``\'l\'`` and upper case ``\'L\'`` are allowed as\nsuffix for long integers, it is strongly recommended to always use\n``\'L\'``, since the letter ``\'l\'`` looks too much like the digit\n``\'1\'``.\n\nPlain integer literals that are above the largest representable plain\ninteger (e.g., 2147483647 when using 32-bit arithmetic) are accepted\nas if they were long integers instead. [1] There is no limit for long\ninteger literals apart from what can be stored in available memory.\n\nSome examples of plain integer literals (first row) and long integer\nliterals (second and third rows):\n\n 7 2147483647 0177\n 3L 79228162514264337593543950336L 0377L 0x100000000L\n 79228162514264337593543950336 0xdeadbeef\n', 'lambda': u'\nLambdas\n*******\n\n lambda_form ::= "lambda" [parameter_list]: expression\n old_lambda_form ::= "lambda" [parameter_list]: old_expression\n\nLambda forms (lambda expressions) have the same syntactic position as\nexpressions. They are a shorthand to create anonymous functions; the\nexpression ``lambda arguments: expression`` yields a function object.\nThe unnamed object behaves like a function object defined with\n\n def name(arguments):\n return expression\n\nSee section *Function definitions* for the syntax of parameter lists.\nNote that functions created with lambda forms cannot contain\nstatements.\n', 'lists': u'\nList displays\n*************\n\nA list display is a possibly empty series of expressions enclosed in\nsquare brackets:\n\n list_display ::= "[" [expression_list | list_comprehension] "]"\n list_comprehension ::= expression list_for\n list_for ::= "for" target_list "in" old_expression_list [list_iter]\n old_expression_list ::= old_expression [("," old_expression)+ [","]]\n old_expression ::= or_test | old_lambda_form\n list_iter ::= list_for | list_if\n list_if ::= "if" old_expression [list_iter]\n\nA list display yields a new list object. Its contents are specified\nby providing either a list of expressions or a list comprehension.\nWhen a comma-separated list of expressions is supplied, its elements\nare evaluated from left to right and placed into the list object in\nthat order. When a list comprehension is supplied, it consists of a\nsingle expression followed by at least one ``for`` clause and zero or\nmore ``for`` or ``if`` clauses. In this case, the elements of the new\nlist are those that would be produced by considering each of the\n``for`` or ``if`` clauses a block, nesting from left to right, and\nevaluating the expression to produce a list element each time the\ninnermost block is reached [1].\n', - 'naming': u"\nNaming and binding\n******************\n\n*Names* refer to objects. Names are introduced by name binding\noperations. Each occurrence of a name in the program text refers to\nthe *binding* of that name established in the innermost function block\ncontaining the use.\n\nA *block* is a piece of Python program text that is executed as a\nunit. The following are blocks: a module, a function body, and a class\ndefinition. Each command typed interactively is a block. A script\nfile (a file given as standard input to the interpreter or specified\non the interpreter command line the first argument) is a code block.\nA script command (a command specified on the interpreter command line\nwith the '**-c**' option) is a code block. The file read by the\nbuilt-in function ``execfile()`` is a code block. The string argument\npassed to the built-in function ``eval()`` and to the ``exec``\nstatement is a code block. The expression read and evaluated by the\nbuilt-in function ``input()`` is a code block.\n\nA code block is executed in an *execution frame*. A frame contains\nsome administrative information (used for debugging) and determines\nwhere and how execution continues after the code block's execution has\ncompleted.\n\nA *scope* defines the visibility of a name within a block. If a local\nvariable is defined in a block, its scope includes that block. If the\ndefinition occurs in a function block, the scope extends to any blocks\ncontained within the defining one, unless a contained block introduces\na different binding for the name. The scope of names defined in a\nclass block is limited to the class block; it does not extend to the\ncode blocks of methods -- this includes generator expressions since\nthey are implemented using a function scope. This means that the\nfollowing will fail:\n\n class A:\n a = 42\n b = list(a + i for i in range(10))\n\nWhen a name is used in a code block, it is resolved using the nearest\nenclosing scope. The set of all such scopes visible to a code block\nis called the block's *environment*.\n\nIf a name is bound in a block, it is a local variable of that block.\nIf a name is bound at the module level, it is a global variable. (The\nvariables of the module code block are local and global.) If a\nvariable is used in a code block but not defined there, it is a *free\nvariable*.\n\nWhen a name is not found at all, a ``NameError`` exception is raised.\nIf the name refers to a local variable that has not been bound, a\n``UnboundLocalError`` exception is raised. ``UnboundLocalError`` is a\nsubclass of ``NameError``.\n\nThe following constructs bind names: formal parameters to functions,\n``import`` statements, class and function definitions (these bind the\nclass or function name in the defining block), and targets that are\nidentifiers if occurring in an assignment, ``for`` loop header, in the\nsecond position of an ``except`` clause header or after ``as`` in a\n``with`` statement. The ``import`` statement of the form ``from ...\nimport *`` binds all names defined in the imported module, except\nthose beginning with an underscore. This form may only be used at the\nmodule level.\n\nA target occurring in a ``del`` statement is also considered bound for\nthis purpose (though the actual semantics are to unbind the name). It\nis illegal to unbind a name that is referenced by an enclosing scope;\nthe compiler will report a ``SyntaxError``.\n\nEach assignment or import statement occurs within a block defined by a\nclass or function definition or at the module level (the top-level\ncode block).\n\nIf a name binding operation occurs anywhere within a code block, all\nuses of the name within the block are treated as references to the\ncurrent block. This can lead to errors when a name is used within a\nblock before it is bound. This rule is subtle. Python lacks\ndeclarations and allows name binding operations to occur anywhere\nwithin a code block. The local variables of a code block can be\ndetermined by scanning the entire text of the block for name binding\noperations.\n\nIf the global statement occurs within a block, all uses of the name\nspecified in the statement refer to the binding of that name in the\ntop-level namespace. Names are resolved in the top-level namespace by\nsearching the global namespace, i.e. the namespace of the module\ncontaining the code block, and the builtins namespace, the namespace\nof the module ``__builtin__``. The global namespace is searched\nfirst. If the name is not found there, the builtins namespace is\nsearched. The global statement must precede all uses of the name.\n\nThe builtins namespace associated with the execution of a code block\nis actually found by looking up the name ``__builtins__`` in its\nglobal namespace; this should be a dictionary or a module (in the\nlatter case the module's dictionary is used). By default, when in the\n``__main__`` module, ``__builtins__`` is the built-in module\n``__builtin__`` (note: no 's'); when in any other module,\n``__builtins__`` is an alias for the dictionary of the ``__builtin__``\nmodule itself. ``__builtins__`` can be set to a user-created\ndictionary to create a weak form of restricted execution.\n\n**CPython implementation detail:** Users should not touch\n``__builtins__``; it is strictly an implementation detail. Users\nwanting to override values in the builtins namespace should ``import``\nthe ``__builtin__`` (no 's') module and modify its attributes\nappropriately.\n\nThe namespace for a module is automatically created the first time a\nmodule is imported. The main module for a script is always called\n``__main__``.\n\nThe global statement has the same scope as a name binding operation in\nthe same block. If the nearest enclosing scope for a free variable\ncontains a global statement, the free variable is treated as a global.\n\nA class definition is an executable statement that may use and define\nnames. These references follow the normal rules for name resolution.\nThe namespace of the class definition becomes the attribute dictionary\nof the class. Names defined at the class scope are not visible in\nmethods.\n\n\nInteraction with dynamic features\n=================================\n\nThere are several cases where Python statements are illegal when used\nin conjunction with nested scopes that contain free variables.\n\nIf a variable is referenced in an enclosing scope, it is illegal to\ndelete the name. An error will be reported at compile time.\n\nIf the wild card form of import --- ``import *`` --- is used in a\nfunction and the function contains or is a nested block with free\nvariables, the compiler will raise a ``SyntaxError``.\n\nIf ``exec`` is used in a function and the function contains or is a\nnested block with free variables, the compiler will raise a\n``SyntaxError`` unless the exec explicitly specifies the local\nnamespace for the ``exec``. (In other words, ``exec obj`` would be\nillegal, but ``exec obj in ns`` would be legal.)\n\nThe ``eval()``, ``execfile()``, and ``input()`` functions and the\n``exec`` statement do not have access to the full environment for\nresolving names. Names may be resolved in the local and global\nnamespaces of the caller. Free variables are not resolved in the\nnearest enclosing namespace, but in the global namespace. [1] The\n``exec`` statement and the ``eval()`` and ``execfile()`` functions\nhave optional arguments to override the global and local namespace.\nIf only one namespace is specified, it is used for both.\n", + 'naming': u"\nNaming and binding\n******************\n\n*Names* refer to objects. Names are introduced by name binding\noperations. Each occurrence of a name in the program text refers to\nthe *binding* of that name established in the innermost function block\ncontaining the use.\n\nA *block* is a piece of Python program text that is executed as a\nunit. The following are blocks: a module, a function body, and a class\ndefinition. Each command typed interactively is a block. A script\nfile (a file given as standard input to the interpreter or specified\non the interpreter command line the first argument) is a code block.\nA script command (a command specified on the interpreter command line\nwith the '**-c**' option) is a code block. The file read by the\nbuilt-in function ``execfile()`` is a code block. The string argument\npassed to the built-in function ``eval()`` and to the ``exec``\nstatement is a code block. The expression read and evaluated by the\nbuilt-in function ``input()`` is a code block.\n\nA code block is executed in an *execution frame*. A frame contains\nsome administrative information (used for debugging) and determines\nwhere and how execution continues after the code block's execution has\ncompleted.\n\nA *scope* defines the visibility of a name within a block. If a local\nvariable is defined in a block, its scope includes that block. If the\ndefinition occurs in a function block, the scope extends to any blocks\ncontained within the defining one, unless a contained block introduces\na different binding for the name. The scope of names defined in a\nclass block is limited to the class block; it does not extend to the\ncode blocks of methods -- this includes generator expressions since\nthey are implemented using a function scope. This means that the\nfollowing will fail:\n\n class A:\n a = 42\n b = list(a + i for i in range(10))\n\nWhen a name is used in a code block, it is resolved using the nearest\nenclosing scope. The set of all such scopes visible to a code block\nis called the block's *environment*.\n\nIf a name is bound in a block, it is a local variable of that block.\nIf a name is bound at the module level, it is a global variable. (The\nvariables of the module code block are local and global.) If a\nvariable is used in a code block but not defined there, it is a *free\nvariable*.\n\nWhen a name is not found at all, a ``NameError`` exception is raised.\nIf the name refers to a local variable that has not been bound, a\n``UnboundLocalError`` exception is raised. ``UnboundLocalError`` is a\nsubclass of ``NameError``.\n\nThe following constructs bind names: formal parameters to functions,\n``import`` statements, class and function definitions (these bind the\nclass or function name in the defining block), and targets that are\nidentifiers if occurring in an assignment, ``for`` loop header, in the\nsecond position of an ``except`` clause header or after ``as`` in a\n``with`` statement. The ``import`` statement of the form ``from ...\nimport *`` binds all names defined in the imported module, except\nthose beginning with an underscore. This form may only be used at the\nmodule level.\n\nA target occurring in a ``del`` statement is also considered bound for\nthis purpose (though the actual semantics are to unbind the name). It\nis illegal to unbind a name that is referenced by an enclosing scope;\nthe compiler will report a ``SyntaxError``.\n\nEach assignment or import statement occurs within a block defined by a\nclass or function definition or at the module level (the top-level\ncode block).\n\nIf a name binding operation occurs anywhere within a code block, all\nuses of the name within the block are treated as references to the\ncurrent block. This can lead to errors when a name is used within a\nblock before it is bound. This rule is subtle. Python lacks\ndeclarations and allows name binding operations to occur anywhere\nwithin a code block. The local variables of a code block can be\ndetermined by scanning the entire text of the block for name binding\noperations.\n\nIf the global statement occurs within a block, all uses of the name\nspecified in the statement refer to the binding of that name in the\ntop-level namespace. Names are resolved in the top-level namespace by\nsearching the global namespace, i.e. the namespace of the module\ncontaining the code block, and the builtins namespace, the namespace\nof the module ``__builtin__``. The global namespace is searched\nfirst. If the name is not found there, the builtins namespace is\nsearched. The global statement must precede all uses of the name.\n\nThe builtins namespace associated with the execution of a code block\nis actually found by looking up the name ``__builtins__`` in its\nglobal namespace; this should be a dictionary or a module (in the\nlatter case the module's dictionary is used). By default, when in the\n``__main__`` module, ``__builtins__`` is the built-in module\n``__builtin__`` (note: no 's'); when in any other module,\n``__builtins__`` is an alias for the dictionary of the ``__builtin__``\nmodule itself. ``__builtins__`` can be set to a user-created\ndictionary to create a weak form of restricted execution.\n\n**CPython implementation detail:** Users should not touch\n``__builtins__``; it is strictly an implementation detail. Users\nwanting to override values in the builtins namespace should ``import``\nthe ``__builtin__`` (no 's') module and modify its attributes\nappropriately.\n\nThe namespace for a module is automatically created the first time a\nmodule is imported. The main module for a script is always called\n``__main__``.\n\nThe ``global`` statement has the same scope as a name binding\noperation in the same block. If the nearest enclosing scope for a\nfree variable contains a global statement, the free variable is\ntreated as a global.\n\nA class definition is an executable statement that may use and define\nnames. These references follow the normal rules for name resolution.\nThe namespace of the class definition becomes the attribute dictionary\nof the class. Names defined at the class scope are not visible in\nmethods.\n\n\nInteraction with dynamic features\n=================================\n\nThere are several cases where Python statements are illegal when used\nin conjunction with nested scopes that contain free variables.\n\nIf a variable is referenced in an enclosing scope, it is illegal to\ndelete the name. An error will be reported at compile time.\n\nIf the wild card form of import --- ``import *`` --- is used in a\nfunction and the function contains or is a nested block with free\nvariables, the compiler will raise a ``SyntaxError``.\n\nIf ``exec`` is used in a function and the function contains or is a\nnested block with free variables, the compiler will raise a\n``SyntaxError`` unless the exec explicitly specifies the local\nnamespace for the ``exec``. (In other words, ``exec obj`` would be\nillegal, but ``exec obj in ns`` would be legal.)\n\nThe ``eval()``, ``execfile()``, and ``input()`` functions and the\n``exec`` statement do not have access to the full environment for\nresolving names. Names may be resolved in the local and global\nnamespaces of the caller. Free variables are not resolved in the\nnearest enclosing namespace, but in the global namespace. [1] The\n``exec`` statement and the ``eval()`` and ``execfile()`` functions\nhave optional arguments to override the global and local namespace.\nIf only one namespace is specified, it is used for both.\n", 'numbers': u"\nNumeric literals\n****************\n\nThere are four types of numeric literals: plain integers, long\nintegers, floating point numbers, and imaginary numbers. There are no\ncomplex literals (complex numbers can be formed by adding a real\nnumber and an imaginary number).\n\nNote that numeric literals do not include a sign; a phrase like ``-1``\nis actually an expression composed of the unary operator '``-``' and\nthe literal ``1``.\n", 'numeric-types': u'\nEmulating numeric types\n***********************\n\nThe following methods can be defined to emulate numeric objects.\nMethods corresponding to operations that are not supported by the\nparticular kind of number implemented (e.g., bitwise operations for\nnon-integral numbers) should be left undefined.\n\nobject.__add__(self, other)\nobject.__sub__(self, other)\nobject.__mul__(self, other)\nobject.__floordiv__(self, other)\nobject.__mod__(self, other)\nobject.__divmod__(self, other)\nobject.__pow__(self, other[, modulo])\nobject.__lshift__(self, other)\nobject.__rshift__(self, other)\nobject.__and__(self, other)\nobject.__xor__(self, other)\nobject.__or__(self, other)\n\n These methods are called to implement the binary arithmetic\n operations (``+``, ``-``, ``*``, ``//``, ``%``, ``divmod()``,\n ``pow()``, ``**``, ``<<``, ``>>``, ``&``, ``^``, ``|``). For\n instance, to evaluate the expression ``x + y``, where *x* is an\n instance of a class that has an ``__add__()`` method,\n ``x.__add__(y)`` is called. The ``__divmod__()`` method should be\n the equivalent to using ``__floordiv__()`` and ``__mod__()``; it\n should not be related to ``__truediv__()`` (described below). Note\n that ``__pow__()`` should be defined to accept an optional third\n argument if the ternary version of the built-in ``pow()`` function\n is to be supported.\n\n If one of those methods does not support the operation with the\n supplied arguments, it should return ``NotImplemented``.\n\nobject.__div__(self, other)\nobject.__truediv__(self, other)\n\n The division operator (``/``) is implemented by these methods. The\n ``__truediv__()`` method is used when ``__future__.division`` is in\n effect, otherwise ``__div__()`` is used. If only one of these two\n methods is defined, the object will not support division in the\n alternate context; ``TypeError`` will be raised instead.\n\nobject.__radd__(self, other)\nobject.__rsub__(self, other)\nobject.__rmul__(self, other)\nobject.__rdiv__(self, other)\nobject.__rtruediv__(self, other)\nobject.__rfloordiv__(self, other)\nobject.__rmod__(self, other)\nobject.__rdivmod__(self, other)\nobject.__rpow__(self, other)\nobject.__rlshift__(self, other)\nobject.__rrshift__(self, other)\nobject.__rand__(self, other)\nobject.__rxor__(self, other)\nobject.__ror__(self, other)\n\n These methods are called to implement the binary arithmetic\n operations (``+``, ``-``, ``*``, ``/``, ``%``, ``divmod()``,\n ``pow()``, ``**``, ``<<``, ``>>``, ``&``, ``^``, ``|``) with\n reflected (swapped) operands. These functions are only called if\n the left operand does not support the corresponding operation and\n the operands are of different types. [2] For instance, to evaluate\n the expression ``x - y``, where *y* is an instance of a class that\n has an ``__rsub__()`` method, ``y.__rsub__(x)`` is called if\n ``x.__sub__(y)`` returns *NotImplemented*.\n\n Note that ternary ``pow()`` will not try calling ``__rpow__()``\n (the coercion rules would become too complicated).\n\n Note: If the right operand\'s type is a subclass of the left operand\'s\n type and that subclass provides the reflected method for the\n operation, this method will be called before the left operand\'s\n non-reflected method. This behavior allows subclasses to\n override their ancestors\' operations.\n\nobject.__iadd__(self, other)\nobject.__isub__(self, other)\nobject.__imul__(self, other)\nobject.__idiv__(self, other)\nobject.__itruediv__(self, other)\nobject.__ifloordiv__(self, other)\nobject.__imod__(self, other)\nobject.__ipow__(self, other[, modulo])\nobject.__ilshift__(self, other)\nobject.__irshift__(self, other)\nobject.__iand__(self, other)\nobject.__ixor__(self, other)\nobject.__ior__(self, other)\n\n These methods are called to implement the augmented arithmetic\n assignments (``+=``, ``-=``, ``*=``, ``/=``, ``//=``, ``%=``,\n ``**=``, ``<<=``, ``>>=``, ``&=``, ``^=``, ``|=``). These methods\n should attempt to do the operation in-place (modifying *self*) and\n return the result (which could be, but does not have to be,\n *self*). If a specific method is not defined, the augmented\n assignment falls back to the normal methods. For instance, to\n execute the statement ``x += y``, where *x* is an instance of a\n class that has an ``__iadd__()`` method, ``x.__iadd__(y)`` is\n called. If *x* is an instance of a class that does not define a\n ``__iadd__()`` method, ``x.__add__(y)`` and ``y.__radd__(x)`` are\n considered, as with the evaluation of ``x + y``.\n\nobject.__neg__(self)\nobject.__pos__(self)\nobject.__abs__(self)\nobject.__invert__(self)\n\n Called to implement the unary arithmetic operations (``-``, ``+``,\n ``abs()`` and ``~``).\n\nobject.__complex__(self)\nobject.__int__(self)\nobject.__long__(self)\nobject.__float__(self)\n\n Called to implement the built-in functions ``complex()``,\n ``int()``, ``long()``, and ``float()``. Should return a value of\n the appropriate type.\n\nobject.__oct__(self)\nobject.__hex__(self)\n\n Called to implement the built-in functions ``oct()`` and ``hex()``.\n Should return a string value.\n\nobject.__index__(self)\n\n Called to implement ``operator.index()``. Also called whenever\n Python needs an integer object (such as in slicing). Must return\n an integer (int or long).\n\n New in version 2.5.\n\nobject.__coerce__(self, other)\n\n Called to implement "mixed-mode" numeric arithmetic. Should either\n return a 2-tuple containing *self* and *other* converted to a\n common numeric type, or ``None`` if conversion is impossible. When\n the common type would be the type of ``other``, it is sufficient to\n return ``None``, since the interpreter will also ask the other\n object to attempt a coercion (but sometimes, if the implementation\n of the other type cannot be changed, it is useful to do the\n conversion to the other type here). A return value of\n ``NotImplemented`` is equivalent to returning ``None``.\n', - 'objects': u'\nObjects, values and types\n*************************\n\n*Objects* are Python\'s abstraction for data. All data in a Python\nprogram is represented by objects or by relations between objects. (In\na sense, and in conformance to Von Neumann\'s model of a "stored\nprogram computer," code is also represented by objects.)\n\nEvery object has an identity, a type and a value. An object\'s\n*identity* never changes once it has been created; you may think of it\nas the object\'s address in memory. The \'``is``\' operator compares the\nidentity of two objects; the ``id()`` function returns an integer\nrepresenting its identity (currently implemented as its address). An\nobject\'s *type* is also unchangeable. [1] An object\'s type determines\nthe operations that the object supports (e.g., "does it have a\nlength?") and also defines the possible values for objects of that\ntype. The ``type()`` function returns an object\'s type (which is an\nobject itself). The *value* of some objects can change. Objects\nwhose value can change are said to be *mutable*; objects whose value\nis unchangeable once they are created are called *immutable*. (The\nvalue of an immutable container object that contains a reference to a\nmutable object can change when the latter\'s value is changed; however\nthe container is still considered immutable, because the collection of\nobjects it contains cannot be changed. So, immutability is not\nstrictly the same as having an unchangeable value, it is more subtle.)\nAn object\'s mutability is determined by its type; for instance,\nnumbers, strings and tuples are immutable, while dictionaries and\nlists are mutable.\n\nObjects are never explicitly destroyed; however, when they become\nunreachable they may be garbage-collected. An implementation is\nallowed to postpone garbage collection or omit it altogether --- it is\na matter of implementation quality how garbage collection is\nimplemented, as long as no objects are collected that are still\nreachable.\n\n**CPython implementation detail:** CPython currently uses a reference-\ncounting scheme with (optional) delayed detection of cyclically linked\ngarbage, which collects most objects as soon as they become\nunreachable, but is not guaranteed to collect garbage containing\ncircular references. See the documentation of the ``gc`` module for\ninformation on controlling the collection of cyclic garbage. Other\nimplementations act differently and CPython may change.\n\nNote that the use of the implementation\'s tracing or debugging\nfacilities may keep objects alive that would normally be collectable.\nAlso note that catching an exception with a \'``try``...``except``\'\nstatement may keep objects alive.\n\nSome objects contain references to "external" resources such as open\nfiles or windows. It is understood that these resources are freed\nwhen the object is garbage-collected, but since garbage collection is\nnot guaranteed to happen, such objects also provide an explicit way to\nrelease the external resource, usually a ``close()`` method. Programs\nare strongly recommended to explicitly close such objects. The\n\'``try``...``finally``\' statement provides a convenient way to do\nthis.\n\nSome objects contain references to other objects; these are called\n*containers*. Examples of containers are tuples, lists and\ndictionaries. The references are part of a container\'s value. In\nmost cases, when we talk about the value of a container, we imply the\nvalues, not the identities of the contained objects; however, when we\ntalk about the mutability of a container, only the identities of the\nimmediately contained objects are implied. So, if an immutable\ncontainer (like a tuple) contains a reference to a mutable object, its\nvalue changes if that mutable object is changed.\n\nTypes affect almost all aspects of object behavior. Even the\nimportance of object identity is affected in some sense: for immutable\ntypes, operations that compute new values may actually return a\nreference to any existing object with the same type and value, while\nfor mutable objects this is not allowed. E.g., after ``a = 1; b =\n1``, ``a`` and ``b`` may or may not refer to the same object with the\nvalue one, depending on the implementation, but after ``c = []; d =\n[]``, ``c`` and ``d`` are guaranteed to refer to two different,\nunique, newly created empty lists. (Note that ``c = d = []`` assigns\nthe same object to both ``c`` and ``d``.)\n', - 'operator-summary': u'\nSummary\n*******\n\nThe following table summarizes the operator precedences in Python,\nfrom lowest precedence (least binding) to highest precedence (most\nbinding). Operators in the same box have the same precedence. Unless\nthe syntax is explicitly given, operators are binary. Operators in\nthe same box group left to right (except for comparisons, including\ntests, which all have the same precedence and chain from left to right\n--- see section *Comparisons* --- and exponentiation, which groups\nfrom right to left).\n\n+-------------------------------------------------+---------------------------------------+\n| Operator | Description |\n+=================================================+=======================================+\n| ``lambda`` | Lambda expression |\n+-------------------------------------------------+---------------------------------------+\n| ``if`` -- ``else`` | Conditional expression |\n+-------------------------------------------------+---------------------------------------+\n| ``or`` | Boolean OR |\n+-------------------------------------------------+---------------------------------------+\n| ``and`` | Boolean AND |\n+-------------------------------------------------+---------------------------------------+\n| ``not`` *x* | Boolean NOT |\n+-------------------------------------------------+---------------------------------------+\n| ``in``, ``not`` ``in``, ``is``, ``is not``, | Comparisons, including membership |\n| ``<``, ``<=``, ``>``, ``>=``, ``<>``, ``!=``, | tests and identity tests, |\n| ``==`` | |\n+-------------------------------------------------+---------------------------------------+\n| ``|`` | Bitwise OR |\n+-------------------------------------------------+---------------------------------------+\n| ``^`` | Bitwise XOR |\n+-------------------------------------------------+---------------------------------------+\n| ``&`` | Bitwise AND |\n+-------------------------------------------------+---------------------------------------+\n| ``<<``, ``>>`` | Shifts |\n+-------------------------------------------------+---------------------------------------+\n| ``+``, ``-`` | Addition and subtraction |\n+-------------------------------------------------+---------------------------------------+\n| ``*``, ``/``, ``//``, ``%`` | Multiplication, division, remainder |\n+-------------------------------------------------+---------------------------------------+\n| ``+x``, ``-x``, ``~x`` | Positive, negative, bitwise NOT |\n+-------------------------------------------------+---------------------------------------+\n| ``**`` | Exponentiation [8] |\n+-------------------------------------------------+---------------------------------------+\n| ``x[index]``, ``x[index:index]``, | Subscription, slicing, call, |\n| ``x(arguments...)``, ``x.attribute`` | attribute reference |\n+-------------------------------------------------+---------------------------------------+\n| ``(expressions...)``, ``[expressions...]``, | Binding or tuple display, list |\n| ``{key:datum...}``, ```expressions...``` | display, dictionary display, string |\n| | conversion |\n+-------------------------------------------------+---------------------------------------+\n\n-[ Footnotes ]-\n\n[1] In Python 2.3 and later releases, a list comprehension "leaks" the\n control variables of each ``for`` it contains into the containing\n scope. However, this behavior is deprecated, and relying on it\n will not work in Python 3.0\n\n[2] While ``abs(x%y) < abs(y)`` is true mathematically, for floats it\n may not be true numerically due to roundoff. For example, and\n assuming a platform on which a Python float is an IEEE 754 double-\n precision number, in order that ``-1e-100 % 1e100`` have the same\n sign as ``1e100``, the computed result is ``-1e-100 + 1e100``,\n which is numerically exactly equal to ``1e100``. Function\n ``fmod()`` in the ``math`` module returns a result whose sign\n matches the sign of the first argument instead, and so returns\n ``-1e-100`` in this case. Which approach is more appropriate\n depends on the application.\n\n[3] If x is very close to an exact integer multiple of y, it\'s\n possible for ``floor(x/y)`` to be one larger than ``(x-x%y)/y``\n due to rounding. In such cases, Python returns the latter result,\n in order to preserve that ``divmod(x,y)[0] * y + x % y`` be very\n close to ``x``.\n\n[4] While comparisons between unicode strings make sense at the byte\n level, they may be counter-intuitive to users. For example, the\n strings ``u"\\u00C7"`` and ``u"\\u0043\\u0327"`` compare differently,\n even though they both represent the same unicode character (LATIN\n CAPITAL LETTER C WITH CEDILLA). To compare strings in a human\n recognizable way, compare using ``unicodedata.normalize()``.\n\n[5] The implementation computes this efficiently, without constructing\n lists or sorting.\n\n[6] Earlier versions of Python used lexicographic comparison of the\n sorted (key, value) lists, but this was very expensive for the\n common case of comparing for equality. An even earlier version of\n Python compared dictionaries by identity only, but this caused\n surprises because people expected to be able to test a dictionary\n for emptiness by comparing it to ``{}``.\n\n[7] Due to automatic garbage-collection, free lists, and the dynamic\n nature of descriptors, you may notice seemingly unusual behaviour\n in certain uses of the ``is`` operator, like those involving\n comparisons between instance methods, or constants. Check their\n documentation for more info.\n\n[8] The power operator ``**`` binds less tightly than an arithmetic or\n bitwise unary operator on its right, that is, ``2**-1`` is\n ``0.5``.\n', + 'objects': u'\nObjects, values and types\n*************************\n\n*Objects* are Python\'s abstraction for data. All data in a Python\nprogram is represented by objects or by relations between objects. (In\na sense, and in conformance to Von Neumann\'s model of a "stored\nprogram computer," code is also represented by objects.)\n\nEvery object has an identity, a type and a value. An object\'s\n*identity* never changes once it has been created; you may think of it\nas the object\'s address in memory. The \'``is``\' operator compares the\nidentity of two objects; the ``id()`` function returns an integer\nrepresenting its identity (currently implemented as its address). An\nobject\'s *type* is also unchangeable. [1] An object\'s type determines\nthe operations that the object supports (e.g., "does it have a\nlength?") and also defines the possible values for objects of that\ntype. The ``type()`` function returns an object\'s type (which is an\nobject itself). The *value* of some objects can change. Objects\nwhose value can change are said to be *mutable*; objects whose value\nis unchangeable once they are created are called *immutable*. (The\nvalue of an immutable container object that contains a reference to a\nmutable object can change when the latter\'s value is changed; however\nthe container is still considered immutable, because the collection of\nobjects it contains cannot be changed. So, immutability is not\nstrictly the same as having an unchangeable value, it is more subtle.)\nAn object\'s mutability is determined by its type; for instance,\nnumbers, strings and tuples are immutable, while dictionaries and\nlists are mutable.\n\nObjects are never explicitly destroyed; however, when they become\nunreachable they may be garbage-collected. An implementation is\nallowed to postpone garbage collection or omit it altogether --- it is\na matter of implementation quality how garbage collection is\nimplemented, as long as no objects are collected that are still\nreachable.\n\n**CPython implementation detail:** CPython currently uses a reference-\ncounting scheme with (optional) delayed detection of cyclically linked\ngarbage, which collects most objects as soon as they become\nunreachable, but is not guaranteed to collect garbage containing\ncircular references. See the documentation of the ``gc`` module for\ninformation on controlling the collection of cyclic garbage. Other\nimplementations act differently and CPython may change. Do not depend\non immediate finalization of objects when they become unreachable (ex:\nalways close files).\n\nNote that the use of the implementation\'s tracing or debugging\nfacilities may keep objects alive that would normally be collectable.\nAlso note that catching an exception with a \'``try``...``except``\'\nstatement may keep objects alive.\n\nSome objects contain references to "external" resources such as open\nfiles or windows. It is understood that these resources are freed\nwhen the object is garbage-collected, but since garbage collection is\nnot guaranteed to happen, such objects also provide an explicit way to\nrelease the external resource, usually a ``close()`` method. Programs\nare strongly recommended to explicitly close such objects. The\n\'``try``...``finally``\' statement provides a convenient way to do\nthis.\n\nSome objects contain references to other objects; these are called\n*containers*. Examples of containers are tuples, lists and\ndictionaries. The references are part of a container\'s value. In\nmost cases, when we talk about the value of a container, we imply the\nvalues, not the identities of the contained objects; however, when we\ntalk about the mutability of a container, only the identities of the\nimmediately contained objects are implied. So, if an immutable\ncontainer (like a tuple) contains a reference to a mutable object, its\nvalue changes if that mutable object is changed.\n\nTypes affect almost all aspects of object behavior. Even the\nimportance of object identity is affected in some sense: for immutable\ntypes, operations that compute new values may actually return a\nreference to any existing object with the same type and value, while\nfor mutable objects this is not allowed. E.g., after ``a = 1; b =\n1``, ``a`` and ``b`` may or may not refer to the same object with the\nvalue one, depending on the implementation, but after ``c = []; d =\n[]``, ``c`` and ``d`` are guaranteed to refer to two different,\nunique, newly created empty lists. (Note that ``c = d = []`` assigns\nthe same object to both ``c`` and ``d``.)\n', + 'operator-summary': u'\nSummary\n*******\n\nThe following table summarizes the operator precedences in Python,\nfrom lowest precedence (least binding) to highest precedence (most\nbinding). Operators in the same box have the same precedence. Unless\nthe syntax is explicitly given, operators are binary. Operators in\nthe same box group left to right (except for comparisons, including\ntests, which all have the same precedence and chain from left to right\n--- see section *Comparisons* --- and exponentiation, which groups\nfrom right to left).\n\n+-------------------------------------------------+---------------------------------------+\n| Operator | Description |\n+=================================================+=======================================+\n| ``lambda`` | Lambda expression |\n+-------------------------------------------------+---------------------------------------+\n| ``if`` -- ``else`` | Conditional expression |\n+-------------------------------------------------+---------------------------------------+\n| ``or`` | Boolean OR |\n+-------------------------------------------------+---------------------------------------+\n| ``and`` | Boolean AND |\n+-------------------------------------------------+---------------------------------------+\n| ``not`` *x* | Boolean NOT |\n+-------------------------------------------------+---------------------------------------+\n| ``in``, ``not`` ``in``, ``is``, ``is not``, | Comparisons, including membership |\n| ``<``, ``<=``, ``>``, ``>=``, ``<>``, ``!=``, | tests and identity tests, |\n| ``==`` | |\n+-------------------------------------------------+---------------------------------------+\n| ``|`` | Bitwise OR |\n+-------------------------------------------------+---------------------------------------+\n| ``^`` | Bitwise XOR |\n+-------------------------------------------------+---------------------------------------+\n| ``&`` | Bitwise AND |\n+-------------------------------------------------+---------------------------------------+\n| ``<<``, ``>>`` | Shifts |\n+-------------------------------------------------+---------------------------------------+\n| ``+``, ``-`` | Addition and subtraction |\n+-------------------------------------------------+---------------------------------------+\n| ``*``, ``/``, ``//``, ``%`` | Multiplication, division, remainder |\n| | [8] |\n+-------------------------------------------------+---------------------------------------+\n| ``+x``, ``-x``, ``~x`` | Positive, negative, bitwise NOT |\n+-------------------------------------------------+---------------------------------------+\n| ``**`` | Exponentiation [9] |\n+-------------------------------------------------+---------------------------------------+\n| ``x[index]``, ``x[index:index]``, | Subscription, slicing, call, |\n| ``x(arguments...)``, ``x.attribute`` | attribute reference |\n+-------------------------------------------------+---------------------------------------+\n| ``(expressions...)``, ``[expressions...]``, | Binding or tuple display, list |\n| ``{key:datum...}``, ```expressions...``` | display, dictionary display, string |\n| | conversion |\n+-------------------------------------------------+---------------------------------------+\n\n-[ Footnotes ]-\n\n[1] In Python 2.3 and later releases, a list comprehension "leaks" the\n control variables of each ``for`` it contains into the containing\n scope. However, this behavior is deprecated, and relying on it\n will not work in Python 3.0\n\n[2] While ``abs(x%y) < abs(y)`` is true mathematically, for floats it\n may not be true numerically due to roundoff. For example, and\n assuming a platform on which a Python float is an IEEE 754 double-\n precision number, in order that ``-1e-100 % 1e100`` have the same\n sign as ``1e100``, the computed result is ``-1e-100 + 1e100``,\n which is numerically exactly equal to ``1e100``. The function\n ``math.fmod()`` returns a result whose sign matches the sign of\n the first argument instead, and so returns ``-1e-100`` in this\n case. Which approach is more appropriate depends on the\n application.\n\n[3] If x is very close to an exact integer multiple of y, it\'s\n possible for ``floor(x/y)`` to be one larger than ``(x-x%y)/y``\n due to rounding. In such cases, Python returns the latter result,\n in order to preserve that ``divmod(x,y)[0] * y + x % y`` be very\n close to ``x``.\n\n[4] While comparisons between unicode strings make sense at the byte\n level, they may be counter-intuitive to users. For example, the\n strings ``u"\\u00C7"`` and ``u"\\u0043\\u0327"`` compare differently,\n even though they both represent the same unicode character (LATIN\n CAPITAL LETTER C WITH CEDILLA). To compare strings in a human\n recognizable way, compare using ``unicodedata.normalize()``.\n\n[5] The implementation computes this efficiently, without constructing\n lists or sorting.\n\n[6] Earlier versions of Python used lexicographic comparison of the\n sorted (key, value) lists, but this was very expensive for the\n common case of comparing for equality. An even earlier version of\n Python compared dictionaries by identity only, but this caused\n surprises because people expected to be able to test a dictionary\n for emptiness by comparing it to ``{}``.\n\n[7] Due to automatic garbage-collection, free lists, and the dynamic\n nature of descriptors, you may notice seemingly unusual behaviour\n in certain uses of the ``is`` operator, like those involving\n comparisons between instance methods, or constants. Check their\n documentation for more info.\n\n[8] The ``%`` operator is also used for string formatting; the same\n precedence applies.\n\n[9] The power operator ``**`` binds less tightly than an arithmetic or\n bitwise unary operator on its right, that is, ``2**-1`` is\n ``0.5``.\n', 'pass': u'\nThe ``pass`` statement\n**********************\n\n pass_stmt ::= "pass"\n\n``pass`` is a null operation --- when it is executed, nothing happens.\nIt is useful as a placeholder when a statement is required\nsyntactically, but no code needs to be executed, for example:\n\n def f(arg): pass # a function that does nothing (yet)\n\n class C: pass # a class with no methods (yet)\n', 'power': u'\nThe power operator\n******************\n\nThe power operator binds more tightly than unary operators on its\nleft; it binds less tightly than unary operators on its right. The\nsyntax is:\n\n power ::= primary ["**" u_expr]\n\nThus, in an unparenthesized sequence of power and unary operators, the\noperators are evaluated from right to left (this does not constrain\nthe evaluation order for the operands): ``-1**2`` results in ``-1``.\n\nThe power operator has the same semantics as the built-in ``pow()``\nfunction, when called with two arguments: it yields its left argument\nraised to the power of its right argument. The numeric arguments are\nfirst converted to a common type. The result type is that of the\narguments after coercion.\n\nWith mixed operand types, the coercion rules for binary arithmetic\noperators apply. For int and long int operands, the result has the\nsame type as the operands (after coercion) unless the second argument\nis negative; in that case, all arguments are converted to float and a\nfloat result is delivered. For example, ``10**2`` returns ``100``, but\n``10**-2`` returns ``0.01``. (This last feature was added in Python\n2.2. In Python 2.1 and before, if both arguments were of integer types\nand the second argument was negative, an exception was raised).\n\nRaising ``0.0`` to a negative power results in a\n``ZeroDivisionError``. Raising a negative number to a fractional power\nresults in a ``ValueError``.\n', 'print': u'\nThe ``print`` statement\n***********************\n\n print_stmt ::= "print" ([expression ("," expression)* [","]]\n | ">>" expression [("," expression)+ [","]])\n\n``print`` evaluates each expression in turn and writes the resulting\nobject to standard output (see below). If an object is not a string,\nit is first converted to a string using the rules for string\nconversions. The (resulting or original) string is then written. A\nspace is written before each object is (converted and) written, unless\nthe output system believes it is positioned at the beginning of a\nline. This is the case (1) when no characters have yet been written\nto standard output, (2) when the last character written to standard\noutput is a whitespace character except ``\' \'``, or (3) when the last\nwrite operation on standard output was not a ``print`` statement. (In\nsome cases it may be functional to write an empty string to standard\noutput for this reason.)\n\nNote: Objects which act like file objects but which are not the built-in\n file objects often do not properly emulate this aspect of the file\n object\'s behavior, so it is best not to rely on this.\n\nA ``\'\\n\'`` character is written at the end, unless the ``print``\nstatement ends with a comma. This is the only action if the statement\ncontains just the keyword ``print``.\n\nStandard output is defined as the file object named ``stdout`` in the\nbuilt-in module ``sys``. If no such object exists, or if it does not\nhave a ``write()`` method, a ``RuntimeError`` exception is raised.\n\n``print`` also has an extended form, defined by the second portion of\nthe syntax described above. This form is sometimes referred to as\n"``print`` chevron." In this form, the first expression after the\n``>>`` must evaluate to a "file-like" object, specifically an object\nthat has a ``write()`` method as described above. With this extended\nform, the subsequent expressions are printed to this file object. If\nthe first expression evaluates to ``None``, then ``sys.stdout`` is\nused as the file for output.\n', @@ -63,21 +63,21 @@ 'shifting': u'\nShifting operations\n*******************\n\nThe shifting operations have lower priority than the arithmetic\noperations:\n\n shift_expr ::= a_expr | shift_expr ( "<<" | ">>" ) a_expr\n\nThese operators accept plain or long integers as arguments. The\narguments are converted to a common type. They shift the first\nargument to the left or right by the number of bits given by the\nsecond argument.\n\nA right shift by *n* bits is defined as division by ``pow(2, n)``. A\nleft shift by *n* bits is defined as multiplication with ``pow(2,\nn)``. Negative shift counts raise a ``ValueError`` exception.\n\nNote: In the current implementation, the right-hand operand is required to\n be at most ``sys.maxsize``. If the right-hand operand is larger\n than ``sys.maxsize`` an ``OverflowError`` exception is raised.\n', 'slicings': u'\nSlicings\n********\n\nA slicing selects a range of items in a sequence object (e.g., a\nstring, tuple or list). Slicings may be used as expressions or as\ntargets in assignment or ``del`` statements. The syntax for a\nslicing:\n\n slicing ::= simple_slicing | extended_slicing\n simple_slicing ::= primary "[" short_slice "]"\n extended_slicing ::= primary "[" slice_list "]"\n slice_list ::= slice_item ("," slice_item)* [","]\n slice_item ::= expression | proper_slice | ellipsis\n proper_slice ::= short_slice | long_slice\n short_slice ::= [lower_bound] ":" [upper_bound]\n long_slice ::= short_slice ":" [stride]\n lower_bound ::= expression\n upper_bound ::= expression\n stride ::= expression\n ellipsis ::= "..."\n\nThere is ambiguity in the formal syntax here: anything that looks like\nan expression list also looks like a slice list, so any subscription\ncan be interpreted as a slicing. Rather than further complicating the\nsyntax, this is disambiguated by defining that in this case the\ninterpretation as a subscription takes priority over the\ninterpretation as a slicing (this is the case if the slice list\ncontains no proper slice nor ellipses). Similarly, when the slice\nlist has exactly one short slice and no trailing comma, the\ninterpretation as a simple slicing takes priority over that as an\nextended slicing.\n\nThe semantics for a simple slicing are as follows. The primary must\nevaluate to a sequence object. The lower and upper bound expressions,\nif present, must evaluate to plain integers; defaults are zero and the\n``sys.maxint``, respectively. If either bound is negative, the\nsequence\'s length is added to it. The slicing now selects all items\nwith index *k* such that ``i <= k < j`` where *i* and *j* are the\nspecified lower and upper bounds. This may be an empty sequence. It\nis not an error if *i* or *j* lie outside the range of valid indexes\n(such items don\'t exist so they aren\'t selected).\n\nThe semantics for an extended slicing are as follows. The primary\nmust evaluate to a mapping object, and it is indexed with a key that\nis constructed from the slice list, as follows. If the slice list\ncontains at least one comma, the key is a tuple containing the\nconversion of the slice items; otherwise, the conversion of the lone\nslice item is the key. The conversion of a slice item that is an\nexpression is that expression. The conversion of an ellipsis slice\nitem is the built-in ``Ellipsis`` object. The conversion of a proper\nslice is a slice object (see section *The standard type hierarchy*)\nwhose ``start``, ``stop`` and ``step`` attributes are the values of\nthe expressions given as lower bound, upper bound and stride,\nrespectively, substituting ``None`` for missing expressions.\n', 'specialattrs': u"\nSpecial Attributes\n******************\n\nThe implementation adds a few special read-only attributes to several\nobject types, where they are relevant. Some of these are not reported\nby the ``dir()`` built-in function.\n\nobject.__dict__\n\n A dictionary or other mapping object used to store an object's\n (writable) attributes.\n\nobject.__methods__\n\n Deprecated since version 2.2: Use the built-in function ``dir()``\n to get a list of an object's attributes. This attribute is no\n longer available.\n\nobject.__members__\n\n Deprecated since version 2.2: Use the built-in function ``dir()``\n to get a list of an object's attributes. This attribute is no\n longer available.\n\ninstance.__class__\n\n The class to which a class instance belongs.\n\nclass.__bases__\n\n The tuple of base classes of a class object.\n\nclass.__name__\n\n The name of the class or type.\n\nThe following attributes are only supported by *new-style class*es.\n\nclass.__mro__\n\n This attribute is a tuple of classes that are considered when\n looking for base classes during method resolution.\n\nclass.mro()\n\n This method can be overridden by a metaclass to customize the\n method resolution order for its instances. It is called at class\n instantiation, and its result is stored in ``__mro__``.\n\nclass.__subclasses__()\n\n Each new-style class keeps a list of weak references to its\n immediate subclasses. This method returns a list of all those\n references still alive. Example:\n\n >>> int.__subclasses__()\n [<type 'bool'>]\n\n-[ Footnotes ]-\n\n[1] Additional information on these special methods may be found in\n the Python Reference Manual (*Basic customization*).\n\n[2] As a consequence, the list ``[1, 2]`` is considered equal to\n ``[1.0, 2.0]``, and similarly for tuples.\n\n[3] They must have since the parser can't tell the type of the\n operands.\n\n[4] To format only a tuple you should therefore provide a singleton\n tuple whose only element is the tuple to be formatted.\n\n[5] The advantage of leaving the newline on is that returning an empty\n string is then an unambiguous EOF indication. It is also possible\n (in cases where it might matter, for example, if you want to make\n an exact copy of a file while scanning its lines) to tell whether\n the last line of a file ended in a newline or not (yes this\n happens!).\n", - 'specialnames': u'\nSpecial method names\n********************\n\nA class can implement certain operations that are invoked by special\nsyntax (such as arithmetic operations or subscripting and slicing) by\ndefining methods with special names. This is Python\'s approach to\n*operator overloading*, allowing classes to define their own behavior\nwith respect to language operators. For instance, if a class defines\na method named ``__getitem__()``, and ``x`` is an instance of this\nclass, then ``x[i]`` is roughly equivalent to ``x.__getitem__(i)`` for\nold-style classes and ``type(x).__getitem__(x, i)`` for new-style\nclasses. Except where mentioned, attempts to execute an operation\nraise an exception when no appropriate method is defined (typically\n``AttributeError`` or ``TypeError``).\n\nWhen implementing a class that emulates any built-in type, it is\nimportant that the emulation only be implemented to the degree that it\nmakes sense for the object being modelled. For example, some\nsequences may work well with retrieval of individual elements, but\nextracting a slice may not make sense. (One example of this is the\n``NodeList`` interface in the W3C\'s Document Object Model.)\n\n\nBasic customization\n===================\n\nobject.__new__(cls[, ...])\n\n Called to create a new instance of class *cls*. ``__new__()`` is a\n static method (special-cased so you need not declare it as such)\n that takes the class of which an instance was requested as its\n first argument. The remaining arguments are those passed to the\n object constructor expression (the call to the class). The return\n value of ``__new__()`` should be the new object instance (usually\n an instance of *cls*).\n\n Typical implementations create a new instance of the class by\n invoking the superclass\'s ``__new__()`` method using\n ``super(currentclass, cls).__new__(cls[, ...])`` with appropriate\n arguments and then modifying the newly-created instance as\n necessary before returning it.\n\n If ``__new__()`` returns an instance of *cls*, then the new\n instance\'s ``__init__()`` method will be invoked like\n ``__init__(self[, ...])``, where *self* is the new instance and the\n remaining arguments are the same as were passed to ``__new__()``.\n\n If ``__new__()`` does not return an instance of *cls*, then the new\n instance\'s ``__init__()`` method will not be invoked.\n\n ``__new__()`` is intended mainly to allow subclasses of immutable\n types (like int, str, or tuple) to customize instance creation. It\n is also commonly overridden in custom metaclasses in order to\n customize class creation.\n\nobject.__init__(self[, ...])\n\n Called when the instance is created. The arguments are those\n passed to the class constructor expression. If a base class has an\n ``__init__()`` method, the derived class\'s ``__init__()`` method,\n if any, must explicitly call it to ensure proper initialization of\n the base class part of the instance; for example:\n ``BaseClass.__init__(self, [args...])``. As a special constraint\n on constructors, no value may be returned; doing so will cause a\n ``TypeError`` to be raised at runtime.\n\nobject.__del__(self)\n\n Called when the instance is about to be destroyed. This is also\n called a destructor. If a base class has a ``__del__()`` method,\n the derived class\'s ``__del__()`` method, if any, must explicitly\n call it to ensure proper deletion of the base class part of the\n instance. Note that it is possible (though not recommended!) for\n the ``__del__()`` method to postpone destruction of the instance by\n creating a new reference to it. It may then be called at a later\n time when this new reference is deleted. It is not guaranteed that\n ``__del__()`` methods are called for objects that still exist when\n the interpreter exits.\n\n Note: ``del x`` doesn\'t directly call ``x.__del__()`` --- the former\n decrements the reference count for ``x`` by one, and the latter\n is only called when ``x``\'s reference count reaches zero. Some\n common situations that may prevent the reference count of an\n object from going to zero include: circular references between\n objects (e.g., a doubly-linked list or a tree data structure with\n parent and child pointers); a reference to the object on the\n stack frame of a function that caught an exception (the traceback\n stored in ``sys.exc_traceback`` keeps the stack frame alive); or\n a reference to the object on the stack frame that raised an\n unhandled exception in interactive mode (the traceback stored in\n ``sys.last_traceback`` keeps the stack frame alive). The first\n situation can only be remedied by explicitly breaking the cycles;\n the latter two situations can be resolved by storing ``None`` in\n ``sys.exc_traceback`` or ``sys.last_traceback``. Circular\n references which are garbage are detected when the option cycle\n detector is enabled (it\'s on by default), but can only be cleaned\n up if there are no Python-level ``__del__()`` methods involved.\n Refer to the documentation for the ``gc`` module for more\n information about how ``__del__()`` methods are handled by the\n cycle detector, particularly the description of the ``garbage``\n value.\n\n Warning: Due to the precarious circumstances under which ``__del__()``\n methods are invoked, exceptions that occur during their execution\n are ignored, and a warning is printed to ``sys.stderr`` instead.\n Also, when ``__del__()`` is invoked in response to a module being\n deleted (e.g., when execution of the program is done), other\n globals referenced by the ``__del__()`` method may already have\n been deleted or in the process of being torn down (e.g. the\n import machinery shutting down). For this reason, ``__del__()``\n methods should do the absolute minimum needed to maintain\n external invariants. Starting with version 1.5, Python\n guarantees that globals whose name begins with a single\n underscore are deleted from their module before other globals are\n deleted; if no other references to such globals exist, this may\n help in assuring that imported modules are still available at the\n time when the ``__del__()`` method is called.\n\nobject.__repr__(self)\n\n Called by the ``repr()`` built-in function and by string\n conversions (reverse quotes) to compute the "official" string\n representation of an object. If at all possible, this should look\n like a valid Python expression that could be used to recreate an\n object with the same value (given an appropriate environment). If\n this is not possible, a string of the form ``<...some useful\n description...>`` should be returned. The return value must be a\n string object. If a class defines ``__repr__()`` but not\n ``__str__()``, then ``__repr__()`` is also used when an "informal"\n string representation of instances of that class is required.\n\n This is typically used for debugging, so it is important that the\n representation is information-rich and unambiguous.\n\nobject.__str__(self)\n\n Called by the ``str()`` built-in function and by the ``print``\n statement to compute the "informal" string representation of an\n object. This differs from ``__repr__()`` in that it does not have\n to be a valid Python expression: a more convenient or concise\n representation may be used instead. The return value must be a\n string object.\n\nobject.__lt__(self, other)\nobject.__le__(self, other)\nobject.__eq__(self, other)\nobject.__ne__(self, other)\nobject.__gt__(self, other)\nobject.__ge__(self, other)\n\n New in version 2.1.\n\n These are the so-called "rich comparison" methods, and are called\n for comparison operators in preference to ``__cmp__()`` below. The\n correspondence between operator symbols and method names is as\n follows: ``x<y`` calls ``x.__lt__(y)``, ``x<=y`` calls\n ``x.__le__(y)``, ``x==y`` calls ``x.__eq__(y)``, ``x!=y`` and\n ``x<>y`` call ``x.__ne__(y)``, ``x>y`` calls ``x.__gt__(y)``, and\n ``x>=y`` calls ``x.__ge__(y)``.\n\n A rich comparison method may return the singleton\n ``NotImplemented`` if it does not implement the operation for a\n given pair of arguments. By convention, ``False`` and ``True`` are\n returned for a successful comparison. However, these methods can\n return any value, so if the comparison operator is used in a\n Boolean context (e.g., in the condition of an ``if`` statement),\n Python will call ``bool()`` on the value to determine if the result\n is true or false.\n\n There are no implied relationships among the comparison operators.\n The truth of ``x==y`` does not imply that ``x!=y`` is false.\n Accordingly, when defining ``__eq__()``, one should also define\n ``__ne__()`` so that the operators will behave as expected. See\n the paragraph on ``__hash__()`` for some important notes on\n creating *hashable* objects which support custom comparison\n operations and are usable as dictionary keys.\n\n There are no swapped-argument versions of these methods (to be used\n when the left argument does not support the operation but the right\n argument does); rather, ``__lt__()`` and ``__gt__()`` are each\n other\'s reflection, ``__le__()`` and ``__ge__()`` are each other\'s\n reflection, and ``__eq__()`` and ``__ne__()`` are their own\n reflection.\n\n Arguments to rich comparison methods are never coerced.\n\n To automatically generate ordering operations from a single root\n operation, see ``functools.total_ordering()``.\n\nobject.__cmp__(self, other)\n\n Called by comparison operations if rich comparison (see above) is\n not defined. Should return a negative integer if ``self < other``,\n zero if ``self == other``, a positive integer if ``self > other``.\n If no ``__cmp__()``, ``__eq__()`` or ``__ne__()`` operation is\n defined, class instances are compared by object identity\n ("address"). See also the description of ``__hash__()`` for some\n important notes on creating *hashable* objects which support custom\n comparison operations and are usable as dictionary keys. (Note: the\n restriction that exceptions are not propagated by ``__cmp__()`` has\n been removed since Python 1.5.)\n\nobject.__rcmp__(self, other)\n\n Changed in version 2.1: No longer supported.\n\nobject.__hash__(self)\n\n Called by built-in function ``hash()`` and for operations on\n members of hashed collections including ``set``, ``frozenset``, and\n ``dict``. ``__hash__()`` should return an integer. The only\n required property is that objects which compare equal have the same\n hash value; it is advised to somehow mix together (e.g. using\n exclusive or) the hash values for the components of the object that\n also play a part in comparison of objects.\n\n If a class does not define a ``__cmp__()`` or ``__eq__()`` method\n it should not define a ``__hash__()`` operation either; if it\n defines ``__cmp__()`` or ``__eq__()`` but not ``__hash__()``, its\n instances will not be usable in hashed collections. If a class\n defines mutable objects and implements a ``__cmp__()`` or\n ``__eq__()`` method, it should not implement ``__hash__()``, since\n hashable collection implementations require that a object\'s hash\n value is immutable (if the object\'s hash value changes, it will be\n in the wrong hash bucket).\n\n User-defined classes have ``__cmp__()`` and ``__hash__()`` methods\n by default; with them, all objects compare unequal (except with\n themselves) and ``x.__hash__()`` returns ``id(x)``.\n\n Classes which inherit a ``__hash__()`` method from a parent class\n but change the meaning of ``__cmp__()`` or ``__eq__()`` such that\n the hash value returned is no longer appropriate (e.g. by switching\n to a value-based concept of equality instead of the default\n identity based equality) can explicitly flag themselves as being\n unhashable by setting ``__hash__ = None`` in the class definition.\n Doing so means that not only will instances of the class raise an\n appropriate ``TypeError`` when a program attempts to retrieve their\n hash value, but they will also be correctly identified as\n unhashable when checking ``isinstance(obj, collections.Hashable)``\n (unlike classes which define their own ``__hash__()`` to explicitly\n raise ``TypeError``).\n\n Changed in version 2.5: ``__hash__()`` may now also return a long\n integer object; the 32-bit integer is then derived from the hash of\n that object.\n\n Changed in version 2.6: ``__hash__`` may now be set to ``None`` to\n explicitly flag instances of a class as unhashable.\n\nobject.__nonzero__(self)\n\n Called to implement truth value testing and the built-in operation\n ``bool()``; should return ``False`` or ``True``, or their integer\n equivalents ``0`` or ``1``. When this method is not defined,\n ``__len__()`` is called, if it is defined, and the object is\n considered true if its result is nonzero. If a class defines\n neither ``__len__()`` nor ``__nonzero__()``, all its instances are\n considered true.\n\nobject.__unicode__(self)\n\n Called to implement ``unicode()`` built-in; should return a Unicode\n object. When this method is not defined, string conversion is\n attempted, and the result of string conversion is converted to\n Unicode using the system default encoding.\n\n\nCustomizing attribute access\n============================\n\nThe following methods can be defined to customize the meaning of\nattribute access (use of, assignment to, or deletion of ``x.name``)\nfor class instances.\n\nobject.__getattr__(self, name)\n\n Called when an attribute lookup has not found the attribute in the\n usual places (i.e. it is not an instance attribute nor is it found\n in the class tree for ``self``). ``name`` is the attribute name.\n This method should return the (computed) attribute value or raise\n an ``AttributeError`` exception.\n\n Note that if the attribute is found through the normal mechanism,\n ``__getattr__()`` is not called. (This is an intentional asymmetry\n between ``__getattr__()`` and ``__setattr__()``.) This is done both\n for efficiency reasons and because otherwise ``__getattr__()``\n would have no way to access other attributes of the instance. Note\n that at least for instance variables, you can fake total control by\n not inserting any values in the instance attribute dictionary (but\n instead inserting them in another object). See the\n ``__getattribute__()`` method below for a way to actually get total\n control in new-style classes.\n\nobject.__setattr__(self, name, value)\n\n Called when an attribute assignment is attempted. This is called\n instead of the normal mechanism (i.e. store the value in the\n instance dictionary). *name* is the attribute name, *value* is the\n value to be assigned to it.\n\n If ``__setattr__()`` wants to assign to an instance attribute, it\n should not simply execute ``self.name = value`` --- this would\n cause a recursive call to itself. Instead, it should insert the\n value in the dictionary of instance attributes, e.g.,\n ``self.__dict__[name] = value``. For new-style classes, rather\n than accessing the instance dictionary, it should call the base\n class method with the same name, for example,\n ``object.__setattr__(self, name, value)``.\n\nobject.__delattr__(self, name)\n\n Like ``__setattr__()`` but for attribute deletion instead of\n assignment. This should only be implemented if ``del obj.name`` is\n meaningful for the object.\n\n\nMore attribute access for new-style classes\n-------------------------------------------\n\nThe following methods only apply to new-style classes.\n\nobject.__getattribute__(self, name)\n\n Called unconditionally to implement attribute accesses for\n instances of the class. If the class also defines\n ``__getattr__()``, the latter will not be called unless\n ``__getattribute__()`` either calls it explicitly or raises an\n ``AttributeError``. This method should return the (computed)\n attribute value or raise an ``AttributeError`` exception. In order\n to avoid infinite recursion in this method, its implementation\n should always call the base class method with the same name to\n access any attributes it needs, for example,\n ``object.__getattribute__(self, name)``.\n\n Note: This method may still be bypassed when looking up special methods\n as the result of implicit invocation via language syntax or\n built-in functions. See *Special method lookup for new-style\n classes*.\n\n\nImplementing Descriptors\n------------------------\n\nThe following methods only apply when an instance of the class\ncontaining the method (a so-called *descriptor* class) appears in the\nclass dictionary of another new-style class, known as the *owner*\nclass. In the examples below, "the attribute" refers to the attribute\nwhose name is the key of the property in the owner class\'\n``__dict__``. Descriptors can only be implemented as new-style\nclasses themselves.\n\nobject.__get__(self, instance, owner)\n\n Called to get the attribute of the owner class (class attribute\n access) or of an instance of that class (instance attribute\n access). *owner* is always the owner class, while *instance* is the\n instance that the attribute was accessed through, or ``None`` when\n the attribute is accessed through the *owner*. This method should\n return the (computed) attribute value or raise an\n ``AttributeError`` exception.\n\nobject.__set__(self, instance, value)\n\n Called to set the attribute on an instance *instance* of the owner\n class to a new value, *value*.\n\nobject.__delete__(self, instance)\n\n Called to delete the attribute on an instance *instance* of the\n owner class.\n\n\nInvoking Descriptors\n--------------------\n\nIn general, a descriptor is an object attribute with "binding\nbehavior", one whose attribute access has been overridden by methods\nin the descriptor protocol: ``__get__()``, ``__set__()``, and\n``__delete__()``. If any of those methods are defined for an object,\nit is said to be a descriptor.\n\nThe default behavior for attribute access is to get, set, or delete\nthe attribute from an object\'s dictionary. For instance, ``a.x`` has a\nlookup chain starting with ``a.__dict__[\'x\']``, then\n``type(a).__dict__[\'x\']``, and continuing through the base classes of\n``type(a)`` excluding metaclasses.\n\nHowever, if the looked-up value is an object defining one of the\ndescriptor methods, then Python may override the default behavior and\ninvoke the descriptor method instead. Where this occurs in the\nprecedence chain depends on which descriptor methods were defined and\nhow they were called. Note that descriptors are only invoked for new\nstyle objects or classes (ones that subclass ``object()`` or\n``type()``).\n\nThe starting point for descriptor invocation is a binding, ``a.x``.\nHow the arguments are assembled depends on ``a``:\n\nDirect Call\n The simplest and least common call is when user code directly\n invokes a descriptor method: ``x.__get__(a)``.\n\nInstance Binding\n If binding to a new-style object instance, ``a.x`` is transformed\n into the call: ``type(a).__dict__[\'x\'].__get__(a, type(a))``.\n\nClass Binding\n If binding to a new-style class, ``A.x`` is transformed into the\n call: ``A.__dict__[\'x\'].__get__(None, A)``.\n\nSuper Binding\n If ``a`` is an instance of ``super``, then the binding ``super(B,\n obj).m()`` searches ``obj.__class__.__mro__`` for the base class\n ``A`` immediately preceding ``B`` and then invokes the descriptor\n with the call: ``A.__dict__[\'m\'].__get__(obj, A)``.\n\nFor instance bindings, the precedence of descriptor invocation depends\non the which descriptor methods are defined. A descriptor can define\nany combination of ``__get__()``, ``__set__()`` and ``__delete__()``.\nIf it does not define ``__get__()``, then accessing the attribute will\nreturn the descriptor object itself unless there is a value in the\nobject\'s instance dictionary. If the descriptor defines ``__set__()``\nand/or ``__delete__()``, it is a data descriptor; if it defines\nneither, it is a non-data descriptor. Normally, data descriptors\ndefine both ``__get__()`` and ``__set__()``, while non-data\ndescriptors have just the ``__get__()`` method. Data descriptors with\n``__set__()`` and ``__get__()`` defined always override a redefinition\nin an instance dictionary. In contrast, non-data descriptors can be\noverridden by instances.\n\nPython methods (including ``staticmethod()`` and ``classmethod()``)\nare implemented as non-data descriptors. Accordingly, instances can\nredefine and override methods. This allows individual instances to\nacquire behaviors that differ from other instances of the same class.\n\nThe ``property()`` function is implemented as a data descriptor.\nAccordingly, instances cannot override the behavior of a property.\n\n\n__slots__\n---------\n\nBy default, instances of both old and new-style classes have a\ndictionary for attribute storage. This wastes space for objects\nhaving very few instance variables. The space consumption can become\nacute when creating large numbers of instances.\n\nThe default can be overridden by defining *__slots__* in a new-style\nclass definition. The *__slots__* declaration takes a sequence of\ninstance variables and reserves just enough space in each instance to\nhold a value for each variable. Space is saved because *__dict__* is\nnot created for each instance.\n\n__slots__\n\n This class variable can be assigned a string, iterable, or sequence\n of strings with variable names used by instances. If defined in a\n new-style class, *__slots__* reserves space for the declared\n variables and prevents the automatic creation of *__dict__* and\n *__weakref__* for each instance.\n\n New in version 2.2.\n\nNotes on using *__slots__*\n\n* When inheriting from a class without *__slots__*, the *__dict__*\n attribute of that class will always be accessible, so a *__slots__*\n definition in the subclass is meaningless.\n\n* Without a *__dict__* variable, instances cannot be assigned new\n variables not listed in the *__slots__* definition. Attempts to\n assign to an unlisted variable name raises ``AttributeError``. If\n dynamic assignment of new variables is desired, then add\n ``\'__dict__\'`` to the sequence of strings in the *__slots__*\n declaration.\n\n Changed in version 2.3: Previously, adding ``\'__dict__\'`` to the\n *__slots__* declaration would not enable the assignment of new\n attributes not specifically listed in the sequence of instance\n variable names.\n\n* Without a *__weakref__* variable for each instance, classes defining\n *__slots__* do not support weak references to its instances. If weak\n reference support is needed, then add ``\'__weakref__\'`` to the\n sequence of strings in the *__slots__* declaration.\n\n Changed in version 2.3: Previously, adding ``\'__weakref__\'`` to the\n *__slots__* declaration would not enable support for weak\n references.\n\n* *__slots__* are implemented at the class level by creating\n descriptors (*Implementing Descriptors*) for each variable name. As\n a result, class attributes cannot be used to set default values for\n instance variables defined by *__slots__*; otherwise, the class\n attribute would overwrite the descriptor assignment.\n\n* The action of a *__slots__* declaration is limited to the class\n where it is defined. As a result, subclasses will have a *__dict__*\n unless they also define *__slots__* (which must only contain names\n of any *additional* slots).\n\n* If a class defines a slot also defined in a base class, the instance\n variable defined by the base class slot is inaccessible (except by\n retrieving its descriptor directly from the base class). This\n renders the meaning of the program undefined. In the future, a\n check may be added to prevent this.\n\n* Nonempty *__slots__* does not work for classes derived from\n "variable-length" built-in types such as ``long``, ``str`` and\n ``tuple``.\n\n* Any non-string iterable may be assigned to *__slots__*. Mappings may\n also be used; however, in the future, special meaning may be\n assigned to the values corresponding to each key.\n\n* *__class__* assignment works only if both classes have the same\n *__slots__*.\n\n Changed in version 2.6: Previously, *__class__* assignment raised an\n error if either new or old class had *__slots__*.\n\n\nCustomizing class creation\n==========================\n\nBy default, new-style classes are constructed using ``type()``. A\nclass definition is read into a separate namespace and the value of\nclass name is bound to the result of ``type(name, bases, dict)``.\n\nWhen the class definition is read, if *__metaclass__* is defined then\nthe callable assigned to it will be called instead of ``type()``. This\nallows classes or functions to be written which monitor or alter the\nclass creation process:\n\n* Modifying the class dictionary prior to the class being created.\n\n* Returning an instance of another class -- essentially performing the\n role of a factory function.\n\nThese steps will have to be performed in the metaclass\'s ``__new__()``\nmethod -- ``type.__new__()`` can then be called from this method to\ncreate a class with different properties. This example adds a new\nelement to the class dictionary before creating the class:\n\n class metacls(type):\n def __new__(mcs, name, bases, dict):\n dict[\'foo\'] = \'metacls was here\'\n return type.__new__(mcs, name, bases, dict)\n\nYou can of course also override other class methods (or add new\nmethods); for example defining a custom ``__call__()`` method in the\nmetaclass allows custom behavior when the class is called, e.g. not\nalways creating a new instance.\n\n__metaclass__\n\n This variable can be any callable accepting arguments for ``name``,\n ``bases``, and ``dict``. Upon class creation, the callable is used\n instead of the built-in ``type()``.\n\n New in version 2.2.\n\nThe appropriate metaclass is determined by the following precedence\nrules:\n\n* If ``dict[\'__metaclass__\']`` exists, it is used.\n\n* Otherwise, if there is at least one base class, its metaclass is\n used (this looks for a *__class__* attribute first and if not found,\n uses its type).\n\n* Otherwise, if a global variable named __metaclass__ exists, it is\n used.\n\n* Otherwise, the old-style, classic metaclass (types.ClassType) is\n used.\n\nThe potential uses for metaclasses are boundless. Some ideas that have\nbeen explored including logging, interface checking, automatic\ndelegation, automatic property creation, proxies, frameworks, and\nautomatic resource locking/synchronization.\n\n\nCustomizing instance and subclass checks\n========================================\n\nNew in version 2.6.\n\nThe following methods are used to override the default behavior of the\n``isinstance()`` and ``issubclass()`` built-in functions.\n\nIn particular, the metaclass ``abc.ABCMeta`` implements these methods\nin order to allow the addition of Abstract Base Classes (ABCs) as\n"virtual base classes" to any class or type (including built-in\ntypes), including other ABCs.\n\nclass.__instancecheck__(self, instance)\n\n Return true if *instance* should be considered a (direct or\n indirect) instance of *class*. If defined, called to implement\n ``isinstance(instance, class)``.\n\nclass.__subclasscheck__(self, subclass)\n\n Return true if *subclass* should be considered a (direct or\n indirect) subclass of *class*. If defined, called to implement\n ``issubclass(subclass, class)``.\n\nNote that these methods are looked up on the type (metaclass) of a\nclass. They cannot be defined as class methods in the actual class.\nThis is consistent with the lookup of special methods that are called\non instances, only in this case the instance is itself a class.\n\nSee also:\n\n **PEP 3119** - Introducing Abstract Base Classes\n Includes the specification for customizing ``isinstance()`` and\n ``issubclass()`` behavior through ``__instancecheck__()`` and\n ``__subclasscheck__()``, with motivation for this functionality\n in the context of adding Abstract Base Classes (see the ``abc``\n module) to the language.\n\n\nEmulating callable objects\n==========================\n\nobject.__call__(self[, args...])\n\n Called when the instance is "called" as a function; if this method\n is defined, ``x(arg1, arg2, ...)`` is a shorthand for\n ``x.__call__(arg1, arg2, ...)``.\n\n\nEmulating container types\n=========================\n\nThe following methods can be defined to implement container objects.\nContainers usually are sequences (such as lists or tuples) or mappings\n(like dictionaries), but can represent other containers as well. The\nfirst set of methods is used either to emulate a sequence or to\nemulate a mapping; the difference is that for a sequence, the\nallowable keys should be the integers *k* for which ``0 <= k < N``\nwhere *N* is the length of the sequence, or slice objects, which\ndefine a range of items. (For backwards compatibility, the method\n``__getslice__()`` (see below) can also be defined to handle simple,\nbut not extended slices.) It is also recommended that mappings provide\nthe methods ``keys()``, ``values()``, ``items()``, ``has_key()``,\n``get()``, ``clear()``, ``setdefault()``, ``iterkeys()``,\n``itervalues()``, ``iteritems()``, ``pop()``, ``popitem()``,\n``copy()``, and ``update()`` behaving similar to those for Python\'s\nstandard dictionary objects. The ``UserDict`` module provides a\n``DictMixin`` class to help create those methods from a base set of\n``__getitem__()``, ``__setitem__()``, ``__delitem__()``, and\n``keys()``. Mutable sequences should provide methods ``append()``,\n``count()``, ``index()``, ``extend()``, ``insert()``, ``pop()``,\n``remove()``, ``reverse()`` and ``sort()``, like Python standard list\nobjects. Finally, sequence types should implement addition (meaning\nconcatenation) and multiplication (meaning repetition) by defining the\nmethods ``__add__()``, ``__radd__()``, ``__iadd__()``, ``__mul__()``,\n``__rmul__()`` and ``__imul__()`` described below; they should not\ndefine ``__coerce__()`` or other numerical operators. It is\nrecommended that both mappings and sequences implement the\n``__contains__()`` method to allow efficient use of the ``in``\noperator; for mappings, ``in`` should be equivalent of ``has_key()``;\nfor sequences, it should search through the values. It is further\nrecommended that both mappings and sequences implement the\n``__iter__()`` method to allow efficient iteration through the\ncontainer; for mappings, ``__iter__()`` should be the same as\n``iterkeys()``; for sequences, it should iterate through the values.\n\nobject.__len__(self)\n\n Called to implement the built-in function ``len()``. Should return\n the length of the object, an integer ``>=`` 0. Also, an object\n that doesn\'t define a ``__nonzero__()`` method and whose\n ``__len__()`` method returns zero is considered to be false in a\n Boolean context.\n\nobject.__getitem__(self, key)\n\n Called to implement evaluation of ``self[key]``. For sequence\n types, the accepted keys should be integers and slice objects.\n Note that the special interpretation of negative indexes (if the\n class wishes to emulate a sequence type) is up to the\n ``__getitem__()`` method. If *key* is of an inappropriate type,\n ``TypeError`` may be raised; if of a value outside the set of\n indexes for the sequence (after any special interpretation of\n negative values), ``IndexError`` should be raised. For mapping\n types, if *key* is missing (not in the container), ``KeyError``\n should be raised.\n\n Note: ``for`` loops expect that an ``IndexError`` will be raised for\n illegal indexes to allow proper detection of the end of the\n sequence.\n\nobject.__setitem__(self, key, value)\n\n Called to implement assignment to ``self[key]``. Same note as for\n ``__getitem__()``. This should only be implemented for mappings if\n the objects support changes to the values for keys, or if new keys\n can be added, or for sequences if elements can be replaced. The\n same exceptions should be raised for improper *key* values as for\n the ``__getitem__()`` method.\n\nobject.__delitem__(self, key)\n\n Called to implement deletion of ``self[key]``. Same note as for\n ``__getitem__()``. This should only be implemented for mappings if\n the objects support removal of keys, or for sequences if elements\n can be removed from the sequence. The same exceptions should be\n raised for improper *key* values as for the ``__getitem__()``\n method.\n\nobject.__iter__(self)\n\n This method is called when an iterator is required for a container.\n This method should return a new iterator object that can iterate\n over all the objects in the container. For mappings, it should\n iterate over the keys of the container, and should also be made\n available as the method ``iterkeys()``.\n\n Iterator objects also need to implement this method; they are\n required to return themselves. For more information on iterator\n objects, see *Iterator Types*.\n\nobject.__reversed__(self)\n\n Called (if present) by the ``reversed()`` built-in to implement\n reverse iteration. It should return a new iterator object that\n iterates over all the objects in the container in reverse order.\n\n If the ``__reversed__()`` method is not provided, the\n ``reversed()`` built-in will fall back to using the sequence\n protocol (``__len__()`` and ``__getitem__()``). Objects that\n support the sequence protocol should only provide\n ``__reversed__()`` if they can provide an implementation that is\n more efficient than the one provided by ``reversed()``.\n\n New in version 2.6.\n\nThe membership test operators (``in`` and ``not in``) are normally\nimplemented as an iteration through a sequence. However, container\nobjects can supply the following special method with a more efficient\nimplementation, which also does not require the object be a sequence.\n\nobject.__contains__(self, item)\n\n Called to implement membership test operators. Should return true\n if *item* is in *self*, false otherwise. For mapping objects, this\n should consider the keys of the mapping rather than the values or\n the key-item pairs.\n\n For objects that don\'t define ``__contains__()``, the membership\n test first tries iteration via ``__iter__()``, then the old\n sequence iteration protocol via ``__getitem__()``, see *this\n section in the language reference*.\n\n\nAdditional methods for emulation of sequence types\n==================================================\n\nThe following optional methods can be defined to further emulate\nsequence objects. Immutable sequences methods should at most only\ndefine ``__getslice__()``; mutable sequences might define all three\nmethods.\n\nobject.__getslice__(self, i, j)\n\n Deprecated since version 2.0: Support slice objects as parameters\n to the ``__getitem__()`` method. (However, built-in types in\n CPython currently still implement ``__getslice__()``. Therefore,\n you have to override it in derived classes when implementing\n slicing.)\n\n Called to implement evaluation of ``self[i:j]``. The returned\n object should be of the same type as *self*. Note that missing *i*\n or *j* in the slice expression are replaced by zero or\n ``sys.maxint``, respectively. If negative indexes are used in the\n slice, the length of the sequence is added to that index. If the\n instance does not implement the ``__len__()`` method, an\n ``AttributeError`` is raised. No guarantee is made that indexes\n adjusted this way are not still negative. Indexes which are\n greater than the length of the sequence are not modified. If no\n ``__getslice__()`` is found, a slice object is created instead, and\n passed to ``__getitem__()`` instead.\n\nobject.__setslice__(self, i, j, sequence)\n\n Called to implement assignment to ``self[i:j]``. Same notes for *i*\n and *j* as for ``__getslice__()``.\n\n This method is deprecated. If no ``__setslice__()`` is found, or\n for extended slicing of the form ``self[i:j:k]``, a slice object is\n created, and passed to ``__setitem__()``, instead of\n ``__setslice__()`` being called.\n\nobject.__delslice__(self, i, j)\n\n Called to implement deletion of ``self[i:j]``. Same notes for *i*\n and *j* as for ``__getslice__()``. This method is deprecated. If no\n ``__delslice__()`` is found, or for extended slicing of the form\n ``self[i:j:k]``, a slice object is created, and passed to\n ``__delitem__()``, instead of ``__delslice__()`` being called.\n\nNotice that these methods are only invoked when a single slice with a\nsingle colon is used, and the slice method is available. For slice\noperations involving extended slice notation, or in absence of the\nslice methods, ``__getitem__()``, ``__setitem__()`` or\n``__delitem__()`` is called with a slice object as argument.\n\nThe following example demonstrate how to make your program or module\ncompatible with earlier versions of Python (assuming that methods\n``__getitem__()``, ``__setitem__()`` and ``__delitem__()`` support\nslice objects as arguments):\n\n class MyClass:\n ...\n def __getitem__(self, index):\n ...\n def __setitem__(self, index, value):\n ...\n def __delitem__(self, index):\n ...\n\n if sys.version_info < (2, 0):\n # They won\'t be defined if version is at least 2.0 final\n\n def __getslice__(self, i, j):\n return self[max(0, i):max(0, j):]\n def __setslice__(self, i, j, seq):\n self[max(0, i):max(0, j):] = seq\n def __delslice__(self, i, j):\n del self[max(0, i):max(0, j):]\n ...\n\nNote the calls to ``max()``; these are necessary because of the\nhandling of negative indices before the ``__*slice__()`` methods are\ncalled. When negative indexes are used, the ``__*item__()`` methods\nreceive them as provided, but the ``__*slice__()`` methods get a\n"cooked" form of the index values. For each negative index value, the\nlength of the sequence is added to the index before calling the method\n(which may still result in a negative index); this is the customary\nhandling of negative indexes by the built-in sequence types, and the\n``__*item__()`` methods are expected to do this as well. However,\nsince they should already be doing that, negative indexes cannot be\npassed in; they must be constrained to the bounds of the sequence\nbefore being passed to the ``__*item__()`` methods. Calling ``max(0,\ni)`` conveniently returns the proper value.\n\n\nEmulating numeric types\n=======================\n\nThe following methods can be defined to emulate numeric objects.\nMethods corresponding to operations that are not supported by the\nparticular kind of number implemented (e.g., bitwise operations for\nnon-integral numbers) should be left undefined.\n\nobject.__add__(self, other)\nobject.__sub__(self, other)\nobject.__mul__(self, other)\nobject.__floordiv__(self, other)\nobject.__mod__(self, other)\nobject.__divmod__(self, other)\nobject.__pow__(self, other[, modulo])\nobject.__lshift__(self, other)\nobject.__rshift__(self, other)\nobject.__and__(self, other)\nobject.__xor__(self, other)\nobject.__or__(self, other)\n\n These methods are called to implement the binary arithmetic\n operations (``+``, ``-``, ``*``, ``//``, ``%``, ``divmod()``,\n ``pow()``, ``**``, ``<<``, ``>>``, ``&``, ``^``, ``|``). For\n instance, to evaluate the expression ``x + y``, where *x* is an\n instance of a class that has an ``__add__()`` method,\n ``x.__add__(y)`` is called. The ``__divmod__()`` method should be\n the equivalent to using ``__floordiv__()`` and ``__mod__()``; it\n should not be related to ``__truediv__()`` (described below). Note\n that ``__pow__()`` should be defined to accept an optional third\n argument if the ternary version of the built-in ``pow()`` function\n is to be supported.\n\n If one of those methods does not support the operation with the\n supplied arguments, it should return ``NotImplemented``.\n\nobject.__div__(self, other)\nobject.__truediv__(self, other)\n\n The division operator (``/``) is implemented by these methods. The\n ``__truediv__()`` method is used when ``__future__.division`` is in\n effect, otherwise ``__div__()`` is used. If only one of these two\n methods is defined, the object will not support division in the\n alternate context; ``TypeError`` will be raised instead.\n\nobject.__radd__(self, other)\nobject.__rsub__(self, other)\nobject.__rmul__(self, other)\nobject.__rdiv__(self, other)\nobject.__rtruediv__(self, other)\nobject.__rfloordiv__(self, other)\nobject.__rmod__(self, other)\nobject.__rdivmod__(self, other)\nobject.__rpow__(self, other)\nobject.__rlshift__(self, other)\nobject.__rrshift__(self, other)\nobject.__rand__(self, other)\nobject.__rxor__(self, other)\nobject.__ror__(self, other)\n\n These methods are called to implement the binary arithmetic\n operations (``+``, ``-``, ``*``, ``/``, ``%``, ``divmod()``,\n ``pow()``, ``**``, ``<<``, ``>>``, ``&``, ``^``, ``|``) with\n reflected (swapped) operands. These functions are only called if\n the left operand does not support the corresponding operation and\n the operands are of different types. [2] For instance, to evaluate\n the expression ``x - y``, where *y* is an instance of a class that\n has an ``__rsub__()`` method, ``y.__rsub__(x)`` is called if\n ``x.__sub__(y)`` returns *NotImplemented*.\n\n Note that ternary ``pow()`` will not try calling ``__rpow__()``\n (the coercion rules would become too complicated).\n\n Note: If the right operand\'s type is a subclass of the left operand\'s\n type and that subclass provides the reflected method for the\n operation, this method will be called before the left operand\'s\n non-reflected method. This behavior allows subclasses to\n override their ancestors\' operations.\n\nobject.__iadd__(self, other)\nobject.__isub__(self, other)\nobject.__imul__(self, other)\nobject.__idiv__(self, other)\nobject.__itruediv__(self, other)\nobject.__ifloordiv__(self, other)\nobject.__imod__(self, other)\nobject.__ipow__(self, other[, modulo])\nobject.__ilshift__(self, other)\nobject.__irshift__(self, other)\nobject.__iand__(self, other)\nobject.__ixor__(self, other)\nobject.__ior__(self, other)\n\n These methods are called to implement the augmented arithmetic\n assignments (``+=``, ``-=``, ``*=``, ``/=``, ``//=``, ``%=``,\n ``**=``, ``<<=``, ``>>=``, ``&=``, ``^=``, ``|=``). These methods\n should attempt to do the operation in-place (modifying *self*) and\n return the result (which could be, but does not have to be,\n *self*). If a specific method is not defined, the augmented\n assignment falls back to the normal methods. For instance, to\n execute the statement ``x += y``, where *x* is an instance of a\n class that has an ``__iadd__()`` method, ``x.__iadd__(y)`` is\n called. If *x* is an instance of a class that does not define a\n ``__iadd__()`` method, ``x.__add__(y)`` and ``y.__radd__(x)`` are\n considered, as with the evaluation of ``x + y``.\n\nobject.__neg__(self)\nobject.__pos__(self)\nobject.__abs__(self)\nobject.__invert__(self)\n\n Called to implement the unary arithmetic operations (``-``, ``+``,\n ``abs()`` and ``~``).\n\nobject.__complex__(self)\nobject.__int__(self)\nobject.__long__(self)\nobject.__float__(self)\n\n Called to implement the built-in functions ``complex()``,\n ``int()``, ``long()``, and ``float()``. Should return a value of\n the appropriate type.\n\nobject.__oct__(self)\nobject.__hex__(self)\n\n Called to implement the built-in functions ``oct()`` and ``hex()``.\n Should return a string value.\n\nobject.__index__(self)\n\n Called to implement ``operator.index()``. Also called whenever\n Python needs an integer object (such as in slicing). Must return\n an integer (int or long).\n\n New in version 2.5.\n\nobject.__coerce__(self, other)\n\n Called to implement "mixed-mode" numeric arithmetic. Should either\n return a 2-tuple containing *self* and *other* converted to a\n common numeric type, or ``None`` if conversion is impossible. When\n the common type would be the type of ``other``, it is sufficient to\n return ``None``, since the interpreter will also ask the other\n object to attempt a coercion (but sometimes, if the implementation\n of the other type cannot be changed, it is useful to do the\n conversion to the other type here). A return value of\n ``NotImplemented`` is equivalent to returning ``None``.\n\n\nCoercion rules\n==============\n\nThis section used to document the rules for coercion. As the language\nhas evolved, the coercion rules have become hard to document\nprecisely; documenting what one version of one particular\nimplementation does is undesirable. Instead, here are some informal\nguidelines regarding coercion. In Python 3.0, coercion will not be\nsupported.\n\n* If the left operand of a % operator is a string or Unicode object,\n no coercion takes place and the string formatting operation is\n invoked instead.\n\n* It is no longer recommended to define a coercion operation. Mixed-\n mode operations on types that don\'t define coercion pass the\n original arguments to the operation.\n\n* New-style classes (those derived from ``object``) never invoke the\n ``__coerce__()`` method in response to a binary operator; the only\n time ``__coerce__()`` is invoked is when the built-in function\n ``coerce()`` is called.\n\n* For most intents and purposes, an operator that returns\n ``NotImplemented`` is treated the same as one that is not\n implemented at all.\n\n* Below, ``__op__()`` and ``__rop__()`` are used to signify the\n generic method names corresponding to an operator; ``__iop__()`` is\n used for the corresponding in-place operator. For example, for the\n operator \'``+``\', ``__add__()`` and ``__radd__()`` are used for the\n left and right variant of the binary operator, and ``__iadd__()``\n for the in-place variant.\n\n* For objects *x* and *y*, first ``x.__op__(y)`` is tried. If this is\n not implemented or returns ``NotImplemented``, ``y.__rop__(x)`` is\n tried. If this is also not implemented or returns\n ``NotImplemented``, a ``TypeError`` exception is raised. But see\n the following exception:\n\n* Exception to the previous item: if the left operand is an instance\n of a built-in type or a new-style class, and the right operand is an\n instance of a proper subclass of that type or class and overrides\n the base\'s ``__rop__()`` method, the right operand\'s ``__rop__()``\n method is tried *before* the left operand\'s ``__op__()`` method.\n\n This is done so that a subclass can completely override binary\n operators. Otherwise, the left operand\'s ``__op__()`` method would\n always accept the right operand: when an instance of a given class\n is expected, an instance of a subclass of that class is always\n acceptable.\n\n* When either operand type defines a coercion, this coercion is called\n before that type\'s ``__op__()`` or ``__rop__()`` method is called,\n but no sooner. If the coercion returns an object of a different\n type for the operand whose coercion is invoked, part of the process\n is redone using the new object.\n\n* When an in-place operator (like \'``+=``\') is used, if the left\n operand implements ``__iop__()``, it is invoked without any\n coercion. When the operation falls back to ``__op__()`` and/or\n ``__rop__()``, the normal coercion rules apply.\n\n* In ``x + y``, if *x* is a sequence that implements sequence\n concatenation, sequence concatenation is invoked.\n\n* In ``x * y``, if one operator is a sequence that implements sequence\n repetition, and the other is an integer (``int`` or ``long``),\n sequence repetition is invoked.\n\n* Rich comparisons (implemented by methods ``__eq__()`` and so on)\n never use coercion. Three-way comparison (implemented by\n ``__cmp__()``) does use coercion under the same conditions as other\n binary operations use it.\n\n* In the current implementation, the built-in numeric types ``int``,\n ``long``, ``float``, and ``complex`` do not use coercion. All these\n types implement a ``__coerce__()`` method, for use by the built-in\n ``coerce()`` function.\n\n Changed in version 2.7.\n\n\nWith Statement Context Managers\n===============================\n\nNew in version 2.5.\n\nA *context manager* is an object that defines the runtime context to\nbe established when executing a ``with`` statement. The context\nmanager handles the entry into, and the exit from, the desired runtime\ncontext for the execution of the block of code. Context managers are\nnormally invoked using the ``with`` statement (described in section\n*The with statement*), but can also be used by directly invoking their\nmethods.\n\nTypical uses of context managers include saving and restoring various\nkinds of global state, locking and unlocking resources, closing opened\nfiles, etc.\n\nFor more information on context managers, see *Context Manager Types*.\n\nobject.__enter__(self)\n\n Enter the runtime context related to this object. The ``with``\n statement will bind this method\'s return value to the target(s)\n specified in the ``as`` clause of the statement, if any.\n\nobject.__exit__(self, exc_type, exc_value, traceback)\n\n Exit the runtime context related to this object. The parameters\n describe the exception that caused the context to be exited. If the\n context was exited without an exception, all three arguments will\n be ``None``.\n\n If an exception is supplied, and the method wishes to suppress the\n exception (i.e., prevent it from being propagated), it should\n return a true value. Otherwise, the exception will be processed\n normally upon exit from this method.\n\n Note that ``__exit__()`` methods should not reraise the passed-in\n exception; this is the caller\'s responsibility.\n\nSee also:\n\n **PEP 0343** - The "with" statement\n The specification, background, and examples for the Python\n ``with`` statement.\n\n\nSpecial method lookup for old-style classes\n===========================================\n\nFor old-style classes, special methods are always looked up in exactly\nthe same way as any other method or attribute. This is the case\nregardless of whether the method is being looked up explicitly as in\n``x.__getitem__(i)`` or implicitly as in ``x[i]``.\n\nThis behaviour means that special methods may exhibit different\nbehaviour for different instances of a single old-style class if the\nappropriate special attributes are set differently:\n\n >>> class C:\n ... pass\n ...\n >>> c1 = C()\n >>> c2 = C()\n >>> c1.__len__ = lambda: 5\n >>> c2.__len__ = lambda: 9\n >>> len(c1)\n 5\n >>> len(c2)\n 9\n\n\nSpecial method lookup for new-style classes\n===========================================\n\nFor new-style classes, implicit invocations of special methods are\nonly guaranteed to work correctly if defined on an object\'s type, not\nin the object\'s instance dictionary. That behaviour is the reason why\nthe following code raises an exception (unlike the equivalent example\nwith old-style classes):\n\n >>> class C(object):\n ... pass\n ...\n >>> c = C()\n >>> c.__len__ = lambda: 5\n >>> len(c)\n Traceback (most recent call last):\n File "<stdin>", line 1, in <module>\n TypeError: object of type \'C\' has no len()\n\nThe rationale behind this behaviour lies with a number of special\nmethods such as ``__hash__()`` and ``__repr__()`` that are implemented\nby all objects, including type objects. If the implicit lookup of\nthese methods used the conventional lookup process, they would fail\nwhen invoked on the type object itself:\n\n >>> 1 .__hash__() == hash(1)\n True\n >>> int.__hash__() == hash(int)\n Traceback (most recent call last):\n File "<stdin>", line 1, in <module>\n TypeError: descriptor \'__hash__\' of \'int\' object needs an argument\n\nIncorrectly attempting to invoke an unbound method of a class in this\nway is sometimes referred to as \'metaclass confusion\', and is avoided\nby bypassing the instance when looking up special methods:\n\n >>> type(1).__hash__(1) == hash(1)\n True\n >>> type(int).__hash__(int) == hash(int)\n True\n\nIn addition to bypassing any instance attributes in the interest of\ncorrectness, implicit special method lookup generally also bypasses\nthe ``__getattribute__()`` method even of the object\'s metaclass:\n\n >>> class Meta(type):\n ... def __getattribute__(*args):\n ... print "Metaclass getattribute invoked"\n ... return type.__getattribute__(*args)\n ...\n >>> class C(object):\n ... __metaclass__ = Meta\n ... def __len__(self):\n ... return 10\n ... def __getattribute__(*args):\n ... print "Class getattribute invoked"\n ... return object.__getattribute__(*args)\n ...\n >>> c = C()\n >>> c.__len__() # Explicit lookup via instance\n Class getattribute invoked\n 10\n >>> type(c).__len__(c) # Explicit lookup via type\n Metaclass getattribute invoked\n 10\n >>> len(c) # Implicit lookup\n 10\n\nBypassing the ``__getattribute__()`` machinery in this fashion\nprovides significant scope for speed optimisations within the\ninterpreter, at the cost of some flexibility in the handling of\nspecial methods (the special method *must* be set on the class object\nitself in order to be consistently invoked by the interpreter).\n\n-[ Footnotes ]-\n\n[1] It *is* possible in some cases to change an object\'s type, under\n certain controlled conditions. It generally isn\'t a good idea\n though, since it can lead to some very strange behaviour if it is\n handled incorrectly.\n\n[2] For operands of the same type, it is assumed that if the non-\n reflected method (such as ``__add__()``) fails the operation is\n not supported, which is why the reflected method is not called.\n', + 'specialnames': u'\nSpecial method names\n********************\n\nA class can implement certain operations that are invoked by special\nsyntax (such as arithmetic operations or subscripting and slicing) by\ndefining methods with special names. This is Python\'s approach to\n*operator overloading*, allowing classes to define their own behavior\nwith respect to language operators. For instance, if a class defines\na method named ``__getitem__()``, and ``x`` is an instance of this\nclass, then ``x[i]`` is roughly equivalent to ``x.__getitem__(i)`` for\nold-style classes and ``type(x).__getitem__(x, i)`` for new-style\nclasses. Except where mentioned, attempts to execute an operation\nraise an exception when no appropriate method is defined (typically\n``AttributeError`` or ``TypeError``).\n\nWhen implementing a class that emulates any built-in type, it is\nimportant that the emulation only be implemented to the degree that it\nmakes sense for the object being modelled. For example, some\nsequences may work well with retrieval of individual elements, but\nextracting a slice may not make sense. (One example of this is the\n``NodeList`` interface in the W3C\'s Document Object Model.)\n\n\nBasic customization\n===================\n\nobject.__new__(cls[, ...])\n\n Called to create a new instance of class *cls*. ``__new__()`` is a\n static method (special-cased so you need not declare it as such)\n that takes the class of which an instance was requested as its\n first argument. The remaining arguments are those passed to the\n object constructor expression (the call to the class). The return\n value of ``__new__()`` should be the new object instance (usually\n an instance of *cls*).\n\n Typical implementations create a new instance of the class by\n invoking the superclass\'s ``__new__()`` method using\n ``super(currentclass, cls).__new__(cls[, ...])`` with appropriate\n arguments and then modifying the newly-created instance as\n necessary before returning it.\n\n If ``__new__()`` returns an instance of *cls*, then the new\n instance\'s ``__init__()`` method will be invoked like\n ``__init__(self[, ...])``, where *self* is the new instance and the\n remaining arguments are the same as were passed to ``__new__()``.\n\n If ``__new__()`` does not return an instance of *cls*, then the new\n instance\'s ``__init__()`` method will not be invoked.\n\n ``__new__()`` is intended mainly to allow subclasses of immutable\n types (like int, str, or tuple) to customize instance creation. It\n is also commonly overridden in custom metaclasses in order to\n customize class creation.\n\nobject.__init__(self[, ...])\n\n Called when the instance is created. The arguments are those\n passed to the class constructor expression. If a base class has an\n ``__init__()`` method, the derived class\'s ``__init__()`` method,\n if any, must explicitly call it to ensure proper initialization of\n the base class part of the instance; for example:\n ``BaseClass.__init__(self, [args...])``. As a special constraint\n on constructors, no value may be returned; doing so will cause a\n ``TypeError`` to be raised at runtime.\n\nobject.__del__(self)\n\n Called when the instance is about to be destroyed. This is also\n called a destructor. If a base class has a ``__del__()`` method,\n the derived class\'s ``__del__()`` method, if any, must explicitly\n call it to ensure proper deletion of the base class part of the\n instance. Note that it is possible (though not recommended!) for\n the ``__del__()`` method to postpone destruction of the instance by\n creating a new reference to it. It may then be called at a later\n time when this new reference is deleted. It is not guaranteed that\n ``__del__()`` methods are called for objects that still exist when\n the interpreter exits.\n\n Note: ``del x`` doesn\'t directly call ``x.__del__()`` --- the former\n decrements the reference count for ``x`` by one, and the latter\n is only called when ``x``\'s reference count reaches zero. Some\n common situations that may prevent the reference count of an\n object from going to zero include: circular references between\n objects (e.g., a doubly-linked list or a tree data structure with\n parent and child pointers); a reference to the object on the\n stack frame of a function that caught an exception (the traceback\n stored in ``sys.exc_traceback`` keeps the stack frame alive); or\n a reference to the object on the stack frame that raised an\n unhandled exception in interactive mode (the traceback stored in\n ``sys.last_traceback`` keeps the stack frame alive). The first\n situation can only be remedied by explicitly breaking the cycles;\n the latter two situations can be resolved by storing ``None`` in\n ``sys.exc_traceback`` or ``sys.last_traceback``. Circular\n references which are garbage are detected when the option cycle\n detector is enabled (it\'s on by default), but can only be cleaned\n up if there are no Python-level ``__del__()`` methods involved.\n Refer to the documentation for the ``gc`` module for more\n information about how ``__del__()`` methods are handled by the\n cycle detector, particularly the description of the ``garbage``\n value.\n\n Warning: Due to the precarious circumstances under which ``__del__()``\n methods are invoked, exceptions that occur during their execution\n are ignored, and a warning is printed to ``sys.stderr`` instead.\n Also, when ``__del__()`` is invoked in response to a module being\n deleted (e.g., when execution of the program is done), other\n globals referenced by the ``__del__()`` method may already have\n been deleted or in the process of being torn down (e.g. the\n import machinery shutting down). For this reason, ``__del__()``\n methods should do the absolute minimum needed to maintain\n external invariants. Starting with version 1.5, Python\n guarantees that globals whose name begins with a single\n underscore are deleted from their module before other globals are\n deleted; if no other references to such globals exist, this may\n help in assuring that imported modules are still available at the\n time when the ``__del__()`` method is called.\n\nobject.__repr__(self)\n\n Called by the ``repr()`` built-in function and by string\n conversions (reverse quotes) to compute the "official" string\n representation of an object. If at all possible, this should look\n like a valid Python expression that could be used to recreate an\n object with the same value (given an appropriate environment). If\n this is not possible, a string of the form ``<...some useful\n description...>`` should be returned. The return value must be a\n string object. If a class defines ``__repr__()`` but not\n ``__str__()``, then ``__repr__()`` is also used when an "informal"\n string representation of instances of that class is required.\n\n This is typically used for debugging, so it is important that the\n representation is information-rich and unambiguous.\n\nobject.__str__(self)\n\n Called by the ``str()`` built-in function and by the ``print``\n statement to compute the "informal" string representation of an\n object. This differs from ``__repr__()`` in that it does not have\n to be a valid Python expression: a more convenient or concise\n representation may be used instead. The return value must be a\n string object.\n\nobject.__lt__(self, other)\nobject.__le__(self, other)\nobject.__eq__(self, other)\nobject.__ne__(self, other)\nobject.__gt__(self, other)\nobject.__ge__(self, other)\n\n New in version 2.1.\n\n These are the so-called "rich comparison" methods, and are called\n for comparison operators in preference to ``__cmp__()`` below. The\n correspondence between operator symbols and method names is as\n follows: ``x<y`` calls ``x.__lt__(y)``, ``x<=y`` calls\n ``x.__le__(y)``, ``x==y`` calls ``x.__eq__(y)``, ``x!=y`` and\n ``x<>y`` call ``x.__ne__(y)``, ``x>y`` calls ``x.__gt__(y)``, and\n ``x>=y`` calls ``x.__ge__(y)``.\n\n A rich comparison method may return the singleton\n ``NotImplemented`` if it does not implement the operation for a\n given pair of arguments. By convention, ``False`` and ``True`` are\n returned for a successful comparison. However, these methods can\n return any value, so if the comparison operator is used in a\n Boolean context (e.g., in the condition of an ``if`` statement),\n Python will call ``bool()`` on the value to determine if the result\n is true or false.\n\n There are no implied relationships among the comparison operators.\n The truth of ``x==y`` does not imply that ``x!=y`` is false.\n Accordingly, when defining ``__eq__()``, one should also define\n ``__ne__()`` so that the operators will behave as expected. See\n the paragraph on ``__hash__()`` for some important notes on\n creating *hashable* objects which support custom comparison\n operations and are usable as dictionary keys.\n\n There are no swapped-argument versions of these methods (to be used\n when the left argument does not support the operation but the right\n argument does); rather, ``__lt__()`` and ``__gt__()`` are each\n other\'s reflection, ``__le__()`` and ``__ge__()`` are each other\'s\n reflection, and ``__eq__()`` and ``__ne__()`` are their own\n reflection.\n\n Arguments to rich comparison methods are never coerced.\n\n To automatically generate ordering operations from a single root\n operation, see ``functools.total_ordering()``.\n\nobject.__cmp__(self, other)\n\n Called by comparison operations if rich comparison (see above) is\n not defined. Should return a negative integer if ``self < other``,\n zero if ``self == other``, a positive integer if ``self > other``.\n If no ``__cmp__()``, ``__eq__()`` or ``__ne__()`` operation is\n defined, class instances are compared by object identity\n ("address"). See also the description of ``__hash__()`` for some\n important notes on creating *hashable* objects which support custom\n comparison operations and are usable as dictionary keys. (Note: the\n restriction that exceptions are not propagated by ``__cmp__()`` has\n been removed since Python 1.5.)\n\nobject.__rcmp__(self, other)\n\n Changed in version 2.1: No longer supported.\n\nobject.__hash__(self)\n\n Called by built-in function ``hash()`` and for operations on\n members of hashed collections including ``set``, ``frozenset``, and\n ``dict``. ``__hash__()`` should return an integer. The only\n required property is that objects which compare equal have the same\n hash value; it is advised to somehow mix together (e.g. using\n exclusive or) the hash values for the components of the object that\n also play a part in comparison of objects.\n\n If a class does not define a ``__cmp__()`` or ``__eq__()`` method\n it should not define a ``__hash__()`` operation either; if it\n defines ``__cmp__()`` or ``__eq__()`` but not ``__hash__()``, its\n instances will not be usable in hashed collections. If a class\n defines mutable objects and implements a ``__cmp__()`` or\n ``__eq__()`` method, it should not implement ``__hash__()``, since\n hashable collection implementations require that a object\'s hash\n value is immutable (if the object\'s hash value changes, it will be\n in the wrong hash bucket).\n\n User-defined classes have ``__cmp__()`` and ``__hash__()`` methods\n by default; with them, all objects compare unequal (except with\n themselves) and ``x.__hash__()`` returns ``id(x)``.\n\n Classes which inherit a ``__hash__()`` method from a parent class\n but change the meaning of ``__cmp__()`` or ``__eq__()`` such that\n the hash value returned is no longer appropriate (e.g. by switching\n to a value-based concept of equality instead of the default\n identity based equality) can explicitly flag themselves as being\n unhashable by setting ``__hash__ = None`` in the class definition.\n Doing so means that not only will instances of the class raise an\n appropriate ``TypeError`` when a program attempts to retrieve their\n hash value, but they will also be correctly identified as\n unhashable when checking ``isinstance(obj, collections.Hashable)``\n (unlike classes which define their own ``__hash__()`` to explicitly\n raise ``TypeError``).\n\n Changed in version 2.5: ``__hash__()`` may now also return a long\n integer object; the 32-bit integer is then derived from the hash of\n that object.\n\n Changed in version 2.6: ``__hash__`` may now be set to ``None`` to\n explicitly flag instances of a class as unhashable.\n\nobject.__nonzero__(self)\n\n Called to implement truth value testing and the built-in operation\n ``bool()``; should return ``False`` or ``True``, or their integer\n equivalents ``0`` or ``1``. When this method is not defined,\n ``__len__()`` is called, if it is defined, and the object is\n considered true if its result is nonzero. If a class defines\n neither ``__len__()`` nor ``__nonzero__()``, all its instances are\n considered true.\n\nobject.__unicode__(self)\n\n Called to implement ``unicode()`` built-in; should return a Unicode\n object. When this method is not defined, string conversion is\n attempted, and the result of string conversion is converted to\n Unicode using the system default encoding.\n\n\nCustomizing attribute access\n============================\n\nThe following methods can be defined to customize the meaning of\nattribute access (use of, assignment to, or deletion of ``x.name``)\nfor class instances.\n\nobject.__getattr__(self, name)\n\n Called when an attribute lookup has not found the attribute in the\n usual places (i.e. it is not an instance attribute nor is it found\n in the class tree for ``self``). ``name`` is the attribute name.\n This method should return the (computed) attribute value or raise\n an ``AttributeError`` exception.\n\n Note that if the attribute is found through the normal mechanism,\n ``__getattr__()`` is not called. (This is an intentional asymmetry\n between ``__getattr__()`` and ``__setattr__()``.) This is done both\n for efficiency reasons and because otherwise ``__getattr__()``\n would have no way to access other attributes of the instance. Note\n that at least for instance variables, you can fake total control by\n not inserting any values in the instance attribute dictionary (but\n instead inserting them in another object). See the\n ``__getattribute__()`` method below for a way to actually get total\n control in new-style classes.\n\nobject.__setattr__(self, name, value)\n\n Called when an attribute assignment is attempted. This is called\n instead of the normal mechanism (i.e. store the value in the\n instance dictionary). *name* is the attribute name, *value* is the\n value to be assigned to it.\n\n If ``__setattr__()`` wants to assign to an instance attribute, it\n should not simply execute ``self.name = value`` --- this would\n cause a recursive call to itself. Instead, it should insert the\n value in the dictionary of instance attributes, e.g.,\n ``self.__dict__[name] = value``. For new-style classes, rather\n than accessing the instance dictionary, it should call the base\n class method with the same name, for example,\n ``object.__setattr__(self, name, value)``.\n\nobject.__delattr__(self, name)\n\n Like ``__setattr__()`` but for attribute deletion instead of\n assignment. This should only be implemented if ``del obj.name`` is\n meaningful for the object.\n\n\nMore attribute access for new-style classes\n-------------------------------------------\n\nThe following methods only apply to new-style classes.\n\nobject.__getattribute__(self, name)\n\n Called unconditionally to implement attribute accesses for\n instances of the class. If the class also defines\n ``__getattr__()``, the latter will not be called unless\n ``__getattribute__()`` either calls it explicitly or raises an\n ``AttributeError``. This method should return the (computed)\n attribute value or raise an ``AttributeError`` exception. In order\n to avoid infinite recursion in this method, its implementation\n should always call the base class method with the same name to\n access any attributes it needs, for example,\n ``object.__getattribute__(self, name)``.\n\n Note: This method may still be bypassed when looking up special methods\n as the result of implicit invocation via language syntax or\n built-in functions. See *Special method lookup for new-style\n classes*.\n\n\nImplementing Descriptors\n------------------------\n\nThe following methods only apply when an instance of the class\ncontaining the method (a so-called *descriptor* class) appears in an\n*owner* class (the descriptor must be in either the owner\'s class\ndictionary or in the class dictionary for one of its parents). In the\nexamples below, "the attribute" refers to the attribute whose name is\nthe key of the property in the owner class\' ``__dict__``.\n\nobject.__get__(self, instance, owner)\n\n Called to get the attribute of the owner class (class attribute\n access) or of an instance of that class (instance attribute\n access). *owner* is always the owner class, while *instance* is the\n instance that the attribute was accessed through, or ``None`` when\n the attribute is accessed through the *owner*. This method should\n return the (computed) attribute value or raise an\n ``AttributeError`` exception.\n\nobject.__set__(self, instance, value)\n\n Called to set the attribute on an instance *instance* of the owner\n class to a new value, *value*.\n\nobject.__delete__(self, instance)\n\n Called to delete the attribute on an instance *instance* of the\n owner class.\n\n\nInvoking Descriptors\n--------------------\n\nIn general, a descriptor is an object attribute with "binding\nbehavior", one whose attribute access has been overridden by methods\nin the descriptor protocol: ``__get__()``, ``__set__()``, and\n``__delete__()``. If any of those methods are defined for an object,\nit is said to be a descriptor.\n\nThe default behavior for attribute access is to get, set, or delete\nthe attribute from an object\'s dictionary. For instance, ``a.x`` has a\nlookup chain starting with ``a.__dict__[\'x\']``, then\n``type(a).__dict__[\'x\']``, and continuing through the base classes of\n``type(a)`` excluding metaclasses.\n\nHowever, if the looked-up value is an object defining one of the\ndescriptor methods, then Python may override the default behavior and\ninvoke the descriptor method instead. Where this occurs in the\nprecedence chain depends on which descriptor methods were defined and\nhow they were called. Note that descriptors are only invoked for new\nstyle objects or classes (ones that subclass ``object()`` or\n``type()``).\n\nThe starting point for descriptor invocation is a binding, ``a.x``.\nHow the arguments are assembled depends on ``a``:\n\nDirect Call\n The simplest and least common call is when user code directly\n invokes a descriptor method: ``x.__get__(a)``.\n\nInstance Binding\n If binding to a new-style object instance, ``a.x`` is transformed\n into the call: ``type(a).__dict__[\'x\'].__get__(a, type(a))``.\n\nClass Binding\n If binding to a new-style class, ``A.x`` is transformed into the\n call: ``A.__dict__[\'x\'].__get__(None, A)``.\n\nSuper Binding\n If ``a`` is an instance of ``super``, then the binding ``super(B,\n obj).m()`` searches ``obj.__class__.__mro__`` for the base class\n ``A`` immediately preceding ``B`` and then invokes the descriptor\n with the call: ``A.__dict__[\'m\'].__get__(obj, obj.__class__)``.\n\nFor instance bindings, the precedence of descriptor invocation depends\non the which descriptor methods are defined. A descriptor can define\nany combination of ``__get__()``, ``__set__()`` and ``__delete__()``.\nIf it does not define ``__get__()``, then accessing the attribute will\nreturn the descriptor object itself unless there is a value in the\nobject\'s instance dictionary. If the descriptor defines ``__set__()``\nand/or ``__delete__()``, it is a data descriptor; if it defines\nneither, it is a non-data descriptor. Normally, data descriptors\ndefine both ``__get__()`` and ``__set__()``, while non-data\ndescriptors have just the ``__get__()`` method. Data descriptors with\n``__set__()`` and ``__get__()`` defined always override a redefinition\nin an instance dictionary. In contrast, non-data descriptors can be\noverridden by instances.\n\nPython methods (including ``staticmethod()`` and ``classmethod()``)\nare implemented as non-data descriptors. Accordingly, instances can\nredefine and override methods. This allows individual instances to\nacquire behaviors that differ from other instances of the same class.\n\nThe ``property()`` function is implemented as a data descriptor.\nAccordingly, instances cannot override the behavior of a property.\n\n\n__slots__\n---------\n\nBy default, instances of both old and new-style classes have a\ndictionary for attribute storage. This wastes space for objects\nhaving very few instance variables. The space consumption can become\nacute when creating large numbers of instances.\n\nThe default can be overridden by defining *__slots__* in a new-style\nclass definition. The *__slots__* declaration takes a sequence of\ninstance variables and reserves just enough space in each instance to\nhold a value for each variable. Space is saved because *__dict__* is\nnot created for each instance.\n\n__slots__\n\n This class variable can be assigned a string, iterable, or sequence\n of strings with variable names used by instances. If defined in a\n new-style class, *__slots__* reserves space for the declared\n variables and prevents the automatic creation of *__dict__* and\n *__weakref__* for each instance.\n\n New in version 2.2.\n\nNotes on using *__slots__*\n\n* When inheriting from a class without *__slots__*, the *__dict__*\n attribute of that class will always be accessible, so a *__slots__*\n definition in the subclass is meaningless.\n\n* Without a *__dict__* variable, instances cannot be assigned new\n variables not listed in the *__slots__* definition. Attempts to\n assign to an unlisted variable name raises ``AttributeError``. If\n dynamic assignment of new variables is desired, then add\n ``\'__dict__\'`` to the sequence of strings in the *__slots__*\n declaration.\n\n Changed in version 2.3: Previously, adding ``\'__dict__\'`` to the\n *__slots__* declaration would not enable the assignment of new\n attributes not specifically listed in the sequence of instance\n variable names.\n\n* Without a *__weakref__* variable for each instance, classes defining\n *__slots__* do not support weak references to its instances. If weak\n reference support is needed, then add ``\'__weakref__\'`` to the\n sequence of strings in the *__slots__* declaration.\n\n Changed in version 2.3: Previously, adding ``\'__weakref__\'`` to the\n *__slots__* declaration would not enable support for weak\n references.\n\n* *__slots__* are implemented at the class level by creating\n descriptors (*Implementing Descriptors*) for each variable name. As\n a result, class attributes cannot be used to set default values for\n instance variables defined by *__slots__*; otherwise, the class\n attribute would overwrite the descriptor assignment.\n\n* The action of a *__slots__* declaration is limited to the class\n where it is defined. As a result, subclasses will have a *__dict__*\n unless they also define *__slots__* (which must only contain names\n of any *additional* slots).\n\n* If a class defines a slot also defined in a base class, the instance\n variable defined by the base class slot is inaccessible (except by\n retrieving its descriptor directly from the base class). This\n renders the meaning of the program undefined. In the future, a\n check may be added to prevent this.\n\n* Nonempty *__slots__* does not work for classes derived from\n "variable-length" built-in types such as ``long``, ``str`` and\n ``tuple``.\n\n* Any non-string iterable may be assigned to *__slots__*. Mappings may\n also be used; however, in the future, special meaning may be\n assigned to the values corresponding to each key.\n\n* *__class__* assignment works only if both classes have the same\n *__slots__*.\n\n Changed in version 2.6: Previously, *__class__* assignment raised an\n error if either new or old class had *__slots__*.\n\n\nCustomizing class creation\n==========================\n\nBy default, new-style classes are constructed using ``type()``. A\nclass definition is read into a separate namespace and the value of\nclass name is bound to the result of ``type(name, bases, dict)``.\n\nWhen the class definition is read, if *__metaclass__* is defined then\nthe callable assigned to it will be called instead of ``type()``. This\nallows classes or functions to be written which monitor or alter the\nclass creation process:\n\n* Modifying the class dictionary prior to the class being created.\n\n* Returning an instance of another class -- essentially performing the\n role of a factory function.\n\nThese steps will have to be performed in the metaclass\'s ``__new__()``\nmethod -- ``type.__new__()`` can then be called from this method to\ncreate a class with different properties. This example adds a new\nelement to the class dictionary before creating the class:\n\n class metacls(type):\n def __new__(mcs, name, bases, dict):\n dict[\'foo\'] = \'metacls was here\'\n return type.__new__(mcs, name, bases, dict)\n\nYou can of course also override other class methods (or add new\nmethods); for example defining a custom ``__call__()`` method in the\nmetaclass allows custom behavior when the class is called, e.g. not\nalways creating a new instance.\n\n__metaclass__\n\n This variable can be any callable accepting arguments for ``name``,\n ``bases``, and ``dict``. Upon class creation, the callable is used\n instead of the built-in ``type()``.\n\n New in version 2.2.\n\nThe appropriate metaclass is determined by the following precedence\nrules:\n\n* If ``dict[\'__metaclass__\']`` exists, it is used.\n\n* Otherwise, if there is at least one base class, its metaclass is\n used (this looks for a *__class__* attribute first and if not found,\n uses its type).\n\n* Otherwise, if a global variable named __metaclass__ exists, it is\n used.\n\n* Otherwise, the old-style, classic metaclass (types.ClassType) is\n used.\n\nThe potential uses for metaclasses are boundless. Some ideas that have\nbeen explored including logging, interface checking, automatic\ndelegation, automatic property creation, proxies, frameworks, and\nautomatic resource locking/synchronization.\n\n\nCustomizing instance and subclass checks\n========================================\n\nNew in version 2.6.\n\nThe following methods are used to override the default behavior of the\n``isinstance()`` and ``issubclass()`` built-in functions.\n\nIn particular, the metaclass ``abc.ABCMeta`` implements these methods\nin order to allow the addition of Abstract Base Classes (ABCs) as\n"virtual base classes" to any class or type (including built-in\ntypes), including other ABCs.\n\nclass.__instancecheck__(self, instance)\n\n Return true if *instance* should be considered a (direct or\n indirect) instance of *class*. If defined, called to implement\n ``isinstance(instance, class)``.\n\nclass.__subclasscheck__(self, subclass)\n\n Return true if *subclass* should be considered a (direct or\n indirect) subclass of *class*. If defined, called to implement\n ``issubclass(subclass, class)``.\n\nNote that these methods are looked up on the type (metaclass) of a\nclass. They cannot be defined as class methods in the actual class.\nThis is consistent with the lookup of special methods that are called\non instances, only in this case the instance is itself a class.\n\nSee also:\n\n **PEP 3119** - Introducing Abstract Base Classes\n Includes the specification for customizing ``isinstance()`` and\n ``issubclass()`` behavior through ``__instancecheck__()`` and\n ``__subclasscheck__()``, with motivation for this functionality\n in the context of adding Abstract Base Classes (see the ``abc``\n module) to the language.\n\n\nEmulating callable objects\n==========================\n\nobject.__call__(self[, args...])\n\n Called when the instance is "called" as a function; if this method\n is defined, ``x(arg1, arg2, ...)`` is a shorthand for\n ``x.__call__(arg1, arg2, ...)``.\n\n\nEmulating container types\n=========================\n\nThe following methods can be defined to implement container objects.\nContainers usually are sequences (such as lists or tuples) or mappings\n(like dictionaries), but can represent other containers as well. The\nfirst set of methods is used either to emulate a sequence or to\nemulate a mapping; the difference is that for a sequence, the\nallowable keys should be the integers *k* for which ``0 <= k < N``\nwhere *N* is the length of the sequence, or slice objects, which\ndefine a range of items. (For backwards compatibility, the method\n``__getslice__()`` (see below) can also be defined to handle simple,\nbut not extended slices.) It is also recommended that mappings provide\nthe methods ``keys()``, ``values()``, ``items()``, ``has_key()``,\n``get()``, ``clear()``, ``setdefault()``, ``iterkeys()``,\n``itervalues()``, ``iteritems()``, ``pop()``, ``popitem()``,\n``copy()``, and ``update()`` behaving similar to those for Python\'s\nstandard dictionary objects. The ``UserDict`` module provides a\n``DictMixin`` class to help create those methods from a base set of\n``__getitem__()``, ``__setitem__()``, ``__delitem__()``, and\n``keys()``. Mutable sequences should provide methods ``append()``,\n``count()``, ``index()``, ``extend()``, ``insert()``, ``pop()``,\n``remove()``, ``reverse()`` and ``sort()``, like Python standard list\nobjects. Finally, sequence types should implement addition (meaning\nconcatenation) and multiplication (meaning repetition) by defining the\nmethods ``__add__()``, ``__radd__()``, ``__iadd__()``, ``__mul__()``,\n``__rmul__()`` and ``__imul__()`` described below; they should not\ndefine ``__coerce__()`` or other numerical operators. It is\nrecommended that both mappings and sequences implement the\n``__contains__()`` method to allow efficient use of the ``in``\noperator; for mappings, ``in`` should be equivalent of ``has_key()``;\nfor sequences, it should search through the values. It is further\nrecommended that both mappings and sequences implement the\n``__iter__()`` method to allow efficient iteration through the\ncontainer; for mappings, ``__iter__()`` should be the same as\n``iterkeys()``; for sequences, it should iterate through the values.\n\nobject.__len__(self)\n\n Called to implement the built-in function ``len()``. Should return\n the length of the object, an integer ``>=`` 0. Also, an object\n that doesn\'t define a ``__nonzero__()`` method and whose\n ``__len__()`` method returns zero is considered to be false in a\n Boolean context.\n\nobject.__getitem__(self, key)\n\n Called to implement evaluation of ``self[key]``. For sequence\n types, the accepted keys should be integers and slice objects.\n Note that the special interpretation of negative indexes (if the\n class wishes to emulate a sequence type) is up to the\n ``__getitem__()`` method. If *key* is of an inappropriate type,\n ``TypeError`` may be raised; if of a value outside the set of\n indexes for the sequence (after any special interpretation of\n negative values), ``IndexError`` should be raised. For mapping\n types, if *key* is missing (not in the container), ``KeyError``\n should be raised.\n\n Note: ``for`` loops expect that an ``IndexError`` will be raised for\n illegal indexes to allow proper detection of the end of the\n sequence.\n\nobject.__setitem__(self, key, value)\n\n Called to implement assignment to ``self[key]``. Same note as for\n ``__getitem__()``. This should only be implemented for mappings if\n the objects support changes to the values for keys, or if new keys\n can be added, or for sequences if elements can be replaced. The\n same exceptions should be raised for improper *key* values as for\n the ``__getitem__()`` method.\n\nobject.__delitem__(self, key)\n\n Called to implement deletion of ``self[key]``. Same note as for\n ``__getitem__()``. This should only be implemented for mappings if\n the objects support removal of keys, or for sequences if elements\n can be removed from the sequence. The same exceptions should be\n raised for improper *key* values as for the ``__getitem__()``\n method.\n\nobject.__iter__(self)\n\n This method is called when an iterator is required for a container.\n This method should return a new iterator object that can iterate\n over all the objects in the container. For mappings, it should\n iterate over the keys of the container, and should also be made\n available as the method ``iterkeys()``.\n\n Iterator objects also need to implement this method; they are\n required to return themselves. For more information on iterator\n objects, see *Iterator Types*.\n\nobject.__reversed__(self)\n\n Called (if present) by the ``reversed()`` built-in to implement\n reverse iteration. It should return a new iterator object that\n iterates over all the objects in the container in reverse order.\n\n If the ``__reversed__()`` method is not provided, the\n ``reversed()`` built-in will fall back to using the sequence\n protocol (``__len__()`` and ``__getitem__()``). Objects that\n support the sequence protocol should only provide\n ``__reversed__()`` if they can provide an implementation that is\n more efficient than the one provided by ``reversed()``.\n\n New in version 2.6.\n\nThe membership test operators (``in`` and ``not in``) are normally\nimplemented as an iteration through a sequence. However, container\nobjects can supply the following special method with a more efficient\nimplementation, which also does not require the object be a sequence.\n\nobject.__contains__(self, item)\n\n Called to implement membership test operators. Should return true\n if *item* is in *self*, false otherwise. For mapping objects, this\n should consider the keys of the mapping rather than the values or\n the key-item pairs.\n\n For objects that don\'t define ``__contains__()``, the membership\n test first tries iteration via ``__iter__()``, then the old\n sequence iteration protocol via ``__getitem__()``, see *this\n section in the language reference*.\n\n\nAdditional methods for emulation of sequence types\n==================================================\n\nThe following optional methods can be defined to further emulate\nsequence objects. Immutable sequences methods should at most only\ndefine ``__getslice__()``; mutable sequences might define all three\nmethods.\n\nobject.__getslice__(self, i, j)\n\n Deprecated since version 2.0: Support slice objects as parameters\n to the ``__getitem__()`` method. (However, built-in types in\n CPython currently still implement ``__getslice__()``. Therefore,\n you have to override it in derived classes when implementing\n slicing.)\n\n Called to implement evaluation of ``self[i:j]``. The returned\n object should be of the same type as *self*. Note that missing *i*\n or *j* in the slice expression are replaced by zero or\n ``sys.maxint``, respectively. If negative indexes are used in the\n slice, the length of the sequence is added to that index. If the\n instance does not implement the ``__len__()`` method, an\n ``AttributeError`` is raised. No guarantee is made that indexes\n adjusted this way are not still negative. Indexes which are\n greater than the length of the sequence are not modified. If no\n ``__getslice__()`` is found, a slice object is created instead, and\n passed to ``__getitem__()`` instead.\n\nobject.__setslice__(self, i, j, sequence)\n\n Called to implement assignment to ``self[i:j]``. Same notes for *i*\n and *j* as for ``__getslice__()``.\n\n This method is deprecated. If no ``__setslice__()`` is found, or\n for extended slicing of the form ``self[i:j:k]``, a slice object is\n created, and passed to ``__setitem__()``, instead of\n ``__setslice__()`` being called.\n\nobject.__delslice__(self, i, j)\n\n Called to implement deletion of ``self[i:j]``. Same notes for *i*\n and *j* as for ``__getslice__()``. This method is deprecated. If no\n ``__delslice__()`` is found, or for extended slicing of the form\n ``self[i:j:k]``, a slice object is created, and passed to\n ``__delitem__()``, instead of ``__delslice__()`` being called.\n\nNotice that these methods are only invoked when a single slice with a\nsingle colon is used, and the slice method is available. For slice\noperations involving extended slice notation, or in absence of the\nslice methods, ``__getitem__()``, ``__setitem__()`` or\n``__delitem__()`` is called with a slice object as argument.\n\nThe following example demonstrate how to make your program or module\ncompatible with earlier versions of Python (assuming that methods\n``__getitem__()``, ``__setitem__()`` and ``__delitem__()`` support\nslice objects as arguments):\n\n class MyClass:\n ...\n def __getitem__(self, index):\n ...\n def __setitem__(self, index, value):\n ...\n def __delitem__(self, index):\n ...\n\n if sys.version_info < (2, 0):\n # They won\'t be defined if version is at least 2.0 final\n\n def __getslice__(self, i, j):\n return self[max(0, i):max(0, j):]\n def __setslice__(self, i, j, seq):\n self[max(0, i):max(0, j):] = seq\n def __delslice__(self, i, j):\n del self[max(0, i):max(0, j):]\n ...\n\nNote the calls to ``max()``; these are necessary because of the\nhandling of negative indices before the ``__*slice__()`` methods are\ncalled. When negative indexes are used, the ``__*item__()`` methods\nreceive them as provided, but the ``__*slice__()`` methods get a\n"cooked" form of the index values. For each negative index value, the\nlength of the sequence is added to the index before calling the method\n(which may still result in a negative index); this is the customary\nhandling of negative indexes by the built-in sequence types, and the\n``__*item__()`` methods are expected to do this as well. However,\nsince they should already be doing that, negative indexes cannot be\npassed in; they must be constrained to the bounds of the sequence\nbefore being passed to the ``__*item__()`` methods. Calling ``max(0,\ni)`` conveniently returns the proper value.\n\n\nEmulating numeric types\n=======================\n\nThe following methods can be defined to emulate numeric objects.\nMethods corresponding to operations that are not supported by the\nparticular kind of number implemented (e.g., bitwise operations for\nnon-integral numbers) should be left undefined.\n\nobject.__add__(self, other)\nobject.__sub__(self, other)\nobject.__mul__(self, other)\nobject.__floordiv__(self, other)\nobject.__mod__(self, other)\nobject.__divmod__(self, other)\nobject.__pow__(self, other[, modulo])\nobject.__lshift__(self, other)\nobject.__rshift__(self, other)\nobject.__and__(self, other)\nobject.__xor__(self, other)\nobject.__or__(self, other)\n\n These methods are called to implement the binary arithmetic\n operations (``+``, ``-``, ``*``, ``//``, ``%``, ``divmod()``,\n ``pow()``, ``**``, ``<<``, ``>>``, ``&``, ``^``, ``|``). For\n instance, to evaluate the expression ``x + y``, where *x* is an\n instance of a class that has an ``__add__()`` method,\n ``x.__add__(y)`` is called. The ``__divmod__()`` method should be\n the equivalent to using ``__floordiv__()`` and ``__mod__()``; it\n should not be related to ``__truediv__()`` (described below). Note\n that ``__pow__()`` should be defined to accept an optional third\n argument if the ternary version of the built-in ``pow()`` function\n is to be supported.\n\n If one of those methods does not support the operation with the\n supplied arguments, it should return ``NotImplemented``.\n\nobject.__div__(self, other)\nobject.__truediv__(self, other)\n\n The division operator (``/``) is implemented by these methods. The\n ``__truediv__()`` method is used when ``__future__.division`` is in\n effect, otherwise ``__div__()`` is used. If only one of these two\n methods is defined, the object will not support division in the\n alternate context; ``TypeError`` will be raised instead.\n\nobject.__radd__(self, other)\nobject.__rsub__(self, other)\nobject.__rmul__(self, other)\nobject.__rdiv__(self, other)\nobject.__rtruediv__(self, other)\nobject.__rfloordiv__(self, other)\nobject.__rmod__(self, other)\nobject.__rdivmod__(self, other)\nobject.__rpow__(self, other)\nobject.__rlshift__(self, other)\nobject.__rrshift__(self, other)\nobject.__rand__(self, other)\nobject.__rxor__(self, other)\nobject.__ror__(self, other)\n\n These methods are called to implement the binary arithmetic\n operations (``+``, ``-``, ``*``, ``/``, ``%``, ``divmod()``,\n ``pow()``, ``**``, ``<<``, ``>>``, ``&``, ``^``, ``|``) with\n reflected (swapped) operands. These functions are only called if\n the left operand does not support the corresponding operation and\n the operands are of different types. [2] For instance, to evaluate\n the expression ``x - y``, where *y* is an instance of a class that\n has an ``__rsub__()`` method, ``y.__rsub__(x)`` is called if\n ``x.__sub__(y)`` returns *NotImplemented*.\n\n Note that ternary ``pow()`` will not try calling ``__rpow__()``\n (the coercion rules would become too complicated).\n\n Note: If the right operand\'s type is a subclass of the left operand\'s\n type and that subclass provides the reflected method for the\n operation, this method will be called before the left operand\'s\n non-reflected method. This behavior allows subclasses to\n override their ancestors\' operations.\n\nobject.__iadd__(self, other)\nobject.__isub__(self, other)\nobject.__imul__(self, other)\nobject.__idiv__(self, other)\nobject.__itruediv__(self, other)\nobject.__ifloordiv__(self, other)\nobject.__imod__(self, other)\nobject.__ipow__(self, other[, modulo])\nobject.__ilshift__(self, other)\nobject.__irshift__(self, other)\nobject.__iand__(self, other)\nobject.__ixor__(self, other)\nobject.__ior__(self, other)\n\n These methods are called to implement the augmented arithmetic\n assignments (``+=``, ``-=``, ``*=``, ``/=``, ``//=``, ``%=``,\n ``**=``, ``<<=``, ``>>=``, ``&=``, ``^=``, ``|=``). These methods\n should attempt to do the operation in-place (modifying *self*) and\n return the result (which could be, but does not have to be,\n *self*). If a specific method is not defined, the augmented\n assignment falls back to the normal methods. For instance, to\n execute the statement ``x += y``, where *x* is an instance of a\n class that has an ``__iadd__()`` method, ``x.__iadd__(y)`` is\n called. If *x* is an instance of a class that does not define a\n ``__iadd__()`` method, ``x.__add__(y)`` and ``y.__radd__(x)`` are\n considered, as with the evaluation of ``x + y``.\n\nobject.__neg__(self)\nobject.__pos__(self)\nobject.__abs__(self)\nobject.__invert__(self)\n\n Called to implement the unary arithmetic operations (``-``, ``+``,\n ``abs()`` and ``~``).\n\nobject.__complex__(self)\nobject.__int__(self)\nobject.__long__(self)\nobject.__float__(self)\n\n Called to implement the built-in functions ``complex()``,\n ``int()``, ``long()``, and ``float()``. Should return a value of\n the appropriate type.\n\nobject.__oct__(self)\nobject.__hex__(self)\n\n Called to implement the built-in functions ``oct()`` and ``hex()``.\n Should return a string value.\n\nobject.__index__(self)\n\n Called to implement ``operator.index()``. Also called whenever\n Python needs an integer object (such as in slicing). Must return\n an integer (int or long).\n\n New in version 2.5.\n\nobject.__coerce__(self, other)\n\n Called to implement "mixed-mode" numeric arithmetic. Should either\n return a 2-tuple containing *self* and *other* converted to a\n common numeric type, or ``None`` if conversion is impossible. When\n the common type would be the type of ``other``, it is sufficient to\n return ``None``, since the interpreter will also ask the other\n object to attempt a coercion (but sometimes, if the implementation\n of the other type cannot be changed, it is useful to do the\n conversion to the other type here). A return value of\n ``NotImplemented`` is equivalent to returning ``None``.\n\n\nCoercion rules\n==============\n\nThis section used to document the rules for coercion. As the language\nhas evolved, the coercion rules have become hard to document\nprecisely; documenting what one version of one particular\nimplementation does is undesirable. Instead, here are some informal\nguidelines regarding coercion. In Python 3.0, coercion will not be\nsupported.\n\n* If the left operand of a % operator is a string or Unicode object,\n no coercion takes place and the string formatting operation is\n invoked instead.\n\n* It is no longer recommended to define a coercion operation. Mixed-\n mode operations on types that don\'t define coercion pass the\n original arguments to the operation.\n\n* New-style classes (those derived from ``object``) never invoke the\n ``__coerce__()`` method in response to a binary operator; the only\n time ``__coerce__()`` is invoked is when the built-in function\n ``coerce()`` is called.\n\n* For most intents and purposes, an operator that returns\n ``NotImplemented`` is treated the same as one that is not\n implemented at all.\n\n* Below, ``__op__()`` and ``__rop__()`` are used to signify the\n generic method names corresponding to an operator; ``__iop__()`` is\n used for the corresponding in-place operator. For example, for the\n operator \'``+``\', ``__add__()`` and ``__radd__()`` are used for the\n left and right variant of the binary operator, and ``__iadd__()``\n for the in-place variant.\n\n* For objects *x* and *y*, first ``x.__op__(y)`` is tried. If this is\n not implemented or returns ``NotImplemented``, ``y.__rop__(x)`` is\n tried. If this is also not implemented or returns\n ``NotImplemented``, a ``TypeError`` exception is raised. But see\n the following exception:\n\n* Exception to the previous item: if the left operand is an instance\n of a built-in type or a new-style class, and the right operand is an\n instance of a proper subclass of that type or class and overrides\n the base\'s ``__rop__()`` method, the right operand\'s ``__rop__()``\n method is tried *before* the left operand\'s ``__op__()`` method.\n\n This is done so that a subclass can completely override binary\n operators. Otherwise, the left operand\'s ``__op__()`` method would\n always accept the right operand: when an instance of a given class\n is expected, an instance of a subclass of that class is always\n acceptable.\n\n* When either operand type defines a coercion, this coercion is called\n before that type\'s ``__op__()`` or ``__rop__()`` method is called,\n but no sooner. If the coercion returns an object of a different\n type for the operand whose coercion is invoked, part of the process\n is redone using the new object.\n\n* When an in-place operator (like \'``+=``\') is used, if the left\n operand implements ``__iop__()``, it is invoked without any\n coercion. When the operation falls back to ``__op__()`` and/or\n ``__rop__()``, the normal coercion rules apply.\n\n* In ``x + y``, if *x* is a sequence that implements sequence\n concatenation, sequence concatenation is invoked.\n\n* In ``x * y``, if one operator is a sequence that implements sequence\n repetition, and the other is an integer (``int`` or ``long``),\n sequence repetition is invoked.\n\n* Rich comparisons (implemented by methods ``__eq__()`` and so on)\n never use coercion. Three-way comparison (implemented by\n ``__cmp__()``) does use coercion under the same conditions as other\n binary operations use it.\n\n* In the current implementation, the built-in numeric types ``int``,\n ``long``, ``float``, and ``complex`` do not use coercion. All these\n types implement a ``__coerce__()`` method, for use by the built-in\n ``coerce()`` function.\n\n Changed in version 2.7.\n\n\nWith Statement Context Managers\n===============================\n\nNew in version 2.5.\n\nA *context manager* is an object that defines the runtime context to\nbe established when executing a ``with`` statement. The context\nmanager handles the entry into, and the exit from, the desired runtime\ncontext for the execution of the block of code. Context managers are\nnormally invoked using the ``with`` statement (described in section\n*The with statement*), but can also be used by directly invoking their\nmethods.\n\nTypical uses of context managers include saving and restoring various\nkinds of global state, locking and unlocking resources, closing opened\nfiles, etc.\n\nFor more information on context managers, see *Context Manager Types*.\n\nobject.__enter__(self)\n\n Enter the runtime context related to this object. The ``with``\n statement will bind this method\'s return value to the target(s)\n specified in the ``as`` clause of the statement, if any.\n\nobject.__exit__(self, exc_type, exc_value, traceback)\n\n Exit the runtime context related to this object. The parameters\n describe the exception that caused the context to be exited. If the\n context was exited without an exception, all three arguments will\n be ``None``.\n\n If an exception is supplied, and the method wishes to suppress the\n exception (i.e., prevent it from being propagated), it should\n return a true value. Otherwise, the exception will be processed\n normally upon exit from this method.\n\n Note that ``__exit__()`` methods should not reraise the passed-in\n exception; this is the caller\'s responsibility.\n\nSee also:\n\n **PEP 0343** - The "with" statement\n The specification, background, and examples for the Python\n ``with`` statement.\n\n\nSpecial method lookup for old-style classes\n===========================================\n\nFor old-style classes, special methods are always looked up in exactly\nthe same way as any other method or attribute. This is the case\nregardless of whether the method is being looked up explicitly as in\n``x.__getitem__(i)`` or implicitly as in ``x[i]``.\n\nThis behaviour means that special methods may exhibit different\nbehaviour for different instances of a single old-style class if the\nappropriate special attributes are set differently:\n\n >>> class C:\n ... pass\n ...\n >>> c1 = C()\n >>> c2 = C()\n >>> c1.__len__ = lambda: 5\n >>> c2.__len__ = lambda: 9\n >>> len(c1)\n 5\n >>> len(c2)\n 9\n\n\nSpecial method lookup for new-style classes\n===========================================\n\nFor new-style classes, implicit invocations of special methods are\nonly guaranteed to work correctly if defined on an object\'s type, not\nin the object\'s instance dictionary. That behaviour is the reason why\nthe following code raises an exception (unlike the equivalent example\nwith old-style classes):\n\n >>> class C(object):\n ... pass\n ...\n >>> c = C()\n >>> c.__len__ = lambda: 5\n >>> len(c)\n Traceback (most recent call last):\n File "<stdin>", line 1, in <module>\n TypeError: object of type \'C\' has no len()\n\nThe rationale behind this behaviour lies with a number of special\nmethods such as ``__hash__()`` and ``__repr__()`` that are implemented\nby all objects, including type objects. If the implicit lookup of\nthese methods used the conventional lookup process, they would fail\nwhen invoked on the type object itself:\n\n >>> 1 .__hash__() == hash(1)\n True\n >>> int.__hash__() == hash(int)\n Traceback (most recent call last):\n File "<stdin>", line 1, in <module>\n TypeError: descriptor \'__hash__\' of \'int\' object needs an argument\n\nIncorrectly attempting to invoke an unbound method of a class in this\nway is sometimes referred to as \'metaclass confusion\', and is avoided\nby bypassing the instance when looking up special methods:\n\n >>> type(1).__hash__(1) == hash(1)\n True\n >>> type(int).__hash__(int) == hash(int)\n True\n\nIn addition to bypassing any instance attributes in the interest of\ncorrectness, implicit special method lookup generally also bypasses\nthe ``__getattribute__()`` method even of the object\'s metaclass:\n\n >>> class Meta(type):\n ... def __getattribute__(*args):\n ... print "Metaclass getattribute invoked"\n ... return type.__getattribute__(*args)\n ...\n >>> class C(object):\n ... __metaclass__ = Meta\n ... def __len__(self):\n ... return 10\n ... def __getattribute__(*args):\n ... print "Class getattribute invoked"\n ... return object.__getattribute__(*args)\n ...\n >>> c = C()\n >>> c.__len__() # Explicit lookup via instance\n Class getattribute invoked\n 10\n >>> type(c).__len__(c) # Explicit lookup via type\n Metaclass getattribute invoked\n 10\n >>> len(c) # Implicit lookup\n 10\n\nBypassing the ``__getattribute__()`` machinery in this fashion\nprovides significant scope for speed optimisations within the\ninterpreter, at the cost of some flexibility in the handling of\nspecial methods (the special method *must* be set on the class object\nitself in order to be consistently invoked by the interpreter).\n\n-[ Footnotes ]-\n\n[1] It *is* possible in some cases to change an object\'s type, under\n certain controlled conditions. It generally isn\'t a good idea\n though, since it can lead to some very strange behaviour if it is\n handled incorrectly.\n\n[2] For operands of the same type, it is assumed that if the non-\n reflected method (such as ``__add__()``) fails the operation is\n not supported, which is why the reflected method is not called.\n', 'string-conversions': u'\nString conversions\n******************\n\nA string conversion is an expression list enclosed in reverse (a.k.a.\nbackward) quotes:\n\n string_conversion ::= "\'" expression_list "\'"\n\nA string conversion evaluates the contained expression list and\nconverts the resulting object into a string according to rules\nspecific to its type.\n\nIf the object is a string, a number, ``None``, or a tuple, list or\ndictionary containing only objects whose type is one of these, the\nresulting string is a valid Python expression which can be passed to\nthe built-in function ``eval()`` to yield an expression with the same\nvalue (or an approximation, if floating point numbers are involved).\n\n(In particular, converting a string adds quotes around it and converts\n"funny" characters to escape sequences that are safe to print.)\n\nRecursive objects (for example, lists or dictionaries that contain a\nreference to themselves, directly or indirectly) use ``...`` to\nindicate a recursive reference, and the result cannot be passed to\n``eval()`` to get an equal value (``SyntaxError`` will be raised\ninstead).\n\nThe built-in function ``repr()`` performs exactly the same conversion\nin its argument as enclosing it in parentheses and reverse quotes\ndoes. The built-in function ``str()`` performs a similar but more\nuser-friendly conversion.\n', - 'string-methods': u'\nString Methods\n**************\n\nBelow are listed the string methods which both 8-bit strings and\nUnicode objects support.\n\nIn addition, Python\'s strings support the sequence type methods\ndescribed in the *Sequence Types --- str, unicode, list, tuple,\nbuffer, xrange* section. To output formatted strings use template\nstrings or the ``%`` operator described in the *String Formatting\nOperations* section. Also, see the ``re`` module for string functions\nbased on regular expressions.\n\nstr.capitalize()\n\n Return a copy of the string with only its first character\n capitalized.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.center(width[, fillchar])\n\n Return centered in a string of length *width*. Padding is done\n using the specified *fillchar* (default is a space).\n\n Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.count(sub[, start[, end]])\n\n Return the number of non-overlapping occurrences of substring *sub*\n in the range [*start*, *end*]. Optional arguments *start* and\n *end* are interpreted as in slice notation.\n\nstr.decode([encoding[, errors]])\n\n Decodes the string using the codec registered for *encoding*.\n *encoding* defaults to the default string encoding. *errors* may\n be given to set a different error handling scheme. The default is\n ``\'strict\'``, meaning that encoding errors raise ``UnicodeError``.\n Other possible values are ``\'ignore\'``, ``\'replace\'`` and any other\n name registered via ``codecs.register_error()``, see section *Codec\n Base Classes*.\n\n New in version 2.2.\n\n Changed in version 2.3: Support for other error handling schemes\n added.\n\n Changed in version 2.7: Support for keyword arguments added.\n\nstr.encode([encoding[, errors]])\n\n Return an encoded version of the string. Default encoding is the\n current default string encoding. *errors* may be given to set a\n different error handling scheme. The default for *errors* is\n ``\'strict\'``, meaning that encoding errors raise a\n ``UnicodeError``. Other possible values are ``\'ignore\'``,\n ``\'replace\'``, ``\'xmlcharrefreplace\'``, ``\'backslashreplace\'`` and\n any other name registered via ``codecs.register_error()``, see\n section *Codec Base Classes*. For a list of possible encodings, see\n section *Standard Encodings*.\n\n New in version 2.0.\n\n Changed in version 2.3: Support for ``\'xmlcharrefreplace\'`` and\n ``\'backslashreplace\'`` and other error handling schemes added.\n\n Changed in version 2.7: Support for keyword arguments added.\n\nstr.endswith(suffix[, start[, end]])\n\n Return ``True`` if the string ends with the specified *suffix*,\n otherwise return ``False``. *suffix* can also be a tuple of\n suffixes to look for. With optional *start*, test beginning at\n that position. With optional *end*, stop comparing at that\n position.\n\n Changed in version 2.5: Accept tuples as *suffix*.\n\nstr.expandtabs([tabsize])\n\n Return a copy of the string where all tab characters are replaced\n by one or more spaces, depending on the current column and the\n given tab size. The column number is reset to zero after each\n newline occurring in the string. If *tabsize* is not given, a tab\n size of ``8`` characters is assumed. This doesn\'t understand other\n non-printing characters or escape sequences.\n\nstr.find(sub[, start[, end]])\n\n Return the lowest index in the string where substring *sub* is\n found, such that *sub* is contained in the slice ``s[start:end]``.\n Optional arguments *start* and *end* are interpreted as in slice\n notation. Return ``-1`` if *sub* is not found.\n\nstr.format(*args, **kwargs)\n\n Perform a string formatting operation. The string on which this\n method is called can contain literal text or replacement fields\n delimited by braces ``{}``. Each replacement field contains either\n the numeric index of a positional argument, or the name of a\n keyword argument. Returns a copy of the string where each\n replacement field is replaced with the string value of the\n corresponding argument.\n\n >>> "The sum of 1 + 2 is {0}".format(1+2)\n \'The sum of 1 + 2 is 3\'\n\n See *Format String Syntax* for a description of the various\n formatting options that can be specified in format strings.\n\n This method of string formatting is the new standard in Python 3.0,\n and should be preferred to the ``%`` formatting described in\n *String Formatting Operations* in new code.\n\n New in version 2.6.\n\nstr.index(sub[, start[, end]])\n\n Like ``find()``, but raise ``ValueError`` when the substring is not\n found.\n\nstr.isalnum()\n\n Return true if all characters in the string are alphanumeric and\n there is at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isalpha()\n\n Return true if all characters in the string are alphabetic and\n there is at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isdigit()\n\n Return true if all characters in the string are digits and there is\n at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.islower()\n\n Return true if all cased characters in the string are lowercase and\n there is at least one cased character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isspace()\n\n Return true if there are only whitespace characters in the string\n and there is at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.istitle()\n\n Return true if the string is a titlecased string and there is at\n least one character, for example uppercase characters may only\n follow uncased characters and lowercase characters only cased ones.\n Return false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isupper()\n\n Return true if all cased characters in the string are uppercase and\n there is at least one cased character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.join(iterable)\n\n Return a string which is the concatenation of the strings in the\n *iterable* *iterable*. The separator between elements is the\n string providing this method.\n\nstr.ljust(width[, fillchar])\n\n Return the string left justified in a string of length *width*.\n Padding is done using the specified *fillchar* (default is a\n space). The original string is returned if *width* is less than\n ``len(s)``.\n\n Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.lower()\n\n Return a copy of the string converted to lowercase.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.lstrip([chars])\n\n Return a copy of the string with leading characters removed. The\n *chars* argument is a string specifying the set of characters to be\n removed. If omitted or ``None``, the *chars* argument defaults to\n removing whitespace. The *chars* argument is not a prefix; rather,\n all combinations of its values are stripped:\n\n >>> \' spacious \'.lstrip()\n \'spacious \'\n >>> \'www.example.com\'.lstrip(\'cmowz.\')\n \'example.com\'\n\n Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.partition(sep)\n\n Split the string at the first occurrence of *sep*, and return a\n 3-tuple containing the part before the separator, the separator\n itself, and the part after the separator. If the separator is not\n found, return a 3-tuple containing the string itself, followed by\n two empty strings.\n\n New in version 2.5.\n\nstr.replace(old, new[, count])\n\n Return a copy of the string with all occurrences of substring *old*\n replaced by *new*. If the optional argument *count* is given, only\n the first *count* occurrences are replaced.\n\nstr.rfind(sub[, start[, end]])\n\n Return the highest index in the string where substring *sub* is\n found, such that *sub* is contained within ``s[start:end]``.\n Optional arguments *start* and *end* are interpreted as in slice\n notation. Return ``-1`` on failure.\n\nstr.rindex(sub[, start[, end]])\n\n Like ``rfind()`` but raises ``ValueError`` when the substring *sub*\n is not found.\n\nstr.rjust(width[, fillchar])\n\n Return the string right justified in a string of length *width*.\n Padding is done using the specified *fillchar* (default is a\n space). The original string is returned if *width* is less than\n ``len(s)``.\n\n Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.rpartition(sep)\n\n Split the string at the last occurrence of *sep*, and return a\n 3-tuple containing the part before the separator, the separator\n itself, and the part after the separator. If the separator is not\n found, return a 3-tuple containing two empty strings, followed by\n the string itself.\n\n New in version 2.5.\n\nstr.rsplit([sep[, maxsplit]])\n\n Return a list of the words in the string, using *sep* as the\n delimiter string. If *maxsplit* is given, at most *maxsplit* splits\n are done, the *rightmost* ones. If *sep* is not specified or\n ``None``, any whitespace string is a separator. Except for\n splitting from the right, ``rsplit()`` behaves like ``split()``\n which is described in detail below.\n\n New in version 2.4.\n\nstr.rstrip([chars])\n\n Return a copy of the string with trailing characters removed. The\n *chars* argument is a string specifying the set of characters to be\n removed. If omitted or ``None``, the *chars* argument defaults to\n removing whitespace. The *chars* argument is not a suffix; rather,\n all combinations of its values are stripped:\n\n >>> \' spacious \'.rstrip()\n \' spacious\'\n >>> \'mississippi\'.rstrip(\'ipz\')\n \'mississ\'\n\n Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.split([sep[, maxsplit]])\n\n Return a list of the words in the string, using *sep* as the\n delimiter string. If *maxsplit* is given, at most *maxsplit*\n splits are done (thus, the list will have at most ``maxsplit+1``\n elements). If *maxsplit* is not specified, then there is no limit\n on the number of splits (all possible splits are made).\n\n If *sep* is given, consecutive delimiters are not grouped together\n and are deemed to delimit empty strings (for example,\n ``\'1,,2\'.split(\',\')`` returns ``[\'1\', \'\', \'2\']``). The *sep*\n argument may consist of multiple characters (for example,\n ``\'1<>2<>3\'.split(\'<>\')`` returns ``[\'1\', \'2\', \'3\']``). Splitting\n an empty string with a specified separator returns ``[\'\']``.\n\n If *sep* is not specified or is ``None``, a different splitting\n algorithm is applied: runs of consecutive whitespace are regarded\n as a single separator, and the result will contain no empty strings\n at the start or end if the string has leading or trailing\n whitespace. Consequently, splitting an empty string or a string\n consisting of just whitespace with a ``None`` separator returns\n ``[]``.\n\n For example, ``\' 1 2 3 \'.split()`` returns ``[\'1\', \'2\', \'3\']``,\n and ``\' 1 2 3 \'.split(None, 1)`` returns ``[\'1\', \'2 3 \']``.\n\nstr.splitlines([keepends])\n\n Return a list of the lines in the string, breaking at line\n boundaries. Line breaks are not included in the resulting list\n unless *keepends* is given and true.\n\nstr.startswith(prefix[, start[, end]])\n\n Return ``True`` if string starts with the *prefix*, otherwise\n return ``False``. *prefix* can also be a tuple of prefixes to look\n for. With optional *start*, test string beginning at that\n position. With optional *end*, stop comparing string at that\n position.\n\n Changed in version 2.5: Accept tuples as *prefix*.\n\nstr.strip([chars])\n\n Return a copy of the string with the leading and trailing\n characters removed. The *chars* argument is a string specifying the\n set of characters to be removed. If omitted or ``None``, the\n *chars* argument defaults to removing whitespace. The *chars*\n argument is not a prefix or suffix; rather, all combinations of its\n values are stripped:\n\n >>> \' spacious \'.strip()\n \'spacious\'\n >>> \'www.example.com\'.strip(\'cmowz.\')\n \'example\'\n\n Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.swapcase()\n\n Return a copy of the string with uppercase characters converted to\n lowercase and vice versa.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.title()\n\n Return a titlecased version of the string where words start with an\n uppercase character and the remaining characters are lowercase.\n\n The algorithm uses a simple language-independent definition of a\n word as groups of consecutive letters. The definition works in\n many contexts but it means that apostrophes in contractions and\n possessives form word boundaries, which may not be the desired\n result:\n\n >>> "they\'re bill\'s friends from the UK".title()\n "They\'Re Bill\'S Friends From The Uk"\n\n A workaround for apostrophes can be constructed using regular\n expressions:\n\n >>> import re\n >>> def titlecase(s):\n return re.sub(r"[A-Za-z]+(\'[A-Za-z]+)?",\n lambda mo: mo.group(0)[0].upper() +\n mo.group(0)[1:].lower(),\n s)\n\n >>> titlecase("they\'re bill\'s friends.")\n "They\'re Bill\'s Friends."\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.translate(table[, deletechars])\n\n Return a copy of the string where all characters occurring in the\n optional argument *deletechars* are removed, and the remaining\n characters have been mapped through the given translation table,\n which must be a string of length 256.\n\n You can use the ``maketrans()`` helper function in the ``string``\n module to create a translation table. For string objects, set the\n *table* argument to ``None`` for translations that only delete\n characters:\n\n >>> \'read this short text\'.translate(None, \'aeiou\')\n \'rd ths shrt txt\'\n\n New in version 2.6: Support for a ``None`` *table* argument.\n\n For Unicode objects, the ``translate()`` method does not accept the\n optional *deletechars* argument. Instead, it returns a copy of the\n *s* where all characters have been mapped through the given\n translation table which must be a mapping of Unicode ordinals to\n Unicode ordinals, Unicode strings or ``None``. Unmapped characters\n are left untouched. Characters mapped to ``None`` are deleted.\n Note, a more flexible approach is to create a custom character\n mapping codec using the ``codecs`` module (see ``encodings.cp1251``\n for an example).\n\nstr.upper()\n\n Return a copy of the string converted to uppercase.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.zfill(width)\n\n Return the numeric string left filled with zeros in a string of\n length *width*. A sign prefix is handled correctly. The original\n string is returned if *width* is less than ``len(s)``.\n\n New in version 2.2.2.\n\nThe following methods are present only on unicode objects:\n\nunicode.isnumeric()\n\n Return ``True`` if there are only numeric characters in S,\n ``False`` otherwise. Numeric characters include digit characters,\n and all characters that have the Unicode numeric value property,\n e.g. U+2155, VULGAR FRACTION ONE FIFTH.\n\nunicode.isdecimal()\n\n Return ``True`` if there are only decimal characters in S,\n ``False`` otherwise. Decimal characters include digit characters,\n and all characters that that can be used to form decimal-radix\n numbers, e.g. U+0660, ARABIC-INDIC DIGIT ZERO.\n', - 'strings': u'\nString literals\n***************\n\nString literals are described by the following lexical definitions:\n\n stringliteral ::= [stringprefix](shortstring | longstring)\n stringprefix ::= "r" | "u" | "ur" | "R" | "U" | "UR" | "Ur" | "uR"\n shortstring ::= "\'" shortstringitem* "\'" | \'"\' shortstringitem* \'"\'\n longstring ::= "\'\'\'" longstringitem* "\'\'\'"\n | \'"""\' longstringitem* \'"""\'\n shortstringitem ::= shortstringchar | escapeseq\n longstringitem ::= longstringchar | escapeseq\n shortstringchar ::= <any source character except "\\" or newline or the quote>\n longstringchar ::= <any source character except "\\">\n escapeseq ::= "\\" <any ASCII character>\n\nOne syntactic restriction not indicated by these productions is that\nwhitespace is not allowed between the **stringprefix** and the rest of\nthe string literal. The source character set is defined by the\nencoding declaration; it is ASCII if no encoding declaration is given\nin the source file; see section *Encoding declarations*.\n\nIn plain English: String literals can be enclosed in matching single\nquotes (``\'``) or double quotes (``"``). They can also be enclosed in\nmatching groups of three single or double quotes (these are generally\nreferred to as *triple-quoted strings*). The backslash (``\\``)\ncharacter is used to escape characters that otherwise have a special\nmeaning, such as newline, backslash itself, or the quote character.\nString literals may optionally be prefixed with a letter ``\'r\'`` or\n``\'R\'``; such strings are called *raw strings* and use different rules\nfor interpreting backslash escape sequences. A prefix of ``\'u\'`` or\n``\'U\'`` makes the string a Unicode string. Unicode strings use the\nUnicode character set as defined by the Unicode Consortium and ISO\n10646. Some additional escape sequences, described below, are\navailable in Unicode strings. The two prefix characters may be\ncombined; in this case, ``\'u\'`` must appear before ``\'r\'``.\n\nIn triple-quoted strings, unescaped newlines and quotes are allowed\n(and are retained), except that three unescaped quotes in a row\nterminate the string. (A "quote" is the character used to open the\nstring, i.e. either ``\'`` or ``"``.)\n\nUnless an ``\'r\'`` or ``\'R\'`` prefix is present, escape sequences in\nstrings are interpreted according to rules similar to those used by\nStandard C. The recognized escape sequences are:\n\n+-------------------+-----------------------------------+---------+\n| Escape Sequence | Meaning | Notes |\n+===================+===================================+=========+\n| ``\\newline`` | Ignored | |\n+-------------------+-----------------------------------+---------+\n| ``\\\\`` | Backslash (``\\``) | |\n+-------------------+-----------------------------------+---------+\n| ``\\\'`` | Single quote (``\'``) | |\n+-------------------+-----------------------------------+---------+\n| ``\\"`` | Double quote (``"``) | |\n+-------------------+-----------------------------------+---------+\n| ``\\a`` | ASCII Bell (BEL) | |\n+-------------------+-----------------------------------+---------+\n| ``\\b`` | ASCII Backspace (BS) | |\n+-------------------+-----------------------------------+---------+\n| ``\\f`` | ASCII Formfeed (FF) | |\n+-------------------+-----------------------------------+---------+\n| ``\\n`` | ASCII Linefeed (LF) | |\n+-------------------+-----------------------------------+---------+\n| ``\\N{name}`` | Character named *name* in the | |\n| | Unicode database (Unicode only) | |\n+-------------------+-----------------------------------+---------+\n| ``\\r`` | ASCII Carriage Return (CR) | |\n+-------------------+-----------------------------------+---------+\n| ``\\t`` | ASCII Horizontal Tab (TAB) | |\n+-------------------+-----------------------------------+---------+\n| ``\\uxxxx`` | Character with 16-bit hex value | (1) |\n| | *xxxx* (Unicode only) | |\n+-------------------+-----------------------------------+---------+\n| ``\\Uxxxxxxxx`` | Character with 32-bit hex value | (2) |\n| | *xxxxxxxx* (Unicode only) | |\n+-------------------+-----------------------------------+---------+\n| ``\\v`` | ASCII Vertical Tab (VT) | |\n+-------------------+-----------------------------------+---------+\n| ``\\ooo`` | Character with octal value *ooo* | (3,5) |\n+-------------------+-----------------------------------+---------+\n| ``\\xhh`` | Character with hex value *hh* | (4,5) |\n+-------------------+-----------------------------------+---------+\n\nNotes:\n\n1. Individual code units which form parts of a surrogate pair can be\n encoded using this escape sequence.\n\n2. Any Unicode character can be encoded this way, but characters\n outside the Basic Multilingual Plane (BMP) will be encoded using a\n surrogate pair if Python is compiled to use 16-bit code units (the\n default). Individual code units which form parts of a surrogate\n pair can be encoded using this escape sequence.\n\n3. As in Standard C, up to three octal digits are accepted.\n\n4. Unlike in Standard C, exactly two hex digits are required.\n\n5. In a string literal, hexadecimal and octal escapes denote the byte\n with the given value; it is not necessary that the byte encodes a\n character in the source character set. In a Unicode literal, these\n escapes denote a Unicode character with the given value.\n\nUnlike Standard C, all unrecognized escape sequences are left in the\nstring unchanged, i.e., *the backslash is left in the string*. (This\nbehavior is useful when debugging: if an escape sequence is mistyped,\nthe resulting output is more easily recognized as broken.) It is also\nimportant to note that the escape sequences marked as "(Unicode only)"\nin the table above fall into the category of unrecognized escapes for\nnon-Unicode string literals.\n\nWhen an ``\'r\'`` or ``\'R\'`` prefix is present, a character following a\nbackslash is included in the string without change, and *all\nbackslashes are left in the string*. For example, the string literal\n``r"\\n"`` consists of two characters: a backslash and a lowercase\n``\'n\'``. String quotes can be escaped with a backslash, but the\nbackslash remains in the string; for example, ``r"\\""`` is a valid\nstring literal consisting of two characters: a backslash and a double\nquote; ``r"\\"`` is not a valid string literal (even a raw string\ncannot end in an odd number of backslashes). Specifically, *a raw\nstring cannot end in a single backslash* (since the backslash would\nescape the following quote character). Note also that a single\nbackslash followed by a newline is interpreted as those two characters\nas part of the string, *not* as a line continuation.\n\nWhen an ``\'r\'`` or ``\'R\'`` prefix is used in conjunction with a\n``\'u\'`` or ``\'U\'`` prefix, then the ``\\uXXXX`` and ``\\UXXXXXXXX``\nescape sequences are processed while *all other backslashes are left\nin the string*. For example, the string literal ``ur"\\u0062\\n"``\nconsists of three Unicode characters: \'LATIN SMALL LETTER B\', \'REVERSE\nSOLIDUS\', and \'LATIN SMALL LETTER N\'. Backslashes can be escaped with\na preceding backslash; however, both remain in the string. As a\nresult, ``\\uXXXX`` escape sequences are only recognized when there are\nan odd number of backslashes.\n', + 'string-methods': u'\nString Methods\n**************\n\nBelow are listed the string methods which both 8-bit strings and\nUnicode objects support. Some of them are also available on\n``bytearray`` objects.\n\nIn addition, Python\'s strings support the sequence type methods\ndescribed in the *Sequence Types --- str, unicode, list, tuple,\nbytearray, buffer, xrange* section. To output formatted strings use\ntemplate strings or the ``%`` operator described in the *String\nFormatting Operations* section. Also, see the ``re`` module for string\nfunctions based on regular expressions.\n\nstr.capitalize()\n\n Return a copy of the string with its first character capitalized\n and the rest lowercased.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.center(width[, fillchar])\n\n Return centered in a string of length *width*. Padding is done\n using the specified *fillchar* (default is a space).\n\n Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.count(sub[, start[, end]])\n\n Return the number of non-overlapping occurrences of substring *sub*\n in the range [*start*, *end*]. Optional arguments *start* and\n *end* are interpreted as in slice notation.\n\nstr.decode([encoding[, errors]])\n\n Decodes the string using the codec registered for *encoding*.\n *encoding* defaults to the default string encoding. *errors* may\n be given to set a different error handling scheme. The default is\n ``\'strict\'``, meaning that encoding errors raise ``UnicodeError``.\n Other possible values are ``\'ignore\'``, ``\'replace\'`` and any other\n name registered via ``codecs.register_error()``, see section *Codec\n Base Classes*.\n\n New in version 2.2.\n\n Changed in version 2.3: Support for other error handling schemes\n added.\n\n Changed in version 2.7: Support for keyword arguments added.\n\nstr.encode([encoding[, errors]])\n\n Return an encoded version of the string. Default encoding is the\n current default string encoding. *errors* may be given to set a\n different error handling scheme. The default for *errors* is\n ``\'strict\'``, meaning that encoding errors raise a\n ``UnicodeError``. Other possible values are ``\'ignore\'``,\n ``\'replace\'``, ``\'xmlcharrefreplace\'``, ``\'backslashreplace\'`` and\n any other name registered via ``codecs.register_error()``, see\n section *Codec Base Classes*. For a list of possible encodings, see\n section *Standard Encodings*.\n\n New in version 2.0.\n\n Changed in version 2.3: Support for ``\'xmlcharrefreplace\'`` and\n ``\'backslashreplace\'`` and other error handling schemes added.\n\n Changed in version 2.7: Support for keyword arguments added.\n\nstr.endswith(suffix[, start[, end]])\n\n Return ``True`` if the string ends with the specified *suffix*,\n otherwise return ``False``. *suffix* can also be a tuple of\n suffixes to look for. With optional *start*, test beginning at\n that position. With optional *end*, stop comparing at that\n position.\n\n Changed in version 2.5: Accept tuples as *suffix*.\n\nstr.expandtabs([tabsize])\n\n Return a copy of the string where all tab characters are replaced\n by one or more spaces, depending on the current column and the\n given tab size. The column number is reset to zero after each\n newline occurring in the string. If *tabsize* is not given, a tab\n size of ``8`` characters is assumed. This doesn\'t understand other\n non-printing characters or escape sequences.\n\nstr.find(sub[, start[, end]])\n\n Return the lowest index in the string where substring *sub* is\n found, such that *sub* is contained in the slice ``s[start:end]``.\n Optional arguments *start* and *end* are interpreted as in slice\n notation. Return ``-1`` if *sub* is not found.\n\n Note: The ``find()`` method should be used only if you need to know the\n position of *sub*. To check if *sub* is a substring or not, use\n the ``in`` operator:\n\n >>> \'Py\' in \'Python\'\n True\n\nstr.format(*args, **kwargs)\n\n Perform a string formatting operation. The string on which this\n method is called can contain literal text or replacement fields\n delimited by braces ``{}``. Each replacement field contains either\n the numeric index of a positional argument, or the name of a\n keyword argument. Returns a copy of the string where each\n replacement field is replaced with the string value of the\n corresponding argument.\n\n >>> "The sum of 1 + 2 is {0}".format(1+2)\n \'The sum of 1 + 2 is 3\'\n\n See *Format String Syntax* for a description of the various\n formatting options that can be specified in format strings.\n\n This method of string formatting is the new standard in Python 3.0,\n and should be preferred to the ``%`` formatting described in\n *String Formatting Operations* in new code.\n\n New in version 2.6.\n\nstr.index(sub[, start[, end]])\n\n Like ``find()``, but raise ``ValueError`` when the substring is not\n found.\n\nstr.isalnum()\n\n Return true if all characters in the string are alphanumeric and\n there is at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isalpha()\n\n Return true if all characters in the string are alphabetic and\n there is at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isdigit()\n\n Return true if all characters in the string are digits and there is\n at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.islower()\n\n Return true if all cased characters in the string are lowercase and\n there is at least one cased character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isspace()\n\n Return true if there are only whitespace characters in the string\n and there is at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.istitle()\n\n Return true if the string is a titlecased string and there is at\n least one character, for example uppercase characters may only\n follow uncased characters and lowercase characters only cased ones.\n Return false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isupper()\n\n Return true if all cased characters in the string are uppercase and\n there is at least one cased character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.join(iterable)\n\n Return a string which is the concatenation of the strings in the\n *iterable* *iterable*. The separator between elements is the\n string providing this method.\n\nstr.ljust(width[, fillchar])\n\n Return the string left justified in a string of length *width*.\n Padding is done using the specified *fillchar* (default is a\n space). The original string is returned if *width* is less than\n ``len(s)``.\n\n Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.lower()\n\n Return a copy of the string converted to lowercase.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.lstrip([chars])\n\n Return a copy of the string with leading characters removed. The\n *chars* argument is a string specifying the set of characters to be\n removed. If omitted or ``None``, the *chars* argument defaults to\n removing whitespace. The *chars* argument is not a prefix; rather,\n all combinations of its values are stripped:\n\n >>> \' spacious \'.lstrip()\n \'spacious \'\n >>> \'www.example.com\'.lstrip(\'cmowz.\')\n \'example.com\'\n\n Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.partition(sep)\n\n Split the string at the first occurrence of *sep*, and return a\n 3-tuple containing the part before the separator, the separator\n itself, and the part after the separator. If the separator is not\n found, return a 3-tuple containing the string itself, followed by\n two empty strings.\n\n New in version 2.5.\n\nstr.replace(old, new[, count])\n\n Return a copy of the string with all occurrences of substring *old*\n replaced by *new*. If the optional argument *count* is given, only\n the first *count* occurrences are replaced.\n\nstr.rfind(sub[, start[, end]])\n\n Return the highest index in the string where substring *sub* is\n found, such that *sub* is contained within ``s[start:end]``.\n Optional arguments *start* and *end* are interpreted as in slice\n notation. Return ``-1`` on failure.\n\nstr.rindex(sub[, start[, end]])\n\n Like ``rfind()`` but raises ``ValueError`` when the substring *sub*\n is not found.\n\nstr.rjust(width[, fillchar])\n\n Return the string right justified in a string of length *width*.\n Padding is done using the specified *fillchar* (default is a\n space). The original string is returned if *width* is less than\n ``len(s)``.\n\n Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.rpartition(sep)\n\n Split the string at the last occurrence of *sep*, and return a\n 3-tuple containing the part before the separator, the separator\n itself, and the part after the separator. If the separator is not\n found, return a 3-tuple containing two empty strings, followed by\n the string itself.\n\n New in version 2.5.\n\nstr.rsplit([sep[, maxsplit]])\n\n Return a list of the words in the string, using *sep* as the\n delimiter string. If *maxsplit* is given, at most *maxsplit* splits\n are done, the *rightmost* ones. If *sep* is not specified or\n ``None``, any whitespace string is a separator. Except for\n splitting from the right, ``rsplit()`` behaves like ``split()``\n which is described in detail below.\n\n New in version 2.4.\n\nstr.rstrip([chars])\n\n Return a copy of the string with trailing characters removed. The\n *chars* argument is a string specifying the set of characters to be\n removed. If omitted or ``None``, the *chars* argument defaults to\n removing whitespace. The *chars* argument is not a suffix; rather,\n all combinations of its values are stripped:\n\n >>> \' spacious \'.rstrip()\n \' spacious\'\n >>> \'mississippi\'.rstrip(\'ipz\')\n \'mississ\'\n\n Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.split([sep[, maxsplit]])\n\n Return a list of the words in the string, using *sep* as the\n delimiter string. If *maxsplit* is given, at most *maxsplit*\n splits are done (thus, the list will have at most ``maxsplit+1``\n elements). If *maxsplit* is not specified, then there is no limit\n on the number of splits (all possible splits are made).\n\n If *sep* is given, consecutive delimiters are not grouped together\n and are deemed to delimit empty strings (for example,\n ``\'1,,2\'.split(\',\')`` returns ``[\'1\', \'\', \'2\']``). The *sep*\n argument may consist of multiple characters (for example,\n ``\'1<>2<>3\'.split(\'<>\')`` returns ``[\'1\', \'2\', \'3\']``). Splitting\n an empty string with a specified separator returns ``[\'\']``.\n\n If *sep* is not specified or is ``None``, a different splitting\n algorithm is applied: runs of consecutive whitespace are regarded\n as a single separator, and the result will contain no empty strings\n at the start or end if the string has leading or trailing\n whitespace. Consequently, splitting an empty string or a string\n consisting of just whitespace with a ``None`` separator returns\n ``[]``.\n\n For example, ``\' 1 2 3 \'.split()`` returns ``[\'1\', \'2\', \'3\']``,\n and ``\' 1 2 3 \'.split(None, 1)`` returns ``[\'1\', \'2 3 \']``.\n\nstr.splitlines([keepends])\n\n Return a list of the lines in the string, breaking at line\n boundaries. Line breaks are not included in the resulting list\n unless *keepends* is given and true.\n\nstr.startswith(prefix[, start[, end]])\n\n Return ``True`` if string starts with the *prefix*, otherwise\n return ``False``. *prefix* can also be a tuple of prefixes to look\n for. With optional *start*, test string beginning at that\n position. With optional *end*, stop comparing string at that\n position.\n\n Changed in version 2.5: Accept tuples as *prefix*.\n\nstr.strip([chars])\n\n Return a copy of the string with the leading and trailing\n characters removed. The *chars* argument is a string specifying the\n set of characters to be removed. If omitted or ``None``, the\n *chars* argument defaults to removing whitespace. The *chars*\n argument is not a prefix or suffix; rather, all combinations of its\n values are stripped:\n\n >>> \' spacious \'.strip()\n \'spacious\'\n >>> \'www.example.com\'.strip(\'cmowz.\')\n \'example\'\n\n Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.swapcase()\n\n Return a copy of the string with uppercase characters converted to\n lowercase and vice versa.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.title()\n\n Return a titlecased version of the string where words start with an\n uppercase character and the remaining characters are lowercase.\n\n The algorithm uses a simple language-independent definition of a\n word as groups of consecutive letters. The definition works in\n many contexts but it means that apostrophes in contractions and\n possessives form word boundaries, which may not be the desired\n result:\n\n >>> "they\'re bill\'s friends from the UK".title()\n "They\'Re Bill\'S Friends From The Uk"\n\n A workaround for apostrophes can be constructed using regular\n expressions:\n\n >>> import re\n >>> def titlecase(s):\n return re.sub(r"[A-Za-z]+(\'[A-Za-z]+)?",\n lambda mo: mo.group(0)[0].upper() +\n mo.group(0)[1:].lower(),\n s)\n\n >>> titlecase("they\'re bill\'s friends.")\n "They\'re Bill\'s Friends."\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.translate(table[, deletechars])\n\n Return a copy of the string where all characters occurring in the\n optional argument *deletechars* are removed, and the remaining\n characters have been mapped through the given translation table,\n which must be a string of length 256.\n\n You can use the ``maketrans()`` helper function in the ``string``\n module to create a translation table. For string objects, set the\n *table* argument to ``None`` for translations that only delete\n characters:\n\n >>> \'read this short text\'.translate(None, \'aeiou\')\n \'rd ths shrt txt\'\n\n New in version 2.6: Support for a ``None`` *table* argument.\n\n For Unicode objects, the ``translate()`` method does not accept the\n optional *deletechars* argument. Instead, it returns a copy of the\n *s* where all characters have been mapped through the given\n translation table which must be a mapping of Unicode ordinals to\n Unicode ordinals, Unicode strings or ``None``. Unmapped characters\n are left untouched. Characters mapped to ``None`` are deleted.\n Note, a more flexible approach is to create a custom character\n mapping codec using the ``codecs`` module (see ``encodings.cp1251``\n for an example).\n\nstr.upper()\n\n Return a copy of the string converted to uppercase.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.zfill(width)\n\n Return the numeric string left filled with zeros in a string of\n length *width*. A sign prefix is handled correctly. The original\n string is returned if *width* is less than ``len(s)``.\n\n New in version 2.2.2.\n\nThe following methods are present only on unicode objects:\n\nunicode.isnumeric()\n\n Return ``True`` if there are only numeric characters in S,\n ``False`` otherwise. Numeric characters include digit characters,\n and all characters that have the Unicode numeric value property,\n e.g. U+2155, VULGAR FRACTION ONE FIFTH.\n\nunicode.isdecimal()\n\n Return ``True`` if there are only decimal characters in S,\n ``False`` otherwise. Decimal characters include digit characters,\n and all characters that that can be used to form decimal-radix\n numbers, e.g. U+0660, ARABIC-INDIC DIGIT ZERO.\n', + 'strings': u'\nString literals\n***************\n\nString literals are described by the following lexical definitions:\n\n stringliteral ::= [stringprefix](shortstring | longstring)\n stringprefix ::= "r" | "u" | "ur" | "R" | "U" | "UR" | "Ur" | "uR"\n | "b" | "B" | "br" | "Br" | "bR" | "BR"\n shortstring ::= "\'" shortstringitem* "\'" | \'"\' shortstringitem* \'"\'\n longstring ::= "\'\'\'" longstringitem* "\'\'\'"\n | \'"""\' longstringitem* \'"""\'\n shortstringitem ::= shortstringchar | escapeseq\n longstringitem ::= longstringchar | escapeseq\n shortstringchar ::= <any source character except "\\" or newline or the quote>\n longstringchar ::= <any source character except "\\">\n escapeseq ::= "\\" <any ASCII character>\n\nOne syntactic restriction not indicated by these productions is that\nwhitespace is not allowed between the **stringprefix** and the rest of\nthe string literal. The source character set is defined by the\nencoding declaration; it is ASCII if no encoding declaration is given\nin the source file; see section *Encoding declarations*.\n\nIn plain English: String literals can be enclosed in matching single\nquotes (``\'``) or double quotes (``"``). They can also be enclosed in\nmatching groups of three single or double quotes (these are generally\nreferred to as *triple-quoted strings*). The backslash (``\\``)\ncharacter is used to escape characters that otherwise have a special\nmeaning, such as newline, backslash itself, or the quote character.\nString literals may optionally be prefixed with a letter ``\'r\'`` or\n``\'R\'``; such strings are called *raw strings* and use different rules\nfor interpreting backslash escape sequences. A prefix of ``\'u\'`` or\n``\'U\'`` makes the string a Unicode string. Unicode strings use the\nUnicode character set as defined by the Unicode Consortium and ISO\n10646. Some additional escape sequences, described below, are\navailable in Unicode strings. A prefix of ``\'b\'`` or ``\'B\'`` is\nignored in Python 2; it indicates that the literal should become a\nbytes literal in Python 3 (e.g. when code is automatically converted\nwith 2to3). A ``\'u\'`` or ``\'b\'`` prefix may be followed by an ``\'r\'``\nprefix.\n\nIn triple-quoted strings, unescaped newlines and quotes are allowed\n(and are retained), except that three unescaped quotes in a row\nterminate the string. (A "quote" is the character used to open the\nstring, i.e. either ``\'`` or ``"``.)\n\nUnless an ``\'r\'`` or ``\'R\'`` prefix is present, escape sequences in\nstrings are interpreted according to rules similar to those used by\nStandard C. The recognized escape sequences are:\n\n+-------------------+-----------------------------------+---------+\n| Escape Sequence | Meaning | Notes |\n+===================+===================================+=========+\n| ``\\newline`` | Ignored | |\n+-------------------+-----------------------------------+---------+\n| ``\\\\`` | Backslash (``\\``) | |\n+-------------------+-----------------------------------+---------+\n| ``\\\'`` | Single quote (``\'``) | |\n+-------------------+-----------------------------------+---------+\n| ``\\"`` | Double quote (``"``) | |\n+-------------------+-----------------------------------+---------+\n| ``\\a`` | ASCII Bell (BEL) | |\n+-------------------+-----------------------------------+---------+\n| ``\\b`` | ASCII Backspace (BS) | |\n+-------------------+-----------------------------------+---------+\n| ``\\f`` | ASCII Formfeed (FF) | |\n+-------------------+-----------------------------------+---------+\n| ``\\n`` | ASCII Linefeed (LF) | |\n+-------------------+-----------------------------------+---------+\n| ``\\N{name}`` | Character named *name* in the | |\n| | Unicode database (Unicode only) | |\n+-------------------+-----------------------------------+---------+\n| ``\\r`` | ASCII Carriage Return (CR) | |\n+-------------------+-----------------------------------+---------+\n| ``\\t`` | ASCII Horizontal Tab (TAB) | |\n+-------------------+-----------------------------------+---------+\n| ``\\uxxxx`` | Character with 16-bit hex value | (1) |\n| | *xxxx* (Unicode only) | |\n+-------------------+-----------------------------------+---------+\n| ``\\Uxxxxxxxx`` | Character with 32-bit hex value | (2) |\n| | *xxxxxxxx* (Unicode only) | |\n+-------------------+-----------------------------------+---------+\n| ``\\v`` | ASCII Vertical Tab (VT) | |\n+-------------------+-----------------------------------+---------+\n| ``\\ooo`` | Character with octal value *ooo* | (3,5) |\n+-------------------+-----------------------------------+---------+\n| ``\\xhh`` | Character with hex value *hh* | (4,5) |\n+-------------------+-----------------------------------+---------+\n\nNotes:\n\n1. Individual code units which form parts of a surrogate pair can be\n encoded using this escape sequence.\n\n2. Any Unicode character can be encoded this way, but characters\n outside the Basic Multilingual Plane (BMP) will be encoded using a\n surrogate pair if Python is compiled to use 16-bit code units (the\n default). Individual code units which form parts of a surrogate\n pair can be encoded using this escape sequence.\n\n3. As in Standard C, up to three octal digits are accepted.\n\n4. Unlike in Standard C, exactly two hex digits are required.\n\n5. In a string literal, hexadecimal and octal escapes denote the byte\n with the given value; it is not necessary that the byte encodes a\n character in the source character set. In a Unicode literal, these\n escapes denote a Unicode character with the given value.\n\nUnlike Standard C, all unrecognized escape sequences are left in the\nstring unchanged, i.e., *the backslash is left in the string*. (This\nbehavior is useful when debugging: if an escape sequence is mistyped,\nthe resulting output is more easily recognized as broken.) It is also\nimportant to note that the escape sequences marked as "(Unicode only)"\nin the table above fall into the category of unrecognized escapes for\nnon-Unicode string literals.\n\nWhen an ``\'r\'`` or ``\'R\'`` prefix is present, a character following a\nbackslash is included in the string without change, and *all\nbackslashes are left in the string*. For example, the string literal\n``r"\\n"`` consists of two characters: a backslash and a lowercase\n``\'n\'``. String quotes can be escaped with a backslash, but the\nbackslash remains in the string; for example, ``r"\\""`` is a valid\nstring literal consisting of two characters: a backslash and a double\nquote; ``r"\\"`` is not a valid string literal (even a raw string\ncannot end in an odd number of backslashes). Specifically, *a raw\nstring cannot end in a single backslash* (since the backslash would\nescape the following quote character). Note also that a single\nbackslash followed by a newline is interpreted as those two characters\nas part of the string, *not* as a line continuation.\n\nWhen an ``\'r\'`` or ``\'R\'`` prefix is used in conjunction with a\n``\'u\'`` or ``\'U\'`` prefix, then the ``\\uXXXX`` and ``\\UXXXXXXXX``\nescape sequences are processed while *all other backslashes are left\nin the string*. For example, the string literal ``ur"\\u0062\\n"``\nconsists of three Unicode characters: \'LATIN SMALL LETTER B\', \'REVERSE\nSOLIDUS\', and \'LATIN SMALL LETTER N\'. Backslashes can be escaped with\na preceding backslash; however, both remain in the string. As a\nresult, ``\\uXXXX`` escape sequences are only recognized when there are\nan odd number of backslashes.\n', 'subscriptions': u'\nSubscriptions\n*************\n\nA subscription selects an item of a sequence (string, tuple or list)\nor mapping (dictionary) object:\n\n subscription ::= primary "[" expression_list "]"\n\nThe primary must evaluate to an object of a sequence or mapping type.\n\nIf the primary is a mapping, the expression list must evaluate to an\nobject whose value is one of the keys of the mapping, and the\nsubscription selects the value in the mapping that corresponds to that\nkey. (The expression list is a tuple except if it has exactly one\nitem.)\n\nIf the primary is a sequence, the expression (list) must evaluate to a\nplain integer. If this value is negative, the length of the sequence\nis added to it (so that, e.g., ``x[-1]`` selects the last item of\n``x``.) The resulting value must be a nonnegative integer less than\nthe number of items in the sequence, and the subscription selects the\nitem whose index is that value (counting from zero).\n\nA string\'s items are characters. A character is not a separate data\ntype but a string of exactly one character.\n', 'truth': u"\nTruth Value Testing\n*******************\n\nAny object can be tested for truth value, for use in an ``if`` or\n``while`` condition or as operand of the Boolean operations below. The\nfollowing values are considered false:\n\n* ``None``\n\n* ``False``\n\n* zero of any numeric type, for example, ``0``, ``0L``, ``0.0``,\n ``0j``.\n\n* any empty sequence, for example, ``''``, ``()``, ``[]``.\n\n* any empty mapping, for example, ``{}``.\n\n* instances of user-defined classes, if the class defines a\n ``__nonzero__()`` or ``__len__()`` method, when that method returns\n the integer zero or ``bool`` value ``False``. [1]\n\nAll other values are considered true --- so objects of many types are\nalways true.\n\nOperations and built-in functions that have a Boolean result always\nreturn ``0`` or ``False`` for false and ``1`` or ``True`` for true,\nunless otherwise stated. (Important exception: the Boolean operations\n``or`` and ``and`` always return one of their operands.)\n", 'try': u'\nThe ``try`` statement\n*********************\n\nThe ``try`` statement specifies exception handlers and/or cleanup code\nfor a group of statements:\n\n try_stmt ::= try1_stmt | try2_stmt\n try1_stmt ::= "try" ":" suite\n ("except" [expression [("as" | ",") target]] ":" suite)+\n ["else" ":" suite]\n ["finally" ":" suite]\n try2_stmt ::= "try" ":" suite\n "finally" ":" suite\n\nChanged in version 2.5: In previous versions of Python,\n``try``...``except``...``finally`` did not work. ``try``...``except``\nhad to be nested in ``try``...``finally``.\n\nThe ``except`` clause(s) specify one or more exception handlers. When\nno exception occurs in the ``try`` clause, no exception handler is\nexecuted. When an exception occurs in the ``try`` suite, a search for\nan exception handler is started. This search inspects the except\nclauses in turn until one is found that matches the exception. An\nexpression-less except clause, if present, must be last; it matches\nany exception. For an except clause with an expression, that\nexpression is evaluated, and the clause matches the exception if the\nresulting object is "compatible" with the exception. An object is\ncompatible with an exception if it is the class or a base class of the\nexception object, a tuple containing an item compatible with the\nexception, or, in the (deprecated) case of string exceptions, is the\nraised string itself (note that the object identities must match, i.e.\nit must be the same string object, not just a string with the same\nvalue).\n\nIf no except clause matches the exception, the search for an exception\nhandler continues in the surrounding code and on the invocation stack.\n[1]\n\nIf the evaluation of an expression in the header of an except clause\nraises an exception, the original search for a handler is canceled and\na search starts for the new exception in the surrounding code and on\nthe call stack (it is treated as if the entire ``try`` statement\nraised the exception).\n\nWhen a matching except clause is found, the exception is assigned to\nthe target specified in that except clause, if present, and the except\nclause\'s suite is executed. All except clauses must have an\nexecutable block. When the end of this block is reached, execution\ncontinues normally after the entire try statement. (This means that\nif two nested handlers exist for the same exception, and the exception\noccurs in the try clause of the inner handler, the outer handler will\nnot handle the exception.)\n\nBefore an except clause\'s suite is executed, details about the\nexception are assigned to three variables in the ``sys`` module:\n``sys.exc_type`` receives the object identifying the exception;\n``sys.exc_value`` receives the exception\'s parameter;\n``sys.exc_traceback`` receives a traceback object (see section *The\nstandard type hierarchy*) identifying the point in the program where\nthe exception occurred. These details are also available through the\n``sys.exc_info()`` function, which returns a tuple ``(exc_type,\nexc_value, exc_traceback)``. Use of the corresponding variables is\ndeprecated in favor of this function, since their use is unsafe in a\nthreaded program. As of Python 1.5, the variables are restored to\ntheir previous values (before the call) when returning from a function\nthat handled an exception.\n\nThe optional ``else`` clause is executed if and when control flows off\nthe end of the ``try`` clause. [2] Exceptions in the ``else`` clause\nare not handled by the preceding ``except`` clauses.\n\nIf ``finally`` is present, it specifies a \'cleanup\' handler. The\n``try`` clause is executed, including any ``except`` and ``else``\nclauses. If an exception occurs in any of the clauses and is not\nhandled, the exception is temporarily saved. The ``finally`` clause is\nexecuted. If there is a saved exception, it is re-raised at the end\nof the ``finally`` clause. If the ``finally`` clause raises another\nexception or executes a ``return`` or ``break`` statement, the saved\nexception is lost. The exception information is not available to the\nprogram during execution of the ``finally`` clause.\n\nWhen a ``return``, ``break`` or ``continue`` statement is executed in\nthe ``try`` suite of a ``try``...``finally`` statement, the\n``finally`` clause is also executed \'on the way out.\' A ``continue``\nstatement is illegal in the ``finally`` clause. (The reason is a\nproblem with the current implementation --- this restriction may be\nlifted in the future).\n\nAdditional information on exceptions can be found in section\n*Exceptions*, and information on using the ``raise`` statement to\ngenerate exceptions may be found in section *The raise statement*.\n', - 'types': u'\nThe standard type hierarchy\n***************************\n\nBelow is a list of the types that are built into Python. Extension\nmodules (written in C, Java, or other languages, depending on the\nimplementation) can define additional types. Future versions of\nPython may add types to the type hierarchy (e.g., rational numbers,\nefficiently stored arrays of integers, etc.).\n\nSome of the type descriptions below contain a paragraph listing\n\'special attributes.\' These are attributes that provide access to the\nimplementation and are not intended for general use. Their definition\nmay change in the future.\n\nNone\n This type has a single value. There is a single object with this\n value. This object is accessed through the built-in name ``None``.\n It is used to signify the absence of a value in many situations,\n e.g., it is returned from functions that don\'t explicitly return\n anything. Its truth value is false.\n\nNotImplemented\n This type has a single value. There is a single object with this\n value. This object is accessed through the built-in name\n ``NotImplemented``. Numeric methods and rich comparison methods may\n return this value if they do not implement the operation for the\n operands provided. (The interpreter will then try the reflected\n operation, or some other fallback, depending on the operator.) Its\n truth value is true.\n\nEllipsis\n This type has a single value. There is a single object with this\n value. This object is accessed through the built-in name\n ``Ellipsis``. It is used to indicate the presence of the ``...``\n syntax in a slice. Its truth value is true.\n\n``numbers.Number``\n These are created by numeric literals and returned as results by\n arithmetic operators and arithmetic built-in functions. Numeric\n objects are immutable; once created their value never changes.\n Python numbers are of course strongly related to mathematical\n numbers, but subject to the limitations of numerical representation\n in computers.\n\n Python distinguishes between integers, floating point numbers, and\n complex numbers:\n\n ``numbers.Integral``\n These represent elements from the mathematical set of integers\n (positive and negative).\n\n There are three types of integers:\n\n Plain integers\n These represent numbers in the range -2147483648 through\n 2147483647. (The range may be larger on machines with a\n larger natural word size, but not smaller.) When the result\n of an operation would fall outside this range, the result is\n normally returned as a long integer (in some cases, the\n exception ``OverflowError`` is raised instead). For the\n purpose of shift and mask operations, integers are assumed to\n have a binary, 2\'s complement notation using 32 or more bits,\n and hiding no bits from the user (i.e., all 4294967296\n different bit patterns correspond to different values).\n\n Long integers\n These represent numbers in an unlimited range, subject to\n available (virtual) memory only. For the purpose of shift\n and mask operations, a binary representation is assumed, and\n negative numbers are represented in a variant of 2\'s\n complement which gives the illusion of an infinite string of\n sign bits extending to the left.\n\n Booleans\n These represent the truth values False and True. The two\n objects representing the values False and True are the only\n Boolean objects. The Boolean type is a subtype of plain\n integers, and Boolean values behave like the values 0 and 1,\n respectively, in almost all contexts, the exception being\n that when converted to a string, the strings ``"False"`` or\n ``"True"`` are returned, respectively.\n\n The rules for integer representation are intended to give the\n most meaningful interpretation of shift and mask operations\n involving negative integers and the least surprises when\n switching between the plain and long integer domains. Any\n operation, if it yields a result in the plain integer domain,\n will yield the same result in the long integer domain or when\n using mixed operands. The switch between domains is transparent\n to the programmer.\n\n ``numbers.Real`` (``float``)\n These represent machine-level double precision floating point\n numbers. You are at the mercy of the underlying machine\n architecture (and C or Java implementation) for the accepted\n range and handling of overflow. Python does not support single-\n precision floating point numbers; the savings in processor and\n memory usage that are usually the reason for using these is\n dwarfed by the overhead of using objects in Python, so there is\n no reason to complicate the language with two kinds of floating\n point numbers.\n\n ``numbers.Complex``\n These represent complex numbers as a pair of machine-level\n double precision floating point numbers. The same caveats apply\n as for floating point numbers. The real and imaginary parts of a\n complex number ``z`` can be retrieved through the read-only\n attributes ``z.real`` and ``z.imag``.\n\nSequences\n These represent finite ordered sets indexed by non-negative\n numbers. The built-in function ``len()`` returns the number of\n items of a sequence. When the length of a sequence is *n*, the\n index set contains the numbers 0, 1, ..., *n*-1. Item *i* of\n sequence *a* is selected by ``a[i]``.\n\n Sequences also support slicing: ``a[i:j]`` selects all items with\n index *k* such that *i* ``<=`` *k* ``<`` *j*. When used as an\n expression, a slice is a sequence of the same type. This implies\n that the index set is renumbered so that it starts at 0.\n\n Some sequences also support "extended slicing" with a third "step"\n parameter: ``a[i:j:k]`` selects all items of *a* with index *x*\n where ``x = i + n*k``, *n* ``>=`` ``0`` and *i* ``<=`` *x* ``<``\n *j*.\n\n Sequences are distinguished according to their mutability:\n\n Immutable sequences\n An object of an immutable sequence type cannot change once it is\n created. (If the object contains references to other objects,\n these other objects may be mutable and may be changed; however,\n the collection of objects directly referenced by an immutable\n object cannot change.)\n\n The following types are immutable sequences:\n\n Strings\n The items of a string are characters. There is no separate\n character type; a character is represented by a string of one\n item. Characters represent (at least) 8-bit bytes. The\n built-in functions ``chr()`` and ``ord()`` convert between\n characters and nonnegative integers representing the byte\n values. Bytes with the values 0-127 usually represent the\n corresponding ASCII values, but the interpretation of values\n is up to the program. The string data type is also used to\n represent arrays of bytes, e.g., to hold data read from a\n file.\n\n (On systems whose native character set is not ASCII, strings\n may use EBCDIC in their internal representation, provided the\n functions ``chr()`` and ``ord()`` implement a mapping between\n ASCII and EBCDIC, and string comparison preserves the ASCII\n order. Or perhaps someone can propose a better rule?)\n\n Unicode\n The items of a Unicode object are Unicode code units. A\n Unicode code unit is represented by a Unicode object of one\n item and can hold either a 16-bit or 32-bit value\n representing a Unicode ordinal (the maximum value for the\n ordinal is given in ``sys.maxunicode``, and depends on how\n Python is configured at compile time). Surrogate pairs may\n be present in the Unicode object, and will be reported as two\n separate items. The built-in functions ``unichr()`` and\n ``ord()`` convert between code units and nonnegative integers\n representing the Unicode ordinals as defined in the Unicode\n Standard 3.0. Conversion from and to other encodings are\n possible through the Unicode method ``encode()`` and the\n built-in function ``unicode()``.\n\n Tuples\n The items of a tuple are arbitrary Python objects. Tuples of\n two or more items are formed by comma-separated lists of\n expressions. A tuple of one item (a \'singleton\') can be\n formed by affixing a comma to an expression (an expression by\n itself does not create a tuple, since parentheses must be\n usable for grouping of expressions). An empty tuple can be\n formed by an empty pair of parentheses.\n\n Mutable sequences\n Mutable sequences can be changed after they are created. The\n subscription and slicing notations can be used as the target of\n assignment and ``del`` (delete) statements.\n\n There are currently two intrinsic mutable sequence types:\n\n Lists\n The items of a list are arbitrary Python objects. Lists are\n formed by placing a comma-separated list of expressions in\n square brackets. (Note that there are no special cases needed\n to form lists of length 0 or 1.)\n\n Byte Arrays\n A bytearray object is a mutable array. They are created by\n the built-in ``bytearray()`` constructor. Aside from being\n mutable (and hence unhashable), byte arrays otherwise provide\n the same interface and functionality as immutable bytes\n objects.\n\n The extension module ``array`` provides an additional example of\n a mutable sequence type.\n\nSet types\n These represent unordered, finite sets of unique, immutable\n objects. As such, they cannot be indexed by any subscript. However,\n they can be iterated over, and the built-in function ``len()``\n returns the number of items in a set. Common uses for sets are fast\n membership testing, removing duplicates from a sequence, and\n computing mathematical operations such as intersection, union,\n difference, and symmetric difference.\n\n For set elements, the same immutability rules apply as for\n dictionary keys. Note that numeric types obey the normal rules for\n numeric comparison: if two numbers compare equal (e.g., ``1`` and\n ``1.0``), only one of them can be contained in a set.\n\n There are currently two intrinsic set types:\n\n Sets\n These represent a mutable set. They are created by the built-in\n ``set()`` constructor and can be modified afterwards by several\n methods, such as ``add()``.\n\n Frozen sets\n These represent an immutable set. They are created by the\n built-in ``frozenset()`` constructor. As a frozenset is\n immutable and *hashable*, it can be used again as an element of\n another set, or as a dictionary key.\n\nMappings\n These represent finite sets of objects indexed by arbitrary index\n sets. The subscript notation ``a[k]`` selects the item indexed by\n ``k`` from the mapping ``a``; this can be used in expressions and\n as the target of assignments or ``del`` statements. The built-in\n function ``len()`` returns the number of items in a mapping.\n\n There is currently a single intrinsic mapping type:\n\n Dictionaries\n These represent finite sets of objects indexed by nearly\n arbitrary values. The only types of values not acceptable as\n keys are values containing lists or dictionaries or other\n mutable types that are compared by value rather than by object\n identity, the reason being that the efficient implementation of\n dictionaries requires a key\'s hash value to remain constant.\n Numeric types used for keys obey the normal rules for numeric\n comparison: if two numbers compare equal (e.g., ``1`` and\n ``1.0``) then they can be used interchangeably to index the same\n dictionary entry.\n\n Dictionaries are mutable; they can be created by the ``{...}``\n notation (see section *Dictionary displays*).\n\n The extension modules ``dbm``, ``gdbm``, and ``bsddb`` provide\n additional examples of mapping types.\n\nCallable types\n These are the types to which the function call operation (see\n section *Calls*) can be applied:\n\n User-defined functions\n A user-defined function object is created by a function\n definition (see section *Function definitions*). It should be\n called with an argument list containing the same number of items\n as the function\'s formal parameter list.\n\n Special attributes:\n\n +-------------------------+---------------------------------+-------------+\n | Attribute | Meaning | |\n +=========================+=================================+=============+\n | ``func_doc`` | The function\'s documentation | Writable |\n | | string, or ``None`` if | |\n | | unavailable | |\n +-------------------------+---------------------------------+-------------+\n | ``__doc__`` | Another way of spelling | Writable |\n | | ``func_doc`` | |\n +-------------------------+---------------------------------+-------------+\n | ``func_name`` | The function\'s name | Writable |\n +-------------------------+---------------------------------+-------------+\n | ``__name__`` | Another way of spelling | Writable |\n | | ``func_name`` | |\n +-------------------------+---------------------------------+-------------+\n | ``__module__`` | The name of the module the | Writable |\n | | function was defined in, or | |\n | | ``None`` if unavailable. | |\n +-------------------------+---------------------------------+-------------+\n | ``func_defaults`` | A tuple containing default | Writable |\n | | argument values for those | |\n | | arguments that have defaults, | |\n | | or ``None`` if no arguments | |\n | | have a default value | |\n +-------------------------+---------------------------------+-------------+\n | ``func_code`` | The code object representing | Writable |\n | | the compiled function body. | |\n +-------------------------+---------------------------------+-------------+\n | ``func_globals`` | A reference to the dictionary | Read-only |\n | | that holds the function\'s | |\n | | global variables --- the global | |\n | | namespace of the module in | |\n | | which the function was defined. | |\n +-------------------------+---------------------------------+-------------+\n | ``func_dict`` | The namespace supporting | Writable |\n | | arbitrary function attributes. | |\n +-------------------------+---------------------------------+-------------+\n | ``func_closure`` | ``None`` or a tuple of cells | Read-only |\n | | that contain bindings for the | |\n | | function\'s free variables. | |\n +-------------------------+---------------------------------+-------------+\n\n Most of the attributes labelled "Writable" check the type of the\n assigned value.\n\n Changed in version 2.4: ``func_name`` is now writable.\n\n Function objects also support getting and setting arbitrary\n attributes, which can be used, for example, to attach metadata\n to functions. Regular attribute dot-notation is used to get and\n set such attributes. *Note that the current implementation only\n supports function attributes on user-defined functions. Function\n attributes on built-in functions may be supported in the\n future.*\n\n Additional information about a function\'s definition can be\n retrieved from its code object; see the description of internal\n types below.\n\n User-defined methods\n A user-defined method object combines a class, a class instance\n (or ``None``) and any callable object (normally a user-defined\n function).\n\n Special read-only attributes: ``im_self`` is the class instance\n object, ``im_func`` is the function object; ``im_class`` is the\n class of ``im_self`` for bound methods or the class that asked\n for the method for unbound methods; ``__doc__`` is the method\'s\n documentation (same as ``im_func.__doc__``); ``__name__`` is the\n method name (same as ``im_func.__name__``); ``__module__`` is\n the name of the module the method was defined in, or ``None`` if\n unavailable.\n\n Changed in version 2.2: ``im_self`` used to refer to the class\n that defined the method.\n\n Changed in version 2.6: For 3.0 forward-compatibility,\n ``im_func`` is also available as ``__func__``, and ``im_self``\n as ``__self__``.\n\n Methods also support accessing (but not setting) the arbitrary\n function attributes on the underlying function object.\n\n User-defined method objects may be created when getting an\n attribute of a class (perhaps via an instance of that class), if\n that attribute is a user-defined function object, an unbound\n user-defined method object, or a class method object. When the\n attribute is a user-defined method object, a new method object\n is only created if the class from which it is being retrieved is\n the same as, or a derived class of, the class stored in the\n original method object; otherwise, the original method object is\n used as it is.\n\n When a user-defined method object is created by retrieving a\n user-defined function object from a class, its ``im_self``\n attribute is ``None`` and the method object is said to be\n unbound. When one is created by retrieving a user-defined\n function object from a class via one of its instances, its\n ``im_self`` attribute is the instance, and the method object is\n said to be bound. In either case, the new method\'s ``im_class``\n attribute is the class from which the retrieval takes place, and\n its ``im_func`` attribute is the original function object.\n\n When a user-defined method object is created by retrieving\n another method object from a class or instance, the behaviour is\n the same as for a function object, except that the ``im_func``\n attribute of the new instance is not the original method object\n but its ``im_func`` attribute.\n\n When a user-defined method object is created by retrieving a\n class method object from a class or instance, its ``im_self``\n attribute is the class itself (the same as the ``im_class``\n attribute), and its ``im_func`` attribute is the function object\n underlying the class method.\n\n When an unbound user-defined method object is called, the\n underlying function (``im_func``) is called, with the\n restriction that the first argument must be an instance of the\n proper class (``im_class``) or of a derived class thereof.\n\n When a bound user-defined method object is called, the\n underlying function (``im_func``) is called, inserting the class\n instance (``im_self``) in front of the argument list. For\n instance, when ``C`` is a class which contains a definition for\n a function ``f()``, and ``x`` is an instance of ``C``, calling\n ``x.f(1)`` is equivalent to calling ``C.f(x, 1)``.\n\n When a user-defined method object is derived from a class method\n object, the "class instance" stored in ``im_self`` will actually\n be the class itself, so that calling either ``x.f(1)`` or\n ``C.f(1)`` is equivalent to calling ``f(C,1)`` where ``f`` is\n the underlying function.\n\n Note that the transformation from function object to (unbound or\n bound) method object happens each time the attribute is\n retrieved from the class or instance. In some cases, a fruitful\n optimization is to assign the attribute to a local variable and\n call that local variable. Also notice that this transformation\n only happens for user-defined functions; other callable objects\n (and all non-callable objects) are retrieved without\n transformation. It is also important to note that user-defined\n functions which are attributes of a class instance are not\n converted to bound methods; this *only* happens when the\n function is an attribute of the class.\n\n Generator functions\n A function or method which uses the ``yield`` statement (see\n section *The yield statement*) is called a *generator function*.\n Such a function, when called, always returns an iterator object\n which can be used to execute the body of the function: calling\n the iterator\'s ``next()`` method will cause the function to\n execute until it provides a value using the ``yield`` statement.\n When the function executes a ``return`` statement or falls off\n the end, a ``StopIteration`` exception is raised and the\n iterator will have reached the end of the set of values to be\n returned.\n\n Built-in functions\n A built-in function object is a wrapper around a C function.\n Examples of built-in functions are ``len()`` and ``math.sin()``\n (``math`` is a standard built-in module). The number and type of\n the arguments are determined by the C function. Special read-\n only attributes: ``__doc__`` is the function\'s documentation\n string, or ``None`` if unavailable; ``__name__`` is the\n function\'s name; ``__self__`` is set to ``None`` (but see the\n next item); ``__module__`` is the name of the module the\n function was defined in or ``None`` if unavailable.\n\n Built-in methods\n This is really a different disguise of a built-in function, this\n time containing an object passed to the C function as an\n implicit extra argument. An example of a built-in method is\n ``alist.append()``, assuming *alist* is a list object. In this\n case, the special read-only attribute ``__self__`` is set to the\n object denoted by *list*.\n\n Class Types\n Class types, or "new-style classes," are callable. These\n objects normally act as factories for new instances of\n themselves, but variations are possible for class types that\n override ``__new__()``. The arguments of the call are passed to\n ``__new__()`` and, in the typical case, to ``__init__()`` to\n initialize the new instance.\n\n Classic Classes\n Class objects are described below. When a class object is\n called, a new class instance (also described below) is created\n and returned. This implies a call to the class\'s ``__init__()``\n method if it has one. Any arguments are passed on to the\n ``__init__()`` method. If there is no ``__init__()`` method,\n the class must be called without arguments.\n\n Class instances\n Class instances are described below. Class instances are\n callable only when the class has a ``__call__()`` method;\n ``x(arguments)`` is a shorthand for ``x.__call__(arguments)``.\n\nModules\n Modules are imported by the ``import`` statement (see section *The\n import statement*). A module object has a namespace implemented by\n a dictionary object (this is the dictionary referenced by the\n func_globals attribute of functions defined in the module).\n Attribute references are translated to lookups in this dictionary,\n e.g., ``m.x`` is equivalent to ``m.__dict__["x"]``. A module object\n does not contain the code object used to initialize the module\n (since it isn\'t needed once the initialization is done).\n\n Attribute assignment updates the module\'s namespace dictionary,\n e.g., ``m.x = 1`` is equivalent to ``m.__dict__["x"] = 1``.\n\n Special read-only attribute: ``__dict__`` is the module\'s namespace\n as a dictionary object.\n\n Predefined (writable) attributes: ``__name__`` is the module\'s\n name; ``__doc__`` is the module\'s documentation string, or ``None``\n if unavailable; ``__file__`` is the pathname of the file from which\n the module was loaded, if it was loaded from a file. The\n ``__file__`` attribute is not present for C modules that are\n statically linked into the interpreter; for extension modules\n loaded dynamically from a shared library, it is the pathname of the\n shared library file.\n\nClasses\n Both class types (new-style classes) and class objects (old-\n style/classic classes) are typically created by class definitions\n (see section *Class definitions*). A class has a namespace\n implemented by a dictionary object. Class attribute references are\n translated to lookups in this dictionary, e.g., ``C.x`` is\n translated to ``C.__dict__["x"]`` (although for new-style classes\n in particular there are a number of hooks which allow for other\n means of locating attributes). When the attribute name is not found\n there, the attribute search continues in the base classes. For\n old-style classes, the search is depth-first, left-to-right in the\n order of occurrence in the base class list. New-style classes use\n the more complex C3 method resolution order which behaves correctly\n even in the presence of \'diamond\' inheritance structures where\n there are multiple inheritance paths leading back to a common\n ancestor. Additional details on the C3 MRO used by new-style\n classes can be found in the documentation accompanying the 2.3\n release at http://www.python.org/download/releases/2.3/mro/.\n\n When a class attribute reference (for class ``C``, say) would yield\n a user-defined function object or an unbound user-defined method\n object whose associated class is either ``C`` or one of its base\n classes, it is transformed into an unbound user-defined method\n object whose ``im_class`` attribute is ``C``. When it would yield a\n class method object, it is transformed into a bound user-defined\n method object whose ``im_class`` and ``im_self`` attributes are\n both ``C``. When it would yield a static method object, it is\n transformed into the object wrapped by the static method object.\n See section *Implementing Descriptors* for another way in which\n attributes retrieved from a class may differ from those actually\n contained in its ``__dict__`` (note that only new-style classes\n support descriptors).\n\n Class attribute assignments update the class\'s dictionary, never\n the dictionary of a base class.\n\n A class object can be called (see above) to yield a class instance\n (see below).\n\n Special attributes: ``__name__`` is the class name; ``__module__``\n is the module name in which the class was defined; ``__dict__`` is\n the dictionary containing the class\'s namespace; ``__bases__`` is a\n tuple (possibly empty or a singleton) containing the base classes,\n in the order of their occurrence in the base class list;\n ``__doc__`` is the class\'s documentation string, or None if\n undefined.\n\nClass instances\n A class instance is created by calling a class object (see above).\n A class instance has a namespace implemented as a dictionary which\n is the first place in which attribute references are searched.\n When an attribute is not found there, and the instance\'s class has\n an attribute by that name, the search continues with the class\n attributes. If a class attribute is found that is a user-defined\n function object or an unbound user-defined method object whose\n associated class is the class (call it ``C``) of the instance for\n which the attribute reference was initiated or one of its bases, it\n is transformed into a bound user-defined method object whose\n ``im_class`` attribute is ``C`` and whose ``im_self`` attribute is\n the instance. Static method and class method objects are also\n transformed, as if they had been retrieved from class ``C``; see\n above under "Classes". See section *Implementing Descriptors* for\n another way in which attributes of a class retrieved via its\n instances may differ from the objects actually stored in the\n class\'s ``__dict__``. If no class attribute is found, and the\n object\'s class has a ``__getattr__()`` method, that is called to\n satisfy the lookup.\n\n Attribute assignments and deletions update the instance\'s\n dictionary, never a class\'s dictionary. If the class has a\n ``__setattr__()`` or ``__delattr__()`` method, this is called\n instead of updating the instance dictionary directly.\n\n Class instances can pretend to be numbers, sequences, or mappings\n if they have methods with certain special names. See section\n *Special method names*.\n\n Special attributes: ``__dict__`` is the attribute dictionary;\n ``__class__`` is the instance\'s class.\n\nFiles\n A file object represents an open file. File objects are created by\n the ``open()`` built-in function, and also by ``os.popen()``,\n ``os.fdopen()``, and the ``makefile()`` method of socket objects\n (and perhaps by other functions or methods provided by extension\n modules). The objects ``sys.stdin``, ``sys.stdout`` and\n ``sys.stderr`` are initialized to file objects corresponding to the\n interpreter\'s standard input, output and error streams. See *File\n Objects* for complete documentation of file objects.\n\nInternal types\n A few types used internally by the interpreter are exposed to the\n user. Their definitions may change with future versions of the\n interpreter, but they are mentioned here for completeness.\n\n Code objects\n Code objects represent *byte-compiled* executable Python code,\n or *bytecode*. The difference between a code object and a\n function object is that the function object contains an explicit\n reference to the function\'s globals (the module in which it was\n defined), while a code object contains no context; also the\n default argument values are stored in the function object, not\n in the code object (because they represent values calculated at\n run-time). Unlike function objects, code objects are immutable\n and contain no references (directly or indirectly) to mutable\n objects.\n\n Special read-only attributes: ``co_name`` gives the function\n name; ``co_argcount`` is the number of positional arguments\n (including arguments with default values); ``co_nlocals`` is the\n number of local variables used by the function (including\n arguments); ``co_varnames`` is a tuple containing the names of\n the local variables (starting with the argument names);\n ``co_cellvars`` is a tuple containing the names of local\n variables that are referenced by nested functions;\n ``co_freevars`` is a tuple containing the names of free\n variables; ``co_code`` is a string representing the sequence of\n bytecode instructions; ``co_consts`` is a tuple containing the\n literals used by the bytecode; ``co_names`` is a tuple\n containing the names used by the bytecode; ``co_filename`` is\n the filename from which the code was compiled;\n ``co_firstlineno`` is the first line number of the function;\n ``co_lnotab`` is a string encoding the mapping from bytecode\n offsets to line numbers (for details see the source code of the\n interpreter); ``co_stacksize`` is the required stack size\n (including local variables); ``co_flags`` is an integer encoding\n a number of flags for the interpreter.\n\n The following flag bits are defined for ``co_flags``: bit\n ``0x04`` is set if the function uses the ``*arguments`` syntax\n to accept an arbitrary number of positional arguments; bit\n ``0x08`` is set if the function uses the ``**keywords`` syntax\n to accept arbitrary keyword arguments; bit ``0x20`` is set if\n the function is a generator.\n\n Future feature declarations (``from __future__ import\n division``) also use bits in ``co_flags`` to indicate whether a\n code object was compiled with a particular feature enabled: bit\n ``0x2000`` is set if the function was compiled with future\n division enabled; bits ``0x10`` and ``0x1000`` were used in\n earlier versions of Python.\n\n Other bits in ``co_flags`` are reserved for internal use.\n\n If a code object represents a function, the first item in\n ``co_consts`` is the documentation string of the function, or\n ``None`` if undefined.\n\n Frame objects\n Frame objects represent execution frames. They may occur in\n traceback objects (see below).\n\n Special read-only attributes: ``f_back`` is to the previous\n stack frame (towards the caller), or ``None`` if this is the\n bottom stack frame; ``f_code`` is the code object being executed\n in this frame; ``f_locals`` is the dictionary used to look up\n local variables; ``f_globals`` is used for global variables;\n ``f_builtins`` is used for built-in (intrinsic) names;\n ``f_restricted`` is a flag indicating whether the function is\n executing in restricted execution mode; ``f_lasti`` gives the\n precise instruction (this is an index into the bytecode string\n of the code object).\n\n Special writable attributes: ``f_trace``, if not ``None``, is a\n function called at the start of each source code line (this is\n used by the debugger); ``f_exc_type``, ``f_exc_value``,\n ``f_exc_traceback`` represent the last exception raised in the\n parent frame provided another exception was ever raised in the\n current frame (in all other cases they are None); ``f_lineno``\n is the current line number of the frame --- writing to this from\n within a trace function jumps to the given line (only for the\n bottom-most frame). A debugger can implement a Jump command\n (aka Set Next Statement) by writing to f_lineno.\n\n Traceback objects\n Traceback objects represent a stack trace of an exception. A\n traceback object is created when an exception occurs. When the\n search for an exception handler unwinds the execution stack, at\n each unwound level a traceback object is inserted in front of\n the current traceback. When an exception handler is entered,\n the stack trace is made available to the program. (See section\n *The try statement*.) It is accessible as ``sys.exc_traceback``,\n and also as the third item of the tuple returned by\n ``sys.exc_info()``. The latter is the preferred interface,\n since it works correctly when the program is using multiple\n threads. When the program contains no suitable handler, the\n stack trace is written (nicely formatted) to the standard error\n stream; if the interpreter is interactive, it is also made\n available to the user as ``sys.last_traceback``.\n\n Special read-only attributes: ``tb_next`` is the next level in\n the stack trace (towards the frame where the exception\n occurred), or ``None`` if there is no next level; ``tb_frame``\n points to the execution frame of the current level;\n ``tb_lineno`` gives the line number where the exception\n occurred; ``tb_lasti`` indicates the precise instruction. The\n line number and last instruction in the traceback may differ\n from the line number of its frame object if the exception\n occurred in a ``try`` statement with no matching except clause\n or with a finally clause.\n\n Slice objects\n Slice objects are used to represent slices when *extended slice\n syntax* is used. This is a slice using two colons, or multiple\n slices or ellipses separated by commas, e.g., ``a[i:j:step]``,\n ``a[i:j, k:l]``, or ``a[..., i:j]``. They are also created by\n the built-in ``slice()`` function.\n\n Special read-only attributes: ``start`` is the lower bound;\n ``stop`` is the upper bound; ``step`` is the step value; each is\n ``None`` if omitted. These attributes can have any type.\n\n Slice objects support one method:\n\n slice.indices(self, length)\n\n This method takes a single integer argument *length* and\n computes information about the extended slice that the slice\n object would describe if applied to a sequence of *length*\n items. It returns a tuple of three integers; respectively\n these are the *start* and *stop* indices and the *step* or\n stride length of the slice. Missing or out-of-bounds indices\n are handled in a manner consistent with regular slices.\n\n New in version 2.3.\n\n Static method objects\n Static method objects provide a way of defeating the\n transformation of function objects to method objects described\n above. A static method object is a wrapper around any other\n object, usually a user-defined method object. When a static\n method object is retrieved from a class or a class instance, the\n object actually returned is the wrapped object, which is not\n subject to any further transformation. Static method objects are\n not themselves callable, although the objects they wrap usually\n are. Static method objects are created by the built-in\n ``staticmethod()`` constructor.\n\n Class method objects\n A class method object, like a static method object, is a wrapper\n around another object that alters the way in which that object\n is retrieved from classes and class instances. The behaviour of\n class method objects upon such retrieval is described above,\n under "User-defined methods". Class method objects are created\n by the built-in ``classmethod()`` constructor.\n', + 'types': u'\nThe standard type hierarchy\n***************************\n\nBelow is a list of the types that are built into Python. Extension\nmodules (written in C, Java, or other languages, depending on the\nimplementation) can define additional types. Future versions of\nPython may add types to the type hierarchy (e.g., rational numbers,\nefficiently stored arrays of integers, etc.).\n\nSome of the type descriptions below contain a paragraph listing\n\'special attributes.\' These are attributes that provide access to the\nimplementation and are not intended for general use. Their definition\nmay change in the future.\n\nNone\n This type has a single value. There is a single object with this\n value. This object is accessed through the built-in name ``None``.\n It is used to signify the absence of a value in many situations,\n e.g., it is returned from functions that don\'t explicitly return\n anything. Its truth value is false.\n\nNotImplemented\n This type has a single value. There is a single object with this\n value. This object is accessed through the built-in name\n ``NotImplemented``. Numeric methods and rich comparison methods may\n return this value if they do not implement the operation for the\n operands provided. (The interpreter will then try the reflected\n operation, or some other fallback, depending on the operator.) Its\n truth value is true.\n\nEllipsis\n This type has a single value. There is a single object with this\n value. This object is accessed through the built-in name\n ``Ellipsis``. It is used to indicate the presence of the ``...``\n syntax in a slice. Its truth value is true.\n\n``numbers.Number``\n These are created by numeric literals and returned as results by\n arithmetic operators and arithmetic built-in functions. Numeric\n objects are immutable; once created their value never changes.\n Python numbers are of course strongly related to mathematical\n numbers, but subject to the limitations of numerical representation\n in computers.\n\n Python distinguishes between integers, floating point numbers, and\n complex numbers:\n\n ``numbers.Integral``\n These represent elements from the mathematical set of integers\n (positive and negative).\n\n There are three types of integers:\n\n Plain integers\n These represent numbers in the range -2147483648 through\n 2147483647. (The range may be larger on machines with a\n larger natural word size, but not smaller.) When the result\n of an operation would fall outside this range, the result is\n normally returned as a long integer (in some cases, the\n exception ``OverflowError`` is raised instead). For the\n purpose of shift and mask operations, integers are assumed to\n have a binary, 2\'s complement notation using 32 or more bits,\n and hiding no bits from the user (i.e., all 4294967296\n different bit patterns correspond to different values).\n\n Long integers\n These represent numbers in an unlimited range, subject to\n available (virtual) memory only. For the purpose of shift\n and mask operations, a binary representation is assumed, and\n negative numbers are represented in a variant of 2\'s\n complement which gives the illusion of an infinite string of\n sign bits extending to the left.\n\n Booleans\n These represent the truth values False and True. The two\n objects representing the values False and True are the only\n Boolean objects. The Boolean type is a subtype of plain\n integers, and Boolean values behave like the values 0 and 1,\n respectively, in almost all contexts, the exception being\n that when converted to a string, the strings ``"False"`` or\n ``"True"`` are returned, respectively.\n\n The rules for integer representation are intended to give the\n most meaningful interpretation of shift and mask operations\n involving negative integers and the least surprises when\n switching between the plain and long integer domains. Any\n operation, if it yields a result in the plain integer domain,\n will yield the same result in the long integer domain or when\n using mixed operands. The switch between domains is transparent\n to the programmer.\n\n ``numbers.Real`` (``float``)\n These represent machine-level double precision floating point\n numbers. You are at the mercy of the underlying machine\n architecture (and C or Java implementation) for the accepted\n range and handling of overflow. Python does not support single-\n precision floating point numbers; the savings in processor and\n memory usage that are usually the reason for using these is\n dwarfed by the overhead of using objects in Python, so there is\n no reason to complicate the language with two kinds of floating\n point numbers.\n\n ``numbers.Complex``\n These represent complex numbers as a pair of machine-level\n double precision floating point numbers. The same caveats apply\n as for floating point numbers. The real and imaginary parts of a\n complex number ``z`` can be retrieved through the read-only\n attributes ``z.real`` and ``z.imag``.\n\nSequences\n These represent finite ordered sets indexed by non-negative\n numbers. The built-in function ``len()`` returns the number of\n items of a sequence. When the length of a sequence is *n*, the\n index set contains the numbers 0, 1, ..., *n*-1. Item *i* of\n sequence *a* is selected by ``a[i]``.\n\n Sequences also support slicing: ``a[i:j]`` selects all items with\n index *k* such that *i* ``<=`` *k* ``<`` *j*. When used as an\n expression, a slice is a sequence of the same type. This implies\n that the index set is renumbered so that it starts at 0.\n\n Some sequences also support "extended slicing" with a third "step"\n parameter: ``a[i:j:k]`` selects all items of *a* with index *x*\n where ``x = i + n*k``, *n* ``>=`` ``0`` and *i* ``<=`` *x* ``<``\n *j*.\n\n Sequences are distinguished according to their mutability:\n\n Immutable sequences\n An object of an immutable sequence type cannot change once it is\n created. (If the object contains references to other objects,\n these other objects may be mutable and may be changed; however,\n the collection of objects directly referenced by an immutable\n object cannot change.)\n\n The following types are immutable sequences:\n\n Strings\n The items of a string are characters. There is no separate\n character type; a character is represented by a string of one\n item. Characters represent (at least) 8-bit bytes. The\n built-in functions ``chr()`` and ``ord()`` convert between\n characters and nonnegative integers representing the byte\n values. Bytes with the values 0-127 usually represent the\n corresponding ASCII values, but the interpretation of values\n is up to the program. The string data type is also used to\n represent arrays of bytes, e.g., to hold data read from a\n file.\n\n (On systems whose native character set is not ASCII, strings\n may use EBCDIC in their internal representation, provided the\n functions ``chr()`` and ``ord()`` implement a mapping between\n ASCII and EBCDIC, and string comparison preserves the ASCII\n order. Or perhaps someone can propose a better rule?)\n\n Unicode\n The items of a Unicode object are Unicode code units. A\n Unicode code unit is represented by a Unicode object of one\n item and can hold either a 16-bit or 32-bit value\n representing a Unicode ordinal (the maximum value for the\n ordinal is given in ``sys.maxunicode``, and depends on how\n Python is configured at compile time). Surrogate pairs may\n be present in the Unicode object, and will be reported as two\n separate items. The built-in functions ``unichr()`` and\n ``ord()`` convert between code units and nonnegative integers\n representing the Unicode ordinals as defined in the Unicode\n Standard 3.0. Conversion from and to other encodings are\n possible through the Unicode method ``encode()`` and the\n built-in function ``unicode()``.\n\n Tuples\n The items of a tuple are arbitrary Python objects. Tuples of\n two or more items are formed by comma-separated lists of\n expressions. A tuple of one item (a \'singleton\') can be\n formed by affixing a comma to an expression (an expression by\n itself does not create a tuple, since parentheses must be\n usable for grouping of expressions). An empty tuple can be\n formed by an empty pair of parentheses.\n\n Mutable sequences\n Mutable sequences can be changed after they are created. The\n subscription and slicing notations can be used as the target of\n assignment and ``del`` (delete) statements.\n\n There are currently two intrinsic mutable sequence types:\n\n Lists\n The items of a list are arbitrary Python objects. Lists are\n formed by placing a comma-separated list of expressions in\n square brackets. (Note that there are no special cases needed\n to form lists of length 0 or 1.)\n\n Byte Arrays\n A bytearray object is a mutable array. They are created by\n the built-in ``bytearray()`` constructor. Aside from being\n mutable (and hence unhashable), byte arrays otherwise provide\n the same interface and functionality as immutable bytes\n objects.\n\n The extension module ``array`` provides an additional example of\n a mutable sequence type.\n\nSet types\n These represent unordered, finite sets of unique, immutable\n objects. As such, they cannot be indexed by any subscript. However,\n they can be iterated over, and the built-in function ``len()``\n returns the number of items in a set. Common uses for sets are fast\n membership testing, removing duplicates from a sequence, and\n computing mathematical operations such as intersection, union,\n difference, and symmetric difference.\n\n For set elements, the same immutability rules apply as for\n dictionary keys. Note that numeric types obey the normal rules for\n numeric comparison: if two numbers compare equal (e.g., ``1`` and\n ``1.0``), only one of them can be contained in a set.\n\n There are currently two intrinsic set types:\n\n Sets\n These represent a mutable set. They are created by the built-in\n ``set()`` constructor and can be modified afterwards by several\n methods, such as ``add()``.\n\n Frozen sets\n These represent an immutable set. They are created by the\n built-in ``frozenset()`` constructor. As a frozenset is\n immutable and *hashable*, it can be used again as an element of\n another set, or as a dictionary key.\n\nMappings\n These represent finite sets of objects indexed by arbitrary index\n sets. The subscript notation ``a[k]`` selects the item indexed by\n ``k`` from the mapping ``a``; this can be used in expressions and\n as the target of assignments or ``del`` statements. The built-in\n function ``len()`` returns the number of items in a mapping.\n\n There is currently a single intrinsic mapping type:\n\n Dictionaries\n These represent finite sets of objects indexed by nearly\n arbitrary values. The only types of values not acceptable as\n keys are values containing lists or dictionaries or other\n mutable types that are compared by value rather than by object\n identity, the reason being that the efficient implementation of\n dictionaries requires a key\'s hash value to remain constant.\n Numeric types used for keys obey the normal rules for numeric\n comparison: if two numbers compare equal (e.g., ``1`` and\n ``1.0``) then they can be used interchangeably to index the same\n dictionary entry.\n\n Dictionaries are mutable; they can be created by the ``{...}``\n notation (see section *Dictionary displays*).\n\n The extension modules ``dbm``, ``gdbm``, and ``bsddb`` provide\n additional examples of mapping types.\n\nCallable types\n These are the types to which the function call operation (see\n section *Calls*) can be applied:\n\n User-defined functions\n A user-defined function object is created by a function\n definition (see section *Function definitions*). It should be\n called with an argument list containing the same number of items\n as the function\'s formal parameter list.\n\n Special attributes:\n\n +-------------------------+---------------------------------+-------------+\n | Attribute | Meaning | |\n +=========================+=================================+=============+\n | ``func_doc`` | The function\'s documentation | Writable |\n | | string, or ``None`` if | |\n | | unavailable | |\n +-------------------------+---------------------------------+-------------+\n | ``__doc__`` | Another way of spelling | Writable |\n | | ``func_doc`` | |\n +-------------------------+---------------------------------+-------------+\n | ``func_name`` | The function\'s name | Writable |\n +-------------------------+---------------------------------+-------------+\n | ``__name__`` | Another way of spelling | Writable |\n | | ``func_name`` | |\n +-------------------------+---------------------------------+-------------+\n | ``__module__`` | The name of the module the | Writable |\n | | function was defined in, or | |\n | | ``None`` if unavailable. | |\n +-------------------------+---------------------------------+-------------+\n | ``func_defaults`` | A tuple containing default | Writable |\n | | argument values for those | |\n | | arguments that have defaults, | |\n | | or ``None`` if no arguments | |\n | | have a default value | |\n +-------------------------+---------------------------------+-------------+\n | ``func_code`` | The code object representing | Writable |\n | | the compiled function body. | |\n +-------------------------+---------------------------------+-------------+\n | ``func_globals`` | A reference to the dictionary | Read-only |\n | | that holds the function\'s | |\n | | global variables --- the global | |\n | | namespace of the module in | |\n | | which the function was defined. | |\n +-------------------------+---------------------------------+-------------+\n | ``func_dict`` | The namespace supporting | Writable |\n | | arbitrary function attributes. | |\n +-------------------------+---------------------------------+-------------+\n | ``func_closure`` | ``None`` or a tuple of cells | Read-only |\n | | that contain bindings for the | |\n | | function\'s free variables. | |\n +-------------------------+---------------------------------+-------------+\n\n Most of the attributes labelled "Writable" check the type of the\n assigned value.\n\n Changed in version 2.4: ``func_name`` is now writable.\n\n Function objects also support getting and setting arbitrary\n attributes, which can be used, for example, to attach metadata\n to functions. Regular attribute dot-notation is used to get and\n set such attributes. *Note that the current implementation only\n supports function attributes on user-defined functions. Function\n attributes on built-in functions may be supported in the\n future.*\n\n Additional information about a function\'s definition can be\n retrieved from its code object; see the description of internal\n types below.\n\n User-defined methods\n A user-defined method object combines a class, a class instance\n (or ``None``) and any callable object (normally a user-defined\n function).\n\n Special read-only attributes: ``im_self`` is the class instance\n object, ``im_func`` is the function object; ``im_class`` is the\n class of ``im_self`` for bound methods or the class that asked\n for the method for unbound methods; ``__doc__`` is the method\'s\n documentation (same as ``im_func.__doc__``); ``__name__`` is the\n method name (same as ``im_func.__name__``); ``__module__`` is\n the name of the module the method was defined in, or ``None`` if\n unavailable.\n\n Changed in version 2.2: ``im_self`` used to refer to the class\n that defined the method.\n\n Changed in version 2.6: For 3.0 forward-compatibility,\n ``im_func`` is also available as ``__func__``, and ``im_self``\n as ``__self__``.\n\n Methods also support accessing (but not setting) the arbitrary\n function attributes on the underlying function object.\n\n User-defined method objects may be created when getting an\n attribute of a class (perhaps via an instance of that class), if\n that attribute is a user-defined function object, an unbound\n user-defined method object, or a class method object. When the\n attribute is a user-defined method object, a new method object\n is only created if the class from which it is being retrieved is\n the same as, or a derived class of, the class stored in the\n original method object; otherwise, the original method object is\n used as it is.\n\n When a user-defined method object is created by retrieving a\n user-defined function object from a class, its ``im_self``\n attribute is ``None`` and the method object is said to be\n unbound. When one is created by retrieving a user-defined\n function object from a class via one of its instances, its\n ``im_self`` attribute is the instance, and the method object is\n said to be bound. In either case, the new method\'s ``im_class``\n attribute is the class from which the retrieval takes place, and\n its ``im_func`` attribute is the original function object.\n\n When a user-defined method object is created by retrieving\n another method object from a class or instance, the behaviour is\n the same as for a function object, except that the ``im_func``\n attribute of the new instance is not the original method object\n but its ``im_func`` attribute.\n\n When a user-defined method object is created by retrieving a\n class method object from a class or instance, its ``im_self``\n attribute is the class itself (the same as the ``im_class``\n attribute), and its ``im_func`` attribute is the function object\n underlying the class method.\n\n When an unbound user-defined method object is called, the\n underlying function (``im_func``) is called, with the\n restriction that the first argument must be an instance of the\n proper class (``im_class``) or of a derived class thereof.\n\n When a bound user-defined method object is called, the\n underlying function (``im_func``) is called, inserting the class\n instance (``im_self``) in front of the argument list. For\n instance, when ``C`` is a class which contains a definition for\n a function ``f()``, and ``x`` is an instance of ``C``, calling\n ``x.f(1)`` is equivalent to calling ``C.f(x, 1)``.\n\n When a user-defined method object is derived from a class method\n object, the "class instance" stored in ``im_self`` will actually\n be the class itself, so that calling either ``x.f(1)`` or\n ``C.f(1)`` is equivalent to calling ``f(C,1)`` where ``f`` is\n the underlying function.\n\n Note that the transformation from function object to (unbound or\n bound) method object happens each time the attribute is\n retrieved from the class or instance. In some cases, a fruitful\n optimization is to assign the attribute to a local variable and\n call that local variable. Also notice that this transformation\n only happens for user-defined functions; other callable objects\n (and all non-callable objects) are retrieved without\n transformation. It is also important to note that user-defined\n functions which are attributes of a class instance are not\n converted to bound methods; this *only* happens when the\n function is an attribute of the class.\n\n Generator functions\n A function or method which uses the ``yield`` statement (see\n section *The yield statement*) is called a *generator function*.\n Such a function, when called, always returns an iterator object\n which can be used to execute the body of the function: calling\n the iterator\'s ``next()`` method will cause the function to\n execute until it provides a value using the ``yield`` statement.\n When the function executes a ``return`` statement or falls off\n the end, a ``StopIteration`` exception is raised and the\n iterator will have reached the end of the set of values to be\n returned.\n\n Built-in functions\n A built-in function object is a wrapper around a C function.\n Examples of built-in functions are ``len()`` and ``math.sin()``\n (``math`` is a standard built-in module). The number and type of\n the arguments are determined by the C function. Special read-\n only attributes: ``__doc__`` is the function\'s documentation\n string, or ``None`` if unavailable; ``__name__`` is the\n function\'s name; ``__self__`` is set to ``None`` (but see the\n next item); ``__module__`` is the name of the module the\n function was defined in or ``None`` if unavailable.\n\n Built-in methods\n This is really a different disguise of a built-in function, this\n time containing an object passed to the C function as an\n implicit extra argument. An example of a built-in method is\n ``alist.append()``, assuming *alist* is a list object. In this\n case, the special read-only attribute ``__self__`` is set to the\n object denoted by *alist*.\n\n Class Types\n Class types, or "new-style classes," are callable. These\n objects normally act as factories for new instances of\n themselves, but variations are possible for class types that\n override ``__new__()``. The arguments of the call are passed to\n ``__new__()`` and, in the typical case, to ``__init__()`` to\n initialize the new instance.\n\n Classic Classes\n Class objects are described below. When a class object is\n called, a new class instance (also described below) is created\n and returned. This implies a call to the class\'s ``__init__()``\n method if it has one. Any arguments are passed on to the\n ``__init__()`` method. If there is no ``__init__()`` method,\n the class must be called without arguments.\n\n Class instances\n Class instances are described below. Class instances are\n callable only when the class has a ``__call__()`` method;\n ``x(arguments)`` is a shorthand for ``x.__call__(arguments)``.\n\nModules\n Modules are imported by the ``import`` statement (see section *The\n import statement*). A module object has a namespace implemented by\n a dictionary object (this is the dictionary referenced by the\n func_globals attribute of functions defined in the module).\n Attribute references are translated to lookups in this dictionary,\n e.g., ``m.x`` is equivalent to ``m.__dict__["x"]``. A module object\n does not contain the code object used to initialize the module\n (since it isn\'t needed once the initialization is done).\n\n Attribute assignment updates the module\'s namespace dictionary,\n e.g., ``m.x = 1`` is equivalent to ``m.__dict__["x"] = 1``.\n\n Special read-only attribute: ``__dict__`` is the module\'s namespace\n as a dictionary object.\n\n **CPython implementation detail:** Because of the way CPython\n clears module dictionaries, the module dictionary will be cleared\n when the module falls out of scope even if the dictionary still has\n live references. To avoid this, copy the dictionary or keep the\n module around while using its dictionary directly.\n\n Predefined (writable) attributes: ``__name__`` is the module\'s\n name; ``__doc__`` is the module\'s documentation string, or ``None``\n if unavailable; ``__file__`` is the pathname of the file from which\n the module was loaded, if it was loaded from a file. The\n ``__file__`` attribute is not present for C modules that are\n statically linked into the interpreter; for extension modules\n loaded dynamically from a shared library, it is the pathname of the\n shared library file.\n\nClasses\n Both class types (new-style classes) and class objects (old-\n style/classic classes) are typically created by class definitions\n (see section *Class definitions*). A class has a namespace\n implemented by a dictionary object. Class attribute references are\n translated to lookups in this dictionary, e.g., ``C.x`` is\n translated to ``C.__dict__["x"]`` (although for new-style classes\n in particular there are a number of hooks which allow for other\n means of locating attributes). When the attribute name is not found\n there, the attribute search continues in the base classes. For\n old-style classes, the search is depth-first, left-to-right in the\n order of occurrence in the base class list. New-style classes use\n the more complex C3 method resolution order which behaves correctly\n even in the presence of \'diamond\' inheritance structures where\n there are multiple inheritance paths leading back to a common\n ancestor. Additional details on the C3 MRO used by new-style\n classes can be found in the documentation accompanying the 2.3\n release at http://www.python.org/download/releases/2.3/mro/.\n\n When a class attribute reference (for class ``C``, say) would yield\n a user-defined function object or an unbound user-defined method\n object whose associated class is either ``C`` or one of its base\n classes, it is transformed into an unbound user-defined method\n object whose ``im_class`` attribute is ``C``. When it would yield a\n class method object, it is transformed into a bound user-defined\n method object whose ``im_class`` and ``im_self`` attributes are\n both ``C``. When it would yield a static method object, it is\n transformed into the object wrapped by the static method object.\n See section *Implementing Descriptors* for another way in which\n attributes retrieved from a class may differ from those actually\n contained in its ``__dict__`` (note that only new-style classes\n support descriptors).\n\n Class attribute assignments update the class\'s dictionary, never\n the dictionary of a base class.\n\n A class object can be called (see above) to yield a class instance\n (see below).\n\n Special attributes: ``__name__`` is the class name; ``__module__``\n is the module name in which the class was defined; ``__dict__`` is\n the dictionary containing the class\'s namespace; ``__bases__`` is a\n tuple (possibly empty or a singleton) containing the base classes,\n in the order of their occurrence in the base class list;\n ``__doc__`` is the class\'s documentation string, or None if\n undefined.\n\nClass instances\n A class instance is created by calling a class object (see above).\n A class instance has a namespace implemented as a dictionary which\n is the first place in which attribute references are searched.\n When an attribute is not found there, and the instance\'s class has\n an attribute by that name, the search continues with the class\n attributes. If a class attribute is found that is a user-defined\n function object or an unbound user-defined method object whose\n associated class is the class (call it ``C``) of the instance for\n which the attribute reference was initiated or one of its bases, it\n is transformed into a bound user-defined method object whose\n ``im_class`` attribute is ``C`` and whose ``im_self`` attribute is\n the instance. Static method and class method objects are also\n transformed, as if they had been retrieved from class ``C``; see\n above under "Classes". See section *Implementing Descriptors* for\n another way in which attributes of a class retrieved via its\n instances may differ from the objects actually stored in the\n class\'s ``__dict__``. If no class attribute is found, and the\n object\'s class has a ``__getattr__()`` method, that is called to\n satisfy the lookup.\n\n Attribute assignments and deletions update the instance\'s\n dictionary, never a class\'s dictionary. If the class has a\n ``__setattr__()`` or ``__delattr__()`` method, this is called\n instead of updating the instance dictionary directly.\n\n Class instances can pretend to be numbers, sequences, or mappings\n if they have methods with certain special names. See section\n *Special method names*.\n\n Special attributes: ``__dict__`` is the attribute dictionary;\n ``__class__`` is the instance\'s class.\n\nFiles\n A file object represents an open file. File objects are created by\n the ``open()`` built-in function, and also by ``os.popen()``,\n ``os.fdopen()``, and the ``makefile()`` method of socket objects\n (and perhaps by other functions or methods provided by extension\n modules). The objects ``sys.stdin``, ``sys.stdout`` and\n ``sys.stderr`` are initialized to file objects corresponding to the\n interpreter\'s standard input, output and error streams. See *File\n Objects* for complete documentation of file objects.\n\nInternal types\n A few types used internally by the interpreter are exposed to the\n user. Their definitions may change with future versions of the\n interpreter, but they are mentioned here for completeness.\n\n Code objects\n Code objects represent *byte-compiled* executable Python code,\n or *bytecode*. The difference between a code object and a\n function object is that the function object contains an explicit\n reference to the function\'s globals (the module in which it was\n defined), while a code object contains no context; also the\n default argument values are stored in the function object, not\n in the code object (because they represent values calculated at\n run-time). Unlike function objects, code objects are immutable\n and contain no references (directly or indirectly) to mutable\n objects.\n\n Special read-only attributes: ``co_name`` gives the function\n name; ``co_argcount`` is the number of positional arguments\n (including arguments with default values); ``co_nlocals`` is the\n number of local variables used by the function (including\n arguments); ``co_varnames`` is a tuple containing the names of\n the local variables (starting with the argument names);\n ``co_cellvars`` is a tuple containing the names of local\n variables that are referenced by nested functions;\n ``co_freevars`` is a tuple containing the names of free\n variables; ``co_code`` is a string representing the sequence of\n bytecode instructions; ``co_consts`` is a tuple containing the\n literals used by the bytecode; ``co_names`` is a tuple\n containing the names used by the bytecode; ``co_filename`` is\n the filename from which the code was compiled;\n ``co_firstlineno`` is the first line number of the function;\n ``co_lnotab`` is a string encoding the mapping from bytecode\n offsets to line numbers (for details see the source code of the\n interpreter); ``co_stacksize`` is the required stack size\n (including local variables); ``co_flags`` is an integer encoding\n a number of flags for the interpreter.\n\n The following flag bits are defined for ``co_flags``: bit\n ``0x04`` is set if the function uses the ``*arguments`` syntax\n to accept an arbitrary number of positional arguments; bit\n ``0x08`` is set if the function uses the ``**keywords`` syntax\n to accept arbitrary keyword arguments; bit ``0x20`` is set if\n the function is a generator.\n\n Future feature declarations (``from __future__ import\n division``) also use bits in ``co_flags`` to indicate whether a\n code object was compiled with a particular feature enabled: bit\n ``0x2000`` is set if the function was compiled with future\n division enabled; bits ``0x10`` and ``0x1000`` were used in\n earlier versions of Python.\n\n Other bits in ``co_flags`` are reserved for internal use.\n\n If a code object represents a function, the first item in\n ``co_consts`` is the documentation string of the function, or\n ``None`` if undefined.\n\n Frame objects\n Frame objects represent execution frames. They may occur in\n traceback objects (see below).\n\n Special read-only attributes: ``f_back`` is to the previous\n stack frame (towards the caller), or ``None`` if this is the\n bottom stack frame; ``f_code`` is the code object being executed\n in this frame; ``f_locals`` is the dictionary used to look up\n local variables; ``f_globals`` is used for global variables;\n ``f_builtins`` is used for built-in (intrinsic) names;\n ``f_restricted`` is a flag indicating whether the function is\n executing in restricted execution mode; ``f_lasti`` gives the\n precise instruction (this is an index into the bytecode string\n of the code object).\n\n Special writable attributes: ``f_trace``, if not ``None``, is a\n function called at the start of each source code line (this is\n used by the debugger); ``f_exc_type``, ``f_exc_value``,\n ``f_exc_traceback`` represent the last exception raised in the\n parent frame provided another exception was ever raised in the\n current frame (in all other cases they are None); ``f_lineno``\n is the current line number of the frame --- writing to this from\n within a trace function jumps to the given line (only for the\n bottom-most frame). A debugger can implement a Jump command\n (aka Set Next Statement) by writing to f_lineno.\n\n Traceback objects\n Traceback objects represent a stack trace of an exception. A\n traceback object is created when an exception occurs. When the\n search for an exception handler unwinds the execution stack, at\n each unwound level a traceback object is inserted in front of\n the current traceback. When an exception handler is entered,\n the stack trace is made available to the program. (See section\n *The try statement*.) It is accessible as ``sys.exc_traceback``,\n and also as the third item of the tuple returned by\n ``sys.exc_info()``. The latter is the preferred interface,\n since it works correctly when the program is using multiple\n threads. When the program contains no suitable handler, the\n stack trace is written (nicely formatted) to the standard error\n stream; if the interpreter is interactive, it is also made\n available to the user as ``sys.last_traceback``.\n\n Special read-only attributes: ``tb_next`` is the next level in\n the stack trace (towards the frame where the exception\n occurred), or ``None`` if there is no next level; ``tb_frame``\n points to the execution frame of the current level;\n ``tb_lineno`` gives the line number where the exception\n occurred; ``tb_lasti`` indicates the precise instruction. The\n line number and last instruction in the traceback may differ\n from the line number of its frame object if the exception\n occurred in a ``try`` statement with no matching except clause\n or with a finally clause.\n\n Slice objects\n Slice objects are used to represent slices when *extended slice\n syntax* is used. This is a slice using two colons, or multiple\n slices or ellipses separated by commas, e.g., ``a[i:j:step]``,\n ``a[i:j, k:l]``, or ``a[..., i:j]``. They are also created by\n the built-in ``slice()`` function.\n\n Special read-only attributes: ``start`` is the lower bound;\n ``stop`` is the upper bound; ``step`` is the step value; each is\n ``None`` if omitted. These attributes can have any type.\n\n Slice objects support one method:\n\n slice.indices(self, length)\n\n This method takes a single integer argument *length* and\n computes information about the extended slice that the slice\n object would describe if applied to a sequence of *length*\n items. It returns a tuple of three integers; respectively\n these are the *start* and *stop* indices and the *step* or\n stride length of the slice. Missing or out-of-bounds indices\n are handled in a manner consistent with regular slices.\n\n New in version 2.3.\n\n Static method objects\n Static method objects provide a way of defeating the\n transformation of function objects to method objects described\n above. A static method object is a wrapper around any other\n object, usually a user-defined method object. When a static\n method object is retrieved from a class or a class instance, the\n object actually returned is the wrapped object, which is not\n subject to any further transformation. Static method objects are\n not themselves callable, although the objects they wrap usually\n are. Static method objects are created by the built-in\n ``staticmethod()`` constructor.\n\n Class method objects\n A class method object, like a static method object, is a wrapper\n around another object that alters the way in which that object\n is retrieved from classes and class instances. The behaviour of\n class method objects upon such retrieval is described above,\n under "User-defined methods". Class method objects are created\n by the built-in ``classmethod()`` constructor.\n', 'typesfunctions': u'\nFunctions\n*********\n\nFunction objects are created by function definitions. The only\noperation on a function object is to call it: ``func(argument-list)``.\n\nThere are really two flavors of function objects: built-in functions\nand user-defined functions. Both support the same operation (to call\nthe function), but the implementation is different, hence the\ndifferent object types.\n\nSee *Function definitions* for more information.\n', - 'typesmapping': u'\nMapping Types --- ``dict``\n**************************\n\nA *mapping* object maps *hashable* values to arbitrary objects.\nMappings are mutable objects. There is currently only one standard\nmapping type, the *dictionary*. (For other containers see the built\nin ``list``, ``set``, and ``tuple`` classes, and the ``collections``\nmodule.)\n\nA dictionary\'s keys are *almost* arbitrary values. Values that are\nnot *hashable*, that is, values containing lists, dictionaries or\nother mutable types (that are compared by value rather than by object\nidentity) may not be used as keys. Numeric types used for keys obey\nthe normal rules for numeric comparison: if two numbers compare equal\n(such as ``1`` and ``1.0``) then they can be used interchangeably to\nindex the same dictionary entry. (Note however, that since computers\nstore floating-point numbers as approximations it is usually unwise to\nuse them as dictionary keys.)\n\nDictionaries can be created by placing a comma-separated list of\n``key: value`` pairs within braces, for example: ``{\'jack\': 4098,\n\'sjoerd\': 4127}`` or ``{4098: \'jack\', 4127: \'sjoerd\'}``, or by the\n``dict`` constructor.\n\nclass class dict([arg])\n\n Return a new dictionary initialized from an optional positional\n argument or from a set of keyword arguments. If no arguments are\n given, return a new empty dictionary. If the positional argument\n *arg* is a mapping object, return a dictionary mapping the same\n keys to the same values as does the mapping object. Otherwise the\n positional argument must be a sequence, a container that supports\n iteration, or an iterator object. The elements of the argument\n must each also be of one of those kinds, and each must in turn\n contain exactly two objects. The first is used as a key in the new\n dictionary, and the second as the key\'s value. If a given key is\n seen more than once, the last value associated with it is retained\n in the new dictionary.\n\n If keyword arguments are given, the keywords themselves with their\n associated values are added as items to the dictionary. If a key is\n specified both in the positional argument and as a keyword\n argument, the value associated with the keyword is retained in the\n dictionary. For example, these all return a dictionary equal to\n ``{"one": 2, "two": 3}``:\n\n * ``dict(one=2, two=3)``\n\n * ``dict({\'one\': 2, \'two\': 3})``\n\n * ``dict(zip((\'one\', \'two\'), (2, 3)))``\n\n * ``dict([[\'two\', 3], [\'one\', 2]])``\n\n The first example only works for keys that are valid Python\n identifiers; the others work with any valid keys.\n\n New in version 2.2.\n\n Changed in version 2.3: Support for building a dictionary from\n keyword arguments added.\n\n These are the operations that dictionaries support (and therefore,\n custom mapping types should support too):\n\n len(d)\n\n Return the number of items in the dictionary *d*.\n\n d[key]\n\n Return the item of *d* with key *key*. Raises a ``KeyError`` if\n *key* is not in the map.\n\n New in version 2.5: If a subclass of dict defines a method\n ``__missing__()``, if the key *key* is not present, the\n ``d[key]`` operation calls that method with the key *key* as\n argument. The ``d[key]`` operation then returns or raises\n whatever is returned or raised by the ``__missing__(key)`` call\n if the key is not present. No other operations or methods invoke\n ``__missing__()``. If ``__missing__()`` is not defined,\n ``KeyError`` is raised. ``__missing__()`` must be a method; it\n cannot be an instance variable. For an example, see\n ``collections.defaultdict``.\n\n d[key] = value\n\n Set ``d[key]`` to *value*.\n\n del d[key]\n\n Remove ``d[key]`` from *d*. Raises a ``KeyError`` if *key* is\n not in the map.\n\n key in d\n\n Return ``True`` if *d* has a key *key*, else ``False``.\n\n New in version 2.2.\n\n key not in d\n\n Equivalent to ``not key in d``.\n\n New in version 2.2.\n\n iter(d)\n\n Return an iterator over the keys of the dictionary. This is a\n shortcut for ``iterkeys()``.\n\n clear()\n\n Remove all items from the dictionary.\n\n copy()\n\n Return a shallow copy of the dictionary.\n\n fromkeys(seq[, value])\n\n Create a new dictionary with keys from *seq* and values set to\n *value*.\n\n ``fromkeys()`` is a class method that returns a new dictionary.\n *value* defaults to ``None``.\n\n New in version 2.3.\n\n get(key[, default])\n\n Return the value for *key* if *key* is in the dictionary, else\n *default*. If *default* is not given, it defaults to ``None``,\n so that this method never raises a ``KeyError``.\n\n has_key(key)\n\n Test for the presence of *key* in the dictionary. ``has_key()``\n is deprecated in favor of ``key in d``.\n\n items()\n\n Return a copy of the dictionary\'s list of ``(key, value)``\n pairs.\n\n **CPython implementation detail:** Keys and values are listed in\n an arbitrary order which is non-random, varies across Python\n implementations, and depends on the dictionary\'s history of\n insertions and deletions.\n\n If ``items()``, ``keys()``, ``values()``, ``iteritems()``,\n ``iterkeys()``, and ``itervalues()`` are called with no\n intervening modifications to the dictionary, the lists will\n directly correspond. This allows the creation of ``(value,\n key)`` pairs using ``zip()``: ``pairs = zip(d.values(),\n d.keys())``. The same relationship holds for the ``iterkeys()``\n and ``itervalues()`` methods: ``pairs = zip(d.itervalues(),\n d.iterkeys())`` provides the same value for ``pairs``. Another\n way to create the same list is ``pairs = [(v, k) for (k, v) in\n d.iteritems()]``.\n\n iteritems()\n\n Return an iterator over the dictionary\'s ``(key, value)`` pairs.\n See the note for ``dict.items()``.\n\n Using ``iteritems()`` while adding or deleting entries in the\n dictionary may raise a ``RuntimeError`` or fail to iterate over\n all entries.\n\n New in version 2.2.\n\n iterkeys()\n\n Return an iterator over the dictionary\'s keys. See the note for\n ``dict.items()``.\n\n Using ``iterkeys()`` while adding or deleting entries in the\n dictionary may raise a ``RuntimeError`` or fail to iterate over\n all entries.\n\n New in version 2.2.\n\n itervalues()\n\n Return an iterator over the dictionary\'s values. See the note\n for ``dict.items()``.\n\n Using ``itervalues()`` while adding or deleting entries in the\n dictionary may raise a ``RuntimeError`` or fail to iterate over\n all entries.\n\n New in version 2.2.\n\n keys()\n\n Return a copy of the dictionary\'s list of keys. See the note\n for ``dict.items()``.\n\n pop(key[, default])\n\n If *key* is in the dictionary, remove it and return its value,\n else return *default*. If *default* is not given and *key* is\n not in the dictionary, a ``KeyError`` is raised.\n\n New in version 2.3.\n\n popitem()\n\n Remove and return an arbitrary ``(key, value)`` pair from the\n dictionary.\n\n ``popitem()`` is useful to destructively iterate over a\n dictionary, as often used in set algorithms. If the dictionary\n is empty, calling ``popitem()`` raises a ``KeyError``.\n\n setdefault(key[, default])\n\n If *key* is in the dictionary, return its value. If not, insert\n *key* with a value of *default* and return *default*. *default*\n defaults to ``None``.\n\n update([other])\n\n Update the dictionary with the key/value pairs from *other*,\n overwriting existing keys. Return ``None``.\n\n ``update()`` accepts either another dictionary object or an\n iterable of key/value pairs (as a tuple or other iterable of\n length two). If keyword arguments are specified, the dictionary\n is then updated with those key/value pairs: ``d.update(red=1,\n blue=2)``.\n\n Changed in version 2.4: Allowed the argument to be an iterable\n of key/value pairs and allowed keyword arguments.\n\n values()\n\n Return a copy of the dictionary\'s list of values. See the note\n for ``dict.items()``.\n\n viewitems()\n\n Return a new view of the dictionary\'s items (``(key, value)``\n pairs). See below for documentation of view objects.\n\n New in version 2.7.\n\n viewkeys()\n\n Return a new view of the dictionary\'s keys. See below for\n documentation of view objects.\n\n New in version 2.7.\n\n viewvalues()\n\n Return a new view of the dictionary\'s values. See below for\n documentation of view objects.\n\n New in version 2.7.\n\n\nDictionary view objects\n=======================\n\nThe objects returned by ``dict.viewkeys()``, ``dict.viewvalues()`` and\n``dict.viewitems()`` are *view objects*. They provide a dynamic view\non the dictionary\'s entries, which means that when the dictionary\nchanges, the view reflects these changes.\n\nDictionary views can be iterated over to yield their respective data,\nand support membership tests:\n\nlen(dictview)\n\n Return the number of entries in the dictionary.\n\niter(dictview)\n\n Return an iterator over the keys, values or items (represented as\n tuples of ``(key, value)``) in the dictionary.\n\n Keys and values are iterated over in an arbitrary order which is\n non-random, varies across Python implementations, and depends on\n the dictionary\'s history of insertions and deletions. If keys,\n values and items views are iterated over with no intervening\n modifications to the dictionary, the order of items will directly\n correspond. This allows the creation of ``(value, key)`` pairs\n using ``zip()``: ``pairs = zip(d.values(), d.keys())``. Another\n way to create the same list is ``pairs = [(v, k) for (k, v) in\n d.items()]``.\n\n Iterating views while adding or deleting entries in the dictionary\n may raise a ``RuntimeError`` or fail to iterate over all entries.\n\nx in dictview\n\n Return ``True`` if *x* is in the underlying dictionary\'s keys,\n values or items (in the latter case, *x* should be a ``(key,\n value)`` tuple).\n\nKeys views are set-like since their entries are unique and hashable.\nIf all values are hashable, so that (key, value) pairs are unique and\nhashable, then the items view is also set-like. (Values views are not\ntreated as set-like since the entries are generally not unique.) Then\nthese set operations are available ("other" refers either to another\nview or a set):\n\ndictview & other\n\n Return the intersection of the dictview and the other object as a\n new set.\n\ndictview | other\n\n Return the union of the dictview and the other object as a new set.\n\ndictview - other\n\n Return the difference between the dictview and the other object\n (all elements in *dictview* that aren\'t in *other*) as a new set.\n\ndictview ^ other\n\n Return the symmetric difference (all elements either in *dictview*\n or *other*, but not in both) of the dictview and the other object\n as a new set.\n\nAn example of dictionary view usage:\n\n >>> dishes = {\'eggs\': 2, \'sausage\': 1, \'bacon\': 1, \'spam\': 500}\n >>> keys = dishes.viewkeys()\n >>> values = dishes.viewvalues()\n\n >>> # iteration\n >>> n = 0\n >>> for val in values:\n ... n += val\n >>> print(n)\n 504\n\n >>> # keys and values are iterated over in the same order\n >>> list(keys)\n [\'eggs\', \'bacon\', \'sausage\', \'spam\']\n >>> list(values)\n [2, 1, 1, 500]\n\n >>> # view objects are dynamic and reflect dict changes\n >>> del dishes[\'eggs\']\n >>> del dishes[\'sausage\']\n >>> list(keys)\n [\'spam\', \'bacon\']\n\n >>> # set operations\n >>> keys & {\'eggs\', \'bacon\', \'salad\'}\n {\'bacon\'}\n', + 'typesmapping': u'\nMapping Types --- ``dict``\n**************************\n\nA *mapping* object maps *hashable* values to arbitrary objects.\nMappings are mutable objects. There is currently only one standard\nmapping type, the *dictionary*. (For other containers see the built\nin ``list``, ``set``, and ``tuple`` classes, and the ``collections``\nmodule.)\n\nA dictionary\'s keys are *almost* arbitrary values. Values that are\nnot *hashable*, that is, values containing lists, dictionaries or\nother mutable types (that are compared by value rather than by object\nidentity) may not be used as keys. Numeric types used for keys obey\nthe normal rules for numeric comparison: if two numbers compare equal\n(such as ``1`` and ``1.0``) then they can be used interchangeably to\nindex the same dictionary entry. (Note however, that since computers\nstore floating-point numbers as approximations it is usually unwise to\nuse them as dictionary keys.)\n\nDictionaries can be created by placing a comma-separated list of\n``key: value`` pairs within braces, for example: ``{\'jack\': 4098,\n\'sjoerd\': 4127}`` or ``{4098: \'jack\', 4127: \'sjoerd\'}``, or by the\n``dict`` constructor.\n\nclass class dict([arg])\n\n Return a new dictionary initialized from an optional positional\n argument or from a set of keyword arguments. If no arguments are\n given, return a new empty dictionary. If the positional argument\n *arg* is a mapping object, return a dictionary mapping the same\n keys to the same values as does the mapping object. Otherwise the\n positional argument must be a sequence, a container that supports\n iteration, or an iterator object. The elements of the argument\n must each also be of one of those kinds, and each must in turn\n contain exactly two objects. The first is used as a key in the new\n dictionary, and the second as the key\'s value. If a given key is\n seen more than once, the last value associated with it is retained\n in the new dictionary.\n\n If keyword arguments are given, the keywords themselves with their\n associated values are added as items to the dictionary. If a key is\n specified both in the positional argument and as a keyword\n argument, the value associated with the keyword is retained in the\n dictionary. For example, these all return a dictionary equal to\n ``{"one": 1, "two": 2}``:\n\n * ``dict(one=1, two=2)``\n\n * ``dict({\'one\': 1, \'two\': 2})``\n\n * ``dict(zip((\'one\', \'two\'), (1, 2)))``\n\n * ``dict([[\'two\', 2], [\'one\', 1]])``\n\n The first example only works for keys that are valid Python\n identifiers; the others work with any valid keys.\n\n New in version 2.2.\n\n Changed in version 2.3: Support for building a dictionary from\n keyword arguments added.\n\n These are the operations that dictionaries support (and therefore,\n custom mapping types should support too):\n\n len(d)\n\n Return the number of items in the dictionary *d*.\n\n d[key]\n\n Return the item of *d* with key *key*. Raises a ``KeyError`` if\n *key* is not in the map.\n\n New in version 2.5: If a subclass of dict defines a method\n ``__missing__()``, if the key *key* is not present, the\n ``d[key]`` operation calls that method with the key *key* as\n argument. The ``d[key]`` operation then returns or raises\n whatever is returned or raised by the ``__missing__(key)`` call\n if the key is not present. No other operations or methods invoke\n ``__missing__()``. If ``__missing__()`` is not defined,\n ``KeyError`` is raised. ``__missing__()`` must be a method; it\n cannot be an instance variable. For an example, see\n ``collections.defaultdict``.\n\n d[key] = value\n\n Set ``d[key]`` to *value*.\n\n del d[key]\n\n Remove ``d[key]`` from *d*. Raises a ``KeyError`` if *key* is\n not in the map.\n\n key in d\n\n Return ``True`` if *d* has a key *key*, else ``False``.\n\n New in version 2.2.\n\n key not in d\n\n Equivalent to ``not key in d``.\n\n New in version 2.2.\n\n iter(d)\n\n Return an iterator over the keys of the dictionary. This is a\n shortcut for ``iterkeys()``.\n\n clear()\n\n Remove all items from the dictionary.\n\n copy()\n\n Return a shallow copy of the dictionary.\n\n fromkeys(seq[, value])\n\n Create a new dictionary with keys from *seq* and values set to\n *value*.\n\n ``fromkeys()`` is a class method that returns a new dictionary.\n *value* defaults to ``None``.\n\n New in version 2.3.\n\n get(key[, default])\n\n Return the value for *key* if *key* is in the dictionary, else\n *default*. If *default* is not given, it defaults to ``None``,\n so that this method never raises a ``KeyError``.\n\n has_key(key)\n\n Test for the presence of *key* in the dictionary. ``has_key()``\n is deprecated in favor of ``key in d``.\n\n items()\n\n Return a copy of the dictionary\'s list of ``(key, value)``\n pairs.\n\n **CPython implementation detail:** Keys and values are listed in\n an arbitrary order which is non-random, varies across Python\n implementations, and depends on the dictionary\'s history of\n insertions and deletions.\n\n If ``items()``, ``keys()``, ``values()``, ``iteritems()``,\n ``iterkeys()``, and ``itervalues()`` are called with no\n intervening modifications to the dictionary, the lists will\n directly correspond. This allows the creation of ``(value,\n key)`` pairs using ``zip()``: ``pairs = zip(d.values(),\n d.keys())``. The same relationship holds for the ``iterkeys()``\n and ``itervalues()`` methods: ``pairs = zip(d.itervalues(),\n d.iterkeys())`` provides the same value for ``pairs``. Another\n way to create the same list is ``pairs = [(v, k) for (k, v) in\n d.iteritems()]``.\n\n iteritems()\n\n Return an iterator over the dictionary\'s ``(key, value)`` pairs.\n See the note for ``dict.items()``.\n\n Using ``iteritems()`` while adding or deleting entries in the\n dictionary may raise a ``RuntimeError`` or fail to iterate over\n all entries.\n\n New in version 2.2.\n\n iterkeys()\n\n Return an iterator over the dictionary\'s keys. See the note for\n ``dict.items()``.\n\n Using ``iterkeys()`` while adding or deleting entries in the\n dictionary may raise a ``RuntimeError`` or fail to iterate over\n all entries.\n\n New in version 2.2.\n\n itervalues()\n\n Return an iterator over the dictionary\'s values. See the note\n for ``dict.items()``.\n\n Using ``itervalues()`` while adding or deleting entries in the\n dictionary may raise a ``RuntimeError`` or fail to iterate over\n all entries.\n\n New in version 2.2.\n\n keys()\n\n Return a copy of the dictionary\'s list of keys. See the note\n for ``dict.items()``.\n\n pop(key[, default])\n\n If *key* is in the dictionary, remove it and return its value,\n else return *default*. If *default* is not given and *key* is\n not in the dictionary, a ``KeyError`` is raised.\n\n New in version 2.3.\n\n popitem()\n\n Remove and return an arbitrary ``(key, value)`` pair from the\n dictionary.\n\n ``popitem()`` is useful to destructively iterate over a\n dictionary, as often used in set algorithms. If the dictionary\n is empty, calling ``popitem()`` raises a ``KeyError``.\n\n setdefault(key[, default])\n\n If *key* is in the dictionary, return its value. If not, insert\n *key* with a value of *default* and return *default*. *default*\n defaults to ``None``.\n\n update([other])\n\n Update the dictionary with the key/value pairs from *other*,\n overwriting existing keys. Return ``None``.\n\n ``update()`` accepts either another dictionary object or an\n iterable of key/value pairs (as tuples or other iterables of\n length two). If keyword arguments are specified, the dictionary\n is then updated with those key/value pairs: ``d.update(red=1,\n blue=2)``.\n\n Changed in version 2.4: Allowed the argument to be an iterable\n of key/value pairs and allowed keyword arguments.\n\n values()\n\n Return a copy of the dictionary\'s list of values. See the note\n for ``dict.items()``.\n\n viewitems()\n\n Return a new view of the dictionary\'s items (``(key, value)``\n pairs). See below for documentation of view objects.\n\n New in version 2.7.\n\n viewkeys()\n\n Return a new view of the dictionary\'s keys. See below for\n documentation of view objects.\n\n New in version 2.7.\n\n viewvalues()\n\n Return a new view of the dictionary\'s values. See below for\n documentation of view objects.\n\n New in version 2.7.\n\n\nDictionary view objects\n=======================\n\nThe objects returned by ``dict.viewkeys()``, ``dict.viewvalues()`` and\n``dict.viewitems()`` are *view objects*. They provide a dynamic view\non the dictionary\'s entries, which means that when the dictionary\nchanges, the view reflects these changes.\n\nDictionary views can be iterated over to yield their respective data,\nand support membership tests:\n\nlen(dictview)\n\n Return the number of entries in the dictionary.\n\niter(dictview)\n\n Return an iterator over the keys, values or items (represented as\n tuples of ``(key, value)``) in the dictionary.\n\n Keys and values are iterated over in an arbitrary order which is\n non-random, varies across Python implementations, and depends on\n the dictionary\'s history of insertions and deletions. If keys,\n values and items views are iterated over with no intervening\n modifications to the dictionary, the order of items will directly\n correspond. This allows the creation of ``(value, key)`` pairs\n using ``zip()``: ``pairs = zip(d.values(), d.keys())``. Another\n way to create the same list is ``pairs = [(v, k) for (k, v) in\n d.items()]``.\n\n Iterating views while adding or deleting entries in the dictionary\n may raise a ``RuntimeError`` or fail to iterate over all entries.\n\nx in dictview\n\n Return ``True`` if *x* is in the underlying dictionary\'s keys,\n values or items (in the latter case, *x* should be a ``(key,\n value)`` tuple).\n\nKeys views are set-like since their entries are unique and hashable.\nIf all values are hashable, so that (key, value) pairs are unique and\nhashable, then the items view is also set-like. (Values views are not\ntreated as set-like since the entries are generally not unique.) Then\nthese set operations are available ("other" refers either to another\nview or a set):\n\ndictview & other\n\n Return the intersection of the dictview and the other object as a\n new set.\n\ndictview | other\n\n Return the union of the dictview and the other object as a new set.\n\ndictview - other\n\n Return the difference between the dictview and the other object\n (all elements in *dictview* that aren\'t in *other*) as a new set.\n\ndictview ^ other\n\n Return the symmetric difference (all elements either in *dictview*\n or *other*, but not in both) of the dictview and the other object\n as a new set.\n\nAn example of dictionary view usage:\n\n >>> dishes = {\'eggs\': 2, \'sausage\': 1, \'bacon\': 1, \'spam\': 500}\n >>> keys = dishes.viewkeys()\n >>> values = dishes.viewvalues()\n\n >>> # iteration\n >>> n = 0\n >>> for val in values:\n ... n += val\n >>> print(n)\n 504\n\n >>> # keys and values are iterated over in the same order\n >>> list(keys)\n [\'eggs\', \'bacon\', \'sausage\', \'spam\']\n >>> list(values)\n [2, 1, 1, 500]\n\n >>> # view objects are dynamic and reflect dict changes\n >>> del dishes[\'eggs\']\n >>> del dishes[\'sausage\']\n >>> list(keys)\n [\'spam\', \'bacon\']\n\n >>> # set operations\n >>> keys & {\'eggs\', \'bacon\', \'salad\'}\n {\'bacon\'}\n', 'typesmethods': u"\nMethods\n*******\n\nMethods are functions that are called using the attribute notation.\nThere are two flavors: built-in methods (such as ``append()`` on\nlists) and class instance methods. Built-in methods are described\nwith the types that support them.\n\nThe implementation adds two special read-only attributes to class\ninstance methods: ``m.im_self`` is the object on which the method\noperates, and ``m.im_func`` is the function implementing the method.\nCalling ``m(arg-1, arg-2, ..., arg-n)`` is completely equivalent to\ncalling ``m.im_func(m.im_self, arg-1, arg-2, ..., arg-n)``.\n\nClass instance methods are either *bound* or *unbound*, referring to\nwhether the method was accessed through an instance or a class,\nrespectively. When a method is unbound, its ``im_self`` attribute\nwill be ``None`` and if called, an explicit ``self`` object must be\npassed as the first argument. In this case, ``self`` must be an\ninstance of the unbound method's class (or a subclass of that class),\notherwise a ``TypeError`` is raised.\n\nLike function objects, methods objects support getting arbitrary\nattributes. However, since method attributes are actually stored on\nthe underlying function object (``meth.im_func``), setting method\nattributes on either bound or unbound methods is disallowed.\nAttempting to set a method attribute results in a ``TypeError`` being\nraised. In order to set a method attribute, you need to explicitly\nset it on the underlying function object:\n\n class C:\n def method(self):\n pass\n\n c = C()\n c.method.im_func.whoami = 'my name is c'\n\nSee *The standard type hierarchy* for more information.\n", 'typesmodules': u"\nModules\n*******\n\nThe only special operation on a module is attribute access:\n``m.name``, where *m* is a module and *name* accesses a name defined\nin *m*'s symbol table. Module attributes can be assigned to. (Note\nthat the ``import`` statement is not, strictly speaking, an operation\non a module object; ``import foo`` does not require a module object\nnamed *foo* to exist, rather it requires an (external) *definition*\nfor a module named *foo* somewhere.)\n\nA special member of every module is ``__dict__``. This is the\ndictionary containing the module's symbol table. Modifying this\ndictionary will actually change the module's symbol table, but direct\nassignment to the ``__dict__`` attribute is not possible (you can\nwrite ``m.__dict__['a'] = 1``, which defines ``m.a`` to be ``1``, but\nyou can't write ``m.__dict__ = {}``). Modifying ``__dict__`` directly\nis not recommended.\n\nModules built into the interpreter are written like this: ``<module\n'sys' (built-in)>``. If loaded from a file, they are written as\n``<module 'os' from '/usr/local/lib/pythonX.Y/os.pyc'>``.\n", - 'typesseq': u'\nSequence Types --- ``str``, ``unicode``, ``list``, ``tuple``, ``buffer``, ``xrange``\n************************************************************************************\n\nThere are six sequence types: strings, Unicode strings, lists, tuples,\nbuffers, and xrange objects.\n\nFor other containers see the built in ``dict`` and ``set`` classes,\nand the ``collections`` module.\n\nString literals are written in single or double quotes: ``\'xyzzy\'``,\n``"frobozz"``. See *String literals* for more about string literals.\nUnicode strings are much like strings, but are specified in the syntax\nusing a preceding ``\'u\'`` character: ``u\'abc\'``, ``u"def"``. In\naddition to the functionality described here, there are also string-\nspecific methods described in the *String Methods* section. Lists are\nconstructed with square brackets, separating items with commas: ``[a,\nb, c]``. Tuples are constructed by the comma operator (not within\nsquare brackets), with or without enclosing parentheses, but an empty\ntuple must have the enclosing parentheses, such as ``a, b, c`` or\n``()``. A single item tuple must have a trailing comma, such as\n``(d,)``.\n\nBuffer objects are not directly supported by Python syntax, but can be\ncreated by calling the built-in function ``buffer()``. They don\'t\nsupport concatenation or repetition.\n\nObjects of type xrange are similar to buffers in that there is no\nspecific syntax to create them, but they are created using the\n``xrange()`` function. They don\'t support slicing, concatenation or\nrepetition, and using ``in``, ``not in``, ``min()`` or ``max()`` on\nthem is inefficient.\n\nMost sequence types support the following operations. The ``in`` and\n``not in`` operations have the same priorities as the comparison\noperations. The ``+`` and ``*`` operations have the same priority as\nthe corresponding numeric operations. [3] Additional methods are\nprovided for *Mutable Sequence Types*.\n\nThis table lists the sequence operations sorted in ascending priority\n(operations in the same box have the same priority). In the table,\n*s* and *t* are sequences of the same type; *n*, *i* and *j* are\nintegers:\n\n+--------------------+----------------------------------+------------+\n| Operation | Result | Notes |\n+====================+==================================+============+\n| ``x in s`` | ``True`` if an item of *s* is | (1) |\n| | equal to *x*, else ``False`` | |\n+--------------------+----------------------------------+------------+\n| ``x not in s`` | ``False`` if an item of *s* is | (1) |\n| | equal to *x*, else ``True`` | |\n+--------------------+----------------------------------+------------+\n| ``s + t`` | the concatenation of *s* and *t* | (6) |\n+--------------------+----------------------------------+------------+\n| ``s * n, n * s`` | *n* shallow copies of *s* | (2) |\n| | concatenated | |\n+--------------------+----------------------------------+------------+\n| ``s[i]`` | *i*\'th item of *s*, origin 0 | (3) |\n+--------------------+----------------------------------+------------+\n| ``s[i:j]`` | slice of *s* from *i* to *j* | (3)(4) |\n+--------------------+----------------------------------+------------+\n| ``s[i:j:k]`` | slice of *s* from *i* to *j* | (3)(5) |\n| | with step *k* | |\n+--------------------+----------------------------------+------------+\n| ``len(s)`` | length of *s* | |\n+--------------------+----------------------------------+------------+\n| ``min(s)`` | smallest item of *s* | |\n+--------------------+----------------------------------+------------+\n| ``max(s)`` | largest item of *s* | |\n+--------------------+----------------------------------+------------+\n\nSequence types also support comparisons. In particular, tuples and\nlists are compared lexicographically by comparing corresponding\nelements. This means that to compare equal, every element must compare\nequal and the two sequences must be of the same type and have the same\nlength. (For full details see *Comparisons* in the language\nreference.)\n\nNotes:\n\n1. When *s* is a string or Unicode string object the ``in`` and ``not\n in`` operations act like a substring test. In Python versions\n before 2.3, *x* had to be a string of length 1. In Python 2.3 and\n beyond, *x* may be a string of any length.\n\n2. Values of *n* less than ``0`` are treated as ``0`` (which yields an\n empty sequence of the same type as *s*). Note also that the copies\n are shallow; nested structures are not copied. This often haunts\n new Python programmers; consider:\n\n >>> lists = [[]] * 3\n >>> lists\n [[], [], []]\n >>> lists[0].append(3)\n >>> lists\n [[3], [3], [3]]\n\n What has happened is that ``[[]]`` is a one-element list containing\n an empty list, so all three elements of ``[[]] * 3`` are (pointers\n to) this single empty list. Modifying any of the elements of\n ``lists`` modifies this single list. You can create a list of\n different lists this way:\n\n >>> lists = [[] for i in range(3)]\n >>> lists[0].append(3)\n >>> lists[1].append(5)\n >>> lists[2].append(7)\n >>> lists\n [[3], [5], [7]]\n\n3. If *i* or *j* is negative, the index is relative to the end of the\n string: ``len(s) + i`` or ``len(s) + j`` is substituted. But note\n that ``-0`` is still ``0``.\n\n4. The slice of *s* from *i* to *j* is defined as the sequence of\n items with index *k* such that ``i <= k < j``. If *i* or *j* is\n greater than ``len(s)``, use ``len(s)``. If *i* is omitted or\n ``None``, use ``0``. If *j* is omitted or ``None``, use\n ``len(s)``. If *i* is greater than or equal to *j*, the slice is\n empty.\n\n5. The slice of *s* from *i* to *j* with step *k* is defined as the\n sequence of items with index ``x = i + n*k`` such that ``0 <= n <\n (j-i)/k``. In other words, the indices are ``i``, ``i+k``,\n ``i+2*k``, ``i+3*k`` and so on, stopping when *j* is reached (but\n never including *j*). If *i* or *j* is greater than ``len(s)``,\n use ``len(s)``. If *i* or *j* are omitted or ``None``, they become\n "end" values (which end depends on the sign of *k*). Note, *k*\n cannot be zero. If *k* is ``None``, it is treated like ``1``.\n\n6. **CPython implementation detail:** If *s* and *t* are both strings,\n some Python implementations such as CPython can usually perform an\n in-place optimization for assignments of the form ``s = s + t`` or\n ``s += t``. When applicable, this optimization makes quadratic\n run-time much less likely. This optimization is both version and\n implementation dependent. For performance sensitive code, it is\n preferable to use the ``str.join()`` method which assures\n consistent linear concatenation performance across versions and\n implementations.\n\n Changed in version 2.4: Formerly, string concatenation never\n occurred in-place.\n\n\nString Methods\n==============\n\nBelow are listed the string methods which both 8-bit strings and\nUnicode objects support.\n\nIn addition, Python\'s strings support the sequence type methods\ndescribed in the *Sequence Types --- str, unicode, list, tuple,\nbuffer, xrange* section. To output formatted strings use template\nstrings or the ``%`` operator described in the *String Formatting\nOperations* section. Also, see the ``re`` module for string functions\nbased on regular expressions.\n\nstr.capitalize()\n\n Return a copy of the string with only its first character\n capitalized.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.center(width[, fillchar])\n\n Return centered in a string of length *width*. Padding is done\n using the specified *fillchar* (default is a space).\n\n Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.count(sub[, start[, end]])\n\n Return the number of non-overlapping occurrences of substring *sub*\n in the range [*start*, *end*]. Optional arguments *start* and\n *end* are interpreted as in slice notation.\n\nstr.decode([encoding[, errors]])\n\n Decodes the string using the codec registered for *encoding*.\n *encoding* defaults to the default string encoding. *errors* may\n be given to set a different error handling scheme. The default is\n ``\'strict\'``, meaning that encoding errors raise ``UnicodeError``.\n Other possible values are ``\'ignore\'``, ``\'replace\'`` and any other\n name registered via ``codecs.register_error()``, see section *Codec\n Base Classes*.\n\n New in version 2.2.\n\n Changed in version 2.3: Support for other error handling schemes\n added.\n\n Changed in version 2.7: Support for keyword arguments added.\n\nstr.encode([encoding[, errors]])\n\n Return an encoded version of the string. Default encoding is the\n current default string encoding. *errors* may be given to set a\n different error handling scheme. The default for *errors* is\n ``\'strict\'``, meaning that encoding errors raise a\n ``UnicodeError``. Other possible values are ``\'ignore\'``,\n ``\'replace\'``, ``\'xmlcharrefreplace\'``, ``\'backslashreplace\'`` and\n any other name registered via ``codecs.register_error()``, see\n section *Codec Base Classes*. For a list of possible encodings, see\n section *Standard Encodings*.\n\n New in version 2.0.\n\n Changed in version 2.3: Support for ``\'xmlcharrefreplace\'`` and\n ``\'backslashreplace\'`` and other error handling schemes added.\n\n Changed in version 2.7: Support for keyword arguments added.\n\nstr.endswith(suffix[, start[, end]])\n\n Return ``True`` if the string ends with the specified *suffix*,\n otherwise return ``False``. *suffix* can also be a tuple of\n suffixes to look for. With optional *start*, test beginning at\n that position. With optional *end*, stop comparing at that\n position.\n\n Changed in version 2.5: Accept tuples as *suffix*.\n\nstr.expandtabs([tabsize])\n\n Return a copy of the string where all tab characters are replaced\n by one or more spaces, depending on the current column and the\n given tab size. The column number is reset to zero after each\n newline occurring in the string. If *tabsize* is not given, a tab\n size of ``8`` characters is assumed. This doesn\'t understand other\n non-printing characters or escape sequences.\n\nstr.find(sub[, start[, end]])\n\n Return the lowest index in the string where substring *sub* is\n found, such that *sub* is contained in the slice ``s[start:end]``.\n Optional arguments *start* and *end* are interpreted as in slice\n notation. Return ``-1`` if *sub* is not found.\n\nstr.format(*args, **kwargs)\n\n Perform a string formatting operation. The string on which this\n method is called can contain literal text or replacement fields\n delimited by braces ``{}``. Each replacement field contains either\n the numeric index of a positional argument, or the name of a\n keyword argument. Returns a copy of the string where each\n replacement field is replaced with the string value of the\n corresponding argument.\n\n >>> "The sum of 1 + 2 is {0}".format(1+2)\n \'The sum of 1 + 2 is 3\'\n\n See *Format String Syntax* for a description of the various\n formatting options that can be specified in format strings.\n\n This method of string formatting is the new standard in Python 3.0,\n and should be preferred to the ``%`` formatting described in\n *String Formatting Operations* in new code.\n\n New in version 2.6.\n\nstr.index(sub[, start[, end]])\n\n Like ``find()``, but raise ``ValueError`` when the substring is not\n found.\n\nstr.isalnum()\n\n Return true if all characters in the string are alphanumeric and\n there is at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isalpha()\n\n Return true if all characters in the string are alphabetic and\n there is at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isdigit()\n\n Return true if all characters in the string are digits and there is\n at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.islower()\n\n Return true if all cased characters in the string are lowercase and\n there is at least one cased character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isspace()\n\n Return true if there are only whitespace characters in the string\n and there is at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.istitle()\n\n Return true if the string is a titlecased string and there is at\n least one character, for example uppercase characters may only\n follow uncased characters and lowercase characters only cased ones.\n Return false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isupper()\n\n Return true if all cased characters in the string are uppercase and\n there is at least one cased character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.join(iterable)\n\n Return a string which is the concatenation of the strings in the\n *iterable* *iterable*. The separator between elements is the\n string providing this method.\n\nstr.ljust(width[, fillchar])\n\n Return the string left justified in a string of length *width*.\n Padding is done using the specified *fillchar* (default is a\n space). The original string is returned if *width* is less than\n ``len(s)``.\n\n Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.lower()\n\n Return a copy of the string converted to lowercase.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.lstrip([chars])\n\n Return a copy of the string with leading characters removed. The\n *chars* argument is a string specifying the set of characters to be\n removed. If omitted or ``None``, the *chars* argument defaults to\n removing whitespace. The *chars* argument is not a prefix; rather,\n all combinations of its values are stripped:\n\n >>> \' spacious \'.lstrip()\n \'spacious \'\n >>> \'www.example.com\'.lstrip(\'cmowz.\')\n \'example.com\'\n\n Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.partition(sep)\n\n Split the string at the first occurrence of *sep*, and return a\n 3-tuple containing the part before the separator, the separator\n itself, and the part after the separator. If the separator is not\n found, return a 3-tuple containing the string itself, followed by\n two empty strings.\n\n New in version 2.5.\n\nstr.replace(old, new[, count])\n\n Return a copy of the string with all occurrences of substring *old*\n replaced by *new*. If the optional argument *count* is given, only\n the first *count* occurrences are replaced.\n\nstr.rfind(sub[, start[, end]])\n\n Return the highest index in the string where substring *sub* is\n found, such that *sub* is contained within ``s[start:end]``.\n Optional arguments *start* and *end* are interpreted as in slice\n notation. Return ``-1`` on failure.\n\nstr.rindex(sub[, start[, end]])\n\n Like ``rfind()`` but raises ``ValueError`` when the substring *sub*\n is not found.\n\nstr.rjust(width[, fillchar])\n\n Return the string right justified in a string of length *width*.\n Padding is done using the specified *fillchar* (default is a\n space). The original string is returned if *width* is less than\n ``len(s)``.\n\n Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.rpartition(sep)\n\n Split the string at the last occurrence of *sep*, and return a\n 3-tuple containing the part before the separator, the separator\n itself, and the part after the separator. If the separator is not\n found, return a 3-tuple containing two empty strings, followed by\n the string itself.\n\n New in version 2.5.\n\nstr.rsplit([sep[, maxsplit]])\n\n Return a list of the words in the string, using *sep* as the\n delimiter string. If *maxsplit* is given, at most *maxsplit* splits\n are done, the *rightmost* ones. If *sep* is not specified or\n ``None``, any whitespace string is a separator. Except for\n splitting from the right, ``rsplit()`` behaves like ``split()``\n which is described in detail below.\n\n New in version 2.4.\n\nstr.rstrip([chars])\n\n Return a copy of the string with trailing characters removed. The\n *chars* argument is a string specifying the set of characters to be\n removed. If omitted or ``None``, the *chars* argument defaults to\n removing whitespace. The *chars* argument is not a suffix; rather,\n all combinations of its values are stripped:\n\n >>> \' spacious \'.rstrip()\n \' spacious\'\n >>> \'mississippi\'.rstrip(\'ipz\')\n \'mississ\'\n\n Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.split([sep[, maxsplit]])\n\n Return a list of the words in the string, using *sep* as the\n delimiter string. If *maxsplit* is given, at most *maxsplit*\n splits are done (thus, the list will have at most ``maxsplit+1``\n elements). If *maxsplit* is not specified, then there is no limit\n on the number of splits (all possible splits are made).\n\n If *sep* is given, consecutive delimiters are not grouped together\n and are deemed to delimit empty strings (for example,\n ``\'1,,2\'.split(\',\')`` returns ``[\'1\', \'\', \'2\']``). The *sep*\n argument may consist of multiple characters (for example,\n ``\'1<>2<>3\'.split(\'<>\')`` returns ``[\'1\', \'2\', \'3\']``). Splitting\n an empty string with a specified separator returns ``[\'\']``.\n\n If *sep* is not specified or is ``None``, a different splitting\n algorithm is applied: runs of consecutive whitespace are regarded\n as a single separator, and the result will contain no empty strings\n at the start or end if the string has leading or trailing\n whitespace. Consequently, splitting an empty string or a string\n consisting of just whitespace with a ``None`` separator returns\n ``[]``.\n\n For example, ``\' 1 2 3 \'.split()`` returns ``[\'1\', \'2\', \'3\']``,\n and ``\' 1 2 3 \'.split(None, 1)`` returns ``[\'1\', \'2 3 \']``.\n\nstr.splitlines([keepends])\n\n Return a list of the lines in the string, breaking at line\n boundaries. Line breaks are not included in the resulting list\n unless *keepends* is given and true.\n\nstr.startswith(prefix[, start[, end]])\n\n Return ``True`` if string starts with the *prefix*, otherwise\n return ``False``. *prefix* can also be a tuple of prefixes to look\n for. With optional *start*, test string beginning at that\n position. With optional *end*, stop comparing string at that\n position.\n\n Changed in version 2.5: Accept tuples as *prefix*.\n\nstr.strip([chars])\n\n Return a copy of the string with the leading and trailing\n characters removed. The *chars* argument is a string specifying the\n set of characters to be removed. If omitted or ``None``, the\n *chars* argument defaults to removing whitespace. The *chars*\n argument is not a prefix or suffix; rather, all combinations of its\n values are stripped:\n\n >>> \' spacious \'.strip()\n \'spacious\'\n >>> \'www.example.com\'.strip(\'cmowz.\')\n \'example\'\n\n Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.swapcase()\n\n Return a copy of the string with uppercase characters converted to\n lowercase and vice versa.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.title()\n\n Return a titlecased version of the string where words start with an\n uppercase character and the remaining characters are lowercase.\n\n The algorithm uses a simple language-independent definition of a\n word as groups of consecutive letters. The definition works in\n many contexts but it means that apostrophes in contractions and\n possessives form word boundaries, which may not be the desired\n result:\n\n >>> "they\'re bill\'s friends from the UK".title()\n "They\'Re Bill\'S Friends From The Uk"\n\n A workaround for apostrophes can be constructed using regular\n expressions:\n\n >>> import re\n >>> def titlecase(s):\n return re.sub(r"[A-Za-z]+(\'[A-Za-z]+)?",\n lambda mo: mo.group(0)[0].upper() +\n mo.group(0)[1:].lower(),\n s)\n\n >>> titlecase("they\'re bill\'s friends.")\n "They\'re Bill\'s Friends."\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.translate(table[, deletechars])\n\n Return a copy of the string where all characters occurring in the\n optional argument *deletechars* are removed, and the remaining\n characters have been mapped through the given translation table,\n which must be a string of length 256.\n\n You can use the ``maketrans()`` helper function in the ``string``\n module to create a translation table. For string objects, set the\n *table* argument to ``None`` for translations that only delete\n characters:\n\n >>> \'read this short text\'.translate(None, \'aeiou\')\n \'rd ths shrt txt\'\n\n New in version 2.6: Support for a ``None`` *table* argument.\n\n For Unicode objects, the ``translate()`` method does not accept the\n optional *deletechars* argument. Instead, it returns a copy of the\n *s* where all characters have been mapped through the given\n translation table which must be a mapping of Unicode ordinals to\n Unicode ordinals, Unicode strings or ``None``. Unmapped characters\n are left untouched. Characters mapped to ``None`` are deleted.\n Note, a more flexible approach is to create a custom character\n mapping codec using the ``codecs`` module (see ``encodings.cp1251``\n for an example).\n\nstr.upper()\n\n Return a copy of the string converted to uppercase.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.zfill(width)\n\n Return the numeric string left filled with zeros in a string of\n length *width*. A sign prefix is handled correctly. The original\n string is returned if *width* is less than ``len(s)``.\n\n New in version 2.2.2.\n\nThe following methods are present only on unicode objects:\n\nunicode.isnumeric()\n\n Return ``True`` if there are only numeric characters in S,\n ``False`` otherwise. Numeric characters include digit characters,\n and all characters that have the Unicode numeric value property,\n e.g. U+2155, VULGAR FRACTION ONE FIFTH.\n\nunicode.isdecimal()\n\n Return ``True`` if there are only decimal characters in S,\n ``False`` otherwise. Decimal characters include digit characters,\n and all characters that that can be used to form decimal-radix\n numbers, e.g. U+0660, ARABIC-INDIC DIGIT ZERO.\n\n\nString Formatting Operations\n============================\n\nString and Unicode objects have one unique built-in operation: the\n``%`` operator (modulo). This is also known as the string\n*formatting* or *interpolation* operator. Given ``format % values``\n(where *format* is a string or Unicode object), ``%`` conversion\nspecifications in *format* are replaced with zero or more elements of\n*values*. The effect is similar to the using ``sprintf()`` in the C\nlanguage. If *format* is a Unicode object, or if any of the objects\nbeing converted using the ``%s`` conversion are Unicode objects, the\nresult will also be a Unicode object.\n\nIf *format* requires a single argument, *values* may be a single non-\ntuple object. [4] Otherwise, *values* must be a tuple with exactly\nthe number of items specified by the format string, or a single\nmapping object (for example, a dictionary).\n\nA conversion specifier contains two or more characters and has the\nfollowing components, which must occur in this order:\n\n1. The ``\'%\'`` character, which marks the start of the specifier.\n\n2. Mapping key (optional), consisting of a parenthesised sequence of\n characters (for example, ``(somename)``).\n\n3. Conversion flags (optional), which affect the result of some\n conversion types.\n\n4. Minimum field width (optional). If specified as an ``\'*\'``\n (asterisk), the actual width is read from the next element of the\n tuple in *values*, and the object to convert comes after the\n minimum field width and optional precision.\n\n5. Precision (optional), given as a ``\'.\'`` (dot) followed by the\n precision. If specified as ``\'*\'`` (an asterisk), the actual width\n is read from the next element of the tuple in *values*, and the\n value to convert comes after the precision.\n\n6. Length modifier (optional).\n\n7. Conversion type.\n\nWhen the right argument is a dictionary (or other mapping type), then\nthe formats in the string *must* include a parenthesised mapping key\ninto that dictionary inserted immediately after the ``\'%\'`` character.\nThe mapping key selects the value to be formatted from the mapping.\nFor example:\n\n>>> print \'%(language)s has %(#)03d quote types.\' % \\\n... {\'language\': "Python", "#": 2}\nPython has 002 quote types.\n\nIn this case no ``*`` specifiers may occur in a format (since they\nrequire a sequential parameter list).\n\nThe conversion flag characters are:\n\n+-----------+-----------------------------------------------------------------------+\n| Flag | Meaning |\n+===========+=======================================================================+\n| ``\'#\'`` | The value conversion will use the "alternate form" (where defined |\n| | below). |\n+-----------+-----------------------------------------------------------------------+\n| ``\'0\'`` | The conversion will be zero padded for numeric values. |\n+-----------+-----------------------------------------------------------------------+\n| ``\'-\'`` | The converted value is left adjusted (overrides the ``\'0\'`` |\n| | conversion if both are given). |\n+-----------+-----------------------------------------------------------------------+\n| ``\' \'`` | (a space) A blank should be left before a positive number (or empty |\n| | string) produced by a signed conversion. |\n+-----------+-----------------------------------------------------------------------+\n| ``\'+\'`` | A sign character (``\'+\'`` or ``\'-\'``) will precede the conversion |\n| | (overrides a "space" flag). |\n+-----------+-----------------------------------------------------------------------+\n\nA length modifier (``h``, ``l``, or ``L``) may be present, but is\nignored as it is not necessary for Python -- so e.g. ``%ld`` is\nidentical to ``%d``.\n\nThe conversion types are:\n\n+--------------+-------------------------------------------------------+---------+\n| Conversion | Meaning | Notes |\n+==============+=======================================================+=========+\n| ``\'d\'`` | Signed integer decimal. | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'i\'`` | Signed integer decimal. | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'o\'`` | Signed octal value. | (1) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'u\'`` | Obsolete type -- it is identical to ``\'d\'``. | (7) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'x\'`` | Signed hexadecimal (lowercase). | (2) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'X\'`` | Signed hexadecimal (uppercase). | (2) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'e\'`` | Floating point exponential format (lowercase). | (3) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'E\'`` | Floating point exponential format (uppercase). | (3) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'f\'`` | Floating point decimal format. | (3) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'F\'`` | Floating point decimal format. | (3) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'g\'`` | Floating point format. Uses lowercase exponential | (4) |\n| | format if exponent is less than -4 or not less than | |\n| | precision, decimal format otherwise. | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'G\'`` | Floating point format. Uses uppercase exponential | (4) |\n| | format if exponent is less than -4 or not less than | |\n| | precision, decimal format otherwise. | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'c\'`` | Single character (accepts integer or single character | |\n| | string). | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'r\'`` | String (converts any Python object using ``repr()``). | (5) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'s\'`` | String (converts any Python object using ``str()``). | (6) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'%\'`` | No argument is converted, results in a ``\'%\'`` | |\n| | character in the result. | |\n+--------------+-------------------------------------------------------+---------+\n\nNotes:\n\n1. The alternate form causes a leading zero (``\'0\'``) to be inserted\n between left-hand padding and the formatting of the number if the\n leading character of the result is not already a zero.\n\n2. The alternate form causes a leading ``\'0x\'`` or ``\'0X\'`` (depending\n on whether the ``\'x\'`` or ``\'X\'`` format was used) to be inserted\n between left-hand padding and the formatting of the number if the\n leading character of the result is not already a zero.\n\n3. The alternate form causes the result to always contain a decimal\n point, even if no digits follow it.\n\n The precision determines the number of digits after the decimal\n point and defaults to 6.\n\n4. The alternate form causes the result to always contain a decimal\n point, and trailing zeroes are not removed as they would otherwise\n be.\n\n The precision determines the number of significant digits before\n and after the decimal point and defaults to 6.\n\n5. The ``%r`` conversion was added in Python 2.0.\n\n The precision determines the maximal number of characters used.\n\n6. If the object or format provided is a ``unicode`` string, the\n resulting string will also be ``unicode``.\n\n The precision determines the maximal number of characters used.\n\n7. See **PEP 237**.\n\nSince Python strings have an explicit length, ``%s`` conversions do\nnot assume that ``\'\\0\'`` is the end of the string.\n\nChanged in version 2.7: ``%f`` conversions for numbers whose absolute\nvalue is over 1e50 are no longer replaced by ``%g`` conversions.\n\nAdditional string operations are defined in standard modules\n``string`` and ``re``.\n\n\nXRange Type\n===========\n\nThe ``xrange`` type is an immutable sequence which is commonly used\nfor looping. The advantage of the ``xrange`` type is that an\n``xrange`` object will always take the same amount of memory, no\nmatter the size of the range it represents. There are no consistent\nperformance advantages.\n\nXRange objects have very little behavior: they only support indexing,\niteration, and the ``len()`` function.\n\n\nMutable Sequence Types\n======================\n\nList objects support additional operations that allow in-place\nmodification of the object. Other mutable sequence types (when added\nto the language) should also support these operations. Strings and\ntuples are immutable sequence types: such objects cannot be modified\nonce created. The following operations are defined on mutable sequence\ntypes (where *x* is an arbitrary object):\n\n+--------------------------------+----------------------------------+-----------------------+\n| Operation | Result | Notes |\n+================================+==================================+=======================+\n| ``s[i] = x`` | item *i* of *s* is replaced by | |\n| | *x* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s[i:j] = t`` | slice of *s* from *i* to *j* is | |\n| | replaced by the contents of the | |\n| | iterable *t* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``del s[i:j]`` | same as ``s[i:j] = []`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s[i:j:k] = t`` | the elements of ``s[i:j:k]`` are | (1) |\n| | replaced by those of *t* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``del s[i:j:k]`` | removes the elements of | |\n| | ``s[i:j:k]`` from the list | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.append(x)`` | same as ``s[len(s):len(s)] = | (2) |\n| | [x]`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.extend(x)`` | same as ``s[len(s):len(s)] = x`` | (3) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.count(x)`` | return number of *i*\'s for which | |\n| | ``s[i] == x`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.index(x[, i[, j]])`` | return smallest *k* such that | (4) |\n| | ``s[k] == x`` and ``i <= k < j`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.insert(i, x)`` | same as ``s[i:i] = [x]`` | (5) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.pop([i])`` | same as ``x = s[i]; del s[i]; | (6) |\n| | return x`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.remove(x)`` | same as ``del s[s.index(x)]`` | (4) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.reverse()`` | reverses the items of *s* in | (7) |\n| | place | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.sort([cmp[, key[, | sort the items of *s* in place | (7)(8)(9)(10) |\n| reverse]]])`` | | |\n+--------------------------------+----------------------------------+-----------------------+\n\nNotes:\n\n1. *t* must have the same length as the slice it is replacing.\n\n2. The C implementation of Python has historically accepted multiple\n parameters and implicitly joined them into a tuple; this no longer\n works in Python 2.0. Use of this misfeature has been deprecated\n since Python 1.4.\n\n3. *x* can be any iterable object.\n\n4. Raises ``ValueError`` when *x* is not found in *s*. When a negative\n index is passed as the second or third parameter to the ``index()``\n method, the list length is added, as for slice indices. If it is\n still negative, it is truncated to zero, as for slice indices.\n\n Changed in version 2.3: Previously, ``index()`` didn\'t have\n arguments for specifying start and stop positions.\n\n5. When a negative index is passed as the first parameter to the\n ``insert()`` method, the list length is added, as for slice\n indices. If it is still negative, it is truncated to zero, as for\n slice indices.\n\n Changed in version 2.3: Previously, all negative indices were\n truncated to zero.\n\n6. The ``pop()`` method is only supported by the list and array types.\n The optional argument *i* defaults to ``-1``, so that by default\n the last item is removed and returned.\n\n7. The ``sort()`` and ``reverse()`` methods modify the list in place\n for economy of space when sorting or reversing a large list. To\n remind you that they operate by side effect, they don\'t return the\n sorted or reversed list.\n\n8. The ``sort()`` method takes optional arguments for controlling the\n comparisons.\n\n *cmp* specifies a custom comparison function of two arguments (list\n items) which should return a negative, zero or positive number\n depending on whether the first argument is considered smaller than,\n equal to, or larger than the second argument: ``cmp=lambda x,y:\n cmp(x.lower(), y.lower())``. The default value is ``None``.\n\n *key* specifies a function of one argument that is used to extract\n a comparison key from each list element: ``key=str.lower``. The\n default value is ``None``.\n\n *reverse* is a boolean value. If set to ``True``, then the list\n elements are sorted as if each comparison were reversed.\n\n In general, the *key* and *reverse* conversion processes are much\n faster than specifying an equivalent *cmp* function. This is\n because *cmp* is called multiple times for each list element while\n *key* and *reverse* touch each element only once. Use\n ``functools.cmp_to_key()`` to convert an old-style *cmp* function\n to a *key* function.\n\n Changed in version 2.3: Support for ``None`` as an equivalent to\n omitting *cmp* was added.\n\n Changed in version 2.4: Support for *key* and *reverse* was added.\n\n9. Starting with Python 2.3, the ``sort()`` method is guaranteed to be\n stable. A sort is stable if it guarantees not to change the\n relative order of elements that compare equal --- this is helpful\n for sorting in multiple passes (for example, sort by department,\n then by salary grade).\n\n10. **CPython implementation detail:** While a list is being sorted,\n the effect of attempting to mutate, or even inspect, the list is\n undefined. The C implementation of Python 2.3 and newer makes the\n list appear empty for the duration, and raises ``ValueError`` if\n it can detect that the list has been mutated during a sort.\n', - 'typesseq-mutable': u"\nMutable Sequence Types\n**********************\n\nList objects support additional operations that allow in-place\nmodification of the object. Other mutable sequence types (when added\nto the language) should also support these operations. Strings and\ntuples are immutable sequence types: such objects cannot be modified\nonce created. The following operations are defined on mutable sequence\ntypes (where *x* is an arbitrary object):\n\n+--------------------------------+----------------------------------+-----------------------+\n| Operation | Result | Notes |\n+================================+==================================+=======================+\n| ``s[i] = x`` | item *i* of *s* is replaced by | |\n| | *x* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s[i:j] = t`` | slice of *s* from *i* to *j* is | |\n| | replaced by the contents of the | |\n| | iterable *t* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``del s[i:j]`` | same as ``s[i:j] = []`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s[i:j:k] = t`` | the elements of ``s[i:j:k]`` are | (1) |\n| | replaced by those of *t* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``del s[i:j:k]`` | removes the elements of | |\n| | ``s[i:j:k]`` from the list | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.append(x)`` | same as ``s[len(s):len(s)] = | (2) |\n| | [x]`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.extend(x)`` | same as ``s[len(s):len(s)] = x`` | (3) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.count(x)`` | return number of *i*'s for which | |\n| | ``s[i] == x`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.index(x[, i[, j]])`` | return smallest *k* such that | (4) |\n| | ``s[k] == x`` and ``i <= k < j`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.insert(i, x)`` | same as ``s[i:i] = [x]`` | (5) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.pop([i])`` | same as ``x = s[i]; del s[i]; | (6) |\n| | return x`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.remove(x)`` | same as ``del s[s.index(x)]`` | (4) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.reverse()`` | reverses the items of *s* in | (7) |\n| | place | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.sort([cmp[, key[, | sort the items of *s* in place | (7)(8)(9)(10) |\n| reverse]]])`` | | |\n+--------------------------------+----------------------------------+-----------------------+\n\nNotes:\n\n1. *t* must have the same length as the slice it is replacing.\n\n2. The C implementation of Python has historically accepted multiple\n parameters and implicitly joined them into a tuple; this no longer\n works in Python 2.0. Use of this misfeature has been deprecated\n since Python 1.4.\n\n3. *x* can be any iterable object.\n\n4. Raises ``ValueError`` when *x* is not found in *s*. When a negative\n index is passed as the second or third parameter to the ``index()``\n method, the list length is added, as for slice indices. If it is\n still negative, it is truncated to zero, as for slice indices.\n\n Changed in version 2.3: Previously, ``index()`` didn't have\n arguments for specifying start and stop positions.\n\n5. When a negative index is passed as the first parameter to the\n ``insert()`` method, the list length is added, as for slice\n indices. If it is still negative, it is truncated to zero, as for\n slice indices.\n\n Changed in version 2.3: Previously, all negative indices were\n truncated to zero.\n\n6. The ``pop()`` method is only supported by the list and array types.\n The optional argument *i* defaults to ``-1``, so that by default\n the last item is removed and returned.\n\n7. The ``sort()`` and ``reverse()`` methods modify the list in place\n for economy of space when sorting or reversing a large list. To\n remind you that they operate by side effect, they don't return the\n sorted or reversed list.\n\n8. The ``sort()`` method takes optional arguments for controlling the\n comparisons.\n\n *cmp* specifies a custom comparison function of two arguments (list\n items) which should return a negative, zero or positive number\n depending on whether the first argument is considered smaller than,\n equal to, or larger than the second argument: ``cmp=lambda x,y:\n cmp(x.lower(), y.lower())``. The default value is ``None``.\n\n *key* specifies a function of one argument that is used to extract\n a comparison key from each list element: ``key=str.lower``. The\n default value is ``None``.\n\n *reverse* is a boolean value. If set to ``True``, then the list\n elements are sorted as if each comparison were reversed.\n\n In general, the *key* and *reverse* conversion processes are much\n faster than specifying an equivalent *cmp* function. This is\n because *cmp* is called multiple times for each list element while\n *key* and *reverse* touch each element only once. Use\n ``functools.cmp_to_key()`` to convert an old-style *cmp* function\n to a *key* function.\n\n Changed in version 2.3: Support for ``None`` as an equivalent to\n omitting *cmp* was added.\n\n Changed in version 2.4: Support for *key* and *reverse* was added.\n\n9. Starting with Python 2.3, the ``sort()`` method is guaranteed to be\n stable. A sort is stable if it guarantees not to change the\n relative order of elements that compare equal --- this is helpful\n for sorting in multiple passes (for example, sort by department,\n then by salary grade).\n\n10. **CPython implementation detail:** While a list is being sorted,\n the effect of attempting to mutate, or even inspect, the list is\n undefined. The C implementation of Python 2.3 and newer makes the\n list appear empty for the duration, and raises ``ValueError`` if\n it can detect that the list has been mutated during a sort.\n", + 'typesseq': u'\nSequence Types --- ``str``, ``unicode``, ``list``, ``tuple``, ``bytearray``, ``buffer``, ``xrange``\n***************************************************************************************************\n\nThere are seven sequence types: strings, Unicode strings, lists,\ntuples, bytearrays, buffers, and xrange objects.\n\nFor other containers see the built in ``dict`` and ``set`` classes,\nand the ``collections`` module.\n\nString literals are written in single or double quotes: ``\'xyzzy\'``,\n``"frobozz"``. See *String literals* for more about string literals.\nUnicode strings are much like strings, but are specified in the syntax\nusing a preceding ``\'u\'`` character: ``u\'abc\'``, ``u"def"``. In\naddition to the functionality described here, there are also string-\nspecific methods described in the *String Methods* section. Lists are\nconstructed with square brackets, separating items with commas: ``[a,\nb, c]``. Tuples are constructed by the comma operator (not within\nsquare brackets), with or without enclosing parentheses, but an empty\ntuple must have the enclosing parentheses, such as ``a, b, c`` or\n``()``. A single item tuple must have a trailing comma, such as\n``(d,)``.\n\nBytearray objects are created with the built-in function\n``bytearray()``.\n\nBuffer objects are not directly supported by Python syntax, but can be\ncreated by calling the built-in function ``buffer()``. They don\'t\nsupport concatenation or repetition.\n\nObjects of type xrange are similar to buffers in that there is no\nspecific syntax to create them, but they are created using the\n``xrange()`` function. They don\'t support slicing, concatenation or\nrepetition, and using ``in``, ``not in``, ``min()`` or ``max()`` on\nthem is inefficient.\n\nMost sequence types support the following operations. The ``in`` and\n``not in`` operations have the same priorities as the comparison\noperations. The ``+`` and ``*`` operations have the same priority as\nthe corresponding numeric operations. [3] Additional methods are\nprovided for *Mutable Sequence Types*.\n\nThis table lists the sequence operations sorted in ascending priority\n(operations in the same box have the same priority). In the table,\n*s* and *t* are sequences of the same type; *n*, *i* and *j* are\nintegers:\n\n+--------------------+----------------------------------+------------+\n| Operation | Result | Notes |\n+====================+==================================+============+\n| ``x in s`` | ``True`` if an item of *s* is | (1) |\n| | equal to *x*, else ``False`` | |\n+--------------------+----------------------------------+------------+\n| ``x not in s`` | ``False`` if an item of *s* is | (1) |\n| | equal to *x*, else ``True`` | |\n+--------------------+----------------------------------+------------+\n| ``s + t`` | the concatenation of *s* and *t* | (6) |\n+--------------------+----------------------------------+------------+\n| ``s * n, n * s`` | *n* shallow copies of *s* | (2) |\n| | concatenated | |\n+--------------------+----------------------------------+------------+\n| ``s[i]`` | *i*\'th item of *s*, origin 0 | (3) |\n+--------------------+----------------------------------+------------+\n| ``s[i:j]`` | slice of *s* from *i* to *j* | (3)(4) |\n+--------------------+----------------------------------+------------+\n| ``s[i:j:k]`` | slice of *s* from *i* to *j* | (3)(5) |\n| | with step *k* | |\n+--------------------+----------------------------------+------------+\n| ``len(s)`` | length of *s* | |\n+--------------------+----------------------------------+------------+\n| ``min(s)`` | smallest item of *s* | |\n+--------------------+----------------------------------+------------+\n| ``max(s)`` | largest item of *s* | |\n+--------------------+----------------------------------+------------+\n| ``s.index(i)`` | index of the first occurence of | |\n| | *i* in *s* | |\n+--------------------+----------------------------------+------------+\n| ``s.count(i)`` | total number of occurences of | |\n| | *i* in *s* | |\n+--------------------+----------------------------------+------------+\n\nSequence types also support comparisons. In particular, tuples and\nlists are compared lexicographically by comparing corresponding\nelements. This means that to compare equal, every element must compare\nequal and the two sequences must be of the same type and have the same\nlength. (For full details see *Comparisons* in the language\nreference.)\n\nNotes:\n\n1. When *s* is a string or Unicode string object the ``in`` and ``not\n in`` operations act like a substring test. In Python versions\n before 2.3, *x* had to be a string of length 1. In Python 2.3 and\n beyond, *x* may be a string of any length.\n\n2. Values of *n* less than ``0`` are treated as ``0`` (which yields an\n empty sequence of the same type as *s*). Note also that the copies\n are shallow; nested structures are not copied. This often haunts\n new Python programmers; consider:\n\n >>> lists = [[]] * 3\n >>> lists\n [[], [], []]\n >>> lists[0].append(3)\n >>> lists\n [[3], [3], [3]]\n\n What has happened is that ``[[]]`` is a one-element list containing\n an empty list, so all three elements of ``[[]] * 3`` are (pointers\n to) this single empty list. Modifying any of the elements of\n ``lists`` modifies this single list. You can create a list of\n different lists this way:\n\n >>> lists = [[] for i in range(3)]\n >>> lists[0].append(3)\n >>> lists[1].append(5)\n >>> lists[2].append(7)\n >>> lists\n [[3], [5], [7]]\n\n3. If *i* or *j* is negative, the index is relative to the end of the\n string: ``len(s) + i`` or ``len(s) + j`` is substituted. But note\n that ``-0`` is still ``0``.\n\n4. The slice of *s* from *i* to *j* is defined as the sequence of\n items with index *k* such that ``i <= k < j``. If *i* or *j* is\n greater than ``len(s)``, use ``len(s)``. If *i* is omitted or\n ``None``, use ``0``. If *j* is omitted or ``None``, use\n ``len(s)``. If *i* is greater than or equal to *j*, the slice is\n empty.\n\n5. The slice of *s* from *i* to *j* with step *k* is defined as the\n sequence of items with index ``x = i + n*k`` such that ``0 <= n <\n (j-i)/k``. In other words, the indices are ``i``, ``i+k``,\n ``i+2*k``, ``i+3*k`` and so on, stopping when *j* is reached (but\n never including *j*). If *i* or *j* is greater than ``len(s)``,\n use ``len(s)``. If *i* or *j* are omitted or ``None``, they become\n "end" values (which end depends on the sign of *k*). Note, *k*\n cannot be zero. If *k* is ``None``, it is treated like ``1``.\n\n6. **CPython implementation detail:** If *s* and *t* are both strings,\n some Python implementations such as CPython can usually perform an\n in-place optimization for assignments of the form ``s = s + t`` or\n ``s += t``. When applicable, this optimization makes quadratic\n run-time much less likely. This optimization is both version and\n implementation dependent. For performance sensitive code, it is\n preferable to use the ``str.join()`` method which assures\n consistent linear concatenation performance across versions and\n implementations.\n\n Changed in version 2.4: Formerly, string concatenation never\n occurred in-place.\n\n\nString Methods\n==============\n\nBelow are listed the string methods which both 8-bit strings and\nUnicode objects support. Some of them are also available on\n``bytearray`` objects.\n\nIn addition, Python\'s strings support the sequence type methods\ndescribed in the *Sequence Types --- str, unicode, list, tuple,\nbytearray, buffer, xrange* section. To output formatted strings use\ntemplate strings or the ``%`` operator described in the *String\nFormatting Operations* section. Also, see the ``re`` module for string\nfunctions based on regular expressions.\n\nstr.capitalize()\n\n Return a copy of the string with its first character capitalized\n and the rest lowercased.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.center(width[, fillchar])\n\n Return centered in a string of length *width*. Padding is done\n using the specified *fillchar* (default is a space).\n\n Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.count(sub[, start[, end]])\n\n Return the number of non-overlapping occurrences of substring *sub*\n in the range [*start*, *end*]. Optional arguments *start* and\n *end* are interpreted as in slice notation.\n\nstr.decode([encoding[, errors]])\n\n Decodes the string using the codec registered for *encoding*.\n *encoding* defaults to the default string encoding. *errors* may\n be given to set a different error handling scheme. The default is\n ``\'strict\'``, meaning that encoding errors raise ``UnicodeError``.\n Other possible values are ``\'ignore\'``, ``\'replace\'`` and any other\n name registered via ``codecs.register_error()``, see section *Codec\n Base Classes*.\n\n New in version 2.2.\n\n Changed in version 2.3: Support for other error handling schemes\n added.\n\n Changed in version 2.7: Support for keyword arguments added.\n\nstr.encode([encoding[, errors]])\n\n Return an encoded version of the string. Default encoding is the\n current default string encoding. *errors* may be given to set a\n different error handling scheme. The default for *errors* is\n ``\'strict\'``, meaning that encoding errors raise a\n ``UnicodeError``. Other possible values are ``\'ignore\'``,\n ``\'replace\'``, ``\'xmlcharrefreplace\'``, ``\'backslashreplace\'`` and\n any other name registered via ``codecs.register_error()``, see\n section *Codec Base Classes*. For a list of possible encodings, see\n section *Standard Encodings*.\n\n New in version 2.0.\n\n Changed in version 2.3: Support for ``\'xmlcharrefreplace\'`` and\n ``\'backslashreplace\'`` and other error handling schemes added.\n\n Changed in version 2.7: Support for keyword arguments added.\n\nstr.endswith(suffix[, start[, end]])\n\n Return ``True`` if the string ends with the specified *suffix*,\n otherwise return ``False``. *suffix* can also be a tuple of\n suffixes to look for. With optional *start*, test beginning at\n that position. With optional *end*, stop comparing at that\n position.\n\n Changed in version 2.5: Accept tuples as *suffix*.\n\nstr.expandtabs([tabsize])\n\n Return a copy of the string where all tab characters are replaced\n by one or more spaces, depending on the current column and the\n given tab size. The column number is reset to zero after each\n newline occurring in the string. If *tabsize* is not given, a tab\n size of ``8`` characters is assumed. This doesn\'t understand other\n non-printing characters or escape sequences.\n\nstr.find(sub[, start[, end]])\n\n Return the lowest index in the string where substring *sub* is\n found, such that *sub* is contained in the slice ``s[start:end]``.\n Optional arguments *start* and *end* are interpreted as in slice\n notation. Return ``-1`` if *sub* is not found.\n\n Note: The ``find()`` method should be used only if you need to know the\n position of *sub*. To check if *sub* is a substring or not, use\n the ``in`` operator:\n\n >>> \'Py\' in \'Python\'\n True\n\nstr.format(*args, **kwargs)\n\n Perform a string formatting operation. The string on which this\n method is called can contain literal text or replacement fields\n delimited by braces ``{}``. Each replacement field contains either\n the numeric index of a positional argument, or the name of a\n keyword argument. Returns a copy of the string where each\n replacement field is replaced with the string value of the\n corresponding argument.\n\n >>> "The sum of 1 + 2 is {0}".format(1+2)\n \'The sum of 1 + 2 is 3\'\n\n See *Format String Syntax* for a description of the various\n formatting options that can be specified in format strings.\n\n This method of string formatting is the new standard in Python 3.0,\n and should be preferred to the ``%`` formatting described in\n *String Formatting Operations* in new code.\n\n New in version 2.6.\n\nstr.index(sub[, start[, end]])\n\n Like ``find()``, but raise ``ValueError`` when the substring is not\n found.\n\nstr.isalnum()\n\n Return true if all characters in the string are alphanumeric and\n there is at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isalpha()\n\n Return true if all characters in the string are alphabetic and\n there is at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isdigit()\n\n Return true if all characters in the string are digits and there is\n at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.islower()\n\n Return true if all cased characters in the string are lowercase and\n there is at least one cased character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isspace()\n\n Return true if there are only whitespace characters in the string\n and there is at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.istitle()\n\n Return true if the string is a titlecased string and there is at\n least one character, for example uppercase characters may only\n follow uncased characters and lowercase characters only cased ones.\n Return false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isupper()\n\n Return true if all cased characters in the string are uppercase and\n there is at least one cased character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.join(iterable)\n\n Return a string which is the concatenation of the strings in the\n *iterable* *iterable*. The separator between elements is the\n string providing this method.\n\nstr.ljust(width[, fillchar])\n\n Return the string left justified in a string of length *width*.\n Padding is done using the specified *fillchar* (default is a\n space). The original string is returned if *width* is less than\n ``len(s)``.\n\n Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.lower()\n\n Return a copy of the string converted to lowercase.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.lstrip([chars])\n\n Return a copy of the string with leading characters removed. The\n *chars* argument is a string specifying the set of characters to be\n removed. If omitted or ``None``, the *chars* argument defaults to\n removing whitespace. The *chars* argument is not a prefix; rather,\n all combinations of its values are stripped:\n\n >>> \' spacious \'.lstrip()\n \'spacious \'\n >>> \'www.example.com\'.lstrip(\'cmowz.\')\n \'example.com\'\n\n Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.partition(sep)\n\n Split the string at the first occurrence of *sep*, and return a\n 3-tuple containing the part before the separator, the separator\n itself, and the part after the separator. If the separator is not\n found, return a 3-tuple containing the string itself, followed by\n two empty strings.\n\n New in version 2.5.\n\nstr.replace(old, new[, count])\n\n Return a copy of the string with all occurrences of substring *old*\n replaced by *new*. If the optional argument *count* is given, only\n the first *count* occurrences are replaced.\n\nstr.rfind(sub[, start[, end]])\n\n Return the highest index in the string where substring *sub* is\n found, such that *sub* is contained within ``s[start:end]``.\n Optional arguments *start* and *end* are interpreted as in slice\n notation. Return ``-1`` on failure.\n\nstr.rindex(sub[, start[, end]])\n\n Like ``rfind()`` but raises ``ValueError`` when the substring *sub*\n is not found.\n\nstr.rjust(width[, fillchar])\n\n Return the string right justified in a string of length *width*.\n Padding is done using the specified *fillchar* (default is a\n space). The original string is returned if *width* is less than\n ``len(s)``.\n\n Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.rpartition(sep)\n\n Split the string at the last occurrence of *sep*, and return a\n 3-tuple containing the part before the separator, the separator\n itself, and the part after the separator. If the separator is not\n found, return a 3-tuple containing two empty strings, followed by\n the string itself.\n\n New in version 2.5.\n\nstr.rsplit([sep[, maxsplit]])\n\n Return a list of the words in the string, using *sep* as the\n delimiter string. If *maxsplit* is given, at most *maxsplit* splits\n are done, the *rightmost* ones. If *sep* is not specified or\n ``None``, any whitespace string is a separator. Except for\n splitting from the right, ``rsplit()`` behaves like ``split()``\n which is described in detail below.\n\n New in version 2.4.\n\nstr.rstrip([chars])\n\n Return a copy of the string with trailing characters removed. The\n *chars* argument is a string specifying the set of characters to be\n removed. If omitted or ``None``, the *chars* argument defaults to\n removing whitespace. The *chars* argument is not a suffix; rather,\n all combinations of its values are stripped:\n\n >>> \' spacious \'.rstrip()\n \' spacious\'\n >>> \'mississippi\'.rstrip(\'ipz\')\n \'mississ\'\n\n Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.split([sep[, maxsplit]])\n\n Return a list of the words in the string, using *sep* as the\n delimiter string. If *maxsplit* is given, at most *maxsplit*\n splits are done (thus, the list will have at most ``maxsplit+1``\n elements). If *maxsplit* is not specified, then there is no limit\n on the number of splits (all possible splits are made).\n\n If *sep* is given, consecutive delimiters are not grouped together\n and are deemed to delimit empty strings (for example,\n ``\'1,,2\'.split(\',\')`` returns ``[\'1\', \'\', \'2\']``). The *sep*\n argument may consist of multiple characters (for example,\n ``\'1<>2<>3\'.split(\'<>\')`` returns ``[\'1\', \'2\', \'3\']``). Splitting\n an empty string with a specified separator returns ``[\'\']``.\n\n If *sep* is not specified or is ``None``, a different splitting\n algorithm is applied: runs of consecutive whitespace are regarded\n as a single separator, and the result will contain no empty strings\n at the start or end if the string has leading or trailing\n whitespace. Consequently, splitting an empty string or a string\n consisting of just whitespace with a ``None`` separator returns\n ``[]``.\n\n For example, ``\' 1 2 3 \'.split()`` returns ``[\'1\', \'2\', \'3\']``,\n and ``\' 1 2 3 \'.split(None, 1)`` returns ``[\'1\', \'2 3 \']``.\n\nstr.splitlines([keepends])\n\n Return a list of the lines in the string, breaking at line\n boundaries. Line breaks are not included in the resulting list\n unless *keepends* is given and true.\n\nstr.startswith(prefix[, start[, end]])\n\n Return ``True`` if string starts with the *prefix*, otherwise\n return ``False``. *prefix* can also be a tuple of prefixes to look\n for. With optional *start*, test string beginning at that\n position. With optional *end*, stop comparing string at that\n position.\n\n Changed in version 2.5: Accept tuples as *prefix*.\n\nstr.strip([chars])\n\n Return a copy of the string with the leading and trailing\n characters removed. The *chars* argument is a string specifying the\n set of characters to be removed. If omitted or ``None``, the\n *chars* argument defaults to removing whitespace. The *chars*\n argument is not a prefix or suffix; rather, all combinations of its\n values are stripped:\n\n >>> \' spacious \'.strip()\n \'spacious\'\n >>> \'www.example.com\'.strip(\'cmowz.\')\n \'example\'\n\n Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.swapcase()\n\n Return a copy of the string with uppercase characters converted to\n lowercase and vice versa.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.title()\n\n Return a titlecased version of the string where words start with an\n uppercase character and the remaining characters are lowercase.\n\n The algorithm uses a simple language-independent definition of a\n word as groups of consecutive letters. The definition works in\n many contexts but it means that apostrophes in contractions and\n possessives form word boundaries, which may not be the desired\n result:\n\n >>> "they\'re bill\'s friends from the UK".title()\n "They\'Re Bill\'S Friends From The Uk"\n\n A workaround for apostrophes can be constructed using regular\n expressions:\n\n >>> import re\n >>> def titlecase(s):\n return re.sub(r"[A-Za-z]+(\'[A-Za-z]+)?",\n lambda mo: mo.group(0)[0].upper() +\n mo.group(0)[1:].lower(),\n s)\n\n >>> titlecase("they\'re bill\'s friends.")\n "They\'re Bill\'s Friends."\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.translate(table[, deletechars])\n\n Return a copy of the string where all characters occurring in the\n optional argument *deletechars* are removed, and the remaining\n characters have been mapped through the given translation table,\n which must be a string of length 256.\n\n You can use the ``maketrans()`` helper function in the ``string``\n module to create a translation table. For string objects, set the\n *table* argument to ``None`` for translations that only delete\n characters:\n\n >>> \'read this short text\'.translate(None, \'aeiou\')\n \'rd ths shrt txt\'\n\n New in version 2.6: Support for a ``None`` *table* argument.\n\n For Unicode objects, the ``translate()`` method does not accept the\n optional *deletechars* argument. Instead, it returns a copy of the\n *s* where all characters have been mapped through the given\n translation table which must be a mapping of Unicode ordinals to\n Unicode ordinals, Unicode strings or ``None``. Unmapped characters\n are left untouched. Characters mapped to ``None`` are deleted.\n Note, a more flexible approach is to create a custom character\n mapping codec using the ``codecs`` module (see ``encodings.cp1251``\n for an example).\n\nstr.upper()\n\n Return a copy of the string converted to uppercase.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.zfill(width)\n\n Return the numeric string left filled with zeros in a string of\n length *width*. A sign prefix is handled correctly. The original\n string is returned if *width* is less than ``len(s)``.\n\n New in version 2.2.2.\n\nThe following methods are present only on unicode objects:\n\nunicode.isnumeric()\n\n Return ``True`` if there are only numeric characters in S,\n ``False`` otherwise. Numeric characters include digit characters,\n and all characters that have the Unicode numeric value property,\n e.g. U+2155, VULGAR FRACTION ONE FIFTH.\n\nunicode.isdecimal()\n\n Return ``True`` if there are only decimal characters in S,\n ``False`` otherwise. Decimal characters include digit characters,\n and all characters that that can be used to form decimal-radix\n numbers, e.g. U+0660, ARABIC-INDIC DIGIT ZERO.\n\n\nString Formatting Operations\n============================\n\nString and Unicode objects have one unique built-in operation: the\n``%`` operator (modulo). This is also known as the string\n*formatting* or *interpolation* operator. Given ``format % values``\n(where *format* is a string or Unicode object), ``%`` conversion\nspecifications in *format* are replaced with zero or more elements of\n*values*. The effect is similar to the using ``sprintf()`` in the C\nlanguage. If *format* is a Unicode object, or if any of the objects\nbeing converted using the ``%s`` conversion are Unicode objects, the\nresult will also be a Unicode object.\n\nIf *format* requires a single argument, *values* may be a single non-\ntuple object. [4] Otherwise, *values* must be a tuple with exactly\nthe number of items specified by the format string, or a single\nmapping object (for example, a dictionary).\n\nA conversion specifier contains two or more characters and has the\nfollowing components, which must occur in this order:\n\n1. The ``\'%\'`` character, which marks the start of the specifier.\n\n2. Mapping key (optional), consisting of a parenthesised sequence of\n characters (for example, ``(somename)``).\n\n3. Conversion flags (optional), which affect the result of some\n conversion types.\n\n4. Minimum field width (optional). If specified as an ``\'*\'``\n (asterisk), the actual width is read from the next element of the\n tuple in *values*, and the object to convert comes after the\n minimum field width and optional precision.\n\n5. Precision (optional), given as a ``\'.\'`` (dot) followed by the\n precision. If specified as ``\'*\'`` (an asterisk), the actual width\n is read from the next element of the tuple in *values*, and the\n value to convert comes after the precision.\n\n6. Length modifier (optional).\n\n7. Conversion type.\n\nWhen the right argument is a dictionary (or other mapping type), then\nthe formats in the string *must* include a parenthesised mapping key\ninto that dictionary inserted immediately after the ``\'%\'`` character.\nThe mapping key selects the value to be formatted from the mapping.\nFor example:\n\n>>> print \'%(language)s has %(number)03d quote types.\' % \\\n... {"language": "Python", "number": 2}\nPython has 002 quote types.\n\nIn this case no ``*`` specifiers may occur in a format (since they\nrequire a sequential parameter list).\n\nThe conversion flag characters are:\n\n+-----------+-----------------------------------------------------------------------+\n| Flag | Meaning |\n+===========+=======================================================================+\n| ``\'#\'`` | The value conversion will use the "alternate form" (where defined |\n| | below). |\n+-----------+-----------------------------------------------------------------------+\n| ``\'0\'`` | The conversion will be zero padded for numeric values. |\n+-----------+-----------------------------------------------------------------------+\n| ``\'-\'`` | The converted value is left adjusted (overrides the ``\'0\'`` |\n| | conversion if both are given). |\n+-----------+-----------------------------------------------------------------------+\n| ``\' \'`` | (a space) A blank should be left before a positive number (or empty |\n| | string) produced by a signed conversion. |\n+-----------+-----------------------------------------------------------------------+\n| ``\'+\'`` | A sign character (``\'+\'`` or ``\'-\'``) will precede the conversion |\n| | (overrides a "space" flag). |\n+-----------+-----------------------------------------------------------------------+\n\nA length modifier (``h``, ``l``, or ``L``) may be present, but is\nignored as it is not necessary for Python -- so e.g. ``%ld`` is\nidentical to ``%d``.\n\nThe conversion types are:\n\n+--------------+-------------------------------------------------------+---------+\n| Conversion | Meaning | Notes |\n+==============+=======================================================+=========+\n| ``\'d\'`` | Signed integer decimal. | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'i\'`` | Signed integer decimal. | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'o\'`` | Signed octal value. | (1) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'u\'`` | Obsolete type -- it is identical to ``\'d\'``. | (7) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'x\'`` | Signed hexadecimal (lowercase). | (2) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'X\'`` | Signed hexadecimal (uppercase). | (2) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'e\'`` | Floating point exponential format (lowercase). | (3) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'E\'`` | Floating point exponential format (uppercase). | (3) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'f\'`` | Floating point decimal format. | (3) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'F\'`` | Floating point decimal format. | (3) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'g\'`` | Floating point format. Uses lowercase exponential | (4) |\n| | format if exponent is less than -4 or not less than | |\n| | precision, decimal format otherwise. | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'G\'`` | Floating point format. Uses uppercase exponential | (4) |\n| | format if exponent is less than -4 or not less than | |\n| | precision, decimal format otherwise. | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'c\'`` | Single character (accepts integer or single character | |\n| | string). | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'r\'`` | String (converts any Python object using ``repr()``). | (5) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'s\'`` | String (converts any Python object using ``str()``). | (6) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'%\'`` | No argument is converted, results in a ``\'%\'`` | |\n| | character in the result. | |\n+--------------+-------------------------------------------------------+---------+\n\nNotes:\n\n1. The alternate form causes a leading zero (``\'0\'``) to be inserted\n between left-hand padding and the formatting of the number if the\n leading character of the result is not already a zero.\n\n2. The alternate form causes a leading ``\'0x\'`` or ``\'0X\'`` (depending\n on whether the ``\'x\'`` or ``\'X\'`` format was used) to be inserted\n between left-hand padding and the formatting of the number if the\n leading character of the result is not already a zero.\n\n3. The alternate form causes the result to always contain a decimal\n point, even if no digits follow it.\n\n The precision determines the number of digits after the decimal\n point and defaults to 6.\n\n4. The alternate form causes the result to always contain a decimal\n point, and trailing zeroes are not removed as they would otherwise\n be.\n\n The precision determines the number of significant digits before\n and after the decimal point and defaults to 6.\n\n5. The ``%r`` conversion was added in Python 2.0.\n\n The precision determines the maximal number of characters used.\n\n6. If the object or format provided is a ``unicode`` string, the\n resulting string will also be ``unicode``.\n\n The precision determines the maximal number of characters used.\n\n7. See **PEP 237**.\n\nSince Python strings have an explicit length, ``%s`` conversions do\nnot assume that ``\'\\0\'`` is the end of the string.\n\nChanged in version 2.7: ``%f`` conversions for numbers whose absolute\nvalue is over 1e50 are no longer replaced by ``%g`` conversions.\n\nAdditional string operations are defined in standard modules\n``string`` and ``re``.\n\n\nXRange Type\n===========\n\nThe ``xrange`` type is an immutable sequence which is commonly used\nfor looping. The advantage of the ``xrange`` type is that an\n``xrange`` object will always take the same amount of memory, no\nmatter the size of the range it represents. There are no consistent\nperformance advantages.\n\nXRange objects have very little behavior: they only support indexing,\niteration, and the ``len()`` function.\n\n\nMutable Sequence Types\n======================\n\nList and ``bytearray`` objects support additional operations that\nallow in-place modification of the object. Other mutable sequence\ntypes (when added to the language) should also support these\noperations. Strings and tuples are immutable sequence types: such\nobjects cannot be modified once created. The following operations are\ndefined on mutable sequence types (where *x* is an arbitrary object):\n\n+--------------------------------+----------------------------------+-----------------------+\n| Operation | Result | Notes |\n+================================+==================================+=======================+\n| ``s[i] = x`` | item *i* of *s* is replaced by | |\n| | *x* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s[i:j] = t`` | slice of *s* from *i* to *j* is | |\n| | replaced by the contents of the | |\n| | iterable *t* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``del s[i:j]`` | same as ``s[i:j] = []`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s[i:j:k] = t`` | the elements of ``s[i:j:k]`` are | (1) |\n| | replaced by those of *t* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``del s[i:j:k]`` | removes the elements of | |\n| | ``s[i:j:k]`` from the list | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.append(x)`` | same as ``s[len(s):len(s)] = | (2) |\n| | [x]`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.extend(x)`` | same as ``s[len(s):len(s)] = x`` | (3) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.count(x)`` | return number of *i*\'s for which | |\n| | ``s[i] == x`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.index(x[, i[, j]])`` | return smallest *k* such that | (4) |\n| | ``s[k] == x`` and ``i <= k < j`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.insert(i, x)`` | same as ``s[i:i] = [x]`` | (5) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.pop([i])`` | same as ``x = s[i]; del s[i]; | (6) |\n| | return x`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.remove(x)`` | same as ``del s[s.index(x)]`` | (4) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.reverse()`` | reverses the items of *s* in | (7) |\n| | place | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.sort([cmp[, key[, | sort the items of *s* in place | (7)(8)(9)(10) |\n| reverse]]])`` | | |\n+--------------------------------+----------------------------------+-----------------------+\n\nNotes:\n\n1. *t* must have the same length as the slice it is replacing.\n\n2. The C implementation of Python has historically accepted multiple\n parameters and implicitly joined them into a tuple; this no longer\n works in Python 2.0. Use of this misfeature has been deprecated\n since Python 1.4.\n\n3. *x* can be any iterable object.\n\n4. Raises ``ValueError`` when *x* is not found in *s*. When a negative\n index is passed as the second or third parameter to the ``index()``\n method, the list length is added, as for slice indices. If it is\n still negative, it is truncated to zero, as for slice indices.\n\n Changed in version 2.3: Previously, ``index()`` didn\'t have\n arguments for specifying start and stop positions.\n\n5. When a negative index is passed as the first parameter to the\n ``insert()`` method, the list length is added, as for slice\n indices. If it is still negative, it is truncated to zero, as for\n slice indices.\n\n Changed in version 2.3: Previously, all negative indices were\n truncated to zero.\n\n6. The ``pop()`` method is only supported by the list and array types.\n The optional argument *i* defaults to ``-1``, so that by default\n the last item is removed and returned.\n\n7. The ``sort()`` and ``reverse()`` methods modify the list in place\n for economy of space when sorting or reversing a large list. To\n remind you that they operate by side effect, they don\'t return the\n sorted or reversed list.\n\n8. The ``sort()`` method takes optional arguments for controlling the\n comparisons.\n\n *cmp* specifies a custom comparison function of two arguments (list\n items) which should return a negative, zero or positive number\n depending on whether the first argument is considered smaller than,\n equal to, or larger than the second argument: ``cmp=lambda x,y:\n cmp(x.lower(), y.lower())``. The default value is ``None``.\n\n *key* specifies a function of one argument that is used to extract\n a comparison key from each list element: ``key=str.lower``. The\n default value is ``None``.\n\n *reverse* is a boolean value. If set to ``True``, then the list\n elements are sorted as if each comparison were reversed.\n\n In general, the *key* and *reverse* conversion processes are much\n faster than specifying an equivalent *cmp* function. This is\n because *cmp* is called multiple times for each list element while\n *key* and *reverse* touch each element only once. Use\n ``functools.cmp_to_key()`` to convert an old-style *cmp* function\n to a *key* function.\n\n Changed in version 2.3: Support for ``None`` as an equivalent to\n omitting *cmp* was added.\n\n Changed in version 2.4: Support for *key* and *reverse* was added.\n\n9. Starting with Python 2.3, the ``sort()`` method is guaranteed to be\n stable. A sort is stable if it guarantees not to change the\n relative order of elements that compare equal --- this is helpful\n for sorting in multiple passes (for example, sort by department,\n then by salary grade).\n\n10. **CPython implementation detail:** While a list is being sorted,\n the effect of attempting to mutate, or even inspect, the list is\n undefined. The C implementation of Python 2.3 and newer makes the\n list appear empty for the duration, and raises ``ValueError`` if\n it can detect that the list has been mutated during a sort.\n', + 'typesseq-mutable': u"\nMutable Sequence Types\n**********************\n\nList and ``bytearray`` objects support additional operations that\nallow in-place modification of the object. Other mutable sequence\ntypes (when added to the language) should also support these\noperations. Strings and tuples are immutable sequence types: such\nobjects cannot be modified once created. The following operations are\ndefined on mutable sequence types (where *x* is an arbitrary object):\n\n+--------------------------------+----------------------------------+-----------------------+\n| Operation | Result | Notes |\n+================================+==================================+=======================+\n| ``s[i] = x`` | item *i* of *s* is replaced by | |\n| | *x* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s[i:j] = t`` | slice of *s* from *i* to *j* is | |\n| | replaced by the contents of the | |\n| | iterable *t* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``del s[i:j]`` | same as ``s[i:j] = []`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s[i:j:k] = t`` | the elements of ``s[i:j:k]`` are | (1) |\n| | replaced by those of *t* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``del s[i:j:k]`` | removes the elements of | |\n| | ``s[i:j:k]`` from the list | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.append(x)`` | same as ``s[len(s):len(s)] = | (2) |\n| | [x]`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.extend(x)`` | same as ``s[len(s):len(s)] = x`` | (3) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.count(x)`` | return number of *i*'s for which | |\n| | ``s[i] == x`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.index(x[, i[, j]])`` | return smallest *k* such that | (4) |\n| | ``s[k] == x`` and ``i <= k < j`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.insert(i, x)`` | same as ``s[i:i] = [x]`` | (5) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.pop([i])`` | same as ``x = s[i]; del s[i]; | (6) |\n| | return x`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.remove(x)`` | same as ``del s[s.index(x)]`` | (4) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.reverse()`` | reverses the items of *s* in | (7) |\n| | place | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.sort([cmp[, key[, | sort the items of *s* in place | (7)(8)(9)(10) |\n| reverse]]])`` | | |\n+--------------------------------+----------------------------------+-----------------------+\n\nNotes:\n\n1. *t* must have the same length as the slice it is replacing.\n\n2. The C implementation of Python has historically accepted multiple\n parameters and implicitly joined them into a tuple; this no longer\n works in Python 2.0. Use of this misfeature has been deprecated\n since Python 1.4.\n\n3. *x* can be any iterable object.\n\n4. Raises ``ValueError`` when *x* is not found in *s*. When a negative\n index is passed as the second or third parameter to the ``index()``\n method, the list length is added, as for slice indices. If it is\n still negative, it is truncated to zero, as for slice indices.\n\n Changed in version 2.3: Previously, ``index()`` didn't have\n arguments for specifying start and stop positions.\n\n5. When a negative index is passed as the first parameter to the\n ``insert()`` method, the list length is added, as for slice\n indices. If it is still negative, it is truncated to zero, as for\n slice indices.\n\n Changed in version 2.3: Previously, all negative indices were\n truncated to zero.\n\n6. The ``pop()`` method is only supported by the list and array types.\n The optional argument *i* defaults to ``-1``, so that by default\n the last item is removed and returned.\n\n7. The ``sort()`` and ``reverse()`` methods modify the list in place\n for economy of space when sorting or reversing a large list. To\n remind you that they operate by side effect, they don't return the\n sorted or reversed list.\n\n8. The ``sort()`` method takes optional arguments for controlling the\n comparisons.\n\n *cmp* specifies a custom comparison function of two arguments (list\n items) which should return a negative, zero or positive number\n depending on whether the first argument is considered smaller than,\n equal to, or larger than the second argument: ``cmp=lambda x,y:\n cmp(x.lower(), y.lower())``. The default value is ``None``.\n\n *key* specifies a function of one argument that is used to extract\n a comparison key from each list element: ``key=str.lower``. The\n default value is ``None``.\n\n *reverse* is a boolean value. If set to ``True``, then the list\n elements are sorted as if each comparison were reversed.\n\n In general, the *key* and *reverse* conversion processes are much\n faster than specifying an equivalent *cmp* function. This is\n because *cmp* is called multiple times for each list element while\n *key* and *reverse* touch each element only once. Use\n ``functools.cmp_to_key()`` to convert an old-style *cmp* function\n to a *key* function.\n\n Changed in version 2.3: Support for ``None`` as an equivalent to\n omitting *cmp* was added.\n\n Changed in version 2.4: Support for *key* and *reverse* was added.\n\n9. Starting with Python 2.3, the ``sort()`` method is guaranteed to be\n stable. A sort is stable if it guarantees not to change the\n relative order of elements that compare equal --- this is helpful\n for sorting in multiple passes (for example, sort by department,\n then by salary grade).\n\n10. **CPython implementation detail:** While a list is being sorted,\n the effect of attempting to mutate, or even inspect, the list is\n undefined. The C implementation of Python 2.3 and newer makes the\n list appear empty for the duration, and raises ``ValueError`` if\n it can detect that the list has been mutated during a sort.\n", 'unary': u'\nUnary arithmetic and bitwise operations\n***************************************\n\nAll unary arithmetic and bitwise operations have the same priority:\n\n u_expr ::= power | "-" u_expr | "+" u_expr | "~" u_expr\n\nThe unary ``-`` (minus) operator yields the negation of its numeric\nargument.\n\nThe unary ``+`` (plus) operator yields its numeric argument unchanged.\n\nThe unary ``~`` (invert) operator yields the bitwise inversion of its\nplain or long integer argument. The bitwise inversion of ``x`` is\ndefined as ``-(x+1)``. It only applies to integral numbers.\n\nIn all three cases, if the argument does not have the proper type, a\n``TypeError`` exception is raised.\n', 'while': u'\nThe ``while`` statement\n***********************\n\nThe ``while`` statement is used for repeated execution as long as an\nexpression is true:\n\n while_stmt ::= "while" expression ":" suite\n ["else" ":" suite]\n\nThis repeatedly tests the expression and, if it is true, executes the\nfirst suite; if the expression is false (which may be the first time\nit is tested) the suite of the ``else`` clause, if present, is\nexecuted and the loop terminates.\n\nA ``break`` statement executed in the first suite terminates the loop\nwithout executing the ``else`` clause\'s suite. A ``continue``\nstatement executed in the first suite skips the rest of the suite and\ngoes back to testing the expression.\n', - 'with': u'\nThe ``with`` statement\n**********************\n\nNew in version 2.5.\n\nThe ``with`` statement is used to wrap the execution of a block with\nmethods defined by a context manager (see section *With Statement\nContext Managers*). This allows common\n``try``...``except``...``finally`` usage patterns to be encapsulated\nfor convenient reuse.\n\n with_stmt ::= "with" with_item ("," with_item)* ":" suite\n with_item ::= expression ["as" target]\n\nThe execution of the ``with`` statement with one "item" proceeds as\nfollows:\n\n1. The context expression is evaluated to obtain a context manager.\n\n2. The context manager\'s ``__exit__()`` is loaded for later use.\n\n3. The context manager\'s ``__enter__()`` method is invoked.\n\n4. If a target was included in the ``with`` statement, the return\n value from ``__enter__()`` is assigned to it.\n\n Note: The ``with`` statement guarantees that if the ``__enter__()``\n method returns without an error, then ``__exit__()`` will always\n be called. Thus, if an error occurs during the assignment to the\n target list, it will be treated the same as an error occurring\n within the suite would be. See step 6 below.\n\n5. The suite is executed.\n\n6. The context manager\'s ``__exit__()`` method is invoked. If an\n exception caused the suite to be exited, its type, value, and\n traceback are passed as arguments to ``__exit__()``. Otherwise,\n three ``None`` arguments are supplied.\n\n If the suite was exited due to an exception, and the return value\n from the ``__exit__()`` method was false, the exception is\n reraised. If the return value was true, the exception is\n suppressed, and execution continues with the statement following\n the ``with`` statement.\n\n If the suite was exited for any reason other than an exception, the\n return value from ``__exit__()`` is ignored, and execution proceeds\n at the normal location for the kind of exit that was taken.\n\nWith more than one item, the context managers are processed as if\nmultiple ``with`` statements were nested:\n\n with A() as a, B() as b:\n suite\n\nis equivalent to\n\n with A() as a:\n with B() as b:\n suite\n\nNote: In Python 2.5, the ``with`` statement is only allowed when the\n ``with_statement`` feature has been enabled. It is always enabled\n in Python 2.6.\n\nChanged in version 2.7: Support for multiple context expressions.\n\nSee also:\n\n **PEP 0343** - The "with" statement\n The specification, background, and examples for the Python\n ``with`` statement.\n', + 'with': u'\nThe ``with`` statement\n**********************\n\nNew in version 2.5.\n\nThe ``with`` statement is used to wrap the execution of a block with\nmethods defined by a context manager (see section *With Statement\nContext Managers*). This allows common\n``try``...``except``...``finally`` usage patterns to be encapsulated\nfor convenient reuse.\n\n with_stmt ::= "with" with_item ("," with_item)* ":" suite\n with_item ::= expression ["as" target]\n\nThe execution of the ``with`` statement with one "item" proceeds as\nfollows:\n\n1. The context expression (the expression given in the **with_item**)\n is evaluated to obtain a context manager.\n\n2. The context manager\'s ``__exit__()`` is loaded for later use.\n\n3. The context manager\'s ``__enter__()`` method is invoked.\n\n4. If a target was included in the ``with`` statement, the return\n value from ``__enter__()`` is assigned to it.\n\n Note: The ``with`` statement guarantees that if the ``__enter__()``\n method returns without an error, then ``__exit__()`` will always\n be called. Thus, if an error occurs during the assignment to the\n target list, it will be treated the same as an error occurring\n within the suite would be. See step 6 below.\n\n5. The suite is executed.\n\n6. The context manager\'s ``__exit__()`` method is invoked. If an\n exception caused the suite to be exited, its type, value, and\n traceback are passed as arguments to ``__exit__()``. Otherwise,\n three ``None`` arguments are supplied.\n\n If the suite was exited due to an exception, and the return value\n from the ``__exit__()`` method was false, the exception is\n reraised. If the return value was true, the exception is\n suppressed, and execution continues with the statement following\n the ``with`` statement.\n\n If the suite was exited for any reason other than an exception, the\n return value from ``__exit__()`` is ignored, and execution proceeds\n at the normal location for the kind of exit that was taken.\n\nWith more than one item, the context managers are processed as if\nmultiple ``with`` statements were nested:\n\n with A() as a, B() as b:\n suite\n\nis equivalent to\n\n with A() as a:\n with B() as b:\n suite\n\nNote: In Python 2.5, the ``with`` statement is only allowed when the\n ``with_statement`` feature has been enabled. It is always enabled\n in Python 2.6.\n\nChanged in version 2.7: Support for multiple context expressions.\n\nSee also:\n\n **PEP 0343** - The "with" statement\n The specification, background, and examples for the Python\n ``with`` statement.\n', 'yield': u'\nThe ``yield`` statement\n***********************\n\n yield_stmt ::= yield_expression\n\nThe ``yield`` statement is only used when defining a generator\nfunction, and is only used in the body of the generator function.\nUsing a ``yield`` statement in a function definition is sufficient to\ncause that definition to create a generator function instead of a\nnormal function.\n\nWhen a generator function is called, it returns an iterator known as a\ngenerator iterator, or more commonly, a generator. The body of the\ngenerator function is executed by calling the generator\'s ``next()``\nmethod repeatedly until it raises an exception.\n\nWhen a ``yield`` statement is executed, the state of the generator is\nfrozen and the value of **expression_list** is returned to\n``next()``\'s caller. By "frozen" we mean that all local state is\nretained, including the current bindings of local variables, the\ninstruction pointer, and the internal evaluation stack: enough\ninformation is saved so that the next time ``next()`` is invoked, the\nfunction can proceed exactly as if the ``yield`` statement were just\nanother external call.\n\nAs of Python version 2.5, the ``yield`` statement is now allowed in\nthe ``try`` clause of a ``try`` ... ``finally`` construct. If the\ngenerator is not resumed before it is finalized (by reaching a zero\nreference count or by being garbage collected), the generator-\niterator\'s ``close()`` method will be called, allowing any pending\n``finally`` clauses to execute.\n\nNote: In Python 2.2, the ``yield`` statement was only allowed when the\n ``generators`` feature has been enabled. This ``__future__`` import\n statement was used to enable the feature:\n\n from __future__ import generators\n\nSee also:\n\n **PEP 0255** - Simple Generators\n The proposal for adding generators and the ``yield`` statement\n to Python.\n\n **PEP 0342** - Coroutines via Enhanced Generators\n The proposal that, among other generator enhancements, proposed\n allowing ``yield`` to appear inside a ``try`` ... ``finally``\n block.\n'} diff --git a/lib-python/2.7/random.py b/lib-python/2.7/random.py --- a/lib-python/2.7/random.py +++ b/lib-python/2.7/random.py @@ -317,7 +317,7 @@ n = len(population) if not 0 <= k <= n: - raise ValueError, "sample larger than population" + raise ValueError("sample larger than population") random = self.random _int = int result = [None] * k @@ -490,6 +490,12 @@ Conditions on the parameters are alpha > 0 and beta > 0. + The probability distribution function is: + + x ** (alpha - 1) * math.exp(-x / beta) + pdf(x) = -------------------------------------- + math.gamma(alpha) * beta ** alpha + """ # alpha > 0, beta > 0, mean is alpha*beta, variance is alpha*beta**2 @@ -592,7 +598,7 @@ ## -------------------- beta -------------------- ## See -## http://sourceforge.net/bugs/?func=detailbug&bug_id=130030&group_id=5470 +## http://mail.python.org/pipermail/python-bugs-list/2001-January/003752.html ## for Ivan Frohne's insightful analysis of why the original implementation: ## ## def betavariate(self, alpha, beta): diff --git a/lib-python/2.7/re.py b/lib-python/2.7/re.py --- a/lib-python/2.7/re.py +++ b/lib-python/2.7/re.py @@ -207,8 +207,7 @@ "Escape all non-alphanumeric characters in pattern." s = list(pattern) alphanum = _alphanum - for i in range(len(pattern)): - c = pattern[i] + for i, c in enumerate(pattern): if c not in alphanum: if c == "\000": s[i] = "\\000" diff --git a/lib-python/2.7/shutil.py b/lib-python/2.7/shutil.py --- a/lib-python/2.7/shutil.py +++ b/lib-python/2.7/shutil.py @@ -277,6 +277,12 @@ """ real_dst = dst if os.path.isdir(dst): + if _samefile(src, dst): + # We might be on a case insensitive filesystem, + # perform the rename anyway. + os.rename(src, dst) + return + real_dst = os.path.join(dst, _basename(src)) if os.path.exists(real_dst): raise Error, "Destination path '%s' already exists" % real_dst @@ -336,7 +342,7 @@ archive that is being built. If not provided, the current owner and group will be used. - The output tar file will be named 'base_dir' + ".tar", possibly plus + The output tar file will be named 'base_name' + ".tar", possibly plus the appropriate compression extension (".gz", or ".bz2"). Returns the output filename. @@ -406,7 +412,7 @@ def _make_zipfile(base_name, base_dir, verbose=0, dry_run=0, logger=None): """Create a zip file from all the files under 'base_dir'. - The output zip file will be named 'base_dir' + ".zip". Uses either the + The output zip file will be named 'base_name' + ".zip". Uses either the "zipfile" Python module (if available) or the InfoZIP "zip" utility (if installed and found on the default search path). If neither tool is available, raises ExecError. Returns the name of the output zip diff --git a/lib-python/2.7/site.py b/lib-python/2.7/site.py --- a/lib-python/2.7/site.py +++ b/lib-python/2.7/site.py @@ -61,6 +61,7 @@ import sys import os import __builtin__ +import traceback # Prefixes for site-packages; add additional prefixes like /usr/local here PREFIXES = [sys.prefix, sys.exec_prefix] @@ -155,17 +156,26 @@ except IOError: return with f: - for line in f: + for n, line in enumerate(f): if line.startswith("#"): continue - if line.startswith(("import ", "import\t")): - exec line - continue - line = line.rstrip() - dir, dircase = makepath(sitedir, line) - if not dircase in known_paths and os.path.exists(dir): - sys.path.append(dir) - known_paths.add(dircase) + try: + if line.startswith(("import ", "import\t")): + exec line + continue + line = line.rstrip() + dir, dircase = makepath(sitedir, line) + if not dircase in known_paths and os.path.exists(dir): + sys.path.append(dir) + known_paths.add(dircase) + except Exception as err: + print >>sys.stderr, "Error processing line {:d} of {}:\n".format( + n+1, fullname) + for record in traceback.format_exception(*sys.exc_info()): + for line in record.splitlines(): + print >>sys.stderr, ' '+line + print >>sys.stderr, "\nRemainder of file ignored" + break if reset: known_paths = None return known_paths diff --git a/lib-python/2.7/smtplib.py b/lib-python/2.7/smtplib.py --- a/lib-python/2.7/smtplib.py +++ b/lib-python/2.7/smtplib.py @@ -49,17 +49,18 @@ from email.base64mime import encode as encode_base64 from sys import stderr -__all__ = ["SMTPException","SMTPServerDisconnected","SMTPResponseException", - "SMTPSenderRefused","SMTPRecipientsRefused","SMTPDataError", - "SMTPConnectError","SMTPHeloError","SMTPAuthenticationError", - "quoteaddr","quotedata","SMTP"] +__all__ = ["SMTPException", "SMTPServerDisconnected", "SMTPResponseException", + "SMTPSenderRefused", "SMTPRecipientsRefused", "SMTPDataError", + "SMTPConnectError", "SMTPHeloError", "SMTPAuthenticationError", + "quoteaddr", "quotedata", "SMTP"] SMTP_PORT = 25 SMTP_SSL_PORT = 465 -CRLF="\r\n" +CRLF = "\r\n" OLDSTYLE_AUTH = re.compile(r"auth=(.*)", re.I) + # Exception classes used by this module. class SMTPException(Exception): """Base class for all exceptions raised by this module.""" @@ -109,7 +110,7 @@ def __init__(self, recipients): self.recipients = recipients - self.args = ( recipients,) + self.args = (recipients,) class SMTPDataError(SMTPResponseException): @@ -128,6 +129,7 @@ combination provided. """ + def quoteaddr(addr): """Quote a subset of the email addresses defined by RFC 821. @@ -138,7 +140,7 @@ m = email.utils.parseaddr(addr)[1] except AttributeError: pass - if m == (None, None): # Indicates parse failure or AttributeError + if m == (None, None): # Indicates parse failure or AttributeError # something weird here.. punt -ddm return "<%s>" % addr elif m is None: @@ -175,7 +177,8 @@ chr = None while chr != "\n": chr = self.sslobj.read(1) - if not chr: break + if not chr: + break str += chr return str @@ -219,6 +222,7 @@ ehlo_msg = "ehlo" ehlo_resp = None does_esmtp = 0 + default_port = SMTP_PORT def __init__(self, host='', port=0, local_hostname=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT): @@ -234,7 +238,6 @@ """ self.timeout = timeout self.esmtp_features = {} - self.default_port = SMTP_PORT if host: (code, msg) = self.connect(host, port) if code != 220: @@ -269,10 +272,11 @@ def _get_socket(self, port, host, timeout): # This makes it simpler for SMTP_SSL to use the SMTP connect code # and just alter the socket connection bit. - if self.debuglevel > 0: print>>stderr, 'connect:', (host, port) + if self.debuglevel > 0: + print>>stderr, 'connect:', (host, port) return socket.create_connection((port, host), timeout) - def connect(self, host='localhost', port = 0): + def connect(self, host='localhost', port=0): """Connect to a host on a given port. If the hostname ends with a colon (`:') followed by a number, and @@ -286,20 +290,25 @@ if not port and (host.find(':') == host.rfind(':')): i = host.rfind(':') if i >= 0: - host, port = host[:i], host[i+1:] - try: port = int(port) + host, port = host[:i], host[i + 1:] + try: + port = int(port) except ValueError: raise socket.error, "nonnumeric port" - if not port: port = self.default_port - if self.debuglevel > 0: print>>stderr, 'connect:', (host, port) + if not port: + port = self.default_port + if self.debuglevel > 0: + print>>stderr, 'connect:', (host, port) self.sock = self._get_socket(host, port, self.timeout) (code, msg) = self.getreply() - if self.debuglevel > 0: print>>stderr, "connect:", msg + if self.debuglevel > 0: + print>>stderr, "connect:", msg return (code, msg) def send(self, str): """Send `str' to the server.""" - if self.debuglevel > 0: print>>stderr, 'send:', repr(str) + if self.debuglevel > 0: + print>>stderr, 'send:', repr(str) if hasattr(self, 'sock') and self.sock: try: self.sock.sendall(str) @@ -330,7 +339,7 @@ Raises SMTPServerDisconnected if end-of-file is reached. """ - resp=[] + resp = [] if self.file is None: self.file = self.sock.makefile('rb') while 1: @@ -341,9 +350,10 @@ if line == '': self.close() raise SMTPServerDisconnected("Connection unexpectedly closed") - if self.debuglevel > 0: print>>stderr, 'reply:', repr(line) + if self.debuglevel > 0: + print>>stderr, 'reply:', repr(line) resp.append(line[4:].strip()) - code=line[:3] + code = line[:3] # Check that the error code is syntactically correct. # Don't attempt to read a continuation line if it is broken. try: @@ -352,17 +362,17 @@ errcode = -1 break # Check if multiline response. - if line[3:4]!="-": + if line[3:4] != "-": break errmsg = "\n".join(resp) if self.debuglevel > 0: - print>>stderr, 'reply: retcode (%s); Msg: %s' % (errcode,errmsg) + print>>stderr, 'reply: retcode (%s); Msg: %s' % (errcode, errmsg) return errcode, errmsg def docmd(self, cmd, args=""): """Send a command, and return its response code.""" - self.putcmd(cmd,args) + self.putcmd(cmd, args) return self.getreply() # std smtp commands @@ -372,9 +382,9 @@ host. """ self.putcmd("helo", name or self.local_hostname) - (code,msg)=self.getreply() - self.helo_resp=msg - return (code,msg) + (code, msg) = self.getreply() + self.helo_resp = msg + return (code, msg) def ehlo(self, name=''): """ SMTP 'ehlo' command. @@ -383,19 +393,19 @@ """ self.esmtp_features = {} self.putcmd(self.ehlo_msg, name or self.local_hostname) - (code,msg)=self.getreply() + (code, msg) = self.getreply() # According to RFC1869 some (badly written) # MTA's will disconnect on an ehlo. Toss an exception if # that happens -ddm if code == -1 and len(msg) == 0: self.close() raise SMTPServerDisconnected("Server not connected") - self.ehlo_resp=msg + self.ehlo_resp = msg if code != 250: - return (code,msg) - self.does_esmtp=1 + return (code, msg) + self.does_esmtp = 1 #parse the ehlo response -ddm - resp=self.ehlo_resp.split('\n') + resp = self.ehlo_resp.split('\n') del resp[0] for each in resp: # To be able to communicate with as many SMTP servers as possible, @@ -415,16 +425,16 @@ # It's actually stricter, in that only spaces are allowed between # parameters, but were not going to check for that here. Note # that the space isn't present if there are no parameters. - m=re.match(r'(?P<feature>[A-Za-z0-9][A-Za-z0-9\-]*) ?',each) + m = re.match(r'(?P<feature>[A-Za-z0-9][A-Za-z0-9\-]*) ?', each) if m: - feature=m.group("feature").lower() - params=m.string[m.end("feature"):].strip() + feature = m.group("feature").lower() + params = m.string[m.end("feature"):].strip() if feature == "auth": self.esmtp_features[feature] = self.esmtp_features.get(feature, "") \ + " " + params else: - self.esmtp_features[feature]=params - return (code,msg) + self.esmtp_features[feature] = params + return (code, msg) def has_extn(self, opt): """Does the server support a given SMTP service extension?""" @@ -444,23 +454,23 @@ """SMTP 'noop' command -- doesn't do anything :>""" return self.docmd("noop") - def mail(self,sender,options=[]): + def mail(self, sender, options=[]): """SMTP 'mail' command -- begins mail xfer session.""" optionlist = '' if options and self.does_esmtp: optionlist = ' ' + ' '.join(options) - self.putcmd("mail", "FROM:%s%s" % (quoteaddr(sender) ,optionlist)) + self.putcmd("mail", "FROM:%s%s" % (quoteaddr(sender), optionlist)) return self.getreply() - def rcpt(self,recip,options=[]): + def rcpt(self, recip, options=[]): """SMTP 'rcpt' command -- indicates 1 recipient for this mail.""" optionlist = '' if options and self.does_esmtp: optionlist = ' ' + ' '.join(options) - self.putcmd("rcpt","TO:%s%s" % (quoteaddr(recip),optionlist)) + self.putcmd("rcpt", "TO:%s%s" % (quoteaddr(recip), optionlist)) return self.getreply() - def data(self,msg): + def data(self, msg): """SMTP 'DATA' command -- sends message data to server. Automatically quotes lines beginning with a period per rfc821. @@ -469,26 +479,28 @@ response code received when the all data is sent. """ self.putcmd("data") - (code,repl)=self.getreply() - if self.debuglevel >0 : print>>stderr, "data:", (code,repl) + (code, repl) = self.getreply() + if self.debuglevel > 0: + print>>stderr, "data:", (code, repl) if code != 354: - raise SMTPDataError(code,repl) + raise SMTPDataError(code, repl) else: q = quotedata(msg) if q[-2:] != CRLF: q = q + CRLF q = q + "." + CRLF self.send(q) - (code,msg)=self.getreply() - if self.debuglevel >0 : print>>stderr, "data:", (code,msg) - return (code,msg) + (code, msg) = self.getreply() + if self.debuglevel > 0: + print>>stderr, "data:", (code, msg) + return (code, msg) def verify(self, address): """SMTP 'verify' command -- checks for address validity.""" self.putcmd("vrfy", quoteaddr(address)) return self.getreply() # a.k.a. - vrfy=verify + vrfy = verify def expn(self, address): """SMTP 'expn' command -- expands a mailing list.""" @@ -592,7 +604,7 @@ raise SMTPAuthenticationError(code, resp) return (code, resp) - def starttls(self, keyfile = None, certfile = None): + def starttls(self, keyfile=None, certfile=None): """Puts the connection to the SMTP server into TLS mode. If there has been no previous EHLO or HELO command this session, this @@ -695,22 +707,22 @@ for option in mail_options: esmtp_opts.append(option) - (code,resp) = self.mail(from_addr, esmtp_opts) + (code, resp) = self.mail(from_addr, esmtp_opts) if code != 250: self.rset() raise SMTPSenderRefused(code, resp, from_addr) - senderrs={} + senderrs = {} if isinstance(to_addrs, basestring): to_addrs = [to_addrs] for each in to_addrs: - (code,resp)=self.rcpt(each, rcpt_options) + (code, resp) = self.rcpt(each, rcpt_options) if (code != 250) and (code != 251): - senderrs[each]=(code,resp) - if len(senderrs)==len(to_addrs): + senderrs[each] = (code, resp) + if len(senderrs) == len(to_addrs): # the server refused all our recipients self.rset() raise SMTPRecipientsRefused(senderrs) - (code,resp) = self.data(msg) + (code, resp) = self.data(msg) if code != 250: self.rset() raise SMTPDataError(code, resp) @@ -744,16 +756,19 @@ are also optional - they can contain a PEM formatted private key and certificate chain file for the SSL connection. """ + + default_port = SMTP_SSL_PORT + def __init__(self, host='', port=0, local_hostname=None, keyfile=None, certfile=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT): self.keyfile = keyfile self.certfile = certfile SMTP.__init__(self, host, port, local_hostname, timeout) - self.default_port = SMTP_SSL_PORT def _get_socket(self, host, port, timeout): - if self.debuglevel > 0: print>>stderr, 'connect:', (host, port) + if self.debuglevel > 0: + print>>stderr, 'connect:', (host, port) new_socket = socket.create_connection((host, port), timeout) new_socket = ssl.wrap_socket(new_socket, self.keyfile, self.certfile) self.file = SSLFakeFile(new_socket) @@ -781,11 +796,11 @@ ehlo_msg = "lhlo" - def __init__(self, host = '', port = LMTP_PORT, local_hostname = None): + def __init__(self, host='', port=LMTP_PORT, local_hostname=None): """Initialize a new instance.""" SMTP.__init__(self, host, port, local_hostname) - def connect(self, host = 'localhost', port = 0): + def connect(self, host='localhost', port=0): """Connect to the LMTP daemon, on either a Unix or a TCP socket.""" if host[0] != '/': return SMTP.connect(self, host, port) @@ -795,13 +810,15 @@ self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) self.sock.connect(host) except socket.error, msg: - if self.debuglevel > 0: print>>stderr, 'connect fail:', host + if self.debuglevel > 0: + print>>stderr, 'connect fail:', host if self.sock: self.sock.close() self.sock = None raise socket.error, msg (code, msg) = self.getreply() - if self.debuglevel > 0: print>>stderr, "connect:", msg + if self.debuglevel > 0: + print>>stderr, "connect:", msg return (code, msg) @@ -815,7 +832,7 @@ return sys.stdin.readline().strip() fromaddr = prompt("From") - toaddrs = prompt("To").split(',') + toaddrs = prompt("To").split(',') print "Enter message, end with ^D:" msg = '' while 1: diff --git a/lib-python/2.7/ssl.py b/lib-python/2.7/ssl.py --- a/lib-python/2.7/ssl.py +++ b/lib-python/2.7/ssl.py @@ -121,9 +121,11 @@ if e.errno != errno.ENOTCONN: raise # no, no connection yet + self._connected = False self._sslobj = None else: # yes, create the SSL object + self._connected = True self._sslobj = _ssl.sslwrap(self._sock, server_side, keyfile, certfile, cert_reqs, ssl_version, ca_certs, @@ -293,21 +295,36 @@ self._sslobj.do_handshake() - def connect(self, addr): - - """Connects to remote ADDR, and then wraps the connection in - an SSL channel.""" - + def _real_connect(self, addr, return_errno): # Here we assume that the socket is client-side, and not # connected at the time of the call. We connect it, then wrap it. - if self._sslobj: + if self._connected: raise ValueError("attempt to connect already-connected SSLSocket!") - socket.connect(self, addr) self._sslobj = _ssl.sslwrap(self._sock, False, self.keyfile, self.certfile, self.cert_reqs, self.ssl_version, self.ca_certs, self.ciphers) - if self.do_handshake_on_connect: - self.do_handshake() + try: + socket.connect(self, addr) + if self.do_handshake_on_connect: + self.do_handshake() + except socket_error as e: + if return_errno: + return e.errno + else: + self._sslobj = None + raise e + self._connected = True + return 0 + + def connect(self, addr): + """Connects to remote ADDR, and then wraps the connection in + an SSL channel.""" + self._real_connect(addr, False) + + def connect_ex(self, addr): + """Connects to remote ADDR, and then wraps the connection in + an SSL channel.""" + return self._real_connect(addr, True) def accept(self): diff --git a/lib-python/2.7/subprocess.py b/lib-python/2.7/subprocess.py --- a/lib-python/2.7/subprocess.py +++ b/lib-python/2.7/subprocess.py @@ -396,6 +396,7 @@ import traceback import gc import signal +import errno # Exception classes used by this module. class CalledProcessError(Exception): @@ -427,7 +428,6 @@ else: import select _has_poll = hasattr(select, 'poll') - import errno import fcntl import pickle @@ -441,8 +441,15 @@ "check_output", "CalledProcessError"] if mswindows: - from _subprocess import CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP - __all__.extend(["CREATE_NEW_CONSOLE", "CREATE_NEW_PROCESS_GROUP"]) + from _subprocess import (CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP, + STD_INPUT_HANDLE, STD_OUTPUT_HANDLE, + STD_ERROR_HANDLE, SW_HIDE, + STARTF_USESTDHANDLES, STARTF_USESHOWWINDOW) + + __all__.extend(["CREATE_NEW_CONSOLE", "CREATE_NEW_PROCESS_GROUP", + "STD_INPUT_HANDLE", "STD_OUTPUT_HANDLE", + "STD_ERROR_HANDLE", "SW_HIDE", + "STARTF_USESTDHANDLES", "STARTF_USESHOWWINDOW"]) try: MAXFD = os.sysconf("SC_OPEN_MAX") except: @@ -726,7 +733,11 @@ stderr = None if self.stdin: if input: - self.stdin.write(input) + try: + self.stdin.write(input) + except IOError as e: + if e.errno != errno.EPIPE and e.errno != errno.EINVAL: + raise self.stdin.close() elif self.stdout: stdout = self.stdout.read() @@ -883,7 +894,7 @@ except pywintypes.error, e: # Translate pywintypes.error to WindowsError, which is # a subclass of OSError. FIXME: We should really - # translate errno using _sys_errlist (or simliar), but + # translate errno using _sys_errlist (or similar), but # how can this be done from Python? raise WindowsError(*e.args) finally: @@ -956,7 +967,11 @@ if self.stdin: if input is not None: - self.stdin.write(input) + try: + self.stdin.write(input) + except IOError as e: + if e.errno != errno.EPIPE: + raise self.stdin.close() if self.stdout: @@ -1051,14 +1066,17 @@ errread, errwrite) - def _set_cloexec_flag(self, fd): + def _set_cloexec_flag(self, fd, cloexec=True): try: cloexec_flag = fcntl.FD_CLOEXEC except AttributeError: cloexec_flag = 1 old = fcntl.fcntl(fd, fcntl.F_GETFD) - fcntl.fcntl(fd, fcntl.F_SETFD, old | cloexec_flag) + if cloexec: + fcntl.fcntl(fd, fcntl.F_SETFD, old | cloexec_flag) + else: + fcntl.fcntl(fd, fcntl.F_SETFD, old & ~cloexec_flag) def _close_fds(self, but): @@ -1128,21 +1146,25 @@ os.close(errpipe_read) # Dup fds for child - if p2cread is not None: - os.dup2(p2cread, 0) - if c2pwrite is not None: - os.dup2(c2pwrite, 1) - if errwrite is not None: - os.dup2(errwrite, 2) + def _dup2(a, b): + # dup2() removes the CLOEXEC flag but + # we must do it ourselves if dup2() + # would be a no-op (issue #10806). + if a == b: + self._set_cloexec_flag(a, False) + elif a is not None: + os.dup2(a, b) + _dup2(p2cread, 0) + _dup2(c2pwrite, 1) + _dup2(errwrite, 2) - # Close pipe fds. Make sure we don't close the same - # fd more than once, or standard fds. - if p2cread is not None and p2cread not in (0,): - os.close(p2cread) - if c2pwrite is not None and c2pwrite not in (p2cread, 1): - os.close(c2pwrite) - if errwrite is not None and errwrite not in (p2cread, c2pwrite, 2): - os.close(errwrite) + # Close pipe fds. Make sure we don't close the + # same fd more than once, or standard fds. + closed = { None } + for fd in [p2cread, c2pwrite, errwrite]: + if fd not in closed and fd > 2: + os.close(fd) + closed.add(fd) # Close all other fds, if asked for if close_fds: @@ -1194,7 +1216,11 @@ os.close(errpipe_read) if data != "": - _eintr_retry_call(os.waitpid, self.pid, 0) + try: + _eintr_retry_call(os.waitpid, self.pid, 0) + except OSError as e: + if e.errno != errno.ECHILD: + raise child_exception = pickle.loads(data) for fd in (p2cwrite, c2pread, errread): if fd is not None: @@ -1240,7 +1266,15 @@ """Wait for child process to terminate. Returns returncode attribute.""" if self.returncode is None: - pid, sts = _eintr_retry_call(os.waitpid, self.pid, 0) + try: + pid, sts = _eintr_retry_call(os.waitpid, self.pid, 0) + except OSError as e: + if e.errno != errno.ECHILD: + raise + # This happens if SIGCLD is set to be ignored or waiting + # for child processes has otherwise been disabled for our + # process. This child is dead, we can't get the status. + sts = 0 self._handle_exitstatus(sts) return self.returncode @@ -1317,9 +1351,16 @@ for fd, mode in ready: if mode & select.POLLOUT: chunk = input[input_offset : input_offset + _PIPE_BUF] - input_offset += os.write(fd, chunk) - if input_offset >= len(input): - close_unregister_and_remove(fd) + try: + input_offset += os.write(fd, chunk) + except OSError as e: + if e.errno == errno.EPIPE: + close_unregister_and_remove(fd) + else: + raise + else: + if input_offset >= len(input): + close_unregister_and_remove(fd) elif mode & select_POLLIN_POLLPRI: data = os.read(fd, 4096) if not data: @@ -1358,11 +1399,19 @@ if self.stdin in wlist: chunk = input[input_offset : input_offset + _PIPE_BUF] - bytes_written = os.write(self.stdin.fileno(), chunk) - input_offset += bytes_written - if input_offset >= len(input): - self.stdin.close() - write_set.remove(self.stdin) + try: + bytes_written = os.write(self.stdin.fileno(), chunk) + except OSError as e: + if e.errno == errno.EPIPE: + self.stdin.close() + write_set.remove(self.stdin) + else: + raise + else: + input_offset += bytes_written + if input_offset >= len(input): + self.stdin.close() + write_set.remove(self.stdin) if self.stdout in rlist: data = os.read(self.stdout.fileno(), 1024) diff --git a/lib-python/2.7/symbol.py b/lib-python/2.7/symbol.py --- a/lib-python/2.7/symbol.py +++ b/lib-python/2.7/symbol.py @@ -82,20 +82,19 @@ sliceop = 325 exprlist = 326 testlist = 327 -dictmaker = 328 -dictorsetmaker = 329 -classdef = 330 -arglist = 331 -argument = 332 -list_iter = 333 -list_for = 334 -list_if = 335 -comp_iter = 336 -comp_for = 337 -comp_if = 338 -testlist1 = 339 -encoding_decl = 340 -yield_expr = 341 +dictorsetmaker = 328 +classdef = 329 +arglist = 330 +argument = 331 +list_iter = 332 +list_for = 333 +list_if = 334 +comp_iter = 335 +comp_for = 336 +comp_if = 337 +testlist1 = 338 +encoding_decl = 339 +yield_expr = 340 #--end constants-- sym_name = {} diff --git a/lib-python/2.7/sysconfig.py b/lib-python/2.7/sysconfig.py --- a/lib-python/2.7/sysconfig.py +++ b/lib-python/2.7/sysconfig.py @@ -271,7 +271,7 @@ def _get_makefile_filename(): if _PYTHON_BUILD: return os.path.join(_PROJECT_BASE, "Makefile") - return os.path.join(get_path('stdlib'), "config", "Makefile") + return os.path.join(get_path('platstdlib'), "config", "Makefile") def _init_posix(vars): @@ -297,21 +297,6 @@ msg = msg + " (%s)" % e.strerror raise IOError(msg) - # On MacOSX we need to check the setting of the environment variable - # MACOSX_DEPLOYMENT_TARGET: configure bases some choices on it so - # it needs to be compatible. - # If it isn't set we set it to the configure-time value - if sys.platform == 'darwin' and 'MACOSX_DEPLOYMENT_TARGET' in vars: - cfg_target = vars['MACOSX_DEPLOYMENT_TARGET'] - cur_target = os.getenv('MACOSX_DEPLOYMENT_TARGET', '') - if cur_target == '': - cur_target = cfg_target - os.putenv('MACOSX_DEPLOYMENT_TARGET', cfg_target) - elif map(int, cfg_target.split('.')) > map(int, cur_target.split('.')): - msg = ('$MACOSX_DEPLOYMENT_TARGET mismatch: now "%s" but "%s" ' - 'during configure' % (cur_target, cfg_target)) - raise IOError(msg) - # On AIX, there are wrong paths to the linker scripts in the Makefile # -- these paths are relative to the Python source, but when installed # the scripts are in another directory. @@ -616,9 +601,7 @@ # machine is going to compile and link as if it were # MACOSX_DEPLOYMENT_TARGET. cfgvars = get_config_vars() - macver = os.environ.get('MACOSX_DEPLOYMENT_TARGET') - if not macver: - macver = cfgvars.get('MACOSX_DEPLOYMENT_TARGET') + macver = cfgvars.get('MACOSX_DEPLOYMENT_TARGET') if 1: # Always calculate the release of the running machine, @@ -639,7 +622,6 @@ m = re.search( r'<key>ProductUserVisibleVersion</key>\s*' + r'<string>(.*?)</string>', f.read()) - f.close() if m is not None: macrelease = '.'.join(m.group(1).split('.')[:2]) # else: fall back to the default behaviour diff --git a/lib-python/2.7/tarfile.py b/lib-python/2.7/tarfile.py --- a/lib-python/2.7/tarfile.py +++ b/lib-python/2.7/tarfile.py @@ -2239,10 +2239,14 @@ if hasattr(os, "symlink") and hasattr(os, "link"): # For systems that support symbolic and hard links. if tarinfo.issym(): + if os.path.lexists(targetpath): + os.unlink(targetpath) os.symlink(tarinfo.linkname, targetpath) else: # See extract(). if os.path.exists(tarinfo._link_target): + if os.path.lexists(targetpath): + os.unlink(targetpath) os.link(tarinfo._link_target, targetpath) else: self._extract_member(self._find_link_target(tarinfo), targetpath) diff --git a/lib-python/2.7/telnetlib.py b/lib-python/2.7/telnetlib.py --- a/lib-python/2.7/telnetlib.py +++ b/lib-python/2.7/telnetlib.py @@ -236,7 +236,7 @@ """ if self.debuglevel > 0: - print 'Telnet(%s,%d):' % (self.host, self.port), + print 'Telnet(%s,%s):' % (self.host, self.port), if args: print msg % args else: diff --git a/lib-python/2.7/test/cjkencodings/big5-utf8.txt b/lib-python/2.7/test/cjkencodings/big5-utf8.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/big5-utf8.txt @@ -0,0 +1,9 @@ +如何在 Python 中使用既有的 C library? +　在資訊科技快速發展的今天, 開發及測試軟體的速度是不容忽視的 +課題. 為加快開發及測試的速度, 我們便常希望能利用一些已開發好的 +library, 並有一個 fast prototyping 的 programming language 可 +供使用. 目前有許許多多的 library 是以 C 寫成, 而 Python 是一個 +fast prototyping 的 programming language. 故我們希望能將既有的 +C library 拿到 Python 的環境中測試及整合. 其中最主要也是我們所 +要討論的問題就是: + diff --git a/lib-python/2.7/test/cjkencodings/big5.txt b/lib-python/2.7/test/cjkencodings/big5.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/big5.txt @@ -0,0 +1,9 @@ +�p��b Python ��ϥάJ�� C library? +�@�b��T��ާֳt�o�i��, �}�o�δ��ճn�骺�t�׬O��e�� +��D. ��[�ֶ}�o�δ��ժ��t��, �ڭ̫K�`�Ʊ��Q�Τ@�Ǥw�}�o�n�� +library, �æ��@�� fast prototyping �� programming language �i +�Ѩϥ�. �ثe��\�\�h�h�� library �O�H C �g��, �� Python �O�@�� +fast prototyping �� programming language. �G�ڭ̧Ʊ��N�J�� +C library �� Python ��Ҥ��դξ�X. �䤤�̥D�n�]�O�ڭ̩� +�n�Q�ת��D�N�O: + diff --git a/lib-python/2.7/test/cjkencodings/big5hkscs-utf8.txt b/lib-python/2.7/test/cjkencodings/big5hkscs-utf8.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/big5hkscs-utf8.txt @@ -0,0 +1,2 @@ +𠄌Ě鵮罓洆 +ÊÊ̄ê êê̄ diff --git a/lib-python/2.7/test/cjkencodings/big5hkscs.txt b/lib-python/2.7/test/cjkencodings/big5hkscs.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/big5hkscs.txt @@ -0,0 +1,2 @@ +�E�\�s�ڍ� +�f�b�� diff --git a/lib-python/2.7/test/cjkencodings/cp949-utf8.txt b/lib-python/2.7/test/cjkencodings/cp949-utf8.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/cp949-utf8.txt @@ -0,0 +1,9 @@ +똠방각하 펲시콜라 + +㉯㉯납!! 因九月패믤릔궈 ⓡⓖ훀¿¿¿ 긍뒙 ⓔ뎨 ㉯. . +亞영ⓔ능횹 . . . . 서울뤄 뎐학乙家훀 ! ! !ㅠ.ㅠ +흐흐흐 ㄱㄱㄱ☆ㅠ_ㅠ 어릨 탸콰긐 뎌응 칑九들乙㉯드긐 +설릌 家훀 . . . . 굴애쉌 ⓔ궈 ⓡ릘㉱긐 因仁川女中까즼 +와쒀훀 ! ! 亞영ⓔ 家능궈 ☆上관 없능궈능 亞능뒈훀 글애듴 +ⓡ려듀九 싀풔숴훀 어릨 因仁川女中싁⑨들앜!! ㉯㉯납♡ ⌒⌒* + diff --git a/lib-python/2.7/test/cjkencodings/cp949.txt b/lib-python/2.7/test/cjkencodings/cp949.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/cp949.txt @@ -0,0 +1,9 @@ +�c�氢�� ݶ� + +��!! ��В�p�� ި��R�� ѵ� ��. . +䬿��Ѵ�� . . . . �� ʫ�R ! ! !��.�� +�� ٤�_��  ��O �� h�� O +��j ʫ�R . . . . ��֚f �ѱ� �ސt�ƒO �� +�;��R ! ! 䬿�� ʫ�ɱ� ��߾�� ɱŴ� 䬴ɵ��R �۾֊� +�޷�� Ǵ��R � ��Ĩ��!! �� ҡ�* + diff --git a/lib-python/2.7/test/cjkencodings/euc_jisx0213-utf8.txt b/lib-python/2.7/test/cjkencodings/euc_jisx0213-utf8.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/euc_jisx0213-utf8.txt @@ -0,0 +1,8 @@ +Python の開発は、1990 年ごろから開始されています。 +開発者の Guido van Rossum は教育用のプログラミング言語「ABC」の開発に参加していましたが、ABC は実用上の目的にはあまり適していませんでした。 +このため、Guido はより実用的なプログラミング言語の開発を開始し、英国 BBS 放送のコメディ番組「モンティパイソン」のファンである Guido はこの言語を「Python」と名づけました。 +このような背景から生まれた Python の言語設計は、「シンプル」で「習得が容易」という目標に重点が置かれています。 +多くのスクリプト系言語ではユーザの目先の利便性を優先して色々な機能を言語要素として取り入れる場合が多いのですが、Python ではそういった小細工が追加されることはあまりありません。 +言語自体の機能は最小限に押さえ、必要な機能は拡張モジュールとして追加する、というのが Python のポリシーです。 + +ノか゚ト゚トキ喝塀𡚴𪎌麀齁𩛰 diff --git a/lib-python/2.7/test/cjkencodings/euc_jisx0213.txt b/lib-python/2.7/test/cjkencodings/euc_jisx0213.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/euc_jisx0213.txt @@ -0,0 +1,8 @@ +Python �γ�ȯ�ϡ�1990 ǯ��鳫�Ϥ��Ƥ��ޤ�� +��ȯ�Ԥ� Guido van Rossum �϶��ѤΥץ��ߥ󥰸��ABC�פγ�ȯ�˻��ä��Ƥ��ޤ��ABC �ϼ��Ѿ��Ū�ˤϤ��ޤ�Ŭ��Ƥ��ޤ��Ǥ�� +��Τ��ᡢGuido �Ϥ��Ū�ʥץ��ߥ󥰸��γ�ȯ�򳫻Ϥ��ѹ� BBS ��Υ��ǥ��ȡ֥��ƥ� �ѥ��פΥե��Ǥ�� Guido �Ϥ��θ��Python�פ�̾�Ť��ޤ�� +��Τ褦��طʤ��ޤ줿 Python �θ��߷פϡ��֥��ץ�פǡֽ��ưספȤ��ɸ�˽��֤��Ƥ��ޤ�� +¿��Υ��ץȷϸ��Ǥϥ桼��ͥ�褷�ƿ��ʵ�ǽ��ǤȤ��Ƽ��礬¿��ΤǤ��Python �ǤϤ��ä��ٹ��ɲä��뤳�ȤϤ��ޤꤢ��ޤ�� +��켫�Τε�ǽ�ϺǾ��¤˲��ɬ�פʵ�ǽ�ϳ�ĥ�⥸�塼��Ȥ��ɲä��롢�Ȥ��Τ� Python �Υݥꥷ��Ǥ�� + +�Τ� �� ȥ�� ԏ�� diff --git a/lib-python/2.7/test/cjkencodings/euc_jp-utf8.txt b/lib-python/2.7/test/cjkencodings/euc_jp-utf8.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/euc_jp-utf8.txt @@ -0,0 +1,7 @@ +Python の開発は、1990 年ごろから開始されています。 +開発者の Guido van Rossum は教育用のプログラミング言語「ABC」の開発に参加していましたが、ABC は実用上の目的にはあまり適していませんでした。 +このため、Guido はより実用的なプログラミング言語の開発を開始し、英国 BBS 放送のコメディ番組「モンティパイソン」のファンである Guido はこの言語を「Python」と名づけました。 +このような背景から生まれた Python の言語設計は、「シンプル」で「習得が容易」という目標に重点が置かれています。 +多くのスクリプト系言語ではユーザの目先の利便性を優先して色々な機能を言語要素として取り入れる場合が多いのですが、Python ではそういった小細工が追加されることはあまりありません。 +言語自体の機能は最小限に押さえ、必要な機能は拡張モジュールとして追加する、というのが Python のポリシーです。 + diff --git a/lib-python/2.7/test/cjkencodings/euc_jp.txt b/lib-python/2.7/test/cjkencodings/euc_jp.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/euc_jp.txt @@ -0,0 +1,7 @@ +Python �γ�ȯ�ϡ�1990 ǯ��鳫�Ϥ��Ƥ��ޤ�� +��ȯ�Ԥ� Guido van Rossum �϶��ѤΥץ��ߥ󥰸��ABC�פγ�ȯ�˻��ä��Ƥ��ޤ��ABC �ϼ��Ѿ��Ū�ˤϤ��ޤ�Ŭ��Ƥ��ޤ��Ǥ�� +��Τ��ᡢGuido �Ϥ��Ū�ʥץ��ߥ󥰸��γ�ȯ�򳫻Ϥ��ѹ� BBS ��Υ��ǥ��ȡ֥��ƥ� �ѥ��פΥե��Ǥ�� Guido �Ϥ��θ��Python�פ�̾�Ť��ޤ�� +��Τ褦��طʤ��ޤ줿 Python �θ��߷פϡ��֥��ץ�פǡֽ��ưספȤ��ɸ�˽��֤��Ƥ��ޤ�� +¿��Υ��ץȷϸ��Ǥϥ桼��ͥ�褷�ƿ��ʵ�ǽ��ǤȤ��Ƽ��礬¿��ΤǤ��Python �ǤϤ��ä��ٹ��ɲä��뤳�ȤϤ��ޤꤢ��ޤ�� +��켫�Τε�ǽ�ϺǾ��¤˲��ɬ�פʵ�ǽ�ϳ�ĥ�⥸�塼��Ȥ��ɲä��롢�Ȥ��Τ� Python �Υݥꥷ��Ǥ�� + diff --git a/lib-python/2.7/test/cjkencodings/euc_kr-utf8.txt b/lib-python/2.7/test/cjkencodings/euc_kr-utf8.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/euc_kr-utf8.txt @@ -0,0 +1,7 @@ +◎ 파이썬(Python)은 배우기 쉽고, 강력한 프로그래밍 언어입니다. 파이썬은 +효율적인 고수준 데이터 구조와 간단하지만 효율적인 객체지향프로그래밍을 +지원합니다. 파이썬의 우아(優雅)한 문법과 동적 타이핑, 그리고 인터프리팅 +환경은 파이썬을 스크립팅과 여러 분야에서와 대부분의 플랫폼에서의 빠른 +애플리케이션 개발을 할 수 있는 이상적인 언어로 만들어줍니다. + +☆첫가끝: 날아라 쓔쓔쓩~ 닁큼! 뜽금없이 전홥니다. 뷁. 그런거 읎다. diff --git a/lib-python/2.7/test/cjkencodings/euc_kr.txt b/lib-python/2.7/test/cjkencodings/euc_kr.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/euc_kr.txt @@ -0,0 +1,7 @@ +�� ̽�(Python)�� , �� α׷�� Դϴ�. ��̽�� +ȿ�� ȿ�� ü��α׷�� +��մϴ�. ��̽�� (��)�� Ÿ��, �׸�� +ȯ�� ̽�� ũ��ð� �� о߿�� κ�� ÷�� +��ø��̼� �� ִ� �̻�� ݴϴ�. + +��ù��: ��ƶ� �Ԥ��ФԤԤ��ФԾ�~ �Ԥ��Ҥ�ŭ! �Ԥ��Ѥ��ݾ�� Ԥ��Ȥ��ϴ�. �Ԥ��Τ�. �׷�� Ԥ��Ѥ��. diff --git a/lib-python/2.7/test/cjkencodings/gb18030-utf8.txt b/lib-python/2.7/test/cjkencodings/gb18030-utf8.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/gb18030-utf8.txt @@ -0,0 +1,15 @@ +Python（派森）语言是一种功能强大而完善的通用型计算机程序设计语言， +已经具有十多年的发展历史，成熟且稳定。这种语言具有非常简捷而清晰 +的语法特点，适合完成各种高层任务，几乎可以在所有的操作系统中 +运行。这种语言简单而强大，适合各种人士学习使用。目前，基于这 +种语言的相关技术正在飞速的发展，用户数量急剧扩大，相关的资源非常多。 +如何在 Python 中使用既有的 C library? +　在資訊科技快速發展的今天, 開發及測試軟體的速度是不容忽視的 +課題. 為加快開發及測試的速度, 我們便常希望能利用一些已開發好的 +library, 並有一個 fast prototyping 的 programming language 可 +供使用. 目前有許許多多的 library 是以 C 寫成, 而 Python 是一個 +fast prototyping 的 programming language. 故我們希望能將既有的 +C library 拿到 Python 的環境中測試及整合. 其中最主要也是我們所 +要討論的問題就是: +파이썬은 강력한 기능을 지닌 범용 컴퓨터 프로그래밍 언어다. + diff --git a/lib-python/2.7/test/cjkencodings/gb18030.txt b/lib-python/2.7/test/cjkencodings/gb18030.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/gb18030.txt @@ -0,0 +1,15 @@ +Python��ɭ��һ�ֹ��ǿ��Ƶ�ͨ��ͼ��ԣ� +�Ѿ��ʮ��ķ�չ��ʷ��ȶ��Ծ��зǳ��ݶ�� +��﷨�ص㣬�ʺ��ɸ��ָ߲��񣬼��еĲ��ϵͳ�� +��С��Լ򵥶�ǿ��ʺϸ��ʿѧϰʹ�á�Ŀǰ�� +��Ե��ؼ��ڷ��ٵķ�չ��û��ص��Դ�ǳ��ࡣ +�� Python ��ʹ�ü��е� C library? +��YӍ�Ƽ��ٰlչ�Ľ��, �_�l��yԇܛ�w��ٶ��ǲ��ݺ�ҕ�� +�n�}. ��ӿ��_�l��yԇ��ٶ�, �҂��㳣ϣ��һЩ��_�l�õ� +library, �K��һ�� fast prototyping �� programming language �� +��ʹ��. Ŀǰ��S�S�� library �� C ��, �� Python ��һ�� +fast prototyping �� programming language. ��҂�ϣ��܌��е� +C library �õ� Python �ĭh��Мyԇ��. ��ҪҲ��҂�� +ҪӑՓ�Ć��}��: +�5�1�3�3�2�1�3�1 �7�6�0�4�6�3 �8�5�8�6�3�5 �3�1�9�5 �0�9�3�0 �4�3�5�7�5�5 �5�5�0�9�8�9�9�3�0�4 �2�9�2�5�9�9. + diff --git a/lib-python/2.7/test/cjkencodings/gb2312-utf8.txt b/lib-python/2.7/test/cjkencodings/gb2312-utf8.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/gb2312-utf8.txt @@ -0,0 +1,6 @@ +Python（派森）语言是一种功能强大而完善的通用型计算机程序设计语言， +已经具有十多年的发展历史，成熟且稳定。这种语言具有非常简捷而清晰 +的语法特点，适合完成各种高层任务，几乎可以在所有的操作系统中 +运行。这种语言简单而强大，适合各种人士学习使用。目前，基于这 +种语言的相关技术正在飞速的发展，用户数量急剧扩大，相关的资源非常多。 + diff --git a/lib-python/2.7/test/cjkencodings/gb2312.txt b/lib-python/2.7/test/cjkencodings/gb2312.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/gb2312.txt @@ -0,0 +1,6 @@ +Python��ɭ��һ�ֹ��ǿ��Ƶ�ͨ��ͼ��ԣ� +�Ѿ��ʮ��ķ�չ��ʷ��ȶ��Ծ��зǳ��ݶ�� +��﷨�ص㣬�ʺ��ɸ��ָ߲��񣬼��еĲ��ϵͳ�� +��С��Լ򵥶�ǿ��ʺϸ��ʿѧϰʹ�á�Ŀǰ�� +��Ե��ؼ��ڷ��ٵķ�չ��û��ص��Դ�ǳ��ࡣ + diff --git a/lib-python/2.7/test/cjkencodings/gbk-utf8.txt b/lib-python/2.7/test/cjkencodings/gbk-utf8.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/gbk-utf8.txt @@ -0,0 +1,14 @@ +Python（派森）语言是一种功能强大而完善的通用型计算机程序设计语言， +已经具有十多年的发展历史，成熟且稳定。这种语言具有非常简捷而清晰 +的语法特点，适合完成各种高层任务，几乎可以在所有的操作系统中 +运行。这种语言简单而强大，适合各种人士学习使用。目前，基于这 +种语言的相关技术正在飞速的发展，用户数量急剧扩大，相关的资源非常多。 +如何在 Python 中使用既有的 C library? +　在資訊科技快速發展的今天, 開發及測試軟體的速度是不容忽視的 +課題. 為加快開發及測試的速度, 我們便常希望能利用一些已開發好的 +library, 並有一個 fast prototyping 的 programming language 可 +供使用. 目前有許許多多的 library 是以 C 寫成, 而 Python 是一個 +fast prototyping 的 programming language. 故我們希望能將既有的 +C library 拿到 Python 的環境中測試及整合. 其中最主要也是我們所 +要討論的問題就是: + diff --git a/lib-python/2.7/test/cjkencodings/gbk.txt b/lib-python/2.7/test/cjkencodings/gbk.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/gbk.txt @@ -0,0 +1,14 @@ +Python��ɭ��һ�ֹ��ǿ��Ƶ�ͨ��ͼ��ԣ� +�Ѿ��ʮ��ķ�չ��ʷ��ȶ��Ծ��зǳ��ݶ�� +��﷨�ص㣬�ʺ��ɸ��ָ߲��񣬼��еĲ��ϵͳ�� +��С��Լ򵥶�ǿ��ʺϸ��ʿѧϰʹ�á�Ŀǰ�� +��Ե��ؼ��ڷ��ٵķ�չ��û��ص��Դ�ǳ��ࡣ +�� Python ��ʹ�ü��е� C library? +��YӍ�Ƽ��ٰlչ�Ľ��, �_�l��yԇܛ�w��ٶ��ǲ��ݺ�ҕ�� +�n�}. ��ӿ��_�l��yԇ��ٶ�, �҂��㳣ϣ��һЩ��_�l�õ� +library, �K��һ�� fast prototyping �� programming language �� +��ʹ��. Ŀǰ��S�S�� library �� C ��, �� Python ��һ�� +fast prototyping �� programming language. ��҂�ϣ��܌��е� +C library �õ� Python �ĭh��Мyԇ��. ��ҪҲ��҂�� +ҪӑՓ�Ć��}��: + diff --git a/lib-python/2.7/test/cjkencodings/hz-utf8.txt b/lib-python/2.7/test/cjkencodings/hz-utf8.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/hz-utf8.txt @@ -0,0 +1,2 @@ +This sentence is in ASCII. +The next sentence is in GB.己所不欲，勿施於人。Bye. diff --git a/lib-python/2.7/test/cjkencodings/hz.txt b/lib-python/2.7/test/cjkencodings/hz.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/hz.txt @@ -0,0 +1,2 @@ +This sentence is in ASCII. +The next sentence is in GB.~{<:Ky2;S{#,NpJ)l6HK!#~}Bye. diff --git a/lib-python/2.7/test/cjkencodings/johab-utf8.txt b/lib-python/2.7/test/cjkencodings/johab-utf8.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/johab-utf8.txt @@ -0,0 +1,9 @@ +똠방각하 펲시콜라 + +㉯㉯납!! 因九月패믤릔궈 ⓡⓖ훀¿¿¿ 긍뒙 ⓔ뎨 ㉯. . +亞영ⓔ능횹 . . . . 서울뤄 뎐학乙家훀 ! ! !ㅠ.ㅠ +흐흐흐 ㄱㄱㄱ☆ㅠ_ㅠ 어릨 탸콰긐 뎌응 칑九들乙㉯드긐 +설릌 家훀 . . . . 굴애쉌 ⓔ궈 ⓡ릘㉱긐 因仁川女中까즼 +와쒀훀 ! ! 亞영ⓔ 家능궈 ☆上관 없능궈능 亞능뒈훀 글애듴 +ⓡ려듀九 싀풔숴훀 어릨 因仁川女中싁⑨들앜!! ㉯㉯납♡ ⌒⌒* + diff --git a/lib-python/2.7/test/cjkencodings/johab.txt b/lib-python/2.7/test/cjkencodings/johab.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/johab.txt @@ -0,0 +1,9 @@ +��w�b�a �\��ũ�a + +��s!! �g��Ú�� zٯٯٯ �w�� ѕ� ��. . +�<�w�ѓw�s . . . . �ᶉ�� e�b�� ;�z ! ! !�A.�A +�a�a�a �A�A�A�i�A_�A �៚ ȡ��z �a�w ×✗i�� a�z +��z �;�z . . . . �� ъ� �ޟ��‹z �g�b�I��a�� +��z ! ! �<�w�� ;�w�� i꾉� ��w��w �<�w��z �i��z +�ޝa�A� ��Ρ��z �៚ �g�b�I��鯂��i�z!! ��sٽ �b�b* + diff --git a/lib-python/2.7/test/cjkencodings/shift_jis-utf8.txt b/lib-python/2.7/test/cjkencodings/shift_jis-utf8.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/shift_jis-utf8.txt @@ -0,0 +1,7 @@ +Python の開発は、1990 年ごろから開始されています。 +開発者の Guido van Rossum は教育用のプログラミング言語「ABC」の開発に参加していましたが、ABC は実用上の目的にはあまり適していませんでした。 +このため、Guido はより実用的なプログラミング言語の開発を開始し、英国 BBS 放送のコメディ番組「モンティパイソン」のファンである Guido はこの言語を「Python」と名づけました。 +このような背景から生まれた Python の言語設計は、「シンプル」で「習得が容易」という目標に重点が置かれています。 +多くのスクリプト系言語ではユーザの目先の利便性を優先して色々な機能を言語要素として取り入れる場合が多いのですが、Python ではそういった小細工が追加されることはあまりありません。 +言語自体の機能は最小限に押さえ、必要な機能は拡張モジュールとして追加する、というのが Python のポリシーです。 + diff --git a/lib-python/2.7/test/cjkencodings/shift_jis.txt b/lib-python/2.7/test/cjkencodings/shift_jis.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/shift_jis.txt @@ -0,0 +1,7 @@ +Python �̊J��́A1990 �N��납��J�n��Ă��܂��B +�J��҂� Guido van Rossum �͋��p�̃v��O��~��O��uABC�v�̊J��ɎQ��Ă��܂��AABC �͎��p��̖ړI�ɂ͂��܂�K��Ă��܂��ł��B +��̂��߁AGuido �͂��p�I�ȃv��O��~��O��̊J��J�n��A�p�� BBS ��̃R��f�B�ԑg�u��e�B �p�C�\��v�̃t�@��ł�� Guido �͂��̌��uPython�v�Ɩ��Â��܂��B +��̂悤�Ȕw�i��琶�܂ꂽ Python �̌��݌v�́A�u�V��v��v�Łu�K��e�Ձv�Ƃ��ڕW�ɏd�_��u��Ă��܂��B +��̃X�N��v�g�n��ł̓��[�U�̖ڐ�̗��֐��D�悵�ĐF�X�ȋ@�\��v�f�Ƃ��Ď��ꍇ��̂ł��APython �ł͂��׍H��ǉ��邱�Ƃ͂��܂肠��܂��B +��ꎩ�̂̋@�\�͍ŏ��ɉ��A�K�v�ȋ@�\�͊g��W��[��Ƃ��Ēǉ��A�Ƃ��̂� Python �̃|��V�[�ł��B + diff --git a/lib-python/2.7/test/cjkencodings/shift_jisx0213-utf8.txt b/lib-python/2.7/test/cjkencodings/shift_jisx0213-utf8.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/shift_jisx0213-utf8.txt @@ -0,0 +1,8 @@ +Python の開発は、1990 年ごろから開始されています。 +開発者の Guido van Rossum は教育用のプログラミング言語「ABC」の開発に参加していましたが、ABC は実用上の目的にはあまり適していませんでした。 +このため、Guido はより実用的なプログラミング言語の開発を開始し、英国 BBS 放送のコメディ番組「モンティパイソン」のファンである Guido はこの言語を「Python」と名づけました。 +このような背景から生まれた Python の言語設計は、「シンプル」で「習得が容易」という目標に重点が置かれています。 +多くのスクリプト系言語ではユーザの目先の利便性を優先して色々な機能を言語要素として取り入れる場合が多いのですが、Python ではそういった小細工が追加されることはあまりありません。 +言語自体の機能は最小限に押さえ、必要な機能は拡張モジュールとして追加する、というのが Python のポリシーです。 + +ノか゚ト゚トキ喝塀𡚴𪎌麀齁𩛰 diff --git a/lib-python/2.7/test/cjkencodings/shift_jisx0213.txt b/lib-python/2.7/test/cjkencodings/shift_jisx0213.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/shift_jisx0213.txt @@ -0,0 +1,8 @@ +Python �̊J��́A1990 �N��납��J�n��Ă��܂��B +�J��҂� Guido van Rossum �͋��p�̃v��O��~��O��uABC�v�̊J��ɎQ��Ă��܂��AABC �͎��p��̖ړI�ɂ͂��܂�K��Ă��܂��ł��B +��̂��߁AGuido �͂��p�I�ȃv��O��~��O��̊J��J�n��A�p�� BBS ��̃R��f�B�ԑg�u��e�B �p�C�\��v�̃t�@��ł�� Guido �͂��̌��uPython�v�Ɩ��Â��܂��B +��̂悤�Ȕw�i��琶�܂ꂽ Python �̌��݌v�́A�u�V��v��v�Łu�K��e�Ձv�Ƃ��ڕW�ɏd�_��u��Ă��܂��B +��̃X�N��v�g�n��ł̓��[�U�̖ڐ�̗��֐��D�悵�ĐF�X�ȋ@�\��v�f�Ƃ��Ď��ꍇ��̂ł��APython �ł͂��׍H��ǉ��邱�Ƃ͂��܂肠��܂��B +��ꎩ�̂̋@�\�͍ŏ��ɉ��A�K�v�ȋ@�\�͊g��W��[��Ƃ��Ēǉ��A�Ƃ��̂� Python �̃|��V�[�ł��B + +�m�� g�L�K�y �� diff --git a/lib-python/2.7/test/cjkencodings_test.py b/lib-python/2.7/test/cjkencodings_test.py deleted file mode 100644 --- a/lib-python/2.7/test/cjkencodings_test.py +++ /dev/null @@ -1,1019 +0,0 @@ -teststring = { -'big5': ( -"\xa6\x70\xa6\xf3\xa6\x62\x20\x50\x79\x74\x68\x6f\x6e\x20\xa4\xa4" -"\xa8\xcf\xa5\xce\xac\x4a\xa6\xb3\xaa\xba\x20\x43\x20\x6c\x69\x62" -"\x72\x61\x72\x79\x3f\x0a\xa1\x40\xa6\x62\xb8\xea\xb0\x54\xac\xec" -"\xa7\xde\xa7\xd6\xb3\x74\xb5\x6f\xae\x69\xaa\xba\xa4\xb5\xa4\xd1" -"\x2c\x20\xb6\x7d\xb5\x6f\xa4\xce\xb4\xfa\xb8\xd5\xb3\x6e\xc5\xe9" -"\xaa\xba\xb3\x74\xab\xd7\xac\x4f\xa4\xa3\xae\x65\xa9\xbf\xb5\xf8" -"\xaa\xba\x0a\xbd\xd2\xc3\x44\x2e\x20\xac\xb0\xa5\x5b\xa7\xd6\xb6" -"\x7d\xb5\x6f\xa4\xce\xb4\xfa\xb8\xd5\xaa\xba\xb3\x74\xab\xd7\x2c" -"\x20\xa7\xda\xad\xcc\xab\x4b\xb1\x60\xa7\xc6\xb1\xe6\xaf\xe0\xa7" -"\x51\xa5\xce\xa4\x40\xa8\xc7\xa4\x77\xb6\x7d\xb5\x6f\xa6\x6e\xaa" -"\xba\x0a\x6c\x69\x62\x72\x61\x72\x79\x2c\x20\xa8\xc3\xa6\xb3\xa4" -"\x40\xad\xd3\x20\x66\x61\x73\x74\x20\x70\x72\x6f\x74\x6f\x74\x79" -"\x70\x69\x6e\x67\x20\xaa\xba\x20\x70\x72\x6f\x67\x72\x61\x6d\x6d" -"\x69\x6e\x67\x20\x6c\x61\x6e\x67\x75\x61\x67\x65\x20\xa5\x69\x0a" -"\xa8\xd1\xa8\xcf\xa5\xce\x2e\x20\xa5\xd8\xab\x65\xa6\xb3\xb3\x5c" -"\xb3\x5c\xa6\x68\xa6\x68\xaa\xba\x20\x6c\x69\x62\x72\x61\x72\x79" -"\x20\xac\x4f\xa5\x48\x20\x43\x20\xbc\x67\xa6\xa8\x2c\x20\xa6\xd3" -"\x20\x50\x79\x74\x68\x6f\x6e\x20\xac\x4f\xa4\x40\xad\xd3\x0a\x66" -"\x61\x73\x74\x20\x70\x72\x6f\x74\x6f\x74\x79\x70\x69\x6e\x67\x20" -"\xaa\xba\x20\x70\x72\x6f\x67\x72\x61\x6d\x6d\x69\x6e\x67\x20\x6c" -"\x61\x6e\x67\x75\x61\x67\x65\x2e\x20\xac\x47\xa7\xda\xad\xcc\xa7" -"\xc6\xb1\xe6\xaf\xe0\xb1\x4e\xac\x4a\xa6\xb3\xaa\xba\x0a\x43\x20" -"\x6c\x69\x62\x72\x61\x72\x79\x20\xae\xb3\xa8\xec\x20\x50\x79\x74" -"\x68\x6f\x6e\x20\xaa\xba\xc0\xf4\xb9\xd2\xa4\xa4\xb4\xfa\xb8\xd5" -"\xa4\xce\xbe\xe3\xa6\x58\x2e\x20\xa8\xe4\xa4\xa4\xb3\xcc\xa5\x44" -"\xad\x6e\xa4\x5d\xac\x4f\xa7\xda\xad\xcc\xa9\xd2\x0a\xad\x6e\xb0" -"\x51\xbd\xd7\xaa\xba\xb0\xdd\xc3\x44\xb4\x4e\xac\x4f\x3a\x0a\x0a", -"\xe5\xa6\x82\xe4\xbd\x95\xe5\x9c\xa8\x20\x50\x79\x74\x68\x6f\x6e" -"\x20\xe4\xb8\xad\xe4\xbd\xbf\xe7\x94\xa8\xe6\x97\xa2\xe6\x9c\x89" -"\xe7\x9a\x84\x20\x43\x20\x6c\x69\x62\x72\x61\x72\x79\x3f\x0a\xe3" -"\x80\x80\xe5\x9c\xa8\xe8\xb3\x87\xe8\xa8\x8a\xe7\xa7\x91\xe6\x8a" -"\x80\xe5\xbf\xab\xe9\x80\x9f\xe7\x99\xbc\xe5\xb1\x95\xe7\x9a\x84" -"\xe4\xbb\x8a\xe5\xa4\xa9\x2c\x20\xe9\x96\x8b\xe7\x99\xbc\xe5\x8f" -"\x8a\xe6\xb8\xac\xe8\xa9\xa6\xe8\xbb\x9f\xe9\xab\x94\xe7\x9a\x84" -"\xe9\x80\x9f\xe5\xba\xa6\xe6\x98\xaf\xe4\xb8\x8d\xe5\xae\xb9\xe5" -"\xbf\xbd\xe8\xa6\x96\xe7\x9a\x84\x0a\xe8\xaa\xb2\xe9\xa1\x8c\x2e" -"\x20\xe7\x82\xba\xe5\x8a\xa0\xe5\xbf\xab\xe9\x96\x8b\xe7\x99\xbc" -"\xe5\x8f\x8a\xe6\xb8\xac\xe8\xa9\xa6\xe7\x9a\x84\xe9\x80\x9f\xe5" -"\xba\xa6\x2c\x20\xe6\x88\x91\xe5\x80\x91\xe4\xbe\xbf\xe5\xb8\xb8" -"\xe5\xb8\x8c\xe6\x9c\x9b\xe8\x83\xbd\xe5\x88\xa9\xe7\x94\xa8\xe4" -"\xb8\x80\xe4\xba\x9b\xe5\xb7\xb2\xe9\x96\x8b\xe7\x99\xbc\xe5\xa5" -"\xbd\xe7\x9a\x84\x0a\x6c\x69\x62\x72\x61\x72\x79\x2c\x20\xe4\xb8" -"\xa6\xe6\x9c\x89\xe4\xb8\x80\xe5\x80\x8b\x20\x66\x61\x73\x74\x20" -"\x70\x72\x6f\x74\x6f\x74\x79\x70\x69\x6e\x67\x20\xe7\x9a\x84\x20" -"\x70\x72\x6f\x67\x72\x61\x6d\x6d\x69\x6e\x67\x20\x6c\x61\x6e\x67" -"\x75\x61\x67\x65\x20\xe5\x8f\xaf\x0a\xe4\xbe\x9b\xe4\xbd\xbf\xe7" -"\x94\xa8\x2e\x20\xe7\x9b\xae\xe5\x89\x8d\xe6\x9c\x89\xe8\xa8\xb1" -"\xe8\xa8\xb1\xe5\xa4\x9a\xe5\xa4\x9a\xe7\x9a\x84\x20\x6c\x69\x62" -"\x72\x61\x72\x79\x20\xe6\x98\xaf\xe4\xbb\xa5\x20\x43\x20\xe5\xaf" -"\xab\xe6\x88\x90\x2c\x20\xe8\x80\x8c\x20\x50\x79\x74\x68\x6f\x6e" -"\x20\xe6\x98\xaf\xe4\xb8\x80\xe5\x80\x8b\x0a\x66\x61\x73\x74\x20" -"\x70\x72\x6f\x74\x6f\x74\x79\x70\x69\x6e\x67\x20\xe7\x9a\x84\x20" -"\x70\x72\x6f\x67\x72\x61\x6d\x6d\x69\x6e\x67\x20\x6c\x61\x6e\x67" -"\x75\x61\x67\x65\x2e\x20\xe6\x95\x85\xe6\x88\x91\xe5\x80\x91\xe5" -"\xb8\x8c\xe6\x9c\x9b\xe8\x83\xbd\xe5\xb0\x87\xe6\x97\xa2\xe6\x9c" -"\x89\xe7\x9a\x84\x0a\x43\x20\x6c\x69\x62\x72\x61\x72\x79\x20\xe6" -"\x8b\xbf\xe5\x88\xb0\x20\x50\x79\x74\x68\x6f\x6e\x20\xe7\x9a\x84" -"\xe7\x92\xb0\xe5\xa2\x83\xe4\xb8\xad\xe6\xb8\xac\xe8\xa9\xa6\xe5" -"\x8f\x8a\xe6\x95\xb4\xe5\x90\x88\x2e\x20\xe5\x85\xb6\xe4\xb8\xad" -"\xe6\x9c\x80\xe4\xb8\xbb\xe8\xa6\x81\xe4\xb9\x9f\xe6\x98\xaf\xe6" -"\x88\x91\xe5\x80\x91\xe6\x89\x80\x0a\xe8\xa6\x81\xe8\xa8\x8e\xe8" -"\xab\x96\xe7\x9a\x84\xe5\x95\x8f\xe9\xa1\x8c\xe5\xb0\xb1\xe6\x98" -"\xaf\x3a\x0a\x0a"), -'big5hkscs': ( -"\x88\x45\x88\x5c\x8a\x73\x8b\xda\x8d\xd8\x0a\x88\x66\x88\x62\x88" -"\xa7\x20\x88\xa7\x88\xa3\x0a", -"\xf0\xa0\x84\x8c\xc4\x9a\xe9\xb5\xae\xe7\xbd\x93\xe6\xb4\x86\x0a" -"\xc3\x8a\xc3\x8a\xcc\x84\xc3\xaa\x20\xc3\xaa\xc3\xaa\xcc\x84\x0a"), -'cp949': ( -"\x8c\x63\xb9\xe6\xb0\xa2\xc7\xcf\x20\xbc\x84\xbd\xc3\xc4\xdd\xb6" -"\xf3\x0a\x0a\xa8\xc0\xa8\xc0\xb3\xb3\x21\x21\x20\xec\xd7\xce\xfa" -"\xea\xc5\xc6\xd0\x92\xe6\x90\x70\xb1\xc5\x20\xa8\xde\xa8\xd3\xc4" -"\x52\xa2\xaf\xa2\xaf\xa2\xaf\x20\xb1\xe0\x8a\x96\x20\xa8\xd1\xb5" -"\xb3\x20\xa8\xc0\x2e\x20\x2e\x0a\xe4\xac\xbf\xb5\xa8\xd1\xb4\xc9" -"\xc8\xc2\x20\x2e\x20\x2e\x20\x2e\x20\x2e\x20\xbc\xad\xbf\xef\xb7" -"\xef\x20\xb5\xaf\xc7\xd0\xeb\xe0\x20\xca\xab\xc4\x52\x20\x21\x20" -"\x21\x20\x21\xa4\xd0\x2e\xa4\xd0\x0a\xc8\xe5\xc8\xe5\xc8\xe5\x20" -"\xa4\xa1\xa4\xa1\xa4\xa1\xa1\xd9\xa4\xd0\x5f\xa4\xd0\x20\xbe\xee" -"\x90\x8a\x20\xc5\xcb\xc4\xe2\x83\x4f\x20\xb5\xae\xc0\xc0\x20\xaf" -"\x68\xce\xfa\xb5\xe9\xeb\xe0\x20\xa8\xc0\xb5\xe5\x83\x4f\x0a\xbc" -"\xb3\x90\x6a\x20\xca\xab\xc4\x52\x20\x2e\x20\x2e\x20\x2e\x20\x2e" -"\x20\xb1\xbc\xbe\xd6\x9a\x66\x20\xa8\xd1\xb1\xc5\x20\xa8\xde\x90" -"\x74\xa8\xc2\x83\x4f\x20\xec\xd7\xec\xd2\xf4\xb9\xe5\xfc\xf1\xe9" -"\xb1\xee\xa3\x8e\x0a\xbf\xcd\xbe\xac\xc4\x52\x20\x21\x20\x21\x20" -"\xe4\xac\xbf\xb5\xa8\xd1\x20\xca\xab\xb4\xc9\xb1\xc5\x20\xa1\xd9" -"\xdf\xbe\xb0\xfc\x20\xbe\xf8\xb4\xc9\xb1\xc5\xb4\xc9\x20\xe4\xac" -"\xb4\xc9\xb5\xd8\xc4\x52\x20\xb1\xdb\xbe\xd6\x8a\xdb\x0a\xa8\xde" -"\xb7\xc1\xb5\xe0\xce\xfa\x20\x9a\xc3\xc7\xb4\xbd\xa4\xc4\x52\x20" -"\xbe\xee\x90\x8a\x20\xec\xd7\xec\xd2\xf4\xb9\xe5\xfc\xf1\xe9\x9a" -"\xc4\xa8\xef\xb5\xe9\x9d\xda\x21\x21\x20\xa8\xc0\xa8\xc0\xb3\xb3" -"\xa2\xbd\x20\xa1\xd2\xa1\xd2\x2a\x0a\x0a", -"\xeb\x98\xa0\xeb\xb0\xa9\xea\xb0\x81\xed\x95\x98\x20\xed\x8e\xb2" -"\xec\x8b\x9c\xec\xbd\x9c\xeb\x9d\xbc\x0a\x0a\xe3\x89\xaf\xe3\x89" -"\xaf\xeb\x82\xa9\x21\x21\x20\xe5\x9b\xa0\xe4\xb9\x9d\xe6\x9c\x88" -"\xed\x8c\xa8\xeb\xaf\xa4\xeb\xa6\x94\xea\xb6\x88\x20\xe2\x93\xa1" -"\xe2\x93\x96\xed\x9b\x80\xc2\xbf\xc2\xbf\xc2\xbf\x20\xea\xb8\x8d" -"\xeb\x92\x99\x20\xe2\x93\x94\xeb\x8e\xa8\x20\xe3\x89\xaf\x2e\x20" -"\x2e\x0a\xe4\xba\x9e\xec\x98\x81\xe2\x93\x94\xeb\x8a\xa5\xed\x9a" -"\xb9\x20\x2e\x20\x2e\x20\x2e\x20\x2e\x20\xec\x84\x9c\xec\x9a\xb8" -"\xeb\xa4\x84\x20\xeb\x8e\x90\xed\x95\x99\xe4\xb9\x99\x20\xe5\xae" -"\xb6\xed\x9b\x80\x20\x21\x20\x21\x20\x21\xe3\x85\xa0\x2e\xe3\x85" -"\xa0\x0a\xed\x9d\x90\xed\x9d\x90\xed\x9d\x90\x20\xe3\x84\xb1\xe3" -"\x84\xb1\xe3\x84\xb1\xe2\x98\x86\xe3\x85\xa0\x5f\xe3\x85\xa0\x20" -"\xec\x96\xb4\xeb\xa6\xa8\x20\xed\x83\xb8\xec\xbd\xb0\xea\xb8\x90" -"\x20\xeb\x8e\x8c\xec\x9d\x91\x20\xec\xb9\x91\xe4\xb9\x9d\xeb\x93" -"\xa4\xe4\xb9\x99\x20\xe3\x89\xaf\xeb\x93\x9c\xea\xb8\x90\x0a\xec" -"\x84\xa4\xeb\xa6\x8c\x20\xe5\xae\xb6\xed\x9b\x80\x20\x2e\x20\x2e" -"\x20\x2e\x20\x2e\x20\xea\xb5\xb4\xec\x95\xa0\xec\x89\x8c\x20\xe2" -"\x93\x94\xea\xb6\x88\x20\xe2\x93\xa1\xeb\xa6\x98\xe3\x89\xb1\xea" -"\xb8\x90\x20\xe5\x9b\xa0\xe4\xbb\x81\xe5\xb7\x9d\xef\xa6\x81\xe4" -"\xb8\xad\xea\xb9\x8c\xec\xa6\xbc\x0a\xec\x99\x80\xec\x92\x80\xed" -"\x9b\x80\x20\x21\x20\x21\x20\xe4\xba\x9e\xec\x98\x81\xe2\x93\x94" -"\x20\xe5\xae\xb6\xeb\x8a\xa5\xea\xb6\x88\x20\xe2\x98\x86\xe4\xb8" -"\x8a\xea\xb4\x80\x20\xec\x97\x86\xeb\x8a\xa5\xea\xb6\x88\xeb\x8a" -"\xa5\x20\xe4\xba\x9e\xeb\x8a\xa5\xeb\x92\x88\xed\x9b\x80\x20\xea" -"\xb8\x80\xec\x95\xa0\xeb\x93\xb4\x0a\xe2\x93\xa1\xeb\xa0\xa4\xeb" -"\x93\x80\xe4\xb9\x9d\x20\xec\x8b\x80\xed\x92\x94\xec\x88\xb4\xed" -"\x9b\x80\x20\xec\x96\xb4\xeb\xa6\xa8\x20\xe5\x9b\xa0\xe4\xbb\x81" -"\xe5\xb7\x9d\xef\xa6\x81\xe4\xb8\xad\xec\x8b\x81\xe2\x91\xa8\xeb" -"\x93\xa4\xec\x95\x9c\x21\x21\x20\xe3\x89\xaf\xe3\x89\xaf\xeb\x82" -"\xa9\xe2\x99\xa1\x20\xe2\x8c\x92\xe2\x8c\x92\x2a\x0a\x0a"), -'euc_jisx0213': ( -"\x50\x79\x74\x68\x6f\x6e\x20\xa4\xce\xb3\xab\xc8\xaf\xa4\xcf\xa1" -"\xa2\x31\x39\x39\x30\x20\xc7\xaf\xa4\xb4\xa4\xed\xa4\xab\xa4\xe9" -"\xb3\xab\xbb\xcf\xa4\xb5\xa4\xec\xa4\xc6\xa4\xa4\xa4\xde\xa4\xb9" -"\xa1\xa3\x0a\xb3\xab\xc8\xaf\xbc\xd4\xa4\xce\x20\x47\x75\x69\x64" -"\x6f\x20\x76\x61\x6e\x20\x52\x6f\x73\x73\x75\x6d\x20\xa4\xcf\xb6" -"\xb5\xb0\xe9\xcd\xd1\xa4\xce\xa5\xd7\xa5\xed\xa5\xb0\xa5\xe9\xa5" -"\xdf\xa5\xf3\xa5\xb0\xb8\xc0\xb8\xec\xa1\xd6\x41\x42\x43\xa1\xd7" -"\xa4\xce\xb3\xab\xc8\xaf\xa4\xcb\xbb\xb2\xb2\xc3\xa4\xb7\xa4\xc6" -"\xa4\xa4\xa4\xde\xa4\xb7\xa4\xbf\xa4\xac\xa1\xa2\x41\x42\x43\x20" -"\xa4\xcf\xbc\xc2\xcd\xd1\xbe\xe5\xa4\xce\xcc\xdc\xc5\xaa\xa4\xcb" -"\xa4\xcf\xa4\xa2\xa4\xde\xa4\xea\xc5\xac\xa4\xb7\xa4\xc6\xa4\xa4" -"\xa4\xde\xa4\xbb\xa4\xf3\xa4\xc7\xa4\xb7\xa4\xbf\xa1\xa3\x0a\xa4" -"\xb3\xa4\xce\xa4\xbf\xa4\xe1\xa1\xa2\x47\x75\x69\x64\x6f\x20\xa4" -"\xcf\xa4\xe8\xa4\xea\xbc\xc2\xcd\xd1\xc5\xaa\xa4\xca\xa5\xd7\xa5" -"\xed\xa5\xb0\xa5\xe9\xa5\xdf\xa5\xf3\xa5\xb0\xb8\xc0\xb8\xec\xa4" -"\xce\xb3\xab\xc8\xaf\xa4\xf2\xb3\xab\xbb\xcf\xa4\xb7\xa1\xa2\xb1" -"\xd1\xb9\xf1\x20\x42\x42\x53\x20\xca\xfc\xc1\xf7\xa4\xce\xa5\xb3" -"\xa5\xe1\xa5\xc7\xa5\xa3\xc8\xd6\xc1\xc8\xa1\xd6\xa5\xe2\xa5\xf3" -"\xa5\xc6\xa5\xa3\x20\xa5\xd1\xa5\xa4\xa5\xbd\xa5\xf3\xa1\xd7\xa4" -"\xce\xa5\xd5\xa5\xa1\xa5\xf3\xa4\xc7\xa4\xa2\xa4\xeb\x20\x47\x75" -"\x69\x64\x6f\x20\xa4\xcf\xa4\xb3\xa4\xce\xb8\xc0\xb8\xec\xa4\xf2" -"\xa1\xd6\x50\x79\x74\x68\x6f\x6e\xa1\xd7\xa4\xc8\xcc\xbe\xa4\xc5" -"\xa4\xb1\xa4\xde\xa4\xb7\xa4\xbf\xa1\xa3\x0a\xa4\xb3\xa4\xce\xa4" -"\xe8\xa4\xa6\xa4\xca\xc7\xd8\xb7\xca\xa4\xab\xa4\xe9\xc0\xb8\xa4" -"\xde\xa4\xec\xa4\xbf\x20\x50\x79\x74\x68\x6f\x6e\x20\xa4\xce\xb8" -"\xc0\xb8\xec\xc0\xdf\xb7\xd7\xa4\xcf\xa1\xa2\xa1\xd6\xa5\xb7\xa5" -"\xf3\xa5\xd7\xa5\xeb\xa1\xd7\xa4\xc7\xa1\xd6\xbd\xac\xc6\xc0\xa4" -"\xac\xcd\xc6\xb0\xd7\xa1\xd7\xa4\xc8\xa4\xa4\xa4\xa6\xcc\xdc\xc9" -"\xb8\xa4\xcb\xbd\xc5\xc5\xc0\xa4\xac\xc3\xd6\xa4\xab\xa4\xec\xa4" -"\xc6\xa4\xa4\xa4\xde\xa4\xb9\xa1\xa3\x0a\xc2\xbf\xa4\xaf\xa4\xce" -"\xa5\xb9\xa5\xaf\xa5\xea\xa5\xd7\xa5\xc8\xb7\xcf\xb8\xc0\xb8\xec" -"\xa4\xc7\xa4\xcf\xa5\xe6\xa1\xbc\xa5\xb6\xa4\xce\xcc\xdc\xc0\xe8" -"\xa4\xce\xcd\xf8\xca\xd8\xc0\xad\xa4\xf2\xcd\xa5\xc0\xe8\xa4\xb7" -"\xa4\xc6\xbf\xa7\xa1\xb9\xa4\xca\xb5\xa1\xc7\xbd\xa4\xf2\xb8\xc0" -"\xb8\xec\xcd\xd7\xc1\xc7\xa4\xc8\xa4\xb7\xa4\xc6\xbc\xe8\xa4\xea" -"\xc6\xfe\xa4\xec\xa4\xeb\xbe\xec\xb9\xe7\xa4\xac\xc2\xbf\xa4\xa4" -"\xa4\xce\xa4\xc7\xa4\xb9\xa4\xac\xa1\xa2\x50\x79\x74\x68\x6f\x6e" -"\x20\xa4\xc7\xa4\xcf\xa4\xbd\xa4\xa6\xa4\xa4\xa4\xc3\xa4\xbf\xbe" -"\xae\xba\xd9\xb9\xa9\xa4\xac\xc4\xc9\xb2\xc3\xa4\xb5\xa4\xec\xa4" -"\xeb\xa4\xb3\xa4\xc8\xa4\xcf\xa4\xa2\xa4\xde\xa4\xea\xa4\xa2\xa4" -"\xea\xa4\xde\xa4\xbb\xa4\xf3\xa1\xa3\x0a\xb8\xc0\xb8\xec\xbc\xab" -"\xc2\xce\xa4\xce\xb5\xa1\xc7\xbd\xa4\xcf\xba\xc7\xbe\xae\xb8\xc2" -"\xa4\xcb\xb2\xa1\xa4\xb5\xa4\xa8\xa1\xa2\xc9\xac\xcd\xd7\xa4\xca" -"\xb5\xa1\xc7\xbd\xa4\xcf\xb3\xc8\xc4\xa5\xa5\xe2\xa5\xb8\xa5\xe5" -"\xa1\xbc\xa5\xeb\xa4\xc8\xa4\xb7\xa4\xc6\xc4\xc9\xb2\xc3\xa4\xb9" -"\xa4\xeb\xa1\xa2\xa4\xc8\xa4\xa4\xa4\xa6\xa4\xce\xa4\xac\x20\x50" -"\x79\x74\x68\x6f\x6e\x20\xa4\xce\xa5\xdd\xa5\xea\xa5\xb7\xa1\xbc" -"\xa4\xc7\xa4\xb9\xa1\xa3\x0a\x0a\xa5\xce\xa4\xf7\x20\xa5\xfe\x20" -"\xa5\xc8\xa5\xad\xaf\xac\xaf\xda\x20\xcf\xe3\x8f\xfe\xd8\x20\x8f" -"\xfe\xd4\x8f\xfe\xe8\x8f\xfc\xd6\x0a", -"\x50\x79\x74\x68\x6f\x6e\x20\xe3\x81\xae\xe9\x96\x8b\xe7\x99\xba" -"\xe3\x81\xaf\xe3\x80\x81\x31\x39\x39\x30\x20\xe5\xb9\xb4\xe3\x81" -"\x94\xe3\x82\x8d\xe3\x81\x8b\xe3\x82\x89\xe9\x96\x8b\xe5\xa7\x8b" -"\xe3\x81\x95\xe3\x82\x8c\xe3\x81\xa6\xe3\x81\x84\xe3\x81\xbe\xe3" -"\x81\x99\xe3\x80\x82\x0a\xe9\x96\x8b\xe7\x99\xba\xe8\x80\x85\xe3" -"\x81\xae\x20\x47\x75\x69\x64\x6f\x20\x76\x61\x6e\x20\x52\x6f\x73" -"\x73\x75\x6d\x20\xe3\x81\xaf\xe6\x95\x99\xe8\x82\xb2\xe7\x94\xa8" -"\xe3\x81\xae\xe3\x83\x97\xe3\x83\xad\xe3\x82\xb0\xe3\x83\xa9\xe3" -"\x83\x9f\xe3\x83\xb3\xe3\x82\xb0\xe8\xa8\x80\xe8\xaa\x9e\xe3\x80" -"\x8c\x41\x42\x43\xe3\x80\x8d\xe3\x81\xae\xe9\x96\x8b\xe7\x99\xba" -"\xe3\x81\xab\xe5\x8f\x82\xe5\x8a\xa0\xe3\x81\x97\xe3\x81\xa6\xe3" -"\x81\x84\xe3\x81\xbe\xe3\x81\x97\xe3\x81\x9f\xe3\x81\x8c\xe3\x80" -"\x81\x41\x42\x43\x20\xe3\x81\xaf\xe5\xae\x9f\xe7\x94\xa8\xe4\xb8" -"\x8a\xe3\x81\xae\xe7\x9b\xae\xe7\x9a\x84\xe3\x81\xab\xe3\x81\xaf" -"\xe3\x81\x82\xe3\x81\xbe\xe3\x82\x8a\xe9\x81\xa9\xe3\x81\x97\xe3" -"\x81\xa6\xe3\x81\x84\xe3\x81\xbe\xe3\x81\x9b\xe3\x82\x93\xe3\x81" -"\xa7\xe3\x81\x97\xe3\x81\x9f\xe3\x80\x82\x0a\xe3\x81\x93\xe3\x81" -"\xae\xe3\x81\x9f\xe3\x82\x81\xe3\x80\x81\x47\x75\x69\x64\x6f\x20" -"\xe3\x81\xaf\xe3\x82\x88\xe3\x82\x8a\xe5\xae\x9f\xe7\x94\xa8\xe7" -"\x9a\x84\xe3\x81\xaa\xe3\x83\x97\xe3\x83\xad\xe3\x82\xb0\xe3\x83" -"\xa9\xe3\x83\x9f\xe3\x83\xb3\xe3\x82\xb0\xe8\xa8\x80\xe8\xaa\x9e" -"\xe3\x81\xae\xe9\x96\x8b\xe7\x99\xba\xe3\x82\x92\xe9\x96\x8b\xe5" -"\xa7\x8b\xe3\x81\x97\xe3\x80\x81\xe8\x8b\xb1\xe5\x9b\xbd\x20\x42" -"\x42\x53\x20\xe6\x94\xbe\xe9\x80\x81\xe3\x81\xae\xe3\x82\xb3\xe3" -"\x83\xa1\xe3\x83\x87\xe3\x82\xa3\xe7\x95\xaa\xe7\xb5\x84\xe3\x80" -"\x8c\xe3\x83\xa2\xe3\x83\xb3\xe3\x83\x86\xe3\x82\xa3\x20\xe3\x83" -"\x91\xe3\x82\xa4\xe3\x82\xbd\xe3\x83\xb3\xe3\x80\x8d\xe3\x81\xae" -"\xe3\x83\x95\xe3\x82\xa1\xe3\x83\xb3\xe3\x81\xa7\xe3\x81\x82\xe3" -"\x82\x8b\x20\x47\x75\x69\x64\x6f\x20\xe3\x81\xaf\xe3\x81\x93\xe3" -"\x81\xae\xe8\xa8\x80\xe8\xaa\x9e\xe3\x82\x92\xe3\x80\x8c\x50\x79" -"\x74\x68\x6f\x6e\xe3\x80\x8d\xe3\x81\xa8\xe5\x90\x8d\xe3\x81\xa5" -"\xe3\x81\x91\xe3\x81\xbe\xe3\x81\x97\xe3\x81\x9f\xe3\x80\x82\x0a" -"\xe3\x81\x93\xe3\x81\xae\xe3\x82\x88\xe3\x81\x86\xe3\x81\xaa\xe8" -"\x83\x8c\xe6\x99\xaf\xe3\x81\x8b\xe3\x82\x89\xe7\x94\x9f\xe3\x81" -"\xbe\xe3\x82\x8c\xe3\x81\x9f\x20\x50\x79\x74\x68\x6f\x6e\x20\xe3" -"\x81\xae\xe8\xa8\x80\xe8\xaa\x9e\xe8\xa8\xad\xe8\xa8\x88\xe3\x81" -"\xaf\xe3\x80\x81\xe3\x80\x8c\xe3\x82\xb7\xe3\x83\xb3\xe3\x83\x97" -"\xe3\x83\xab\xe3\x80\x8d\xe3\x81\xa7\xe3\x80\x8c\xe7\xbf\x92\xe5" -"\xbe\x97\xe3\x81\x8c\xe5\xae\xb9\xe6\x98\x93\xe3\x80\x8d\xe3\x81" -"\xa8\xe3\x81\x84\xe3\x81\x86\xe7\x9b\xae\xe6\xa8\x99\xe3\x81\xab" -"\xe9\x87\x8d\xe7\x82\xb9\xe3\x81\x8c\xe7\xbd\xae\xe3\x81\x8b\xe3" -"\x82\x8c\xe3\x81\xa6\xe3\x81\x84\xe3\x81\xbe\xe3\x81\x99\xe3\x80" -"\x82\x0a\xe5\xa4\x9a\xe3\x81\x8f\xe3\x81\xae\xe3\x82\xb9\xe3\x82" -"\xaf\xe3\x83\xaa\xe3\x83\x97\xe3\x83\x88\xe7\xb3\xbb\xe8\xa8\x80" -"\xe8\xaa\x9e\xe3\x81\xa7\xe3\x81\xaf\xe3\x83\xa6\xe3\x83\xbc\xe3" -"\x82\xb6\xe3\x81\xae\xe7\x9b\xae\xe5\x85\x88\xe3\x81\xae\xe5\x88" -"\xa9\xe4\xbe\xbf\xe6\x80\xa7\xe3\x82\x92\xe5\x84\xaa\xe5\x85\x88" -"\xe3\x81\x97\xe3\x81\xa6\xe8\x89\xb2\xe3\x80\x85\xe3\x81\xaa\xe6" -"\xa9\x9f\xe8\x83\xbd\xe3\x82\x92\xe8\xa8\x80\xe8\xaa\x9e\xe8\xa6" -"\x81\xe7\xb4\xa0\xe3\x81\xa8\xe3\x81\x97\xe3\x81\xa6\xe5\x8f\x96" -"\xe3\x82\x8a\xe5\x85\xa5\xe3\x82\x8c\xe3\x82\x8b\xe5\xa0\xb4\xe5" -"\x90\x88\xe3\x81\x8c\xe5\xa4\x9a\xe3\x81\x84\xe3\x81\xae\xe3\x81" -"\xa7\xe3\x81\x99\xe3\x81\x8c\xe3\x80\x81\x50\x79\x74\x68\x6f\x6e" -"\x20\xe3\x81\xa7\xe3\x81\xaf\xe3\x81\x9d\xe3\x81\x86\xe3\x81\x84" -"\xe3\x81\xa3\xe3\x81\x9f\xe5\xb0\x8f\xe7\xb4\xb0\xe5\xb7\xa5\xe3" -"\x81\x8c\xe8\xbf\xbd\xe5\x8a\xa0\xe3\x81\x95\xe3\x82\x8c\xe3\x82" -"\x8b\xe3\x81\x93\xe3\x81\xa8\xe3\x81\xaf\xe3\x81\x82\xe3\x81\xbe" -"\xe3\x82\x8a\xe3\x81\x82\xe3\x82\x8a\xe3\x81\xbe\xe3\x81\x9b\xe3" -"\x82\x93\xe3\x80\x82\x0a\xe8\xa8\x80\xe8\xaa\x9e\xe8\x87\xaa\xe4" -"\xbd\x93\xe3\x81\xae\xe6\xa9\x9f\xe8\x83\xbd\xe3\x81\xaf\xe6\x9c" -"\x80\xe5\xb0\x8f\xe9\x99\x90\xe3\x81\xab\xe6\x8a\xbc\xe3\x81\x95" -"\xe3\x81\x88\xe3\x80\x81\xe5\xbf\x85\xe8\xa6\x81\xe3\x81\xaa\xe6" -"\xa9\x9f\xe8\x83\xbd\xe3\x81\xaf\xe6\x8b\xa1\xe5\xbc\xb5\xe3\x83" -"\xa2\xe3\x82\xb8\xe3\x83\xa5\xe3\x83\xbc\xe3\x83\xab\xe3\x81\xa8" -"\xe3\x81\x97\xe3\x81\xa6\xe8\xbf\xbd\xe5\x8a\xa0\xe3\x81\x99\xe3" -"\x82\x8b\xe3\x80\x81\xe3\x81\xa8\xe3\x81\x84\xe3\x81\x86\xe3\x81" -"\xae\xe3\x81\x8c\x20\x50\x79\x74\x68\x6f\x6e\x20\xe3\x81\xae\xe3" -"\x83\x9d\xe3\x83\xaa\xe3\x82\xb7\xe3\x83\xbc\xe3\x81\xa7\xe3\x81" -"\x99\xe3\x80\x82\x0a\x0a\xe3\x83\x8e\xe3\x81\x8b\xe3\x82\x9a\x20" -"\xe3\x83\x88\xe3\x82\x9a\x20\xe3\x83\x88\xe3\x82\xad\xef\xa8\xb6" -"\xef\xa8\xb9\x20\xf0\xa1\x9a\xb4\xf0\xaa\x8e\x8c\x20\xe9\xba\x80" -"\xe9\xbd\x81\xf0\xa9\x9b\xb0\x0a"), -'euc_jp': ( -"\x50\x79\x74\x68\x6f\x6e\x20\xa4\xce\xb3\xab\xc8\xaf\xa4\xcf\xa1" -"\xa2\x31\x39\x39\x30\x20\xc7\xaf\xa4\xb4\xa4\xed\xa4\xab\xa4\xe9" -"\xb3\xab\xbb\xcf\xa4\xb5\xa4\xec\xa4\xc6\xa4\xa4\xa4\xde\xa4\xb9" -"\xa1\xa3\x0a\xb3\xab\xc8\xaf\xbc\xd4\xa4\xce\x20\x47\x75\x69\x64" -"\x6f\x20\x76\x61\x6e\x20\x52\x6f\x73\x73\x75\x6d\x20\xa4\xcf\xb6" -"\xb5\xb0\xe9\xcd\xd1\xa4\xce\xa5\xd7\xa5\xed\xa5\xb0\xa5\xe9\xa5" -"\xdf\xa5\xf3\xa5\xb0\xb8\xc0\xb8\xec\xa1\xd6\x41\x42\x43\xa1\xd7" -"\xa4\xce\xb3\xab\xc8\xaf\xa4\xcb\xbb\xb2\xb2\xc3\xa4\xb7\xa4\xc6" -"\xa4\xa4\xa4\xde\xa4\xb7\xa4\xbf\xa4\xac\xa1\xa2\x41\x42\x43\x20" -"\xa4\xcf\xbc\xc2\xcd\xd1\xbe\xe5\xa4\xce\xcc\xdc\xc5\xaa\xa4\xcb" -"\xa4\xcf\xa4\xa2\xa4\xde\xa4\xea\xc5\xac\xa4\xb7\xa4\xc6\xa4\xa4" -"\xa4\xde\xa4\xbb\xa4\xf3\xa4\xc7\xa4\xb7\xa4\xbf\xa1\xa3\x0a\xa4" -"\xb3\xa4\xce\xa4\xbf\xa4\xe1\xa1\xa2\x47\x75\x69\x64\x6f\x20\xa4" -"\xcf\xa4\xe8\xa4\xea\xbc\xc2\xcd\xd1\xc5\xaa\xa4\xca\xa5\xd7\xa5" -"\xed\xa5\xb0\xa5\xe9\xa5\xdf\xa5\xf3\xa5\xb0\xb8\xc0\xb8\xec\xa4" -"\xce\xb3\xab\xc8\xaf\xa4\xf2\xb3\xab\xbb\xcf\xa4\xb7\xa1\xa2\xb1" -"\xd1\xb9\xf1\x20\x42\x42\x53\x20\xca\xfc\xc1\xf7\xa4\xce\xa5\xb3" -"\xa5\xe1\xa5\xc7\xa5\xa3\xc8\xd6\xc1\xc8\xa1\xd6\xa5\xe2\xa5\xf3" -"\xa5\xc6\xa5\xa3\x20\xa5\xd1\xa5\xa4\xa5\xbd\xa5\xf3\xa1\xd7\xa4" -"\xce\xa5\xd5\xa5\xa1\xa5\xf3\xa4\xc7\xa4\xa2\xa4\xeb\x20\x47\x75" -"\x69\x64\x6f\x20\xa4\xcf\xa4\xb3\xa4\xce\xb8\xc0\xb8\xec\xa4\xf2" -"\xa1\xd6\x50\x79\x74\x68\x6f\x6e\xa1\xd7\xa4\xc8\xcc\xbe\xa4\xc5" -"\xa4\xb1\xa4\xde\xa4\xb7\xa4\xbf\xa1\xa3\x0a\xa4\xb3\xa4\xce\xa4" -"\xe8\xa4\xa6\xa4\xca\xc7\xd8\xb7\xca\xa4\xab\xa4\xe9\xc0\xb8\xa4" -"\xde\xa4\xec\xa4\xbf\x20\x50\x79\x74\x68\x6f\x6e\x20\xa4\xce\xb8" -"\xc0\xb8\xec\xc0\xdf\xb7\xd7\xa4\xcf\xa1\xa2\xa1\xd6\xa5\xb7\xa5" -"\xf3\xa5\xd7\xa5\xeb\xa1\xd7\xa4\xc7\xa1\xd6\xbd\xac\xc6\xc0\xa4" -"\xac\xcd\xc6\xb0\xd7\xa1\xd7\xa4\xc8\xa4\xa4\xa4\xa6\xcc\xdc\xc9" -"\xb8\xa4\xcb\xbd\xc5\xc5\xc0\xa4\xac\xc3\xd6\xa4\xab\xa4\xec\xa4" -"\xc6\xa4\xa4\xa4\xde\xa4\xb9\xa1\xa3\x0a\xc2\xbf\xa4\xaf\xa4\xce" -"\xa5\xb9\xa5\xaf\xa5\xea\xa5\xd7\xa5\xc8\xb7\xcf\xb8\xc0\xb8\xec" -"\xa4\xc7\xa4\xcf\xa5\xe6\xa1\xbc\xa5\xb6\xa4\xce\xcc\xdc\xc0\xe8" -"\xa4\xce\xcd\xf8\xca\xd8\xc0\xad\xa4\xf2\xcd\xa5\xc0\xe8\xa4\xb7" -"\xa4\xc6\xbf\xa7\xa1\xb9\xa4\xca\xb5\xa1\xc7\xbd\xa4\xf2\xb8\xc0" -"\xb8\xec\xcd\xd7\xc1\xc7\xa4\xc8\xa4\xb7\xa4\xc6\xbc\xe8\xa4\xea" -"\xc6\xfe\xa4\xec\xa4\xeb\xbe\xec\xb9\xe7\xa4\xac\xc2\xbf\xa4\xa4" -"\xa4\xce\xa4\xc7\xa4\xb9\xa4\xac\xa1\xa2\x50\x79\x74\x68\x6f\x6e" -"\x20\xa4\xc7\xa4\xcf\xa4\xbd\xa4\xa6\xa4\xa4\xa4\xc3\xa4\xbf\xbe" -"\xae\xba\xd9\xb9\xa9\xa4\xac\xc4\xc9\xb2\xc3\xa4\xb5\xa4\xec\xa4" -"\xeb\xa4\xb3\xa4\xc8\xa4\xcf\xa4\xa2\xa4\xde\xa4\xea\xa4\xa2\xa4" -"\xea\xa4\xde\xa4\xbb\xa4\xf3\xa1\xa3\x0a\xb8\xc0\xb8\xec\xbc\xab" -"\xc2\xce\xa4\xce\xb5\xa1\xc7\xbd\xa4\xcf\xba\xc7\xbe\xae\xb8\xc2" -"\xa4\xcb\xb2\xa1\xa4\xb5\xa4\xa8\xa1\xa2\xc9\xac\xcd\xd7\xa4\xca" -"\xb5\xa1\xc7\xbd\xa4\xcf\xb3\xc8\xc4\xa5\xa5\xe2\xa5\xb8\xa5\xe5" -"\xa1\xbc\xa5\xeb\xa4\xc8\xa4\xb7\xa4\xc6\xc4\xc9\xb2\xc3\xa4\xb9" -"\xa4\xeb\xa1\xa2\xa4\xc8\xa4\xa4\xa4\xa6\xa4\xce\xa4\xac\x20\x50" -"\x79\x74\x68\x6f\x6e\x20\xa4\xce\xa5\xdd\xa5\xea\xa5\xb7\xa1\xbc" -"\xa4\xc7\xa4\xb9\xa1\xa3\x0a\x0a", -"\x50\x79\x74\x68\x6f\x6e\x20\xe3\x81\xae\xe9\x96\x8b\xe7\x99\xba" -"\xe3\x81\xaf\xe3\x80\x81\x31\x39\x39\x30\x20\xe5\xb9\xb4\xe3\x81" -"\x94\xe3\x82\x8d\xe3\x81\x8b\xe3\x82\x89\xe9\x96\x8b\xe5\xa7\x8b" -"\xe3\x81\x95\xe3\x82\x8c\xe3\x81\xa6\xe3\x81\x84\xe3\x81\xbe\xe3" -"\x81\x99\xe3\x80\x82\x0a\xe9\x96\x8b\xe7\x99\xba\xe8\x80\x85\xe3" -"\x81\xae\x20\x47\x75\x69\x64\x6f\x20\x76\x61\x6e\x20\x52\x6f\x73" -"\x73\x75\x6d\x20\xe3\x81\xaf\xe6\x95\x99\xe8\x82\xb2\xe7\x94\xa8" -"\xe3\x81\xae\xe3\x83\x97\xe3\x83\xad\xe3\x82\xb0\xe3\x83\xa9\xe3" -"\x83\x9f\xe3\x83\xb3\xe3\x82\xb0\xe8\xa8\x80\xe8\xaa\x9e\xe3\x80" -"\x8c\x41\x42\x43\xe3\x80\x8d\xe3\x81\xae\xe9\x96\x8b\xe7\x99\xba" -"\xe3\x81\xab\xe5\x8f\x82\xe5\x8a\xa0\xe3\x81\x97\xe3\x81\xa6\xe3" -"\x81\x84\xe3\x81\xbe\xe3\x81\x97\xe3\x81\x9f\xe3\x81\x8c\xe3\x80" -"\x81\x41\x42\x43\x20\xe3\x81\xaf\xe5\xae\x9f\xe7\x94\xa8\xe4\xb8" -"\x8a\xe3\x81\xae\xe7\x9b\xae\xe7\x9a\x84\xe3\x81\xab\xe3\x81\xaf" -"\xe3\x81\x82\xe3\x81\xbe\xe3\x82\x8a\xe9\x81\xa9\xe3\x81\x97\xe3" -"\x81\xa6\xe3\x81\x84\xe3\x81\xbe\xe3\x81\x9b\xe3\x82\x93\xe3\x81" -"\xa7\xe3\x81\x97\xe3\x81\x9f\xe3\x80\x82\x0a\xe3\x81\x93\xe3\x81" -"\xae\xe3\x81\x9f\xe3\x82\x81\xe3\x80\x81\x47\x75\x69\x64\x6f\x20" -"\xe3\x81\xaf\xe3\x82\x88\xe3\x82\x8a\xe5\xae\x9f\xe7\x94\xa8\xe7" -"\x9a\x84\xe3\x81\xaa\xe3\x83\x97\xe3\x83\xad\xe3\x82\xb0\xe3\x83" -"\xa9\xe3\x83\x9f\xe3\x83\xb3\xe3\x82\xb0\xe8\xa8\x80\xe8\xaa\x9e" -"\xe3\x81\xae\xe9\x96\x8b\xe7\x99\xba\xe3\x82\x92\xe9\x96\x8b\xe5" -"\xa7\x8b\xe3\x81\x97\xe3\x80\x81\xe8\x8b\xb1\xe5\x9b\xbd\x20\x42" -"\x42\x53\x20\xe6\x94\xbe\xe9\x80\x81\xe3\x81\xae\xe3\x82\xb3\xe3" -"\x83\xa1\xe3\x83\x87\xe3\x82\xa3\xe7\x95\xaa\xe7\xb5\x84\xe3\x80" -"\x8c\xe3\x83\xa2\xe3\x83\xb3\xe3\x83\x86\xe3\x82\xa3\x20\xe3\x83" -"\x91\xe3\x82\xa4\xe3\x82\xbd\xe3\x83\xb3\xe3\x80\x8d\xe3\x81\xae" -"\xe3\x83\x95\xe3\x82\xa1\xe3\x83\xb3\xe3\x81\xa7\xe3\x81\x82\xe3" -"\x82\x8b\x20\x47\x75\x69\x64\x6f\x20\xe3\x81\xaf\xe3\x81\x93\xe3" -"\x81\xae\xe8\xa8\x80\xe8\xaa\x9e\xe3\x82\x92\xe3\x80\x8c\x50\x79" -"\x74\x68\x6f\x6e\xe3\x80\x8d\xe3\x81\xa8\xe5\x90\x8d\xe3\x81\xa5" -"\xe3\x81\x91\xe3\x81\xbe\xe3\x81\x97\xe3\x81\x9f\xe3\x80\x82\x0a" -"\xe3\x81\x93\xe3\x81\xae\xe3\x82\x88\xe3\x81\x86\xe3\x81\xaa\xe8" -"\x83\x8c\xe6\x99\xaf\xe3\x81\x8b\xe3\x82\x89\xe7\x94\x9f\xe3\x81" -"\xbe\xe3\x82\x8c\xe3\x81\x9f\x20\x50\x79\x74\x68\x6f\x6e\x20\xe3" -"\x81\xae\xe8\xa8\x80\xe8\xaa\x9e\xe8\xa8\xad\xe8\xa8\x88\xe3\x81" -"\xaf\xe3\x80\x81\xe3\x80\x8c\xe3\x82\xb7\xe3\x83\xb3\xe3\x83\x97" -"\xe3\x83\xab\xe3\x80\x8d\xe3\x81\xa7\xe3\x80\x8c\xe7\xbf\x92\xe5" -"\xbe\x97\xe3\x81\x8c\xe5\xae\xb9\xe6\x98\x93\xe3\x80\x8d\xe3\x81" -"\xa8\xe3\x81\x84\xe3\x81\x86\xe7\x9b\xae\xe6\xa8\x99\xe3\x81\xab" -"\xe9\x87\x8d\xe7\x82\xb9\xe3\x81\x8c\xe7\xbd\xae\xe3\x81\x8b\xe3" -"\x82\x8c\xe3\x81\xa6\xe3\x81\x84\xe3\x81\xbe\xe3\x81\x99\xe3\x80" -"\x82\x0a\xe5\xa4\x9a\xe3\x81\x8f\xe3\x81\xae\xe3\x82\xb9\xe3\x82" -"\xaf\xe3\x83\xaa\xe3\x83\x97\xe3\x83\x88\xe7\xb3\xbb\xe8\xa8\x80" -"\xe8\xaa\x9e\xe3\x81\xa7\xe3\x81\xaf\xe3\x83\xa6\xe3\x83\xbc\xe3" -"\x82\xb6\xe3\x81\xae\xe7\x9b\xae\xe5\x85\x88\xe3\x81\xae\xe5\x88" -"\xa9\xe4\xbe\xbf\xe6\x80\xa7\xe3\x82\x92\xe5\x84\xaa\xe5\x85\x88" -"\xe3\x81\x97\xe3\x81\xa6\xe8\x89\xb2\xe3\x80\x85\xe3\x81\xaa\xe6" -"\xa9\x9f\xe8\x83\xbd\xe3\x82\x92\xe8\xa8\x80\xe8\xaa\x9e\xe8\xa6" -"\x81\xe7\xb4\xa0\xe3\x81\xa8\xe3\x81\x97\xe3\x81\xa6\xe5\x8f\x96" -"\xe3\x82\x8a\xe5\x85\xa5\xe3\x82\x8c\xe3\x82\x8b\xe5\xa0\xb4\xe5" -"\x90\x88\xe3\x81\x8c\xe5\xa4\x9a\xe3\x81\x84\xe3\x81\xae\xe3\x81" -"\xa7\xe3\x81\x99\xe3\x81\x8c\xe3\x80\x81\x50\x79\x74\x68\x6f\x6e" -"\x20\xe3\x81\xa7\xe3\x81\xaf\xe3\x81\x9d\xe3\x81\x86\xe3\x81\x84" -"\xe3\x81\xa3\xe3\x81\x9f\xe5\xb0\x8f\xe7\xb4\xb0\xe5\xb7\xa5\xe3" -"\x81\x8c\xe8\xbf\xbd\xe5\x8a\xa0\xe3\x81\x95\xe3\x82\x8c\xe3\x82" -"\x8b\xe3\x81\x93\xe3\x81\xa8\xe3\x81\xaf\xe3\x81\x82\xe3\x81\xbe" -"\xe3\x82\x8a\xe3\x81\x82\xe3\x82\x8a\xe3\x81\xbe\xe3\x81\x9b\xe3" -"\x82\x93\xe3\x80\x82\x0a\xe8\xa8\x80\xe8\xaa\x9e\xe8\x87\xaa\xe4" -"\xbd\x93\xe3\x81\xae\xe6\xa9\x9f\xe8\x83\xbd\xe3\x81\xaf\xe6\x9c" -"\x80\xe5\xb0\x8f\xe9\x99\x90\xe3\x81\xab\xe6\x8a\xbc\xe3\x81\x95" -"\xe3\x81\x88\xe3\x80\x81\xe5\xbf\x85\xe8\xa6\x81\xe3\x81\xaa\xe6" -"\xa9\x9f\xe8\x83\xbd\xe3\x81\xaf\xe6\x8b\xa1\xe5\xbc\xb5\xe3\x83" -"\xa2\xe3\x82\xb8\xe3\x83\xa5\xe3\x83\xbc\xe3\x83\xab\xe3\x81\xa8" -"\xe3\x81\x97\xe3\x81\xa6\xe8\xbf\xbd\xe5\x8a\xa0\xe3\x81\x99\xe3" -"\x82\x8b\xe3\x80\x81\xe3\x81\xa8\xe3\x81\x84\xe3\x81\x86\xe3\x81" -"\xae\xe3\x81\x8c\x20\x50\x79\x74\x68\x6f\x6e\x20\xe3\x81\xae\xe3" -"\x83\x9d\xe3\x83\xaa\xe3\x82\xb7\xe3\x83\xbc\xe3\x81\xa7\xe3\x81" -"\x99\xe3\x80\x82\x0a\x0a"), -'euc_kr': ( -"\xa1\xdd\x20\xc6\xc4\xc0\xcc\xbd\xe3\x28\x50\x79\x74\x68\x6f\x6e" -"\x29\xc0\xba\x20\xb9\xe8\xbf\xec\xb1\xe2\x20\xbd\xb1\xb0\xed\x2c" -"\x20\xb0\xad\xb7\xc2\xc7\xd1\x20\xc7\xc1\xb7\xce\xb1\xd7\xb7\xa1" -"\xb9\xd6\x20\xbe\xf0\xbe\xee\xc0\xd4\xb4\xcf\xb4\xd9\x2e\x20\xc6" -"\xc4\xc0\xcc\xbd\xe3\xc0\xba\x0a\xc8\xbf\xc0\xb2\xc0\xfb\xc0\xce" -"\x20\xb0\xed\xbc\xf6\xc1\xd8\x20\xb5\xa5\xc0\xcc\xc5\xcd\x20\xb1" -"\xb8\xc1\xb6\xbf\xcd\x20\xb0\xa3\xb4\xdc\xc7\xcf\xc1\xf6\xb8\xb8" -"\x20\xc8\xbf\xc0\xb2\xc0\xfb\xc0\xce\x20\xb0\xb4\xc3\xbc\xc1\xf6" -"\xc7\xe2\xc7\xc1\xb7\xce\xb1\xd7\xb7\xa1\xb9\xd6\xc0\xbb\x0a\xc1" -"\xf6\xbf\xf8\xc7\xd5\xb4\xcf\xb4\xd9\x2e\x20\xc6\xc4\xc0\xcc\xbd" -"\xe3\xc0\xc7\x20\xbf\xec\xbe\xc6\x28\xe9\xd0\xe4\xba\x29\xc7\xd1" -"\x20\xb9\xae\xb9\xfd\xb0\xfa\x20\xb5\xbf\xc0\xfb\x20\xc5\xb8\xc0" -"\xcc\xc7\xce\x2c\x20\xb1\xd7\xb8\xae\xb0\xed\x20\xc0\xce\xc5\xcd" -"\xc7\xc1\xb8\xae\xc6\xc3\x0a\xc8\xaf\xb0\xe6\xc0\xba\x20\xc6\xc4" -"\xc0\xcc\xbd\xe3\xc0\xbb\x20\xbd\xba\xc5\xa9\xb8\xb3\xc6\xc3\xb0" -"\xfa\x20\xbf\xa9\xb7\xaf\x20\xba\xd0\xbe\xdf\xbf\xa1\xbc\xad\xbf" -"\xcd\x20\xb4\xeb\xba\xce\xba\xd0\xc0\xc7\x20\xc7\xc3\xb7\xa7\xc6" -"\xfb\xbf\xa1\xbc\xad\xc0\xc7\x20\xba\xfc\xb8\xa5\x0a\xbe\xd6\xc7" -"\xc3\xb8\xae\xc4\xc9\xc0\xcc\xbc\xc7\x20\xb0\xb3\xb9\xdf\xc0\xbb" -"\x20\xc7\xd2\x20\xbc\xf6\x20\xc0\xd6\xb4\xc2\x20\xc0\xcc\xbb\xf3" -"\xc0\xfb\xc0\xce\x20\xbe\xf0\xbe\xee\xb7\xce\x20\xb8\xb8\xb5\xe9" -"\xbe\xee\xc1\xdd\xb4\xcf\xb4\xd9\x2e\x0a\x0a\xa1\xd9\xc3\xb9\xb0" -"\xa1\xb3\xa1\x3a\x20\xb3\xaf\xbe\xc6\xb6\xf3\x20\xa4\xd4\xa4\xb6" -"\xa4\xd0\xa4\xd4\xa4\xd4\xa4\xb6\xa4\xd0\xa4\xd4\xbe\xb1\x7e\x20" -"\xa4\xd4\xa4\xa4\xa4\xd2\xa4\xb7\xc5\xad\x21\x20\xa4\xd4\xa4\xa8" -"\xa4\xd1\xa4\xb7\xb1\xdd\xbe\xf8\xc0\xcc\x20\xc0\xfc\xa4\xd4\xa4" -"\xbe\xa4\xc8\xa4\xb2\xb4\xcf\xb4\xd9\x2e\x20\xa4\xd4\xa4\xb2\xa4" -"\xce\xa4\xaa\x2e\x20\xb1\xd7\xb7\xb1\xb0\xc5\x20\xa4\xd4\xa4\xb7" -"\xa4\xd1\xa4\xb4\xb4\xd9\x2e\x0a", -"\xe2\x97\x8e\x20\xed\x8c\x8c\xec\x9d\xb4\xec\x8d\xac\x28\x50\x79" -"\x74\x68\x6f\x6e\x29\xec\x9d\x80\x20\xeb\xb0\xb0\xec\x9a\xb0\xea" -"\xb8\xb0\x20\xec\x89\xbd\xea\xb3\xa0\x2c\x20\xea\xb0\x95\xeb\xa0" -"\xa5\xed\x95\x9c\x20\xed\x94\x84\xeb\xa1\x9c\xea\xb7\xb8\xeb\x9e" -"\x98\xeb\xb0\x8d\x20\xec\x96\xb8\xec\x96\xb4\xec\x9e\x85\xeb\x8b" -"\x88\xeb\x8b\xa4\x2e\x20\xed\x8c\x8c\xec\x9d\xb4\xec\x8d\xac\xec" -"\x9d\x80\x0a\xed\x9a\xa8\xec\x9c\xa8\xec\xa0\x81\xec\x9d\xb8\x20" -"\xea\xb3\xa0\xec\x88\x98\xec\xa4\x80\x20\xeb\x8d\xb0\xec\x9d\xb4" -"\xed\x84\xb0\x20\xea\xb5\xac\xec\xa1\xb0\xec\x99\x80\x20\xea\xb0" -"\x84\xeb\x8b\xa8\xed\x95\x98\xec\xa7\x80\xeb\xa7\x8c\x20\xed\x9a" -"\xa8\xec\x9c\xa8\xec\xa0\x81\xec\x9d\xb8\x20\xea\xb0\x9d\xec\xb2" -"\xb4\xec\xa7\x80\xed\x96\xa5\xed\x94\x84\xeb\xa1\x9c\xea\xb7\xb8" -"\xeb\x9e\x98\xeb\xb0\x8d\xec\x9d\x84\x0a\xec\xa7\x80\xec\x9b\x90" -"\xed\x95\xa9\xeb\x8b\x88\xeb\x8b\xa4\x2e\x20\xed\x8c\x8c\xec\x9d" -"\xb4\xec\x8d\xac\xec\x9d\x98\x20\xec\x9a\xb0\xec\x95\x84\x28\xe5" -"\x84\xaa\xe9\x9b\x85\x29\xed\x95\x9c\x20\xeb\xac\xb8\xeb\xb2\x95" -"\xea\xb3\xbc\x20\xeb\x8f\x99\xec\xa0\x81\x20\xed\x83\x80\xec\x9d" -"\xb4\xed\x95\x91\x2c\x20\xea\xb7\xb8\xeb\xa6\xac\xea\xb3\xa0\x20" -"\xec\x9d\xb8\xed\x84\xb0\xed\x94\x84\xeb\xa6\xac\xed\x8c\x85\x0a" -"\xed\x99\x98\xea\xb2\xbd\xec\x9d\x80\x20\xed\x8c\x8c\xec\x9d\xb4" -"\xec\x8d\xac\xec\x9d\x84\x20\xec\x8a\xa4\xed\x81\xac\xeb\xa6\xbd" -"\xed\x8c\x85\xea\xb3\xbc\x20\xec\x97\xac\xeb\x9f\xac\x20\xeb\xb6" -"\x84\xec\x95\xbc\xec\x97\x90\xec\x84\x9c\xec\x99\x80\x20\xeb\x8c" -"\x80\xeb\xb6\x80\xeb\xb6\x84\xec\x9d\x98\x20\xed\x94\x8c\xeb\x9e" -"\xab\xed\x8f\xbc\xec\x97\x90\xec\x84\x9c\xec\x9d\x98\x20\xeb\xb9" -"\xa0\xeb\xa5\xb8\x0a\xec\x95\xa0\xed\x94\x8c\xeb\xa6\xac\xec\xbc" -"\x80\xec\x9d\xb4\xec\x85\x98\x20\xea\xb0\x9c\xeb\xb0\x9c\xec\x9d" -"\x84\x20\xed\x95\xa0\x20\xec\x88\x98\x20\xec\x9e\x88\xeb\x8a\x94" -"\x20\xec\x9d\xb4\xec\x83\x81\xec\xa0\x81\xec\x9d\xb8\x20\xec\x96" -"\xb8\xec\x96\xb4\xeb\xa1\x9c\x20\xeb\xa7\x8c\xeb\x93\xa4\xec\x96" -"\xb4\xec\xa4\x8d\xeb\x8b\x88\xeb\x8b\xa4\x2e\x0a\x0a\xe2\x98\x86" -"\xec\xb2\xab\xea\xb0\x80\xeb\x81\x9d\x3a\x20\xeb\x82\xa0\xec\x95" -"\x84\xeb\x9d\xbc\x20\xec\x93\x94\xec\x93\x94\xec\x93\xa9\x7e\x20" -"\xeb\x8b\x81\xed\x81\xbc\x21\x20\xeb\x9c\xbd\xea\xb8\x88\xec\x97" -"\x86\xec\x9d\xb4\x20\xec\xa0\x84\xed\x99\xa5\xeb\x8b\x88\xeb\x8b" -"\xa4\x2e\x20\xeb\xb7\x81\x2e\x20\xea\xb7\xb8\xeb\x9f\xb0\xea\xb1" -"\xb0\x20\xec\x9d\x8e\xeb\x8b\xa4\x2e\x0a"), -'gb18030': ( -"\x50\x79\x74\x68\x6f\x6e\xa3\xa8\xc5\xc9\xc9\xad\xa3\xa9\xd3\xef" -"\xd1\xd4\xca\xc7\xd2\xbb\xd6\xd6\xb9\xa6\xc4\xdc\xc7\xbf\xb4\xf3" -"\xb6\xf8\xcd\xea\xc9\xc6\xb5\xc4\xcd\xa8\xd3\xc3\xd0\xcd\xbc\xc6" -"\xcb\xe3\xbb\xfa\xb3\xcc\xd0\xf2\xc9\xe8\xbc\xc6\xd3\xef\xd1\xd4" -"\xa3\xac\x0a\xd2\xd1\xbe\xad\xbe\xdf\xd3\xd0\xca\xae\xb6\xe0\xc4" -"\xea\xb5\xc4\xb7\xa2\xd5\xb9\xc0\xfa\xca\xb7\xa3\xac\xb3\xc9\xca" -"\xec\xc7\xd2\xce\xc8\xb6\xa8\xa1\xa3\xd5\xe2\xd6\xd6\xd3\xef\xd1" -"\xd4\xbe\xdf\xd3\xd0\xb7\xc7\xb3\xa3\xbc\xf2\xbd\xdd\xb6\xf8\xc7" -"\xe5\xce\xfa\x0a\xb5\xc4\xd3\xef\xb7\xa8\xcc\xd8\xb5\xe3\xa3\xac" -"\xca\xca\xba\xcf\xcd\xea\xb3\xc9\xb8\xf7\xd6\xd6\xb8\xdf\xb2\xe3" -"\xc8\xce\xce\xf1\xa3\xac\xbc\xb8\xba\xf5\xbf\xc9\xd2\xd4\xd4\xda" -"\xcb\xf9\xd3\xd0\xb5\xc4\xb2\xd9\xd7\xf7\xcf\xb5\xcd\xb3\xd6\xd0" -"\x0a\xd4\xcb\xd0\xd0\xa1\xa3\xd5\xe2\xd6\xd6\xd3\xef\xd1\xd4\xbc" -"\xf2\xb5\xa5\xb6\xf8\xc7\xbf\xb4\xf3\xa3\xac\xca\xca\xba\xcf\xb8" -"\xf7\xd6\xd6\xc8\xcb\xca\xbf\xd1\xa7\xcf\xb0\xca\xb9\xd3\xc3\xa1" -"\xa3\xc4\xbf\xc7\xb0\xa3\xac\xbb\xf9\xd3\xda\xd5\xe2\x0a\xd6\xd6" -"\xd3\xef\xd1\xd4\xb5\xc4\xcf\xe0\xb9\xd8\xbc\xbc\xca\xf5\xd5\xfd" -"\xd4\xda\xb7\xc9\xcb\xd9\xb5\xc4\xb7\xa2\xd5\xb9\xa3\xac\xd3\xc3" -"\xbb\xa7\xca\xfd\xc1\xbf\xbc\xb1\xbe\xe7\xc0\xa9\xb4\xf3\xa3\xac" -"\xcf\xe0\xb9\xd8\xb5\xc4\xd7\xca\xd4\xb4\xb7\xc7\xb3\xa3\xb6\xe0" -"\xa1\xa3\x0a\xc8\xe7\xba\xce\xd4\xda\x20\x50\x79\x74\x68\x6f\x6e" -"\x20\xd6\xd0\xca\xb9\xd3\xc3\xbc\xc8\xd3\xd0\xb5\xc4\x20\x43\x20" -"\x6c\x69\x62\x72\x61\x72\x79\x3f\x0a\xa1\xa1\xd4\xda\xd9\x59\xd3" -"\x8d\xbf\xc6\xbc\xbc\xbf\xec\xcb\xd9\xb0\x6c\xd5\xb9\xb5\xc4\xbd" -"\xf1\xcc\xec\x2c\x20\xe9\x5f\xb0\x6c\xbc\xb0\x9c\x79\xd4\x87\xdc" -"\x9b\xf3\x77\xb5\xc4\xcb\xd9\xb6\xc8\xca\xc7\xb2\xbb\xc8\xdd\xba" -"\xf6\xd2\x95\xb5\xc4\x0a\xd5\x6e\xee\x7d\x2e\x20\x9e\xe9\xbc\xd3" -"\xbf\xec\xe9\x5f\xb0\x6c\xbc\xb0\x9c\x79\xd4\x87\xb5\xc4\xcb\xd9" -"\xb6\xc8\x2c\x20\xce\xd2\x82\x83\xb1\xe3\xb3\xa3\xcf\xa3\xcd\xfb" -"\xc4\xdc\xc0\xfb\xd3\xc3\xd2\xbb\xd0\xa9\xd2\xd1\xe9\x5f\xb0\x6c" -"\xba\xc3\xb5\xc4\x0a\x6c\x69\x62\x72\x61\x72\x79\x2c\x20\x81\x4b" -"\xd3\xd0\xd2\xbb\x82\x80\x20\x66\x61\x73\x74\x20\x70\x72\x6f\x74" -"\x6f\x74\x79\x70\x69\x6e\x67\x20\xb5\xc4\x20\x70\x72\x6f\x67\x72" -"\x61\x6d\x6d\x69\x6e\x67\x20\x6c\x61\x6e\x67\x75\x61\x67\x65\x20" -"\xbf\xc9\x0a\xb9\xa9\xca\xb9\xd3\xc3\x2e\x20\xc4\xbf\xc7\xb0\xd3" -"\xd0\xd4\x53\xd4\x53\xb6\xe0\xb6\xe0\xb5\xc4\x20\x6c\x69\x62\x72" -"\x61\x72\x79\x20\xca\xc7\xd2\xd4\x20\x43\x20\x8c\x91\xb3\xc9\x2c" -"\x20\xb6\xf8\x20\x50\x79\x74\x68\x6f\x6e\x20\xca\xc7\xd2\xbb\x82" -"\x80\x0a\x66\x61\x73\x74\x20\x70\x72\x6f\x74\x6f\x74\x79\x70\x69" -"\x6e\x67\x20\xb5\xc4\x20\x70\x72\x6f\x67\x72\x61\x6d\x6d\x69\x6e" -"\x67\x20\x6c\x61\x6e\x67\x75\x61\x67\x65\x2e\x20\xb9\xca\xce\xd2" -"\x82\x83\xcf\xa3\xcd\xfb\xc4\xdc\x8c\xa2\xbc\xc8\xd3\xd0\xb5\xc4" -"\x0a\x43\x20\x6c\x69\x62\x72\x61\x72\x79\x20\xc4\xc3\xb5\xbd\x20" -"\x50\x79\x74\x68\x6f\x6e\x20\xb5\xc4\xad\x68\xbe\xb3\xd6\xd0\x9c" -"\x79\xd4\x87\xbc\xb0\xd5\xfb\xba\xcf\x2e\x20\xc6\xe4\xd6\xd0\xd7" -"\xee\xd6\xf7\xd2\xaa\xd2\xb2\xca\xc7\xce\xd2\x82\x83\xcb\xf9\x0a" -"\xd2\xaa\xd3\x91\xd5\x93\xb5\xc4\x86\x96\xee\x7d\xbe\xcd\xca\xc7" -"\x3a\x0a\x83\x35\xc7\x31\x83\x33\x9a\x33\x83\x32\xb1\x31\x83\x33" -"\x95\x31\x20\x82\x37\xd1\x36\x83\x30\x8c\x34\x83\x36\x84\x33\x20" -"\x82\x38\x89\x35\x82\x38\xfb\x36\x83\x33\x95\x35\x20\x83\x33\xd5" -"\x31\x82\x39\x81\x35\x20\x83\x30\xfd\x39\x83\x33\x86\x30\x20\x83" -"\x34\xdc\x33\x83\x35\xf6\x37\x83\x35\x97\x35\x20\x83\x35\xf9\x35" -"\x83\x30\x91\x39\x82\x38\x83\x39\x82\x39\xfc\x33\x83\x30\xf0\x34" -"\x20\x83\x32\xeb\x39\x83\x32\xeb\x35\x82\x39\x83\x39\x2e\x0a\x0a", -"\x50\x79\x74\x68\x6f\x6e\xef\xbc\x88\xe6\xb4\xbe\xe6\xa3\xae\xef" -"\xbc\x89\xe8\xaf\xad\xe8\xa8\x80\xe6\x98\xaf\xe4\xb8\x80\xe7\xa7" -"\x8d\xe5\x8a\x9f\xe8\x83\xbd\xe5\xbc\xba\xe5\xa4\xa7\xe8\x80\x8c" -"\xe5\xae\x8c\xe5\x96\x84\xe7\x9a\x84\xe9\x80\x9a\xe7\x94\xa8\xe5" -"\x9e\x8b\xe8\xae\xa1\xe7\xae\x97\xe6\x9c\xba\xe7\xa8\x8b\xe5\xba" -"\x8f\xe8\xae\xbe\xe8\xae\xa1\xe8\xaf\xad\xe8\xa8\x80\xef\xbc\x8c" -"\x0a\xe5\xb7\xb2\xe7\xbb\x8f\xe5\x85\xb7\xe6\x9c\x89\xe5\x8d\x81" -"\xe5\xa4\x9a\xe5\xb9\xb4\xe7\x9a\x84\xe5\x8f\x91\xe5\xb1\x95\xe5" -"\x8e\x86\xe5\x8f\xb2\xef\xbc\x8c\xe6\x88\x90\xe7\x86\x9f\xe4\xb8" -"\x94\xe7\xa8\xb3\xe5\xae\x9a\xe3\x80\x82\xe8\xbf\x99\xe7\xa7\x8d" -"\xe8\xaf\xad\xe8\xa8\x80\xe5\x85\xb7\xe6\x9c\x89\xe9\x9d\x9e\xe5" -"\xb8\xb8\xe7\xae\x80\xe6\x8d\xb7\xe8\x80\x8c\xe6\xb8\x85\xe6\x99" -"\xb0\x0a\xe7\x9a\x84\xe8\xaf\xad\xe6\xb3\x95\xe7\x89\xb9\xe7\x82" -"\xb9\xef\xbc\x8c\xe9\x80\x82\xe5\x90\x88\xe5\xae\x8c\xe6\x88\x90" -"\xe5\x90\x84\xe7\xa7\x8d\xe9\xab\x98\xe5\xb1\x82\xe4\xbb\xbb\xe5" -"\x8a\xa1\xef\xbc\x8c\xe5\x87\xa0\xe4\xb9\x8e\xe5\x8f\xaf\xe4\xbb" -"\xa5\xe5\x9c\xa8\xe6\x89\x80\xe6\x9c\x89\xe7\x9a\x84\xe6\x93\x8d" -"\xe4\xbd\x9c\xe7\xb3\xbb\xe7\xbb\x9f\xe4\xb8\xad\x0a\xe8\xbf\x90" -"\xe8\xa1\x8c\xe3\x80\x82\xe8\xbf\x99\xe7\xa7\x8d\xe8\xaf\xad\xe8" -"\xa8\x80\xe7\xae\x80\xe5\x8d\x95\xe8\x80\x8c\xe5\xbc\xba\xe5\xa4" -"\xa7\xef\xbc\x8c\xe9\x80\x82\xe5\x90\x88\xe5\x90\x84\xe7\xa7\x8d" -"\xe4\xba\xba\xe5\xa3\xab\xe5\xad\xa6\xe4\xb9\xa0\xe4\xbd\xbf\xe7" -"\x94\xa8\xe3\x80\x82\xe7\x9b\xae\xe5\x89\x8d\xef\xbc\x8c\xe5\x9f" -"\xba\xe4\xba\x8e\xe8\xbf\x99\x0a\xe7\xa7\x8d\xe8\xaf\xad\xe8\xa8" -"\x80\xe7\x9a\x84\xe7\x9b\xb8\xe5\x85\xb3\xe6\x8a\x80\xe6\x9c\xaf" -"\xe6\xad\xa3\xe5\x9c\xa8\xe9\xa3\x9e\xe9\x80\x9f\xe7\x9a\x84\xe5" -"\x8f\x91\xe5\xb1\x95\xef\xbc\x8c\xe7\x94\xa8\xe6\x88\xb7\xe6\x95" -"\xb0\xe9\x87\x8f\xe6\x80\xa5\xe5\x89\xa7\xe6\x89\xa9\xe5\xa4\xa7" -"\xef\xbc\x8c\xe7\x9b\xb8\xe5\x85\xb3\xe7\x9a\x84\xe8\xb5\x84\xe6" -"\xba\x90\xe9\x9d\x9e\xe5\xb8\xb8\xe5\xa4\x9a\xe3\x80\x82\x0a\xe5" -"\xa6\x82\xe4\xbd\x95\xe5\x9c\xa8\x20\x50\x79\x74\x68\x6f\x6e\x20" -"\xe4\xb8\xad\xe4\xbd\xbf\xe7\x94\xa8\xe6\x97\xa2\xe6\x9c\x89\xe7" -"\x9a\x84\x20\x43\x20\x6c\x69\x62\x72\x61\x72\x79\x3f\x0a\xe3\x80" -"\x80\xe5\x9c\xa8\xe8\xb3\x87\xe8\xa8\x8a\xe7\xa7\x91\xe6\x8a\x80" -"\xe5\xbf\xab\xe9\x80\x9f\xe7\x99\xbc\xe5\xb1\x95\xe7\x9a\x84\xe4" -"\xbb\x8a\xe5\xa4\xa9\x2c\x20\xe9\x96\x8b\xe7\x99\xbc\xe5\x8f\x8a" -"\xe6\xb8\xac\xe8\xa9\xa6\xe8\xbb\x9f\xe9\xab\x94\xe7\x9a\x84\xe9" -"\x80\x9f\xe5\xba\xa6\xe6\x98\xaf\xe4\xb8\x8d\xe5\xae\xb9\xe5\xbf" -"\xbd\xe8\xa6\x96\xe7\x9a\x84\x0a\xe8\xaa\xb2\xe9\xa1\x8c\x2e\x20" -"\xe7\x82\xba\xe5\x8a\xa0\xe5\xbf\xab\xe9\x96\x8b\xe7\x99\xbc\xe5" -"\x8f\x8a\xe6\xb8\xac\xe8\xa9\xa6\xe7\x9a\x84\xe9\x80\x9f\xe5\xba" -"\xa6\x2c\x20\xe6\x88\x91\xe5\x80\x91\xe4\xbe\xbf\xe5\xb8\xb8\xe5" -"\xb8\x8c\xe6\x9c\x9b\xe8\x83\xbd\xe5\x88\xa9\xe7\x94\xa8\xe4\xb8" -"\x80\xe4\xba\x9b\xe5\xb7\xb2\xe9\x96\x8b\xe7\x99\xbc\xe5\xa5\xbd" -"\xe7\x9a\x84\x0a\x6c\x69\x62\x72\x61\x72\x79\x2c\x20\xe4\xb8\xa6" -"\xe6\x9c\x89\xe4\xb8\x80\xe5\x80\x8b\x20\x66\x61\x73\x74\x20\x70" -"\x72\x6f\x74\x6f\x74\x79\x70\x69\x6e\x67\x20\xe7\x9a\x84\x20\x70" -"\x72\x6f\x67\x72\x61\x6d\x6d\x69\x6e\x67\x20\x6c\x61\x6e\x67\x75" -"\x61\x67\x65\x20\xe5\x8f\xaf\x0a\xe4\xbe\x9b\xe4\xbd\xbf\xe7\x94" -"\xa8\x2e\x20\xe7\x9b\xae\xe5\x89\x8d\xe6\x9c\x89\xe8\xa8\xb1\xe8" -"\xa8\xb1\xe5\xa4\x9a\xe5\xa4\x9a\xe7\x9a\x84\x20\x6c\x69\x62\x72" -"\x61\x72\x79\x20\xe6\x98\xaf\xe4\xbb\xa5\x20\x43\x20\xe5\xaf\xab" -"\xe6\x88\x90\x2c\x20\xe8\x80\x8c\x20\x50\x79\x74\x68\x6f\x6e\x20" -"\xe6\x98\xaf\xe4\xb8\x80\xe5\x80\x8b\x0a\x66\x61\x73\x74\x20\x70" -"\x72\x6f\x74\x6f\x74\x79\x70\x69\x6e\x67\x20\xe7\x9a\x84\x20\x70" -"\x72\x6f\x67\x72\x61\x6d\x6d\x69\x6e\x67\x20\x6c\x61\x6e\x67\x75" -"\x61\x67\x65\x2e\x20\xe6\x95\x85\xe6\x88\x91\xe5\x80\x91\xe5\xb8" -"\x8c\xe6\x9c\x9b\xe8\x83\xbd\xe5\xb0\x87\xe6\x97\xa2\xe6\x9c\x89" -"\xe7\x9a\x84\x0a\x43\x20\x6c\x69\x62\x72\x61\x72\x79\x20\xe6\x8b" -"\xbf\xe5\x88\xb0\x20\x50\x79\x74\x68\x6f\x6e\x20\xe7\x9a\x84\xe7" -"\x92\xb0\xe5\xa2\x83\xe4\xb8\xad\xe6\xb8\xac\xe8\xa9\xa6\xe5\x8f" -"\x8a\xe6\x95\xb4\xe5\x90\x88\x2e\x20\xe5\x85\xb6\xe4\xb8\xad\xe6" -"\x9c\x80\xe4\xb8\xbb\xe8\xa6\x81\xe4\xb9\x9f\xe6\x98\xaf\xe6\x88" -"\x91\xe5\x80\x91\xe6\x89\x80\x0a\xe8\xa6\x81\xe8\xa8\x8e\xe8\xab" -"\x96\xe7\x9a\x84\xe5\x95\x8f\xe9\xa1\x8c\xe5\xb0\xb1\xe6\x98\xaf" -"\x3a\x0a\xed\x8c\x8c\xec\x9d\xb4\xec\x8d\xac\xec\x9d\x80\x20\xea" -"\xb0\x95\xeb\xa0\xa5\xed\x95\x9c\x20\xea\xb8\xb0\xeb\x8a\xa5\xec" -"\x9d\x84\x20\xec\xa7\x80\xeb\x8b\x8c\x20\xeb\xb2\x94\xec\x9a\xa9" -"\x20\xec\xbb\xb4\xed\x93\xa8\xed\x84\xb0\x20\xed\x94\x84\xeb\xa1" -"\x9c\xea\xb7\xb8\xeb\x9e\x98\xeb\xb0\x8d\x20\xec\x96\xb8\xec\x96" -"\xb4\xeb\x8b\xa4\x2e\x0a\x0a"), -'gb2312': ( -"\x50\x79\x74\x68\x6f\x6e\xa3\xa8\xc5\xc9\xc9\xad\xa3\xa9\xd3\xef" -"\xd1\xd4\xca\xc7\xd2\xbb\xd6\xd6\xb9\xa6\xc4\xdc\xc7\xbf\xb4\xf3" -"\xb6\xf8\xcd\xea\xc9\xc6\xb5\xc4\xcd\xa8\xd3\xc3\xd0\xcd\xbc\xc6" -"\xcb\xe3\xbb\xfa\xb3\xcc\xd0\xf2\xc9\xe8\xbc\xc6\xd3\xef\xd1\xd4" -"\xa3\xac\x0a\xd2\xd1\xbe\xad\xbe\xdf\xd3\xd0\xca\xae\xb6\xe0\xc4" -"\xea\xb5\xc4\xb7\xa2\xd5\xb9\xc0\xfa\xca\xb7\xa3\xac\xb3\xc9\xca" -"\xec\xc7\xd2\xce\xc8\xb6\xa8\xa1\xa3\xd5\xe2\xd6\xd6\xd3\xef\xd1" -"\xd4\xbe\xdf\xd3\xd0\xb7\xc7\xb3\xa3\xbc\xf2\xbd\xdd\xb6\xf8\xc7" -"\xe5\xce\xfa\x0a\xb5\xc4\xd3\xef\xb7\xa8\xcc\xd8\xb5\xe3\xa3\xac" -"\xca\xca\xba\xcf\xcd\xea\xb3\xc9\xb8\xf7\xd6\xd6\xb8\xdf\xb2\xe3" -"\xc8\xce\xce\xf1\xa3\xac\xbc\xb8\xba\xf5\xbf\xc9\xd2\xd4\xd4\xda" -"\xcb\xf9\xd3\xd0\xb5\xc4\xb2\xd9\xd7\xf7\xcf\xb5\xcd\xb3\xd6\xd0" -"\x0a\xd4\xcb\xd0\xd0\xa1\xa3\xd5\xe2\xd6\xd6\xd3\xef\xd1\xd4\xbc" -"\xf2\xb5\xa5\xb6\xf8\xc7\xbf\xb4\xf3\xa3\xac\xca\xca\xba\xcf\xb8" -"\xf7\xd6\xd6\xc8\xcb\xca\xbf\xd1\xa7\xcf\xb0\xca\xb9\xd3\xc3\xa1" -"\xa3\xc4\xbf\xc7\xb0\xa3\xac\xbb\xf9\xd3\xda\xd5\xe2\x0a\xd6\xd6" -"\xd3\xef\xd1\xd4\xb5\xc4\xcf\xe0\xb9\xd8\xbc\xbc\xca\xf5\xd5\xfd" -"\xd4\xda\xb7\xc9\xcb\xd9\xb5\xc4\xb7\xa2\xd5\xb9\xa3\xac\xd3\xc3" -"\xbb\xa7\xca\xfd\xc1\xbf\xbc\xb1\xbe\xe7\xc0\xa9\xb4\xf3\xa3\xac" -"\xcf\xe0\xb9\xd8\xb5\xc4\xd7\xca\xd4\xb4\xb7\xc7\xb3\xa3\xb6\xe0" -"\xa1\xa3\x0a\x0a", -"\x50\x79\x74\x68\x6f\x6e\xef\xbc\x88\xe6\xb4\xbe\xe6\xa3\xae\xef" -"\xbc\x89\xe8\xaf\xad\xe8\xa8\x80\xe6\x98\xaf\xe4\xb8\x80\xe7\xa7" -"\x8d\xe5\x8a\x9f\xe8\x83\xbd\xe5\xbc\xba\xe5\xa4\xa7\xe8\x80\x8c" -"\xe5\xae\x8c\xe5\x96\x84\xe7\x9a\x84\xe9\x80\x9a\xe7\x94\xa8\xe5" -"\x9e\x8b\xe8\xae\xa1\xe7\xae\x97\xe6\x9c\xba\xe7\xa8\x8b\xe5\xba" -"\x8f\xe8\xae\xbe\xe8\xae\xa1\xe8\xaf\xad\xe8\xa8\x80\xef\xbc\x8c" -"\x0a\xe5\xb7\xb2\xe7\xbb\x8f\xe5\x85\xb7\xe6\x9c\x89\xe5\x8d\x81" -"\xe5\xa4\x9a\xe5\xb9\xb4\xe7\x9a\x84\xe5\x8f\x91\xe5\xb1\x95\xe5" -"\x8e\x86\xe5\x8f\xb2\xef\xbc\x8c\xe6\x88\x90\xe7\x86\x9f\xe4\xb8" -"\x94\xe7\xa8\xb3\xe5\xae\x9a\xe3\x80\x82\xe8\xbf\x99\xe7\xa7\x8d" -"\xe8\xaf\xad\xe8\xa8\x80\xe5\x85\xb7\xe6\x9c\x89\xe9\x9d\x9e\xe5" -"\xb8\xb8\xe7\xae\x80\xe6\x8d\xb7\xe8\x80\x8c\xe6\xb8\x85\xe6\x99" -"\xb0\x0a\xe7\x9a\x84\xe8\xaf\xad\xe6\xb3\x95\xe7\x89\xb9\xe7\x82" -"\xb9\xef\xbc\x8c\xe9\x80\x82\xe5\x90\x88\xe5\xae\x8c\xe6\x88\x90" -"\xe5\x90\x84\xe7\xa7\x8d\xe9\xab\x98\xe5\xb1\x82\xe4\xbb\xbb\xe5" -"\x8a\xa1\xef\xbc\x8c\xe5\x87\xa0\xe4\xb9\x8e\xe5\x8f\xaf\xe4\xbb" -"\xa5\xe5\x9c\xa8\xe6\x89\x80\xe6\x9c\x89\xe7\x9a\x84\xe6\x93\x8d" -"\xe4\xbd\x9c\xe7\xb3\xbb\xe7\xbb\x9f\xe4\xb8\xad\x0a\xe8\xbf\x90" -"\xe8\xa1\x8c\xe3\x80\x82\xe8\xbf\x99\xe7\xa7\x8d\xe8\xaf\xad\xe8" -"\xa8\x80\xe7\xae\x80\xe5\x8d\x95\xe8\x80\x8c\xe5\xbc\xba\xe5\xa4" -"\xa7\xef\xbc\x8c\xe9\x80\x82\xe5\x90\x88\xe5\x90\x84\xe7\xa7\x8d" -"\xe4\xba\xba\xe5\xa3\xab\xe5\xad\xa6\xe4\xb9\xa0\xe4\xbd\xbf\xe7" -"\x94\xa8\xe3\x80\x82\xe7\x9b\xae\xe5\x89\x8d\xef\xbc\x8c\xe5\x9f" -"\xba\xe4\xba\x8e\xe8\xbf\x99\x0a\xe7\xa7\x8d\xe8\xaf\xad\xe8\xa8" -"\x80\xe7\x9a\x84\xe7\x9b\xb8\xe5\x85\xb3\xe6\x8a\x80\xe6\x9c\xaf" -"\xe6\xad\xa3\xe5\x9c\xa8\xe9\xa3\x9e\xe9\x80\x9f\xe7\x9a\x84\xe5" -"\x8f\x91\xe5\xb1\x95\xef\xbc\x8c\xe7\x94\xa8\xe6\x88\xb7\xe6\x95" -"\xb0\xe9\x87\x8f\xe6\x80\xa5\xe5\x89\xa7\xe6\x89\xa9\xe5\xa4\xa7" -"\xef\xbc\x8c\xe7\x9b\xb8\xe5\x85\xb3\xe7\x9a\x84\xe8\xb5\x84\xe6" -"\xba\x90\xe9\x9d\x9e\xe5\xb8\xb8\xe5\xa4\x9a\xe3\x80\x82\x0a\x0a"), -'gbk': ( -"\x50\x79\x74\x68\x6f\x6e\xa3\xa8\xc5\xc9\xc9\xad\xa3\xa9\xd3\xef" -"\xd1\xd4\xca\xc7\xd2\xbb\xd6\xd6\xb9\xa6\xc4\xdc\xc7\xbf\xb4\xf3" -"\xb6\xf8\xcd\xea\xc9\xc6\xb5\xc4\xcd\xa8\xd3\xc3\xd0\xcd\xbc\xc6" -"\xcb\xe3\xbb\xfa\xb3\xcc\xd0\xf2\xc9\xe8\xbc\xc6\xd3\xef\xd1\xd4" -"\xa3\xac\x0a\xd2\xd1\xbe\xad\xbe\xdf\xd3\xd0\xca\xae\xb6\xe0\xc4" -"\xea\xb5\xc4\xb7\xa2\xd5\xb9\xc0\xfa\xca\xb7\xa3\xac\xb3\xc9\xca" -"\xec\xc7\xd2\xce\xc8\xb6\xa8\xa1\xa3\xd5\xe2\xd6\xd6\xd3\xef\xd1" -"\xd4\xbe\xdf\xd3\xd0\xb7\xc7\xb3\xa3\xbc\xf2\xbd\xdd\xb6\xf8\xc7" -"\xe5\xce\xfa\x0a\xb5\xc4\xd3\xef\xb7\xa8\xcc\xd8\xb5\xe3\xa3\xac" -"\xca\xca\xba\xcf\xcd\xea\xb3\xc9\xb8\xf7\xd6\xd6\xb8\xdf\xb2\xe3" -"\xc8\xce\xce\xf1\xa3\xac\xbc\xb8\xba\xf5\xbf\xc9\xd2\xd4\xd4\xda" -"\xcb\xf9\xd3\xd0\xb5\xc4\xb2\xd9\xd7\xf7\xcf\xb5\xcd\xb3\xd6\xd0" -"\x0a\xd4\xcb\xd0\xd0\xa1\xa3\xd5\xe2\xd6\xd6\xd3\xef\xd1\xd4\xbc" -"\xf2\xb5\xa5\xb6\xf8\xc7\xbf\xb4\xf3\xa3\xac\xca\xca\xba\xcf\xb8" -"\xf7\xd6\xd6\xc8\xcb\xca\xbf\xd1\xa7\xcf\xb0\xca\xb9\xd3\xc3\xa1" -"\xa3\xc4\xbf\xc7\xb0\xa3\xac\xbb\xf9\xd3\xda\xd5\xe2\x0a\xd6\xd6" -"\xd3\xef\xd1\xd4\xb5\xc4\xcf\xe0\xb9\xd8\xbc\xbc\xca\xf5\xd5\xfd" -"\xd4\xda\xb7\xc9\xcb\xd9\xb5\xc4\xb7\xa2\xd5\xb9\xa3\xac\xd3\xc3" -"\xbb\xa7\xca\xfd\xc1\xbf\xbc\xb1\xbe\xe7\xc0\xa9\xb4\xf3\xa3\xac" -"\xcf\xe0\xb9\xd8\xb5\xc4\xd7\xca\xd4\xb4\xb7\xc7\xb3\xa3\xb6\xe0" -"\xa1\xa3\x0a\xc8\xe7\xba\xce\xd4\xda\x20\x50\x79\x74\x68\x6f\x6e" -"\x20\xd6\xd0\xca\xb9\xd3\xc3\xbc\xc8\xd3\xd0\xb5\xc4\x20\x43\x20" -"\x6c\x69\x62\x72\x61\x72\x79\x3f\x0a\xa1\xa1\xd4\xda\xd9\x59\xd3" -"\x8d\xbf\xc6\xbc\xbc\xbf\xec\xcb\xd9\xb0\x6c\xd5\xb9\xb5\xc4\xbd" -"\xf1\xcc\xec\x2c\x20\xe9\x5f\xb0\x6c\xbc\xb0\x9c\x79\xd4\x87\xdc" -"\x9b\xf3\x77\xb5\xc4\xcb\xd9\xb6\xc8\xca\xc7\xb2\xbb\xc8\xdd\xba" -"\xf6\xd2\x95\xb5\xc4\x0a\xd5\x6e\xee\x7d\x2e\x20\x9e\xe9\xbc\xd3" -"\xbf\xec\xe9\x5f\xb0\x6c\xbc\xb0\x9c\x79\xd4\x87\xb5\xc4\xcb\xd9" -"\xb6\xc8\x2c\x20\xce\xd2\x82\x83\xb1\xe3\xb3\xa3\xcf\xa3\xcd\xfb" -"\xc4\xdc\xc0\xfb\xd3\xc3\xd2\xbb\xd0\xa9\xd2\xd1\xe9\x5f\xb0\x6c" -"\xba\xc3\xb5\xc4\x0a\x6c\x69\x62\x72\x61\x72\x79\x2c\x20\x81\x4b" -"\xd3\xd0\xd2\xbb\x82\x80\x20\x66\x61\x73\x74\x20\x70\x72\x6f\x74" -"\x6f\x74\x79\x70\x69\x6e\x67\x20\xb5\xc4\x20\x70\x72\x6f\x67\x72" -"\x61\x6d\x6d\x69\x6e\x67\x20\x6c\x61\x6e\x67\x75\x61\x67\x65\x20" -"\xbf\xc9\x0a\xb9\xa9\xca\xb9\xd3\xc3\x2e\x20\xc4\xbf\xc7\xb0\xd3" -"\xd0\xd4\x53\xd4\x53\xb6\xe0\xb6\xe0\xb5\xc4\x20\x6c\x69\x62\x72" -"\x61\x72\x79\x20\xca\xc7\xd2\xd4\x20\x43\x20\x8c\x91\xb3\xc9\x2c" -"\x20\xb6\xf8\x20\x50\x79\x74\x68\x6f\x6e\x20\xca\xc7\xd2\xbb\x82" -"\x80\x0a\x66\x61\x73\x74\x20\x70\x72\x6f\x74\x6f\x74\x79\x70\x69" -"\x6e\x67\x20\xb5\xc4\x20\x70\x72\x6f\x67\x72\x61\x6d\x6d\x69\x6e" -"\x67\x20\x6c\x61\x6e\x67\x75\x61\x67\x65\x2e\x20\xb9\xca\xce\xd2" -"\x82\x83\xcf\xa3\xcd\xfb\xc4\xdc\x8c\xa2\xbc\xc8\xd3\xd0\xb5\xc4" -"\x0a\x43\x20\x6c\x69\x62\x72\x61\x72\x79\x20\xc4\xc3\xb5\xbd\x20" -"\x50\x79\x74\x68\x6f\x6e\x20\xb5\xc4\xad\x68\xbe\xb3\xd6\xd0\x9c" -"\x79\xd4\x87\xbc\xb0\xd5\xfb\xba\xcf\x2e\x20\xc6\xe4\xd6\xd0\xd7" -"\xee\xd6\xf7\xd2\xaa\xd2\xb2\xca\xc7\xce\xd2\x82\x83\xcb\xf9\x0a" -"\xd2\xaa\xd3\x91\xd5\x93\xb5\xc4\x86\x96\xee\x7d\xbe\xcd\xca\xc7" -"\x3a\x0a\x0a", -"\x50\x79\x74\x68\x6f\x6e\xef\xbc\x88\xe6\xb4\xbe\xe6\xa3\xae\xef" -"\xbc\x89\xe8\xaf\xad\xe8\xa8\x80\xe6\x98\xaf\xe4\xb8\x80\xe7\xa7" -"\x8d\xe5\x8a\x9f\xe8\x83\xbd\xe5\xbc\xba\xe5\xa4\xa7\xe8\x80\x8c" -"\xe5\xae\x8c\xe5\x96\x84\xe7\x9a\x84\xe9\x80\x9a\xe7\x94\xa8\xe5" -"\x9e\x8b\xe8\xae\xa1\xe7\xae\x97\xe6\x9c\xba\xe7\xa8\x8b\xe5\xba" -"\x8f\xe8\xae\xbe\xe8\xae\xa1\xe8\xaf\xad\xe8\xa8\x80\xef\xbc\x8c" -"\x0a\xe5\xb7\xb2\xe7\xbb\x8f\xe5\x85\xb7\xe6\x9c\x89\xe5\x8d\x81" -"\xe5\xa4\x9a\xe5\xb9\xb4\xe7\x9a\x84\xe5\x8f\x91\xe5\xb1\x95\xe5" -"\x8e\x86\xe5\x8f\xb2\xef\xbc\x8c\xe6\x88\x90\xe7\x86\x9f\xe4\xb8" -"\x94\xe7\xa8\xb3\xe5\xae\x9a\xe3\x80\x82\xe8\xbf\x99\xe7\xa7\x8d" -"\xe8\xaf\xad\xe8\xa8\x80\xe5\x85\xb7\xe6\x9c\x89\xe9\x9d\x9e\xe5" -"\xb8\xb8\xe7\xae\x80\xe6\x8d\xb7\xe8\x80\x8c\xe6\xb8\x85\xe6\x99" -"\xb0\x0a\xe7\x9a\x84\xe8\xaf\xad\xe6\xb3\x95\xe7\x89\xb9\xe7\x82" -"\xb9\xef\xbc\x8c\xe9\x80\x82\xe5\x90\x88\xe5\xae\x8c\xe6\x88\x90" -"\xe5\x90\x84\xe7\xa7\x8d\xe9\xab\x98\xe5\xb1\x82\xe4\xbb\xbb\xe5" -"\x8a\xa1\xef\xbc\x8c\xe5\x87\xa0\xe4\xb9\x8e\xe5\x8f\xaf\xe4\xbb" -"\xa5\xe5\x9c\xa8\xe6\x89\x80\xe6\x9c\x89\xe7\x9a\x84\xe6\x93\x8d" -"\xe4\xbd\x9c\xe7\xb3\xbb\xe7\xbb\x9f\xe4\xb8\xad\x0a\xe8\xbf\x90" -"\xe8\xa1\x8c\xe3\x80\x82\xe8\xbf\x99\xe7\xa7\x8d\xe8\xaf\xad\xe8" -"\xa8\x80\xe7\xae\x80\xe5\x8d\x95\xe8\x80\x8c\xe5\xbc\xba\xe5\xa4" -"\xa7\xef\xbc\x8c\xe9\x80\x82\xe5\x90\x88\xe5\x90\x84\xe7\xa7\x8d" -"\xe4\xba\xba\xe5\xa3\xab\xe5\xad\xa6\xe4\xb9\xa0\xe4\xbd\xbf\xe7" -"\x94\xa8\xe3\x80\x82\xe7\x9b\xae\xe5\x89\x8d\xef\xbc\x8c\xe5\x9f" -"\xba\xe4\xba\x8e\xe8\xbf\x99\x0a\xe7\xa7\x8d\xe8\xaf\xad\xe8\xa8" -"\x80\xe7\x9a\x84\xe7\x9b\xb8\xe5\x85\xb3\xe6\x8a\x80\xe6\x9c\xaf" -"\xe6\xad\xa3\xe5\x9c\xa8\xe9\xa3\x9e\xe9\x80\x9f\xe7\x9a\x84\xe5" -"\x8f\x91\xe5\xb1\x95\xef\xbc\x8c\xe7\x94\xa8\xe6\x88\xb7\xe6\x95" -"\xb0\xe9\x87\x8f\xe6\x80\xa5\xe5\x89\xa7\xe6\x89\xa9\xe5\xa4\xa7" -"\xef\xbc\x8c\xe7\x9b\xb8\xe5\x85\xb3\xe7\x9a\x84\xe8\xb5\x84\xe6" -"\xba\x90\xe9\x9d\x9e\xe5\xb8\xb8\xe5\xa4\x9a\xe3\x80\x82\x0a\xe5" -"\xa6\x82\xe4\xbd\x95\xe5\x9c\xa8\x20\x50\x79\x74\x68\x6f\x6e\x20" -"\xe4\xb8\xad\xe4\xbd\xbf\xe7\x94\xa8\xe6\x97\xa2\xe6\x9c\x89\xe7" -"\x9a\x84\x20\x43\x20\x6c\x69\x62\x72\x61\x72\x79\x3f\x0a\xe3\x80" -"\x80\xe5\x9c\xa8\xe8\xb3\x87\xe8\xa8\x8a\xe7\xa7\x91\xe6\x8a\x80" -"\xe5\xbf\xab\xe9\x80\x9f\xe7\x99\xbc\xe5\xb1\x95\xe7\x9a\x84\xe4" -"\xbb\x8a\xe5\xa4\xa9\x2c\x20\xe9\x96\x8b\xe7\x99\xbc\xe5\x8f\x8a" -"\xe6\xb8\xac\xe8\xa9\xa6\xe8\xbb\x9f\xe9\xab\x94\xe7\x9a\x84\xe9" -"\x80\x9f\xe5\xba\xa6\xe6\x98\xaf\xe4\xb8\x8d\xe5\xae\xb9\xe5\xbf" -"\xbd\xe8\xa6\x96\xe7\x9a\x84\x0a\xe8\xaa\xb2\xe9\xa1\x8c\x2e\x20" -"\xe7\x82\xba\xe5\x8a\xa0\xe5\xbf\xab\xe9\x96\x8b\xe7\x99\xbc\xe5" -"\x8f\x8a\xe6\xb8\xac\xe8\xa9\xa6\xe7\x9a\x84\xe9\x80\x9f\xe5\xba" -"\xa6\x2c\x20\xe6\x88\x91\xe5\x80\x91\xe4\xbe\xbf\xe5\xb8\xb8\xe5" -"\xb8\x8c\xe6\x9c\x9b\xe8\x83\xbd\xe5\x88\xa9\xe7\x94\xa8\xe4\xb8" -"\x80\xe4\xba\x9b\xe5\xb7\xb2\xe9\x96\x8b\xe7\x99\xbc\xe5\xa5\xbd" -"\xe7\x9a\x84\x0a\x6c\x69\x62\x72\x61\x72\x79\x2c\x20\xe4\xb8\xa6" -"\xe6\x9c\x89\xe4\xb8\x80\xe5\x80\x8b\x20\x66\x61\x73\x74\x20\x70" -"\x72\x6f\x74\x6f\x74\x79\x70\x69\x6e\x67\x20\xe7\x9a\x84\x20\x70" -"\x72\x6f\x67\x72\x61\x6d\x6d\x69\x6e\x67\x20\x6c\x61\x6e\x67\x75" -"\x61\x67\x65\x20\xe5\x8f\xaf\x0a\xe4\xbe\x9b\xe4\xbd\xbf\xe7\x94" -"\xa8\x2e\x20\xe7\x9b\xae\xe5\x89\x8d\xe6\x9c\x89\xe8\xa8\xb1\xe8" -"\xa8\xb1\xe5\xa4\x9a\xe5\xa4\x9a\xe7\x9a\x84\x20\x6c\x69\x62\x72" -"\x61\x72\x79\x20\xe6\x98\xaf\xe4\xbb\xa5\x20\x43\x20\xe5\xaf\xab" -"\xe6\x88\x90\x2c\x20\xe8\x80\x8c\x20\x50\x79\x74\x68\x6f\x6e\x20" -"\xe6\x98\xaf\xe4\xb8\x80\xe5\x80\x8b\x0a\x66\x61\x73\x74\x20\x70" -"\x72\x6f\x74\x6f\x74\x79\x70\x69\x6e\x67\x20\xe7\x9a\x84\x20\x70" -"\x72\x6f\x67\x72\x61\x6d\x6d\x69\x6e\x67\x20\x6c\x61\x6e\x67\x75" -"\x61\x67\x65\x2e\x20\xe6\x95\x85\xe6\x88\x91\xe5\x80\x91\xe5\xb8" -"\x8c\xe6\x9c\x9b\xe8\x83\xbd\xe5\xb0\x87\xe6\x97\xa2\xe6\x9c\x89" -"\xe7\x9a\x84\x0a\x43\x20\x6c\x69\x62\x72\x61\x72\x79\x20\xe6\x8b" -"\xbf\xe5\x88\xb0\x20\x50\x79\x74\x68\x6f\x6e\x20\xe7\x9a\x84\xe7" -"\x92\xb0\xe5\xa2\x83\xe4\xb8\xad\xe6\xb8\xac\xe8\xa9\xa6\xe5\x8f" -"\x8a\xe6\x95\xb4\xe5\x90\x88\x2e\x20\xe5\x85\xb6\xe4\xb8\xad\xe6" -"\x9c\x80\xe4\xb8\xbb\xe8\xa6\x81\xe4\xb9\x9f\xe6\x98\xaf\xe6\x88" -"\x91\xe5\x80\x91\xe6\x89\x80\x0a\xe8\xa6\x81\xe8\xa8\x8e\xe8\xab" -"\x96\xe7\x9a\x84\xe5\x95\x8f\xe9\xa1\x8c\xe5\xb0\xb1\xe6\x98\xaf" -"\x3a\x0a\x0a"), -'johab': ( -"\x99\xb1\xa4\x77\x88\x62\xd0\x61\x20\xcd\x5c\xaf\xa1\xc5\xa9\x9c" -"\x61\x0a\x0a\xdc\xc0\xdc\xc0\x90\x73\x21\x21\x20\xf1\x67\xe2\x9c" -"\xf0\x55\xcc\x81\xa3\x89\x9f\x85\x8a\xa1\x20\xdc\xde\xdc\xd3\xd2" -"\x7a\xd9\xaf\xd9\xaf\xd9\xaf\x20\x8b\x77\x96\xd3\x20\xdc\xd1\x95" -"\x81\x20\xdc\xc0\x2e\x20\x2e\x0a\xed\x3c\xb5\x77\xdc\xd1\x93\x77" -"\xd2\x73\x20\x2e\x20\x2e\x20\x2e\x20\x2e\x20\xac\xe1\xb6\x89\x9e" -"\xa1\x20\x95\x65\xd0\x62\xf0\xe0\x20\xe0\x3b\xd2\x7a\x20\x21\x20" -"\x21\x20\x21\x87\x41\x2e\x87\x41\x0a\xd3\x61\xd3\x61\xd3\x61\x20" -"\x88\x41\x88\x41\x88\x41\xd9\x69\x87\x41\x5f\x87\x41\x20\xb4\xe1" -"\x9f\x9a\x20\xc8\xa1\xc5\xc1\x8b\x7a\x20\x95\x61\xb7\x77\x20\xc3" -"\x97\xe2\x9c\x97\x69\xf0\xe0\x20\xdc\xc0\x97\x61\x8b\x7a\x0a\xac" -"\xe9\x9f\x7a\x20\xe0\x3b\xd2\x7a\x20\x2e\x20\x2e\x20\x2e\x20\x2e" -"\x20\x8a\x89\xb4\x81\xae\xba\x20\xdc\xd1\x8a\xa1\x20\xdc\xde\x9f" -"\x89\xdc\xc2\x8b\x7a\x20\xf1\x67\xf1\x62\xf5\x49\xed\xfc\xf3\xe9" -"\x8c\x61\xbb\x9a\x0a\xb5\xc1\xb2\xa1\xd2\x7a\x20\x21\x20\x21\x20" -"\xed\x3c\xb5\x77\xdc\xd1\x20\xe0\x3b\x93\x77\x8a\xa1\x20\xd9\x69" -"\xea\xbe\x89\xc5\x20\xb4\xf4\x93\x77\x8a\xa1\x93\x77\x20\xed\x3c" -"\x93\x77\x96\xc1\xd2\x7a\x20\x8b\x69\xb4\x81\x97\x7a\x0a\xdc\xde" -"\x9d\x61\x97\x41\xe2\x9c\x20\xaf\x81\xce\xa1\xae\xa1\xd2\x7a\x20" -"\xb4\xe1\x9f\x9a\x20\xf1\x67\xf1\x62\xf5\x49\xed\xfc\xf3\xe9\xaf" -"\x82\xdc\xef\x97\x69\xb4\x7a\x21\x21\x20\xdc\xc0\xdc\xc0\x90\x73" -"\xd9\xbd\x20\xd9\x62\xd9\x62\x2a\x0a\x0a", -"\xeb\x98\xa0\xeb\xb0\xa9\xea\xb0\x81\xed\x95\x98\x20\xed\x8e\xb2" -"\xec\x8b\x9c\xec\xbd\x9c\xeb\x9d\xbc\x0a\x0a\xe3\x89\xaf\xe3\x89" -"\xaf\xeb\x82\xa9\x21\x21\x20\xe5\x9b\xa0\xe4\xb9\x9d\xe6\x9c\x88" -"\xed\x8c\xa8\xeb\xaf\xa4\xeb\xa6\x94\xea\xb6\x88\x20\xe2\x93\xa1" -"\xe2\x93\x96\xed\x9b\x80\xc2\xbf\xc2\xbf\xc2\xbf\x20\xea\xb8\x8d" -"\xeb\x92\x99\x20\xe2\x93\x94\xeb\x8e\xa8\x20\xe3\x89\xaf\x2e\x20" -"\x2e\x0a\xe4\xba\x9e\xec\x98\x81\xe2\x93\x94\xeb\x8a\xa5\xed\x9a" -"\xb9\x20\x2e\x20\x2e\x20\x2e\x20\x2e\x20\xec\x84\x9c\xec\x9a\xb8" -"\xeb\xa4\x84\x20\xeb\x8e\x90\xed\x95\x99\xe4\xb9\x99\x20\xe5\xae" -"\xb6\xed\x9b\x80\x20\x21\x20\x21\x20\x21\xe3\x85\xa0\x2e\xe3\x85" -"\xa0\x0a\xed\x9d\x90\xed\x9d\x90\xed\x9d\x90\x20\xe3\x84\xb1\xe3" -"\x84\xb1\xe3\x84\xb1\xe2\x98\x86\xe3\x85\xa0\x5f\xe3\x85\xa0\x20" -"\xec\x96\xb4\xeb\xa6\xa8\x20\xed\x83\xb8\xec\xbd\xb0\xea\xb8\x90" -"\x20\xeb\x8e\x8c\xec\x9d\x91\x20\xec\xb9\x91\xe4\xb9\x9d\xeb\x93" -"\xa4\xe4\xb9\x99\x20\xe3\x89\xaf\xeb\x93\x9c\xea\xb8\x90\x0a\xec" -"\x84\xa4\xeb\xa6\x8c\x20\xe5\xae\xb6\xed\x9b\x80\x20\x2e\x20\x2e" -"\x20\x2e\x20\x2e\x20\xea\xb5\xb4\xec\x95\xa0\xec\x89\x8c\x20\xe2" -"\x93\x94\xea\xb6\x88\x20\xe2\x93\xa1\xeb\xa6\x98\xe3\x89\xb1\xea" -"\xb8\x90\x20\xe5\x9b\xa0\xe4\xbb\x81\xe5\xb7\x9d\xef\xa6\x81\xe4" -"\xb8\xad\xea\xb9\x8c\xec\xa6\xbc\x0a\xec\x99\x80\xec\x92\x80\xed" -"\x9b\x80\x20\x21\x20\x21\x20\xe4\xba\x9e\xec\x98\x81\xe2\x93\x94" -"\x20\xe5\xae\xb6\xeb\x8a\xa5\xea\xb6\x88\x20\xe2\x98\x86\xe4\xb8" -"\x8a\xea\xb4\x80\x20\xec\x97\x86\xeb\x8a\xa5\xea\xb6\x88\xeb\x8a" -"\xa5\x20\xe4\xba\x9e\xeb\x8a\xa5\xeb\x92\x88\xed\x9b\x80\x20\xea" -"\xb8\x80\xec\x95\xa0\xeb\x93\xb4\x0a\xe2\x93\xa1\xeb\xa0\xa4\xeb" -"\x93\x80\xe4\xb9\x9d\x20\xec\x8b\x80\xed\x92\x94\xec\x88\xb4\xed" -"\x9b\x80\x20\xec\x96\xb4\xeb\xa6\xa8\x20\xe5\x9b\xa0\xe4\xbb\x81" -"\xe5\xb7\x9d\xef\xa6\x81\xe4\xb8\xad\xec\x8b\x81\xe2\x91\xa8\xeb" -"\x93\xa4\xec\x95\x9c\x21\x21\x20\xe3\x89\xaf\xe3\x89\xaf\xeb\x82" -"\xa9\xe2\x99\xa1\x20\xe2\x8c\x92\xe2\x8c\x92\x2a\x0a\x0a"), -'shift_jis': ( -"\x50\x79\x74\x68\x6f\x6e\x20\x82\xcc\x8a\x4a\x94\xad\x82\xcd\x81" -"\x41\x31\x39\x39\x30\x20\x94\x4e\x82\xb2\x82\xeb\x82\xa9\x82\xe7" -"\x8a\x4a\x8e\x6e\x82\xb3\x82\xea\x82\xc4\x82\xa2\x82\xdc\x82\xb7" -"\x81\x42\x0a\x8a\x4a\x94\xad\x8e\xd2\x82\xcc\x20\x47\x75\x69\x64" -"\x6f\x20\x76\x61\x6e\x20\x52\x6f\x73\x73\x75\x6d\x20\x82\xcd\x8b" -"\xb3\x88\xe7\x97\x70\x82\xcc\x83\x76\x83\x8d\x83\x4f\x83\x89\x83" -"\x7e\x83\x93\x83\x4f\x8c\xbe\x8c\xea\x81\x75\x41\x42\x43\x81\x76" -"\x82\xcc\x8a\x4a\x94\xad\x82\xc9\x8e\x51\x89\xc1\x82\xb5\x82\xc4" -"\x82\xa2\x82\xdc\x82\xb5\x82\xbd\x82\xaa\x81\x41\x41\x42\x43\x20" -"\x82\xcd\x8e\xc0\x97\x70\x8f\xe3\x82\xcc\x96\xda\x93\x49\x82\xc9" -"\x82\xcd\x82\xa0\x82\xdc\x82\xe8\x93\x4b\x82\xb5\x82\xc4\x82\xa2" -"\x82\xdc\x82\xb9\x82\xf1\x82\xc5\x82\xb5\x82\xbd\x81\x42\x0a\x82" -"\xb1\x82\xcc\x82\xbd\x82\xdf\x81\x41\x47\x75\x69\x64\x6f\x20\x82" -"\xcd\x82\xe6\x82\xe8\x8e\xc0\x97\x70\x93\x49\x82\xc8\x83\x76\x83" -"\x8d\x83\x4f\x83\x89\x83\x7e\x83\x93\x83\x4f\x8c\xbe\x8c\xea\x82" -"\xcc\x8a\x4a\x94\xad\x82\xf0\x8a\x4a\x8e\x6e\x82\xb5\x81\x41\x89" -"\x70\x8d\x91\x20\x42\x42\x53\x20\x95\xfa\x91\x97\x82\xcc\x83\x52" -"\x83\x81\x83\x66\x83\x42\x94\xd4\x91\x67\x81\x75\x83\x82\x83\x93" -"\x83\x65\x83\x42\x20\x83\x70\x83\x43\x83\x5c\x83\x93\x81\x76\x82" -"\xcc\x83\x74\x83\x40\x83\x93\x82\xc5\x82\xa0\x82\xe9\x20\x47\x75" -"\x69\x64\x6f\x20\x82\xcd\x82\xb1\x82\xcc\x8c\xbe\x8c\xea\x82\xf0" -"\x81\x75\x50\x79\x74\x68\x6f\x6e\x81\x76\x82\xc6\x96\xbc\x82\xc3" -"\x82\xaf\x82\xdc\x82\xb5\x82\xbd\x81\x42\x0a\x82\xb1\x82\xcc\x82" -"\xe6\x82\xa4\x82\xc8\x94\x77\x8c\x69\x82\xa9\x82\xe7\x90\xb6\x82" -"\xdc\x82\xea\x82\xbd\x20\x50\x79\x74\x68\x6f\x6e\x20\x82\xcc\x8c" -"\xbe\x8c\xea\x90\xdd\x8c\x76\x82\xcd\x81\x41\x81\x75\x83\x56\x83" -"\x93\x83\x76\x83\x8b\x81\x76\x82\xc5\x81\x75\x8f\x4b\x93\xbe\x82" -"\xaa\x97\x65\x88\xd5\x81\x76\x82\xc6\x82\xa2\x82\xa4\x96\xda\x95" -"\x57\x82\xc9\x8f\x64\x93\x5f\x82\xaa\x92\x75\x82\xa9\x82\xea\x82" -"\xc4\x82\xa2\x82\xdc\x82\xb7\x81\x42\x0a\x91\xbd\x82\xad\x82\xcc" -"\x83\x58\x83\x4e\x83\x8a\x83\x76\x83\x67\x8c\x6e\x8c\xbe\x8c\xea" -"\x82\xc5\x82\xcd\x83\x86\x81\x5b\x83\x55\x82\xcc\x96\xda\x90\xe6" -"\x82\xcc\x97\x98\x95\xd6\x90\xab\x82\xf0\x97\x44\x90\xe6\x82\xb5" -"\x82\xc4\x90\x46\x81\x58\x82\xc8\x8b\x40\x94\x5c\x82\xf0\x8c\xbe" -"\x8c\xea\x97\x76\x91\x66\x82\xc6\x82\xb5\x82\xc4\x8e\xe6\x82\xe8" -"\x93\xfc\x82\xea\x82\xe9\x8f\xea\x8d\x87\x82\xaa\x91\xbd\x82\xa2" -"\x82\xcc\x82\xc5\x82\xb7\x82\xaa\x81\x41\x50\x79\x74\x68\x6f\x6e" -"\x20\x82\xc5\x82\xcd\x82\xbb\x82\xa4\x82\xa2\x82\xc1\x82\xbd\x8f" -"\xac\x8d\xd7\x8d\x48\x82\xaa\x92\xc7\x89\xc1\x82\xb3\x82\xea\x82" -"\xe9\x82\xb1\x82\xc6\x82\xcd\x82\xa0\x82\xdc\x82\xe8\x82\xa0\x82" -"\xe8\x82\xdc\x82\xb9\x82\xf1\x81\x42\x0a\x8c\xbe\x8c\xea\x8e\xa9" -"\x91\xcc\x82\xcc\x8b\x40\x94\x5c\x82\xcd\x8d\xc5\x8f\xac\x8c\xc0" -"\x82\xc9\x89\x9f\x82\xb3\x82\xa6\x81\x41\x95\x4b\x97\x76\x82\xc8" -"\x8b\x40\x94\x5c\x82\xcd\x8a\x67\x92\xa3\x83\x82\x83\x57\x83\x85" -"\x81\x5b\x83\x8b\x82\xc6\x82\xb5\x82\xc4\x92\xc7\x89\xc1\x82\xb7" -"\x82\xe9\x81\x41\x82\xc6\x82\xa2\x82\xa4\x82\xcc\x82\xaa\x20\x50" -"\x79\x74\x68\x6f\x6e\x20\x82\xcc\x83\x7c\x83\x8a\x83\x56\x81\x5b" -"\x82\xc5\x82\xb7\x81\x42\x0a\x0a", -"\x50\x79\x74\x68\x6f\x6e\x20\xe3\x81\xae\xe9\x96\x8b\xe7\x99\xba" -"\xe3\x81\xaf\xe3\x80\x81\x31\x39\x39\x30\x20\xe5\xb9\xb4\xe3\x81" -"\x94\xe3\x82\x8d\xe3\x81\x8b\xe3\x82\x89\xe9\x96\x8b\xe5\xa7\x8b" -"\xe3\x81\x95\xe3\x82\x8c\xe3\x81\xa6\xe3\x81\x84\xe3\x81\xbe\xe3" -"\x81\x99\xe3\x80\x82\x0a\xe9\x96\x8b\xe7\x99\xba\xe8\x80\x85\xe3" -"\x81\xae\x20\x47\x75\x69\x64\x6f\x20\x76\x61\x6e\x20\x52\x6f\x73" -"\x73\x75\x6d\x20\xe3\x81\xaf\xe6\x95\x99\xe8\x82\xb2\xe7\x94\xa8" -"\xe3\x81\xae\xe3\x83\x97\xe3\x83\xad\xe3\x82\xb0\xe3\x83\xa9\xe3" -"\x83\x9f\xe3\x83\xb3\xe3\x82\xb0\xe8\xa8\x80\xe8\xaa\x9e\xe3\x80" -"\x8c\x41\x42\x43\xe3\x80\x8d\xe3\x81\xae\xe9\x96\x8b\xe7\x99\xba" -"\xe3\x81\xab\xe5\x8f\x82\xe5\x8a\xa0\xe3\x81\x97\xe3\x81\xa6\xe3" -"\x81\x84\xe3\x81\xbe\xe3\x81\x97\xe3\x81\x9f\xe3\x81\x8c\xe3\x80" -"\x81\x41\x42\x43\x20\xe3\x81\xaf\xe5\xae\x9f\xe7\x94\xa8\xe4\xb8" -"\x8a\xe3\x81\xae\xe7\x9b\xae\xe7\x9a\x84\xe3\x81\xab\xe3\x81\xaf" -"\xe3\x81\x82\xe3\x81\xbe\xe3\x82\x8a\xe9\x81\xa9\xe3\x81\x97\xe3" -"\x81\xa6\xe3\x81\x84\xe3\x81\xbe\xe3\x81\x9b\xe3\x82\x93\xe3\x81" -"\xa7\xe3\x81\x97\xe3\x81\x9f\xe3\x80\x82\x0a\xe3\x81\x93\xe3\x81" -"\xae\xe3\x81\x9f\xe3\x82\x81\xe3\x80\x81\x47\x75\x69\x64\x6f\x20" -"\xe3\x81\xaf\xe3\x82\x88\xe3\x82\x8a\xe5\xae\x9f\xe7\x94\xa8\xe7" -"\x9a\x84\xe3\x81\xaa\xe3\x83\x97\xe3\x83\xad\xe3\x82\xb0\xe3\x83" -"\xa9\xe3\x83\x9f\xe3\x83\xb3\xe3\x82\xb0\xe8\xa8\x80\xe8\xaa\x9e" -"\xe3\x81\xae\xe9\x96\x8b\xe7\x99\xba\xe3\x82\x92\xe9\x96\x8b\xe5" -"\xa7\x8b\xe3\x81\x97\xe3\x80\x81\xe8\x8b\xb1\xe5\x9b\xbd\x20\x42" -"\x42\x53\x20\xe6\x94\xbe\xe9\x80\x81\xe3\x81\xae\xe3\x82\xb3\xe3" -"\x83\xa1\xe3\x83\x87\xe3\x82\xa3\xe7\x95\xaa\xe7\xb5\x84\xe3\x80" -"\x8c\xe3\x83\xa2\xe3\x83\xb3\xe3\x83\x86\xe3\x82\xa3\x20\xe3\x83" -"\x91\xe3\x82\xa4\xe3\x82\xbd\xe3\x83\xb3\xe3\x80\x8d\xe3\x81\xae" -"\xe3\x83\x95\xe3\x82\xa1\xe3\x83\xb3\xe3\x81\xa7\xe3\x81\x82\xe3" -"\x82\x8b\x20\x47\x75\x69\x64\x6f\x20\xe3\x81\xaf\xe3\x81\x93\xe3" -"\x81\xae\xe8\xa8\x80\xe8\xaa\x9e\xe3\x82\x92\xe3\x80\x8c\x50\x79" -"\x74\x68\x6f\x6e\xe3\x80\x8d\xe3\x81\xa8\xe5\x90\x8d\xe3\x81\xa5" -"\xe3\x81\x91\xe3\x81\xbe\xe3\x81\x97\xe3\x81\x9f\xe3\x80\x82\x0a" -"\xe3\x81\x93\xe3\x81\xae\xe3\x82\x88\xe3\x81\x86\xe3\x81\xaa\xe8" -"\x83\x8c\xe6\x99\xaf\xe3\x81\x8b\xe3\x82\x89\xe7\x94\x9f\xe3\x81" -"\xbe\xe3\x82\x8c\xe3\x81\x9f\x20\x50\x79\x74\x68\x6f\x6e\x20\xe3" -"\x81\xae\xe8\xa8\x80\xe8\xaa\x9e\xe8\xa8\xad\xe8\xa8\x88\xe3\x81" -"\xaf\xe3\x80\x81\xe3\x80\x8c\xe3\x82\xb7\xe3\x83\xb3\xe3\x83\x97" -"\xe3\x83\xab\xe3\x80\x8d\xe3\x81\xa7\xe3\x80\x8c\xe7\xbf\x92\xe5" -"\xbe\x97\xe3\x81\x8c\xe5\xae\xb9\xe6\x98\x93\xe3\x80\x8d\xe3\x81" -"\xa8\xe3\x81\x84\xe3\x81\x86\xe7\x9b\xae\xe6\xa8\x99\xe3\x81\xab" -"\xe9\x87\x8d\xe7\x82\xb9\xe3\x81\x8c\xe7\xbd\xae\xe3\x81\x8b\xe3" -"\x82\x8c\xe3\x81\xa6\xe3\x81\x84\xe3\x81\xbe\xe3\x81\x99\xe3\x80" -"\x82\x0a\xe5\xa4\x9a\xe3\x81\x8f\xe3\x81\xae\xe3\x82\xb9\xe3\x82" -"\xaf\xe3\x83\xaa\xe3\x83\x97\xe3\x83\x88\xe7\xb3\xbb\xe8\xa8\x80" -"\xe8\xaa\x9e\xe3\x81\xa7\xe3\x81\xaf\xe3\x83\xa6\xe3\x83\xbc\xe3" -"\x82\xb6\xe3\x81\xae\xe7\x9b\xae\xe5\x85\x88\xe3\x81\xae\xe5\x88" -"\xa9\xe4\xbe\xbf\xe6\x80\xa7\xe3\x82\x92\xe5\x84\xaa\xe5\x85\x88" -"\xe3\x81\x97\xe3\x81\xa6\xe8\x89\xb2\xe3\x80\x85\xe3\x81\xaa\xe6" -"\xa9\x9f\xe8\x83\xbd\xe3\x82\x92\xe8\xa8\x80\xe8\xaa\x9e\xe8\xa6" -"\x81\xe7\xb4\xa0\xe3\x81\xa8\xe3\x81\x97\xe3\x81\xa6\xe5\x8f\x96" -"\xe3\x82\x8a\xe5\x85\xa5\xe3\x82\x8c\xe3\x82\x8b\xe5\xa0\xb4\xe5" -"\x90\x88\xe3\x81\x8c\xe5\xa4\x9a\xe3\x81\x84\xe3\x81\xae\xe3\x81" -"\xa7\xe3\x81\x99\xe3\x81\x8c\xe3\x80\x81\x50\x79\x74\x68\x6f\x6e" -"\x20\xe3\x81\xa7\xe3\x81\xaf\xe3\x81\x9d\xe3\x81\x86\xe3\x81\x84" -"\xe3\x81\xa3\xe3\x81\x9f\xe5\xb0\x8f\xe7\xb4\xb0\xe5\xb7\xa5\xe3" -"\x81\x8c\xe8\xbf\xbd\xe5\x8a\xa0\xe3\x81\x95\xe3\x82\x8c\xe3\x82" -"\x8b\xe3\x81\x93\xe3\x81\xa8\xe3\x81\xaf\xe3\x81\x82\xe3\x81\xbe" -"\xe3\x82\x8a\xe3\x81\x82\xe3\x82\x8a\xe3\x81\xbe\xe3\x81\x9b\xe3" -"\x82\x93\xe3\x80\x82\x0a\xe8\xa8\x80\xe8\xaa\x9e\xe8\x87\xaa\xe4" -"\xbd\x93\xe3\x81\xae\xe6\xa9\x9f\xe8\x83\xbd\xe3\x81\xaf\xe6\x9c" -"\x80\xe5\xb0\x8f\xe9\x99\x90\xe3\x81\xab\xe6\x8a\xbc\xe3\x81\x95" -"\xe3\x81\x88\xe3\x80\x81\xe5\xbf\x85\xe8\xa6\x81\xe3\x81\xaa\xe6" -"\xa9\x9f\xe8\x83\xbd\xe3\x81\xaf\xe6\x8b\xa1\xe5\xbc\xb5\xe3\x83" -"\xa2\xe3\x82\xb8\xe3\x83\xa5\xe3\x83\xbc\xe3\x83\xab\xe3\x81\xa8" -"\xe3\x81\x97\xe3\x81\xa6\xe8\xbf\xbd\xe5\x8a\xa0\xe3\x81\x99\xe3" -"\x82\x8b\xe3\x80\x81\xe3\x81\xa8\xe3\x81\x84\xe3\x81\x86\xe3\x81" -"\xae\xe3\x81\x8c\x20\x50\x79\x74\x68\x6f\x6e\x20\xe3\x81\xae\xe3" -"\x83\x9d\xe3\x83\xaa\xe3\x82\xb7\xe3\x83\xbc\xe3\x81\xa7\xe3\x81" -"\x99\xe3\x80\x82\x0a\x0a"), -'shift_jisx0213': ( -"\x50\x79\x74\x68\x6f\x6e\x20\x82\xcc\x8a\x4a\x94\xad\x82\xcd\x81" -"\x41\x31\x39\x39\x30\x20\x94\x4e\x82\xb2\x82\xeb\x82\xa9\x82\xe7" -"\x8a\x4a\x8e\x6e\x82\xb3\x82\xea\x82\xc4\x82\xa2\x82\xdc\x82\xb7" -"\x81\x42\x0a\x8a\x4a\x94\xad\x8e\xd2\x82\xcc\x20\x47\x75\x69\x64" -"\x6f\x20\x76\x61\x6e\x20\x52\x6f\x73\x73\x75\x6d\x20\x82\xcd\x8b" -"\xb3\x88\xe7\x97\x70\x82\xcc\x83\x76\x83\x8d\x83\x4f\x83\x89\x83" -"\x7e\x83\x93\x83\x4f\x8c\xbe\x8c\xea\x81\x75\x41\x42\x43\x81\x76" -"\x82\xcc\x8a\x4a\x94\xad\x82\xc9\x8e\x51\x89\xc1\x82\xb5\x82\xc4" -"\x82\xa2\x82\xdc\x82\xb5\x82\xbd\x82\xaa\x81\x41\x41\x42\x43\x20" -"\x82\xcd\x8e\xc0\x97\x70\x8f\xe3\x82\xcc\x96\xda\x93\x49\x82\xc9" -"\x82\xcd\x82\xa0\x82\xdc\x82\xe8\x93\x4b\x82\xb5\x82\xc4\x82\xa2" -"\x82\xdc\x82\xb9\x82\xf1\x82\xc5\x82\xb5\x82\xbd\x81\x42\x0a\x82" -"\xb1\x82\xcc\x82\xbd\x82\xdf\x81\x41\x47\x75\x69\x64\x6f\x20\x82" -"\xcd\x82\xe6\x82\xe8\x8e\xc0\x97\x70\x93\x49\x82\xc8\x83\x76\x83" -"\x8d\x83\x4f\x83\x89\x83\x7e\x83\x93\x83\x4f\x8c\xbe\x8c\xea\x82" -"\xcc\x8a\x4a\x94\xad\x82\xf0\x8a\x4a\x8e\x6e\x82\xb5\x81\x41\x89" -"\x70\x8d\x91\x20\x42\x42\x53\x20\x95\xfa\x91\x97\x82\xcc\x83\x52" -"\x83\x81\x83\x66\x83\x42\x94\xd4\x91\x67\x81\x75\x83\x82\x83\x93" -"\x83\x65\x83\x42\x20\x83\x70\x83\x43\x83\x5c\x83\x93\x81\x76\x82" -"\xcc\x83\x74\x83\x40\x83\x93\x82\xc5\x82\xa0\x82\xe9\x20\x47\x75" -"\x69\x64\x6f\x20\x82\xcd\x82\xb1\x82\xcc\x8c\xbe\x8c\xea\x82\xf0" -"\x81\x75\x50\x79\x74\x68\x6f\x6e\x81\x76\x82\xc6\x96\xbc\x82\xc3" -"\x82\xaf\x82\xdc\x82\xb5\x82\xbd\x81\x42\x0a\x82\xb1\x82\xcc\x82" -"\xe6\x82\xa4\x82\xc8\x94\x77\x8c\x69\x82\xa9\x82\xe7\x90\xb6\x82" -"\xdc\x82\xea\x82\xbd\x20\x50\x79\x74\x68\x6f\x6e\x20\x82\xcc\x8c" -"\xbe\x8c\xea\x90\xdd\x8c\x76\x82\xcd\x81\x41\x81\x75\x83\x56\x83" -"\x93\x83\x76\x83\x8b\x81\x76\x82\xc5\x81\x75\x8f\x4b\x93\xbe\x82" -"\xaa\x97\x65\x88\xd5\x81\x76\x82\xc6\x82\xa2\x82\xa4\x96\xda\x95" -"\x57\x82\xc9\x8f\x64\x93\x5f\x82\xaa\x92\x75\x82\xa9\x82\xea\x82" -"\xc4\x82\xa2\x82\xdc\x82\xb7\x81\x42\x0a\x91\xbd\x82\xad\x82\xcc" -"\x83\x58\x83\x4e\x83\x8a\x83\x76\x83\x67\x8c\x6e\x8c\xbe\x8c\xea" -"\x82\xc5\x82\xcd\x83\x86\x81\x5b\x83\x55\x82\xcc\x96\xda\x90\xe6" -"\x82\xcc\x97\x98\x95\xd6\x90\xab\x82\xf0\x97\x44\x90\xe6\x82\xb5" -"\x82\xc4\x90\x46\x81\x58\x82\xc8\x8b\x40\x94\x5c\x82\xf0\x8c\xbe" -"\x8c\xea\x97\x76\x91\x66\x82\xc6\x82\xb5\x82\xc4\x8e\xe6\x82\xe8" -"\x93\xfc\x82\xea\x82\xe9\x8f\xea\x8d\x87\x82\xaa\x91\xbd\x82\xa2" -"\x82\xcc\x82\xc5\x82\xb7\x82\xaa\x81\x41\x50\x79\x74\x68\x6f\x6e" -"\x20\x82\xc5\x82\xcd\x82\xbb\x82\xa4\x82\xa2\x82\xc1\x82\xbd\x8f" -"\xac\x8d\xd7\x8d\x48\x82\xaa\x92\xc7\x89\xc1\x82\xb3\x82\xea\x82" -"\xe9\x82\xb1\x82\xc6\x82\xcd\x82\xa0\x82\xdc\x82\xe8\x82\xa0\x82" -"\xe8\x82\xdc\x82\xb9\x82\xf1\x81\x42\x0a\x8c\xbe\x8c\xea\x8e\xa9" -"\x91\xcc\x82\xcc\x8b\x40\x94\x5c\x82\xcd\x8d\xc5\x8f\xac\x8c\xc0" -"\x82\xc9\x89\x9f\x82\xb3\x82\xa6\x81\x41\x95\x4b\x97\x76\x82\xc8" -"\x8b\x40\x94\x5c\x82\xcd\x8a\x67\x92\xa3\x83\x82\x83\x57\x83\x85" -"\x81\x5b\x83\x8b\x82\xc6\x82\xb5\x82\xc4\x92\xc7\x89\xc1\x82\xb7" -"\x82\xe9\x81\x41\x82\xc6\x82\xa2\x82\xa4\x82\xcc\x82\xaa\x20\x50" -"\x79\x74\x68\x6f\x6e\x20\x82\xcc\x83\x7c\x83\x8a\x83\x56\x81\x5b" -"\x82\xc5\x82\xb7\x81\x42\x0a\x0a\x83\x6d\x82\xf5\x20\x83\x9e\x20" -"\x83\x67\x83\x4c\x88\x4b\x88\x79\x20\x98\x83\xfc\xd6\x20\xfc\xd2" -"\xfc\xe6\xfb\xd4\x0a", -"\x50\x79\x74\x68\x6f\x6e\x20\xe3\x81\xae\xe9\x96\x8b\xe7\x99\xba" -"\xe3\x81\xaf\xe3\x80\x81\x31\x39\x39\x30\x20\xe5\xb9\xb4\xe3\x81" -"\x94\xe3\x82\x8d\xe3\x81\x8b\xe3\x82\x89\xe9\x96\x8b\xe5\xa7\x8b" -"\xe3\x81\x95\xe3\x82\x8c\xe3\x81\xa6\xe3\x81\x84\xe3\x81\xbe\xe3" -"\x81\x99\xe3\x80\x82\x0a\xe9\x96\x8b\xe7\x99\xba\xe8\x80\x85\xe3" -"\x81\xae\x20\x47\x75\x69\x64\x6f\x20\x76\x61\x6e\x20\x52\x6f\x73" -"\x73\x75\x6d\x20\xe3\x81\xaf\xe6\x95\x99\xe8\x82\xb2\xe7\x94\xa8" -"\xe3\x81\xae\xe3\x83\x97\xe3\x83\xad\xe3\x82\xb0\xe3\x83\xa9\xe3" -"\x83\x9f\xe3\x83\xb3\xe3\x82\xb0\xe8\xa8\x80\xe8\xaa\x9e\xe3\x80" -"\x8c\x41\x42\x43\xe3\x80\x8d\xe3\x81\xae\xe9\x96\x8b\xe7\x99\xba" -"\xe3\x81\xab\xe5\x8f\x82\xe5\x8a\xa0\xe3\x81\x97\xe3\x81\xa6\xe3" -"\x81\x84\xe3\x81\xbe\xe3\x81\x97\xe3\x81\x9f\xe3\x81\x8c\xe3\x80" -"\x81\x41\x42\x43\x20\xe3\x81\xaf\xe5\xae\x9f\xe7\x94\xa8\xe4\xb8" -"\x8a\xe3\x81\xae\xe7\x9b\xae\xe7\x9a\x84\xe3\x81\xab\xe3\x81\xaf" -"\xe3\x81\x82\xe3\x81\xbe\xe3\x82\x8a\xe9\x81\xa9\xe3\x81\x97\xe3" -"\x81\xa6\xe3\x81\x84\xe3\x81\xbe\xe3\x81\x9b\xe3\x82\x93\xe3\x81" -"\xa7\xe3\x81\x97\xe3\x81\x9f\xe3\x80\x82\x0a\xe3\x81\x93\xe3\x81" -"\xae\xe3\x81\x9f\xe3\x82\x81\xe3\x80\x81\x47\x75\x69\x64\x6f\x20" -"\xe3\x81\xaf\xe3\x82\x88\xe3\x82\x8a\xe5\xae\x9f\xe7\x94\xa8\xe7" -"\x9a\x84\xe3\x81\xaa\xe3\x83\x97\xe3\x83\xad\xe3\x82\xb0\xe3\x83" -"\xa9\xe3\x83\x9f\xe3\x83\xb3\xe3\x82\xb0\xe8\xa8\x80\xe8\xaa\x9e" -"\xe3\x81\xae\xe9\x96\x8b\xe7\x99\xba\xe3\x82\x92\xe9\x96\x8b\xe5" -"\xa7\x8b\xe3\x81\x97\xe3\x80\x81\xe8\x8b\xb1\xe5\x9b\xbd\x20\x42" -"\x42\x53\x20\xe6\x94\xbe\xe9\x80\x81\xe3\x81\xae\xe3\x82\xb3\xe3" -"\x83\xa1\xe3\x83\x87\xe3\x82\xa3\xe7\x95\xaa\xe7\xb5\x84\xe3\x80" -"\x8c\xe3\x83\xa2\xe3\x83\xb3\xe3\x83\x86\xe3\x82\xa3\x20\xe3\x83" -"\x91\xe3\x82\xa4\xe3\x82\xbd\xe3\x83\xb3\xe3\x80\x8d\xe3\x81\xae" -"\xe3\x83\x95\xe3\x82\xa1\xe3\x83\xb3\xe3\x81\xa7\xe3\x81\x82\xe3" -"\x82\x8b\x20\x47\x75\x69\x64\x6f\x20\xe3\x81\xaf\xe3\x81\x93\xe3" -"\x81\xae\xe8\xa8\x80\xe8\xaa\x9e\xe3\x82\x92\xe3\x80\x8c\x50\x79" -"\x74\x68\x6f\x6e\xe3\x80\x8d\xe3\x81\xa8\xe5\x90\x8d\xe3\x81\xa5" -"\xe3\x81\x91\xe3\x81\xbe\xe3\x81\x97\xe3\x81\x9f\xe3\x80\x82\x0a" -"\xe3\x81\x93\xe3\x81\xae\xe3\x82\x88\xe3\x81\x86\xe3\x81\xaa\xe8" -"\x83\x8c\xe6\x99\xaf\xe3\x81\x8b\xe3\x82\x89\xe7\x94\x9f\xe3\x81" -"\xbe\xe3\x82\x8c\xe3\x81\x9f\x20\x50\x79\x74\x68\x6f\x6e\x20\xe3" -"\x81\xae\xe8\xa8\x80\xe8\xaa\x9e\xe8\xa8\xad\xe8\xa8\x88\xe3\x81" -"\xaf\xe3\x80\x81\xe3\x80\x8c\xe3\x82\xb7\xe3\x83\xb3\xe3\x83\x97" -"\xe3\x83\xab\xe3\x80\x8d\xe3\x81\xa7\xe3\x80\x8c\xe7\xbf\x92\xe5" -"\xbe\x97\xe3\x81\x8c\xe5\xae\xb9\xe6\x98\x93\xe3\x80\x8d\xe3\x81" -"\xa8\xe3\x81\x84\xe3\x81\x86\xe7\x9b\xae\xe6\xa8\x99\xe3\x81\xab" -"\xe9\x87\x8d\xe7\x82\xb9\xe3\x81\x8c\xe7\xbd\xae\xe3\x81\x8b\xe3" -"\x82\x8c\xe3\x81\xa6\xe3\x81\x84\xe3\x81\xbe\xe3\x81\x99\xe3\x80" -"\x82\x0a\xe5\xa4\x9a\xe3\x81\x8f\xe3\x81\xae\xe3\x82\xb9\xe3\x82" -"\xaf\xe3\x83\xaa\xe3\x83\x97\xe3\x83\x88\xe7\xb3\xbb\xe8\xa8\x80" -"\xe8\xaa\x9e\xe3\x81\xa7\xe3\x81\xaf\xe3\x83\xa6\xe3\x83\xbc\xe3" -"\x82\xb6\xe3\x81\xae\xe7\x9b\xae\xe5\x85\x88\xe3\x81\xae\xe5\x88" -"\xa9\xe4\xbe\xbf\xe6\x80\xa7\xe3\x82\x92\xe5\x84\xaa\xe5\x85\x88" -"\xe3\x81\x97\xe3\x81\xa6\xe8\x89\xb2\xe3\x80\x85\xe3\x81\xaa\xe6" -"\xa9\x9f\xe8\x83\xbd\xe3\x82\x92\xe8\xa8\x80\xe8\xaa\x9e\xe8\xa6" -"\x81\xe7\xb4\xa0\xe3\x81\xa8\xe3\x81\x97\xe3\x81\xa6\xe5\x8f\x96" -"\xe3\x82\x8a\xe5\x85\xa5\xe3\x82\x8c\xe3\x82\x8b\xe5\xa0\xb4\xe5" -"\x90\x88\xe3\x81\x8c\xe5\xa4\x9a\xe3\x81\x84\xe3\x81\xae\xe3\x81" -"\xa7\xe3\x81\x99\xe3\x81\x8c\xe3\x80\x81\x50\x79\x74\x68\x6f\x6e" -"\x20\xe3\x81\xa7\xe3\x81\xaf\xe3\x81\x9d\xe3\x81\x86\xe3\x81\x84" -"\xe3\x81\xa3\xe3\x81\x9f\xe5\xb0\x8f\xe7\xb4\xb0\xe5\xb7\xa5\xe3" -"\x81\x8c\xe8\xbf\xbd\xe5\x8a\xa0\xe3\x81\x95\xe3\x82\x8c\xe3\x82" -"\x8b\xe3\x81\x93\xe3\x81\xa8\xe3\x81\xaf\xe3\x81\x82\xe3\x81\xbe" -"\xe3\x82\x8a\xe3\x81\x82\xe3\x82\x8a\xe3\x81\xbe\xe3\x81\x9b\xe3" -"\x82\x93\xe3\x80\x82\x0a\xe8\xa8\x80\xe8\xaa\x9e\xe8\x87\xaa\xe4" -"\xbd\x93\xe3\x81\xae\xe6\xa9\x9f\xe8\x83\xbd\xe3\x81\xaf\xe6\x9c" -"\x80\xe5\xb0\x8f\xe9\x99\x90\xe3\x81\xab\xe6\x8a\xbc\xe3\x81\x95" -"\xe3\x81\x88\xe3\x80\x81\xe5\xbf\x85\xe8\xa6\x81\xe3\x81\xaa\xe6" -"\xa9\x9f\xe8\x83\xbd\xe3\x81\xaf\xe6\x8b\xa1\xe5\xbc\xb5\xe3\x83" -"\xa2\xe3\x82\xb8\xe3\x83\xa5\xe3\x83\xbc\xe3\x83\xab\xe3\x81\xa8" -"\xe3\x81\x97\xe3\x81\xa6\xe8\xbf\xbd\xe5\x8a\xa0\xe3\x81\x99\xe3" -"\x82\x8b\xe3\x80\x81\xe3\x81\xa8\xe3\x81\x84\xe3\x81\x86\xe3\x81" -"\xae\xe3\x81\x8c\x20\x50\x79\x74\x68\x6f\x6e\x20\xe3\x81\xae\xe3" -"\x83\x9d\xe3\x83\xaa\xe3\x82\xb7\xe3\x83\xbc\xe3\x81\xa7\xe3\x81" -"\x99\xe3\x80\x82\x0a\x0a\xe3\x83\x8e\xe3\x81\x8b\xe3\x82\x9a\x20" -"\xe3\x83\x88\xe3\x82\x9a\x20\xe3\x83\x88\xe3\x82\xad\xef\xa8\xb6" -"\xef\xa8\xb9\x20\xf0\xa1\x9a\xb4\xf0\xaa\x8e\x8c\x20\xe9\xba\x80" -"\xe9\xbd\x81\xf0\xa9\x9b\xb0\x0a"), -} diff --git a/lib-python/2.7/test/crashers/README b/lib-python/2.7/test/crashers/README --- a/lib-python/2.7/test/crashers/README +++ b/lib-python/2.7/test/crashers/README @@ -1,20 +1,16 @@ -This directory only contains tests for outstanding bugs that cause -the interpreter to segfault. Ideally this directory should always -be empty. Sometimes it may not be easy to fix the underlying cause. +This directory only contains tests for outstanding bugs that cause the +interpreter to segfault. Ideally this directory should always be empty, but +sometimes it may not be easy to fix the underlying cause and the bug is deemed +too obscure to invest the effort. Each test should fail when run from the command line: ./python Lib/test/crashers/weakref_in_del.py -Each test should have a link to the bug report: +Put as much info into a docstring or comments to help determine the cause of the +failure, as well as a bugs.python.org issue number if it exists. Particularly +note if the cause is system or environment dependent and what the variables are. - # http://python.org/sf/BUG# - -Put as much info into a docstring or comments to help determine -the cause of the failure. Particularly note if the cause is -system or environment dependent and what the variables are. - -Once the crash is fixed, the test case should be moved into an appropriate -test (even if it was originally from the test suite). This ensures the -regression doesn't happen again. And if it does, it should be easier -to track down. +Once the crash is fixed, the test case should be moved into an appropriate test +(even if it was originally from the test suite). This ensures the regression +doesn't happen again. And if it does, it should be easier to track down. diff --git a/lib-python/2.7/test/crashers/recursion_limit_too_high.py b/lib-python/2.7/test/crashers/recursion_limit_too_high.py --- a/lib-python/2.7/test/crashers/recursion_limit_too_high.py +++ b/lib-python/2.7/test/crashers/recursion_limit_too_high.py @@ -5,7 +5,7 @@ # file handles. # The point of this example is to show that sys.setrecursionlimit() is a -# hack, and not a robust solution. This example simply exercices a path +# hack, and not a robust solution. This example simply exercises a path # where it takes many C-level recursions, consuming a lot of stack # space, for each Python-level recursion. So 1000 times this amount of # stack space may be too much for standard platforms already. diff --git a/lib-python/2.7/test/decimaltestdata/and.decTest b/lib-python/2.7/test/decimaltestdata/and.decTest --- a/lib-python/2.7/test/decimaltestdata/and.decTest +++ b/lib-python/2.7/test/decimaltestdata/and.decTest @@ -1,338 +1,338 @@ ------------------------------------------------------------------------- --- and.decTest -- digitwise logical AND -- --- Copyright (c) IBM Corporation, 1981, 2008. All rights reserved. -- ------------------------------------------------------------------------- --- Please see the document "General Decimal Arithmetic Testcases" -- --- at http://www2.hursley.ibm.com/decimal for the description of -- --- these testcases. -- --- -- --- These testcases are experimental ('beta' versions), and they -- --- may contain errors. They are offered on an as-is basis. In -- --- particular, achieving the same results as the tests here is not -- --- a guarantee that an implementation complies with any Standard -- --- or specification. The tests are not exhaustive. -- --- -- --- Please send comments, suggestions, and corrections to the author: -- --- Mike Cowlishaw, IBM Fellow -- --- IBM UK, PO Box 31, Birmingham Road, Warwick CV34 5JL, UK -- --- mfc at uk.ibm.com -- ------------------------------------------------------------------------- -version: 2.59 - -extended: 1 -precision: 9 -rounding: half_up -maxExponent: 999 -minExponent: -999 - --- Sanity check (truth table) -andx001 and 0 0 -> 0 -andx002 and 0 1 -> 0 -andx003 and 1 0 -> 0 -andx004 and 1 1 -> 1 -andx005 and 1100 1010 -> 1000 -andx006 and 1111 10 -> 10 -andx007 and 1111 1010 -> 1010 - --- and at msd and msd-1 -andx010 and 000000000 000000000 -> 0 -andx011 and 000000000 100000000 -> 0 -andx012 and 100000000 000000000 -> 0 -andx013 and 100000000 100000000 -> 100000000 -andx014 and 000000000 000000000 -> 0 -andx015 and 000000000 010000000 -> 0 -andx016 and 010000000 000000000 -> 0 -andx017 and 010000000 010000000 -> 10000000 - --- Various lengths --- 123456789 123456789 123456789 -andx021 and 111111111 111111111 -> 111111111 -andx022 and 111111111111 111111111 -> 111111111 -andx023 and 111111111111 11111111 -> 11111111 -andx024 and 111111111 11111111 -> 11111111 -andx025 and 111111111 1111111 -> 1111111 -andx026 and 111111111111 111111 -> 111111 -andx027 and 111111111111 11111 -> 11111 -andx028 and 111111111111 1111 -> 1111 -andx029 and 111111111111 111 -> 111 -andx031 and 111111111111 11 -> 11 -andx032 and 111111111111 1 -> 1 -andx033 and 111111111111 1111111111 -> 111111111 -andx034 and 11111111111 11111111111 -> 111111111 -andx035 and 1111111111 111111111111 -> 111111111 -andx036 and 111111111 1111111111111 -> 111111111 - -andx040 and 111111111 111111111111 -> 111111111 -andx041 and 11111111 111111111111 -> 11111111 -andx042 and 11111111 111111111 -> 11111111 -andx043 and 1111111 111111111 -> 1111111 -andx044 and 111111 111111111 -> 111111 -andx045 and 11111 111111111 -> 11111 -andx046 and 1111 111111111 -> 1111 -andx047 and 111 111111111 -> 111 -andx048 and 11 111111111 -> 11 -andx049 and 1 111111111 -> 1 - -andx050 and 1111111111 1 -> 1 -andx051 and 111111111 1 -> 1 -andx052 and 11111111 1 -> 1 -andx053 and 1111111 1 -> 1 -andx054 and 111111 1 -> 1 -andx055 and 11111 1 -> 1 -andx056 and 1111 1 -> 1 -andx057 and 111 1 -> 1 -andx058 and 11 1 -> 1 -andx059 and 1 1 -> 1 - -andx060 and 1111111111 0 -> 0 -andx061 and 111111111 0 -> 0 -andx062 and 11111111 0 -> 0 -andx063 and 1111111 0 -> 0 -andx064 and 111111 0 -> 0 -andx065 and 11111 0 -> 0 -andx066 and 1111 0 -> 0 -andx067 and 111 0 -> 0 -andx068 and 11 0 -> 0 -andx069 and 1 0 -> 0 - -andx070 and 1 1111111111 -> 1 -andx071 and 1 111111111 -> 1 -andx072 and 1 11111111 -> 1 -andx073 and 1 1111111 -> 1 -andx074 and 1 111111 -> 1 -andx075 and 1 11111 -> 1 -andx076 and 1 1111 -> 1 -andx077 and 1 111 -> 1 -andx078 and 1 11 -> 1 -andx079 and 1 1 -> 1 - -andx080 and 0 1111111111 -> 0 -andx081 and 0 111111111 -> 0 -andx082 and 0 11111111 -> 0 -andx083 and 0 1111111 -> 0 -andx084 and 0 111111 -> 0 -andx085 and 0 11111 -> 0 -andx086 and 0 1111 -> 0 -andx087 and 0 111 -> 0 -andx088 and 0 11 -> 0 -andx089 and 0 1 -> 0 - -andx090 and 011111111 111111111 -> 11111111 -andx091 and 101111111 111111111 -> 101111111 -andx092 and 110111111 111111111 -> 110111111 -andx093 and 111011111 111111111 -> 111011111 -andx094 and 111101111 111111111 -> 111101111 -andx095 and 111110111 111111111 -> 111110111 -andx096 and 111111011 111111111 -> 111111011 -andx097 and 111111101 111111111 -> 111111101 -andx098 and 111111110 111111111 -> 111111110 - -andx100 and 111111111 011111111 -> 11111111 -andx101 and 111111111 101111111 -> 101111111 -andx102 and 111111111 110111111 -> 110111111 -andx103 and 111111111 111011111 -> 111011111 -andx104 and 111111111 111101111 -> 111101111 -andx105 and 111111111 111110111 -> 111110111 -andx106 and 111111111 111111011 -> 111111011 -andx107 and 111111111 111111101 -> 111111101 -andx108 and 111111111 111111110 -> 111111110 - --- non-0/1 should not be accepted, nor should signs -andx220 and 111111112 111111111 -> NaN Invalid_operation -andx221 and 333333333 333333333 -> NaN Invalid_operation -andx222 and 555555555 555555555 -> NaN Invalid_operation -andx223 and 777777777 777777777 -> NaN Invalid_operation -andx224 and 999999999 999999999 -> NaN Invalid_operation -andx225 and 222222222 999999999 -> NaN Invalid_operation -andx226 and 444444444 999999999 -> NaN Invalid_operation -andx227 and 666666666 999999999 -> NaN Invalid_operation -andx228 and 888888888 999999999 -> NaN Invalid_operation -andx229 and 999999999 222222222 -> NaN Invalid_operation -andx230 and 999999999 444444444 -> NaN Invalid_operation -andx231 and 999999999 666666666 -> NaN Invalid_operation -andx232 and 999999999 888888888 -> NaN Invalid_operation --- a few randoms -andx240 and 567468689 -934981942 -> NaN Invalid_operation -andx241 and 567367689 934981942 -> NaN Invalid_operation -andx242 and -631917772 -706014634 -> NaN Invalid_operation -andx243 and -756253257 138579234 -> NaN Invalid_operation -andx244 and 835590149 567435400 -> NaN Invalid_operation --- test MSD -andx250 and 200000000 100000000 -> NaN Invalid_operation -andx251 and 700000000 100000000 -> NaN Invalid_operation -andx252 and 800000000 100000000 -> NaN Invalid_operation -andx253 and 900000000 100000000 -> NaN Invalid_operation -andx254 and 200000000 000000000 -> NaN Invalid_operation -andx255 and 700000000 000000000 -> NaN Invalid_operation -andx256 and 800000000 000000000 -> NaN Invalid_operation -andx257 and 900000000 000000000 -> NaN Invalid_operation -andx258 and 100000000 200000000 -> NaN Invalid_operation -andx259 and 100000000 700000000 -> NaN Invalid_operation -andx260 and 100000000 800000000 -> NaN Invalid_operation -andx261 and 100000000 900000000 -> NaN Invalid_operation -andx262 and 000000000 200000000 -> NaN Invalid_operation -andx263 and 000000000 700000000 -> NaN Invalid_operation -andx264 and 000000000 800000000 -> NaN Invalid_operation -andx265 and 000000000 900000000 -> NaN Invalid_operation --- test MSD-1 -andx270 and 020000000 100000000 -> NaN Invalid_operation -andx271 and 070100000 100000000 -> NaN Invalid_operation -andx272 and 080010000 100000001 -> NaN Invalid_operation -andx273 and 090001000 100000010 -> NaN Invalid_operation -andx274 and 100000100 020010100 -> NaN Invalid_operation -andx275 and 100000000 070001000 -> NaN Invalid_operation -andx276 and 100000010 080010100 -> NaN Invalid_operation -andx277 and 100000000 090000010 -> NaN Invalid_operation --- test LSD -andx280 and 001000002 100000000 -> NaN Invalid_operation -andx281 and 000000007 100000000 -> NaN Invalid_operation -andx282 and 000000008 100000000 -> NaN Invalid_operation -andx283 and 000000009 100000000 -> NaN Invalid_operation -andx284 and 100000000 000100002 -> NaN Invalid_operation -andx285 and 100100000 001000007 -> NaN Invalid_operation -andx286 and 100010000 010000008 -> NaN Invalid_operation -andx287 and 100001000 100000009 -> NaN Invalid_operation --- test Middie -andx288 and 001020000 100000000 -> NaN Invalid_operation -andx289 and 000070001 100000000 -> NaN Invalid_operation -andx290 and 000080000 100010000 -> NaN Invalid_operation -andx291 and 000090000 100001000 -> NaN Invalid_operation -andx292 and 100000010 000020100 -> NaN Invalid_operation -andx293 and 100100000 000070010 -> NaN Invalid_operation -andx294 and 100010100 000080001 -> NaN Invalid_operation -andx295 and 100001000 000090000 -> NaN Invalid_operation --- signs -andx296 and -100001000 -000000000 -> NaN Invalid_operation -andx297 and -100001000 000010000 -> NaN Invalid_operation -andx298 and 100001000 -000000000 -> NaN Invalid_operation -andx299 and 100001000 000011000 -> 1000 - --- Nmax, Nmin, Ntiny -andx331 and 2 9.99999999E+999 -> NaN Invalid_operation -andx332 and 3 1E-999 -> NaN Invalid_operation -andx333 and 4 1.00000000E-999 -> NaN Invalid_operation -andx334 and 5 1E-1007 -> NaN Invalid_operation -andx335 and 6 -1E-1007 -> NaN Invalid_operation -andx336 and 7 -1.00000000E-999 -> NaN Invalid_operation -andx337 and 8 -1E-999 -> NaN Invalid_operation -andx338 and 9 -9.99999999E+999 -> NaN Invalid_operation -andx341 and 9.99999999E+999 -18 -> NaN Invalid_operation -andx342 and 1E-999 01 -> NaN Invalid_operation -andx343 and 1.00000000E-999 -18 -> NaN Invalid_operation -andx344 and 1E-1007 18 -> NaN Invalid_operation -andx345 and -1E-1007 -10 -> NaN Invalid_operation -andx346 and -1.00000000E-999 18 -> NaN Invalid_operation -andx347 and -1E-999 10 -> NaN Invalid_operation -andx348 and -9.99999999E+999 -18 -> NaN Invalid_operation - --- A few other non-integers -andx361 and 1.0 1 -> NaN Invalid_operation -andx362 and 1E+1 1 -> NaN Invalid_operation -andx363 and 0.0 1 -> NaN Invalid_operation -andx364 and 0E+1 1 -> NaN Invalid_operation -andx365 and 9.9 1 -> NaN Invalid_operation -andx366 and 9E+1 1 -> NaN Invalid_operation -andx371 and 0 1.0 -> NaN Invalid_operation -andx372 and 0 1E+1 -> NaN Invalid_operation -andx373 and 0 0.0 -> NaN Invalid_operation -andx374 and 0 0E+1 -> NaN Invalid_operation -andx375 and 0 9.9 -> NaN Invalid_operation -andx376 and 0 9E+1 -> NaN Invalid_operation - --- All Specials are in error -andx780 and -Inf -Inf -> NaN Invalid_operation -andx781 and -Inf -1000 -> NaN Invalid_operation -andx782 and -Inf -1 -> NaN Invalid_operation -andx783 and -Inf -0 -> NaN Invalid_operation -andx784 and -Inf 0 -> NaN Invalid_operation -andx785 and -Inf 1 -> NaN Invalid_operation -andx786 and -Inf 1000 -> NaN Invalid_operation -andx787 and -1000 -Inf -> NaN Invalid_operation -andx788 and -Inf -Inf -> NaN Invalid_operation -andx789 and -1 -Inf -> NaN Invalid_operation -andx790 and -0 -Inf -> NaN Invalid_operation -andx791 and 0 -Inf -> NaN Invalid_operation -andx792 and 1 -Inf -> NaN Invalid_operation -andx793 and 1000 -Inf -> NaN Invalid_operation -andx794 and Inf -Inf -> NaN Invalid_operation - -andx800 and Inf -Inf -> NaN Invalid_operation -andx801 and Inf -1000 -> NaN Invalid_operation -andx802 and Inf -1 -> NaN Invalid_operation -andx803 and Inf -0 -> NaN Invalid_operation -andx804 and Inf 0 -> NaN Invalid_operation -andx805 and Inf 1 -> NaN Invalid_operation -andx806 and Inf 1000 -> NaN Invalid_operation -andx807 and Inf Inf -> NaN Invalid_operation -andx808 and -1000 Inf -> NaN Invalid_operation -andx809 and -Inf Inf -> NaN Invalid_operation -andx810 and -1 Inf -> NaN Invalid_operation -andx811 and -0 Inf -> NaN Invalid_operation -andx812 and 0 Inf -> NaN Invalid_operation -andx813 and 1 Inf -> NaN Invalid_operation -andx814 and 1000 Inf -> NaN Invalid_operation -andx815 and Inf Inf -> NaN Invalid_operation - -andx821 and NaN -Inf -> NaN Invalid_operation -andx822 and NaN -1000 -> NaN Invalid_operation -andx823 and NaN -1 -> NaN Invalid_operation -andx824 and NaN -0 -> NaN Invalid_operation -andx825 and NaN 0 -> NaN Invalid_operation -andx826 and NaN 1 -> NaN Invalid_operation -andx827 and NaN 1000 -> NaN Invalid_operation -andx828 and NaN Inf -> NaN Invalid_operation -andx829 and NaN NaN -> NaN Invalid_operation -andx830 and -Inf NaN -> NaN Invalid_operation -andx831 and -1000 NaN -> NaN Invalid_operation -andx832 and -1 NaN -> NaN Invalid_operation -andx833 and -0 NaN -> NaN Invalid_operation -andx834 and 0 NaN -> NaN Invalid_operation -andx835 and 1 NaN -> NaN Invalid_operation -andx836 and 1000 NaN -> NaN Invalid_operation -andx837 and Inf NaN -> NaN Invalid_operation - -andx841 and sNaN -Inf -> NaN Invalid_operation -andx842 and sNaN -1000 -> NaN Invalid_operation -andx843 and sNaN -1 -> NaN Invalid_operation -andx844 and sNaN -0 -> NaN Invalid_operation -andx845 and sNaN 0 -> NaN Invalid_operation -andx846 and sNaN 1 -> NaN Invalid_operation -andx847 and sNaN 1000 -> NaN Invalid_operation -andx848 and sNaN NaN -> NaN Invalid_operation -andx849 and sNaN sNaN -> NaN Invalid_operation -andx850 and NaN sNaN -> NaN Invalid_operation -andx851 and -Inf sNaN -> NaN Invalid_operation -andx852 and -1000 sNaN -> NaN Invalid_operation -andx853 and -1 sNaN -> NaN Invalid_operation -andx854 and -0 sNaN -> NaN Invalid_operation -andx855 and 0 sNaN -> NaN Invalid_operation -andx856 and 1 sNaN -> NaN Invalid_operation -andx857 and 1000 sNaN -> NaN Invalid_operation -andx858 and Inf sNaN -> NaN Invalid_operation -andx859 and NaN sNaN -> NaN Invalid_operation - --- propagating NaNs -andx861 and NaN1 -Inf -> NaN Invalid_operation -andx862 and +NaN2 -1000 -> NaN Invalid_operation -andx863 and NaN3 1000 -> NaN Invalid_operation -andx864 and NaN4 Inf -> NaN Invalid_operation -andx865 and NaN5 +NaN6 -> NaN Invalid_operation -andx866 and -Inf NaN7 -> NaN Invalid_operation -andx867 and -1000 NaN8 -> NaN Invalid_operation -andx868 and 1000 NaN9 -> NaN Invalid_operation -andx869 and Inf +NaN10 -> NaN Invalid_operation -andx871 and sNaN11 -Inf -> NaN Invalid_operation -andx872 and sNaN12 -1000 -> NaN Invalid_operation -andx873 and sNaN13 1000 -> NaN Invalid_operation -andx874 and sNaN14 NaN17 -> NaN Invalid_operation -andx875 and sNaN15 sNaN18 -> NaN Invalid_operation -andx876 and NaN16 sNaN19 -> NaN Invalid_operation -andx877 and -Inf +sNaN20 -> NaN Invalid_operation -andx878 and -1000 sNaN21 -> NaN Invalid_operation -andx879 and 1000 sNaN22 -> NaN Invalid_operation -andx880 and Inf sNaN23 -> NaN Invalid_operation -andx881 and +NaN25 +sNaN24 -> NaN Invalid_operation -andx882 and -NaN26 NaN28 -> NaN Invalid_operation -andx883 and -sNaN27 sNaN29 -> NaN Invalid_operation -andx884 and 1000 -NaN30 -> NaN Invalid_operation -andx885 and 1000 -sNaN31 -> NaN Invalid_operation +------------------------------------------------------------------------ +-- and.decTest -- digitwise logical AND -- +-- Copyright (c) IBM Corporation, 1981, 2008. All rights reserved. -- +------------------------------------------------------------------------ +-- Please see the document "General Decimal Arithmetic Testcases" -- +-- at http://www2.hursley.ibm.com/decimal for the description of -- +-- these testcases. -- +-- -- +-- These testcases are experimental ('beta' versions), and they -- +-- may contain errors. They are offered on an as-is basis. In -- +-- particular, achieving the same results as the tests here is not -- +-- a guarantee that an implementation complies with any Standard -- +-- or specification. The tests are not exhaustive. -- +-- -- +-- Please send comments, suggestions, and corrections to the author: -- +-- Mike Cowlishaw, IBM Fellow -- +-- IBM UK, PO Box 31, Birmingham Road, Warwick CV34 5JL, UK -- +-- mfc at uk.ibm.com -- +------------------------------------------------------------------------ +version: 2.59 + +extended: 1 +precision: 9 +rounding: half_up +maxExponent: 999 +minExponent: -999 + +-- Sanity check (truth table) +andx001 and 0 0 -> 0 +andx002 and 0 1 -> 0 +andx003 and 1 0 -> 0 +andx004 and 1 1 -> 1 +andx005 and 1100 1010 -> 1000 +andx006 and 1111 10 -> 10 +andx007 and 1111 1010 -> 1010 + +-- and at msd and msd-1 +andx010 and 000000000 000000000 -> 0 +andx011 and 000000000 100000000 -> 0 +andx012 and 100000000 000000000 -> 0 +andx013 and 100000000 100000000 -> 100000000 +andx014 and 000000000 000000000 -> 0 +andx015 and 000000000 010000000 -> 0 +andx016 and 010000000 000000000 -> 0 +andx017 and 010000000 010000000 -> 10000000 + +-- Various lengths +-- 123456789 123456789 123456789 +andx021 and 111111111 111111111 -> 111111111 +andx022 and 111111111111 111111111 -> 111111111 +andx023 and 111111111111 11111111 -> 11111111 +andx024 and 111111111 11111111 -> 11111111 +andx025 and 111111111 1111111 -> 1111111 +andx026 and 111111111111 111111 -> 111111 +andx027 and 111111111111 11111 -> 11111 +andx028 and 111111111111 1111 -> 1111 +andx029 and 111111111111 111 -> 111 +andx031 and 111111111111 11 -> 11 +andx032 and 111111111111 1 -> 1 +andx033 and 111111111111 1111111111 -> 111111111 +andx034 and 11111111111 11111111111 -> 111111111 +andx035 and 1111111111 111111111111 -> 111111111 +andx036 and 111111111 1111111111111 -> 111111111 + +andx040 and 111111111 111111111111 -> 111111111 +andx041 and 11111111 111111111111 -> 11111111 +andx042 and 11111111 111111111 -> 11111111 +andx043 and 1111111 111111111 -> 1111111 +andx044 and 111111 111111111 -> 111111 +andx045 and 11111 111111111 -> 11111 +andx046 and 1111 111111111 -> 1111 +andx047 and 111 111111111 -> 111 +andx048 and 11 111111111 -> 11 +andx049 and 1 111111111 -> 1 + +andx050 and 1111111111 1 -> 1 +andx051 and 111111111 1 -> 1 +andx052 and 11111111 1 -> 1 +andx053 and 1111111 1 -> 1 +andx054 and 111111 1 -> 1 +andx055 and 11111 1 -> 1 +andx056 and 1111 1 -> 1 +andx057 and 111 1 -> 1 +andx058 and 11 1 -> 1 +andx059 and 1 1 -> 1 + +andx060 and 1111111111 0 -> 0 +andx061 and 111111111 0 -> 0 +andx062 and 11111111 0 -> 0 +andx063 and 1111111 0 -> 0 +andx064 and 111111 0 -> 0 +andx065 and 11111 0 -> 0 +andx066 and 1111 0 -> 0 +andx067 and 111 0 -> 0 +andx068 and 11 0 -> 0 +andx069 and 1 0 -> 0 + +andx070 and 1 1111111111 -> 1 +andx071 and 1 111111111 -> 1 +andx072 and 1 11111111 -> 1 +andx073 and 1 1111111 -> 1 +andx074 and 1 111111 -> 1 +andx075 and 1 11111 -> 1 +andx076 and 1 1111 -> 1 +andx077 and 1 111 -> 1 +andx078 and 1 11 -> 1 +andx079 and 1 1 -> 1 + +andx080 and 0 1111111111 -> 0 +andx081 and 0 111111111 -> 0 +andx082 and 0 11111111 -> 0 +andx083 and 0 1111111 -> 0 +andx084 and 0 111111 -> 0 +andx085 and 0 11111 -> 0 +andx086 and 0 1111 -> 0 +andx087 and 0 111 -> 0 +andx088 and 0 11 -> 0 +andx089 and 0 1 -> 0 + +andx090 and 011111111 111111111 -> 11111111 +andx091 and 101111111 111111111 -> 101111111 +andx092 and 110111111 111111111 -> 110111111 +andx093 and 111011111 111111111 -> 111011111 +andx094 and 111101111 111111111 -> 111101111 +andx095 and 111110111 111111111 -> 111110111 +andx096 and 111111011 111111111 -> 111111011 +andx097 and 111111101 111111111 -> 111111101 +andx098 and 111111110 111111111 -> 111111110 + +andx100 and 111111111 011111111 -> 11111111 +andx101 and 111111111 101111111 -> 101111111 +andx102 and 111111111 110111111 -> 110111111 +andx103 and 111111111 111011111 -> 111011111 +andx104 and 111111111 111101111 -> 111101111 +andx105 and 111111111 111110111 -> 111110111 +andx106 and 111111111 111111011 -> 111111011 +andx107 and 111111111 111111101 -> 111111101 +andx108 and 111111111 111111110 -> 111111110 + +-- non-0/1 should not be accepted, nor should signs +andx220 and 111111112 111111111 -> NaN Invalid_operation +andx221 and 333333333 333333333 -> NaN Invalid_operation +andx222 and 555555555 555555555 -> NaN Invalid_operation +andx223 and 777777777 777777777 -> NaN Invalid_operation +andx224 and 999999999 999999999 -> NaN Invalid_operation +andx225 and 222222222 999999999 -> NaN Invalid_operation +andx226 and 444444444 999999999 -> NaN Invalid_operation +andx227 and 666666666 999999999 -> NaN Invalid_operation +andx228 and 888888888 999999999 -> NaN Invalid_operation +andx229 and 999999999 222222222 -> NaN Invalid_operation +andx230 and 999999999 444444444 -> NaN Invalid_operation +andx231 and 999999999 666666666 -> NaN Invalid_operation +andx232 and 999999999 888888888 -> NaN Invalid_operation +-- a few randoms +andx240 and 567468689 -934981942 -> NaN Invalid_operation +andx241 and 567367689 934981942 -> NaN Invalid_operation +andx242 and -631917772 -706014634 -> NaN Invalid_operation +andx243 and -756253257 138579234 -> NaN Invalid_operation +andx244 and 835590149 567435400 -> NaN Invalid_operation +-- test MSD +andx250 and 200000000 100000000 -> NaN Invalid_operation +andx251 and 700000000 100000000 -> NaN Invalid_operation +andx252 and 800000000 100000000 -> NaN Invalid_operation +andx253 and 900000000 100000000 -> NaN Invalid_operation +andx254 and 200000000 000000000 -> NaN Invalid_operation +andx255 and 700000000 000000000 -> NaN Invalid_operation +andx256 and 800000000 000000000 -> NaN Invalid_operation +andx257 and 900000000 000000000 -> NaN Invalid_operation +andx258 and 100000000 200000000 -> NaN Invalid_operation +andx259 and 100000000 700000000 -> NaN Invalid_operation +andx260 and 100000000 800000000 -> NaN Invalid_operation +andx261 and 100000000 900000000 -> NaN Invalid_operation +andx262 and 000000000 200000000 -> NaN Invalid_operation +andx263 and 000000000 700000000 -> NaN Invalid_operation +andx264 and 000000000 800000000 -> NaN Invalid_operation +andx265 and 000000000 900000000 -> NaN Invalid_operation +-- test MSD-1 +andx270 and 020000000 100000000 -> NaN Invalid_operation +andx271 and 070100000 100000000 -> NaN Invalid_operation +andx272 and 080010000 100000001 -> NaN Invalid_operation +andx273 and 090001000 100000010 -> NaN Invalid_operation +andx274 and 100000100 020010100 -> NaN Invalid_operation +andx275 and 100000000 070001000 -> NaN Invalid_operation +andx276 and 100000010 080010100 -> NaN Invalid_operation +andx277 and 100000000 090000010 -> NaN Invalid_operation +-- test LSD +andx280 and 001000002 100000000 -> NaN Invalid_operation +andx281 and 000000007 100000000 -> NaN Invalid_operation +andx282 and 000000008 100000000 -> NaN Invalid_operation +andx283 and 000000009 100000000 -> NaN Invalid_operation +andx284 and 100000000 000100002 -> NaN Invalid_operation +andx285 and 100100000 001000007 -> NaN Invalid_operation +andx286 and 100010000 010000008 -> NaN Invalid_operation +andx287 and 100001000 100000009 -> NaN Invalid_operation +-- test Middie +andx288 and 001020000 100000000 -> NaN Invalid_operation +andx289 and 000070001 100000000 -> NaN Invalid_operation +andx290 and 000080000 100010000 -> NaN Invalid_operation +andx291 and 000090000 100001000 -> NaN Invalid_operation +andx292 and 100000010 000020100 -> NaN Invalid_operation +andx293 and 100100000 000070010 -> NaN Invalid_operation +andx294 and 100010100 000080001 -> NaN Invalid_operation +andx295 and 100001000 000090000 -> NaN Invalid_operation +-- signs +andx296 and -100001000 -000000000 -> NaN Invalid_operation +andx297 and -100001000 000010000 -> NaN Invalid_operation +andx298 and 100001000 -000000000 -> NaN Invalid_operation +andx299 and 100001000 000011000 -> 1000 + +-- Nmax, Nmin, Ntiny +andx331 and 2 9.99999999E+999 -> NaN Invalid_operation +andx332 and 3 1E-999 -> NaN Invalid_operation +andx333 and 4 1.00000000E-999 -> NaN Invalid_operation +andx334 and 5 1E-1007 -> NaN Invalid_operation +andx335 and 6 -1E-1007 -> NaN Invalid_operation +andx336 and 7 -1.00000000E-999 -> NaN Invalid_operation +andx337 and 8 -1E-999 -> NaN Invalid_operation +andx338 and 9 -9.99999999E+999 -> NaN Invalid_operation +andx341 and 9.99999999E+999 -18 -> NaN Invalid_operation +andx342 and 1E-999 01 -> NaN Invalid_operation +andx343 and 1.00000000E-999 -18 -> NaN Invalid_operation +andx344 and 1E-1007 18 -> NaN Invalid_operation +andx345 and -1E-1007 -10 -> NaN Invalid_operation +andx346 and -1.00000000E-999 18 -> NaN Invalid_operation +andx347 and -1E-999 10 -> NaN Invalid_operation +andx348 and -9.99999999E+999 -18 -> NaN Invalid_operation + +-- A few other non-integers +andx361 and 1.0 1 -> NaN Invalid_operation +andx362 and 1E+1 1 -> NaN Invalid_operation +andx363 and 0.0 1 -> NaN Invalid_operation +andx364 and 0E+1 1 -> NaN Invalid_operation +andx365 and 9.9 1 -> NaN Invalid_operation +andx366 and 9E+1 1 -> NaN Invalid_operation +andx371 and 0 1.0 -> NaN Invalid_operation +andx372 and 0 1E+1 -> NaN Invalid_operation +andx373 and 0 0.0 -> NaN Invalid_operation +andx374 and 0 0E+1 -> NaN Invalid_operation +andx375 and 0 9.9 -> NaN Invalid_operation +andx376 and 0 9E+1 -> NaN Invalid_operation + +-- All Specials are in error +andx780 and -Inf -Inf -> NaN Invalid_operation +andx781 and -Inf -1000 -> NaN Invalid_operation +andx782 and -Inf -1 -> NaN Invalid_operation +andx783 and -Inf -0 -> NaN Invalid_operation +andx784 and -Inf 0 -> NaN Invalid_operation +andx785 and -Inf 1 -> NaN Invalid_operation +andx786 and -Inf 1000 -> NaN Invalid_operation +andx787 and -1000 -Inf -> NaN Invalid_operation +andx788 and -Inf -Inf -> NaN Invalid_operation +andx789 and -1 -Inf -> NaN Invalid_operation +andx790 and -0 -Inf -> NaN Invalid_operation +andx791 and 0 -Inf -> NaN Invalid_operation +andx792 and 1 -Inf -> NaN Invalid_operation +andx793 and 1000 -Inf -> NaN Invalid_operation +andx794 and Inf -Inf -> NaN Invalid_operation + +andx800 and Inf -Inf -> NaN Invalid_operation +andx801 and Inf -1000 -> NaN Invalid_operation +andx802 and Inf -1 -> NaN Invalid_operation +andx803 and Inf -0 -> NaN Invalid_operation +andx804 and Inf 0 -> NaN Invalid_operation +andx805 and Inf 1 -> NaN Invalid_operation +andx806 and Inf 1000 -> NaN Invalid_operation +andx807 and Inf Inf -> NaN Invalid_operation +andx808 and -1000 Inf -> NaN Invalid_operation +andx809 and -Inf Inf -> NaN Invalid_operation +andx810 and -1 Inf -> NaN Invalid_operation +andx811 and -0 Inf -> NaN Invalid_operation +andx812 and 0 Inf -> NaN Invalid_operation +andx813 and 1 Inf -> NaN Invalid_operation +andx814 and 1000 Inf -> NaN Invalid_operation +andx815 and Inf Inf -> NaN Invalid_operation + +andx821 and NaN -Inf -> NaN Invalid_operation +andx822 and NaN -1000 -> NaN Invalid_operation +andx823 and NaN -1 -> NaN Invalid_operation +andx824 and NaN -0 -> NaN Invalid_operation +andx825 and NaN 0 -> NaN Invalid_operation +andx826 and NaN 1 -> NaN Invalid_operation +andx827 and NaN 1000 -> NaN Invalid_operation +andx828 and NaN Inf -> NaN Invalid_operation +andx829 and NaN NaN -> NaN Invalid_operation +andx830 and -Inf NaN -> NaN Invalid_operation +andx831 and -1000 NaN -> NaN Invalid_operation +andx832 and -1 NaN -> NaN Invalid_operation +andx833 and -0 NaN -> NaN Invalid_operation +andx834 and 0 NaN -> NaN Invalid_operation +andx835 and 1 NaN -> NaN Invalid_operation +andx836 and 1000 NaN -> NaN Invalid_operation +andx837 and Inf NaN -> NaN Invalid_operation + +andx841 and sNaN -Inf -> NaN Invalid_operation +andx842 and sNaN -1000 -> NaN Invalid_operation +andx843 and sNaN -1 -> NaN Invalid_operation +andx844 and sNaN -0 -> NaN Invalid_operation +andx845 and sNaN 0 -> NaN Invalid_operation +andx846 and sNaN 1 -> NaN Invalid_operation +andx847 and sNaN 1000 -> NaN Invalid_operation +andx848 and sNaN NaN -> NaN Invalid_operation +andx849 and sNaN sNaN -> NaN Invalid_operation +andx850 and NaN sNaN -> NaN Invalid_operation +andx851 and -Inf sNaN -> NaN Invalid_operation +andx852 and -1000 sNaN -> NaN Invalid_operation +andx853 and -1 sNaN -> NaN Invalid_operation +andx854 and -0 sNaN -> NaN Invalid_operation +andx855 and 0 sNaN -> NaN Invalid_operation +andx856 and 1 sNaN -> NaN Invalid_operation +andx857 and 1000 sNaN -> NaN Invalid_operation +andx858 and Inf sNaN -> NaN Invalid_operation +andx859 and NaN sNaN -> NaN Invalid_operation + +-- propagating NaNs +andx861 and NaN1 -Inf -> NaN Invalid_operation +andx862 and +NaN2 -1000 -> NaN Invalid_operation +andx863 and NaN3 1000 -> NaN Invalid_operation +andx864 and NaN4 Inf -> NaN Invalid_operation +andx865 and NaN5 +NaN6 -> NaN Invalid_operation +andx866 and -Inf NaN7 -> NaN Invalid_operation +andx867 and -1000 NaN8 -> NaN Invalid_operation +andx868 and 1000 NaN9 -> NaN Invalid_operation +andx869 and Inf +NaN10 -> NaN Invalid_operation +andx871 and sNaN11 -Inf -> NaN Invalid_operation +andx872 and sNaN12 -1000 -> NaN Invalid_operation +andx873 and sNaN13 1000 -> NaN Invalid_operation +andx874 and sNaN14 NaN17 -> NaN Invalid_operation +andx875 and sNaN15 sNaN18 -> NaN Invalid_operation +andx876 and NaN16 sNaN19 -> NaN Invalid_operation +andx877 and -Inf +sNaN20 -> NaN Invalid_operation +andx878 and -1000 sNaN21 -> NaN Invalid_operation +andx879 and 1000 sNaN22 -> NaN Invalid_operation +andx880 and Inf sNaN23 -> NaN Invalid_operation +andx881 and +NaN25 +sNaN24 -> NaN Invalid_operation +andx882 and -NaN26 NaN28 -> NaN Invalid_operation +andx883 and -sNaN27 sNaN29 -> NaN Invalid_operation +andx884 and 1000 -NaN30 -> NaN Invalid_operation +andx885 and 1000 -sNaN31 -> NaN Invalid_operation diff --git a/lib-python/2.7/test/decimaltestdata/class.decTest b/lib-python/2.7/test/decimaltestdata/class.decTest --- a/lib-python/2.7/test/decimaltestdata/class.decTest +++ b/lib-python/2.7/test/decimaltestdata/class.decTest @@ -1,131 +1,131 @@ ------------------------------------------------------------------------- --- class.decTest -- Class operations -- --- Copyright (c) IBM Corporation, 1981, 2008. All rights reserved. -- ------------------------------------------------------------------------- --- Please see the document "General Decimal Arithmetic Testcases" -- --- at http://www2.hursley.ibm.com/decimal for the description of -- --- these testcases. -- --- -- --- These testcases are experimental ('beta' versions), and they -- --- may contain errors. They are offered on an as-is basis. In -- --- particular, achieving the same results as the tests here is not -- --- a guarantee that an implementation complies with any Standard -- --- or specification. The tests are not exhaustive. -- --- -- --- Please send comments, suggestions, and corrections to the author: -- --- Mike Cowlishaw, IBM Fellow -- --- IBM UK, PO Box 31, Birmingham Road, Warwick CV34 5JL, UK -- --- mfc at uk.ibm.com -- ------------------------------------------------------------------------- -version: 2.59 - --- [New 2006.11.27] - -precision: 9 -maxExponent: 999 -minExponent: -999 -extended: 1 -clamp: 1 -rounding: half_even - -clasx001 class 0 -> +Zero -clasx002 class 0.00 -> +Zero -clasx003 class 0E+5 -> +Zero -clasx004 class 1E-1007 -> +Subnormal -clasx005 class 0.1E-999 -> +Subnormal -clasx006 class 0.99999999E-999 -> +Subnormal -clasx007 class 1.00000000E-999 -> +Normal -clasx008 class 1E-999 -> +Normal -clasx009 class 1E-100 -> +Normal -clasx010 class 1E-10 -> +Normal -clasx012 class 1E-1 -> +Normal -clasx013 class 1 -> +Normal -clasx014 class 2.50 -> +Normal -clasx015 class 100.100 -> +Normal -clasx016 class 1E+30 -> +Normal -clasx017 class 1E+999 -> +Normal -clasx018 class 9.99999999E+999 -> +Normal -clasx019 class Inf -> +Infinity - -clasx021 class -0 -> -Zero -clasx022 class -0.00 -> -Zero -clasx023 class -0E+5 -> -Zero -clasx024 class -1E-1007 -> -Subnormal -clasx025 class -0.1E-999 -> -Subnormal -clasx026 class -0.99999999E-999 -> -Subnormal -clasx027 class -1.00000000E-999 -> -Normal -clasx028 class -1E-999 -> -Normal -clasx029 class -1E-100 -> -Normal -clasx030 class -1E-10 -> -Normal -clasx032 class -1E-1 -> -Normal -clasx033 class -1 -> -Normal -clasx034 class -2.50 -> -Normal -clasx035 class -100.100 -> -Normal -clasx036 class -1E+30 -> -Normal -clasx037 class -1E+999 -> -Normal -clasx038 class -9.99999999E+999 -> -Normal -clasx039 class -Inf -> -Infinity - -clasx041 class NaN -> NaN -clasx042 class -NaN -> NaN -clasx043 class +NaN12345 -> NaN -clasx044 class sNaN -> sNaN -clasx045 class -sNaN -> sNaN -clasx046 class +sNaN12345 -> sNaN - - --- decimal64 bounds - -precision: 16 -maxExponent: 384 -minExponent: -383 -clamp: 1 -rounding: half_even - -clasx201 class 0 -> +Zero -clasx202 class 0.00 -> +Zero -clasx203 class 0E+5 -> +Zero -clasx204 class 1E-396 -> +Subnormal -clasx205 class 0.1E-383 -> +Subnormal -clasx206 class 0.999999999999999E-383 -> +Subnormal -clasx207 class 1.000000000000000E-383 -> +Normal -clasx208 class 1E-383 -> +Normal -clasx209 class 1E-100 -> +Normal -clasx210 class 1E-10 -> +Normal -clasx212 class 1E-1 -> +Normal -clasx213 class 1 -> +Normal -clasx214 class 2.50 -> +Normal -clasx215 class 100.100 -> +Normal -clasx216 class 1E+30 -> +Normal -clasx217 class 1E+384 -> +Normal -clasx218 class 9.999999999999999E+384 -> +Normal -clasx219 class Inf -> +Infinity - -clasx221 class -0 -> -Zero -clasx222 class -0.00 -> -Zero -clasx223 class -0E+5 -> -Zero -clasx224 class -1E-396 -> -Subnormal -clasx225 class -0.1E-383 -> -Subnormal -clasx226 class -0.999999999999999E-383 -> -Subnormal -clasx227 class -1.000000000000000E-383 -> -Normal -clasx228 class -1E-383 -> -Normal -clasx229 class -1E-100 -> -Normal -clasx230 class -1E-10 -> -Normal -clasx232 class -1E-1 -> -Normal -clasx233 class -1 -> -Normal -clasx234 class -2.50 -> -Normal -clasx235 class -100.100 -> -Normal -clasx236 class -1E+30 -> -Normal -clasx237 class -1E+384 -> -Normal -clasx238 class -9.999999999999999E+384 -> -Normal -clasx239 class -Inf -> -Infinity - -clasx241 class NaN -> NaN -clasx242 class -NaN -> NaN -clasx243 class +NaN12345 -> NaN -clasx244 class sNaN -> sNaN -clasx245 class -sNaN -> sNaN -clasx246 class +sNaN12345 -> sNaN - - - +------------------------------------------------------------------------ +-- class.decTest -- Class operations -- +-- Copyright (c) IBM Corporation, 1981, 2008. All rights reserved. -- +------------------------------------------------------------------------ +-- Please see the document "General Decimal Arithmetic Testcases" -- +-- at http://www2.hursley.ibm.com/decimal for the description of -- +-- these testcases. -- +-- -- +-- These testcases are experimental ('beta' versions), and they -- +-- may contain errors. They are offered on an as-is basis. In -- +-- particular, achieving the same results as the tests here is not -- +-- a guarantee that an implementation complies with any Standard -- +-- or specification. The tests are not exhaustive. -- +-- -- +-- Please send comments, suggestions, and corrections to the author: -- +-- Mike Cowlishaw, IBM Fellow -- +-- IBM UK, PO Box 31, Birmingham Road, Warwick CV34 5JL, UK -- +-- mfc at uk.ibm.com -- +------------------------------------------------------------------------ +version: 2.59 + +-- [New 2006.11.27] + +precision: 9 +maxExponent: 999 +minExponent: -999 +extended: 1 +clamp: 1 +rounding: half_even + +clasx001 class 0 -> +Zero +clasx002 class 0.00 -> +Zero +clasx003 class 0E+5 -> +Zero +clasx004 class 1E-1007 -> +Subnormal +clasx005 class 0.1E-999 -> +Subnormal +clasx006 class 0.99999999E-999 -> +Subnormal +clasx007 class 1.00000000E-999 -> +Normal +clasx008 class 1E-999 -> +Normal +clasx009 class 1E-100 -> +Normal +clasx010 class 1E-10 -> +Normal +clasx012 class 1E-1 -> +Normal +clasx013 class 1 -> +Normal +clasx014 class 2.50 -> +Normal +clasx015 class 100.100 -> +Normal +clasx016 class 1E+30 -> +Normal +clasx017 class 1E+999 -> +Normal +clasx018 class 9.99999999E+999 -> +Normal +clasx019 class Inf -> +Infinity + +clasx021 class -0 -> -Zero +clasx022 class -0.00 -> -Zero +clasx023 class -0E+5 -> -Zero +clasx024 class -1E-1007 -> -Subnormal +clasx025 class -0.1E-999 -> -Subnormal +clasx026 class -0.99999999E-999 -> -Subnormal +clasx027 class -1.00000000E-999 -> -Normal +clasx028 class -1E-999 -> -Normal +clasx029 class -1E-100 -> -Normal +clasx030 class -1E-10 -> -Normal +clasx032 class -1E-1 -> -Normal +clasx033 class -1 -> -Normal +clasx034 class -2.50 -> -Normal +clasx035 class -100.100 -> -Normal +clasx036 class -1E+30 -> -Normal +clasx037 class -1E+999 -> -Normal +clasx038 class -9.99999999E+999 -> -Normal +clasx039 class -Inf -> -Infinity + +clasx041 class NaN -> NaN +clasx042 class -NaN -> NaN +clasx043 class +NaN12345 -> NaN +clasx044 class sNaN -> sNaN +clasx045 class -sNaN -> sNaN +clasx046 class +sNaN12345 -> sNaN + + +-- decimal64 bounds + +precision: 16 +maxExponent: 384 +minExponent: -383 +clamp: 1 +rounding: half_even + +clasx201 class 0 -> +Zero +clasx202 class 0.00 -> +Zero +clasx203 class 0E+5 -> +Zero +clasx204 class 1E-396 -> +Subnormal +clasx205 class 0.1E-383 -> +Subnormal +clasx206 class 0.999999999999999E-383 -> +Subnormal +clasx207 class 1.000000000000000E-383 -> +Normal +clasx208 class 1E-383 -> +Normal +clasx209 class 1E-100 -> +Normal +clasx210 class 1E-10 -> +Normal +clasx212 class 1E-1 -> +Normal +clasx213 class 1 -> +Normal +clasx214 class 2.50 -> +Normal +clasx215 class 100.100 -> +Normal +clasx216 class 1E+30 -> +Normal +clasx217 class 1E+384 -> +Normal +clasx218 class 9.999999999999999E+384 -> +Normal +clasx219 class Inf -> +Infinity + +clasx221 class -0 -> -Zero +clasx222 class -0.00 -> -Zero +clasx223 class -0E+5 -> -Zero +clasx224 class -1E-396 -> -Subnormal +clasx225 class -0.1E-383 -> -Subnormal +clasx226 class -0.999999999999999E-383 -> -Subnormal +clasx227 class -1.000000000000000E-383 -> -Normal +clasx228 class -1E-383 -> -Normal +clasx229 class -1E-100 -> -Normal +clasx230 class -1E-10 -> -Normal +clasx232 class -1E-1 -> -Normal +clasx233 class -1 -> -Normal +clasx234 class -2.50 -> -Normal +clasx235 class -100.100 -> -Normal +clasx236 class -1E+30 -> -Normal +clasx237 class -1E+384 -> -Normal +clasx238 class -9.999999999999999E+384 -> -Normal +clasx239 class -Inf -> -Infinity + +clasx241 class NaN -> NaN +clasx242 class -NaN -> NaN +clasx243 class +NaN12345 -> NaN +clasx244 class sNaN -> sNaN +clasx245 class -sNaN -> sNaN +clasx246 class +sNaN12345 -> sNaN + + + diff --git a/lib-python/2.7/test/decimaltestdata/comparetotal.decTest b/lib-python/2.7/test/decimaltestdata/comparetotal.decTest --- a/lib-python/2.7/test/decimaltestdata/comparetotal.decTest +++ b/lib-python/2.7/test/decimaltestdata/comparetotal.decTest @@ -1,798 +1,798 @@ ------------------------------------------------------------------------- --- comparetotal.decTest -- decimal comparison using total ordering -- --- Copyright (c) IBM Corporation, 1981, 2008. All rights reserved. -- ------------------------------------------------------------------------- --- Please see the document "General Decimal Arithmetic Testcases" -- --- at http://www2.hursley.ibm.com/decimal for the description of -- --- these testcases. -- --- -- --- These testcases are experimental ('beta' versions), and they -- --- may contain errors. They are offered on an as-is basis. In -- --- particular, achieving the same results as the tests here is not -- --- a guarantee that an implementation complies with any Standard -- --- or specification. The tests are not exhaustive. -- --- -- --- Please send comments, suggestions, and corrections to the author: -- --- Mike Cowlishaw, IBM Fellow -- --- IBM UK, PO Box 31, Birmingham Road, Warwick CV34 5JL, UK -- --- mfc at uk.ibm.com -- ------------------------------------------------------------------------- -version: 2.59 - --- Note that we cannot assume add/subtract tests cover paths adequately, --- here, because the code might be quite different (comparison cannot --- overflow or underflow, so actual subtractions are not necessary). --- Similarly, comparetotal will have some radically different paths --- than compare. - -extended: 1 -precision: 16 -rounding: half_up -maxExponent: 384 -minExponent: -383 - --- sanity checks -cotx001 comparetotal -2 -2 -> 0 -cotx002 comparetotal -2 -1 -> -1 -cotx003 comparetotal -2 0 -> -1 -cotx004 comparetotal -2 1 -> -1 -cotx005 comparetotal -2 2 -> -1 -cotx006 comparetotal -1 -2 -> 1 -cotx007 comparetotal -1 -1 -> 0 -cotx008 comparetotal -1 0 -> -1 -cotx009 comparetotal -1 1 -> -1 -cotx010 comparetotal -1 2 -> -1 -cotx011 comparetotal 0 -2 -> 1 -cotx012 comparetotal 0 -1 -> 1 -cotx013 comparetotal 0 0 -> 0 -cotx014 comparetotal 0 1 -> -1 -cotx015 comparetotal 0 2 -> -1 -cotx016 comparetotal 1 -2 -> 1 -cotx017 comparetotal 1 -1 -> 1 -cotx018 comparetotal 1 0 -> 1 -cotx019 comparetotal 1 1 -> 0 -cotx020 comparetotal 1 2 -> -1 -cotx021 comparetotal 2 -2 -> 1 -cotx022 comparetotal 2 -1 -> 1 -cotx023 comparetotal 2 0 -> 1 -cotx025 comparetotal 2 1 -> 1 -cotx026 comparetotal 2 2 -> 0 - -cotx031 comparetotal -20 -20 -> 0 -cotx032 comparetotal -20 -10 -> -1 -cotx033 comparetotal -20 00 -> -1 -cotx034 comparetotal -20 10 -> -1 -cotx035 comparetotal -20 20 -> -1 -cotx036 comparetotal -10 -20 -> 1 -cotx037 comparetotal -10 -10 -> 0 -cotx038 comparetotal -10 00 -> -1 -cotx039 comparetotal -10 10 -> -1 -cotx040 comparetotal -10 20 -> -1 -cotx041 comparetotal 00 -20 -> 1 -cotx042 comparetotal 00 -10 -> 1 -cotx043 comparetotal 00 00 -> 0 -cotx044 comparetotal 00 10 -> -1 -cotx045 comparetotal 00 20 -> -1 -cotx046 comparetotal 10 -20 -> 1 -cotx047 comparetotal 10 -10 -> 1 -cotx048 comparetotal 10 00 -> 1 -cotx049 comparetotal 10 10 -> 0 -cotx050 comparetotal 10 20 -> -1 -cotx051 comparetotal 20 -20 -> 1 -cotx052 comparetotal 20 -10 -> 1 -cotx053 comparetotal 20 00 -> 1 -cotx055 comparetotal 20 10 -> 1 -cotx056 comparetotal 20 20 -> 0 - -cotx061 comparetotal -2.0 -2.0 -> 0 -cotx062 comparetotal -2.0 -1.0 -> -1 -cotx063 comparetotal -2.0 0.0 -> -1 -cotx064 comparetotal -2.0 1.0 -> -1 -cotx065 comparetotal -2.0 2.0 -> -1 -cotx066 comparetotal -1.0 -2.0 -> 1 -cotx067 comparetotal -1.0 -1.0 -> 0 -cotx068 comparetotal -1.0 0.0 -> -1 -cotx069 comparetotal -1.0 1.0 -> -1 -cotx070 comparetotal -1.0 2.0 -> -1 -cotx071 comparetotal 0.0 -2.0 -> 1 -cotx072 comparetotal 0.0 -1.0 -> 1 -cotx073 comparetotal 0.0 0.0 -> 0 -cotx074 comparetotal 0.0 1.0 -> -1 -cotx075 comparetotal 0.0 2.0 -> -1 -cotx076 comparetotal 1.0 -2.0 -> 1 -cotx077 comparetotal 1.0 -1.0 -> 1 -cotx078 comparetotal 1.0 0.0 -> 1 -cotx079 comparetotal 1.0 1.0 -> 0 -cotx080 comparetotal 1.0 2.0 -> -1 -cotx081 comparetotal 2.0 -2.0 -> 1 -cotx082 comparetotal 2.0 -1.0 -> 1 -cotx083 comparetotal 2.0 0.0 -> 1 -cotx085 comparetotal 2.0 1.0 -> 1 -cotx086 comparetotal 2.0 2.0 -> 0 - --- now some cases which might overflow if subtract were used -maxexponent: 999999999 -minexponent: -999999999 -cotx090 comparetotal 9.99999999E+999999999 9.99999999E+999999999 -> 0 -cotx091 comparetotal -9.99999999E+999999999 9.99999999E+999999999 -> -1 -cotx092 comparetotal 9.99999999E+999999999 -9.99999999E+999999999 -> 1 -cotx093 comparetotal -9.99999999E+999999999 -9.99999999E+999999999 -> 0 - --- Examples -cotx094 comparetotal 12.73 127.9 -> -1 -cotx095 comparetotal -127 12 -> -1 -cotx096 comparetotal 12.30 12.3 -> -1 -cotx097 comparetotal 12.30 12.30 -> 0 -cotx098 comparetotal 12.3 12.300 -> 1 -cotx099 comparetotal 12.3 NaN -> -1 - --- some differing length/exponent cases --- in this first group, compare would compare all equal -cotx100 comparetotal 7.0 7.0 -> 0 -cotx101 comparetotal 7.0 7 -> -1 -cotx102 comparetotal 7 7.0 -> 1 -cotx103 comparetotal 7E+0 7.0 -> 1 -cotx104 comparetotal 70E-1 7.0 -> 0 -cotx105 comparetotal 0.7E+1 7 -> 0 -cotx106 comparetotal 70E-1 7 -> -1 -cotx107 comparetotal 7.0 7E+0 -> -1 -cotx108 comparetotal 7.0 70E-1 -> 0 -cotx109 comparetotal 7 0.7E+1 -> 0 -cotx110 comparetotal 7 70E-1 -> 1 - -cotx120 comparetotal 8.0 7.0 -> 1 -cotx121 comparetotal 8.0 7 -> 1 -cotx122 comparetotal 8 7.0 -> 1 -cotx123 comparetotal 8E+0 7.0 -> 1 -cotx124 comparetotal 80E-1 7.0 -> 1 -cotx125 comparetotal 0.8E+1 7 -> 1 -cotx126 comparetotal 80E-1 7 -> 1 -cotx127 comparetotal 8.0 7E+0 -> 1 -cotx128 comparetotal 8.0 70E-1 -> 1 -cotx129 comparetotal 8 0.7E+1 -> 1 -cotx130 comparetotal 8 70E-1 -> 1 - -cotx140 comparetotal 8.0 9.0 -> -1 -cotx141 comparetotal 8.0 9 -> -1 -cotx142 comparetotal 8 9.0 -> -1 -cotx143 comparetotal 8E+0 9.0 -> -1 -cotx144 comparetotal 80E-1 9.0 -> -1 -cotx145 comparetotal 0.8E+1 9 -> -1 -cotx146 comparetotal 80E-1 9 -> -1 -cotx147 comparetotal 8.0 9E+0 -> -1 -cotx148 comparetotal 8.0 90E-1 -> -1 -cotx149 comparetotal 8 0.9E+1 -> -1 -cotx150 comparetotal 8 90E-1 -> -1 - --- and again, with sign changes -+ .. -cotx200 comparetotal -7.0 7.0 -> -1 -cotx201 comparetotal -7.0 7 -> -1 -cotx202 comparetotal -7 7.0 -> -1 -cotx203 comparetotal -7E+0 7.0 -> -1 -cotx204 comparetotal -70E-1 7.0 -> -1 -cotx205 comparetotal -0.7E+1 7 -> -1 -cotx206 comparetotal -70E-1 7 -> -1 -cotx207 comparetotal -7.0 7E+0 -> -1 -cotx208 comparetotal -7.0 70E-1 -> -1 -cotx209 comparetotal -7 0.7E+1 -> -1 -cotx210 comparetotal -7 70E-1 -> -1 - -cotx220 comparetotal -8.0 7.0 -> -1 -cotx221 comparetotal -8.0 7 -> -1 -cotx222 comparetotal -8 7.0 -> -1 -cotx223 comparetotal -8E+0 7.0 -> -1 -cotx224 comparetotal -80E-1 7.0 -> -1 -cotx225 comparetotal -0.8E+1 7 -> -1 -cotx226 comparetotal -80E-1 7 -> -1 -cotx227 comparetotal -8.0 7E+0 -> -1 -cotx228 comparetotal -8.0 70E-1 -> -1 -cotx229 comparetotal -8 0.7E+1 -> -1 -cotx230 comparetotal -8 70E-1 -> -1 - -cotx240 comparetotal -8.0 9.0 -> -1 -cotx241 comparetotal -8.0 9 -> -1 -cotx242 comparetotal -8 9.0 -> -1 -cotx243 comparetotal -8E+0 9.0 -> -1 -cotx244 comparetotal -80E-1 9.0 -> -1 -cotx245 comparetotal -0.8E+1 9 -> -1 -cotx246 comparetotal -80E-1 9 -> -1 -cotx247 comparetotal -8.0 9E+0 -> -1 -cotx248 comparetotal -8.0 90E-1 -> -1 -cotx249 comparetotal -8 0.9E+1 -> -1 -cotx250 comparetotal -8 90E-1 -> -1 - --- and again, with sign changes +- .. -cotx300 comparetotal 7.0 -7.0 -> 1 -cotx301 comparetotal 7.0 -7 -> 1 -cotx302 comparetotal 7 -7.0 -> 1 -cotx303 comparetotal 7E+0 -7.0 -> 1 -cotx304 comparetotal 70E-1 -7.0 -> 1 -cotx305 comparetotal .7E+1 -7 -> 1 -cotx306 comparetotal 70E-1 -7 -> 1 -cotx307 comparetotal 7.0 -7E+0 -> 1 -cotx308 comparetotal 7.0 -70E-1 -> 1 -cotx309 comparetotal 7 -.7E+1 -> 1 -cotx310 comparetotal 7 -70E-1 -> 1 - -cotx320 comparetotal 8.0 -7.0 -> 1 -cotx321 comparetotal 8.0 -7 -> 1 -cotx322 comparetotal 8 -7.0 -> 1 -cotx323 comparetotal 8E+0 -7.0 -> 1 -cotx324 comparetotal 80E-1 -7.0 -> 1 -cotx325 comparetotal .8E+1 -7 -> 1 -cotx326 comparetotal 80E-1 -7 -> 1 -cotx327 comparetotal 8.0 -7E+0 -> 1 -cotx328 comparetotal 8.0 -70E-1 -> 1 -cotx329 comparetotal 8 -.7E+1 -> 1 -cotx330 comparetotal 8 -70E-1 -> 1 - -cotx340 comparetotal 8.0 -9.0 -> 1 -cotx341 comparetotal 8.0 -9 -> 1 -cotx342 comparetotal 8 -9.0 -> 1 -cotx343 comparetotal 8E+0 -9.0 -> 1 -cotx344 comparetotal 80E-1 -9.0 -> 1 -cotx345 comparetotal .8E+1 -9 -> 1 -cotx346 comparetotal 80E-1 -9 -> 1 -cotx347 comparetotal 8.0 -9E+0 -> 1 -cotx348 comparetotal 8.0 -90E-1 -> 1 -cotx349 comparetotal 8 -.9E+1 -> 1 -cotx350 comparetotal 8 -90E-1 -> 1 - --- and again, with sign changes -- .. -cotx400 comparetotal -7.0 -7.0 -> 0 -cotx401 comparetotal -7.0 -7 -> 1 -cotx402 comparetotal -7 -7.0 -> -1 -cotx403 comparetotal -7E+0 -7.0 -> -1 -cotx404 comparetotal -70E-1 -7.0 -> 0 -cotx405 comparetotal -.7E+1 -7 -> 0 -cotx406 comparetotal -70E-1 -7 -> 1 -cotx407 comparetotal -7.0 -7E+0 -> 1 -cotx408 comparetotal -7.0 -70E-1 -> 0 -cotx409 comparetotal -7 -.7E+1 -> 0 From noreply at buildbot.pypy.org Tue Jan 31 16:07:17 2012 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 31 Jan 2012 16:07:17 +0100 (CET) Subject: [pypy-commit] pypy stm: Add a test and, for now, fix the emulation to match the real code's Message-ID: <20120131150717.107507106F3@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: stm Changeset: r51992:2cbc49a8a169 Date: 2012-01-31 16:06 +0100 http://bitbucket.org/pypy/pypy/changeset/2cbc49a8a169/ Log: Add a test and, for now, fix the emulation to match the real code's behavior. Maybe at some point we want instead the opposite behavior, but it's unclear. diff --git a/lib_pypy/transaction.py b/lib_pypy/transaction.py --- a/lib_pypy/transaction.py +++ b/lib_pypy/transaction.py @@ -30,6 +30,9 @@ def run(): pending = _pending - while pending: - _, (f, args) = pending.popitem() - f(*args) + try: + while pending: + _, (f, args) = pending.popitem() + f(*args) + finally: + pending.clear() # this is the behavior we get with interp_transaction diff --git a/pypy/module/transaction/test/test_transaction.py b/pypy/module/transaction/test/test_transaction.py --- a/pypy/module/transaction/test/test_transaction.py +++ b/pypy/module/transaction/test/test_transaction.py @@ -60,6 +60,21 @@ assert len(lst) == 1 assert lst[0] == e.args[0] + def test_clear_pending_transactions(self): + import transaction + class Foo(Exception): + pass + def raiseme(): + raise Foo + for i in range(20): + transaction.add(raiseme) + try: + transaction.run() + assert 0, "should have raised Foo" + except Foo: + pass + transaction.run() # all the other 'raiseme's should have been cleared + class AppTestTransactionEmulator(AppTestTransaction): def setup_class(cls): From noreply at buildbot.pypy.org Tue Jan 31 17:05:35 2012 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 31 Jan 2012 17:05:35 +0100 (CET) Subject: [pypy-commit] pypy stm: Ignore epoll_wait() returning EINTR. Message-ID: <20120131160535.0DD2882B67@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: stm Changeset: r51993:889be87a6951 Date: 2012-01-31 16:54 +0100 http://bitbucket.org/pypy/pypy/changeset/889be87a6951/ Log: Ignore epoll_wait() returning EINTR. diff --git a/pypy/module/transaction/interp_epoll.py b/pypy/module/transaction/interp_epoll.py --- a/pypy/module/transaction/interp_epoll.py +++ b/pypy/module/transaction/interp_epoll.py @@ -3,6 +3,7 @@ from __future__ import with_statement import os +from errno import EINTR from pypy.rpython.lltypesystem import lltype, rffi from pypy.interpreter.gateway import unwrap_spec from pypy.interpreter.error import OperationError @@ -29,9 +30,13 @@ with lltype.scoped_alloc(rffi.CArray(epoll_event), maxevents) as evs: nfds = epoll_wait(self.epoller.epfd, evs, maxevents, int(timeout)) if nfds < 0: - state.got_exception_errno = rposix.get_errno() - state.must_reraise_exception(_reraise_from_errno) - return + errno = rposix.get_errno() + if errno == EINTR: + nfds = 0 # ignore, just wait for more later + else: + state.got_exception_errno = errno + state.must_reraise_exception(_reraise_from_errno) + return for i in range(nfds): event = evs[i] fd = rffi.cast(lltype.Signed, event.c_data.c_fd) From noreply at buildbot.pypy.org Tue Jan 31 17:17:23 2012 From: noreply at buildbot.pypy.org (hager) Date: Tue, 31 Jan 2012 17:17:23 +0100 (CET) Subject: [pypy-commit] pypy ppc-jit-backend: don't use make_sure_var_in_reg in prepare_same_as Message-ID: <20120131161723.A43CF7106F3@wyvern.cs.uni-duesseldorf.de> Author: hager <sven.hager at uni-duesseldorf.de> Branch: ppc-jit-backend Changeset: r51994:4e9490069131 Date: 2012-01-31 08:01 -0800 http://bitbucket.org/pypy/pypy/changeset/4e9490069131/ Log: don't use make_sure_var_in_reg in prepare_same_as diff --git a/pypy/jit/backend/ppc/ppcgen/regalloc.py b/pypy/jit/backend/ppc/ppcgen/regalloc.py --- a/pypy/jit/backend/ppc/ppcgen/regalloc.py +++ b/pypy/jit/backend/ppc/ppcgen/regalloc.py @@ -109,7 +109,7 @@ self.assembler.load(loc, immvalue) else: loc = self.make_sure_var_in_reg(thing, - forbidden_vars=forbidden_vars) + forbidden_vars=self.temp_boxes + forbidden_vars) return loc def allocate_scratch_reg(self, type=INT, selected_reg=None, forbidden_vars=None): @@ -765,11 +765,7 @@ def prepare_same_as(self, op): arg = op.getarg(0) - imm_arg = _check_imm_arg(arg) - if imm_arg: - argloc = self.make_sure_var_in_reg(arg) - else: - argloc = self._ensure_value_is_boxed(arg) + argloc = self._ensure_value_is_boxed(arg) self.possibly_free_vars_for_op(op) self.free_temp_vars() resloc = self.force_allocate_reg(op.result) From noreply at buildbot.pypy.org Tue Jan 31 17:17:24 2012 From: noreply at buildbot.pypy.org (hager) Date: Tue, 31 Jan 2012 17:17:24 +0100 (CET) Subject: [pypy-commit] pypy ppc-jit-backend: don't use make_sure_var_in_reg in helper/regalloc.py Message-ID: <20120131161724.D99957106F3@wyvern.cs.uni-duesseldorf.de> Author: hager <sven.hager at uni-duesseldorf.de> Branch: ppc-jit-backend Changeset: r51995:8a15fad80415 Date: 2012-01-31 08:16 -0800 http://bitbucket.org/pypy/pypy/changeset/8a15fad80415/ Log: don't use make_sure_var_in_reg in helper/regalloc.py diff --git a/pypy/jit/backend/ppc/ppcgen/helper/regalloc.py b/pypy/jit/backend/ppc/ppcgen/helper/regalloc.py --- a/pypy/jit/backend/ppc/ppcgen/helper/regalloc.py +++ b/pypy/jit/backend/ppc/ppcgen/helper/regalloc.py @@ -30,7 +30,7 @@ l0 = self._ensure_value_is_boxed(arg0, forbidden_vars=boxes) if imm_a1 and not imm_a0: - l1 = self.make_sure_var_in_reg(arg1, boxes) + l1 = self._ensure_value_is_boxed(arg1, boxes) else: l1 = self._ensure_value_is_boxed(arg1, forbidden_vars=boxes) @@ -44,7 +44,7 @@ def f(self, op): a0 = op.getarg(0) assert isinstance(a0, Box) - reg = self.make_sure_var_in_reg(a0) + reg = self._ensure_value_is_boxed(a0) self.possibly_free_vars_for_op(op) res = self.force_allocate_reg(op.result, [a0]) return [reg, res] @@ -65,15 +65,8 @@ b0, b1 = boxes imm_b0 = _check_imm_arg(b0) imm_b1 = _check_imm_arg(b1) - if not imm_b0 and imm_b1: - l0 = self._ensure_value_is_boxed(b0) - l1 = self.make_sure_var_in_reg(b1, boxes) - elif imm_b0 and not imm_b1: - l0 = self.make_sure_var_in_reg(b0) - l1 = self._ensure_value_is_boxed(b1, boxes) - else: - l0 = self._ensure_value_is_boxed(b0) - l1 = self._ensure_value_is_boxed(b1, boxes) + l0 = self._ensure_value_is_boxed(b0, boxes) + l1 = self._ensure_value_is_boxed(b1, boxes) locs = [l0, l1] self.possibly_free_vars_for_op(op) self.free_temp_vars() From noreply at buildbot.pypy.org Tue Jan 31 17:27:45 2012 From: noreply at buildbot.pypy.org (hager) Date: Tue, 31 Jan 2012 17:27:45 +0100 (CET) Subject: [pypy-commit] pypy ppc-jit-backend: remove further uses of make_sure_var_in_reg Message-ID: <20120131162745.6771082B67@wyvern.cs.uni-duesseldorf.de> Author: hager <sven.hager at uni-duesseldorf.de> Branch: ppc-jit-backend Changeset: r51996:fa607e2b04a2 Date: 2012-01-31 08:26 -0800 http://bitbucket.org/pypy/pypy/changeset/fa607e2b04a2/ Log: remove further uses of make_sure_var_in_reg diff --git a/pypy/jit/backend/ppc/ppcgen/regalloc.py b/pypy/jit/backend/ppc/ppcgen/regalloc.py --- a/pypy/jit/backend/ppc/ppcgen/regalloc.py +++ b/pypy/jit/backend/ppc/ppcgen/regalloc.py @@ -448,12 +448,8 @@ def prepare_guard_value(self, op): boxes = list(op.getarglist()) a0, a1 = boxes - imm_a1 = check_imm_box(a1) l0 = self._ensure_value_is_boxed(a0, boxes) - if not imm_a1: - l1 = self._ensure_value_is_boxed(a1, boxes) - else: - l1 = self.make_sure_var_in_reg(a1, boxes) + l1 = self._ensure_value_is_boxed(a1, boxes) assert op.result is None arglocs = self._prepare_guard(op, [l0, l1]) self.possibly_free_vars(op.getarglist()) @@ -693,11 +689,7 @@ base_loc = self._ensure_value_is_boxed(boxes[0]) a1 = boxes[1] - imm_a1 = check_imm_box(a1) - if imm_a1: - ofs_loc = self.make_sure_var_in_reg(a1, boxes) - else: - ofs_loc = self._ensure_value_is_boxed(a1, boxes) + ofs_loc = self._ensure_value_is_boxed(a1, boxes) self.possibly_free_vars_for_op(op) self.free_temp_vars() From noreply at buildbot.pypy.org Tue Jan 31 17:36:50 2012 From: noreply at buildbot.pypy.org (hager) Date: Tue, 31 Jan 2012 17:36:50 +0100 (CET) Subject: [pypy-commit] pypy ppc-jit-backend: kill _malloc_varsize and _regalloc_malloc_varsize Message-ID: <20120131163650.ABA337106F3@wyvern.cs.uni-duesseldorf.de> Author: hager <sven.hager at uni-duesseldorf.de> Branch: ppc-jit-backend Changeset: r51997:f14a20d7af69 Date: 2012-01-31 08:35 -0800 http://bitbucket.org/pypy/pypy/changeset/f14a20d7af69/ Log: kill _malloc_varsize and _regalloc_malloc_varsize diff --git a/pypy/jit/backend/ppc/ppcgen/opassembler.py b/pypy/jit/backend/ppc/ppcgen/opassembler.py --- a/pypy/jit/backend/ppc/ppcgen/opassembler.py +++ b/pypy/jit/backend/ppc/ppcgen/opassembler.py @@ -854,24 +854,6 @@ self.emit_call(op, arglocs, regalloc) self.propagate_memoryerror_if_r3_is_null() - # from: ../x86/regalloc.py:750 - # called from regalloc - # XXX kill this function at some point - def _regalloc_malloc_varsize(self, size, size_box, vloc, vbox, - ofs_items_loc, regalloc, result): - if IS_PPC_32: - self.mc.mullw(size.value, size.value, vloc.value) - else: - self.mc.mulld(size.value, size.value, vloc.value) - if ofs_items_loc.is_imm(): - self.mc.addi(size.value, size.value, ofs_items_loc.value) - else: - self.mc.add(size.value, size.value, ofs_items_loc.value) - force_index = self.write_new_force_index() - regalloc.force_spill_var(vbox) - self._emit_call(force_index, self.malloc_func_addr, [size_box], regalloc, - result=result) - def set_vtable(self, box, vtable): if self.cpu.vtable_offset is not None: adr = rffi.cast(lltype.Signed, vtable) diff --git a/pypy/jit/backend/ppc/ppcgen/regalloc.py b/pypy/jit/backend/ppc/ppcgen/regalloc.py --- a/pypy/jit/backend/ppc/ppcgen/regalloc.py +++ b/pypy/jit/backend/ppc/ppcgen/regalloc.py @@ -882,32 +882,6 @@ arglocs.append(t) return arglocs - def _malloc_varsize(self, ofs_items, ofs_length, itemsize, op): - v = op.getarg(0) - res_v = op.result - boxes = [v, res_v] - itemsize_box = ConstInt(itemsize) - ofs_items_box = ConstInt(ofs_items) - if _check_imm_arg(ofs_items_box): - ofs_items_loc = self.convert_to_imm(ofs_items_box) - else: - ofs_items_loc, ofs_items_box = self._ensure_value_is_boxed(ofs_items_box, boxes) - boxes.append(ofs_items_box) - vloc, vbox = self._ensure_value_is_boxed(v, [res_v]) - boxes.append(vbox) - size, size_box = self._ensure_value_is_boxed(itemsize_box, boxes) - boxes.append(size_box) - self.assembler._regalloc_malloc_varsize(size, size_box, - vloc, vbox, ofs_items_loc, self, res_v) - base_loc = self.make_sure_var_in_reg(res_v) - - value_loc, vbox = self._ensure_value_is_boxed(v, [res_v]) - boxes.append(vbox) - self.possibly_free_vars(boxes) - assert value_loc.is_reg() - assert base_loc.is_reg() - return [value_loc, base_loc, imm(ofs_length)] - # from ../x86/regalloc.py:791 def _unpack_fielddescr(self, fielddescr): assert isinstance(fielddescr, BaseFieldDescr) From noreply at buildbot.pypy.org Tue Jan 31 17:56:25 2012 From: noreply at buildbot.pypy.org (hager) Date: Tue, 31 Jan 2012 17:56:25 +0100 (CET) Subject: [pypy-commit] pypy ppc-jit-backend: adjust prepare_setarrayitemgc and prepare_getarrayitemgc to ARM backend Message-ID: <20120131165625.636F582B67@wyvern.cs.uni-duesseldorf.de> Author: hager <sven.hager at uni-duesseldorf.de> Branch: ppc-jit-backend Changeset: r51998:5e8969c29bbc Date: 2012-01-31 08:55 -0800 http://bitbucket.org/pypy/pypy/changeset/5e8969c29bbc/ Log: adjust prepare_setarrayitemgc and prepare_getarrayitemgc to ARM backend diff --git a/pypy/jit/backend/ppc/ppcgen/regalloc.py b/pypy/jit/backend/ppc/ppcgen/regalloc.py --- a/pypy/jit/backend/ppc/ppcgen/regalloc.py +++ b/pypy/jit/backend/ppc/ppcgen/regalloc.py @@ -634,33 +634,30 @@ return [res, base_loc, imm(ofs)] def prepare_setarrayitem_gc(self, op): - a0, a1, a2 = list(op.getarglist()) size, ofs, _ = unpack_arraydescr(op.getdescr()) scale = get_scale(size) args = op.getarglist() - base_loc = self._ensure_value_is_boxed(a0, args) - ofs_loc = self._ensure_value_is_boxed(a1, args) - scratch_loc = self.rm.get_scratch_reg(INT, [base_loc, ofs_loc]) - value_loc = self._ensure_value_is_boxed(a2, args) + base_loc = self._ensure_value_is_boxed(args[0], args) + ofs_loc = self._ensure_value_is_boxed(args[1], args) + value_loc = self._ensure_value_is_boxed(args[2], args) + scratch_loc = self.rm.get_scratch_reg(INT, + [base_loc, ofs_loc, value_loc]) assert _check_imm_arg(ofs) - return [value_loc, base_loc, ofs_loc, scratch_loc, - imm(scale), imm(ofs)] - + return [value_loc, base_loc, ofs_loc, scratch_loc, imm(scale), imm(ofs)] prepare_setarrayitem_raw = prepare_setarrayitem_gc def prepare_getarrayitem_gc(self, op): - a0, a1 = boxes = list(op.getarglist()) + boxes = op.getarglist() size, ofs, _ = unpack_arraydescr(op.getdescr()) scale = get_scale(size) - base_loc = self._ensure_value_is_boxed(a0, boxes) - ofs_loc = self._ensure_value_is_boxed(a1, boxes) + base_loc = self._ensure_value_is_boxed(boxes[0], boxes) + ofs_loc = self._ensure_value_is_boxed(boxes[1], boxes) scratch_loc = self.rm.get_scratch_reg(INT, [base_loc, ofs_loc]) self.possibly_free_vars_for_op(op) self.free_temp_vars() res = self.force_allocate_reg(op.result) assert _check_imm_arg(ofs) - return [res, base_loc, ofs_loc, scratch_loc, - imm(scale), imm(ofs)] + return [res, base_loc, ofs_loc, scratch_loc, imm(scale), imm(ofs)] prepare_getarrayitem_raw = prepare_getarrayitem_gc prepare_getarrayitem_gc_pure = prepare_getarrayitem_gc From noreply at buildbot.pypy.org Tue Jan 31 18:09:32 2012 From: noreply at buildbot.pypy.org (hager) Date: Tue, 31 Jan 2012 18:09:32 +0100 (CET) Subject: [pypy-commit] pypy ppc-jit-backend: don't copy arglist in prepare_strgetitem and prepare_strsetitem Message-ID: <20120131170932.437FC7106F3@wyvern.cs.uni-duesseldorf.de> Author: hager <sven.hager at uni-duesseldorf.de> Branch: ppc-jit-backend Changeset: r51999:e8e75c828faa Date: 2012-01-31 09:08 -0800 http://bitbucket.org/pypy/pypy/changeset/e8e75c828faa/ Log: don't copy arglist in prepare_strgetitem and prepare_strsetitem diff --git a/pypy/jit/backend/ppc/ppcgen/regalloc.py b/pypy/jit/backend/ppc/ppcgen/regalloc.py --- a/pypy/jit/backend/ppc/ppcgen/regalloc.py +++ b/pypy/jit/backend/ppc/ppcgen/regalloc.py @@ -682,7 +682,7 @@ return [l0, l1, res] def prepare_strgetitem(self, op): - boxes = list(op.getarglist()) + boxes = op.getarglist() base_loc = self._ensure_value_is_boxed(boxes[0]) a1 = boxes[1] @@ -698,7 +698,7 @@ return [res, base_loc, ofs_loc, imm(basesize)] def prepare_strsetitem(self, op): - boxes = list(op.getarglist()) + boxes = op.getarglist() base_loc = self._ensure_value_is_boxed(boxes[0], boxes) ofs_loc = self._ensure_value_is_boxed(boxes[1], boxes) value_loc = self._ensure_value_is_boxed(boxes[2], boxes) From noreply at buildbot.pypy.org Tue Jan 31 18:57:10 2012 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 31 Jan 2012 18:57:10 +0100 (CET) Subject: [pypy-commit] extradoc extradoc: First-step planning. Message-ID: <20120131175710.53BBD7106F3@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo <arigo at tunes.org> Branch: extradoc Changeset: r4070:9e7b09df3f36 Date: 2012-01-31 18:56 +0100 http://bitbucket.org/pypy/extradoc/changeset/9e7b09df3f36/ Log: First-step planning. diff --git a/planning/stm.txt b/planning/stm.txt --- a/planning/stm.txt +++ b/planning/stm.txt @@ -2,6 +2,9 @@ STM planning ============ +Comments in << >> describe the next thing to work on. + + Overview -------- @@ -72,6 +75,9 @@ use 4-5 bits, where in addition we use some "thread hash" value if there is only one copy. +<< NOW: think of a minimal GC model with these properties. We probably +need GC_GLOBAL, a single bit of GC_WAS_COPIED, and the version number. >> + stm_read -------- @@ -96,6 +102,10 @@ depending on cases). And if the read is accepted then we need to remember in a local list that we've read that object. +<< NOW: implement the thread's local dictionary in C, as say a search +tree. Should be easy enough if we don't try to be as efficient as +possible. The rest of the logic here is straightforward. >> + stm_write --------- @@ -114,6 +124,8 @@ consistent copy (i.e. nobody changed the object in the middle of us reading it). If it is too recent, then we might have to abort. +<< NOW: straightforward >> + End-of-transaction collections ------------------------------ @@ -132,6 +144,9 @@ We need to check that each of these global objects' versions have not been modified in the meantime. +<< NOW: should be easy, but with unclear interactions between the C +code and the GC. >> + Annotator support ----------------- @@ -149,6 +164,8 @@ of a localobj are themselves localobjs. This would be useful for 'PyFrame.fastlocals_w': it should also be known to always be a localobj. +<< do later >> + Local collections ----------------- @@ -173,6 +190,9 @@ all global reads done --- "compress" in the sense of removing duplicates. +<< do later; memory usage grows unboundedly during one transaction for +now. >> + Global collections ------------------ @@ -218,6 +238,10 @@ * Parallelism: there are multiple threads all doing something GC-related, like all scanning the heap together. +<< at first the global area keeps growing unboundedly. The next step +will be to add the LIL but run the global collection by keeping all +other threads blocked. >> + When not running transactively ------------------------------ @@ -240,11 +264,11 @@ is called, we can try to do such a collection, but what about the pinned objects? -Some intermediate solution would be to let this mode be rather slow: +<< NOW: let this mode be rather slow. To implement this mode, we would have only global objects, and have the stm_write barrier of 'obj' return -'obj'. Do only global collections. Allocation would allocate -immediately a global object, mostly without being able to benefit from -bump-pointer allocation. +'obj'. Do only global collections (one we have them; at first, don't +collect at all). Allocation would allocate immediately a global object, +without being able to benefit from bump-pointer allocation. >> Pointer equality @@ -261,6 +285,8 @@ dictionary if they map to each other. And we need to take care of the cases of NULL pointers. +<< NOW: straightforward, if we're careful not to forget cases >> + From noreply at buildbot.pypy.org Tue Jan 31 19:39:53 2012 From: noreply at buildbot.pypy.org (hager) Date: Tue, 31 Jan 2012 19:39:53 +0100 (CET) Subject: [pypy-commit] pypy ppc-jit-backend: don't copy arglist in prepare_unicodesetitem and prepare_unicodegetitem Message-ID: <20120131183953.3605982B67@wyvern.cs.uni-duesseldorf.de> Author: hager <sven.hager at uni-duesseldorf.de> Branch: ppc-jit-backend Changeset: r52000:de37273d21e7 Date: 2012-01-31 10:39 -0800 http://bitbucket.org/pypy/pypy/changeset/de37273d21e7/ Log: don't copy arglist in prepare_unicodesetitem and prepare_unicodegetitem diff --git a/pypy/jit/backend/ppc/ppcgen/regalloc.py b/pypy/jit/backend/ppc/ppcgen/regalloc.py --- a/pypy/jit/backend/ppc/ppcgen/regalloc.py +++ b/pypy/jit/backend/ppc/ppcgen/regalloc.py @@ -727,7 +727,7 @@ return [l0, l1, res] def prepare_unicodegetitem(self, op): - boxes = list(op.getarglist()) + boxes = op.getarglist() base_loc = self._ensure_value_is_boxed(boxes[0], boxes) ofs_loc = self._ensure_value_is_boxed(boxes[1], boxes) @@ -742,7 +742,7 @@ imm(scale), imm(basesize), imm(itemsize)] def prepare_unicodesetitem(self, op): - boxes = list(op.getarglist()) + boxes = op.getarglist() base_loc = self._ensure_value_is_boxed(boxes[0], boxes) ofs_loc = self._ensure_value_is_boxed(boxes[1], boxes) value_loc = self._ensure_value_is_boxed(boxes[2], boxes) From noreply at buildbot.pypy.org Tue Jan 31 23:52:36 2012 From: noreply at buildbot.pypy.org (mattip) Date: Tue, 31 Jan 2012 23:52:36 +0100 (CET) Subject: [pypy-commit] pypy win32-cleanup: put Defined in correct place Message-ID: <20120131225236.0E92082B67@wyvern.cs.uni-duesseldorf.de> Author: mattip Branch: win32-cleanup Changeset: r52001:b54fc297932d Date: 2012-01-31 23:33 +0200 http://bitbucket.org/pypy/pypy/changeset/b54fc297932d/ Log: put Defined in correct place diff --git a/pypy/rlib/ropenssl.py b/pypy/rlib/ropenssl.py --- a/pypy/rlib/ropenssl.py +++ b/pypy/rlib/ropenssl.py @@ -118,6 +118,8 @@ ('block_size', rffi.INT)]) EVP_MD_SIZE = rffi_platform.SizeOf('EVP_MD') EVP_MD_CTX_SIZE = rffi_platform.SizeOf('EVP_MD_CTX') + OPENSSL_EXPORT_VAR_AS_FUNCTION = rffi_platform.Defined( + "OPENSSL_EXPORT_VAR_AS_FUNCTION") for k, v in rffi_platform.configure(CConfig).items(): @@ -224,7 +226,7 @@ ssl_external('i2a_ASN1_INTEGER', [BIO, ASN1_INTEGER], rffi.INT) ssl_external('ASN1_item_d2i', [rffi.VOIDP, rffi.CCHARPP, rffi.LONG, ASN1_ITEM], rffi.VOIDP) -if rffi_platform.Defined("OPENSSL_EXPORT_VAR_AS_FUNCTION"): +if OPENSSL_EXPORT_VAR_AS_FUNCTION: ssl_external('ASN1_ITEM_ptr', [ASN1_ITEM_EXP], ASN1_ITEM, macro=True) else: ssl_external('ASN1_ITEM_ptr', [rffi.VOIDP], ASN1_ITEM, macro=True) From noreply at buildbot.pypy.org Tue Jan 31 23:52:37 2012 From: noreply at buildbot.pypy.org (mattip) Date: Tue, 31 Jan 2012 23:52:37 +0100 (CET) Subject: [pypy-commit] pypy win32-cleanup: cause test to report Error and not crash testing process Message-ID: <20120131225237.447CA82B68@wyvern.cs.uni-duesseldorf.de> Author: mattip Branch: win32-cleanup Changeset: r52002:f90a69310775 Date: 2012-02-01 00:48 +0200 http://bitbucket.org/pypy/pypy/changeset/f90a69310775/ Log: cause test to report Error and not crash testing process diff --git a/pypy/module/_file/test/test_file.py b/pypy/module/_file/test/test_file.py --- a/pypy/module/_file/test/test_file.py +++ b/pypy/module/_file/test/test_file.py @@ -265,6 +265,13 @@ if option.runappdirect: py.test.skip("works with internals of _file impl on py.py") + import platform + if platform.system() == 'Windows': + # XXX This test crashes until someone implements something like + # XXX verify_fd from + # XXX http://hg.python.org/cpython/file/80ddbd822227/Modules/posixmodule.c#l434 + # XXX and adds it to fopen + assert False state = [0] def read(fd, n=None): From noreply at buildbot.pypy.org Tue Jan 31 23:52:43 2012 From: noreply at buildbot.pypy.org (mattip) Date: Tue, 31 Jan 2012 23:52:43 +0100 (CET) Subject: [pypy-commit] pypy win32-cleanup: merge with default Message-ID: <20120131225243.E9AF582B67@wyvern.cs.uni-duesseldorf.de> Author: mattip Branch: win32-cleanup Changeset: r52003:c496f04019ad Date: 2012-02-01 00:50 +0200 http://bitbucket.org/pypy/pypy/changeset/c496f04019ad/ Log: merge with default diff too long, truncating to 10000 out of 144189 lines diff --git a/lib-python/2.7/BaseHTTPServer.py b/lib-python/2.7/BaseHTTPServer.py --- a/lib-python/2.7/BaseHTTPServer.py +++ b/lib-python/2.7/BaseHTTPServer.py @@ -310,7 +310,13 @@ """ try: - self.raw_requestline = self.rfile.readline() + self.raw_requestline = self.rfile.readline(65537) + if len(self.raw_requestline) > 65536: + self.requestline = '' + self.request_version = '' + self.command = '' + self.send_error(414) + return if not self.raw_requestline: self.close_connection = 1 return diff --git a/lib-python/2.7/ConfigParser.py b/lib-python/2.7/ConfigParser.py --- a/lib-python/2.7/ConfigParser.py +++ b/lib-python/2.7/ConfigParser.py @@ -545,6 +545,38 @@ if isinstance(val, list): options[name] = '\n'.join(val) +import UserDict as _UserDict + +class _Chainmap(_UserDict.DictMixin): + """Combine multiple mappings for successive lookups. + + For example, to emulate Python's normal lookup sequence: + + import __builtin__ + pylookup = _Chainmap(locals(), globals(), vars(__builtin__)) + """ + + def __init__(self, *maps): + self._maps = maps + + def __getitem__(self, key): + for mapping in self._maps: + try: + return mapping[key] + except KeyError: + pass + raise KeyError(key) + + def keys(self): + result = [] + seen = set() + for mapping in self_maps: + for key in mapping: + if key not in seen: + result.append(key) + seen.add(key) + return result + class ConfigParser(RawConfigParser): def get(self, section, option, raw=False, vars=None): @@ -559,16 +591,18 @@ The section DEFAULT is special. """ - d = self._defaults.copy() + sectiondict = {} try: - d.update(self._sections[section]) + sectiondict = self._sections[section] except KeyError: if section != DEFAULTSECT: raise NoSectionError(section) # Update with the entry specific variables + vardict = {} if vars: for key, value in vars.items(): - d[self.optionxform(key)] = value + vardict[self.optionxform(key)] = value + d = _Chainmap(vardict, sectiondict, self._defaults) option = self.optionxform(option) try: value = d[option] diff --git a/lib-python/2.7/Cookie.py b/lib-python/2.7/Cookie.py --- a/lib-python/2.7/Cookie.py +++ b/lib-python/2.7/Cookie.py @@ -258,6 +258,11 @@ '\033' : '\\033', '\034' : '\\034', '\035' : '\\035', '\036' : '\\036', '\037' : '\\037', + # Because of the way browsers really handle cookies (as opposed + # to what the RFC says) we also encode , and ; + + ',' : '\\054', ';' : '\\073', + '"' : '\\"', '\\' : '\\\\', '\177' : '\\177', '\200' : '\\200', '\201' : '\\201', diff --git a/lib-python/2.7/HTMLParser.py b/lib-python/2.7/HTMLParser.py --- a/lib-python/2.7/HTMLParser.py +++ b/lib-python/2.7/HTMLParser.py @@ -26,7 +26,7 @@ tagfind = re.compile('[a-zA-Z][-.a-zA-Z0-9:_]*') attrfind = re.compile( r'\s*([a-zA-Z_][-.:a-zA-Z_0-9]*)(\s*=\s*' - r'(\'[^\']*\'|"[^"]*"|[-a-zA-Z0-9./,:;+*%?!&$_#=~@]*))?') + r'(\'[^\']*\'|"[^"]*"|[^\s"\'=<>`]*))?') locatestarttagend = re.compile(r""" <[a-zA-Z][-.a-zA-Z0-9:_]* # tag name @@ -99,7 +99,7 @@ markupbase.ParserBase.reset(self) def feed(self, data): - """Feed data to the parser. + r"""Feed data to the parser. Call this as often as you want, with as little or as much text as you want (may include '\n'). @@ -367,13 +367,16 @@ return s def replaceEntities(s): s = s.groups()[0] - if s[0] == "#": - s = s[1:] - if s[0] in ['x','X']: - c = int(s[1:], 16) - else: - c = int(s) - return unichr(c) + try: + if s[0] == "#": + s = s[1:] + if s[0] in ['x','X']: + c = int(s[1:], 16) + else: + c = int(s) + return unichr(c) + except ValueError: + return '&#'+s+';' else: # Cannot use name2codepoint directly, because HTMLParser supports apos, # which is not part of HTML 4 diff --git a/lib-python/2.7/SimpleHTTPServer.py b/lib-python/2.7/SimpleHTTPServer.py --- a/lib-python/2.7/SimpleHTTPServer.py +++ b/lib-python/2.7/SimpleHTTPServer.py @@ -15,6 +15,7 @@ import BaseHTTPServer import urllib import cgi +import sys import shutil import mimetypes try: @@ -131,7 +132,8 @@ length = f.tell() f.seek(0) self.send_response(200) - self.send_header("Content-type", "text/html") + encoding = sys.getfilesystemencoding() + self.send_header("Content-type", "text/html; charset=%s" % encoding) self.send_header("Content-Length", str(length)) self.end_headers() return f diff --git a/lib-python/2.7/SimpleXMLRPCServer.py b/lib-python/2.7/SimpleXMLRPCServer.py --- a/lib-python/2.7/SimpleXMLRPCServer.py +++ b/lib-python/2.7/SimpleXMLRPCServer.py @@ -246,7 +246,7 @@ marshalled data. For backwards compatibility, a dispatch function can be provided as an argument (see comment in SimpleXMLRPCRequestHandler.do_POST) but overriding the - existing method through subclassing is the prefered means + existing method through subclassing is the preferred means of changing method dispatch behavior. """ diff --git a/lib-python/2.7/SocketServer.py b/lib-python/2.7/SocketServer.py --- a/lib-python/2.7/SocketServer.py +++ b/lib-python/2.7/SocketServer.py @@ -675,7 +675,7 @@ # A timeout to apply to the request socket, if not None. timeout = None - # Disable nagle algoritm for this socket, if True. + # Disable nagle algorithm for this socket, if True. # Use only when wbufsize != 0, to avoid small packets. disable_nagle_algorithm = False diff --git a/lib-python/2.7/StringIO.py b/lib-python/2.7/StringIO.py --- a/lib-python/2.7/StringIO.py +++ b/lib-python/2.7/StringIO.py @@ -266,6 +266,7 @@ 8th bit) will cause a UnicodeError to be raised when getvalue() is called. """ + _complain_ifclosed(self.closed) if self.buflist: self.buf += ''.join(self.buflist) self.buflist = [] diff --git a/lib-python/2.7/_abcoll.py b/lib-python/2.7/_abcoll.py --- a/lib-python/2.7/_abcoll.py +++ b/lib-python/2.7/_abcoll.py @@ -82,7 +82,7 @@ @classmethod def __subclasshook__(cls, C): if cls is Iterator: - if _hasattr(C, "next"): + if _hasattr(C, "next") and _hasattr(C, "__iter__"): return True return NotImplemented diff --git a/lib-python/2.7/_pyio.py b/lib-python/2.7/_pyio.py --- a/lib-python/2.7/_pyio.py +++ b/lib-python/2.7/_pyio.py @@ -16,6 +16,7 @@ import io from io import (__all__, SEEK_SET, SEEK_CUR, SEEK_END) +from errno import EINTR __metaclass__ = type @@ -559,7 +560,11 @@ if not data: break res += data - return bytes(res) + if res: + return bytes(res) + else: + # b'' or None + return data def readinto(self, b): """Read up to len(b) bytes into b. @@ -678,7 +683,7 @@ """ def __init__(self, raw): - self.raw = raw + self._raw = raw ### Positioning ### @@ -722,8 +727,8 @@ if self.raw is None: raise ValueError("raw stream already detached") self.flush() - raw = self.raw - self.raw = None + raw = self._raw + self._raw = None return raw ### Inquiries ### @@ -738,6 +743,10 @@ return self.raw.writable() @property + def raw(self): + return self._raw + + @property def closed(self): return self.raw.closed @@ -933,7 +942,12 @@ current_size = 0 while True: # Read until EOF or until read() would block. - chunk = self.raw.read() + try: + chunk = self.raw.read() + except IOError as e: + if e.errno != EINTR: + raise + continue if chunk in empty_values: nodata_val = chunk break @@ -952,7 +966,12 @@ chunks = [buf[pos:]] wanted = max(self.buffer_size, n) while avail < n: - chunk = self.raw.read(wanted) + try: + chunk = self.raw.read(wanted) + except IOError as e: + if e.errno != EINTR: + raise + continue if chunk in empty_values: nodata_val = chunk break @@ -981,7 +1000,14 @@ have = len(self._read_buf) - self._read_pos if have < want or have <= 0: to_read = self.buffer_size - have - current = self.raw.read(to_read) + while True: + try: + current = self.raw.read(to_read) + except IOError as e: + if e.errno != EINTR: + raise + continue + break if current: self._read_buf = self._read_buf[self._read_pos:] + current self._read_pos = 0 @@ -1088,7 +1114,12 @@ written = 0 try: while self._write_buf: - n = self.raw.write(self._write_buf) + try: + n = self.raw.write(self._write_buf) + except IOError as e: + if e.errno != EINTR: + raise + continue if n > len(self._write_buf) or n < 0: raise IOError("write() returned incorrect number of bytes") del self._write_buf[:n] @@ -1456,7 +1487,7 @@ if not isinstance(errors, basestring): raise ValueError("invalid errors: %r" % errors) - self.buffer = buffer + self._buffer = buffer self._line_buffering = line_buffering self._encoding = encoding self._errors = errors @@ -1511,6 +1542,10 @@ def line_buffering(self): return self._line_buffering + @property + def buffer(self): + return self._buffer + def seekable(self): return self._seekable @@ -1724,8 +1759,8 @@ if self.buffer is None: raise ValueError("buffer is already detached") self.flush() - buffer = self.buffer - self.buffer = None + buffer = self._buffer + self._buffer = None return buffer def seek(self, cookie, whence=0): diff --git a/lib-python/2.7/_weakrefset.py b/lib-python/2.7/_weakrefset.py --- a/lib-python/2.7/_weakrefset.py +++ b/lib-python/2.7/_weakrefset.py @@ -66,7 +66,11 @@ return sum(x() is not None for x in self.data) def __contains__(self, item): - return ref(item) in self.data + try: + wr = ref(item) + except TypeError: + return False + return wr in self.data def __reduce__(self): return (self.__class__, (list(self),), diff --git a/lib-python/2.7/anydbm.py b/lib-python/2.7/anydbm.py --- a/lib-python/2.7/anydbm.py +++ b/lib-python/2.7/anydbm.py @@ -29,17 +29,8 @@ list = d.keys() # return a list of all existing keys (slow!) Future versions may change the order in which implementations are -tested for existence, add interfaces to other dbm-like +tested for existence, and add interfaces to other dbm-like implementations. - -The open function has an optional second argument. This can be 'r', -for read-only access, 'w', for read-write access of an existing -database, 'c' for read-write access to a new or existing database, and -'n' for read-write access to a new database. The default is 'r'. - -Note: 'r' and 'w' fail if the database doesn't exist; 'c' creates it -only if it doesn't exist; and 'n' always creates a new database. - """ class error(Exception): @@ -63,7 +54,18 @@ error = tuple(_errors) -def open(file, flag = 'r', mode = 0666): +def open(file, flag='r', mode=0666): + """Open or create database at path given by *file*. + + Optional argument *flag* can be 'r' (default) for read-only access, 'w' + for read-write access of an existing database, 'c' for read-write access + to a new or existing database, and 'n' for read-write access to a new + database. + + Note: 'r' and 'w' fail if the database doesn't exist; 'c' creates it + only if it doesn't exist; and 'n' always creates a new database. + """ + # guess the type of an existing database from whichdb import whichdb result=whichdb(file) diff --git a/lib-python/2.7/argparse.py b/lib-python/2.7/argparse.py --- a/lib-python/2.7/argparse.py +++ b/lib-python/2.7/argparse.py @@ -82,6 +82,7 @@ ] +import collections as _collections import copy as _copy import os as _os import re as _re @@ -1037,7 +1038,7 @@ self._prog_prefix = prog self._parser_class = parser_class - self._name_parser_map = {} + self._name_parser_map = _collections.OrderedDict() self._choices_actions = [] super(_SubParsersAction, self).__init__( @@ -1080,7 +1081,7 @@ parser = self._name_parser_map[parser_name] except KeyError: tup = parser_name, ', '.join(self._name_parser_map) - msg = _('unknown parser %r (choices: %s)' % tup) + msg = _('unknown parser %r (choices: %s)') % tup raise ArgumentError(self, msg) # parse all the remaining options into the namespace @@ -1109,7 +1110,7 @@ the builtin open() function. """ - def __init__(self, mode='r', bufsize=None): + def __init__(self, mode='r', bufsize=-1): self._mode = mode self._bufsize = bufsize @@ -1121,18 +1122,19 @@ elif 'w' in self._mode: return _sys.stdout else: - msg = _('argument "-" with mode %r' % self._mode) + msg = _('argument "-" with mode %r') % self._mode raise ValueError(msg) # all other arguments are used as file names - if self._bufsize: + try: return open(string, self._mode, self._bufsize) - else: - return open(string, self._mode) + except IOError as e: + message = _("can't open '%s': %s") + raise ArgumentTypeError(message % (string, e)) def __repr__(self): - args = [self._mode, self._bufsize] - args_str = ', '.join([repr(arg) for arg in args if arg is not None]) + args = self._mode, self._bufsize + args_str = ', '.join(repr(arg) for arg in args if arg != -1) return '%s(%s)' % (type(self).__name__, args_str) # =========================== @@ -1275,13 +1277,20 @@ # create the action object, and add it to the parser action_class = self._pop_action_class(kwargs) if not _callable(action_class): - raise ValueError('unknown action "%s"' % action_class) + raise ValueError('unknown action "%s"' % (action_class,)) action = action_class(**kwargs) # raise an error if the action type is not callable type_func = self._registry_get('type', action.type, action.type) if not _callable(type_func): - raise ValueError('%r is not callable' % type_func) + raise ValueError('%r is not callable' % (type_func,)) + + # raise an error if the metavar does not match the type + if hasattr(self, "_get_formatter"): + try: + self._get_formatter()._format_args(action, None) + except TypeError: + raise ValueError("length of metavar tuple does not match nargs") return self._add_action(action) @@ -1481,6 +1490,7 @@ self._defaults = container._defaults self._has_negative_number_optionals = \ container._has_negative_number_optionals + self._mutually_exclusive_groups = container._mutually_exclusive_groups def _add_action(self, action): action = super(_ArgumentGroup, self)._add_action(action) diff --git a/lib-python/2.7/ast.py b/lib-python/2.7/ast.py --- a/lib-python/2.7/ast.py +++ b/lib-python/2.7/ast.py @@ -29,12 +29,12 @@ from _ast import __version__ -def parse(expr, filename='<unknown>', mode='exec'): +def parse(source, filename='<unknown>', mode='exec'): """ - Parse an expression into an AST node. - Equivalent to compile(expr, filename, mode, PyCF_ONLY_AST). + Parse the source into an AST node. + Equivalent to compile(source, filename, mode, PyCF_ONLY_AST). """ - return compile(expr, filename, mode, PyCF_ONLY_AST) + return compile(source, filename, mode, PyCF_ONLY_AST) def literal_eval(node_or_string): @@ -152,8 +152,6 @@ Increment the line number of each node in the tree starting at *node* by *n*. This is useful to "move code" to a different location in a file. """ - if 'lineno' in node._attributes: - node.lineno = getattr(node, 'lineno', 0) + n for child in walk(node): if 'lineno' in child._attributes: child.lineno = getattr(child, 'lineno', 0) + n @@ -204,9 +202,9 @@ def walk(node): """ - Recursively yield all child nodes of *node*, in no specified order. This is - useful if you only want to modify nodes in place and don't care about the - context. + Recursively yield all descendant nodes in the tree starting at *node* + (including *node* itself), in no specified order. This is useful if you + only want to modify nodes in place and don't care about the context. """ from collections import deque todo = deque([node]) diff --git a/lib-python/2.7/asyncore.py b/lib-python/2.7/asyncore.py --- a/lib-python/2.7/asyncore.py +++ b/lib-python/2.7/asyncore.py @@ -54,7 +54,11 @@ import os from errno import EALREADY, EINPROGRESS, EWOULDBLOCK, ECONNRESET, EINVAL, \ - ENOTCONN, ESHUTDOWN, EINTR, EISCONN, EBADF, ECONNABORTED, errorcode + ENOTCONN, ESHUTDOWN, EINTR, EISCONN, EBADF, ECONNABORTED, EPIPE, EAGAIN, \ + errorcode + +_DISCONNECTED = frozenset((ECONNRESET, ENOTCONN, ESHUTDOWN, ECONNABORTED, EPIPE, + EBADF)) try: socket_map @@ -109,7 +113,7 @@ if flags & (select.POLLHUP | select.POLLERR | select.POLLNVAL): obj.handle_close() except socket.error, e: - if e.args[0] not in (EBADF, ECONNRESET, ENOTCONN, ESHUTDOWN, ECONNABORTED): + if e.args[0] not in _DISCONNECTED: obj.handle_error() else: obj.handle_close() @@ -353,7 +357,7 @@ except TypeError: return None except socket.error as why: - if why.args[0] in (EWOULDBLOCK, ECONNABORTED): + if why.args[0] in (EWOULDBLOCK, ECONNABORTED, EAGAIN): return None else: raise @@ -367,7 +371,7 @@ except socket.error, why: if why.args[0] == EWOULDBLOCK: return 0 - elif why.args[0] in (ECONNRESET, ENOTCONN, ESHUTDOWN, ECONNABORTED): + elif why.args[0] in _DISCONNECTED: self.handle_close() return 0 else: @@ -385,7 +389,7 @@ return data except socket.error, why: # winsock sometimes throws ENOTCONN - if why.args[0] in [ECONNRESET, ENOTCONN, ESHUTDOWN, ECONNABORTED]: + if why.args[0] in _DISCONNECTED: self.handle_close() return '' else: diff --git a/lib-python/2.7/bdb.py b/lib-python/2.7/bdb.py --- a/lib-python/2.7/bdb.py +++ b/lib-python/2.7/bdb.py @@ -250,6 +250,12 @@ list.append(lineno) bp = Breakpoint(filename, lineno, temporary, cond, funcname) + def _prune_breaks(self, filename, lineno): + if (filename, lineno) not in Breakpoint.bplist: + self.breaks[filename].remove(lineno) + if not self.breaks[filename]: + del self.breaks[filename] + def clear_break(self, filename, lineno): filename = self.canonic(filename) if not filename in self.breaks: @@ -261,10 +267,7 @@ # pair, then remove the breaks entry for bp in Breakpoint.bplist[filename, lineno][:]: bp.deleteMe() - if (filename, lineno) not in Breakpoint.bplist: - self.breaks[filename].remove(lineno) - if not self.breaks[filename]: - del self.breaks[filename] + self._prune_breaks(filename, lineno) def clear_bpbynumber(self, arg): try: @@ -277,7 +280,8 @@ return 'Breakpoint number (%d) out of range' % number if not bp: return 'Breakpoint (%d) already deleted' % number - self.clear_break(bp.file, bp.line) + bp.deleteMe() + self._prune_breaks(bp.file, bp.line) def clear_all_file_breaks(self, filename): filename = self.canonic(filename) diff --git a/lib-python/2.7/collections.py b/lib-python/2.7/collections.py --- a/lib-python/2.7/collections.py +++ b/lib-python/2.7/collections.py @@ -6,59 +6,38 @@ __all__ += _abcoll.__all__ from _collections import deque, defaultdict -from operator import itemgetter as _itemgetter, eq as _eq +from operator import itemgetter as _itemgetter from keyword import iskeyword as _iskeyword import sys as _sys import heapq as _heapq -from itertools import repeat as _repeat, chain as _chain, starmap as _starmap, \ - ifilter as _ifilter, imap as _imap +from itertools import repeat as _repeat, chain as _chain, starmap as _starmap + try: - from thread import get_ident + from thread import get_ident as _get_ident except ImportError: - from dummy_thread import get_ident - -def _recursive_repr(user_function): - 'Decorator to make a repr function return "..." for a recursive call' - repr_running = set() - - def wrapper(self): - key = id(self), get_ident() - if key in repr_running: - return '...' - repr_running.add(key) - try: - result = user_function(self) - finally: - repr_running.discard(key) - return result - - # Can't use functools.wraps() here because of bootstrap issues - wrapper.__module__ = getattr(user_function, '__module__') - wrapper.__doc__ = getattr(user_function, '__doc__') - wrapper.__name__ = getattr(user_function, '__name__') - return wrapper + from dummy_thread import get_ident as _get_ident ################################################################################ ### OrderedDict ################################################################################ -class OrderedDict(dict, MutableMapping): +class OrderedDict(dict): 'Dictionary that remembers insertion order' # An inherited dict maps keys to values. # The inherited dict provides __getitem__, __len__, __contains__, and get. # The remaining methods are order-aware. - # Big-O running times for all methods are the same as for regular dictionaries. + # Big-O running times for all methods are the same as regular dictionaries. - # The internal self.__map dictionary maps keys to links in a doubly linked list. + # The internal self.__map dict maps keys to links in a doubly linked list. # The circular doubly linked list starts and ends with a sentinel element. # The sentinel element never gets deleted (this simplifies the algorithm). # Each link is stored as a list of length three: [PREV, NEXT, KEY]. def __init__(self, *args, **kwds): - '''Initialize an ordered dictionary. Signature is the same as for - regular dictionaries, but keyword arguments are not recommended - because their insertion order is arbitrary. + '''Initialize an ordered dictionary. The signature is the same as + regular dictionaries, but keyword arguments are not recommended because + their insertion order is arbitrary. ''' if len(args) > 1: @@ -66,17 +45,15 @@ try: self.__root except AttributeError: - self.__root = root = [None, None, None] # sentinel node - PREV = 0 - NEXT = 1 - root[PREV] = root[NEXT] = root + self.__root = root = [] # sentinel node + root[:] = [root, root, None] self.__map = {} - self.update(*args, **kwds) + self.__update(*args, **kwds) def __setitem__(self, key, value, PREV=0, NEXT=1, dict_setitem=dict.__setitem__): 'od.__setitem__(i, y) <==> od[i]=y' - # Setting a new item creates a new link which goes at the end of the linked - # list, and the inherited dictionary is updated with the new key/value pair. + # Setting a new item creates a new link at the end of the linked list, + # and the inherited dictionary is updated with the new key/value pair. if key not in self: root = self.__root last = root[PREV] @@ -85,65 +62,160 @@ def __delitem__(self, key, PREV=0, NEXT=1, dict_delitem=dict.__delitem__): 'od.__delitem__(y) <==> del od[y]' - # Deleting an existing item uses self.__map to find the link which is - # then removed by updating the links in the predecessor and successor nodes. + # Deleting an existing item uses self.__map to find the link which gets + # removed by updating the links in the predecessor and successor nodes. dict_delitem(self, key) - link = self.__map.pop(key) - link_prev = link[PREV] - link_next = link[NEXT] + link_prev, link_next, key = self.__map.pop(key) link_prev[NEXT] = link_next link_next[PREV] = link_prev - def __iter__(self, NEXT=1, KEY=2): + def __iter__(self): 'od.__iter__() <==> iter(od)' # Traverse the linked list in order. + NEXT, KEY = 1, 2 root = self.__root curr = root[NEXT] while curr is not root: yield curr[KEY] curr = curr[NEXT] - def __reversed__(self, PREV=0, KEY=2): + def __reversed__(self): 'od.__reversed__() <==> reversed(od)' # Traverse the linked list in reverse order. + PREV, KEY = 0, 2 root = self.__root curr = root[PREV] while curr is not root: yield curr[KEY] curr = curr[PREV] + def clear(self): + 'od.clear() -> None. Remove all items from od.' + for node in self.__map.itervalues(): + del node[:] + root = self.__root + root[:] = [root, root, None] + self.__map.clear() + dict.clear(self) + + # -- the following methods do not depend on the internal structure -- + + def keys(self): + 'od.keys() -> list of keys in od' + return list(self) + + def values(self): + 'od.values() -> list of values in od' + return [self[key] for key in self] + + def items(self): + 'od.items() -> list of (key, value) pairs in od' + return [(key, self[key]) for key in self] + + def iterkeys(self): + 'od.iterkeys() -> an iterator over the keys in od' + return iter(self) + + def itervalues(self): + 'od.itervalues -> an iterator over the values in od' + for k in self: + yield self[k] + + def iteritems(self): + 'od.iteritems -> an iterator over the (key, value) pairs in od' + for k in self: + yield (k, self[k]) + + update = MutableMapping.update + + __update = update # let subclasses override update without breaking __init__ + + __marker = object() + + def pop(self, key, default=__marker): + '''od.pop(k[,d]) -> v, remove specified key and return the corresponding + value. If key is not found, d is returned if given, otherwise KeyError + is raised. + + ''' + if key in self: + result = self[key] + del self[key] + return result + if default is self.__marker: + raise KeyError(key) + return default + + def setdefault(self, key, default=None): + 'od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od' + if key in self: + return self[key] + self[key] = default + return default + + def popitem(self, last=True): + '''od.popitem() -> (k, v), return and remove a (key, value) pair. + Pairs are returned in LIFO order if last is true or FIFO order if false. + + ''' + if not self: + raise KeyError('dictionary is empty') + key = next(reversed(self) if last else iter(self)) + value = self.pop(key) + return key, value + + def __repr__(self, _repr_running={}): + 'od.__repr__() <==> repr(od)' + call_key = id(self), _get_ident() + if call_key in _repr_running: + return '...' + _repr_running[call_key] = 1 + try: + if not self: + return '%s()' % (self.__class__.__name__,) + return '%s(%r)' % (self.__class__.__name__, self.items()) + finally: + del _repr_running[call_key] + def __reduce__(self): 'Return state information for pickling' items = [[k, self[k]] for k in self] - tmp = self.__map, self.__root - del self.__map, self.__root inst_dict = vars(self).copy() - self.__map, self.__root = tmp + for k in vars(OrderedDict()): + inst_dict.pop(k, None) if inst_dict: return (self.__class__, (items,), inst_dict) return self.__class__, (items,) - def clear(self): - 'od.clear() -> None. Remove all items from od.' - try: - for node in self.__map.itervalues(): - del node[:] - self.__root[:] = [self.__root, self.__root, None] - self.__map.clear() - except AttributeError: - pass - dict.clear(self) + def copy(self): + 'od.copy() -> a shallow copy of od' + return self.__class__(self) - setdefault = MutableMapping.setdefault - update = MutableMapping.update - pop = MutableMapping.pop - keys = MutableMapping.keys - values = MutableMapping.values - items = MutableMapping.items - iterkeys = MutableMapping.iterkeys - itervalues = MutableMapping.itervalues - iteritems = MutableMapping.iteritems - __ne__ = MutableMapping.__ne__ + @classmethod + def fromkeys(cls, iterable, value=None): + '''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S. + If not specified, the value defaults to None. + + ''' + self = cls() + for key in iterable: + self[key] = value + return self + + def __eq__(self, other): + '''od.__eq__(y) <==> od==y. Comparison to another OD is order-sensitive + while comparison to a regular mapping is order-insensitive. + + ''' + if isinstance(other, OrderedDict): + return len(self)==len(other) and self.items() == other.items() + return dict.__eq__(self, other) + + def __ne__(self, other): + 'od.__ne__(y) <==> od!=y' + return not self == other + + # -- the following methods support python 3.x style dictionary views -- def viewkeys(self): "od.viewkeys() -> a set-like object providing a view on od's keys" @@ -157,49 +229,6 @@ "od.viewitems() -> a set-like object providing a view on od's items" return ItemsView(self) - def popitem(self, last=True): - '''od.popitem() -> (k, v), return and remove a (key, value) pair. - Pairs are returned in LIFO order if last is true or FIFO order if false. - - ''' - if not self: - raise KeyError('dictionary is empty') - key = next(reversed(self) if last else iter(self)) - value = self.pop(key) - return key, value - - @_recursive_repr - def __repr__(self): - 'od.__repr__() <==> repr(od)' - if not self: - return '%s()' % (self.__class__.__name__,) - return '%s(%r)' % (self.__class__.__name__, self.items()) - - def copy(self): - 'od.copy() -> a shallow copy of od' - return self.__class__(self) - - @classmethod - def fromkeys(cls, iterable, value=None): - '''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S - and values equal to v (which defaults to None). - - ''' - d = cls() - for key in iterable: - d[key] = value - return d - - def __eq__(self, other): - '''od.__eq__(y) <==> od==y. Comparison to another OD is order-sensitive - while comparison to a regular mapping is order-insensitive. - - ''' - if isinstance(other, OrderedDict): - return len(self)==len(other) and \ - all(_imap(_eq, self.iteritems(), other.iteritems())) - return dict.__eq__(self, other) - ################################################################################ ### namedtuple @@ -328,16 +357,16 @@ or multiset. Elements are stored as dictionary keys and their counts are stored as dictionary values. - >>> c = Counter('abracadabra') # count elements from a string + >>> c = Counter('abcdeabcdabcaba') # count elements from a string >>> c.most_common(3) # three most common elements - [('a', 5), ('r', 2), ('b', 2)] + [('a', 5), ('b', 4), ('c', 3)] >>> sorted(c) # list all unique elements - ['a', 'b', 'c', 'd', 'r'] + ['a', 'b', 'c', 'd', 'e'] >>> ''.join(sorted(c.elements())) # list elements with repetitions - 'aaaaabbcdrr' + 'aaaaabbbbcccdde' >>> sum(c.values()) # total of all counts - 11 + 15 >>> c['a'] # count of letter 'a' 5 @@ -345,8 +374,8 @@ ... c[elem] += 1 # by adding 1 to each element's count >>> c['a'] # now there are seven 'a' 7 - >>> del c['r'] # remove all 'r' - >>> c['r'] # now there are zero 'r' + >>> del c['b'] # remove all 'b' + >>> c['b'] # now there are zero 'b' 0 >>> d = Counter('simsalabim') # make another counter @@ -385,6 +414,7 @@ >>> c = Counter(a=4, b=2) # a new counter from keyword args ''' + super(Counter, self).__init__() self.update(iterable, **kwds) def __missing__(self, key): @@ -396,8 +426,8 @@ '''List the n most common elements and their counts from the most common to the least. If n is None, then list all element counts. - >>> Counter('abracadabra').most_common(3) - [('a', 5), ('r', 2), ('b', 2)] + >>> Counter('abcdeabcdabcaba').most_common(3) + [('a', 5), ('b', 4), ('c', 3)] ''' # Emulate Bag.sortedByCount from Smalltalk @@ -463,7 +493,7 @@ for elem, count in iterable.iteritems(): self[elem] = self_get(elem, 0) + count else: - dict.update(self, iterable) # fast path when counter is empty + super(Counter, self).update(iterable) # fast path when counter is empty else: self_get = self.get for elem in iterable: @@ -499,13 +529,16 @@ self.subtract(kwds) def copy(self): - 'Like dict.copy() but returns a Counter instance instead of a dict.' - return Counter(self) + 'Return a shallow copy.' + return self.__class__(self) + + def __reduce__(self): + return self.__class__, (dict(self),) def __delitem__(self, elem): 'Like dict.__delitem__() but does not raise KeyError for missing values.' if elem in self: - dict.__delitem__(self, elem) + super(Counter, self).__delitem__(elem) def __repr__(self): if not self: @@ -532,10 +565,13 @@ if not isinstance(other, Counter): return NotImplemented result = Counter() - for elem in set(self) | set(other): - newcount = self[elem] + other[elem] + for elem, count in self.items(): + newcount = count + other[elem] if newcount > 0: result[elem] = newcount + for elem, count in other.items(): + if elem not in self and count > 0: + result[elem] = count return result def __sub__(self, other): @@ -548,10 +584,13 @@ if not isinstance(other, Counter): return NotImplemented result = Counter() - for elem in set(self) | set(other): - newcount = self[elem] - other[elem] + for elem, count in self.items(): + newcount = count - other[elem] if newcount > 0: result[elem] = newcount + for elem, count in other.items(): + if elem not in self and count < 0: + result[elem] = 0 - count return result def __or__(self, other): @@ -564,11 +603,14 @@ if not isinstance(other, Counter): return NotImplemented result = Counter() - for elem in set(self) | set(other): - p, q = self[elem], other[elem] - newcount = q if p < q else p + for elem, count in self.items(): + other_count = other[elem] + newcount = other_count if count < other_count else count if newcount > 0: result[elem] = newcount + for elem, count in other.items(): + if elem not in self and count > 0: + result[elem] = count return result def __and__(self, other): @@ -581,11 +623,9 @@ if not isinstance(other, Counter): return NotImplemented result = Counter() - if len(self) < len(other): - self, other = other, self - for elem in _ifilter(self.__contains__, other): - p, q = self[elem], other[elem] - newcount = p if p < q else q + for elem, count in self.items(): + other_count = other[elem] + newcount = count if count < other_count else other_count if newcount > 0: result[elem] = newcount return result diff --git a/lib-python/2.7/compileall.py b/lib-python/2.7/compileall.py --- a/lib-python/2.7/compileall.py +++ b/lib-python/2.7/compileall.py @@ -9,7 +9,6 @@ packages -- for now, you'll have to deal with packages separately.) See module py_compile for details of the actual byte-compilation. - """ import os import sys @@ -31,7 +30,6 @@ directory name that will show up in error messages) force: if 1, force compilation, even if timestamps are up-to-date quiet: if 1, be quiet during compilation - """ if not quiet: print 'Listing', dir, '...' @@ -61,15 +59,16 @@ return success def compile_file(fullname, ddir=None, force=0, rx=None, quiet=0): - """Byte-compile file. - file: the file to byte-compile + """Byte-compile one file. + + Arguments (only fullname is required): + + fullname: the file to byte-compile ddir: if given, purported directory name (this is the directory name that will show up in error messages) force: if 1, force compilation, even if timestamps are up-to-date quiet: if 1, be quiet during compilation - """ - success = 1 name = os.path.basename(fullname) if ddir is not None: @@ -120,7 +119,6 @@ maxlevels: max recursion level (default 0) force: as for compile_dir() (default 0) quiet: as for compile_dir() (default 0) - """ success = 1 for dir in sys.path: diff --git a/lib-python/2.7/csv.py b/lib-python/2.7/csv.py --- a/lib-python/2.7/csv.py +++ b/lib-python/2.7/csv.py @@ -281,7 +281,7 @@ an all or nothing approach, so we allow for small variations in this number. 1) build a table of the frequency of each character on every line. - 2) build a table of freqencies of this frequency (meta-frequency?), + 2) build a table of frequencies of this frequency (meta-frequency?), e.g. 'x occurred 5 times in 10 rows, 6 times in 1000 rows, 7 times in 2 rows' 3) use the mode of the meta-frequency to determine the /expected/ diff --git a/lib-python/2.7/ctypes/test/test_arrays.py b/lib-python/2.7/ctypes/test/test_arrays.py --- a/lib-python/2.7/ctypes/test/test_arrays.py +++ b/lib-python/2.7/ctypes/test/test_arrays.py @@ -37,7 +37,7 @@ values = [ia[i] for i in range(len(init))] self.assertEqual(values, [0] * len(init)) - # Too many in itializers should be caught + # Too many initializers should be caught self.assertRaises(IndexError, int_array, *range(alen*2)) CharArray = ARRAY(c_char, 3) diff --git a/lib-python/2.7/ctypes/test/test_as_parameter.py b/lib-python/2.7/ctypes/test/test_as_parameter.py --- a/lib-python/2.7/ctypes/test/test_as_parameter.py +++ b/lib-python/2.7/ctypes/test/test_as_parameter.py @@ -187,6 +187,18 @@ self.assertEqual((s8i.a, s8i.b, s8i.c, s8i.d, s8i.e, s8i.f, s8i.g, s8i.h), (9*2, 8*3, 7*4, 6*5, 5*6, 4*7, 3*8, 2*9)) + def test_recursive_as_param(self): + from ctypes import c_int + + class A(object): + pass + + a = A() + a._as_parameter_ = a + with self.assertRaises(RuntimeError): + c_int.from_param(a) + + #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ class AsParamWrapper(object): diff --git a/lib-python/2.7/ctypes/test/test_callbacks.py b/lib-python/2.7/ctypes/test/test_callbacks.py --- a/lib-python/2.7/ctypes/test/test_callbacks.py +++ b/lib-python/2.7/ctypes/test/test_callbacks.py @@ -206,6 +206,42 @@ windll.user32.EnumWindows(EnumWindowsCallbackFunc, 0) + def test_callback_register_int(self): + # Issue #8275: buggy handling of callback args under Win64 + # NOTE: should be run on release builds as well + dll = CDLL(_ctypes_test.__file__) + CALLBACK = CFUNCTYPE(c_int, c_int, c_int, c_int, c_int, c_int) + # All this function does is call the callback with its args squared + func = dll._testfunc_cbk_reg_int + func.argtypes = (c_int, c_int, c_int, c_int, c_int, CALLBACK) + func.restype = c_int + + def callback(a, b, c, d, e): + return a + b + c + d + e + + result = func(2, 3, 4, 5, 6, CALLBACK(callback)) + self.assertEqual(result, callback(2*2, 3*3, 4*4, 5*5, 6*6)) + + def test_callback_register_double(self): + # Issue #8275: buggy handling of callback args under Win64 + # NOTE: should be run on release builds as well + dll = CDLL(_ctypes_test.__file__) + CALLBACK = CFUNCTYPE(c_double, c_double, c_double, c_double, + c_double, c_double) + # All this function does is call the callback with its args squared + func = dll._testfunc_cbk_reg_double + func.argtypes = (c_double, c_double, c_double, + c_double, c_double, CALLBACK) + func.restype = c_double + + def callback(a, b, c, d, e): + return a + b + c + d + e + + result = func(1.1, 2.2, 3.3, 4.4, 5.5, CALLBACK(callback)) + self.assertEqual(result, + callback(1.1*1.1, 2.2*2.2, 3.3*3.3, 4.4*4.4, 5.5*5.5)) + + ################################################################ if __name__ == '__main__': diff --git a/lib-python/2.7/ctypes/test/test_functions.py b/lib-python/2.7/ctypes/test/test_functions.py --- a/lib-python/2.7/ctypes/test/test_functions.py +++ b/lib-python/2.7/ctypes/test/test_functions.py @@ -116,7 +116,7 @@ self.assertEqual(result, 21) self.assertEqual(type(result), int) - # You cannot assing character format codes as restype any longer + # You cannot assign character format codes as restype any longer self.assertRaises(TypeError, setattr, f, "restype", "i") def test_floatresult(self): diff --git a/lib-python/2.7/ctypes/test/test_init.py b/lib-python/2.7/ctypes/test/test_init.py --- a/lib-python/2.7/ctypes/test/test_init.py +++ b/lib-python/2.7/ctypes/test/test_init.py @@ -27,7 +27,7 @@ self.assertEqual((y.x.a, y.x.b), (0, 0)) self.assertEqual(y.x.new_was_called, False) - # But explicitely creating an X structure calls __new__ and __init__, of course. + # But explicitly creating an X structure calls __new__ and __init__, of course. x = X() self.assertEqual((x.a, x.b), (9, 12)) self.assertEqual(x.new_was_called, True) diff --git a/lib-python/2.7/ctypes/test/test_numbers.py b/lib-python/2.7/ctypes/test/test_numbers.py --- a/lib-python/2.7/ctypes/test/test_numbers.py +++ b/lib-python/2.7/ctypes/test/test_numbers.py @@ -157,7 +157,7 @@ def test_int_from_address(self): from array import array for t in signed_types + unsigned_types: - # the array module doesn't suppport all format codes + # the array module doesn't support all format codes # (no 'q' or 'Q') try: array(t._type_) diff --git a/lib-python/2.7/ctypes/test/test_win32.py b/lib-python/2.7/ctypes/test/test_win32.py --- a/lib-python/2.7/ctypes/test/test_win32.py +++ b/lib-python/2.7/ctypes/test/test_win32.py @@ -17,7 +17,7 @@ # ValueError: Procedure probably called with not enough arguments (4 bytes missing) self.assertRaises(ValueError, IsWindow) - # This one should succeeed... + # This one should succeed... self.assertEqual(0, IsWindow(0)) # ValueError: Procedure probably called with too many arguments (8 bytes in excess) diff --git a/lib-python/2.7/curses/wrapper.py b/lib-python/2.7/curses/wrapper.py --- a/lib-python/2.7/curses/wrapper.py +++ b/lib-python/2.7/curses/wrapper.py @@ -43,7 +43,8 @@ return func(stdscr, *args, **kwds) finally: # Set everything back to normal - stdscr.keypad(0) - curses.echo() - curses.nocbreak() - curses.endwin() + if 'stdscr' in locals(): + stdscr.keypad(0) + curses.echo() + curses.nocbreak() + curses.endwin() diff --git a/lib-python/2.7/decimal.py b/lib-python/2.7/decimal.py --- a/lib-python/2.7/decimal.py +++ b/lib-python/2.7/decimal.py @@ -1068,14 +1068,16 @@ if ans: return ans - if not self: - # -Decimal('0') is Decimal('0'), not Decimal('-0') + if context is None: + context = getcontext() + + if not self and context.rounding != ROUND_FLOOR: + # -Decimal('0') is Decimal('0'), not Decimal('-0'), except + # in ROUND_FLOOR rounding mode. ans = self.copy_abs() else: ans = self.copy_negate() - if context is None: - context = getcontext() return ans._fix(context) def __pos__(self, context=None): @@ -1088,14 +1090,15 @@ if ans: return ans - if not self: - # + (-0) = 0 + if context is None: + context = getcontext() + + if not self and context.rounding != ROUND_FLOOR: + # + (-0) = 0, except in ROUND_FLOOR rounding mode. ans = self.copy_abs() else: ans = Decimal(self) - if context is None: - context = getcontext() return ans._fix(context) def __abs__(self, round=True, context=None): @@ -1680,7 +1683,7 @@ self = _dec_from_triple(self._sign, '1', exp_min-1) digits = 0 rounding_method = self._pick_rounding_function[context.rounding] - changed = getattr(self, rounding_method)(digits) + changed = rounding_method(self, digits) coeff = self._int[:digits] or '0' if changed > 0: coeff = str(int(coeff)+1) @@ -1720,8 +1723,6 @@ # here self was representable to begin with; return unchanged return Decimal(self) - _pick_rounding_function = {} - # for each of the rounding functions below: # self is a finite, nonzero Decimal # prec is an integer satisfying 0 <= prec < len(self._int) @@ -1788,6 +1789,17 @@ else: return -self._round_down(prec) + _pick_rounding_function = dict( + ROUND_DOWN = _round_down, + ROUND_UP = _round_up, + ROUND_HALF_UP = _round_half_up, + ROUND_HALF_DOWN = _round_half_down, + ROUND_HALF_EVEN = _round_half_even, + ROUND_CEILING = _round_ceiling, + ROUND_FLOOR = _round_floor, + ROUND_05UP = _round_05up, + ) + def fma(self, other, third, context=None): """Fused multiply-add. @@ -2492,8 +2504,8 @@ if digits < 0: self = _dec_from_triple(self._sign, '1', exp-1) digits = 0 - this_function = getattr(self, self._pick_rounding_function[rounding]) - changed = this_function(digits) + this_function = self._pick_rounding_function[rounding] + changed = this_function(self, digits) coeff = self._int[:digits] or '0' if changed == 1: coeff = str(int(coeff)+1) @@ -3705,18 +3717,6 @@ ##### Context class ####################################################### - -# get rounding method function: -rounding_functions = [name for name in Decimal.__dict__.keys() - if name.startswith('_round_')] -for name in rounding_functions: - # name is like _round_half_even, goes to the global ROUND_HALF_EVEN value. - globalname = name[1:].upper() - val = globals()[globalname] - Decimal._pick_rounding_function[val] = name - -del name, val, globalname, rounding_functions - class _ContextManager(object): """Context manager class to support localcontext(). @@ -5990,7 +5990,7 @@ def _format_align(sign, body, spec): """Given an unpadded, non-aligned numeric string 'body' and sign - string 'sign', add padding and aligment conforming to the given + string 'sign', add padding and alignment conforming to the given format specifier dictionary 'spec' (as produced by parse_format_specifier). diff --git a/lib-python/2.7/difflib.py b/lib-python/2.7/difflib.py --- a/lib-python/2.7/difflib.py +++ b/lib-python/2.7/difflib.py @@ -1140,6 +1140,21 @@ return ch in ws +######################################################################## +### Unified Diff +######################################################################## + +def _format_range_unified(start, stop): + 'Convert range to the "ed" format' + # Per the diff spec at http://www.unix.org/single_unix_specification/ + beginning = start + 1 # lines start numbering with one + length = stop - start + if length == 1: + return '{}'.format(beginning) + if not length: + beginning -= 1 # empty ranges begin at line just before the range + return '{},{}'.format(beginning, length) + def unified_diff(a, b, fromfile='', tofile='', fromfiledate='', tofiledate='', n=3, lineterm='\n'): r""" @@ -1184,25 +1199,45 @@ started = False for group in SequenceMatcher(None,a,b).get_grouped_opcodes(n): if not started: - fromdate = '\t%s' % fromfiledate if fromfiledate else '' - todate = '\t%s' % tofiledate if tofiledate else '' - yield '--- %s%s%s' % (fromfile, fromdate, lineterm) - yield '+++ %s%s%s' % (tofile, todate, lineterm) started = True - i1, i2, j1, j2 = group[0][1], group[-1][2], group[0][3], group[-1][4] - yield "@@ -%d,%d +%d,%d @@%s" % (i1+1, i2-i1, j1+1, j2-j1, lineterm) + fromdate = '\t{}'.format(fromfiledate) if fromfiledate else '' + todate = '\t{}'.format(tofiledate) if tofiledate else '' + yield '--- {}{}{}'.format(fromfile, fromdate, lineterm) + yield '+++ {}{}{}'.format(tofile, todate, lineterm) + + first, last = group[0], group[-1] + file1_range = _format_range_unified(first[1], last[2]) + file2_range = _format_range_unified(first[3], last[4]) + yield '@@ -{} +{} @@{}'.format(file1_range, file2_range, lineterm) + for tag, i1, i2, j1, j2 in group: if tag == 'equal': for line in a[i1:i2]: yield ' ' + line continue - if tag == 'replace' or tag == 'delete': + if tag in ('replace', 'delete'): for line in a[i1:i2]: yield '-' + line - if tag == 'replace' or tag == 'insert': + if tag in ('replace', 'insert'): for line in b[j1:j2]: yield '+' + line + +######################################################################## +### Context Diff +######################################################################## + +def _format_range_context(start, stop): + 'Convert range to the "ed" format' + # Per the diff spec at http://www.unix.org/single_unix_specification/ + beginning = start + 1 # lines start numbering with one + length = stop - start + if not length: + beginning -= 1 # empty ranges begin at line just before the range + if length <= 1: + return '{}'.format(beginning) + return '{},{}'.format(beginning, beginning + length - 1) + # See http://www.unix.org/single_unix_specification/ def context_diff(a, b, fromfile='', tofile='', fromfiledate='', tofiledate='', n=3, lineterm='\n'): @@ -1247,38 +1282,36 @@ four """ + prefix = dict(insert='+ ', delete='- ', replace='! ', equal=' ') started = False - prefixmap = {'insert':'+ ', 'delete':'- ', 'replace':'! ', 'equal':' '} for group in SequenceMatcher(None,a,b).get_grouped_opcodes(n): if not started: - fromdate = '\t%s' % fromfiledate if fromfiledate else '' - todate = '\t%s' % tofiledate if tofiledate else '' - yield '*** %s%s%s' % (fromfile, fromdate, lineterm) - yield '--- %s%s%s' % (tofile, todate, lineterm) started = True + fromdate = '\t{}'.format(fromfiledate) if fromfiledate else '' + todate = '\t{}'.format(tofiledate) if tofiledate else '' + yield '*** {}{}{}'.format(fromfile, fromdate, lineterm) + yield '--- {}{}{}'.format(tofile, todate, lineterm) - yield '***************%s' % (lineterm,) - if group[-1][2] - group[0][1] >= 2: - yield '*** %d,%d ****%s' % (group[0][1]+1, group[-1][2], lineterm) - else: - yield '*** %d ****%s' % (group[-1][2], lineterm) - visiblechanges = [e for e in group if e[0] in ('replace', 'delete')] - if visiblechanges: + first, last = group[0], group[-1] + yield '***************' + lineterm + + file1_range = _format_range_context(first[1], last[2]) + yield '*** {} ****{}'.format(file1_range, lineterm) + + if any(tag in ('replace', 'delete') for tag, _, _, _, _ in group): for tag, i1, i2, _, _ in group: if tag != 'insert': for line in a[i1:i2]: - yield prefixmap[tag] + line + yield prefix[tag] + line - if group[-1][4] - group[0][3] >= 2: - yield '--- %d,%d ----%s' % (group[0][3]+1, group[-1][4], lineterm) - else: - yield '--- %d ----%s' % (group[-1][4], lineterm) - visiblechanges = [e for e in group if e[0] in ('replace', 'insert')] - if visiblechanges: + file2_range = _format_range_context(first[3], last[4]) + yield '--- {} ----{}'.format(file2_range, lineterm) + + if any(tag in ('replace', 'insert') for tag, _, _, _, _ in group): for tag, _, _, j1, j2 in group: if tag != 'delete': for line in b[j1:j2]: - yield prefixmap[tag] + line + yield prefix[tag] + line def ndiff(a, b, linejunk=None, charjunk=IS_CHARACTER_JUNK): r""" @@ -1714,7 +1747,7 @@ line = line.replace(' ','\0') # expand tabs into spaces line = line.expandtabs(self._tabsize) - # relace spaces from expanded tabs back into tab characters + # replace spaces from expanded tabs back into tab characters # (we'll replace them with markup after we do differencing) line = line.replace(' ','\t') return line.replace('\0',' ').rstrip('\n') diff --git a/lib-python/2.7/distutils/__init__.py b/lib-python/2.7/distutils/__init__.py --- a/lib-python/2.7/distutils/__init__.py +++ b/lib-python/2.7/distutils/__init__.py @@ -15,5 +15,5 @@ # Updated automatically by the Python release process. # #--start constants-- -__version__ = "2.7.1" +__version__ = "2.7.2" #--end constants-- diff --git a/lib-python/2.7/distutils/archive_util.py b/lib-python/2.7/distutils/archive_util.py --- a/lib-python/2.7/distutils/archive_util.py +++ b/lib-python/2.7/distutils/archive_util.py @@ -121,7 +121,7 @@ def make_zipfile(base_name, base_dir, verbose=0, dry_run=0): """Create a zip file from all the files under 'base_dir'. - The output zip file will be named 'base_dir' + ".zip". Uses either the + The output zip file will be named 'base_name' + ".zip". Uses either the "zipfile" Python module (if available) or the InfoZIP "zip" utility (if installed and found on the default search path). If neither tool is available, raises DistutilsExecError. Returns the name of the output zip diff --git a/lib-python/2.7/distutils/cmd.py b/lib-python/2.7/distutils/cmd.py --- a/lib-python/2.7/distutils/cmd.py +++ b/lib-python/2.7/distutils/cmd.py @@ -377,7 +377,7 @@ dry_run=self.dry_run) def move_file (self, src, dst, level=1): - """Move a file respectin dry-run flag.""" + """Move a file respecting dry-run flag.""" return file_util.move_file(src, dst, dry_run = self.dry_run) def spawn (self, cmd, search_path=1, level=1): diff --git a/lib-python/2.7/distutils/command/build_ext.py b/lib-python/2.7/distutils/command/build_ext.py --- a/lib-python/2.7/distutils/command/build_ext.py +++ b/lib-python/2.7/distutils/command/build_ext.py @@ -207,7 +207,7 @@ elif MSVC_VERSION == 8: self.library_dirs.append(os.path.join(sys.exec_prefix, - 'PC', 'VS8.0', 'win32release')) + 'PC', 'VS8.0')) elif MSVC_VERSION == 7: self.library_dirs.append(os.path.join(sys.exec_prefix, 'PC', 'VS7.1')) diff --git a/lib-python/2.7/distutils/command/sdist.py b/lib-python/2.7/distutils/command/sdist.py --- a/lib-python/2.7/distutils/command/sdist.py +++ b/lib-python/2.7/distutils/command/sdist.py @@ -306,17 +306,20 @@ rstrip_ws=1, collapse_join=1) - while 1: - line = template.readline() - if line is None: # end of file - break + try: + while 1: + line = template.readline() + if line is None: # end of file + break - try: - self.filelist.process_template_line(line) - except DistutilsTemplateError, msg: - self.warn("%s, line %d: %s" % (template.filename, - template.current_line, - msg)) + try: + self.filelist.process_template_line(line) + except DistutilsTemplateError, msg: + self.warn("%s, line %d: %s" % (template.filename, + template.current_line, + msg)) + finally: + template.close() def prune_file_list(self): """Prune off branches that might slip into the file list as created diff --git a/lib-python/2.7/distutils/command/upload.py b/lib-python/2.7/distutils/command/upload.py --- a/lib-python/2.7/distutils/command/upload.py +++ b/lib-python/2.7/distutils/command/upload.py @@ -176,6 +176,9 @@ result = urlopen(request) status = result.getcode() reason = result.msg + if self.show_response: + msg = '\n'.join(('-' * 75, r.read(), '-' * 75)) + self.announce(msg, log.INFO) except socket.error, e: self.announce(str(e), log.ERROR) return @@ -189,6 +192,3 @@ else: self.announce('Upload failed (%s): %s' % (status, reason), log.ERROR) - if self.show_response: - msg = '\n'.join(('-' * 75, r.read(), '-' * 75)) - self.announce(msg, log.INFO) diff --git a/lib-python/2.7/distutils/sysconfig.py b/lib-python/2.7/distutils/sysconfig.py --- a/lib-python/2.7/distutils/sysconfig.py +++ b/lib-python/2.7/distutils/sysconfig.py @@ -389,7 +389,7 @@ cur_target = os.getenv('MACOSX_DEPLOYMENT_TARGET', '') if cur_target == '': cur_target = cfg_target - os.putenv('MACOSX_DEPLOYMENT_TARGET', cfg_target) + os.environ['MACOSX_DEPLOYMENT_TARGET'] = cfg_target elif map(int, cfg_target.split('.')) > map(int, cur_target.split('.')): my_msg = ('$MACOSX_DEPLOYMENT_TARGET mismatch: now "%s" but "%s" during configure' % (cur_target, cfg_target)) diff --git a/lib-python/2.7/distutils/tests/__init__.py b/lib-python/2.7/distutils/tests/__init__.py --- a/lib-python/2.7/distutils/tests/__init__.py +++ b/lib-python/2.7/distutils/tests/__init__.py @@ -15,9 +15,10 @@ import os import sys import unittest +from test.test_support import run_unittest -here = os.path.dirname(__file__) +here = os.path.dirname(__file__) or os.curdir def test_suite(): @@ -32,4 +33,4 @@ if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_archive_util.py b/lib-python/2.7/distutils/tests/test_archive_util.py --- a/lib-python/2.7/distutils/tests/test_archive_util.py +++ b/lib-python/2.7/distutils/tests/test_archive_util.py @@ -12,7 +12,7 @@ ARCHIVE_FORMATS) from distutils.spawn import find_executable, spawn from distutils.tests import support -from test.test_support import check_warnings +from test.test_support import check_warnings, run_unittest try: import grp @@ -281,4 +281,4 @@ return unittest.makeSuite(ArchiveUtilTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_bdist_msi.py b/lib-python/2.7/distutils/tests/test_bdist_msi.py --- a/lib-python/2.7/distutils/tests/test_bdist_msi.py +++ b/lib-python/2.7/distutils/tests/test_bdist_msi.py @@ -11,7 +11,7 @@ support.LoggingSilencer, unittest.TestCase): - def test_minial(self): + def test_minimal(self): # minimal test XXX need more tests from distutils.command.bdist_msi import bdist_msi pkg_pth, dist = self.create_dist() diff --git a/lib-python/2.7/distutils/tests/test_build.py b/lib-python/2.7/distutils/tests/test_build.py --- a/lib-python/2.7/distutils/tests/test_build.py +++ b/lib-python/2.7/distutils/tests/test_build.py @@ -2,6 +2,7 @@ import unittest import os import sys +from test.test_support import run_unittest from distutils.command.build import build from distutils.tests import support @@ -51,4 +52,4 @@ return unittest.makeSuite(BuildTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_build_clib.py b/lib-python/2.7/distutils/tests/test_build_clib.py --- a/lib-python/2.7/distutils/tests/test_build_clib.py +++ b/lib-python/2.7/distutils/tests/test_build_clib.py @@ -3,6 +3,8 @@ import os import sys +from test.test_support import run_unittest + from distutils.command.build_clib import build_clib from distutils.errors import DistutilsSetupError from distutils.tests import support @@ -140,4 +142,4 @@ return unittest.makeSuite(BuildCLibTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_build_ext.py b/lib-python/2.7/distutils/tests/test_build_ext.py --- a/lib-python/2.7/distutils/tests/test_build_ext.py +++ b/lib-python/2.7/distutils/tests/test_build_ext.py @@ -3,12 +3,13 @@ import tempfile import shutil from StringIO import StringIO +import textwrap from distutils.core import Extension, Distribution from distutils.command.build_ext import build_ext from distutils import sysconfig from distutils.tests import support -from distutils.errors import DistutilsSetupError +from distutils.errors import DistutilsSetupError, CompileError import unittest from test import test_support @@ -430,6 +431,59 @@ wanted = os.path.join(cmd.build_lib, 'UpdateManager', 'fdsend' + ext) self.assertEqual(ext_path, wanted) + @unittest.skipUnless(sys.platform == 'darwin', 'test only relevant for MacOSX') + def test_deployment_target(self): + self._try_compile_deployment_target() + + orig_environ = os.environ + os.environ = orig_environ.copy() + self.addCleanup(setattr, os, 'environ', orig_environ) + + os.environ['MACOSX_DEPLOYMENT_TARGET']='10.1' + self._try_compile_deployment_target() + + + def _try_compile_deployment_target(self): + deptarget_c = os.path.join(self.tmp_dir, 'deptargetmodule.c') + + with open(deptarget_c, 'w') as fp: + fp.write(textwrap.dedent('''\ + #include <AvailabilityMacros.h> + + int dummy; + + #if TARGET != MAC_OS_X_VERSION_MIN_REQUIRED + #error "Unexpected target" + #endif + + ''')) + + target = sysconfig.get_config_var('MACOSX_DEPLOYMENT_TARGET') + target = tuple(map(int, target.split('.'))) + target = '%02d%01d0' % target + + deptarget_ext = Extension( + 'deptarget', + [deptarget_c], + extra_compile_args=['-DTARGET=%s'%(target,)], + ) + dist = Distribution({ + 'name': 'deptarget', + 'ext_modules': [deptarget_ext] + }) + dist.package_dir = self.tmp_dir + cmd = build_ext(dist) + cmd.build_lib = self.tmp_dir + cmd.build_temp = self.tmp_dir + + try: + old_stdout = sys.stdout + cmd.ensure_finalized() + cmd.run() + + except CompileError: + self.fail("Wrong deployment target during compilation") + def test_suite(): return unittest.makeSuite(BuildExtTestCase) diff --git a/lib-python/2.7/distutils/tests/test_build_py.py b/lib-python/2.7/distutils/tests/test_build_py.py --- a/lib-python/2.7/distutils/tests/test_build_py.py +++ b/lib-python/2.7/distutils/tests/test_build_py.py @@ -10,13 +10,14 @@ from distutils.errors import DistutilsFileError from distutils.tests import support +from test.test_support import run_unittest class BuildPyTestCase(support.TempdirManager, support.LoggingSilencer, unittest.TestCase): - def _setup_package_data(self): + def test_package_data(self): sources = self.mkdtemp() f = open(os.path.join(sources, "__init__.py"), "w") try: @@ -56,20 +57,15 @@ self.assertEqual(len(cmd.get_outputs()), 3) pkgdest = os.path.join(destination, "pkg") files = os.listdir(pkgdest) - return files + self.assertIn("__init__.py", files) + self.assertIn("README.txt", files) + # XXX even with -O, distutils writes pyc, not pyo; bug? + if sys.dont_write_bytecode: + self.assertNotIn("__init__.pyc", files) + else: + self.assertIn("__init__.pyc", files) - def test_package_data(self): - files = self._setup_package_data() - self.assertTrue("__init__.py" in files) - self.assertTrue("README.txt" in files) - - @unittest.skipIf(sys.flags.optimize >= 2, - "pyc files are not written with -O2 and above") - def test_package_data_pyc(self): - files = self._setup_package_data() - self.assertTrue("__init__.pyc" in files) - - def test_empty_package_dir (self): + def test_empty_package_dir(self): # See SF 1668596/1720897. cwd = os.getcwd() @@ -117,10 +113,10 @@ finally: sys.dont_write_bytecode = old_dont_write_bytecode - self.assertTrue('byte-compiling is disabled' in self.logs[0][1]) + self.assertIn('byte-compiling is disabled', self.logs[0][1]) def test_suite(): return unittest.makeSuite(BuildPyTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_build_scripts.py b/lib-python/2.7/distutils/tests/test_build_scripts.py --- a/lib-python/2.7/distutils/tests/test_build_scripts.py +++ b/lib-python/2.7/distutils/tests/test_build_scripts.py @@ -8,6 +8,7 @@ import sysconfig from distutils.tests import support +from test.test_support import run_unittest class BuildScriptsTestCase(support.TempdirManager, @@ -108,4 +109,4 @@ return unittest.makeSuite(BuildScriptsTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_check.py b/lib-python/2.7/distutils/tests/test_check.py --- a/lib-python/2.7/distutils/tests/test_check.py +++ b/lib-python/2.7/distutils/tests/test_check.py @@ -1,5 +1,6 @@ """Tests for distutils.command.check.""" import unittest +from test.test_support import run_unittest from distutils.command.check import check, HAS_DOCUTILS from distutils.tests import support @@ -95,4 +96,4 @@ return unittest.makeSuite(CheckTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_clean.py b/lib-python/2.7/distutils/tests/test_clean.py --- a/lib-python/2.7/distutils/tests/test_clean.py +++ b/lib-python/2.7/distutils/tests/test_clean.py @@ -6,6 +6,7 @@ from distutils.command.clean import clean from distutils.tests import support +from test.test_support import run_unittest class cleanTestCase(support.TempdirManager, support.LoggingSilencer, @@ -38,7 +39,7 @@ self.assertTrue(not os.path.exists(path), '%s was not removed' % path) - # let's run the command again (should spit warnings but suceed) + # let's run the command again (should spit warnings but succeed) cmd.all = 1 cmd.ensure_finalized() cmd.run() @@ -47,4 +48,4 @@ return unittest.makeSuite(cleanTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_cmd.py b/lib-python/2.7/distutils/tests/test_cmd.py --- a/lib-python/2.7/distutils/tests/test_cmd.py +++ b/lib-python/2.7/distutils/tests/test_cmd.py @@ -99,7 +99,7 @@ def test_ensure_dirname(self): cmd = self.cmd - cmd.option1 = os.path.dirname(__file__) + cmd.option1 = os.path.dirname(__file__) or os.curdir cmd.ensure_dirname('option1') cmd.option2 = 'xxx' self.assertRaises(DistutilsOptionError, cmd.ensure_dirname, 'option2') diff --git a/lib-python/2.7/distutils/tests/test_config.py b/lib-python/2.7/distutils/tests/test_config.py --- a/lib-python/2.7/distutils/tests/test_config.py +++ b/lib-python/2.7/distutils/tests/test_config.py @@ -11,6 +11,7 @@ from distutils.log import WARN from distutils.tests import support +from test.test_support import run_unittest PYPIRC = """\ [distutils] @@ -119,4 +120,4 @@ return unittest.makeSuite(PyPIRCCommandTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_config_cmd.py b/lib-python/2.7/distutils/tests/test_config_cmd.py --- a/lib-python/2.7/distutils/tests/test_config_cmd.py +++ b/lib-python/2.7/distutils/tests/test_config_cmd.py @@ -2,6 +2,7 @@ import unittest import os import sys +from test.test_support import run_unittest from distutils.command.config import dump_file, config from distutils.tests import support @@ -86,4 +87,4 @@ return unittest.makeSuite(ConfigTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_core.py b/lib-python/2.7/distutils/tests/test_core.py --- a/lib-python/2.7/distutils/tests/test_core.py +++ b/lib-python/2.7/distutils/tests/test_core.py @@ -6,7 +6,7 @@ import shutil import sys import test.test_support -from test.test_support import captured_stdout +from test.test_support import captured_stdout, run_unittest import unittest from distutils.tests import support @@ -105,4 +105,4 @@ return unittest.makeSuite(CoreTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_dep_util.py b/lib-python/2.7/distutils/tests/test_dep_util.py --- a/lib-python/2.7/distutils/tests/test_dep_util.py +++ b/lib-python/2.7/distutils/tests/test_dep_util.py @@ -6,6 +6,7 @@ from distutils.dep_util import newer, newer_pairwise, newer_group from distutils.errors import DistutilsFileError from distutils.tests import support +from test.test_support import run_unittest class DepUtilTestCase(support.TempdirManager, unittest.TestCase): @@ -77,4 +78,4 @@ return unittest.makeSuite(DepUtilTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_dir_util.py b/lib-python/2.7/distutils/tests/test_dir_util.py --- a/lib-python/2.7/distutils/tests/test_dir_util.py +++ b/lib-python/2.7/distutils/tests/test_dir_util.py @@ -10,6 +10,7 @@ from distutils import log from distutils.tests import support +from test.test_support import run_unittest class DirUtilTestCase(support.TempdirManager, unittest.TestCase): @@ -112,4 +113,4 @@ return unittest.makeSuite(DirUtilTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_dist.py b/lib-python/2.7/distutils/tests/test_dist.py --- a/lib-python/2.7/distutils/tests/test_dist.py +++ b/lib-python/2.7/distutils/tests/test_dist.py @@ -11,7 +11,7 @@ from distutils.dist import Distribution, fix_help_options, DistributionMetadata from distutils.cmd import Command import distutils.dist -from test.test_support import TESTFN, captured_stdout +from test.test_support import TESTFN, captured_stdout, run_unittest from distutils.tests import support class test_dist(Command): @@ -433,4 +433,4 @@ return suite if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_file_util.py b/lib-python/2.7/distutils/tests/test_file_util.py --- a/lib-python/2.7/distutils/tests/test_file_util.py +++ b/lib-python/2.7/distutils/tests/test_file_util.py @@ -6,6 +6,7 @@ from distutils.file_util import move_file, write_file, copy_file from distutils import log from distutils.tests import support +from test.test_support import run_unittest class FileUtilTestCase(support.TempdirManager, unittest.TestCase): @@ -77,4 +78,4 @@ return unittest.makeSuite(FileUtilTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_filelist.py b/lib-python/2.7/distutils/tests/test_filelist.py --- a/lib-python/2.7/distutils/tests/test_filelist.py +++ b/lib-python/2.7/distutils/tests/test_filelist.py @@ -1,7 +1,7 @@ """Tests for distutils.filelist.""" from os.path import join import unittest -from test.test_support import captured_stdout +from test.test_support import captured_stdout, run_unittest from distutils.filelist import glob_to_re, FileList from distutils import debug @@ -82,4 +82,4 @@ return unittest.makeSuite(FileListTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_install.py b/lib-python/2.7/distutils/tests/test_install.py --- a/lib-python/2.7/distutils/tests/test_install.py +++ b/lib-python/2.7/distutils/tests/test_install.py @@ -3,6 +3,8 @@ import os import unittest +from test.test_support import run_unittest + from distutils.command.install import install from distutils.core import Distribution @@ -52,4 +54,4 @@ return unittest.makeSuite(InstallTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_install_data.py b/lib-python/2.7/distutils/tests/test_install_data.py --- a/lib-python/2.7/distutils/tests/test_install_data.py +++ b/lib-python/2.7/distutils/tests/test_install_data.py @@ -6,6 +6,7 @@ from distutils.command.install_data import install_data from distutils.tests import support +from test.test_support import run_unittest class InstallDataTestCase(support.TempdirManager, support.LoggingSilencer, @@ -73,4 +74,4 @@ return unittest.makeSuite(InstallDataTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_install_headers.py b/lib-python/2.7/distutils/tests/test_install_headers.py --- a/lib-python/2.7/distutils/tests/test_install_headers.py +++ b/lib-python/2.7/distutils/tests/test_install_headers.py @@ -6,6 +6,7 @@ from distutils.command.install_headers import install_headers from distutils.tests import support +from test.test_support import run_unittest class InstallHeadersTestCase(support.TempdirManager, support.LoggingSilencer, @@ -37,4 +38,4 @@ return unittest.makeSuite(InstallHeadersTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_install_lib.py b/lib-python/2.7/distutils/tests/test_install_lib.py --- a/lib-python/2.7/distutils/tests/test_install_lib.py +++ b/lib-python/2.7/distutils/tests/test_install_lib.py @@ -7,6 +7,7 @@ from distutils.extension import Extension from distutils.tests import support from distutils.errors import DistutilsOptionError +from test.test_support import run_unittest class InstallLibTestCase(support.TempdirManager, support.LoggingSilencer, @@ -103,4 +104,4 @@ return unittest.makeSuite(InstallLibTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_install_scripts.py b/lib-python/2.7/distutils/tests/test_install_scripts.py --- a/lib-python/2.7/distutils/tests/test_install_scripts.py +++ b/lib-python/2.7/distutils/tests/test_install_scripts.py @@ -7,6 +7,7 @@ from distutils.core import Distribution from distutils.tests import support +from test.test_support import run_unittest class InstallScriptsTestCase(support.TempdirManager, @@ -78,4 +79,4 @@ return unittest.makeSuite(InstallScriptsTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_msvc9compiler.py b/lib-python/2.7/distutils/tests/test_msvc9compiler.py --- a/lib-python/2.7/distutils/tests/test_msvc9compiler.py +++ b/lib-python/2.7/distutils/tests/test_msvc9compiler.py @@ -5,6 +5,7 @@ from distutils.errors import DistutilsPlatformError from distutils.tests import support +from test.test_support import run_unittest _MANIFEST = """\ <?xml version="1.0" encoding="UTF-8" standalone="yes"?> @@ -137,4 +138,4 @@ return unittest.makeSuite(msvc9compilerTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_register.py b/lib-python/2.7/distutils/tests/test_register.py --- a/lib-python/2.7/distutils/tests/test_register.py +++ b/lib-python/2.7/distutils/tests/test_register.py @@ -7,7 +7,7 @@ import urllib2 import warnings -from test.test_support import check_warnings +from test.test_support import check_warnings, run_unittest from distutils.command import register as register_module from distutils.command.register import register @@ -138,7 +138,7 @@ # let's see what the server received : we should # have 2 similar requests - self.assertTrue(self.conn.reqs, 2) + self.assertEqual(len(self.conn.reqs), 2) req1 = dict(self.conn.reqs[0].headers) req2 = dict(self.conn.reqs[1].headers) self.assertEqual(req2['Content-length'], req1['Content-length']) @@ -168,7 +168,7 @@ del register_module.raw_input # we should have send a request - self.assertTrue(self.conn.reqs, 1) + self.assertEqual(len(self.conn.reqs), 1) req = self.conn.reqs[0] headers = dict(req.headers) self.assertEqual(headers['Content-length'], '608') @@ -186,7 +186,7 @@ del register_module.raw_input # we should have send a request - self.assertTrue(self.conn.reqs, 1) + self.assertEqual(len(self.conn.reqs), 1) req = self.conn.reqs[0] headers = dict(req.headers) self.assertEqual(headers['Content-length'], '290') @@ -258,4 +258,4 @@ return unittest.makeSuite(RegisterTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_sdist.py b/lib-python/2.7/distutils/tests/test_sdist.py --- a/lib-python/2.7/distutils/tests/test_sdist.py +++ b/lib-python/2.7/distutils/tests/test_sdist.py @@ -24,11 +24,9 @@ import tempfile import warnings -from test.test_support import check_warnings -from test.test_support import captured_stdout +from test.test_support import captured_stdout, check_warnings, run_unittest -from distutils.command.sdist import sdist -from distutils.command.sdist import show_formats +from distutils.command.sdist import sdist, show_formats from distutils.core import Distribution from distutils.tests.test_config import PyPIRCCommandTestCase from distutils.errors import DistutilsExecError, DistutilsOptionError @@ -372,7 +370,7 @@ # adding a file self.write_file((self.tmp_dir, 'somecode', 'doc2.txt'), '#') - # make sure build_py is reinitinialized, like a fresh run + # make sure build_py is reinitialized, like a fresh run build_py = dist.get_command_obj('build_py') build_py.finalized = False build_py.ensure_finalized() @@ -390,6 +388,7 @@ self.assertEqual(len(manifest2), 6) self.assertIn('doc2.txt', manifest2[-1]) + @unittest.skipUnless(zlib, "requires zlib") def test_manifest_marker(self): # check that autogenerated MANIFESTs have a marker dist, cmd = self.get_cmd() @@ -406,6 +405,7 @@ self.assertEqual(manifest[0], '# file GENERATED by distutils, do NOT edit') + @unittest.skipUnless(zlib, "requires zlib") def test_manual_manifest(self): # check that a MANIFEST without a marker is left alone dist, cmd = self.get_cmd() @@ -426,4 +426,4 @@ return unittest.makeSuite(SDistTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_spawn.py b/lib-python/2.7/distutils/tests/test_spawn.py --- a/lib-python/2.7/distutils/tests/test_spawn.py +++ b/lib-python/2.7/distutils/tests/test_spawn.py @@ -2,7 +2,7 @@ import unittest import os import time -from test.test_support import captured_stdout +from test.test_support import captured_stdout, run_unittest from distutils.spawn import _nt_quote_args from distutils.spawn import spawn, find_executable @@ -57,4 +57,4 @@ return unittest.makeSuite(SpawnTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_text_file.py b/lib-python/2.7/distutils/tests/test_text_file.py --- a/lib-python/2.7/distutils/tests/test_text_file.py +++ b/lib-python/2.7/distutils/tests/test_text_file.py @@ -3,6 +3,7 @@ import unittest from distutils.text_file import TextFile from distutils.tests import support +from test.test_support import run_unittest TEST_DATA = """# test file @@ -103,4 +104,4 @@ return unittest.makeSuite(TextFileTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_unixccompiler.py b/lib-python/2.7/distutils/tests/test_unixccompiler.py --- a/lib-python/2.7/distutils/tests/test_unixccompiler.py +++ b/lib-python/2.7/distutils/tests/test_unixccompiler.py @@ -1,6 +1,7 @@ """Tests for distutils.unixccompiler.""" import sys import unittest +from test.test_support import run_unittest from distutils import sysconfig from distutils.unixccompiler import UnixCCompiler @@ -126,4 +127,4 @@ return unittest.makeSuite(UnixCCompilerTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_upload.py b/lib-python/2.7/distutils/tests/test_upload.py --- a/lib-python/2.7/distutils/tests/test_upload.py +++ b/lib-python/2.7/distutils/tests/test_upload.py @@ -1,14 +1,13 @@ +# -*- encoding: utf8 -*- """Tests for distutils.command.upload.""" -# -*- encoding: utf8 -*- -import sys import os import unittest +from test.test_support import run_unittest from distutils.command import upload as upload_mod from distutils.command.upload import upload from distutils.core import Distribution -from distutils.tests import support from distutils.tests.test_config import PYPIRC, PyPIRCCommandTestCase PYPIRC_LONG_PASSWORD = """\ @@ -129,4 +128,4 @@ return unittest.makeSuite(uploadTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_util.py b/lib-python/2.7/distutils/tests/test_util.py --- a/lib-python/2.7/distutils/tests/test_util.py +++ b/lib-python/2.7/distutils/tests/test_util.py @@ -1,6 +1,7 @@ """Tests for distutils.util.""" import sys import unittest +from test.test_support import run_unittest from distutils.errors import DistutilsPlatformError, DistutilsByteCompileError from distutils.util import byte_compile @@ -21,4 +22,4 @@ return unittest.makeSuite(UtilTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_version.py b/lib-python/2.7/distutils/tests/test_version.py --- a/lib-python/2.7/distutils/tests/test_version.py +++ b/lib-python/2.7/distutils/tests/test_version.py @@ -2,6 +2,7 @@ import unittest from distutils.version import LooseVersion from distutils.version import StrictVersion +from test.test_support import run_unittest class VersionTestCase(unittest.TestCase): @@ -67,4 +68,4 @@ return unittest.makeSuite(VersionTestCase) if __name__ == "__main__": - unittest.main(defaultTest="test_suite") + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/tests/test_versionpredicate.py b/lib-python/2.7/distutils/tests/test_versionpredicate.py --- a/lib-python/2.7/distutils/tests/test_versionpredicate.py +++ b/lib-python/2.7/distutils/tests/test_versionpredicate.py @@ -4,6 +4,10 @@ import distutils.versionpredicate import doctest +from test.test_support import run_unittest def test_suite(): return doctest.DocTestSuite(distutils.versionpredicate) + +if __name__ == '__main__': + run_unittest(test_suite()) diff --git a/lib-python/2.7/distutils/util.py b/lib-python/2.7/distutils/util.py --- a/lib-python/2.7/distutils/util.py +++ b/lib-python/2.7/distutils/util.py @@ -97,9 +97,7 @@ from distutils.sysconfig import get_config_vars cfgvars = get_config_vars() - macver = os.environ.get('MACOSX_DEPLOYMENT_TARGET') - if not macver: - macver = cfgvars.get('MACOSX_DEPLOYMENT_TARGET') + macver = cfgvars.get('MACOSX_DEPLOYMENT_TARGET') if 1: # Always calculate the release of the running machine, diff --git a/lib-python/2.7/doctest.py b/lib-python/2.7/doctest.py --- a/lib-python/2.7/doctest.py +++ b/lib-python/2.7/doctest.py @@ -1217,7 +1217,7 @@ # Process each example. for examplenum, example in enumerate(test.examples): - # If REPORT_ONLY_FIRST_FAILURE is set, then supress + # If REPORT_ONLY_FIRST_FAILURE is set, then suppress # reporting after the first failure. quiet = (self.optionflags & REPORT_ONLY_FIRST_FAILURE and failures > 0) @@ -2186,7 +2186,7 @@ caller can catch the errors and initiate post-mortem debugging. The DocTestCase provides a debug method that raises - UnexpectedException errors if there is an unexepcted + UnexpectedException errors if there is an unexpected exception: >>> test = DocTestParser().get_doctest('>>> raise KeyError\n42', diff --git a/lib-python/2.7/email/charset.py b/lib-python/2.7/email/charset.py --- a/lib-python/2.7/email/charset.py +++ b/lib-python/2.7/email/charset.py @@ -209,7 +209,7 @@ input_charset = unicode(input_charset, 'ascii') except UnicodeError: raise errors.CharsetError(input_charset) - input_charset = input_charset.lower() + input_charset = input_charset.lower().encode('ascii') # Set the input charset after filtering through the aliases and/or codecs if not (input_charset in ALIASES or input_charset in CHARSETS): try: diff --git a/lib-python/2.7/email/generator.py b/lib-python/2.7/email/generator.py --- a/lib-python/2.7/email/generator.py +++ b/lib-python/2.7/email/generator.py @@ -202,18 +202,13 @@ g = self.clone(s) g.flatten(part, unixfrom=False) msgtexts.append(s.getvalue()) - # Now make sure the boundary we've selected doesn't appear in any of - # the message texts. - alltext = NL.join(msgtexts) # BAW: What about boundaries that are wrapped in double-quotes? - boundary = msg.get_boundary(failobj=_make_boundary(alltext)) - # If we had to calculate a new boundary because the body text - # contained that string, set the new boundary. We don't do it - # unconditionally because, while set_boundary() preserves order, it - # doesn't preserve newlines/continuations in headers. This is no big - # deal in practice, but turns out to be inconvenient for the unittest - # suite. - if msg.get_boundary() != boundary: + boundary = msg.get_boundary() + if not boundary: + # Create a boundary that doesn't appear in any of the + # message texts. + alltext = NL.join(msgtexts) + boundary = _make_boundary(alltext) msg.set_boundary(boundary) # If there's a preamble, write it out, with a trailing CRLF if msg.preamble is not None: @@ -292,7 +287,7 @@ _FMT = '[Non-text (%(type)s) part of message omitted, filename %(filename)s]' class DecodedGenerator(Generator): - """Generator a text representation of a message. + """Generates a text representation of a message. Like the Generator base class, except that non-text parts are substituted with a format string representing the part. diff --git a/lib-python/2.7/email/header.py b/lib-python/2.7/email/header.py --- a/lib-python/2.7/email/header.py +++ b/lib-python/2.7/email/header.py @@ -47,6 +47,10 @@ # For use with .match() fcre = re.compile(r'[\041-\176]+:$') +# Find a header embedded in a putative header value. Used to check for +# header injection attack. +_embeded_header = re.compile(r'\n[^ \t]+:') + # Helpers @@ -403,7 +407,11 @@ newchunks += self._split(s, charset, targetlen, splitchars) lastchunk, lastcharset = newchunks[-1] lastlen = lastcharset.encoded_header_len(lastchunk) - return self._encode_chunks(newchunks, maxlinelen) + value = self._encode_chunks(newchunks, maxlinelen) + if _embeded_header.search(value): + raise HeaderParseError("header value appears to contain " + "an embedded header: {!r}".format(value)) + return value diff --git a/lib-python/2.7/email/message.py b/lib-python/2.7/email/message.py --- a/lib-python/2.7/email/message.py +++ b/lib-python/2.7/email/message.py @@ -38,7 +38,9 @@ def _formatparam(param, value=None, quote=True): """Convenience function to format and return a key=value pair. - This will quote the value if needed or if quote is true. + This will quote the value if needed or if quote is true. If value is a + three tuple (charset, language, value), it will be encoded according + to RFC2231 rules. """ if value is not None and len(value) > 0: # A tuple is used for RFC 2231 encoded parameter values where items @@ -97,7 +99,7 @@ objects, otherwise it is a string. Message objects implement part of the `mapping' interface, which assumes - there is exactly one occurrance of the header per message. Some headers + there is exactly one occurrence of the header per message. Some headers do in fact appear multiple times (e.g. Received) and for those headers, you must use the explicit API to set or get all the headers. Not all of the mapping methods are implemented. @@ -286,7 +288,7 @@ Return None if the header is missing instead of raising an exception. Note that if the header appeared multiple times, exactly which - occurrance gets returned is undefined. Use get_all() to get all + occurrence gets returned is undefined. Use get_all() to get all the values matching a header field name. """ return self.get(name) @@ -389,7 +391,10 @@ name is the header field to add. keyword arguments can be used to set additional parameters for the header field, with underscores converted to dashes. Normally the parameter will be added as key="value" unless - value is None, in which case only the key will be added. + value is None, in which case only the key will be added. If a + parameter value contains non-ASCII characters it must be specified as a + three-tuple of (charset, language, value), in which case it will be + encoded according to RFC2231 rules. Example: diff --git a/lib-python/2.7/email/mime/application.py b/lib-python/2.7/email/mime/application.py --- a/lib-python/2.7/email/mime/application.py +++ b/lib-python/2.7/email/mime/application.py @@ -17,7 +17,7 @@ _encoder=encoders.encode_base64, **_params): """Create an application/* type MIME document. - _data is a string containing the raw applicatoin data. + _data is a string containing the raw application data. _subtype is the MIME content type subtype, defaulting to 'octet-stream'. diff --git a/lib-python/2.7/email/test/data/msg_26.txt b/lib-python/2.7/email/test/data/msg_26.txt --- a/lib-python/2.7/email/test/data/msg_26.txt +++ b/lib-python/2.7/email/test/data/msg_26.txt @@ -42,4 +42,4 @@ MzMAAAAACH97tzAAAAALu3c3gAAAAAAL+7tzDABAu7f7cAAAAAAACA+3MA7EQAv/sIAA AAAAAAAIAAAAAAAAAIAAAAAA ---1618492860--2051301190--113853680-- +--1618492860--2051301190--113853680-- \ No newline at end of file diff --git a/lib-python/2.7/email/test/test_email.py b/lib-python/2.7/email/test/test_email.py --- a/lib-python/2.7/email/test/test_email.py +++ b/lib-python/2.7/email/test/test_email.py @@ -179,6 +179,17 @@ self.assertRaises(Errors.HeaderParseError, msg.set_boundary, 'BOUNDARY') + def test_make_boundary(self): + msg = MIMEMultipart('form-data') + # Note that when the boundary gets created is an implementation + # detail and might change. + self.assertEqual(msg.items()[0][1], 'multipart/form-data') + # Trigger creation of boundary + msg.as_string() + self.assertEqual(msg.items()[0][1][:33], + 'multipart/form-data; boundary="==') + # XXX: there ought to be tests of the uniqueness of the boundary, too. + def test_message_rfc822_only(self): # Issue 7970: message/rfc822 not in multipart parsed by # HeaderParser caused an exception when flattened. @@ -542,6 +553,17 @@ msg.set_charset(u'us-ascii') self.assertEqual('us-ascii', msg.get_content_charset()) + # Issue 5871: reject an attempt to embed a header inside a header value + # (header injection attack). + def test_embeded_header_via_Header_rejected(self): + msg = Message() + msg['Dummy'] = Header('dummy\nX-Injected-Header: test') + self.assertRaises(Errors.HeaderParseError, msg.as_string) + + def test_embeded_header_via_string_rejected(self): + msg = Message() + msg['Dummy'] = 'dummy\nX-Injected-Header: test' + self.assertRaises(Errors.HeaderParseError, msg.as_string) # Test the email.Encoders module @@ -3113,6 +3135,28 @@ s = 'Subject: =?EUC-KR?B?CSixpLDtKSC/7Liuvsax4iC6uLmwMcijIKHaILzSwd/H0SC8+LCjwLsgv7W/+Mj3I ?=' raises(Errors.HeaderParseError, decode_header, s) + # Issue 1078919 + def test_ascii_add_header(self): + msg = Message() + msg.add_header('Content-Disposition', 'attachment', + filename='bud.gif') + self.assertEqual('attachment; filename="bud.gif"', + msg['Content-Disposition']) + + def test_nonascii_add_header_via_triple(self): + msg = Message() + msg.add_header('Content-Disposition', 'attachment', + filename=('iso-8859-1', '', 'Fu\xdfballer.ppt')) + self.assertEqual( + 'attachment; filename*="iso-8859-1\'\'Fu%DFballer.ppt"', + msg['Content-Disposition']) + + def test_encode_unaliased_charset(self): + # Issue 1379416: when the charset has no output conversion, + # output was accidentally getting coerced to unicode. + res = Header('abc','iso-8859-2').encode() + self.assertEqual(res, '=?iso-8859-2?q?abc?=') + self.assertIsInstance(res, str) # Test RFC 2231 header parameters (en/de)coding diff --git a/lib-python/2.7/ftplib.py b/lib-python/2.7/ftplib.py --- a/lib-python/2.7/ftplib.py +++ b/lib-python/2.7/ftplib.py @@ -599,7 +599,7 @@ Usage example: >>> from ftplib import FTP_TLS >>> ftps = FTP_TLS('ftp.python.org') - >>> ftps.login() # login anonimously previously securing control channel + >>> ftps.login() # login anonymously previously securing control channel '230 Guest login ok, access restrictions apply.' >>> ftps.prot_p() # switch to secure data connection '200 Protection level set to P' diff --git a/lib-python/2.7/functools.py b/lib-python/2.7/functools.py --- a/lib-python/2.7/functools.py +++ b/lib-python/2.7/functools.py @@ -53,17 +53,17 @@ def total_ordering(cls): """Class decorator that fills in missing ordering methods""" convert = { - '__lt__': [('__gt__', lambda self, other: other < self), - ('__le__', lambda self, other: not other < self), + '__lt__': [('__gt__', lambda self, other: not (self < other or self == other)), + ('__le__', lambda self, other: self < other or self == other), ('__ge__', lambda self, other: not self < other)], - '__le__': [('__ge__', lambda self, other: other <= self), - ('__lt__', lambda self, other: not other <= self), + '__le__': [('__ge__', lambda self, other: not self <= other or self == other), + ('__lt__', lambda self, other: self <= other and not self == other), ('__gt__', lambda self, other: not self <= other)], - '__gt__': [('__lt__', lambda self, other: other > self), - ('__ge__', lambda self, other: not other > self), + '__gt__': [('__lt__', lambda self, other: not (self > other or self == other)), + ('__ge__', lambda self, other: self > other or self == other), ('__le__', lambda self, other: not self > other)], - '__ge__': [('__le__', lambda self, other: other >= self), - ('__gt__', lambda self, other: not other >= self), + '__ge__': [('__le__', lambda self, other: (not self >= other) or self == other), + ('__gt__', lambda self, other: self >= other and not self == other), ('__lt__', lambda self, other: not self >= other)] } roots = set(dir(cls)) & set(convert) @@ -80,6 +80,7 @@ def cmp_to_key(mycmp): """Convert a cmp= function into a key= function""" class K(object): + __slots__ = ['obj'] def __init__(self, obj, *args): self.obj = obj def __lt__(self, other): diff --git a/lib-python/2.7/getpass.py b/lib-python/2.7/getpass.py --- a/lib-python/2.7/getpass.py +++ b/lib-python/2.7/getpass.py @@ -62,7 +62,7 @@ try: old = termios.tcgetattr(fd) # a copy to save new = old[:] - new[3] &= ~(termios.ECHO|termios.ISIG) # 3 == 'lflags' + new[3] &= ~termios.ECHO # 3 == 'lflags' tcsetattr_flags = termios.TCSAFLUSH if hasattr(termios, 'TCSASOFT'): tcsetattr_flags |= termios.TCSASOFT diff --git a/lib-python/2.7/gettext.py b/lib-python/2.7/gettext.py --- a/lib-python/2.7/gettext.py +++ b/lib-python/2.7/gettext.py @@ -316,7 +316,7 @@ # Note: we unconditionally convert both msgids and msgstrs to # Unicode using the character encoding specified in the charset # parameter of the Content-Type header. The gettext documentation - # strongly encourages msgids to be us-ascii, but some appliations + # strongly encourages msgids to be us-ascii, but some applications # require alternative encodings (e.g. Zope's ZCML and ZPT). For # traditional gettext applications, the msgid conversion will # cause no problems since us-ascii should always be a subset of diff --git a/lib-python/2.7/hashlib.py b/lib-python/2.7/hashlib.py --- a/lib-python/2.7/hashlib.py +++ b/lib-python/2.7/hashlib.py @@ -64,26 +64,29 @@ def __get_builtin_constructor(name): - if name in ('SHA1', 'sha1'): - import _sha - return _sha.new - elif name in ('MD5', 'md5'): - import _md5 - return _md5.new - elif name in ('SHA256', 'sha256', 'SHA224', 'sha224'): - import _sha256 - bs = name[3:] - if bs == '256': - return _sha256.sha256 - elif bs == '224': - return _sha256.sha224 - elif name in ('SHA512', 'sha512', 'SHA384', 'sha384'): - import _sha512 - bs = name[3:] - if bs == '512': - return _sha512.sha512 - elif bs == '384': - return _sha512.sha384 + try: + if name in ('SHA1', 'sha1'): + import _sha + return _sha.new + elif name in ('MD5', 'md5'): + import _md5 + return _md5.new + elif name in ('SHA256', 'sha256', 'SHA224', 'sha224'): + import _sha256 + bs = name[3:] + if bs == '256': + return _sha256.sha256 + elif bs == '224': + return _sha256.sha224 + elif name in ('SHA512', 'sha512', 'SHA384', 'sha384'): + import _sha512 + bs = name[3:] + if bs == '512': + return _sha512.sha512 + elif bs == '384': + return _sha512.sha384 + except ImportError: + pass # no extension module, this hash is unsupported. raise ValueError('unsupported hash type %s' % name) diff --git a/lib-python/2.7/heapq.py b/lib-python/2.7/heapq.py --- a/lib-python/2.7/heapq.py +++ b/lib-python/2.7/heapq.py @@ -133,6 +133,11 @@ from operator import itemgetter import bisect +def cmp_lt(x, y): + # Use __lt__ if available; otherwise, try __le__. + # In Py3.x, only __lt__ will be called. + return (x < y) if hasattr(x, '__lt__') else (not y <= x) + def heappush(heap, item): """Push item onto heap, maintaining the heap invariant.""" heap.append(item) @@ -167,13 +172,13 @@ def heappushpop(heap, item): """Fast version of a heappush followed by a heappop.""" - if heap and heap[0] < item: + if heap and cmp_lt(heap[0], item): item, heap[0] = heap[0], item _siftup(heap, 0) return item def heapify(x): - """Transform list into a heap, in-place, in O(len(heap)) time.""" + """Transform list into a heap, in-place, in O(len(x)) time.""" n = len(x) # Transform bottom-up. The largest index there's any point to looking at # is the largest with a child index in-range, so must have 2*i + 1 < n, @@ -215,11 +220,10 @@ pop = result.pop los = result[-1] # los --> Largest of the nsmallest for elem in it: - if los <= elem: - continue - insort(result, elem) - pop() - los = result[-1] + if cmp_lt(elem, los): + insort(result, elem) + pop() + los = result[-1] return result # An alternative approach manifests the whole iterable in memory but # saves comparisons by heapifying all at once. Also, saves time @@ -240,7 +244,7 @@ while pos > startpos: parentpos = (pos - 1) >> 1 parent = heap[parentpos] - if newitem < parent: + if cmp_lt(newitem, parent): heap[pos] = parent pos = parentpos continue @@ -295,7 +299,7 @@ while childpos < endpos: # Set childpos to index of smaller child. rightpos = childpos + 1 - if rightpos < endpos and not heap[childpos] < heap[rightpos]: + if rightpos < endpos and not cmp_lt(heap[childpos], heap[rightpos]): childpos = rightpos # Move the smaller child up. heap[pos] = heap[childpos] @@ -364,7 +368,7 @@ return [min(chain(head, it))] return [min(chain(head, it), key=key)] - # When n>=size, it's faster to use sort() + # When n>=size, it's faster to use sorted() try: size = len(iterable) except (TypeError, AttributeError): @@ -402,7 +406,7 @@ return [max(chain(head, it))] return [max(chain(head, it), key=key)] - # When n>=size, it's faster to use sort() + # When n>=size, it's faster to use sorted() try: size = len(iterable) except (TypeError, AttributeError): diff --git a/lib-python/2.7/httplib.py b/lib-python/2.7/httplib.py --- a/lib-python/2.7/httplib.py +++ b/lib-python/2.7/httplib.py @@ -212,6 +212,9 @@ # maximal amount of data to read at one time in _safe_read MAXAMOUNT = 1048576 +# maximal line length when calling readline(). +_MAXLINE = 65536 + class HTTPMessage(mimetools.Message): def addheader(self, key, value): @@ -274,7 +277,9 @@ except IOError: startofline = tell = None self.seekable = 0 - line = self.fp.readline() + line = self.fp.readline(_MAXLINE + 1) + if len(line) > _MAXLINE: + raise LineTooLong("header line") if not line: self.status = 'EOF in headers' break @@ -404,7 +409,10 @@ break # skip the header from the 100 response while True: - skip = self.fp.readline().strip() + skip = self.fp.readline(_MAXLINE + 1) + if len(skip) > _MAXLINE: + raise LineTooLong("header line") + skip = skip.strip() if not skip: break if self.debuglevel > 0: @@ -563,7 +571,9 @@ value = [] while True: if chunk_left is None: - line = self.fp.readline() + line = self.fp.readline(_MAXLINE + 1) + if len(line) > _MAXLINE: + raise LineTooLong("chunk size") i = line.find(';') if i >= 0: line = line[:i] # strip chunk-extensions @@ -598,7 +608,9 @@ # read and discard trailer up to the CRLF terminator ### note: we shouldn't have any trailers! while True: - line = self.fp.readline() + line = self.fp.readline(_MAXLINE + 1) + if len(line) > _MAXLINE: + raise LineTooLong("trailer line") if not line: # a vanishingly small number of sites EOF without # sending the trailer @@ -730,7 +742,9 @@ raise socket.error("Tunnel connection failed: %d %s" % (code, message.strip())) while True: - line = response.fp.readline() + line = response.fp.readline(_MAXLINE + 1) + if len(line) > _MAXLINE: + raise LineTooLong("header line") if line == '\r\n': break @@ -790,7 +804,7 @@ del self._buffer[:] # If msg and message_body are sent in a single send() call, # it will avoid performance problems caused by the interaction - # between delayed ack and the Nagle algorithim. + # between delayed ack and the Nagle algorithm. if isinstance(message_body, str): msg += message_body message_body = None @@ -1233,6 +1247,11 @@ self.args = line, self.line = line +class LineTooLong(HTTPException): + def __init__(self, line_type): + HTTPException.__init__(self, "got more than %d bytes when reading %s" + % (_MAXLINE, line_type)) + # for backwards compatibility error = HTTPException diff --git a/lib-python/2.7/idlelib/Bindings.py b/lib-python/2.7/idlelib/Bindings.py --- a/lib-python/2.7/idlelib/Bindings.py +++ b/lib-python/2.7/idlelib/Bindings.py @@ -98,14 +98,6 @@ # menu del menudefs[-1][1][0:2] - menudefs.insert(0, - ('application', [ - ('About IDLE', '<<about-idle>>'), - None, - ('_Preferences....', '<<open-config-dialog>>'), - ])) - - default_keydefs = idleConf.GetCurrentKeySet() del sys diff --git a/lib-python/2.7/idlelib/EditorWindow.py b/lib-python/2.7/idlelib/EditorWindow.py --- a/lib-python/2.7/idlelib/EditorWindow.py +++ b/lib-python/2.7/idlelib/EditorWindow.py @@ -48,6 +48,21 @@ path = module.__path__ except AttributeError: raise ImportError, 'No source for module ' + module.__name__ + if descr[2] != imp.PY_SOURCE: + # If all of the above fails and didn't raise an exception,fallback + # to a straight import which can find __init__.py in a package. + m = __import__(fullname) + try: + filename = m.__file__ + except AttributeError: + pass + else: + file = None + base, ext = os.path.splitext(filename) + if ext == '.pyc': + ext = '.py' + filename = base + ext + descr = filename, None, imp.PY_SOURCE return file, filename, descr class EditorWindow(object): @@ -102,8 +117,8 @@ self.top = top = WindowList.ListedToplevel(root, menu=self.menubar) if flist: self.tkinter_vars = flist.vars - #self.top.instance_dict makes flist.inversedict avalable to - #configDialog.py so it can access all EditorWindow instaces + #self.top.instance_dict makes flist.inversedict available to + #configDialog.py so it can access all EditorWindow instances self.top.instance_dict = flist.inversedict else: self.tkinter_vars = {} # keys: Tkinter event names @@ -136,6 +151,14 @@ if macosxSupport.runningAsOSXApp(): # Command-W on editorwindows doesn't work without this. text.bind('<<close-window>>', self.close_event) + # Some OS X systems have only one mouse button, + # so use control-click for pulldown menus there. + # (Note, AquaTk defines <2> as the right button if + # present and the Tk Text widget already binds <2>.) + text.bind("<Control-Button-1>",self.right_menu_event) + else: + # Elsewhere, use right-click for pulldown menus. + text.bind("<3>",self.right_menu_event) text.bind("<<cut>>", self.cut) text.bind("<<copy>>", self.copy) text.bind("<<paste>>", self.paste) @@ -154,7 +177,6 @@ text.bind("<<find-selection>>", self.find_selection_event) text.bind("<<replace>>", self.replace_event) text.bind("<<goto-line>>", self.goto_line_event) - text.bind("<3>", self.right_menu_event) text.bind("<<smart-backspace>>",self.smart_backspace_event) text.bind("<<newline-and-indent>>",self.newline_and_indent_event) text.bind("<<smart-indent>>",self.smart_indent_event) @@ -300,13 +322,13 @@ return "break" def home_callback(self, event): - if (event.state & 12) != 0 and event.keysym == "Home": - # state&1==shift, state&4==control, state&8==alt - return # <Modifier-Home>; fall back to class binding - + if (event.state & 4) != 0 and event.keysym == "Home": + # state&4==Control. If <Control-Home>, use the Tk binding. + return if self.text.index("iomark") and \ self.text.compare("iomark", "<=", "insert lineend") and \ self.text.compare("insert linestart", "<=", "iomark"): + # In Shell on input line, go to just after prompt insertpt = int(self.text.index("iomark").split(".")[1]) else: line = self.text.get("insert linestart", "insert lineend") @@ -315,30 +337,27 @@ break else: insertpt=len(line) - lineat = int(self.text.index("insert").split('.')[1]) - if insertpt == lineat: insertpt = 0 - dest = "insert linestart+"+str(insertpt)+"c" - if (event.state&1) == 0: - # shift not pressed + # shift was not pressed self.text.tag_remove("sel", "1.0", "end") else: if not self.text.index("sel.first"): - self.text.mark_set("anchor","insert") - + self.text.mark_set("my_anchor", "insert") # there was no previous selection + else: + if self.text.compare(self.text.index("sel.first"), "<", self.text.index("insert")): + self.text.mark_set("my_anchor", "sel.first") # extend back + else: + self.text.mark_set("my_anchor", "sel.last") # extend forward first = self.text.index(dest) - last = self.text.index("anchor") - + last = self.text.index("my_anchor") if self.text.compare(first,">",last): first,last = last,first - self.text.tag_remove("sel", "1.0", "end") self.text.tag_add("sel", first, last) - self.text.mark_set("insert", dest) self.text.see("insert") return "break" @@ -385,7 +404,7 @@ menudict[name] = menu = Menu(mbar, name=name) mbar.add_cascade(label=label, menu=menu, underline=underline) - if macosxSupport.runningAsOSXApp(): + if macosxSupport.isCarbonAquaTk(self.root): # Insert the application menu menudict['application'] = menu = Menu(mbar, name='apple') mbar.add_cascade(label='IDLE', menu=menu) @@ -445,7 +464,11 @@ def python_docs(self, event=None): if sys.platform[:3] == 'win': - os.startfile(self.help_url) + try: + os.startfile(self.help_url) + except WindowsError as why: + tkMessageBox.showerror(title='Document Start Failure', + message=str(why), parent=self.text) else: webbrowser.open(self.help_url) return "break" @@ -740,9 +763,13 @@ "Create a callback with the helpfile value frozen at definition time" def display_extra_help(helpfile=helpfile): if not helpfile.startswith(('www', 'http')): - url = os.path.normpath(helpfile) + helpfile = os.path.normpath(helpfile) if sys.platform[:3] == 'win': - os.startfile(helpfile) + try: + os.startfile(helpfile) + except WindowsError as why: + tkMessageBox.showerror(title='Document Start Failure', + message=str(why), parent=self.text) else: webbrowser.open(helpfile) return display_extra_help @@ -1526,7 +1553,12 @@ def get_accelerator(keydefs, eventname): keylist = keydefs.get(eventname) - if not keylist: + # issue10940: temporary workaround to prevent hang with OS X Cocoa Tk 8.5 + # if not keylist: + if (not keylist) or (macosxSupport.runningAsOSXApp() and eventname in { + "<<open-module>>", + "<<goto-line>>", + "<<change-indentwidth>>"}): return "" s = keylist[0] s = re.sub(r"-[a-z]\b", lambda m: m.group().upper(), s) diff --git a/lib-python/2.7/idlelib/FileList.py b/lib-python/2.7/idlelib/FileList.py --- a/lib-python/2.7/idlelib/FileList.py +++ b/lib-python/2.7/idlelib/FileList.py @@ -43,7 +43,7 @@ def new(self, filename=None): return self.EditorWindow(self, filename) - def close_all_callback(self, event): + def close_all_callback(self, *args, **kwds): for edit in self.inversedict.keys(): reply = edit.close() if reply == "cancel": diff --git a/lib-python/2.7/idlelib/FormatParagraph.py b/lib-python/2.7/idlelib/FormatParagraph.py --- a/lib-python/2.7/idlelib/FormatParagraph.py +++ b/lib-python/2.7/idlelib/FormatParagraph.py @@ -54,7 +54,7 @@ # If the block ends in a \n, we dont want the comment # prefix inserted after it. (Im not sure it makes sense to # reformat a comment block that isnt made of complete - # lines, but whatever!) Can't think of a clean soltution, + # lines, but whatever!) Can't think of a clean solution, # so we hack away block_suffix = "" if not newdata[-1]: diff --git a/lib-python/2.7/idlelib/HISTORY.txt b/lib-python/2.7/idlelib/HISTORY.txt --- a/lib-python/2.7/idlelib/HISTORY.txt +++ b/lib-python/2.7/idlelib/HISTORY.txt @@ -13,7 +13,7 @@ - New tarball released as a result of the 'revitalisation' of the IDLEfork project. -- This release requires python 2.1 or better. Compatability with earlier +- This release requires python 2.1 or better. Compatibility with earlier versions of python (especially ancient ones like 1.5x) is no longer a priority in IDLEfork development. diff --git a/lib-python/2.7/idlelib/IOBinding.py b/lib-python/2.7/idlelib/IOBinding.py --- a/lib-python/2.7/idlelib/IOBinding.py +++ b/lib-python/2.7/idlelib/IOBinding.py @@ -320,17 +320,20 @@ return "yes" message = "Do you want to save %s before closing?" % ( self.filename or "this untitled document") - m = tkMessageBox.Message( - title="Save On Close", - message=message, - icon=tkMessageBox.QUESTION, - type=tkMessageBox.YESNOCANCEL, - master=self.text) - reply = m.show() - if reply == "yes": + confirm = tkMessageBox.askyesnocancel( + title="Save On Close", + message=message, + default=tkMessageBox.YES, + master=self.text) + if confirm: + reply = "yes" self.save(None) if not self.get_saved(): reply = "cancel" + elif confirm is None: + reply = "cancel" + else: + reply = "no" self.text.focus_set() return reply @@ -339,7 +342,7 @@ self.save_as(event) else: if self.writefile(self.filename): - self.set_saved(1) + self.set_saved(True) try: self.editwin.store_file_breaks() except AttributeError: # may be a PyShell @@ -465,15 +468,12 @@ self.text.insert("end-1c", "\n") def print_window(self, event): - m = tkMessageBox.Message( - title="Print", - message="Print to Default Printer", - icon=tkMessageBox.QUESTION, - type=tkMessageBox.OKCANCEL, - default=tkMessageBox.OK, - master=self.text) - reply = m.show() - if reply != tkMessageBox.OK: + confirm = tkMessageBox.askokcancel( + title="Print", + message="Print to Default Printer", + default=tkMessageBox.OK, + master=self.text) + if not confirm: self.text.focus_set() return "break" tempfilename = None @@ -488,8 +488,8 @@ if not self.writefile(tempfilename): os.unlink(tempfilename) return "break" - platform=os.name - printPlatform=1 + platform = os.name + printPlatform = True if platform == 'posix': #posix platform command = idleConf.GetOption('main','General', 'print-command-posix') @@ -497,7 +497,7 @@ elif platform == 'nt': #win32 platform command = idleConf.GetOption('main','General','print-command-win') else: #no printing for this platform - printPlatform=0 + printPlatform = False if printPlatform: #we can try to print for this platform command = command % filename pipe = os.popen(command, "r") @@ -511,7 +511,7 @@ output = "Printing command: %s\n" % repr(command) + output tkMessageBox.showerror("Print status", output, master=self.text) else: #no printing for this platform - message="Printing is not enabled for this platform: %s" % platform + message = "Printing is not enabled for this platform: %s" % platform tkMessageBox.showinfo("Print status", message, master=self.text) if tempfilename: os.unlink(tempfilename) diff --git a/lib-python/2.7/idlelib/NEWS.txt b/lib-python/2.7/idlelib/NEWS.txt --- a/lib-python/2.7/idlelib/NEWS.txt +++ b/lib-python/2.7/idlelib/NEWS.txt @@ -1,3 +1,18 @@ +What's New in IDLE 2.7.2? +======================= + +*Release date: 29-May-2011* + +- Issue #6378: Further adjust idle.bat to start associated Python + +- Issue #11896: Save on Close failed despite selecting "Yes" in dialog. + +- <Home> toggle failing on Tk 8.5, causing IDLE exits and strange selection + behavior. Issue 4676. Improve selection extension behaviour. + +- <Home> toggle non-functional when NumLock set on Windows. Issue 3851. + + What's New in IDLE 2.7? ======================= @@ -21,7 +36,7 @@ - Tk 8.5 Text widget requires 'wordprocessor' tabstyle attr to handle mixed space/tab properly. Issue 5129, patch by Guilherme Polo. - + - Issue #3549: On MacOS the preferences menu was not present diff --git a/lib-python/2.7/idlelib/PyShell.py b/lib-python/2.7/idlelib/PyShell.py --- a/lib-python/2.7/idlelib/PyShell.py +++ b/lib-python/2.7/idlelib/PyShell.py @@ -1432,6 +1432,13 @@ shell.interp.prepend_syspath(script) shell.interp.execfile(script) + # Check for problematic OS X Tk versions and print a warning message + # in the IDLE shell window; this is less intrusive than always opening + # a separate window. + tkversionwarning = macosxSupport.tkVersionWarning(root) + if tkversionwarning: + shell.interp.runcommand(''.join(("print('", tkversionwarning, "')"))) + root.mainloop() root.destroy() diff --git a/lib-python/2.7/idlelib/ScriptBinding.py b/lib-python/2.7/idlelib/ScriptBinding.py --- a/lib-python/2.7/idlelib/ScriptBinding.py +++ b/lib-python/2.7/idlelib/ScriptBinding.py @@ -26,6 +26,7 @@ from idlelib import PyShell from idlelib.configHandler import idleConf +from idlelib import macosxSupport IDENTCHARS = string.ascii_letters + string.digits + "_" @@ -53,6 +54,9 @@ self.flist = self.editwin.flist self.root = self.editwin.root + if macosxSupport.runningAsOSXApp(): + self.editwin.text_frame.bind('<<run-module-event-2>>', self._run_module_event) + def check_module_event(self, event): filename = self.getfilename() if not filename: @@ -166,6 +170,19 @@ interp.runcode(code) return 'break' + if macosxSupport.runningAsOSXApp(): + # Tk-Cocoa in MacOSX is broken until at least + # Tk 8.5.9, and without this rather + # crude workaround IDLE would hang when a user + # tries to run a module using the keyboard shortcut + # (the menu item works fine). + _run_module_event = run_module_event + + def run_module_event(self, event): + self.editwin.text_frame.after(200, + lambda: self.editwin.text_frame.event_generate('<<run-module-event-2>>')) + return 'break' + def getfilename(self): """Get source filename. If not saved, offer to save (or create) file @@ -184,9 +201,9 @@ if autosave and filename: self.editwin.io.save(None) else: - reply = self.ask_save_dialog() + confirm = self.ask_save_dialog() self.editwin.text.focus_set() - if reply == "ok": + if confirm: self.editwin.io.save(None) filename = self.editwin.io.filename else: @@ -195,13 +212,11 @@ def ask_save_dialog(self): msg = "Source Must Be Saved\n" + 5*' ' + "OK to Save?" - mb = tkMessageBox.Message(title="Save Before Run or Check", - message=msg, - icon=tkMessageBox.QUESTION, - type=tkMessageBox.OKCANCEL, - default=tkMessageBox.OK, - master=self.editwin.text) - return mb.show() + confirm = tkMessageBox.askokcancel(title="Save Before Run or Check", + message=msg, + default=tkMessageBox.OK, + master=self.editwin.text) + return confirm def errorbox(self, title, message): # XXX This should really be a function of EditorWindow... diff --git a/lib-python/2.7/idlelib/config-keys.def b/lib-python/2.7/idlelib/config-keys.def --- a/lib-python/2.7/idlelib/config-keys.def +++ b/lib-python/2.7/idlelib/config-keys.def @@ -176,7 +176,7 @@ redo = <Shift-Command-Key-Z> close-window = <Command-Key-w> restart-shell = <Control-Key-F6> -save-window-as-file = <Command-Key-S> +save-window-as-file = <Shift-Command-Key-S> close-all-windows = <Command-Key-q> view-restart = <Key-F6> tabify-region = <Control-Key-5> @@ -208,7 +208,7 @@ open-module = <Command-Key-m> find-selection = <Shift-Command-Key-F3> python-context-help = <Shift-Key-F1> -save-copy-of-window-as-file = <Shift-Command-Key-s> +save-copy-of-window-as-file = <Option-Command-Key-s> open-window-from-file = <Command-Key-o> python-docs = <Key-F1> diff --git a/lib-python/2.7/idlelib/extend.txt b/lib-python/2.7/idlelib/extend.txt --- a/lib-python/2.7/idlelib/extend.txt +++ b/lib-python/2.7/idlelib/extend.txt @@ -18,7 +18,7 @@ An IDLE extension class is instantiated with a single argument, `editwin', an EditorWindow instance. The extension cannot assume much -about this argument, but it is guarateed to have the following instance +about this argument, but it is guaranteed to have the following instance variables: text a Text instance (a widget) diff --git a/lib-python/2.7/idlelib/idle.bat b/lib-python/2.7/idlelib/idle.bat --- a/lib-python/2.7/idlelib/idle.bat +++ b/lib-python/2.7/idlelib/idle.bat @@ -1,4 +1,4 @@ @echo off rem Start IDLE using the appropriate Python interpreter set CURRDIR=%~dp0 -start "%CURRDIR%..\..\pythonw.exe" "%CURRDIR%idle.pyw" %1 %2 %3 %4 %5 %6 %7 %8 %9 +start "IDLE" "%CURRDIR%..\..\pythonw.exe" "%CURRDIR%idle.pyw" %1 %2 %3 %4 %5 %6 %7 %8 %9 diff --git a/lib-python/2.7/idlelib/idlever.py b/lib-python/2.7/idlelib/idlever.py --- a/lib-python/2.7/idlelib/idlever.py +++ b/lib-python/2.7/idlelib/idlever.py @@ -1,1 +1,1 @@ -IDLE_VERSION = "2.7.1" +IDLE_VERSION = "2.7.2" diff --git a/lib-python/2.7/idlelib/macosxSupport.py b/lib-python/2.7/idlelib/macosxSupport.py --- a/lib-python/2.7/idlelib/macosxSupport.py +++ b/lib-python/2.7/idlelib/macosxSupport.py @@ -4,6 +4,7 @@ """ import sys import Tkinter +from os import path _appbundle = None @@ -19,10 +20,41 @@ _appbundle = (sys.platform == 'darwin' and '.app' in sys.executable) return _appbundle +_carbonaquatk = None + +def isCarbonAquaTk(root): + """ + Returns True if IDLE is using a Carbon Aqua Tk (instead of the + newer Cocoa Aqua Tk). + """ + global _carbonaquatk + if _carbonaquatk is None: + _carbonaquatk = (runningAsOSXApp() and + 'aqua' in root.tk.call('tk', 'windowingsystem') and + 'AppKit' not in root.tk.call('winfo', 'server', '.')) + return _carbonaquatk + +def tkVersionWarning(root): + """ + Returns a string warning message if the Tk version in use appears to + be one known to cause problems with IDLE. The Apple Cocoa-based Tk 8.5 + that was shipped with Mac OS X 10.6. + """ + + if (runningAsOSXApp() and + ('AppKit' in root.tk.call('winfo', 'server', '.')) and + (root.tk.call('info', 'patchlevel') == '8.5.7') ): + return (r"WARNING: The version of Tcl/Tk (8.5.7) in use may" + r" be unstable.\n" + r"Visit http://www.python.org/download/mac/tcltk/" + r" for current information.") + else: + return False + def addOpenEventSupport(root, flist): """ - This ensures that the application will respont to open AppleEvents, which - makes is feaseable to use IDLE as the default application for python files. + This ensures that the application will respond to open AppleEvents, which + makes is feasible to use IDLE as the default application for python files. """ def doOpenFile(*args): for fn in args: @@ -79,9 +111,6 @@ WindowList.add_windows_to_menu(menu) WindowList.register_callback(postwindowsmenu) - menudict['application'] = menu = Menu(menubar, name='apple') - menubar.add_cascade(label='IDLE', menu=menu) - def about_dialog(event=None): from idlelib import aboutDialog aboutDialog.AboutDialog(root, 'About IDLE') @@ -91,41 +120,45 @@ root.instance_dict = flist.inversedict configDialog.ConfigDialog(root, 'Settings') + def help_dialog(event=None): + from idlelib import textView + fn = path.join(path.abspath(path.dirname(__file__)), 'help.txt') + textView.view_file(root, 'Help', fn) root.bind('<<about-idle>>', about_dialog) root.bind('<<open-config-dialog>>', config_dialog) + root.createcommand('::tk::mac::ShowPreferences', config_dialog) if flist: root.bind('<<close-all-windows>>', flist.close_all_callback) + # The binding above doesn't reliably work on all versions of Tk + # on MacOSX. Adding command definition below does seem to do the + # right thing for now. + root.createcommand('exit', flist.close_all_callback) - ###check if Tk version >= 8.4.14; if so, use hard-coded showprefs binding - tkversion = root.tk.eval('info patchlevel') - # Note: we cannot check if the string tkversion >= '8.4.14', because - # the string '8.4.7' is greater than the string '8.4.14'. - if tuple(map(int, tkversion.split('.'))) >= (8, 4, 14): - Bindings.menudefs[0] = ('application', [ + if isCarbonAquaTk(root): + # for Carbon AquaTk, replace the default Tk apple menu + menudict['application'] = menu = Menu(menubar, name='apple') + menubar.add_cascade(label='IDLE', menu=menu) + Bindings.menudefs.insert(0, + ('application', [ ('About IDLE', '<<about-idle>>'), - None, - ]) - root.createcommand('::tk::mac::ShowPreferences', config_dialog) + None, + ])) + tkversion = root.tk.eval('info patchlevel') + if tuple(map(int, tkversion.split('.'))) < (8, 4, 14): + # for earlier AquaTk versions, supply a Preferences menu item + Bindings.menudefs[0][1].append( + ('_Preferences....', '<<open-config-dialog>>'), + ) else: - for mname, entrylist in Bindings.menudefs: - menu = menudict.get(mname) - if not menu: - continue - else: - for entry in entrylist: - if not entry: - menu.add_separator() - else: - label, eventname = entry - underline, label = prepstr(label) - accelerator = get_accelerator(Bindings.default_keydefs, - eventname) - def command(text=root, eventname=eventname): - text.event_generate(eventname) - menu.add_command(label=label, underline=underline, - command=command, accelerator=accelerator) + # assume Cocoa AquaTk + # replace default About dialog with About IDLE one + root.createcommand('tkAboutDialog', about_dialog) + # replace default "Help" item in Help menu + root.createcommand('::tk::mac::ShowHelp', help_dialog) + # remove redundant "IDLE Help" from menu + del Bindings.menudefs[-1][1][0] def setupApp(root, flist): """ diff --git a/lib-python/2.7/imaplib.py b/lib-python/2.7/imaplib.py --- a/lib-python/2.7/imaplib.py +++ b/lib-python/2.7/imaplib.py @@ -1158,28 +1158,17 @@ self.port = port self.sock = socket.create_connection((host, port)) self.sslobj = ssl.wrap_socket(self.sock, self.keyfile, self.certfile) + self.file = self.sslobj.makefile('rb') def read(self, size): """Read 'size' bytes from remote.""" - # sslobj.read() sometimes returns < size bytes - chunks = [] - read = 0 - while read < size: - data = self.sslobj.read(min(size-read, 16384)) - read += len(data) - chunks.append(data) - - return ''.join(chunks) + return self.file.read(size) def readline(self): """Read line from remote.""" - line = [] - while 1: - char = self.sslobj.read(1) - line.append(char) - if char in ("\n", ""): return ''.join(line) + return self.file.readline() def send(self, data): @@ -1195,6 +1184,7 @@ def shutdown(self): """Close I/O established in "open".""" + self.file.close() self.sock.close() @@ -1321,9 +1311,10 @@ 'Jul': 7, 'Aug': 8, 'Sep': 9, 'Oct': 10, 'Nov': 11, 'Dec': 12} def Internaldate2tuple(resp): - """Convert IMAP4 INTERNALDATE to UT. + """Parse an IMAP4 INTERNALDATE string. - Returns Python time module tuple. + Return corresponding local time. The return value is a + time.struct_time instance or None if the string has wrong format. """ mo = InternalDate.match(resp) @@ -1390,9 +1381,14 @@ def Time2Internaldate(date_time): - """Convert 'date_time' to IMAP4 INTERNALDATE representation. + """Convert date_time to IMAP4 INTERNALDATE representation. - Return string in form: '"DD-Mmm-YYYY HH:MM:SS +HHMM"' + Return string in form: '"DD-Mmm-YYYY HH:MM:SS +HHMM"'. The + date_time argument can be a number (int or float) representing + seconds since epoch (as returned by time.time()), a 9-tuple + representing local time (as returned by time.localtime()), or a + double-quoted string. In the last case, it is assumed to already + be in the correct format. """ if isinstance(date_time, (int, float)): diff --git a/lib-python/2.7/inspect.py b/lib-python/2.7/inspect.py --- a/lib-python/2.7/inspect.py +++ b/lib-python/2.7/inspect.py @@ -943,8 +943,14 @@ f_name, 'at most' if defaults else 'exactly', num_args, 'arguments' if num_args > 1 else 'argument', num_total)) elif num_args == 0 and num_total: - raise TypeError('%s() takes no arguments (%d given)' % - (f_name, num_total)) + if varkw: + if num_pos: + # XXX: We should use num_pos, but Python also uses num_total: + raise TypeError('%s() takes exactly 0 arguments ' + '(%d given)' % (f_name, num_total)) + else: + raise TypeError('%s() takes no arguments (%d given)' % + (f_name, num_total)) for arg in args: if isinstance(arg, str) and arg in named: if is_assigned(arg): diff --git a/lib-python/2.7/json/decoder.py b/lib-python/2.7/json/decoder.py --- a/lib-python/2.7/json/decoder.py +++ b/lib-python/2.7/json/decoder.py @@ -4,7 +4,7 @@ import sys import struct -from json.scanner import make_scanner +from json import scanner try: from _json import scanstring as c_scanstring except ImportError: @@ -161,6 +161,12 @@ nextchar = s[end:end + 1] # Trivial empty object if nextchar == '}': + if object_pairs_hook is not None: + result = object_pairs_hook(pairs) + return result, end + pairs = {} + if object_hook is not None: + pairs = object_hook(pairs) return pairs, end + 1 elif nextchar != '"': raise ValueError(errmsg("Expecting property name", s, end)) @@ -350,7 +356,7 @@ self.parse_object = JSONObject self.parse_array = JSONArray self.parse_string = scanstring - self.scan_once = make_scanner(self) + self.scan_once = scanner.make_scanner(self) def decode(self, s, _w=WHITESPACE.match): """Return the Python representation of ``s`` (a ``str`` or ``unicode`` diff --git a/lib-python/2.7/json/encoder.py b/lib-python/2.7/json/encoder.py --- a/lib-python/2.7/json/encoder.py +++ b/lib-python/2.7/json/encoder.py @@ -251,7 +251,7 @@ if (_one_shot and c_make_encoder is not None - and not self.indent and not self.sort_keys): + and self.indent is None and not self.sort_keys): _iterencode = c_make_encoder( markers, self.default, _encoder, self.indent, self.key_separator, self.item_separator, self.sort_keys, diff --git a/lib-python/2.7/json/tests/__init__.py b/lib-python/2.7/json/tests/__init__.py --- a/lib-python/2.7/json/tests/__init__.py +++ b/lib-python/2.7/json/tests/__init__.py @@ -1,7 +1,46 @@ import os import sys +import json +import doctest import unittest -import doctest + +from test import test_support + +# import json with and without accelerations +cjson = test_support.import_fresh_module('json', fresh=['_json']) +pyjson = test_support.import_fresh_module('json', blocked=['_json']) + +# create two base classes that will be used by the other tests +class PyTest(unittest.TestCase): + json = pyjson + loads = staticmethod(pyjson.loads) + dumps = staticmethod(pyjson.dumps) + + at unittest.skipUnless(cjson, 'requires _json') +class CTest(unittest.TestCase): + if cjson is not None: + json = cjson + loads = staticmethod(cjson.loads) + dumps = staticmethod(cjson.dumps) + +# test PyTest and CTest checking if the functions come from the right module +class TestPyTest(PyTest): + def test_pyjson(self): + self.assertEqual(self.json.scanner.make_scanner.__module__, + 'json.scanner') + self.assertEqual(self.json.decoder.scanstring.__module__, + 'json.decoder') + self.assertEqual(self.json.encoder.encode_basestring_ascii.__module__, + 'json.encoder') + +class TestCTest(CTest): + def test_cjson(self): + self.assertEqual(self.json.scanner.make_scanner.__module__, '_json') + self.assertEqual(self.json.decoder.scanstring.__module__, '_json') + self.assertEqual(self.json.encoder.c_make_encoder.__module__, '_json') + self.assertEqual(self.json.encoder.encode_basestring_ascii.__module__, + '_json') + here = os.path.dirname(__file__) @@ -17,12 +56,11 @@ return suite def additional_tests(): - import json - import json.encoder - import json.decoder suite = unittest.TestSuite() for mod in (json, json.encoder, json.decoder): suite.addTest(doctest.DocTestSuite(mod)) + suite.addTest(TestPyTest('test_pyjson')) + suite.addTest(TestCTest('test_cjson')) return suite def main(): diff --git a/lib-python/2.7/json/tests/test_check_circular.py b/lib-python/2.7/json/tests/test_check_circular.py --- a/lib-python/2.7/json/tests/test_check_circular.py +++ b/lib-python/2.7/json/tests/test_check_circular.py @@ -1,30 +1,34 @@ -from unittest import TestCase -import json +from json.tests import PyTest, CTest + def default_iterable(obj): return list(obj) -class TestCheckCircular(TestCase): +class TestCheckCircular(object): def test_circular_dict(self): dct = {} dct['a'] = dct - self.assertRaises(ValueError, json.dumps, dct) + self.assertRaises(ValueError, self.dumps, dct) def test_circular_list(self): lst = [] lst.append(lst) - self.assertRaises(ValueError, json.dumps, lst) + self.assertRaises(ValueError, self.dumps, lst) def test_circular_composite(self): dct2 = {} dct2['a'] = [] dct2['a'].append(dct2) - self.assertRaises(ValueError, json.dumps, dct2) + self.assertRaises(ValueError, self.dumps, dct2) def test_circular_default(self): - json.dumps([set()], default=default_iterable) - self.assertRaises(TypeError, json.dumps, [set()]) + self.dumps([set()], default=default_iterable) + self.assertRaises(TypeError, self.dumps, [set()]) def test_circular_off_default(self): - json.dumps([set()], default=default_iterable, check_circular=False) - self.assertRaises(TypeError, json.dumps, [set()], check_circular=False) + self.dumps([set()], default=default_iterable, check_circular=False) + self.assertRaises(TypeError, self.dumps, [set()], check_circular=False) + + +class TestPyCheckCircular(TestCheckCircular, PyTest): pass +class TestCCheckCircular(TestCheckCircular, CTest): pass diff --git a/lib-python/2.7/json/tests/test_decode.py b/lib-python/2.7/json/tests/test_decode.py --- a/lib-python/2.7/json/tests/test_decode.py +++ b/lib-python/2.7/json/tests/test_decode.py @@ -1,18 +1,17 @@ import decimal -from unittest import TestCase from StringIO import StringIO +from collections import OrderedDict +from json.tests import PyTest, CTest -import json -from collections import OrderedDict -class TestDecode(TestCase): +class TestDecode(object): def test_decimal(self): - rval = json.loads('1.1', parse_float=decimal.Decimal) + rval = self.loads('1.1', parse_float=decimal.Decimal) self.assertTrue(isinstance(rval, decimal.Decimal)) self.assertEqual(rval, decimal.Decimal('1.1')) def test_float(self): - rval = json.loads('1', parse_int=float) + rval = self.loads('1', parse_int=float) self.assertTrue(isinstance(rval, float)) self.assertEqual(rval, 1.0) @@ -20,22 +19,32 @@ # Several optimizations were made that skip over calls to # the whitespace regex, so this test is designed to try and # exercise the uncommon cases. The array cases are already covered. - rval = json.loads('{ "key" : "value" , "k":"v" }') + rval = self.loads('{ "key" : "value" , "k":"v" }') self.assertEqual(rval, {"key":"value", "k":"v"}) + def test_empty_objects(self): + self.assertEqual(self.loads('{}'), {}) + self.assertEqual(self.loads('[]'), []) + self.assertEqual(self.loads('""'), u"") + self.assertIsInstance(self.loads('""'), unicode) + def test_object_pairs_hook(self): s = '{"xkd":1, "kcw":2, "art":3, "hxm":4, "qrt":5, "pad":6, "hoy":7}' p = [("xkd", 1), ("kcw", 2), ("art", 3), ("hxm", 4), ("qrt", 5), ("pad", 6), ("hoy", 7)] - self.assertEqual(json.loads(s), eval(s)) - self.assertEqual(json.loads(s, object_pairs_hook=lambda x: x), p) - self.assertEqual(json.load(StringIO(s), - object_pairs_hook=lambda x: x), p) - od = json.loads(s, object_pairs_hook=OrderedDict) + self.assertEqual(self.loads(s), eval(s)) + self.assertEqual(self.loads(s, object_pairs_hook=lambda x: x), p) + self.assertEqual(self.json.load(StringIO(s), + object_pairs_hook=lambda x: x), p) + od = self.loads(s, object_pairs_hook=OrderedDict) self.assertEqual(od, OrderedDict(p)) self.assertEqual(type(od), OrderedDict) # the object_pairs_hook takes priority over the object_hook - self.assertEqual(json.loads(s, + self.assertEqual(self.loads(s, object_pairs_hook=OrderedDict, object_hook=lambda x: None), OrderedDict(p)) + + +class TestPyDecode(TestDecode, PyTest): pass +class TestCDecode(TestDecode, CTest): pass diff --git a/lib-python/2.7/json/tests/test_default.py b/lib-python/2.7/json/tests/test_default.py --- a/lib-python/2.7/json/tests/test_default.py +++ b/lib-python/2.7/json/tests/test_default.py @@ -1,9 +1,12 @@ -from unittest import TestCase +from json.tests import PyTest, CTest -import json -class TestDefault(TestCase): +class TestDefault(object): def test_default(self): self.assertEqual( - json.dumps(type, default=repr), - json.dumps(repr(type))) + self.dumps(type, default=repr), + self.dumps(repr(type))) + + +class TestPyDefault(TestDefault, PyTest): pass +class TestCDefault(TestDefault, CTest): pass diff --git a/lib-python/2.7/json/tests/test_dump.py b/lib-python/2.7/json/tests/test_dump.py --- a/lib-python/2.7/json/tests/test_dump.py +++ b/lib-python/2.7/json/tests/test_dump.py @@ -1,21 +1,23 @@ -from unittest import TestCase from cStringIO import StringIO +from json.tests import PyTest, CTest -import json -class TestDump(TestCase): +class TestDump(object): def test_dump(self): sio = StringIO() - json.dump({}, sio) + self.json.dump({}, sio) self.assertEqual(sio.getvalue(), '{}') def test_dumps(self): - self.assertEqual(json.dumps({}), '{}') + self.assertEqual(self.dumps({}), '{}') def test_encode_truefalse(self): - self.assertEqual(json.dumps( + self.assertEqual(self.dumps( {True: False, False: True}, sort_keys=True), '{"false": true, "true": false}') - self.assertEqual(json.dumps( + self.assertEqual(self.dumps( {2: 3.0, 4.0: 5L, False: 1, 6L: True}, sort_keys=True), '{"false": 1, "2": 3.0, "4.0": 5, "6": true}') + +class TestPyDump(TestDump, PyTest): pass +class TestCDump(TestDump, CTest): pass diff --git a/lib-python/2.7/json/tests/test_encode_basestring_ascii.py b/lib-python/2.7/json/tests/test_encode_basestring_ascii.py --- a/lib-python/2.7/json/tests/test_encode_basestring_ascii.py +++ b/lib-python/2.7/json/tests/test_encode_basestring_ascii.py @@ -1,8 +1,6 @@ -from unittest import TestCase +from collections import OrderedDict +from json.tests import PyTest, CTest -import json.encoder -from json import dumps -from collections import OrderedDict CASES = [ (u'/\\"\ucafe\ubabe\uab98\ufcde\ubcda\uef4a\x08\x0c\n\r\t`1~!@#$%^&*()_+-=[]{}|;:\',./<>?', '"/\\\\\\"\\ucafe\\ubabe\\uab98\\ufcde\\ubcda\\uef4a\\b\\f\\n\\r\\t`1~!@#$%^&*()_+-=[]{}|;:\',./<>?"'), @@ -23,19 +21,11 @@ (u'\u0123\u4567\u89ab\ucdef\uabcd\uef4a', '"\\u0123\\u4567\\u89ab\\ucdef\\uabcd\\uef4a"'), ] -class TestEncodeBaseStringAscii(TestCase): - def test_py_encode_basestring_ascii(self): - self._test_encode_basestring_ascii(json.encoder.py_encode_basestring_ascii) - - def test_c_encode_basestring_ascii(self): - if not json.encoder.c_encode_basestring_ascii: - return - self._test_encode_basestring_ascii(json.encoder.c_encode_basestring_ascii) - - def _test_encode_basestring_ascii(self, encode_basestring_ascii): - fname = encode_basestring_ascii.__name__ +class TestEncodeBasestringAscii(object): + def test_encode_basestring_ascii(self): + fname = self.json.encoder.encode_basestring_ascii.__name__ for input_string, expect in CASES: - result = encode_basestring_ascii(input_string) + result = self.json.encoder.encode_basestring_ascii(input_string) self.assertEqual(result, expect, '{0!r} != {1!r} for {2}({3!r})'.format( result, expect, fname, input_string)) @@ -43,5 +33,9 @@ def test_ordered_dict(self): # See issue 6105 items = [('one', 1), ('two', 2), ('three', 3), ('four', 4), ('five', 5)] - s = json.dumps(OrderedDict(items)) + s = self.dumps(OrderedDict(items)) self.assertEqual(s, '{"one": 1, "two": 2, "three": 3, "four": 4, "five": 5}') + + +class TestPyEncodeBasestringAscii(TestEncodeBasestringAscii, PyTest): pass +class TestCEncodeBasestringAscii(TestEncodeBasestringAscii, CTest): pass diff --git a/lib-python/2.7/json/tests/test_fail.py b/lib-python/2.7/json/tests/test_fail.py --- a/lib-python/2.7/json/tests/test_fail.py +++ b/lib-python/2.7/json/tests/test_fail.py @@ -1,6 +1,4 @@ -from unittest import TestCase - -import json +from json.tests import PyTest, CTest # Fri Dec 30 18:57:26 2005 JSONDOCS = [ @@ -61,15 +59,15 @@ 18: "spec doesn't specify any nesting limitations", } -class TestFail(TestCase): +class TestFail(object): def test_failures(self): for idx, doc in enumerate(JSONDOCS): idx = idx + 1 if idx in SKIPS: - json.loads(doc) + self.loads(doc) continue try: - json.loads(doc) + self.loads(doc) except ValueError: pass else: @@ -79,7 +77,11 @@ data = {'a' : 1, (1, 2) : 2} #This is for c encoder - self.assertRaises(TypeError, json.dumps, data) + self.assertRaises(TypeError, self.dumps, data) #This is for python encoder - self.assertRaises(TypeError, json.dumps, data, indent=True) + self.assertRaises(TypeError, self.dumps, data, indent=True) + + +class TestPyFail(TestFail, PyTest): pass +class TestCFail(TestFail, CTest): pass diff --git a/lib-python/2.7/json/tests/test_float.py b/lib-python/2.7/json/tests/test_float.py --- a/lib-python/2.7/json/tests/test_float.py +++ b/lib-python/2.7/json/tests/test_float.py @@ -1,19 +1,22 @@ import math -from unittest import TestCase +from json.tests import PyTest, CTest -import json -class TestFloat(TestCase): +class TestFloat(object): def test_floats(self): for num in [1617161771.7650001, math.pi, math.pi**100, math.pi**-100, 3.1]: - self.assertEqual(float(json.dumps(num)), num) - self.assertEqual(json.loads(json.dumps(num)), num) - self.assertEqual(json.loads(unicode(json.dumps(num))), num) + self.assertEqual(float(self.dumps(num)), num) + self.assertEqual(self.loads(self.dumps(num)), num) + self.assertEqual(self.loads(unicode(self.dumps(num))), num) def test_ints(self): for num in [1, 1L, 1<<32, 1<<64]: - self.assertEqual(json.dumps(num), str(num)) - self.assertEqual(int(json.dumps(num)), num) - self.assertEqual(json.loads(json.dumps(num)), num) - self.assertEqual(json.loads(unicode(json.dumps(num))), num) + self.assertEqual(self.dumps(num), str(num)) + self.assertEqual(int(self.dumps(num)), num) + self.assertEqual(self.loads(self.dumps(num)), num) + self.assertEqual(self.loads(unicode(self.dumps(num))), num) + + +class TestPyFloat(TestFloat, PyTest): pass +class TestCFloat(TestFloat, CTest): pass diff --git a/lib-python/2.7/json/tests/test_indent.py b/lib-python/2.7/json/tests/test_indent.py --- a/lib-python/2.7/json/tests/test_indent.py +++ b/lib-python/2.7/json/tests/test_indent.py @@ -1,9 +1,9 @@ -from unittest import TestCase +import textwrap +from StringIO import StringIO +from json.tests import PyTest, CTest -import json -import textwrap -class TestIndent(TestCase): +class TestIndent(object): def test_indent(self): h = [['blorpie'], ['whoops'], [], 'd-shtaeou', 'd-nthiouh', 'i-vhbjkhnth', {'nifty': 87}, {'field': 'yes', 'morefield': False} ] @@ -30,12 +30,31 @@ ]""") - d1 = json.dumps(h) - d2 = json.dumps(h, indent=2, sort_keys=True, separators=(',', ': ')) + d1 = self.dumps(h) + d2 = self.dumps(h, indent=2, sort_keys=True, separators=(',', ': ')) - h1 = json.loads(d1) - h2 = json.loads(d2) + h1 = self.loads(d1) + h2 = self.loads(d2) self.assertEqual(h1, h) self.assertEqual(h2, h) self.assertEqual(d2, expect) + + def test_indent0(self): + h = {3: 1} + def check(indent, expected): + d1 = self.dumps(h, indent=indent) + self.assertEqual(d1, expected) + + sio = StringIO() + self.json.dump(h, sio, indent=indent) + self.assertEqual(sio.getvalue(), expected) + + # indent=0 should emit newlines + check(0, '{\n"3": 1\n}') + # indent=None is more compact + check(None, '{"3": 1}') + + +class TestPyIndent(TestIndent, PyTest): pass +class TestCIndent(TestIndent, CTest): pass diff --git a/lib-python/2.7/json/tests/test_pass1.py b/lib-python/2.7/json/tests/test_pass1.py --- a/lib-python/2.7/json/tests/test_pass1.py +++ b/lib-python/2.7/json/tests/test_pass1.py @@ -1,6 +1,5 @@ -from unittest import TestCase +from json.tests import PyTest, CTest -import json # from http://json.org/JSON_checker/test/pass1.json JSON = r''' @@ -62,15 +61,19 @@ ,"rosebud"] ''' -class TestPass1(TestCase): +class TestPass1(object): def test_parse(self): # test in/out equivalence and parsing - res = json.loads(JSON) - out = json.dumps(res) - self.assertEqual(res, json.loads(out)) + res = self.loads(JSON) + out = self.dumps(res) + self.assertEqual(res, self.loads(out)) try: - json.dumps(res, allow_nan=False) + self.dumps(res, allow_nan=False) except ValueError: pass else: self.fail("23456789012E666 should be out of range") + + +class TestPyPass1(TestPass1, PyTest): pass +class TestCPass1(TestPass1, CTest): pass diff --git a/lib-python/2.7/json/tests/test_pass2.py b/lib-python/2.7/json/tests/test_pass2.py --- a/lib-python/2.7/json/tests/test_pass2.py +++ b/lib-python/2.7/json/tests/test_pass2.py @@ -1,14 +1,18 @@ -from unittest import TestCase -import json +from json.tests import PyTest, CTest + # from http://json.org/JSON_checker/test/pass2.json JSON = r''' [[[[[[[[[[[[[[[[[[["Not too deep"]]]]]]]]]]]]]]]]]]] ''' -class TestPass2(TestCase): +class TestPass2(object): def test_parse(self): # test in/out equivalence and parsing - res = json.loads(JSON) - out = json.dumps(res) - self.assertEqual(res, json.loads(out)) + res = self.loads(JSON) + out = self.dumps(res) + self.assertEqual(res, self.loads(out)) + + +class TestPyPass2(TestPass2, PyTest): pass +class TestCPass2(TestPass2, CTest): pass diff --git a/lib-python/2.7/json/tests/test_pass3.py b/lib-python/2.7/json/tests/test_pass3.py --- a/lib-python/2.7/json/tests/test_pass3.py +++ b/lib-python/2.7/json/tests/test_pass3.py @@ -1,6 +1,5 @@ -from unittest import TestCase +from json.tests import PyTest, CTest -import json # from http://json.org/JSON_checker/test/pass3.json JSON = r''' @@ -12,9 +11,14 @@ } ''' -class TestPass3(TestCase): + +class TestPass3(object): def test_parse(self): # test in/out equivalence and parsing - res = json.loads(JSON) - out = json.dumps(res) - self.assertEqual(res, json.loads(out)) + res = self.loads(JSON) + out = self.dumps(res) + self.assertEqual(res, self.loads(out)) + + +class TestPyPass3(TestPass3, PyTest): pass +class TestCPass3(TestPass3, CTest): pass diff --git a/lib-python/2.7/json/tests/test_recursion.py b/lib-python/2.7/json/tests/test_recursion.py --- a/lib-python/2.7/json/tests/test_recursion.py +++ b/lib-python/2.7/json/tests/test_recursion.py @@ -1,28 +1,16 @@ -from unittest import TestCase +from json.tests import PyTest, CTest -import json class JSONTestObject: pass -class RecursiveJSONEncoder(json.JSONEncoder): - recurse = False - def default(self, o): - if o is JSONTestObject: - if self.recurse: - return [JSONTestObject] - else: - return 'JSONTestObject' - return json.JSONEncoder.default(o) - - -class TestRecursion(TestCase): +class TestRecursion(object): def test_listrecursion(self): x = [] x.append(x) try: - json.dumps(x) + self.dumps(x) except ValueError: pass else: @@ -31,7 +19,7 @@ y = [x] x.append(y) try: - json.dumps(x) + self.dumps(x) except ValueError: pass else: @@ -39,13 +27,13 @@ y = [] x = [y, y] # ensure that the marker is cleared - json.dumps(x) + self.dumps(x) def test_dictrecursion(self): x = {} x["test"] = x try: - json.dumps(x) + self.dumps(x) except ValueError: pass else: @@ -53,9 +41,19 @@ x = {} y = {"a": x, "b": x} # ensure that the marker is cleared - json.dumps(x) + self.dumps(x) def test_defaultrecursion(self): + class RecursiveJSONEncoder(self.json.JSONEncoder): + recurse = False + def default(self, o): + if o is JSONTestObject: + if self.recurse: + return [JSONTestObject] + else: + return 'JSONTestObject' + return pyjson.JSONEncoder.default(o) + enc = RecursiveJSONEncoder() self.assertEqual(enc.encode(JSONTestObject), '"JSONTestObject"') enc.recurse = True @@ -65,3 +63,46 @@ pass else: self.fail("didn't raise ValueError on default recursion") + + + def test_highly_nested_objects_decoding(self): + # test that loading highly-nested objects doesn't segfault when C + # accelerations are used. See #12017 + # str + with self.assertRaises(RuntimeError): + self.loads('{"a":' * 100000 + '1' + '}' * 100000) + with self.assertRaises(RuntimeError): + self.loads('{"a":' * 100000 + '[1]' + '}' * 100000) + with self.assertRaises(RuntimeError): + self.loads('[' * 100000 + '1' + ']' * 100000) + # unicode + with self.assertRaises(RuntimeError): + self.loads(u'{"a":' * 100000 + u'1' + u'}' * 100000) + with self.assertRaises(RuntimeError): + self.loads(u'{"a":' * 100000 + u'[1]' + u'}' * 100000) + with self.assertRaises(RuntimeError): + self.loads(u'[' * 100000 + u'1' + u']' * 100000) + + def test_highly_nested_objects_encoding(self): + # See #12051 + l, d = [], {} + for x in xrange(100000): + l, d = [l], {'k':d} + with self.assertRaises(RuntimeError): + self.dumps(l) + with self.assertRaises(RuntimeError): + self.dumps(d) + + def test_endless_recursion(self): + # See #12051 + class EndlessJSONEncoder(self.json.JSONEncoder): + def default(self, o): + """If check_circular is False, this will keep adding another list.""" + return [o] + + with self.assertRaises(RuntimeError): + EndlessJSONEncoder(check_circular=False).encode(5j) + + +class TestPyRecursion(TestRecursion, PyTest): pass +class TestCRecursion(TestRecursion, CTest): pass diff --git a/lib-python/2.7/json/tests/test_scanstring.py b/lib-python/2.7/json/tests/test_scanstring.py --- a/lib-python/2.7/json/tests/test_scanstring.py +++ b/lib-python/2.7/json/tests/test_scanstring.py @@ -1,18 +1,10 @@ import sys -import decimal -from unittest import TestCase +from json.tests import PyTest, CTest -import json -import json.decoder -class TestScanString(TestCase): - def test_py_scanstring(self): - self._test_scanstring(json.decoder.py_scanstring) - - def test_c_scanstring(self): - self._test_scanstring(json.decoder.c_scanstring) - - def _test_scanstring(self, scanstring): +class TestScanstring(object): + def test_scanstring(self): + scanstring = self.json.decoder.scanstring self.assertEqual( scanstring('"z\\ud834\\udd20x"', 1, None, True), (u'z\U0001d120x', 16)) @@ -103,10 +95,15 @@ (u'Bad value', 12)) def test_issue3623(self): - self.assertRaises(ValueError, json.decoder.scanstring, b"xxx", 1, + self.assertRaises(ValueError, self.json.decoder.scanstring, b"xxx", 1, "xxx") self.assertRaises(UnicodeDecodeError, - json.encoder.encode_basestring_ascii, b"xx\xff") + self.json.encoder.encode_basestring_ascii, b"xx\xff") def test_overflow(self): - self.assertRaises(OverflowError, json.decoder.scanstring, b"xxx", sys.maxsize+1) + with self.assertRaises(OverflowError): + self.json.decoder.scanstring(b"xxx", sys.maxsize+1) + + +class TestPyScanstring(TestScanstring, PyTest): pass +class TestCScanstring(TestScanstring, CTest): pass diff --git a/lib-python/2.7/json/tests/test_separators.py b/lib-python/2.7/json/tests/test_separators.py --- a/lib-python/2.7/json/tests/test_separators.py +++ b/lib-python/2.7/json/tests/test_separators.py @@ -1,10 +1,8 @@ import textwrap -from unittest import TestCase +from json.tests import PyTest, CTest -import json - -class TestSeparators(TestCase): +class TestSeparators(object): def test_separators(self): h = [['blorpie'], ['whoops'], [], 'd-shtaeou', 'd-nthiouh', 'i-vhbjkhnth', {'nifty': 87}, {'field': 'yes', 'morefield': False} ] @@ -31,12 +29,16 @@ ]""") - d1 = json.dumps(h) - d2 = json.dumps(h, indent=2, sort_keys=True, separators=(' ,', ' : ')) + d1 = self.dumps(h) + d2 = self.dumps(h, indent=2, sort_keys=True, separators=(' ,', ' : ')) - h1 = json.loads(d1) - h2 = json.loads(d2) + h1 = self.loads(d1) + h2 = self.loads(d2) self.assertEqual(h1, h) self.assertEqual(h2, h) self.assertEqual(d2, expect) + + +class TestPySeparators(TestSeparators, PyTest): pass +class TestCSeparators(TestSeparators, CTest): pass diff --git a/lib-python/2.7/json/tests/test_speedups.py b/lib-python/2.7/json/tests/test_speedups.py --- a/lib-python/2.7/json/tests/test_speedups.py +++ b/lib-python/2.7/json/tests/test_speedups.py @@ -1,24 +1,23 @@ -import decimal -from unittest import TestCase +from json.tests import CTest -from json import decoder, encoder, scanner -class TestSpeedups(TestCase): +class TestSpeedups(CTest): def test_scanstring(self): - self.assertEqual(decoder.scanstring.__module__, "_json") - self.assertTrue(decoder.scanstring is decoder.c_scanstring) + self.assertEqual(self.json.decoder.scanstring.__module__, "_json") + self.assertIs(self.json.decoder.scanstring, self.json.decoder.c_scanstring) def test_encode_basestring_ascii(self): - self.assertEqual(encoder.encode_basestring_ascii.__module__, "_json") - self.assertTrue(encoder.encode_basestring_ascii is - encoder.c_encode_basestring_ascii) + self.assertEqual(self.json.encoder.encode_basestring_ascii.__module__, + "_json") + self.assertIs(self.json.encoder.encode_basestring_ascii, + self.json.encoder.c_encode_basestring_ascii) -class TestDecode(TestCase): +class TestDecode(CTest): def test_make_scanner(self): - self.assertRaises(AttributeError, scanner.c_make_scanner, 1) + self.assertRaises(AttributeError, self.json.scanner.c_make_scanner, 1) def test_make_encoder(self): - self.assertRaises(TypeError, encoder.c_make_encoder, + self.assertRaises(TypeError, self.json.encoder.c_make_encoder, None, "\xCD\x7D\x3D\x4E\x12\x4C\xF9\x79\xD7\x52\xBA\x82\xF2\x27\x4A\x7D\xA0\xCA\x75", None) diff --git a/lib-python/2.7/json/tests/test_unicode.py b/lib-python/2.7/json/tests/test_unicode.py --- a/lib-python/2.7/json/tests/test_unicode.py +++ b/lib-python/2.7/json/tests/test_unicode.py @@ -1,11 +1,10 @@ -from unittest import TestCase +from collections import OrderedDict +from json.tests import PyTest, CTest -import json -from collections import OrderedDict -class TestUnicode(TestCase): +class TestUnicode(object): def test_encoding1(self): - encoder = json.JSONEncoder(encoding='utf-8') + encoder = self.json.JSONEncoder(encoding='utf-8') u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' s = u.encode('utf-8') ju = encoder.encode(u) @@ -15,68 +14,72 @@ def test_encoding2(self): u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' s = u.encode('utf-8') - ju = json.dumps(u, encoding='utf-8') - js = json.dumps(s, encoding='utf-8') + ju = self.dumps(u, encoding='utf-8') + js = self.dumps(s, encoding='utf-8') self.assertEqual(ju, js) def test_encoding3(self): u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' - j = json.dumps(u) + j = self.dumps(u) self.assertEqual(j, '"\\u03b1\\u03a9"') def test_encoding4(self): u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' - j = json.dumps([u]) + j = self.dumps([u]) self.assertEqual(j, '["\\u03b1\\u03a9"]') def test_encoding5(self): u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' - j = json.dumps(u, ensure_ascii=False) + j = self.dumps(u, ensure_ascii=False) self.assertEqual(j, u'"{0}"'.format(u)) def test_encoding6(self): u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' - j = json.dumps([u], ensure_ascii=False) + j = self.dumps([u], ensure_ascii=False) self.assertEqual(j, u'["{0}"]'.format(u)) def test_big_unicode_encode(self): u = u'\U0001d120' - self.assertEqual(json.dumps(u), '"\\ud834\\udd20"') - self.assertEqual(json.dumps(u, ensure_ascii=False), u'"\U0001d120"') + self.assertEqual(self.dumps(u), '"\\ud834\\udd20"') + self.assertEqual(self.dumps(u, ensure_ascii=False), u'"\U0001d120"') def test_big_unicode_decode(self): u = u'z\U0001d120x' - self.assertEqual(json.loads('"' + u + '"'), u) - self.assertEqual(json.loads('"z\\ud834\\udd20x"'), u) + self.assertEqual(self.loads('"' + u + '"'), u) + self.assertEqual(self.loads('"z\\ud834\\udd20x"'), u) def test_unicode_decode(self): for i in range(0, 0xd7ff): u = unichr(i) s = '"\\u{0:04x}"'.format(i) - self.assertEqual(json.loads(s), u) + self.assertEqual(self.loads(s), u) def test_object_pairs_hook_with_unicode(self): s = u'{"xkd":1, "kcw":2, "art":3, "hxm":4, "qrt":5, "pad":6, "hoy":7}' p = [(u"xkd", 1), (u"kcw", 2), (u"art", 3), (u"hxm", 4), (u"qrt", 5), (u"pad", 6), (u"hoy", 7)] - self.assertEqual(json.loads(s), eval(s)) - self.assertEqual(json.loads(s, object_pairs_hook = lambda x: x), p) - od = json.loads(s, object_pairs_hook = OrderedDict) + self.assertEqual(self.loads(s), eval(s)) + self.assertEqual(self.loads(s, object_pairs_hook = lambda x: x), p) + od = self.loads(s, object_pairs_hook = OrderedDict) self.assertEqual(od, OrderedDict(p)) self.assertEqual(type(od), OrderedDict) # the object_pairs_hook takes priority over the object_hook - self.assertEqual(json.loads(s, + self.assertEqual(self.loads(s, object_pairs_hook = OrderedDict, object_hook = lambda x: None), OrderedDict(p)) def test_default_encoding(self): - self.assertEqual(json.loads(u'{"a": "\xe9"}'.encode('utf-8')), + self.assertEqual(self.loads(u'{"a": "\xe9"}'.encode('utf-8')), {'a': u'\xe9'}) def test_unicode_preservation(self): - self.assertEqual(type(json.loads(u'""')), unicode) - self.assertEqual(type(json.loads(u'"a"')), unicode) - self.assertEqual(type(json.loads(u'["a"]')[0]), unicode) + self.assertEqual(type(self.loads(u'""')), unicode) + self.assertEqual(type(self.loads(u'"a"')), unicode) + self.assertEqual(type(self.loads(u'["a"]')[0]), unicode) # Issue 10038. - self.assertEqual(type(json.loads('"foo"')), unicode) + self.assertEqual(type(self.loads('"foo"')), unicode) + + +class TestPyUnicode(TestUnicode, PyTest): pass +class TestCUnicode(TestUnicode, CTest): pass diff --git a/lib-python/2.7/lib-tk/Tix.py b/lib-python/2.7/lib-tk/Tix.py --- a/lib-python/2.7/lib-tk/Tix.py +++ b/lib-python/2.7/lib-tk/Tix.py @@ -163,7 +163,7 @@ extensions) exist, then the image type is chosen according to the depth of the X display: xbm images are chosen on monochrome displays and color images are chosen on color displays. By using - tix_ getimage, you can advoid hard coding the pathnames of the + tix_ getimage, you can avoid hard coding the pathnames of the image files in your application. When successful, this command returns the name of the newly created image, which can be used to configure the -image option of the Tk and Tix widgets. @@ -171,7 +171,7 @@ return self.tk.call('tix', 'getimage', name) def tix_option_get(self, name): - """Gets the options manitained by the Tix + """Gets the options maintained by the Tix scheme mechanism. Available options include: active_bg active_fg bg @@ -576,7 +576,7 @@ class ComboBox(TixWidget): """ComboBox - an Entry field with a dropdown menu. The user can select a - choice by either typing in the entry subwdget or selecting from the + choice by either typing in the entry subwidget or selecting from the listbox subwidget. Subwidget Class @@ -869,7 +869,7 @@ """HList - Hierarchy display widget can be used to display any data that have a hierarchical structure, for example, file system directory trees. The list entries are indented and connected by branch lines - according to their places in the hierachy. + according to their places in the hierarchy. Subwidgets - None""" @@ -1520,7 +1520,7 @@ self.tk.call(self._w, 'selection', 'set', first, last) class Tree(TixWidget): - """Tree - The tixTree widget can be used to display hierachical + """Tree - The tixTree widget can be used to display hierarchical data in a tree form. The user can adjust the view of the tree by opening or closing parts of the tree.""" diff --git a/lib-python/2.7/lib-tk/Tkinter.py b/lib-python/2.7/lib-tk/Tkinter.py --- a/lib-python/2.7/lib-tk/Tkinter.py +++ b/lib-python/2.7/lib-tk/Tkinter.py @@ -1660,7 +1660,7 @@ class Tk(Misc, Wm): """Toplevel widget of Tk which represents mostly the main window - of an appliation. It has an associated Tcl interpreter.""" + of an application. It has an associated Tcl interpreter.""" _w = '.' def __init__(self, screenName=None, baseName=None, className='Tk', useTk=1, sync=0, use=None): diff --git a/lib-python/2.7/lib-tk/test/test_ttk/test_functions.py b/lib-python/2.7/lib-tk/test/test_ttk/test_functions.py --- a/lib-python/2.7/lib-tk/test/test_ttk/test_functions.py +++ b/lib-python/2.7/lib-tk/test/test_ttk/test_functions.py @@ -136,7 +136,7 @@ # minimum acceptable for image type self.assertEqual(ttk._format_elemcreate('image', False, 'test'), ("test ", ())) - # specifiyng a state spec + # specifying a state spec self.assertEqual(ttk._format_elemcreate('image', False, 'test', ('', 'a')), ("test {} a", ())) # state spec with multiple states diff --git a/lib-python/2.7/lib-tk/ttk.py b/lib-python/2.7/lib-tk/ttk.py --- a/lib-python/2.7/lib-tk/ttk.py +++ b/lib-python/2.7/lib-tk/ttk.py @@ -707,7 +707,7 @@ textvariable, values, width """ # The "values" option may need special formatting, so leave to - # _format_optdict the responsability to format it + # _format_optdict the responsibility to format it if "values" in kw: kw["values"] = _format_optdict({'v': kw["values"]})[1] @@ -993,7 +993,7 @@ pane is either an integer index or the name of a managed subwindow. If kw is not given, returns a dict of the pane option values. If option is specified then the value for that option is returned. - Otherwise, sets the options to the correspoding values.""" + Otherwise, sets the options to the corresponding values.""" if option is not None: kw[option] = None return _val_or_dict(kw, self.tk.call, self._w, "pane", pane) diff --git a/lib-python/2.7/lib-tk/turtle.py b/lib-python/2.7/lib-tk/turtle.py --- a/lib-python/2.7/lib-tk/turtle.py +++ b/lib-python/2.7/lib-tk/turtle.py @@ -1385,7 +1385,7 @@ Optional argument: picname -- a string, name of a gif-file or "nopic". - If picname is a filename, set the corresponing image as background. + If picname is a filename, set the corresponding image as background. If picname is "nopic", delete backgroundimage, if present. If picname is None, return the filename of the current backgroundimage. @@ -1409,7 +1409,7 @@ Optional arguments: canvwidth -- positive integer, new width of canvas in pixels canvheight -- positive integer, new height of canvas in pixels - bg -- colorstring or color-tupel, new backgroundcolor + bg -- colorstring or color-tuple, new backgroundcolor If no arguments are given, return current (canvaswidth, canvasheight) Do not alter the drawing window. To observe hidden parts of @@ -3079,9 +3079,9 @@ fill="", width=ps) # Turtle now at position old, self._position = old - ## if undo is done during crating a polygon, the last vertex - ## will be deleted. if the polygon is entirel deleted, - ## creatigPoly will be set to False. + ## if undo is done during creating a polygon, the last vertex + ## will be deleted. if the polygon is entirely deleted, + ## creatingPoly will be set to False. ## Polygons created before the last one will not be affected by undo() if self._creatingPoly: if len(self._poly) > 0: @@ -3221,7 +3221,7 @@ def dot(self, size=None, *color): """Draw a dot with diameter size, using color. - Optional argumentS: + Optional arguments: size -- an integer >= 1 (if given) color -- a colorstring or a numeric color tuple @@ -3691,7 +3691,7 @@ class Turtle(RawTurtle): - """RawTurtle auto-crating (scrolled) canvas. + """RawTurtle auto-creating (scrolled) canvas. When a Turtle object is created or a function derived from some Turtle method is called a TurtleScreen object is automatically created. @@ -3731,7 +3731,7 @@ filename -- a string, used as filename default value is turtle_docstringdict - Has to be called explicitely, (not used by the turtle-graphics classes) + Has to be called explicitly, (not used by the turtle-graphics classes) The docstring dictionary will be written to the Python script <filname>.py It is intended to serve as a template for translation of the docstrings into different languages. diff --git a/lib-python/2.7/lib2to3/__main__.py b/lib-python/2.7/lib2to3/__main__.py new file mode 100644 --- /dev/null +++ b/lib-python/2.7/lib2to3/__main__.py @@ -0,0 +1,4 @@ +import sys +from .main import main + +sys.exit(main("lib2to3.fixes")) diff --git a/lib-python/2.7/lib2to3/fixes/fix_itertools.py b/lib-python/2.7/lib2to3/fixes/fix_itertools.py --- a/lib-python/2.7/lib2to3/fixes/fix_itertools.py +++ b/lib-python/2.7/lib2to3/fixes/fix_itertools.py @@ -13,7 +13,7 @@ class FixItertools(fixer_base.BaseFix): BM_compatible = True - it_funcs = "('imap'|'ifilter'|'izip'|'ifilterfalse')" + it_funcs = "('imap'|'ifilter'|'izip'|'izip_longest'|'ifilterfalse')" PATTERN = """ power< it='itertools' trailer< @@ -28,7 +28,8 @@ def transform(self, node, results): prefix = None func = results['func'][0] - if 'it' in results and func.value != u'ifilterfalse': + if ('it' in results and + func.value not in (u'ifilterfalse', u'izip_longest')): dot, it = (results['dot'], results['it']) # Remove the 'itertools' prefix = it.prefix diff --git a/lib-python/2.7/lib2to3/fixes/fix_itertools_imports.py b/lib-python/2.7/lib2to3/fixes/fix_itertools_imports.py --- a/lib-python/2.7/lib2to3/fixes/fix_itertools_imports.py +++ b/lib-python/2.7/lib2to3/fixes/fix_itertools_imports.py @@ -31,9 +31,10 @@ if member_name in (u'imap', u'izip', u'ifilter'): child.value = None child.remove() - elif member_name == u'ifilterfalse': + elif member_name in (u'ifilterfalse', u'izip_longest'): node.changed() - name_node.value = u'filterfalse' + name_node.value = (u'filterfalse' if member_name[1] == u'f' + else u'zip_longest') # Make sure the import statement is still sane children = imports.children[:] or [imports] diff --git a/lib-python/2.7/lib2to3/fixes/fix_metaclass.py b/lib-python/2.7/lib2to3/fixes/fix_metaclass.py --- a/lib-python/2.7/lib2to3/fixes/fix_metaclass.py +++ b/lib-python/2.7/lib2to3/fixes/fix_metaclass.py @@ -48,7 +48,7 @@ """ for node in cls_node.children: if node.type == syms.suite: - # already in the prefered format, do nothing + # already in the preferred format, do nothing return # !%@#! oneliners have no suite node, we have to fake one up diff --git a/lib-python/2.7/lib2to3/fixes/fix_urllib.py b/lib-python/2.7/lib2to3/fixes/fix_urllib.py --- a/lib-python/2.7/lib2to3/fixes/fix_urllib.py +++ b/lib-python/2.7/lib2to3/fixes/fix_urllib.py @@ -12,7 +12,7 @@ MAPPING = {"urllib": [ ("urllib.request", - ["URLOpener", "FancyURLOpener", "urlretrieve", + ["URLopener", "FancyURLopener", "urlretrieve", "_urlopener", "urlopen", "urlcleanup", "pathname2url", "url2pathname"]), ("urllib.parse", diff --git a/lib-python/2.7/lib2to3/main.py b/lib-python/2.7/lib2to3/main.py --- a/lib-python/2.7/lib2to3/main.py +++ b/lib-python/2.7/lib2to3/main.py @@ -101,7 +101,7 @@ parser.add_option("-j", "--processes", action="store", default=1, type="int", help="Run 2to3 concurrently") parser.add_option("-x", "--nofix", action="append", default=[], - help="Prevent a fixer from being run.") + help="Prevent a transformation from being run") parser.add_option("-l", "--list-fixes", action="store_true", help="List available transformations") parser.add_option("-p", "--print-function", action="store_true", @@ -113,7 +113,7 @@ parser.add_option("-w", "--write", action="store_true", help="Write back modified files") parser.add_option("-n", "--nobackups", action="store_true", default=False, - help="Don't write backups for modified files.") + help="Don't write backups for modified files") # Parse command line arguments refactor_stdin = False diff --git a/lib-python/2.7/lib2to3/patcomp.py b/lib-python/2.7/lib2to3/patcomp.py --- a/lib-python/2.7/lib2to3/patcomp.py +++ b/lib-python/2.7/lib2to3/patcomp.py @@ -12,6 +12,7 @@ # Python imports import os +import StringIO # Fairly local imports from .pgen2 import driver, literals, token, tokenize, parse, grammar @@ -32,7 +33,7 @@ def tokenize_wrapper(input): """Tokenizes a string suppressing significant whitespace.""" skip = set((token.NEWLINE, token.INDENT, token.DEDENT)) - tokens = tokenize.generate_tokens(driver.generate_lines(input).next) + tokens = tokenize.generate_tokens(StringIO.StringIO(input).readline) for quintuple in tokens: type, value, start, end, line_text = quintuple if type not in skip: diff --git a/lib-python/2.7/lib2to3/pgen2/conv.py b/lib-python/2.7/lib2to3/pgen2/conv.py --- a/lib-python/2.7/lib2to3/pgen2/conv.py +++ b/lib-python/2.7/lib2to3/pgen2/conv.py @@ -51,7 +51,7 @@ self.finish_off() def parse_graminit_h(self, filename): - """Parse the .h file writen by pgen. (Internal) + """Parse the .h file written by pgen. (Internal) This file is a sequence of #define statements defining the nonterminals of the grammar as numbers. We build two tables @@ -82,7 +82,7 @@ return True def parse_graminit_c(self, filename): - """Parse the .c file writen by pgen. (Internal) + """Parse the .c file written by pgen. (Internal) The file looks as follows. The first two lines are always this: diff --git a/lib-python/2.7/lib2to3/pgen2/driver.py b/lib-python/2.7/lib2to3/pgen2/driver.py --- a/lib-python/2.7/lib2to3/pgen2/driver.py +++ b/lib-python/2.7/lib2to3/pgen2/driver.py @@ -19,6 +19,7 @@ import codecs import os import logging +import StringIO import sys # Pgen imports @@ -101,18 +102,10 @@ def parse_string(self, text, debug=False): """Parse a string and return the syntax tree.""" - tokens = tokenize.generate_tokens(generate_lines(text).next) + tokens = tokenize.generate_tokens(StringIO.StringIO(text).readline) return self.parse_tokens(tokens, debug) -def generate_lines(text): - """Generator that behaves like readline without using StringIO.""" - for line in text.splitlines(True): - yield line - while True: - yield "" - - def load_grammar(gt="Grammar.txt", gp=None, save=True, force=False, logger=None): """Load the grammar (maybe from a pickle).""" diff --git a/lib-python/2.7/lib2to3/pytree.py b/lib-python/2.7/lib2to3/pytree.py --- a/lib-python/2.7/lib2to3/pytree.py +++ b/lib-python/2.7/lib2to3/pytree.py @@ -658,8 +658,8 @@ content: optional sequence of subsequences of patterns; if absent, matches one node; if present, each subsequence is an alternative [*] - min: optinal minumum number of times to match, default 0 - max: optional maximum number of times tro match, default HUGE + min: optional minimum number of times to match, default 0 + max: optional maximum number of times to match, default HUGE name: optional name assigned to this match [*] Thus, if content is [[a, b, c], [d, e], [f, g, h]] this is @@ -743,9 +743,11 @@ else: # The reason for this is that hitting the recursion limit usually # results in some ugly messages about how RuntimeErrors are being - # ignored. - save_stderr = sys.stderr - sys.stderr = StringIO() + # ignored. We don't do this on non-CPython implementation because + # they don't have this problem. + if hasattr(sys, "getrefcount"): + save_stderr = sys.stderr + sys.stderr = StringIO() try: for count, r in self._recursive_matches(nodes, 0): if self.name: @@ -759,7 +761,8 @@ r[self.name] = nodes[:count] yield count, r finally: - sys.stderr = save_stderr + if hasattr(sys, "getrefcount"): + sys.stderr = save_stderr def _iterative_matches(self, nodes): """Helper to iteratively yield the matches.""" diff --git a/lib-python/2.7/lib2to3/refactor.py b/lib-python/2.7/lib2to3/refactor.py --- a/lib-python/2.7/lib2to3/refactor.py +++ b/lib-python/2.7/lib2to3/refactor.py @@ -302,13 +302,14 @@ Files and subdirectories starting with '.' are skipped. """ + py_ext = os.extsep + "py" for dirpath, dirnames, filenames in os.walk(dir_name): self.log_debug("Descending into %s", dirpath) dirnames.sort() filenames.sort() for name in filenames: - if not name.startswith(".") and \ - os.path.splitext(name)[1].endswith("py"): + if (not name.startswith(".") and + os.path.splitext(name)[1] == py_ext): fullname = os.path.join(dirpath, name) self.refactor_file(fullname, write, doctests_only) # Modify dirnames in-place to remove subdirs with leading dots diff --git a/lib-python/2.7/lib2to3/tests/data/py2_test_grammar.py b/lib-python/2.7/lib2to3/tests/data/py2_test_grammar.py --- a/lib-python/2.7/lib2to3/tests/data/py2_test_grammar.py +++ b/lib-python/2.7/lib2to3/tests/data/py2_test_grammar.py @@ -316,7 +316,7 @@ ### simple_stmt: small_stmt (';' small_stmt)* [';'] x = 1; pass; del x def foo(): - # verify statments that end with semi-colons + # verify statements that end with semi-colons x = 1; pass; del x; foo() diff --git a/lib-python/2.7/lib2to3/tests/data/py3_test_grammar.py b/lib-python/2.7/lib2to3/tests/data/py3_test_grammar.py --- a/lib-python/2.7/lib2to3/tests/data/py3_test_grammar.py +++ b/lib-python/2.7/lib2to3/tests/data/py3_test_grammar.py @@ -356,7 +356,7 @@ ### simple_stmt: small_stmt (';' small_stmt)* [';'] x = 1; pass; del x def foo(): - # verify statments that end with semi-colons + # verify statements that end with semi-colons x = 1; pass; del x; foo() diff --git a/lib-python/2.7/lib2to3/tests/test_fixers.py b/lib-python/2.7/lib2to3/tests/test_fixers.py --- a/lib-python/2.7/lib2to3/tests/test_fixers.py +++ b/lib-python/2.7/lib2to3/tests/test_fixers.py @@ -3623,16 +3623,24 @@ a = """%s(f, a)""" self.checkall(b, a) - def test_2(self): + def test_qualified(self): b = """itertools.ifilterfalse(a, b)""" a = """itertools.filterfalse(a, b)""" self.check(b, a) - def test_4(self): + b = """itertools.izip_longest(a, b)""" + a = """itertools.zip_longest(a, b)""" + self.check(b, a) + + def test_2(self): b = """ifilterfalse(a, b)""" a = """filterfalse(a, b)""" self.check(b, a) + b = """izip_longest(a, b)""" + a = """zip_longest(a, b)""" + self.check(b, a) + def test_space_1(self): b = """ %s(f, a)""" a = """ %s(f, a)""" @@ -3643,9 +3651,14 @@ a = """ itertools.filterfalse(a, b)""" self.check(b, a) + b = """ itertools.izip_longest(a, b)""" + a = """ itertools.zip_longest(a, b)""" + self.check(b, a) + def test_run_order(self): self.assert_runs_after('map', 'zip', 'filter') + class Test_itertools_imports(FixerTestCase): fixer = 'itertools_imports' @@ -3696,18 +3709,19 @@ s = "from itertools import bar as bang" self.unchanged(s) - def test_ifilter(self): - b = "from itertools import ifilterfalse" - a = "from itertools import filterfalse" - self.check(b, a) - - b = "from itertools import imap, ifilterfalse, foo" - a = "from itertools import filterfalse, foo" - self.check(b, a) - - b = "from itertools import bar, ifilterfalse, foo" - a = "from itertools import bar, filterfalse, foo" - self.check(b, a) + def test_ifilter_and_zip_longest(self): + for name in "filterfalse", "zip_longest": + b = "from itertools import i%s" % (name,) + a = "from itertools import %s" % (name,) + self.check(b, a) + + b = "from itertools import imap, i%s, foo" % (name,) + a = "from itertools import %s, foo" % (name,) + self.check(b, a) + + b = "from itertools import bar, i%s, foo" % (name,) + a = "from itertools import bar, %s, foo" % (name,) + self.check(b, a) def test_import_star(self): s = "from itertools import *" diff --git a/lib-python/2.7/lib2to3/tests/test_parser.py b/lib-python/2.7/lib2to3/tests/test_parser.py --- a/lib-python/2.7/lib2to3/tests/test_parser.py +++ b/lib-python/2.7/lib2to3/tests/test_parser.py @@ -19,6 +19,16 @@ # Local imports from lib2to3.pgen2 import tokenize from ..pgen2.parse import ParseError +from lib2to3.pygram import python_symbols as syms + + +class TestDriver(support.TestCase): + + def test_formfeed(self): + s = """print 1\n\x0Cprint 2\n""" + t = driver.parse_string(s) + self.assertEqual(t.children[0].children[0].type, syms.print_stmt) + self.assertEqual(t.children[1].children[0].type, syms.print_stmt) class GrammarTest(support.TestCase): diff --git a/lib-python/2.7/lib2to3/tests/test_refactor.py b/lib-python/2.7/lib2to3/tests/test_refactor.py --- a/lib-python/2.7/lib2to3/tests/test_refactor.py +++ b/lib-python/2.7/lib2to3/tests/test_refactor.py @@ -223,6 +223,7 @@ "hi.py", ".dumb", ".after.py", + "notpy.npy", "sappy"] expected = ["hi.py"] check(tree, expected) diff --git a/lib-python/2.7/lib2to3/tests/test_util.py b/lib-python/2.7/lib2to3/tests/test_util.py --- a/lib-python/2.7/lib2to3/tests/test_util.py +++ b/lib-python/2.7/lib2to3/tests/test_util.py @@ -568,8 +568,8 @@ def test_from_import(self): node = parse('bar()') - fixer_util.touch_import("cgi", "escape", node) - self.assertEqual(str(node), 'from cgi import escape\nbar()\n\n') + fixer_util.touch_import("html", "escape", node) + self.assertEqual(str(node), 'from html import escape\nbar()\n\n') def test_name_import(self): node = parse('bar()') diff --git a/lib-python/2.7/locale.py b/lib-python/2.7/locale.py --- a/lib-python/2.7/locale.py +++ b/lib-python/2.7/locale.py @@ -621,7 +621,7 @@ 'tactis': 'TACTIS', 'euc_jp': 'eucJP', 'euc_kr': 'eucKR', - 'utf_8': 'UTF8', + 'utf_8': 'UTF-8', 'koi8_r': 'KOI8-R', 'koi8_u': 'KOI8-U', # XXX This list is still incomplete. If you know more diff --git a/lib-python/2.7/logging/__init__.py b/lib-python/2.7/logging/__init__.py --- a/lib-python/2.7/logging/__init__.py +++ b/lib-python/2.7/logging/__init__.py @@ -1627,6 +1627,7 @@ h = wr() if h: try: + h.acquire() h.flush() h.close() except (IOError, ValueError): @@ -1635,6 +1636,8 @@ # references to them are still around at # application exit. pass + finally: + h.release() except: if raiseExceptions: raise diff --git a/lib-python/2.7/logging/config.py b/lib-python/2.7/logging/config.py --- a/lib-python/2.7/logging/config.py +++ b/lib-python/2.7/logging/config.py @@ -226,14 +226,14 @@ propagate = 1 logger = logging.getLogger(qn) if qn in existing: - i = existing.index(qn) + i = existing.index(qn) + 1 # start with the entry after qn prefixed = qn + "." pflen = len(prefixed) num_existing = len(existing) - i = i + 1 # look at the entry after qn - while (i < num_existing) and (existing[i][:pflen] == prefixed): - child_loggers.append(existing[i]) - i = i + 1 + while i < num_existing: + if existing[i][:pflen] == prefixed: + child_loggers.append(existing[i]) + i += 1 existing.remove(qn) if "level" in opts: level = cp.get(sectname, "level") diff --git a/lib-python/2.7/logging/handlers.py b/lib-python/2.7/logging/handlers.py --- a/lib-python/2.7/logging/handlers.py +++ b/lib-python/2.7/logging/handlers.py @@ -125,6 +125,7 @@ """ if self.stream: self.stream.close() + self.stream = None if self.backupCount > 0: for i in range(self.backupCount - 1, 0, -1): sfn = "%s.%d" % (self.baseFilename, i) @@ -324,6 +325,7 @@ """ if self.stream: self.stream.close() + self.stream = None # get the time that this sequence started at and make it a TimeTuple t = self.rolloverAt - self.interval if self.utc: diff --git a/lib-python/2.7/mailbox.py b/lib-python/2.7/mailbox.py --- a/lib-python/2.7/mailbox.py +++ b/lib-python/2.7/mailbox.py @@ -234,27 +234,35 @@ def __init__(self, dirname, factory=rfc822.Message, create=True): """Initialize a Maildir instance.""" Mailbox.__init__(self, dirname, factory, create) + self._paths = { + 'tmp': os.path.join(self._path, 'tmp'), + 'new': os.path.join(self._path, 'new'), + 'cur': os.path.join(self._path, 'cur'), + } if not os.path.exists(self._path): if create: os.mkdir(self._path, 0700) - os.mkdir(os.path.join(self._path, 'tmp'), 0700) - os.mkdir(os.path.join(self._path, 'new'), 0700) - os.mkdir(os.path.join(self._path, 'cur'), 0700) + for path in self._paths.values(): + os.mkdir(path, 0o700) else: raise NoSuchMailboxError(self._path) self._toc = {} - self._last_read = None # Records last time we read cur/new - # NOTE: we manually invalidate _last_read each time we do any - # modifications ourselves, otherwise we might get tripped up by - # bogus mtime behaviour on some systems (see issue #6896). + self._toc_mtimes = {} + for subdir in ('cur', 'new'): + self._toc_mtimes[subdir] = os.path.getmtime(self._paths[subdir]) + self._last_read = time.time() # Records last time we read cur/new + self._skewfactor = 0.1 # Adjust if os/fs clocks are skewing def add(self, message): """Add message and return assigned key.""" tmp_file = self._create_tmp() try: self._dump_message(message, tmp_file) - finally: - _sync_close(tmp_file) + except BaseException: + tmp_file.close() + os.remove(tmp_file.name) + raise + _sync_close(tmp_file) if isinstance(message, MaildirMessage): subdir = message.get_subdir() suffix = self.colon + message.get_info() @@ -280,15 +288,11 @@ raise if isinstance(message, MaildirMessage): os.utime(dest, (os.path.getatime(dest), message.get_date())) - # Invalidate cached toc - self._last_read = None return uniq def remove(self, key): """Remove the keyed message; raise KeyError if it doesn't exist.""" os.remove(os.path.join(self._path, self._lookup(key))) - # Invalidate cached toc (only on success) - self._last_read = None def discard(self, key): """If the keyed message exists, remove it.""" @@ -323,8 +327,6 @@ if isinstance(message, MaildirMessage): os.utime(new_path, (os.path.getatime(new_path), message.get_date())) - # Invalidate cached toc - self._last_read = None def get_message(self, key): """Return a Message representation or raise a KeyError.""" @@ -380,8 +382,8 @@ def flush(self): """Write any pending changes to disk.""" # Maildir changes are always written immediately, so there's nothing - # to do except invalidate our cached toc. - self._last_read = None + # to do. + pass def lock(self): """Lock the mailbox.""" @@ -479,36 +481,39 @@ def _refresh(self): """Update table of contents mapping.""" - if self._last_read is not None: - for subdir in ('new', 'cur'): - mtime = os.path.getmtime(os.path.join(self._path, subdir)) - if mtime > self._last_read: - break - else: + # If it has been less than two seconds since the last _refresh() call, + # we have to unconditionally re-read the mailbox just in case it has + # been modified, because os.path.mtime() has a 2 sec resolution in the + # most common worst case (FAT) and a 1 sec resolution typically. This + # results in a few unnecessary re-reads when _refresh() is called + # multiple times in that interval, but once the clock ticks over, we + # will only re-read as needed. Because the filesystem might be being + # served by an independent system with its own clock, we record and + # compare with the mtimes from the filesystem. Because the other + # system's clock might be skewing relative to our clock, we add an + # extra delta to our wait. The default is one tenth second, but is an + # instance variable and so can be adjusted if dealing with a + # particularly skewed or irregular system. + if time.time() - self._last_read > 2 + self._skewfactor: + refresh = False + for subdir in self._toc_mtimes: + mtime = os.path.getmtime(self._paths[subdir]) + if mtime > self._toc_mtimes[subdir]: + refresh = True + self._toc_mtimes[subdir] = mtime + if not refresh: return - - # We record the current time - 1sec so that, if _refresh() is called - # again in the same second, we will always re-read the mailbox - # just in case it's been modified. (os.path.mtime() only has - # 1sec resolution.) This results in a few unnecessary re-reads - # when _refresh() is called multiple times in the same second, - # but once the clock ticks over, we will only re-read as needed. - now = time.time() - 1 - + # Refresh toc self._toc = {} - def update_dir (subdir): - path = os.path.join(self._path, subdir) + for subdir in self._toc_mtimes: + path = self._paths[subdir] for entry in os.listdir(path): p = os.path.join(path, entry) if os.path.isdir(p): continue uniq = entry.split(self.colon)[0] self._toc[uniq] = os.path.join(subdir, entry) - - update_dir('new') - update_dir('cur') - - self._last_read = now + self._last_read = time.time() def _lookup(self, key): """Use TOC to return subpath for given key, or raise a KeyError.""" @@ -551,7 +556,7 @@ f = open(self._path, 'wb+') else: raise NoSuchMailboxError(self._path) - elif e.errno == errno.EACCES: + elif e.errno in (errno.EACCES, errno.EROFS): f = open(self._path, 'rb') else: raise @@ -700,9 +705,14 @@ def _append_message(self, message): """Append message to mailbox and return (start, stop) offsets.""" self._file.seek(0, 2) - self._pre_message_hook(self._file) - offsets = self._install_message(message) - self._post_message_hook(self._file) + before = self._file.tell() + try: + self._pre_message_hook(self._file) + offsets = self._install_message(message) + self._post_message_hook(self._file) + except BaseException: + self._file.truncate(before) + raise self._file.flush() self._file_length = self._file.tell() # Record current length of mailbox return offsets @@ -868,18 +878,29 @@ new_key = max(keys) + 1 new_path = os.path.join(self._path, str(new_key)) f = _create_carefully(new_path) + closed = False try: if self._locked: _lock_file(f) try: - self._dump_message(message, f) + try: + self._dump_message(message, f) + except BaseException: + # Unlock and close so it can be deleted on Windows + if self._locked: + _unlock_file(f) + _sync_close(f) + closed = True + os.remove(new_path) + raise if isinstance(message, MHMessage): self._dump_sequences(message, new_key) finally: if self._locked: _unlock_file(f) finally: - _sync_close(f) + if not closed: + _sync_close(f) return new_key def remove(self, key): @@ -1886,7 +1907,7 @@ try: fcntl.lockf(f, fcntl.LOCK_EX | fcntl.LOCK_NB) except IOError, e: - if e.errno in (errno.EAGAIN, errno.EACCES): + if e.errno in (errno.EAGAIN, errno.EACCES, errno.EROFS): raise ExternalClashError('lockf: lock unavailable: %s' % f.name) else: @@ -1896,7 +1917,7 @@ pre_lock = _create_temporary(f.name + '.lock') pre_lock.close() except IOError, e: - if e.errno == errno.EACCES: + if e.errno in (errno.EACCES, errno.EROFS): return # Without write access, just skip dotlocking. else: raise diff --git a/lib-python/2.7/msilib/__init__.py b/lib-python/2.7/msilib/__init__.py --- a/lib-python/2.7/msilib/__init__.py +++ b/lib-python/2.7/msilib/__init__.py @@ -173,11 +173,10 @@ add_data(db, table, getattr(module, table)) def make_id(str): - #str = str.replace(".", "_") # colons are allowed - str = str.replace(" ", "_") - str = str.replace("-", "_") - if str[0] in string.digits: - str = "_"+str + identifier_chars = string.ascii_letters + string.digits + "._" + str = "".join([c if c in identifier_chars else "_" for c in str]) + if str[0] in (string.digits + "."): + str = "_" + str assert re.match("^[A-Za-z_][A-Za-z0-9_.]*$", str), "FILE"+str return str @@ -285,19 +284,28 @@ [(feature.id, component)]) def make_short(self, file): + oldfile = file + file = file.replace('+', '_') + file = ''.join(c for c in file if not c in ' "/\[]:;=,') parts = file.split(".") - if len(parts)>1: + if len(parts) > 1: + prefix = "".join(parts[:-1]).upper() suffix = parts[-1].upper() + if not prefix: + prefix = suffix + suffix = None else: + prefix = file.upper() suffix = None - prefix = parts[0].upper() - if len(prefix) <= 8 and (not suffix or len(suffix)<=3): + if len(parts) < 3 and len(prefix) <= 8 and file == oldfile and ( + not suffix or len(suffix) <= 3): if suffix: file = prefix+"."+suffix else: file = prefix - assert file not in self.short_names else: + file = None + if file is None or file in self.short_names: prefix = prefix[:6] if suffix: suffix = suffix[:3] diff --git a/lib-python/2.7/multiprocessing/__init__.py b/lib-python/2.7/multiprocessing/__init__.py --- a/lib-python/2.7/multiprocessing/__init__.py +++ b/lib-python/2.7/multiprocessing/__init__.py @@ -38,6 +38,7 @@ # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. # __version__ = '0.70a1' @@ -115,8 +116,11 @@ except (ValueError, KeyError): num = 0 elif 'bsd' in sys.platform or sys.platform == 'darwin': + comm = '/sbin/sysctl -n hw.ncpu' + if sys.platform == 'darwin': + comm = '/usr' + comm try: - with os.popen('sysctl -n hw.ncpu') as p: + with os.popen(comm) as p: num = int(p.read()) except ValueError: num = 0 diff --git a/lib-python/2.7/multiprocessing/connection.py b/lib-python/2.7/multiprocessing/connection.py --- a/lib-python/2.7/multiprocessing/connection.py +++ b/lib-python/2.7/multiprocessing/connection.py @@ -3,7 +3,33 @@ # # multiprocessing/connection.py # -# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt +# Copyright (c) 2006-2008, R Oudkerk +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of author nor the names of any contributors may be +# used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. # __all__ = [ 'Client', 'Listener', 'Pipe' ] diff --git a/lib-python/2.7/multiprocessing/dummy/__init__.py b/lib-python/2.7/multiprocessing/dummy/__init__.py --- a/lib-python/2.7/multiprocessing/dummy/__init__.py +++ b/lib-python/2.7/multiprocessing/dummy/__init__.py @@ -3,7 +3,33 @@ # # multiprocessing/dummy/__init__.py # -# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt +# Copyright (c) 2006-2008, R Oudkerk +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of author nor the names of any contributors may be +# used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. # __all__ = [ diff --git a/lib-python/2.7/multiprocessing/dummy/connection.py b/lib-python/2.7/multiprocessing/dummy/connection.py --- a/lib-python/2.7/multiprocessing/dummy/connection.py +++ b/lib-python/2.7/multiprocessing/dummy/connection.py @@ -3,7 +3,33 @@ # # multiprocessing/dummy/connection.py # -# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt +# Copyright (c) 2006-2008, R Oudkerk +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of author nor the names of any contributors may be +# used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. # __all__ = [ 'Client', 'Listener', 'Pipe' ] diff --git a/lib-python/2.7/multiprocessing/forking.py b/lib-python/2.7/multiprocessing/forking.py --- a/lib-python/2.7/multiprocessing/forking.py +++ b/lib-python/2.7/multiprocessing/forking.py @@ -3,7 +3,33 @@ # # multiprocessing/forking.py # -# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt +# Copyright (c) 2006-2008, R Oudkerk +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of author nor the names of any contributors may be +# used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. # import os @@ -172,6 +198,7 @@ TERMINATE = 0x10000 WINEXE = (sys.platform == 'win32' and getattr(sys, 'frozen', False)) + WINSERVICE = sys.executable.lower().endswith("pythonservice.exe") exit = win32.ExitProcess close = win32.CloseHandle @@ -181,7 +208,7 @@ # People embedding Python want to modify it. # - if sys.executable.lower().endswith('pythonservice.exe'): + if WINSERVICE: _python_exe = os.path.join(sys.exec_prefix, 'python.exe') else: _python_exe = sys.executable @@ -371,7 +398,7 @@ if _logger is not None: d['log_level'] = _logger.getEffectiveLevel() - if not WINEXE: + if not WINEXE and not WINSERVICE: main_path = getattr(sys.modules['__main__'], '__file__', None) if not main_path and sys.argv[0] not in ('', '-c'): main_path = sys.argv[0] diff --git a/lib-python/2.7/multiprocessing/heap.py b/lib-python/2.7/multiprocessing/heap.py --- a/lib-python/2.7/multiprocessing/heap.py +++ b/lib-python/2.7/multiprocessing/heap.py @@ -3,7 +3,33 @@ # # multiprocessing/heap.py # -# Copyright (c) 2007-2008, R Oudkerk --- see COPYING.txt +# Copyright (c) 2006-2008, R Oudkerk +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of author nor the names of any contributors may be +# used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. # import bisect diff --git a/lib-python/2.7/multiprocessing/managers.py b/lib-python/2.7/multiprocessing/managers.py --- a/lib-python/2.7/multiprocessing/managers.py +++ b/lib-python/2.7/multiprocessing/managers.py @@ -4,7 +4,33 @@ # # multiprocessing/managers.py # -# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt +# Copyright (c) 2006-2008, R Oudkerk +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of author nor the names of any contributors may be +# used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. # __all__ = [ 'BaseManager', 'SyncManager', 'BaseProxy', 'Token' ] diff --git a/lib-python/2.7/multiprocessing/pool.py b/lib-python/2.7/multiprocessing/pool.py --- a/lib-python/2.7/multiprocessing/pool.py +++ b/lib-python/2.7/multiprocessing/pool.py @@ -3,7 +3,33 @@ # # multiprocessing/pool.py # -# Copyright (c) 2007-2008, R Oudkerk --- see COPYING.txt +# Copyright (c) 2006-2008, R Oudkerk +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of author nor the names of any contributors may be +# used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. # __all__ = ['Pool'] @@ -269,6 +295,8 @@ while pool._worker_handler._state == RUN and pool._state == RUN: pool._maintain_pool() time.sleep(0.1) + # send sentinel to stop workers + pool._taskqueue.put(None) debug('worker handler exiting') @staticmethod @@ -387,7 +415,6 @@ if self._state == RUN: self._state = CLOSE self._worker_handler._state = CLOSE - self._taskqueue.put(None) def terminate(self): debug('terminating pool') @@ -421,7 +448,6 @@ worker_handler._state = TERMINATE task_handler._state = TERMINATE - taskqueue.put(None) # sentinel debug('helping task handler/workers to finish') cls._help_stuff_finish(inqueue, task_handler, len(pool)) @@ -431,6 +457,11 @@ result_handler._state = TERMINATE outqueue.put(None) # sentinel + # We must wait for the worker handler to exit before terminating + # workers because we don't want workers to be restarted behind our back. + debug('joining worker handler') + worker_handler.join() + # Terminate workers which haven't already finished. if pool and hasattr(pool[0], 'terminate'): debug('terminating workers') diff --git a/lib-python/2.7/multiprocessing/process.py b/lib-python/2.7/multiprocessing/process.py --- a/lib-python/2.7/multiprocessing/process.py +++ b/lib-python/2.7/multiprocessing/process.py @@ -3,7 +3,33 @@ # # multiprocessing/process.py # -# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt +# Copyright (c) 2006-2008, R Oudkerk +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of author nor the names of any contributors may be +# used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. # __all__ = ['Process', 'current_process', 'active_children'] diff --git a/lib-python/2.7/multiprocessing/queues.py b/lib-python/2.7/multiprocessing/queues.py --- a/lib-python/2.7/multiprocessing/queues.py +++ b/lib-python/2.7/multiprocessing/queues.py @@ -3,7 +3,33 @@ # # multiprocessing/queues.py # -# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt +# Copyright (c) 2006-2008, R Oudkerk +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of author nor the names of any contributors may be +# used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. # __all__ = ['Queue', 'SimpleQueue', 'JoinableQueue'] diff --git a/lib-python/2.7/multiprocessing/reduction.py b/lib-python/2.7/multiprocessing/reduction.py --- a/lib-python/2.7/multiprocessing/reduction.py +++ b/lib-python/2.7/multiprocessing/reduction.py @@ -4,7 +4,33 @@ # # multiprocessing/reduction.py # -# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt +# Copyright (c) 2006-2008, R Oudkerk +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of author nor the names of any contributors may be +# used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. # __all__ = [] diff --git a/lib-python/2.7/multiprocessing/sharedctypes.py b/lib-python/2.7/multiprocessing/sharedctypes.py --- a/lib-python/2.7/multiprocessing/sharedctypes.py +++ b/lib-python/2.7/multiprocessing/sharedctypes.py @@ -3,7 +3,33 @@ # # multiprocessing/sharedctypes.py # -# Copyright (c) 2007-2008, R Oudkerk --- see COPYING.txt +# Copyright (c) 2006-2008, R Oudkerk +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of author nor the names of any contributors may be +# used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. # import sys @@ -52,9 +78,11 @@ Returns a ctypes array allocated from shared memory ''' type_ = typecode_to_type.get(typecode_or_type, typecode_or_type) - if isinstance(size_or_initializer, int): + if isinstance(size_or_initializer, (int, long)): type_ = type_ * size_or_initializer - return _new_value(type_) + obj = _new_value(type_) + ctypes.memset(ctypes.addressof(obj), 0, ctypes.sizeof(obj)) + return obj else: type_ = type_ * len(size_or_initializer) result = _new_value(type_) diff --git a/lib-python/2.7/multiprocessing/synchronize.py b/lib-python/2.7/multiprocessing/synchronize.py --- a/lib-python/2.7/multiprocessing/synchronize.py +++ b/lib-python/2.7/multiprocessing/synchronize.py @@ -3,7 +3,33 @@ # # multiprocessing/synchronize.py # -# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt +# Copyright (c) 2006-2008, R Oudkerk +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of author nor the names of any contributors may be +# used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. # __all__ = [ diff --git a/lib-python/2.7/multiprocessing/util.py b/lib-python/2.7/multiprocessing/util.py --- a/lib-python/2.7/multiprocessing/util.py +++ b/lib-python/2.7/multiprocessing/util.py @@ -3,7 +3,33 @@ # # multiprocessing/util.py # -# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt +# Copyright (c) 2006-2008, R Oudkerk +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of author nor the names of any contributors may be +# used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. # import itertools diff --git a/lib-python/2.7/netrc.py b/lib-python/2.7/netrc.py --- a/lib-python/2.7/netrc.py +++ b/lib-python/2.7/netrc.py @@ -34,11 +34,19 @@ def _parse(self, file, fp): lexer = shlex.shlex(fp) lexer.wordchars += r"""!"#$%&'()*+,-./:;<=>?@[\]^_`{|}~""" + lexer.commenters = lexer.commenters.replace('#', '') while 1: # Look for a machine, default, or macdef top-level keyword toplevel = tt = lexer.get_token() if not tt: break + elif tt[0] == '#': + # seek to beginning of comment, in case reading the token put + # us on a new line, and then skip the rest of the line. + pos = len(tt) + 1 + lexer.instream.seek(-pos, 1) + lexer.instream.readline() + continue elif tt == 'machine': entryname = lexer.get_token() elif tt == 'default': @@ -64,8 +72,8 @@ self.hosts[entryname] = {} while 1: tt = lexer.get_token() - if (tt=='' or tt == 'machine' or - tt == 'default' or tt =='macdef'): + if (tt.startswith('#') or + tt in {'', 'machine', 'default', 'macdef'}): if password: self.hosts[entryname] = (login, account, password) lexer.push_token(tt) diff --git a/lib-python/2.7/nntplib.py b/lib-python/2.7/nntplib.py --- a/lib-python/2.7/nntplib.py +++ b/lib-python/2.7/nntplib.py @@ -103,7 +103,7 @@ readermode is sometimes necessary if you are connecting to an NNTP server on the local machine and intend to call - reader-specific comamnds, such as `group'. If you get + reader-specific commands, such as `group'. If you get unexpected NNTPPermanentErrors, you might need to set readermode. """ diff --git a/lib-python/2.7/ntpath.py b/lib-python/2.7/ntpath.py --- a/lib-python/2.7/ntpath.py +++ b/lib-python/2.7/ntpath.py @@ -310,7 +310,7 @@ # - $varname is accepted. # - %varname% is accepted. # - varnames can be made out of letters, digits and the characters '_-' -# (though is not verifed in the ${varname} and %varname% cases) +# (though is not verified in the ${varname} and %varname% cases) # XXX With COMMAND.COM you can use any characters in a variable name, # XXX except '^|<>='. diff --git a/lib-python/2.7/nturl2path.py b/lib-python/2.7/nturl2path.py --- a/lib-python/2.7/nturl2path.py +++ b/lib-python/2.7/nturl2path.py @@ -25,11 +25,14 @@ error = 'Bad URL: ' + url raise IOError, error drive = comp[0][-1].upper() + path = drive + ':' components = comp[1].split('/') - path = drive + ':' - for comp in components: + for comp in components: if comp: path = path + '\\' + urllib.unquote(comp) + # Issue #11474: url like '/C|/' should convert into 'C:\\' + if path.endswith(':') and url.endswith('/'): + path += '\\' return path def pathname2url(p): diff --git a/lib-python/2.7/numbers.py b/lib-python/2.7/numbers.py --- a/lib-python/2.7/numbers.py +++ b/lib-python/2.7/numbers.py @@ -63,7 +63,7 @@ @abstractproperty def imag(self): - """Retrieve the real component of this number. + """Retrieve the imaginary component of this number. This should subclass Real. """ diff --git a/lib-python/2.7/optparse.py b/lib-python/2.7/optparse.py --- a/lib-python/2.7/optparse.py +++ b/lib-python/2.7/optparse.py @@ -1131,6 +1131,11 @@ prog : string the name of the current program (to override os.path.basename(sys.argv[0])). + description : string + A paragraph of text giving a brief overview of your program. + optparse reformats this paragraph to fit the current terminal + width and prints it when the user requests help (after usage, + but before the list of options). epilog : string paragraph of help text to print after option help diff --git a/lib-python/2.7/pickletools.py b/lib-python/2.7/pickletools.py --- a/lib-python/2.7/pickletools.py +++ b/lib-python/2.7/pickletools.py @@ -1370,7 +1370,7 @@ proto=0, doc="""Read an object from the memo and push it on the stack. - The index of the memo object to push is given by the newline-teriminated + The index of the memo object to push is given by the newline-terminated decimal string following. BINGET and LONG_BINGET are space-optimized versions. """), diff --git a/lib-python/2.7/pkgutil.py b/lib-python/2.7/pkgutil.py --- a/lib-python/2.7/pkgutil.py +++ b/lib-python/2.7/pkgutil.py @@ -11,7 +11,7 @@ __all__ = [ 'get_importer', 'iter_importers', 'get_loader', 'find_loader', - 'walk_packages', 'iter_modules', + 'walk_packages', 'iter_modules', 'get_data', 'ImpImporter', 'ImpLoader', 'read_code', 'extend_path', ] diff --git a/lib-python/2.7/platform.py b/lib-python/2.7/platform.py --- a/lib-python/2.7/platform.py +++ b/lib-python/2.7/platform.py @@ -503,7 +503,7 @@ info = pipe.read() if pipe.close(): raise os.error,'command failed' - # XXX How can I supress shell errors from being written + # XXX How can I suppress shell errors from being written # to stderr ? except os.error,why: #print 'Command %s failed: %s' % (cmd,why) @@ -1448,9 +1448,10 @@ """ Returns a string identifying the Python implementation. Currently, the following implementations are identified: - 'CPython' (C implementation of Python), - 'IronPython' (.NET implementation of Python), - 'Jython' (Java implementation of Python). + 'CPython' (C implementation of Python), + 'IronPython' (.NET implementation of Python), + 'Jython' (Java implementation of Python), + 'PyPy' (Python implementation of Python). """ return _sys_version()[0] diff --git a/lib-python/2.7/pydoc.py b/lib-python/2.7/pydoc.py --- a/lib-python/2.7/pydoc.py +++ b/lib-python/2.7/pydoc.py @@ -156,7 +156,7 @@ no.append(x) return yes, no -def visiblename(name, all=None): +def visiblename(name, all=None, obj=None): """Decide whether to show documentation on a variable.""" # Certain special names are redundant. _hidden_names = ('__builtins__', '__doc__', '__file__', '__path__', @@ -164,6 +164,9 @@ if name in _hidden_names: return 0 # Private names are hidden, but special names are displayed. if name.startswith('__') and name.endswith('__'): return 1 + # Namedtuples have public fields and methods with a single leading underscore + if name.startswith('_') and hasattr(obj, '_fields'): + return 1 if all is not None: # only document that which the programmer exported in __all__ return name in all @@ -475,9 +478,9 @@ def multicolumn(self, list, format, cols=4): """Format a list of items into a multi-column list.""" result = '' - rows = (len(list)+cols-1)/cols + rows = (len(list)+cols-1)//cols for col in range(cols): - result = result + '<td width="%d%%" valign=top>' % (100/cols) + result = result + '<td width="%d%%" valign=top>' % (100//cols) for i in range(rows*col, rows*col+rows): if i < len(list): result = result + format(list[i]) + '<br>\n' @@ -627,7 +630,7 @@ # if __all__ exists, believe it. Otherwise use old heuristic. if (all is not None or (inspect.getmodule(value) or object) is object): - if visiblename(key, all): + if visiblename(key, all, object): classes.append((key, value)) cdict[key] = cdict[value] = '#' + key for key, value in classes: @@ -643,13 +646,13 @@ # if __all__ exists, believe it. Otherwise use old heuristic. if (all is not None or inspect.isbuiltin(value) or inspect.getmodule(value) is object): - if visiblename(key, all): + if visiblename(key, all, object): funcs.append((key, value)) fdict[key] = '#-' + key if inspect.isfunction(value): fdict[value] = fdict[key] data = [] for key, value in inspect.getmembers(object, isdata): - if visiblename(key, all): + if visiblename(key, all, object): data.append((key, value)) doc = self.markup(getdoc(object), self.preformat, fdict, cdict) @@ -773,7 +776,7 @@ push('\n') return attrs - attrs = filter(lambda data: visiblename(data[0]), + attrs = filter(lambda data: visiblename(data[0], obj=object), classify_class_attrs(object)) mdict = {} for key, kind, homecls, value in attrs: @@ -1042,18 +1045,18 @@ # if __all__ exists, believe it. Otherwise use old heuristic. if (all is not None or (inspect.getmodule(value) or object) is object): - if visiblename(key, all): + if visiblename(key, all, object): classes.append((key, value)) funcs = [] for key, value in inspect.getmembers(object, inspect.isroutine): # if __all__ exists, believe it. Otherwise use old heuristic. if (all is not None or inspect.isbuiltin(value) or inspect.getmodule(value) is object): - if visiblename(key, all): + if visiblename(key, all, object): funcs.append((key, value)) data = [] for key, value in inspect.getmembers(object, isdata): - if visiblename(key, all): + if visiblename(key, all, object): data.append((key, value)) modpkgs = [] @@ -1113,7 +1116,7 @@ result = result + self.section('CREDITS', str(object.__credits__)) return result - def docclass(self, object, name=None, mod=None): + def docclass(self, object, name=None, mod=None, *ignored): """Produce text documentation for a given class object.""" realname = object.__name__ name = name or realname @@ -1186,7 +1189,7 @@ name, mod, maxlen=70, doc=doc) + '\n') return attrs - attrs = filter(lambda data: visiblename(data[0]), + attrs = filter(lambda data: visiblename(data[0], obj=object), classify_class_attrs(object)) while attrs: if mro: @@ -1718,8 +1721,9 @@ return '' return '<pydoc.Helper instance>' - def __call__(self, request=None): - if request is not None: + _GoInteractive = object() + def __call__(self, request=_GoInteractive): + if request is not self._GoInteractive: self.help(request) else: self.intro() diff --git a/lib-python/2.7/pydoc_data/topics.py b/lib-python/2.7/pydoc_data/topics.py --- a/lib-python/2.7/pydoc_data/topics.py +++ b/lib-python/2.7/pydoc_data/topics.py @@ -1,16 +1,16 @@ -# Autogenerated by Sphinx on Sat Jul 3 08:52:04 2010 +# Autogenerated by Sphinx on Sat Jun 11 09:49:30 2011 topics = {'assert': u'\nThe ``assert`` statement\n************************\n\nAssert statements are a convenient way to insert debugging assertions\ninto a program:\n\n assert_stmt ::= "assert" expression ["," expression]\n\nThe simple form, ``assert expression``, is equivalent to\n\n if __debug__:\n if not expression: raise AssertionError\n\nThe extended form, ``assert expression1, expression2``, is equivalent\nto\n\n if __debug__:\n if not expression1: raise AssertionError(expression2)\n\nThese equivalences assume that ``__debug__`` and ``AssertionError``\nrefer to the built-in variables with those names. In the current\nimplementation, the built-in variable ``__debug__`` is ``True`` under\nnormal circumstances, ``False`` when optimization is requested\n(command line option -O). The current code generator emits no code\nfor an assert statement when optimization is requested at compile\ntime. Note that it is unnecessary to include the source code for the\nexpression that failed in the error message; it will be displayed as\npart of the stack trace.\n\nAssignments to ``__debug__`` are illegal. The value for the built-in\nvariable is determined when the interpreter starts.\n', - 'assignment': u'\nAssignment statements\n*********************\n\nAssignment statements are used to (re)bind names to values and to\nmodify attributes or items of mutable objects:\n\n assignment_stmt ::= (target_list "=")+ (expression_list | yield_expression)\n target_list ::= target ("," target)* [","]\n target ::= identifier\n | "(" target_list ")"\n | "[" target_list "]"\n | attributeref\n | subscription\n | slicing\n\n(See section *Primaries* for the syntax definitions for the last three\nsymbols.)\n\nAn assignment statement evaluates the expression list (remember that\nthis can be a single expression or a comma-separated list, the latter\nyielding a tuple) and assigns the single resulting object to each of\nthe target lists, from left to right.\n\nAssignment is defined recursively depending on the form of the target\n(list). When a target is part of a mutable object (an attribute\nreference, subscription or slicing), the mutable object must\nultimately perform the assignment and decide about its validity, and\nmay raise an exception if the assignment is unacceptable. The rules\nobserved by various types and the exceptions raised are given with the\ndefinition of the object types (see section *The standard type\nhierarchy*).\n\nAssignment of an object to a target list is recursively defined as\nfollows.\n\n* If the target list is a single target: The object is assigned to\n that target.\n\n* If the target list is a comma-separated list of targets: The object\n must be an iterable with the same number of items as there are\n targets in the target list, and the items are assigned, from left to\n right, to the corresponding targets. (This rule is relaxed as of\n Python 1.5; in earlier versions, the object had to be a tuple.\n Since strings are sequences, an assignment like ``a, b = "xy"`` is\n now legal as long as the string has the right length.)\n\nAssignment of an object to a single target is recursively defined as\nfollows.\n\n* If the target is an identifier (name):\n\n * If the name does not occur in a ``global`` statement in the\n current code block: the name is bound to the object in the current\n local namespace.\n\n * Otherwise: the name is bound to the object in the current global\n namespace.\n\n The name is rebound if it was already bound. This may cause the\n reference count for the object previously bound to the name to reach\n zero, causing the object to be deallocated and its destructor (if it\n has one) to be called.\n\n* If the target is a target list enclosed in parentheses or in square\n brackets: The object must be an iterable with the same number of\n items as there are targets in the target list, and its items are\n assigned, from left to right, to the corresponding targets.\n\n* If the target is an attribute reference: The primary expression in\n the reference is evaluated. It should yield an object with\n assignable attributes; if this is not the case, ``TypeError`` is\n raised. That object is then asked to assign the assigned object to\n the given attribute; if it cannot perform the assignment, it raises\n an exception (usually but not necessarily ``AttributeError``).\n\n Note: If the object is a class instance and the attribute reference\n occurs on both sides of the assignment operator, the RHS expression,\n ``a.x`` can access either an instance attribute or (if no instance\n attribute exists) a class attribute. The LHS target ``a.x`` is\n always set as an instance attribute, creating it if necessary.\n Thus, the two occurrences of ``a.x`` do not necessarily refer to the\n same attribute: if the RHS expression refers to a class attribute,\n the LHS creates a new instance attribute as the target of the\n assignment:\n\n class Cls:\n x = 3 # class variable\n inst = Cls()\n inst.x = inst.x + 1 # writes inst.x as 4 leaving Cls.x as 3\n\n This description does not necessarily apply to descriptor\n attributes, such as properties created with ``property()``.\n\n* If the target is a subscription: The primary expression in the\n reference is evaluated. It should yield either a mutable sequence\n object (such as a list) or a mapping object (such as a dictionary).\n Next, the subscript expression is evaluated.\n\n If the primary is a mutable sequence object (such as a list), the\n subscript must yield a plain integer. If it is negative, the\n sequence\'s length is added to it. The resulting value must be a\n nonnegative integer less than the sequence\'s length, and the\n sequence is asked to assign the assigned object to its item with\n that index. If the index is out of range, ``IndexError`` is raised\n (assignment to a subscripted sequence cannot add new items to a\n list).\n\n If the primary is a mapping object (such as a dictionary), the\n subscript must have a type compatible with the mapping\'s key type,\n and the mapping is then asked to create a key/datum pair which maps\n the subscript to the assigned object. This can either replace an\n existing key/value pair with the same key value, or insert a new\n key/value pair (if no key with the same value existed).\n\n* If the target is a slicing: The primary expression in the reference\n is evaluated. It should yield a mutable sequence object (such as a\n list). The assigned object should be a sequence object of the same\n type. Next, the lower and upper bound expressions are evaluated,\n insofar they are present; defaults are zero and the sequence\'s\n length. The bounds should evaluate to (small) integers. If either\n bound is negative, the sequence\'s length is added to it. The\n resulting bounds are clipped to lie between zero and the sequence\'s\n length, inclusive. Finally, the sequence object is asked to replace\n the slice with the items of the assigned sequence. The length of\n the slice may be different from the length of the assigned sequence,\n thus changing the length of the target sequence, if the object\n allows it.\n\n**CPython implementation detail:** In the current implementation, the\nsyntax for targets is taken to be the same as for expressions, and\ninvalid syntax is rejected during the code generation phase, causing\nless detailed error messages.\n\nWARNING: Although the definition of assignment implies that overlaps\nbetween the left-hand side and the right-hand side are \'safe\' (for\nexample ``a, b = b, a`` swaps two variables), overlaps *within* the\ncollection of assigned-to variables are not safe! For instance, the\nfollowing program prints ``[0, 2]``:\n\n x = [0, 1]\n i = 0\n i, x[i] = 1, 2\n print x\n\n\nAugmented assignment statements\n===============================\n\nAugmented assignment is the combination, in a single statement, of a\nbinary operation and an assignment statement:\n\n augmented_assignment_stmt ::= augtarget augop (expression_list | yield_expression)\n augtarget ::= identifier | attributeref | subscription | slicing\n augop ::= "+=" | "-=" | "*=" | "/=" | "//=" | "%=" | "**="\n | ">>=" | "<<=" | "&=" | "^=" | "|="\n\n(See section *Primaries* for the syntax definitions for the last three\nsymbols.)\n\nAn augmented assignment evaluates the target (which, unlike normal\nassignment statements, cannot be an unpacking) and the expression\nlist, performs the binary operation specific to the type of assignment\non the two operands, and assigns the result to the original target.\nThe target is only evaluated once.\n\nAn augmented assignment expression like ``x += 1`` can be rewritten as\n``x = x + 1`` to achieve a similar, but not exactly equal effect. In\nthe augmented version, ``x`` is only evaluated once. Also, when\npossible, the actual operation is performed *in-place*, meaning that\nrather than creating a new object and assigning that to the target,\nthe old object is modified instead.\n\nWith the exception of assigning to tuples and multiple targets in a\nsingle statement, the assignment done by augmented assignment\nstatements is handled the same way as normal assignments. Similarly,\nwith the exception of the possible *in-place* behavior, the binary\noperation performed by augmented assignment is the same as the normal\nbinary operations.\n\nFor targets which are attribute references, the same *caveat about\nclass and instance attributes* applies as for regular assignments.\n', + 'assignment': u'\nAssignment statements\n*********************\n\nAssignment statements are used to (re)bind names to values and to\nmodify attributes or items of mutable objects:\n\n assignment_stmt ::= (target_list "=")+ (expression_list | yield_expression)\n target_list ::= target ("," target)* [","]\n target ::= identifier\n | "(" target_list ")"\n | "[" target_list "]"\n | attributeref\n | subscription\n | slicing\n\n(See section *Primaries* for the syntax definitions for the last three\nsymbols.)\n\nAn assignment statement evaluates the expression list (remember that\nthis can be a single expression or a comma-separated list, the latter\nyielding a tuple) and assigns the single resulting object to each of\nthe target lists, from left to right.\n\nAssignment is defined recursively depending on the form of the target\n(list). When a target is part of a mutable object (an attribute\nreference, subscription or slicing), the mutable object must\nultimately perform the assignment and decide about its validity, and\nmay raise an exception if the assignment is unacceptable. The rules\nobserved by various types and the exceptions raised are given with the\ndefinition of the object types (see section *The standard type\nhierarchy*).\n\nAssignment of an object to a target list is recursively defined as\nfollows.\n\n* If the target list is a single target: The object is assigned to\n that target.\n\n* If the target list is a comma-separated list of targets: The object\n must be an iterable with the same number of items as there are\n targets in the target list, and the items are assigned, from left to\n right, to the corresponding targets.\n\nAssignment of an object to a single target is recursively defined as\nfollows.\n\n* If the target is an identifier (name):\n\n * If the name does not occur in a ``global`` statement in the\n current code block: the name is bound to the object in the current\n local namespace.\n\n * Otherwise: the name is bound to the object in the current global\n namespace.\n\n The name is rebound if it was already bound. This may cause the\n reference count for the object previously bound to the name to reach\n zero, causing the object to be deallocated and its destructor (if it\n has one) to be called.\n\n* If the target is a target list enclosed in parentheses or in square\n brackets: The object must be an iterable with the same number of\n items as there are targets in the target list, and its items are\n assigned, from left to right, to the corresponding targets.\n\n* If the target is an attribute reference: The primary expression in\n the reference is evaluated. It should yield an object with\n assignable attributes; if this is not the case, ``TypeError`` is\n raised. That object is then asked to assign the assigned object to\n the given attribute; if it cannot perform the assignment, it raises\n an exception (usually but not necessarily ``AttributeError``).\n\n Note: If the object is a class instance and the attribute reference\n occurs on both sides of the assignment operator, the RHS expression,\n ``a.x`` can access either an instance attribute or (if no instance\n attribute exists) a class attribute. The LHS target ``a.x`` is\n always set as an instance attribute, creating it if necessary.\n Thus, the two occurrences of ``a.x`` do not necessarily refer to the\n same attribute: if the RHS expression refers to a class attribute,\n the LHS creates a new instance attribute as the target of the\n assignment:\n\n class Cls:\n x = 3 # class variable\n inst = Cls()\n inst.x = inst.x + 1 # writes inst.x as 4 leaving Cls.x as 3\n\n This description does not necessarily apply to descriptor\n attributes, such as properties created with ``property()``.\n\n* If the target is a subscription: The primary expression in the\n reference is evaluated. It should yield either a mutable sequence\n object (such as a list) or a mapping object (such as a dictionary).\n Next, the subscript expression is evaluated.\n\n If the primary is a mutable sequence object (such as a list), the\n subscript must yield a plain integer. If it is negative, the\n sequence\'s length is added to it. The resulting value must be a\n nonnegative integer less than the sequence\'s length, and the\n sequence is asked to assign the assigned object to its item with\n that index. If the index is out of range, ``IndexError`` is raised\n (assignment to a subscripted sequence cannot add new items to a\n list).\n\n If the primary is a mapping object (such as a dictionary), the\n subscript must have a type compatible with the mapping\'s key type,\n and the mapping is then asked to create a key/datum pair which maps\n the subscript to the assigned object. This can either replace an\n existing key/value pair with the same key value, or insert a new\n key/value pair (if no key with the same value existed).\n\n* If the target is a slicing: The primary expression in the reference\n is evaluated. It should yield a mutable sequence object (such as a\n list). The assigned object should be a sequence object of the same\n type. Next, the lower and upper bound expressions are evaluated,\n insofar they are present; defaults are zero and the sequence\'s\n length. The bounds should evaluate to (small) integers. If either\n bound is negative, the sequence\'s length is added to it. The\n resulting bounds are clipped to lie between zero and the sequence\'s\n length, inclusive. Finally, the sequence object is asked to replace\n the slice with the items of the assigned sequence. The length of\n the slice may be different from the length of the assigned sequence,\n thus changing the length of the target sequence, if the object\n allows it.\n\n**CPython implementation detail:** In the current implementation, the\nsyntax for targets is taken to be the same as for expressions, and\ninvalid syntax is rejected during the code generation phase, causing\nless detailed error messages.\n\nWARNING: Although the definition of assignment implies that overlaps\nbetween the left-hand side and the right-hand side are \'safe\' (for\nexample ``a, b = b, a`` swaps two variables), overlaps *within* the\ncollection of assigned-to variables are not safe! For instance, the\nfollowing program prints ``[0, 2]``:\n\n x = [0, 1]\n i = 0\n i, x[i] = 1, 2\n print x\n\n\nAugmented assignment statements\n===============================\n\nAugmented assignment is the combination, in a single statement, of a\nbinary operation and an assignment statement:\n\n augmented_assignment_stmt ::= augtarget augop (expression_list | yield_expression)\n augtarget ::= identifier | attributeref | subscription | slicing\n augop ::= "+=" | "-=" | "*=" | "/=" | "//=" | "%=" | "**="\n | ">>=" | "<<=" | "&=" | "^=" | "|="\n\n(See section *Primaries* for the syntax definitions for the last three\nsymbols.)\n\nAn augmented assignment evaluates the target (which, unlike normal\nassignment statements, cannot be an unpacking) and the expression\nlist, performs the binary operation specific to the type of assignment\non the two operands, and assigns the result to the original target.\nThe target is only evaluated once.\n\nAn augmented assignment expression like ``x += 1`` can be rewritten as\n``x = x + 1`` to achieve a similar, but not exactly equal effect. In\nthe augmented version, ``x`` is only evaluated once. Also, when\npossible, the actual operation is performed *in-place*, meaning that\nrather than creating a new object and assigning that to the target,\nthe old object is modified instead.\n\nWith the exception of assigning to tuples and multiple targets in a\nsingle statement, the assignment done by augmented assignment\nstatements is handled the same way as normal assignments. Similarly,\nwith the exception of the possible *in-place* behavior, the binary\noperation performed by augmented assignment is the same as the normal\nbinary operations.\n\nFor targets which are attribute references, the same *caveat about\nclass and instance attributes* applies as for regular assignments.\n', 'atom-identifiers': u'\nIdentifiers (Names)\n*******************\n\nAn identifier occurring as an atom is a name. See section\n*Identifiers and keywords* for lexical definition and section *Naming\nand binding* for documentation of naming and binding.\n\nWhen the name is bound to an object, evaluation of the atom yields\nthat object. When a name is not bound, an attempt to evaluate it\nraises a ``NameError`` exception.\n\n**Private name mangling:** When an identifier that textually occurs in\na class definition begins with two or more underscore characters and\ndoes not end in two or more underscores, it is considered a *private\nname* of that class. Private names are transformed to a longer form\nbefore code is generated for them. The transformation inserts the\nclass name in front of the name, with leading underscores removed, and\na single underscore inserted in front of the class name. For example,\nthe identifier ``__spam`` occurring in a class named ``Ham`` will be\ntransformed to ``_Ham__spam``. This transformation is independent of\nthe syntactical context in which the identifier is used. If the\ntransformed name is extremely long (longer than 255 characters),\nimplementation defined truncation may happen. If the class name\nconsists only of underscores, no transformation is done.\n', 'atom-literals': u"\nLiterals\n********\n\nPython supports string literals and various numeric literals:\n\n literal ::= stringliteral | integer | longinteger\n | floatnumber | imagnumber\n\nEvaluation of a literal yields an object of the given type (string,\ninteger, long integer, floating point number, complex number) with the\ngiven value. The value may be approximated in the case of floating\npoint and imaginary (complex) literals. See section *Literals* for\ndetails.\n\nAll literals correspond to immutable data types, and hence the\nobject's identity is less important than its value. Multiple\nevaluations of literals with the same value (either the same\noccurrence in the program text or a different occurrence) may obtain\nthe same object or a different object with the same value.\n", - 'attribute-access': u'\nCustomizing attribute access\n****************************\n\nThe following methods can be defined to customize the meaning of\nattribute access (use of, assignment to, or deletion of ``x.name``)\nfor class instances.\n\nobject.__getattr__(self, name)\n\n Called when an attribute lookup has not found the attribute in the\n usual places (i.e. it is not an instance attribute nor is it found\n in the class tree for ``self``). ``name`` is the attribute name.\n This method should return the (computed) attribute value or raise\n an ``AttributeError`` exception.\n\n Note that if the attribute is found through the normal mechanism,\n ``__getattr__()`` is not called. (This is an intentional asymmetry\n between ``__getattr__()`` and ``__setattr__()``.) This is done both\n for efficiency reasons and because otherwise ``__getattr__()``\n would have no way to access other attributes of the instance. Note\n that at least for instance variables, you can fake total control by\n not inserting any values in the instance attribute dictionary (but\n instead inserting them in another object). See the\n ``__getattribute__()`` method below for a way to actually get total\n control in new-style classes.\n\nobject.__setattr__(self, name, value)\n\n Called when an attribute assignment is attempted. This is called\n instead of the normal mechanism (i.e. store the value in the\n instance dictionary). *name* is the attribute name, *value* is the\n value to be assigned to it.\n\n If ``__setattr__()`` wants to assign to an instance attribute, it\n should not simply execute ``self.name = value`` --- this would\n cause a recursive call to itself. Instead, it should insert the\n value in the dictionary of instance attributes, e.g.,\n ``self.__dict__[name] = value``. For new-style classes, rather\n than accessing the instance dictionary, it should call the base\n class method with the same name, for example,\n ``object.__setattr__(self, name, value)``.\n\nobject.__delattr__(self, name)\n\n Like ``__setattr__()`` but for attribute deletion instead of\n assignment. This should only be implemented if ``del obj.name`` is\n meaningful for the object.\n\n\nMore attribute access for new-style classes\n===========================================\n\nThe following methods only apply to new-style classes.\n\nobject.__getattribute__(self, name)\n\n Called unconditionally to implement attribute accesses for\n instances of the class. If the class also defines\n ``__getattr__()``, the latter will not be called unless\n ``__getattribute__()`` either calls it explicitly or raises an\n ``AttributeError``. This method should return the (computed)\n attribute value or raise an ``AttributeError`` exception. In order\n to avoid infinite recursion in this method, its implementation\n should always call the base class method with the same name to\n access any attributes it needs, for example,\n ``object.__getattribute__(self, name)``.\n\n Note: This method may still be bypassed when looking up special methods\n as the result of implicit invocation via language syntax or\n built-in functions. See *Special method lookup for new-style\n classes*.\n\n\nImplementing Descriptors\n========================\n\nThe following methods only apply when an instance of the class\ncontaining the method (a so-called *descriptor* class) appears in the\nclass dictionary of another new-style class, known as the *owner*\nclass. In the examples below, "the attribute" refers to the attribute\nwhose name is the key of the property in the owner class\'\n``__dict__``. Descriptors can only be implemented as new-style\nclasses themselves.\n\nobject.__get__(self, instance, owner)\n\n Called to get the attribute of the owner class (class attribute\n access) or of an instance of that class (instance attribute\n access). *owner* is always the owner class, while *instance* is the\n instance that the attribute was accessed through, or ``None`` when\n the attribute is accessed through the *owner*. This method should\n return the (computed) attribute value or raise an\n ``AttributeError`` exception.\n\nobject.__set__(self, instance, value)\n\n Called to set the attribute on an instance *instance* of the owner\n class to a new value, *value*.\n\nobject.__delete__(self, instance)\n\n Called to delete the attribute on an instance *instance* of the\n owner class.\n\n\nInvoking Descriptors\n====================\n\nIn general, a descriptor is an object attribute with "binding\nbehavior", one whose attribute access has been overridden by methods\nin the descriptor protocol: ``__get__()``, ``__set__()``, and\n``__delete__()``. If any of those methods are defined for an object,\nit is said to be a descriptor.\n\nThe default behavior for attribute access is to get, set, or delete\nthe attribute from an object\'s dictionary. For instance, ``a.x`` has a\nlookup chain starting with ``a.__dict__[\'x\']``, then\n``type(a).__dict__[\'x\']``, and continuing through the base classes of\n``type(a)`` excluding metaclasses.\n\nHowever, if the looked-up value is an object defining one of the\ndescriptor methods, then Python may override the default behavior and\ninvoke the descriptor method instead. Where this occurs in the\nprecedence chain depends on which descriptor methods were defined and\nhow they were called. Note that descriptors are only invoked for new\nstyle objects or classes (ones that subclass ``object()`` or\n``type()``).\n\nThe starting point for descriptor invocation is a binding, ``a.x``.\nHow the arguments are assembled depends on ``a``:\n\nDirect Call\n The simplest and least common call is when user code directly\n invokes a descriptor method: ``x.__get__(a)``.\n\nInstance Binding\n If binding to a new-style object instance, ``a.x`` is transformed\n into the call: ``type(a).__dict__[\'x\'].__get__(a, type(a))``.\n\nClass Binding\n If binding to a new-style class, ``A.x`` is transformed into the\n call: ``A.__dict__[\'x\'].__get__(None, A)``.\n\nSuper Binding\n If ``a`` is an instance of ``super``, then the binding ``super(B,\n obj).m()`` searches ``obj.__class__.__mro__`` for the base class\n ``A`` immediately preceding ``B`` and then invokes the descriptor\n with the call: ``A.__dict__[\'m\'].__get__(obj, A)``.\n\nFor instance bindings, the precedence of descriptor invocation depends\non the which descriptor methods are defined. A descriptor can define\nany combination of ``__get__()``, ``__set__()`` and ``__delete__()``.\nIf it does not define ``__get__()``, then accessing the attribute will\nreturn the descriptor object itself unless there is a value in the\nobject\'s instance dictionary. If the descriptor defines ``__set__()``\nand/or ``__delete__()``, it is a data descriptor; if it defines\nneither, it is a non-data descriptor. Normally, data descriptors\ndefine both ``__get__()`` and ``__set__()``, while non-data\ndescriptors have just the ``__get__()`` method. Data descriptors with\n``__set__()`` and ``__get__()`` defined always override a redefinition\nin an instance dictionary. In contrast, non-data descriptors can be\noverridden by instances.\n\nPython methods (including ``staticmethod()`` and ``classmethod()``)\nare implemented as non-data descriptors. Accordingly, instances can\nredefine and override methods. This allows individual instances to\nacquire behaviors that differ from other instances of the same class.\n\nThe ``property()`` function is implemented as a data descriptor.\nAccordingly, instances cannot override the behavior of a property.\n\n\n__slots__\n=========\n\nBy default, instances of both old and new-style classes have a\ndictionary for attribute storage. This wastes space for objects\nhaving very few instance variables. The space consumption can become\nacute when creating large numbers of instances.\n\nThe default can be overridden by defining *__slots__* in a new-style\nclass definition. The *__slots__* declaration takes a sequence of\ninstance variables and reserves just enough space in each instance to\nhold a value for each variable. Space is saved because *__dict__* is\nnot created for each instance.\n\n__slots__\n\n This class variable can be assigned a string, iterable, or sequence\n of strings with variable names used by instances. If defined in a\n new-style class, *__slots__* reserves space for the declared\n variables and prevents the automatic creation of *__dict__* and\n *__weakref__* for each instance.\n\n New in version 2.2.\n\nNotes on using *__slots__*\n\n* When inheriting from a class without *__slots__*, the *__dict__*\n attribute of that class will always be accessible, so a *__slots__*\n definition in the subclass is meaningless.\n\n* Without a *__dict__* variable, instances cannot be assigned new\n variables not listed in the *__slots__* definition. Attempts to\n assign to an unlisted variable name raises ``AttributeError``. If\n dynamic assignment of new variables is desired, then add\n ``\'__dict__\'`` to the sequence of strings in the *__slots__*\n declaration.\n\n Changed in version 2.3: Previously, adding ``\'__dict__\'`` to the\n *__slots__* declaration would not enable the assignment of new\n attributes not specifically listed in the sequence of instance\n variable names.\n\n* Without a *__weakref__* variable for each instance, classes defining\n *__slots__* do not support weak references to its instances. If weak\n reference support is needed, then add ``\'__weakref__\'`` to the\n sequence of strings in the *__slots__* declaration.\n\n Changed in version 2.3: Previously, adding ``\'__weakref__\'`` to the\n *__slots__* declaration would not enable support for weak\n references.\n\n* *__slots__* are implemented at the class level by creating\n descriptors (*Implementing Descriptors*) for each variable name. As\n a result, class attributes cannot be used to set default values for\n instance variables defined by *__slots__*; otherwise, the class\n attribute would overwrite the descriptor assignment.\n\n* The action of a *__slots__* declaration is limited to the class\n where it is defined. As a result, subclasses will have a *__dict__*\n unless they also define *__slots__* (which must only contain names\n of any *additional* slots).\n\n* If a class defines a slot also defined in a base class, the instance\n variable defined by the base class slot is inaccessible (except by\n retrieving its descriptor directly from the base class). This\n renders the meaning of the program undefined. In the future, a\n check may be added to prevent this.\n\n* Nonempty *__slots__* does not work for classes derived from\n "variable-length" built-in types such as ``long``, ``str`` and\n ``tuple``.\n\n* Any non-string iterable may be assigned to *__slots__*. Mappings may\n also be used; however, in the future, special meaning may be\n assigned to the values corresponding to each key.\n\n* *__class__* assignment works only if both classes have the same\n *__slots__*.\n\n Changed in version 2.6: Previously, *__class__* assignment raised an\n error if either new or old class had *__slots__*.\n', + 'attribute-access': u'\nCustomizing attribute access\n****************************\n\nThe following methods can be defined to customize the meaning of\nattribute access (use of, assignment to, or deletion of ``x.name``)\nfor class instances.\n\nobject.__getattr__(self, name)\n\n Called when an attribute lookup has not found the attribute in the\n usual places (i.e. it is not an instance attribute nor is it found\n in the class tree for ``self``). ``name`` is the attribute name.\n This method should return the (computed) attribute value or raise\n an ``AttributeError`` exception.\n\n Note that if the attribute is found through the normal mechanism,\n ``__getattr__()`` is not called. (This is an intentional asymmetry\n between ``__getattr__()`` and ``__setattr__()``.) This is done both\n for efficiency reasons and because otherwise ``__getattr__()``\n would have no way to access other attributes of the instance. Note\n that at least for instance variables, you can fake total control by\n not inserting any values in the instance attribute dictionary (but\n instead inserting them in another object). See the\n ``__getattribute__()`` method below for a way to actually get total\n control in new-style classes.\n\nobject.__setattr__(self, name, value)\n\n Called when an attribute assignment is attempted. This is called\n instead of the normal mechanism (i.e. store the value in the\n instance dictionary). *name* is the attribute name, *value* is the\n value to be assigned to it.\n\n If ``__setattr__()`` wants to assign to an instance attribute, it\n should not simply execute ``self.name = value`` --- this would\n cause a recursive call to itself. Instead, it should insert the\n value in the dictionary of instance attributes, e.g.,\n ``self.__dict__[name] = value``. For new-style classes, rather\n than accessing the instance dictionary, it should call the base\n class method with the same name, for example,\n ``object.__setattr__(self, name, value)``.\n\nobject.__delattr__(self, name)\n\n Like ``__setattr__()`` but for attribute deletion instead of\n assignment. This should only be implemented if ``del obj.name`` is\n meaningful for the object.\n\n\nMore attribute access for new-style classes\n===========================================\n\nThe following methods only apply to new-style classes.\n\nobject.__getattribute__(self, name)\n\n Called unconditionally to implement attribute accesses for\n instances of the class. If the class also defines\n ``__getattr__()``, the latter will not be called unless\n ``__getattribute__()`` either calls it explicitly or raises an\n ``AttributeError``. This method should return the (computed)\n attribute value or raise an ``AttributeError`` exception. In order\n to avoid infinite recursion in this method, its implementation\n should always call the base class method with the same name to\n access any attributes it needs, for example,\n ``object.__getattribute__(self, name)``.\n\n Note: This method may still be bypassed when looking up special methods\n as the result of implicit invocation via language syntax or\n built-in functions. See *Special method lookup for new-style\n classes*.\n\n\nImplementing Descriptors\n========================\n\nThe following methods only apply when an instance of the class\ncontaining the method (a so-called *descriptor* class) appears in an\n*owner* class (the descriptor must be in either the owner\'s class\ndictionary or in the class dictionary for one of its parents). In the\nexamples below, "the attribute" refers to the attribute whose name is\nthe key of the property in the owner class\' ``__dict__``.\n\nobject.__get__(self, instance, owner)\n\n Called to get the attribute of the owner class (class attribute\n access) or of an instance of that class (instance attribute\n access). *owner* is always the owner class, while *instance* is the\n instance that the attribute was accessed through, or ``None`` when\n the attribute is accessed through the *owner*. This method should\n return the (computed) attribute value or raise an\n ``AttributeError`` exception.\n\nobject.__set__(self, instance, value)\n\n Called to set the attribute on an instance *instance* of the owner\n class to a new value, *value*.\n\nobject.__delete__(self, instance)\n\n Called to delete the attribute on an instance *instance* of the\n owner class.\n\n\nInvoking Descriptors\n====================\n\nIn general, a descriptor is an object attribute with "binding\nbehavior", one whose attribute access has been overridden by methods\nin the descriptor protocol: ``__get__()``, ``__set__()``, and\n``__delete__()``. If any of those methods are defined for an object,\nit is said to be a descriptor.\n\nThe default behavior for attribute access is to get, set, or delete\nthe attribute from an object\'s dictionary. For instance, ``a.x`` has a\nlookup chain starting with ``a.__dict__[\'x\']``, then\n``type(a).__dict__[\'x\']``, and continuing through the base classes of\n``type(a)`` excluding metaclasses.\n\nHowever, if the looked-up value is an object defining one of the\ndescriptor methods, then Python may override the default behavior and\ninvoke the descriptor method instead. Where this occurs in the\nprecedence chain depends on which descriptor methods were defined and\nhow they were called. Note that descriptors are only invoked for new\nstyle objects or classes (ones that subclass ``object()`` or\n``type()``).\n\nThe starting point for descriptor invocation is a binding, ``a.x``.\nHow the arguments are assembled depends on ``a``:\n\nDirect Call\n The simplest and least common call is when user code directly\n invokes a descriptor method: ``x.__get__(a)``.\n\nInstance Binding\n If binding to a new-style object instance, ``a.x`` is transformed\n into the call: ``type(a).__dict__[\'x\'].__get__(a, type(a))``.\n\nClass Binding\n If binding to a new-style class, ``A.x`` is transformed into the\n call: ``A.__dict__[\'x\'].__get__(None, A)``.\n\nSuper Binding\n If ``a`` is an instance of ``super``, then the binding ``super(B,\n obj).m()`` searches ``obj.__class__.__mro__`` for the base class\n ``A`` immediately preceding ``B`` and then invokes the descriptor\n with the call: ``A.__dict__[\'m\'].__get__(obj, obj.__class__)``.\n\nFor instance bindings, the precedence of descriptor invocation depends\non the which descriptor methods are defined. A descriptor can define\nany combination of ``__get__()``, ``__set__()`` and ``__delete__()``.\nIf it does not define ``__get__()``, then accessing the attribute will\nreturn the descriptor object itself unless there is a value in the\nobject\'s instance dictionary. If the descriptor defines ``__set__()``\nand/or ``__delete__()``, it is a data descriptor; if it defines\nneither, it is a non-data descriptor. Normally, data descriptors\ndefine both ``__get__()`` and ``__set__()``, while non-data\ndescriptors have just the ``__get__()`` method. Data descriptors with\n``__set__()`` and ``__get__()`` defined always override a redefinition\nin an instance dictionary. In contrast, non-data descriptors can be\noverridden by instances.\n\nPython methods (including ``staticmethod()`` and ``classmethod()``)\nare implemented as non-data descriptors. Accordingly, instances can\nredefine and override methods. This allows individual instances to\nacquire behaviors that differ from other instances of the same class.\n\nThe ``property()`` function is implemented as a data descriptor.\nAccordingly, instances cannot override the behavior of a property.\n\n\n__slots__\n=========\n\nBy default, instances of both old and new-style classes have a\ndictionary for attribute storage. This wastes space for objects\nhaving very few instance variables. The space consumption can become\nacute when creating large numbers of instances.\n\nThe default can be overridden by defining *__slots__* in a new-style\nclass definition. The *__slots__* declaration takes a sequence of\ninstance variables and reserves just enough space in each instance to\nhold a value for each variable. Space is saved because *__dict__* is\nnot created for each instance.\n\n__slots__\n\n This class variable can be assigned a string, iterable, or sequence\n of strings with variable names used by instances. If defined in a\n new-style class, *__slots__* reserves space for the declared\n variables and prevents the automatic creation of *__dict__* and\n *__weakref__* for each instance.\n\n New in version 2.2.\n\nNotes on using *__slots__*\n\n* When inheriting from a class without *__slots__*, the *__dict__*\n attribute of that class will always be accessible, so a *__slots__*\n definition in the subclass is meaningless.\n\n* Without a *__dict__* variable, instances cannot be assigned new\n variables not listed in the *__slots__* definition. Attempts to\n assign to an unlisted variable name raises ``AttributeError``. If\n dynamic assignment of new variables is desired, then add\n ``\'__dict__\'`` to the sequence of strings in the *__slots__*\n declaration.\n\n Changed in version 2.3: Previously, adding ``\'__dict__\'`` to the\n *__slots__* declaration would not enable the assignment of new\n attributes not specifically listed in the sequence of instance\n variable names.\n\n* Without a *__weakref__* variable for each instance, classes defining\n *__slots__* do not support weak references to its instances. If weak\n reference support is needed, then add ``\'__weakref__\'`` to the\n sequence of strings in the *__slots__* declaration.\n\n Changed in version 2.3: Previously, adding ``\'__weakref__\'`` to the\n *__slots__* declaration would not enable support for weak\n references.\n\n* *__slots__* are implemented at the class level by creating\n descriptors (*Implementing Descriptors*) for each variable name. As\n a result, class attributes cannot be used to set default values for\n instance variables defined by *__slots__*; otherwise, the class\n attribute would overwrite the descriptor assignment.\n\n* The action of a *__slots__* declaration is limited to the class\n where it is defined. As a result, subclasses will have a *__dict__*\n unless they also define *__slots__* (which must only contain names\n of any *additional* slots).\n\n* If a class defines a slot also defined in a base class, the instance\n variable defined by the base class slot is inaccessible (except by\n retrieving its descriptor directly from the base class). This\n renders the meaning of the program undefined. In the future, a\n check may be added to prevent this.\n\n* Nonempty *__slots__* does not work for classes derived from\n "variable-length" built-in types such as ``long``, ``str`` and\n ``tuple``.\n\n* Any non-string iterable may be assigned to *__slots__*. Mappings may\n also be used; however, in the future, special meaning may be\n assigned to the values corresponding to each key.\n\n* *__class__* assignment works only if both classes have the same\n *__slots__*.\n\n Changed in version 2.6: Previously, *__class__* assignment raised an\n error if either new or old class had *__slots__*.\n', 'attribute-references': u'\nAttribute references\n********************\n\nAn attribute reference is a primary followed by a period and a name:\n\n attributeref ::= primary "." identifier\n\nThe primary must evaluate to an object of a type that supports\nattribute references, e.g., a module, list, or an instance. This\nobject is then asked to produce the attribute whose name is the\nidentifier. If this attribute is not available, the exception\n``AttributeError`` is raised. Otherwise, the type and value of the\nobject produced is determined by the object. Multiple evaluations of\nthe same attribute reference may yield different objects.\n', 'augassign': u'\nAugmented assignment statements\n*******************************\n\nAugmented assignment is the combination, in a single statement, of a\nbinary operation and an assignment statement:\n\n augmented_assignment_stmt ::= augtarget augop (expression_list | yield_expression)\n augtarget ::= identifier | attributeref | subscription | slicing\n augop ::= "+=" | "-=" | "*=" | "/=" | "//=" | "%=" | "**="\n | ">>=" | "<<=" | "&=" | "^=" | "|="\n\n(See section *Primaries* for the syntax definitions for the last three\nsymbols.)\n\nAn augmented assignment evaluates the target (which, unlike normal\nassignment statements, cannot be an unpacking) and the expression\nlist, performs the binary operation specific to the type of assignment\non the two operands, and assigns the result to the original target.\nThe target is only evaluated once.\n\nAn augmented assignment expression like ``x += 1`` can be rewritten as\n``x = x + 1`` to achieve a similar, but not exactly equal effect. In\nthe augmented version, ``x`` is only evaluated once. Also, when\npossible, the actual operation is performed *in-place*, meaning that\nrather than creating a new object and assigning that to the target,\nthe old object is modified instead.\n\nWith the exception of assigning to tuples and multiple targets in a\nsingle statement, the assignment done by augmented assignment\nstatements is handled the same way as normal assignments. Similarly,\nwith the exception of the possible *in-place* behavior, the binary\noperation performed by augmented assignment is the same as the normal\nbinary operations.\n\nFor targets which are attribute references, the same *caveat about\nclass and instance attributes* applies as for regular assignments.\n', 'binary': u'\nBinary arithmetic operations\n****************************\n\nThe binary arithmetic operations have the conventional priority\nlevels. Note that some of these operations also apply to certain non-\nnumeric types. Apart from the power operator, there are only two\nlevels, one for multiplicative operators and one for additive\noperators:\n\n m_expr ::= u_expr | m_expr "*" u_expr | m_expr "//" u_expr | m_expr "/" u_expr\n | m_expr "%" u_expr\n a_expr ::= m_expr | a_expr "+" m_expr | a_expr "-" m_expr\n\nThe ``*`` (multiplication) operator yields the product of its\narguments. The arguments must either both be numbers, or one argument\nmust be an integer (plain or long) and the other must be a sequence.\nIn the former case, the numbers are converted to a common type and\nthen multiplied together. In the latter case, sequence repetition is\nperformed; a negative repetition factor yields an empty sequence.\n\nThe ``/`` (division) and ``//`` (floor division) operators yield the\nquotient of their arguments. The numeric arguments are first\nconverted to a common type. Plain or long integer division yields an\ninteger of the same type; the result is that of mathematical division\nwith the \'floor\' function applied to the result. Division by zero\nraises the ``ZeroDivisionError`` exception.\n\nThe ``%`` (modulo) operator yields the remainder from the division of\nthe first argument by the second. The numeric arguments are first\nconverted to a common type. A zero right argument raises the\n``ZeroDivisionError`` exception. The arguments may be floating point\nnumbers, e.g., ``3.14%0.7`` equals ``0.34`` (since ``3.14`` equals\n``4*0.7 + 0.34``.) The modulo operator always yields a result with\nthe same sign as its second operand (or zero); the absolute value of\nthe result is strictly smaller than the absolute value of the second\noperand [2].\n\nThe integer division and modulo operators are connected by the\nfollowing identity: ``x == (x/y)*y + (x%y)``. Integer division and\nmodulo are also connected with the built-in function ``divmod()``:\n``divmod(x, y) == (x/y, x%y)``. These identities don\'t hold for\nfloating point numbers; there similar identities hold approximately\nwhere ``x/y`` is replaced by ``floor(x/y)`` or ``floor(x/y) - 1`` [3].\n\nIn addition to performing the modulo operation on numbers, the ``%``\noperator is also overloaded by string and unicode objects to perform\nstring formatting (also known as interpolation). The syntax for string\nformatting is described in the Python Library Reference, section\n*String Formatting Operations*.\n\nDeprecated since version 2.3: The floor division operator, the modulo\noperator, and the ``divmod()`` function are no longer defined for\ncomplex numbers. Instead, convert to a floating point number using\nthe ``abs()`` function if appropriate.\n\nThe ``+`` (addition) operator yields the sum of its arguments. The\narguments must either both be numbers or both sequences of the same\ntype. In the former case, the numbers are converted to a common type\nand then added together. In the latter case, the sequences are\nconcatenated.\n\nThe ``-`` (subtraction) operator yields the difference of its\narguments. The numeric arguments are first converted to a common\ntype.\n', 'bitwise': u'\nBinary bitwise operations\n*************************\n\nEach of the three bitwise operations has a different priority level:\n\n and_expr ::= shift_expr | and_expr "&" shift_expr\n xor_expr ::= and_expr | xor_expr "^" and_expr\n or_expr ::= xor_expr | or_expr "|" xor_expr\n\nThe ``&`` operator yields the bitwise AND of its arguments, which must\nbe plain or long integers. The arguments are converted to a common\ntype.\n\nThe ``^`` operator yields the bitwise XOR (exclusive OR) of its\narguments, which must be plain or long integers. The arguments are\nconverted to a common type.\n\nThe ``|`` operator yields the bitwise (inclusive) OR of its arguments,\nwhich must be plain or long integers. The arguments are converted to\na common type.\n', 'bltin-code-objects': u'\nCode Objects\n************\n\nCode objects are used by the implementation to represent "pseudo-\ncompiled" executable Python code such as a function body. They differ\nfrom function objects because they don\'t contain a reference to their\nglobal execution environment. Code objects are returned by the built-\nin ``compile()`` function and can be extracted from function objects\nthrough their ``func_code`` attribute. See also the ``code`` module.\n\nA code object can be executed or evaluated by passing it (instead of a\nsource string) to the ``exec`` statement or the built-in ``eval()``\nfunction.\n\nSee *The standard type hierarchy* for more information.\n', 'bltin-ellipsis-object': u'\nThe Ellipsis Object\n*******************\n\nThis object is used by extended slice notation (see *Slicings*). It\nsupports no special operations. There is exactly one ellipsis object,\nnamed ``Ellipsis`` (a built-in name).\n\nIt is written as ``Ellipsis``.\n', - 'bltin-file-objects': u'\nFile Objects\n************\n\nFile objects are implemented using C\'s ``stdio`` package and can be\ncreated with the built-in ``open()`` function. File objects are also\nreturned by some other built-in functions and methods, such as\n``os.popen()`` and ``os.fdopen()`` and the ``makefile()`` method of\nsocket objects. Temporary files can be created using the ``tempfile``\nmodule, and high-level file operations such as copying, moving, and\ndeleting files and directories can be achieved with the ``shutil``\nmodule.\n\nWhen a file operation fails for an I/O-related reason, the exception\n``IOError`` is raised. This includes situations where the operation\nis not defined for some reason, like ``seek()`` on a tty device or\nwriting a file opened for reading.\n\nFiles have the following methods:\n\nfile.close()\n\n Close the file. A closed file cannot be read or written any more.\n Any operation which requires that the file be open will raise a\n ``ValueError`` after the file has been closed. Calling ``close()``\n more than once is allowed.\n\n As of Python 2.5, you can avoid having to call this method\n explicitly if you use the ``with`` statement. For example, the\n following code will automatically close *f* when the ``with`` block\n is exited:\n\n from __future__ import with_statement # This isn\'t required in Python 2.6\n\n with open("hello.txt") as f:\n for line in f:\n print line\n\n In older versions of Python, you would have needed to do this to\n get the same effect:\n\n f = open("hello.txt")\n try:\n for line in f:\n print line\n finally:\n f.close()\n\n Note: Not all "file-like" types in Python support use as a context\n manager for the ``with`` statement. If your code is intended to\n work with any file-like object, you can use the function\n ``contextlib.closing()`` instead of using the object directly.\n\nfile.flush()\n\n Flush the internal buffer, like ``stdio``\'s ``fflush()``. This may\n be a no-op on some file-like objects.\n\n Note: ``flush()`` does not necessarily write the file\'s data to disk.\n Use ``flush()`` followed by ``os.fsync()`` to ensure this\n behavior.\n\nfile.fileno()\n\n Return the integer "file descriptor" that is used by the underlying\n implementation to request I/O operations from the operating system.\n This can be useful for other, lower level interfaces that use file\n descriptors, such as the ``fcntl`` module or ``os.read()`` and\n friends.\n\n Note: File-like objects which do not have a real file descriptor should\n *not* provide this method!\n\nfile.isatty()\n\n Return ``True`` if the file is connected to a tty(-like) device,\n else ``False``.\n\n Note: If a file-like object is not associated with a real file, this\n method should *not* be implemented.\n\nfile.next()\n\n A file object is its own iterator, for example ``iter(f)`` returns\n *f* (unless *f* is closed). When a file is used as an iterator,\n typically in a ``for`` loop (for example, ``for line in f: print\n line``), the ``next()`` method is called repeatedly. This method\n returns the next input line, or raises ``StopIteration`` when EOF\n is hit when the file is open for reading (behavior is undefined\n when the file is open for writing). In order to make a ``for``\n loop the most efficient way of looping over the lines of a file (a\n very common operation), the ``next()`` method uses a hidden read-\n ahead buffer. As a consequence of using a read-ahead buffer,\n combining ``next()`` with other file methods (like ``readline()``)\n does not work right. However, using ``seek()`` to reposition the\n file to an absolute position will flush the read-ahead buffer.\n\n New in version 2.3.\n\nfile.read([size])\n\n Read at most *size* bytes from the file (less if the read hits EOF\n before obtaining *size* bytes). If the *size* argument is negative\n or omitted, read all data until EOF is reached. The bytes are\n returned as a string object. An empty string is returned when EOF\n is encountered immediately. (For certain files, like ttys, it\n makes sense to continue reading after an EOF is hit.) Note that\n this method may call the underlying C function ``fread()`` more\n than once in an effort to acquire as close to *size* bytes as\n possible. Also note that when in non-blocking mode, less data than\n was requested may be returned, even if no *size* parameter was\n given.\n\n Note: This function is simply a wrapper for the underlying ``fread()``\n C function, and will behave the same in corner cases, such as\n whether the EOF value is cached.\n\nfile.readline([size])\n\n Read one entire line from the file. A trailing newline character\n is kept in the string (but may be absent when a file ends with an\n incomplete line). [5] If the *size* argument is present and non-\n negative, it is a maximum byte count (including the trailing\n newline) and an incomplete line may be returned. An empty string is\n returned *only* when EOF is encountered immediately.\n\n Note: Unlike ``stdio``\'s ``fgets()``, the returned string contains null\n characters (``\'\\0\'``) if they occurred in the input.\n\nfile.readlines([sizehint])\n\n Read until EOF using ``readline()`` and return a list containing\n the lines thus read. If the optional *sizehint* argument is\n present, instead of reading up to EOF, whole lines totalling\n approximately *sizehint* bytes (possibly after rounding up to an\n internal buffer size) are read. Objects implementing a file-like\n interface may choose to ignore *sizehint* if it cannot be\n implemented, or cannot be implemented efficiently.\n\nfile.xreadlines()\n\n This method returns the same thing as ``iter(f)``.\n\n New in version 2.1.\n\n Deprecated since version 2.3: Use ``for line in file`` instead.\n\nfile.seek(offset[, whence])\n\n Set the file\'s current position, like ``stdio``\'s ``fseek()``. The\n *whence* argument is optional and defaults to ``os.SEEK_SET`` or\n ``0`` (absolute file positioning); other values are ``os.SEEK_CUR``\n or ``1`` (seek relative to the current position) and\n ``os.SEEK_END`` or ``2`` (seek relative to the file\'s end). There\n is no return value.\n\n For example, ``f.seek(2, os.SEEK_CUR)`` advances the position by\n two and ``f.seek(-3, os.SEEK_END)`` sets the position to the third\n to last.\n\n Note that if the file is opened for appending (mode ``\'a\'`` or\n ``\'a+\'``), any ``seek()`` operations will be undone at the next\n write. If the file is only opened for writing in append mode (mode\n ``\'a\'``), this method is essentially a no-op, but it remains useful\n for files opened in append mode with reading enabled (mode\n ``\'a+\'``). If the file is opened in text mode (without ``\'b\'``),\n only offsets returned by ``tell()`` are legal. Use of other\n offsets causes undefined behavior.\n\n Note that not all file objects are seekable.\n\n Changed in version 2.6: Passing float values as offset has been\n deprecated.\n\nfile.tell()\n\n Return the file\'s current position, like ``stdio``\'s ``ftell()``.\n\n Note: On Windows, ``tell()`` can return illegal values (after an\n ``fgets()``) when reading files with Unix-style line-endings. Use\n binary mode (``\'rb\'``) to circumvent this problem.\n\nfile.truncate([size])\n\n Truncate the file\'s size. If the optional *size* argument is\n present, the file is truncated to (at most) that size. The size\n defaults to the current position. The current file position is not\n changed. Note that if a specified size exceeds the file\'s current\n size, the result is platform-dependent: possibilities include that\n the file may remain unchanged, increase to the specified size as if\n zero-filled, or increase to the specified size with undefined new\n content. Availability: Windows, many Unix variants.\n\nfile.write(str)\n\n Write a string to the file. There is no return value. Due to\n buffering, the string may not actually show up in the file until\n the ``flush()`` or ``close()`` method is called.\n\nfile.writelines(sequence)\n\n Write a sequence of strings to the file. The sequence can be any\n iterable object producing strings, typically a list of strings.\n There is no return value. (The name is intended to match\n ``readlines()``; ``writelines()`` does not add line separators.)\n\nFiles support the iterator protocol. Each iteration returns the same\nresult as ``file.readline()``, and iteration ends when the\n``readline()`` method returns an empty string.\n\nFile objects also offer a number of other interesting attributes.\nThese are not required for file-like objects, but should be\nimplemented if they make sense for the particular object.\n\nfile.closed\n\n bool indicating the current state of the file object. This is a\n read-only attribute; the ``close()`` method changes the value. It\n may not be available on all file-like objects.\n\nfile.encoding\n\n The encoding that this file uses. When Unicode strings are written\n to a file, they will be converted to byte strings using this\n encoding. In addition, when the file is connected to a terminal,\n the attribute gives the encoding that the terminal is likely to use\n (that information might be incorrect if the user has misconfigured\n the terminal). The attribute is read-only and may not be present\n on all file-like objects. It may also be ``None``, in which case\n the file uses the system default encoding for converting Unicode\n strings.\n\n New in version 2.3.\n\nfile.errors\n\n The Unicode error handler used along with the encoding.\n\n New in version 2.6.\n\nfile.mode\n\n The I/O mode for the file. If the file was created using the\n ``open()`` built-in function, this will be the value of the *mode*\n parameter. This is a read-only attribute and may not be present on\n all file-like objects.\n\nfile.name\n\n If the file object was created using ``open()``, the name of the\n file. Otherwise, some string that indicates the source of the file\n object, of the form ``<...>``. This is a read-only attribute and\n may not be present on all file-like objects.\n\nfile.newlines\n\n If Python was built with the *--with-universal-newlines* option to\n **configure** (the default) this read-only attribute exists, and\n for files opened in universal newline read mode it keeps track of\n the types of newlines encountered while reading the file. The\n values it can take are ``\'\\r\'``, ``\'\\n\'``, ``\'\\r\\n\'``, ``None``\n (unknown, no newlines read yet) or a tuple containing all the\n newline types seen, to indicate that multiple newline conventions\n were encountered. For files not opened in universal newline read\n mode the value of this attribute will be ``None``.\n\nfile.softspace\n\n Boolean that indicates whether a space character needs to be\n printed before another value when using the ``print`` statement.\n Classes that are trying to simulate a file object should also have\n a writable ``softspace`` attribute, which should be initialized to\n zero. This will be automatic for most classes implemented in\n Python (care may be needed for objects that override attribute\n access); types implemented in C will have to provide a writable\n ``softspace`` attribute.\n\n Note: This attribute is not used to control the ``print`` statement,\n but to allow the implementation of ``print`` to keep track of its\n internal state.\n', + 'bltin-file-objects': u'\nFile Objects\n************\n\nFile objects are implemented using C\'s ``stdio`` package and can be\ncreated with the built-in ``open()`` function. File objects are also\nreturned by some other built-in functions and methods, such as\n``os.popen()`` and ``os.fdopen()`` and the ``makefile()`` method of\nsocket objects. Temporary files can be created using the ``tempfile``\nmodule, and high-level file operations such as copying, moving, and\ndeleting files and directories can be achieved with the ``shutil``\nmodule.\n\nWhen a file operation fails for an I/O-related reason, the exception\n``IOError`` is raised. This includes situations where the operation\nis not defined for some reason, like ``seek()`` on a tty device or\nwriting a file opened for reading.\n\nFiles have the following methods:\n\nfile.close()\n\n Close the file. A closed file cannot be read or written any more.\n Any operation which requires that the file be open will raise a\n ``ValueError`` after the file has been closed. Calling ``close()``\n more than once is allowed.\n\n As of Python 2.5, you can avoid having to call this method\n explicitly if you use the ``with`` statement. For example, the\n following code will automatically close *f* when the ``with`` block\n is exited:\n\n from __future__ import with_statement # This isn\'t required in Python 2.6\n\n with open("hello.txt") as f:\n for line in f:\n print line\n\n In older versions of Python, you would have needed to do this to\n get the same effect:\n\n f = open("hello.txt")\n try:\n for line in f:\n print line\n finally:\n f.close()\n\n Note: Not all "file-like" types in Python support use as a context\n manager for the ``with`` statement. If your code is intended to\n work with any file-like object, you can use the function\n ``contextlib.closing()`` instead of using the object directly.\n\nfile.flush()\n\n Flush the internal buffer, like ``stdio``\'s ``fflush()``. This may\n be a no-op on some file-like objects.\n\n Note: ``flush()`` does not necessarily write the file\'s data to disk.\n Use ``flush()`` followed by ``os.fsync()`` to ensure this\n behavior.\n\nfile.fileno()\n\n Return the integer "file descriptor" that is used by the underlying\n implementation to request I/O operations from the operating system.\n This can be useful for other, lower level interfaces that use file\n descriptors, such as the ``fcntl`` module or ``os.read()`` and\n friends.\n\n Note: File-like objects which do not have a real file descriptor should\n *not* provide this method!\n\nfile.isatty()\n\n Return ``True`` if the file is connected to a tty(-like) device,\n else ``False``.\n\n Note: If a file-like object is not associated with a real file, this\n method should *not* be implemented.\n\nfile.next()\n\n A file object is its own iterator, for example ``iter(f)`` returns\n *f* (unless *f* is closed). When a file is used as an iterator,\n typically in a ``for`` loop (for example, ``for line in f: print\n line``), the ``next()`` method is called repeatedly. This method\n returns the next input line, or raises ``StopIteration`` when EOF\n is hit when the file is open for reading (behavior is undefined\n when the file is open for writing). In order to make a ``for``\n loop the most efficient way of looping over the lines of a file (a\n very common operation), the ``next()`` method uses a hidden read-\n ahead buffer. As a consequence of using a read-ahead buffer,\n combining ``next()`` with other file methods (like ``readline()``)\n does not work right. However, using ``seek()`` to reposition the\n file to an absolute position will flush the read-ahead buffer.\n\n New in version 2.3.\n\nfile.read([size])\n\n Read at most *size* bytes from the file (less if the read hits EOF\n before obtaining *size* bytes). If the *size* argument is negative\n or omitted, read all data until EOF is reached. The bytes are\n returned as a string object. An empty string is returned when EOF\n is encountered immediately. (For certain files, like ttys, it\n makes sense to continue reading after an EOF is hit.) Note that\n this method may call the underlying C function ``fread()`` more\n than once in an effort to acquire as close to *size* bytes as\n possible. Also note that when in non-blocking mode, less data than\n was requested may be returned, even if no *size* parameter was\n given.\n\n Note: This function is simply a wrapper for the underlying ``fread()``\n C function, and will behave the same in corner cases, such as\n whether the EOF value is cached.\n\nfile.readline([size])\n\n Read one entire line from the file. A trailing newline character\n is kept in the string (but may be absent when a file ends with an\n incomplete line). [5] If the *size* argument is present and non-\n negative, it is a maximum byte count (including the trailing\n newline) and an incomplete line may be returned. When *size* is not\n 0, an empty string is returned *only* when EOF is encountered\n immediately.\n\n Note: Unlike ``stdio``\'s ``fgets()``, the returned string contains null\n characters (``\'\\0\'``) if they occurred in the input.\n\nfile.readlines([sizehint])\n\n Read until EOF using ``readline()`` and return a list containing\n the lines thus read. If the optional *sizehint* argument is\n present, instead of reading up to EOF, whole lines totalling\n approximately *sizehint* bytes (possibly after rounding up to an\n internal buffer size) are read. Objects implementing a file-like\n interface may choose to ignore *sizehint* if it cannot be\n implemented, or cannot be implemented efficiently.\n\nfile.xreadlines()\n\n This method returns the same thing as ``iter(f)``.\n\n New in version 2.1.\n\n Deprecated since version 2.3: Use ``for line in file`` instead.\n\nfile.seek(offset[, whence])\n\n Set the file\'s current position, like ``stdio``\'s ``fseek()``. The\n *whence* argument is optional and defaults to ``os.SEEK_SET`` or\n ``0`` (absolute file positioning); other values are ``os.SEEK_CUR``\n or ``1`` (seek relative to the current position) and\n ``os.SEEK_END`` or ``2`` (seek relative to the file\'s end). There\n is no return value.\n\n For example, ``f.seek(2, os.SEEK_CUR)`` advances the position by\n two and ``f.seek(-3, os.SEEK_END)`` sets the position to the third\n to last.\n\n Note that if the file is opened for appending (mode ``\'a\'`` or\n ``\'a+\'``), any ``seek()`` operations will be undone at the next\n write. If the file is only opened for writing in append mode (mode\n ``\'a\'``), this method is essentially a no-op, but it remains useful\n for files opened in append mode with reading enabled (mode\n ``\'a+\'``). If the file is opened in text mode (without ``\'b\'``),\n only offsets returned by ``tell()`` are legal. Use of other\n offsets causes undefined behavior.\n\n Note that not all file objects are seekable.\n\n Changed in version 2.6: Passing float values as offset has been\n deprecated.\n\nfile.tell()\n\n Return the file\'s current position, like ``stdio``\'s ``ftell()``.\n\n Note: On Windows, ``tell()`` can return illegal values (after an\n ``fgets()``) when reading files with Unix-style line-endings. Use\n binary mode (``\'rb\'``) to circumvent this problem.\n\nfile.truncate([size])\n\n Truncate the file\'s size. If the optional *size* argument is\n present, the file is truncated to (at most) that size. The size\n defaults to the current position. The current file position is not\n changed. Note that if a specified size exceeds the file\'s current\n size, the result is platform-dependent: possibilities include that\n the file may remain unchanged, increase to the specified size as if\n zero-filled, or increase to the specified size with undefined new\n content. Availability: Windows, many Unix variants.\n\nfile.write(str)\n\n Write a string to the file. There is no return value. Due to\n buffering, the string may not actually show up in the file until\n the ``flush()`` or ``close()`` method is called.\n\nfile.writelines(sequence)\n\n Write a sequence of strings to the file. The sequence can be any\n iterable object producing strings, typically a list of strings.\n There is no return value. (The name is intended to match\n ``readlines()``; ``writelines()`` does not add line separators.)\n\nFiles support the iterator protocol. Each iteration returns the same\nresult as ``file.readline()``, and iteration ends when the\n``readline()`` method returns an empty string.\n\nFile objects also offer a number of other interesting attributes.\nThese are not required for file-like objects, but should be\nimplemented if they make sense for the particular object.\n\nfile.closed\n\n bool indicating the current state of the file object. This is a\n read-only attribute; the ``close()`` method changes the value. It\n may not be available on all file-like objects.\n\nfile.encoding\n\n The encoding that this file uses. When Unicode strings are written\n to a file, they will be converted to byte strings using this\n encoding. In addition, when the file is connected to a terminal,\n the attribute gives the encoding that the terminal is likely to use\n (that information might be incorrect if the user has misconfigured\n the terminal). The attribute is read-only and may not be present\n on all file-like objects. It may also be ``None``, in which case\n the file uses the system default encoding for converting Unicode\n strings.\n\n New in version 2.3.\n\nfile.errors\n\n The Unicode error handler used along with the encoding.\n\n New in version 2.6.\n\nfile.mode\n\n The I/O mode for the file. If the file was created using the\n ``open()`` built-in function, this will be the value of the *mode*\n parameter. This is a read-only attribute and may not be present on\n all file-like objects.\n\nfile.name\n\n If the file object was created using ``open()``, the name of the\n file. Otherwise, some string that indicates the source of the file\n object, of the form ``<...>``. This is a read-only attribute and\n may not be present on all file-like objects.\n\nfile.newlines\n\n If Python was built with universal newlines enabled (the default)\n this read-only attribute exists, and for files opened in universal\n newline read mode it keeps track of the types of newlines\n encountered while reading the file. The values it can take are\n ``\'\\r\'``, ``\'\\n\'``, ``\'\\r\\n\'``, ``None`` (unknown, no newlines read\n yet) or a tuple containing all the newline types seen, to indicate\n that multiple newline conventions were encountered. For files not\n opened in universal newline read mode the value of this attribute\n will be ``None``.\n\nfile.softspace\n\n Boolean that indicates whether a space character needs to be\n printed before another value when using the ``print`` statement.\n Classes that are trying to simulate a file object should also have\n a writable ``softspace`` attribute, which should be initialized to\n zero. This will be automatic for most classes implemented in\n Python (care may be needed for objects that override attribute\n access); types implemented in C will have to provide a writable\n ``softspace`` attribute.\n\n Note: This attribute is not used to control the ``print`` statement,\n but to allow the implementation of ``print`` to keep track of its\n internal state.\n', 'bltin-null-object': u"\nThe Null Object\n***************\n\nThis object is returned by functions that don't explicitly return a\nvalue. It supports no special operations. There is exactly one null\nobject, named ``None`` (a built-in name).\n\nIt is written as ``None``.\n", 'bltin-type-objects': u"\nType Objects\n************\n\nType objects represent the various object types. An object's type is\naccessed by the built-in function ``type()``. There are no special\noperations on types. The standard module ``types`` defines names for\nall standard built-in types.\n\nTypes are written like this: ``<type 'int'>``.\n", 'booleans': u'\nBoolean operations\n******************\n\n or_test ::= and_test | or_test "or" and_test\n and_test ::= not_test | and_test "and" not_test\n not_test ::= comparison | "not" not_test\n\nIn the context of Boolean operations, and also when expressions are\nused by control flow statements, the following values are interpreted\nas false: ``False``, ``None``, numeric zero of all types, and empty\nstrings and containers (including strings, tuples, lists,\ndictionaries, sets and frozensets). All other values are interpreted\nas true. (See the ``__nonzero__()`` special method for a way to\nchange this.)\n\nThe operator ``not`` yields ``True`` if its argument is false,\n``False`` otherwise.\n\nThe expression ``x and y`` first evaluates *x*; if *x* is false, its\nvalue is returned; otherwise, *y* is evaluated and the resulting value\nis returned.\n\nThe expression ``x or y`` first evaluates *x*; if *x* is true, its\nvalue is returned; otherwise, *y* is evaluated and the resulting value\nis returned.\n\n(Note that neither ``and`` nor ``or`` restrict the value and type they\nreturn to ``False`` and ``True``, but rather return the last evaluated\nargument. This is sometimes useful, e.g., if ``s`` is a string that\nshould be replaced by a default value if it is empty, the expression\n``s or \'foo\'`` yields the desired value. Because ``not`` has to\ninvent a value anyway, it does not bother to return a value of the\nsame type as its argument, so e.g., ``not \'foo\'`` yields ``False``,\nnot ``\'\'``.)\n', @@ -20,39 +20,39 @@ 'class': u'\nClass definitions\n*****************\n\nA class definition defines a class object (see section *The standard\ntype hierarchy*):\n\n classdef ::= "class" classname [inheritance] ":" suite\n inheritance ::= "(" [expression_list] ")"\n classname ::= identifier\n\nA class definition is an executable statement. It first evaluates the\ninheritance list, if present. Each item in the inheritance list\nshould evaluate to a class object or class type which allows\nsubclassing. The class\'s suite is then executed in a new execution\nframe (see section *Naming and binding*), using a newly created local\nnamespace and the original global namespace. (Usually, the suite\ncontains only function definitions.) When the class\'s suite finishes\nexecution, its execution frame is discarded but its local namespace is\nsaved. [4] A class object is then created using the inheritance list\nfor the base classes and the saved local namespace for the attribute\ndictionary. The class name is bound to this class object in the\noriginal local namespace.\n\n**Programmer\'s note:** Variables defined in the class definition are\nclass variables; they are shared by all instances. To create instance\nvariables, they can be set in a method with ``self.name = value``.\nBoth class and instance variables are accessible through the notation\n"``self.name``", and an instance variable hides a class variable with\nthe same name when accessed in this way. Class variables can be used\nas defaults for instance variables, but using mutable values there can\nlead to unexpected results. For *new-style class*es, descriptors can\nbe used to create instance variables with different implementation\ndetails.\n\nClass definitions, like function definitions, may be wrapped by one or\nmore *decorator* expressions. The evaluation rules for the decorator\nexpressions are the same as for functions. The result must be a class\nobject, which is then bound to the class name.\n\n-[ Footnotes ]-\n\n[1] The exception is propagated to the invocation stack only if there\n is no ``finally`` clause that negates the exception.\n\n[2] Currently, control "flows off the end" except in the case of an\n exception or the execution of a ``return``, ``continue``, or\n ``break`` statement.\n\n[3] A string literal appearing as the first statement in the function\n body is transformed into the function\'s ``__doc__`` attribute and\n therefore the function\'s *docstring*.\n\n[4] A string literal appearing as the first statement in the class\n body is transformed into the namespace\'s ``__doc__`` item and\n therefore the class\'s *docstring*.\n', 'coercion-rules': u"\nCoercion rules\n**************\n\nThis section used to document the rules for coercion. As the language\nhas evolved, the coercion rules have become hard to document\nprecisely; documenting what one version of one particular\nimplementation does is undesirable. Instead, here are some informal\nguidelines regarding coercion. In Python 3.0, coercion will not be\nsupported.\n\n* If the left operand of a % operator is a string or Unicode object,\n no coercion takes place and the string formatting operation is\n invoked instead.\n\n* It is no longer recommended to define a coercion operation. Mixed-\n mode operations on types that don't define coercion pass the\n original arguments to the operation.\n\n* New-style classes (those derived from ``object``) never invoke the\n ``__coerce__()`` method in response to a binary operator; the only\n time ``__coerce__()`` is invoked is when the built-in function\n ``coerce()`` is called.\n\n* For most intents and purposes, an operator that returns\n ``NotImplemented`` is treated the same as one that is not\n implemented at all.\n\n* Below, ``__op__()`` and ``__rop__()`` are used to signify the\n generic method names corresponding to an operator; ``__iop__()`` is\n used for the corresponding in-place operator. For example, for the\n operator '``+``', ``__add__()`` and ``__radd__()`` are used for the\n left and right variant of the binary operator, and ``__iadd__()``\n for the in-place variant.\n\n* For objects *x* and *y*, first ``x.__op__(y)`` is tried. If this is\n not implemented or returns ``NotImplemented``, ``y.__rop__(x)`` is\n tried. If this is also not implemented or returns\n ``NotImplemented``, a ``TypeError`` exception is raised. But see\n the following exception:\n\n* Exception to the previous item: if the left operand is an instance\n of a built-in type or a new-style class, and the right operand is an\n instance of a proper subclass of that type or class and overrides\n the base's ``__rop__()`` method, the right operand's ``__rop__()``\n method is tried *before* the left operand's ``__op__()`` method.\n\n This is done so that a subclass can completely override binary\n operators. Otherwise, the left operand's ``__op__()`` method would\n always accept the right operand: when an instance of a given class\n is expected, an instance of a subclass of that class is always\n acceptable.\n\n* When either operand type defines a coercion, this coercion is called\n before that type's ``__op__()`` or ``__rop__()`` method is called,\n but no sooner. If the coercion returns an object of a different\n type for the operand whose coercion is invoked, part of the process\n is redone using the new object.\n\n* When an in-place operator (like '``+=``') is used, if the left\n operand implements ``__iop__()``, it is invoked without any\n coercion. When the operation falls back to ``__op__()`` and/or\n ``__rop__()``, the normal coercion rules apply.\n\n* In ``x + y``, if *x* is a sequence that implements sequence\n concatenation, sequence concatenation is invoked.\n\n* In ``x * y``, if one operator is a sequence that implements sequence\n repetition, and the other is an integer (``int`` or ``long``),\n sequence repetition is invoked.\n\n* Rich comparisons (implemented by methods ``__eq__()`` and so on)\n never use coercion. Three-way comparison (implemented by\n ``__cmp__()``) does use coercion under the same conditions as other\n binary operations use it.\n\n* In the current implementation, the built-in numeric types ``int``,\n ``long``, ``float``, and ``complex`` do not use coercion. All these\n types implement a ``__coerce__()`` method, for use by the built-in\n ``coerce()`` function.\n\n Changed in version 2.7.\n", 'comparisons': u'\nComparisons\n***********\n\nUnlike C, all comparison operations in Python have the same priority,\nwhich is lower than that of any arithmetic, shifting or bitwise\noperation. Also unlike C, expressions like ``a < b < c`` have the\ninterpretation that is conventional in mathematics:\n\n comparison ::= or_expr ( comp_operator or_expr )*\n comp_operator ::= "<" | ">" | "==" | ">=" | "<=" | "<>" | "!="\n | "is" ["not"] | ["not"] "in"\n\nComparisons yield boolean values: ``True`` or ``False``.\n\nComparisons can be chained arbitrarily, e.g., ``x < y <= z`` is\nequivalent to ``x < y and y <= z``, except that ``y`` is evaluated\nonly once (but in both cases ``z`` is not evaluated at all when ``x <\ny`` is found to be false).\n\nFormally, if *a*, *b*, *c*, ..., *y*, *z* are expressions and *op1*,\n*op2*, ..., *opN* are comparison operators, then ``a op1 b op2 c ... y\nopN z`` is equivalent to ``a op1 b and b op2 c and ... y opN z``,\nexcept that each expression is evaluated at most once.\n\nNote that ``a op1 b op2 c`` doesn\'t imply any kind of comparison\nbetween *a* and *c*, so that, e.g., ``x < y > z`` is perfectly legal\n(though perhaps not pretty).\n\nThe forms ``<>`` and ``!=`` are equivalent; for consistency with C,\n``!=`` is preferred; where ``!=`` is mentioned below ``<>`` is also\naccepted. The ``<>`` spelling is considered obsolescent.\n\nThe operators ``<``, ``>``, ``==``, ``>=``, ``<=``, and ``!=`` compare\nthe values of two objects. The objects need not have the same type.\nIf both are numbers, they are converted to a common type. Otherwise,\nobjects of different types *always* compare unequal, and are ordered\nconsistently but arbitrarily. You can control comparison behavior of\nobjects of non-built-in types by defining a ``__cmp__`` method or rich\ncomparison methods like ``__gt__``, described in section *Special\nmethod names*.\n\n(This unusual definition of comparison was used to simplify the\ndefinition of operations like sorting and the ``in`` and ``not in``\noperators. In the future, the comparison rules for objects of\ndifferent types are likely to change.)\n\nComparison of objects of the same type depends on the type:\n\n* Numbers are compared arithmetically.\n\n* Strings are compared lexicographically using the numeric equivalents\n (the result of the built-in function ``ord()``) of their characters.\n Unicode and 8-bit strings are fully interoperable in this behavior.\n [4]\n\n* Tuples and lists are compared lexicographically using comparison of\n corresponding elements. This means that to compare equal, each\n element must compare equal and the two sequences must be of the same\n type and have the same length.\n\n If not equal, the sequences are ordered the same as their first\n differing elements. For example, ``cmp([1,2,x], [1,2,y])`` returns\n the same as ``cmp(x,y)``. If the corresponding element does not\n exist, the shorter sequence is ordered first (for example, ``[1,2] <\n [1,2,3]``).\n\n* Mappings (dictionaries) compare equal if and only if their sorted\n (key, value) lists compare equal. [5] Outcomes other than equality\n are resolved consistently, but are not otherwise defined. [6]\n\n* Most other objects of built-in types compare unequal unless they are\n the same object; the choice whether one object is considered smaller\n or larger than another one is made arbitrarily but consistently\n within one execution of a program.\n\nThe operators ``in`` and ``not in`` test for collection membership.\n``x in s`` evaluates to true if *x* is a member of the collection *s*,\nand false otherwise. ``x not in s`` returns the negation of ``x in\ns``. The collection membership test has traditionally been bound to\nsequences; an object is a member of a collection if the collection is\na sequence and contains an element equal to that object. However, it\nmake sense for many other object types to support membership tests\nwithout being a sequence. In particular, dictionaries (for keys) and\nsets support membership testing.\n\nFor the list and tuple types, ``x in y`` is true if and only if there\nexists an index *i* such that ``x == y[i]`` is true.\n\nFor the Unicode and string types, ``x in y`` is true if and only if\n*x* is a substring of *y*. An equivalent test is ``y.find(x) != -1``.\nNote, *x* and *y* need not be the same type; consequently, ``u\'ab\' in\n\'abc\'`` will return ``True``. Empty strings are always considered to\nbe a substring of any other string, so ``"" in "abc"`` will return\n``True``.\n\nChanged in version 2.3: Previously, *x* was required to be a string of\nlength ``1``.\n\nFor user-defined classes which define the ``__contains__()`` method,\n``x in y`` is true if and only if ``y.__contains__(x)`` is true.\n\nFor user-defined classes which do not define ``__contains__()`` but do\ndefine ``__iter__()``, ``x in y`` is true if some value ``z`` with ``x\n== z`` is produced while iterating over ``y``. If an exception is\nraised during the iteration, it is as if ``in`` raised that exception.\n\nLastly, the old-style iteration protocol is tried: if a class defines\n``__getitem__()``, ``x in y`` is true if and only if there is a non-\nnegative integer index *i* such that ``x == y[i]``, and all lower\ninteger indices do not raise ``IndexError`` exception. (If any other\nexception is raised, it is as if ``in`` raised that exception).\n\nThe operator ``not in`` is defined to have the inverse true value of\n``in``.\n\nThe operators ``is`` and ``is not`` test for object identity: ``x is\ny`` is true if and only if *x* and *y* are the same object. ``x is\nnot y`` yields the inverse truth value. [7]\n', - 'compound': u'\nCompound statements\n*******************\n\nCompound statements contain (groups of) other statements; they affect\nor control the execution of those other statements in some way. In\ngeneral, compound statements span multiple lines, although in simple\nincarnations a whole compound statement may be contained in one line.\n\nThe ``if``, ``while`` and ``for`` statements implement traditional\ncontrol flow constructs. ``try`` specifies exception handlers and/or\ncleanup code for a group of statements. Function and class\ndefinitions are also syntactically compound statements.\n\nCompound statements consist of one or more \'clauses.\' A clause\nconsists of a header and a \'suite.\' The clause headers of a\nparticular compound statement are all at the same indentation level.\nEach clause header begins with a uniquely identifying keyword and ends\nwith a colon. A suite is a group of statements controlled by a\nclause. A suite can be one or more semicolon-separated simple\nstatements on the same line as the header, following the header\'s\ncolon, or it can be one or more indented statements on subsequent\nlines. Only the latter form of suite can contain nested compound\nstatements; the following is illegal, mostly because it wouldn\'t be\nclear to which ``if`` clause a following ``else`` clause would belong:\n\n if test1: if test2: print x\n\nAlso note that the semicolon binds tighter than the colon in this\ncontext, so that in the following example, either all or none of the\n``print`` statements are executed:\n\n if x < y < z: print x; print y; print z\n\nSummarizing:\n\n compound_stmt ::= if_stmt\n | while_stmt\n | for_stmt\n | try_stmt\n | with_stmt\n | funcdef\n | classdef\n | decorated\n suite ::= stmt_list NEWLINE | NEWLINE INDENT statement+ DEDENT\n statement ::= stmt_list NEWLINE | compound_stmt\n stmt_list ::= simple_stmt (";" simple_stmt)* [";"]\n\nNote that statements always end in a ``NEWLINE`` possibly followed by\na ``DEDENT``. Also note that optional continuation clauses always\nbegin with a keyword that cannot start a statement, thus there are no\nambiguities (the \'dangling ``else``\' problem is solved in Python by\nrequiring nested ``if`` statements to be indented).\n\nThe formatting of the grammar rules in the following sections places\neach clause on a separate line for clarity.\n\n\nThe ``if`` statement\n====================\n\nThe ``if`` statement is used for conditional execution:\n\n if_stmt ::= "if" expression ":" suite\n ( "elif" expression ":" suite )*\n ["else" ":" suite]\n\nIt selects exactly one of the suites by evaluating the expressions one\nby one until one is found to be true (see section *Boolean operations*\nfor the definition of true and false); then that suite is executed\n(and no other part of the ``if`` statement is executed or evaluated).\nIf all expressions are false, the suite of the ``else`` clause, if\npresent, is executed.\n\n\nThe ``while`` statement\n=======================\n\nThe ``while`` statement is used for repeated execution as long as an\nexpression is true:\n\n while_stmt ::= "while" expression ":" suite\n ["else" ":" suite]\n\nThis repeatedly tests the expression and, if it is true, executes the\nfirst suite; if the expression is false (which may be the first time\nit is tested) the suite of the ``else`` clause, if present, is\nexecuted and the loop terminates.\n\nA ``break`` statement executed in the first suite terminates the loop\nwithout executing the ``else`` clause\'s suite. A ``continue``\nstatement executed in the first suite skips the rest of the suite and\ngoes back to testing the expression.\n\n\nThe ``for`` statement\n=====================\n\nThe ``for`` statement is used to iterate over the elements of a\nsequence (such as a string, tuple or list) or other iterable object:\n\n for_stmt ::= "for" target_list "in" expression_list ":" suite\n ["else" ":" suite]\n\nThe expression list is evaluated once; it should yield an iterable\nobject. An iterator is created for the result of the\n``expression_list``. The suite is then executed once for each item\nprovided by the iterator, in the order of ascending indices. Each\nitem in turn is assigned to the target list using the standard rules\nfor assignments, and then the suite is executed. When the items are\nexhausted (which is immediately when the sequence is empty), the suite\nin the ``else`` clause, if present, is executed, and the loop\nterminates.\n\nA ``break`` statement executed in the first suite terminates the loop\nwithout executing the ``else`` clause\'s suite. A ``continue``\nstatement executed in the first suite skips the rest of the suite and\ncontinues with the next item, or with the ``else`` clause if there was\nno next item.\n\nThe suite may assign to the variable(s) in the target list; this does\nnot affect the next item assigned to it.\n\nThe target list is not deleted when the loop is finished, but if the\nsequence is empty, it will not have been assigned to at all by the\nloop. Hint: the built-in function ``range()`` returns a sequence of\nintegers suitable to emulate the effect of Pascal\'s ``for i := a to b\ndo``; e.g., ``range(3)`` returns the list ``[0, 1, 2]``.\n\nNote: There is a subtlety when the sequence is being modified by the loop\n (this can only occur for mutable sequences, i.e. lists). An internal\n counter is used to keep track of which item is used next, and this\n is incremented on each iteration. When this counter has reached the\n length of the sequence the loop terminates. This means that if the\n suite deletes the current (or a previous) item from the sequence,\n the next item will be skipped (since it gets the index of the\n current item which has already been treated). Likewise, if the\n suite inserts an item in the sequence before the current item, the\n current item will be treated again the next time through the loop.\n This can lead to nasty bugs that can be avoided by making a\n temporary copy using a slice of the whole sequence, e.g.,\n\n for x in a[:]:\n if x < 0: a.remove(x)\n\n\nThe ``try`` statement\n=====================\n\nThe ``try`` statement specifies exception handlers and/or cleanup code\nfor a group of statements:\n\n try_stmt ::= try1_stmt | try2_stmt\n try1_stmt ::= "try" ":" suite\n ("except" [expression [("as" | ",") target]] ":" suite)+\n ["else" ":" suite]\n ["finally" ":" suite]\n try2_stmt ::= "try" ":" suite\n "finally" ":" suite\n\nChanged in version 2.5: In previous versions of Python,\n``try``...``except``...``finally`` did not work. ``try``...``except``\nhad to be nested in ``try``...``finally``.\n\nThe ``except`` clause(s) specify one or more exception handlers. When\nno exception occurs in the ``try`` clause, no exception handler is\nexecuted. When an exception occurs in the ``try`` suite, a search for\nan exception handler is started. This search inspects the except\nclauses in turn until one is found that matches the exception. An\nexpression-less except clause, if present, must be last; it matches\nany exception. For an except clause with an expression, that\nexpression is evaluated, and the clause matches the exception if the\nresulting object is "compatible" with the exception. An object is\ncompatible with an exception if it is the class or a base class of the\nexception object, a tuple containing an item compatible with the\nexception, or, in the (deprecated) case of string exceptions, is the\nraised string itself (note that the object identities must match, i.e.\nit must be the same string object, not just a string with the same\nvalue).\n\nIf no except clause matches the exception, the search for an exception\nhandler continues in the surrounding code and on the invocation stack.\n[1]\n\nIf the evaluation of an expression in the header of an except clause\nraises an exception, the original search for a handler is canceled and\na search starts for the new exception in the surrounding code and on\nthe call stack (it is treated as if the entire ``try`` statement\nraised the exception).\n\nWhen a matching except clause is found, the exception is assigned to\nthe target specified in that except clause, if present, and the except\nclause\'s suite is executed. All except clauses must have an\nexecutable block. When the end of this block is reached, execution\ncontinues normally after the entire try statement. (This means that\nif two nested handlers exist for the same exception, and the exception\noccurs in the try clause of the inner handler, the outer handler will\nnot handle the exception.)\n\nBefore an except clause\'s suite is executed, details about the\nexception are assigned to three variables in the ``sys`` module:\n``sys.exc_type`` receives the object identifying the exception;\n``sys.exc_value`` receives the exception\'s parameter;\n``sys.exc_traceback`` receives a traceback object (see section *The\nstandard type hierarchy*) identifying the point in the program where\nthe exception occurred. These details are also available through the\n``sys.exc_info()`` function, which returns a tuple ``(exc_type,\nexc_value, exc_traceback)``. Use of the corresponding variables is\ndeprecated in favor of this function, since their use is unsafe in a\nthreaded program. As of Python 1.5, the variables are restored to\ntheir previous values (before the call) when returning from a function\nthat handled an exception.\n\nThe optional ``else`` clause is executed if and when control flows off\nthe end of the ``try`` clause. [2] Exceptions in the ``else`` clause\nare not handled by the preceding ``except`` clauses.\n\nIf ``finally`` is present, it specifies a \'cleanup\' handler. The\n``try`` clause is executed, including any ``except`` and ``else``\nclauses. If an exception occurs in any of the clauses and is not\nhandled, the exception is temporarily saved. The ``finally`` clause is\nexecuted. If there is a saved exception, it is re-raised at the end\nof the ``finally`` clause. If the ``finally`` clause raises another\nexception or executes a ``return`` or ``break`` statement, the saved\nexception is lost. The exception information is not available to the\nprogram during execution of the ``finally`` clause.\n\nWhen a ``return``, ``break`` or ``continue`` statement is executed in\nthe ``try`` suite of a ``try``...``finally`` statement, the\n``finally`` clause is also executed \'on the way out.\' A ``continue``\nstatement is illegal in the ``finally`` clause. (The reason is a\nproblem with the current implementation --- this restriction may be\nlifted in the future).\n\nAdditional information on exceptions can be found in section\n*Exceptions*, and information on using the ``raise`` statement to\ngenerate exceptions may be found in section *The raise statement*.\n\n\nThe ``with`` statement\n======================\n\nNew in version 2.5.\n\nThe ``with`` statement is used to wrap the execution of a block with\nmethods defined by a context manager (see section *With Statement\nContext Managers*). This allows common\n``try``...``except``...``finally`` usage patterns to be encapsulated\nfor convenient reuse.\n\n with_stmt ::= "with" with_item ("," with_item)* ":" suite\n with_item ::= expression ["as" target]\n\nThe execution of the ``with`` statement with one "item" proceeds as\nfollows:\n\n1. The context expression is evaluated to obtain a context manager.\n\n2. The context manager\'s ``__exit__()`` is loaded for later use.\n\n3. The context manager\'s ``__enter__()`` method is invoked.\n\n4. If a target was included in the ``with`` statement, the return\n value from ``__enter__()`` is assigned to it.\n\n Note: The ``with`` statement guarantees that if the ``__enter__()``\n method returns without an error, then ``__exit__()`` will always\n be called. Thus, if an error occurs during the assignment to the\n target list, it will be treated the same as an error occurring\n within the suite would be. See step 6 below.\n\n5. The suite is executed.\n\n6. The context manager\'s ``__exit__()`` method is invoked. If an\n exception caused the suite to be exited, its type, value, and\n traceback are passed as arguments to ``__exit__()``. Otherwise,\n three ``None`` arguments are supplied.\n\n If the suite was exited due to an exception, and the return value\n from the ``__exit__()`` method was false, the exception is\n reraised. If the return value was true, the exception is\n suppressed, and execution continues with the statement following\n the ``with`` statement.\n\n If the suite was exited for any reason other than an exception, the\n return value from ``__exit__()`` is ignored, and execution proceeds\n at the normal location for the kind of exit that was taken.\n\nWith more than one item, the context managers are processed as if\nmultiple ``with`` statements were nested:\n\n with A() as a, B() as b:\n suite\n\nis equivalent to\n\n with A() as a:\n with B() as b:\n suite\n\nNote: In Python 2.5, the ``with`` statement is only allowed when the\n ``with_statement`` feature has been enabled. It is always enabled\n in Python 2.6.\n\nChanged in version 2.7: Support for multiple context expressions.\n\nSee also:\n\n **PEP 0343** - The "with" statement\n The specification, background, and examples for the Python\n ``with`` statement.\n\n\nFunction definitions\n====================\n\nA function definition defines a user-defined function object (see\nsection *The standard type hierarchy*):\n\n decorated ::= decorators (classdef | funcdef)\n decorators ::= decorator+\n decorator ::= "@" dotted_name ["(" [argument_list [","]] ")"] NEWLINE\n funcdef ::= "def" funcname "(" [parameter_list] ")" ":" suite\n dotted_name ::= identifier ("." identifier)*\n parameter_list ::= (defparameter ",")*\n ( "*" identifier [, "**" identifier]\n | "**" identifier\n | defparameter [","] )\n defparameter ::= parameter ["=" expression]\n sublist ::= parameter ("," parameter)* [","]\n parameter ::= identifier | "(" sublist ")"\n funcname ::= identifier\n\nA function definition is an executable statement. Its execution binds\nthe function name in the current local namespace to a function object\n(a wrapper around the executable code for the function). This\nfunction object contains a reference to the current global namespace\nas the global namespace to be used when the function is called.\n\nThe function definition does not execute the function body; this gets\nexecuted only when the function is called. [3]\n\nA function definition may be wrapped by one or more *decorator*\nexpressions. Decorator expressions are evaluated when the function is\ndefined, in the scope that contains the function definition. The\nresult must be a callable, which is invoked with the function object\nas the only argument. The returned value is bound to the function name\ninstead of the function object. Multiple decorators are applied in\nnested fashion. For example, the following code:\n\n @f1(arg)\n @f2\n def func(): pass\n\nis equivalent to:\n\n def func(): pass\n func = f1(arg)(f2(func))\n\nWhen one or more top-level parameters have the form *parameter* ``=``\n*expression*, the function is said to have "default parameter values."\nFor a parameter with a default value, the corresponding argument may\nbe omitted from a call, in which case the parameter\'s default value is\nsubstituted. If a parameter has a default value, all following\nparameters must also have a default value --- this is a syntactic\nrestriction that is not expressed by the grammar.\n\n**Default parameter values are evaluated when the function definition\nis executed.** This means that the expression is evaluated once, when\nthe function is defined, and that that same "pre-computed" value is\nused for each call. This is especially important to understand when a\ndefault parameter is a mutable object, such as a list or a dictionary:\nif the function modifies the object (e.g. by appending an item to a\nlist), the default value is in effect modified. This is generally not\nwhat was intended. A way around this is to use ``None`` as the\ndefault, and explicitly test for it in the body of the function, e.g.:\n\n def whats_on_the_telly(penguin=None):\n if penguin is None:\n penguin = []\n penguin.append("property of the zoo")\n return penguin\n\nFunction call semantics are described in more detail in section\n*Calls*. A function call always assigns values to all parameters\nmentioned in the parameter list, either from position arguments, from\nkeyword arguments, or from default values. If the form\n"``*identifier``" is present, it is initialized to a tuple receiving\nany excess positional parameters, defaulting to the empty tuple. If\nthe form "``**identifier``" is present, it is initialized to a new\ndictionary receiving any excess keyword arguments, defaulting to a new\nempty dictionary.\n\nIt is also possible to create anonymous functions (functions not bound\nto a name), for immediate use in expressions. This uses lambda forms,\ndescribed in section *Lambdas*. Note that the lambda form is merely a\nshorthand for a simplified function definition; a function defined in\na "``def``" statement can be passed around or assigned to another name\njust like a function defined by a lambda form. The "``def``" form is\nactually more powerful since it allows the execution of multiple\nstatements.\n\n**Programmer\'s note:** Functions are first-class objects. A "``def``"\nform executed inside a function definition defines a local function\nthat can be returned or passed around. Free variables used in the\nnested function can access the local variables of the function\ncontaining the def. See section *Naming and binding* for details.\n\n\nClass definitions\n=================\n\nA class definition defines a class object (see section *The standard\ntype hierarchy*):\n\n classdef ::= "class" classname [inheritance] ":" suite\n inheritance ::= "(" [expression_list] ")"\n classname ::= identifier\n\nA class definition is an executable statement. It first evaluates the\ninheritance list, if present. Each item in the inheritance list\nshould evaluate to a class object or class type which allows\nsubclassing. The class\'s suite is then executed in a new execution\nframe (see section *Naming and binding*), using a newly created local\nnamespace and the original global namespace. (Usually, the suite\ncontains only function definitions.) When the class\'s suite finishes\nexecution, its execution frame is discarded but its local namespace is\nsaved. [4] A class object is then created using the inheritance list\nfor the base classes and the saved local namespace for the attribute\ndictionary. The class name is bound to this class object in the\noriginal local namespace.\n\n**Programmer\'s note:** Variables defined in the class definition are\nclass variables; they are shared by all instances. To create instance\nvariables, they can be set in a method with ``self.name = value``.\nBoth class and instance variables are accessible through the notation\n"``self.name``", and an instance variable hides a class variable with\nthe same name when accessed in this way. Class variables can be used\nas defaults for instance variables, but using mutable values there can\nlead to unexpected results. For *new-style class*es, descriptors can\nbe used to create instance variables with different implementation\ndetails.\n\nClass definitions, like function definitions, may be wrapped by one or\nmore *decorator* expressions. The evaluation rules for the decorator\nexpressions are the same as for functions. The result must be a class\nobject, which is then bound to the class name.\n\n-[ Footnotes ]-\n\n[1] The exception is propagated to the invocation stack only if there\n is no ``finally`` clause that negates the exception.\n\n[2] Currently, control "flows off the end" except in the case of an\n exception or the execution of a ``return``, ``continue``, or\n ``break`` statement.\n\n[3] A string literal appearing as the first statement in the function\n body is transformed into the function\'s ``__doc__`` attribute and\n therefore the function\'s *docstring*.\n\n[4] A string literal appearing as the first statement in the class\n body is transformed into the namespace\'s ``__doc__`` item and\n therefore the class\'s *docstring*.\n', + 'compound': u'\nCompound statements\n*******************\n\nCompound statements contain (groups of) other statements; they affect\nor control the execution of those other statements in some way. In\ngeneral, compound statements span multiple lines, although in simple\nincarnations a whole compound statement may be contained in one line.\n\nThe ``if``, ``while`` and ``for`` statements implement traditional\ncontrol flow constructs. ``try`` specifies exception handlers and/or\ncleanup code for a group of statements. Function and class\ndefinitions are also syntactically compound statements.\n\nCompound statements consist of one or more \'clauses.\' A clause\nconsists of a header and a \'suite.\' The clause headers of a\nparticular compound statement are all at the same indentation level.\nEach clause header begins with a uniquely identifying keyword and ends\nwith a colon. A suite is a group of statements controlled by a\nclause. A suite can be one or more semicolon-separated simple\nstatements on the same line as the header, following the header\'s\ncolon, or it can be one or more indented statements on subsequent\nlines. Only the latter form of suite can contain nested compound\nstatements; the following is illegal, mostly because it wouldn\'t be\nclear to which ``if`` clause a following ``else`` clause would belong:\n\n if test1: if test2: print x\n\nAlso note that the semicolon binds tighter than the colon in this\ncontext, so that in the following example, either all or none of the\n``print`` statements are executed:\n\n if x < y < z: print x; print y; print z\n\nSummarizing:\n\n compound_stmt ::= if_stmt\n | while_stmt\n | for_stmt\n | try_stmt\n | with_stmt\n | funcdef\n | classdef\n | decorated\n suite ::= stmt_list NEWLINE | NEWLINE INDENT statement+ DEDENT\n statement ::= stmt_list NEWLINE | compound_stmt\n stmt_list ::= simple_stmt (";" simple_stmt)* [";"]\n\nNote that statements always end in a ``NEWLINE`` possibly followed by\na ``DEDENT``. Also note that optional continuation clauses always\nbegin with a keyword that cannot start a statement, thus there are no\nambiguities (the \'dangling ``else``\' problem is solved in Python by\nrequiring nested ``if`` statements to be indented).\n\nThe formatting of the grammar rules in the following sections places\neach clause on a separate line for clarity.\n\n\nThe ``if`` statement\n====================\n\nThe ``if`` statement is used for conditional execution:\n\n if_stmt ::= "if" expression ":" suite\n ( "elif" expression ":" suite )*\n ["else" ":" suite]\n\nIt selects exactly one of the suites by evaluating the expressions one\nby one until one is found to be true (see section *Boolean operations*\nfor the definition of true and false); then that suite is executed\n(and no other part of the ``if`` statement is executed or evaluated).\nIf all expressions are false, the suite of the ``else`` clause, if\npresent, is executed.\n\n\nThe ``while`` statement\n=======================\n\nThe ``while`` statement is used for repeated execution as long as an\nexpression is true:\n\n while_stmt ::= "while" expression ":" suite\n ["else" ":" suite]\n\nThis repeatedly tests the expression and, if it is true, executes the\nfirst suite; if the expression is false (which may be the first time\nit is tested) the suite of the ``else`` clause, if present, is\nexecuted and the loop terminates.\n\nA ``break`` statement executed in the first suite terminates the loop\nwithout executing the ``else`` clause\'s suite. A ``continue``\nstatement executed in the first suite skips the rest of the suite and\ngoes back to testing the expression.\n\n\nThe ``for`` statement\n=====================\n\nThe ``for`` statement is used to iterate over the elements of a\nsequence (such as a string, tuple or list) or other iterable object:\n\n for_stmt ::= "for" target_list "in" expression_list ":" suite\n ["else" ":" suite]\n\nThe expression list is evaluated once; it should yield an iterable\nobject. An iterator is created for the result of the\n``expression_list``. The suite is then executed once for each item\nprovided by the iterator, in the order of ascending indices. Each\nitem in turn is assigned to the target list using the standard rules\nfor assignments, and then the suite is executed. When the items are\nexhausted (which is immediately when the sequence is empty), the suite\nin the ``else`` clause, if present, is executed, and the loop\nterminates.\n\nA ``break`` statement executed in the first suite terminates the loop\nwithout executing the ``else`` clause\'s suite. A ``continue``\nstatement executed in the first suite skips the rest of the suite and\ncontinues with the next item, or with the ``else`` clause if there was\nno next item.\n\nThe suite may assign to the variable(s) in the target list; this does\nnot affect the next item assigned to it.\n\nThe target list is not deleted when the loop is finished, but if the\nsequence is empty, it will not have been assigned to at all by the\nloop. Hint: the built-in function ``range()`` returns a sequence of\nintegers suitable to emulate the effect of Pascal\'s ``for i := a to b\ndo``; e.g., ``range(3)`` returns the list ``[0, 1, 2]``.\n\nNote: There is a subtlety when the sequence is being modified by the loop\n (this can only occur for mutable sequences, i.e. lists). An internal\n counter is used to keep track of which item is used next, and this\n is incremented on each iteration. When this counter has reached the\n length of the sequence the loop terminates. This means that if the\n suite deletes the current (or a previous) item from the sequence,\n the next item will be skipped (since it gets the index of the\n current item which has already been treated). Likewise, if the\n suite inserts an item in the sequence before the current item, the\n current item will be treated again the next time through the loop.\n This can lead to nasty bugs that can be avoided by making a\n temporary copy using a slice of the whole sequence, e.g.,\n\n for x in a[:]:\n if x < 0: a.remove(x)\n\n\nThe ``try`` statement\n=====================\n\nThe ``try`` statement specifies exception handlers and/or cleanup code\nfor a group of statements:\n\n try_stmt ::= try1_stmt | try2_stmt\n try1_stmt ::= "try" ":" suite\n ("except" [expression [("as" | ",") target]] ":" suite)+\n ["else" ":" suite]\n ["finally" ":" suite]\n try2_stmt ::= "try" ":" suite\n "finally" ":" suite\n\nChanged in version 2.5: In previous versions of Python,\n``try``...``except``...``finally`` did not work. ``try``...``except``\nhad to be nested in ``try``...``finally``.\n\nThe ``except`` clause(s) specify one or more exception handlers. When\nno exception occurs in the ``try`` clause, no exception handler is\nexecuted. When an exception occurs in the ``try`` suite, a search for\nan exception handler is started. This search inspects the except\nclauses in turn until one is found that matches the exception. An\nexpression-less except clause, if present, must be last; it matches\nany exception. For an except clause with an expression, that\nexpression is evaluated, and the clause matches the exception if the\nresulting object is "compatible" with the exception. An object is\ncompatible with an exception if it is the class or a base class of the\nexception object, a tuple containing an item compatible with the\nexception, or, in the (deprecated) case of string exceptions, is the\nraised string itself (note that the object identities must match, i.e.\nit must be the same string object, not just a string with the same\nvalue).\n\nIf no except clause matches the exception, the search for an exception\nhandler continues in the surrounding code and on the invocation stack.\n[1]\n\nIf the evaluation of an expression in the header of an except clause\nraises an exception, the original search for a handler is canceled and\na search starts for the new exception in the surrounding code and on\nthe call stack (it is treated as if the entire ``try`` statement\nraised the exception).\n\nWhen a matching except clause is found, the exception is assigned to\nthe target specified in that except clause, if present, and the except\nclause\'s suite is executed. All except clauses must have an\nexecutable block. When the end of this block is reached, execution\ncontinues normally after the entire try statement. (This means that\nif two nested handlers exist for the same exception, and the exception\noccurs in the try clause of the inner handler, the outer handler will\nnot handle the exception.)\n\nBefore an except clause\'s suite is executed, details about the\nexception are assigned to three variables in the ``sys`` module:\n``sys.exc_type`` receives the object identifying the exception;\n``sys.exc_value`` receives the exception\'s parameter;\n``sys.exc_traceback`` receives a traceback object (see section *The\nstandard type hierarchy*) identifying the point in the program where\nthe exception occurred. These details are also available through the\n``sys.exc_info()`` function, which returns a tuple ``(exc_type,\nexc_value, exc_traceback)``. Use of the corresponding variables is\ndeprecated in favor of this function, since their use is unsafe in a\nthreaded program. As of Python 1.5, the variables are restored to\ntheir previous values (before the call) when returning from a function\nthat handled an exception.\n\nThe optional ``else`` clause is executed if and when control flows off\nthe end of the ``try`` clause. [2] Exceptions in the ``else`` clause\nare not handled by the preceding ``except`` clauses.\n\nIf ``finally`` is present, it specifies a \'cleanup\' handler. The\n``try`` clause is executed, including any ``except`` and ``else``\nclauses. If an exception occurs in any of the clauses and is not\nhandled, the exception is temporarily saved. The ``finally`` clause is\nexecuted. If there is a saved exception, it is re-raised at the end\nof the ``finally`` clause. If the ``finally`` clause raises another\nexception or executes a ``return`` or ``break`` statement, the saved\nexception is lost. The exception information is not available to the\nprogram during execution of the ``finally`` clause.\n\nWhen a ``return``, ``break`` or ``continue`` statement is executed in\nthe ``try`` suite of a ``try``...``finally`` statement, the\n``finally`` clause is also executed \'on the way out.\' A ``continue``\nstatement is illegal in the ``finally`` clause. (The reason is a\nproblem with the current implementation --- this restriction may be\nlifted in the future).\n\nAdditional information on exceptions can be found in section\n*Exceptions*, and information on using the ``raise`` statement to\ngenerate exceptions may be found in section *The raise statement*.\n\n\nThe ``with`` statement\n======================\n\nNew in version 2.5.\n\nThe ``with`` statement is used to wrap the execution of a block with\nmethods defined by a context manager (see section *With Statement\nContext Managers*). This allows common\n``try``...``except``...``finally`` usage patterns to be encapsulated\nfor convenient reuse.\n\n with_stmt ::= "with" with_item ("," with_item)* ":" suite\n with_item ::= expression ["as" target]\n\nThe execution of the ``with`` statement with one "item" proceeds as\nfollows:\n\n1. The context expression (the expression given in the **with_item**)\n is evaluated to obtain a context manager.\n\n2. The context manager\'s ``__exit__()`` is loaded for later use.\n\n3. The context manager\'s ``__enter__()`` method is invoked.\n\n4. If a target was included in the ``with`` statement, the return\n value from ``__enter__()`` is assigned to it.\n\n Note: The ``with`` statement guarantees that if the ``__enter__()``\n method returns without an error, then ``__exit__()`` will always\n be called. Thus, if an error occurs during the assignment to the\n target list, it will be treated the same as an error occurring\n within the suite would be. See step 6 below.\n\n5. The suite is executed.\n\n6. The context manager\'s ``__exit__()`` method is invoked. If an\n exception caused the suite to be exited, its type, value, and\n traceback are passed as arguments to ``__exit__()``. Otherwise,\n three ``None`` arguments are supplied.\n\n If the suite was exited due to an exception, and the return value\n from the ``__exit__()`` method was false, the exception is\n reraised. If the return value was true, the exception is\n suppressed, and execution continues with the statement following\n the ``with`` statement.\n\n If the suite was exited for any reason other than an exception, the\n return value from ``__exit__()`` is ignored, and execution proceeds\n at the normal location for the kind of exit that was taken.\n\nWith more than one item, the context managers are processed as if\nmultiple ``with`` statements were nested:\n\n with A() as a, B() as b:\n suite\n\nis equivalent to\n\n with A() as a:\n with B() as b:\n suite\n\nNote: In Python 2.5, the ``with`` statement is only allowed when the\n ``with_statement`` feature has been enabled. It is always enabled\n in Python 2.6.\n\nChanged in version 2.7: Support for multiple context expressions.\n\nSee also:\n\n **PEP 0343** - The "with" statement\n The specification, background, and examples for the Python\n ``with`` statement.\n\n\nFunction definitions\n====================\n\nA function definition defines a user-defined function object (see\nsection *The standard type hierarchy*):\n\n decorated ::= decorators (classdef | funcdef)\n decorators ::= decorator+\n decorator ::= "@" dotted_name ["(" [argument_list [","]] ")"] NEWLINE\n funcdef ::= "def" funcname "(" [parameter_list] ")" ":" suite\n dotted_name ::= identifier ("." identifier)*\n parameter_list ::= (defparameter ",")*\n ( "*" identifier [, "**" identifier]\n | "**" identifier\n | defparameter [","] )\n defparameter ::= parameter ["=" expression]\n sublist ::= parameter ("," parameter)* [","]\n parameter ::= identifier | "(" sublist ")"\n funcname ::= identifier\n\nA function definition is an executable statement. Its execution binds\nthe function name in the current local namespace to a function object\n(a wrapper around the executable code for the function). This\nfunction object contains a reference to the current global namespace\nas the global namespace to be used when the function is called.\n\nThe function definition does not execute the function body; this gets\nexecuted only when the function is called. [3]\n\nA function definition may be wrapped by one or more *decorator*\nexpressions. Decorator expressions are evaluated when the function is\ndefined, in the scope that contains the function definition. The\nresult must be a callable, which is invoked with the function object\nas the only argument. The returned value is bound to the function name\ninstead of the function object. Multiple decorators are applied in\nnested fashion. For example, the following code:\n\n @f1(arg)\n @f2\n def func(): pass\n\nis equivalent to:\n\n def func(): pass\n func = f1(arg)(f2(func))\n\nWhen one or more top-level parameters have the form *parameter* ``=``\n*expression*, the function is said to have "default parameter values."\nFor a parameter with a default value, the corresponding argument may\nbe omitted from a call, in which case the parameter\'s default value is\nsubstituted. If a parameter has a default value, all following\nparameters must also have a default value --- this is a syntactic\nrestriction that is not expressed by the grammar.\n\n**Default parameter values are evaluated when the function definition\nis executed.** This means that the expression is evaluated once, when\nthe function is defined, and that that same "pre-computed" value is\nused for each call. This is especially important to understand when a\ndefault parameter is a mutable object, such as a list or a dictionary:\nif the function modifies the object (e.g. by appending an item to a\nlist), the default value is in effect modified. This is generally not\nwhat was intended. A way around this is to use ``None`` as the\ndefault, and explicitly test for it in the body of the function, e.g.:\n\n def whats_on_the_telly(penguin=None):\n if penguin is None:\n penguin = []\n penguin.append("property of the zoo")\n return penguin\n\nFunction call semantics are described in more detail in section\n*Calls*. A function call always assigns values to all parameters\nmentioned in the parameter list, either from position arguments, from\nkeyword arguments, or from default values. If the form\n"``*identifier``" is present, it is initialized to a tuple receiving\nany excess positional parameters, defaulting to the empty tuple. If\nthe form "``**identifier``" is present, it is initialized to a new\ndictionary receiving any excess keyword arguments, defaulting to a new\nempty dictionary.\n\nIt is also possible to create anonymous functions (functions not bound\nto a name), for immediate use in expressions. This uses lambda forms,\ndescribed in section *Lambdas*. Note that the lambda form is merely a\nshorthand for a simplified function definition; a function defined in\na "``def``" statement can be passed around or assigned to another name\njust like a function defined by a lambda form. The "``def``" form is\nactually more powerful since it allows the execution of multiple\nstatements.\n\n**Programmer\'s note:** Functions are first-class objects. A "``def``"\nform executed inside a function definition defines a local function\nthat can be returned or passed around. Free variables used in the\nnested function can access the local variables of the function\ncontaining the def. See section *Naming and binding* for details.\n\n\nClass definitions\n=================\n\nA class definition defines a class object (see section *The standard\ntype hierarchy*):\n\n classdef ::= "class" classname [inheritance] ":" suite\n inheritance ::= "(" [expression_list] ")"\n classname ::= identifier\n\nA class definition is an executable statement. It first evaluates the\ninheritance list, if present. Each item in the inheritance list\nshould evaluate to a class object or class type which allows\nsubclassing. The class\'s suite is then executed in a new execution\nframe (see section *Naming and binding*), using a newly created local\nnamespace and the original global namespace. (Usually, the suite\ncontains only function definitions.) When the class\'s suite finishes\nexecution, its execution frame is discarded but its local namespace is\nsaved. [4] A class object is then created using the inheritance list\nfor the base classes and the saved local namespace for the attribute\ndictionary. The class name is bound to this class object in the\noriginal local namespace.\n\n**Programmer\'s note:** Variables defined in the class definition are\nclass variables; they are shared by all instances. To create instance\nvariables, they can be set in a method with ``self.name = value``.\nBoth class and instance variables are accessible through the notation\n"``self.name``", and an instance variable hides a class variable with\nthe same name when accessed in this way. Class variables can be used\nas defaults for instance variables, but using mutable values there can\nlead to unexpected results. For *new-style class*es, descriptors can\nbe used to create instance variables with different implementation\ndetails.\n\nClass definitions, like function definitions, may be wrapped by one or\nmore *decorator* expressions. The evaluation rules for the decorator\nexpressions are the same as for functions. The result must be a class\nobject, which is then bound to the class name.\n\n-[ Footnotes ]-\n\n[1] The exception is propagated to the invocation stack only if there\n is no ``finally`` clause that negates the exception.\n\n[2] Currently, control "flows off the end" except in the case of an\n exception or the execution of a ``return``, ``continue``, or\n ``break`` statement.\n\n[3] A string literal appearing as the first statement in the function\n body is transformed into the function\'s ``__doc__`` attribute and\n therefore the function\'s *docstring*.\n\n[4] A string literal appearing as the first statement in the class\n body is transformed into the namespace\'s ``__doc__`` item and\n therefore the class\'s *docstring*.\n', 'context-managers': u'\nWith Statement Context Managers\n*******************************\n\nNew in version 2.5.\n\nA *context manager* is an object that defines the runtime context to\nbe established when executing a ``with`` statement. The context\nmanager handles the entry into, and the exit from, the desired runtime\ncontext for the execution of the block of code. Context managers are\nnormally invoked using the ``with`` statement (described in section\n*The with statement*), but can also be used by directly invoking their\nmethods.\n\nTypical uses of context managers include saving and restoring various\nkinds of global state, locking and unlocking resources, closing opened\nfiles, etc.\n\nFor more information on context managers, see *Context Manager Types*.\n\nobject.__enter__(self)\n\n Enter the runtime context related to this object. The ``with``\n statement will bind this method\'s return value to the target(s)\n specified in the ``as`` clause of the statement, if any.\n\nobject.__exit__(self, exc_type, exc_value, traceback)\n\n Exit the runtime context related to this object. The parameters\n describe the exception that caused the context to be exited. If the\n context was exited without an exception, all three arguments will\n be ``None``.\n\n If an exception is supplied, and the method wishes to suppress the\n exception (i.e., prevent it from being propagated), it should\n return a true value. Otherwise, the exception will be processed\n normally upon exit from this method.\n\n Note that ``__exit__()`` methods should not reraise the passed-in\n exception; this is the caller\'s responsibility.\n\nSee also:\n\n **PEP 0343** - The "with" statement\n The specification, background, and examples for the Python\n ``with`` statement.\n', 'continue': u'\nThe ``continue`` statement\n**************************\n\n continue_stmt ::= "continue"\n\n``continue`` may only occur syntactically nested in a ``for`` or\n``while`` loop, but not nested in a function or class definition or\n``finally`` clause within that loop. It continues with the next cycle\nof the nearest enclosing loop.\n\nWhen ``continue`` passes control out of a ``try`` statement with a\n``finally`` clause, that ``finally`` clause is executed before really\nstarting the next loop cycle.\n', 'conversions': u'\nArithmetic conversions\n**********************\n\nWhen a description of an arithmetic operator below uses the phrase\n"the numeric arguments are converted to a common type," the arguments\nare coerced using the coercion rules listed at *Coercion rules*. If\nboth arguments are standard numeric types, the following coercions are\napplied:\n\n* If either argument is a complex number, the other is converted to\n complex;\n\n* otherwise, if either argument is a floating point number, the other\n is converted to floating point;\n\n* otherwise, if either argument is a long integer, the other is\n converted to long integer;\n\n* otherwise, both must be plain integers and no conversion is\n necessary.\n\nSome additional rules apply for certain operators (e.g., a string left\nargument to the \'%\' operator). Extensions can define their own\ncoercions.\n', 'customization': u'\nBasic customization\n*******************\n\nobject.__new__(cls[, ...])\n\n Called to create a new instance of class *cls*. ``__new__()`` is a\n static method (special-cased so you need not declare it as such)\n that takes the class of which an instance was requested as its\n first argument. The remaining arguments are those passed to the\n object constructor expression (the call to the class). The return\n value of ``__new__()`` should be the new object instance (usually\n an instance of *cls*).\n\n Typical implementations create a new instance of the class by\n invoking the superclass\'s ``__new__()`` method using\n ``super(currentclass, cls).__new__(cls[, ...])`` with appropriate\n arguments and then modifying the newly-created instance as\n necessary before returning it.\n\n If ``__new__()`` returns an instance of *cls*, then the new\n instance\'s ``__init__()`` method will be invoked like\n ``__init__(self[, ...])``, where *self* is the new instance and the\n remaining arguments are the same as were passed to ``__new__()``.\n\n If ``__new__()`` does not return an instance of *cls*, then the new\n instance\'s ``__init__()`` method will not be invoked.\n\n ``__new__()`` is intended mainly to allow subclasses of immutable\n types (like int, str, or tuple) to customize instance creation. It\n is also commonly overridden in custom metaclasses in order to\n customize class creation.\n\nobject.__init__(self[, ...])\n\n Called when the instance is created. The arguments are those\n passed to the class constructor expression. If a base class has an\n ``__init__()`` method, the derived class\'s ``__init__()`` method,\n if any, must explicitly call it to ensure proper initialization of\n the base class part of the instance; for example:\n ``BaseClass.__init__(self, [args...])``. As a special constraint\n on constructors, no value may be returned; doing so will cause a\n ``TypeError`` to be raised at runtime.\n\nobject.__del__(self)\n\n Called when the instance is about to be destroyed. This is also\n called a destructor. If a base class has a ``__del__()`` method,\n the derived class\'s ``__del__()`` method, if any, must explicitly\n call it to ensure proper deletion of the base class part of the\n instance. Note that it is possible (though not recommended!) for\n the ``__del__()`` method to postpone destruction of the instance by\n creating a new reference to it. It may then be called at a later\n time when this new reference is deleted. It is not guaranteed that\n ``__del__()`` methods are called for objects that still exist when\n the interpreter exits.\n\n Note: ``del x`` doesn\'t directly call ``x.__del__()`` --- the former\n decrements the reference count for ``x`` by one, and the latter\n is only called when ``x``\'s reference count reaches zero. Some\n common situations that may prevent the reference count of an\n object from going to zero include: circular references between\n objects (e.g., a doubly-linked list or a tree data structure with\n parent and child pointers); a reference to the object on the\n stack frame of a function that caught an exception (the traceback\n stored in ``sys.exc_traceback`` keeps the stack frame alive); or\n a reference to the object on the stack frame that raised an\n unhandled exception in interactive mode (the traceback stored in\n ``sys.last_traceback`` keeps the stack frame alive). The first\n situation can only be remedied by explicitly breaking the cycles;\n the latter two situations can be resolved by storing ``None`` in\n ``sys.exc_traceback`` or ``sys.last_traceback``. Circular\n references which are garbage are detected when the option cycle\n detector is enabled (it\'s on by default), but can only be cleaned\n up if there are no Python-level ``__del__()`` methods involved.\n Refer to the documentation for the ``gc`` module for more\n information about how ``__del__()`` methods are handled by the\n cycle detector, particularly the description of the ``garbage``\n value.\n\n Warning: Due to the precarious circumstances under which ``__del__()``\n methods are invoked, exceptions that occur during their execution\n are ignored, and a warning is printed to ``sys.stderr`` instead.\n Also, when ``__del__()`` is invoked in response to a module being\n deleted (e.g., when execution of the program is done), other\n globals referenced by the ``__del__()`` method may already have\n been deleted or in the process of being torn down (e.g. the\n import machinery shutting down). For this reason, ``__del__()``\n methods should do the absolute minimum needed to maintain\n external invariants. Starting with version 1.5, Python\n guarantees that globals whose name begins with a single\n underscore are deleted from their module before other globals are\n deleted; if no other references to such globals exist, this may\n help in assuring that imported modules are still available at the\n time when the ``__del__()`` method is called.\n\nobject.__repr__(self)\n\n Called by the ``repr()`` built-in function and by string\n conversions (reverse quotes) to compute the "official" string\n representation of an object. If at all possible, this should look\n like a valid Python expression that could be used to recreate an\n object with the same value (given an appropriate environment). If\n this is not possible, a string of the form ``<...some useful\n description...>`` should be returned. The return value must be a\n string object. If a class defines ``__repr__()`` but not\n ``__str__()``, then ``__repr__()`` is also used when an "informal"\n string representation of instances of that class is required.\n\n This is typically used for debugging, so it is important that the\n representation is information-rich and unambiguous.\n\nobject.__str__(self)\n\n Called by the ``str()`` built-in function and by the ``print``\n statement to compute the "informal" string representation of an\n object. This differs from ``__repr__()`` in that it does not have\n to be a valid Python expression: a more convenient or concise\n representation may be used instead. The return value must be a\n string object.\n\nobject.__lt__(self, other)\nobject.__le__(self, other)\nobject.__eq__(self, other)\nobject.__ne__(self, other)\nobject.__gt__(self, other)\nobject.__ge__(self, other)\n\n New in version 2.1.\n\n These are the so-called "rich comparison" methods, and are called\n for comparison operators in preference to ``__cmp__()`` below. The\n correspondence between operator symbols and method names is as\n follows: ``x<y`` calls ``x.__lt__(y)``, ``x<=y`` calls\n ``x.__le__(y)``, ``x==y`` calls ``x.__eq__(y)``, ``x!=y`` and\n ``x<>y`` call ``x.__ne__(y)``, ``x>y`` calls ``x.__gt__(y)``, and\n ``x>=y`` calls ``x.__ge__(y)``.\n\n A rich comparison method may return the singleton\n ``NotImplemented`` if it does not implement the operation for a\n given pair of arguments. By convention, ``False`` and ``True`` are\n returned for a successful comparison. However, these methods can\n return any value, so if the comparison operator is used in a\n Boolean context (e.g., in the condition of an ``if`` statement),\n Python will call ``bool()`` on the value to determine if the result\n is true or false.\n\n There are no implied relationships among the comparison operators.\n The truth of ``x==y`` does not imply that ``x!=y`` is false.\n Accordingly, when defining ``__eq__()``, one should also define\n ``__ne__()`` so that the operators will behave as expected. See\n the paragraph on ``__hash__()`` for some important notes on\n creating *hashable* objects which support custom comparison\n operations and are usable as dictionary keys.\n\n There are no swapped-argument versions of these methods (to be used\n when the left argument does not support the operation but the right\n argument does); rather, ``__lt__()`` and ``__gt__()`` are each\n other\'s reflection, ``__le__()`` and ``__ge__()`` are each other\'s\n reflection, and ``__eq__()`` and ``__ne__()`` are their own\n reflection.\n\n Arguments to rich comparison methods are never coerced.\n\n To automatically generate ordering operations from a single root\n operation, see ``functools.total_ordering()``.\n\nobject.__cmp__(self, other)\n\n Called by comparison operations if rich comparison (see above) is\n not defined. Should return a negative integer if ``self < other``,\n zero if ``self == other``, a positive integer if ``self > other``.\n If no ``__cmp__()``, ``__eq__()`` or ``__ne__()`` operation is\n defined, class instances are compared by object identity\n ("address"). See also the description of ``__hash__()`` for some\n important notes on creating *hashable* objects which support custom\n comparison operations and are usable as dictionary keys. (Note: the\n restriction that exceptions are not propagated by ``__cmp__()`` has\n been removed since Python 1.5.)\n\nobject.__rcmp__(self, other)\n\n Changed in version 2.1: No longer supported.\n\nobject.__hash__(self)\n\n Called by built-in function ``hash()`` and for operations on\n members of hashed collections including ``set``, ``frozenset``, and\n ``dict``. ``__hash__()`` should return an integer. The only\n required property is that objects which compare equal have the same\n hash value; it is advised to somehow mix together (e.g. using\n exclusive or) the hash values for the components of the object that\n also play a part in comparison of objects.\n\n If a class does not define a ``__cmp__()`` or ``__eq__()`` method\n it should not define a ``__hash__()`` operation either; if it\n defines ``__cmp__()`` or ``__eq__()`` but not ``__hash__()``, its\n instances will not be usable in hashed collections. If a class\n defines mutable objects and implements a ``__cmp__()`` or\n ``__eq__()`` method, it should not implement ``__hash__()``, since\n hashable collection implementations require that a object\'s hash\n value is immutable (if the object\'s hash value changes, it will be\n in the wrong hash bucket).\n\n User-defined classes have ``__cmp__()`` and ``__hash__()`` methods\n by default; with them, all objects compare unequal (except with\n themselves) and ``x.__hash__()`` returns ``id(x)``.\n\n Classes which inherit a ``__hash__()`` method from a parent class\n but change the meaning of ``__cmp__()`` or ``__eq__()`` such that\n the hash value returned is no longer appropriate (e.g. by switching\n to a value-based concept of equality instead of the default\n identity based equality) can explicitly flag themselves as being\n unhashable by setting ``__hash__ = None`` in the class definition.\n Doing so means that not only will instances of the class raise an\n appropriate ``TypeError`` when a program attempts to retrieve their\n hash value, but they will also be correctly identified as\n unhashable when checking ``isinstance(obj, collections.Hashable)``\n (unlike classes which define their own ``__hash__()`` to explicitly\n raise ``TypeError``).\n\n Changed in version 2.5: ``__hash__()`` may now also return a long\n integer object; the 32-bit integer is then derived from the hash of\n that object.\n\n Changed in version 2.6: ``__hash__`` may now be set to ``None`` to\n explicitly flag instances of a class as unhashable.\n\nobject.__nonzero__(self)\n\n Called to implement truth value testing and the built-in operation\n ``bool()``; should return ``False`` or ``True``, or their integer\n equivalents ``0`` or ``1``. When this method is not defined,\n ``__len__()`` is called, if it is defined, and the object is\n considered true if its result is nonzero. If a class defines\n neither ``__len__()`` nor ``__nonzero__()``, all its instances are\n considered true.\n\nobject.__unicode__(self)\n\n Called to implement ``unicode()`` built-in; should return a Unicode\n object. When this method is not defined, string conversion is\n attempted, and the result of string conversion is converted to\n Unicode using the system default encoding.\n', - 'debugger': u'\n``pdb`` --- The Python Debugger\n*******************************\n\nThe module ``pdb`` defines an interactive source code debugger for\nPython programs. It supports setting (conditional) breakpoints and\nsingle stepping at the source line level, inspection of stack frames,\nsource code listing, and evaluation of arbitrary Python code in the\ncontext of any stack frame. It also supports post-mortem debugging\nand can be called under program control.\n\nThe debugger is extensible --- it is actually defined as the class\n``Pdb``. This is currently undocumented but easily understood by\nreading the source. The extension interface uses the modules ``bdb``\nand ``cmd``.\n\nThe debugger\'s prompt is ``(Pdb)``. Typical usage to run a program\nunder control of the debugger is:\n\n >>> import pdb\n >>> import mymodule\n >>> pdb.run(\'mymodule.test()\')\n > <string>(0)?()\n (Pdb) continue\n > <string>(1)?()\n (Pdb) continue\n NameError: \'spam\'\n > <string>(1)?()\n (Pdb)\n\n``pdb.py`` can also be invoked as a script to debug other scripts.\nFor example:\n\n python -m pdb myscript.py\n\nWhen invoked as a script, pdb will automatically enter post-mortem\ndebugging if the program being debugged exits abnormally. After post-\nmortem debugging (or after normal exit of the program), pdb will\nrestart the program. Automatic restarting preserves pdb\'s state (such\nas breakpoints) and in most cases is more useful than quitting the\ndebugger upon program\'s exit.\n\nNew in version 2.4: Restarting post-mortem behavior added.\n\nThe typical usage to break into the debugger from a running program is\nto insert\n\n import pdb; pdb.set_trace()\n\nat the location you want to break into the debugger. You can then\nstep through the code following this statement, and continue running\nwithout the debugger using the ``c`` command.\n\nThe typical usage to inspect a crashed program is:\n\n >>> import pdb\n >>> import mymodule\n >>> mymodule.test()\n Traceback (most recent call last):\n File "<stdin>", line 1, in ?\n File "./mymodule.py", line 4, in test\n test2()\n File "./mymodule.py", line 3, in test2\n print spam\n NameError: spam\n >>> pdb.pm()\n > ./mymodule.py(3)test2()\n -> print spam\n (Pdb)\n\nThe module defines the following functions; each enters the debugger\nin a slightly different way:\n\npdb.run(statement[, globals[, locals]])\n\n Execute the *statement* (given as a string) under debugger control.\n The debugger prompt appears before any code is executed; you can\n set breakpoints and type ``continue``, or you can step through the\n statement using ``step`` or ``next`` (all these commands are\n explained below). The optional *globals* and *locals* arguments\n specify the environment in which the code is executed; by default\n the dictionary of the module ``__main__`` is used. (See the\n explanation of the ``exec`` statement or the ``eval()`` built-in\n function.)\n\npdb.runeval(expression[, globals[, locals]])\n\n Evaluate the *expression* (given as a string) under debugger\n control. When ``runeval()`` returns, it returns the value of the\n expression. Otherwise this function is similar to ``run()``.\n\npdb.runcall(function[, argument, ...])\n\n Call the *function* (a function or method object, not a string)\n with the given arguments. When ``runcall()`` returns, it returns\n whatever the function call returned. The debugger prompt appears\n as soon as the function is entered.\n\npdb.set_trace()\n\n Enter the debugger at the calling stack frame. This is useful to\n hard-code a breakpoint at a given point in a program, even if the\n code is not otherwise being debugged (e.g. when an assertion\n fails).\n\npdb.post_mortem([traceback])\n\n Enter post-mortem debugging of the given *traceback* object. If no\n *traceback* is given, it uses the one of the exception that is\n currently being handled (an exception must be being handled if the\n default is to be used).\n\npdb.pm()\n\n Enter post-mortem debugging of the traceback found in\n ``sys.last_traceback``.\n\nThe ``run_*`` functions and ``set_trace()`` are aliases for\ninstantiating the ``Pdb`` class and calling the method of the same\nname. If you want to access further features, you have to do this\nyourself:\n\nclass class pdb.Pdb(completekey=\'tab\', stdin=None, stdout=None, skip=None)\n\n ``Pdb`` is the debugger class.\n\n The *completekey*, *stdin* and *stdout* arguments are passed to the\n underlying ``cmd.Cmd`` class; see the description there.\n\n The *skip* argument, if given, must be an iterable of glob-style\n module name patterns. The debugger will not step into frames that\n originate in a module that matches one of these patterns. [1]\n\n Example call to enable tracing with *skip*:\n\n import pdb; pdb.Pdb(skip=[\'django.*\']).set_trace()\n\n New in version 2.7: The *skip* argument.\n\n run(statement[, globals[, locals]])\n runeval(expression[, globals[, locals]])\n runcall(function[, argument, ...])\n set_trace()\n\n See the documentation for the functions explained above.\n', + 'debugger': u'\n``pdb`` --- The Python Debugger\n*******************************\n\nThe module ``pdb`` defines an interactive source code debugger for\nPython programs. It supports setting (conditional) breakpoints and\nsingle stepping at the source line level, inspection of stack frames,\nsource code listing, and evaluation of arbitrary Python code in the\ncontext of any stack frame. It also supports post-mortem debugging\nand can be called under program control.\n\nThe debugger is extensible --- it is actually defined as the class\n``Pdb``. This is currently undocumented but easily understood by\nreading the source. The extension interface uses the modules ``bdb``\nand ``cmd``.\n\nThe debugger\'s prompt is ``(Pdb)``. Typical usage to run a program\nunder control of the debugger is:\n\n >>> import pdb\n >>> import mymodule\n >>> pdb.run(\'mymodule.test()\')\n > <string>(0)?()\n (Pdb) continue\n > <string>(1)?()\n (Pdb) continue\n NameError: \'spam\'\n > <string>(1)?()\n (Pdb)\n\n``pdb.py`` can also be invoked as a script to debug other scripts.\nFor example:\n\n python -m pdb myscript.py\n\nWhen invoked as a script, pdb will automatically enter post-mortem\ndebugging if the program being debugged exits abnormally. After post-\nmortem debugging (or after normal exit of the program), pdb will\nrestart the program. Automatic restarting preserves pdb\'s state (such\nas breakpoints) and in most cases is more useful than quitting the\ndebugger upon program\'s exit.\n\nNew in version 2.4: Restarting post-mortem behavior added.\n\nThe typical usage to break into the debugger from a running program is\nto insert\n\n import pdb; pdb.set_trace()\n\nat the location you want to break into the debugger. You can then\nstep through the code following this statement, and continue running\nwithout the debugger using the ``c`` command.\n\nThe typical usage to inspect a crashed program is:\n\n >>> import pdb\n >>> import mymodule\n >>> mymodule.test()\n Traceback (most recent call last):\n File "<stdin>", line 1, in ?\n File "./mymodule.py", line 4, in test\n test2()\n File "./mymodule.py", line 3, in test2\n print spam\n NameError: spam\n >>> pdb.pm()\n > ./mymodule.py(3)test2()\n -> print spam\n (Pdb)\n\nThe module defines the following functions; each enters the debugger\nin a slightly different way:\n\npdb.run(statement[, globals[, locals]])\n\n Execute the *statement* (given as a string) under debugger control.\n The debugger prompt appears before any code is executed; you can\n set breakpoints and type ``continue``, or you can step through the\n statement using ``step`` or ``next`` (all these commands are\n explained below). The optional *globals* and *locals* arguments\n specify the environment in which the code is executed; by default\n the dictionary of the module ``__main__`` is used. (See the\n explanation of the ``exec`` statement or the ``eval()`` built-in\n function.)\n\npdb.runeval(expression[, globals[, locals]])\n\n Evaluate the *expression* (given as a string) under debugger\n control. When ``runeval()`` returns, it returns the value of the\n expression. Otherwise this function is similar to ``run()``.\n\npdb.runcall(function[, argument, ...])\n\n Call the *function* (a function or method object, not a string)\n with the given arguments. When ``runcall()`` returns, it returns\n whatever the function call returned. The debugger prompt appears\n as soon as the function is entered.\n\npdb.set_trace()\n\n Enter the debugger at the calling stack frame. This is useful to\n hard-code a breakpoint at a given point in a program, even if the\n code is not otherwise being debugged (e.g. when an assertion\n fails).\n\npdb.post_mortem([traceback])\n\n Enter post-mortem debugging of the given *traceback* object. If no\n *traceback* is given, it uses the one of the exception that is\n currently being handled (an exception must be being handled if the\n default is to be used).\n\npdb.pm()\n\n Enter post-mortem debugging of the traceback found in\n ``sys.last_traceback``.\n\nThe ``run*`` functions and ``set_trace()`` are aliases for\ninstantiating the ``Pdb`` class and calling the method of the same\nname. If you want to access further features, you have to do this\nyourself:\n\nclass class pdb.Pdb(completekey=\'tab\', stdin=None, stdout=None, skip=None)\n\n ``Pdb`` is the debugger class.\n\n The *completekey*, *stdin* and *stdout* arguments are passed to the\n underlying ``cmd.Cmd`` class; see the description there.\n\n The *skip* argument, if given, must be an iterable of glob-style\n module name patterns. The debugger will not step into frames that\n originate in a module that matches one of these patterns. [1]\n\n Example call to enable tracing with *skip*:\n\n import pdb; pdb.Pdb(skip=[\'django.*\']).set_trace()\n\n New in version 2.7: The *skip* argument.\n\n run(statement[, globals[, locals]])\n runeval(expression[, globals[, locals]])\n runcall(function[, argument, ...])\n set_trace()\n\n See the documentation for the functions explained above.\n', 'del': u'\nThe ``del`` statement\n*********************\n\n del_stmt ::= "del" target_list\n\nDeletion is recursively defined very similar to the way assignment is\ndefined. Rather that spelling it out in full details, here are some\nhints.\n\nDeletion of a target list recursively deletes each target, from left\nto right.\n\nDeletion of a name removes the binding of that name from the local or\nglobal namespace, depending on whether the name occurs in a ``global``\nstatement in the same code block. If the name is unbound, a\n``NameError`` exception will be raised.\n\nIt is illegal to delete a name from the local namespace if it occurs\nas a free variable in a nested block.\n\nDeletion of attribute references, subscriptions and slicings is passed\nto the primary object involved; deletion of a slicing is in general\nequivalent to assignment of an empty slice of the right type (but even\nthis is determined by the sliced object).\n', 'dict': u'\nDictionary displays\n*******************\n\nA dictionary display is a possibly empty series of key/datum pairs\nenclosed in curly braces:\n\n dict_display ::= "{" [key_datum_list | dict_comprehension] "}"\n key_datum_list ::= key_datum ("," key_datum)* [","]\n key_datum ::= expression ":" expression\n dict_comprehension ::= expression ":" expression comp_for\n\nA dictionary display yields a new dictionary object.\n\nIf a comma-separated sequence of key/datum pairs is given, they are\nevaluated from left to right to define the entries of the dictionary:\neach key object is used as a key into the dictionary to store the\ncorresponding datum. This means that you can specify the same key\nmultiple times in the key/datum list, and the final dictionary\'s value\nfor that key will be the last one given.\n\nA dict comprehension, in contrast to list and set comprehensions,\nneeds two expressions separated with a colon followed by the usual\n"for" and "if" clauses. When the comprehension is run, the resulting\nkey and value elements are inserted in the new dictionary in the order\nthey are produced.\n\nRestrictions on the types of the key values are listed earlier in\nsection *The standard type hierarchy*. (To summarize, the key type\nshould be *hashable*, which excludes all mutable objects.) Clashes\nbetween duplicate keys are not detected; the last datum (textually\nrightmost in the display) stored for a given key value prevails.\n', 'dynamic-features': u'\nInteraction with dynamic features\n*********************************\n\nThere are several cases where Python statements are illegal when used\nin conjunction with nested scopes that contain free variables.\n\nIf a variable is referenced in an enclosing scope, it is illegal to\ndelete the name. An error will be reported at compile time.\n\nIf the wild card form of import --- ``import *`` --- is used in a\nfunction and the function contains or is a nested block with free\nvariables, the compiler will raise a ``SyntaxError``.\n\nIf ``exec`` is used in a function and the function contains or is a\nnested block with free variables, the compiler will raise a\n``SyntaxError`` unless the exec explicitly specifies the local\nnamespace for the ``exec``. (In other words, ``exec obj`` would be\nillegal, but ``exec obj in ns`` would be legal.)\n\nThe ``eval()``, ``execfile()``, and ``input()`` functions and the\n``exec`` statement do not have access to the full environment for\nresolving names. Names may be resolved in the local and global\nnamespaces of the caller. Free variables are not resolved in the\nnearest enclosing namespace, but in the global namespace. [1] The\n``exec`` statement and the ``eval()`` and ``execfile()`` functions\nhave optional arguments to override the global and local namespace.\nIf only one namespace is specified, it is used for both.\n', 'else': u'\nThe ``if`` statement\n********************\n\nThe ``if`` statement is used for conditional execution:\n\n if_stmt ::= "if" expression ":" suite\n ( "elif" expression ":" suite )*\n ["else" ":" suite]\n\nIt selects exactly one of the suites by evaluating the expressions one\nby one until one is found to be true (see section *Boolean operations*\nfor the definition of true and false); then that suite is executed\n(and no other part of the ``if`` statement is executed or evaluated).\nIf all expressions are false, the suite of the ``else`` clause, if\npresent, is executed.\n', 'exceptions': u'\nExceptions\n**********\n\nExceptions are a means of breaking out of the normal flow of control\nof a code block in order to handle errors or other exceptional\nconditions. An exception is *raised* at the point where the error is\ndetected; it may be *handled* by the surrounding code block or by any\ncode block that directly or indirectly invoked the code block where\nthe error occurred.\n\nThe Python interpreter raises an exception when it detects a run-time\nerror (such as division by zero). A Python program can also\nexplicitly raise an exception with the ``raise`` statement. Exception\nhandlers are specified with the ``try`` ... ``except`` statement. The\n``finally`` clause of such a statement can be used to specify cleanup\ncode which does not handle the exception, but is executed whether an\nexception occurred or not in the preceding code.\n\nPython uses the "termination" model of error handling: an exception\nhandler can find out what happened and continue execution at an outer\nlevel, but it cannot repair the cause of the error and retry the\nfailing operation (except by re-entering the offending piece of code\nfrom the top).\n\nWhen an exception is not handled at all, the interpreter terminates\nexecution of the program, or returns to its interactive main loop. In\neither case, it prints a stack backtrace, except when the exception is\n``SystemExit``.\n\nExceptions are identified by class instances. The ``except`` clause\nis selected depending on the class of the instance: it must reference\nthe class of the instance or a base class thereof. The instance can\nbe received by the handler and can carry additional information about\nthe exceptional condition.\n\nExceptions can also be identified by strings, in which case the\n``except`` clause is selected by object identity. An arbitrary value\ncan be raised along with the identifying string which can be passed to\nthe handler.\n\nNote: Messages to exceptions are not part of the Python API. Their\n contents may change from one version of Python to the next without\n warning and should not be relied on by code which will run under\n multiple versions of the interpreter.\n\nSee also the description of the ``try`` statement in section *The try\nstatement* and ``raise`` statement in section *The raise statement*.\n\n-[ Footnotes ]-\n\n[1] This limitation occurs because the code that is executed by these\n operations is not available at the time the module is compiled.\n', 'exec': u'\nThe ``exec`` statement\n**********************\n\n exec_stmt ::= "exec" or_expr ["in" expression ["," expression]]\n\nThis statement supports dynamic execution of Python code. The first\nexpression should evaluate to either a string, an open file object, or\na code object. If it is a string, the string is parsed as a suite of\nPython statements which is then executed (unless a syntax error\noccurs). [1] If it is an open file, the file is parsed until EOF and\nexecuted. If it is a code object, it is simply executed. In all\ncases, the code that\'s executed is expected to be valid as file input\n(see section *File input*). Be aware that the ``return`` and\n``yield`` statements may not be used outside of function definitions\neven within the context of code passed to the ``exec`` statement.\n\nIn all cases, if the optional parts are omitted, the code is executed\nin the current scope. If only the first expression after ``in`` is\nspecified, it should be a dictionary, which will be used for both the\nglobal and the local variables. If two expressions are given, they\nare used for the global and local variables, respectively. If\nprovided, *locals* can be any mapping object.\n\nChanged in version 2.4: Formerly, *locals* was required to be a\ndictionary.\n\nAs a side effect, an implementation may insert additional keys into\nthe dictionaries given besides those corresponding to variable names\nset by the executed code. For example, the current implementation may\nadd a reference to the dictionary of the built-in module\n``__builtin__`` under the key ``__builtins__`` (!).\n\n**Programmer\'s hints:** dynamic evaluation of expressions is supported\nby the built-in function ``eval()``. The built-in functions\n``globals()`` and ``locals()`` return the current global and local\ndictionary, respectively, which may be useful to pass around for use\nby ``exec``.\n\n-[ Footnotes ]-\n\n[1] Note that the parser only accepts the Unix-style end of line\n convention. If you are reading the code from a file, make sure to\n use universal newline mode to convert Windows or Mac-style\n newlines.\n', - 'execmodel': u'\nExecution model\n***************\n\n\nNaming and binding\n==================\n\n*Names* refer to objects. Names are introduced by name binding\noperations. Each occurrence of a name in the program text refers to\nthe *binding* of that name established in the innermost function block\ncontaining the use.\n\nA *block* is a piece of Python program text that is executed as a\nunit. The following are blocks: a module, a function body, and a class\ndefinition. Each command typed interactively is a block. A script\nfile (a file given as standard input to the interpreter or specified\non the interpreter command line the first argument) is a code block.\nA script command (a command specified on the interpreter command line\nwith the \'**-c**\' option) is a code block. The file read by the\nbuilt-in function ``execfile()`` is a code block. The string argument\npassed to the built-in function ``eval()`` and to the ``exec``\nstatement is a code block. The expression read and evaluated by the\nbuilt-in function ``input()`` is a code block.\n\nA code block is executed in an *execution frame*. A frame contains\nsome administrative information (used for debugging) and determines\nwhere and how execution continues after the code block\'s execution has\ncompleted.\n\nA *scope* defines the visibility of a name within a block. If a local\nvariable is defined in a block, its scope includes that block. If the\ndefinition occurs in a function block, the scope extends to any blocks\ncontained within the defining one, unless a contained block introduces\na different binding for the name. The scope of names defined in a\nclass block is limited to the class block; it does not extend to the\ncode blocks of methods -- this includes generator expressions since\nthey are implemented using a function scope. This means that the\nfollowing will fail:\n\n class A:\n a = 42\n b = list(a + i for i in range(10))\n\nWhen a name is used in a code block, it is resolved using the nearest\nenclosing scope. The set of all such scopes visible to a code block\nis called the block\'s *environment*.\n\nIf a name is bound in a block, it is a local variable of that block.\nIf a name is bound at the module level, it is a global variable. (The\nvariables of the module code block are local and global.) If a\nvariable is used in a code block but not defined there, it is a *free\nvariable*.\n\nWhen a name is not found at all, a ``NameError`` exception is raised.\nIf the name refers to a local variable that has not been bound, a\n``UnboundLocalError`` exception is raised. ``UnboundLocalError`` is a\nsubclass of ``NameError``.\n\nThe following constructs bind names: formal parameters to functions,\n``import`` statements, class and function definitions (these bind the\nclass or function name in the defining block), and targets that are\nidentifiers if occurring in an assignment, ``for`` loop header, in the\nsecond position of an ``except`` clause header or after ``as`` in a\n``with`` statement. The ``import`` statement of the form ``from ...\nimport *`` binds all names defined in the imported module, except\nthose beginning with an underscore. This form may only be used at the\nmodule level.\n\nA target occurring in a ``del`` statement is also considered bound for\nthis purpose (though the actual semantics are to unbind the name). It\nis illegal to unbind a name that is referenced by an enclosing scope;\nthe compiler will report a ``SyntaxError``.\n\nEach assignment or import statement occurs within a block defined by a\nclass or function definition or at the module level (the top-level\ncode block).\n\nIf a name binding operation occurs anywhere within a code block, all\nuses of the name within the block are treated as references to the\ncurrent block. This can lead to errors when a name is used within a\nblock before it is bound. This rule is subtle. Python lacks\ndeclarations and allows name binding operations to occur anywhere\nwithin a code block. The local variables of a code block can be\ndetermined by scanning the entire text of the block for name binding\noperations.\n\nIf the global statement occurs within a block, all uses of the name\nspecified in the statement refer to the binding of that name in the\ntop-level namespace. Names are resolved in the top-level namespace by\nsearching the global namespace, i.e. the namespace of the module\ncontaining the code block, and the builtins namespace, the namespace\nof the module ``__builtin__``. The global namespace is searched\nfirst. If the name is not found there, the builtins namespace is\nsearched. The global statement must precede all uses of the name.\n\nThe builtins namespace associated with the execution of a code block\nis actually found by looking up the name ``__builtins__`` in its\nglobal namespace; this should be a dictionary or a module (in the\nlatter case the module\'s dictionary is used). By default, when in the\n``__main__`` module, ``__builtins__`` is the built-in module\n``__builtin__`` (note: no \'s\'); when in any other module,\n``__builtins__`` is an alias for the dictionary of the ``__builtin__``\nmodule itself. ``__builtins__`` can be set to a user-created\ndictionary to create a weak form of restricted execution.\n\n**CPython implementation detail:** Users should not touch\n``__builtins__``; it is strictly an implementation detail. Users\nwanting to override values in the builtins namespace should ``import``\nthe ``__builtin__`` (no \'s\') module and modify its attributes\nappropriately.\n\nThe namespace for a module is automatically created the first time a\nmodule is imported. The main module for a script is always called\n``__main__``.\n\nThe global statement has the same scope as a name binding operation in\nthe same block. If the nearest enclosing scope for a free variable\ncontains a global statement, the free variable is treated as a global.\n\nA class definition is an executable statement that may use and define\nnames. These references follow the normal rules for name resolution.\nThe namespace of the class definition becomes the attribute dictionary\nof the class. Names defined at the class scope are not visible in\nmethods.\n\n\nInteraction with dynamic features\n---------------------------------\n\nThere are several cases where Python statements are illegal when used\nin conjunction with nested scopes that contain free variables.\n\nIf a variable is referenced in an enclosing scope, it is illegal to\ndelete the name. An error will be reported at compile time.\n\nIf the wild card form of import --- ``import *`` --- is used in a\nfunction and the function contains or is a nested block with free\nvariables, the compiler will raise a ``SyntaxError``.\n\nIf ``exec`` is used in a function and the function contains or is a\nnested block with free variables, the compiler will raise a\n``SyntaxError`` unless the exec explicitly specifies the local\nnamespace for the ``exec``. (In other words, ``exec obj`` would be\nillegal, but ``exec obj in ns`` would be legal.)\n\nThe ``eval()``, ``execfile()``, and ``input()`` functions and the\n``exec`` statement do not have access to the full environment for\nresolving names. Names may be resolved in the local and global\nnamespaces of the caller. Free variables are not resolved in the\nnearest enclosing namespace, but in the global namespace. [1] The\n``exec`` statement and the ``eval()`` and ``execfile()`` functions\nhave optional arguments to override the global and local namespace.\nIf only one namespace is specified, it is used for both.\n\n\nExceptions\n==========\n\nExceptions are a means of breaking out of the normal flow of control\nof a code block in order to handle errors or other exceptional\nconditions. An exception is *raised* at the point where the error is\ndetected; it may be *handled* by the surrounding code block or by any\ncode block that directly or indirectly invoked the code block where\nthe error occurred.\n\nThe Python interpreter raises an exception when it detects a run-time\nerror (such as division by zero). A Python program can also\nexplicitly raise an exception with the ``raise`` statement. Exception\nhandlers are specified with the ``try`` ... ``except`` statement. The\n``finally`` clause of such a statement can be used to specify cleanup\ncode which does not handle the exception, but is executed whether an\nexception occurred or not in the preceding code.\n\nPython uses the "termination" model of error handling: an exception\nhandler can find out what happened and continue execution at an outer\nlevel, but it cannot repair the cause of the error and retry the\nfailing operation (except by re-entering the offending piece of code\nfrom the top).\n\nWhen an exception is not handled at all, the interpreter terminates\nexecution of the program, or returns to its interactive main loop. In\neither case, it prints a stack backtrace, except when the exception is\n``SystemExit``.\n\nExceptions are identified by class instances. The ``except`` clause\nis selected depending on the class of the instance: it must reference\nthe class of the instance or a base class thereof. The instance can\nbe received by the handler and can carry additional information about\nthe exceptional condition.\n\nExceptions can also be identified by strings, in which case the\n``except`` clause is selected by object identity. An arbitrary value\ncan be raised along with the identifying string which can be passed to\nthe handler.\n\nNote: Messages to exceptions are not part of the Python API. Their\n contents may change from one version of Python to the next without\n warning and should not be relied on by code which will run under\n multiple versions of the interpreter.\n\nSee also the description of the ``try`` statement in section *The try\nstatement* and ``raise`` statement in section *The raise statement*.\n\n-[ Footnotes ]-\n\n[1] This limitation occurs because the code that is executed by these\n operations is not available at the time the module is compiled.\n', + 'execmodel': u'\nExecution model\n***************\n\n\nNaming and binding\n==================\n\n*Names* refer to objects. Names are introduced by name binding\noperations. Each occurrence of a name in the program text refers to\nthe *binding* of that name established in the innermost function block\ncontaining the use.\n\nA *block* is a piece of Python program text that is executed as a\nunit. The following are blocks: a module, a function body, and a class\ndefinition. Each command typed interactively is a block. A script\nfile (a file given as standard input to the interpreter or specified\non the interpreter command line the first argument) is a code block.\nA script command (a command specified on the interpreter command line\nwith the \'**-c**\' option) is a code block. The file read by the\nbuilt-in function ``execfile()`` is a code block. The string argument\npassed to the built-in function ``eval()`` and to the ``exec``\nstatement is a code block. The expression read and evaluated by the\nbuilt-in function ``input()`` is a code block.\n\nA code block is executed in an *execution frame*. A frame contains\nsome administrative information (used for debugging) and determines\nwhere and how execution continues after the code block\'s execution has\ncompleted.\n\nA *scope* defines the visibility of a name within a block. If a local\nvariable is defined in a block, its scope includes that block. If the\ndefinition occurs in a function block, the scope extends to any blocks\ncontained within the defining one, unless a contained block introduces\na different binding for the name. The scope of names defined in a\nclass block is limited to the class block; it does not extend to the\ncode blocks of methods -- this includes generator expressions since\nthey are implemented using a function scope. This means that the\nfollowing will fail:\n\n class A:\n a = 42\n b = list(a + i for i in range(10))\n\nWhen a name is used in a code block, it is resolved using the nearest\nenclosing scope. The set of all such scopes visible to a code block\nis called the block\'s *environment*.\n\nIf a name is bound in a block, it is a local variable of that block.\nIf a name is bound at the module level, it is a global variable. (The\nvariables of the module code block are local and global.) If a\nvariable is used in a code block but not defined there, it is a *free\nvariable*.\n\nWhen a name is not found at all, a ``NameError`` exception is raised.\nIf the name refers to a local variable that has not been bound, a\n``UnboundLocalError`` exception is raised. ``UnboundLocalError`` is a\nsubclass of ``NameError``.\n\nThe following constructs bind names: formal parameters to functions,\n``import`` statements, class and function definitions (these bind the\nclass or function name in the defining block), and targets that are\nidentifiers if occurring in an assignment, ``for`` loop header, in the\nsecond position of an ``except`` clause header or after ``as`` in a\n``with`` statement. The ``import`` statement of the form ``from ...\nimport *`` binds all names defined in the imported module, except\nthose beginning with an underscore. This form may only be used at the\nmodule level.\n\nA target occurring in a ``del`` statement is also considered bound for\nthis purpose (though the actual semantics are to unbind the name). It\nis illegal to unbind a name that is referenced by an enclosing scope;\nthe compiler will report a ``SyntaxError``.\n\nEach assignment or import statement occurs within a block defined by a\nclass or function definition or at the module level (the top-level\ncode block).\n\nIf a name binding operation occurs anywhere within a code block, all\nuses of the name within the block are treated as references to the\ncurrent block. This can lead to errors when a name is used within a\nblock before it is bound. This rule is subtle. Python lacks\ndeclarations and allows name binding operations to occur anywhere\nwithin a code block. The local variables of a code block can be\ndetermined by scanning the entire text of the block for name binding\noperations.\n\nIf the global statement occurs within a block, all uses of the name\nspecified in the statement refer to the binding of that name in the\ntop-level namespace. Names are resolved in the top-level namespace by\nsearching the global namespace, i.e. the namespace of the module\ncontaining the code block, and the builtins namespace, the namespace\nof the module ``__builtin__``. The global namespace is searched\nfirst. If the name is not found there, the builtins namespace is\nsearched. The global statement must precede all uses of the name.\n\nThe builtins namespace associated with the execution of a code block\nis actually found by looking up the name ``__builtins__`` in its\nglobal namespace; this should be a dictionary or a module (in the\nlatter case the module\'s dictionary is used). By default, when in the\n``__main__`` module, ``__builtins__`` is the built-in module\n``__builtin__`` (note: no \'s\'); when in any other module,\n``__builtins__`` is an alias for the dictionary of the ``__builtin__``\nmodule itself. ``__builtins__`` can be set to a user-created\ndictionary to create a weak form of restricted execution.\n\n**CPython implementation detail:** Users should not touch\n``__builtins__``; it is strictly an implementation detail. Users\nwanting to override values in the builtins namespace should ``import``\nthe ``__builtin__`` (no \'s\') module and modify its attributes\nappropriately.\n\nThe namespace for a module is automatically created the first time a\nmodule is imported. The main module for a script is always called\n``__main__``.\n\nThe ``global`` statement has the same scope as a name binding\noperation in the same block. If the nearest enclosing scope for a\nfree variable contains a global statement, the free variable is\ntreated as a global.\n\nA class definition is an executable statement that may use and define\nnames. These references follow the normal rules for name resolution.\nThe namespace of the class definition becomes the attribute dictionary\nof the class. Names defined at the class scope are not visible in\nmethods.\n\n\nInteraction with dynamic features\n---------------------------------\n\nThere are several cases where Python statements are illegal when used\nin conjunction with nested scopes that contain free variables.\n\nIf a variable is referenced in an enclosing scope, it is illegal to\ndelete the name. An error will be reported at compile time.\n\nIf the wild card form of import --- ``import *`` --- is used in a\nfunction and the function contains or is a nested block with free\nvariables, the compiler will raise a ``SyntaxError``.\n\nIf ``exec`` is used in a function and the function contains or is a\nnested block with free variables, the compiler will raise a\n``SyntaxError`` unless the exec explicitly specifies the local\nnamespace for the ``exec``. (In other words, ``exec obj`` would be\nillegal, but ``exec obj in ns`` would be legal.)\n\nThe ``eval()``, ``execfile()``, and ``input()`` functions and the\n``exec`` statement do not have access to the full environment for\nresolving names. Names may be resolved in the local and global\nnamespaces of the caller. Free variables are not resolved in the\nnearest enclosing namespace, but in the global namespace. [1] The\n``exec`` statement and the ``eval()`` and ``execfile()`` functions\nhave optional arguments to override the global and local namespace.\nIf only one namespace is specified, it is used for both.\n\n\nExceptions\n==========\n\nExceptions are a means of breaking out of the normal flow of control\nof a code block in order to handle errors or other exceptional\nconditions. An exception is *raised* at the point where the error is\ndetected; it may be *handled* by the surrounding code block or by any\ncode block that directly or indirectly invoked the code block where\nthe error occurred.\n\nThe Python interpreter raises an exception when it detects a run-time\nerror (such as division by zero). A Python program can also\nexplicitly raise an exception with the ``raise`` statement. Exception\nhandlers are specified with the ``try`` ... ``except`` statement. The\n``finally`` clause of such a statement can be used to specify cleanup\ncode which does not handle the exception, but is executed whether an\nexception occurred or not in the preceding code.\n\nPython uses the "termination" model of error handling: an exception\nhandler can find out what happened and continue execution at an outer\nlevel, but it cannot repair the cause of the error and retry the\nfailing operation (except by re-entering the offending piece of code\nfrom the top).\n\nWhen an exception is not handled at all, the interpreter terminates\nexecution of the program, or returns to its interactive main loop. In\neither case, it prints a stack backtrace, except when the exception is\n``SystemExit``.\n\nExceptions are identified by class instances. The ``except`` clause\nis selected depending on the class of the instance: it must reference\nthe class of the instance or a base class thereof. The instance can\nbe received by the handler and can carry additional information about\nthe exceptional condition.\n\nExceptions can also be identified by strings, in which case the\n``except`` clause is selected by object identity. An arbitrary value\ncan be raised along with the identifying string which can be passed to\nthe handler.\n\nNote: Messages to exceptions are not part of the Python API. Their\n contents may change from one version of Python to the next without\n warning and should not be relied on by code which will run under\n multiple versions of the interpreter.\n\nSee also the description of the ``try`` statement in section *The try\nstatement* and ``raise`` statement in section *The raise statement*.\n\n-[ Footnotes ]-\n\n[1] This limitation occurs because the code that is executed by these\n operations is not available at the time the module is compiled.\n', 'exprlists': u'\nExpression lists\n****************\n\n expression_list ::= expression ( "," expression )* [","]\n\nAn expression list containing at least one comma yields a tuple. The\nlength of the tuple is the number of expressions in the list. The\nexpressions are evaluated from left to right.\n\nThe trailing comma is required only to create a single tuple (a.k.a. a\n*singleton*); it is optional in all other cases. A single expression\nwithout a trailing comma doesn\'t create a tuple, but rather yields the\nvalue of that expression. (To create an empty tuple, use an empty pair\nof parentheses: ``()``.)\n', 'floating': u'\nFloating point literals\n***********************\n\nFloating point literals are described by the following lexical\ndefinitions:\n\n floatnumber ::= pointfloat | exponentfloat\n pointfloat ::= [intpart] fraction | intpart "."\n exponentfloat ::= (intpart | pointfloat) exponent\n intpart ::= digit+\n fraction ::= "." digit+\n exponent ::= ("e" | "E") ["+" | "-"] digit+\n\nNote that the integer and exponent parts of floating point numbers can\nlook like octal integers, but are interpreted using radix 10. For\nexample, ``077e010`` is legal, and denotes the same number as\n``77e10``. The allowed range of floating point literals is\nimplementation-dependent. Some examples of floating point literals:\n\n 3.14 10. .001 1e100 3.14e-10 0e0\n\nNote that numeric literals do not include a sign; a phrase like ``-1``\nis actually an expression composed of the unary operator ``-`` and the\nliteral ``1``.\n', 'for': u'\nThe ``for`` statement\n*********************\n\nThe ``for`` statement is used to iterate over the elements of a\nsequence (such as a string, tuple or list) or other iterable object:\n\n for_stmt ::= "for" target_list "in" expression_list ":" suite\n ["else" ":" suite]\n\nThe expression list is evaluated once; it should yield an iterable\nobject. An iterator is created for the result of the\n``expression_list``. The suite is then executed once for each item\nprovided by the iterator, in the order of ascending indices. Each\nitem in turn is assigned to the target list using the standard rules\nfor assignments, and then the suite is executed. When the items are\nexhausted (which is immediately when the sequence is empty), the suite\nin the ``else`` clause, if present, is executed, and the loop\nterminates.\n\nA ``break`` statement executed in the first suite terminates the loop\nwithout executing the ``else`` clause\'s suite. A ``continue``\nstatement executed in the first suite skips the rest of the suite and\ncontinues with the next item, or with the ``else`` clause if there was\nno next item.\n\nThe suite may assign to the variable(s) in the target list; this does\nnot affect the next item assigned to it.\n\nThe target list is not deleted when the loop is finished, but if the\nsequence is empty, it will not have been assigned to at all by the\nloop. Hint: the built-in function ``range()`` returns a sequence of\nintegers suitable to emulate the effect of Pascal\'s ``for i := a to b\ndo``; e.g., ``range(3)`` returns the list ``[0, 1, 2]``.\n\nNote: There is a subtlety when the sequence is being modified by the loop\n (this can only occur for mutable sequences, i.e. lists). An internal\n counter is used to keep track of which item is used next, and this\n is incremented on each iteration. When this counter has reached the\n length of the sequence the loop terminates. This means that if the\n suite deletes the current (or a previous) item from the sequence,\n the next item will be skipped (since it gets the index of the\n current item which has already been treated). Likewise, if the\n suite inserts an item in the sequence before the current item, the\n current item will be treated again the next time through the loop.\n This can lead to nasty bugs that can be avoided by making a\n temporary copy using a slice of the whole sequence, e.g.,\n\n for x in a[:]:\n if x < 0: a.remove(x)\n', - 'formatstrings': u'\nFormat String Syntax\n********************\n\nThe ``str.format()`` method and the ``Formatter`` class share the same\nsyntax for format strings (although in the case of ``Formatter``,\nsubclasses can define their own format string syntax).\n\nFormat strings contain "replacement fields" surrounded by curly braces\n``{}``. Anything that is not contained in braces is considered literal\ntext, which is copied unchanged to the output. If you need to include\na brace character in the literal text, it can be escaped by doubling:\n``{{`` and ``}}``.\n\nThe grammar for a replacement field is as follows:\n\n replacement_field ::= "{" [field_name] ["!" conversion] [":" format_spec] "}"\n field_name ::= arg_name ("." attribute_name | "[" element_index "]")*\n arg_name ::= [identifier | integer]\n attribute_name ::= identifier\n element_index ::= integer | index_string\n index_string ::= <any source character except "]"> +\n conversion ::= "r" | "s"\n format_spec ::= <described in the next section>\n\nIn less formal terms, the replacement field can start with a\n*field_name* that specifies the object whose value is to be formatted\nand inserted into the output instead of the replacement field. The\n*field_name* is optionally followed by a *conversion* field, which is\npreceded by an exclamation point ``\'!\'``, and a *format_spec*, which\nis preceded by a colon ``\':\'``. These specify a non-default format\nfor the replacement value.\n\nSee also the *Format Specification Mini-Language* section.\n\nThe *field_name* itself begins with an *arg_name* that is either\neither a number or a keyword. If it\'s a number, it refers to a\npositional argument, and if it\'s a keyword, it refers to a named\nkeyword argument. If the numerical arg_names in a format string are\n0, 1, 2, ... in sequence, they can all be omitted (not just some) and\nthe numbers 0, 1, 2, ... will be automatically inserted in that order.\nThe *arg_name* can be followed by any number of index or attribute\nexpressions. An expression of the form ``\'.name\'`` selects the named\nattribute using ``getattr()``, while an expression of the form\n``\'[index]\'`` does an index lookup using ``__getitem__()``.\n\nChanged in version 2.7: The positional argument specifiers can be\nomitted, so ``\'{} {}\'`` is equivalent to ``\'{0} {1}\'``.\n\nSome simple format string examples:\n\n "First, thou shalt count to {0}" # References first positional argument\n "Bring me a {}" # Implicitly references the first positional argument\n "From {} to {}" # Same as "From {0} to {1}"\n "My quest is {name}" # References keyword argument \'name\'\n "Weight in tons {0.weight}" # \'weight\' attribute of first positional arg\n "Units destroyed: {players[0]}" # First element of keyword argument \'players\'.\n\nThe *conversion* field causes a type coercion before formatting.\nNormally, the job of formatting a value is done by the\n``__format__()`` method of the value itself. However, in some cases\nit is desirable to force a type to be formatted as a string,\noverriding its own definition of formatting. By converting the value\nto a string before calling ``__format__()``, the normal formatting\nlogic is bypassed.\n\nTwo conversion flags are currently supported: ``\'!s\'`` which calls\n``str()`` on the value, and ``\'!r\'`` which calls ``repr()``.\n\nSome examples:\n\n "Harold\'s a clever {0!s}" # Calls str() on the argument first\n "Bring out the holy {name!r}" # Calls repr() on the argument first\n\nThe *format_spec* field contains a specification of how the value\nshould be presented, including such details as field width, alignment,\npadding, decimal precision and so on. Each value type can define its\nown "formatting mini-language" or interpretation of the *format_spec*.\n\nMost built-in types support a common formatting mini-language, which\nis described in the next section.\n\nA *format_spec* field can also include nested replacement fields\nwithin it. These nested replacement fields can contain only a field\nname; conversion flags and format specifications are not allowed. The\nreplacement fields within the format_spec are substituted before the\n*format_spec* string is interpreted. This allows the formatting of a\nvalue to be dynamically specified.\n\nSee the *Format examples* section for some examples.\n\n\nFormat Specification Mini-Language\n==================================\n\n"Format specifications" are used within replacement fields contained\nwithin a format string to define how individual values are presented\n(see *Format String Syntax*). They can also be passed directly to the\nbuilt-in ``format()`` function. Each formattable type may define how\nthe format specification is to be interpreted.\n\nMost built-in types implement the following options for format\nspecifications, although some of the formatting options are only\nsupported by the numeric types.\n\nA general convention is that an empty format string (``""``) produces\nthe same result as if you had called ``str()`` on the value. A non-\nempty format string typically modifies the result.\n\nThe general form of a *standard format specifier* is:\n\n format_spec ::= [[fill]align][sign][#][0][width][,][.precision][type]\n fill ::= <a character other than \'}\'>\n align ::= "<" | ">" | "=" | "^"\n sign ::= "+" | "-" | " "\n width ::= integer\n precision ::= integer\n type ::= "b" | "c" | "d" | "e" | "E" | "f" | "F" | "g" | "G" | "n" | "o" | "s" | "x" | "X" | "%"\n\nThe *fill* character can be any character other than \'}\' (which\nsignifies the end of the field). The presence of a fill character is\nsignaled by the *next* character, which must be one of the alignment\noptions. If the second character of *format_spec* is not a valid\nalignment option, then it is assumed that both the fill character and\nthe alignment option are absent.\n\nThe meaning of the various alignment options is as follows:\n\n +-----------+------------------------------------------------------------+\n | Option | Meaning |\n +===========+============================================================+\n | ``\'<\'`` | Forces the field to be left-aligned within the available |\n | | space (this is the default). |\n +-----------+------------------------------------------------------------+\n | ``\'>\'`` | Forces the field to be right-aligned within the available |\n | | space. |\n +-----------+------------------------------------------------------------+\n | ``\'=\'`` | Forces the padding to be placed after the sign (if any) |\n | | but before the digits. This is used for printing fields |\n | | in the form \'+000000120\'. This alignment option is only |\n | | valid for numeric types. |\n +-----------+------------------------------------------------------------+\n | ``\'^\'`` | Forces the field to be centered within the available |\n | | space. |\n +-----------+------------------------------------------------------------+\n\nNote that unless a minimum field width is defined, the field width\nwill always be the same size as the data to fill it, so that the\nalignment option has no meaning in this case.\n\nThe *sign* option is only valid for number types, and can be one of\nthe following:\n\n +-----------+------------------------------------------------------------+\n | Option | Meaning |\n +===========+============================================================+\n | ``\'+\'`` | indicates that a sign should be used for both positive as |\n | | well as negative numbers. |\n +-----------+------------------------------------------------------------+\n | ``\'-\'`` | indicates that a sign should be used only for negative |\n | | numbers (this is the default behavior). |\n +-----------+------------------------------------------------------------+\n | space | indicates that a leading space should be used on positive |\n | | numbers, and a minus sign on negative numbers. |\n +-----------+------------------------------------------------------------+\n\nThe ``\'#\'`` option is only valid for integers, and only for binary,\noctal, or hexadecimal output. If present, it specifies that the\noutput will be prefixed by ``\'0b\'``, ``\'0o\'``, or ``\'0x\'``,\nrespectively.\n\nThe ``\',\'`` option signals the use of a comma for a thousands\nseparator. For a locale aware separator, use the ``\'n\'`` integer\npresentation type instead.\n\nChanged in version 2.7: Added the ``\',\'`` option (see also **PEP\n378**).\n\n*width* is a decimal integer defining the minimum field width. If not\nspecified, then the field width will be determined by the content.\n\nIf the *width* field is preceded by a zero (``\'0\'``) character, this\nenables zero-padding. This is equivalent to an *alignment* type of\n``\'=\'`` and a *fill* character of ``\'0\'``.\n\nThe *precision* is a decimal number indicating how many digits should\nbe displayed after the decimal point for a floating point value\nformatted with ``\'f\'`` and ``\'F\'``, or before and after the decimal\npoint for a floating point value formatted with ``\'g\'`` or ``\'G\'``.\nFor non-number types the field indicates the maximum field size - in\nother words, how many characters will be used from the field content.\nThe *precision* is not allowed for integer values.\n\nFinally, the *type* determines how the data should be presented.\n\nThe available string presentation types are:\n\n +-----------+------------------------------------------------------------+\n | Type | Meaning |\n +===========+============================================================+\n | ``\'s\'`` | String format. This is the default type for strings and |\n | | may be omitted. |\n +-----------+------------------------------------------------------------+\n | None | The same as ``\'s\'``. |\n +-----------+------------------------------------------------------------+\n\nThe available integer presentation types are:\n\n +-----------+------------------------------------------------------------+\n | Type | Meaning |\n +===========+============================================================+\n | ``\'b\'`` | Binary format. Outputs the number in base 2. |\n +-----------+------------------------------------------------------------+\n | ``\'c\'`` | Character. Converts the integer to the corresponding |\n | | unicode character before printing. |\n +-----------+------------------------------------------------------------+\n | ``\'d\'`` | Decimal Integer. Outputs the number in base 10. |\n +-----------+------------------------------------------------------------+\n | ``\'o\'`` | Octal format. Outputs the number in base 8. |\n +-----------+------------------------------------------------------------+\n | ``\'x\'`` | Hex format. Outputs the number in base 16, using lower- |\n | | case letters for the digits above 9. |\n +-----------+------------------------------------------------------------+\n | ``\'X\'`` | Hex format. Outputs the number in base 16, using upper- |\n | | case letters for the digits above 9. |\n +-----------+------------------------------------------------------------+\n | ``\'n\'`` | Number. This is the same as ``\'d\'``, except that it uses |\n | | the current locale setting to insert the appropriate |\n | | number separator characters. |\n +-----------+------------------------------------------------------------+\n | None | The same as ``\'d\'``. |\n +-----------+------------------------------------------------------------+\n\nIn addition to the above presentation types, integers can be formatted\nwith the floating point presentation types listed below (except\n``\'n\'`` and None). When doing so, ``float()`` is used to convert the\ninteger to a floating point number before formatting.\n\nThe available presentation types for floating point and decimal values\nare:\n\n +-----------+------------------------------------------------------------+\n | Type | Meaning |\n +===========+============================================================+\n | ``\'e\'`` | Exponent notation. Prints the number in scientific |\n | | notation using the letter \'e\' to indicate the exponent. |\n +-----------+------------------------------------------------------------+\n | ``\'E\'`` | Exponent notation. Same as ``\'e\'`` except it uses an upper |\n | | case \'E\' as the separator character. |\n +-----------+------------------------------------------------------------+\n | ``\'f\'`` | Fixed point. Displays the number as a fixed-point number. |\n +-----------+------------------------------------------------------------+\n | ``\'F\'`` | Fixed point. Same as ``\'f\'``. |\n +-----------+------------------------------------------------------------+\n | ``\'g\'`` | General format. For a given precision ``p >= 1``, this |\n | | rounds the number to ``p`` significant digits and then |\n | | formats the result in either fixed-point format or in |\n | | scientific notation, depending on its magnitude. The |\n | | precise rules are as follows: suppose that the result |\n | | formatted with presentation type ``\'e\'`` and precision |\n | | ``p-1`` would have exponent ``exp``. Then if ``-4 <= exp |\n | | < p``, the number is formatted with presentation type |\n | | ``\'f\'`` and precision ``p-1-exp``. Otherwise, the number |\n | | is formatted with presentation type ``\'e\'`` and precision |\n | | ``p-1``. In both cases insignificant trailing zeros are |\n | | removed from the significand, and the decimal point is |\n | | also removed if there are no remaining digits following |\n | | it. Postive and negative infinity, positive and negative |\n | | zero, and nans, are formatted as ``inf``, ``-inf``, ``0``, |\n | | ``-0`` and ``nan`` respectively, regardless of the |\n | | precision. A precision of ``0`` is treated as equivalent |\n | | to a precision of ``1``. |\n +-----------+------------------------------------------------------------+\n | ``\'G\'`` | General format. Same as ``\'g\'`` except switches to ``\'E\'`` |\n | | if the number gets too large. The representations of |\n | | infinity and NaN are uppercased, too. |\n +-----------+------------------------------------------------------------+\n | ``\'n\'`` | Number. This is the same as ``\'g\'``, except that it uses |\n | | the current locale setting to insert the appropriate |\n | | number separator characters. |\n +-----------+------------------------------------------------------------+\n | ``\'%\'`` | Percentage. Multiplies the number by 100 and displays in |\n | | fixed (``\'f\'``) format, followed by a percent sign. |\n +-----------+------------------------------------------------------------+\n | None | The same as ``\'g\'``. |\n +-----------+------------------------------------------------------------+\n\n\nFormat examples\n===============\n\nThis section contains examples of the new format syntax and comparison\nwith the old ``%``-formatting.\n\nIn most of the cases the syntax is similar to the old\n``%``-formatting, with the addition of the ``{}`` and with ``:`` used\ninstead of ``%``. For example, ``\'%03.2f\'`` can be translated to\n``\'{:03.2f}\'``.\n\nThe new format syntax also supports new and different options, shown\nin the follow examples.\n\nAccessing arguments by position:\n\n >>> \'{0}, {1}, {2}\'.format(\'a\', \'b\', \'c\')\n \'a, b, c\'\n >>> \'{}, {}, {}\'.format(\'a\', \'b\', \'c\') # 2.7+ only\n \'a, b, c\'\n >>> \'{2}, {1}, {0}\'.format(\'a\', \'b\', \'c\')\n \'c, b, a\'\n >>> \'{2}, {1}, {0}\'.format(*\'abc\') # unpacking argument sequence\n \'c, b, a\'\n >>> \'{0}{1}{0}\'.format(\'abra\', \'cad\') # arguments\' indices can be repeated\n \'abracadabra\'\n\nAccessing arguments by name:\n\n >>> \'Coordinates: {latitude}, {longitude}\'.format(latitude=\'37.24N\', longitude=\'-115.81W\')\n \'Coordinates: 37.24N, -115.81W\'\n >>> coord = {\'latitude\': \'37.24N\', \'longitude\': \'-115.81W\'}\n >>> \'Coordinates: {latitude}, {longitude}\'.format(**coord)\n \'Coordinates: 37.24N, -115.81W\'\n\nAccessing arguments\' attributes:\n\n >>> c = 3-5j\n >>> (\'The complex number {0} is formed from the real part {0.real} \'\n ... \'and the imaginary part {0.imag}.\').format(c)\n \'The complex number (3-5j) is formed from the real part 3.0 and the imaginary part -5.0.\'\n >>> class Point(object):\n ... def __init__(self, x, y):\n ... self.x, self.y = x, y\n ... def __str__(self):\n ... return \'Point({self.x}, {self.y})\'.format(self=self)\n ...\n >>> str(Point(4, 2))\n \'Point(4, 2)\'\n\nAccessing arguments\' items:\n\n >>> coord = (3, 5)\n >>> \'X: {0[0]}; Y: {0[1]}\'.format(coord)\n \'X: 3; Y: 5\'\n\nReplacing ``%s`` and ``%r``:\n\n >>> "repr() shows quotes: {!r}; str() doesn\'t: {!s}".format(\'test1\', \'test2\')\n "repr() shows quotes: \'test1\'; str() doesn\'t: test2"\n\nAligning the text and specifying a width:\n\n >>> \'{:<30}\'.format(\'left aligned\')\n \'left aligned \'\n >>> \'{:>30}\'.format(\'right aligned\')\n \' right aligned\'\n >>> \'{:^30}\'.format(\'centered\')\n \' centered \'\n >>> \'{:*^30}\'.format(\'centered\') # use \'*\' as a fill char\n \'***********centered***********\'\n\nReplacing ``%+f``, ``%-f``, and ``% f`` and specifying a sign:\n\n >>> \'{:+f}; {:+f}\'.format(3.14, -3.14) # show it always\n \'+3.140000; -3.140000\'\n >>> \'{: f}; {: f}\'.format(3.14, -3.14) # show a space for positive numbers\n \' 3.140000; -3.140000\'\n >>> \'{:-f}; {:-f}\'.format(3.14, -3.14) # show only the minus -- same as \'{:f}; {:f}\'\n \'3.140000; -3.140000\'\n\nReplacing ``%x`` and ``%o`` and converting the value to different\nbases:\n\n >>> # format also supports binary numbers\n >>> "int: {0:d}; hex: {0:x}; oct: {0:o}; bin: {0:b}".format(42)\n \'int: 42; hex: 2a; oct: 52; bin: 101010\'\n >>> # with 0x, 0o, or 0b as prefix:\n >>> "int: {0:d}; hex: {0:#x}; oct: {0:#o}; bin: {0:#b}".format(42)\n \'int: 42; hex: 0x2a; oct: 0o52; bin: 0b101010\'\n\nUsing the comma as a thousands separator:\n\n >>> \'{:,}\'.format(1234567890)\n \'1,234,567,890\'\n\nExpressing a percentage:\n\n >>> points = 19.5\n >>> total = 22\n >>> \'Correct answers: {:.2%}.\'.format(points/total)\n \'Correct answers: 88.64%\'\n\nUsing type-specific formatting:\n\n >>> import datetime\n >>> d = datetime.datetime(2010, 7, 4, 12, 15, 58)\n >>> \'{:%Y-%m-%d %H:%M:%S}\'.format(d)\n \'2010-07-04 12:15:58\'\n\nNesting arguments and more complex examples:\n\n >>> for align, text in zip(\'<^>\', [\'left\', \'center\', \'right\']):\n ... \'{0:{align}{fill}16}\'.format(text, fill=align, align=align)\n ...\n \'left<<<<<<<<<<<<\'\n \'^^^^^center^^^^^\'\n \'>>>>>>>>>>>right\'\n >>>\n >>> octets = [192, 168, 0, 1]\n >>> \'{:02X}{:02X}{:02X}{:02X}\'.format(*octets)\n \'C0A80001\'\n >>> int(_, 16)\n 3232235521\n >>>\n >>> width = 5\n >>> for num in range(5,12):\n ... for base in \'dXob\':\n ... print \'{0:{width}{base}}\'.format(num, base=base, width=width),\n ... print\n ...\n 5 5 5 101\n 6 6 6 110\n 7 7 7 111\n 8 8 10 1000\n 9 9 11 1001\n 10 A 12 1010\n 11 B 13 1011\n', + 'formatstrings': u'\nFormat String Syntax\n********************\n\nThe ``str.format()`` method and the ``Formatter`` class share the same\nsyntax for format strings (although in the case of ``Formatter``,\nsubclasses can define their own format string syntax).\n\nFormat strings contain "replacement fields" surrounded by curly braces\n``{}``. Anything that is not contained in braces is considered literal\ntext, which is copied unchanged to the output. If you need to include\na brace character in the literal text, it can be escaped by doubling:\n``{{`` and ``}}``.\n\nThe grammar for a replacement field is as follows:\n\n replacement_field ::= "{" [field_name] ["!" conversion] [":" format_spec] "}"\n field_name ::= arg_name ("." attribute_name | "[" element_index "]")*\n arg_name ::= [identifier | integer]\n attribute_name ::= identifier\n element_index ::= integer | index_string\n index_string ::= <any source character except "]"> +\n conversion ::= "r" | "s"\n format_spec ::= <described in the next section>\n\nIn less formal terms, the replacement field can start with a\n*field_name* that specifies the object whose value is to be formatted\nand inserted into the output instead of the replacement field. The\n*field_name* is optionally followed by a *conversion* field, which is\npreceded by an exclamation point ``\'!\'``, and a *format_spec*, which\nis preceded by a colon ``\':\'``. These specify a non-default format\nfor the replacement value.\n\nSee also the *Format Specification Mini-Language* section.\n\nThe *field_name* itself begins with an *arg_name* that is either\neither a number or a keyword. If it\'s a number, it refers to a\npositional argument, and if it\'s a keyword, it refers to a named\nkeyword argument. If the numerical arg_names in a format string are\n0, 1, 2, ... in sequence, they can all be omitted (not just some) and\nthe numbers 0, 1, 2, ... will be automatically inserted in that order.\nThe *arg_name* can be followed by any number of index or attribute\nexpressions. An expression of the form ``\'.name\'`` selects the named\nattribute using ``getattr()``, while an expression of the form\n``\'[index]\'`` does an index lookup using ``__getitem__()``.\n\nChanged in version 2.7: The positional argument specifiers can be\nomitted, so ``\'{} {}\'`` is equivalent to ``\'{0} {1}\'``.\n\nSome simple format string examples:\n\n "First, thou shalt count to {0}" # References first positional argument\n "Bring me a {}" # Implicitly references the first positional argument\n "From {} to {}" # Same as "From {0} to {1}"\n "My quest is {name}" # References keyword argument \'name\'\n "Weight in tons {0.weight}" # \'weight\' attribute of first positional arg\n "Units destroyed: {players[0]}" # First element of keyword argument \'players\'.\n\nThe *conversion* field causes a type coercion before formatting.\nNormally, the job of formatting a value is done by the\n``__format__()`` method of the value itself. However, in some cases\nit is desirable to force a type to be formatted as a string,\noverriding its own definition of formatting. By converting the value\nto a string before calling ``__format__()``, the normal formatting\nlogic is bypassed.\n\nTwo conversion flags are currently supported: ``\'!s\'`` which calls\n``str()`` on the value, and ``\'!r\'`` which calls ``repr()``.\n\nSome examples:\n\n "Harold\'s a clever {0!s}" # Calls str() on the argument first\n "Bring out the holy {name!r}" # Calls repr() on the argument first\n\nThe *format_spec* field contains a specification of how the value\nshould be presented, including such details as field width, alignment,\npadding, decimal precision and so on. Each value type can define its\nown "formatting mini-language" or interpretation of the *format_spec*.\n\nMost built-in types support a common formatting mini-language, which\nis described in the next section.\n\nA *format_spec* field can also include nested replacement fields\nwithin it. These nested replacement fields can contain only a field\nname; conversion flags and format specifications are not allowed. The\nreplacement fields within the format_spec are substituted before the\n*format_spec* string is interpreted. This allows the formatting of a\nvalue to be dynamically specified.\n\nSee the *Format examples* section for some examples.\n\n\nFormat Specification Mini-Language\n==================================\n\n"Format specifications" are used within replacement fields contained\nwithin a format string to define how individual values are presented\n(see *Format String Syntax*). They can also be passed directly to the\nbuilt-in ``format()`` function. Each formattable type may define how\nthe format specification is to be interpreted.\n\nMost built-in types implement the following options for format\nspecifications, although some of the formatting options are only\nsupported by the numeric types.\n\nA general convention is that an empty format string (``""``) produces\nthe same result as if you had called ``str()`` on the value. A non-\nempty format string typically modifies the result.\n\nThe general form of a *standard format specifier* is:\n\n format_spec ::= [[fill]align][sign][#][0][width][,][.precision][type]\n fill ::= <a character other than \'}\'>\n align ::= "<" | ">" | "=" | "^"\n sign ::= "+" | "-" | " "\n width ::= integer\n precision ::= integer\n type ::= "b" | "c" | "d" | "e" | "E" | "f" | "F" | "g" | "G" | "n" | "o" | "s" | "x" | "X" | "%"\n\nThe *fill* character can be any character other than \'{\' or \'}\'. The\npresence of a fill character is signaled by the character following\nit, which must be one of the alignment options. If the second\ncharacter of *format_spec* is not a valid alignment option, then it is\nassumed that both the fill character and the alignment option are\nabsent.\n\nThe meaning of the various alignment options is as follows:\n\n +-----------+------------------------------------------------------------+\n | Option | Meaning |\n +===========+============================================================+\n | ``\'<\'`` | Forces the field to be left-aligned within the available |\n | | space (this is the default for most objects). |\n +-----------+------------------------------------------------------------+\n | ``\'>\'`` | Forces the field to be right-aligned within the available |\n | | space (this is the default for numbers). |\n +-----------+------------------------------------------------------------+\n | ``\'=\'`` | Forces the padding to be placed after the sign (if any) |\n | | but before the digits. This is used for printing fields |\n | | in the form \'+000000120\'. This alignment option is only |\n | | valid for numeric types. |\n +-----------+------------------------------------------------------------+\n | ``\'^\'`` | Forces the field to be centered within the available |\n | | space. |\n +-----------+------------------------------------------------------------+\n\nNote that unless a minimum field width is defined, the field width\nwill always be the same size as the data to fill it, so that the\nalignment option has no meaning in this case.\n\nThe *sign* option is only valid for number types, and can be one of\nthe following:\n\n +-----------+------------------------------------------------------------+\n | Option | Meaning |\n +===========+============================================================+\n | ``\'+\'`` | indicates that a sign should be used for both positive as |\n | | well as negative numbers. |\n +-----------+------------------------------------------------------------+\n | ``\'-\'`` | indicates that a sign should be used only for negative |\n | | numbers (this is the default behavior). |\n +-----------+------------------------------------------------------------+\n | space | indicates that a leading space should be used on positive |\n | | numbers, and a minus sign on negative numbers. |\n +-----------+------------------------------------------------------------+\n\nThe ``\'#\'`` option is only valid for integers, and only for binary,\noctal, or hexadecimal output. If present, it specifies that the\noutput will be prefixed by ``\'0b\'``, ``\'0o\'``, or ``\'0x\'``,\nrespectively.\n\nThe ``\',\'`` option signals the use of a comma for a thousands\nseparator. For a locale aware separator, use the ``\'n\'`` integer\npresentation type instead.\n\nChanged in version 2.7: Added the ``\',\'`` option (see also **PEP\n378**).\n\n*width* is a decimal integer defining the minimum field width. If not\nspecified, then the field width will be determined by the content.\n\nIf the *width* field is preceded by a zero (``\'0\'``) character, this\nenables zero-padding. This is equivalent to an *alignment* type of\n``\'=\'`` and a *fill* character of ``\'0\'``.\n\nThe *precision* is a decimal number indicating how many digits should\nbe displayed after the decimal point for a floating point value\nformatted with ``\'f\'`` and ``\'F\'``, or before and after the decimal\npoint for a floating point value formatted with ``\'g\'`` or ``\'G\'``.\nFor non-number types the field indicates the maximum field size - in\nother words, how many characters will be used from the field content.\nThe *precision* is not allowed for integer values.\n\nFinally, the *type* determines how the data should be presented.\n\nThe available string presentation types are:\n\n +-----------+------------------------------------------------------------+\n | Type | Meaning |\n +===========+============================================================+\n | ``\'s\'`` | String format. This is the default type for strings and |\n | | may be omitted. |\n +-----------+------------------------------------------------------------+\n | None | The same as ``\'s\'``. |\n +-----------+------------------------------------------------------------+\n\nThe available integer presentation types are:\n\n +-----------+------------------------------------------------------------+\n | Type | Meaning |\n +===========+============================================================+\n | ``\'b\'`` | Binary format. Outputs the number in base 2. |\n +-----------+------------------------------------------------------------+\n | ``\'c\'`` | Character. Converts the integer to the corresponding |\n | | unicode character before printing. |\n +-----------+------------------------------------------------------------+\n | ``\'d\'`` | Decimal Integer. Outputs the number in base 10. |\n +-----------+------------------------------------------------------------+\n | ``\'o\'`` | Octal format. Outputs the number in base 8. |\n +-----------+------------------------------------------------------------+\n | ``\'x\'`` | Hex format. Outputs the number in base 16, using lower- |\n | | case letters for the digits above 9. |\n +-----------+------------------------------------------------------------+\n | ``\'X\'`` | Hex format. Outputs the number in base 16, using upper- |\n | | case letters for the digits above 9. |\n +-----------+------------------------------------------------------------+\n | ``\'n\'`` | Number. This is the same as ``\'d\'``, except that it uses |\n | | the current locale setting to insert the appropriate |\n | | number separator characters. |\n +-----------+------------------------------------------------------------+\n | None | The same as ``\'d\'``. |\n +-----------+------------------------------------------------------------+\n\nIn addition to the above presentation types, integers can be formatted\nwith the floating point presentation types listed below (except\n``\'n\'`` and None). When doing so, ``float()`` is used to convert the\ninteger to a floating point number before formatting.\n\nThe available presentation types for floating point and decimal values\nare:\n\n +-----------+------------------------------------------------------------+\n | Type | Meaning |\n +===========+============================================================+\n | ``\'e\'`` | Exponent notation. Prints the number in scientific |\n | | notation using the letter \'e\' to indicate the exponent. |\n +-----------+------------------------------------------------------------+\n | ``\'E\'`` | Exponent notation. Same as ``\'e\'`` except it uses an upper |\n | | case \'E\' as the separator character. |\n +-----------+------------------------------------------------------------+\n | ``\'f\'`` | Fixed point. Displays the number as a fixed-point number. |\n +-----------+------------------------------------------------------------+\n | ``\'F\'`` | Fixed point. Same as ``\'f\'``. |\n +-----------+------------------------------------------------------------+\n | ``\'g\'`` | General format. For a given precision ``p >= 1``, this |\n | | rounds the number to ``p`` significant digits and then |\n | | formats the result in either fixed-point format or in |\n | | scientific notation, depending on its magnitude. The |\n | | precise rules are as follows: suppose that the result |\n | | formatted with presentation type ``\'e\'`` and precision |\n | | ``p-1`` would have exponent ``exp``. Then if ``-4 <= exp |\n | | < p``, the number is formatted with presentation type |\n | | ``\'f\'`` and precision ``p-1-exp``. Otherwise, the number |\n | | is formatted with presentation type ``\'e\'`` and precision |\n | | ``p-1``. In both cases insignificant trailing zeros are |\n | | removed from the significand, and the decimal point is |\n | | also removed if there are no remaining digits following |\n | | it. Positive and negative infinity, positive and negative |\n | | zero, and nans, are formatted as ``inf``, ``-inf``, ``0``, |\n | | ``-0`` and ``nan`` respectively, regardless of the |\n | | precision. A precision of ``0`` is treated as equivalent |\n | | to a precision of ``1``. |\n +-----------+------------------------------------------------------------+\n | ``\'G\'`` | General format. Same as ``\'g\'`` except switches to ``\'E\'`` |\n | | if the number gets too large. The representations of |\n | | infinity and NaN are uppercased, too. |\n +-----------+------------------------------------------------------------+\n | ``\'n\'`` | Number. This is the same as ``\'g\'``, except that it uses |\n | | the current locale setting to insert the appropriate |\n | | number separator characters. |\n +-----------+------------------------------------------------------------+\n | ``\'%\'`` | Percentage. Multiplies the number by 100 and displays in |\n | | fixed (``\'f\'``) format, followed by a percent sign. |\n +-----------+------------------------------------------------------------+\n | None | The same as ``\'g\'``. |\n +-----------+------------------------------------------------------------+\n\n\nFormat examples\n===============\n\nThis section contains examples of the new format syntax and comparison\nwith the old ``%``-formatting.\n\nIn most of the cases the syntax is similar to the old\n``%``-formatting, with the addition of the ``{}`` and with ``:`` used\ninstead of ``%``. For example, ``\'%03.2f\'`` can be translated to\n``\'{:03.2f}\'``.\n\nThe new format syntax also supports new and different options, shown\nin the follow examples.\n\nAccessing arguments by position:\n\n >>> \'{0}, {1}, {2}\'.format(\'a\', \'b\', \'c\')\n \'a, b, c\'\n >>> \'{}, {}, {}\'.format(\'a\', \'b\', \'c\') # 2.7+ only\n \'a, b, c\'\n >>> \'{2}, {1}, {0}\'.format(\'a\', \'b\', \'c\')\n \'c, b, a\'\n >>> \'{2}, {1}, {0}\'.format(*\'abc\') # unpacking argument sequence\n \'c, b, a\'\n >>> \'{0}{1}{0}\'.format(\'abra\', \'cad\') # arguments\' indices can be repeated\n \'abracadabra\'\n\nAccessing arguments by name:\n\n >>> \'Coordinates: {latitude}, {longitude}\'.format(latitude=\'37.24N\', longitude=\'-115.81W\')\n \'Coordinates: 37.24N, -115.81W\'\n >>> coord = {\'latitude\': \'37.24N\', \'longitude\': \'-115.81W\'}\n >>> \'Coordinates: {latitude}, {longitude}\'.format(**coord)\n \'Coordinates: 37.24N, -115.81W\'\n\nAccessing arguments\' attributes:\n\n >>> c = 3-5j\n >>> (\'The complex number {0} is formed from the real part {0.real} \'\n ... \'and the imaginary part {0.imag}.\').format(c)\n \'The complex number (3-5j) is formed from the real part 3.0 and the imaginary part -5.0.\'\n >>> class Point(object):\n ... def __init__(self, x, y):\n ... self.x, self.y = x, y\n ... def __str__(self):\n ... return \'Point({self.x}, {self.y})\'.format(self=self)\n ...\n >>> str(Point(4, 2))\n \'Point(4, 2)\'\n\nAccessing arguments\' items:\n\n >>> coord = (3, 5)\n >>> \'X: {0[0]}; Y: {0[1]}\'.format(coord)\n \'X: 3; Y: 5\'\n\nReplacing ``%s`` and ``%r``:\n\n >>> "repr() shows quotes: {!r}; str() doesn\'t: {!s}".format(\'test1\', \'test2\')\n "repr() shows quotes: \'test1\'; str() doesn\'t: test2"\n\nAligning the text and specifying a width:\n\n >>> \'{:<30}\'.format(\'left aligned\')\n \'left aligned \'\n >>> \'{:>30}\'.format(\'right aligned\')\n \' right aligned\'\n >>> \'{:^30}\'.format(\'centered\')\n \' centered \'\n >>> \'{:*^30}\'.format(\'centered\') # use \'*\' as a fill char\n \'***********centered***********\'\n\nReplacing ``%+f``, ``%-f``, and ``% f`` and specifying a sign:\n\n >>> \'{:+f}; {:+f}\'.format(3.14, -3.14) # show it always\n \'+3.140000; -3.140000\'\n >>> \'{: f}; {: f}\'.format(3.14, -3.14) # show a space for positive numbers\n \' 3.140000; -3.140000\'\n >>> \'{:-f}; {:-f}\'.format(3.14, -3.14) # show only the minus -- same as \'{:f}; {:f}\'\n \'3.140000; -3.140000\'\n\nReplacing ``%x`` and ``%o`` and converting the value to different\nbases:\n\n >>> # format also supports binary numbers\n >>> "int: {0:d}; hex: {0:x}; oct: {0:o}; bin: {0:b}".format(42)\n \'int: 42; hex: 2a; oct: 52; bin: 101010\'\n >>> # with 0x, 0o, or 0b as prefix:\n >>> "int: {0:d}; hex: {0:#x}; oct: {0:#o}; bin: {0:#b}".format(42)\n \'int: 42; hex: 0x2a; oct: 0o52; bin: 0b101010\'\n\nUsing the comma as a thousands separator:\n\n >>> \'{:,}\'.format(1234567890)\n \'1,234,567,890\'\n\nExpressing a percentage:\n\n >>> points = 19.5\n >>> total = 22\n >>> \'Correct answers: {:.2%}.\'.format(points/total)\n \'Correct answers: 88.64%\'\n\nUsing type-specific formatting:\n\n >>> import datetime\n >>> d = datetime.datetime(2010, 7, 4, 12, 15, 58)\n >>> \'{:%Y-%m-%d %H:%M:%S}\'.format(d)\n \'2010-07-04 12:15:58\'\n\nNesting arguments and more complex examples:\n\n >>> for align, text in zip(\'<^>\', [\'left\', \'center\', \'right\']):\n ... \'{0:{fill}{align}16}\'.format(text, fill=align, align=align)\n ...\n \'left<<<<<<<<<<<<\'\n \'^^^^^center^^^^^\'\n \'>>>>>>>>>>>right\'\n >>>\n >>> octets = [192, 168, 0, 1]\n >>> \'{:02X}{:02X}{:02X}{:02X}\'.format(*octets)\n \'C0A80001\'\n >>> int(_, 16)\n 3232235521\n >>>\n >>> width = 5\n >>> for num in range(5,12):\n ... for base in \'dXob\':\n ... print \'{0:{width}{base}}\'.format(num, base=base, width=width),\n ... print\n ...\n 5 5 5 101\n 6 6 6 110\n 7 7 7 111\n 8 8 10 1000\n 9 9 11 1001\n 10 A 12 1010\n 11 B 13 1011\n', 'function': u'\nFunction definitions\n********************\n\nA function definition defines a user-defined function object (see\nsection *The standard type hierarchy*):\n\n decorated ::= decorators (classdef | funcdef)\n decorators ::= decorator+\n decorator ::= "@" dotted_name ["(" [argument_list [","]] ")"] NEWLINE\n funcdef ::= "def" funcname "(" [parameter_list] ")" ":" suite\n dotted_name ::= identifier ("." identifier)*\n parameter_list ::= (defparameter ",")*\n ( "*" identifier [, "**" identifier]\n | "**" identifier\n | defparameter [","] )\n defparameter ::= parameter ["=" expression]\n sublist ::= parameter ("," parameter)* [","]\n parameter ::= identifier | "(" sublist ")"\n funcname ::= identifier\n\nA function definition is an executable statement. Its execution binds\nthe function name in the current local namespace to a function object\n(a wrapper around the executable code for the function). This\nfunction object contains a reference to the current global namespace\nas the global namespace to be used when the function is called.\n\nThe function definition does not execute the function body; this gets\nexecuted only when the function is called. [3]\n\nA function definition may be wrapped by one or more *decorator*\nexpressions. Decorator expressions are evaluated when the function is\ndefined, in the scope that contains the function definition. The\nresult must be a callable, which is invoked with the function object\nas the only argument. The returned value is bound to the function name\ninstead of the function object. Multiple decorators are applied in\nnested fashion. For example, the following code:\n\n @f1(arg)\n @f2\n def func(): pass\n\nis equivalent to:\n\n def func(): pass\n func = f1(arg)(f2(func))\n\nWhen one or more top-level parameters have the form *parameter* ``=``\n*expression*, the function is said to have "default parameter values."\nFor a parameter with a default value, the corresponding argument may\nbe omitted from a call, in which case the parameter\'s default value is\nsubstituted. If a parameter has a default value, all following\nparameters must also have a default value --- this is a syntactic\nrestriction that is not expressed by the grammar.\n\n**Default parameter values are evaluated when the function definition\nis executed.** This means that the expression is evaluated once, when\nthe function is defined, and that that same "pre-computed" value is\nused for each call. This is especially important to understand when a\ndefault parameter is a mutable object, such as a list or a dictionary:\nif the function modifies the object (e.g. by appending an item to a\nlist), the default value is in effect modified. This is generally not\nwhat was intended. A way around this is to use ``None`` as the\ndefault, and explicitly test for it in the body of the function, e.g.:\n\n def whats_on_the_telly(penguin=None):\n if penguin is None:\n penguin = []\n penguin.append("property of the zoo")\n return penguin\n\nFunction call semantics are described in more detail in section\n*Calls*. A function call always assigns values to all parameters\nmentioned in the parameter list, either from position arguments, from\nkeyword arguments, or from default values. If the form\n"``*identifier``" is present, it is initialized to a tuple receiving\nany excess positional parameters, defaulting to the empty tuple. If\nthe form "``**identifier``" is present, it is initialized to a new\ndictionary receiving any excess keyword arguments, defaulting to a new\nempty dictionary.\n\nIt is also possible to create anonymous functions (functions not bound\nto a name), for immediate use in expressions. This uses lambda forms,\ndescribed in section *Lambdas*. Note that the lambda form is merely a\nshorthand for a simplified function definition; a function defined in\na "``def``" statement can be passed around or assigned to another name\njust like a function defined by a lambda form. The "``def``" form is\nactually more powerful since it allows the execution of multiple\nstatements.\n\n**Programmer\'s note:** Functions are first-class objects. A "``def``"\nform executed inside a function definition defines a local function\nthat can be returned or passed around. Free variables used in the\nnested function can access the local variables of the function\ncontaining the def. See section *Naming and binding* for details.\n', 'global': u'\nThe ``global`` statement\n************************\n\n global_stmt ::= "global" identifier ("," identifier)*\n\nThe ``global`` statement is a declaration which holds for the entire\ncurrent code block. It means that the listed identifiers are to be\ninterpreted as globals. It would be impossible to assign to a global\nvariable without ``global``, although free variables may refer to\nglobals without being declared global.\n\nNames listed in a ``global`` statement must not be used in the same\ncode block textually preceding that ``global`` statement.\n\nNames listed in a ``global`` statement must not be defined as formal\nparameters or in a ``for`` loop control target, ``class`` definition,\nfunction definition, or ``import`` statement.\n\n**CPython implementation detail:** The current implementation does not\nenforce the latter two restrictions, but programs should not abuse\nthis freedom, as future implementations may enforce them or silently\nchange the meaning of the program.\n\n**Programmer\'s note:** the ``global`` is a directive to the parser.\nIt applies only to code parsed at the same time as the ``global``\nstatement. In particular, a ``global`` statement contained in an\n``exec`` statement does not affect the code block *containing* the\n``exec`` statement, and code contained in an ``exec`` statement is\nunaffected by ``global`` statements in the code containing the\n``exec`` statement. The same applies to the ``eval()``,\n``execfile()`` and ``compile()`` functions.\n', - 'id-classes': u'\nReserved classes of identifiers\n*******************************\n\nCertain classes of identifiers (besides keywords) have special\nmeanings. These classes are identified by the patterns of leading and\ntrailing underscore characters:\n\n``_*``\n Not imported by ``from module import *``. The special identifier\n ``_`` is used in the interactive interpreter to store the result of\n the last evaluation; it is stored in the ``__builtin__`` module.\n When not in interactive mode, ``_`` has no special meaning and is\n not defined. See section *The import statement*.\n\n Note: The name ``_`` is often used in conjunction with\n internationalization; refer to the documentation for the\n ``gettext`` module for more information on this convention.\n\n``__*__``\n System-defined names. These names are defined by the interpreter\n and its implementation (including the standard library);\n applications should not expect to define additional names using\n this convention. The set of names of this class defined by Python\n may be extended in future versions. See section *Special method\n names*.\n\n``__*``\n Class-private names. Names in this category, when used within the\n context of a class definition, are re-written to use a mangled form\n to help avoid name clashes between "private" attributes of base and\n derived classes. See section *Identifiers (Names)*.\n', - 'identifiers': u'\nIdentifiers and keywords\n************************\n\nIdentifiers (also referred to as *names*) are described by the\nfollowing lexical definitions:\n\n identifier ::= (letter|"_") (letter | digit | "_")*\n letter ::= lowercase | uppercase\n lowercase ::= "a"..."z"\n uppercase ::= "A"..."Z"\n digit ::= "0"..."9"\n\nIdentifiers are unlimited in length. Case is significant.\n\n\nKeywords\n========\n\nThe following identifiers are used as reserved words, or *keywords* of\nthe language, and cannot be used as ordinary identifiers. They must\nbe spelled exactly as written here:\n\n and del from not while\n as elif global or with\n assert else if pass yield\n break except import print\n class exec in raise\n continue finally is return\n def for lambda try\n\nChanged in version 2.4: ``None`` became a constant and is now\nrecognized by the compiler as a name for the built-in object ``None``.\nAlthough it is not a keyword, you cannot assign a different object to\nit.\n\nChanged in version 2.5: Both ``as`` and ``with`` are only recognized\nwhen the ``with_statement`` future feature has been enabled. It will\nalways be enabled in Python 2.6. See section *The with statement* for\ndetails. Note that using ``as`` and ``with`` as identifiers will\nalways issue a warning, even when the ``with_statement`` future\ndirective is not in effect.\n\n\nReserved classes of identifiers\n===============================\n\nCertain classes of identifiers (besides keywords) have special\nmeanings. These classes are identified by the patterns of leading and\ntrailing underscore characters:\n\n``_*``\n Not imported by ``from module import *``. The special identifier\n ``_`` is used in the interactive interpreter to store the result of\n the last evaluation; it is stored in the ``__builtin__`` module.\n When not in interactive mode, ``_`` has no special meaning and is\n not defined. See section *The import statement*.\n\n Note: The name ``_`` is often used in conjunction with\n internationalization; refer to the documentation for the\n ``gettext`` module for more information on this convention.\n\n``__*__``\n System-defined names. These names are defined by the interpreter\n and its implementation (including the standard library);\n applications should not expect to define additional names using\n this convention. The set of names of this class defined by Python\n may be extended in future versions. See section *Special method\n names*.\n\n``__*``\n Class-private names. Names in this category, when used within the\n context of a class definition, are re-written to use a mangled form\n to help avoid name clashes between "private" attributes of base and\n derived classes. See section *Identifiers (Names)*.\n', + 'id-classes': u'\nReserved classes of identifiers\n*******************************\n\nCertain classes of identifiers (besides keywords) have special\nmeanings. These classes are identified by the patterns of leading and\ntrailing underscore characters:\n\n``_*``\n Not imported by ``from module import *``. The special identifier\n ``_`` is used in the interactive interpreter to store the result of\n the last evaluation; it is stored in the ``__builtin__`` module.\n When not in interactive mode, ``_`` has no special meaning and is\n not defined. See section *The import statement*.\n\n Note: The name ``_`` is often used in conjunction with\n internationalization; refer to the documentation for the\n ``gettext`` module for more information on this convention.\n\n``__*__``\n System-defined names. These names are defined by the interpreter\n and its implementation (including the standard library). Current\n system names are discussed in the *Special method names* section\n and elsewhere. More will likely be defined in future versions of\n Python. *Any* use of ``__*__`` names, in any context, that does\n not follow explicitly documented use, is subject to breakage\n without warning.\n\n``__*``\n Class-private names. Names in this category, when used within the\n context of a class definition, are re-written to use a mangled form\n to help avoid name clashes between "private" attributes of base and\n derived classes. See section *Identifiers (Names)*.\n', + 'identifiers': u'\nIdentifiers and keywords\n************************\n\nIdentifiers (also referred to as *names*) are described by the\nfollowing lexical definitions:\n\n identifier ::= (letter|"_") (letter | digit | "_")*\n letter ::= lowercase | uppercase\n lowercase ::= "a"..."z"\n uppercase ::= "A"..."Z"\n digit ::= "0"..."9"\n\nIdentifiers are unlimited in length. Case is significant.\n\n\nKeywords\n========\n\nThe following identifiers are used as reserved words, or *keywords* of\nthe language, and cannot be used as ordinary identifiers. They must\nbe spelled exactly as written here:\n\n and del from not while\n as elif global or with\n assert else if pass yield\n break except import print\n class exec in raise\n continue finally is return\n def for lambda try\n\nChanged in version 2.4: ``None`` became a constant and is now\nrecognized by the compiler as a name for the built-in object ``None``.\nAlthough it is not a keyword, you cannot assign a different object to\nit.\n\nChanged in version 2.5: Both ``as`` and ``with`` are only recognized\nwhen the ``with_statement`` future feature has been enabled. It will\nalways be enabled in Python 2.6. See section *The with statement* for\ndetails. Note that using ``as`` and ``with`` as identifiers will\nalways issue a warning, even when the ``with_statement`` future\ndirective is not in effect.\n\n\nReserved classes of identifiers\n===============================\n\nCertain classes of identifiers (besides keywords) have special\nmeanings. These classes are identified by the patterns of leading and\ntrailing underscore characters:\n\n``_*``\n Not imported by ``from module import *``. The special identifier\n ``_`` is used in the interactive interpreter to store the result of\n the last evaluation; it is stored in the ``__builtin__`` module.\n When not in interactive mode, ``_`` has no special meaning and is\n not defined. See section *The import statement*.\n\n Note: The name ``_`` is often used in conjunction with\n internationalization; refer to the documentation for the\n ``gettext`` module for more information on this convention.\n\n``__*__``\n System-defined names. These names are defined by the interpreter\n and its implementation (including the standard library). Current\n system names are discussed in the *Special method names* section\n and elsewhere. More will likely be defined in future versions of\n Python. *Any* use of ``__*__`` names, in any context, that does\n not follow explicitly documented use, is subject to breakage\n without warning.\n\n``__*``\n Class-private names. Names in this category, when used within the\n context of a class definition, are re-written to use a mangled form\n to help avoid name clashes between "private" attributes of base and\n derived classes. See section *Identifiers (Names)*.\n', 'if': u'\nThe ``if`` statement\n********************\n\nThe ``if`` statement is used for conditional execution:\n\n if_stmt ::= "if" expression ":" suite\n ( "elif" expression ":" suite )*\n ["else" ":" suite]\n\nIt selects exactly one of the suites by evaluating the expressions one\nby one until one is found to be true (see section *Boolean operations*\nfor the definition of true and false); then that suite is executed\n(and no other part of the ``if`` statement is executed or evaluated).\nIf all expressions are false, the suite of the ``else`` clause, if\npresent, is executed.\n', 'imaginary': u'\nImaginary literals\n******************\n\nImaginary literals are described by the following lexical definitions:\n\n imagnumber ::= (floatnumber | intpart) ("j" | "J")\n\nAn imaginary literal yields a complex number with a real part of 0.0.\nComplex numbers are represented as a pair of floating point numbers\nand have the same restrictions on their range. To create a complex\nnumber with a nonzero real part, add a floating point number to it,\ne.g., ``(3+4j)``. Some examples of imaginary literals:\n\n 3.14j 10.j 10j .001j 1e100j 3.14e-10j\n', - 'import': u'\nThe ``import`` statement\n************************\n\n import_stmt ::= "import" module ["as" name] ( "," module ["as" name] )*\n | "from" relative_module "import" identifier ["as" name]\n ( "," identifier ["as" name] )*\n | "from" relative_module "import" "(" identifier ["as" name]\n ( "," identifier ["as" name] )* [","] ")"\n | "from" module "import" "*"\n module ::= (identifier ".")* identifier\n relative_module ::= "."* module | "."+\n name ::= identifier\n\nImport statements are executed in two steps: (1) find a module, and\ninitialize it if necessary; (2) define a name or names in the local\nnamespace (of the scope where the ``import`` statement occurs). The\nstatement comes in two forms differing on whether it uses the ``from``\nkeyword. The first form (without ``from``) repeats these steps for\neach identifier in the list. The form with ``from`` performs step (1)\nonce, and then performs step (2) repeatedly.\n\nTo understand how step (1) occurs, one must first understand how\nPython handles hierarchical naming of modules. To help organize\nmodules and provide a hierarchy in naming, Python has a concept of\npackages. A package can contain other packages and modules while\nmodules cannot contain other modules or packages. From a file system\nperspective, packages are directories and modules are files. The\noriginal specification for packages is still available to read,\nalthough minor details have changed since the writing of that\ndocument.\n\nOnce the name of the module is known (unless otherwise specified, the\nterm "module" will refer to both packages and modules), searching for\nthe module or package can begin. The first place checked is\n``sys.modules``, the cache of all modules that have been imported\npreviously. If the module is found there then it is used in step (2)\nof import.\n\nIf the module is not found in the cache, then ``sys.meta_path`` is\nsearched (the specification for ``sys.meta_path`` can be found in\n**PEP 302**). The object is a list of *finder* objects which are\nqueried in order as to whether they know how to load the module by\ncalling their ``find_module()`` method with the name of the module. If\nthe module happens to be contained within a package (as denoted by the\nexistence of a dot in the name), then a second argument to\n``find_module()`` is given as the value of the ``__path__`` attribute\nfrom the parent package (everything up to the last dot in the name of\nthe module being imported). If a finder can find the module it returns\na *loader* (discussed later) or returns ``None``.\n\nIf none of the finders on ``sys.meta_path`` are able to find the\nmodule then some implicitly defined finders are queried.\nImplementations of Python vary in what implicit meta path finders are\ndefined. The one they all do define, though, is one that handles\n``sys.path_hooks``, ``sys.path_importer_cache``, and ``sys.path``.\n\nThe implicit finder searches for the requested module in the "paths"\nspecified in one of two places ("paths" do not have to be file system\npaths). If the module being imported is supposed to be contained\nwithin a package then the second argument passed to ``find_module()``,\n``__path__`` on the parent package, is used as the source of paths. If\nthe module is not contained in a package then ``sys.path`` is used as\nthe source of paths.\n\nOnce the source of paths is chosen it is iterated over to find a\nfinder that can handle that path. The dict at\n``sys.path_importer_cache`` caches finders for paths and is checked\nfor a finder. If the path does not have a finder cached then\n``sys.path_hooks`` is searched by calling each object in the list with\na single argument of the path, returning a finder or raises\n``ImportError``. If a finder is returned then it is cached in\n``sys.path_importer_cache`` and then used for that path entry. If no\nfinder can be found but the path exists then a value of ``None`` is\nstored in ``sys.path_importer_cache`` to signify that an implicit,\nfile-based finder that handles modules stored as individual files\nshould be used for that path. If the path does not exist then a finder\nwhich always returns ``None`` is placed in the cache for the path.\n\nIf no finder can find the module then ``ImportError`` is raised.\nOtherwise some finder returned a loader whose ``load_module()`` method\nis called with the name of the module to load (see **PEP 302** for the\noriginal definition of loaders). A loader has several responsibilities\nto perform on a module it loads. First, if the module already exists\nin ``sys.modules`` (a possibility if the loader is called outside of\nthe import machinery) then it is to use that module for initialization\nand not a new module. But if the module does not exist in\n``sys.modules`` then it is to be added to that dict before\ninitialization begins. If an error occurs during loading of the module\nand it was added to ``sys.modules`` it is to be removed from the dict.\nIf an error occurs but the module was already in ``sys.modules`` it is\nleft in the dict.\n\nThe loader must set several attributes on the module. ``__name__`` is\nto be set to the name of the module. ``__file__`` is to be the "path"\nto the file unless the module is built-in (and thus listed in\n``sys.builtin_module_names``) in which case the attribute is not set.\nIf what is being imported is a package then ``__path__`` is to be set\nto a list of paths to be searched when looking for modules and\npackages contained within the package being imported. ``__package__``\nis optional but should be set to the name of package that contains the\nmodule or package (the empty string is used for module not contained\nin a package). ``__loader__`` is also optional but should be set to\nthe loader object that is loading the module.\n\nIf an error occurs during loading then the loader raises\n``ImportError`` if some other exception is not already being\npropagated. Otherwise the loader returns the module that was loaded\nand initialized.\n\nWhen step (1) finishes without raising an exception, step (2) can\nbegin.\n\nThe first form of ``import`` statement binds the module name in the\nlocal namespace to the module object, and then goes on to import the\nnext identifier, if any. If the module name is followed by ``as``,\nthe name following ``as`` is used as the local name for the module.\n\nThe ``from`` form does not bind the module name: it goes through the\nlist of identifiers, looks each one of them up in the module found in\nstep (1), and binds the name in the local namespace to the object thus\nfound. As with the first form of ``import``, an alternate local name\ncan be supplied by specifying "``as`` localname". If a name is not\nfound, ``ImportError`` is raised. If the list of identifiers is\nreplaced by a star (``\'*\'``), all public names defined in the module\nare bound in the local namespace of the ``import`` statement..\n\nThe *public names* defined by a module are determined by checking the\nmodule\'s namespace for a variable named ``__all__``; if defined, it\nmust be a sequence of strings which are names defined or imported by\nthat module. The names given in ``__all__`` are all considered public\nand are required to exist. If ``__all__`` is not defined, the set of\npublic names includes all names found in the module\'s namespace which\ndo not begin with an underscore character (``\'_\'``). ``__all__``\nshould contain the entire public API. It is intended to avoid\naccidentally exporting items that are not part of the API (such as\nlibrary modules which were imported and used within the module).\n\nThe ``from`` form with ``*`` may only occur in a module scope. If the\nwild card form of import --- ``import *`` --- is used in a function\nand the function contains or is a nested block with free variables,\nthe compiler will raise a ``SyntaxError``.\n\nWhen specifying what module to import you do not have to specify the\nabsolute name of the module. When a module or package is contained\nwithin another package it is possible to make a relative import within\nthe same top package without having to mention the package name. By\nusing leading dots in the specified module or package after ``from``\nyou can specify how high to traverse up the current package hierarchy\nwithout specifying exact names. One leading dot means the current\npackage where the module making the import exists. Two dots means up\none package level. Three dots is up two levels, etc. So if you execute\n``from . import mod`` from a module in the ``pkg`` package then you\nwill end up importing ``pkg.mod``. If you execute ``from ..subpkg2\nimprt mod`` from within ``pkg.subpkg1`` you will import\n``pkg.subpkg2.mod``. The specification for relative imports is\ncontained within **PEP 328**.\n\n``importlib.import_module()`` is provided to support applications that\ndetermine which modules need to be loaded dynamically.\n\n\nFuture statements\n=================\n\nA *future statement* is a directive to the compiler that a particular\nmodule should be compiled using syntax or semantics that will be\navailable in a specified future release of Python. The future\nstatement is intended to ease migration to future versions of Python\nthat introduce incompatible changes to the language. It allows use of\nthe new features on a per-module basis before the release in which the\nfeature becomes standard.\n\n future_statement ::= "from" "__future__" "import" feature ["as" name]\n ("," feature ["as" name])*\n | "from" "__future__" "import" "(" feature ["as" name]\n ("," feature ["as" name])* [","] ")"\n feature ::= identifier\n name ::= identifier\n\nA future statement must appear near the top of the module. The only\nlines that can appear before a future statement are:\n\n* the module docstring (if any),\n\n* comments,\n\n* blank lines, and\n\n* other future statements.\n\nThe features recognized by Python 2.6 are ``unicode_literals``,\n``print_function``, ``absolute_import``, ``division``, ``generators``,\n``nested_scopes`` and ``with_statement``. ``generators``,\n``with_statement``, ``nested_scopes`` are redundant in Python version\n2.6 and above because they are always enabled.\n\nA future statement is recognized and treated specially at compile\ntime: Changes to the semantics of core constructs are often\nimplemented by generating different code. It may even be the case\nthat a new feature introduces new incompatible syntax (such as a new\nreserved word), in which case the compiler may need to parse the\nmodule differently. Such decisions cannot be pushed off until\nruntime.\n\nFor any given release, the compiler knows which feature names have\nbeen defined, and raises a compile-time error if a future statement\ncontains a feature not known to it.\n\nThe direct runtime semantics are the same as for any import statement:\nthere is a standard module ``__future__``, described later, and it\nwill be imported in the usual way at the time the future statement is\nexecuted.\n\nThe interesting runtime semantics depend on the specific feature\nenabled by the future statement.\n\nNote that there is nothing special about the statement:\n\n import __future__ [as name]\n\nThat is not a future statement; it\'s an ordinary import statement with\nno special semantics or syntax restrictions.\n\nCode compiled by an ``exec`` statement or calls to the built-in\nfunctions ``compile()`` and ``execfile()`` that occur in a module\n``M`` containing a future statement will, by default, use the new\nsyntax or semantics associated with the future statement. This can,\nstarting with Python 2.2 be controlled by optional arguments to\n``compile()`` --- see the documentation of that function for details.\n\nA future statement typed at an interactive interpreter prompt will\ntake effect for the rest of the interpreter session. If an\ninterpreter is started with the *-i* option, is passed a script name\nto execute, and the script includes a future statement, it will be in\neffect in the interactive session started after the script is\nexecuted.\n\nSee also:\n\n **PEP 236** - Back to the __future__\n The original proposal for the __future__ mechanism.\n', + 'import': u'\nThe ``import`` statement\n************************\n\n import_stmt ::= "import" module ["as" name] ( "," module ["as" name] )*\n | "from" relative_module "import" identifier ["as" name]\n ( "," identifier ["as" name] )*\n | "from" relative_module "import" "(" identifier ["as" name]\n ( "," identifier ["as" name] )* [","] ")"\n | "from" module "import" "*"\n module ::= (identifier ".")* identifier\n relative_module ::= "."* module | "."+\n name ::= identifier\n\nImport statements are executed in two steps: (1) find a module, and\ninitialize it if necessary; (2) define a name or names in the local\nnamespace (of the scope where the ``import`` statement occurs). The\nstatement comes in two forms differing on whether it uses the ``from``\nkeyword. The first form (without ``from``) repeats these steps for\neach identifier in the list. The form with ``from`` performs step (1)\nonce, and then performs step (2) repeatedly.\n\nTo understand how step (1) occurs, one must first understand how\nPython handles hierarchical naming of modules. To help organize\nmodules and provide a hierarchy in naming, Python has a concept of\npackages. A package can contain other packages and modules while\nmodules cannot contain other modules or packages. From a file system\nperspective, packages are directories and modules are files. The\noriginal specification for packages is still available to read,\nalthough minor details have changed since the writing of that\ndocument.\n\nOnce the name of the module is known (unless otherwise specified, the\nterm "module" will refer to both packages and modules), searching for\nthe module or package can begin. The first place checked is\n``sys.modules``, the cache of all modules that have been imported\npreviously. If the module is found there then it is used in step (2)\nof import.\n\nIf the module is not found in the cache, then ``sys.meta_path`` is\nsearched (the specification for ``sys.meta_path`` can be found in\n**PEP 302**). The object is a list of *finder* objects which are\nqueried in order as to whether they know how to load the module by\ncalling their ``find_module()`` method with the name of the module. If\nthe module happens to be contained within a package (as denoted by the\nexistence of a dot in the name), then a second argument to\n``find_module()`` is given as the value of the ``__path__`` attribute\nfrom the parent package (everything up to the last dot in the name of\nthe module being imported). If a finder can find the module it returns\na *loader* (discussed later) or returns ``None``.\n\nIf none of the finders on ``sys.meta_path`` are able to find the\nmodule then some implicitly defined finders are queried.\nImplementations of Python vary in what implicit meta path finders are\ndefined. The one they all do define, though, is one that handles\n``sys.path_hooks``, ``sys.path_importer_cache``, and ``sys.path``.\n\nThe implicit finder searches for the requested module in the "paths"\nspecified in one of two places ("paths" do not have to be file system\npaths). If the module being imported is supposed to be contained\nwithin a package then the second argument passed to ``find_module()``,\n``__path__`` on the parent package, is used as the source of paths. If\nthe module is not contained in a package then ``sys.path`` is used as\nthe source of paths.\n\nOnce the source of paths is chosen it is iterated over to find a\nfinder that can handle that path. The dict at\n``sys.path_importer_cache`` caches finders for paths and is checked\nfor a finder. If the path does not have a finder cached then\n``sys.path_hooks`` is searched by calling each object in the list with\na single argument of the path, returning a finder or raises\n``ImportError``. If a finder is returned then it is cached in\n``sys.path_importer_cache`` and then used for that path entry. If no\nfinder can be found but the path exists then a value of ``None`` is\nstored in ``sys.path_importer_cache`` to signify that an implicit,\nfile-based finder that handles modules stored as individual files\nshould be used for that path. If the path does not exist then a finder\nwhich always returns ``None`` is placed in the cache for the path.\n\nIf no finder can find the module then ``ImportError`` is raised.\nOtherwise some finder returned a loader whose ``load_module()`` method\nis called with the name of the module to load (see **PEP 302** for the\noriginal definition of loaders). A loader has several responsibilities\nto perform on a module it loads. First, if the module already exists\nin ``sys.modules`` (a possibility if the loader is called outside of\nthe import machinery) then it is to use that module for initialization\nand not a new module. But if the module does not exist in\n``sys.modules`` then it is to be added to that dict before\ninitialization begins. If an error occurs during loading of the module\nand it was added to ``sys.modules`` it is to be removed from the dict.\nIf an error occurs but the module was already in ``sys.modules`` it is\nleft in the dict.\n\nThe loader must set several attributes on the module. ``__name__`` is\nto be set to the name of the module. ``__file__`` is to be the "path"\nto the file unless the module is built-in (and thus listed in\n``sys.builtin_module_names``) in which case the attribute is not set.\nIf what is being imported is a package then ``__path__`` is to be set\nto a list of paths to be searched when looking for modules and\npackages contained within the package being imported. ``__package__``\nis optional but should be set to the name of package that contains the\nmodule or package (the empty string is used for module not contained\nin a package). ``__loader__`` is also optional but should be set to\nthe loader object that is loading the module.\n\nIf an error occurs during loading then the loader raises\n``ImportError`` if some other exception is not already being\npropagated. Otherwise the loader returns the module that was loaded\nand initialized.\n\nWhen step (1) finishes without raising an exception, step (2) can\nbegin.\n\nThe first form of ``import`` statement binds the module name in the\nlocal namespace to the module object, and then goes on to import the\nnext identifier, if any. If the module name is followed by ``as``,\nthe name following ``as`` is used as the local name for the module.\n\nThe ``from`` form does not bind the module name: it goes through the\nlist of identifiers, looks each one of them up in the module found in\nstep (1), and binds the name in the local namespace to the object thus\nfound. As with the first form of ``import``, an alternate local name\ncan be supplied by specifying "``as`` localname". If a name is not\nfound, ``ImportError`` is raised. If the list of identifiers is\nreplaced by a star (``\'*\'``), all public names defined in the module\nare bound in the local namespace of the ``import`` statement..\n\nThe *public names* defined by a module are determined by checking the\nmodule\'s namespace for a variable named ``__all__``; if defined, it\nmust be a sequence of strings which are names defined or imported by\nthat module. The names given in ``__all__`` are all considered public\nand are required to exist. If ``__all__`` is not defined, the set of\npublic names includes all names found in the module\'s namespace which\ndo not begin with an underscore character (``\'_\'``). ``__all__``\nshould contain the entire public API. It is intended to avoid\naccidentally exporting items that are not part of the API (such as\nlibrary modules which were imported and used within the module).\n\nThe ``from`` form with ``*`` may only occur in a module scope. If the\nwild card form of import --- ``import *`` --- is used in a function\nand the function contains or is a nested block with free variables,\nthe compiler will raise a ``SyntaxError``.\n\nWhen specifying what module to import you do not have to specify the\nabsolute name of the module. When a module or package is contained\nwithin another package it is possible to make a relative import within\nthe same top package without having to mention the package name. By\nusing leading dots in the specified module or package after ``from``\nyou can specify how high to traverse up the current package hierarchy\nwithout specifying exact names. One leading dot means the current\npackage where the module making the import exists. Two dots means up\none package level. Three dots is up two levels, etc. So if you execute\n``from . import mod`` from a module in the ``pkg`` package then you\nwill end up importing ``pkg.mod``. If you execute ``from ..subpkg2\nimport mod`` from within ``pkg.subpkg1`` you will import\n``pkg.subpkg2.mod``. The specification for relative imports is\ncontained within **PEP 328**.\n\n``importlib.import_module()`` is provided to support applications that\ndetermine which modules need to be loaded dynamically.\n\n\nFuture statements\n=================\n\nA *future statement* is a directive to the compiler that a particular\nmodule should be compiled using syntax or semantics that will be\navailable in a specified future release of Python. The future\nstatement is intended to ease migration to future versions of Python\nthat introduce incompatible changes to the language. It allows use of\nthe new features on a per-module basis before the release in which the\nfeature becomes standard.\n\n future_statement ::= "from" "__future__" "import" feature ["as" name]\n ("," feature ["as" name])*\n | "from" "__future__" "import" "(" feature ["as" name]\n ("," feature ["as" name])* [","] ")"\n feature ::= identifier\n name ::= identifier\n\nA future statement must appear near the top of the module. The only\nlines that can appear before a future statement are:\n\n* the module docstring (if any),\n\n* comments,\n\n* blank lines, and\n\n* other future statements.\n\nThe features recognized by Python 2.6 are ``unicode_literals``,\n``print_function``, ``absolute_import``, ``division``, ``generators``,\n``nested_scopes`` and ``with_statement``. ``generators``,\n``with_statement``, ``nested_scopes`` are redundant in Python version\n2.6 and above because they are always enabled.\n\nA future statement is recognized and treated specially at compile\ntime: Changes to the semantics of core constructs are often\nimplemented by generating different code. It may even be the case\nthat a new feature introduces new incompatible syntax (such as a new\nreserved word), in which case the compiler may need to parse the\nmodule differently. Such decisions cannot be pushed off until\nruntime.\n\nFor any given release, the compiler knows which feature names have\nbeen defined, and raises a compile-time error if a future statement\ncontains a feature not known to it.\n\nThe direct runtime semantics are the same as for any import statement:\nthere is a standard module ``__future__``, described later, and it\nwill be imported in the usual way at the time the future statement is\nexecuted.\n\nThe interesting runtime semantics depend on the specific feature\nenabled by the future statement.\n\nNote that there is nothing special about the statement:\n\n import __future__ [as name]\n\nThat is not a future statement; it\'s an ordinary import statement with\nno special semantics or syntax restrictions.\n\nCode compiled by an ``exec`` statement or calls to the built-in\nfunctions ``compile()`` and ``execfile()`` that occur in a module\n``M`` containing a future statement will, by default, use the new\nsyntax or semantics associated with the future statement. This can,\nstarting with Python 2.2 be controlled by optional arguments to\n``compile()`` --- see the documentation of that function for details.\n\nA future statement typed at an interactive interpreter prompt will\ntake effect for the rest of the interpreter session. If an\ninterpreter is started with the *-i* option, is passed a script name\nto execute, and the script includes a future statement, it will be in\neffect in the interactive session started after the script is\nexecuted.\n\nSee also:\n\n **PEP 236** - Back to the __future__\n The original proposal for the __future__ mechanism.\n', 'in': u'\nComparisons\n***********\n\nUnlike C, all comparison operations in Python have the same priority,\nwhich is lower than that of any arithmetic, shifting or bitwise\noperation. Also unlike C, expressions like ``a < b < c`` have the\ninterpretation that is conventional in mathematics:\n\n comparison ::= or_expr ( comp_operator or_expr )*\n comp_operator ::= "<" | ">" | "==" | ">=" | "<=" | "<>" | "!="\n | "is" ["not"] | ["not"] "in"\n\nComparisons yield boolean values: ``True`` or ``False``.\n\nComparisons can be chained arbitrarily, e.g., ``x < y <= z`` is\nequivalent to ``x < y and y <= z``, except that ``y`` is evaluated\nonly once (but in both cases ``z`` is not evaluated at all when ``x <\ny`` is found to be false).\n\nFormally, if *a*, *b*, *c*, ..., *y*, *z* are expressions and *op1*,\n*op2*, ..., *opN* are comparison operators, then ``a op1 b op2 c ... y\nopN z`` is equivalent to ``a op1 b and b op2 c and ... y opN z``,\nexcept that each expression is evaluated at most once.\n\nNote that ``a op1 b op2 c`` doesn\'t imply any kind of comparison\nbetween *a* and *c*, so that, e.g., ``x < y > z`` is perfectly legal\n(though perhaps not pretty).\n\nThe forms ``<>`` and ``!=`` are equivalent; for consistency with C,\n``!=`` is preferred; where ``!=`` is mentioned below ``<>`` is also\naccepted. The ``<>`` spelling is considered obsolescent.\n\nThe operators ``<``, ``>``, ``==``, ``>=``, ``<=``, and ``!=`` compare\nthe values of two objects. The objects need not have the same type.\nIf both are numbers, they are converted to a common type. Otherwise,\nobjects of different types *always* compare unequal, and are ordered\nconsistently but arbitrarily. You can control comparison behavior of\nobjects of non-built-in types by defining a ``__cmp__`` method or rich\ncomparison methods like ``__gt__``, described in section *Special\nmethod names*.\n\n(This unusual definition of comparison was used to simplify the\ndefinition of operations like sorting and the ``in`` and ``not in``\noperators. In the future, the comparison rules for objects of\ndifferent types are likely to change.)\n\nComparison of objects of the same type depends on the type:\n\n* Numbers are compared arithmetically.\n\n* Strings are compared lexicographically using the numeric equivalents\n (the result of the built-in function ``ord()``) of their characters.\n Unicode and 8-bit strings are fully interoperable in this behavior.\n [4]\n\n* Tuples and lists are compared lexicographically using comparison of\n corresponding elements. This means that to compare equal, each\n element must compare equal and the two sequences must be of the same\n type and have the same length.\n\n If not equal, the sequences are ordered the same as their first\n differing elements. For example, ``cmp([1,2,x], [1,2,y])`` returns\n the same as ``cmp(x,y)``. If the corresponding element does not\n exist, the shorter sequence is ordered first (for example, ``[1,2] <\n [1,2,3]``).\n\n* Mappings (dictionaries) compare equal if and only if their sorted\n (key, value) lists compare equal. [5] Outcomes other than equality\n are resolved consistently, but are not otherwise defined. [6]\n\n* Most other objects of built-in types compare unequal unless they are\n the same object; the choice whether one object is considered smaller\n or larger than another one is made arbitrarily but consistently\n within one execution of a program.\n\nThe operators ``in`` and ``not in`` test for collection membership.\n``x in s`` evaluates to true if *x* is a member of the collection *s*,\nand false otherwise. ``x not in s`` returns the negation of ``x in\ns``. The collection membership test has traditionally been bound to\nsequences; an object is a member of a collection if the collection is\na sequence and contains an element equal to that object. However, it\nmake sense for many other object types to support membership tests\nwithout being a sequence. In particular, dictionaries (for keys) and\nsets support membership testing.\n\nFor the list and tuple types, ``x in y`` is true if and only if there\nexists an index *i* such that ``x == y[i]`` is true.\n\nFor the Unicode and string types, ``x in y`` is true if and only if\n*x* is a substring of *y*. An equivalent test is ``y.find(x) != -1``.\nNote, *x* and *y* need not be the same type; consequently, ``u\'ab\' in\n\'abc\'`` will return ``True``. Empty strings are always considered to\nbe a substring of any other string, so ``"" in "abc"`` will return\n``True``.\n\nChanged in version 2.3: Previously, *x* was required to be a string of\nlength ``1``.\n\nFor user-defined classes which define the ``__contains__()`` method,\n``x in y`` is true if and only if ``y.__contains__(x)`` is true.\n\nFor user-defined classes which do not define ``__contains__()`` but do\ndefine ``__iter__()``, ``x in y`` is true if some value ``z`` with ``x\n== z`` is produced while iterating over ``y``. If an exception is\nraised during the iteration, it is as if ``in`` raised that exception.\n\nLastly, the old-style iteration protocol is tried: if a class defines\n``__getitem__()``, ``x in y`` is true if and only if there is a non-\nnegative integer index *i* such that ``x == y[i]``, and all lower\ninteger indices do not raise ``IndexError`` exception. (If any other\nexception is raised, it is as if ``in`` raised that exception).\n\nThe operator ``not in`` is defined to have the inverse true value of\n``in``.\n\nThe operators ``is`` and ``is not`` test for object identity: ``x is\ny`` is true if and only if *x* and *y* are the same object. ``x is\nnot y`` yields the inverse truth value. [7]\n', 'integers': u'\nInteger and long integer literals\n*********************************\n\nInteger and long integer literals are described by the following\nlexical definitions:\n\n longinteger ::= integer ("l" | "L")\n integer ::= decimalinteger | octinteger | hexinteger | bininteger\n decimalinteger ::= nonzerodigit digit* | "0"\n octinteger ::= "0" ("o" | "O") octdigit+ | "0" octdigit+\n hexinteger ::= "0" ("x" | "X") hexdigit+\n bininteger ::= "0" ("b" | "B") bindigit+\n nonzerodigit ::= "1"..."9"\n octdigit ::= "0"..."7"\n bindigit ::= "0" | "1"\n hexdigit ::= digit | "a"..."f" | "A"..."F"\n\nAlthough both lower case ``\'l\'`` and upper case ``\'L\'`` are allowed as\nsuffix for long integers, it is strongly recommended to always use\n``\'L\'``, since the letter ``\'l\'`` looks too much like the digit\n``\'1\'``.\n\nPlain integer literals that are above the largest representable plain\ninteger (e.g., 2147483647 when using 32-bit arithmetic) are accepted\nas if they were long integers instead. [1] There is no limit for long\ninteger literals apart from what can be stored in available memory.\n\nSome examples of plain integer literals (first row) and long integer\nliterals (second and third rows):\n\n 7 2147483647 0177\n 3L 79228162514264337593543950336L 0377L 0x100000000L\n 79228162514264337593543950336 0xdeadbeef\n', 'lambda': u'\nLambdas\n*******\n\n lambda_form ::= "lambda" [parameter_list]: expression\n old_lambda_form ::= "lambda" [parameter_list]: old_expression\n\nLambda forms (lambda expressions) have the same syntactic position as\nexpressions. They are a shorthand to create anonymous functions; the\nexpression ``lambda arguments: expression`` yields a function object.\nThe unnamed object behaves like a function object defined with\n\n def name(arguments):\n return expression\n\nSee section *Function definitions* for the syntax of parameter lists.\nNote that functions created with lambda forms cannot contain\nstatements.\n', 'lists': u'\nList displays\n*************\n\nA list display is a possibly empty series of expressions enclosed in\nsquare brackets:\n\n list_display ::= "[" [expression_list | list_comprehension] "]"\n list_comprehension ::= expression list_for\n list_for ::= "for" target_list "in" old_expression_list [list_iter]\n old_expression_list ::= old_expression [("," old_expression)+ [","]]\n old_expression ::= or_test | old_lambda_form\n list_iter ::= list_for | list_if\n list_if ::= "if" old_expression [list_iter]\n\nA list display yields a new list object. Its contents are specified\nby providing either a list of expressions or a list comprehension.\nWhen a comma-separated list of expressions is supplied, its elements\nare evaluated from left to right and placed into the list object in\nthat order. When a list comprehension is supplied, it consists of a\nsingle expression followed by at least one ``for`` clause and zero or\nmore ``for`` or ``if`` clauses. In this case, the elements of the new\nlist are those that would be produced by considering each of the\n``for`` or ``if`` clauses a block, nesting from left to right, and\nevaluating the expression to produce a list element each time the\ninnermost block is reached [1].\n', - 'naming': u"\nNaming and binding\n******************\n\n*Names* refer to objects. Names are introduced by name binding\noperations. Each occurrence of a name in the program text refers to\nthe *binding* of that name established in the innermost function block\ncontaining the use.\n\nA *block* is a piece of Python program text that is executed as a\nunit. The following are blocks: a module, a function body, and a class\ndefinition. Each command typed interactively is a block. A script\nfile (a file given as standard input to the interpreter or specified\non the interpreter command line the first argument) is a code block.\nA script command (a command specified on the interpreter command line\nwith the '**-c**' option) is a code block. The file read by the\nbuilt-in function ``execfile()`` is a code block. The string argument\npassed to the built-in function ``eval()`` and to the ``exec``\nstatement is a code block. The expression read and evaluated by the\nbuilt-in function ``input()`` is a code block.\n\nA code block is executed in an *execution frame*. A frame contains\nsome administrative information (used for debugging) and determines\nwhere and how execution continues after the code block's execution has\ncompleted.\n\nA *scope* defines the visibility of a name within a block. If a local\nvariable is defined in a block, its scope includes that block. If the\ndefinition occurs in a function block, the scope extends to any blocks\ncontained within the defining one, unless a contained block introduces\na different binding for the name. The scope of names defined in a\nclass block is limited to the class block; it does not extend to the\ncode blocks of methods -- this includes generator expressions since\nthey are implemented using a function scope. This means that the\nfollowing will fail:\n\n class A:\n a = 42\n b = list(a + i for i in range(10))\n\nWhen a name is used in a code block, it is resolved using the nearest\nenclosing scope. The set of all such scopes visible to a code block\nis called the block's *environment*.\n\nIf a name is bound in a block, it is a local variable of that block.\nIf a name is bound at the module level, it is a global variable. (The\nvariables of the module code block are local and global.) If a\nvariable is used in a code block but not defined there, it is a *free\nvariable*.\n\nWhen a name is not found at all, a ``NameError`` exception is raised.\nIf the name refers to a local variable that has not been bound, a\n``UnboundLocalError`` exception is raised. ``UnboundLocalError`` is a\nsubclass of ``NameError``.\n\nThe following constructs bind names: formal parameters to functions,\n``import`` statements, class and function definitions (these bind the\nclass or function name in the defining block), and targets that are\nidentifiers if occurring in an assignment, ``for`` loop header, in the\nsecond position of an ``except`` clause header or after ``as`` in a\n``with`` statement. The ``import`` statement of the form ``from ...\nimport *`` binds all names defined in the imported module, except\nthose beginning with an underscore. This form may only be used at the\nmodule level.\n\nA target occurring in a ``del`` statement is also considered bound for\nthis purpose (though the actual semantics are to unbind the name). It\nis illegal to unbind a name that is referenced by an enclosing scope;\nthe compiler will report a ``SyntaxError``.\n\nEach assignment or import statement occurs within a block defined by a\nclass or function definition or at the module level (the top-level\ncode block).\n\nIf a name binding operation occurs anywhere within a code block, all\nuses of the name within the block are treated as references to the\ncurrent block. This can lead to errors when a name is used within a\nblock before it is bound. This rule is subtle. Python lacks\ndeclarations and allows name binding operations to occur anywhere\nwithin a code block. The local variables of a code block can be\ndetermined by scanning the entire text of the block for name binding\noperations.\n\nIf the global statement occurs within a block, all uses of the name\nspecified in the statement refer to the binding of that name in the\ntop-level namespace. Names are resolved in the top-level namespace by\nsearching the global namespace, i.e. the namespace of the module\ncontaining the code block, and the builtins namespace, the namespace\nof the module ``__builtin__``. The global namespace is searched\nfirst. If the name is not found there, the builtins namespace is\nsearched. The global statement must precede all uses of the name.\n\nThe builtins namespace associated with the execution of a code block\nis actually found by looking up the name ``__builtins__`` in its\nglobal namespace; this should be a dictionary or a module (in the\nlatter case the module's dictionary is used). By default, when in the\n``__main__`` module, ``__builtins__`` is the built-in module\n``__builtin__`` (note: no 's'); when in any other module,\n``__builtins__`` is an alias for the dictionary of the ``__builtin__``\nmodule itself. ``__builtins__`` can be set to a user-created\ndictionary to create a weak form of restricted execution.\n\n**CPython implementation detail:** Users should not touch\n``__builtins__``; it is strictly an implementation detail. Users\nwanting to override values in the builtins namespace should ``import``\nthe ``__builtin__`` (no 's') module and modify its attributes\nappropriately.\n\nThe namespace for a module is automatically created the first time a\nmodule is imported. The main module for a script is always called\n``__main__``.\n\nThe global statement has the same scope as a name binding operation in\nthe same block. If the nearest enclosing scope for a free variable\ncontains a global statement, the free variable is treated as a global.\n\nA class definition is an executable statement that may use and define\nnames. These references follow the normal rules for name resolution.\nThe namespace of the class definition becomes the attribute dictionary\nof the class. Names defined at the class scope are not visible in\nmethods.\n\n\nInteraction with dynamic features\n=================================\n\nThere are several cases where Python statements are illegal when used\nin conjunction with nested scopes that contain free variables.\n\nIf a variable is referenced in an enclosing scope, it is illegal to\ndelete the name. An error will be reported at compile time.\n\nIf the wild card form of import --- ``import *`` --- is used in a\nfunction and the function contains or is a nested block with free\nvariables, the compiler will raise a ``SyntaxError``.\n\nIf ``exec`` is used in a function and the function contains or is a\nnested block with free variables, the compiler will raise a\n``SyntaxError`` unless the exec explicitly specifies the local\nnamespace for the ``exec``. (In other words, ``exec obj`` would be\nillegal, but ``exec obj in ns`` would be legal.)\n\nThe ``eval()``, ``execfile()``, and ``input()`` functions and the\n``exec`` statement do not have access to the full environment for\nresolving names. Names may be resolved in the local and global\nnamespaces of the caller. Free variables are not resolved in the\nnearest enclosing namespace, but in the global namespace. [1] The\n``exec`` statement and the ``eval()`` and ``execfile()`` functions\nhave optional arguments to override the global and local namespace.\nIf only one namespace is specified, it is used for both.\n", + 'naming': u"\nNaming and binding\n******************\n\n*Names* refer to objects. Names are introduced by name binding\noperations. Each occurrence of a name in the program text refers to\nthe *binding* of that name established in the innermost function block\ncontaining the use.\n\nA *block* is a piece of Python program text that is executed as a\nunit. The following are blocks: a module, a function body, and a class\ndefinition. Each command typed interactively is a block. A script\nfile (a file given as standard input to the interpreter or specified\non the interpreter command line the first argument) is a code block.\nA script command (a command specified on the interpreter command line\nwith the '**-c**' option) is a code block. The file read by the\nbuilt-in function ``execfile()`` is a code block. The string argument\npassed to the built-in function ``eval()`` and to the ``exec``\nstatement is a code block. The expression read and evaluated by the\nbuilt-in function ``input()`` is a code block.\n\nA code block is executed in an *execution frame*. A frame contains\nsome administrative information (used for debugging) and determines\nwhere and how execution continues after the code block's execution has\ncompleted.\n\nA *scope* defines the visibility of a name within a block. If a local\nvariable is defined in a block, its scope includes that block. If the\ndefinition occurs in a function block, the scope extends to any blocks\ncontained within the defining one, unless a contained block introduces\na different binding for the name. The scope of names defined in a\nclass block is limited to the class block; it does not extend to the\ncode blocks of methods -- this includes generator expressions since\nthey are implemented using a function scope. This means that the\nfollowing will fail:\n\n class A:\n a = 42\n b = list(a + i for i in range(10))\n\nWhen a name is used in a code block, it is resolved using the nearest\nenclosing scope. The set of all such scopes visible to a code block\nis called the block's *environment*.\n\nIf a name is bound in a block, it is a local variable of that block.\nIf a name is bound at the module level, it is a global variable. (The\nvariables of the module code block are local and global.) If a\nvariable is used in a code block but not defined there, it is a *free\nvariable*.\n\nWhen a name is not found at all, a ``NameError`` exception is raised.\nIf the name refers to a local variable that has not been bound, a\n``UnboundLocalError`` exception is raised. ``UnboundLocalError`` is a\nsubclass of ``NameError``.\n\nThe following constructs bind names: formal parameters to functions,\n``import`` statements, class and function definitions (these bind the\nclass or function name in the defining block), and targets that are\nidentifiers if occurring in an assignment, ``for`` loop header, in the\nsecond position of an ``except`` clause header or after ``as`` in a\n``with`` statement. The ``import`` statement of the form ``from ...\nimport *`` binds all names defined in the imported module, except\nthose beginning with an underscore. This form may only be used at the\nmodule level.\n\nA target occurring in a ``del`` statement is also considered bound for\nthis purpose (though the actual semantics are to unbind the name). It\nis illegal to unbind a name that is referenced by an enclosing scope;\nthe compiler will report a ``SyntaxError``.\n\nEach assignment or import statement occurs within a block defined by a\nclass or function definition or at the module level (the top-level\ncode block).\n\nIf a name binding operation occurs anywhere within a code block, all\nuses of the name within the block are treated as references to the\ncurrent block. This can lead to errors when a name is used within a\nblock before it is bound. This rule is subtle. Python lacks\ndeclarations and allows name binding operations to occur anywhere\nwithin a code block. The local variables of a code block can be\ndetermined by scanning the entire text of the block for name binding\noperations.\n\nIf the global statement occurs within a block, all uses of the name\nspecified in the statement refer to the binding of that name in the\ntop-level namespace. Names are resolved in the top-level namespace by\nsearching the global namespace, i.e. the namespace of the module\ncontaining the code block, and the builtins namespace, the namespace\nof the module ``__builtin__``. The global namespace is searched\nfirst. If the name is not found there, the builtins namespace is\nsearched. The global statement must precede all uses of the name.\n\nThe builtins namespace associated with the execution of a code block\nis actually found by looking up the name ``__builtins__`` in its\nglobal namespace; this should be a dictionary or a module (in the\nlatter case the module's dictionary is used). By default, when in the\n``__main__`` module, ``__builtins__`` is the built-in module\n``__builtin__`` (note: no 's'); when in any other module,\n``__builtins__`` is an alias for the dictionary of the ``__builtin__``\nmodule itself. ``__builtins__`` can be set to a user-created\ndictionary to create a weak form of restricted execution.\n\n**CPython implementation detail:** Users should not touch\n``__builtins__``; it is strictly an implementation detail. Users\nwanting to override values in the builtins namespace should ``import``\nthe ``__builtin__`` (no 's') module and modify its attributes\nappropriately.\n\nThe namespace for a module is automatically created the first time a\nmodule is imported. The main module for a script is always called\n``__main__``.\n\nThe ``global`` statement has the same scope as a name binding\noperation in the same block. If the nearest enclosing scope for a\nfree variable contains a global statement, the free variable is\ntreated as a global.\n\nA class definition is an executable statement that may use and define\nnames. These references follow the normal rules for name resolution.\nThe namespace of the class definition becomes the attribute dictionary\nof the class. Names defined at the class scope are not visible in\nmethods.\n\n\nInteraction with dynamic features\n=================================\n\nThere are several cases where Python statements are illegal when used\nin conjunction with nested scopes that contain free variables.\n\nIf a variable is referenced in an enclosing scope, it is illegal to\ndelete the name. An error will be reported at compile time.\n\nIf the wild card form of import --- ``import *`` --- is used in a\nfunction and the function contains or is a nested block with free\nvariables, the compiler will raise a ``SyntaxError``.\n\nIf ``exec`` is used in a function and the function contains or is a\nnested block with free variables, the compiler will raise a\n``SyntaxError`` unless the exec explicitly specifies the local\nnamespace for the ``exec``. (In other words, ``exec obj`` would be\nillegal, but ``exec obj in ns`` would be legal.)\n\nThe ``eval()``, ``execfile()``, and ``input()`` functions and the\n``exec`` statement do not have access to the full environment for\nresolving names. Names may be resolved in the local and global\nnamespaces of the caller. Free variables are not resolved in the\nnearest enclosing namespace, but in the global namespace. [1] The\n``exec`` statement and the ``eval()`` and ``execfile()`` functions\nhave optional arguments to override the global and local namespace.\nIf only one namespace is specified, it is used for both.\n", 'numbers': u"\nNumeric literals\n****************\n\nThere are four types of numeric literals: plain integers, long\nintegers, floating point numbers, and imaginary numbers. There are no\ncomplex literals (complex numbers can be formed by adding a real\nnumber and an imaginary number).\n\nNote that numeric literals do not include a sign; a phrase like ``-1``\nis actually an expression composed of the unary operator '``-``' and\nthe literal ``1``.\n", 'numeric-types': u'\nEmulating numeric types\n***********************\n\nThe following methods can be defined to emulate numeric objects.\nMethods corresponding to operations that are not supported by the\nparticular kind of number implemented (e.g., bitwise operations for\nnon-integral numbers) should be left undefined.\n\nobject.__add__(self, other)\nobject.__sub__(self, other)\nobject.__mul__(self, other)\nobject.__floordiv__(self, other)\nobject.__mod__(self, other)\nobject.__divmod__(self, other)\nobject.__pow__(self, other[, modulo])\nobject.__lshift__(self, other)\nobject.__rshift__(self, other)\nobject.__and__(self, other)\nobject.__xor__(self, other)\nobject.__or__(self, other)\n\n These methods are called to implement the binary arithmetic\n operations (``+``, ``-``, ``*``, ``//``, ``%``, ``divmod()``,\n ``pow()``, ``**``, ``<<``, ``>>``, ``&``, ``^``, ``|``). For\n instance, to evaluate the expression ``x + y``, where *x* is an\n instance of a class that has an ``__add__()`` method,\n ``x.__add__(y)`` is called. The ``__divmod__()`` method should be\n the equivalent to using ``__floordiv__()`` and ``__mod__()``; it\n should not be related to ``__truediv__()`` (described below). Note\n that ``__pow__()`` should be defined to accept an optional third\n argument if the ternary version of the built-in ``pow()`` function\n is to be supported.\n\n If one of those methods does not support the operation with the\n supplied arguments, it should return ``NotImplemented``.\n\nobject.__div__(self, other)\nobject.__truediv__(self, other)\n\n The division operator (``/``) is implemented by these methods. The\n ``__truediv__()`` method is used when ``__future__.division`` is in\n effect, otherwise ``__div__()`` is used. If only one of these two\n methods is defined, the object will not support division in the\n alternate context; ``TypeError`` will be raised instead.\n\nobject.__radd__(self, other)\nobject.__rsub__(self, other)\nobject.__rmul__(self, other)\nobject.__rdiv__(self, other)\nobject.__rtruediv__(self, other)\nobject.__rfloordiv__(self, other)\nobject.__rmod__(self, other)\nobject.__rdivmod__(self, other)\nobject.__rpow__(self, other)\nobject.__rlshift__(self, other)\nobject.__rrshift__(self, other)\nobject.__rand__(self, other)\nobject.__rxor__(self, other)\nobject.__ror__(self, other)\n\n These methods are called to implement the binary arithmetic\n operations (``+``, ``-``, ``*``, ``/``, ``%``, ``divmod()``,\n ``pow()``, ``**``, ``<<``, ``>>``, ``&``, ``^``, ``|``) with\n reflected (swapped) operands. These functions are only called if\n the left operand does not support the corresponding operation and\n the operands are of different types. [2] For instance, to evaluate\n the expression ``x - y``, where *y* is an instance of a class that\n has an ``__rsub__()`` method, ``y.__rsub__(x)`` is called if\n ``x.__sub__(y)`` returns *NotImplemented*.\n\n Note that ternary ``pow()`` will not try calling ``__rpow__()``\n (the coercion rules would become too complicated).\n\n Note: If the right operand\'s type is a subclass of the left operand\'s\n type and that subclass provides the reflected method for the\n operation, this method will be called before the left operand\'s\n non-reflected method. This behavior allows subclasses to\n override their ancestors\' operations.\n\nobject.__iadd__(self, other)\nobject.__isub__(self, other)\nobject.__imul__(self, other)\nobject.__idiv__(self, other)\nobject.__itruediv__(self, other)\nobject.__ifloordiv__(self, other)\nobject.__imod__(self, other)\nobject.__ipow__(self, other[, modulo])\nobject.__ilshift__(self, other)\nobject.__irshift__(self, other)\nobject.__iand__(self, other)\nobject.__ixor__(self, other)\nobject.__ior__(self, other)\n\n These methods are called to implement the augmented arithmetic\n assignments (``+=``, ``-=``, ``*=``, ``/=``, ``//=``, ``%=``,\n ``**=``, ``<<=``, ``>>=``, ``&=``, ``^=``, ``|=``). These methods\n should attempt to do the operation in-place (modifying *self*) and\n return the result (which could be, but does not have to be,\n *self*). If a specific method is not defined, the augmented\n assignment falls back to the normal methods. For instance, to\n execute the statement ``x += y``, where *x* is an instance of a\n class that has an ``__iadd__()`` method, ``x.__iadd__(y)`` is\n called. If *x* is an instance of a class that does not define a\n ``__iadd__()`` method, ``x.__add__(y)`` and ``y.__radd__(x)`` are\n considered, as with the evaluation of ``x + y``.\n\nobject.__neg__(self)\nobject.__pos__(self)\nobject.__abs__(self)\nobject.__invert__(self)\n\n Called to implement the unary arithmetic operations (``-``, ``+``,\n ``abs()`` and ``~``).\n\nobject.__complex__(self)\nobject.__int__(self)\nobject.__long__(self)\nobject.__float__(self)\n\n Called to implement the built-in functions ``complex()``,\n ``int()``, ``long()``, and ``float()``. Should return a value of\n the appropriate type.\n\nobject.__oct__(self)\nobject.__hex__(self)\n\n Called to implement the built-in functions ``oct()`` and ``hex()``.\n Should return a string value.\n\nobject.__index__(self)\n\n Called to implement ``operator.index()``. Also called whenever\n Python needs an integer object (such as in slicing). Must return\n an integer (int or long).\n\n New in version 2.5.\n\nobject.__coerce__(self, other)\n\n Called to implement "mixed-mode" numeric arithmetic. Should either\n return a 2-tuple containing *self* and *other* converted to a\n common numeric type, or ``None`` if conversion is impossible. When\n the common type would be the type of ``other``, it is sufficient to\n return ``None``, since the interpreter will also ask the other\n object to attempt a coercion (but sometimes, if the implementation\n of the other type cannot be changed, it is useful to do the\n conversion to the other type here). A return value of\n ``NotImplemented`` is equivalent to returning ``None``.\n', - 'objects': u'\nObjects, values and types\n*************************\n\n*Objects* are Python\'s abstraction for data. All data in a Python\nprogram is represented by objects or by relations between objects. (In\na sense, and in conformance to Von Neumann\'s model of a "stored\nprogram computer," code is also represented by objects.)\n\nEvery object has an identity, a type and a value. An object\'s\n*identity* never changes once it has been created; you may think of it\nas the object\'s address in memory. The \'``is``\' operator compares the\nidentity of two objects; the ``id()`` function returns an integer\nrepresenting its identity (currently implemented as its address). An\nobject\'s *type* is also unchangeable. [1] An object\'s type determines\nthe operations that the object supports (e.g., "does it have a\nlength?") and also defines the possible values for objects of that\ntype. The ``type()`` function returns an object\'s type (which is an\nobject itself). The *value* of some objects can change. Objects\nwhose value can change are said to be *mutable*; objects whose value\nis unchangeable once they are created are called *immutable*. (The\nvalue of an immutable container object that contains a reference to a\nmutable object can change when the latter\'s value is changed; however\nthe container is still considered immutable, because the collection of\nobjects it contains cannot be changed. So, immutability is not\nstrictly the same as having an unchangeable value, it is more subtle.)\nAn object\'s mutability is determined by its type; for instance,\nnumbers, strings and tuples are immutable, while dictionaries and\nlists are mutable.\n\nObjects are never explicitly destroyed; however, when they become\nunreachable they may be garbage-collected. An implementation is\nallowed to postpone garbage collection or omit it altogether --- it is\na matter of implementation quality how garbage collection is\nimplemented, as long as no objects are collected that are still\nreachable.\n\n**CPython implementation detail:** CPython currently uses a reference-\ncounting scheme with (optional) delayed detection of cyclically linked\ngarbage, which collects most objects as soon as they become\nunreachable, but is not guaranteed to collect garbage containing\ncircular references. See the documentation of the ``gc`` module for\ninformation on controlling the collection of cyclic garbage. Other\nimplementations act differently and CPython may change.\n\nNote that the use of the implementation\'s tracing or debugging\nfacilities may keep objects alive that would normally be collectable.\nAlso note that catching an exception with a \'``try``...``except``\'\nstatement may keep objects alive.\n\nSome objects contain references to "external" resources such as open\nfiles or windows. It is understood that these resources are freed\nwhen the object is garbage-collected, but since garbage collection is\nnot guaranteed to happen, such objects also provide an explicit way to\nrelease the external resource, usually a ``close()`` method. Programs\nare strongly recommended to explicitly close such objects. The\n\'``try``...``finally``\' statement provides a convenient way to do\nthis.\n\nSome objects contain references to other objects; these are called\n*containers*. Examples of containers are tuples, lists and\ndictionaries. The references are part of a container\'s value. In\nmost cases, when we talk about the value of a container, we imply the\nvalues, not the identities of the contained objects; however, when we\ntalk about the mutability of a container, only the identities of the\nimmediately contained objects are implied. So, if an immutable\ncontainer (like a tuple) contains a reference to a mutable object, its\nvalue changes if that mutable object is changed.\n\nTypes affect almost all aspects of object behavior. Even the\nimportance of object identity is affected in some sense: for immutable\ntypes, operations that compute new values may actually return a\nreference to any existing object with the same type and value, while\nfor mutable objects this is not allowed. E.g., after ``a = 1; b =\n1``, ``a`` and ``b`` may or may not refer to the same object with the\nvalue one, depending on the implementation, but after ``c = []; d =\n[]``, ``c`` and ``d`` are guaranteed to refer to two different,\nunique, newly created empty lists. (Note that ``c = d = []`` assigns\nthe same object to both ``c`` and ``d``.)\n', - 'operator-summary': u'\nSummary\n*******\n\nThe following table summarizes the operator precedences in Python,\nfrom lowest precedence (least binding) to highest precedence (most\nbinding). Operators in the same box have the same precedence. Unless\nthe syntax is explicitly given, operators are binary. Operators in\nthe same box group left to right (except for comparisons, including\ntests, which all have the same precedence and chain from left to right\n--- see section *Comparisons* --- and exponentiation, which groups\nfrom right to left).\n\n+-------------------------------------------------+---------------------------------------+\n| Operator | Description |\n+=================================================+=======================================+\n| ``lambda`` | Lambda expression |\n+-------------------------------------------------+---------------------------------------+\n| ``if`` -- ``else`` | Conditional expression |\n+-------------------------------------------------+---------------------------------------+\n| ``or`` | Boolean OR |\n+-------------------------------------------------+---------------------------------------+\n| ``and`` | Boolean AND |\n+-------------------------------------------------+---------------------------------------+\n| ``not`` *x* | Boolean NOT |\n+-------------------------------------------------+---------------------------------------+\n| ``in``, ``not`` ``in``, ``is``, ``is not``, | Comparisons, including membership |\n| ``<``, ``<=``, ``>``, ``>=``, ``<>``, ``!=``, | tests and identity tests, |\n| ``==`` | |\n+-------------------------------------------------+---------------------------------------+\n| ``|`` | Bitwise OR |\n+-------------------------------------------------+---------------------------------------+\n| ``^`` | Bitwise XOR |\n+-------------------------------------------------+---------------------------------------+\n| ``&`` | Bitwise AND |\n+-------------------------------------------------+---------------------------------------+\n| ``<<``, ``>>`` | Shifts |\n+-------------------------------------------------+---------------------------------------+\n| ``+``, ``-`` | Addition and subtraction |\n+-------------------------------------------------+---------------------------------------+\n| ``*``, ``/``, ``//``, ``%`` | Multiplication, division, remainder |\n+-------------------------------------------------+---------------------------------------+\n| ``+x``, ``-x``, ``~x`` | Positive, negative, bitwise NOT |\n+-------------------------------------------------+---------------------------------------+\n| ``**`` | Exponentiation [8] |\n+-------------------------------------------------+---------------------------------------+\n| ``x[index]``, ``x[index:index]``, | Subscription, slicing, call, |\n| ``x(arguments...)``, ``x.attribute`` | attribute reference |\n+-------------------------------------------------+---------------------------------------+\n| ``(expressions...)``, ``[expressions...]``, | Binding or tuple display, list |\n| ``{key:datum...}``, ```expressions...``` | display, dictionary display, string |\n| | conversion |\n+-------------------------------------------------+---------------------------------------+\n\n-[ Footnotes ]-\n\n[1] In Python 2.3 and later releases, a list comprehension "leaks" the\n control variables of each ``for`` it contains into the containing\n scope. However, this behavior is deprecated, and relying on it\n will not work in Python 3.0\n\n[2] While ``abs(x%y) < abs(y)`` is true mathematically, for floats it\n may not be true numerically due to roundoff. For example, and\n assuming a platform on which a Python float is an IEEE 754 double-\n precision number, in order that ``-1e-100 % 1e100`` have the same\n sign as ``1e100``, the computed result is ``-1e-100 + 1e100``,\n which is numerically exactly equal to ``1e100``. Function\n ``fmod()`` in the ``math`` module returns a result whose sign\n matches the sign of the first argument instead, and so returns\n ``-1e-100`` in this case. Which approach is more appropriate\n depends on the application.\n\n[3] If x is very close to an exact integer multiple of y, it\'s\n possible for ``floor(x/y)`` to be one larger than ``(x-x%y)/y``\n due to rounding. In such cases, Python returns the latter result,\n in order to preserve that ``divmod(x,y)[0] * y + x % y`` be very\n close to ``x``.\n\n[4] While comparisons between unicode strings make sense at the byte\n level, they may be counter-intuitive to users. For example, the\n strings ``u"\\u00C7"`` and ``u"\\u0043\\u0327"`` compare differently,\n even though they both represent the same unicode character (LATIN\n CAPITAL LETTER C WITH CEDILLA). To compare strings in a human\n recognizable way, compare using ``unicodedata.normalize()``.\n\n[5] The implementation computes this efficiently, without constructing\n lists or sorting.\n\n[6] Earlier versions of Python used lexicographic comparison of the\n sorted (key, value) lists, but this was very expensive for the\n common case of comparing for equality. An even earlier version of\n Python compared dictionaries by identity only, but this caused\n surprises because people expected to be able to test a dictionary\n for emptiness by comparing it to ``{}``.\n\n[7] Due to automatic garbage-collection, free lists, and the dynamic\n nature of descriptors, you may notice seemingly unusual behaviour\n in certain uses of the ``is`` operator, like those involving\n comparisons between instance methods, or constants. Check their\n documentation for more info.\n\n[8] The power operator ``**`` binds less tightly than an arithmetic or\n bitwise unary operator on its right, that is, ``2**-1`` is\n ``0.5``.\n', + 'objects': u'\nObjects, values and types\n*************************\n\n*Objects* are Python\'s abstraction for data. All data in a Python\nprogram is represented by objects or by relations between objects. (In\na sense, and in conformance to Von Neumann\'s model of a "stored\nprogram computer," code is also represented by objects.)\n\nEvery object has an identity, a type and a value. An object\'s\n*identity* never changes once it has been created; you may think of it\nas the object\'s address in memory. The \'``is``\' operator compares the\nidentity of two objects; the ``id()`` function returns an integer\nrepresenting its identity (currently implemented as its address). An\nobject\'s *type* is also unchangeable. [1] An object\'s type determines\nthe operations that the object supports (e.g., "does it have a\nlength?") and also defines the possible values for objects of that\ntype. The ``type()`` function returns an object\'s type (which is an\nobject itself). The *value* of some objects can change. Objects\nwhose value can change are said to be *mutable*; objects whose value\nis unchangeable once they are created are called *immutable*. (The\nvalue of an immutable container object that contains a reference to a\nmutable object can change when the latter\'s value is changed; however\nthe container is still considered immutable, because the collection of\nobjects it contains cannot be changed. So, immutability is not\nstrictly the same as having an unchangeable value, it is more subtle.)\nAn object\'s mutability is determined by its type; for instance,\nnumbers, strings and tuples are immutable, while dictionaries and\nlists are mutable.\n\nObjects are never explicitly destroyed; however, when they become\nunreachable they may be garbage-collected. An implementation is\nallowed to postpone garbage collection or omit it altogether --- it is\na matter of implementation quality how garbage collection is\nimplemented, as long as no objects are collected that are still\nreachable.\n\n**CPython implementation detail:** CPython currently uses a reference-\ncounting scheme with (optional) delayed detection of cyclically linked\ngarbage, which collects most objects as soon as they become\nunreachable, but is not guaranteed to collect garbage containing\ncircular references. See the documentation of the ``gc`` module for\ninformation on controlling the collection of cyclic garbage. Other\nimplementations act differently and CPython may change. Do not depend\non immediate finalization of objects when they become unreachable (ex:\nalways close files).\n\nNote that the use of the implementation\'s tracing or debugging\nfacilities may keep objects alive that would normally be collectable.\nAlso note that catching an exception with a \'``try``...``except``\'\nstatement may keep objects alive.\n\nSome objects contain references to "external" resources such as open\nfiles or windows. It is understood that these resources are freed\nwhen the object is garbage-collected, but since garbage collection is\nnot guaranteed to happen, such objects also provide an explicit way to\nrelease the external resource, usually a ``close()`` method. Programs\nare strongly recommended to explicitly close such objects. The\n\'``try``...``finally``\' statement provides a convenient way to do\nthis.\n\nSome objects contain references to other objects; these are called\n*containers*. Examples of containers are tuples, lists and\ndictionaries. The references are part of a container\'s value. In\nmost cases, when we talk about the value of a container, we imply the\nvalues, not the identities of the contained objects; however, when we\ntalk about the mutability of a container, only the identities of the\nimmediately contained objects are implied. So, if an immutable\ncontainer (like a tuple) contains a reference to a mutable object, its\nvalue changes if that mutable object is changed.\n\nTypes affect almost all aspects of object behavior. Even the\nimportance of object identity is affected in some sense: for immutable\ntypes, operations that compute new values may actually return a\nreference to any existing object with the same type and value, while\nfor mutable objects this is not allowed. E.g., after ``a = 1; b =\n1``, ``a`` and ``b`` may or may not refer to the same object with the\nvalue one, depending on the implementation, but after ``c = []; d =\n[]``, ``c`` and ``d`` are guaranteed to refer to two different,\nunique, newly created empty lists. (Note that ``c = d = []`` assigns\nthe same object to both ``c`` and ``d``.)\n', + 'operator-summary': u'\nSummary\n*******\n\nThe following table summarizes the operator precedences in Python,\nfrom lowest precedence (least binding) to highest precedence (most\nbinding). Operators in the same box have the same precedence. Unless\nthe syntax is explicitly given, operators are binary. Operators in\nthe same box group left to right (except for comparisons, including\ntests, which all have the same precedence and chain from left to right\n--- see section *Comparisons* --- and exponentiation, which groups\nfrom right to left).\n\n+-------------------------------------------------+---------------------------------------+\n| Operator | Description |\n+=================================================+=======================================+\n| ``lambda`` | Lambda expression |\n+-------------------------------------------------+---------------------------------------+\n| ``if`` -- ``else`` | Conditional expression |\n+-------------------------------------------------+---------------------------------------+\n| ``or`` | Boolean OR |\n+-------------------------------------------------+---------------------------------------+\n| ``and`` | Boolean AND |\n+-------------------------------------------------+---------------------------------------+\n| ``not`` *x* | Boolean NOT |\n+-------------------------------------------------+---------------------------------------+\n| ``in``, ``not`` ``in``, ``is``, ``is not``, | Comparisons, including membership |\n| ``<``, ``<=``, ``>``, ``>=``, ``<>``, ``!=``, | tests and identity tests, |\n| ``==`` | |\n+-------------------------------------------------+---------------------------------------+\n| ``|`` | Bitwise OR |\n+-------------------------------------------------+---------------------------------------+\n| ``^`` | Bitwise XOR |\n+-------------------------------------------------+---------------------------------------+\n| ``&`` | Bitwise AND |\n+-------------------------------------------------+---------------------------------------+\n| ``<<``, ``>>`` | Shifts |\n+-------------------------------------------------+---------------------------------------+\n| ``+``, ``-`` | Addition and subtraction |\n+-------------------------------------------------+---------------------------------------+\n| ``*``, ``/``, ``//``, ``%`` | Multiplication, division, remainder |\n| | [8] |\n+-------------------------------------------------+---------------------------------------+\n| ``+x``, ``-x``, ``~x`` | Positive, negative, bitwise NOT |\n+-------------------------------------------------+---------------------------------------+\n| ``**`` | Exponentiation [9] |\n+-------------------------------------------------+---------------------------------------+\n| ``x[index]``, ``x[index:index]``, | Subscription, slicing, call, |\n| ``x(arguments...)``, ``x.attribute`` | attribute reference |\n+-------------------------------------------------+---------------------------------------+\n| ``(expressions...)``, ``[expressions...]``, | Binding or tuple display, list |\n| ``{key:datum...}``, ```expressions...``` | display, dictionary display, string |\n| | conversion |\n+-------------------------------------------------+---------------------------------------+\n\n-[ Footnotes ]-\n\n[1] In Python 2.3 and later releases, a list comprehension "leaks" the\n control variables of each ``for`` it contains into the containing\n scope. However, this behavior is deprecated, and relying on it\n will not work in Python 3.0\n\n[2] While ``abs(x%y) < abs(y)`` is true mathematically, for floats it\n may not be true numerically due to roundoff. For example, and\n assuming a platform on which a Python float is an IEEE 754 double-\n precision number, in order that ``-1e-100 % 1e100`` have the same\n sign as ``1e100``, the computed result is ``-1e-100 + 1e100``,\n which is numerically exactly equal to ``1e100``. The function\n ``math.fmod()`` returns a result whose sign matches the sign of\n the first argument instead, and so returns ``-1e-100`` in this\n case. Which approach is more appropriate depends on the\n application.\n\n[3] If x is very close to an exact integer multiple of y, it\'s\n possible for ``floor(x/y)`` to be one larger than ``(x-x%y)/y``\n due to rounding. In such cases, Python returns the latter result,\n in order to preserve that ``divmod(x,y)[0] * y + x % y`` be very\n close to ``x``.\n\n[4] While comparisons between unicode strings make sense at the byte\n level, they may be counter-intuitive to users. For example, the\n strings ``u"\\u00C7"`` and ``u"\\u0043\\u0327"`` compare differently,\n even though they both represent the same unicode character (LATIN\n CAPITAL LETTER C WITH CEDILLA). To compare strings in a human\n recognizable way, compare using ``unicodedata.normalize()``.\n\n[5] The implementation computes this efficiently, without constructing\n lists or sorting.\n\n[6] Earlier versions of Python used lexicographic comparison of the\n sorted (key, value) lists, but this was very expensive for the\n common case of comparing for equality. An even earlier version of\n Python compared dictionaries by identity only, but this caused\n surprises because people expected to be able to test a dictionary\n for emptiness by comparing it to ``{}``.\n\n[7] Due to automatic garbage-collection, free lists, and the dynamic\n nature of descriptors, you may notice seemingly unusual behaviour\n in certain uses of the ``is`` operator, like those involving\n comparisons between instance methods, or constants. Check their\n documentation for more info.\n\n[8] The ``%`` operator is also used for string formatting; the same\n precedence applies.\n\n[9] The power operator ``**`` binds less tightly than an arithmetic or\n bitwise unary operator on its right, that is, ``2**-1`` is\n ``0.5``.\n', 'pass': u'\nThe ``pass`` statement\n**********************\n\n pass_stmt ::= "pass"\n\n``pass`` is a null operation --- when it is executed, nothing happens.\nIt is useful as a placeholder when a statement is required\nsyntactically, but no code needs to be executed, for example:\n\n def f(arg): pass # a function that does nothing (yet)\n\n class C: pass # a class with no methods (yet)\n', 'power': u'\nThe power operator\n******************\n\nThe power operator binds more tightly than unary operators on its\nleft; it binds less tightly than unary operators on its right. The\nsyntax is:\n\n power ::= primary ["**" u_expr]\n\nThus, in an unparenthesized sequence of power and unary operators, the\noperators are evaluated from right to left (this does not constrain\nthe evaluation order for the operands): ``-1**2`` results in ``-1``.\n\nThe power operator has the same semantics as the built-in ``pow()``\nfunction, when called with two arguments: it yields its left argument\nraised to the power of its right argument. The numeric arguments are\nfirst converted to a common type. The result type is that of the\narguments after coercion.\n\nWith mixed operand types, the coercion rules for binary arithmetic\noperators apply. For int and long int operands, the result has the\nsame type as the operands (after coercion) unless the second argument\nis negative; in that case, all arguments are converted to float and a\nfloat result is delivered. For example, ``10**2`` returns ``100``, but\n``10**-2`` returns ``0.01``. (This last feature was added in Python\n2.2. In Python 2.1 and before, if both arguments were of integer types\nand the second argument was negative, an exception was raised).\n\nRaising ``0.0`` to a negative power results in a\n``ZeroDivisionError``. Raising a negative number to a fractional power\nresults in a ``ValueError``.\n', 'print': u'\nThe ``print`` statement\n***********************\n\n print_stmt ::= "print" ([expression ("," expression)* [","]]\n | ">>" expression [("," expression)+ [","]])\n\n``print`` evaluates each expression in turn and writes the resulting\nobject to standard output (see below). If an object is not a string,\nit is first converted to a string using the rules for string\nconversions. The (resulting or original) string is then written. A\nspace is written before each object is (converted and) written, unless\nthe output system believes it is positioned at the beginning of a\nline. This is the case (1) when no characters have yet been written\nto standard output, (2) when the last character written to standard\noutput is a whitespace character except ``\' \'``, or (3) when the last\nwrite operation on standard output was not a ``print`` statement. (In\nsome cases it may be functional to write an empty string to standard\noutput for this reason.)\n\nNote: Objects which act like file objects but which are not the built-in\n file objects often do not properly emulate this aspect of the file\n object\'s behavior, so it is best not to rely on this.\n\nA ``\'\\n\'`` character is written at the end, unless the ``print``\nstatement ends with a comma. This is the only action if the statement\ncontains just the keyword ``print``.\n\nStandard output is defined as the file object named ``stdout`` in the\nbuilt-in module ``sys``. If no such object exists, or if it does not\nhave a ``write()`` method, a ``RuntimeError`` exception is raised.\n\n``print`` also has an extended form, defined by the second portion of\nthe syntax described above. This form is sometimes referred to as\n"``print`` chevron." In this form, the first expression after the\n``>>`` must evaluate to a "file-like" object, specifically an object\nthat has a ``write()`` method as described above. With this extended\nform, the subsequent expressions are printed to this file object. If\nthe first expression evaluates to ``None``, then ``sys.stdout`` is\nused as the file for output.\n', @@ -63,21 +63,21 @@ 'shifting': u'\nShifting operations\n*******************\n\nThe shifting operations have lower priority than the arithmetic\noperations:\n\n shift_expr ::= a_expr | shift_expr ( "<<" | ">>" ) a_expr\n\nThese operators accept plain or long integers as arguments. The\narguments are converted to a common type. They shift the first\nargument to the left or right by the number of bits given by the\nsecond argument.\n\nA right shift by *n* bits is defined as division by ``pow(2, n)``. A\nleft shift by *n* bits is defined as multiplication with ``pow(2,\nn)``. Negative shift counts raise a ``ValueError`` exception.\n\nNote: In the current implementation, the right-hand operand is required to\n be at most ``sys.maxsize``. If the right-hand operand is larger\n than ``sys.maxsize`` an ``OverflowError`` exception is raised.\n', 'slicings': u'\nSlicings\n********\n\nA slicing selects a range of items in a sequence object (e.g., a\nstring, tuple or list). Slicings may be used as expressions or as\ntargets in assignment or ``del`` statements. The syntax for a\nslicing:\n\n slicing ::= simple_slicing | extended_slicing\n simple_slicing ::= primary "[" short_slice "]"\n extended_slicing ::= primary "[" slice_list "]"\n slice_list ::= slice_item ("," slice_item)* [","]\n slice_item ::= expression | proper_slice | ellipsis\n proper_slice ::= short_slice | long_slice\n short_slice ::= [lower_bound] ":" [upper_bound]\n long_slice ::= short_slice ":" [stride]\n lower_bound ::= expression\n upper_bound ::= expression\n stride ::= expression\n ellipsis ::= "..."\n\nThere is ambiguity in the formal syntax here: anything that looks like\nan expression list also looks like a slice list, so any subscription\ncan be interpreted as a slicing. Rather than further complicating the\nsyntax, this is disambiguated by defining that in this case the\ninterpretation as a subscription takes priority over the\ninterpretation as a slicing (this is the case if the slice list\ncontains no proper slice nor ellipses). Similarly, when the slice\nlist has exactly one short slice and no trailing comma, the\ninterpretation as a simple slicing takes priority over that as an\nextended slicing.\n\nThe semantics for a simple slicing are as follows. The primary must\nevaluate to a sequence object. The lower and upper bound expressions,\nif present, must evaluate to plain integers; defaults are zero and the\n``sys.maxint``, respectively. If either bound is negative, the\nsequence\'s length is added to it. The slicing now selects all items\nwith index *k* such that ``i <= k < j`` where *i* and *j* are the\nspecified lower and upper bounds. This may be an empty sequence. It\nis not an error if *i* or *j* lie outside the range of valid indexes\n(such items don\'t exist so they aren\'t selected).\n\nThe semantics for an extended slicing are as follows. The primary\nmust evaluate to a mapping object, and it is indexed with a key that\nis constructed from the slice list, as follows. If the slice list\ncontains at least one comma, the key is a tuple containing the\nconversion of the slice items; otherwise, the conversion of the lone\nslice item is the key. The conversion of a slice item that is an\nexpression is that expression. The conversion of an ellipsis slice\nitem is the built-in ``Ellipsis`` object. The conversion of a proper\nslice is a slice object (see section *The standard type hierarchy*)\nwhose ``start``, ``stop`` and ``step`` attributes are the values of\nthe expressions given as lower bound, upper bound and stride,\nrespectively, substituting ``None`` for missing expressions.\n', 'specialattrs': u"\nSpecial Attributes\n******************\n\nThe implementation adds a few special read-only attributes to several\nobject types, where they are relevant. Some of these are not reported\nby the ``dir()`` built-in function.\n\nobject.__dict__\n\n A dictionary or other mapping object used to store an object's\n (writable) attributes.\n\nobject.__methods__\n\n Deprecated since version 2.2: Use the built-in function ``dir()``\n to get a list of an object's attributes. This attribute is no\n longer available.\n\nobject.__members__\n\n Deprecated since version 2.2: Use the built-in function ``dir()``\n to get a list of an object's attributes. This attribute is no\n longer available.\n\ninstance.__class__\n\n The class to which a class instance belongs.\n\nclass.__bases__\n\n The tuple of base classes of a class object.\n\nclass.__name__\n\n The name of the class or type.\n\nThe following attributes are only supported by *new-style class*es.\n\nclass.__mro__\n\n This attribute is a tuple of classes that are considered when\n looking for base classes during method resolution.\n\nclass.mro()\n\n This method can be overridden by a metaclass to customize the\n method resolution order for its instances. It is called at class\n instantiation, and its result is stored in ``__mro__``.\n\nclass.__subclasses__()\n\n Each new-style class keeps a list of weak references to its\n immediate subclasses. This method returns a list of all those\n references still alive. Example:\n\n >>> int.__subclasses__()\n [<type 'bool'>]\n\n-[ Footnotes ]-\n\n[1] Additional information on these special methods may be found in\n the Python Reference Manual (*Basic customization*).\n\n[2] As a consequence, the list ``[1, 2]`` is considered equal to\n ``[1.0, 2.0]``, and similarly for tuples.\n\n[3] They must have since the parser can't tell the type of the\n operands.\n\n[4] To format only a tuple you should therefore provide a singleton\n tuple whose only element is the tuple to be formatted.\n\n[5] The advantage of leaving the newline on is that returning an empty\n string is then an unambiguous EOF indication. It is also possible\n (in cases where it might matter, for example, if you want to make\n an exact copy of a file while scanning its lines) to tell whether\n the last line of a file ended in a newline or not (yes this\n happens!).\n", - 'specialnames': u'\nSpecial method names\n********************\n\nA class can implement certain operations that are invoked by special\nsyntax (such as arithmetic operations or subscripting and slicing) by\ndefining methods with special names. This is Python\'s approach to\n*operator overloading*, allowing classes to define their own behavior\nwith respect to language operators. For instance, if a class defines\na method named ``__getitem__()``, and ``x`` is an instance of this\nclass, then ``x[i]`` is roughly equivalent to ``x.__getitem__(i)`` for\nold-style classes and ``type(x).__getitem__(x, i)`` for new-style\nclasses. Except where mentioned, attempts to execute an operation\nraise an exception when no appropriate method is defined (typically\n``AttributeError`` or ``TypeError``).\n\nWhen implementing a class that emulates any built-in type, it is\nimportant that the emulation only be implemented to the degree that it\nmakes sense for the object being modelled. For example, some\nsequences may work well with retrieval of individual elements, but\nextracting a slice may not make sense. (One example of this is the\n``NodeList`` interface in the W3C\'s Document Object Model.)\n\n\nBasic customization\n===================\n\nobject.__new__(cls[, ...])\n\n Called to create a new instance of class *cls*. ``__new__()`` is a\n static method (special-cased so you need not declare it as such)\n that takes the class of which an instance was requested as its\n first argument. The remaining arguments are those passed to the\n object constructor expression (the call to the class). The return\n value of ``__new__()`` should be the new object instance (usually\n an instance of *cls*).\n\n Typical implementations create a new instance of the class by\n invoking the superclass\'s ``__new__()`` method using\n ``super(currentclass, cls).__new__(cls[, ...])`` with appropriate\n arguments and then modifying the newly-created instance as\n necessary before returning it.\n\n If ``__new__()`` returns an instance of *cls*, then the new\n instance\'s ``__init__()`` method will be invoked like\n ``__init__(self[, ...])``, where *self* is the new instance and the\n remaining arguments are the same as were passed to ``__new__()``.\n\n If ``__new__()`` does not return an instance of *cls*, then the new\n instance\'s ``__init__()`` method will not be invoked.\n\n ``__new__()`` is intended mainly to allow subclasses of immutable\n types (like int, str, or tuple) to customize instance creation. It\n is also commonly overridden in custom metaclasses in order to\n customize class creation.\n\nobject.__init__(self[, ...])\n\n Called when the instance is created. The arguments are those\n passed to the class constructor expression. If a base class has an\n ``__init__()`` method, the derived class\'s ``__init__()`` method,\n if any, must explicitly call it to ensure proper initialization of\n the base class part of the instance; for example:\n ``BaseClass.__init__(self, [args...])``. As a special constraint\n on constructors, no value may be returned; doing so will cause a\n ``TypeError`` to be raised at runtime.\n\nobject.__del__(self)\n\n Called when the instance is about to be destroyed. This is also\n called a destructor. If a base class has a ``__del__()`` method,\n the derived class\'s ``__del__()`` method, if any, must explicitly\n call it to ensure proper deletion of the base class part of the\n instance. Note that it is possible (though not recommended!) for\n the ``__del__()`` method to postpone destruction of the instance by\n creating a new reference to it. It may then be called at a later\n time when this new reference is deleted. It is not guaranteed that\n ``__del__()`` methods are called for objects that still exist when\n the interpreter exits.\n\n Note: ``del x`` doesn\'t directly call ``x.__del__()`` --- the former\n decrements the reference count for ``x`` by one, and the latter\n is only called when ``x``\'s reference count reaches zero. Some\n common situations that may prevent the reference count of an\n object from going to zero include: circular references between\n objects (e.g., a doubly-linked list or a tree data structure with\n parent and child pointers); a reference to the object on the\n stack frame of a function that caught an exception (the traceback\n stored in ``sys.exc_traceback`` keeps the stack frame alive); or\n a reference to the object on the stack frame that raised an\n unhandled exception in interactive mode (the traceback stored in\n ``sys.last_traceback`` keeps the stack frame alive). The first\n situation can only be remedied by explicitly breaking the cycles;\n the latter two situations can be resolved by storing ``None`` in\n ``sys.exc_traceback`` or ``sys.last_traceback``. Circular\n references which are garbage are detected when the option cycle\n detector is enabled (it\'s on by default), but can only be cleaned\n up if there are no Python-level ``__del__()`` methods involved.\n Refer to the documentation for the ``gc`` module for more\n information about how ``__del__()`` methods are handled by the\n cycle detector, particularly the description of the ``garbage``\n value.\n\n Warning: Due to the precarious circumstances under which ``__del__()``\n methods are invoked, exceptions that occur during their execution\n are ignored, and a warning is printed to ``sys.stderr`` instead.\n Also, when ``__del__()`` is invoked in response to a module being\n deleted (e.g., when execution of the program is done), other\n globals referenced by the ``__del__()`` method may already have\n been deleted or in the process of being torn down (e.g. the\n import machinery shutting down). For this reason, ``__del__()``\n methods should do the absolute minimum needed to maintain\n external invariants. Starting with version 1.5, Python\n guarantees that globals whose name begins with a single\n underscore are deleted from their module before other globals are\n deleted; if no other references to such globals exist, this may\n help in assuring that imported modules are still available at the\n time when the ``__del__()`` method is called.\n\nobject.__repr__(self)\n\n Called by the ``repr()`` built-in function and by string\n conversions (reverse quotes) to compute the "official" string\n representation of an object. If at all possible, this should look\n like a valid Python expression that could be used to recreate an\n object with the same value (given an appropriate environment). If\n this is not possible, a string of the form ``<...some useful\n description...>`` should be returned. The return value must be a\n string object. If a class defines ``__repr__()`` but not\n ``__str__()``, then ``__repr__()`` is also used when an "informal"\n string representation of instances of that class is required.\n\n This is typically used for debugging, so it is important that the\n representation is information-rich and unambiguous.\n\nobject.__str__(self)\n\n Called by the ``str()`` built-in function and by the ``print``\n statement to compute the "informal" string representation of an\n object. This differs from ``__repr__()`` in that it does not have\n to be a valid Python expression: a more convenient or concise\n representation may be used instead. The return value must be a\n string object.\n\nobject.__lt__(self, other)\nobject.__le__(self, other)\nobject.__eq__(self, other)\nobject.__ne__(self, other)\nobject.__gt__(self, other)\nobject.__ge__(self, other)\n\n New in version 2.1.\n\n These are the so-called "rich comparison" methods, and are called\n for comparison operators in preference to ``__cmp__()`` below. The\n correspondence between operator symbols and method names is as\n follows: ``x<y`` calls ``x.__lt__(y)``, ``x<=y`` calls\n ``x.__le__(y)``, ``x==y`` calls ``x.__eq__(y)``, ``x!=y`` and\n ``x<>y`` call ``x.__ne__(y)``, ``x>y`` calls ``x.__gt__(y)``, and\n ``x>=y`` calls ``x.__ge__(y)``.\n\n A rich comparison method may return the singleton\n ``NotImplemented`` if it does not implement the operation for a\n given pair of arguments. By convention, ``False`` and ``True`` are\n returned for a successful comparison. However, these methods can\n return any value, so if the comparison operator is used in a\n Boolean context (e.g., in the condition of an ``if`` statement),\n Python will call ``bool()`` on the value to determine if the result\n is true or false.\n\n There are no implied relationships among the comparison operators.\n The truth of ``x==y`` does not imply that ``x!=y`` is false.\n Accordingly, when defining ``__eq__()``, one should also define\n ``__ne__()`` so that the operators will behave as expected. See\n the paragraph on ``__hash__()`` for some important notes on\n creating *hashable* objects which support custom comparison\n operations and are usable as dictionary keys.\n\n There are no swapped-argument versions of these methods (to be used\n when the left argument does not support the operation but the right\n argument does); rather, ``__lt__()`` and ``__gt__()`` are each\n other\'s reflection, ``__le__()`` and ``__ge__()`` are each other\'s\n reflection, and ``__eq__()`` and ``__ne__()`` are their own\n reflection.\n\n Arguments to rich comparison methods are never coerced.\n\n To automatically generate ordering operations from a single root\n operation, see ``functools.total_ordering()``.\n\nobject.__cmp__(self, other)\n\n Called by comparison operations if rich comparison (see above) is\n not defined. Should return a negative integer if ``self < other``,\n zero if ``self == other``, a positive integer if ``self > other``.\n If no ``__cmp__()``, ``__eq__()`` or ``__ne__()`` operation is\n defined, class instances are compared by object identity\n ("address"). See also the description of ``__hash__()`` for some\n important notes on creating *hashable* objects which support custom\n comparison operations and are usable as dictionary keys. (Note: the\n restriction that exceptions are not propagated by ``__cmp__()`` has\n been removed since Python 1.5.)\n\nobject.__rcmp__(self, other)\n\n Changed in version 2.1: No longer supported.\n\nobject.__hash__(self)\n\n Called by built-in function ``hash()`` and for operations on\n members of hashed collections including ``set``, ``frozenset``, and\n ``dict``. ``__hash__()`` should return an integer. The only\n required property is that objects which compare equal have the same\n hash value; it is advised to somehow mix together (e.g. using\n exclusive or) the hash values for the components of the object that\n also play a part in comparison of objects.\n\n If a class does not define a ``__cmp__()`` or ``__eq__()`` method\n it should not define a ``__hash__()`` operation either; if it\n defines ``__cmp__()`` or ``__eq__()`` but not ``__hash__()``, its\n instances will not be usable in hashed collections. If a class\n defines mutable objects and implements a ``__cmp__()`` or\n ``__eq__()`` method, it should not implement ``__hash__()``, since\n hashable collection implementations require that a object\'s hash\n value is immutable (if the object\'s hash value changes, it will be\n in the wrong hash bucket).\n\n User-defined classes have ``__cmp__()`` and ``__hash__()`` methods\n by default; with them, all objects compare unequal (except with\n themselves) and ``x.__hash__()`` returns ``id(x)``.\n\n Classes which inherit a ``__hash__()`` method from a parent class\n but change the meaning of ``__cmp__()`` or ``__eq__()`` such that\n the hash value returned is no longer appropriate (e.g. by switching\n to a value-based concept of equality instead of the default\n identity based equality) can explicitly flag themselves as being\n unhashable by setting ``__hash__ = None`` in the class definition.\n Doing so means that not only will instances of the class raise an\n appropriate ``TypeError`` when a program attempts to retrieve their\n hash value, but they will also be correctly identified as\n unhashable when checking ``isinstance(obj, collections.Hashable)``\n (unlike classes which define their own ``__hash__()`` to explicitly\n raise ``TypeError``).\n\n Changed in version 2.5: ``__hash__()`` may now also return a long\n integer object; the 32-bit integer is then derived from the hash of\n that object.\n\n Changed in version 2.6: ``__hash__`` may now be set to ``None`` to\n explicitly flag instances of a class as unhashable.\n\nobject.__nonzero__(self)\n\n Called to implement truth value testing and the built-in operation\n ``bool()``; should return ``False`` or ``True``, or their integer\n equivalents ``0`` or ``1``. When this method is not defined,\n ``__len__()`` is called, if it is defined, and the object is\n considered true if its result is nonzero. If a class defines\n neither ``__len__()`` nor ``__nonzero__()``, all its instances are\n considered true.\n\nobject.__unicode__(self)\n\n Called to implement ``unicode()`` built-in; should return a Unicode\n object. When this method is not defined, string conversion is\n attempted, and the result of string conversion is converted to\n Unicode using the system default encoding.\n\n\nCustomizing attribute access\n============================\n\nThe following methods can be defined to customize the meaning of\nattribute access (use of, assignment to, or deletion of ``x.name``)\nfor class instances.\n\nobject.__getattr__(self, name)\n\n Called when an attribute lookup has not found the attribute in the\n usual places (i.e. it is not an instance attribute nor is it found\n in the class tree for ``self``). ``name`` is the attribute name.\n This method should return the (computed) attribute value or raise\n an ``AttributeError`` exception.\n\n Note that if the attribute is found through the normal mechanism,\n ``__getattr__()`` is not called. (This is an intentional asymmetry\n between ``__getattr__()`` and ``__setattr__()``.) This is done both\n for efficiency reasons and because otherwise ``__getattr__()``\n would have no way to access other attributes of the instance. Note\n that at least for instance variables, you can fake total control by\n not inserting any values in the instance attribute dictionary (but\n instead inserting them in another object). See the\n ``__getattribute__()`` method below for a way to actually get total\n control in new-style classes.\n\nobject.__setattr__(self, name, value)\n\n Called when an attribute assignment is attempted. This is called\n instead of the normal mechanism (i.e. store the value in the\n instance dictionary). *name* is the attribute name, *value* is the\n value to be assigned to it.\n\n If ``__setattr__()`` wants to assign to an instance attribute, it\n should not simply execute ``self.name = value`` --- this would\n cause a recursive call to itself. Instead, it should insert the\n value in the dictionary of instance attributes, e.g.,\n ``self.__dict__[name] = value``. For new-style classes, rather\n than accessing the instance dictionary, it should call the base\n class method with the same name, for example,\n ``object.__setattr__(self, name, value)``.\n\nobject.__delattr__(self, name)\n\n Like ``__setattr__()`` but for attribute deletion instead of\n assignment. This should only be implemented if ``del obj.name`` is\n meaningful for the object.\n\n\nMore attribute access for new-style classes\n-------------------------------------------\n\nThe following methods only apply to new-style classes.\n\nobject.__getattribute__(self, name)\n\n Called unconditionally to implement attribute accesses for\n instances of the class. If the class also defines\n ``__getattr__()``, the latter will not be called unless\n ``__getattribute__()`` either calls it explicitly or raises an\n ``AttributeError``. This method should return the (computed)\n attribute value or raise an ``AttributeError`` exception. In order\n to avoid infinite recursion in this method, its implementation\n should always call the base class method with the same name to\n access any attributes it needs, for example,\n ``object.__getattribute__(self, name)``.\n\n Note: This method may still be bypassed when looking up special methods\n as the result of implicit invocation via language syntax or\n built-in functions. See *Special method lookup for new-style\n classes*.\n\n\nImplementing Descriptors\n------------------------\n\nThe following methods only apply when an instance of the class\ncontaining the method (a so-called *descriptor* class) appears in the\nclass dictionary of another new-style class, known as the *owner*\nclass. In the examples below, "the attribute" refers to the attribute\nwhose name is the key of the property in the owner class\'\n``__dict__``. Descriptors can only be implemented as new-style\nclasses themselves.\n\nobject.__get__(self, instance, owner)\n\n Called to get the attribute of the owner class (class attribute\n access) or of an instance of that class (instance attribute\n access). *owner* is always the owner class, while *instance* is the\n instance that the attribute was accessed through, or ``None`` when\n the attribute is accessed through the *owner*. This method should\n return the (computed) attribute value or raise an\n ``AttributeError`` exception.\n\nobject.__set__(self, instance, value)\n\n Called to set the attribute on an instance *instance* of the owner\n class to a new value, *value*.\n\nobject.__delete__(self, instance)\n\n Called to delete the attribute on an instance *instance* of the\n owner class.\n\n\nInvoking Descriptors\n--------------------\n\nIn general, a descriptor is an object attribute with "binding\nbehavior", one whose attribute access has been overridden by methods\nin the descriptor protocol: ``__get__()``, ``__set__()``, and\n``__delete__()``. If any of those methods are defined for an object,\nit is said to be a descriptor.\n\nThe default behavior for attribute access is to get, set, or delete\nthe attribute from an object\'s dictionary. For instance, ``a.x`` has a\nlookup chain starting with ``a.__dict__[\'x\']``, then\n``type(a).__dict__[\'x\']``, and continuing through the base classes of\n``type(a)`` excluding metaclasses.\n\nHowever, if the looked-up value is an object defining one of the\ndescriptor methods, then Python may override the default behavior and\ninvoke the descriptor method instead. Where this occurs in the\nprecedence chain depends on which descriptor methods were defined and\nhow they were called. Note that descriptors are only invoked for new\nstyle objects or classes (ones that subclass ``object()`` or\n``type()``).\n\nThe starting point for descriptor invocation is a binding, ``a.x``.\nHow the arguments are assembled depends on ``a``:\n\nDirect Call\n The simplest and least common call is when user code directly\n invokes a descriptor method: ``x.__get__(a)``.\n\nInstance Binding\n If binding to a new-style object instance, ``a.x`` is transformed\n into the call: ``type(a).__dict__[\'x\'].__get__(a, type(a))``.\n\nClass Binding\n If binding to a new-style class, ``A.x`` is transformed into the\n call: ``A.__dict__[\'x\'].__get__(None, A)``.\n\nSuper Binding\n If ``a`` is an instance of ``super``, then the binding ``super(B,\n obj).m()`` searches ``obj.__class__.__mro__`` for the base class\n ``A`` immediately preceding ``B`` and then invokes the descriptor\n with the call: ``A.__dict__[\'m\'].__get__(obj, A)``.\n\nFor instance bindings, the precedence of descriptor invocation depends\non the which descriptor methods are defined. A descriptor can define\nany combination of ``__get__()``, ``__set__()`` and ``__delete__()``.\nIf it does not define ``__get__()``, then accessing the attribute will\nreturn the descriptor object itself unless there is a value in the\nobject\'s instance dictionary. If the descriptor defines ``__set__()``\nand/or ``__delete__()``, it is a data descriptor; if it defines\nneither, it is a non-data descriptor. Normally, data descriptors\ndefine both ``__get__()`` and ``__set__()``, while non-data\ndescriptors have just the ``__get__()`` method. Data descriptors with\n``__set__()`` and ``__get__()`` defined always override a redefinition\nin an instance dictionary. In contrast, non-data descriptors can be\noverridden by instances.\n\nPython methods (including ``staticmethod()`` and ``classmethod()``)\nare implemented as non-data descriptors. Accordingly, instances can\nredefine and override methods. This allows individual instances to\nacquire behaviors that differ from other instances of the same class.\n\nThe ``property()`` function is implemented as a data descriptor.\nAccordingly, instances cannot override the behavior of a property.\n\n\n__slots__\n---------\n\nBy default, instances of both old and new-style classes have a\ndictionary for attribute storage. This wastes space for objects\nhaving very few instance variables. The space consumption can become\nacute when creating large numbers of instances.\n\nThe default can be overridden by defining *__slots__* in a new-style\nclass definition. The *__slots__* declaration takes a sequence of\ninstance variables and reserves just enough space in each instance to\nhold a value for each variable. Space is saved because *__dict__* is\nnot created for each instance.\n\n__slots__\n\n This class variable can be assigned a string, iterable, or sequence\n of strings with variable names used by instances. If defined in a\n new-style class, *__slots__* reserves space for the declared\n variables and prevents the automatic creation of *__dict__* and\n *__weakref__* for each instance.\n\n New in version 2.2.\n\nNotes on using *__slots__*\n\n* When inheriting from a class without *__slots__*, the *__dict__*\n attribute of that class will always be accessible, so a *__slots__*\n definition in the subclass is meaningless.\n\n* Without a *__dict__* variable, instances cannot be assigned new\n variables not listed in the *__slots__* definition. Attempts to\n assign to an unlisted variable name raises ``AttributeError``. If\n dynamic assignment of new variables is desired, then add\n ``\'__dict__\'`` to the sequence of strings in the *__slots__*\n declaration.\n\n Changed in version 2.3: Previously, adding ``\'__dict__\'`` to the\n *__slots__* declaration would not enable the assignment of new\n attributes not specifically listed in the sequence of instance\n variable names.\n\n* Without a *__weakref__* variable for each instance, classes defining\n *__slots__* do not support weak references to its instances. If weak\n reference support is needed, then add ``\'__weakref__\'`` to the\n sequence of strings in the *__slots__* declaration.\n\n Changed in version 2.3: Previously, adding ``\'__weakref__\'`` to the\n *__slots__* declaration would not enable support for weak\n references.\n\n* *__slots__* are implemented at the class level by creating\n descriptors (*Implementing Descriptors*) for each variable name. As\n a result, class attributes cannot be used to set default values for\n instance variables defined by *__slots__*; otherwise, the class\n attribute would overwrite the descriptor assignment.\n\n* The action of a *__slots__* declaration is limited to the class\n where it is defined. As a result, subclasses will have a *__dict__*\n unless they also define *__slots__* (which must only contain names\n of any *additional* slots).\n\n* If a class defines a slot also defined in a base class, the instance\n variable defined by the base class slot is inaccessible (except by\n retrieving its descriptor directly from the base class). This\n renders the meaning of the program undefined. In the future, a\n check may be added to prevent this.\n\n* Nonempty *__slots__* does not work for classes derived from\n "variable-length" built-in types such as ``long``, ``str`` and\n ``tuple``.\n\n* Any non-string iterable may be assigned to *__slots__*. Mappings may\n also be used; however, in the future, special meaning may be\n assigned to the values corresponding to each key.\n\n* *__class__* assignment works only if both classes have the same\n *__slots__*.\n\n Changed in version 2.6: Previously, *__class__* assignment raised an\n error if either new or old class had *__slots__*.\n\n\nCustomizing class creation\n==========================\n\nBy default, new-style classes are constructed using ``type()``. A\nclass definition is read into a separate namespace and the value of\nclass name is bound to the result of ``type(name, bases, dict)``.\n\nWhen the class definition is read, if *__metaclass__* is defined then\nthe callable assigned to it will be called instead of ``type()``. This\nallows classes or functions to be written which monitor or alter the\nclass creation process:\n\n* Modifying the class dictionary prior to the class being created.\n\n* Returning an instance of another class -- essentially performing the\n role of a factory function.\n\nThese steps will have to be performed in the metaclass\'s ``__new__()``\nmethod -- ``type.__new__()`` can then be called from this method to\ncreate a class with different properties. This example adds a new\nelement to the class dictionary before creating the class:\n\n class metacls(type):\n def __new__(mcs, name, bases, dict):\n dict[\'foo\'] = \'metacls was here\'\n return type.__new__(mcs, name, bases, dict)\n\nYou can of course also override other class methods (or add new\nmethods); for example defining a custom ``__call__()`` method in the\nmetaclass allows custom behavior when the class is called, e.g. not\nalways creating a new instance.\n\n__metaclass__\n\n This variable can be any callable accepting arguments for ``name``,\n ``bases``, and ``dict``. Upon class creation, the callable is used\n instead of the built-in ``type()``.\n\n New in version 2.2.\n\nThe appropriate metaclass is determined by the following precedence\nrules:\n\n* If ``dict[\'__metaclass__\']`` exists, it is used.\n\n* Otherwise, if there is at least one base class, its metaclass is\n used (this looks for a *__class__* attribute first and if not found,\n uses its type).\n\n* Otherwise, if a global variable named __metaclass__ exists, it is\n used.\n\n* Otherwise, the old-style, classic metaclass (types.ClassType) is\n used.\n\nThe potential uses for metaclasses are boundless. Some ideas that have\nbeen explored including logging, interface checking, automatic\ndelegation, automatic property creation, proxies, frameworks, and\nautomatic resource locking/synchronization.\n\n\nCustomizing instance and subclass checks\n========================================\n\nNew in version 2.6.\n\nThe following methods are used to override the default behavior of the\n``isinstance()`` and ``issubclass()`` built-in functions.\n\nIn particular, the metaclass ``abc.ABCMeta`` implements these methods\nin order to allow the addition of Abstract Base Classes (ABCs) as\n"virtual base classes" to any class or type (including built-in\ntypes), including other ABCs.\n\nclass.__instancecheck__(self, instance)\n\n Return true if *instance* should be considered a (direct or\n indirect) instance of *class*. If defined, called to implement\n ``isinstance(instance, class)``.\n\nclass.__subclasscheck__(self, subclass)\n\n Return true if *subclass* should be considered a (direct or\n indirect) subclass of *class*. If defined, called to implement\n ``issubclass(subclass, class)``.\n\nNote that these methods are looked up on the type (metaclass) of a\nclass. They cannot be defined as class methods in the actual class.\nThis is consistent with the lookup of special methods that are called\non instances, only in this case the instance is itself a class.\n\nSee also:\n\n **PEP 3119** - Introducing Abstract Base Classes\n Includes the specification for customizing ``isinstance()`` and\n ``issubclass()`` behavior through ``__instancecheck__()`` and\n ``__subclasscheck__()``, with motivation for this functionality\n in the context of adding Abstract Base Classes (see the ``abc``\n module) to the language.\n\n\nEmulating callable objects\n==========================\n\nobject.__call__(self[, args...])\n\n Called when the instance is "called" as a function; if this method\n is defined, ``x(arg1, arg2, ...)`` is a shorthand for\n ``x.__call__(arg1, arg2, ...)``.\n\n\nEmulating container types\n=========================\n\nThe following methods can be defined to implement container objects.\nContainers usually are sequences (such as lists or tuples) or mappings\n(like dictionaries), but can represent other containers as well. The\nfirst set of methods is used either to emulate a sequence or to\nemulate a mapping; the difference is that for a sequence, the\nallowable keys should be the integers *k* for which ``0 <= k < N``\nwhere *N* is the length of the sequence, or slice objects, which\ndefine a range of items. (For backwards compatibility, the method\n``__getslice__()`` (see below) can also be defined to handle simple,\nbut not extended slices.) It is also recommended that mappings provide\nthe methods ``keys()``, ``values()``, ``items()``, ``has_key()``,\n``get()``, ``clear()``, ``setdefault()``, ``iterkeys()``,\n``itervalues()``, ``iteritems()``, ``pop()``, ``popitem()``,\n``copy()``, and ``update()`` behaving similar to those for Python\'s\nstandard dictionary objects. The ``UserDict`` module provides a\n``DictMixin`` class to help create those methods from a base set of\n``__getitem__()``, ``__setitem__()``, ``__delitem__()``, and\n``keys()``. Mutable sequences should provide methods ``append()``,\n``count()``, ``index()``, ``extend()``, ``insert()``, ``pop()``,\n``remove()``, ``reverse()`` and ``sort()``, like Python standard list\nobjects. Finally, sequence types should implement addition (meaning\nconcatenation) and multiplication (meaning repetition) by defining the\nmethods ``__add__()``, ``__radd__()``, ``__iadd__()``, ``__mul__()``,\n``__rmul__()`` and ``__imul__()`` described below; they should not\ndefine ``__coerce__()`` or other numerical operators. It is\nrecommended that both mappings and sequences implement the\n``__contains__()`` method to allow efficient use of the ``in``\noperator; for mappings, ``in`` should be equivalent of ``has_key()``;\nfor sequences, it should search through the values. It is further\nrecommended that both mappings and sequences implement the\n``__iter__()`` method to allow efficient iteration through the\ncontainer; for mappings, ``__iter__()`` should be the same as\n``iterkeys()``; for sequences, it should iterate through the values.\n\nobject.__len__(self)\n\n Called to implement the built-in function ``len()``. Should return\n the length of the object, an integer ``>=`` 0. Also, an object\n that doesn\'t define a ``__nonzero__()`` method and whose\n ``__len__()`` method returns zero is considered to be false in a\n Boolean context.\n\nobject.__getitem__(self, key)\n\n Called to implement evaluation of ``self[key]``. For sequence\n types, the accepted keys should be integers and slice objects.\n Note that the special interpretation of negative indexes (if the\n class wishes to emulate a sequence type) is up to the\n ``__getitem__()`` method. If *key* is of an inappropriate type,\n ``TypeError`` may be raised; if of a value outside the set of\n indexes for the sequence (after any special interpretation of\n negative values), ``IndexError`` should be raised. For mapping\n types, if *key* is missing (not in the container), ``KeyError``\n should be raised.\n\n Note: ``for`` loops expect that an ``IndexError`` will be raised for\n illegal indexes to allow proper detection of the end of the\n sequence.\n\nobject.__setitem__(self, key, value)\n\n Called to implement assignment to ``self[key]``. Same note as for\n ``__getitem__()``. This should only be implemented for mappings if\n the objects support changes to the values for keys, or if new keys\n can be added, or for sequences if elements can be replaced. The\n same exceptions should be raised for improper *key* values as for\n the ``__getitem__()`` method.\n\nobject.__delitem__(self, key)\n\n Called to implement deletion of ``self[key]``. Same note as for\n ``__getitem__()``. This should only be implemented for mappings if\n the objects support removal of keys, or for sequences if elements\n can be removed from the sequence. The same exceptions should be\n raised for improper *key* values as for the ``__getitem__()``\n method.\n\nobject.__iter__(self)\n\n This method is called when an iterator is required for a container.\n This method should return a new iterator object that can iterate\n over all the objects in the container. For mappings, it should\n iterate over the keys of the container, and should also be made\n available as the method ``iterkeys()``.\n\n Iterator objects also need to implement this method; they are\n required to return themselves. For more information on iterator\n objects, see *Iterator Types*.\n\nobject.__reversed__(self)\n\n Called (if present) by the ``reversed()`` built-in to implement\n reverse iteration. It should return a new iterator object that\n iterates over all the objects in the container in reverse order.\n\n If the ``__reversed__()`` method is not provided, the\n ``reversed()`` built-in will fall back to using the sequence\n protocol (``__len__()`` and ``__getitem__()``). Objects that\n support the sequence protocol should only provide\n ``__reversed__()`` if they can provide an implementation that is\n more efficient than the one provided by ``reversed()``.\n\n New in version 2.6.\n\nThe membership test operators (``in`` and ``not in``) are normally\nimplemented as an iteration through a sequence. However, container\nobjects can supply the following special method with a more efficient\nimplementation, which also does not require the object be a sequence.\n\nobject.__contains__(self, item)\n\n Called to implement membership test operators. Should return true\n if *item* is in *self*, false otherwise. For mapping objects, this\n should consider the keys of the mapping rather than the values or\n the key-item pairs.\n\n For objects that don\'t define ``__contains__()``, the membership\n test first tries iteration via ``__iter__()``, then the old\n sequence iteration protocol via ``__getitem__()``, see *this\n section in the language reference*.\n\n\nAdditional methods for emulation of sequence types\n==================================================\n\nThe following optional methods can be defined to further emulate\nsequence objects. Immutable sequences methods should at most only\ndefine ``__getslice__()``; mutable sequences might define all three\nmethods.\n\nobject.__getslice__(self, i, j)\n\n Deprecated since version 2.0: Support slice objects as parameters\n to the ``__getitem__()`` method. (However, built-in types in\n CPython currently still implement ``__getslice__()``. Therefore,\n you have to override it in derived classes when implementing\n slicing.)\n\n Called to implement evaluation of ``self[i:j]``. The returned\n object should be of the same type as *self*. Note that missing *i*\n or *j* in the slice expression are replaced by zero or\n ``sys.maxint``, respectively. If negative indexes are used in the\n slice, the length of the sequence is added to that index. If the\n instance does not implement the ``__len__()`` method, an\n ``AttributeError`` is raised. No guarantee is made that indexes\n adjusted this way are not still negative. Indexes which are\n greater than the length of the sequence are not modified. If no\n ``__getslice__()`` is found, a slice object is created instead, and\n passed to ``__getitem__()`` instead.\n\nobject.__setslice__(self, i, j, sequence)\n\n Called to implement assignment to ``self[i:j]``. Same notes for *i*\n and *j* as for ``__getslice__()``.\n\n This method is deprecated. If no ``__setslice__()`` is found, or\n for extended slicing of the form ``self[i:j:k]``, a slice object is\n created, and passed to ``__setitem__()``, instead of\n ``__setslice__()`` being called.\n\nobject.__delslice__(self, i, j)\n\n Called to implement deletion of ``self[i:j]``. Same notes for *i*\n and *j* as for ``__getslice__()``. This method is deprecated. If no\n ``__delslice__()`` is found, or for extended slicing of the form\n ``self[i:j:k]``, a slice object is created, and passed to\n ``__delitem__()``, instead of ``__delslice__()`` being called.\n\nNotice that these methods are only invoked when a single slice with a\nsingle colon is used, and the slice method is available. For slice\noperations involving extended slice notation, or in absence of the\nslice methods, ``__getitem__()``, ``__setitem__()`` or\n``__delitem__()`` is called with a slice object as argument.\n\nThe following example demonstrate how to make your program or module\ncompatible with earlier versions of Python (assuming that methods\n``__getitem__()``, ``__setitem__()`` and ``__delitem__()`` support\nslice objects as arguments):\n\n class MyClass:\n ...\n def __getitem__(self, index):\n ...\n def __setitem__(self, index, value):\n ...\n def __delitem__(self, index):\n ...\n\n if sys.version_info < (2, 0):\n # They won\'t be defined if version is at least 2.0 final\n\n def __getslice__(self, i, j):\n return self[max(0, i):max(0, j):]\n def __setslice__(self, i, j, seq):\n self[max(0, i):max(0, j):] = seq\n def __delslice__(self, i, j):\n del self[max(0, i):max(0, j):]\n ...\n\nNote the calls to ``max()``; these are necessary because of the\nhandling of negative indices before the ``__*slice__()`` methods are\ncalled. When negative indexes are used, the ``__*item__()`` methods\nreceive them as provided, but the ``__*slice__()`` methods get a\n"cooked" form of the index values. For each negative index value, the\nlength of the sequence is added to the index before calling the method\n(which may still result in a negative index); this is the customary\nhandling of negative indexes by the built-in sequence types, and the\n``__*item__()`` methods are expected to do this as well. However,\nsince they should already be doing that, negative indexes cannot be\npassed in; they must be constrained to the bounds of the sequence\nbefore being passed to the ``__*item__()`` methods. Calling ``max(0,\ni)`` conveniently returns the proper value.\n\n\nEmulating numeric types\n=======================\n\nThe following methods can be defined to emulate numeric objects.\nMethods corresponding to operations that are not supported by the\nparticular kind of number implemented (e.g., bitwise operations for\nnon-integral numbers) should be left undefined.\n\nobject.__add__(self, other)\nobject.__sub__(self, other)\nobject.__mul__(self, other)\nobject.__floordiv__(self, other)\nobject.__mod__(self, other)\nobject.__divmod__(self, other)\nobject.__pow__(self, other[, modulo])\nobject.__lshift__(self, other)\nobject.__rshift__(self, other)\nobject.__and__(self, other)\nobject.__xor__(self, other)\nobject.__or__(self, other)\n\n These methods are called to implement the binary arithmetic\n operations (``+``, ``-``, ``*``, ``//``, ``%``, ``divmod()``,\n ``pow()``, ``**``, ``<<``, ``>>``, ``&``, ``^``, ``|``). For\n instance, to evaluate the expression ``x + y``, where *x* is an\n instance of a class that has an ``__add__()`` method,\n ``x.__add__(y)`` is called. The ``__divmod__()`` method should be\n the equivalent to using ``__floordiv__()`` and ``__mod__()``; it\n should not be related to ``__truediv__()`` (described below). Note\n that ``__pow__()`` should be defined to accept an optional third\n argument if the ternary version of the built-in ``pow()`` function\n is to be supported.\n\n If one of those methods does not support the operation with the\n supplied arguments, it should return ``NotImplemented``.\n\nobject.__div__(self, other)\nobject.__truediv__(self, other)\n\n The division operator (``/``) is implemented by these methods. The\n ``__truediv__()`` method is used when ``__future__.division`` is in\n effect, otherwise ``__div__()`` is used. If only one of these two\n methods is defined, the object will not support division in the\n alternate context; ``TypeError`` will be raised instead.\n\nobject.__radd__(self, other)\nobject.__rsub__(self, other)\nobject.__rmul__(self, other)\nobject.__rdiv__(self, other)\nobject.__rtruediv__(self, other)\nobject.__rfloordiv__(self, other)\nobject.__rmod__(self, other)\nobject.__rdivmod__(self, other)\nobject.__rpow__(self, other)\nobject.__rlshift__(self, other)\nobject.__rrshift__(self, other)\nobject.__rand__(self, other)\nobject.__rxor__(self, other)\nobject.__ror__(self, other)\n\n These methods are called to implement the binary arithmetic\n operations (``+``, ``-``, ``*``, ``/``, ``%``, ``divmod()``,\n ``pow()``, ``**``, ``<<``, ``>>``, ``&``, ``^``, ``|``) with\n reflected (swapped) operands. These functions are only called if\n the left operand does not support the corresponding operation and\n the operands are of different types. [2] For instance, to evaluate\n the expression ``x - y``, where *y* is an instance of a class that\n has an ``__rsub__()`` method, ``y.__rsub__(x)`` is called if\n ``x.__sub__(y)`` returns *NotImplemented*.\n\n Note that ternary ``pow()`` will not try calling ``__rpow__()``\n (the coercion rules would become too complicated).\n\n Note: If the right operand\'s type is a subclass of the left operand\'s\n type and that subclass provides the reflected method for the\n operation, this method will be called before the left operand\'s\n non-reflected method. This behavior allows subclasses to\n override their ancestors\' operations.\n\nobject.__iadd__(self, other)\nobject.__isub__(self, other)\nobject.__imul__(self, other)\nobject.__idiv__(self, other)\nobject.__itruediv__(self, other)\nobject.__ifloordiv__(self, other)\nobject.__imod__(self, other)\nobject.__ipow__(self, other[, modulo])\nobject.__ilshift__(self, other)\nobject.__irshift__(self, other)\nobject.__iand__(self, other)\nobject.__ixor__(self, other)\nobject.__ior__(self, other)\n\n These methods are called to implement the augmented arithmetic\n assignments (``+=``, ``-=``, ``*=``, ``/=``, ``//=``, ``%=``,\n ``**=``, ``<<=``, ``>>=``, ``&=``, ``^=``, ``|=``). These methods\n should attempt to do the operation in-place (modifying *self*) and\n return the result (which could be, but does not have to be,\n *self*). If a specific method is not defined, the augmented\n assignment falls back to the normal methods. For instance, to\n execute the statement ``x += y``, where *x* is an instance of a\n class that has an ``__iadd__()`` method, ``x.__iadd__(y)`` is\n called. If *x* is an instance of a class that does not define a\n ``__iadd__()`` method, ``x.__add__(y)`` and ``y.__radd__(x)`` are\n considered, as with the evaluation of ``x + y``.\n\nobject.__neg__(self)\nobject.__pos__(self)\nobject.__abs__(self)\nobject.__invert__(self)\n\n Called to implement the unary arithmetic operations (``-``, ``+``,\n ``abs()`` and ``~``).\n\nobject.__complex__(self)\nobject.__int__(self)\nobject.__long__(self)\nobject.__float__(self)\n\n Called to implement the built-in functions ``complex()``,\n ``int()``, ``long()``, and ``float()``. Should return a value of\n the appropriate type.\n\nobject.__oct__(self)\nobject.__hex__(self)\n\n Called to implement the built-in functions ``oct()`` and ``hex()``.\n Should return a string value.\n\nobject.__index__(self)\n\n Called to implement ``operator.index()``. Also called whenever\n Python needs an integer object (such as in slicing). Must return\n an integer (int or long).\n\n New in version 2.5.\n\nobject.__coerce__(self, other)\n\n Called to implement "mixed-mode" numeric arithmetic. Should either\n return a 2-tuple containing *self* and *other* converted to a\n common numeric type, or ``None`` if conversion is impossible. When\n the common type would be the type of ``other``, it is sufficient to\n return ``None``, since the interpreter will also ask the other\n object to attempt a coercion (but sometimes, if the implementation\n of the other type cannot be changed, it is useful to do the\n conversion to the other type here). A return value of\n ``NotImplemented`` is equivalent to returning ``None``.\n\n\nCoercion rules\n==============\n\nThis section used to document the rules for coercion. As the language\nhas evolved, the coercion rules have become hard to document\nprecisely; documenting what one version of one particular\nimplementation does is undesirable. Instead, here are some informal\nguidelines regarding coercion. In Python 3.0, coercion will not be\nsupported.\n\n* If the left operand of a % operator is a string or Unicode object,\n no coercion takes place and the string formatting operation is\n invoked instead.\n\n* It is no longer recommended to define a coercion operation. Mixed-\n mode operations on types that don\'t define coercion pass the\n original arguments to the operation.\n\n* New-style classes (those derived from ``object``) never invoke the\n ``__coerce__()`` method in response to a binary operator; the only\n time ``__coerce__()`` is invoked is when the built-in function\n ``coerce()`` is called.\n\n* For most intents and purposes, an operator that returns\n ``NotImplemented`` is treated the same as one that is not\n implemented at all.\n\n* Below, ``__op__()`` and ``__rop__()`` are used to signify the\n generic method names corresponding to an operator; ``__iop__()`` is\n used for the corresponding in-place operator. For example, for the\n operator \'``+``\', ``__add__()`` and ``__radd__()`` are used for the\n left and right variant of the binary operator, and ``__iadd__()``\n for the in-place variant.\n\n* For objects *x* and *y*, first ``x.__op__(y)`` is tried. If this is\n not implemented or returns ``NotImplemented``, ``y.__rop__(x)`` is\n tried. If this is also not implemented or returns\n ``NotImplemented``, a ``TypeError`` exception is raised. But see\n the following exception:\n\n* Exception to the previous item: if the left operand is an instance\n of a built-in type or a new-style class, and the right operand is an\n instance of a proper subclass of that type or class and overrides\n the base\'s ``__rop__()`` method, the right operand\'s ``__rop__()``\n method is tried *before* the left operand\'s ``__op__()`` method.\n\n This is done so that a subclass can completely override binary\n operators. Otherwise, the left operand\'s ``__op__()`` method would\n always accept the right operand: when an instance of a given class\n is expected, an instance of a subclass of that class is always\n acceptable.\n\n* When either operand type defines a coercion, this coercion is called\n before that type\'s ``__op__()`` or ``__rop__()`` method is called,\n but no sooner. If the coercion returns an object of a different\n type for the operand whose coercion is invoked, part of the process\n is redone using the new object.\n\n* When an in-place operator (like \'``+=``\') is used, if the left\n operand implements ``__iop__()``, it is invoked without any\n coercion. When the operation falls back to ``__op__()`` and/or\n ``__rop__()``, the normal coercion rules apply.\n\n* In ``x + y``, if *x* is a sequence that implements sequence\n concatenation, sequence concatenation is invoked.\n\n* In ``x * y``, if one operator is a sequence that implements sequence\n repetition, and the other is an integer (``int`` or ``long``),\n sequence repetition is invoked.\n\n* Rich comparisons (implemented by methods ``__eq__()`` and so on)\n never use coercion. Three-way comparison (implemented by\n ``__cmp__()``) does use coercion under the same conditions as other\n binary operations use it.\n\n* In the current implementation, the built-in numeric types ``int``,\n ``long``, ``float``, and ``complex`` do not use coercion. All these\n types implement a ``__coerce__()`` method, for use by the built-in\n ``coerce()`` function.\n\n Changed in version 2.7.\n\n\nWith Statement Context Managers\n===============================\n\nNew in version 2.5.\n\nA *context manager* is an object that defines the runtime context to\nbe established when executing a ``with`` statement. The context\nmanager handles the entry into, and the exit from, the desired runtime\ncontext for the execution of the block of code. Context managers are\nnormally invoked using the ``with`` statement (described in section\n*The with statement*), but can also be used by directly invoking their\nmethods.\n\nTypical uses of context managers include saving and restoring various\nkinds of global state, locking and unlocking resources, closing opened\nfiles, etc.\n\nFor more information on context managers, see *Context Manager Types*.\n\nobject.__enter__(self)\n\n Enter the runtime context related to this object. The ``with``\n statement will bind this method\'s return value to the target(s)\n specified in the ``as`` clause of the statement, if any.\n\nobject.__exit__(self, exc_type, exc_value, traceback)\n\n Exit the runtime context related to this object. The parameters\n describe the exception that caused the context to be exited. If the\n context was exited without an exception, all three arguments will\n be ``None``.\n\n If an exception is supplied, and the method wishes to suppress the\n exception (i.e., prevent it from being propagated), it should\n return a true value. Otherwise, the exception will be processed\n normally upon exit from this method.\n\n Note that ``__exit__()`` methods should not reraise the passed-in\n exception; this is the caller\'s responsibility.\n\nSee also:\n\n **PEP 0343** - The "with" statement\n The specification, background, and examples for the Python\n ``with`` statement.\n\n\nSpecial method lookup for old-style classes\n===========================================\n\nFor old-style classes, special methods are always looked up in exactly\nthe same way as any other method or attribute. This is the case\nregardless of whether the method is being looked up explicitly as in\n``x.__getitem__(i)`` or implicitly as in ``x[i]``.\n\nThis behaviour means that special methods may exhibit different\nbehaviour for different instances of a single old-style class if the\nappropriate special attributes are set differently:\n\n >>> class C:\n ... pass\n ...\n >>> c1 = C()\n >>> c2 = C()\n >>> c1.__len__ = lambda: 5\n >>> c2.__len__ = lambda: 9\n >>> len(c1)\n 5\n >>> len(c2)\n 9\n\n\nSpecial method lookup for new-style classes\n===========================================\n\nFor new-style classes, implicit invocations of special methods are\nonly guaranteed to work correctly if defined on an object\'s type, not\nin the object\'s instance dictionary. That behaviour is the reason why\nthe following code raises an exception (unlike the equivalent example\nwith old-style classes):\n\n >>> class C(object):\n ... pass\n ...\n >>> c = C()\n >>> c.__len__ = lambda: 5\n >>> len(c)\n Traceback (most recent call last):\n File "<stdin>", line 1, in <module>\n TypeError: object of type \'C\' has no len()\n\nThe rationale behind this behaviour lies with a number of special\nmethods such as ``__hash__()`` and ``__repr__()`` that are implemented\nby all objects, including type objects. If the implicit lookup of\nthese methods used the conventional lookup process, they would fail\nwhen invoked on the type object itself:\n\n >>> 1 .__hash__() == hash(1)\n True\n >>> int.__hash__() == hash(int)\n Traceback (most recent call last):\n File "<stdin>", line 1, in <module>\n TypeError: descriptor \'__hash__\' of \'int\' object needs an argument\n\nIncorrectly attempting to invoke an unbound method of a class in this\nway is sometimes referred to as \'metaclass confusion\', and is avoided\nby bypassing the instance when looking up special methods:\n\n >>> type(1).__hash__(1) == hash(1)\n True\n >>> type(int).__hash__(int) == hash(int)\n True\n\nIn addition to bypassing any instance attributes in the interest of\ncorrectness, implicit special method lookup generally also bypasses\nthe ``__getattribute__()`` method even of the object\'s metaclass:\n\n >>> class Meta(type):\n ... def __getattribute__(*args):\n ... print "Metaclass getattribute invoked"\n ... return type.__getattribute__(*args)\n ...\n >>> class C(object):\n ... __metaclass__ = Meta\n ... def __len__(self):\n ... return 10\n ... def __getattribute__(*args):\n ... print "Class getattribute invoked"\n ... return object.__getattribute__(*args)\n ...\n >>> c = C()\n >>> c.__len__() # Explicit lookup via instance\n Class getattribute invoked\n 10\n >>> type(c).__len__(c) # Explicit lookup via type\n Metaclass getattribute invoked\n 10\n >>> len(c) # Implicit lookup\n 10\n\nBypassing the ``__getattribute__()`` machinery in this fashion\nprovides significant scope for speed optimisations within the\ninterpreter, at the cost of some flexibility in the handling of\nspecial methods (the special method *must* be set on the class object\nitself in order to be consistently invoked by the interpreter).\n\n-[ Footnotes ]-\n\n[1] It *is* possible in some cases to change an object\'s type, under\n certain controlled conditions. It generally isn\'t a good idea\n though, since it can lead to some very strange behaviour if it is\n handled incorrectly.\n\n[2] For operands of the same type, it is assumed that if the non-\n reflected method (such as ``__add__()``) fails the operation is\n not supported, which is why the reflected method is not called.\n', + 'specialnames': u'\nSpecial method names\n********************\n\nA class can implement certain operations that are invoked by special\nsyntax (such as arithmetic operations or subscripting and slicing) by\ndefining methods with special names. This is Python\'s approach to\n*operator overloading*, allowing classes to define their own behavior\nwith respect to language operators. For instance, if a class defines\na method named ``__getitem__()``, and ``x`` is an instance of this\nclass, then ``x[i]`` is roughly equivalent to ``x.__getitem__(i)`` for\nold-style classes and ``type(x).__getitem__(x, i)`` for new-style\nclasses. Except where mentioned, attempts to execute an operation\nraise an exception when no appropriate method is defined (typically\n``AttributeError`` or ``TypeError``).\n\nWhen implementing a class that emulates any built-in type, it is\nimportant that the emulation only be implemented to the degree that it\nmakes sense for the object being modelled. For example, some\nsequences may work well with retrieval of individual elements, but\nextracting a slice may not make sense. (One example of this is the\n``NodeList`` interface in the W3C\'s Document Object Model.)\n\n\nBasic customization\n===================\n\nobject.__new__(cls[, ...])\n\n Called to create a new instance of class *cls*. ``__new__()`` is a\n static method (special-cased so you need not declare it as such)\n that takes the class of which an instance was requested as its\n first argument. The remaining arguments are those passed to the\n object constructor expression (the call to the class). The return\n value of ``__new__()`` should be the new object instance (usually\n an instance of *cls*).\n\n Typical implementations create a new instance of the class by\n invoking the superclass\'s ``__new__()`` method using\n ``super(currentclass, cls).__new__(cls[, ...])`` with appropriate\n arguments and then modifying the newly-created instance as\n necessary before returning it.\n\n If ``__new__()`` returns an instance of *cls*, then the new\n instance\'s ``__init__()`` method will be invoked like\n ``__init__(self[, ...])``, where *self* is the new instance and the\n remaining arguments are the same as were passed to ``__new__()``.\n\n If ``__new__()`` does not return an instance of *cls*, then the new\n instance\'s ``__init__()`` method will not be invoked.\n\n ``__new__()`` is intended mainly to allow subclasses of immutable\n types (like int, str, or tuple) to customize instance creation. It\n is also commonly overridden in custom metaclasses in order to\n customize class creation.\n\nobject.__init__(self[, ...])\n\n Called when the instance is created. The arguments are those\n passed to the class constructor expression. If a base class has an\n ``__init__()`` method, the derived class\'s ``__init__()`` method,\n if any, must explicitly call it to ensure proper initialization of\n the base class part of the instance; for example:\n ``BaseClass.__init__(self, [args...])``. As a special constraint\n on constructors, no value may be returned; doing so will cause a\n ``TypeError`` to be raised at runtime.\n\nobject.__del__(self)\n\n Called when the instance is about to be destroyed. This is also\n called a destructor. If a base class has a ``__del__()`` method,\n the derived class\'s ``__del__()`` method, if any, must explicitly\n call it to ensure proper deletion of the base class part of the\n instance. Note that it is possible (though not recommended!) for\n the ``__del__()`` method to postpone destruction of the instance by\n creating a new reference to it. It may then be called at a later\n time when this new reference is deleted. It is not guaranteed that\n ``__del__()`` methods are called for objects that still exist when\n the interpreter exits.\n\n Note: ``del x`` doesn\'t directly call ``x.__del__()`` --- the former\n decrements the reference count for ``x`` by one, and the latter\n is only called when ``x``\'s reference count reaches zero. Some\n common situations that may prevent the reference count of an\n object from going to zero include: circular references between\n objects (e.g., a doubly-linked list or a tree data structure with\n parent and child pointers); a reference to the object on the\n stack frame of a function that caught an exception (the traceback\n stored in ``sys.exc_traceback`` keeps the stack frame alive); or\n a reference to the object on the stack frame that raised an\n unhandled exception in interactive mode (the traceback stored in\n ``sys.last_traceback`` keeps the stack frame alive). The first\n situation can only be remedied by explicitly breaking the cycles;\n the latter two situations can be resolved by storing ``None`` in\n ``sys.exc_traceback`` or ``sys.last_traceback``. Circular\n references which are garbage are detected when the option cycle\n detector is enabled (it\'s on by default), but can only be cleaned\n up if there are no Python-level ``__del__()`` methods involved.\n Refer to the documentation for the ``gc`` module for more\n information about how ``__del__()`` methods are handled by the\n cycle detector, particularly the description of the ``garbage``\n value.\n\n Warning: Due to the precarious circumstances under which ``__del__()``\n methods are invoked, exceptions that occur during their execution\n are ignored, and a warning is printed to ``sys.stderr`` instead.\n Also, when ``__del__()`` is invoked in response to a module being\n deleted (e.g., when execution of the program is done), other\n globals referenced by the ``__del__()`` method may already have\n been deleted or in the process of being torn down (e.g. the\n import machinery shutting down). For this reason, ``__del__()``\n methods should do the absolute minimum needed to maintain\n external invariants. Starting with version 1.5, Python\n guarantees that globals whose name begins with a single\n underscore are deleted from their module before other globals are\n deleted; if no other references to such globals exist, this may\n help in assuring that imported modules are still available at the\n time when the ``__del__()`` method is called.\n\nobject.__repr__(self)\n\n Called by the ``repr()`` built-in function and by string\n conversions (reverse quotes) to compute the "official" string\n representation of an object. If at all possible, this should look\n like a valid Python expression that could be used to recreate an\n object with the same value (given an appropriate environment). If\n this is not possible, a string of the form ``<...some useful\n description...>`` should be returned. The return value must be a\n string object. If a class defines ``__repr__()`` but not\n ``__str__()``, then ``__repr__()`` is also used when an "informal"\n string representation of instances of that class is required.\n\n This is typically used for debugging, so it is important that the\n representation is information-rich and unambiguous.\n\nobject.__str__(self)\n\n Called by the ``str()`` built-in function and by the ``print``\n statement to compute the "informal" string representation of an\n object. This differs from ``__repr__()`` in that it does not have\n to be a valid Python expression: a more convenient or concise\n representation may be used instead. The return value must be a\n string object.\n\nobject.__lt__(self, other)\nobject.__le__(self, other)\nobject.__eq__(self, other)\nobject.__ne__(self, other)\nobject.__gt__(self, other)\nobject.__ge__(self, other)\n\n New in version 2.1.\n\n These are the so-called "rich comparison" methods, and are called\n for comparison operators in preference to ``__cmp__()`` below. The\n correspondence between operator symbols and method names is as\n follows: ``x<y`` calls ``x.__lt__(y)``, ``x<=y`` calls\n ``x.__le__(y)``, ``x==y`` calls ``x.__eq__(y)``, ``x!=y`` and\n ``x<>y`` call ``x.__ne__(y)``, ``x>y`` calls ``x.__gt__(y)``, and\n ``x>=y`` calls ``x.__ge__(y)``.\n\n A rich comparison method may return the singleton\n ``NotImplemented`` if it does not implement the operation for a\n given pair of arguments. By convention, ``False`` and ``True`` are\n returned for a successful comparison. However, these methods can\n return any value, so if the comparison operator is used in a\n Boolean context (e.g., in the condition of an ``if`` statement),\n Python will call ``bool()`` on the value to determine if the result\n is true or false.\n\n There are no implied relationships among the comparison operators.\n The truth of ``x==y`` does not imply that ``x!=y`` is false.\n Accordingly, when defining ``__eq__()``, one should also define\n ``__ne__()`` so that the operators will behave as expected. See\n the paragraph on ``__hash__()`` for some important notes on\n creating *hashable* objects which support custom comparison\n operations and are usable as dictionary keys.\n\n There are no swapped-argument versions of these methods (to be used\n when the left argument does not support the operation but the right\n argument does); rather, ``__lt__()`` and ``__gt__()`` are each\n other\'s reflection, ``__le__()`` and ``__ge__()`` are each other\'s\n reflection, and ``__eq__()`` and ``__ne__()`` are their own\n reflection.\n\n Arguments to rich comparison methods are never coerced.\n\n To automatically generate ordering operations from a single root\n operation, see ``functools.total_ordering()``.\n\nobject.__cmp__(self, other)\n\n Called by comparison operations if rich comparison (see above) is\n not defined. Should return a negative integer if ``self < other``,\n zero if ``self == other``, a positive integer if ``self > other``.\n If no ``__cmp__()``, ``__eq__()`` or ``__ne__()`` operation is\n defined, class instances are compared by object identity\n ("address"). See also the description of ``__hash__()`` for some\n important notes on creating *hashable* objects which support custom\n comparison operations and are usable as dictionary keys. (Note: the\n restriction that exceptions are not propagated by ``__cmp__()`` has\n been removed since Python 1.5.)\n\nobject.__rcmp__(self, other)\n\n Changed in version 2.1: No longer supported.\n\nobject.__hash__(self)\n\n Called by built-in function ``hash()`` and for operations on\n members of hashed collections including ``set``, ``frozenset``, and\n ``dict``. ``__hash__()`` should return an integer. The only\n required property is that objects which compare equal have the same\n hash value; it is advised to somehow mix together (e.g. using\n exclusive or) the hash values for the components of the object that\n also play a part in comparison of objects.\n\n If a class does not define a ``__cmp__()`` or ``__eq__()`` method\n it should not define a ``__hash__()`` operation either; if it\n defines ``__cmp__()`` or ``__eq__()`` but not ``__hash__()``, its\n instances will not be usable in hashed collections. If a class\n defines mutable objects and implements a ``__cmp__()`` or\n ``__eq__()`` method, it should not implement ``__hash__()``, since\n hashable collection implementations require that a object\'s hash\n value is immutable (if the object\'s hash value changes, it will be\n in the wrong hash bucket).\n\n User-defined classes have ``__cmp__()`` and ``__hash__()`` methods\n by default; with them, all objects compare unequal (except with\n themselves) and ``x.__hash__()`` returns ``id(x)``.\n\n Classes which inherit a ``__hash__()`` method from a parent class\n but change the meaning of ``__cmp__()`` or ``__eq__()`` such that\n the hash value returned is no longer appropriate (e.g. by switching\n to a value-based concept of equality instead of the default\n identity based equality) can explicitly flag themselves as being\n unhashable by setting ``__hash__ = None`` in the class definition.\n Doing so means that not only will instances of the class raise an\n appropriate ``TypeError`` when a program attempts to retrieve their\n hash value, but they will also be correctly identified as\n unhashable when checking ``isinstance(obj, collections.Hashable)``\n (unlike classes which define their own ``__hash__()`` to explicitly\n raise ``TypeError``).\n\n Changed in version 2.5: ``__hash__()`` may now also return a long\n integer object; the 32-bit integer is then derived from the hash of\n that object.\n\n Changed in version 2.6: ``__hash__`` may now be set to ``None`` to\n explicitly flag instances of a class as unhashable.\n\nobject.__nonzero__(self)\n\n Called to implement truth value testing and the built-in operation\n ``bool()``; should return ``False`` or ``True``, or their integer\n equivalents ``0`` or ``1``. When this method is not defined,\n ``__len__()`` is called, if it is defined, and the object is\n considered true if its result is nonzero. If a class defines\n neither ``__len__()`` nor ``__nonzero__()``, all its instances are\n considered true.\n\nobject.__unicode__(self)\n\n Called to implement ``unicode()`` built-in; should return a Unicode\n object. When this method is not defined, string conversion is\n attempted, and the result of string conversion is converted to\n Unicode using the system default encoding.\n\n\nCustomizing attribute access\n============================\n\nThe following methods can be defined to customize the meaning of\nattribute access (use of, assignment to, or deletion of ``x.name``)\nfor class instances.\n\nobject.__getattr__(self, name)\n\n Called when an attribute lookup has not found the attribute in the\n usual places (i.e. it is not an instance attribute nor is it found\n in the class tree for ``self``). ``name`` is the attribute name.\n This method should return the (computed) attribute value or raise\n an ``AttributeError`` exception.\n\n Note that if the attribute is found through the normal mechanism,\n ``__getattr__()`` is not called. (This is an intentional asymmetry\n between ``__getattr__()`` and ``__setattr__()``.) This is done both\n for efficiency reasons and because otherwise ``__getattr__()``\n would have no way to access other attributes of the instance. Note\n that at least for instance variables, you can fake total control by\n not inserting any values in the instance attribute dictionary (but\n instead inserting them in another object). See the\n ``__getattribute__()`` method below for a way to actually get total\n control in new-style classes.\n\nobject.__setattr__(self, name, value)\n\n Called when an attribute assignment is attempted. This is called\n instead of the normal mechanism (i.e. store the value in the\n instance dictionary). *name* is the attribute name, *value* is the\n value to be assigned to it.\n\n If ``__setattr__()`` wants to assign to an instance attribute, it\n should not simply execute ``self.name = value`` --- this would\n cause a recursive call to itself. Instead, it should insert the\n value in the dictionary of instance attributes, e.g.,\n ``self.__dict__[name] = value``. For new-style classes, rather\n than accessing the instance dictionary, it should call the base\n class method with the same name, for example,\n ``object.__setattr__(self, name, value)``.\n\nobject.__delattr__(self, name)\n\n Like ``__setattr__()`` but for attribute deletion instead of\n assignment. This should only be implemented if ``del obj.name`` is\n meaningful for the object.\n\n\nMore attribute access for new-style classes\n-------------------------------------------\n\nThe following methods only apply to new-style classes.\n\nobject.__getattribute__(self, name)\n\n Called unconditionally to implement attribute accesses for\n instances of the class. If the class also defines\n ``__getattr__()``, the latter will not be called unless\n ``__getattribute__()`` either calls it explicitly or raises an\n ``AttributeError``. This method should return the (computed)\n attribute value or raise an ``AttributeError`` exception. In order\n to avoid infinite recursion in this method, its implementation\n should always call the base class method with the same name to\n access any attributes it needs, for example,\n ``object.__getattribute__(self, name)``.\n\n Note: This method may still be bypassed when looking up special methods\n as the result of implicit invocation via language syntax or\n built-in functions. See *Special method lookup for new-style\n classes*.\n\n\nImplementing Descriptors\n------------------------\n\nThe following methods only apply when an instance of the class\ncontaining the method (a so-called *descriptor* class) appears in an\n*owner* class (the descriptor must be in either the owner\'s class\ndictionary or in the class dictionary for one of its parents). In the\nexamples below, "the attribute" refers to the attribute whose name is\nthe key of the property in the owner class\' ``__dict__``.\n\nobject.__get__(self, instance, owner)\n\n Called to get the attribute of the owner class (class attribute\n access) or of an instance of that class (instance attribute\n access). *owner* is always the owner class, while *instance* is the\n instance that the attribute was accessed through, or ``None`` when\n the attribute is accessed through the *owner*. This method should\n return the (computed) attribute value or raise an\n ``AttributeError`` exception.\n\nobject.__set__(self, instance, value)\n\n Called to set the attribute on an instance *instance* of the owner\n class to a new value, *value*.\n\nobject.__delete__(self, instance)\n\n Called to delete the attribute on an instance *instance* of the\n owner class.\n\n\nInvoking Descriptors\n--------------------\n\nIn general, a descriptor is an object attribute with "binding\nbehavior", one whose attribute access has been overridden by methods\nin the descriptor protocol: ``__get__()``, ``__set__()``, and\n``__delete__()``. If any of those methods are defined for an object,\nit is said to be a descriptor.\n\nThe default behavior for attribute access is to get, set, or delete\nthe attribute from an object\'s dictionary. For instance, ``a.x`` has a\nlookup chain starting with ``a.__dict__[\'x\']``, then\n``type(a).__dict__[\'x\']``, and continuing through the base classes of\n``type(a)`` excluding metaclasses.\n\nHowever, if the looked-up value is an object defining one of the\ndescriptor methods, then Python may override the default behavior and\ninvoke the descriptor method instead. Where this occurs in the\nprecedence chain depends on which descriptor methods were defined and\nhow they were called. Note that descriptors are only invoked for new\nstyle objects or classes (ones that subclass ``object()`` or\n``type()``).\n\nThe starting point for descriptor invocation is a binding, ``a.x``.\nHow the arguments are assembled depends on ``a``:\n\nDirect Call\n The simplest and least common call is when user code directly\n invokes a descriptor method: ``x.__get__(a)``.\n\nInstance Binding\n If binding to a new-style object instance, ``a.x`` is transformed\n into the call: ``type(a).__dict__[\'x\'].__get__(a, type(a))``.\n\nClass Binding\n If binding to a new-style class, ``A.x`` is transformed into the\n call: ``A.__dict__[\'x\'].__get__(None, A)``.\n\nSuper Binding\n If ``a`` is an instance of ``super``, then the binding ``super(B,\n obj).m()`` searches ``obj.__class__.__mro__`` for the base class\n ``A`` immediately preceding ``B`` and then invokes the descriptor\n with the call: ``A.__dict__[\'m\'].__get__(obj, obj.__class__)``.\n\nFor instance bindings, the precedence of descriptor invocation depends\non the which descriptor methods are defined. A descriptor can define\nany combination of ``__get__()``, ``__set__()`` and ``__delete__()``.\nIf it does not define ``__get__()``, then accessing the attribute will\nreturn the descriptor object itself unless there is a value in the\nobject\'s instance dictionary. If the descriptor defines ``__set__()``\nand/or ``__delete__()``, it is a data descriptor; if it defines\nneither, it is a non-data descriptor. Normally, data descriptors\ndefine both ``__get__()`` and ``__set__()``, while non-data\ndescriptors have just the ``__get__()`` method. Data descriptors with\n``__set__()`` and ``__get__()`` defined always override a redefinition\nin an instance dictionary. In contrast, non-data descriptors can be\noverridden by instances.\n\nPython methods (including ``staticmethod()`` and ``classmethod()``)\nare implemented as non-data descriptors. Accordingly, instances can\nredefine and override methods. This allows individual instances to\nacquire behaviors that differ from other instances of the same class.\n\nThe ``property()`` function is implemented as a data descriptor.\nAccordingly, instances cannot override the behavior of a property.\n\n\n__slots__\n---------\n\nBy default, instances of both old and new-style classes have a\ndictionary for attribute storage. This wastes space for objects\nhaving very few instance variables. The space consumption can become\nacute when creating large numbers of instances.\n\nThe default can be overridden by defining *__slots__* in a new-style\nclass definition. The *__slots__* declaration takes a sequence of\ninstance variables and reserves just enough space in each instance to\nhold a value for each variable. Space is saved because *__dict__* is\nnot created for each instance.\n\n__slots__\n\n This class variable can be assigned a string, iterable, or sequence\n of strings with variable names used by instances. If defined in a\n new-style class, *__slots__* reserves space for the declared\n variables and prevents the automatic creation of *__dict__* and\n *__weakref__* for each instance.\n\n New in version 2.2.\n\nNotes on using *__slots__*\n\n* When inheriting from a class without *__slots__*, the *__dict__*\n attribute of that class will always be accessible, so a *__slots__*\n definition in the subclass is meaningless.\n\n* Without a *__dict__* variable, instances cannot be assigned new\n variables not listed in the *__slots__* definition. Attempts to\n assign to an unlisted variable name raises ``AttributeError``. If\n dynamic assignment of new variables is desired, then add\n ``\'__dict__\'`` to the sequence of strings in the *__slots__*\n declaration.\n\n Changed in version 2.3: Previously, adding ``\'__dict__\'`` to the\n *__slots__* declaration would not enable the assignment of new\n attributes not specifically listed in the sequence of instance\n variable names.\n\n* Without a *__weakref__* variable for each instance, classes defining\n *__slots__* do not support weak references to its instances. If weak\n reference support is needed, then add ``\'__weakref__\'`` to the\n sequence of strings in the *__slots__* declaration.\n\n Changed in version 2.3: Previously, adding ``\'__weakref__\'`` to the\n *__slots__* declaration would not enable support for weak\n references.\n\n* *__slots__* are implemented at the class level by creating\n descriptors (*Implementing Descriptors*) for each variable name. As\n a result, class attributes cannot be used to set default values for\n instance variables defined by *__slots__*; otherwise, the class\n attribute would overwrite the descriptor assignment.\n\n* The action of a *__slots__* declaration is limited to the class\n where it is defined. As a result, subclasses will have a *__dict__*\n unless they also define *__slots__* (which must only contain names\n of any *additional* slots).\n\n* If a class defines a slot also defined in a base class, the instance\n variable defined by the base class slot is inaccessible (except by\n retrieving its descriptor directly from the base class). This\n renders the meaning of the program undefined. In the future, a\n check may be added to prevent this.\n\n* Nonempty *__slots__* does not work for classes derived from\n "variable-length" built-in types such as ``long``, ``str`` and\n ``tuple``.\n\n* Any non-string iterable may be assigned to *__slots__*. Mappings may\n also be used; however, in the future, special meaning may be\n assigned to the values corresponding to each key.\n\n* *__class__* assignment works only if both classes have the same\n *__slots__*.\n\n Changed in version 2.6: Previously, *__class__* assignment raised an\n error if either new or old class had *__slots__*.\n\n\nCustomizing class creation\n==========================\n\nBy default, new-style classes are constructed using ``type()``. A\nclass definition is read into a separate namespace and the value of\nclass name is bound to the result of ``type(name, bases, dict)``.\n\nWhen the class definition is read, if *__metaclass__* is defined then\nthe callable assigned to it will be called instead of ``type()``. This\nallows classes or functions to be written which monitor or alter the\nclass creation process:\n\n* Modifying the class dictionary prior to the class being created.\n\n* Returning an instance of another class -- essentially performing the\n role of a factory function.\n\nThese steps will have to be performed in the metaclass\'s ``__new__()``\nmethod -- ``type.__new__()`` can then be called from this method to\ncreate a class with different properties. This example adds a new\nelement to the class dictionary before creating the class:\n\n class metacls(type):\n def __new__(mcs, name, bases, dict):\n dict[\'foo\'] = \'metacls was here\'\n return type.__new__(mcs, name, bases, dict)\n\nYou can of course also override other class methods (or add new\nmethods); for example defining a custom ``__call__()`` method in the\nmetaclass allows custom behavior when the class is called, e.g. not\nalways creating a new instance.\n\n__metaclass__\n\n This variable can be any callable accepting arguments for ``name``,\n ``bases``, and ``dict``. Upon class creation, the callable is used\n instead of the built-in ``type()``.\n\n New in version 2.2.\n\nThe appropriate metaclass is determined by the following precedence\nrules:\n\n* If ``dict[\'__metaclass__\']`` exists, it is used.\n\n* Otherwise, if there is at least one base class, its metaclass is\n used (this looks for a *__class__* attribute first and if not found,\n uses its type).\n\n* Otherwise, if a global variable named __metaclass__ exists, it is\n used.\n\n* Otherwise, the old-style, classic metaclass (types.ClassType) is\n used.\n\nThe potential uses for metaclasses are boundless. Some ideas that have\nbeen explored including logging, interface checking, automatic\ndelegation, automatic property creation, proxies, frameworks, and\nautomatic resource locking/synchronization.\n\n\nCustomizing instance and subclass checks\n========================================\n\nNew in version 2.6.\n\nThe following methods are used to override the default behavior of the\n``isinstance()`` and ``issubclass()`` built-in functions.\n\nIn particular, the metaclass ``abc.ABCMeta`` implements these methods\nin order to allow the addition of Abstract Base Classes (ABCs) as\n"virtual base classes" to any class or type (including built-in\ntypes), including other ABCs.\n\nclass.__instancecheck__(self, instance)\n\n Return true if *instance* should be considered a (direct or\n indirect) instance of *class*. If defined, called to implement\n ``isinstance(instance, class)``.\n\nclass.__subclasscheck__(self, subclass)\n\n Return true if *subclass* should be considered a (direct or\n indirect) subclass of *class*. If defined, called to implement\n ``issubclass(subclass, class)``.\n\nNote that these methods are looked up on the type (metaclass) of a\nclass. They cannot be defined as class methods in the actual class.\nThis is consistent with the lookup of special methods that are called\non instances, only in this case the instance is itself a class.\n\nSee also:\n\n **PEP 3119** - Introducing Abstract Base Classes\n Includes the specification for customizing ``isinstance()`` and\n ``issubclass()`` behavior through ``__instancecheck__()`` and\n ``__subclasscheck__()``, with motivation for this functionality\n in the context of adding Abstract Base Classes (see the ``abc``\n module) to the language.\n\n\nEmulating callable objects\n==========================\n\nobject.__call__(self[, args...])\n\n Called when the instance is "called" as a function; if this method\n is defined, ``x(arg1, arg2, ...)`` is a shorthand for\n ``x.__call__(arg1, arg2, ...)``.\n\n\nEmulating container types\n=========================\n\nThe following methods can be defined to implement container objects.\nContainers usually are sequences (such as lists or tuples) or mappings\n(like dictionaries), but can represent other containers as well. The\nfirst set of methods is used either to emulate a sequence or to\nemulate a mapping; the difference is that for a sequence, the\nallowable keys should be the integers *k* for which ``0 <= k < N``\nwhere *N* is the length of the sequence, or slice objects, which\ndefine a range of items. (For backwards compatibility, the method\n``__getslice__()`` (see below) can also be defined to handle simple,\nbut not extended slices.) It is also recommended that mappings provide\nthe methods ``keys()``, ``values()``, ``items()``, ``has_key()``,\n``get()``, ``clear()``, ``setdefault()``, ``iterkeys()``,\n``itervalues()``, ``iteritems()``, ``pop()``, ``popitem()``,\n``copy()``, and ``update()`` behaving similar to those for Python\'s\nstandard dictionary objects. The ``UserDict`` module provides a\n``DictMixin`` class to help create those methods from a base set of\n``__getitem__()``, ``__setitem__()``, ``__delitem__()``, and\n``keys()``. Mutable sequences should provide methods ``append()``,\n``count()``, ``index()``, ``extend()``, ``insert()``, ``pop()``,\n``remove()``, ``reverse()`` and ``sort()``, like Python standard list\nobjects. Finally, sequence types should implement addition (meaning\nconcatenation) and multiplication (meaning repetition) by defining the\nmethods ``__add__()``, ``__radd__()``, ``__iadd__()``, ``__mul__()``,\n``__rmul__()`` and ``__imul__()`` described below; they should not\ndefine ``__coerce__()`` or other numerical operators. It is\nrecommended that both mappings and sequences implement the\n``__contains__()`` method to allow efficient use of the ``in``\noperator; for mappings, ``in`` should be equivalent of ``has_key()``;\nfor sequences, it should search through the values. It is further\nrecommended that both mappings and sequences implement the\n``__iter__()`` method to allow efficient iteration through the\ncontainer; for mappings, ``__iter__()`` should be the same as\n``iterkeys()``; for sequences, it should iterate through the values.\n\nobject.__len__(self)\n\n Called to implement the built-in function ``len()``. Should return\n the length of the object, an integer ``>=`` 0. Also, an object\n that doesn\'t define a ``__nonzero__()`` method and whose\n ``__len__()`` method returns zero is considered to be false in a\n Boolean context.\n\nobject.__getitem__(self, key)\n\n Called to implement evaluation of ``self[key]``. For sequence\n types, the accepted keys should be integers and slice objects.\n Note that the special interpretation of negative indexes (if the\n class wishes to emulate a sequence type) is up to the\n ``__getitem__()`` method. If *key* is of an inappropriate type,\n ``TypeError`` may be raised; if of a value outside the set of\n indexes for the sequence (after any special interpretation of\n negative values), ``IndexError`` should be raised. For mapping\n types, if *key* is missing (not in the container), ``KeyError``\n should be raised.\n\n Note: ``for`` loops expect that an ``IndexError`` will be raised for\n illegal indexes to allow proper detection of the end of the\n sequence.\n\nobject.__setitem__(self, key, value)\n\n Called to implement assignment to ``self[key]``. Same note as for\n ``__getitem__()``. This should only be implemented for mappings if\n the objects support changes to the values for keys, or if new keys\n can be added, or for sequences if elements can be replaced. The\n same exceptions should be raised for improper *key* values as for\n the ``__getitem__()`` method.\n\nobject.__delitem__(self, key)\n\n Called to implement deletion of ``self[key]``. Same note as for\n ``__getitem__()``. This should only be implemented for mappings if\n the objects support removal of keys, or for sequences if elements\n can be removed from the sequence. The same exceptions should be\n raised for improper *key* values as for the ``__getitem__()``\n method.\n\nobject.__iter__(self)\n\n This method is called when an iterator is required for a container.\n This method should return a new iterator object that can iterate\n over all the objects in the container. For mappings, it should\n iterate over the keys of the container, and should also be made\n available as the method ``iterkeys()``.\n\n Iterator objects also need to implement this method; they are\n required to return themselves. For more information on iterator\n objects, see *Iterator Types*.\n\nobject.__reversed__(self)\n\n Called (if present) by the ``reversed()`` built-in to implement\n reverse iteration. It should return a new iterator object that\n iterates over all the objects in the container in reverse order.\n\n If the ``__reversed__()`` method is not provided, the\n ``reversed()`` built-in will fall back to using the sequence\n protocol (``__len__()`` and ``__getitem__()``). Objects that\n support the sequence protocol should only provide\n ``__reversed__()`` if they can provide an implementation that is\n more efficient than the one provided by ``reversed()``.\n\n New in version 2.6.\n\nThe membership test operators (``in`` and ``not in``) are normally\nimplemented as an iteration through a sequence. However, container\nobjects can supply the following special method with a more efficient\nimplementation, which also does not require the object be a sequence.\n\nobject.__contains__(self, item)\n\n Called to implement membership test operators. Should return true\n if *item* is in *self*, false otherwise. For mapping objects, this\n should consider the keys of the mapping rather than the values or\n the key-item pairs.\n\n For objects that don\'t define ``__contains__()``, the membership\n test first tries iteration via ``__iter__()``, then the old\n sequence iteration protocol via ``__getitem__()``, see *this\n section in the language reference*.\n\n\nAdditional methods for emulation of sequence types\n==================================================\n\nThe following optional methods can be defined to further emulate\nsequence objects. Immutable sequences methods should at most only\ndefine ``__getslice__()``; mutable sequences might define all three\nmethods.\n\nobject.__getslice__(self, i, j)\n\n Deprecated since version 2.0: Support slice objects as parameters\n to the ``__getitem__()`` method. (However, built-in types in\n CPython currently still implement ``__getslice__()``. Therefore,\n you have to override it in derived classes when implementing\n slicing.)\n\n Called to implement evaluation of ``self[i:j]``. The returned\n object should be of the same type as *self*. Note that missing *i*\n or *j* in the slice expression are replaced by zero or\n ``sys.maxint``, respectively. If negative indexes are used in the\n slice, the length of the sequence is added to that index. If the\n instance does not implement the ``__len__()`` method, an\n ``AttributeError`` is raised. No guarantee is made that indexes\n adjusted this way are not still negative. Indexes which are\n greater than the length of the sequence are not modified. If no\n ``__getslice__()`` is found, a slice object is created instead, and\n passed to ``__getitem__()`` instead.\n\nobject.__setslice__(self, i, j, sequence)\n\n Called to implement assignment to ``self[i:j]``. Same notes for *i*\n and *j* as for ``__getslice__()``.\n\n This method is deprecated. If no ``__setslice__()`` is found, or\n for extended slicing of the form ``self[i:j:k]``, a slice object is\n created, and passed to ``__setitem__()``, instead of\n ``__setslice__()`` being called.\n\nobject.__delslice__(self, i, j)\n\n Called to implement deletion of ``self[i:j]``. Same notes for *i*\n and *j* as for ``__getslice__()``. This method is deprecated. If no\n ``__delslice__()`` is found, or for extended slicing of the form\n ``self[i:j:k]``, a slice object is created, and passed to\n ``__delitem__()``, instead of ``__delslice__()`` being called.\n\nNotice that these methods are only invoked when a single slice with a\nsingle colon is used, and the slice method is available. For slice\noperations involving extended slice notation, or in absence of the\nslice methods, ``__getitem__()``, ``__setitem__()`` or\n``__delitem__()`` is called with a slice object as argument.\n\nThe following example demonstrate how to make your program or module\ncompatible with earlier versions of Python (assuming that methods\n``__getitem__()``, ``__setitem__()`` and ``__delitem__()`` support\nslice objects as arguments):\n\n class MyClass:\n ...\n def __getitem__(self, index):\n ...\n def __setitem__(self, index, value):\n ...\n def __delitem__(self, index):\n ...\n\n if sys.version_info < (2, 0):\n # They won\'t be defined if version is at least 2.0 final\n\n def __getslice__(self, i, j):\n return self[max(0, i):max(0, j):]\n def __setslice__(self, i, j, seq):\n self[max(0, i):max(0, j):] = seq\n def __delslice__(self, i, j):\n del self[max(0, i):max(0, j):]\n ...\n\nNote the calls to ``max()``; these are necessary because of the\nhandling of negative indices before the ``__*slice__()`` methods are\ncalled. When negative indexes are used, the ``__*item__()`` methods\nreceive them as provided, but the ``__*slice__()`` methods get a\n"cooked" form of the index values. For each negative index value, the\nlength of the sequence is added to the index before calling the method\n(which may still result in a negative index); this is the customary\nhandling of negative indexes by the built-in sequence types, and the\n``__*item__()`` methods are expected to do this as well. However,\nsince they should already be doing that, negative indexes cannot be\npassed in; they must be constrained to the bounds of the sequence\nbefore being passed to the ``__*item__()`` methods. Calling ``max(0,\ni)`` conveniently returns the proper value.\n\n\nEmulating numeric types\n=======================\n\nThe following methods can be defined to emulate numeric objects.\nMethods corresponding to operations that are not supported by the\nparticular kind of number implemented (e.g., bitwise operations for\nnon-integral numbers) should be left undefined.\n\nobject.__add__(self, other)\nobject.__sub__(self, other)\nobject.__mul__(self, other)\nobject.__floordiv__(self, other)\nobject.__mod__(self, other)\nobject.__divmod__(self, other)\nobject.__pow__(self, other[, modulo])\nobject.__lshift__(self, other)\nobject.__rshift__(self, other)\nobject.__and__(self, other)\nobject.__xor__(self, other)\nobject.__or__(self, other)\n\n These methods are called to implement the binary arithmetic\n operations (``+``, ``-``, ``*``, ``//``, ``%``, ``divmod()``,\n ``pow()``, ``**``, ``<<``, ``>>``, ``&``, ``^``, ``|``). For\n instance, to evaluate the expression ``x + y``, where *x* is an\n instance of a class that has an ``__add__()`` method,\n ``x.__add__(y)`` is called. The ``__divmod__()`` method should be\n the equivalent to using ``__floordiv__()`` and ``__mod__()``; it\n should not be related to ``__truediv__()`` (described below). Note\n that ``__pow__()`` should be defined to accept an optional third\n argument if the ternary version of the built-in ``pow()`` function\n is to be supported.\n\n If one of those methods does not support the operation with the\n supplied arguments, it should return ``NotImplemented``.\n\nobject.__div__(self, other)\nobject.__truediv__(self, other)\n\n The division operator (``/``) is implemented by these methods. The\n ``__truediv__()`` method is used when ``__future__.division`` is in\n effect, otherwise ``__div__()`` is used. If only one of these two\n methods is defined, the object will not support division in the\n alternate context; ``TypeError`` will be raised instead.\n\nobject.__radd__(self, other)\nobject.__rsub__(self, other)\nobject.__rmul__(self, other)\nobject.__rdiv__(self, other)\nobject.__rtruediv__(self, other)\nobject.__rfloordiv__(self, other)\nobject.__rmod__(self, other)\nobject.__rdivmod__(self, other)\nobject.__rpow__(self, other)\nobject.__rlshift__(self, other)\nobject.__rrshift__(self, other)\nobject.__rand__(self, other)\nobject.__rxor__(self, other)\nobject.__ror__(self, other)\n\n These methods are called to implement the binary arithmetic\n operations (``+``, ``-``, ``*``, ``/``, ``%``, ``divmod()``,\n ``pow()``, ``**``, ``<<``, ``>>``, ``&``, ``^``, ``|``) with\n reflected (swapped) operands. These functions are only called if\n the left operand does not support the corresponding operation and\n the operands are of different types. [2] For instance, to evaluate\n the expression ``x - y``, where *y* is an instance of a class that\n has an ``__rsub__()`` method, ``y.__rsub__(x)`` is called if\n ``x.__sub__(y)`` returns *NotImplemented*.\n\n Note that ternary ``pow()`` will not try calling ``__rpow__()``\n (the coercion rules would become too complicated).\n\n Note: If the right operand\'s type is a subclass of the left operand\'s\n type and that subclass provides the reflected method for the\n operation, this method will be called before the left operand\'s\n non-reflected method. This behavior allows subclasses to\n override their ancestors\' operations.\n\nobject.__iadd__(self, other)\nobject.__isub__(self, other)\nobject.__imul__(self, other)\nobject.__idiv__(self, other)\nobject.__itruediv__(self, other)\nobject.__ifloordiv__(self, other)\nobject.__imod__(self, other)\nobject.__ipow__(self, other[, modulo])\nobject.__ilshift__(self, other)\nobject.__irshift__(self, other)\nobject.__iand__(self, other)\nobject.__ixor__(self, other)\nobject.__ior__(self, other)\n\n These methods are called to implement the augmented arithmetic\n assignments (``+=``, ``-=``, ``*=``, ``/=``, ``//=``, ``%=``,\n ``**=``, ``<<=``, ``>>=``, ``&=``, ``^=``, ``|=``). These methods\n should attempt to do the operation in-place (modifying *self*) and\n return the result (which could be, but does not have to be,\n *self*). If a specific method is not defined, the augmented\n assignment falls back to the normal methods. For instance, to\n execute the statement ``x += y``, where *x* is an instance of a\n class that has an ``__iadd__()`` method, ``x.__iadd__(y)`` is\n called. If *x* is an instance of a class that does not define a\n ``__iadd__()`` method, ``x.__add__(y)`` and ``y.__radd__(x)`` are\n considered, as with the evaluation of ``x + y``.\n\nobject.__neg__(self)\nobject.__pos__(self)\nobject.__abs__(self)\nobject.__invert__(self)\n\n Called to implement the unary arithmetic operations (``-``, ``+``,\n ``abs()`` and ``~``).\n\nobject.__complex__(self)\nobject.__int__(self)\nobject.__long__(self)\nobject.__float__(self)\n\n Called to implement the built-in functions ``complex()``,\n ``int()``, ``long()``, and ``float()``. Should return a value of\n the appropriate type.\n\nobject.__oct__(self)\nobject.__hex__(self)\n\n Called to implement the built-in functions ``oct()`` and ``hex()``.\n Should return a string value.\n\nobject.__index__(self)\n\n Called to implement ``operator.index()``. Also called whenever\n Python needs an integer object (such as in slicing). Must return\n an integer (int or long).\n\n New in version 2.5.\n\nobject.__coerce__(self, other)\n\n Called to implement "mixed-mode" numeric arithmetic. Should either\n return a 2-tuple containing *self* and *other* converted to a\n common numeric type, or ``None`` if conversion is impossible. When\n the common type would be the type of ``other``, it is sufficient to\n return ``None``, since the interpreter will also ask the other\n object to attempt a coercion (but sometimes, if the implementation\n of the other type cannot be changed, it is useful to do the\n conversion to the other type here). A return value of\n ``NotImplemented`` is equivalent to returning ``None``.\n\n\nCoercion rules\n==============\n\nThis section used to document the rules for coercion. As the language\nhas evolved, the coercion rules have become hard to document\nprecisely; documenting what one version of one particular\nimplementation does is undesirable. Instead, here are some informal\nguidelines regarding coercion. In Python 3.0, coercion will not be\nsupported.\n\n* If the left operand of a % operator is a string or Unicode object,\n no coercion takes place and the string formatting operation is\n invoked instead.\n\n* It is no longer recommended to define a coercion operation. Mixed-\n mode operations on types that don\'t define coercion pass the\n original arguments to the operation.\n\n* New-style classes (those derived from ``object``) never invoke the\n ``__coerce__()`` method in response to a binary operator; the only\n time ``__coerce__()`` is invoked is when the built-in function\n ``coerce()`` is called.\n\n* For most intents and purposes, an operator that returns\n ``NotImplemented`` is treated the same as one that is not\n implemented at all.\n\n* Below, ``__op__()`` and ``__rop__()`` are used to signify the\n generic method names corresponding to an operator; ``__iop__()`` is\n used for the corresponding in-place operator. For example, for the\n operator \'``+``\', ``__add__()`` and ``__radd__()`` are used for the\n left and right variant of the binary operator, and ``__iadd__()``\n for the in-place variant.\n\n* For objects *x* and *y*, first ``x.__op__(y)`` is tried. If this is\n not implemented or returns ``NotImplemented``, ``y.__rop__(x)`` is\n tried. If this is also not implemented or returns\n ``NotImplemented``, a ``TypeError`` exception is raised. But see\n the following exception:\n\n* Exception to the previous item: if the left operand is an instance\n of a built-in type or a new-style class, and the right operand is an\n instance of a proper subclass of that type or class and overrides\n the base\'s ``__rop__()`` method, the right operand\'s ``__rop__()``\n method is tried *before* the left operand\'s ``__op__()`` method.\n\n This is done so that a subclass can completely override binary\n operators. Otherwise, the left operand\'s ``__op__()`` method would\n always accept the right operand: when an instance of a given class\n is expected, an instance of a subclass of that class is always\n acceptable.\n\n* When either operand type defines a coercion, this coercion is called\n before that type\'s ``__op__()`` or ``__rop__()`` method is called,\n but no sooner. If the coercion returns an object of a different\n type for the operand whose coercion is invoked, part of the process\n is redone using the new object.\n\n* When an in-place operator (like \'``+=``\') is used, if the left\n operand implements ``__iop__()``, it is invoked without any\n coercion. When the operation falls back to ``__op__()`` and/or\n ``__rop__()``, the normal coercion rules apply.\n\n* In ``x + y``, if *x* is a sequence that implements sequence\n concatenation, sequence concatenation is invoked.\n\n* In ``x * y``, if one operator is a sequence that implements sequence\n repetition, and the other is an integer (``int`` or ``long``),\n sequence repetition is invoked.\n\n* Rich comparisons (implemented by methods ``__eq__()`` and so on)\n never use coercion. Three-way comparison (implemented by\n ``__cmp__()``) does use coercion under the same conditions as other\n binary operations use it.\n\n* In the current implementation, the built-in numeric types ``int``,\n ``long``, ``float``, and ``complex`` do not use coercion. All these\n types implement a ``__coerce__()`` method, for use by the built-in\n ``coerce()`` function.\n\n Changed in version 2.7.\n\n\nWith Statement Context Managers\n===============================\n\nNew in version 2.5.\n\nA *context manager* is an object that defines the runtime context to\nbe established when executing a ``with`` statement. The context\nmanager handles the entry into, and the exit from, the desired runtime\ncontext for the execution of the block of code. Context managers are\nnormally invoked using the ``with`` statement (described in section\n*The with statement*), but can also be used by directly invoking their\nmethods.\n\nTypical uses of context managers include saving and restoring various\nkinds of global state, locking and unlocking resources, closing opened\nfiles, etc.\n\nFor more information on context managers, see *Context Manager Types*.\n\nobject.__enter__(self)\n\n Enter the runtime context related to this object. The ``with``\n statement will bind this method\'s return value to the target(s)\n specified in the ``as`` clause of the statement, if any.\n\nobject.__exit__(self, exc_type, exc_value, traceback)\n\n Exit the runtime context related to this object. The parameters\n describe the exception that caused the context to be exited. If the\n context was exited without an exception, all three arguments will\n be ``None``.\n\n If an exception is supplied, and the method wishes to suppress the\n exception (i.e., prevent it from being propagated), it should\n return a true value. Otherwise, the exception will be processed\n normally upon exit from this method.\n\n Note that ``__exit__()`` methods should not reraise the passed-in\n exception; this is the caller\'s responsibility.\n\nSee also:\n\n **PEP 0343** - The "with" statement\n The specification, background, and examples for the Python\n ``with`` statement.\n\n\nSpecial method lookup for old-style classes\n===========================================\n\nFor old-style classes, special methods are always looked up in exactly\nthe same way as any other method or attribute. This is the case\nregardless of whether the method is being looked up explicitly as in\n``x.__getitem__(i)`` or implicitly as in ``x[i]``.\n\nThis behaviour means that special methods may exhibit different\nbehaviour for different instances of a single old-style class if the\nappropriate special attributes are set differently:\n\n >>> class C:\n ... pass\n ...\n >>> c1 = C()\n >>> c2 = C()\n >>> c1.__len__ = lambda: 5\n >>> c2.__len__ = lambda: 9\n >>> len(c1)\n 5\n >>> len(c2)\n 9\n\n\nSpecial method lookup for new-style classes\n===========================================\n\nFor new-style classes, implicit invocations of special methods are\nonly guaranteed to work correctly if defined on an object\'s type, not\nin the object\'s instance dictionary. That behaviour is the reason why\nthe following code raises an exception (unlike the equivalent example\nwith old-style classes):\n\n >>> class C(object):\n ... pass\n ...\n >>> c = C()\n >>> c.__len__ = lambda: 5\n >>> len(c)\n Traceback (most recent call last):\n File "<stdin>", line 1, in <module>\n TypeError: object of type \'C\' has no len()\n\nThe rationale behind this behaviour lies with a number of special\nmethods such as ``__hash__()`` and ``__repr__()`` that are implemented\nby all objects, including type objects. If the implicit lookup of\nthese methods used the conventional lookup process, they would fail\nwhen invoked on the type object itself:\n\n >>> 1 .__hash__() == hash(1)\n True\n >>> int.__hash__() == hash(int)\n Traceback (most recent call last):\n File "<stdin>", line 1, in <module>\n TypeError: descriptor \'__hash__\' of \'int\' object needs an argument\n\nIncorrectly attempting to invoke an unbound method of a class in this\nway is sometimes referred to as \'metaclass confusion\', and is avoided\nby bypassing the instance when looking up special methods:\n\n >>> type(1).__hash__(1) == hash(1)\n True\n >>> type(int).__hash__(int) == hash(int)\n True\n\nIn addition to bypassing any instance attributes in the interest of\ncorrectness, implicit special method lookup generally also bypasses\nthe ``__getattribute__()`` method even of the object\'s metaclass:\n\n >>> class Meta(type):\n ... def __getattribute__(*args):\n ... print "Metaclass getattribute invoked"\n ... return type.__getattribute__(*args)\n ...\n >>> class C(object):\n ... __metaclass__ = Meta\n ... def __len__(self):\n ... return 10\n ... def __getattribute__(*args):\n ... print "Class getattribute invoked"\n ... return object.__getattribute__(*args)\n ...\n >>> c = C()\n >>> c.__len__() # Explicit lookup via instance\n Class getattribute invoked\n 10\n >>> type(c).__len__(c) # Explicit lookup via type\n Metaclass getattribute invoked\n 10\n >>> len(c) # Implicit lookup\n 10\n\nBypassing the ``__getattribute__()`` machinery in this fashion\nprovides significant scope for speed optimisations within the\ninterpreter, at the cost of some flexibility in the handling of\nspecial methods (the special method *must* be set on the class object\nitself in order to be consistently invoked by the interpreter).\n\n-[ Footnotes ]-\n\n[1] It *is* possible in some cases to change an object\'s type, under\n certain controlled conditions. It generally isn\'t a good idea\n though, since it can lead to some very strange behaviour if it is\n handled incorrectly.\n\n[2] For operands of the same type, it is assumed that if the non-\n reflected method (such as ``__add__()``) fails the operation is\n not supported, which is why the reflected method is not called.\n', 'string-conversions': u'\nString conversions\n******************\n\nA string conversion is an expression list enclosed in reverse (a.k.a.\nbackward) quotes:\n\n string_conversion ::= "\'" expression_list "\'"\n\nA string conversion evaluates the contained expression list and\nconverts the resulting object into a string according to rules\nspecific to its type.\n\nIf the object is a string, a number, ``None``, or a tuple, list or\ndictionary containing only objects whose type is one of these, the\nresulting string is a valid Python expression which can be passed to\nthe built-in function ``eval()`` to yield an expression with the same\nvalue (or an approximation, if floating point numbers are involved).\n\n(In particular, converting a string adds quotes around it and converts\n"funny" characters to escape sequences that are safe to print.)\n\nRecursive objects (for example, lists or dictionaries that contain a\nreference to themselves, directly or indirectly) use ``...`` to\nindicate a recursive reference, and the result cannot be passed to\n``eval()`` to get an equal value (``SyntaxError`` will be raised\ninstead).\n\nThe built-in function ``repr()`` performs exactly the same conversion\nin its argument as enclosing it in parentheses and reverse quotes\ndoes. The built-in function ``str()`` performs a similar but more\nuser-friendly conversion.\n', - 'string-methods': u'\nString Methods\n**************\n\nBelow are listed the string methods which both 8-bit strings and\nUnicode objects support.\n\nIn addition, Python\'s strings support the sequence type methods\ndescribed in the *Sequence Types --- str, unicode, list, tuple,\nbuffer, xrange* section. To output formatted strings use template\nstrings or the ``%`` operator described in the *String Formatting\nOperations* section. Also, see the ``re`` module for string functions\nbased on regular expressions.\n\nstr.capitalize()\n\n Return a copy of the string with only its first character\n capitalized.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.center(width[, fillchar])\n\n Return centered in a string of length *width*. Padding is done\n using the specified *fillchar* (default is a space).\n\n Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.count(sub[, start[, end]])\n\n Return the number of non-overlapping occurrences of substring *sub*\n in the range [*start*, *end*]. Optional arguments *start* and\n *end* are interpreted as in slice notation.\n\nstr.decode([encoding[, errors]])\n\n Decodes the string using the codec registered for *encoding*.\n *encoding* defaults to the default string encoding. *errors* may\n be given to set a different error handling scheme. The default is\n ``\'strict\'``, meaning that encoding errors raise ``UnicodeError``.\n Other possible values are ``\'ignore\'``, ``\'replace\'`` and any other\n name registered via ``codecs.register_error()``, see section *Codec\n Base Classes*.\n\n New in version 2.2.\n\n Changed in version 2.3: Support for other error handling schemes\n added.\n\n Changed in version 2.7: Support for keyword arguments added.\n\nstr.encode([encoding[, errors]])\n\n Return an encoded version of the string. Default encoding is the\n current default string encoding. *errors* may be given to set a\n different error handling scheme. The default for *errors* is\n ``\'strict\'``, meaning that encoding errors raise a\n ``UnicodeError``. Other possible values are ``\'ignore\'``,\n ``\'replace\'``, ``\'xmlcharrefreplace\'``, ``\'backslashreplace\'`` and\n any other name registered via ``codecs.register_error()``, see\n section *Codec Base Classes*. For a list of possible encodings, see\n section *Standard Encodings*.\n\n New in version 2.0.\n\n Changed in version 2.3: Support for ``\'xmlcharrefreplace\'`` and\n ``\'backslashreplace\'`` and other error handling schemes added.\n\n Changed in version 2.7: Support for keyword arguments added.\n\nstr.endswith(suffix[, start[, end]])\n\n Return ``True`` if the string ends with the specified *suffix*,\n otherwise return ``False``. *suffix* can also be a tuple of\n suffixes to look for. With optional *start*, test beginning at\n that position. With optional *end*, stop comparing at that\n position.\n\n Changed in version 2.5: Accept tuples as *suffix*.\n\nstr.expandtabs([tabsize])\n\n Return a copy of the string where all tab characters are replaced\n by one or more spaces, depending on the current column and the\n given tab size. The column number is reset to zero after each\n newline occurring in the string. If *tabsize* is not given, a tab\n size of ``8`` characters is assumed. This doesn\'t understand other\n non-printing characters or escape sequences.\n\nstr.find(sub[, start[, end]])\n\n Return the lowest index in the string where substring *sub* is\n found, such that *sub* is contained in the slice ``s[start:end]``.\n Optional arguments *start* and *end* are interpreted as in slice\n notation. Return ``-1`` if *sub* is not found.\n\nstr.format(*args, **kwargs)\n\n Perform a string formatting operation. The string on which this\n method is called can contain literal text or replacement fields\n delimited by braces ``{}``. Each replacement field contains either\n the numeric index of a positional argument, or the name of a\n keyword argument. Returns a copy of the string where each\n replacement field is replaced with the string value of the\n corresponding argument.\n\n >>> "The sum of 1 + 2 is {0}".format(1+2)\n \'The sum of 1 + 2 is 3\'\n\n See *Format String Syntax* for a description of the various\n formatting options that can be specified in format strings.\n\n This method of string formatting is the new standard in Python 3.0,\n and should be preferred to the ``%`` formatting described in\n *String Formatting Operations* in new code.\n\n New in version 2.6.\n\nstr.index(sub[, start[, end]])\n\n Like ``find()``, but raise ``ValueError`` when the substring is not\n found.\n\nstr.isalnum()\n\n Return true if all characters in the string are alphanumeric and\n there is at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isalpha()\n\n Return true if all characters in the string are alphabetic and\n there is at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isdigit()\n\n Return true if all characters in the string are digits and there is\n at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.islower()\n\n Return true if all cased characters in the string are lowercase and\n there is at least one cased character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isspace()\n\n Return true if there are only whitespace characters in the string\n and there is at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.istitle()\n\n Return true if the string is a titlecased string and there is at\n least one character, for example uppercase characters may only\n follow uncased characters and lowercase characters only cased ones.\n Return false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isupper()\n\n Return true if all cased characters in the string are uppercase and\n there is at least one cased character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.join(iterable)\n\n Return a string which is the concatenation of the strings in the\n *iterable* *iterable*. The separator between elements is the\n string providing this method.\n\nstr.ljust(width[, fillchar])\n\n Return the string left justified in a string of length *width*.\n Padding is done using the specified *fillchar* (default is a\n space). The original string is returned if *width* is less than\n ``len(s)``.\n\n Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.lower()\n\n Return a copy of the string converted to lowercase.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.lstrip([chars])\n\n Return a copy of the string with leading characters removed. The\n *chars* argument is a string specifying the set of characters to be\n removed. If omitted or ``None``, the *chars* argument defaults to\n removing whitespace. The *chars* argument is not a prefix; rather,\n all combinations of its values are stripped:\n\n >>> \' spacious \'.lstrip()\n \'spacious \'\n >>> \'www.example.com\'.lstrip(\'cmowz.\')\n \'example.com\'\n\n Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.partition(sep)\n\n Split the string at the first occurrence of *sep*, and return a\n 3-tuple containing the part before the separator, the separator\n itself, and the part after the separator. If the separator is not\n found, return a 3-tuple containing the string itself, followed by\n two empty strings.\n\n New in version 2.5.\n\nstr.replace(old, new[, count])\n\n Return a copy of the string with all occurrences of substring *old*\n replaced by *new*. If the optional argument *count* is given, only\n the first *count* occurrences are replaced.\n\nstr.rfind(sub[, start[, end]])\n\n Return the highest index in the string where substring *sub* is\n found, such that *sub* is contained within ``s[start:end]``.\n Optional arguments *start* and *end* are interpreted as in slice\n notation. Return ``-1`` on failure.\n\nstr.rindex(sub[, start[, end]])\n\n Like ``rfind()`` but raises ``ValueError`` when the substring *sub*\n is not found.\n\nstr.rjust(width[, fillchar])\n\n Return the string right justified in a string of length *width*.\n Padding is done using the specified *fillchar* (default is a\n space). The original string is returned if *width* is less than\n ``len(s)``.\n\n Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.rpartition(sep)\n\n Split the string at the last occurrence of *sep*, and return a\n 3-tuple containing the part before the separator, the separator\n itself, and the part after the separator. If the separator is not\n found, return a 3-tuple containing two empty strings, followed by\n the string itself.\n\n New in version 2.5.\n\nstr.rsplit([sep[, maxsplit]])\n\n Return a list of the words in the string, using *sep* as the\n delimiter string. If *maxsplit* is given, at most *maxsplit* splits\n are done, the *rightmost* ones. If *sep* is not specified or\n ``None``, any whitespace string is a separator. Except for\n splitting from the right, ``rsplit()`` behaves like ``split()``\n which is described in detail below.\n\n New in version 2.4.\n\nstr.rstrip([chars])\n\n Return a copy of the string with trailing characters removed. The\n *chars* argument is a string specifying the set of characters to be\n removed. If omitted or ``None``, the *chars* argument defaults to\n removing whitespace. The *chars* argument is not a suffix; rather,\n all combinations of its values are stripped:\n\n >>> \' spacious \'.rstrip()\n \' spacious\'\n >>> \'mississippi\'.rstrip(\'ipz\')\n \'mississ\'\n\n Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.split([sep[, maxsplit]])\n\n Return a list of the words in the string, using *sep* as the\n delimiter string. If *maxsplit* is given, at most *maxsplit*\n splits are done (thus, the list will have at most ``maxsplit+1``\n elements). If *maxsplit* is not specified, then there is no limit\n on the number of splits (all possible splits are made).\n\n If *sep* is given, consecutive delimiters are not grouped together\n and are deemed to delimit empty strings (for example,\n ``\'1,,2\'.split(\',\')`` returns ``[\'1\', \'\', \'2\']``). The *sep*\n argument may consist of multiple characters (for example,\n ``\'1<>2<>3\'.split(\'<>\')`` returns ``[\'1\', \'2\', \'3\']``). Splitting\n an empty string with a specified separator returns ``[\'\']``.\n\n If *sep* is not specified or is ``None``, a different splitting\n algorithm is applied: runs of consecutive whitespace are regarded\n as a single separator, and the result will contain no empty strings\n at the start or end if the string has leading or trailing\n whitespace. Consequently, splitting an empty string or a string\n consisting of just whitespace with a ``None`` separator returns\n ``[]``.\n\n For example, ``\' 1 2 3 \'.split()`` returns ``[\'1\', \'2\', \'3\']``,\n and ``\' 1 2 3 \'.split(None, 1)`` returns ``[\'1\', \'2 3 \']``.\n\nstr.splitlines([keepends])\n\n Return a list of the lines in the string, breaking at line\n boundaries. Line breaks are not included in the resulting list\n unless *keepends* is given and true.\n\nstr.startswith(prefix[, start[, end]])\n\n Return ``True`` if string starts with the *prefix*, otherwise\n return ``False``. *prefix* can also be a tuple of prefixes to look\n for. With optional *start*, test string beginning at that\n position. With optional *end*, stop comparing string at that\n position.\n\n Changed in version 2.5: Accept tuples as *prefix*.\n\nstr.strip([chars])\n\n Return a copy of the string with the leading and trailing\n characters removed. The *chars* argument is a string specifying the\n set of characters to be removed. If omitted or ``None``, the\n *chars* argument defaults to removing whitespace. The *chars*\n argument is not a prefix or suffix; rather, all combinations of its\n values are stripped:\n\n >>> \' spacious \'.strip()\n \'spacious\'\n >>> \'www.example.com\'.strip(\'cmowz.\')\n \'example\'\n\n Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.swapcase()\n\n Return a copy of the string with uppercase characters converted to\n lowercase and vice versa.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.title()\n\n Return a titlecased version of the string where words start with an\n uppercase character and the remaining characters are lowercase.\n\n The algorithm uses a simple language-independent definition of a\n word as groups of consecutive letters. The definition works in\n many contexts but it means that apostrophes in contractions and\n possessives form word boundaries, which may not be the desired\n result:\n\n >>> "they\'re bill\'s friends from the UK".title()\n "They\'Re Bill\'S Friends From The Uk"\n\n A workaround for apostrophes can be constructed using regular\n expressions:\n\n >>> import re\n >>> def titlecase(s):\n return re.sub(r"[A-Za-z]+(\'[A-Za-z]+)?",\n lambda mo: mo.group(0)[0].upper() +\n mo.group(0)[1:].lower(),\n s)\n\n >>> titlecase("they\'re bill\'s friends.")\n "They\'re Bill\'s Friends."\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.translate(table[, deletechars])\n\n Return a copy of the string where all characters occurring in the\n optional argument *deletechars* are removed, and the remaining\n characters have been mapped through the given translation table,\n which must be a string of length 256.\n\n You can use the ``maketrans()`` helper function in the ``string``\n module to create a translation table. For string objects, set the\n *table* argument to ``None`` for translations that only delete\n characters:\n\n >>> \'read this short text\'.translate(None, \'aeiou\')\n \'rd ths shrt txt\'\n\n New in version 2.6: Support for a ``None`` *table* argument.\n\n For Unicode objects, the ``translate()`` method does not accept the\n optional *deletechars* argument. Instead, it returns a copy of the\n *s* where all characters have been mapped through the given\n translation table which must be a mapping of Unicode ordinals to\n Unicode ordinals, Unicode strings or ``None``. Unmapped characters\n are left untouched. Characters mapped to ``None`` are deleted.\n Note, a more flexible approach is to create a custom character\n mapping codec using the ``codecs`` module (see ``encodings.cp1251``\n for an example).\n\nstr.upper()\n\n Return a copy of the string converted to uppercase.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.zfill(width)\n\n Return the numeric string left filled with zeros in a string of\n length *width*. A sign prefix is handled correctly. The original\n string is returned if *width* is less than ``len(s)``.\n\n New in version 2.2.2.\n\nThe following methods are present only on unicode objects:\n\nunicode.isnumeric()\n\n Return ``True`` if there are only numeric characters in S,\n ``False`` otherwise. Numeric characters include digit characters,\n and all characters that have the Unicode numeric value property,\n e.g. U+2155, VULGAR FRACTION ONE FIFTH.\n\nunicode.isdecimal()\n\n Return ``True`` if there are only decimal characters in S,\n ``False`` otherwise. Decimal characters include digit characters,\n and all characters that that can be used to form decimal-radix\n numbers, e.g. U+0660, ARABIC-INDIC DIGIT ZERO.\n', - 'strings': u'\nString literals\n***************\n\nString literals are described by the following lexical definitions:\n\n stringliteral ::= [stringprefix](shortstring | longstring)\n stringprefix ::= "r" | "u" | "ur" | "R" | "U" | "UR" | "Ur" | "uR"\n shortstring ::= "\'" shortstringitem* "\'" | \'"\' shortstringitem* \'"\'\n longstring ::= "\'\'\'" longstringitem* "\'\'\'"\n | \'"""\' longstringitem* \'"""\'\n shortstringitem ::= shortstringchar | escapeseq\n longstringitem ::= longstringchar | escapeseq\n shortstringchar ::= <any source character except "\\" or newline or the quote>\n longstringchar ::= <any source character except "\\">\n escapeseq ::= "\\" <any ASCII character>\n\nOne syntactic restriction not indicated by these productions is that\nwhitespace is not allowed between the **stringprefix** and the rest of\nthe string literal. The source character set is defined by the\nencoding declaration; it is ASCII if no encoding declaration is given\nin the source file; see section *Encoding declarations*.\n\nIn plain English: String literals can be enclosed in matching single\nquotes (``\'``) or double quotes (``"``). They can also be enclosed in\nmatching groups of three single or double quotes (these are generally\nreferred to as *triple-quoted strings*). The backslash (``\\``)\ncharacter is used to escape characters that otherwise have a special\nmeaning, such as newline, backslash itself, or the quote character.\nString literals may optionally be prefixed with a letter ``\'r\'`` or\n``\'R\'``; such strings are called *raw strings* and use different rules\nfor interpreting backslash escape sequences. A prefix of ``\'u\'`` or\n``\'U\'`` makes the string a Unicode string. Unicode strings use the\nUnicode character set as defined by the Unicode Consortium and ISO\n10646. Some additional escape sequences, described below, are\navailable in Unicode strings. The two prefix characters may be\ncombined; in this case, ``\'u\'`` must appear before ``\'r\'``.\n\nIn triple-quoted strings, unescaped newlines and quotes are allowed\n(and are retained), except that three unescaped quotes in a row\nterminate the string. (A "quote" is the character used to open the\nstring, i.e. either ``\'`` or ``"``.)\n\nUnless an ``\'r\'`` or ``\'R\'`` prefix is present, escape sequences in\nstrings are interpreted according to rules similar to those used by\nStandard C. The recognized escape sequences are:\n\n+-------------------+-----------------------------------+---------+\n| Escape Sequence | Meaning | Notes |\n+===================+===================================+=========+\n| ``\\newline`` | Ignored | |\n+-------------------+-----------------------------------+---------+\n| ``\\\\`` | Backslash (``\\``) | |\n+-------------------+-----------------------------------+---------+\n| ``\\\'`` | Single quote (``\'``) | |\n+-------------------+-----------------------------------+---------+\n| ``\\"`` | Double quote (``"``) | |\n+-------------------+-----------------------------------+---------+\n| ``\\a`` | ASCII Bell (BEL) | |\n+-------------------+-----------------------------------+---------+\n| ``\\b`` | ASCII Backspace (BS) | |\n+-------------------+-----------------------------------+---------+\n| ``\\f`` | ASCII Formfeed (FF) | |\n+-------------------+-----------------------------------+---------+\n| ``\\n`` | ASCII Linefeed (LF) | |\n+-------------------+-----------------------------------+---------+\n| ``\\N{name}`` | Character named *name* in the | |\n| | Unicode database (Unicode only) | |\n+-------------------+-----------------------------------+---------+\n| ``\\r`` | ASCII Carriage Return (CR) | |\n+-------------------+-----------------------------------+---------+\n| ``\\t`` | ASCII Horizontal Tab (TAB) | |\n+-------------------+-----------------------------------+---------+\n| ``\\uxxxx`` | Character with 16-bit hex value | (1) |\n| | *xxxx* (Unicode only) | |\n+-------------------+-----------------------------------+---------+\n| ``\\Uxxxxxxxx`` | Character with 32-bit hex value | (2) |\n| | *xxxxxxxx* (Unicode only) | |\n+-------------------+-----------------------------------+---------+\n| ``\\v`` | ASCII Vertical Tab (VT) | |\n+-------------------+-----------------------------------+---------+\n| ``\\ooo`` | Character with octal value *ooo* | (3,5) |\n+-------------------+-----------------------------------+---------+\n| ``\\xhh`` | Character with hex value *hh* | (4,5) |\n+-------------------+-----------------------------------+---------+\n\nNotes:\n\n1. Individual code units which form parts of a surrogate pair can be\n encoded using this escape sequence.\n\n2. Any Unicode character can be encoded this way, but characters\n outside the Basic Multilingual Plane (BMP) will be encoded using a\n surrogate pair if Python is compiled to use 16-bit code units (the\n default). Individual code units which form parts of a surrogate\n pair can be encoded using this escape sequence.\n\n3. As in Standard C, up to three octal digits are accepted.\n\n4. Unlike in Standard C, exactly two hex digits are required.\n\n5. In a string literal, hexadecimal and octal escapes denote the byte\n with the given value; it is not necessary that the byte encodes a\n character in the source character set. In a Unicode literal, these\n escapes denote a Unicode character with the given value.\n\nUnlike Standard C, all unrecognized escape sequences are left in the\nstring unchanged, i.e., *the backslash is left in the string*. (This\nbehavior is useful when debugging: if an escape sequence is mistyped,\nthe resulting output is more easily recognized as broken.) It is also\nimportant to note that the escape sequences marked as "(Unicode only)"\nin the table above fall into the category of unrecognized escapes for\nnon-Unicode string literals.\n\nWhen an ``\'r\'`` or ``\'R\'`` prefix is present, a character following a\nbackslash is included in the string without change, and *all\nbackslashes are left in the string*. For example, the string literal\n``r"\\n"`` consists of two characters: a backslash and a lowercase\n``\'n\'``. String quotes can be escaped with a backslash, but the\nbackslash remains in the string; for example, ``r"\\""`` is a valid\nstring literal consisting of two characters: a backslash and a double\nquote; ``r"\\"`` is not a valid string literal (even a raw string\ncannot end in an odd number of backslashes). Specifically, *a raw\nstring cannot end in a single backslash* (since the backslash would\nescape the following quote character). Note also that a single\nbackslash followed by a newline is interpreted as those two characters\nas part of the string, *not* as a line continuation.\n\nWhen an ``\'r\'`` or ``\'R\'`` prefix is used in conjunction with a\n``\'u\'`` or ``\'U\'`` prefix, then the ``\\uXXXX`` and ``\\UXXXXXXXX``\nescape sequences are processed while *all other backslashes are left\nin the string*. For example, the string literal ``ur"\\u0062\\n"``\nconsists of three Unicode characters: \'LATIN SMALL LETTER B\', \'REVERSE\nSOLIDUS\', and \'LATIN SMALL LETTER N\'. Backslashes can be escaped with\na preceding backslash; however, both remain in the string. As a\nresult, ``\\uXXXX`` escape sequences are only recognized when there are\nan odd number of backslashes.\n', + 'string-methods': u'\nString Methods\n**************\n\nBelow are listed the string methods which both 8-bit strings and\nUnicode objects support. Some of them are also available on\n``bytearray`` objects.\n\nIn addition, Python\'s strings support the sequence type methods\ndescribed in the *Sequence Types --- str, unicode, list, tuple,\nbytearray, buffer, xrange* section. To output formatted strings use\ntemplate strings or the ``%`` operator described in the *String\nFormatting Operations* section. Also, see the ``re`` module for string\nfunctions based on regular expressions.\n\nstr.capitalize()\n\n Return a copy of the string with its first character capitalized\n and the rest lowercased.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.center(width[, fillchar])\n\n Return centered in a string of length *width*. Padding is done\n using the specified *fillchar* (default is a space).\n\n Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.count(sub[, start[, end]])\n\n Return the number of non-overlapping occurrences of substring *sub*\n in the range [*start*, *end*]. Optional arguments *start* and\n *end* are interpreted as in slice notation.\n\nstr.decode([encoding[, errors]])\n\n Decodes the string using the codec registered for *encoding*.\n *encoding* defaults to the default string encoding. *errors* may\n be given to set a different error handling scheme. The default is\n ``\'strict\'``, meaning that encoding errors raise ``UnicodeError``.\n Other possible values are ``\'ignore\'``, ``\'replace\'`` and any other\n name registered via ``codecs.register_error()``, see section *Codec\n Base Classes*.\n\n New in version 2.2.\n\n Changed in version 2.3: Support for other error handling schemes\n added.\n\n Changed in version 2.7: Support for keyword arguments added.\n\nstr.encode([encoding[, errors]])\n\n Return an encoded version of the string. Default encoding is the\n current default string encoding. *errors* may be given to set a\n different error handling scheme. The default for *errors* is\n ``\'strict\'``, meaning that encoding errors raise a\n ``UnicodeError``. Other possible values are ``\'ignore\'``,\n ``\'replace\'``, ``\'xmlcharrefreplace\'``, ``\'backslashreplace\'`` and\n any other name registered via ``codecs.register_error()``, see\n section *Codec Base Classes*. For a list of possible encodings, see\n section *Standard Encodings*.\n\n New in version 2.0.\n\n Changed in version 2.3: Support for ``\'xmlcharrefreplace\'`` and\n ``\'backslashreplace\'`` and other error handling schemes added.\n\n Changed in version 2.7: Support for keyword arguments added.\n\nstr.endswith(suffix[, start[, end]])\n\n Return ``True`` if the string ends with the specified *suffix*,\n otherwise return ``False``. *suffix* can also be a tuple of\n suffixes to look for. With optional *start*, test beginning at\n that position. With optional *end*, stop comparing at that\n position.\n\n Changed in version 2.5: Accept tuples as *suffix*.\n\nstr.expandtabs([tabsize])\n\n Return a copy of the string where all tab characters are replaced\n by one or more spaces, depending on the current column and the\n given tab size. The column number is reset to zero after each\n newline occurring in the string. If *tabsize* is not given, a tab\n size of ``8`` characters is assumed. This doesn\'t understand other\n non-printing characters or escape sequences.\n\nstr.find(sub[, start[, end]])\n\n Return the lowest index in the string where substring *sub* is\n found, such that *sub* is contained in the slice ``s[start:end]``.\n Optional arguments *start* and *end* are interpreted as in slice\n notation. Return ``-1`` if *sub* is not found.\n\n Note: The ``find()`` method should be used only if you need to know the\n position of *sub*. To check if *sub* is a substring or not, use\n the ``in`` operator:\n\n >>> \'Py\' in \'Python\'\n True\n\nstr.format(*args, **kwargs)\n\n Perform a string formatting operation. The string on which this\n method is called can contain literal text or replacement fields\n delimited by braces ``{}``. Each replacement field contains either\n the numeric index of a positional argument, or the name of a\n keyword argument. Returns a copy of the string where each\n replacement field is replaced with the string value of the\n corresponding argument.\n\n >>> "The sum of 1 + 2 is {0}".format(1+2)\n \'The sum of 1 + 2 is 3\'\n\n See *Format String Syntax* for a description of the various\n formatting options that can be specified in format strings.\n\n This method of string formatting is the new standard in Python 3.0,\n and should be preferred to the ``%`` formatting described in\n *String Formatting Operations* in new code.\n\n New in version 2.6.\n\nstr.index(sub[, start[, end]])\n\n Like ``find()``, but raise ``ValueError`` when the substring is not\n found.\n\nstr.isalnum()\n\n Return true if all characters in the string are alphanumeric and\n there is at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isalpha()\n\n Return true if all characters in the string are alphabetic and\n there is at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isdigit()\n\n Return true if all characters in the string are digits and there is\n at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.islower()\n\n Return true if all cased characters in the string are lowercase and\n there is at least one cased character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isspace()\n\n Return true if there are only whitespace characters in the string\n and there is at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.istitle()\n\n Return true if the string is a titlecased string and there is at\n least one character, for example uppercase characters may only\n follow uncased characters and lowercase characters only cased ones.\n Return false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isupper()\n\n Return true if all cased characters in the string are uppercase and\n there is at least one cased character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.join(iterable)\n\n Return a string which is the concatenation of the strings in the\n *iterable* *iterable*. The separator between elements is the\n string providing this method.\n\nstr.ljust(width[, fillchar])\n\n Return the string left justified in a string of length *width*.\n Padding is done using the specified *fillchar* (default is a\n space). The original string is returned if *width* is less than\n ``len(s)``.\n\n Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.lower()\n\n Return a copy of the string converted to lowercase.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.lstrip([chars])\n\n Return a copy of the string with leading characters removed. The\n *chars* argument is a string specifying the set of characters to be\n removed. If omitted or ``None``, the *chars* argument defaults to\n removing whitespace. The *chars* argument is not a prefix; rather,\n all combinations of its values are stripped:\n\n >>> \' spacious \'.lstrip()\n \'spacious \'\n >>> \'www.example.com\'.lstrip(\'cmowz.\')\n \'example.com\'\n\n Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.partition(sep)\n\n Split the string at the first occurrence of *sep*, and return a\n 3-tuple containing the part before the separator, the separator\n itself, and the part after the separator. If the separator is not\n found, return a 3-tuple containing the string itself, followed by\n two empty strings.\n\n New in version 2.5.\n\nstr.replace(old, new[, count])\n\n Return a copy of the string with all occurrences of substring *old*\n replaced by *new*. If the optional argument *count* is given, only\n the first *count* occurrences are replaced.\n\nstr.rfind(sub[, start[, end]])\n\n Return the highest index in the string where substring *sub* is\n found, such that *sub* is contained within ``s[start:end]``.\n Optional arguments *start* and *end* are interpreted as in slice\n notation. Return ``-1`` on failure.\n\nstr.rindex(sub[, start[, end]])\n\n Like ``rfind()`` but raises ``ValueError`` when the substring *sub*\n is not found.\n\nstr.rjust(width[, fillchar])\n\n Return the string right justified in a string of length *width*.\n Padding is done using the specified *fillchar* (default is a\n space). The original string is returned if *width* is less than\n ``len(s)``.\n\n Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.rpartition(sep)\n\n Split the string at the last occurrence of *sep*, and return a\n 3-tuple containing the part before the separator, the separator\n itself, and the part after the separator. If the separator is not\n found, return a 3-tuple containing two empty strings, followed by\n the string itself.\n\n New in version 2.5.\n\nstr.rsplit([sep[, maxsplit]])\n\n Return a list of the words in the string, using *sep* as the\n delimiter string. If *maxsplit* is given, at most *maxsplit* splits\n are done, the *rightmost* ones. If *sep* is not specified or\n ``None``, any whitespace string is a separator. Except for\n splitting from the right, ``rsplit()`` behaves like ``split()``\n which is described in detail below.\n\n New in version 2.4.\n\nstr.rstrip([chars])\n\n Return a copy of the string with trailing characters removed. The\n *chars* argument is a string specifying the set of characters to be\n removed. If omitted or ``None``, the *chars* argument defaults to\n removing whitespace. The *chars* argument is not a suffix; rather,\n all combinations of its values are stripped:\n\n >>> \' spacious \'.rstrip()\n \' spacious\'\n >>> \'mississippi\'.rstrip(\'ipz\')\n \'mississ\'\n\n Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.split([sep[, maxsplit]])\n\n Return a list of the words in the string, using *sep* as the\n delimiter string. If *maxsplit* is given, at most *maxsplit*\n splits are done (thus, the list will have at most ``maxsplit+1``\n elements). If *maxsplit* is not specified, then there is no limit\n on the number of splits (all possible splits are made).\n\n If *sep* is given, consecutive delimiters are not grouped together\n and are deemed to delimit empty strings (for example,\n ``\'1,,2\'.split(\',\')`` returns ``[\'1\', \'\', \'2\']``). The *sep*\n argument may consist of multiple characters (for example,\n ``\'1<>2<>3\'.split(\'<>\')`` returns ``[\'1\', \'2\', \'3\']``). Splitting\n an empty string with a specified separator returns ``[\'\']``.\n\n If *sep* is not specified or is ``None``, a different splitting\n algorithm is applied: runs of consecutive whitespace are regarded\n as a single separator, and the result will contain no empty strings\n at the start or end if the string has leading or trailing\n whitespace. Consequently, splitting an empty string or a string\n consisting of just whitespace with a ``None`` separator returns\n ``[]``.\n\n For example, ``\' 1 2 3 \'.split()`` returns ``[\'1\', \'2\', \'3\']``,\n and ``\' 1 2 3 \'.split(None, 1)`` returns ``[\'1\', \'2 3 \']``.\n\nstr.splitlines([keepends])\n\n Return a list of the lines in the string, breaking at line\n boundaries. Line breaks are not included in the resulting list\n unless *keepends* is given and true.\n\nstr.startswith(prefix[, start[, end]])\n\n Return ``True`` if string starts with the *prefix*, otherwise\n return ``False``. *prefix* can also be a tuple of prefixes to look\n for. With optional *start*, test string beginning at that\n position. With optional *end*, stop comparing string at that\n position.\n\n Changed in version 2.5: Accept tuples as *prefix*.\n\nstr.strip([chars])\n\n Return a copy of the string with the leading and trailing\n characters removed. The *chars* argument is a string specifying the\n set of characters to be removed. If omitted or ``None``, the\n *chars* argument defaults to removing whitespace. The *chars*\n argument is not a prefix or suffix; rather, all combinations of its\n values are stripped:\n\n >>> \' spacious \'.strip()\n \'spacious\'\n >>> \'www.example.com\'.strip(\'cmowz.\')\n \'example\'\n\n Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.swapcase()\n\n Return a copy of the string with uppercase characters converted to\n lowercase and vice versa.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.title()\n\n Return a titlecased version of the string where words start with an\n uppercase character and the remaining characters are lowercase.\n\n The algorithm uses a simple language-independent definition of a\n word as groups of consecutive letters. The definition works in\n many contexts but it means that apostrophes in contractions and\n possessives form word boundaries, which may not be the desired\n result:\n\n >>> "they\'re bill\'s friends from the UK".title()\n "They\'Re Bill\'S Friends From The Uk"\n\n A workaround for apostrophes can be constructed using regular\n expressions:\n\n >>> import re\n >>> def titlecase(s):\n return re.sub(r"[A-Za-z]+(\'[A-Za-z]+)?",\n lambda mo: mo.group(0)[0].upper() +\n mo.group(0)[1:].lower(),\n s)\n\n >>> titlecase("they\'re bill\'s friends.")\n "They\'re Bill\'s Friends."\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.translate(table[, deletechars])\n\n Return a copy of the string where all characters occurring in the\n optional argument *deletechars* are removed, and the remaining\n characters have been mapped through the given translation table,\n which must be a string of length 256.\n\n You can use the ``maketrans()`` helper function in the ``string``\n module to create a translation table. For string objects, set the\n *table* argument to ``None`` for translations that only delete\n characters:\n\n >>> \'read this short text\'.translate(None, \'aeiou\')\n \'rd ths shrt txt\'\n\n New in version 2.6: Support for a ``None`` *table* argument.\n\n For Unicode objects, the ``translate()`` method does not accept the\n optional *deletechars* argument. Instead, it returns a copy of the\n *s* where all characters have been mapped through the given\n translation table which must be a mapping of Unicode ordinals to\n Unicode ordinals, Unicode strings or ``None``. Unmapped characters\n are left untouched. Characters mapped to ``None`` are deleted.\n Note, a more flexible approach is to create a custom character\n mapping codec using the ``codecs`` module (see ``encodings.cp1251``\n for an example).\n\nstr.upper()\n\n Return a copy of the string converted to uppercase.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.zfill(width)\n\n Return the numeric string left filled with zeros in a string of\n length *width*. A sign prefix is handled correctly. The original\n string is returned if *width* is less than ``len(s)``.\n\n New in version 2.2.2.\n\nThe following methods are present only on unicode objects:\n\nunicode.isnumeric()\n\n Return ``True`` if there are only numeric characters in S,\n ``False`` otherwise. Numeric characters include digit characters,\n and all characters that have the Unicode numeric value property,\n e.g. U+2155, VULGAR FRACTION ONE FIFTH.\n\nunicode.isdecimal()\n\n Return ``True`` if there are only decimal characters in S,\n ``False`` otherwise. Decimal characters include digit characters,\n and all characters that that can be used to form decimal-radix\n numbers, e.g. U+0660, ARABIC-INDIC DIGIT ZERO.\n', + 'strings': u'\nString literals\n***************\n\nString literals are described by the following lexical definitions:\n\n stringliteral ::= [stringprefix](shortstring | longstring)\n stringprefix ::= "r" | "u" | "ur" | "R" | "U" | "UR" | "Ur" | "uR"\n | "b" | "B" | "br" | "Br" | "bR" | "BR"\n shortstring ::= "\'" shortstringitem* "\'" | \'"\' shortstringitem* \'"\'\n longstring ::= "\'\'\'" longstringitem* "\'\'\'"\n | \'"""\' longstringitem* \'"""\'\n shortstringitem ::= shortstringchar | escapeseq\n longstringitem ::= longstringchar | escapeseq\n shortstringchar ::= <any source character except "\\" or newline or the quote>\n longstringchar ::= <any source character except "\\">\n escapeseq ::= "\\" <any ASCII character>\n\nOne syntactic restriction not indicated by these productions is that\nwhitespace is not allowed between the **stringprefix** and the rest of\nthe string literal. The source character set is defined by the\nencoding declaration; it is ASCII if no encoding declaration is given\nin the source file; see section *Encoding declarations*.\n\nIn plain English: String literals can be enclosed in matching single\nquotes (``\'``) or double quotes (``"``). They can also be enclosed in\nmatching groups of three single or double quotes (these are generally\nreferred to as *triple-quoted strings*). The backslash (``\\``)\ncharacter is used to escape characters that otherwise have a special\nmeaning, such as newline, backslash itself, or the quote character.\nString literals may optionally be prefixed with a letter ``\'r\'`` or\n``\'R\'``; such strings are called *raw strings* and use different rules\nfor interpreting backslash escape sequences. A prefix of ``\'u\'`` or\n``\'U\'`` makes the string a Unicode string. Unicode strings use the\nUnicode character set as defined by the Unicode Consortium and ISO\n10646. Some additional escape sequences, described below, are\navailable in Unicode strings. A prefix of ``\'b\'`` or ``\'B\'`` is\nignored in Python 2; it indicates that the literal should become a\nbytes literal in Python 3 (e.g. when code is automatically converted\nwith 2to3). A ``\'u\'`` or ``\'b\'`` prefix may be followed by an ``\'r\'``\nprefix.\n\nIn triple-quoted strings, unescaped newlines and quotes are allowed\n(and are retained), except that three unescaped quotes in a row\nterminate the string. (A "quote" is the character used to open the\nstring, i.e. either ``\'`` or ``"``.)\n\nUnless an ``\'r\'`` or ``\'R\'`` prefix is present, escape sequences in\nstrings are interpreted according to rules similar to those used by\nStandard C. The recognized escape sequences are:\n\n+-------------------+-----------------------------------+---------+\n| Escape Sequence | Meaning | Notes |\n+===================+===================================+=========+\n| ``\\newline`` | Ignored | |\n+-------------------+-----------------------------------+---------+\n| ``\\\\`` | Backslash (``\\``) | |\n+-------------------+-----------------------------------+---------+\n| ``\\\'`` | Single quote (``\'``) | |\n+-------------------+-----------------------------------+---------+\n| ``\\"`` | Double quote (``"``) | |\n+-------------------+-----------------------------------+---------+\n| ``\\a`` | ASCII Bell (BEL) | |\n+-------------------+-----------------------------------+---------+\n| ``\\b`` | ASCII Backspace (BS) | |\n+-------------------+-----------------------------------+---------+\n| ``\\f`` | ASCII Formfeed (FF) | |\n+-------------------+-----------------------------------+---------+\n| ``\\n`` | ASCII Linefeed (LF) | |\n+-------------------+-----------------------------------+---------+\n| ``\\N{name}`` | Character named *name* in the | |\n| | Unicode database (Unicode only) | |\n+-------------------+-----------------------------------+---------+\n| ``\\r`` | ASCII Carriage Return (CR) | |\n+-------------------+-----------------------------------+---------+\n| ``\\t`` | ASCII Horizontal Tab (TAB) | |\n+-------------------+-----------------------------------+---------+\n| ``\\uxxxx`` | Character with 16-bit hex value | (1) |\n| | *xxxx* (Unicode only) | |\n+-------------------+-----------------------------------+---------+\n| ``\\Uxxxxxxxx`` | Character with 32-bit hex value | (2) |\n| | *xxxxxxxx* (Unicode only) | |\n+-------------------+-----------------------------------+---------+\n| ``\\v`` | ASCII Vertical Tab (VT) | |\n+-------------------+-----------------------------------+---------+\n| ``\\ooo`` | Character with octal value *ooo* | (3,5) |\n+-------------------+-----------------------------------+---------+\n| ``\\xhh`` | Character with hex value *hh* | (4,5) |\n+-------------------+-----------------------------------+---------+\n\nNotes:\n\n1. Individual code units which form parts of a surrogate pair can be\n encoded using this escape sequence.\n\n2. Any Unicode character can be encoded this way, but characters\n outside the Basic Multilingual Plane (BMP) will be encoded using a\n surrogate pair if Python is compiled to use 16-bit code units (the\n default). Individual code units which form parts of a surrogate\n pair can be encoded using this escape sequence.\n\n3. As in Standard C, up to three octal digits are accepted.\n\n4. Unlike in Standard C, exactly two hex digits are required.\n\n5. In a string literal, hexadecimal and octal escapes denote the byte\n with the given value; it is not necessary that the byte encodes a\n character in the source character set. In a Unicode literal, these\n escapes denote a Unicode character with the given value.\n\nUnlike Standard C, all unrecognized escape sequences are left in the\nstring unchanged, i.e., *the backslash is left in the string*. (This\nbehavior is useful when debugging: if an escape sequence is mistyped,\nthe resulting output is more easily recognized as broken.) It is also\nimportant to note that the escape sequences marked as "(Unicode only)"\nin the table above fall into the category of unrecognized escapes for\nnon-Unicode string literals.\n\nWhen an ``\'r\'`` or ``\'R\'`` prefix is present, a character following a\nbackslash is included in the string without change, and *all\nbackslashes are left in the string*. For example, the string literal\n``r"\\n"`` consists of two characters: a backslash and a lowercase\n``\'n\'``. String quotes can be escaped with a backslash, but the\nbackslash remains in the string; for example, ``r"\\""`` is a valid\nstring literal consisting of two characters: a backslash and a double\nquote; ``r"\\"`` is not a valid string literal (even a raw string\ncannot end in an odd number of backslashes). Specifically, *a raw\nstring cannot end in a single backslash* (since the backslash would\nescape the following quote character). Note also that a single\nbackslash followed by a newline is interpreted as those two characters\nas part of the string, *not* as a line continuation.\n\nWhen an ``\'r\'`` or ``\'R\'`` prefix is used in conjunction with a\n``\'u\'`` or ``\'U\'`` prefix, then the ``\\uXXXX`` and ``\\UXXXXXXXX``\nescape sequences are processed while *all other backslashes are left\nin the string*. For example, the string literal ``ur"\\u0062\\n"``\nconsists of three Unicode characters: \'LATIN SMALL LETTER B\', \'REVERSE\nSOLIDUS\', and \'LATIN SMALL LETTER N\'. Backslashes can be escaped with\na preceding backslash; however, both remain in the string. As a\nresult, ``\\uXXXX`` escape sequences are only recognized when there are\nan odd number of backslashes.\n', 'subscriptions': u'\nSubscriptions\n*************\n\nA subscription selects an item of a sequence (string, tuple or list)\nor mapping (dictionary) object:\n\n subscription ::= primary "[" expression_list "]"\n\nThe primary must evaluate to an object of a sequence or mapping type.\n\nIf the primary is a mapping, the expression list must evaluate to an\nobject whose value is one of the keys of the mapping, and the\nsubscription selects the value in the mapping that corresponds to that\nkey. (The expression list is a tuple except if it has exactly one\nitem.)\n\nIf the primary is a sequence, the expression (list) must evaluate to a\nplain integer. If this value is negative, the length of the sequence\nis added to it (so that, e.g., ``x[-1]`` selects the last item of\n``x``.) The resulting value must be a nonnegative integer less than\nthe number of items in the sequence, and the subscription selects the\nitem whose index is that value (counting from zero).\n\nA string\'s items are characters. A character is not a separate data\ntype but a string of exactly one character.\n', 'truth': u"\nTruth Value Testing\n*******************\n\nAny object can be tested for truth value, for use in an ``if`` or\n``while`` condition or as operand of the Boolean operations below. The\nfollowing values are considered false:\n\n* ``None``\n\n* ``False``\n\n* zero of any numeric type, for example, ``0``, ``0L``, ``0.0``,\n ``0j``.\n\n* any empty sequence, for example, ``''``, ``()``, ``[]``.\n\n* any empty mapping, for example, ``{}``.\n\n* instances of user-defined classes, if the class defines a\n ``__nonzero__()`` or ``__len__()`` method, when that method returns\n the integer zero or ``bool`` value ``False``. [1]\n\nAll other values are considered true --- so objects of many types are\nalways true.\n\nOperations and built-in functions that have a Boolean result always\nreturn ``0`` or ``False`` for false and ``1`` or ``True`` for true,\nunless otherwise stated. (Important exception: the Boolean operations\n``or`` and ``and`` always return one of their operands.)\n", 'try': u'\nThe ``try`` statement\n*********************\n\nThe ``try`` statement specifies exception handlers and/or cleanup code\nfor a group of statements:\n\n try_stmt ::= try1_stmt | try2_stmt\n try1_stmt ::= "try" ":" suite\n ("except" [expression [("as" | ",") target]] ":" suite)+\n ["else" ":" suite]\n ["finally" ":" suite]\n try2_stmt ::= "try" ":" suite\n "finally" ":" suite\n\nChanged in version 2.5: In previous versions of Python,\n``try``...``except``...``finally`` did not work. ``try``...``except``\nhad to be nested in ``try``...``finally``.\n\nThe ``except`` clause(s) specify one or more exception handlers. When\nno exception occurs in the ``try`` clause, no exception handler is\nexecuted. When an exception occurs in the ``try`` suite, a search for\nan exception handler is started. This search inspects the except\nclauses in turn until one is found that matches the exception. An\nexpression-less except clause, if present, must be last; it matches\nany exception. For an except clause with an expression, that\nexpression is evaluated, and the clause matches the exception if the\nresulting object is "compatible" with the exception. An object is\ncompatible with an exception if it is the class or a base class of the\nexception object, a tuple containing an item compatible with the\nexception, or, in the (deprecated) case of string exceptions, is the\nraised string itself (note that the object identities must match, i.e.\nit must be the same string object, not just a string with the same\nvalue).\n\nIf no except clause matches the exception, the search for an exception\nhandler continues in the surrounding code and on the invocation stack.\n[1]\n\nIf the evaluation of an expression in the header of an except clause\nraises an exception, the original search for a handler is canceled and\na search starts for the new exception in the surrounding code and on\nthe call stack (it is treated as if the entire ``try`` statement\nraised the exception).\n\nWhen a matching except clause is found, the exception is assigned to\nthe target specified in that except clause, if present, and the except\nclause\'s suite is executed. All except clauses must have an\nexecutable block. When the end of this block is reached, execution\ncontinues normally after the entire try statement. (This means that\nif two nested handlers exist for the same exception, and the exception\noccurs in the try clause of the inner handler, the outer handler will\nnot handle the exception.)\n\nBefore an except clause\'s suite is executed, details about the\nexception are assigned to three variables in the ``sys`` module:\n``sys.exc_type`` receives the object identifying the exception;\n``sys.exc_value`` receives the exception\'s parameter;\n``sys.exc_traceback`` receives a traceback object (see section *The\nstandard type hierarchy*) identifying the point in the program where\nthe exception occurred. These details are also available through the\n``sys.exc_info()`` function, which returns a tuple ``(exc_type,\nexc_value, exc_traceback)``. Use of the corresponding variables is\ndeprecated in favor of this function, since their use is unsafe in a\nthreaded program. As of Python 1.5, the variables are restored to\ntheir previous values (before the call) when returning from a function\nthat handled an exception.\n\nThe optional ``else`` clause is executed if and when control flows off\nthe end of the ``try`` clause. [2] Exceptions in the ``else`` clause\nare not handled by the preceding ``except`` clauses.\n\nIf ``finally`` is present, it specifies a \'cleanup\' handler. The\n``try`` clause is executed, including any ``except`` and ``else``\nclauses. If an exception occurs in any of the clauses and is not\nhandled, the exception is temporarily saved. The ``finally`` clause is\nexecuted. If there is a saved exception, it is re-raised at the end\nof the ``finally`` clause. If the ``finally`` clause raises another\nexception or executes a ``return`` or ``break`` statement, the saved\nexception is lost. The exception information is not available to the\nprogram during execution of the ``finally`` clause.\n\nWhen a ``return``, ``break`` or ``continue`` statement is executed in\nthe ``try`` suite of a ``try``...``finally`` statement, the\n``finally`` clause is also executed \'on the way out.\' A ``continue``\nstatement is illegal in the ``finally`` clause. (The reason is a\nproblem with the current implementation --- this restriction may be\nlifted in the future).\n\nAdditional information on exceptions can be found in section\n*Exceptions*, and information on using the ``raise`` statement to\ngenerate exceptions may be found in section *The raise statement*.\n', - 'types': u'\nThe standard type hierarchy\n***************************\n\nBelow is a list of the types that are built into Python. Extension\nmodules (written in C, Java, or other languages, depending on the\nimplementation) can define additional types. Future versions of\nPython may add types to the type hierarchy (e.g., rational numbers,\nefficiently stored arrays of integers, etc.).\n\nSome of the type descriptions below contain a paragraph listing\n\'special attributes.\' These are attributes that provide access to the\nimplementation and are not intended for general use. Their definition\nmay change in the future.\n\nNone\n This type has a single value. There is a single object with this\n value. This object is accessed through the built-in name ``None``.\n It is used to signify the absence of a value in many situations,\n e.g., it is returned from functions that don\'t explicitly return\n anything. Its truth value is false.\n\nNotImplemented\n This type has a single value. There is a single object with this\n value. This object is accessed through the built-in name\n ``NotImplemented``. Numeric methods and rich comparison methods may\n return this value if they do not implement the operation for the\n operands provided. (The interpreter will then try the reflected\n operation, or some other fallback, depending on the operator.) Its\n truth value is true.\n\nEllipsis\n This type has a single value. There is a single object with this\n value. This object is accessed through the built-in name\n ``Ellipsis``. It is used to indicate the presence of the ``...``\n syntax in a slice. Its truth value is true.\n\n``numbers.Number``\n These are created by numeric literals and returned as results by\n arithmetic operators and arithmetic built-in functions. Numeric\n objects are immutable; once created their value never changes.\n Python numbers are of course strongly related to mathematical\n numbers, but subject to the limitations of numerical representation\n in computers.\n\n Python distinguishes between integers, floating point numbers, and\n complex numbers:\n\n ``numbers.Integral``\n These represent elements from the mathematical set of integers\n (positive and negative).\n\n There are three types of integers:\n\n Plain integers\n These represent numbers in the range -2147483648 through\n 2147483647. (The range may be larger on machines with a\n larger natural word size, but not smaller.) When the result\n of an operation would fall outside this range, the result is\n normally returned as a long integer (in some cases, the\n exception ``OverflowError`` is raised instead). For the\n purpose of shift and mask operations, integers are assumed to\n have a binary, 2\'s complement notation using 32 or more bits,\n and hiding no bits from the user (i.e., all 4294967296\n different bit patterns correspond to different values).\n\n Long integers\n These represent numbers in an unlimited range, subject to\n available (virtual) memory only. For the purpose of shift\n and mask operations, a binary representation is assumed, and\n negative numbers are represented in a variant of 2\'s\n complement which gives the illusion of an infinite string of\n sign bits extending to the left.\n\n Booleans\n These represent the truth values False and True. The two\n objects representing the values False and True are the only\n Boolean objects. The Boolean type is a subtype of plain\n integers, and Boolean values behave like the values 0 and 1,\n respectively, in almost all contexts, the exception being\n that when converted to a string, the strings ``"False"`` or\n ``"True"`` are returned, respectively.\n\n The rules for integer representation are intended to give the\n most meaningful interpretation of shift and mask operations\n involving negative integers and the least surprises when\n switching between the plain and long integer domains. Any\n operation, if it yields a result in the plain integer domain,\n will yield the same result in the long integer domain or when\n using mixed operands. The switch between domains is transparent\n to the programmer.\n\n ``numbers.Real`` (``float``)\n These represent machine-level double precision floating point\n numbers. You are at the mercy of the underlying machine\n architecture (and C or Java implementation) for the accepted\n range and handling of overflow. Python does not support single-\n precision floating point numbers; the savings in processor and\n memory usage that are usually the reason for using these is\n dwarfed by the overhead of using objects in Python, so there is\n no reason to complicate the language with two kinds of floating\n point numbers.\n\n ``numbers.Complex``\n These represent complex numbers as a pair of machine-level\n double precision floating point numbers. The same caveats apply\n as for floating point numbers. The real and imaginary parts of a\n complex number ``z`` can be retrieved through the read-only\n attributes ``z.real`` and ``z.imag``.\n\nSequences\n These represent finite ordered sets indexed by non-negative\n numbers. The built-in function ``len()`` returns the number of\n items of a sequence. When the length of a sequence is *n*, the\n index set contains the numbers 0, 1, ..., *n*-1. Item *i* of\n sequence *a* is selected by ``a[i]``.\n\n Sequences also support slicing: ``a[i:j]`` selects all items with\n index *k* such that *i* ``<=`` *k* ``<`` *j*. When used as an\n expression, a slice is a sequence of the same type. This implies\n that the index set is renumbered so that it starts at 0.\n\n Some sequences also support "extended slicing" with a third "step"\n parameter: ``a[i:j:k]`` selects all items of *a* with index *x*\n where ``x = i + n*k``, *n* ``>=`` ``0`` and *i* ``<=`` *x* ``<``\n *j*.\n\n Sequences are distinguished according to their mutability:\n\n Immutable sequences\n An object of an immutable sequence type cannot change once it is\n created. (If the object contains references to other objects,\n these other objects may be mutable and may be changed; however,\n the collection of objects directly referenced by an immutable\n object cannot change.)\n\n The following types are immutable sequences:\n\n Strings\n The items of a string are characters. There is no separate\n character type; a character is represented by a string of one\n item. Characters represent (at least) 8-bit bytes. The\n built-in functions ``chr()`` and ``ord()`` convert between\n characters and nonnegative integers representing the byte\n values. Bytes with the values 0-127 usually represent the\n corresponding ASCII values, but the interpretation of values\n is up to the program. The string data type is also used to\n represent arrays of bytes, e.g., to hold data read from a\n file.\n\n (On systems whose native character set is not ASCII, strings\n may use EBCDIC in their internal representation, provided the\n functions ``chr()`` and ``ord()`` implement a mapping between\n ASCII and EBCDIC, and string comparison preserves the ASCII\n order. Or perhaps someone can propose a better rule?)\n\n Unicode\n The items of a Unicode object are Unicode code units. A\n Unicode code unit is represented by a Unicode object of one\n item and can hold either a 16-bit or 32-bit value\n representing a Unicode ordinal (the maximum value for the\n ordinal is given in ``sys.maxunicode``, and depends on how\n Python is configured at compile time). Surrogate pairs may\n be present in the Unicode object, and will be reported as two\n separate items. The built-in functions ``unichr()`` and\n ``ord()`` convert between code units and nonnegative integers\n representing the Unicode ordinals as defined in the Unicode\n Standard 3.0. Conversion from and to other encodings are\n possible through the Unicode method ``encode()`` and the\n built-in function ``unicode()``.\n\n Tuples\n The items of a tuple are arbitrary Python objects. Tuples of\n two or more items are formed by comma-separated lists of\n expressions. A tuple of one item (a \'singleton\') can be\n formed by affixing a comma to an expression (an expression by\n itself does not create a tuple, since parentheses must be\n usable for grouping of expressions). An empty tuple can be\n formed by an empty pair of parentheses.\n\n Mutable sequences\n Mutable sequences can be changed after they are created. The\n subscription and slicing notations can be used as the target of\n assignment and ``del`` (delete) statements.\n\n There are currently two intrinsic mutable sequence types:\n\n Lists\n The items of a list are arbitrary Python objects. Lists are\n formed by placing a comma-separated list of expressions in\n square brackets. (Note that there are no special cases needed\n to form lists of length 0 or 1.)\n\n Byte Arrays\n A bytearray object is a mutable array. They are created by\n the built-in ``bytearray()`` constructor. Aside from being\n mutable (and hence unhashable), byte arrays otherwise provide\n the same interface and functionality as immutable bytes\n objects.\n\n The extension module ``array`` provides an additional example of\n a mutable sequence type.\n\nSet types\n These represent unordered, finite sets of unique, immutable\n objects. As such, they cannot be indexed by any subscript. However,\n they can be iterated over, and the built-in function ``len()``\n returns the number of items in a set. Common uses for sets are fast\n membership testing, removing duplicates from a sequence, and\n computing mathematical operations such as intersection, union,\n difference, and symmetric difference.\n\n For set elements, the same immutability rules apply as for\n dictionary keys. Note that numeric types obey the normal rules for\n numeric comparison: if two numbers compare equal (e.g., ``1`` and\n ``1.0``), only one of them can be contained in a set.\n\n There are currently two intrinsic set types:\n\n Sets\n These represent a mutable set. They are created by the built-in\n ``set()`` constructor and can be modified afterwards by several\n methods, such as ``add()``.\n\n Frozen sets\n These represent an immutable set. They are created by the\n built-in ``frozenset()`` constructor. As a frozenset is\n immutable and *hashable*, it can be used again as an element of\n another set, or as a dictionary key.\n\nMappings\n These represent finite sets of objects indexed by arbitrary index\n sets. The subscript notation ``a[k]`` selects the item indexed by\n ``k`` from the mapping ``a``; this can be used in expressions and\n as the target of assignments or ``del`` statements. The built-in\n function ``len()`` returns the number of items in a mapping.\n\n There is currently a single intrinsic mapping type:\n\n Dictionaries\n These represent finite sets of objects indexed by nearly\n arbitrary values. The only types of values not acceptable as\n keys are values containing lists or dictionaries or other\n mutable types that are compared by value rather than by object\n identity, the reason being that the efficient implementation of\n dictionaries requires a key\'s hash value to remain constant.\n Numeric types used for keys obey the normal rules for numeric\n comparison: if two numbers compare equal (e.g., ``1`` and\n ``1.0``) then they can be used interchangeably to index the same\n dictionary entry.\n\n Dictionaries are mutable; they can be created by the ``{...}``\n notation (see section *Dictionary displays*).\n\n The extension modules ``dbm``, ``gdbm``, and ``bsddb`` provide\n additional examples of mapping types.\n\nCallable types\n These are the types to which the function call operation (see\n section *Calls*) can be applied:\n\n User-defined functions\n A user-defined function object is created by a function\n definition (see section *Function definitions*). It should be\n called with an argument list containing the same number of items\n as the function\'s formal parameter list.\n\n Special attributes:\n\n +-------------------------+---------------------------------+-------------+\n | Attribute | Meaning | |\n +=========================+=================================+=============+\n | ``func_doc`` | The function\'s documentation | Writable |\n | | string, or ``None`` if | |\n | | unavailable | |\n +-------------------------+---------------------------------+-------------+\n | ``__doc__`` | Another way of spelling | Writable |\n | | ``func_doc`` | |\n +-------------------------+---------------------------------+-------------+\n | ``func_name`` | The function\'s name | Writable |\n +-------------------------+---------------------------------+-------------+\n | ``__name__`` | Another way of spelling | Writable |\n | | ``func_name`` | |\n +-------------------------+---------------------------------+-------------+\n | ``__module__`` | The name of the module the | Writable |\n | | function was defined in, or | |\n | | ``None`` if unavailable. | |\n +-------------------------+---------------------------------+-------------+\n | ``func_defaults`` | A tuple containing default | Writable |\n | | argument values for those | |\n | | arguments that have defaults, | |\n | | or ``None`` if no arguments | |\n | | have a default value | |\n +-------------------------+---------------------------------+-------------+\n | ``func_code`` | The code object representing | Writable |\n | | the compiled function body. | |\n +-------------------------+---------------------------------+-------------+\n | ``func_globals`` | A reference to the dictionary | Read-only |\n | | that holds the function\'s | |\n | | global variables --- the global | |\n | | namespace of the module in | |\n | | which the function was defined. | |\n +-------------------------+---------------------------------+-------------+\n | ``func_dict`` | The namespace supporting | Writable |\n | | arbitrary function attributes. | |\n +-------------------------+---------------------------------+-------------+\n | ``func_closure`` | ``None`` or a tuple of cells | Read-only |\n | | that contain bindings for the | |\n | | function\'s free variables. | |\n +-------------------------+---------------------------------+-------------+\n\n Most of the attributes labelled "Writable" check the type of the\n assigned value.\n\n Changed in version 2.4: ``func_name`` is now writable.\n\n Function objects also support getting and setting arbitrary\n attributes, which can be used, for example, to attach metadata\n to functions. Regular attribute dot-notation is used to get and\n set such attributes. *Note that the current implementation only\n supports function attributes on user-defined functions. Function\n attributes on built-in functions may be supported in the\n future.*\n\n Additional information about a function\'s definition can be\n retrieved from its code object; see the description of internal\n types below.\n\n User-defined methods\n A user-defined method object combines a class, a class instance\n (or ``None``) and any callable object (normally a user-defined\n function).\n\n Special read-only attributes: ``im_self`` is the class instance\n object, ``im_func`` is the function object; ``im_class`` is the\n class of ``im_self`` for bound methods or the class that asked\n for the method for unbound methods; ``__doc__`` is the method\'s\n documentation (same as ``im_func.__doc__``); ``__name__`` is the\n method name (same as ``im_func.__name__``); ``__module__`` is\n the name of the module the method was defined in, or ``None`` if\n unavailable.\n\n Changed in version 2.2: ``im_self`` used to refer to the class\n that defined the method.\n\n Changed in version 2.6: For 3.0 forward-compatibility,\n ``im_func`` is also available as ``__func__``, and ``im_self``\n as ``__self__``.\n\n Methods also support accessing (but not setting) the arbitrary\n function attributes on the underlying function object.\n\n User-defined method objects may be created when getting an\n attribute of a class (perhaps via an instance of that class), if\n that attribute is a user-defined function object, an unbound\n user-defined method object, or a class method object. When the\n attribute is a user-defined method object, a new method object\n is only created if the class from which it is being retrieved is\n the same as, or a derived class of, the class stored in the\n original method object; otherwise, the original method object is\n used as it is.\n\n When a user-defined method object is created by retrieving a\n user-defined function object from a class, its ``im_self``\n attribute is ``None`` and the method object is said to be\n unbound. When one is created by retrieving a user-defined\n function object from a class via one of its instances, its\n ``im_self`` attribute is the instance, and the method object is\n said to be bound. In either case, the new method\'s ``im_class``\n attribute is the class from which the retrieval takes place, and\n its ``im_func`` attribute is the original function object.\n\n When a user-defined method object is created by retrieving\n another method object from a class or instance, the behaviour is\n the same as for a function object, except that the ``im_func``\n attribute of the new instance is not the original method object\n but its ``im_func`` attribute.\n\n When a user-defined method object is created by retrieving a\n class method object from a class or instance, its ``im_self``\n attribute is the class itself (the same as the ``im_class``\n attribute), and its ``im_func`` attribute is the function object\n underlying the class method.\n\n When an unbound user-defined method object is called, the\n underlying function (``im_func``) is called, with the\n restriction that the first argument must be an instance of the\n proper class (``im_class``) or of a derived class thereof.\n\n When a bound user-defined method object is called, the\n underlying function (``im_func``) is called, inserting the class\n instance (``im_self``) in front of the argument list. For\n instance, when ``C`` is a class which contains a definition for\n a function ``f()``, and ``x`` is an instance of ``C``, calling\n ``x.f(1)`` is equivalent to calling ``C.f(x, 1)``.\n\n When a user-defined method object is derived from a class method\n object, the "class instance" stored in ``im_self`` will actually\n be the class itself, so that calling either ``x.f(1)`` or\n ``C.f(1)`` is equivalent to calling ``f(C,1)`` where ``f`` is\n the underlying function.\n\n Note that the transformation from function object to (unbound or\n bound) method object happens each time the attribute is\n retrieved from the class or instance. In some cases, a fruitful\n optimization is to assign the attribute to a local variable and\n call that local variable. Also notice that this transformation\n only happens for user-defined functions; other callable objects\n (and all non-callable objects) are retrieved without\n transformation. It is also important to note that user-defined\n functions which are attributes of a class instance are not\n converted to bound methods; this *only* happens when the\n function is an attribute of the class.\n\n Generator functions\n A function or method which uses the ``yield`` statement (see\n section *The yield statement*) is called a *generator function*.\n Such a function, when called, always returns an iterator object\n which can be used to execute the body of the function: calling\n the iterator\'s ``next()`` method will cause the function to\n execute until it provides a value using the ``yield`` statement.\n When the function executes a ``return`` statement or falls off\n the end, a ``StopIteration`` exception is raised and the\n iterator will have reached the end of the set of values to be\n returned.\n\n Built-in functions\n A built-in function object is a wrapper around a C function.\n Examples of built-in functions are ``len()`` and ``math.sin()``\n (``math`` is a standard built-in module). The number and type of\n the arguments are determined by the C function. Special read-\n only attributes: ``__doc__`` is the function\'s documentation\n string, or ``None`` if unavailable; ``__name__`` is the\n function\'s name; ``__self__`` is set to ``None`` (but see the\n next item); ``__module__`` is the name of the module the\n function was defined in or ``None`` if unavailable.\n\n Built-in methods\n This is really a different disguise of a built-in function, this\n time containing an object passed to the C function as an\n implicit extra argument. An example of a built-in method is\n ``alist.append()``, assuming *alist* is a list object. In this\n case, the special read-only attribute ``__self__`` is set to the\n object denoted by *list*.\n\n Class Types\n Class types, or "new-style classes," are callable. These\n objects normally act as factories for new instances of\n themselves, but variations are possible for class types that\n override ``__new__()``. The arguments of the call are passed to\n ``__new__()`` and, in the typical case, to ``__init__()`` to\n initialize the new instance.\n\n Classic Classes\n Class objects are described below. When a class object is\n called, a new class instance (also described below) is created\n and returned. This implies a call to the class\'s ``__init__()``\n method if it has one. Any arguments are passed on to the\n ``__init__()`` method. If there is no ``__init__()`` method,\n the class must be called without arguments.\n\n Class instances\n Class instances are described below. Class instances are\n callable only when the class has a ``__call__()`` method;\n ``x(arguments)`` is a shorthand for ``x.__call__(arguments)``.\n\nModules\n Modules are imported by the ``import`` statement (see section *The\n import statement*). A module object has a namespace implemented by\n a dictionary object (this is the dictionary referenced by the\n func_globals attribute of functions defined in the module).\n Attribute references are translated to lookups in this dictionary,\n e.g., ``m.x`` is equivalent to ``m.__dict__["x"]``. A module object\n does not contain the code object used to initialize the module\n (since it isn\'t needed once the initialization is done).\n\n Attribute assignment updates the module\'s namespace dictionary,\n e.g., ``m.x = 1`` is equivalent to ``m.__dict__["x"] = 1``.\n\n Special read-only attribute: ``__dict__`` is the module\'s namespace\n as a dictionary object.\n\n Predefined (writable) attributes: ``__name__`` is the module\'s\n name; ``__doc__`` is the module\'s documentation string, or ``None``\n if unavailable; ``__file__`` is the pathname of the file from which\n the module was loaded, if it was loaded from a file. The\n ``__file__`` attribute is not present for C modules that are\n statically linked into the interpreter; for extension modules\n loaded dynamically from a shared library, it is the pathname of the\n shared library file.\n\nClasses\n Both class types (new-style classes) and class objects (old-\n style/classic classes) are typically created by class definitions\n (see section *Class definitions*). A class has a namespace\n implemented by a dictionary object. Class attribute references are\n translated to lookups in this dictionary, e.g., ``C.x`` is\n translated to ``C.__dict__["x"]`` (although for new-style classes\n in particular there are a number of hooks which allow for other\n means of locating attributes). When the attribute name is not found\n there, the attribute search continues in the base classes. For\n old-style classes, the search is depth-first, left-to-right in the\n order of occurrence in the base class list. New-style classes use\n the more complex C3 method resolution order which behaves correctly\n even in the presence of \'diamond\' inheritance structures where\n there are multiple inheritance paths leading back to a common\n ancestor. Additional details on the C3 MRO used by new-style\n classes can be found in the documentation accompanying the 2.3\n release at http://www.python.org/download/releases/2.3/mro/.\n\n When a class attribute reference (for class ``C``, say) would yield\n a user-defined function object or an unbound user-defined method\n object whose associated class is either ``C`` or one of its base\n classes, it is transformed into an unbound user-defined method\n object whose ``im_class`` attribute is ``C``. When it would yield a\n class method object, it is transformed into a bound user-defined\n method object whose ``im_class`` and ``im_self`` attributes are\n both ``C``. When it would yield a static method object, it is\n transformed into the object wrapped by the static method object.\n See section *Implementing Descriptors* for another way in which\n attributes retrieved from a class may differ from those actually\n contained in its ``__dict__`` (note that only new-style classes\n support descriptors).\n\n Class attribute assignments update the class\'s dictionary, never\n the dictionary of a base class.\n\n A class object can be called (see above) to yield a class instance\n (see below).\n\n Special attributes: ``__name__`` is the class name; ``__module__``\n is the module name in which the class was defined; ``__dict__`` is\n the dictionary containing the class\'s namespace; ``__bases__`` is a\n tuple (possibly empty or a singleton) containing the base classes,\n in the order of their occurrence in the base class list;\n ``__doc__`` is the class\'s documentation string, or None if\n undefined.\n\nClass instances\n A class instance is created by calling a class object (see above).\n A class instance has a namespace implemented as a dictionary which\n is the first place in which attribute references are searched.\n When an attribute is not found there, and the instance\'s class has\n an attribute by that name, the search continues with the class\n attributes. If a class attribute is found that is a user-defined\n function object or an unbound user-defined method object whose\n associated class is the class (call it ``C``) of the instance for\n which the attribute reference was initiated or one of its bases, it\n is transformed into a bound user-defined method object whose\n ``im_class`` attribute is ``C`` and whose ``im_self`` attribute is\n the instance. Static method and class method objects are also\n transformed, as if they had been retrieved from class ``C``; see\n above under "Classes". See section *Implementing Descriptors* for\n another way in which attributes of a class retrieved via its\n instances may differ from the objects actually stored in the\n class\'s ``__dict__``. If no class attribute is found, and the\n object\'s class has a ``__getattr__()`` method, that is called to\n satisfy the lookup.\n\n Attribute assignments and deletions update the instance\'s\n dictionary, never a class\'s dictionary. If the class has a\n ``__setattr__()`` or ``__delattr__()`` method, this is called\n instead of updating the instance dictionary directly.\n\n Class instances can pretend to be numbers, sequences, or mappings\n if they have methods with certain special names. See section\n *Special method names*.\n\n Special attributes: ``__dict__`` is the attribute dictionary;\n ``__class__`` is the instance\'s class.\n\nFiles\n A file object represents an open file. File objects are created by\n the ``open()`` built-in function, and also by ``os.popen()``,\n ``os.fdopen()``, and the ``makefile()`` method of socket objects\n (and perhaps by other functions or methods provided by extension\n modules). The objects ``sys.stdin``, ``sys.stdout`` and\n ``sys.stderr`` are initialized to file objects corresponding to the\n interpreter\'s standard input, output and error streams. See *File\n Objects* for complete documentation of file objects.\n\nInternal types\n A few types used internally by the interpreter are exposed to the\n user. Their definitions may change with future versions of the\n interpreter, but they are mentioned here for completeness.\n\n Code objects\n Code objects represent *byte-compiled* executable Python code,\n or *bytecode*. The difference between a code object and a\n function object is that the function object contains an explicit\n reference to the function\'s globals (the module in which it was\n defined), while a code object contains no context; also the\n default argument values are stored in the function object, not\n in the code object (because they represent values calculated at\n run-time). Unlike function objects, code objects are immutable\n and contain no references (directly or indirectly) to mutable\n objects.\n\n Special read-only attributes: ``co_name`` gives the function\n name; ``co_argcount`` is the number of positional arguments\n (including arguments with default values); ``co_nlocals`` is the\n number of local variables used by the function (including\n arguments); ``co_varnames`` is a tuple containing the names of\n the local variables (starting with the argument names);\n ``co_cellvars`` is a tuple containing the names of local\n variables that are referenced by nested functions;\n ``co_freevars`` is a tuple containing the names of free\n variables; ``co_code`` is a string representing the sequence of\n bytecode instructions; ``co_consts`` is a tuple containing the\n literals used by the bytecode; ``co_names`` is a tuple\n containing the names used by the bytecode; ``co_filename`` is\n the filename from which the code was compiled;\n ``co_firstlineno`` is the first line number of the function;\n ``co_lnotab`` is a string encoding the mapping from bytecode\n offsets to line numbers (for details see the source code of the\n interpreter); ``co_stacksize`` is the required stack size\n (including local variables); ``co_flags`` is an integer encoding\n a number of flags for the interpreter.\n\n The following flag bits are defined for ``co_flags``: bit\n ``0x04`` is set if the function uses the ``*arguments`` syntax\n to accept an arbitrary number of positional arguments; bit\n ``0x08`` is set if the function uses the ``**keywords`` syntax\n to accept arbitrary keyword arguments; bit ``0x20`` is set if\n the function is a generator.\n\n Future feature declarations (``from __future__ import\n division``) also use bits in ``co_flags`` to indicate whether a\n code object was compiled with a particular feature enabled: bit\n ``0x2000`` is set if the function was compiled with future\n division enabled; bits ``0x10`` and ``0x1000`` were used in\n earlier versions of Python.\n\n Other bits in ``co_flags`` are reserved for internal use.\n\n If a code object represents a function, the first item in\n ``co_consts`` is the documentation string of the function, or\n ``None`` if undefined.\n\n Frame objects\n Frame objects represent execution frames. They may occur in\n traceback objects (see below).\n\n Special read-only attributes: ``f_back`` is to the previous\n stack frame (towards the caller), or ``None`` if this is the\n bottom stack frame; ``f_code`` is the code object being executed\n in this frame; ``f_locals`` is the dictionary used to look up\n local variables; ``f_globals`` is used for global variables;\n ``f_builtins`` is used for built-in (intrinsic) names;\n ``f_restricted`` is a flag indicating whether the function is\n executing in restricted execution mode; ``f_lasti`` gives the\n precise instruction (this is an index into the bytecode string\n of the code object).\n\n Special writable attributes: ``f_trace``, if not ``None``, is a\n function called at the start of each source code line (this is\n used by the debugger); ``f_exc_type``, ``f_exc_value``,\n ``f_exc_traceback`` represent the last exception raised in the\n parent frame provided another exception was ever raised in the\n current frame (in all other cases they are None); ``f_lineno``\n is the current line number of the frame --- writing to this from\n within a trace function jumps to the given line (only for the\n bottom-most frame). A debugger can implement a Jump command\n (aka Set Next Statement) by writing to f_lineno.\n\n Traceback objects\n Traceback objects represent a stack trace of an exception. A\n traceback object is created when an exception occurs. When the\n search for an exception handler unwinds the execution stack, at\n each unwound level a traceback object is inserted in front of\n the current traceback. When an exception handler is entered,\n the stack trace is made available to the program. (See section\n *The try statement*.) It is accessible as ``sys.exc_traceback``,\n and also as the third item of the tuple returned by\n ``sys.exc_info()``. The latter is the preferred interface,\n since it works correctly when the program is using multiple\n threads. When the program contains no suitable handler, the\n stack trace is written (nicely formatted) to the standard error\n stream; if the interpreter is interactive, it is also made\n available to the user as ``sys.last_traceback``.\n\n Special read-only attributes: ``tb_next`` is the next level in\n the stack trace (towards the frame where the exception\n occurred), or ``None`` if there is no next level; ``tb_frame``\n points to the execution frame of the current level;\n ``tb_lineno`` gives the line number where the exception\n occurred; ``tb_lasti`` indicates the precise instruction. The\n line number and last instruction in the traceback may differ\n from the line number of its frame object if the exception\n occurred in a ``try`` statement with no matching except clause\n or with a finally clause.\n\n Slice objects\n Slice objects are used to represent slices when *extended slice\n syntax* is used. This is a slice using two colons, or multiple\n slices or ellipses separated by commas, e.g., ``a[i:j:step]``,\n ``a[i:j, k:l]``, or ``a[..., i:j]``. They are also created by\n the built-in ``slice()`` function.\n\n Special read-only attributes: ``start`` is the lower bound;\n ``stop`` is the upper bound; ``step`` is the step value; each is\n ``None`` if omitted. These attributes can have any type.\n\n Slice objects support one method:\n\n slice.indices(self, length)\n\n This method takes a single integer argument *length* and\n computes information about the extended slice that the slice\n object would describe if applied to a sequence of *length*\n items. It returns a tuple of three integers; respectively\n these are the *start* and *stop* indices and the *step* or\n stride length of the slice. Missing or out-of-bounds indices\n are handled in a manner consistent with regular slices.\n\n New in version 2.3.\n\n Static method objects\n Static method objects provide a way of defeating the\n transformation of function objects to method objects described\n above. A static method object is a wrapper around any other\n object, usually a user-defined method object. When a static\n method object is retrieved from a class or a class instance, the\n object actually returned is the wrapped object, which is not\n subject to any further transformation. Static method objects are\n not themselves callable, although the objects they wrap usually\n are. Static method objects are created by the built-in\n ``staticmethod()`` constructor.\n\n Class method objects\n A class method object, like a static method object, is a wrapper\n around another object that alters the way in which that object\n is retrieved from classes and class instances. The behaviour of\n class method objects upon such retrieval is described above,\n under "User-defined methods". Class method objects are created\n by the built-in ``classmethod()`` constructor.\n', + 'types': u'\nThe standard type hierarchy\n***************************\n\nBelow is a list of the types that are built into Python. Extension\nmodules (written in C, Java, or other languages, depending on the\nimplementation) can define additional types. Future versions of\nPython may add types to the type hierarchy (e.g., rational numbers,\nefficiently stored arrays of integers, etc.).\n\nSome of the type descriptions below contain a paragraph listing\n\'special attributes.\' These are attributes that provide access to the\nimplementation and are not intended for general use. Their definition\nmay change in the future.\n\nNone\n This type has a single value. There is a single object with this\n value. This object is accessed through the built-in name ``None``.\n It is used to signify the absence of a value in many situations,\n e.g., it is returned from functions that don\'t explicitly return\n anything. Its truth value is false.\n\nNotImplemented\n This type has a single value. There is a single object with this\n value. This object is accessed through the built-in name\n ``NotImplemented``. Numeric methods and rich comparison methods may\n return this value if they do not implement the operation for the\n operands provided. (The interpreter will then try the reflected\n operation, or some other fallback, depending on the operator.) Its\n truth value is true.\n\nEllipsis\n This type has a single value. There is a single object with this\n value. This object is accessed through the built-in name\n ``Ellipsis``. It is used to indicate the presence of the ``...``\n syntax in a slice. Its truth value is true.\n\n``numbers.Number``\n These are created by numeric literals and returned as results by\n arithmetic operators and arithmetic built-in functions. Numeric\n objects are immutable; once created their value never changes.\n Python numbers are of course strongly related to mathematical\n numbers, but subject to the limitations of numerical representation\n in computers.\n\n Python distinguishes between integers, floating point numbers, and\n complex numbers:\n\n ``numbers.Integral``\n These represent elements from the mathematical set of integers\n (positive and negative).\n\n There are three types of integers:\n\n Plain integers\n These represent numbers in the range -2147483648 through\n 2147483647. (The range may be larger on machines with a\n larger natural word size, but not smaller.) When the result\n of an operation would fall outside this range, the result is\n normally returned as a long integer (in some cases, the\n exception ``OverflowError`` is raised instead). For the\n purpose of shift and mask operations, integers are assumed to\n have a binary, 2\'s complement notation using 32 or more bits,\n and hiding no bits from the user (i.e., all 4294967296\n different bit patterns correspond to different values).\n\n Long integers\n These represent numbers in an unlimited range, subject to\n available (virtual) memory only. For the purpose of shift\n and mask operations, a binary representation is assumed, and\n negative numbers are represented in a variant of 2\'s\n complement which gives the illusion of an infinite string of\n sign bits extending to the left.\n\n Booleans\n These represent the truth values False and True. The two\n objects representing the values False and True are the only\n Boolean objects. The Boolean type is a subtype of plain\n integers, and Boolean values behave like the values 0 and 1,\n respectively, in almost all contexts, the exception being\n that when converted to a string, the strings ``"False"`` or\n ``"True"`` are returned, respectively.\n\n The rules for integer representation are intended to give the\n most meaningful interpretation of shift and mask operations\n involving negative integers and the least surprises when\n switching between the plain and long integer domains. Any\n operation, if it yields a result in the plain integer domain,\n will yield the same result in the long integer domain or when\n using mixed operands. The switch between domains is transparent\n to the programmer.\n\n ``numbers.Real`` (``float``)\n These represent machine-level double precision floating point\n numbers. You are at the mercy of the underlying machine\n architecture (and C or Java implementation) for the accepted\n range and handling of overflow. Python does not support single-\n precision floating point numbers; the savings in processor and\n memory usage that are usually the reason for using these is\n dwarfed by the overhead of using objects in Python, so there is\n no reason to complicate the language with two kinds of floating\n point numbers.\n\n ``numbers.Complex``\n These represent complex numbers as a pair of machine-level\n double precision floating point numbers. The same caveats apply\n as for floating point numbers. The real and imaginary parts of a\n complex number ``z`` can be retrieved through the read-only\n attributes ``z.real`` and ``z.imag``.\n\nSequences\n These represent finite ordered sets indexed by non-negative\n numbers. The built-in function ``len()`` returns the number of\n items of a sequence. When the length of a sequence is *n*, the\n index set contains the numbers 0, 1, ..., *n*-1. Item *i* of\n sequence *a* is selected by ``a[i]``.\n\n Sequences also support slicing: ``a[i:j]`` selects all items with\n index *k* such that *i* ``<=`` *k* ``<`` *j*. When used as an\n expression, a slice is a sequence of the same type. This implies\n that the index set is renumbered so that it starts at 0.\n\n Some sequences also support "extended slicing" with a third "step"\n parameter: ``a[i:j:k]`` selects all items of *a* with index *x*\n where ``x = i + n*k``, *n* ``>=`` ``0`` and *i* ``<=`` *x* ``<``\n *j*.\n\n Sequences are distinguished according to their mutability:\n\n Immutable sequences\n An object of an immutable sequence type cannot change once it is\n created. (If the object contains references to other objects,\n these other objects may be mutable and may be changed; however,\n the collection of objects directly referenced by an immutable\n object cannot change.)\n\n The following types are immutable sequences:\n\n Strings\n The items of a string are characters. There is no separate\n character type; a character is represented by a string of one\n item. Characters represent (at least) 8-bit bytes. The\n built-in functions ``chr()`` and ``ord()`` convert between\n characters and nonnegative integers representing the byte\n values. Bytes with the values 0-127 usually represent the\n corresponding ASCII values, but the interpretation of values\n is up to the program. The string data type is also used to\n represent arrays of bytes, e.g., to hold data read from a\n file.\n\n (On systems whose native character set is not ASCII, strings\n may use EBCDIC in their internal representation, provided the\n functions ``chr()`` and ``ord()`` implement a mapping between\n ASCII and EBCDIC, and string comparison preserves the ASCII\n order. Or perhaps someone can propose a better rule?)\n\n Unicode\n The items of a Unicode object are Unicode code units. A\n Unicode code unit is represented by a Unicode object of one\n item and can hold either a 16-bit or 32-bit value\n representing a Unicode ordinal (the maximum value for the\n ordinal is given in ``sys.maxunicode``, and depends on how\n Python is configured at compile time). Surrogate pairs may\n be present in the Unicode object, and will be reported as two\n separate items. The built-in functions ``unichr()`` and\n ``ord()`` convert between code units and nonnegative integers\n representing the Unicode ordinals as defined in the Unicode\n Standard 3.0. Conversion from and to other encodings are\n possible through the Unicode method ``encode()`` and the\n built-in function ``unicode()``.\n\n Tuples\n The items of a tuple are arbitrary Python objects. Tuples of\n two or more items are formed by comma-separated lists of\n expressions. A tuple of one item (a \'singleton\') can be\n formed by affixing a comma to an expression (an expression by\n itself does not create a tuple, since parentheses must be\n usable for grouping of expressions). An empty tuple can be\n formed by an empty pair of parentheses.\n\n Mutable sequences\n Mutable sequences can be changed after they are created. The\n subscription and slicing notations can be used as the target of\n assignment and ``del`` (delete) statements.\n\n There are currently two intrinsic mutable sequence types:\n\n Lists\n The items of a list are arbitrary Python objects. Lists are\n formed by placing a comma-separated list of expressions in\n square brackets. (Note that there are no special cases needed\n to form lists of length 0 or 1.)\n\n Byte Arrays\n A bytearray object is a mutable array. They are created by\n the built-in ``bytearray()`` constructor. Aside from being\n mutable (and hence unhashable), byte arrays otherwise provide\n the same interface and functionality as immutable bytes\n objects.\n\n The extension module ``array`` provides an additional example of\n a mutable sequence type.\n\nSet types\n These represent unordered, finite sets of unique, immutable\n objects. As such, they cannot be indexed by any subscript. However,\n they can be iterated over, and the built-in function ``len()``\n returns the number of items in a set. Common uses for sets are fast\n membership testing, removing duplicates from a sequence, and\n computing mathematical operations such as intersection, union,\n difference, and symmetric difference.\n\n For set elements, the same immutability rules apply as for\n dictionary keys. Note that numeric types obey the normal rules for\n numeric comparison: if two numbers compare equal (e.g., ``1`` and\n ``1.0``), only one of them can be contained in a set.\n\n There are currently two intrinsic set types:\n\n Sets\n These represent a mutable set. They are created by the built-in\n ``set()`` constructor and can be modified afterwards by several\n methods, such as ``add()``.\n\n Frozen sets\n These represent an immutable set. They are created by the\n built-in ``frozenset()`` constructor. As a frozenset is\n immutable and *hashable*, it can be used again as an element of\n another set, or as a dictionary key.\n\nMappings\n These represent finite sets of objects indexed by arbitrary index\n sets. The subscript notation ``a[k]`` selects the item indexed by\n ``k`` from the mapping ``a``; this can be used in expressions and\n as the target of assignments or ``del`` statements. The built-in\n function ``len()`` returns the number of items in a mapping.\n\n There is currently a single intrinsic mapping type:\n\n Dictionaries\n These represent finite sets of objects indexed by nearly\n arbitrary values. The only types of values not acceptable as\n keys are values containing lists or dictionaries or other\n mutable types that are compared by value rather than by object\n identity, the reason being that the efficient implementation of\n dictionaries requires a key\'s hash value to remain constant.\n Numeric types used for keys obey the normal rules for numeric\n comparison: if two numbers compare equal (e.g., ``1`` and\n ``1.0``) then they can be used interchangeably to index the same\n dictionary entry.\n\n Dictionaries are mutable; they can be created by the ``{...}``\n notation (see section *Dictionary displays*).\n\n The extension modules ``dbm``, ``gdbm``, and ``bsddb`` provide\n additional examples of mapping types.\n\nCallable types\n These are the types to which the function call operation (see\n section *Calls*) can be applied:\n\n User-defined functions\n A user-defined function object is created by a function\n definition (see section *Function definitions*). It should be\n called with an argument list containing the same number of items\n as the function\'s formal parameter list.\n\n Special attributes:\n\n +-------------------------+---------------------------------+-------------+\n | Attribute | Meaning | |\n +=========================+=================================+=============+\n | ``func_doc`` | The function\'s documentation | Writable |\n | | string, or ``None`` if | |\n | | unavailable | |\n +-------------------------+---------------------------------+-------------+\n | ``__doc__`` | Another way of spelling | Writable |\n | | ``func_doc`` | |\n +-------------------------+---------------------------------+-------------+\n | ``func_name`` | The function\'s name | Writable |\n +-------------------------+---------------------------------+-------------+\n | ``__name__`` | Another way of spelling | Writable |\n | | ``func_name`` | |\n +-------------------------+---------------------------------+-------------+\n | ``__module__`` | The name of the module the | Writable |\n | | function was defined in, or | |\n | | ``None`` if unavailable. | |\n +-------------------------+---------------------------------+-------------+\n | ``func_defaults`` | A tuple containing default | Writable |\n | | argument values for those | |\n | | arguments that have defaults, | |\n | | or ``None`` if no arguments | |\n | | have a default value | |\n +-------------------------+---------------------------------+-------------+\n | ``func_code`` | The code object representing | Writable |\n | | the compiled function body. | |\n +-------------------------+---------------------------------+-------------+\n | ``func_globals`` | A reference to the dictionary | Read-only |\n | | that holds the function\'s | |\n | | global variables --- the global | |\n | | namespace of the module in | |\n | | which the function was defined. | |\n +-------------------------+---------------------------------+-------------+\n | ``func_dict`` | The namespace supporting | Writable |\n | | arbitrary function attributes. | |\n +-------------------------+---------------------------------+-------------+\n | ``func_closure`` | ``None`` or a tuple of cells | Read-only |\n | | that contain bindings for the | |\n | | function\'s free variables. | |\n +-------------------------+---------------------------------+-------------+\n\n Most of the attributes labelled "Writable" check the type of the\n assigned value.\n\n Changed in version 2.4: ``func_name`` is now writable.\n\n Function objects also support getting and setting arbitrary\n attributes, which can be used, for example, to attach metadata\n to functions. Regular attribute dot-notation is used to get and\n set such attributes. *Note that the current implementation only\n supports function attributes on user-defined functions. Function\n attributes on built-in functions may be supported in the\n future.*\n\n Additional information about a function\'s definition can be\n retrieved from its code object; see the description of internal\n types below.\n\n User-defined methods\n A user-defined method object combines a class, a class instance\n (or ``None``) and any callable object (normally a user-defined\n function).\n\n Special read-only attributes: ``im_self`` is the class instance\n object, ``im_func`` is the function object; ``im_class`` is the\n class of ``im_self`` for bound methods or the class that asked\n for the method for unbound methods; ``__doc__`` is the method\'s\n documentation (same as ``im_func.__doc__``); ``__name__`` is the\n method name (same as ``im_func.__name__``); ``__module__`` is\n the name of the module the method was defined in, or ``None`` if\n unavailable.\n\n Changed in version 2.2: ``im_self`` used to refer to the class\n that defined the method.\n\n Changed in version 2.6: For 3.0 forward-compatibility,\n ``im_func`` is also available as ``__func__``, and ``im_self``\n as ``__self__``.\n\n Methods also support accessing (but not setting) the arbitrary\n function attributes on the underlying function object.\n\n User-defined method objects may be created when getting an\n attribute of a class (perhaps via an instance of that class), if\n that attribute is a user-defined function object, an unbound\n user-defined method object, or a class method object. When the\n attribute is a user-defined method object, a new method object\n is only created if the class from which it is being retrieved is\n the same as, or a derived class of, the class stored in the\n original method object; otherwise, the original method object is\n used as it is.\n\n When a user-defined method object is created by retrieving a\n user-defined function object from a class, its ``im_self``\n attribute is ``None`` and the method object is said to be\n unbound. When one is created by retrieving a user-defined\n function object from a class via one of its instances, its\n ``im_self`` attribute is the instance, and the method object is\n said to be bound. In either case, the new method\'s ``im_class``\n attribute is the class from which the retrieval takes place, and\n its ``im_func`` attribute is the original function object.\n\n When a user-defined method object is created by retrieving\n another method object from a class or instance, the behaviour is\n the same as for a function object, except that the ``im_func``\n attribute of the new instance is not the original method object\n but its ``im_func`` attribute.\n\n When a user-defined method object is created by retrieving a\n class method object from a class or instance, its ``im_self``\n attribute is the class itself (the same as the ``im_class``\n attribute), and its ``im_func`` attribute is the function object\n underlying the class method.\n\n When an unbound user-defined method object is called, the\n underlying function (``im_func``) is called, with the\n restriction that the first argument must be an instance of the\n proper class (``im_class``) or of a derived class thereof.\n\n When a bound user-defined method object is called, the\n underlying function (``im_func``) is called, inserting the class\n instance (``im_self``) in front of the argument list. For\n instance, when ``C`` is a class which contains a definition for\n a function ``f()``, and ``x`` is an instance of ``C``, calling\n ``x.f(1)`` is equivalent to calling ``C.f(x, 1)``.\n\n When a user-defined method object is derived from a class method\n object, the "class instance" stored in ``im_self`` will actually\n be the class itself, so that calling either ``x.f(1)`` or\n ``C.f(1)`` is equivalent to calling ``f(C,1)`` where ``f`` is\n the underlying function.\n\n Note that the transformation from function object to (unbound or\n bound) method object happens each time the attribute is\n retrieved from the class or instance. In some cases, a fruitful\n optimization is to assign the attribute to a local variable and\n call that local variable. Also notice that this transformation\n only happens for user-defined functions; other callable objects\n (and all non-callable objects) are retrieved without\n transformation. It is also important to note that user-defined\n functions which are attributes of a class instance are not\n converted to bound methods; this *only* happens when the\n function is an attribute of the class.\n\n Generator functions\n A function or method which uses the ``yield`` statement (see\n section *The yield statement*) is called a *generator function*.\n Such a function, when called, always returns an iterator object\n which can be used to execute the body of the function: calling\n the iterator\'s ``next()`` method will cause the function to\n execute until it provides a value using the ``yield`` statement.\n When the function executes a ``return`` statement or falls off\n the end, a ``StopIteration`` exception is raised and the\n iterator will have reached the end of the set of values to be\n returned.\n\n Built-in functions\n A built-in function object is a wrapper around a C function.\n Examples of built-in functions are ``len()`` and ``math.sin()``\n (``math`` is a standard built-in module). The number and type of\n the arguments are determined by the C function. Special read-\n only attributes: ``__doc__`` is the function\'s documentation\n string, or ``None`` if unavailable; ``__name__`` is the\n function\'s name; ``__self__`` is set to ``None`` (but see the\n next item); ``__module__`` is the name of the module the\n function was defined in or ``None`` if unavailable.\n\n Built-in methods\n This is really a different disguise of a built-in function, this\n time containing an object passed to the C function as an\n implicit extra argument. An example of a built-in method is\n ``alist.append()``, assuming *alist* is a list object. In this\n case, the special read-only attribute ``__self__`` is set to the\n object denoted by *alist*.\n\n Class Types\n Class types, or "new-style classes," are callable. These\n objects normally act as factories for new instances of\n themselves, but variations are possible for class types that\n override ``__new__()``. The arguments of the call are passed to\n ``__new__()`` and, in the typical case, to ``__init__()`` to\n initialize the new instance.\n\n Classic Classes\n Class objects are described below. When a class object is\n called, a new class instance (also described below) is created\n and returned. This implies a call to the class\'s ``__init__()``\n method if it has one. Any arguments are passed on to the\n ``__init__()`` method. If there is no ``__init__()`` method,\n the class must be called without arguments.\n\n Class instances\n Class instances are described below. Class instances are\n callable only when the class has a ``__call__()`` method;\n ``x(arguments)`` is a shorthand for ``x.__call__(arguments)``.\n\nModules\n Modules are imported by the ``import`` statement (see section *The\n import statement*). A module object has a namespace implemented by\n a dictionary object (this is the dictionary referenced by the\n func_globals attribute of functions defined in the module).\n Attribute references are translated to lookups in this dictionary,\n e.g., ``m.x`` is equivalent to ``m.__dict__["x"]``. A module object\n does not contain the code object used to initialize the module\n (since it isn\'t needed once the initialization is done).\n\n Attribute assignment updates the module\'s namespace dictionary,\n e.g., ``m.x = 1`` is equivalent to ``m.__dict__["x"] = 1``.\n\n Special read-only attribute: ``__dict__`` is the module\'s namespace\n as a dictionary object.\n\n **CPython implementation detail:** Because of the way CPython\n clears module dictionaries, the module dictionary will be cleared\n when the module falls out of scope even if the dictionary still has\n live references. To avoid this, copy the dictionary or keep the\n module around while using its dictionary directly.\n\n Predefined (writable) attributes: ``__name__`` is the module\'s\n name; ``__doc__`` is the module\'s documentation string, or ``None``\n if unavailable; ``__file__`` is the pathname of the file from which\n the module was loaded, if it was loaded from a file. The\n ``__file__`` attribute is not present for C modules that are\n statically linked into the interpreter; for extension modules\n loaded dynamically from a shared library, it is the pathname of the\n shared library file.\n\nClasses\n Both class types (new-style classes) and class objects (old-\n style/classic classes) are typically created by class definitions\n (see section *Class definitions*). A class has a namespace\n implemented by a dictionary object. Class attribute references are\n translated to lookups in this dictionary, e.g., ``C.x`` is\n translated to ``C.__dict__["x"]`` (although for new-style classes\n in particular there are a number of hooks which allow for other\n means of locating attributes). When the attribute name is not found\n there, the attribute search continues in the base classes. For\n old-style classes, the search is depth-first, left-to-right in the\n order of occurrence in the base class list. New-style classes use\n the more complex C3 method resolution order which behaves correctly\n even in the presence of \'diamond\' inheritance structures where\n there are multiple inheritance paths leading back to a common\n ancestor. Additional details on the C3 MRO used by new-style\n classes can be found in the documentation accompanying the 2.3\n release at http://www.python.org/download/releases/2.3/mro/.\n\n When a class attribute reference (for class ``C``, say) would yield\n a user-defined function object or an unbound user-defined method\n object whose associated class is either ``C`` or one of its base\n classes, it is transformed into an unbound user-defined method\n object whose ``im_class`` attribute is ``C``. When it would yield a\n class method object, it is transformed into a bound user-defined\n method object whose ``im_class`` and ``im_self`` attributes are\n both ``C``. When it would yield a static method object, it is\n transformed into the object wrapped by the static method object.\n See section *Implementing Descriptors* for another way in which\n attributes retrieved from a class may differ from those actually\n contained in its ``__dict__`` (note that only new-style classes\n support descriptors).\n\n Class attribute assignments update the class\'s dictionary, never\n the dictionary of a base class.\n\n A class object can be called (see above) to yield a class instance\n (see below).\n\n Special attributes: ``__name__`` is the class name; ``__module__``\n is the module name in which the class was defined; ``__dict__`` is\n the dictionary containing the class\'s namespace; ``__bases__`` is a\n tuple (possibly empty or a singleton) containing the base classes,\n in the order of their occurrence in the base class list;\n ``__doc__`` is the class\'s documentation string, or None if\n undefined.\n\nClass instances\n A class instance is created by calling a class object (see above).\n A class instance has a namespace implemented as a dictionary which\n is the first place in which attribute references are searched.\n When an attribute is not found there, and the instance\'s class has\n an attribute by that name, the search continues with the class\n attributes. If a class attribute is found that is a user-defined\n function object or an unbound user-defined method object whose\n associated class is the class (call it ``C``) of the instance for\n which the attribute reference was initiated or one of its bases, it\n is transformed into a bound user-defined method object whose\n ``im_class`` attribute is ``C`` and whose ``im_self`` attribute is\n the instance. Static method and class method objects are also\n transformed, as if they had been retrieved from class ``C``; see\n above under "Classes". See section *Implementing Descriptors* for\n another way in which attributes of a class retrieved via its\n instances may differ from the objects actually stored in the\n class\'s ``__dict__``. If no class attribute is found, and the\n object\'s class has a ``__getattr__()`` method, that is called to\n satisfy the lookup.\n\n Attribute assignments and deletions update the instance\'s\n dictionary, never a class\'s dictionary. If the class has a\n ``__setattr__()`` or ``__delattr__()`` method, this is called\n instead of updating the instance dictionary directly.\n\n Class instances can pretend to be numbers, sequences, or mappings\n if they have methods with certain special names. See section\n *Special method names*.\n\n Special attributes: ``__dict__`` is the attribute dictionary;\n ``__class__`` is the instance\'s class.\n\nFiles\n A file object represents an open file. File objects are created by\n the ``open()`` built-in function, and also by ``os.popen()``,\n ``os.fdopen()``, and the ``makefile()`` method of socket objects\n (and perhaps by other functions or methods provided by extension\n modules). The objects ``sys.stdin``, ``sys.stdout`` and\n ``sys.stderr`` are initialized to file objects corresponding to the\n interpreter\'s standard input, output and error streams. See *File\n Objects* for complete documentation of file objects.\n\nInternal types\n A few types used internally by the interpreter are exposed to the\n user. Their definitions may change with future versions of the\n interpreter, but they are mentioned here for completeness.\n\n Code objects\n Code objects represent *byte-compiled* executable Python code,\n or *bytecode*. The difference between a code object and a\n function object is that the function object contains an explicit\n reference to the function\'s globals (the module in which it was\n defined), while a code object contains no context; also the\n default argument values are stored in the function object, not\n in the code object (because they represent values calculated at\n run-time). Unlike function objects, code objects are immutable\n and contain no references (directly or indirectly) to mutable\n objects.\n\n Special read-only attributes: ``co_name`` gives the function\n name; ``co_argcount`` is the number of positional arguments\n (including arguments with default values); ``co_nlocals`` is the\n number of local variables used by the function (including\n arguments); ``co_varnames`` is a tuple containing the names of\n the local variables (starting with the argument names);\n ``co_cellvars`` is a tuple containing the names of local\n variables that are referenced by nested functions;\n ``co_freevars`` is a tuple containing the names of free\n variables; ``co_code`` is a string representing the sequence of\n bytecode instructions; ``co_consts`` is a tuple containing the\n literals used by the bytecode; ``co_names`` is a tuple\n containing the names used by the bytecode; ``co_filename`` is\n the filename from which the code was compiled;\n ``co_firstlineno`` is the first line number of the function;\n ``co_lnotab`` is a string encoding the mapping from bytecode\n offsets to line numbers (for details see the source code of the\n interpreter); ``co_stacksize`` is the required stack size\n (including local variables); ``co_flags`` is an integer encoding\n a number of flags for the interpreter.\n\n The following flag bits are defined for ``co_flags``: bit\n ``0x04`` is set if the function uses the ``*arguments`` syntax\n to accept an arbitrary number of positional arguments; bit\n ``0x08`` is set if the function uses the ``**keywords`` syntax\n to accept arbitrary keyword arguments; bit ``0x20`` is set if\n the function is a generator.\n\n Future feature declarations (``from __future__ import\n division``) also use bits in ``co_flags`` to indicate whether a\n code object was compiled with a particular feature enabled: bit\n ``0x2000`` is set if the function was compiled with future\n division enabled; bits ``0x10`` and ``0x1000`` were used in\n earlier versions of Python.\n\n Other bits in ``co_flags`` are reserved for internal use.\n\n If a code object represents a function, the first item in\n ``co_consts`` is the documentation string of the function, or\n ``None`` if undefined.\n\n Frame objects\n Frame objects represent execution frames. They may occur in\n traceback objects (see below).\n\n Special read-only attributes: ``f_back`` is to the previous\n stack frame (towards the caller), or ``None`` if this is the\n bottom stack frame; ``f_code`` is the code object being executed\n in this frame; ``f_locals`` is the dictionary used to look up\n local variables; ``f_globals`` is used for global variables;\n ``f_builtins`` is used for built-in (intrinsic) names;\n ``f_restricted`` is a flag indicating whether the function is\n executing in restricted execution mode; ``f_lasti`` gives the\n precise instruction (this is an index into the bytecode string\n of the code object).\n\n Special writable attributes: ``f_trace``, if not ``None``, is a\n function called at the start of each source code line (this is\n used by the debugger); ``f_exc_type``, ``f_exc_value``,\n ``f_exc_traceback`` represent the last exception raised in the\n parent frame provided another exception was ever raised in the\n current frame (in all other cases they are None); ``f_lineno``\n is the current line number of the frame --- writing to this from\n within a trace function jumps to the given line (only for the\n bottom-most frame). A debugger can implement a Jump command\n (aka Set Next Statement) by writing to f_lineno.\n\n Traceback objects\n Traceback objects represent a stack trace of an exception. A\n traceback object is created when an exception occurs. When the\n search for an exception handler unwinds the execution stack, at\n each unwound level a traceback object is inserted in front of\n the current traceback. When an exception handler is entered,\n the stack trace is made available to the program. (See section\n *The try statement*.) It is accessible as ``sys.exc_traceback``,\n and also as the third item of the tuple returned by\n ``sys.exc_info()``. The latter is the preferred interface,\n since it works correctly when the program is using multiple\n threads. When the program contains no suitable handler, the\n stack trace is written (nicely formatted) to the standard error\n stream; if the interpreter is interactive, it is also made\n available to the user as ``sys.last_traceback``.\n\n Special read-only attributes: ``tb_next`` is the next level in\n the stack trace (towards the frame where the exception\n occurred), or ``None`` if there is no next level; ``tb_frame``\n points to the execution frame of the current level;\n ``tb_lineno`` gives the line number where the exception\n occurred; ``tb_lasti`` indicates the precise instruction. The\n line number and last instruction in the traceback may differ\n from the line number of its frame object if the exception\n occurred in a ``try`` statement with no matching except clause\n or with a finally clause.\n\n Slice objects\n Slice objects are used to represent slices when *extended slice\n syntax* is used. This is a slice using two colons, or multiple\n slices or ellipses separated by commas, e.g., ``a[i:j:step]``,\n ``a[i:j, k:l]``, or ``a[..., i:j]``. They are also created by\n the built-in ``slice()`` function.\n\n Special read-only attributes: ``start`` is the lower bound;\n ``stop`` is the upper bound; ``step`` is the step value; each is\n ``None`` if omitted. These attributes can have any type.\n\n Slice objects support one method:\n\n slice.indices(self, length)\n\n This method takes a single integer argument *length* and\n computes information about the extended slice that the slice\n object would describe if applied to a sequence of *length*\n items. It returns a tuple of three integers; respectively\n these are the *start* and *stop* indices and the *step* or\n stride length of the slice. Missing or out-of-bounds indices\n are handled in a manner consistent with regular slices.\n\n New in version 2.3.\n\n Static method objects\n Static method objects provide a way of defeating the\n transformation of function objects to method objects described\n above. A static method object is a wrapper around any other\n object, usually a user-defined method object. When a static\n method object is retrieved from a class or a class instance, the\n object actually returned is the wrapped object, which is not\n subject to any further transformation. Static method objects are\n not themselves callable, although the objects they wrap usually\n are. Static method objects are created by the built-in\n ``staticmethod()`` constructor.\n\n Class method objects\n A class method object, like a static method object, is a wrapper\n around another object that alters the way in which that object\n is retrieved from classes and class instances. The behaviour of\n class method objects upon such retrieval is described above,\n under "User-defined methods". Class method objects are created\n by the built-in ``classmethod()`` constructor.\n', 'typesfunctions': u'\nFunctions\n*********\n\nFunction objects are created by function definitions. The only\noperation on a function object is to call it: ``func(argument-list)``.\n\nThere are really two flavors of function objects: built-in functions\nand user-defined functions. Both support the same operation (to call\nthe function), but the implementation is different, hence the\ndifferent object types.\n\nSee *Function definitions* for more information.\n', - 'typesmapping': u'\nMapping Types --- ``dict``\n**************************\n\nA *mapping* object maps *hashable* values to arbitrary objects.\nMappings are mutable objects. There is currently only one standard\nmapping type, the *dictionary*. (For other containers see the built\nin ``list``, ``set``, and ``tuple`` classes, and the ``collections``\nmodule.)\n\nA dictionary\'s keys are *almost* arbitrary values. Values that are\nnot *hashable*, that is, values containing lists, dictionaries or\nother mutable types (that are compared by value rather than by object\nidentity) may not be used as keys. Numeric types used for keys obey\nthe normal rules for numeric comparison: if two numbers compare equal\n(such as ``1`` and ``1.0``) then they can be used interchangeably to\nindex the same dictionary entry. (Note however, that since computers\nstore floating-point numbers as approximations it is usually unwise to\nuse them as dictionary keys.)\n\nDictionaries can be created by placing a comma-separated list of\n``key: value`` pairs within braces, for example: ``{\'jack\': 4098,\n\'sjoerd\': 4127}`` or ``{4098: \'jack\', 4127: \'sjoerd\'}``, or by the\n``dict`` constructor.\n\nclass class dict([arg])\n\n Return a new dictionary initialized from an optional positional\n argument or from a set of keyword arguments. If no arguments are\n given, return a new empty dictionary. If the positional argument\n *arg* is a mapping object, return a dictionary mapping the same\n keys to the same values as does the mapping object. Otherwise the\n positional argument must be a sequence, a container that supports\n iteration, or an iterator object. The elements of the argument\n must each also be of one of those kinds, and each must in turn\n contain exactly two objects. The first is used as a key in the new\n dictionary, and the second as the key\'s value. If a given key is\n seen more than once, the last value associated with it is retained\n in the new dictionary.\n\n If keyword arguments are given, the keywords themselves with their\n associated values are added as items to the dictionary. If a key is\n specified both in the positional argument and as a keyword\n argument, the value associated with the keyword is retained in the\n dictionary. For example, these all return a dictionary equal to\n ``{"one": 2, "two": 3}``:\n\n * ``dict(one=2, two=3)``\n\n * ``dict({\'one\': 2, \'two\': 3})``\n\n * ``dict(zip((\'one\', \'two\'), (2, 3)))``\n\n * ``dict([[\'two\', 3], [\'one\', 2]])``\n\n The first example only works for keys that are valid Python\n identifiers; the others work with any valid keys.\n\n New in version 2.2.\n\n Changed in version 2.3: Support for building a dictionary from\n keyword arguments added.\n\n These are the operations that dictionaries support (and therefore,\n custom mapping types should support too):\n\n len(d)\n\n Return the number of items in the dictionary *d*.\n\n d[key]\n\n Return the item of *d* with key *key*. Raises a ``KeyError`` if\n *key* is not in the map.\n\n New in version 2.5: If a subclass of dict defines a method\n ``__missing__()``, if the key *key* is not present, the\n ``d[key]`` operation calls that method with the key *key* as\n argument. The ``d[key]`` operation then returns or raises\n whatever is returned or raised by the ``__missing__(key)`` call\n if the key is not present. No other operations or methods invoke\n ``__missing__()``. If ``__missing__()`` is not defined,\n ``KeyError`` is raised. ``__missing__()`` must be a method; it\n cannot be an instance variable. For an example, see\n ``collections.defaultdict``.\n\n d[key] = value\n\n Set ``d[key]`` to *value*.\n\n del d[key]\n\n Remove ``d[key]`` from *d*. Raises a ``KeyError`` if *key* is\n not in the map.\n\n key in d\n\n Return ``True`` if *d* has a key *key*, else ``False``.\n\n New in version 2.2.\n\n key not in d\n\n Equivalent to ``not key in d``.\n\n New in version 2.2.\n\n iter(d)\n\n Return an iterator over the keys of the dictionary. This is a\n shortcut for ``iterkeys()``.\n\n clear()\n\n Remove all items from the dictionary.\n\n copy()\n\n Return a shallow copy of the dictionary.\n\n fromkeys(seq[, value])\n\n Create a new dictionary with keys from *seq* and values set to\n *value*.\n\n ``fromkeys()`` is a class method that returns a new dictionary.\n *value* defaults to ``None``.\n\n New in version 2.3.\n\n get(key[, default])\n\n Return the value for *key* if *key* is in the dictionary, else\n *default*. If *default* is not given, it defaults to ``None``,\n so that this method never raises a ``KeyError``.\n\n has_key(key)\n\n Test for the presence of *key* in the dictionary. ``has_key()``\n is deprecated in favor of ``key in d``.\n\n items()\n\n Return a copy of the dictionary\'s list of ``(key, value)``\n pairs.\n\n **CPython implementation detail:** Keys and values are listed in\n an arbitrary order which is non-random, varies across Python\n implementations, and depends on the dictionary\'s history of\n insertions and deletions.\n\n If ``items()``, ``keys()``, ``values()``, ``iteritems()``,\n ``iterkeys()``, and ``itervalues()`` are called with no\n intervening modifications to the dictionary, the lists will\n directly correspond. This allows the creation of ``(value,\n key)`` pairs using ``zip()``: ``pairs = zip(d.values(),\n d.keys())``. The same relationship holds for the ``iterkeys()``\n and ``itervalues()`` methods: ``pairs = zip(d.itervalues(),\n d.iterkeys())`` provides the same value for ``pairs``. Another\n way to create the same list is ``pairs = [(v, k) for (k, v) in\n d.iteritems()]``.\n\n iteritems()\n\n Return an iterator over the dictionary\'s ``(key, value)`` pairs.\n See the note for ``dict.items()``.\n\n Using ``iteritems()`` while adding or deleting entries in the\n dictionary may raise a ``RuntimeError`` or fail to iterate over\n all entries.\n\n New in version 2.2.\n\n iterkeys()\n\n Return an iterator over the dictionary\'s keys. See the note for\n ``dict.items()``.\n\n Using ``iterkeys()`` while adding or deleting entries in the\n dictionary may raise a ``RuntimeError`` or fail to iterate over\n all entries.\n\n New in version 2.2.\n\n itervalues()\n\n Return an iterator over the dictionary\'s values. See the note\n for ``dict.items()``.\n\n Using ``itervalues()`` while adding or deleting entries in the\n dictionary may raise a ``RuntimeError`` or fail to iterate over\n all entries.\n\n New in version 2.2.\n\n keys()\n\n Return a copy of the dictionary\'s list of keys. See the note\n for ``dict.items()``.\n\n pop(key[, default])\n\n If *key* is in the dictionary, remove it and return its value,\n else return *default*. If *default* is not given and *key* is\n not in the dictionary, a ``KeyError`` is raised.\n\n New in version 2.3.\n\n popitem()\n\n Remove and return an arbitrary ``(key, value)`` pair from the\n dictionary.\n\n ``popitem()`` is useful to destructively iterate over a\n dictionary, as often used in set algorithms. If the dictionary\n is empty, calling ``popitem()`` raises a ``KeyError``.\n\n setdefault(key[, default])\n\n If *key* is in the dictionary, return its value. If not, insert\n *key* with a value of *default* and return *default*. *default*\n defaults to ``None``.\n\n update([other])\n\n Update the dictionary with the key/value pairs from *other*,\n overwriting existing keys. Return ``None``.\n\n ``update()`` accepts either another dictionary object or an\n iterable of key/value pairs (as a tuple or other iterable of\n length two). If keyword arguments are specified, the dictionary\n is then updated with those key/value pairs: ``d.update(red=1,\n blue=2)``.\n\n Changed in version 2.4: Allowed the argument to be an iterable\n of key/value pairs and allowed keyword arguments.\n\n values()\n\n Return a copy of the dictionary\'s list of values. See the note\n for ``dict.items()``.\n\n viewitems()\n\n Return a new view of the dictionary\'s items (``(key, value)``\n pairs). See below for documentation of view objects.\n\n New in version 2.7.\n\n viewkeys()\n\n Return a new view of the dictionary\'s keys. See below for\n documentation of view objects.\n\n New in version 2.7.\n\n viewvalues()\n\n Return a new view of the dictionary\'s values. See below for\n documentation of view objects.\n\n New in version 2.7.\n\n\nDictionary view objects\n=======================\n\nThe objects returned by ``dict.viewkeys()``, ``dict.viewvalues()`` and\n``dict.viewitems()`` are *view objects*. They provide a dynamic view\non the dictionary\'s entries, which means that when the dictionary\nchanges, the view reflects these changes.\n\nDictionary views can be iterated over to yield their respective data,\nand support membership tests:\n\nlen(dictview)\n\n Return the number of entries in the dictionary.\n\niter(dictview)\n\n Return an iterator over the keys, values or items (represented as\n tuples of ``(key, value)``) in the dictionary.\n\n Keys and values are iterated over in an arbitrary order which is\n non-random, varies across Python implementations, and depends on\n the dictionary\'s history of insertions and deletions. If keys,\n values and items views are iterated over with no intervening\n modifications to the dictionary, the order of items will directly\n correspond. This allows the creation of ``(value, key)`` pairs\n using ``zip()``: ``pairs = zip(d.values(), d.keys())``. Another\n way to create the same list is ``pairs = [(v, k) for (k, v) in\n d.items()]``.\n\n Iterating views while adding or deleting entries in the dictionary\n may raise a ``RuntimeError`` or fail to iterate over all entries.\n\nx in dictview\n\n Return ``True`` if *x* is in the underlying dictionary\'s keys,\n values or items (in the latter case, *x* should be a ``(key,\n value)`` tuple).\n\nKeys views are set-like since their entries are unique and hashable.\nIf all values are hashable, so that (key, value) pairs are unique and\nhashable, then the items view is also set-like. (Values views are not\ntreated as set-like since the entries are generally not unique.) Then\nthese set operations are available ("other" refers either to another\nview or a set):\n\ndictview & other\n\n Return the intersection of the dictview and the other object as a\n new set.\n\ndictview | other\n\n Return the union of the dictview and the other object as a new set.\n\ndictview - other\n\n Return the difference between the dictview and the other object\n (all elements in *dictview* that aren\'t in *other*) as a new set.\n\ndictview ^ other\n\n Return the symmetric difference (all elements either in *dictview*\n or *other*, but not in both) of the dictview and the other object\n as a new set.\n\nAn example of dictionary view usage:\n\n >>> dishes = {\'eggs\': 2, \'sausage\': 1, \'bacon\': 1, \'spam\': 500}\n >>> keys = dishes.viewkeys()\n >>> values = dishes.viewvalues()\n\n >>> # iteration\n >>> n = 0\n >>> for val in values:\n ... n += val\n >>> print(n)\n 504\n\n >>> # keys and values are iterated over in the same order\n >>> list(keys)\n [\'eggs\', \'bacon\', \'sausage\', \'spam\']\n >>> list(values)\n [2, 1, 1, 500]\n\n >>> # view objects are dynamic and reflect dict changes\n >>> del dishes[\'eggs\']\n >>> del dishes[\'sausage\']\n >>> list(keys)\n [\'spam\', \'bacon\']\n\n >>> # set operations\n >>> keys & {\'eggs\', \'bacon\', \'salad\'}\n {\'bacon\'}\n', + 'typesmapping': u'\nMapping Types --- ``dict``\n**************************\n\nA *mapping* object maps *hashable* values to arbitrary objects.\nMappings are mutable objects. There is currently only one standard\nmapping type, the *dictionary*. (For other containers see the built\nin ``list``, ``set``, and ``tuple`` classes, and the ``collections``\nmodule.)\n\nA dictionary\'s keys are *almost* arbitrary values. Values that are\nnot *hashable*, that is, values containing lists, dictionaries or\nother mutable types (that are compared by value rather than by object\nidentity) may not be used as keys. Numeric types used for keys obey\nthe normal rules for numeric comparison: if two numbers compare equal\n(such as ``1`` and ``1.0``) then they can be used interchangeably to\nindex the same dictionary entry. (Note however, that since computers\nstore floating-point numbers as approximations it is usually unwise to\nuse them as dictionary keys.)\n\nDictionaries can be created by placing a comma-separated list of\n``key: value`` pairs within braces, for example: ``{\'jack\': 4098,\n\'sjoerd\': 4127}`` or ``{4098: \'jack\', 4127: \'sjoerd\'}``, or by the\n``dict`` constructor.\n\nclass class dict([arg])\n\n Return a new dictionary initialized from an optional positional\n argument or from a set of keyword arguments. If no arguments are\n given, return a new empty dictionary. If the positional argument\n *arg* is a mapping object, return a dictionary mapping the same\n keys to the same values as does the mapping object. Otherwise the\n positional argument must be a sequence, a container that supports\n iteration, or an iterator object. The elements of the argument\n must each also be of one of those kinds, and each must in turn\n contain exactly two objects. The first is used as a key in the new\n dictionary, and the second as the key\'s value. If a given key is\n seen more than once, the last value associated with it is retained\n in the new dictionary.\n\n If keyword arguments are given, the keywords themselves with their\n associated values are added as items to the dictionary. If a key is\n specified both in the positional argument and as a keyword\n argument, the value associated with the keyword is retained in the\n dictionary. For example, these all return a dictionary equal to\n ``{"one": 1, "two": 2}``:\n\n * ``dict(one=1, two=2)``\n\n * ``dict({\'one\': 1, \'two\': 2})``\n\n * ``dict(zip((\'one\', \'two\'), (1, 2)))``\n\n * ``dict([[\'two\', 2], [\'one\', 1]])``\n\n The first example only works for keys that are valid Python\n identifiers; the others work with any valid keys.\n\n New in version 2.2.\n\n Changed in version 2.3: Support for building a dictionary from\n keyword arguments added.\n\n These are the operations that dictionaries support (and therefore,\n custom mapping types should support too):\n\n len(d)\n\n Return the number of items in the dictionary *d*.\n\n d[key]\n\n Return the item of *d* with key *key*. Raises a ``KeyError`` if\n *key* is not in the map.\n\n New in version 2.5: If a subclass of dict defines a method\n ``__missing__()``, if the key *key* is not present, the\n ``d[key]`` operation calls that method with the key *key* as\n argument. The ``d[key]`` operation then returns or raises\n whatever is returned or raised by the ``__missing__(key)`` call\n if the key is not present. No other operations or methods invoke\n ``__missing__()``. If ``__missing__()`` is not defined,\n ``KeyError`` is raised. ``__missing__()`` must be a method; it\n cannot be an instance variable. For an example, see\n ``collections.defaultdict``.\n\n d[key] = value\n\n Set ``d[key]`` to *value*.\n\n del d[key]\n\n Remove ``d[key]`` from *d*. Raises a ``KeyError`` if *key* is\n not in the map.\n\n key in d\n\n Return ``True`` if *d* has a key *key*, else ``False``.\n\n New in version 2.2.\n\n key not in d\n\n Equivalent to ``not key in d``.\n\n New in version 2.2.\n\n iter(d)\n\n Return an iterator over the keys of the dictionary. This is a\n shortcut for ``iterkeys()``.\n\n clear()\n\n Remove all items from the dictionary.\n\n copy()\n\n Return a shallow copy of the dictionary.\n\n fromkeys(seq[, value])\n\n Create a new dictionary with keys from *seq* and values set to\n *value*.\n\n ``fromkeys()`` is a class method that returns a new dictionary.\n *value* defaults to ``None``.\n\n New in version 2.3.\n\n get(key[, default])\n\n Return the value for *key* if *key* is in the dictionary, else\n *default*. If *default* is not given, it defaults to ``None``,\n so that this method never raises a ``KeyError``.\n\n has_key(key)\n\n Test for the presence of *key* in the dictionary. ``has_key()``\n is deprecated in favor of ``key in d``.\n\n items()\n\n Return a copy of the dictionary\'s list of ``(key, value)``\n pairs.\n\n **CPython implementation detail:** Keys and values are listed in\n an arbitrary order which is non-random, varies across Python\n implementations, and depends on the dictionary\'s history of\n insertions and deletions.\n\n If ``items()``, ``keys()``, ``values()``, ``iteritems()``,\n ``iterkeys()``, and ``itervalues()`` are called with no\n intervening modifications to the dictionary, the lists will\n directly correspond. This allows the creation of ``(value,\n key)`` pairs using ``zip()``: ``pairs = zip(d.values(),\n d.keys())``. The same relationship holds for the ``iterkeys()``\n and ``itervalues()`` methods: ``pairs = zip(d.itervalues(),\n d.iterkeys())`` provides the same value for ``pairs``. Another\n way to create the same list is ``pairs = [(v, k) for (k, v) in\n d.iteritems()]``.\n\n iteritems()\n\n Return an iterator over the dictionary\'s ``(key, value)`` pairs.\n See the note for ``dict.items()``.\n\n Using ``iteritems()`` while adding or deleting entries in the\n dictionary may raise a ``RuntimeError`` or fail to iterate over\n all entries.\n\n New in version 2.2.\n\n iterkeys()\n\n Return an iterator over the dictionary\'s keys. See the note for\n ``dict.items()``.\n\n Using ``iterkeys()`` while adding or deleting entries in the\n dictionary may raise a ``RuntimeError`` or fail to iterate over\n all entries.\n\n New in version 2.2.\n\n itervalues()\n\n Return an iterator over the dictionary\'s values. See the note\n for ``dict.items()``.\n\n Using ``itervalues()`` while adding or deleting entries in the\n dictionary may raise a ``RuntimeError`` or fail to iterate over\n all entries.\n\n New in version 2.2.\n\n keys()\n\n Return a copy of the dictionary\'s list of keys. See the note\n for ``dict.items()``.\n\n pop(key[, default])\n\n If *key* is in the dictionary, remove it and return its value,\n else return *default*. If *default* is not given and *key* is\n not in the dictionary, a ``KeyError`` is raised.\n\n New in version 2.3.\n\n popitem()\n\n Remove and return an arbitrary ``(key, value)`` pair from the\n dictionary.\n\n ``popitem()`` is useful to destructively iterate over a\n dictionary, as often used in set algorithms. If the dictionary\n is empty, calling ``popitem()`` raises a ``KeyError``.\n\n setdefault(key[, default])\n\n If *key* is in the dictionary, return its value. If not, insert\n *key* with a value of *default* and return *default*. *default*\n defaults to ``None``.\n\n update([other])\n\n Update the dictionary with the key/value pairs from *other*,\n overwriting existing keys. Return ``None``.\n\n ``update()`` accepts either another dictionary object or an\n iterable of key/value pairs (as tuples or other iterables of\n length two). If keyword arguments are specified, the dictionary\n is then updated with those key/value pairs: ``d.update(red=1,\n blue=2)``.\n\n Changed in version 2.4: Allowed the argument to be an iterable\n of key/value pairs and allowed keyword arguments.\n\n values()\n\n Return a copy of the dictionary\'s list of values. See the note\n for ``dict.items()``.\n\n viewitems()\n\n Return a new view of the dictionary\'s items (``(key, value)``\n pairs). See below for documentation of view objects.\n\n New in version 2.7.\n\n viewkeys()\n\n Return a new view of the dictionary\'s keys. See below for\n documentation of view objects.\n\n New in version 2.7.\n\n viewvalues()\n\n Return a new view of the dictionary\'s values. See below for\n documentation of view objects.\n\n New in version 2.7.\n\n\nDictionary view objects\n=======================\n\nThe objects returned by ``dict.viewkeys()``, ``dict.viewvalues()`` and\n``dict.viewitems()`` are *view objects*. They provide a dynamic view\non the dictionary\'s entries, which means that when the dictionary\nchanges, the view reflects these changes.\n\nDictionary views can be iterated over to yield their respective data,\nand support membership tests:\n\nlen(dictview)\n\n Return the number of entries in the dictionary.\n\niter(dictview)\n\n Return an iterator over the keys, values or items (represented as\n tuples of ``(key, value)``) in the dictionary.\n\n Keys and values are iterated over in an arbitrary order which is\n non-random, varies across Python implementations, and depends on\n the dictionary\'s history of insertions and deletions. If keys,\n values and items views are iterated over with no intervening\n modifications to the dictionary, the order of items will directly\n correspond. This allows the creation of ``(value, key)`` pairs\n using ``zip()``: ``pairs = zip(d.values(), d.keys())``. Another\n way to create the same list is ``pairs = [(v, k) for (k, v) in\n d.items()]``.\n\n Iterating views while adding or deleting entries in the dictionary\n may raise a ``RuntimeError`` or fail to iterate over all entries.\n\nx in dictview\n\n Return ``True`` if *x* is in the underlying dictionary\'s keys,\n values or items (in the latter case, *x* should be a ``(key,\n value)`` tuple).\n\nKeys views are set-like since their entries are unique and hashable.\nIf all values are hashable, so that (key, value) pairs are unique and\nhashable, then the items view is also set-like. (Values views are not\ntreated as set-like since the entries are generally not unique.) Then\nthese set operations are available ("other" refers either to another\nview or a set):\n\ndictview & other\n\n Return the intersection of the dictview and the other object as a\n new set.\n\ndictview | other\n\n Return the union of the dictview and the other object as a new set.\n\ndictview - other\n\n Return the difference between the dictview and the other object\n (all elements in *dictview* that aren\'t in *other*) as a new set.\n\ndictview ^ other\n\n Return the symmetric difference (all elements either in *dictview*\n or *other*, but not in both) of the dictview and the other object\n as a new set.\n\nAn example of dictionary view usage:\n\n >>> dishes = {\'eggs\': 2, \'sausage\': 1, \'bacon\': 1, \'spam\': 500}\n >>> keys = dishes.viewkeys()\n >>> values = dishes.viewvalues()\n\n >>> # iteration\n >>> n = 0\n >>> for val in values:\n ... n += val\n >>> print(n)\n 504\n\n >>> # keys and values are iterated over in the same order\n >>> list(keys)\n [\'eggs\', \'bacon\', \'sausage\', \'spam\']\n >>> list(values)\n [2, 1, 1, 500]\n\n >>> # view objects are dynamic and reflect dict changes\n >>> del dishes[\'eggs\']\n >>> del dishes[\'sausage\']\n >>> list(keys)\n [\'spam\', \'bacon\']\n\n >>> # set operations\n >>> keys & {\'eggs\', \'bacon\', \'salad\'}\n {\'bacon\'}\n', 'typesmethods': u"\nMethods\n*******\n\nMethods are functions that are called using the attribute notation.\nThere are two flavors: built-in methods (such as ``append()`` on\nlists) and class instance methods. Built-in methods are described\nwith the types that support them.\n\nThe implementation adds two special read-only attributes to class\ninstance methods: ``m.im_self`` is the object on which the method\noperates, and ``m.im_func`` is the function implementing the method.\nCalling ``m(arg-1, arg-2, ..., arg-n)`` is completely equivalent to\ncalling ``m.im_func(m.im_self, arg-1, arg-2, ..., arg-n)``.\n\nClass instance methods are either *bound* or *unbound*, referring to\nwhether the method was accessed through an instance or a class,\nrespectively. When a method is unbound, its ``im_self`` attribute\nwill be ``None`` and if called, an explicit ``self`` object must be\npassed as the first argument. In this case, ``self`` must be an\ninstance of the unbound method's class (or a subclass of that class),\notherwise a ``TypeError`` is raised.\n\nLike function objects, methods objects support getting arbitrary\nattributes. However, since method attributes are actually stored on\nthe underlying function object (``meth.im_func``), setting method\nattributes on either bound or unbound methods is disallowed.\nAttempting to set a method attribute results in a ``TypeError`` being\nraised. In order to set a method attribute, you need to explicitly\nset it on the underlying function object:\n\n class C:\n def method(self):\n pass\n\n c = C()\n c.method.im_func.whoami = 'my name is c'\n\nSee *The standard type hierarchy* for more information.\n", 'typesmodules': u"\nModules\n*******\n\nThe only special operation on a module is attribute access:\n``m.name``, where *m* is a module and *name* accesses a name defined\nin *m*'s symbol table. Module attributes can be assigned to. (Note\nthat the ``import`` statement is not, strictly speaking, an operation\non a module object; ``import foo`` does not require a module object\nnamed *foo* to exist, rather it requires an (external) *definition*\nfor a module named *foo* somewhere.)\n\nA special member of every module is ``__dict__``. This is the\ndictionary containing the module's symbol table. Modifying this\ndictionary will actually change the module's symbol table, but direct\nassignment to the ``__dict__`` attribute is not possible (you can\nwrite ``m.__dict__['a'] = 1``, which defines ``m.a`` to be ``1``, but\nyou can't write ``m.__dict__ = {}``). Modifying ``__dict__`` directly\nis not recommended.\n\nModules built into the interpreter are written like this: ``<module\n'sys' (built-in)>``. If loaded from a file, they are written as\n``<module 'os' from '/usr/local/lib/pythonX.Y/os.pyc'>``.\n", - 'typesseq': u'\nSequence Types --- ``str``, ``unicode``, ``list``, ``tuple``, ``buffer``, ``xrange``\n************************************************************************************\n\nThere are six sequence types: strings, Unicode strings, lists, tuples,\nbuffers, and xrange objects.\n\nFor other containers see the built in ``dict`` and ``set`` classes,\nand the ``collections`` module.\n\nString literals are written in single or double quotes: ``\'xyzzy\'``,\n``"frobozz"``. See *String literals* for more about string literals.\nUnicode strings are much like strings, but are specified in the syntax\nusing a preceding ``\'u\'`` character: ``u\'abc\'``, ``u"def"``. In\naddition to the functionality described here, there are also string-\nspecific methods described in the *String Methods* section. Lists are\nconstructed with square brackets, separating items with commas: ``[a,\nb, c]``. Tuples are constructed by the comma operator (not within\nsquare brackets), with or without enclosing parentheses, but an empty\ntuple must have the enclosing parentheses, such as ``a, b, c`` or\n``()``. A single item tuple must have a trailing comma, such as\n``(d,)``.\n\nBuffer objects are not directly supported by Python syntax, but can be\ncreated by calling the built-in function ``buffer()``. They don\'t\nsupport concatenation or repetition.\n\nObjects of type xrange are similar to buffers in that there is no\nspecific syntax to create them, but they are created using the\n``xrange()`` function. They don\'t support slicing, concatenation or\nrepetition, and using ``in``, ``not in``, ``min()`` or ``max()`` on\nthem is inefficient.\n\nMost sequence types support the following operations. The ``in`` and\n``not in`` operations have the same priorities as the comparison\noperations. The ``+`` and ``*`` operations have the same priority as\nthe corresponding numeric operations. [3] Additional methods are\nprovided for *Mutable Sequence Types*.\n\nThis table lists the sequence operations sorted in ascending priority\n(operations in the same box have the same priority). In the table,\n*s* and *t* are sequences of the same type; *n*, *i* and *j* are\nintegers:\n\n+--------------------+----------------------------------+------------+\n| Operation | Result | Notes |\n+====================+==================================+============+\n| ``x in s`` | ``True`` if an item of *s* is | (1) |\n| | equal to *x*, else ``False`` | |\n+--------------------+----------------------------------+------------+\n| ``x not in s`` | ``False`` if an item of *s* is | (1) |\n| | equal to *x*, else ``True`` | |\n+--------------------+----------------------------------+------------+\n| ``s + t`` | the concatenation of *s* and *t* | (6) |\n+--------------------+----------------------------------+------------+\n| ``s * n, n * s`` | *n* shallow copies of *s* | (2) |\n| | concatenated | |\n+--------------------+----------------------------------+------------+\n| ``s[i]`` | *i*\'th item of *s*, origin 0 | (3) |\n+--------------------+----------------------------------+------------+\n| ``s[i:j]`` | slice of *s* from *i* to *j* | (3)(4) |\n+--------------------+----------------------------------+------------+\n| ``s[i:j:k]`` | slice of *s* from *i* to *j* | (3)(5) |\n| | with step *k* | |\n+--------------------+----------------------------------+------------+\n| ``len(s)`` | length of *s* | |\n+--------------------+----------------------------------+------------+\n| ``min(s)`` | smallest item of *s* | |\n+--------------------+----------------------------------+------------+\n| ``max(s)`` | largest item of *s* | |\n+--------------------+----------------------------------+------------+\n\nSequence types also support comparisons. In particular, tuples and\nlists are compared lexicographically by comparing corresponding\nelements. This means that to compare equal, every element must compare\nequal and the two sequences must be of the same type and have the same\nlength. (For full details see *Comparisons* in the language\nreference.)\n\nNotes:\n\n1. When *s* is a string or Unicode string object the ``in`` and ``not\n in`` operations act like a substring test. In Python versions\n before 2.3, *x* had to be a string of length 1. In Python 2.3 and\n beyond, *x* may be a string of any length.\n\n2. Values of *n* less than ``0`` are treated as ``0`` (which yields an\n empty sequence of the same type as *s*). Note also that the copies\n are shallow; nested structures are not copied. This often haunts\n new Python programmers; consider:\n\n >>> lists = [[]] * 3\n >>> lists\n [[], [], []]\n >>> lists[0].append(3)\n >>> lists\n [[3], [3], [3]]\n\n What has happened is that ``[[]]`` is a one-element list containing\n an empty list, so all three elements of ``[[]] * 3`` are (pointers\n to) this single empty list. Modifying any of the elements of\n ``lists`` modifies this single list. You can create a list of\n different lists this way:\n\n >>> lists = [[] for i in range(3)]\n >>> lists[0].append(3)\n >>> lists[1].append(5)\n >>> lists[2].append(7)\n >>> lists\n [[3], [5], [7]]\n\n3. If *i* or *j* is negative, the index is relative to the end of the\n string: ``len(s) + i`` or ``len(s) + j`` is substituted. But note\n that ``-0`` is still ``0``.\n\n4. The slice of *s* from *i* to *j* is defined as the sequence of\n items with index *k* such that ``i <= k < j``. If *i* or *j* is\n greater than ``len(s)``, use ``len(s)``. If *i* is omitted or\n ``None``, use ``0``. If *j* is omitted or ``None``, use\n ``len(s)``. If *i* is greater than or equal to *j*, the slice is\n empty.\n\n5. The slice of *s* from *i* to *j* with step *k* is defined as the\n sequence of items with index ``x = i + n*k`` such that ``0 <= n <\n (j-i)/k``. In other words, the indices are ``i``, ``i+k``,\n ``i+2*k``, ``i+3*k`` and so on, stopping when *j* is reached (but\n never including *j*). If *i* or *j* is greater than ``len(s)``,\n use ``len(s)``. If *i* or *j* are omitted or ``None``, they become\n "end" values (which end depends on the sign of *k*). Note, *k*\n cannot be zero. If *k* is ``None``, it is treated like ``1``.\n\n6. **CPython implementation detail:** If *s* and *t* are both strings,\n some Python implementations such as CPython can usually perform an\n in-place optimization for assignments of the form ``s = s + t`` or\n ``s += t``. When applicable, this optimization makes quadratic\n run-time much less likely. This optimization is both version and\n implementation dependent. For performance sensitive code, it is\n preferable to use the ``str.join()`` method which assures\n consistent linear concatenation performance across versions and\n implementations.\n\n Changed in version 2.4: Formerly, string concatenation never\n occurred in-place.\n\n\nString Methods\n==============\n\nBelow are listed the string methods which both 8-bit strings and\nUnicode objects support.\n\nIn addition, Python\'s strings support the sequence type methods\ndescribed in the *Sequence Types --- str, unicode, list, tuple,\nbuffer, xrange* section. To output formatted strings use template\nstrings or the ``%`` operator described in the *String Formatting\nOperations* section. Also, see the ``re`` module for string functions\nbased on regular expressions.\n\nstr.capitalize()\n\n Return a copy of the string with only its first character\n capitalized.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.center(width[, fillchar])\n\n Return centered in a string of length *width*. Padding is done\n using the specified *fillchar* (default is a space).\n\n Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.count(sub[, start[, end]])\n\n Return the number of non-overlapping occurrences of substring *sub*\n in the range [*start*, *end*]. Optional arguments *start* and\n *end* are interpreted as in slice notation.\n\nstr.decode([encoding[, errors]])\n\n Decodes the string using the codec registered for *encoding*.\n *encoding* defaults to the default string encoding. *errors* may\n be given to set a different error handling scheme. The default is\n ``\'strict\'``, meaning that encoding errors raise ``UnicodeError``.\n Other possible values are ``\'ignore\'``, ``\'replace\'`` and any other\n name registered via ``codecs.register_error()``, see section *Codec\n Base Classes*.\n\n New in version 2.2.\n\n Changed in version 2.3: Support for other error handling schemes\n added.\n\n Changed in version 2.7: Support for keyword arguments added.\n\nstr.encode([encoding[, errors]])\n\n Return an encoded version of the string. Default encoding is the\n current default string encoding. *errors* may be given to set a\n different error handling scheme. The default for *errors* is\n ``\'strict\'``, meaning that encoding errors raise a\n ``UnicodeError``. Other possible values are ``\'ignore\'``,\n ``\'replace\'``, ``\'xmlcharrefreplace\'``, ``\'backslashreplace\'`` and\n any other name registered via ``codecs.register_error()``, see\n section *Codec Base Classes*. For a list of possible encodings, see\n section *Standard Encodings*.\n\n New in version 2.0.\n\n Changed in version 2.3: Support for ``\'xmlcharrefreplace\'`` and\n ``\'backslashreplace\'`` and other error handling schemes added.\n\n Changed in version 2.7: Support for keyword arguments added.\n\nstr.endswith(suffix[, start[, end]])\n\n Return ``True`` if the string ends with the specified *suffix*,\n otherwise return ``False``. *suffix* can also be a tuple of\n suffixes to look for. With optional *start*, test beginning at\n that position. With optional *end*, stop comparing at that\n position.\n\n Changed in version 2.5: Accept tuples as *suffix*.\n\nstr.expandtabs([tabsize])\n\n Return a copy of the string where all tab characters are replaced\n by one or more spaces, depending on the current column and the\n given tab size. The column number is reset to zero after each\n newline occurring in the string. If *tabsize* is not given, a tab\n size of ``8`` characters is assumed. This doesn\'t understand other\n non-printing characters or escape sequences.\n\nstr.find(sub[, start[, end]])\n\n Return the lowest index in the string where substring *sub* is\n found, such that *sub* is contained in the slice ``s[start:end]``.\n Optional arguments *start* and *end* are interpreted as in slice\n notation. Return ``-1`` if *sub* is not found.\n\nstr.format(*args, **kwargs)\n\n Perform a string formatting operation. The string on which this\n method is called can contain literal text or replacement fields\n delimited by braces ``{}``. Each replacement field contains either\n the numeric index of a positional argument, or the name of a\n keyword argument. Returns a copy of the string where each\n replacement field is replaced with the string value of the\n corresponding argument.\n\n >>> "The sum of 1 + 2 is {0}".format(1+2)\n \'The sum of 1 + 2 is 3\'\n\n See *Format String Syntax* for a description of the various\n formatting options that can be specified in format strings.\n\n This method of string formatting is the new standard in Python 3.0,\n and should be preferred to the ``%`` formatting described in\n *String Formatting Operations* in new code.\n\n New in version 2.6.\n\nstr.index(sub[, start[, end]])\n\n Like ``find()``, but raise ``ValueError`` when the substring is not\n found.\n\nstr.isalnum()\n\n Return true if all characters in the string are alphanumeric and\n there is at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isalpha()\n\n Return true if all characters in the string are alphabetic and\n there is at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isdigit()\n\n Return true if all characters in the string are digits and there is\n at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.islower()\n\n Return true if all cased characters in the string are lowercase and\n there is at least one cased character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isspace()\n\n Return true if there are only whitespace characters in the string\n and there is at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.istitle()\n\n Return true if the string is a titlecased string and there is at\n least one character, for example uppercase characters may only\n follow uncased characters and lowercase characters only cased ones.\n Return false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isupper()\n\n Return true if all cased characters in the string are uppercase and\n there is at least one cased character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.join(iterable)\n\n Return a string which is the concatenation of the strings in the\n *iterable* *iterable*. The separator between elements is the\n string providing this method.\n\nstr.ljust(width[, fillchar])\n\n Return the string left justified in a string of length *width*.\n Padding is done using the specified *fillchar* (default is a\n space). The original string is returned if *width* is less than\n ``len(s)``.\n\n Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.lower()\n\n Return a copy of the string converted to lowercase.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.lstrip([chars])\n\n Return a copy of the string with leading characters removed. The\n *chars* argument is a string specifying the set of characters to be\n removed. If omitted or ``None``, the *chars* argument defaults to\n removing whitespace. The *chars* argument is not a prefix; rather,\n all combinations of its values are stripped:\n\n >>> \' spacious \'.lstrip()\n \'spacious \'\n >>> \'www.example.com\'.lstrip(\'cmowz.\')\n \'example.com\'\n\n Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.partition(sep)\n\n Split the string at the first occurrence of *sep*, and return a\n 3-tuple containing the part before the separator, the separator\n itself, and the part after the separator. If the separator is not\n found, return a 3-tuple containing the string itself, followed by\n two empty strings.\n\n New in version 2.5.\n\nstr.replace(old, new[, count])\n\n Return a copy of the string with all occurrences of substring *old*\n replaced by *new*. If the optional argument *count* is given, only\n the first *count* occurrences are replaced.\n\nstr.rfind(sub[, start[, end]])\n\n Return the highest index in the string where substring *sub* is\n found, such that *sub* is contained within ``s[start:end]``.\n Optional arguments *start* and *end* are interpreted as in slice\n notation. Return ``-1`` on failure.\n\nstr.rindex(sub[, start[, end]])\n\n Like ``rfind()`` but raises ``ValueError`` when the substring *sub*\n is not found.\n\nstr.rjust(width[, fillchar])\n\n Return the string right justified in a string of length *width*.\n Padding is done using the specified *fillchar* (default is a\n space). The original string is returned if *width* is less than\n ``len(s)``.\n\n Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.rpartition(sep)\n\n Split the string at the last occurrence of *sep*, and return a\n 3-tuple containing the part before the separator, the separator\n itself, and the part after the separator. If the separator is not\n found, return a 3-tuple containing two empty strings, followed by\n the string itself.\n\n New in version 2.5.\n\nstr.rsplit([sep[, maxsplit]])\n\n Return a list of the words in the string, using *sep* as the\n delimiter string. If *maxsplit* is given, at most *maxsplit* splits\n are done, the *rightmost* ones. If *sep* is not specified or\n ``None``, any whitespace string is a separator. Except for\n splitting from the right, ``rsplit()`` behaves like ``split()``\n which is described in detail below.\n\n New in version 2.4.\n\nstr.rstrip([chars])\n\n Return a copy of the string with trailing characters removed. The\n *chars* argument is a string specifying the set of characters to be\n removed. If omitted or ``None``, the *chars* argument defaults to\n removing whitespace. The *chars* argument is not a suffix; rather,\n all combinations of its values are stripped:\n\n >>> \' spacious \'.rstrip()\n \' spacious\'\n >>> \'mississippi\'.rstrip(\'ipz\')\n \'mississ\'\n\n Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.split([sep[, maxsplit]])\n\n Return a list of the words in the string, using *sep* as the\n delimiter string. If *maxsplit* is given, at most *maxsplit*\n splits are done (thus, the list will have at most ``maxsplit+1``\n elements). If *maxsplit* is not specified, then there is no limit\n on the number of splits (all possible splits are made).\n\n If *sep* is given, consecutive delimiters are not grouped together\n and are deemed to delimit empty strings (for example,\n ``\'1,,2\'.split(\',\')`` returns ``[\'1\', \'\', \'2\']``). The *sep*\n argument may consist of multiple characters (for example,\n ``\'1<>2<>3\'.split(\'<>\')`` returns ``[\'1\', \'2\', \'3\']``). Splitting\n an empty string with a specified separator returns ``[\'\']``.\n\n If *sep* is not specified or is ``None``, a different splitting\n algorithm is applied: runs of consecutive whitespace are regarded\n as a single separator, and the result will contain no empty strings\n at the start or end if the string has leading or trailing\n whitespace. Consequently, splitting an empty string or a string\n consisting of just whitespace with a ``None`` separator returns\n ``[]``.\n\n For example, ``\' 1 2 3 \'.split()`` returns ``[\'1\', \'2\', \'3\']``,\n and ``\' 1 2 3 \'.split(None, 1)`` returns ``[\'1\', \'2 3 \']``.\n\nstr.splitlines([keepends])\n\n Return a list of the lines in the string, breaking at line\n boundaries. Line breaks are not included in the resulting list\n unless *keepends* is given and true.\n\nstr.startswith(prefix[, start[, end]])\n\n Return ``True`` if string starts with the *prefix*, otherwise\n return ``False``. *prefix* can also be a tuple of prefixes to look\n for. With optional *start*, test string beginning at that\n position. With optional *end*, stop comparing string at that\n position.\n\n Changed in version 2.5: Accept tuples as *prefix*.\n\nstr.strip([chars])\n\n Return a copy of the string with the leading and trailing\n characters removed. The *chars* argument is a string specifying the\n set of characters to be removed. If omitted or ``None``, the\n *chars* argument defaults to removing whitespace. The *chars*\n argument is not a prefix or suffix; rather, all combinations of its\n values are stripped:\n\n >>> \' spacious \'.strip()\n \'spacious\'\n >>> \'www.example.com\'.strip(\'cmowz.\')\n \'example\'\n\n Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.swapcase()\n\n Return a copy of the string with uppercase characters converted to\n lowercase and vice versa.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.title()\n\n Return a titlecased version of the string where words start with an\n uppercase character and the remaining characters are lowercase.\n\n The algorithm uses a simple language-independent definition of a\n word as groups of consecutive letters. The definition works in\n many contexts but it means that apostrophes in contractions and\n possessives form word boundaries, which may not be the desired\n result:\n\n >>> "they\'re bill\'s friends from the UK".title()\n "They\'Re Bill\'S Friends From The Uk"\n\n A workaround for apostrophes can be constructed using regular\n expressions:\n\n >>> import re\n >>> def titlecase(s):\n return re.sub(r"[A-Za-z]+(\'[A-Za-z]+)?",\n lambda mo: mo.group(0)[0].upper() +\n mo.group(0)[1:].lower(),\n s)\n\n >>> titlecase("they\'re bill\'s friends.")\n "They\'re Bill\'s Friends."\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.translate(table[, deletechars])\n\n Return a copy of the string where all characters occurring in the\n optional argument *deletechars* are removed, and the remaining\n characters have been mapped through the given translation table,\n which must be a string of length 256.\n\n You can use the ``maketrans()`` helper function in the ``string``\n module to create a translation table. For string objects, set the\n *table* argument to ``None`` for translations that only delete\n characters:\n\n >>> \'read this short text\'.translate(None, \'aeiou\')\n \'rd ths shrt txt\'\n\n New in version 2.6: Support for a ``None`` *table* argument.\n\n For Unicode objects, the ``translate()`` method does not accept the\n optional *deletechars* argument. Instead, it returns a copy of the\n *s* where all characters have been mapped through the given\n translation table which must be a mapping of Unicode ordinals to\n Unicode ordinals, Unicode strings or ``None``. Unmapped characters\n are left untouched. Characters mapped to ``None`` are deleted.\n Note, a more flexible approach is to create a custom character\n mapping codec using the ``codecs`` module (see ``encodings.cp1251``\n for an example).\n\nstr.upper()\n\n Return a copy of the string converted to uppercase.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.zfill(width)\n\n Return the numeric string left filled with zeros in a string of\n length *width*. A sign prefix is handled correctly. The original\n string is returned if *width* is less than ``len(s)``.\n\n New in version 2.2.2.\n\nThe following methods are present only on unicode objects:\n\nunicode.isnumeric()\n\n Return ``True`` if there are only numeric characters in S,\n ``False`` otherwise. Numeric characters include digit characters,\n and all characters that have the Unicode numeric value property,\n e.g. U+2155, VULGAR FRACTION ONE FIFTH.\n\nunicode.isdecimal()\n\n Return ``True`` if there are only decimal characters in S,\n ``False`` otherwise. Decimal characters include digit characters,\n and all characters that that can be used to form decimal-radix\n numbers, e.g. U+0660, ARABIC-INDIC DIGIT ZERO.\n\n\nString Formatting Operations\n============================\n\nString and Unicode objects have one unique built-in operation: the\n``%`` operator (modulo). This is also known as the string\n*formatting* or *interpolation* operator. Given ``format % values``\n(where *format* is a string or Unicode object), ``%`` conversion\nspecifications in *format* are replaced with zero or more elements of\n*values*. The effect is similar to the using ``sprintf()`` in the C\nlanguage. If *format* is a Unicode object, or if any of the objects\nbeing converted using the ``%s`` conversion are Unicode objects, the\nresult will also be a Unicode object.\n\nIf *format* requires a single argument, *values* may be a single non-\ntuple object. [4] Otherwise, *values* must be a tuple with exactly\nthe number of items specified by the format string, or a single\nmapping object (for example, a dictionary).\n\nA conversion specifier contains two or more characters and has the\nfollowing components, which must occur in this order:\n\n1. The ``\'%\'`` character, which marks the start of the specifier.\n\n2. Mapping key (optional), consisting of a parenthesised sequence of\n characters (for example, ``(somename)``).\n\n3. Conversion flags (optional), which affect the result of some\n conversion types.\n\n4. Minimum field width (optional). If specified as an ``\'*\'``\n (asterisk), the actual width is read from the next element of the\n tuple in *values*, and the object to convert comes after the\n minimum field width and optional precision.\n\n5. Precision (optional), given as a ``\'.\'`` (dot) followed by the\n precision. If specified as ``\'*\'`` (an asterisk), the actual width\n is read from the next element of the tuple in *values*, and the\n value to convert comes after the precision.\n\n6. Length modifier (optional).\n\n7. Conversion type.\n\nWhen the right argument is a dictionary (or other mapping type), then\nthe formats in the string *must* include a parenthesised mapping key\ninto that dictionary inserted immediately after the ``\'%\'`` character.\nThe mapping key selects the value to be formatted from the mapping.\nFor example:\n\n>>> print \'%(language)s has %(#)03d quote types.\' % \\\n... {\'language\': "Python", "#": 2}\nPython has 002 quote types.\n\nIn this case no ``*`` specifiers may occur in a format (since they\nrequire a sequential parameter list).\n\nThe conversion flag characters are:\n\n+-----------+-----------------------------------------------------------------------+\n| Flag | Meaning |\n+===========+=======================================================================+\n| ``\'#\'`` | The value conversion will use the "alternate form" (where defined |\n| | below). |\n+-----------+-----------------------------------------------------------------------+\n| ``\'0\'`` | The conversion will be zero padded for numeric values. |\n+-----------+-----------------------------------------------------------------------+\n| ``\'-\'`` | The converted value is left adjusted (overrides the ``\'0\'`` |\n| | conversion if both are given). |\n+-----------+-----------------------------------------------------------------------+\n| ``\' \'`` | (a space) A blank should be left before a positive number (or empty |\n| | string) produced by a signed conversion. |\n+-----------+-----------------------------------------------------------------------+\n| ``\'+\'`` | A sign character (``\'+\'`` or ``\'-\'``) will precede the conversion |\n| | (overrides a "space" flag). |\n+-----------+-----------------------------------------------------------------------+\n\nA length modifier (``h``, ``l``, or ``L``) may be present, but is\nignored as it is not necessary for Python -- so e.g. ``%ld`` is\nidentical to ``%d``.\n\nThe conversion types are:\n\n+--------------+-------------------------------------------------------+---------+\n| Conversion | Meaning | Notes |\n+==============+=======================================================+=========+\n| ``\'d\'`` | Signed integer decimal. | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'i\'`` | Signed integer decimal. | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'o\'`` | Signed octal value. | (1) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'u\'`` | Obsolete type -- it is identical to ``\'d\'``. | (7) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'x\'`` | Signed hexadecimal (lowercase). | (2) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'X\'`` | Signed hexadecimal (uppercase). | (2) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'e\'`` | Floating point exponential format (lowercase). | (3) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'E\'`` | Floating point exponential format (uppercase). | (3) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'f\'`` | Floating point decimal format. | (3) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'F\'`` | Floating point decimal format. | (3) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'g\'`` | Floating point format. Uses lowercase exponential | (4) |\n| | format if exponent is less than -4 or not less than | |\n| | precision, decimal format otherwise. | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'G\'`` | Floating point format. Uses uppercase exponential | (4) |\n| | format if exponent is less than -4 or not less than | |\n| | precision, decimal format otherwise. | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'c\'`` | Single character (accepts integer or single character | |\n| | string). | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'r\'`` | String (converts any Python object using ``repr()``). | (5) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'s\'`` | String (converts any Python object using ``str()``). | (6) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'%\'`` | No argument is converted, results in a ``\'%\'`` | |\n| | character in the result. | |\n+--------------+-------------------------------------------------------+---------+\n\nNotes:\n\n1. The alternate form causes a leading zero (``\'0\'``) to be inserted\n between left-hand padding and the formatting of the number if the\n leading character of the result is not already a zero.\n\n2. The alternate form causes a leading ``\'0x\'`` or ``\'0X\'`` (depending\n on whether the ``\'x\'`` or ``\'X\'`` format was used) to be inserted\n between left-hand padding and the formatting of the number if the\n leading character of the result is not already a zero.\n\n3. The alternate form causes the result to always contain a decimal\n point, even if no digits follow it.\n\n The precision determines the number of digits after the decimal\n point and defaults to 6.\n\n4. The alternate form causes the result to always contain a decimal\n point, and trailing zeroes are not removed as they would otherwise\n be.\n\n The precision determines the number of significant digits before\n and after the decimal point and defaults to 6.\n\n5. The ``%r`` conversion was added in Python 2.0.\n\n The precision determines the maximal number of characters used.\n\n6. If the object or format provided is a ``unicode`` string, the\n resulting string will also be ``unicode``.\n\n The precision determines the maximal number of characters used.\n\n7. See **PEP 237**.\n\nSince Python strings have an explicit length, ``%s`` conversions do\nnot assume that ``\'\\0\'`` is the end of the string.\n\nChanged in version 2.7: ``%f`` conversions for numbers whose absolute\nvalue is over 1e50 are no longer replaced by ``%g`` conversions.\n\nAdditional string operations are defined in standard modules\n``string`` and ``re``.\n\n\nXRange Type\n===========\n\nThe ``xrange`` type is an immutable sequence which is commonly used\nfor looping. The advantage of the ``xrange`` type is that an\n``xrange`` object will always take the same amount of memory, no\nmatter the size of the range it represents. There are no consistent\nperformance advantages.\n\nXRange objects have very little behavior: they only support indexing,\niteration, and the ``len()`` function.\n\n\nMutable Sequence Types\n======================\n\nList objects support additional operations that allow in-place\nmodification of the object. Other mutable sequence types (when added\nto the language) should also support these operations. Strings and\ntuples are immutable sequence types: such objects cannot be modified\nonce created. The following operations are defined on mutable sequence\ntypes (where *x* is an arbitrary object):\n\n+--------------------------------+----------------------------------+-----------------------+\n| Operation | Result | Notes |\n+================================+==================================+=======================+\n| ``s[i] = x`` | item *i* of *s* is replaced by | |\n| | *x* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s[i:j] = t`` | slice of *s* from *i* to *j* is | |\n| | replaced by the contents of the | |\n| | iterable *t* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``del s[i:j]`` | same as ``s[i:j] = []`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s[i:j:k] = t`` | the elements of ``s[i:j:k]`` are | (1) |\n| | replaced by those of *t* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``del s[i:j:k]`` | removes the elements of | |\n| | ``s[i:j:k]`` from the list | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.append(x)`` | same as ``s[len(s):len(s)] = | (2) |\n| | [x]`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.extend(x)`` | same as ``s[len(s):len(s)] = x`` | (3) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.count(x)`` | return number of *i*\'s for which | |\n| | ``s[i] == x`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.index(x[, i[, j]])`` | return smallest *k* such that | (4) |\n| | ``s[k] == x`` and ``i <= k < j`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.insert(i, x)`` | same as ``s[i:i] = [x]`` | (5) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.pop([i])`` | same as ``x = s[i]; del s[i]; | (6) |\n| | return x`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.remove(x)`` | same as ``del s[s.index(x)]`` | (4) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.reverse()`` | reverses the items of *s* in | (7) |\n| | place | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.sort([cmp[, key[, | sort the items of *s* in place | (7)(8)(9)(10) |\n| reverse]]])`` | | |\n+--------------------------------+----------------------------------+-----------------------+\n\nNotes:\n\n1. *t* must have the same length as the slice it is replacing.\n\n2. The C implementation of Python has historically accepted multiple\n parameters and implicitly joined them into a tuple; this no longer\n works in Python 2.0. Use of this misfeature has been deprecated\n since Python 1.4.\n\n3. *x* can be any iterable object.\n\n4. Raises ``ValueError`` when *x* is not found in *s*. When a negative\n index is passed as the second or third parameter to the ``index()``\n method, the list length is added, as for slice indices. If it is\n still negative, it is truncated to zero, as for slice indices.\n\n Changed in version 2.3: Previously, ``index()`` didn\'t have\n arguments for specifying start and stop positions.\n\n5. When a negative index is passed as the first parameter to the\n ``insert()`` method, the list length is added, as for slice\n indices. If it is still negative, it is truncated to zero, as for\n slice indices.\n\n Changed in version 2.3: Previously, all negative indices were\n truncated to zero.\n\n6. The ``pop()`` method is only supported by the list and array types.\n The optional argument *i* defaults to ``-1``, so that by default\n the last item is removed and returned.\n\n7. The ``sort()`` and ``reverse()`` methods modify the list in place\n for economy of space when sorting or reversing a large list. To\n remind you that they operate by side effect, they don\'t return the\n sorted or reversed list.\n\n8. The ``sort()`` method takes optional arguments for controlling the\n comparisons.\n\n *cmp* specifies a custom comparison function of two arguments (list\n items) which should return a negative, zero or positive number\n depending on whether the first argument is considered smaller than,\n equal to, or larger than the second argument: ``cmp=lambda x,y:\n cmp(x.lower(), y.lower())``. The default value is ``None``.\n\n *key* specifies a function of one argument that is used to extract\n a comparison key from each list element: ``key=str.lower``. The\n default value is ``None``.\n\n *reverse* is a boolean value. If set to ``True``, then the list\n elements are sorted as if each comparison were reversed.\n\n In general, the *key* and *reverse* conversion processes are much\n faster than specifying an equivalent *cmp* function. This is\n because *cmp* is called multiple times for each list element while\n *key* and *reverse* touch each element only once. Use\n ``functools.cmp_to_key()`` to convert an old-style *cmp* function\n to a *key* function.\n\n Changed in version 2.3: Support for ``None`` as an equivalent to\n omitting *cmp* was added.\n\n Changed in version 2.4: Support for *key* and *reverse* was added.\n\n9. Starting with Python 2.3, the ``sort()`` method is guaranteed to be\n stable. A sort is stable if it guarantees not to change the\n relative order of elements that compare equal --- this is helpful\n for sorting in multiple passes (for example, sort by department,\n then by salary grade).\n\n10. **CPython implementation detail:** While a list is being sorted,\n the effect of attempting to mutate, or even inspect, the list is\n undefined. The C implementation of Python 2.3 and newer makes the\n list appear empty for the duration, and raises ``ValueError`` if\n it can detect that the list has been mutated during a sort.\n', - 'typesseq-mutable': u"\nMutable Sequence Types\n**********************\n\nList objects support additional operations that allow in-place\nmodification of the object. Other mutable sequence types (when added\nto the language) should also support these operations. Strings and\ntuples are immutable sequence types: such objects cannot be modified\nonce created. The following operations are defined on mutable sequence\ntypes (where *x* is an arbitrary object):\n\n+--------------------------------+----------------------------------+-----------------------+\n| Operation | Result | Notes |\n+================================+==================================+=======================+\n| ``s[i] = x`` | item *i* of *s* is replaced by | |\n| | *x* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s[i:j] = t`` | slice of *s* from *i* to *j* is | |\n| | replaced by the contents of the | |\n| | iterable *t* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``del s[i:j]`` | same as ``s[i:j] = []`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s[i:j:k] = t`` | the elements of ``s[i:j:k]`` are | (1) |\n| | replaced by those of *t* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``del s[i:j:k]`` | removes the elements of | |\n| | ``s[i:j:k]`` from the list | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.append(x)`` | same as ``s[len(s):len(s)] = | (2) |\n| | [x]`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.extend(x)`` | same as ``s[len(s):len(s)] = x`` | (3) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.count(x)`` | return number of *i*'s for which | |\n| | ``s[i] == x`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.index(x[, i[, j]])`` | return smallest *k* such that | (4) |\n| | ``s[k] == x`` and ``i <= k < j`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.insert(i, x)`` | same as ``s[i:i] = [x]`` | (5) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.pop([i])`` | same as ``x = s[i]; del s[i]; | (6) |\n| | return x`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.remove(x)`` | same as ``del s[s.index(x)]`` | (4) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.reverse()`` | reverses the items of *s* in | (7) |\n| | place | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.sort([cmp[, key[, | sort the items of *s* in place | (7)(8)(9)(10) |\n| reverse]]])`` | | |\n+--------------------------------+----------------------------------+-----------------------+\n\nNotes:\n\n1. *t* must have the same length as the slice it is replacing.\n\n2. The C implementation of Python has historically accepted multiple\n parameters and implicitly joined them into a tuple; this no longer\n works in Python 2.0. Use of this misfeature has been deprecated\n since Python 1.4.\n\n3. *x* can be any iterable object.\n\n4. Raises ``ValueError`` when *x* is not found in *s*. When a negative\n index is passed as the second or third parameter to the ``index()``\n method, the list length is added, as for slice indices. If it is\n still negative, it is truncated to zero, as for slice indices.\n\n Changed in version 2.3: Previously, ``index()`` didn't have\n arguments for specifying start and stop positions.\n\n5. When a negative index is passed as the first parameter to the\n ``insert()`` method, the list length is added, as for slice\n indices. If it is still negative, it is truncated to zero, as for\n slice indices.\n\n Changed in version 2.3: Previously, all negative indices were\n truncated to zero.\n\n6. The ``pop()`` method is only supported by the list and array types.\n The optional argument *i* defaults to ``-1``, so that by default\n the last item is removed and returned.\n\n7. The ``sort()`` and ``reverse()`` methods modify the list in place\n for economy of space when sorting or reversing a large list. To\n remind you that they operate by side effect, they don't return the\n sorted or reversed list.\n\n8. The ``sort()`` method takes optional arguments for controlling the\n comparisons.\n\n *cmp* specifies a custom comparison function of two arguments (list\n items) which should return a negative, zero or positive number\n depending on whether the first argument is considered smaller than,\n equal to, or larger than the second argument: ``cmp=lambda x,y:\n cmp(x.lower(), y.lower())``. The default value is ``None``.\n\n *key* specifies a function of one argument that is used to extract\n a comparison key from each list element: ``key=str.lower``. The\n default value is ``None``.\n\n *reverse* is a boolean value. If set to ``True``, then the list\n elements are sorted as if each comparison were reversed.\n\n In general, the *key* and *reverse* conversion processes are much\n faster than specifying an equivalent *cmp* function. This is\n because *cmp* is called multiple times for each list element while\n *key* and *reverse* touch each element only once. Use\n ``functools.cmp_to_key()`` to convert an old-style *cmp* function\n to a *key* function.\n\n Changed in version 2.3: Support for ``None`` as an equivalent to\n omitting *cmp* was added.\n\n Changed in version 2.4: Support for *key* and *reverse* was added.\n\n9. Starting with Python 2.3, the ``sort()`` method is guaranteed to be\n stable. A sort is stable if it guarantees not to change the\n relative order of elements that compare equal --- this is helpful\n for sorting in multiple passes (for example, sort by department,\n then by salary grade).\n\n10. **CPython implementation detail:** While a list is being sorted,\n the effect of attempting to mutate, or even inspect, the list is\n undefined. The C implementation of Python 2.3 and newer makes the\n list appear empty for the duration, and raises ``ValueError`` if\n it can detect that the list has been mutated during a sort.\n", + 'typesseq': u'\nSequence Types --- ``str``, ``unicode``, ``list``, ``tuple``, ``bytearray``, ``buffer``, ``xrange``\n***************************************************************************************************\n\nThere are seven sequence types: strings, Unicode strings, lists,\ntuples, bytearrays, buffers, and xrange objects.\n\nFor other containers see the built in ``dict`` and ``set`` classes,\nand the ``collections`` module.\n\nString literals are written in single or double quotes: ``\'xyzzy\'``,\n``"frobozz"``. See *String literals* for more about string literals.\nUnicode strings are much like strings, but are specified in the syntax\nusing a preceding ``\'u\'`` character: ``u\'abc\'``, ``u"def"``. In\naddition to the functionality described here, there are also string-\nspecific methods described in the *String Methods* section. Lists are\nconstructed with square brackets, separating items with commas: ``[a,\nb, c]``. Tuples are constructed by the comma operator (not within\nsquare brackets), with or without enclosing parentheses, but an empty\ntuple must have the enclosing parentheses, such as ``a, b, c`` or\n``()``. A single item tuple must have a trailing comma, such as\n``(d,)``.\n\nBytearray objects are created with the built-in function\n``bytearray()``.\n\nBuffer objects are not directly supported by Python syntax, but can be\ncreated by calling the built-in function ``buffer()``. They don\'t\nsupport concatenation or repetition.\n\nObjects of type xrange are similar to buffers in that there is no\nspecific syntax to create them, but they are created using the\n``xrange()`` function. They don\'t support slicing, concatenation or\nrepetition, and using ``in``, ``not in``, ``min()`` or ``max()`` on\nthem is inefficient.\n\nMost sequence types support the following operations. The ``in`` and\n``not in`` operations have the same priorities as the comparison\noperations. The ``+`` and ``*`` operations have the same priority as\nthe corresponding numeric operations. [3] Additional methods are\nprovided for *Mutable Sequence Types*.\n\nThis table lists the sequence operations sorted in ascending priority\n(operations in the same box have the same priority). In the table,\n*s* and *t* are sequences of the same type; *n*, *i* and *j* are\nintegers:\n\n+--------------------+----------------------------------+------------+\n| Operation | Result | Notes |\n+====================+==================================+============+\n| ``x in s`` | ``True`` if an item of *s* is | (1) |\n| | equal to *x*, else ``False`` | |\n+--------------------+----------------------------------+------------+\n| ``x not in s`` | ``False`` if an item of *s* is | (1) |\n| | equal to *x*, else ``True`` | |\n+--------------------+----------------------------------+------------+\n| ``s + t`` | the concatenation of *s* and *t* | (6) |\n+--------------------+----------------------------------+------------+\n| ``s * n, n * s`` | *n* shallow copies of *s* | (2) |\n| | concatenated | |\n+--------------------+----------------------------------+------------+\n| ``s[i]`` | *i*\'th item of *s*, origin 0 | (3) |\n+--------------------+----------------------------------+------------+\n| ``s[i:j]`` | slice of *s* from *i* to *j* | (3)(4) |\n+--------------------+----------------------------------+------------+\n| ``s[i:j:k]`` | slice of *s* from *i* to *j* | (3)(5) |\n| | with step *k* | |\n+--------------------+----------------------------------+------------+\n| ``len(s)`` | length of *s* | |\n+--------------------+----------------------------------+------------+\n| ``min(s)`` | smallest item of *s* | |\n+--------------------+----------------------------------+------------+\n| ``max(s)`` | largest item of *s* | |\n+--------------------+----------------------------------+------------+\n| ``s.index(i)`` | index of the first occurence of | |\n| | *i* in *s* | |\n+--------------------+----------------------------------+------------+\n| ``s.count(i)`` | total number of occurences of | |\n| | *i* in *s* | |\n+--------------------+----------------------------------+------------+\n\nSequence types also support comparisons. In particular, tuples and\nlists are compared lexicographically by comparing corresponding\nelements. This means that to compare equal, every element must compare\nequal and the two sequences must be of the same type and have the same\nlength. (For full details see *Comparisons* in the language\nreference.)\n\nNotes:\n\n1. When *s* is a string or Unicode string object the ``in`` and ``not\n in`` operations act like a substring test. In Python versions\n before 2.3, *x* had to be a string of length 1. In Python 2.3 and\n beyond, *x* may be a string of any length.\n\n2. Values of *n* less than ``0`` are treated as ``0`` (which yields an\n empty sequence of the same type as *s*). Note also that the copies\n are shallow; nested structures are not copied. This often haunts\n new Python programmers; consider:\n\n >>> lists = [[]] * 3\n >>> lists\n [[], [], []]\n >>> lists[0].append(3)\n >>> lists\n [[3], [3], [3]]\n\n What has happened is that ``[[]]`` is a one-element list containing\n an empty list, so all three elements of ``[[]] * 3`` are (pointers\n to) this single empty list. Modifying any of the elements of\n ``lists`` modifies this single list. You can create a list of\n different lists this way:\n\n >>> lists = [[] for i in range(3)]\n >>> lists[0].append(3)\n >>> lists[1].append(5)\n >>> lists[2].append(7)\n >>> lists\n [[3], [5], [7]]\n\n3. If *i* or *j* is negative, the index is relative to the end of the\n string: ``len(s) + i`` or ``len(s) + j`` is substituted. But note\n that ``-0`` is still ``0``.\n\n4. The slice of *s* from *i* to *j* is defined as the sequence of\n items with index *k* such that ``i <= k < j``. If *i* or *j* is\n greater than ``len(s)``, use ``len(s)``. If *i* is omitted or\n ``None``, use ``0``. If *j* is omitted or ``None``, use\n ``len(s)``. If *i* is greater than or equal to *j*, the slice is\n empty.\n\n5. The slice of *s* from *i* to *j* with step *k* is defined as the\n sequence of items with index ``x = i + n*k`` such that ``0 <= n <\n (j-i)/k``. In other words, the indices are ``i``, ``i+k``,\n ``i+2*k``, ``i+3*k`` and so on, stopping when *j* is reached (but\n never including *j*). If *i* or *j* is greater than ``len(s)``,\n use ``len(s)``. If *i* or *j* are omitted or ``None``, they become\n "end" values (which end depends on the sign of *k*). Note, *k*\n cannot be zero. If *k* is ``None``, it is treated like ``1``.\n\n6. **CPython implementation detail:** If *s* and *t* are both strings,\n some Python implementations such as CPython can usually perform an\n in-place optimization for assignments of the form ``s = s + t`` or\n ``s += t``. When applicable, this optimization makes quadratic\n run-time much less likely. This optimization is both version and\n implementation dependent. For performance sensitive code, it is\n preferable to use the ``str.join()`` method which assures\n consistent linear concatenation performance across versions and\n implementations.\n\n Changed in version 2.4: Formerly, string concatenation never\n occurred in-place.\n\n\nString Methods\n==============\n\nBelow are listed the string methods which both 8-bit strings and\nUnicode objects support. Some of them are also available on\n``bytearray`` objects.\n\nIn addition, Python\'s strings support the sequence type methods\ndescribed in the *Sequence Types --- str, unicode, list, tuple,\nbytearray, buffer, xrange* section. To output formatted strings use\ntemplate strings or the ``%`` operator described in the *String\nFormatting Operations* section. Also, see the ``re`` module for string\nfunctions based on regular expressions.\n\nstr.capitalize()\n\n Return a copy of the string with its first character capitalized\n and the rest lowercased.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.center(width[, fillchar])\n\n Return centered in a string of length *width*. Padding is done\n using the specified *fillchar* (default is a space).\n\n Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.count(sub[, start[, end]])\n\n Return the number of non-overlapping occurrences of substring *sub*\n in the range [*start*, *end*]. Optional arguments *start* and\n *end* are interpreted as in slice notation.\n\nstr.decode([encoding[, errors]])\n\n Decodes the string using the codec registered for *encoding*.\n *encoding* defaults to the default string encoding. *errors* may\n be given to set a different error handling scheme. The default is\n ``\'strict\'``, meaning that encoding errors raise ``UnicodeError``.\n Other possible values are ``\'ignore\'``, ``\'replace\'`` and any other\n name registered via ``codecs.register_error()``, see section *Codec\n Base Classes*.\n\n New in version 2.2.\n\n Changed in version 2.3: Support for other error handling schemes\n added.\n\n Changed in version 2.7: Support for keyword arguments added.\n\nstr.encode([encoding[, errors]])\n\n Return an encoded version of the string. Default encoding is the\n current default string encoding. *errors* may be given to set a\n different error handling scheme. The default for *errors* is\n ``\'strict\'``, meaning that encoding errors raise a\n ``UnicodeError``. Other possible values are ``\'ignore\'``,\n ``\'replace\'``, ``\'xmlcharrefreplace\'``, ``\'backslashreplace\'`` and\n any other name registered via ``codecs.register_error()``, see\n section *Codec Base Classes*. For a list of possible encodings, see\n section *Standard Encodings*.\n\n New in version 2.0.\n\n Changed in version 2.3: Support for ``\'xmlcharrefreplace\'`` and\n ``\'backslashreplace\'`` and other error handling schemes added.\n\n Changed in version 2.7: Support for keyword arguments added.\n\nstr.endswith(suffix[, start[, end]])\n\n Return ``True`` if the string ends with the specified *suffix*,\n otherwise return ``False``. *suffix* can also be a tuple of\n suffixes to look for. With optional *start*, test beginning at\n that position. With optional *end*, stop comparing at that\n position.\n\n Changed in version 2.5: Accept tuples as *suffix*.\n\nstr.expandtabs([tabsize])\n\n Return a copy of the string where all tab characters are replaced\n by one or more spaces, depending on the current column and the\n given tab size. The column number is reset to zero after each\n newline occurring in the string. If *tabsize* is not given, a tab\n size of ``8`` characters is assumed. This doesn\'t understand other\n non-printing characters or escape sequences.\n\nstr.find(sub[, start[, end]])\n\n Return the lowest index in the string where substring *sub* is\n found, such that *sub* is contained in the slice ``s[start:end]``.\n Optional arguments *start* and *end* are interpreted as in slice\n notation. Return ``-1`` if *sub* is not found.\n\n Note: The ``find()`` method should be used only if you need to know the\n position of *sub*. To check if *sub* is a substring or not, use\n the ``in`` operator:\n\n >>> \'Py\' in \'Python\'\n True\n\nstr.format(*args, **kwargs)\n\n Perform a string formatting operation. The string on which this\n method is called can contain literal text or replacement fields\n delimited by braces ``{}``. Each replacement field contains either\n the numeric index of a positional argument, or the name of a\n keyword argument. Returns a copy of the string where each\n replacement field is replaced with the string value of the\n corresponding argument.\n\n >>> "The sum of 1 + 2 is {0}".format(1+2)\n \'The sum of 1 + 2 is 3\'\n\n See *Format String Syntax* for a description of the various\n formatting options that can be specified in format strings.\n\n This method of string formatting is the new standard in Python 3.0,\n and should be preferred to the ``%`` formatting described in\n *String Formatting Operations* in new code.\n\n New in version 2.6.\n\nstr.index(sub[, start[, end]])\n\n Like ``find()``, but raise ``ValueError`` when the substring is not\n found.\n\nstr.isalnum()\n\n Return true if all characters in the string are alphanumeric and\n there is at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isalpha()\n\n Return true if all characters in the string are alphabetic and\n there is at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isdigit()\n\n Return true if all characters in the string are digits and there is\n at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.islower()\n\n Return true if all cased characters in the string are lowercase and\n there is at least one cased character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isspace()\n\n Return true if there are only whitespace characters in the string\n and there is at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.istitle()\n\n Return true if the string is a titlecased string and there is at\n least one character, for example uppercase characters may only\n follow uncased characters and lowercase characters only cased ones.\n Return false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isupper()\n\n Return true if all cased characters in the string are uppercase and\n there is at least one cased character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.join(iterable)\n\n Return a string which is the concatenation of the strings in the\n *iterable* *iterable*. The separator between elements is the\n string providing this method.\n\nstr.ljust(width[, fillchar])\n\n Return the string left justified in a string of length *width*.\n Padding is done using the specified *fillchar* (default is a\n space). The original string is returned if *width* is less than\n ``len(s)``.\n\n Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.lower()\n\n Return a copy of the string converted to lowercase.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.lstrip([chars])\n\n Return a copy of the string with leading characters removed. The\n *chars* argument is a string specifying the set of characters to be\n removed. If omitted or ``None``, the *chars* argument defaults to\n removing whitespace. The *chars* argument is not a prefix; rather,\n all combinations of its values are stripped:\n\n >>> \' spacious \'.lstrip()\n \'spacious \'\n >>> \'www.example.com\'.lstrip(\'cmowz.\')\n \'example.com\'\n\n Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.partition(sep)\n\n Split the string at the first occurrence of *sep*, and return a\n 3-tuple containing the part before the separator, the separator\n itself, and the part after the separator. If the separator is not\n found, return a 3-tuple containing the string itself, followed by\n two empty strings.\n\n New in version 2.5.\n\nstr.replace(old, new[, count])\n\n Return a copy of the string with all occurrences of substring *old*\n replaced by *new*. If the optional argument *count* is given, only\n the first *count* occurrences are replaced.\n\nstr.rfind(sub[, start[, end]])\n\n Return the highest index in the string where substring *sub* is\n found, such that *sub* is contained within ``s[start:end]``.\n Optional arguments *start* and *end* are interpreted as in slice\n notation. Return ``-1`` on failure.\n\nstr.rindex(sub[, start[, end]])\n\n Like ``rfind()`` but raises ``ValueError`` when the substring *sub*\n is not found.\n\nstr.rjust(width[, fillchar])\n\n Return the string right justified in a string of length *width*.\n Padding is done using the specified *fillchar* (default is a\n space). The original string is returned if *width* is less than\n ``len(s)``.\n\n Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.rpartition(sep)\n\n Split the string at the last occurrence of *sep*, and return a\n 3-tuple containing the part before the separator, the separator\n itself, and the part after the separator. If the separator is not\n found, return a 3-tuple containing two empty strings, followed by\n the string itself.\n\n New in version 2.5.\n\nstr.rsplit([sep[, maxsplit]])\n\n Return a list of the words in the string, using *sep* as the\n delimiter string. If *maxsplit* is given, at most *maxsplit* splits\n are done, the *rightmost* ones. If *sep* is not specified or\n ``None``, any whitespace string is a separator. Except for\n splitting from the right, ``rsplit()`` behaves like ``split()``\n which is described in detail below.\n\n New in version 2.4.\n\nstr.rstrip([chars])\n\n Return a copy of the string with trailing characters removed. The\n *chars* argument is a string specifying the set of characters to be\n removed. If omitted or ``None``, the *chars* argument defaults to\n removing whitespace. The *chars* argument is not a suffix; rather,\n all combinations of its values are stripped:\n\n >>> \' spacious \'.rstrip()\n \' spacious\'\n >>> \'mississippi\'.rstrip(\'ipz\')\n \'mississ\'\n\n Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.split([sep[, maxsplit]])\n\n Return a list of the words in the string, using *sep* as the\n delimiter string. If *maxsplit* is given, at most *maxsplit*\n splits are done (thus, the list will have at most ``maxsplit+1``\n elements). If *maxsplit* is not specified, then there is no limit\n on the number of splits (all possible splits are made).\n\n If *sep* is given, consecutive delimiters are not grouped together\n and are deemed to delimit empty strings (for example,\n ``\'1,,2\'.split(\',\')`` returns ``[\'1\', \'\', \'2\']``). The *sep*\n argument may consist of multiple characters (for example,\n ``\'1<>2<>3\'.split(\'<>\')`` returns ``[\'1\', \'2\', \'3\']``). Splitting\n an empty string with a specified separator returns ``[\'\']``.\n\n If *sep* is not specified or is ``None``, a different splitting\n algorithm is applied: runs of consecutive whitespace are regarded\n as a single separator, and the result will contain no empty strings\n at the start or end if the string has leading or trailing\n whitespace. Consequently, splitting an empty string or a string\n consisting of just whitespace with a ``None`` separator returns\n ``[]``.\n\n For example, ``\' 1 2 3 \'.split()`` returns ``[\'1\', \'2\', \'3\']``,\n and ``\' 1 2 3 \'.split(None, 1)`` returns ``[\'1\', \'2 3 \']``.\n\nstr.splitlines([keepends])\n\n Return a list of the lines in the string, breaking at line\n boundaries. Line breaks are not included in the resulting list\n unless *keepends* is given and true.\n\nstr.startswith(prefix[, start[, end]])\n\n Return ``True`` if string starts with the *prefix*, otherwise\n return ``False``. *prefix* can also be a tuple of prefixes to look\n for. With optional *start*, test string beginning at that\n position. With optional *end*, stop comparing string at that\n position.\n\n Changed in version 2.5: Accept tuples as *prefix*.\n\nstr.strip([chars])\n\n Return a copy of the string with the leading and trailing\n characters removed. The *chars* argument is a string specifying the\n set of characters to be removed. If omitted or ``None``, the\n *chars* argument defaults to removing whitespace. The *chars*\n argument is not a prefix or suffix; rather, all combinations of its\n values are stripped:\n\n >>> \' spacious \'.strip()\n \'spacious\'\n >>> \'www.example.com\'.strip(\'cmowz.\')\n \'example\'\n\n Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.swapcase()\n\n Return a copy of the string with uppercase characters converted to\n lowercase and vice versa.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.title()\n\n Return a titlecased version of the string where words start with an\n uppercase character and the remaining characters are lowercase.\n\n The algorithm uses a simple language-independent definition of a\n word as groups of consecutive letters. The definition works in\n many contexts but it means that apostrophes in contractions and\n possessives form word boundaries, which may not be the desired\n result:\n\n >>> "they\'re bill\'s friends from the UK".title()\n "They\'Re Bill\'S Friends From The Uk"\n\n A workaround for apostrophes can be constructed using regular\n expressions:\n\n >>> import re\n >>> def titlecase(s):\n return re.sub(r"[A-Za-z]+(\'[A-Za-z]+)?",\n lambda mo: mo.group(0)[0].upper() +\n mo.group(0)[1:].lower(),\n s)\n\n >>> titlecase("they\'re bill\'s friends.")\n "They\'re Bill\'s Friends."\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.translate(table[, deletechars])\n\n Return a copy of the string where all characters occurring in the\n optional argument *deletechars* are removed, and the remaining\n characters have been mapped through the given translation table,\n which must be a string of length 256.\n\n You can use the ``maketrans()`` helper function in the ``string``\n module to create a translation table. For string objects, set the\n *table* argument to ``None`` for translations that only delete\n characters:\n\n >>> \'read this short text\'.translate(None, \'aeiou\')\n \'rd ths shrt txt\'\n\n New in version 2.6: Support for a ``None`` *table* argument.\n\n For Unicode objects, the ``translate()`` method does not accept the\n optional *deletechars* argument. Instead, it returns a copy of the\n *s* where all characters have been mapped through the given\n translation table which must be a mapping of Unicode ordinals to\n Unicode ordinals, Unicode strings or ``None``. Unmapped characters\n are left untouched. Characters mapped to ``None`` are deleted.\n Note, a more flexible approach is to create a custom character\n mapping codec using the ``codecs`` module (see ``encodings.cp1251``\n for an example).\n\nstr.upper()\n\n Return a copy of the string converted to uppercase.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.zfill(width)\n\n Return the numeric string left filled with zeros in a string of\n length *width*. A sign prefix is handled correctly. The original\n string is returned if *width* is less than ``len(s)``.\n\n New in version 2.2.2.\n\nThe following methods are present only on unicode objects:\n\nunicode.isnumeric()\n\n Return ``True`` if there are only numeric characters in S,\n ``False`` otherwise. Numeric characters include digit characters,\n and all characters that have the Unicode numeric value property,\n e.g. U+2155, VULGAR FRACTION ONE FIFTH.\n\nunicode.isdecimal()\n\n Return ``True`` if there are only decimal characters in S,\n ``False`` otherwise. Decimal characters include digit characters,\n and all characters that that can be used to form decimal-radix\n numbers, e.g. U+0660, ARABIC-INDIC DIGIT ZERO.\n\n\nString Formatting Operations\n============================\n\nString and Unicode objects have one unique built-in operation: the\n``%`` operator (modulo). This is also known as the string\n*formatting* or *interpolation* operator. Given ``format % values``\n(where *format* is a string or Unicode object), ``%`` conversion\nspecifications in *format* are replaced with zero or more elements of\n*values*. The effect is similar to the using ``sprintf()`` in the C\nlanguage. If *format* is a Unicode object, or if any of the objects\nbeing converted using the ``%s`` conversion are Unicode objects, the\nresult will also be a Unicode object.\n\nIf *format* requires a single argument, *values* may be a single non-\ntuple object. [4] Otherwise, *values* must be a tuple with exactly\nthe number of items specified by the format string, or a single\nmapping object (for example, a dictionary).\n\nA conversion specifier contains two or more characters and has the\nfollowing components, which must occur in this order:\n\n1. The ``\'%\'`` character, which marks the start of the specifier.\n\n2. Mapping key (optional), consisting of a parenthesised sequence of\n characters (for example, ``(somename)``).\n\n3. Conversion flags (optional), which affect the result of some\n conversion types.\n\n4. Minimum field width (optional). If specified as an ``\'*\'``\n (asterisk), the actual width is read from the next element of the\n tuple in *values*, and the object to convert comes after the\n minimum field width and optional precision.\n\n5. Precision (optional), given as a ``\'.\'`` (dot) followed by the\n precision. If specified as ``\'*\'`` (an asterisk), the actual width\n is read from the next element of the tuple in *values*, and the\n value to convert comes after the precision.\n\n6. Length modifier (optional).\n\n7. Conversion type.\n\nWhen the right argument is a dictionary (or other mapping type), then\nthe formats in the string *must* include a parenthesised mapping key\ninto that dictionary inserted immediately after the ``\'%\'`` character.\nThe mapping key selects the value to be formatted from the mapping.\nFor example:\n\n>>> print \'%(language)s has %(number)03d quote types.\' % \\\n... {"language": "Python", "number": 2}\nPython has 002 quote types.\n\nIn this case no ``*`` specifiers may occur in a format (since they\nrequire a sequential parameter list).\n\nThe conversion flag characters are:\n\n+-----------+-----------------------------------------------------------------------+\n| Flag | Meaning |\n+===========+=======================================================================+\n| ``\'#\'`` | The value conversion will use the "alternate form" (where defined |\n| | below). |\n+-----------+-----------------------------------------------------------------------+\n| ``\'0\'`` | The conversion will be zero padded for numeric values. |\n+-----------+-----------------------------------------------------------------------+\n| ``\'-\'`` | The converted value is left adjusted (overrides the ``\'0\'`` |\n| | conversion if both are given). |\n+-----------+-----------------------------------------------------------------------+\n| ``\' \'`` | (a space) A blank should be left before a positive number (or empty |\n| | string) produced by a signed conversion. |\n+-----------+-----------------------------------------------------------------------+\n| ``\'+\'`` | A sign character (``\'+\'`` or ``\'-\'``) will precede the conversion |\n| | (overrides a "space" flag). |\n+-----------+-----------------------------------------------------------------------+\n\nA length modifier (``h``, ``l``, or ``L``) may be present, but is\nignored as it is not necessary for Python -- so e.g. ``%ld`` is\nidentical to ``%d``.\n\nThe conversion types are:\n\n+--------------+-------------------------------------------------------+---------+\n| Conversion | Meaning | Notes |\n+==============+=======================================================+=========+\n| ``\'d\'`` | Signed integer decimal. | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'i\'`` | Signed integer decimal. | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'o\'`` | Signed octal value. | (1) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'u\'`` | Obsolete type -- it is identical to ``\'d\'``. | (7) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'x\'`` | Signed hexadecimal (lowercase). | (2) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'X\'`` | Signed hexadecimal (uppercase). | (2) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'e\'`` | Floating point exponential format (lowercase). | (3) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'E\'`` | Floating point exponential format (uppercase). | (3) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'f\'`` | Floating point decimal format. | (3) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'F\'`` | Floating point decimal format. | (3) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'g\'`` | Floating point format. Uses lowercase exponential | (4) |\n| | format if exponent is less than -4 or not less than | |\n| | precision, decimal format otherwise. | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'G\'`` | Floating point format. Uses uppercase exponential | (4) |\n| | format if exponent is less than -4 or not less than | |\n| | precision, decimal format otherwise. | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'c\'`` | Single character (accepts integer or single character | |\n| | string). | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'r\'`` | String (converts any Python object using ``repr()``). | (5) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'s\'`` | String (converts any Python object using ``str()``). | (6) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'%\'`` | No argument is converted, results in a ``\'%\'`` | |\n| | character in the result. | |\n+--------------+-------------------------------------------------------+---------+\n\nNotes:\n\n1. The alternate form causes a leading zero (``\'0\'``) to be inserted\n between left-hand padding and the formatting of the number if the\n leading character of the result is not already a zero.\n\n2. The alternate form causes a leading ``\'0x\'`` or ``\'0X\'`` (depending\n on whether the ``\'x\'`` or ``\'X\'`` format was used) to be inserted\n between left-hand padding and the formatting of the number if the\n leading character of the result is not already a zero.\n\n3. The alternate form causes the result to always contain a decimal\n point, even if no digits follow it.\n\n The precision determines the number of digits after the decimal\n point and defaults to 6.\n\n4. The alternate form causes the result to always contain a decimal\n point, and trailing zeroes are not removed as they would otherwise\n be.\n\n The precision determines the number of significant digits before\n and after the decimal point and defaults to 6.\n\n5. The ``%r`` conversion was added in Python 2.0.\n\n The precision determines the maximal number of characters used.\n\n6. If the object or format provided is a ``unicode`` string, the\n resulting string will also be ``unicode``.\n\n The precision determines the maximal number of characters used.\n\n7. See **PEP 237**.\n\nSince Python strings have an explicit length, ``%s`` conversions do\nnot assume that ``\'\\0\'`` is the end of the string.\n\nChanged in version 2.7: ``%f`` conversions for numbers whose absolute\nvalue is over 1e50 are no longer replaced by ``%g`` conversions.\n\nAdditional string operations are defined in standard modules\n``string`` and ``re``.\n\n\nXRange Type\n===========\n\nThe ``xrange`` type is an immutable sequence which is commonly used\nfor looping. The advantage of the ``xrange`` type is that an\n``xrange`` object will always take the same amount of memory, no\nmatter the size of the range it represents. There are no consistent\nperformance advantages.\n\nXRange objects have very little behavior: they only support indexing,\niteration, and the ``len()`` function.\n\n\nMutable Sequence Types\n======================\n\nList and ``bytearray`` objects support additional operations that\nallow in-place modification of the object. Other mutable sequence\ntypes (when added to the language) should also support these\noperations. Strings and tuples are immutable sequence types: such\nobjects cannot be modified once created. The following operations are\ndefined on mutable sequence types (where *x* is an arbitrary object):\n\n+--------------------------------+----------------------------------+-----------------------+\n| Operation | Result | Notes |\n+================================+==================================+=======================+\n| ``s[i] = x`` | item *i* of *s* is replaced by | |\n| | *x* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s[i:j] = t`` | slice of *s* from *i* to *j* is | |\n| | replaced by the contents of the | |\n| | iterable *t* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``del s[i:j]`` | same as ``s[i:j] = []`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s[i:j:k] = t`` | the elements of ``s[i:j:k]`` are | (1) |\n| | replaced by those of *t* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``del s[i:j:k]`` | removes the elements of | |\n| | ``s[i:j:k]`` from the list | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.append(x)`` | same as ``s[len(s):len(s)] = | (2) |\n| | [x]`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.extend(x)`` | same as ``s[len(s):len(s)] = x`` | (3) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.count(x)`` | return number of *i*\'s for which | |\n| | ``s[i] == x`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.index(x[, i[, j]])`` | return smallest *k* such that | (4) |\n| | ``s[k] == x`` and ``i <= k < j`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.insert(i, x)`` | same as ``s[i:i] = [x]`` | (5) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.pop([i])`` | same as ``x = s[i]; del s[i]; | (6) |\n| | return x`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.remove(x)`` | same as ``del s[s.index(x)]`` | (4) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.reverse()`` | reverses the items of *s* in | (7) |\n| | place | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.sort([cmp[, key[, | sort the items of *s* in place | (7)(8)(9)(10) |\n| reverse]]])`` | | |\n+--------------------------------+----------------------------------+-----------------------+\n\nNotes:\n\n1. *t* must have the same length as the slice it is replacing.\n\n2. The C implementation of Python has historically accepted multiple\n parameters and implicitly joined them into a tuple; this no longer\n works in Python 2.0. Use of this misfeature has been deprecated\n since Python 1.4.\n\n3. *x* can be any iterable object.\n\n4. Raises ``ValueError`` when *x* is not found in *s*. When a negative\n index is passed as the second or third parameter to the ``index()``\n method, the list length is added, as for slice indices. If it is\n still negative, it is truncated to zero, as for slice indices.\n\n Changed in version 2.3: Previously, ``index()`` didn\'t have\n arguments for specifying start and stop positions.\n\n5. When a negative index is passed as the first parameter to the\n ``insert()`` method, the list length is added, as for slice\n indices. If it is still negative, it is truncated to zero, as for\n slice indices.\n\n Changed in version 2.3: Previously, all negative indices were\n truncated to zero.\n\n6. The ``pop()`` method is only supported by the list and array types.\n The optional argument *i* defaults to ``-1``, so that by default\n the last item is removed and returned.\n\n7. The ``sort()`` and ``reverse()`` methods modify the list in place\n for economy of space when sorting or reversing a large list. To\n remind you that they operate by side effect, they don\'t return the\n sorted or reversed list.\n\n8. The ``sort()`` method takes optional arguments for controlling the\n comparisons.\n\n *cmp* specifies a custom comparison function of two arguments (list\n items) which should return a negative, zero or positive number\n depending on whether the first argument is considered smaller than,\n equal to, or larger than the second argument: ``cmp=lambda x,y:\n cmp(x.lower(), y.lower())``. The default value is ``None``.\n\n *key* specifies a function of one argument that is used to extract\n a comparison key from each list element: ``key=str.lower``. The\n default value is ``None``.\n\n *reverse* is a boolean value. If set to ``True``, then the list\n elements are sorted as if each comparison were reversed.\n\n In general, the *key* and *reverse* conversion processes are much\n faster than specifying an equivalent *cmp* function. This is\n because *cmp* is called multiple times for each list element while\n *key* and *reverse* touch each element only once. Use\n ``functools.cmp_to_key()`` to convert an old-style *cmp* function\n to a *key* function.\n\n Changed in version 2.3: Support for ``None`` as an equivalent to\n omitting *cmp* was added.\n\n Changed in version 2.4: Support for *key* and *reverse* was added.\n\n9. Starting with Python 2.3, the ``sort()`` method is guaranteed to be\n stable. A sort is stable if it guarantees not to change the\n relative order of elements that compare equal --- this is helpful\n for sorting in multiple passes (for example, sort by department,\n then by salary grade).\n\n10. **CPython implementation detail:** While a list is being sorted,\n the effect of attempting to mutate, or even inspect, the list is\n undefined. The C implementation of Python 2.3 and newer makes the\n list appear empty for the duration, and raises ``ValueError`` if\n it can detect that the list has been mutated during a sort.\n', + 'typesseq-mutable': u"\nMutable Sequence Types\n**********************\n\nList and ``bytearray`` objects support additional operations that\nallow in-place modification of the object. Other mutable sequence\ntypes (when added to the language) should also support these\noperations. Strings and tuples are immutable sequence types: such\nobjects cannot be modified once created. The following operations are\ndefined on mutable sequence types (where *x* is an arbitrary object):\n\n+--------------------------------+----------------------------------+-----------------------+\n| Operation | Result | Notes |\n+================================+==================================+=======================+\n| ``s[i] = x`` | item *i* of *s* is replaced by | |\n| | *x* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s[i:j] = t`` | slice of *s* from *i* to *j* is | |\n| | replaced by the contents of the | |\n| | iterable *t* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``del s[i:j]`` | same as ``s[i:j] = []`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s[i:j:k] = t`` | the elements of ``s[i:j:k]`` are | (1) |\n| | replaced by those of *t* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``del s[i:j:k]`` | removes the elements of | |\n| | ``s[i:j:k]`` from the list | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.append(x)`` | same as ``s[len(s):len(s)] = | (2) |\n| | [x]`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.extend(x)`` | same as ``s[len(s):len(s)] = x`` | (3) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.count(x)`` | return number of *i*'s for which | |\n| | ``s[i] == x`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.index(x[, i[, j]])`` | return smallest *k* such that | (4) |\n| | ``s[k] == x`` and ``i <= k < j`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.insert(i, x)`` | same as ``s[i:i] = [x]`` | (5) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.pop([i])`` | same as ``x = s[i]; del s[i]; | (6) |\n| | return x`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.remove(x)`` | same as ``del s[s.index(x)]`` | (4) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.reverse()`` | reverses the items of *s* in | (7) |\n| | place | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.sort([cmp[, key[, | sort the items of *s* in place | (7)(8)(9)(10) |\n| reverse]]])`` | | |\n+--------------------------------+----------------------------------+-----------------------+\n\nNotes:\n\n1. *t* must have the same length as the slice it is replacing.\n\n2. The C implementation of Python has historically accepted multiple\n parameters and implicitly joined them into a tuple; this no longer\n works in Python 2.0. Use of this misfeature has been deprecated\n since Python 1.4.\n\n3. *x* can be any iterable object.\n\n4. Raises ``ValueError`` when *x* is not found in *s*. When a negative\n index is passed as the second or third parameter to the ``index()``\n method, the list length is added, as for slice indices. If it is\n still negative, it is truncated to zero, as for slice indices.\n\n Changed in version 2.3: Previously, ``index()`` didn't have\n arguments for specifying start and stop positions.\n\n5. When a negative index is passed as the first parameter to the\n ``insert()`` method, the list length is added, as for slice\n indices. If it is still negative, it is truncated to zero, as for\n slice indices.\n\n Changed in version 2.3: Previously, all negative indices were\n truncated to zero.\n\n6. The ``pop()`` method is only supported by the list and array types.\n The optional argument *i* defaults to ``-1``, so that by default\n the last item is removed and returned.\n\n7. The ``sort()`` and ``reverse()`` methods modify the list in place\n for economy of space when sorting or reversing a large list. To\n remind you that they operate by side effect, they don't return the\n sorted or reversed list.\n\n8. The ``sort()`` method takes optional arguments for controlling the\n comparisons.\n\n *cmp* specifies a custom comparison function of two arguments (list\n items) which should return a negative, zero or positive number\n depending on whether the first argument is considered smaller than,\n equal to, or larger than the second argument: ``cmp=lambda x,y:\n cmp(x.lower(), y.lower())``. The default value is ``None``.\n\n *key* specifies a function of one argument that is used to extract\n a comparison key from each list element: ``key=str.lower``. The\n default value is ``None``.\n\n *reverse* is a boolean value. If set to ``True``, then the list\n elements are sorted as if each comparison were reversed.\n\n In general, the *key* and *reverse* conversion processes are much\n faster than specifying an equivalent *cmp* function. This is\n because *cmp* is called multiple times for each list element while\n *key* and *reverse* touch each element only once. Use\n ``functools.cmp_to_key()`` to convert an old-style *cmp* function\n to a *key* function.\n\n Changed in version 2.3: Support for ``None`` as an equivalent to\n omitting *cmp* was added.\n\n Changed in version 2.4: Support for *key* and *reverse* was added.\n\n9. Starting with Python 2.3, the ``sort()`` method is guaranteed to be\n stable. A sort is stable if it guarantees not to change the\n relative order of elements that compare equal --- this is helpful\n for sorting in multiple passes (for example, sort by department,\n then by salary grade).\n\n10. **CPython implementation detail:** While a list is being sorted,\n the effect of attempting to mutate, or even inspect, the list is\n undefined. The C implementation of Python 2.3 and newer makes the\n list appear empty for the duration, and raises ``ValueError`` if\n it can detect that the list has been mutated during a sort.\n", 'unary': u'\nUnary arithmetic and bitwise operations\n***************************************\n\nAll unary arithmetic and bitwise operations have the same priority:\n\n u_expr ::= power | "-" u_expr | "+" u_expr | "~" u_expr\n\nThe unary ``-`` (minus) operator yields the negation of its numeric\nargument.\n\nThe unary ``+`` (plus) operator yields its numeric argument unchanged.\n\nThe unary ``~`` (invert) operator yields the bitwise inversion of its\nplain or long integer argument. The bitwise inversion of ``x`` is\ndefined as ``-(x+1)``. It only applies to integral numbers.\n\nIn all three cases, if the argument does not have the proper type, a\n``TypeError`` exception is raised.\n', 'while': u'\nThe ``while`` statement\n***********************\n\nThe ``while`` statement is used for repeated execution as long as an\nexpression is true:\n\n while_stmt ::= "while" expression ":" suite\n ["else" ":" suite]\n\nThis repeatedly tests the expression and, if it is true, executes the\nfirst suite; if the expression is false (which may be the first time\nit is tested) the suite of the ``else`` clause, if present, is\nexecuted and the loop terminates.\n\nA ``break`` statement executed in the first suite terminates the loop\nwithout executing the ``else`` clause\'s suite. A ``continue``\nstatement executed in the first suite skips the rest of the suite and\ngoes back to testing the expression.\n', - 'with': u'\nThe ``with`` statement\n**********************\n\nNew in version 2.5.\n\nThe ``with`` statement is used to wrap the execution of a block with\nmethods defined by a context manager (see section *With Statement\nContext Managers*). This allows common\n``try``...``except``...``finally`` usage patterns to be encapsulated\nfor convenient reuse.\n\n with_stmt ::= "with" with_item ("," with_item)* ":" suite\n with_item ::= expression ["as" target]\n\nThe execution of the ``with`` statement with one "item" proceeds as\nfollows:\n\n1. The context expression is evaluated to obtain a context manager.\n\n2. The context manager\'s ``__exit__()`` is loaded for later use.\n\n3. The context manager\'s ``__enter__()`` method is invoked.\n\n4. If a target was included in the ``with`` statement, the return\n value from ``__enter__()`` is assigned to it.\n\n Note: The ``with`` statement guarantees that if the ``__enter__()``\n method returns without an error, then ``__exit__()`` will always\n be called. Thus, if an error occurs during the assignment to the\n target list, it will be treated the same as an error occurring\n within the suite would be. See step 6 below.\n\n5. The suite is executed.\n\n6. The context manager\'s ``__exit__()`` method is invoked. If an\n exception caused the suite to be exited, its type, value, and\n traceback are passed as arguments to ``__exit__()``. Otherwise,\n three ``None`` arguments are supplied.\n\n If the suite was exited due to an exception, and the return value\n from the ``__exit__()`` method was false, the exception is\n reraised. If the return value was true, the exception is\n suppressed, and execution continues with the statement following\n the ``with`` statement.\n\n If the suite was exited for any reason other than an exception, the\n return value from ``__exit__()`` is ignored, and execution proceeds\n at the normal location for the kind of exit that was taken.\n\nWith more than one item, the context managers are processed as if\nmultiple ``with`` statements were nested:\n\n with A() as a, B() as b:\n suite\n\nis equivalent to\n\n with A() as a:\n with B() as b:\n suite\n\nNote: In Python 2.5, the ``with`` statement is only allowed when the\n ``with_statement`` feature has been enabled. It is always enabled\n in Python 2.6.\n\nChanged in version 2.7: Support for multiple context expressions.\n\nSee also:\n\n **PEP 0343** - The "with" statement\n The specification, background, and examples for the Python\n ``with`` statement.\n', + 'with': u'\nThe ``with`` statement\n**********************\n\nNew in version 2.5.\n\nThe ``with`` statement is used to wrap the execution of a block with\nmethods defined by a context manager (see section *With Statement\nContext Managers*). This allows common\n``try``...``except``...``finally`` usage patterns to be encapsulated\nfor convenient reuse.\n\n with_stmt ::= "with" with_item ("," with_item)* ":" suite\n with_item ::= expression ["as" target]\n\nThe execution of the ``with`` statement with one "item" proceeds as\nfollows:\n\n1. The context expression (the expression given in the **with_item**)\n is evaluated to obtain a context manager.\n\n2. The context manager\'s ``__exit__()`` is loaded for later use.\n\n3. The context manager\'s ``__enter__()`` method is invoked.\n\n4. If a target was included in the ``with`` statement, the return\n value from ``__enter__()`` is assigned to it.\n\n Note: The ``with`` statement guarantees that if the ``__enter__()``\n method returns without an error, then ``__exit__()`` will always\n be called. Thus, if an error occurs during the assignment to the\n target list, it will be treated the same as an error occurring\n within the suite would be. See step 6 below.\n\n5. The suite is executed.\n\n6. The context manager\'s ``__exit__()`` method is invoked. If an\n exception caused the suite to be exited, its type, value, and\n traceback are passed as arguments to ``__exit__()``. Otherwise,\n three ``None`` arguments are supplied.\n\n If the suite was exited due to an exception, and the return value\n from the ``__exit__()`` method was false, the exception is\n reraised. If the return value was true, the exception is\n suppressed, and execution continues with the statement following\n the ``with`` statement.\n\n If the suite was exited for any reason other than an exception, the\n return value from ``__exit__()`` is ignored, and execution proceeds\n at the normal location for the kind of exit that was taken.\n\nWith more than one item, the context managers are processed as if\nmultiple ``with`` statements were nested:\n\n with A() as a, B() as b:\n suite\n\nis equivalent to\n\n with A() as a:\n with B() as b:\n suite\n\nNote: In Python 2.5, the ``with`` statement is only allowed when the\n ``with_statement`` feature has been enabled. It is always enabled\n in Python 2.6.\n\nChanged in version 2.7: Support for multiple context expressions.\n\nSee also:\n\n **PEP 0343** - The "with" statement\n The specification, background, and examples for the Python\n ``with`` statement.\n', 'yield': u'\nThe ``yield`` statement\n***********************\n\n yield_stmt ::= yield_expression\n\nThe ``yield`` statement is only used when defining a generator\nfunction, and is only used in the body of the generator function.\nUsing a ``yield`` statement in a function definition is sufficient to\ncause that definition to create a generator function instead of a\nnormal function.\n\nWhen a generator function is called, it returns an iterator known as a\ngenerator iterator, or more commonly, a generator. The body of the\ngenerator function is executed by calling the generator\'s ``next()``\nmethod repeatedly until it raises an exception.\n\nWhen a ``yield`` statement is executed, the state of the generator is\nfrozen and the value of **expression_list** is returned to\n``next()``\'s caller. By "frozen" we mean that all local state is\nretained, including the current bindings of local variables, the\ninstruction pointer, and the internal evaluation stack: enough\ninformation is saved so that the next time ``next()`` is invoked, the\nfunction can proceed exactly as if the ``yield`` statement were just\nanother external call.\n\nAs of Python version 2.5, the ``yield`` statement is now allowed in\nthe ``try`` clause of a ``try`` ... ``finally`` construct. If the\ngenerator is not resumed before it is finalized (by reaching a zero\nreference count or by being garbage collected), the generator-\niterator\'s ``close()`` method will be called, allowing any pending\n``finally`` clauses to execute.\n\nNote: In Python 2.2, the ``yield`` statement was only allowed when the\n ``generators`` feature has been enabled. This ``__future__`` import\n statement was used to enable the feature:\n\n from __future__ import generators\n\nSee also:\n\n **PEP 0255** - Simple Generators\n The proposal for adding generators and the ``yield`` statement\n to Python.\n\n **PEP 0342** - Coroutines via Enhanced Generators\n The proposal that, among other generator enhancements, proposed\n allowing ``yield`` to appear inside a ``try`` ... ``finally``\n block.\n'} diff --git a/lib-python/2.7/random.py b/lib-python/2.7/random.py --- a/lib-python/2.7/random.py +++ b/lib-python/2.7/random.py @@ -317,7 +317,7 @@ n = len(population) if not 0 <= k <= n: - raise ValueError, "sample larger than population" + raise ValueError("sample larger than population") random = self.random _int = int result = [None] * k @@ -490,6 +490,12 @@ Conditions on the parameters are alpha > 0 and beta > 0. + The probability distribution function is: + + x ** (alpha - 1) * math.exp(-x / beta) + pdf(x) = -------------------------------------- + math.gamma(alpha) * beta ** alpha + """ # alpha > 0, beta > 0, mean is alpha*beta, variance is alpha*beta**2 @@ -592,7 +598,7 @@ ## -------------------- beta -------------------- ## See -## http://sourceforge.net/bugs/?func=detailbug&bug_id=130030&group_id=5470 +## http://mail.python.org/pipermail/python-bugs-list/2001-January/003752.html ## for Ivan Frohne's insightful analysis of why the original implementation: ## ## def betavariate(self, alpha, beta): diff --git a/lib-python/2.7/re.py b/lib-python/2.7/re.py --- a/lib-python/2.7/re.py +++ b/lib-python/2.7/re.py @@ -207,8 +207,7 @@ "Escape all non-alphanumeric characters in pattern." s = list(pattern) alphanum = _alphanum - for i in range(len(pattern)): - c = pattern[i] + for i, c in enumerate(pattern): if c not in alphanum: if c == "\000": s[i] = "\\000" diff --git a/lib-python/2.7/shutil.py b/lib-python/2.7/shutil.py --- a/lib-python/2.7/shutil.py +++ b/lib-python/2.7/shutil.py @@ -277,6 +277,12 @@ """ real_dst = dst if os.path.isdir(dst): + if _samefile(src, dst): + # We might be on a case insensitive filesystem, + # perform the rename anyway. + os.rename(src, dst) + return + real_dst = os.path.join(dst, _basename(src)) if os.path.exists(real_dst): raise Error, "Destination path '%s' already exists" % real_dst @@ -336,7 +342,7 @@ archive that is being built. If not provided, the current owner and group will be used. - The output tar file will be named 'base_dir' + ".tar", possibly plus + The output tar file will be named 'base_name' + ".tar", possibly plus the appropriate compression extension (".gz", or ".bz2"). Returns the output filename. @@ -406,7 +412,7 @@ def _make_zipfile(base_name, base_dir, verbose=0, dry_run=0, logger=None): """Create a zip file from all the files under 'base_dir'. - The output zip file will be named 'base_dir' + ".zip". Uses either the + The output zip file will be named 'base_name' + ".zip". Uses either the "zipfile" Python module (if available) or the InfoZIP "zip" utility (if installed and found on the default search path). If neither tool is available, raises ExecError. Returns the name of the output zip diff --git a/lib-python/2.7/site.py b/lib-python/2.7/site.py --- a/lib-python/2.7/site.py +++ b/lib-python/2.7/site.py @@ -61,6 +61,7 @@ import sys import os import __builtin__ +import traceback # Prefixes for site-packages; add additional prefixes like /usr/local here PREFIXES = [sys.prefix, sys.exec_prefix] @@ -155,17 +156,26 @@ except IOError: return with f: - for line in f: + for n, line in enumerate(f): if line.startswith("#"): continue - if line.startswith(("import ", "import\t")): - exec line - continue - line = line.rstrip() - dir, dircase = makepath(sitedir, line) - if not dircase in known_paths and os.path.exists(dir): - sys.path.append(dir) - known_paths.add(dircase) + try: + if line.startswith(("import ", "import\t")): + exec line + continue + line = line.rstrip() + dir, dircase = makepath(sitedir, line) + if not dircase in known_paths and os.path.exists(dir): + sys.path.append(dir) + known_paths.add(dircase) + except Exception as err: + print >>sys.stderr, "Error processing line {:d} of {}:\n".format( + n+1, fullname) + for record in traceback.format_exception(*sys.exc_info()): + for line in record.splitlines(): + print >>sys.stderr, ' '+line + print >>sys.stderr, "\nRemainder of file ignored" + break if reset: known_paths = None return known_paths diff --git a/lib-python/2.7/smtplib.py b/lib-python/2.7/smtplib.py --- a/lib-python/2.7/smtplib.py +++ b/lib-python/2.7/smtplib.py @@ -49,17 +49,18 @@ from email.base64mime import encode as encode_base64 from sys import stderr -__all__ = ["SMTPException","SMTPServerDisconnected","SMTPResponseException", - "SMTPSenderRefused","SMTPRecipientsRefused","SMTPDataError", - "SMTPConnectError","SMTPHeloError","SMTPAuthenticationError", - "quoteaddr","quotedata","SMTP"] +__all__ = ["SMTPException", "SMTPServerDisconnected", "SMTPResponseException", + "SMTPSenderRefused", "SMTPRecipientsRefused", "SMTPDataError", + "SMTPConnectError", "SMTPHeloError", "SMTPAuthenticationError", + "quoteaddr", "quotedata", "SMTP"] SMTP_PORT = 25 SMTP_SSL_PORT = 465 -CRLF="\r\n" +CRLF = "\r\n" OLDSTYLE_AUTH = re.compile(r"auth=(.*)", re.I) + # Exception classes used by this module. class SMTPException(Exception): """Base class for all exceptions raised by this module.""" @@ -109,7 +110,7 @@ def __init__(self, recipients): self.recipients = recipients - self.args = ( recipients,) + self.args = (recipients,) class SMTPDataError(SMTPResponseException): @@ -128,6 +129,7 @@ combination provided. """ + def quoteaddr(addr): """Quote a subset of the email addresses defined by RFC 821. @@ -138,7 +140,7 @@ m = email.utils.parseaddr(addr)[1] except AttributeError: pass - if m == (None, None): # Indicates parse failure or AttributeError + if m == (None, None): # Indicates parse failure or AttributeError # something weird here.. punt -ddm return "<%s>" % addr elif m is None: @@ -175,7 +177,8 @@ chr = None while chr != "\n": chr = self.sslobj.read(1) - if not chr: break + if not chr: + break str += chr return str @@ -219,6 +222,7 @@ ehlo_msg = "ehlo" ehlo_resp = None does_esmtp = 0 + default_port = SMTP_PORT def __init__(self, host='', port=0, local_hostname=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT): @@ -234,7 +238,6 @@ """ self.timeout = timeout self.esmtp_features = {} - self.default_port = SMTP_PORT if host: (code, msg) = self.connect(host, port) if code != 220: @@ -269,10 +272,11 @@ def _get_socket(self, port, host, timeout): # This makes it simpler for SMTP_SSL to use the SMTP connect code # and just alter the socket connection bit. - if self.debuglevel > 0: print>>stderr, 'connect:', (host, port) + if self.debuglevel > 0: + print>>stderr, 'connect:', (host, port) return socket.create_connection((port, host), timeout) - def connect(self, host='localhost', port = 0): + def connect(self, host='localhost', port=0): """Connect to a host on a given port. If the hostname ends with a colon (`:') followed by a number, and @@ -286,20 +290,25 @@ if not port and (host.find(':') == host.rfind(':')): i = host.rfind(':') if i >= 0: - host, port = host[:i], host[i+1:] - try: port = int(port) + host, port = host[:i], host[i + 1:] + try: + port = int(port) except ValueError: raise socket.error, "nonnumeric port" - if not port: port = self.default_port - if self.debuglevel > 0: print>>stderr, 'connect:', (host, port) + if not port: + port = self.default_port + if self.debuglevel > 0: + print>>stderr, 'connect:', (host, port) self.sock = self._get_socket(host, port, self.timeout) (code, msg) = self.getreply() - if self.debuglevel > 0: print>>stderr, "connect:", msg + if self.debuglevel > 0: + print>>stderr, "connect:", msg return (code, msg) def send(self, str): """Send `str' to the server.""" - if self.debuglevel > 0: print>>stderr, 'send:', repr(str) + if self.debuglevel > 0: + print>>stderr, 'send:', repr(str) if hasattr(self, 'sock') and self.sock: try: self.sock.sendall(str) @@ -330,7 +339,7 @@ Raises SMTPServerDisconnected if end-of-file is reached. """ - resp=[] + resp = [] if self.file is None: self.file = self.sock.makefile('rb') while 1: @@ -341,9 +350,10 @@ if line == '': self.close() raise SMTPServerDisconnected("Connection unexpectedly closed") - if self.debuglevel > 0: print>>stderr, 'reply:', repr(line) + if self.debuglevel > 0: + print>>stderr, 'reply:', repr(line) resp.append(line[4:].strip()) - code=line[:3] + code = line[:3] # Check that the error code is syntactically correct. # Don't attempt to read a continuation line if it is broken. try: @@ -352,17 +362,17 @@ errcode = -1 break # Check if multiline response. - if line[3:4]!="-": + if line[3:4] != "-": break errmsg = "\n".join(resp) if self.debuglevel > 0: - print>>stderr, 'reply: retcode (%s); Msg: %s' % (errcode,errmsg) + print>>stderr, 'reply: retcode (%s); Msg: %s' % (errcode, errmsg) return errcode, errmsg def docmd(self, cmd, args=""): """Send a command, and return its response code.""" - self.putcmd(cmd,args) + self.putcmd(cmd, args) return self.getreply() # std smtp commands @@ -372,9 +382,9 @@ host. """ self.putcmd("helo", name or self.local_hostname) - (code,msg)=self.getreply() - self.helo_resp=msg - return (code,msg) + (code, msg) = self.getreply() + self.helo_resp = msg + return (code, msg) def ehlo(self, name=''): """ SMTP 'ehlo' command. @@ -383,19 +393,19 @@ """ self.esmtp_features = {} self.putcmd(self.ehlo_msg, name or self.local_hostname) - (code,msg)=self.getreply() + (code, msg) = self.getreply() # According to RFC1869 some (badly written) # MTA's will disconnect on an ehlo. Toss an exception if # that happens -ddm if code == -1 and len(msg) == 0: self.close() raise SMTPServerDisconnected("Server not connected") - self.ehlo_resp=msg + self.ehlo_resp = msg if code != 250: - return (code,msg) - self.does_esmtp=1 + return (code, msg) + self.does_esmtp = 1 #parse the ehlo response -ddm - resp=self.ehlo_resp.split('\n') + resp = self.ehlo_resp.split('\n') del resp[0] for each in resp: # To be able to communicate with as many SMTP servers as possible, @@ -415,16 +425,16 @@ # It's actually stricter, in that only spaces are allowed between # parameters, but were not going to check for that here. Note # that the space isn't present if there are no parameters. - m=re.match(r'(?P<feature>[A-Za-z0-9][A-Za-z0-9\-]*) ?',each) + m = re.match(r'(?P<feature>[A-Za-z0-9][A-Za-z0-9\-]*) ?', each) if m: - feature=m.group("feature").lower() - params=m.string[m.end("feature"):].strip() + feature = m.group("feature").lower() + params = m.string[m.end("feature"):].strip() if feature == "auth": self.esmtp_features[feature] = self.esmtp_features.get(feature, "") \ + " " + params else: - self.esmtp_features[feature]=params - return (code,msg) + self.esmtp_features[feature] = params + return (code, msg) def has_extn(self, opt): """Does the server support a given SMTP service extension?""" @@ -444,23 +454,23 @@ """SMTP 'noop' command -- doesn't do anything :>""" return self.docmd("noop") - def mail(self,sender,options=[]): + def mail(self, sender, options=[]): """SMTP 'mail' command -- begins mail xfer session.""" optionlist = '' if options and self.does_esmtp: optionlist = ' ' + ' '.join(options) - self.putcmd("mail", "FROM:%s%s" % (quoteaddr(sender) ,optionlist)) + self.putcmd("mail", "FROM:%s%s" % (quoteaddr(sender), optionlist)) return self.getreply() - def rcpt(self,recip,options=[]): + def rcpt(self, recip, options=[]): """SMTP 'rcpt' command -- indicates 1 recipient for this mail.""" optionlist = '' if options and self.does_esmtp: optionlist = ' ' + ' '.join(options) - self.putcmd("rcpt","TO:%s%s" % (quoteaddr(recip),optionlist)) + self.putcmd("rcpt", "TO:%s%s" % (quoteaddr(recip), optionlist)) return self.getreply() - def data(self,msg): + def data(self, msg): """SMTP 'DATA' command -- sends message data to server. Automatically quotes lines beginning with a period per rfc821. @@ -469,26 +479,28 @@ response code received when the all data is sent. """ self.putcmd("data") - (code,repl)=self.getreply() - if self.debuglevel >0 : print>>stderr, "data:", (code,repl) + (code, repl) = self.getreply() + if self.debuglevel > 0: + print>>stderr, "data:", (code, repl) if code != 354: - raise SMTPDataError(code,repl) + raise SMTPDataError(code, repl) else: q = quotedata(msg) if q[-2:] != CRLF: q = q + CRLF q = q + "." + CRLF self.send(q) - (code,msg)=self.getreply() - if self.debuglevel >0 : print>>stderr, "data:", (code,msg) - return (code,msg) + (code, msg) = self.getreply() + if self.debuglevel > 0: + print>>stderr, "data:", (code, msg) + return (code, msg) def verify(self, address): """SMTP 'verify' command -- checks for address validity.""" self.putcmd("vrfy", quoteaddr(address)) return self.getreply() # a.k.a. - vrfy=verify + vrfy = verify def expn(self, address): """SMTP 'expn' command -- expands a mailing list.""" @@ -592,7 +604,7 @@ raise SMTPAuthenticationError(code, resp) return (code, resp) - def starttls(self, keyfile = None, certfile = None): + def starttls(self, keyfile=None, certfile=None): """Puts the connection to the SMTP server into TLS mode. If there has been no previous EHLO or HELO command this session, this @@ -695,22 +707,22 @@ for option in mail_options: esmtp_opts.append(option) - (code,resp) = self.mail(from_addr, esmtp_opts) + (code, resp) = self.mail(from_addr, esmtp_opts) if code != 250: self.rset() raise SMTPSenderRefused(code, resp, from_addr) - senderrs={} + senderrs = {} if isinstance(to_addrs, basestring): to_addrs = [to_addrs] for each in to_addrs: - (code,resp)=self.rcpt(each, rcpt_options) + (code, resp) = self.rcpt(each, rcpt_options) if (code != 250) and (code != 251): - senderrs[each]=(code,resp) - if len(senderrs)==len(to_addrs): + senderrs[each] = (code, resp) + if len(senderrs) == len(to_addrs): # the server refused all our recipients self.rset() raise SMTPRecipientsRefused(senderrs) - (code,resp) = self.data(msg) + (code, resp) = self.data(msg) if code != 250: self.rset() raise SMTPDataError(code, resp) @@ -744,16 +756,19 @@ are also optional - they can contain a PEM formatted private key and certificate chain file for the SSL connection. """ + + default_port = SMTP_SSL_PORT + def __init__(self, host='', port=0, local_hostname=None, keyfile=None, certfile=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT): self.keyfile = keyfile self.certfile = certfile SMTP.__init__(self, host, port, local_hostname, timeout) - self.default_port = SMTP_SSL_PORT def _get_socket(self, host, port, timeout): - if self.debuglevel > 0: print>>stderr, 'connect:', (host, port) + if self.debuglevel > 0: + print>>stderr, 'connect:', (host, port) new_socket = socket.create_connection((host, port), timeout) new_socket = ssl.wrap_socket(new_socket, self.keyfile, self.certfile) self.file = SSLFakeFile(new_socket) @@ -781,11 +796,11 @@ ehlo_msg = "lhlo" - def __init__(self, host = '', port = LMTP_PORT, local_hostname = None): + def __init__(self, host='', port=LMTP_PORT, local_hostname=None): """Initialize a new instance.""" SMTP.__init__(self, host, port, local_hostname) - def connect(self, host = 'localhost', port = 0): + def connect(self, host='localhost', port=0): """Connect to the LMTP daemon, on either a Unix or a TCP socket.""" if host[0] != '/': return SMTP.connect(self, host, port) @@ -795,13 +810,15 @@ self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) self.sock.connect(host) except socket.error, msg: - if self.debuglevel > 0: print>>stderr, 'connect fail:', host + if self.debuglevel > 0: + print>>stderr, 'connect fail:', host if self.sock: self.sock.close() self.sock = None raise socket.error, msg (code, msg) = self.getreply() - if self.debuglevel > 0: print>>stderr, "connect:", msg + if self.debuglevel > 0: + print>>stderr, "connect:", msg return (code, msg) @@ -815,7 +832,7 @@ return sys.stdin.readline().strip() fromaddr = prompt("From") - toaddrs = prompt("To").split(',') + toaddrs = prompt("To").split(',') print "Enter message, end with ^D:" msg = '' while 1: diff --git a/lib-python/2.7/ssl.py b/lib-python/2.7/ssl.py --- a/lib-python/2.7/ssl.py +++ b/lib-python/2.7/ssl.py @@ -121,9 +121,11 @@ if e.errno != errno.ENOTCONN: raise # no, no connection yet + self._connected = False self._sslobj = None else: # yes, create the SSL object + self._connected = True self._sslobj = _ssl.sslwrap(self._sock, server_side, keyfile, certfile, cert_reqs, ssl_version, ca_certs, @@ -293,21 +295,36 @@ self._sslobj.do_handshake() - def connect(self, addr): - - """Connects to remote ADDR, and then wraps the connection in - an SSL channel.""" - + def _real_connect(self, addr, return_errno): # Here we assume that the socket is client-side, and not # connected at the time of the call. We connect it, then wrap it. - if self._sslobj: + if self._connected: raise ValueError("attempt to connect already-connected SSLSocket!") - socket.connect(self, addr) self._sslobj = _ssl.sslwrap(self._sock, False, self.keyfile, self.certfile, self.cert_reqs, self.ssl_version, self.ca_certs, self.ciphers) - if self.do_handshake_on_connect: - self.do_handshake() + try: + socket.connect(self, addr) + if self.do_handshake_on_connect: + self.do_handshake() + except socket_error as e: + if return_errno: + return e.errno + else: + self._sslobj = None + raise e + self._connected = True + return 0 + + def connect(self, addr): + """Connects to remote ADDR, and then wraps the connection in + an SSL channel.""" + self._real_connect(addr, False) + + def connect_ex(self, addr): + """Connects to remote ADDR, and then wraps the connection in + an SSL channel.""" + return self._real_connect(addr, True) def accept(self): diff --git a/lib-python/2.7/subprocess.py b/lib-python/2.7/subprocess.py --- a/lib-python/2.7/subprocess.py +++ b/lib-python/2.7/subprocess.py @@ -396,6 +396,7 @@ import traceback import gc import signal +import errno # Exception classes used by this module. class CalledProcessError(Exception): @@ -427,7 +428,6 @@ else: import select _has_poll = hasattr(select, 'poll') - import errno import fcntl import pickle @@ -441,8 +441,15 @@ "check_output", "CalledProcessError"] if mswindows: - from _subprocess import CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP - __all__.extend(["CREATE_NEW_CONSOLE", "CREATE_NEW_PROCESS_GROUP"]) + from _subprocess import (CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP, + STD_INPUT_HANDLE, STD_OUTPUT_HANDLE, + STD_ERROR_HANDLE, SW_HIDE, + STARTF_USESTDHANDLES, STARTF_USESHOWWINDOW) + + __all__.extend(["CREATE_NEW_CONSOLE", "CREATE_NEW_PROCESS_GROUP", + "STD_INPUT_HANDLE", "STD_OUTPUT_HANDLE", + "STD_ERROR_HANDLE", "SW_HIDE", + "STARTF_USESTDHANDLES", "STARTF_USESHOWWINDOW"]) try: MAXFD = os.sysconf("SC_OPEN_MAX") except: @@ -726,7 +733,11 @@ stderr = None if self.stdin: if input: - self.stdin.write(input) + try: + self.stdin.write(input) + except IOError as e: + if e.errno != errno.EPIPE and e.errno != errno.EINVAL: + raise self.stdin.close() elif self.stdout: stdout = self.stdout.read() @@ -883,7 +894,7 @@ except pywintypes.error, e: # Translate pywintypes.error to WindowsError, which is # a subclass of OSError. FIXME: We should really - # translate errno using _sys_errlist (or simliar), but + # translate errno using _sys_errlist (or similar), but # how can this be done from Python? raise WindowsError(*e.args) finally: @@ -956,7 +967,11 @@ if self.stdin: if input is not None: - self.stdin.write(input) + try: + self.stdin.write(input) + except IOError as e: + if e.errno != errno.EPIPE: + raise self.stdin.close() if self.stdout: @@ -1051,14 +1066,17 @@ errread, errwrite) - def _set_cloexec_flag(self, fd): + def _set_cloexec_flag(self, fd, cloexec=True): try: cloexec_flag = fcntl.FD_CLOEXEC except AttributeError: cloexec_flag = 1 old = fcntl.fcntl(fd, fcntl.F_GETFD) - fcntl.fcntl(fd, fcntl.F_SETFD, old | cloexec_flag) + if cloexec: + fcntl.fcntl(fd, fcntl.F_SETFD, old | cloexec_flag) + else: + fcntl.fcntl(fd, fcntl.F_SETFD, old & ~cloexec_flag) def _close_fds(self, but): @@ -1128,21 +1146,25 @@ os.close(errpipe_read) # Dup fds for child - if p2cread is not None: - os.dup2(p2cread, 0) - if c2pwrite is not None: - os.dup2(c2pwrite, 1) - if errwrite is not None: - os.dup2(errwrite, 2) + def _dup2(a, b): + # dup2() removes the CLOEXEC flag but + # we must do it ourselves if dup2() + # would be a no-op (issue #10806). + if a == b: + self._set_cloexec_flag(a, False) + elif a is not None: + os.dup2(a, b) + _dup2(p2cread, 0) + _dup2(c2pwrite, 1) + _dup2(errwrite, 2) - # Close pipe fds. Make sure we don't close the same - # fd more than once, or standard fds. - if p2cread is not None and p2cread not in (0,): - os.close(p2cread) - if c2pwrite is not None and c2pwrite not in (p2cread, 1): - os.close(c2pwrite) - if errwrite is not None and errwrite not in (p2cread, c2pwrite, 2): - os.close(errwrite) + # Close pipe fds. Make sure we don't close the + # same fd more than once, or standard fds. + closed = { None } + for fd in [p2cread, c2pwrite, errwrite]: + if fd not in closed and fd > 2: + os.close(fd) + closed.add(fd) # Close all other fds, if asked for if close_fds: @@ -1194,7 +1216,11 @@ os.close(errpipe_read) if data != "": - _eintr_retry_call(os.waitpid, self.pid, 0) + try: + _eintr_retry_call(os.waitpid, self.pid, 0) + except OSError as e: + if e.errno != errno.ECHILD: + raise child_exception = pickle.loads(data) for fd in (p2cwrite, c2pread, errread): if fd is not None: @@ -1240,7 +1266,15 @@ """Wait for child process to terminate. Returns returncode attribute.""" if self.returncode is None: - pid, sts = _eintr_retry_call(os.waitpid, self.pid, 0) + try: + pid, sts = _eintr_retry_call(os.waitpid, self.pid, 0) + except OSError as e: + if e.errno != errno.ECHILD: + raise + # This happens if SIGCLD is set to be ignored or waiting + # for child processes has otherwise been disabled for our + # process. This child is dead, we can't get the status. + sts = 0 self._handle_exitstatus(sts) return self.returncode @@ -1317,9 +1351,16 @@ for fd, mode in ready: if mode & select.POLLOUT: chunk = input[input_offset : input_offset + _PIPE_BUF] - input_offset += os.write(fd, chunk) - if input_offset >= len(input): - close_unregister_and_remove(fd) + try: + input_offset += os.write(fd, chunk) + except OSError as e: + if e.errno == errno.EPIPE: + close_unregister_and_remove(fd) + else: + raise + else: + if input_offset >= len(input): + close_unregister_and_remove(fd) elif mode & select_POLLIN_POLLPRI: data = os.read(fd, 4096) if not data: @@ -1358,11 +1399,19 @@ if self.stdin in wlist: chunk = input[input_offset : input_offset + _PIPE_BUF] - bytes_written = os.write(self.stdin.fileno(), chunk) - input_offset += bytes_written - if input_offset >= len(input): - self.stdin.close() - write_set.remove(self.stdin) + try: + bytes_written = os.write(self.stdin.fileno(), chunk) + except OSError as e: + if e.errno == errno.EPIPE: + self.stdin.close() + write_set.remove(self.stdin) + else: + raise + else: + input_offset += bytes_written + if input_offset >= len(input): + self.stdin.close() + write_set.remove(self.stdin) if self.stdout in rlist: data = os.read(self.stdout.fileno(), 1024) diff --git a/lib-python/2.7/symbol.py b/lib-python/2.7/symbol.py --- a/lib-python/2.7/symbol.py +++ b/lib-python/2.7/symbol.py @@ -82,20 +82,19 @@ sliceop = 325 exprlist = 326 testlist = 327 -dictmaker = 328 -dictorsetmaker = 329 -classdef = 330 -arglist = 331 -argument = 332 -list_iter = 333 -list_for = 334 -list_if = 335 -comp_iter = 336 -comp_for = 337 -comp_if = 338 -testlist1 = 339 -encoding_decl = 340 -yield_expr = 341 +dictorsetmaker = 328 +classdef = 329 +arglist = 330 +argument = 331 +list_iter = 332 +list_for = 333 +list_if = 334 +comp_iter = 335 +comp_for = 336 +comp_if = 337 +testlist1 = 338 +encoding_decl = 339 +yield_expr = 340 #--end constants-- sym_name = {} diff --git a/lib-python/2.7/sysconfig.py b/lib-python/2.7/sysconfig.py --- a/lib-python/2.7/sysconfig.py +++ b/lib-python/2.7/sysconfig.py @@ -271,7 +271,7 @@ def _get_makefile_filename(): if _PYTHON_BUILD: return os.path.join(_PROJECT_BASE, "Makefile") - return os.path.join(get_path('stdlib'), "config", "Makefile") + return os.path.join(get_path('platstdlib'), "config", "Makefile") def _init_posix(vars): @@ -297,21 +297,6 @@ msg = msg + " (%s)" % e.strerror raise IOError(msg) - # On MacOSX we need to check the setting of the environment variable - # MACOSX_DEPLOYMENT_TARGET: configure bases some choices on it so - # it needs to be compatible. - # If it isn't set we set it to the configure-time value - if sys.platform == 'darwin' and 'MACOSX_DEPLOYMENT_TARGET' in vars: - cfg_target = vars['MACOSX_DEPLOYMENT_TARGET'] - cur_target = os.getenv('MACOSX_DEPLOYMENT_TARGET', '') - if cur_target == '': - cur_target = cfg_target - os.putenv('MACOSX_DEPLOYMENT_TARGET', cfg_target) - elif map(int, cfg_target.split('.')) > map(int, cur_target.split('.')): - msg = ('$MACOSX_DEPLOYMENT_TARGET mismatch: now "%s" but "%s" ' - 'during configure' % (cur_target, cfg_target)) - raise IOError(msg) - # On AIX, there are wrong paths to the linker scripts in the Makefile # -- these paths are relative to the Python source, but when installed # the scripts are in another directory. @@ -616,9 +601,7 @@ # machine is going to compile and link as if it were # MACOSX_DEPLOYMENT_TARGET. cfgvars = get_config_vars() - macver = os.environ.get('MACOSX_DEPLOYMENT_TARGET') - if not macver: - macver = cfgvars.get('MACOSX_DEPLOYMENT_TARGET') + macver = cfgvars.get('MACOSX_DEPLOYMENT_TARGET') if 1: # Always calculate the release of the running machine, @@ -639,7 +622,6 @@ m = re.search( r'<key>ProductUserVisibleVersion</key>\s*' + r'<string>(.*?)</string>', f.read()) - f.close() if m is not None: macrelease = '.'.join(m.group(1).split('.')[:2]) # else: fall back to the default behaviour diff --git a/lib-python/2.7/tarfile.py b/lib-python/2.7/tarfile.py --- a/lib-python/2.7/tarfile.py +++ b/lib-python/2.7/tarfile.py @@ -2239,10 +2239,14 @@ if hasattr(os, "symlink") and hasattr(os, "link"): # For systems that support symbolic and hard links. if tarinfo.issym(): + if os.path.lexists(targetpath): + os.unlink(targetpath) os.symlink(tarinfo.linkname, targetpath) else: # See extract(). if os.path.exists(tarinfo._link_target): + if os.path.lexists(targetpath): + os.unlink(targetpath) os.link(tarinfo._link_target, targetpath) else: self._extract_member(self._find_link_target(tarinfo), targetpath) diff --git a/lib-python/2.7/telnetlib.py b/lib-python/2.7/telnetlib.py --- a/lib-python/2.7/telnetlib.py +++ b/lib-python/2.7/telnetlib.py @@ -236,7 +236,7 @@ """ if self.debuglevel > 0: - print 'Telnet(%s,%d):' % (self.host, self.port), + print 'Telnet(%s,%s):' % (self.host, self.port), if args: print msg % args else: diff --git a/lib-python/2.7/test/cjkencodings/big5-utf8.txt b/lib-python/2.7/test/cjkencodings/big5-utf8.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/big5-utf8.txt @@ -0,0 +1,9 @@ +如何在 Python 中使用既有的 C library? +　在資訊科技快速發展的今天, 開發及測試軟體的速度是不容忽視的 +課題. 為加快開發及測試的速度, 我們便常希望能利用一些已開發好的 +library, 並有一個 fast prototyping 的 programming language 可 +供使用. 目前有許許多多的 library 是以 C 寫成, 而 Python 是一個 +fast prototyping 的 programming language. 故我們希望能將既有的 +C library 拿到 Python 的環境中測試及整合. 其中最主要也是我們所 +要討論的問題就是: + diff --git a/lib-python/2.7/test/cjkencodings/big5.txt b/lib-python/2.7/test/cjkencodings/big5.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/big5.txt @@ -0,0 +1,9 @@ +�p��b Python ��ϥάJ�� C library? +�@�b��T��ާֳt�o�i��, �}�o�δ��ճn�骺�t�׬O��e�� +��D. ��[�ֶ}�o�δ��ժ��t��, �ڭ̫K�`�Ʊ��Q�Τ@�Ǥw�}�o�n�� +library, �æ��@�� fast prototyping �� programming language �i +�Ѩϥ�. �ثe��\�\�h�h�� library �O�H C �g��, �� Python �O�@�� +fast prototyping �� programming language. �G�ڭ̧Ʊ��N�J�� +C library �� Python ��Ҥ��դξ�X. �䤤�̥D�n�]�O�ڭ̩� +�n�Q�ת��D�N�O: + diff --git a/lib-python/2.7/test/cjkencodings/big5hkscs-utf8.txt b/lib-python/2.7/test/cjkencodings/big5hkscs-utf8.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/big5hkscs-utf8.txt @@ -0,0 +1,2 @@ +𠄌Ě鵮罓洆 +ÊÊ̄ê êê̄ diff --git a/lib-python/2.7/test/cjkencodings/big5hkscs.txt b/lib-python/2.7/test/cjkencodings/big5hkscs.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/big5hkscs.txt @@ -0,0 +1,2 @@ +�E�\�s�ڍ� +�f�b�� diff --git a/lib-python/2.7/test/cjkencodings/cp949-utf8.txt b/lib-python/2.7/test/cjkencodings/cp949-utf8.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/cp949-utf8.txt @@ -0,0 +1,9 @@ +똠방각하 펲시콜라 + +㉯㉯납!! 因九月패믤릔궈 ⓡⓖ훀¿¿¿ 긍뒙 ⓔ뎨 ㉯. . +亞영ⓔ능횹 . . . . 서울뤄 뎐학乙家훀 ! ! !ㅠ.ㅠ +흐흐흐 ㄱㄱㄱ☆ㅠ_ㅠ 어릨 탸콰긐 뎌응 칑九들乙㉯드긐 +설릌 家훀 . . . . 굴애쉌 ⓔ궈 ⓡ릘㉱긐 因仁川女中까즼 +와쒀훀 ! ! 亞영ⓔ 家능궈 ☆上관 없능궈능 亞능뒈훀 글애듴 +ⓡ려듀九 싀풔숴훀 어릨 因仁川女中싁⑨들앜!! ㉯㉯납♡ ⌒⌒* + diff --git a/lib-python/2.7/test/cjkencodings/cp949.txt b/lib-python/2.7/test/cjkencodings/cp949.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/cp949.txt @@ -0,0 +1,9 @@ +�c�氢�� ݶ� + +��!! ��В�p�� ި��R�� ѵ� ��. . +䬿��Ѵ�� . . . . �� ʫ�R ! ! !��.�� +�� ٤�_��  ��O �� h�� O +��j ʫ�R . . . . ��֚f �ѱ� �ސt�ƒO �� +�;��R ! ! 䬿�� ʫ�ɱ� ��߾�� ɱŴ� 䬴ɵ��R �۾֊� +�޷�� Ǵ��R � ��Ĩ��!! �� ҡ�* + diff --git a/lib-python/2.7/test/cjkencodings/euc_jisx0213-utf8.txt b/lib-python/2.7/test/cjkencodings/euc_jisx0213-utf8.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/euc_jisx0213-utf8.txt @@ -0,0 +1,8 @@ +Python の開発は、1990 年ごろから開始されています。 +開発者の Guido van Rossum は教育用のプログラミング言語「ABC」の開発に参加していましたが、ABC は実用上の目的にはあまり適していませんでした。 +このため、Guido はより実用的なプログラミング言語の開発を開始し、英国 BBS 放送のコメディ番組「モンティパイソン」のファンである Guido はこの言語を「Python」と名づけました。 +このような背景から生まれた Python の言語設計は、「シンプル」で「習得が容易」という目標に重点が置かれています。 +多くのスクリプト系言語ではユーザの目先の利便性を優先して色々な機能を言語要素として取り入れる場合が多いのですが、Python ではそういった小細工が追加されることはあまりありません。 +言語自体の機能は最小限に押さえ、必要な機能は拡張モジュールとして追加する、というのが Python のポリシーです。 + +ノか゚ト゚トキ喝塀𡚴𪎌麀齁𩛰 diff --git a/lib-python/2.7/test/cjkencodings/euc_jisx0213.txt b/lib-python/2.7/test/cjkencodings/euc_jisx0213.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/euc_jisx0213.txt @@ -0,0 +1,8 @@ +Python �γ�ȯ�ϡ�1990 ǯ��鳫�Ϥ��Ƥ��ޤ�� +��ȯ�Ԥ� Guido van Rossum �϶��ѤΥץ��ߥ󥰸��ABC�פγ�ȯ�˻��ä��Ƥ��ޤ��ABC �ϼ��Ѿ��Ū�ˤϤ��ޤ�Ŭ��Ƥ��ޤ��Ǥ�� +��Τ��ᡢGuido �Ϥ��Ū�ʥץ��ߥ󥰸��γ�ȯ�򳫻Ϥ��ѹ� BBS ��Υ��ǥ��ȡ֥��ƥ� �ѥ��פΥե��Ǥ�� Guido �Ϥ��θ��Python�פ�̾�Ť��ޤ�� +��Τ褦��طʤ��ޤ줿 Python �θ��߷פϡ��֥��ץ�פǡֽ��ưספȤ��ɸ�˽��֤��Ƥ��ޤ�� +¿��Υ��ץȷϸ��Ǥϥ桼��ͥ�褷�ƿ��ʵ�ǽ��ǤȤ��Ƽ��礬¿��ΤǤ��Python �ǤϤ��ä��ٹ��ɲä��뤳�ȤϤ��ޤꤢ��ޤ�� +��켫�Τε�ǽ�ϺǾ��¤˲��ɬ�פʵ�ǽ�ϳ�ĥ�⥸�塼��Ȥ��ɲä��롢�Ȥ��Τ� Python �Υݥꥷ��Ǥ�� + +�Τ� �� ȥ�� ԏ�� diff --git a/lib-python/2.7/test/cjkencodings/euc_jp-utf8.txt b/lib-python/2.7/test/cjkencodings/euc_jp-utf8.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/euc_jp-utf8.txt @@ -0,0 +1,7 @@ +Python の開発は、1990 年ごろから開始されています。 +開発者の Guido van Rossum は教育用のプログラミング言語「ABC」の開発に参加していましたが、ABC は実用上の目的にはあまり適していませんでした。 +このため、Guido はより実用的なプログラミング言語の開発を開始し、英国 BBS 放送のコメディ番組「モンティパイソン」のファンである Guido はこの言語を「Python」と名づけました。 +このような背景から生まれた Python の言語設計は、「シンプル」で「習得が容易」という目標に重点が置かれています。 +多くのスクリプト系言語ではユーザの目先の利便性を優先して色々な機能を言語要素として取り入れる場合が多いのですが、Python ではそういった小細工が追加されることはあまりありません。 +言語自体の機能は最小限に押さえ、必要な機能は拡張モジュールとして追加する、というのが Python のポリシーです。 + diff --git a/lib-python/2.7/test/cjkencodings/euc_jp.txt b/lib-python/2.7/test/cjkencodings/euc_jp.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/euc_jp.txt @@ -0,0 +1,7 @@ +Python �γ�ȯ�ϡ�1990 ǯ��鳫�Ϥ��Ƥ��ޤ�� +��ȯ�Ԥ� Guido van Rossum �϶��ѤΥץ��ߥ󥰸��ABC�פγ�ȯ�˻��ä��Ƥ��ޤ��ABC �ϼ��Ѿ��Ū�ˤϤ��ޤ�Ŭ��Ƥ��ޤ��Ǥ�� +��Τ��ᡢGuido �Ϥ��Ū�ʥץ��ߥ󥰸��γ�ȯ�򳫻Ϥ��ѹ� BBS ��Υ��ǥ��ȡ֥��ƥ� �ѥ��פΥե��Ǥ�� Guido �Ϥ��θ��Python�פ�̾�Ť��ޤ�� +��Τ褦��طʤ��ޤ줿 Python �θ��߷פϡ��֥��ץ�פǡֽ��ưספȤ��ɸ�˽��֤��Ƥ��ޤ�� +¿��Υ��ץȷϸ��Ǥϥ桼��ͥ�褷�ƿ��ʵ�ǽ��ǤȤ��Ƽ��礬¿��ΤǤ��Python �ǤϤ��ä��ٹ��ɲä��뤳�ȤϤ��ޤꤢ��ޤ�� +��켫�Τε�ǽ�ϺǾ��¤˲��ɬ�פʵ�ǽ�ϳ�ĥ�⥸�塼��Ȥ��ɲä��롢�Ȥ��Τ� Python �Υݥꥷ��Ǥ�� + diff --git a/lib-python/2.7/test/cjkencodings/euc_kr-utf8.txt b/lib-python/2.7/test/cjkencodings/euc_kr-utf8.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/euc_kr-utf8.txt @@ -0,0 +1,7 @@ +◎ 파이썬(Python)은 배우기 쉽고, 강력한 프로그래밍 언어입니다. 파이썬은 +효율적인 고수준 데이터 구조와 간단하지만 효율적인 객체지향프로그래밍을 +지원합니다. 파이썬의 우아(優雅)한 문법과 동적 타이핑, 그리고 인터프리팅 +환경은 파이썬을 스크립팅과 여러 분야에서와 대부분의 플랫폼에서의 빠른 +애플리케이션 개발을 할 수 있는 이상적인 언어로 만들어줍니다. + +☆첫가끝: 날아라 쓔쓔쓩~ 닁큼! 뜽금없이 전홥니다. 뷁. 그런거 읎다. diff --git a/lib-python/2.7/test/cjkencodings/euc_kr.txt b/lib-python/2.7/test/cjkencodings/euc_kr.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/euc_kr.txt @@ -0,0 +1,7 @@ +�� ̽�(Python)�� , �� α׷�� Դϴ�. ��̽�� +ȿ�� ȿ�� ü��α׷�� +��մϴ�. ��̽�� (��)�� Ÿ��, �׸�� +ȯ�� ̽�� ũ��ð� �� о߿�� κ�� ÷�� +��ø��̼� �� ִ� �̻�� ݴϴ�. + +��ù��: ��ƶ� �Ԥ��ФԤԤ��ФԾ�~ �Ԥ��Ҥ�ŭ! �Ԥ��Ѥ��ݾ�� Ԥ��Ȥ��ϴ�. �Ԥ��Τ�. �׷�� Ԥ��Ѥ��. diff --git a/lib-python/2.7/test/cjkencodings/gb18030-utf8.txt b/lib-python/2.7/test/cjkencodings/gb18030-utf8.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/gb18030-utf8.txt @@ -0,0 +1,15 @@ +Python（派森）语言是一种功能强大而完善的通用型计算机程序设计语言， +已经具有十多年的发展历史，成熟且稳定。这种语言具有非常简捷而清晰 +的语法特点，适合完成各种高层任务，几乎可以在所有的操作系统中 +运行。这种语言简单而强大，适合各种人士学习使用。目前，基于这 +种语言的相关技术正在飞速的发展，用户数量急剧扩大，相关的资源非常多。 +如何在 Python 中使用既有的 C library? +　在資訊科技快速發展的今天, 開發及測試軟體的速度是不容忽視的 +課題. 為加快開發及測試的速度, 我們便常希望能利用一些已開發好的 +library, 並有一個 fast prototyping 的 programming language 可 +供使用. 目前有許許多多的 library 是以 C 寫成, 而 Python 是一個 +fast prototyping 的 programming language. 故我們希望能將既有的 +C library 拿到 Python 的環境中測試及整合. 其中最主要也是我們所 +要討論的問題就是: +파이썬은 강력한 기능을 지닌 범용 컴퓨터 프로그래밍 언어다. + diff --git a/lib-python/2.7/test/cjkencodings/gb18030.txt b/lib-python/2.7/test/cjkencodings/gb18030.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/gb18030.txt @@ -0,0 +1,15 @@ +Python��ɭ��һ�ֹ��ǿ��Ƶ�ͨ��ͼ��ԣ� +�Ѿ��ʮ��ķ�չ��ʷ��ȶ��Ծ��зǳ��ݶ�� +��﷨�ص㣬�ʺ��ɸ��ָ߲��񣬼��еĲ��ϵͳ�� +��С��Լ򵥶�ǿ��ʺϸ��ʿѧϰʹ�á�Ŀǰ�� +��Ե��ؼ��ڷ��ٵķ�չ��û��ص��Դ�ǳ��ࡣ +�� Python ��ʹ�ü��е� C library? +��YӍ�Ƽ��ٰlչ�Ľ��, �_�l��yԇܛ�w��ٶ��ǲ��ݺ�ҕ�� +�n�}. ��ӿ��_�l��yԇ��ٶ�, �҂��㳣ϣ��һЩ��_�l�õ� +library, �K��һ�� fast prototyping �� programming language �� +��ʹ��. Ŀǰ��S�S�� library �� C ��, �� Python ��һ�� +fast prototyping �� programming language. ��҂�ϣ��܌��е� +C library �õ� Python �ĭh��Мyԇ��. ��ҪҲ��҂�� +ҪӑՓ�Ć��}��: +�5�1�3�3�2�1�3�1 �7�6�0�4�6�3 �8�5�8�6�3�5 �3�1�9�5 �0�9�3�0 �4�3�5�7�5�5 �5�5�0�9�8�9�9�3�0�4 �2�9�2�5�9�9. + diff --git a/lib-python/2.7/test/cjkencodings/gb2312-utf8.txt b/lib-python/2.7/test/cjkencodings/gb2312-utf8.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/gb2312-utf8.txt @@ -0,0 +1,6 @@ +Python（派森）语言是一种功能强大而完善的通用型计算机程序设计语言， +已经具有十多年的发展历史，成熟且稳定。这种语言具有非常简捷而清晰 +的语法特点，适合完成各种高层任务，几乎可以在所有的操作系统中 +运行。这种语言简单而强大，适合各种人士学习使用。目前，基于这 +种语言的相关技术正在飞速的发展，用户数量急剧扩大，相关的资源非常多。 + diff --git a/lib-python/2.7/test/cjkencodings/gb2312.txt b/lib-python/2.7/test/cjkencodings/gb2312.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/gb2312.txt @@ -0,0 +1,6 @@ +Python��ɭ��һ�ֹ��ǿ��Ƶ�ͨ��ͼ��ԣ� +�Ѿ��ʮ��ķ�չ��ʷ��ȶ��Ծ��зǳ��ݶ�� +��﷨�ص㣬�ʺ��ɸ��ָ߲��񣬼��еĲ��ϵͳ�� +��С��Լ򵥶�ǿ��ʺϸ��ʿѧϰʹ�á�Ŀǰ�� +��Ե��ؼ��ڷ��ٵķ�չ��û��ص��Դ�ǳ��ࡣ + diff --git a/lib-python/2.7/test/cjkencodings/gbk-utf8.txt b/lib-python/2.7/test/cjkencodings/gbk-utf8.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/gbk-utf8.txt @@ -0,0 +1,14 @@ +Python（派森）语言是一种功能强大而完善的通用型计算机程序设计语言， +已经具有十多年的发展历史，成熟且稳定。这种语言具有非常简捷而清晰 +的语法特点，适合完成各种高层任务，几乎可以在所有的操作系统中 +运行。这种语言简单而强大，适合各种人士学习使用。目前，基于这 +种语言的相关技术正在飞速的发展，用户数量急剧扩大，相关的资源非常多。 +如何在 Python 中使用既有的 C library? +　在資訊科技快速發展的今天, 開發及測試軟體的速度是不容忽視的 +課題. 為加快開發及測試的速度, 我們便常希望能利用一些已開發好的 +library, 並有一個 fast prototyping 的 programming language 可 +供使用. 目前有許許多多的 library 是以 C 寫成, 而 Python 是一個 +fast prototyping 的 programming language. 故我們希望能將既有的 +C library 拿到 Python 的環境中測試及整合. 其中最主要也是我們所 +要討論的問題就是: + diff --git a/lib-python/2.7/test/cjkencodings/gbk.txt b/lib-python/2.7/test/cjkencodings/gbk.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/gbk.txt @@ -0,0 +1,14 @@ +Python��ɭ��һ�ֹ��ǿ��Ƶ�ͨ��ͼ��ԣ� +�Ѿ��ʮ��ķ�չ��ʷ��ȶ��Ծ��зǳ��ݶ�� +��﷨�ص㣬�ʺ��ɸ��ָ߲��񣬼��еĲ��ϵͳ�� +��С��Լ򵥶�ǿ��ʺϸ��ʿѧϰʹ�á�Ŀǰ�� +��Ե��ؼ��ڷ��ٵķ�չ��û��ص��Դ�ǳ��ࡣ +�� Python ��ʹ�ü��е� C library? +��YӍ�Ƽ��ٰlչ�Ľ��, �_�l��yԇܛ�w��ٶ��ǲ��ݺ�ҕ�� +�n�}. ��ӿ��_�l��yԇ��ٶ�, �҂��㳣ϣ��һЩ��_�l�õ� +library, �K��һ�� fast prototyping �� programming language �� +��ʹ��. Ŀǰ��S�S�� library �� C ��, �� Python ��һ�� +fast prototyping �� programming language. ��҂�ϣ��܌��е� +C library �õ� Python �ĭh��Мyԇ��. ��ҪҲ��҂�� +ҪӑՓ�Ć��}��: + diff --git a/lib-python/2.7/test/cjkencodings/hz-utf8.txt b/lib-python/2.7/test/cjkencodings/hz-utf8.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/hz-utf8.txt @@ -0,0 +1,2 @@ +This sentence is in ASCII. +The next sentence is in GB.己所不欲，勿施於人。Bye. diff --git a/lib-python/2.7/test/cjkencodings/hz.txt b/lib-python/2.7/test/cjkencodings/hz.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/hz.txt @@ -0,0 +1,2 @@ +This sentence is in ASCII. +The next sentence is in GB.~{<:Ky2;S{#,NpJ)l6HK!#~}Bye. diff --git a/lib-python/2.7/test/cjkencodings/johab-utf8.txt b/lib-python/2.7/test/cjkencodings/johab-utf8.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/johab-utf8.txt @@ -0,0 +1,9 @@ +똠방각하 펲시콜라 + +㉯㉯납!! 因九月패믤릔궈 ⓡⓖ훀¿¿¿ 긍뒙 ⓔ뎨 ㉯. . +亞영ⓔ능횹 . . . . 서울뤄 뎐학乙家훀 ! ! !ㅠ.ㅠ +흐흐흐 ㄱㄱㄱ☆ㅠ_ㅠ 어릨 탸콰긐 뎌응 칑九들乙㉯드긐 +설릌 家훀 . . . . 굴애쉌 ⓔ궈 ⓡ릘㉱긐 因仁川女中까즼 +와쒀훀 ! ! 亞영ⓔ 家능궈 ☆上관 없능궈능 亞능뒈훀 글애듴 +ⓡ려듀九 싀풔숴훀 어릨 因仁川女中싁⑨들앜!! ㉯㉯납♡ ⌒⌒* + diff --git a/lib-python/2.7/test/cjkencodings/johab.txt b/lib-python/2.7/test/cjkencodings/johab.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/johab.txt @@ -0,0 +1,9 @@ +��w�b�a �\��ũ�a + +��s!! �g��Ú�� zٯٯٯ �w�� ѕ� ��. . +�<�w�ѓw�s . . . . �ᶉ�� e�b�� ;�z ! ! !�A.�A +�a�a�a �A�A�A�i�A_�A �៚ ȡ��z �a�w ×✗i�� a�z +��z �;�z . . . . �� ъ� �ޟ��‹z �g�b�I��a�� +��z ! ! �<�w�� ;�w�� i꾉� ��w��w �<�w��z �i��z +�ޝa�A� ��Ρ��z �៚ �g�b�I��鯂��i�z!! ��sٽ �b�b* + diff --git a/lib-python/2.7/test/cjkencodings/shift_jis-utf8.txt b/lib-python/2.7/test/cjkencodings/shift_jis-utf8.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/shift_jis-utf8.txt @@ -0,0 +1,7 @@ +Python の開発は、1990 年ごろから開始されています。 +開発者の Guido van Rossum は教育用のプログラミング言語「ABC」の開発に参加していましたが、ABC は実用上の目的にはあまり適していませんでした。 +このため、Guido はより実用的なプログラミング言語の開発を開始し、英国 BBS 放送のコメディ番組「モンティパイソン」のファンである Guido はこの言語を「Python」と名づけました。 +このような背景から生まれた Python の言語設計は、「シンプル」で「習得が容易」という目標に重点が置かれています。 +多くのスクリプト系言語ではユーザの目先の利便性を優先して色々な機能を言語要素として取り入れる場合が多いのですが、Python ではそういった小細工が追加されることはあまりありません。 +言語自体の機能は最小限に押さえ、必要な機能は拡張モジュールとして追加する、というのが Python のポリシーです。 + diff --git a/lib-python/2.7/test/cjkencodings/shift_jis.txt b/lib-python/2.7/test/cjkencodings/shift_jis.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/shift_jis.txt @@ -0,0 +1,7 @@ +Python �̊J��́A1990 �N��납��J�n��Ă��܂��B +�J��҂� Guido van Rossum �͋��p�̃v��O��~��O��uABC�v�̊J��ɎQ��Ă��܂��AABC �͎��p��̖ړI�ɂ͂��܂�K��Ă��܂��ł��B +��̂��߁AGuido �͂��p�I�ȃv��O��~��O��̊J��J�n��A�p�� BBS ��̃R��f�B�ԑg�u��e�B �p�C�\��v�̃t�@��ł�� Guido �͂��̌��uPython�v�Ɩ��Â��܂��B +��̂悤�Ȕw�i��琶�܂ꂽ Python �̌��݌v�́A�u�V��v��v�Łu�K��e�Ձv�Ƃ��ڕW�ɏd�_��u��Ă��܂��B +��̃X�N��v�g�n��ł̓��[�U�̖ڐ�̗��֐��D�悵�ĐF�X�ȋ@�\��v�f�Ƃ��Ď��ꍇ��̂ł��APython �ł͂��׍H��ǉ��邱�Ƃ͂��܂肠��܂��B +��ꎩ�̂̋@�\�͍ŏ��ɉ��A�K�v�ȋ@�\�͊g��W��[��Ƃ��Ēǉ��A�Ƃ��̂� Python �̃|��V�[�ł��B + diff --git a/lib-python/2.7/test/cjkencodings/shift_jisx0213-utf8.txt b/lib-python/2.7/test/cjkencodings/shift_jisx0213-utf8.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/shift_jisx0213-utf8.txt @@ -0,0 +1,8 @@ +Python の開発は、1990 年ごろから開始されています。 +開発者の Guido van Rossum は教育用のプログラミング言語「ABC」の開発に参加していましたが、ABC は実用上の目的にはあまり適していませんでした。 +このため、Guido はより実用的なプログラミング言語の開発を開始し、英国 BBS 放送のコメディ番組「モンティパイソン」のファンである Guido はこの言語を「Python」と名づけました。 +このような背景から生まれた Python の言語設計は、「シンプル」で「習得が容易」という目標に重点が置かれています。 +多くのスクリプト系言語ではユーザの目先の利便性を優先して色々な機能を言語要素として取り入れる場合が多いのですが、Python ではそういった小細工が追加されることはあまりありません。 +言語自体の機能は最小限に押さえ、必要な機能は拡張モジュールとして追加する、というのが Python のポリシーです。 + +ノか゚ト゚トキ喝塀𡚴𪎌麀齁𩛰 diff --git a/lib-python/2.7/test/cjkencodings/shift_jisx0213.txt b/lib-python/2.7/test/cjkencodings/shift_jisx0213.txt new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/cjkencodings/shift_jisx0213.txt @@ -0,0 +1,8 @@ +Python �̊J��́A1990 �N��납��J�n��Ă��܂��B +�J��҂� Guido van Rossum �͋��p�̃v��O��~��O��uABC�v�̊J��ɎQ��Ă��܂��AABC �͎��p��̖ړI�ɂ͂��܂�K��Ă��܂��ł��B +��̂��߁AGuido �͂��p�I�ȃv��O��~��O��̊J��J�n��A�p�� BBS ��̃R��f�B�ԑg�u��e�B �p�C�\��v�̃t�@��ł�� Guido �͂��̌��uPython�v�Ɩ��Â��܂��B +��̂悤�Ȕw�i��琶�܂ꂽ Python �̌��݌v�́A�u�V��v��v�Łu�K��e�Ձv�Ƃ��ڕW�ɏd�_��u��Ă��܂��B +��̃X�N��v�g�n��ł̓��[�U�̖ڐ�̗��֐��D�悵�ĐF�X�ȋ@�\��v�f�Ƃ��Ď��ꍇ��̂ł��APython �ł͂��׍H��ǉ��邱�Ƃ͂��܂肠��܂��B +��ꎩ�̂̋@�\�͍ŏ��ɉ��A�K�v�ȋ@�\�͊g��W��[��Ƃ��Ēǉ��A�Ƃ��̂� Python �̃|��V�[�ł��B + +�m�� g�L�K�y �� diff --git a/lib-python/2.7/test/cjkencodings_test.py b/lib-python/2.7/test/cjkencodings_test.py deleted file mode 100644 --- a/lib-python/2.7/test/cjkencodings_test.py +++ /dev/null @@ -1,1019 +0,0 @@ -teststring = { -'big5': ( -"\xa6\x70\xa6\xf3\xa6\x62\x20\x50\x79\x74\x68\x6f\x6e\x20\xa4\xa4" -"\xa8\xcf\xa5\xce\xac\x4a\xa6\xb3\xaa\xba\x20\x43\x20\x6c\x69\x62" -"\x72\x61\x72\x79\x3f\x0a\xa1\x40\xa6\x62\xb8\xea\xb0\x54\xac\xec" -"\xa7\xde\xa7\xd6\xb3\x74\xb5\x6f\xae\x69\xaa\xba\xa4\xb5\xa4\xd1" -"\x2c\x20\xb6\x7d\xb5\x6f\xa4\xce\xb4\xfa\xb8\xd5\xb3\x6e\xc5\xe9" -"\xaa\xba\xb3\x74\xab\xd7\xac\x4f\xa4\xa3\xae\x65\xa9\xbf\xb5\xf8" -"\xaa\xba\x0a\xbd\xd2\xc3\x44\x2e\x20\xac\xb0\xa5\x5b\xa7\xd6\xb6" -"\x7d\xb5\x6f\xa4\xce\xb4\xfa\xb8\xd5\xaa\xba\xb3\x74\xab\xd7\x2c" -"\x20\xa7\xda\xad\xcc\xab\x4b\xb1\x60\xa7\xc6\xb1\xe6\xaf\xe0\xa7" -"\x51\xa5\xce\xa4\x40\xa8\xc7\xa4\x77\xb6\x7d\xb5\x6f\xa6\x6e\xaa" -"\xba\x0a\x6c\x69\x62\x72\x61\x72\x79\x2c\x20\xa8\xc3\xa6\xb3\xa4" -"\x40\xad\xd3\x20\x66\x61\x73\x74\x20\x70\x72\x6f\x74\x6f\x74\x79" -"\x70\x69\x6e\x67\x20\xaa\xba\x20\x70\x72\x6f\x67\x72\x61\x6d\x6d" -"\x69\x6e\x67\x20\x6c\x61\x6e\x67\x75\x61\x67\x65\x20\xa5\x69\x0a" -"\xa8\xd1\xa8\xcf\xa5\xce\x2e\x20\xa5\xd8\xab\x65\xa6\xb3\xb3\x5c" -"\xb3\x5c\xa6\x68\xa6\x68\xaa\xba\x20\x6c\x69\x62\x72\x61\x72\x79" -"\x20\xac\x4f\xa5\x48\x20\x43\x20\xbc\x67\xa6\xa8\x2c\x20\xa6\xd3" -"\x20\x50\x79\x74\x68\x6f\x6e\x20\xac\x4f\xa4\x40\xad\xd3\x0a\x66" -"\x61\x73\x74\x20\x70\x72\x6f\x74\x6f\x74\x79\x70\x69\x6e\x67\x20" -"\xaa\xba\x20\x70\x72\x6f\x67\x72\x61\x6d\x6d\x69\x6e\x67\x20\x6c" -"\x61\x6e\x67\x75\x61\x67\x65\x2e\x20\xac\x47\xa7\xda\xad\xcc\xa7" -"\xc6\xb1\xe6\xaf\xe0\xb1\x4e\xac\x4a\xa6\xb3\xaa\xba\x0a\x43\x20" -"\x6c\x69\x62\x72\x61\x72\x79\x20\xae\xb3\xa8\xec\x20\x50\x79\x74" -"\x68\x6f\x6e\x20\xaa\xba\xc0\xf4\xb9\xd2\xa4\xa4\xb4\xfa\xb8\xd5" -"\xa4\xce\xbe\xe3\xa6\x58\x2e\x20\xa8\xe4\xa4\xa4\xb3\xcc\xa5\x44" -"\xad\x6e\xa4\x5d\xac\x4f\xa7\xda\xad\xcc\xa9\xd2\x0a\xad\x6e\xb0" -"\x51\xbd\xd7\xaa\xba\xb0\xdd\xc3\x44\xb4\x4e\xac\x4f\x3a\x0a\x0a", -"\xe5\xa6\x82\xe4\xbd\x95\xe5\x9c\xa8\x20\x50\x79\x74\x68\x6f\x6e" -"\x20\xe4\xb8\xad\xe4\xbd\xbf\xe7\x94\xa8\xe6\x97\xa2\xe6\x9c\x89" -"\xe7\x9a\x84\x20\x43\x20\x6c\x69\x62\x72\x61\x72\x79\x3f\x0a\xe3" -"\x80\x80\xe5\x9c\xa8\xe8\xb3\x87\xe8\xa8\x8a\xe7\xa7\x91\xe6\x8a" -"\x80\xe5\xbf\xab\xe9\x80\x9f\xe7\x99\xbc\xe5\xb1\x95\xe7\x9a\x84" -"\xe4\xbb\x8a\xe5\xa4\xa9\x2c\x20\xe9\x96\x8b\xe7\x99\xbc\xe5\x8f" -"\x8a\xe6\xb8\xac\xe8\xa9\xa6\xe8\xbb\x9f\xe9\xab\x94\xe7\x9a\x84" -"\xe9\x80\x9f\xe5\xba\xa6\xe6\x98\xaf\xe4\xb8\x8d\xe5\xae\xb9\xe5" -"\xbf\xbd\xe8\xa6\x96\xe7\x9a\x84\x0a\xe8\xaa\xb2\xe9\xa1\x8c\x2e" -"\x20\xe7\x82\xba\xe5\x8a\xa0\xe5\xbf\xab\xe9\x96\x8b\xe7\x99\xbc" -"\xe5\x8f\x8a\xe6\xb8\xac\xe8\xa9\xa6\xe7\x9a\x84\xe9\x80\x9f\xe5" -"\xba\xa6\x2c\x20\xe6\x88\x91\xe5\x80\x91\xe4\xbe\xbf\xe5\xb8\xb8" -"\xe5\xb8\x8c\xe6\x9c\x9b\xe8\x83\xbd\xe5\x88\xa9\xe7\x94\xa8\xe4" -"\xb8\x80\xe4\xba\x9b\xe5\xb7\xb2\xe9\x96\x8b\xe7\x99\xbc\xe5\xa5" -"\xbd\xe7\x9a\x84\x0a\x6c\x69\x62\x72\x61\x72\x79\x2c\x20\xe4\xb8" -"\xa6\xe6\x9c\x89\xe4\xb8\x80\xe5\x80\x8b\x20\x66\x61\x73\x74\x20" -"\x70\x72\x6f\x74\x6f\x74\x79\x70\x69\x6e\x67\x20\xe7\x9a\x84\x20" -"\x70\x72\x6f\x67\x72\x61\x6d\x6d\x69\x6e\x67\x20\x6c\x61\x6e\x67" -"\x75\x61\x67\x65\x20\xe5\x8f\xaf\x0a\xe4\xbe\x9b\xe4\xbd\xbf\xe7" -"\x94\xa8\x2e\x20\xe7\x9b\xae\xe5\x89\x8d\xe6\x9c\x89\xe8\xa8\xb1" -"\xe8\xa8\xb1\xe5\xa4\x9a\xe5\xa4\x9a\xe7\x9a\x84\x20\x6c\x69\x62" -"\x72\x61\x72\x79\x20\xe6\x98\xaf\xe4\xbb\xa5\x20\x43\x20\xe5\xaf" -"\xab\xe6\x88\x90\x2c\x20\xe8\x80\x8c\x20\x50\x79\x74\x68\x6f\x6e" -"\x20\xe6\x98\xaf\xe4\xb8\x80\xe5\x80\x8b\x0a\x66\x61\x73\x74\x20" -"\x70\x72\x6f\x74\x6f\x74\x79\x70\x69\x6e\x67\x20\xe7\x9a\x84\x20" -"\x70\x72\x6f\x67\x72\x61\x6d\x6d\x69\x6e\x67\x20\x6c\x61\x6e\x67" -"\x75\x61\x67\x65\x2e\x20\xe6\x95\x85\xe6\x88\x91\xe5\x80\x91\xe5" -"\xb8\x8c\xe6\x9c\x9b\xe8\x83\xbd\xe5\xb0\x87\xe6\x97\xa2\xe6\x9c" -"\x89\xe7\x9a\x84\x0a\x43\x20\x6c\x69\x62\x72\x61\x72\x79\x20\xe6" -"\x8b\xbf\xe5\x88\xb0\x20\x50\x79\x74\x68\x6f\x6e\x20\xe7\x9a\x84" -"\xe7\x92\xb0\xe5\xa2\x83\xe4\xb8\xad\xe6\xb8\xac\xe8\xa9\xa6\xe5" -"\x8f\x8a\xe6\x95\xb4\xe5\x90\x88\x2e\x20\xe5\x85\xb6\xe4\xb8\xad" -"\xe6\x9c\x80\xe4\xb8\xbb\xe8\xa6\x81\xe4\xb9\x9f\xe6\x98\xaf\xe6" -"\x88\x91\xe5\x80\x91\xe6\x89\x80\x0a\xe8\xa6\x81\xe8\xa8\x8e\xe8" -"\xab\x96\xe7\x9a\x84\xe5\x95\x8f\xe9\xa1\x8c\xe5\xb0\xb1\xe6\x98" -"\xaf\x3a\x0a\x0a"), -'big5hkscs': ( -"\x88\x45\x88\x5c\x8a\x73\x8b\xda\x8d\xd8\x0a\x88\x66\x88\x62\x88" -"\xa7\x20\x88\xa7\x88\xa3\x0a", -"\xf0\xa0\x84\x8c\xc4\x9a\xe9\xb5\xae\xe7\xbd\x93\xe6\xb4\x86\x0a" -"\xc3\x8a\xc3\x8a\xcc\x84\xc3\xaa\x20\xc3\xaa\xc3\xaa\xcc\x84\x0a"), -'cp949': ( -"\x8c\x63\xb9\xe6\xb0\xa2\xc7\xcf\x20\xbc\x84\xbd\xc3\xc4\xdd\xb6" -"\xf3\x0a\x0a\xa8\xc0\xa8\xc0\xb3\xb3\x21\x21\x20\xec\xd7\xce\xfa" -"\xea\xc5\xc6\xd0\x92\xe6\x90\x70\xb1\xc5\x20\xa8\xde\xa8\xd3\xc4" -"\x52\xa2\xaf\xa2\xaf\xa2\xaf\x20\xb1\xe0\x8a\x96\x20\xa8\xd1\xb5" -"\xb3\x20\xa8\xc0\x2e\x20\x2e\x0a\xe4\xac\xbf\xb5\xa8\xd1\xb4\xc9" -"\xc8\xc2\x20\x2e\x20\x2e\x20\x2e\x20\x2e\x20\xbc\xad\xbf\xef\xb7" -"\xef\x20\xb5\xaf\xc7\xd0\xeb\xe0\x20\xca\xab\xc4\x52\x20\x21\x20" -"\x21\x20\x21\xa4\xd0\x2e\xa4\xd0\x0a\xc8\xe5\xc8\xe5\xc8\xe5\x20" -"\xa4\xa1\xa4\xa1\xa4\xa1\xa1\xd9\xa4\xd0\x5f\xa4\xd0\x20\xbe\xee" -"\x90\x8a\x20\xc5\xcb\xc4\xe2\x83\x4f\x20\xb5\xae\xc0\xc0\x20\xaf" -"\x68\xce\xfa\xb5\xe9\xeb\xe0\x20\xa8\xc0\xb5\xe5\x83\x4f\x0a\xbc" -"\xb3\x90\x6a\x20\xca\xab\xc4\x52\x20\x2e\x20\x2e\x20\x2e\x20\x2e" -"\x20\xb1\xbc\xbe\xd6\x9a\x66\x20\xa8\xd1\xb1\xc5\x20\xa8\xde\x90" -"\x74\xa8\xc2\x83\x4f\x20\xec\xd7\xec\xd2\xf4\xb9\xe5\xfc\xf1\xe9" -"\xb1\xee\xa3\x8e\x0a\xbf\xcd\xbe\xac\xc4\x52\x20\x21\x20\x21\x20" -"\xe4\xac\xbf\xb5\xa8\xd1\x20\xca\xab\xb4\xc9\xb1\xc5\x20\xa1\xd9" -"\xdf\xbe\xb0\xfc\x20\xbe\xf8\xb4\xc9\xb1\xc5\xb4\xc9\x20\xe4\xac" -"\xb4\xc9\xb5\xd8\xc4\x52\x20\xb1\xdb\xbe\xd6\x8a\xdb\x0a\xa8\xde" -"\xb7\xc1\xb5\xe0\xce\xfa\x20\x9a\xc3\xc7\xb4\xbd\xa4\xc4\x52\x20" -"\xbe\xee\x90\x8a\x20\xec\xd7\xec\xd2\xf4\xb9\xe5\xfc\xf1\xe9\x9a" -"\xc4\xa8\xef\xb5\xe9\x9d\xda\x21\x21\x20\xa8\xc0\xa8\xc0\xb3\xb3" -"\xa2\xbd\x20\xa1\xd2\xa1\xd2\x2a\x0a\x0a", -"\xeb\x98\xa0\xeb\xb0\xa9\xea\xb0\x81\xed\x95\x98\x20\xed\x8e\xb2" -"\xec\x8b\x9c\xec\xbd\x9c\xeb\x9d\xbc\x0a\x0a\xe3\x89\xaf\xe3\x89" -"\xaf\xeb\x82\xa9\x21\x21\x20\xe5\x9b\xa0\xe4\xb9\x9d\xe6\x9c\x88" -"\xed\x8c\xa8\xeb\xaf\xa4\xeb\xa6\x94\xea\xb6\x88\x20\xe2\x93\xa1" -"\xe2\x93\x96\xed\x9b\x80\xc2\xbf\xc2\xbf\xc2\xbf\x20\xea\xb8\x8d" -"\xeb\x92\x99\x20\xe2\x93\x94\xeb\x8e\xa8\x20\xe3\x89\xaf\x2e\x20" -"\x2e\x0a\xe4\xba\x9e\xec\x98\x81\xe2\x93\x94\xeb\x8a\xa5\xed\x9a" -"\xb9\x20\x2e\x20\x2e\x20\x2e\x20\x2e\x20\xec\x84\x9c\xec\x9a\xb8" -"\xeb\xa4\x84\x20\xeb\x8e\x90\xed\x95\x99\xe4\xb9\x99\x20\xe5\xae" -"\xb6\xed\x9b\x80\x20\x21\x20\x21\x20\x21\xe3\x85\xa0\x2e\xe3\x85" -"\xa0\x0a\xed\x9d\x90\xed\x9d\x90\xed\x9d\x90\x20\xe3\x84\xb1\xe3" -"\x84\xb1\xe3\x84\xb1\xe2\x98\x86\xe3\x85\xa0\x5f\xe3\x85\xa0\x20" -"\xec\x96\xb4\xeb\xa6\xa8\x20\xed\x83\xb8\xec\xbd\xb0\xea\xb8\x90" -"\x20\xeb\x8e\x8c\xec\x9d\x91\x20\xec\xb9\x91\xe4\xb9\x9d\xeb\x93" -"\xa4\xe4\xb9\x99\x20\xe3\x89\xaf\xeb\x93\x9c\xea\xb8\x90\x0a\xec" -"\x84\xa4\xeb\xa6\x8c\x20\xe5\xae\xb6\xed\x9b\x80\x20\x2e\x20\x2e" -"\x20\x2e\x20\x2e\x20\xea\xb5\xb4\xec\x95\xa0\xec\x89\x8c\x20\xe2" -"\x93\x94\xea\xb6\x88\x20\xe2\x93\xa1\xeb\xa6\x98\xe3\x89\xb1\xea" -"\xb8\x90\x20\xe5\x9b\xa0\xe4\xbb\x81\xe5\xb7\x9d\xef\xa6\x81\xe4" -"\xb8\xad\xea\xb9\x8c\xec\xa6\xbc\x0a\xec\x99\x80\xec\x92\x80\xed" -"\x9b\x80\x20\x21\x20\x21\x20\xe4\xba\x9e\xec\x98\x81\xe2\x93\x94" -"\x20\xe5\xae\xb6\xeb\x8a\xa5\xea\xb6\x88\x20\xe2\x98\x86\xe4\xb8" -"\x8a\xea\xb4\x80\x20\xec\x97\x86\xeb\x8a\xa5\xea\xb6\x88\xeb\x8a" -"\xa5\x20\xe4\xba\x9e\xeb\x8a\xa5\xeb\x92\x88\xed\x9b\x80\x20\xea" -"\xb8\x80\xec\x95\xa0\xeb\x93\xb4\x0a\xe2\x93\xa1\xeb\xa0\xa4\xeb" -"\x93\x80\xe4\xb9\x9d\x20\xec\x8b\x80\xed\x92\x94\xec\x88\xb4\xed" -"\x9b\x80\x20\xec\x96\xb4\xeb\xa6\xa8\x20\xe5\x9b\xa0\xe4\xbb\x81" -"\xe5\xb7\x9d\xef\xa6\x81\xe4\xb8\xad\xec\x8b\x81\xe2\x91\xa8\xeb" -"\x93\xa4\xec\x95\x9c\x21\x21\x20\xe3\x89\xaf\xe3\x89\xaf\xeb\x82" -"\xa9\xe2\x99\xa1\x20\xe2\x8c\x92\xe2\x8c\x92\x2a\x0a\x0a"), -'euc_jisx0213': ( -"\x50\x79\x74\x68\x6f\x6e\x20\xa4\xce\xb3\xab\xc8\xaf\xa4\xcf\xa1" -"\xa2\x31\x39\x39\x30\x20\xc7\xaf\xa4\xb4\xa4\xed\xa4\xab\xa4\xe9" -"\xb3\xab\xbb\xcf\xa4\xb5\xa4\xec\xa4\xc6\xa4\xa4\xa4\xde\xa4\xb9" -"\xa1\xa3\x0a\xb3\xab\xc8\xaf\xbc\xd4\xa4\xce\x20\x47\x75\x69\x64" -"\x6f\x20\x76\x61\x6e\x20\x52\x6f\x73\x73\x75\x6d\x20\xa4\xcf\xb6" -"\xb5\xb0\xe9\xcd\xd1\xa4\xce\xa5\xd7\xa5\xed\xa5\xb0\xa5\xe9\xa5" -"\xdf\xa5\xf3\xa5\xb0\xb8\xc0\xb8\xec\xa1\xd6\x41\x42\x43\xa1\xd7" -"\xa4\xce\xb3\xab\xc8\xaf\xa4\xcb\xbb\xb2\xb2\xc3\xa4\xb7\xa4\xc6" -"\xa4\xa4\xa4\xde\xa4\xb7\xa4\xbf\xa4\xac\xa1\xa2\x41\x42\x43\x20" -"\xa4\xcf\xbc\xc2\xcd\xd1\xbe\xe5\xa4\xce\xcc\xdc\xc5\xaa\xa4\xcb" -"\xa4\xcf\xa4\xa2\xa4\xde\xa4\xea\xc5\xac\xa4\xb7\xa4\xc6\xa4\xa4" -"\xa4\xde\xa4\xbb\xa4\xf3\xa4\xc7\xa4\xb7\xa4\xbf\xa1\xa3\x0a\xa4" -"\xb3\xa4\xce\xa4\xbf\xa4\xe1\xa1\xa2\x47\x75\x69\x64\x6f\x20\xa4" -"\xcf\xa4\xe8\xa4\xea\xbc\xc2\xcd\xd1\xc5\xaa\xa4\xca\xa5\xd7\xa5" -"\xed\xa5\xb0\xa5\xe9\xa5\xdf\xa5\xf3\xa5\xb0\xb8\xc0\xb8\xec\xa4" -"\xce\xb3\xab\xc8\xaf\xa4\xf2\xb3\xab\xbb\xcf\xa4\xb7\xa1\xa2\xb1" -"\xd1\xb9\xf1\x20\x42\x42\x53\x20\xca\xfc\xc1\xf7\xa4\xce\xa5\xb3" -"\xa5\xe1\xa5\xc7\xa5\xa3\xc8\xd6\xc1\xc8\xa1\xd6\xa5\xe2\xa5\xf3" -"\xa5\xc6\xa5\xa3\x20\xa5\xd1\xa5\xa4\xa5\xbd\xa5\xf3\xa1\xd7\xa4" -"\xce\xa5\xd5\xa5\xa1\xa5\xf3\xa4\xc7\xa4\xa2\xa4\xeb\x20\x47\x75" -"\x69\x64\x6f\x20\xa4\xcf\xa4\xb3\xa4\xce\xb8\xc0\xb8\xec\xa4\xf2" -"\xa1\xd6\x50\x79\x74\x68\x6f\x6e\xa1\xd7\xa4\xc8\xcc\xbe\xa4\xc5" -"\xa4\xb1\xa4\xde\xa4\xb7\xa4\xbf\xa1\xa3\x0a\xa4\xb3\xa4\xce\xa4" -"\xe8\xa4\xa6\xa4\xca\xc7\xd8\xb7\xca\xa4\xab\xa4\xe9\xc0\xb8\xa4" -"\xde\xa4\xec\xa4\xbf\x20\x50\x79\x74\x68\x6f\x6e\x20\xa4\xce\xb8" -"\xc0\xb8\xec\xc0\xdf\xb7\xd7\xa4\xcf\xa1\xa2\xa1\xd6\xa5\xb7\xa5" -"\xf3\xa5\xd7\xa5\xeb\xa1\xd7\xa4\xc7\xa1\xd6\xbd\xac\xc6\xc0\xa4" -"\xac\xcd\xc6\xb0\xd7\xa1\xd7\xa4\xc8\xa4\xa4\xa4\xa6\xcc\xdc\xc9" -"\xb8\xa4\xcb\xbd\xc5\xc5\xc0\xa4\xac\xc3\xd6\xa4\xab\xa4\xec\xa4" -"\xc6\xa4\xa4\xa4\xde\xa4\xb9\xa1\xa3\x0a\xc2\xbf\xa4\xaf\xa4\xce" -"\xa5\xb9\xa5\xaf\xa5\xea\xa5\xd7\xa5\xc8\xb7\xcf\xb8\xc0\xb8\xec" -"\xa4\xc7\xa4\xcf\xa5\xe6\xa1\xbc\xa5\xb6\xa4\xce\xcc\xdc\xc0\xe8" -"\xa4\xce\xcd\xf8\xca\xd8\xc0\xad\xa4\xf2\xcd\xa5\xc0\xe8\xa4\xb7" -"\xa4\xc6\xbf\xa7\xa1\xb9\xa4\xca\xb5\xa1\xc7\xbd\xa4\xf2\xb8\xc0" -"\xb8\xec\xcd\xd7\xc1\xc7\xa4\xc8\xa4\xb7\xa4\xc6\xbc\xe8\xa4\xea" -"\xc6\xfe\xa4\xec\xa4\xeb\xbe\xec\xb9\xe7\xa4\xac\xc2\xbf\xa4\xa4" -"\xa4\xce\xa4\xc7\xa4\xb9\xa4\xac\xa1\xa2\x50\x79\x74\x68\x6f\x6e" -"\x20\xa4\xc7\xa4\xcf\xa4\xbd\xa4\xa6\xa4\xa4\xa4\xc3\xa4\xbf\xbe" -"\xae\xba\xd9\xb9\xa9\xa4\xac\xc4\xc9\xb2\xc3\xa4\xb5\xa4\xec\xa4" -"\xeb\xa4\xb3\xa4\xc8\xa4\xcf\xa4\xa2\xa4\xde\xa4\xea\xa4\xa2\xa4" -"\xea\xa4\xde\xa4\xbb\xa4\xf3\xa1\xa3\x0a\xb8\xc0\xb8\xec\xbc\xab" -"\xc2\xce\xa4\xce\xb5\xa1\xc7\xbd\xa4\xcf\xba\xc7\xbe\xae\xb8\xc2" -"\xa4\xcb\xb2\xa1\xa4\xb5\xa4\xa8\xa1\xa2\xc9\xac\xcd\xd7\xa4\xca" -"\xb5\xa1\xc7\xbd\xa4\xcf\xb3\xc8\xc4\xa5\xa5\xe2\xa5\xb8\xa5\xe5" -"\xa1\xbc\xa5\xeb\xa4\xc8\xa4\xb7\xa4\xc6\xc4\xc9\xb2\xc3\xa4\xb9" -"\xa4\xeb\xa1\xa2\xa4\xc8\xa4\xa4\xa4\xa6\xa4\xce\xa4\xac\x20\x50" -"\x79\x74\x68\x6f\x6e\x20\xa4\xce\xa5\xdd\xa5\xea\xa5\xb7\xa1\xbc" -"\xa4\xc7\xa4\xb9\xa1\xa3\x0a\x0a\xa5\xce\xa4\xf7\x20\xa5\xfe\x20" -"\xa5\xc8\xa5\xad\xaf\xac\xaf\xda\x20\xcf\xe3\x8f\xfe\xd8\x20\x8f" -"\xfe\xd4\x8f\xfe\xe8\x8f\xfc\xd6\x0a", -"\x50\x79\x74\x68\x6f\x6e\x20\xe3\x81\xae\xe9\x96\x8b\xe7\x99\xba" -"\xe3\x81\xaf\xe3\x80\x81\x31\x39\x39\x30\x20\xe5\xb9\xb4\xe3\x81" -"\x94\xe3\x82\x8d\xe3\x81\x8b\xe3\x82\x89\xe9\x96\x8b\xe5\xa7\x8b" -"\xe3\x81\x95\xe3\x82\x8c\xe3\x81\xa6\xe3\x81\x84\xe3\x81\xbe\xe3" -"\x81\x99\xe3\x80\x82\x0a\xe9\x96\x8b\xe7\x99\xba\xe8\x80\x85\xe3" -"\x81\xae\x20\x47\x75\x69\x64\x6f\x20\x76\x61\x6e\x20\x52\x6f\x73" -"\x73\x75\x6d\x20\xe3\x81\xaf\xe6\x95\x99\xe8\x82\xb2\xe7\x94\xa8" -"\xe3\x81\xae\xe3\x83\x97\xe3\x83\xad\xe3\x82\xb0\xe3\x83\xa9\xe3" -"\x83\x9f\xe3\x83\xb3\xe3\x82\xb0\xe8\xa8\x80\xe8\xaa\x9e\xe3\x80" -"\x8c\x41\x42\x43\xe3\x80\x8d\xe3\x81\xae\xe9\x96\x8b\xe7\x99\xba" -"\xe3\x81\xab\xe5\x8f\x82\xe5\x8a\xa0\xe3\x81\x97\xe3\x81\xa6\xe3" -"\x81\x84\xe3\x81\xbe\xe3\x81\x97\xe3\x81\x9f\xe3\x81\x8c\xe3\x80" -"\x81\x41\x42\x43\x20\xe3\x81\xaf\xe5\xae\x9f\xe7\x94\xa8\xe4\xb8" -"\x8a\xe3\x81\xae\xe7\x9b\xae\xe7\x9a\x84\xe3\x81\xab\xe3\x81\xaf" -"\xe3\x81\x82\xe3\x81\xbe\xe3\x82\x8a\xe9\x81\xa9\xe3\x81\x97\xe3" -"\x81\xa6\xe3\x81\x84\xe3\x81\xbe\xe3\x81\x9b\xe3\x82\x93\xe3\x81" -"\xa7\xe3\x81\x97\xe3\x81\x9f\xe3\x80\x82\x0a\xe3\x81\x93\xe3\x81" -"\xae\xe3\x81\x9f\xe3\x82\x81\xe3\x80\x81\x47\x75\x69\x64\x6f\x20" -"\xe3\x81\xaf\xe3\x82\x88\xe3\x82\x8a\xe5\xae\x9f\xe7\x94\xa8\xe7" -"\x9a\x84\xe3\x81\xaa\xe3\x83\x97\xe3\x83\xad\xe3\x82\xb0\xe3\x83" -"\xa9\xe3\x83\x9f\xe3\x83\xb3\xe3\x82\xb0\xe8\xa8\x80\xe8\xaa\x9e" -"\xe3\x81\xae\xe9\x96\x8b\xe7\x99\xba\xe3\x82\x92\xe9\x96\x8b\xe5" -"\xa7\x8b\xe3\x81\x97\xe3\x80\x81\xe8\x8b\xb1\xe5\x9b\xbd\x20\x42" -"\x42\x53\x20\xe6\x94\xbe\xe9\x80\x81\xe3\x81\xae\xe3\x82\xb3\xe3" -"\x83\xa1\xe3\x83\x87\xe3\x82\xa3\xe7\x95\xaa\xe7\xb5\x84\xe3\x80" -"\x8c\xe3\x83\xa2\xe3\x83\xb3\xe3\x83\x86\xe3\x82\xa3\x20\xe3\x83" -"\x91\xe3\x82\xa4\xe3\x82\xbd\xe3\x83\xb3\xe3\x80\x8d\xe3\x81\xae" -"\xe3\x83\x95\xe3\x82\xa1\xe3\x83\xb3\xe3\x81\xa7\xe3\x81\x82\xe3" -"\x82\x8b\x20\x47\x75\x69\x64\x6f\x20\xe3\x81\xaf\xe3\x81\x93\xe3" -"\x81\xae\xe8\xa8\x80\xe8\xaa\x9e\xe3\x82\x92\xe3\x80\x8c\x50\x79" -"\x74\x68\x6f\x6e\xe3\x80\x8d\xe3\x81\xa8\xe5\x90\x8d\xe3\x81\xa5" -"\xe3\x81\x91\xe3\x81\xbe\xe3\x81\x97\xe3\x81\x9f\xe3\x80\x82\x0a" -"\xe3\x81\x93\xe3\x81\xae\xe3\x82\x88\xe3\x81\x86\xe3\x81\xaa\xe8" -"\x83\x8c\xe6\x99\xaf\xe3\x81\x8b\xe3\x82\x89\xe7\x94\x9f\xe3\x81" -"\xbe\xe3\x82\x8c\xe3\x81\x9f\x20\x50\x79\x74\x68\x6f\x6e\x20\xe3" -"\x81\xae\xe8\xa8\x80\xe8\xaa\x9e\xe8\xa8\xad\xe8\xa8\x88\xe3\x81" -"\xaf\xe3\x80\x81\xe3\x80\x8c\xe3\x82\xb7\xe3\x83\xb3\xe3\x83\x97" -"\xe3\x83\xab\xe3\x80\x8d\xe3\x81\xa7\xe3\x80\x8c\xe7\xbf\x92\xe5" -"\xbe\x97\xe3\x81\x8c\xe5\xae\xb9\xe6\x98\x93\xe3\x80\x8d\xe3\x81" -"\xa8\xe3\x81\x84\xe3\x81\x86\xe7\x9b\xae\xe6\xa8\x99\xe3\x81\xab" -"\xe9\x87\x8d\xe7\x82\xb9\xe3\x81\x8c\xe7\xbd\xae\xe3\x81\x8b\xe3" -"\x82\x8c\xe3\x81\xa6\xe3\x81\x84\xe3\x81\xbe\xe3\x81\x99\xe3\x80" -"\x82\x0a\xe5\xa4\x9a\xe3\x81\x8f\xe3\x81\xae\xe3\x82\xb9\xe3\x82" -"\xaf\xe3\x83\xaa\xe3\x83\x97\xe3\x83\x88\xe7\xb3\xbb\xe8\xa8\x80" -"\xe8\xaa\x9e\xe3\x81\xa7\xe3\x81\xaf\xe3\x83\xa6\xe3\x83\xbc\xe3" -"\x82\xb6\xe3\x81\xae\xe7\x9b\xae\xe5\x85\x88\xe3\x81\xae\xe5\x88" -"\xa9\xe4\xbe\xbf\xe6\x80\xa7\xe3\x82\x92\xe5\x84\xaa\xe5\x85\x88" -"\xe3\x81\x97\xe3\x81\xa6\xe8\x89\xb2\xe3\x80\x85\xe3\x81\xaa\xe6" -"\xa9\x9f\xe8\x83\xbd\xe3\x82\x92\xe8\xa8\x80\xe8\xaa\x9e\xe8\xa6" -"\x81\xe7\xb4\xa0\xe3\x81\xa8\xe3\x81\x97\xe3\x81\xa6\xe5\x8f\x96" -"\xe3\x82\x8a\xe5\x85\xa5\xe3\x82\x8c\xe3\x82\x8b\xe5\xa0\xb4\xe5" -"\x90\x88\xe3\x81\x8c\xe5\xa4\x9a\xe3\x81\x84\xe3\x81\xae\xe3\x81" -"\xa7\xe3\x81\x99\xe3\x81\x8c\xe3\x80\x81\x50\x79\x74\x68\x6f\x6e" -"\x20\xe3\x81\xa7\xe3\x81\xaf\xe3\x81\x9d\xe3\x81\x86\xe3\x81\x84" -"\xe3\x81\xa3\xe3\x81\x9f\xe5\xb0\x8f\xe7\xb4\xb0\xe5\xb7\xa5\xe3" -"\x81\x8c\xe8\xbf\xbd\xe5\x8a\xa0\xe3\x81\x95\xe3\x82\x8c\xe3\x82" -"\x8b\xe3\x81\x93\xe3\x81\xa8\xe3\x81\xaf\xe3\x81\x82\xe3\x81\xbe" -"\xe3\x82\x8a\xe3\x81\x82\xe3\x82\x8a\xe3\x81\xbe\xe3\x81\x9b\xe3" -"\x82\x93\xe3\x80\x82\x0a\xe8\xa8\x80\xe8\xaa\x9e\xe8\x87\xaa\xe4" -"\xbd\x93\xe3\x81\xae\xe6\xa9\x9f\xe8\x83\xbd\xe3\x81\xaf\xe6\x9c" -"\x80\xe5\xb0\x8f\xe9\x99\x90\xe3\x81\xab\xe6\x8a\xbc\xe3\x81\x95" -"\xe3\x81\x88\xe3\x80\x81\xe5\xbf\x85\xe8\xa6\x81\xe3\x81\xaa\xe6" -"\xa9\x9f\xe8\x83\xbd\xe3\x81\xaf\xe6\x8b\xa1\xe5\xbc\xb5\xe3\x83" -"\xa2\xe3\x82\xb8\xe3\x83\xa5\xe3\x83\xbc\xe3\x83\xab\xe3\x81\xa8" -"\xe3\x81\x97\xe3\x81\xa6\xe8\xbf\xbd\xe5\x8a\xa0\xe3\x81\x99\xe3" -"\x82\x8b\xe3\x80\x81\xe3\x81\xa8\xe3\x81\x84\xe3\x81\x86\xe3\x81" -"\xae\xe3\x81\x8c\x20\x50\x79\x74\x68\x6f\x6e\x20\xe3\x81\xae\xe3" -"\x83\x9d\xe3\x83\xaa\xe3\x82\xb7\xe3\x83\xbc\xe3\x81\xa7\xe3\x81" -"\x99\xe3\x80\x82\x0a\x0a\xe3\x83\x8e\xe3\x81\x8b\xe3\x82\x9a\x20" -"\xe3\x83\x88\xe3\x82\x9a\x20\xe3\x83\x88\xe3\x82\xad\xef\xa8\xb6" -"\xef\xa8\xb9\x20\xf0\xa1\x9a\xb4\xf0\xaa\x8e\x8c\x20\xe9\xba\x80" -"\xe9\xbd\x81\xf0\xa9\x9b\xb0\x0a"), -'euc_jp': ( -"\x50\x79\x74\x68\x6f\x6e\x20\xa4\xce\xb3\xab\xc8\xaf\xa4\xcf\xa1" -"\xa2\x31\x39\x39\x30\x20\xc7\xaf\xa4\xb4\xa4\xed\xa4\xab\xa4\xe9" -"\xb3\xab\xbb\xcf\xa4\xb5\xa4\xec\xa4\xc6\xa4\xa4\xa4\xde\xa4\xb9" -"\xa1\xa3\x0a\xb3\xab\xc8\xaf\xbc\xd4\xa4\xce\x20\x47\x75\x69\x64" -"\x6f\x20\x76\x61\x6e\x20\x52\x6f\x73\x73\x75\x6d\x20\xa4\xcf\xb6" -"\xb5\xb0\xe9\xcd\xd1\xa4\xce\xa5\xd7\xa5\xed\xa5\xb0\xa5\xe9\xa5" -"\xdf\xa5\xf3\xa5\xb0\xb8\xc0\xb8\xec\xa1\xd6\x41\x42\x43\xa1\xd7" -"\xa4\xce\xb3\xab\xc8\xaf\xa4\xcb\xbb\xb2\xb2\xc3\xa4\xb7\xa4\xc6" -"\xa4\xa4\xa4\xde\xa4\xb7\xa4\xbf\xa4\xac\xa1\xa2\x41\x42\x43\x20" -"\xa4\xcf\xbc\xc2\xcd\xd1\xbe\xe5\xa4\xce\xcc\xdc\xc5\xaa\xa4\xcb" -"\xa4\xcf\xa4\xa2\xa4\xde\xa4\xea\xc5\xac\xa4\xb7\xa4\xc6\xa4\xa4" -"\xa4\xde\xa4\xbb\xa4\xf3\xa4\xc7\xa4\xb7\xa4\xbf\xa1\xa3\x0a\xa4" -"\xb3\xa4\xce\xa4\xbf\xa4\xe1\xa1\xa2\x47\x75\x69\x64\x6f\x20\xa4" -"\xcf\xa4\xe8\xa4\xea\xbc\xc2\xcd\xd1\xc5\xaa\xa4\xca\xa5\xd7\xa5" -"\xed\xa5\xb0\xa5\xe9\xa5\xdf\xa5\xf3\xa5\xb0\xb8\xc0\xb8\xec\xa4" -"\xce\xb3\xab\xc8\xaf\xa4\xf2\xb3\xab\xbb\xcf\xa4\xb7\xa1\xa2\xb1" -"\xd1\xb9\xf1\x20\x42\x42\x53\x20\xca\xfc\xc1\xf7\xa4\xce\xa5\xb3" -"\xa5\xe1\xa5\xc7\xa5\xa3\xc8\xd6\xc1\xc8\xa1\xd6\xa5\xe2\xa5\xf3" -"\xa5\xc6\xa5\xa3\x20\xa5\xd1\xa5\xa4\xa5\xbd\xa5\xf3\xa1\xd7\xa4" -"\xce\xa5\xd5\xa5\xa1\xa5\xf3\xa4\xc7\xa4\xa2\xa4\xeb\x20\x47\x75" -"\x69\x64\x6f\x20\xa4\xcf\xa4\xb3\xa4\xce\xb8\xc0\xb8\xec\xa4\xf2" -"\xa1\xd6\x50\x79\x74\x68\x6f\x6e\xa1\xd7\xa4\xc8\xcc\xbe\xa4\xc5" -"\xa4\xb1\xa4\xde\xa4\xb7\xa4\xbf\xa1\xa3\x0a\xa4\xb3\xa4\xce\xa4" -"\xe8\xa4\xa6\xa4\xca\xc7\xd8\xb7\xca\xa4\xab\xa4\xe9\xc0\xb8\xa4" -"\xde\xa4\xec\xa4\xbf\x20\x50\x79\x74\x68\x6f\x6e\x20\xa4\xce\xb8" -"\xc0\xb8\xec\xc0\xdf\xb7\xd7\xa4\xcf\xa1\xa2\xa1\xd6\xa5\xb7\xa5" -"\xf3\xa5\xd7\xa5\xeb\xa1\xd7\xa4\xc7\xa1\xd6\xbd\xac\xc6\xc0\xa4" -"\xac\xcd\xc6\xb0\xd7\xa1\xd7\xa4\xc8\xa4\xa4\xa4\xa6\xcc\xdc\xc9" -"\xb8\xa4\xcb\xbd\xc5\xc5\xc0\xa4\xac\xc3\xd6\xa4\xab\xa4\xec\xa4" -"\xc6\xa4\xa4\xa4\xde\xa4\xb9\xa1\xa3\x0a\xc2\xbf\xa4\xaf\xa4\xce" -"\xa5\xb9\xa5\xaf\xa5\xea\xa5\xd7\xa5\xc8\xb7\xcf\xb8\xc0\xb8\xec" -"\xa4\xc7\xa4\xcf\xa5\xe6\xa1\xbc\xa5\xb6\xa4\xce\xcc\xdc\xc0\xe8" -"\xa4\xce\xcd\xf8\xca\xd8\xc0\xad\xa4\xf2\xcd\xa5\xc0\xe8\xa4\xb7" -"\xa4\xc6\xbf\xa7\xa1\xb9\xa4\xca\xb5\xa1\xc7\xbd\xa4\xf2\xb8\xc0" -"\xb8\xec\xcd\xd7\xc1\xc7\xa4\xc8\xa4\xb7\xa4\xc6\xbc\xe8\xa4\xea" -"\xc6\xfe\xa4\xec\xa4\xeb\xbe\xec\xb9\xe7\xa4\xac\xc2\xbf\xa4\xa4" -"\xa4\xce\xa4\xc7\xa4\xb9\xa4\xac\xa1\xa2\x50\x79\x74\x68\x6f\x6e" -"\x20\xa4\xc7\xa4\xcf\xa4\xbd\xa4\xa6\xa4\xa4\xa4\xc3\xa4\xbf\xbe" -"\xae\xba\xd9\xb9\xa9\xa4\xac\xc4\xc9\xb2\xc3\xa4\xb5\xa4\xec\xa4" -"\xeb\xa4\xb3\xa4\xc8\xa4\xcf\xa4\xa2\xa4\xde\xa4\xea\xa4\xa2\xa4" -"\xea\xa4\xde\xa4\xbb\xa4\xf3\xa1\xa3\x0a\xb8\xc0\xb8\xec\xbc\xab" -"\xc2\xce\xa4\xce\xb5\xa1\xc7\xbd\xa4\xcf\xba\xc7\xbe\xae\xb8\xc2" -"\xa4\xcb\xb2\xa1\xa4\xb5\xa4\xa8\xa1\xa2\xc9\xac\xcd\xd7\xa4\xca" -"\xb5\xa1\xc7\xbd\xa4\xcf\xb3\xc8\xc4\xa5\xa5\xe2\xa5\xb8\xa5\xe5" -"\xa1\xbc\xa5\xeb\xa4\xc8\xa4\xb7\xa4\xc6\xc4\xc9\xb2\xc3\xa4\xb9" -"\xa4\xeb\xa1\xa2\xa4\xc8\xa4\xa4\xa4\xa6\xa4\xce\xa4\xac\x20\x50" -"\x79\x74\x68\x6f\x6e\x20\xa4\xce\xa5\xdd\xa5\xea\xa5\xb7\xa1\xbc" -"\xa4\xc7\xa4\xb9\xa1\xa3\x0a\x0a", -"\x50\x79\x74\x68\x6f\x6e\x20\xe3\x81\xae\xe9\x96\x8b\xe7\x99\xba" -"\xe3\x81\xaf\xe3\x80\x81\x31\x39\x39\x30\x20\xe5\xb9\xb4\xe3\x81" -"\x94\xe3\x82\x8d\xe3\x81\x8b\xe3\x82\x89\xe9\x96\x8b\xe5\xa7\x8b" -"\xe3\x81\x95\xe3\x82\x8c\xe3\x81\xa6\xe3\x81\x84\xe3\x81\xbe\xe3" -"\x81\x99\xe3\x80\x82\x0a\xe9\x96\x8b\xe7\x99\xba\xe8\x80\x85\xe3" -"\x81\xae\x20\x47\x75\x69\x64\x6f\x20\x76\x61\x6e\x20\x52\x6f\x73" -"\x73\x75\x6d\x20\xe3\x81\xaf\xe6\x95\x99\xe8\x82\xb2\xe7\x94\xa8" -"\xe3\x81\xae\xe3\x83\x97\xe3\x83\xad\xe3\x82\xb0\xe3\x83\xa9\xe3" -"\x83\x9f\xe3\x83\xb3\xe3\x82\xb0\xe8\xa8\x80\xe8\xaa\x9e\xe3\x80" -"\x8c\x41\x42\x43\xe3\x80\x8d\xe3\x81\xae\xe9\x96\x8b\xe7\x99\xba" -"\xe3\x81\xab\xe5\x8f\x82\xe5\x8a\xa0\xe3\x81\x97\xe3\x81\xa6\xe3" -"\x81\x84\xe3\x81\xbe\xe3\x81\x97\xe3\x81\x9f\xe3\x81\x8c\xe3\x80" -"\x81\x41\x42\x43\x20\xe3\x81\xaf\xe5\xae\x9f\xe7\x94\xa8\xe4\xb8" -"\x8a\xe3\x81\xae\xe7\x9b\xae\xe7\x9a\x84\xe3\x81\xab\xe3\x81\xaf" -"\xe3\x81\x82\xe3\x81\xbe\xe3\x82\x8a\xe9\x81\xa9\xe3\x81\x97\xe3" -"\x81\xa6\xe3\x81\x84\xe3\x81\xbe\xe3\x81\x9b\xe3\x82\x93\xe3\x81" -"\xa7\xe3\x81\x97\xe3\x81\x9f\xe3\x80\x82\x0a\xe3\x81\x93\xe3\x81" -"\xae\xe3\x81\x9f\xe3\x82\x81\xe3\x80\x81\x47\x75\x69\x64\x6f\x20" -"\xe3\x81\xaf\xe3\x82\x88\xe3\x82\x8a\xe5\xae\x9f\xe7\x94\xa8\xe7" -"\x9a\x84\xe3\x81\xaa\xe3\x83\x97\xe3\x83\xad\xe3\x82\xb0\xe3\x83" -"\xa9\xe3\x83\x9f\xe3\x83\xb3\xe3\x82\xb0\xe8\xa8\x80\xe8\xaa\x9e" -"\xe3\x81\xae\xe9\x96\x8b\xe7\x99\xba\xe3\x82\x92\xe9\x96\x8b\xe5" -"\xa7\x8b\xe3\x81\x97\xe3\x80\x81\xe8\x8b\xb1\xe5\x9b\xbd\x20\x42" -"\x42\x53\x20\xe6\x94\xbe\xe9\x80\x81\xe3\x81\xae\xe3\x82\xb3\xe3" -"\x83\xa1\xe3\x83\x87\xe3\x82\xa3\xe7\x95\xaa\xe7\xb5\x84\xe3\x80" -"\x8c\xe3\x83\xa2\xe3\x83\xb3\xe3\x83\x86\xe3\x82\xa3\x20\xe3\x83" -"\x91\xe3\x82\xa4\xe3\x82\xbd\xe3\x83\xb3\xe3\x80\x8d\xe3\x81\xae" -"\xe3\x83\x95\xe3\x82\xa1\xe3\x83\xb3\xe3\x81\xa7\xe3\x81\x82\xe3" -"\x82\x8b\x20\x47\x75\x69\x64\x6f\x20\xe3\x81\xaf\xe3\x81\x93\xe3" -"\x81\xae\xe8\xa8\x80\xe8\xaa\x9e\xe3\x82\x92\xe3\x80\x8c\x50\x79" -"\x74\x68\x6f\x6e\xe3\x80\x8d\xe3\x81\xa8\xe5\x90\x8d\xe3\x81\xa5" -"\xe3\x81\x91\xe3\x81\xbe\xe3\x81\x97\xe3\x81\x9f\xe3\x80\x82\x0a" -"\xe3\x81\x93\xe3\x81\xae\xe3\x82\x88\xe3\x81\x86\xe3\x81\xaa\xe8" -"\x83\x8c\xe6\x99\xaf\xe3\x81\x8b\xe3\x82\x89\xe7\x94\x9f\xe3\x81" -"\xbe\xe3\x82\x8c\xe3\x81\x9f\x20\x50\x79\x74\x68\x6f\x6e\x20\xe3" -"\x81\xae\xe8\xa8\x80\xe8\xaa\x9e\xe8\xa8\xad\xe8\xa8\x88\xe3\x81" -"\xaf\xe3\x80\x81\xe3\x80\x8c\xe3\x82\xb7\xe3\x83\xb3\xe3\x83\x97" -"\xe3\x83\xab\xe3\x80\x8d\xe3\x81\xa7\xe3\x80\x8c\xe7\xbf\x92\xe5" -"\xbe\x97\xe3\x81\x8c\xe5\xae\xb9\xe6\x98\x93\xe3\x80\x8d\xe3\x81" -"\xa8\xe3\x81\x84\xe3\x81\x86\xe7\x9b\xae\xe6\xa8\x99\xe3\x81\xab" -"\xe9\x87\x8d\xe7\x82\xb9\xe3\x81\x8c\xe7\xbd\xae\xe3\x81\x8b\xe3" -"\x82\x8c\xe3\x81\xa6\xe3\x81\x84\xe3\x81\xbe\xe3\x81\x99\xe3\x80" -"\x82\x0a\xe5\xa4\x9a\xe3\x81\x8f\xe3\x81\xae\xe3\x82\xb9\xe3\x82" -"\xaf\xe3\x83\xaa\xe3\x83\x97\xe3\x83\x88\xe7\xb3\xbb\xe8\xa8\x80" -"\xe8\xaa\x9e\xe3\x81\xa7\xe3\x81\xaf\xe3\x83\xa6\xe3\x83\xbc\xe3" -"\x82\xb6\xe3\x81\xae\xe7\x9b\xae\xe5\x85\x88\xe3\x81\xae\xe5\x88" -"\xa9\xe4\xbe\xbf\xe6\x80\xa7\xe3\x82\x92\xe5\x84\xaa\xe5\x85\x88" -"\xe3\x81\x97\xe3\x81\xa6\xe8\x89\xb2\xe3\x80\x85\xe3\x81\xaa\xe6" -"\xa9\x9f\xe8\x83\xbd\xe3\x82\x92\xe8\xa8\x80\xe8\xaa\x9e\xe8\xa6" -"\x81\xe7\xb4\xa0\xe3\x81\xa8\xe3\x81\x97\xe3\x81\xa6\xe5\x8f\x96" -"\xe3\x82\x8a\xe5\x85\xa5\xe3\x82\x8c\xe3\x82\x8b\xe5\xa0\xb4\xe5" -"\x90\x88\xe3\x81\x8c\xe5\xa4\x9a\xe3\x81\x84\xe3\x81\xae\xe3\x81" -"\xa7\xe3\x81\x99\xe3\x81\x8c\xe3\x80\x81\x50\x79\x74\x68\x6f\x6e" -"\x20\xe3\x81\xa7\xe3\x81\xaf\xe3\x81\x9d\xe3\x81\x86\xe3\x81\x84" -"\xe3\x81\xa3\xe3\x81\x9f\xe5\xb0\x8f\xe7\xb4\xb0\xe5\xb7\xa5\xe3" -"\x81\x8c\xe8\xbf\xbd\xe5\x8a\xa0\xe3\x81\x95\xe3\x82\x8c\xe3\x82" -"\x8b\xe3\x81\x93\xe3\x81\xa8\xe3\x81\xaf\xe3\x81\x82\xe3\x81\xbe" -"\xe3\x82\x8a\xe3\x81\x82\xe3\x82\x8a\xe3\x81\xbe\xe3\x81\x9b\xe3" -"\x82\x93\xe3\x80\x82\x0a\xe8\xa8\x80\xe8\xaa\x9e\xe8\x87\xaa\xe4" -"\xbd\x93\xe3\x81\xae\xe6\xa9\x9f\xe8\x83\xbd\xe3\x81\xaf\xe6\x9c" -"\x80\xe5\xb0\x8f\xe9\x99\x90\xe3\x81\xab\xe6\x8a\xbc\xe3\x81\x95" -"\xe3\x81\x88\xe3\x80\x81\xe5\xbf\x85\xe8\xa6\x81\xe3\x81\xaa\xe6" -"\xa9\x9f\xe8\x83\xbd\xe3\x81\xaf\xe6\x8b\xa1\xe5\xbc\xb5\xe3\x83" -"\xa2\xe3\x82\xb8\xe3\x83\xa5\xe3\x83\xbc\xe3\x83\xab\xe3\x81\xa8" -"\xe3\x81\x97\xe3\x81\xa6\xe8\xbf\xbd\xe5\x8a\xa0\xe3\x81\x99\xe3" -"\x82\x8b\xe3\x80\x81\xe3\x81\xa8\xe3\x81\x84\xe3\x81\x86\xe3\x81" -"\xae\xe3\x81\x8c\x20\x50\x79\x74\x68\x6f\x6e\x20\xe3\x81\xae\xe3" -"\x83\x9d\xe3\x83\xaa\xe3\x82\xb7\xe3\x83\xbc\xe3\x81\xa7\xe3\x81" -"\x99\xe3\x80\x82\x0a\x0a"), -'euc_kr': ( -"\xa1\xdd\x20\xc6\xc4\xc0\xcc\xbd\xe3\x28\x50\x79\x74\x68\x6f\x6e" -"\x29\xc0\xba\x20\xb9\xe8\xbf\xec\xb1\xe2\x20\xbd\xb1\xb0\xed\x2c" -"\x20\xb0\xad\xb7\xc2\xc7\xd1\x20\xc7\xc1\xb7\xce\xb1\xd7\xb7\xa1" -"\xb9\xd6\x20\xbe\xf0\xbe\xee\xc0\xd4\xb4\xcf\xb4\xd9\x2e\x20\xc6" -"\xc4\xc0\xcc\xbd\xe3\xc0\xba\x0a\xc8\xbf\xc0\xb2\xc0\xfb\xc0\xce" -"\x20\xb0\xed\xbc\xf6\xc1\xd8\x20\xb5\xa5\xc0\xcc\xc5\xcd\x20\xb1" -"\xb8\xc1\xb6\xbf\xcd\x20\xb0\xa3\xb4\xdc\xc7\xcf\xc1\xf6\xb8\xb8" -"\x20\xc8\xbf\xc0\xb2\xc0\xfb\xc0\xce\x20\xb0\xb4\xc3\xbc\xc1\xf6" -"\xc7\xe2\xc7\xc1\xb7\xce\xb1\xd7\xb7\xa1\xb9\xd6\xc0\xbb\x0a\xc1" -"\xf6\xbf\xf8\xc7\xd5\xb4\xcf\xb4\xd9\x2e\x20\xc6\xc4\xc0\xcc\xbd" -"\xe3\xc0\xc7\x20\xbf\xec\xbe\xc6\x28\xe9\xd0\xe4\xba\x29\xc7\xd1" -"\x20\xb9\xae\xb9\xfd\xb0\xfa\x20\xb5\xbf\xc0\xfb\x20\xc5\xb8\xc0" -"\xcc\xc7\xce\x2c\x20\xb1\xd7\xb8\xae\xb0\xed\x20\xc0\xce\xc5\xcd" -"\xc7\xc1\xb8\xae\xc6\xc3\x0a\xc8\xaf\xb0\xe6\xc0\xba\x20\xc6\xc4" -"\xc0\xcc\xbd\xe3\xc0\xbb\x20\xbd\xba\xc5\xa9\xb8\xb3\xc6\xc3\xb0" -"\xfa\x20\xbf\xa9\xb7\xaf\x20\xba\xd0\xbe\xdf\xbf\xa1\xbc\xad\xbf" -"\xcd\x20\xb4\xeb\xba\xce\xba\xd0\xc0\xc7\x20\xc7\xc3\xb7\xa7\xc6" -"\xfb\xbf\xa1\xbc\xad\xc0\xc7\x20\xba\xfc\xb8\xa5\x0a\xbe\xd6\xc7" -"\xc3\xb8\xae\xc4\xc9\xc0\xcc\xbc\xc7\x20\xb0\xb3\xb9\xdf\xc0\xbb" -"\x20\xc7\xd2\x20\xbc\xf6\x20\xc0\xd6\xb4\xc2\x20\xc0\xcc\xbb\xf3" -"\xc0\xfb\xc0\xce\x20\xbe\xf0\xbe\xee\xb7\xce\x20\xb8\xb8\xb5\xe9" -"\xbe\xee\xc1\xdd\xb4\xcf\xb4\xd9\x2e\x0a\x0a\xa1\xd9\xc3\xb9\xb0" -"\xa1\xb3\xa1\x3a\x20\xb3\xaf\xbe\xc6\xb6\xf3\x20\xa4\xd4\xa4\xb6" -"\xa4\xd0\xa4\xd4\xa4\xd4\xa4\xb6\xa4\xd0\xa4\xd4\xbe\xb1\x7e\x20" -"\xa4\xd4\xa4\xa4\xa4\xd2\xa4\xb7\xc5\xad\x21\x20\xa4\xd4\xa4\xa8" -"\xa4\xd1\xa4\xb7\xb1\xdd\xbe\xf8\xc0\xcc\x20\xc0\xfc\xa4\xd4\xa4" -"\xbe\xa4\xc8\xa4\xb2\xb4\xcf\xb4\xd9\x2e\x20\xa4\xd4\xa4\xb2\xa4" -"\xce\xa4\xaa\x2e\x20\xb1\xd7\xb7\xb1\xb0\xc5\x20\xa4\xd4\xa4\xb7" -"\xa4\xd1\xa4\xb4\xb4\xd9\x2e\x0a", -"\xe2\x97\x8e\x20\xed\x8c\x8c\xec\x9d\xb4\xec\x8d\xac\x28\x50\x79" -"\x74\x68\x6f\x6e\x29\xec\x9d\x80\x20\xeb\xb0\xb0\xec\x9a\xb0\xea" -"\xb8\xb0\x20\xec\x89\xbd\xea\xb3\xa0\x2c\x20\xea\xb0\x95\xeb\xa0" -"\xa5\xed\x95\x9c\x20\xed\x94\x84\xeb\xa1\x9c\xea\xb7\xb8\xeb\x9e" -"\x98\xeb\xb0\x8d\x20\xec\x96\xb8\xec\x96\xb4\xec\x9e\x85\xeb\x8b" -"\x88\xeb\x8b\xa4\x2e\x20\xed\x8c\x8c\xec\x9d\xb4\xec\x8d\xac\xec" -"\x9d\x80\x0a\xed\x9a\xa8\xec\x9c\xa8\xec\xa0\x81\xec\x9d\xb8\x20" -"\xea\xb3\xa0\xec\x88\x98\xec\xa4\x80\x20\xeb\x8d\xb0\xec\x9d\xb4" -"\xed\x84\xb0\x20\xea\xb5\xac\xec\xa1\xb0\xec\x99\x80\x20\xea\xb0" -"\x84\xeb\x8b\xa8\xed\x95\x98\xec\xa7\x80\xeb\xa7\x8c\x20\xed\x9a" -"\xa8\xec\x9c\xa8\xec\xa0\x81\xec\x9d\xb8\x20\xea\xb0\x9d\xec\xb2" -"\xb4\xec\xa7\x80\xed\x96\xa5\xed\x94\x84\xeb\xa1\x9c\xea\xb7\xb8" -"\xeb\x9e\x98\xeb\xb0\x8d\xec\x9d\x84\x0a\xec\xa7\x80\xec\x9b\x90" -"\xed\x95\xa9\xeb\x8b\x88\xeb\x8b\xa4\x2e\x20\xed\x8c\x8c\xec\x9d" -"\xb4\xec\x8d\xac\xec\x9d\x98\x20\xec\x9a\xb0\xec\x95\x84\x28\xe5" -"\x84\xaa\xe9\x9b\x85\x29\xed\x95\x9c\x20\xeb\xac\xb8\xeb\xb2\x95" -"\xea\xb3\xbc\x20\xeb\x8f\x99\xec\xa0\x81\x20\xed\x83\x80\xec\x9d" -"\xb4\xed\x95\x91\x2c\x20\xea\xb7\xb8\xeb\xa6\xac\xea\xb3\xa0\x20" -"\xec\x9d\xb8\xed\x84\xb0\xed\x94\x84\xeb\xa6\xac\xed\x8c\x85\x0a" -"\xed\x99\x98\xea\xb2\xbd\xec\x9d\x80\x20\xed\x8c\x8c\xec\x9d\xb4" -"\xec\x8d\xac\xec\x9d\x84\x20\xec\x8a\xa4\xed\x81\xac\xeb\xa6\xbd" -"\xed\x8c\x85\xea\xb3\xbc\x20\xec\x97\xac\xeb\x9f\xac\x20\xeb\xb6" -"\x84\xec\x95\xbc\xec\x97\x90\xec\x84\x9c\xec\x99\x80\x20\xeb\x8c" -"\x80\xeb\xb6\x80\xeb\xb6\x84\xec\x9d\x98\x20\xed\x94\x8c\xeb\x9e" -"\xab\xed\x8f\xbc\xec\x97\x90\xec\x84\x9c\xec\x9d\x98\x20\xeb\xb9" -"\xa0\xeb\xa5\xb8\x0a\xec\x95\xa0\xed\x94\x8c\xeb\xa6\xac\xec\xbc" -"\x80\xec\x9d\xb4\xec\x85\x98\x20\xea\xb0\x9c\xeb\xb0\x9c\xec\x9d" -"\x84\x20\xed\x95\xa0\x20\xec\x88\x98\x20\xec\x9e\x88\xeb\x8a\x94" -"\x20\xec\x9d\xb4\xec\x83\x81\xec\xa0\x81\xec\x9d\xb8\x20\xec\x96" -"\xb8\xec\x96\xb4\xeb\xa1\x9c\x20\xeb\xa7\x8c\xeb\x93\xa4\xec\x96" -"\xb4\xec\xa4\x8d\xeb\x8b\x88\xeb\x8b\xa4\x2e\x0a\x0a\xe2\x98\x86" -"\xec\xb2\xab\xea\xb0\x80\xeb\x81\x9d\x3a\x20\xeb\x82\xa0\xec\x95" -"\x84\xeb\x9d\xbc\x20\xec\x93\x94\xec\x93\x94\xec\x93\xa9\x7e\x20" -"\xeb\x8b\x81\xed\x81\xbc\x21\x20\xeb\x9c\xbd\xea\xb8\x88\xec\x97" -"\x86\xec\x9d\xb4\x20\xec\xa0\x84\xed\x99\xa5\xeb\x8b\x88\xeb\x8b" -"\xa4\x2e\x20\xeb\xb7\x81\x2e\x20\xea\xb7\xb8\xeb\x9f\xb0\xea\xb1" -"\xb0\x20\xec\x9d\x8e\xeb\x8b\xa4\x2e\x0a"), -'gb18030': ( -"\x50\x79\x74\x68\x6f\x6e\xa3\xa8\xc5\xc9\xc9\xad\xa3\xa9\xd3\xef" -"\xd1\xd4\xca\xc7\xd2\xbb\xd6\xd6\xb9\xa6\xc4\xdc\xc7\xbf\xb4\xf3" -"\xb6\xf8\xcd\xea\xc9\xc6\xb5\xc4\xcd\xa8\xd3\xc3\xd0\xcd\xbc\xc6" -"\xcb\xe3\xbb\xfa\xb3\xcc\xd0\xf2\xc9\xe8\xbc\xc6\xd3\xef\xd1\xd4" -"\xa3\xac\x0a\xd2\xd1\xbe\xad\xbe\xdf\xd3\xd0\xca\xae\xb6\xe0\xc4" -"\xea\xb5\xc4\xb7\xa2\xd5\xb9\xc0\xfa\xca\xb7\xa3\xac\xb3\xc9\xca" -"\xec\xc7\xd2\xce\xc8\xb6\xa8\xa1\xa3\xd5\xe2\xd6\xd6\xd3\xef\xd1" -"\xd4\xbe\xdf\xd3\xd0\xb7\xc7\xb3\xa3\xbc\xf2\xbd\xdd\xb6\xf8\xc7" -"\xe5\xce\xfa\x0a\xb5\xc4\xd3\xef\xb7\xa8\xcc\xd8\xb5\xe3\xa3\xac" -"\xca\xca\xba\xcf\xcd\xea\xb3\xc9\xb8\xf7\xd6\xd6\xb8\xdf\xb2\xe3" -"\xc8\xce\xce\xf1\xa3\xac\xbc\xb8\xba\xf5\xbf\xc9\xd2\xd4\xd4\xda" -"\xcb\xf9\xd3\xd0\xb5\xc4\xb2\xd9\xd7\xf7\xcf\xb5\xcd\xb3\xd6\xd0" -"\x0a\xd4\xcb\xd0\xd0\xa1\xa3\xd5\xe2\xd6\xd6\xd3\xef\xd1\xd4\xbc" -"\xf2\xb5\xa5\xb6\xf8\xc7\xbf\xb4\xf3\xa3\xac\xca\xca\xba\xcf\xb8" -"\xf7\xd6\xd6\xc8\xcb\xca\xbf\xd1\xa7\xcf\xb0\xca\xb9\xd3\xc3\xa1" -"\xa3\xc4\xbf\xc7\xb0\xa3\xac\xbb\xf9\xd3\xda\xd5\xe2\x0a\xd6\xd6" -"\xd3\xef\xd1\xd4\xb5\xc4\xcf\xe0\xb9\xd8\xbc\xbc\xca\xf5\xd5\xfd" -"\xd4\xda\xb7\xc9\xcb\xd9\xb5\xc4\xb7\xa2\xd5\xb9\xa3\xac\xd3\xc3" -"\xbb\xa7\xca\xfd\xc1\xbf\xbc\xb1\xbe\xe7\xc0\xa9\xb4\xf3\xa3\xac" -"\xcf\xe0\xb9\xd8\xb5\xc4\xd7\xca\xd4\xb4\xb7\xc7\xb3\xa3\xb6\xe0" -"\xa1\xa3\x0a\xc8\xe7\xba\xce\xd4\xda\x20\x50\x79\x74\x68\x6f\x6e" -"\x20\xd6\xd0\xca\xb9\xd3\xc3\xbc\xc8\xd3\xd0\xb5\xc4\x20\x43\x20" -"\x6c\x69\x62\x72\x61\x72\x79\x3f\x0a\xa1\xa1\xd4\xda\xd9\x59\xd3" -"\x8d\xbf\xc6\xbc\xbc\xbf\xec\xcb\xd9\xb0\x6c\xd5\xb9\xb5\xc4\xbd" -"\xf1\xcc\xec\x2c\x20\xe9\x5f\xb0\x6c\xbc\xb0\x9c\x79\xd4\x87\xdc" -"\x9b\xf3\x77\xb5\xc4\xcb\xd9\xb6\xc8\xca\xc7\xb2\xbb\xc8\xdd\xba" -"\xf6\xd2\x95\xb5\xc4\x0a\xd5\x6e\xee\x7d\x2e\x20\x9e\xe9\xbc\xd3" -"\xbf\xec\xe9\x5f\xb0\x6c\xbc\xb0\x9c\x79\xd4\x87\xb5\xc4\xcb\xd9" -"\xb6\xc8\x2c\x20\xce\xd2\x82\x83\xb1\xe3\xb3\xa3\xcf\xa3\xcd\xfb" -"\xc4\xdc\xc0\xfb\xd3\xc3\xd2\xbb\xd0\xa9\xd2\xd1\xe9\x5f\xb0\x6c" -"\xba\xc3\xb5\xc4\x0a\x6c\x69\x62\x72\x61\x72\x79\x2c\x20\x81\x4b" -"\xd3\xd0\xd2\xbb\x82\x80\x20\x66\x61\x73\x74\x20\x70\x72\x6f\x74" -"\x6f\x74\x79\x70\x69\x6e\x67\x20\xb5\xc4\x20\x70\x72\x6f\x67\x72" -"\x61\x6d\x6d\x69\x6e\x67\x20\x6c\x61\x6e\x67\x75\x61\x67\x65\x20" -"\xbf\xc9\x0a\xb9\xa9\xca\xb9\xd3\xc3\x2e\x20\xc4\xbf\xc7\xb0\xd3" -"\xd0\xd4\x53\xd4\x53\xb6\xe0\xb6\xe0\xb5\xc4\x20\x6c\x69\x62\x72" -"\x61\x72\x79\x20\xca\xc7\xd2\xd4\x20\x43\x20\x8c\x91\xb3\xc9\x2c" -"\x20\xb6\xf8\x20\x50\x79\x74\x68\x6f\x6e\x20\xca\xc7\xd2\xbb\x82" -"\x80\x0a\x66\x61\x73\x74\x20\x70\x72\x6f\x74\x6f\x74\x79\x70\x69" -"\x6e\x67\x20\xb5\xc4\x20\x70\x72\x6f\x67\x72\x61\x6d\x6d\x69\x6e" -"\x67\x20\x6c\x61\x6e\x67\x75\x61\x67\x65\x2e\x20\xb9\xca\xce\xd2" -"\x82\x83\xcf\xa3\xcd\xfb\xc4\xdc\x8c\xa2\xbc\xc8\xd3\xd0\xb5\xc4" -"\x0a\x43\x20\x6c\x69\x62\x72\x61\x72\x79\x20\xc4\xc3\xb5\xbd\x20" -"\x50\x79\x74\x68\x6f\x6e\x20\xb5\xc4\xad\x68\xbe\xb3\xd6\xd0\x9c" -"\x79\xd4\x87\xbc\xb0\xd5\xfb\xba\xcf\x2e\x20\xc6\xe4\xd6\xd0\xd7" -"\xee\xd6\xf7\xd2\xaa\xd2\xb2\xca\xc7\xce\xd2\x82\x83\xcb\xf9\x0a" -"\xd2\xaa\xd3\x91\xd5\x93\xb5\xc4\x86\x96\xee\x7d\xbe\xcd\xca\xc7" -"\x3a\x0a\x83\x35\xc7\x31\x83\x33\x9a\x33\x83\x32\xb1\x31\x83\x33" -"\x95\x31\x20\x82\x37\xd1\x36\x83\x30\x8c\x34\x83\x36\x84\x33\x20" -"\x82\x38\x89\x35\x82\x38\xfb\x36\x83\x33\x95\x35\x20\x83\x33\xd5" -"\x31\x82\x39\x81\x35\x20\x83\x30\xfd\x39\x83\x33\x86\x30\x20\x83" -"\x34\xdc\x33\x83\x35\xf6\x37\x83\x35\x97\x35\x20\x83\x35\xf9\x35" -"\x83\x30\x91\x39\x82\x38\x83\x39\x82\x39\xfc\x33\x83\x30\xf0\x34" -"\x20\x83\x32\xeb\x39\x83\x32\xeb\x35\x82\x39\x83\x39\x2e\x0a\x0a", -"\x50\x79\x74\x68\x6f\x6e\xef\xbc\x88\xe6\xb4\xbe\xe6\xa3\xae\xef" -"\xbc\x89\xe8\xaf\xad\xe8\xa8\x80\xe6\x98\xaf\xe4\xb8\x80\xe7\xa7" -"\x8d\xe5\x8a\x9f\xe8\x83\xbd\xe5\xbc\xba\xe5\xa4\xa7\xe8\x80\x8c" -"\xe5\xae\x8c\xe5\x96\x84\xe7\x9a\x84\xe9\x80\x9a\xe7\x94\xa8\xe5" -"\x9e\x8b\xe8\xae\xa1\xe7\xae\x97\xe6\x9c\xba\xe7\xa8\x8b\xe5\xba" -"\x8f\xe8\xae\xbe\xe8\xae\xa1\xe8\xaf\xad\xe8\xa8\x80\xef\xbc\x8c" -"\x0a\xe5\xb7\xb2\xe7\xbb\x8f\xe5\x85\xb7\xe6\x9c\x89\xe5\x8d\x81" -"\xe5\xa4\x9a\xe5\xb9\xb4\xe7\x9a\x84\xe5\x8f\x91\xe5\xb1\x95\xe5" -"\x8e\x86\xe5\x8f\xb2\xef\xbc\x8c\xe6\x88\x90\xe7\x86\x9f\xe4\xb8" -"\x94\xe7\xa8\xb3\xe5\xae\x9a\xe3\x80\x82\xe8\xbf\x99\xe7\xa7\x8d" -"\xe8\xaf\xad\xe8\xa8\x80\xe5\x85\xb7\xe6\x9c\x89\xe9\x9d\x9e\xe5" -"\xb8\xb8\xe7\xae\x80\xe6\x8d\xb7\xe8\x80\x8c\xe6\xb8\x85\xe6\x99" -"\xb0\x0a\xe7\x9a\x84\xe8\xaf\xad\xe6\xb3\x95\xe7\x89\xb9\xe7\x82" -"\xb9\xef\xbc\x8c\xe9\x80\x82\xe5\x90\x88\xe5\xae\x8c\xe6\x88\x90" -"\xe5\x90\x84\xe7\xa7\x8d\xe9\xab\x98\xe5\xb1\x82\xe4\xbb\xbb\xe5" -"\x8a\xa1\xef\xbc\x8c\xe5\x87\xa0\xe4\xb9\x8e\xe5\x8f\xaf\xe4\xbb" -"\xa5\xe5\x9c\xa8\xe6\x89\x80\xe6\x9c\x89\xe7\x9a\x84\xe6\x93\x8d" -"\xe4\xbd\x9c\xe7\xb3\xbb\xe7\xbb\x9f\xe4\xb8\xad\x0a\xe8\xbf\x90" -"\xe8\xa1\x8c\xe3\x80\x82\xe8\xbf\x99\xe7\xa7\x8d\xe8\xaf\xad\xe8" -"\xa8\x80\xe7\xae\x80\xe5\x8d\x95\xe8\x80\x8c\xe5\xbc\xba\xe5\xa4" -"\xa7\xef\xbc\x8c\xe9\x80\x82\xe5\x90\x88\xe5\x90\x84\xe7\xa7\x8d" -"\xe4\xba\xba\xe5\xa3\xab\xe5\xad\xa6\xe4\xb9\xa0\xe4\xbd\xbf\xe7" -"\x94\xa8\xe3\x80\x82\xe7\x9b\xae\xe5\x89\x8d\xef\xbc\x8c\xe5\x9f" -"\xba\xe4\xba\x8e\xe8\xbf\x99\x0a\xe7\xa7\x8d\xe8\xaf\xad\xe8\xa8" -"\x80\xe7\x9a\x84\xe7\x9b\xb8\xe5\x85\xb3\xe6\x8a\x80\xe6\x9c\xaf" -"\xe6\xad\xa3\xe5\x9c\xa8\xe9\xa3\x9e\xe9\x80\x9f\xe7\x9a\x84\xe5" -"\x8f\x91\xe5\xb1\x95\xef\xbc\x8c\xe7\x94\xa8\xe6\x88\xb7\xe6\x95" -"\xb0\xe9\x87\x8f\xe6\x80\xa5\xe5\x89\xa7\xe6\x89\xa9\xe5\xa4\xa7" -"\xef\xbc\x8c\xe7\x9b\xb8\xe5\x85\xb3\xe7\x9a\x84\xe8\xb5\x84\xe6" -"\xba\x90\xe9\x9d\x9e\xe5\xb8\xb8\xe5\xa4\x9a\xe3\x80\x82\x0a\xe5" -"\xa6\x82\xe4\xbd\x95\xe5\x9c\xa8\x20\x50\x79\x74\x68\x6f\x6e\x20" -"\xe4\xb8\xad\xe4\xbd\xbf\xe7\x94\xa8\xe6\x97\xa2\xe6\x9c\x89\xe7" -"\x9a\x84\x20\x43\x20\x6c\x69\x62\x72\x61\x72\x79\x3f\x0a\xe3\x80" -"\x80\xe5\x9c\xa8\xe8\xb3\x87\xe8\xa8\x8a\xe7\xa7\x91\xe6\x8a\x80" -"\xe5\xbf\xab\xe9\x80\x9f\xe7\x99\xbc\xe5\xb1\x95\xe7\x9a\x84\xe4" -"\xbb\x8a\xe5\xa4\xa9\x2c\x20\xe9\x96\x8b\xe7\x99\xbc\xe5\x8f\x8a" -"\xe6\xb8\xac\xe8\xa9\xa6\xe8\xbb\x9f\xe9\xab\x94\xe7\x9a\x84\xe9" -"\x80\x9f\xe5\xba\xa6\xe6\x98\xaf\xe4\xb8\x8d\xe5\xae\xb9\xe5\xbf" -"\xbd\xe8\xa6\x96\xe7\x9a\x84\x0a\xe8\xaa\xb2\xe9\xa1\x8c\x2e\x20" -"\xe7\x82\xba\xe5\x8a\xa0\xe5\xbf\xab\xe9\x96\x8b\xe7\x99\xbc\xe5" -"\x8f\x8a\xe6\xb8\xac\xe8\xa9\xa6\xe7\x9a\x84\xe9\x80\x9f\xe5\xba" -"\xa6\x2c\x20\xe6\x88\x91\xe5\x80\x91\xe4\xbe\xbf\xe5\xb8\xb8\xe5" -"\xb8\x8c\xe6\x9c\x9b\xe8\x83\xbd\xe5\x88\xa9\xe7\x94\xa8\xe4\xb8" -"\x80\xe4\xba\x9b\xe5\xb7\xb2\xe9\x96\x8b\xe7\x99\xbc\xe5\xa5\xbd" -"\xe7\x9a\x84\x0a\x6c\x69\x62\x72\x61\x72\x79\x2c\x20\xe4\xb8\xa6" -"\xe6\x9c\x89\xe4\xb8\x80\xe5\x80\x8b\x20\x66\x61\x73\x74\x20\x70" -"\x72\x6f\x74\x6f\x74\x79\x70\x69\x6e\x67\x20\xe7\x9a\x84\x20\x70" -"\x72\x6f\x67\x72\x61\x6d\x6d\x69\x6e\x67\x20\x6c\x61\x6e\x67\x75" -"\x61\x67\x65\x20\xe5\x8f\xaf\x0a\xe4\xbe\x9b\xe4\xbd\xbf\xe7\x94" -"\xa8\x2e\x20\xe7\x9b\xae\xe5\x89\x8d\xe6\x9c\x89\xe8\xa8\xb1\xe8" -"\xa8\xb1\xe5\xa4\x9a\xe5\xa4\x9a\xe7\x9a\x84\x20\x6c\x69\x62\x72" -"\x61\x72\x79\x20\xe6\x98\xaf\xe4\xbb\xa5\x20\x43\x20\xe5\xaf\xab" -"\xe6\x88\x90\x2c\x20\xe8\x80\x8c\x20\x50\x79\x74\x68\x6f\x6e\x20" -"\xe6\x98\xaf\xe4\xb8\x80\xe5\x80\x8b\x0a\x66\x61\x73\x74\x20\x70" -"\x72\x6f\x74\x6f\x74\x79\x70\x69\x6e\x67\x20\xe7\x9a\x84\x20\x70" -"\x72\x6f\x67\x72\x61\x6d\x6d\x69\x6e\x67\x20\x6c\x61\x6e\x67\x75" -"\x61\x67\x65\x2e\x20\xe6\x95\x85\xe6\x88\x91\xe5\x80\x91\xe5\xb8" -"\x8c\xe6\x9c\x9b\xe8\x83\xbd\xe5\xb0\x87\xe6\x97\xa2\xe6\x9c\x89" -"\xe7\x9a\x84\x0a\x43\x20\x6c\x69\x62\x72\x61\x72\x79\x20\xe6\x8b" -"\xbf\xe5\x88\xb0\x20\x50\x79\x74\x68\x6f\x6e\x20\xe7\x9a\x84\xe7" -"\x92\xb0\xe5\xa2\x83\xe4\xb8\xad\xe6\xb8\xac\xe8\xa9\xa6\xe5\x8f" -"\x8a\xe6\x95\xb4\xe5\x90\x88\x2e\x20\xe5\x85\xb6\xe4\xb8\xad\xe6" -"\x9c\x80\xe4\xb8\xbb\xe8\xa6\x81\xe4\xb9\x9f\xe6\x98\xaf\xe6\x88" -"\x91\xe5\x80\x91\xe6\x89\x80\x0a\xe8\xa6\x81\xe8\xa8\x8e\xe8\xab" -"\x96\xe7\x9a\x84\xe5\x95\x8f\xe9\xa1\x8c\xe5\xb0\xb1\xe6\x98\xaf" -"\x3a\x0a\xed\x8c\x8c\xec\x9d\xb4\xec\x8d\xac\xec\x9d\x80\x20\xea" -"\xb0\x95\xeb\xa0\xa5\xed\x95\x9c\x20\xea\xb8\xb0\xeb\x8a\xa5\xec" -"\x9d\x84\x20\xec\xa7\x80\xeb\x8b\x8c\x20\xeb\xb2\x94\xec\x9a\xa9" -"\x20\xec\xbb\xb4\xed\x93\xa8\xed\x84\xb0\x20\xed\x94\x84\xeb\xa1" -"\x9c\xea\xb7\xb8\xeb\x9e\x98\xeb\xb0\x8d\x20\xec\x96\xb8\xec\x96" -"\xb4\xeb\x8b\xa4\x2e\x0a\x0a"), -'gb2312': ( -"\x50\x79\x74\x68\x6f\x6e\xa3\xa8\xc5\xc9\xc9\xad\xa3\xa9\xd3\xef" -"\xd1\xd4\xca\xc7\xd2\xbb\xd6\xd6\xb9\xa6\xc4\xdc\xc7\xbf\xb4\xf3" -"\xb6\xf8\xcd\xea\xc9\xc6\xb5\xc4\xcd\xa8\xd3\xc3\xd0\xcd\xbc\xc6" -"\xcb\xe3\xbb\xfa\xb3\xcc\xd0\xf2\xc9\xe8\xbc\xc6\xd3\xef\xd1\xd4" -"\xa3\xac\x0a\xd2\xd1\xbe\xad\xbe\xdf\xd3\xd0\xca\xae\xb6\xe0\xc4" -"\xea\xb5\xc4\xb7\xa2\xd5\xb9\xc0\xfa\xca\xb7\xa3\xac\xb3\xc9\xca" -"\xec\xc7\xd2\xce\xc8\xb6\xa8\xa1\xa3\xd5\xe2\xd6\xd6\xd3\xef\xd1" -"\xd4\xbe\xdf\xd3\xd0\xb7\xc7\xb3\xa3\xbc\xf2\xbd\xdd\xb6\xf8\xc7" -"\xe5\xce\xfa\x0a\xb5\xc4\xd3\xef\xb7\xa8\xcc\xd8\xb5\xe3\xa3\xac" -"\xca\xca\xba\xcf\xcd\xea\xb3\xc9\xb8\xf7\xd6\xd6\xb8\xdf\xb2\xe3" -"\xc8\xce\xce\xf1\xa3\xac\xbc\xb8\xba\xf5\xbf\xc9\xd2\xd4\xd4\xda" -"\xcb\xf9\xd3\xd0\xb5\xc4\xb2\xd9\xd7\xf7\xcf\xb5\xcd\xb3\xd6\xd0" -"\x0a\xd4\xcb\xd0\xd0\xa1\xa3\xd5\xe2\xd6\xd6\xd3\xef\xd1\xd4\xbc" -"\xf2\xb5\xa5\xb6\xf8\xc7\xbf\xb4\xf3\xa3\xac\xca\xca\xba\xcf\xb8" -"\xf7\xd6\xd6\xc8\xcb\xca\xbf\xd1\xa7\xcf\xb0\xca\xb9\xd3\xc3\xa1" -"\xa3\xc4\xbf\xc7\xb0\xa3\xac\xbb\xf9\xd3\xda\xd5\xe2\x0a\xd6\xd6" -"\xd3\xef\xd1\xd4\xb5\xc4\xcf\xe0\xb9\xd8\xbc\xbc\xca\xf5\xd5\xfd" -"\xd4\xda\xb7\xc9\xcb\xd9\xb5\xc4\xb7\xa2\xd5\xb9\xa3\xac\xd3\xc3" -"\xbb\xa7\xca\xfd\xc1\xbf\xbc\xb1\xbe\xe7\xc0\xa9\xb4\xf3\xa3\xac" -"\xcf\xe0\xb9\xd8\xb5\xc4\xd7\xca\xd4\xb4\xb7\xc7\xb3\xa3\xb6\xe0" -"\xa1\xa3\x0a\x0a", -"\x50\x79\x74\x68\x6f\x6e\xef\xbc\x88\xe6\xb4\xbe\xe6\xa3\xae\xef" -"\xbc\x89\xe8\xaf\xad\xe8\xa8\x80\xe6\x98\xaf\xe4\xb8\x80\xe7\xa7" -"\x8d\xe5\x8a\x9f\xe8\x83\xbd\xe5\xbc\xba\xe5\xa4\xa7\xe8\x80\x8c" -"\xe5\xae\x8c\xe5\x96\x84\xe7\x9a\x84\xe9\x80\x9a\xe7\x94\xa8\xe5" -"\x9e\x8b\xe8\xae\xa1\xe7\xae\x97\xe6\x9c\xba\xe7\xa8\x8b\xe5\xba" -"\x8f\xe8\xae\xbe\xe8\xae\xa1\xe8\xaf\xad\xe8\xa8\x80\xef\xbc\x8c" -"\x0a\xe5\xb7\xb2\xe7\xbb\x8f\xe5\x85\xb7\xe6\x9c\x89\xe5\x8d\x81" -"\xe5\xa4\x9a\xe5\xb9\xb4\xe7\x9a\x84\xe5\x8f\x91\xe5\xb1\x95\xe5" -"\x8e\x86\xe5\x8f\xb2\xef\xbc\x8c\xe6\x88\x90\xe7\x86\x9f\xe4\xb8" -"\x94\xe7\xa8\xb3\xe5\xae\x9a\xe3\x80\x82\xe8\xbf\x99\xe7\xa7\x8d" -"\xe8\xaf\xad\xe8\xa8\x80\xe5\x85\xb7\xe6\x9c\x89\xe9\x9d\x9e\xe5" -"\xb8\xb8\xe7\xae\x80\xe6\x8d\xb7\xe8\x80\x8c\xe6\xb8\x85\xe6\x99" -"\xb0\x0a\xe7\x9a\x84\xe8\xaf\xad\xe6\xb3\x95\xe7\x89\xb9\xe7\x82" -"\xb9\xef\xbc\x8c\xe9\x80\x82\xe5\x90\x88\xe5\xae\x8c\xe6\x88\x90" -"\xe5\x90\x84\xe7\xa7\x8d\xe9\xab\x98\xe5\xb1\x82\xe4\xbb\xbb\xe5" -"\x8a\xa1\xef\xbc\x8c\xe5\x87\xa0\xe4\xb9\x8e\xe5\x8f\xaf\xe4\xbb" -"\xa5\xe5\x9c\xa8\xe6\x89\x80\xe6\x9c\x89\xe7\x9a\x84\xe6\x93\x8d" -"\xe4\xbd\x9c\xe7\xb3\xbb\xe7\xbb\x9f\xe4\xb8\xad\x0a\xe8\xbf\x90" -"\xe8\xa1\x8c\xe3\x80\x82\xe8\xbf\x99\xe7\xa7\x8d\xe8\xaf\xad\xe8" -"\xa8\x80\xe7\xae\x80\xe5\x8d\x95\xe8\x80\x8c\xe5\xbc\xba\xe5\xa4" -"\xa7\xef\xbc\x8c\xe9\x80\x82\xe5\x90\x88\xe5\x90\x84\xe7\xa7\x8d" -"\xe4\xba\xba\xe5\xa3\xab\xe5\xad\xa6\xe4\xb9\xa0\xe4\xbd\xbf\xe7" -"\x94\xa8\xe3\x80\x82\xe7\x9b\xae\xe5\x89\x8d\xef\xbc\x8c\xe5\x9f" -"\xba\xe4\xba\x8e\xe8\xbf\x99\x0a\xe7\xa7\x8d\xe8\xaf\xad\xe8\xa8" -"\x80\xe7\x9a\x84\xe7\x9b\xb8\xe5\x85\xb3\xe6\x8a\x80\xe6\x9c\xaf" -"\xe6\xad\xa3\xe5\x9c\xa8\xe9\xa3\x9e\xe9\x80\x9f\xe7\x9a\x84\xe5" -"\x8f\x91\xe5\xb1\x95\xef\xbc\x8c\xe7\x94\xa8\xe6\x88\xb7\xe6\x95" -"\xb0\xe9\x87\x8f\xe6\x80\xa5\xe5\x89\xa7\xe6\x89\xa9\xe5\xa4\xa7" -"\xef\xbc\x8c\xe7\x9b\xb8\xe5\x85\xb3\xe7\x9a\x84\xe8\xb5\x84\xe6" -"\xba\x90\xe9\x9d\x9e\xe5\xb8\xb8\xe5\xa4\x9a\xe3\x80\x82\x0a\x0a"), -'gbk': ( -"\x50\x79\x74\x68\x6f\x6e\xa3\xa8\xc5\xc9\xc9\xad\xa3\xa9\xd3\xef" -"\xd1\xd4\xca\xc7\xd2\xbb\xd6\xd6\xb9\xa6\xc4\xdc\xc7\xbf\xb4\xf3" -"\xb6\xf8\xcd\xea\xc9\xc6\xb5\xc4\xcd\xa8\xd3\xc3\xd0\xcd\xbc\xc6" -"\xcb\xe3\xbb\xfa\xb3\xcc\xd0\xf2\xc9\xe8\xbc\xc6\xd3\xef\xd1\xd4" -"\xa3\xac\x0a\xd2\xd1\xbe\xad\xbe\xdf\xd3\xd0\xca\xae\xb6\xe0\xc4" -"\xea\xb5\xc4\xb7\xa2\xd5\xb9\xc0\xfa\xca\xb7\xa3\xac\xb3\xc9\xca" -"\xec\xc7\xd2\xce\xc8\xb6\xa8\xa1\xa3\xd5\xe2\xd6\xd6\xd3\xef\xd1" -"\xd4\xbe\xdf\xd3\xd0\xb7\xc7\xb3\xa3\xbc\xf2\xbd\xdd\xb6\xf8\xc7" -"\xe5\xce\xfa\x0a\xb5\xc4\xd3\xef\xb7\xa8\xcc\xd8\xb5\xe3\xa3\xac" -"\xca\xca\xba\xcf\xcd\xea\xb3\xc9\xb8\xf7\xd6\xd6\xb8\xdf\xb2\xe3" -"\xc8\xce\xce\xf1\xa3\xac\xbc\xb8\xba\xf5\xbf\xc9\xd2\xd4\xd4\xda" -"\xcb\xf9\xd3\xd0\xb5\xc4\xb2\xd9\xd7\xf7\xcf\xb5\xcd\xb3\xd6\xd0" -"\x0a\xd4\xcb\xd0\xd0\xa1\xa3\xd5\xe2\xd6\xd6\xd3\xef\xd1\xd4\xbc" -"\xf2\xb5\xa5\xb6\xf8\xc7\xbf\xb4\xf3\xa3\xac\xca\xca\xba\xcf\xb8" -"\xf7\xd6\xd6\xc8\xcb\xca\xbf\xd1\xa7\xcf\xb0\xca\xb9\xd3\xc3\xa1" -"\xa3\xc4\xbf\xc7\xb0\xa3\xac\xbb\xf9\xd3\xda\xd5\xe2\x0a\xd6\xd6" -"\xd3\xef\xd1\xd4\xb5\xc4\xcf\xe0\xb9\xd8\xbc\xbc\xca\xf5\xd5\xfd" -"\xd4\xda\xb7\xc9\xcb\xd9\xb5\xc4\xb7\xa2\xd5\xb9\xa3\xac\xd3\xc3" -"\xbb\xa7\xca\xfd\xc1\xbf\xbc\xb1\xbe\xe7\xc0\xa9\xb4\xf3\xa3\xac" -"\xcf\xe0\xb9\xd8\xb5\xc4\xd7\xca\xd4\xb4\xb7\xc7\xb3\xa3\xb6\xe0" -"\xa1\xa3\x0a\xc8\xe7\xba\xce\xd4\xda\x20\x50\x79\x74\x68\x6f\x6e" -"\x20\xd6\xd0\xca\xb9\xd3\xc3\xbc\xc8\xd3\xd0\xb5\xc4\x20\x43\x20" -"\x6c\x69\x62\x72\x61\x72\x79\x3f\x0a\xa1\xa1\xd4\xda\xd9\x59\xd3" -"\x8d\xbf\xc6\xbc\xbc\xbf\xec\xcb\xd9\xb0\x6c\xd5\xb9\xb5\xc4\xbd" -"\xf1\xcc\xec\x2c\x20\xe9\x5f\xb0\x6c\xbc\xb0\x9c\x79\xd4\x87\xdc" -"\x9b\xf3\x77\xb5\xc4\xcb\xd9\xb6\xc8\xca\xc7\xb2\xbb\xc8\xdd\xba" -"\xf6\xd2\x95\xb5\xc4\x0a\xd5\x6e\xee\x7d\x2e\x20\x9e\xe9\xbc\xd3" -"\xbf\xec\xe9\x5f\xb0\x6c\xbc\xb0\x9c\x79\xd4\x87\xb5\xc4\xcb\xd9" -"\xb6\xc8\x2c\x20\xce\xd2\x82\x83\xb1\xe3\xb3\xa3\xcf\xa3\xcd\xfb" -"\xc4\xdc\xc0\xfb\xd3\xc3\xd2\xbb\xd0\xa9\xd2\xd1\xe9\x5f\xb0\x6c" -"\xba\xc3\xb5\xc4\x0a\x6c\x69\x62\x72\x61\x72\x79\x2c\x20\x81\x4b" -"\xd3\xd0\xd2\xbb\x82\x80\x20\x66\x61\x73\x74\x20\x70\x72\x6f\x74" -"\x6f\x74\x79\x70\x69\x6e\x67\x20\xb5\xc4\x20\x70\x72\x6f\x67\x72" -"\x61\x6d\x6d\x69\x6e\x67\x20\x6c\x61\x6e\x67\x75\x61\x67\x65\x20" -"\xbf\xc9\x0a\xb9\xa9\xca\xb9\xd3\xc3\x2e\x20\xc4\xbf\xc7\xb0\xd3" -"\xd0\xd4\x53\xd4\x53\xb6\xe0\xb6\xe0\xb5\xc4\x20\x6c\x69\x62\x72" -"\x61\x72\x79\x20\xca\xc7\xd2\xd4\x20\x43\x20\x8c\x91\xb3\xc9\x2c" -"\x20\xb6\xf8\x20\x50\x79\x74\x68\x6f\x6e\x20\xca\xc7\xd2\xbb\x82" -"\x80\x0a\x66\x61\x73\x74\x20\x70\x72\x6f\x74\x6f\x74\x79\x70\x69" -"\x6e\x67\x20\xb5\xc4\x20\x70\x72\x6f\x67\x72\x61\x6d\x6d\x69\x6e" -"\x67\x20\x6c\x61\x6e\x67\x75\x61\x67\x65\x2e\x20\xb9\xca\xce\xd2" -"\x82\x83\xcf\xa3\xcd\xfb\xc4\xdc\x8c\xa2\xbc\xc8\xd3\xd0\xb5\xc4" -"\x0a\x43\x20\x6c\x69\x62\x72\x61\x72\x79\x20\xc4\xc3\xb5\xbd\x20" -"\x50\x79\x74\x68\x6f\x6e\x20\xb5\xc4\xad\x68\xbe\xb3\xd6\xd0\x9c" -"\x79\xd4\x87\xbc\xb0\xd5\xfb\xba\xcf\x2e\x20\xc6\xe4\xd6\xd0\xd7" -"\xee\xd6\xf7\xd2\xaa\xd2\xb2\xca\xc7\xce\xd2\x82\x83\xcb\xf9\x0a" -"\xd2\xaa\xd3\x91\xd5\x93\xb5\xc4\x86\x96\xee\x7d\xbe\xcd\xca\xc7" -"\x3a\x0a\x0a", -"\x50\x79\x74\x68\x6f\x6e\xef\xbc\x88\xe6\xb4\xbe\xe6\xa3\xae\xef" -"\xbc\x89\xe8\xaf\xad\xe8\xa8\x80\xe6\x98\xaf\xe4\xb8\x80\xe7\xa7" -"\x8d\xe5\x8a\x9f\xe8\x83\xbd\xe5\xbc\xba\xe5\xa4\xa7\xe8\x80\x8c" -"\xe5\xae\x8c\xe5\x96\x84\xe7\x9a\x84\xe9\x80\x9a\xe7\x94\xa8\xe5" -"\x9e\x8b\xe8\xae\xa1\xe7\xae\x97\xe6\x9c\xba\xe7\xa8\x8b\xe5\xba" -"\x8f\xe8\xae\xbe\xe8\xae\xa1\xe8\xaf\xad\xe8\xa8\x80\xef\xbc\x8c" -"\x0a\xe5\xb7\xb2\xe7\xbb\x8f\xe5\x85\xb7\xe6\x9c\x89\xe5\x8d\x81" -"\xe5\xa4\x9a\xe5\xb9\xb4\xe7\x9a\x84\xe5\x8f\x91\xe5\xb1\x95\xe5" -"\x8e\x86\xe5\x8f\xb2\xef\xbc\x8c\xe6\x88\x90\xe7\x86\x9f\xe4\xb8" -"\x94\xe7\xa8\xb3\xe5\xae\x9a\xe3\x80\x82\xe8\xbf\x99\xe7\xa7\x8d" -"\xe8\xaf\xad\xe8\xa8\x80\xe5\x85\xb7\xe6\x9c\x89\xe9\x9d\x9e\xe5" -"\xb8\xb8\xe7\xae\x80\xe6\x8d\xb7\xe8\x80\x8c\xe6\xb8\x85\xe6\x99" -"\xb0\x0a\xe7\x9a\x84\xe8\xaf\xad\xe6\xb3\x95\xe7\x89\xb9\xe7\x82" -"\xb9\xef\xbc\x8c\xe9\x80\x82\xe5\x90\x88\xe5\xae\x8c\xe6\x88\x90" -"\xe5\x90\x84\xe7\xa7\x8d\xe9\xab\x98\xe5\xb1\x82\xe4\xbb\xbb\xe5" -"\x8a\xa1\xef\xbc\x8c\xe5\x87\xa0\xe4\xb9\x8e\xe5\x8f\xaf\xe4\xbb" -"\xa5\xe5\x9c\xa8\xe6\x89\x80\xe6\x9c\x89\xe7\x9a\x84\xe6\x93\x8d" -"\xe4\xbd\x9c\xe7\xb3\xbb\xe7\xbb\x9f\xe4\xb8\xad\x0a\xe8\xbf\x90" -"\xe8\xa1\x8c\xe3\x80\x82\xe8\xbf\x99\xe7\xa7\x8d\xe8\xaf\xad\xe8" -"\xa8\x80\xe7\xae\x80\xe5\x8d\x95\xe8\x80\x8c\xe5\xbc\xba\xe5\xa4" -"\xa7\xef\xbc\x8c\xe9\x80\x82\xe5\x90\x88\xe5\x90\x84\xe7\xa7\x8d" -"\xe4\xba\xba\xe5\xa3\xab\xe5\xad\xa6\xe4\xb9\xa0\xe4\xbd\xbf\xe7" -"\x94\xa8\xe3\x80\x82\xe7\x9b\xae\xe5\x89\x8d\xef\xbc\x8c\xe5\x9f" -"\xba\xe4\xba\x8e\xe8\xbf\x99\x0a\xe7\xa7\x8d\xe8\xaf\xad\xe8\xa8" -"\x80\xe7\x9a\x84\xe7\x9b\xb8\xe5\x85\xb3\xe6\x8a\x80\xe6\x9c\xaf" -"\xe6\xad\xa3\xe5\x9c\xa8\xe9\xa3\x9e\xe9\x80\x9f\xe7\x9a\x84\xe5" -"\x8f\x91\xe5\xb1\x95\xef\xbc\x8c\xe7\x94\xa8\xe6\x88\xb7\xe6\x95" -"\xb0\xe9\x87\x8f\xe6\x80\xa5\xe5\x89\xa7\xe6\x89\xa9\xe5\xa4\xa7" -"\xef\xbc\x8c\xe7\x9b\xb8\xe5\x85\xb3\xe7\x9a\x84\xe8\xb5\x84\xe6" -"\xba\x90\xe9\x9d\x9e\xe5\xb8\xb8\xe5\xa4\x9a\xe3\x80\x82\x0a\xe5" -"\xa6\x82\xe4\xbd\x95\xe5\x9c\xa8\x20\x50\x79\x74\x68\x6f\x6e\x20" -"\xe4\xb8\xad\xe4\xbd\xbf\xe7\x94\xa8\xe6\x97\xa2\xe6\x9c\x89\xe7" -"\x9a\x84\x20\x43\x20\x6c\x69\x62\x72\x61\x72\x79\x3f\x0a\xe3\x80" -"\x80\xe5\x9c\xa8\xe8\xb3\x87\xe8\xa8\x8a\xe7\xa7\x91\xe6\x8a\x80" -"\xe5\xbf\xab\xe9\x80\x9f\xe7\x99\xbc\xe5\xb1\x95\xe7\x9a\x84\xe4" -"\xbb\x8a\xe5\xa4\xa9\x2c\x20\xe9\x96\x8b\xe7\x99\xbc\xe5\x8f\x8a" -"\xe6\xb8\xac\xe8\xa9\xa6\xe8\xbb\x9f\xe9\xab\x94\xe7\x9a\x84\xe9" -"\x80\x9f\xe5\xba\xa6\xe6\x98\xaf\xe4\xb8\x8d\xe5\xae\xb9\xe5\xbf" -"\xbd\xe8\xa6\x96\xe7\x9a\x84\x0a\xe8\xaa\xb2\xe9\xa1\x8c\x2e\x20" -"\xe7\x82\xba\xe5\x8a\xa0\xe5\xbf\xab\xe9\x96\x8b\xe7\x99\xbc\xe5" -"\x8f\x8a\xe6\xb8\xac\xe8\xa9\xa6\xe7\x9a\x84\xe9\x80\x9f\xe5\xba" -"\xa6\x2c\x20\xe6\x88\x91\xe5\x80\x91\xe4\xbe\xbf\xe5\xb8\xb8\xe5" -"\xb8\x8c\xe6\x9c\x9b\xe8\x83\xbd\xe5\x88\xa9\xe7\x94\xa8\xe4\xb8" -"\x80\xe4\xba\x9b\xe5\xb7\xb2\xe9\x96\x8b\xe7\x99\xbc\xe5\xa5\xbd" -"\xe7\x9a\x84\x0a\x6c\x69\x62\x72\x61\x72\x79\x2c\x20\xe4\xb8\xa6" -"\xe6\x9c\x89\xe4\xb8\x80\xe5\x80\x8b\x20\x66\x61\x73\x74\x20\x70" -"\x72\x6f\x74\x6f\x74\x79\x70\x69\x6e\x67\x20\xe7\x9a\x84\x20\x70" -"\x72\x6f\x67\x72\x61\x6d\x6d\x69\x6e\x67\x20\x6c\x61\x6e\x67\x75" -"\x61\x67\x65\x20\xe5\x8f\xaf\x0a\xe4\xbe\x9b\xe4\xbd\xbf\xe7\x94" -"\xa8\x2e\x20\xe7\x9b\xae\xe5\x89\x8d\xe6\x9c\x89\xe8\xa8\xb1\xe8" -"\xa8\xb1\xe5\xa4\x9a\xe5\xa4\x9a\xe7\x9a\x84\x20\x6c\x69\x62\x72" -"\x61\x72\x79\x20\xe6\x98\xaf\xe4\xbb\xa5\x20\x43\x20\xe5\xaf\xab" -"\xe6\x88\x90\x2c\x20\xe8\x80\x8c\x20\x50\x79\x74\x68\x6f\x6e\x20" -"\xe6\x98\xaf\xe4\xb8\x80\xe5\x80\x8b\x0a\x66\x61\x73\x74\x20\x70" -"\x72\x6f\x74\x6f\x74\x79\x70\x69\x6e\x67\x20\xe7\x9a\x84\x20\x70" -"\x72\x6f\x67\x72\x61\x6d\x6d\x69\x6e\x67\x20\x6c\x61\x6e\x67\x75" -"\x61\x67\x65\x2e\x20\xe6\x95\x85\xe6\x88\x91\xe5\x80\x91\xe5\xb8" -"\x8c\xe6\x9c\x9b\xe8\x83\xbd\xe5\xb0\x87\xe6\x97\xa2\xe6\x9c\x89" -"\xe7\x9a\x84\x0a\x43\x20\x6c\x69\x62\x72\x61\x72\x79\x20\xe6\x8b" -"\xbf\xe5\x88\xb0\x20\x50\x79\x74\x68\x6f\x6e\x20\xe7\x9a\x84\xe7" -"\x92\xb0\xe5\xa2\x83\xe4\xb8\xad\xe6\xb8\xac\xe8\xa9\xa6\xe5\x8f" -"\x8a\xe6\x95\xb4\xe5\x90\x88\x2e\x20\xe5\x85\xb6\xe4\xb8\xad\xe6" -"\x9c\x80\xe4\xb8\xbb\xe8\xa6\x81\xe4\xb9\x9f\xe6\x98\xaf\xe6\x88" -"\x91\xe5\x80\x91\xe6\x89\x80\x0a\xe8\xa6\x81\xe8\xa8\x8e\xe8\xab" -"\x96\xe7\x9a\x84\xe5\x95\x8f\xe9\xa1\x8c\xe5\xb0\xb1\xe6\x98\xaf" -"\x3a\x0a\x0a"), -'johab': ( -"\x99\xb1\xa4\x77\x88\x62\xd0\x61\x20\xcd\x5c\xaf\xa1\xc5\xa9\x9c" -"\x61\x0a\x0a\xdc\xc0\xdc\xc0\x90\x73\x21\x21\x20\xf1\x67\xe2\x9c" -"\xf0\x55\xcc\x81\xa3\x89\x9f\x85\x8a\xa1\x20\xdc\xde\xdc\xd3\xd2" -"\x7a\xd9\xaf\xd9\xaf\xd9\xaf\x20\x8b\x77\x96\xd3\x20\xdc\xd1\x95" -"\x81\x20\xdc\xc0\x2e\x20\x2e\x0a\xed\x3c\xb5\x77\xdc\xd1\x93\x77" -"\xd2\x73\x20\x2e\x20\x2e\x20\x2e\x20\x2e\x20\xac\xe1\xb6\x89\x9e" -"\xa1\x20\x95\x65\xd0\x62\xf0\xe0\x20\xe0\x3b\xd2\x7a\x20\x21\x20" -"\x21\x20\x21\x87\x41\x2e\x87\x41\x0a\xd3\x61\xd3\x61\xd3\x61\x20" -"\x88\x41\x88\x41\x88\x41\xd9\x69\x87\x41\x5f\x87\x41\x20\xb4\xe1" -"\x9f\x9a\x20\xc8\xa1\xc5\xc1\x8b\x7a\x20\x95\x61\xb7\x77\x20\xc3" -"\x97\xe2\x9c\x97\x69\xf0\xe0\x20\xdc\xc0\x97\x61\x8b\x7a\x0a\xac" -"\xe9\x9f\x7a\x20\xe0\x3b\xd2\x7a\x20\x2e\x20\x2e\x20\x2e\x20\x2e" -"\x20\x8a\x89\xb4\x81\xae\xba\x20\xdc\xd1\x8a\xa1\x20\xdc\xde\x9f" -"\x89\xdc\xc2\x8b\x7a\x20\xf1\x67\xf1\x62\xf5\x49\xed\xfc\xf3\xe9" -"\x8c\x61\xbb\x9a\x0a\xb5\xc1\xb2\xa1\xd2\x7a\x20\x21\x20\x21\x20" -"\xed\x3c\xb5\x77\xdc\xd1\x20\xe0\x3b\x93\x77\x8a\xa1\x20\xd9\x69" -"\xea\xbe\x89\xc5\x20\xb4\xf4\x93\x77\x8a\xa1\x93\x77\x20\xed\x3c" -"\x93\x77\x96\xc1\xd2\x7a\x20\x8b\x69\xb4\x81\x97\x7a\x0a\xdc\xde" -"\x9d\x61\x97\x41\xe2\x9c\x20\xaf\x81\xce\xa1\xae\xa1\xd2\x7a\x20" -"\xb4\xe1\x9f\x9a\x20\xf1\x67\xf1\x62\xf5\x49\xed\xfc\xf3\xe9\xaf" -"\x82\xdc\xef\x97\x69\xb4\x7a\x21\x21\x20\xdc\xc0\xdc\xc0\x90\x73" -"\xd9\xbd\x20\xd9\x62\xd9\x62\x2a\x0a\x0a", -"\xeb\x98\xa0\xeb\xb0\xa9\xea\xb0\x81\xed\x95\x98\x20\xed\x8e\xb2" -"\xec\x8b\x9c\xec\xbd\x9c\xeb\x9d\xbc\x0a\x0a\xe3\x89\xaf\xe3\x89" -"\xaf\xeb\x82\xa9\x21\x21\x20\xe5\x9b\xa0\xe4\xb9\x9d\xe6\x9c\x88" -"\xed\x8c\xa8\xeb\xaf\xa4\xeb\xa6\x94\xea\xb6\x88\x20\xe2\x93\xa1" -"\xe2\x93\x96\xed\x9b\x80\xc2\xbf\xc2\xbf\xc2\xbf\x20\xea\xb8\x8d" -"\xeb\x92\x99\x20\xe2\x93\x94\xeb\x8e\xa8\x20\xe3\x89\xaf\x2e\x20" -"\x2e\x0a\xe4\xba\x9e\xec\x98\x81\xe2\x93\x94\xeb\x8a\xa5\xed\x9a" -"\xb9\x20\x2e\x20\x2e\x20\x2e\x20\x2e\x20\xec\x84\x9c\xec\x9a\xb8" -"\xeb\xa4\x84\x20\xeb\x8e\x90\xed\x95\x99\xe4\xb9\x99\x20\xe5\xae" -"\xb6\xed\x9b\x80\x20\x21\x20\x21\x20\x21\xe3\x85\xa0\x2e\xe3\x85" -"\xa0\x0a\xed\x9d\x90\xed\x9d\x90\xed\x9d\x90\x20\xe3\x84\xb1\xe3" -"\x84\xb1\xe3\x84\xb1\xe2\x98\x86\xe3\x85\xa0\x5f\xe3\x85\xa0\x20" -"\xec\x96\xb4\xeb\xa6\xa8\x20\xed\x83\xb8\xec\xbd\xb0\xea\xb8\x90" -"\x20\xeb\x8e\x8c\xec\x9d\x91\x20\xec\xb9\x91\xe4\xb9\x9d\xeb\x93" -"\xa4\xe4\xb9\x99\x20\xe3\x89\xaf\xeb\x93\x9c\xea\xb8\x90\x0a\xec" -"\x84\xa4\xeb\xa6\x8c\x20\xe5\xae\xb6\xed\x9b\x80\x20\x2e\x20\x2e" -"\x20\x2e\x20\x2e\x20\xea\xb5\xb4\xec\x95\xa0\xec\x89\x8c\x20\xe2" -"\x93\x94\xea\xb6\x88\x20\xe2\x93\xa1\xeb\xa6\x98\xe3\x89\xb1\xea" -"\xb8\x90\x20\xe5\x9b\xa0\xe4\xbb\x81\xe5\xb7\x9d\xef\xa6\x81\xe4" -"\xb8\xad\xea\xb9\x8c\xec\xa6\xbc\x0a\xec\x99\x80\xec\x92\x80\xed" -"\x9b\x80\x20\x21\x20\x21\x20\xe4\xba\x9e\xec\x98\x81\xe2\x93\x94" -"\x20\xe5\xae\xb6\xeb\x8a\xa5\xea\xb6\x88\x20\xe2\x98\x86\xe4\xb8" -"\x8a\xea\xb4\x80\x20\xec\x97\x86\xeb\x8a\xa5\xea\xb6\x88\xeb\x8a" -"\xa5\x20\xe4\xba\x9e\xeb\x8a\xa5\xeb\x92\x88\xed\x9b\x80\x20\xea" -"\xb8\x80\xec\x95\xa0\xeb\x93\xb4\x0a\xe2\x93\xa1\xeb\xa0\xa4\xeb" -"\x93\x80\xe4\xb9\x9d\x20\xec\x8b\x80\xed\x92\x94\xec\x88\xb4\xed" -"\x9b\x80\x20\xec\x96\xb4\xeb\xa6\xa8\x20\xe5\x9b\xa0\xe4\xbb\x81" -"\xe5\xb7\x9d\xef\xa6\x81\xe4\xb8\xad\xec\x8b\x81\xe2\x91\xa8\xeb" -"\x93\xa4\xec\x95\x9c\x21\x21\x20\xe3\x89\xaf\xe3\x89\xaf\xeb\x82" -"\xa9\xe2\x99\xa1\x20\xe2\x8c\x92\xe2\x8c\x92\x2a\x0a\x0a"), -'shift_jis': ( -"\x50\x79\x74\x68\x6f\x6e\x20\x82\xcc\x8a\x4a\x94\xad\x82\xcd\x81" -"\x41\x31\x39\x39\x30\x20\x94\x4e\x82\xb2\x82\xeb\x82\xa9\x82\xe7" -"\x8a\x4a\x8e\x6e\x82\xb3\x82\xea\x82\xc4\x82\xa2\x82\xdc\x82\xb7" -"\x81\x42\x0a\x8a\x4a\x94\xad\x8e\xd2\x82\xcc\x20\x47\x75\x69\x64" -"\x6f\x20\x76\x61\x6e\x20\x52\x6f\x73\x73\x75\x6d\x20\x82\xcd\x8b" -"\xb3\x88\xe7\x97\x70\x82\xcc\x83\x76\x83\x8d\x83\x4f\x83\x89\x83" -"\x7e\x83\x93\x83\x4f\x8c\xbe\x8c\xea\x81\x75\x41\x42\x43\x81\x76" -"\x82\xcc\x8a\x4a\x94\xad\x82\xc9\x8e\x51\x89\xc1\x82\xb5\x82\xc4" -"\x82\xa2\x82\xdc\x82\xb5\x82\xbd\x82\xaa\x81\x41\x41\x42\x43\x20" -"\x82\xcd\x8e\xc0\x97\x70\x8f\xe3\x82\xcc\x96\xda\x93\x49\x82\xc9" -"\x82\xcd\x82\xa0\x82\xdc\x82\xe8\x93\x4b\x82\xb5\x82\xc4\x82\xa2" -"\x82\xdc\x82\xb9\x82\xf1\x82\xc5\x82\xb5\x82\xbd\x81\x42\x0a\x82" -"\xb1\x82\xcc\x82\xbd\x82\xdf\x81\x41\x47\x75\x69\x64\x6f\x20\x82" -"\xcd\x82\xe6\x82\xe8\x8e\xc0\x97\x70\x93\x49\x82\xc8\x83\x76\x83" -"\x8d\x83\x4f\x83\x89\x83\x7e\x83\x93\x83\x4f\x8c\xbe\x8c\xea\x82" -"\xcc\x8a\x4a\x94\xad\x82\xf0\x8a\x4a\x8e\x6e\x82\xb5\x81\x41\x89" -"\x70\x8d\x91\x20\x42\x42\x53\x20\x95\xfa\x91\x97\x82\xcc\x83\x52" -"\x83\x81\x83\x66\x83\x42\x94\xd4\x91\x67\x81\x75\x83\x82\x83\x93" -"\x83\x65\x83\x42\x20\x83\x70\x83\x43\x83\x5c\x83\x93\x81\x76\x82" -"\xcc\x83\x74\x83\x40\x83\x93\x82\xc5\x82\xa0\x82\xe9\x20\x47\x75" -"\x69\x64\x6f\x20\x82\xcd\x82\xb1\x82\xcc\x8c\xbe\x8c\xea\x82\xf0" -"\x81\x75\x50\x79\x74\x68\x6f\x6e\x81\x76\x82\xc6\x96\xbc\x82\xc3" -"\x82\xaf\x82\xdc\x82\xb5\x82\xbd\x81\x42\x0a\x82\xb1\x82\xcc\x82" -"\xe6\x82\xa4\x82\xc8\x94\x77\x8c\x69\x82\xa9\x82\xe7\x90\xb6\x82" -"\xdc\x82\xea\x82\xbd\x20\x50\x79\x74\x68\x6f\x6e\x20\x82\xcc\x8c" -"\xbe\x8c\xea\x90\xdd\x8c\x76\x82\xcd\x81\x41\x81\x75\x83\x56\x83" -"\x93\x83\x76\x83\x8b\x81\x76\x82\xc5\x81\x75\x8f\x4b\x93\xbe\x82" -"\xaa\x97\x65\x88\xd5\x81\x76\x82\xc6\x82\xa2\x82\xa4\x96\xda\x95" -"\x57\x82\xc9\x8f\x64\x93\x5f\x82\xaa\x92\x75\x82\xa9\x82\xea\x82" -"\xc4\x82\xa2\x82\xdc\x82\xb7\x81\x42\x0a\x91\xbd\x82\xad\x82\xcc" -"\x83\x58\x83\x4e\x83\x8a\x83\x76\x83\x67\x8c\x6e\x8c\xbe\x8c\xea" -"\x82\xc5\x82\xcd\x83\x86\x81\x5b\x83\x55\x82\xcc\x96\xda\x90\xe6" -"\x82\xcc\x97\x98\x95\xd6\x90\xab\x82\xf0\x97\x44\x90\xe6\x82\xb5" -"\x82\xc4\x90\x46\x81\x58\x82\xc8\x8b\x40\x94\x5c\x82\xf0\x8c\xbe" -"\x8c\xea\x97\x76\x91\x66\x82\xc6\x82\xb5\x82\xc4\x8e\xe6\x82\xe8" -"\x93\xfc\x82\xea\x82\xe9\x8f\xea\x8d\x87\x82\xaa\x91\xbd\x82\xa2" -"\x82\xcc\x82\xc5\x82\xb7\x82\xaa\x81\x41\x50\x79\x74\x68\x6f\x6e" -"\x20\x82\xc5\x82\xcd\x82\xbb\x82\xa4\x82\xa2\x82\xc1\x82\xbd\x8f" -"\xac\x8d\xd7\x8d\x48\x82\xaa\x92\xc7\x89\xc1\x82\xb3\x82\xea\x82" -"\xe9\x82\xb1\x82\xc6\x82\xcd\x82\xa0\x82\xdc\x82\xe8\x82\xa0\x82" -"\xe8\x82\xdc\x82\xb9\x82\xf1\x81\x42\x0a\x8c\xbe\x8c\xea\x8e\xa9" -"\x91\xcc\x82\xcc\x8b\x40\x94\x5c\x82\xcd\x8d\xc5\x8f\xac\x8c\xc0" -"\x82\xc9\x89\x9f\x82\xb3\x82\xa6\x81\x41\x95\x4b\x97\x76\x82\xc8" -"\x8b\x40\x94\x5c\x82\xcd\x8a\x67\x92\xa3\x83\x82\x83\x57\x83\x85" -"\x81\x5b\x83\x8b\x82\xc6\x82\xb5\x82\xc4\x92\xc7\x89\xc1\x82\xb7" -"\x82\xe9\x81\x41\x82\xc6\x82\xa2\x82\xa4\x82\xcc\x82\xaa\x20\x50" -"\x79\x74\x68\x6f\x6e\x20\x82\xcc\x83\x7c\x83\x8a\x83\x56\x81\x5b" -"\x82\xc5\x82\xb7\x81\x42\x0a\x0a", -"\x50\x79\x74\x68\x6f\x6e\x20\xe3\x81\xae\xe9\x96\x8b\xe7\x99\xba" -"\xe3\x81\xaf\xe3\x80\x81\x31\x39\x39\x30\x20\xe5\xb9\xb4\xe3\x81" -"\x94\xe3\x82\x8d\xe3\x81\x8b\xe3\x82\x89\xe9\x96\x8b\xe5\xa7\x8b" -"\xe3\x81\x95\xe3\x82\x8c\xe3\x81\xa6\xe3\x81\x84\xe3\x81\xbe\xe3" -"\x81\x99\xe3\x80\x82\x0a\xe9\x96\x8b\xe7\x99\xba\xe8\x80\x85\xe3" -"\x81\xae\x20\x47\x75\x69\x64\x6f\x20\x76\x61\x6e\x20\x52\x6f\x73" -"\x73\x75\x6d\x20\xe3\x81\xaf\xe6\x95\x99\xe8\x82\xb2\xe7\x94\xa8" -"\xe3\x81\xae\xe3\x83\x97\xe3\x83\xad\xe3\x82\xb0\xe3\x83\xa9\xe3" -"\x83\x9f\xe3\x83\xb3\xe3\x82\xb0\xe8\xa8\x80\xe8\xaa\x9e\xe3\x80" -"\x8c\x41\x42\x43\xe3\x80\x8d\xe3\x81\xae\xe9\x96\x8b\xe7\x99\xba" -"\xe3\x81\xab\xe5\x8f\x82\xe5\x8a\xa0\xe3\x81\x97\xe3\x81\xa6\xe3" -"\x81\x84\xe3\x81\xbe\xe3\x81\x97\xe3\x81\x9f\xe3\x81\x8c\xe3\x80" -"\x81\x41\x42\x43\x20\xe3\x81\xaf\xe5\xae\x9f\xe7\x94\xa8\xe4\xb8" -"\x8a\xe3\x81\xae\xe7\x9b\xae\xe7\x9a\x84\xe3\x81\xab\xe3\x81\xaf" -"\xe3\x81\x82\xe3\x81\xbe\xe3\x82\x8a\xe9\x81\xa9\xe3\x81\x97\xe3" -"\x81\xa6\xe3\x81\x84\xe3\x81\xbe\xe3\x81\x9b\xe3\x82\x93\xe3\x81" -"\xa7\xe3\x81\x97\xe3\x81\x9f\xe3\x80\x82\x0a\xe3\x81\x93\xe3\x81" -"\xae\xe3\x81\x9f\xe3\x82\x81\xe3\x80\x81\x47\x75\x69\x64\x6f\x20" -"\xe3\x81\xaf\xe3\x82\x88\xe3\x82\x8a\xe5\xae\x9f\xe7\x94\xa8\xe7" -"\x9a\x84\xe3\x81\xaa\xe3\x83\x97\xe3\x83\xad\xe3\x82\xb0\xe3\x83" -"\xa9\xe3\x83\x9f\xe3\x83\xb3\xe3\x82\xb0\xe8\xa8\x80\xe8\xaa\x9e" -"\xe3\x81\xae\xe9\x96\x8b\xe7\x99\xba\xe3\x82\x92\xe9\x96\x8b\xe5" -"\xa7\x8b\xe3\x81\x97\xe3\x80\x81\xe8\x8b\xb1\xe5\x9b\xbd\x20\x42" -"\x42\x53\x20\xe6\x94\xbe\xe9\x80\x81\xe3\x81\xae\xe3\x82\xb3\xe3" -"\x83\xa1\xe3\x83\x87\xe3\x82\xa3\xe7\x95\xaa\xe7\xb5\x84\xe3\x80" -"\x8c\xe3\x83\xa2\xe3\x83\xb3\xe3\x83\x86\xe3\x82\xa3\x20\xe3\x83" -"\x91\xe3\x82\xa4\xe3\x82\xbd\xe3\x83\xb3\xe3\x80\x8d\xe3\x81\xae" -"\xe3\x83\x95\xe3\x82\xa1\xe3\x83\xb3\xe3\x81\xa7\xe3\x81\x82\xe3" -"\x82\x8b\x20\x47\x75\x69\x64\x6f\x20\xe3\x81\xaf\xe3\x81\x93\xe3" -"\x81\xae\xe8\xa8\x80\xe8\xaa\x9e\xe3\x82\x92\xe3\x80\x8c\x50\x79" -"\x74\x68\x6f\x6e\xe3\x80\x8d\xe3\x81\xa8\xe5\x90\x8d\xe3\x81\xa5" -"\xe3\x81\x91\xe3\x81\xbe\xe3\x81\x97\xe3\x81\x9f\xe3\x80\x82\x0a" -"\xe3\x81\x93\xe3\x81\xae\xe3\x82\x88\xe3\x81\x86\xe3\x81\xaa\xe8" -"\x83\x8c\xe6\x99\xaf\xe3\x81\x8b\xe3\x82\x89\xe7\x94\x9f\xe3\x81" -"\xbe\xe3\x82\x8c\xe3\x81\x9f\x20\x50\x79\x74\x68\x6f\x6e\x20\xe3" -"\x81\xae\xe8\xa8\x80\xe8\xaa\x9e\xe8\xa8\xad\xe8\xa8\x88\xe3\x81" -"\xaf\xe3\x80\x81\xe3\x80\x8c\xe3\x82\xb7\xe3\x83\xb3\xe3\x83\x97" -"\xe3\x83\xab\xe3\x80\x8d\xe3\x81\xa7\xe3\x80\x8c\xe7\xbf\x92\xe5" -"\xbe\x97\xe3\x81\x8c\xe5\xae\xb9\xe6\x98\x93\xe3\x80\x8d\xe3\x81" -"\xa8\xe3\x81\x84\xe3\x81\x86\xe7\x9b\xae\xe6\xa8\x99\xe3\x81\xab" -"\xe9\x87\x8d\xe7\x82\xb9\xe3\x81\x8c\xe7\xbd\xae\xe3\x81\x8b\xe3" -"\x82\x8c\xe3\x81\xa6\xe3\x81\x84\xe3\x81\xbe\xe3\x81\x99\xe3\x80" -"\x82\x0a\xe5\xa4\x9a\xe3\x81\x8f\xe3\x81\xae\xe3\x82\xb9\xe3\x82" -"\xaf\xe3\x83\xaa\xe3\x83\x97\xe3\x83\x88\xe7\xb3\xbb\xe8\xa8\x80" -"\xe8\xaa\x9e\xe3\x81\xa7\xe3\x81\xaf\xe3\x83\xa6\xe3\x83\xbc\xe3" -"\x82\xb6\xe3\x81\xae\xe7\x9b\xae\xe5\x85\x88\xe3\x81\xae\xe5\x88" -"\xa9\xe4\xbe\xbf\xe6\x80\xa7\xe3\x82\x92\xe5\x84\xaa\xe5\x85\x88" -"\xe3\x81\x97\xe3\x81\xa6\xe8\x89\xb2\xe3\x80\x85\xe3\x81\xaa\xe6" -"\xa9\x9f\xe8\x83\xbd\xe3\x82\x92\xe8\xa8\x80\xe8\xaa\x9e\xe8\xa6" -"\x81\xe7\xb4\xa0\xe3\x81\xa8\xe3\x81\x97\xe3\x81\xa6\xe5\x8f\x96" -"\xe3\x82\x8a\xe5\x85\xa5\xe3\x82\x8c\xe3\x82\x8b\xe5\xa0\xb4\xe5" -"\x90\x88\xe3\x81\x8c\xe5\xa4\x9a\xe3\x81\x84\xe3\x81\xae\xe3\x81" -"\xa7\xe3\x81\x99\xe3\x81\x8c\xe3\x80\x81\x50\x79\x74\x68\x6f\x6e" -"\x20\xe3\x81\xa7\xe3\x81\xaf\xe3\x81\x9d\xe3\x81\x86\xe3\x81\x84" -"\xe3\x81\xa3\xe3\x81\x9f\xe5\xb0\x8f\xe7\xb4\xb0\xe5\xb7\xa5\xe3" -"\x81\x8c\xe8\xbf\xbd\xe5\x8a\xa0\xe3\x81\x95\xe3\x82\x8c\xe3\x82" -"\x8b\xe3\x81\x93\xe3\x81\xa8\xe3\x81\xaf\xe3\x81\x82\xe3\x81\xbe" -"\xe3\x82\x8a\xe3\x81\x82\xe3\x82\x8a\xe3\x81\xbe\xe3\x81\x9b\xe3" -"\x82\x93\xe3\x80\x82\x0a\xe8\xa8\x80\xe8\xaa\x9e\xe8\x87\xaa\xe4" -"\xbd\x93\xe3\x81\xae\xe6\xa9\x9f\xe8\x83\xbd\xe3\x81\xaf\xe6\x9c" -"\x80\xe5\xb0\x8f\xe9\x99\x90\xe3\x81\xab\xe6\x8a\xbc\xe3\x81\x95" -"\xe3\x81\x88\xe3\x80\x81\xe5\xbf\x85\xe8\xa6\x81\xe3\x81\xaa\xe6" -"\xa9\x9f\xe8\x83\xbd\xe3\x81\xaf\xe6\x8b\xa1\xe5\xbc\xb5\xe3\x83" -"\xa2\xe3\x82\xb8\xe3\x83\xa5\xe3\x83\xbc\xe3\x83\xab\xe3\x81\xa8" -"\xe3\x81\x97\xe3\x81\xa6\xe8\xbf\xbd\xe5\x8a\xa0\xe3\x81\x99\xe3" -"\x82\x8b\xe3\x80\x81\xe3\x81\xa8\xe3\x81\x84\xe3\x81\x86\xe3\x81" -"\xae\xe3\x81\x8c\x20\x50\x79\x74\x68\x6f\x6e\x20\xe3\x81\xae\xe3" -"\x83\x9d\xe3\x83\xaa\xe3\x82\xb7\xe3\x83\xbc\xe3\x81\xa7\xe3\x81" -"\x99\xe3\x80\x82\x0a\x0a"), -'shift_jisx0213': ( -"\x50\x79\x74\x68\x6f\x6e\x20\x82\xcc\x8a\x4a\x94\xad\x82\xcd\x81" -"\x41\x31\x39\x39\x30\x20\x94\x4e\x82\xb2\x82\xeb\x82\xa9\x82\xe7" -"\x8a\x4a\x8e\x6e\x82\xb3\x82\xea\x82\xc4\x82\xa2\x82\xdc\x82\xb7" -"\x81\x42\x0a\x8a\x4a\x94\xad\x8e\xd2\x82\xcc\x20\x47\x75\x69\x64" -"\x6f\x20\x76\x61\x6e\x20\x52\x6f\x73\x73\x75\x6d\x20\x82\xcd\x8b" -"\xb3\x88\xe7\x97\x70\x82\xcc\x83\x76\x83\x8d\x83\x4f\x83\x89\x83" -"\x7e\x83\x93\x83\x4f\x8c\xbe\x8c\xea\x81\x75\x41\x42\x43\x81\x76" -"\x82\xcc\x8a\x4a\x94\xad\x82\xc9\x8e\x51\x89\xc1\x82\xb5\x82\xc4" -"\x82\xa2\x82\xdc\x82\xb5\x82\xbd\x82\xaa\x81\x41\x41\x42\x43\x20" -"\x82\xcd\x8e\xc0\x97\x70\x8f\xe3\x82\xcc\x96\xda\x93\x49\x82\xc9" -"\x82\xcd\x82\xa0\x82\xdc\x82\xe8\x93\x4b\x82\xb5\x82\xc4\x82\xa2" -"\x82\xdc\x82\xb9\x82\xf1\x82\xc5\x82\xb5\x82\xbd\x81\x42\x0a\x82" -"\xb1\x82\xcc\x82\xbd\x82\xdf\x81\x41\x47\x75\x69\x64\x6f\x20\x82" -"\xcd\x82\xe6\x82\xe8\x8e\xc0\x97\x70\x93\x49\x82\xc8\x83\x76\x83" -"\x8d\x83\x4f\x83\x89\x83\x7e\x83\x93\x83\x4f\x8c\xbe\x8c\xea\x82" -"\xcc\x8a\x4a\x94\xad\x82\xf0\x8a\x4a\x8e\x6e\x82\xb5\x81\x41\x89" -"\x70\x8d\x91\x20\x42\x42\x53\x20\x95\xfa\x91\x97\x82\xcc\x83\x52" -"\x83\x81\x83\x66\x83\x42\x94\xd4\x91\x67\x81\x75\x83\x82\x83\x93" -"\x83\x65\x83\x42\x20\x83\x70\x83\x43\x83\x5c\x83\x93\x81\x76\x82" -"\xcc\x83\x74\x83\x40\x83\x93\x82\xc5\x82\xa0\x82\xe9\x20\x47\x75" -"\x69\x64\x6f\x20\x82\xcd\x82\xb1\x82\xcc\x8c\xbe\x8c\xea\x82\xf0" -"\x81\x75\x50\x79\x74\x68\x6f\x6e\x81\x76\x82\xc6\x96\xbc\x82\xc3" -"\x82\xaf\x82\xdc\x82\xb5\x82\xbd\x81\x42\x0a\x82\xb1\x82\xcc\x82" -"\xe6\x82\xa4\x82\xc8\x94\x77\x8c\x69\x82\xa9\x82\xe7\x90\xb6\x82" -"\xdc\x82\xea\x82\xbd\x20\x50\x79\x74\x68\x6f\x6e\x20\x82\xcc\x8c" -"\xbe\x8c\xea\x90\xdd\x8c\x76\x82\xcd\x81\x41\x81\x75\x83\x56\x83" -"\x93\x83\x76\x83\x8b\x81\x76\x82\xc5\x81\x75\x8f\x4b\x93\xbe\x82" -"\xaa\x97\x65\x88\xd5\x81\x76\x82\xc6\x82\xa2\x82\xa4\x96\xda\x95" -"\x57\x82\xc9\x8f\x64\x93\x5f\x82\xaa\x92\x75\x82\xa9\x82\xea\x82" -"\xc4\x82\xa2\x82\xdc\x82\xb7\x81\x42\x0a\x91\xbd\x82\xad\x82\xcc" -"\x83\x58\x83\x4e\x83\x8a\x83\x76\x83\x67\x8c\x6e\x8c\xbe\x8c\xea" -"\x82\xc5\x82\xcd\x83\x86\x81\x5b\x83\x55\x82\xcc\x96\xda\x90\xe6" -"\x82\xcc\x97\x98\x95\xd6\x90\xab\x82\xf0\x97\x44\x90\xe6\x82\xb5" -"\x82\xc4\x90\x46\x81\x58\x82\xc8\x8b\x40\x94\x5c\x82\xf0\x8c\xbe" -"\x8c\xea\x97\x76\x91\x66\x82\xc6\x82\xb5\x82\xc4\x8e\xe6\x82\xe8" -"\x93\xfc\x82\xea\x82\xe9\x8f\xea\x8d\x87\x82\xaa\x91\xbd\x82\xa2" -"\x82\xcc\x82\xc5\x82\xb7\x82\xaa\x81\x41\x50\x79\x74\x68\x6f\x6e" -"\x20\x82\xc5\x82\xcd\x82\xbb\x82\xa4\x82\xa2\x82\xc1\x82\xbd\x8f" -"\xac\x8d\xd7\x8d\x48\x82\xaa\x92\xc7\x89\xc1\x82\xb3\x82\xea\x82" -"\xe9\x82\xb1\x82\xc6\x82\xcd\x82\xa0\x82\xdc\x82\xe8\x82\xa0\x82" -"\xe8\x82\xdc\x82\xb9\x82\xf1\x81\x42\x0a\x8c\xbe\x8c\xea\x8e\xa9" -"\x91\xcc\x82\xcc\x8b\x40\x94\x5c\x82\xcd\x8d\xc5\x8f\xac\x8c\xc0" -"\x82\xc9\x89\x9f\x82\xb3\x82\xa6\x81\x41\x95\x4b\x97\x76\x82\xc8" -"\x8b\x40\x94\x5c\x82\xcd\x8a\x67\x92\xa3\x83\x82\x83\x57\x83\x85" -"\x81\x5b\x83\x8b\x82\xc6\x82\xb5\x82\xc4\x92\xc7\x89\xc1\x82\xb7" -"\x82\xe9\x81\x41\x82\xc6\x82\xa2\x82\xa4\x82\xcc\x82\xaa\x20\x50" -"\x79\x74\x68\x6f\x6e\x20\x82\xcc\x83\x7c\x83\x8a\x83\x56\x81\x5b" -"\x82\xc5\x82\xb7\x81\x42\x0a\x0a\x83\x6d\x82\xf5\x20\x83\x9e\x20" -"\x83\x67\x83\x4c\x88\x4b\x88\x79\x20\x98\x83\xfc\xd6\x20\xfc\xd2" -"\xfc\xe6\xfb\xd4\x0a", -"\x50\x79\x74\x68\x6f\x6e\x20\xe3\x81\xae\xe9\x96\x8b\xe7\x99\xba" -"\xe3\x81\xaf\xe3\x80\x81\x31\x39\x39\x30\x20\xe5\xb9\xb4\xe3\x81" -"\x94\xe3\x82\x8d\xe3\x81\x8b\xe3\x82\x89\xe9\x96\x8b\xe5\xa7\x8b" -"\xe3\x81\x95\xe3\x82\x8c\xe3\x81\xa6\xe3\x81\x84\xe3\x81\xbe\xe3" -"\x81\x99\xe3\x80\x82\x0a\xe9\x96\x8b\xe7\x99\xba\xe8\x80\x85\xe3" -"\x81\xae\x20\x47\x75\x69\x64\x6f\x20\x76\x61\x6e\x20\x52\x6f\x73" -"\x73\x75\x6d\x20\xe3\x81\xaf\xe6\x95\x99\xe8\x82\xb2\xe7\x94\xa8" -"\xe3\x81\xae\xe3\x83\x97\xe3\x83\xad\xe3\x82\xb0\xe3\x83\xa9\xe3" -"\x83\x9f\xe3\x83\xb3\xe3\x82\xb0\xe8\xa8\x80\xe8\xaa\x9e\xe3\x80" -"\x8c\x41\x42\x43\xe3\x80\x8d\xe3\x81\xae\xe9\x96\x8b\xe7\x99\xba" -"\xe3\x81\xab\xe5\x8f\x82\xe5\x8a\xa0\xe3\x81\x97\xe3\x81\xa6\xe3" -"\x81\x84\xe3\x81\xbe\xe3\x81\x97\xe3\x81\x9f\xe3\x81\x8c\xe3\x80" -"\x81\x41\x42\x43\x20\xe3\x81\xaf\xe5\xae\x9f\xe7\x94\xa8\xe4\xb8" -"\x8a\xe3\x81\xae\xe7\x9b\xae\xe7\x9a\x84\xe3\x81\xab\xe3\x81\xaf" -"\xe3\x81\x82\xe3\x81\xbe\xe3\x82\x8a\xe9\x81\xa9\xe3\x81\x97\xe3" -"\x81\xa6\xe3\x81\x84\xe3\x81\xbe\xe3\x81\x9b\xe3\x82\x93\xe3\x81" -"\xa7\xe3\x81\x97\xe3\x81\x9f\xe3\x80\x82\x0a\xe3\x81\x93\xe3\x81" -"\xae\xe3\x81\x9f\xe3\x82\x81\xe3\x80\x81\x47\x75\x69\x64\x6f\x20" -"\xe3\x81\xaf\xe3\x82\x88\xe3\x82\x8a\xe5\xae\x9f\xe7\x94\xa8\xe7" -"\x9a\x84\xe3\x81\xaa\xe3\x83\x97\xe3\x83\xad\xe3\x82\xb0\xe3\x83" -"\xa9\xe3\x83\x9f\xe3\x83\xb3\xe3\x82\xb0\xe8\xa8\x80\xe8\xaa\x9e" -"\xe3\x81\xae\xe9\x96\x8b\xe7\x99\xba\xe3\x82\x92\xe9\x96\x8b\xe5" -"\xa7\x8b\xe3\x81\x97\xe3\x80\x81\xe8\x8b\xb1\xe5\x9b\xbd\x20\x42" -"\x42\x53\x20\xe6\x94\xbe\xe9\x80\x81\xe3\x81\xae\xe3\x82\xb3\xe3" -"\x83\xa1\xe3\x83\x87\xe3\x82\xa3\xe7\x95\xaa\xe7\xb5\x84\xe3\x80" -"\x8c\xe3\x83\xa2\xe3\x83\xb3\xe3\x83\x86\xe3\x82\xa3\x20\xe3\x83" -"\x91\xe3\x82\xa4\xe3\x82\xbd\xe3\x83\xb3\xe3\x80\x8d\xe3\x81\xae" -"\xe3\x83\x95\xe3\x82\xa1\xe3\x83\xb3\xe3\x81\xa7\xe3\x81\x82\xe3" -"\x82\x8b\x20\x47\x75\x69\x64\x6f\x20\xe3\x81\xaf\xe3\x81\x93\xe3" -"\x81\xae\xe8\xa8\x80\xe8\xaa\x9e\xe3\x82\x92\xe3\x80\x8c\x50\x79" -"\x74\x68\x6f\x6e\xe3\x80\x8d\xe3\x81\xa8\xe5\x90\x8d\xe3\x81\xa5" -"\xe3\x81\x91\xe3\x81\xbe\xe3\x81\x97\xe3\x81\x9f\xe3\x80\x82\x0a" -"\xe3\x81\x93\xe3\x81\xae\xe3\x82\x88\xe3\x81\x86\xe3\x81\xaa\xe8" -"\x83\x8c\xe6\x99\xaf\xe3\x81\x8b\xe3\x82\x89\xe7\x94\x9f\xe3\x81" -"\xbe\xe3\x82\x8c\xe3\x81\x9f\x20\x50\x79\x74\x68\x6f\x6e\x20\xe3" -"\x81\xae\xe8\xa8\x80\xe8\xaa\x9e\xe8\xa8\xad\xe8\xa8\x88\xe3\x81" -"\xaf\xe3\x80\x81\xe3\x80\x8c\xe3\x82\xb7\xe3\x83\xb3\xe3\x83\x97" -"\xe3\x83\xab\xe3\x80\x8d\xe3\x81\xa7\xe3\x80\x8c\xe7\xbf\x92\xe5" -"\xbe\x97\xe3\x81\x8c\xe5\xae\xb9\xe6\x98\x93\xe3\x80\x8d\xe3\x81" -"\xa8\xe3\x81\x84\xe3\x81\x86\xe7\x9b\xae\xe6\xa8\x99\xe3\x81\xab" -"\xe9\x87\x8d\xe7\x82\xb9\xe3\x81\x8c\xe7\xbd\xae\xe3\x81\x8b\xe3" -"\x82\x8c\xe3\x81\xa6\xe3\x81\x84\xe3\x81\xbe\xe3\x81\x99\xe3\x80" -"\x82\x0a\xe5\xa4\x9a\xe3\x81\x8f\xe3\x81\xae\xe3\x82\xb9\xe3\x82" -"\xaf\xe3\x83\xaa\xe3\x83\x97\xe3\x83\x88\xe7\xb3\xbb\xe8\xa8\x80" -"\xe8\xaa\x9e\xe3\x81\xa7\xe3\x81\xaf\xe3\x83\xa6\xe3\x83\xbc\xe3" -"\x82\xb6\xe3\x81\xae\xe7\x9b\xae\xe5\x85\x88\xe3\x81\xae\xe5\x88" -"\xa9\xe4\xbe\xbf\xe6\x80\xa7\xe3\x82\x92\xe5\x84\xaa\xe5\x85\x88" -"\xe3\x81\x97\xe3\x81\xa6\xe8\x89\xb2\xe3\x80\x85\xe3\x81\xaa\xe6" -"\xa9\x9f\xe8\x83\xbd\xe3\x82\x92\xe8\xa8\x80\xe8\xaa\x9e\xe8\xa6" -"\x81\xe7\xb4\xa0\xe3\x81\xa8\xe3\x81\x97\xe3\x81\xa6\xe5\x8f\x96" -"\xe3\x82\x8a\xe5\x85\xa5\xe3\x82\x8c\xe3\x82\x8b\xe5\xa0\xb4\xe5" -"\x90\x88\xe3\x81\x8c\xe5\xa4\x9a\xe3\x81\x84\xe3\x81\xae\xe3\x81" -"\xa7\xe3\x81\x99\xe3\x81\x8c\xe3\x80\x81\x50\x79\x74\x68\x6f\x6e" -"\x20\xe3\x81\xa7\xe3\x81\xaf\xe3\x81\x9d\xe3\x81\x86\xe3\x81\x84" -"\xe3\x81\xa3\xe3\x81\x9f\xe5\xb0\x8f\xe7\xb4\xb0\xe5\xb7\xa5\xe3" -"\x81\x8c\xe8\xbf\xbd\xe5\x8a\xa0\xe3\x81\x95\xe3\x82\x8c\xe3\x82" -"\x8b\xe3\x81\x93\xe3\x81\xa8\xe3\x81\xaf\xe3\x81\x82\xe3\x81\xbe" -"\xe3\x82\x8a\xe3\x81\x82\xe3\x82\x8a\xe3\x81\xbe\xe3\x81\x9b\xe3" -"\x82\x93\xe3\x80\x82\x0a\xe8\xa8\x80\xe8\xaa\x9e\xe8\x87\xaa\xe4" -"\xbd\x93\xe3\x81\xae\xe6\xa9\x9f\xe8\x83\xbd\xe3\x81\xaf\xe6\x9c" -"\x80\xe5\xb0\x8f\xe9\x99\x90\xe3\x81\xab\xe6\x8a\xbc\xe3\x81\x95" -"\xe3\x81\x88\xe3\x80\x81\xe5\xbf\x85\xe8\xa6\x81\xe3\x81\xaa\xe6" -"\xa9\x9f\xe8\x83\xbd\xe3\x81\xaf\xe6\x8b\xa1\xe5\xbc\xb5\xe3\x83" -"\xa2\xe3\x82\xb8\xe3\x83\xa5\xe3\x83\xbc\xe3\x83\xab\xe3\x81\xa8" -"\xe3\x81\x97\xe3\x81\xa6\xe8\xbf\xbd\xe5\x8a\xa0\xe3\x81\x99\xe3" -"\x82\x8b\xe3\x80\x81\xe3\x81\xa8\xe3\x81\x84\xe3\x81\x86\xe3\x81" -"\xae\xe3\x81\x8c\x20\x50\x79\x74\x68\x6f\x6e\x20\xe3\x81\xae\xe3" -"\x83\x9d\xe3\x83\xaa\xe3\x82\xb7\xe3\x83\xbc\xe3\x81\xa7\xe3\x81" -"\x99\xe3\x80\x82\x0a\x0a\xe3\x83\x8e\xe3\x81\x8b\xe3\x82\x9a\x20" -"\xe3\x83\x88\xe3\x82\x9a\x20\xe3\x83\x88\xe3\x82\xad\xef\xa8\xb6" -"\xef\xa8\xb9\x20\xf0\xa1\x9a\xb4\xf0\xaa\x8e\x8c\x20\xe9\xba\x80" -"\xe9\xbd\x81\xf0\xa9\x9b\xb0\x0a"), -} diff --git a/lib-python/2.7/test/crashers/README b/lib-python/2.7/test/crashers/README --- a/lib-python/2.7/test/crashers/README +++ b/lib-python/2.7/test/crashers/README @@ -1,20 +1,16 @@ -This directory only contains tests for outstanding bugs that cause -the interpreter to segfault. Ideally this directory should always -be empty. Sometimes it may not be easy to fix the underlying cause. +This directory only contains tests for outstanding bugs that cause the +interpreter to segfault. Ideally this directory should always be empty, but +sometimes it may not be easy to fix the underlying cause and the bug is deemed +too obscure to invest the effort. Each test should fail when run from the command line: ./python Lib/test/crashers/weakref_in_del.py -Each test should have a link to the bug report: +Put as much info into a docstring or comments to help determine the cause of the +failure, as well as a bugs.python.org issue number if it exists. Particularly +note if the cause is system or environment dependent and what the variables are. - # http://python.org/sf/BUG# - -Put as much info into a docstring or comments to help determine -the cause of the failure. Particularly note if the cause is -system or environment dependent and what the variables are. - -Once the crash is fixed, the test case should be moved into an appropriate -test (even if it was originally from the test suite). This ensures the -regression doesn't happen again. And if it does, it should be easier -to track down. +Once the crash is fixed, the test case should be moved into an appropriate test +(even if it was originally from the test suite). This ensures the regression +doesn't happen again. And if it does, it should be easier to track down. diff --git a/lib-python/2.7/test/crashers/recursion_limit_too_high.py b/lib-python/2.7/test/crashers/recursion_limit_too_high.py --- a/lib-python/2.7/test/crashers/recursion_limit_too_high.py +++ b/lib-python/2.7/test/crashers/recursion_limit_too_high.py @@ -5,7 +5,7 @@ # file handles. # The point of this example is to show that sys.setrecursionlimit() is a -# hack, and not a robust solution. This example simply exercices a path +# hack, and not a robust solution. This example simply exercises a path # where it takes many C-level recursions, consuming a lot of stack # space, for each Python-level recursion. So 1000 times this amount of # stack space may be too much for standard platforms already. diff --git a/lib-python/2.7/test/decimaltestdata/and.decTest b/lib-python/2.7/test/decimaltestdata/and.decTest --- a/lib-python/2.7/test/decimaltestdata/and.decTest +++ b/lib-python/2.7/test/decimaltestdata/and.decTest @@ -1,338 +1,338 @@ ------------------------------------------------------------------------- --- and.decTest -- digitwise logical AND -- --- Copyright (c) IBM Corporation, 1981, 2008. All rights reserved. -- ------------------------------------------------------------------------- --- Please see the document "General Decimal Arithmetic Testcases" -- --- at http://www2.hursley.ibm.com/decimal for the description of -- --- these testcases. -- --- -- --- These testcases are experimental ('beta' versions), and they -- --- may contain errors. They are offered on an as-is basis. In -- --- particular, achieving the same results as the tests here is not -- --- a guarantee that an implementation complies with any Standard -- --- or specification. The tests are not exhaustive. -- --- -- --- Please send comments, suggestions, and corrections to the author: -- --- Mike Cowlishaw, IBM Fellow -- --- IBM UK, PO Box 31, Birmingham Road, Warwick CV34 5JL, UK -- --- mfc at uk.ibm.com -- ------------------------------------------------------------------------- -version: 2.59 - -extended: 1 -precision: 9 -rounding: half_up -maxExponent: 999 -minExponent: -999 - --- Sanity check (truth table) -andx001 and 0 0 -> 0 -andx002 and 0 1 -> 0 -andx003 and 1 0 -> 0 -andx004 and 1 1 -> 1 -andx005 and 1100 1010 -> 1000 -andx006 and 1111 10 -> 10 -andx007 and 1111 1010 -> 1010 - --- and at msd and msd-1 -andx010 and 000000000 000000000 -> 0 -andx011 and 000000000 100000000 -> 0 -andx012 and 100000000 000000000 -> 0 -andx013 and 100000000 100000000 -> 100000000 -andx014 and 000000000 000000000 -> 0 -andx015 and 000000000 010000000 -> 0 -andx016 and 010000000 000000000 -> 0 -andx017 and 010000000 010000000 -> 10000000 - --- Various lengths --- 123456789 123456789 123456789 -andx021 and 111111111 111111111 -> 111111111 -andx022 and 111111111111 111111111 -> 111111111 -andx023 and 111111111111 11111111 -> 11111111 -andx024 and 111111111 11111111 -> 11111111 -andx025 and 111111111 1111111 -> 1111111 -andx026 and 111111111111 111111 -> 111111 -andx027 and 111111111111 11111 -> 11111 -andx028 and 111111111111 1111 -> 1111 -andx029 and 111111111111 111 -> 111 -andx031 and 111111111111 11 -> 11 -andx032 and 111111111111 1 -> 1 -andx033 and 111111111111 1111111111 -> 111111111 -andx034 and 11111111111 11111111111 -> 111111111 -andx035 and 1111111111 111111111111 -> 111111111 -andx036 and 111111111 1111111111111 -> 111111111 - -andx040 and 111111111 111111111111 -> 111111111 -andx041 and 11111111 111111111111 -> 11111111 -andx042 and 11111111 111111111 -> 11111111 -andx043 and 1111111 111111111 -> 1111111 -andx044 and 111111 111111111 -> 111111 -andx045 and 11111 111111111 -> 11111 -andx046 and 1111 111111111 -> 1111 -andx047 and 111 111111111 -> 111 -andx048 and 11 111111111 -> 11 -andx049 and 1 111111111 -> 1 - -andx050 and 1111111111 1 -> 1 -andx051 and 111111111 1 -> 1 -andx052 and 11111111 1 -> 1 -andx053 and 1111111 1 -> 1 -andx054 and 111111 1 -> 1 -andx055 and 11111 1 -> 1 -andx056 and 1111 1 -> 1 -andx057 and 111 1 -> 1 -andx058 and 11 1 -> 1 -andx059 and 1 1 -> 1 - -andx060 and 1111111111 0 -> 0 -andx061 and 111111111 0 -> 0 -andx062 and 11111111 0 -> 0 -andx063 and 1111111 0 -> 0 -andx064 and 111111 0 -> 0 -andx065 and 11111 0 -> 0 -andx066 and 1111 0 -> 0 -andx067 and 111 0 -> 0 -andx068 and 11 0 -> 0 -andx069 and 1 0 -> 0 - -andx070 and 1 1111111111 -> 1 -andx071 and 1 111111111 -> 1 -andx072 and 1 11111111 -> 1 -andx073 and 1 1111111 -> 1 -andx074 and 1 111111 -> 1 -andx075 and 1 11111 -> 1 -andx076 and 1 1111 -> 1 -andx077 and 1 111 -> 1 -andx078 and 1 11 -> 1 -andx079 and 1 1 -> 1 - -andx080 and 0 1111111111 -> 0 -andx081 and 0 111111111 -> 0 -andx082 and 0 11111111 -> 0 -andx083 and 0 1111111 -> 0 -andx084 and 0 111111 -> 0 -andx085 and 0 11111 -> 0 -andx086 and 0 1111 -> 0 -andx087 and 0 111 -> 0 -andx088 and 0 11 -> 0 -andx089 and 0 1 -> 0 - -andx090 and 011111111 111111111 -> 11111111 -andx091 and 101111111 111111111 -> 101111111 -andx092 and 110111111 111111111 -> 110111111 -andx093 and 111011111 111111111 -> 111011111 -andx094 and 111101111 111111111 -> 111101111 -andx095 and 111110111 111111111 -> 111110111 -andx096 and 111111011 111111111 -> 111111011 -andx097 and 111111101 111111111 -> 111111101 -andx098 and 111111110 111111111 -> 111111110 - -andx100 and 111111111 011111111 -> 11111111 -andx101 and 111111111 101111111 -> 101111111 -andx102 and 111111111 110111111 -> 110111111 -andx103 and 111111111 111011111 -> 111011111 -andx104 and 111111111 111101111 -> 111101111 -andx105 and 111111111 111110111 -> 111110111 -andx106 and 111111111 111111011 -> 111111011 -andx107 and 111111111 111111101 -> 111111101 -andx108 and 111111111 111111110 -> 111111110 - --- non-0/1 should not be accepted, nor should signs -andx220 and 111111112 111111111 -> NaN Invalid_operation -andx221 and 333333333 333333333 -> NaN Invalid_operation -andx222 and 555555555 555555555 -> NaN Invalid_operation -andx223 and 777777777 777777777 -> NaN Invalid_operation -andx224 and 999999999 999999999 -> NaN Invalid_operation -andx225 and 222222222 999999999 -> NaN Invalid_operation -andx226 and 444444444 999999999 -> NaN Invalid_operation -andx227 and 666666666 999999999 -> NaN Invalid_operation -andx228 and 888888888 999999999 -> NaN Invalid_operation -andx229 and 999999999 222222222 -> NaN Invalid_operation -andx230 and 999999999 444444444 -> NaN Invalid_operation -andx231 and 999999999 666666666 -> NaN Invalid_operation -andx232 and 999999999 888888888 -> NaN Invalid_operation --- a few randoms -andx240 and 567468689 -934981942 -> NaN Invalid_operation -andx241 and 567367689 934981942 -> NaN Invalid_operation -andx242 and -631917772 -706014634 -> NaN Invalid_operation -andx243 and -756253257 138579234 -> NaN Invalid_operation -andx244 and 835590149 567435400 -> NaN Invalid_operation --- test MSD -andx250 and 200000000 100000000 -> NaN Invalid_operation -andx251 and 700000000 100000000 -> NaN Invalid_operation -andx252 and 800000000 100000000 -> NaN Invalid_operation -andx253 and 900000000 100000000 -> NaN Invalid_operation -andx254 and 200000000 000000000 -> NaN Invalid_operation -andx255 and 700000000 000000000 -> NaN Invalid_operation -andx256 and 800000000 000000000 -> NaN Invalid_operation -andx257 and 900000000 000000000 -> NaN Invalid_operation -andx258 and 100000000 200000000 -> NaN Invalid_operation -andx259 and 100000000 700000000 -> NaN Invalid_operation -andx260 and 100000000 800000000 -> NaN Invalid_operation -andx261 and 100000000 900000000 -> NaN Invalid_operation -andx262 and 000000000 200000000 -> NaN Invalid_operation -andx263 and 000000000 700000000 -> NaN Invalid_operation -andx264 and 000000000 800000000 -> NaN Invalid_operation -andx265 and 000000000 900000000 -> NaN Invalid_operation --- test MSD-1 -andx270 and 020000000 100000000 -> NaN Invalid_operation -andx271 and 070100000 100000000 -> NaN Invalid_operation -andx272 and 080010000 100000001 -> NaN Invalid_operation -andx273 and 090001000 100000010 -> NaN Invalid_operation -andx274 and 100000100 020010100 -> NaN Invalid_operation -andx275 and 100000000 070001000 -> NaN Invalid_operation -andx276 and 100000010 080010100 -> NaN Invalid_operation -andx277 and 100000000 090000010 -> NaN Invalid_operation --- test LSD -andx280 and 001000002 100000000 -> NaN Invalid_operation -andx281 and 000000007 100000000 -> NaN Invalid_operation -andx282 and 000000008 100000000 -> NaN Invalid_operation -andx283 and 000000009 100000000 -> NaN Invalid_operation -andx284 and 100000000 000100002 -> NaN Invalid_operation -andx285 and 100100000 001000007 -> NaN Invalid_operation -andx286 and 100010000 010000008 -> NaN Invalid_operation -andx287 and 100001000 100000009 -> NaN Invalid_operation --- test Middie -andx288 and 001020000 100000000 -> NaN Invalid_operation -andx289 and 000070001 100000000 -> NaN Invalid_operation -andx290 and 000080000 100010000 -> NaN Invalid_operation -andx291 and 000090000 100001000 -> NaN Invalid_operation -andx292 and 100000010 000020100 -> NaN Invalid_operation -andx293 and 100100000 000070010 -> NaN Invalid_operation -andx294 and 100010100 000080001 -> NaN Invalid_operation -andx295 and 100001000 000090000 -> NaN Invalid_operation --- signs -andx296 and -100001000 -000000000 -> NaN Invalid_operation -andx297 and -100001000 000010000 -> NaN Invalid_operation -andx298 and 100001000 -000000000 -> NaN Invalid_operation -andx299 and 100001000 000011000 -> 1000 - --- Nmax, Nmin, Ntiny -andx331 and 2 9.99999999E+999 -> NaN Invalid_operation -andx332 and 3 1E-999 -> NaN Invalid_operation -andx333 and 4 1.00000000E-999 -> NaN Invalid_operation -andx334 and 5 1E-1007 -> NaN Invalid_operation -andx335 and 6 -1E-1007 -> NaN Invalid_operation -andx336 and 7 -1.00000000E-999 -> NaN Invalid_operation -andx337 and 8 -1E-999 -> NaN Invalid_operation -andx338 and 9 -9.99999999E+999 -> NaN Invalid_operation -andx341 and 9.99999999E+999 -18 -> NaN Invalid_operation -andx342 and 1E-999 01 -> NaN Invalid_operation -andx343 and 1.00000000E-999 -18 -> NaN Invalid_operation -andx344 and 1E-1007 18 -> NaN Invalid_operation -andx345 and -1E-1007 -10 -> NaN Invalid_operation -andx346 and -1.00000000E-999 18 -> NaN Invalid_operation -andx347 and -1E-999 10 -> NaN Invalid_operation -andx348 and -9.99999999E+999 -18 -> NaN Invalid_operation - --- A few other non-integers -andx361 and 1.0 1 -> NaN Invalid_operation -andx362 and 1E+1 1 -> NaN Invalid_operation -andx363 and 0.0 1 -> NaN Invalid_operation -andx364 and 0E+1 1 -> NaN Invalid_operation -andx365 and 9.9 1 -> NaN Invalid_operation -andx366 and 9E+1 1 -> NaN Invalid_operation -andx371 and 0 1.0 -> NaN Invalid_operation -andx372 and 0 1E+1 -> NaN Invalid_operation -andx373 and 0 0.0 -> NaN Invalid_operation -andx374 and 0 0E+1 -> NaN Invalid_operation -andx375 and 0 9.9 -> NaN Invalid_operation -andx376 and 0 9E+1 -> NaN Invalid_operation - --- All Specials are in error -andx780 and -Inf -Inf -> NaN Invalid_operation -andx781 and -Inf -1000 -> NaN Invalid_operation -andx782 and -Inf -1 -> NaN Invalid_operation -andx783 and -Inf -0 -> NaN Invalid_operation -andx784 and -Inf 0 -> NaN Invalid_operation -andx785 and -Inf 1 -> NaN Invalid_operation -andx786 and -Inf 1000 -> NaN Invalid_operation -andx787 and -1000 -Inf -> NaN Invalid_operation -andx788 and -Inf -Inf -> NaN Invalid_operation -andx789 and -1 -Inf -> NaN Invalid_operation -andx790 and -0 -Inf -> NaN Invalid_operation -andx791 and 0 -Inf -> NaN Invalid_operation -andx792 and 1 -Inf -> NaN Invalid_operation -andx793 and 1000 -Inf -> NaN Invalid_operation -andx794 and Inf -Inf -> NaN Invalid_operation - -andx800 and Inf -Inf -> NaN Invalid_operation -andx801 and Inf -1000 -> NaN Invalid_operation -andx802 and Inf -1 -> NaN Invalid_operation -andx803 and Inf -0 -> NaN Invalid_operation -andx804 and Inf 0 -> NaN Invalid_operation -andx805 and Inf 1 -> NaN Invalid_operation -andx806 and Inf 1000 -> NaN Invalid_operation -andx807 and Inf Inf -> NaN Invalid_operation -andx808 and -1000 Inf -> NaN Invalid_operation -andx809 and -Inf Inf -> NaN Invalid_operation -andx810 and -1 Inf -> NaN Invalid_operation -andx811 and -0 Inf -> NaN Invalid_operation -andx812 and 0 Inf -> NaN Invalid_operation -andx813 and 1 Inf -> NaN Invalid_operation -andx814 and 1000 Inf -> NaN Invalid_operation -andx815 and Inf Inf -> NaN Invalid_operation - -andx821 and NaN -Inf -> NaN Invalid_operation -andx822 and NaN -1000 -> NaN Invalid_operation -andx823 and NaN -1 -> NaN Invalid_operation -andx824 and NaN -0 -> NaN Invalid_operation -andx825 and NaN 0 -> NaN Invalid_operation -andx826 and NaN 1 -> NaN Invalid_operation -andx827 and NaN 1000 -> NaN Invalid_operation -andx828 and NaN Inf -> NaN Invalid_operation -andx829 and NaN NaN -> NaN Invalid_operation -andx830 and -Inf NaN -> NaN Invalid_operation -andx831 and -1000 NaN -> NaN Invalid_operation -andx832 and -1 NaN -> NaN Invalid_operation -andx833 and -0 NaN -> NaN Invalid_operation -andx834 and 0 NaN -> NaN Invalid_operation -andx835 and 1 NaN -> NaN Invalid_operation -andx836 and 1000 NaN -> NaN Invalid_operation -andx837 and Inf NaN -> NaN Invalid_operation - -andx841 and sNaN -Inf -> NaN Invalid_operation -andx842 and sNaN -1000 -> NaN Invalid_operation -andx843 and sNaN -1 -> NaN Invalid_operation -andx844 and sNaN -0 -> NaN Invalid_operation -andx845 and sNaN 0 -> NaN Invalid_operation -andx846 and sNaN 1 -> NaN Invalid_operation -andx847 and sNaN 1000 -> NaN Invalid_operation -andx848 and sNaN NaN -> NaN Invalid_operation -andx849 and sNaN sNaN -> NaN Invalid_operation -andx850 and NaN sNaN -> NaN Invalid_operation -andx851 and -Inf sNaN -> NaN Invalid_operation -andx852 and -1000 sNaN -> NaN Invalid_operation -andx853 and -1 sNaN -> NaN Invalid_operation -andx854 and -0 sNaN -> NaN Invalid_operation -andx855 and 0 sNaN -> NaN Invalid_operation -andx856 and 1 sNaN -> NaN Invalid_operation -andx857 and 1000 sNaN -> NaN Invalid_operation -andx858 and Inf sNaN -> NaN Invalid_operation -andx859 and NaN sNaN -> NaN Invalid_operation - --- propagating NaNs -andx861 and NaN1 -Inf -> NaN Invalid_operation -andx862 and +NaN2 -1000 -> NaN Invalid_operation -andx863 and NaN3 1000 -> NaN Invalid_operation -andx864 and NaN4 Inf -> NaN Invalid_operation -andx865 and NaN5 +NaN6 -> NaN Invalid_operation -andx866 and -Inf NaN7 -> NaN Invalid_operation -andx867 and -1000 NaN8 -> NaN Invalid_operation -andx868 and 1000 NaN9 -> NaN Invalid_operation -andx869 and Inf +NaN10 -> NaN Invalid_operation -andx871 and sNaN11 -Inf -> NaN Invalid_operation -andx872 and sNaN12 -1000 -> NaN Invalid_operation -andx873 and sNaN13 1000 -> NaN Invalid_operation -andx874 and sNaN14 NaN17 -> NaN Invalid_operation -andx875 and sNaN15 sNaN18 -> NaN Invalid_operation -andx876 and NaN16 sNaN19 -> NaN Invalid_operation -andx877 and -Inf +sNaN20 -> NaN Invalid_operation -andx878 and -1000 sNaN21 -> NaN Invalid_operation -andx879 and 1000 sNaN22 -> NaN Invalid_operation -andx880 and Inf sNaN23 -> NaN Invalid_operation -andx881 and +NaN25 +sNaN24 -> NaN Invalid_operation -andx882 and -NaN26 NaN28 -> NaN Invalid_operation -andx883 and -sNaN27 sNaN29 -> NaN Invalid_operation -andx884 and 1000 -NaN30 -> NaN Invalid_operation -andx885 and 1000 -sNaN31 -> NaN Invalid_operation +------------------------------------------------------------------------ +-- and.decTest -- digitwise logical AND -- +-- Copyright (c) IBM Corporation, 1981, 2008. All rights reserved. -- +------------------------------------------------------------------------ +-- Please see the document "General Decimal Arithmetic Testcases" -- +-- at http://www2.hursley.ibm.com/decimal for the description of -- +-- these testcases. -- +-- -- +-- These testcases are experimental ('beta' versions), and they -- +-- may contain errors. They are offered on an as-is basis. In -- +-- particular, achieving the same results as the tests here is not -- +-- a guarantee that an implementation complies with any Standard -- +-- or specification. The tests are not exhaustive. -- +-- -- +-- Please send comments, suggestions, and corrections to the author: -- +-- Mike Cowlishaw, IBM Fellow -- +-- IBM UK, PO Box 31, Birmingham Road, Warwick CV34 5JL, UK -- +-- mfc at uk.ibm.com -- +------------------------------------------------------------------------ +version: 2.59 + +extended: 1 +precision: 9 +rounding: half_up +maxExponent: 999 +minExponent: -999 + +-- Sanity check (truth table) +andx001 and 0 0 -> 0 +andx002 and 0 1 -> 0 +andx003 and 1 0 -> 0 +andx004 and 1 1 -> 1 +andx005 and 1100 1010 -> 1000 +andx006 and 1111 10 -> 10 +andx007 and 1111 1010 -> 1010 + +-- and at msd and msd-1 +andx010 and 000000000 000000000 -> 0 +andx011 and 000000000 100000000 -> 0 +andx012 and 100000000 000000000 -> 0 +andx013 and 100000000 100000000 -> 100000000 +andx014 and 000000000 000000000 -> 0 +andx015 and 000000000 010000000 -> 0 +andx016 and 010000000 000000000 -> 0 +andx017 and 010000000 010000000 -> 10000000 + +-- Various lengths +-- 123456789 123456789 123456789 +andx021 and 111111111 111111111 -> 111111111 +andx022 and 111111111111 111111111 -> 111111111 +andx023 and 111111111111 11111111 -> 11111111 +andx024 and 111111111 11111111 -> 11111111 +andx025 and 111111111 1111111 -> 1111111 +andx026 and 111111111111 111111 -> 111111 +andx027 and 111111111111 11111 -> 11111 +andx028 and 111111111111 1111 -> 1111 +andx029 and 111111111111 111 -> 111 +andx031 and 111111111111 11 -> 11 +andx032 and 111111111111 1 -> 1 +andx033 and 111111111111 1111111111 -> 111111111 +andx034 and 11111111111 11111111111 -> 111111111 +andx035 and 1111111111 111111111111 -> 111111111 +andx036 and 111111111 1111111111111 -> 111111111 + +andx040 and 111111111 111111111111 -> 111111111 +andx041 and 11111111 111111111111 -> 11111111 +andx042 and 11111111 111111111 -> 11111111 +andx043 and 1111111 111111111 -> 1111111 +andx044 and 111111 111111111 -> 111111 +andx045 and 11111 111111111 -> 11111 +andx046 and 1111 111111111 -> 1111 +andx047 and 111 111111111 -> 111 +andx048 and 11 111111111 -> 11 +andx049 and 1 111111111 -> 1 + +andx050 and 1111111111 1 -> 1 +andx051 and 111111111 1 -> 1 +andx052 and 11111111 1 -> 1 +andx053 and 1111111 1 -> 1 +andx054 and 111111 1 -> 1 +andx055 and 11111 1 -> 1 +andx056 and 1111 1 -> 1 +andx057 and 111 1 -> 1 +andx058 and 11 1 -> 1 +andx059 and 1 1 -> 1 + +andx060 and 1111111111 0 -> 0 +andx061 and 111111111 0 -> 0 +andx062 and 11111111 0 -> 0 +andx063 and 1111111 0 -> 0 +andx064 and 111111 0 -> 0 +andx065 and 11111 0 -> 0 +andx066 and 1111 0 -> 0 +andx067 and 111 0 -> 0 +andx068 and 11 0 -> 0 +andx069 and 1 0 -> 0 + +andx070 and 1 1111111111 -> 1 +andx071 and 1 111111111 -> 1 +andx072 and 1 11111111 -> 1 +andx073 and 1 1111111 -> 1 +andx074 and 1 111111 -> 1 +andx075 and 1 11111 -> 1 +andx076 and 1 1111 -> 1 +andx077 and 1 111 -> 1 +andx078 and 1 11 -> 1 +andx079 and 1 1 -> 1 + +andx080 and 0 1111111111 -> 0 +andx081 and 0 111111111 -> 0 +andx082 and 0 11111111 -> 0 +andx083 and 0 1111111 -> 0 +andx084 and 0 111111 -> 0 +andx085 and 0 11111 -> 0 +andx086 and 0 1111 -> 0 +andx087 and 0 111 -> 0 +andx088 and 0 11 -> 0 +andx089 and 0 1 -> 0 + +andx090 and 011111111 111111111 -> 11111111 +andx091 and 101111111 111111111 -> 101111111 +andx092 and 110111111 111111111 -> 110111111 +andx093 and 111011111 111111111 -> 111011111 +andx094 and 111101111 111111111 -> 111101111 +andx095 and 111110111 111111111 -> 111110111 +andx096 and 111111011 111111111 -> 111111011 +andx097 and 111111101 111111111 -> 111111101 +andx098 and 111111110 111111111 -> 111111110 + +andx100 and 111111111 011111111 -> 11111111 +andx101 and 111111111 101111111 -> 101111111 +andx102 and 111111111 110111111 -> 110111111 +andx103 and 111111111 111011111 -> 111011111 +andx104 and 111111111 111101111 -> 111101111 +andx105 and 111111111 111110111 -> 111110111 +andx106 and 111111111 111111011 -> 111111011 +andx107 and 111111111 111111101 -> 111111101 +andx108 and 111111111 111111110 -> 111111110 + +-- non-0/1 should not be accepted, nor should signs +andx220 and 111111112 111111111 -> NaN Invalid_operation +andx221 and 333333333 333333333 -> NaN Invalid_operation +andx222 and 555555555 555555555 -> NaN Invalid_operation +andx223 and 777777777 777777777 -> NaN Invalid_operation +andx224 and 999999999 999999999 -> NaN Invalid_operation +andx225 and 222222222 999999999 -> NaN Invalid_operation +andx226 and 444444444 999999999 -> NaN Invalid_operation +andx227 and 666666666 999999999 -> NaN Invalid_operation +andx228 and 888888888 999999999 -> NaN Invalid_operation +andx229 and 999999999 222222222 -> NaN Invalid_operation +andx230 and 999999999 444444444 -> NaN Invalid_operation +andx231 and 999999999 666666666 -> NaN Invalid_operation +andx232 and 999999999 888888888 -> NaN Invalid_operation +-- a few randoms +andx240 and 567468689 -934981942 -> NaN Invalid_operation +andx241 and 567367689 934981942 -> NaN Invalid_operation +andx242 and -631917772 -706014634 -> NaN Invalid_operation +andx243 and -756253257 138579234 -> NaN Invalid_operation +andx244 and 835590149 567435400 -> NaN Invalid_operation +-- test MSD +andx250 and 200000000 100000000 -> NaN Invalid_operation +andx251 and 700000000 100000000 -> NaN Invalid_operation +andx252 and 800000000 100000000 -> NaN Invalid_operation +andx253 and 900000000 100000000 -> NaN Invalid_operation +andx254 and 200000000 000000000 -> NaN Invalid_operation +andx255 and 700000000 000000000 -> NaN Invalid_operation +andx256 and 800000000 000000000 -> NaN Invalid_operation +andx257 and 900000000 000000000 -> NaN Invalid_operation +andx258 and 100000000 200000000 -> NaN Invalid_operation +andx259 and 100000000 700000000 -> NaN Invalid_operation +andx260 and 100000000 800000000 -> NaN Invalid_operation +andx261 and 100000000 900000000 -> NaN Invalid_operation +andx262 and 000000000 200000000 -> NaN Invalid_operation +andx263 and 000000000 700000000 -> NaN Invalid_operation +andx264 and 000000000 800000000 -> NaN Invalid_operation +andx265 and 000000000 900000000 -> NaN Invalid_operation +-- test MSD-1 +andx270 and 020000000 100000000 -> NaN Invalid_operation +andx271 and 070100000 100000000 -> NaN Invalid_operation +andx272 and 080010000 100000001 -> NaN Invalid_operation +andx273 and 090001000 100000010 -> NaN Invalid_operation +andx274 and 100000100 020010100 -> NaN Invalid_operation +andx275 and 100000000 070001000 -> NaN Invalid_operation +andx276 and 100000010 080010100 -> NaN Invalid_operation +andx277 and 100000000 090000010 -> NaN Invalid_operation +-- test LSD +andx280 and 001000002 100000000 -> NaN Invalid_operation +andx281 and 000000007 100000000 -> NaN Invalid_operation +andx282 and 000000008 100000000 -> NaN Invalid_operation +andx283 and 000000009 100000000 -> NaN Invalid_operation +andx284 and 100000000 000100002 -> NaN Invalid_operation +andx285 and 100100000 001000007 -> NaN Invalid_operation +andx286 and 100010000 010000008 -> NaN Invalid_operation +andx287 and 100001000 100000009 -> NaN Invalid_operation +-- test Middie +andx288 and 001020000 100000000 -> NaN Invalid_operation +andx289 and 000070001 100000000 -> NaN Invalid_operation +andx290 and 000080000 100010000 -> NaN Invalid_operation +andx291 and 000090000 100001000 -> NaN Invalid_operation +andx292 and 100000010 000020100 -> NaN Invalid_operation +andx293 and 100100000 000070010 -> NaN Invalid_operation +andx294 and 100010100 000080001 -> NaN Invalid_operation +andx295 and 100001000 000090000 -> NaN Invalid_operation +-- signs +andx296 and -100001000 -000000000 -> NaN Invalid_operation +andx297 and -100001000 000010000 -> NaN Invalid_operation +andx298 and 100001000 -000000000 -> NaN Invalid_operation +andx299 and 100001000 000011000 -> 1000 + +-- Nmax, Nmin, Ntiny +andx331 and 2 9.99999999E+999 -> NaN Invalid_operation +andx332 and 3 1E-999 -> NaN Invalid_operation +andx333 and 4 1.00000000E-999 -> NaN Invalid_operation +andx334 and 5 1E-1007 -> NaN Invalid_operation +andx335 and 6 -1E-1007 -> NaN Invalid_operation +andx336 and 7 -1.00000000E-999 -> NaN Invalid_operation +andx337 and 8 -1E-999 -> NaN Invalid_operation +andx338 and 9 -9.99999999E+999 -> NaN Invalid_operation +andx341 and 9.99999999E+999 -18 -> NaN Invalid_operation +andx342 and 1E-999 01 -> NaN Invalid_operation +andx343 and 1.00000000E-999 -18 -> NaN Invalid_operation +andx344 and 1E-1007 18 -> NaN Invalid_operation +andx345 and -1E-1007 -10 -> NaN Invalid_operation +andx346 and -1.00000000E-999 18 -> NaN Invalid_operation +andx347 and -1E-999 10 -> NaN Invalid_operation +andx348 and -9.99999999E+999 -18 -> NaN Invalid_operation + +-- A few other non-integers +andx361 and 1.0 1 -> NaN Invalid_operation +andx362 and 1E+1 1 -> NaN Invalid_operation +andx363 and 0.0 1 -> NaN Invalid_operation +andx364 and 0E+1 1 -> NaN Invalid_operation +andx365 and 9.9 1 -> NaN Invalid_operation +andx366 and 9E+1 1 -> NaN Invalid_operation +andx371 and 0 1.0 -> NaN Invalid_operation +andx372 and 0 1E+1 -> NaN Invalid_operation +andx373 and 0 0.0 -> NaN Invalid_operation +andx374 and 0 0E+1 -> NaN Invalid_operation +andx375 and 0 9.9 -> NaN Invalid_operation +andx376 and 0 9E+1 -> NaN Invalid_operation + +-- All Specials are in error +andx780 and -Inf -Inf -> NaN Invalid_operation +andx781 and -Inf -1000 -> NaN Invalid_operation +andx782 and -Inf -1 -> NaN Invalid_operation +andx783 and -Inf -0 -> NaN Invalid_operation +andx784 and -Inf 0 -> NaN Invalid_operation +andx785 and -Inf 1 -> NaN Invalid_operation +andx786 and -Inf 1000 -> NaN Invalid_operation +andx787 and -1000 -Inf -> NaN Invalid_operation +andx788 and -Inf -Inf -> NaN Invalid_operation +andx789 and -1 -Inf -> NaN Invalid_operation +andx790 and -0 -Inf -> NaN Invalid_operation +andx791 and 0 -Inf -> NaN Invalid_operation +andx792 and 1 -Inf -> NaN Invalid_operation +andx793 and 1000 -Inf -> NaN Invalid_operation +andx794 and Inf -Inf -> NaN Invalid_operation + +andx800 and Inf -Inf -> NaN Invalid_operation +andx801 and Inf -1000 -> NaN Invalid_operation +andx802 and Inf -1 -> NaN Invalid_operation +andx803 and Inf -0 -> NaN Invalid_operation +andx804 and Inf 0 -> NaN Invalid_operation +andx805 and Inf 1 -> NaN Invalid_operation +andx806 and Inf 1000 -> NaN Invalid_operation +andx807 and Inf Inf -> NaN Invalid_operation +andx808 and -1000 Inf -> NaN Invalid_operation +andx809 and -Inf Inf -> NaN Invalid_operation +andx810 and -1 Inf -> NaN Invalid_operation +andx811 and -0 Inf -> NaN Invalid_operation +andx812 and 0 Inf -> NaN Invalid_operation +andx813 and 1 Inf -> NaN Invalid_operation +andx814 and 1000 Inf -> NaN Invalid_operation +andx815 and Inf Inf -> NaN Invalid_operation + +andx821 and NaN -Inf -> NaN Invalid_operation +andx822 and NaN -1000 -> NaN Invalid_operation +andx823 and NaN -1 -> NaN Invalid_operation +andx824 and NaN -0 -> NaN Invalid_operation +andx825 and NaN 0 -> NaN Invalid_operation +andx826 and NaN 1 -> NaN Invalid_operation +andx827 and NaN 1000 -> NaN Invalid_operation +andx828 and NaN Inf -> NaN Invalid_operation +andx829 and NaN NaN -> NaN Invalid_operation +andx830 and -Inf NaN -> NaN Invalid_operation +andx831 and -1000 NaN -> NaN Invalid_operation +andx832 and -1 NaN -> NaN Invalid_operation +andx833 and -0 NaN -> NaN Invalid_operation +andx834 and 0 NaN -> NaN Invalid_operation +andx835 and 1 NaN -> NaN Invalid_operation +andx836 and 1000 NaN -> NaN Invalid_operation +andx837 and Inf NaN -> NaN Invalid_operation + +andx841 and sNaN -Inf -> NaN Invalid_operation +andx842 and sNaN -1000 -> NaN Invalid_operation +andx843 and sNaN -1 -> NaN Invalid_operation +andx844 and sNaN -0 -> NaN Invalid_operation +andx845 and sNaN 0 -> NaN Invalid_operation +andx846 and sNaN 1 -> NaN Invalid_operation +andx847 and sNaN 1000 -> NaN Invalid_operation +andx848 and sNaN NaN -> NaN Invalid_operation +andx849 and sNaN sNaN -> NaN Invalid_operation +andx850 and NaN sNaN -> NaN Invalid_operation +andx851 and -Inf sNaN -> NaN Invalid_operation +andx852 and -1000 sNaN -> NaN Invalid_operation +andx853 and -1 sNaN -> NaN Invalid_operation +andx854 and -0 sNaN -> NaN Invalid_operation +andx855 and 0 sNaN -> NaN Invalid_operation +andx856 and 1 sNaN -> NaN Invalid_operation +andx857 and 1000 sNaN -> NaN Invalid_operation +andx858 and Inf sNaN -> NaN Invalid_operation +andx859 and NaN sNaN -> NaN Invalid_operation + +-- propagating NaNs +andx861 and NaN1 -Inf -> NaN Invalid_operation +andx862 and +NaN2 -1000 -> NaN Invalid_operation +andx863 and NaN3 1000 -> NaN Invalid_operation +andx864 and NaN4 Inf -> NaN Invalid_operation +andx865 and NaN5 +NaN6 -> NaN Invalid_operation +andx866 and -Inf NaN7 -> NaN Invalid_operation +andx867 and -1000 NaN8 -> NaN Invalid_operation +andx868 and 1000 NaN9 -> NaN Invalid_operation +andx869 and Inf +NaN10 -> NaN Invalid_operation +andx871 and sNaN11 -Inf -> NaN Invalid_operation +andx872 and sNaN12 -1000 -> NaN Invalid_operation +andx873 and sNaN13 1000 -> NaN Invalid_operation +andx874 and sNaN14 NaN17 -> NaN Invalid_operation +andx875 and sNaN15 sNaN18 -> NaN Invalid_operation +andx876 and NaN16 sNaN19 -> NaN Invalid_operation +andx877 and -Inf +sNaN20 -> NaN Invalid_operation +andx878 and -1000 sNaN21 -> NaN Invalid_operation +andx879 and 1000 sNaN22 -> NaN Invalid_operation +andx880 and Inf sNaN23 -> NaN Invalid_operation +andx881 and +NaN25 +sNaN24 -> NaN Invalid_operation +andx882 and -NaN26 NaN28 -> NaN Invalid_operation +andx883 and -sNaN27 sNaN29 -> NaN Invalid_operation +andx884 and 1000 -NaN30 -> NaN Invalid_operation +andx885 and 1000 -sNaN31 -> NaN Invalid_operation diff --git a/lib-python/2.7/test/decimaltestdata/class.decTest b/lib-python/2.7/test/decimaltestdata/class.decTest --- a/lib-python/2.7/test/decimaltestdata/class.decTest +++ b/lib-python/2.7/test/decimaltestdata/class.decTest @@ -1,131 +1,131 @@ ------------------------------------------------------------------------- --- class.decTest -- Class operations -- --- Copyright (c) IBM Corporation, 1981, 2008. All rights reserved. -- ------------------------------------------------------------------------- --- Please see the document "General Decimal Arithmetic Testcases" -- --- at http://www2.hursley.ibm.com/decimal for the description of -- --- these testcases. -- --- -- --- These testcases are experimental ('beta' versions), and they -- --- may contain errors. They are offered on an as-is basis. In -- --- particular, achieving the same results as the tests here is not -- --- a guarantee that an implementation complies with any Standard -- --- or specification. The tests are not exhaustive. -- --- -- --- Please send comments, suggestions, and corrections to the author: -- --- Mike Cowlishaw, IBM Fellow -- --- IBM UK, PO Box 31, Birmingham Road, Warwick CV34 5JL, UK -- --- mfc at uk.ibm.com -- ------------------------------------------------------------------------- -version: 2.59 - --- [New 2006.11.27] - -precision: 9 -maxExponent: 999 -minExponent: -999 -extended: 1 -clamp: 1 -rounding: half_even - -clasx001 class 0 -> +Zero -clasx002 class 0.00 -> +Zero -clasx003 class 0E+5 -> +Zero -clasx004 class 1E-1007 -> +Subnormal -clasx005 class 0.1E-999 -> +Subnormal -clasx006 class 0.99999999E-999 -> +Subnormal -clasx007 class 1.00000000E-999 -> +Normal -clasx008 class 1E-999 -> +Normal -clasx009 class 1E-100 -> +Normal -clasx010 class 1E-10 -> +Normal -clasx012 class 1E-1 -> +Normal -clasx013 class 1 -> +Normal -clasx014 class 2.50 -> +Normal -clasx015 class 100.100 -> +Normal -clasx016 class 1E+30 -> +Normal -clasx017 class 1E+999 -> +Normal -clasx018 class 9.99999999E+999 -> +Normal -clasx019 class Inf -> +Infinity - -clasx021 class -0 -> -Zero -clasx022 class -0.00 -> -Zero -clasx023 class -0E+5 -> -Zero -clasx024 class -1E-1007 -> -Subnormal -clasx025 class -0.1E-999 -> -Subnormal -clasx026 class -0.99999999E-999 -> -Subnormal -clasx027 class -1.00000000E-999 -> -Normal -clasx028 class -1E-999 -> -Normal -clasx029 class -1E-100 -> -Normal -clasx030 class -1E-10 -> -Normal -clasx032 class -1E-1 -> -Normal -clasx033 class -1 -> -Normal -clasx034 class -2.50 -> -Normal -clasx035 class -100.100 -> -Normal -clasx036 class -1E+30 -> -Normal -clasx037 class -1E+999 -> -Normal -clasx038 class -9.99999999E+999 -> -Normal -clasx039 class -Inf -> -Infinity - -clasx041 class NaN -> NaN -clasx042 class -NaN -> NaN -clasx043 class +NaN12345 -> NaN -clasx044 class sNaN -> sNaN -clasx045 class -sNaN -> sNaN -clasx046 class +sNaN12345 -> sNaN - - --- decimal64 bounds - -precision: 16 -maxExponent: 384 -minExponent: -383 -clamp: 1 -rounding: half_even - -clasx201 class 0 -> +Zero -clasx202 class 0.00 -> +Zero -clasx203 class 0E+5 -> +Zero -clasx204 class 1E-396 -> +Subnormal -clasx205 class 0.1E-383 -> +Subnormal -clasx206 class 0.999999999999999E-383 -> +Subnormal -clasx207 class 1.000000000000000E-383 -> +Normal -clasx208 class 1E-383 -> +Normal -clasx209 class 1E-100 -> +Normal -clasx210 class 1E-10 -> +Normal -clasx212 class 1E-1 -> +Normal -clasx213 class 1 -> +Normal -clasx214 class 2.50 -> +Normal -clasx215 class 100.100 -> +Normal -clasx216 class 1E+30 -> +Normal -clasx217 class 1E+384 -> +Normal -clasx218 class 9.999999999999999E+384 -> +Normal -clasx219 class Inf -> +Infinity - -clasx221 class -0 -> -Zero -clasx222 class -0.00 -> -Zero -clasx223 class -0E+5 -> -Zero -clasx224 class -1E-396 -> -Subnormal -clasx225 class -0.1E-383 -> -Subnormal -clasx226 class -0.999999999999999E-383 -> -Subnormal -clasx227 class -1.000000000000000E-383 -> -Normal -clasx228 class -1E-383 -> -Normal -clasx229 class -1E-100 -> -Normal -clasx230 class -1E-10 -> -Normal -clasx232 class -1E-1 -> -Normal -clasx233 class -1 -> -Normal -clasx234 class -2.50 -> -Normal -clasx235 class -100.100 -> -Normal -clasx236 class -1E+30 -> -Normal -clasx237 class -1E+384 -> -Normal -clasx238 class -9.999999999999999E+384 -> -Normal -clasx239 class -Inf -> -Infinity - -clasx241 class NaN -> NaN -clasx242 class -NaN -> NaN -clasx243 class +NaN12345 -> NaN -clasx244 class sNaN -> sNaN -clasx245 class -sNaN -> sNaN -clasx246 class +sNaN12345 -> sNaN - - - +------------------------------------------------------------------------ +-- class.decTest -- Class operations -- +-- Copyright (c) IBM Corporation, 1981, 2008. All rights reserved. -- +------------------------------------------------------------------------ +-- Please see the document "General Decimal Arithmetic Testcases" -- +-- at http://www2.hursley.ibm.com/decimal for the description of -- +-- these testcases. -- +-- -- +-- These testcases are experimental ('beta' versions), and they -- +-- may contain errors. They are offered on an as-is basis. In -- +-- particular, achieving the same results as the tests here is not -- +-- a guarantee that an implementation complies with any Standard -- +-- or specification. The tests are not exhaustive. -- +-- -- +-- Please send comments, suggestions, and corrections to the author: -- +-- Mike Cowlishaw, IBM Fellow -- +-- IBM UK, PO Box 31, Birmingham Road, Warwick CV34 5JL, UK -- +-- mfc at uk.ibm.com -- +------------------------------------------------------------------------ +version: 2.59 + +-- [New 2006.11.27] + +precision: 9 +maxExponent: 999 +minExponent: -999 +extended: 1 +clamp: 1 +rounding: half_even + +clasx001 class 0 -> +Zero +clasx002 class 0.00 -> +Zero +clasx003 class 0E+5 -> +Zero +clasx004 class 1E-1007 -> +Subnormal +clasx005 class 0.1E-999 -> +Subnormal +clasx006 class 0.99999999E-999 -> +Subnormal +clasx007 class 1.00000000E-999 -> +Normal +clasx008 class 1E-999 -> +Normal +clasx009 class 1E-100 -> +Normal +clasx010 class 1E-10 -> +Normal +clasx012 class 1E-1 -> +Normal +clasx013 class 1 -> +Normal +clasx014 class 2.50 -> +Normal +clasx015 class 100.100 -> +Normal +clasx016 class 1E+30 -> +Normal +clasx017 class 1E+999 -> +Normal +clasx018 class 9.99999999E+999 -> +Normal +clasx019 class Inf -> +Infinity + +clasx021 class -0 -> -Zero +clasx022 class -0.00 -> -Zero +clasx023 class -0E+5 -> -Zero +clasx024 class -1E-1007 -> -Subnormal +clasx025 class -0.1E-999 -> -Subnormal +clasx026 class -0.99999999E-999 -> -Subnormal +clasx027 class -1.00000000E-999 -> -Normal +clasx028 class -1E-999 -> -Normal +clasx029 class -1E-100 -> -Normal +clasx030 class -1E-10 -> -Normal +clasx032 class -1E-1 -> -Normal +clasx033 class -1 -> -Normal +clasx034 class -2.50 -> -Normal +clasx035 class -100.100 -> -Normal +clasx036 class -1E+30 -> -Normal +clasx037 class -1E+999 -> -Normal +clasx038 class -9.99999999E+999 -> -Normal +clasx039 class -Inf -> -Infinity + +clasx041 class NaN -> NaN +clasx042 class -NaN -> NaN +clasx043 class +NaN12345 -> NaN +clasx044 class sNaN -> sNaN +clasx045 class -sNaN -> sNaN +clasx046 class +sNaN12345 -> sNaN + + +-- decimal64 bounds + +precision: 16 +maxExponent: 384 +minExponent: -383 +clamp: 1 +rounding: half_even + +clasx201 class 0 -> +Zero +clasx202 class 0.00 -> +Zero +clasx203 class 0E+5 -> +Zero +clasx204 class 1E-396 -> +Subnormal +clasx205 class 0.1E-383 -> +Subnormal +clasx206 class 0.999999999999999E-383 -> +Subnormal +clasx207 class 1.000000000000000E-383 -> +Normal +clasx208 class 1E-383 -> +Normal +clasx209 class 1E-100 -> +Normal +clasx210 class 1E-10 -> +Normal +clasx212 class 1E-1 -> +Normal +clasx213 class 1 -> +Normal +clasx214 class 2.50 -> +Normal +clasx215 class 100.100 -> +Normal +clasx216 class 1E+30 -> +Normal +clasx217 class 1E+384 -> +Normal +clasx218 class 9.999999999999999E+384 -> +Normal +clasx219 class Inf -> +Infinity + +clasx221 class -0 -> -Zero +clasx222 class -0.00 -> -Zero +clasx223 class -0E+5 -> -Zero +clasx224 class -1E-396 -> -Subnormal +clasx225 class -0.1E-383 -> -Subnormal +clasx226 class -0.999999999999999E-383 -> -Subnormal +clasx227 class -1.000000000000000E-383 -> -Normal +clasx228 class -1E-383 -> -Normal +clasx229 class -1E-100 -> -Normal +clasx230 class -1E-10 -> -Normal +clasx232 class -1E-1 -> -Normal +clasx233 class -1 -> -Normal +clasx234 class -2.50 -> -Normal +clasx235 class -100.100 -> -Normal +clasx236 class -1E+30 -> -Normal +clasx237 class -1E+384 -> -Normal +clasx238 class -9.999999999999999E+384 -> -Normal +clasx239 class -Inf -> -Infinity + +clasx241 class NaN -> NaN +clasx242 class -NaN -> NaN +clasx243 class +NaN12345 -> NaN +clasx244 class sNaN -> sNaN +clasx245 class -sNaN -> sNaN +clasx246 class +sNaN12345 -> sNaN + + + diff --git a/lib-python/2.7/test/decimaltestdata/comparetotal.decTest b/lib-python/2.7/test/decimaltestdata/comparetotal.decTest --- a/lib-python/2.7/test/decimaltestdata/comparetotal.decTest +++ b/lib-python/2.7/test/decimaltestdata/comparetotal.decTest @@ -1,798 +1,798 @@ ------------------------------------------------------------------------- --- comparetotal.decTest -- decimal comparison using total ordering -- --- Copyright (c) IBM Corporation, 1981, 2008. All rights reserved. -- ------------------------------------------------------------------------- --- Please see the document "General Decimal Arithmetic Testcases" -- --- at http://www2.hursley.ibm.com/decimal for the description of -- --- these testcases. -- --- -- --- These testcases are experimental ('beta' versions), and they -- --- may contain errors. They are offered on an as-is basis. In -- --- particular, achieving the same results as the tests here is not -- --- a guarantee that an implementation complies with any Standard -- --- or specification. The tests are not exhaustive. -- --- -- --- Please send comments, suggestions, and corrections to the author: -- --- Mike Cowlishaw, IBM Fellow -- --- IBM UK, PO Box 31, Birmingham Road, Warwick CV34 5JL, UK -- --- mfc at uk.ibm.com -- ------------------------------------------------------------------------- -version: 2.59 - --- Note that we cannot assume add/subtract tests cover paths adequately, --- here, because the code might be quite different (comparison cannot --- overflow or underflow, so actual subtractions are not necessary). --- Similarly, comparetotal will have some radically different paths --- than compare. - -extended: 1 -precision: 16 -rounding: half_up -maxExponent: 384 -minExponent: -383 - --- sanity checks -cotx001 comparetotal -2 -2 -> 0 -cotx002 comparetotal -2 -1 -> -1 -cotx003 comparetotal -2 0 -> -1 -cotx004 comparetotal -2 1 -> -1 -cotx005 comparetotal -2 2 -> -1 -cotx006 comparetotal -1 -2 -> 1 -cotx007 comparetotal -1 -1 -> 0 -cotx008 comparetotal -1 0 -> -1 -cotx009 comparetotal -1 1 -> -1 -cotx010 comparetotal -1 2 -> -1 -cotx011 comparetotal 0 -2 -> 1 -cotx012 comparetotal 0 -1 -> 1 -cotx013 comparetotal 0 0 -> 0 -cotx014 comparetotal 0 1 -> -1 -cotx015 comparetotal 0 2 -> -1 -cotx016 comparetotal 1 -2 -> 1 -cotx017 comparetotal 1 -1 -> 1 -cotx018 comparetotal 1 0 -> 1 -cotx019 comparetotal 1 1 -> 0 -cotx020 comparetotal 1 2 -> -1 -cotx021 comparetotal 2 -2 -> 1 -cotx022 comparetotal 2 -1 -> 1 -cotx023 comparetotal 2 0 -> 1 -cotx025 comparetotal 2 1 -> 1 -cotx026 comparetotal 2 2 -> 0 - -cotx031 comparetotal -20 -20 -> 0 -cotx032 comparetotal -20 -10 -> -1 -cotx033 comparetotal -20 00 -> -1 -cotx034 comparetotal -20 10 -> -1 -cotx035 comparetotal -20 20 -> -1 -cotx036 comparetotal -10 -20 -> 1 -cotx037 comparetotal -10 -10 -> 0 -cotx038 comparetotal -10 00 -> -1 -cotx039 comparetotal -10 10 -> -1 -cotx040 comparetotal -10 20 -> -1 -cotx041 comparetotal 00 -20 -> 1 -cotx042 comparetotal 00 -10 -> 1 -cotx043 comparetotal 00 00 -> 0 -cotx044 comparetotal 00 10 -> -1 -cotx045 comparetotal 00 20 -> -1 -cotx046 comparetotal 10 -20 -> 1 -cotx047 comparetotal 10 -10 -> 1 -cotx048 comparetotal 10 00 -> 1 -cotx049 comparetotal 10 10 -> 0 -cotx050 comparetotal 10 20 -> -1 -cotx051 comparetotal 20 -20 -> 1 -cotx052 comparetotal 20 -10 -> 1 -cotx053 comparetotal 20 00 -> 1 -cotx055 comparetotal 20 10 -> 1 -cotx056 comparetotal 20 20 -> 0 - -cotx061 comparetotal -2.0 -2.0 -> 0 -cotx062 comparetotal -2.0 -1.0 -> -1 -cotx063 comparetotal -2.0 0.0 -> -1 -cotx064 comparetotal -2.0 1.0 -> -1 -cotx065 comparetotal -2.0 2.0 -> -1 -cotx066 comparetotal -1.0 -2.0 -> 1 -cotx067 comparetotal -1.0 -1.0 -> 0 -cotx068 comparetotal -1.0 0.0 -> -1 -cotx069 comparetotal -1.0 1.0 -> -1 -cotx070 comparetotal -1.0 2.0 -> -1 -cotx071 comparetotal 0.0 -2.0 -> 1 -cotx072 comparetotal 0.0 -1.0 -> 1 -cotx073 comparetotal 0.0 0.0 -> 0 -cotx074 comparetotal 0.0 1.0 -> -1 -cotx075 comparetotal 0.0 2.0 -> -1 -cotx076 comparetotal 1.0 -2.0 -> 1 -cotx077 comparetotal 1.0 -1.0 -> 1 -cotx078 comparetotal 1.0 0.0 -> 1 -cotx079 comparetotal 1.0 1.0 -> 0 -cotx080 comparetotal 1.0 2.0 -> -1 -cotx081 comparetotal 2.0 -2.0 -> 1 -cotx082 comparetotal 2.0 -1.0 -> 1 -cotx083 comparetotal 2.0 0.0 -> 1 -cotx085 comparetotal 2.0 1.0 -> 1 -cotx086 comparetotal 2.0 2.0 -> 0 - --- now some cases which might overflow if subtract were used -maxexponent: 999999999 -minexponent: -999999999 -cotx090 comparetotal 9.99999999E+999999999 9.99999999E+999999999 -> 0 -cotx091 comparetotal -9.99999999E+999999999 9.99999999E+999999999 -> -1 -cotx092 comparetotal 9.99999999E+999999999 -9.99999999E+999999999 -> 1 -cotx093 comparetotal -9.99999999E+999999999 -9.99999999E+999999999 -> 0 - --- Examples -cotx094 comparetotal 12.73 127.9 -> -1 -cotx095 comparetotal -127 12 -> -1 -cotx096 comparetotal 12.30 12.3 -> -1 -cotx097 comparetotal 12.30 12.30 -> 0 -cotx098 comparetotal 12.3 12.300 -> 1 -cotx099 comparetotal 12.3 NaN -> -1 - --- some differing length/exponent cases --- in this first group, compare would compare all equal -cotx100 comparetotal 7.0 7.0 -> 0 -cotx101 comparetotal 7.0 7 -> -1 -cotx102 comparetotal 7 7.0 -> 1 -cotx103 comparetotal 7E+0 7.0 -> 1 -cotx104 comparetotal 70E-1 7.0 -> 0 -cotx105 comparetotal 0.7E+1 7 -> 0 -cotx106 comparetotal 70E-1 7 -> -1 -cotx107 comparetotal 7.0 7E+0 -> -1 -cotx108 comparetotal 7.0 70E-1 -> 0 -cotx109 comparetotal 7 0.7E+1 -> 0 -cotx110 comparetotal 7 70E-1 -> 1 - -cotx120 comparetotal 8.0 7.0 -> 1 -cotx121 comparetotal 8.0 7 -> 1 -cotx122 comparetotal 8 7.0 -> 1 -cotx123 comparetotal 8E+0 7.0 -> 1 -cotx124 comparetotal 80E-1 7.0 -> 1 -cotx125 comparetotal 0.8E+1 7 -> 1 -cotx126 comparetotal 80E-1 7 -> 1 -cotx127 comparetotal 8.0 7E+0 -> 1 -cotx128 comparetotal 8.0 70E-1 -> 1 -cotx129 comparetotal 8 0.7E+1 -> 1 -cotx130 comparetotal 8 70E-1 -> 1 - -cotx140 comparetotal 8.0 9.0 -> -1 -cotx141 comparetotal 8.0 9 -> -1 -cotx142 comparetotal 8 9.0 -> -1 -cotx143 comparetotal 8E+0 9.0 -> -1 -cotx144 comparetotal 80E-1 9.0 -> -1 -cotx145 comparetotal 0.8E+1 9 -> -1 -cotx146 comparetotal 80E-1 9 -> -1 -cotx147 comparetotal 8.0 9E+0 -> -1 -cotx148 comparetotal 8.0 90E-1 -> -1 -cotx149 comparetotal 8 0.9E+1 -> -1 -cotx150 comparetotal 8 90E-1 -> -1 - --- and again, with sign changes -+ .. -cotx200 comparetotal -7.0 7.0 -> -1 -cotx201 comparetotal -7.0 7 -> -1 -cotx202 comparetotal -7 7.0 -> -1 -cotx203 comparetotal -7E+0 7.0 -> -1 -cotx204 comparetotal -70E-1 7.0 -> -1 -cotx205 comparetotal -0.7E+1 7 -> -1 -cotx206 comparetotal -70E-1 7 -> -1 -cotx207 comparetotal -7.0 7E+0 -> -1 -cotx208 comparetotal -7.0 70E-1 -> -1 -cotx209 comparetotal -7 0.7E+1 -> -1 -cotx210 comparetotal -7 70E-1 -> -1 - -cotx220 comparetotal -8.0 7.0 -> -1 -cotx221 comparetotal -8.0 7 -> -1 -cotx222 comparetotal -8 7.0 -> -1 -cotx223 comparetotal -8E+0 7.0 -> -1 -cotx224 comparetotal -80E-1 7.0 -> -1 -cotx225 comparetotal -0.8E+1 7 -> -1 -cotx226 comparetotal -80E-1 7 -> -1 -cotx227 comparetotal -8.0 7E+0 -> -1 -cotx228 comparetotal -8.0 70E-1 -> -1 -cotx229 comparetotal -8 0.7E+1 -> -1 -cotx230 comparetotal -8 70E-1 -> -1 - -cotx240 comparetotal -8.0 9.0 -> -1 -cotx241 comparetotal -8.0 9 -> -1 -cotx242 comparetotal -8 9.0 -> -1 -cotx243 comparetotal -8E+0 9.0 -> -1 -cotx244 comparetotal -80E-1 9.0 -> -1 -cotx245 comparetotal -0.8E+1 9 -> -1 -cotx246 comparetotal -80E-1 9 -> -1 -cotx247 comparetotal -8.0 9E+0 -> -1 -cotx248 comparetotal -8.0 90E-1 -> -1 -cotx249 comparetotal -8 0.9E+1 -> -1 -cotx250 comparetotal -8 90E-1 -> -1 - --- and again, with sign changes +- .. -cotx300 comparetotal 7.0 -7.0 -> 1 -cotx301 comparetotal 7.0 -7 -> 1 -cotx302 comparetotal 7 -7.0 -> 1 -cotx303 comparetotal 7E+0 -7.0 -> 1 -cotx304 comparetotal 70E-1 -7.0 -> 1 -cotx305 comparetotal .7E+1 -7 -> 1 -cotx306 comparetotal 70E-1 -7 -> 1 -cotx307 comparetotal 7.0 -7E+0 -> 1 -cotx308 comparetotal 7.0 -70E-1 -> 1 -cotx309 comparetotal 7 -.7E+1 -> 1 -cotx310 comparetotal 7 -70E-1 -> 1 - -cotx320 comparetotal 8.0 -7.0 -> 1 -cotx321 comparetotal 8.0 -7 -> 1 -cotx322 comparetotal 8 -7.0 -> 1 -cotx323 comparetotal 8E+0 -7.0 -> 1 -cotx324 comparetotal 80E-1 -7.0 -> 1 -cotx325 comparetotal .8E+1 -7 -> 1 -cotx326 comparetotal 80E-1 -7 -> 1 -cotx327 comparetotal 8.0 -7E+0 -> 1 -cotx328 comparetotal 8.0 -70E-1 -> 1 -cotx329 comparetotal 8 -.7E+1 -> 1 -cotx330 comparetotal 8 -70E-1 -> 1 - -cotx340 comparetotal 8.0 -9.0 -> 1 -cotx341 comparetotal 8.0 -9 -> 1 -cotx342 comparetotal 8 -9.0 -> 1 -cotx343 comparetotal 8E+0 -9.0 -> 1 -cotx344 comparetotal 80E-1 -9.0 -> 1 -cotx345 comparetotal .8E+1 -9 -> 1 -cotx346 comparetotal 80E-1 -9 -> 1 -cotx347 comparetotal 8.0 -9E+0 -> 1 -cotx348 comparetotal 8.0 -90E-1 -> 1 -cotx349 comparetotal 8 -.9E+1 -> 1 -cotx350 comparetotal 8 -90E-1 -> 1 - --- and again, with sign changes -- .. -cotx400 comparetotal -7.0 -7.0 -> 0 -cotx401 comparetotal -7.0 -7 -> 1 -cotx402 comparetotal -7 -7.0 -> -1 -cotx403 comparetotal -7E+0 -7.0 -> -1 -cotx404 comparetotal -70E-1 -7.0 -> 0 -cotx405 comparetotal -.7E+1 -7 -> 0 -cotx406 comparetotal -70E-1 -7 -> 1 -cotx407 comparetotal -7.0 -7E+0 -> 1 -cotx408 comparetotal -7.0 -70E-1 -> 0 -cotx409 comparetotal -7 -.7E+1 -> 0 From noreply at buildbot.pypy.org Tue Jan 31 23:55:05 2012 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Tue, 31 Jan 2012 23:55:05 +0100 (CET) Subject: [pypy-commit] pypy default: expose the current call depth on DebugMergePoint Message-ID: <20120131225505.57C8882B67@wyvern.cs.uni-duesseldorf.de> Author: Alex Gaynor <alex.gaynor at gmail.com> Branch: Changeset: r52004:32425967effa Date: 2012-01-31 17:54 -0500 http://bitbucket.org/pypy/pypy/changeset/32425967effa/ Log: expose the current call depth on DebugMergePoint diff --git a/pypy/module/pypyjit/interp_resop.py b/pypy/module/pypyjit/interp_resop.py --- a/pypy/module/pypyjit/interp_resop.py +++ b/pypy/module/pypyjit/interp_resop.py @@ -127,6 +127,7 @@ l_w.append(DebugMergePoint(space, jit_hooks._cast_to_gcref(op), logops.repr_of_resop(op), jd_sd.jitdriver.name, + op.getarg(1).getint(), w_greenkey)) else: l_w.append(WrappedOp(jit_hooks._cast_to_gcref(op), ofs, @@ -163,14 +164,14 @@ llres = res.llbox return WrappedOp(jit_hooks.resop_new(num, args, llres), offset, repr) - at unwrap_spec(repr=str, jd_name=str) -def descr_new_dmp(space, w_tp, w_args, repr, jd_name, w_greenkey): + at unwrap_spec(repr=str, jd_name=str, call_depth=int) +def descr_new_dmp(space, w_tp, w_args, repr, jd_name, call_depth, w_greenkey): args = [space.interp_w(WrappedBox, w_arg).llbox for w_arg in space.listview(w_args)] num = rop.DEBUG_MERGE_POINT return DebugMergePoint(space, jit_hooks.resop_new(num, args, jit_hooks.emptyval()), - repr, jd_name, w_greenkey) + repr, jd_name, call_depth, w_greenkey) class WrappedOp(Wrappable): """ A class representing a single ResOperation, wrapped nicely @@ -205,10 +206,11 @@ jit_hooks.resop_setresult(self.op, box.llbox) class DebugMergePoint(WrappedOp): - def __init__(self, space, op, repr_of_resop, jd_name, w_greenkey): + def __init__(self, space, op, repr_of_resop, jd_name, call_depth, w_greenkey): WrappedOp.__init__(self, op, -1, repr_of_resop) + self.jd_name = jd_name + self.call_depth = call_depth self.w_greenkey = w_greenkey - self.jd_name = jd_name def get_pycode(self, space): if self.jd_name == pypyjitdriver.name: @@ -243,6 +245,7 @@ greenkey = interp_attrproperty_w("w_greenkey", cls=DebugMergePoint), pycode = GetSetProperty(DebugMergePoint.get_pycode), bytecode_no = GetSetProperty(DebugMergePoint.get_bytecode_no), + call_depth = interp_attrproperty("call_depth", cls=DebugMergePoint), jitdriver_name = GetSetProperty(DebugMergePoint.get_jitdriver_name), ) DebugMergePoint.acceptable_as_base_class = False diff --git a/pypy/module/pypyjit/test/test_jit_hook.py b/pypy/module/pypyjit/test/test_jit_hook.py --- a/pypy/module/pypyjit/test/test_jit_hook.py +++ b/pypy/module/pypyjit/test/test_jit_hook.py @@ -122,7 +122,8 @@ assert isinstance(dmp, pypyjit.DebugMergePoint) assert dmp.pycode is self.f.func_code assert dmp.greenkey == (self.f.func_code, 0, False) - #assert int_add.name == 'int_add' + assert dmp.call_depth == 0 + assert int_add.name == 'int_add' assert int_add.num == self.int_add_num self.on_compile_bridge() assert len(all) == 2 @@ -223,11 +224,13 @@ def f(): pass - op = DebugMergePoint([Box(0)], 'repr', 'pypyjit', (f.func_code, 0, 0)) + op = DebugMergePoint([Box(0)], 'repr', 'pypyjit', 2, (f.func_code, 0, 0)) assert op.bytecode_no == 0 assert op.pycode is f.func_code assert repr(op) == 'repr' assert op.jitdriver_name == 'pypyjit' assert op.num == self.dmp_num - op = DebugMergePoint([Box(0)], 'repr', 'notmain', ('str',)) + assert op.call_depth == 2 + op = DebugMergePoint([Box(0)], 'repr', 'notmain', 5, ('str',)) raises(AttributeError, 'op.pycode') + assert op.call_depth == 5

Stackless

PyPy is also available in a separate Stackless version that includes -support for micro-threads for massive concurrency. Read more about -it at the Stackless main site (we provide the same interface as the -standard Stackless Python), and at the greenlets page.

Support for Stackless and greenlets are now integrated in the normal +PyPy. More detailed information is available here.

Note that there is still an important performance hit for programs using +Stackless features.

Other features

django

Sandboxing: PyPy provides the ability to run untrusted code in a fully secure way.

Stackless: PyPy can be configured to run in stackless mode, +

Stackless: PyPy comes by default with support for stackless mode, providing micro-threads for massive concurrency.

As well as other features.

Branch: arm-backend-2 Changeset: r51309:bb83f459ff90 Date: 2012-01-13 14:58 +0100 http://bitbucket.org/pypy/pypy/changeset/bb83f459ff90/ Log: Implement case of moving a const float to the stack diff --git a/pypy/jit/backend/arm/assembler.py b/pypy/jit/backend/arm/assembler.py --- a/pypy/jit/backend/arm/assembler.py +++ b/pypy/jit/backend/arm/assembler.py @@ -856,12 +856,18 @@ assert 0, 'unsupported case' def _mov_imm_float_to_loc(self, prev_loc, loc, cond=c.AL): - if not loc.is_vfp_reg(): + if loc.is_vfp_reg(): + self.mc.PUSH([r.ip.value], cond=cond) + self.mc.gen_load_int(r.ip.value, prev_loc.getint(), cond=cond) + self.mc.VLDR(loc.value, r.ip.value, cond=cond) + self.mc.POP([r.ip.value], cond=cond) + elif loc.is_stack(): + self.regalloc_push(r.vfp_ip) + self.regalloc_mov(prev_loc, r.vfp_ip, cond) + self.regalloc_mov(r.vfp_ip, loc, cond) + self.regalloc_pop(r.vfp_ip) + else: assert 0, 'unsupported case' - self.mc.PUSH([r.ip.value], cond=cond) - self.mc.gen_load_int(r.ip.value, prev_loc.getint(), cond=cond) - self.mc.VLDR(loc.value, r.ip.value, cond=cond) - self.mc.POP([r.ip.value], cond=cond) def _mov_vfp_reg_to_loc(self, prev_loc, loc, cond=c.AL): if loc.is_vfp_reg(): From noreply at buildbot.pypy.org Sat Jan 14 17:19:25 2012 From: noreply at buildbot.pypy.org (bivab) Date: Sat, 14 Jan 2012 17:19:25 +0100 (CET) Subject: [pypy-commit] pypy arm-backend-2: implement as_key method for const float locations required for using const floats in jumps Message-ID: <20120114161925.6C7FA82B12@wyvern.cs.uni-duesseldorf.de> Author: David Schneider Branch: arm-backend-2 Changeset: r51310:44a83931f5cb Date: 2012-01-13 14:59 +0100 http://bitbucket.org/pypy/pypy/changeset/44a83931f5cb/ Log: implement as_key method for const float locations required for using const floats in jumps diff --git a/pypy/jit/backend/arm/locations.py b/pypy/jit/backend/arm/locations.py --- a/pypy/jit/backend/arm/locations.py +++ b/pypy/jit/backend/arm/locations.py @@ -100,6 +100,9 @@ def is_imm_float(self): return True + def as_key(self): + return self.value + class StackLocation(AssemblerLocation): _immutable_ = True From noreply at buildbot.pypy.org Sat Jan 14 17:19:26 2012 From: noreply at buildbot.pypy.org (bivab) Date: Sat, 14 Jan 2012 17:19:26 +0100 (CET) Subject: [pypy-commit] pypy arm-backend-2: remove a call that was done twice Message-ID: <20120114161926.990EF82B12@wyvern.cs.uni-duesseldorf.de> Author: David Schneider Branch: arm-backend-2 Changeset: r51311:8ff3eb80d78e Date: 2012-01-13 16:38 +0100 http://bitbucket.org/pypy/pypy/changeset/8ff3eb80d78e/ Log: remove a call that was done twice diff --git a/pypy/jit/backend/arm/regalloc.py b/pypy/jit/backend/arm/regalloc.py --- a/pypy/jit/backend/arm/regalloc.py +++ b/pypy/jit/backend/arm/regalloc.py @@ -951,14 +951,13 @@ self.rm.force_allocate_reg(t, selected_reg=r.r1) self.possibly_free_var(op.result) self.possibly_free_var(t) + return [imm(size)] - return [imm(size)] def get_mark_gc_roots(self, gcrootmap, use_copy_area=False): shape = gcrootmap.get_basic_shape(False) for v, val in self.frame_manager.bindings.items(): if (isinstance(v, BoxPtr) and self.rm.stays_alive(v)): assert val.is_stack() - gcrootmap.add_frame_offset(shape, val.position * -WORD) gcrootmap.add_frame_offset(shape, -val.value) for v, reg in self.rm.reg_bindings.items(): if reg is r.r0: From noreply at buildbot.pypy.org Sat Jan 14 17:19:27 2012 From: noreply at buildbot.pypy.org (bivab) Date: Sat, 14 Jan 2012 17:19:27 +0100 (CET) Subject: [pypy-commit] pypy arm-backend-2: as pointed out by David Edelsohn and Maciej, get rid of redundant calls to list Message-ID: <20120114161927.D70DE82B12@wyvern.cs.uni-duesseldorf.de> Author: David Schneider Branch: arm-backend-2 Changeset: r51312:483ce484be66 Date: 2012-01-14 16:34 +0100 http://bitbucket.org/pypy/pypy/changeset/483ce484be66/ Log: as pointed out by David Edelsohn and Maciej, get rid of redundant calls to list when accessing the list of inputargs of an operation. Also replace all calls to make_sure_var_in_reg with calls to _ensure_value_is_boxes which makes sure that all temporary boxes are also considered as forbidden. diff --git a/pypy/jit/backend/arm/helper/regalloc.py b/pypy/jit/backend/arm/helper/regalloc.py --- a/pypy/jit/backend/arm/helper/regalloc.py +++ b/pypy/jit/backend/arm/helper/regalloc.py @@ -33,9 +33,9 @@ imm_a1 = check_imm_box(a1, imm_size, allow_zero=allow_zero) if not imm_a0 and imm_a1: l0 = self._ensure_value_is_boxed(a0) - l1 = self.make_sure_var_in_reg(a1, boxes) + l1 = self._ensure_value_is_boxed(a1, boxes) elif commutative and imm_a0 and not imm_a1: - l1 = self.make_sure_var_in_reg(a0, boxes) + l1 = self._ensure_value_is_boxed(a0, boxes) l0 = self._ensure_value_is_boxed(a1, boxes) else: l0 = self._ensure_value_is_boxed(a0, boxes) @@ -90,8 +90,8 @@ assert fcond is not None a0 = op.getarg(0) a1 = op.getarg(1) - arg1 = self.make_sure_var_in_reg(a0, selected_reg=r.r0) - arg2 = self.make_sure_var_in_reg(a1, selected_reg=r.r1) + arg1 = self.rm.make_sure_var_in_reg(a0, selected_reg=r.r0) + arg2 = self.rm.make_sure_var_in_reg(a1, selected_reg=r.r1) assert arg1 == r.r0 assert arg2 == r.r1 if isinstance(a0, Box) and self.stays_alive(a0): @@ -113,7 +113,7 @@ l0 = self._ensure_value_is_boxed(arg0, forbidden_vars=boxes) if imm_a1: - l1 = self.make_sure_var_in_reg(arg1, boxes) + l1 = self._ensure_value_is_boxed(arg1, boxes) else: l1 = self._ensure_value_is_boxed(arg1, forbidden_vars=boxes) @@ -134,7 +134,7 @@ assert fcond is not None a0 = op.getarg(0) assert isinstance(a0, Box) - reg = self.make_sure_var_in_reg(a0) + reg = self._ensure_value_is_boxed(a0) self.possibly_free_vars_for_op(op) if guard_op is None: res = self.force_allocate_reg(op.result, [a0]) diff --git a/pypy/jit/backend/arm/opassembler.py b/pypy/jit/backend/arm/opassembler.py --- a/pypy/jit/backend/arm/opassembler.py +++ b/pypy/jit/backend/arm/opassembler.py @@ -892,7 +892,7 @@ # need the box here if isinstance(args[4], Box): length_box = args[4] - length_loc = regalloc.make_sure_var_in_reg(args[4], forbidden_vars) + length_loc = regalloc._ensure_value_is_boxed(args[4], forbidden_vars) else: length_box = TempInt() length_loc = regalloc.force_allocate_reg(length_box, diff --git a/pypy/jit/backend/arm/regalloc.py b/pypy/jit/backend/arm/regalloc.py --- a/pypy/jit/backend/arm/regalloc.py +++ b/pypy/jit/backend/arm/regalloc.py @@ -183,7 +183,7 @@ self.assembler.load(loc, immvalue) else: loc = self.make_sure_var_in_reg(thing, - forbidden_vars=forbidden_vars) + forbidden_vars=self.temp_boxes + forbidden_vars) return loc def get_scratch_reg(self, type=INT, forbidden_vars=[], selected_reg=None): @@ -285,12 +285,7 @@ def make_sure_var_in_reg(self, var, forbidden_vars=[], selected_reg=None, need_lower_byte=False): - if var.type == FLOAT: - return self.vfprm.make_sure_var_in_reg(var, forbidden_vars, - selected_reg, need_lower_byte) - else: - return self.rm.make_sure_var_in_reg(var, forbidden_vars, - selected_reg, need_lower_byte) + assert 0, 'should not be called directly' def convert_to_imm(self, value): if isinstance(value, ConstInt): @@ -312,7 +307,7 @@ def prepare_loop(self, inputargs, operations): self._prepare(inputargs, operations) self._set_initial_bindings(inputargs) - self.possibly_free_vars(list(inputargs)) + self.possibly_free_vars(inputargs) def prepare_bridge(self, inputargs, arglocs, ops): self._prepare(inputargs, ops) @@ -410,18 +405,18 @@ self.rm._sync_var(v) def _prepare_op_int_add(self, op, fcond): - boxes = list(op.getarglist()) + boxes = op.getarglist() a0, a1 = boxes imm_a0 = check_imm_box(a0) imm_a1 = check_imm_box(a1) if not imm_a0 and imm_a1: - l0 = self._ensure_value_is_boxed(a0) - l1 = self.make_sure_var_in_reg(a1, boxes) + l0 = self._ensure_value_is_boxed(a0, boxes) + l1 = self._ensure_value_is_boxed(a1, boxes) elif imm_a0 and not imm_a1: - l0 = self.make_sure_var_in_reg(a0) + l0 = self._ensure_value_is_boxed(a0, boxes) l1 = self._ensure_value_is_boxed(a1, boxes) else: - l0 = self._ensure_value_is_boxed(a0) + l0 = self._ensure_value_is_boxed(a0, boxes) l1 = self._ensure_value_is_boxed(a1, boxes) return [l0, l1] @@ -438,9 +433,9 @@ imm_a1 = check_imm_box(a1) if not imm_a0 and imm_a1: l0 = self._ensure_value_is_boxed(a0, boxes) - l1 = self.make_sure_var_in_reg(a1, boxes) + l1 = self._ensure_value_is_boxed(a1, boxes) elif imm_a0 and not imm_a1: - l0 = self.make_sure_var_in_reg(a0, boxes) + l0 = self._ensure_value_is_boxed(a0, boxes) l1 = self._ensure_value_is_boxed(a1, boxes) else: l0 = self._ensure_value_is_boxed(a0, boxes) @@ -455,7 +450,7 @@ return locs + [res] def prepare_op_int_mul(self, op, fcond): - boxes = list(op.getarglist()) + boxes = op.getarglist() a0, a1 = boxes reg1 = self._ensure_value_is_boxed(a0, forbidden_vars=boxes) @@ -597,14 +592,14 @@ prepare_op_guard_isnull = prepare_op_guard_true def prepare_op_guard_value(self, op, fcond): - boxes = list(op.getarglist()) + boxes = op.getarglist() a0, a1 = boxes imm_a1 = check_imm_box(a1) l0 = self._ensure_value_is_boxed(a0, boxes) if not imm_a1: l1 = self._ensure_value_is_boxed(a1, boxes) else: - l1 = self.make_sure_var_in_reg(a1, boxes) + l1 = self._ensure_value_is_boxed(a1, boxes) assert op.result is None arglocs = self._prepare_guard(op, [l0, l1]) self.possibly_free_vars(op.getarglist()) @@ -620,7 +615,7 @@ prepare_op_guard_not_invalidated = prepare_op_guard_no_overflow def prepare_op_guard_exception(self, op, fcond): - boxes = list(op.getarglist()) + boxes = op.getarglist() arg0 = ConstInt(rffi.cast(lltype.Signed, op.getarg(0).getint())) loc = self._ensure_value_is_boxed(arg0) loc1 = self.get_scratch_reg(INT, boxes) @@ -648,7 +643,7 @@ def _prepare_guard_class(self, op, fcond): assert isinstance(op.getarg(0), Box) - boxes = list(op.getarglist()) + boxes = op.getarglist() x = self._ensure_value_is_boxed(boxes[0], boxes) y = self.get_scratch_reg(REF, forbidden_vars=boxes) @@ -727,7 +722,7 @@ return [] def prepare_op_setfield_gc(self, op, fcond): - boxes = list(op.getarglist()) + boxes = op.getarglist() a0, a1 = boxes ofs, size, sign = unpack_fielddescr(op.getdescr()) base_loc = self._ensure_value_is_boxed(a0, boxes) @@ -806,23 +801,22 @@ return [res, base_loc, imm(ofs)] def prepare_op_setarrayitem_gc(self, op, fcond): - a0, a1, a2 = list(op.getarglist()) size, ofs, _ = unpack_arraydescr(op.getdescr()) scale = get_scale(size) args = op.getarglist() - base_loc = self._ensure_value_is_boxed(a0, args) - ofs_loc = self._ensure_value_is_boxed(a1, args) - value_loc = self._ensure_value_is_boxed(a2, args) + base_loc = self._ensure_value_is_boxed(args[0], args) + ofs_loc = self._ensure_value_is_boxed(args[1], args) + value_loc = self._ensure_value_is_boxed(args[2], args) assert check_imm_arg(ofs) return [value_loc, base_loc, ofs_loc, imm(scale), imm(ofs)] prepare_op_setarrayitem_raw = prepare_op_setarrayitem_gc def prepare_op_getarrayitem_gc(self, op, fcond): - a0, a1 = boxes = list(op.getarglist()) + boxes = op.getarglist() size, ofs, _ = unpack_arraydescr(op.getdescr()) scale = get_scale(size) - base_loc = self._ensure_value_is_boxed(a0, boxes) - ofs_loc = self._ensure_value_is_boxed(a1, boxes) + base_loc = self._ensure_value_is_boxed(boxes[0], boxes) + ofs_loc = self._ensure_value_is_boxed(boxes[1], boxes) self.possibly_free_vars_for_op(op) self.free_temp_vars() res = self.force_allocate_reg(op.result) @@ -852,13 +846,13 @@ return [l0, l1, res] def prepare_op_strgetitem(self, op, fcond): - boxes = list(op.getarglist()) + boxes = op.getarglist() base_loc = self._ensure_value_is_boxed(boxes[0]) a1 = boxes[1] imm_a1 = check_imm_box(a1) if imm_a1: - ofs_loc = self.make_sure_var_in_reg(a1, boxes) + ofs_loc = self._ensure_value_is_boxed(a1, boxes) else: ofs_loc = self._ensure_value_is_boxed(a1, boxes) @@ -872,7 +866,7 @@ return [res, base_loc, ofs_loc, imm(basesize)] def prepare_op_strsetitem(self, op, fcond): - boxes = list(op.getarglist()) + boxes = op.getarglist() base_loc = self._ensure_value_is_boxed(boxes[0], boxes) ofs_loc = self._ensure_value_is_boxed(boxes[1], boxes) value_loc = self._ensure_value_is_boxed(boxes[2], boxes) @@ -901,7 +895,7 @@ return [l0, l1, res] def prepare_op_unicodegetitem(self, op, fcond): - boxes = list(op.getarglist()) + boxes = op.getarglist() base_loc = self._ensure_value_is_boxed(boxes[0], boxes) ofs_loc = self._ensure_value_is_boxed(boxes[1], boxes) @@ -916,7 +910,7 @@ imm(scale), imm(basesize), imm(itemsize)] def prepare_op_unicodesetitem(self, op, fcond): - boxes = list(op.getarglist()) + boxes = op.getarglist() base_loc = self._ensure_value_is_boxed(boxes[0], boxes) ofs_loc = self._ensure_value_is_boxed(boxes[1], boxes) value_loc = self._ensure_value_is_boxed(boxes[2], boxes) @@ -930,7 +924,7 @@ arg = op.getarg(0) imm_arg = check_imm_box(arg) if imm_arg: - argloc = self.make_sure_var_in_reg(arg) + argloc = self._ensure_value_is_boxed(arg) else: argloc = self._ensure_value_is_boxed(arg) self.possibly_free_vars_for_op(op) From noreply at buildbot.pypy.org Sat Jan 14 20:07:50 2012 From: noreply at buildbot.pypy.org (stefanor) Date: Sat, 14 Jan 2012 20:07:50 +0100 (CET) Subject: [pypy-commit] pypy default: Big endian test case for test_utf_16_encode_decode Message-ID: <20120114190750.A77FA82B12@wyvern.cs.uni-duesseldorf.de> Author: Stefano Rivera Branch: Changeset: r51313:c6a06cbab53d Date: 2012-01-14 21:07 +0200 http://bitbucket.org/pypy/pypy/changeset/c6a06cbab53d/ Log: Big endian test case for test_utf_16_encode_decode diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py --- a/pypy/module/_codecs/test/test_codecs.py +++ b/pypy/module/_codecs/test/test_codecs.py @@ -588,10 +588,18 @@ raises(UnicodeDecodeError, '+3ADYAA-'.decode, 'utf-7') def test_utf_16_encode_decode(self): - import codecs + import codecs, sys x = u'123abc' - assert codecs.getencoder('utf-16')(x) == ('\xff\xfe1\x002\x003\x00a\x00b\x00c\x00', 6) - assert codecs.getdecoder('utf-16')('\xff\xfe1\x002\x003\x00a\x00b\x00c\x00') == (x, 14) + if sys.byteorder == 'big': + assert codecs.getencoder('utf-16')(x) == ( + '\xfe\xff\x001\x002\x003\x00a\x00b\x00c', 6) + assert codecs.getdecoder('utf-16')( + '\xfe\xff\x001\x002\x003\x00a\x00b\x00c') == (x, 14) + else: + assert codecs.getencoder('utf-16')(x) == ( + '\xff\xfe1\x002\x003\x00a\x00b\x00c\x00', 6) + assert codecs.getdecoder('utf-16')( + '\xff\xfe1\x002\x003\x00a\x00b\x00c\x00') == (x, 14) def test_unicode_escape(self): assert u'\\'.encode('unicode-escape') == '\\\\' From noreply at buildbot.pypy.org Sat Jan 14 20:37:17 2012 From: noreply at buildbot.pypy.org (mattip) Date: Sat, 14 Jan 2012 20:37:17 +0100 (CET) Subject: [pypy-commit] pypy numpypy-reshape: fix for 1-length shapes Message-ID: <20120114193717.D139082B12@wyvern.cs.uni-duesseldorf.de> Author: mattip Branch: numpypy-reshape Changeset: r51314:38824caea2ca Date: 2012-01-14 21:34 +0200 http://bitbucket.org/pypy/pypy/changeset/38824caea2ca/ Log: fix for 1-length shapes diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -157,9 +157,6 @@ # (meaning that the realignment of elements crosses from one step into another) # return None so that the caller can raise an exception. def calc_new_strides(new_shape, old_shape, old_strides): - # Return the proper strides for new_shape, or None if the mapping crosses - # stepping boundaries - # Assumes that prod(old_shape) == prod(new_shape), len(old_shape) > 1, and # len(new_shape) > 0 steps = [] @@ -167,6 +164,7 @@ oldI = 0 new_strides = [] if old_strides[0] < old_strides[-1]: + #Start at old_shape[0], old_stides[0] for i in range(len(old_shape)): steps.append(old_strides[i] / last_step) last_step *= old_shape[i] @@ -184,10 +182,11 @@ if n_new_elems_used == n_old_elems_to_use: oldI += 1 if oldI >= len(old_shape): - break + continue cur_step = steps[oldI] n_old_elems_to_use *= old_shape[oldI] else: + #Start at old_shape[-1], old_strides[-1] for i in range(len(old_shape) - 1, -1, -1): steps.insert(0, old_strides[i] / last_step) last_step *= old_shape[i] @@ -207,7 +206,7 @@ if n_new_elems_used == n_old_elems_to_use: oldI -= 1 if oldI < -len(old_shape): - break + continue cur_step = steps[oldI] n_old_elems_to_use *= old_shape[oldI] return new_strides diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -157,6 +157,8 @@ assert calc_new_strides([2, 3, 4], [8, 3], [1, 16]) is None assert calc_new_strides([24], [2, 4, 3], [48, 6, 1]) is None assert calc_new_strides([24], [2, 4, 3], [24, 6, 2]) == [2] + assert calc_new_strides([105, 1], [3, 5, 7], [35, 7, 1]) == [1, 1] + assert calc_new_strides([1, 105], [3, 5, 7], [35, 7, 1]) == [105, 1] class AppTestNumArray(BaseNumpyAppTest): @@ -765,7 +767,6 @@ assert (a[:, 1, :].sum(1) == [70, 315, 560]).all() raises (ValueError, 'a[:, 1, :].sum(2)') assert ((a + a).T.sum(2).T == (a + a).sum(0)).all() - skip("Those are broken on reshape, fix!") assert (a.reshape(1,-1).sum(0) == range(105)).all() assert (a.reshape(1,-1).sum(1) == 5460) @@ -1556,3 +1557,7 @@ a = range(12) b = reshape(a, (3, 4)) assert b.shape == (3, 4) + a = array(range(105)).reshape(3, 5, 7) + assert a.reshape(1, -1).shape == (1, 105) + assert a.reshape(1, 1, -1).shape == (1, 1, 105) + assert a.reshape(-1, 1, 1).shape == (105, 1, 1) From noreply at buildbot.pypy.org Sat Jan 14 20:40:47 2012 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 14 Jan 2012 20:40:47 +0100 (CET) Subject: [pypy-commit] pypy default: merge numpypy-reshape Message-ID: <20120114194047.83ACA82B12@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r51315:c9f77f542246 Date: 2012-01-14 21:40 +0200 http://bitbucket.org/pypy/pypy/changeset/c9f77f542246/ Log: merge numpypy-reshape diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -157,9 +157,6 @@ # (meaning that the realignment of elements crosses from one step into another) # return None so that the caller can raise an exception. def calc_new_strides(new_shape, old_shape, old_strides): - # Return the proper strides for new_shape, or None if the mapping crosses - # stepping boundaries - # Assumes that prod(old_shape) == prod(new_shape), len(old_shape) > 1, and # len(new_shape) > 0 steps = [] @@ -167,6 +164,7 @@ oldI = 0 new_strides = [] if old_strides[0] < old_strides[-1]: + #Start at old_shape[0], old_stides[0] for i in range(len(old_shape)): steps.append(old_strides[i] / last_step) last_step *= old_shape[i] @@ -184,10 +182,11 @@ if n_new_elems_used == n_old_elems_to_use: oldI += 1 if oldI >= len(old_shape): - break + continue cur_step = steps[oldI] n_old_elems_to_use *= old_shape[oldI] else: + #Start at old_shape[-1], old_strides[-1] for i in range(len(old_shape) - 1, -1, -1): steps.insert(0, old_strides[i] / last_step) last_step *= old_shape[i] @@ -207,7 +206,7 @@ if n_new_elems_used == n_old_elems_to_use: oldI -= 1 if oldI < -len(old_shape): - break + continue cur_step = steps[oldI] n_old_elems_to_use *= old_shape[oldI] return new_strides diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -157,6 +157,8 @@ assert calc_new_strides([2, 3, 4], [8, 3], [1, 16]) is None assert calc_new_strides([24], [2, 4, 3], [48, 6, 1]) is None assert calc_new_strides([24], [2, 4, 3], [24, 6, 2]) == [2] + assert calc_new_strides([105, 1], [3, 5, 7], [35, 7, 1]) == [1, 1] + assert calc_new_strides([1, 105], [3, 5, 7], [35, 7, 1]) == [105, 1] class AppTestNumArray(BaseNumpyAppTest): @@ -765,7 +767,6 @@ assert (a[:, 1, :].sum(1) == [70, 315, 560]).all() raises (ValueError, 'a[:, 1, :].sum(2)') assert ((a + a).T.sum(2).T == (a + a).sum(0)).all() - skip("Those are broken on reshape, fix!") assert (a.reshape(1,-1).sum(0) == range(105)).all() assert (a.reshape(1,-1).sum(1) == 5460) @@ -1556,3 +1557,7 @@ a = range(12) b = reshape(a, (3, 4)) assert b.shape == (3, 4) + a = array(range(105)).reshape(3, 5, 7) + assert a.reshape(1, -1).shape == (1, 105) + assert a.reshape(1, 1, -1).shape == (1, 1, 105) + assert a.reshape(-1, 1, 1).shape == (105, 1, 1) From noreply at buildbot.pypy.org Sat Jan 14 20:40:48 2012 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 14 Jan 2012 20:40:48 +0100 (CET) Subject: [pypy-commit] pypy numpypy-reshape: close merged branch Message-ID: <20120114194048.AF82C82B12@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: numpypy-reshape Changeset: r51316:edc019f4b1b8 Date: 2012-01-14 21:40 +0200 http://bitbucket.org/pypy/pypy/changeset/edc019f4b1b8/ Log: close merged branch From notifications-noreply at bitbucket.org Sat Jan 14 21:26:56 2012 From: notifications-noreply at bitbucket.org (Bitbucket) Date: Sat, 14 Jan 2012 20:26:56 -0000 Subject: [pypy-commit] Notification: pypy Message-ID: <20120114202656.17002.30760@bitbucket05.managed.contegix.com> You have received a notification from benol. Hi, I forked pypy. My fork is at https://bitbucket.org/benol/pypy. -- Disable notifications at https://bitbucket.org/account/notifications/ From noreply at buildbot.pypy.org Sat Jan 14 21:48:25 2012 From: noreply at buildbot.pypy.org (amauryfa) Date: Sat, 14 Jan 2012 21:48:25 +0100 (CET) Subject: [pypy-commit] pypy py3k: New Unicode database properties: .isxidstart and .isxidcontinue Message-ID: <20120114204825.91B0282B12@wyvern.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3k Changeset: r51317:9779b8a0e896 Date: 2011-12-22 10:32 +0100 http://bitbucket.org/pypy/pypy/changeset/9779b8a0e896/ Log: New Unicode database properties: .isxidstart and .isxidcontinue diff too long, truncating to 10000 out of 116033 lines diff --git a/pypy/module/unicodedata/DerivedCoreProperties-5.2.0.txt b/pypy/module/unicodedata/DerivedCoreProperties-5.2.0.txt new file mode 100644 --- /dev/null +++ b/pypy/module/unicodedata/DerivedCoreProperties-5.2.0.txt @@ -0,0 +1,9243 @@ +# DerivedCoreProperties-5.2.0.txt +# Date: 2009-08-26, 00:45:22 GMT [MD] +# +# Unicode Character Database +# Copyright (c) 1991-2009 Unicode, Inc. +# For terms of use, see http://www.unicode.org/terms_of_use.html +# For documentation, see http://www.unicode.org/reports/tr44/ + +# ================================================ + +# Derived Property: Math +# Generated from: Sm + Other_Math + +002B ; Math # Sm PLUS SIGN +003C..003E ; Math # Sm [3] LESS-THAN SIGN..GREATER-THAN SIGN +005E ; Math # Sk CIRCUMFLEX ACCENT +007C ; Math # Sm VERTICAL LINE +007E ; Math # Sm TILDE +00AC ; Math # Sm NOT SIGN +00B1 ; Math # Sm PLUS-MINUS SIGN +00D7 ; Math # Sm MULTIPLICATION SIGN +00F7 ; Math # Sm DIVISION SIGN +03D0..03D2 ; Math # L& [3] GREEK BETA SYMBOL..GREEK UPSILON WITH HOOK SYMBOL +03D5 ; Math # L& GREEK PHI SYMBOL +03F0..03F1 ; Math # L& [2] GREEK KAPPA SYMBOL..GREEK RHO SYMBOL +03F4..03F5 ; Math # L& [2] GREEK CAPITAL THETA SYMBOL..GREEK LUNATE EPSILON SYMBOL +03F6 ; Math # Sm GREEK REVERSED LUNATE EPSILON SYMBOL +0606..0608 ; Math # Sm [3] ARABIC-INDIC CUBE ROOT..ARABIC RAY +2016 ; Math # Po DOUBLE VERTICAL LINE +2032..2034 ; Math # Po [3] PRIME..TRIPLE PRIME +2040 ; Math # Pc CHARACTER TIE +2044 ; Math # Sm FRACTION SLASH +2052 ; Math # Sm COMMERCIAL MINUS SIGN +2061..2064 ; Math # Cf [4] FUNCTION APPLICATION..INVISIBLE PLUS +207A..207C ; Math # Sm [3] SUPERSCRIPT PLUS SIGN..SUPERSCRIPT EQUALS SIGN +207D ; Math # Ps SUPERSCRIPT LEFT PARENTHESIS +207E ; Math # Pe SUPERSCRIPT RIGHT PARENTHESIS +208A..208C ; Math # Sm [3] SUBSCRIPT PLUS SIGN..SUBSCRIPT EQUALS SIGN +208D ; Math # Ps SUBSCRIPT LEFT PARENTHESIS +208E ; Math # Pe SUBSCRIPT RIGHT PARENTHESIS +20D0..20DC ; Math # Mn [13] COMBINING LEFT HARPOON ABOVE..COMBINING FOUR DOTS ABOVE +20E1 ; Math # Mn COMBINING LEFT RIGHT ARROW ABOVE +20E5..20E6 ; Math # Mn [2] COMBINING REVERSE SOLIDUS OVERLAY..COMBINING DOUBLE VERTICAL STROKE OVERLAY +20EB..20EF ; Math # Mn [5] COMBINING LONG DOUBLE SOLIDUS OVERLAY..COMBINING RIGHT ARROW BELOW +2102 ; Math # L& DOUBLE-STRUCK CAPITAL C +210A..2113 ; Math # L& [10] SCRIPT SMALL G..SCRIPT SMALL L +2115 ; Math # L& DOUBLE-STRUCK CAPITAL N +2119..211D ; Math # L& [5] DOUBLE-STRUCK CAPITAL P..DOUBLE-STRUCK CAPITAL R +2124 ; Math # L& DOUBLE-STRUCK CAPITAL Z +2128 ; Math # L& BLACK-LETTER CAPITAL Z +2129 ; Math # So TURNED GREEK SMALL LETTER IOTA +212C..212D ; Math # L& [2] SCRIPT CAPITAL B..BLACK-LETTER CAPITAL C +212F..2131 ; Math # L& [3] SCRIPT SMALL E..SCRIPT CAPITAL F +2133..2134 ; Math # L& [2] SCRIPT CAPITAL M..SCRIPT SMALL O +2135..2138 ; Math # Lo [4] ALEF SYMBOL..DALET SYMBOL +213C..213F ; Math # L& [4] DOUBLE-STRUCK SMALL PI..DOUBLE-STRUCK CAPITAL PI +2140..2144 ; Math # Sm [5] DOUBLE-STRUCK N-ARY SUMMATION..TURNED SANS-SERIF CAPITAL Y +2145..2149 ; Math # L& [5] DOUBLE-STRUCK ITALIC CAPITAL D..DOUBLE-STRUCK ITALIC SMALL J +214B ; Math # Sm TURNED AMPERSAND +2190..2194 ; Math # Sm [5] LEFTWARDS ARROW..LEFT RIGHT ARROW +2195..2199 ; Math # So [5] UP DOWN ARROW..SOUTH WEST ARROW +219A..219B ; Math # Sm [2] LEFTWARDS ARROW WITH STROKE..RIGHTWARDS ARROW WITH STROKE +219C..219F ; Math # So [4] LEFTWARDS WAVE ARROW..UPWARDS TWO HEADED ARROW +21A0 ; Math # Sm RIGHTWARDS TWO HEADED ARROW +21A1..21A2 ; Math # So [2] DOWNWARDS TWO HEADED ARROW..LEFTWARDS ARROW WITH TAIL +21A3 ; Math # Sm RIGHTWARDS ARROW WITH TAIL +21A4..21A5 ; Math # So [2] LEFTWARDS ARROW FROM BAR..UPWARDS ARROW FROM BAR +21A6 ; Math # Sm RIGHTWARDS ARROW FROM BAR +21A7 ; Math # So DOWNWARDS ARROW FROM BAR +21A9..21AD ; Math # So [5] LEFTWARDS ARROW WITH HOOK..LEFT RIGHT WAVE ARROW +21AE ; Math # Sm LEFT RIGHT ARROW WITH STROKE +21B0..21B1 ; Math # So [2] UPWARDS ARROW WITH TIP LEFTWARDS..UPWARDS ARROW WITH TIP RIGHTWARDS +21B6..21B7 ; Math # So [2] ANTICLOCKWISE TOP SEMICIRCLE ARROW..CLOCKWISE TOP SEMICIRCLE ARROW +21BC..21CD ; Math # So [18] LEFTWARDS HARPOON WITH BARB UPWARDS..LEFTWARDS DOUBLE ARROW WITH STROKE +21CE..21CF ; Math # Sm [2] LEFT RIGHT DOUBLE ARROW WITH STROKE..RIGHTWARDS DOUBLE ARROW WITH STROKE +21D0..21D1 ; Math # So [2] LEFTWARDS DOUBLE ARROW..UPWARDS DOUBLE ARROW +21D2 ; Math # Sm RIGHTWARDS DOUBLE ARROW +21D3 ; Math # So DOWNWARDS DOUBLE ARROW +21D4 ; Math # Sm LEFT RIGHT DOUBLE ARROW +21D5..21DB ; Math # So [7] UP DOWN DOUBLE ARROW..RIGHTWARDS TRIPLE ARROW +21DD ; Math # So RIGHTWARDS SQUIGGLE ARROW +21E4..21E5 ; Math # So [2] LEFTWARDS ARROW TO BAR..RIGHTWARDS ARROW TO BAR +21F4..22FF ; Math # Sm [268] RIGHT ARROW WITH SMALL CIRCLE..Z NOTATION BAG MEMBERSHIP +2308..230B ; Math # Sm [4] LEFT CEILING..RIGHT FLOOR +2320..2321 ; Math # Sm [2] TOP HALF INTEGRAL..BOTTOM HALF INTEGRAL +237C ; Math # Sm RIGHT ANGLE WITH DOWNWARDS ZIGZAG ARROW +239B..23B3 ; Math # Sm [25] LEFT PARENTHESIS UPPER HOOK..SUMMATION BOTTOM +23B4..23B5 ; Math # So [2] TOP SQUARE BRACKET..BOTTOM SQUARE BRACKET +23B7 ; Math # So RADICAL SYMBOL BOTTOM +23D0 ; Math # So VERTICAL LINE EXTENSION +23DC..23E1 ; Math # Sm [6] TOP PARENTHESIS..BOTTOM TORTOISE SHELL BRACKET +23E2 ; Math # So WHITE TRAPEZIUM +25A0..25A1 ; Math # So [2] BLACK SQUARE..WHITE SQUARE +25AE..25B6 ; Math # So [9] BLACK VERTICAL RECTANGLE..BLACK RIGHT-POINTING TRIANGLE +25B7 ; Math # Sm WHITE RIGHT-POINTING TRIANGLE +25BC..25C0 ; Math # So [5] BLACK DOWN-POINTING TRIANGLE..BLACK LEFT-POINTING TRIANGLE +25C1 ; Math # Sm WHITE LEFT-POINTING TRIANGLE +25C6..25C7 ; Math # So [2] BLACK DIAMOND..WHITE DIAMOND +25CA..25CB ; Math # So [2] LOZENGE..WHITE CIRCLE +25CF..25D3 ; Math # So [5] BLACK CIRCLE..CIRCLE WITH UPPER HALF BLACK +25E2 ; Math # So BLACK LOWER RIGHT TRIANGLE +25E4 ; Math # So BLACK UPPER LEFT TRIANGLE +25E7..25EC ; Math # So [6] SQUARE WITH LEFT HALF BLACK..WHITE UP-POINTING TRIANGLE WITH DOT +25F8..25FF ; Math # Sm [8] UPPER LEFT TRIANGLE..LOWER RIGHT TRIANGLE +2605..2606 ; Math # So [2] BLACK STAR..WHITE STAR +2640 ; Math # So FEMALE SIGN +2642 ; Math # So MALE SIGN +2660..2663 ; Math # So [4] BLACK SPADE SUIT..BLACK CLUB SUIT +266D..266E ; Math # So [2] MUSIC FLAT SIGN..MUSIC NATURAL SIGN +266F ; Math # Sm MUSIC SHARP SIGN +27C0..27C4 ; Math # Sm [5] THREE DIMENSIONAL ANGLE..OPEN SUPERSET +27C5 ; Math # Ps LEFT S-SHAPED BAG DELIMITER +27C6 ; Math # Pe RIGHT S-SHAPED BAG DELIMITER +27C7..27CA ; Math # Sm [4] OR WITH DOT INSIDE..VERTICAL BAR WITH HORIZONTAL STROKE +27CC ; Math # Sm LONG DIVISION +27D0..27E5 ; Math # Sm [22] WHITE DIAMOND WITH CENTRED DOT..WHITE SQUARE WITH RIGHTWARDS TICK +27E6 ; Math # Ps MATHEMATICAL LEFT WHITE SQUARE BRACKET +27E7 ; Math # Pe MATHEMATICAL RIGHT WHITE SQUARE BRACKET +27E8 ; Math # Ps MATHEMATICAL LEFT ANGLE BRACKET +27E9 ; Math # Pe MATHEMATICAL RIGHT ANGLE BRACKET +27EA ; Math # Ps MATHEMATICAL LEFT DOUBLE ANGLE BRACKET +27EB ; Math # Pe MATHEMATICAL RIGHT DOUBLE ANGLE BRACKET +27EC ; Math # Ps MATHEMATICAL LEFT WHITE TORTOISE SHELL BRACKET +27ED ; Math # Pe MATHEMATICAL RIGHT WHITE TORTOISE SHELL BRACKET +27EE ; Math # Ps MATHEMATICAL LEFT FLATTENED PARENTHESIS +27EF ; Math # Pe MATHEMATICAL RIGHT FLATTENED PARENTHESIS +27F0..27FF ; Math # Sm [16] UPWARDS QUADRUPLE ARROW..LONG RIGHTWARDS SQUIGGLE ARROW +2900..2982 ; Math # Sm [131] RIGHTWARDS TWO-HEADED ARROW WITH VERTICAL STROKE..Z NOTATION TYPE COLON +2983 ; Math # Ps LEFT WHITE CURLY BRACKET +2984 ; Math # Pe RIGHT WHITE CURLY BRACKET +2985 ; Math # Ps LEFT WHITE PARENTHESIS +2986 ; Math # Pe RIGHT WHITE PARENTHESIS +2987 ; Math # Ps Z NOTATION LEFT IMAGE BRACKET +2988 ; Math # Pe Z NOTATION RIGHT IMAGE BRACKET +2989 ; Math # Ps Z NOTATION LEFT BINDING BRACKET +298A ; Math # Pe Z NOTATION RIGHT BINDING BRACKET +298B ; Math # Ps LEFT SQUARE BRACKET WITH UNDERBAR +298C ; Math # Pe RIGHT SQUARE BRACKET WITH UNDERBAR +298D ; Math # Ps LEFT SQUARE BRACKET WITH TICK IN TOP CORNER +298E ; Math # Pe RIGHT SQUARE BRACKET WITH TICK IN BOTTOM CORNER +298F ; Math # Ps LEFT SQUARE BRACKET WITH TICK IN BOTTOM CORNER +2990 ; Math # Pe RIGHT SQUARE BRACKET WITH TICK IN TOP CORNER +2991 ; Math # Ps LEFT ANGLE BRACKET WITH DOT +2992 ; Math # Pe RIGHT ANGLE BRACKET WITH DOT +2993 ; Math # Ps LEFT ARC LESS-THAN BRACKET +2994 ; Math # Pe RIGHT ARC GREATER-THAN BRACKET +2995 ; Math # Ps DOUBLE LEFT ARC GREATER-THAN BRACKET +2996 ; Math # Pe DOUBLE RIGHT ARC LESS-THAN BRACKET +2997 ; Math # Ps LEFT BLACK TORTOISE SHELL BRACKET +2998 ; Math # Pe RIGHT BLACK TORTOISE SHELL BRACKET +2999..29D7 ; Math # Sm [63] DOTTED FENCE..BLACK HOURGLASS +29D8 ; Math # Ps LEFT WIGGLY FENCE +29D9 ; Math # Pe RIGHT WIGGLY FENCE +29DA ; Math # Ps LEFT DOUBLE WIGGLY FENCE +29DB ; Math # Pe RIGHT DOUBLE WIGGLY FENCE +29DC..29FB ; Math # Sm [32] INCOMPLETE INFINITY..TRIPLE PLUS +29FC ; Math # Ps LEFT-POINTING CURVED ANGLE BRACKET +29FD ; Math # Pe RIGHT-POINTING CURVED ANGLE BRACKET +29FE..2AFF ; Math # Sm [258] TINY..N-ARY WHITE VERTICAL BAR +2B30..2B44 ; Math # Sm [21] LEFT ARROW WITH SMALL CIRCLE..RIGHTWARDS ARROW THROUGH SUPERSET +2B47..2B4C ; Math # Sm [6] REVERSE TILDE OPERATOR ABOVE RIGHTWARDS ARROW..RIGHTWARDS ARROW ABOVE REVERSE TILDE OPERATOR +FB29 ; Math # Sm HEBREW LETTER ALTERNATIVE PLUS SIGN +FE61 ; Math # Po SMALL ASTERISK +FE62 ; Math # Sm SMALL PLUS SIGN +FE63 ; Math # Pd SMALL HYPHEN-MINUS +FE64..FE66 ; Math # Sm [3] SMALL LESS-THAN SIGN..SMALL EQUALS SIGN +FE68 ; Math # Po SMALL REVERSE SOLIDUS +FF0B ; Math # Sm FULLWIDTH PLUS SIGN +FF1C..FF1E ; Math # Sm [3] FULLWIDTH LESS-THAN SIGN..FULLWIDTH GREATER-THAN SIGN +FF3C ; Math # Po FULLWIDTH REVERSE SOLIDUS +FF3E ; Math # Sk FULLWIDTH CIRCUMFLEX ACCENT +FF5C ; Math # Sm FULLWIDTH VERTICAL LINE +FF5E ; Math # Sm FULLWIDTH TILDE +FFE2 ; Math # Sm FULLWIDTH NOT SIGN +FFE9..FFEC ; Math # Sm [4] HALFWIDTH LEFTWARDS ARROW..HALFWIDTH DOWNWARDS ARROW +1D400..1D454 ; Math # L& [85] MATHEMATICAL BOLD CAPITAL A..MATHEMATICAL ITALIC SMALL G +1D456..1D49C ; Math # L& [71] MATHEMATICAL ITALIC SMALL I..MATHEMATICAL SCRIPT CAPITAL A +1D49E..1D49F ; Math # L& [2] MATHEMATICAL SCRIPT CAPITAL C..MATHEMATICAL SCRIPT CAPITAL D +1D4A2 ; Math # L& MATHEMATICAL SCRIPT CAPITAL G +1D4A5..1D4A6 ; Math # L& [2] MATHEMATICAL SCRIPT CAPITAL J..MATHEMATICAL SCRIPT CAPITAL K +1D4A9..1D4AC ; Math # L& [4] MATHEMATICAL SCRIPT CAPITAL N..MATHEMATICAL SCRIPT CAPITAL Q +1D4AE..1D4B9 ; Math # L& [12] MATHEMATICAL SCRIPT CAPITAL S..MATHEMATICAL SCRIPT SMALL D +1D4BB ; Math # L& MATHEMATICAL SCRIPT SMALL F +1D4BD..1D4C3 ; Math # L& [7] MATHEMATICAL SCRIPT SMALL H..MATHEMATICAL SCRIPT SMALL N +1D4C5..1D505 ; Math # L& [65] MATHEMATICAL SCRIPT SMALL P..MATHEMATICAL FRAKTUR CAPITAL B +1D507..1D50A ; Math # L& [4] MATHEMATICAL FRAKTUR CAPITAL D..MATHEMATICAL FRAKTUR CAPITAL G +1D50D..1D514 ; Math # L& [8] MATHEMATICAL FRAKTUR CAPITAL J..MATHEMATICAL FRAKTUR CAPITAL Q +1D516..1D51C ; Math # L& [7] MATHEMATICAL FRAKTUR CAPITAL S..MATHEMATICAL FRAKTUR CAPITAL Y +1D51E..1D539 ; Math # L& [28] MATHEMATICAL FRAKTUR SMALL A..MATHEMATICAL DOUBLE-STRUCK CAPITAL B +1D53B..1D53E ; Math # L& [4] MATHEMATICAL DOUBLE-STRUCK CAPITAL D..MATHEMATICAL DOUBLE-STRUCK CAPITAL G +1D540..1D544 ; Math # L& [5] MATHEMATICAL DOUBLE-STRUCK CAPITAL I..MATHEMATICAL DOUBLE-STRUCK CAPITAL M +1D546 ; Math # L& MATHEMATICAL DOUBLE-STRUCK CAPITAL O +1D54A..1D550 ; Math # L& [7] MATHEMATICAL DOUBLE-STRUCK CAPITAL S..MATHEMATICAL DOUBLE-STRUCK CAPITAL Y +1D552..1D6A5 ; Math # L& [340] MATHEMATICAL DOUBLE-STRUCK SMALL A..MATHEMATICAL ITALIC SMALL DOTLESS J +1D6A8..1D6C0 ; Math # L& [25] MATHEMATICAL BOLD CAPITAL ALPHA..MATHEMATICAL BOLD CAPITAL OMEGA +1D6C1 ; Math # Sm MATHEMATICAL BOLD NABLA +1D6C2..1D6DA ; Math # L& [25] MATHEMATICAL BOLD SMALL ALPHA..MATHEMATICAL BOLD SMALL OMEGA +1D6DB ; Math # Sm MATHEMATICAL BOLD PARTIAL DIFFERENTIAL +1D6DC..1D6FA ; Math # L& [31] MATHEMATICAL BOLD EPSILON SYMBOL..MATHEMATICAL ITALIC CAPITAL OMEGA +1D6FB ; Math # Sm MATHEMATICAL ITALIC NABLA +1D6FC..1D714 ; Math # L& [25] MATHEMATICAL ITALIC SMALL ALPHA..MATHEMATICAL ITALIC SMALL OMEGA +1D715 ; Math # Sm MATHEMATICAL ITALIC PARTIAL DIFFERENTIAL +1D716..1D734 ; Math # L& [31] MATHEMATICAL ITALIC EPSILON SYMBOL..MATHEMATICAL BOLD ITALIC CAPITAL OMEGA +1D735 ; Math # Sm MATHEMATICAL BOLD ITALIC NABLA +1D736..1D74E ; Math # L& [25] MATHEMATICAL BOLD ITALIC SMALL ALPHA..MATHEMATICAL BOLD ITALIC SMALL OMEGA +1D74F ; Math # Sm MATHEMATICAL BOLD ITALIC PARTIAL DIFFERENTIAL +1D750..1D76E ; Math # L& [31] MATHEMATICAL BOLD ITALIC EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD CAPITAL OMEGA +1D76F ; Math # Sm MATHEMATICAL SANS-SERIF BOLD NABLA +1D770..1D788 ; Math # L& [25] MATHEMATICAL SANS-SERIF BOLD SMALL ALPHA..MATHEMATICAL SANS-SERIF BOLD SMALL OMEGA +1D789 ; Math # Sm MATHEMATICAL SANS-SERIF BOLD PARTIAL DIFFERENTIAL +1D78A..1D7A8 ; Math # L& [31] MATHEMATICAL SANS-SERIF BOLD EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL OMEGA +1D7A9 ; Math # Sm MATHEMATICAL SANS-SERIF BOLD ITALIC NABLA +1D7AA..1D7C2 ; Math # L& [25] MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL ALPHA..MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL OMEGA +1D7C3 ; Math # Sm MATHEMATICAL SANS-SERIF BOLD ITALIC PARTIAL DIFFERENTIAL +1D7C4..1D7CB ; Math # L& [8] MATHEMATICAL SANS-SERIF BOLD ITALIC EPSILON SYMBOL..MATHEMATICAL BOLD SMALL DIGAMMA +1D7CE..1D7FF ; Math # Nd [50] MATHEMATICAL BOLD DIGIT ZERO..MATHEMATICAL MONOSPACE DIGIT NINE + +# Total code points: 2161 + +# ================================================ + +# Derived Property: Alphabetic +# Generated from: Lu+Ll+Lt+Lm+Lo+Nl + Other_Alphabetic + +0041..005A ; Alphabetic # L& [26] LATIN CAPITAL LETTER A..LATIN CAPITAL LETTER Z +0061..007A ; Alphabetic # L& [26] LATIN SMALL LETTER A..LATIN SMALL LETTER Z +00AA ; Alphabetic # L& FEMININE ORDINAL INDICATOR +00B5 ; Alphabetic # L& MICRO SIGN +00BA ; Alphabetic # L& MASCULINE ORDINAL INDICATOR +00C0..00D6 ; Alphabetic # L& [23] LATIN CAPITAL LETTER A WITH GRAVE..LATIN CAPITAL LETTER O WITH DIAERESIS +00D8..00F6 ; Alphabetic # L& [31] LATIN CAPITAL LETTER O WITH STROKE..LATIN SMALL LETTER O WITH DIAERESIS +00F8..01BA ; Alphabetic # L& [195] LATIN SMALL LETTER O WITH STROKE..LATIN SMALL LETTER EZH WITH TAIL +01BB ; Alphabetic # Lo LATIN LETTER TWO WITH STROKE +01BC..01BF ; Alphabetic # L& [4] LATIN CAPITAL LETTER TONE FIVE..LATIN LETTER WYNN +01C0..01C3 ; Alphabetic # Lo [4] LATIN LETTER DENTAL CLICK..LATIN LETTER RETROFLEX CLICK +01C4..0293 ; Alphabetic # L& [208] LATIN CAPITAL LETTER DZ WITH CARON..LATIN SMALL LETTER EZH WITH CURL +0294 ; Alphabetic # Lo LATIN LETTER GLOTTAL STOP +0295..02AF ; Alphabetic # L& [27] LATIN LETTER PHARYNGEAL VOICED FRICATIVE..LATIN SMALL LETTER TURNED H WITH FISHHOOK AND TAIL +02B0..02C1 ; Alphabetic # Lm [18] MODIFIER LETTER SMALL H..MODIFIER LETTER REVERSED GLOTTAL STOP +02C6..02D1 ; Alphabetic # Lm [12] MODIFIER LETTER CIRCUMFLEX ACCENT..MODIFIER LETTER HALF TRIANGULAR COLON +02E0..02E4 ; Alphabetic # Lm [5] MODIFIER LETTER SMALL GAMMA..MODIFIER LETTER SMALL REVERSED GLOTTAL STOP +02EC ; Alphabetic # Lm MODIFIER LETTER VOICING +02EE ; Alphabetic # Lm MODIFIER LETTER DOUBLE APOSTROPHE +0345 ; Alphabetic # Mn COMBINING GREEK YPOGEGRAMMENI +0370..0373 ; Alphabetic # L& [4] GREEK CAPITAL LETTER HETA..GREEK SMALL LETTER ARCHAIC SAMPI +0374 ; Alphabetic # Lm GREEK NUMERAL SIGN +0376..0377 ; Alphabetic # L& [2] GREEK CAPITAL LETTER PAMPHYLIAN DIGAMMA..GREEK SMALL LETTER PAMPHYLIAN DIGAMMA +037A ; Alphabetic # Lm GREEK YPOGEGRAMMENI +037B..037D ; Alphabetic # L& [3] GREEK SMALL REVERSED LUNATE SIGMA SYMBOL..GREEK SMALL REVERSED DOTTED LUNATE SIGMA SYMBOL +0386 ; Alphabetic # L& GREEK CAPITAL LETTER ALPHA WITH TONOS +0388..038A ; Alphabetic # L& [3] GREEK CAPITAL LETTER EPSILON WITH TONOS..GREEK CAPITAL LETTER IOTA WITH TONOS +038C ; Alphabetic # L& GREEK CAPITAL LETTER OMICRON WITH TONOS +038E..03A1 ; Alphabetic # L& [20] GREEK CAPITAL LETTER UPSILON WITH TONOS..GREEK CAPITAL LETTER RHO +03A3..03F5 ; Alphabetic # L& [83] GREEK CAPITAL LETTER SIGMA..GREEK LUNATE EPSILON SYMBOL +03F7..0481 ; Alphabetic # L& [139] GREEK CAPITAL LETTER SHO..CYRILLIC SMALL LETTER KOPPA +048A..0525 ; Alphabetic # L& [156] CYRILLIC CAPITAL LETTER SHORT I WITH TAIL..CYRILLIC SMALL LETTER PE WITH DESCENDER +0531..0556 ; Alphabetic # L& [38] ARMENIAN CAPITAL LETTER AYB..ARMENIAN CAPITAL LETTER FEH +0559 ; Alphabetic # Lm ARMENIAN MODIFIER LETTER LEFT HALF RING +0561..0587 ; Alphabetic # L& [39] ARMENIAN SMALL LETTER AYB..ARMENIAN SMALL LIGATURE ECH YIWN +05B0..05BD ; Alphabetic # Mn [14] HEBREW POINT SHEVA..HEBREW POINT METEG +05BF ; Alphabetic # Mn HEBREW POINT RAFE +05C1..05C2 ; Alphabetic # Mn [2] HEBREW POINT SHIN DOT..HEBREW POINT SIN DOT +05C4..05C5 ; Alphabetic # Mn [2] HEBREW MARK UPPER DOT..HEBREW MARK LOWER DOT +05C7 ; Alphabetic # Mn HEBREW POINT QAMATS QATAN +05D0..05EA ; Alphabetic # Lo [27] HEBREW LETTER ALEF..HEBREW LETTER TAV +05F0..05F2 ; Alphabetic # Lo [3] HEBREW LIGATURE YIDDISH DOUBLE VAV..HEBREW LIGATURE YIDDISH DOUBLE YOD +0610..061A ; Alphabetic # Mn [11] ARABIC SIGN SALLALLAHOU ALAYHE WASSALLAM..ARABIC SMALL KASRA +0621..063F ; Alphabetic # Lo [31] ARABIC LETTER HAMZA..ARABIC LETTER FARSI YEH WITH THREE DOTS ABOVE +0640 ; Alphabetic # Lm ARABIC TATWEEL +0641..064A ; Alphabetic # Lo [10] ARABIC LETTER FEH..ARABIC LETTER YEH +064B..0657 ; Alphabetic # Mn [13] ARABIC FATHATAN..ARABIC INVERTED DAMMA +0659..065E ; Alphabetic # Mn [6] ARABIC ZWARAKAY..ARABIC FATHA WITH TWO DOTS +066E..066F ; Alphabetic # Lo [2] ARABIC LETTER DOTLESS BEH..ARABIC LETTER DOTLESS QAF +0670 ; Alphabetic # Mn ARABIC LETTER SUPERSCRIPT ALEF +0671..06D3 ; Alphabetic # Lo [99] ARABIC LETTER ALEF WASLA..ARABIC LETTER YEH BARREE WITH HAMZA ABOVE +06D5 ; Alphabetic # Lo ARABIC LETTER AE +06D6..06DC ; Alphabetic # Mn [7] ARABIC SMALL HIGH LIGATURE SAD WITH LAM WITH ALEF MAKSURA..ARABIC SMALL HIGH SEEN +06E1..06E4 ; Alphabetic # Mn [4] ARABIC SMALL HIGH DOTLESS HEAD OF KHAH..ARABIC SMALL HIGH MADDA +06E5..06E6 ; Alphabetic # Lm [2] ARABIC SMALL WAW..ARABIC SMALL YEH +06E7..06E8 ; Alphabetic # Mn [2] ARABIC SMALL HIGH YEH..ARABIC SMALL HIGH NOON +06ED ; Alphabetic # Mn ARABIC SMALL LOW MEEM +06EE..06EF ; Alphabetic # Lo [2] ARABIC LETTER DAL WITH INVERTED V..ARABIC LETTER REH WITH INVERTED V +06FA..06FC ; Alphabetic # Lo [3] ARABIC LETTER SHEEN WITH DOT BELOW..ARABIC LETTER GHAIN WITH DOT BELOW +06FF ; Alphabetic # Lo ARABIC LETTER HEH WITH INVERTED V +0710 ; Alphabetic # Lo SYRIAC LETTER ALAPH +0711 ; Alphabetic # Mn SYRIAC LETTER SUPERSCRIPT ALAPH +0712..072F ; Alphabetic # Lo [30] SYRIAC LETTER BETH..SYRIAC LETTER PERSIAN DHALATH +0730..073F ; Alphabetic # Mn [16] SYRIAC PTHAHA ABOVE..SYRIAC RWAHA +074D..07A5 ; Alphabetic # Lo [89] SYRIAC LETTER SOGDIAN ZHAIN..THAANA LETTER WAAVU +07A6..07B0 ; Alphabetic # Mn [11] THAANA ABAFILI..THAANA SUKUN +07B1 ; Alphabetic # Lo THAANA LETTER NAA +07CA..07EA ; Alphabetic # Lo [33] NKO LETTER A..NKO LETTER JONA RA +07F4..07F5 ; Alphabetic # Lm [2] NKO HIGH TONE APOSTROPHE..NKO LOW TONE APOSTROPHE +07FA ; Alphabetic # Lm NKO LAJANYALAN +0800..0815 ; Alphabetic # Lo [22] SAMARITAN LETTER ALAF..SAMARITAN LETTER TAAF +0816..0817 ; Alphabetic # Mn [2] SAMARITAN MARK IN..SAMARITAN MARK IN-ALAF +081A ; Alphabetic # Lm SAMARITAN MODIFIER LETTER EPENTHETIC YUT +081B..0823 ; Alphabetic # Mn [9] SAMARITAN MARK EPENTHETIC YUT..SAMARITAN VOWEL SIGN A +0824 ; Alphabetic # Lm SAMARITAN MODIFIER LETTER SHORT A +0825..0827 ; Alphabetic # Mn [3] SAMARITAN VOWEL SIGN SHORT A..SAMARITAN VOWEL SIGN U +0828 ; Alphabetic # Lm SAMARITAN MODIFIER LETTER I +0829..082C ; Alphabetic # Mn [4] SAMARITAN VOWEL SIGN LONG I..SAMARITAN VOWEL SIGN SUKUN +0900..0902 ; Alphabetic # Mn [3] DEVANAGARI SIGN INVERTED CANDRABINDU..DEVANAGARI SIGN ANUSVARA +0903 ; Alphabetic # Mc DEVANAGARI SIGN VISARGA +0904..0939 ; Alphabetic # Lo [54] DEVANAGARI LETTER SHORT A..DEVANAGARI LETTER HA +093D ; Alphabetic # Lo DEVANAGARI SIGN AVAGRAHA +093E..0940 ; Alphabetic # Mc [3] DEVANAGARI VOWEL SIGN AA..DEVANAGARI VOWEL SIGN II +0941..0948 ; Alphabetic # Mn [8] DEVANAGARI VOWEL SIGN U..DEVANAGARI VOWEL SIGN AI +0949..094C ; Alphabetic # Mc [4] DEVANAGARI VOWEL SIGN CANDRA O..DEVANAGARI VOWEL SIGN AU +094E ; Alphabetic # Mc DEVANAGARI VOWEL SIGN PRISHTHAMATRA E +0950 ; Alphabetic # Lo DEVANAGARI OM +0955 ; Alphabetic # Mn DEVANAGARI VOWEL SIGN CANDRA LONG E +0958..0961 ; Alphabetic # Lo [10] DEVANAGARI LETTER QA..DEVANAGARI LETTER VOCALIC LL +0962..0963 ; Alphabetic # Mn [2] DEVANAGARI VOWEL SIGN VOCALIC L..DEVANAGARI VOWEL SIGN VOCALIC LL +0971 ; Alphabetic # Lm DEVANAGARI SIGN HIGH SPACING DOT +0972 ; Alphabetic # Lo DEVANAGARI LETTER CANDRA A +0979..097F ; Alphabetic # Lo [7] DEVANAGARI LETTER ZHA..DEVANAGARI LETTER BBA +0981 ; Alphabetic # Mn BENGALI SIGN CANDRABINDU +0982..0983 ; Alphabetic # Mc [2] BENGALI SIGN ANUSVARA..BENGALI SIGN VISARGA +0985..098C ; Alphabetic # Lo [8] BENGALI LETTER A..BENGALI LETTER VOCALIC L +098F..0990 ; Alphabetic # Lo [2] BENGALI LETTER E..BENGALI LETTER AI +0993..09A8 ; Alphabetic # Lo [22] BENGALI LETTER O..BENGALI LETTER NA +09AA..09B0 ; Alphabetic # Lo [7] BENGALI LETTER PA..BENGALI LETTER RA +09B2 ; Alphabetic # Lo BENGALI LETTER LA +09B6..09B9 ; Alphabetic # Lo [4] BENGALI LETTER SHA..BENGALI LETTER HA +09BD ; Alphabetic # Lo BENGALI SIGN AVAGRAHA +09BE..09C0 ; Alphabetic # Mc [3] BENGALI VOWEL SIGN AA..BENGALI VOWEL SIGN II +09C1..09C4 ; Alphabetic # Mn [4] BENGALI VOWEL SIGN U..BENGALI VOWEL SIGN VOCALIC RR +09C7..09C8 ; Alphabetic # Mc [2] BENGALI VOWEL SIGN E..BENGALI VOWEL SIGN AI +09CB..09CC ; Alphabetic # Mc [2] BENGALI VOWEL SIGN O..BENGALI VOWEL SIGN AU +09CE ; Alphabetic # Lo BENGALI LETTER KHANDA TA +09D7 ; Alphabetic # Mc BENGALI AU LENGTH MARK +09DC..09DD ; Alphabetic # Lo [2] BENGALI LETTER RRA..BENGALI LETTER RHA +09DF..09E1 ; Alphabetic # Lo [3] BENGALI LETTER YYA..BENGALI LETTER VOCALIC LL +09E2..09E3 ; Alphabetic # Mn [2] BENGALI VOWEL SIGN VOCALIC L..BENGALI VOWEL SIGN VOCALIC LL +09F0..09F1 ; Alphabetic # Lo [2] BENGALI LETTER RA WITH MIDDLE DIAGONAL..BENGALI LETTER RA WITH LOWER DIAGONAL +0A01..0A02 ; Alphabetic # Mn [2] GURMUKHI SIGN ADAK BINDI..GURMUKHI SIGN BINDI +0A03 ; Alphabetic # Mc GURMUKHI SIGN VISARGA +0A05..0A0A ; Alphabetic # Lo [6] GURMUKHI LETTER A..GURMUKHI LETTER UU +0A0F..0A10 ; Alphabetic # Lo [2] GURMUKHI LETTER EE..GURMUKHI LETTER AI +0A13..0A28 ; Alphabetic # Lo [22] GURMUKHI LETTER OO..GURMUKHI LETTER NA +0A2A..0A30 ; Alphabetic # Lo [7] GURMUKHI LETTER PA..GURMUKHI LETTER RA +0A32..0A33 ; Alphabetic # Lo [2] GURMUKHI LETTER LA..GURMUKHI LETTER LLA +0A35..0A36 ; Alphabetic # Lo [2] GURMUKHI LETTER VA..GURMUKHI LETTER SHA +0A38..0A39 ; Alphabetic # Lo [2] GURMUKHI LETTER SA..GURMUKHI LETTER HA +0A3E..0A40 ; Alphabetic # Mc [3] GURMUKHI VOWEL SIGN AA..GURMUKHI VOWEL SIGN II +0A41..0A42 ; Alphabetic # Mn [2] GURMUKHI VOWEL SIGN U..GURMUKHI VOWEL SIGN UU +0A47..0A48 ; Alphabetic # Mn [2] GURMUKHI VOWEL SIGN EE..GURMUKHI VOWEL SIGN AI +0A4B..0A4C ; Alphabetic # Mn [2] GURMUKHI VOWEL SIGN OO..GURMUKHI VOWEL SIGN AU +0A51 ; Alphabetic # Mn GURMUKHI SIGN UDAAT +0A59..0A5C ; Alphabetic # Lo [4] GURMUKHI LETTER KHHA..GURMUKHI LETTER RRA +0A5E ; Alphabetic # Lo GURMUKHI LETTER FA +0A70..0A71 ; Alphabetic # Mn [2] GURMUKHI TIPPI..GURMUKHI ADDAK +0A72..0A74 ; Alphabetic # Lo [3] GURMUKHI IRI..GURMUKHI EK ONKAR +0A75 ; Alphabetic # Mn GURMUKHI SIGN YAKASH +0A81..0A82 ; Alphabetic # Mn [2] GUJARATI SIGN CANDRABINDU..GUJARATI SIGN ANUSVARA +0A83 ; Alphabetic # Mc GUJARATI SIGN VISARGA +0A85..0A8D ; Alphabetic # Lo [9] GUJARATI LETTER A..GUJARATI VOWEL CANDRA E +0A8F..0A91 ; Alphabetic # Lo [3] GUJARATI LETTER E..GUJARATI VOWEL CANDRA O +0A93..0AA8 ; Alphabetic # Lo [22] GUJARATI LETTER O..GUJARATI LETTER NA +0AAA..0AB0 ; Alphabetic # Lo [7] GUJARATI LETTER PA..GUJARATI LETTER RA +0AB2..0AB3 ; Alphabetic # Lo [2] GUJARATI LETTER LA..GUJARATI LETTER LLA +0AB5..0AB9 ; Alphabetic # Lo [5] GUJARATI LETTER VA..GUJARATI LETTER HA +0ABD ; Alphabetic # Lo GUJARATI SIGN AVAGRAHA +0ABE..0AC0 ; Alphabetic # Mc [3] GUJARATI VOWEL SIGN AA..GUJARATI VOWEL SIGN II +0AC1..0AC5 ; Alphabetic # Mn [5] GUJARATI VOWEL SIGN U..GUJARATI VOWEL SIGN CANDRA E +0AC7..0AC8 ; Alphabetic # Mn [2] GUJARATI VOWEL SIGN E..GUJARATI VOWEL SIGN AI +0AC9 ; Alphabetic # Mc GUJARATI VOWEL SIGN CANDRA O +0ACB..0ACC ; Alphabetic # Mc [2] GUJARATI VOWEL SIGN O..GUJARATI VOWEL SIGN AU +0AD0 ; Alphabetic # Lo GUJARATI OM +0AE0..0AE1 ; Alphabetic # Lo [2] GUJARATI LETTER VOCALIC RR..GUJARATI LETTER VOCALIC LL +0AE2..0AE3 ; Alphabetic # Mn [2] GUJARATI VOWEL SIGN VOCALIC L..GUJARATI VOWEL SIGN VOCALIC LL +0B01 ; Alphabetic # Mn ORIYA SIGN CANDRABINDU +0B02..0B03 ; Alphabetic # Mc [2] ORIYA SIGN ANUSVARA..ORIYA SIGN VISARGA +0B05..0B0C ; Alphabetic # Lo [8] ORIYA LETTER A..ORIYA LETTER VOCALIC L +0B0F..0B10 ; Alphabetic # Lo [2] ORIYA LETTER E..ORIYA LETTER AI +0B13..0B28 ; Alphabetic # Lo [22] ORIYA LETTER O..ORIYA LETTER NA +0B2A..0B30 ; Alphabetic # Lo [7] ORIYA LETTER PA..ORIYA LETTER RA +0B32..0B33 ; Alphabetic # Lo [2] ORIYA LETTER LA..ORIYA LETTER LLA +0B35..0B39 ; Alphabetic # Lo [5] ORIYA LETTER VA..ORIYA LETTER HA +0B3D ; Alphabetic # Lo ORIYA SIGN AVAGRAHA +0B3E ; Alphabetic # Mc ORIYA VOWEL SIGN AA +0B3F ; Alphabetic # Mn ORIYA VOWEL SIGN I +0B40 ; Alphabetic # Mc ORIYA VOWEL SIGN II +0B41..0B44 ; Alphabetic # Mn [4] ORIYA VOWEL SIGN U..ORIYA VOWEL SIGN VOCALIC RR +0B47..0B48 ; Alphabetic # Mc [2] ORIYA VOWEL SIGN E..ORIYA VOWEL SIGN AI +0B4B..0B4C ; Alphabetic # Mc [2] ORIYA VOWEL SIGN O..ORIYA VOWEL SIGN AU +0B56 ; Alphabetic # Mn ORIYA AI LENGTH MARK +0B57 ; Alphabetic # Mc ORIYA AU LENGTH MARK +0B5C..0B5D ; Alphabetic # Lo [2] ORIYA LETTER RRA..ORIYA LETTER RHA +0B5F..0B61 ; Alphabetic # Lo [3] ORIYA LETTER YYA..ORIYA LETTER VOCALIC LL +0B62..0B63 ; Alphabetic # Mn [2] ORIYA VOWEL SIGN VOCALIC L..ORIYA VOWEL SIGN VOCALIC LL +0B71 ; Alphabetic # Lo ORIYA LETTER WA +0B82 ; Alphabetic # Mn TAMIL SIGN ANUSVARA +0B83 ; Alphabetic # Lo TAMIL SIGN VISARGA +0B85..0B8A ; Alphabetic # Lo [6] TAMIL LETTER A..TAMIL LETTER UU +0B8E..0B90 ; Alphabetic # Lo [3] TAMIL LETTER E..TAMIL LETTER AI +0B92..0B95 ; Alphabetic # Lo [4] TAMIL LETTER O..TAMIL LETTER KA +0B99..0B9A ; Alphabetic # Lo [2] TAMIL LETTER NGA..TAMIL LETTER CA +0B9C ; Alphabetic # Lo TAMIL LETTER JA +0B9E..0B9F ; Alphabetic # Lo [2] TAMIL LETTER NYA..TAMIL LETTER TTA +0BA3..0BA4 ; Alphabetic # Lo [2] TAMIL LETTER NNA..TAMIL LETTER TA +0BA8..0BAA ; Alphabetic # Lo [3] TAMIL LETTER NA..TAMIL LETTER PA +0BAE..0BB9 ; Alphabetic # Lo [12] TAMIL LETTER MA..TAMIL LETTER HA +0BBE..0BBF ; Alphabetic # Mc [2] TAMIL VOWEL SIGN AA..TAMIL VOWEL SIGN I +0BC0 ; Alphabetic # Mn TAMIL VOWEL SIGN II +0BC1..0BC2 ; Alphabetic # Mc [2] TAMIL VOWEL SIGN U..TAMIL VOWEL SIGN UU +0BC6..0BC8 ; Alphabetic # Mc [3] TAMIL VOWEL SIGN E..TAMIL VOWEL SIGN AI +0BCA..0BCC ; Alphabetic # Mc [3] TAMIL VOWEL SIGN O..TAMIL VOWEL SIGN AU +0BD0 ; Alphabetic # Lo TAMIL OM +0BD7 ; Alphabetic # Mc TAMIL AU LENGTH MARK +0C01..0C03 ; Alphabetic # Mc [3] TELUGU SIGN CANDRABINDU..TELUGU SIGN VISARGA +0C05..0C0C ; Alphabetic # Lo [8] TELUGU LETTER A..TELUGU LETTER VOCALIC L +0C0E..0C10 ; Alphabetic # Lo [3] TELUGU LETTER E..TELUGU LETTER AI +0C12..0C28 ; Alphabetic # Lo [23] TELUGU LETTER O..TELUGU LETTER NA +0C2A..0C33 ; Alphabetic # Lo [10] TELUGU LETTER PA..TELUGU LETTER LLA +0C35..0C39 ; Alphabetic # Lo [5] TELUGU LETTER VA..TELUGU LETTER HA +0C3D ; Alphabetic # Lo TELUGU SIGN AVAGRAHA +0C3E..0C40 ; Alphabetic # Mn [3] TELUGU VOWEL SIGN AA..TELUGU VOWEL SIGN II +0C41..0C44 ; Alphabetic # Mc [4] TELUGU VOWEL SIGN U..TELUGU VOWEL SIGN VOCALIC RR +0C46..0C48 ; Alphabetic # Mn [3] TELUGU VOWEL SIGN E..TELUGU VOWEL SIGN AI +0C4A..0C4C ; Alphabetic # Mn [3] TELUGU VOWEL SIGN O..TELUGU VOWEL SIGN AU +0C55..0C56 ; Alphabetic # Mn [2] TELUGU LENGTH MARK..TELUGU AI LENGTH MARK +0C58..0C59 ; Alphabetic # Lo [2] TELUGU LETTER TSA..TELUGU LETTER DZA +0C60..0C61 ; Alphabetic # Lo [2] TELUGU LETTER VOCALIC RR..TELUGU LETTER VOCALIC LL +0C62..0C63 ; Alphabetic # Mn [2] TELUGU VOWEL SIGN VOCALIC L..TELUGU VOWEL SIGN VOCALIC LL +0C82..0C83 ; Alphabetic # Mc [2] KANNADA SIGN ANUSVARA..KANNADA SIGN VISARGA +0C85..0C8C ; Alphabetic # Lo [8] KANNADA LETTER A..KANNADA LETTER VOCALIC L +0C8E..0C90 ; Alphabetic # Lo [3] KANNADA LETTER E..KANNADA LETTER AI +0C92..0CA8 ; Alphabetic # Lo [23] KANNADA LETTER O..KANNADA LETTER NA +0CAA..0CB3 ; Alphabetic # Lo [10] KANNADA LETTER PA..KANNADA LETTER LLA +0CB5..0CB9 ; Alphabetic # Lo [5] KANNADA LETTER VA..KANNADA LETTER HA +0CBD ; Alphabetic # Lo KANNADA SIGN AVAGRAHA +0CBE ; Alphabetic # Mc KANNADA VOWEL SIGN AA +0CBF ; Alphabetic # Mn KANNADA VOWEL SIGN I +0CC0..0CC4 ; Alphabetic # Mc [5] KANNADA VOWEL SIGN II..KANNADA VOWEL SIGN VOCALIC RR +0CC6 ; Alphabetic # Mn KANNADA VOWEL SIGN E +0CC7..0CC8 ; Alphabetic # Mc [2] KANNADA VOWEL SIGN EE..KANNADA VOWEL SIGN AI +0CCA..0CCB ; Alphabetic # Mc [2] KANNADA VOWEL SIGN O..KANNADA VOWEL SIGN OO +0CCC ; Alphabetic # Mn KANNADA VOWEL SIGN AU +0CD5..0CD6 ; Alphabetic # Mc [2] KANNADA LENGTH MARK..KANNADA AI LENGTH MARK +0CDE ; Alphabetic # Lo KANNADA LETTER FA +0CE0..0CE1 ; Alphabetic # Lo [2] KANNADA LETTER VOCALIC RR..KANNADA LETTER VOCALIC LL +0CE2..0CE3 ; Alphabetic # Mn [2] KANNADA VOWEL SIGN VOCALIC L..KANNADA VOWEL SIGN VOCALIC LL +0D02..0D03 ; Alphabetic # Mc [2] MALAYALAM SIGN ANUSVARA..MALAYALAM SIGN VISARGA +0D05..0D0C ; Alphabetic # Lo [8] MALAYALAM LETTER A..MALAYALAM LETTER VOCALIC L +0D0E..0D10 ; Alphabetic # Lo [3] MALAYALAM LETTER E..MALAYALAM LETTER AI +0D12..0D28 ; Alphabetic # Lo [23] MALAYALAM LETTER O..MALAYALAM LETTER NA +0D2A..0D39 ; Alphabetic # Lo [16] MALAYALAM LETTER PA..MALAYALAM LETTER HA +0D3D ; Alphabetic # Lo MALAYALAM SIGN AVAGRAHA +0D3E..0D40 ; Alphabetic # Mc [3] MALAYALAM VOWEL SIGN AA..MALAYALAM VOWEL SIGN II +0D41..0D44 ; Alphabetic # Mn [4] MALAYALAM VOWEL SIGN U..MALAYALAM VOWEL SIGN VOCALIC RR +0D46..0D48 ; Alphabetic # Mc [3] MALAYALAM VOWEL SIGN E..MALAYALAM VOWEL SIGN AI +0D4A..0D4C ; Alphabetic # Mc [3] MALAYALAM VOWEL SIGN O..MALAYALAM VOWEL SIGN AU +0D57 ; Alphabetic # Mc MALAYALAM AU LENGTH MARK +0D60..0D61 ; Alphabetic # Lo [2] MALAYALAM LETTER VOCALIC RR..MALAYALAM LETTER VOCALIC LL +0D62..0D63 ; Alphabetic # Mn [2] MALAYALAM VOWEL SIGN VOCALIC L..MALAYALAM VOWEL SIGN VOCALIC LL +0D7A..0D7F ; Alphabetic # Lo [6] MALAYALAM LETTER CHILLU NN..MALAYALAM LETTER CHILLU K +0D82..0D83 ; Alphabetic # Mc [2] SINHALA SIGN ANUSVARAYA..SINHALA SIGN VISARGAYA +0D85..0D96 ; Alphabetic # Lo [18] SINHALA LETTER AYANNA..SINHALA LETTER AUYANNA +0D9A..0DB1 ; Alphabetic # Lo [24] SINHALA LETTER ALPAPRAANA KAYANNA..SINHALA LETTER DANTAJA NAYANNA +0DB3..0DBB ; Alphabetic # Lo [9] SINHALA LETTER SANYAKA DAYANNA..SINHALA LETTER RAYANNA +0DBD ; Alphabetic # Lo SINHALA LETTER DANTAJA LAYANNA +0DC0..0DC6 ; Alphabetic # Lo [7] SINHALA LETTER VAYANNA..SINHALA LETTER FAYANNA +0DCF..0DD1 ; Alphabetic # Mc [3] SINHALA VOWEL SIGN AELA-PILLA..SINHALA VOWEL SIGN DIGA AEDA-PILLA +0DD2..0DD4 ; Alphabetic # Mn [3] SINHALA VOWEL SIGN KETTI IS-PILLA..SINHALA VOWEL SIGN KETTI PAA-PILLA +0DD6 ; Alphabetic # Mn SINHALA VOWEL SIGN DIGA PAA-PILLA +0DD8..0DDF ; Alphabetic # Mc [8] SINHALA VOWEL SIGN GAETTA-PILLA..SINHALA VOWEL SIGN GAYANUKITTA +0DF2..0DF3 ; Alphabetic # Mc [2] SINHALA VOWEL SIGN DIGA GAETTA-PILLA..SINHALA VOWEL SIGN DIGA GAYANUKITTA +0E01..0E30 ; Alphabetic # Lo [48] THAI CHARACTER KO KAI..THAI CHARACTER SARA A +0E31 ; Alphabetic # Mn THAI CHARACTER MAI HAN-AKAT +0E32..0E33 ; Alphabetic # Lo [2] THAI CHARACTER SARA AA..THAI CHARACTER SARA AM +0E34..0E3A ; Alphabetic # Mn [7] THAI CHARACTER SARA I..THAI CHARACTER PHINTHU +0E40..0E45 ; Alphabetic # Lo [6] THAI CHARACTER SARA E..THAI CHARACTER LAKKHANGYAO +0E46 ; Alphabetic # Lm THAI CHARACTER MAIYAMOK +0E4D ; Alphabetic # Mn THAI CHARACTER NIKHAHIT +0E81..0E82 ; Alphabetic # Lo [2] LAO LETTER KO..LAO LETTER KHO SUNG +0E84 ; Alphabetic # Lo LAO LETTER KHO TAM +0E87..0E88 ; Alphabetic # Lo [2] LAO LETTER NGO..LAO LETTER CO +0E8A ; Alphabetic # Lo LAO LETTER SO TAM +0E8D ; Alphabetic # Lo LAO LETTER NYO +0E94..0E97 ; Alphabetic # Lo [4] LAO LETTER DO..LAO LETTER THO TAM +0E99..0E9F ; Alphabetic # Lo [7] LAO LETTER NO..LAO LETTER FO SUNG +0EA1..0EA3 ; Alphabetic # Lo [3] LAO LETTER MO..LAO LETTER LO LING +0EA5 ; Alphabetic # Lo LAO LETTER LO LOOT +0EA7 ; Alphabetic # Lo LAO LETTER WO +0EAA..0EAB ; Alphabetic # Lo [2] LAO LETTER SO SUNG..LAO LETTER HO SUNG +0EAD..0EB0 ; Alphabetic # Lo [4] LAO LETTER O..LAO VOWEL SIGN A +0EB1 ; Alphabetic # Mn LAO VOWEL SIGN MAI KAN +0EB2..0EB3 ; Alphabetic # Lo [2] LAO VOWEL SIGN AA..LAO VOWEL SIGN AM +0EB4..0EB9 ; Alphabetic # Mn [6] LAO VOWEL SIGN I..LAO VOWEL SIGN UU +0EBB..0EBC ; Alphabetic # Mn [2] LAO VOWEL SIGN MAI KON..LAO SEMIVOWEL SIGN LO +0EBD ; Alphabetic # Lo LAO SEMIVOWEL SIGN NYO +0EC0..0EC4 ; Alphabetic # Lo [5] LAO VOWEL SIGN E..LAO VOWEL SIGN AI +0EC6 ; Alphabetic # Lm LAO KO LA +0ECD ; Alphabetic # Mn LAO NIGGAHITA +0EDC..0EDD ; Alphabetic # Lo [2] LAO HO NO..LAO HO MO +0F00 ; Alphabetic # Lo TIBETAN SYLLABLE OM +0F40..0F47 ; Alphabetic # Lo [8] TIBETAN LETTER KA..TIBETAN LETTER JA +0F49..0F6C ; Alphabetic # Lo [36] TIBETAN LETTER NYA..TIBETAN LETTER RRA +0F71..0F7E ; Alphabetic # Mn [14] TIBETAN VOWEL SIGN AA..TIBETAN SIGN RJES SU NGA RO +0F7F ; Alphabetic # Mc TIBETAN SIGN RNAM BCAD +0F80..0F81 ; Alphabetic # Mn [2] TIBETAN VOWEL SIGN REVERSED I..TIBETAN VOWEL SIGN REVERSED II +0F88..0F8B ; Alphabetic # Lo [4] TIBETAN SIGN LCE TSA CAN..TIBETAN SIGN GRU MED RGYINGS +0F90..0F97 ; Alphabetic # Mn [8] TIBETAN SUBJOINED LETTER KA..TIBETAN SUBJOINED LETTER JA +0F99..0FBC ; Alphabetic # Mn [36] TIBETAN SUBJOINED LETTER NYA..TIBETAN SUBJOINED LETTER FIXED-FORM RA +1000..102A ; Alphabetic # Lo [43] MYANMAR LETTER KA..MYANMAR LETTER AU +102B..102C ; Alphabetic # Mc [2] MYANMAR VOWEL SIGN TALL AA..MYANMAR VOWEL SIGN AA +102D..1030 ; Alphabetic # Mn [4] MYANMAR VOWEL SIGN I..MYANMAR VOWEL SIGN UU +1031 ; Alphabetic # Mc MYANMAR VOWEL SIGN E +1032..1036 ; Alphabetic # Mn [5] MYANMAR VOWEL SIGN AI..MYANMAR SIGN ANUSVARA +1038 ; Alphabetic # Mc MYANMAR SIGN VISARGA +103B..103C ; Alphabetic # Mc [2] MYANMAR CONSONANT SIGN MEDIAL YA..MYANMAR CONSONANT SIGN MEDIAL RA +103D..103E ; Alphabetic # Mn [2] MYANMAR CONSONANT SIGN MEDIAL WA..MYANMAR CONSONANT SIGN MEDIAL HA +103F ; Alphabetic # Lo MYANMAR LETTER GREAT SA +1050..1055 ; Alphabetic # Lo [6] MYANMAR LETTER SHA..MYANMAR LETTER VOCALIC LL +1056..1057 ; Alphabetic # Mc [2] MYANMAR VOWEL SIGN VOCALIC R..MYANMAR VOWEL SIGN VOCALIC RR +1058..1059 ; Alphabetic # Mn [2] MYANMAR VOWEL SIGN VOCALIC L..MYANMAR VOWEL SIGN VOCALIC LL +105A..105D ; Alphabetic # Lo [4] MYANMAR LETTER MON NGA..MYANMAR LETTER MON BBE +105E..1060 ; Alphabetic # Mn [3] MYANMAR CONSONANT SIGN MON MEDIAL NA..MYANMAR CONSONANT SIGN MON MEDIAL LA +1061 ; Alphabetic # Lo MYANMAR LETTER SGAW KAREN SHA +1062 ; Alphabetic # Mc MYANMAR VOWEL SIGN SGAW KAREN EU +1065..1066 ; Alphabetic # Lo [2] MYANMAR LETTER WESTERN PWO KAREN THA..MYANMAR LETTER WESTERN PWO KAREN PWA +1067..1068 ; Alphabetic # Mc [2] MYANMAR VOWEL SIGN WESTERN PWO KAREN EU..MYANMAR VOWEL SIGN WESTERN PWO KAREN UE +106E..1070 ; Alphabetic # Lo [3] MYANMAR LETTER EASTERN PWO KAREN NNA..MYANMAR LETTER EASTERN PWO KAREN GHWA +1071..1074 ; Alphabetic # Mn [4] MYANMAR VOWEL SIGN GEBA KAREN I..MYANMAR VOWEL SIGN KAYAH EE +1075..1081 ; Alphabetic # Lo [13] MYANMAR LETTER SHAN KA..MYANMAR LETTER SHAN HA +1082 ; Alphabetic # Mn MYANMAR CONSONANT SIGN SHAN MEDIAL WA +1083..1084 ; Alphabetic # Mc [2] MYANMAR VOWEL SIGN SHAN AA..MYANMAR VOWEL SIGN SHAN E +1085..1086 ; Alphabetic # Mn [2] MYANMAR VOWEL SIGN SHAN E ABOVE..MYANMAR VOWEL SIGN SHAN FINAL Y +108E ; Alphabetic # Lo MYANMAR LETTER RUMAI PALAUNG FA +109C ; Alphabetic # Mc MYANMAR VOWEL SIGN AITON A +109D ; Alphabetic # Mn MYANMAR VOWEL SIGN AITON AI +10A0..10C5 ; Alphabetic # L& [38] GEORGIAN CAPITAL LETTER AN..GEORGIAN CAPITAL LETTER HOE +10D0..10FA ; Alphabetic # Lo [43] GEORGIAN LETTER AN..GEORGIAN LETTER AIN +10FC ; Alphabetic # Lm MODIFIER LETTER GEORGIAN NAR +1100..1248 ; Alphabetic # Lo [329] HANGUL CHOSEONG KIYEOK..ETHIOPIC SYLLABLE QWA +124A..124D ; Alphabetic # Lo [4] ETHIOPIC SYLLABLE QWI..ETHIOPIC SYLLABLE QWE +1250..1256 ; Alphabetic # Lo [7] ETHIOPIC SYLLABLE QHA..ETHIOPIC SYLLABLE QHO +1258 ; Alphabetic # Lo ETHIOPIC SYLLABLE QHWA +125A..125D ; Alphabetic # Lo [4] ETHIOPIC SYLLABLE QHWI..ETHIOPIC SYLLABLE QHWE +1260..1288 ; Alphabetic # Lo [41] ETHIOPIC SYLLABLE BA..ETHIOPIC SYLLABLE XWA +128A..128D ; Alphabetic # Lo [4] ETHIOPIC SYLLABLE XWI..ETHIOPIC SYLLABLE XWE +1290..12B0 ; Alphabetic # Lo [33] ETHIOPIC SYLLABLE NA..ETHIOPIC SYLLABLE KWA +12B2..12B5 ; Alphabetic # Lo [4] ETHIOPIC SYLLABLE KWI..ETHIOPIC SYLLABLE KWE +12B8..12BE ; Alphabetic # Lo [7] ETHIOPIC SYLLABLE KXA..ETHIOPIC SYLLABLE KXO +12C0 ; Alphabetic # Lo ETHIOPIC SYLLABLE KXWA +12C2..12C5 ; Alphabetic # Lo [4] ETHIOPIC SYLLABLE KXWI..ETHIOPIC SYLLABLE KXWE +12C8..12D6 ; Alphabetic # Lo [15] ETHIOPIC SYLLABLE WA..ETHIOPIC SYLLABLE PHARYNGEAL O +12D8..1310 ; Alphabetic # Lo [57] ETHIOPIC SYLLABLE ZA..ETHIOPIC SYLLABLE GWA +1312..1315 ; Alphabetic # Lo [4] ETHIOPIC SYLLABLE GWI..ETHIOPIC SYLLABLE GWE +1318..135A ; Alphabetic # Lo [67] ETHIOPIC SYLLABLE GGA..ETHIOPIC SYLLABLE FYA +135F ; Alphabetic # Mn ETHIOPIC COMBINING GEMINATION MARK +1380..138F ; Alphabetic # Lo [16] ETHIOPIC SYLLABLE SEBATBEIT MWA..ETHIOPIC SYLLABLE PWE +13A0..13F4 ; Alphabetic # Lo [85] CHEROKEE LETTER A..CHEROKEE LETTER YV +1401..166C ; Alphabetic # Lo [620] CANADIAN SYLLABICS E..CANADIAN SYLLABICS CARRIER TTSA +166F..167F ; Alphabetic # Lo [17] CANADIAN SYLLABICS QAI..CANADIAN SYLLABICS BLACKFOOT W +1681..169A ; Alphabetic # Lo [26] OGHAM LETTER BEITH..OGHAM LETTER PEITH +16A0..16EA ; Alphabetic # Lo [75] RUNIC LETTER FEHU FEOH FE F..RUNIC LETTER X +16EE..16F0 ; Alphabetic # Nl [3] RUNIC ARLAUG SYMBOL..RUNIC BELGTHOR SYMBOL +1700..170C ; Alphabetic # Lo [13] TAGALOG LETTER A..TAGALOG LETTER YA +170E..1711 ; Alphabetic # Lo [4] TAGALOG LETTER LA..TAGALOG LETTER HA +1712..1713 ; Alphabetic # Mn [2] TAGALOG VOWEL SIGN I..TAGALOG VOWEL SIGN U +1720..1731 ; Alphabetic # Lo [18] HANUNOO LETTER A..HANUNOO LETTER HA +1732..1733 ; Alphabetic # Mn [2] HANUNOO VOWEL SIGN I..HANUNOO VOWEL SIGN U +1740..1751 ; Alphabetic # Lo [18] BUHID LETTER A..BUHID LETTER HA +1752..1753 ; Alphabetic # Mn [2] BUHID VOWEL SIGN I..BUHID VOWEL SIGN U +1760..176C ; Alphabetic # Lo [13] TAGBANWA LETTER A..TAGBANWA LETTER YA +176E..1770 ; Alphabetic # Lo [3] TAGBANWA LETTER LA..TAGBANWA LETTER SA +1772..1773 ; Alphabetic # Mn [2] TAGBANWA VOWEL SIGN I..TAGBANWA VOWEL SIGN U +1780..17B3 ; Alphabetic # Lo [52] KHMER LETTER KA..KHMER INDEPENDENT VOWEL QAU +17B6 ; Alphabetic # Mc KHMER VOWEL SIGN AA +17B7..17BD ; Alphabetic # Mn [7] KHMER VOWEL SIGN I..KHMER VOWEL SIGN UA +17BE..17C5 ; Alphabetic # Mc [8] KHMER VOWEL SIGN OE..KHMER VOWEL SIGN AU +17C6 ; Alphabetic # Mn KHMER SIGN NIKAHIT +17C7..17C8 ; Alphabetic # Mc [2] KHMER SIGN REAHMUK..KHMER SIGN YUUKALEAPINTU +17D7 ; Alphabetic # Lm KHMER SIGN LEK TOO +17DC ; Alphabetic # Lo KHMER SIGN AVAKRAHASANYA +1820..1842 ; Alphabetic # Lo [35] MONGOLIAN LETTER A..MONGOLIAN LETTER CHI +1843 ; Alphabetic # Lm MONGOLIAN LETTER TODO LONG VOWEL SIGN +1844..1877 ; Alphabetic # Lo [52] MONGOLIAN LETTER TODO E..MONGOLIAN LETTER MANCHU ZHA +1880..18A8 ; Alphabetic # Lo [41] MONGOLIAN LETTER ALI GALI ANUSVARA ONE..MONGOLIAN LETTER MANCHU ALI GALI BHA +18A9 ; Alphabetic # Mn MONGOLIAN LETTER ALI GALI DAGALGA +18AA ; Alphabetic # Lo MONGOLIAN LETTER MANCHU ALI GALI LHA +18B0..18F5 ; Alphabetic # Lo [70] CANADIAN SYLLABICS OY..CANADIAN SYLLABICS CARRIER DENTAL S +1900..191C ; Alphabetic # Lo [29] LIMBU VOWEL-CARRIER LETTER..LIMBU LETTER HA +1920..1922 ; Alphabetic # Mn [3] LIMBU VOWEL SIGN A..LIMBU VOWEL SIGN U +1923..1926 ; Alphabetic # Mc [4] LIMBU VOWEL SIGN EE..LIMBU VOWEL SIGN AU +1927..1928 ; Alphabetic # Mn [2] LIMBU VOWEL SIGN E..LIMBU VOWEL SIGN O +1929..192B ; Alphabetic # Mc [3] LIMBU SUBJOINED LETTER YA..LIMBU SUBJOINED LETTER WA +1930..1931 ; Alphabetic # Mc [2] LIMBU SMALL LETTER KA..LIMBU SMALL LETTER NGA +1932 ; Alphabetic # Mn LIMBU SMALL LETTER ANUSVARA +1933..1938 ; Alphabetic # Mc [6] LIMBU SMALL LETTER TA..LIMBU SMALL LETTER LA +1950..196D ; Alphabetic # Lo [30] TAI LE LETTER KA..TAI LE LETTER AI +1970..1974 ; Alphabetic # Lo [5] TAI LE LETTER TONE-2..TAI LE LETTER TONE-6 +1980..19AB ; Alphabetic # Lo [44] NEW TAI LUE LETTER HIGH QA..NEW TAI LUE LETTER LOW SUA +19B0..19C0 ; Alphabetic # Mc [17] NEW TAI LUE VOWEL SIGN VOWEL SHORTENER..NEW TAI LUE VOWEL SIGN IY +19C1..19C7 ; Alphabetic # Lo [7] NEW TAI LUE LETTER FINAL V..NEW TAI LUE LETTER FINAL B +19C8..19C9 ; Alphabetic # Mc [2] NEW TAI LUE TONE MARK-1..NEW TAI LUE TONE MARK-2 +1A00..1A16 ; Alphabetic # Lo [23] BUGINESE LETTER KA..BUGINESE LETTER HA +1A17..1A18 ; Alphabetic # Mn [2] BUGINESE VOWEL SIGN I..BUGINESE VOWEL SIGN U +1A19..1A1B ; Alphabetic # Mc [3] BUGINESE VOWEL SIGN E..BUGINESE VOWEL SIGN AE +1A20..1A54 ; Alphabetic # Lo [53] TAI THAM LETTER HIGH KA..TAI THAM LETTER GREAT SA +1A55 ; Alphabetic # Mc TAI THAM CONSONANT SIGN MEDIAL RA +1A56 ; Alphabetic # Mn TAI THAM CONSONANT SIGN MEDIAL LA +1A57 ; Alphabetic # Mc TAI THAM CONSONANT SIGN LA TANG LAI +1A58..1A5E ; Alphabetic # Mn [7] TAI THAM SIGN MAI KANG LAI..TAI THAM CONSONANT SIGN SA +1A61 ; Alphabetic # Mc TAI THAM VOWEL SIGN A +1A62 ; Alphabetic # Mn TAI THAM VOWEL SIGN MAI SAT +1A63..1A64 ; Alphabetic # Mc [2] TAI THAM VOWEL SIGN AA..TAI THAM VOWEL SIGN TALL AA +1A65..1A6C ; Alphabetic # Mn [8] TAI THAM VOWEL SIGN I..TAI THAM VOWEL SIGN OA BELOW +1A6D..1A72 ; Alphabetic # Mc [6] TAI THAM VOWEL SIGN OY..TAI THAM VOWEL SIGN THAM AI +1A73..1A74 ; Alphabetic # Mn [2] TAI THAM VOWEL SIGN OA ABOVE..TAI THAM SIGN MAI KANG +1AA7 ; Alphabetic # Lm TAI THAM SIGN MAI YAMOK +1B00..1B03 ; Alphabetic # Mn [4] BALINESE SIGN ULU RICEM..BALINESE SIGN SURANG +1B04 ; Alphabetic # Mc BALINESE SIGN BISAH +1B05..1B33 ; Alphabetic # Lo [47] BALINESE LETTER AKARA..BALINESE LETTER HA +1B35 ; Alphabetic # Mc BALINESE VOWEL SIGN TEDUNG +1B36..1B3A ; Alphabetic # Mn [5] BALINESE VOWEL SIGN ULU..BALINESE VOWEL SIGN RA REPA +1B3B ; Alphabetic # Mc BALINESE VOWEL SIGN RA REPA TEDUNG +1B3C ; Alphabetic # Mn BALINESE VOWEL SIGN LA LENGA +1B3D..1B41 ; Alphabetic # Mc [5] BALINESE VOWEL SIGN LA LENGA TEDUNG..BALINESE VOWEL SIGN TALING REPA TEDUNG +1B42 ; Alphabetic # Mn BALINESE VOWEL SIGN PEPET +1B43 ; Alphabetic # Mc BALINESE VOWEL SIGN PEPET TEDUNG +1B45..1B4B ; Alphabetic # Lo [7] BALINESE LETTER KAF SASAK..BALINESE LETTER ASYURA SASAK +1B80..1B81 ; Alphabetic # Mn [2] SUNDANESE SIGN PANYECEK..SUNDANESE SIGN PANGLAYAR +1B82 ; Alphabetic # Mc SUNDANESE SIGN PANGWISAD +1B83..1BA0 ; Alphabetic # Lo [30] SUNDANESE LETTER A..SUNDANESE LETTER HA +1BA1 ; Alphabetic # Mc SUNDANESE CONSONANT SIGN PAMINGKAL +1BA2..1BA5 ; Alphabetic # Mn [4] SUNDANESE CONSONANT SIGN PANYAKRA..SUNDANESE VOWEL SIGN PANYUKU +1BA6..1BA7 ; Alphabetic # Mc [2] SUNDANESE VOWEL SIGN PANAELAENG..SUNDANESE VOWEL SIGN PANOLONG +1BA8..1BA9 ; Alphabetic # Mn [2] SUNDANESE VOWEL SIGN PAMEPET..SUNDANESE VOWEL SIGN PANEULEUNG +1BAE..1BAF ; Alphabetic # Lo [2] SUNDANESE LETTER KHA..SUNDANESE LETTER SYA +1C00..1C23 ; Alphabetic # Lo [36] LEPCHA LETTER KA..LEPCHA LETTER A +1C24..1C2B ; Alphabetic # Mc [8] LEPCHA SUBJOINED LETTER YA..LEPCHA VOWEL SIGN UU +1C2C..1C33 ; Alphabetic # Mn [8] LEPCHA VOWEL SIGN E..LEPCHA CONSONANT SIGN T +1C34..1C35 ; Alphabetic # Mc [2] LEPCHA CONSONANT SIGN NYIN-DO..LEPCHA CONSONANT SIGN KANG +1C4D..1C4F ; Alphabetic # Lo [3] LEPCHA LETTER TTA..LEPCHA LETTER DDA +1C5A..1C77 ; Alphabetic # Lo [30] OL CHIKI LETTER LA..OL CHIKI LETTER OH +1C78..1C7D ; Alphabetic # Lm [6] OL CHIKI MU TTUDDAG..OL CHIKI AHAD +1CE9..1CEC ; Alphabetic # Lo [4] VEDIC SIGN ANUSVARA ANTARGOMUKHA..VEDIC SIGN ANUSVARA VAMAGOMUKHA WITH TAIL +1CEE..1CF1 ; Alphabetic # Lo [4] VEDIC SIGN HEXIFORM LONG ANUSVARA..VEDIC SIGN ANUSVARA UBHAYATO MUKHA +1CF2 ; Alphabetic # Mc VEDIC SIGN ARDHAVISARGA +1D00..1D2B ; Alphabetic # L& [44] LATIN LETTER SMALL CAPITAL A..CYRILLIC LETTER SMALL CAPITAL EL +1D2C..1D61 ; Alphabetic # Lm [54] MODIFIER LETTER CAPITAL A..MODIFIER LETTER SMALL CHI +1D62..1D77 ; Alphabetic # L& [22] LATIN SUBSCRIPT SMALL LETTER I..LATIN SMALL LETTER TURNED G +1D78 ; Alphabetic # Lm MODIFIER LETTER CYRILLIC EN +1D79..1D9A ; Alphabetic # L& [34] LATIN SMALL LETTER INSULAR G..LATIN SMALL LETTER EZH WITH RETROFLEX HOOK +1D9B..1DBF ; Alphabetic # Lm [37] MODIFIER LETTER SMALL TURNED ALPHA..MODIFIER LETTER SMALL THETA +1E00..1F15 ; Alphabetic # L& [278] LATIN CAPITAL LETTER A WITH RING BELOW..GREEK SMALL LETTER EPSILON WITH DASIA AND OXIA +1F18..1F1D ; Alphabetic # L& [6] GREEK CAPITAL LETTER EPSILON WITH PSILI..GREEK CAPITAL LETTER EPSILON WITH DASIA AND OXIA +1F20..1F45 ; Alphabetic # L& [38] GREEK SMALL LETTER ETA WITH PSILI..GREEK SMALL LETTER OMICRON WITH DASIA AND OXIA +1F48..1F4D ; Alphabetic # L& [6] GREEK CAPITAL LETTER OMICRON WITH PSILI..GREEK CAPITAL LETTER OMICRON WITH DASIA AND OXIA +1F50..1F57 ; Alphabetic # L& [8] GREEK SMALL LETTER UPSILON WITH PSILI..GREEK SMALL LETTER UPSILON WITH DASIA AND PERISPOMENI +1F59 ; Alphabetic # L& GREEK CAPITAL LETTER UPSILON WITH DASIA +1F5B ; Alphabetic # L& GREEK CAPITAL LETTER UPSILON WITH DASIA AND VARIA +1F5D ; Alphabetic # L& GREEK CAPITAL LETTER UPSILON WITH DASIA AND OXIA +1F5F..1F7D ; Alphabetic # L& [31] GREEK CAPITAL LETTER UPSILON WITH DASIA AND PERISPOMENI..GREEK SMALL LETTER OMEGA WITH OXIA +1F80..1FB4 ; Alphabetic # L& [53] GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI..GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI +1FB6..1FBC ; Alphabetic # L& [7] GREEK SMALL LETTER ALPHA WITH PERISPOMENI..GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI +1FBE ; Alphabetic # L& GREEK PROSGEGRAMMENI +1FC2..1FC4 ; Alphabetic # L& [3] GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI..GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI +1FC6..1FCC ; Alphabetic # L& [7] GREEK SMALL LETTER ETA WITH PERISPOMENI..GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI +1FD0..1FD3 ; Alphabetic # L& [4] GREEK SMALL LETTER IOTA WITH VRACHY..GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA +1FD6..1FDB ; Alphabetic # L& [6] GREEK SMALL LETTER IOTA WITH PERISPOMENI..GREEK CAPITAL LETTER IOTA WITH OXIA +1FE0..1FEC ; Alphabetic # L& [13] GREEK SMALL LETTER UPSILON WITH VRACHY..GREEK CAPITAL LETTER RHO WITH DASIA +1FF2..1FF4 ; Alphabetic # L& [3] GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI..GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI +1FF6..1FFC ; Alphabetic # L& [7] GREEK SMALL LETTER OMEGA WITH PERISPOMENI..GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI +2071 ; Alphabetic # Lm SUPERSCRIPT LATIN SMALL LETTER I +207F ; Alphabetic # Lm SUPERSCRIPT LATIN SMALL LETTER N +2090..2094 ; Alphabetic # Lm [5] LATIN SUBSCRIPT SMALL LETTER A..LATIN SUBSCRIPT SMALL LETTER SCHWA +2102 ; Alphabetic # L& DOUBLE-STRUCK CAPITAL C +2107 ; Alphabetic # L& EULER CONSTANT +210A..2113 ; Alphabetic # L& [10] SCRIPT SMALL G..SCRIPT SMALL L +2115 ; Alphabetic # L& DOUBLE-STRUCK CAPITAL N +2119..211D ; Alphabetic # L& [5] DOUBLE-STRUCK CAPITAL P..DOUBLE-STRUCK CAPITAL R +2124 ; Alphabetic # L& DOUBLE-STRUCK CAPITAL Z +2126 ; Alphabetic # L& OHM SIGN +2128 ; Alphabetic # L& BLACK-LETTER CAPITAL Z +212A..212D ; Alphabetic # L& [4] KELVIN SIGN..BLACK-LETTER CAPITAL C +212F..2134 ; Alphabetic # L& [6] SCRIPT SMALL E..SCRIPT SMALL O +2135..2138 ; Alphabetic # Lo [4] ALEF SYMBOL..DALET SYMBOL +2139 ; Alphabetic # L& INFORMATION SOURCE +213C..213F ; Alphabetic # L& [4] DOUBLE-STRUCK SMALL PI..DOUBLE-STRUCK CAPITAL PI +2145..2149 ; Alphabetic # L& [5] DOUBLE-STRUCK ITALIC CAPITAL D..DOUBLE-STRUCK ITALIC SMALL J +214E ; Alphabetic # L& TURNED SMALL F +2160..2182 ; Alphabetic # Nl [35] ROMAN NUMERAL ONE..ROMAN NUMERAL TEN THOUSAND +2183..2184 ; Alphabetic # L& [2] ROMAN NUMERAL REVERSED ONE HUNDRED..LATIN SMALL LETTER REVERSED C +2185..2188 ; Alphabetic # Nl [4] ROMAN NUMERAL SIX LATE FORM..ROMAN NUMERAL ONE HUNDRED THOUSAND +24B6..24E9 ; Alphabetic # So [52] CIRCLED LATIN CAPITAL LETTER A..CIRCLED LATIN SMALL LETTER Z +2C00..2C2E ; Alphabetic # L& [47] GLAGOLITIC CAPITAL LETTER AZU..GLAGOLITIC CAPITAL LETTER LATINATE MYSLITE +2C30..2C5E ; Alphabetic # L& [47] GLAGOLITIC SMALL LETTER AZU..GLAGOLITIC SMALL LETTER LATINATE MYSLITE +2C60..2C7C ; Alphabetic # L& [29] LATIN CAPITAL LETTER L WITH DOUBLE BAR..LATIN SUBSCRIPT SMALL LETTER J +2C7D ; Alphabetic # Lm MODIFIER LETTER CAPITAL V +2C7E..2CE4 ; Alphabetic # L& [103] LATIN CAPITAL LETTER S WITH SWASH TAIL..COPTIC SYMBOL KAI +2CEB..2CEE ; Alphabetic # L& [4] COPTIC CAPITAL LETTER CRYPTOGRAMMIC SHEI..COPTIC SMALL LETTER CRYPTOGRAMMIC GANGIA +2D00..2D25 ; Alphabetic # L& [38] GEORGIAN SMALL LETTER AN..GEORGIAN SMALL LETTER HOE +2D30..2D65 ; Alphabetic # Lo [54] TIFINAGH LETTER YA..TIFINAGH LETTER YAZZ +2D6F ; Alphabetic # Lm TIFINAGH MODIFIER LETTER LABIALIZATION MARK +2D80..2D96 ; Alphabetic # Lo [23] ETHIOPIC SYLLABLE LOA..ETHIOPIC SYLLABLE GGWE +2DA0..2DA6 ; Alphabetic # Lo [7] ETHIOPIC SYLLABLE SSA..ETHIOPIC SYLLABLE SSO +2DA8..2DAE ; Alphabetic # Lo [7] ETHIOPIC SYLLABLE CCA..ETHIOPIC SYLLABLE CCO +2DB0..2DB6 ; Alphabetic # Lo [7] ETHIOPIC SYLLABLE ZZA..ETHIOPIC SYLLABLE ZZO +2DB8..2DBE ; Alphabetic # Lo [7] ETHIOPIC SYLLABLE CCHA..ETHIOPIC SYLLABLE CCHO +2DC0..2DC6 ; Alphabetic # Lo [7] ETHIOPIC SYLLABLE QYA..ETHIOPIC SYLLABLE QYO +2DC8..2DCE ; Alphabetic # Lo [7] ETHIOPIC SYLLABLE KYA..ETHIOPIC SYLLABLE KYO +2DD0..2DD6 ; Alphabetic # Lo [7] ETHIOPIC SYLLABLE XYA..ETHIOPIC SYLLABLE XYO +2DD8..2DDE ; Alphabetic # Lo [7] ETHIOPIC SYLLABLE GYA..ETHIOPIC SYLLABLE GYO +2DE0..2DFF ; Alphabetic # Mn [32] COMBINING CYRILLIC LETTER BE..COMBINING CYRILLIC LETTER IOTIFIED BIG YUS +2E2F ; Alphabetic # Lm VERTICAL TILDE +3005 ; Alphabetic # Lm IDEOGRAPHIC ITERATION MARK +3006 ; Alphabetic # Lo IDEOGRAPHIC CLOSING MARK +3007 ; Alphabetic # Nl IDEOGRAPHIC NUMBER ZERO +3021..3029 ; Alphabetic # Nl [9] HANGZHOU NUMERAL ONE..HANGZHOU NUMERAL NINE +3031..3035 ; Alphabetic # Lm [5] VERTICAL KANA REPEAT MARK..VERTICAL KANA REPEAT MARK LOWER HALF +3038..303A ; Alphabetic # Nl [3] HANGZHOU NUMERAL TEN..HANGZHOU NUMERAL THIRTY +303B ; Alphabetic # Lm VERTICAL IDEOGRAPHIC ITERATION MARK +303C ; Alphabetic # Lo MASU MARK +3041..3096 ; Alphabetic # Lo [86] HIRAGANA LETTER SMALL A..HIRAGANA LETTER SMALL KE +309D..309E ; Alphabetic # Lm [2] HIRAGANA ITERATION MARK..HIRAGANA VOICED ITERATION MARK +309F ; Alphabetic # Lo HIRAGANA DIGRAPH YORI +30A1..30FA ; Alphabetic # Lo [90] KATAKANA LETTER SMALL A..KATAKANA LETTER VO +30FC..30FE ; Alphabetic # Lm [3] KATAKANA-HIRAGANA PROLONGED SOUND MARK..KATAKANA VOICED ITERATION MARK +30FF ; Alphabetic # Lo KATAKANA DIGRAPH KOTO +3105..312D ; Alphabetic # Lo [41] BOPOMOFO LETTER B..BOPOMOFO LETTER IH +3131..318E ; Alphabetic # Lo [94] HANGUL LETTER KIYEOK..HANGUL LETTER ARAEAE +31A0..31B7 ; Alphabetic # Lo [24] BOPOMOFO LETTER BU..BOPOMOFO FINAL LETTER H +31F0..31FF ; Alphabetic # Lo [16] KATAKANA LETTER SMALL KU..KATAKANA LETTER SMALL RO +3400..4DB5 ; Alphabetic # Lo [6582] CJK UNIFIED IDEOGRAPH-3400..CJK UNIFIED IDEOGRAPH-4DB5 +4E00..9FCB ; Alphabetic # Lo [20940] CJK UNIFIED IDEOGRAPH-4E00..CJK UNIFIED IDEOGRAPH-9FCB +A000..A014 ; Alphabetic # Lo [21] YI SYLLABLE IT..YI SYLLABLE E +A015 ; Alphabetic # Lm YI SYLLABLE WU +A016..A48C ; Alphabetic # Lo [1143] YI SYLLABLE BIT..YI SYLLABLE YYR +A4D0..A4F7 ; Alphabetic # Lo [40] LISU LETTER BA..LISU LETTER OE +A4F8..A4FD ; Alphabetic # Lm [6] LISU LETTER TONE MYA TI..LISU LETTER TONE MYA JEU +A500..A60B ; Alphabetic # Lo [268] VAI SYLLABLE EE..VAI SYLLABLE NG +A60C ; Alphabetic # Lm VAI SYLLABLE LENGTHENER +A610..A61F ; Alphabetic # Lo [16] VAI SYLLABLE NDOLE FA..VAI SYMBOL JONG +A62A..A62B ; Alphabetic # Lo [2] VAI SYLLABLE NDOLE MA..VAI SYLLABLE NDOLE DO +A640..A65F ; Alphabetic # L& [32] CYRILLIC CAPITAL LETTER ZEMLYA..CYRILLIC SMALL LETTER YN +A662..A66D ; Alphabetic # L& [12] CYRILLIC CAPITAL LETTER SOFT DE..CYRILLIC SMALL LETTER DOUBLE MONOCULAR O +A66E ; Alphabetic # Lo CYRILLIC LETTER MULTIOCULAR O +A67F ; Alphabetic # Lm CYRILLIC PAYEROK +A680..A697 ; Alphabetic # L& [24] CYRILLIC CAPITAL LETTER DWE..CYRILLIC SMALL LETTER SHWE +A6A0..A6E5 ; Alphabetic # Lo [70] BAMUM LETTER A..BAMUM LETTER KI +A6E6..A6EF ; Alphabetic # Nl [10] BAMUM LETTER MO..BAMUM LETTER KOGHOM +A717..A71F ; Alphabetic # Lm [9] MODIFIER LETTER DOT VERTICAL BAR..MODIFIER LETTER LOW INVERTED EXCLAMATION MARK +A722..A76F ; Alphabetic # L& [78] LATIN CAPITAL LETTER EGYPTOLOGICAL ALEF..LATIN SMALL LETTER CON +A770 ; Alphabetic # Lm MODIFIER LETTER US +A771..A787 ; Alphabetic # L& [23] LATIN SMALL LETTER DUM..LATIN SMALL LETTER INSULAR T +A788 ; Alphabetic # Lm MODIFIER LETTER LOW CIRCUMFLEX ACCENT +A78B..A78C ; Alphabetic # L& [2] LATIN CAPITAL LETTER SALTILLO..LATIN SMALL LETTER SALTILLO +A7FB..A801 ; Alphabetic # Lo [7] LATIN EPIGRAPHIC LETTER REVERSED F..SYLOTI NAGRI LETTER I +A803..A805 ; Alphabetic # Lo [3] SYLOTI NAGRI LETTER U..SYLOTI NAGRI LETTER O +A807..A80A ; Alphabetic # Lo [4] SYLOTI NAGRI LETTER KO..SYLOTI NAGRI LETTER GHO +A80C..A822 ; Alphabetic # Lo [23] SYLOTI NAGRI LETTER CO..SYLOTI NAGRI LETTER HO +A823..A824 ; Alphabetic # Mc [2] SYLOTI NAGRI VOWEL SIGN A..SYLOTI NAGRI VOWEL SIGN I +A825..A826 ; Alphabetic # Mn [2] SYLOTI NAGRI VOWEL SIGN U..SYLOTI NAGRI VOWEL SIGN E +A827 ; Alphabetic # Mc SYLOTI NAGRI VOWEL SIGN OO +A840..A873 ; Alphabetic # Lo [52] PHAGS-PA LETTER KA..PHAGS-PA LETTER CANDRABINDU +A880..A881 ; Alphabetic # Mc [2] SAURASHTRA SIGN ANUSVARA..SAURASHTRA SIGN VISARGA +A882..A8B3 ; Alphabetic # Lo [50] SAURASHTRA LETTER A..SAURASHTRA LETTER LLA +A8B4..A8C3 ; Alphabetic # Mc [16] SAURASHTRA CONSONANT SIGN HAARU..SAURASHTRA VOWEL SIGN AU +A8F2..A8F7 ; Alphabetic # Lo [6] DEVANAGARI SIGN SPACING CANDRABINDU..DEVANAGARI SIGN CANDRABINDU AVAGRAHA +A8FB ; Alphabetic # Lo DEVANAGARI HEADSTROKE +A90A..A925 ; Alphabetic # Lo [28] KAYAH LI LETTER KA..KAYAH LI LETTER OO +A926..A92A ; Alphabetic # Mn [5] KAYAH LI VOWEL UE..KAYAH LI VOWEL O +A930..A946 ; Alphabetic # Lo [23] REJANG LETTER KA..REJANG LETTER A +A947..A951 ; Alphabetic # Mn [11] REJANG VOWEL SIGN I..REJANG CONSONANT SIGN R +A952 ; Alphabetic # Mc REJANG CONSONANT SIGN H +A960..A97C ; Alphabetic # Lo [29] HANGUL CHOSEONG TIKEUT-MIEUM..HANGUL CHOSEONG SSANGYEORINHIEUH +A980..A982 ; Alphabetic # Mn [3] JAVANESE SIGN PANYANGGA..JAVANESE SIGN LAYAR +A983 ; Alphabetic # Mc JAVANESE SIGN WIGNYAN +A984..A9B2 ; Alphabetic # Lo [47] JAVANESE LETTER A..JAVANESE LETTER HA +A9B3 ; Alphabetic # Mn JAVANESE SIGN CECAK TELU +A9B4..A9B5 ; Alphabetic # Mc [2] JAVANESE VOWEL SIGN TARUNG..JAVANESE VOWEL SIGN TOLONG +A9B6..A9B9 ; Alphabetic # Mn [4] JAVANESE VOWEL SIGN WULU..JAVANESE VOWEL SIGN SUKU MENDUT +A9BA..A9BB ; Alphabetic # Mc [2] JAVANESE VOWEL SIGN TALING..JAVANESE VOWEL SIGN DIRGA MURE +A9BC ; Alphabetic # Mn JAVANESE VOWEL SIGN PEPET +A9BD..A9BF ; Alphabetic # Mc [3] JAVANESE CONSONANT SIGN KERET..JAVANESE CONSONANT SIGN CAKRA +A9CF ; Alphabetic # Lm JAVANESE PANGRANGKEP +AA00..AA28 ; Alphabetic # Lo [41] CHAM LETTER A..CHAM LETTER HA +AA29..AA2E ; Alphabetic # Mn [6] CHAM VOWEL SIGN AA..CHAM VOWEL SIGN OE +AA2F..AA30 ; Alphabetic # Mc [2] CHAM VOWEL SIGN O..CHAM VOWEL SIGN AI +AA31..AA32 ; Alphabetic # Mn [2] CHAM VOWEL SIGN AU..CHAM VOWEL SIGN UE +AA33..AA34 ; Alphabetic # Mc [2] CHAM CONSONANT SIGN YA..CHAM CONSONANT SIGN RA +AA35..AA36 ; Alphabetic # Mn [2] CHAM CONSONANT SIGN LA..CHAM CONSONANT SIGN WA +AA40..AA42 ; Alphabetic # Lo [3] CHAM LETTER FINAL K..CHAM LETTER FINAL NG +AA43 ; Alphabetic # Mn CHAM CONSONANT SIGN FINAL NG +AA44..AA4B ; Alphabetic # Lo [8] CHAM LETTER FINAL CH..CHAM LETTER FINAL SS +AA4C ; Alphabetic # Mn CHAM CONSONANT SIGN FINAL M +AA4D ; Alphabetic # Mc CHAM CONSONANT SIGN FINAL H +AA60..AA6F ; Alphabetic # Lo [16] MYANMAR LETTER KHAMTI GA..MYANMAR LETTER KHAMTI FA +AA70 ; Alphabetic # Lm MYANMAR MODIFIER LETTER KHAMTI REDUPLICATION +AA71..AA76 ; Alphabetic # Lo [6] MYANMAR LETTER KHAMTI XA..MYANMAR LOGOGRAM KHAMTI HM +AA7A ; Alphabetic # Lo MYANMAR LETTER AITON RA +AA80..AAAF ; Alphabetic # Lo [48] TAI VIET LETTER LOW KO..TAI VIET LETTER HIGH O +AAB0 ; Alphabetic # Mn TAI VIET MAI KANG +AAB1 ; Alphabetic # Lo TAI VIET VOWEL AA +AAB2..AAB4 ; Alphabetic # Mn [3] TAI VIET VOWEL I..TAI VIET VOWEL U +AAB5..AAB6 ; Alphabetic # Lo [2] TAI VIET VOWEL E..TAI VIET VOWEL O +AAB7..AAB8 ; Alphabetic # Mn [2] TAI VIET MAI KHIT..TAI VIET VOWEL IA +AAB9..AABD ; Alphabetic # Lo [5] TAI VIET VOWEL UEA..TAI VIET VOWEL AN +AABE ; Alphabetic # Mn TAI VIET VOWEL AM +AAC0 ; Alphabetic # Lo TAI VIET TONE MAI NUENG +AAC2 ; Alphabetic # Lo TAI VIET TONE MAI SONG +AADB..AADC ; Alphabetic # Lo [2] TAI VIET SYMBOL KON..TAI VIET SYMBOL NUENG +AADD ; Alphabetic # Lm TAI VIET SYMBOL SAM +ABC0..ABE2 ; Alphabetic # Lo [35] MEETEI MAYEK LETTER KOK..MEETEI MAYEK LETTER I LONSUM +ABE3..ABE4 ; Alphabetic # Mc [2] MEETEI MAYEK VOWEL SIGN ONAP..MEETEI MAYEK VOWEL SIGN INAP +ABE5 ; Alphabetic # Mn MEETEI MAYEK VOWEL SIGN ANAP +ABE6..ABE7 ; Alphabetic # Mc [2] MEETEI MAYEK VOWEL SIGN YENAP..MEETEI MAYEK VOWEL SIGN SOUNAP +ABE8 ; Alphabetic # Mn MEETEI MAYEK VOWEL SIGN UNAP +ABE9..ABEA ; Alphabetic # Mc [2] MEETEI MAYEK VOWEL SIGN CHEINAP..MEETEI MAYEK VOWEL SIGN NUNG +AC00..D7A3 ; Alphabetic # Lo [11172] HANGUL SYLLABLE GA..HANGUL SYLLABLE HIH +D7B0..D7C6 ; Alphabetic # Lo [23] HANGUL JUNGSEONG O-YEO..HANGUL JUNGSEONG ARAEA-E +D7CB..D7FB ; Alphabetic # Lo [49] HANGUL JONGSEONG NIEUN-RIEUL..HANGUL JONGSEONG PHIEUPH-THIEUTH +F900..FA2D ; Alphabetic # Lo [302] CJK COMPATIBILITY IDEOGRAPH-F900..CJK COMPATIBILITY IDEOGRAPH-FA2D +FA30..FA6D ; Alphabetic # Lo [62] CJK COMPATIBILITY IDEOGRAPH-FA30..CJK COMPATIBILITY IDEOGRAPH-FA6D +FA70..FAD9 ; Alphabetic # Lo [106] CJK COMPATIBILITY IDEOGRAPH-FA70..CJK COMPATIBILITY IDEOGRAPH-FAD9 +FB00..FB06 ; Alphabetic # L& [7] LATIN SMALL LIGATURE FF..LATIN SMALL LIGATURE ST +FB13..FB17 ; Alphabetic # L& [5] ARMENIAN SMALL LIGATURE MEN NOW..ARMENIAN SMALL LIGATURE MEN XEH +FB1D ; Alphabetic # Lo HEBREW LETTER YOD WITH HIRIQ +FB1E ; Alphabetic # Mn HEBREW POINT JUDEO-SPANISH VARIKA +FB1F..FB28 ; Alphabetic # Lo [10] HEBREW LIGATURE YIDDISH YOD YOD PATAH..HEBREW LETTER WIDE TAV +FB2A..FB36 ; Alphabetic # Lo [13] HEBREW LETTER SHIN WITH SHIN DOT..HEBREW LETTER ZAYIN WITH DAGESH +FB38..FB3C ; Alphabetic # Lo [5] HEBREW LETTER TET WITH DAGESH..HEBREW LETTER LAMED WITH DAGESH +FB3E ; Alphabetic # Lo HEBREW LETTER MEM WITH DAGESH +FB40..FB41 ; Alphabetic # Lo [2] HEBREW LETTER NUN WITH DAGESH..HEBREW LETTER SAMEKH WITH DAGESH +FB43..FB44 ; Alphabetic # Lo [2] HEBREW LETTER FINAL PE WITH DAGESH..HEBREW LETTER PE WITH DAGESH +FB46..FBB1 ; Alphabetic # Lo [108] HEBREW LETTER TSADI WITH DAGESH..ARABIC LETTER YEH BARREE WITH HAMZA ABOVE FINAL FORM +FBD3..FD3D ; Alphabetic # Lo [363] ARABIC LETTER NG ISOLATED FORM..ARABIC LIGATURE ALEF WITH FATHATAN ISOLATED FORM +FD50..FD8F ; Alphabetic # Lo [64] ARABIC LIGATURE TEH WITH JEEM WITH MEEM INITIAL FORM..ARABIC LIGATURE MEEM WITH KHAH WITH MEEM INITIAL FORM +FD92..FDC7 ; Alphabetic # Lo [54] ARABIC LIGATURE MEEM WITH JEEM WITH KHAH INITIAL FORM..ARABIC LIGATURE NOON WITH JEEM WITH YEH FINAL FORM +FDF0..FDFB ; Alphabetic # Lo [12] ARABIC LIGATURE SALLA USED AS KORANIC STOP SIGN ISOLATED FORM..ARABIC LIGATURE JALLAJALALOUHOU +FE70..FE74 ; Alphabetic # Lo [5] ARABIC FATHATAN ISOLATED FORM..ARABIC KASRATAN ISOLATED FORM +FE76..FEFC ; Alphabetic # Lo [135] ARABIC FATHA ISOLATED FORM..ARABIC LIGATURE LAM WITH ALEF FINAL FORM +FF21..FF3A ; Alphabetic # L& [26] FULLWIDTH LATIN CAPITAL LETTER A..FULLWIDTH LATIN CAPITAL LETTER Z +FF41..FF5A ; Alphabetic # L& [26] FULLWIDTH LATIN SMALL LETTER A..FULLWIDTH LATIN SMALL LETTER Z +FF66..FF6F ; Alphabetic # Lo [10] HALFWIDTH KATAKANA LETTER WO..HALFWIDTH KATAKANA LETTER SMALL TU +FF70 ; Alphabetic # Lm HALFWIDTH KATAKANA-HIRAGANA PROLONGED SOUND MARK +FF71..FF9D ; Alphabetic # Lo [45] HALFWIDTH KATAKANA LETTER A..HALFWIDTH KATAKANA LETTER N +FF9E..FF9F ; Alphabetic # Lm [2] HALFWIDTH KATAKANA VOICED SOUND MARK..HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK +FFA0..FFBE ; Alphabetic # Lo [31] HALFWIDTH HANGUL FILLER..HALFWIDTH HANGUL LETTER HIEUH +FFC2..FFC7 ; Alphabetic # Lo [6] HALFWIDTH HANGUL LETTER A..HALFWIDTH HANGUL LETTER E +FFCA..FFCF ; Alphabetic # Lo [6] HALFWIDTH HANGUL LETTER YEO..HALFWIDTH HANGUL LETTER OE +FFD2..FFD7 ; Alphabetic # Lo [6] HALFWIDTH HANGUL LETTER YO..HALFWIDTH HANGUL LETTER YU +FFDA..FFDC ; Alphabetic # Lo [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANGUL LETTER I +10000..1000B ; Alphabetic # Lo [12] LINEAR B SYLLABLE B008 A..LINEAR B SYLLABLE B046 JE +1000D..10026 ; Alphabetic # Lo [26] LINEAR B SYLLABLE B036 JO..LINEAR B SYLLABLE B032 QO +10028..1003A ; Alphabetic # Lo [19] LINEAR B SYLLABLE B060 RA..LINEAR B SYLLABLE B042 WO +1003C..1003D ; Alphabetic # Lo [2] LINEAR B SYLLABLE B017 ZA..LINEAR B SYLLABLE B074 ZE +1003F..1004D ; Alphabetic # Lo [15] LINEAR B SYLLABLE B020 ZO..LINEAR B SYLLABLE B091 TWO +10050..1005D ; Alphabetic # Lo [14] LINEAR B SYMBOL B018..LINEAR B SYMBOL B089 +10080..100FA ; Alphabetic # Lo [123] LINEAR B IDEOGRAM B100 MAN..LINEAR B IDEOGRAM VESSEL B305 +10140..10174 ; Alphabetic # Nl [53] GREEK ACROPHONIC ATTIC ONE QUARTER..GREEK ACROPHONIC STRATIAN FIFTY MNAS +10280..1029C ; Alphabetic # Lo [29] LYCIAN LETTER A..LYCIAN LETTER X +102A0..102D0 ; Alphabetic # Lo [49] CARIAN LETTER A..CARIAN LETTER UUU3 +10300..1031E ; Alphabetic # Lo [31] OLD ITALIC LETTER A..OLD ITALIC LETTER UU +10330..10340 ; Alphabetic # Lo [17] GOTHIC LETTER AHSA..GOTHIC LETTER PAIRTHRA +10341 ; Alphabetic # Nl GOTHIC LETTER NINETY +10342..10349 ; Alphabetic # Lo [8] GOTHIC LETTER RAIDA..GOTHIC LETTER OTHAL +1034A ; Alphabetic # Nl GOTHIC LETTER NINE HUNDRED +10380..1039D ; Alphabetic # Lo [30] UGARITIC LETTER ALPA..UGARITIC LETTER SSU +103A0..103C3 ; Alphabetic # Lo [36] OLD PERSIAN SIGN A..OLD PERSIAN SIGN HA +103C8..103CF ; Alphabetic # Lo [8] OLD PERSIAN SIGN AURAMAZDAA..OLD PERSIAN SIGN BUUMISH +103D1..103D5 ; Alphabetic # Nl [5] OLD PERSIAN NUMBER ONE..OLD PERSIAN NUMBER HUNDRED +10400..1044F ; Alphabetic # L& [80] DESERET CAPITAL LETTER LONG I..DESERET SMALL LETTER EW +10450..1049D ; Alphabetic # Lo [78] SHAVIAN LETTER PEEP..OSMANYA LETTER OO +10800..10805 ; Alphabetic # Lo [6] CYPRIOT SYLLABLE A..CYPRIOT SYLLABLE JA +10808 ; Alphabetic # Lo CYPRIOT SYLLABLE JO +1080A..10835 ; Alphabetic # Lo [44] CYPRIOT SYLLABLE KA..CYPRIOT SYLLABLE WO +10837..10838 ; Alphabetic # Lo [2] CYPRIOT SYLLABLE XA..CYPRIOT SYLLABLE XE +1083C ; Alphabetic # Lo CYPRIOT SYLLABLE ZA +1083F..10855 ; Alphabetic # Lo [23] CYPRIOT SYLLABLE ZO..IMPERIAL ARAMAIC LETTER TAW +10900..10915 ; Alphabetic # Lo [22] PHOENICIAN LETTER ALF..PHOENICIAN LETTER TAU +10920..10939 ; Alphabetic # Lo [26] LYDIAN LETTER A..LYDIAN LETTER C +10A00 ; Alphabetic # Lo KHAROSHTHI LETTER A +10A01..10A03 ; Alphabetic # Mn [3] KHAROSHTHI VOWEL SIGN I..KHAROSHTHI VOWEL SIGN VOCALIC R +10A05..10A06 ; Alphabetic # Mn [2] KHAROSHTHI VOWEL SIGN E..KHAROSHTHI VOWEL SIGN O +10A0C..10A0F ; Alphabetic # Mn [4] KHAROSHTHI VOWEL LENGTH MARK..KHAROSHTHI SIGN VISARGA +10A10..10A13 ; Alphabetic # Lo [4] KHAROSHTHI LETTER KA..KHAROSHTHI LETTER GHA +10A15..10A17 ; Alphabetic # Lo [3] KHAROSHTHI LETTER CA..KHAROSHTHI LETTER JA +10A19..10A33 ; Alphabetic # Lo [27] KHAROSHTHI LETTER NYA..KHAROSHTHI LETTER TTTHA +10A60..10A7C ; Alphabetic # Lo [29] OLD SOUTH ARABIAN LETTER HE..OLD SOUTH ARABIAN LETTER THETH +10B00..10B35 ; Alphabetic # Lo [54] AVESTAN LETTER A..AVESTAN LETTER HE +10B40..10B55 ; Alphabetic # Lo [22] INSCRIPTIONAL PARTHIAN LETTER ALEPH..INSCRIPTIONAL PARTHIAN LETTER TAW +10B60..10B72 ; Alphabetic # Lo [19] INSCRIPTIONAL PAHLAVI LETTER ALEPH..INSCRIPTIONAL PAHLAVI LETTER TAW +10C00..10C48 ; Alphabetic # Lo [73] OLD TURKIC LETTER ORKHON A..OLD TURKIC LETTER ORKHON BASH +11082 ; Alphabetic # Mc KAITHI SIGN VISARGA +11083..110AF ; Alphabetic # Lo [45] KAITHI LETTER A..KAITHI LETTER HA +110B0..110B2 ; Alphabetic # Mc [3] KAITHI VOWEL SIGN AA..KAITHI VOWEL SIGN II +110B3..110B6 ; Alphabetic # Mn [4] KAITHI VOWEL SIGN U..KAITHI VOWEL SIGN AI +110B7..110B8 ; Alphabetic # Mc [2] KAITHI VOWEL SIGN O..KAITHI VOWEL SIGN AU +12000..1236E ; Alphabetic # Lo [879] CUNEIFORM SIGN A..CUNEIFORM SIGN ZUM +12400..12462 ; Alphabetic # Nl [99] CUNEIFORM NUMERIC SIGN TWO ASH..CUNEIFORM NUMERIC SIGN OLD ASSYRIAN ONE QUARTER +13000..1342E ; Alphabetic # Lo [1071] EGYPTIAN HIEROGLYPH A001..EGYPTIAN HIEROGLYPH AA032 +1D400..1D454 ; Alphabetic # L& [85] MATHEMATICAL BOLD CAPITAL A..MATHEMATICAL ITALIC SMALL G +1D456..1D49C ; Alphabetic # L& [71] MATHEMATICAL ITALIC SMALL I..MATHEMATICAL SCRIPT CAPITAL A +1D49E..1D49F ; Alphabetic # L& [2] MATHEMATICAL SCRIPT CAPITAL C..MATHEMATICAL SCRIPT CAPITAL D +1D4A2 ; Alphabetic # L& MATHEMATICAL SCRIPT CAPITAL G +1D4A5..1D4A6 ; Alphabetic # L& [2] MATHEMATICAL SCRIPT CAPITAL J..MATHEMATICAL SCRIPT CAPITAL K +1D4A9..1D4AC ; Alphabetic # L& [4] MATHEMATICAL SCRIPT CAPITAL N..MATHEMATICAL SCRIPT CAPITAL Q +1D4AE..1D4B9 ; Alphabetic # L& [12] MATHEMATICAL SCRIPT CAPITAL S..MATHEMATICAL SCRIPT SMALL D +1D4BB ; Alphabetic # L& MATHEMATICAL SCRIPT SMALL F +1D4BD..1D4C3 ; Alphabetic # L& [7] MATHEMATICAL SCRIPT SMALL H..MATHEMATICAL SCRIPT SMALL N +1D4C5..1D505 ; Alphabetic # L& [65] MATHEMATICAL SCRIPT SMALL P..MATHEMATICAL FRAKTUR CAPITAL B +1D507..1D50A ; Alphabetic # L& [4] MATHEMATICAL FRAKTUR CAPITAL D..MATHEMATICAL FRAKTUR CAPITAL G +1D50D..1D514 ; Alphabetic # L& [8] MATHEMATICAL FRAKTUR CAPITAL J..MATHEMATICAL FRAKTUR CAPITAL Q +1D516..1D51C ; Alphabetic # L& [7] MATHEMATICAL FRAKTUR CAPITAL S..MATHEMATICAL FRAKTUR CAPITAL Y +1D51E..1D539 ; Alphabetic # L& [28] MATHEMATICAL FRAKTUR SMALL A..MATHEMATICAL DOUBLE-STRUCK CAPITAL B +1D53B..1D53E ; Alphabetic # L& [4] MATHEMATICAL DOUBLE-STRUCK CAPITAL D..MATHEMATICAL DOUBLE-STRUCK CAPITAL G +1D540..1D544 ; Alphabetic # L& [5] MATHEMATICAL DOUBLE-STRUCK CAPITAL I..MATHEMATICAL DOUBLE-STRUCK CAPITAL M +1D546 ; Alphabetic # L& MATHEMATICAL DOUBLE-STRUCK CAPITAL O +1D54A..1D550 ; Alphabetic # L& [7] MATHEMATICAL DOUBLE-STRUCK CAPITAL S..MATHEMATICAL DOUBLE-STRUCK CAPITAL Y +1D552..1D6A5 ; Alphabetic # L& [340] MATHEMATICAL DOUBLE-STRUCK SMALL A..MATHEMATICAL ITALIC SMALL DOTLESS J +1D6A8..1D6C0 ; Alphabetic # L& [25] MATHEMATICAL BOLD CAPITAL ALPHA..MATHEMATICAL BOLD CAPITAL OMEGA +1D6C2..1D6DA ; Alphabetic # L& [25] MATHEMATICAL BOLD SMALL ALPHA..MATHEMATICAL BOLD SMALL OMEGA +1D6DC..1D6FA ; Alphabetic # L& [31] MATHEMATICAL BOLD EPSILON SYMBOL..MATHEMATICAL ITALIC CAPITAL OMEGA +1D6FC..1D714 ; Alphabetic # L& [25] MATHEMATICAL ITALIC SMALL ALPHA..MATHEMATICAL ITALIC SMALL OMEGA +1D716..1D734 ; Alphabetic # L& [31] MATHEMATICAL ITALIC EPSILON SYMBOL..MATHEMATICAL BOLD ITALIC CAPITAL OMEGA +1D736..1D74E ; Alphabetic # L& [25] MATHEMATICAL BOLD ITALIC SMALL ALPHA..MATHEMATICAL BOLD ITALIC SMALL OMEGA +1D750..1D76E ; Alphabetic # L& [31] MATHEMATICAL BOLD ITALIC EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD CAPITAL OMEGA +1D770..1D788 ; Alphabetic # L& [25] MATHEMATICAL SANS-SERIF BOLD SMALL ALPHA..MATHEMATICAL SANS-SERIF BOLD SMALL OMEGA +1D78A..1D7A8 ; Alphabetic # L& [31] MATHEMATICAL SANS-SERIF BOLD EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL OMEGA +1D7AA..1D7C2 ; Alphabetic # L& [25] MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL ALPHA..MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL OMEGA +1D7C4..1D7CB ; Alphabetic # L& [8] MATHEMATICAL SANS-SERIF BOLD ITALIC EPSILON SYMBOL..MATHEMATICAL BOLD SMALL DIGAMMA +20000..2A6D6 ; Alphabetic # Lo [42711] CJK UNIFIED IDEOGRAPH-20000..CJK UNIFIED IDEOGRAPH-2A6D6 +2A700..2B734 ; Alphabetic # Lo [4149] CJK UNIFIED IDEOGRAPH-2A700..CJK UNIFIED IDEOGRAPH-2B734 +2F800..2FA1D ; Alphabetic # Lo [542] CJK COMPATIBILITY IDEOGRAPH-2F800..CJK COMPATIBILITY IDEOGRAPH-2FA1D + +# Total code points: 100520 + +# ================================================ + +# Derived Property: Lowercase +# Generated from: Ll + Other_Lowercase + +0061..007A ; Lowercase # L& [26] LATIN SMALL LETTER A..LATIN SMALL LETTER Z +00AA ; Lowercase # L& FEMININE ORDINAL INDICATOR +00B5 ; Lowercase # L& MICRO SIGN +00BA ; Lowercase # L& MASCULINE ORDINAL INDICATOR +00DF..00F6 ; Lowercase # L& [24] LATIN SMALL LETTER SHARP S..LATIN SMALL LETTER O WITH DIAERESIS +00F8..00FF ; Lowercase # L& [8] LATIN SMALL LETTER O WITH STROKE..LATIN SMALL LETTER Y WITH DIAERESIS +0101 ; Lowercase # L& LATIN SMALL LETTER A WITH MACRON +0103 ; Lowercase # L& LATIN SMALL LETTER A WITH BREVE +0105 ; Lowercase # L& LATIN SMALL LETTER A WITH OGONEK +0107 ; Lowercase # L& LATIN SMALL LETTER C WITH ACUTE +0109 ; Lowercase # L& LATIN SMALL LETTER C WITH CIRCUMFLEX +010B ; Lowercase # L& LATIN SMALL LETTER C WITH DOT ABOVE +010D ; Lowercase # L& LATIN SMALL LETTER C WITH CARON +010F ; Lowercase # L& LATIN SMALL LETTER D WITH CARON +0111 ; Lowercase # L& LATIN SMALL LETTER D WITH STROKE +0113 ; Lowercase # L& LATIN SMALL LETTER E WITH MACRON +0115 ; Lowercase # L& LATIN SMALL LETTER E WITH BREVE +0117 ; Lowercase # L& LATIN SMALL LETTER E WITH DOT ABOVE +0119 ; Lowercase # L& LATIN SMALL LETTER E WITH OGONEK +011B ; Lowercase # L& LATIN SMALL LETTER E WITH CARON +011D ; Lowercase # L& LATIN SMALL LETTER G WITH CIRCUMFLEX +011F ; Lowercase # L& LATIN SMALL LETTER G WITH BREVE +0121 ; Lowercase # L& LATIN SMALL LETTER G WITH DOT ABOVE +0123 ; Lowercase # L& LATIN SMALL LETTER G WITH CEDILLA +0125 ; Lowercase # L& LATIN SMALL LETTER H WITH CIRCUMFLEX +0127 ; Lowercase # L& LATIN SMALL LETTER H WITH STROKE +0129 ; Lowercase # L& LATIN SMALL LETTER I WITH TILDE +012B ; Lowercase # L& LATIN SMALL LETTER I WITH MACRON +012D ; Lowercase # L& LATIN SMALL LETTER I WITH BREVE +012F ; Lowercase # L& LATIN SMALL LETTER I WITH OGONEK +0131 ; Lowercase # L& LATIN SMALL LETTER DOTLESS I +0133 ; Lowercase # L& LATIN SMALL LIGATURE IJ +0135 ; Lowercase # L& LATIN SMALL LETTER J WITH CIRCUMFLEX +0137..0138 ; Lowercase # L& [2] LATIN SMALL LETTER K WITH CEDILLA..LATIN SMALL LETTER KRA +013A ; Lowercase # L& LATIN SMALL LETTER L WITH ACUTE +013C ; Lowercase # L& LATIN SMALL LETTER L WITH CEDILLA +013E ; Lowercase # L& LATIN SMALL LETTER L WITH CARON +0140 ; Lowercase # L& LATIN SMALL LETTER L WITH MIDDLE DOT +0142 ; Lowercase # L& LATIN SMALL LETTER L WITH STROKE +0144 ; Lowercase # L& LATIN SMALL LETTER N WITH ACUTE +0146 ; Lowercase # L& LATIN SMALL LETTER N WITH CEDILLA +0148..0149 ; Lowercase # L& [2] LATIN SMALL LETTER N WITH CARON..LATIN SMALL LETTER N PRECEDED BY APOSTROPHE +014B ; Lowercase # L& LATIN SMALL LETTER ENG +014D ; Lowercase # L& LATIN SMALL LETTER O WITH MACRON +014F ; Lowercase # L& LATIN SMALL LETTER O WITH BREVE +0151 ; Lowercase # L& LATIN SMALL LETTER O WITH DOUBLE ACUTE +0153 ; Lowercase # L& LATIN SMALL LIGATURE OE +0155 ; Lowercase # L& LATIN SMALL LETTER R WITH ACUTE +0157 ; Lowercase # L& LATIN SMALL LETTER R WITH CEDILLA +0159 ; Lowercase # L& LATIN SMALL LETTER R WITH CARON +015B ; Lowercase # L& LATIN SMALL LETTER S WITH ACUTE +015D ; Lowercase # L& LATIN SMALL LETTER S WITH CIRCUMFLEX +015F ; Lowercase # L& LATIN SMALL LETTER S WITH CEDILLA +0161 ; Lowercase # L& LATIN SMALL LETTER S WITH CARON +0163 ; Lowercase # L& LATIN SMALL LETTER T WITH CEDILLA +0165 ; Lowercase # L& LATIN SMALL LETTER T WITH CARON +0167 ; Lowercase # L& LATIN SMALL LETTER T WITH STROKE +0169 ; Lowercase # L& LATIN SMALL LETTER U WITH TILDE +016B ; Lowercase # L& LATIN SMALL LETTER U WITH MACRON +016D ; Lowercase # L& LATIN SMALL LETTER U WITH BREVE +016F ; Lowercase # L& LATIN SMALL LETTER U WITH RING ABOVE +0171 ; Lowercase # L& LATIN SMALL LETTER U WITH DOUBLE ACUTE +0173 ; Lowercase # L& LATIN SMALL LETTER U WITH OGONEK +0175 ; Lowercase # L& LATIN SMALL LETTER W WITH CIRCUMFLEX +0177 ; Lowercase # L& LATIN SMALL LETTER Y WITH CIRCUMFLEX +017A ; Lowercase # L& LATIN SMALL LETTER Z WITH ACUTE +017C ; Lowercase # L& LATIN SMALL LETTER Z WITH DOT ABOVE +017E..0180 ; Lowercase # L& [3] LATIN SMALL LETTER Z WITH CARON..LATIN SMALL LETTER B WITH STROKE +0183 ; Lowercase # L& LATIN SMALL LETTER B WITH TOPBAR +0185 ; Lowercase # L& LATIN SMALL LETTER TONE SIX +0188 ; Lowercase # L& LATIN SMALL LETTER C WITH HOOK +018C..018D ; Lowercase # L& [2] LATIN SMALL LETTER D WITH TOPBAR..LATIN SMALL LETTER TURNED DELTA +0192 ; Lowercase # L& LATIN SMALL LETTER F WITH HOOK +0195 ; Lowercase # L& LATIN SMALL LETTER HV +0199..019B ; Lowercase # L& [3] LATIN SMALL LETTER K WITH HOOK..LATIN SMALL LETTER LAMBDA WITH STROKE +019E ; Lowercase # L& LATIN SMALL LETTER N WITH LONG RIGHT LEG +01A1 ; Lowercase # L& LATIN SMALL LETTER O WITH HORN +01A3 ; Lowercase # L& LATIN SMALL LETTER OI +01A5 ; Lowercase # L& LATIN SMALL LETTER P WITH HOOK +01A8 ; Lowercase # L& LATIN SMALL LETTER TONE TWO +01AA..01AB ; Lowercase # L& [2] LATIN LETTER REVERSED ESH LOOP..LATIN SMALL LETTER T WITH PALATAL HOOK +01AD ; Lowercase # L& LATIN SMALL LETTER T WITH HOOK +01B0 ; Lowercase # L& LATIN SMALL LETTER U WITH HORN +01B4 ; Lowercase # L& LATIN SMALL LETTER Y WITH HOOK +01B6 ; Lowercase # L& LATIN SMALL LETTER Z WITH STROKE +01B9..01BA ; Lowercase # L& [2] LATIN SMALL LETTER EZH REVERSED..LATIN SMALL LETTER EZH WITH TAIL +01BD..01BF ; Lowercase # L& [3] LATIN SMALL LETTER TONE FIVE..LATIN LETTER WYNN +01C6 ; Lowercase # L& LATIN SMALL LETTER DZ WITH CARON +01C9 ; Lowercase # L& LATIN SMALL LETTER LJ +01CC ; Lowercase # L& LATIN SMALL LETTER NJ +01CE ; Lowercase # L& LATIN SMALL LETTER A WITH CARON +01D0 ; Lowercase # L& LATIN SMALL LETTER I WITH CARON +01D2 ; Lowercase # L& LATIN SMALL LETTER O WITH CARON +01D4 ; Lowercase # L& LATIN SMALL LETTER U WITH CARON +01D6 ; Lowercase # L& LATIN SMALL LETTER U WITH DIAERESIS AND MACRON +01D8 ; Lowercase # L& LATIN SMALL LETTER U WITH DIAERESIS AND ACUTE +01DA ; Lowercase # L& LATIN SMALL LETTER U WITH DIAERESIS AND CARON +01DC..01DD ; Lowercase # L& [2] LATIN SMALL LETTER U WITH DIAERESIS AND GRAVE..LATIN SMALL LETTER TURNED E +01DF ; Lowercase # L& LATIN SMALL LETTER A WITH DIAERESIS AND MACRON +01E1 ; Lowercase # L& LATIN SMALL LETTER A WITH DOT ABOVE AND MACRON +01E3 ; Lowercase # L& LATIN SMALL LETTER AE WITH MACRON +01E5 ; Lowercase # L& LATIN SMALL LETTER G WITH STROKE +01E7 ; Lowercase # L& LATIN SMALL LETTER G WITH CARON +01E9 ; Lowercase # L& LATIN SMALL LETTER K WITH CARON +01EB ; Lowercase # L& LATIN SMALL LETTER O WITH OGONEK +01ED ; Lowercase # L& LATIN SMALL LETTER O WITH OGONEK AND MACRON +01EF..01F0 ; Lowercase # L& [2] LATIN SMALL LETTER EZH WITH CARON..LATIN SMALL LETTER J WITH CARON +01F3 ; Lowercase # L& LATIN SMALL LETTER DZ +01F5 ; Lowercase # L& LATIN SMALL LETTER G WITH ACUTE +01F9 ; Lowercase # L& LATIN SMALL LETTER N WITH GRAVE +01FB ; Lowercase # L& LATIN SMALL LETTER A WITH RING ABOVE AND ACUTE +01FD ; Lowercase # L& LATIN SMALL LETTER AE WITH ACUTE +01FF ; Lowercase # L& LATIN SMALL LETTER O WITH STROKE AND ACUTE +0201 ; Lowercase # L& LATIN SMALL LETTER A WITH DOUBLE GRAVE +0203 ; Lowercase # L& LATIN SMALL LETTER A WITH INVERTED BREVE +0205 ; Lowercase # L& LATIN SMALL LETTER E WITH DOUBLE GRAVE +0207 ; Lowercase # L& LATIN SMALL LETTER E WITH INVERTED BREVE +0209 ; Lowercase # L& LATIN SMALL LETTER I WITH DOUBLE GRAVE +020B ; Lowercase # L& LATIN SMALL LETTER I WITH INVERTED BREVE +020D ; Lowercase # L& LATIN SMALL LETTER O WITH DOUBLE GRAVE +020F ; Lowercase # L& LATIN SMALL LETTER O WITH INVERTED BREVE +0211 ; Lowercase # L& LATIN SMALL LETTER R WITH DOUBLE GRAVE +0213 ; Lowercase # L& LATIN SMALL LETTER R WITH INVERTED BREVE +0215 ; Lowercase # L& LATIN SMALL LETTER U WITH DOUBLE GRAVE +0217 ; Lowercase # L& LATIN SMALL LETTER U WITH INVERTED BREVE +0219 ; Lowercase # L& LATIN SMALL LETTER S WITH COMMA BELOW +021B ; Lowercase # L& LATIN SMALL LETTER T WITH COMMA BELOW +021D ; Lowercase # L& LATIN SMALL LETTER YOGH +021F ; Lowercase # L& LATIN SMALL LETTER H WITH CARON +0221 ; Lowercase # L& LATIN SMALL LETTER D WITH CURL +0223 ; Lowercase # L& LATIN SMALL LETTER OU +0225 ; Lowercase # L& LATIN SMALL LETTER Z WITH HOOK +0227 ; Lowercase # L& LATIN SMALL LETTER A WITH DOT ABOVE +0229 ; Lowercase # L& LATIN SMALL LETTER E WITH CEDILLA +022B ; Lowercase # L& LATIN SMALL LETTER O WITH DIAERESIS AND MACRON +022D ; Lowercase # L& LATIN SMALL LETTER O WITH TILDE AND MACRON +022F ; Lowercase # L& LATIN SMALL LETTER O WITH DOT ABOVE +0231 ; Lowercase # L& LATIN SMALL LETTER O WITH DOT ABOVE AND MACRON +0233..0239 ; Lowercase # L& [7] LATIN SMALL LETTER Y WITH MACRON..LATIN SMALL LETTER QP DIGRAPH +023C ; Lowercase # L& LATIN SMALL LETTER C WITH STROKE +023F..0240 ; Lowercase # L& [2] LATIN SMALL LETTER S WITH SWASH TAIL..LATIN SMALL LETTER Z WITH SWASH TAIL +0242 ; Lowercase # L& LATIN SMALL LETTER GLOTTAL STOP +0247 ; Lowercase # L& LATIN SMALL LETTER E WITH STROKE +0249 ; Lowercase # L& LATIN SMALL LETTER J WITH STROKE +024B ; Lowercase # L& LATIN SMALL LETTER Q WITH HOOK TAIL +024D ; Lowercase # L& LATIN SMALL LETTER R WITH STROKE +024F..0293 ; Lowercase # L& [69] LATIN SMALL LETTER Y WITH STROKE..LATIN SMALL LETTER EZH WITH CURL +0295..02AF ; Lowercase # L& [27] LATIN LETTER PHARYNGEAL VOICED FRICATIVE..LATIN SMALL LETTER TURNED H WITH FISHHOOK AND TAIL +02B0..02B8 ; Lowercase # Lm [9] MODIFIER LETTER SMALL H..MODIFIER LETTER SMALL Y +02C0..02C1 ; Lowercase # Lm [2] MODIFIER LETTER GLOTTAL STOP..MODIFIER LETTER REVERSED GLOTTAL STOP +02E0..02E4 ; Lowercase # Lm [5] MODIFIER LETTER SMALL GAMMA..MODIFIER LETTER SMALL REVERSED GLOTTAL STOP +0345 ; Lowercase # Mn COMBINING GREEK YPOGEGRAMMENI +0371 ; Lowercase # L& GREEK SMALL LETTER HETA +0373 ; Lowercase # L& GREEK SMALL LETTER ARCHAIC SAMPI +0377 ; Lowercase # L& GREEK SMALL LETTER PAMPHYLIAN DIGAMMA +037A ; Lowercase # Lm GREEK YPOGEGRAMMENI +037B..037D ; Lowercase # L& [3] GREEK SMALL REVERSED LUNATE SIGMA SYMBOL..GREEK SMALL REVERSED DOTTED LUNATE SIGMA SYMBOL +0390 ; Lowercase # L& GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS +03AC..03CE ; Lowercase # L& [35] GREEK SMALL LETTER ALPHA WITH TONOS..GREEK SMALL LETTER OMEGA WITH TONOS +03D0..03D1 ; Lowercase # L& [2] GREEK BETA SYMBOL..GREEK THETA SYMBOL +03D5..03D7 ; Lowercase # L& [3] GREEK PHI SYMBOL..GREEK KAI SYMBOL +03D9 ; Lowercase # L& GREEK SMALL LETTER ARCHAIC KOPPA +03DB ; Lowercase # L& GREEK SMALL LETTER STIGMA +03DD ; Lowercase # L& GREEK SMALL LETTER DIGAMMA +03DF ; Lowercase # L& GREEK SMALL LETTER KOPPA +03E1 ; Lowercase # L& GREEK SMALL LETTER SAMPI +03E3 ; Lowercase # L& COPTIC SMALL LETTER SHEI +03E5 ; Lowercase # L& COPTIC SMALL LETTER FEI +03E7 ; Lowercase # L& COPTIC SMALL LETTER KHEI +03E9 ; Lowercase # L& COPTIC SMALL LETTER HORI +03EB ; Lowercase # L& COPTIC SMALL LETTER GANGIA +03ED ; Lowercase # L& COPTIC SMALL LETTER SHIMA +03EF..03F3 ; Lowercase # L& [5] COPTIC SMALL LETTER DEI..GREEK LETTER YOT +03F5 ; Lowercase # L& GREEK LUNATE EPSILON SYMBOL +03F8 ; Lowercase # L& GREEK SMALL LETTER SHO +03FB..03FC ; Lowercase # L& [2] GREEK SMALL LETTER SAN..GREEK RHO WITH STROKE SYMBOL +0430..045F ; Lowercase # L& [48] CYRILLIC SMALL LETTER A..CYRILLIC SMALL LETTER DZHE +0461 ; Lowercase # L& CYRILLIC SMALL LETTER OMEGA +0463 ; Lowercase # L& CYRILLIC SMALL LETTER YAT +0465 ; Lowercase # L& CYRILLIC SMALL LETTER IOTIFIED E +0467 ; Lowercase # L& CYRILLIC SMALL LETTER LITTLE YUS +0469 ; Lowercase # L& CYRILLIC SMALL LETTER IOTIFIED LITTLE YUS +046B ; Lowercase # L& CYRILLIC SMALL LETTER BIG YUS +046D ; Lowercase # L& CYRILLIC SMALL LETTER IOTIFIED BIG YUS +046F ; Lowercase # L& CYRILLIC SMALL LETTER KSI +0471 ; Lowercase # L& CYRILLIC SMALL LETTER PSI +0473 ; Lowercase # L& CYRILLIC SMALL LETTER FITA +0475 ; Lowercase # L& CYRILLIC SMALL LETTER IZHITSA +0477 ; Lowercase # L& CYRILLIC SMALL LETTER IZHITSA WITH DOUBLE GRAVE ACCENT +0479 ; Lowercase # L& CYRILLIC SMALL LETTER UK +047B ; Lowercase # L& CYRILLIC SMALL LETTER ROUND OMEGA +047D ; Lowercase # L& CYRILLIC SMALL LETTER OMEGA WITH TITLO +047F ; Lowercase # L& CYRILLIC SMALL LETTER OT +0481 ; Lowercase # L& CYRILLIC SMALL LETTER KOPPA +048B ; Lowercase # L& CYRILLIC SMALL LETTER SHORT I WITH TAIL +048D ; Lowercase # L& CYRILLIC SMALL LETTER SEMISOFT SIGN +048F ; Lowercase # L& CYRILLIC SMALL LETTER ER WITH TICK +0491 ; Lowercase # L& CYRILLIC SMALL LETTER GHE WITH UPTURN +0493 ; Lowercase # L& CYRILLIC SMALL LETTER GHE WITH STROKE +0495 ; Lowercase # L& CYRILLIC SMALL LETTER GHE WITH MIDDLE HOOK +0497 ; Lowercase # L& CYRILLIC SMALL LETTER ZHE WITH DESCENDER +0499 ; Lowercase # L& CYRILLIC SMALL LETTER ZE WITH DESCENDER +049B ; Lowercase # L& CYRILLIC SMALL LETTER KA WITH DESCENDER +049D ; Lowercase # L& CYRILLIC SMALL LETTER KA WITH VERTICAL STROKE +049F ; Lowercase # L& CYRILLIC SMALL LETTER KA WITH STROKE +04A1 ; Lowercase # L& CYRILLIC SMALL LETTER BASHKIR KA +04A3 ; Lowercase # L& CYRILLIC SMALL LETTER EN WITH DESCENDER +04A5 ; Lowercase # L& CYRILLIC SMALL LIGATURE EN GHE +04A7 ; Lowercase # L& CYRILLIC SMALL LETTER PE WITH MIDDLE HOOK +04A9 ; Lowercase # L& CYRILLIC SMALL LETTER ABKHASIAN HA +04AB ; Lowercase # L& CYRILLIC SMALL LETTER ES WITH DESCENDER +04AD ; Lowercase # L& CYRILLIC SMALL LETTER TE WITH DESCENDER +04AF ; Lowercase # L& CYRILLIC SMALL LETTER STRAIGHT U +04B1 ; Lowercase # L& CYRILLIC SMALL LETTER STRAIGHT U WITH STROKE +04B3 ; Lowercase # L& CYRILLIC SMALL LETTER HA WITH DESCENDER +04B5 ; Lowercase # L& CYRILLIC SMALL LIGATURE TE TSE +04B7 ; Lowercase # L& CYRILLIC SMALL LETTER CHE WITH DESCENDER +04B9 ; Lowercase # L& CYRILLIC SMALL LETTER CHE WITH VERTICAL STROKE +04BB ; Lowercase # L& CYRILLIC SMALL LETTER SHHA +04BD ; Lowercase # L& CYRILLIC SMALL LETTER ABKHASIAN CHE +04BF ; Lowercase # L& CYRILLIC SMALL LETTER ABKHASIAN CHE WITH DESCENDER +04C2 ; Lowercase # L& CYRILLIC SMALL LETTER ZHE WITH BREVE +04C4 ; Lowercase # L& CYRILLIC SMALL LETTER KA WITH HOOK +04C6 ; Lowercase # L& CYRILLIC SMALL LETTER EL WITH TAIL +04C8 ; Lowercase # L& CYRILLIC SMALL LETTER EN WITH HOOK +04CA ; Lowercase # L& CYRILLIC SMALL LETTER EN WITH TAIL +04CC ; Lowercase # L& CYRILLIC SMALL LETTER KHAKASSIAN CHE +04CE..04CF ; Lowercase # L& [2] CYRILLIC SMALL LETTER EM WITH TAIL..CYRILLIC SMALL LETTER PALOCHKA +04D1 ; Lowercase # L& CYRILLIC SMALL LETTER A WITH BREVE +04D3 ; Lowercase # L& CYRILLIC SMALL LETTER A WITH DIAERESIS +04D5 ; Lowercase # L& CYRILLIC SMALL LIGATURE A IE +04D7 ; Lowercase # L& CYRILLIC SMALL LETTER IE WITH BREVE +04D9 ; Lowercase # L& CYRILLIC SMALL LETTER SCHWA +04DB ; Lowercase # L& CYRILLIC SMALL LETTER SCHWA WITH DIAERESIS +04DD ; Lowercase # L& CYRILLIC SMALL LETTER ZHE WITH DIAERESIS +04DF ; Lowercase # L& CYRILLIC SMALL LETTER ZE WITH DIAERESIS +04E1 ; Lowercase # L& CYRILLIC SMALL LETTER ABKHASIAN DZE +04E3 ; Lowercase # L& CYRILLIC SMALL LETTER I WITH MACRON +04E5 ; Lowercase # L& CYRILLIC SMALL LETTER I WITH DIAERESIS +04E7 ; Lowercase # L& CYRILLIC SMALL LETTER O WITH DIAERESIS +04E9 ; Lowercase # L& CYRILLIC SMALL LETTER BARRED O +04EB ; Lowercase # L& CYRILLIC SMALL LETTER BARRED O WITH DIAERESIS +04ED ; Lowercase # L& CYRILLIC SMALL LETTER E WITH DIAERESIS +04EF ; Lowercase # L& CYRILLIC SMALL LETTER U WITH MACRON +04F1 ; Lowercase # L& CYRILLIC SMALL LETTER U WITH DIAERESIS +04F3 ; Lowercase # L& CYRILLIC SMALL LETTER U WITH DOUBLE ACUTE +04F5 ; Lowercase # L& CYRILLIC SMALL LETTER CHE WITH DIAERESIS +04F7 ; Lowercase # L& CYRILLIC SMALL LETTER GHE WITH DESCENDER +04F9 ; Lowercase # L& CYRILLIC SMALL LETTER YERU WITH DIAERESIS +04FB ; Lowercase # L& CYRILLIC SMALL LETTER GHE WITH STROKE AND HOOK +04FD ; Lowercase # L& CYRILLIC SMALL LETTER HA WITH HOOK +04FF ; Lowercase # L& CYRILLIC SMALL LETTER HA WITH STROKE +0501 ; Lowercase # L& CYRILLIC SMALL LETTER KOMI DE +0503 ; Lowercase # L& CYRILLIC SMALL LETTER KOMI DJE +0505 ; Lowercase # L& CYRILLIC SMALL LETTER KOMI ZJE +0507 ; Lowercase # L& CYRILLIC SMALL LETTER KOMI DZJE +0509 ; Lowercase # L& CYRILLIC SMALL LETTER KOMI LJE +050B ; Lowercase # L& CYRILLIC SMALL LETTER KOMI NJE +050D ; Lowercase # L& CYRILLIC SMALL LETTER KOMI SJE +050F ; Lowercase # L& CYRILLIC SMALL LETTER KOMI TJE +0511 ; Lowercase # L& CYRILLIC SMALL LETTER REVERSED ZE +0513 ; Lowercase # L& CYRILLIC SMALL LETTER EL WITH HOOK +0515 ; Lowercase # L& CYRILLIC SMALL LETTER LHA +0517 ; Lowercase # L& CYRILLIC SMALL LETTER RHA +0519 ; Lowercase # L& CYRILLIC SMALL LETTER YAE +051B ; Lowercase # L& CYRILLIC SMALL LETTER QA +051D ; Lowercase # L& CYRILLIC SMALL LETTER WE +051F ; Lowercase # L& CYRILLIC SMALL LETTER ALEUT KA +0521 ; Lowercase # L& CYRILLIC SMALL LETTER EL WITH MIDDLE HOOK +0523 ; Lowercase # L& CYRILLIC SMALL LETTER EN WITH MIDDLE HOOK +0525 ; Lowercase # L& CYRILLIC SMALL LETTER PE WITH DESCENDER +0561..0587 ; Lowercase # L& [39] ARMENIAN SMALL LETTER AYB..ARMENIAN SMALL LIGATURE ECH YIWN +1D00..1D2B ; Lowercase # L& [44] LATIN LETTER SMALL CAPITAL A..CYRILLIC LETTER SMALL CAPITAL EL +1D2C..1D61 ; Lowercase # Lm [54] MODIFIER LETTER CAPITAL A..MODIFIER LETTER SMALL CHI +1D62..1D77 ; Lowercase # L& [22] LATIN SUBSCRIPT SMALL LETTER I..LATIN SMALL LETTER TURNED G +1D78 ; Lowercase # Lm MODIFIER LETTER CYRILLIC EN +1D79..1D9A ; Lowercase # L& [34] LATIN SMALL LETTER INSULAR G..LATIN SMALL LETTER EZH WITH RETROFLEX HOOK +1D9B..1DBF ; Lowercase # Lm [37] MODIFIER LETTER SMALL TURNED ALPHA..MODIFIER LETTER SMALL THETA +1E01 ; Lowercase # L& LATIN SMALL LETTER A WITH RING BELOW +1E03 ; Lowercase # L& LATIN SMALL LETTER B WITH DOT ABOVE +1E05 ; Lowercase # L& LATIN SMALL LETTER B WITH DOT BELOW +1E07 ; Lowercase # L& LATIN SMALL LETTER B WITH LINE BELOW +1E09 ; Lowercase # L& LATIN SMALL LETTER C WITH CEDILLA AND ACUTE +1E0B ; Lowercase # L& LATIN SMALL LETTER D WITH DOT ABOVE +1E0D ; Lowercase # L& LATIN SMALL LETTER D WITH DOT BELOW +1E0F ; Lowercase # L& LATIN SMALL LETTER D WITH LINE BELOW +1E11 ; Lowercase # L& LATIN SMALL LETTER D WITH CEDILLA +1E13 ; Lowercase # L& LATIN SMALL LETTER D WITH CIRCUMFLEX BELOW +1E15 ; Lowercase # L& LATIN SMALL LETTER E WITH MACRON AND GRAVE +1E17 ; Lowercase # L& LATIN SMALL LETTER E WITH MACRON AND ACUTE +1E19 ; Lowercase # L& LATIN SMALL LETTER E WITH CIRCUMFLEX BELOW +1E1B ; Lowercase # L& LATIN SMALL LETTER E WITH TILDE BELOW +1E1D ; Lowercase # L& LATIN SMALL LETTER E WITH CEDILLA AND BREVE +1E1F ; Lowercase # L& LATIN SMALL LETTER F WITH DOT ABOVE +1E21 ; Lowercase # L& LATIN SMALL LETTER G WITH MACRON +1E23 ; Lowercase # L& LATIN SMALL LETTER H WITH DOT ABOVE +1E25 ; Lowercase # L& LATIN SMALL LETTER H WITH DOT BELOW +1E27 ; Lowercase # L& LATIN SMALL LETTER H WITH DIAERESIS +1E29 ; Lowercase # L& LATIN SMALL LETTER H WITH CEDILLA +1E2B ; Lowercase # L& LATIN SMALL LETTER H WITH BREVE BELOW +1E2D ; Lowercase # L& LATIN SMALL LETTER I WITH TILDE BELOW +1E2F ; Lowercase # L& LATIN SMALL LETTER I WITH DIAERESIS AND ACUTE +1E31 ; Lowercase # L& LATIN SMALL LETTER K WITH ACUTE +1E33 ; Lowercase # L& LATIN SMALL LETTER K WITH DOT BELOW +1E35 ; Lowercase # L& LATIN SMALL LETTER K WITH LINE BELOW +1E37 ; Lowercase # L& LATIN SMALL LETTER L WITH DOT BELOW +1E39 ; Lowercase # L& LATIN SMALL LETTER L WITH DOT BELOW AND MACRON +1E3B ; Lowercase # L& LATIN SMALL LETTER L WITH LINE BELOW +1E3D ; Lowercase # L& LATIN SMALL LETTER L WITH CIRCUMFLEX BELOW +1E3F ; Lowercase # L& LATIN SMALL LETTER M WITH ACUTE +1E41 ; Lowercase # L& LATIN SMALL LETTER M WITH DOT ABOVE +1E43 ; Lowercase # L& LATIN SMALL LETTER M WITH DOT BELOW +1E45 ; Lowercase # L& LATIN SMALL LETTER N WITH DOT ABOVE +1E47 ; Lowercase # L& LATIN SMALL LETTER N WITH DOT BELOW +1E49 ; Lowercase # L& LATIN SMALL LETTER N WITH LINE BELOW +1E4B ; Lowercase # L& LATIN SMALL LETTER N WITH CIRCUMFLEX BELOW +1E4D ; Lowercase # L& LATIN SMALL LETTER O WITH TILDE AND ACUTE +1E4F ; Lowercase # L& LATIN SMALL LETTER O WITH TILDE AND DIAERESIS +1E51 ; Lowercase # L& LATIN SMALL LETTER O WITH MACRON AND GRAVE +1E53 ; Lowercase # L& LATIN SMALL LETTER O WITH MACRON AND ACUTE +1E55 ; Lowercase # L& LATIN SMALL LETTER P WITH ACUTE +1E57 ; Lowercase # L& LATIN SMALL LETTER P WITH DOT ABOVE +1E59 ; Lowercase # L& LATIN SMALL LETTER R WITH DOT ABOVE +1E5B ; Lowercase # L& LATIN SMALL LETTER R WITH DOT BELOW +1E5D ; Lowercase # L& LATIN SMALL LETTER R WITH DOT BELOW AND MACRON +1E5F ; Lowercase # L& LATIN SMALL LETTER R WITH LINE BELOW +1E61 ; Lowercase # L& LATIN SMALL LETTER S WITH DOT ABOVE +1E63 ; Lowercase # L& LATIN SMALL LETTER S WITH DOT BELOW +1E65 ; Lowercase # L& LATIN SMALL LETTER S WITH ACUTE AND DOT ABOVE +1E67 ; Lowercase # L& LATIN SMALL LETTER S WITH CARON AND DOT ABOVE +1E69 ; Lowercase # L& LATIN SMALL LETTER S WITH DOT BELOW AND DOT ABOVE +1E6B ; Lowercase # L& LATIN SMALL LETTER T WITH DOT ABOVE +1E6D ; Lowercase # L& LATIN SMALL LETTER T WITH DOT BELOW +1E6F ; Lowercase # L& LATIN SMALL LETTER T WITH LINE BELOW +1E71 ; Lowercase # L& LATIN SMALL LETTER T WITH CIRCUMFLEX BELOW +1E73 ; Lowercase # L& LATIN SMALL LETTER U WITH DIAERESIS BELOW +1E75 ; Lowercase # L& LATIN SMALL LETTER U WITH TILDE BELOW +1E77 ; Lowercase # L& LATIN SMALL LETTER U WITH CIRCUMFLEX BELOW +1E79 ; Lowercase # L& LATIN SMALL LETTER U WITH TILDE AND ACUTE +1E7B ; Lowercase # L& LATIN SMALL LETTER U WITH MACRON AND DIAERESIS +1E7D ; Lowercase # L& LATIN SMALL LETTER V WITH TILDE +1E7F ; Lowercase # L& LATIN SMALL LETTER V WITH DOT BELOW +1E81 ; Lowercase # L& LATIN SMALL LETTER W WITH GRAVE +1E83 ; Lowercase # L& LATIN SMALL LETTER W WITH ACUTE +1E85 ; Lowercase # L& LATIN SMALL LETTER W WITH DIAERESIS +1E87 ; Lowercase # L& LATIN SMALL LETTER W WITH DOT ABOVE +1E89 ; Lowercase # L& LATIN SMALL LETTER W WITH DOT BELOW +1E8B ; Lowercase # L& LATIN SMALL LETTER X WITH DOT ABOVE +1E8D ; Lowercase # L& LATIN SMALL LETTER X WITH DIAERESIS +1E8F ; Lowercase # L& LATIN SMALL LETTER Y WITH DOT ABOVE +1E91 ; Lowercase # L& LATIN SMALL LETTER Z WITH CIRCUMFLEX +1E93 ; Lowercase # L& LATIN SMALL LETTER Z WITH DOT BELOW +1E95..1E9D ; Lowercase # L& [9] LATIN SMALL LETTER Z WITH LINE BELOW..LATIN SMALL LETTER LONG S WITH HIGH STROKE +1E9F ; Lowercase # L& LATIN SMALL LETTER DELTA +1EA1 ; Lowercase # L& LATIN SMALL LETTER A WITH DOT BELOW +1EA3 ; Lowercase # L& LATIN SMALL LETTER A WITH HOOK ABOVE +1EA5 ; Lowercase # L& LATIN SMALL LETTER A WITH CIRCUMFLEX AND ACUTE +1EA7 ; Lowercase # L& LATIN SMALL LETTER A WITH CIRCUMFLEX AND GRAVE +1EA9 ; Lowercase # L& LATIN SMALL LETTER A WITH CIRCUMFLEX AND HOOK ABOVE +1EAB ; Lowercase # L& LATIN SMALL LETTER A WITH CIRCUMFLEX AND TILDE +1EAD ; Lowercase # L& LATIN SMALL LETTER A WITH CIRCUMFLEX AND DOT BELOW +1EAF ; Lowercase # L& LATIN SMALL LETTER A WITH BREVE AND ACUTE +1EB1 ; Lowercase # L& LATIN SMALL LETTER A WITH BREVE AND GRAVE +1EB3 ; Lowercase # L& LATIN SMALL LETTER A WITH BREVE AND HOOK ABOVE +1EB5 ; Lowercase # L& LATIN SMALL LETTER A WITH BREVE AND TILDE +1EB7 ; Lowercase # L& LATIN SMALL LETTER A WITH BREVE AND DOT BELOW +1EB9 ; Lowercase # L& LATIN SMALL LETTER E WITH DOT BELOW +1EBB ; Lowercase # L& LATIN SMALL LETTER E WITH HOOK ABOVE +1EBD ; Lowercase # L& LATIN SMALL LETTER E WITH TILDE +1EBF ; Lowercase # L& LATIN SMALL LETTER E WITH CIRCUMFLEX AND ACUTE +1EC1 ; Lowercase # L& LATIN SMALL LETTER E WITH CIRCUMFLEX AND GRAVE +1EC3 ; Lowercase # L& LATIN SMALL LETTER E WITH CIRCUMFLEX AND HOOK ABOVE +1EC5 ; Lowercase # L& LATIN SMALL LETTER E WITH CIRCUMFLEX AND TILDE +1EC7 ; Lowercase # L& LATIN SMALL LETTER E WITH CIRCUMFLEX AND DOT BELOW +1EC9 ; Lowercase # L& LATIN SMALL LETTER I WITH HOOK ABOVE +1ECB ; Lowercase # L& LATIN SMALL LETTER I WITH DOT BELOW +1ECD ; Lowercase # L& LATIN SMALL LETTER O WITH DOT BELOW +1ECF ; Lowercase # L& LATIN SMALL LETTER O WITH HOOK ABOVE +1ED1 ; Lowercase # L& LATIN SMALL LETTER O WITH CIRCUMFLEX AND ACUTE +1ED3 ; Lowercase # L& LATIN SMALL LETTER O WITH CIRCUMFLEX AND GRAVE +1ED5 ; Lowercase # L& LATIN SMALL LETTER O WITH CIRCUMFLEX AND HOOK ABOVE +1ED7 ; Lowercase # L& LATIN SMALL LETTER O WITH CIRCUMFLEX AND TILDE +1ED9 ; Lowercase # L& LATIN SMALL LETTER O WITH CIRCUMFLEX AND DOT BELOW +1EDB ; Lowercase # L& LATIN SMALL LETTER O WITH HORN AND ACUTE +1EDD ; Lowercase # L& LATIN SMALL LETTER O WITH HORN AND GRAVE +1EDF ; Lowercase # L& LATIN SMALL LETTER O WITH HORN AND HOOK ABOVE +1EE1 ; Lowercase # L& LATIN SMALL LETTER O WITH HORN AND TILDE +1EE3 ; Lowercase # L& LATIN SMALL LETTER O WITH HORN AND DOT BELOW +1EE5 ; Lowercase # L& LATIN SMALL LETTER U WITH DOT BELOW +1EE7 ; Lowercase # L& LATIN SMALL LETTER U WITH HOOK ABOVE +1EE9 ; Lowercase # L& LATIN SMALL LETTER U WITH HORN AND ACUTE +1EEB ; Lowercase # L& LATIN SMALL LETTER U WITH HORN AND GRAVE +1EED ; Lowercase # L& LATIN SMALL LETTER U WITH HORN AND HOOK ABOVE +1EEF ; Lowercase # L& LATIN SMALL LETTER U WITH HORN AND TILDE +1EF1 ; Lowercase # L& LATIN SMALL LETTER U WITH HORN AND DOT BELOW +1EF3 ; Lowercase # L& LATIN SMALL LETTER Y WITH GRAVE +1EF5 ; Lowercase # L& LATIN SMALL LETTER Y WITH DOT BELOW +1EF7 ; Lowercase # L& LATIN SMALL LETTER Y WITH HOOK ABOVE +1EF9 ; Lowercase # L& LATIN SMALL LETTER Y WITH TILDE +1EFB ; Lowercase # L& LATIN SMALL LETTER MIDDLE-WELSH LL +1EFD ; Lowercase # L& LATIN SMALL LETTER MIDDLE-WELSH V +1EFF..1F07 ; Lowercase # L& [9] LATIN SMALL LETTER Y WITH LOOP..GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI +1F10..1F15 ; Lowercase # L& [6] GREEK SMALL LETTER EPSILON WITH PSILI..GREEK SMALL LETTER EPSILON WITH DASIA AND OXIA +1F20..1F27 ; Lowercase # L& [8] GREEK SMALL LETTER ETA WITH PSILI..GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI +1F30..1F37 ; Lowercase # L& [8] GREEK SMALL LETTER IOTA WITH PSILI..GREEK SMALL LETTER IOTA WITH DASIA AND PERISPOMENI +1F40..1F45 ; Lowercase # L& [6] GREEK SMALL LETTER OMICRON WITH PSILI..GREEK SMALL LETTER OMICRON WITH DASIA AND OXIA +1F50..1F57 ; Lowercase # L& [8] GREEK SMALL LETTER UPSILON WITH PSILI..GREEK SMALL LETTER UPSILON WITH DASIA AND PERISPOMENI +1F60..1F67 ; Lowercase # L& [8] GREEK SMALL LETTER OMEGA WITH PSILI..GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI +1F70..1F7D ; Lowercase # L& [14] GREEK SMALL LETTER ALPHA WITH VARIA..GREEK SMALL LETTER OMEGA WITH OXIA +1F80..1F87 ; Lowercase # L& [8] GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI..GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI +1F90..1F97 ; Lowercase # L& [8] GREEK SMALL LETTER ETA WITH PSILI AND YPOGEGRAMMENI..GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI +1FA0..1FA7 ; Lowercase # L& [8] GREEK SMALL LETTER OMEGA WITH PSILI AND YPOGEGRAMMENI..GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI +1FB0..1FB4 ; Lowercase # L& [5] GREEK SMALL LETTER ALPHA WITH VRACHY..GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI +1FB6..1FB7 ; Lowercase # L& [2] GREEK SMALL LETTER ALPHA WITH PERISPOMENI..GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI +1FBE ; Lowercase # L& GREEK PROSGEGRAMMENI +1FC2..1FC4 ; Lowercase # L& [3] GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI..GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI +1FC6..1FC7 ; Lowercase # L& [2] GREEK SMALL LETTER ETA WITH PERISPOMENI..GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI +1FD0..1FD3 ; Lowercase # L& [4] GREEK SMALL LETTER IOTA WITH VRACHY..GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA +1FD6..1FD7 ; Lowercase # L& [2] GREEK SMALL LETTER IOTA WITH PERISPOMENI..GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI +1FE0..1FE7 ; Lowercase # L& [8] GREEK SMALL LETTER UPSILON WITH VRACHY..GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI +1FF2..1FF4 ; Lowercase # L& [3] GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI..GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI +1FF6..1FF7 ; Lowercase # L& [2] GREEK SMALL LETTER OMEGA WITH PERISPOMENI..GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI +2090..2094 ; Lowercase # Lm [5] LATIN SUBSCRIPT SMALL LETTER A..LATIN SUBSCRIPT SMALL LETTER SCHWA +210A ; Lowercase # L& SCRIPT SMALL G +210E..210F ; Lowercase # L& [2] PLANCK CONSTANT..PLANCK CONSTANT OVER TWO PI +2113 ; Lowercase # L& SCRIPT SMALL L +212F ; Lowercase # L& SCRIPT SMALL E +2134 ; Lowercase # L& SCRIPT SMALL O +2139 ; Lowercase # L& INFORMATION SOURCE +213C..213D ; Lowercase # L& [2] DOUBLE-STRUCK SMALL PI..DOUBLE-STRUCK SMALL GAMMA +2146..2149 ; Lowercase # L& [4] DOUBLE-STRUCK ITALIC SMALL D..DOUBLE-STRUCK ITALIC SMALL J +214E ; Lowercase # L& TURNED SMALL F +2170..217F ; Lowercase # Nl [16] SMALL ROMAN NUMERAL ONE..SMALL ROMAN NUMERAL ONE THOUSAND +2184 ; Lowercase # L& LATIN SMALL LETTER REVERSED C +24D0..24E9 ; Lowercase # So [26] CIRCLED LATIN SMALL LETTER A..CIRCLED LATIN SMALL LETTER Z +2C30..2C5E ; Lowercase # L& [47] GLAGOLITIC SMALL LETTER AZU..GLAGOLITIC SMALL LETTER LATINATE MYSLITE +2C61 ; Lowercase # L& LATIN SMALL LETTER L WITH DOUBLE BAR +2C65..2C66 ; Lowercase # L& [2] LATIN SMALL LETTER A WITH STROKE..LATIN SMALL LETTER T WITH DIAGONAL STROKE +2C68 ; Lowercase # L& LATIN SMALL LETTER H WITH DESCENDER +2C6A ; Lowercase # L& LATIN SMALL LETTER K WITH DESCENDER +2C6C ; Lowercase # L& LATIN SMALL LETTER Z WITH DESCENDER +2C71 ; Lowercase # L& LATIN SMALL LETTER V WITH RIGHT HOOK +2C73..2C74 ; Lowercase # L& [2] LATIN SMALL LETTER W WITH HOOK..LATIN SMALL LETTER V WITH CURL +2C76..2C7C ; Lowercase # L& [7] LATIN SMALL LETTER HALF H..LATIN SUBSCRIPT SMALL LETTER J +2C7D ; Lowercase # Lm MODIFIER LETTER CAPITAL V +2C81 ; Lowercase # L& COPTIC SMALL LETTER ALFA +2C83 ; Lowercase # L& COPTIC SMALL LETTER VIDA +2C85 ; Lowercase # L& COPTIC SMALL LETTER GAMMA +2C87 ; Lowercase # L& COPTIC SMALL LETTER DALDA +2C89 ; Lowercase # L& COPTIC SMALL LETTER EIE +2C8B ; Lowercase # L& COPTIC SMALL LETTER SOU +2C8D ; Lowercase # L& COPTIC SMALL LETTER ZATA +2C8F ; Lowercase # L& COPTIC SMALL LETTER HATE +2C91 ; Lowercase # L& COPTIC SMALL LETTER THETHE +2C93 ; Lowercase # L& COPTIC SMALL LETTER IAUDA +2C95 ; Lowercase # L& COPTIC SMALL LETTER KAPA +2C97 ; Lowercase # L& COPTIC SMALL LETTER LAULA +2C99 ; Lowercase # L& COPTIC SMALL LETTER MI +2C9B ; Lowercase # L& COPTIC SMALL LETTER NI +2C9D ; Lowercase # L& COPTIC SMALL LETTER KSI +2C9F ; Lowercase # L& COPTIC SMALL LETTER O +2CA1 ; Lowercase # L& COPTIC SMALL LETTER PI +2CA3 ; Lowercase # L& COPTIC SMALL LETTER RO +2CA5 ; Lowercase # L& COPTIC SMALL LETTER SIMA +2CA7 ; Lowercase # L& COPTIC SMALL LETTER TAU +2CA9 ; Lowercase # L& COPTIC SMALL LETTER UA +2CAB ; Lowercase # L& COPTIC SMALL LETTER FI +2CAD ; Lowercase # L& COPTIC SMALL LETTER KHI +2CAF ; Lowercase # L& COPTIC SMALL LETTER PSI +2CB1 ; Lowercase # L& COPTIC SMALL LETTER OOU +2CB3 ; Lowercase # L& COPTIC SMALL LETTER DIALECT-P ALEF +2CB5 ; Lowercase # L& COPTIC SMALL LETTER OLD COPTIC AIN +2CB7 ; Lowercase # L& COPTIC SMALL LETTER CRYPTOGRAMMIC EIE +2CB9 ; Lowercase # L& COPTIC SMALL LETTER DIALECT-P KAPA +2CBB ; Lowercase # L& COPTIC SMALL LETTER DIALECT-P NI +2CBD ; Lowercase # L& COPTIC SMALL LETTER CRYPTOGRAMMIC NI +2CBF ; Lowercase # L& COPTIC SMALL LETTER OLD COPTIC OOU +2CC1 ; Lowercase # L& COPTIC SMALL LETTER SAMPI +2CC3 ; Lowercase # L& COPTIC SMALL LETTER CROSSED SHEI +2CC5 ; Lowercase # L& COPTIC SMALL LETTER OLD COPTIC SHEI +2CC7 ; Lowercase # L& COPTIC SMALL LETTER OLD COPTIC ESH +2CC9 ; Lowercase # L& COPTIC SMALL LETTER AKHMIMIC KHEI +2CCB ; Lowercase # L& COPTIC SMALL LETTER DIALECT-P HORI +2CCD ; Lowercase # L& COPTIC SMALL LETTER OLD COPTIC HORI +2CCF ; Lowercase # L& COPTIC SMALL LETTER OLD COPTIC HA +2CD1 ; Lowercase # L& COPTIC SMALL LETTER L-SHAPED HA +2CD3 ; Lowercase # L& COPTIC SMALL LETTER OLD COPTIC HEI +2CD5 ; Lowercase # L& COPTIC SMALL LETTER OLD COPTIC HAT +2CD7 ; Lowercase # L& COPTIC SMALL LETTER OLD COPTIC GANGIA +2CD9 ; Lowercase # L& COPTIC SMALL LETTER OLD COPTIC DJA +2CDB ; Lowercase # L& COPTIC SMALL LETTER OLD COPTIC SHIMA +2CDD ; Lowercase # L& COPTIC SMALL LETTER OLD NUBIAN SHIMA +2CDF ; Lowercase # L& COPTIC SMALL LETTER OLD NUBIAN NGI +2CE1 ; Lowercase # L& COPTIC SMALL LETTER OLD NUBIAN NYI +2CE3..2CE4 ; Lowercase # L& [2] COPTIC SMALL LETTER OLD NUBIAN WAU..COPTIC SYMBOL KAI +2CEC ; Lowercase # L& COPTIC SMALL LETTER CRYPTOGRAMMIC SHEI +2CEE ; Lowercase # L& COPTIC SMALL LETTER CRYPTOGRAMMIC GANGIA +2D00..2D25 ; Lowercase # L& [38] GEORGIAN SMALL LETTER AN..GEORGIAN SMALL LETTER HOE +A641 ; Lowercase # L& CYRILLIC SMALL LETTER ZEMLYA +A643 ; Lowercase # L& CYRILLIC SMALL LETTER DZELO +A645 ; Lowercase # L& CYRILLIC SMALL LETTER REVERSED DZE +A647 ; Lowercase # L& CYRILLIC SMALL LETTER IOTA +A649 ; Lowercase # L& CYRILLIC SMALL LETTER DJERV +A64B ; Lowercase # L& CYRILLIC SMALL LETTER MONOGRAPH UK +A64D ; Lowercase # L& CYRILLIC SMALL LETTER BROAD OMEGA +A64F ; Lowercase # L& CYRILLIC SMALL LETTER NEUTRAL YER +A651 ; Lowercase # L& CYRILLIC SMALL LETTER YERU WITH BACK YER +A653 ; Lowercase # L& CYRILLIC SMALL LETTER IOTIFIED YAT +A655 ; Lowercase # L& CYRILLIC SMALL LETTER REVERSED YU +A657 ; Lowercase # L& CYRILLIC SMALL LETTER IOTIFIED A +A659 ; Lowercase # L& CYRILLIC SMALL LETTER CLOSED LITTLE YUS +A65B ; Lowercase # L& CYRILLIC SMALL LETTER BLENDED YUS +A65D ; Lowercase # L& CYRILLIC SMALL LETTER IOTIFIED CLOSED LITTLE YUS +A65F ; Lowercase # L& CYRILLIC SMALL LETTER YN +A663 ; Lowercase # L& CYRILLIC SMALL LETTER SOFT DE +A665 ; Lowercase # L& CYRILLIC SMALL LETTER SOFT EL +A667 ; Lowercase # L& CYRILLIC SMALL LETTER SOFT EM +A669 ; Lowercase # L& CYRILLIC SMALL LETTER MONOCULAR O +A66B ; Lowercase # L& CYRILLIC SMALL LETTER BINOCULAR O +A66D ; Lowercase # L& CYRILLIC SMALL LETTER DOUBLE MONOCULAR O +A681 ; Lowercase # L& CYRILLIC SMALL LETTER DWE +A683 ; Lowercase # L& CYRILLIC SMALL LETTER DZWE +A685 ; Lowercase # L& CYRILLIC SMALL LETTER ZHWE +A687 ; Lowercase # L& CYRILLIC SMALL LETTER CCHE +A689 ; Lowercase # L& CYRILLIC SMALL LETTER DZZE +A68B ; Lowercase # L& CYRILLIC SMALL LETTER TE WITH MIDDLE HOOK +A68D ; Lowercase # L& CYRILLIC SMALL LETTER TWE +A68F ; Lowercase # L& CYRILLIC SMALL LETTER TSWE +A691 ; Lowercase # L& CYRILLIC SMALL LETTER TSSE +A693 ; Lowercase # L& CYRILLIC SMALL LETTER TCHE +A695 ; Lowercase # L& CYRILLIC SMALL LETTER HWE +A697 ; Lowercase # L& CYRILLIC SMALL LETTER SHWE +A723 ; Lowercase # L& LATIN SMALL LETTER EGYPTOLOGICAL ALEF +A725 ; Lowercase # L& LATIN SMALL LETTER EGYPTOLOGICAL AIN +A727 ; Lowercase # L& LATIN SMALL LETTER HENG +A729 ; Lowercase # L& LATIN SMALL LETTER TZ +A72B ; Lowercase # L& LATIN SMALL LETTER TRESILLO +A72D ; Lowercase # L& LATIN SMALL LETTER CUATRILLO +A72F..A731 ; Lowercase # L& [3] LATIN SMALL LETTER CUATRILLO WITH COMMA..LATIN LETTER SMALL CAPITAL S +A733 ; Lowercase # L& LATIN SMALL LETTER AA +A735 ; Lowercase # L& LATIN SMALL LETTER AO +A737 ; Lowercase # L& LATIN SMALL LETTER AU +A739 ; Lowercase # L& LATIN SMALL LETTER AV +A73B ; Lowercase # L& LATIN SMALL LETTER AV WITH HORIZONTAL BAR +A73D ; Lowercase # L& LATIN SMALL LETTER AY +A73F ; Lowercase # L& LATIN SMALL LETTER REVERSED C WITH DOT +A741 ; Lowercase # L& LATIN SMALL LETTER K WITH STROKE +A743 ; Lowercase # L& LATIN SMALL LETTER K WITH DIAGONAL STROKE +A745 ; Lowercase # L& LATIN SMALL LETTER K WITH STROKE AND DIAGONAL STROKE +A747 ; Lowercase # L& LATIN SMALL LETTER BROKEN L +A749 ; Lowercase # L& LATIN SMALL LETTER L WITH HIGH STROKE +A74B ; Lowercase # L& LATIN SMALL LETTER O WITH LONG STROKE OVERLAY +A74D ; Lowercase # L& LATIN SMALL LETTER O WITH LOOP +A74F ; Lowercase # L& LATIN SMALL LETTER OO +A751 ; Lowercase # L& LATIN SMALL LETTER P WITH STROKE THROUGH DESCENDER +A753 ; Lowercase # L& LATIN SMALL LETTER P WITH FLOURISH +A755 ; Lowercase # L& LATIN SMALL LETTER P WITH SQUIRREL TAIL +A757 ; Lowercase # L& LATIN SMALL LETTER Q WITH STROKE THROUGH DESCENDER +A759 ; Lowercase # L& LATIN SMALL LETTER Q WITH DIAGONAL STROKE +A75B ; Lowercase # L& LATIN SMALL LETTER R ROTUNDA +A75D ; Lowercase # L& LATIN SMALL LETTER RUM ROTUNDA +A75F ; Lowercase # L& LATIN SMALL LETTER V WITH DIAGONAL STROKE +A761 ; Lowercase # L& LATIN SMALL LETTER VY +A763 ; Lowercase # L& LATIN SMALL LETTER VISIGOTHIC Z +A765 ; Lowercase # L& LATIN SMALL LETTER THORN WITH STROKE +A767 ; Lowercase # L& LATIN SMALL LETTER THORN WITH STROKE THROUGH DESCENDER +A769 ; Lowercase # L& LATIN SMALL LETTER VEND +A76B ; Lowercase # L& LATIN SMALL LETTER ET +A76D ; Lowercase # L& LATIN SMALL LETTER IS +A76F ; Lowercase # L& LATIN SMALL LETTER CON +A770 ; Lowercase # Lm MODIFIER LETTER US +A771..A778 ; Lowercase # L& [8] LATIN SMALL LETTER DUM..LATIN SMALL LETTER UM +A77A ; Lowercase # L& LATIN SMALL LETTER INSULAR D +A77C ; Lowercase # L& LATIN SMALL LETTER INSULAR F +A77F ; Lowercase # L& LATIN SMALL LETTER TURNED INSULAR G +A781 ; Lowercase # L& LATIN SMALL LETTER TURNED L +A783 ; Lowercase # L& LATIN SMALL LETTER INSULAR R +A785 ; Lowercase # L& LATIN SMALL LETTER INSULAR S +A787 ; Lowercase # L& LATIN SMALL LETTER INSULAR T +A78C ; Lowercase # L& LATIN SMALL LETTER SALTILLO +FB00..FB06 ; Lowercase # L& [7] LATIN SMALL LIGATURE FF..LATIN SMALL LIGATURE ST +FB13..FB17 ; Lowercase # L& [5] ARMENIAN SMALL LIGATURE MEN NOW..ARMENIAN SMALL LIGATURE MEN XEH +FF41..FF5A ; Lowercase # L& [26] FULLWIDTH LATIN SMALL LETTER A..FULLWIDTH LATIN SMALL LETTER Z +10428..1044F ; Lowercase # L& [40] DESERET SMALL LETTER LONG I..DESERET SMALL LETTER EW +1D41A..1D433 ; Lowercase # L& [26] MATHEMATICAL BOLD SMALL A..MATHEMATICAL BOLD SMALL Z +1D44E..1D454 ; Lowercase # L& [7] MATHEMATICAL ITALIC SMALL A..MATHEMATICAL ITALIC SMALL G +1D456..1D467 ; Lowercase # L& [18] MATHEMATICAL ITALIC SMALL I..MATHEMATICAL ITALIC SMALL Z +1D482..1D49B ; Lowercase # L& [26] MATHEMATICAL BOLD ITALIC SMALL A..MATHEMATICAL BOLD ITALIC SMALL Z +1D4B6..1D4B9 ; Lowercase # L& [4] MATHEMATICAL SCRIPT SMALL A..MATHEMATICAL SCRIPT SMALL D +1D4BB ; Lowercase # L& MATHEMATICAL SCRIPT SMALL F +1D4BD..1D4C3 ; Lowercase # L& [7] MATHEMATICAL SCRIPT SMALL H..MATHEMATICAL SCRIPT SMALL N +1D4C5..1D4CF ; Lowercase # L& [11] MATHEMATICAL SCRIPT SMALL P..MATHEMATICAL SCRIPT SMALL Z +1D4EA..1D503 ; Lowercase # L& [26] MATHEMATICAL BOLD SCRIPT SMALL A..MATHEMATICAL BOLD SCRIPT SMALL Z +1D51E..1D537 ; Lowercase # L& [26] MATHEMATICAL FRAKTUR SMALL A..MATHEMATICAL FRAKTUR SMALL Z +1D552..1D56B ; Lowercase # L& [26] MATHEMATICAL DOUBLE-STRUCK SMALL A..MATHEMATICAL DOUBLE-STRUCK SMALL Z +1D586..1D59F ; Lowercase # L& [26] MATHEMATICAL BOLD FRAKTUR SMALL A..MATHEMATICAL BOLD FRAKTUR SMALL Z +1D5BA..1D5D3 ; Lowercase # L& [26] MATHEMATICAL SANS-SERIF SMALL A..MATHEMATICAL SANS-SERIF SMALL Z +1D5EE..1D607 ; Lowercase # L& [26] MATHEMATICAL SANS-SERIF BOLD SMALL A..MATHEMATICAL SANS-SERIF BOLD SMALL Z +1D622..1D63B ; Lowercase # L& [26] MATHEMATICAL SANS-SERIF ITALIC SMALL A..MATHEMATICAL SANS-SERIF ITALIC SMALL Z +1D656..1D66F ; Lowercase # L& [26] MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL A..MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL Z +1D68A..1D6A5 ; Lowercase # L& [28] MATHEMATICAL MONOSPACE SMALL A..MATHEMATICAL ITALIC SMALL DOTLESS J +1D6C2..1D6DA ; Lowercase # L& [25] MATHEMATICAL BOLD SMALL ALPHA..MATHEMATICAL BOLD SMALL OMEGA +1D6DC..1D6E1 ; Lowercase # L& [6] MATHEMATICAL BOLD EPSILON SYMBOL..MATHEMATICAL BOLD PI SYMBOL +1D6FC..1D714 ; Lowercase # L& [25] MATHEMATICAL ITALIC SMALL ALPHA..MATHEMATICAL ITALIC SMALL OMEGA +1D716..1D71B ; Lowercase # L& [6] MATHEMATICAL ITALIC EPSILON SYMBOL..MATHEMATICAL ITALIC PI SYMBOL +1D736..1D74E ; Lowercase # L& [25] MATHEMATICAL BOLD ITALIC SMALL ALPHA..MATHEMATICAL BOLD ITALIC SMALL OMEGA +1D750..1D755 ; Lowercase # L& [6] MATHEMATICAL BOLD ITALIC EPSILON SYMBOL..MATHEMATICAL BOLD ITALIC PI SYMBOL +1D770..1D788 ; Lowercase # L& [25] MATHEMATICAL SANS-SERIF BOLD SMALL ALPHA..MATHEMATICAL SANS-SERIF BOLD SMALL OMEGA +1D78A..1D78F ; Lowercase # L& [6] MATHEMATICAL SANS-SERIF BOLD EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD PI SYMBOL +1D7AA..1D7C2 ; Lowercase # L& [25] MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL ALPHA..MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL OMEGA +1D7C4..1D7C9 ; Lowercase # L& [6] MATHEMATICAL SANS-SERIF BOLD ITALIC EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD ITALIC PI SYMBOL +1D7CB ; Lowercase # L& MATHEMATICAL BOLD SMALL DIGAMMA + +# Total code points: 1908 + +# ================================================ + +# Derived Property: Uppercase +# Generated from: Lu + Other_Uppercase + +0041..005A ; Uppercase # L& [26] LATIN CAPITAL LETTER A..LATIN CAPITAL LETTER Z +00C0..00D6 ; Uppercase # L& [23] LATIN CAPITAL LETTER A WITH GRAVE..LATIN CAPITAL LETTER O WITH DIAERESIS +00D8..00DE ; Uppercase # L& [7] LATIN CAPITAL LETTER O WITH STROKE..LATIN CAPITAL LETTER THORN +0100 ; Uppercase # L& LATIN CAPITAL LETTER A WITH MACRON +0102 ; Uppercase # L& LATIN CAPITAL LETTER A WITH BREVE +0104 ; Uppercase # L& LATIN CAPITAL LETTER A WITH OGONEK +0106 ; Uppercase # L& LATIN CAPITAL LETTER C WITH ACUTE +0108 ; Uppercase # L& LATIN CAPITAL LETTER C WITH CIRCUMFLEX +010A ; Uppercase # L& LATIN CAPITAL LETTER C WITH DOT ABOVE +010C ; Uppercase # L& LATIN CAPITAL LETTER C WITH CARON +010E ; Uppercase # L& LATIN CAPITAL LETTER D WITH CARON +0110 ; Uppercase # L& LATIN CAPITAL LETTER D WITH STROKE +0112 ; Uppercase # L& LATIN CAPITAL LETTER E WITH MACRON +0114 ; Uppercase # L& LATIN CAPITAL LETTER E WITH BREVE +0116 ; Uppercase # L& LATIN CAPITAL LETTER E WITH DOT ABOVE +0118 ; Uppercase # L& LATIN CAPITAL LETTER E WITH OGONEK +011A ; Uppercase # L& LATIN CAPITAL LETTER E WITH CARON +011C ; Uppercase # L& LATIN CAPITAL LETTER G WITH CIRCUMFLEX +011E ; Uppercase # L& LATIN CAPITAL LETTER G WITH BREVE +0120 ; Uppercase # L& LATIN CAPITAL LETTER G WITH DOT ABOVE +0122 ; Uppercase # L& LATIN CAPITAL LETTER G WITH CEDILLA +0124 ; Uppercase # L& LATIN CAPITAL LETTER H WITH CIRCUMFLEX +0126 ; Uppercase # L& LATIN CAPITAL LETTER H WITH STROKE +0128 ; Uppercase # L& LATIN CAPITAL LETTER I WITH TILDE +012A ; Uppercase # L& LATIN CAPITAL LETTER I WITH MACRON +012C ; Uppercase # L& LATIN CAPITAL LETTER I WITH BREVE +012E ; Uppercase # L& LATIN CAPITAL LETTER I WITH OGONEK +0130 ; Uppercase # L& LATIN CAPITAL LETTER I WITH DOT ABOVE +0132 ; Uppercase # L& LATIN CAPITAL LIGATURE IJ +0134 ; Uppercase # L& LATIN CAPITAL LETTER J WITH CIRCUMFLEX +0136 ; Uppercase # L& LATIN CAPITAL LETTER K WITH CEDILLA +0139 ; Uppercase # L& LATIN CAPITAL LETTER L WITH ACUTE +013B ; Uppercase # L& LATIN CAPITAL LETTER L WITH CEDILLA +013D ; Uppercase # L& LATIN CAPITAL LETTER L WITH CARON +013F ; Uppercase # L& LATIN CAPITAL LETTER L WITH MIDDLE DOT +0141 ; Uppercase # L& LATIN CAPITAL LETTER L WITH STROKE +0143 ; Uppercase # L& LATIN CAPITAL LETTER N WITH ACUTE +0145 ; Uppercase # L& LATIN CAPITAL LETTER N WITH CEDILLA +0147 ; Uppercase # L& LATIN CAPITAL LETTER N WITH CARON +014A ; Uppercase # L& LATIN CAPITAL LETTER ENG +014C ; Uppercase # L& LATIN CAPITAL LETTER O WITH MACRON +014E ; Uppercase # L& LATIN CAPITAL LETTER O WITH BREVE +0150 ; Uppercase # L& LATIN CAPITAL LETTER O WITH DOUBLE ACUTE +0152 ; Uppercase # L& LATIN CAPITAL LIGATURE OE +0154 ; Uppercase # L& LATIN CAPITAL LETTER R WITH ACUTE +0156 ; Uppercase # L& LATIN CAPITAL LETTER R WITH CEDILLA +0158 ; Uppercase # L& LATIN CAPITAL LETTER R WITH CARON +015A ; Uppercase # L& LATIN CAPITAL LETTER S WITH ACUTE +015C ; Uppercase # L& LATIN CAPITAL LETTER S WITH CIRCUMFLEX +015E ; Uppercase # L& LATIN CAPITAL LETTER S WITH CEDILLA +0160 ; Uppercase # L& LATIN CAPITAL LETTER S WITH CARON +0162 ; Uppercase # L& LATIN CAPITAL LETTER T WITH CEDILLA +0164 ; Uppercase # L& LATIN CAPITAL LETTER T WITH CARON +0166 ; Uppercase # L& LATIN CAPITAL LETTER T WITH STROKE +0168 ; Uppercase # L& LATIN CAPITAL LETTER U WITH TILDE +016A ; Uppercase # L& LATIN CAPITAL LETTER U WITH MACRON +016C ; Uppercase # L& LATIN CAPITAL LETTER U WITH BREVE +016E ; Uppercase # L& LATIN CAPITAL LETTER U WITH RING ABOVE +0170 ; Uppercase # L& LATIN CAPITAL LETTER U WITH DOUBLE ACUTE +0172 ; Uppercase # L& LATIN CAPITAL LETTER U WITH OGONEK +0174 ; Uppercase # L& LATIN CAPITAL LETTER W WITH CIRCUMFLEX +0176 ; Uppercase # L& LATIN CAPITAL LETTER Y WITH CIRCUMFLEX +0178..0179 ; Uppercase # L& [2] LATIN CAPITAL LETTER Y WITH DIAERESIS..LATIN CAPITAL LETTER Z WITH ACUTE +017B ; Uppercase # L& LATIN CAPITAL LETTER Z WITH DOT ABOVE +017D ; Uppercase # L& LATIN CAPITAL LETTER Z WITH CARON +0181..0182 ; Uppercase # L& [2] LATIN CAPITAL LETTER B WITH HOOK..LATIN CAPITAL LETTER B WITH TOPBAR +0184 ; Uppercase # L& LATIN CAPITAL LETTER TONE SIX +0186..0187 ; Uppercase # L& [2] LATIN CAPITAL LETTER OPEN O..LATIN CAPITAL LETTER C WITH HOOK +0189..018B ; Uppercase # L& [3] LATIN CAPITAL LETTER AFRICAN D..LATIN CAPITAL LETTER D WITH TOPBAR +018E..0191 ; Uppercase # L& [4] LATIN CAPITAL LETTER REVERSED E..LATIN CAPITAL LETTER F WITH HOOK +0193..0194 ; Uppercase # L& [2] LATIN CAPITAL LETTER G WITH HOOK..LATIN CAPITAL LETTER GAMMA +0196..0198 ; Uppercase # L& [3] LATIN CAPITAL LETTER IOTA..LATIN CAPITAL LETTER K WITH HOOK +019C..019D ; Uppercase # L& [2] LATIN CAPITAL LETTER TURNED M..LATIN CAPITAL LETTER N WITH LEFT HOOK +019F..01A0 ; Uppercase # L& [2] LATIN CAPITAL LETTER O WITH MIDDLE TILDE..LATIN CAPITAL LETTER O WITH HORN +01A2 ; Uppercase # L& LATIN CAPITAL LETTER OI +01A4 ; Uppercase # L& LATIN CAPITAL LETTER P WITH HOOK +01A6..01A7 ; Uppercase # L& [2] LATIN LETTER YR..LATIN CAPITAL LETTER TONE TWO +01A9 ; Uppercase # L& LATIN CAPITAL LETTER ESH +01AC ; Uppercase # L& LATIN CAPITAL LETTER T WITH HOOK +01AE..01AF ; Uppercase # L& [2] LATIN CAPITAL LETTER T WITH RETROFLEX HOOK..LATIN CAPITAL LETTER U WITH HORN +01B1..01B3 ; Uppercase # L& [3] LATIN CAPITAL LETTER UPSILON..LATIN CAPITAL LETTER Y WITH HOOK +01B5 ; Uppercase # L& LATIN CAPITAL LETTER Z WITH STROKE +01B7..01B8 ; Uppercase # L& [2] LATIN CAPITAL LETTER EZH..LATIN CAPITAL LETTER EZH REVERSED +01BC ; Uppercase # L& LATIN CAPITAL LETTER TONE FIVE +01C4 ; Uppercase # L& LATIN CAPITAL LETTER DZ WITH CARON +01C7 ; Uppercase # L& LATIN CAPITAL LETTER LJ +01CA ; Uppercase # L& LATIN CAPITAL LETTER NJ +01CD ; Uppercase # L& LATIN CAPITAL LETTER A WITH CARON +01CF ; Uppercase # L& LATIN CAPITAL LETTER I WITH CARON +01D1 ; Uppercase # L& LATIN CAPITAL LETTER O WITH CARON +01D3 ; Uppercase # L& LATIN CAPITAL LETTER U WITH CARON +01D5 ; Uppercase # L& LATIN CAPITAL LETTER U WITH DIAERESIS AND MACRON +01D7 ; Uppercase # L& LATIN CAPITAL LETTER U WITH DIAERESIS AND ACUTE +01D9 ; Uppercase # L& LATIN CAPITAL LETTER U WITH DIAERESIS AND CARON +01DB ; Uppercase # L& LATIN CAPITAL LETTER U WITH DIAERESIS AND GRAVE +01DE ; Uppercase # L& LATIN CAPITAL LETTER A WITH DIAERESIS AND MACRON +01E0 ; Uppercase # L& LATIN CAPITAL LETTER A WITH DOT ABOVE AND MACRON +01E2 ; Uppercase # L& LATIN CAPITAL LETTER AE WITH MACRON +01E4 ; Uppercase # L& LATIN CAPITAL LETTER G WITH STROKE +01E6 ; Uppercase # L& LATIN CAPITAL LETTER G WITH CARON +01E8 ; Uppercase # L& LATIN CAPITAL LETTER K WITH CARON +01EA ; Uppercase # L& LATIN CAPITAL LETTER O WITH OGONEK +01EC ; Uppercase # L& LATIN CAPITAL LETTER O WITH OGONEK AND MACRON +01EE ; Uppercase # L& LATIN CAPITAL LETTER EZH WITH CARON +01F1 ; Uppercase # L& LATIN CAPITAL LETTER DZ +01F4 ; Uppercase # L& LATIN CAPITAL LETTER G WITH ACUTE +01F6..01F8 ; Uppercase # L& [3] LATIN CAPITAL LETTER HWAIR..LATIN CAPITAL LETTER N WITH GRAVE +01FA ; Uppercase # L& LATIN CAPITAL LETTER A WITH RING ABOVE AND ACUTE +01FC ; Uppercase # L& LATIN CAPITAL LETTER AE WITH ACUTE +01FE ; Uppercase # L& LATIN CAPITAL LETTER O WITH STROKE AND ACUTE +0200 ; Uppercase # L& LATIN CAPITAL LETTER A WITH DOUBLE GRAVE +0202 ; Uppercase # L& LATIN CAPITAL LETTER A WITH INVERTED BREVE +0204 ; Uppercase # L& LATIN CAPITAL LETTER E WITH DOUBLE GRAVE +0206 ; Uppercase # L& LATIN CAPITAL LETTER E WITH INVERTED BREVE +0208 ; Uppercase # L& LATIN CAPITAL LETTER I WITH DOUBLE GRAVE +020A ; Uppercase # L& LATIN CAPITAL LETTER I WITH INVERTED BREVE +020C ; Uppercase # L& LATIN CAPITAL LETTER O WITH DOUBLE GRAVE +020E ; Uppercase # L& LATIN CAPITAL LETTER O WITH INVERTED BREVE +0210 ; Uppercase # L& LATIN CAPITAL LETTER R WITH DOUBLE GRAVE +0212 ; Uppercase # L& LATIN CAPITAL LETTER R WITH INVERTED BREVE +0214 ; Uppercase # L& LATIN CAPITAL LETTER U WITH DOUBLE GRAVE +0216 ; Uppercase # L& LATIN CAPITAL LETTER U WITH INVERTED BREVE +0218 ; Uppercase # L& LATIN CAPITAL LETTER S WITH COMMA BELOW +021A ; Uppercase # L& LATIN CAPITAL LETTER T WITH COMMA BELOW +021C ; Uppercase # L& LATIN CAPITAL LETTER YOGH +021E ; Uppercase # L& LATIN CAPITAL LETTER H WITH CARON +0220 ; Uppercase # L& LATIN CAPITAL LETTER N WITH LONG RIGHT LEG +0222 ; Uppercase # L& LATIN CAPITAL LETTER OU +0224 ; Uppercase # L& LATIN CAPITAL LETTER Z WITH HOOK +0226 ; Uppercase # L& LATIN CAPITAL LETTER A WITH DOT ABOVE +0228 ; Uppercase # L& LATIN CAPITAL LETTER E WITH CEDILLA +022A ; Uppercase # L& LATIN CAPITAL LETTER O WITH DIAERESIS AND MACRON +022C ; Uppercase # L& LATIN CAPITAL LETTER O WITH TILDE AND MACRON +022E ; Uppercase # L& LATIN CAPITAL LETTER O WITH DOT ABOVE +0230 ; Uppercase # L& LATIN CAPITAL LETTER O WITH DOT ABOVE AND MACRON +0232 ; Uppercase # L& LATIN CAPITAL LETTER Y WITH MACRON +023A..023B ; Uppercase # L& [2] LATIN CAPITAL LETTER A WITH STROKE..LATIN CAPITAL LETTER C WITH STROKE +023D..023E ; Uppercase # L& [2] LATIN CAPITAL LETTER L WITH BAR..LATIN CAPITAL LETTER T WITH DIAGONAL STROKE +0241 ; Uppercase # L& LATIN CAPITAL LETTER GLOTTAL STOP +0243..0246 ; Uppercase # L& [4] LATIN CAPITAL LETTER B WITH STROKE..LATIN CAPITAL LETTER E WITH STROKE +0248 ; Uppercase # L& LATIN CAPITAL LETTER J WITH STROKE +024A ; Uppercase # L& LATIN CAPITAL LETTER SMALL Q WITH HOOK TAIL +024C ; Uppercase # L& LATIN CAPITAL LETTER R WITH STROKE +024E ; Uppercase # L& LATIN CAPITAL LETTER Y WITH STROKE +0370 ; Uppercase # L& GREEK CAPITAL LETTER HETA +0372 ; Uppercase # L& GREEK CAPITAL LETTER ARCHAIC SAMPI +0376 ; Uppercase # L& GREEK CAPITAL LETTER PAMPHYLIAN DIGAMMA +0386 ; Uppercase # L& GREEK CAPITAL LETTER ALPHA WITH TONOS +0388..038A ; Uppercase # L& [3] GREEK CAPITAL LETTER EPSILON WITH TONOS..GREEK CAPITAL LETTER IOTA WITH TONOS +038C ; Uppercase # L& GREEK CAPITAL LETTER OMICRON WITH TONOS +038E..038F ; Uppercase # L& [2] GREEK CAPITAL LETTER UPSILON WITH TONOS..GREEK CAPITAL LETTER OMEGA WITH TONOS +0391..03A1 ; Uppercase # L& [17] GREEK CAPITAL LETTER ALPHA..GREEK CAPITAL LETTER RHO +03A3..03AB ; Uppercase # L& [9] GREEK CAPITAL LETTER SIGMA..GREEK CAPITAL LETTER UPSILON WITH DIALYTIKA +03CF ; Uppercase # L& GREEK CAPITAL KAI SYMBOL +03D2..03D4 ; Uppercase # L& [3] GREEK UPSILON WITH HOOK SYMBOL..GREEK UPSILON WITH DIAERESIS AND HOOK SYMBOL +03D8 ; Uppercase # L& GREEK LETTER ARCHAIC KOPPA +03DA ; Uppercase # L& GREEK LETTER STIGMA +03DC ; Uppercase # L& GREEK LETTER DIGAMMA +03DE ; Uppercase # L& GREEK LETTER KOPPA +03E0 ; Uppercase # L& GREEK LETTER SAMPI +03E2 ; Uppercase # L& COPTIC CAPITAL LETTER SHEI +03E4 ; Uppercase # L& COPTIC CAPITAL LETTER FEI +03E6 ; Uppercase # L& COPTIC CAPITAL LETTER KHEI +03E8 ; Uppercase # L& COPTIC CAPITAL LETTER HORI +03EA ; Uppercase # L& COPTIC CAPITAL LETTER GANGIA +03EC ; Uppercase # L& COPTIC CAPITAL LETTER SHIMA +03EE ; Uppercase # L& COPTIC CAPITAL LETTER DEI +03F4 ; Uppercase # L& GREEK CAPITAL THETA SYMBOL +03F7 ; Uppercase # L& GREEK CAPITAL LETTER SHO +03F9..03FA ; Uppercase # L& [2] GREEK CAPITAL LUNATE SIGMA SYMBOL..GREEK CAPITAL LETTER SAN +03FD..042F ; Uppercase # L& [51] GREEK CAPITAL REVERSED LUNATE SIGMA SYMBOL..CYRILLIC CAPITAL LETTER YA +0460 ; Uppercase # L& CYRILLIC CAPITAL LETTER OMEGA +0462 ; Uppercase # L& CYRILLIC CAPITAL LETTER YAT +0464 ; Uppercase # L& CYRILLIC CAPITAL LETTER IOTIFIED E +0466 ; Uppercase # L& CYRILLIC CAPITAL LETTER LITTLE YUS +0468 ; Uppercase # L& CYRILLIC CAPITAL LETTER IOTIFIED LITTLE YUS +046A ; Uppercase # L& CYRILLIC CAPITAL LETTER BIG YUS +046C ; Uppercase # L& CYRILLIC CAPITAL LETTER IOTIFIED BIG YUS +046E ; Uppercase # L& CYRILLIC CAPITAL LETTER KSI +0470 ; Uppercase # L& CYRILLIC CAPITAL LETTER PSI +0472 ; Uppercase # L& CYRILLIC CAPITAL LETTER FITA +0474 ; Uppercase # L& CYRILLIC CAPITAL LETTER IZHITSA +0476 ; Uppercase # L& CYRILLIC CAPITAL LETTER IZHITSA WITH DOUBLE GRAVE ACCENT +0478 ; Uppercase # L& CYRILLIC CAPITAL LETTER UK +047A ; Uppercase # L& CYRILLIC CAPITAL LETTER ROUND OMEGA +047C ; Uppercase # L& CYRILLIC CAPITAL LETTER OMEGA WITH TITLO +047E ; Uppercase # L& CYRILLIC CAPITAL LETTER OT +0480 ; Uppercase # L& CYRILLIC CAPITAL LETTER KOPPA +048A ; Uppercase # L& CYRILLIC CAPITAL LETTER SHORT I WITH TAIL +048C ; Uppercase # L& CYRILLIC CAPITAL LETTER SEMISOFT SIGN +048E ; Uppercase # L& CYRILLIC CAPITAL LETTER ER WITH TICK +0490 ; Uppercase # L& CYRILLIC CAPITAL LETTER GHE WITH UPTURN +0492 ; Uppercase # L& CYRILLIC CAPITAL LETTER GHE WITH STROKE +0494 ; Uppercase # L& CYRILLIC CAPITAL LETTER GHE WITH MIDDLE HOOK +0496 ; Uppercase # L& CYRILLIC CAPITAL LETTER ZHE WITH DESCENDER +0498 ; Uppercase # L& CYRILLIC CAPITAL LETTER ZE WITH DESCENDER +049A ; Uppercase # L& CYRILLIC CAPITAL LETTER KA WITH DESCENDER +049C ; Uppercase # L& CYRILLIC CAPITAL LETTER KA WITH VERTICAL STROKE +049E ; Uppercase # L& CYRILLIC CAPITAL LETTER KA WITH STROKE +04A0 ; Uppercase # L& CYRILLIC CAPITAL LETTER BASHKIR KA +04A2 ; Uppercase # L& CYRILLIC CAPITAL LETTER EN WITH DESCENDER +04A4 ; Uppercase # L& CYRILLIC CAPITAL LIGATURE EN GHE +04A6 ; Uppercase # L& CYRILLIC CAPITAL LETTER PE WITH MIDDLE HOOK +04A8 ; Uppercase # L& CYRILLIC CAPITAL LETTER ABKHASIAN HA +04AA ; Uppercase # L& CYRILLIC CAPITAL LETTER ES WITH DESCENDER +04AC ; Uppercase # L& CYRILLIC CAPITAL LETTER TE WITH DESCENDER +04AE ; Uppercase # L& CYRILLIC CAPITAL LETTER STRAIGHT U +04B0 ; Uppercase # L& CYRILLIC CAPITAL LETTER STRAIGHT U WITH STROKE +04B2 ; Uppercase # L& CYRILLIC CAPITAL LETTER HA WITH DESCENDER +04B4 ; Uppercase # L& CYRILLIC CAPITAL LIGATURE TE TSE +04B6 ; Uppercase # L& CYRILLIC CAPITAL LETTER CHE WITH DESCENDER +04B8 ; Uppercase # L& CYRILLIC CAPITAL LETTER CHE WITH VERTICAL STROKE +04BA ; Uppercase # L& CYRILLIC CAPITAL LETTER SHHA +04BC ; Uppercase # L& CYRILLIC CAPITAL LETTER ABKHASIAN CHE +04BE ; Uppercase # L& CYRILLIC CAPITAL LETTER ABKHASIAN CHE WITH DESCENDER +04C0..04C1 ; Uppercase # L& [2] CYRILLIC LETTER PALOCHKA..CYRILLIC CAPITAL LETTER ZHE WITH BREVE +04C3 ; Uppercase # L& CYRILLIC CAPITAL LETTER KA WITH HOOK +04C5 ; Uppercase # L& CYRILLIC CAPITAL LETTER EL WITH TAIL +04C7 ; Uppercase # L& CYRILLIC CAPITAL LETTER EN WITH HOOK +04C9 ; Uppercase # L& CYRILLIC CAPITAL LETTER EN WITH TAIL +04CB ; Uppercase # L& CYRILLIC CAPITAL LETTER KHAKASSIAN CHE +04CD ; Uppercase # L& CYRILLIC CAPITAL LETTER EM WITH TAIL +04D0 ; Uppercase # L& CYRILLIC CAPITAL LETTER A WITH BREVE +04D2 ; Uppercase # L& CYRILLIC CAPITAL LETTER A WITH DIAERESIS +04D4 ; Uppercase # L& CYRILLIC CAPITAL LIGATURE A IE +04D6 ; Uppercase # L& CYRILLIC CAPITAL LETTER IE WITH BREVE +04D8 ; Uppercase # L& CYRILLIC CAPITAL LETTER SCHWA +04DA ; Uppercase # L& CYRILLIC CAPITAL LETTER SCHWA WITH DIAERESIS +04DC ; Uppercase # L& CYRILLIC CAPITAL LETTER ZHE WITH DIAERESIS +04DE ; Uppercase # L& CYRILLIC CAPITAL LETTER ZE WITH DIAERESIS +04E0 ; Uppercase # L& CYRILLIC CAPITAL LETTER ABKHASIAN DZE +04E2 ; Uppercase # L& CYRILLIC CAPITAL LETTER I WITH MACRON +04E4 ; Uppercase # L& CYRILLIC CAPITAL LETTER I WITH DIAERESIS +04E6 ; Uppercase # L& CYRILLIC CAPITAL LETTER O WITH DIAERESIS +04E8 ; Uppercase # L& CYRILLIC CAPITAL LETTER BARRED O +04EA ; Uppercase # L& CYRILLIC CAPITAL LETTER BARRED O WITH DIAERESIS +04EC ; Uppercase # L& CYRILLIC CAPITAL LETTER E WITH DIAERESIS +04EE ; Uppercase # L& CYRILLIC CAPITAL LETTER U WITH MACRON +04F0 ; Uppercase # L& CYRILLIC CAPITAL LETTER U WITH DIAERESIS +04F2 ; Uppercase # L& CYRILLIC CAPITAL LETTER U WITH DOUBLE ACUTE +04F4 ; Uppercase # L& CYRILLIC CAPITAL LETTER CHE WITH DIAERESIS +04F6 ; Uppercase # L& CYRILLIC CAPITAL LETTER GHE WITH DESCENDER +04F8 ; Uppercase # L& CYRILLIC CAPITAL LETTER YERU WITH DIAERESIS +04FA ; Uppercase # L& CYRILLIC CAPITAL LETTER GHE WITH STROKE AND HOOK +04FC ; Uppercase # L& CYRILLIC CAPITAL LETTER HA WITH HOOK +04FE ; Uppercase # L& CYRILLIC CAPITAL LETTER HA WITH STROKE +0500 ; Uppercase # L& CYRILLIC CAPITAL LETTER KOMI DE +0502 ; Uppercase # L& CYRILLIC CAPITAL LETTER KOMI DJE +0504 ; Uppercase # L& CYRILLIC CAPITAL LETTER KOMI ZJE +0506 ; Uppercase # L& CYRILLIC CAPITAL LETTER KOMI DZJE +0508 ; Uppercase # L& CYRILLIC CAPITAL LETTER KOMI LJE +050A ; Uppercase # L& CYRILLIC CAPITAL LETTER KOMI NJE +050C ; Uppercase # L& CYRILLIC CAPITAL LETTER KOMI SJE +050E ; Uppercase # L& CYRILLIC CAPITAL LETTER KOMI TJE +0510 ; Uppercase # L& CYRILLIC CAPITAL LETTER REVERSED ZE +0512 ; Uppercase # L& CYRILLIC CAPITAL LETTER EL WITH HOOK +0514 ; Uppercase # L& CYRILLIC CAPITAL LETTER LHA +0516 ; Uppercase # L& CYRILLIC CAPITAL LETTER RHA +0518 ; Uppercase # L& CYRILLIC CAPITAL LETTER YAE +051A ; Uppercase # L& CYRILLIC CAPITAL LETTER QA +051C ; Uppercase # L& CYRILLIC CAPITAL LETTER WE +051E ; Uppercase # L& CYRILLIC CAPITAL LETTER ALEUT KA +0520 ; Uppercase # L& CYRILLIC CAPITAL LETTER EL WITH MIDDLE HOOK +0522 ; Uppercase # L& CYRILLIC CAPITAL LETTER EN WITH MIDDLE HOOK +0524 ; Uppercase # L& CYRILLIC CAPITAL LETTER PE WITH DESCENDER +0531..0556 ; Uppercase # L& [38] ARMENIAN CAPITAL LETTER AYB..ARMENIAN CAPITAL LETTER FEH +10A0..10C5 ; Uppercase # L& [38] GEORGIAN CAPITAL LETTER AN..GEORGIAN CAPITAL LETTER HOE +1E00 ; Uppercase # L& LATIN CAPITAL LETTER A WITH RING BELOW +1E02 ; Uppercase # L& LATIN CAPITAL LETTER B WITH DOT ABOVE +1E04 ; Uppercase # L& LATIN CAPITAL LETTER B WITH DOT BELOW +1E06 ; Uppercase # L& LATIN CAPITAL LETTER B WITH LINE BELOW +1E08 ; Uppercase # L& LATIN CAPITAL LETTER C WITH CEDILLA AND ACUTE +1E0A ; Uppercase # L& LATIN CAPITAL LETTER D WITH DOT ABOVE +1E0C ; Uppercase # L& LATIN CAPITAL LETTER D WITH DOT BELOW +1E0E ; Uppercase # L& LATIN CAPITAL LETTER D WITH LINE BELOW +1E10 ; Uppercase # L& LATIN CAPITAL LETTER D WITH CEDILLA +1E12 ; Uppercase # L& LATIN CAPITAL LETTER D WITH CIRCUMFLEX BELOW +1E14 ; Uppercase # L& LATIN CAPITAL LETTER E WITH MACRON AND GRAVE +1E16 ; Uppercase # L& LATIN CAPITAL LETTER E WITH MACRON AND ACUTE +1E18 ; Uppercase # L& LATIN CAPITAL LETTER E WITH CIRCUMFLEX BELOW +1E1A ; Uppercase # L& LATIN CAPITAL LETTER E WITH TILDE BELOW +1E1C ; Uppercase # L& LATIN CAPITAL LETTER E WITH CEDILLA AND BREVE +1E1E ; Uppercase # L& LATIN CAPITAL LETTER F WITH DOT ABOVE +1E20 ; Uppercase # L& LATIN CAPITAL LETTER G WITH MACRON +1E22 ; Uppercase # L& LATIN CAPITAL LETTER H WITH DOT ABOVE +1E24 ; Uppercase # L& LATIN CAPITAL LETTER H WITH DOT BELOW +1E26 ; Uppercase # L& LATIN CAPITAL LETTER H WITH DIAERESIS +1E28 ; Uppercase # L& LATIN CAPITAL LETTER H WITH CEDILLA +1E2A ; Uppercase # L& LATIN CAPITAL LETTER H WITH BREVE BELOW +1E2C ; Uppercase # L& LATIN CAPITAL LETTER I WITH TILDE BELOW +1E2E ; Uppercase # L& LATIN CAPITAL LETTER I WITH DIAERESIS AND ACUTE +1E30 ; Uppercase # L& LATIN CAPITAL LETTER K WITH ACUTE +1E32 ; Uppercase # L& LATIN CAPITAL LETTER K WITH DOT BELOW +1E34 ; Uppercase # L& LATIN CAPITAL LETTER K WITH LINE BELOW +1E36 ; Uppercase # L& LATIN CAPITAL LETTER L WITH DOT BELOW +1E38 ; Uppercase # L& LATIN CAPITAL LETTER L WITH DOT BELOW AND MACRON +1E3A ; Uppercase # L& LATIN CAPITAL LETTER L WITH LINE BELOW +1E3C ; Uppercase # L& LATIN CAPITAL LETTER L WITH CIRCUMFLEX BELOW +1E3E ; Uppercase # L& LATIN CAPITAL LETTER M WITH ACUTE +1E40 ; Uppercase # L& LATIN CAPITAL LETTER M WITH DOT ABOVE +1E42 ; Uppercase # L& LATIN CAPITAL LETTER M WITH DOT BELOW +1E44 ; Uppercase # L& LATIN CAPITAL LETTER N WITH DOT ABOVE +1E46 ; Uppercase # L& LATIN CAPITAL LETTER N WITH DOT BELOW +1E48 ; Uppercase # L& LATIN CAPITAL LETTER N WITH LINE BELOW +1E4A ; Uppercase # L& LATIN CAPITAL LETTER N WITH CIRCUMFLEX BELOW +1E4C ; Uppercase # L& LATIN CAPITAL LETTER O WITH TILDE AND ACUTE +1E4E ; Uppercase # L& LATIN CAPITAL LETTER O WITH TILDE AND DIAERESIS +1E50 ; Uppercase # L& LATIN CAPITAL LETTER O WITH MACRON AND GRAVE +1E52 ; Uppercase # L& LATIN CAPITAL LETTER O WITH MACRON AND ACUTE +1E54 ; Uppercase # L& LATIN CAPITAL LETTER P WITH ACUTE +1E56 ; Uppercase # L& LATIN CAPITAL LETTER P WITH DOT ABOVE +1E58 ; Uppercase # L& LATIN CAPITAL LETTER R WITH DOT ABOVE +1E5A ; Uppercase # L& LATIN CAPITAL LETTER R WITH DOT BELOW +1E5C ; Uppercase # L& LATIN CAPITAL LETTER R WITH DOT BELOW AND MACRON +1E5E ; Uppercase # L& LATIN CAPITAL LETTER R WITH LINE BELOW +1E60 ; Uppercase # L& LATIN CAPITAL LETTER S WITH DOT ABOVE +1E62 ; Uppercase # L& LATIN CAPITAL LETTER S WITH DOT BELOW +1E64 ; Uppercase # L& LATIN CAPITAL LETTER S WITH ACUTE AND DOT ABOVE +1E66 ; Uppercase # L& LATIN CAPITAL LETTER S WITH CARON AND DOT ABOVE +1E68 ; Uppercase # L& LATIN CAPITAL LETTER S WITH DOT BELOW AND DOT ABOVE +1E6A ; Uppercase # L& LATIN CAPITAL LETTER T WITH DOT ABOVE +1E6C ; Uppercase # L& LATIN CAPITAL LETTER T WITH DOT BELOW +1E6E ; Uppercase # L& LATIN CAPITAL LETTER T WITH LINE BELOW +1E70 ; Uppercase # L& LATIN CAPITAL LETTER T WITH CIRCUMFLEX BELOW +1E72 ; Uppercase # L& LATIN CAPITAL LETTER U WITH DIAERESIS BELOW +1E74 ; Uppercase # L& LATIN CAPITAL LETTER U WITH TILDE BELOW +1E76 ; Uppercase # L& LATIN CAPITAL LETTER U WITH CIRCUMFLEX BELOW +1E78 ; Uppercase # L& LATIN CAPITAL LETTER U WITH TILDE AND ACUTE +1E7A ; Uppercase # L& LATIN CAPITAL LETTER U WITH MACRON AND DIAERESIS +1E7C ; Uppercase # L& LATIN CAPITAL LETTER V WITH TILDE +1E7E ; Uppercase # L& LATIN CAPITAL LETTER V WITH DOT BELOW +1E80 ; Uppercase # L& LATIN CAPITAL LETTER W WITH GRAVE +1E82 ; Uppercase # L& LATIN CAPITAL LETTER W WITH ACUTE +1E84 ; Uppercase # L& LATIN CAPITAL LETTER W WITH DIAERESIS +1E86 ; Uppercase # L& LATIN CAPITAL LETTER W WITH DOT ABOVE +1E88 ; Uppercase # L& LATIN CAPITAL LETTER W WITH DOT BELOW +1E8A ; Uppercase # L& LATIN CAPITAL LETTER X WITH DOT ABOVE +1E8C ; Uppercase # L& LATIN CAPITAL LETTER X WITH DIAERESIS +1E8E ; Uppercase # L& LATIN CAPITAL LETTER Y WITH DOT ABOVE +1E90 ; Uppercase # L& LATIN CAPITAL LETTER Z WITH CIRCUMFLEX +1E92 ; Uppercase # L& LATIN CAPITAL LETTER Z WITH DOT BELOW +1E94 ; Uppercase # L& LATIN CAPITAL LETTER Z WITH LINE BELOW +1E9E ; Uppercase # L& LATIN CAPITAL LETTER SHARP S +1EA0 ; Uppercase # L& LATIN CAPITAL LETTER A WITH DOT BELOW +1EA2 ; Uppercase # L& LATIN CAPITAL LETTER A WITH HOOK ABOVE +1EA4 ; Uppercase # L& LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND ACUTE +1EA6 ; Uppercase # L& LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND GRAVE +1EA8 ; Uppercase # L& LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND HOOK ABOVE +1EAA ; Uppercase # L& LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND TILDE +1EAC ; Uppercase # L& LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND DOT BELOW +1EAE ; Uppercase # L& LATIN CAPITAL LETTER A WITH BREVE AND ACUTE +1EB0 ; Uppercase # L& LATIN CAPITAL LETTER A WITH BREVE AND GRAVE +1EB2 ; Uppercase # L& LATIN CAPITAL LETTER A WITH BREVE AND HOOK ABOVE +1EB4 ; Uppercase # L& LATIN CAPITAL LETTER A WITH BREVE AND TILDE +1EB6 ; Uppercase # L& LATIN CAPITAL LETTER A WITH BREVE AND DOT BELOW +1EB8 ; Uppercase # L& LATIN CAPITAL LETTER E WITH DOT BELOW +1EBA ; Uppercase # L& LATIN CAPITAL LETTER E WITH HOOK ABOVE +1EBC ; Uppercase # L& LATIN CAPITAL LETTER E WITH TILDE +1EBE ; Uppercase # L& LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND ACUTE +1EC0 ; Uppercase # L& LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND GRAVE +1EC2 ; Uppercase # L& LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND HOOK ABOVE +1EC4 ; Uppercase # L& LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND TILDE +1EC6 ; Uppercase # L& LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND DOT BELOW +1EC8 ; Uppercase # L& LATIN CAPITAL LETTER I WITH HOOK ABOVE +1ECA ; Uppercase # L& LATIN CAPITAL LETTER I WITH DOT BELOW +1ECC ; Uppercase # L& LATIN CAPITAL LETTER O WITH DOT BELOW +1ECE ; Uppercase # L& LATIN CAPITAL LETTER O WITH HOOK ABOVE +1ED0 ; Uppercase # L& LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND ACUTE +1ED2 ; Uppercase # L& LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND GRAVE +1ED4 ; Uppercase # L& LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND HOOK ABOVE +1ED6 ; Uppercase # L& LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND TILDE +1ED8 ; Uppercase # L& LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND DOT BELOW +1EDA ; Uppercase # L& LATIN CAPITAL LETTER O WITH HORN AND ACUTE +1EDC ; Uppercase # L& LATIN CAPITAL LETTER O WITH HORN AND GRAVE +1EDE ; Uppercase # L& LATIN CAPITAL LETTER O WITH HORN AND HOOK ABOVE +1EE0 ; Uppercase # L& LATIN CAPITAL LETTER O WITH HORN AND TILDE +1EE2 ; Uppercase # L& LATIN CAPITAL LETTER O WITH HORN AND DOT BELOW +1EE4 ; Uppercase # L& LATIN CAPITAL LETTER U WITH DOT BELOW +1EE6 ; Uppercase # L& LATIN CAPITAL LETTER U WITH HOOK ABOVE +1EE8 ; Uppercase # L& LATIN CAPITAL LETTER U WITH HORN AND ACUTE +1EEA ; Uppercase # L& LATIN CAPITAL LETTER U WITH HORN AND GRAVE +1EEC ; Uppercase # L& LATIN CAPITAL LETTER U WITH HORN AND HOOK ABOVE +1EEE ; Uppercase # L& LATIN CAPITAL LETTER U WITH HORN AND TILDE +1EF0 ; Uppercase # L& LATIN CAPITAL LETTER U WITH HORN AND DOT BELOW +1EF2 ; Uppercase # L& LATIN CAPITAL LETTER Y WITH GRAVE +1EF4 ; Uppercase # L& LATIN CAPITAL LETTER Y WITH DOT BELOW +1EF6 ; Uppercase # L& LATIN CAPITAL LETTER Y WITH HOOK ABOVE +1EF8 ; Uppercase # L& LATIN CAPITAL LETTER Y WITH TILDE +1EFA ; Uppercase # L& LATIN CAPITAL LETTER MIDDLE-WELSH LL +1EFC ; Uppercase # L& LATIN CAPITAL LETTER MIDDLE-WELSH V +1EFE ; Uppercase # L& LATIN CAPITAL LETTER Y WITH LOOP +1F08..1F0F ; Uppercase # L& [8] GREEK CAPITAL LETTER ALPHA WITH PSILI..GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI +1F18..1F1D ; Uppercase # L& [6] GREEK CAPITAL LETTER EPSILON WITH PSILI..GREEK CAPITAL LETTER EPSILON WITH DASIA AND OXIA +1F28..1F2F ; Uppercase # L& [8] GREEK CAPITAL LETTER ETA WITH PSILI..GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI +1F38..1F3F ; Uppercase # L& [8] GREEK CAPITAL LETTER IOTA WITH PSILI..GREEK CAPITAL LETTER IOTA WITH DASIA AND PERISPOMENI +1F48..1F4D ; Uppercase # L& [6] GREEK CAPITAL LETTER OMICRON WITH PSILI..GREEK CAPITAL LETTER OMICRON WITH DASIA AND OXIA +1F59 ; Uppercase # L& GREEK CAPITAL LETTER UPSILON WITH DASIA +1F5B ; Uppercase # L& GREEK CAPITAL LETTER UPSILON WITH DASIA AND VARIA +1F5D ; Uppercase # L& GREEK CAPITAL LETTER UPSILON WITH DASIA AND OXIA +1F5F ; Uppercase # L& GREEK CAPITAL LETTER UPSILON WITH DASIA AND PERISPOMENI +1F68..1F6F ; Uppercase # L& [8] GREEK CAPITAL LETTER OMEGA WITH PSILI..GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI +1FB8..1FBB ; Uppercase # L& [4] GREEK CAPITAL LETTER ALPHA WITH VRACHY..GREEK CAPITAL LETTER ALPHA WITH OXIA +1FC8..1FCB ; Uppercase # L& [4] GREEK CAPITAL LETTER EPSILON WITH VARIA..GREEK CAPITAL LETTER ETA WITH OXIA +1FD8..1FDB ; Uppercase # L& [4] GREEK CAPITAL LETTER IOTA WITH VRACHY..GREEK CAPITAL LETTER IOTA WITH OXIA +1FE8..1FEC ; Uppercase # L& [5] GREEK CAPITAL LETTER UPSILON WITH VRACHY..GREEK CAPITAL LETTER RHO WITH DASIA +1FF8..1FFB ; Uppercase # L& [4] GREEK CAPITAL LETTER OMICRON WITH VARIA..GREEK CAPITAL LETTER OMEGA WITH OXIA +2102 ; Uppercase # L& DOUBLE-STRUCK CAPITAL C +2107 ; Uppercase # L& EULER CONSTANT +210B..210D ; Uppercase # L& [3] SCRIPT CAPITAL H..DOUBLE-STRUCK CAPITAL H +2110..2112 ; Uppercase # L& [3] SCRIPT CAPITAL I..SCRIPT CAPITAL L +2115 ; Uppercase # L& DOUBLE-STRUCK CAPITAL N +2119..211D ; Uppercase # L& [5] DOUBLE-STRUCK CAPITAL P..DOUBLE-STRUCK CAPITAL R +2124 ; Uppercase # L& DOUBLE-STRUCK CAPITAL Z +2126 ; Uppercase # L& OHM SIGN +2128 ; Uppercase # L& BLACK-LETTER CAPITAL Z +212A..212D ; Uppercase # L& [4] KELVIN SIGN..BLACK-LETTER CAPITAL C +2130..2133 ; Uppercase # L& [4] SCRIPT CAPITAL E..SCRIPT CAPITAL M +213E..213F ; Uppercase # L& [2] DOUBLE-STRUCK CAPITAL GAMMA..DOUBLE-STRUCK CAPITAL PI +2145 ; Uppercase # L& DOUBLE-STRUCK ITALIC CAPITAL D +2160..216F ; Uppercase # Nl [16] ROMAN NUMERAL ONE..ROMAN NUMERAL ONE THOUSAND +2183 ; Uppercase # L& ROMAN NUMERAL REVERSED ONE HUNDRED +24B6..24CF ; Uppercase # So [26] CIRCLED LATIN CAPITAL LETTER A..CIRCLED LATIN CAPITAL LETTER Z +2C00..2C2E ; Uppercase # L& [47] GLAGOLITIC CAPITAL LETTER AZU..GLAGOLITIC CAPITAL LETTER LATINATE MYSLITE +2C60 ; Uppercase # L& LATIN CAPITAL LETTER L WITH DOUBLE BAR +2C62..2C64 ; Uppercase # L& [3] LATIN CAPITAL LETTER L WITH MIDDLE TILDE..LATIN CAPITAL LETTER R WITH TAIL +2C67 ; Uppercase # L& LATIN CAPITAL LETTER H WITH DESCENDER +2C69 ; Uppercase # L& LATIN CAPITAL LETTER K WITH DESCENDER +2C6B ; Uppercase # L& LATIN CAPITAL LETTER Z WITH DESCENDER +2C6D..2C70 ; Uppercase # L& [4] LATIN CAPITAL LETTER ALPHA..LATIN CAPITAL LETTER TURNED ALPHA +2C72 ; Uppercase # L& LATIN CAPITAL LETTER W WITH HOOK +2C75 ; Uppercase # L& LATIN CAPITAL LETTER HALF H +2C7E..2C80 ; Uppercase # L& [3] LATIN CAPITAL LETTER S WITH SWASH TAIL..COPTIC CAPITAL LETTER ALFA +2C82 ; Uppercase # L& COPTIC CAPITAL LETTER VIDA +2C84 ; Uppercase # L& COPTIC CAPITAL LETTER GAMMA +2C86 ; Uppercase # L& COPTIC CAPITAL LETTER DALDA +2C88 ; Uppercase # L& COPTIC CAPITAL LETTER EIE +2C8A ; Uppercase # L& COPTIC CAPITAL LETTER SOU +2C8C ; Uppercase # L& COPTIC CAPITAL LETTER ZATA +2C8E ; Uppercase # L& COPTIC CAPITAL LETTER HATE +2C90 ; Uppercase # L& COPTIC CAPITAL LETTER THETHE +2C92 ; Uppercase # L& COPTIC CAPITAL LETTER IAUDA +2C94 ; Uppercase # L& COPTIC CAPITAL LETTER KAPA +2C96 ; Uppercase # L& COPTIC CAPITAL LETTER LAULA +2C98 ; Uppercase # L& COPTIC CAPITAL LETTER MI +2C9A ; Uppercase # L& COPTIC CAPITAL LETTER NI +2C9C ; Uppercase # L& COPTIC CAPITAL LETTER KSI +2C9E ; Uppercase # L& COPTIC CAPITAL LETTER O +2CA0 ; Uppercase # L& COPTIC CAPITAL LETTER PI +2CA2 ; Uppercase # L& COPTIC CAPITAL LETTER RO +2CA4 ; Uppercase # L& COPTIC CAPITAL LETTER SIMA +2CA6 ; Uppercase # L& COPTIC CAPITAL LETTER TAU +2CA8 ; Uppercase # L& COPTIC CAPITAL LETTER UA +2CAA ; Uppercase # L& COPTIC CAPITAL LETTER FI +2CAC ; Uppercase # L& COPTIC CAPITAL LETTER KHI +2CAE ; Uppercase # L& COPTIC CAPITAL LETTER PSI +2CB0 ; Uppercase # L& COPTIC CAPITAL LETTER OOU +2CB2 ; Uppercase # L& COPTIC CAPITAL LETTER DIALECT-P ALEF +2CB4 ; Uppercase # L& COPTIC CAPITAL LETTER OLD COPTIC AIN +2CB6 ; Uppercase # L& COPTIC CAPITAL LETTER CRYPTOGRAMMIC EIE +2CB8 ; Uppercase # L& COPTIC CAPITAL LETTER DIALECT-P KAPA +2CBA ; Uppercase # L& COPTIC CAPITAL LETTER DIALECT-P NI +2CBC ; Uppercase # L& COPTIC CAPITAL LETTER CRYPTOGRAMMIC NI +2CBE ; Uppercase # L& COPTIC CAPITAL LETTER OLD COPTIC OOU +2CC0 ; Uppercase # L& COPTIC CAPITAL LETTER SAMPI +2CC2 ; Uppercase # L& COPTIC CAPITAL LETTER CROSSED SHEI +2CC4 ; Uppercase # L& COPTIC CAPITAL LETTER OLD COPTIC SHEI +2CC6 ; Uppercase # L& COPTIC CAPITAL LETTER OLD COPTIC ESH +2CC8 ; Uppercase # L& COPTIC CAPITAL LETTER AKHMIMIC KHEI +2CCA ; Uppercase # L& COPTIC CAPITAL LETTER DIALECT-P HORI +2CCC ; Uppercase # L& COPTIC CAPITAL LETTER OLD COPTIC HORI +2CCE ; Uppercase # L& COPTIC CAPITAL LETTER OLD COPTIC HA +2CD0 ; Uppercase # L& COPTIC CAPITAL LETTER L-SHAPED HA +2CD2 ; Uppercase # L& COPTIC CAPITAL LETTER OLD COPTIC HEI +2CD4 ; Uppercase # L& COPTIC CAPITAL LETTER OLD COPTIC HAT +2CD6 ; Uppercase # L& COPTIC CAPITAL LETTER OLD COPTIC GANGIA +2CD8 ; Uppercase # L& COPTIC CAPITAL LETTER OLD COPTIC DJA +2CDA ; Uppercase # L& COPTIC CAPITAL LETTER OLD COPTIC SHIMA +2CDC ; Uppercase # L& COPTIC CAPITAL LETTER OLD NUBIAN SHIMA +2CDE ; Uppercase # L& COPTIC CAPITAL LETTER OLD NUBIAN NGI +2CE0 ; Uppercase # L& COPTIC CAPITAL LETTER OLD NUBIAN NYI +2CE2 ; Uppercase # L& COPTIC CAPITAL LETTER OLD NUBIAN WAU +2CEB ; Uppercase # L& COPTIC CAPITAL LETTER CRYPTOGRAMMIC SHEI +2CED ; Uppercase # L& COPTIC CAPITAL LETTER CRYPTOGRAMMIC GANGIA +A640 ; Uppercase # L& CYRILLIC CAPITAL LETTER ZEMLYA +A642 ; Uppercase # L& CYRILLIC CAPITAL LETTER DZELO +A644 ; Uppercase # L& CYRILLIC CAPITAL LETTER REVERSED DZE +A646 ; Uppercase # L& CYRILLIC CAPITAL LETTER IOTA +A648 ; Uppercase # L& CYRILLIC CAPITAL LETTER DJERV +A64A ; Uppercase # L& CYRILLIC CAPITAL LETTER MONOGRAPH UK +A64C ; Uppercase # L& CYRILLIC CAPITAL LETTER BROAD OMEGA +A64E ; Uppercase # L& CYRILLIC CAPITAL LETTER NEUTRAL YER +A650 ; Uppercase # L& CYRILLIC CAPITAL LETTER YERU WITH BACK YER +A652 ; Uppercase # L& CYRILLIC CAPITAL LETTER IOTIFIED YAT +A654 ; Uppercase # L& CYRILLIC CAPITAL LETTER REVERSED YU +A656 ; Uppercase # L& CYRILLIC CAPITAL LETTER IOTIFIED A +A658 ; Uppercase # L& CYRILLIC CAPITAL LETTER CLOSED LITTLE YUS +A65A ; Uppercase # L& CYRILLIC CAPITAL LETTER BLENDED YUS +A65C ; Uppercase # L& CYRILLIC CAPITAL LETTER IOTIFIED CLOSED LITTLE YUS +A65E ; Uppercase # L& CYRILLIC CAPITAL LETTER YN +A662 ; Uppercase # L& CYRILLIC CAPITAL LETTER SOFT DE +A664 ; Uppercase # L& CYRILLIC CAPITAL LETTER SOFT EL +A666 ; Uppercase # L& CYRILLIC CAPITAL LETTER SOFT EM +A668 ; Uppercase # L& CYRILLIC CAPITAL LETTER MONOCULAR O +A66A ; Uppercase # L& CYRILLIC CAPITAL LETTER BINOCULAR O +A66C ; Uppercase # L& CYRILLIC CAPITAL LETTER DOUBLE MONOCULAR O +A680 ; Uppercase # L& CYRILLIC CAPITAL LETTER DWE +A682 ; Uppercase # L& CYRILLIC CAPITAL LETTER DZWE +A684 ; Uppercase # L& CYRILLIC CAPITAL LETTER ZHWE +A686 ; Uppercase # L& CYRILLIC CAPITAL LETTER CCHE +A688 ; Uppercase # L& CYRILLIC CAPITAL LETTER DZZE +A68A ; Uppercase # L& CYRILLIC CAPITAL LETTER TE WITH MIDDLE HOOK +A68C ; Uppercase # L& CYRILLIC CAPITAL LETTER TWE +A68E ; Uppercase # L& CYRILLIC CAPITAL LETTER TSWE +A690 ; Uppercase # L& CYRILLIC CAPITAL LETTER TSSE +A692 ; Uppercase # L& CYRILLIC CAPITAL LETTER TCHE +A694 ; Uppercase # L& CYRILLIC CAPITAL LETTER HWE +A696 ; Uppercase # L& CYRILLIC CAPITAL LETTER SHWE +A722 ; Uppercase # L& LATIN CAPITAL LETTER EGYPTOLOGICAL ALEF +A724 ; Uppercase # L& LATIN CAPITAL LETTER EGYPTOLOGICAL AIN +A726 ; Uppercase # L& LATIN CAPITAL LETTER HENG +A728 ; Uppercase # L& LATIN CAPITAL LETTER TZ +A72A ; Uppercase # L& LATIN CAPITAL LETTER TRESILLO +A72C ; Uppercase # L& LATIN CAPITAL LETTER CUATRILLO +A72E ; Uppercase # L& LATIN CAPITAL LETTER CUATRILLO WITH COMMA +A732 ; Uppercase # L& LATIN CAPITAL LETTER AA +A734 ; Uppercase # L& LATIN CAPITAL LETTER AO +A736 ; Uppercase # L& LATIN CAPITAL LETTER AU +A738 ; Uppercase # L& LATIN CAPITAL LETTER AV +A73A ; Uppercase # L& LATIN CAPITAL LETTER AV WITH HORIZONTAL BAR +A73C ; Uppercase # L& LATIN CAPITAL LETTER AY +A73E ; Uppercase # L& LATIN CAPITAL LETTER REVERSED C WITH DOT +A740 ; Uppercase # L& LATIN CAPITAL LETTER K WITH STROKE +A742 ; Uppercase # L& LATIN CAPITAL LETTER K WITH DIAGONAL STROKE +A744 ; Uppercase # L& LATIN CAPITAL LETTER K WITH STROKE AND DIAGONAL STROKE +A746 ; Uppercase # L& LATIN CAPITAL LETTER BROKEN L +A748 ; Uppercase # L& LATIN CAPITAL LETTER L WITH HIGH STROKE +A74A ; Uppercase # L& LATIN CAPITAL LETTER O WITH LONG STROKE OVERLAY +A74C ; Uppercase # L& LATIN CAPITAL LETTER O WITH LOOP +A74E ; Uppercase # L& LATIN CAPITAL LETTER OO +A750 ; Uppercase # L& LATIN CAPITAL LETTER P WITH STROKE THROUGH DESCENDER +A752 ; Uppercase # L& LATIN CAPITAL LETTER P WITH FLOURISH +A754 ; Uppercase # L& LATIN CAPITAL LETTER P WITH SQUIRREL TAIL +A756 ; Uppercase # L& LATIN CAPITAL LETTER Q WITH STROKE THROUGH DESCENDER +A758 ; Uppercase # L& LATIN CAPITAL LETTER Q WITH DIAGONAL STROKE +A75A ; Uppercase # L& LATIN CAPITAL LETTER R ROTUNDA +A75C ; Uppercase # L& LATIN CAPITAL LETTER RUM ROTUNDA +A75E ; Uppercase # L& LATIN CAPITAL LETTER V WITH DIAGONAL STROKE +A760 ; Uppercase # L& LATIN CAPITAL LETTER VY +A762 ; Uppercase # L& LATIN CAPITAL LETTER VISIGOTHIC Z +A764 ; Uppercase # L& LATIN CAPITAL LETTER THORN WITH STROKE +A766 ; Uppercase # L& LATIN CAPITAL LETTER THORN WITH STROKE THROUGH DESCENDER +A768 ; Uppercase # L& LATIN CAPITAL LETTER VEND +A76A ; Uppercase # L& LATIN CAPITAL LETTER ET +A76C ; Uppercase # L& LATIN CAPITAL LETTER IS +A76E ; Uppercase # L& LATIN CAPITAL LETTER CON +A779 ; Uppercase # L& LATIN CAPITAL LETTER INSULAR D +A77B ; Uppercase # L& LATIN CAPITAL LETTER INSULAR F +A77D..A77E ; Uppercase # L& [2] LATIN CAPITAL LETTER INSULAR G..LATIN CAPITAL LETTER TURNED INSULAR G +A780 ; Uppercase # L& LATIN CAPITAL LETTER TURNED L +A782 ; Uppercase # L& LATIN CAPITAL LETTER INSULAR R +A784 ; Uppercase # L& LATIN CAPITAL LETTER INSULAR S +A786 ; Uppercase # L& LATIN CAPITAL LETTER INSULAR T +A78B ; Uppercase # L& LATIN CAPITAL LETTER SALTILLO +FF21..FF3A ; Uppercase # L& [26] FULLWIDTH LATIN CAPITAL LETTER A..FULLWIDTH LATIN CAPITAL LETTER Z +10400..10427 ; Uppercase # L& [40] DESERET CAPITAL LETTER LONG I..DESERET CAPITAL LETTER EW +1D400..1D419 ; Uppercase # L& [26] MATHEMATICAL BOLD CAPITAL A..MATHEMATICAL BOLD CAPITAL Z +1D434..1D44D ; Uppercase # L& [26] MATHEMATICAL ITALIC CAPITAL A..MATHEMATICAL ITALIC CAPITAL Z +1D468..1D481 ; Uppercase # L& [26] MATHEMATICAL BOLD ITALIC CAPITAL A..MATHEMATICAL BOLD ITALIC CAPITAL Z +1D49C ; Uppercase # L& MATHEMATICAL SCRIPT CAPITAL A +1D49E..1D49F ; Uppercase # L& [2] MATHEMATICAL SCRIPT CAPITAL C..MATHEMATICAL SCRIPT CAPITAL D +1D4A2 ; Uppercase # L& MATHEMATICAL SCRIPT CAPITAL G +1D4A5..1D4A6 ; Uppercase # L& [2] MATHEMATICAL SCRIPT CAPITAL J..MATHEMATICAL SCRIPT CAPITAL K +1D4A9..1D4AC ; Uppercase # L& [4] MATHEMATICAL SCRIPT CAPITAL N..MATHEMATICAL SCRIPT CAPITAL Q +1D4AE..1D4B5 ; Uppercase # L& [8] MATHEMATICAL SCRIPT CAPITAL S..MATHEMATICAL SCRIPT CAPITAL Z +1D4D0..1D4E9 ; Uppercase # L& [26] MATHEMATICAL BOLD SCRIPT CAPITAL A..MATHEMATICAL BOLD SCRIPT CAPITAL Z +1D504..1D505 ; Uppercase # L& [2] MATHEMATICAL FRAKTUR CAPITAL A..MATHEMATICAL FRAKTUR CAPITAL B +1D507..1D50A ; Uppercase # L& [4] MATHEMATICAL FRAKTUR CAPITAL D..MATHEMATICAL FRAKTUR CAPITAL G +1D50D..1D514 ; Uppercase # L& [8] MATHEMATICAL FRAKTUR CAPITAL J..MATHEMATICAL FRAKTUR CAPITAL Q +1D516..1D51C ; Uppercase # L& [7] MATHEMATICAL FRAKTUR CAPITAL S..MATHEMATICAL FRAKTUR CAPITAL Y +1D538..1D539 ; Uppercase # L& [2] MATHEMATICAL DOUBLE-STRUCK CAPITAL A..MATHEMATICAL DOUBLE-STRUCK CAPITAL B +1D53B..1D53E ; Uppercase # L& [4] MATHEMATICAL DOUBLE-STRUCK CAPITAL D..MATHEMATICAL DOUBLE-STRUCK CAPITAL G +1D540..1D544 ; Uppercase # L& [5] MATHEMATICAL DOUBLE-STRUCK CAPITAL I..MATHEMATICAL DOUBLE-STRUCK CAPITAL M +1D546 ; Uppercase # L& MATHEMATICAL DOUBLE-STRUCK CAPITAL O +1D54A..1D550 ; Uppercase # L& [7] MATHEMATICAL DOUBLE-STRUCK CAPITAL S..MATHEMATICAL DOUBLE-STRUCK CAPITAL Y +1D56C..1D585 ; Uppercase # L& [26] MATHEMATICAL BOLD FRAKTUR CAPITAL A..MATHEMATICAL BOLD FRAKTUR CAPITAL Z +1D5A0..1D5B9 ; Uppercase # L& [26] MATHEMATICAL SANS-SERIF CAPITAL A..MATHEMATICAL SANS-SERIF CAPITAL Z +1D5D4..1D5ED ; Uppercase # L& [26] MATHEMATICAL SANS-SERIF BOLD CAPITAL A..MATHEMATICAL SANS-SERIF BOLD CAPITAL Z +1D608..1D621 ; Uppercase # L& [26] MATHEMATICAL SANS-SERIF ITALIC CAPITAL A..MATHEMATICAL SANS-SERIF ITALIC CAPITAL Z +1D63C..1D655 ; Uppercase # L& [26] MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL A..MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL Z +1D670..1D689 ; Uppercase # L& [26] MATHEMATICAL MONOSPACE CAPITAL A..MATHEMATICAL MONOSPACE CAPITAL Z +1D6A8..1D6C0 ; Uppercase # L& [25] MATHEMATICAL BOLD CAPITAL ALPHA..MATHEMATICAL BOLD CAPITAL OMEGA +1D6E2..1D6FA ; Uppercase # L& [25] MATHEMATICAL ITALIC CAPITAL ALPHA..MATHEMATICAL ITALIC CAPITAL OMEGA +1D71C..1D734 ; Uppercase # L& [25] MATHEMATICAL BOLD ITALIC CAPITAL ALPHA..MATHEMATICAL BOLD ITALIC CAPITAL OMEGA +1D756..1D76E ; Uppercase # L& [25] MATHEMATICAL SANS-SERIF BOLD CAPITAL ALPHA..MATHEMATICAL SANS-SERIF BOLD CAPITAL OMEGA +1D790..1D7A8 ; Uppercase # L& [25] MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL ALPHA..MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL OMEGA +1D7CA ; Uppercase # L& MATHEMATICAL BOLD CAPITAL DIGAMMA + +# Total code points: 1469 + +# ================================================ + +# Derived Property: Cased (Cased) +# As defined by Unicode Standard Definition D120 +# C has the Lowercase or Uppercase property or has a General_Category value of Titlecase_Letter. + +0041..005A ; Cased # L& [26] LATIN CAPITAL LETTER A..LATIN CAPITAL LETTER Z +0061..007A ; Cased # L& [26] LATIN SMALL LETTER A..LATIN SMALL LETTER Z +00AA ; Cased # L& FEMININE ORDINAL INDICATOR +00B5 ; Cased # L& MICRO SIGN +00BA ; Cased # L& MASCULINE ORDINAL INDICATOR +00C0..00D6 ; Cased # L& [23] LATIN CAPITAL LETTER A WITH GRAVE..LATIN CAPITAL LETTER O WITH DIAERESIS +00D8..00F6 ; Cased # L& [31] LATIN CAPITAL LETTER O WITH STROKE..LATIN SMALL LETTER O WITH DIAERESIS +00F8..01BA ; Cased # L& [195] LATIN SMALL LETTER O WITH STROKE..LATIN SMALL LETTER EZH WITH TAIL +01BC..01BF ; Cased # L& [4] LATIN CAPITAL LETTER TONE FIVE..LATIN LETTER WYNN +01C4..0293 ; Cased # L& [208] LATIN CAPITAL LETTER DZ WITH CARON..LATIN SMALL LETTER EZH WITH CURL +0295..02AF ; Cased # L& [27] LATIN LETTER PHARYNGEAL VOICED FRICATIVE..LATIN SMALL LETTER TURNED H WITH FISHHOOK AND TAIL +02B0..02B8 ; Cased # Lm [9] MODIFIER LETTER SMALL H..MODIFIER LETTER SMALL Y +02C0..02C1 ; Cased # Lm [2] MODIFIER LETTER GLOTTAL STOP..MODIFIER LETTER REVERSED GLOTTAL STOP +02E0..02E4 ; Cased # Lm [5] MODIFIER LETTER SMALL GAMMA..MODIFIER LETTER SMALL REVERSED GLOTTAL STOP +0345 ; Cased # Mn COMBINING GREEK YPOGEGRAMMENI +0370..0373 ; Cased # L& [4] GREEK CAPITAL LETTER HETA..GREEK SMALL LETTER ARCHAIC SAMPI +0376..0377 ; Cased # L& [2] GREEK CAPITAL LETTER PAMPHYLIAN DIGAMMA..GREEK SMALL LETTER PAMPHYLIAN DIGAMMA +037A ; Cased # Lm GREEK YPOGEGRAMMENI +037B..037D ; Cased # L& [3] GREEK SMALL REVERSED LUNATE SIGMA SYMBOL..GREEK SMALL REVERSED DOTTED LUNATE SIGMA SYMBOL +0386 ; Cased # L& GREEK CAPITAL LETTER ALPHA WITH TONOS +0388..038A ; Cased # L& [3] GREEK CAPITAL LETTER EPSILON WITH TONOS..GREEK CAPITAL LETTER IOTA WITH TONOS +038C ; Cased # L& GREEK CAPITAL LETTER OMICRON WITH TONOS +038E..03A1 ; Cased # L& [20] GREEK CAPITAL LETTER UPSILON WITH TONOS..GREEK CAPITAL LETTER RHO +03A3..03F5 ; Cased # L& [83] GREEK CAPITAL LETTER SIGMA..GREEK LUNATE EPSILON SYMBOL +03F7..0481 ; Cased # L& [139] GREEK CAPITAL LETTER SHO..CYRILLIC SMALL LETTER KOPPA +048A..0525 ; Cased # L& [156] CYRILLIC CAPITAL LETTER SHORT I WITH TAIL..CYRILLIC SMALL LETTER PE WITH DESCENDER +0531..0556 ; Cased # L& [38] ARMENIAN CAPITAL LETTER AYB..ARMENIAN CAPITAL LETTER FEH +0561..0587 ; Cased # L& [39] ARMENIAN SMALL LETTER AYB..ARMENIAN SMALL LIGATURE ECH YIWN +10A0..10C5 ; Cased # L& [38] GEORGIAN CAPITAL LETTER AN..GEORGIAN CAPITAL LETTER HOE +1D00..1D2B ; Cased # L& [44] LATIN LETTER SMALL CAPITAL A..CYRILLIC LETTER SMALL CAPITAL EL +1D2C..1D61 ; Cased # Lm [54] MODIFIER LETTER CAPITAL A..MODIFIER LETTER SMALL CHI +1D62..1D77 ; Cased # L& [22] LATIN SUBSCRIPT SMALL LETTER I..LATIN SMALL LETTER TURNED G +1D78 ; Cased # Lm MODIFIER LETTER CYRILLIC EN +1D79..1D9A ; Cased # L& [34] LATIN SMALL LETTER INSULAR G..LATIN SMALL LETTER EZH WITH RETROFLEX HOOK +1D9B..1DBF ; Cased # Lm [37] MODIFIER LETTER SMALL TURNED ALPHA..MODIFIER LETTER SMALL THETA +1E00..1F15 ; Cased # L& [278] LATIN CAPITAL LETTER A WITH RING BELOW..GREEK SMALL LETTER EPSILON WITH DASIA AND OXIA +1F18..1F1D ; Cased # L& [6] GREEK CAPITAL LETTER EPSILON WITH PSILI..GREEK CAPITAL LETTER EPSILON WITH DASIA AND OXIA +1F20..1F45 ; Cased # L& [38] GREEK SMALL LETTER ETA WITH PSILI..GREEK SMALL LETTER OMICRON WITH DASIA AND OXIA +1F48..1F4D ; Cased # L& [6] GREEK CAPITAL LETTER OMICRON WITH PSILI..GREEK CAPITAL LETTER OMICRON WITH DASIA AND OXIA +1F50..1F57 ; Cased # L& [8] GREEK SMALL LETTER UPSILON WITH PSILI..GREEK SMALL LETTER UPSILON WITH DASIA AND PERISPOMENI +1F59 ; Cased # L& GREEK CAPITAL LETTER UPSILON WITH DASIA +1F5B ; Cased # L& GREEK CAPITAL LETTER UPSILON WITH DASIA AND VARIA +1F5D ; Cased # L& GREEK CAPITAL LETTER UPSILON WITH DASIA AND OXIA +1F5F..1F7D ; Cased # L& [31] GREEK CAPITAL LETTER UPSILON WITH DASIA AND PERISPOMENI..GREEK SMALL LETTER OMEGA WITH OXIA +1F80..1FB4 ; Cased # L& [53] GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI..GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI +1FB6..1FBC ; Cased # L& [7] GREEK SMALL LETTER ALPHA WITH PERISPOMENI..GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI +1FBE ; Cased # L& GREEK PROSGEGRAMMENI +1FC2..1FC4 ; Cased # L& [3] GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI..GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI +1FC6..1FCC ; Cased # L& [7] GREEK SMALL LETTER ETA WITH PERISPOMENI..GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI +1FD0..1FD3 ; Cased # L& [4] GREEK SMALL LETTER IOTA WITH VRACHY..GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA +1FD6..1FDB ; Cased # L& [6] GREEK SMALL LETTER IOTA WITH PERISPOMENI..GREEK CAPITAL LETTER IOTA WITH OXIA +1FE0..1FEC ; Cased # L& [13] GREEK SMALL LETTER UPSILON WITH VRACHY..GREEK CAPITAL LETTER RHO WITH DASIA +1FF2..1FF4 ; Cased # L& [3] GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI..GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI +1FF6..1FFC ; Cased # L& [7] GREEK SMALL LETTER OMEGA WITH PERISPOMENI..GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI +2090..2094 ; Cased # Lm [5] LATIN SUBSCRIPT SMALL LETTER A..LATIN SUBSCRIPT SMALL LETTER SCHWA +2102 ; Cased # L& DOUBLE-STRUCK CAPITAL C +2107 ; Cased # L& EULER CONSTANT +210A..2113 ; Cased # L& [10] SCRIPT SMALL G..SCRIPT SMALL L +2115 ; Cased # L& DOUBLE-STRUCK CAPITAL N +2119..211D ; Cased # L& [5] DOUBLE-STRUCK CAPITAL P..DOUBLE-STRUCK CAPITAL R +2124 ; Cased # L& DOUBLE-STRUCK CAPITAL Z +2126 ; Cased # L& OHM SIGN +2128 ; Cased # L& BLACK-LETTER CAPITAL Z +212A..212D ; Cased # L& [4] KELVIN SIGN..BLACK-LETTER CAPITAL C +212F..2134 ; Cased # L& [6] SCRIPT SMALL E..SCRIPT SMALL O +2139 ; Cased # L& INFORMATION SOURCE +213C..213F ; Cased # L& [4] DOUBLE-STRUCK SMALL PI..DOUBLE-STRUCK CAPITAL PI +2145..2149 ; Cased # L& [5] DOUBLE-STRUCK ITALIC CAPITAL D..DOUBLE-STRUCK ITALIC SMALL J +214E ; Cased # L& TURNED SMALL F +2160..217F ; Cased # Nl [32] ROMAN NUMERAL ONE..SMALL ROMAN NUMERAL ONE THOUSAND +2183..2184 ; Cased # L& [2] ROMAN NUMERAL REVERSED ONE HUNDRED..LATIN SMALL LETTER REVERSED C +24B6..24E9 ; Cased # So [52] CIRCLED LATIN CAPITAL LETTER A..CIRCLED LATIN SMALL LETTER Z +2C00..2C2E ; Cased # L& [47] GLAGOLITIC CAPITAL LETTER AZU..GLAGOLITIC CAPITAL LETTER LATINATE MYSLITE +2C30..2C5E ; Cased # L& [47] GLAGOLITIC SMALL LETTER AZU..GLAGOLITIC SMALL LETTER LATINATE MYSLITE +2C60..2C7C ; Cased # L& [29] LATIN CAPITAL LETTER L WITH DOUBLE BAR..LATIN SUBSCRIPT SMALL LETTER J +2C7D ; Cased # Lm MODIFIER LETTER CAPITAL V +2C7E..2CE4 ; Cased # L& [103] LATIN CAPITAL LETTER S WITH SWASH TAIL..COPTIC SYMBOL KAI +2CEB..2CEE ; Cased # L& [4] COPTIC CAPITAL LETTER CRYPTOGRAMMIC SHEI..COPTIC SMALL LETTER CRYPTOGRAMMIC GANGIA +2D00..2D25 ; Cased # L& [38] GEORGIAN SMALL LETTER AN..GEORGIAN SMALL LETTER HOE +A640..A65F ; Cased # L& [32] CYRILLIC CAPITAL LETTER ZEMLYA..CYRILLIC SMALL LETTER YN +A662..A66D ; Cased # L& [12] CYRILLIC CAPITAL LETTER SOFT DE..CYRILLIC SMALL LETTER DOUBLE MONOCULAR O +A680..A697 ; Cased # L& [24] CYRILLIC CAPITAL LETTER DWE..CYRILLIC SMALL LETTER SHWE +A722..A76F ; Cased # L& [78] LATIN CAPITAL LETTER EGYPTOLOGICAL ALEF..LATIN SMALL LETTER CON +A770 ; Cased # Lm MODIFIER LETTER US +A771..A787 ; Cased # L& [23] LATIN SMALL LETTER DUM..LATIN SMALL LETTER INSULAR T +A78B..A78C ; Cased # L& [2] LATIN CAPITAL LETTER SALTILLO..LATIN SMALL LETTER SALTILLO +FB00..FB06 ; Cased # L& [7] LATIN SMALL LIGATURE FF..LATIN SMALL LIGATURE ST +FB13..FB17 ; Cased # L& [5] ARMENIAN SMALL LIGATURE MEN NOW..ARMENIAN SMALL LIGATURE MEN XEH +FF21..FF3A ; Cased # L& [26] FULLWIDTH LATIN CAPITAL LETTER A..FULLWIDTH LATIN CAPITAL LETTER Z +FF41..FF5A ; Cased # L& [26] FULLWIDTH LATIN SMALL LETTER A..FULLWIDTH LATIN SMALL LETTER Z +10400..1044F ; Cased # L& [80] DESERET CAPITAL LETTER LONG I..DESERET SMALL LETTER EW +1D400..1D454 ; Cased # L& [85] MATHEMATICAL BOLD CAPITAL A..MATHEMATICAL ITALIC SMALL G +1D456..1D49C ; Cased # L& [71] MATHEMATICAL ITALIC SMALL I..MATHEMATICAL SCRIPT CAPITAL A +1D49E..1D49F ; Cased # L& [2] MATHEMATICAL SCRIPT CAPITAL C..MATHEMATICAL SCRIPT CAPITAL D +1D4A2 ; Cased # L& MATHEMATICAL SCRIPT CAPITAL G +1D4A5..1D4A6 ; Cased # L& [2] MATHEMATICAL SCRIPT CAPITAL J..MATHEMATICAL SCRIPT CAPITAL K +1D4A9..1D4AC ; Cased # L& [4] MATHEMATICAL SCRIPT CAPITAL N..MATHEMATICAL SCRIPT CAPITAL Q +1D4AE..1D4B9 ; Cased # L& [12] MATHEMATICAL SCRIPT CAPITAL S..MATHEMATICAL SCRIPT SMALL D +1D4BB ; Cased # L& MATHEMATICAL SCRIPT SMALL F +1D4BD..1D4C3 ; Cased # L& [7] MATHEMATICAL SCRIPT SMALL H..MATHEMATICAL SCRIPT SMALL N +1D4C5..1D505 ; Cased # L& [65] MATHEMATICAL SCRIPT SMALL P..MATHEMATICAL FRAKTUR CAPITAL B +1D507..1D50A ; Cased # L& [4] MATHEMATICAL FRAKTUR CAPITAL D..MATHEMATICAL FRAKTUR CAPITAL G +1D50D..1D514 ; Cased # L& [8] MATHEMATICAL FRAKTUR CAPITAL J..MATHEMATICAL FRAKTUR CAPITAL Q +1D516..1D51C ; Cased # L& [7] MATHEMATICAL FRAKTUR CAPITAL S..MATHEMATICAL FRAKTUR CAPITAL Y +1D51E..1D539 ; Cased # L& [28] MATHEMATICAL FRAKTUR SMALL A..MATHEMATICAL DOUBLE-STRUCK CAPITAL B +1D53B..1D53E ; Cased # L& [4] MATHEMATICAL DOUBLE-STRUCK CAPITAL D..MATHEMATICAL DOUBLE-STRUCK CAPITAL G +1D540..1D544 ; Cased # L& [5] MATHEMATICAL DOUBLE-STRUCK CAPITAL I..MATHEMATICAL DOUBLE-STRUCK CAPITAL M +1D546 ; Cased # L& MATHEMATICAL DOUBLE-STRUCK CAPITAL O +1D54A..1D550 ; Cased # L& [7] MATHEMATICAL DOUBLE-STRUCK CAPITAL S..MATHEMATICAL DOUBLE-STRUCK CAPITAL Y +1D552..1D6A5 ; Cased # L& [340] MATHEMATICAL DOUBLE-STRUCK SMALL A..MATHEMATICAL ITALIC SMALL DOTLESS J +1D6A8..1D6C0 ; Cased # L& [25] MATHEMATICAL BOLD CAPITAL ALPHA..MATHEMATICAL BOLD CAPITAL OMEGA +1D6C2..1D6DA ; Cased # L& [25] MATHEMATICAL BOLD SMALL ALPHA..MATHEMATICAL BOLD SMALL OMEGA +1D6DC..1D6FA ; Cased # L& [31] MATHEMATICAL BOLD EPSILON SYMBOL..MATHEMATICAL ITALIC CAPITAL OMEGA +1D6FC..1D714 ; Cased # L& [25] MATHEMATICAL ITALIC SMALL ALPHA..MATHEMATICAL ITALIC SMALL OMEGA +1D716..1D734 ; Cased # L& [31] MATHEMATICAL ITALIC EPSILON SYMBOL..MATHEMATICAL BOLD ITALIC CAPITAL OMEGA +1D736..1D74E ; Cased # L& [25] MATHEMATICAL BOLD ITALIC SMALL ALPHA..MATHEMATICAL BOLD ITALIC SMALL OMEGA +1D750..1D76E ; Cased # L& [31] MATHEMATICAL BOLD ITALIC EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD CAPITAL OMEGA +1D770..1D788 ; Cased # L& [25] MATHEMATICAL SANS-SERIF BOLD SMALL ALPHA..MATHEMATICAL SANS-SERIF BOLD SMALL OMEGA +1D78A..1D7A8 ; Cased # L& [31] MATHEMATICAL SANS-SERIF BOLD EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL OMEGA +1D7AA..1D7C2 ; Cased # L& [25] MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL ALPHA..MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL OMEGA +1D7C4..1D7CB ; Cased # L& [8] MATHEMATICAL SANS-SERIF BOLD ITALIC EPSILON SYMBOL..MATHEMATICAL BOLD SMALL DIGAMMA + +# Total code points: 3408 + +# ================================================ + +# Derived Property: Case_Ignorable (CI) +# As defined by Unicode Standard Definition D121 +# C is defined to be case-ignorable if +# Word_Break(C) = MidLetter or MidNumLet, or +# General_Category(C) = Nonspacing_Mark (Mn), Enclosing_Mark (Me), Format (Cf), Modifier_Letter (Lm), or Modifier_Symbol (Sk). + +0027 ; Case_Ignorable # Po APOSTROPHE +002E ; Case_Ignorable # Po FULL STOP +003A ; Case_Ignorable # Po COLON +005E ; Case_Ignorable # Sk CIRCUMFLEX ACCENT +0060 ; Case_Ignorable # Sk GRAVE ACCENT +00A8 ; Case_Ignorable # Sk DIAERESIS +00AD ; Case_Ignorable # Cf SOFT HYPHEN +00AF ; Case_Ignorable # Sk MACRON +00B4 ; Case_Ignorable # Sk ACUTE ACCENT +00B7 ; Case_Ignorable # Po MIDDLE DOT +00B8 ; Case_Ignorable # Sk CEDILLA +02B0..02C1 ; Case_Ignorable # Lm [18] MODIFIER LETTER SMALL H..MODIFIER LETTER REVERSED GLOTTAL STOP +02C2..02C5 ; Case_Ignorable # Sk [4] MODIFIER LETTER LEFT ARROWHEAD..MODIFIER LETTER DOWN ARROWHEAD +02C6..02D1 ; Case_Ignorable # Lm [12] MODIFIER LETTER CIRCUMFLEX ACCENT..MODIFIER LETTER HALF TRIANGULAR COLON +02D2..02DF ; Case_Ignorable # Sk [14] MODIFIER LETTER CENTRED RIGHT HALF RING..MODIFIER LETTER CROSS ACCENT +02E0..02E4 ; Case_Ignorable # Lm [5] MODIFIER LETTER SMALL GAMMA..MODIFIER LETTER SMALL REVERSED GLOTTAL STOP +02E5..02EB ; Case_Ignorable # Sk [7] MODIFIER LETTER EXTRA-HIGH TONE BAR..MODIFIER LETTER YANG DEPARTING TONE MARK +02EC ; Case_Ignorable # Lm MODIFIER LETTER VOICING +02ED ; Case_Ignorable # Sk MODIFIER LETTER UNASPIRATED +02EE ; Case_Ignorable # Lm MODIFIER LETTER DOUBLE APOSTROPHE +02EF..02FF ; Case_Ignorable # Sk [17] MODIFIER LETTER LOW DOWN ARROWHEAD..MODIFIER LETTER LOW LEFT ARROW +0300..036F ; Case_Ignorable # Mn [112] COMBINING GRAVE ACCENT..COMBINING LATIN SMALL LETTER X +0374 ; Case_Ignorable # Lm GREEK NUMERAL SIGN +0375 ; Case_Ignorable # Sk GREEK LOWER NUMERAL SIGN +037A ; Case_Ignorable # Lm GREEK YPOGEGRAMMENI +0384..0385 ; Case_Ignorable # Sk [2] GREEK TONOS..GREEK DIALYTIKA TONOS +0387 ; Case_Ignorable # Po GREEK ANO TELEIA +0483..0487 ; Case_Ignorable # Mn [5] COMBINING CYRILLIC TITLO..COMBINING CYRILLIC POKRYTIE +0488..0489 ; Case_Ignorable # Me [2] COMBINING CYRILLIC HUNDRED THOUSANDS SIGN..COMBINING CYRILLIC MILLIONS SIGN +0559 ; Case_Ignorable # Lm ARMENIAN MODIFIER LETTER LEFT HALF RING +0591..05BD ; Case_Ignorable # Mn [45] HEBREW ACCENT ETNAHTA..HEBREW POINT METEG +05BF ; Case_Ignorable # Mn HEBREW POINT RAFE +05C1..05C2 ; Case_Ignorable # Mn [2] HEBREW POINT SHIN DOT..HEBREW POINT SIN DOT +05C4..05C5 ; Case_Ignorable # Mn [2] HEBREW MARK UPPER DOT..HEBREW MARK LOWER DOT +05C7 ; Case_Ignorable # Mn HEBREW POINT QAMATS QATAN +05F4 ; Case_Ignorable # Po HEBREW PUNCTUATION GERSHAYIM +0600..0603 ; Case_Ignorable # Cf [4] ARABIC NUMBER SIGN..ARABIC SIGN SAFHA +0610..061A ; Case_Ignorable # Mn [11] ARABIC SIGN SALLALLAHOU ALAYHE WASSALLAM..ARABIC SMALL KASRA +0640 ; Case_Ignorable # Lm ARABIC TATWEEL +064B..065E ; Case_Ignorable # Mn [20] ARABIC FATHATAN..ARABIC FATHA WITH TWO DOTS +0670 ; Case_Ignorable # Mn ARABIC LETTER SUPERSCRIPT ALEF +06D6..06DC ; Case_Ignorable # Mn [7] ARABIC SMALL HIGH LIGATURE SAD WITH LAM WITH ALEF MAKSURA..ARABIC SMALL HIGH SEEN +06DD ; Case_Ignorable # Cf ARABIC END OF AYAH +06DE ; Case_Ignorable # Me ARABIC START OF RUB EL HIZB +06DF..06E4 ; Case_Ignorable # Mn [6] ARABIC SMALL HIGH ROUNDED ZERO..ARABIC SMALL HIGH MADDA +06E5..06E6 ; Case_Ignorable # Lm [2] ARABIC SMALL WAW..ARABIC SMALL YEH +06E7..06E8 ; Case_Ignorable # Mn [2] ARABIC SMALL HIGH YEH..ARABIC SMALL HIGH NOON +06EA..06ED ; Case_Ignorable # Mn [4] ARABIC EMPTY CENTRE LOW STOP..ARABIC SMALL LOW MEEM +070F ; Case_Ignorable # Cf SYRIAC ABBREVIATION MARK +0711 ; Case_Ignorable # Mn SYRIAC LETTER SUPERSCRIPT ALAPH +0730..074A ; Case_Ignorable # Mn [27] SYRIAC PTHAHA ABOVE..SYRIAC BARREKH +07A6..07B0 ; Case_Ignorable # Mn [11] THAANA ABAFILI..THAANA SUKUN +07EB..07F3 ; Case_Ignorable # Mn [9] NKO COMBINING SHORT HIGH TONE..NKO COMBINING DOUBLE DOT ABOVE +07F4..07F5 ; Case_Ignorable # Lm [2] NKO HIGH TONE APOSTROPHE..NKO LOW TONE APOSTROPHE +07FA ; Case_Ignorable # Lm NKO LAJANYALAN +0816..0819 ; Case_Ignorable # Mn [4] SAMARITAN MARK IN..SAMARITAN MARK DAGESH +081A ; Case_Ignorable # Lm SAMARITAN MODIFIER LETTER EPENTHETIC YUT +081B..0823 ; Case_Ignorable # Mn [9] SAMARITAN MARK EPENTHETIC YUT..SAMARITAN VOWEL SIGN A +0824 ; Case_Ignorable # Lm SAMARITAN MODIFIER LETTER SHORT A +0825..0827 ; Case_Ignorable # Mn [3] SAMARITAN VOWEL SIGN SHORT A..SAMARITAN VOWEL SIGN U +0828 ; Case_Ignorable # Lm SAMARITAN MODIFIER LETTER I +0829..082D ; Case_Ignorable # Mn [5] SAMARITAN VOWEL SIGN LONG I..SAMARITAN MARK NEQUDAA +0900..0902 ; Case_Ignorable # Mn [3] DEVANAGARI SIGN INVERTED CANDRABINDU..DEVANAGARI SIGN ANUSVARA +093C ; Case_Ignorable # Mn DEVANAGARI SIGN NUKTA +0941..0948 ; Case_Ignorable # Mn [8] DEVANAGARI VOWEL SIGN U..DEVANAGARI VOWEL SIGN AI +094D ; Case_Ignorable # Mn DEVANAGARI SIGN VIRAMA +0951..0955 ; Case_Ignorable # Mn [5] DEVANAGARI STRESS SIGN UDATTA..DEVANAGARI VOWEL SIGN CANDRA LONG E +0962..0963 ; Case_Ignorable # Mn [2] DEVANAGARI VOWEL SIGN VOCALIC L..DEVANAGARI VOWEL SIGN VOCALIC LL +0971 ; Case_Ignorable # Lm DEVANAGARI SIGN HIGH SPACING DOT +0981 ; Case_Ignorable # Mn BENGALI SIGN CANDRABINDU +09BC ; Case_Ignorable # Mn BENGALI SIGN NUKTA +09C1..09C4 ; Case_Ignorable # Mn [4] BENGALI VOWEL SIGN U..BENGALI VOWEL SIGN VOCALIC RR +09CD ; Case_Ignorable # Mn BENGALI SIGN VIRAMA +09E2..09E3 ; Case_Ignorable # Mn [2] BENGALI VOWEL SIGN VOCALIC L..BENGALI VOWEL SIGN VOCALIC LL +0A01..0A02 ; Case_Ignorable # Mn [2] GURMUKHI SIGN ADAK BINDI..GURMUKHI SIGN BINDI +0A3C ; Case_Ignorable # Mn GURMUKHI SIGN NUKTA +0A41..0A42 ; Case_Ignorable # Mn [2] GURMUKHI VOWEL SIGN U..GURMUKHI VOWEL SIGN UU +0A47..0A48 ; Case_Ignorable # Mn [2] GURMUKHI VOWEL SIGN EE..GURMUKHI VOWEL SIGN AI +0A4B..0A4D ; Case_Ignorable # Mn [3] GURMUKHI VOWEL SIGN OO..GURMUKHI SIGN VIRAMA +0A51 ; Case_Ignorable # Mn GURMUKHI SIGN UDAAT +0A70..0A71 ; Case_Ignorable # Mn [2] GURMUKHI TIPPI..GURMUKHI ADDAK +0A75 ; Case_Ignorable # Mn GURMUKHI SIGN YAKASH +0A81..0A82 ; Case_Ignorable # Mn [2] GUJARATI SIGN CANDRABINDU..GUJARATI SIGN ANUSVARA +0ABC ; Case_Ignorable # Mn GUJARATI SIGN NUKTA +0AC1..0AC5 ; Case_Ignorable # Mn [5] GUJARATI VOWEL SIGN U..GUJARATI VOWEL SIGN CANDRA E +0AC7..0AC8 ; Case_Ignorable # Mn [2] GUJARATI VOWEL SIGN E..GUJARATI VOWEL SIGN AI +0ACD ; Case_Ignorable # Mn GUJARATI SIGN VIRAMA +0AE2..0AE3 ; Case_Ignorable # Mn [2] GUJARATI VOWEL SIGN VOCALIC L..GUJARATI VOWEL SIGN VOCALIC LL +0B01 ; Case_Ignorable # Mn ORIYA SIGN CANDRABINDU +0B3C ; Case_Ignorable # Mn ORIYA SIGN NUKTA +0B3F ; Case_Ignorable # Mn ORIYA VOWEL SIGN I +0B41..0B44 ; Case_Ignorable # Mn [4] ORIYA VOWEL SIGN U..ORIYA VOWEL SIGN VOCALIC RR +0B4D ; Case_Ignorable # Mn ORIYA SIGN VIRAMA +0B56 ; Case_Ignorable # Mn ORIYA AI LENGTH MARK +0B62..0B63 ; Case_Ignorable # Mn [2] ORIYA VOWEL SIGN VOCALIC L..ORIYA VOWEL SIGN VOCALIC LL +0B82 ; Case_Ignorable # Mn TAMIL SIGN ANUSVARA +0BC0 ; Case_Ignorable # Mn TAMIL VOWEL SIGN II +0BCD ; Case_Ignorable # Mn TAMIL SIGN VIRAMA +0C3E..0C40 ; Case_Ignorable # Mn [3] TELUGU VOWEL SIGN AA..TELUGU VOWEL SIGN II +0C46..0C48 ; Case_Ignorable # Mn [3] TELUGU VOWEL SIGN E..TELUGU VOWEL SIGN AI +0C4A..0C4D ; Case_Ignorable # Mn [4] TELUGU VOWEL SIGN O..TELUGU SIGN VIRAMA +0C55..0C56 ; Case_Ignorable # Mn [2] TELUGU LENGTH MARK..TELUGU AI LENGTH MARK +0C62..0C63 ; Case_Ignorable # Mn [2] TELUGU VOWEL SIGN VOCALIC L..TELUGU VOWEL SIGN VOCALIC LL +0CBC ; Case_Ignorable # Mn KANNADA SIGN NUKTA +0CBF ; Case_Ignorable # Mn KANNADA VOWEL SIGN I +0CC6 ; Case_Ignorable # Mn KANNADA VOWEL SIGN E +0CCC..0CCD ; Case_Ignorable # Mn [2] KANNADA VOWEL SIGN AU..KANNADA SIGN VIRAMA +0CE2..0CE3 ; Case_Ignorable # Mn [2] KANNADA VOWEL SIGN VOCALIC L..KANNADA VOWEL SIGN VOCALIC LL +0D41..0D44 ; Case_Ignorable # Mn [4] MALAYALAM VOWEL SIGN U..MALAYALAM VOWEL SIGN VOCALIC RR +0D4D ; Case_Ignorable # Mn MALAYALAM SIGN VIRAMA +0D62..0D63 ; Case_Ignorable # Mn [2] MALAYALAM VOWEL SIGN VOCALIC L..MALAYALAM VOWEL SIGN VOCALIC LL +0DCA ; Case_Ignorable # Mn SINHALA SIGN AL-LAKUNA +0DD2..0DD4 ; Case_Ignorable # Mn [3] SINHALA VOWEL SIGN KETTI IS-PILLA..SINHALA VOWEL SIGN KETTI PAA-PILLA +0DD6 ; Case_Ignorable # Mn SINHALA VOWEL SIGN DIGA PAA-PILLA +0E31 ; Case_Ignorable # Mn THAI CHARACTER MAI HAN-AKAT +0E34..0E3A ; Case_Ignorable # Mn [7] THAI CHARACTER SARA I..THAI CHARACTER PHINTHU +0E46 ; Case_Ignorable # Lm THAI CHARACTER MAIYAMOK +0E47..0E4E ; Case_Ignorable # Mn [8] THAI CHARACTER MAITAIKHU..THAI CHARACTER YAMAKKAN +0EB1 ; Case_Ignorable # Mn LAO VOWEL SIGN MAI KAN +0EB4..0EB9 ; Case_Ignorable # Mn [6] LAO VOWEL SIGN I..LAO VOWEL SIGN UU +0EBB..0EBC ; Case_Ignorable # Mn [2] LAO VOWEL SIGN MAI KON..LAO SEMIVOWEL SIGN LO +0EC6 ; Case_Ignorable # Lm LAO KO LA +0EC8..0ECD ; Case_Ignorable # Mn [6] LAO TONE MAI EK..LAO NIGGAHITA +0F18..0F19 ; Case_Ignorable # Mn [2] TIBETAN ASTROLOGICAL SIGN -KHYUD PA..TIBETAN ASTROLOGICAL SIGN SDONG TSHUGS +0F35 ; Case_Ignorable # Mn TIBETAN MARK NGAS BZUNG NYI ZLA +0F37 ; Case_Ignorable # Mn TIBETAN MARK NGAS BZUNG SGOR RTAGS +0F39 ; Case_Ignorable # Mn TIBETAN MARK TSA -PHRU +0F71..0F7E ; Case_Ignorable # Mn [14] TIBETAN VOWEL SIGN AA..TIBETAN SIGN RJES SU NGA RO +0F80..0F84 ; Case_Ignorable # Mn [5] TIBETAN VOWEL SIGN REVERSED I..TIBETAN MARK HALANTA +0F86..0F87 ; Case_Ignorable # Mn [2] TIBETAN SIGN LCI RTAGS..TIBETAN SIGN YANG RTAGS +0F90..0F97 ; Case_Ignorable # Mn [8] TIBETAN SUBJOINED LETTER KA..TIBETAN SUBJOINED LETTER JA +0F99..0FBC ; Case_Ignorable # Mn [36] TIBETAN SUBJOINED LETTER NYA..TIBETAN SUBJOINED LETTER FIXED-FORM RA +0FC6 ; Case_Ignorable # Mn TIBETAN SYMBOL PADMA GDAN +102D..1030 ; Case_Ignorable # Mn [4] MYANMAR VOWEL SIGN I..MYANMAR VOWEL SIGN UU +1032..1037 ; Case_Ignorable # Mn [6] MYANMAR VOWEL SIGN AI..MYANMAR SIGN DOT BELOW +1039..103A ; Case_Ignorable # Mn [2] MYANMAR SIGN VIRAMA..MYANMAR SIGN ASAT +103D..103E ; Case_Ignorable # Mn [2] MYANMAR CONSONANT SIGN MEDIAL WA..MYANMAR CONSONANT SIGN MEDIAL HA +1058..1059 ; Case_Ignorable # Mn [2] MYANMAR VOWEL SIGN VOCALIC L..MYANMAR VOWEL SIGN VOCALIC LL +105E..1060 ; Case_Ignorable # Mn [3] MYANMAR CONSONANT SIGN MON MEDIAL NA..MYANMAR CONSONANT SIGN MON MEDIAL LA +1071..1074 ; Case_Ignorable # Mn [4] MYANMAR VOWEL SIGN GEBA KAREN I..MYANMAR VOWEL SIGN KAYAH EE +1082 ; Case_Ignorable # Mn MYANMAR CONSONANT SIGN SHAN MEDIAL WA +1085..1086 ; Case_Ignorable # Mn [2] MYANMAR VOWEL SIGN SHAN E ABOVE..MYANMAR VOWEL SIGN SHAN FINAL Y +108D ; Case_Ignorable # Mn MYANMAR SIGN SHAN COUNCIL EMPHATIC TONE +109D ; Case_Ignorable # Mn MYANMAR VOWEL SIGN AITON AI +10FC ; Case_Ignorable # Lm MODIFIER LETTER GEORGIAN NAR +135F ; Case_Ignorable # Mn ETHIOPIC COMBINING GEMINATION MARK +1712..1714 ; Case_Ignorable # Mn [3] TAGALOG VOWEL SIGN I..TAGALOG SIGN VIRAMA +1732..1734 ; Case_Ignorable # Mn [3] HANUNOO VOWEL SIGN I..HANUNOO SIGN PAMUDPOD +1752..1753 ; Case_Ignorable # Mn [2] BUHID VOWEL SIGN I..BUHID VOWEL SIGN U +1772..1773 ; Case_Ignorable # Mn [2] TAGBANWA VOWEL SIGN I..TAGBANWA VOWEL SIGN U +17B4..17B5 ; Case_Ignorable # Cf [2] KHMER VOWEL INHERENT AQ..KHMER VOWEL INHERENT AA +17B7..17BD ; Case_Ignorable # Mn [7] KHMER VOWEL SIGN I..KHMER VOWEL SIGN UA +17C6 ; Case_Ignorable # Mn KHMER SIGN NIKAHIT +17C9..17D3 ; Case_Ignorable # Mn [11] KHMER SIGN MUUSIKATOAN..KHMER SIGN BATHAMASAT +17D7 ; Case_Ignorable # Lm KHMER SIGN LEK TOO +17DD ; Case_Ignorable # Mn KHMER SIGN ATTHACAN +180B..180D ; Case_Ignorable # Mn [3] MONGOLIAN FREE VARIATION SELECTOR ONE..MONGOLIAN FREE VARIATION SELECTOR THREE +1843 ; Case_Ignorable # Lm MONGOLIAN LETTER TODO LONG VOWEL SIGN +18A9 ; Case_Ignorable # Mn MONGOLIAN LETTER ALI GALI DAGALGA +1920..1922 ; Case_Ignorable # Mn [3] LIMBU VOWEL SIGN A..LIMBU VOWEL SIGN U +1927..1928 ; Case_Ignorable # Mn [2] LIMBU VOWEL SIGN E..LIMBU VOWEL SIGN O +1932 ; Case_Ignorable # Mn LIMBU SMALL LETTER ANUSVARA +1939..193B ; Case_Ignorable # Mn [3] LIMBU SIGN MUKPHRENG..LIMBU SIGN SA-I +1A17..1A18 ; Case_Ignorable # Mn [2] BUGINESE VOWEL SIGN I..BUGINESE VOWEL SIGN U +1A56 ; Case_Ignorable # Mn TAI THAM CONSONANT SIGN MEDIAL LA +1A58..1A5E ; Case_Ignorable # Mn [7] TAI THAM SIGN MAI KANG LAI..TAI THAM CONSONANT SIGN SA +1A60 ; Case_Ignorable # Mn TAI THAM SIGN SAKOT +1A62 ; Case_Ignorable # Mn TAI THAM VOWEL SIGN MAI SAT +1A65..1A6C ; Case_Ignorable # Mn [8] TAI THAM VOWEL SIGN I..TAI THAM VOWEL SIGN OA BELOW +1A73..1A7C ; Case_Ignorable # Mn [10] TAI THAM VOWEL SIGN OA ABOVE..TAI THAM SIGN KHUEN-LUE KARAN +1A7F ; Case_Ignorable # Mn TAI THAM COMBINING CRYPTOGRAMMIC DOT +1AA7 ; Case_Ignorable # Lm TAI THAM SIGN MAI YAMOK +1B00..1B03 ; Case_Ignorable # Mn [4] BALINESE SIGN ULU RICEM..BALINESE SIGN SURANG +1B34 ; Case_Ignorable # Mn BALINESE SIGN REREKAN +1B36..1B3A ; Case_Ignorable # Mn [5] BALINESE VOWEL SIGN ULU..BALINESE VOWEL SIGN RA REPA +1B3C ; Case_Ignorable # Mn BALINESE VOWEL SIGN LA LENGA +1B42 ; Case_Ignorable # Mn BALINESE VOWEL SIGN PEPET +1B6B..1B73 ; Case_Ignorable # Mn [9] BALINESE MUSICAL SYMBOL COMBINING TEGEH..BALINESE MUSICAL SYMBOL COMBINING GONG +1B80..1B81 ; Case_Ignorable # Mn [2] SUNDANESE SIGN PANYECEK..SUNDANESE SIGN PANGLAYAR +1BA2..1BA5 ; Case_Ignorable # Mn [4] SUNDANESE CONSONANT SIGN PANYAKRA..SUNDANESE VOWEL SIGN PANYUKU +1BA8..1BA9 ; Case_Ignorable # Mn [2] SUNDANESE VOWEL SIGN PAMEPET..SUNDANESE VOWEL SIGN PANEULEUNG +1C2C..1C33 ; Case_Ignorable # Mn [8] LEPCHA VOWEL SIGN E..LEPCHA CONSONANT SIGN T +1C36..1C37 ; Case_Ignorable # Mn [2] LEPCHA SIGN RAN..LEPCHA SIGN NUKTA +1C78..1C7D ; Case_Ignorable # Lm [6] OL CHIKI MU TTUDDAG..OL CHIKI AHAD +1CD0..1CD2 ; Case_Ignorable # Mn [3] VEDIC TONE KARSHANA..VEDIC TONE PRENKHA +1CD4..1CE0 ; Case_Ignorable # Mn [13] VEDIC SIGN YAJURVEDIC MIDLINE SVARITA..VEDIC TONE RIGVEDIC KASHMIRI INDEPENDENT SVARITA +1CE2..1CE8 ; Case_Ignorable # Mn [7] VEDIC SIGN VISARGA SVARITA..VEDIC SIGN VISARGA ANUDATTA WITH TAIL +1CED ; Case_Ignorable # Mn VEDIC SIGN TIRYAK +1D2C..1D61 ; Case_Ignorable # Lm [54] MODIFIER LETTER CAPITAL A..MODIFIER LETTER SMALL CHI +1D78 ; Case_Ignorable # Lm MODIFIER LETTER CYRILLIC EN +1D9B..1DBF ; Case_Ignorable # Lm [37] MODIFIER LETTER SMALL TURNED ALPHA..MODIFIER LETTER SMALL THETA +1DC0..1DE6 ; Case_Ignorable # Mn [39] COMBINING DOTTED GRAVE ACCENT..COMBINING LATIN SMALL LETTER Z +1DFD..1DFF ; Case_Ignorable # Mn [3] COMBINING ALMOST EQUAL TO BELOW..COMBINING RIGHT ARROWHEAD AND DOWN ARROWHEAD BELOW +1FBD ; Case_Ignorable # Sk GREEK KORONIS +1FBF..1FC1 ; Case_Ignorable # Sk [3] GREEK PSILI..GREEK DIALYTIKA AND PERISPOMENI +1FCD..1FCF ; Case_Ignorable # Sk [3] GREEK PSILI AND VARIA..GREEK PSILI AND PERISPOMENI +1FDD..1FDF ; Case_Ignorable # Sk [3] GREEK DASIA AND VARIA..GREEK DASIA AND PERISPOMENI +1FED..1FEF ; Case_Ignorable # Sk [3] GREEK DIALYTIKA AND VARIA..GREEK VARIA +1FFD..1FFE ; Case_Ignorable # Sk [2] GREEK OXIA..GREEK DASIA +200B..200F ; Case_Ignorable # Cf [5] ZERO WIDTH SPACE..RIGHT-TO-LEFT MARK +2018 ; Case_Ignorable # Pi LEFT SINGLE QUOTATION MARK +2019 ; Case_Ignorable # Pf RIGHT SINGLE QUOTATION MARK +2024 ; Case_Ignorable # Po ONE DOT LEADER +2027 ; Case_Ignorable # Po HYPHENATION POINT +202A..202E ; Case_Ignorable # Cf [5] LEFT-TO-RIGHT EMBEDDING..RIGHT-TO-LEFT OVERRIDE +2060..2064 ; Case_Ignorable # Cf [5] WORD JOINER..INVISIBLE PLUS +206A..206F ; Case_Ignorable # Cf [6] INHIBIT SYMMETRIC SWAPPING..NOMINAL DIGIT SHAPES +2071 ; Case_Ignorable # Lm SUPERSCRIPT LATIN SMALL LETTER I +207F ; Case_Ignorable # Lm SUPERSCRIPT LATIN SMALL LETTER N +2090..2094 ; Case_Ignorable # Lm [5] LATIN SUBSCRIPT SMALL LETTER A..LATIN SUBSCRIPT SMALL LETTER SCHWA +20D0..20DC ; Case_Ignorable # Mn [13] COMBINING LEFT HARPOON ABOVE..COMBINING FOUR DOTS ABOVE +20DD..20E0 ; Case_Ignorable # Me [4] COMBINING ENCLOSING CIRCLE..COMBINING ENCLOSING CIRCLE BACKSLASH +20E1 ; Case_Ignorable # Mn COMBINING LEFT RIGHT ARROW ABOVE +20E2..20E4 ; Case_Ignorable # Me [3] COMBINING ENCLOSING SCREEN..COMBINING ENCLOSING UPWARD POINTING TRIANGLE +20E5..20F0 ; Case_Ignorable # Mn [12] COMBINING REVERSE SOLIDUS OVERLAY..COMBINING ASTERISK ABOVE +2C7D ; Case_Ignorable # Lm MODIFIER LETTER CAPITAL V +2CEF..2CF1 ; Case_Ignorable # Mn [3] COPTIC COMBINING NI ABOVE..COPTIC COMBINING SPIRITUS LENIS +2D6F ; Case_Ignorable # Lm TIFINAGH MODIFIER LETTER LABIALIZATION MARK +2DE0..2DFF ; Case_Ignorable # Mn [32] COMBINING CYRILLIC LETTER BE..COMBINING CYRILLIC LETTER IOTIFIED BIG YUS +2E2F ; Case_Ignorable # Lm VERTICAL TILDE +3005 ; Case_Ignorable # Lm IDEOGRAPHIC ITERATION MARK +302A..302F ; Case_Ignorable # Mn [6] IDEOGRAPHIC LEVEL TONE MARK..HANGUL DOUBLE DOT TONE MARK +3031..3035 ; Case_Ignorable # Lm [5] VERTICAL KANA REPEAT MARK..VERTICAL KANA REPEAT MARK LOWER HALF +303B ; Case_Ignorable # Lm VERTICAL IDEOGRAPHIC ITERATION MARK +3099..309A ; Case_Ignorable # Mn [2] COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK..COMBINING KATAKANA-HIRAGANA SEMI-VOICED SOUND MARK +309B..309C ; Case_Ignorable # Sk [2] KATAKANA-HIRAGANA VOICED SOUND MARK..KATAKANA-HIRAGANA SEMI-VOICED SOUND MARK +309D..309E ; Case_Ignorable # Lm [2] HIRAGANA ITERATION MARK..HIRAGANA VOICED ITERATION MARK +30FC..30FE ; Case_Ignorable # Lm [3] KATAKANA-HIRAGANA PROLONGED SOUND MARK..KATAKANA VOICED ITERATION MARK +A015 ; Case_Ignorable # Lm YI SYLLABLE WU +A4F8..A4FD ; Case_Ignorable # Lm [6] LISU LETTER TONE MYA TI..LISU LETTER TONE MYA JEU +A60C ; Case_Ignorable # Lm VAI SYLLABLE LENGTHENER +A66F ; Case_Ignorable # Mn COMBINING CYRILLIC VZMET +A670..A672 ; Case_Ignorable # Me [3] COMBINING CYRILLIC TEN MILLIONS SIGN..COMBINING CYRILLIC THOUSAND MILLIONS SIGN +A67C..A67D ; Case_Ignorable # Mn [2] COMBINING CYRILLIC KAVYKA..COMBINING CYRILLIC PAYEROK +A67F ; Case_Ignorable # Lm CYRILLIC PAYEROK +A6F0..A6F1 ; Case_Ignorable # Mn [2] BAMUM COMBINING MARK KOQNDON..BAMUM COMBINING MARK TUKWENTIS +A700..A716 ; Case_Ignorable # Sk [23] MODIFIER LETTER CHINESE TONE YIN PING..MODIFIER LETTER EXTRA-LOW LEFT-STEM TONE BAR +A717..A71F ; Case_Ignorable # Lm [9] MODIFIER LETTER DOT VERTICAL BAR..MODIFIER LETTER LOW INVERTED EXCLAMATION MARK +A720..A721 ; Case_Ignorable # Sk [2] MODIFIER LETTER STRESS AND HIGH TONE..MODIFIER LETTER STRESS AND LOW TONE +A770 ; Case_Ignorable # Lm MODIFIER LETTER US +A788 ; Case_Ignorable # Lm MODIFIER LETTER LOW CIRCUMFLEX ACCENT +A789..A78A ; Case_Ignorable # Sk [2] MODIFIER LETTER COLON..MODIFIER LETTER SHORT EQUALS SIGN +A802 ; Case_Ignorable # Mn SYLOTI NAGRI SIGN DVISVARA +A806 ; Case_Ignorable # Mn SYLOTI NAGRI SIGN HASANTA +A80B ; Case_Ignorable # Mn SYLOTI NAGRI SIGN ANUSVARA +A825..A826 ; Case_Ignorable # Mn [2] SYLOTI NAGRI VOWEL SIGN U..SYLOTI NAGRI VOWEL SIGN E +A8C4 ; Case_Ignorable # Mn SAURASHTRA SIGN VIRAMA +A8E0..A8F1 ; Case_Ignorable # Mn [18] COMBINING DEVANAGARI DIGIT ZERO..COMBINING DEVANAGARI SIGN AVAGRAHA +A926..A92D ; Case_Ignorable # Mn [8] KAYAH LI VOWEL UE..KAYAH LI TONE CALYA PLOPHU +A947..A951 ; Case_Ignorable # Mn [11] REJANG VOWEL SIGN I..REJANG CONSONANT SIGN R +A980..A982 ; Case_Ignorable # Mn [3] JAVANESE SIGN PANYANGGA..JAVANESE SIGN LAYAR +A9B3 ; Case_Ignorable # Mn JAVANESE SIGN CECAK TELU +A9B6..A9B9 ; Case_Ignorable # Mn [4] JAVANESE VOWEL SIGN WULU..JAVANESE VOWEL SIGN SUKU MENDUT +A9BC ; Case_Ignorable # Mn JAVANESE VOWEL SIGN PEPET +A9CF ; Case_Ignorable # Lm JAVANESE PANGRANGKEP +AA29..AA2E ; Case_Ignorable # Mn [6] CHAM VOWEL SIGN AA..CHAM VOWEL SIGN OE +AA31..AA32 ; Case_Ignorable # Mn [2] CHAM VOWEL SIGN AU..CHAM VOWEL SIGN UE +AA35..AA36 ; Case_Ignorable # Mn [2] CHAM CONSONANT SIGN LA..CHAM CONSONANT SIGN WA +AA43 ; Case_Ignorable # Mn CHAM CONSONANT SIGN FINAL NG +AA4C ; Case_Ignorable # Mn CHAM CONSONANT SIGN FINAL M +AA70 ; Case_Ignorable # Lm MYANMAR MODIFIER LETTER KHAMTI REDUPLICATION +AAB0 ; Case_Ignorable # Mn TAI VIET MAI KANG +AAB2..AAB4 ; Case_Ignorable # Mn [3] TAI VIET VOWEL I..TAI VIET VOWEL U +AAB7..AAB8 ; Case_Ignorable # Mn [2] TAI VIET MAI KHIT..TAI VIET VOWEL IA +AABE..AABF ; Case_Ignorable # Mn [2] TAI VIET VOWEL AM..TAI VIET TONE MAI EK +AAC1 ; Case_Ignorable # Mn TAI VIET TONE MAI THO +AADD ; Case_Ignorable # Lm TAI VIET SYMBOL SAM +ABE5 ; Case_Ignorable # Mn MEETEI MAYEK VOWEL SIGN ANAP +ABE8 ; Case_Ignorable # Mn MEETEI MAYEK VOWEL SIGN UNAP +ABED ; Case_Ignorable # Mn MEETEI MAYEK APUN IYEK +FB1E ; Case_Ignorable # Mn HEBREW POINT JUDEO-SPANISH VARIKA +FE00..FE0F ; Case_Ignorable # Mn [16] VARIATION SELECTOR-1..VARIATION SELECTOR-16 +FE13 ; Case_Ignorable # Po PRESENTATION FORM FOR VERTICAL COLON +FE20..FE26 ; Case_Ignorable # Mn [7] COMBINING LIGATURE LEFT HALF..COMBINING CONJOINING MACRON +FE52 ; Case_Ignorable # Po SMALL FULL STOP +FE55 ; Case_Ignorable # Po SMALL COLON +FEFF ; Case_Ignorable # Cf ZERO WIDTH NO-BREAK SPACE +FF07 ; Case_Ignorable # Po FULLWIDTH APOSTROPHE +FF0E ; Case_Ignorable # Po FULLWIDTH FULL STOP +FF1A ; Case_Ignorable # Po FULLWIDTH COLON +FF3E ; Case_Ignorable # Sk FULLWIDTH CIRCUMFLEX ACCENT +FF40 ; Case_Ignorable # Sk FULLWIDTH GRAVE ACCENT +FF70 ; Case_Ignorable # Lm HALFWIDTH KATAKANA-HIRAGANA PROLONGED SOUND MARK +FF9E..FF9F ; Case_Ignorable # Lm [2] HALFWIDTH KATAKANA VOICED SOUND MARK..HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK +FFE3 ; Case_Ignorable # Sk FULLWIDTH MACRON +FFF9..FFFB ; Case_Ignorable # Cf [3] INTERLINEAR ANNOTATION ANCHOR..INTERLINEAR ANNOTATION TERMINATOR +101FD ; Case_Ignorable # Mn PHAISTOS DISC SIGN COMBINING OBLIQUE STROKE +10A01..10A03 ; Case_Ignorable # Mn [3] KHAROSHTHI VOWEL SIGN I..KHAROSHTHI VOWEL SIGN VOCALIC R +10A05..10A06 ; Case_Ignorable # Mn [2] KHAROSHTHI VOWEL SIGN E..KHAROSHTHI VOWEL SIGN O +10A0C..10A0F ; Case_Ignorable # Mn [4] KHAROSHTHI VOWEL LENGTH MARK..KHAROSHTHI SIGN VISARGA +10A38..10A3A ; Case_Ignorable # Mn [3] KHAROSHTHI SIGN BAR ABOVE..KHAROSHTHI SIGN DOT BELOW +10A3F ; Case_Ignorable # Mn KHAROSHTHI VIRAMA +11080..11081 ; Case_Ignorable # Mn [2] KAITHI SIGN CANDRABINDU..KAITHI SIGN ANUSVARA +110B3..110B6 ; Case_Ignorable # Mn [4] KAITHI VOWEL SIGN U..KAITHI VOWEL SIGN AI +110B9..110BA ; Case_Ignorable # Mn [2] KAITHI SIGN VIRAMA..KAITHI SIGN NUKTA +110BD ; Case_Ignorable # Cf KAITHI NUMBER SIGN +1D167..1D169 ; Case_Ignorable # Mn [3] MUSICAL SYMBOL COMBINING TREMOLO-1..MUSICAL SYMBOL COMBINING TREMOLO-3 +1D173..1D17A ; Case_Ignorable # Cf [8] MUSICAL SYMBOL BEGIN BEAM..MUSICAL SYMBOL END PHRASE +1D17B..1D182 ; Case_Ignorable # Mn [8] MUSICAL SYMBOL COMBINING ACCENT..MUSICAL SYMBOL COMBINING LOURE +1D185..1D18B ; Case_Ignorable # Mn [7] MUSICAL SYMBOL COMBINING DOIT..MUSICAL SYMBOL COMBINING TRIPLE TONGUE +1D1AA..1D1AD ; Case_Ignorable # Mn [4] MUSICAL SYMBOL COMBINING DOWN BOW..MUSICAL SYMBOL COMBINING SNAP PIZZICATO +1D242..1D244 ; Case_Ignorable # Mn [3] COMBINING GREEK MUSICAL TRISEME..COMBINING GREEK MUSICAL PENTASEME +E0001 ; Case_Ignorable # Cf LANGUAGE TAG +E0020..E007F ; Case_Ignorable # Cf [96] TAG SPACE..CANCEL TAG +E0100..E01EF ; Case_Ignorable # Mn [240] VARIATION SELECTOR-17..VARIATION SELECTOR-256 + +# Total code points: 1632 + +# ================================================ + +# Derived Property: Changes_When_Lowercased (CWL) +# Characters whose normalized forms are not stable under a toLowercase mapping. +# For more information, see D124 in Section 3.13, "Default Case Algorithms". +# Changes_When_Lowercased(X) is true when toLowercase(toNFD(X)) != toNFD(X) + +0041..005A ; Changes_When_Lowercased # L& [26] LATIN CAPITAL LETTER A..LATIN CAPITAL LETTER Z +00C0..00D6 ; Changes_When_Lowercased # L& [23] LATIN CAPITAL LETTER A WITH GRAVE..LATIN CAPITAL LETTER O WITH DIAERESIS +00D8..00DE ; Changes_When_Lowercased # L& [7] LATIN CAPITAL LETTER O WITH STROKE..LATIN CAPITAL LETTER THORN +0100 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER A WITH MACRON +0102 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER A WITH BREVE +0104 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER A WITH OGONEK +0106 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER C WITH ACUTE +0108 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER C WITH CIRCUMFLEX +010A ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER C WITH DOT ABOVE +010C ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER C WITH CARON +010E ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER D WITH CARON +0110 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER D WITH STROKE +0112 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER E WITH MACRON +0114 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER E WITH BREVE +0116 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER E WITH DOT ABOVE +0118 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER E WITH OGONEK +011A ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER E WITH CARON +011C ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER G WITH CIRCUMFLEX +011E ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER G WITH BREVE +0120 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER G WITH DOT ABOVE +0122 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER G WITH CEDILLA +0124 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER H WITH CIRCUMFLEX +0126 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER H WITH STROKE +0128 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER I WITH TILDE +012A ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER I WITH MACRON +012C ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER I WITH BREVE +012E ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER I WITH OGONEK +0130 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER I WITH DOT ABOVE +0132 ; Changes_When_Lowercased # L& LATIN CAPITAL LIGATURE IJ +0134 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER J WITH CIRCUMFLEX +0136 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER K WITH CEDILLA +0139 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER L WITH ACUTE +013B ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER L WITH CEDILLA +013D ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER L WITH CARON +013F ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER L WITH MIDDLE DOT +0141 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER L WITH STROKE +0143 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER N WITH ACUTE +0145 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER N WITH CEDILLA +0147 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER N WITH CARON +014A ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER ENG +014C ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER O WITH MACRON +014E ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER O WITH BREVE +0150 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER O WITH DOUBLE ACUTE +0152 ; Changes_When_Lowercased # L& LATIN CAPITAL LIGATURE OE +0154 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER R WITH ACUTE +0156 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER R WITH CEDILLA +0158 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER R WITH CARON +015A ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER S WITH ACUTE +015C ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER S WITH CIRCUMFLEX +015E ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER S WITH CEDILLA +0160 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER S WITH CARON +0162 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER T WITH CEDILLA +0164 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER T WITH CARON +0166 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER T WITH STROKE +0168 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER U WITH TILDE +016A ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER U WITH MACRON +016C ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER U WITH BREVE +016E ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER U WITH RING ABOVE +0170 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER U WITH DOUBLE ACUTE +0172 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER U WITH OGONEK +0174 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER W WITH CIRCUMFLEX +0176 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER Y WITH CIRCUMFLEX +0178..0179 ; Changes_When_Lowercased # L& [2] LATIN CAPITAL LETTER Y WITH DIAERESIS..LATIN CAPITAL LETTER Z WITH ACUTE +017B ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER Z WITH DOT ABOVE +017D ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER Z WITH CARON +0181..0182 ; Changes_When_Lowercased # L& [2] LATIN CAPITAL LETTER B WITH HOOK..LATIN CAPITAL LETTER B WITH TOPBAR +0184 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER TONE SIX +0186..0187 ; Changes_When_Lowercased # L& [2] LATIN CAPITAL LETTER OPEN O..LATIN CAPITAL LETTER C WITH HOOK +0189..018B ; Changes_When_Lowercased # L& [3] LATIN CAPITAL LETTER AFRICAN D..LATIN CAPITAL LETTER D WITH TOPBAR +018E..0191 ; Changes_When_Lowercased # L& [4] LATIN CAPITAL LETTER REVERSED E..LATIN CAPITAL LETTER F WITH HOOK +0193..0194 ; Changes_When_Lowercased # L& [2] LATIN CAPITAL LETTER G WITH HOOK..LATIN CAPITAL LETTER GAMMA +0196..0198 ; Changes_When_Lowercased # L& [3] LATIN CAPITAL LETTER IOTA..LATIN CAPITAL LETTER K WITH HOOK +019C..019D ; Changes_When_Lowercased # L& [2] LATIN CAPITAL LETTER TURNED M..LATIN CAPITAL LETTER N WITH LEFT HOOK +019F..01A0 ; Changes_When_Lowercased # L& [2] LATIN CAPITAL LETTER O WITH MIDDLE TILDE..LATIN CAPITAL LETTER O WITH HORN +01A2 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER OI +01A4 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER P WITH HOOK +01A6..01A7 ; Changes_When_Lowercased # L& [2] LATIN LETTER YR..LATIN CAPITAL LETTER TONE TWO +01A9 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER ESH +01AC ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER T WITH HOOK +01AE..01AF ; Changes_When_Lowercased # L& [2] LATIN CAPITAL LETTER T WITH RETROFLEX HOOK..LATIN CAPITAL LETTER U WITH HORN +01B1..01B3 ; Changes_When_Lowercased # L& [3] LATIN CAPITAL LETTER UPSILON..LATIN CAPITAL LETTER Y WITH HOOK +01B5 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER Z WITH STROKE +01B7..01B8 ; Changes_When_Lowercased # L& [2] LATIN CAPITAL LETTER EZH..LATIN CAPITAL LETTER EZH REVERSED +01BC ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER TONE FIVE +01C4..01C5 ; Changes_When_Lowercased # L& [2] LATIN CAPITAL LETTER DZ WITH CARON..LATIN CAPITAL LETTER D WITH SMALL LETTER Z WITH CARON +01C7..01C8 ; Changes_When_Lowercased # L& [2] LATIN CAPITAL LETTER LJ..LATIN CAPITAL LETTER L WITH SMALL LETTER J +01CA..01CB ; Changes_When_Lowercased # L& [2] LATIN CAPITAL LETTER NJ..LATIN CAPITAL LETTER N WITH SMALL LETTER J +01CD ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER A WITH CARON +01CF ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER I WITH CARON +01D1 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER O WITH CARON +01D3 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER U WITH CARON +01D5 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER U WITH DIAERESIS AND MACRON +01D7 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER U WITH DIAERESIS AND ACUTE +01D9 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER U WITH DIAERESIS AND CARON +01DB ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER U WITH DIAERESIS AND GRAVE +01DE ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER A WITH DIAERESIS AND MACRON +01E0 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER A WITH DOT ABOVE AND MACRON +01E2 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER AE WITH MACRON +01E4 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER G WITH STROKE +01E6 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER G WITH CARON +01E8 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER K WITH CARON +01EA ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER O WITH OGONEK +01EC ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER O WITH OGONEK AND MACRON +01EE ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER EZH WITH CARON +01F1..01F2 ; Changes_When_Lowercased # L& [2] LATIN CAPITAL LETTER DZ..LATIN CAPITAL LETTER D WITH SMALL LETTER Z +01F4 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER G WITH ACUTE +01F6..01F8 ; Changes_When_Lowercased # L& [3] LATIN CAPITAL LETTER HWAIR..LATIN CAPITAL LETTER N WITH GRAVE +01FA ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER A WITH RING ABOVE AND ACUTE +01FC ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER AE WITH ACUTE +01FE ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER O WITH STROKE AND ACUTE +0200 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER A WITH DOUBLE GRAVE +0202 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER A WITH INVERTED BREVE +0204 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER E WITH DOUBLE GRAVE +0206 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER E WITH INVERTED BREVE +0208 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER I WITH DOUBLE GRAVE +020A ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER I WITH INVERTED BREVE +020C ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER O WITH DOUBLE GRAVE +020E ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER O WITH INVERTED BREVE +0210 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER R WITH DOUBLE GRAVE +0212 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER R WITH INVERTED BREVE +0214 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER U WITH DOUBLE GRAVE +0216 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER U WITH INVERTED BREVE +0218 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER S WITH COMMA BELOW +021A ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER T WITH COMMA BELOW +021C ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER YOGH +021E ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER H WITH CARON +0220 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER N WITH LONG RIGHT LEG +0222 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER OU +0224 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER Z WITH HOOK +0226 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER A WITH DOT ABOVE +0228 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER E WITH CEDILLA +022A ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER O WITH DIAERESIS AND MACRON +022C ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER O WITH TILDE AND MACRON +022E ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER O WITH DOT ABOVE +0230 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER O WITH DOT ABOVE AND MACRON +0232 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER Y WITH MACRON +023A..023B ; Changes_When_Lowercased # L& [2] LATIN CAPITAL LETTER A WITH STROKE..LATIN CAPITAL LETTER C WITH STROKE +023D..023E ; Changes_When_Lowercased # L& [2] LATIN CAPITAL LETTER L WITH BAR..LATIN CAPITAL LETTER T WITH DIAGONAL STROKE +0241 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER GLOTTAL STOP +0243..0246 ; Changes_When_Lowercased # L& [4] LATIN CAPITAL LETTER B WITH STROKE..LATIN CAPITAL LETTER E WITH STROKE +0248 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER J WITH STROKE +024A ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER SMALL Q WITH HOOK TAIL +024C ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER R WITH STROKE +024E ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER Y WITH STROKE +0370 ; Changes_When_Lowercased # L& GREEK CAPITAL LETTER HETA +0372 ; Changes_When_Lowercased # L& GREEK CAPITAL LETTER ARCHAIC SAMPI +0376 ; Changes_When_Lowercased # L& GREEK CAPITAL LETTER PAMPHYLIAN DIGAMMA +0386 ; Changes_When_Lowercased # L& GREEK CAPITAL LETTER ALPHA WITH TONOS +0388..038A ; Changes_When_Lowercased # L& [3] GREEK CAPITAL LETTER EPSILON WITH TONOS..GREEK CAPITAL LETTER IOTA WITH TONOS +038C ; Changes_When_Lowercased # L& GREEK CAPITAL LETTER OMICRON WITH TONOS +038E..038F ; Changes_When_Lowercased # L& [2] GREEK CAPITAL LETTER UPSILON WITH TONOS..GREEK CAPITAL LETTER OMEGA WITH TONOS +0391..03A1 ; Changes_When_Lowercased # L& [17] GREEK CAPITAL LETTER ALPHA..GREEK CAPITAL LETTER RHO +03A3..03AB ; Changes_When_Lowercased # L& [9] GREEK CAPITAL LETTER SIGMA..GREEK CAPITAL LETTER UPSILON WITH DIALYTIKA +03CF ; Changes_When_Lowercased # L& GREEK CAPITAL KAI SYMBOL +03D8 ; Changes_When_Lowercased # L& GREEK LETTER ARCHAIC KOPPA +03DA ; Changes_When_Lowercased # L& GREEK LETTER STIGMA +03DC ; Changes_When_Lowercased # L& GREEK LETTER DIGAMMA +03DE ; Changes_When_Lowercased # L& GREEK LETTER KOPPA +03E0 ; Changes_When_Lowercased # L& GREEK LETTER SAMPI +03E2 ; Changes_When_Lowercased # L& COPTIC CAPITAL LETTER SHEI +03E4 ; Changes_When_Lowercased # L& COPTIC CAPITAL LETTER FEI +03E6 ; Changes_When_Lowercased # L& COPTIC CAPITAL LETTER KHEI +03E8 ; Changes_When_Lowercased # L& COPTIC CAPITAL LETTER HORI +03EA ; Changes_When_Lowercased # L& COPTIC CAPITAL LETTER GANGIA +03EC ; Changes_When_Lowercased # L& COPTIC CAPITAL LETTER SHIMA +03EE ; Changes_When_Lowercased # L& COPTIC CAPITAL LETTER DEI +03F4 ; Changes_When_Lowercased # L& GREEK CAPITAL THETA SYMBOL +03F7 ; Changes_When_Lowercased # L& GREEK CAPITAL LETTER SHO +03F9..03FA ; Changes_When_Lowercased # L& [2] GREEK CAPITAL LUNATE SIGMA SYMBOL..GREEK CAPITAL LETTER SAN +03FD..042F ; Changes_When_Lowercased # L& [51] GREEK CAPITAL REVERSED LUNATE SIGMA SYMBOL..CYRILLIC CAPITAL LETTER YA +0460 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER OMEGA +0462 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER YAT +0464 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER IOTIFIED E +0466 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER LITTLE YUS +0468 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER IOTIFIED LITTLE YUS +046A ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER BIG YUS +046C ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER IOTIFIED BIG YUS +046E ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER KSI +0470 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER PSI +0472 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER FITA +0474 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER IZHITSA +0476 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER IZHITSA WITH DOUBLE GRAVE ACCENT +0478 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER UK +047A ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER ROUND OMEGA +047C ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER OMEGA WITH TITLO +047E ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER OT +0480 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER KOPPA +048A ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER SHORT I WITH TAIL +048C ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER SEMISOFT SIGN +048E ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER ER WITH TICK +0490 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER GHE WITH UPTURN +0492 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER GHE WITH STROKE +0494 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER GHE WITH MIDDLE HOOK +0496 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER ZHE WITH DESCENDER +0498 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER ZE WITH DESCENDER +049A ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER KA WITH DESCENDER +049C ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER KA WITH VERTICAL STROKE +049E ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER KA WITH STROKE +04A0 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER BASHKIR KA +04A2 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER EN WITH DESCENDER +04A4 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LIGATURE EN GHE +04A6 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER PE WITH MIDDLE HOOK +04A8 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER ABKHASIAN HA +04AA ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER ES WITH DESCENDER +04AC ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER TE WITH DESCENDER +04AE ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER STRAIGHT U +04B0 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER STRAIGHT U WITH STROKE +04B2 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER HA WITH DESCENDER +04B4 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LIGATURE TE TSE +04B6 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER CHE WITH DESCENDER +04B8 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER CHE WITH VERTICAL STROKE +04BA ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER SHHA +04BC ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER ABKHASIAN CHE +04BE ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER ABKHASIAN CHE WITH DESCENDER +04C0..04C1 ; Changes_When_Lowercased # L& [2] CYRILLIC LETTER PALOCHKA..CYRILLIC CAPITAL LETTER ZHE WITH BREVE +04C3 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER KA WITH HOOK +04C5 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER EL WITH TAIL +04C7 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER EN WITH HOOK +04C9 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER EN WITH TAIL +04CB ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER KHAKASSIAN CHE +04CD ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER EM WITH TAIL +04D0 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER A WITH BREVE +04D2 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER A WITH DIAERESIS +04D4 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LIGATURE A IE +04D6 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER IE WITH BREVE +04D8 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER SCHWA +04DA ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER SCHWA WITH DIAERESIS +04DC ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER ZHE WITH DIAERESIS +04DE ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER ZE WITH DIAERESIS +04E0 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER ABKHASIAN DZE +04E2 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER I WITH MACRON +04E4 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER I WITH DIAERESIS +04E6 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER O WITH DIAERESIS +04E8 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER BARRED O +04EA ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER BARRED O WITH DIAERESIS +04EC ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER E WITH DIAERESIS +04EE ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER U WITH MACRON +04F0 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER U WITH DIAERESIS +04F2 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER U WITH DOUBLE ACUTE +04F4 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER CHE WITH DIAERESIS +04F6 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER GHE WITH DESCENDER +04F8 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER YERU WITH DIAERESIS +04FA ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER GHE WITH STROKE AND HOOK +04FC ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER HA WITH HOOK +04FE ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER HA WITH STROKE +0500 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER KOMI DE +0502 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER KOMI DJE +0504 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER KOMI ZJE +0506 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER KOMI DZJE +0508 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER KOMI LJE +050A ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER KOMI NJE +050C ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER KOMI SJE +050E ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER KOMI TJE +0510 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER REVERSED ZE +0512 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER EL WITH HOOK +0514 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER LHA +0516 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER RHA +0518 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER YAE +051A ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER QA +051C ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER WE +051E ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER ALEUT KA +0520 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER EL WITH MIDDLE HOOK +0522 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER EN WITH MIDDLE HOOK +0524 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER PE WITH DESCENDER +0531..0556 ; Changes_When_Lowercased # L& [38] ARMENIAN CAPITAL LETTER AYB..ARMENIAN CAPITAL LETTER FEH +10A0..10C5 ; Changes_When_Lowercased # L& [38] GEORGIAN CAPITAL LETTER AN..GEORGIAN CAPITAL LETTER HOE +1E00 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER A WITH RING BELOW +1E02 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER B WITH DOT ABOVE +1E04 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER B WITH DOT BELOW +1E06 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER B WITH LINE BELOW +1E08 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER C WITH CEDILLA AND ACUTE +1E0A ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER D WITH DOT ABOVE +1E0C ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER D WITH DOT BELOW +1E0E ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER D WITH LINE BELOW +1E10 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER D WITH CEDILLA +1E12 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER D WITH CIRCUMFLEX BELOW +1E14 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER E WITH MACRON AND GRAVE +1E16 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER E WITH MACRON AND ACUTE +1E18 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER E WITH CIRCUMFLEX BELOW +1E1A ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER E WITH TILDE BELOW +1E1C ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER E WITH CEDILLA AND BREVE +1E1E ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER F WITH DOT ABOVE +1E20 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER G WITH MACRON +1E22 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER H WITH DOT ABOVE +1E24 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER H WITH DOT BELOW +1E26 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER H WITH DIAERESIS +1E28 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER H WITH CEDILLA +1E2A ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER H WITH BREVE BELOW +1E2C ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER I WITH TILDE BELOW +1E2E ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER I WITH DIAERESIS AND ACUTE +1E30 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER K WITH ACUTE +1E32 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER K WITH DOT BELOW +1E34 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER K WITH LINE BELOW +1E36 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER L WITH DOT BELOW +1E38 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER L WITH DOT BELOW AND MACRON +1E3A ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER L WITH LINE BELOW +1E3C ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER L WITH CIRCUMFLEX BELOW +1E3E ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER M WITH ACUTE +1E40 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER M WITH DOT ABOVE +1E42 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER M WITH DOT BELOW +1E44 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER N WITH DOT ABOVE +1E46 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER N WITH DOT BELOW +1E48 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER N WITH LINE BELOW +1E4A ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER N WITH CIRCUMFLEX BELOW +1E4C ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER O WITH TILDE AND ACUTE +1E4E ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER O WITH TILDE AND DIAERESIS +1E50 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER O WITH MACRON AND GRAVE +1E52 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER O WITH MACRON AND ACUTE +1E54 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER P WITH ACUTE +1E56 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER P WITH DOT ABOVE +1E58 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER R WITH DOT ABOVE +1E5A ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER R WITH DOT BELOW +1E5C ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER R WITH DOT BELOW AND MACRON +1E5E ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER R WITH LINE BELOW +1E60 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER S WITH DOT ABOVE +1E62 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER S WITH DOT BELOW +1E64 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER S WITH ACUTE AND DOT ABOVE +1E66 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER S WITH CARON AND DOT ABOVE +1E68 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER S WITH DOT BELOW AND DOT ABOVE +1E6A ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER T WITH DOT ABOVE +1E6C ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER T WITH DOT BELOW +1E6E ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER T WITH LINE BELOW +1E70 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER T WITH CIRCUMFLEX BELOW +1E72 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER U WITH DIAERESIS BELOW +1E74 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER U WITH TILDE BELOW +1E76 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER U WITH CIRCUMFLEX BELOW +1E78 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER U WITH TILDE AND ACUTE +1E7A ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER U WITH MACRON AND DIAERESIS +1E7C ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER V WITH TILDE +1E7E ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER V WITH DOT BELOW +1E80 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER W WITH GRAVE +1E82 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER W WITH ACUTE +1E84 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER W WITH DIAERESIS +1E86 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER W WITH DOT ABOVE +1E88 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER W WITH DOT BELOW +1E8A ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER X WITH DOT ABOVE +1E8C ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER X WITH DIAERESIS +1E8E ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER Y WITH DOT ABOVE +1E90 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER Z WITH CIRCUMFLEX +1E92 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER Z WITH DOT BELOW +1E94 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER Z WITH LINE BELOW +1E9E ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER SHARP S +1EA0 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER A WITH DOT BELOW +1EA2 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER A WITH HOOK ABOVE +1EA4 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND ACUTE +1EA6 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND GRAVE +1EA8 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND HOOK ABOVE +1EAA ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND TILDE +1EAC ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND DOT BELOW +1EAE ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER A WITH BREVE AND ACUTE +1EB0 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER A WITH BREVE AND GRAVE +1EB2 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER A WITH BREVE AND HOOK ABOVE +1EB4 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER A WITH BREVE AND TILDE +1EB6 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER A WITH BREVE AND DOT BELOW +1EB8 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER E WITH DOT BELOW +1EBA ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER E WITH HOOK ABOVE +1EBC ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER E WITH TILDE +1EBE ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND ACUTE +1EC0 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND GRAVE +1EC2 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND HOOK ABOVE +1EC4 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND TILDE +1EC6 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND DOT BELOW +1EC8 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER I WITH HOOK ABOVE +1ECA ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER I WITH DOT BELOW +1ECC ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER O WITH DOT BELOW +1ECE ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER O WITH HOOK ABOVE +1ED0 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND ACUTE +1ED2 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND GRAVE +1ED4 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND HOOK ABOVE +1ED6 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND TILDE +1ED8 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND DOT BELOW +1EDA ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER O WITH HORN AND ACUTE +1EDC ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER O WITH HORN AND GRAVE +1EDE ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER O WITH HORN AND HOOK ABOVE +1EE0 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER O WITH HORN AND TILDE +1EE2 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER O WITH HORN AND DOT BELOW +1EE4 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER U WITH DOT BELOW +1EE6 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER U WITH HOOK ABOVE +1EE8 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER U WITH HORN AND ACUTE +1EEA ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER U WITH HORN AND GRAVE +1EEC ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER U WITH HORN AND HOOK ABOVE +1EEE ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER U WITH HORN AND TILDE +1EF0 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER U WITH HORN AND DOT BELOW +1EF2 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER Y WITH GRAVE +1EF4 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER Y WITH DOT BELOW +1EF6 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER Y WITH HOOK ABOVE +1EF8 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER Y WITH TILDE +1EFA ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER MIDDLE-WELSH LL +1EFC ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER MIDDLE-WELSH V +1EFE ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER Y WITH LOOP +1F08..1F0F ; Changes_When_Lowercased # L& [8] GREEK CAPITAL LETTER ALPHA WITH PSILI..GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI +1F18..1F1D ; Changes_When_Lowercased # L& [6] GREEK CAPITAL LETTER EPSILON WITH PSILI..GREEK CAPITAL LETTER EPSILON WITH DASIA AND OXIA +1F28..1F2F ; Changes_When_Lowercased # L& [8] GREEK CAPITAL LETTER ETA WITH PSILI..GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI +1F38..1F3F ; Changes_When_Lowercased # L& [8] GREEK CAPITAL LETTER IOTA WITH PSILI..GREEK CAPITAL LETTER IOTA WITH DASIA AND PERISPOMENI +1F48..1F4D ; Changes_When_Lowercased # L& [6] GREEK CAPITAL LETTER OMICRON WITH PSILI..GREEK CAPITAL LETTER OMICRON WITH DASIA AND OXIA +1F59 ; Changes_When_Lowercased # L& GREEK CAPITAL LETTER UPSILON WITH DASIA +1F5B ; Changes_When_Lowercased # L& GREEK CAPITAL LETTER UPSILON WITH DASIA AND VARIA +1F5D ; Changes_When_Lowercased # L& GREEK CAPITAL LETTER UPSILON WITH DASIA AND OXIA +1F5F ; Changes_When_Lowercased # L& GREEK CAPITAL LETTER UPSILON WITH DASIA AND PERISPOMENI +1F68..1F6F ; Changes_When_Lowercased # L& [8] GREEK CAPITAL LETTER OMEGA WITH PSILI..GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI +1F88..1F8F ; Changes_When_Lowercased # L& [8] GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI..GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI +1F98..1F9F ; Changes_When_Lowercased # L& [8] GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI..GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI +1FA8..1FAF ; Changes_When_Lowercased # L& [8] GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI..GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI +1FB8..1FBC ; Changes_When_Lowercased # L& [5] GREEK CAPITAL LETTER ALPHA WITH VRACHY..GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI +1FC8..1FCC ; Changes_When_Lowercased # L& [5] GREEK CAPITAL LETTER EPSILON WITH VARIA..GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI +1FD8..1FDB ; Changes_When_Lowercased # L& [4] GREEK CAPITAL LETTER IOTA WITH VRACHY..GREEK CAPITAL LETTER IOTA WITH OXIA +1FE8..1FEC ; Changes_When_Lowercased # L& [5] GREEK CAPITAL LETTER UPSILON WITH VRACHY..GREEK CAPITAL LETTER RHO WITH DASIA +1FF8..1FFC ; Changes_When_Lowercased # L& [5] GREEK CAPITAL LETTER OMICRON WITH VARIA..GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI +2126 ; Changes_When_Lowercased # L& OHM SIGN +212A..212B ; Changes_When_Lowercased # L& [2] KELVIN SIGN..ANGSTROM SIGN +2132 ; Changes_When_Lowercased # L& TURNED CAPITAL F +2160..216F ; Changes_When_Lowercased # Nl [16] ROMAN NUMERAL ONE..ROMAN NUMERAL ONE THOUSAND +2183 ; Changes_When_Lowercased # L& ROMAN NUMERAL REVERSED ONE HUNDRED +24B6..24CF ; Changes_When_Lowercased # So [26] CIRCLED LATIN CAPITAL LETTER A..CIRCLED LATIN CAPITAL LETTER Z +2C00..2C2E ; Changes_When_Lowercased # L& [47] GLAGOLITIC CAPITAL LETTER AZU..GLAGOLITIC CAPITAL LETTER LATINATE MYSLITE +2C60 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER L WITH DOUBLE BAR +2C62..2C64 ; Changes_When_Lowercased # L& [3] LATIN CAPITAL LETTER L WITH MIDDLE TILDE..LATIN CAPITAL LETTER R WITH TAIL +2C67 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER H WITH DESCENDER +2C69 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER K WITH DESCENDER +2C6B ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER Z WITH DESCENDER +2C6D..2C70 ; Changes_When_Lowercased # L& [4] LATIN CAPITAL LETTER ALPHA..LATIN CAPITAL LETTER TURNED ALPHA +2C72 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER W WITH HOOK +2C75 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER HALF H +2C7E..2C80 ; Changes_When_Lowercased # L& [3] LATIN CAPITAL LETTER S WITH SWASH TAIL..COPTIC CAPITAL LETTER ALFA +2C82 ; Changes_When_Lowercased # L& COPTIC CAPITAL LETTER VIDA +2C84 ; Changes_When_Lowercased # L& COPTIC CAPITAL LETTER GAMMA +2C86 ; Changes_When_Lowercased # L& COPTIC CAPITAL LETTER DALDA +2C88 ; Changes_When_Lowercased # L& COPTIC CAPITAL LETTER EIE +2C8A ; Changes_When_Lowercased # L& COPTIC CAPITAL LETTER SOU +2C8C ; Changes_When_Lowercased # L& COPTIC CAPITAL LETTER ZATA +2C8E ; Changes_When_Lowercased # L& COPTIC CAPITAL LETTER HATE +2C90 ; Changes_When_Lowercased # L& COPTIC CAPITAL LETTER THETHE +2C92 ; Changes_When_Lowercased # L& COPTIC CAPITAL LETTER IAUDA +2C94 ; Changes_When_Lowercased # L& COPTIC CAPITAL LETTER KAPA +2C96 ; Changes_When_Lowercased # L& COPTIC CAPITAL LETTER LAULA +2C98 ; Changes_When_Lowercased # L& COPTIC CAPITAL LETTER MI +2C9A ; Changes_When_Lowercased # L& COPTIC CAPITAL LETTER NI +2C9C ; Changes_When_Lowercased # L& COPTIC CAPITAL LETTER KSI +2C9E ; Changes_When_Lowercased # L& COPTIC CAPITAL LETTER O +2CA0 ; Changes_When_Lowercased # L& COPTIC CAPITAL LETTER PI +2CA2 ; Changes_When_Lowercased # L& COPTIC CAPITAL LETTER RO +2CA4 ; Changes_When_Lowercased # L& COPTIC CAPITAL LETTER SIMA +2CA6 ; Changes_When_Lowercased # L& COPTIC CAPITAL LETTER TAU +2CA8 ; Changes_When_Lowercased # L& COPTIC CAPITAL LETTER UA +2CAA ; Changes_When_Lowercased # L& COPTIC CAPITAL LETTER FI +2CAC ; Changes_When_Lowercased # L& COPTIC CAPITAL LETTER KHI +2CAE ; Changes_When_Lowercased # L& COPTIC CAPITAL LETTER PSI +2CB0 ; Changes_When_Lowercased # L& COPTIC CAPITAL LETTER OOU +2CB2 ; Changes_When_Lowercased # L& COPTIC CAPITAL LETTER DIALECT-P ALEF +2CB4 ; Changes_When_Lowercased # L& COPTIC CAPITAL LETTER OLD COPTIC AIN +2CB6 ; Changes_When_Lowercased # L& COPTIC CAPITAL LETTER CRYPTOGRAMMIC EIE +2CB8 ; Changes_When_Lowercased # L& COPTIC CAPITAL LETTER DIALECT-P KAPA +2CBA ; Changes_When_Lowercased # L& COPTIC CAPITAL LETTER DIALECT-P NI +2CBC ; Changes_When_Lowercased # L& COPTIC CAPITAL LETTER CRYPTOGRAMMIC NI +2CBE ; Changes_When_Lowercased # L& COPTIC CAPITAL LETTER OLD COPTIC OOU +2CC0 ; Changes_When_Lowercased # L& COPTIC CAPITAL LETTER SAMPI +2CC2 ; Changes_When_Lowercased # L& COPTIC CAPITAL LETTER CROSSED SHEI +2CC4 ; Changes_When_Lowercased # L& COPTIC CAPITAL LETTER OLD COPTIC SHEI +2CC6 ; Changes_When_Lowercased # L& COPTIC CAPITAL LETTER OLD COPTIC ESH +2CC8 ; Changes_When_Lowercased # L& COPTIC CAPITAL LETTER AKHMIMIC KHEI +2CCA ; Changes_When_Lowercased # L& COPTIC CAPITAL LETTER DIALECT-P HORI +2CCC ; Changes_When_Lowercased # L& COPTIC CAPITAL LETTER OLD COPTIC HORI +2CCE ; Changes_When_Lowercased # L& COPTIC CAPITAL LETTER OLD COPTIC HA +2CD0 ; Changes_When_Lowercased # L& COPTIC CAPITAL LETTER L-SHAPED HA +2CD2 ; Changes_When_Lowercased # L& COPTIC CAPITAL LETTER OLD COPTIC HEI +2CD4 ; Changes_When_Lowercased # L& COPTIC CAPITAL LETTER OLD COPTIC HAT +2CD6 ; Changes_When_Lowercased # L& COPTIC CAPITAL LETTER OLD COPTIC GANGIA +2CD8 ; Changes_When_Lowercased # L& COPTIC CAPITAL LETTER OLD COPTIC DJA +2CDA ; Changes_When_Lowercased # L& COPTIC CAPITAL LETTER OLD COPTIC SHIMA +2CDC ; Changes_When_Lowercased # L& COPTIC CAPITAL LETTER OLD NUBIAN SHIMA +2CDE ; Changes_When_Lowercased # L& COPTIC CAPITAL LETTER OLD NUBIAN NGI +2CE0 ; Changes_When_Lowercased # L& COPTIC CAPITAL LETTER OLD NUBIAN NYI +2CE2 ; Changes_When_Lowercased # L& COPTIC CAPITAL LETTER OLD NUBIAN WAU +2CEB ; Changes_When_Lowercased # L& COPTIC CAPITAL LETTER CRYPTOGRAMMIC SHEI +2CED ; Changes_When_Lowercased # L& COPTIC CAPITAL LETTER CRYPTOGRAMMIC GANGIA +A640 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER ZEMLYA +A642 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER DZELO +A644 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER REVERSED DZE +A646 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER IOTA +A648 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER DJERV +A64A ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER MONOGRAPH UK +A64C ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER BROAD OMEGA +A64E ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER NEUTRAL YER +A650 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER YERU WITH BACK YER +A652 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER IOTIFIED YAT +A654 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER REVERSED YU +A656 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER IOTIFIED A +A658 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER CLOSED LITTLE YUS +A65A ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER BLENDED YUS +A65C ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER IOTIFIED CLOSED LITTLE YUS +A65E ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER YN +A662 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER SOFT DE +A664 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER SOFT EL +A666 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER SOFT EM +A668 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER MONOCULAR O +A66A ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER BINOCULAR O +A66C ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER DOUBLE MONOCULAR O +A680 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER DWE +A682 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER DZWE +A684 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER ZHWE +A686 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER CCHE +A688 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER DZZE +A68A ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER TE WITH MIDDLE HOOK +A68C ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER TWE +A68E ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER TSWE +A690 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER TSSE +A692 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER TCHE +A694 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER HWE +A696 ; Changes_When_Lowercased # L& CYRILLIC CAPITAL LETTER SHWE +A722 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER EGYPTOLOGICAL ALEF +A724 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER EGYPTOLOGICAL AIN +A726 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER HENG +A728 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER TZ +A72A ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER TRESILLO +A72C ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER CUATRILLO +A72E ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER CUATRILLO WITH COMMA +A732 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER AA +A734 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER AO +A736 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER AU +A738 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER AV +A73A ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER AV WITH HORIZONTAL BAR +A73C ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER AY +A73E ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER REVERSED C WITH DOT +A740 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER K WITH STROKE +A742 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER K WITH DIAGONAL STROKE +A744 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER K WITH STROKE AND DIAGONAL STROKE +A746 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER BROKEN L +A748 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER L WITH HIGH STROKE +A74A ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER O WITH LONG STROKE OVERLAY +A74C ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER O WITH LOOP +A74E ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER OO +A750 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER P WITH STROKE THROUGH DESCENDER +A752 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER P WITH FLOURISH +A754 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER P WITH SQUIRREL TAIL +A756 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER Q WITH STROKE THROUGH DESCENDER +A758 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER Q WITH DIAGONAL STROKE +A75A ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER R ROTUNDA +A75C ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER RUM ROTUNDA +A75E ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER V WITH DIAGONAL STROKE +A760 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER VY +A762 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER VISIGOTHIC Z +A764 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER THORN WITH STROKE +A766 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER THORN WITH STROKE THROUGH DESCENDER +A768 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER VEND +A76A ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER ET +A76C ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER IS +A76E ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER CON +A779 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER INSULAR D +A77B ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER INSULAR F +A77D..A77E ; Changes_When_Lowercased # L& [2] LATIN CAPITAL LETTER INSULAR G..LATIN CAPITAL LETTER TURNED INSULAR G +A780 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER TURNED L +A782 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER INSULAR R +A784 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER INSULAR S +A786 ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER INSULAR T +A78B ; Changes_When_Lowercased # L& LATIN CAPITAL LETTER SALTILLO +FF21..FF3A ; Changes_When_Lowercased # L& [26] FULLWIDTH LATIN CAPITAL LETTER A..FULLWIDTH LATIN CAPITAL LETTER Z +10400..10427 ; Changes_When_Lowercased # L& [40] DESERET CAPITAL LETTER LONG I..DESERET CAPITAL LETTER EW + +# Total code points: 1029 + +# ================================================ + +# Derived Property: Changes_When_Uppercased (CWU) +# Characters whose normalized forms are not stable under a toUppercase mapping. +# For more information, see D125 in Section 3.13, "Default Case Algorithms". +# Changes_When_Uppercased(X) is true when toUppercase(toNFD(X)) != toNFD(X) + +0061..007A ; Changes_When_Uppercased # L& [26] LATIN SMALL LETTER A..LATIN SMALL LETTER Z +00B5 ; Changes_When_Uppercased # L& MICRO SIGN +00DF..00F6 ; Changes_When_Uppercased # L& [24] LATIN SMALL LETTER SHARP S..LATIN SMALL LETTER O WITH DIAERESIS +00F8..00FF ; Changes_When_Uppercased # L& [8] LATIN SMALL LETTER O WITH STROKE..LATIN SMALL LETTER Y WITH DIAERESIS +0101 ; Changes_When_Uppercased # L& LATIN SMALL LETTER A WITH MACRON +0103 ; Changes_When_Uppercased # L& LATIN SMALL LETTER A WITH BREVE +0105 ; Changes_When_Uppercased # L& LATIN SMALL LETTER A WITH OGONEK +0107 ; Changes_When_Uppercased # L& LATIN SMALL LETTER C WITH ACUTE +0109 ; Changes_When_Uppercased # L& LATIN SMALL LETTER C WITH CIRCUMFLEX +010B ; Changes_When_Uppercased # L& LATIN SMALL LETTER C WITH DOT ABOVE +010D ; Changes_When_Uppercased # L& LATIN SMALL LETTER C WITH CARON +010F ; Changes_When_Uppercased # L& LATIN SMALL LETTER D WITH CARON +0111 ; Changes_When_Uppercased # L& LATIN SMALL LETTER D WITH STROKE +0113 ; Changes_When_Uppercased # L& LATIN SMALL LETTER E WITH MACRON +0115 ; Changes_When_Uppercased # L& LATIN SMALL LETTER E WITH BREVE +0117 ; Changes_When_Uppercased # L& LATIN SMALL LETTER E WITH DOT ABOVE +0119 ; Changes_When_Uppercased # L& LATIN SMALL LETTER E WITH OGONEK +011B ; Changes_When_Uppercased # L& LATIN SMALL LETTER E WITH CARON +011D ; Changes_When_Uppercased # L& LATIN SMALL LETTER G WITH CIRCUMFLEX +011F ; Changes_When_Uppercased # L& LATIN SMALL LETTER G WITH BREVE +0121 ; Changes_When_Uppercased # L& LATIN SMALL LETTER G WITH DOT ABOVE +0123 ; Changes_When_Uppercased # L& LATIN SMALL LETTER G WITH CEDILLA +0125 ; Changes_When_Uppercased # L& LATIN SMALL LETTER H WITH CIRCUMFLEX +0127 ; Changes_When_Uppercased # L& LATIN SMALL LETTER H WITH STROKE +0129 ; Changes_When_Uppercased # L& LATIN SMALL LETTER I WITH TILDE +012B ; Changes_When_Uppercased # L& LATIN SMALL LETTER I WITH MACRON +012D ; Changes_When_Uppercased # L& LATIN SMALL LETTER I WITH BREVE +012F ; Changes_When_Uppercased # L& LATIN SMALL LETTER I WITH OGONEK +0131 ; Changes_When_Uppercased # L& LATIN SMALL LETTER DOTLESS I +0133 ; Changes_When_Uppercased # L& LATIN SMALL LIGATURE IJ +0135 ; Changes_When_Uppercased # L& LATIN SMALL LETTER J WITH CIRCUMFLEX +0137 ; Changes_When_Uppercased # L& LATIN SMALL LETTER K WITH CEDILLA +013A ; Changes_When_Uppercased # L& LATIN SMALL LETTER L WITH ACUTE +013C ; Changes_When_Uppercased # L& LATIN SMALL LETTER L WITH CEDILLA +013E ; Changes_When_Uppercased # L& LATIN SMALL LETTER L WITH CARON +0140 ; Changes_When_Uppercased # L& LATIN SMALL LETTER L WITH MIDDLE DOT +0142 ; Changes_When_Uppercased # L& LATIN SMALL LETTER L WITH STROKE +0144 ; Changes_When_Uppercased # L& LATIN SMALL LETTER N WITH ACUTE +0146 ; Changes_When_Uppercased # L& LATIN SMALL LETTER N WITH CEDILLA +0148..0149 ; Changes_When_Uppercased # L& [2] LATIN SMALL LETTER N WITH CARON..LATIN SMALL LETTER N PRECEDED BY APOSTROPHE +014B ; Changes_When_Uppercased # L& LATIN SMALL LETTER ENG +014D ; Changes_When_Uppercased # L& LATIN SMALL LETTER O WITH MACRON +014F ; Changes_When_Uppercased # L& LATIN SMALL LETTER O WITH BREVE +0151 ; Changes_When_Uppercased # L& LATIN SMALL LETTER O WITH DOUBLE ACUTE +0153 ; Changes_When_Uppercased # L& LATIN SMALL LIGATURE OE +0155 ; Changes_When_Uppercased # L& LATIN SMALL LETTER R WITH ACUTE +0157 ; Changes_When_Uppercased # L& LATIN SMALL LETTER R WITH CEDILLA +0159 ; Changes_When_Uppercased # L& LATIN SMALL LETTER R WITH CARON +015B ; Changes_When_Uppercased # L& LATIN SMALL LETTER S WITH ACUTE +015D ; Changes_When_Uppercased # L& LATIN SMALL LETTER S WITH CIRCUMFLEX +015F ; Changes_When_Uppercased # L& LATIN SMALL LETTER S WITH CEDILLA +0161 ; Changes_When_Uppercased # L& LATIN SMALL LETTER S WITH CARON +0163 ; Changes_When_Uppercased # L& LATIN SMALL LETTER T WITH CEDILLA +0165 ; Changes_When_Uppercased # L& LATIN SMALL LETTER T WITH CARON +0167 ; Changes_When_Uppercased # L& LATIN SMALL LETTER T WITH STROKE +0169 ; Changes_When_Uppercased # L& LATIN SMALL LETTER U WITH TILDE +016B ; Changes_When_Uppercased # L& LATIN SMALL LETTER U WITH MACRON +016D ; Changes_When_Uppercased # L& LATIN SMALL LETTER U WITH BREVE +016F ; Changes_When_Uppercased # L& LATIN SMALL LETTER U WITH RING ABOVE +0171 ; Changes_When_Uppercased # L& LATIN SMALL LETTER U WITH DOUBLE ACUTE +0173 ; Changes_When_Uppercased # L& LATIN SMALL LETTER U WITH OGONEK +0175 ; Changes_When_Uppercased # L& LATIN SMALL LETTER W WITH CIRCUMFLEX +0177 ; Changes_When_Uppercased # L& LATIN SMALL LETTER Y WITH CIRCUMFLEX +017A ; Changes_When_Uppercased # L& LATIN SMALL LETTER Z WITH ACUTE +017C ; Changes_When_Uppercased # L& LATIN SMALL LETTER Z WITH DOT ABOVE +017E..0180 ; Changes_When_Uppercased # L& [3] LATIN SMALL LETTER Z WITH CARON..LATIN SMALL LETTER B WITH STROKE +0183 ; Changes_When_Uppercased # L& LATIN SMALL LETTER B WITH TOPBAR +0185 ; Changes_When_Uppercased # L& LATIN SMALL LETTER TONE SIX +0188 ; Changes_When_Uppercased # L& LATIN SMALL LETTER C WITH HOOK +018C ; Changes_When_Uppercased # L& LATIN SMALL LETTER D WITH TOPBAR +0192 ; Changes_When_Uppercased # L& LATIN SMALL LETTER F WITH HOOK +0195 ; Changes_When_Uppercased # L& LATIN SMALL LETTER HV +0199..019A ; Changes_When_Uppercased # L& [2] LATIN SMALL LETTER K WITH HOOK..LATIN SMALL LETTER L WITH BAR +019E ; Changes_When_Uppercased # L& LATIN SMALL LETTER N WITH LONG RIGHT LEG +01A1 ; Changes_When_Uppercased # L& LATIN SMALL LETTER O WITH HORN +01A3 ; Changes_When_Uppercased # L& LATIN SMALL LETTER OI +01A5 ; Changes_When_Uppercased # L& LATIN SMALL LETTER P WITH HOOK +01A8 ; Changes_When_Uppercased # L& LATIN SMALL LETTER TONE TWO +01AD ; Changes_When_Uppercased # L& LATIN SMALL LETTER T WITH HOOK +01B0 ; Changes_When_Uppercased # L& LATIN SMALL LETTER U WITH HORN +01B4 ; Changes_When_Uppercased # L& LATIN SMALL LETTER Y WITH HOOK +01B6 ; Changes_When_Uppercased # L& LATIN SMALL LETTER Z WITH STROKE +01B9 ; Changes_When_Uppercased # L& LATIN SMALL LETTER EZH REVERSED +01BD ; Changes_When_Uppercased # L& LATIN SMALL LETTER TONE FIVE +01BF ; Changes_When_Uppercased # L& LATIN LETTER WYNN +01C5..01C6 ; Changes_When_Uppercased # L& [2] LATIN CAPITAL LETTER D WITH SMALL LETTER Z WITH CARON..LATIN SMALL LETTER DZ WITH CARON +01C8..01C9 ; Changes_When_Uppercased # L& [2] LATIN CAPITAL LETTER L WITH SMALL LETTER J..LATIN SMALL LETTER LJ +01CB..01CC ; Changes_When_Uppercased # L& [2] LATIN CAPITAL LETTER N WITH SMALL LETTER J..LATIN SMALL LETTER NJ +01CE ; Changes_When_Uppercased # L& LATIN SMALL LETTER A WITH CARON +01D0 ; Changes_When_Uppercased # L& LATIN SMALL LETTER I WITH CARON +01D2 ; Changes_When_Uppercased # L& LATIN SMALL LETTER O WITH CARON +01D4 ; Changes_When_Uppercased # L& LATIN SMALL LETTER U WITH CARON +01D6 ; Changes_When_Uppercased # L& LATIN SMALL LETTER U WITH DIAERESIS AND MACRON +01D8 ; Changes_When_Uppercased # L& LATIN SMALL LETTER U WITH DIAERESIS AND ACUTE +01DA ; Changes_When_Uppercased # L& LATIN SMALL LETTER U WITH DIAERESIS AND CARON +01DC..01DD ; Changes_When_Uppercased # L& [2] LATIN SMALL LETTER U WITH DIAERESIS AND GRAVE..LATIN SMALL LETTER TURNED E +01DF ; Changes_When_Uppercased # L& LATIN SMALL LETTER A WITH DIAERESIS AND MACRON +01E1 ; Changes_When_Uppercased # L& LATIN SMALL LETTER A WITH DOT ABOVE AND MACRON +01E3 ; Changes_When_Uppercased # L& LATIN SMALL LETTER AE WITH MACRON +01E5 ; Changes_When_Uppercased # L& LATIN SMALL LETTER G WITH STROKE +01E7 ; Changes_When_Uppercased # L& LATIN SMALL LETTER G WITH CARON +01E9 ; Changes_When_Uppercased # L& LATIN SMALL LETTER K WITH CARON +01EB ; Changes_When_Uppercased # L& LATIN SMALL LETTER O WITH OGONEK +01ED ; Changes_When_Uppercased # L& LATIN SMALL LETTER O WITH OGONEK AND MACRON +01EF..01F0 ; Changes_When_Uppercased # L& [2] LATIN SMALL LETTER EZH WITH CARON..LATIN SMALL LETTER J WITH CARON +01F2..01F3 ; Changes_When_Uppercased # L& [2] LATIN CAPITAL LETTER D WITH SMALL LETTER Z..LATIN SMALL LETTER DZ +01F5 ; Changes_When_Uppercased # L& LATIN SMALL LETTER G WITH ACUTE +01F9 ; Changes_When_Uppercased # L& LATIN SMALL LETTER N WITH GRAVE +01FB ; Changes_When_Uppercased # L& LATIN SMALL LETTER A WITH RING ABOVE AND ACUTE +01FD ; Changes_When_Uppercased # L& LATIN SMALL LETTER AE WITH ACUTE +01FF ; Changes_When_Uppercased # L& LATIN SMALL LETTER O WITH STROKE AND ACUTE +0201 ; Changes_When_Uppercased # L& LATIN SMALL LETTER A WITH DOUBLE GRAVE +0203 ; Changes_When_Uppercased # L& LATIN SMALL LETTER A WITH INVERTED BREVE +0205 ; Changes_When_Uppercased # L& LATIN SMALL LETTER E WITH DOUBLE GRAVE +0207 ; Changes_When_Uppercased # L& LATIN SMALL LETTER E WITH INVERTED BREVE +0209 ; Changes_When_Uppercased # L& LATIN SMALL LETTER I WITH DOUBLE GRAVE +020B ; Changes_When_Uppercased # L& LATIN SMALL LETTER I WITH INVERTED BREVE +020D ; Changes_When_Uppercased # L& LATIN SMALL LETTER O WITH DOUBLE GRAVE +020F ; Changes_When_Uppercased # L& LATIN SMALL LETTER O WITH INVERTED BREVE +0211 ; Changes_When_Uppercased # L& LATIN SMALL LETTER R WITH DOUBLE GRAVE +0213 ; Changes_When_Uppercased # L& LATIN SMALL LETTER R WITH INVERTED BREVE +0215 ; Changes_When_Uppercased # L& LATIN SMALL LETTER U WITH DOUBLE GRAVE +0217 ; Changes_When_Uppercased # L& LATIN SMALL LETTER U WITH INVERTED BREVE +0219 ; Changes_When_Uppercased # L& LATIN SMALL LETTER S WITH COMMA BELOW +021B ; Changes_When_Uppercased # L& LATIN SMALL LETTER T WITH COMMA BELOW +021D ; Changes_When_Uppercased # L& LATIN SMALL LETTER YOGH +021F ; Changes_When_Uppercased # L& LATIN SMALL LETTER H WITH CARON +0223 ; Changes_When_Uppercased # L& LATIN SMALL LETTER OU +0225 ; Changes_When_Uppercased # L& LATIN SMALL LETTER Z WITH HOOK +0227 ; Changes_When_Uppercased # L& LATIN SMALL LETTER A WITH DOT ABOVE +0229 ; Changes_When_Uppercased # L& LATIN SMALL LETTER E WITH CEDILLA +022B ; Changes_When_Uppercased # L& LATIN SMALL LETTER O WITH DIAERESIS AND MACRON +022D ; Changes_When_Uppercased # L& LATIN SMALL LETTER O WITH TILDE AND MACRON +022F ; Changes_When_Uppercased # L& LATIN SMALL LETTER O WITH DOT ABOVE +0231 ; Changes_When_Uppercased # L& LATIN SMALL LETTER O WITH DOT ABOVE AND MACRON +0233 ; Changes_When_Uppercased # L& LATIN SMALL LETTER Y WITH MACRON +023C ; Changes_When_Uppercased # L& LATIN SMALL LETTER C WITH STROKE +023F..0240 ; Changes_When_Uppercased # L& [2] LATIN SMALL LETTER S WITH SWASH TAIL..LATIN SMALL LETTER Z WITH SWASH TAIL +0242 ; Changes_When_Uppercased # L& LATIN SMALL LETTER GLOTTAL STOP +0247 ; Changes_When_Uppercased # L& LATIN SMALL LETTER E WITH STROKE +0249 ; Changes_When_Uppercased # L& LATIN SMALL LETTER J WITH STROKE +024B ; Changes_When_Uppercased # L& LATIN SMALL LETTER Q WITH HOOK TAIL +024D ; Changes_When_Uppercased # L& LATIN SMALL LETTER R WITH STROKE +024F..0254 ; Changes_When_Uppercased # L& [6] LATIN SMALL LETTER Y WITH STROKE..LATIN SMALL LETTER OPEN O +0256..0257 ; Changes_When_Uppercased # L& [2] LATIN SMALL LETTER D WITH TAIL..LATIN SMALL LETTER D WITH HOOK +0259 ; Changes_When_Uppercased # L& LATIN SMALL LETTER SCHWA +025B ; Changes_When_Uppercased # L& LATIN SMALL LETTER OPEN E +0260 ; Changes_When_Uppercased # L& LATIN SMALL LETTER G WITH HOOK +0263 ; Changes_When_Uppercased # L& LATIN SMALL LETTER GAMMA +0268..0269 ; Changes_When_Uppercased # L& [2] LATIN SMALL LETTER I WITH STROKE..LATIN SMALL LETTER IOTA +026B ; Changes_When_Uppercased # L& LATIN SMALL LETTER L WITH MIDDLE TILDE +026F ; Changes_When_Uppercased # L& LATIN SMALL LETTER TURNED M +0271..0272 ; Changes_When_Uppercased # L& [2] LATIN SMALL LETTER M WITH HOOK..LATIN SMALL LETTER N WITH LEFT HOOK +0275 ; Changes_When_Uppercased # L& LATIN SMALL LETTER BARRED O +027D ; Changes_When_Uppercased # L& LATIN SMALL LETTER R WITH TAIL +0280 ; Changes_When_Uppercased # L& LATIN LETTER SMALL CAPITAL R +0283 ; Changes_When_Uppercased # L& LATIN SMALL LETTER ESH +0288..028C ; Changes_When_Uppercased # L& [5] LATIN SMALL LETTER T WITH RETROFLEX HOOK..LATIN SMALL LETTER TURNED V +0292 ; Changes_When_Uppercased # L& LATIN SMALL LETTER EZH +0345 ; Changes_When_Uppercased # Mn COMBINING GREEK YPOGEGRAMMENI +0371 ; Changes_When_Uppercased # L& GREEK SMALL LETTER HETA +0373 ; Changes_When_Uppercased # L& GREEK SMALL LETTER ARCHAIC SAMPI +0377 ; Changes_When_Uppercased # L& GREEK SMALL LETTER PAMPHYLIAN DIGAMMA +037B..037D ; Changes_When_Uppercased # L& [3] GREEK SMALL REVERSED LUNATE SIGMA SYMBOL..GREEK SMALL REVERSED DOTTED LUNATE SIGMA SYMBOL +0390 ; Changes_When_Uppercased # L& GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS +03AC..03CE ; Changes_When_Uppercased # L& [35] GREEK SMALL LETTER ALPHA WITH TONOS..GREEK SMALL LETTER OMEGA WITH TONOS +03D0..03D1 ; Changes_When_Uppercased # L& [2] GREEK BETA SYMBOL..GREEK THETA SYMBOL +03D5..03D7 ; Changes_When_Uppercased # L& [3] GREEK PHI SYMBOL..GREEK KAI SYMBOL +03D9 ; Changes_When_Uppercased # L& GREEK SMALL LETTER ARCHAIC KOPPA +03DB ; Changes_When_Uppercased # L& GREEK SMALL LETTER STIGMA +03DD ; Changes_When_Uppercased # L& GREEK SMALL LETTER DIGAMMA +03DF ; Changes_When_Uppercased # L& GREEK SMALL LETTER KOPPA +03E1 ; Changes_When_Uppercased # L& GREEK SMALL LETTER SAMPI +03E3 ; Changes_When_Uppercased # L& COPTIC SMALL LETTER SHEI +03E5 ; Changes_When_Uppercased # L& COPTIC SMALL LETTER FEI +03E7 ; Changes_When_Uppercased # L& COPTIC SMALL LETTER KHEI +03E9 ; Changes_When_Uppercased # L& COPTIC SMALL LETTER HORI +03EB ; Changes_When_Uppercased # L& COPTIC SMALL LETTER GANGIA +03ED ; Changes_When_Uppercased # L& COPTIC SMALL LETTER SHIMA +03EF..03F2 ; Changes_When_Uppercased # L& [4] COPTIC SMALL LETTER DEI..GREEK LUNATE SIGMA SYMBOL +03F5 ; Changes_When_Uppercased # L& GREEK LUNATE EPSILON SYMBOL +03F8 ; Changes_When_Uppercased # L& GREEK SMALL LETTER SHO +03FB ; Changes_When_Uppercased # L& GREEK SMALL LETTER SAN +0430..045F ; Changes_When_Uppercased # L& [48] CYRILLIC SMALL LETTER A..CYRILLIC SMALL LETTER DZHE +0461 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER OMEGA +0463 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER YAT +0465 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER IOTIFIED E +0467 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER LITTLE YUS +0469 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER IOTIFIED LITTLE YUS +046B ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER BIG YUS +046D ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER IOTIFIED BIG YUS +046F ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER KSI +0471 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER PSI +0473 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER FITA +0475 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER IZHITSA +0477 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER IZHITSA WITH DOUBLE GRAVE ACCENT +0479 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER UK +047B ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER ROUND OMEGA +047D ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER OMEGA WITH TITLO +047F ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER OT +0481 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER KOPPA +048B ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER SHORT I WITH TAIL +048D ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER SEMISOFT SIGN +048F ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER ER WITH TICK +0491 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER GHE WITH UPTURN +0493 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER GHE WITH STROKE +0495 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER GHE WITH MIDDLE HOOK +0497 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER ZHE WITH DESCENDER +0499 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER ZE WITH DESCENDER +049B ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER KA WITH DESCENDER +049D ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER KA WITH VERTICAL STROKE +049F ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER KA WITH STROKE +04A1 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER BASHKIR KA +04A3 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER EN WITH DESCENDER +04A5 ; Changes_When_Uppercased # L& CYRILLIC SMALL LIGATURE EN GHE +04A7 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER PE WITH MIDDLE HOOK +04A9 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER ABKHASIAN HA +04AB ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER ES WITH DESCENDER +04AD ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER TE WITH DESCENDER +04AF ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER STRAIGHT U +04B1 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER STRAIGHT U WITH STROKE +04B3 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER HA WITH DESCENDER +04B5 ; Changes_When_Uppercased # L& CYRILLIC SMALL LIGATURE TE TSE +04B7 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER CHE WITH DESCENDER +04B9 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER CHE WITH VERTICAL STROKE +04BB ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER SHHA +04BD ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER ABKHASIAN CHE +04BF ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER ABKHASIAN CHE WITH DESCENDER +04C2 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER ZHE WITH BREVE +04C4 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER KA WITH HOOK +04C6 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER EL WITH TAIL +04C8 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER EN WITH HOOK +04CA ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER EN WITH TAIL +04CC ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER KHAKASSIAN CHE +04CE..04CF ; Changes_When_Uppercased # L& [2] CYRILLIC SMALL LETTER EM WITH TAIL..CYRILLIC SMALL LETTER PALOCHKA +04D1 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER A WITH BREVE +04D3 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER A WITH DIAERESIS +04D5 ; Changes_When_Uppercased # L& CYRILLIC SMALL LIGATURE A IE +04D7 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER IE WITH BREVE +04D9 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER SCHWA +04DB ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER SCHWA WITH DIAERESIS +04DD ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER ZHE WITH DIAERESIS +04DF ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER ZE WITH DIAERESIS +04E1 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER ABKHASIAN DZE +04E3 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER I WITH MACRON +04E5 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER I WITH DIAERESIS +04E7 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER O WITH DIAERESIS +04E9 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER BARRED O +04EB ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER BARRED O WITH DIAERESIS +04ED ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER E WITH DIAERESIS +04EF ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER U WITH MACRON +04F1 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER U WITH DIAERESIS +04F3 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER U WITH DOUBLE ACUTE +04F5 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER CHE WITH DIAERESIS +04F7 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER GHE WITH DESCENDER +04F9 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER YERU WITH DIAERESIS +04FB ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER GHE WITH STROKE AND HOOK +04FD ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER HA WITH HOOK +04FF ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER HA WITH STROKE +0501 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER KOMI DE +0503 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER KOMI DJE +0505 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER KOMI ZJE +0507 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER KOMI DZJE +0509 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER KOMI LJE +050B ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER KOMI NJE +050D ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER KOMI SJE +050F ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER KOMI TJE +0511 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER REVERSED ZE +0513 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER EL WITH HOOK +0515 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER LHA +0517 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER RHA +0519 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER YAE +051B ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER QA +051D ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER WE +051F ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER ALEUT KA +0521 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER EL WITH MIDDLE HOOK +0523 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER EN WITH MIDDLE HOOK +0525 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER PE WITH DESCENDER +0561..0587 ; Changes_When_Uppercased # L& [39] ARMENIAN SMALL LETTER AYB..ARMENIAN SMALL LIGATURE ECH YIWN +1D79 ; Changes_When_Uppercased # L& LATIN SMALL LETTER INSULAR G +1D7D ; Changes_When_Uppercased # L& LATIN SMALL LETTER P WITH STROKE +1E01 ; Changes_When_Uppercased # L& LATIN SMALL LETTER A WITH RING BELOW +1E03 ; Changes_When_Uppercased # L& LATIN SMALL LETTER B WITH DOT ABOVE +1E05 ; Changes_When_Uppercased # L& LATIN SMALL LETTER B WITH DOT BELOW +1E07 ; Changes_When_Uppercased # L& LATIN SMALL LETTER B WITH LINE BELOW +1E09 ; Changes_When_Uppercased # L& LATIN SMALL LETTER C WITH CEDILLA AND ACUTE +1E0B ; Changes_When_Uppercased # L& LATIN SMALL LETTER D WITH DOT ABOVE +1E0D ; Changes_When_Uppercased # L& LATIN SMALL LETTER D WITH DOT BELOW +1E0F ; Changes_When_Uppercased # L& LATIN SMALL LETTER D WITH LINE BELOW +1E11 ; Changes_When_Uppercased # L& LATIN SMALL LETTER D WITH CEDILLA +1E13 ; Changes_When_Uppercased # L& LATIN SMALL LETTER D WITH CIRCUMFLEX BELOW +1E15 ; Changes_When_Uppercased # L& LATIN SMALL LETTER E WITH MACRON AND GRAVE +1E17 ; Changes_When_Uppercased # L& LATIN SMALL LETTER E WITH MACRON AND ACUTE +1E19 ; Changes_When_Uppercased # L& LATIN SMALL LETTER E WITH CIRCUMFLEX BELOW +1E1B ; Changes_When_Uppercased # L& LATIN SMALL LETTER E WITH TILDE BELOW +1E1D ; Changes_When_Uppercased # L& LATIN SMALL LETTER E WITH CEDILLA AND BREVE +1E1F ; Changes_When_Uppercased # L& LATIN SMALL LETTER F WITH DOT ABOVE +1E21 ; Changes_When_Uppercased # L& LATIN SMALL LETTER G WITH MACRON +1E23 ; Changes_When_Uppercased # L& LATIN SMALL LETTER H WITH DOT ABOVE +1E25 ; Changes_When_Uppercased # L& LATIN SMALL LETTER H WITH DOT BELOW +1E27 ; Changes_When_Uppercased # L& LATIN SMALL LETTER H WITH DIAERESIS +1E29 ; Changes_When_Uppercased # L& LATIN SMALL LETTER H WITH CEDILLA +1E2B ; Changes_When_Uppercased # L& LATIN SMALL LETTER H WITH BREVE BELOW +1E2D ; Changes_When_Uppercased # L& LATIN SMALL LETTER I WITH TILDE BELOW +1E2F ; Changes_When_Uppercased # L& LATIN SMALL LETTER I WITH DIAERESIS AND ACUTE +1E31 ; Changes_When_Uppercased # L& LATIN SMALL LETTER K WITH ACUTE +1E33 ; Changes_When_Uppercased # L& LATIN SMALL LETTER K WITH DOT BELOW +1E35 ; Changes_When_Uppercased # L& LATIN SMALL LETTER K WITH LINE BELOW +1E37 ; Changes_When_Uppercased # L& LATIN SMALL LETTER L WITH DOT BELOW +1E39 ; Changes_When_Uppercased # L& LATIN SMALL LETTER L WITH DOT BELOW AND MACRON +1E3B ; Changes_When_Uppercased # L& LATIN SMALL LETTER L WITH LINE BELOW +1E3D ; Changes_When_Uppercased # L& LATIN SMALL LETTER L WITH CIRCUMFLEX BELOW +1E3F ; Changes_When_Uppercased # L& LATIN SMALL LETTER M WITH ACUTE +1E41 ; Changes_When_Uppercased # L& LATIN SMALL LETTER M WITH DOT ABOVE +1E43 ; Changes_When_Uppercased # L& LATIN SMALL LETTER M WITH DOT BELOW +1E45 ; Changes_When_Uppercased # L& LATIN SMALL LETTER N WITH DOT ABOVE +1E47 ; Changes_When_Uppercased # L& LATIN SMALL LETTER N WITH DOT BELOW +1E49 ; Changes_When_Uppercased # L& LATIN SMALL LETTER N WITH LINE BELOW +1E4B ; Changes_When_Uppercased # L& LATIN SMALL LETTER N WITH CIRCUMFLEX BELOW +1E4D ; Changes_When_Uppercased # L& LATIN SMALL LETTER O WITH TILDE AND ACUTE +1E4F ; Changes_When_Uppercased # L& LATIN SMALL LETTER O WITH TILDE AND DIAERESIS +1E51 ; Changes_When_Uppercased # L& LATIN SMALL LETTER O WITH MACRON AND GRAVE +1E53 ; Changes_When_Uppercased # L& LATIN SMALL LETTER O WITH MACRON AND ACUTE +1E55 ; Changes_When_Uppercased # L& LATIN SMALL LETTER P WITH ACUTE +1E57 ; Changes_When_Uppercased # L& LATIN SMALL LETTER P WITH DOT ABOVE +1E59 ; Changes_When_Uppercased # L& LATIN SMALL LETTER R WITH DOT ABOVE +1E5B ; Changes_When_Uppercased # L& LATIN SMALL LETTER R WITH DOT BELOW +1E5D ; Changes_When_Uppercased # L& LATIN SMALL LETTER R WITH DOT BELOW AND MACRON +1E5F ; Changes_When_Uppercased # L& LATIN SMALL LETTER R WITH LINE BELOW +1E61 ; Changes_When_Uppercased # L& LATIN SMALL LETTER S WITH DOT ABOVE +1E63 ; Changes_When_Uppercased # L& LATIN SMALL LETTER S WITH DOT BELOW +1E65 ; Changes_When_Uppercased # L& LATIN SMALL LETTER S WITH ACUTE AND DOT ABOVE +1E67 ; Changes_When_Uppercased # L& LATIN SMALL LETTER S WITH CARON AND DOT ABOVE +1E69 ; Changes_When_Uppercased # L& LATIN SMALL LETTER S WITH DOT BELOW AND DOT ABOVE +1E6B ; Changes_When_Uppercased # L& LATIN SMALL LETTER T WITH DOT ABOVE +1E6D ; Changes_When_Uppercased # L& LATIN SMALL LETTER T WITH DOT BELOW +1E6F ; Changes_When_Uppercased # L& LATIN SMALL LETTER T WITH LINE BELOW +1E71 ; Changes_When_Uppercased # L& LATIN SMALL LETTER T WITH CIRCUMFLEX BELOW +1E73 ; Changes_When_Uppercased # L& LATIN SMALL LETTER U WITH DIAERESIS BELOW +1E75 ; Changes_When_Uppercased # L& LATIN SMALL LETTER U WITH TILDE BELOW +1E77 ; Changes_When_Uppercased # L& LATIN SMALL LETTER U WITH CIRCUMFLEX BELOW +1E79 ; Changes_When_Uppercased # L& LATIN SMALL LETTER U WITH TILDE AND ACUTE +1E7B ; Changes_When_Uppercased # L& LATIN SMALL LETTER U WITH MACRON AND DIAERESIS +1E7D ; Changes_When_Uppercased # L& LATIN SMALL LETTER V WITH TILDE +1E7F ; Changes_When_Uppercased # L& LATIN SMALL LETTER V WITH DOT BELOW +1E81 ; Changes_When_Uppercased # L& LATIN SMALL LETTER W WITH GRAVE +1E83 ; Changes_When_Uppercased # L& LATIN SMALL LETTER W WITH ACUTE +1E85 ; Changes_When_Uppercased # L& LATIN SMALL LETTER W WITH DIAERESIS +1E87 ; Changes_When_Uppercased # L& LATIN SMALL LETTER W WITH DOT ABOVE +1E89 ; Changes_When_Uppercased # L& LATIN SMALL LETTER W WITH DOT BELOW +1E8B ; Changes_When_Uppercased # L& LATIN SMALL LETTER X WITH DOT ABOVE +1E8D ; Changes_When_Uppercased # L& LATIN SMALL LETTER X WITH DIAERESIS +1E8F ; Changes_When_Uppercased # L& LATIN SMALL LETTER Y WITH DOT ABOVE +1E91 ; Changes_When_Uppercased # L& LATIN SMALL LETTER Z WITH CIRCUMFLEX +1E93 ; Changes_When_Uppercased # L& LATIN SMALL LETTER Z WITH DOT BELOW +1E95..1E9B ; Changes_When_Uppercased # L& [7] LATIN SMALL LETTER Z WITH LINE BELOW..LATIN SMALL LETTER LONG S WITH DOT ABOVE +1EA1 ; Changes_When_Uppercased # L& LATIN SMALL LETTER A WITH DOT BELOW +1EA3 ; Changes_When_Uppercased # L& LATIN SMALL LETTER A WITH HOOK ABOVE +1EA5 ; Changes_When_Uppercased # L& LATIN SMALL LETTER A WITH CIRCUMFLEX AND ACUTE +1EA7 ; Changes_When_Uppercased # L& LATIN SMALL LETTER A WITH CIRCUMFLEX AND GRAVE +1EA9 ; Changes_When_Uppercased # L& LATIN SMALL LETTER A WITH CIRCUMFLEX AND HOOK ABOVE +1EAB ; Changes_When_Uppercased # L& LATIN SMALL LETTER A WITH CIRCUMFLEX AND TILDE +1EAD ; Changes_When_Uppercased # L& LATIN SMALL LETTER A WITH CIRCUMFLEX AND DOT BELOW +1EAF ; Changes_When_Uppercased # L& LATIN SMALL LETTER A WITH BREVE AND ACUTE +1EB1 ; Changes_When_Uppercased # L& LATIN SMALL LETTER A WITH BREVE AND GRAVE +1EB3 ; Changes_When_Uppercased # L& LATIN SMALL LETTER A WITH BREVE AND HOOK ABOVE +1EB5 ; Changes_When_Uppercased # L& LATIN SMALL LETTER A WITH BREVE AND TILDE +1EB7 ; Changes_When_Uppercased # L& LATIN SMALL LETTER A WITH BREVE AND DOT BELOW +1EB9 ; Changes_When_Uppercased # L& LATIN SMALL LETTER E WITH DOT BELOW +1EBB ; Changes_When_Uppercased # L& LATIN SMALL LETTER E WITH HOOK ABOVE +1EBD ; Changes_When_Uppercased # L& LATIN SMALL LETTER E WITH TILDE +1EBF ; Changes_When_Uppercased # L& LATIN SMALL LETTER E WITH CIRCUMFLEX AND ACUTE +1EC1 ; Changes_When_Uppercased # L& LATIN SMALL LETTER E WITH CIRCUMFLEX AND GRAVE +1EC3 ; Changes_When_Uppercased # L& LATIN SMALL LETTER E WITH CIRCUMFLEX AND HOOK ABOVE +1EC5 ; Changes_When_Uppercased # L& LATIN SMALL LETTER E WITH CIRCUMFLEX AND TILDE +1EC7 ; Changes_When_Uppercased # L& LATIN SMALL LETTER E WITH CIRCUMFLEX AND DOT BELOW +1EC9 ; Changes_When_Uppercased # L& LATIN SMALL LETTER I WITH HOOK ABOVE +1ECB ; Changes_When_Uppercased # L& LATIN SMALL LETTER I WITH DOT BELOW +1ECD ; Changes_When_Uppercased # L& LATIN SMALL LETTER O WITH DOT BELOW +1ECF ; Changes_When_Uppercased # L& LATIN SMALL LETTER O WITH HOOK ABOVE +1ED1 ; Changes_When_Uppercased # L& LATIN SMALL LETTER O WITH CIRCUMFLEX AND ACUTE +1ED3 ; Changes_When_Uppercased # L& LATIN SMALL LETTER O WITH CIRCUMFLEX AND GRAVE +1ED5 ; Changes_When_Uppercased # L& LATIN SMALL LETTER O WITH CIRCUMFLEX AND HOOK ABOVE +1ED7 ; Changes_When_Uppercased # L& LATIN SMALL LETTER O WITH CIRCUMFLEX AND TILDE +1ED9 ; Changes_When_Uppercased # L& LATIN SMALL LETTER O WITH CIRCUMFLEX AND DOT BELOW +1EDB ; Changes_When_Uppercased # L& LATIN SMALL LETTER O WITH HORN AND ACUTE +1EDD ; Changes_When_Uppercased # L& LATIN SMALL LETTER O WITH HORN AND GRAVE +1EDF ; Changes_When_Uppercased # L& LATIN SMALL LETTER O WITH HORN AND HOOK ABOVE +1EE1 ; Changes_When_Uppercased # L& LATIN SMALL LETTER O WITH HORN AND TILDE +1EE3 ; Changes_When_Uppercased # L& LATIN SMALL LETTER O WITH HORN AND DOT BELOW +1EE5 ; Changes_When_Uppercased # L& LATIN SMALL LETTER U WITH DOT BELOW +1EE7 ; Changes_When_Uppercased # L& LATIN SMALL LETTER U WITH HOOK ABOVE +1EE9 ; Changes_When_Uppercased # L& LATIN SMALL LETTER U WITH HORN AND ACUTE +1EEB ; Changes_When_Uppercased # L& LATIN SMALL LETTER U WITH HORN AND GRAVE +1EED ; Changes_When_Uppercased # L& LATIN SMALL LETTER U WITH HORN AND HOOK ABOVE +1EEF ; Changes_When_Uppercased # L& LATIN SMALL LETTER U WITH HORN AND TILDE +1EF1 ; Changes_When_Uppercased # L& LATIN SMALL LETTER U WITH HORN AND DOT BELOW +1EF3 ; Changes_When_Uppercased # L& LATIN SMALL LETTER Y WITH GRAVE +1EF5 ; Changes_When_Uppercased # L& LATIN SMALL LETTER Y WITH DOT BELOW +1EF7 ; Changes_When_Uppercased # L& LATIN SMALL LETTER Y WITH HOOK ABOVE +1EF9 ; Changes_When_Uppercased # L& LATIN SMALL LETTER Y WITH TILDE +1EFB ; Changes_When_Uppercased # L& LATIN SMALL LETTER MIDDLE-WELSH LL +1EFD ; Changes_When_Uppercased # L& LATIN SMALL LETTER MIDDLE-WELSH V +1EFF..1F07 ; Changes_When_Uppercased # L& [9] LATIN SMALL LETTER Y WITH LOOP..GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI +1F10..1F15 ; Changes_When_Uppercased # L& [6] GREEK SMALL LETTER EPSILON WITH PSILI..GREEK SMALL LETTER EPSILON WITH DASIA AND OXIA +1F20..1F27 ; Changes_When_Uppercased # L& [8] GREEK SMALL LETTER ETA WITH PSILI..GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI +1F30..1F37 ; Changes_When_Uppercased # L& [8] GREEK SMALL LETTER IOTA WITH PSILI..GREEK SMALL LETTER IOTA WITH DASIA AND PERISPOMENI +1F40..1F45 ; Changes_When_Uppercased # L& [6] GREEK SMALL LETTER OMICRON WITH PSILI..GREEK SMALL LETTER OMICRON WITH DASIA AND OXIA +1F50..1F57 ; Changes_When_Uppercased # L& [8] GREEK SMALL LETTER UPSILON WITH PSILI..GREEK SMALL LETTER UPSILON WITH DASIA AND PERISPOMENI +1F60..1F67 ; Changes_When_Uppercased # L& [8] GREEK SMALL LETTER OMEGA WITH PSILI..GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI +1F70..1F7D ; Changes_When_Uppercased # L& [14] GREEK SMALL LETTER ALPHA WITH VARIA..GREEK SMALL LETTER OMEGA WITH OXIA +1F80..1FB4 ; Changes_When_Uppercased # L& [53] GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI..GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI +1FB6..1FB7 ; Changes_When_Uppercased # L& [2] GREEK SMALL LETTER ALPHA WITH PERISPOMENI..GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI +1FBC ; Changes_When_Uppercased # L& GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI +1FBE ; Changes_When_Uppercased # L& GREEK PROSGEGRAMMENI +1FC2..1FC4 ; Changes_When_Uppercased # L& [3] GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI..GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI +1FC6..1FC7 ; Changes_When_Uppercased # L& [2] GREEK SMALL LETTER ETA WITH PERISPOMENI..GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI +1FCC ; Changes_When_Uppercased # L& GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI +1FD0..1FD3 ; Changes_When_Uppercased # L& [4] GREEK SMALL LETTER IOTA WITH VRACHY..GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA +1FD6..1FD7 ; Changes_When_Uppercased # L& [2] GREEK SMALL LETTER IOTA WITH PERISPOMENI..GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI +1FE0..1FE7 ; Changes_When_Uppercased # L& [8] GREEK SMALL LETTER UPSILON WITH VRACHY..GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI +1FF2..1FF4 ; Changes_When_Uppercased # L& [3] GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI..GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI +1FF6..1FF7 ; Changes_When_Uppercased # L& [2] GREEK SMALL LETTER OMEGA WITH PERISPOMENI..GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI +1FFC ; Changes_When_Uppercased # L& GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI +214E ; Changes_When_Uppercased # L& TURNED SMALL F +2170..217F ; Changes_When_Uppercased # Nl [16] SMALL ROMAN NUMERAL ONE..SMALL ROMAN NUMERAL ONE THOUSAND +2184 ; Changes_When_Uppercased # L& LATIN SMALL LETTER REVERSED C +24D0..24E9 ; Changes_When_Uppercased # So [26] CIRCLED LATIN SMALL LETTER A..CIRCLED LATIN SMALL LETTER Z +2C30..2C5E ; Changes_When_Uppercased # L& [47] GLAGOLITIC SMALL LETTER AZU..GLAGOLITIC SMALL LETTER LATINATE MYSLITE +2C61 ; Changes_When_Uppercased # L& LATIN SMALL LETTER L WITH DOUBLE BAR +2C65..2C66 ; Changes_When_Uppercased # L& [2] LATIN SMALL LETTER A WITH STROKE..LATIN SMALL LETTER T WITH DIAGONAL STROKE +2C68 ; Changes_When_Uppercased # L& LATIN SMALL LETTER H WITH DESCENDER +2C6A ; Changes_When_Uppercased # L& LATIN SMALL LETTER K WITH DESCENDER +2C6C ; Changes_When_Uppercased # L& LATIN SMALL LETTER Z WITH DESCENDER +2C73 ; Changes_When_Uppercased # L& LATIN SMALL LETTER W WITH HOOK +2C76 ; Changes_When_Uppercased # L& LATIN SMALL LETTER HALF H +2C81 ; Changes_When_Uppercased # L& COPTIC SMALL LETTER ALFA +2C83 ; Changes_When_Uppercased # L& COPTIC SMALL LETTER VIDA +2C85 ; Changes_When_Uppercased # L& COPTIC SMALL LETTER GAMMA +2C87 ; Changes_When_Uppercased # L& COPTIC SMALL LETTER DALDA +2C89 ; Changes_When_Uppercased # L& COPTIC SMALL LETTER EIE +2C8B ; Changes_When_Uppercased # L& COPTIC SMALL LETTER SOU +2C8D ; Changes_When_Uppercased # L& COPTIC SMALL LETTER ZATA +2C8F ; Changes_When_Uppercased # L& COPTIC SMALL LETTER HATE +2C91 ; Changes_When_Uppercased # L& COPTIC SMALL LETTER THETHE +2C93 ; Changes_When_Uppercased # L& COPTIC SMALL LETTER IAUDA +2C95 ; Changes_When_Uppercased # L& COPTIC SMALL LETTER KAPA +2C97 ; Changes_When_Uppercased # L& COPTIC SMALL LETTER LAULA +2C99 ; Changes_When_Uppercased # L& COPTIC SMALL LETTER MI +2C9B ; Changes_When_Uppercased # L& COPTIC SMALL LETTER NI +2C9D ; Changes_When_Uppercased # L& COPTIC SMALL LETTER KSI +2C9F ; Changes_When_Uppercased # L& COPTIC SMALL LETTER O +2CA1 ; Changes_When_Uppercased # L& COPTIC SMALL LETTER PI +2CA3 ; Changes_When_Uppercased # L& COPTIC SMALL LETTER RO +2CA5 ; Changes_When_Uppercased # L& COPTIC SMALL LETTER SIMA +2CA7 ; Changes_When_Uppercased # L& COPTIC SMALL LETTER TAU +2CA9 ; Changes_When_Uppercased # L& COPTIC SMALL LETTER UA +2CAB ; Changes_When_Uppercased # L& COPTIC SMALL LETTER FI +2CAD ; Changes_When_Uppercased # L& COPTIC SMALL LETTER KHI +2CAF ; Changes_When_Uppercased # L& COPTIC SMALL LETTER PSI +2CB1 ; Changes_When_Uppercased # L& COPTIC SMALL LETTER OOU +2CB3 ; Changes_When_Uppercased # L& COPTIC SMALL LETTER DIALECT-P ALEF +2CB5 ; Changes_When_Uppercased # L& COPTIC SMALL LETTER OLD COPTIC AIN +2CB7 ; Changes_When_Uppercased # L& COPTIC SMALL LETTER CRYPTOGRAMMIC EIE +2CB9 ; Changes_When_Uppercased # L& COPTIC SMALL LETTER DIALECT-P KAPA +2CBB ; Changes_When_Uppercased # L& COPTIC SMALL LETTER DIALECT-P NI +2CBD ; Changes_When_Uppercased # L& COPTIC SMALL LETTER CRYPTOGRAMMIC NI +2CBF ; Changes_When_Uppercased # L& COPTIC SMALL LETTER OLD COPTIC OOU +2CC1 ; Changes_When_Uppercased # L& COPTIC SMALL LETTER SAMPI +2CC3 ; Changes_When_Uppercased # L& COPTIC SMALL LETTER CROSSED SHEI +2CC5 ; Changes_When_Uppercased # L& COPTIC SMALL LETTER OLD COPTIC SHEI +2CC7 ; Changes_When_Uppercased # L& COPTIC SMALL LETTER OLD COPTIC ESH +2CC9 ; Changes_When_Uppercased # L& COPTIC SMALL LETTER AKHMIMIC KHEI +2CCB ; Changes_When_Uppercased # L& COPTIC SMALL LETTER DIALECT-P HORI +2CCD ; Changes_When_Uppercased # L& COPTIC SMALL LETTER OLD COPTIC HORI +2CCF ; Changes_When_Uppercased # L& COPTIC SMALL LETTER OLD COPTIC HA +2CD1 ; Changes_When_Uppercased # L& COPTIC SMALL LETTER L-SHAPED HA +2CD3 ; Changes_When_Uppercased # L& COPTIC SMALL LETTER OLD COPTIC HEI +2CD5 ; Changes_When_Uppercased # L& COPTIC SMALL LETTER OLD COPTIC HAT +2CD7 ; Changes_When_Uppercased # L& COPTIC SMALL LETTER OLD COPTIC GANGIA +2CD9 ; Changes_When_Uppercased # L& COPTIC SMALL LETTER OLD COPTIC DJA +2CDB ; Changes_When_Uppercased # L& COPTIC SMALL LETTER OLD COPTIC SHIMA +2CDD ; Changes_When_Uppercased # L& COPTIC SMALL LETTER OLD NUBIAN SHIMA +2CDF ; Changes_When_Uppercased # L& COPTIC SMALL LETTER OLD NUBIAN NGI +2CE1 ; Changes_When_Uppercased # L& COPTIC SMALL LETTER OLD NUBIAN NYI +2CE3 ; Changes_When_Uppercased # L& COPTIC SMALL LETTER OLD NUBIAN WAU +2CEC ; Changes_When_Uppercased # L& COPTIC SMALL LETTER CRYPTOGRAMMIC SHEI +2CEE ; Changes_When_Uppercased # L& COPTIC SMALL LETTER CRYPTOGRAMMIC GANGIA +2D00..2D25 ; Changes_When_Uppercased # L& [38] GEORGIAN SMALL LETTER AN..GEORGIAN SMALL LETTER HOE +A641 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER ZEMLYA +A643 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER DZELO +A645 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER REVERSED DZE +A647 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER IOTA +A649 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER DJERV +A64B ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER MONOGRAPH UK +A64D ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER BROAD OMEGA +A64F ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER NEUTRAL YER +A651 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER YERU WITH BACK YER +A653 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER IOTIFIED YAT +A655 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER REVERSED YU +A657 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER IOTIFIED A +A659 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER CLOSED LITTLE YUS +A65B ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER BLENDED YUS +A65D ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER IOTIFIED CLOSED LITTLE YUS +A65F ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER YN +A663 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER SOFT DE +A665 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER SOFT EL +A667 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER SOFT EM +A669 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER MONOCULAR O +A66B ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER BINOCULAR O +A66D ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER DOUBLE MONOCULAR O +A681 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER DWE +A683 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER DZWE +A685 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER ZHWE +A687 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER CCHE +A689 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER DZZE +A68B ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER TE WITH MIDDLE HOOK +A68D ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER TWE +A68F ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER TSWE +A691 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER TSSE +A693 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER TCHE +A695 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER HWE +A697 ; Changes_When_Uppercased # L& CYRILLIC SMALL LETTER SHWE +A723 ; Changes_When_Uppercased # L& LATIN SMALL LETTER EGYPTOLOGICAL ALEF +A725 ; Changes_When_Uppercased # L& LATIN SMALL LETTER EGYPTOLOGICAL AIN +A727 ; Changes_When_Uppercased # L& LATIN SMALL LETTER HENG +A729 ; Changes_When_Uppercased # L& LATIN SMALL LETTER TZ +A72B ; Changes_When_Uppercased # L& LATIN SMALL LETTER TRESILLO +A72D ; Changes_When_Uppercased # L& LATIN SMALL LETTER CUATRILLO +A72F ; Changes_When_Uppercased # L& LATIN SMALL LETTER CUATRILLO WITH COMMA +A733 ; Changes_When_Uppercased # L& LATIN SMALL LETTER AA +A735 ; Changes_When_Uppercased # L& LATIN SMALL LETTER AO +A737 ; Changes_When_Uppercased # L& LATIN SMALL LETTER AU +A739 ; Changes_When_Uppercased # L& LATIN SMALL LETTER AV +A73B ; Changes_When_Uppercased # L& LATIN SMALL LETTER AV WITH HORIZONTAL BAR +A73D ; Changes_When_Uppercased # L& LATIN SMALL LETTER AY +A73F ; Changes_When_Uppercased # L& LATIN SMALL LETTER REVERSED C WITH DOT +A741 ; Changes_When_Uppercased # L& LATIN SMALL LETTER K WITH STROKE +A743 ; Changes_When_Uppercased # L& LATIN SMALL LETTER K WITH DIAGONAL STROKE +A745 ; Changes_When_Uppercased # L& LATIN SMALL LETTER K WITH STROKE AND DIAGONAL STROKE +A747 ; Changes_When_Uppercased # L& LATIN SMALL LETTER BROKEN L +A749 ; Changes_When_Uppercased # L& LATIN SMALL LETTER L WITH HIGH STROKE +A74B ; Changes_When_Uppercased # L& LATIN SMALL LETTER O WITH LONG STROKE OVERLAY +A74D ; Changes_When_Uppercased # L& LATIN SMALL LETTER O WITH LOOP +A74F ; Changes_When_Uppercased # L& LATIN SMALL LETTER OO +A751 ; Changes_When_Uppercased # L& LATIN SMALL LETTER P WITH STROKE THROUGH DESCENDER +A753 ; Changes_When_Uppercased # L& LATIN SMALL LETTER P WITH FLOURISH +A755 ; Changes_When_Uppercased # L& LATIN SMALL LETTER P WITH SQUIRREL TAIL +A757 ; Changes_When_Uppercased # L& LATIN SMALL LETTER Q WITH STROKE THROUGH DESCENDER +A759 ; Changes_When_Uppercased # L& LATIN SMALL LETTER Q WITH DIAGONAL STROKE +A75B ; Changes_When_Uppercased # L& LATIN SMALL LETTER R ROTUNDA +A75D ; Changes_When_Uppercased # L& LATIN SMALL LETTER RUM ROTUNDA +A75F ; Changes_When_Uppercased # L& LATIN SMALL LETTER V WITH DIAGONAL STROKE +A761 ; Changes_When_Uppercased # L& LATIN SMALL LETTER VY +A763 ; Changes_When_Uppercased # L& LATIN SMALL LETTER VISIGOTHIC Z +A765 ; Changes_When_Uppercased # L& LATIN SMALL LETTER THORN WITH STROKE +A767 ; Changes_When_Uppercased # L& LATIN SMALL LETTER THORN WITH STROKE THROUGH DESCENDER +A769 ; Changes_When_Uppercased # L& LATIN SMALL LETTER VEND +A76B ; Changes_When_Uppercased # L& LATIN SMALL LETTER ET +A76D ; Changes_When_Uppercased # L& LATIN SMALL LETTER IS +A76F ; Changes_When_Uppercased # L& LATIN SMALL LETTER CON +A77A ; Changes_When_Uppercased # L& LATIN SMALL LETTER INSULAR D +A77C ; Changes_When_Uppercased # L& LATIN SMALL LETTER INSULAR F +A77F ; Changes_When_Uppercased # L& LATIN SMALL LETTER TURNED INSULAR G +A781 ; Changes_When_Uppercased # L& LATIN SMALL LETTER TURNED L +A783 ; Changes_When_Uppercased # L& LATIN SMALL LETTER INSULAR R +A785 ; Changes_When_Uppercased # L& LATIN SMALL LETTER INSULAR S +A787 ; Changes_When_Uppercased # L& LATIN SMALL LETTER INSULAR T +A78C ; Changes_When_Uppercased # L& LATIN SMALL LETTER SALTILLO +FB00..FB06 ; Changes_When_Uppercased # L& [7] LATIN SMALL LIGATURE FF..LATIN SMALL LIGATURE ST +FB13..FB17 ; Changes_When_Uppercased # L& [5] ARMENIAN SMALL LIGATURE MEN NOW..ARMENIAN SMALL LIGATURE MEN XEH +FF41..FF5A ; Changes_When_Uppercased # L& [26] FULLWIDTH LATIN SMALL LETTER A..FULLWIDTH LATIN SMALL LETTER Z +10428..1044F ; Changes_When_Uppercased # L& [40] DESERET SMALL LETTER LONG I..DESERET SMALL LETTER EW + +# Total code points: 1112 + +# ================================================ + +# Derived Property: Changes_When_Titlecased (CWT) +# Characters whose normalized forms are not stable under a toTitlecase mapping. +# For more information, see D126 in Section 3.13, "Default Case Algorithms". +# Changes_When_Titlecased(X) is true when toTitlecase(toNFD(X)) != toNFD(X) + +0061..007A ; Changes_When_Titlecased # L& [26] LATIN SMALL LETTER A..LATIN SMALL LETTER Z +00B5 ; Changes_When_Titlecased # L& MICRO SIGN +00DF..00F6 ; Changes_When_Titlecased # L& [24] LATIN SMALL LETTER SHARP S..LATIN SMALL LETTER O WITH DIAERESIS +00F8..00FF ; Changes_When_Titlecased # L& [8] LATIN SMALL LETTER O WITH STROKE..LATIN SMALL LETTER Y WITH DIAERESIS +0101 ; Changes_When_Titlecased # L& LATIN SMALL LETTER A WITH MACRON +0103 ; Changes_When_Titlecased # L& LATIN SMALL LETTER A WITH BREVE +0105 ; Changes_When_Titlecased # L& LATIN SMALL LETTER A WITH OGONEK +0107 ; Changes_When_Titlecased # L& LATIN SMALL LETTER C WITH ACUTE +0109 ; Changes_When_Titlecased # L& LATIN SMALL LETTER C WITH CIRCUMFLEX +010B ; Changes_When_Titlecased # L& LATIN SMALL LETTER C WITH DOT ABOVE +010D ; Changes_When_Titlecased # L& LATIN SMALL LETTER C WITH CARON +010F ; Changes_When_Titlecased # L& LATIN SMALL LETTER D WITH CARON +0111 ; Changes_When_Titlecased # L& LATIN SMALL LETTER D WITH STROKE +0113 ; Changes_When_Titlecased # L& LATIN SMALL LETTER E WITH MACRON +0115 ; Changes_When_Titlecased # L& LATIN SMALL LETTER E WITH BREVE +0117 ; Changes_When_Titlecased # L& LATIN SMALL LETTER E WITH DOT ABOVE +0119 ; Changes_When_Titlecased # L& LATIN SMALL LETTER E WITH OGONEK +011B ; Changes_When_Titlecased # L& LATIN SMALL LETTER E WITH CARON +011D ; Changes_When_Titlecased # L& LATIN SMALL LETTER G WITH CIRCUMFLEX +011F ; Changes_When_Titlecased # L& LATIN SMALL LETTER G WITH BREVE +0121 ; Changes_When_Titlecased # L& LATIN SMALL LETTER G WITH DOT ABOVE +0123 ; Changes_When_Titlecased # L& LATIN SMALL LETTER G WITH CEDILLA +0125 ; Changes_When_Titlecased # L& LATIN SMALL LETTER H WITH CIRCUMFLEX +0127 ; Changes_When_Titlecased # L& LATIN SMALL LETTER H WITH STROKE +0129 ; Changes_When_Titlecased # L& LATIN SMALL LETTER I WITH TILDE +012B ; Changes_When_Titlecased # L& LATIN SMALL LETTER I WITH MACRON +012D ; Changes_When_Titlecased # L& LATIN SMALL LETTER I WITH BREVE +012F ; Changes_When_Titlecased # L& LATIN SMALL LETTER I WITH OGONEK +0131 ; Changes_When_Titlecased # L& LATIN SMALL LETTER DOTLESS I +0133 ; Changes_When_Titlecased # L& LATIN SMALL LIGATURE IJ +0135 ; Changes_When_Titlecased # L& LATIN SMALL LETTER J WITH CIRCUMFLEX +0137 ; Changes_When_Titlecased # L& LATIN SMALL LETTER K WITH CEDILLA +013A ; Changes_When_Titlecased # L& LATIN SMALL LETTER L WITH ACUTE +013C ; Changes_When_Titlecased # L& LATIN SMALL LETTER L WITH CEDILLA +013E ; Changes_When_Titlecased # L& LATIN SMALL LETTER L WITH CARON +0140 ; Changes_When_Titlecased # L& LATIN SMALL LETTER L WITH MIDDLE DOT +0142 ; Changes_When_Titlecased # L& LATIN SMALL LETTER L WITH STROKE +0144 ; Changes_When_Titlecased # L& LATIN SMALL LETTER N WITH ACUTE +0146 ; Changes_When_Titlecased # L& LATIN SMALL LETTER N WITH CEDILLA +0148..0149 ; Changes_When_Titlecased # L& [2] LATIN SMALL LETTER N WITH CARON..LATIN SMALL LETTER N PRECEDED BY APOSTROPHE +014B ; Changes_When_Titlecased # L& LATIN SMALL LETTER ENG +014D ; Changes_When_Titlecased # L& LATIN SMALL LETTER O WITH MACRON +014F ; Changes_When_Titlecased # L& LATIN SMALL LETTER O WITH BREVE +0151 ; Changes_When_Titlecased # L& LATIN SMALL LETTER O WITH DOUBLE ACUTE +0153 ; Changes_When_Titlecased # L& LATIN SMALL LIGATURE OE +0155 ; Changes_When_Titlecased # L& LATIN SMALL LETTER R WITH ACUTE +0157 ; Changes_When_Titlecased # L& LATIN SMALL LETTER R WITH CEDILLA +0159 ; Changes_When_Titlecased # L& LATIN SMALL LETTER R WITH CARON +015B ; Changes_When_Titlecased # L& LATIN SMALL LETTER S WITH ACUTE +015D ; Changes_When_Titlecased # L& LATIN SMALL LETTER S WITH CIRCUMFLEX +015F ; Changes_When_Titlecased # L& LATIN SMALL LETTER S WITH CEDILLA +0161 ; Changes_When_Titlecased # L& LATIN SMALL LETTER S WITH CARON +0163 ; Changes_When_Titlecased # L& LATIN SMALL LETTER T WITH CEDILLA +0165 ; Changes_When_Titlecased # L& LATIN SMALL LETTER T WITH CARON +0167 ; Changes_When_Titlecased # L& LATIN SMALL LETTER T WITH STROKE +0169 ; Changes_When_Titlecased # L& LATIN SMALL LETTER U WITH TILDE +016B ; Changes_When_Titlecased # L& LATIN SMALL LETTER U WITH MACRON +016D ; Changes_When_Titlecased # L& LATIN SMALL LETTER U WITH BREVE +016F ; Changes_When_Titlecased # L& LATIN SMALL LETTER U WITH RING ABOVE +0171 ; Changes_When_Titlecased # L& LATIN SMALL LETTER U WITH DOUBLE ACUTE +0173 ; Changes_When_Titlecased # L& LATIN SMALL LETTER U WITH OGONEK +0175 ; Changes_When_Titlecased # L& LATIN SMALL LETTER W WITH CIRCUMFLEX +0177 ; Changes_When_Titlecased # L& LATIN SMALL LETTER Y WITH CIRCUMFLEX +017A ; Changes_When_Titlecased # L& LATIN SMALL LETTER Z WITH ACUTE +017C ; Changes_When_Titlecased # L& LATIN SMALL LETTER Z WITH DOT ABOVE +017E..0180 ; Changes_When_Titlecased # L& [3] LATIN SMALL LETTER Z WITH CARON..LATIN SMALL LETTER B WITH STROKE +0183 ; Changes_When_Titlecased # L& LATIN SMALL LETTER B WITH TOPBAR +0185 ; Changes_When_Titlecased # L& LATIN SMALL LETTER TONE SIX +0188 ; Changes_When_Titlecased # L& LATIN SMALL LETTER C WITH HOOK +018C ; Changes_When_Titlecased # L& LATIN SMALL LETTER D WITH TOPBAR +0192 ; Changes_When_Titlecased # L& LATIN SMALL LETTER F WITH HOOK +0195 ; Changes_When_Titlecased # L& LATIN SMALL LETTER HV +0199..019A ; Changes_When_Titlecased # L& [2] LATIN SMALL LETTER K WITH HOOK..LATIN SMALL LETTER L WITH BAR +019E ; Changes_When_Titlecased # L& LATIN SMALL LETTER N WITH LONG RIGHT LEG +01A1 ; Changes_When_Titlecased # L& LATIN SMALL LETTER O WITH HORN +01A3 ; Changes_When_Titlecased # L& LATIN SMALL LETTER OI +01A5 ; Changes_When_Titlecased # L& LATIN SMALL LETTER P WITH HOOK +01A8 ; Changes_When_Titlecased # L& LATIN SMALL LETTER TONE TWO +01AD ; Changes_When_Titlecased # L& LATIN SMALL LETTER T WITH HOOK +01B0 ; Changes_When_Titlecased # L& LATIN SMALL LETTER U WITH HORN +01B4 ; Changes_When_Titlecased # L& LATIN SMALL LETTER Y WITH HOOK +01B6 ; Changes_When_Titlecased # L& LATIN SMALL LETTER Z WITH STROKE +01B9 ; Changes_When_Titlecased # L& LATIN SMALL LETTER EZH REVERSED +01BD ; Changes_When_Titlecased # L& LATIN SMALL LETTER TONE FIVE +01BF ; Changes_When_Titlecased # L& LATIN LETTER WYNN +01C4 ; Changes_When_Titlecased # L& LATIN CAPITAL LETTER DZ WITH CARON +01C6..01C7 ; Changes_When_Titlecased # L& [2] LATIN SMALL LETTER DZ WITH CARON..LATIN CAPITAL LETTER LJ +01C9..01CA ; Changes_When_Titlecased # L& [2] LATIN SMALL LETTER LJ..LATIN CAPITAL LETTER NJ +01CC ; Changes_When_Titlecased # L& LATIN SMALL LETTER NJ +01CE ; Changes_When_Titlecased # L& LATIN SMALL LETTER A WITH CARON +01D0 ; Changes_When_Titlecased # L& LATIN SMALL LETTER I WITH CARON +01D2 ; Changes_When_Titlecased # L& LATIN SMALL LETTER O WITH CARON +01D4 ; Changes_When_Titlecased # L& LATIN SMALL LETTER U WITH CARON +01D6 ; Changes_When_Titlecased # L& LATIN SMALL LETTER U WITH DIAERESIS AND MACRON +01D8 ; Changes_When_Titlecased # L& LATIN SMALL LETTER U WITH DIAERESIS AND ACUTE +01DA ; Changes_When_Titlecased # L& LATIN SMALL LETTER U WITH DIAERESIS AND CARON +01DC..01DD ; Changes_When_Titlecased # L& [2] LATIN SMALL LETTER U WITH DIAERESIS AND GRAVE..LATIN SMALL LETTER TURNED E +01DF ; Changes_When_Titlecased # L& LATIN SMALL LETTER A WITH DIAERESIS AND MACRON +01E1 ; Changes_When_Titlecased # L& LATIN SMALL LETTER A WITH DOT ABOVE AND MACRON +01E3 ; Changes_When_Titlecased # L& LATIN SMALL LETTER AE WITH MACRON +01E5 ; Changes_When_Titlecased # L& LATIN SMALL LETTER G WITH STROKE +01E7 ; Changes_When_Titlecased # L& LATIN SMALL LETTER G WITH CARON +01E9 ; Changes_When_Titlecased # L& LATIN SMALL LETTER K WITH CARON +01EB ; Changes_When_Titlecased # L& LATIN SMALL LETTER O WITH OGONEK +01ED ; Changes_When_Titlecased # L& LATIN SMALL LETTER O WITH OGONEK AND MACRON +01EF..01F1 ; Changes_When_Titlecased # L& [3] LATIN SMALL LETTER EZH WITH CARON..LATIN CAPITAL LETTER DZ +01F3 ; Changes_When_Titlecased # L& LATIN SMALL LETTER DZ +01F5 ; Changes_When_Titlecased # L& LATIN SMALL LETTER G WITH ACUTE +01F9 ; Changes_When_Titlecased # L& LATIN SMALL LETTER N WITH GRAVE +01FB ; Changes_When_Titlecased # L& LATIN SMALL LETTER A WITH RING ABOVE AND ACUTE +01FD ; Changes_When_Titlecased # L& LATIN SMALL LETTER AE WITH ACUTE +01FF ; Changes_When_Titlecased # L& LATIN SMALL LETTER O WITH STROKE AND ACUTE +0201 ; Changes_When_Titlecased # L& LATIN SMALL LETTER A WITH DOUBLE GRAVE +0203 ; Changes_When_Titlecased # L& LATIN SMALL LETTER A WITH INVERTED BREVE +0205 ; Changes_When_Titlecased # L& LATIN SMALL LETTER E WITH DOUBLE GRAVE +0207 ; Changes_When_Titlecased # L& LATIN SMALL LETTER E WITH INVERTED BREVE +0209 ; Changes_When_Titlecased # L& LATIN SMALL LETTER I WITH DOUBLE GRAVE +020B ; Changes_When_Titlecased # L& LATIN SMALL LETTER I WITH INVERTED BREVE +020D ; Changes_When_Titlecased # L& LATIN SMALL LETTER O WITH DOUBLE GRAVE +020F ; Changes_When_Titlecased # L& LATIN SMALL LETTER O WITH INVERTED BREVE +0211 ; Changes_When_Titlecased # L& LATIN SMALL LETTER R WITH DOUBLE GRAVE +0213 ; Changes_When_Titlecased # L& LATIN SMALL LETTER R WITH INVERTED BREVE +0215 ; Changes_When_Titlecased # L& LATIN SMALL LETTER U WITH DOUBLE GRAVE +0217 ; Changes_When_Titlecased # L& LATIN SMALL LETTER U WITH INVERTED BREVE +0219 ; Changes_When_Titlecased # L& LATIN SMALL LETTER S WITH COMMA BELOW +021B ; Changes_When_Titlecased # L& LATIN SMALL LETTER T WITH COMMA BELOW +021D ; Changes_When_Titlecased # L& LATIN SMALL LETTER YOGH +021F ; Changes_When_Titlecased # L& LATIN SMALL LETTER H WITH CARON +0223 ; Changes_When_Titlecased # L& LATIN SMALL LETTER OU +0225 ; Changes_When_Titlecased # L& LATIN SMALL LETTER Z WITH HOOK +0227 ; Changes_When_Titlecased # L& LATIN SMALL LETTER A WITH DOT ABOVE +0229 ; Changes_When_Titlecased # L& LATIN SMALL LETTER E WITH CEDILLA +022B ; Changes_When_Titlecased # L& LATIN SMALL LETTER O WITH DIAERESIS AND MACRON +022D ; Changes_When_Titlecased # L& LATIN SMALL LETTER O WITH TILDE AND MACRON +022F ; Changes_When_Titlecased # L& LATIN SMALL LETTER O WITH DOT ABOVE +0231 ; Changes_When_Titlecased # L& LATIN SMALL LETTER O WITH DOT ABOVE AND MACRON +0233 ; Changes_When_Titlecased # L& LATIN SMALL LETTER Y WITH MACRON +023C ; Changes_When_Titlecased # L& LATIN SMALL LETTER C WITH STROKE +023F..0240 ; Changes_When_Titlecased # L& [2] LATIN SMALL LETTER S WITH SWASH TAIL..LATIN SMALL LETTER Z WITH SWASH TAIL +0242 ; Changes_When_Titlecased # L& LATIN SMALL LETTER GLOTTAL STOP +0247 ; Changes_When_Titlecased # L& LATIN SMALL LETTER E WITH STROKE +0249 ; Changes_When_Titlecased # L& LATIN SMALL LETTER J WITH STROKE +024B ; Changes_When_Titlecased # L& LATIN SMALL LETTER Q WITH HOOK TAIL +024D ; Changes_When_Titlecased # L& LATIN SMALL LETTER R WITH STROKE +024F..0254 ; Changes_When_Titlecased # L& [6] LATIN SMALL LETTER Y WITH STROKE..LATIN SMALL LETTER OPEN O +0256..0257 ; Changes_When_Titlecased # L& [2] LATIN SMALL LETTER D WITH TAIL..LATIN SMALL LETTER D WITH HOOK +0259 ; Changes_When_Titlecased # L& LATIN SMALL LETTER SCHWA +025B ; Changes_When_Titlecased # L& LATIN SMALL LETTER OPEN E +0260 ; Changes_When_Titlecased # L& LATIN SMALL LETTER G WITH HOOK +0263 ; Changes_When_Titlecased # L& LATIN SMALL LETTER GAMMA +0268..0269 ; Changes_When_Titlecased # L& [2] LATIN SMALL LETTER I WITH STROKE..LATIN SMALL LETTER IOTA +026B ; Changes_When_Titlecased # L& LATIN SMALL LETTER L WITH MIDDLE TILDE +026F ; Changes_When_Titlecased # L& LATIN SMALL LETTER TURNED M +0271..0272 ; Changes_When_Titlecased # L& [2] LATIN SMALL LETTER M WITH HOOK..LATIN SMALL LETTER N WITH LEFT HOOK +0275 ; Changes_When_Titlecased # L& LATIN SMALL LETTER BARRED O +027D ; Changes_When_Titlecased # L& LATIN SMALL LETTER R WITH TAIL +0280 ; Changes_When_Titlecased # L& LATIN LETTER SMALL CAPITAL R +0283 ; Changes_When_Titlecased # L& LATIN SMALL LETTER ESH +0288..028C ; Changes_When_Titlecased # L& [5] LATIN SMALL LETTER T WITH RETROFLEX HOOK..LATIN SMALL LETTER TURNED V +0292 ; Changes_When_Titlecased # L& LATIN SMALL LETTER EZH +0345 ; Changes_When_Titlecased # Mn COMBINING GREEK YPOGEGRAMMENI +0371 ; Changes_When_Titlecased # L& GREEK SMALL LETTER HETA +0373 ; Changes_When_Titlecased # L& GREEK SMALL LETTER ARCHAIC SAMPI +0377 ; Changes_When_Titlecased # L& GREEK SMALL LETTER PAMPHYLIAN DIGAMMA +037B..037D ; Changes_When_Titlecased # L& [3] GREEK SMALL REVERSED LUNATE SIGMA SYMBOL..GREEK SMALL REVERSED DOTTED LUNATE SIGMA SYMBOL +0390 ; Changes_When_Titlecased # L& GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS +03AC..03CE ; Changes_When_Titlecased # L& [35] GREEK SMALL LETTER ALPHA WITH TONOS..GREEK SMALL LETTER OMEGA WITH TONOS +03D0..03D1 ; Changes_When_Titlecased # L& [2] GREEK BETA SYMBOL..GREEK THETA SYMBOL +03D5..03D7 ; Changes_When_Titlecased # L& [3] GREEK PHI SYMBOL..GREEK KAI SYMBOL +03D9 ; Changes_When_Titlecased # L& GREEK SMALL LETTER ARCHAIC KOPPA +03DB ; Changes_When_Titlecased # L& GREEK SMALL LETTER STIGMA +03DD ; Changes_When_Titlecased # L& GREEK SMALL LETTER DIGAMMA +03DF ; Changes_When_Titlecased # L& GREEK SMALL LETTER KOPPA +03E1 ; Changes_When_Titlecased # L& GREEK SMALL LETTER SAMPI +03E3 ; Changes_When_Titlecased # L& COPTIC SMALL LETTER SHEI +03E5 ; Changes_When_Titlecased # L& COPTIC SMALL LETTER FEI +03E7 ; Changes_When_Titlecased # L& COPTIC SMALL LETTER KHEI +03E9 ; Changes_When_Titlecased # L& COPTIC SMALL LETTER HORI +03EB ; Changes_When_Titlecased # L& COPTIC SMALL LETTER GANGIA +03ED ; Changes_When_Titlecased # L& COPTIC SMALL LETTER SHIMA +03EF..03F2 ; Changes_When_Titlecased # L& [4] COPTIC SMALL LETTER DEI..GREEK LUNATE SIGMA SYMBOL +03F5 ; Changes_When_Titlecased # L& GREEK LUNATE EPSILON SYMBOL +03F8 ; Changes_When_Titlecased # L& GREEK SMALL LETTER SHO +03FB ; Changes_When_Titlecased # L& GREEK SMALL LETTER SAN +0430..045F ; Changes_When_Titlecased # L& [48] CYRILLIC SMALL LETTER A..CYRILLIC SMALL LETTER DZHE +0461 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER OMEGA +0463 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER YAT +0465 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER IOTIFIED E +0467 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER LITTLE YUS +0469 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER IOTIFIED LITTLE YUS +046B ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER BIG YUS +046D ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER IOTIFIED BIG YUS +046F ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER KSI +0471 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER PSI +0473 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER FITA +0475 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER IZHITSA +0477 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER IZHITSA WITH DOUBLE GRAVE ACCENT +0479 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER UK +047B ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER ROUND OMEGA +047D ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER OMEGA WITH TITLO +047F ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER OT +0481 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER KOPPA +048B ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER SHORT I WITH TAIL +048D ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER SEMISOFT SIGN +048F ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER ER WITH TICK +0491 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER GHE WITH UPTURN +0493 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER GHE WITH STROKE +0495 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER GHE WITH MIDDLE HOOK +0497 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER ZHE WITH DESCENDER +0499 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER ZE WITH DESCENDER +049B ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER KA WITH DESCENDER +049D ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER KA WITH VERTICAL STROKE +049F ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER KA WITH STROKE +04A1 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER BASHKIR KA +04A3 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER EN WITH DESCENDER +04A5 ; Changes_When_Titlecased # L& CYRILLIC SMALL LIGATURE EN GHE +04A7 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER PE WITH MIDDLE HOOK +04A9 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER ABKHASIAN HA +04AB ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER ES WITH DESCENDER +04AD ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER TE WITH DESCENDER +04AF ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER STRAIGHT U +04B1 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER STRAIGHT U WITH STROKE +04B3 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER HA WITH DESCENDER +04B5 ; Changes_When_Titlecased # L& CYRILLIC SMALL LIGATURE TE TSE +04B7 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER CHE WITH DESCENDER +04B9 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER CHE WITH VERTICAL STROKE +04BB ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER SHHA +04BD ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER ABKHASIAN CHE +04BF ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER ABKHASIAN CHE WITH DESCENDER +04C2 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER ZHE WITH BREVE +04C4 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER KA WITH HOOK +04C6 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER EL WITH TAIL +04C8 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER EN WITH HOOK +04CA ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER EN WITH TAIL +04CC ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER KHAKASSIAN CHE +04CE..04CF ; Changes_When_Titlecased # L& [2] CYRILLIC SMALL LETTER EM WITH TAIL..CYRILLIC SMALL LETTER PALOCHKA +04D1 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER A WITH BREVE +04D3 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER A WITH DIAERESIS +04D5 ; Changes_When_Titlecased # L& CYRILLIC SMALL LIGATURE A IE +04D7 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER IE WITH BREVE +04D9 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER SCHWA +04DB ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER SCHWA WITH DIAERESIS +04DD ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER ZHE WITH DIAERESIS +04DF ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER ZE WITH DIAERESIS +04E1 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER ABKHASIAN DZE +04E3 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER I WITH MACRON +04E5 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER I WITH DIAERESIS +04E7 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER O WITH DIAERESIS +04E9 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER BARRED O +04EB ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER BARRED O WITH DIAERESIS +04ED ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER E WITH DIAERESIS +04EF ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER U WITH MACRON +04F1 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER U WITH DIAERESIS +04F3 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER U WITH DOUBLE ACUTE +04F5 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER CHE WITH DIAERESIS +04F7 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER GHE WITH DESCENDER +04F9 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER YERU WITH DIAERESIS +04FB ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER GHE WITH STROKE AND HOOK +04FD ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER HA WITH HOOK +04FF ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER HA WITH STROKE +0501 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER KOMI DE +0503 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER KOMI DJE +0505 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER KOMI ZJE +0507 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER KOMI DZJE +0509 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER KOMI LJE +050B ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER KOMI NJE +050D ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER KOMI SJE +050F ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER KOMI TJE +0511 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER REVERSED ZE +0513 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER EL WITH HOOK +0515 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER LHA +0517 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER RHA +0519 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER YAE +051B ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER QA +051D ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER WE +051F ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER ALEUT KA +0521 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER EL WITH MIDDLE HOOK +0523 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER EN WITH MIDDLE HOOK +0525 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER PE WITH DESCENDER +0561..0587 ; Changes_When_Titlecased # L& [39] ARMENIAN SMALL LETTER AYB..ARMENIAN SMALL LIGATURE ECH YIWN +1D79 ; Changes_When_Titlecased # L& LATIN SMALL LETTER INSULAR G +1D7D ; Changes_When_Titlecased # L& LATIN SMALL LETTER P WITH STROKE +1E01 ; Changes_When_Titlecased # L& LATIN SMALL LETTER A WITH RING BELOW +1E03 ; Changes_When_Titlecased # L& LATIN SMALL LETTER B WITH DOT ABOVE +1E05 ; Changes_When_Titlecased # L& LATIN SMALL LETTER B WITH DOT BELOW +1E07 ; Changes_When_Titlecased # L& LATIN SMALL LETTER B WITH LINE BELOW +1E09 ; Changes_When_Titlecased # L& LATIN SMALL LETTER C WITH CEDILLA AND ACUTE +1E0B ; Changes_When_Titlecased # L& LATIN SMALL LETTER D WITH DOT ABOVE +1E0D ; Changes_When_Titlecased # L& LATIN SMALL LETTER D WITH DOT BELOW +1E0F ; Changes_When_Titlecased # L& LATIN SMALL LETTER D WITH LINE BELOW +1E11 ; Changes_When_Titlecased # L& LATIN SMALL LETTER D WITH CEDILLA +1E13 ; Changes_When_Titlecased # L& LATIN SMALL LETTER D WITH CIRCUMFLEX BELOW +1E15 ; Changes_When_Titlecased # L& LATIN SMALL LETTER E WITH MACRON AND GRAVE +1E17 ; Changes_When_Titlecased # L& LATIN SMALL LETTER E WITH MACRON AND ACUTE +1E19 ; Changes_When_Titlecased # L& LATIN SMALL LETTER E WITH CIRCUMFLEX BELOW +1E1B ; Changes_When_Titlecased # L& LATIN SMALL LETTER E WITH TILDE BELOW +1E1D ; Changes_When_Titlecased # L& LATIN SMALL LETTER E WITH CEDILLA AND BREVE +1E1F ; Changes_When_Titlecased # L& LATIN SMALL LETTER F WITH DOT ABOVE +1E21 ; Changes_When_Titlecased # L& LATIN SMALL LETTER G WITH MACRON +1E23 ; Changes_When_Titlecased # L& LATIN SMALL LETTER H WITH DOT ABOVE +1E25 ; Changes_When_Titlecased # L& LATIN SMALL LETTER H WITH DOT BELOW +1E27 ; Changes_When_Titlecased # L& LATIN SMALL LETTER H WITH DIAERESIS +1E29 ; Changes_When_Titlecased # L& LATIN SMALL LETTER H WITH CEDILLA +1E2B ; Changes_When_Titlecased # L& LATIN SMALL LETTER H WITH BREVE BELOW +1E2D ; Changes_When_Titlecased # L& LATIN SMALL LETTER I WITH TILDE BELOW +1E2F ; Changes_When_Titlecased # L& LATIN SMALL LETTER I WITH DIAERESIS AND ACUTE +1E31 ; Changes_When_Titlecased # L& LATIN SMALL LETTER K WITH ACUTE +1E33 ; Changes_When_Titlecased # L& LATIN SMALL LETTER K WITH DOT BELOW +1E35 ; Changes_When_Titlecased # L& LATIN SMALL LETTER K WITH LINE BELOW +1E37 ; Changes_When_Titlecased # L& LATIN SMALL LETTER L WITH DOT BELOW +1E39 ; Changes_When_Titlecased # L& LATIN SMALL LETTER L WITH DOT BELOW AND MACRON +1E3B ; Changes_When_Titlecased # L& LATIN SMALL LETTER L WITH LINE BELOW +1E3D ; Changes_When_Titlecased # L& LATIN SMALL LETTER L WITH CIRCUMFLEX BELOW +1E3F ; Changes_When_Titlecased # L& LATIN SMALL LETTER M WITH ACUTE +1E41 ; Changes_When_Titlecased # L& LATIN SMALL LETTER M WITH DOT ABOVE +1E43 ; Changes_When_Titlecased # L& LATIN SMALL LETTER M WITH DOT BELOW +1E45 ; Changes_When_Titlecased # L& LATIN SMALL LETTER N WITH DOT ABOVE +1E47 ; Changes_When_Titlecased # L& LATIN SMALL LETTER N WITH DOT BELOW +1E49 ; Changes_When_Titlecased # L& LATIN SMALL LETTER N WITH LINE BELOW +1E4B ; Changes_When_Titlecased # L& LATIN SMALL LETTER N WITH CIRCUMFLEX BELOW +1E4D ; Changes_When_Titlecased # L& LATIN SMALL LETTER O WITH TILDE AND ACUTE +1E4F ; Changes_When_Titlecased # L& LATIN SMALL LETTER O WITH TILDE AND DIAERESIS +1E51 ; Changes_When_Titlecased # L& LATIN SMALL LETTER O WITH MACRON AND GRAVE +1E53 ; Changes_When_Titlecased # L& LATIN SMALL LETTER O WITH MACRON AND ACUTE +1E55 ; Changes_When_Titlecased # L& LATIN SMALL LETTER P WITH ACUTE +1E57 ; Changes_When_Titlecased # L& LATIN SMALL LETTER P WITH DOT ABOVE +1E59 ; Changes_When_Titlecased # L& LATIN SMALL LETTER R WITH DOT ABOVE +1E5B ; Changes_When_Titlecased # L& LATIN SMALL LETTER R WITH DOT BELOW +1E5D ; Changes_When_Titlecased # L& LATIN SMALL LETTER R WITH DOT BELOW AND MACRON +1E5F ; Changes_When_Titlecased # L& LATIN SMALL LETTER R WITH LINE BELOW +1E61 ; Changes_When_Titlecased # L& LATIN SMALL LETTER S WITH DOT ABOVE +1E63 ; Changes_When_Titlecased # L& LATIN SMALL LETTER S WITH DOT BELOW +1E65 ; Changes_When_Titlecased # L& LATIN SMALL LETTER S WITH ACUTE AND DOT ABOVE +1E67 ; Changes_When_Titlecased # L& LATIN SMALL LETTER S WITH CARON AND DOT ABOVE +1E69 ; Changes_When_Titlecased # L& LATIN SMALL LETTER S WITH DOT BELOW AND DOT ABOVE +1E6B ; Changes_When_Titlecased # L& LATIN SMALL LETTER T WITH DOT ABOVE +1E6D ; Changes_When_Titlecased # L& LATIN SMALL LETTER T WITH DOT BELOW +1E6F ; Changes_When_Titlecased # L& LATIN SMALL LETTER T WITH LINE BELOW +1E71 ; Changes_When_Titlecased # L& LATIN SMALL LETTER T WITH CIRCUMFLEX BELOW +1E73 ; Changes_When_Titlecased # L& LATIN SMALL LETTER U WITH DIAERESIS BELOW +1E75 ; Changes_When_Titlecased # L& LATIN SMALL LETTER U WITH TILDE BELOW +1E77 ; Changes_When_Titlecased # L& LATIN SMALL LETTER U WITH CIRCUMFLEX BELOW +1E79 ; Changes_When_Titlecased # L& LATIN SMALL LETTER U WITH TILDE AND ACUTE +1E7B ; Changes_When_Titlecased # L& LATIN SMALL LETTER U WITH MACRON AND DIAERESIS +1E7D ; Changes_When_Titlecased # L& LATIN SMALL LETTER V WITH TILDE +1E7F ; Changes_When_Titlecased # L& LATIN SMALL LETTER V WITH DOT BELOW +1E81 ; Changes_When_Titlecased # L& LATIN SMALL LETTER W WITH GRAVE +1E83 ; Changes_When_Titlecased # L& LATIN SMALL LETTER W WITH ACUTE +1E85 ; Changes_When_Titlecased # L& LATIN SMALL LETTER W WITH DIAERESIS +1E87 ; Changes_When_Titlecased # L& LATIN SMALL LETTER W WITH DOT ABOVE +1E89 ; Changes_When_Titlecased # L& LATIN SMALL LETTER W WITH DOT BELOW +1E8B ; Changes_When_Titlecased # L& LATIN SMALL LETTER X WITH DOT ABOVE +1E8D ; Changes_When_Titlecased # L& LATIN SMALL LETTER X WITH DIAERESIS +1E8F ; Changes_When_Titlecased # L& LATIN SMALL LETTER Y WITH DOT ABOVE +1E91 ; Changes_When_Titlecased # L& LATIN SMALL LETTER Z WITH CIRCUMFLEX +1E93 ; Changes_When_Titlecased # L& LATIN SMALL LETTER Z WITH DOT BELOW +1E95..1E9B ; Changes_When_Titlecased # L& [7] LATIN SMALL LETTER Z WITH LINE BELOW..LATIN SMALL LETTER LONG S WITH DOT ABOVE +1EA1 ; Changes_When_Titlecased # L& LATIN SMALL LETTER A WITH DOT BELOW +1EA3 ; Changes_When_Titlecased # L& LATIN SMALL LETTER A WITH HOOK ABOVE +1EA5 ; Changes_When_Titlecased # L& LATIN SMALL LETTER A WITH CIRCUMFLEX AND ACUTE +1EA7 ; Changes_When_Titlecased # L& LATIN SMALL LETTER A WITH CIRCUMFLEX AND GRAVE +1EA9 ; Changes_When_Titlecased # L& LATIN SMALL LETTER A WITH CIRCUMFLEX AND HOOK ABOVE +1EAB ; Changes_When_Titlecased # L& LATIN SMALL LETTER A WITH CIRCUMFLEX AND TILDE +1EAD ; Changes_When_Titlecased # L& LATIN SMALL LETTER A WITH CIRCUMFLEX AND DOT BELOW +1EAF ; Changes_When_Titlecased # L& LATIN SMALL LETTER A WITH BREVE AND ACUTE +1EB1 ; Changes_When_Titlecased # L& LATIN SMALL LETTER A WITH BREVE AND GRAVE +1EB3 ; Changes_When_Titlecased # L& LATIN SMALL LETTER A WITH BREVE AND HOOK ABOVE +1EB5 ; Changes_When_Titlecased # L& LATIN SMALL LETTER A WITH BREVE AND TILDE +1EB7 ; Changes_When_Titlecased # L& LATIN SMALL LETTER A WITH BREVE AND DOT BELOW +1EB9 ; Changes_When_Titlecased # L& LATIN SMALL LETTER E WITH DOT BELOW +1EBB ; Changes_When_Titlecased # L& LATIN SMALL LETTER E WITH HOOK ABOVE +1EBD ; Changes_When_Titlecased # L& LATIN SMALL LETTER E WITH TILDE +1EBF ; Changes_When_Titlecased # L& LATIN SMALL LETTER E WITH CIRCUMFLEX AND ACUTE +1EC1 ; Changes_When_Titlecased # L& LATIN SMALL LETTER E WITH CIRCUMFLEX AND GRAVE +1EC3 ; Changes_When_Titlecased # L& LATIN SMALL LETTER E WITH CIRCUMFLEX AND HOOK ABOVE +1EC5 ; Changes_When_Titlecased # L& LATIN SMALL LETTER E WITH CIRCUMFLEX AND TILDE +1EC7 ; Changes_When_Titlecased # L& LATIN SMALL LETTER E WITH CIRCUMFLEX AND DOT BELOW +1EC9 ; Changes_When_Titlecased # L& LATIN SMALL LETTER I WITH HOOK ABOVE +1ECB ; Changes_When_Titlecased # L& LATIN SMALL LETTER I WITH DOT BELOW +1ECD ; Changes_When_Titlecased # L& LATIN SMALL LETTER O WITH DOT BELOW +1ECF ; Changes_When_Titlecased # L& LATIN SMALL LETTER O WITH HOOK ABOVE +1ED1 ; Changes_When_Titlecased # L& LATIN SMALL LETTER O WITH CIRCUMFLEX AND ACUTE +1ED3 ; Changes_When_Titlecased # L& LATIN SMALL LETTER O WITH CIRCUMFLEX AND GRAVE +1ED5 ; Changes_When_Titlecased # L& LATIN SMALL LETTER O WITH CIRCUMFLEX AND HOOK ABOVE +1ED7 ; Changes_When_Titlecased # L& LATIN SMALL LETTER O WITH CIRCUMFLEX AND TILDE +1ED9 ; Changes_When_Titlecased # L& LATIN SMALL LETTER O WITH CIRCUMFLEX AND DOT BELOW +1EDB ; Changes_When_Titlecased # L& LATIN SMALL LETTER O WITH HORN AND ACUTE +1EDD ; Changes_When_Titlecased # L& LATIN SMALL LETTER O WITH HORN AND GRAVE +1EDF ; Changes_When_Titlecased # L& LATIN SMALL LETTER O WITH HORN AND HOOK ABOVE +1EE1 ; Changes_When_Titlecased # L& LATIN SMALL LETTER O WITH HORN AND TILDE +1EE3 ; Changes_When_Titlecased # L& LATIN SMALL LETTER O WITH HORN AND DOT BELOW +1EE5 ; Changes_When_Titlecased # L& LATIN SMALL LETTER U WITH DOT BELOW +1EE7 ; Changes_When_Titlecased # L& LATIN SMALL LETTER U WITH HOOK ABOVE +1EE9 ; Changes_When_Titlecased # L& LATIN SMALL LETTER U WITH HORN AND ACUTE +1EEB ; Changes_When_Titlecased # L& LATIN SMALL LETTER U WITH HORN AND GRAVE +1EED ; Changes_When_Titlecased # L& LATIN SMALL LETTER U WITH HORN AND HOOK ABOVE +1EEF ; Changes_When_Titlecased # L& LATIN SMALL LETTER U WITH HORN AND TILDE +1EF1 ; Changes_When_Titlecased # L& LATIN SMALL LETTER U WITH HORN AND DOT BELOW +1EF3 ; Changes_When_Titlecased # L& LATIN SMALL LETTER Y WITH GRAVE +1EF5 ; Changes_When_Titlecased # L& LATIN SMALL LETTER Y WITH DOT BELOW +1EF7 ; Changes_When_Titlecased # L& LATIN SMALL LETTER Y WITH HOOK ABOVE +1EF9 ; Changes_When_Titlecased # L& LATIN SMALL LETTER Y WITH TILDE +1EFB ; Changes_When_Titlecased # L& LATIN SMALL LETTER MIDDLE-WELSH LL +1EFD ; Changes_When_Titlecased # L& LATIN SMALL LETTER MIDDLE-WELSH V +1EFF..1F07 ; Changes_When_Titlecased # L& [9] LATIN SMALL LETTER Y WITH LOOP..GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI +1F10..1F15 ; Changes_When_Titlecased # L& [6] GREEK SMALL LETTER EPSILON WITH PSILI..GREEK SMALL LETTER EPSILON WITH DASIA AND OXIA +1F20..1F27 ; Changes_When_Titlecased # L& [8] GREEK SMALL LETTER ETA WITH PSILI..GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI +1F30..1F37 ; Changes_When_Titlecased # L& [8] GREEK SMALL LETTER IOTA WITH PSILI..GREEK SMALL LETTER IOTA WITH DASIA AND PERISPOMENI +1F40..1F45 ; Changes_When_Titlecased # L& [6] GREEK SMALL LETTER OMICRON WITH PSILI..GREEK SMALL LETTER OMICRON WITH DASIA AND OXIA +1F50..1F57 ; Changes_When_Titlecased # L& [8] GREEK SMALL LETTER UPSILON WITH PSILI..GREEK SMALL LETTER UPSILON WITH DASIA AND PERISPOMENI +1F60..1F67 ; Changes_When_Titlecased # L& [8] GREEK SMALL LETTER OMEGA WITH PSILI..GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI +1F70..1F7D ; Changes_When_Titlecased # L& [14] GREEK SMALL LETTER ALPHA WITH VARIA..GREEK SMALL LETTER OMEGA WITH OXIA +1F80..1F87 ; Changes_When_Titlecased # L& [8] GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI..GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI +1F90..1F97 ; Changes_When_Titlecased # L& [8] GREEK SMALL LETTER ETA WITH PSILI AND YPOGEGRAMMENI..GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI +1FA0..1FA7 ; Changes_When_Titlecased # L& [8] GREEK SMALL LETTER OMEGA WITH PSILI AND YPOGEGRAMMENI..GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI +1FB0..1FB4 ; Changes_When_Titlecased # L& [5] GREEK SMALL LETTER ALPHA WITH VRACHY..GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI +1FB6..1FB7 ; Changes_When_Titlecased # L& [2] GREEK SMALL LETTER ALPHA WITH PERISPOMENI..GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI +1FBE ; Changes_When_Titlecased # L& GREEK PROSGEGRAMMENI +1FC2..1FC4 ; Changes_When_Titlecased # L& [3] GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI..GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI +1FC6..1FC7 ; Changes_When_Titlecased # L& [2] GREEK SMALL LETTER ETA WITH PERISPOMENI..GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI +1FD0..1FD3 ; Changes_When_Titlecased # L& [4] GREEK SMALL LETTER IOTA WITH VRACHY..GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA +1FD6..1FD7 ; Changes_When_Titlecased # L& [2] GREEK SMALL LETTER IOTA WITH PERISPOMENI..GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI +1FE0..1FE7 ; Changes_When_Titlecased # L& [8] GREEK SMALL LETTER UPSILON WITH VRACHY..GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI +1FF2..1FF4 ; Changes_When_Titlecased # L& [3] GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI..GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI +1FF6..1FF7 ; Changes_When_Titlecased # L& [2] GREEK SMALL LETTER OMEGA WITH PERISPOMENI..GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI +214E ; Changes_When_Titlecased # L& TURNED SMALL F +2170..217F ; Changes_When_Titlecased # Nl [16] SMALL ROMAN NUMERAL ONE..SMALL ROMAN NUMERAL ONE THOUSAND +2184 ; Changes_When_Titlecased # L& LATIN SMALL LETTER REVERSED C +24D0..24E9 ; Changes_When_Titlecased # So [26] CIRCLED LATIN SMALL LETTER A..CIRCLED LATIN SMALL LETTER Z +2C30..2C5E ; Changes_When_Titlecased # L& [47] GLAGOLITIC SMALL LETTER AZU..GLAGOLITIC SMALL LETTER LATINATE MYSLITE +2C61 ; Changes_When_Titlecased # L& LATIN SMALL LETTER L WITH DOUBLE BAR +2C65..2C66 ; Changes_When_Titlecased # L& [2] LATIN SMALL LETTER A WITH STROKE..LATIN SMALL LETTER T WITH DIAGONAL STROKE +2C68 ; Changes_When_Titlecased # L& LATIN SMALL LETTER H WITH DESCENDER +2C6A ; Changes_When_Titlecased # L& LATIN SMALL LETTER K WITH DESCENDER +2C6C ; Changes_When_Titlecased # L& LATIN SMALL LETTER Z WITH DESCENDER +2C73 ; Changes_When_Titlecased # L& LATIN SMALL LETTER W WITH HOOK +2C76 ; Changes_When_Titlecased # L& LATIN SMALL LETTER HALF H +2C81 ; Changes_When_Titlecased # L& COPTIC SMALL LETTER ALFA +2C83 ; Changes_When_Titlecased # L& COPTIC SMALL LETTER VIDA +2C85 ; Changes_When_Titlecased # L& COPTIC SMALL LETTER GAMMA +2C87 ; Changes_When_Titlecased # L& COPTIC SMALL LETTER DALDA +2C89 ; Changes_When_Titlecased # L& COPTIC SMALL LETTER EIE +2C8B ; Changes_When_Titlecased # L& COPTIC SMALL LETTER SOU +2C8D ; Changes_When_Titlecased # L& COPTIC SMALL LETTER ZATA +2C8F ; Changes_When_Titlecased # L& COPTIC SMALL LETTER HATE +2C91 ; Changes_When_Titlecased # L& COPTIC SMALL LETTER THETHE +2C93 ; Changes_When_Titlecased # L& COPTIC SMALL LETTER IAUDA +2C95 ; Changes_When_Titlecased # L& COPTIC SMALL LETTER KAPA +2C97 ; Changes_When_Titlecased # L& COPTIC SMALL LETTER LAULA +2C99 ; Changes_When_Titlecased # L& COPTIC SMALL LETTER MI +2C9B ; Changes_When_Titlecased # L& COPTIC SMALL LETTER NI +2C9D ; Changes_When_Titlecased # L& COPTIC SMALL LETTER KSI +2C9F ; Changes_When_Titlecased # L& COPTIC SMALL LETTER O +2CA1 ; Changes_When_Titlecased # L& COPTIC SMALL LETTER PI +2CA3 ; Changes_When_Titlecased # L& COPTIC SMALL LETTER RO +2CA5 ; Changes_When_Titlecased # L& COPTIC SMALL LETTER SIMA +2CA7 ; Changes_When_Titlecased # L& COPTIC SMALL LETTER TAU +2CA9 ; Changes_When_Titlecased # L& COPTIC SMALL LETTER UA +2CAB ; Changes_When_Titlecased # L& COPTIC SMALL LETTER FI +2CAD ; Changes_When_Titlecased # L& COPTIC SMALL LETTER KHI +2CAF ; Changes_When_Titlecased # L& COPTIC SMALL LETTER PSI +2CB1 ; Changes_When_Titlecased # L& COPTIC SMALL LETTER OOU +2CB3 ; Changes_When_Titlecased # L& COPTIC SMALL LETTER DIALECT-P ALEF +2CB5 ; Changes_When_Titlecased # L& COPTIC SMALL LETTER OLD COPTIC AIN +2CB7 ; Changes_When_Titlecased # L& COPTIC SMALL LETTER CRYPTOGRAMMIC EIE +2CB9 ; Changes_When_Titlecased # L& COPTIC SMALL LETTER DIALECT-P KAPA +2CBB ; Changes_When_Titlecased # L& COPTIC SMALL LETTER DIALECT-P NI +2CBD ; Changes_When_Titlecased # L& COPTIC SMALL LETTER CRYPTOGRAMMIC NI +2CBF ; Changes_When_Titlecased # L& COPTIC SMALL LETTER OLD COPTIC OOU +2CC1 ; Changes_When_Titlecased # L& COPTIC SMALL LETTER SAMPI +2CC3 ; Changes_When_Titlecased # L& COPTIC SMALL LETTER CROSSED SHEI +2CC5 ; Changes_When_Titlecased # L& COPTIC SMALL LETTER OLD COPTIC SHEI +2CC7 ; Changes_When_Titlecased # L& COPTIC SMALL LETTER OLD COPTIC ESH +2CC9 ; Changes_When_Titlecased # L& COPTIC SMALL LETTER AKHMIMIC KHEI +2CCB ; Changes_When_Titlecased # L& COPTIC SMALL LETTER DIALECT-P HORI +2CCD ; Changes_When_Titlecased # L& COPTIC SMALL LETTER OLD COPTIC HORI +2CCF ; Changes_When_Titlecased # L& COPTIC SMALL LETTER OLD COPTIC HA +2CD1 ; Changes_When_Titlecased # L& COPTIC SMALL LETTER L-SHAPED HA +2CD3 ; Changes_When_Titlecased # L& COPTIC SMALL LETTER OLD COPTIC HEI +2CD5 ; Changes_When_Titlecased # L& COPTIC SMALL LETTER OLD COPTIC HAT +2CD7 ; Changes_When_Titlecased # L& COPTIC SMALL LETTER OLD COPTIC GANGIA +2CD9 ; Changes_When_Titlecased # L& COPTIC SMALL LETTER OLD COPTIC DJA +2CDB ; Changes_When_Titlecased # L& COPTIC SMALL LETTER OLD COPTIC SHIMA +2CDD ; Changes_When_Titlecased # L& COPTIC SMALL LETTER OLD NUBIAN SHIMA +2CDF ; Changes_When_Titlecased # L& COPTIC SMALL LETTER OLD NUBIAN NGI +2CE1 ; Changes_When_Titlecased # L& COPTIC SMALL LETTER OLD NUBIAN NYI +2CE3 ; Changes_When_Titlecased # L& COPTIC SMALL LETTER OLD NUBIAN WAU +2CEC ; Changes_When_Titlecased # L& COPTIC SMALL LETTER CRYPTOGRAMMIC SHEI +2CEE ; Changes_When_Titlecased # L& COPTIC SMALL LETTER CRYPTOGRAMMIC GANGIA +2D00..2D25 ; Changes_When_Titlecased # L& [38] GEORGIAN SMALL LETTER AN..GEORGIAN SMALL LETTER HOE +A641 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER ZEMLYA +A643 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER DZELO +A645 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER REVERSED DZE +A647 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER IOTA +A649 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER DJERV +A64B ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER MONOGRAPH UK +A64D ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER BROAD OMEGA +A64F ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER NEUTRAL YER +A651 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER YERU WITH BACK YER +A653 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER IOTIFIED YAT +A655 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER REVERSED YU +A657 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER IOTIFIED A +A659 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER CLOSED LITTLE YUS +A65B ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER BLENDED YUS +A65D ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER IOTIFIED CLOSED LITTLE YUS +A65F ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER YN +A663 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER SOFT DE +A665 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER SOFT EL +A667 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER SOFT EM +A669 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER MONOCULAR O +A66B ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER BINOCULAR O +A66D ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER DOUBLE MONOCULAR O +A681 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER DWE +A683 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER DZWE +A685 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER ZHWE +A687 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER CCHE +A689 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER DZZE +A68B ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER TE WITH MIDDLE HOOK +A68D ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER TWE +A68F ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER TSWE +A691 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER TSSE +A693 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER TCHE +A695 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER HWE +A697 ; Changes_When_Titlecased # L& CYRILLIC SMALL LETTER SHWE +A723 ; Changes_When_Titlecased # L& LATIN SMALL LETTER EGYPTOLOGICAL ALEF +A725 ; Changes_When_Titlecased # L& LATIN SMALL LETTER EGYPTOLOGICAL AIN +A727 ; Changes_When_Titlecased # L& LATIN SMALL LETTER HENG +A729 ; Changes_When_Titlecased # L& LATIN SMALL LETTER TZ +A72B ; Changes_When_Titlecased # L& LATIN SMALL LETTER TRESILLO +A72D ; Changes_When_Titlecased # L& LATIN SMALL LETTER CUATRILLO +A72F ; Changes_When_Titlecased # L& LATIN SMALL LETTER CUATRILLO WITH COMMA +A733 ; Changes_When_Titlecased # L& LATIN SMALL LETTER AA +A735 ; Changes_When_Titlecased # L& LATIN SMALL LETTER AO +A737 ; Changes_When_Titlecased # L& LATIN SMALL LETTER AU +A739 ; Changes_When_Titlecased # L& LATIN SMALL LETTER AV +A73B ; Changes_When_Titlecased # L& LATIN SMALL LETTER AV WITH HORIZONTAL BAR +A73D ; Changes_When_Titlecased # L& LATIN SMALL LETTER AY +A73F ; Changes_When_Titlecased # L& LATIN SMALL LETTER REVERSED C WITH DOT +A741 ; Changes_When_Titlecased # L& LATIN SMALL LETTER K WITH STROKE +A743 ; Changes_When_Titlecased # L& LATIN SMALL LETTER K WITH DIAGONAL STROKE +A745 ; Changes_When_Titlecased # L& LATIN SMALL LETTER K WITH STROKE AND DIAGONAL STROKE +A747 ; Changes_When_Titlecased # L& LATIN SMALL LETTER BROKEN L +A749 ; Changes_When_Titlecased # L& LATIN SMALL LETTER L WITH HIGH STROKE +A74B ; Changes_When_Titlecased # L& LATIN SMALL LETTER O WITH LONG STROKE OVERLAY +A74D ; Changes_When_Titlecased # L& LATIN SMALL LETTER O WITH LOOP +A74F ; Changes_When_Titlecased # L& LATIN SMALL LETTER OO +A751 ; Changes_When_Titlecased # L& LATIN SMALL LETTER P WITH STROKE THROUGH DESCENDER +A753 ; Changes_When_Titlecased # L& LATIN SMALL LETTER P WITH FLOURISH +A755 ; Changes_When_Titlecased # L& LATIN SMALL LETTER P WITH SQUIRREL TAIL +A757 ; Changes_When_Titlecased # L& LATIN SMALL LETTER Q WITH STROKE THROUGH DESCENDER +A759 ; Changes_When_Titlecased # L& LATIN SMALL LETTER Q WITH DIAGONAL STROKE +A75B ; Changes_When_Titlecased # L& LATIN SMALL LETTER R ROTUNDA +A75D ; Changes_When_Titlecased # L& LATIN SMALL LETTER RUM ROTUNDA +A75F ; Changes_When_Titlecased # L& LATIN SMALL LETTER V WITH DIAGONAL STROKE +A761 ; Changes_When_Titlecased # L& LATIN SMALL LETTER VY +A763 ; Changes_When_Titlecased # L& LATIN SMALL LETTER VISIGOTHIC Z +A765 ; Changes_When_Titlecased # L& LATIN SMALL LETTER THORN WITH STROKE +A767 ; Changes_When_Titlecased # L& LATIN SMALL LETTER THORN WITH STROKE THROUGH DESCENDER +A769 ; Changes_When_Titlecased # L& LATIN SMALL LETTER VEND +A76B ; Changes_When_Titlecased # L& LATIN SMALL LETTER ET +A76D ; Changes_When_Titlecased # L& LATIN SMALL LETTER IS +A76F ; Changes_When_Titlecased # L& LATIN SMALL LETTER CON +A77A ; Changes_When_Titlecased # L& LATIN SMALL LETTER INSULAR D +A77C ; Changes_When_Titlecased # L& LATIN SMALL LETTER INSULAR F +A77F ; Changes_When_Titlecased # L& LATIN SMALL LETTER TURNED INSULAR G +A781 ; Changes_When_Titlecased # L& LATIN SMALL LETTER TURNED L +A783 ; Changes_When_Titlecased # L& LATIN SMALL LETTER INSULAR R +A785 ; Changes_When_Titlecased # L& LATIN SMALL LETTER INSULAR S +A787 ; Changes_When_Titlecased # L& LATIN SMALL LETTER INSULAR T +A78C ; Changes_When_Titlecased # L& LATIN SMALL LETTER SALTILLO +FB00..FB06 ; Changes_When_Titlecased # L& [7] LATIN SMALL LIGATURE FF..LATIN SMALL LIGATURE ST +FB13..FB17 ; Changes_When_Titlecased # L& [5] ARMENIAN SMALL LIGATURE MEN NOW..ARMENIAN SMALL LIGATURE MEN XEH +FF41..FF5A ; Changes_When_Titlecased # L& [26] FULLWIDTH LATIN SMALL LETTER A..FULLWIDTH LATIN SMALL LETTER Z +10428..1044F ; Changes_When_Titlecased # L& [40] DESERET SMALL LETTER LONG I..DESERET SMALL LETTER EW + +# Total code points: 1085 + +# ================================================ + +# Derived Property: Changes_When_Casefolded (CWCF) +# Characters whose normalized forms are not stable under case folding. +# For more information, see D127 in Section 3.13, "Default Case Algorithms". +# Changes_When_Casefolded(X) is true when toCasefold(toNFD(X)) != toNFD(X) + +0041..005A ; Changes_When_Casefolded # L& [26] LATIN CAPITAL LETTER A..LATIN CAPITAL LETTER Z +00B5 ; Changes_When_Casefolded # L& MICRO SIGN +00C0..00D6 ; Changes_When_Casefolded # L& [23] LATIN CAPITAL LETTER A WITH GRAVE..LATIN CAPITAL LETTER O WITH DIAERESIS +00D8..00DF ; Changes_When_Casefolded # L& [8] LATIN CAPITAL LETTER O WITH STROKE..LATIN SMALL LETTER SHARP S +0100 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER A WITH MACRON +0102 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER A WITH BREVE +0104 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER A WITH OGONEK +0106 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER C WITH ACUTE +0108 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER C WITH CIRCUMFLEX +010A ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER C WITH DOT ABOVE +010C ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER C WITH CARON +010E ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER D WITH CARON +0110 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER D WITH STROKE +0112 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER E WITH MACRON +0114 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER E WITH BREVE +0116 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER E WITH DOT ABOVE +0118 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER E WITH OGONEK +011A ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER E WITH CARON +011C ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER G WITH CIRCUMFLEX +011E ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER G WITH BREVE +0120 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER G WITH DOT ABOVE +0122 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER G WITH CEDILLA +0124 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER H WITH CIRCUMFLEX +0126 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER H WITH STROKE +0128 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER I WITH TILDE +012A ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER I WITH MACRON +012C ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER I WITH BREVE +012E ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER I WITH OGONEK +0130 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER I WITH DOT ABOVE +0132 ; Changes_When_Casefolded # L& LATIN CAPITAL LIGATURE IJ +0134 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER J WITH CIRCUMFLEX +0136 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER K WITH CEDILLA +0139 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER L WITH ACUTE +013B ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER L WITH CEDILLA +013D ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER L WITH CARON +013F ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER L WITH MIDDLE DOT +0141 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER L WITH STROKE +0143 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER N WITH ACUTE +0145 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER N WITH CEDILLA +0147 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER N WITH CARON +0149..014A ; Changes_When_Casefolded # L& [2] LATIN SMALL LETTER N PRECEDED BY APOSTROPHE..LATIN CAPITAL LETTER ENG +014C ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER O WITH MACRON +014E ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER O WITH BREVE +0150 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER O WITH DOUBLE ACUTE +0152 ; Changes_When_Casefolded # L& LATIN CAPITAL LIGATURE OE +0154 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER R WITH ACUTE +0156 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER R WITH CEDILLA +0158 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER R WITH CARON +015A ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER S WITH ACUTE +015C ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER S WITH CIRCUMFLEX +015E ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER S WITH CEDILLA +0160 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER S WITH CARON +0162 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER T WITH CEDILLA +0164 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER T WITH CARON +0166 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER T WITH STROKE +0168 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER U WITH TILDE +016A ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER U WITH MACRON +016C ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER U WITH BREVE +016E ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER U WITH RING ABOVE +0170 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER U WITH DOUBLE ACUTE +0172 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER U WITH OGONEK +0174 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER W WITH CIRCUMFLEX +0176 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER Y WITH CIRCUMFLEX +0178..0179 ; Changes_When_Casefolded # L& [2] LATIN CAPITAL LETTER Y WITH DIAERESIS..LATIN CAPITAL LETTER Z WITH ACUTE +017B ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER Z WITH DOT ABOVE +017D ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER Z WITH CARON +017F ; Changes_When_Casefolded # L& LATIN SMALL LETTER LONG S +0181..0182 ; Changes_When_Casefolded # L& [2] LATIN CAPITAL LETTER B WITH HOOK..LATIN CAPITAL LETTER B WITH TOPBAR +0184 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER TONE SIX +0186..0187 ; Changes_When_Casefolded # L& [2] LATIN CAPITAL LETTER OPEN O..LATIN CAPITAL LETTER C WITH HOOK +0189..018B ; Changes_When_Casefolded # L& [3] LATIN CAPITAL LETTER AFRICAN D..LATIN CAPITAL LETTER D WITH TOPBAR +018E..0191 ; Changes_When_Casefolded # L& [4] LATIN CAPITAL LETTER REVERSED E..LATIN CAPITAL LETTER F WITH HOOK +0193..0194 ; Changes_When_Casefolded # L& [2] LATIN CAPITAL LETTER G WITH HOOK..LATIN CAPITAL LETTER GAMMA +0196..0198 ; Changes_When_Casefolded # L& [3] LATIN CAPITAL LETTER IOTA..LATIN CAPITAL LETTER K WITH HOOK +019C..019D ; Changes_When_Casefolded # L& [2] LATIN CAPITAL LETTER TURNED M..LATIN CAPITAL LETTER N WITH LEFT HOOK +019F..01A0 ; Changes_When_Casefolded # L& [2] LATIN CAPITAL LETTER O WITH MIDDLE TILDE..LATIN CAPITAL LETTER O WITH HORN +01A2 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER OI +01A4 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER P WITH HOOK +01A6..01A7 ; Changes_When_Casefolded # L& [2] LATIN LETTER YR..LATIN CAPITAL LETTER TONE TWO +01A9 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER ESH +01AC ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER T WITH HOOK +01AE..01AF ; Changes_When_Casefolded # L& [2] LATIN CAPITAL LETTER T WITH RETROFLEX HOOK..LATIN CAPITAL LETTER U WITH HORN +01B1..01B3 ; Changes_When_Casefolded # L& [3] LATIN CAPITAL LETTER UPSILON..LATIN CAPITAL LETTER Y WITH HOOK +01B5 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER Z WITH STROKE +01B7..01B8 ; Changes_When_Casefolded # L& [2] LATIN CAPITAL LETTER EZH..LATIN CAPITAL LETTER EZH REVERSED +01BC ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER TONE FIVE +01C4..01C5 ; Changes_When_Casefolded # L& [2] LATIN CAPITAL LETTER DZ WITH CARON..LATIN CAPITAL LETTER D WITH SMALL LETTER Z WITH CARON +01C7..01C8 ; Changes_When_Casefolded # L& [2] LATIN CAPITAL LETTER LJ..LATIN CAPITAL LETTER L WITH SMALL LETTER J +01CA..01CB ; Changes_When_Casefolded # L& [2] LATIN CAPITAL LETTER NJ..LATIN CAPITAL LETTER N WITH SMALL LETTER J +01CD ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER A WITH CARON +01CF ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER I WITH CARON +01D1 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER O WITH CARON +01D3 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER U WITH CARON +01D5 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER U WITH DIAERESIS AND MACRON +01D7 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER U WITH DIAERESIS AND ACUTE +01D9 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER U WITH DIAERESIS AND CARON +01DB ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER U WITH DIAERESIS AND GRAVE +01DE ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER A WITH DIAERESIS AND MACRON +01E0 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER A WITH DOT ABOVE AND MACRON +01E2 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER AE WITH MACRON +01E4 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER G WITH STROKE +01E6 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER G WITH CARON +01E8 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER K WITH CARON +01EA ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER O WITH OGONEK +01EC ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER O WITH OGONEK AND MACRON +01EE ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER EZH WITH CARON +01F1..01F2 ; Changes_When_Casefolded # L& [2] LATIN CAPITAL LETTER DZ..LATIN CAPITAL LETTER D WITH SMALL LETTER Z +01F4 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER G WITH ACUTE +01F6..01F8 ; Changes_When_Casefolded # L& [3] LATIN CAPITAL LETTER HWAIR..LATIN CAPITAL LETTER N WITH GRAVE +01FA ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER A WITH RING ABOVE AND ACUTE +01FC ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER AE WITH ACUTE +01FE ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER O WITH STROKE AND ACUTE +0200 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER A WITH DOUBLE GRAVE +0202 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER A WITH INVERTED BREVE +0204 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER E WITH DOUBLE GRAVE +0206 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER E WITH INVERTED BREVE +0208 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER I WITH DOUBLE GRAVE +020A ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER I WITH INVERTED BREVE +020C ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER O WITH DOUBLE GRAVE +020E ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER O WITH INVERTED BREVE +0210 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER R WITH DOUBLE GRAVE +0212 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER R WITH INVERTED BREVE +0214 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER U WITH DOUBLE GRAVE +0216 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER U WITH INVERTED BREVE +0218 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER S WITH COMMA BELOW +021A ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER T WITH COMMA BELOW +021C ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER YOGH +021E ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER H WITH CARON +0220 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER N WITH LONG RIGHT LEG +0222 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER OU +0224 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER Z WITH HOOK +0226 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER A WITH DOT ABOVE +0228 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER E WITH CEDILLA +022A ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER O WITH DIAERESIS AND MACRON +022C ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER O WITH TILDE AND MACRON +022E ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER O WITH DOT ABOVE +0230 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER O WITH DOT ABOVE AND MACRON +0232 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER Y WITH MACRON +023A..023B ; Changes_When_Casefolded # L& [2] LATIN CAPITAL LETTER A WITH STROKE..LATIN CAPITAL LETTER C WITH STROKE +023D..023E ; Changes_When_Casefolded # L& [2] LATIN CAPITAL LETTER L WITH BAR..LATIN CAPITAL LETTER T WITH DIAGONAL STROKE +0241 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER GLOTTAL STOP +0243..0246 ; Changes_When_Casefolded # L& [4] LATIN CAPITAL LETTER B WITH STROKE..LATIN CAPITAL LETTER E WITH STROKE +0248 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER J WITH STROKE +024A ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER SMALL Q WITH HOOK TAIL +024C ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER R WITH STROKE +024E ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER Y WITH STROKE +0345 ; Changes_When_Casefolded # Mn COMBINING GREEK YPOGEGRAMMENI +0370 ; Changes_When_Casefolded # L& GREEK CAPITAL LETTER HETA +0372 ; Changes_When_Casefolded # L& GREEK CAPITAL LETTER ARCHAIC SAMPI +0376 ; Changes_When_Casefolded # L& GREEK CAPITAL LETTER PAMPHYLIAN DIGAMMA +0386 ; Changes_When_Casefolded # L& GREEK CAPITAL LETTER ALPHA WITH TONOS +0388..038A ; Changes_When_Casefolded # L& [3] GREEK CAPITAL LETTER EPSILON WITH TONOS..GREEK CAPITAL LETTER IOTA WITH TONOS +038C ; Changes_When_Casefolded # L& GREEK CAPITAL LETTER OMICRON WITH TONOS +038E..038F ; Changes_When_Casefolded # L& [2] GREEK CAPITAL LETTER UPSILON WITH TONOS..GREEK CAPITAL LETTER OMEGA WITH TONOS +0391..03A1 ; Changes_When_Casefolded # L& [17] GREEK CAPITAL LETTER ALPHA..GREEK CAPITAL LETTER RHO +03A3..03AB ; Changes_When_Casefolded # L& [9] GREEK CAPITAL LETTER SIGMA..GREEK CAPITAL LETTER UPSILON WITH DIALYTIKA +03C2 ; Changes_When_Casefolded # L& GREEK SMALL LETTER FINAL SIGMA +03CF..03D1 ; Changes_When_Casefolded # L& [3] GREEK CAPITAL KAI SYMBOL..GREEK THETA SYMBOL +03D5..03D6 ; Changes_When_Casefolded # L& [2] GREEK PHI SYMBOL..GREEK PI SYMBOL +03D8 ; Changes_When_Casefolded # L& GREEK LETTER ARCHAIC KOPPA +03DA ; Changes_When_Casefolded # L& GREEK LETTER STIGMA +03DC ; Changes_When_Casefolded # L& GREEK LETTER DIGAMMA +03DE ; Changes_When_Casefolded # L& GREEK LETTER KOPPA +03E0 ; Changes_When_Casefolded # L& GREEK LETTER SAMPI +03E2 ; Changes_When_Casefolded # L& COPTIC CAPITAL LETTER SHEI +03E4 ; Changes_When_Casefolded # L& COPTIC CAPITAL LETTER FEI +03E6 ; Changes_When_Casefolded # L& COPTIC CAPITAL LETTER KHEI +03E8 ; Changes_When_Casefolded # L& COPTIC CAPITAL LETTER HORI +03EA ; Changes_When_Casefolded # L& COPTIC CAPITAL LETTER GANGIA +03EC ; Changes_When_Casefolded # L& COPTIC CAPITAL LETTER SHIMA +03EE ; Changes_When_Casefolded # L& COPTIC CAPITAL LETTER DEI +03F0..03F1 ; Changes_When_Casefolded # L& [2] GREEK KAPPA SYMBOL..GREEK RHO SYMBOL +03F4..03F5 ; Changes_When_Casefolded # L& [2] GREEK CAPITAL THETA SYMBOL..GREEK LUNATE EPSILON SYMBOL +03F7 ; Changes_When_Casefolded # L& GREEK CAPITAL LETTER SHO +03F9..03FA ; Changes_When_Casefolded # L& [2] GREEK CAPITAL LUNATE SIGMA SYMBOL..GREEK CAPITAL LETTER SAN +03FD..042F ; Changes_When_Casefolded # L& [51] GREEK CAPITAL REVERSED LUNATE SIGMA SYMBOL..CYRILLIC CAPITAL LETTER YA +0460 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER OMEGA +0462 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER YAT +0464 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER IOTIFIED E +0466 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER LITTLE YUS +0468 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER IOTIFIED LITTLE YUS +046A ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER BIG YUS +046C ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER IOTIFIED BIG YUS +046E ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER KSI +0470 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER PSI +0472 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER FITA +0474 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER IZHITSA +0476 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER IZHITSA WITH DOUBLE GRAVE ACCENT +0478 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER UK +047A ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER ROUND OMEGA +047C ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER OMEGA WITH TITLO +047E ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER OT +0480 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER KOPPA +048A ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER SHORT I WITH TAIL +048C ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER SEMISOFT SIGN +048E ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER ER WITH TICK +0490 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER GHE WITH UPTURN +0492 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER GHE WITH STROKE +0494 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER GHE WITH MIDDLE HOOK +0496 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER ZHE WITH DESCENDER +0498 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER ZE WITH DESCENDER +049A ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER KA WITH DESCENDER +049C ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER KA WITH VERTICAL STROKE +049E ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER KA WITH STROKE +04A0 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER BASHKIR KA +04A2 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER EN WITH DESCENDER +04A4 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LIGATURE EN GHE +04A6 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER PE WITH MIDDLE HOOK +04A8 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER ABKHASIAN HA +04AA ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER ES WITH DESCENDER +04AC ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER TE WITH DESCENDER +04AE ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER STRAIGHT U +04B0 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER STRAIGHT U WITH STROKE +04B2 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER HA WITH DESCENDER +04B4 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LIGATURE TE TSE +04B6 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER CHE WITH DESCENDER +04B8 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER CHE WITH VERTICAL STROKE +04BA ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER SHHA +04BC ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER ABKHASIAN CHE +04BE ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER ABKHASIAN CHE WITH DESCENDER +04C0..04C1 ; Changes_When_Casefolded # L& [2] CYRILLIC LETTER PALOCHKA..CYRILLIC CAPITAL LETTER ZHE WITH BREVE +04C3 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER KA WITH HOOK +04C5 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER EL WITH TAIL +04C7 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER EN WITH HOOK +04C9 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER EN WITH TAIL +04CB ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER KHAKASSIAN CHE +04CD ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER EM WITH TAIL +04D0 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER A WITH BREVE +04D2 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER A WITH DIAERESIS +04D4 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LIGATURE A IE +04D6 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER IE WITH BREVE +04D8 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER SCHWA +04DA ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER SCHWA WITH DIAERESIS +04DC ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER ZHE WITH DIAERESIS +04DE ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER ZE WITH DIAERESIS +04E0 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER ABKHASIAN DZE +04E2 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER I WITH MACRON +04E4 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER I WITH DIAERESIS +04E6 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER O WITH DIAERESIS +04E8 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER BARRED O +04EA ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER BARRED O WITH DIAERESIS +04EC ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER E WITH DIAERESIS +04EE ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER U WITH MACRON +04F0 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER U WITH DIAERESIS +04F2 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER U WITH DOUBLE ACUTE +04F4 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER CHE WITH DIAERESIS +04F6 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER GHE WITH DESCENDER +04F8 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER YERU WITH DIAERESIS +04FA ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER GHE WITH STROKE AND HOOK +04FC ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER HA WITH HOOK +04FE ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER HA WITH STROKE +0500 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER KOMI DE +0502 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER KOMI DJE +0504 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER KOMI ZJE +0506 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER KOMI DZJE +0508 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER KOMI LJE +050A ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER KOMI NJE +050C ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER KOMI SJE +050E ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER KOMI TJE +0510 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER REVERSED ZE +0512 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER EL WITH HOOK +0514 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER LHA +0516 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER RHA +0518 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER YAE +051A ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER QA +051C ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER WE +051E ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER ALEUT KA +0520 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER EL WITH MIDDLE HOOK +0522 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER EN WITH MIDDLE HOOK +0524 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER PE WITH DESCENDER +0531..0556 ; Changes_When_Casefolded # L& [38] ARMENIAN CAPITAL LETTER AYB..ARMENIAN CAPITAL LETTER FEH +0587 ; Changes_When_Casefolded # L& ARMENIAN SMALL LIGATURE ECH YIWN +10A0..10C5 ; Changes_When_Casefolded # L& [38] GEORGIAN CAPITAL LETTER AN..GEORGIAN CAPITAL LETTER HOE +1E00 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER A WITH RING BELOW +1E02 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER B WITH DOT ABOVE +1E04 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER B WITH DOT BELOW +1E06 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER B WITH LINE BELOW +1E08 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER C WITH CEDILLA AND ACUTE +1E0A ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER D WITH DOT ABOVE +1E0C ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER D WITH DOT BELOW +1E0E ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER D WITH LINE BELOW +1E10 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER D WITH CEDILLA +1E12 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER D WITH CIRCUMFLEX BELOW +1E14 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER E WITH MACRON AND GRAVE +1E16 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER E WITH MACRON AND ACUTE +1E18 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER E WITH CIRCUMFLEX BELOW +1E1A ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER E WITH TILDE BELOW +1E1C ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER E WITH CEDILLA AND BREVE +1E1E ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER F WITH DOT ABOVE +1E20 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER G WITH MACRON +1E22 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER H WITH DOT ABOVE +1E24 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER H WITH DOT BELOW +1E26 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER H WITH DIAERESIS +1E28 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER H WITH CEDILLA +1E2A ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER H WITH BREVE BELOW +1E2C ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER I WITH TILDE BELOW +1E2E ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER I WITH DIAERESIS AND ACUTE +1E30 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER K WITH ACUTE +1E32 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER K WITH DOT BELOW +1E34 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER K WITH LINE BELOW +1E36 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER L WITH DOT BELOW +1E38 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER L WITH DOT BELOW AND MACRON +1E3A ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER L WITH LINE BELOW +1E3C ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER L WITH CIRCUMFLEX BELOW +1E3E ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER M WITH ACUTE +1E40 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER M WITH DOT ABOVE +1E42 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER M WITH DOT BELOW +1E44 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER N WITH DOT ABOVE +1E46 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER N WITH DOT BELOW +1E48 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER N WITH LINE BELOW +1E4A ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER N WITH CIRCUMFLEX BELOW +1E4C ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER O WITH TILDE AND ACUTE +1E4E ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER O WITH TILDE AND DIAERESIS +1E50 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER O WITH MACRON AND GRAVE +1E52 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER O WITH MACRON AND ACUTE +1E54 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER P WITH ACUTE +1E56 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER P WITH DOT ABOVE +1E58 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER R WITH DOT ABOVE +1E5A ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER R WITH DOT BELOW +1E5C ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER R WITH DOT BELOW AND MACRON +1E5E ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER R WITH LINE BELOW +1E60 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER S WITH DOT ABOVE +1E62 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER S WITH DOT BELOW +1E64 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER S WITH ACUTE AND DOT ABOVE +1E66 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER S WITH CARON AND DOT ABOVE +1E68 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER S WITH DOT BELOW AND DOT ABOVE +1E6A ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER T WITH DOT ABOVE +1E6C ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER T WITH DOT BELOW +1E6E ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER T WITH LINE BELOW +1E70 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER T WITH CIRCUMFLEX BELOW +1E72 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER U WITH DIAERESIS BELOW +1E74 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER U WITH TILDE BELOW +1E76 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER U WITH CIRCUMFLEX BELOW +1E78 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER U WITH TILDE AND ACUTE +1E7A ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER U WITH MACRON AND DIAERESIS +1E7C ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER V WITH TILDE +1E7E ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER V WITH DOT BELOW +1E80 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER W WITH GRAVE +1E82 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER W WITH ACUTE +1E84 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER W WITH DIAERESIS +1E86 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER W WITH DOT ABOVE +1E88 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER W WITH DOT BELOW +1E8A ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER X WITH DOT ABOVE +1E8C ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER X WITH DIAERESIS +1E8E ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER Y WITH DOT ABOVE +1E90 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER Z WITH CIRCUMFLEX +1E92 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER Z WITH DOT BELOW +1E94 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER Z WITH LINE BELOW +1E9A..1E9B ; Changes_When_Casefolded # L& [2] LATIN SMALL LETTER A WITH RIGHT HALF RING..LATIN SMALL LETTER LONG S WITH DOT ABOVE +1E9E ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER SHARP S +1EA0 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER A WITH DOT BELOW +1EA2 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER A WITH HOOK ABOVE +1EA4 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND ACUTE +1EA6 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND GRAVE +1EA8 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND HOOK ABOVE +1EAA ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND TILDE +1EAC ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND DOT BELOW +1EAE ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER A WITH BREVE AND ACUTE +1EB0 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER A WITH BREVE AND GRAVE +1EB2 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER A WITH BREVE AND HOOK ABOVE +1EB4 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER A WITH BREVE AND TILDE +1EB6 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER A WITH BREVE AND DOT BELOW +1EB8 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER E WITH DOT BELOW +1EBA ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER E WITH HOOK ABOVE +1EBC ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER E WITH TILDE +1EBE ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND ACUTE +1EC0 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND GRAVE +1EC2 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND HOOK ABOVE +1EC4 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND TILDE +1EC6 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND DOT BELOW +1EC8 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER I WITH HOOK ABOVE +1ECA ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER I WITH DOT BELOW +1ECC ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER O WITH DOT BELOW +1ECE ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER O WITH HOOK ABOVE +1ED0 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND ACUTE +1ED2 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND GRAVE +1ED4 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND HOOK ABOVE +1ED6 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND TILDE +1ED8 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND DOT BELOW +1EDA ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER O WITH HORN AND ACUTE +1EDC ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER O WITH HORN AND GRAVE +1EDE ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER O WITH HORN AND HOOK ABOVE +1EE0 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER O WITH HORN AND TILDE +1EE2 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER O WITH HORN AND DOT BELOW +1EE4 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER U WITH DOT BELOW +1EE6 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER U WITH HOOK ABOVE +1EE8 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER U WITH HORN AND ACUTE +1EEA ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER U WITH HORN AND GRAVE +1EEC ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER U WITH HORN AND HOOK ABOVE +1EEE ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER U WITH HORN AND TILDE +1EF0 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER U WITH HORN AND DOT BELOW +1EF2 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER Y WITH GRAVE +1EF4 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER Y WITH DOT BELOW +1EF6 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER Y WITH HOOK ABOVE +1EF8 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER Y WITH TILDE +1EFA ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER MIDDLE-WELSH LL +1EFC ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER MIDDLE-WELSH V +1EFE ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER Y WITH LOOP +1F08..1F0F ; Changes_When_Casefolded # L& [8] GREEK CAPITAL LETTER ALPHA WITH PSILI..GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI +1F18..1F1D ; Changes_When_Casefolded # L& [6] GREEK CAPITAL LETTER EPSILON WITH PSILI..GREEK CAPITAL LETTER EPSILON WITH DASIA AND OXIA +1F28..1F2F ; Changes_When_Casefolded # L& [8] GREEK CAPITAL LETTER ETA WITH PSILI..GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI +1F38..1F3F ; Changes_When_Casefolded # L& [8] GREEK CAPITAL LETTER IOTA WITH PSILI..GREEK CAPITAL LETTER IOTA WITH DASIA AND PERISPOMENI +1F48..1F4D ; Changes_When_Casefolded # L& [6] GREEK CAPITAL LETTER OMICRON WITH PSILI..GREEK CAPITAL LETTER OMICRON WITH DASIA AND OXIA +1F59 ; Changes_When_Casefolded # L& GREEK CAPITAL LETTER UPSILON WITH DASIA +1F5B ; Changes_When_Casefolded # L& GREEK CAPITAL LETTER UPSILON WITH DASIA AND VARIA +1F5D ; Changes_When_Casefolded # L& GREEK CAPITAL LETTER UPSILON WITH DASIA AND OXIA +1F5F ; Changes_When_Casefolded # L& GREEK CAPITAL LETTER UPSILON WITH DASIA AND PERISPOMENI +1F68..1F6F ; Changes_When_Casefolded # L& [8] GREEK CAPITAL LETTER OMEGA WITH PSILI..GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI +1F80..1FAF ; Changes_When_Casefolded # L& [48] GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI..GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI +1FB2..1FB4 ; Changes_When_Casefolded # L& [3] GREEK SMALL LETTER ALPHA WITH VARIA AND YPOGEGRAMMENI..GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI +1FB7..1FBC ; Changes_When_Casefolded # L& [6] GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI..GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI +1FC2..1FC4 ; Changes_When_Casefolded # L& [3] GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI..GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI +1FC7..1FCC ; Changes_When_Casefolded # L& [6] GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI..GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI +1FD8..1FDB ; Changes_When_Casefolded # L& [4] GREEK CAPITAL LETTER IOTA WITH VRACHY..GREEK CAPITAL LETTER IOTA WITH OXIA +1FE8..1FEC ; Changes_When_Casefolded # L& [5] GREEK CAPITAL LETTER UPSILON WITH VRACHY..GREEK CAPITAL LETTER RHO WITH DASIA +1FF2..1FF4 ; Changes_When_Casefolded # L& [3] GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI..GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI +1FF7..1FFC ; Changes_When_Casefolded # L& [6] GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI..GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI +2126 ; Changes_When_Casefolded # L& OHM SIGN +212A..212B ; Changes_When_Casefolded # L& [2] KELVIN SIGN..ANGSTROM SIGN +2132 ; Changes_When_Casefolded # L& TURNED CAPITAL F +2160..216F ; Changes_When_Casefolded # Nl [16] ROMAN NUMERAL ONE..ROMAN NUMERAL ONE THOUSAND +2183 ; Changes_When_Casefolded # L& ROMAN NUMERAL REVERSED ONE HUNDRED +24B6..24CF ; Changes_When_Casefolded # So [26] CIRCLED LATIN CAPITAL LETTER A..CIRCLED LATIN CAPITAL LETTER Z +2C00..2C2E ; Changes_When_Casefolded # L& [47] GLAGOLITIC CAPITAL LETTER AZU..GLAGOLITIC CAPITAL LETTER LATINATE MYSLITE +2C60 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER L WITH DOUBLE BAR +2C62..2C64 ; Changes_When_Casefolded # L& [3] LATIN CAPITAL LETTER L WITH MIDDLE TILDE..LATIN CAPITAL LETTER R WITH TAIL +2C67 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER H WITH DESCENDER +2C69 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER K WITH DESCENDER +2C6B ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER Z WITH DESCENDER +2C6D..2C70 ; Changes_When_Casefolded # L& [4] LATIN CAPITAL LETTER ALPHA..LATIN CAPITAL LETTER TURNED ALPHA +2C72 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER W WITH HOOK +2C75 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER HALF H +2C7E..2C80 ; Changes_When_Casefolded # L& [3] LATIN CAPITAL LETTER S WITH SWASH TAIL..COPTIC CAPITAL LETTER ALFA +2C82 ; Changes_When_Casefolded # L& COPTIC CAPITAL LETTER VIDA +2C84 ; Changes_When_Casefolded # L& COPTIC CAPITAL LETTER GAMMA +2C86 ; Changes_When_Casefolded # L& COPTIC CAPITAL LETTER DALDA +2C88 ; Changes_When_Casefolded # L& COPTIC CAPITAL LETTER EIE +2C8A ; Changes_When_Casefolded # L& COPTIC CAPITAL LETTER SOU +2C8C ; Changes_When_Casefolded # L& COPTIC CAPITAL LETTER ZATA +2C8E ; Changes_When_Casefolded # L& COPTIC CAPITAL LETTER HATE +2C90 ; Changes_When_Casefolded # L& COPTIC CAPITAL LETTER THETHE +2C92 ; Changes_When_Casefolded # L& COPTIC CAPITAL LETTER IAUDA +2C94 ; Changes_When_Casefolded # L& COPTIC CAPITAL LETTER KAPA +2C96 ; Changes_When_Casefolded # L& COPTIC CAPITAL LETTER LAULA +2C98 ; Changes_When_Casefolded # L& COPTIC CAPITAL LETTER MI +2C9A ; Changes_When_Casefolded # L& COPTIC CAPITAL LETTER NI +2C9C ; Changes_When_Casefolded # L& COPTIC CAPITAL LETTER KSI +2C9E ; Changes_When_Casefolded # L& COPTIC CAPITAL LETTER O +2CA0 ; Changes_When_Casefolded # L& COPTIC CAPITAL LETTER PI +2CA2 ; Changes_When_Casefolded # L& COPTIC CAPITAL LETTER RO +2CA4 ; Changes_When_Casefolded # L& COPTIC CAPITAL LETTER SIMA +2CA6 ; Changes_When_Casefolded # L& COPTIC CAPITAL LETTER TAU +2CA8 ; Changes_When_Casefolded # L& COPTIC CAPITAL LETTER UA +2CAA ; Changes_When_Casefolded # L& COPTIC CAPITAL LETTER FI +2CAC ; Changes_When_Casefolded # L& COPTIC CAPITAL LETTER KHI +2CAE ; Changes_When_Casefolded # L& COPTIC CAPITAL LETTER PSI +2CB0 ; Changes_When_Casefolded # L& COPTIC CAPITAL LETTER OOU +2CB2 ; Changes_When_Casefolded # L& COPTIC CAPITAL LETTER DIALECT-P ALEF +2CB4 ; Changes_When_Casefolded # L& COPTIC CAPITAL LETTER OLD COPTIC AIN +2CB6 ; Changes_When_Casefolded # L& COPTIC CAPITAL LETTER CRYPTOGRAMMIC EIE +2CB8 ; Changes_When_Casefolded # L& COPTIC CAPITAL LETTER DIALECT-P KAPA +2CBA ; Changes_When_Casefolded # L& COPTIC CAPITAL LETTER DIALECT-P NI +2CBC ; Changes_When_Casefolded # L& COPTIC CAPITAL LETTER CRYPTOGRAMMIC NI +2CBE ; Changes_When_Casefolded # L& COPTIC CAPITAL LETTER OLD COPTIC OOU +2CC0 ; Changes_When_Casefolded # L& COPTIC CAPITAL LETTER SAMPI +2CC2 ; Changes_When_Casefolded # L& COPTIC CAPITAL LETTER CROSSED SHEI +2CC4 ; Changes_When_Casefolded # L& COPTIC CAPITAL LETTER OLD COPTIC SHEI +2CC6 ; Changes_When_Casefolded # L& COPTIC CAPITAL LETTER OLD COPTIC ESH +2CC8 ; Changes_When_Casefolded # L& COPTIC CAPITAL LETTER AKHMIMIC KHEI +2CCA ; Changes_When_Casefolded # L& COPTIC CAPITAL LETTER DIALECT-P HORI +2CCC ; Changes_When_Casefolded # L& COPTIC CAPITAL LETTER OLD COPTIC HORI +2CCE ; Changes_When_Casefolded # L& COPTIC CAPITAL LETTER OLD COPTIC HA +2CD0 ; Changes_When_Casefolded # L& COPTIC CAPITAL LETTER L-SHAPED HA +2CD2 ; Changes_When_Casefolded # L& COPTIC CAPITAL LETTER OLD COPTIC HEI +2CD4 ; Changes_When_Casefolded # L& COPTIC CAPITAL LETTER OLD COPTIC HAT +2CD6 ; Changes_When_Casefolded # L& COPTIC CAPITAL LETTER OLD COPTIC GANGIA +2CD8 ; Changes_When_Casefolded # L& COPTIC CAPITAL LETTER OLD COPTIC DJA +2CDA ; Changes_When_Casefolded # L& COPTIC CAPITAL LETTER OLD COPTIC SHIMA +2CDC ; Changes_When_Casefolded # L& COPTIC CAPITAL LETTER OLD NUBIAN SHIMA +2CDE ; Changes_When_Casefolded # L& COPTIC CAPITAL LETTER OLD NUBIAN NGI +2CE0 ; Changes_When_Casefolded # L& COPTIC CAPITAL LETTER OLD NUBIAN NYI +2CE2 ; Changes_When_Casefolded # L& COPTIC CAPITAL LETTER OLD NUBIAN WAU +2CEB ; Changes_When_Casefolded # L& COPTIC CAPITAL LETTER CRYPTOGRAMMIC SHEI +2CED ; Changes_When_Casefolded # L& COPTIC CAPITAL LETTER CRYPTOGRAMMIC GANGIA +A640 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER ZEMLYA +A642 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER DZELO +A644 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER REVERSED DZE +A646 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER IOTA +A648 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER DJERV +A64A ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER MONOGRAPH UK +A64C ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER BROAD OMEGA +A64E ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER NEUTRAL YER +A650 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER YERU WITH BACK YER +A652 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER IOTIFIED YAT +A654 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER REVERSED YU +A656 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER IOTIFIED A +A658 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER CLOSED LITTLE YUS +A65A ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER BLENDED YUS +A65C ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER IOTIFIED CLOSED LITTLE YUS +A65E ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER YN +A662 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER SOFT DE +A664 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER SOFT EL +A666 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER SOFT EM +A668 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER MONOCULAR O +A66A ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER BINOCULAR O +A66C ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER DOUBLE MONOCULAR O +A680 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER DWE +A682 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER DZWE +A684 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER ZHWE +A686 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER CCHE +A688 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER DZZE +A68A ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER TE WITH MIDDLE HOOK +A68C ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER TWE +A68E ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER TSWE +A690 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER TSSE +A692 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER TCHE +A694 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER HWE +A696 ; Changes_When_Casefolded # L& CYRILLIC CAPITAL LETTER SHWE +A722 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER EGYPTOLOGICAL ALEF +A724 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER EGYPTOLOGICAL AIN +A726 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER HENG +A728 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER TZ +A72A ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER TRESILLO +A72C ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER CUATRILLO +A72E ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER CUATRILLO WITH COMMA +A732 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER AA +A734 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER AO +A736 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER AU +A738 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER AV +A73A ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER AV WITH HORIZONTAL BAR +A73C ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER AY +A73E ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER REVERSED C WITH DOT +A740 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER K WITH STROKE +A742 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER K WITH DIAGONAL STROKE +A744 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER K WITH STROKE AND DIAGONAL STROKE +A746 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER BROKEN L +A748 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER L WITH HIGH STROKE +A74A ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER O WITH LONG STROKE OVERLAY +A74C ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER O WITH LOOP +A74E ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER OO +A750 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER P WITH STROKE THROUGH DESCENDER +A752 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER P WITH FLOURISH +A754 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER P WITH SQUIRREL TAIL +A756 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER Q WITH STROKE THROUGH DESCENDER +A758 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER Q WITH DIAGONAL STROKE +A75A ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER R ROTUNDA +A75C ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER RUM ROTUNDA +A75E ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER V WITH DIAGONAL STROKE +A760 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER VY +A762 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER VISIGOTHIC Z +A764 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER THORN WITH STROKE +A766 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER THORN WITH STROKE THROUGH DESCENDER +A768 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER VEND +A76A ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER ET +A76C ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER IS +A76E ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER CON +A779 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER INSULAR D +A77B ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER INSULAR F +A77D..A77E ; Changes_When_Casefolded # L& [2] LATIN CAPITAL LETTER INSULAR G..LATIN CAPITAL LETTER TURNED INSULAR G +A780 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER TURNED L +A782 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER INSULAR R +A784 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER INSULAR S +A786 ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER INSULAR T +A78B ; Changes_When_Casefolded # L& LATIN CAPITAL LETTER SALTILLO +FB00..FB06 ; Changes_When_Casefolded # L& [7] LATIN SMALL LIGATURE FF..LATIN SMALL LIGATURE ST +FB13..FB17 ; Changes_When_Casefolded # L& [5] ARMENIAN SMALL LIGATURE MEN NOW..ARMENIAN SMALL LIGATURE MEN XEH +FF21..FF3A ; Changes_When_Casefolded # L& [26] FULLWIDTH LATIN CAPITAL LETTER A..FULLWIDTH LATIN CAPITAL LETTER Z +10400..10427 ; Changes_When_Casefolded # L& [40] DESERET CAPITAL LETTER LONG I..DESERET CAPITAL LETTER EW + +# Total code points: 1093 + +# ================================================ + +# Derived Property: Changes_When_Casemapped (CWCM) +# Characters whose normalized forms are not stable under case mapping. +# For more information, see D128 in Section 3.13, "Default Case Algorithms". +# Changes_When_Casemapped(X) is true when CWL(X), or CWT(X), or CWU(X) + +0041..005A ; Changes_When_Casemapped # L& [26] LATIN CAPITAL LETTER A..LATIN CAPITAL LETTER Z +0061..007A ; Changes_When_Casemapped # L& [26] LATIN SMALL LETTER A..LATIN SMALL LETTER Z +00B5 ; Changes_When_Casemapped # L& MICRO SIGN +00C0..00D6 ; Changes_When_Casemapped # L& [23] LATIN CAPITAL LETTER A WITH GRAVE..LATIN CAPITAL LETTER O WITH DIAERESIS +00D8..00F6 ; Changes_When_Casemapped # L& [31] LATIN CAPITAL LETTER O WITH STROKE..LATIN SMALL LETTER O WITH DIAERESIS +00F8..0137 ; Changes_When_Casemapped # L& [64] LATIN SMALL LETTER O WITH STROKE..LATIN SMALL LETTER K WITH CEDILLA +0139..018C ; Changes_When_Casemapped # L& [84] LATIN CAPITAL LETTER L WITH ACUTE..LATIN SMALL LETTER D WITH TOPBAR +018E..019A ; Changes_When_Casemapped # L& [13] LATIN CAPITAL LETTER REVERSED E..LATIN SMALL LETTER L WITH BAR +019C..01A9 ; Changes_When_Casemapped # L& [14] LATIN CAPITAL LETTER TURNED M..LATIN CAPITAL LETTER ESH +01AC..01B9 ; Changes_When_Casemapped # L& [14] LATIN CAPITAL LETTER T WITH HOOK..LATIN SMALL LETTER EZH REVERSED +01BC..01BD ; Changes_When_Casemapped # L& [2] LATIN CAPITAL LETTER TONE FIVE..LATIN SMALL LETTER TONE FIVE +01BF ; Changes_When_Casemapped # L& LATIN LETTER WYNN +01C4..0220 ; Changes_When_Casemapped # L& [93] LATIN CAPITAL LETTER DZ WITH CARON..LATIN CAPITAL LETTER N WITH LONG RIGHT LEG +0222..0233 ; Changes_When_Casemapped # L& [18] LATIN CAPITAL LETTER OU..LATIN SMALL LETTER Y WITH MACRON +023A..0254 ; Changes_When_Casemapped # L& [27] LATIN CAPITAL LETTER A WITH STROKE..LATIN SMALL LETTER OPEN O +0256..0257 ; Changes_When_Casemapped # L& [2] LATIN SMALL LETTER D WITH TAIL..LATIN SMALL LETTER D WITH HOOK +0259 ; Changes_When_Casemapped # L& LATIN SMALL LETTER SCHWA +025B ; Changes_When_Casemapped # L& LATIN SMALL LETTER OPEN E +0260 ; Changes_When_Casemapped # L& LATIN SMALL LETTER G WITH HOOK +0263 ; Changes_When_Casemapped # L& LATIN SMALL LETTER GAMMA +0268..0269 ; Changes_When_Casemapped # L& [2] LATIN SMALL LETTER I WITH STROKE..LATIN SMALL LETTER IOTA +026B ; Changes_When_Casemapped # L& LATIN SMALL LETTER L WITH MIDDLE TILDE +026F ; Changes_When_Casemapped # L& LATIN SMALL LETTER TURNED M +0271..0272 ; Changes_When_Casemapped # L& [2] LATIN SMALL LETTER M WITH HOOK..LATIN SMALL LETTER N WITH LEFT HOOK +0275 ; Changes_When_Casemapped # L& LATIN SMALL LETTER BARRED O +027D ; Changes_When_Casemapped # L& LATIN SMALL LETTER R WITH TAIL +0280 ; Changes_When_Casemapped # L& LATIN LETTER SMALL CAPITAL R +0283 ; Changes_When_Casemapped # L& LATIN SMALL LETTER ESH +0288..028C ; Changes_When_Casemapped # L& [5] LATIN SMALL LETTER T WITH RETROFLEX HOOK..LATIN SMALL LETTER TURNED V +0292 ; Changes_When_Casemapped # L& LATIN SMALL LETTER EZH +0345 ; Changes_When_Casemapped # Mn COMBINING GREEK YPOGEGRAMMENI +0370..0373 ; Changes_When_Casemapped # L& [4] GREEK CAPITAL LETTER HETA..GREEK SMALL LETTER ARCHAIC SAMPI +0376..0377 ; Changes_When_Casemapped # L& [2] GREEK CAPITAL LETTER PAMPHYLIAN DIGAMMA..GREEK SMALL LETTER PAMPHYLIAN DIGAMMA +037B..037D ; Changes_When_Casemapped # L& [3] GREEK SMALL REVERSED LUNATE SIGMA SYMBOL..GREEK SMALL REVERSED DOTTED LUNATE SIGMA SYMBOL +0386 ; Changes_When_Casemapped # L& GREEK CAPITAL LETTER ALPHA WITH TONOS +0388..038A ; Changes_When_Casemapped # L& [3] GREEK CAPITAL LETTER EPSILON WITH TONOS..GREEK CAPITAL LETTER IOTA WITH TONOS +038C ; Changes_When_Casemapped # L& GREEK CAPITAL LETTER OMICRON WITH TONOS +038E..03A1 ; Changes_When_Casemapped # L& [20] GREEK CAPITAL LETTER UPSILON WITH TONOS..GREEK CAPITAL LETTER RHO +03A3..03D1 ; Changes_When_Casemapped # L& [47] GREEK CAPITAL LETTER SIGMA..GREEK THETA SYMBOL +03D5..03F2 ; Changes_When_Casemapped # L& [30] GREEK PHI SYMBOL..GREEK LUNATE SIGMA SYMBOL +03F4..03F5 ; Changes_When_Casemapped # L& [2] GREEK CAPITAL THETA SYMBOL..GREEK LUNATE EPSILON SYMBOL +03F7..03FB ; Changes_When_Casemapped # L& [5] GREEK CAPITAL LETTER SHO..GREEK SMALL LETTER SAN +03FD..0481 ; Changes_When_Casemapped # L& [133] GREEK CAPITAL REVERSED LUNATE SIGMA SYMBOL..CYRILLIC SMALL LETTER KOPPA +048A..0525 ; Changes_When_Casemapped # L& [156] CYRILLIC CAPITAL LETTER SHORT I WITH TAIL..CYRILLIC SMALL LETTER PE WITH DESCENDER +0531..0556 ; Changes_When_Casemapped # L& [38] ARMENIAN CAPITAL LETTER AYB..ARMENIAN CAPITAL LETTER FEH +0561..0587 ; Changes_When_Casemapped # L& [39] ARMENIAN SMALL LETTER AYB..ARMENIAN SMALL LIGATURE ECH YIWN +10A0..10C5 ; Changes_When_Casemapped # L& [38] GEORGIAN CAPITAL LETTER AN..GEORGIAN CAPITAL LETTER HOE +1D79 ; Changes_When_Casemapped # L& LATIN SMALL LETTER INSULAR G +1D7D ; Changes_When_Casemapped # L& LATIN SMALL LETTER P WITH STROKE +1E00..1E9B ; Changes_When_Casemapped # L& [156] LATIN CAPITAL LETTER A WITH RING BELOW..LATIN SMALL LETTER LONG S WITH DOT ABOVE +1E9E ; Changes_When_Casemapped # L& LATIN CAPITAL LETTER SHARP S +1EA0..1F15 ; Changes_When_Casemapped # L& [118] LATIN CAPITAL LETTER A WITH DOT BELOW..GREEK SMALL LETTER EPSILON WITH DASIA AND OXIA +1F18..1F1D ; Changes_When_Casemapped # L& [6] GREEK CAPITAL LETTER EPSILON WITH PSILI..GREEK CAPITAL LETTER EPSILON WITH DASIA AND OXIA +1F20..1F45 ; Changes_When_Casemapped # L& [38] GREEK SMALL LETTER ETA WITH PSILI..GREEK SMALL LETTER OMICRON WITH DASIA AND OXIA +1F48..1F4D ; Changes_When_Casemapped # L& [6] GREEK CAPITAL LETTER OMICRON WITH PSILI..GREEK CAPITAL LETTER OMICRON WITH DASIA AND OXIA +1F50..1F57 ; Changes_When_Casemapped # L& [8] GREEK SMALL LETTER UPSILON WITH PSILI..GREEK SMALL LETTER UPSILON WITH DASIA AND PERISPOMENI +1F59 ; Changes_When_Casemapped # L& GREEK CAPITAL LETTER UPSILON WITH DASIA +1F5B ; Changes_When_Casemapped # L& GREEK CAPITAL LETTER UPSILON WITH DASIA AND VARIA +1F5D ; Changes_When_Casemapped # L& GREEK CAPITAL LETTER UPSILON WITH DASIA AND OXIA +1F5F..1F7D ; Changes_When_Casemapped # L& [31] GREEK CAPITAL LETTER UPSILON WITH DASIA AND PERISPOMENI..GREEK SMALL LETTER OMEGA WITH OXIA +1F80..1FB4 ; Changes_When_Casemapped # L& [53] GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI..GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI +1FB6..1FBC ; Changes_When_Casemapped # L& [7] GREEK SMALL LETTER ALPHA WITH PERISPOMENI..GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI +1FBE ; Changes_When_Casemapped # L& GREEK PROSGEGRAMMENI +1FC2..1FC4 ; Changes_When_Casemapped # L& [3] GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI..GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI +1FC6..1FCC ; Changes_When_Casemapped # L& [7] GREEK SMALL LETTER ETA WITH PERISPOMENI..GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI +1FD0..1FD3 ; Changes_When_Casemapped # L& [4] GREEK SMALL LETTER IOTA WITH VRACHY..GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA +1FD6..1FDB ; Changes_When_Casemapped # L& [6] GREEK SMALL LETTER IOTA WITH PERISPOMENI..GREEK CAPITAL LETTER IOTA WITH OXIA +1FE0..1FEC ; Changes_When_Casemapped # L& [13] GREEK SMALL LETTER UPSILON WITH VRACHY..GREEK CAPITAL LETTER RHO WITH DASIA +1FF2..1FF4 ; Changes_When_Casemapped # L& [3] GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI..GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI +1FF6..1FFC ; Changes_When_Casemapped # L& [7] GREEK SMALL LETTER OMEGA WITH PERISPOMENI..GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI +2126 ; Changes_When_Casemapped # L& OHM SIGN +212A..212B ; Changes_When_Casemapped # L& [2] KELVIN SIGN..ANGSTROM SIGN +2132 ; Changes_When_Casemapped # L& TURNED CAPITAL F +214E ; Changes_When_Casemapped # L& TURNED SMALL F +2160..217F ; Changes_When_Casemapped # Nl [32] ROMAN NUMERAL ONE..SMALL ROMAN NUMERAL ONE THOUSAND +2183..2184 ; Changes_When_Casemapped # L& [2] ROMAN NUMERAL REVERSED ONE HUNDRED..LATIN SMALL LETTER REVERSED C +24B6..24E9 ; Changes_When_Casemapped # So [52] CIRCLED LATIN CAPITAL LETTER A..CIRCLED LATIN SMALL LETTER Z +2C00..2C2E ; Changes_When_Casemapped # L& [47] GLAGOLITIC CAPITAL LETTER AZU..GLAGOLITIC CAPITAL LETTER LATINATE MYSLITE +2C30..2C5E ; Changes_When_Casemapped # L& [47] GLAGOLITIC SMALL LETTER AZU..GLAGOLITIC SMALL LETTER LATINATE MYSLITE +2C60..2C70 ; Changes_When_Casemapped # L& [17] LATIN CAPITAL LETTER L WITH DOUBLE BAR..LATIN CAPITAL LETTER TURNED ALPHA +2C72..2C73 ; Changes_When_Casemapped # L& [2] LATIN CAPITAL LETTER W WITH HOOK..LATIN SMALL LETTER W WITH HOOK +2C75..2C76 ; Changes_When_Casemapped # L& [2] LATIN CAPITAL LETTER HALF H..LATIN SMALL LETTER HALF H +2C7E..2CE3 ; Changes_When_Casemapped # L& [102] LATIN CAPITAL LETTER S WITH SWASH TAIL..COPTIC SMALL LETTER OLD NUBIAN WAU +2CEB..2CEE ; Changes_When_Casemapped # L& [4] COPTIC CAPITAL LETTER CRYPTOGRAMMIC SHEI..COPTIC SMALL LETTER CRYPTOGRAMMIC GANGIA +2D00..2D25 ; Changes_When_Casemapped # L& [38] GEORGIAN SMALL LETTER AN..GEORGIAN SMALL LETTER HOE +A640..A65F ; Changes_When_Casemapped # L& [32] CYRILLIC CAPITAL LETTER ZEMLYA..CYRILLIC SMALL LETTER YN +A662..A66D ; Changes_When_Casemapped # L& [12] CYRILLIC CAPITAL LETTER SOFT DE..CYRILLIC SMALL LETTER DOUBLE MONOCULAR O +A680..A697 ; Changes_When_Casemapped # L& [24] CYRILLIC CAPITAL LETTER DWE..CYRILLIC SMALL LETTER SHWE +A722..A72F ; Changes_When_Casemapped # L& [14] LATIN CAPITAL LETTER EGYPTOLOGICAL ALEF..LATIN SMALL LETTER CUATRILLO WITH COMMA +A732..A76F ; Changes_When_Casemapped # L& [62] LATIN CAPITAL LETTER AA..LATIN SMALL LETTER CON +A779..A787 ; Changes_When_Casemapped # L& [15] LATIN CAPITAL LETTER INSULAR D..LATIN SMALL LETTER INSULAR T +A78B..A78C ; Changes_When_Casemapped # L& [2] LATIN CAPITAL LETTER SALTILLO..LATIN SMALL LETTER SALTILLO +FB00..FB06 ; Changes_When_Casemapped # L& [7] LATIN SMALL LIGATURE FF..LATIN SMALL LIGATURE ST +FB13..FB17 ; Changes_When_Casemapped # L& [5] ARMENIAN SMALL LIGATURE MEN NOW..ARMENIAN SMALL LIGATURE MEN XEH +FF21..FF3A ; Changes_When_Casemapped # L& [26] FULLWIDTH LATIN CAPITAL LETTER A..FULLWIDTH LATIN CAPITAL LETTER Z +FF41..FF5A ; Changes_When_Casemapped # L& [26] FULLWIDTH LATIN SMALL LETTER A..FULLWIDTH LATIN SMALL LETTER Z +10400..1044F ; Changes_When_Casemapped # L& [80] DESERET CAPITAL LETTER LONG I..DESERET SMALL LETTER EW + +# Total code points: 2110 + +# ================================================ + +# Derived Property: ID_Start +# Characters that can start an identifier. +# Generated from: +# Lu + Ll + Lt + Lm + Lo + Nl +# + Other_ID_Start +# - Pattern_Syntax +# - Pattern_White_Space +# NOTE: See UAX #31 for more information + +0041..005A ; ID_Start # L& [26] LATIN CAPITAL LETTER A..LATIN CAPITAL LETTER Z +0061..007A ; ID_Start # L& [26] LATIN SMALL LETTER A..LATIN SMALL LETTER Z +00AA ; ID_Start # L& FEMININE ORDINAL INDICATOR +00B5 ; ID_Start # L& MICRO SIGN +00BA ; ID_Start # L& MASCULINE ORDINAL INDICATOR +00C0..00D6 ; ID_Start # L& [23] LATIN CAPITAL LETTER A WITH GRAVE..LATIN CAPITAL LETTER O WITH DIAERESIS +00D8..00F6 ; ID_Start # L& [31] LATIN CAPITAL LETTER O WITH STROKE..LATIN SMALL LETTER O WITH DIAERESIS +00F8..01BA ; ID_Start # L& [195] LATIN SMALL LETTER O WITH STROKE..LATIN SMALL LETTER EZH WITH TAIL +01BB ; ID_Start # Lo LATIN LETTER TWO WITH STROKE +01BC..01BF ; ID_Start # L& [4] LATIN CAPITAL LETTER TONE FIVE..LATIN LETTER WYNN +01C0..01C3 ; ID_Start # Lo [4] LATIN LETTER DENTAL CLICK..LATIN LETTER RETROFLEX CLICK +01C4..0293 ; ID_Start # L& [208] LATIN CAPITAL LETTER DZ WITH CARON..LATIN SMALL LETTER EZH WITH CURL +0294 ; ID_Start # Lo LATIN LETTER GLOTTAL STOP +0295..02AF ; ID_Start # L& [27] LATIN LETTER PHARYNGEAL VOICED FRICATIVE..LATIN SMALL LETTER TURNED H WITH FISHHOOK AND TAIL +02B0..02C1 ; ID_Start # Lm [18] MODIFIER LETTER SMALL H..MODIFIER LETTER REVERSED GLOTTAL STOP +02C6..02D1 ; ID_Start # Lm [12] MODIFIER LETTER CIRCUMFLEX ACCENT..MODIFIER LETTER HALF TRIANGULAR COLON +02E0..02E4 ; ID_Start # Lm [5] MODIFIER LETTER SMALL GAMMA..MODIFIER LETTER SMALL REVERSED GLOTTAL STOP +02EC ; ID_Start # Lm MODIFIER LETTER VOICING +02EE ; ID_Start # Lm MODIFIER LETTER DOUBLE APOSTROPHE +0370..0373 ; ID_Start # L& [4] GREEK CAPITAL LETTER HETA..GREEK SMALL LETTER ARCHAIC SAMPI +0374 ; ID_Start # Lm GREEK NUMERAL SIGN +0376..0377 ; ID_Start # L& [2] GREEK CAPITAL LETTER PAMPHYLIAN DIGAMMA..GREEK SMALL LETTER PAMPHYLIAN DIGAMMA +037A ; ID_Start # Lm GREEK YPOGEGRAMMENI +037B..037D ; ID_Start # L& [3] GREEK SMALL REVERSED LUNATE SIGMA SYMBOL..GREEK SMALL REVERSED DOTTED LUNATE SIGMA SYMBOL +0386 ; ID_Start # L& GREEK CAPITAL LETTER ALPHA WITH TONOS +0388..038A ; ID_Start # L& [3] GREEK CAPITAL LETTER EPSILON WITH TONOS..GREEK CAPITAL LETTER IOTA WITH TONOS +038C ; ID_Start # L& GREEK CAPITAL LETTER OMICRON WITH TONOS +038E..03A1 ; ID_Start # L& [20] GREEK CAPITAL LETTER UPSILON WITH TONOS..GREEK CAPITAL LETTER RHO +03A3..03F5 ; ID_Start # L& [83] GREEK CAPITAL LETTER SIGMA..GREEK LUNATE EPSILON SYMBOL +03F7..0481 ; ID_Start # L& [139] GREEK CAPITAL LETTER SHO..CYRILLIC SMALL LETTER KOPPA +048A..0525 ; ID_Start # L& [156] CYRILLIC CAPITAL LETTER SHORT I WITH TAIL..CYRILLIC SMALL LETTER PE WITH DESCENDER +0531..0556 ; ID_Start # L& [38] ARMENIAN CAPITAL LETTER AYB..ARMENIAN CAPITAL LETTER FEH +0559 ; ID_Start # Lm ARMENIAN MODIFIER LETTER LEFT HALF RING +0561..0587 ; ID_Start # L& [39] ARMENIAN SMALL LETTER AYB..ARMENIAN SMALL LIGATURE ECH YIWN +05D0..05EA ; ID_Start # Lo [27] HEBREW LETTER ALEF..HEBREW LETTER TAV +05F0..05F2 ; ID_Start # Lo [3] HEBREW LIGATURE YIDDISH DOUBLE VAV..HEBREW LIGATURE YIDDISH DOUBLE YOD +0621..063F ; ID_Start # Lo [31] ARABIC LETTER HAMZA..ARABIC LETTER FARSI YEH WITH THREE DOTS ABOVE +0640 ; ID_Start # Lm ARABIC TATWEEL +0641..064A ; ID_Start # Lo [10] ARABIC LETTER FEH..ARABIC LETTER YEH +066E..066F ; ID_Start # Lo [2] ARABIC LETTER DOTLESS BEH..ARABIC LETTER DOTLESS QAF +0671..06D3 ; ID_Start # Lo [99] ARABIC LETTER ALEF WASLA..ARABIC LETTER YEH BARREE WITH HAMZA ABOVE +06D5 ; ID_Start # Lo ARABIC LETTER AE +06E5..06E6 ; ID_Start # Lm [2] ARABIC SMALL WAW..ARABIC SMALL YEH +06EE..06EF ; ID_Start # Lo [2] ARABIC LETTER DAL WITH INVERTED V..ARABIC LETTER REH WITH INVERTED V +06FA..06FC ; ID_Start # Lo [3] ARABIC LETTER SHEEN WITH DOT BELOW..ARABIC LETTER GHAIN WITH DOT BELOW +06FF ; ID_Start # Lo ARABIC LETTER HEH WITH INVERTED V +0710 ; ID_Start # Lo SYRIAC LETTER ALAPH +0712..072F ; ID_Start # Lo [30] SYRIAC LETTER BETH..SYRIAC LETTER PERSIAN DHALATH +074D..07A5 ; ID_Start # Lo [89] SYRIAC LETTER SOGDIAN ZHAIN..THAANA LETTER WAAVU +07B1 ; ID_Start # Lo THAANA LETTER NAA +07CA..07EA ; ID_Start # Lo [33] NKO LETTER A..NKO LETTER JONA RA +07F4..07F5 ; ID_Start # Lm [2] NKO HIGH TONE APOSTROPHE..NKO LOW TONE APOSTROPHE +07FA ; ID_Start # Lm NKO LAJANYALAN +0800..0815 ; ID_Start # Lo [22] SAMARITAN LETTER ALAF..SAMARITAN LETTER TAAF +081A ; ID_Start # Lm SAMARITAN MODIFIER LETTER EPENTHETIC YUT +0824 ; ID_Start # Lm SAMARITAN MODIFIER LETTER SHORT A +0828 ; ID_Start # Lm SAMARITAN MODIFIER LETTER I +0904..0939 ; ID_Start # Lo [54] DEVANAGARI LETTER SHORT A..DEVANAGARI LETTER HA +093D ; ID_Start # Lo DEVANAGARI SIGN AVAGRAHA +0950 ; ID_Start # Lo DEVANAGARI OM +0958..0961 ; ID_Start # Lo [10] DEVANAGARI LETTER QA..DEVANAGARI LETTER VOCALIC LL +0971 ; ID_Start # Lm DEVANAGARI SIGN HIGH SPACING DOT +0972 ; ID_Start # Lo DEVANAGARI LETTER CANDRA A +0979..097F ; ID_Start # Lo [7] DEVANAGARI LETTER ZHA..DEVANAGARI LETTER BBA +0985..098C ; ID_Start # Lo [8] BENGALI LETTER A..BENGALI LETTER VOCALIC L +098F..0990 ; ID_Start # Lo [2] BENGALI LETTER E..BENGALI LETTER AI +0993..09A8 ; ID_Start # Lo [22] BENGALI LETTER O..BENGALI LETTER NA +09AA..09B0 ; ID_Start # Lo [7] BENGALI LETTER PA..BENGALI LETTER RA +09B2 ; ID_Start # Lo BENGALI LETTER LA +09B6..09B9 ; ID_Start # Lo [4] BENGALI LETTER SHA..BENGALI LETTER HA +09BD ; ID_Start # Lo BENGALI SIGN AVAGRAHA +09CE ; ID_Start # Lo BENGALI LETTER KHANDA TA +09DC..09DD ; ID_Start # Lo [2] BENGALI LETTER RRA..BENGALI LETTER RHA +09DF..09E1 ; ID_Start # Lo [3] BENGALI LETTER YYA..BENGALI LETTER VOCALIC LL +09F0..09F1 ; ID_Start # Lo [2] BENGALI LETTER RA WITH MIDDLE DIAGONAL..BENGALI LETTER RA WITH LOWER DIAGONAL +0A05..0A0A ; ID_Start # Lo [6] GURMUKHI LETTER A..GURMUKHI LETTER UU +0A0F..0A10 ; ID_Start # Lo [2] GURMUKHI LETTER EE..GURMUKHI LETTER AI +0A13..0A28 ; ID_Start # Lo [22] GURMUKHI LETTER OO..GURMUKHI LETTER NA +0A2A..0A30 ; ID_Start # Lo [7] GURMUKHI LETTER PA..GURMUKHI LETTER RA +0A32..0A33 ; ID_Start # Lo [2] GURMUKHI LETTER LA..GURMUKHI LETTER LLA +0A35..0A36 ; ID_Start # Lo [2] GURMUKHI LETTER VA..GURMUKHI LETTER SHA +0A38..0A39 ; ID_Start # Lo [2] GURMUKHI LETTER SA..GURMUKHI LETTER HA +0A59..0A5C ; ID_Start # Lo [4] GURMUKHI LETTER KHHA..GURMUKHI LETTER RRA +0A5E ; ID_Start # Lo GURMUKHI LETTER FA +0A72..0A74 ; ID_Start # Lo [3] GURMUKHI IRI..GURMUKHI EK ONKAR +0A85..0A8D ; ID_Start # Lo [9] GUJARATI LETTER A..GUJARATI VOWEL CANDRA E +0A8F..0A91 ; ID_Start # Lo [3] GUJARATI LETTER E..GUJARATI VOWEL CANDRA O +0A93..0AA8 ; ID_Start # Lo [22] GUJARATI LETTER O..GUJARATI LETTER NA +0AAA..0AB0 ; ID_Start # Lo [7] GUJARATI LETTER PA..GUJARATI LETTER RA +0AB2..0AB3 ; ID_Start # Lo [2] GUJARATI LETTER LA..GUJARATI LETTER LLA +0AB5..0AB9 ; ID_Start # Lo [5] GUJARATI LETTER VA..GUJARATI LETTER HA +0ABD ; ID_Start # Lo GUJARATI SIGN AVAGRAHA +0AD0 ; ID_Start # Lo GUJARATI OM +0AE0..0AE1 ; ID_Start # Lo [2] GUJARATI LETTER VOCALIC RR..GUJARATI LETTER VOCALIC LL +0B05..0B0C ; ID_Start # Lo [8] ORIYA LETTER A..ORIYA LETTER VOCALIC L +0B0F..0B10 ; ID_Start # Lo [2] ORIYA LETTER E..ORIYA LETTER AI +0B13..0B28 ; ID_Start # Lo [22] ORIYA LETTER O..ORIYA LETTER NA +0B2A..0B30 ; ID_Start # Lo [7] ORIYA LETTER PA..ORIYA LETTER RA +0B32..0B33 ; ID_Start # Lo [2] ORIYA LETTER LA..ORIYA LETTER LLA +0B35..0B39 ; ID_Start # Lo [5] ORIYA LETTER VA..ORIYA LETTER HA +0B3D ; ID_Start # Lo ORIYA SIGN AVAGRAHA +0B5C..0B5D ; ID_Start # Lo [2] ORIYA LETTER RRA..ORIYA LETTER RHA +0B5F..0B61 ; ID_Start # Lo [3] ORIYA LETTER YYA..ORIYA LETTER VOCALIC LL +0B71 ; ID_Start # Lo ORIYA LETTER WA +0B83 ; ID_Start # Lo TAMIL SIGN VISARGA +0B85..0B8A ; ID_Start # Lo [6] TAMIL LETTER A..TAMIL LETTER UU +0B8E..0B90 ; ID_Start # Lo [3] TAMIL LETTER E..TAMIL LETTER AI +0B92..0B95 ; ID_Start # Lo [4] TAMIL LETTER O..TAMIL LETTER KA +0B99..0B9A ; ID_Start # Lo [2] TAMIL LETTER NGA..TAMIL LETTER CA +0B9C ; ID_Start # Lo TAMIL LETTER JA +0B9E..0B9F ; ID_Start # Lo [2] TAMIL LETTER NYA..TAMIL LETTER TTA +0BA3..0BA4 ; ID_Start # Lo [2] TAMIL LETTER NNA..TAMIL LETTER TA +0BA8..0BAA ; ID_Start # Lo [3] TAMIL LETTER NA..TAMIL LETTER PA +0BAE..0BB9 ; ID_Start # Lo [12] TAMIL LETTER MA..TAMIL LETTER HA +0BD0 ; ID_Start # Lo TAMIL OM +0C05..0C0C ; ID_Start # Lo [8] TELUGU LETTER A..TELUGU LETTER VOCALIC L +0C0E..0C10 ; ID_Start # Lo [3] TELUGU LETTER E..TELUGU LETTER AI +0C12..0C28 ; ID_Start # Lo [23] TELUGU LETTER O..TELUGU LETTER NA +0C2A..0C33 ; ID_Start # Lo [10] TELUGU LETTER PA..TELUGU LETTER LLA +0C35..0C39 ; ID_Start # Lo [5] TELUGU LETTER VA..TELUGU LETTER HA +0C3D ; ID_Start # Lo TELUGU SIGN AVAGRAHA +0C58..0C59 ; ID_Start # Lo [2] TELUGU LETTER TSA..TELUGU LETTER DZA +0C60..0C61 ; ID_Start # Lo [2] TELUGU LETTER VOCALIC RR..TELUGU LETTER VOCALIC LL +0C85..0C8C ; ID_Start # Lo [8] KANNADA LETTER A..KANNADA LETTER VOCALIC L +0C8E..0C90 ; ID_Start # Lo [3] KANNADA LETTER E..KANNADA LETTER AI +0C92..0CA8 ; ID_Start # Lo [23] KANNADA LETTER O..KANNADA LETTER NA +0CAA..0CB3 ; ID_Start # Lo [10] KANNADA LETTER PA..KANNADA LETTER LLA +0CB5..0CB9 ; ID_Start # Lo [5] KANNADA LETTER VA..KANNADA LETTER HA +0CBD ; ID_Start # Lo KANNADA SIGN AVAGRAHA +0CDE ; ID_Start # Lo KANNADA LETTER FA +0CE0..0CE1 ; ID_Start # Lo [2] KANNADA LETTER VOCALIC RR..KANNADA LETTER VOCALIC LL +0D05..0D0C ; ID_Start # Lo [8] MALAYALAM LETTER A..MALAYALAM LETTER VOCALIC L +0D0E..0D10 ; ID_Start # Lo [3] MALAYALAM LETTER E..MALAYALAM LETTER AI +0D12..0D28 ; ID_Start # Lo [23] MALAYALAM LETTER O..MALAYALAM LETTER NA +0D2A..0D39 ; ID_Start # Lo [16] MALAYALAM LETTER PA..MALAYALAM LETTER HA +0D3D ; ID_Start # Lo MALAYALAM SIGN AVAGRAHA +0D60..0D61 ; ID_Start # Lo [2] MALAYALAM LETTER VOCALIC RR..MALAYALAM LETTER VOCALIC LL +0D7A..0D7F ; ID_Start # Lo [6] MALAYALAM LETTER CHILLU NN..MALAYALAM LETTER CHILLU K +0D85..0D96 ; ID_Start # Lo [18] SINHALA LETTER AYANNA..SINHALA LETTER AUYANNA +0D9A..0DB1 ; ID_Start # Lo [24] SINHALA LETTER ALPAPRAANA KAYANNA..SINHALA LETTER DANTAJA NAYANNA +0DB3..0DBB ; ID_Start # Lo [9] SINHALA LETTER SANYAKA DAYANNA..SINHALA LETTER RAYANNA +0DBD ; ID_Start # Lo SINHALA LETTER DANTAJA LAYANNA +0DC0..0DC6 ; ID_Start # Lo [7] SINHALA LETTER VAYANNA..SINHALA LETTER FAYANNA +0E01..0E30 ; ID_Start # Lo [48] THAI CHARACTER KO KAI..THAI CHARACTER SARA A +0E32..0E33 ; ID_Start # Lo [2] THAI CHARACTER SARA AA..THAI CHARACTER SARA AM +0E40..0E45 ; ID_Start # Lo [6] THAI CHARACTER SARA E..THAI CHARACTER LAKKHANGYAO +0E46 ; ID_Start # Lm THAI CHARACTER MAIYAMOK +0E81..0E82 ; ID_Start # Lo [2] LAO LETTER KO..LAO LETTER KHO SUNG +0E84 ; ID_Start # Lo LAO LETTER KHO TAM +0E87..0E88 ; ID_Start # Lo [2] LAO LETTER NGO..LAO LETTER CO +0E8A ; ID_Start # Lo LAO LETTER SO TAM +0E8D ; ID_Start # Lo LAO LETTER NYO +0E94..0E97 ; ID_Start # Lo [4] LAO LETTER DO..LAO LETTER THO TAM +0E99..0E9F ; ID_Start # Lo [7] LAO LETTER NO..LAO LETTER FO SUNG +0EA1..0EA3 ; ID_Start # Lo [3] LAO LETTER MO..LAO LETTER LO LING +0EA5 ; ID_Start # Lo LAO LETTER LO LOOT +0EA7 ; ID_Start # Lo LAO LETTER WO +0EAA..0EAB ; ID_Start # Lo [2] LAO LETTER SO SUNG..LAO LETTER HO SUNG +0EAD..0EB0 ; ID_Start # Lo [4] LAO LETTER O..LAO VOWEL SIGN A +0EB2..0EB3 ; ID_Start # Lo [2] LAO VOWEL SIGN AA..LAO VOWEL SIGN AM +0EBD ; ID_Start # Lo LAO SEMIVOWEL SIGN NYO +0EC0..0EC4 ; ID_Start # Lo [5] LAO VOWEL SIGN E..LAO VOWEL SIGN AI +0EC6 ; ID_Start # Lm LAO KO LA +0EDC..0EDD ; ID_Start # Lo [2] LAO HO NO..LAO HO MO +0F00 ; ID_Start # Lo TIBETAN SYLLABLE OM +0F40..0F47 ; ID_Start # Lo [8] TIBETAN LETTER KA..TIBETAN LETTER JA +0F49..0F6C ; ID_Start # Lo [36] TIBETAN LETTER NYA..TIBETAN LETTER RRA +0F88..0F8B ; ID_Start # Lo [4] TIBETAN SIGN LCE TSA CAN..TIBETAN SIGN GRU MED RGYINGS +1000..102A ; ID_Start # Lo [43] MYANMAR LETTER KA..MYANMAR LETTER AU +103F ; ID_Start # Lo MYANMAR LETTER GREAT SA +1050..1055 ; ID_Start # Lo [6] MYANMAR LETTER SHA..MYANMAR LETTER VOCALIC LL +105A..105D ; ID_Start # Lo [4] MYANMAR LETTER MON NGA..MYANMAR LETTER MON BBE +1061 ; ID_Start # Lo MYANMAR LETTER SGAW KAREN SHA +1065..1066 ; ID_Start # Lo [2] MYANMAR LETTER WESTERN PWO KAREN THA..MYANMAR LETTER WESTERN PWO KAREN PWA +106E..1070 ; ID_Start # Lo [3] MYANMAR LETTER EASTERN PWO KAREN NNA..MYANMAR LETTER EASTERN PWO KAREN GHWA +1075..1081 ; ID_Start # Lo [13] MYANMAR LETTER SHAN KA..MYANMAR LETTER SHAN HA +108E ; ID_Start # Lo MYANMAR LETTER RUMAI PALAUNG FA +10A0..10C5 ; ID_Start # L& [38] GEORGIAN CAPITAL LETTER AN..GEORGIAN CAPITAL LETTER HOE +10D0..10FA ; ID_Start # Lo [43] GEORGIAN LETTER AN..GEORGIAN LETTER AIN +10FC ; ID_Start # Lm MODIFIER LETTER GEORGIAN NAR +1100..1248 ; ID_Start # Lo [329] HANGUL CHOSEONG KIYEOK..ETHIOPIC SYLLABLE QWA +124A..124D ; ID_Start # Lo [4] ETHIOPIC SYLLABLE QWI..ETHIOPIC SYLLABLE QWE +1250..1256 ; ID_Start # Lo [7] ETHIOPIC SYLLABLE QHA..ETHIOPIC SYLLABLE QHO +1258 ; ID_Start # Lo ETHIOPIC SYLLABLE QHWA +125A..125D ; ID_Start # Lo [4] ETHIOPIC SYLLABLE QHWI..ETHIOPIC SYLLABLE QHWE +1260..1288 ; ID_Start # Lo [41] ETHIOPIC SYLLABLE BA..ETHIOPIC SYLLABLE XWA +128A..128D ; ID_Start # Lo [4] ETHIOPIC SYLLABLE XWI..ETHIOPIC SYLLABLE XWE +1290..12B0 ; ID_Start # Lo [33] ETHIOPIC SYLLABLE NA..ETHIOPIC SYLLABLE KWA +12B2..12B5 ; ID_Start # Lo [4] ETHIOPIC SYLLABLE KWI..ETHIOPIC SYLLABLE KWE +12B8..12BE ; ID_Start # Lo [7] ETHIOPIC SYLLABLE KXA..ETHIOPIC SYLLABLE KXO +12C0 ; ID_Start # Lo ETHIOPIC SYLLABLE KXWA +12C2..12C5 ; ID_Start # Lo [4] ETHIOPIC SYLLABLE KXWI..ETHIOPIC SYLLABLE KXWE +12C8..12D6 ; ID_Start # Lo [15] ETHIOPIC SYLLABLE WA..ETHIOPIC SYLLABLE PHARYNGEAL O +12D8..1310 ; ID_Start # Lo [57] ETHIOPIC SYLLABLE ZA..ETHIOPIC SYLLABLE GWA +1312..1315 ; ID_Start # Lo [4] ETHIOPIC SYLLABLE GWI..ETHIOPIC SYLLABLE GWE +1318..135A ; ID_Start # Lo [67] ETHIOPIC SYLLABLE GGA..ETHIOPIC SYLLABLE FYA +1380..138F ; ID_Start # Lo [16] ETHIOPIC SYLLABLE SEBATBEIT MWA..ETHIOPIC SYLLABLE PWE +13A0..13F4 ; ID_Start # Lo [85] CHEROKEE LETTER A..CHEROKEE LETTER YV +1401..166C ; ID_Start # Lo [620] CANADIAN SYLLABICS E..CANADIAN SYLLABICS CARRIER TTSA +166F..167F ; ID_Start # Lo [17] CANADIAN SYLLABICS QAI..CANADIAN SYLLABICS BLACKFOOT W +1681..169A ; ID_Start # Lo [26] OGHAM LETTER BEITH..OGHAM LETTER PEITH +16A0..16EA ; ID_Start # Lo [75] RUNIC LETTER FEHU FEOH FE F..RUNIC LETTER X +16EE..16F0 ; ID_Start # Nl [3] RUNIC ARLAUG SYMBOL..RUNIC BELGTHOR SYMBOL +1700..170C ; ID_Start # Lo [13] TAGALOG LETTER A..TAGALOG LETTER YA +170E..1711 ; ID_Start # Lo [4] TAGALOG LETTER LA..TAGALOG LETTER HA +1720..1731 ; ID_Start # Lo [18] HANUNOO LETTER A..HANUNOO LETTER HA +1740..1751 ; ID_Start # Lo [18] BUHID LETTER A..BUHID LETTER HA +1760..176C ; ID_Start # Lo [13] TAGBANWA LETTER A..TAGBANWA LETTER YA +176E..1770 ; ID_Start # Lo [3] TAGBANWA LETTER LA..TAGBANWA LETTER SA +1780..17B3 ; ID_Start # Lo [52] KHMER LETTER KA..KHMER INDEPENDENT VOWEL QAU +17D7 ; ID_Start # Lm KHMER SIGN LEK TOO +17DC ; ID_Start # Lo KHMER SIGN AVAKRAHASANYA +1820..1842 ; ID_Start # Lo [35] MONGOLIAN LETTER A..MONGOLIAN LETTER CHI +1843 ; ID_Start # Lm MONGOLIAN LETTER TODO LONG VOWEL SIGN +1844..1877 ; ID_Start # Lo [52] MONGOLIAN LETTER TODO E..MONGOLIAN LETTER MANCHU ZHA +1880..18A8 ; ID_Start # Lo [41] MONGOLIAN LETTER ALI GALI ANUSVARA ONE..MONGOLIAN LETTER MANCHU ALI GALI BHA +18AA ; ID_Start # Lo MONGOLIAN LETTER MANCHU ALI GALI LHA +18B0..18F5 ; ID_Start # Lo [70] CANADIAN SYLLABICS OY..CANADIAN SYLLABICS CARRIER DENTAL S +1900..191C ; ID_Start # Lo [29] LIMBU VOWEL-CARRIER LETTER..LIMBU LETTER HA +1950..196D ; ID_Start # Lo [30] TAI LE LETTER KA..TAI LE LETTER AI +1970..1974 ; ID_Start # Lo [5] TAI LE LETTER TONE-2..TAI LE LETTER TONE-6 +1980..19AB ; ID_Start # Lo [44] NEW TAI LUE LETTER HIGH QA..NEW TAI LUE LETTER LOW SUA +19C1..19C7 ; ID_Start # Lo [7] NEW TAI LUE LETTER FINAL V..NEW TAI LUE LETTER FINAL B +1A00..1A16 ; ID_Start # Lo [23] BUGINESE LETTER KA..BUGINESE LETTER HA +1A20..1A54 ; ID_Start # Lo [53] TAI THAM LETTER HIGH KA..TAI THAM LETTER GREAT SA +1AA7 ; ID_Start # Lm TAI THAM SIGN MAI YAMOK +1B05..1B33 ; ID_Start # Lo [47] BALINESE LETTER AKARA..BALINESE LETTER HA +1B45..1B4B ; ID_Start # Lo [7] BALINESE LETTER KAF SASAK..BALINESE LETTER ASYURA SASAK +1B83..1BA0 ; ID_Start # Lo [30] SUNDANESE LETTER A..SUNDANESE LETTER HA +1BAE..1BAF ; ID_Start # Lo [2] SUNDANESE LETTER KHA..SUNDANESE LETTER SYA +1C00..1C23 ; ID_Start # Lo [36] LEPCHA LETTER KA..LEPCHA LETTER A +1C4D..1C4F ; ID_Start # Lo [3] LEPCHA LETTER TTA..LEPCHA LETTER DDA +1C5A..1C77 ; ID_Start # Lo [30] OL CHIKI LETTER LA..OL CHIKI LETTER OH +1C78..1C7D ; ID_Start # Lm [6] OL CHIKI MU TTUDDAG..OL CHIKI AHAD +1CE9..1CEC ; ID_Start # Lo [4] VEDIC SIGN ANUSVARA ANTARGOMUKHA..VEDIC SIGN ANUSVARA VAMAGOMUKHA WITH TAIL +1CEE..1CF1 ; ID_Start # Lo [4] VEDIC SIGN HEXIFORM LONG ANUSVARA..VEDIC SIGN ANUSVARA UBHAYATO MUKHA +1D00..1D2B ; ID_Start # L& [44] LATIN LETTER SMALL CAPITAL A..CYRILLIC LETTER SMALL CAPITAL EL +1D2C..1D61 ; ID_Start # Lm [54] MODIFIER LETTER CAPITAL A..MODIFIER LETTER SMALL CHI +1D62..1D77 ; ID_Start # L& [22] LATIN SUBSCRIPT SMALL LETTER I..LATIN SMALL LETTER TURNED G +1D78 ; ID_Start # Lm MODIFIER LETTER CYRILLIC EN +1D79..1D9A ; ID_Start # L& [34] LATIN SMALL LETTER INSULAR G..LATIN SMALL LETTER EZH WITH RETROFLEX HOOK +1D9B..1DBF ; ID_Start # Lm [37] MODIFIER LETTER SMALL TURNED ALPHA..MODIFIER LETTER SMALL THETA +1E00..1F15 ; ID_Start # L& [278] LATIN CAPITAL LETTER A WITH RING BELOW..GREEK SMALL LETTER EPSILON WITH DASIA AND OXIA +1F18..1F1D ; ID_Start # L& [6] GREEK CAPITAL LETTER EPSILON WITH PSILI..GREEK CAPITAL LETTER EPSILON WITH DASIA AND OXIA +1F20..1F45 ; ID_Start # L& [38] GREEK SMALL LETTER ETA WITH PSILI..GREEK SMALL LETTER OMICRON WITH DASIA AND OXIA +1F48..1F4D ; ID_Start # L& [6] GREEK CAPITAL LETTER OMICRON WITH PSILI..GREEK CAPITAL LETTER OMICRON WITH DASIA AND OXIA +1F50..1F57 ; ID_Start # L& [8] GREEK SMALL LETTER UPSILON WITH PSILI..GREEK SMALL LETTER UPSILON WITH DASIA AND PERISPOMENI +1F59 ; ID_Start # L& GREEK CAPITAL LETTER UPSILON WITH DASIA +1F5B ; ID_Start # L& GREEK CAPITAL LETTER UPSILON WITH DASIA AND VARIA +1F5D ; ID_Start # L& GREEK CAPITAL LETTER UPSILON WITH DASIA AND OXIA +1F5F..1F7D ; ID_Start # L& [31] GREEK CAPITAL LETTER UPSILON WITH DASIA AND PERISPOMENI..GREEK SMALL LETTER OMEGA WITH OXIA +1F80..1FB4 ; ID_Start # L& [53] GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI..GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI +1FB6..1FBC ; ID_Start # L& [7] GREEK SMALL LETTER ALPHA WITH PERISPOMENI..GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI +1FBE ; ID_Start # L& GREEK PROSGEGRAMMENI +1FC2..1FC4 ; ID_Start # L& [3] GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI..GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI +1FC6..1FCC ; ID_Start # L& [7] GREEK SMALL LETTER ETA WITH PERISPOMENI..GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI +1FD0..1FD3 ; ID_Start # L& [4] GREEK SMALL LETTER IOTA WITH VRACHY..GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA +1FD6..1FDB ; ID_Start # L& [6] GREEK SMALL LETTER IOTA WITH PERISPOMENI..GREEK CAPITAL LETTER IOTA WITH OXIA +1FE0..1FEC ; ID_Start # L& [13] GREEK SMALL LETTER UPSILON WITH VRACHY..GREEK CAPITAL LETTER RHO WITH DASIA +1FF2..1FF4 ; ID_Start # L& [3] GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI..GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI +1FF6..1FFC ; ID_Start # L& [7] GREEK SMALL LETTER OMEGA WITH PERISPOMENI..GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI +2071 ; ID_Start # Lm SUPERSCRIPT LATIN SMALL LETTER I +207F ; ID_Start # Lm SUPERSCRIPT LATIN SMALL LETTER N +2090..2094 ; ID_Start # Lm [5] LATIN SUBSCRIPT SMALL LETTER A..LATIN SUBSCRIPT SMALL LETTER SCHWA +2102 ; ID_Start # L& DOUBLE-STRUCK CAPITAL C +2107 ; ID_Start # L& EULER CONSTANT +210A..2113 ; ID_Start # L& [10] SCRIPT SMALL G..SCRIPT SMALL L +2115 ; ID_Start # L& DOUBLE-STRUCK CAPITAL N +2118 ; ID_Start # So SCRIPT CAPITAL P +2119..211D ; ID_Start # L& [5] DOUBLE-STRUCK CAPITAL P..DOUBLE-STRUCK CAPITAL R +2124 ; ID_Start # L& DOUBLE-STRUCK CAPITAL Z +2126 ; ID_Start # L& OHM SIGN +2128 ; ID_Start # L& BLACK-LETTER CAPITAL Z +212A..212D ; ID_Start # L& [4] KELVIN SIGN..BLACK-LETTER CAPITAL C +212E ; ID_Start # So ESTIMATED SYMBOL +212F..2134 ; ID_Start # L& [6] SCRIPT SMALL E..SCRIPT SMALL O +2135..2138 ; ID_Start # Lo [4] ALEF SYMBOL..DALET SYMBOL +2139 ; ID_Start # L& INFORMATION SOURCE +213C..213F ; ID_Start # L& [4] DOUBLE-STRUCK SMALL PI..DOUBLE-STRUCK CAPITAL PI +2145..2149 ; ID_Start # L& [5] DOUBLE-STRUCK ITALIC CAPITAL D..DOUBLE-STRUCK ITALIC SMALL J +214E ; ID_Start # L& TURNED SMALL F +2160..2182 ; ID_Start # Nl [35] ROMAN NUMERAL ONE..ROMAN NUMERAL TEN THOUSAND +2183..2184 ; ID_Start # L& [2] ROMAN NUMERAL REVERSED ONE HUNDRED..LATIN SMALL LETTER REVERSED C +2185..2188 ; ID_Start # Nl [4] ROMAN NUMERAL SIX LATE FORM..ROMAN NUMERAL ONE HUNDRED THOUSAND +2C00..2C2E ; ID_Start # L& [47] GLAGOLITIC CAPITAL LETTER AZU..GLAGOLITIC CAPITAL LETTER LATINATE MYSLITE +2C30..2C5E ; ID_Start # L& [47] GLAGOLITIC SMALL LETTER AZU..GLAGOLITIC SMALL LETTER LATINATE MYSLITE +2C60..2C7C ; ID_Start # L& [29] LATIN CAPITAL LETTER L WITH DOUBLE BAR..LATIN SUBSCRIPT SMALL LETTER J +2C7D ; ID_Start # Lm MODIFIER LETTER CAPITAL V +2C7E..2CE4 ; ID_Start # L& [103] LATIN CAPITAL LETTER S WITH SWASH TAIL..COPTIC SYMBOL KAI +2CEB..2CEE ; ID_Start # L& [4] COPTIC CAPITAL LETTER CRYPTOGRAMMIC SHEI..COPTIC SMALL LETTER CRYPTOGRAMMIC GANGIA +2D00..2D25 ; ID_Start # L& [38] GEORGIAN SMALL LETTER AN..GEORGIAN SMALL LETTER HOE +2D30..2D65 ; ID_Start # Lo [54] TIFINAGH LETTER YA..TIFINAGH LETTER YAZZ +2D6F ; ID_Start # Lm TIFINAGH MODIFIER LETTER LABIALIZATION MARK +2D80..2D96 ; ID_Start # Lo [23] ETHIOPIC SYLLABLE LOA..ETHIOPIC SYLLABLE GGWE +2DA0..2DA6 ; ID_Start # Lo [7] ETHIOPIC SYLLABLE SSA..ETHIOPIC SYLLABLE SSO +2DA8..2DAE ; ID_Start # Lo [7] ETHIOPIC SYLLABLE CCA..ETHIOPIC SYLLABLE CCO +2DB0..2DB6 ; ID_Start # Lo [7] ETHIOPIC SYLLABLE ZZA..ETHIOPIC SYLLABLE ZZO +2DB8..2DBE ; ID_Start # Lo [7] ETHIOPIC SYLLABLE CCHA..ETHIOPIC SYLLABLE CCHO +2DC0..2DC6 ; ID_Start # Lo [7] ETHIOPIC SYLLABLE QYA..ETHIOPIC SYLLABLE QYO +2DC8..2DCE ; ID_Start # Lo [7] ETHIOPIC SYLLABLE KYA..ETHIOPIC SYLLABLE KYO +2DD0..2DD6 ; ID_Start # Lo [7] ETHIOPIC SYLLABLE XYA..ETHIOPIC SYLLABLE XYO +2DD8..2DDE ; ID_Start # Lo [7] ETHIOPIC SYLLABLE GYA..ETHIOPIC SYLLABLE GYO +3005 ; ID_Start # Lm IDEOGRAPHIC ITERATION MARK +3006 ; ID_Start # Lo IDEOGRAPHIC CLOSING MARK +3007 ; ID_Start # Nl IDEOGRAPHIC NUMBER ZERO +3021..3029 ; ID_Start # Nl [9] HANGZHOU NUMERAL ONE..HANGZHOU NUMERAL NINE +3031..3035 ; ID_Start # Lm [5] VERTICAL KANA REPEAT MARK..VERTICAL KANA REPEAT MARK LOWER HALF +3038..303A ; ID_Start # Nl [3] HANGZHOU NUMERAL TEN..HANGZHOU NUMERAL THIRTY +303B ; ID_Start # Lm VERTICAL IDEOGRAPHIC ITERATION MARK +303C ; ID_Start # Lo MASU MARK +3041..3096 ; ID_Start # Lo [86] HIRAGANA LETTER SMALL A..HIRAGANA LETTER SMALL KE +309B..309C ; ID_Start # Sk [2] KATAKANA-HIRAGANA VOICED SOUND MARK..KATAKANA-HIRAGANA SEMI-VOICED SOUND MARK +309D..309E ; ID_Start # Lm [2] HIRAGANA ITERATION MARK..HIRAGANA VOICED ITERATION MARK +309F ; ID_Start # Lo HIRAGANA DIGRAPH YORI +30A1..30FA ; ID_Start # Lo [90] KATAKANA LETTER SMALL A..KATAKANA LETTER VO +30FC..30FE ; ID_Start # Lm [3] KATAKANA-HIRAGANA PROLONGED SOUND MARK..KATAKANA VOICED ITERATION MARK +30FF ; ID_Start # Lo KATAKANA DIGRAPH KOTO +3105..312D ; ID_Start # Lo [41] BOPOMOFO LETTER B..BOPOMOFO LETTER IH +3131..318E ; ID_Start # Lo [94] HANGUL LETTER KIYEOK..HANGUL LETTER ARAEAE +31A0..31B7 ; ID_Start # Lo [24] BOPOMOFO LETTER BU..BOPOMOFO FINAL LETTER H +31F0..31FF ; ID_Start # Lo [16] KATAKANA LETTER SMALL KU..KATAKANA LETTER SMALL RO +3400..4DB5 ; ID_Start # Lo [6582] CJK UNIFIED IDEOGRAPH-3400..CJK UNIFIED IDEOGRAPH-4DB5 +4E00..9FCB ; ID_Start # Lo [20940] CJK UNIFIED IDEOGRAPH-4E00..CJK UNIFIED IDEOGRAPH-9FCB +A000..A014 ; ID_Start # Lo [21] YI SYLLABLE IT..YI SYLLABLE E +A015 ; ID_Start # Lm YI SYLLABLE WU +A016..A48C ; ID_Start # Lo [1143] YI SYLLABLE BIT..YI SYLLABLE YYR +A4D0..A4F7 ; ID_Start # Lo [40] LISU LETTER BA..LISU LETTER OE +A4F8..A4FD ; ID_Start # Lm [6] LISU LETTER TONE MYA TI..LISU LETTER TONE MYA JEU +A500..A60B ; ID_Start # Lo [268] VAI SYLLABLE EE..VAI SYLLABLE NG +A60C ; ID_Start # Lm VAI SYLLABLE LENGTHENER +A610..A61F ; ID_Start # Lo [16] VAI SYLLABLE NDOLE FA..VAI SYMBOL JONG +A62A..A62B ; ID_Start # Lo [2] VAI SYLLABLE NDOLE MA..VAI SYLLABLE NDOLE DO +A640..A65F ; ID_Start # L& [32] CYRILLIC CAPITAL LETTER ZEMLYA..CYRILLIC SMALL LETTER YN +A662..A66D ; ID_Start # L& [12] CYRILLIC CAPITAL LETTER SOFT DE..CYRILLIC SMALL LETTER DOUBLE MONOCULAR O +A66E ; ID_Start # Lo CYRILLIC LETTER MULTIOCULAR O +A67F ; ID_Start # Lm CYRILLIC PAYEROK +A680..A697 ; ID_Start # L& [24] CYRILLIC CAPITAL LETTER DWE..CYRILLIC SMALL LETTER SHWE +A6A0..A6E5 ; ID_Start # Lo [70] BAMUM LETTER A..BAMUM LETTER KI +A6E6..A6EF ; ID_Start # Nl [10] BAMUM LETTER MO..BAMUM LETTER KOGHOM +A717..A71F ; ID_Start # Lm [9] MODIFIER LETTER DOT VERTICAL BAR..MODIFIER LETTER LOW INVERTED EXCLAMATION MARK +A722..A76F ; ID_Start # L& [78] LATIN CAPITAL LETTER EGYPTOLOGICAL ALEF..LATIN SMALL LETTER CON +A770 ; ID_Start # Lm MODIFIER LETTER US +A771..A787 ; ID_Start # L& [23] LATIN SMALL LETTER DUM..LATIN SMALL LETTER INSULAR T +A788 ; ID_Start # Lm MODIFIER LETTER LOW CIRCUMFLEX ACCENT +A78B..A78C ; ID_Start # L& [2] LATIN CAPITAL LETTER SALTILLO..LATIN SMALL LETTER SALTILLO +A7FB..A801 ; ID_Start # Lo [7] LATIN EPIGRAPHIC LETTER REVERSED F..SYLOTI NAGRI LETTER I +A803..A805 ; ID_Start # Lo [3] SYLOTI NAGRI LETTER U..SYLOTI NAGRI LETTER O +A807..A80A ; ID_Start # Lo [4] SYLOTI NAGRI LETTER KO..SYLOTI NAGRI LETTER GHO +A80C..A822 ; ID_Start # Lo [23] SYLOTI NAGRI LETTER CO..SYLOTI NAGRI LETTER HO +A840..A873 ; ID_Start # Lo [52] PHAGS-PA LETTER KA..PHAGS-PA LETTER CANDRABINDU +A882..A8B3 ; ID_Start # Lo [50] SAURASHTRA LETTER A..SAURASHTRA LETTER LLA +A8F2..A8F7 ; ID_Start # Lo [6] DEVANAGARI SIGN SPACING CANDRABINDU..DEVANAGARI SIGN CANDRABINDU AVAGRAHA +A8FB ; ID_Start # Lo DEVANAGARI HEADSTROKE +A90A..A925 ; ID_Start # Lo [28] KAYAH LI LETTER KA..KAYAH LI LETTER OO +A930..A946 ; ID_Start # Lo [23] REJANG LETTER KA..REJANG LETTER A +A960..A97C ; ID_Start # Lo [29] HANGUL CHOSEONG TIKEUT-MIEUM..HANGUL CHOSEONG SSANGYEORINHIEUH +A984..A9B2 ; ID_Start # Lo [47] JAVANESE LETTER A..JAVANESE LETTER HA +A9CF ; ID_Start # Lm JAVANESE PANGRANGKEP +AA00..AA28 ; ID_Start # Lo [41] CHAM LETTER A..CHAM LETTER HA +AA40..AA42 ; ID_Start # Lo [3] CHAM LETTER FINAL K..CHAM LETTER FINAL NG +AA44..AA4B ; ID_Start # Lo [8] CHAM LETTER FINAL CH..CHAM LETTER FINAL SS +AA60..AA6F ; ID_Start # Lo [16] MYANMAR LETTER KHAMTI GA..MYANMAR LETTER KHAMTI FA +AA70 ; ID_Start # Lm MYANMAR MODIFIER LETTER KHAMTI REDUPLICATION +AA71..AA76 ; ID_Start # Lo [6] MYANMAR LETTER KHAMTI XA..MYANMAR LOGOGRAM KHAMTI HM +AA7A ; ID_Start # Lo MYANMAR LETTER AITON RA +AA80..AAAF ; ID_Start # Lo [48] TAI VIET LETTER LOW KO..TAI VIET LETTER HIGH O +AAB1 ; ID_Start # Lo TAI VIET VOWEL AA +AAB5..AAB6 ; ID_Start # Lo [2] TAI VIET VOWEL E..TAI VIET VOWEL O +AAB9..AABD ; ID_Start # Lo [5] TAI VIET VOWEL UEA..TAI VIET VOWEL AN +AAC0 ; ID_Start # Lo TAI VIET TONE MAI NUENG +AAC2 ; ID_Start # Lo TAI VIET TONE MAI SONG +AADB..AADC ; ID_Start # Lo [2] TAI VIET SYMBOL KON..TAI VIET SYMBOL NUENG +AADD ; ID_Start # Lm TAI VIET SYMBOL SAM +ABC0..ABE2 ; ID_Start # Lo [35] MEETEI MAYEK LETTER KOK..MEETEI MAYEK LETTER I LONSUM +AC00..D7A3 ; ID_Start # Lo [11172] HANGUL SYLLABLE GA..HANGUL SYLLABLE HIH +D7B0..D7C6 ; ID_Start # Lo [23] HANGUL JUNGSEONG O-YEO..HANGUL JUNGSEONG ARAEA-E +D7CB..D7FB ; ID_Start # Lo [49] HANGUL JONGSEONG NIEUN-RIEUL..HANGUL JONGSEONG PHIEUPH-THIEUTH +F900..FA2D ; ID_Start # Lo [302] CJK COMPATIBILITY IDEOGRAPH-F900..CJK COMPATIBILITY IDEOGRAPH-FA2D +FA30..FA6D ; ID_Start # Lo [62] CJK COMPATIBILITY IDEOGRAPH-FA30..CJK COMPATIBILITY IDEOGRAPH-FA6D +FA70..FAD9 ; ID_Start # Lo [106] CJK COMPATIBILITY IDEOGRAPH-FA70..CJK COMPATIBILITY IDEOGRAPH-FAD9 +FB00..FB06 ; ID_Start # L& [7] LATIN SMALL LIGATURE FF..LATIN SMALL LIGATURE ST +FB13..FB17 ; ID_Start # L& [5] ARMENIAN SMALL LIGATURE MEN NOW..ARMENIAN SMALL LIGATURE MEN XEH +FB1D ; ID_Start # Lo HEBREW LETTER YOD WITH HIRIQ +FB1F..FB28 ; ID_Start # Lo [10] HEBREW LIGATURE YIDDISH YOD YOD PATAH..HEBREW LETTER WIDE TAV +FB2A..FB36 ; ID_Start # Lo [13] HEBREW LETTER SHIN WITH SHIN DOT..HEBREW LETTER ZAYIN WITH DAGESH +FB38..FB3C ; ID_Start # Lo [5] HEBREW LETTER TET WITH DAGESH..HEBREW LETTER LAMED WITH DAGESH +FB3E ; ID_Start # Lo HEBREW LETTER MEM WITH DAGESH +FB40..FB41 ; ID_Start # Lo [2] HEBREW LETTER NUN WITH DAGESH..HEBREW LETTER SAMEKH WITH DAGESH +FB43..FB44 ; ID_Start # Lo [2] HEBREW LETTER FINAL PE WITH DAGESH..HEBREW LETTER PE WITH DAGESH +FB46..FBB1 ; ID_Start # Lo [108] HEBREW LETTER TSADI WITH DAGESH..ARABIC LETTER YEH BARREE WITH HAMZA ABOVE FINAL FORM +FBD3..FD3D ; ID_Start # Lo [363] ARABIC LETTER NG ISOLATED FORM..ARABIC LIGATURE ALEF WITH FATHATAN ISOLATED FORM +FD50..FD8F ; ID_Start # Lo [64] ARABIC LIGATURE TEH WITH JEEM WITH MEEM INITIAL FORM..ARABIC LIGATURE MEEM WITH KHAH WITH MEEM INITIAL FORM +FD92..FDC7 ; ID_Start # Lo [54] ARABIC LIGATURE MEEM WITH JEEM WITH KHAH INITIAL FORM..ARABIC LIGATURE NOON WITH JEEM WITH YEH FINAL FORM +FDF0..FDFB ; ID_Start # Lo [12] ARABIC LIGATURE SALLA USED AS KORANIC STOP SIGN ISOLATED FORM..ARABIC LIGATURE JALLAJALALOUHOU +FE70..FE74 ; ID_Start # Lo [5] ARABIC FATHATAN ISOLATED FORM..ARABIC KASRATAN ISOLATED FORM +FE76..FEFC ; ID_Start # Lo [135] ARABIC FATHA ISOLATED FORM..ARABIC LIGATURE LAM WITH ALEF FINAL FORM +FF21..FF3A ; ID_Start # L& [26] FULLWIDTH LATIN CAPITAL LETTER A..FULLWIDTH LATIN CAPITAL LETTER Z +FF41..FF5A ; ID_Start # L& [26] FULLWIDTH LATIN SMALL LETTER A..FULLWIDTH LATIN SMALL LETTER Z +FF66..FF6F ; ID_Start # Lo [10] HALFWIDTH KATAKANA LETTER WO..HALFWIDTH KATAKANA LETTER SMALL TU +FF70 ; ID_Start # Lm HALFWIDTH KATAKANA-HIRAGANA PROLONGED SOUND MARK +FF71..FF9D ; ID_Start # Lo [45] HALFWIDTH KATAKANA LETTER A..HALFWIDTH KATAKANA LETTER N +FF9E..FF9F ; ID_Start # Lm [2] HALFWIDTH KATAKANA VOICED SOUND MARK..HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK +FFA0..FFBE ; ID_Start # Lo [31] HALFWIDTH HANGUL FILLER..HALFWIDTH HANGUL LETTER HIEUH +FFC2..FFC7 ; ID_Start # Lo [6] HALFWIDTH HANGUL LETTER A..HALFWIDTH HANGUL LETTER E +FFCA..FFCF ; ID_Start # Lo [6] HALFWIDTH HANGUL LETTER YEO..HALFWIDTH HANGUL LETTER OE +FFD2..FFD7 ; ID_Start # Lo [6] HALFWIDTH HANGUL LETTER YO..HALFWIDTH HANGUL LETTER YU +FFDA..FFDC ; ID_Start # Lo [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANGUL LETTER I +10000..1000B ; ID_Start # Lo [12] LINEAR B SYLLABLE B008 A..LINEAR B SYLLABLE B046 JE +1000D..10026 ; ID_Start # Lo [26] LINEAR B SYLLABLE B036 JO..LINEAR B SYLLABLE B032 QO +10028..1003A ; ID_Start # Lo [19] LINEAR B SYLLABLE B060 RA..LINEAR B SYLLABLE B042 WO +1003C..1003D ; ID_Start # Lo [2] LINEAR B SYLLABLE B017 ZA..LINEAR B SYLLABLE B074 ZE +1003F..1004D ; ID_Start # Lo [15] LINEAR B SYLLABLE B020 ZO..LINEAR B SYLLABLE B091 TWO +10050..1005D ; ID_Start # Lo [14] LINEAR B SYMBOL B018..LINEAR B SYMBOL B089 +10080..100FA ; ID_Start # Lo [123] LINEAR B IDEOGRAM B100 MAN..LINEAR B IDEOGRAM VESSEL B305 +10140..10174 ; ID_Start # Nl [53] GREEK ACROPHONIC ATTIC ONE QUARTER..GREEK ACROPHONIC STRATIAN FIFTY MNAS +10280..1029C ; ID_Start # Lo [29] LYCIAN LETTER A..LYCIAN LETTER X +102A0..102D0 ; ID_Start # Lo [49] CARIAN LETTER A..CARIAN LETTER UUU3 +10300..1031E ; ID_Start # Lo [31] OLD ITALIC LETTER A..OLD ITALIC LETTER UU +10330..10340 ; ID_Start # Lo [17] GOTHIC LETTER AHSA..GOTHIC LETTER PAIRTHRA +10341 ; ID_Start # Nl GOTHIC LETTER NINETY +10342..10349 ; ID_Start # Lo [8] GOTHIC LETTER RAIDA..GOTHIC LETTER OTHAL +1034A ; ID_Start # Nl GOTHIC LETTER NINE HUNDRED +10380..1039D ; ID_Start # Lo [30] UGARITIC LETTER ALPA..UGARITIC LETTER SSU +103A0..103C3 ; ID_Start # Lo [36] OLD PERSIAN SIGN A..OLD PERSIAN SIGN HA +103C8..103CF ; ID_Start # Lo [8] OLD PERSIAN SIGN AURAMAZDAA..OLD PERSIAN SIGN BUUMISH +103D1..103D5 ; ID_Start # Nl [5] OLD PERSIAN NUMBER ONE..OLD PERSIAN NUMBER HUNDRED +10400..1044F ; ID_Start # L& [80] DESERET CAPITAL LETTER LONG I..DESERET SMALL LETTER EW +10450..1049D ; ID_Start # Lo [78] SHAVIAN LETTER PEEP..OSMANYA LETTER OO +10800..10805 ; ID_Start # Lo [6] CYPRIOT SYLLABLE A..CYPRIOT SYLLABLE JA +10808 ; ID_Start # Lo CYPRIOT SYLLABLE JO +1080A..10835 ; ID_Start # Lo [44] CYPRIOT SYLLABLE KA..CYPRIOT SYLLABLE WO +10837..10838 ; ID_Start # Lo [2] CYPRIOT SYLLABLE XA..CYPRIOT SYLLABLE XE +1083C ; ID_Start # Lo CYPRIOT SYLLABLE ZA +1083F..10855 ; ID_Start # Lo [23] CYPRIOT SYLLABLE ZO..IMPERIAL ARAMAIC LETTER TAW +10900..10915 ; ID_Start # Lo [22] PHOENICIAN LETTER ALF..PHOENICIAN LETTER TAU +10920..10939 ; ID_Start # Lo [26] LYDIAN LETTER A..LYDIAN LETTER C +10A00 ; ID_Start # Lo KHAROSHTHI LETTER A +10A10..10A13 ; ID_Start # Lo [4] KHAROSHTHI LETTER KA..KHAROSHTHI LETTER GHA +10A15..10A17 ; ID_Start # Lo [3] KHAROSHTHI LETTER CA..KHAROSHTHI LETTER JA +10A19..10A33 ; ID_Start # Lo [27] KHAROSHTHI LETTER NYA..KHAROSHTHI LETTER TTTHA +10A60..10A7C ; ID_Start # Lo [29] OLD SOUTH ARABIAN LETTER HE..OLD SOUTH ARABIAN LETTER THETH +10B00..10B35 ; ID_Start # Lo [54] AVESTAN LETTER A..AVESTAN LETTER HE +10B40..10B55 ; ID_Start # Lo [22] INSCRIPTIONAL PARTHIAN LETTER ALEPH..INSCRIPTIONAL PARTHIAN LETTER TAW +10B60..10B72 ; ID_Start # Lo [19] INSCRIPTIONAL PAHLAVI LETTER ALEPH..INSCRIPTIONAL PAHLAVI LETTER TAW +10C00..10C48 ; ID_Start # Lo [73] OLD TURKIC LETTER ORKHON A..OLD TURKIC LETTER ORKHON BASH +11083..110AF ; ID_Start # Lo [45] KAITHI LETTER A..KAITHI LETTER HA +12000..1236E ; ID_Start # Lo [879] CUNEIFORM SIGN A..CUNEIFORM SIGN ZUM +12400..12462 ; ID_Start # Nl [99] CUNEIFORM NUMERIC SIGN TWO ASH..CUNEIFORM NUMERIC SIGN OLD ASSYRIAN ONE QUARTER +13000..1342E ; ID_Start # Lo [1071] EGYPTIAN HIEROGLYPH A001..EGYPTIAN HIEROGLYPH AA032 +1D400..1D454 ; ID_Start # L& [85] MATHEMATICAL BOLD CAPITAL A..MATHEMATICAL ITALIC SMALL G +1D456..1D49C ; ID_Start # L& [71] MATHEMATICAL ITALIC SMALL I..MATHEMATICAL SCRIPT CAPITAL A +1D49E..1D49F ; ID_Start # L& [2] MATHEMATICAL SCRIPT CAPITAL C..MATHEMATICAL SCRIPT CAPITAL D +1D4A2 ; ID_Start # L& MATHEMATICAL SCRIPT CAPITAL G +1D4A5..1D4A6 ; ID_Start # L& [2] MATHEMATICAL SCRIPT CAPITAL J..MATHEMATICAL SCRIPT CAPITAL K +1D4A9..1D4AC ; ID_Start # L& [4] MATHEMATICAL SCRIPT CAPITAL N..MATHEMATICAL SCRIPT CAPITAL Q +1D4AE..1D4B9 ; ID_Start # L& [12] MATHEMATICAL SCRIPT CAPITAL S..MATHEMATICAL SCRIPT SMALL D +1D4BB ; ID_Start # L& MATHEMATICAL SCRIPT SMALL F +1D4BD..1D4C3 ; ID_Start # L& [7] MATHEMATICAL SCRIPT SMALL H..MATHEMATICAL SCRIPT SMALL N +1D4C5..1D505 ; ID_Start # L& [65] MATHEMATICAL SCRIPT SMALL P..MATHEMATICAL FRAKTUR CAPITAL B +1D507..1D50A ; ID_Start # L& [4] MATHEMATICAL FRAKTUR CAPITAL D..MATHEMATICAL FRAKTUR CAPITAL G +1D50D..1D514 ; ID_Start # L& [8] MATHEMATICAL FRAKTUR CAPITAL J..MATHEMATICAL FRAKTUR CAPITAL Q +1D516..1D51C ; ID_Start # L& [7] MATHEMATICAL FRAKTUR CAPITAL S..MATHEMATICAL FRAKTUR CAPITAL Y +1D51E..1D539 ; ID_Start # L& [28] MATHEMATICAL FRAKTUR SMALL A..MATHEMATICAL DOUBLE-STRUCK CAPITAL B +1D53B..1D53E ; ID_Start # L& [4] MATHEMATICAL DOUBLE-STRUCK CAPITAL D..MATHEMATICAL DOUBLE-STRUCK CAPITAL G +1D540..1D544 ; ID_Start # L& [5] MATHEMATICAL DOUBLE-STRUCK CAPITAL I..MATHEMATICAL DOUBLE-STRUCK CAPITAL M +1D546 ; ID_Start # L& MATHEMATICAL DOUBLE-STRUCK CAPITAL O +1D54A..1D550 ; ID_Start # L& [7] MATHEMATICAL DOUBLE-STRUCK CAPITAL S..MATHEMATICAL DOUBLE-STRUCK CAPITAL Y +1D552..1D6A5 ; ID_Start # L& [340] MATHEMATICAL DOUBLE-STRUCK SMALL A..MATHEMATICAL ITALIC SMALL DOTLESS J +1D6A8..1D6C0 ; ID_Start # L& [25] MATHEMATICAL BOLD CAPITAL ALPHA..MATHEMATICAL BOLD CAPITAL OMEGA +1D6C2..1D6DA ; ID_Start # L& [25] MATHEMATICAL BOLD SMALL ALPHA..MATHEMATICAL BOLD SMALL OMEGA +1D6DC..1D6FA ; ID_Start # L& [31] MATHEMATICAL BOLD EPSILON SYMBOL..MATHEMATICAL ITALIC CAPITAL OMEGA +1D6FC..1D714 ; ID_Start # L& [25] MATHEMATICAL ITALIC SMALL ALPHA..MATHEMATICAL ITALIC SMALL OMEGA +1D716..1D734 ; ID_Start # L& [31] MATHEMATICAL ITALIC EPSILON SYMBOL..MATHEMATICAL BOLD ITALIC CAPITAL OMEGA +1D736..1D74E ; ID_Start # L& [25] MATHEMATICAL BOLD ITALIC SMALL ALPHA..MATHEMATICAL BOLD ITALIC SMALL OMEGA +1D750..1D76E ; ID_Start # L& [31] MATHEMATICAL BOLD ITALIC EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD CAPITAL OMEGA +1D770..1D788 ; ID_Start # L& [25] MATHEMATICAL SANS-SERIF BOLD SMALL ALPHA..MATHEMATICAL SANS-SERIF BOLD SMALL OMEGA +1D78A..1D7A8 ; ID_Start # L& [31] MATHEMATICAL SANS-SERIF BOLD EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL OMEGA +1D7AA..1D7C2 ; ID_Start # L& [25] MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL ALPHA..MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL OMEGA +1D7C4..1D7CB ; ID_Start # L& [8] MATHEMATICAL SANS-SERIF BOLD ITALIC EPSILON SYMBOL..MATHEMATICAL BOLD SMALL DIGAMMA +20000..2A6D6 ; ID_Start # Lo [42711] CJK UNIFIED IDEOGRAPH-20000..CJK UNIFIED IDEOGRAPH-2A6D6 +2A700..2B734 ; ID_Start # Lo [4149] CJK UNIFIED IDEOGRAPH-2A700..CJK UNIFIED IDEOGRAPH-2B734 +2F800..2FA1D ; ID_Start # Lo [542] CJK COMPATIBILITY IDEOGRAPH-2F800..CJK COMPATIBILITY IDEOGRAPH-2FA1D + +# Total code points: 99764 + +# ================================================ + +# Derived Property: ID_Continue +# Characters that can continue an identifier. +# Generated from: +# ID_Start +# + Mn + Mc + Nd + Pc +# + Other_ID_Continue +# - Pattern_Syntax +# - Pattern_White_Space +# NOTE: See UAX #31 for more information + +0030..0039 ; ID_Continue # Nd [10] DIGIT ZERO..DIGIT NINE +0041..005A ; ID_Continue # L& [26] LATIN CAPITAL LETTER A..LATIN CAPITAL LETTER Z +005F ; ID_Continue # Pc LOW LINE +0061..007A ; ID_Continue # L& [26] LATIN SMALL LETTER A..LATIN SMALL LETTER Z +00AA ; ID_Continue # L& FEMININE ORDINAL INDICATOR +00B5 ; ID_Continue # L& MICRO SIGN +00B7 ; ID_Continue # Po MIDDLE DOT +00BA ; ID_Continue # L& MASCULINE ORDINAL INDICATOR +00C0..00D6 ; ID_Continue # L& [23] LATIN CAPITAL LETTER A WITH GRAVE..LATIN CAPITAL LETTER O WITH DIAERESIS +00D8..00F6 ; ID_Continue # L& [31] LATIN CAPITAL LETTER O WITH STROKE..LATIN SMALL LETTER O WITH DIAERESIS +00F8..01BA ; ID_Continue # L& [195] LATIN SMALL LETTER O WITH STROKE..LATIN SMALL LETTER EZH WITH TAIL +01BB ; ID_Continue # Lo LATIN LETTER TWO WITH STROKE +01BC..01BF ; ID_Continue # L& [4] LATIN CAPITAL LETTER TONE FIVE..LATIN LETTER WYNN +01C0..01C3 ; ID_Continue # Lo [4] LATIN LETTER DENTAL CLICK..LATIN LETTER RETROFLEX CLICK +01C4..0293 ; ID_Continue # L& [208] LATIN CAPITAL LETTER DZ WITH CARON..LATIN SMALL LETTER EZH WITH CURL +0294 ; ID_Continue # Lo LATIN LETTER GLOTTAL STOP +0295..02AF ; ID_Continue # L& [27] LATIN LETTER PHARYNGEAL VOICED FRICATIVE..LATIN SMALL LETTER TURNED H WITH FISHHOOK AND TAIL +02B0..02C1 ; ID_Continue # Lm [18] MODIFIER LETTER SMALL H..MODIFIER LETTER REVERSED GLOTTAL STOP +02C6..02D1 ; ID_Continue # Lm [12] MODIFIER LETTER CIRCUMFLEX ACCENT..MODIFIER LETTER HALF TRIANGULAR COLON +02E0..02E4 ; ID_Continue # Lm [5] MODIFIER LETTER SMALL GAMMA..MODIFIER LETTER SMALL REVERSED GLOTTAL STOP +02EC ; ID_Continue # Lm MODIFIER LETTER VOICING +02EE ; ID_Continue # Lm MODIFIER LETTER DOUBLE APOSTROPHE +0300..036F ; ID_Continue # Mn [112] COMBINING GRAVE ACCENT..COMBINING LATIN SMALL LETTER X +0370..0373 ; ID_Continue # L& [4] GREEK CAPITAL LETTER HETA..GREEK SMALL LETTER ARCHAIC SAMPI +0374 ; ID_Continue # Lm GREEK NUMERAL SIGN +0376..0377 ; ID_Continue # L& [2] GREEK CAPITAL LETTER PAMPHYLIAN DIGAMMA..GREEK SMALL LETTER PAMPHYLIAN DIGAMMA +037A ; ID_Continue # Lm GREEK YPOGEGRAMMENI +037B..037D ; ID_Continue # L& [3] GREEK SMALL REVERSED LUNATE SIGMA SYMBOL..GREEK SMALL REVERSED DOTTED LUNATE SIGMA SYMBOL +0386 ; ID_Continue # L& GREEK CAPITAL LETTER ALPHA WITH TONOS +0387 ; ID_Continue # Po GREEK ANO TELEIA +0388..038A ; ID_Continue # L& [3] GREEK CAPITAL LETTER EPSILON WITH TONOS..GREEK CAPITAL LETTER IOTA WITH TONOS +038C ; ID_Continue # L& GREEK CAPITAL LETTER OMICRON WITH TONOS +038E..03A1 ; ID_Continue # L& [20] GREEK CAPITAL LETTER UPSILON WITH TONOS..GREEK CAPITAL LETTER RHO +03A3..03F5 ; ID_Continue # L& [83] GREEK CAPITAL LETTER SIGMA..GREEK LUNATE EPSILON SYMBOL +03F7..0481 ; ID_Continue # L& [139] GREEK CAPITAL LETTER SHO..CYRILLIC SMALL LETTER KOPPA +0483..0487 ; ID_Continue # Mn [5] COMBINING CYRILLIC TITLO..COMBINING CYRILLIC POKRYTIE +048A..0525 ; ID_Continue # L& [156] CYRILLIC CAPITAL LETTER SHORT I WITH TAIL..CYRILLIC SMALL LETTER PE WITH DESCENDER +0531..0556 ; ID_Continue # L& [38] ARMENIAN CAPITAL LETTER AYB..ARMENIAN CAPITAL LETTER FEH +0559 ; ID_Continue # Lm ARMENIAN MODIFIER LETTER LEFT HALF RING +0561..0587 ; ID_Continue # L& [39] ARMENIAN SMALL LETTER AYB..ARMENIAN SMALL LIGATURE ECH YIWN +0591..05BD ; ID_Continue # Mn [45] HEBREW ACCENT ETNAHTA..HEBREW POINT METEG +05BF ; ID_Continue # Mn HEBREW POINT RAFE +05C1..05C2 ; ID_Continue # Mn [2] HEBREW POINT SHIN DOT..HEBREW POINT SIN DOT +05C4..05C5 ; ID_Continue # Mn [2] HEBREW MARK UPPER DOT..HEBREW MARK LOWER DOT +05C7 ; ID_Continue # Mn HEBREW POINT QAMATS QATAN +05D0..05EA ; ID_Continue # Lo [27] HEBREW LETTER ALEF..HEBREW LETTER TAV +05F0..05F2 ; ID_Continue # Lo [3] HEBREW LIGATURE YIDDISH DOUBLE VAV..HEBREW LIGATURE YIDDISH DOUBLE YOD +0610..061A ; ID_Continue # Mn [11] ARABIC SIGN SALLALLAHOU ALAYHE WASSALLAM..ARABIC SMALL KASRA +0621..063F ; ID_Continue # Lo [31] ARABIC LETTER HAMZA..ARABIC LETTER FARSI YEH WITH THREE DOTS ABOVE +0640 ; ID_Continue # Lm ARABIC TATWEEL +0641..064A ; ID_Continue # Lo [10] ARABIC LETTER FEH..ARABIC LETTER YEH +064B..065E ; ID_Continue # Mn [20] ARABIC FATHATAN..ARABIC FATHA WITH TWO DOTS +0660..0669 ; ID_Continue # Nd [10] ARABIC-INDIC DIGIT ZERO..ARABIC-INDIC DIGIT NINE +066E..066F ; ID_Continue # Lo [2] ARABIC LETTER DOTLESS BEH..ARABIC LETTER DOTLESS QAF +0670 ; ID_Continue # Mn ARABIC LETTER SUPERSCRIPT ALEF +0671..06D3 ; ID_Continue # Lo [99] ARABIC LETTER ALEF WASLA..ARABIC LETTER YEH BARREE WITH HAMZA ABOVE +06D5 ; ID_Continue # Lo ARABIC LETTER AE +06D6..06DC ; ID_Continue # Mn [7] ARABIC SMALL HIGH LIGATURE SAD WITH LAM WITH ALEF MAKSURA..ARABIC SMALL HIGH SEEN +06DF..06E4 ; ID_Continue # Mn [6] ARABIC SMALL HIGH ROUNDED ZERO..ARABIC SMALL HIGH MADDA +06E5..06E6 ; ID_Continue # Lm [2] ARABIC SMALL WAW..ARABIC SMALL YEH +06E7..06E8 ; ID_Continue # Mn [2] ARABIC SMALL HIGH YEH..ARABIC SMALL HIGH NOON +06EA..06ED ; ID_Continue # Mn [4] ARABIC EMPTY CENTRE LOW STOP..ARABIC SMALL LOW MEEM +06EE..06EF ; ID_Continue # Lo [2] ARABIC LETTER DAL WITH INVERTED V..ARABIC LETTER REH WITH INVERTED V +06F0..06F9 ; ID_Continue # Nd [10] EXTENDED ARABIC-INDIC DIGIT ZERO..EXTENDED ARABIC-INDIC DIGIT NINE +06FA..06FC ; ID_Continue # Lo [3] ARABIC LETTER SHEEN WITH DOT BELOW..ARABIC LETTER GHAIN WITH DOT BELOW +06FF ; ID_Continue # Lo ARABIC LETTER HEH WITH INVERTED V +0710 ; ID_Continue # Lo SYRIAC LETTER ALAPH +0711 ; ID_Continue # Mn SYRIAC LETTER SUPERSCRIPT ALAPH +0712..072F ; ID_Continue # Lo [30] SYRIAC LETTER BETH..SYRIAC LETTER PERSIAN DHALATH +0730..074A ; ID_Continue # Mn [27] SYRIAC PTHAHA ABOVE..SYRIAC BARREKH +074D..07A5 ; ID_Continue # Lo [89] SYRIAC LETTER SOGDIAN ZHAIN..THAANA LETTER WAAVU +07A6..07B0 ; ID_Continue # Mn [11] THAANA ABAFILI..THAANA SUKUN +07B1 ; ID_Continue # Lo THAANA LETTER NAA +07C0..07C9 ; ID_Continue # Nd [10] NKO DIGIT ZERO..NKO DIGIT NINE +07CA..07EA ; ID_Continue # Lo [33] NKO LETTER A..NKO LETTER JONA RA +07EB..07F3 ; ID_Continue # Mn [9] NKO COMBINING SHORT HIGH TONE..NKO COMBINING DOUBLE DOT ABOVE +07F4..07F5 ; ID_Continue # Lm [2] NKO HIGH TONE APOSTROPHE..NKO LOW TONE APOSTROPHE +07FA ; ID_Continue # Lm NKO LAJANYALAN +0800..0815 ; ID_Continue # Lo [22] SAMARITAN LETTER ALAF..SAMARITAN LETTER TAAF +0816..0819 ; ID_Continue # Mn [4] SAMARITAN MARK IN..SAMARITAN MARK DAGESH +081A ; ID_Continue # Lm SAMARITAN MODIFIER LETTER EPENTHETIC YUT +081B..0823 ; ID_Continue # Mn [9] SAMARITAN MARK EPENTHETIC YUT..SAMARITAN VOWEL SIGN A +0824 ; ID_Continue # Lm SAMARITAN MODIFIER LETTER SHORT A +0825..0827 ; ID_Continue # Mn [3] SAMARITAN VOWEL SIGN SHORT A..SAMARITAN VOWEL SIGN U +0828 ; ID_Continue # Lm SAMARITAN MODIFIER LETTER I +0829..082D ; ID_Continue # Mn [5] SAMARITAN VOWEL SIGN LONG I..SAMARITAN MARK NEQUDAA +0900..0902 ; ID_Continue # Mn [3] DEVANAGARI SIGN INVERTED CANDRABINDU..DEVANAGARI SIGN ANUSVARA +0903 ; ID_Continue # Mc DEVANAGARI SIGN VISARGA +0904..0939 ; ID_Continue # Lo [54] DEVANAGARI LETTER SHORT A..DEVANAGARI LETTER HA +093C ; ID_Continue # Mn DEVANAGARI SIGN NUKTA +093D ; ID_Continue # Lo DEVANAGARI SIGN AVAGRAHA +093E..0940 ; ID_Continue # Mc [3] DEVANAGARI VOWEL SIGN AA..DEVANAGARI VOWEL SIGN II +0941..0948 ; ID_Continue # Mn [8] DEVANAGARI VOWEL SIGN U..DEVANAGARI VOWEL SIGN AI +0949..094C ; ID_Continue # Mc [4] DEVANAGARI VOWEL SIGN CANDRA O..DEVANAGARI VOWEL SIGN AU +094D ; ID_Continue # Mn DEVANAGARI SIGN VIRAMA +094E ; ID_Continue # Mc DEVANAGARI VOWEL SIGN PRISHTHAMATRA E +0950 ; ID_Continue # Lo DEVANAGARI OM +0951..0955 ; ID_Continue # Mn [5] DEVANAGARI STRESS SIGN UDATTA..DEVANAGARI VOWEL SIGN CANDRA LONG E +0958..0961 ; ID_Continue # Lo [10] DEVANAGARI LETTER QA..DEVANAGARI LETTER VOCALIC LL +0962..0963 ; ID_Continue # Mn [2] DEVANAGARI VOWEL SIGN VOCALIC L..DEVANAGARI VOWEL SIGN VOCALIC LL +0966..096F ; ID_Continue # Nd [10] DEVANAGARI DIGIT ZERO..DEVANAGARI DIGIT NINE +0971 ; ID_Continue # Lm DEVANAGARI SIGN HIGH SPACING DOT +0972 ; ID_Continue # Lo DEVANAGARI LETTER CANDRA A +0979..097F ; ID_Continue # Lo [7] DEVANAGARI LETTER ZHA..DEVANAGARI LETTER BBA +0981 ; ID_Continue # Mn BENGALI SIGN CANDRABINDU +0982..0983 ; ID_Continue # Mc [2] BENGALI SIGN ANUSVARA..BENGALI SIGN VISARGA +0985..098C ; ID_Continue # Lo [8] BENGALI LETTER A..BENGALI LETTER VOCALIC L +098F..0990 ; ID_Continue # Lo [2] BENGALI LETTER E..BENGALI LETTER AI +0993..09A8 ; ID_Continue # Lo [22] BENGALI LETTER O..BENGALI LETTER NA +09AA..09B0 ; ID_Continue # Lo [7] BENGALI LETTER PA..BENGALI LETTER RA +09B2 ; ID_Continue # Lo BENGALI LETTER LA +09B6..09B9 ; ID_Continue # Lo [4] BENGALI LETTER SHA..BENGALI LETTER HA +09BC ; ID_Continue # Mn BENGALI SIGN NUKTA +09BD ; ID_Continue # Lo BENGALI SIGN AVAGRAHA +09BE..09C0 ; ID_Continue # Mc [3] BENGALI VOWEL SIGN AA..BENGALI VOWEL SIGN II +09C1..09C4 ; ID_Continue # Mn [4] BENGALI VOWEL SIGN U..BENGALI VOWEL SIGN VOCALIC RR +09C7..09C8 ; ID_Continue # Mc [2] BENGALI VOWEL SIGN E..BENGALI VOWEL SIGN AI +09CB..09CC ; ID_Continue # Mc [2] BENGALI VOWEL SIGN O..BENGALI VOWEL SIGN AU +09CD ; ID_Continue # Mn BENGALI SIGN VIRAMA +09CE ; ID_Continue # Lo BENGALI LETTER KHANDA TA +09D7 ; ID_Continue # Mc BENGALI AU LENGTH MARK +09DC..09DD ; ID_Continue # Lo [2] BENGALI LETTER RRA..BENGALI LETTER RHA +09DF..09E1 ; ID_Continue # Lo [3] BENGALI LETTER YYA..BENGALI LETTER VOCALIC LL +09E2..09E3 ; ID_Continue # Mn [2] BENGALI VOWEL SIGN VOCALIC L..BENGALI VOWEL SIGN VOCALIC LL +09E6..09EF ; ID_Continue # Nd [10] BENGALI DIGIT ZERO..BENGALI DIGIT NINE +09F0..09F1 ; ID_Continue # Lo [2] BENGALI LETTER RA WITH MIDDLE DIAGONAL..BENGALI LETTER RA WITH LOWER DIAGONAL +0A01..0A02 ; ID_Continue # Mn [2] GURMUKHI SIGN ADAK BINDI..GURMUKHI SIGN BINDI +0A03 ; ID_Continue # Mc GURMUKHI SIGN VISARGA +0A05..0A0A ; ID_Continue # Lo [6] GURMUKHI LETTER A..GURMUKHI LETTER UU +0A0F..0A10 ; ID_Continue # Lo [2] GURMUKHI LETTER EE..GURMUKHI LETTER AI +0A13..0A28 ; ID_Continue # Lo [22] GURMUKHI LETTER OO..GURMUKHI LETTER NA +0A2A..0A30 ; ID_Continue # Lo [7] GURMUKHI LETTER PA..GURMUKHI LETTER RA +0A32..0A33 ; ID_Continue # Lo [2] GURMUKHI LETTER LA..GURMUKHI LETTER LLA +0A35..0A36 ; ID_Continue # Lo [2] GURMUKHI LETTER VA..GURMUKHI LETTER SHA +0A38..0A39 ; ID_Continue # Lo [2] GURMUKHI LETTER SA..GURMUKHI LETTER HA +0A3C ; ID_Continue # Mn GURMUKHI SIGN NUKTA +0A3E..0A40 ; ID_Continue # Mc [3] GURMUKHI VOWEL SIGN AA..GURMUKHI VOWEL SIGN II +0A41..0A42 ; ID_Continue # Mn [2] GURMUKHI VOWEL SIGN U..GURMUKHI VOWEL SIGN UU +0A47..0A48 ; ID_Continue # Mn [2] GURMUKHI VOWEL SIGN EE..GURMUKHI VOWEL SIGN AI +0A4B..0A4D ; ID_Continue # Mn [3] GURMUKHI VOWEL SIGN OO..GURMUKHI SIGN VIRAMA +0A51 ; ID_Continue # Mn GURMUKHI SIGN UDAAT +0A59..0A5C ; ID_Continue # Lo [4] GURMUKHI LETTER KHHA..GURMUKHI LETTER RRA +0A5E ; ID_Continue # Lo GURMUKHI LETTER FA +0A66..0A6F ; ID_Continue # Nd [10] GURMUKHI DIGIT ZERO..GURMUKHI DIGIT NINE +0A70..0A71 ; ID_Continue # Mn [2] GURMUKHI TIPPI..GURMUKHI ADDAK +0A72..0A74 ; ID_Continue # Lo [3] GURMUKHI IRI..GURMUKHI EK ONKAR +0A75 ; ID_Continue # Mn GURMUKHI SIGN YAKASH +0A81..0A82 ; ID_Continue # Mn [2] GUJARATI SIGN CANDRABINDU..GUJARATI SIGN ANUSVARA +0A83 ; ID_Continue # Mc GUJARATI SIGN VISARGA +0A85..0A8D ; ID_Continue # Lo [9] GUJARATI LETTER A..GUJARATI VOWEL CANDRA E +0A8F..0A91 ; ID_Continue # Lo [3] GUJARATI LETTER E..GUJARATI VOWEL CANDRA O +0A93..0AA8 ; ID_Continue # Lo [22] GUJARATI LETTER O..GUJARATI LETTER NA +0AAA..0AB0 ; ID_Continue # Lo [7] GUJARATI LETTER PA..GUJARATI LETTER RA +0AB2..0AB3 ; ID_Continue # Lo [2] GUJARATI LETTER LA..GUJARATI LETTER LLA +0AB5..0AB9 ; ID_Continue # Lo [5] GUJARATI LETTER VA..GUJARATI LETTER HA +0ABC ; ID_Continue # Mn GUJARATI SIGN NUKTA +0ABD ; ID_Continue # Lo GUJARATI SIGN AVAGRAHA +0ABE..0AC0 ; ID_Continue # Mc [3] GUJARATI VOWEL SIGN AA..GUJARATI VOWEL SIGN II +0AC1..0AC5 ; ID_Continue # Mn [5] GUJARATI VOWEL SIGN U..GUJARATI VOWEL SIGN CANDRA E +0AC7..0AC8 ; ID_Continue # Mn [2] GUJARATI VOWEL SIGN E..GUJARATI VOWEL SIGN AI +0AC9 ; ID_Continue # Mc GUJARATI VOWEL SIGN CANDRA O +0ACB..0ACC ; ID_Continue # Mc [2] GUJARATI VOWEL SIGN O..GUJARATI VOWEL SIGN AU +0ACD ; ID_Continue # Mn GUJARATI SIGN VIRAMA +0AD0 ; ID_Continue # Lo GUJARATI OM +0AE0..0AE1 ; ID_Continue # Lo [2] GUJARATI LETTER VOCALIC RR..GUJARATI LETTER VOCALIC LL +0AE2..0AE3 ; ID_Continue # Mn [2] GUJARATI VOWEL SIGN VOCALIC L..GUJARATI VOWEL SIGN VOCALIC LL +0AE6..0AEF ; ID_Continue # Nd [10] GUJARATI DIGIT ZERO..GUJARATI DIGIT NINE +0B01 ; ID_Continue # Mn ORIYA SIGN CANDRABINDU +0B02..0B03 ; ID_Continue # Mc [2] ORIYA SIGN ANUSVARA..ORIYA SIGN VISARGA +0B05..0B0C ; ID_Continue # Lo [8] ORIYA LETTER A..ORIYA LETTER VOCALIC L +0B0F..0B10 ; ID_Continue # Lo [2] ORIYA LETTER E..ORIYA LETTER AI +0B13..0B28 ; ID_Continue # Lo [22] ORIYA LETTER O..ORIYA LETTER NA +0B2A..0B30 ; ID_Continue # Lo [7] ORIYA LETTER PA..ORIYA LETTER RA +0B32..0B33 ; ID_Continue # Lo [2] ORIYA LETTER LA..ORIYA LETTER LLA +0B35..0B39 ; ID_Continue # Lo [5] ORIYA LETTER VA..ORIYA LETTER HA +0B3C ; ID_Continue # Mn ORIYA SIGN NUKTA +0B3D ; ID_Continue # Lo ORIYA SIGN AVAGRAHA +0B3E ; ID_Continue # Mc ORIYA VOWEL SIGN AA +0B3F ; ID_Continue # Mn ORIYA VOWEL SIGN I +0B40 ; ID_Continue # Mc ORIYA VOWEL SIGN II +0B41..0B44 ; ID_Continue # Mn [4] ORIYA VOWEL SIGN U..ORIYA VOWEL SIGN VOCALIC RR +0B47..0B48 ; ID_Continue # Mc [2] ORIYA VOWEL SIGN E..ORIYA VOWEL SIGN AI +0B4B..0B4C ; ID_Continue # Mc [2] ORIYA VOWEL SIGN O..ORIYA VOWEL SIGN AU +0B4D ; ID_Continue # Mn ORIYA SIGN VIRAMA +0B56 ; ID_Continue # Mn ORIYA AI LENGTH MARK +0B57 ; ID_Continue # Mc ORIYA AU LENGTH MARK +0B5C..0B5D ; ID_Continue # Lo [2] ORIYA LETTER RRA..ORIYA LETTER RHA +0B5F..0B61 ; ID_Continue # Lo [3] ORIYA LETTER YYA..ORIYA LETTER VOCALIC LL +0B62..0B63 ; ID_Continue # Mn [2] ORIYA VOWEL SIGN VOCALIC L..ORIYA VOWEL SIGN VOCALIC LL +0B66..0B6F ; ID_Continue # Nd [10] ORIYA DIGIT ZERO..ORIYA DIGIT NINE +0B71 ; ID_Continue # Lo ORIYA LETTER WA +0B82 ; ID_Continue # Mn TAMIL SIGN ANUSVARA +0B83 ; ID_Continue # Lo TAMIL SIGN VISARGA +0B85..0B8A ; ID_Continue # Lo [6] TAMIL LETTER A..TAMIL LETTER UU +0B8E..0B90 ; ID_Continue # Lo [3] TAMIL LETTER E..TAMIL LETTER AI +0B92..0B95 ; ID_Continue # Lo [4] TAMIL LETTER O..TAMIL LETTER KA +0B99..0B9A ; ID_Continue # Lo [2] TAMIL LETTER NGA..TAMIL LETTER CA +0B9C ; ID_Continue # Lo TAMIL LETTER JA +0B9E..0B9F ; ID_Continue # Lo [2] TAMIL LETTER NYA..TAMIL LETTER TTA +0BA3..0BA4 ; ID_Continue # Lo [2] TAMIL LETTER NNA..TAMIL LETTER TA +0BA8..0BAA ; ID_Continue # Lo [3] TAMIL LETTER NA..TAMIL LETTER PA +0BAE..0BB9 ; ID_Continue # Lo [12] TAMIL LETTER MA..TAMIL LETTER HA +0BBE..0BBF ; ID_Continue # Mc [2] TAMIL VOWEL SIGN AA..TAMIL VOWEL SIGN I +0BC0 ; ID_Continue # Mn TAMIL VOWEL SIGN II +0BC1..0BC2 ; ID_Continue # Mc [2] TAMIL VOWEL SIGN U..TAMIL VOWEL SIGN UU +0BC6..0BC8 ; ID_Continue # Mc [3] TAMIL VOWEL SIGN E..TAMIL VOWEL SIGN AI +0BCA..0BCC ; ID_Continue # Mc [3] TAMIL VOWEL SIGN O..TAMIL VOWEL SIGN AU +0BCD ; ID_Continue # Mn TAMIL SIGN VIRAMA +0BD0 ; ID_Continue # Lo TAMIL OM +0BD7 ; ID_Continue # Mc TAMIL AU LENGTH MARK +0BE6..0BEF ; ID_Continue # Nd [10] TAMIL DIGIT ZERO..TAMIL DIGIT NINE +0C01..0C03 ; ID_Continue # Mc [3] TELUGU SIGN CANDRABINDU..TELUGU SIGN VISARGA +0C05..0C0C ; ID_Continue # Lo [8] TELUGU LETTER A..TELUGU LETTER VOCALIC L +0C0E..0C10 ; ID_Continue # Lo [3] TELUGU LETTER E..TELUGU LETTER AI +0C12..0C28 ; ID_Continue # Lo [23] TELUGU LETTER O..TELUGU LETTER NA +0C2A..0C33 ; ID_Continue # Lo [10] TELUGU LETTER PA..TELUGU LETTER LLA +0C35..0C39 ; ID_Continue # Lo [5] TELUGU LETTER VA..TELUGU LETTER HA +0C3D ; ID_Continue # Lo TELUGU SIGN AVAGRAHA +0C3E..0C40 ; ID_Continue # Mn [3] TELUGU VOWEL SIGN AA..TELUGU VOWEL SIGN II +0C41..0C44 ; ID_Continue # Mc [4] TELUGU VOWEL SIGN U..TELUGU VOWEL SIGN VOCALIC RR +0C46..0C48 ; ID_Continue # Mn [3] TELUGU VOWEL SIGN E..TELUGU VOWEL SIGN AI +0C4A..0C4D ; ID_Continue # Mn [4] TELUGU VOWEL SIGN O..TELUGU SIGN VIRAMA +0C55..0C56 ; ID_Continue # Mn [2] TELUGU LENGTH MARK..TELUGU AI LENGTH MARK +0C58..0C59 ; ID_Continue # Lo [2] TELUGU LETTER TSA..TELUGU LETTER DZA +0C60..0C61 ; ID_Continue # Lo [2] TELUGU LETTER VOCALIC RR..TELUGU LETTER VOCALIC LL +0C62..0C63 ; ID_Continue # Mn [2] TELUGU VOWEL SIGN VOCALIC L..TELUGU VOWEL SIGN VOCALIC LL +0C66..0C6F ; ID_Continue # Nd [10] TELUGU DIGIT ZERO..TELUGU DIGIT NINE +0C82..0C83 ; ID_Continue # Mc [2] KANNADA SIGN ANUSVARA..KANNADA SIGN VISARGA +0C85..0C8C ; ID_Continue # Lo [8] KANNADA LETTER A..KANNADA LETTER VOCALIC L +0C8E..0C90 ; ID_Continue # Lo [3] KANNADA LETTER E..KANNADA LETTER AI +0C92..0CA8 ; ID_Continue # Lo [23] KANNADA LETTER O..KANNADA LETTER NA +0CAA..0CB3 ; ID_Continue # Lo [10] KANNADA LETTER PA..KANNADA LETTER LLA +0CB5..0CB9 ; ID_Continue # Lo [5] KANNADA LETTER VA..KANNADA LETTER HA +0CBC ; ID_Continue # Mn KANNADA SIGN NUKTA +0CBD ; ID_Continue # Lo KANNADA SIGN AVAGRAHA +0CBE ; ID_Continue # Mc KANNADA VOWEL SIGN AA +0CBF ; ID_Continue # Mn KANNADA VOWEL SIGN I +0CC0..0CC4 ; ID_Continue # Mc [5] KANNADA VOWEL SIGN II..KANNADA VOWEL SIGN VOCALIC RR +0CC6 ; ID_Continue # Mn KANNADA VOWEL SIGN E +0CC7..0CC8 ; ID_Continue # Mc [2] KANNADA VOWEL SIGN EE..KANNADA VOWEL SIGN AI +0CCA..0CCB ; ID_Continue # Mc [2] KANNADA VOWEL SIGN O..KANNADA VOWEL SIGN OO +0CCC..0CCD ; ID_Continue # Mn [2] KANNADA VOWEL SIGN AU..KANNADA SIGN VIRAMA +0CD5..0CD6 ; ID_Continue # Mc [2] KANNADA LENGTH MARK..KANNADA AI LENGTH MARK +0CDE ; ID_Continue # Lo KANNADA LETTER FA +0CE0..0CE1 ; ID_Continue # Lo [2] KANNADA LETTER VOCALIC RR..KANNADA LETTER VOCALIC LL +0CE2..0CE3 ; ID_Continue # Mn [2] KANNADA VOWEL SIGN VOCALIC L..KANNADA VOWEL SIGN VOCALIC LL +0CE6..0CEF ; ID_Continue # Nd [10] KANNADA DIGIT ZERO..KANNADA DIGIT NINE +0D02..0D03 ; ID_Continue # Mc [2] MALAYALAM SIGN ANUSVARA..MALAYALAM SIGN VISARGA +0D05..0D0C ; ID_Continue # Lo [8] MALAYALAM LETTER A..MALAYALAM LETTER VOCALIC L +0D0E..0D10 ; ID_Continue # Lo [3] MALAYALAM LETTER E..MALAYALAM LETTER AI +0D12..0D28 ; ID_Continue # Lo [23] MALAYALAM LETTER O..MALAYALAM LETTER NA +0D2A..0D39 ; ID_Continue # Lo [16] MALAYALAM LETTER PA..MALAYALAM LETTER HA +0D3D ; ID_Continue # Lo MALAYALAM SIGN AVAGRAHA +0D3E..0D40 ; ID_Continue # Mc [3] MALAYALAM VOWEL SIGN AA..MALAYALAM VOWEL SIGN II +0D41..0D44 ; ID_Continue # Mn [4] MALAYALAM VOWEL SIGN U..MALAYALAM VOWEL SIGN VOCALIC RR +0D46..0D48 ; ID_Continue # Mc [3] MALAYALAM VOWEL SIGN E..MALAYALAM VOWEL SIGN AI +0D4A..0D4C ; ID_Continue # Mc [3] MALAYALAM VOWEL SIGN O..MALAYALAM VOWEL SIGN AU +0D4D ; ID_Continue # Mn MALAYALAM SIGN VIRAMA +0D57 ; ID_Continue # Mc MALAYALAM AU LENGTH MARK +0D60..0D61 ; ID_Continue # Lo [2] MALAYALAM LETTER VOCALIC RR..MALAYALAM LETTER VOCALIC LL +0D62..0D63 ; ID_Continue # Mn [2] MALAYALAM VOWEL SIGN VOCALIC L..MALAYALAM VOWEL SIGN VOCALIC LL +0D66..0D6F ; ID_Continue # Nd [10] MALAYALAM DIGIT ZERO..MALAYALAM DIGIT NINE +0D7A..0D7F ; ID_Continue # Lo [6] MALAYALAM LETTER CHILLU NN..MALAYALAM LETTER CHILLU K +0D82..0D83 ; ID_Continue # Mc [2] SINHALA SIGN ANUSVARAYA..SINHALA SIGN VISARGAYA +0D85..0D96 ; ID_Continue # Lo [18] SINHALA LETTER AYANNA..SINHALA LETTER AUYANNA +0D9A..0DB1 ; ID_Continue # Lo [24] SINHALA LETTER ALPAPRAANA KAYANNA..SINHALA LETTER DANTAJA NAYANNA +0DB3..0DBB ; ID_Continue # Lo [9] SINHALA LETTER SANYAKA DAYANNA..SINHALA LETTER RAYANNA +0DBD ; ID_Continue # Lo SINHALA LETTER DANTAJA LAYANNA +0DC0..0DC6 ; ID_Continue # Lo [7] SINHALA LETTER VAYANNA..SINHALA LETTER FAYANNA +0DCA ; ID_Continue # Mn SINHALA SIGN AL-LAKUNA +0DCF..0DD1 ; ID_Continue # Mc [3] SINHALA VOWEL SIGN AELA-PILLA..SINHALA VOWEL SIGN DIGA AEDA-PILLA +0DD2..0DD4 ; ID_Continue # Mn [3] SINHALA VOWEL SIGN KETTI IS-PILLA..SINHALA VOWEL SIGN KETTI PAA-PILLA +0DD6 ; ID_Continue # Mn SINHALA VOWEL SIGN DIGA PAA-PILLA +0DD8..0DDF ; ID_Continue # Mc [8] SINHALA VOWEL SIGN GAETTA-PILLA..SINHALA VOWEL SIGN GAYANUKITTA +0DF2..0DF3 ; ID_Continue # Mc [2] SINHALA VOWEL SIGN DIGA GAETTA-PILLA..SINHALA VOWEL SIGN DIGA GAYANUKITTA +0E01..0E30 ; ID_Continue # Lo [48] THAI CHARACTER KO KAI..THAI CHARACTER SARA A +0E31 ; ID_Continue # Mn THAI CHARACTER MAI HAN-AKAT +0E32..0E33 ; ID_Continue # Lo [2] THAI CHARACTER SARA AA..THAI CHARACTER SARA AM +0E34..0E3A ; ID_Continue # Mn [7] THAI CHARACTER SARA I..THAI CHARACTER PHINTHU +0E40..0E45 ; ID_Continue # Lo [6] THAI CHARACTER SARA E..THAI CHARACTER LAKKHANGYAO +0E46 ; ID_Continue # Lm THAI CHARACTER MAIYAMOK +0E47..0E4E ; ID_Continue # Mn [8] THAI CHARACTER MAITAIKHU..THAI CHARACTER YAMAKKAN +0E50..0E59 ; ID_Continue # Nd [10] THAI DIGIT ZERO..THAI DIGIT NINE +0E81..0E82 ; ID_Continue # Lo [2] LAO LETTER KO..LAO LETTER KHO SUNG +0E84 ; ID_Continue # Lo LAO LETTER KHO TAM +0E87..0E88 ; ID_Continue # Lo [2] LAO LETTER NGO..LAO LETTER CO +0E8A ; ID_Continue # Lo LAO LETTER SO TAM +0E8D ; ID_Continue # Lo LAO LETTER NYO +0E94..0E97 ; ID_Continue # Lo [4] LAO LETTER DO..LAO LETTER THO TAM +0E99..0E9F ; ID_Continue # Lo [7] LAO LETTER NO..LAO LETTER FO SUNG +0EA1..0EA3 ; ID_Continue # Lo [3] LAO LETTER MO..LAO LETTER LO LING +0EA5 ; ID_Continue # Lo LAO LETTER LO LOOT +0EA7 ; ID_Continue # Lo LAO LETTER WO +0EAA..0EAB ; ID_Continue # Lo [2] LAO LETTER SO SUNG..LAO LETTER HO SUNG +0EAD..0EB0 ; ID_Continue # Lo [4] LAO LETTER O..LAO VOWEL SIGN A +0EB1 ; ID_Continue # Mn LAO VOWEL SIGN MAI KAN +0EB2..0EB3 ; ID_Continue # Lo [2] LAO VOWEL SIGN AA..LAO VOWEL SIGN AM +0EB4..0EB9 ; ID_Continue # Mn [6] LAO VOWEL SIGN I..LAO VOWEL SIGN UU +0EBB..0EBC ; ID_Continue # Mn [2] LAO VOWEL SIGN MAI KON..LAO SEMIVOWEL SIGN LO +0EBD ; ID_Continue # Lo LAO SEMIVOWEL SIGN NYO +0EC0..0EC4 ; ID_Continue # Lo [5] LAO VOWEL SIGN E..LAO VOWEL SIGN AI +0EC6 ; ID_Continue # Lm LAO KO LA +0EC8..0ECD ; ID_Continue # Mn [6] LAO TONE MAI EK..LAO NIGGAHITA +0ED0..0ED9 ; ID_Continue # Nd [10] LAO DIGIT ZERO..LAO DIGIT NINE +0EDC..0EDD ; ID_Continue # Lo [2] LAO HO NO..LAO HO MO +0F00 ; ID_Continue # Lo TIBETAN SYLLABLE OM +0F18..0F19 ; ID_Continue # Mn [2] TIBETAN ASTROLOGICAL SIGN -KHYUD PA..TIBETAN ASTROLOGICAL SIGN SDONG TSHUGS +0F20..0F29 ; ID_Continue # Nd [10] TIBETAN DIGIT ZERO..TIBETAN DIGIT NINE +0F35 ; ID_Continue # Mn TIBETAN MARK NGAS BZUNG NYI ZLA +0F37 ; ID_Continue # Mn TIBETAN MARK NGAS BZUNG SGOR RTAGS +0F39 ; ID_Continue # Mn TIBETAN MARK TSA -PHRU +0F3E..0F3F ; ID_Continue # Mc [2] TIBETAN SIGN YAR TSHES..TIBETAN SIGN MAR TSHES +0F40..0F47 ; ID_Continue # Lo [8] TIBETAN LETTER KA..TIBETAN LETTER JA +0F49..0F6C ; ID_Continue # Lo [36] TIBETAN LETTER NYA..TIBETAN LETTER RRA +0F71..0F7E ; ID_Continue # Mn [14] TIBETAN VOWEL SIGN AA..TIBETAN SIGN RJES SU NGA RO +0F7F ; ID_Continue # Mc TIBETAN SIGN RNAM BCAD +0F80..0F84 ; ID_Continue # Mn [5] TIBETAN VOWEL SIGN REVERSED I..TIBETAN MARK HALANTA +0F86..0F87 ; ID_Continue # Mn [2] TIBETAN SIGN LCI RTAGS..TIBETAN SIGN YANG RTAGS +0F88..0F8B ; ID_Continue # Lo [4] TIBETAN SIGN LCE TSA CAN..TIBETAN SIGN GRU MED RGYINGS +0F90..0F97 ; ID_Continue # Mn [8] TIBETAN SUBJOINED LETTER KA..TIBETAN SUBJOINED LETTER JA +0F99..0FBC ; ID_Continue # Mn [36] TIBETAN SUBJOINED LETTER NYA..TIBETAN SUBJOINED LETTER FIXED-FORM RA +0FC6 ; ID_Continue # Mn TIBETAN SYMBOL PADMA GDAN +1000..102A ; ID_Continue # Lo [43] MYANMAR LETTER KA..MYANMAR LETTER AU +102B..102C ; ID_Continue # Mc [2] MYANMAR VOWEL SIGN TALL AA..MYANMAR VOWEL SIGN AA +102D..1030 ; ID_Continue # Mn [4] MYANMAR VOWEL SIGN I..MYANMAR VOWEL SIGN UU +1031 ; ID_Continue # Mc MYANMAR VOWEL SIGN E +1032..1037 ; ID_Continue # Mn [6] MYANMAR VOWEL SIGN AI..MYANMAR SIGN DOT BELOW +1038 ; ID_Continue # Mc MYANMAR SIGN VISARGA +1039..103A ; ID_Continue # Mn [2] MYANMAR SIGN VIRAMA..MYANMAR SIGN ASAT +103B..103C ; ID_Continue # Mc [2] MYANMAR CONSONANT SIGN MEDIAL YA..MYANMAR CONSONANT SIGN MEDIAL RA +103D..103E ; ID_Continue # Mn [2] MYANMAR CONSONANT SIGN MEDIAL WA..MYANMAR CONSONANT SIGN MEDIAL HA +103F ; ID_Continue # Lo MYANMAR LETTER GREAT SA +1040..1049 ; ID_Continue # Nd [10] MYANMAR DIGIT ZERO..MYANMAR DIGIT NINE +1050..1055 ; ID_Continue # Lo [6] MYANMAR LETTER SHA..MYANMAR LETTER VOCALIC LL +1056..1057 ; ID_Continue # Mc [2] MYANMAR VOWEL SIGN VOCALIC R..MYANMAR VOWEL SIGN VOCALIC RR +1058..1059 ; ID_Continue # Mn [2] MYANMAR VOWEL SIGN VOCALIC L..MYANMAR VOWEL SIGN VOCALIC LL +105A..105D ; ID_Continue # Lo [4] MYANMAR LETTER MON NGA..MYANMAR LETTER MON BBE +105E..1060 ; ID_Continue # Mn [3] MYANMAR CONSONANT SIGN MON MEDIAL NA..MYANMAR CONSONANT SIGN MON MEDIAL LA +1061 ; ID_Continue # Lo MYANMAR LETTER SGAW KAREN SHA +1062..1064 ; ID_Continue # Mc [3] MYANMAR VOWEL SIGN SGAW KAREN EU..MYANMAR TONE MARK SGAW KAREN KE PHO +1065..1066 ; ID_Continue # Lo [2] MYANMAR LETTER WESTERN PWO KAREN THA..MYANMAR LETTER WESTERN PWO KAREN PWA +1067..106D ; ID_Continue # Mc [7] MYANMAR VOWEL SIGN WESTERN PWO KAREN EU..MYANMAR SIGN WESTERN PWO KAREN TONE-5 +106E..1070 ; ID_Continue # Lo [3] MYANMAR LETTER EASTERN PWO KAREN NNA..MYANMAR LETTER EASTERN PWO KAREN GHWA +1071..1074 ; ID_Continue # Mn [4] MYANMAR VOWEL SIGN GEBA KAREN I..MYANMAR VOWEL SIGN KAYAH EE +1075..1081 ; ID_Continue # Lo [13] MYANMAR LETTER SHAN KA..MYANMAR LETTER SHAN HA +1082 ; ID_Continue # Mn MYANMAR CONSONANT SIGN SHAN MEDIAL WA +1083..1084 ; ID_Continue # Mc [2] MYANMAR VOWEL SIGN SHAN AA..MYANMAR VOWEL SIGN SHAN E +1085..1086 ; ID_Continue # Mn [2] MYANMAR VOWEL SIGN SHAN E ABOVE..MYANMAR VOWEL SIGN SHAN FINAL Y +1087..108C ; ID_Continue # Mc [6] MYANMAR SIGN SHAN TONE-2..MYANMAR SIGN SHAN COUNCIL TONE-3 +108D ; ID_Continue # Mn MYANMAR SIGN SHAN COUNCIL EMPHATIC TONE +108E ; ID_Continue # Lo MYANMAR LETTER RUMAI PALAUNG FA +108F ; ID_Continue # Mc MYANMAR SIGN RUMAI PALAUNG TONE-5 +1090..1099 ; ID_Continue # Nd [10] MYANMAR SHAN DIGIT ZERO..MYANMAR SHAN DIGIT NINE +109A..109C ; ID_Continue # Mc [3] MYANMAR SIGN KHAMTI TONE-1..MYANMAR VOWEL SIGN AITON A +109D ; ID_Continue # Mn MYANMAR VOWEL SIGN AITON AI +10A0..10C5 ; ID_Continue # L& [38] GEORGIAN CAPITAL LETTER AN..GEORGIAN CAPITAL LETTER HOE +10D0..10FA ; ID_Continue # Lo [43] GEORGIAN LETTER AN..GEORGIAN LETTER AIN +10FC ; ID_Continue # Lm MODIFIER LETTER GEORGIAN NAR +1100..1248 ; ID_Continue # Lo [329] HANGUL CHOSEONG KIYEOK..ETHIOPIC SYLLABLE QWA +124A..124D ; ID_Continue # Lo [4] ETHIOPIC SYLLABLE QWI..ETHIOPIC SYLLABLE QWE +1250..1256 ; ID_Continue # Lo [7] ETHIOPIC SYLLABLE QHA..ETHIOPIC SYLLABLE QHO +1258 ; ID_Continue # Lo ETHIOPIC SYLLABLE QHWA +125A..125D ; ID_Continue # Lo [4] ETHIOPIC SYLLABLE QHWI..ETHIOPIC SYLLABLE QHWE +1260..1288 ; ID_Continue # Lo [41] ETHIOPIC SYLLABLE BA..ETHIOPIC SYLLABLE XWA +128A..128D ; ID_Continue # Lo [4] ETHIOPIC SYLLABLE XWI..ETHIOPIC SYLLABLE XWE +1290..12B0 ; ID_Continue # Lo [33] ETHIOPIC SYLLABLE NA..ETHIOPIC SYLLABLE KWA +12B2..12B5 ; ID_Continue # Lo [4] ETHIOPIC SYLLABLE KWI..ETHIOPIC SYLLABLE KWE +12B8..12BE ; ID_Continue # Lo [7] ETHIOPIC SYLLABLE KXA..ETHIOPIC SYLLABLE KXO +12C0 ; ID_Continue # Lo ETHIOPIC SYLLABLE KXWA +12C2..12C5 ; ID_Continue # Lo [4] ETHIOPIC SYLLABLE KXWI..ETHIOPIC SYLLABLE KXWE +12C8..12D6 ; ID_Continue # Lo [15] ETHIOPIC SYLLABLE WA..ETHIOPIC SYLLABLE PHARYNGEAL O +12D8..1310 ; ID_Continue # Lo [57] ETHIOPIC SYLLABLE ZA..ETHIOPIC SYLLABLE GWA +1312..1315 ; ID_Continue # Lo [4] ETHIOPIC SYLLABLE GWI..ETHIOPIC SYLLABLE GWE +1318..135A ; ID_Continue # Lo [67] ETHIOPIC SYLLABLE GGA..ETHIOPIC SYLLABLE FYA +135F ; ID_Continue # Mn ETHIOPIC COMBINING GEMINATION MARK +1369..1371 ; ID_Continue # No [9] ETHIOPIC DIGIT ONE..ETHIOPIC DIGIT NINE +1380..138F ; ID_Continue # Lo [16] ETHIOPIC SYLLABLE SEBATBEIT MWA..ETHIOPIC SYLLABLE PWE +13A0..13F4 ; ID_Continue # Lo [85] CHEROKEE LETTER A..CHEROKEE LETTER YV +1401..166C ; ID_Continue # Lo [620] CANADIAN SYLLABICS E..CANADIAN SYLLABICS CARRIER TTSA +166F..167F ; ID_Continue # Lo [17] CANADIAN SYLLABICS QAI..CANADIAN SYLLABICS BLACKFOOT W +1681..169A ; ID_Continue # Lo [26] OGHAM LETTER BEITH..OGHAM LETTER PEITH +16A0..16EA ; ID_Continue # Lo [75] RUNIC LETTER FEHU FEOH FE F..RUNIC LETTER X +16EE..16F0 ; ID_Continue # Nl [3] RUNIC ARLAUG SYMBOL..RUNIC BELGTHOR SYMBOL +1700..170C ; ID_Continue # Lo [13] TAGALOG LETTER A..TAGALOG LETTER YA +170E..1711 ; ID_Continue # Lo [4] TAGALOG LETTER LA..TAGALOG LETTER HA +1712..1714 ; ID_Continue # Mn [3] TAGALOG VOWEL SIGN I..TAGALOG SIGN VIRAMA +1720..1731 ; ID_Continue # Lo [18] HANUNOO LETTER A..HANUNOO LETTER HA +1732..1734 ; ID_Continue # Mn [3] HANUNOO VOWEL SIGN I..HANUNOO SIGN PAMUDPOD +1740..1751 ; ID_Continue # Lo [18] BUHID LETTER A..BUHID LETTER HA +1752..1753 ; ID_Continue # Mn [2] BUHID VOWEL SIGN I..BUHID VOWEL SIGN U +1760..176C ; ID_Continue # Lo [13] TAGBANWA LETTER A..TAGBANWA LETTER YA +176E..1770 ; ID_Continue # Lo [3] TAGBANWA LETTER LA..TAGBANWA LETTER SA +1772..1773 ; ID_Continue # Mn [2] TAGBANWA VOWEL SIGN I..TAGBANWA VOWEL SIGN U +1780..17B3 ; ID_Continue # Lo [52] KHMER LETTER KA..KHMER INDEPENDENT VOWEL QAU +17B6 ; ID_Continue # Mc KHMER VOWEL SIGN AA +17B7..17BD ; ID_Continue # Mn [7] KHMER VOWEL SIGN I..KHMER VOWEL SIGN UA +17BE..17C5 ; ID_Continue # Mc [8] KHMER VOWEL SIGN OE..KHMER VOWEL SIGN AU +17C6 ; ID_Continue # Mn KHMER SIGN NIKAHIT +17C7..17C8 ; ID_Continue # Mc [2] KHMER SIGN REAHMUK..KHMER SIGN YUUKALEAPINTU +17C9..17D3 ; ID_Continue # Mn [11] KHMER SIGN MUUSIKATOAN..KHMER SIGN BATHAMASAT +17D7 ; ID_Continue # Lm KHMER SIGN LEK TOO +17DC ; ID_Continue # Lo KHMER SIGN AVAKRAHASANYA +17DD ; ID_Continue # Mn KHMER SIGN ATTHACAN +17E0..17E9 ; ID_Continue # Nd [10] KHMER DIGIT ZERO..KHMER DIGIT NINE +180B..180D ; ID_Continue # Mn [3] MONGOLIAN FREE VARIATION SELECTOR ONE..MONGOLIAN FREE VARIATION SELECTOR THREE +1810..1819 ; ID_Continue # Nd [10] MONGOLIAN DIGIT ZERO..MONGOLIAN DIGIT NINE +1820..1842 ; ID_Continue # Lo [35] MONGOLIAN LETTER A..MONGOLIAN LETTER CHI +1843 ; ID_Continue # Lm MONGOLIAN LETTER TODO LONG VOWEL SIGN +1844..1877 ; ID_Continue # Lo [52] MONGOLIAN LETTER TODO E..MONGOLIAN LETTER MANCHU ZHA +1880..18A8 ; ID_Continue # Lo [41] MONGOLIAN LETTER ALI GALI ANUSVARA ONE..MONGOLIAN LETTER MANCHU ALI GALI BHA +18A9 ; ID_Continue # Mn MONGOLIAN LETTER ALI GALI DAGALGA +18AA ; ID_Continue # Lo MONGOLIAN LETTER MANCHU ALI GALI LHA +18B0..18F5 ; ID_Continue # Lo [70] CANADIAN SYLLABICS OY..CANADIAN SYLLABICS CARRIER DENTAL S +1900..191C ; ID_Continue # Lo [29] LIMBU VOWEL-CARRIER LETTER..LIMBU LETTER HA +1920..1922 ; ID_Continue # Mn [3] LIMBU VOWEL SIGN A..LIMBU VOWEL SIGN U +1923..1926 ; ID_Continue # Mc [4] LIMBU VOWEL SIGN EE..LIMBU VOWEL SIGN AU +1927..1928 ; ID_Continue # Mn [2] LIMBU VOWEL SIGN E..LIMBU VOWEL SIGN O +1929..192B ; ID_Continue # Mc [3] LIMBU SUBJOINED LETTER YA..LIMBU SUBJOINED LETTER WA +1930..1931 ; ID_Continue # Mc [2] LIMBU SMALL LETTER KA..LIMBU SMALL LETTER NGA +1932 ; ID_Continue # Mn LIMBU SMALL LETTER ANUSVARA +1933..1938 ; ID_Continue # Mc [6] LIMBU SMALL LETTER TA..LIMBU SMALL LETTER LA +1939..193B ; ID_Continue # Mn [3] LIMBU SIGN MUKPHRENG..LIMBU SIGN SA-I +1946..194F ; ID_Continue # Nd [10] LIMBU DIGIT ZERO..LIMBU DIGIT NINE +1950..196D ; ID_Continue # Lo [30] TAI LE LETTER KA..TAI LE LETTER AI +1970..1974 ; ID_Continue # Lo [5] TAI LE LETTER TONE-2..TAI LE LETTER TONE-6 +1980..19AB ; ID_Continue # Lo [44] NEW TAI LUE LETTER HIGH QA..NEW TAI LUE LETTER LOW SUA +19B0..19C0 ; ID_Continue # Mc [17] NEW TAI LUE VOWEL SIGN VOWEL SHORTENER..NEW TAI LUE VOWEL SIGN IY +19C1..19C7 ; ID_Continue # Lo [7] NEW TAI LUE LETTER FINAL V..NEW TAI LUE LETTER FINAL B +19C8..19C9 ; ID_Continue # Mc [2] NEW TAI LUE TONE MARK-1..NEW TAI LUE TONE MARK-2 +19D0..19DA ; ID_Continue # Nd [11] NEW TAI LUE DIGIT ZERO..NEW TAI LUE THAM DIGIT ONE +1A00..1A16 ; ID_Continue # Lo [23] BUGINESE LETTER KA..BUGINESE LETTER HA +1A17..1A18 ; ID_Continue # Mn [2] BUGINESE VOWEL SIGN I..BUGINESE VOWEL SIGN U +1A19..1A1B ; ID_Continue # Mc [3] BUGINESE VOWEL SIGN E..BUGINESE VOWEL SIGN AE +1A20..1A54 ; ID_Continue # Lo [53] TAI THAM LETTER HIGH KA..TAI THAM LETTER GREAT SA +1A55 ; ID_Continue # Mc TAI THAM CONSONANT SIGN MEDIAL RA +1A56 ; ID_Continue # Mn TAI THAM CONSONANT SIGN MEDIAL LA +1A57 ; ID_Continue # Mc TAI THAM CONSONANT SIGN LA TANG LAI +1A58..1A5E ; ID_Continue # Mn [7] TAI THAM SIGN MAI KANG LAI..TAI THAM CONSONANT SIGN SA +1A60 ; ID_Continue # Mn TAI THAM SIGN SAKOT +1A61 ; ID_Continue # Mc TAI THAM VOWEL SIGN A +1A62 ; ID_Continue # Mn TAI THAM VOWEL SIGN MAI SAT +1A63..1A64 ; ID_Continue # Mc [2] TAI THAM VOWEL SIGN AA..TAI THAM VOWEL SIGN TALL AA +1A65..1A6C ; ID_Continue # Mn [8] TAI THAM VOWEL SIGN I..TAI THAM VOWEL SIGN OA BELOW +1A6D..1A72 ; ID_Continue # Mc [6] TAI THAM VOWEL SIGN OY..TAI THAM VOWEL SIGN THAM AI +1A73..1A7C ; ID_Continue # Mn [10] TAI THAM VOWEL SIGN OA ABOVE..TAI THAM SIGN KHUEN-LUE KARAN +1A7F ; ID_Continue # Mn TAI THAM COMBINING CRYPTOGRAMMIC DOT +1A80..1A89 ; ID_Continue # Nd [10] TAI THAM HORA DIGIT ZERO..TAI THAM HORA DIGIT NINE +1A90..1A99 ; ID_Continue # Nd [10] TAI THAM THAM DIGIT ZERO..TAI THAM THAM DIGIT NINE +1AA7 ; ID_Continue # Lm TAI THAM SIGN MAI YAMOK +1B00..1B03 ; ID_Continue # Mn [4] BALINESE SIGN ULU RICEM..BALINESE SIGN SURANG +1B04 ; ID_Continue # Mc BALINESE SIGN BISAH +1B05..1B33 ; ID_Continue # Lo [47] BALINESE LETTER AKARA..BALINESE LETTER HA +1B34 ; ID_Continue # Mn BALINESE SIGN REREKAN +1B35 ; ID_Continue # Mc BALINESE VOWEL SIGN TEDUNG +1B36..1B3A ; ID_Continue # Mn [5] BALINESE VOWEL SIGN ULU..BALINESE VOWEL SIGN RA REPA +1B3B ; ID_Continue # Mc BALINESE VOWEL SIGN RA REPA TEDUNG +1B3C ; ID_Continue # Mn BALINESE VOWEL SIGN LA LENGA +1B3D..1B41 ; ID_Continue # Mc [5] BALINESE VOWEL SIGN LA LENGA TEDUNG..BALINESE VOWEL SIGN TALING REPA TEDUNG +1B42 ; ID_Continue # Mn BALINESE VOWEL SIGN PEPET +1B43..1B44 ; ID_Continue # Mc [2] BALINESE VOWEL SIGN PEPET TEDUNG..BALINESE ADEG ADEG +1B45..1B4B ; ID_Continue # Lo [7] BALINESE LETTER KAF SASAK..BALINESE LETTER ASYURA SASAK +1B50..1B59 ; ID_Continue # Nd [10] BALINESE DIGIT ZERO..BALINESE DIGIT NINE +1B6B..1B73 ; ID_Continue # Mn [9] BALINESE MUSICAL SYMBOL COMBINING TEGEH..BALINESE MUSICAL SYMBOL COMBINING GONG +1B80..1B81 ; ID_Continue # Mn [2] SUNDANESE SIGN PANYECEK..SUNDANESE SIGN PANGLAYAR +1B82 ; ID_Continue # Mc SUNDANESE SIGN PANGWISAD +1B83..1BA0 ; ID_Continue # Lo [30] SUNDANESE LETTER A..SUNDANESE LETTER HA +1BA1 ; ID_Continue # Mc SUNDANESE CONSONANT SIGN PAMINGKAL +1BA2..1BA5 ; ID_Continue # Mn [4] SUNDANESE CONSONANT SIGN PANYAKRA..SUNDANESE VOWEL SIGN PANYUKU +1BA6..1BA7 ; ID_Continue # Mc [2] SUNDANESE VOWEL SIGN PANAELAENG..SUNDANESE VOWEL SIGN PANOLONG +1BA8..1BA9 ; ID_Continue # Mn [2] SUNDANESE VOWEL SIGN PAMEPET..SUNDANESE VOWEL SIGN PANEULEUNG +1BAA ; ID_Continue # Mc SUNDANESE SIGN PAMAAEH +1BAE..1BAF ; ID_Continue # Lo [2] SUNDANESE LETTER KHA..SUNDANESE LETTER SYA +1BB0..1BB9 ; ID_Continue # Nd [10] SUNDANESE DIGIT ZERO..SUNDANESE DIGIT NINE +1C00..1C23 ; ID_Continue # Lo [36] LEPCHA LETTER KA..LEPCHA LETTER A +1C24..1C2B ; ID_Continue # Mc [8] LEPCHA SUBJOINED LETTER YA..LEPCHA VOWEL SIGN UU +1C2C..1C33 ; ID_Continue # Mn [8] LEPCHA VOWEL SIGN E..LEPCHA CONSONANT SIGN T +1C34..1C35 ; ID_Continue # Mc [2] LEPCHA CONSONANT SIGN NYIN-DO..LEPCHA CONSONANT SIGN KANG +1C36..1C37 ; ID_Continue # Mn [2] LEPCHA SIGN RAN..LEPCHA SIGN NUKTA +1C40..1C49 ; ID_Continue # Nd [10] LEPCHA DIGIT ZERO..LEPCHA DIGIT NINE +1C4D..1C4F ; ID_Continue # Lo [3] LEPCHA LETTER TTA..LEPCHA LETTER DDA +1C50..1C59 ; ID_Continue # Nd [10] OL CHIKI DIGIT ZERO..OL CHIKI DIGIT NINE +1C5A..1C77 ; ID_Continue # Lo [30] OL CHIKI LETTER LA..OL CHIKI LETTER OH +1C78..1C7D ; ID_Continue # Lm [6] OL CHIKI MU TTUDDAG..OL CHIKI AHAD +1CD0..1CD2 ; ID_Continue # Mn [3] VEDIC TONE KARSHANA..VEDIC TONE PRENKHA +1CD4..1CE0 ; ID_Continue # Mn [13] VEDIC SIGN YAJURVEDIC MIDLINE SVARITA..VEDIC TONE RIGVEDIC KASHMIRI INDEPENDENT SVARITA +1CE1 ; ID_Continue # Mc VEDIC TONE ATHARVAVEDIC INDEPENDENT SVARITA +1CE2..1CE8 ; ID_Continue # Mn [7] VEDIC SIGN VISARGA SVARITA..VEDIC SIGN VISARGA ANUDATTA WITH TAIL +1CE9..1CEC ; ID_Continue # Lo [4] VEDIC SIGN ANUSVARA ANTARGOMUKHA..VEDIC SIGN ANUSVARA VAMAGOMUKHA WITH TAIL +1CED ; ID_Continue # Mn VEDIC SIGN TIRYAK +1CEE..1CF1 ; ID_Continue # Lo [4] VEDIC SIGN HEXIFORM LONG ANUSVARA..VEDIC SIGN ANUSVARA UBHAYATO MUKHA +1CF2 ; ID_Continue # Mc VEDIC SIGN ARDHAVISARGA +1D00..1D2B ; ID_Continue # L& [44] LATIN LETTER SMALL CAPITAL A..CYRILLIC LETTER SMALL CAPITAL EL +1D2C..1D61 ; ID_Continue # Lm [54] MODIFIER LETTER CAPITAL A..MODIFIER LETTER SMALL CHI +1D62..1D77 ; ID_Continue # L& [22] LATIN SUBSCRIPT SMALL LETTER I..LATIN SMALL LETTER TURNED G +1D78 ; ID_Continue # Lm MODIFIER LETTER CYRILLIC EN +1D79..1D9A ; ID_Continue # L& [34] LATIN SMALL LETTER INSULAR G..LATIN SMALL LETTER EZH WITH RETROFLEX HOOK +1D9B..1DBF ; ID_Continue # Lm [37] MODIFIER LETTER SMALL TURNED ALPHA..MODIFIER LETTER SMALL THETA +1DC0..1DE6 ; ID_Continue # Mn [39] COMBINING DOTTED GRAVE ACCENT..COMBINING LATIN SMALL LETTER Z +1DFD..1DFF ; ID_Continue # Mn [3] COMBINING ALMOST EQUAL TO BELOW..COMBINING RIGHT ARROWHEAD AND DOWN ARROWHEAD BELOW +1E00..1F15 ; ID_Continue # L& [278] LATIN CAPITAL LETTER A WITH RING BELOW..GREEK SMALL LETTER EPSILON WITH DASIA AND OXIA +1F18..1F1D ; ID_Continue # L& [6] GREEK CAPITAL LETTER EPSILON WITH PSILI..GREEK CAPITAL LETTER EPSILON WITH DASIA AND OXIA +1F20..1F45 ; ID_Continue # L& [38] GREEK SMALL LETTER ETA WITH PSILI..GREEK SMALL LETTER OMICRON WITH DASIA AND OXIA +1F48..1F4D ; ID_Continue # L& [6] GREEK CAPITAL LETTER OMICRON WITH PSILI..GREEK CAPITAL LETTER OMICRON WITH DASIA AND OXIA +1F50..1F57 ; ID_Continue # L& [8] GREEK SMALL LETTER UPSILON WITH PSILI..GREEK SMALL LETTER UPSILON WITH DASIA AND PERISPOMENI +1F59 ; ID_Continue # L& GREEK CAPITAL LETTER UPSILON WITH DASIA +1F5B ; ID_Continue # L& GREEK CAPITAL LETTER UPSILON WITH DASIA AND VARIA +1F5D ; ID_Continue # L& GREEK CAPITAL LETTER UPSILON WITH DASIA AND OXIA +1F5F..1F7D ; ID_Continue # L& [31] GREEK CAPITAL LETTER UPSILON WITH DASIA AND PERISPOMENI..GREEK SMALL LETTER OMEGA WITH OXIA +1F80..1FB4 ; ID_Continue # L& [53] GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI..GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI +1FB6..1FBC ; ID_Continue # L& [7] GREEK SMALL LETTER ALPHA WITH PERISPOMENI..GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI +1FBE ; ID_Continue # L& GREEK PROSGEGRAMMENI +1FC2..1FC4 ; ID_Continue # L& [3] GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI..GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI +1FC6..1FCC ; ID_Continue # L& [7] GREEK SMALL LETTER ETA WITH PERISPOMENI..GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI +1FD0..1FD3 ; ID_Continue # L& [4] GREEK SMALL LETTER IOTA WITH VRACHY..GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA +1FD6..1FDB ; ID_Continue # L& [6] GREEK SMALL LETTER IOTA WITH PERISPOMENI..GREEK CAPITAL LETTER IOTA WITH OXIA +1FE0..1FEC ; ID_Continue # L& [13] GREEK SMALL LETTER UPSILON WITH VRACHY..GREEK CAPITAL LETTER RHO WITH DASIA +1FF2..1FF4 ; ID_Continue # L& [3] GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI..GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI +1FF6..1FFC ; ID_Continue # L& [7] GREEK SMALL LETTER OMEGA WITH PERISPOMENI..GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI +203F..2040 ; ID_Continue # Pc [2] UNDERTIE..CHARACTER TIE +2054 ; ID_Continue # Pc INVERTED UNDERTIE +2071 ; ID_Continue # Lm SUPERSCRIPT LATIN SMALL LETTER I +207F ; ID_Continue # Lm SUPERSCRIPT LATIN SMALL LETTER N +2090..2094 ; ID_Continue # Lm [5] LATIN SUBSCRIPT SMALL LETTER A..LATIN SUBSCRIPT SMALL LETTER SCHWA +20D0..20DC ; ID_Continue # Mn [13] COMBINING LEFT HARPOON ABOVE..COMBINING FOUR DOTS ABOVE +20E1 ; ID_Continue # Mn COMBINING LEFT RIGHT ARROW ABOVE +20E5..20F0 ; ID_Continue # Mn [12] COMBINING REVERSE SOLIDUS OVERLAY..COMBINING ASTERISK ABOVE +2102 ; ID_Continue # L& DOUBLE-STRUCK CAPITAL C +2107 ; ID_Continue # L& EULER CONSTANT +210A..2113 ; ID_Continue # L& [10] SCRIPT SMALL G..SCRIPT SMALL L +2115 ; ID_Continue # L& DOUBLE-STRUCK CAPITAL N +2118 ; ID_Continue # So SCRIPT CAPITAL P +2119..211D ; ID_Continue # L& [5] DOUBLE-STRUCK CAPITAL P..DOUBLE-STRUCK CAPITAL R +2124 ; ID_Continue # L& DOUBLE-STRUCK CAPITAL Z +2126 ; ID_Continue # L& OHM SIGN +2128 ; ID_Continue # L& BLACK-LETTER CAPITAL Z +212A..212D ; ID_Continue # L& [4] KELVIN SIGN..BLACK-LETTER CAPITAL C +212E ; ID_Continue # So ESTIMATED SYMBOL +212F..2134 ; ID_Continue # L& [6] SCRIPT SMALL E..SCRIPT SMALL O +2135..2138 ; ID_Continue # Lo [4] ALEF SYMBOL..DALET SYMBOL +2139 ; ID_Continue # L& INFORMATION SOURCE +213C..213F ; ID_Continue # L& [4] DOUBLE-STRUCK SMALL PI..DOUBLE-STRUCK CAPITAL PI +2145..2149 ; ID_Continue # L& [5] DOUBLE-STRUCK ITALIC CAPITAL D..DOUBLE-STRUCK ITALIC SMALL J +214E ; ID_Continue # L& TURNED SMALL F +2160..2182 ; ID_Continue # Nl [35] ROMAN NUMERAL ONE..ROMAN NUMERAL TEN THOUSAND +2183..2184 ; ID_Continue # L& [2] ROMAN NUMERAL REVERSED ONE HUNDRED..LATIN SMALL LETTER REVERSED C +2185..2188 ; ID_Continue # Nl [4] ROMAN NUMERAL SIX LATE FORM..ROMAN NUMERAL ONE HUNDRED THOUSAND +2C00..2C2E ; ID_Continue # L& [47] GLAGOLITIC CAPITAL LETTER AZU..GLAGOLITIC CAPITAL LETTER LATINATE MYSLITE +2C30..2C5E ; ID_Continue # L& [47] GLAGOLITIC SMALL LETTER AZU..GLAGOLITIC SMALL LETTER LATINATE MYSLITE +2C60..2C7C ; ID_Continue # L& [29] LATIN CAPITAL LETTER L WITH DOUBLE BAR..LATIN SUBSCRIPT SMALL LETTER J +2C7D ; ID_Continue # Lm MODIFIER LETTER CAPITAL V +2C7E..2CE4 ; ID_Continue # L& [103] LATIN CAPITAL LETTER S WITH SWASH TAIL..COPTIC SYMBOL KAI +2CEB..2CEE ; ID_Continue # L& [4] COPTIC CAPITAL LETTER CRYPTOGRAMMIC SHEI..COPTIC SMALL LETTER CRYPTOGRAMMIC GANGIA +2CEF..2CF1 ; ID_Continue # Mn [3] COPTIC COMBINING NI ABOVE..COPTIC COMBINING SPIRITUS LENIS +2D00..2D25 ; ID_Continue # L& [38] GEORGIAN SMALL LETTER AN..GEORGIAN SMALL LETTER HOE +2D30..2D65 ; ID_Continue # Lo [54] TIFINAGH LETTER YA..TIFINAGH LETTER YAZZ +2D6F ; ID_Continue # Lm TIFINAGH MODIFIER LETTER LABIALIZATION MARK +2D80..2D96 ; ID_Continue # Lo [23] ETHIOPIC SYLLABLE LOA..ETHIOPIC SYLLABLE GGWE +2DA0..2DA6 ; ID_Continue # Lo [7] ETHIOPIC SYLLABLE SSA..ETHIOPIC SYLLABLE SSO +2DA8..2DAE ; ID_Continue # Lo [7] ETHIOPIC SYLLABLE CCA..ETHIOPIC SYLLABLE CCO +2DB0..2DB6 ; ID_Continue # Lo [7] ETHIOPIC SYLLABLE ZZA..ETHIOPIC SYLLABLE ZZO +2DB8..2DBE ; ID_Continue # Lo [7] ETHIOPIC SYLLABLE CCHA..ETHIOPIC SYLLABLE CCHO +2DC0..2DC6 ; ID_Continue # Lo [7] ETHIOPIC SYLLABLE QYA..ETHIOPIC SYLLABLE QYO +2DC8..2DCE ; ID_Continue # Lo [7] ETHIOPIC SYLLABLE KYA..ETHIOPIC SYLLABLE KYO +2DD0..2DD6 ; ID_Continue # Lo [7] ETHIOPIC SYLLABLE XYA..ETHIOPIC SYLLABLE XYO +2DD8..2DDE ; ID_Continue # Lo [7] ETHIOPIC SYLLABLE GYA..ETHIOPIC SYLLABLE GYO +2DE0..2DFF ; ID_Continue # Mn [32] COMBINING CYRILLIC LETTER BE..COMBINING CYRILLIC LETTER IOTIFIED BIG YUS +3005 ; ID_Continue # Lm IDEOGRAPHIC ITERATION MARK +3006 ; ID_Continue # Lo IDEOGRAPHIC CLOSING MARK +3007 ; ID_Continue # Nl IDEOGRAPHIC NUMBER ZERO +3021..3029 ; ID_Continue # Nl [9] HANGZHOU NUMERAL ONE..HANGZHOU NUMERAL NINE +302A..302F ; ID_Continue # Mn [6] IDEOGRAPHIC LEVEL TONE MARK..HANGUL DOUBLE DOT TONE MARK +3031..3035 ; ID_Continue # Lm [5] VERTICAL KANA REPEAT MARK..VERTICAL KANA REPEAT MARK LOWER HALF +3038..303A ; ID_Continue # Nl [3] HANGZHOU NUMERAL TEN..HANGZHOU NUMERAL THIRTY +303B ; ID_Continue # Lm VERTICAL IDEOGRAPHIC ITERATION MARK +303C ; ID_Continue # Lo MASU MARK +3041..3096 ; ID_Continue # Lo [86] HIRAGANA LETTER SMALL A..HIRAGANA LETTER SMALL KE +3099..309A ; ID_Continue # Mn [2] COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK..COMBINING KATAKANA-HIRAGANA SEMI-VOICED SOUND MARK +309B..309C ; ID_Continue # Sk [2] KATAKANA-HIRAGANA VOICED SOUND MARK..KATAKANA-HIRAGANA SEMI-VOICED SOUND MARK +309D..309E ; ID_Continue # Lm [2] HIRAGANA ITERATION MARK..HIRAGANA VOICED ITERATION MARK +309F ; ID_Continue # Lo HIRAGANA DIGRAPH YORI +30A1..30FA ; ID_Continue # Lo [90] KATAKANA LETTER SMALL A..KATAKANA LETTER VO +30FC..30FE ; ID_Continue # Lm [3] KATAKANA-HIRAGANA PROLONGED SOUND MARK..KATAKANA VOICED ITERATION MARK +30FF ; ID_Continue # Lo KATAKANA DIGRAPH KOTO +3105..312D ; ID_Continue # Lo [41] BOPOMOFO LETTER B..BOPOMOFO LETTER IH +3131..318E ; ID_Continue # Lo [94] HANGUL LETTER KIYEOK..HANGUL LETTER ARAEAE +31A0..31B7 ; ID_Continue # Lo [24] BOPOMOFO LETTER BU..BOPOMOFO FINAL LETTER H +31F0..31FF ; ID_Continue # Lo [16] KATAKANA LETTER SMALL KU..KATAKANA LETTER SMALL RO +3400..4DB5 ; ID_Continue # Lo [6582] CJK UNIFIED IDEOGRAPH-3400..CJK UNIFIED IDEOGRAPH-4DB5 +4E00..9FCB ; ID_Continue # Lo [20940] CJK UNIFIED IDEOGRAPH-4E00..CJK UNIFIED IDEOGRAPH-9FCB +A000..A014 ; ID_Continue # Lo [21] YI SYLLABLE IT..YI SYLLABLE E +A015 ; ID_Continue # Lm YI SYLLABLE WU +A016..A48C ; ID_Continue # Lo [1143] YI SYLLABLE BIT..YI SYLLABLE YYR +A4D0..A4F7 ; ID_Continue # Lo [40] LISU LETTER BA..LISU LETTER OE +A4F8..A4FD ; ID_Continue # Lm [6] LISU LETTER TONE MYA TI..LISU LETTER TONE MYA JEU +A500..A60B ; ID_Continue # Lo [268] VAI SYLLABLE EE..VAI SYLLABLE NG +A60C ; ID_Continue # Lm VAI SYLLABLE LENGTHENER +A610..A61F ; ID_Continue # Lo [16] VAI SYLLABLE NDOLE FA..VAI SYMBOL JONG +A620..A629 ; ID_Continue # Nd [10] VAI DIGIT ZERO..VAI DIGIT NINE +A62A..A62B ; ID_Continue # Lo [2] VAI SYLLABLE NDOLE MA..VAI SYLLABLE NDOLE DO +A640..A65F ; ID_Continue # L& [32] CYRILLIC CAPITAL LETTER ZEMLYA..CYRILLIC SMALL LETTER YN +A662..A66D ; ID_Continue # L& [12] CYRILLIC CAPITAL LETTER SOFT DE..CYRILLIC SMALL LETTER DOUBLE MONOCULAR O +A66E ; ID_Continue # Lo CYRILLIC LETTER MULTIOCULAR O +A66F ; ID_Continue # Mn COMBINING CYRILLIC VZMET +A67C..A67D ; ID_Continue # Mn [2] COMBINING CYRILLIC KAVYKA..COMBINING CYRILLIC PAYEROK +A67F ; ID_Continue # Lm CYRILLIC PAYEROK +A680..A697 ; ID_Continue # L& [24] CYRILLIC CAPITAL LETTER DWE..CYRILLIC SMALL LETTER SHWE +A6A0..A6E5 ; ID_Continue # Lo [70] BAMUM LETTER A..BAMUM LETTER KI +A6E6..A6EF ; ID_Continue # Nl [10] BAMUM LETTER MO..BAMUM LETTER KOGHOM +A6F0..A6F1 ; ID_Continue # Mn [2] BAMUM COMBINING MARK KOQNDON..BAMUM COMBINING MARK TUKWENTIS +A717..A71F ; ID_Continue # Lm [9] MODIFIER LETTER DOT VERTICAL BAR..MODIFIER LETTER LOW INVERTED EXCLAMATION MARK +A722..A76F ; ID_Continue # L& [78] LATIN CAPITAL LETTER EGYPTOLOGICAL ALEF..LATIN SMALL LETTER CON +A770 ; ID_Continue # Lm MODIFIER LETTER US +A771..A787 ; ID_Continue # L& [23] LATIN SMALL LETTER DUM..LATIN SMALL LETTER INSULAR T +A788 ; ID_Continue # Lm MODIFIER LETTER LOW CIRCUMFLEX ACCENT +A78B..A78C ; ID_Continue # L& [2] LATIN CAPITAL LETTER SALTILLO..LATIN SMALL LETTER SALTILLO +A7FB..A801 ; ID_Continue # Lo [7] LATIN EPIGRAPHIC LETTER REVERSED F..SYLOTI NAGRI LETTER I +A802 ; ID_Continue # Mn SYLOTI NAGRI SIGN DVISVARA +A803..A805 ; ID_Continue # Lo [3] SYLOTI NAGRI LETTER U..SYLOTI NAGRI LETTER O +A806 ; ID_Continue # Mn SYLOTI NAGRI SIGN HASANTA +A807..A80A ; ID_Continue # Lo [4] SYLOTI NAGRI LETTER KO..SYLOTI NAGRI LETTER GHO +A80B ; ID_Continue # Mn SYLOTI NAGRI SIGN ANUSVARA +A80C..A822 ; ID_Continue # Lo [23] SYLOTI NAGRI LETTER CO..SYLOTI NAGRI LETTER HO +A823..A824 ; ID_Continue # Mc [2] SYLOTI NAGRI VOWEL SIGN A..SYLOTI NAGRI VOWEL SIGN I +A825..A826 ; ID_Continue # Mn [2] SYLOTI NAGRI VOWEL SIGN U..SYLOTI NAGRI VOWEL SIGN E +A827 ; ID_Continue # Mc SYLOTI NAGRI VOWEL SIGN OO +A840..A873 ; ID_Continue # Lo [52] PHAGS-PA LETTER KA..PHAGS-PA LETTER CANDRABINDU +A880..A881 ; ID_Continue # Mc [2] SAURASHTRA SIGN ANUSVARA..SAURASHTRA SIGN VISARGA +A882..A8B3 ; ID_Continue # Lo [50] SAURASHTRA LETTER A..SAURASHTRA LETTER LLA +A8B4..A8C3 ; ID_Continue # Mc [16] SAURASHTRA CONSONANT SIGN HAARU..SAURASHTRA VOWEL SIGN AU +A8C4 ; ID_Continue # Mn SAURASHTRA SIGN VIRAMA +A8D0..A8D9 ; ID_Continue # Nd [10] SAURASHTRA DIGIT ZERO..SAURASHTRA DIGIT NINE +A8E0..A8F1 ; ID_Continue # Mn [18] COMBINING DEVANAGARI DIGIT ZERO..COMBINING DEVANAGARI SIGN AVAGRAHA +A8F2..A8F7 ; ID_Continue # Lo [6] DEVANAGARI SIGN SPACING CANDRABINDU..DEVANAGARI SIGN CANDRABINDU AVAGRAHA +A8FB ; ID_Continue # Lo DEVANAGARI HEADSTROKE +A900..A909 ; ID_Continue # Nd [10] KAYAH LI DIGIT ZERO..KAYAH LI DIGIT NINE +A90A..A925 ; ID_Continue # Lo [28] KAYAH LI LETTER KA..KAYAH LI LETTER OO +A926..A92D ; ID_Continue # Mn [8] KAYAH LI VOWEL UE..KAYAH LI TONE CALYA PLOPHU +A930..A946 ; ID_Continue # Lo [23] REJANG LETTER KA..REJANG LETTER A +A947..A951 ; ID_Continue # Mn [11] REJANG VOWEL SIGN I..REJANG CONSONANT SIGN R +A952..A953 ; ID_Continue # Mc [2] REJANG CONSONANT SIGN H..REJANG VIRAMA +A960..A97C ; ID_Continue # Lo [29] HANGUL CHOSEONG TIKEUT-MIEUM..HANGUL CHOSEONG SSANGYEORINHIEUH +A980..A982 ; ID_Continue # Mn [3] JAVANESE SIGN PANYANGGA..JAVANESE SIGN LAYAR +A983 ; ID_Continue # Mc JAVANESE SIGN WIGNYAN +A984..A9B2 ; ID_Continue # Lo [47] JAVANESE LETTER A..JAVANESE LETTER HA +A9B3 ; ID_Continue # Mn JAVANESE SIGN CECAK TELU +A9B4..A9B5 ; ID_Continue # Mc [2] JAVANESE VOWEL SIGN TARUNG..JAVANESE VOWEL SIGN TOLONG +A9B6..A9B9 ; ID_Continue # Mn [4] JAVANESE VOWEL SIGN WULU..JAVANESE VOWEL SIGN SUKU MENDUT +A9BA..A9BB ; ID_Continue # Mc [2] JAVANESE VOWEL SIGN TALING..JAVANESE VOWEL SIGN DIRGA MURE +A9BC ; ID_Continue # Mn JAVANESE VOWEL SIGN PEPET +A9BD..A9C0 ; ID_Continue # Mc [4] JAVANESE CONSONANT SIGN KERET..JAVANESE PANGKON +A9CF ; ID_Continue # Lm JAVANESE PANGRANGKEP +A9D0..A9D9 ; ID_Continue # Nd [10] JAVANESE DIGIT ZERO..JAVANESE DIGIT NINE +AA00..AA28 ; ID_Continue # Lo [41] CHAM LETTER A..CHAM LETTER HA +AA29..AA2E ; ID_Continue # Mn [6] CHAM VOWEL SIGN AA..CHAM VOWEL SIGN OE +AA2F..AA30 ; ID_Continue # Mc [2] CHAM VOWEL SIGN O..CHAM VOWEL SIGN AI +AA31..AA32 ; ID_Continue # Mn [2] CHAM VOWEL SIGN AU..CHAM VOWEL SIGN UE +AA33..AA34 ; ID_Continue # Mc [2] CHAM CONSONANT SIGN YA..CHAM CONSONANT SIGN RA +AA35..AA36 ; ID_Continue # Mn [2] CHAM CONSONANT SIGN LA..CHAM CONSONANT SIGN WA +AA40..AA42 ; ID_Continue # Lo [3] CHAM LETTER FINAL K..CHAM LETTER FINAL NG +AA43 ; ID_Continue # Mn CHAM CONSONANT SIGN FINAL NG +AA44..AA4B ; ID_Continue # Lo [8] CHAM LETTER FINAL CH..CHAM LETTER FINAL SS +AA4C ; ID_Continue # Mn CHAM CONSONANT SIGN FINAL M +AA4D ; ID_Continue # Mc CHAM CONSONANT SIGN FINAL H +AA50..AA59 ; ID_Continue # Nd [10] CHAM DIGIT ZERO..CHAM DIGIT NINE +AA60..AA6F ; ID_Continue # Lo [16] MYANMAR LETTER KHAMTI GA..MYANMAR LETTER KHAMTI FA +AA70 ; ID_Continue # Lm MYANMAR MODIFIER LETTER KHAMTI REDUPLICATION +AA71..AA76 ; ID_Continue # Lo [6] MYANMAR LETTER KHAMTI XA..MYANMAR LOGOGRAM KHAMTI HM +AA7A ; ID_Continue # Lo MYANMAR LETTER AITON RA +AA7B ; ID_Continue # Mc MYANMAR SIGN PAO KAREN TONE +AA80..AAAF ; ID_Continue # Lo [48] TAI VIET LETTER LOW KO..TAI VIET LETTER HIGH O +AAB0 ; ID_Continue # Mn TAI VIET MAI KANG +AAB1 ; ID_Continue # Lo TAI VIET VOWEL AA +AAB2..AAB4 ; ID_Continue # Mn [3] TAI VIET VOWEL I..TAI VIET VOWEL U +AAB5..AAB6 ; ID_Continue # Lo [2] TAI VIET VOWEL E..TAI VIET VOWEL O +AAB7..AAB8 ; ID_Continue # Mn [2] TAI VIET MAI KHIT..TAI VIET VOWEL IA +AAB9..AABD ; ID_Continue # Lo [5] TAI VIET VOWEL UEA..TAI VIET VOWEL AN +AABE..AABF ; ID_Continue # Mn [2] TAI VIET VOWEL AM..TAI VIET TONE MAI EK +AAC0 ; ID_Continue # Lo TAI VIET TONE MAI NUENG +AAC1 ; ID_Continue # Mn TAI VIET TONE MAI THO +AAC2 ; ID_Continue # Lo TAI VIET TONE MAI SONG +AADB..AADC ; ID_Continue # Lo [2] TAI VIET SYMBOL KON..TAI VIET SYMBOL NUENG +AADD ; ID_Continue # Lm TAI VIET SYMBOL SAM +ABC0..ABE2 ; ID_Continue # Lo [35] MEETEI MAYEK LETTER KOK..MEETEI MAYEK LETTER I LONSUM +ABE3..ABE4 ; ID_Continue # Mc [2] MEETEI MAYEK VOWEL SIGN ONAP..MEETEI MAYEK VOWEL SIGN INAP +ABE5 ; ID_Continue # Mn MEETEI MAYEK VOWEL SIGN ANAP +ABE6..ABE7 ; ID_Continue # Mc [2] MEETEI MAYEK VOWEL SIGN YENAP..MEETEI MAYEK VOWEL SIGN SOUNAP +ABE8 ; ID_Continue # Mn MEETEI MAYEK VOWEL SIGN UNAP +ABE9..ABEA ; ID_Continue # Mc [2] MEETEI MAYEK VOWEL SIGN CHEINAP..MEETEI MAYEK VOWEL SIGN NUNG +ABEC ; ID_Continue # Mc MEETEI MAYEK LUM IYEK +ABED ; ID_Continue # Mn MEETEI MAYEK APUN IYEK +ABF0..ABF9 ; ID_Continue # Nd [10] MEETEI MAYEK DIGIT ZERO..MEETEI MAYEK DIGIT NINE +AC00..D7A3 ; ID_Continue # Lo [11172] HANGUL SYLLABLE GA..HANGUL SYLLABLE HIH +D7B0..D7C6 ; ID_Continue # Lo [23] HANGUL JUNGSEONG O-YEO..HANGUL JUNGSEONG ARAEA-E +D7CB..D7FB ; ID_Continue # Lo [49] HANGUL JONGSEONG NIEUN-RIEUL..HANGUL JONGSEONG PHIEUPH-THIEUTH +F900..FA2D ; ID_Continue # Lo [302] CJK COMPATIBILITY IDEOGRAPH-F900..CJK COMPATIBILITY IDEOGRAPH-FA2D +FA30..FA6D ; ID_Continue # Lo [62] CJK COMPATIBILITY IDEOGRAPH-FA30..CJK COMPATIBILITY IDEOGRAPH-FA6D +FA70..FAD9 ; ID_Continue # Lo [106] CJK COMPATIBILITY IDEOGRAPH-FA70..CJK COMPATIBILITY IDEOGRAPH-FAD9 +FB00..FB06 ; ID_Continue # L& [7] LATIN SMALL LIGATURE FF..LATIN SMALL LIGATURE ST +FB13..FB17 ; ID_Continue # L& [5] ARMENIAN SMALL LIGATURE MEN NOW..ARMENIAN SMALL LIGATURE MEN XEH +FB1D ; ID_Continue # Lo HEBREW LETTER YOD WITH HIRIQ +FB1E ; ID_Continue # Mn HEBREW POINT JUDEO-SPANISH VARIKA +FB1F..FB28 ; ID_Continue # Lo [10] HEBREW LIGATURE YIDDISH YOD YOD PATAH..HEBREW LETTER WIDE TAV +FB2A..FB36 ; ID_Continue # Lo [13] HEBREW LETTER SHIN WITH SHIN DOT..HEBREW LETTER ZAYIN WITH DAGESH +FB38..FB3C ; ID_Continue # Lo [5] HEBREW LETTER TET WITH DAGESH..HEBREW LETTER LAMED WITH DAGESH +FB3E ; ID_Continue # Lo HEBREW LETTER MEM WITH DAGESH +FB40..FB41 ; ID_Continue # Lo [2] HEBREW LETTER NUN WITH DAGESH..HEBREW LETTER SAMEKH WITH DAGESH +FB43..FB44 ; ID_Continue # Lo [2] HEBREW LETTER FINAL PE WITH DAGESH..HEBREW LETTER PE WITH DAGESH +FB46..FBB1 ; ID_Continue # Lo [108] HEBREW LETTER TSADI WITH DAGESH..ARABIC LETTER YEH BARREE WITH HAMZA ABOVE FINAL FORM +FBD3..FD3D ; ID_Continue # Lo [363] ARABIC LETTER NG ISOLATED FORM..ARABIC LIGATURE ALEF WITH FATHATAN ISOLATED FORM +FD50..FD8F ; ID_Continue # Lo [64] ARABIC LIGATURE TEH WITH JEEM WITH MEEM INITIAL FORM..ARABIC LIGATURE MEEM WITH KHAH WITH MEEM INITIAL FORM +FD92..FDC7 ; ID_Continue # Lo [54] ARABIC LIGATURE MEEM WITH JEEM WITH KHAH INITIAL FORM..ARABIC LIGATURE NOON WITH JEEM WITH YEH FINAL FORM +FDF0..FDFB ; ID_Continue # Lo [12] ARABIC LIGATURE SALLA USED AS KORANIC STOP SIGN ISOLATED FORM..ARABIC LIGATURE JALLAJALALOUHOU +FE00..FE0F ; ID_Continue # Mn [16] VARIATION SELECTOR-1..VARIATION SELECTOR-16 +FE20..FE26 ; ID_Continue # Mn [7] COMBINING LIGATURE LEFT HALF..COMBINING CONJOINING MACRON +FE33..FE34 ; ID_Continue # Pc [2] PRESENTATION FORM FOR VERTICAL LOW LINE..PRESENTATION FORM FOR VERTICAL WAVY LOW LINE +FE4D..FE4F ; ID_Continue # Pc [3] DASHED LOW LINE..WAVY LOW LINE +FE70..FE74 ; ID_Continue # Lo [5] ARABIC FATHATAN ISOLATED FORM..ARABIC KASRATAN ISOLATED FORM +FE76..FEFC ; ID_Continue # Lo [135] ARABIC FATHA ISOLATED FORM..ARABIC LIGATURE LAM WITH ALEF FINAL FORM +FF10..FF19 ; ID_Continue # Nd [10] FULLWIDTH DIGIT ZERO..FULLWIDTH DIGIT NINE +FF21..FF3A ; ID_Continue # L& [26] FULLWIDTH LATIN CAPITAL LETTER A..FULLWIDTH LATIN CAPITAL LETTER Z +FF3F ; ID_Continue # Pc FULLWIDTH LOW LINE +FF41..FF5A ; ID_Continue # L& [26] FULLWIDTH LATIN SMALL LETTER A..FULLWIDTH LATIN SMALL LETTER Z +FF66..FF6F ; ID_Continue # Lo [10] HALFWIDTH KATAKANA LETTER WO..HALFWIDTH KATAKANA LETTER SMALL TU +FF70 ; ID_Continue # Lm HALFWIDTH KATAKANA-HIRAGANA PROLONGED SOUND MARK +FF71..FF9D ; ID_Continue # Lo [45] HALFWIDTH KATAKANA LETTER A..HALFWIDTH KATAKANA LETTER N +FF9E..FF9F ; ID_Continue # Lm [2] HALFWIDTH KATAKANA VOICED SOUND MARK..HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK +FFA0..FFBE ; ID_Continue # Lo [31] HALFWIDTH HANGUL FILLER..HALFWIDTH HANGUL LETTER HIEUH +FFC2..FFC7 ; ID_Continue # Lo [6] HALFWIDTH HANGUL LETTER A..HALFWIDTH HANGUL LETTER E +FFCA..FFCF ; ID_Continue # Lo [6] HALFWIDTH HANGUL LETTER YEO..HALFWIDTH HANGUL LETTER OE +FFD2..FFD7 ; ID_Continue # Lo [6] HALFWIDTH HANGUL LETTER YO..HALFWIDTH HANGUL LETTER YU +FFDA..FFDC ; ID_Continue # Lo [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANGUL LETTER I +10000..1000B ; ID_Continue # Lo [12] LINEAR B SYLLABLE B008 A..LINEAR B SYLLABLE B046 JE +1000D..10026 ; ID_Continue # Lo [26] LINEAR B SYLLABLE B036 JO..LINEAR B SYLLABLE B032 QO +10028..1003A ; ID_Continue # Lo [19] LINEAR B SYLLABLE B060 RA..LINEAR B SYLLABLE B042 WO +1003C..1003D ; ID_Continue # Lo [2] LINEAR B SYLLABLE B017 ZA..LINEAR B SYLLABLE B074 ZE +1003F..1004D ; ID_Continue # Lo [15] LINEAR B SYLLABLE B020 ZO..LINEAR B SYLLABLE B091 TWO +10050..1005D ; ID_Continue # Lo [14] LINEAR B SYMBOL B018..LINEAR B SYMBOL B089 +10080..100FA ; ID_Continue # Lo [123] LINEAR B IDEOGRAM B100 MAN..LINEAR B IDEOGRAM VESSEL B305 +10140..10174 ; ID_Continue # Nl [53] GREEK ACROPHONIC ATTIC ONE QUARTER..GREEK ACROPHONIC STRATIAN FIFTY MNAS +101FD ; ID_Continue # Mn PHAISTOS DISC SIGN COMBINING OBLIQUE STROKE +10280..1029C ; ID_Continue # Lo [29] LYCIAN LETTER A..LYCIAN LETTER X +102A0..102D0 ; ID_Continue # Lo [49] CARIAN LETTER A..CARIAN LETTER UUU3 +10300..1031E ; ID_Continue # Lo [31] OLD ITALIC LETTER A..OLD ITALIC LETTER UU +10330..10340 ; ID_Continue # Lo [17] GOTHIC LETTER AHSA..GOTHIC LETTER PAIRTHRA +10341 ; ID_Continue # Nl GOTHIC LETTER NINETY +10342..10349 ; ID_Continue # Lo [8] GOTHIC LETTER RAIDA..GOTHIC LETTER OTHAL +1034A ; ID_Continue # Nl GOTHIC LETTER NINE HUNDRED +10380..1039D ; ID_Continue # Lo [30] UGARITIC LETTER ALPA..UGARITIC LETTER SSU +103A0..103C3 ; ID_Continue # Lo [36] OLD PERSIAN SIGN A..OLD PERSIAN SIGN HA +103C8..103CF ; ID_Continue # Lo [8] OLD PERSIAN SIGN AURAMAZDAA..OLD PERSIAN SIGN BUUMISH +103D1..103D5 ; ID_Continue # Nl [5] OLD PERSIAN NUMBER ONE..OLD PERSIAN NUMBER HUNDRED +10400..1044F ; ID_Continue # L& [80] DESERET CAPITAL LETTER LONG I..DESERET SMALL LETTER EW +10450..1049D ; ID_Continue # Lo [78] SHAVIAN LETTER PEEP..OSMANYA LETTER OO +104A0..104A9 ; ID_Continue # Nd [10] OSMANYA DIGIT ZERO..OSMANYA DIGIT NINE +10800..10805 ; ID_Continue # Lo [6] CYPRIOT SYLLABLE A..CYPRIOT SYLLABLE JA +10808 ; ID_Continue # Lo CYPRIOT SYLLABLE JO +1080A..10835 ; ID_Continue # Lo [44] CYPRIOT SYLLABLE KA..CYPRIOT SYLLABLE WO +10837..10838 ; ID_Continue # Lo [2] CYPRIOT SYLLABLE XA..CYPRIOT SYLLABLE XE +1083C ; ID_Continue # Lo CYPRIOT SYLLABLE ZA +1083F..10855 ; ID_Continue # Lo [23] CYPRIOT SYLLABLE ZO..IMPERIAL ARAMAIC LETTER TAW +10900..10915 ; ID_Continue # Lo [22] PHOENICIAN LETTER ALF..PHOENICIAN LETTER TAU +10920..10939 ; ID_Continue # Lo [26] LYDIAN LETTER A..LYDIAN LETTER C +10A00 ; ID_Continue # Lo KHAROSHTHI LETTER A +10A01..10A03 ; ID_Continue # Mn [3] KHAROSHTHI VOWEL SIGN I..KHAROSHTHI VOWEL SIGN VOCALIC R +10A05..10A06 ; ID_Continue # Mn [2] KHAROSHTHI VOWEL SIGN E..KHAROSHTHI VOWEL SIGN O +10A0C..10A0F ; ID_Continue # Mn [4] KHAROSHTHI VOWEL LENGTH MARK..KHAROSHTHI SIGN VISARGA +10A10..10A13 ; ID_Continue # Lo [4] KHAROSHTHI LETTER KA..KHAROSHTHI LETTER GHA +10A15..10A17 ; ID_Continue # Lo [3] KHAROSHTHI LETTER CA..KHAROSHTHI LETTER JA +10A19..10A33 ; ID_Continue # Lo [27] KHAROSHTHI LETTER NYA..KHAROSHTHI LETTER TTTHA +10A38..10A3A ; ID_Continue # Mn [3] KHAROSHTHI SIGN BAR ABOVE..KHAROSHTHI SIGN DOT BELOW +10A3F ; ID_Continue # Mn KHAROSHTHI VIRAMA +10A60..10A7C ; ID_Continue # Lo [29] OLD SOUTH ARABIAN LETTER HE..OLD SOUTH ARABIAN LETTER THETH +10B00..10B35 ; ID_Continue # Lo [54] AVESTAN LETTER A..AVESTAN LETTER HE +10B40..10B55 ; ID_Continue # Lo [22] INSCRIPTIONAL PARTHIAN LETTER ALEPH..INSCRIPTIONAL PARTHIAN LETTER TAW +10B60..10B72 ; ID_Continue # Lo [19] INSCRIPTIONAL PAHLAVI LETTER ALEPH..INSCRIPTIONAL PAHLAVI LETTER TAW +10C00..10C48 ; ID_Continue # Lo [73] OLD TURKIC LETTER ORKHON A..OLD TURKIC LETTER ORKHON BASH +11080..11081 ; ID_Continue # Mn [2] KAITHI SIGN CANDRABINDU..KAITHI SIGN ANUSVARA +11082 ; ID_Continue # Mc KAITHI SIGN VISARGA +11083..110AF ; ID_Continue # Lo [45] KAITHI LETTER A..KAITHI LETTER HA +110B0..110B2 ; ID_Continue # Mc [3] KAITHI VOWEL SIGN AA..KAITHI VOWEL SIGN II +110B3..110B6 ; ID_Continue # Mn [4] KAITHI VOWEL SIGN U..KAITHI VOWEL SIGN AI +110B7..110B8 ; ID_Continue # Mc [2] KAITHI VOWEL SIGN O..KAITHI VOWEL SIGN AU +110B9..110BA ; ID_Continue # Mn [2] KAITHI SIGN VIRAMA..KAITHI SIGN NUKTA +12000..1236E ; ID_Continue # Lo [879] CUNEIFORM SIGN A..CUNEIFORM SIGN ZUM +12400..12462 ; ID_Continue # Nl [99] CUNEIFORM NUMERIC SIGN TWO ASH..CUNEIFORM NUMERIC SIGN OLD ASSYRIAN ONE QUARTER +13000..1342E ; ID_Continue # Lo [1071] EGYPTIAN HIEROGLYPH A001..EGYPTIAN HIEROGLYPH AA032 +1D165..1D166 ; ID_Continue # Mc [2] MUSICAL SYMBOL COMBINING STEM..MUSICAL SYMBOL COMBINING SPRECHGESANG STEM +1D167..1D169 ; ID_Continue # Mn [3] MUSICAL SYMBOL COMBINING TREMOLO-1..MUSICAL SYMBOL COMBINING TREMOLO-3 +1D16D..1D172 ; ID_Continue # Mc [6] MUSICAL SYMBOL COMBINING AUGMENTATION DOT..MUSICAL SYMBOL COMBINING FLAG-5 +1D17B..1D182 ; ID_Continue # Mn [8] MUSICAL SYMBOL COMBINING ACCENT..MUSICAL SYMBOL COMBINING LOURE +1D185..1D18B ; ID_Continue # Mn [7] MUSICAL SYMBOL COMBINING DOIT..MUSICAL SYMBOL COMBINING TRIPLE TONGUE +1D1AA..1D1AD ; ID_Continue # Mn [4] MUSICAL SYMBOL COMBINING DOWN BOW..MUSICAL SYMBOL COMBINING SNAP PIZZICATO +1D242..1D244 ; ID_Continue # Mn [3] COMBINING GREEK MUSICAL TRISEME..COMBINING GREEK MUSICAL PENTASEME +1D400..1D454 ; ID_Continue # L& [85] MATHEMATICAL BOLD CAPITAL A..MATHEMATICAL ITALIC SMALL G +1D456..1D49C ; ID_Continue # L& [71] MATHEMATICAL ITALIC SMALL I..MATHEMATICAL SCRIPT CAPITAL A +1D49E..1D49F ; ID_Continue # L& [2] MATHEMATICAL SCRIPT CAPITAL C..MATHEMATICAL SCRIPT CAPITAL D +1D4A2 ; ID_Continue # L& MATHEMATICAL SCRIPT CAPITAL G +1D4A5..1D4A6 ; ID_Continue # L& [2] MATHEMATICAL SCRIPT CAPITAL J..MATHEMATICAL SCRIPT CAPITAL K +1D4A9..1D4AC ; ID_Continue # L& [4] MATHEMATICAL SCRIPT CAPITAL N..MATHEMATICAL SCRIPT CAPITAL Q +1D4AE..1D4B9 ; ID_Continue # L& [12] MATHEMATICAL SCRIPT CAPITAL S..MATHEMATICAL SCRIPT SMALL D +1D4BB ; ID_Continue # L& MATHEMATICAL SCRIPT SMALL F +1D4BD..1D4C3 ; ID_Continue # L& [7] MATHEMATICAL SCRIPT SMALL H..MATHEMATICAL SCRIPT SMALL N +1D4C5..1D505 ; ID_Continue # L& [65] MATHEMATICAL SCRIPT SMALL P..MATHEMATICAL FRAKTUR CAPITAL B +1D507..1D50A ; ID_Continue # L& [4] MATHEMATICAL FRAKTUR CAPITAL D..MATHEMATICAL FRAKTUR CAPITAL G +1D50D..1D514 ; ID_Continue # L& [8] MATHEMATICAL FRAKTUR CAPITAL J..MATHEMATICAL FRAKTUR CAPITAL Q +1D516..1D51C ; ID_Continue # L& [7] MATHEMATICAL FRAKTUR CAPITAL S..MATHEMATICAL FRAKTUR CAPITAL Y +1D51E..1D539 ; ID_Continue # L& [28] MATHEMATICAL FRAKTUR SMALL A..MATHEMATICAL DOUBLE-STRUCK CAPITAL B +1D53B..1D53E ; ID_Continue # L& [4] MATHEMATICAL DOUBLE-STRUCK CAPITAL D..MATHEMATICAL DOUBLE-STRUCK CAPITAL G +1D540..1D544 ; ID_Continue # L& [5] MATHEMATICAL DOUBLE-STRUCK CAPITAL I..MATHEMATICAL DOUBLE-STRUCK CAPITAL M +1D546 ; ID_Continue # L& MATHEMATICAL DOUBLE-STRUCK CAPITAL O +1D54A..1D550 ; ID_Continue # L& [7] MATHEMATICAL DOUBLE-STRUCK CAPITAL S..MATHEMATICAL DOUBLE-STRUCK CAPITAL Y +1D552..1D6A5 ; ID_Continue # L& [340] MATHEMATICAL DOUBLE-STRUCK SMALL A..MATHEMATICAL ITALIC SMALL DOTLESS J +1D6A8..1D6C0 ; ID_Continue # L& [25] MATHEMATICAL BOLD CAPITAL ALPHA..MATHEMATICAL BOLD CAPITAL OMEGA +1D6C2..1D6DA ; ID_Continue # L& [25] MATHEMATICAL BOLD SMALL ALPHA..MATHEMATICAL BOLD SMALL OMEGA +1D6DC..1D6FA ; ID_Continue # L& [31] MATHEMATICAL BOLD EPSILON SYMBOL..MATHEMATICAL ITALIC CAPITAL OMEGA +1D6FC..1D714 ; ID_Continue # L& [25] MATHEMATICAL ITALIC SMALL ALPHA..MATHEMATICAL ITALIC SMALL OMEGA +1D716..1D734 ; ID_Continue # L& [31] MATHEMATICAL ITALIC EPSILON SYMBOL..MATHEMATICAL BOLD ITALIC CAPITAL OMEGA +1D736..1D74E ; ID_Continue # L& [25] MATHEMATICAL BOLD ITALIC SMALL ALPHA..MATHEMATICAL BOLD ITALIC SMALL OMEGA +1D750..1D76E ; ID_Continue # L& [31] MATHEMATICAL BOLD ITALIC EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD CAPITAL OMEGA +1D770..1D788 ; ID_Continue # L& [25] MATHEMATICAL SANS-SERIF BOLD SMALL ALPHA..MATHEMATICAL SANS-SERIF BOLD SMALL OMEGA +1D78A..1D7A8 ; ID_Continue # L& [31] MATHEMATICAL SANS-SERIF BOLD EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL OMEGA +1D7AA..1D7C2 ; ID_Continue # L& [25] MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL ALPHA..MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL OMEGA +1D7C4..1D7CB ; ID_Continue # L& [8] MATHEMATICAL SANS-SERIF BOLD ITALIC EPSILON SYMBOL..MATHEMATICAL BOLD SMALL DIGAMMA +1D7CE..1D7FF ; ID_Continue # Nd [50] MATHEMATICAL BOLD DIGIT ZERO..MATHEMATICAL MONOSPACE DIGIT NINE +20000..2A6D6 ; ID_Continue # Lo [42711] CJK UNIFIED IDEOGRAPH-20000..CJK UNIFIED IDEOGRAPH-2A6D6 +2A700..2B734 ; ID_Continue # Lo [4149] CJK UNIFIED IDEOGRAPH-2A700..CJK UNIFIED IDEOGRAPH-2B734 +2F800..2FA1D ; ID_Continue # Lo [542] CJK COMPATIBILITY IDEOGRAPH-2F800..CJK COMPATIBILITY IDEOGRAPH-2FA1D +E0100..E01EF ; ID_Continue # Mn [240] VARIATION SELECTOR-17..VARIATION SELECTOR-256 + +# Total code points: 101634 + +# ================================================ + +# Derived Property: XID_Start +# ID_Start modified for closure under NFKx +# Modified as described in UAX #15 +# NOTE: Does NOT remove the non-NFKx characters. +# Merely ensures that if isIdentifer(string) then isIdentifier(NFKx(string)) +# NOTE: See UAX #31 for more information + +0041..005A ; XID_Start # L& [26] LATIN CAPITAL LETTER A..LATIN CAPITAL LETTER Z +0061..007A ; XID_Start # L& [26] LATIN SMALL LETTER A..LATIN SMALL LETTER Z +00AA ; XID_Start # L& FEMININE ORDINAL INDICATOR +00B5 ; XID_Start # L& MICRO SIGN +00BA ; XID_Start # L& MASCULINE ORDINAL INDICATOR +00C0..00D6 ; XID_Start # L& [23] LATIN CAPITAL LETTER A WITH GRAVE..LATIN CAPITAL LETTER O WITH DIAERESIS +00D8..00F6 ; XID_Start # L& [31] LATIN CAPITAL LETTER O WITH STROKE..LATIN SMALL LETTER O WITH DIAERESIS +00F8..01BA ; XID_Start # L& [195] LATIN SMALL LETTER O WITH STROKE..LATIN SMALL LETTER EZH WITH TAIL +01BB ; XID_Start # Lo LATIN LETTER TWO WITH STROKE +01BC..01BF ; XID_Start # L& [4] LATIN CAPITAL LETTER TONE FIVE..LATIN LETTER WYNN +01C0..01C3 ; XID_Start # Lo [4] LATIN LETTER DENTAL CLICK..LATIN LETTER RETROFLEX CLICK +01C4..0293 ; XID_Start # L& [208] LATIN CAPITAL LETTER DZ WITH CARON..LATIN SMALL LETTER EZH WITH CURL +0294 ; XID_Start # Lo LATIN LETTER GLOTTAL STOP +0295..02AF ; XID_Start # L& [27] LATIN LETTER PHARYNGEAL VOICED FRICATIVE..LATIN SMALL LETTER TURNED H WITH FISHHOOK AND TAIL +02B0..02C1 ; XID_Start # Lm [18] MODIFIER LETTER SMALL H..MODIFIER LETTER REVERSED GLOTTAL STOP +02C6..02D1 ; XID_Start # Lm [12] MODIFIER LETTER CIRCUMFLEX ACCENT..MODIFIER LETTER HALF TRIANGULAR COLON +02E0..02E4 ; XID_Start # Lm [5] MODIFIER LETTER SMALL GAMMA..MODIFIER LETTER SMALL REVERSED GLOTTAL STOP +02EC ; XID_Start # Lm MODIFIER LETTER VOICING +02EE ; XID_Start # Lm MODIFIER LETTER DOUBLE APOSTROPHE +0370..0373 ; XID_Start # L& [4] GREEK CAPITAL LETTER HETA..GREEK SMALL LETTER ARCHAIC SAMPI +0374 ; XID_Start # Lm GREEK NUMERAL SIGN +0376..0377 ; XID_Start # L& [2] GREEK CAPITAL LETTER PAMPHYLIAN DIGAMMA..GREEK SMALL LETTER PAMPHYLIAN DIGAMMA +037B..037D ; XID_Start # L& [3] GREEK SMALL REVERSED LUNATE SIGMA SYMBOL..GREEK SMALL REVERSED DOTTED LUNATE SIGMA SYMBOL +0386 ; XID_Start # L& GREEK CAPITAL LETTER ALPHA WITH TONOS +0388..038A ; XID_Start # L& [3] GREEK CAPITAL LETTER EPSILON WITH TONOS..GREEK CAPITAL LETTER IOTA WITH TONOS +038C ; XID_Start # L& GREEK CAPITAL LETTER OMICRON WITH TONOS +038E..03A1 ; XID_Start # L& [20] GREEK CAPITAL LETTER UPSILON WITH TONOS..GREEK CAPITAL LETTER RHO +03A3..03F5 ; XID_Start # L& [83] GREEK CAPITAL LETTER SIGMA..GREEK LUNATE EPSILON SYMBOL +03F7..0481 ; XID_Start # L& [139] GREEK CAPITAL LETTER SHO..CYRILLIC SMALL LETTER KOPPA +048A..0525 ; XID_Start # L& [156] CYRILLIC CAPITAL LETTER SHORT I WITH TAIL..CYRILLIC SMALL LETTER PE WITH DESCENDER +0531..0556 ; XID_Start # L& [38] ARMENIAN CAPITAL LETTER AYB..ARMENIAN CAPITAL LETTER FEH +0559 ; XID_Start # Lm ARMENIAN MODIFIER LETTER LEFT HALF RING +0561..0587 ; XID_Start # L& [39] ARMENIAN SMALL LETTER AYB..ARMENIAN SMALL LIGATURE ECH YIWN +05D0..05EA ; XID_Start # Lo [27] HEBREW LETTER ALEF..HEBREW LETTER TAV +05F0..05F2 ; XID_Start # Lo [3] HEBREW LIGATURE YIDDISH DOUBLE VAV..HEBREW LIGATURE YIDDISH DOUBLE YOD +0621..063F ; XID_Start # Lo [31] ARABIC LETTER HAMZA..ARABIC LETTER FARSI YEH WITH THREE DOTS ABOVE +0640 ; XID_Start # Lm ARABIC TATWEEL +0641..064A ; XID_Start # Lo [10] ARABIC LETTER FEH..ARABIC LETTER YEH +066E..066F ; XID_Start # Lo [2] ARABIC LETTER DOTLESS BEH..ARABIC LETTER DOTLESS QAF +0671..06D3 ; XID_Start # Lo [99] ARABIC LETTER ALEF WASLA..ARABIC LETTER YEH BARREE WITH HAMZA ABOVE +06D5 ; XID_Start # Lo ARABIC LETTER AE +06E5..06E6 ; XID_Start # Lm [2] ARABIC SMALL WAW..ARABIC SMALL YEH +06EE..06EF ; XID_Start # Lo [2] ARABIC LETTER DAL WITH INVERTED V..ARABIC LETTER REH WITH INVERTED V +06FA..06FC ; XID_Start # Lo [3] ARABIC LETTER SHEEN WITH DOT BELOW..ARABIC LETTER GHAIN WITH DOT BELOW +06FF ; XID_Start # Lo ARABIC LETTER HEH WITH INVERTED V +0710 ; XID_Start # Lo SYRIAC LETTER ALAPH +0712..072F ; XID_Start # Lo [30] SYRIAC LETTER BETH..SYRIAC LETTER PERSIAN DHALATH +074D..07A5 ; XID_Start # Lo [89] SYRIAC LETTER SOGDIAN ZHAIN..THAANA LETTER WAAVU +07B1 ; XID_Start # Lo THAANA LETTER NAA +07CA..07EA ; XID_Start # Lo [33] NKO LETTER A..NKO LETTER JONA RA +07F4..07F5 ; XID_Start # Lm [2] NKO HIGH TONE APOSTROPHE..NKO LOW TONE APOSTROPHE +07FA ; XID_Start # Lm NKO LAJANYALAN +0800..0815 ; XID_Start # Lo [22] SAMARITAN LETTER ALAF..SAMARITAN LETTER TAAF +081A ; XID_Start # Lm SAMARITAN MODIFIER LETTER EPENTHETIC YUT +0824 ; XID_Start # Lm SAMARITAN MODIFIER LETTER SHORT A +0828 ; XID_Start # Lm SAMARITAN MODIFIER LETTER I +0904..0939 ; XID_Start # Lo [54] DEVANAGARI LETTER SHORT A..DEVANAGARI LETTER HA +093D ; XID_Start # Lo DEVANAGARI SIGN AVAGRAHA +0950 ; XID_Start # Lo DEVANAGARI OM +0958..0961 ; XID_Start # Lo [10] DEVANAGARI LETTER QA..DEVANAGARI LETTER VOCALIC LL +0971 ; XID_Start # Lm DEVANAGARI SIGN HIGH SPACING DOT +0972 ; XID_Start # Lo DEVANAGARI LETTER CANDRA A +0979..097F ; XID_Start # Lo [7] DEVANAGARI LETTER ZHA..DEVANAGARI LETTER BBA +0985..098C ; XID_Start # Lo [8] BENGALI LETTER A..BENGALI LETTER VOCALIC L +098F..0990 ; XID_Start # Lo [2] BENGALI LETTER E..BENGALI LETTER AI +0993..09A8 ; XID_Start # Lo [22] BENGALI LETTER O..BENGALI LETTER NA +09AA..09B0 ; XID_Start # Lo [7] BENGALI LETTER PA..BENGALI LETTER RA +09B2 ; XID_Start # Lo BENGALI LETTER LA +09B6..09B9 ; XID_Start # Lo [4] BENGALI LETTER SHA..BENGALI LETTER HA +09BD ; XID_Start # Lo BENGALI SIGN AVAGRAHA +09CE ; XID_Start # Lo BENGALI LETTER KHANDA TA +09DC..09DD ; XID_Start # Lo [2] BENGALI LETTER RRA..BENGALI LETTER RHA +09DF..09E1 ; XID_Start # Lo [3] BENGALI LETTER YYA..BENGALI LETTER VOCALIC LL +09F0..09F1 ; XID_Start # Lo [2] BENGALI LETTER RA WITH MIDDLE DIAGONAL..BENGALI LETTER RA WITH LOWER DIAGONAL +0A05..0A0A ; XID_Start # Lo [6] GURMUKHI LETTER A..GURMUKHI LETTER UU +0A0F..0A10 ; XID_Start # Lo [2] GURMUKHI LETTER EE..GURMUKHI LETTER AI +0A13..0A28 ; XID_Start # Lo [22] GURMUKHI LETTER OO..GURMUKHI LETTER NA +0A2A..0A30 ; XID_Start # Lo [7] GURMUKHI LETTER PA..GURMUKHI LETTER RA +0A32..0A33 ; XID_Start # Lo [2] GURMUKHI LETTER LA..GURMUKHI LETTER LLA +0A35..0A36 ; XID_Start # Lo [2] GURMUKHI LETTER VA..GURMUKHI LETTER SHA +0A38..0A39 ; XID_Start # Lo [2] GURMUKHI LETTER SA..GURMUKHI LETTER HA +0A59..0A5C ; XID_Start # Lo [4] GURMUKHI LETTER KHHA..GURMUKHI LETTER RRA +0A5E ; XID_Start # Lo GURMUKHI LETTER FA +0A72..0A74 ; XID_Start # Lo [3] GURMUKHI IRI..GURMUKHI EK ONKAR +0A85..0A8D ; XID_Start # Lo [9] GUJARATI LETTER A..GUJARATI VOWEL CANDRA E +0A8F..0A91 ; XID_Start # Lo [3] GUJARATI LETTER E..GUJARATI VOWEL CANDRA O +0A93..0AA8 ; XID_Start # Lo [22] GUJARATI LETTER O..GUJARATI LETTER NA +0AAA..0AB0 ; XID_Start # Lo [7] GUJARATI LETTER PA..GUJARATI LETTER RA +0AB2..0AB3 ; XID_Start # Lo [2] GUJARATI LETTER LA..GUJARATI LETTER LLA +0AB5..0AB9 ; XID_Start # Lo [5] GUJARATI LETTER VA..GUJARATI LETTER HA +0ABD ; XID_Start # Lo GUJARATI SIGN AVAGRAHA +0AD0 ; XID_Start # Lo GUJARATI OM +0AE0..0AE1 ; XID_Start # Lo [2] GUJARATI LETTER VOCALIC RR..GUJARATI LETTER VOCALIC LL +0B05..0B0C ; XID_Start # Lo [8] ORIYA LETTER A..ORIYA LETTER VOCALIC L +0B0F..0B10 ; XID_Start # Lo [2] ORIYA LETTER E..ORIYA LETTER AI +0B13..0B28 ; XID_Start # Lo [22] ORIYA LETTER O..ORIYA LETTER NA +0B2A..0B30 ; XID_Start # Lo [7] ORIYA LETTER PA..ORIYA LETTER RA +0B32..0B33 ; XID_Start # Lo [2] ORIYA LETTER LA..ORIYA LETTER LLA +0B35..0B39 ; XID_Start # Lo [5] ORIYA LETTER VA..ORIYA LETTER HA +0B3D ; XID_Start # Lo ORIYA SIGN AVAGRAHA +0B5C..0B5D ; XID_Start # Lo [2] ORIYA LETTER RRA..ORIYA LETTER RHA +0B5F..0B61 ; XID_Start # Lo [3] ORIYA LETTER YYA..ORIYA LETTER VOCALIC LL +0B71 ; XID_Start # Lo ORIYA LETTER WA +0B83 ; XID_Start # Lo TAMIL SIGN VISARGA +0B85..0B8A ; XID_Start # Lo [6] TAMIL LETTER A..TAMIL LETTER UU +0B8E..0B90 ; XID_Start # Lo [3] TAMIL LETTER E..TAMIL LETTER AI +0B92..0B95 ; XID_Start # Lo [4] TAMIL LETTER O..TAMIL LETTER KA +0B99..0B9A ; XID_Start # Lo [2] TAMIL LETTER NGA..TAMIL LETTER CA +0B9C ; XID_Start # Lo TAMIL LETTER JA +0B9E..0B9F ; XID_Start # Lo [2] TAMIL LETTER NYA..TAMIL LETTER TTA +0BA3..0BA4 ; XID_Start # Lo [2] TAMIL LETTER NNA..TAMIL LETTER TA +0BA8..0BAA ; XID_Start # Lo [3] TAMIL LETTER NA..TAMIL LETTER PA +0BAE..0BB9 ; XID_Start # Lo [12] TAMIL LETTER MA..TAMIL LETTER HA +0BD0 ; XID_Start # Lo TAMIL OM +0C05..0C0C ; XID_Start # Lo [8] TELUGU LETTER A..TELUGU LETTER VOCALIC L +0C0E..0C10 ; XID_Start # Lo [3] TELUGU LETTER E..TELUGU LETTER AI +0C12..0C28 ; XID_Start # Lo [23] TELUGU LETTER O..TELUGU LETTER NA +0C2A..0C33 ; XID_Start # Lo [10] TELUGU LETTER PA..TELUGU LETTER LLA +0C35..0C39 ; XID_Start # Lo [5] TELUGU LETTER VA..TELUGU LETTER HA +0C3D ; XID_Start # Lo TELUGU SIGN AVAGRAHA +0C58..0C59 ; XID_Start # Lo [2] TELUGU LETTER TSA..TELUGU LETTER DZA +0C60..0C61 ; XID_Start # Lo [2] TELUGU LETTER VOCALIC RR..TELUGU LETTER VOCALIC LL +0C85..0C8C ; XID_Start # Lo [8] KANNADA LETTER A..KANNADA LETTER VOCALIC L +0C8E..0C90 ; XID_Start # Lo [3] KANNADA LETTER E..KANNADA LETTER AI +0C92..0CA8 ; XID_Start # Lo [23] KANNADA LETTER O..KANNADA LETTER NA +0CAA..0CB3 ; XID_Start # Lo [10] KANNADA LETTER PA..KANNADA LETTER LLA +0CB5..0CB9 ; XID_Start # Lo [5] KANNADA LETTER VA..KANNADA LETTER HA +0CBD ; XID_Start # Lo KANNADA SIGN AVAGRAHA +0CDE ; XID_Start # Lo KANNADA LETTER FA +0CE0..0CE1 ; XID_Start # Lo [2] KANNADA LETTER VOCALIC RR..KANNADA LETTER VOCALIC LL +0D05..0D0C ; XID_Start # Lo [8] MALAYALAM LETTER A..MALAYALAM LETTER VOCALIC L +0D0E..0D10 ; XID_Start # Lo [3] MALAYALAM LETTER E..MALAYALAM LETTER AI +0D12..0D28 ; XID_Start # Lo [23] MALAYALAM LETTER O..MALAYALAM LETTER NA +0D2A..0D39 ; XID_Start # Lo [16] MALAYALAM LETTER PA..MALAYALAM LETTER HA +0D3D ; XID_Start # Lo MALAYALAM SIGN AVAGRAHA +0D60..0D61 ; XID_Start # Lo [2] MALAYALAM LETTER VOCALIC RR..MALAYALAM LETTER VOCALIC LL +0D7A..0D7F ; XID_Start # Lo [6] MALAYALAM LETTER CHILLU NN..MALAYALAM LETTER CHILLU K +0D85..0D96 ; XID_Start # Lo [18] SINHALA LETTER AYANNA..SINHALA LETTER AUYANNA +0D9A..0DB1 ; XID_Start # Lo [24] SINHALA LETTER ALPAPRAANA KAYANNA..SINHALA LETTER DANTAJA NAYANNA +0DB3..0DBB ; XID_Start # Lo [9] SINHALA LETTER SANYAKA DAYANNA..SINHALA LETTER RAYANNA +0DBD ; XID_Start # Lo SINHALA LETTER DANTAJA LAYANNA +0DC0..0DC6 ; XID_Start # Lo [7] SINHALA LETTER VAYANNA..SINHALA LETTER FAYANNA +0E01..0E30 ; XID_Start # Lo [48] THAI CHARACTER KO KAI..THAI CHARACTER SARA A +0E32 ; XID_Start # Lo THAI CHARACTER SARA AA +0E40..0E45 ; XID_Start # Lo [6] THAI CHARACTER SARA E..THAI CHARACTER LAKKHANGYAO +0E46 ; XID_Start # Lm THAI CHARACTER MAIYAMOK +0E81..0E82 ; XID_Start # Lo [2] LAO LETTER KO..LAO LETTER KHO SUNG +0E84 ; XID_Start # Lo LAO LETTER KHO TAM +0E87..0E88 ; XID_Start # Lo [2] LAO LETTER NGO..LAO LETTER CO +0E8A ; XID_Start # Lo LAO LETTER SO TAM +0E8D ; XID_Start # Lo LAO LETTER NYO +0E94..0E97 ; XID_Start # Lo [4] LAO LETTER DO..LAO LETTER THO TAM +0E99..0E9F ; XID_Start # Lo [7] LAO LETTER NO..LAO LETTER FO SUNG +0EA1..0EA3 ; XID_Start # Lo [3] LAO LETTER MO..LAO LETTER LO LING +0EA5 ; XID_Start # Lo LAO LETTER LO LOOT +0EA7 ; XID_Start # Lo LAO LETTER WO +0EAA..0EAB ; XID_Start # Lo [2] LAO LETTER SO SUNG..LAO LETTER HO SUNG +0EAD..0EB0 ; XID_Start # Lo [4] LAO LETTER O..LAO VOWEL SIGN A +0EB2 ; XID_Start # Lo LAO VOWEL SIGN AA +0EBD ; XID_Start # Lo LAO SEMIVOWEL SIGN NYO +0EC0..0EC4 ; XID_Start # Lo [5] LAO VOWEL SIGN E..LAO VOWEL SIGN AI +0EC6 ; XID_Start # Lm LAO KO LA +0EDC..0EDD ; XID_Start # Lo [2] LAO HO NO..LAO HO MO +0F00 ; XID_Start # Lo TIBETAN SYLLABLE OM +0F40..0F47 ; XID_Start # Lo [8] TIBETAN LETTER KA..TIBETAN LETTER JA +0F49..0F6C ; XID_Start # Lo [36] TIBETAN LETTER NYA..TIBETAN LETTER RRA +0F88..0F8B ; XID_Start # Lo [4] TIBETAN SIGN LCE TSA CAN..TIBETAN SIGN GRU MED RGYINGS +1000..102A ; XID_Start # Lo [43] MYANMAR LETTER KA..MYANMAR LETTER AU +103F ; XID_Start # Lo MYANMAR LETTER GREAT SA +1050..1055 ; XID_Start # Lo [6] MYANMAR LETTER SHA..MYANMAR LETTER VOCALIC LL +105A..105D ; XID_Start # Lo [4] MYANMAR LETTER MON NGA..MYANMAR LETTER MON BBE +1061 ; XID_Start # Lo MYANMAR LETTER SGAW KAREN SHA +1065..1066 ; XID_Start # Lo [2] MYANMAR LETTER WESTERN PWO KAREN THA..MYANMAR LETTER WESTERN PWO KAREN PWA +106E..1070 ; XID_Start # Lo [3] MYANMAR LETTER EASTERN PWO KAREN NNA..MYANMAR LETTER EASTERN PWO KAREN GHWA +1075..1081 ; XID_Start # Lo [13] MYANMAR LETTER SHAN KA..MYANMAR LETTER SHAN HA +108E ; XID_Start # Lo MYANMAR LETTER RUMAI PALAUNG FA +10A0..10C5 ; XID_Start # L& [38] GEORGIAN CAPITAL LETTER AN..GEORGIAN CAPITAL LETTER HOE +10D0..10FA ; XID_Start # Lo [43] GEORGIAN LETTER AN..GEORGIAN LETTER AIN +10FC ; XID_Start # Lm MODIFIER LETTER GEORGIAN NAR +1100..1248 ; XID_Start # Lo [329] HANGUL CHOSEONG KIYEOK..ETHIOPIC SYLLABLE QWA +124A..124D ; XID_Start # Lo [4] ETHIOPIC SYLLABLE QWI..ETHIOPIC SYLLABLE QWE +1250..1256 ; XID_Start # Lo [7] ETHIOPIC SYLLABLE QHA..ETHIOPIC SYLLABLE QHO +1258 ; XID_Start # Lo ETHIOPIC SYLLABLE QHWA +125A..125D ; XID_Start # Lo [4] ETHIOPIC SYLLABLE QHWI..ETHIOPIC SYLLABLE QHWE +1260..1288 ; XID_Start # Lo [41] ETHIOPIC SYLLABLE BA..ETHIOPIC SYLLABLE XWA +128A..128D ; XID_Start # Lo [4] ETHIOPIC SYLLABLE XWI..ETHIOPIC SYLLABLE XWE +1290..12B0 ; XID_Start # Lo [33] ETHIOPIC SYLLABLE NA..ETHIOPIC SYLLABLE KWA +12B2..12B5 ; XID_Start # Lo [4] ETHIOPIC SYLLABLE KWI..ETHIOPIC SYLLABLE KWE +12B8..12BE ; XID_Start # Lo [7] ETHIOPIC SYLLABLE KXA..ETHIOPIC SYLLABLE KXO +12C0 ; XID_Start # Lo ETHIOPIC SYLLABLE KXWA +12C2..12C5 ; XID_Start # Lo [4] ETHIOPIC SYLLABLE KXWI..ETHIOPIC SYLLABLE KXWE +12C8..12D6 ; XID_Start # Lo [15] ETHIOPIC SYLLABLE WA..ETHIOPIC SYLLABLE PHARYNGEAL O +12D8..1310 ; XID_Start # Lo [57] ETHIOPIC SYLLABLE ZA..ETHIOPIC SYLLABLE GWA +1312..1315 ; XID_Start # Lo [4] ETHIOPIC SYLLABLE GWI..ETHIOPIC SYLLABLE GWE +1318..135A ; XID_Start # Lo [67] ETHIOPIC SYLLABLE GGA..ETHIOPIC SYLLABLE FYA +1380..138F ; XID_Start # Lo [16] ETHIOPIC SYLLABLE SEBATBEIT MWA..ETHIOPIC SYLLABLE PWE +13A0..13F4 ; XID_Start # Lo [85] CHEROKEE LETTER A..CHEROKEE LETTER YV +1401..166C ; XID_Start # Lo [620] CANADIAN SYLLABICS E..CANADIAN SYLLABICS CARRIER TTSA +166F..167F ; XID_Start # Lo [17] CANADIAN SYLLABICS QAI..CANADIAN SYLLABICS BLACKFOOT W +1681..169A ; XID_Start # Lo [26] OGHAM LETTER BEITH..OGHAM LETTER PEITH +16A0..16EA ; XID_Start # Lo [75] RUNIC LETTER FEHU FEOH FE F..RUNIC LETTER X +16EE..16F0 ; XID_Start # Nl [3] RUNIC ARLAUG SYMBOL..RUNIC BELGTHOR SYMBOL +1700..170C ; XID_Start # Lo [13] TAGALOG LETTER A..TAGALOG LETTER YA +170E..1711 ; XID_Start # Lo [4] TAGALOG LETTER LA..TAGALOG LETTER HA +1720..1731 ; XID_Start # Lo [18] HANUNOO LETTER A..HANUNOO LETTER HA +1740..1751 ; XID_Start # Lo [18] BUHID LETTER A..BUHID LETTER HA +1760..176C ; XID_Start # Lo [13] TAGBANWA LETTER A..TAGBANWA LETTER YA +176E..1770 ; XID_Start # Lo [3] TAGBANWA LETTER LA..TAGBANWA LETTER SA +1780..17B3 ; XID_Start # Lo [52] KHMER LETTER KA..KHMER INDEPENDENT VOWEL QAU +17D7 ; XID_Start # Lm KHMER SIGN LEK TOO +17DC ; XID_Start # Lo KHMER SIGN AVAKRAHASANYA +1820..1842 ; XID_Start # Lo [35] MONGOLIAN LETTER A..MONGOLIAN LETTER CHI +1843 ; XID_Start # Lm MONGOLIAN LETTER TODO LONG VOWEL SIGN +1844..1877 ; XID_Start # Lo [52] MONGOLIAN LETTER TODO E..MONGOLIAN LETTER MANCHU ZHA +1880..18A8 ; XID_Start # Lo [41] MONGOLIAN LETTER ALI GALI ANUSVARA ONE..MONGOLIAN LETTER MANCHU ALI GALI BHA +18AA ; XID_Start # Lo MONGOLIAN LETTER MANCHU ALI GALI LHA +18B0..18F5 ; XID_Start # Lo [70] CANADIAN SYLLABICS OY..CANADIAN SYLLABICS CARRIER DENTAL S +1900..191C ; XID_Start # Lo [29] LIMBU VOWEL-CARRIER LETTER..LIMBU LETTER HA +1950..196D ; XID_Start # Lo [30] TAI LE LETTER KA..TAI LE LETTER AI +1970..1974 ; XID_Start # Lo [5] TAI LE LETTER TONE-2..TAI LE LETTER TONE-6 +1980..19AB ; XID_Start # Lo [44] NEW TAI LUE LETTER HIGH QA..NEW TAI LUE LETTER LOW SUA +19C1..19C7 ; XID_Start # Lo [7] NEW TAI LUE LETTER FINAL V..NEW TAI LUE LETTER FINAL B +1A00..1A16 ; XID_Start # Lo [23] BUGINESE LETTER KA..BUGINESE LETTER HA +1A20..1A54 ; XID_Start # Lo [53] TAI THAM LETTER HIGH KA..TAI THAM LETTER GREAT SA +1AA7 ; XID_Start # Lm TAI THAM SIGN MAI YAMOK +1B05..1B33 ; XID_Start # Lo [47] BALINESE LETTER AKARA..BALINESE LETTER HA +1B45..1B4B ; XID_Start # Lo [7] BALINESE LETTER KAF SASAK..BALINESE LETTER ASYURA SASAK +1B83..1BA0 ; XID_Start # Lo [30] SUNDANESE LETTER A..SUNDANESE LETTER HA +1BAE..1BAF ; XID_Start # Lo [2] SUNDANESE LETTER KHA..SUNDANESE LETTER SYA +1C00..1C23 ; XID_Start # Lo [36] LEPCHA LETTER KA..LEPCHA LETTER A +1C4D..1C4F ; XID_Start # Lo [3] LEPCHA LETTER TTA..LEPCHA LETTER DDA +1C5A..1C77 ; XID_Start # Lo [30] OL CHIKI LETTER LA..OL CHIKI LETTER OH +1C78..1C7D ; XID_Start # Lm [6] OL CHIKI MU TTUDDAG..OL CHIKI AHAD +1CE9..1CEC ; XID_Start # Lo [4] VEDIC SIGN ANUSVARA ANTARGOMUKHA..VEDIC SIGN ANUSVARA VAMAGOMUKHA WITH TAIL +1CEE..1CF1 ; XID_Start # Lo [4] VEDIC SIGN HEXIFORM LONG ANUSVARA..VEDIC SIGN ANUSVARA UBHAYATO MUKHA +1D00..1D2B ; XID_Start # L& [44] LATIN LETTER SMALL CAPITAL A..CYRILLIC LETTER SMALL CAPITAL EL +1D2C..1D61 ; XID_Start # Lm [54] MODIFIER LETTER CAPITAL A..MODIFIER LETTER SMALL CHI +1D62..1D77 ; XID_Start # L& [22] LATIN SUBSCRIPT SMALL LETTER I..LATIN SMALL LETTER TURNED G +1D78 ; XID_Start # Lm MODIFIER LETTER CYRILLIC EN +1D79..1D9A ; XID_Start # L& [34] LATIN SMALL LETTER INSULAR G..LATIN SMALL LETTER EZH WITH RETROFLEX HOOK +1D9B..1DBF ; XID_Start # Lm [37] MODIFIER LETTER SMALL TURNED ALPHA..MODIFIER LETTER SMALL THETA +1E00..1F15 ; XID_Start # L& [278] LATIN CAPITAL LETTER A WITH RING BELOW..GREEK SMALL LETTER EPSILON WITH DASIA AND OXIA +1F18..1F1D ; XID_Start # L& [6] GREEK CAPITAL LETTER EPSILON WITH PSILI..GREEK CAPITAL LETTER EPSILON WITH DASIA AND OXIA +1F20..1F45 ; XID_Start # L& [38] GREEK SMALL LETTER ETA WITH PSILI..GREEK SMALL LETTER OMICRON WITH DASIA AND OXIA +1F48..1F4D ; XID_Start # L& [6] GREEK CAPITAL LETTER OMICRON WITH PSILI..GREEK CAPITAL LETTER OMICRON WITH DASIA AND OXIA +1F50..1F57 ; XID_Start # L& [8] GREEK SMALL LETTER UPSILON WITH PSILI..GREEK SMALL LETTER UPSILON WITH DASIA AND PERISPOMENI +1F59 ; XID_Start # L& GREEK CAPITAL LETTER UPSILON WITH DASIA +1F5B ; XID_Start # L& GREEK CAPITAL LETTER UPSILON WITH DASIA AND VARIA +1F5D ; XID_Start # L& GREEK CAPITAL LETTER UPSILON WITH DASIA AND OXIA +1F5F..1F7D ; XID_Start # L& [31] GREEK CAPITAL LETTER UPSILON WITH DASIA AND PERISPOMENI..GREEK SMALL LETTER OMEGA WITH OXIA +1F80..1FB4 ; XID_Start # L& [53] GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI..GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI +1FB6..1FBC ; XID_Start # L& [7] GREEK SMALL LETTER ALPHA WITH PERISPOMENI..GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI +1FBE ; XID_Start # L& GREEK PROSGEGRAMMENI +1FC2..1FC4 ; XID_Start # L& [3] GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI..GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI +1FC6..1FCC ; XID_Start # L& [7] GREEK SMALL LETTER ETA WITH PERISPOMENI..GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI +1FD0..1FD3 ; XID_Start # L& [4] GREEK SMALL LETTER IOTA WITH VRACHY..GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA +1FD6..1FDB ; XID_Start # L& [6] GREEK SMALL LETTER IOTA WITH PERISPOMENI..GREEK CAPITAL LETTER IOTA WITH OXIA +1FE0..1FEC ; XID_Start # L& [13] GREEK SMALL LETTER UPSILON WITH VRACHY..GREEK CAPITAL LETTER RHO WITH DASIA +1FF2..1FF4 ; XID_Start # L& [3] GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI..GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI +1FF6..1FFC ; XID_Start # L& [7] GREEK SMALL LETTER OMEGA WITH PERISPOMENI..GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI +2071 ; XID_Start # Lm SUPERSCRIPT LATIN SMALL LETTER I +207F ; XID_Start # Lm SUPERSCRIPT LATIN SMALL LETTER N +2090..2094 ; XID_Start # Lm [5] LATIN SUBSCRIPT SMALL LETTER A..LATIN SUBSCRIPT SMALL LETTER SCHWA +2102 ; XID_Start # L& DOUBLE-STRUCK CAPITAL C +2107 ; XID_Start # L& EULER CONSTANT +210A..2113 ; XID_Start # L& [10] SCRIPT SMALL G..SCRIPT SMALL L +2115 ; XID_Start # L& DOUBLE-STRUCK CAPITAL N +2118 ; XID_Start # So SCRIPT CAPITAL P +2119..211D ; XID_Start # L& [5] DOUBLE-STRUCK CAPITAL P..DOUBLE-STRUCK CAPITAL R +2124 ; XID_Start # L& DOUBLE-STRUCK CAPITAL Z +2126 ; XID_Start # L& OHM SIGN +2128 ; XID_Start # L& BLACK-LETTER CAPITAL Z +212A..212D ; XID_Start # L& [4] KELVIN SIGN..BLACK-LETTER CAPITAL C +212E ; XID_Start # So ESTIMATED SYMBOL +212F..2134 ; XID_Start # L& [6] SCRIPT SMALL E..SCRIPT SMALL O +2135..2138 ; XID_Start # Lo [4] ALEF SYMBOL..DALET SYMBOL +2139 ; XID_Start # L& INFORMATION SOURCE +213C..213F ; XID_Start # L& [4] DOUBLE-STRUCK SMALL PI..DOUBLE-STRUCK CAPITAL PI +2145..2149 ; XID_Start # L& [5] DOUBLE-STRUCK ITALIC CAPITAL D..DOUBLE-STRUCK ITALIC SMALL J +214E ; XID_Start # L& TURNED SMALL F +2160..2182 ; XID_Start # Nl [35] ROMAN NUMERAL ONE..ROMAN NUMERAL TEN THOUSAND +2183..2184 ; XID_Start # L& [2] ROMAN NUMERAL REVERSED ONE HUNDRED..LATIN SMALL LETTER REVERSED C +2185..2188 ; XID_Start # Nl [4] ROMAN NUMERAL SIX LATE FORM..ROMAN NUMERAL ONE HUNDRED THOUSAND +2C00..2C2E ; XID_Start # L& [47] GLAGOLITIC CAPITAL LETTER AZU..GLAGOLITIC CAPITAL LETTER LATINATE MYSLITE +2C30..2C5E ; XID_Start # L& [47] GLAGOLITIC SMALL LETTER AZU..GLAGOLITIC SMALL LETTER LATINATE MYSLITE +2C60..2C7C ; XID_Start # L& [29] LATIN CAPITAL LETTER L WITH DOUBLE BAR..LATIN SUBSCRIPT SMALL LETTER J +2C7D ; XID_Start # Lm MODIFIER LETTER CAPITAL V +2C7E..2CE4 ; XID_Start # L& [103] LATIN CAPITAL LETTER S WITH SWASH TAIL..COPTIC SYMBOL KAI +2CEB..2CEE ; XID_Start # L& [4] COPTIC CAPITAL LETTER CRYPTOGRAMMIC SHEI..COPTIC SMALL LETTER CRYPTOGRAMMIC GANGIA +2D00..2D25 ; XID_Start # L& [38] GEORGIAN SMALL LETTER AN..GEORGIAN SMALL LETTER HOE +2D30..2D65 ; XID_Start # Lo [54] TIFINAGH LETTER YA..TIFINAGH LETTER YAZZ +2D6F ; XID_Start # Lm TIFINAGH MODIFIER LETTER LABIALIZATION MARK +2D80..2D96 ; XID_Start # Lo [23] ETHIOPIC SYLLABLE LOA..ETHIOPIC SYLLABLE GGWE +2DA0..2DA6 ; XID_Start # Lo [7] ETHIOPIC SYLLABLE SSA..ETHIOPIC SYLLABLE SSO +2DA8..2DAE ; XID_Start # Lo [7] ETHIOPIC SYLLABLE CCA..ETHIOPIC SYLLABLE CCO +2DB0..2DB6 ; XID_Start # Lo [7] ETHIOPIC SYLLABLE ZZA..ETHIOPIC SYLLABLE ZZO +2DB8..2DBE ; XID_Start # Lo [7] ETHIOPIC SYLLABLE CCHA..ETHIOPIC SYLLABLE CCHO +2DC0..2DC6 ; XID_Start # Lo [7] ETHIOPIC SYLLABLE QYA..ETHIOPIC SYLLABLE QYO +2DC8..2DCE ; XID_Start # Lo [7] ETHIOPIC SYLLABLE KYA..ETHIOPIC SYLLABLE KYO +2DD0..2DD6 ; XID_Start # Lo [7] ETHIOPIC SYLLABLE XYA..ETHIOPIC SYLLABLE XYO +2DD8..2DDE ; XID_Start # Lo [7] ETHIOPIC SYLLABLE GYA..ETHIOPIC SYLLABLE GYO +3005 ; XID_Start # Lm IDEOGRAPHIC ITERATION MARK +3006 ; XID_Start # Lo IDEOGRAPHIC CLOSING MARK +3007 ; XID_Start # Nl IDEOGRAPHIC NUMBER ZERO +3021..3029 ; XID_Start # Nl [9] HANGZHOU NUMERAL ONE..HANGZHOU NUMERAL NINE +3031..3035 ; XID_Start # Lm [5] VERTICAL KANA REPEAT MARK..VERTICAL KANA REPEAT MARK LOWER HALF +3038..303A ; XID_Start # Nl [3] HANGZHOU NUMERAL TEN..HANGZHOU NUMERAL THIRTY +303B ; XID_Start # Lm VERTICAL IDEOGRAPHIC ITERATION MARK +303C ; XID_Start # Lo MASU MARK +3041..3096 ; XID_Start # Lo [86] HIRAGANA LETTER SMALL A..HIRAGANA LETTER SMALL KE +309D..309E ; XID_Start # Lm [2] HIRAGANA ITERATION MARK..HIRAGANA VOICED ITERATION MARK +309F ; XID_Start # Lo HIRAGANA DIGRAPH YORI +30A1..30FA ; XID_Start # Lo [90] KATAKANA LETTER SMALL A..KATAKANA LETTER VO +30FC..30FE ; XID_Start # Lm [3] KATAKANA-HIRAGANA PROLONGED SOUND MARK..KATAKANA VOICED ITERATION MARK +30FF ; XID_Start # Lo KATAKANA DIGRAPH KOTO +3105..312D ; XID_Start # Lo [41] BOPOMOFO LETTER B..BOPOMOFO LETTER IH +3131..318E ; XID_Start # Lo [94] HANGUL LETTER KIYEOK..HANGUL LETTER ARAEAE +31A0..31B7 ; XID_Start # Lo [24] BOPOMOFO LETTER BU..BOPOMOFO FINAL LETTER H +31F0..31FF ; XID_Start # Lo [16] KATAKANA LETTER SMALL KU..KATAKANA LETTER SMALL RO +3400..4DB5 ; XID_Start # Lo [6582] CJK UNIFIED IDEOGRAPH-3400..CJK UNIFIED IDEOGRAPH-4DB5 +4E00..9FCB ; XID_Start # Lo [20940] CJK UNIFIED IDEOGRAPH-4E00..CJK UNIFIED IDEOGRAPH-9FCB +A000..A014 ; XID_Start # Lo [21] YI SYLLABLE IT..YI SYLLABLE E +A015 ; XID_Start # Lm YI SYLLABLE WU +A016..A48C ; XID_Start # Lo [1143] YI SYLLABLE BIT..YI SYLLABLE YYR +A4D0..A4F7 ; XID_Start # Lo [40] LISU LETTER BA..LISU LETTER OE +A4F8..A4FD ; XID_Start # Lm [6] LISU LETTER TONE MYA TI..LISU LETTER TONE MYA JEU +A500..A60B ; XID_Start # Lo [268] VAI SYLLABLE EE..VAI SYLLABLE NG +A60C ; XID_Start # Lm VAI SYLLABLE LENGTHENER +A610..A61F ; XID_Start # Lo [16] VAI SYLLABLE NDOLE FA..VAI SYMBOL JONG +A62A..A62B ; XID_Start # Lo [2] VAI SYLLABLE NDOLE MA..VAI SYLLABLE NDOLE DO +A640..A65F ; XID_Start # L& [32] CYRILLIC CAPITAL LETTER ZEMLYA..CYRILLIC SMALL LETTER YN +A662..A66D ; XID_Start # L& [12] CYRILLIC CAPITAL LETTER SOFT DE..CYRILLIC SMALL LETTER DOUBLE MONOCULAR O +A66E ; XID_Start # Lo CYRILLIC LETTER MULTIOCULAR O +A67F ; XID_Start # Lm CYRILLIC PAYEROK +A680..A697 ; XID_Start # L& [24] CYRILLIC CAPITAL LETTER DWE..CYRILLIC SMALL LETTER SHWE +A6A0..A6E5 ; XID_Start # Lo [70] BAMUM LETTER A..BAMUM LETTER KI +A6E6..A6EF ; XID_Start # Nl [10] BAMUM LETTER MO..BAMUM LETTER KOGHOM +A717..A71F ; XID_Start # Lm [9] MODIFIER LETTER DOT VERTICAL BAR..MODIFIER LETTER LOW INVERTED EXCLAMATION MARK +A722..A76F ; XID_Start # L& [78] LATIN CAPITAL LETTER EGYPTOLOGICAL ALEF..LATIN SMALL LETTER CON +A770 ; XID_Start # Lm MODIFIER LETTER US +A771..A787 ; XID_Start # L& [23] LATIN SMALL LETTER DUM..LATIN SMALL LETTER INSULAR T +A788 ; XID_Start # Lm MODIFIER LETTER LOW CIRCUMFLEX ACCENT +A78B..A78C ; XID_Start # L& [2] LATIN CAPITAL LETTER SALTILLO..LATIN SMALL LETTER SALTILLO +A7FB..A801 ; XID_Start # Lo [7] LATIN EPIGRAPHIC LETTER REVERSED F..SYLOTI NAGRI LETTER I +A803..A805 ; XID_Start # Lo [3] SYLOTI NAGRI LETTER U..SYLOTI NAGRI LETTER O +A807..A80A ; XID_Start # Lo [4] SYLOTI NAGRI LETTER KO..SYLOTI NAGRI LETTER GHO +A80C..A822 ; XID_Start # Lo [23] SYLOTI NAGRI LETTER CO..SYLOTI NAGRI LETTER HO +A840..A873 ; XID_Start # Lo [52] PHAGS-PA LETTER KA..PHAGS-PA LETTER CANDRABINDU +A882..A8B3 ; XID_Start # Lo [50] SAURASHTRA LETTER A..SAURASHTRA LETTER LLA +A8F2..A8F7 ; XID_Start # Lo [6] DEVANAGARI SIGN SPACING CANDRABINDU..DEVANAGARI SIGN CANDRABINDU AVAGRAHA +A8FB ; XID_Start # Lo DEVANAGARI HEADSTROKE +A90A..A925 ; XID_Start # Lo [28] KAYAH LI LETTER KA..KAYAH LI LETTER OO +A930..A946 ; XID_Start # Lo [23] REJANG LETTER KA..REJANG LETTER A +A960..A97C ; XID_Start # Lo [29] HANGUL CHOSEONG TIKEUT-MIEUM..HANGUL CHOSEONG SSANGYEORINHIEUH +A984..A9B2 ; XID_Start # Lo [47] JAVANESE LETTER A..JAVANESE LETTER HA +A9CF ; XID_Start # Lm JAVANESE PANGRANGKEP +AA00..AA28 ; XID_Start # Lo [41] CHAM LETTER A..CHAM LETTER HA +AA40..AA42 ; XID_Start # Lo [3] CHAM LETTER FINAL K..CHAM LETTER FINAL NG +AA44..AA4B ; XID_Start # Lo [8] CHAM LETTER FINAL CH..CHAM LETTER FINAL SS +AA60..AA6F ; XID_Start # Lo [16] MYANMAR LETTER KHAMTI GA..MYANMAR LETTER KHAMTI FA +AA70 ; XID_Start # Lm MYANMAR MODIFIER LETTER KHAMTI REDUPLICATION +AA71..AA76 ; XID_Start # Lo [6] MYANMAR LETTER KHAMTI XA..MYANMAR LOGOGRAM KHAMTI HM +AA7A ; XID_Start # Lo MYANMAR LETTER AITON RA +AA80..AAAF ; XID_Start # Lo [48] TAI VIET LETTER LOW KO..TAI VIET LETTER HIGH O +AAB1 ; XID_Start # Lo TAI VIET VOWEL AA +AAB5..AAB6 ; XID_Start # Lo [2] TAI VIET VOWEL E..TAI VIET VOWEL O +AAB9..AABD ; XID_Start # Lo [5] TAI VIET VOWEL UEA..TAI VIET VOWEL AN +AAC0 ; XID_Start # Lo TAI VIET TONE MAI NUENG +AAC2 ; XID_Start # Lo TAI VIET TONE MAI SONG +AADB..AADC ; XID_Start # Lo [2] TAI VIET SYMBOL KON..TAI VIET SYMBOL NUENG +AADD ; XID_Start # Lm TAI VIET SYMBOL SAM +ABC0..ABE2 ; XID_Start # Lo [35] MEETEI MAYEK LETTER KOK..MEETEI MAYEK LETTER I LONSUM +AC00..D7A3 ; XID_Start # Lo [11172] HANGUL SYLLABLE GA..HANGUL SYLLABLE HIH +D7B0..D7C6 ; XID_Start # Lo [23] HANGUL JUNGSEONG O-YEO..HANGUL JUNGSEONG ARAEA-E +D7CB..D7FB ; XID_Start # Lo [49] HANGUL JONGSEONG NIEUN-RIEUL..HANGUL JONGSEONG PHIEUPH-THIEUTH +F900..FA2D ; XID_Start # Lo [302] CJK COMPATIBILITY IDEOGRAPH-F900..CJK COMPATIBILITY IDEOGRAPH-FA2D +FA30..FA6D ; XID_Start # Lo [62] CJK COMPATIBILITY IDEOGRAPH-FA30..CJK COMPATIBILITY IDEOGRAPH-FA6D +FA70..FAD9 ; XID_Start # Lo [106] CJK COMPATIBILITY IDEOGRAPH-FA70..CJK COMPATIBILITY IDEOGRAPH-FAD9 +FB00..FB06 ; XID_Start # L& [7] LATIN SMALL LIGATURE FF..LATIN SMALL LIGATURE ST +FB13..FB17 ; XID_Start # L& [5] ARMENIAN SMALL LIGATURE MEN NOW..ARMENIAN SMALL LIGATURE MEN XEH +FB1D ; XID_Start # Lo HEBREW LETTER YOD WITH HIRIQ +FB1F..FB28 ; XID_Start # Lo [10] HEBREW LIGATURE YIDDISH YOD YOD PATAH..HEBREW LETTER WIDE TAV +FB2A..FB36 ; XID_Start # Lo [13] HEBREW LETTER SHIN WITH SHIN DOT..HEBREW LETTER ZAYIN WITH DAGESH +FB38..FB3C ; XID_Start # Lo [5] HEBREW LETTER TET WITH DAGESH..HEBREW LETTER LAMED WITH DAGESH +FB3E ; XID_Start # Lo HEBREW LETTER MEM WITH DAGESH +FB40..FB41 ; XID_Start # Lo [2] HEBREW LETTER NUN WITH DAGESH..HEBREW LETTER SAMEKH WITH DAGESH +FB43..FB44 ; XID_Start # Lo [2] HEBREW LETTER FINAL PE WITH DAGESH..HEBREW LETTER PE WITH DAGESH +FB46..FBB1 ; XID_Start # Lo [108] HEBREW LETTER TSADI WITH DAGESH..ARABIC LETTER YEH BARREE WITH HAMZA ABOVE FINAL FORM +FBD3..FC5D ; XID_Start # Lo [139] ARABIC LETTER NG ISOLATED FORM..ARABIC LIGATURE ALEF MAKSURA WITH SUPERSCRIPT ALEF ISOLATED FORM +FC64..FD3D ; XID_Start # Lo [218] ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH REH FINAL FORM..ARABIC LIGATURE ALEF WITH FATHATAN ISOLATED FORM +FD50..FD8F ; XID_Start # Lo [64] ARABIC LIGATURE TEH WITH JEEM WITH MEEM INITIAL FORM..ARABIC LIGATURE MEEM WITH KHAH WITH MEEM INITIAL FORM +FD92..FDC7 ; XID_Start # Lo [54] ARABIC LIGATURE MEEM WITH JEEM WITH KHAH INITIAL FORM..ARABIC LIGATURE NOON WITH JEEM WITH YEH FINAL FORM +FDF0..FDF9 ; XID_Start # Lo [10] ARABIC LIGATURE SALLA USED AS KORANIC STOP SIGN ISOLATED FORM..ARABIC LIGATURE SALLA ISOLATED FORM +FE71 ; XID_Start # Lo ARABIC TATWEEL WITH FATHATAN ABOVE +FE73 ; XID_Start # Lo ARABIC TAIL FRAGMENT +FE77 ; XID_Start # Lo ARABIC FATHA MEDIAL FORM +FE79 ; XID_Start # Lo ARABIC DAMMA MEDIAL FORM +FE7B ; XID_Start # Lo ARABIC KASRA MEDIAL FORM +FE7D ; XID_Start # Lo ARABIC SHADDA MEDIAL FORM +FE7F..FEFC ; XID_Start # Lo [126] ARABIC SUKUN MEDIAL FORM..ARABIC LIGATURE LAM WITH ALEF FINAL FORM +FF21..FF3A ; XID_Start # L& [26] FULLWIDTH LATIN CAPITAL LETTER A..FULLWIDTH LATIN CAPITAL LETTER Z +FF41..FF5A ; XID_Start # L& [26] FULLWIDTH LATIN SMALL LETTER A..FULLWIDTH LATIN SMALL LETTER Z +FF66..FF6F ; XID_Start # Lo [10] HALFWIDTH KATAKANA LETTER WO..HALFWIDTH KATAKANA LETTER SMALL TU +FF70 ; XID_Start # Lm HALFWIDTH KATAKANA-HIRAGANA PROLONGED SOUND MARK +FF71..FF9D ; XID_Start # Lo [45] HALFWIDTH KATAKANA LETTER A..HALFWIDTH KATAKANA LETTER N +FFA0..FFBE ; XID_Start # Lo [31] HALFWIDTH HANGUL FILLER..HALFWIDTH HANGUL LETTER HIEUH +FFC2..FFC7 ; XID_Start # Lo [6] HALFWIDTH HANGUL LETTER A..HALFWIDTH HANGUL LETTER E +FFCA..FFCF ; XID_Start # Lo [6] HALFWIDTH HANGUL LETTER YEO..HALFWIDTH HANGUL LETTER OE +FFD2..FFD7 ; XID_Start # Lo [6] HALFWIDTH HANGUL LETTER YO..HALFWIDTH HANGUL LETTER YU +FFDA..FFDC ; XID_Start # Lo [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANGUL LETTER I +10000..1000B ; XID_Start # Lo [12] LINEAR B SYLLABLE B008 A..LINEAR B SYLLABLE B046 JE +1000D..10026 ; XID_Start # Lo [26] LINEAR B SYLLABLE B036 JO..LINEAR B SYLLABLE B032 QO +10028..1003A ; XID_Start # Lo [19] LINEAR B SYLLABLE B060 RA..LINEAR B SYLLABLE B042 WO +1003C..1003D ; XID_Start # Lo [2] LINEAR B SYLLABLE B017 ZA..LINEAR B SYLLABLE B074 ZE +1003F..1004D ; XID_Start # Lo [15] LINEAR B SYLLABLE B020 ZO..LINEAR B SYLLABLE B091 TWO +10050..1005D ; XID_Start # Lo [14] LINEAR B SYMBOL B018..LINEAR B SYMBOL B089 +10080..100FA ; XID_Start # Lo [123] LINEAR B IDEOGRAM B100 MAN..LINEAR B IDEOGRAM VESSEL B305 +10140..10174 ; XID_Start # Nl [53] GREEK ACROPHONIC ATTIC ONE QUARTER..GREEK ACROPHONIC STRATIAN FIFTY MNAS +10280..1029C ; XID_Start # Lo [29] LYCIAN LETTER A..LYCIAN LETTER X +102A0..102D0 ; XID_Start # Lo [49] CARIAN LETTER A..CARIAN LETTER UUU3 +10300..1031E ; XID_Start # Lo [31] OLD ITALIC LETTER A..OLD ITALIC LETTER UU +10330..10340 ; XID_Start # Lo [17] GOTHIC LETTER AHSA..GOTHIC LETTER PAIRTHRA +10341 ; XID_Start # Nl GOTHIC LETTER NINETY +10342..10349 ; XID_Start # Lo [8] GOTHIC LETTER RAIDA..GOTHIC LETTER OTHAL +1034A ; XID_Start # Nl GOTHIC LETTER NINE HUNDRED +10380..1039D ; XID_Start # Lo [30] UGARITIC LETTER ALPA..UGARITIC LETTER SSU +103A0..103C3 ; XID_Start # Lo [36] OLD PERSIAN SIGN A..OLD PERSIAN SIGN HA +103C8..103CF ; XID_Start # Lo [8] OLD PERSIAN SIGN AURAMAZDAA..OLD PERSIAN SIGN BUUMISH +103D1..103D5 ; XID_Start # Nl [5] OLD PERSIAN NUMBER ONE..OLD PERSIAN NUMBER HUNDRED +10400..1044F ; XID_Start # L& [80] DESERET CAPITAL LETTER LONG I..DESERET SMALL LETTER EW +10450..1049D ; XID_Start # Lo [78] SHAVIAN LETTER PEEP..OSMANYA LETTER OO +10800..10805 ; XID_Start # Lo [6] CYPRIOT SYLLABLE A..CYPRIOT SYLLABLE JA +10808 ; XID_Start # Lo CYPRIOT SYLLABLE JO +1080A..10835 ; XID_Start # Lo [44] CYPRIOT SYLLABLE KA..CYPRIOT SYLLABLE WO +10837..10838 ; XID_Start # Lo [2] CYPRIOT SYLLABLE XA..CYPRIOT SYLLABLE XE +1083C ; XID_Start # Lo CYPRIOT SYLLABLE ZA +1083F..10855 ; XID_Start # Lo [23] CYPRIOT SYLLABLE ZO..IMPERIAL ARAMAIC LETTER TAW +10900..10915 ; XID_Start # Lo [22] PHOENICIAN LETTER ALF..PHOENICIAN LETTER TAU +10920..10939 ; XID_Start # Lo [26] LYDIAN LETTER A..LYDIAN LETTER C +10A00 ; XID_Start # Lo KHAROSHTHI LETTER A +10A10..10A13 ; XID_Start # Lo [4] KHAROSHTHI LETTER KA..KHAROSHTHI LETTER GHA +10A15..10A17 ; XID_Start # Lo [3] KHAROSHTHI LETTER CA..KHAROSHTHI LETTER JA +10A19..10A33 ; XID_Start # Lo [27] KHAROSHTHI LETTER NYA..KHAROSHTHI LETTER TTTHA +10A60..10A7C ; XID_Start # Lo [29] OLD SOUTH ARABIAN LETTER HE..OLD SOUTH ARABIAN LETTER THETH +10B00..10B35 ; XID_Start # Lo [54] AVESTAN LETTER A..AVESTAN LETTER HE +10B40..10B55 ; XID_Start # Lo [22] INSCRIPTIONAL PARTHIAN LETTER ALEPH..INSCRIPTIONAL PARTHIAN LETTER TAW +10B60..10B72 ; XID_Start # Lo [19] INSCRIPTIONAL PAHLAVI LETTER ALEPH..INSCRIPTIONAL PAHLAVI LETTER TAW +10C00..10C48 ; XID_Start # Lo [73] OLD TURKIC LETTER ORKHON A..OLD TURKIC LETTER ORKHON BASH +11083..110AF ; XID_Start # Lo [45] KAITHI LETTER A..KAITHI LETTER HA +12000..1236E ; XID_Start # Lo [879] CUNEIFORM SIGN A..CUNEIFORM SIGN ZUM +12400..12462 ; XID_Start # Nl [99] CUNEIFORM NUMERIC SIGN TWO ASH..CUNEIFORM NUMERIC SIGN OLD ASSYRIAN ONE QUARTER +13000..1342E ; XID_Start # Lo [1071] EGYPTIAN HIEROGLYPH A001..EGYPTIAN HIEROGLYPH AA032 +1D400..1D454 ; XID_Start # L& [85] MATHEMATICAL BOLD CAPITAL A..MATHEMATICAL ITALIC SMALL G +1D456..1D49C ; XID_Start # L& [71] MATHEMATICAL ITALIC SMALL I..MATHEMATICAL SCRIPT CAPITAL A +1D49E..1D49F ; XID_Start # L& [2] MATHEMATICAL SCRIPT CAPITAL C..MATHEMATICAL SCRIPT CAPITAL D +1D4A2 ; XID_Start # L& MATHEMATICAL SCRIPT CAPITAL G +1D4A5..1D4A6 ; XID_Start # L& [2] MATHEMATICAL SCRIPT CAPITAL J..MATHEMATICAL SCRIPT CAPITAL K +1D4A9..1D4AC ; XID_Start # L& [4] MATHEMATICAL SCRIPT CAPITAL N..MATHEMATICAL SCRIPT CAPITAL Q +1D4AE..1D4B9 ; XID_Start # L& [12] MATHEMATICAL SCRIPT CAPITAL S..MATHEMATICAL SCRIPT SMALL D +1D4BB ; XID_Start # L& MATHEMATICAL SCRIPT SMALL F +1D4BD..1D4C3 ; XID_Start # L& [7] MATHEMATICAL SCRIPT SMALL H..MATHEMATICAL SCRIPT SMALL N +1D4C5..1D505 ; XID_Start # L& [65] MATHEMATICAL SCRIPT SMALL P..MATHEMATICAL FRAKTUR CAPITAL B +1D507..1D50A ; XID_Start # L& [4] MATHEMATICAL FRAKTUR CAPITAL D..MATHEMATICAL FRAKTUR CAPITAL G +1D50D..1D514 ; XID_Start # L& [8] MATHEMATICAL FRAKTUR CAPITAL J..MATHEMATICAL FRAKTUR CAPITAL Q +1D516..1D51C ; XID_Start # L& [7] MATHEMATICAL FRAKTUR CAPITAL S..MATHEMATICAL FRAKTUR CAPITAL Y +1D51E..1D539 ; XID_Start # L& [28] MATHEMATICAL FRAKTUR SMALL A..MATHEMATICAL DOUBLE-STRUCK CAPITAL B +1D53B..1D53E ; XID_Start # L& [4] MATHEMATICAL DOUBLE-STRUCK CAPITAL D..MATHEMATICAL DOUBLE-STRUCK CAPITAL G +1D540..1D544 ; XID_Start # L& [5] MATHEMATICAL DOUBLE-STRUCK CAPITAL I..MATHEMATICAL DOUBLE-STRUCK CAPITAL M +1D546 ; XID_Start # L& MATHEMATICAL DOUBLE-STRUCK CAPITAL O +1D54A..1D550 ; XID_Start # L& [7] MATHEMATICAL DOUBLE-STRUCK CAPITAL S..MATHEMATICAL DOUBLE-STRUCK CAPITAL Y +1D552..1D6A5 ; XID_Start # L& [340] MATHEMATICAL DOUBLE-STRUCK SMALL A..MATHEMATICAL ITALIC SMALL DOTLESS J +1D6A8..1D6C0 ; XID_Start # L& [25] MATHEMATICAL BOLD CAPITAL ALPHA..MATHEMATICAL BOLD CAPITAL OMEGA +1D6C2..1D6DA ; XID_Start # L& [25] MATHEMATICAL BOLD SMALL ALPHA..MATHEMATICAL BOLD SMALL OMEGA +1D6DC..1D6FA ; XID_Start # L& [31] MATHEMATICAL BOLD EPSILON SYMBOL..MATHEMATICAL ITALIC CAPITAL OMEGA +1D6FC..1D714 ; XID_Start # L& [25] MATHEMATICAL ITALIC SMALL ALPHA..MATHEMATICAL ITALIC SMALL OMEGA +1D716..1D734 ; XID_Start # L& [31] MATHEMATICAL ITALIC EPSILON SYMBOL..MATHEMATICAL BOLD ITALIC CAPITAL OMEGA +1D736..1D74E ; XID_Start # L& [25] MATHEMATICAL BOLD ITALIC SMALL ALPHA..MATHEMATICAL BOLD ITALIC SMALL OMEGA +1D750..1D76E ; XID_Start # L& [31] MATHEMATICAL BOLD ITALIC EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD CAPITAL OMEGA +1D770..1D788 ; XID_Start # L& [25] MATHEMATICAL SANS-SERIF BOLD SMALL ALPHA..MATHEMATICAL SANS-SERIF BOLD SMALL OMEGA +1D78A..1D7A8 ; XID_Start # L& [31] MATHEMATICAL SANS-SERIF BOLD EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL OMEGA +1D7AA..1D7C2 ; XID_Start # L& [25] MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL ALPHA..MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL OMEGA +1D7C4..1D7CB ; XID_Start # L& [8] MATHEMATICAL SANS-SERIF BOLD ITALIC EPSILON SYMBOL..MATHEMATICAL BOLD SMALL DIGAMMA +20000..2A6D6 ; XID_Start # Lo [42711] CJK UNIFIED IDEOGRAPH-20000..CJK UNIFIED IDEOGRAPH-2A6D6 +2A700..2B734 ; XID_Start # Lo [4149] CJK UNIFIED IDEOGRAPH-2A700..CJK UNIFIED IDEOGRAPH-2B734 +2F800..2FA1D ; XID_Start # Lo [542] CJK COMPATIBILITY IDEOGRAPH-2F800..CJK COMPATIBILITY IDEOGRAPH-2FA1D + +# Total code points: 99741 + +# ================================================ + +# Derived Property: XID_Continue +# Mod_ID_Continue modified for closure under NFKx +# Modified as described in UAX #15 +# NOTE: Cf characters should be filtered out. +# NOTE: Does NOT remove the non-NFKx characters. +# Merely ensures that if isIdentifer(string) then isIdentifier(NFKx(string)) +# NOTE: See UAX #31 for more information + +0030..0039 ; XID_Continue # Nd [10] DIGIT ZERO..DIGIT NINE +0041..005A ; XID_Continue # L& [26] LATIN CAPITAL LETTER A..LATIN CAPITAL LETTER Z +005F ; XID_Continue # Pc LOW LINE +0061..007A ; XID_Continue # L& [26] LATIN SMALL LETTER A..LATIN SMALL LETTER Z +00AA ; XID_Continue # L& FEMININE ORDINAL INDICATOR +00B5 ; XID_Continue # L& MICRO SIGN +00B7 ; XID_Continue # Po MIDDLE DOT +00BA ; XID_Continue # L& MASCULINE ORDINAL INDICATOR +00C0..00D6 ; XID_Continue # L& [23] LATIN CAPITAL LETTER A WITH GRAVE..LATIN CAPITAL LETTER O WITH DIAERESIS +00D8..00F6 ; XID_Continue # L& [31] LATIN CAPITAL LETTER O WITH STROKE..LATIN SMALL LETTER O WITH DIAERESIS +00F8..01BA ; XID_Continue # L& [195] LATIN SMALL LETTER O WITH STROKE..LATIN SMALL LETTER EZH WITH TAIL +01BB ; XID_Continue # Lo LATIN LETTER TWO WITH STROKE +01BC..01BF ; XID_Continue # L& [4] LATIN CAPITAL LETTER TONE FIVE..LATIN LETTER WYNN +01C0..01C3 ; XID_Continue # Lo [4] LATIN LETTER DENTAL CLICK..LATIN LETTER RETROFLEX CLICK +01C4..0293 ; XID_Continue # L& [208] LATIN CAPITAL LETTER DZ WITH CARON..LATIN SMALL LETTER EZH WITH CURL +0294 ; XID_Continue # Lo LATIN LETTER GLOTTAL STOP +0295..02AF ; XID_Continue # L& [27] LATIN LETTER PHARYNGEAL VOICED FRICATIVE..LATIN SMALL LETTER TURNED H WITH FISHHOOK AND TAIL +02B0..02C1 ; XID_Continue # Lm [18] MODIFIER LETTER SMALL H..MODIFIER LETTER REVERSED GLOTTAL STOP +02C6..02D1 ; XID_Continue # Lm [12] MODIFIER LETTER CIRCUMFLEX ACCENT..MODIFIER LETTER HALF TRIANGULAR COLON +02E0..02E4 ; XID_Continue # Lm [5] MODIFIER LETTER SMALL GAMMA..MODIFIER LETTER SMALL REVERSED GLOTTAL STOP +02EC ; XID_Continue # Lm MODIFIER LETTER VOICING +02EE ; XID_Continue # Lm MODIFIER LETTER DOUBLE APOSTROPHE +0300..036F ; XID_Continue # Mn [112] COMBINING GRAVE ACCENT..COMBINING LATIN SMALL LETTER X +0370..0373 ; XID_Continue # L& [4] GREEK CAPITAL LETTER HETA..GREEK SMALL LETTER ARCHAIC SAMPI +0374 ; XID_Continue # Lm GREEK NUMERAL SIGN +0376..0377 ; XID_Continue # L& [2] GREEK CAPITAL LETTER PAMPHYLIAN DIGAMMA..GREEK SMALL LETTER PAMPHYLIAN DIGAMMA +037B..037D ; XID_Continue # L& [3] GREEK SMALL REVERSED LUNATE SIGMA SYMBOL..GREEK SMALL REVERSED DOTTED LUNATE SIGMA SYMBOL +0386 ; XID_Continue # L& GREEK CAPITAL LETTER ALPHA WITH TONOS +0387 ; XID_Continue # Po GREEK ANO TELEIA +0388..038A ; XID_Continue # L& [3] GREEK CAPITAL LETTER EPSILON WITH TONOS..GREEK CAPITAL LETTER IOTA WITH TONOS +038C ; XID_Continue # L& GREEK CAPITAL LETTER OMICRON WITH TONOS +038E..03A1 ; XID_Continue # L& [20] GREEK CAPITAL LETTER UPSILON WITH TONOS..GREEK CAPITAL LETTER RHO +03A3..03F5 ; XID_Continue # L& [83] GREEK CAPITAL LETTER SIGMA..GREEK LUNATE EPSILON SYMBOL +03F7..0481 ; XID_Continue # L& [139] GREEK CAPITAL LETTER SHO..CYRILLIC SMALL LETTER KOPPA +0483..0487 ; XID_Continue # Mn [5] COMBINING CYRILLIC TITLO..COMBINING CYRILLIC POKRYTIE +048A..0525 ; XID_Continue # L& [156] CYRILLIC CAPITAL LETTER SHORT I WITH TAIL..CYRILLIC SMALL LETTER PE WITH DESCENDER +0531..0556 ; XID_Continue # L& [38] ARMENIAN CAPITAL LETTER AYB..ARMENIAN CAPITAL LETTER FEH +0559 ; XID_Continue # Lm ARMENIAN MODIFIER LETTER LEFT HALF RING +0561..0587 ; XID_Continue # L& [39] ARMENIAN SMALL LETTER AYB..ARMENIAN SMALL LIGATURE ECH YIWN +0591..05BD ; XID_Continue # Mn [45] HEBREW ACCENT ETNAHTA..HEBREW POINT METEG +05BF ; XID_Continue # Mn HEBREW POINT RAFE +05C1..05C2 ; XID_Continue # Mn [2] HEBREW POINT SHIN DOT..HEBREW POINT SIN DOT +05C4..05C5 ; XID_Continue # Mn [2] HEBREW MARK UPPER DOT..HEBREW MARK LOWER DOT +05C7 ; XID_Continue # Mn HEBREW POINT QAMATS QATAN +05D0..05EA ; XID_Continue # Lo [27] HEBREW LETTER ALEF..HEBREW LETTER TAV +05F0..05F2 ; XID_Continue # Lo [3] HEBREW LIGATURE YIDDISH DOUBLE VAV..HEBREW LIGATURE YIDDISH DOUBLE YOD +0610..061A ; XID_Continue # Mn [11] ARABIC SIGN SALLALLAHOU ALAYHE WASSALLAM..ARABIC SMALL KASRA +0621..063F ; XID_Continue # Lo [31] ARABIC LETTER HAMZA..ARABIC LETTER FARSI YEH WITH THREE DOTS ABOVE +0640 ; XID_Continue # Lm ARABIC TATWEEL +0641..064A ; XID_Continue # Lo [10] ARABIC LETTER FEH..ARABIC LETTER YEH +064B..065E ; XID_Continue # Mn [20] ARABIC FATHATAN..ARABIC FATHA WITH TWO DOTS +0660..0669 ; XID_Continue # Nd [10] ARABIC-INDIC DIGIT ZERO..ARABIC-INDIC DIGIT NINE +066E..066F ; XID_Continue # Lo [2] ARABIC LETTER DOTLESS BEH..ARABIC LETTER DOTLESS QAF +0670 ; XID_Continue # Mn ARABIC LETTER SUPERSCRIPT ALEF +0671..06D3 ; XID_Continue # Lo [99] ARABIC LETTER ALEF WASLA..ARABIC LETTER YEH BARREE WITH HAMZA ABOVE +06D5 ; XID_Continue # Lo ARABIC LETTER AE +06D6..06DC ; XID_Continue # Mn [7] ARABIC SMALL HIGH LIGATURE SAD WITH LAM WITH ALEF MAKSURA..ARABIC SMALL HIGH SEEN +06DF..06E4 ; XID_Continue # Mn [6] ARABIC SMALL HIGH ROUNDED ZERO..ARABIC SMALL HIGH MADDA +06E5..06E6 ; XID_Continue # Lm [2] ARABIC SMALL WAW..ARABIC SMALL YEH +06E7..06E8 ; XID_Continue # Mn [2] ARABIC SMALL HIGH YEH..ARABIC SMALL HIGH NOON +06EA..06ED ; XID_Continue # Mn [4] ARABIC EMPTY CENTRE LOW STOP..ARABIC SMALL LOW MEEM +06EE..06EF ; XID_Continue # Lo [2] ARABIC LETTER DAL WITH INVERTED V..ARABIC LETTER REH WITH INVERTED V +06F0..06F9 ; XID_Continue # Nd [10] EXTENDED ARABIC-INDIC DIGIT ZERO..EXTENDED ARABIC-INDIC DIGIT NINE +06FA..06FC ; XID_Continue # Lo [3] ARABIC LETTER SHEEN WITH DOT BELOW..ARABIC LETTER GHAIN WITH DOT BELOW +06FF ; XID_Continue # Lo ARABIC LETTER HEH WITH INVERTED V +0710 ; XID_Continue # Lo SYRIAC LETTER ALAPH +0711 ; XID_Continue # Mn SYRIAC LETTER SUPERSCRIPT ALAPH +0712..072F ; XID_Continue # Lo [30] SYRIAC LETTER BETH..SYRIAC LETTER PERSIAN DHALATH +0730..074A ; XID_Continue # Mn [27] SYRIAC PTHAHA ABOVE..SYRIAC BARREKH +074D..07A5 ; XID_Continue # Lo [89] SYRIAC LETTER SOGDIAN ZHAIN..THAANA LETTER WAAVU +07A6..07B0 ; XID_Continue # Mn [11] THAANA ABAFILI..THAANA SUKUN +07B1 ; XID_Continue # Lo THAANA LETTER NAA +07C0..07C9 ; XID_Continue # Nd [10] NKO DIGIT ZERO..NKO DIGIT NINE +07CA..07EA ; XID_Continue # Lo [33] NKO LETTER A..NKO LETTER JONA RA +07EB..07F3 ; XID_Continue # Mn [9] NKO COMBINING SHORT HIGH TONE..NKO COMBINING DOUBLE DOT ABOVE +07F4..07F5 ; XID_Continue # Lm [2] NKO HIGH TONE APOSTROPHE..NKO LOW TONE APOSTROPHE +07FA ; XID_Continue # Lm NKO LAJANYALAN +0800..0815 ; XID_Continue # Lo [22] SAMARITAN LETTER ALAF..SAMARITAN LETTER TAAF +0816..0819 ; XID_Continue # Mn [4] SAMARITAN MARK IN..SAMARITAN MARK DAGESH +081A ; XID_Continue # Lm SAMARITAN MODIFIER LETTER EPENTHETIC YUT +081B..0823 ; XID_Continue # Mn [9] SAMARITAN MARK EPENTHETIC YUT..SAMARITAN VOWEL SIGN A +0824 ; XID_Continue # Lm SAMARITAN MODIFIER LETTER SHORT A +0825..0827 ; XID_Continue # Mn [3] SAMARITAN VOWEL SIGN SHORT A..SAMARITAN VOWEL SIGN U +0828 ; XID_Continue # Lm SAMARITAN MODIFIER LETTER I +0829..082D ; XID_Continue # Mn [5] SAMARITAN VOWEL SIGN LONG I..SAMARITAN MARK NEQUDAA +0900..0902 ; XID_Continue # Mn [3] DEVANAGARI SIGN INVERTED CANDRABINDU..DEVANAGARI SIGN ANUSVARA +0903 ; XID_Continue # Mc DEVANAGARI SIGN VISARGA +0904..0939 ; XID_Continue # Lo [54] DEVANAGARI LETTER SHORT A..DEVANAGARI LETTER HA +093C ; XID_Continue # Mn DEVANAGARI SIGN NUKTA +093D ; XID_Continue # Lo DEVANAGARI SIGN AVAGRAHA +093E..0940 ; XID_Continue # Mc [3] DEVANAGARI VOWEL SIGN AA..DEVANAGARI VOWEL SIGN II +0941..0948 ; XID_Continue # Mn [8] DEVANAGARI VOWEL SIGN U..DEVANAGARI VOWEL SIGN AI +0949..094C ; XID_Continue # Mc [4] DEVANAGARI VOWEL SIGN CANDRA O..DEVANAGARI VOWEL SIGN AU +094D ; XID_Continue # Mn DEVANAGARI SIGN VIRAMA +094E ; XID_Continue # Mc DEVANAGARI VOWEL SIGN PRISHTHAMATRA E +0950 ; XID_Continue # Lo DEVANAGARI OM +0951..0955 ; XID_Continue # Mn [5] DEVANAGARI STRESS SIGN UDATTA..DEVANAGARI VOWEL SIGN CANDRA LONG E +0958..0961 ; XID_Continue # Lo [10] DEVANAGARI LETTER QA..DEVANAGARI LETTER VOCALIC LL +0962..0963 ; XID_Continue # Mn [2] DEVANAGARI VOWEL SIGN VOCALIC L..DEVANAGARI VOWEL SIGN VOCALIC LL +0966..096F ; XID_Continue # Nd [10] DEVANAGARI DIGIT ZERO..DEVANAGARI DIGIT NINE +0971 ; XID_Continue # Lm DEVANAGARI SIGN HIGH SPACING DOT +0972 ; XID_Continue # Lo DEVANAGARI LETTER CANDRA A +0979..097F ; XID_Continue # Lo [7] DEVANAGARI LETTER ZHA..DEVANAGARI LETTER BBA +0981 ; XID_Continue # Mn BENGALI SIGN CANDRABINDU +0982..0983 ; XID_Continue # Mc [2] BENGALI SIGN ANUSVARA..BENGALI SIGN VISARGA +0985..098C ; XID_Continue # Lo [8] BENGALI LETTER A..BENGALI LETTER VOCALIC L +098F..0990 ; XID_Continue # Lo [2] BENGALI LETTER E..BENGALI LETTER AI +0993..09A8 ; XID_Continue # Lo [22] BENGALI LETTER O..BENGALI LETTER NA +09AA..09B0 ; XID_Continue # Lo [7] BENGALI LETTER PA..BENGALI LETTER RA +09B2 ; XID_Continue # Lo BENGALI LETTER LA +09B6..09B9 ; XID_Continue # Lo [4] BENGALI LETTER SHA..BENGALI LETTER HA +09BC ; XID_Continue # Mn BENGALI SIGN NUKTA +09BD ; XID_Continue # Lo BENGALI SIGN AVAGRAHA +09BE..09C0 ; XID_Continue # Mc [3] BENGALI VOWEL SIGN AA..BENGALI VOWEL SIGN II +09C1..09C4 ; XID_Continue # Mn [4] BENGALI VOWEL SIGN U..BENGALI VOWEL SIGN VOCALIC RR +09C7..09C8 ; XID_Continue # Mc [2] BENGALI VOWEL SIGN E..BENGALI VOWEL SIGN AI +09CB..09CC ; XID_Continue # Mc [2] BENGALI VOWEL SIGN O..BENGALI VOWEL SIGN AU +09CD ; XID_Continue # Mn BENGALI SIGN VIRAMA +09CE ; XID_Continue # Lo BENGALI LETTER KHANDA TA +09D7 ; XID_Continue # Mc BENGALI AU LENGTH MARK +09DC..09DD ; XID_Continue # Lo [2] BENGALI LETTER RRA..BENGALI LETTER RHA +09DF..09E1 ; XID_Continue # Lo [3] BENGALI LETTER YYA..BENGALI LETTER VOCALIC LL +09E2..09E3 ; XID_Continue # Mn [2] BENGALI VOWEL SIGN VOCALIC L..BENGALI VOWEL SIGN VOCALIC LL +09E6..09EF ; XID_Continue # Nd [10] BENGALI DIGIT ZERO..BENGALI DIGIT NINE +09F0..09F1 ; XID_Continue # Lo [2] BENGALI LETTER RA WITH MIDDLE DIAGONAL..BENGALI LETTER RA WITH LOWER DIAGONAL +0A01..0A02 ; XID_Continue # Mn [2] GURMUKHI SIGN ADAK BINDI..GURMUKHI SIGN BINDI +0A03 ; XID_Continue # Mc GURMUKHI SIGN VISARGA +0A05..0A0A ; XID_Continue # Lo [6] GURMUKHI LETTER A..GURMUKHI LETTER UU +0A0F..0A10 ; XID_Continue # Lo [2] GURMUKHI LETTER EE..GURMUKHI LETTER AI +0A13..0A28 ; XID_Continue # Lo [22] GURMUKHI LETTER OO..GURMUKHI LETTER NA +0A2A..0A30 ; XID_Continue # Lo [7] GURMUKHI LETTER PA..GURMUKHI LETTER RA +0A32..0A33 ; XID_Continue # Lo [2] GURMUKHI LETTER LA..GURMUKHI LETTER LLA +0A35..0A36 ; XID_Continue # Lo [2] GURMUKHI LETTER VA..GURMUKHI LETTER SHA +0A38..0A39 ; XID_Continue # Lo [2] GURMUKHI LETTER SA..GURMUKHI LETTER HA +0A3C ; XID_Continue # Mn GURMUKHI SIGN NUKTA +0A3E..0A40 ; XID_Continue # Mc [3] GURMUKHI VOWEL SIGN AA..GURMUKHI VOWEL SIGN II +0A41..0A42 ; XID_Continue # Mn [2] GURMUKHI VOWEL SIGN U..GURMUKHI VOWEL SIGN UU +0A47..0A48 ; XID_Continue # Mn [2] GURMUKHI VOWEL SIGN EE..GURMUKHI VOWEL SIGN AI +0A4B..0A4D ; XID_Continue # Mn [3] GURMUKHI VOWEL SIGN OO..GURMUKHI SIGN VIRAMA +0A51 ; XID_Continue # Mn GURMUKHI SIGN UDAAT +0A59..0A5C ; XID_Continue # Lo [4] GURMUKHI LETTER KHHA..GURMUKHI LETTER RRA +0A5E ; XID_Continue # Lo GURMUKHI LETTER FA +0A66..0A6F ; XID_Continue # Nd [10] GURMUKHI DIGIT ZERO..GURMUKHI DIGIT NINE +0A70..0A71 ; XID_Continue # Mn [2] GURMUKHI TIPPI..GURMUKHI ADDAK +0A72..0A74 ; XID_Continue # Lo [3] GURMUKHI IRI..GURMUKHI EK ONKAR +0A75 ; XID_Continue # Mn GURMUKHI SIGN YAKASH +0A81..0A82 ; XID_Continue # Mn [2] GUJARATI SIGN CANDRABINDU..GUJARATI SIGN ANUSVARA +0A83 ; XID_Continue # Mc GUJARATI SIGN VISARGA +0A85..0A8D ; XID_Continue # Lo [9] GUJARATI LETTER A..GUJARATI VOWEL CANDRA E +0A8F..0A91 ; XID_Continue # Lo [3] GUJARATI LETTER E..GUJARATI VOWEL CANDRA O +0A93..0AA8 ; XID_Continue # Lo [22] GUJARATI LETTER O..GUJARATI LETTER NA +0AAA..0AB0 ; XID_Continue # Lo [7] GUJARATI LETTER PA..GUJARATI LETTER RA +0AB2..0AB3 ; XID_Continue # Lo [2] GUJARATI LETTER LA..GUJARATI LETTER LLA +0AB5..0AB9 ; XID_Continue # Lo [5] GUJARATI LETTER VA..GUJARATI LETTER HA +0ABC ; XID_Continue # Mn GUJARATI SIGN NUKTA +0ABD ; XID_Continue # Lo GUJARATI SIGN AVAGRAHA +0ABE..0AC0 ; XID_Continue # Mc [3] GUJARATI VOWEL SIGN AA..GUJARATI VOWEL SIGN II +0AC1..0AC5 ; XID_Continue # Mn [5] GUJARATI VOWEL SIGN U..GUJARATI VOWEL SIGN CANDRA E +0AC7..0AC8 ; XID_Continue # Mn [2] GUJARATI VOWEL SIGN E..GUJARATI VOWEL SIGN AI +0AC9 ; XID_Continue # Mc GUJARATI VOWEL SIGN CANDRA O +0ACB..0ACC ; XID_Continue # Mc [2] GUJARATI VOWEL SIGN O..GUJARATI VOWEL SIGN AU +0ACD ; XID_Continue # Mn GUJARATI SIGN VIRAMA +0AD0 ; XID_Continue # Lo GUJARATI OM +0AE0..0AE1 ; XID_Continue # Lo [2] GUJARATI LETTER VOCALIC RR..GUJARATI LETTER VOCALIC LL +0AE2..0AE3 ; XID_Continue # Mn [2] GUJARATI VOWEL SIGN VOCALIC L..GUJARATI VOWEL SIGN VOCALIC LL +0AE6..0AEF ; XID_Continue # Nd [10] GUJARATI DIGIT ZERO..GUJARATI DIGIT NINE +0B01 ; XID_Continue # Mn ORIYA SIGN CANDRABINDU +0B02..0B03 ; XID_Continue # Mc [2] ORIYA SIGN ANUSVARA..ORIYA SIGN VISARGA +0B05..0B0C ; XID_Continue # Lo [8] ORIYA LETTER A..ORIYA LETTER VOCALIC L +0B0F..0B10 ; XID_Continue # Lo [2] ORIYA LETTER E..ORIYA LETTER AI +0B13..0B28 ; XID_Continue # Lo [22] ORIYA LETTER O..ORIYA LETTER NA +0B2A..0B30 ; XID_Continue # Lo [7] ORIYA LETTER PA..ORIYA LETTER RA +0B32..0B33 ; XID_Continue # Lo [2] ORIYA LETTER LA..ORIYA LETTER LLA +0B35..0B39 ; XID_Continue # Lo [5] ORIYA LETTER VA..ORIYA LETTER HA +0B3C ; XID_Continue # Mn ORIYA SIGN NUKTA +0B3D ; XID_Continue # Lo ORIYA SIGN AVAGRAHA +0B3E ; XID_Continue # Mc ORIYA VOWEL SIGN AA +0B3F ; XID_Continue # Mn ORIYA VOWEL SIGN I +0B40 ; XID_Continue # Mc ORIYA VOWEL SIGN II +0B41..0B44 ; XID_Continue # Mn [4] ORIYA VOWEL SIGN U..ORIYA VOWEL SIGN VOCALIC RR +0B47..0B48 ; XID_Continue # Mc [2] ORIYA VOWEL SIGN E..ORIYA VOWEL SIGN AI +0B4B..0B4C ; XID_Continue # Mc [2] ORIYA VOWEL SIGN O..ORIYA VOWEL SIGN AU +0B4D ; XID_Continue # Mn ORIYA SIGN VIRAMA +0B56 ; XID_Continue # Mn ORIYA AI LENGTH MARK +0B57 ; XID_Continue # Mc ORIYA AU LENGTH MARK +0B5C..0B5D ; XID_Continue # Lo [2] ORIYA LETTER RRA..ORIYA LETTER RHA +0B5F..0B61 ; XID_Continue # Lo [3] ORIYA LETTER YYA..ORIYA LETTER VOCALIC LL +0B62..0B63 ; XID_Continue # Mn [2] ORIYA VOWEL SIGN VOCALIC L..ORIYA VOWEL SIGN VOCALIC LL +0B66..0B6F ; XID_Continue # Nd [10] ORIYA DIGIT ZERO..ORIYA DIGIT NINE +0B71 ; XID_Continue # Lo ORIYA LETTER WA +0B82 ; XID_Continue # Mn TAMIL SIGN ANUSVARA +0B83 ; XID_Continue # Lo TAMIL SIGN VISARGA +0B85..0B8A ; XID_Continue # Lo [6] TAMIL LETTER A..TAMIL LETTER UU +0B8E..0B90 ; XID_Continue # Lo [3] TAMIL LETTER E..TAMIL LETTER AI +0B92..0B95 ; XID_Continue # Lo [4] TAMIL LETTER O..TAMIL LETTER KA +0B99..0B9A ; XID_Continue # Lo [2] TAMIL LETTER NGA..TAMIL LETTER CA +0B9C ; XID_Continue # Lo TAMIL LETTER JA +0B9E..0B9F ; XID_Continue # Lo [2] TAMIL LETTER NYA..TAMIL LETTER TTA +0BA3..0BA4 ; XID_Continue # Lo [2] TAMIL LETTER NNA..TAMIL LETTER TA +0BA8..0BAA ; XID_Continue # Lo [3] TAMIL LETTER NA..TAMIL LETTER PA +0BAE..0BB9 ; XID_Continue # Lo [12] TAMIL LETTER MA..TAMIL LETTER HA +0BBE..0BBF ; XID_Continue # Mc [2] TAMIL VOWEL SIGN AA..TAMIL VOWEL SIGN I +0BC0 ; XID_Continue # Mn TAMIL VOWEL SIGN II +0BC1..0BC2 ; XID_Continue # Mc [2] TAMIL VOWEL SIGN U..TAMIL VOWEL SIGN UU +0BC6..0BC8 ; XID_Continue # Mc [3] TAMIL VOWEL SIGN E..TAMIL VOWEL SIGN AI +0BCA..0BCC ; XID_Continue # Mc [3] TAMIL VOWEL SIGN O..TAMIL VOWEL SIGN AU +0BCD ; XID_Continue # Mn TAMIL SIGN VIRAMA +0BD0 ; XID_Continue # Lo TAMIL OM +0BD7 ; XID_Continue # Mc TAMIL AU LENGTH MARK +0BE6..0BEF ; XID_Continue # Nd [10] TAMIL DIGIT ZERO..TAMIL DIGIT NINE +0C01..0C03 ; XID_Continue # Mc [3] TELUGU SIGN CANDRABINDU..TELUGU SIGN VISARGA +0C05..0C0C ; XID_Continue # Lo [8] TELUGU LETTER A..TELUGU LETTER VOCALIC L +0C0E..0C10 ; XID_Continue # Lo [3] TELUGU LETTER E..TELUGU LETTER AI +0C12..0C28 ; XID_Continue # Lo [23] TELUGU LETTER O..TELUGU LETTER NA +0C2A..0C33 ; XID_Continue # Lo [10] TELUGU LETTER PA..TELUGU LETTER LLA +0C35..0C39 ; XID_Continue # Lo [5] TELUGU LETTER VA..TELUGU LETTER HA +0C3D ; XID_Continue # Lo TELUGU SIGN AVAGRAHA +0C3E..0C40 ; XID_Continue # Mn [3] TELUGU VOWEL SIGN AA..TELUGU VOWEL SIGN II +0C41..0C44 ; XID_Continue # Mc [4] TELUGU VOWEL SIGN U..TELUGU VOWEL SIGN VOCALIC RR +0C46..0C48 ; XID_Continue # Mn [3] TELUGU VOWEL SIGN E..TELUGU VOWEL SIGN AI +0C4A..0C4D ; XID_Continue # Mn [4] TELUGU VOWEL SIGN O..TELUGU SIGN VIRAMA +0C55..0C56 ; XID_Continue # Mn [2] TELUGU LENGTH MARK..TELUGU AI LENGTH MARK +0C58..0C59 ; XID_Continue # Lo [2] TELUGU LETTER TSA..TELUGU LETTER DZA +0C60..0C61 ; XID_Continue # Lo [2] TELUGU LETTER VOCALIC RR..TELUGU LETTER VOCALIC LL +0C62..0C63 ; XID_Continue # Mn [2] TELUGU VOWEL SIGN VOCALIC L..TELUGU VOWEL SIGN VOCALIC LL +0C66..0C6F ; XID_Continue # Nd [10] TELUGU DIGIT ZERO..TELUGU DIGIT NINE +0C82..0C83 ; XID_Continue # Mc [2] KANNADA SIGN ANUSVARA..KANNADA SIGN VISARGA +0C85..0C8C ; XID_Continue # Lo [8] KANNADA LETTER A..KANNADA LETTER VOCALIC L +0C8E..0C90 ; XID_Continue # Lo [3] KANNADA LETTER E..KANNADA LETTER AI +0C92..0CA8 ; XID_Continue # Lo [23] KANNADA LETTER O..KANNADA LETTER NA +0CAA..0CB3 ; XID_Continue # Lo [10] KANNADA LETTER PA..KANNADA LETTER LLA +0CB5..0CB9 ; XID_Continue # Lo [5] KANNADA LETTER VA..KANNADA LETTER HA +0CBC ; XID_Continue # Mn KANNADA SIGN NUKTA +0CBD ; XID_Continue # Lo KANNADA SIGN AVAGRAHA +0CBE ; XID_Continue # Mc KANNADA VOWEL SIGN AA +0CBF ; XID_Continue # Mn KANNADA VOWEL SIGN I +0CC0..0CC4 ; XID_Continue # Mc [5] KANNADA VOWEL SIGN II..KANNADA VOWEL SIGN VOCALIC RR +0CC6 ; XID_Continue # Mn KANNADA VOWEL SIGN E +0CC7..0CC8 ; XID_Continue # Mc [2] KANNADA VOWEL SIGN EE..KANNADA VOWEL SIGN AI +0CCA..0CCB ; XID_Continue # Mc [2] KANNADA VOWEL SIGN O..KANNADA VOWEL SIGN OO +0CCC..0CCD ; XID_Continue # Mn [2] KANNADA VOWEL SIGN AU..KANNADA SIGN VIRAMA +0CD5..0CD6 ; XID_Continue # Mc [2] KANNADA LENGTH MARK..KANNADA AI LENGTH MARK +0CDE ; XID_Continue # Lo KANNADA LETTER FA +0CE0..0CE1 ; XID_Continue # Lo [2] KANNADA LETTER VOCALIC RR..KANNADA LETTER VOCALIC LL +0CE2..0CE3 ; XID_Continue # Mn [2] KANNADA VOWEL SIGN VOCALIC L..KANNADA VOWEL SIGN VOCALIC LL +0CE6..0CEF ; XID_Continue # Nd [10] KANNADA DIGIT ZERO..KANNADA DIGIT NINE +0D02..0D03 ; XID_Continue # Mc [2] MALAYALAM SIGN ANUSVARA..MALAYALAM SIGN VISARGA +0D05..0D0C ; XID_Continue # Lo [8] MALAYALAM LETTER A..MALAYALAM LETTER VOCALIC L +0D0E..0D10 ; XID_Continue # Lo [3] MALAYALAM LETTER E..MALAYALAM LETTER AI +0D12..0D28 ; XID_Continue # Lo [23] MALAYALAM LETTER O..MALAYALAM LETTER NA +0D2A..0D39 ; XID_Continue # Lo [16] MALAYALAM LETTER PA..MALAYALAM LETTER HA +0D3D ; XID_Continue # Lo MALAYALAM SIGN AVAGRAHA +0D3E..0D40 ; XID_Continue # Mc [3] MALAYALAM VOWEL SIGN AA..MALAYALAM VOWEL SIGN II +0D41..0D44 ; XID_Continue # Mn [4] MALAYALAM VOWEL SIGN U..MALAYALAM VOWEL SIGN VOCALIC RR +0D46..0D48 ; XID_Continue # Mc [3] MALAYALAM VOWEL SIGN E..MALAYALAM VOWEL SIGN AI +0D4A..0D4C ; XID_Continue # Mc [3] MALAYALAM VOWEL SIGN O..MALAYALAM VOWEL SIGN AU +0D4D ; XID_Continue # Mn MALAYALAM SIGN VIRAMA +0D57 ; XID_Continue # Mc MALAYALAM AU LENGTH MARK +0D60..0D61 ; XID_Continue # Lo [2] MALAYALAM LETTER VOCALIC RR..MALAYALAM LETTER VOCALIC LL +0D62..0D63 ; XID_Continue # Mn [2] MALAYALAM VOWEL SIGN VOCALIC L..MALAYALAM VOWEL SIGN VOCALIC LL +0D66..0D6F ; XID_Continue # Nd [10] MALAYALAM DIGIT ZERO..MALAYALAM DIGIT NINE +0D7A..0D7F ; XID_Continue # Lo [6] MALAYALAM LETTER CHILLU NN..MALAYALAM LETTER CHILLU K +0D82..0D83 ; XID_Continue # Mc [2] SINHALA SIGN ANUSVARAYA..SINHALA SIGN VISARGAYA +0D85..0D96 ; XID_Continue # Lo [18] SINHALA LETTER AYANNA..SINHALA LETTER AUYANNA +0D9A..0DB1 ; XID_Continue # Lo [24] SINHALA LETTER ALPAPRAANA KAYANNA..SINHALA LETTER DANTAJA NAYANNA +0DB3..0DBB ; XID_Continue # Lo [9] SINHALA LETTER SANYAKA DAYANNA..SINHALA LETTER RAYANNA +0DBD ; XID_Continue # Lo SINHALA LETTER DANTAJA LAYANNA +0DC0..0DC6 ; XID_Continue # Lo [7] SINHALA LETTER VAYANNA..SINHALA LETTER FAYANNA +0DCA ; XID_Continue # Mn SINHALA SIGN AL-LAKUNA +0DCF..0DD1 ; XID_Continue # Mc [3] SINHALA VOWEL SIGN AELA-PILLA..SINHALA VOWEL SIGN DIGA AEDA-PILLA +0DD2..0DD4 ; XID_Continue # Mn [3] SINHALA VOWEL SIGN KETTI IS-PILLA..SINHALA VOWEL SIGN KETTI PAA-PILLA +0DD6 ; XID_Continue # Mn SINHALA VOWEL SIGN DIGA PAA-PILLA +0DD8..0DDF ; XID_Continue # Mc [8] SINHALA VOWEL SIGN GAETTA-PILLA..SINHALA VOWEL SIGN GAYANUKITTA +0DF2..0DF3 ; XID_Continue # Mc [2] SINHALA VOWEL SIGN DIGA GAETTA-PILLA..SINHALA VOWEL SIGN DIGA GAYANUKITTA +0E01..0E30 ; XID_Continue # Lo [48] THAI CHARACTER KO KAI..THAI CHARACTER SARA A +0E31 ; XID_Continue # Mn THAI CHARACTER MAI HAN-AKAT +0E32..0E33 ; XID_Continue # Lo [2] THAI CHARACTER SARA AA..THAI CHARACTER SARA AM +0E34..0E3A ; XID_Continue # Mn [7] THAI CHARACTER SARA I..THAI CHARACTER PHINTHU +0E40..0E45 ; XID_Continue # Lo [6] THAI CHARACTER SARA E..THAI CHARACTER LAKKHANGYAO +0E46 ; XID_Continue # Lm THAI CHARACTER MAIYAMOK +0E47..0E4E ; XID_Continue # Mn [8] THAI CHARACTER MAITAIKHU..THAI CHARACTER YAMAKKAN +0E50..0E59 ; XID_Continue # Nd [10] THAI DIGIT ZERO..THAI DIGIT NINE +0E81..0E82 ; XID_Continue # Lo [2] LAO LETTER KO..LAO LETTER KHO SUNG +0E84 ; XID_Continue # Lo LAO LETTER KHO TAM +0E87..0E88 ; XID_Continue # Lo [2] LAO LETTER NGO..LAO LETTER CO +0E8A ; XID_Continue # Lo LAO LETTER SO TAM +0E8D ; XID_Continue # Lo LAO LETTER NYO +0E94..0E97 ; XID_Continue # Lo [4] LAO LETTER DO..LAO LETTER THO TAM +0E99..0E9F ; XID_Continue # Lo [7] LAO LETTER NO..LAO LETTER FO SUNG +0EA1..0EA3 ; XID_Continue # Lo [3] LAO LETTER MO..LAO LETTER LO LING +0EA5 ; XID_Continue # Lo LAO LETTER LO LOOT +0EA7 ; XID_Continue # Lo LAO LETTER WO +0EAA..0EAB ; XID_Continue # Lo [2] LAO LETTER SO SUNG..LAO LETTER HO SUNG +0EAD..0EB0 ; XID_Continue # Lo [4] LAO LETTER O..LAO VOWEL SIGN A +0EB1 ; XID_Continue # Mn LAO VOWEL SIGN MAI KAN +0EB2..0EB3 ; XID_Continue # Lo [2] LAO VOWEL SIGN AA..LAO VOWEL SIGN AM +0EB4..0EB9 ; XID_Continue # Mn [6] LAO VOWEL SIGN I..LAO VOWEL SIGN UU +0EBB..0EBC ; XID_Continue # Mn [2] LAO VOWEL SIGN MAI KON..LAO SEMIVOWEL SIGN LO +0EBD ; XID_Continue # Lo LAO SEMIVOWEL SIGN NYO +0EC0..0EC4 ; XID_Continue # Lo [5] LAO VOWEL SIGN E..LAO VOWEL SIGN AI +0EC6 ; XID_Continue # Lm LAO KO LA +0EC8..0ECD ; XID_Continue # Mn [6] LAO TONE MAI EK..LAO NIGGAHITA +0ED0..0ED9 ; XID_Continue # Nd [10] LAO DIGIT ZERO..LAO DIGIT NINE +0EDC..0EDD ; XID_Continue # Lo [2] LAO HO NO..LAO HO MO +0F00 ; XID_Continue # Lo TIBETAN SYLLABLE OM +0F18..0F19 ; XID_Continue # Mn [2] TIBETAN ASTROLOGICAL SIGN -KHYUD PA..TIBETAN ASTROLOGICAL SIGN SDONG TSHUGS +0F20..0F29 ; XID_Continue # Nd [10] TIBETAN DIGIT ZERO..TIBETAN DIGIT NINE +0F35 ; XID_Continue # Mn TIBETAN MARK NGAS BZUNG NYI ZLA +0F37 ; XID_Continue # Mn TIBETAN MARK NGAS BZUNG SGOR RTAGS +0F39 ; XID_Continue # Mn TIBETAN MARK TSA -PHRU +0F3E..0F3F ; XID_Continue # Mc [2] TIBETAN SIGN YAR TSHES..TIBETAN SIGN MAR TSHES +0F40..0F47 ; XID_Continue # Lo [8] TIBETAN LETTER KA..TIBETAN LETTER JA +0F49..0F6C ; XID_Continue # Lo [36] TIBETAN LETTER NYA..TIBETAN LETTER RRA +0F71..0F7E ; XID_Continue # Mn [14] TIBETAN VOWEL SIGN AA..TIBETAN SIGN RJES SU NGA RO +0F7F ; XID_Continue # Mc TIBETAN SIGN RNAM BCAD +0F80..0F84 ; XID_Continue # Mn [5] TIBETAN VOWEL SIGN REVERSED I..TIBETAN MARK HALANTA +0F86..0F87 ; XID_Continue # Mn [2] TIBETAN SIGN LCI RTAGS..TIBETAN SIGN YANG RTAGS +0F88..0F8B ; XID_Continue # Lo [4] TIBETAN SIGN LCE TSA CAN..TIBETAN SIGN GRU MED RGYINGS +0F90..0F97 ; XID_Continue # Mn [8] TIBETAN SUBJOINED LETTER KA..TIBETAN SUBJOINED LETTER JA +0F99..0FBC ; XID_Continue # Mn [36] TIBETAN SUBJOINED LETTER NYA..TIBETAN SUBJOINED LETTER FIXED-FORM RA +0FC6 ; XID_Continue # Mn TIBETAN SYMBOL PADMA GDAN +1000..102A ; XID_Continue # Lo [43] MYANMAR LETTER KA..MYANMAR LETTER AU +102B..102C ; XID_Continue # Mc [2] MYANMAR VOWEL SIGN TALL AA..MYANMAR VOWEL SIGN AA +102D..1030 ; XID_Continue # Mn [4] MYANMAR VOWEL SIGN I..MYANMAR VOWEL SIGN UU +1031 ; XID_Continue # Mc MYANMAR VOWEL SIGN E +1032..1037 ; XID_Continue # Mn [6] MYANMAR VOWEL SIGN AI..MYANMAR SIGN DOT BELOW +1038 ; XID_Continue # Mc MYANMAR SIGN VISARGA +1039..103A ; XID_Continue # Mn [2] MYANMAR SIGN VIRAMA..MYANMAR SIGN ASAT +103B..103C ; XID_Continue # Mc [2] MYANMAR CONSONANT SIGN MEDIAL YA..MYANMAR CONSONANT SIGN MEDIAL RA +103D..103E ; XID_Continue # Mn [2] MYANMAR CONSONANT SIGN MEDIAL WA..MYANMAR CONSONANT SIGN MEDIAL HA +103F ; XID_Continue # Lo MYANMAR LETTER GREAT SA +1040..1049 ; XID_Continue # Nd [10] MYANMAR DIGIT ZERO..MYANMAR DIGIT NINE +1050..1055 ; XID_Continue # Lo [6] MYANMAR LETTER SHA..MYANMAR LETTER VOCALIC LL +1056..1057 ; XID_Continue # Mc [2] MYANMAR VOWEL SIGN VOCALIC R..MYANMAR VOWEL SIGN VOCALIC RR +1058..1059 ; XID_Continue # Mn [2] MYANMAR VOWEL SIGN VOCALIC L..MYANMAR VOWEL SIGN VOCALIC LL +105A..105D ; XID_Continue # Lo [4] MYANMAR LETTER MON NGA..MYANMAR LETTER MON BBE +105E..1060 ; XID_Continue # Mn [3] MYANMAR CONSONANT SIGN MON MEDIAL NA..MYANMAR CONSONANT SIGN MON MEDIAL LA +1061 ; XID_Continue # Lo MYANMAR LETTER SGAW KAREN SHA +1062..1064 ; XID_Continue # Mc [3] MYANMAR VOWEL SIGN SGAW KAREN EU..MYANMAR TONE MARK SGAW KAREN KE PHO +1065..1066 ; XID_Continue # Lo [2] MYANMAR LETTER WESTERN PWO KAREN THA..MYANMAR LETTER WESTERN PWO KAREN PWA +1067..106D ; XID_Continue # Mc [7] MYANMAR VOWEL SIGN WESTERN PWO KAREN EU..MYANMAR SIGN WESTERN PWO KAREN TONE-5 +106E..1070 ; XID_Continue # Lo [3] MYANMAR LETTER EASTERN PWO KAREN NNA..MYANMAR LETTER EASTERN PWO KAREN GHWA +1071..1074 ; XID_Continue # Mn [4] MYANMAR VOWEL SIGN GEBA KAREN I..MYANMAR VOWEL SIGN KAYAH EE +1075..1081 ; XID_Continue # Lo [13] MYANMAR LETTER SHAN KA..MYANMAR LETTER SHAN HA +1082 ; XID_Continue # Mn MYANMAR CONSONANT SIGN SHAN MEDIAL WA +1083..1084 ; XID_Continue # Mc [2] MYANMAR VOWEL SIGN SHAN AA..MYANMAR VOWEL SIGN SHAN E +1085..1086 ; XID_Continue # Mn [2] MYANMAR VOWEL SIGN SHAN E ABOVE..MYANMAR VOWEL SIGN SHAN FINAL Y +1087..108C ; XID_Continue # Mc [6] MYANMAR SIGN SHAN TONE-2..MYANMAR SIGN SHAN COUNCIL TONE-3 +108D ; XID_Continue # Mn MYANMAR SIGN SHAN COUNCIL EMPHATIC TONE +108E ; XID_Continue # Lo MYANMAR LETTER RUMAI PALAUNG FA +108F ; XID_Continue # Mc MYANMAR SIGN RUMAI PALAUNG TONE-5 +1090..1099 ; XID_Continue # Nd [10] MYANMAR SHAN DIGIT ZERO..MYANMAR SHAN DIGIT NINE +109A..109C ; XID_Continue # Mc [3] MYANMAR SIGN KHAMTI TONE-1..MYANMAR VOWEL SIGN AITON A +109D ; XID_Continue # Mn MYANMAR VOWEL SIGN AITON AI +10A0..10C5 ; XID_Continue # L& [38] GEORGIAN CAPITAL LETTER AN..GEORGIAN CAPITAL LETTER HOE +10D0..10FA ; XID_Continue # Lo [43] GEORGIAN LETTER AN..GEORGIAN LETTER AIN +10FC ; XID_Continue # Lm MODIFIER LETTER GEORGIAN NAR +1100..1248 ; XID_Continue # Lo [329] HANGUL CHOSEONG KIYEOK..ETHIOPIC SYLLABLE QWA +124A..124D ; XID_Continue # Lo [4] ETHIOPIC SYLLABLE QWI..ETHIOPIC SYLLABLE QWE +1250..1256 ; XID_Continue # Lo [7] ETHIOPIC SYLLABLE QHA..ETHIOPIC SYLLABLE QHO +1258 ; XID_Continue # Lo ETHIOPIC SYLLABLE QHWA +125A..125D ; XID_Continue # Lo [4] ETHIOPIC SYLLABLE QHWI..ETHIOPIC SYLLABLE QHWE +1260..1288 ; XID_Continue # Lo [41] ETHIOPIC SYLLABLE BA..ETHIOPIC SYLLABLE XWA +128A..128D ; XID_Continue # Lo [4] ETHIOPIC SYLLABLE XWI..ETHIOPIC SYLLABLE XWE +1290..12B0 ; XID_Continue # Lo [33] ETHIOPIC SYLLABLE NA..ETHIOPIC SYLLABLE KWA +12B2..12B5 ; XID_Continue # Lo [4] ETHIOPIC SYLLABLE KWI..ETHIOPIC SYLLABLE KWE +12B8..12BE ; XID_Continue # Lo [7] ETHIOPIC SYLLABLE KXA..ETHIOPIC SYLLABLE KXO +12C0 ; XID_Continue # Lo ETHIOPIC SYLLABLE KXWA +12C2..12C5 ; XID_Continue # Lo [4] ETHIOPIC SYLLABLE KXWI..ETHIOPIC SYLLABLE KXWE +12C8..12D6 ; XID_Continue # Lo [15] ETHIOPIC SYLLABLE WA..ETHIOPIC SYLLABLE PHARYNGEAL O +12D8..1310 ; XID_Continue # Lo [57] ETHIOPIC SYLLABLE ZA..ETHIOPIC SYLLABLE GWA +1312..1315 ; XID_Continue # Lo [4] ETHIOPIC SYLLABLE GWI..ETHIOPIC SYLLABLE GWE +1318..135A ; XID_Continue # Lo [67] ETHIOPIC SYLLABLE GGA..ETHIOPIC SYLLABLE FYA +135F ; XID_Continue # Mn ETHIOPIC COMBINING GEMINATION MARK +1369..1371 ; XID_Continue # No [9] ETHIOPIC DIGIT ONE..ETHIOPIC DIGIT NINE +1380..138F ; XID_Continue # Lo [16] ETHIOPIC SYLLABLE SEBATBEIT MWA..ETHIOPIC SYLLABLE PWE +13A0..13F4 ; XID_Continue # Lo [85] CHEROKEE LETTER A..CHEROKEE LETTER YV +1401..166C ; XID_Continue # Lo [620] CANADIAN SYLLABICS E..CANADIAN SYLLABICS CARRIER TTSA +166F..167F ; XID_Continue # Lo [17] CANADIAN SYLLABICS QAI..CANADIAN SYLLABICS BLACKFOOT W +1681..169A ; XID_Continue # Lo [26] OGHAM LETTER BEITH..OGHAM LETTER PEITH +16A0..16EA ; XID_Continue # Lo [75] RUNIC LETTER FEHU FEOH FE F..RUNIC LETTER X +16EE..16F0 ; XID_Continue # Nl [3] RUNIC ARLAUG SYMBOL..RUNIC BELGTHOR SYMBOL +1700..170C ; XID_Continue # Lo [13] TAGALOG LETTER A..TAGALOG LETTER YA +170E..1711 ; XID_Continue # Lo [4] TAGALOG LETTER LA..TAGALOG LETTER HA +1712..1714 ; XID_Continue # Mn [3] TAGALOG VOWEL SIGN I..TAGALOG SIGN VIRAMA +1720..1731 ; XID_Continue # Lo [18] HANUNOO LETTER A..HANUNOO LETTER HA +1732..1734 ; XID_Continue # Mn [3] HANUNOO VOWEL SIGN I..HANUNOO SIGN PAMUDPOD +1740..1751 ; XID_Continue # Lo [18] BUHID LETTER A..BUHID LETTER HA +1752..1753 ; XID_Continue # Mn [2] BUHID VOWEL SIGN I..BUHID VOWEL SIGN U +1760..176C ; XID_Continue # Lo [13] TAGBANWA LETTER A..TAGBANWA LETTER YA +176E..1770 ; XID_Continue # Lo [3] TAGBANWA LETTER LA..TAGBANWA LETTER SA +1772..1773 ; XID_Continue # Mn [2] TAGBANWA VOWEL SIGN I..TAGBANWA VOWEL SIGN U +1780..17B3 ; XID_Continue # Lo [52] KHMER LETTER KA..KHMER INDEPENDENT VOWEL QAU +17B6 ; XID_Continue # Mc KHMER VOWEL SIGN AA +17B7..17BD ; XID_Continue # Mn [7] KHMER VOWEL SIGN I..KHMER VOWEL SIGN UA +17BE..17C5 ; XID_Continue # Mc [8] KHMER VOWEL SIGN OE..KHMER VOWEL SIGN AU +17C6 ; XID_Continue # Mn KHMER SIGN NIKAHIT +17C7..17C8 ; XID_Continue # Mc [2] KHMER SIGN REAHMUK..KHMER SIGN YUUKALEAPINTU +17C9..17D3 ; XID_Continue # Mn [11] KHMER SIGN MUUSIKATOAN..KHMER SIGN BATHAMASAT +17D7 ; XID_Continue # Lm KHMER SIGN LEK TOO +17DC ; XID_Continue # Lo KHMER SIGN AVAKRAHASANYA +17DD ; XID_Continue # Mn KHMER SIGN ATTHACAN +17E0..17E9 ; XID_Continue # Nd [10] KHMER DIGIT ZERO..KHMER DIGIT NINE +180B..180D ; XID_Continue # Mn [3] MONGOLIAN FREE VARIATION SELECTOR ONE..MONGOLIAN FREE VARIATION SELECTOR THREE +1810..1819 ; XID_Continue # Nd [10] MONGOLIAN DIGIT ZERO..MONGOLIAN DIGIT NINE +1820..1842 ; XID_Continue # Lo [35] MONGOLIAN LETTER A..MONGOLIAN LETTER CHI +1843 ; XID_Continue # Lm MONGOLIAN LETTER TODO LONG VOWEL SIGN +1844..1877 ; XID_Continue # Lo [52] MONGOLIAN LETTER TODO E..MONGOLIAN LETTER MANCHU ZHA +1880..18A8 ; XID_Continue # Lo [41] MONGOLIAN LETTER ALI GALI ANUSVARA ONE..MONGOLIAN LETTER MANCHU ALI GALI BHA +18A9 ; XID_Continue # Mn MONGOLIAN LETTER ALI GALI DAGALGA +18AA ; XID_Continue # Lo MONGOLIAN LETTER MANCHU ALI GALI LHA +18B0..18F5 ; XID_Continue # Lo [70] CANADIAN SYLLABICS OY..CANADIAN SYLLABICS CARRIER DENTAL S +1900..191C ; XID_Continue # Lo [29] LIMBU VOWEL-CARRIER LETTER..LIMBU LETTER HA +1920..1922 ; XID_Continue # Mn [3] LIMBU VOWEL SIGN A..LIMBU VOWEL SIGN U +1923..1926 ; XID_Continue # Mc [4] LIMBU VOWEL SIGN EE..LIMBU VOWEL SIGN AU +1927..1928 ; XID_Continue # Mn [2] LIMBU VOWEL SIGN E..LIMBU VOWEL SIGN O +1929..192B ; XID_Continue # Mc [3] LIMBU SUBJOINED LETTER YA..LIMBU SUBJOINED LETTER WA +1930..1931 ; XID_Continue # Mc [2] LIMBU SMALL LETTER KA..LIMBU SMALL LETTER NGA +1932 ; XID_Continue # Mn LIMBU SMALL LETTER ANUSVARA +1933..1938 ; XID_Continue # Mc [6] LIMBU SMALL LETTER TA..LIMBU SMALL LETTER LA +1939..193B ; XID_Continue # Mn [3] LIMBU SIGN MUKPHRENG..LIMBU SIGN SA-I +1946..194F ; XID_Continue # Nd [10] LIMBU DIGIT ZERO..LIMBU DIGIT NINE +1950..196D ; XID_Continue # Lo [30] TAI LE LETTER KA..TAI LE LETTER AI +1970..1974 ; XID_Continue # Lo [5] TAI LE LETTER TONE-2..TAI LE LETTER TONE-6 +1980..19AB ; XID_Continue # Lo [44] NEW TAI LUE LETTER HIGH QA..NEW TAI LUE LETTER LOW SUA +19B0..19C0 ; XID_Continue # Mc [17] NEW TAI LUE VOWEL SIGN VOWEL SHORTENER..NEW TAI LUE VOWEL SIGN IY +19C1..19C7 ; XID_Continue # Lo [7] NEW TAI LUE LETTER FINAL V..NEW TAI LUE LETTER FINAL B +19C8..19C9 ; XID_Continue # Mc [2] NEW TAI LUE TONE MARK-1..NEW TAI LUE TONE MARK-2 +19D0..19DA ; XID_Continue # Nd [11] NEW TAI LUE DIGIT ZERO..NEW TAI LUE THAM DIGIT ONE +1A00..1A16 ; XID_Continue # Lo [23] BUGINESE LETTER KA..BUGINESE LETTER HA +1A17..1A18 ; XID_Continue # Mn [2] BUGINESE VOWEL SIGN I..BUGINESE VOWEL SIGN U +1A19..1A1B ; XID_Continue # Mc [3] BUGINESE VOWEL SIGN E..BUGINESE VOWEL SIGN AE +1A20..1A54 ; XID_Continue # Lo [53] TAI THAM LETTER HIGH KA..TAI THAM LETTER GREAT SA +1A55 ; XID_Continue # Mc TAI THAM CONSONANT SIGN MEDIAL RA +1A56 ; XID_Continue # Mn TAI THAM CONSONANT SIGN MEDIAL LA +1A57 ; XID_Continue # Mc TAI THAM CONSONANT SIGN LA TANG LAI +1A58..1A5E ; XID_Continue # Mn [7] TAI THAM SIGN MAI KANG LAI..TAI THAM CONSONANT SIGN SA +1A60 ; XID_Continue # Mn TAI THAM SIGN SAKOT +1A61 ; XID_Continue # Mc TAI THAM VOWEL SIGN A +1A62 ; XID_Continue # Mn TAI THAM VOWEL SIGN MAI SAT +1A63..1A64 ; XID_Continue # Mc [2] TAI THAM VOWEL SIGN AA..TAI THAM VOWEL SIGN TALL AA +1A65..1A6C ; XID_Continue # Mn [8] TAI THAM VOWEL SIGN I..TAI THAM VOWEL SIGN OA BELOW +1A6D..1A72 ; XID_Continue # Mc [6] TAI THAM VOWEL SIGN OY..TAI THAM VOWEL SIGN THAM AI +1A73..1A7C ; XID_Continue # Mn [10] TAI THAM VOWEL SIGN OA ABOVE..TAI THAM SIGN KHUEN-LUE KARAN +1A7F ; XID_Continue # Mn TAI THAM COMBINING CRYPTOGRAMMIC DOT +1A80..1A89 ; XID_Continue # Nd [10] TAI THAM HORA DIGIT ZERO..TAI THAM HORA DIGIT NINE +1A90..1A99 ; XID_Continue # Nd [10] TAI THAM THAM DIGIT ZERO..TAI THAM THAM DIGIT NINE +1AA7 ; XID_Continue # Lm TAI THAM SIGN MAI YAMOK +1B00..1B03 ; XID_Continue # Mn [4] BALINESE SIGN ULU RICEM..BALINESE SIGN SURANG +1B04 ; XID_Continue # Mc BALINESE SIGN BISAH +1B05..1B33 ; XID_Continue # Lo [47] BALINESE LETTER AKARA..BALINESE LETTER HA +1B34 ; XID_Continue # Mn BALINESE SIGN REREKAN +1B35 ; XID_Continue # Mc BALINESE VOWEL SIGN TEDUNG +1B36..1B3A ; XID_Continue # Mn [5] BALINESE VOWEL SIGN ULU..BALINESE VOWEL SIGN RA REPA +1B3B ; XID_Continue # Mc BALINESE VOWEL SIGN RA REPA TEDUNG +1B3C ; XID_Continue # Mn BALINESE VOWEL SIGN LA LENGA +1B3D..1B41 ; XID_Continue # Mc [5] BALINESE VOWEL SIGN LA LENGA TEDUNG..BALINESE VOWEL SIGN TALING REPA TEDUNG +1B42 ; XID_Continue # Mn BALINESE VOWEL SIGN PEPET +1B43..1B44 ; XID_Continue # Mc [2] BALINESE VOWEL SIGN PEPET TEDUNG..BALINESE ADEG ADEG +1B45..1B4B ; XID_Continue # Lo [7] BALINESE LETTER KAF SASAK..BALINESE LETTER ASYURA SASAK +1B50..1B59 ; XID_Continue # Nd [10] BALINESE DIGIT ZERO..BALINESE DIGIT NINE +1B6B..1B73 ; XID_Continue # Mn [9] BALINESE MUSICAL SYMBOL COMBINING TEGEH..BALINESE MUSICAL SYMBOL COMBINING GONG +1B80..1B81 ; XID_Continue # Mn [2] SUNDANESE SIGN PANYECEK..SUNDANESE SIGN PANGLAYAR +1B82 ; XID_Continue # Mc SUNDANESE SIGN PANGWISAD +1B83..1BA0 ; XID_Continue # Lo [30] SUNDANESE LETTER A..SUNDANESE LETTER HA +1BA1 ; XID_Continue # Mc SUNDANESE CONSONANT SIGN PAMINGKAL +1BA2..1BA5 ; XID_Continue # Mn [4] SUNDANESE CONSONANT SIGN PANYAKRA..SUNDANESE VOWEL SIGN PANYUKU +1BA6..1BA7 ; XID_Continue # Mc [2] SUNDANESE VOWEL SIGN PANAELAENG..SUNDANESE VOWEL SIGN PANOLONG +1BA8..1BA9 ; XID_Continue # Mn [2] SUNDANESE VOWEL SIGN PAMEPET..SUNDANESE VOWEL SIGN PANEULEUNG +1BAA ; XID_Continue # Mc SUNDANESE SIGN PAMAAEH +1BAE..1BAF ; XID_Continue # Lo [2] SUNDANESE LETTER KHA..SUNDANESE LETTER SYA +1BB0..1BB9 ; XID_Continue # Nd [10] SUNDANESE DIGIT ZERO..SUNDANESE DIGIT NINE +1C00..1C23 ; XID_Continue # Lo [36] LEPCHA LETTER KA..LEPCHA LETTER A +1C24..1C2B ; XID_Continue # Mc [8] LEPCHA SUBJOINED LETTER YA..LEPCHA VOWEL SIGN UU +1C2C..1C33 ; XID_Continue # Mn [8] LEPCHA VOWEL SIGN E..LEPCHA CONSONANT SIGN T +1C34..1C35 ; XID_Continue # Mc [2] LEPCHA CONSONANT SIGN NYIN-DO..LEPCHA CONSONANT SIGN KANG +1C36..1C37 ; XID_Continue # Mn [2] LEPCHA SIGN RAN..LEPCHA SIGN NUKTA +1C40..1C49 ; XID_Continue # Nd [10] LEPCHA DIGIT ZERO..LEPCHA DIGIT NINE +1C4D..1C4F ; XID_Continue # Lo [3] LEPCHA LETTER TTA..LEPCHA LETTER DDA +1C50..1C59 ; XID_Continue # Nd [10] OL CHIKI DIGIT ZERO..OL CHIKI DIGIT NINE +1C5A..1C77 ; XID_Continue # Lo [30] OL CHIKI LETTER LA..OL CHIKI LETTER OH +1C78..1C7D ; XID_Continue # Lm [6] OL CHIKI MU TTUDDAG..OL CHIKI AHAD +1CD0..1CD2 ; XID_Continue # Mn [3] VEDIC TONE KARSHANA..VEDIC TONE PRENKHA +1CD4..1CE0 ; XID_Continue # Mn [13] VEDIC SIGN YAJURVEDIC MIDLINE SVARITA..VEDIC TONE RIGVEDIC KASHMIRI INDEPENDENT SVARITA +1CE1 ; XID_Continue # Mc VEDIC TONE ATHARVAVEDIC INDEPENDENT SVARITA +1CE2..1CE8 ; XID_Continue # Mn [7] VEDIC SIGN VISARGA SVARITA..VEDIC SIGN VISARGA ANUDATTA WITH TAIL +1CE9..1CEC ; XID_Continue # Lo [4] VEDIC SIGN ANUSVARA ANTARGOMUKHA..VEDIC SIGN ANUSVARA VAMAGOMUKHA WITH TAIL +1CED ; XID_Continue # Mn VEDIC SIGN TIRYAK +1CEE..1CF1 ; XID_Continue # Lo [4] VEDIC SIGN HEXIFORM LONG ANUSVARA..VEDIC SIGN ANUSVARA UBHAYATO MUKHA +1CF2 ; XID_Continue # Mc VEDIC SIGN ARDHAVISARGA +1D00..1D2B ; XID_Continue # L& [44] LATIN LETTER SMALL CAPITAL A..CYRILLIC LETTER SMALL CAPITAL EL +1D2C..1D61 ; XID_Continue # Lm [54] MODIFIER LETTER CAPITAL A..MODIFIER LETTER SMALL CHI +1D62..1D77 ; XID_Continue # L& [22] LATIN SUBSCRIPT SMALL LETTER I..LATIN SMALL LETTER TURNED G +1D78 ; XID_Continue # Lm MODIFIER LETTER CYRILLIC EN +1D79..1D9A ; XID_Continue # L& [34] LATIN SMALL LETTER INSULAR G..LATIN SMALL LETTER EZH WITH RETROFLEX HOOK +1D9B..1DBF ; XID_Continue # Lm [37] MODIFIER LETTER SMALL TURNED ALPHA..MODIFIER LETTER SMALL THETA +1DC0..1DE6 ; XID_Continue # Mn [39] COMBINING DOTTED GRAVE ACCENT..COMBINING LATIN SMALL LETTER Z +1DFD..1DFF ; XID_Continue # Mn [3] COMBINING ALMOST EQUAL TO BELOW..COMBINING RIGHT ARROWHEAD AND DOWN ARROWHEAD BELOW +1E00..1F15 ; XID_Continue # L& [278] LATIN CAPITAL LETTER A WITH RING BELOW..GREEK SMALL LETTER EPSILON WITH DASIA AND OXIA +1F18..1F1D ; XID_Continue # L& [6] GREEK CAPITAL LETTER EPSILON WITH PSILI..GREEK CAPITAL LETTER EPSILON WITH DASIA AND OXIA +1F20..1F45 ; XID_Continue # L& [38] GREEK SMALL LETTER ETA WITH PSILI..GREEK SMALL LETTER OMICRON WITH DASIA AND OXIA +1F48..1F4D ; XID_Continue # L& [6] GREEK CAPITAL LETTER OMICRON WITH PSILI..GREEK CAPITAL LETTER OMICRON WITH DASIA AND OXIA +1F50..1F57 ; XID_Continue # L& [8] GREEK SMALL LETTER UPSILON WITH PSILI..GREEK SMALL LETTER UPSILON WITH DASIA AND PERISPOMENI +1F59 ; XID_Continue # L& GREEK CAPITAL LETTER UPSILON WITH DASIA +1F5B ; XID_Continue # L& GREEK CAPITAL LETTER UPSILON WITH DASIA AND VARIA +1F5D ; XID_Continue # L& GREEK CAPITAL LETTER UPSILON WITH DASIA AND OXIA +1F5F..1F7D ; XID_Continue # L& [31] GREEK CAPITAL LETTER UPSILON WITH DASIA AND PERISPOMENI..GREEK SMALL LETTER OMEGA WITH OXIA +1F80..1FB4 ; XID_Continue # L& [53] GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI..GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI +1FB6..1FBC ; XID_Continue # L& [7] GREEK SMALL LETTER ALPHA WITH PERISPOMENI..GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI +1FBE ; XID_Continue # L& GREEK PROSGEGRAMMENI +1FC2..1FC4 ; XID_Continue # L& [3] GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI..GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI +1FC6..1FCC ; XID_Continue # L& [7] GREEK SMALL LETTER ETA WITH PERISPOMENI..GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI +1FD0..1FD3 ; XID_Continue # L& [4] GREEK SMALL LETTER IOTA WITH VRACHY..GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA +1FD6..1FDB ; XID_Continue # L& [6] GREEK SMALL LETTER IOTA WITH PERISPOMENI..GREEK CAPITAL LETTER IOTA WITH OXIA +1FE0..1FEC ; XID_Continue # L& [13] GREEK SMALL LETTER UPSILON WITH VRACHY..GREEK CAPITAL LETTER RHO WITH DASIA +1FF2..1FF4 ; XID_Continue # L& [3] GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI..GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI +1FF6..1FFC ; XID_Continue # L& [7] GREEK SMALL LETTER OMEGA WITH PERISPOMENI..GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI +203F..2040 ; XID_Continue # Pc [2] UNDERTIE..CHARACTER TIE +2054 ; XID_Continue # Pc INVERTED UNDERTIE +2071 ; XID_Continue # Lm SUPERSCRIPT LATIN SMALL LETTER I +207F ; XID_Continue # Lm SUPERSCRIPT LATIN SMALL LETTER N +2090..2094 ; XID_Continue # Lm [5] LATIN SUBSCRIPT SMALL LETTER A..LATIN SUBSCRIPT SMALL LETTER SCHWA +20D0..20DC ; XID_Continue # Mn [13] COMBINING LEFT HARPOON ABOVE..COMBINING FOUR DOTS ABOVE +20E1 ; XID_Continue # Mn COMBINING LEFT RIGHT ARROW ABOVE +20E5..20F0 ; XID_Continue # Mn [12] COMBINING REVERSE SOLIDUS OVERLAY..COMBINING ASTERISK ABOVE +2102 ; XID_Continue # L& DOUBLE-STRUCK CAPITAL C +2107 ; XID_Continue # L& EULER CONSTANT +210A..2113 ; XID_Continue # L& [10] SCRIPT SMALL G..SCRIPT SMALL L +2115 ; XID_Continue # L& DOUBLE-STRUCK CAPITAL N +2118 ; XID_Continue # So SCRIPT CAPITAL P +2119..211D ; XID_Continue # L& [5] DOUBLE-STRUCK CAPITAL P..DOUBLE-STRUCK CAPITAL R +2124 ; XID_Continue # L& DOUBLE-STRUCK CAPITAL Z +2126 ; XID_Continue # L& OHM SIGN +2128 ; XID_Continue # L& BLACK-LETTER CAPITAL Z +212A..212D ; XID_Continue # L& [4] KELVIN SIGN..BLACK-LETTER CAPITAL C +212E ; XID_Continue # So ESTIMATED SYMBOL +212F..2134 ; XID_Continue # L& [6] SCRIPT SMALL E..SCRIPT SMALL O +2135..2138 ; XID_Continue # Lo [4] ALEF SYMBOL..DALET SYMBOL +2139 ; XID_Continue # L& INFORMATION SOURCE +213C..213F ; XID_Continue # L& [4] DOUBLE-STRUCK SMALL PI..DOUBLE-STRUCK CAPITAL PI +2145..2149 ; XID_Continue # L& [5] DOUBLE-STRUCK ITALIC CAPITAL D..DOUBLE-STRUCK ITALIC SMALL J +214E ; XID_Continue # L& TURNED SMALL F +2160..2182 ; XID_Continue # Nl [35] ROMAN NUMERAL ONE..ROMAN NUMERAL TEN THOUSAND +2183..2184 ; XID_Continue # L& [2] ROMAN NUMERAL REVERSED ONE HUNDRED..LATIN SMALL LETTER REVERSED C +2185..2188 ; XID_Continue # Nl [4] ROMAN NUMERAL SIX LATE FORM..ROMAN NUMERAL ONE HUNDRED THOUSAND +2C00..2C2E ; XID_Continue # L& [47] GLAGOLITIC CAPITAL LETTER AZU..GLAGOLITIC CAPITAL LETTER LATINATE MYSLITE +2C30..2C5E ; XID_Continue # L& [47] GLAGOLITIC SMALL LETTER AZU..GLAGOLITIC SMALL LETTER LATINATE MYSLITE +2C60..2C7C ; XID_Continue # L& [29] LATIN CAPITAL LETTER L WITH DOUBLE BAR..LATIN SUBSCRIPT SMALL LETTER J +2C7D ; XID_Continue # Lm MODIFIER LETTER CAPITAL V +2C7E..2CE4 ; XID_Continue # L& [103] LATIN CAPITAL LETTER S WITH SWASH TAIL..COPTIC SYMBOL KAI +2CEB..2CEE ; XID_Continue # L& [4] COPTIC CAPITAL LETTER CRYPTOGRAMMIC SHEI..COPTIC SMALL LETTER CRYPTOGRAMMIC GANGIA +2CEF..2CF1 ; XID_Continue # Mn [3] COPTIC COMBINING NI ABOVE..COPTIC COMBINING SPIRITUS LENIS +2D00..2D25 ; XID_Continue # L& [38] GEORGIAN SMALL LETTER AN..GEORGIAN SMALL LETTER HOE +2D30..2D65 ; XID_Continue # Lo [54] TIFINAGH LETTER YA..TIFINAGH LETTER YAZZ +2D6F ; XID_Continue # Lm TIFINAGH MODIFIER LETTER LABIALIZATION MARK +2D80..2D96 ; XID_Continue # Lo [23] ETHIOPIC SYLLABLE LOA..ETHIOPIC SYLLABLE GGWE +2DA0..2DA6 ; XID_Continue # Lo [7] ETHIOPIC SYLLABLE SSA..ETHIOPIC SYLLABLE SSO +2DA8..2DAE ; XID_Continue # Lo [7] ETHIOPIC SYLLABLE CCA..ETHIOPIC SYLLABLE CCO +2DB0..2DB6 ; XID_Continue # Lo [7] ETHIOPIC SYLLABLE ZZA..ETHIOPIC SYLLABLE ZZO +2DB8..2DBE ; XID_Continue # Lo [7] ETHIOPIC SYLLABLE CCHA..ETHIOPIC SYLLABLE CCHO +2DC0..2DC6 ; XID_Continue # Lo [7] ETHIOPIC SYLLABLE QYA..ETHIOPIC SYLLABLE QYO +2DC8..2DCE ; XID_Continue # Lo [7] ETHIOPIC SYLLABLE KYA..ETHIOPIC SYLLABLE KYO +2DD0..2DD6 ; XID_Continue # Lo [7] ETHIOPIC SYLLABLE XYA..ETHIOPIC SYLLABLE XYO +2DD8..2DDE ; XID_Continue # Lo [7] ETHIOPIC SYLLABLE GYA..ETHIOPIC SYLLABLE GYO +2DE0..2DFF ; XID_Continue # Mn [32] COMBINING CYRILLIC LETTER BE..COMBINING CYRILLIC LETTER IOTIFIED BIG YUS +3005 ; XID_Continue # Lm IDEOGRAPHIC ITERATION MARK +3006 ; XID_Continue # Lo IDEOGRAPHIC CLOSING MARK +3007 ; XID_Continue # Nl IDEOGRAPHIC NUMBER ZERO +3021..3029 ; XID_Continue # Nl [9] HANGZHOU NUMERAL ONE..HANGZHOU NUMERAL NINE +302A..302F ; XID_Continue # Mn [6] IDEOGRAPHIC LEVEL TONE MARK..HANGUL DOUBLE DOT TONE MARK +3031..3035 ; XID_Continue # Lm [5] VERTICAL KANA REPEAT MARK..VERTICAL KANA REPEAT MARK LOWER HALF +3038..303A ; XID_Continue # Nl [3] HANGZHOU NUMERAL TEN..HANGZHOU NUMERAL THIRTY +303B ; XID_Continue # Lm VERTICAL IDEOGRAPHIC ITERATION MARK +303C ; XID_Continue # Lo MASU MARK +3041..3096 ; XID_Continue # Lo [86] HIRAGANA LETTER SMALL A..HIRAGANA LETTER SMALL KE +3099..309A ; XID_Continue # Mn [2] COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK..COMBINING KATAKANA-HIRAGANA SEMI-VOICED SOUND MARK +309D..309E ; XID_Continue # Lm [2] HIRAGANA ITERATION MARK..HIRAGANA VOICED ITERATION MARK +309F ; XID_Continue # Lo HIRAGANA DIGRAPH YORI +30A1..30FA ; XID_Continue # Lo [90] KATAKANA LETTER SMALL A..KATAKANA LETTER VO +30FC..30FE ; XID_Continue # Lm [3] KATAKANA-HIRAGANA PROLONGED SOUND MARK..KATAKANA VOICED ITERATION MARK +30FF ; XID_Continue # Lo KATAKANA DIGRAPH KOTO +3105..312D ; XID_Continue # Lo [41] BOPOMOFO LETTER B..BOPOMOFO LETTER IH +3131..318E ; XID_Continue # Lo [94] HANGUL LETTER KIYEOK..HANGUL LETTER ARAEAE +31A0..31B7 ; XID_Continue # Lo [24] BOPOMOFO LETTER BU..BOPOMOFO FINAL LETTER H +31F0..31FF ; XID_Continue # Lo [16] KATAKANA LETTER SMALL KU..KATAKANA LETTER SMALL RO +3400..4DB5 ; XID_Continue # Lo [6582] CJK UNIFIED IDEOGRAPH-3400..CJK UNIFIED IDEOGRAPH-4DB5 +4E00..9FCB ; XID_Continue # Lo [20940] CJK UNIFIED IDEOGRAPH-4E00..CJK UNIFIED IDEOGRAPH-9FCB +A000..A014 ; XID_Continue # Lo [21] YI SYLLABLE IT..YI SYLLABLE E +A015 ; XID_Continue # Lm YI SYLLABLE WU +A016..A48C ; XID_Continue # Lo [1143] YI SYLLABLE BIT..YI SYLLABLE YYR +A4D0..A4F7 ; XID_Continue # Lo [40] LISU LETTER BA..LISU LETTER OE +A4F8..A4FD ; XID_Continue # Lm [6] LISU LETTER TONE MYA TI..LISU LETTER TONE MYA JEU +A500..A60B ; XID_Continue # Lo [268] VAI SYLLABLE EE..VAI SYLLABLE NG +A60C ; XID_Continue # Lm VAI SYLLABLE LENGTHENER +A610..A61F ; XID_Continue # Lo [16] VAI SYLLABLE NDOLE FA..VAI SYMBOL JONG +A620..A629 ; XID_Continue # Nd [10] VAI DIGIT ZERO..VAI DIGIT NINE +A62A..A62B ; XID_Continue # Lo [2] VAI SYLLABLE NDOLE MA..VAI SYLLABLE NDOLE DO +A640..A65F ; XID_Continue # L& [32] CYRILLIC CAPITAL LETTER ZEMLYA..CYRILLIC SMALL LETTER YN +A662..A66D ; XID_Continue # L& [12] CYRILLIC CAPITAL LETTER SOFT DE..CYRILLIC SMALL LETTER DOUBLE MONOCULAR O +A66E ; XID_Continue # Lo CYRILLIC LETTER MULTIOCULAR O +A66F ; XID_Continue # Mn COMBINING CYRILLIC VZMET +A67C..A67D ; XID_Continue # Mn [2] COMBINING CYRILLIC KAVYKA..COMBINING CYRILLIC PAYEROK +A67F ; XID_Continue # Lm CYRILLIC PAYEROK +A680..A697 ; XID_Continue # L& [24] CYRILLIC CAPITAL LETTER DWE..CYRILLIC SMALL LETTER SHWE +A6A0..A6E5 ; XID_Continue # Lo [70] BAMUM LETTER A..BAMUM LETTER KI +A6E6..A6EF ; XID_Continue # Nl [10] BAMUM LETTER MO..BAMUM LETTER KOGHOM +A6F0..A6F1 ; XID_Continue # Mn [2] BAMUM COMBINING MARK KOQNDON..BAMUM COMBINING MARK TUKWENTIS +A717..A71F ; XID_Continue # Lm [9] MODIFIER LETTER DOT VERTICAL BAR..MODIFIER LETTER LOW INVERTED EXCLAMATION MARK +A722..A76F ; XID_Continue # L& [78] LATIN CAPITAL LETTER EGYPTOLOGICAL ALEF..LATIN SMALL LETTER CON +A770 ; XID_Continue # Lm MODIFIER LETTER US +A771..A787 ; XID_Continue # L& [23] LATIN SMALL LETTER DUM..LATIN SMALL LETTER INSULAR T +A788 ; XID_Continue # Lm MODIFIER LETTER LOW CIRCUMFLEX ACCENT +A78B..A78C ; XID_Continue # L& [2] LATIN CAPITAL LETTER SALTILLO..LATIN SMALL LETTER SALTILLO +A7FB..A801 ; XID_Continue # Lo [7] LATIN EPIGRAPHIC LETTER REVERSED F..SYLOTI NAGRI LETTER I +A802 ; XID_Continue # Mn SYLOTI NAGRI SIGN DVISVARA +A803..A805 ; XID_Continue # Lo [3] SYLOTI NAGRI LETTER U..SYLOTI NAGRI LETTER O +A806 ; XID_Continue # Mn SYLOTI NAGRI SIGN HASANTA +A807..A80A ; XID_Continue # Lo [4] SYLOTI NAGRI LETTER KO..SYLOTI NAGRI LETTER GHO +A80B ; XID_Continue # Mn SYLOTI NAGRI SIGN ANUSVARA +A80C..A822 ; XID_Continue # Lo [23] SYLOTI NAGRI LETTER CO..SYLOTI NAGRI LETTER HO +A823..A824 ; XID_Continue # Mc [2] SYLOTI NAGRI VOWEL SIGN A..SYLOTI NAGRI VOWEL SIGN I +A825..A826 ; XID_Continue # Mn [2] SYLOTI NAGRI VOWEL SIGN U..SYLOTI NAGRI VOWEL SIGN E +A827 ; XID_Continue # Mc SYLOTI NAGRI VOWEL SIGN OO +A840..A873 ; XID_Continue # Lo [52] PHAGS-PA LETTER KA..PHAGS-PA LETTER CANDRABINDU +A880..A881 ; XID_Continue # Mc [2] SAURASHTRA SIGN ANUSVARA..SAURASHTRA SIGN VISARGA +A882..A8B3 ; XID_Continue # Lo [50] SAURASHTRA LETTER A..SAURASHTRA LETTER LLA +A8B4..A8C3 ; XID_Continue # Mc [16] SAURASHTRA CONSONANT SIGN HAARU..SAURASHTRA VOWEL SIGN AU +A8C4 ; XID_Continue # Mn SAURASHTRA SIGN VIRAMA +A8D0..A8D9 ; XID_Continue # Nd [10] SAURASHTRA DIGIT ZERO..SAURASHTRA DIGIT NINE +A8E0..A8F1 ; XID_Continue # Mn [18] COMBINING DEVANAGARI DIGIT ZERO..COMBINING DEVANAGARI SIGN AVAGRAHA +A8F2..A8F7 ; XID_Continue # Lo [6] DEVANAGARI SIGN SPACING CANDRABINDU..DEVANAGARI SIGN CANDRABINDU AVAGRAHA +A8FB ; XID_Continue # Lo DEVANAGARI HEADSTROKE +A900..A909 ; XID_Continue # Nd [10] KAYAH LI DIGIT ZERO..KAYAH LI DIGIT NINE +A90A..A925 ; XID_Continue # Lo [28] KAYAH LI LETTER KA..KAYAH LI LETTER OO +A926..A92D ; XID_Continue # Mn [8] KAYAH LI VOWEL UE..KAYAH LI TONE CALYA PLOPHU +A930..A946 ; XID_Continue # Lo [23] REJANG LETTER KA..REJANG LETTER A +A947..A951 ; XID_Continue # Mn [11] REJANG VOWEL SIGN I..REJANG CONSONANT SIGN R +A952..A953 ; XID_Continue # Mc [2] REJANG CONSONANT SIGN H..REJANG VIRAMA +A960..A97C ; XID_Continue # Lo [29] HANGUL CHOSEONG TIKEUT-MIEUM..HANGUL CHOSEONG SSANGYEORINHIEUH +A980..A982 ; XID_Continue # Mn [3] JAVANESE SIGN PANYANGGA..JAVANESE SIGN LAYAR +A983 ; XID_Continue # Mc JAVANESE SIGN WIGNYAN +A984..A9B2 ; XID_Continue # Lo [47] JAVANESE LETTER A..JAVANESE LETTER HA +A9B3 ; XID_Continue # Mn JAVANESE SIGN CECAK TELU +A9B4..A9B5 ; XID_Continue # Mc [2] JAVANESE VOWEL SIGN TARUNG..JAVANESE VOWEL SIGN TOLONG +A9B6..A9B9 ; XID_Continue # Mn [4] JAVANESE VOWEL SIGN WULU..JAVANESE VOWEL SIGN SUKU MENDUT +A9BA..A9BB ; XID_Continue # Mc [2] JAVANESE VOWEL SIGN TALING..JAVANESE VOWEL SIGN DIRGA MURE +A9BC ; XID_Continue # Mn JAVANESE VOWEL SIGN PEPET +A9BD..A9C0 ; XID_Continue # Mc [4] JAVANESE CONSONANT SIGN KERET..JAVANESE PANGKON +A9CF ; XID_Continue # Lm JAVANESE PANGRANGKEP +A9D0..A9D9 ; XID_Continue # Nd [10] JAVANESE DIGIT ZERO..JAVANESE DIGIT NINE +AA00..AA28 ; XID_Continue # Lo [41] CHAM LETTER A..CHAM LETTER HA +AA29..AA2E ; XID_Continue # Mn [6] CHAM VOWEL SIGN AA..CHAM VOWEL SIGN OE +AA2F..AA30 ; XID_Continue # Mc [2] CHAM VOWEL SIGN O..CHAM VOWEL SIGN AI +AA31..AA32 ; XID_Continue # Mn [2] CHAM VOWEL SIGN AU..CHAM VOWEL SIGN UE +AA33..AA34 ; XID_Continue # Mc [2] CHAM CONSONANT SIGN YA..CHAM CONSONANT SIGN RA +AA35..AA36 ; XID_Continue # Mn [2] CHAM CONSONANT SIGN LA..CHAM CONSONANT SIGN WA +AA40..AA42 ; XID_Continue # Lo [3] CHAM LETTER FINAL K..CHAM LETTER FINAL NG +AA43 ; XID_Continue # Mn CHAM CONSONANT SIGN FINAL NG +AA44..AA4B ; XID_Continue # Lo [8] CHAM LETTER FINAL CH..CHAM LETTER FINAL SS +AA4C ; XID_Continue # Mn CHAM CONSONANT SIGN FINAL M +AA4D ; XID_Continue # Mc CHAM CONSONANT SIGN FINAL H +AA50..AA59 ; XID_Continue # Nd [10] CHAM DIGIT ZERO..CHAM DIGIT NINE +AA60..AA6F ; XID_Continue # Lo [16] MYANMAR LETTER KHAMTI GA..MYANMAR LETTER KHAMTI FA +AA70 ; XID_Continue # Lm MYANMAR MODIFIER LETTER KHAMTI REDUPLICATION +AA71..AA76 ; XID_Continue # Lo [6] MYANMAR LETTER KHAMTI XA..MYANMAR LOGOGRAM KHAMTI HM +AA7A ; XID_Continue # Lo MYANMAR LETTER AITON RA +AA7B ; XID_Continue # Mc MYANMAR SIGN PAO KAREN TONE +AA80..AAAF ; XID_Continue # Lo [48] TAI VIET LETTER LOW KO..TAI VIET LETTER HIGH O +AAB0 ; XID_Continue # Mn TAI VIET MAI KANG +AAB1 ; XID_Continue # Lo TAI VIET VOWEL AA +AAB2..AAB4 ; XID_Continue # Mn [3] TAI VIET VOWEL I..TAI VIET VOWEL U +AAB5..AAB6 ; XID_Continue # Lo [2] TAI VIET VOWEL E..TAI VIET VOWEL O +AAB7..AAB8 ; XID_Continue # Mn [2] TAI VIET MAI KHIT..TAI VIET VOWEL IA +AAB9..AABD ; XID_Continue # Lo [5] TAI VIET VOWEL UEA..TAI VIET VOWEL AN +AABE..AABF ; XID_Continue # Mn [2] TAI VIET VOWEL AM..TAI VIET TONE MAI EK +AAC0 ; XID_Continue # Lo TAI VIET TONE MAI NUENG +AAC1 ; XID_Continue # Mn TAI VIET TONE MAI THO +AAC2 ; XID_Continue # Lo TAI VIET TONE MAI SONG +AADB..AADC ; XID_Continue # Lo [2] TAI VIET SYMBOL KON..TAI VIET SYMBOL NUENG +AADD ; XID_Continue # Lm TAI VIET SYMBOL SAM +ABC0..ABE2 ; XID_Continue # Lo [35] MEETEI MAYEK LETTER KOK..MEETEI MAYEK LETTER I LONSUM +ABE3..ABE4 ; XID_Continue # Mc [2] MEETEI MAYEK VOWEL SIGN ONAP..MEETEI MAYEK VOWEL SIGN INAP +ABE5 ; XID_Continue # Mn MEETEI MAYEK VOWEL SIGN ANAP +ABE6..ABE7 ; XID_Continue # Mc [2] MEETEI MAYEK VOWEL SIGN YENAP..MEETEI MAYEK VOWEL SIGN SOUNAP +ABE8 ; XID_Continue # Mn MEETEI MAYEK VOWEL SIGN UNAP +ABE9..ABEA ; XID_Continue # Mc [2] MEETEI MAYEK VOWEL SIGN CHEINAP..MEETEI MAYEK VOWEL SIGN NUNG +ABEC ; XID_Continue # Mc MEETEI MAYEK LUM IYEK +ABED ; XID_Continue # Mn MEETEI MAYEK APUN IYEK +ABF0..ABF9 ; XID_Continue # Nd [10] MEETEI MAYEK DIGIT ZERO..MEETEI MAYEK DIGIT NINE +AC00..D7A3 ; XID_Continue # Lo [11172] HANGUL SYLLABLE GA..HANGUL SYLLABLE HIH +D7B0..D7C6 ; XID_Continue # Lo [23] HANGUL JUNGSEONG O-YEO..HANGUL JUNGSEONG ARAEA-E +D7CB..D7FB ; XID_Continue # Lo [49] HANGUL JONGSEONG NIEUN-RIEUL..HANGUL JONGSEONG PHIEUPH-THIEUTH +F900..FA2D ; XID_Continue # Lo [302] CJK COMPATIBILITY IDEOGRAPH-F900..CJK COMPATIBILITY IDEOGRAPH-FA2D +FA30..FA6D ; XID_Continue # Lo [62] CJK COMPATIBILITY IDEOGRAPH-FA30..CJK COMPATIBILITY IDEOGRAPH-FA6D +FA70..FAD9 ; XID_Continue # Lo [106] CJK COMPATIBILITY IDEOGRAPH-FA70..CJK COMPATIBILITY IDEOGRAPH-FAD9 +FB00..FB06 ; XID_Continue # L& [7] LATIN SMALL LIGATURE FF..LATIN SMALL LIGATURE ST +FB13..FB17 ; XID_Continue # L& [5] ARMENIAN SMALL LIGATURE MEN NOW..ARMENIAN SMALL LIGATURE MEN XEH +FB1D ; XID_Continue # Lo HEBREW LETTER YOD WITH HIRIQ +FB1E ; XID_Continue # Mn HEBREW POINT JUDEO-SPANISH VARIKA +FB1F..FB28 ; XID_Continue # Lo [10] HEBREW LIGATURE YIDDISH YOD YOD PATAH..HEBREW LETTER WIDE TAV +FB2A..FB36 ; XID_Continue # Lo [13] HEBREW LETTER SHIN WITH SHIN DOT..HEBREW LETTER ZAYIN WITH DAGESH +FB38..FB3C ; XID_Continue # Lo [5] HEBREW LETTER TET WITH DAGESH..HEBREW LETTER LAMED WITH DAGESH +FB3E ; XID_Continue # Lo HEBREW LETTER MEM WITH DAGESH +FB40..FB41 ; XID_Continue # Lo [2] HEBREW LETTER NUN WITH DAGESH..HEBREW LETTER SAMEKH WITH DAGESH +FB43..FB44 ; XID_Continue # Lo [2] HEBREW LETTER FINAL PE WITH DAGESH..HEBREW LETTER PE WITH DAGESH +FB46..FBB1 ; XID_Continue # Lo [108] HEBREW LETTER TSADI WITH DAGESH..ARABIC LETTER YEH BARREE WITH HAMZA ABOVE FINAL FORM +FBD3..FC5D ; XID_Continue # Lo [139] ARABIC LETTER NG ISOLATED FORM..ARABIC LIGATURE ALEF MAKSURA WITH SUPERSCRIPT ALEF ISOLATED FORM +FC64..FD3D ; XID_Continue # Lo [218] ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH REH FINAL FORM..ARABIC LIGATURE ALEF WITH FATHATAN ISOLATED FORM +FD50..FD8F ; XID_Continue # Lo [64] ARABIC LIGATURE TEH WITH JEEM WITH MEEM INITIAL FORM..ARABIC LIGATURE MEEM WITH KHAH WITH MEEM INITIAL FORM +FD92..FDC7 ; XID_Continue # Lo [54] ARABIC LIGATURE MEEM WITH JEEM WITH KHAH INITIAL FORM..ARABIC LIGATURE NOON WITH JEEM WITH YEH FINAL FORM +FDF0..FDF9 ; XID_Continue # Lo [10] ARABIC LIGATURE SALLA USED AS KORANIC STOP SIGN ISOLATED FORM..ARABIC LIGATURE SALLA ISOLATED FORM +FE00..FE0F ; XID_Continue # Mn [16] VARIATION SELECTOR-1..VARIATION SELECTOR-16 +FE20..FE26 ; XID_Continue # Mn [7] COMBINING LIGATURE LEFT HALF..COMBINING CONJOINING MACRON +FE33..FE34 ; XID_Continue # Pc [2] PRESENTATION FORM FOR VERTICAL LOW LINE..PRESENTATION FORM FOR VERTICAL WAVY LOW LINE +FE4D..FE4F ; XID_Continue # Pc [3] DASHED LOW LINE..WAVY LOW LINE +FE71 ; XID_Continue # Lo ARABIC TATWEEL WITH FATHATAN ABOVE +FE73 ; XID_Continue # Lo ARABIC TAIL FRAGMENT +FE77 ; XID_Continue # Lo ARABIC FATHA MEDIAL FORM +FE79 ; XID_Continue # Lo ARABIC DAMMA MEDIAL FORM +FE7B ; XID_Continue # Lo ARABIC KASRA MEDIAL FORM +FE7D ; XID_Continue # Lo ARABIC SHADDA MEDIAL FORM +FE7F..FEFC ; XID_Continue # Lo [126] ARABIC SUKUN MEDIAL FORM..ARABIC LIGATURE LAM WITH ALEF FINAL FORM +FF10..FF19 ; XID_Continue # Nd [10] FULLWIDTH DIGIT ZERO..FULLWIDTH DIGIT NINE +FF21..FF3A ; XID_Continue # L& [26] FULLWIDTH LATIN CAPITAL LETTER A..FULLWIDTH LATIN CAPITAL LETTER Z +FF3F ; XID_Continue # Pc FULLWIDTH LOW LINE +FF41..FF5A ; XID_Continue # L& [26] FULLWIDTH LATIN SMALL LETTER A..FULLWIDTH LATIN SMALL LETTER Z +FF66..FF6F ; XID_Continue # Lo [10] HALFWIDTH KATAKANA LETTER WO..HALFWIDTH KATAKANA LETTER SMALL TU +FF70 ; XID_Continue # Lm HALFWIDTH KATAKANA-HIRAGANA PROLONGED SOUND MARK +FF71..FF9D ; XID_Continue # Lo [45] HALFWIDTH KATAKANA LETTER A..HALFWIDTH KATAKANA LETTER N +FF9E..FF9F ; XID_Continue # Lm [2] HALFWIDTH KATAKANA VOICED SOUND MARK..HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK +FFA0..FFBE ; XID_Continue # Lo [31] HALFWIDTH HANGUL FILLER..HALFWIDTH HANGUL LETTER HIEUH +FFC2..FFC7 ; XID_Continue # Lo [6] HALFWIDTH HANGUL LETTER A..HALFWIDTH HANGUL LETTER E +FFCA..FFCF ; XID_Continue # Lo [6] HALFWIDTH HANGUL LETTER YEO..HALFWIDTH HANGUL LETTER OE +FFD2..FFD7 ; XID_Continue # Lo [6] HALFWIDTH HANGUL LETTER YO..HALFWIDTH HANGUL LETTER YU +FFDA..FFDC ; XID_Continue # Lo [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANGUL LETTER I +10000..1000B ; XID_Continue # Lo [12] LINEAR B SYLLABLE B008 A..LINEAR B SYLLABLE B046 JE +1000D..10026 ; XID_Continue # Lo [26] LINEAR B SYLLABLE B036 JO..LINEAR B SYLLABLE B032 QO +10028..1003A ; XID_Continue # Lo [19] LINEAR B SYLLABLE B060 RA..LINEAR B SYLLABLE B042 WO +1003C..1003D ; XID_Continue # Lo [2] LINEAR B SYLLABLE B017 ZA..LINEAR B SYLLABLE B074 ZE +1003F..1004D ; XID_Continue # Lo [15] LINEAR B SYLLABLE B020 ZO..LINEAR B SYLLABLE B091 TWO +10050..1005D ; XID_Continue # Lo [14] LINEAR B SYMBOL B018..LINEAR B SYMBOL B089 +10080..100FA ; XID_Continue # Lo [123] LINEAR B IDEOGRAM B100 MAN..LINEAR B IDEOGRAM VESSEL B305 +10140..10174 ; XID_Continue # Nl [53] GREEK ACROPHONIC ATTIC ONE QUARTER..GREEK ACROPHONIC STRATIAN FIFTY MNAS +101FD ; XID_Continue # Mn PHAISTOS DISC SIGN COMBINING OBLIQUE STROKE +10280..1029C ; XID_Continue # Lo [29] LYCIAN LETTER A..LYCIAN LETTER X +102A0..102D0 ; XID_Continue # Lo [49] CARIAN LETTER A..CARIAN LETTER UUU3 +10300..1031E ; XID_Continue # Lo [31] OLD ITALIC LETTER A..OLD ITALIC LETTER UU +10330..10340 ; XID_Continue # Lo [17] GOTHIC LETTER AHSA..GOTHIC LETTER PAIRTHRA +10341 ; XID_Continue # Nl GOTHIC LETTER NINETY +10342..10349 ; XID_Continue # Lo [8] GOTHIC LETTER RAIDA..GOTHIC LETTER OTHAL +1034A ; XID_Continue # Nl GOTHIC LETTER NINE HUNDRED +10380..1039D ; XID_Continue # Lo [30] UGARITIC LETTER ALPA..UGARITIC LETTER SSU +103A0..103C3 ; XID_Continue # Lo [36] OLD PERSIAN SIGN A..OLD PERSIAN SIGN HA +103C8..103CF ; XID_Continue # Lo [8] OLD PERSIAN SIGN AURAMAZDAA..OLD PERSIAN SIGN BUUMISH +103D1..103D5 ; XID_Continue # Nl [5] OLD PERSIAN NUMBER ONE..OLD PERSIAN NUMBER HUNDRED +10400..1044F ; XID_Continue # L& [80] DESERET CAPITAL LETTER LONG I..DESERET SMALL LETTER EW +10450..1049D ; XID_Continue # Lo [78] SHAVIAN LETTER PEEP..OSMANYA LETTER OO +104A0..104A9 ; XID_Continue # Nd [10] OSMANYA DIGIT ZERO..OSMANYA DIGIT NINE +10800..10805 ; XID_Continue # Lo [6] CYPRIOT SYLLABLE A..CYPRIOT SYLLABLE JA +10808 ; XID_Continue # Lo CYPRIOT SYLLABLE JO +1080A..10835 ; XID_Continue # Lo [44] CYPRIOT SYLLABLE KA..CYPRIOT SYLLABLE WO +10837..10838 ; XID_Continue # Lo [2] CYPRIOT SYLLABLE XA..CYPRIOT SYLLABLE XE +1083C ; XID_Continue # Lo CYPRIOT SYLLABLE ZA +1083F..10855 ; XID_Continue # Lo [23] CYPRIOT SYLLABLE ZO..IMPERIAL ARAMAIC LETTER TAW +10900..10915 ; XID_Continue # Lo [22] PHOENICIAN LETTER ALF..PHOENICIAN LETTER TAU +10920..10939 ; XID_Continue # Lo [26] LYDIAN LETTER A..LYDIAN LETTER C +10A00 ; XID_Continue # Lo KHAROSHTHI LETTER A +10A01..10A03 ; XID_Continue # Mn [3] KHAROSHTHI VOWEL SIGN I..KHAROSHTHI VOWEL SIGN VOCALIC R +10A05..10A06 ; XID_Continue # Mn [2] KHAROSHTHI VOWEL SIGN E..KHAROSHTHI VOWEL SIGN O +10A0C..10A0F ; XID_Continue # Mn [4] KHAROSHTHI VOWEL LENGTH MARK..KHAROSHTHI SIGN VISARGA +10A10..10A13 ; XID_Continue # Lo [4] KHAROSHTHI LETTER KA..KHAROSHTHI LETTER GHA +10A15..10A17 ; XID_Continue # Lo [3] KHAROSHTHI LETTER CA..KHAROSHTHI LETTER JA +10A19..10A33 ; XID_Continue # Lo [27] KHAROSHTHI LETTER NYA..KHAROSHTHI LETTER TTTHA +10A38..10A3A ; XID_Continue # Mn [3] KHAROSHTHI SIGN BAR ABOVE..KHAROSHTHI SIGN DOT BELOW +10A3F ; XID_Continue # Mn KHAROSHTHI VIRAMA +10A60..10A7C ; XID_Continue # Lo [29] OLD SOUTH ARABIAN LETTER HE..OLD SOUTH ARABIAN LETTER THETH +10B00..10B35 ; XID_Continue # Lo [54] AVESTAN LETTER A..AVESTAN LETTER HE +10B40..10B55 ; XID_Continue # Lo [22] INSCRIPTIONAL PARTHIAN LETTER ALEPH..INSCRIPTIONAL PARTHIAN LETTER TAW +10B60..10B72 ; XID_Continue # Lo [19] INSCRIPTIONAL PAHLAVI LETTER ALEPH..INSCRIPTIONAL PAHLAVI LETTER TAW +10C00..10C48 ; XID_Continue # Lo [73] OLD TURKIC LETTER ORKHON A..OLD TURKIC LETTER ORKHON BASH +11080..11081 ; XID_Continue # Mn [2] KAITHI SIGN CANDRABINDU..KAITHI SIGN ANUSVARA +11082 ; XID_Continue # Mc KAITHI SIGN VISARGA +11083..110AF ; XID_Continue # Lo [45] KAITHI LETTER A..KAITHI LETTER HA +110B0..110B2 ; XID_Continue # Mc [3] KAITHI VOWEL SIGN AA..KAITHI VOWEL SIGN II +110B3..110B6 ; XID_Continue # Mn [4] KAITHI VOWEL SIGN U..KAITHI VOWEL SIGN AI +110B7..110B8 ; XID_Continue # Mc [2] KAITHI VOWEL SIGN O..KAITHI VOWEL SIGN AU +110B9..110BA ; XID_Continue # Mn [2] KAITHI SIGN VIRAMA..KAITHI SIGN NUKTA +12000..1236E ; XID_Continue # Lo [879] CUNEIFORM SIGN A..CUNEIFORM SIGN ZUM +12400..12462 ; XID_Continue # Nl [99] CUNEIFORM NUMERIC SIGN TWO ASH..CUNEIFORM NUMERIC SIGN OLD ASSYRIAN ONE QUARTER +13000..1342E ; XID_Continue # Lo [1071] EGYPTIAN HIEROGLYPH A001..EGYPTIAN HIEROGLYPH AA032 +1D165..1D166 ; XID_Continue # Mc [2] MUSICAL SYMBOL COMBINING STEM..MUSICAL SYMBOL COMBINING SPRECHGESANG STEM +1D167..1D169 ; XID_Continue # Mn [3] MUSICAL SYMBOL COMBINING TREMOLO-1..MUSICAL SYMBOL COMBINING TREMOLO-3 +1D16D..1D172 ; XID_Continue # Mc [6] MUSICAL SYMBOL COMBINING AUGMENTATION DOT..MUSICAL SYMBOL COMBINING FLAG-5 +1D17B..1D182 ; XID_Continue # Mn [8] MUSICAL SYMBOL COMBINING ACCENT..MUSICAL SYMBOL COMBINING LOURE +1D185..1D18B ; XID_Continue # Mn [7] MUSICAL SYMBOL COMBINING DOIT..MUSICAL SYMBOL COMBINING TRIPLE TONGUE +1D1AA..1D1AD ; XID_Continue # Mn [4] MUSICAL SYMBOL COMBINING DOWN BOW..MUSICAL SYMBOL COMBINING SNAP PIZZICATO +1D242..1D244 ; XID_Continue # Mn [3] COMBINING GREEK MUSICAL TRISEME..COMBINING GREEK MUSICAL PENTASEME +1D400..1D454 ; XID_Continue # L& [85] MATHEMATICAL BOLD CAPITAL A..MATHEMATICAL ITALIC SMALL G +1D456..1D49C ; XID_Continue # L& [71] MATHEMATICAL ITALIC SMALL I..MATHEMATICAL SCRIPT CAPITAL A +1D49E..1D49F ; XID_Continue # L& [2] MATHEMATICAL SCRIPT CAPITAL C..MATHEMATICAL SCRIPT CAPITAL D +1D4A2 ; XID_Continue # L& MATHEMATICAL SCRIPT CAPITAL G +1D4A5..1D4A6 ; XID_Continue # L& [2] MATHEMATICAL SCRIPT CAPITAL J..MATHEMATICAL SCRIPT CAPITAL K +1D4A9..1D4AC ; XID_Continue # L& [4] MATHEMATICAL SCRIPT CAPITAL N..MATHEMATICAL SCRIPT CAPITAL Q +1D4AE..1D4B9 ; XID_Continue # L& [12] MATHEMATICAL SCRIPT CAPITAL S..MATHEMATICAL SCRIPT SMALL D +1D4BB ; XID_Continue # L& MATHEMATICAL SCRIPT SMALL F +1D4BD..1D4C3 ; XID_Continue # L& [7] MATHEMATICAL SCRIPT SMALL H..MATHEMATICAL SCRIPT SMALL N +1D4C5..1D505 ; XID_Continue # L& [65] MATHEMATICAL SCRIPT SMALL P..MATHEMATICAL FRAKTUR CAPITAL B +1D507..1D50A ; XID_Continue # L& [4] MATHEMATICAL FRAKTUR CAPITAL D..MATHEMATICAL FRAKTUR CAPITAL G +1D50D..1D514 ; XID_Continue # L& [8] MATHEMATICAL FRAKTUR CAPITAL J..MATHEMATICAL FRAKTUR CAPITAL Q +1D516..1D51C ; XID_Continue # L& [7] MATHEMATICAL FRAKTUR CAPITAL S..MATHEMATICAL FRAKTUR CAPITAL Y +1D51E..1D539 ; XID_Continue # L& [28] MATHEMATICAL FRAKTUR SMALL A..MATHEMATICAL DOUBLE-STRUCK CAPITAL B +1D53B..1D53E ; XID_Continue # L& [4] MATHEMATICAL DOUBLE-STRUCK CAPITAL D..MATHEMATICAL DOUBLE-STRUCK CAPITAL G +1D540..1D544 ; XID_Continue # L& [5] MATHEMATICAL DOUBLE-STRUCK CAPITAL I..MATHEMATICAL DOUBLE-STRUCK CAPITAL M +1D546 ; XID_Continue # L& MATHEMATICAL DOUBLE-STRUCK CAPITAL O +1D54A..1D550 ; XID_Continue # L& [7] MATHEMATICAL DOUBLE-STRUCK CAPITAL S..MATHEMATICAL DOUBLE-STRUCK CAPITAL Y +1D552..1D6A5 ; XID_Continue # L& [340] MATHEMATICAL DOUBLE-STRUCK SMALL A..MATHEMATICAL ITALIC SMALL DOTLESS J +1D6A8..1D6C0 ; XID_Continue # L& [25] MATHEMATICAL BOLD CAPITAL ALPHA..MATHEMATICAL BOLD CAPITAL OMEGA +1D6C2..1D6DA ; XID_Continue # L& [25] MATHEMATICAL BOLD SMALL ALPHA..MATHEMATICAL BOLD SMALL OMEGA +1D6DC..1D6FA ; XID_Continue # L& [31] MATHEMATICAL BOLD EPSILON SYMBOL..MATHEMATICAL ITALIC CAPITAL OMEGA +1D6FC..1D714 ; XID_Continue # L& [25] MATHEMATICAL ITALIC SMALL ALPHA..MATHEMATICAL ITALIC SMALL OMEGA +1D716..1D734 ; XID_Continue # L& [31] MATHEMATICAL ITALIC EPSILON SYMBOL..MATHEMATICAL BOLD ITALIC CAPITAL OMEGA +1D736..1D74E ; XID_Continue # L& [25] MATHEMATICAL BOLD ITALIC SMALL ALPHA..MATHEMATICAL BOLD ITALIC SMALL OMEGA +1D750..1D76E ; XID_Continue # L& [31] MATHEMATICAL BOLD ITALIC EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD CAPITAL OMEGA +1D770..1D788 ; XID_Continue # L& [25] MATHEMATICAL SANS-SERIF BOLD SMALL ALPHA..MATHEMATICAL SANS-SERIF BOLD SMALL OMEGA +1D78A..1D7A8 ; XID_Continue # L& [31] MATHEMATICAL SANS-SERIF BOLD EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL OMEGA +1D7AA..1D7C2 ; XID_Continue # L& [25] MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL ALPHA..MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL OMEGA +1D7C4..1D7CB ; XID_Continue # L& [8] MATHEMATICAL SANS-SERIF BOLD ITALIC EPSILON SYMBOL..MATHEMATICAL BOLD SMALL DIGAMMA +1D7CE..1D7FF ; XID_Continue # Nd [50] MATHEMATICAL BOLD DIGIT ZERO..MATHEMATICAL MONOSPACE DIGIT NINE +20000..2A6D6 ; XID_Continue # Lo [42711] CJK UNIFIED IDEOGRAPH-20000..CJK UNIFIED IDEOGRAPH-2A6D6 +2A700..2B734 ; XID_Continue # Lo [4149] CJK UNIFIED IDEOGRAPH-2A700..CJK UNIFIED IDEOGRAPH-2B734 +2F800..2FA1D ; XID_Continue # Lo [542] CJK COMPATIBILITY IDEOGRAPH-2F800..CJK COMPATIBILITY IDEOGRAPH-2FA1D +E0100..E01EF ; XID_Continue # Mn [240] VARIATION SELECTOR-17..VARIATION SELECTOR-256 + +# Total code points: 101615 + +# ================================================ + +# Derived Property: Default_Ignorable_Code_Point +# Generated from +# Other_Default_Ignorable_Code_Point +# + Cf (Format characters) +# + Variation_Selector +# - White_Space +# - FFF9..FFFB (Annotation Characters) +# - 0600..0603, 06DD, 070F (exceptional Cf characters that should be visible) + +00AD ; Default_Ignorable_Code_Point # Cf SOFT HYPHEN +034F ; Default_Ignorable_Code_Point # Mn COMBINING GRAPHEME JOINER +115F..1160 ; Default_Ignorable_Code_Point # Lo [2] HANGUL CHOSEONG FILLER..HANGUL JUNGSEONG FILLER +17B4..17B5 ; Default_Ignorable_Code_Point # Cf [2] KHMER VOWEL INHERENT AQ..KHMER VOWEL INHERENT AA +180B..180D ; Default_Ignorable_Code_Point # Mn [3] MONGOLIAN FREE VARIATION SELECTOR ONE..MONGOLIAN FREE VARIATION SELECTOR THREE +200B..200F ; Default_Ignorable_Code_Point # Cf [5] ZERO WIDTH SPACE..RIGHT-TO-LEFT MARK +202A..202E ; Default_Ignorable_Code_Point # Cf [5] LEFT-TO-RIGHT EMBEDDING..RIGHT-TO-LEFT OVERRIDE +2060..2064 ; Default_Ignorable_Code_Point # Cf [5] WORD JOINER..INVISIBLE PLUS +2065..2069 ; Default_Ignorable_Code_Point # Cn [5] .. +206A..206F ; Default_Ignorable_Code_Point # Cf [6] INHIBIT SYMMETRIC SWAPPING..NOMINAL DIGIT SHAPES +3164 ; Default_Ignorable_Code_Point # Lo HANGUL FILLER +FE00..FE0F ; Default_Ignorable_Code_Point # Mn [16] VARIATION SELECTOR-1..VARIATION SELECTOR-16 +FEFF ; Default_Ignorable_Code_Point # Cf ZERO WIDTH NO-BREAK SPACE +FFA0 ; Default_Ignorable_Code_Point # Lo HALFWIDTH HANGUL FILLER +FFF0..FFF8 ; Default_Ignorable_Code_Point # Cn [9] .. +1D173..1D17A ; Default_Ignorable_Code_Point # Cf [8] MUSICAL SYMBOL BEGIN BEAM..MUSICAL SYMBOL END PHRASE +E0000 ; Default_Ignorable_Code_Point # Cn +E0001 ; Default_Ignorable_Code_Point # Cf LANGUAGE TAG +E0002..E001F ; Default_Ignorable_Code_Point # Cn [30] .. +E0020..E007F ; Default_Ignorable_Code_Point # Cf [96] TAG SPACE..CANCEL TAG +E0080..E00FF ; Default_Ignorable_Code_Point # Cn [128] .. +E0100..E01EF ; Default_Ignorable_Code_Point # Mn [240] VARIATION SELECTOR-17..VARIATION SELECTOR-256 +E01F0..E0FFF ; Default_Ignorable_Code_Point # Cn [3600] .. + +# Total code points: 4167 + +# ================================================ + +# Derived Property: Grapheme_Extend +# Generated from: Me + Mn + Other_Grapheme_Extend +# Note: depending on an application's interpretation of Co (private use), +# they may be either in Grapheme_Base, or in Grapheme_Extend, or in neither. + +0300..036F ; Grapheme_Extend # Mn [112] COMBINING GRAVE ACCENT..COMBINING LATIN SMALL LETTER X +0483..0487 ; Grapheme_Extend # Mn [5] COMBINING CYRILLIC TITLO..COMBINING CYRILLIC POKRYTIE +0488..0489 ; Grapheme_Extend # Me [2] COMBINING CYRILLIC HUNDRED THOUSANDS SIGN..COMBINING CYRILLIC MILLIONS SIGN +0591..05BD ; Grapheme_Extend # Mn [45] HEBREW ACCENT ETNAHTA..HEBREW POINT METEG +05BF ; Grapheme_Extend # Mn HEBREW POINT RAFE +05C1..05C2 ; Grapheme_Extend # Mn [2] HEBREW POINT SHIN DOT..HEBREW POINT SIN DOT +05C4..05C5 ; Grapheme_Extend # Mn [2] HEBREW MARK UPPER DOT..HEBREW MARK LOWER DOT +05C7 ; Grapheme_Extend # Mn HEBREW POINT QAMATS QATAN +0610..061A ; Grapheme_Extend # Mn [11] ARABIC SIGN SALLALLAHOU ALAYHE WASSALLAM..ARABIC SMALL KASRA +064B..065E ; Grapheme_Extend # Mn [20] ARABIC FATHATAN..ARABIC FATHA WITH TWO DOTS +0670 ; Grapheme_Extend # Mn ARABIC LETTER SUPERSCRIPT ALEF +06D6..06DC ; Grapheme_Extend # Mn [7] ARABIC SMALL HIGH LIGATURE SAD WITH LAM WITH ALEF MAKSURA..ARABIC SMALL HIGH SEEN +06DE ; Grapheme_Extend # Me ARABIC START OF RUB EL HIZB +06DF..06E4 ; Grapheme_Extend # Mn [6] ARABIC SMALL HIGH ROUNDED ZERO..ARABIC SMALL HIGH MADDA +06E7..06E8 ; Grapheme_Extend # Mn [2] ARABIC SMALL HIGH YEH..ARABIC SMALL HIGH NOON +06EA..06ED ; Grapheme_Extend # Mn [4] ARABIC EMPTY CENTRE LOW STOP..ARABIC SMALL LOW MEEM +0711 ; Grapheme_Extend # Mn SYRIAC LETTER SUPERSCRIPT ALAPH +0730..074A ; Grapheme_Extend # Mn [27] SYRIAC PTHAHA ABOVE..SYRIAC BARREKH +07A6..07B0 ; Grapheme_Extend # Mn [11] THAANA ABAFILI..THAANA SUKUN +07EB..07F3 ; Grapheme_Extend # Mn [9] NKO COMBINING SHORT HIGH TONE..NKO COMBINING DOUBLE DOT ABOVE +0816..0819 ; Grapheme_Extend # Mn [4] SAMARITAN MARK IN..SAMARITAN MARK DAGESH +081B..0823 ; Grapheme_Extend # Mn [9] SAMARITAN MARK EPENTHETIC YUT..SAMARITAN VOWEL SIGN A +0825..0827 ; Grapheme_Extend # Mn [3] SAMARITAN VOWEL SIGN SHORT A..SAMARITAN VOWEL SIGN U +0829..082D ; Grapheme_Extend # Mn [5] SAMARITAN VOWEL SIGN LONG I..SAMARITAN MARK NEQUDAA +0900..0902 ; Grapheme_Extend # Mn [3] DEVANAGARI SIGN INVERTED CANDRABINDU..DEVANAGARI SIGN ANUSVARA +093C ; Grapheme_Extend # Mn DEVANAGARI SIGN NUKTA +0941..0948 ; Grapheme_Extend # Mn [8] DEVANAGARI VOWEL SIGN U..DEVANAGARI VOWEL SIGN AI +094D ; Grapheme_Extend # Mn DEVANAGARI SIGN VIRAMA +0951..0955 ; Grapheme_Extend # Mn [5] DEVANAGARI STRESS SIGN UDATTA..DEVANAGARI VOWEL SIGN CANDRA LONG E +0962..0963 ; Grapheme_Extend # Mn [2] DEVANAGARI VOWEL SIGN VOCALIC L..DEVANAGARI VOWEL SIGN VOCALIC LL +0981 ; Grapheme_Extend # Mn BENGALI SIGN CANDRABINDU +09BC ; Grapheme_Extend # Mn BENGALI SIGN NUKTA +09BE ; Grapheme_Extend # Mc BENGALI VOWEL SIGN AA +09C1..09C4 ; Grapheme_Extend # Mn [4] BENGALI VOWEL SIGN U..BENGALI VOWEL SIGN VOCALIC RR +09CD ; Grapheme_Extend # Mn BENGALI SIGN VIRAMA +09D7 ; Grapheme_Extend # Mc BENGALI AU LENGTH MARK +09E2..09E3 ; Grapheme_Extend # Mn [2] BENGALI VOWEL SIGN VOCALIC L..BENGALI VOWEL SIGN VOCALIC LL +0A01..0A02 ; Grapheme_Extend # Mn [2] GURMUKHI SIGN ADAK BINDI..GURMUKHI SIGN BINDI +0A3C ; Grapheme_Extend # Mn GURMUKHI SIGN NUKTA +0A41..0A42 ; Grapheme_Extend # Mn [2] GURMUKHI VOWEL SIGN U..GURMUKHI VOWEL SIGN UU +0A47..0A48 ; Grapheme_Extend # Mn [2] GURMUKHI VOWEL SIGN EE..GURMUKHI VOWEL SIGN AI +0A4B..0A4D ; Grapheme_Extend # Mn [3] GURMUKHI VOWEL SIGN OO..GURMUKHI SIGN VIRAMA +0A51 ; Grapheme_Extend # Mn GURMUKHI SIGN UDAAT +0A70..0A71 ; Grapheme_Extend # Mn [2] GURMUKHI TIPPI..GURMUKHI ADDAK +0A75 ; Grapheme_Extend # Mn GURMUKHI SIGN YAKASH +0A81..0A82 ; Grapheme_Extend # Mn [2] GUJARATI SIGN CANDRABINDU..GUJARATI SIGN ANUSVARA +0ABC ; Grapheme_Extend # Mn GUJARATI SIGN NUKTA +0AC1..0AC5 ; Grapheme_Extend # Mn [5] GUJARATI VOWEL SIGN U..GUJARATI VOWEL SIGN CANDRA E +0AC7..0AC8 ; Grapheme_Extend # Mn [2] GUJARATI VOWEL SIGN E..GUJARATI VOWEL SIGN AI +0ACD ; Grapheme_Extend # Mn GUJARATI SIGN VIRAMA +0AE2..0AE3 ; Grapheme_Extend # Mn [2] GUJARATI VOWEL SIGN VOCALIC L..GUJARATI VOWEL SIGN VOCALIC LL +0B01 ; Grapheme_Extend # Mn ORIYA SIGN CANDRABINDU +0B3C ; Grapheme_Extend # Mn ORIYA SIGN NUKTA +0B3E ; Grapheme_Extend # Mc ORIYA VOWEL SIGN AA +0B3F ; Grapheme_Extend # Mn ORIYA VOWEL SIGN I +0B41..0B44 ; Grapheme_Extend # Mn [4] ORIYA VOWEL SIGN U..ORIYA VOWEL SIGN VOCALIC RR +0B4D ; Grapheme_Extend # Mn ORIYA SIGN VIRAMA +0B56 ; Grapheme_Extend # Mn ORIYA AI LENGTH MARK +0B57 ; Grapheme_Extend # Mc ORIYA AU LENGTH MARK +0B62..0B63 ; Grapheme_Extend # Mn [2] ORIYA VOWEL SIGN VOCALIC L..ORIYA VOWEL SIGN VOCALIC LL +0B82 ; Grapheme_Extend # Mn TAMIL SIGN ANUSVARA +0BBE ; Grapheme_Extend # Mc TAMIL VOWEL SIGN AA +0BC0 ; Grapheme_Extend # Mn TAMIL VOWEL SIGN II +0BCD ; Grapheme_Extend # Mn TAMIL SIGN VIRAMA +0BD7 ; Grapheme_Extend # Mc TAMIL AU LENGTH MARK +0C3E..0C40 ; Grapheme_Extend # Mn [3] TELUGU VOWEL SIGN AA..TELUGU VOWEL SIGN II +0C46..0C48 ; Grapheme_Extend # Mn [3] TELUGU VOWEL SIGN E..TELUGU VOWEL SIGN AI +0C4A..0C4D ; Grapheme_Extend # Mn [4] TELUGU VOWEL SIGN O..TELUGU SIGN VIRAMA +0C55..0C56 ; Grapheme_Extend # Mn [2] TELUGU LENGTH MARK..TELUGU AI LENGTH MARK +0C62..0C63 ; Grapheme_Extend # Mn [2] TELUGU VOWEL SIGN VOCALIC L..TELUGU VOWEL SIGN VOCALIC LL +0CBC ; Grapheme_Extend # Mn KANNADA SIGN NUKTA +0CBF ; Grapheme_Extend # Mn KANNADA VOWEL SIGN I +0CC2 ; Grapheme_Extend # Mc KANNADA VOWEL SIGN UU +0CC6 ; Grapheme_Extend # Mn KANNADA VOWEL SIGN E +0CCC..0CCD ; Grapheme_Extend # Mn [2] KANNADA VOWEL SIGN AU..KANNADA SIGN VIRAMA +0CD5..0CD6 ; Grapheme_Extend # Mc [2] KANNADA LENGTH MARK..KANNADA AI LENGTH MARK +0CE2..0CE3 ; Grapheme_Extend # Mn [2] KANNADA VOWEL SIGN VOCALIC L..KANNADA VOWEL SIGN VOCALIC LL +0D3E ; Grapheme_Extend # Mc MALAYALAM VOWEL SIGN AA +0D41..0D44 ; Grapheme_Extend # Mn [4] MALAYALAM VOWEL SIGN U..MALAYALAM VOWEL SIGN VOCALIC RR +0D4D ; Grapheme_Extend # Mn MALAYALAM SIGN VIRAMA +0D57 ; Grapheme_Extend # Mc MALAYALAM AU LENGTH MARK +0D62..0D63 ; Grapheme_Extend # Mn [2] MALAYALAM VOWEL SIGN VOCALIC L..MALAYALAM VOWEL SIGN VOCALIC LL +0DCA ; Grapheme_Extend # Mn SINHALA SIGN AL-LAKUNA +0DCF ; Grapheme_Extend # Mc SINHALA VOWEL SIGN AELA-PILLA +0DD2..0DD4 ; Grapheme_Extend # Mn [3] SINHALA VOWEL SIGN KETTI IS-PILLA..SINHALA VOWEL SIGN KETTI PAA-PILLA +0DD6 ; Grapheme_Extend # Mn SINHALA VOWEL SIGN DIGA PAA-PILLA +0DDF ; Grapheme_Extend # Mc SINHALA VOWEL SIGN GAYANUKITTA +0E31 ; Grapheme_Extend # Mn THAI CHARACTER MAI HAN-AKAT +0E34..0E3A ; Grapheme_Extend # Mn [7] THAI CHARACTER SARA I..THAI CHARACTER PHINTHU +0E47..0E4E ; Grapheme_Extend # Mn [8] THAI CHARACTER MAITAIKHU..THAI CHARACTER YAMAKKAN +0EB1 ; Grapheme_Extend # Mn LAO VOWEL SIGN MAI KAN +0EB4..0EB9 ; Grapheme_Extend # Mn [6] LAO VOWEL SIGN I..LAO VOWEL SIGN UU +0EBB..0EBC ; Grapheme_Extend # Mn [2] LAO VOWEL SIGN MAI KON..LAO SEMIVOWEL SIGN LO +0EC8..0ECD ; Grapheme_Extend # Mn [6] LAO TONE MAI EK..LAO NIGGAHITA +0F18..0F19 ; Grapheme_Extend # Mn [2] TIBETAN ASTROLOGICAL SIGN -KHYUD PA..TIBETAN ASTROLOGICAL SIGN SDONG TSHUGS +0F35 ; Grapheme_Extend # Mn TIBETAN MARK NGAS BZUNG NYI ZLA +0F37 ; Grapheme_Extend # Mn TIBETAN MARK NGAS BZUNG SGOR RTAGS +0F39 ; Grapheme_Extend # Mn TIBETAN MARK TSA -PHRU +0F71..0F7E ; Grapheme_Extend # Mn [14] TIBETAN VOWEL SIGN AA..TIBETAN SIGN RJES SU NGA RO +0F80..0F84 ; Grapheme_Extend # Mn [5] TIBETAN VOWEL SIGN REVERSED I..TIBETAN MARK HALANTA +0F86..0F87 ; Grapheme_Extend # Mn [2] TIBETAN SIGN LCI RTAGS..TIBETAN SIGN YANG RTAGS +0F90..0F97 ; Grapheme_Extend # Mn [8] TIBETAN SUBJOINED LETTER KA..TIBETAN SUBJOINED LETTER JA +0F99..0FBC ; Grapheme_Extend # Mn [36] TIBETAN SUBJOINED LETTER NYA..TIBETAN SUBJOINED LETTER FIXED-FORM RA +0FC6 ; Grapheme_Extend # Mn TIBETAN SYMBOL PADMA GDAN +102D..1030 ; Grapheme_Extend # Mn [4] MYANMAR VOWEL SIGN I..MYANMAR VOWEL SIGN UU +1032..1037 ; Grapheme_Extend # Mn [6] MYANMAR VOWEL SIGN AI..MYANMAR SIGN DOT BELOW +1039..103A ; Grapheme_Extend # Mn [2] MYANMAR SIGN VIRAMA..MYANMAR SIGN ASAT +103D..103E ; Grapheme_Extend # Mn [2] MYANMAR CONSONANT SIGN MEDIAL WA..MYANMAR CONSONANT SIGN MEDIAL HA +1058..1059 ; Grapheme_Extend # Mn [2] MYANMAR VOWEL SIGN VOCALIC L..MYANMAR VOWEL SIGN VOCALIC LL +105E..1060 ; Grapheme_Extend # Mn [3] MYANMAR CONSONANT SIGN MON MEDIAL NA..MYANMAR CONSONANT SIGN MON MEDIAL LA +1071..1074 ; Grapheme_Extend # Mn [4] MYANMAR VOWEL SIGN GEBA KAREN I..MYANMAR VOWEL SIGN KAYAH EE +1082 ; Grapheme_Extend # Mn MYANMAR CONSONANT SIGN SHAN MEDIAL WA +1085..1086 ; Grapheme_Extend # Mn [2] MYANMAR VOWEL SIGN SHAN E ABOVE..MYANMAR VOWEL SIGN SHAN FINAL Y +108D ; Grapheme_Extend # Mn MYANMAR SIGN SHAN COUNCIL EMPHATIC TONE +109D ; Grapheme_Extend # Mn MYANMAR VOWEL SIGN AITON AI +135F ; Grapheme_Extend # Mn ETHIOPIC COMBINING GEMINATION MARK +1712..1714 ; Grapheme_Extend # Mn [3] TAGALOG VOWEL SIGN I..TAGALOG SIGN VIRAMA +1732..1734 ; Grapheme_Extend # Mn [3] HANUNOO VOWEL SIGN I..HANUNOO SIGN PAMUDPOD +1752..1753 ; Grapheme_Extend # Mn [2] BUHID VOWEL SIGN I..BUHID VOWEL SIGN U +1772..1773 ; Grapheme_Extend # Mn [2] TAGBANWA VOWEL SIGN I..TAGBANWA VOWEL SIGN U +17B7..17BD ; Grapheme_Extend # Mn [7] KHMER VOWEL SIGN I..KHMER VOWEL SIGN UA +17C6 ; Grapheme_Extend # Mn KHMER SIGN NIKAHIT +17C9..17D3 ; Grapheme_Extend # Mn [11] KHMER SIGN MUUSIKATOAN..KHMER SIGN BATHAMASAT +17DD ; Grapheme_Extend # Mn KHMER SIGN ATTHACAN +180B..180D ; Grapheme_Extend # Mn [3] MONGOLIAN FREE VARIATION SELECTOR ONE..MONGOLIAN FREE VARIATION SELECTOR THREE +18A9 ; Grapheme_Extend # Mn MONGOLIAN LETTER ALI GALI DAGALGA +1920..1922 ; Grapheme_Extend # Mn [3] LIMBU VOWEL SIGN A..LIMBU VOWEL SIGN U +1927..1928 ; Grapheme_Extend # Mn [2] LIMBU VOWEL SIGN E..LIMBU VOWEL SIGN O +1932 ; Grapheme_Extend # Mn LIMBU SMALL LETTER ANUSVARA +1939..193B ; Grapheme_Extend # Mn [3] LIMBU SIGN MUKPHRENG..LIMBU SIGN SA-I +1A17..1A18 ; Grapheme_Extend # Mn [2] BUGINESE VOWEL SIGN I..BUGINESE VOWEL SIGN U +1A56 ; Grapheme_Extend # Mn TAI THAM CONSONANT SIGN MEDIAL LA +1A58..1A5E ; Grapheme_Extend # Mn [7] TAI THAM SIGN MAI KANG LAI..TAI THAM CONSONANT SIGN SA +1A60 ; Grapheme_Extend # Mn TAI THAM SIGN SAKOT +1A62 ; Grapheme_Extend # Mn TAI THAM VOWEL SIGN MAI SAT +1A65..1A6C ; Grapheme_Extend # Mn [8] TAI THAM VOWEL SIGN I..TAI THAM VOWEL SIGN OA BELOW +1A73..1A7C ; Grapheme_Extend # Mn [10] TAI THAM VOWEL SIGN OA ABOVE..TAI THAM SIGN KHUEN-LUE KARAN +1A7F ; Grapheme_Extend # Mn TAI THAM COMBINING CRYPTOGRAMMIC DOT +1B00..1B03 ; Grapheme_Extend # Mn [4] BALINESE SIGN ULU RICEM..BALINESE SIGN SURANG +1B34 ; Grapheme_Extend # Mn BALINESE SIGN REREKAN +1B36..1B3A ; Grapheme_Extend # Mn [5] BALINESE VOWEL SIGN ULU..BALINESE VOWEL SIGN RA REPA +1B3C ; Grapheme_Extend # Mn BALINESE VOWEL SIGN LA LENGA +1B42 ; Grapheme_Extend # Mn BALINESE VOWEL SIGN PEPET +1B6B..1B73 ; Grapheme_Extend # Mn [9] BALINESE MUSICAL SYMBOL COMBINING TEGEH..BALINESE MUSICAL SYMBOL COMBINING GONG +1B80..1B81 ; Grapheme_Extend # Mn [2] SUNDANESE SIGN PANYECEK..SUNDANESE SIGN PANGLAYAR +1BA2..1BA5 ; Grapheme_Extend # Mn [4] SUNDANESE CONSONANT SIGN PANYAKRA..SUNDANESE VOWEL SIGN PANYUKU +1BA8..1BA9 ; Grapheme_Extend # Mn [2] SUNDANESE VOWEL SIGN PAMEPET..SUNDANESE VOWEL SIGN PANEULEUNG +1C2C..1C33 ; Grapheme_Extend # Mn [8] LEPCHA VOWEL SIGN E..LEPCHA CONSONANT SIGN T +1C36..1C37 ; Grapheme_Extend # Mn [2] LEPCHA SIGN RAN..LEPCHA SIGN NUKTA +1CD0..1CD2 ; Grapheme_Extend # Mn [3] VEDIC TONE KARSHANA..VEDIC TONE PRENKHA +1CD4..1CE0 ; Grapheme_Extend # Mn [13] VEDIC SIGN YAJURVEDIC MIDLINE SVARITA..VEDIC TONE RIGVEDIC KASHMIRI INDEPENDENT SVARITA +1CE2..1CE8 ; Grapheme_Extend # Mn [7] VEDIC SIGN VISARGA SVARITA..VEDIC SIGN VISARGA ANUDATTA WITH TAIL +1CED ; Grapheme_Extend # Mn VEDIC SIGN TIRYAK +1DC0..1DE6 ; Grapheme_Extend # Mn [39] COMBINING DOTTED GRAVE ACCENT..COMBINING LATIN SMALL LETTER Z +1DFD..1DFF ; Grapheme_Extend # Mn [3] COMBINING ALMOST EQUAL TO BELOW..COMBINING RIGHT ARROWHEAD AND DOWN ARROWHEAD BELOW +200C..200D ; Grapheme_Extend # Cf [2] ZERO WIDTH NON-JOINER..ZERO WIDTH JOINER +20D0..20DC ; Grapheme_Extend # Mn [13] COMBINING LEFT HARPOON ABOVE..COMBINING FOUR DOTS ABOVE +20DD..20E0 ; Grapheme_Extend # Me [4] COMBINING ENCLOSING CIRCLE..COMBINING ENCLOSING CIRCLE BACKSLASH +20E1 ; Grapheme_Extend # Mn COMBINING LEFT RIGHT ARROW ABOVE +20E2..20E4 ; Grapheme_Extend # Me [3] COMBINING ENCLOSING SCREEN..COMBINING ENCLOSING UPWARD POINTING TRIANGLE +20E5..20F0 ; Grapheme_Extend # Mn [12] COMBINING REVERSE SOLIDUS OVERLAY..COMBINING ASTERISK ABOVE +2CEF..2CF1 ; Grapheme_Extend # Mn [3] COPTIC COMBINING NI ABOVE..COPTIC COMBINING SPIRITUS LENIS +2DE0..2DFF ; Grapheme_Extend # Mn [32] COMBINING CYRILLIC LETTER BE..COMBINING CYRILLIC LETTER IOTIFIED BIG YUS +302A..302F ; Grapheme_Extend # Mn [6] IDEOGRAPHIC LEVEL TONE MARK..HANGUL DOUBLE DOT TONE MARK +3099..309A ; Grapheme_Extend # Mn [2] COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK..COMBINING KATAKANA-HIRAGANA SEMI-VOICED SOUND MARK +A66F ; Grapheme_Extend # Mn COMBINING CYRILLIC VZMET +A670..A672 ; Grapheme_Extend # Me [3] COMBINING CYRILLIC TEN MILLIONS SIGN..COMBINING CYRILLIC THOUSAND MILLIONS SIGN +A67C..A67D ; Grapheme_Extend # Mn [2] COMBINING CYRILLIC KAVYKA..COMBINING CYRILLIC PAYEROK +A6F0..A6F1 ; Grapheme_Extend # Mn [2] BAMUM COMBINING MARK KOQNDON..BAMUM COMBINING MARK TUKWENTIS +A802 ; Grapheme_Extend # Mn SYLOTI NAGRI SIGN DVISVARA +A806 ; Grapheme_Extend # Mn SYLOTI NAGRI SIGN HASANTA +A80B ; Grapheme_Extend # Mn SYLOTI NAGRI SIGN ANUSVARA +A825..A826 ; Grapheme_Extend # Mn [2] SYLOTI NAGRI VOWEL SIGN U..SYLOTI NAGRI VOWEL SIGN E +A8C4 ; Grapheme_Extend # Mn SAURASHTRA SIGN VIRAMA +A8E0..A8F1 ; Grapheme_Extend # Mn [18] COMBINING DEVANAGARI DIGIT ZERO..COMBINING DEVANAGARI SIGN AVAGRAHA +A926..A92D ; Grapheme_Extend # Mn [8] KAYAH LI VOWEL UE..KAYAH LI TONE CALYA PLOPHU +A947..A951 ; Grapheme_Extend # Mn [11] REJANG VOWEL SIGN I..REJANG CONSONANT SIGN R +A980..A982 ; Grapheme_Extend # Mn [3] JAVANESE SIGN PANYANGGA..JAVANESE SIGN LAYAR +A9B3 ; Grapheme_Extend # Mn JAVANESE SIGN CECAK TELU +A9B6..A9B9 ; Grapheme_Extend # Mn [4] JAVANESE VOWEL SIGN WULU..JAVANESE VOWEL SIGN SUKU MENDUT +A9BC ; Grapheme_Extend # Mn JAVANESE VOWEL SIGN PEPET +AA29..AA2E ; Grapheme_Extend # Mn [6] CHAM VOWEL SIGN AA..CHAM VOWEL SIGN OE +AA31..AA32 ; Grapheme_Extend # Mn [2] CHAM VOWEL SIGN AU..CHAM VOWEL SIGN UE +AA35..AA36 ; Grapheme_Extend # Mn [2] CHAM CONSONANT SIGN LA..CHAM CONSONANT SIGN WA +AA43 ; Grapheme_Extend # Mn CHAM CONSONANT SIGN FINAL NG +AA4C ; Grapheme_Extend # Mn CHAM CONSONANT SIGN FINAL M +AAB0 ; Grapheme_Extend # Mn TAI VIET MAI KANG +AAB2..AAB4 ; Grapheme_Extend # Mn [3] TAI VIET VOWEL I..TAI VIET VOWEL U +AAB7..AAB8 ; Grapheme_Extend # Mn [2] TAI VIET MAI KHIT..TAI VIET VOWEL IA +AABE..AABF ; Grapheme_Extend # Mn [2] TAI VIET VOWEL AM..TAI VIET TONE MAI EK +AAC1 ; Grapheme_Extend # Mn TAI VIET TONE MAI THO +ABE5 ; Grapheme_Extend # Mn MEETEI MAYEK VOWEL SIGN ANAP +ABE8 ; Grapheme_Extend # Mn MEETEI MAYEK VOWEL SIGN UNAP +ABED ; Grapheme_Extend # Mn MEETEI MAYEK APUN IYEK +FB1E ; Grapheme_Extend # Mn HEBREW POINT JUDEO-SPANISH VARIKA +FE00..FE0F ; Grapheme_Extend # Mn [16] VARIATION SELECTOR-1..VARIATION SELECTOR-16 +FE20..FE26 ; Grapheme_Extend # Mn [7] COMBINING LIGATURE LEFT HALF..COMBINING CONJOINING MACRON +FF9E..FF9F ; Grapheme_Extend # Lm [2] HALFWIDTH KATAKANA VOICED SOUND MARK..HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK +101FD ; Grapheme_Extend # Mn PHAISTOS DISC SIGN COMBINING OBLIQUE STROKE +10A01..10A03 ; Grapheme_Extend # Mn [3] KHAROSHTHI VOWEL SIGN I..KHAROSHTHI VOWEL SIGN VOCALIC R +10A05..10A06 ; Grapheme_Extend # Mn [2] KHAROSHTHI VOWEL SIGN E..KHAROSHTHI VOWEL SIGN O +10A0C..10A0F ; Grapheme_Extend # Mn [4] KHAROSHTHI VOWEL LENGTH MARK..KHAROSHTHI SIGN VISARGA +10A38..10A3A ; Grapheme_Extend # Mn [3] KHAROSHTHI SIGN BAR ABOVE..KHAROSHTHI SIGN DOT BELOW +10A3F ; Grapheme_Extend # Mn KHAROSHTHI VIRAMA +11080..11081 ; Grapheme_Extend # Mn [2] KAITHI SIGN CANDRABINDU..KAITHI SIGN ANUSVARA +110B3..110B6 ; Grapheme_Extend # Mn [4] KAITHI VOWEL SIGN U..KAITHI VOWEL SIGN AI +110B9..110BA ; Grapheme_Extend # Mn [2] KAITHI SIGN VIRAMA..KAITHI SIGN NUKTA +1D165 ; Grapheme_Extend # Mc MUSICAL SYMBOL COMBINING STEM +1D167..1D169 ; Grapheme_Extend # Mn [3] MUSICAL SYMBOL COMBINING TREMOLO-1..MUSICAL SYMBOL COMBINING TREMOLO-3 +1D16E..1D172 ; Grapheme_Extend # Mc [5] MUSICAL SYMBOL COMBINING FLAG-1..MUSICAL SYMBOL COMBINING FLAG-5 +1D17B..1D182 ; Grapheme_Extend # Mn [8] MUSICAL SYMBOL COMBINING ACCENT..MUSICAL SYMBOL COMBINING LOURE +1D185..1D18B ; Grapheme_Extend # Mn [7] MUSICAL SYMBOL COMBINING DOIT..MUSICAL SYMBOL COMBINING TRIPLE TONGUE +1D1AA..1D1AD ; Grapheme_Extend # Mn [4] MUSICAL SYMBOL COMBINING DOWN BOW..MUSICAL SYMBOL COMBINING SNAP PIZZICATO +1D242..1D244 ; Grapheme_Extend # Mn [3] COMBINING GREEK MUSICAL TRISEME..COMBINING GREEK MUSICAL PENTASEME +E0100..E01EF ; Grapheme_Extend # Mn [240] VARIATION SELECTOR-17..VARIATION SELECTOR-256 + +# Total code points: 1198 + +# ================================================ + +# Derived Property: Grapheme_Base +# Generated from: [0..10FFFF] - Cc - Cf - Cs - Co - Cn - Zl - Zp - Grapheme_Extend +# Note: depending on an application's interpretation of Co (private use), +# they may be either in Grapheme_Base, or in Grapheme_Extend, or in neither. + +0020 ; Grapheme_Base # Zs SPACE +0021..0023 ; Grapheme_Base # Po [3] EXCLAMATION MARK..NUMBER SIGN +0024 ; Grapheme_Base # Sc DOLLAR SIGN +0025..0027 ; Grapheme_Base # Po [3] PERCENT SIGN..APOSTROPHE +0028 ; Grapheme_Base # Ps LEFT PARENTHESIS +0029 ; Grapheme_Base # Pe RIGHT PARENTHESIS +002A ; Grapheme_Base # Po ASTERISK +002B ; Grapheme_Base # Sm PLUS SIGN +002C ; Grapheme_Base # Po COMMA +002D ; Grapheme_Base # Pd HYPHEN-MINUS +002E..002F ; Grapheme_Base # Po [2] FULL STOP..SOLIDUS +0030..0039 ; Grapheme_Base # Nd [10] DIGIT ZERO..DIGIT NINE +003A..003B ; Grapheme_Base # Po [2] COLON..SEMICOLON +003C..003E ; Grapheme_Base # Sm [3] LESS-THAN SIGN..GREATER-THAN SIGN +003F..0040 ; Grapheme_Base # Po [2] QUESTION MARK..COMMERCIAL AT +0041..005A ; Grapheme_Base # L& [26] LATIN CAPITAL LETTER A..LATIN CAPITAL LETTER Z +005B ; Grapheme_Base # Ps LEFT SQUARE BRACKET +005C ; Grapheme_Base # Po REVERSE SOLIDUS +005D ; Grapheme_Base # Pe RIGHT SQUARE BRACKET +005E ; Grapheme_Base # Sk CIRCUMFLEX ACCENT +005F ; Grapheme_Base # Pc LOW LINE +0060 ; Grapheme_Base # Sk GRAVE ACCENT +0061..007A ; Grapheme_Base # L& [26] LATIN SMALL LETTER A..LATIN SMALL LETTER Z +007B ; Grapheme_Base # Ps LEFT CURLY BRACKET +007C ; Grapheme_Base # Sm VERTICAL LINE +007D ; Grapheme_Base # Pe RIGHT CURLY BRACKET +007E ; Grapheme_Base # Sm TILDE +00A0 ; Grapheme_Base # Zs NO-BREAK SPACE +00A1 ; Grapheme_Base # Po INVERTED EXCLAMATION MARK +00A2..00A5 ; Grapheme_Base # Sc [4] CENT SIGN..YEN SIGN +00A6..00A7 ; Grapheme_Base # So [2] BROKEN BAR..SECTION SIGN +00A8 ; Grapheme_Base # Sk DIAERESIS +00A9 ; Grapheme_Base # So COPYRIGHT SIGN +00AA ; Grapheme_Base # L& FEMININE ORDINAL INDICATOR +00AB ; Grapheme_Base # Pi LEFT-POINTING DOUBLE ANGLE QUOTATION MARK +00AC ; Grapheme_Base # Sm NOT SIGN +00AE ; Grapheme_Base # So REGISTERED SIGN +00AF ; Grapheme_Base # Sk MACRON +00B0 ; Grapheme_Base # So DEGREE SIGN +00B1 ; Grapheme_Base # Sm PLUS-MINUS SIGN +00B2..00B3 ; Grapheme_Base # No [2] SUPERSCRIPT TWO..SUPERSCRIPT THREE +00B4 ; Grapheme_Base # Sk ACUTE ACCENT +00B5 ; Grapheme_Base # L& MICRO SIGN +00B6 ; Grapheme_Base # So PILCROW SIGN +00B7 ; Grapheme_Base # Po MIDDLE DOT +00B8 ; Grapheme_Base # Sk CEDILLA +00B9 ; Grapheme_Base # No SUPERSCRIPT ONE +00BA ; Grapheme_Base # L& MASCULINE ORDINAL INDICATOR +00BB ; Grapheme_Base # Pf RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK +00BC..00BE ; Grapheme_Base # No [3] VULGAR FRACTION ONE QUARTER..VULGAR FRACTION THREE QUARTERS +00BF ; Grapheme_Base # Po INVERTED QUESTION MARK +00C0..00D6 ; Grapheme_Base # L& [23] LATIN CAPITAL LETTER A WITH GRAVE..LATIN CAPITAL LETTER O WITH DIAERESIS +00D7 ; Grapheme_Base # Sm MULTIPLICATION SIGN +00D8..00F6 ; Grapheme_Base # L& [31] LATIN CAPITAL LETTER O WITH STROKE..LATIN SMALL LETTER O WITH DIAERESIS +00F7 ; Grapheme_Base # Sm DIVISION SIGN +00F8..01BA ; Grapheme_Base # L& [195] LATIN SMALL LETTER O WITH STROKE..LATIN SMALL LETTER EZH WITH TAIL +01BB ; Grapheme_Base # Lo LATIN LETTER TWO WITH STROKE +01BC..01BF ; Grapheme_Base # L& [4] LATIN CAPITAL LETTER TONE FIVE..LATIN LETTER WYNN +01C0..01C3 ; Grapheme_Base # Lo [4] LATIN LETTER DENTAL CLICK..LATIN LETTER RETROFLEX CLICK +01C4..0293 ; Grapheme_Base # L& [208] LATIN CAPITAL LETTER DZ WITH CARON..LATIN SMALL LETTER EZH WITH CURL +0294 ; Grapheme_Base # Lo LATIN LETTER GLOTTAL STOP +0295..02AF ; Grapheme_Base # L& [27] LATIN LETTER PHARYNGEAL VOICED FRICATIVE..LATIN SMALL LETTER TURNED H WITH FISHHOOK AND TAIL +02B0..02C1 ; Grapheme_Base # Lm [18] MODIFIER LETTER SMALL H..MODIFIER LETTER REVERSED GLOTTAL STOP +02C2..02C5 ; Grapheme_Base # Sk [4] MODIFIER LETTER LEFT ARROWHEAD..MODIFIER LETTER DOWN ARROWHEAD +02C6..02D1 ; Grapheme_Base # Lm [12] MODIFIER LETTER CIRCUMFLEX ACCENT..MODIFIER LETTER HALF TRIANGULAR COLON +02D2..02DF ; Grapheme_Base # Sk [14] MODIFIER LETTER CENTRED RIGHT HALF RING..MODIFIER LETTER CROSS ACCENT +02E0..02E4 ; Grapheme_Base # Lm [5] MODIFIER LETTER SMALL GAMMA..MODIFIER LETTER SMALL REVERSED GLOTTAL STOP +02E5..02EB ; Grapheme_Base # Sk [7] MODIFIER LETTER EXTRA-HIGH TONE BAR..MODIFIER LETTER YANG DEPARTING TONE MARK +02EC ; Grapheme_Base # Lm MODIFIER LETTER VOICING +02ED ; Grapheme_Base # Sk MODIFIER LETTER UNASPIRATED +02EE ; Grapheme_Base # Lm MODIFIER LETTER DOUBLE APOSTROPHE +02EF..02FF ; Grapheme_Base # Sk [17] MODIFIER LETTER LOW DOWN ARROWHEAD..MODIFIER LETTER LOW LEFT ARROW +0370..0373 ; Grapheme_Base # L& [4] GREEK CAPITAL LETTER HETA..GREEK SMALL LETTER ARCHAIC SAMPI +0374 ; Grapheme_Base # Lm GREEK NUMERAL SIGN +0375 ; Grapheme_Base # Sk GREEK LOWER NUMERAL SIGN +0376..0377 ; Grapheme_Base # L& [2] GREEK CAPITAL LETTER PAMPHYLIAN DIGAMMA..GREEK SMALL LETTER PAMPHYLIAN DIGAMMA +037A ; Grapheme_Base # Lm GREEK YPOGEGRAMMENI +037B..037D ; Grapheme_Base # L& [3] GREEK SMALL REVERSED LUNATE SIGMA SYMBOL..GREEK SMALL REVERSED DOTTED LUNATE SIGMA SYMBOL +037E ; Grapheme_Base # Po GREEK QUESTION MARK +0384..0385 ; Grapheme_Base # Sk [2] GREEK TONOS..GREEK DIALYTIKA TONOS +0386 ; Grapheme_Base # L& GREEK CAPITAL LETTER ALPHA WITH TONOS +0387 ; Grapheme_Base # Po GREEK ANO TELEIA +0388..038A ; Grapheme_Base # L& [3] GREEK CAPITAL LETTER EPSILON WITH TONOS..GREEK CAPITAL LETTER IOTA WITH TONOS +038C ; Grapheme_Base # L& GREEK CAPITAL LETTER OMICRON WITH TONOS +038E..03A1 ; Grapheme_Base # L& [20] GREEK CAPITAL LETTER UPSILON WITH TONOS..GREEK CAPITAL LETTER RHO +03A3..03F5 ; Grapheme_Base # L& [83] GREEK CAPITAL LETTER SIGMA..GREEK LUNATE EPSILON SYMBOL +03F6 ; Grapheme_Base # Sm GREEK REVERSED LUNATE EPSILON SYMBOL +03F7..0481 ; Grapheme_Base # L& [139] GREEK CAPITAL LETTER SHO..CYRILLIC SMALL LETTER KOPPA +0482 ; Grapheme_Base # So CYRILLIC THOUSANDS SIGN +048A..0525 ; Grapheme_Base # L& [156] CYRILLIC CAPITAL LETTER SHORT I WITH TAIL..CYRILLIC SMALL LETTER PE WITH DESCENDER +0531..0556 ; Grapheme_Base # L& [38] ARMENIAN CAPITAL LETTER AYB..ARMENIAN CAPITAL LETTER FEH +0559 ; Grapheme_Base # Lm ARMENIAN MODIFIER LETTER LEFT HALF RING +055A..055F ; Grapheme_Base # Po [6] ARMENIAN APOSTROPHE..ARMENIAN ABBREVIATION MARK +0561..0587 ; Grapheme_Base # L& [39] ARMENIAN SMALL LETTER AYB..ARMENIAN SMALL LIGATURE ECH YIWN +0589 ; Grapheme_Base # Po ARMENIAN FULL STOP +058A ; Grapheme_Base # Pd ARMENIAN HYPHEN +05BE ; Grapheme_Base # Pd HEBREW PUNCTUATION MAQAF +05C0 ; Grapheme_Base # Po HEBREW PUNCTUATION PASEQ +05C3 ; Grapheme_Base # Po HEBREW PUNCTUATION SOF PASUQ +05C6 ; Grapheme_Base # Po HEBREW PUNCTUATION NUN HAFUKHA +05D0..05EA ; Grapheme_Base # Lo [27] HEBREW LETTER ALEF..HEBREW LETTER TAV +05F0..05F2 ; Grapheme_Base # Lo [3] HEBREW LIGATURE YIDDISH DOUBLE VAV..HEBREW LIGATURE YIDDISH DOUBLE YOD +05F3..05F4 ; Grapheme_Base # Po [2] HEBREW PUNCTUATION GERESH..HEBREW PUNCTUATION GERSHAYIM +0606..0608 ; Grapheme_Base # Sm [3] ARABIC-INDIC CUBE ROOT..ARABIC RAY +0609..060A ; Grapheme_Base # Po [2] ARABIC-INDIC PER MILLE SIGN..ARABIC-INDIC PER TEN THOUSAND SIGN +060B ; Grapheme_Base # Sc AFGHANI SIGN +060C..060D ; Grapheme_Base # Po [2] ARABIC COMMA..ARABIC DATE SEPARATOR +060E..060F ; Grapheme_Base # So [2] ARABIC POETIC VERSE SIGN..ARABIC SIGN MISRA +061B ; Grapheme_Base # Po ARABIC SEMICOLON +061E..061F ; Grapheme_Base # Po [2] ARABIC TRIPLE DOT PUNCTUATION MARK..ARABIC QUESTION MARK +0621..063F ; Grapheme_Base # Lo [31] ARABIC LETTER HAMZA..ARABIC LETTER FARSI YEH WITH THREE DOTS ABOVE +0640 ; Grapheme_Base # Lm ARABIC TATWEEL +0641..064A ; Grapheme_Base # Lo [10] ARABIC LETTER FEH..ARABIC LETTER YEH +0660..0669 ; Grapheme_Base # Nd [10] ARABIC-INDIC DIGIT ZERO..ARABIC-INDIC DIGIT NINE +066A..066D ; Grapheme_Base # Po [4] ARABIC PERCENT SIGN..ARABIC FIVE POINTED STAR +066E..066F ; Grapheme_Base # Lo [2] ARABIC LETTER DOTLESS BEH..ARABIC LETTER DOTLESS QAF +0671..06D3 ; Grapheme_Base # Lo [99] ARABIC LETTER ALEF WASLA..ARABIC LETTER YEH BARREE WITH HAMZA ABOVE +06D4 ; Grapheme_Base # Po ARABIC FULL STOP +06D5 ; Grapheme_Base # Lo ARABIC LETTER AE +06E5..06E6 ; Grapheme_Base # Lm [2] ARABIC SMALL WAW..ARABIC SMALL YEH +06E9 ; Grapheme_Base # So ARABIC PLACE OF SAJDAH +06EE..06EF ; Grapheme_Base # Lo [2] ARABIC LETTER DAL WITH INVERTED V..ARABIC LETTER REH WITH INVERTED V +06F0..06F9 ; Grapheme_Base # Nd [10] EXTENDED ARABIC-INDIC DIGIT ZERO..EXTENDED ARABIC-INDIC DIGIT NINE +06FA..06FC ; Grapheme_Base # Lo [3] ARABIC LETTER SHEEN WITH DOT BELOW..ARABIC LETTER GHAIN WITH DOT BELOW +06FD..06FE ; Grapheme_Base # So [2] ARABIC SIGN SINDHI AMPERSAND..ARABIC SIGN SINDHI POSTPOSITION MEN +06FF ; Grapheme_Base # Lo ARABIC LETTER HEH WITH INVERTED V +0700..070D ; Grapheme_Base # Po [14] SYRIAC END OF PARAGRAPH..SYRIAC HARKLEAN ASTERISCUS +0710 ; Grapheme_Base # Lo SYRIAC LETTER ALAPH +0712..072F ; Grapheme_Base # Lo [30] SYRIAC LETTER BETH..SYRIAC LETTER PERSIAN DHALATH +074D..07A5 ; Grapheme_Base # Lo [89] SYRIAC LETTER SOGDIAN ZHAIN..THAANA LETTER WAAVU +07B1 ; Grapheme_Base # Lo THAANA LETTER NAA +07C0..07C9 ; Grapheme_Base # Nd [10] NKO DIGIT ZERO..NKO DIGIT NINE +07CA..07EA ; Grapheme_Base # Lo [33] NKO LETTER A..NKO LETTER JONA RA +07F4..07F5 ; Grapheme_Base # Lm [2] NKO HIGH TONE APOSTROPHE..NKO LOW TONE APOSTROPHE +07F6 ; Grapheme_Base # So NKO SYMBOL OO DENNEN +07F7..07F9 ; Grapheme_Base # Po [3] NKO SYMBOL GBAKURUNEN..NKO EXCLAMATION MARK +07FA ; Grapheme_Base # Lm NKO LAJANYALAN +0800..0815 ; Grapheme_Base # Lo [22] SAMARITAN LETTER ALAF..SAMARITAN LETTER TAAF +081A ; Grapheme_Base # Lm SAMARITAN MODIFIER LETTER EPENTHETIC YUT +0824 ; Grapheme_Base # Lm SAMARITAN MODIFIER LETTER SHORT A +0828 ; Grapheme_Base # Lm SAMARITAN MODIFIER LETTER I +0830..083E ; Grapheme_Base # Po [15] SAMARITAN PUNCTUATION NEQUDAA..SAMARITAN PUNCTUATION ANNAAU +0903 ; Grapheme_Base # Mc DEVANAGARI SIGN VISARGA +0904..0939 ; Grapheme_Base # Lo [54] DEVANAGARI LETTER SHORT A..DEVANAGARI LETTER HA +093D ; Grapheme_Base # Lo DEVANAGARI SIGN AVAGRAHA +093E..0940 ; Grapheme_Base # Mc [3] DEVANAGARI VOWEL SIGN AA..DEVANAGARI VOWEL SIGN II +0949..094C ; Grapheme_Base # Mc [4] DEVANAGARI VOWEL SIGN CANDRA O..DEVANAGARI VOWEL SIGN AU +094E ; Grapheme_Base # Mc DEVANAGARI VOWEL SIGN PRISHTHAMATRA E +0950 ; Grapheme_Base # Lo DEVANAGARI OM +0958..0961 ; Grapheme_Base # Lo [10] DEVANAGARI LETTER QA..DEVANAGARI LETTER VOCALIC LL +0964..0965 ; Grapheme_Base # Po [2] DEVANAGARI DANDA..DEVANAGARI DOUBLE DANDA +0966..096F ; Grapheme_Base # Nd [10] DEVANAGARI DIGIT ZERO..DEVANAGARI DIGIT NINE +0970 ; Grapheme_Base # Po DEVANAGARI ABBREVIATION SIGN +0971 ; Grapheme_Base # Lm DEVANAGARI SIGN HIGH SPACING DOT +0972 ; Grapheme_Base # Lo DEVANAGARI LETTER CANDRA A +0979..097F ; Grapheme_Base # Lo [7] DEVANAGARI LETTER ZHA..DEVANAGARI LETTER BBA +0982..0983 ; Grapheme_Base # Mc [2] BENGALI SIGN ANUSVARA..BENGALI SIGN VISARGA +0985..098C ; Grapheme_Base # Lo [8] BENGALI LETTER A..BENGALI LETTER VOCALIC L +098F..0990 ; Grapheme_Base # Lo [2] BENGALI LETTER E..BENGALI LETTER AI +0993..09A8 ; Grapheme_Base # Lo [22] BENGALI LETTER O..BENGALI LETTER NA +09AA..09B0 ; Grapheme_Base # Lo [7] BENGALI LETTER PA..BENGALI LETTER RA +09B2 ; Grapheme_Base # Lo BENGALI LETTER LA +09B6..09B9 ; Grapheme_Base # Lo [4] BENGALI LETTER SHA..BENGALI LETTER HA +09BD ; Grapheme_Base # Lo BENGALI SIGN AVAGRAHA +09BF..09C0 ; Grapheme_Base # Mc [2] BENGALI VOWEL SIGN I..BENGALI VOWEL SIGN II +09C7..09C8 ; Grapheme_Base # Mc [2] BENGALI VOWEL SIGN E..BENGALI VOWEL SIGN AI +09CB..09CC ; Grapheme_Base # Mc [2] BENGALI VOWEL SIGN O..BENGALI VOWEL SIGN AU +09CE ; Grapheme_Base # Lo BENGALI LETTER KHANDA TA +09DC..09DD ; Grapheme_Base # Lo [2] BENGALI LETTER RRA..BENGALI LETTER RHA +09DF..09E1 ; Grapheme_Base # Lo [3] BENGALI LETTER YYA..BENGALI LETTER VOCALIC LL +09E6..09EF ; Grapheme_Base # Nd [10] BENGALI DIGIT ZERO..BENGALI DIGIT NINE +09F0..09F1 ; Grapheme_Base # Lo [2] BENGALI LETTER RA WITH MIDDLE DIAGONAL..BENGALI LETTER RA WITH LOWER DIAGONAL +09F2..09F3 ; Grapheme_Base # Sc [2] BENGALI RUPEE MARK..BENGALI RUPEE SIGN +09F4..09F9 ; Grapheme_Base # No [6] BENGALI CURRENCY NUMERATOR ONE..BENGALI CURRENCY DENOMINATOR SIXTEEN +09FA ; Grapheme_Base # So BENGALI ISSHAR +09FB ; Grapheme_Base # Sc BENGALI GANDA MARK +0A03 ; Grapheme_Base # Mc GURMUKHI SIGN VISARGA +0A05..0A0A ; Grapheme_Base # Lo [6] GURMUKHI LETTER A..GURMUKHI LETTER UU +0A0F..0A10 ; Grapheme_Base # Lo [2] GURMUKHI LETTER EE..GURMUKHI LETTER AI +0A13..0A28 ; Grapheme_Base # Lo [22] GURMUKHI LETTER OO..GURMUKHI LETTER NA +0A2A..0A30 ; Grapheme_Base # Lo [7] GURMUKHI LETTER PA..GURMUKHI LETTER RA +0A32..0A33 ; Grapheme_Base # Lo [2] GURMUKHI LETTER LA..GURMUKHI LETTER LLA +0A35..0A36 ; Grapheme_Base # Lo [2] GURMUKHI LETTER VA..GURMUKHI LETTER SHA +0A38..0A39 ; Grapheme_Base # Lo [2] GURMUKHI LETTER SA..GURMUKHI LETTER HA +0A3E..0A40 ; Grapheme_Base # Mc [3] GURMUKHI VOWEL SIGN AA..GURMUKHI VOWEL SIGN II +0A59..0A5C ; Grapheme_Base # Lo [4] GURMUKHI LETTER KHHA..GURMUKHI LETTER RRA +0A5E ; Grapheme_Base # Lo GURMUKHI LETTER FA +0A66..0A6F ; Grapheme_Base # Nd [10] GURMUKHI DIGIT ZERO..GURMUKHI DIGIT NINE +0A72..0A74 ; Grapheme_Base # Lo [3] GURMUKHI IRI..GURMUKHI EK ONKAR +0A83 ; Grapheme_Base # Mc GUJARATI SIGN VISARGA +0A85..0A8D ; Grapheme_Base # Lo [9] GUJARATI LETTER A..GUJARATI VOWEL CANDRA E +0A8F..0A91 ; Grapheme_Base # Lo [3] GUJARATI LETTER E..GUJARATI VOWEL CANDRA O +0A93..0AA8 ; Grapheme_Base # Lo [22] GUJARATI LETTER O..GUJARATI LETTER NA +0AAA..0AB0 ; Grapheme_Base # Lo [7] GUJARATI LETTER PA..GUJARATI LETTER RA +0AB2..0AB3 ; Grapheme_Base # Lo [2] GUJARATI LETTER LA..GUJARATI LETTER LLA +0AB5..0AB9 ; Grapheme_Base # Lo [5] GUJARATI LETTER VA..GUJARATI LETTER HA +0ABD ; Grapheme_Base # Lo GUJARATI SIGN AVAGRAHA +0ABE..0AC0 ; Grapheme_Base # Mc [3] GUJARATI VOWEL SIGN AA..GUJARATI VOWEL SIGN II +0AC9 ; Grapheme_Base # Mc GUJARATI VOWEL SIGN CANDRA O +0ACB..0ACC ; Grapheme_Base # Mc [2] GUJARATI VOWEL SIGN O..GUJARATI VOWEL SIGN AU +0AD0 ; Grapheme_Base # Lo GUJARATI OM +0AE0..0AE1 ; Grapheme_Base # Lo [2] GUJARATI LETTER VOCALIC RR..GUJARATI LETTER VOCALIC LL +0AE6..0AEF ; Grapheme_Base # Nd [10] GUJARATI DIGIT ZERO..GUJARATI DIGIT NINE +0AF1 ; Grapheme_Base # Sc GUJARATI RUPEE SIGN +0B02..0B03 ; Grapheme_Base # Mc [2] ORIYA SIGN ANUSVARA..ORIYA SIGN VISARGA +0B05..0B0C ; Grapheme_Base # Lo [8] ORIYA LETTER A..ORIYA LETTER VOCALIC L +0B0F..0B10 ; Grapheme_Base # Lo [2] ORIYA LETTER E..ORIYA LETTER AI +0B13..0B28 ; Grapheme_Base # Lo [22] ORIYA LETTER O..ORIYA LETTER NA +0B2A..0B30 ; Grapheme_Base # Lo [7] ORIYA LETTER PA..ORIYA LETTER RA +0B32..0B33 ; Grapheme_Base # Lo [2] ORIYA LETTER LA..ORIYA LETTER LLA +0B35..0B39 ; Grapheme_Base # Lo [5] ORIYA LETTER VA..ORIYA LETTER HA +0B3D ; Grapheme_Base # Lo ORIYA SIGN AVAGRAHA +0B40 ; Grapheme_Base # Mc ORIYA VOWEL SIGN II +0B47..0B48 ; Grapheme_Base # Mc [2] ORIYA VOWEL SIGN E..ORIYA VOWEL SIGN AI +0B4B..0B4C ; Grapheme_Base # Mc [2] ORIYA VOWEL SIGN O..ORIYA VOWEL SIGN AU +0B5C..0B5D ; Grapheme_Base # Lo [2] ORIYA LETTER RRA..ORIYA LETTER RHA +0B5F..0B61 ; Grapheme_Base # Lo [3] ORIYA LETTER YYA..ORIYA LETTER VOCALIC LL +0B66..0B6F ; Grapheme_Base # Nd [10] ORIYA DIGIT ZERO..ORIYA DIGIT NINE +0B70 ; Grapheme_Base # So ORIYA ISSHAR +0B71 ; Grapheme_Base # Lo ORIYA LETTER WA +0B83 ; Grapheme_Base # Lo TAMIL SIGN VISARGA +0B85..0B8A ; Grapheme_Base # Lo [6] TAMIL LETTER A..TAMIL LETTER UU +0B8E..0B90 ; Grapheme_Base # Lo [3] TAMIL LETTER E..TAMIL LETTER AI +0B92..0B95 ; Grapheme_Base # Lo [4] TAMIL LETTER O..TAMIL LETTER KA +0B99..0B9A ; Grapheme_Base # Lo [2] TAMIL LETTER NGA..TAMIL LETTER CA +0B9C ; Grapheme_Base # Lo TAMIL LETTER JA +0B9E..0B9F ; Grapheme_Base # Lo [2] TAMIL LETTER NYA..TAMIL LETTER TTA +0BA3..0BA4 ; Grapheme_Base # Lo [2] TAMIL LETTER NNA..TAMIL LETTER TA +0BA8..0BAA ; Grapheme_Base # Lo [3] TAMIL LETTER NA..TAMIL LETTER PA +0BAE..0BB9 ; Grapheme_Base # Lo [12] TAMIL LETTER MA..TAMIL LETTER HA +0BBF ; Grapheme_Base # Mc TAMIL VOWEL SIGN I +0BC1..0BC2 ; Grapheme_Base # Mc [2] TAMIL VOWEL SIGN U..TAMIL VOWEL SIGN UU +0BC6..0BC8 ; Grapheme_Base # Mc [3] TAMIL VOWEL SIGN E..TAMIL VOWEL SIGN AI +0BCA..0BCC ; Grapheme_Base # Mc [3] TAMIL VOWEL SIGN O..TAMIL VOWEL SIGN AU +0BD0 ; Grapheme_Base # Lo TAMIL OM +0BE6..0BEF ; Grapheme_Base # Nd [10] TAMIL DIGIT ZERO..TAMIL DIGIT NINE +0BF0..0BF2 ; Grapheme_Base # No [3] TAMIL NUMBER TEN..TAMIL NUMBER ONE THOUSAND +0BF3..0BF8 ; Grapheme_Base # So [6] TAMIL DAY SIGN..TAMIL AS ABOVE SIGN +0BF9 ; Grapheme_Base # Sc TAMIL RUPEE SIGN +0BFA ; Grapheme_Base # So TAMIL NUMBER SIGN +0C01..0C03 ; Grapheme_Base # Mc [3] TELUGU SIGN CANDRABINDU..TELUGU SIGN VISARGA +0C05..0C0C ; Grapheme_Base # Lo [8] TELUGU LETTER A..TELUGU LETTER VOCALIC L +0C0E..0C10 ; Grapheme_Base # Lo [3] TELUGU LETTER E..TELUGU LETTER AI +0C12..0C28 ; Grapheme_Base # Lo [23] TELUGU LETTER O..TELUGU LETTER NA +0C2A..0C33 ; Grapheme_Base # Lo [10] TELUGU LETTER PA..TELUGU LETTER LLA +0C35..0C39 ; Grapheme_Base # Lo [5] TELUGU LETTER VA..TELUGU LETTER HA +0C3D ; Grapheme_Base # Lo TELUGU SIGN AVAGRAHA +0C41..0C44 ; Grapheme_Base # Mc [4] TELUGU VOWEL SIGN U..TELUGU VOWEL SIGN VOCALIC RR +0C58..0C59 ; Grapheme_Base # Lo [2] TELUGU LETTER TSA..TELUGU LETTER DZA +0C60..0C61 ; Grapheme_Base # Lo [2] TELUGU LETTER VOCALIC RR..TELUGU LETTER VOCALIC LL +0C66..0C6F ; Grapheme_Base # Nd [10] TELUGU DIGIT ZERO..TELUGU DIGIT NINE +0C78..0C7E ; Grapheme_Base # No [7] TELUGU FRACTION DIGIT ZERO FOR ODD POWERS OF FOUR..TELUGU FRACTION DIGIT THREE FOR EVEN POWERS OF FOUR +0C7F ; Grapheme_Base # So TELUGU SIGN TUUMU +0C82..0C83 ; Grapheme_Base # Mc [2] KANNADA SIGN ANUSVARA..KANNADA SIGN VISARGA +0C85..0C8C ; Grapheme_Base # Lo [8] KANNADA LETTER A..KANNADA LETTER VOCALIC L +0C8E..0C90 ; Grapheme_Base # Lo [3] KANNADA LETTER E..KANNADA LETTER AI +0C92..0CA8 ; Grapheme_Base # Lo [23] KANNADA LETTER O..KANNADA LETTER NA +0CAA..0CB3 ; Grapheme_Base # Lo [10] KANNADA LETTER PA..KANNADA LETTER LLA +0CB5..0CB9 ; Grapheme_Base # Lo [5] KANNADA LETTER VA..KANNADA LETTER HA +0CBD ; Grapheme_Base # Lo KANNADA SIGN AVAGRAHA +0CBE ; Grapheme_Base # Mc KANNADA VOWEL SIGN AA +0CC0..0CC1 ; Grapheme_Base # Mc [2] KANNADA VOWEL SIGN II..KANNADA VOWEL SIGN U +0CC3..0CC4 ; Grapheme_Base # Mc [2] KANNADA VOWEL SIGN VOCALIC R..KANNADA VOWEL SIGN VOCALIC RR +0CC7..0CC8 ; Grapheme_Base # Mc [2] KANNADA VOWEL SIGN EE..KANNADA VOWEL SIGN AI +0CCA..0CCB ; Grapheme_Base # Mc [2] KANNADA VOWEL SIGN O..KANNADA VOWEL SIGN OO +0CDE ; Grapheme_Base # Lo KANNADA LETTER FA +0CE0..0CE1 ; Grapheme_Base # Lo [2] KANNADA LETTER VOCALIC RR..KANNADA LETTER VOCALIC LL +0CE6..0CEF ; Grapheme_Base # Nd [10] KANNADA DIGIT ZERO..KANNADA DIGIT NINE +0CF1..0CF2 ; Grapheme_Base # So [2] KANNADA SIGN JIHVAMULIYA..KANNADA SIGN UPADHMANIYA +0D02..0D03 ; Grapheme_Base # Mc [2] MALAYALAM SIGN ANUSVARA..MALAYALAM SIGN VISARGA +0D05..0D0C ; Grapheme_Base # Lo [8] MALAYALAM LETTER A..MALAYALAM LETTER VOCALIC L +0D0E..0D10 ; Grapheme_Base # Lo [3] MALAYALAM LETTER E..MALAYALAM LETTER AI +0D12..0D28 ; Grapheme_Base # Lo [23] MALAYALAM LETTER O..MALAYALAM LETTER NA +0D2A..0D39 ; Grapheme_Base # Lo [16] MALAYALAM LETTER PA..MALAYALAM LETTER HA +0D3D ; Grapheme_Base # Lo MALAYALAM SIGN AVAGRAHA +0D3F..0D40 ; Grapheme_Base # Mc [2] MALAYALAM VOWEL SIGN I..MALAYALAM VOWEL SIGN II +0D46..0D48 ; Grapheme_Base # Mc [3] MALAYALAM VOWEL SIGN E..MALAYALAM VOWEL SIGN AI +0D4A..0D4C ; Grapheme_Base # Mc [3] MALAYALAM VOWEL SIGN O..MALAYALAM VOWEL SIGN AU +0D60..0D61 ; Grapheme_Base # Lo [2] MALAYALAM LETTER VOCALIC RR..MALAYALAM LETTER VOCALIC LL +0D66..0D6F ; Grapheme_Base # Nd [10] MALAYALAM DIGIT ZERO..MALAYALAM DIGIT NINE +0D70..0D75 ; Grapheme_Base # No [6] MALAYALAM NUMBER TEN..MALAYALAM FRACTION THREE QUARTERS +0D79 ; Grapheme_Base # So MALAYALAM DATE MARK +0D7A..0D7F ; Grapheme_Base # Lo [6] MALAYALAM LETTER CHILLU NN..MALAYALAM LETTER CHILLU K +0D82..0D83 ; Grapheme_Base # Mc [2] SINHALA SIGN ANUSVARAYA..SINHALA SIGN VISARGAYA +0D85..0D96 ; Grapheme_Base # Lo [18] SINHALA LETTER AYANNA..SINHALA LETTER AUYANNA +0D9A..0DB1 ; Grapheme_Base # Lo [24] SINHALA LETTER ALPAPRAANA KAYANNA..SINHALA LETTER DANTAJA NAYANNA +0DB3..0DBB ; Grapheme_Base # Lo [9] SINHALA LETTER SANYAKA DAYANNA..SINHALA LETTER RAYANNA +0DBD ; Grapheme_Base # Lo SINHALA LETTER DANTAJA LAYANNA +0DC0..0DC6 ; Grapheme_Base # Lo [7] SINHALA LETTER VAYANNA..SINHALA LETTER FAYANNA +0DD0..0DD1 ; Grapheme_Base # Mc [2] SINHALA VOWEL SIGN KETTI AEDA-PILLA..SINHALA VOWEL SIGN DIGA AEDA-PILLA +0DD8..0DDE ; Grapheme_Base # Mc [7] SINHALA VOWEL SIGN GAETTA-PILLA..SINHALA VOWEL SIGN KOMBUVA HAA GAYANUKITTA +0DF2..0DF3 ; Grapheme_Base # Mc [2] SINHALA VOWEL SIGN DIGA GAETTA-PILLA..SINHALA VOWEL SIGN DIGA GAYANUKITTA +0DF4 ; Grapheme_Base # Po SINHALA PUNCTUATION KUNDDALIYA +0E01..0E30 ; Grapheme_Base # Lo [48] THAI CHARACTER KO KAI..THAI CHARACTER SARA A +0E32..0E33 ; Grapheme_Base # Lo [2] THAI CHARACTER SARA AA..THAI CHARACTER SARA AM +0E3F ; Grapheme_Base # Sc THAI CURRENCY SYMBOL BAHT +0E40..0E45 ; Grapheme_Base # Lo [6] THAI CHARACTER SARA E..THAI CHARACTER LAKKHANGYAO +0E46 ; Grapheme_Base # Lm THAI CHARACTER MAIYAMOK +0E4F ; Grapheme_Base # Po THAI CHARACTER FONGMAN +0E50..0E59 ; Grapheme_Base # Nd [10] THAI DIGIT ZERO..THAI DIGIT NINE +0E5A..0E5B ; Grapheme_Base # Po [2] THAI CHARACTER ANGKHANKHU..THAI CHARACTER KHOMUT +0E81..0E82 ; Grapheme_Base # Lo [2] LAO LETTER KO..LAO LETTER KHO SUNG +0E84 ; Grapheme_Base # Lo LAO LETTER KHO TAM +0E87..0E88 ; Grapheme_Base # Lo [2] LAO LETTER NGO..LAO LETTER CO +0E8A ; Grapheme_Base # Lo LAO LETTER SO TAM +0E8D ; Grapheme_Base # Lo LAO LETTER NYO +0E94..0E97 ; Grapheme_Base # Lo [4] LAO LETTER DO..LAO LETTER THO TAM +0E99..0E9F ; Grapheme_Base # Lo [7] LAO LETTER NO..LAO LETTER FO SUNG +0EA1..0EA3 ; Grapheme_Base # Lo [3] LAO LETTER MO..LAO LETTER LO LING +0EA5 ; Grapheme_Base # Lo LAO LETTER LO LOOT +0EA7 ; Grapheme_Base # Lo LAO LETTER WO +0EAA..0EAB ; Grapheme_Base # Lo [2] LAO LETTER SO SUNG..LAO LETTER HO SUNG +0EAD..0EB0 ; Grapheme_Base # Lo [4] LAO LETTER O..LAO VOWEL SIGN A +0EB2..0EB3 ; Grapheme_Base # Lo [2] LAO VOWEL SIGN AA..LAO VOWEL SIGN AM +0EBD ; Grapheme_Base # Lo LAO SEMIVOWEL SIGN NYO +0EC0..0EC4 ; Grapheme_Base # Lo [5] LAO VOWEL SIGN E..LAO VOWEL SIGN AI +0EC6 ; Grapheme_Base # Lm LAO KO LA +0ED0..0ED9 ; Grapheme_Base # Nd [10] LAO DIGIT ZERO..LAO DIGIT NINE +0EDC..0EDD ; Grapheme_Base # Lo [2] LAO HO NO..LAO HO MO +0F00 ; Grapheme_Base # Lo TIBETAN SYLLABLE OM +0F01..0F03 ; Grapheme_Base # So [3] TIBETAN MARK GTER YIG MGO TRUNCATED A..TIBETAN MARK GTER YIG MGO -UM GTER TSHEG MA +0F04..0F12 ; Grapheme_Base # Po [15] TIBETAN MARK INITIAL YIG MGO MDUN MA..TIBETAN MARK RGYA GRAM SHAD +0F13..0F17 ; Grapheme_Base # So [5] TIBETAN MARK CARET -DZUD RTAGS ME LONG CAN..TIBETAN ASTROLOGICAL SIGN SGRA GCAN -CHAR RTAGS +0F1A..0F1F ; Grapheme_Base # So [6] TIBETAN SIGN RDEL DKAR GCIG..TIBETAN SIGN RDEL DKAR RDEL NAG +0F20..0F29 ; Grapheme_Base # Nd [10] TIBETAN DIGIT ZERO..TIBETAN DIGIT NINE +0F2A..0F33 ; Grapheme_Base # No [10] TIBETAN DIGIT HALF ONE..TIBETAN DIGIT HALF ZERO +0F34 ; Grapheme_Base # So TIBETAN MARK BSDUS RTAGS +0F36 ; Grapheme_Base # So TIBETAN MARK CARET -DZUD RTAGS BZHI MIG CAN +0F38 ; Grapheme_Base # So TIBETAN MARK CHE MGO +0F3A ; Grapheme_Base # Ps TIBETAN MARK GUG RTAGS GYON +0F3B ; Grapheme_Base # Pe TIBETAN MARK GUG RTAGS GYAS +0F3C ; Grapheme_Base # Ps TIBETAN MARK ANG KHANG GYON +0F3D ; Grapheme_Base # Pe TIBETAN MARK ANG KHANG GYAS +0F3E..0F3F ; Grapheme_Base # Mc [2] TIBETAN SIGN YAR TSHES..TIBETAN SIGN MAR TSHES +0F40..0F47 ; Grapheme_Base # Lo [8] TIBETAN LETTER KA..TIBETAN LETTER JA +0F49..0F6C ; Grapheme_Base # Lo [36] TIBETAN LETTER NYA..TIBETAN LETTER RRA +0F7F ; Grapheme_Base # Mc TIBETAN SIGN RNAM BCAD +0F85 ; Grapheme_Base # Po TIBETAN MARK PALUTA +0F88..0F8B ; Grapheme_Base # Lo [4] TIBETAN SIGN LCE TSA CAN..TIBETAN SIGN GRU MED RGYINGS +0FBE..0FC5 ; Grapheme_Base # So [8] TIBETAN KU RU KHA..TIBETAN SYMBOL RDO RJE +0FC7..0FCC ; Grapheme_Base # So [6] TIBETAN SYMBOL RDO RJE RGYA GRAM..TIBETAN SYMBOL NOR BU BZHI -KHYIL +0FCE..0FCF ; Grapheme_Base # So [2] TIBETAN SIGN RDEL NAG RDEL DKAR..TIBETAN SIGN RDEL NAG GSUM +0FD0..0FD4 ; Grapheme_Base # Po [5] TIBETAN MARK BSKA- SHOG GI MGO RGYAN..TIBETAN MARK CLOSING BRDA RNYING YIG MGO SGAB MA +0FD5..0FD8 ; Grapheme_Base # So [4] RIGHT-FACING SVASTI SIGN..LEFT-FACING SVASTI SIGN WITH DOTS +1000..102A ; Grapheme_Base # Lo [43] MYANMAR LETTER KA..MYANMAR LETTER AU +102B..102C ; Grapheme_Base # Mc [2] MYANMAR VOWEL SIGN TALL AA..MYANMAR VOWEL SIGN AA +1031 ; Grapheme_Base # Mc MYANMAR VOWEL SIGN E +1038 ; Grapheme_Base # Mc MYANMAR SIGN VISARGA +103B..103C ; Grapheme_Base # Mc [2] MYANMAR CONSONANT SIGN MEDIAL YA..MYANMAR CONSONANT SIGN MEDIAL RA +103F ; Grapheme_Base # Lo MYANMAR LETTER GREAT SA +1040..1049 ; Grapheme_Base # Nd [10] MYANMAR DIGIT ZERO..MYANMAR DIGIT NINE +104A..104F ; Grapheme_Base # Po [6] MYANMAR SIGN LITTLE SECTION..MYANMAR SYMBOL GENITIVE +1050..1055 ; Grapheme_Base # Lo [6] MYANMAR LETTER SHA..MYANMAR LETTER VOCALIC LL +1056..1057 ; Grapheme_Base # Mc [2] MYANMAR VOWEL SIGN VOCALIC R..MYANMAR VOWEL SIGN VOCALIC RR +105A..105D ; Grapheme_Base # Lo [4] MYANMAR LETTER MON NGA..MYANMAR LETTER MON BBE +1061 ; Grapheme_Base # Lo MYANMAR LETTER SGAW KAREN SHA +1062..1064 ; Grapheme_Base # Mc [3] MYANMAR VOWEL SIGN SGAW KAREN EU..MYANMAR TONE MARK SGAW KAREN KE PHO +1065..1066 ; Grapheme_Base # Lo [2] MYANMAR LETTER WESTERN PWO KAREN THA..MYANMAR LETTER WESTERN PWO KAREN PWA +1067..106D ; Grapheme_Base # Mc [7] MYANMAR VOWEL SIGN WESTERN PWO KAREN EU..MYANMAR SIGN WESTERN PWO KAREN TONE-5 +106E..1070 ; Grapheme_Base # Lo [3] MYANMAR LETTER EASTERN PWO KAREN NNA..MYANMAR LETTER EASTERN PWO KAREN GHWA +1075..1081 ; Grapheme_Base # Lo [13] MYANMAR LETTER SHAN KA..MYANMAR LETTER SHAN HA +1083..1084 ; Grapheme_Base # Mc [2] MYANMAR VOWEL SIGN SHAN AA..MYANMAR VOWEL SIGN SHAN E +1087..108C ; Grapheme_Base # Mc [6] MYANMAR SIGN SHAN TONE-2..MYANMAR SIGN SHAN COUNCIL TONE-3 +108E ; Grapheme_Base # Lo MYANMAR LETTER RUMAI PALAUNG FA +108F ; Grapheme_Base # Mc MYANMAR SIGN RUMAI PALAUNG TONE-5 +1090..1099 ; Grapheme_Base # Nd [10] MYANMAR SHAN DIGIT ZERO..MYANMAR SHAN DIGIT NINE +109A..109C ; Grapheme_Base # Mc [3] MYANMAR SIGN KHAMTI TONE-1..MYANMAR VOWEL SIGN AITON A +109E..109F ; Grapheme_Base # So [2] MYANMAR SYMBOL SHAN ONE..MYANMAR SYMBOL SHAN EXCLAMATION +10A0..10C5 ; Grapheme_Base # L& [38] GEORGIAN CAPITAL LETTER AN..GEORGIAN CAPITAL LETTER HOE +10D0..10FA ; Grapheme_Base # Lo [43] GEORGIAN LETTER AN..GEORGIAN LETTER AIN +10FB ; Grapheme_Base # Po GEORGIAN PARAGRAPH SEPARATOR +10FC ; Grapheme_Base # Lm MODIFIER LETTER GEORGIAN NAR +1100..1248 ; Grapheme_Base # Lo [329] HANGUL CHOSEONG KIYEOK..ETHIOPIC SYLLABLE QWA +124A..124D ; Grapheme_Base # Lo [4] ETHIOPIC SYLLABLE QWI..ETHIOPIC SYLLABLE QWE +1250..1256 ; Grapheme_Base # Lo [7] ETHIOPIC SYLLABLE QHA..ETHIOPIC SYLLABLE QHO +1258 ; Grapheme_Base # Lo ETHIOPIC SYLLABLE QHWA +125A..125D ; Grapheme_Base # Lo [4] ETHIOPIC SYLLABLE QHWI..ETHIOPIC SYLLABLE QHWE +1260..1288 ; Grapheme_Base # Lo [41] ETHIOPIC SYLLABLE BA..ETHIOPIC SYLLABLE XWA +128A..128D ; Grapheme_Base # Lo [4] ETHIOPIC SYLLABLE XWI..ETHIOPIC SYLLABLE XWE +1290..12B0 ; Grapheme_Base # Lo [33] ETHIOPIC SYLLABLE NA..ETHIOPIC SYLLABLE KWA +12B2..12B5 ; Grapheme_Base # Lo [4] ETHIOPIC SYLLABLE KWI..ETHIOPIC SYLLABLE KWE +12B8..12BE ; Grapheme_Base # Lo [7] ETHIOPIC SYLLABLE KXA..ETHIOPIC SYLLABLE KXO +12C0 ; Grapheme_Base # Lo ETHIOPIC SYLLABLE KXWA +12C2..12C5 ; Grapheme_Base # Lo [4] ETHIOPIC SYLLABLE KXWI..ETHIOPIC SYLLABLE KXWE +12C8..12D6 ; Grapheme_Base # Lo [15] ETHIOPIC SYLLABLE WA..ETHIOPIC SYLLABLE PHARYNGEAL O +12D8..1310 ; Grapheme_Base # Lo [57] ETHIOPIC SYLLABLE ZA..ETHIOPIC SYLLABLE GWA +1312..1315 ; Grapheme_Base # Lo [4] ETHIOPIC SYLLABLE GWI..ETHIOPIC SYLLABLE GWE +1318..135A ; Grapheme_Base # Lo [67] ETHIOPIC SYLLABLE GGA..ETHIOPIC SYLLABLE FYA +1360 ; Grapheme_Base # So ETHIOPIC SECTION MARK +1361..1368 ; Grapheme_Base # Po [8] ETHIOPIC WORDSPACE..ETHIOPIC PARAGRAPH SEPARATOR +1369..137C ; Grapheme_Base # No [20] ETHIOPIC DIGIT ONE..ETHIOPIC NUMBER TEN THOUSAND +1380..138F ; Grapheme_Base # Lo [16] ETHIOPIC SYLLABLE SEBATBEIT MWA..ETHIOPIC SYLLABLE PWE +1390..1399 ; Grapheme_Base # So [10] ETHIOPIC TONAL MARK YIZET..ETHIOPIC TONAL MARK KURT +13A0..13F4 ; Grapheme_Base # Lo [85] CHEROKEE LETTER A..CHEROKEE LETTER YV +1400 ; Grapheme_Base # Pd CANADIAN SYLLABICS HYPHEN +1401..166C ; Grapheme_Base # Lo [620] CANADIAN SYLLABICS E..CANADIAN SYLLABICS CARRIER TTSA +166D..166E ; Grapheme_Base # Po [2] CANADIAN SYLLABICS CHI SIGN..CANADIAN SYLLABICS FULL STOP +166F..167F ; Grapheme_Base # Lo [17] CANADIAN SYLLABICS QAI..CANADIAN SYLLABICS BLACKFOOT W +1680 ; Grapheme_Base # Zs OGHAM SPACE MARK +1681..169A ; Grapheme_Base # Lo [26] OGHAM LETTER BEITH..OGHAM LETTER PEITH +169B ; Grapheme_Base # Ps OGHAM FEATHER MARK +169C ; Grapheme_Base # Pe OGHAM REVERSED FEATHER MARK +16A0..16EA ; Grapheme_Base # Lo [75] RUNIC LETTER FEHU FEOH FE F..RUNIC LETTER X +16EB..16ED ; Grapheme_Base # Po [3] RUNIC SINGLE PUNCTUATION..RUNIC CROSS PUNCTUATION +16EE..16F0 ; Grapheme_Base # Nl [3] RUNIC ARLAUG SYMBOL..RUNIC BELGTHOR SYMBOL +1700..170C ; Grapheme_Base # Lo [13] TAGALOG LETTER A..TAGALOG LETTER YA +170E..1711 ; Grapheme_Base # Lo [4] TAGALOG LETTER LA..TAGALOG LETTER HA +1720..1731 ; Grapheme_Base # Lo [18] HANUNOO LETTER A..HANUNOO LETTER HA +1735..1736 ; Grapheme_Base # Po [2] PHILIPPINE SINGLE PUNCTUATION..PHILIPPINE DOUBLE PUNCTUATION +1740..1751 ; Grapheme_Base # Lo [18] BUHID LETTER A..BUHID LETTER HA +1760..176C ; Grapheme_Base # Lo [13] TAGBANWA LETTER A..TAGBANWA LETTER YA +176E..1770 ; Grapheme_Base # Lo [3] TAGBANWA LETTER LA..TAGBANWA LETTER SA +1780..17B3 ; Grapheme_Base # Lo [52] KHMER LETTER KA..KHMER INDEPENDENT VOWEL QAU +17B6 ; Grapheme_Base # Mc KHMER VOWEL SIGN AA +17BE..17C5 ; Grapheme_Base # Mc [8] KHMER VOWEL SIGN OE..KHMER VOWEL SIGN AU +17C7..17C8 ; Grapheme_Base # Mc [2] KHMER SIGN REAHMUK..KHMER SIGN YUUKALEAPINTU +17D4..17D6 ; Grapheme_Base # Po [3] KHMER SIGN KHAN..KHMER SIGN CAMNUC PII KUUH +17D7 ; Grapheme_Base # Lm KHMER SIGN LEK TOO +17D8..17DA ; Grapheme_Base # Po [3] KHMER SIGN BEYYAL..KHMER SIGN KOOMUUT +17DB ; Grapheme_Base # Sc KHMER CURRENCY SYMBOL RIEL +17DC ; Grapheme_Base # Lo KHMER SIGN AVAKRAHASANYA +17E0..17E9 ; Grapheme_Base # Nd [10] KHMER DIGIT ZERO..KHMER DIGIT NINE +17F0..17F9 ; Grapheme_Base # No [10] KHMER SYMBOL LEK ATTAK SON..KHMER SYMBOL LEK ATTAK PRAM-BUON +1800..1805 ; Grapheme_Base # Po [6] MONGOLIAN BIRGA..MONGOLIAN FOUR DOTS +1806 ; Grapheme_Base # Pd MONGOLIAN TODO SOFT HYPHEN +1807..180A ; Grapheme_Base # Po [4] MONGOLIAN SIBE SYLLABLE BOUNDARY MARKER..MONGOLIAN NIRUGU +180E ; Grapheme_Base # Zs MONGOLIAN VOWEL SEPARATOR +1810..1819 ; Grapheme_Base # Nd [10] MONGOLIAN DIGIT ZERO..MONGOLIAN DIGIT NINE +1820..1842 ; Grapheme_Base # Lo [35] MONGOLIAN LETTER A..MONGOLIAN LETTER CHI +1843 ; Grapheme_Base # Lm MONGOLIAN LETTER TODO LONG VOWEL SIGN +1844..1877 ; Grapheme_Base # Lo [52] MONGOLIAN LETTER TODO E..MONGOLIAN LETTER MANCHU ZHA +1880..18A8 ; Grapheme_Base # Lo [41] MONGOLIAN LETTER ALI GALI ANUSVARA ONE..MONGOLIAN LETTER MANCHU ALI GALI BHA +18AA ; Grapheme_Base # Lo MONGOLIAN LETTER MANCHU ALI GALI LHA +18B0..18F5 ; Grapheme_Base # Lo [70] CANADIAN SYLLABICS OY..CANADIAN SYLLABICS CARRIER DENTAL S +1900..191C ; Grapheme_Base # Lo [29] LIMBU VOWEL-CARRIER LETTER..LIMBU LETTER HA +1923..1926 ; Grapheme_Base # Mc [4] LIMBU VOWEL SIGN EE..LIMBU VOWEL SIGN AU +1929..192B ; Grapheme_Base # Mc [3] LIMBU SUBJOINED LETTER YA..LIMBU SUBJOINED LETTER WA +1930..1931 ; Grapheme_Base # Mc [2] LIMBU SMALL LETTER KA..LIMBU SMALL LETTER NGA +1933..1938 ; Grapheme_Base # Mc [6] LIMBU SMALL LETTER TA..LIMBU SMALL LETTER LA +1940 ; Grapheme_Base # So LIMBU SIGN LOO +1944..1945 ; Grapheme_Base # Po [2] LIMBU EXCLAMATION MARK..LIMBU QUESTION MARK +1946..194F ; Grapheme_Base # Nd [10] LIMBU DIGIT ZERO..LIMBU DIGIT NINE +1950..196D ; Grapheme_Base # Lo [30] TAI LE LETTER KA..TAI LE LETTER AI +1970..1974 ; Grapheme_Base # Lo [5] TAI LE LETTER TONE-2..TAI LE LETTER TONE-6 +1980..19AB ; Grapheme_Base # Lo [44] NEW TAI LUE LETTER HIGH QA..NEW TAI LUE LETTER LOW SUA +19B0..19C0 ; Grapheme_Base # Mc [17] NEW TAI LUE VOWEL SIGN VOWEL SHORTENER..NEW TAI LUE VOWEL SIGN IY +19C1..19C7 ; Grapheme_Base # Lo [7] NEW TAI LUE LETTER FINAL V..NEW TAI LUE LETTER FINAL B +19C8..19C9 ; Grapheme_Base # Mc [2] NEW TAI LUE TONE MARK-1..NEW TAI LUE TONE MARK-2 +19D0..19DA ; Grapheme_Base # Nd [11] NEW TAI LUE DIGIT ZERO..NEW TAI LUE THAM DIGIT ONE +19DE..19DF ; Grapheme_Base # Po [2] NEW TAI LUE SIGN LAE..NEW TAI LUE SIGN LAEV +19E0..19FF ; Grapheme_Base # So [32] KHMER SYMBOL PATHAMASAT..KHMER SYMBOL DAP-PRAM ROC +1A00..1A16 ; Grapheme_Base # Lo [23] BUGINESE LETTER KA..BUGINESE LETTER HA +1A19..1A1B ; Grapheme_Base # Mc [3] BUGINESE VOWEL SIGN E..BUGINESE VOWEL SIGN AE +1A1E..1A1F ; Grapheme_Base # Po [2] BUGINESE PALLAWA..BUGINESE END OF SECTION +1A20..1A54 ; Grapheme_Base # Lo [53] TAI THAM LETTER HIGH KA..TAI THAM LETTER GREAT SA +1A55 ; Grapheme_Base # Mc TAI THAM CONSONANT SIGN MEDIAL RA +1A57 ; Grapheme_Base # Mc TAI THAM CONSONANT SIGN LA TANG LAI +1A61 ; Grapheme_Base # Mc TAI THAM VOWEL SIGN A +1A63..1A64 ; Grapheme_Base # Mc [2] TAI THAM VOWEL SIGN AA..TAI THAM VOWEL SIGN TALL AA +1A6D..1A72 ; Grapheme_Base # Mc [6] TAI THAM VOWEL SIGN OY..TAI THAM VOWEL SIGN THAM AI +1A80..1A89 ; Grapheme_Base # Nd [10] TAI THAM HORA DIGIT ZERO..TAI THAM HORA DIGIT NINE +1A90..1A99 ; Grapheme_Base # Nd [10] TAI THAM THAM DIGIT ZERO..TAI THAM THAM DIGIT NINE +1AA0..1AA6 ; Grapheme_Base # Po [7] TAI THAM SIGN WIANG..TAI THAM SIGN REVERSED ROTATED RANA +1AA7 ; Grapheme_Base # Lm TAI THAM SIGN MAI YAMOK +1AA8..1AAD ; Grapheme_Base # Po [6] TAI THAM SIGN KAAN..TAI THAM SIGN CAANG +1B04 ; Grapheme_Base # Mc BALINESE SIGN BISAH +1B05..1B33 ; Grapheme_Base # Lo [47] BALINESE LETTER AKARA..BALINESE LETTER HA +1B35 ; Grapheme_Base # Mc BALINESE VOWEL SIGN TEDUNG +1B3B ; Grapheme_Base # Mc BALINESE VOWEL SIGN RA REPA TEDUNG +1B3D..1B41 ; Grapheme_Base # Mc [5] BALINESE VOWEL SIGN LA LENGA TEDUNG..BALINESE VOWEL SIGN TALING REPA TEDUNG +1B43..1B44 ; Grapheme_Base # Mc [2] BALINESE VOWEL SIGN PEPET TEDUNG..BALINESE ADEG ADEG +1B45..1B4B ; Grapheme_Base # Lo [7] BALINESE LETTER KAF SASAK..BALINESE LETTER ASYURA SASAK +1B50..1B59 ; Grapheme_Base # Nd [10] BALINESE DIGIT ZERO..BALINESE DIGIT NINE +1B5A..1B60 ; Grapheme_Base # Po [7] BALINESE PANTI..BALINESE PAMENENG +1B61..1B6A ; Grapheme_Base # So [10] BALINESE MUSICAL SYMBOL DONG..BALINESE MUSICAL SYMBOL DANG GEDE +1B74..1B7C ; Grapheme_Base # So [9] BALINESE MUSICAL SYMBOL RIGHT-HAND OPEN DUG..BALINESE MUSICAL SYMBOL LEFT-HAND OPEN PING +1B82 ; Grapheme_Base # Mc SUNDANESE SIGN PANGWISAD +1B83..1BA0 ; Grapheme_Base # Lo [30] SUNDANESE LETTER A..SUNDANESE LETTER HA +1BA1 ; Grapheme_Base # Mc SUNDANESE CONSONANT SIGN PAMINGKAL +1BA6..1BA7 ; Grapheme_Base # Mc [2] SUNDANESE VOWEL SIGN PANAELAENG..SUNDANESE VOWEL SIGN PANOLONG +1BAA ; Grapheme_Base # Mc SUNDANESE SIGN PAMAAEH +1BAE..1BAF ; Grapheme_Base # Lo [2] SUNDANESE LETTER KHA..SUNDANESE LETTER SYA +1BB0..1BB9 ; Grapheme_Base # Nd [10] SUNDANESE DIGIT ZERO..SUNDANESE DIGIT NINE +1C00..1C23 ; Grapheme_Base # Lo [36] LEPCHA LETTER KA..LEPCHA LETTER A +1C24..1C2B ; Grapheme_Base # Mc [8] LEPCHA SUBJOINED LETTER YA..LEPCHA VOWEL SIGN UU +1C34..1C35 ; Grapheme_Base # Mc [2] LEPCHA CONSONANT SIGN NYIN-DO..LEPCHA CONSONANT SIGN KANG +1C3B..1C3F ; Grapheme_Base # Po [5] LEPCHA PUNCTUATION TA-ROL..LEPCHA PUNCTUATION TSHOOK +1C40..1C49 ; Grapheme_Base # Nd [10] LEPCHA DIGIT ZERO..LEPCHA DIGIT NINE +1C4D..1C4F ; Grapheme_Base # Lo [3] LEPCHA LETTER TTA..LEPCHA LETTER DDA +1C50..1C59 ; Grapheme_Base # Nd [10] OL CHIKI DIGIT ZERO..OL CHIKI DIGIT NINE +1C5A..1C77 ; Grapheme_Base # Lo [30] OL CHIKI LETTER LA..OL CHIKI LETTER OH +1C78..1C7D ; Grapheme_Base # Lm [6] OL CHIKI MU TTUDDAG..OL CHIKI AHAD +1C7E..1C7F ; Grapheme_Base # Po [2] OL CHIKI PUNCTUATION MUCAAD..OL CHIKI PUNCTUATION DOUBLE MUCAAD +1CD3 ; Grapheme_Base # Po VEDIC SIGN NIHSHVASA +1CE1 ; Grapheme_Base # Mc VEDIC TONE ATHARVAVEDIC INDEPENDENT SVARITA +1CE9..1CEC ; Grapheme_Base # Lo [4] VEDIC SIGN ANUSVARA ANTARGOMUKHA..VEDIC SIGN ANUSVARA VAMAGOMUKHA WITH TAIL +1CEE..1CF1 ; Grapheme_Base # Lo [4] VEDIC SIGN HEXIFORM LONG ANUSVARA..VEDIC SIGN ANUSVARA UBHAYATO MUKHA +1CF2 ; Grapheme_Base # Mc VEDIC SIGN ARDHAVISARGA +1D00..1D2B ; Grapheme_Base # L& [44] LATIN LETTER SMALL CAPITAL A..CYRILLIC LETTER SMALL CAPITAL EL +1D2C..1D61 ; Grapheme_Base # Lm [54] MODIFIER LETTER CAPITAL A..MODIFIER LETTER SMALL CHI +1D62..1D77 ; Grapheme_Base # L& [22] LATIN SUBSCRIPT SMALL LETTER I..LATIN SMALL LETTER TURNED G +1D78 ; Grapheme_Base # Lm MODIFIER LETTER CYRILLIC EN +1D79..1D9A ; Grapheme_Base # L& [34] LATIN SMALL LETTER INSULAR G..LATIN SMALL LETTER EZH WITH RETROFLEX HOOK +1D9B..1DBF ; Grapheme_Base # Lm [37] MODIFIER LETTER SMALL TURNED ALPHA..MODIFIER LETTER SMALL THETA +1E00..1F15 ; Grapheme_Base # L& [278] LATIN CAPITAL LETTER A WITH RING BELOW..GREEK SMALL LETTER EPSILON WITH DASIA AND OXIA +1F18..1F1D ; Grapheme_Base # L& [6] GREEK CAPITAL LETTER EPSILON WITH PSILI..GREEK CAPITAL LETTER EPSILON WITH DASIA AND OXIA +1F20..1F45 ; Grapheme_Base # L& [38] GREEK SMALL LETTER ETA WITH PSILI..GREEK SMALL LETTER OMICRON WITH DASIA AND OXIA +1F48..1F4D ; Grapheme_Base # L& [6] GREEK CAPITAL LETTER OMICRON WITH PSILI..GREEK CAPITAL LETTER OMICRON WITH DASIA AND OXIA +1F50..1F57 ; Grapheme_Base # L& [8] GREEK SMALL LETTER UPSILON WITH PSILI..GREEK SMALL LETTER UPSILON WITH DASIA AND PERISPOMENI +1F59 ; Grapheme_Base # L& GREEK CAPITAL LETTER UPSILON WITH DASIA +1F5B ; Grapheme_Base # L& GREEK CAPITAL LETTER UPSILON WITH DASIA AND VARIA +1F5D ; Grapheme_Base # L& GREEK CAPITAL LETTER UPSILON WITH DASIA AND OXIA +1F5F..1F7D ; Grapheme_Base # L& [31] GREEK CAPITAL LETTER UPSILON WITH DASIA AND PERISPOMENI..GREEK SMALL LETTER OMEGA WITH OXIA +1F80..1FB4 ; Grapheme_Base # L& [53] GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI..GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI +1FB6..1FBC ; Grapheme_Base # L& [7] GREEK SMALL LETTER ALPHA WITH PERISPOMENI..GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI +1FBD ; Grapheme_Base # Sk GREEK KORONIS +1FBE ; Grapheme_Base # L& GREEK PROSGEGRAMMENI +1FBF..1FC1 ; Grapheme_Base # Sk [3] GREEK PSILI..GREEK DIALYTIKA AND PERISPOMENI +1FC2..1FC4 ; Grapheme_Base # L& [3] GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI..GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI +1FC6..1FCC ; Grapheme_Base # L& [7] GREEK SMALL LETTER ETA WITH PERISPOMENI..GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI +1FCD..1FCF ; Grapheme_Base # Sk [3] GREEK PSILI AND VARIA..GREEK PSILI AND PERISPOMENI +1FD0..1FD3 ; Grapheme_Base # L& [4] GREEK SMALL LETTER IOTA WITH VRACHY..GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA +1FD6..1FDB ; Grapheme_Base # L& [6] GREEK SMALL LETTER IOTA WITH PERISPOMENI..GREEK CAPITAL LETTER IOTA WITH OXIA +1FDD..1FDF ; Grapheme_Base # Sk [3] GREEK DASIA AND VARIA..GREEK DASIA AND PERISPOMENI +1FE0..1FEC ; Grapheme_Base # L& [13] GREEK SMALL LETTER UPSILON WITH VRACHY..GREEK CAPITAL LETTER RHO WITH DASIA +1FED..1FEF ; Grapheme_Base # Sk [3] GREEK DIALYTIKA AND VARIA..GREEK VARIA +1FF2..1FF4 ; Grapheme_Base # L& [3] GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI..GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI +1FF6..1FFC ; Grapheme_Base # L& [7] GREEK SMALL LETTER OMEGA WITH PERISPOMENI..GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI +1FFD..1FFE ; Grapheme_Base # Sk [2] GREEK OXIA..GREEK DASIA +2000..200A ; Grapheme_Base # Zs [11] EN QUAD..HAIR SPACE +2010..2015 ; Grapheme_Base # Pd [6] HYPHEN..HORIZONTAL BAR +2016..2017 ; Grapheme_Base # Po [2] DOUBLE VERTICAL LINE..DOUBLE LOW LINE +2018 ; Grapheme_Base # Pi LEFT SINGLE QUOTATION MARK +2019 ; Grapheme_Base # Pf RIGHT SINGLE QUOTATION MARK +201A ; Grapheme_Base # Ps SINGLE LOW-9 QUOTATION MARK +201B..201C ; Grapheme_Base # Pi [2] SINGLE HIGH-REVERSED-9 QUOTATION MARK..LEFT DOUBLE QUOTATION MARK +201D ; Grapheme_Base # Pf RIGHT DOUBLE QUOTATION MARK +201E ; Grapheme_Base # Ps DOUBLE LOW-9 QUOTATION MARK +201F ; Grapheme_Base # Pi DOUBLE HIGH-REVERSED-9 QUOTATION MARK +2020..2027 ; Grapheme_Base # Po [8] DAGGER..HYPHENATION POINT +202F ; Grapheme_Base # Zs NARROW NO-BREAK SPACE +2030..2038 ; Grapheme_Base # Po [9] PER MILLE SIGN..CARET +2039 ; Grapheme_Base # Pi SINGLE LEFT-POINTING ANGLE QUOTATION MARK +203A ; Grapheme_Base # Pf SINGLE RIGHT-POINTING ANGLE QUOTATION MARK +203B..203E ; Grapheme_Base # Po [4] REFERENCE MARK..OVERLINE +203F..2040 ; Grapheme_Base # Pc [2] UNDERTIE..CHARACTER TIE +2041..2043 ; Grapheme_Base # Po [3] CARET INSERTION POINT..HYPHEN BULLET +2044 ; Grapheme_Base # Sm FRACTION SLASH +2045 ; Grapheme_Base # Ps LEFT SQUARE BRACKET WITH QUILL +2046 ; Grapheme_Base # Pe RIGHT SQUARE BRACKET WITH QUILL +2047..2051 ; Grapheme_Base # Po [11] DOUBLE QUESTION MARK..TWO ASTERISKS ALIGNED VERTICALLY +2052 ; Grapheme_Base # Sm COMMERCIAL MINUS SIGN +2053 ; Grapheme_Base # Po SWUNG DASH +2054 ; Grapheme_Base # Pc INVERTED UNDERTIE +2055..205E ; Grapheme_Base # Po [10] FLOWER PUNCTUATION MARK..VERTICAL FOUR DOTS +205F ; Grapheme_Base # Zs MEDIUM MATHEMATICAL SPACE +2070 ; Grapheme_Base # No SUPERSCRIPT ZERO +2071 ; Grapheme_Base # Lm SUPERSCRIPT LATIN SMALL LETTER I +2074..2079 ; Grapheme_Base # No [6] SUPERSCRIPT FOUR..SUPERSCRIPT NINE +207A..207C ; Grapheme_Base # Sm [3] SUPERSCRIPT PLUS SIGN..SUPERSCRIPT EQUALS SIGN +207D ; Grapheme_Base # Ps SUPERSCRIPT LEFT PARENTHESIS +207E ; Grapheme_Base # Pe SUPERSCRIPT RIGHT PARENTHESIS +207F ; Grapheme_Base # Lm SUPERSCRIPT LATIN SMALL LETTER N +2080..2089 ; Grapheme_Base # No [10] SUBSCRIPT ZERO..SUBSCRIPT NINE +208A..208C ; Grapheme_Base # Sm [3] SUBSCRIPT PLUS SIGN..SUBSCRIPT EQUALS SIGN +208D ; Grapheme_Base # Ps SUBSCRIPT LEFT PARENTHESIS +208E ; Grapheme_Base # Pe SUBSCRIPT RIGHT PARENTHESIS +2090..2094 ; Grapheme_Base # Lm [5] LATIN SUBSCRIPT SMALL LETTER A..LATIN SUBSCRIPT SMALL LETTER SCHWA +20A0..20B8 ; Grapheme_Base # Sc [25] EURO-CURRENCY SIGN..TENGE SIGN +2100..2101 ; Grapheme_Base # So [2] ACCOUNT OF..ADDRESSED TO THE SUBJECT +2102 ; Grapheme_Base # L& DOUBLE-STRUCK CAPITAL C +2103..2106 ; Grapheme_Base # So [4] DEGREE CELSIUS..CADA UNA +2107 ; Grapheme_Base # L& EULER CONSTANT +2108..2109 ; Grapheme_Base # So [2] SCRUPLE..DEGREE FAHRENHEIT +210A..2113 ; Grapheme_Base # L& [10] SCRIPT SMALL G..SCRIPT SMALL L +2114 ; Grapheme_Base # So L B BAR SYMBOL +2115 ; Grapheme_Base # L& DOUBLE-STRUCK CAPITAL N +2116..2118 ; Grapheme_Base # So [3] NUMERO SIGN..SCRIPT CAPITAL P +2119..211D ; Grapheme_Base # L& [5] DOUBLE-STRUCK CAPITAL P..DOUBLE-STRUCK CAPITAL R +211E..2123 ; Grapheme_Base # So [6] PRESCRIPTION TAKE..VERSICLE +2124 ; Grapheme_Base # L& DOUBLE-STRUCK CAPITAL Z +2125 ; Grapheme_Base # So OUNCE SIGN +2126 ; Grapheme_Base # L& OHM SIGN +2127 ; Grapheme_Base # So INVERTED OHM SIGN +2128 ; Grapheme_Base # L& BLACK-LETTER CAPITAL Z +2129 ; Grapheme_Base # So TURNED GREEK SMALL LETTER IOTA +212A..212D ; Grapheme_Base # L& [4] KELVIN SIGN..BLACK-LETTER CAPITAL C +212E ; Grapheme_Base # So ESTIMATED SYMBOL +212F..2134 ; Grapheme_Base # L& [6] SCRIPT SMALL E..SCRIPT SMALL O +2135..2138 ; Grapheme_Base # Lo [4] ALEF SYMBOL..DALET SYMBOL +2139 ; Grapheme_Base # L& INFORMATION SOURCE +213A..213B ; Grapheme_Base # So [2] ROTATED CAPITAL Q..FACSIMILE SIGN +213C..213F ; Grapheme_Base # L& [4] DOUBLE-STRUCK SMALL PI..DOUBLE-STRUCK CAPITAL PI +2140..2144 ; Grapheme_Base # Sm [5] DOUBLE-STRUCK N-ARY SUMMATION..TURNED SANS-SERIF CAPITAL Y +2145..2149 ; Grapheme_Base # L& [5] DOUBLE-STRUCK ITALIC CAPITAL D..DOUBLE-STRUCK ITALIC SMALL J +214A ; Grapheme_Base # So PROPERTY LINE +214B ; Grapheme_Base # Sm TURNED AMPERSAND +214C..214D ; Grapheme_Base # So [2] PER SIGN..AKTIESELSKAB +214E ; Grapheme_Base # L& TURNED SMALL F +214F ; Grapheme_Base # So SYMBOL FOR SAMARITAN SOURCE +2150..215F ; Grapheme_Base # No [16] VULGAR FRACTION ONE SEVENTH..FRACTION NUMERATOR ONE +2160..2182 ; Grapheme_Base # Nl [35] ROMAN NUMERAL ONE..ROMAN NUMERAL TEN THOUSAND +2183..2184 ; Grapheme_Base # L& [2] ROMAN NUMERAL REVERSED ONE HUNDRED..LATIN SMALL LETTER REVERSED C +2185..2188 ; Grapheme_Base # Nl [4] ROMAN NUMERAL SIX LATE FORM..ROMAN NUMERAL ONE HUNDRED THOUSAND +2189 ; Grapheme_Base # No VULGAR FRACTION ZERO THIRDS +2190..2194 ; Grapheme_Base # Sm [5] LEFTWARDS ARROW..LEFT RIGHT ARROW +2195..2199 ; Grapheme_Base # So [5] UP DOWN ARROW..SOUTH WEST ARROW +219A..219B ; Grapheme_Base # Sm [2] LEFTWARDS ARROW WITH STROKE..RIGHTWARDS ARROW WITH STROKE +219C..219F ; Grapheme_Base # So [4] LEFTWARDS WAVE ARROW..UPWARDS TWO HEADED ARROW +21A0 ; Grapheme_Base # Sm RIGHTWARDS TWO HEADED ARROW +21A1..21A2 ; Grapheme_Base # So [2] DOWNWARDS TWO HEADED ARROW..LEFTWARDS ARROW WITH TAIL +21A3 ; Grapheme_Base # Sm RIGHTWARDS ARROW WITH TAIL +21A4..21A5 ; Grapheme_Base # So [2] LEFTWARDS ARROW FROM BAR..UPWARDS ARROW FROM BAR +21A6 ; Grapheme_Base # Sm RIGHTWARDS ARROW FROM BAR +21A7..21AD ; Grapheme_Base # So [7] DOWNWARDS ARROW FROM BAR..LEFT RIGHT WAVE ARROW +21AE ; Grapheme_Base # Sm LEFT RIGHT ARROW WITH STROKE +21AF..21CD ; Grapheme_Base # So [31] DOWNWARDS ZIGZAG ARROW..LEFTWARDS DOUBLE ARROW WITH STROKE +21CE..21CF ; Grapheme_Base # Sm [2] LEFT RIGHT DOUBLE ARROW WITH STROKE..RIGHTWARDS DOUBLE ARROW WITH STROKE +21D0..21D1 ; Grapheme_Base # So [2] LEFTWARDS DOUBLE ARROW..UPWARDS DOUBLE ARROW +21D2 ; Grapheme_Base # Sm RIGHTWARDS DOUBLE ARROW +21D3 ; Grapheme_Base # So DOWNWARDS DOUBLE ARROW +21D4 ; Grapheme_Base # Sm LEFT RIGHT DOUBLE ARROW +21D5..21F3 ; Grapheme_Base # So [31] UP DOWN DOUBLE ARROW..UP DOWN WHITE ARROW +21F4..22FF ; Grapheme_Base # Sm [268] RIGHT ARROW WITH SMALL CIRCLE..Z NOTATION BAG MEMBERSHIP +2300..2307 ; Grapheme_Base # So [8] DIAMETER SIGN..WAVY LINE +2308..230B ; Grapheme_Base # Sm [4] LEFT CEILING..RIGHT FLOOR +230C..231F ; Grapheme_Base # So [20] BOTTOM RIGHT CROP..BOTTOM RIGHT CORNER +2320..2321 ; Grapheme_Base # Sm [2] TOP HALF INTEGRAL..BOTTOM HALF INTEGRAL +2322..2328 ; Grapheme_Base # So [7] FROWN..KEYBOARD +2329 ; Grapheme_Base # Ps LEFT-POINTING ANGLE BRACKET +232A ; Grapheme_Base # Pe RIGHT-POINTING ANGLE BRACKET +232B..237B ; Grapheme_Base # So [81] ERASE TO THE LEFT..NOT CHECK MARK +237C ; Grapheme_Base # Sm RIGHT ANGLE WITH DOWNWARDS ZIGZAG ARROW +237D..239A ; Grapheme_Base # So [30] SHOULDERED OPEN BOX..CLEAR SCREEN SYMBOL +239B..23B3 ; Grapheme_Base # Sm [25] LEFT PARENTHESIS UPPER HOOK..SUMMATION BOTTOM +23B4..23DB ; Grapheme_Base # So [40] TOP SQUARE BRACKET..FUSE +23DC..23E1 ; Grapheme_Base # Sm [6] TOP PARENTHESIS..BOTTOM TORTOISE SHELL BRACKET +23E2..23E8 ; Grapheme_Base # So [7] WHITE TRAPEZIUM..DECIMAL EXPONENT SYMBOL +2400..2426 ; Grapheme_Base # So [39] SYMBOL FOR NULL..SYMBOL FOR SUBSTITUTE FORM TWO +2440..244A ; Grapheme_Base # So [11] OCR HOOK..OCR DOUBLE BACKSLASH +2460..249B ; Grapheme_Base # No [60] CIRCLED DIGIT ONE..NUMBER TWENTY FULL STOP +249C..24E9 ; Grapheme_Base # So [78] PARENTHESIZED LATIN SMALL LETTER A..CIRCLED LATIN SMALL LETTER Z +24EA..24FF ; Grapheme_Base # No [22] CIRCLED DIGIT ZERO..NEGATIVE CIRCLED DIGIT ZERO +2500..25B6 ; Grapheme_Base # So [183] BOX DRAWINGS LIGHT HORIZONTAL..BLACK RIGHT-POINTING TRIANGLE +25B7 ; Grapheme_Base # Sm WHITE RIGHT-POINTING TRIANGLE +25B8..25C0 ; Grapheme_Base # So [9] BLACK RIGHT-POINTING SMALL TRIANGLE..BLACK LEFT-POINTING TRIANGLE +25C1 ; Grapheme_Base # Sm WHITE LEFT-POINTING TRIANGLE +25C2..25F7 ; Grapheme_Base # So [54] BLACK LEFT-POINTING SMALL TRIANGLE..WHITE CIRCLE WITH UPPER RIGHT QUADRANT +25F8..25FF ; Grapheme_Base # Sm [8] UPPER LEFT TRIANGLE..LOWER RIGHT TRIANGLE +2600..266E ; Grapheme_Base # So [111] BLACK SUN WITH RAYS..MUSIC NATURAL SIGN +266F ; Grapheme_Base # Sm MUSIC SHARP SIGN +2670..26CD ; Grapheme_Base # So [94] WEST SYRIAC CROSS..DISABLED CAR +26CF..26E1 ; Grapheme_Base # So [19] PICK..RESTRICTED LEFT ENTRY-2 +26E3 ; Grapheme_Base # So HEAVY CIRCLE WITH STROKE AND TWO DOTS ABOVE +26E8..26FF ; Grapheme_Base # So [24] BLACK CROSS ON SHIELD..WHITE FLAG WITH HORIZONTAL MIDDLE BLACK STRIPE +2701..2704 ; Grapheme_Base # So [4] UPPER BLADE SCISSORS..WHITE SCISSORS +2706..2709 ; Grapheme_Base # So [4] TELEPHONE LOCATION SIGN..ENVELOPE +270C..2727 ; Grapheme_Base # So [28] VICTORY HAND..WHITE FOUR POINTED STAR +2729..274B ; Grapheme_Base # So [35] STRESS OUTLINED WHITE STAR..HEAVY EIGHT TEARDROP-SPOKED PROPELLER ASTERISK +274D ; Grapheme_Base # So SHADOWED WHITE CIRCLE +274F..2752 ; Grapheme_Base # So [4] LOWER RIGHT DROP-SHADOWED WHITE SQUARE..UPPER RIGHT SHADOWED WHITE SQUARE +2756..275E ; Grapheme_Base # So [9] BLACK DIAMOND MINUS WHITE X..HEAVY DOUBLE COMMA QUOTATION MARK ORNAMENT +2761..2767 ; Grapheme_Base # So [7] CURVED STEM PARAGRAPH SIGN ORNAMENT..ROTATED FLORAL HEART BULLET +2768 ; Grapheme_Base # Ps MEDIUM LEFT PARENTHESIS ORNAMENT +2769 ; Grapheme_Base # Pe MEDIUM RIGHT PARENTHESIS ORNAMENT +276A ; Grapheme_Base # Ps MEDIUM FLATTENED LEFT PARENTHESIS ORNAMENT +276B ; Grapheme_Base # Pe MEDIUM FLATTENED RIGHT PARENTHESIS ORNAMENT +276C ; Grapheme_Base # Ps MEDIUM LEFT-POINTING ANGLE BRACKET ORNAMENT +276D ; Grapheme_Base # Pe MEDIUM RIGHT-POINTING ANGLE BRACKET ORNAMENT +276E ; Grapheme_Base # Ps HEAVY LEFT-POINTING ANGLE QUOTATION MARK ORNAMENT +276F ; Grapheme_Base # Pe HEAVY RIGHT-POINTING ANGLE QUOTATION MARK ORNAMENT +2770 ; Grapheme_Base # Ps HEAVY LEFT-POINTING ANGLE BRACKET ORNAMENT +2771 ; Grapheme_Base # Pe HEAVY RIGHT-POINTING ANGLE BRACKET ORNAMENT +2772 ; Grapheme_Base # Ps LIGHT LEFT TORTOISE SHELL BRACKET ORNAMENT +2773 ; Grapheme_Base # Pe LIGHT RIGHT TORTOISE SHELL BRACKET ORNAMENT +2774 ; Grapheme_Base # Ps MEDIUM LEFT CURLY BRACKET ORNAMENT +2775 ; Grapheme_Base # Pe MEDIUM RIGHT CURLY BRACKET ORNAMENT +2776..2793 ; Grapheme_Base # No [30] DINGBAT NEGATIVE CIRCLED DIGIT ONE..DINGBAT NEGATIVE CIRCLED SANS-SERIF NUMBER TEN +2794 ; Grapheme_Base # So HEAVY WIDE-HEADED RIGHTWARDS ARROW +2798..27AF ; Grapheme_Base # So [24] HEAVY SOUTH EAST ARROW..NOTCHED LOWER RIGHT-SHADOWED WHITE RIGHTWARDS ARROW +27B1..27BE ; Grapheme_Base # So [14] NOTCHED UPPER RIGHT-SHADOWED WHITE RIGHTWARDS ARROW..OPEN-OUTLINED RIGHTWARDS ARROW +27C0..27C4 ; Grapheme_Base # Sm [5] THREE DIMENSIONAL ANGLE..OPEN SUPERSET +27C5 ; Grapheme_Base # Ps LEFT S-SHAPED BAG DELIMITER +27C6 ; Grapheme_Base # Pe RIGHT S-SHAPED BAG DELIMITER +27C7..27CA ; Grapheme_Base # Sm [4] OR WITH DOT INSIDE..VERTICAL BAR WITH HORIZONTAL STROKE +27CC ; Grapheme_Base # Sm LONG DIVISION +27D0..27E5 ; Grapheme_Base # Sm [22] WHITE DIAMOND WITH CENTRED DOT..WHITE SQUARE WITH RIGHTWARDS TICK +27E6 ; Grapheme_Base # Ps MATHEMATICAL LEFT WHITE SQUARE BRACKET +27E7 ; Grapheme_Base # Pe MATHEMATICAL RIGHT WHITE SQUARE BRACKET +27E8 ; Grapheme_Base # Ps MATHEMATICAL LEFT ANGLE BRACKET +27E9 ; Grapheme_Base # Pe MATHEMATICAL RIGHT ANGLE BRACKET +27EA ; Grapheme_Base # Ps MATHEMATICAL LEFT DOUBLE ANGLE BRACKET +27EB ; Grapheme_Base # Pe MATHEMATICAL RIGHT DOUBLE ANGLE BRACKET +27EC ; Grapheme_Base # Ps MATHEMATICAL LEFT WHITE TORTOISE SHELL BRACKET +27ED ; Grapheme_Base # Pe MATHEMATICAL RIGHT WHITE TORTOISE SHELL BRACKET +27EE ; Grapheme_Base # Ps MATHEMATICAL LEFT FLATTENED PARENTHESIS +27EF ; Grapheme_Base # Pe MATHEMATICAL RIGHT FLATTENED PARENTHESIS +27F0..27FF ; Grapheme_Base # Sm [16] UPWARDS QUADRUPLE ARROW..LONG RIGHTWARDS SQUIGGLE ARROW +2800..28FF ; Grapheme_Base # So [256] BRAILLE PATTERN BLANK..BRAILLE PATTERN DOTS-12345678 +2900..2982 ; Grapheme_Base # Sm [131] RIGHTWARDS TWO-HEADED ARROW WITH VERTICAL STROKE..Z NOTATION TYPE COLON +2983 ; Grapheme_Base # Ps LEFT WHITE CURLY BRACKET +2984 ; Grapheme_Base # Pe RIGHT WHITE CURLY BRACKET +2985 ; Grapheme_Base # Ps LEFT WHITE PARENTHESIS +2986 ; Grapheme_Base # Pe RIGHT WHITE PARENTHESIS +2987 ; Grapheme_Base # Ps Z NOTATION LEFT IMAGE BRACKET +2988 ; Grapheme_Base # Pe Z NOTATION RIGHT IMAGE BRACKET +2989 ; Grapheme_Base # Ps Z NOTATION LEFT BINDING BRACKET +298A ; Grapheme_Base # Pe Z NOTATION RIGHT BINDING BRACKET +298B ; Grapheme_Base # Ps LEFT SQUARE BRACKET WITH UNDERBAR +298C ; Grapheme_Base # Pe RIGHT SQUARE BRACKET WITH UNDERBAR +298D ; Grapheme_Base # Ps LEFT SQUARE BRACKET WITH TICK IN TOP CORNER +298E ; Grapheme_Base # Pe RIGHT SQUARE BRACKET WITH TICK IN BOTTOM CORNER +298F ; Grapheme_Base # Ps LEFT SQUARE BRACKET WITH TICK IN BOTTOM CORNER +2990 ; Grapheme_Base # Pe RIGHT SQUARE BRACKET WITH TICK IN TOP CORNER +2991 ; Grapheme_Base # Ps LEFT ANGLE BRACKET WITH DOT +2992 ; Grapheme_Base # Pe RIGHT ANGLE BRACKET WITH DOT +2993 ; Grapheme_Base # Ps LEFT ARC LESS-THAN BRACKET +2994 ; Grapheme_Base # Pe RIGHT ARC GREATER-THAN BRACKET +2995 ; Grapheme_Base # Ps DOUBLE LEFT ARC GREATER-THAN BRACKET +2996 ; Grapheme_Base # Pe DOUBLE RIGHT ARC LESS-THAN BRACKET +2997 ; Grapheme_Base # Ps LEFT BLACK TORTOISE SHELL BRACKET +2998 ; Grapheme_Base # Pe RIGHT BLACK TORTOISE SHELL BRACKET +2999..29D7 ; Grapheme_Base # Sm [63] DOTTED FENCE..BLACK HOURGLASS +29D8 ; Grapheme_Base # Ps LEFT WIGGLY FENCE +29D9 ; Grapheme_Base # Pe RIGHT WIGGLY FENCE +29DA ; Grapheme_Base # Ps LEFT DOUBLE WIGGLY FENCE +29DB ; Grapheme_Base # Pe RIGHT DOUBLE WIGGLY FENCE +29DC..29FB ; Grapheme_Base # Sm [32] INCOMPLETE INFINITY..TRIPLE PLUS +29FC ; Grapheme_Base # Ps LEFT-POINTING CURVED ANGLE BRACKET +29FD ; Grapheme_Base # Pe RIGHT-POINTING CURVED ANGLE BRACKET +29FE..2AFF ; Grapheme_Base # Sm [258] TINY..N-ARY WHITE VERTICAL BAR +2B00..2B2F ; Grapheme_Base # So [48] NORTH EAST WHITE ARROW..WHITE VERTICAL ELLIPSE +2B30..2B44 ; Grapheme_Base # Sm [21] LEFT ARROW WITH SMALL CIRCLE..RIGHTWARDS ARROW THROUGH SUPERSET +2B45..2B46 ; Grapheme_Base # So [2] LEFTWARDS QUADRUPLE ARROW..RIGHTWARDS QUADRUPLE ARROW +2B47..2B4C ; Grapheme_Base # Sm [6] REVERSE TILDE OPERATOR ABOVE RIGHTWARDS ARROW..RIGHTWARDS ARROW ABOVE REVERSE TILDE OPERATOR +2B50..2B59 ; Grapheme_Base # So [10] WHITE MEDIUM STAR..HEAVY CIRCLED SALTIRE +2C00..2C2E ; Grapheme_Base # L& [47] GLAGOLITIC CAPITAL LETTER AZU..GLAGOLITIC CAPITAL LETTER LATINATE MYSLITE +2C30..2C5E ; Grapheme_Base # L& [47] GLAGOLITIC SMALL LETTER AZU..GLAGOLITIC SMALL LETTER LATINATE MYSLITE +2C60..2C7C ; Grapheme_Base # L& [29] LATIN CAPITAL LETTER L WITH DOUBLE BAR..LATIN SUBSCRIPT SMALL LETTER J +2C7D ; Grapheme_Base # Lm MODIFIER LETTER CAPITAL V +2C7E..2CE4 ; Grapheme_Base # L& [103] LATIN CAPITAL LETTER S WITH SWASH TAIL..COPTIC SYMBOL KAI +2CE5..2CEA ; Grapheme_Base # So [6] COPTIC SYMBOL MI RO..COPTIC SYMBOL SHIMA SIMA +2CEB..2CEE ; Grapheme_Base # L& [4] COPTIC CAPITAL LETTER CRYPTOGRAMMIC SHEI..COPTIC SMALL LETTER CRYPTOGRAMMIC GANGIA +2CF9..2CFC ; Grapheme_Base # Po [4] COPTIC OLD NUBIAN FULL STOP..COPTIC OLD NUBIAN VERSE DIVIDER +2CFD ; Grapheme_Base # No COPTIC FRACTION ONE HALF +2CFE..2CFF ; Grapheme_Base # Po [2] COPTIC FULL STOP..COPTIC MORPHOLOGICAL DIVIDER +2D00..2D25 ; Grapheme_Base # L& [38] GEORGIAN SMALL LETTER AN..GEORGIAN SMALL LETTER HOE +2D30..2D65 ; Grapheme_Base # Lo [54] TIFINAGH LETTER YA..TIFINAGH LETTER YAZZ +2D6F ; Grapheme_Base # Lm TIFINAGH MODIFIER LETTER LABIALIZATION MARK +2D80..2D96 ; Grapheme_Base # Lo [23] ETHIOPIC SYLLABLE LOA..ETHIOPIC SYLLABLE GGWE +2DA0..2DA6 ; Grapheme_Base # Lo [7] ETHIOPIC SYLLABLE SSA..ETHIOPIC SYLLABLE SSO +2DA8..2DAE ; Grapheme_Base # Lo [7] ETHIOPIC SYLLABLE CCA..ETHIOPIC SYLLABLE CCO +2DB0..2DB6 ; Grapheme_Base # Lo [7] ETHIOPIC SYLLABLE ZZA..ETHIOPIC SYLLABLE ZZO +2DB8..2DBE ; Grapheme_Base # Lo [7] ETHIOPIC SYLLABLE CCHA..ETHIOPIC SYLLABLE CCHO +2DC0..2DC6 ; Grapheme_Base # Lo [7] ETHIOPIC SYLLABLE QYA..ETHIOPIC SYLLABLE QYO +2DC8..2DCE ; Grapheme_Base # Lo [7] ETHIOPIC SYLLABLE KYA..ETHIOPIC SYLLABLE KYO +2DD0..2DD6 ; Grapheme_Base # Lo [7] ETHIOPIC SYLLABLE XYA..ETHIOPIC SYLLABLE XYO +2DD8..2DDE ; Grapheme_Base # Lo [7] ETHIOPIC SYLLABLE GYA..ETHIOPIC SYLLABLE GYO +2E00..2E01 ; Grapheme_Base # Po [2] RIGHT ANGLE SUBSTITUTION MARKER..RIGHT ANGLE DOTTED SUBSTITUTION MARKER +2E02 ; Grapheme_Base # Pi LEFT SUBSTITUTION BRACKET +2E03 ; Grapheme_Base # Pf RIGHT SUBSTITUTION BRACKET +2E04 ; Grapheme_Base # Pi LEFT DOTTED SUBSTITUTION BRACKET +2E05 ; Grapheme_Base # Pf RIGHT DOTTED SUBSTITUTION BRACKET +2E06..2E08 ; Grapheme_Base # Po [3] RAISED INTERPOLATION MARKER..DOTTED TRANSPOSITION MARKER +2E09 ; Grapheme_Base # Pi LEFT TRANSPOSITION BRACKET +2E0A ; Grapheme_Base # Pf RIGHT TRANSPOSITION BRACKET +2E0B ; Grapheme_Base # Po RAISED SQUARE +2E0C ; Grapheme_Base # Pi LEFT RAISED OMISSION BRACKET +2E0D ; Grapheme_Base # Pf RIGHT RAISED OMISSION BRACKET +2E0E..2E16 ; Grapheme_Base # Po [9] EDITORIAL CORONIS..DOTTED RIGHT-POINTING ANGLE +2E17 ; Grapheme_Base # Pd DOUBLE OBLIQUE HYPHEN +2E18..2E19 ; Grapheme_Base # Po [2] INVERTED INTERROBANG..PALM BRANCH +2E1A ; Grapheme_Base # Pd HYPHEN WITH DIAERESIS +2E1B ; Grapheme_Base # Po TILDE WITH RING ABOVE +2E1C ; Grapheme_Base # Pi LEFT LOW PARAPHRASE BRACKET +2E1D ; Grapheme_Base # Pf RIGHT LOW PARAPHRASE BRACKET +2E1E..2E1F ; Grapheme_Base # Po [2] TILDE WITH DOT ABOVE..TILDE WITH DOT BELOW +2E20 ; Grapheme_Base # Pi LEFT VERTICAL BAR WITH QUILL +2E21 ; Grapheme_Base # Pf RIGHT VERTICAL BAR WITH QUILL +2E22 ; Grapheme_Base # Ps TOP LEFT HALF BRACKET +2E23 ; Grapheme_Base # Pe TOP RIGHT HALF BRACKET +2E24 ; Grapheme_Base # Ps BOTTOM LEFT HALF BRACKET +2E25 ; Grapheme_Base # Pe BOTTOM RIGHT HALF BRACKET +2E26 ; Grapheme_Base # Ps LEFT SIDEWAYS U BRACKET +2E27 ; Grapheme_Base # Pe RIGHT SIDEWAYS U BRACKET +2E28 ; Grapheme_Base # Ps LEFT DOUBLE PARENTHESIS +2E29 ; Grapheme_Base # Pe RIGHT DOUBLE PARENTHESIS +2E2A..2E2E ; Grapheme_Base # Po [5] TWO DOTS OVER ONE DOT PUNCTUATION..REVERSED QUESTION MARK +2E2F ; Grapheme_Base # Lm VERTICAL TILDE +2E30..2E31 ; Grapheme_Base # Po [2] RING POINT..WORD SEPARATOR MIDDLE DOT +2E80..2E99 ; Grapheme_Base # So [26] CJK RADICAL REPEAT..CJK RADICAL RAP +2E9B..2EF3 ; Grapheme_Base # So [89] CJK RADICAL CHOKE..CJK RADICAL C-SIMPLIFIED TURTLE +2F00..2FD5 ; Grapheme_Base # So [214] KANGXI RADICAL ONE..KANGXI RADICAL FLUTE +2FF0..2FFB ; Grapheme_Base # So [12] IDEOGRAPHIC DESCRIPTION CHARACTER LEFT TO RIGHT..IDEOGRAPHIC DESCRIPTION CHARACTER OVERLAID +3000 ; Grapheme_Base # Zs IDEOGRAPHIC SPACE +3001..3003 ; Grapheme_Base # Po [3] IDEOGRAPHIC COMMA..DITTO MARK +3004 ; Grapheme_Base # So JAPANESE INDUSTRIAL STANDARD SYMBOL +3005 ; Grapheme_Base # Lm IDEOGRAPHIC ITERATION MARK +3006 ; Grapheme_Base # Lo IDEOGRAPHIC CLOSING MARK +3007 ; Grapheme_Base # Nl IDEOGRAPHIC NUMBER ZERO +3008 ; Grapheme_Base # Ps LEFT ANGLE BRACKET +3009 ; Grapheme_Base # Pe RIGHT ANGLE BRACKET +300A ; Grapheme_Base # Ps LEFT DOUBLE ANGLE BRACKET +300B ; Grapheme_Base # Pe RIGHT DOUBLE ANGLE BRACKET +300C ; Grapheme_Base # Ps LEFT CORNER BRACKET +300D ; Grapheme_Base # Pe RIGHT CORNER BRACKET +300E ; Grapheme_Base # Ps LEFT WHITE CORNER BRACKET +300F ; Grapheme_Base # Pe RIGHT WHITE CORNER BRACKET +3010 ; Grapheme_Base # Ps LEFT BLACK LENTICULAR BRACKET +3011 ; Grapheme_Base # Pe RIGHT BLACK LENTICULAR BRACKET +3012..3013 ; Grapheme_Base # So [2] POSTAL MARK..GETA MARK +3014 ; Grapheme_Base # Ps LEFT TORTOISE SHELL BRACKET +3015 ; Grapheme_Base # Pe RIGHT TORTOISE SHELL BRACKET +3016 ; Grapheme_Base # Ps LEFT WHITE LENTICULAR BRACKET +3017 ; Grapheme_Base # Pe RIGHT WHITE LENTICULAR BRACKET +3018 ; Grapheme_Base # Ps LEFT WHITE TORTOISE SHELL BRACKET +3019 ; Grapheme_Base # Pe RIGHT WHITE TORTOISE SHELL BRACKET +301A ; Grapheme_Base # Ps LEFT WHITE SQUARE BRACKET +301B ; Grapheme_Base # Pe RIGHT WHITE SQUARE BRACKET +301C ; Grapheme_Base # Pd WAVE DASH +301D ; Grapheme_Base # Ps REVERSED DOUBLE PRIME QUOTATION MARK +301E..301F ; Grapheme_Base # Pe [2] DOUBLE PRIME QUOTATION MARK..LOW DOUBLE PRIME QUOTATION MARK +3020 ; Grapheme_Base # So POSTAL MARK FACE +3021..3029 ; Grapheme_Base # Nl [9] HANGZHOU NUMERAL ONE..HANGZHOU NUMERAL NINE +3030 ; Grapheme_Base # Pd WAVY DASH +3031..3035 ; Grapheme_Base # Lm [5] VERTICAL KANA REPEAT MARK..VERTICAL KANA REPEAT MARK LOWER HALF +3036..3037 ; Grapheme_Base # So [2] CIRCLED POSTAL MARK..IDEOGRAPHIC TELEGRAPH LINE FEED SEPARATOR SYMBOL +3038..303A ; Grapheme_Base # Nl [3] HANGZHOU NUMERAL TEN..HANGZHOU NUMERAL THIRTY +303B ; Grapheme_Base # Lm VERTICAL IDEOGRAPHIC ITERATION MARK +303C ; Grapheme_Base # Lo MASU MARK +303D ; Grapheme_Base # Po PART ALTERNATION MARK +303E..303F ; Grapheme_Base # So [2] IDEOGRAPHIC VARIATION INDICATOR..IDEOGRAPHIC HALF FILL SPACE +3041..3096 ; Grapheme_Base # Lo [86] HIRAGANA LETTER SMALL A..HIRAGANA LETTER SMALL KE +309B..309C ; Grapheme_Base # Sk [2] KATAKANA-HIRAGANA VOICED SOUND MARK..KATAKANA-HIRAGANA SEMI-VOICED SOUND MARK +309D..309E ; Grapheme_Base # Lm [2] HIRAGANA ITERATION MARK..HIRAGANA VOICED ITERATION MARK +309F ; Grapheme_Base # Lo HIRAGANA DIGRAPH YORI +30A0 ; Grapheme_Base # Pd KATAKANA-HIRAGANA DOUBLE HYPHEN +30A1..30FA ; Grapheme_Base # Lo [90] KATAKANA LETTER SMALL A..KATAKANA LETTER VO +30FB ; Grapheme_Base # Po KATAKANA MIDDLE DOT +30FC..30FE ; Grapheme_Base # Lm [3] KATAKANA-HIRAGANA PROLONGED SOUND MARK..KATAKANA VOICED ITERATION MARK +30FF ; Grapheme_Base # Lo KATAKANA DIGRAPH KOTO +3105..312D ; Grapheme_Base # Lo [41] BOPOMOFO LETTER B..BOPOMOFO LETTER IH +3131..318E ; Grapheme_Base # Lo [94] HANGUL LETTER KIYEOK..HANGUL LETTER ARAEAE +3190..3191 ; Grapheme_Base # So [2] IDEOGRAPHIC ANNOTATION LINKING MARK..IDEOGRAPHIC ANNOTATION REVERSE MARK +3192..3195 ; Grapheme_Base # No [4] IDEOGRAPHIC ANNOTATION ONE MARK..IDEOGRAPHIC ANNOTATION FOUR MARK +3196..319F ; Grapheme_Base # So [10] IDEOGRAPHIC ANNOTATION TOP MARK..IDEOGRAPHIC ANNOTATION MAN MARK +31A0..31B7 ; Grapheme_Base # Lo [24] BOPOMOFO LETTER BU..BOPOMOFO FINAL LETTER H +31C0..31E3 ; Grapheme_Base # So [36] CJK STROKE T..CJK STROKE Q +31F0..31FF ; Grapheme_Base # Lo [16] KATAKANA LETTER SMALL KU..KATAKANA LETTER SMALL RO +3200..321E ; Grapheme_Base # So [31] PARENTHESIZED HANGUL KIYEOK..PARENTHESIZED KOREAN CHARACTER O HU +3220..3229 ; Grapheme_Base # No [10] PARENTHESIZED IDEOGRAPH ONE..PARENTHESIZED IDEOGRAPH TEN +322A..3250 ; Grapheme_Base # So [39] PARENTHESIZED IDEOGRAPH MOON..PARTNERSHIP SIGN +3251..325F ; Grapheme_Base # No [15] CIRCLED NUMBER TWENTY ONE..CIRCLED NUMBER THIRTY FIVE +3260..327F ; Grapheme_Base # So [32] CIRCLED HANGUL KIYEOK..KOREAN STANDARD SYMBOL +3280..3289 ; Grapheme_Base # No [10] CIRCLED IDEOGRAPH ONE..CIRCLED IDEOGRAPH TEN +328A..32B0 ; Grapheme_Base # So [39] CIRCLED IDEOGRAPH MOON..CIRCLED IDEOGRAPH NIGHT +32B1..32BF ; Grapheme_Base # No [15] CIRCLED NUMBER THIRTY SIX..CIRCLED NUMBER FIFTY +32C0..32FE ; Grapheme_Base # So [63] IDEOGRAPHIC TELEGRAPH SYMBOL FOR JANUARY..CIRCLED KATAKANA WO +3300..33FF ; Grapheme_Base # So [256] SQUARE APAATO..SQUARE GAL +3400..4DB5 ; Grapheme_Base # Lo [6582] CJK UNIFIED IDEOGRAPH-3400..CJK UNIFIED IDEOGRAPH-4DB5 +4DC0..4DFF ; Grapheme_Base # So [64] HEXAGRAM FOR THE CREATIVE HEAVEN..HEXAGRAM FOR BEFORE COMPLETION +4E00..9FCB ; Grapheme_Base # Lo [20940] CJK UNIFIED IDEOGRAPH-4E00..CJK UNIFIED IDEOGRAPH-9FCB +A000..A014 ; Grapheme_Base # Lo [21] YI SYLLABLE IT..YI SYLLABLE E +A015 ; Grapheme_Base # Lm YI SYLLABLE WU +A016..A48C ; Grapheme_Base # Lo [1143] YI SYLLABLE BIT..YI SYLLABLE YYR +A490..A4C6 ; Grapheme_Base # So [55] YI RADICAL QOT..YI RADICAL KE +A4D0..A4F7 ; Grapheme_Base # Lo [40] LISU LETTER BA..LISU LETTER OE +A4F8..A4FD ; Grapheme_Base # Lm [6] LISU LETTER TONE MYA TI..LISU LETTER TONE MYA JEU +A4FE..A4FF ; Grapheme_Base # Po [2] LISU PUNCTUATION COMMA..LISU PUNCTUATION FULL STOP +A500..A60B ; Grapheme_Base # Lo [268] VAI SYLLABLE EE..VAI SYLLABLE NG +A60C ; Grapheme_Base # Lm VAI SYLLABLE LENGTHENER +A60D..A60F ; Grapheme_Base # Po [3] VAI COMMA..VAI QUESTION MARK +A610..A61F ; Grapheme_Base # Lo [16] VAI SYLLABLE NDOLE FA..VAI SYMBOL JONG +A620..A629 ; Grapheme_Base # Nd [10] VAI DIGIT ZERO..VAI DIGIT NINE +A62A..A62B ; Grapheme_Base # Lo [2] VAI SYLLABLE NDOLE MA..VAI SYLLABLE NDOLE DO +A640..A65F ; Grapheme_Base # L& [32] CYRILLIC CAPITAL LETTER ZEMLYA..CYRILLIC SMALL LETTER YN +A662..A66D ; Grapheme_Base # L& [12] CYRILLIC CAPITAL LETTER SOFT DE..CYRILLIC SMALL LETTER DOUBLE MONOCULAR O +A66E ; Grapheme_Base # Lo CYRILLIC LETTER MULTIOCULAR O +A673 ; Grapheme_Base # Po SLAVONIC ASTERISK +A67E ; Grapheme_Base # Po CYRILLIC KAVYKA +A67F ; Grapheme_Base # Lm CYRILLIC PAYEROK +A680..A697 ; Grapheme_Base # L& [24] CYRILLIC CAPITAL LETTER DWE..CYRILLIC SMALL LETTER SHWE +A6A0..A6E5 ; Grapheme_Base # Lo [70] BAMUM LETTER A..BAMUM LETTER KI +A6E6..A6EF ; Grapheme_Base # Nl [10] BAMUM LETTER MO..BAMUM LETTER KOGHOM +A6F2..A6F7 ; Grapheme_Base # Po [6] BAMUM NJAEMLI..BAMUM QUESTION MARK +A700..A716 ; Grapheme_Base # Sk [23] MODIFIER LETTER CHINESE TONE YIN PING..MODIFIER LETTER EXTRA-LOW LEFT-STEM TONE BAR +A717..A71F ; Grapheme_Base # Lm [9] MODIFIER LETTER DOT VERTICAL BAR..MODIFIER LETTER LOW INVERTED EXCLAMATION MARK +A720..A721 ; Grapheme_Base # Sk [2] MODIFIER LETTER STRESS AND HIGH TONE..MODIFIER LETTER STRESS AND LOW TONE +A722..A76F ; Grapheme_Base # L& [78] LATIN CAPITAL LETTER EGYPTOLOGICAL ALEF..LATIN SMALL LETTER CON +A770 ; Grapheme_Base # Lm MODIFIER LETTER US +A771..A787 ; Grapheme_Base # L& [23] LATIN SMALL LETTER DUM..LATIN SMALL LETTER INSULAR T +A788 ; Grapheme_Base # Lm MODIFIER LETTER LOW CIRCUMFLEX ACCENT +A789..A78A ; Grapheme_Base # Sk [2] MODIFIER LETTER COLON..MODIFIER LETTER SHORT EQUALS SIGN +A78B..A78C ; Grapheme_Base # L& [2] LATIN CAPITAL LETTER SALTILLO..LATIN SMALL LETTER SALTILLO +A7FB..A801 ; Grapheme_Base # Lo [7] LATIN EPIGRAPHIC LETTER REVERSED F..SYLOTI NAGRI LETTER I +A803..A805 ; Grapheme_Base # Lo [3] SYLOTI NAGRI LETTER U..SYLOTI NAGRI LETTER O +A807..A80A ; Grapheme_Base # Lo [4] SYLOTI NAGRI LETTER KO..SYLOTI NAGRI LETTER GHO +A80C..A822 ; Grapheme_Base # Lo [23] SYLOTI NAGRI LETTER CO..SYLOTI NAGRI LETTER HO +A823..A824 ; Grapheme_Base # Mc [2] SYLOTI NAGRI VOWEL SIGN A..SYLOTI NAGRI VOWEL SIGN I +A827 ; Grapheme_Base # Mc SYLOTI NAGRI VOWEL SIGN OO +A828..A82B ; Grapheme_Base # So [4] SYLOTI NAGRI POETRY MARK-1..SYLOTI NAGRI POETRY MARK-4 +A830..A835 ; Grapheme_Base # No [6] NORTH INDIC FRACTION ONE QUARTER..NORTH INDIC FRACTION THREE SIXTEENTHS +A836..A837 ; Grapheme_Base # So [2] NORTH INDIC QUARTER MARK..NORTH INDIC PLACEHOLDER MARK +A838 ; Grapheme_Base # Sc NORTH INDIC RUPEE MARK +A839 ; Grapheme_Base # So NORTH INDIC QUANTITY MARK +A840..A873 ; Grapheme_Base # Lo [52] PHAGS-PA LETTER KA..PHAGS-PA LETTER CANDRABINDU +A874..A877 ; Grapheme_Base # Po [4] PHAGS-PA SINGLE HEAD MARK..PHAGS-PA MARK DOUBLE SHAD +A880..A881 ; Grapheme_Base # Mc [2] SAURASHTRA SIGN ANUSVARA..SAURASHTRA SIGN VISARGA +A882..A8B3 ; Grapheme_Base # Lo [50] SAURASHTRA LETTER A..SAURASHTRA LETTER LLA +A8B4..A8C3 ; Grapheme_Base # Mc [16] SAURASHTRA CONSONANT SIGN HAARU..SAURASHTRA VOWEL SIGN AU +A8CE..A8CF ; Grapheme_Base # Po [2] SAURASHTRA DANDA..SAURASHTRA DOUBLE DANDA +A8D0..A8D9 ; Grapheme_Base # Nd [10] SAURASHTRA DIGIT ZERO..SAURASHTRA DIGIT NINE +A8F2..A8F7 ; Grapheme_Base # Lo [6] DEVANAGARI SIGN SPACING CANDRABINDU..DEVANAGARI SIGN CANDRABINDU AVAGRAHA +A8F8..A8FA ; Grapheme_Base # Po [3] DEVANAGARI SIGN PUSHPIKA..DEVANAGARI CARET +A8FB ; Grapheme_Base # Lo DEVANAGARI HEADSTROKE +A900..A909 ; Grapheme_Base # Nd [10] KAYAH LI DIGIT ZERO..KAYAH LI DIGIT NINE +A90A..A925 ; Grapheme_Base # Lo [28] KAYAH LI LETTER KA..KAYAH LI LETTER OO +A92E..A92F ; Grapheme_Base # Po [2] KAYAH LI SIGN CWI..KAYAH LI SIGN SHYA +A930..A946 ; Grapheme_Base # Lo [23] REJANG LETTER KA..REJANG LETTER A +A952..A953 ; Grapheme_Base # Mc [2] REJANG CONSONANT SIGN H..REJANG VIRAMA +A95F ; Grapheme_Base # Po REJANG SECTION MARK +A960..A97C ; Grapheme_Base # Lo [29] HANGUL CHOSEONG TIKEUT-MIEUM..HANGUL CHOSEONG SSANGYEORINHIEUH +A983 ; Grapheme_Base # Mc JAVANESE SIGN WIGNYAN +A984..A9B2 ; Grapheme_Base # Lo [47] JAVANESE LETTER A..JAVANESE LETTER HA +A9B4..A9B5 ; Grapheme_Base # Mc [2] JAVANESE VOWEL SIGN TARUNG..JAVANESE VOWEL SIGN TOLONG +A9BA..A9BB ; Grapheme_Base # Mc [2] JAVANESE VOWEL SIGN TALING..JAVANESE VOWEL SIGN DIRGA MURE +A9BD..A9C0 ; Grapheme_Base # Mc [4] JAVANESE CONSONANT SIGN KERET..JAVANESE PANGKON +A9C1..A9CD ; Grapheme_Base # Po [13] JAVANESE LEFT RERENGGAN..JAVANESE TURNED PADA PISELEH +A9CF ; Grapheme_Base # Lm JAVANESE PANGRANGKEP +A9D0..A9D9 ; Grapheme_Base # Nd [10] JAVANESE DIGIT ZERO..JAVANESE DIGIT NINE +A9DE..A9DF ; Grapheme_Base # Po [2] JAVANESE PADA TIRTA TUMETES..JAVANESE PADA ISEN-ISEN +AA00..AA28 ; Grapheme_Base # Lo [41] CHAM LETTER A..CHAM LETTER HA +AA2F..AA30 ; Grapheme_Base # Mc [2] CHAM VOWEL SIGN O..CHAM VOWEL SIGN AI +AA33..AA34 ; Grapheme_Base # Mc [2] CHAM CONSONANT SIGN YA..CHAM CONSONANT SIGN RA +AA40..AA42 ; Grapheme_Base # Lo [3] CHAM LETTER FINAL K..CHAM LETTER FINAL NG +AA44..AA4B ; Grapheme_Base # Lo [8] CHAM LETTER FINAL CH..CHAM LETTER FINAL SS +AA4D ; Grapheme_Base # Mc CHAM CONSONANT SIGN FINAL H +AA50..AA59 ; Grapheme_Base # Nd [10] CHAM DIGIT ZERO..CHAM DIGIT NINE +AA5C..AA5F ; Grapheme_Base # Po [4] CHAM PUNCTUATION SPIRAL..CHAM PUNCTUATION TRIPLE DANDA +AA60..AA6F ; Grapheme_Base # Lo [16] MYANMAR LETTER KHAMTI GA..MYANMAR LETTER KHAMTI FA +AA70 ; Grapheme_Base # Lm MYANMAR MODIFIER LETTER KHAMTI REDUPLICATION +AA71..AA76 ; Grapheme_Base # Lo [6] MYANMAR LETTER KHAMTI XA..MYANMAR LOGOGRAM KHAMTI HM +AA77..AA79 ; Grapheme_Base # So [3] MYANMAR SYMBOL AITON EXCLAMATION..MYANMAR SYMBOL AITON TWO +AA7A ; Grapheme_Base # Lo MYANMAR LETTER AITON RA +AA7B ; Grapheme_Base # Mc MYANMAR SIGN PAO KAREN TONE +AA80..AAAF ; Grapheme_Base # Lo [48] TAI VIET LETTER LOW KO..TAI VIET LETTER HIGH O +AAB1 ; Grapheme_Base # Lo TAI VIET VOWEL AA +AAB5..AAB6 ; Grapheme_Base # Lo [2] TAI VIET VOWEL E..TAI VIET VOWEL O +AAB9..AABD ; Grapheme_Base # Lo [5] TAI VIET VOWEL UEA..TAI VIET VOWEL AN +AAC0 ; Grapheme_Base # Lo TAI VIET TONE MAI NUENG +AAC2 ; Grapheme_Base # Lo TAI VIET TONE MAI SONG +AADB..AADC ; Grapheme_Base # Lo [2] TAI VIET SYMBOL KON..TAI VIET SYMBOL NUENG +AADD ; Grapheme_Base # Lm TAI VIET SYMBOL SAM +AADE..AADF ; Grapheme_Base # Po [2] TAI VIET SYMBOL HO HOI..TAI VIET SYMBOL KOI KOI +ABC0..ABE2 ; Grapheme_Base # Lo [35] MEETEI MAYEK LETTER KOK..MEETEI MAYEK LETTER I LONSUM +ABE3..ABE4 ; Grapheme_Base # Mc [2] MEETEI MAYEK VOWEL SIGN ONAP..MEETEI MAYEK VOWEL SIGN INAP +ABE6..ABE7 ; Grapheme_Base # Mc [2] MEETEI MAYEK VOWEL SIGN YENAP..MEETEI MAYEK VOWEL SIGN SOUNAP +ABE9..ABEA ; Grapheme_Base # Mc [2] MEETEI MAYEK VOWEL SIGN CHEINAP..MEETEI MAYEK VOWEL SIGN NUNG +ABEB ; Grapheme_Base # Po MEETEI MAYEK CHEIKHEI +ABEC ; Grapheme_Base # Mc MEETEI MAYEK LUM IYEK +ABF0..ABF9 ; Grapheme_Base # Nd [10] MEETEI MAYEK DIGIT ZERO..MEETEI MAYEK DIGIT NINE +AC00..D7A3 ; Grapheme_Base # Lo [11172] HANGUL SYLLABLE GA..HANGUL SYLLABLE HIH +D7B0..D7C6 ; Grapheme_Base # Lo [23] HANGUL JUNGSEONG O-YEO..HANGUL JUNGSEONG ARAEA-E +D7CB..D7FB ; Grapheme_Base # Lo [49] HANGUL JONGSEONG NIEUN-RIEUL..HANGUL JONGSEONG PHIEUPH-THIEUTH +F900..FA2D ; Grapheme_Base # Lo [302] CJK COMPATIBILITY IDEOGRAPH-F900..CJK COMPATIBILITY IDEOGRAPH-FA2D +FA30..FA6D ; Grapheme_Base # Lo [62] CJK COMPATIBILITY IDEOGRAPH-FA30..CJK COMPATIBILITY IDEOGRAPH-FA6D +FA70..FAD9 ; Grapheme_Base # Lo [106] CJK COMPATIBILITY IDEOGRAPH-FA70..CJK COMPATIBILITY IDEOGRAPH-FAD9 +FB00..FB06 ; Grapheme_Base # L& [7] LATIN SMALL LIGATURE FF..LATIN SMALL LIGATURE ST +FB13..FB17 ; Grapheme_Base # L& [5] ARMENIAN SMALL LIGATURE MEN NOW..ARMENIAN SMALL LIGATURE MEN XEH +FB1D ; Grapheme_Base # Lo HEBREW LETTER YOD WITH HIRIQ +FB1F..FB28 ; Grapheme_Base # Lo [10] HEBREW LIGATURE YIDDISH YOD YOD PATAH..HEBREW LETTER WIDE TAV +FB29 ; Grapheme_Base # Sm HEBREW LETTER ALTERNATIVE PLUS SIGN +FB2A..FB36 ; Grapheme_Base # Lo [13] HEBREW LETTER SHIN WITH SHIN DOT..HEBREW LETTER ZAYIN WITH DAGESH +FB38..FB3C ; Grapheme_Base # Lo [5] HEBREW LETTER TET WITH DAGESH..HEBREW LETTER LAMED WITH DAGESH +FB3E ; Grapheme_Base # Lo HEBREW LETTER MEM WITH DAGESH +FB40..FB41 ; Grapheme_Base # Lo [2] HEBREW LETTER NUN WITH DAGESH..HEBREW LETTER SAMEKH WITH DAGESH +FB43..FB44 ; Grapheme_Base # Lo [2] HEBREW LETTER FINAL PE WITH DAGESH..HEBREW LETTER PE WITH DAGESH +FB46..FBB1 ; Grapheme_Base # Lo [108] HEBREW LETTER TSADI WITH DAGESH..ARABIC LETTER YEH BARREE WITH HAMZA ABOVE FINAL FORM +FBD3..FD3D ; Grapheme_Base # Lo [363] ARABIC LETTER NG ISOLATED FORM..ARABIC LIGATURE ALEF WITH FATHATAN ISOLATED FORM +FD3E ; Grapheme_Base # Ps ORNATE LEFT PARENTHESIS +FD3F ; Grapheme_Base # Pe ORNATE RIGHT PARENTHESIS +FD50..FD8F ; Grapheme_Base # Lo [64] ARABIC LIGATURE TEH WITH JEEM WITH MEEM INITIAL FORM..ARABIC LIGATURE MEEM WITH KHAH WITH MEEM INITIAL FORM +FD92..FDC7 ; Grapheme_Base # Lo [54] ARABIC LIGATURE MEEM WITH JEEM WITH KHAH INITIAL FORM..ARABIC LIGATURE NOON WITH JEEM WITH YEH FINAL FORM +FDF0..FDFB ; Grapheme_Base # Lo [12] ARABIC LIGATURE SALLA USED AS KORANIC STOP SIGN ISOLATED FORM..ARABIC LIGATURE JALLAJALALOUHOU +FDFC ; Grapheme_Base # Sc RIAL SIGN +FDFD ; Grapheme_Base # So ARABIC LIGATURE BISMILLAH AR-RAHMAN AR-RAHEEM +FE10..FE16 ; Grapheme_Base # Po [7] PRESENTATION FORM FOR VERTICAL COMMA..PRESENTATION FORM FOR VERTICAL QUESTION MARK +FE17 ; Grapheme_Base # Ps PRESENTATION FORM FOR VERTICAL LEFT WHITE LENTICULAR BRACKET +FE18 ; Grapheme_Base # Pe PRESENTATION FORM FOR VERTICAL RIGHT WHITE LENTICULAR BRAKCET +FE19 ; Grapheme_Base # Po PRESENTATION FORM FOR VERTICAL HORIZONTAL ELLIPSIS +FE30 ; Grapheme_Base # Po PRESENTATION FORM FOR VERTICAL TWO DOT LEADER +FE31..FE32 ; Grapheme_Base # Pd [2] PRESENTATION FORM FOR VERTICAL EM DASH..PRESENTATION FORM FOR VERTICAL EN DASH +FE33..FE34 ; Grapheme_Base # Pc [2] PRESENTATION FORM FOR VERTICAL LOW LINE..PRESENTATION FORM FOR VERTICAL WAVY LOW LINE +FE35 ; Grapheme_Base # Ps PRESENTATION FORM FOR VERTICAL LEFT PARENTHESIS +FE36 ; Grapheme_Base # Pe PRESENTATION FORM FOR VERTICAL RIGHT PARENTHESIS +FE37 ; Grapheme_Base # Ps PRESENTATION FORM FOR VERTICAL LEFT CURLY BRACKET +FE38 ; Grapheme_Base # Pe PRESENTATION FORM FOR VERTICAL RIGHT CURLY BRACKET +FE39 ; Grapheme_Base # Ps PRESENTATION FORM FOR VERTICAL LEFT TORTOISE SHELL BRACKET +FE3A ; Grapheme_Base # Pe PRESENTATION FORM FOR VERTICAL RIGHT TORTOISE SHELL BRACKET +FE3B ; Grapheme_Base # Ps PRESENTATION FORM FOR VERTICAL LEFT BLACK LENTICULAR BRACKET +FE3C ; Grapheme_Base # Pe PRESENTATION FORM FOR VERTICAL RIGHT BLACK LENTICULAR BRACKET +FE3D ; Grapheme_Base # Ps PRESENTATION FORM FOR VERTICAL LEFT DOUBLE ANGLE BRACKET +FE3E ; Grapheme_Base # Pe PRESENTATION FORM FOR VERTICAL RIGHT DOUBLE ANGLE BRACKET +FE3F ; Grapheme_Base # Ps PRESENTATION FORM FOR VERTICAL LEFT ANGLE BRACKET +FE40 ; Grapheme_Base # Pe PRESENTATION FORM FOR VERTICAL RIGHT ANGLE BRACKET +FE41 ; Grapheme_Base # Ps PRESENTATION FORM FOR VERTICAL LEFT CORNER BRACKET +FE42 ; Grapheme_Base # Pe PRESENTATION FORM FOR VERTICAL RIGHT CORNER BRACKET +FE43 ; Grapheme_Base # Ps PRESENTATION FORM FOR VERTICAL LEFT WHITE CORNER BRACKET +FE44 ; Grapheme_Base # Pe PRESENTATION FORM FOR VERTICAL RIGHT WHITE CORNER BRACKET +FE45..FE46 ; Grapheme_Base # Po [2] SESAME DOT..WHITE SESAME DOT +FE47 ; Grapheme_Base # Ps PRESENTATION FORM FOR VERTICAL LEFT SQUARE BRACKET +FE48 ; Grapheme_Base # Pe PRESENTATION FORM FOR VERTICAL RIGHT SQUARE BRACKET +FE49..FE4C ; Grapheme_Base # Po [4] DASHED OVERLINE..DOUBLE WAVY OVERLINE +FE4D..FE4F ; Grapheme_Base # Pc [3] DASHED LOW LINE..WAVY LOW LINE +FE50..FE52 ; Grapheme_Base # Po [3] SMALL COMMA..SMALL FULL STOP +FE54..FE57 ; Grapheme_Base # Po [4] SMALL SEMICOLON..SMALL EXCLAMATION MARK +FE58 ; Grapheme_Base # Pd SMALL EM DASH +FE59 ; Grapheme_Base # Ps SMALL LEFT PARENTHESIS +FE5A ; Grapheme_Base # Pe SMALL RIGHT PARENTHESIS +FE5B ; Grapheme_Base # Ps SMALL LEFT CURLY BRACKET +FE5C ; Grapheme_Base # Pe SMALL RIGHT CURLY BRACKET +FE5D ; Grapheme_Base # Ps SMALL LEFT TORTOISE SHELL BRACKET +FE5E ; Grapheme_Base # Pe SMALL RIGHT TORTOISE SHELL BRACKET +FE5F..FE61 ; Grapheme_Base # Po [3] SMALL NUMBER SIGN..SMALL ASTERISK +FE62 ; Grapheme_Base # Sm SMALL PLUS SIGN +FE63 ; Grapheme_Base # Pd SMALL HYPHEN-MINUS +FE64..FE66 ; Grapheme_Base # Sm [3] SMALL LESS-THAN SIGN..SMALL EQUALS SIGN +FE68 ; Grapheme_Base # Po SMALL REVERSE SOLIDUS +FE69 ; Grapheme_Base # Sc SMALL DOLLAR SIGN +FE6A..FE6B ; Grapheme_Base # Po [2] SMALL PERCENT SIGN..SMALL COMMERCIAL AT +FE70..FE74 ; Grapheme_Base # Lo [5] ARABIC FATHATAN ISOLATED FORM..ARABIC KASRATAN ISOLATED FORM +FE76..FEFC ; Grapheme_Base # Lo [135] ARABIC FATHA ISOLATED FORM..ARABIC LIGATURE LAM WITH ALEF FINAL FORM +FF01..FF03 ; Grapheme_Base # Po [3] FULLWIDTH EXCLAMATION MARK..FULLWIDTH NUMBER SIGN +FF04 ; Grapheme_Base # Sc FULLWIDTH DOLLAR SIGN +FF05..FF07 ; Grapheme_Base # Po [3] FULLWIDTH PERCENT SIGN..FULLWIDTH APOSTROPHE +FF08 ; Grapheme_Base # Ps FULLWIDTH LEFT PARENTHESIS +FF09 ; Grapheme_Base # Pe FULLWIDTH RIGHT PARENTHESIS +FF0A ; Grapheme_Base # Po FULLWIDTH ASTERISK +FF0B ; Grapheme_Base # Sm FULLWIDTH PLUS SIGN +FF0C ; Grapheme_Base # Po FULLWIDTH COMMA +FF0D ; Grapheme_Base # Pd FULLWIDTH HYPHEN-MINUS +FF0E..FF0F ; Grapheme_Base # Po [2] FULLWIDTH FULL STOP..FULLWIDTH SOLIDUS +FF10..FF19 ; Grapheme_Base # Nd [10] FULLWIDTH DIGIT ZERO..FULLWIDTH DIGIT NINE +FF1A..FF1B ; Grapheme_Base # Po [2] FULLWIDTH COLON..FULLWIDTH SEMICOLON +FF1C..FF1E ; Grapheme_Base # Sm [3] FULLWIDTH LESS-THAN SIGN..FULLWIDTH GREATER-THAN SIGN +FF1F..FF20 ; Grapheme_Base # Po [2] FULLWIDTH QUESTION MARK..FULLWIDTH COMMERCIAL AT +FF21..FF3A ; Grapheme_Base # L& [26] FULLWIDTH LATIN CAPITAL LETTER A..FULLWIDTH LATIN CAPITAL LETTER Z +FF3B ; Grapheme_Base # Ps FULLWIDTH LEFT SQUARE BRACKET +FF3C ; Grapheme_Base # Po FULLWIDTH REVERSE SOLIDUS +FF3D ; Grapheme_Base # Pe FULLWIDTH RIGHT SQUARE BRACKET +FF3E ; Grapheme_Base # Sk FULLWIDTH CIRCUMFLEX ACCENT +FF3F ; Grapheme_Base # Pc FULLWIDTH LOW LINE +FF40 ; Grapheme_Base # Sk FULLWIDTH GRAVE ACCENT +FF41..FF5A ; Grapheme_Base # L& [26] FULLWIDTH LATIN SMALL LETTER A..FULLWIDTH LATIN SMALL LETTER Z +FF5B ; Grapheme_Base # Ps FULLWIDTH LEFT CURLY BRACKET +FF5C ; Grapheme_Base # Sm FULLWIDTH VERTICAL LINE +FF5D ; Grapheme_Base # Pe FULLWIDTH RIGHT CURLY BRACKET +FF5E ; Grapheme_Base # Sm FULLWIDTH TILDE +FF5F ; Grapheme_Base # Ps FULLWIDTH LEFT WHITE PARENTHESIS +FF60 ; Grapheme_Base # Pe FULLWIDTH RIGHT WHITE PARENTHESIS +FF61 ; Grapheme_Base # Po HALFWIDTH IDEOGRAPHIC FULL STOP +FF62 ; Grapheme_Base # Ps HALFWIDTH LEFT CORNER BRACKET +FF63 ; Grapheme_Base # Pe HALFWIDTH RIGHT CORNER BRACKET +FF64..FF65 ; Grapheme_Base # Po [2] HALFWIDTH IDEOGRAPHIC COMMA..HALFWIDTH KATAKANA MIDDLE DOT +FF66..FF6F ; Grapheme_Base # Lo [10] HALFWIDTH KATAKANA LETTER WO..HALFWIDTH KATAKANA LETTER SMALL TU +FF70 ; Grapheme_Base # Lm HALFWIDTH KATAKANA-HIRAGANA PROLONGED SOUND MARK +FF71..FF9D ; Grapheme_Base # Lo [45] HALFWIDTH KATAKANA LETTER A..HALFWIDTH KATAKANA LETTER N +FFA0..FFBE ; Grapheme_Base # Lo [31] HALFWIDTH HANGUL FILLER..HALFWIDTH HANGUL LETTER HIEUH +FFC2..FFC7 ; Grapheme_Base # Lo [6] HALFWIDTH HANGUL LETTER A..HALFWIDTH HANGUL LETTER E +FFCA..FFCF ; Grapheme_Base # Lo [6] HALFWIDTH HANGUL LETTER YEO..HALFWIDTH HANGUL LETTER OE +FFD2..FFD7 ; Grapheme_Base # Lo [6] HALFWIDTH HANGUL LETTER YO..HALFWIDTH HANGUL LETTER YU +FFDA..FFDC ; Grapheme_Base # Lo [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANGUL LETTER I +FFE0..FFE1 ; Grapheme_Base # Sc [2] FULLWIDTH CENT SIGN..FULLWIDTH POUND SIGN +FFE2 ; Grapheme_Base # Sm FULLWIDTH NOT SIGN +FFE3 ; Grapheme_Base # Sk FULLWIDTH MACRON +FFE4 ; Grapheme_Base # So FULLWIDTH BROKEN BAR +FFE5..FFE6 ; Grapheme_Base # Sc [2] FULLWIDTH YEN SIGN..FULLWIDTH WON SIGN +FFE8 ; Grapheme_Base # So HALFWIDTH FORMS LIGHT VERTICAL +FFE9..FFEC ; Grapheme_Base # Sm [4] HALFWIDTH LEFTWARDS ARROW..HALFWIDTH DOWNWARDS ARROW +FFED..FFEE ; Grapheme_Base # So [2] HALFWIDTH BLACK SQUARE..HALFWIDTH WHITE CIRCLE +FFFC..FFFD ; Grapheme_Base # So [2] OBJECT REPLACEMENT CHARACTER..REPLACEMENT CHARACTER +10000..1000B ; Grapheme_Base # Lo [12] LINEAR B SYLLABLE B008 A..LINEAR B SYLLABLE B046 JE +1000D..10026 ; Grapheme_Base # Lo [26] LINEAR B SYLLABLE B036 JO..LINEAR B SYLLABLE B032 QO +10028..1003A ; Grapheme_Base # Lo [19] LINEAR B SYLLABLE B060 RA..LINEAR B SYLLABLE B042 WO +1003C..1003D ; Grapheme_Base # Lo [2] LINEAR B SYLLABLE B017 ZA..LINEAR B SYLLABLE B074 ZE +1003F..1004D ; Grapheme_Base # Lo [15] LINEAR B SYLLABLE B020 ZO..LINEAR B SYLLABLE B091 TWO +10050..1005D ; Grapheme_Base # Lo [14] LINEAR B SYMBOL B018..LINEAR B SYMBOL B089 +10080..100FA ; Grapheme_Base # Lo [123] LINEAR B IDEOGRAM B100 MAN..LINEAR B IDEOGRAM VESSEL B305 +10100..10101 ; Grapheme_Base # Po [2] AEGEAN WORD SEPARATOR LINE..AEGEAN WORD SEPARATOR DOT +10102 ; Grapheme_Base # So AEGEAN CHECK MARK +10107..10133 ; Grapheme_Base # No [45] AEGEAN NUMBER ONE..AEGEAN NUMBER NINETY THOUSAND +10137..1013F ; Grapheme_Base # So [9] AEGEAN WEIGHT BASE UNIT..AEGEAN MEASURE THIRD SUBUNIT +10140..10174 ; Grapheme_Base # Nl [53] GREEK ACROPHONIC ATTIC ONE QUARTER..GREEK ACROPHONIC STRATIAN FIFTY MNAS +10175..10178 ; Grapheme_Base # No [4] GREEK ONE HALF SIGN..GREEK THREE QUARTERS SIGN +10179..10189 ; Grapheme_Base # So [17] GREEK YEAR SIGN..GREEK TRYBLION BASE SIGN +1018A ; Grapheme_Base # No GREEK ZERO SIGN +10190..1019B ; Grapheme_Base # So [12] ROMAN SEXTANS SIGN..ROMAN CENTURIAL SIGN +101D0..101FC ; Grapheme_Base # So [45] PHAISTOS DISC SIGN PEDESTRIAN..PHAISTOS DISC SIGN WAVY BAND +10280..1029C ; Grapheme_Base # Lo [29] LYCIAN LETTER A..LYCIAN LETTER X +102A0..102D0 ; Grapheme_Base # Lo [49] CARIAN LETTER A..CARIAN LETTER UUU3 +10300..1031E ; Grapheme_Base # Lo [31] OLD ITALIC LETTER A..OLD ITALIC LETTER UU +10320..10323 ; Grapheme_Base # No [4] OLD ITALIC NUMERAL ONE..OLD ITALIC NUMERAL FIFTY +10330..10340 ; Grapheme_Base # Lo [17] GOTHIC LETTER AHSA..GOTHIC LETTER PAIRTHRA +10341 ; Grapheme_Base # Nl GOTHIC LETTER NINETY +10342..10349 ; Grapheme_Base # Lo [8] GOTHIC LETTER RAIDA..GOTHIC LETTER OTHAL +1034A ; Grapheme_Base # Nl GOTHIC LETTER NINE HUNDRED +10380..1039D ; Grapheme_Base # Lo [30] UGARITIC LETTER ALPA..UGARITIC LETTER SSU +1039F ; Grapheme_Base # Po UGARITIC WORD DIVIDER +103A0..103C3 ; Grapheme_Base # Lo [36] OLD PERSIAN SIGN A..OLD PERSIAN SIGN HA +103C8..103CF ; Grapheme_Base # Lo [8] OLD PERSIAN SIGN AURAMAZDAA..OLD PERSIAN SIGN BUUMISH +103D0 ; Grapheme_Base # Po OLD PERSIAN WORD DIVIDER +103D1..103D5 ; Grapheme_Base # Nl [5] OLD PERSIAN NUMBER ONE..OLD PERSIAN NUMBER HUNDRED +10400..1044F ; Grapheme_Base # L& [80] DESERET CAPITAL LETTER LONG I..DESERET SMALL LETTER EW +10450..1049D ; Grapheme_Base # Lo [78] SHAVIAN LETTER PEEP..OSMANYA LETTER OO +104A0..104A9 ; Grapheme_Base # Nd [10] OSMANYA DIGIT ZERO..OSMANYA DIGIT NINE +10800..10805 ; Grapheme_Base # Lo [6] CYPRIOT SYLLABLE A..CYPRIOT SYLLABLE JA +10808 ; Grapheme_Base # Lo CYPRIOT SYLLABLE JO +1080A..10835 ; Grapheme_Base # Lo [44] CYPRIOT SYLLABLE KA..CYPRIOT SYLLABLE WO +10837..10838 ; Grapheme_Base # Lo [2] CYPRIOT SYLLABLE XA..CYPRIOT SYLLABLE XE +1083C ; Grapheme_Base # Lo CYPRIOT SYLLABLE ZA +1083F..10855 ; Grapheme_Base # Lo [23] CYPRIOT SYLLABLE ZO..IMPERIAL ARAMAIC LETTER TAW +10857 ; Grapheme_Base # Po IMPERIAL ARAMAIC SECTION SIGN +10858..1085F ; Grapheme_Base # No [8] IMPERIAL ARAMAIC NUMBER ONE..IMPERIAL ARAMAIC NUMBER TEN THOUSAND +10900..10915 ; Grapheme_Base # Lo [22] PHOENICIAN LETTER ALF..PHOENICIAN LETTER TAU +10916..1091B ; Grapheme_Base # No [6] PHOENICIAN NUMBER ONE..PHOENICIAN NUMBER THREE +1091F ; Grapheme_Base # Po PHOENICIAN WORD SEPARATOR +10920..10939 ; Grapheme_Base # Lo [26] LYDIAN LETTER A..LYDIAN LETTER C +1093F ; Grapheme_Base # Po LYDIAN TRIANGULAR MARK +10A00 ; Grapheme_Base # Lo KHAROSHTHI LETTER A +10A10..10A13 ; Grapheme_Base # Lo [4] KHAROSHTHI LETTER KA..KHAROSHTHI LETTER GHA +10A15..10A17 ; Grapheme_Base # Lo [3] KHAROSHTHI LETTER CA..KHAROSHTHI LETTER JA +10A19..10A33 ; Grapheme_Base # Lo [27] KHAROSHTHI LETTER NYA..KHAROSHTHI LETTER TTTHA +10A40..10A47 ; Grapheme_Base # No [8] KHAROSHTHI DIGIT ONE..KHAROSHTHI NUMBER ONE THOUSAND +10A50..10A58 ; Grapheme_Base # Po [9] KHAROSHTHI PUNCTUATION DOT..KHAROSHTHI PUNCTUATION LINES +10A60..10A7C ; Grapheme_Base # Lo [29] OLD SOUTH ARABIAN LETTER HE..OLD SOUTH ARABIAN LETTER THETH +10A7D..10A7E ; Grapheme_Base # No [2] OLD SOUTH ARABIAN NUMBER ONE..OLD SOUTH ARABIAN NUMBER FIFTY +10A7F ; Grapheme_Base # Po OLD SOUTH ARABIAN NUMERIC INDICATOR +10B00..10B35 ; Grapheme_Base # Lo [54] AVESTAN LETTER A..AVESTAN LETTER HE +10B39..10B3F ; Grapheme_Base # Po [7] AVESTAN ABBREVIATION MARK..LARGE ONE RING OVER TWO RINGS PUNCTUATION +10B40..10B55 ; Grapheme_Base # Lo [22] INSCRIPTIONAL PARTHIAN LETTER ALEPH..INSCRIPTIONAL PARTHIAN LETTER TAW +10B58..10B5F ; Grapheme_Base # No [8] INSCRIPTIONAL PARTHIAN NUMBER ONE..INSCRIPTIONAL PARTHIAN NUMBER ONE THOUSAND +10B60..10B72 ; Grapheme_Base # Lo [19] INSCRIPTIONAL PAHLAVI LETTER ALEPH..INSCRIPTIONAL PAHLAVI LETTER TAW +10B78..10B7F ; Grapheme_Base # No [8] INSCRIPTIONAL PAHLAVI NUMBER ONE..INSCRIPTIONAL PAHLAVI NUMBER ONE THOUSAND +10C00..10C48 ; Grapheme_Base # Lo [73] OLD TURKIC LETTER ORKHON A..OLD TURKIC LETTER ORKHON BASH +10E60..10E7E ; Grapheme_Base # No [31] RUMI DIGIT ONE..RUMI FRACTION TWO THIRDS +11082 ; Grapheme_Base # Mc KAITHI SIGN VISARGA +11083..110AF ; Grapheme_Base # Lo [45] KAITHI LETTER A..KAITHI LETTER HA +110B0..110B2 ; Grapheme_Base # Mc [3] KAITHI VOWEL SIGN AA..KAITHI VOWEL SIGN II +110B7..110B8 ; Grapheme_Base # Mc [2] KAITHI VOWEL SIGN O..KAITHI VOWEL SIGN AU +110BB..110BC ; Grapheme_Base # Po [2] KAITHI ABBREVIATION SIGN..KAITHI ENUMERATION SIGN +110BE..110C1 ; Grapheme_Base # Po [4] KAITHI SECTION MARK..KAITHI DOUBLE DANDA +12000..1236E ; Grapheme_Base # Lo [879] CUNEIFORM SIGN A..CUNEIFORM SIGN ZUM +12400..12462 ; Grapheme_Base # Nl [99] CUNEIFORM NUMERIC SIGN TWO ASH..CUNEIFORM NUMERIC SIGN OLD ASSYRIAN ONE QUARTER +12470..12473 ; Grapheme_Base # Po [4] CUNEIFORM PUNCTUATION SIGN OLD ASSYRIAN WORD DIVIDER..CUNEIFORM PUNCTUATION SIGN DIAGONAL TRICOLON +13000..1342E ; Grapheme_Base # Lo [1071] EGYPTIAN HIEROGLYPH A001..EGYPTIAN HIEROGLYPH AA032 +1D000..1D0F5 ; Grapheme_Base # So [246] BYZANTINE MUSICAL SYMBOL PSILI..BYZANTINE MUSICAL SYMBOL GORGON NEO KATO +1D100..1D126 ; Grapheme_Base # So [39] MUSICAL SYMBOL SINGLE BARLINE..MUSICAL SYMBOL DRUM CLEF-2 +1D129..1D164 ; Grapheme_Base # So [60] MUSICAL SYMBOL MULTIPLE MEASURE REST..MUSICAL SYMBOL ONE HUNDRED TWENTY-EIGHTH NOTE +1D166 ; Grapheme_Base # Mc MUSICAL SYMBOL COMBINING SPRECHGESANG STEM +1D16A..1D16C ; Grapheme_Base # So [3] MUSICAL SYMBOL FINGERED TREMOLO-1..MUSICAL SYMBOL FINGERED TREMOLO-3 +1D16D ; Grapheme_Base # Mc MUSICAL SYMBOL COMBINING AUGMENTATION DOT +1D183..1D184 ; Grapheme_Base # So [2] MUSICAL SYMBOL ARPEGGIATO UP..MUSICAL SYMBOL ARPEGGIATO DOWN +1D18C..1D1A9 ; Grapheme_Base # So [30] MUSICAL SYMBOL RINFORZANDO..MUSICAL SYMBOL DEGREE SLASH +1D1AE..1D1DD ; Grapheme_Base # So [48] MUSICAL SYMBOL PEDAL MARK..MUSICAL SYMBOL PES SUBPUNCTIS +1D200..1D241 ; Grapheme_Base # So [66] GREEK VOCAL NOTATION SYMBOL-1..GREEK INSTRUMENTAL NOTATION SYMBOL-54 +1D245 ; Grapheme_Base # So GREEK MUSICAL LEIMMA +1D300..1D356 ; Grapheme_Base # So [87] MONOGRAM FOR EARTH..TETRAGRAM FOR FOSTERING +1D360..1D371 ; Grapheme_Base # No [18] COUNTING ROD UNIT DIGIT ONE..COUNTING ROD TENS DIGIT NINE +1D400..1D454 ; Grapheme_Base # L& [85] MATHEMATICAL BOLD CAPITAL A..MATHEMATICAL ITALIC SMALL G +1D456..1D49C ; Grapheme_Base # L& [71] MATHEMATICAL ITALIC SMALL I..MATHEMATICAL SCRIPT CAPITAL A +1D49E..1D49F ; Grapheme_Base # L& [2] MATHEMATICAL SCRIPT CAPITAL C..MATHEMATICAL SCRIPT CAPITAL D +1D4A2 ; Grapheme_Base # L& MATHEMATICAL SCRIPT CAPITAL G +1D4A5..1D4A6 ; Grapheme_Base # L& [2] MATHEMATICAL SCRIPT CAPITAL J..MATHEMATICAL SCRIPT CAPITAL K +1D4A9..1D4AC ; Grapheme_Base # L& [4] MATHEMATICAL SCRIPT CAPITAL N..MATHEMATICAL SCRIPT CAPITAL Q +1D4AE..1D4B9 ; Grapheme_Base # L& [12] MATHEMATICAL SCRIPT CAPITAL S..MATHEMATICAL SCRIPT SMALL D +1D4BB ; Grapheme_Base # L& MATHEMATICAL SCRIPT SMALL F +1D4BD..1D4C3 ; Grapheme_Base # L& [7] MATHEMATICAL SCRIPT SMALL H..MATHEMATICAL SCRIPT SMALL N +1D4C5..1D505 ; Grapheme_Base # L& [65] MATHEMATICAL SCRIPT SMALL P..MATHEMATICAL FRAKTUR CAPITAL B +1D507..1D50A ; Grapheme_Base # L& [4] MATHEMATICAL FRAKTUR CAPITAL D..MATHEMATICAL FRAKTUR CAPITAL G +1D50D..1D514 ; Grapheme_Base # L& [8] MATHEMATICAL FRAKTUR CAPITAL J..MATHEMATICAL FRAKTUR CAPITAL Q +1D516..1D51C ; Grapheme_Base # L& [7] MATHEMATICAL FRAKTUR CAPITAL S..MATHEMATICAL FRAKTUR CAPITAL Y +1D51E..1D539 ; Grapheme_Base # L& [28] MATHEMATICAL FRAKTUR SMALL A..MATHEMATICAL DOUBLE-STRUCK CAPITAL B +1D53B..1D53E ; Grapheme_Base # L& [4] MATHEMATICAL DOUBLE-STRUCK CAPITAL D..MATHEMATICAL DOUBLE-STRUCK CAPITAL G +1D540..1D544 ; Grapheme_Base # L& [5] MATHEMATICAL DOUBLE-STRUCK CAPITAL I..MATHEMATICAL DOUBLE-STRUCK CAPITAL M +1D546 ; Grapheme_Base # L& MATHEMATICAL DOUBLE-STRUCK CAPITAL O +1D54A..1D550 ; Grapheme_Base # L& [7] MATHEMATICAL DOUBLE-STRUCK CAPITAL S..MATHEMATICAL DOUBLE-STRUCK CAPITAL Y +1D552..1D6A5 ; Grapheme_Base # L& [340] MATHEMATICAL DOUBLE-STRUCK SMALL A..MATHEMATICAL ITALIC SMALL DOTLESS J +1D6A8..1D6C0 ; Grapheme_Base # L& [25] MATHEMATICAL BOLD CAPITAL ALPHA..MATHEMATICAL BOLD CAPITAL OMEGA +1D6C1 ; Grapheme_Base # Sm MATHEMATICAL BOLD NABLA +1D6C2..1D6DA ; Grapheme_Base # L& [25] MATHEMATICAL BOLD SMALL ALPHA..MATHEMATICAL BOLD SMALL OMEGA +1D6DB ; Grapheme_Base # Sm MATHEMATICAL BOLD PARTIAL DIFFERENTIAL +1D6DC..1D6FA ; Grapheme_Base # L& [31] MATHEMATICAL BOLD EPSILON SYMBOL..MATHEMATICAL ITALIC CAPITAL OMEGA +1D6FB ; Grapheme_Base # Sm MATHEMATICAL ITALIC NABLA +1D6FC..1D714 ; Grapheme_Base # L& [25] MATHEMATICAL ITALIC SMALL ALPHA..MATHEMATICAL ITALIC SMALL OMEGA +1D715 ; Grapheme_Base # Sm MATHEMATICAL ITALIC PARTIAL DIFFERENTIAL +1D716..1D734 ; Grapheme_Base # L& [31] MATHEMATICAL ITALIC EPSILON SYMBOL..MATHEMATICAL BOLD ITALIC CAPITAL OMEGA +1D735 ; Grapheme_Base # Sm MATHEMATICAL BOLD ITALIC NABLA +1D736..1D74E ; Grapheme_Base # L& [25] MATHEMATICAL BOLD ITALIC SMALL ALPHA..MATHEMATICAL BOLD ITALIC SMALL OMEGA +1D74F ; Grapheme_Base # Sm MATHEMATICAL BOLD ITALIC PARTIAL DIFFERENTIAL +1D750..1D76E ; Grapheme_Base # L& [31] MATHEMATICAL BOLD ITALIC EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD CAPITAL OMEGA +1D76F ; Grapheme_Base # Sm MATHEMATICAL SANS-SERIF BOLD NABLA +1D770..1D788 ; Grapheme_Base # L& [25] MATHEMATICAL SANS-SERIF BOLD SMALL ALPHA..MATHEMATICAL SANS-SERIF BOLD SMALL OMEGA +1D789 ; Grapheme_Base # Sm MATHEMATICAL SANS-SERIF BOLD PARTIAL DIFFERENTIAL +1D78A..1D7A8 ; Grapheme_Base # L& [31] MATHEMATICAL SANS-SERIF BOLD EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL OMEGA +1D7A9 ; Grapheme_Base # Sm MATHEMATICAL SANS-SERIF BOLD ITALIC NABLA +1D7AA..1D7C2 ; Grapheme_Base # L& [25] MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL ALPHA..MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL OMEGA +1D7C3 ; Grapheme_Base # Sm MATHEMATICAL SANS-SERIF BOLD ITALIC PARTIAL DIFFERENTIAL +1D7C4..1D7CB ; Grapheme_Base # L& [8] MATHEMATICAL SANS-SERIF BOLD ITALIC EPSILON SYMBOL..MATHEMATICAL BOLD SMALL DIGAMMA +1D7CE..1D7FF ; Grapheme_Base # Nd [50] MATHEMATICAL BOLD DIGIT ZERO..MATHEMATICAL MONOSPACE DIGIT NINE +1F000..1F02B ; Grapheme_Base # So [44] MAHJONG TILE EAST WIND..MAHJONG TILE BACK +1F030..1F093 ; Grapheme_Base # So [100] DOMINO TILE HORIZONTAL BACK..DOMINO TILE VERTICAL-06-06 +1F100..1F10A ; Grapheme_Base # No [11] DIGIT ZERO FULL STOP..DIGIT NINE COMMA +1F110..1F12E ; Grapheme_Base # So [31] PARENTHESIZED LATIN CAPITAL LETTER A..CIRCLED WZ +1F131 ; Grapheme_Base # So SQUARED LATIN CAPITAL LETTER B +1F13D ; Grapheme_Base # So SQUARED LATIN CAPITAL LETTER N +1F13F ; Grapheme_Base # So SQUARED LATIN CAPITAL LETTER P +1F142 ; Grapheme_Base # So SQUARED LATIN CAPITAL LETTER S +1F146 ; Grapheme_Base # So SQUARED LATIN CAPITAL LETTER W +1F14A..1F14E ; Grapheme_Base # So [5] SQUARED HV..SQUARED PPV +1F157 ; Grapheme_Base # So NEGATIVE CIRCLED LATIN CAPITAL LETTER H +1F15F ; Grapheme_Base # So NEGATIVE CIRCLED LATIN CAPITAL LETTER P +1F179 ; Grapheme_Base # So NEGATIVE SQUARED LATIN CAPITAL LETTER J +1F17B..1F17C ; Grapheme_Base # So [2] NEGATIVE SQUARED LATIN CAPITAL LETTER L..NEGATIVE SQUARED LATIN CAPITAL LETTER M +1F17F ; Grapheme_Base # So NEGATIVE SQUARED LATIN CAPITAL LETTER P +1F18A..1F18D ; Grapheme_Base # So [4] CROSSED NEGATIVE SQUARED LATIN CAPITAL LETTER P..NEGATIVE SQUARED SA +1F190 ; Grapheme_Base # So SQUARE DJ +1F200 ; Grapheme_Base # So SQUARE HIRAGANA HOKA +1F210..1F231 ; Grapheme_Base # So [34] SQUARED CJK UNIFIED IDEOGRAPH-624B..SQUARED CJK UNIFIED IDEOGRAPH-6253 +1F240..1F248 ; Grapheme_Base # So [9] TORTOISE SHELL BRACKETED CJK UNIFIED IDEOGRAPH-672C..TORTOISE SHELL BRACKETED CJK UNIFIED IDEOGRAPH-6557 +20000..2A6D6 ; Grapheme_Base # Lo [42711] CJK UNIFIED IDEOGRAPH-20000..CJK UNIFIED IDEOGRAPH-2A6D6 +2A700..2B734 ; Grapheme_Base # Lo [4149] CJK UNIFIED IDEOGRAPH-2A700..CJK UNIFIED IDEOGRAPH-2B734 +2F800..2FA1D ; Grapheme_Base # Lo [542] CJK COMPATIBILITY IDEOGRAPH-2F800..CJK COMPATIBILITY IDEOGRAPH-2FA1D + +# Total code points: 105958 + +# ================================================ + +# Derived Property: Grapheme_Link (deprecated) +# Generated from: Canonical_Combining_Class=Virama +# Use Canonical_Combining_Class=Virama directly instead + +094D ; Grapheme_Link # Mn DEVANAGARI SIGN VIRAMA +09CD ; Grapheme_Link # Mn BENGALI SIGN VIRAMA +0A4D ; Grapheme_Link # Mn GURMUKHI SIGN VIRAMA +0ACD ; Grapheme_Link # Mn GUJARATI SIGN VIRAMA +0B4D ; Grapheme_Link # Mn ORIYA SIGN VIRAMA +0BCD ; Grapheme_Link # Mn TAMIL SIGN VIRAMA +0C4D ; Grapheme_Link # Mn TELUGU SIGN VIRAMA +0CCD ; Grapheme_Link # Mn KANNADA SIGN VIRAMA +0D4D ; Grapheme_Link # Mn MALAYALAM SIGN VIRAMA +0DCA ; Grapheme_Link # Mn SINHALA SIGN AL-LAKUNA +0E3A ; Grapheme_Link # Mn THAI CHARACTER PHINTHU +0F84 ; Grapheme_Link # Mn TIBETAN MARK HALANTA +1039..103A ; Grapheme_Link # Mn [2] MYANMAR SIGN VIRAMA..MYANMAR SIGN ASAT +1714 ; Grapheme_Link # Mn TAGALOG SIGN VIRAMA +1734 ; Grapheme_Link # Mn HANUNOO SIGN PAMUDPOD +17D2 ; Grapheme_Link # Mn KHMER SIGN COENG +1A60 ; Grapheme_Link # Mn TAI THAM SIGN SAKOT +1B44 ; Grapheme_Link # Mc BALINESE ADEG ADEG +1BAA ; Grapheme_Link # Mc SUNDANESE SIGN PAMAAEH +A806 ; Grapheme_Link # Mn SYLOTI NAGRI SIGN HASANTA +A8C4 ; Grapheme_Link # Mn SAURASHTRA SIGN VIRAMA +A953 ; Grapheme_Link # Mc REJANG VIRAMA +A9C0 ; Grapheme_Link # Mc JAVANESE PANGKON +ABED ; Grapheme_Link # Mn MEETEI MAYEK APUN IYEK +10A3F ; Grapheme_Link # Mn KHAROSHTHI VIRAMA +110B9 ; Grapheme_Link # Mn KAITHI SIGN VIRAMA + +# Total code points: 27 + +# EOF diff --git a/pypy/module/unicodedata/generate_unicodedb.py b/pypy/module/unicodedata/generate_unicodedb.py --- a/pypy/module/unicodedata/generate_unicodedb.py +++ b/pypy/module/unicodedata/generate_unicodedb.py @@ -37,6 +37,7 @@ self.excluded = False self.linebreak = False self.decompositionTag = '' + self.properties = () if data[5]: self.raw_decomposition = data[5] if data[5][0] == '<': @@ -95,7 +96,8 @@ return table[code].canonical_decomp def read_unicodedata(unicodedata_file, exclusions_file, east_asian_width_file, - unihan_file=None, linebreak_file=None): + unihan_file=None, linebreak_file=None, + derived_core_properties_file=None): rangeFirst = {} rangeLast = {} table = [None] * (MAXUNICODE + 1) @@ -164,6 +166,27 @@ else: table[int(code, 16)].east_asian_width = width + # Read Derived Core Properties: + for line in derived_core_properties_file: + line = line.split('#', 1)[0].strip() + if not line: + continue + + r, p = line.split(";") + r = r.strip() + p = p.strip() + if ".." in r: + first, last = [int(c, 16) for c in r.split('..')] + chars = list(range(first, last+1)) + else: + chars = [int(r, 16)] + for char in chars: + if not table[char]: + # Some properties (e.g. Default_Ignorable_Code_Point) + # apply to unassigned code points; ignore them + continue + table[char].properties += (p,) + # Expand ranges for (first, last), char in ranges.iteritems(): for code in range(first, last + 1): @@ -257,6 +280,8 @@ IS_DIGIT = 128 IS_DECIMAL = 256 IS_MIRRORED = 512 + IS_XID_START = 1024 + IS_XID_CONTINUE = 2048 # Create the records db_records = {} for code in range(len(table)): @@ -282,6 +307,10 @@ flags |= IS_LOWER if char.mirrored: flags |= IS_MIRRORED + if "XID_Start" in char.properties: + flags |= IS_XID_START + if "XID_Continue" in char.properties: + flags |= IS_XID_CONTINUE char.db_record = (char.category, char.bidirectional, char.east_asian_width, flags, char.combining) db_records[char.db_record] = 1 db_records = db_records.keys() @@ -335,6 +364,8 @@ print >> outfile, 'def istitle(code): return _get_record(code)[3] & %d != 0'% IS_TITLE print >> outfile, 'def islower(code): return _get_record(code)[3] & %d != 0'% IS_LOWER print >> outfile, 'def iscased(code): return _get_record(code)[3] & %d != 0'% (IS_UPPER | IS_TITLE | IS_LOWER) + print >> outfile, 'def isxidstart(code): return _get_record(code)[3] & %d != 0'% (IS_XID_START) + print >> outfile, 'def isxidcontinue(code): return _get_record(code)[3] & %d != 0'% (IS_XID_CONTINUE) print >> outfile, 'def mirrored(code): return _get_record(code)[3] & %d != 0'% IS_MIRRORED print >> outfile, 'def combining(code): return _get_record(code)[4]' @@ -681,9 +712,11 @@ east_asian_width = open('EastAsianWidth-%s.txt' % options.unidata_version) unihan = open('UnihanNumeric-%s.txt' % options.unidata_version) linebreak = open('LineBreak-%s.txt' % options.unidata_version) + derived_core_properties = open('DerivedCoreProperties-%s.txt' % + options.unidata_version) table = read_unicodedata(infile, exclusions, east_asian_width, unihan, - linebreak) + linebreak, derived_core_properties) print >> outfile, '# UNICODE CHARACTER DATABASE' print >> outfile, '# This file was generated with the command:' print >> outfile, '# ', ' '.join(sys.argv) diff --git a/pypy/module/unicodedata/unicodedb_5_2_0.py b/pypy/module/unicodedata/unicodedb_5_2_0.py --- a/pypy/module/unicodedata/unicodedb_5_2_0.py +++ b/pypy/module/unicodedata/unicodedb_5_2_0.py @@ -221,7 +221,7 @@ '\x043 RI' '\x05MUOY ' '\x05HEAD ' -'\x08AVY BAND' +'\x08RDEL NAG' '\x07R TSHES' '\x12KBAR ISOLATED FORM' '\x07KYLISMA' @@ -304,13 +304,12 @@ '\x06WIDTH ' '\x0cN ELEMENT OF' '\x0fDAGESH OR MAPIQ' -'\x08CANDICUS' +'!RIGHTWARDS HARPOON WITH BARB DOWN' '\x0cTHODOX CROSS' '\x10AM ISOLATED FORM' '\x08ULL STOP' '\x0eDEYTEROS ICHOS' '\x04ZERO' -'!ALL BUT UPPER LEFT QUADRANT BLACK' '\x05OSS O' '\x04ERET' '\x10VERTICALLY ABOVE' @@ -617,7 +616,6 @@ '\x05EAGLE' '\x05TIMES' '\x05BREW ' -'\x04AMMA' '\x0eWITH DIAERESIS' '\x06OR EQU' '\x08CRO SIGN' @@ -697,7 +695,7 @@ '\x03IRT' '\x03IRU' '\tNTERPRISE' -'\x03IRI' +'\x07DISIMOU' '\x1bKATHAKA INDEPENDENT SVARITA' '\x05TYR T' '\x03IRO' @@ -766,7 +764,7 @@ '\x02NG' '\x02NY' '\tIRCUMFLEX' -'\x02NT' +'\x07AS SIGN' '\x02NU' '\x02NW' '\x0fFOUR DOTS ABOVE' @@ -804,7 +802,6 @@ '\x0cMBELLISHMENT' '\tNCLOSING ' '\x07MINIMA ' -'\x07SHORT A' '\rEAVENLY EARTH' '\x0cEMISOFT SIGN' '\x043 MI' @@ -887,7 +884,7 @@ '\x1dDOTS OVER ONE DOT PUNCTUATION' '\x04IXTH' '\x03UR-' -'\x08RDEL NAG' +'\x08AVY BAND' '\x0bBLACK ARROW' '\x073 AREPA' '\x03URE' @@ -918,7 +915,7 @@ '\x04 ALL' '\x05THIRD' '\x04GIBA' -'\nLET SYMBOL' +'\x03OVE' '\rFULL SURROUND' '\n TIMES LAL' '\x18RIGHTWARDS THEN CURVING ' @@ -1067,7 +1064,6 @@ '\x0eGYPTOLOGICAL A' '\x04GHWA' '\x15ALATALIZED HOOK BELOW' -'\x13TRICTED LEFT ENTRY-' '\x070 SPEAR' '\x16HEXIFORM LONG ANUSVARA' '\x02OO' @@ -1189,7 +1185,7 @@ '\x06RGLASS' '\x05CREEN' '\x04MON ' -'\x08NEIFORM ' +'\x12 THROUGH DESCENDER' '\x05LEFT ' '\x05A-ROL' '\x0fQUADRUPLE ARROW' @@ -1197,7 +1193,7 @@ '\x08NG RTAGS' '\x07 TEDUNG' '\x0cNUMERAL SIGN' -'\x03OVE' +'\nLET SYMBOL' '\x06TO BAR' '\x11TAN ISOLATED FORM' '\x03D70' @@ -1232,7 +1228,7 @@ '\rGYA GRAM SHAD' '\x07RESILLO' '\x10FALLING DIAGONAL' -'\x12 THROUGH DESCENDER' +'\x13TRICTED LEFT ENTRY-' '\x08THAMASAT' '\x14SINGLE DOT TONE MARK' '\x08 STATERS' @@ -1391,6 +1387,7 @@ '\x06TAUROS' '\x03CR ' '\x03OPO' +'\x12LEFT-STEM TONE BAR' '\x05 FACE' '\x08FROM BAR' '\x17YELORUSSIAN-UKRAINIAN I' @@ -1472,7 +1469,7 @@ '\x03O B' '\x03O N' '\x18LINE HORIZONTAL ELLIPSIS' -'!RIGHTWARDS HARPOON WITH BARB DOWN' +'\x08CANDICUS' '\x03O Y' '\x04MINI' '\x0cENOS CHRONOU' @@ -1642,7 +1639,7 @@ '\x05SHARA' '\x05ESIS ' '\rWITH UNDERBAR' -'\x19 WITH DOUBLE GRAVE ACCENT' +'\x19OUTLINED RIGHTWARDS ARROW' '\x05SHARP' '\x05SHARU' "'USED AS KORANIC STOP SIGN ISOLATED FORM" @@ -2369,7 +2366,9 @@ '\x05CHECK' '\x07SIGN PA' '\x08BOL FOR ' +'\x08NEIFORM ' '\x05 HAA ' +'\x04 RHO' '\nPLUS NAGA ' '\x04GRU ' '\x05TONE ' @@ -2968,8 +2967,8 @@ '\x03H00' '\x02U0' '\x02U3' -'\x05NADA ' -'\x03EEL' +'\x02U2' +'\x0cCIAN LETTER ' '\x10CORNER DOWNWARDS' '\x03EEN' '\x03EEE' @@ -3750,7 +3749,7 @@ '\x10SLANTED EQUAL TO' '\x0bIAN LETTER ' '\x12KOREAN CHARACTER O' -'\x16OVER LEFTWARDS HARPOON' +'\x0cGEBA KAREN I' '\tLVIN SIGN' '\x06BISHOP' '\x0cPER-EM SPACE' @@ -3772,7 +3771,7 @@ '\x03TAS' '\x05LAMED' '\x08EQUAL TO' -'\x03TAA' +'\rAWELLEMET YAZ' '\x03TAB' '\x03TAL' '\x03TAM' @@ -3782,7 +3781,7 @@ '\x05PRIME' '\x0cUN WITH RAYS' '\x04KAKO' -'\x19OUTLINED RIGHTWARDS ARROW' +'\x19 WITH DOUBLE GRAVE ACCENT' '\x18E PLUS A PLUS SU PLUS NA' '\x0bQUARTER ASH' '\x03DJA' @@ -3937,7 +3936,7 @@ '\x02EP' '\x02ER' '\x04AMIL' -'\x07DISIMOU' +'\x03IRI' '\tERCIAL AT' '\x1aMARRIED PARTNERSHIP SYMBOL' '\x06NTOGEN' @@ -4084,6 +4083,7 @@ '\x18VOICED LARYNGEAL SPIRANT' '\t PLUS GUD' '\x05TAL S' +'\x0fEFTWARDS ARROWS' '\x0cRAH BEN YOMO' '\t2 GARMENT' '\x11NAUDIZ NYD NAUD N' @@ -4318,7 +4318,7 @@ '\x07STROKE ' '\r-OR-PLUS SIGN' '\x0eJOINED SQUARES' -'\x07AS SIGN' +'\x02NT' '\nRUPEE SIGN' '\x1fUPWARDS HARPOON WITH BARB RIGHT' '\x08 CURRENT' @@ -4342,7 +4342,7 @@ '\x15INVERTED GLOTTAL STOP' '\x05RIGHT' '\nCTION MARK' -'\x12LEFT-STEM TONE BAR' +'\x04AMMA' '\x0bERCENT SIGN' '\x07HIN DOT' '\x08OVER SAG' @@ -4459,7 +4459,7 @@ '\x04ARGI' '\x05DIAL ' '\x04ARGA' -'\x0cGEBA KAREN I' +'\x16OVER LEFTWARDS HARPOON' '\x04KING' '\x05ICHON' '\x03OSS' @@ -4597,7 +4597,7 @@ '\x0bUNCTUATION ' '\x06ANCHOR' '\x07PRICORN' -'\rAWELLEMET YAZ' +'\x03TAA' '\t PLUS ASH' '\x05NGLE ' '\x0bTHEMATICAL ' @@ -4655,7 +4655,6 @@ '\x0eTHAKA ANUDATTA' '\x15AKIA TELOUS ICHIMATOS' '\x04HORN' -'\x04 RHO' '\x05OKARA' '\x03MUG' '\x10HEAVY AND RIGHT ' @@ -4869,7 +4868,7 @@ '\x06WN BOX' '\x1cLIQUID MEASURE FIRST SUBUNIT' '\x0cMETA STAVROU' -'\x08MAKSURA ' +'\x0cINITIAL FORM' '\x07 LONSUM' '\x04TIKI' '\x15VOICED ITERATION MARK' @@ -4917,7 +4916,7 @@ '\x0bTRUNCATED A' '\x05JEEM ' '\x07OVERBAR' -'\x0fEFTWARDS ARROWS' +'!ALL BUT UPPER LEFT QUADRANT BLACK' '\x19INVERTED EXCLAMATION MARK' '\x06-IEUNG' '\t DRACHMAS' @@ -5041,6 +5040,7 @@ '\tCAPITAL D' '\tCAPITAL F' '\x1dDOWN HEAVY AND RIGHT UP LIGHT' +'\x0fGH VOLTAGE SIGN' '\nETA SYMBOL' '\x03GOU' '\x03AY ' @@ -5066,7 +5066,7 @@ '\x13VARIANT FORM ILIMMU' '\x01H' '\x03EQU' -'\x0cINITIAL FORM' +'\x08MAKSURA ' '\x04ADDA' '\x05SHED ' '\x08MEM-QOPH' @@ -5136,7 +5136,7 @@ '\x02IE' '\x02ID' '\x02IG' -'\x0fGH VOLTAGE SIGN' +'\x07SHORT A' '\x13ARTIAL DIFFERENTIAL' '\x04AME ' '\x04ZATA' @@ -5560,10 +5560,10 @@ '\x0bIOR PRODUCT' '\x06IGAMMA' '\x04MUIN' -'\x02U2' +'\x05NADA ' '\x0bRIPLE PRIME' '\x0bMIDDLE HOOK' -'\x0cCIAN LETTER ' +'\x03EEL' '\x05MAI K' '\x04NGIA' '\x16HORT HORIZONTAL STROKE' @@ -5985,3286 +5985,3286 @@ '\x06ALENTS' ) _charnodes =[70758, - -33936, + -33944, -1, 132371, - 34747, + 34739, -1, 197694, - 104236, + 104224, -1, 262727, - 169245, + 169237, -1, 327957, - 251140, + 251146, -1, 393238, - 280647, + 280619, -1, -65529, - 346183, + 346155, 195071, -65528, - 396588, + 396579, 195070, -65527, - 506107, + 506087, 195069, -65526, - 555888, + 555880, 195068, -65525, - 605632, + 605604, 195067, -65524, 656179, 195066, -65523, - 775428, + 775434, 195065, -65522, - 825353, + 825341, 195064, -65521, - 874626, + 874612, 195063, -65520, - 925298, + 925285, 195062, -65519, - 1034822, + 1034828, 195061, -65518, - 1084557, + 1084549, 195060, -65517, - 1134189, + 1134161, 195059, -65516, - 1184784, + 1184775, 195058, -65515, - 1293992, + 1293994, 195057, -1, - 1344078, + 1344070, 195056, 1507367, - 331052, + 331043, -1, -65512, - 1460295, + 1460267, 195055, -65511, - 1510700, + 1510691, 195054, -65510, - 1620219, + 1620199, 195053, -65509, - 1670000, + 1669992, 195052, -65508, - 1719744, + 1719716, 195051, -65507, 1770291, 195050, -65506, - 1889540, + 1889546, 195049, -65505, - 1939465, + 1939453, 195048, -65504, - 1988738, + 1988724, 195047, -65503, - 2039410, + 2039397, 195046, -65502, - 2148934, + 2148940, 195045, -65501, - 2198669, + 2198661, 195044, -65500, - 2248301, + 2248273, 195043, -65499, - 2298896, + 2298887, 195042, -65498, - 2408104, + 2408106, 195041, -1, - 2458190, + 2458182, 195040, 2621496, - 1489147, + 1489127, -1, -65495, - 2574407, + 2574379, 195039, -65494, - 2624812, + 2624803, 195038, -65493, - 2734331, + 2734311, 195037, -65492, - 2784112, + 2784104, 195036, -65491, - 2833856, + 2833828, 195035, -65490, 2884403, 195034, -65489, - 3003652, + 3003658, 195033, -65488, - 3053577, + 3053565, 195032, -65487, - 3102850, + 3102836, 195031, -65486, - 3153522, + 3153509, 195030, -65485, - 3263046, + 3263052, 195029, -65484, - 3312781, + 3312773, 195028, -65483, - 3362413, + 3362385, 195027, -65482, - 3413008, + 3412999, 195026, -65481, - 3522216, + 3522218, 195025, -1, - 3572302, + 3572294, 195024, 3735625, - 2587504, + 2587496, -1, -65478, - 3688519, + 3688491, 195023, -65477, - 3738924, + 3738915, 195022, -65476, - 3848443, + 3848423, 195021, -65475, - 3898224, + 3898216, 195020, -65474, - 3947968, + 3947940, 195019, -65473, 3998515, 195018, -65472, - 4117764, + 4117770, 195017, -65471, - 4167689, + 4167677, 195016, -65470, - 4216962, + 4216948, 195015, -65469, - 4267634, + 4267621, 195014, -65468, - 4377158, + 4377164, 195013, -65467, - 4426893, + 4426885, 195012, -65466, - 4476525, + 4476497, 195011, -65465, - 4527120, + 4527111, 195010, -65464, - 4636328, + 4636330, 195009, -1, - 4686414, + 4686406, 195008, 4849754, - 3685824, + 3685796, -1, -65461, - 4802631, + 4802603, 195007, -65460, - 4853036, + 4853027, 195006, -65459, - 4962555, + 4962535, 195005, -65458, - 5012336, + 5012328, 195004, -65457, - 5062080, + 5062052, 195003, -65456, 5112627, 195002, -65455, - 5231876, + 5231882, 195001, -65454, - 5281801, + 5281789, 195000, -65453, - 5331074, + 5331060, 194999, -65452, - 5381746, + 5381733, 194998, -65451, - 5491270, + 5491276, 194997, -65450, - 5541005, + 5540997, 194996, -65449, - 5590637, + 5590609, 194995, -65448, - 5641232, + 5641223, 194994, -65447, - 5750440, + 5750442, 194993, -1, From noreply at buildbot.pypy.org Sat Jan 14 21:48:26 2012 From: noreply at buildbot.pypy.org (amauryfa) Date: Sat, 14 Jan 2012 21:48:26 +0100 (CET) Subject: [pypy-commit] pypy py3k: Implement str.isidentifier() Message-ID: <20120114204826.EEDDE82B12@wyvern.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3k Changeset: r51318:c863145453cb Date: 2011-12-22 10:44 +0100 http://bitbucket.org/pypy/pypy/changeset/c863145453cb/ Log: Implement str.isidentifier() diff --git a/pypy/objspace/std/test/test_unicodeobject.py b/pypy/objspace/std/test/test_unicodeobject.py --- a/pypy/objspace/std/test/test_unicodeobject.py +++ b/pypy/objspace/std/test/test_unicodeobject.py @@ -162,6 +162,13 @@ assert "!Brown Fox".istitle() == True assert "Brow&&&&N Fox".istitle() == True assert "!Brow&&&&n Fox".istitle() == False + + def test_isidentifier(self): + assert "".isidentifier() is False + assert "a4".isidentifier() is True + assert "_var".isidentifier() is True + assert "_!var".isidentifier() is False + assert "3abc".isidentifier() is False def test_capitalize(self): assert "brown fox".capitalize() == "Brown fox" diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -279,6 +279,27 @@ previous_is_cased = False return space.newbool(cased) +def unicode_isidentifier__Unicode(space, w_unicode): + v = w_unicode._value + if len(v) == 0: + return space.w_False + + # PEP 3131 says that the first character must be in XID_Start and + # subsequent characters in XID_Continue, and for the ASCII range, + # the 2.x rules apply (i.e start with letters and underscore, + # continue with letters, digits, underscore). However, given the + # current definition of XID_Start and XID_Continue, it is + # sufficient to check just for these, except that _ must be + # allowed as starting an identifier. + first = v[0] + if not (unicodedb.isxidstart(ord(first)) or first == u'_'): + return space.w_False + + for i in range(1, len(v)): + if not unicodedb.isxidcontinue(ord(v[i])): + return space.w_False + return space.w_True + def _strip(space, w_self, w_chars, left, right): "internal function called by str_xstrip methods" u_self = w_self._value diff --git a/pypy/objspace/std/unicodetype.py b/pypy/objspace/std/unicodetype.py --- a/pypy/objspace/std/unicodetype.py +++ b/pypy/objspace/std/unicodetype.py @@ -107,6 +107,10 @@ ' characters in S are uppercase and there is\nat' ' least one cased character in S, False' ' otherwise.') +unicode_isidentifier = SMM('isidentifier', 1, + doc='S.isidentifier() -> bool\n\nReturn True if S is' + ' a valid identifier according\nto the language' + ' definition.') unicode_join = SMM('join', 2, doc='S.join(sequence) -> unicode\n\nReturn a string' ' which is the concatenation of the strings in' From noreply at buildbot.pypy.org Sat Jan 14 21:48:28 2012 From: noreply at buildbot.pypy.org (amauryfa) Date: Sat, 14 Jan 2012 21:48:28 +0100 (CET) Subject: [pypy-commit] pypy py3k: fixes in pure-Python implementation of sha1 and md5: Message-ID: <20120114204828.3926482B12@wyvern.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3k Changeset: r51319:5f39f911a130 Date: 2011-12-22 11:06 +0100 http://bitbucket.org/pypy/pypy/changeset/5f39f911a130/ Log: fixes in pure-Python implementation of sha1 and md5: they accept bytes, not strings diff --git a/lib_pypy/_md5.py b/lib_pypy/_md5.py --- a/lib_pypy/_md5.py +++ b/lib_pypy/_md5.py @@ -47,16 +47,16 @@ def _bytelist2long(list): "Transform a list of characters into a list of longs." - imax = len(list)/4 + imax = len(list) // 4 hl = [0L] * imax j = 0 i = 0 while i < imax: - b0 = long(ord(list[j])) - b1 = (long(ord(list[j+1]))) << 8 - b2 = (long(ord(list[j+2]))) << 16 - b3 = (long(ord(list[j+3]))) << 24 + b0 = list[j] + b1 = list[j+1] << 8 + b2 = list[j+2] << 16 + b3 = list[j+3] << 24 hl[i] = b0 | b1 |b2 | b3 i = i+1 j = j+4 @@ -118,7 +118,7 @@ digest_size = digestsize = 16 block_size = 64 - def __init__(self): + def __init__(self, arg=None): "Initialisation." # Initial message length in bits(!). @@ -132,6 +132,9 @@ # to start from scratch on the same object. self.init() + if arg: + self.update(arg) + def init(self): "Initialize the message-digest and set all fields to zero." @@ -316,7 +319,7 @@ else: padLen = 120 - index - padding = ['\200'] + ['\000'] * 63 + padding = [0o200] + [0] * 63 self.update(padding[:padLen]) # Append length (before padding). @@ -346,7 +349,7 @@ binary environments. """ - return ''.join(['%02x' % ord(c) for c in self.digest()]) + return ''.join(['%02x' % c for c in self.digest()]) def copy(self): """Return a clone object. diff --git a/lib_pypy/_sha1.py b/lib_pypy/_sha1.py --- a/lib_pypy/_sha1.py +++ b/lib_pypy/_sha1.py @@ -35,7 +35,7 @@ """ # After much testing, this algorithm was deemed to be the fastest. - s = '' + s = b'' pack = struct.pack while n > 0: s = pack('>I', n & 0xffffffffL) + s @@ -69,10 +69,10 @@ j = 0 i = 0 while i < imax: - b0 = long(ord(list[j])) << 24 - b1 = long(ord(list[j+1])) << 16 - b2 = long(ord(list[j+2])) << 8 - b3 = long(ord(list[j+3])) + b0 = list[j] << 24 + b1 = list[j+1] << 16 + b2 = list[j+2] << 8 + b3 = list[j+3] hl[i] = b0 | b1 | b2 | b3 i = i+1 j = j+4 @@ -280,7 +280,7 @@ else: padLen = 120 - index - padding = ['\200'] + ['\000'] * 63 + padding = [0o200] + [0] * 63 self.update(padding[:padLen]) # Append length (before padding). @@ -314,7 +314,7 @@ used to exchange the value safely in email or other non- binary environments. """ - return ''.join(['%02x' % ord(c) for c in self.digest()]) + return ''.join(['%02x' % c for c in self.digest()]) def copy(self): """Return a clone object. From noreply at buildbot.pypy.org Sat Jan 14 21:48:29 2012 From: noreply at buildbot.pypy.org (amauryfa) Date: Sat, 14 Jan 2012 21:48:29 +0100 (CET) Subject: [pypy-commit] pypy py3k: Fix tests in module/thread Message-ID: <20120114204829.7C32F82B12@wyvern.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3k Changeset: r51320:36af87a5ee11 Date: 2011-12-22 23:52 +0100 http://bitbucket.org/pypy/pypy/changeset/36af87a5ee11/ Log: Fix tests in module/thread diff --git a/pypy/module/_io/interp_stringio.py b/pypy/module/_io/interp_stringio.py --- a/pypy/module/_io/interp_stringio.py +++ b/pypy/module/_io/interp_stringio.py @@ -12,7 +12,7 @@ self.buf = [] self.pos = 0 - def descr_init(self, space, w_initvalue=None, w_newline="\n"): + def descr_init(self, space, w_initvalue=None, w_newline=u"\n"): # In case __init__ is called multiple times self.buf = [] self.pos = 0 diff --git a/pypy/module/thread/test/test_import_lock.py b/pypy/module/thread/test/test_import_lock.py --- a/pypy/module/thread/test/test_import_lock.py +++ b/pypy/module/thread/test/test_import_lock.py @@ -66,9 +66,6 @@ def test_lock(self, space, monkeypatch): from pypy.module.imp.importing import getimportlock, importhook - # Force importing the module _file now - space.builtin.get('file') - # Monkeypatch the import lock and add a counter importlock = getimportlock(space) original_acquire = importlock.acquire_lock @@ -82,9 +79,9 @@ importhook(space, 'sys') assert importlock.count == 0 # A new module - importhook(space, 're') - assert importlock.count == 7 + importhook(space, 'pprint') + assert importlock.count == 1 # Import it again previous_count = importlock.count - importhook(space, 're') + importhook(space, 'pprint') assert importlock.count == previous_count From noreply at buildbot.pypy.org Sat Jan 14 21:48:30 2012 From: noreply at buildbot.pypy.org (amauryfa) Date: Sat, 14 Jan 2012 21:48:30 +0100 (CET) Subject: [pypy-commit] pypy py3k: Add _thread.RLock Message-ID: <20120114204830.AB7EB82B12@wyvern.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3k Changeset: r51321:ae023502cd1a Date: 2011-12-26 17:19 +0100 http://bitbucket.org/pypy/pypy/changeset/ae023502cd1a/ Log: Add _thread.RLock diff --git a/pypy/module/thread/__init__.py b/pypy/module/thread/__init__.py --- a/pypy/module/thread/__init__.py +++ b/pypy/module/thread/__init__.py @@ -20,6 +20,7 @@ 'allocate_lock': 'os_lock.allocate_lock', 'allocate': 'os_lock.allocate_lock', # obsolete synonym 'LockType': 'os_lock.Lock', + 'RLock': 'os_lock.W_RLock', '_local': 'os_local.Local', 'TIMEOUT_MAX': 'space.wrap(float(os_lock.TIMEOUT_MAX) / 1000000.0)', 'error': 'space.fromcache(error.Cache).w_error', diff --git a/pypy/module/thread/os_lock.py b/pypy/module/thread/os_lock.py --- a/pypy/module/thread/os_lock.py +++ b/pypy/module/thread/os_lock.py @@ -6,9 +6,9 @@ from pypy.module.thread.error import wrap_thread_error from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.gateway import interp2app, unwrap_spec -from pypy.interpreter.typedef import TypeDef +from pypy.interpreter.typedef import TypeDef, make_weakref_descr from pypy.interpreter.error import OperationError -from pypy.rlib.rarithmetic import r_longlong +from pypy.rlib.rarithmetic import r_longlong, r_uint, ovfcheck # Force the declaration of the type 'thread.LockType' for RPython #import pypy.module.thread.rpython.exttable @@ -100,14 +100,8 @@ def __exit__(self, *args): self.descr_lock_release(self.space) -descr_acquire = interp2app(Lock.descr_lock_acquire) -descr_release = interp2app(Lock.descr_lock_release) -descr_locked = interp2app(Lock.descr_lock_locked) -descr__enter__ = interp2app(Lock.descr__enter__) -descr__exit__ = interp2app(Lock.descr__exit__) - - -Lock.typedef = TypeDef("thread.lock", +Lock.typedef = TypeDef( + "_thread.lock", __doc__ = """\ A lock object is a synchronization primitive. To create a lock, call the thread.allocate_lock() function. Methods are: @@ -119,15 +113,15 @@ A lock is not owned by the thread that locked it; another thread may unlock it. A thread attempting to lock a lock that it has already locked will block until another thread unlocks it. Deadlocks may ensue.""", - acquire = descr_acquire, - release = descr_release, - locked = descr_locked, - __enter__ = descr__enter__, - __exit__ = descr__exit__, + acquire = interp2app(Lock.descr_lock_acquire), + release = interp2app(Lock.descr_lock_release), + locked = interp2app(Lock.descr_lock_locked), + __enter__ = interp2app(Lock.descr__enter__), + __exit__ = interp2app(Lock.descr__exit__), # Obsolete synonyms - acquire_lock = descr_acquire, - release_lock = descr_release, - locked_lock = descr_locked, + acquire_lock = interp2app(Lock.descr_lock_acquire), + release_lock = interp2app(Lock.descr_lock_release), + locked_lock = interp2app(Lock.descr_lock_locked), ) @@ -135,3 +129,123 @@ """Create a new lock object. (allocate() is an obsolete synonym.) See LockType.__doc__ for information about locks.""" return space.wrap(Lock(space)) + + +class W_RLock(Wrappable): + def __init__(self, space): + self.rlock_count = 0 + self.rlock_owner = 0 + try: + self.lock = thread.allocate_lock() + except thread.error: + raise wrap_thread_error(space, "cannot allocate lock") + + def descr__new__(space, w_subtype): + self = space.allocate_instance(W_RLock, w_subtype) + W_RLock.__init__(self, space) + return space.wrap(self) + + def descr__repr__(self): + typename = space.type(self).getname(space) + return space.wrap("<%s owner=%d count=%d>" % ( + typename, self.rlock_owner, self.rlock_count)) + + @unwrap_spec(blocking=bool) + def acquire_w(self, space, blocking=True): + """Lock the lock. `blocking` indicates whether we should wait + for the lock to be available or not. If `blocking` is False + and another thread holds the lock, the method will return False + immediately. If `blocking` is True and another thread holds + the lock, the method will wait for the lock to be released, + take it and then return True. + (note: the blocking operation is not interruptible.) + + In all other cases, the method will return True immediately. + Precisely, if the current thread already holds the lock, its + internal counter is simply incremented. If nobody holds the lock, + the lock is taken and its internal counter initialized to 1.""" + tid = thread.get_ident() + if self.rlock_count > 0 and tid == self.rlock_owner: + try: + self.rlock_count = ovfcheck(self.rlock_count + 1) + except OverflowError: + raise OperationError(space.w_OverflowError, space.wrap( + 'internal lock count overflowed')) + return space.w_True + + r = True + if self.rlock_count > 0 or not self.lock.acquire(False): + if not blocking: + return space.w_False + r = self.lock.acquire(True) + if r: + assert self.rlock_count == 0 + self.rlock_owner = tid + self.rlock_count = 1 + + return space.wrap(r) + + + def release_w(self, space): + """Release the lock, allowing another thread that is blocked waiting for + the lock to acquire the lock. The lock must be in the locked state, + and must be locked by the same thread that unlocks it; otherwise a + `RuntimeError` is raised. + + Do note that if the lock was acquire()d several times in a row by the + current thread, release() needs to be called as many times for the lock + to be available for other threads.""" + tid = thread.get_ident() + if self.rlock_count == 0 or self.rlock_owner != tid: + raise OperationError(space.w_RuntimeError, space.wrap( + "cannot release un-acquired lock")) + self.rlock_count -= 1 + if self.rlock_count == 0: + self.rlock_owner == 0 + self.lock.release() + + def is_owned_w(self, space): + """For internal use by `threading.Condition`.""" + tid = thread.get_ident() + if self.rlock_count > 0 and self.rlock_owner == tid: + return space.w_True + else: + return space.w_False + + @unwrap_spec(count=r_uint, owner=int) + def acquire_restore_w(self, space, count, owner): + """For internal use by `threading.Condition`.""" + r = True + if not self.lock.acquire(False): + r = self.lock.acquire(True) + if not r: + raise wrap_thread_error(space, "coult not acquire lock") + assert self.rlock_count == 0 + self.rlock_owner = owner + self.rlock_count = count + + def release_save_w(self, space): + """For internal use by `threading.Condition`.""" + count, self.rlock_count = self.rlock_count, 0 + owner, self.rlock_owner = self.rlock_owner, 0 + return space.newtuple([space.wrap(count), space.wrap(owner)]) + + def descr__enter__(self, space): + self.acquire_w(space) + return self + + def descr__exit__(self, space, *args): + self.release_w(space) + +W_RLock.typedef = TypeDef( + "_thread.RLock", + __new__ = interp2app(W_RLock.descr__new__.im_func), + acquire = interp2app(W_RLock.acquire_w), + release = interp2app(W_RLock.release_w), + _is_owned = interp2app(W_RLock.is_owned_w), + _acquire_restore = interp2app(W_RLock.acquire_restore_w), + _release_save = interp2app(W_RLock.release_save_w), + __enter__ = interp2app(W_RLock.descr__enter__), + __exit__ = interp2app(W_RLock.descr__exit__), + __weakref__ = make_weakref_descr(W_RLock), + ) diff --git a/pypy/module/thread/test/test_lock.py b/pypy/module/thread/test/test_lock.py --- a/pypy/module/thread/test/test_lock.py +++ b/pypy/module/thread/test/test_lock.py @@ -80,3 +80,45 @@ class AppTestLockAgain(GenericTestThread): # test it at app-level again to detect strange interactions test_lock_again = AppTestLock.test_lock.im_func + + +class AppTestRLock(GenericTestThread): + """ + Tests for recursive locks. + """ + def test_reacquire(self): + import _thread + lock = _thread.RLock() + lock.acquire() + lock.acquire() + lock.release() + lock.acquire() + lock.release() + lock.release() + + def test_release_unacquired(self): + # Cannot release an unacquired lock + import _thread + lock = _thread.RLock() + raises(RuntimeError, lock.release) + lock.acquire() + lock.acquire() + lock.release() + lock.acquire() + lock.release() + lock.release() + raises(RuntimeError, lock.release) + + def test__is_owned(self): + import _thread + lock = _thread.RLock() + assert lock._is_owned() is False + lock.acquire() + assert lock._is_owned() is True + lock.acquire() + assert lock._is_owned() is True + lock.release() + assert lock._is_owned() is True + lock.release() + assert lock._is_owned() is False + From noreply at buildbot.pypy.org Sat Jan 14 21:48:31 2012 From: noreply at buildbot.pypy.org (amauryfa) Date: Sat, 14 Jan 2012 21:48:31 +0100 (CET) Subject: [pypy-commit] pypy py3k: Fix for repr of the empty set: set(), not {} which is a dict... Message-ID: <20120114204831.DCC7482B12@wyvern.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3k Changeset: r51322:a69b99cf0083 Date: 2011-12-26 17:26 +0100 http://bitbucket.org/pypy/pypy/changeset/a69b99cf0083/ Log: Fix for repr of the empty set: set(), not {} which is a dict... diff --git a/pypy/objspace/std/setobject.py b/pypy/objspace/std/setobject.py --- a/pypy/objspace/std/setobject.py +++ b/pypy/objspace/std/setobject.py @@ -640,6 +640,8 @@ if set_id in currently_in_repr: return '%s(...)' % (s.__class__.__name__,) currently_in_repr[set_id] = 1 + if not s: + return '%s()' % (s.__class__.__name__,) try: listrepr = repr([x for x in s]) if type(s) is set: diff --git a/pypy/objspace/std/test/test_setobject.py b/pypy/objspace/std/test/test_setobject.py --- a/pypy/objspace/std/test/test_setobject.py +++ b/pypy/objspace/std/test/test_setobject.py @@ -52,7 +52,7 @@ def test_space_newset(self): s = self.space.newset() - assert self.space.str_w(self.space.repr(s)) == '{}' + assert self.space.str_w(self.space.repr(s)) == 'set()' class AppTestAppSetTest: def test_subtype(self): From noreply at buildbot.pypy.org Sat Jan 14 21:48:33 2012 From: noreply at buildbot.pypy.org (amauryfa) Date: Sat, 14 Jan 2012 21:48:33 +0100 (CET) Subject: [pypy-commit] pypy py3k: CPython issue #1721812: Binary operations and copy operations on Message-ID: <20120114204833.1C54582B12@wyvern.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3k Changeset: r51323:3924c71a5c0a Date: 2011-12-26 17:49 +0100 http://bitbucket.org/pypy/pypy/changeset/3924c71a5c0a/ Log: CPython issue #1721812: Binary operations and copy operations on set/frozenset subclasses need to return the base type, not the subclass itself. diff --git a/pypy/objspace/std/setobject.py b/pypy/objspace/std/setobject.py --- a/pypy/objspace/std/setobject.py +++ b/pypy/objspace/std/setobject.py @@ -44,12 +44,7 @@ def _newobj(w_self, space, rdict_w): """Make a new set by taking ownership of 'rdict_w'.""" - if type(w_self) is W_SetObject: - return W_SetObject(space, rdict_w) - w_type = space.type(w_self) - w_obj = space.allocate_instance(W_SetObject, w_type) - W_SetObject.__init__(w_obj, space, rdict_w) - return w_obj + return W_SetObject(space, rdict_w) class W_FrozensetObject(W_BaseSetObject): from pypy.objspace.std.frozensettype import frozenset_typedef as typedef @@ -57,12 +52,7 @@ def _newobj(w_self, space, rdict_w): """Make a new frozenset by taking ownership of 'rdict_w'.""" - if type(w_self) is W_FrozensetObject: - return W_FrozensetObject(space, rdict_w) - w_type = space.type(w_self) - w_obj = space.allocate_instance(W_FrozensetObject, w_type) - W_FrozensetObject.__init__(w_obj, space, rdict_w) - return w_obj + return W_FrozensetObject(space, rdict_w) registerimplementation(W_BaseSetObject) registerimplementation(W_SetObject) diff --git a/pypy/objspace/std/test/test_setobject.py b/pypy/objspace/std/test/test_setobject.py --- a/pypy/objspace/std/test/test_setobject.py +++ b/pypy/objspace/std/test/test_setobject.py @@ -59,7 +59,7 @@ class subset(set):pass a = subset() b = a | set('abc') - assert type(b) is subset + assert type(b) is set def test_init_new_behavior(self): s = set.__new__(set, 'abc') @@ -77,14 +77,14 @@ b = subset('abc') subset.__new__ = lambda *args: foobar # not called b = b.copy() - assert type(b) is subset + assert type(b) is set assert set(b) == set('abc') # class frozensubset(frozenset): pass b = frozensubset('abc') frozensubset.__new__ = lambda *args: foobar # not called b = b.copy() - assert type(b) is frozensubset + assert type(b) is frozenset assert frozenset(b) == frozenset('abc') def test_union(self): @@ -160,7 +160,7 @@ s2 = s1.copy() assert s1 is not s2 assert s1 == s2 - assert type(s2) is myfrozen + assert type(s2) is frozenset def test_update(self): s1 = set('abc') @@ -317,8 +317,7 @@ s = subset([2]) assert s.x == ([2],) t = s | base([5]) - # obscure CPython behavior: - assert type(t) is subset + assert type(t) is base assert not hasattr(t, 'x') def test_isdisjoint(self): From noreply at buildbot.pypy.org Sat Jan 14 21:48:34 2012 From: noreply at buildbot.pypy.org (amauryfa) Date: Sat, 14 Jan 2012 21:48:34 +0100 (CET) Subject: [pypy-commit] pypy py3k: Fix checkmodule.py, and run it on the thread module. Message-ID: <20120114204834.50DDB82B12@wyvern.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3k Changeset: r51324:65597104c440 Date: 2011-12-26 18:39 +0100 http://bitbucket.org/pypy/pypy/changeset/65597104c440/ Log: Fix checkmodule.py, and run it on the thread module. diff --git a/pypy/module/thread/os_lock.py b/pypy/module/thread/os_lock.py --- a/pypy/module/thread/os_lock.py +++ b/pypy/module/thread/os_lock.py @@ -8,7 +8,7 @@ from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.interpreter.typedef import TypeDef, make_weakref_descr from pypy.interpreter.error import OperationError -from pypy.rlib.rarithmetic import r_longlong, r_uint, ovfcheck +from pypy.rlib.rarithmetic import r_longlong, ovfcheck # Force the declaration of the type 'thread.LockType' for RPython #import pypy.module.thread.rpython.exttable @@ -212,7 +212,7 @@ else: return space.w_False - @unwrap_spec(count=r_uint, owner=int) + @unwrap_spec(count=int, owner=int) def acquire_restore_w(self, space, count, owner): """For internal use by `threading.Condition`.""" r = True diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py --- a/pypy/objspace/fake/objspace.py +++ b/pypy/objspace/fake/objspace.py @@ -267,7 +267,7 @@ ObjSpace.ExceptionTable + ['int', 'str', 'float', 'long', 'tuple', 'list', 'dict', 'unicode', 'complex', 'slice', 'bool', - 'type', 'basestring']): + 'type', 'text']): setattr(FakeObjSpace, 'w_' + name, w_some_obj()) # for (name, _, arity, _) in ObjSpace.MethodTable: From noreply at buildbot.pypy.org Sat Jan 14 21:48:35 2012 From: noreply at buildbot.pypy.org (amauryfa) Date: Sat, 14 Jan 2012 21:48:35 +0100 (CET) Subject: [pypy-commit] pypy py3k: Break everything and unify int and long! Message-ID: <20120114204835.AACF182B12@wyvern.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3k Changeset: r51325:f4af6649c12f Date: 2011-12-27 17:22 +0100 http://bitbucket.org/pypy/pypy/changeset/f4af6649c12f/ Log: Break everything and unify int and long! diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py --- a/pypy/interpreter/astcompiler/astbuilder.py +++ b/pypy/interpreter/astcompiler/astbuilder.py @@ -1047,10 +1047,7 @@ w_num_str = self.space.wrap(raw) w_index = None w_base = self.space.wrap(base) - if raw[-1] in "lL": - tp = self.space.w_long - return self.space.call_function(tp, w_num_str, w_base) - elif raw[-1] in "jJ": + if raw[-1] in "jJ": tp = self.space.w_complex return self.space.call_function(tp, w_num_str) try: diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -1409,8 +1409,7 @@ # not os.close(). It's likely designed for 'select'. It's irregular # in the sense that it expects either a real int/long or an object # with a fileno(), but not an object with an __int__(). - if (not self.isinstance_w(w_fd, self.w_int) and - not self.isinstance_w(w_fd, self.w_long)): + if not self.isinstance_w(w_fd, self.w_int): try: w_fileno = self.getattr(w_fd, self.wrap("fileno")) except OperationError, e: @@ -1519,7 +1518,6 @@ ('int', 'int', 1, ['__int__']), ('index', 'index', 1, ['__index__']), ('float', 'float', 1, ['__float__']), - ('long', 'long', 1, ['__long__']), ('inplace_add', '+=', 2, ['__iadd__']), ('inplace_sub', '-=', 2, ['__isub__']), ('inplace_mul', '*=', 2, ['__imul__']), diff --git a/pypy/module/_io/interp_io.py b/pypy/module/_io/interp_io.py --- a/pypy/module/_io/interp_io.py +++ b/pypy/module/_io/interp_io.py @@ -41,8 +41,7 @@ if not (space.isinstance_w(w_file, space.w_unicode) or space.isinstance_w(w_file, space.w_str) or - space.isinstance_w(w_file, space.w_int) or - space.isinstance_w(w_file, space.w_long)): + space.isinstance_w(w_file, space.w_int)): raise operationerrfmt(space.w_TypeError, "invalid file: %s", space.str_w(space.repr(w_file)) ) diff --git a/pypy/module/_random/interp_random.py b/pypy/module/_random/interp_random.py --- a/pypy/module/_random/interp_random.py +++ b/pypy/module/_random/interp_random.py @@ -28,8 +28,6 @@ else: if space.is_true(space.isinstance(w_n, space.w_int)): w_n = space.abs(w_n) - elif space.is_true(space.isinstance(w_n, space.w_long)): - w_n = space.abs(w_n) else: # XXX not perfectly like CPython w_n = space.abs(space.hash(w_n)) @@ -76,11 +74,13 @@ self._rnd.index = space.int_w(w_item) def jumpahead(self, space, w_n): - if space.is_true(space.isinstance(w_n, space.w_long)): + try: + n = space.int_w(w_n) + except OperationError, e: + if not e.match(space, space.w_TypeError): + raise num = space.bigint_w(w_n) n = intmask(num.uintmask()) - else: - n = space.int_w(w_n) self._rnd.jumpahead(n) assert rbigint.SHIFT <= 32 diff --git a/pypy/module/_winreg/interp_winreg.py b/pypy/module/_winreg/interp_winreg.py --- a/pypy/module/_winreg/interp_winreg.py +++ b/pypy/module/_winreg/interp_winreg.py @@ -109,9 +109,14 @@ elif isinstance(w_hkey, W_HKEY): return w_hkey.hkey elif space.is_true(space.isinstance(w_hkey, space.w_int)): - return rffi.cast(rwinreg.HKEY, space.int_w(w_hkey)) - elif space.is_true(space.isinstance(w_hkey, space.w_long)): - return rffi.cast(rwinreg.HKEY, space.uint_w(w_hkey)) + try: + value = space.int_w(w_hkey) + except OperationError, e: + if not e.match(space, space.w_TypeError): + raise + return rffi.cast(rwinreg.HKEY, space.uint_w(w_hkey)) + else: + return rffi.cast(rwinreg.HKEY, space.int_w(w_hkey)) else: errstring = space.wrap("The object is not a PyHKEY object") raise OperationError(space.w_TypeError, errstring) diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -377,10 +377,9 @@ "Dict": "space.w_dict", "Tuple": "space.w_tuple", "List": "space.w_list", - "Int": "space.w_int", "Bool": "space.w_bool", "Float": "space.w_float", - "Long": "space.w_long", + "Long": "space.w_int", "Complex": "space.w_complex", "BaseObject": "space.w_object", 'None': 'space.type(space.w_None)', diff --git a/pypy/module/cpyext/longobject.py b/pypy/module/cpyext/longobject.py --- a/pypy/module/cpyext/longobject.py +++ b/pypy/module/cpyext/longobject.py @@ -126,7 +126,7 @@ @cpython_api([lltype.Float], PyObject) def PyLong_FromDouble(space, val): """Return a new PyLongObject object from v, or NULL on failure.""" - return space.long(space.wrap(val)) + return space.int(space.wrap(val)) @cpython_api([PyObject], lltype.Float, error=-1.0) def PyLong_AsDouble(space, w_long): @@ -151,7 +151,7 @@ w_base = space.wrap(rffi.cast(lltype.Signed, base)) if pend: pend[0] = rffi.ptradd(str, len(s)) - return space.call_function(space.w_long, w_str, w_base) + return space.call_function(space.w_int, w_str, w_base) @cpython_api([rffi.VOIDP], PyObject) def PyLong_FromVoidPtr(space, p): diff --git a/pypy/module/cpyext/number.py b/pypy/module/cpyext/number.py --- a/pypy/module/cpyext/number.py +++ b/pypy/module/cpyext/number.py @@ -47,7 +47,7 @@ def PyNumber_Long(space, w_obj): """Returns the o converted to a long integer object on success, or NULL on failure. This is the equivalent of the Python expression long(o).""" - return space.long(w_obj) + return space.int(w_obj) @cpython_api([PyObject], PyObject) def PyNumber_Index(space, w_obj): diff --git a/pypy/module/math/interp_math.py b/pypy/module/math/interp_math.py --- a/pypy/module/math/interp_math.py +++ b/pypy/module/math/interp_math.py @@ -101,8 +101,7 @@ """ldexp(x, i) -> x * (2**i) """ x = _get_double(space, w_x) - if (space.isinstance_w(w_i, space.w_int) or - space.isinstance_w(w_i, space.w_long)): + if space.isinstance_w(w_i, space.w_int): try: exp = space.int_w(w_i) except OperationError, e: @@ -189,7 +188,7 @@ def _log_any(space, w_x, base): # base is supposed to be positive or 0.0, which means we use e try: - if space.is_true(space.isinstance(w_x, space.w_long)): + if space.is_true(space.isinstance(w_x, space.w_int)): # special case to support log(extremely-large-long) num = space.bigint_w(w_x) result = num.log(base) diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py --- a/pypy/module/micronumpy/interp_dtype.py +++ b/pypy/module/micronumpy/interp_dtype.py @@ -187,7 +187,7 @@ name="int64", char="q", w_box_type = space.gettypefor(interp_boxes.W_Int64Box), - alternate_constructors=[space.w_long], + alternate_constructors=[space.w_int], ) self.w_uint64dtype = W_Dtype( types.UInt64(), @@ -228,4 +228,4 @@ ) def get_dtype_cache(space): - return space.fromcache(DtypeCache) \ No newline at end of file + return space.fromcache(DtypeCache) diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py --- a/pypy/module/micronumpy/interp_ufuncs.py +++ b/pypy/module/micronumpy/interp_ufuncs.py @@ -270,11 +270,6 @@ return current_guess elif space.isinstance_w(w_obj, space.w_int): if (current_guess is None or current_guess is bool_dtype or - current_guess is long_dtype): - return long_dtype - return current_guess - elif space.isinstance_w(w_obj, space.w_long): - if (current_guess is None or current_guess is bool_dtype or current_guess is long_dtype or current_guess is int64_dtype): return int64_dtype return current_guess diff --git a/pypy/module/operator/interp_operator.py b/pypy/module/operator/interp_operator.py --- a/pypy/module/operator/interp_operator.py +++ b/pypy/module/operator/interp_operator.py @@ -226,8 +226,7 @@ raise OperationError(space.w_TypeError, space.wrap("non-sequence object can't be repeated")) - if not (space.is_true(space.isinstance(w_obj2, space.w_int)) or \ - space.is_true(space.isinstance(w_obj2, space.w_long))): + if not space.is_true(space.isinstance(w_obj2, space.w_int)): # second arg has to be int/long raise OperationError(space.w_TypeError, space.wrap('an integer is required')) diff --git a/pypy/module/struct/formatiterator.py b/pypy/module/struct/formatiterator.py --- a/pypy/module/struct/formatiterator.py +++ b/pypy/module/struct/formatiterator.py @@ -64,8 +64,7 @@ def _accept_integral(self, meth): space = self.space w_obj = self.accept_obj_arg() - if (space.isinstance_w(w_obj, space.w_int) or - space.isinstance_w(w_obj, space.w_long)): + if space.isinstance_w(w_obj, space.w_int): w_index = w_obj else: w_index = None diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py --- a/pypy/objspace/descroperation.py +++ b/pypy/objspace/descroperation.py @@ -428,18 +428,12 @@ w_resulttype = space.type(w_result) if space.is_w(w_resulttype, space.w_int): return w_result - elif space.is_w(w_resulttype, space.w_long): - return space.hash(w_result) elif space.is_true(space.isinstance(w_result, space.w_int)): # be careful about subclasses of 'int'... return space.wrap(space.int_w(w_result)) - elif space.is_true(space.isinstance(w_result, space.w_long)): - # be careful about subclasses of 'long'... - bigint = space.bigint_w(w_result) - return space.wrap(bigint.hash()) else: raise OperationError(space.w_TypeError, - space.wrap("__hash__() should return an int or long")) + space.wrap("__hash__() should return an int")) def userdel(space, w_obj): w_del = space.lookup(w_obj, '__del__') @@ -724,9 +718,8 @@ # more of the above manually-coded operations as well) for targetname, specialname, checkerspec in [ - ('int', '__int__', ("space.w_int", "space.w_long")), - ('index', '__index__', ("space.w_int", "space.w_long")), - ('long', '__long__', ("space.w_int", "space.w_long")), + ('int', '__int__', ("space.w_int",)), + ('index', '__index__', ("space.w_int",)), ('float', '__float__', ("space.w_float",))]: l = ["space.is_true(space.isinstance(w_result, %s))" % x diff --git a/pypy/objspace/std/booltype.py b/pypy/objspace/std/booltype.py --- a/pypy/objspace/std/booltype.py +++ b/pypy/objspace/std/booltype.py @@ -1,6 +1,6 @@ from pypy.interpreter import gateway from pypy.objspace.std.stdtypedef import StdTypeDef -from pypy.objspace.std.inttype import int_typedef +from pypy.objspace.std.longtype import long_typedef def descr__new__(space, w_booltype, w_obj=None): space.w_bool.check_user_subclass(w_booltype) @@ -11,7 +11,7 @@ # ____________________________________________________________ -bool_typedef = StdTypeDef("bool", int_typedef, +bool_typedef = StdTypeDef("bool", long_typedef, __doc__ = '''bool(x) -> bool Returns True when the argument x is true, False otherwise. diff --git a/pypy/objspace/std/floatobject.py b/pypy/objspace/std/floatobject.py --- a/pypy/objspace/std/floatobject.py +++ b/pypy/objspace/std/floatobject.py @@ -93,15 +93,13 @@ try: value = ovfcheck_float_to_int(w_value.floatval) except OverflowError: - return space.long(w_value) + pass else: return space.newint(value) - -def long__Float(space, w_floatobj): try: - return W_LongObject.fromfloat(space, w_floatobj.floatval) + return W_LongObject.fromfloat(space, w_value.floatval) except OverflowError: - if isnan(w_floatobj.floatval): + if isnan(w_value.floatval): raise OperationError( space.w_ValueError, space.wrap("cannot convert float NaN to integer")) @@ -112,7 +110,7 @@ try: value = ovfcheck_float_to_int(whole) except OverflowError: - return long__Float(space, w_floatobj) + return int__Float(space, w_floatobj) else: return space.newint(value) diff --git a/pypy/objspace/std/intobject.py b/pypy/objspace/std/intobject.py --- a/pypy/objspace/std/intobject.py +++ b/pypy/objspace/std/intobject.py @@ -24,7 +24,7 @@ return False if self.user_overridden_class or w_other.user_overridden_class: return self is w_other - return space.int_w(self) == space.int_w(w_other) + return space.bigint_w(self).eq(space.bigint_w(w_other)) def immutable_unique_id(self, space): if self.user_overridden_class: @@ -39,7 +39,7 @@ __slots__ = 'intval' _immutable_fields_ = ['intval'] - from pypy.objspace.std.inttype import int_typedef as typedef + from pypy.objspace.std.longtype import long_typedef as typedef def __init__(w_self, intval): w_self.intval = intval diff --git a/pypy/objspace/std/inttype.py b/pypy/objspace/std/inttype.py --- a/pypy/objspace/std/inttype.py +++ b/pypy/objspace/std/inttype.py @@ -36,7 +36,7 @@ @gateway.unwrap_spec(s='bufferstr', byteorder=str) def descr_from_bytes(space, w_cls, s, byteorder): from pypy.objspace.std.longtype import descr_from_bytes - return descr_from_bytes(space, space.w_long, s, byteorder) + return descr_from_bytes(space, space.w_int, s, byteorder) def wrapint(space, x): if space.config.objspace.std.withsmallint: diff --git a/pypy/objspace/std/longobject.py b/pypy/objspace/std/longobject.py --- a/pypy/objspace/std/longobject.py +++ b/pypy/objspace/std/longobject.py @@ -4,30 +4,12 @@ from pypy.objspace.std.model import registerimplementation, W_Object from pypy.objspace.std.register_all import register_all from pypy.objspace.std.multimethod import FailedToImplementArgs -from pypy.objspace.std.intobject import W_IntObject +from pypy.objspace.std.intobject import W_IntObject, W_AbstractIntObject from pypy.objspace.std.noneobject import W_NoneObject from pypy.rlib.rbigint import rbigint, SHIFT -class W_AbstractLongObject(W_Object): - __slots__ = () - def is_w(self, space, w_other): - if not isinstance(w_other, W_AbstractLongObject): - return False - if self.user_overridden_class or w_other.user_overridden_class: - return self is w_other - return space.bigint_w(self).eq(space.bigint_w(w_other)) - - def immutable_unique_id(self, space): - if self.user_overridden_class: - return None - from pypy.objspace.std.model import IDTAG_LONG as tag - b = space.bigint_w(self) - b = b.lshift(3).or_(rbigint.fromint(tag)) - return space.newlong_from_rbigint(b) - - -class W_LongObject(W_AbstractLongObject): +class W_LongObject(W_AbstractIntObject): """This is a wrapper of rbigint.""" from pypy.objspace.std.longtype import long_typedef as typedef _immutable_fields_ = ['num'] @@ -114,27 +96,18 @@ return W_LongObject.fromint(space, w_intobj.intval) -# long__Long is supposed to do nothing, unless it has +# int__Long is supposed to do nothing, unless it has # a derived long object, where it should return # an exact one. -def long__Long(space, w_long1): - if space.is_w(space.type(w_long1), space.w_long): +def int__Long(space, w_long1): + if space.is_w(space.type(w_long1), space.w_int): return w_long1 l = w_long1.num return W_LongObject(l) -trunc__Long = long__Long - -def long__Int(space, w_intobj): - return space.newlong(w_intobj.intval) - -def int__Long(space, w_value): - try: - return space.newint(w_value.num.toint()) - except OverflowError: - return long__Long(space, w_value) +trunc__Long = int__Long def index__Long(space, w_value): - return long__Long(space, w_value) + return int__Long(space, w_value) def float__Long(space, w_longobj): try: @@ -275,7 +248,7 @@ return W_LongObject(w_long1.num.neg()) def pos__Long(space, w_long): - return long__Long(space, w_long) + return int__Long(space, w_long) def abs__Long(space, w_long): return W_LongObject(w_long.num.abs()) diff --git a/pypy/objspace/std/longtype.py b/pypy/objspace/std/longtype.py --- a/pypy/objspace/std/longtype.py +++ b/pypy/objspace/std/longtype.py @@ -5,7 +5,7 @@ from pypy.objspace.std.strutil import string_to_bigint, ParseStringError def descr_conjugate(space, w_int): - return space.long(w_int) + return space.int(w_int) def descr__new__(space, w_longtype, w_x=0, w_base=gateway.NoneNotWrapped): @@ -20,7 +20,7 @@ if w_base is None: # check for easy cases if (W_SmallLongObject and type(w_value) is W_SmallLongObject - and space.is_w(w_longtype, space.w_long)): + and space.is_w(w_longtype, space.w_int)): return w_value elif type(w_value) is W_LongObject: return newbigint(space, w_longtype, w_value.num) @@ -34,20 +34,13 @@ return string_to_w_long(space, w_longtype, unicode_to_decimal_w(space, w_value)) else: - # otherwise, use the __long__() or the __trunc__ methods + # otherwise, use the __int__() or the __trunc__ methods w_obj = w_value - if (space.lookup(w_obj, '__long__') is not None or - space.lookup(w_obj, '__int__') is not None): - w_obj = space.long(w_obj) + if space.lookup(w_obj, '__int__') is not None: + w_obj = space.int(w_obj) else: w_obj = space.trunc(w_obj) - # :-( blame CPython 2.7 - if space.lookup(w_obj, '__long__') is not None: - w_obj = space.long(w_obj) - else: - w_obj = space.int(w_obj) - bigint = space.bigint_w(w_obj) - return newbigint(space, w_longtype, bigint) + return w_obj else: base = space.int_w(w_base) @@ -80,7 +73,7 @@ longobject.py, but takes an explicit w_longtype argument. """ if (space.config.objspace.std.withsmalllong - and space.is_w(w_longtype, space.w_long)): + and space.is_w(w_longtype, space.w_int)): try: z = bigint.tolonglong() except OverflowError: @@ -94,13 +87,13 @@ return w_obj def descr_get_numerator(space, w_obj): - return space.long(w_obj) + return space.int(w_obj) def descr_get_denominator(space, w_obj): return space.newlong(1) def descr_get_real(space, w_obj): - return space.long(w_obj) + return space.int(w_obj) def descr_get_imag(space, w_obj): return space.newlong(0) @@ -124,8 +117,8 @@ # ____________________________________________________________ -long_typedef = StdTypeDef("long", - __doc__ = '''long(x[, base]) -> integer +long_typedef = StdTypeDef("int", + __doc__ = '''int(x[, base]) -> integer Convert a string or number to a long integer, if possible. A floating point argument will be truncated towards zero (this does not include a diff --git a/pypy/objspace/std/model.py b/pypy/objspace/std/model.py --- a/pypy/objspace/std/model.py +++ b/pypy/objspace/std/model.py @@ -31,7 +31,7 @@ } IDTAG_INT = 1 -IDTAG_LONG = 3 +#IDTAG_LONG = 3 IDTAG_FLOAT = 5 IDTAG_COMPLEX = 7 @@ -44,7 +44,7 @@ class result: from pypy.objspace.std.objecttype import object_typedef from pypy.objspace.std.booltype import bool_typedef - from pypy.objspace.std.inttype import int_typedef + #from pypy.objspace.std.inttype import int_typedef from pypy.objspace.std.floattype import float_typedef from pypy.objspace.std.complextype import complex_typedef from pypy.objspace.std.settype import set_typedef @@ -184,8 +184,8 @@ (complexobject.W_ComplexObject, complexobject.delegate_Bool2Complex), ] self.typeorder[intobject.W_IntObject] += [ + (longobject.W_LongObject, longobject.delegate_Int2Long), (floatobject.W_FloatObject, floatobject.delegate_Int2Float), - (longobject.W_LongObject, longobject.delegate_Int2Long), (complexobject.W_ComplexObject, complexobject.delegate_Int2Complex), ] if config.objspace.std.withsmalllong: diff --git a/pypy/objspace/std/smallintobject.py b/pypy/objspace/std/smallintobject.py --- a/pypy/objspace/std/smallintobject.py +++ b/pypy/objspace/std/smallintobject.py @@ -16,7 +16,7 @@ class W_SmallIntObject(W_AbstractIntObject, UnboxedValue): __slots__ = 'intval' - from pypy.objspace.std.inttype import int_typedef as typedef + from pypy.objspace.std.longtype import long_typedef as typedef def unwrap(w_self, space): return int(w_self.intval) diff --git a/pypy/objspace/std/smalllongobject.py b/pypy/objspace/std/smalllongobject.py --- a/pypy/objspace/std/smalllongobject.py +++ b/pypy/objspace/std/smalllongobject.py @@ -9,7 +9,7 @@ from pypy.rlib.rarithmetic import r_longlong, r_int, r_uint from pypy.rlib.rarithmetic import intmask, LONGLONG_BIT from pypy.rlib.rbigint import rbigint -from pypy.objspace.std.longobject import W_AbstractLongObject, W_LongObject +from pypy.objspace.std.longobject import W_AbstractIntObject, W_LongObject from pypy.objspace.std.intobject import W_IntObject from pypy.objspace.std.noneobject import W_NoneObject from pypy.interpreter.error import OperationError @@ -17,7 +17,7 @@ LONGLONG_MIN = r_longlong((-1) << (LONGLONG_BIT-1)) -class W_SmallLongObject(W_AbstractLongObject): +class W_SmallLongObject(W_AbstractIntObject): from pypy.objspace.std.longtype import long_typedef as typedef _immutable_fields_ = ['longlong'] @@ -109,9 +109,6 @@ return space.newcomplex(float(w_small.longlong), 0.0) -def long__SmallLong(space, w_value): - return w_value - def int__SmallLong(space, w_value): a = w_value.longlong b = intmask(a) diff --git a/pypy/objspace/std/test/test_intobject.py b/pypy/objspace/std/test/test_intobject.py --- a/pypy/objspace/std/test/test_intobject.py +++ b/pypy/objspace/std/test/test_intobject.py @@ -314,10 +314,7 @@ def test_int_string(self): assert 42 == int("42") - assert 10000000000 == long("10000000000") - - def test_int_unicode(self): - assert 42 == int(unicode('42')) + assert 10000000000 == int("10000000000") def test_int_float(self): assert 4 == int(4.2) @@ -348,7 +345,7 @@ def test_overflow(self): import sys n = sys.maxint + 1 - assert isinstance(n, long) + assert isinstance(n, int) def test_pow(self): assert pow(2, -10) == 1/1024. @@ -369,8 +366,6 @@ assert j("100") == 100 assert j("100",2) == 4 assert isinstance(j("100",2),j) - raises(OverflowError,j,sys.maxint+1) - raises(OverflowError,j,str(sys.maxint+1)) def test_int_subclass_ops(self): import sys @@ -402,7 +397,7 @@ assert (5 + j(100), type(5 + j(100))) == ( 105, int) assert (5 - j(100), type(5 - j(100))) == ( -95, int) assert (5 * j(100), type(5 * j(100))) == ( 500, int) - assert (5 << j(100), type(5 << j(100))) == (5 << 100, long) + assert (5 << j(100), type(5 << j(100))) == (5 << 100, int) assert (j(100) >> 2, type(j(100) >> 2)) == ( 25, int) def test_int_subclass_int(self): @@ -413,17 +408,15 @@ return '' class subint(int): pass - class sublong(long): - pass value = 42L assert int(j()) == 42 value = 4200000000000000000000000000000000L assert int(j()) == 4200000000000000000000000000000000L value = subint(42) assert int(j()) == 42 and type(int(j())) is subint - value = sublong(4200000000000000000000000000000000L) + value = subint(4200000000000000000000000000000000L) assert (int(j()) == 4200000000000000000000000000000000L - and type(int(j())) is sublong) + and type(int(j())) is subint) value = 42.0 raises(TypeError, int, j()) value = "foo" @@ -444,16 +437,16 @@ def test_special_long(self): class a(object): - def __long__(self): + def __int__(self): self.ar = True return None inst = a() - raises(TypeError, long, inst) + raises(TypeError, int, inst) assert inst.ar == True class b(object): pass - raises((AttributeError,TypeError), long, b()) + raises((AttributeError,TypeError), int, b()) def test_just_trunc(self): class myint(object): diff --git a/pypy/objspace/std/test/test_longobject.py b/pypy/objspace/std/test/test_longobject.py --- a/pypy/objspace/std/test/test_longobject.py +++ b/pypy/objspace/std/test/test_longobject.py @@ -63,7 +63,7 @@ assert x * 2 ** 40 == x << 40 def test_truediv(self): - exec "from __future__ import division; a = 31415926L / 10000000L" + a = 31415926L / 10000000L assert a == 3.1415926 def test_floordiv(self): @@ -155,22 +155,22 @@ assert not (int(BIG)-1 >= BIG) def test_conversion(self): - class long2(long): + class long2(int): pass x = 1L x = long2(x<<100) y = int(x) - assert type(y) == long - assert type(+long2(5)) is long - assert type(long2(5) << 0) is long - assert type(long2(5) >> 0) is long - assert type(long2(5) + 0) is long - assert type(long2(5) - 0) is long - assert type(long2(5) * 1) is long - assert type(1 * long2(5)) is long - assert type(0 + long2(5)) is long - assert type(-long2(0)) is long - assert type(long2(5) // 1) is long + assert type(y) == int + assert type(+long2(5)) is int + assert type(long2(5) << 0) is int + assert type(long2(5) >> 0) is int + assert type(long2(5) + 0) is int + assert type(long2(5) - 0) is int + assert type(long2(5) * 1) is int + assert type(1 * long2(5)) is int + assert type(0 + long2(5)) is int + assert type(-long2(0)) is int + assert type(long2(5) // 1) is int def test_pow(self): x = 0L @@ -194,7 +194,7 @@ assert y < r <= 0 for x in [-1L, 0L, 1L, 2L ** 100 - 1, -2L ** 100 - 1]: for y in [-105566530L, -1L, 1L, 1034522340L]: - print "checking division for %s, %s" % (x, y) + print("checking division for %s, %s" % (x, y)) check_division(x, y) # special case from python tests: s1 = 33 @@ -209,7 +209,7 @@ raises(ZeroDivisionError, "x // 0L") def test_format(self): - assert repr(12345678901234567890) == '12345678901234567890L' + assert repr(12345678901234567890) == '12345678901234567890' assert str(12345678901234567890) == '12345678901234567890' assert hex(0x1234567890ABCDEFL) == '0x1234567890abcdefL' assert oct(01234567012345670L) == '01234567012345670L' @@ -227,7 +227,7 @@ def test_hash(self): # ints have the same hash as equal longs for i in range(-4, 14): - assert hash(i) == hash(long(i)) + assert hash(i) == hash(int(i)) # might check too much -- it's ok to change the hashing algorithm assert hash(123456789L) == 123456789 assert hash(1234567890123456789L) in ( @@ -247,8 +247,8 @@ def test_long(self): import sys n = -sys.maxint-1 - assert long(n) == n - assert str(long(n)) == str(n) + assert int(n) == n + assert str(int(n)) == str(n) def test_huge_longs(self): import operator @@ -262,45 +262,41 @@ class myint(object): def __trunc__(self): return 42 - assert long(myint()) == 42 + assert int(myint()) == 42 - def test_override___long__(self): - class mylong(long): - def __long__(self): + def test_override___int__(self): + class myint(int): + def __int__(self): return 42L - assert long(mylong(21)) == 42L - class myotherlong(long): + assert int(myint(21)) == 42L + class myotherint(int): pass - assert long(myotherlong(21)) == 21L + assert int(myotherint(21)) == 21L - def test___long__(self): + def test___int__(self): class A(object): - def __long__(self): - return 42 - assert long(A()) == 42L - class B(object): def __int__(self): return 42 - raises(TypeError, long, B()) + assert int(A()) == 42L # but!: (blame CPython 2.7) class Integral(object): def __int__(self): return 42 - class TruncReturnsNonLong(object): + class TruncReturnsNonInt(object): def __trunc__(self): return Integral() - assert long(TruncReturnsNonLong()) == 42 + assert int(TruncReturnsNonInt()) == 42 def test_conjugate(self): assert (7L).conjugate() == 7L assert (-7L).conjugate() == -7L - class L(long): + class L(int): pass - assert type(L(7).conjugate()) is long + assert type(L(7).conjugate()) is int - class L(long): + class L(int): def __pos__(self): return 43 assert L(7).conjugate() == 7L @@ -311,27 +307,14 @@ assert ((2**31)-1).bit_length() == 31 def test_from_bytes(self): - assert long.from_bytes(b'c', 'little') == 99 - assert long.from_bytes(b'\x01\x01', 'little') == 257 + assert int.from_bytes(b'c', 'little') == 99 + assert int.from_bytes(b'\x01\x01', 'little') == 257 def test_negative_zero(self): x = eval("-0L") assert x == 0L - def test_mix_int_and_long(self): - class IntLongMixClass(object): - def __int__(self): - return 42L - - def __long__(self): - return 64 - - mixIntAndLong = IntLongMixClass() - as_long = long(mixIntAndLong) - assert type(as_long) is long - assert as_long == 64 - def test_long_real(self): - class A(long): pass + class A(int): pass b = A(5).real - assert type(b) is long + assert type(b) is int diff --git a/pypy/objspace/std/test/test_newformat.py b/pypy/objspace/std/test/test_newformat.py --- a/pypy/objspace/std/test/test_newformat.py +++ b/pypy/objspace/std/test/test_newformat.py @@ -269,12 +269,6 @@ cls.w_i = cls.space.w_int -class AppTestLongFormatting(BaseIntegralFormattingTest): - - def setup_class(cls): - cls.w_i = cls.space.w_long - - class AppTestFloatFormatting: def setup_class(cls): cls.space = gettestobjspace(usemodules=('_locale',)) diff --git a/pypy/rlib/rbigint.py b/pypy/rlib/rbigint.py --- a/pypy/rlib/rbigint.py +++ b/pypy/rlib/rbigint.py @@ -293,7 +293,7 @@ return _format(self, digits, prefix, suffix) def repr(self): - return _format(self, BASE10, '', 'L') + return _format(self, BASE10) def str(self): return _format(self, BASE10) diff --git a/pypy/translator/geninterplevel.py b/pypy/translator/geninterplevel.py --- a/pypy/translator/geninterplevel.py +++ b/pypy/translator/geninterplevel.py @@ -844,7 +844,7 @@ typename_mapping = { object: 'space.w_object', int: 'space.w_int', - long: 'space.w_long', + long: 'space.w_int', bool: 'space.w_bool', list: 'space.w_list', tuple: 'space.w_tuple', From noreply at buildbot.pypy.org Sat Jan 14 21:48:36 2012 From: noreply at buildbot.pypy.org (amauryfa) Date: Sat, 14 Jan 2012 21:48:36 +0100 (CET) Subject: [pypy-commit] pypy py3k: Remove 'L' suffix from the output of hex() and oct() Message-ID: <20120114204836.DEB9A82B12@wyvern.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3k Changeset: r51326:83fc38544bca Date: 2011-12-26 23:48 +0100 http://bitbucket.org/pypy/pypy/changeset/83fc38544bca/ Log: Remove 'L' suffix from the output of hex() and oct() diff --git a/pypy/objspace/std/test/test_longobject.py b/pypy/objspace/std/test/test_longobject.py --- a/pypy/objspace/std/test/test_longobject.py +++ b/pypy/objspace/std/test/test_longobject.py @@ -211,8 +211,8 @@ def test_format(self): assert repr(12345678901234567890) == '12345678901234567890' assert str(12345678901234567890) == '12345678901234567890' - assert hex(0x1234567890ABCDEFL) == '0x1234567890abcdefL' - assert oct(01234567012345670L) == '01234567012345670L' + assert hex(0x1234567890ABCDEF) == '0x1234567890abcdef' + assert oct(01234567012345670) == '01234567012345670' def test_bits(self): x = 0xAAAAAAAAL diff --git a/pypy/rlib/rbigint.py b/pypy/rlib/rbigint.py --- a/pypy/rlib/rbigint.py +++ b/pypy/rlib/rbigint.py @@ -287,10 +287,10 @@ def tofloat(self): return _AsDouble(self) - def format(self, digits, prefix='', suffix=''): + def format(self, digits, prefix=''): # 'digits' is a string whose length is the base to use, # and where each character is the corresponding digit. - return _format(self, digits, prefix, suffix) + return _format(self, digits, prefix) def repr(self): return _format(self, BASE10) @@ -603,10 +603,10 @@ if self.sign == 0: return '0L' else: - return _format(self, BASE8, '0', 'L') + return _format(self, BASE8, '0') def hex(self): - return _format(self, BASE16, '0x', 'L') + return _format(self, BASE16, '0x') def log(self, base): # base is supposed to be positive or 0.0, which means we use e @@ -1586,7 +1586,7 @@ BASE10 = '0123456789' BASE16 = '0123456789abcdef' -def _format(a, digits, prefix='', suffix=''): +def _format(a, digits, prefix=''): """ Convert a bigint object to a string, using a given conversion base. Return a string object. @@ -1602,14 +1602,9 @@ while i > 1: bits += 1 i >>= 1 - i = 5 + len(prefix) + len(suffix) + (size_a*SHIFT + bits-1) // bits + i = 5 + len(prefix) + (size_a*SHIFT + bits-1) // bits s = [chr(0)] * i p = i - j = len(suffix) - while j > 0: - p -= 1 - j -= 1 - s[p] = suffix[j] if a.sign == 0: p -= 1 From noreply at buildbot.pypy.org Sat Jan 14 21:48:38 2012 From: noreply at buildbot.pypy.org (amauryfa) Date: Sat, 14 Jan 2012 21:48:38 +0100 (CET) Subject: [pypy-commit] pypy py3k: don't parse '0L' anymore Message-ID: <20120114204838.1DC7882B12@wyvern.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3k Changeset: r51327:40196a1cc504 Date: 2011-12-27 00:03 +0100 http://bitbucket.org/pypy/pypy/changeset/40196a1cc504/ Log: don't parse '0L' anymore diff --git a/pypy/interpreter/pyparser/genpytokenize.py b/pypy/interpreter/pyparser/genpytokenize.py --- a/pypy/interpreter/pyparser/genpytokenize.py +++ b/pypy/interpreter/pyparser/genpytokenize.py @@ -60,25 +60,21 @@ newArcPair(states, "0"), groupStr(states, "xX"), atleastonce(states, - groupStr(states, "0123456789abcdefABCDEF")), - maybe(states, groupStr(states, "lL"))) + groupStr(states, "0123456789abcdefABCDEF"))) octNumber = chain(states, newArcPair(states, "0"), maybe(states, chain(states, groupStr(states, "oO"), groupStr(states, "01234567"))), - any(states, groupStr(states, "01234567")), - maybe(states, groupStr(states, "lL"))) + any(states, groupStr(states, "01234567"))) binNumber = chain(states, newArcPair(states, "0"), groupStr(states, "bB"), - atleastonce(states, groupStr(states, "01")), - maybe(states, groupStr(states, "lL"))) + atleastonce(states, groupStr(states, "01"))) decNumber = chain(states, groupStr(states, "123456789"), - any(states, makeDigits()), - maybe(states, groupStr(states, "lL"))) + any(states, makeDigits())) intNumber = group(states, hexNumber, octNumber, binNumber, decNumber) # ____________________________________________________________ # Exponents diff --git a/pypy/interpreter/pyparser/pytokenize.py b/pypy/interpreter/pyparser/pytokenize.py --- a/pypy/interpreter/pyparser/pytokenize.py +++ b/pypy/interpreter/pyparser/pytokenize.py @@ -113,15 +113,14 @@ {'.': 24, '0': 21, '1': 21, '2': 21, '3': 21, '4': 21, '5': 21, '6': 21, '7': 21, '8': 23, '9': 23, 'B': 22, - 'E': 25, 'J': 13, 'L': 13, 'O': 20, - 'X': 19, 'b': 22, 'e': 25, 'j': 13, - 'l': 13, 'o': 20, 'x': 19}, + 'E': 25, 'J': 13, 'O': 20, 'X': 19, + 'b': 22, 'e': 25, 'j': 13, 'o': 20, + 'x': 19}, # 5 {'.': 24, '0': 5, '1': 5, '2': 5, '3': 5, '4': 5, '5': 5, '6': 5, '7': 5, '8': 5, '9': 5, 'E': 25, - 'J': 13, 'L': 13, 'e': 25, 'j': 13, - 'l': 13}, + 'J': 13, 'e': 25, 'j': 13}, # 6 {'0': 26, '1': 26, '2': 26, '3': 26, '4': 26, '5': 26, '6': 26, '7': 26, @@ -166,8 +165,7 @@ {'.': 24, '0': 21, '1': 21, '2': 21, '3': 21, '4': 21, '5': 21, '6': 21, '7': 21, '8': 23, '9': 23, 'E': 25, - 'J': 13, 'L': 13, 'e': 25, 'j': 13, - 'l': 13}, + 'J': 13, 'e': 25, 'j': 13}, # 22 {'0': 36, '1': 36}, # 23 @@ -210,14 +208,13 @@ '4': 34, '5': 34, '6': 34, '7': 34, '8': 34, '9': 34, 'A': 34, 'B': 34, 'C': 34, 'D': 34, 'E': 34, 'F': 34, - 'L': 13, 'a': 34, 'b': 34, 'c': 34, - 'd': 34, 'e': 34, 'f': 34, 'l': 13}, + 'a': 34, 'b': 34, 'c': 34, 'd': 34, + 'e': 34, 'f': 34}, # 35 {'0': 35, '1': 35, '2': 35, '3': 35, - '4': 35, '5': 35, '6': 35, '7': 35, - 'L': 13, 'l': 13}, + '4': 35, '5': 35, '6': 35, '7': 35}, # 36 - {'0': 36, '1': 36, 'L': 13, 'l': 13}, + {'0': 36, '1': 36}, # 37 {'+': 42, '-': 42, '0': 43, '1': 43, '2': 43, '3': 43, '4': 43, '5': 43, diff --git a/pypy/objspace/std/test/test_intobject.py b/pypy/objspace/std/test/test_intobject.py --- a/pypy/objspace/std/test/test_intobject.py +++ b/pypy/objspace/std/test/test_intobject.py @@ -351,8 +351,8 @@ assert pow(2, -10) == 1/1024. def test_int_w_long_arg(self): - assert int(10000000000) == 10000000000L - assert int("10000000000") == 10000000000l + assert int(10000000000) == 10000000000 + assert int("10000000000") == 10000000000 raises(ValueError, int, "10000000000JUNK") raises(ValueError, int, "10000000000JUNK", 10) @@ -362,7 +362,7 @@ pass assert j(100) == 100 assert isinstance(j(100),j) - assert j(100L) == 100 + assert j(100) == 100 assert j("100") == 100 assert j("100",2) == 4 assert isinstance(j("100",2),j) @@ -408,14 +408,14 @@ return '' class subint(int): pass - value = 42L + value = 42 assert int(j()) == 42 - value = 4200000000000000000000000000000000L - assert int(j()) == 4200000000000000000000000000000000L + value = 4200000000000000000000000000000000 + assert int(j()) == 4200000000000000000000000000000000 value = subint(42) assert int(j()) == 42 and type(int(j())) is subint - value = subint(4200000000000000000000000000000000L) - assert (int(j()) == 4200000000000000000000000000000000L + value = subint(4200000000000000000000000000000000) + assert (int(j()) == 4200000000000000000000000000000000 and type(int(j())) is subint) value = 42.0 raises(TypeError, int, j()) diff --git a/pypy/objspace/std/test/test_longobject.py b/pypy/objspace/std/test/test_longobject.py --- a/pypy/objspace/std/test/test_longobject.py +++ b/pypy/objspace/std/test/test_longobject.py @@ -43,44 +43,44 @@ def test_trunc(self): import math - assert math.trunc(1L) == 1L - assert math.trunc(-1L) == -1L + assert math.trunc(1) == 1 + assert math.trunc(-1) == -1 def test_add(self): - x = 123L - assert int(x + 12443L) == 123 + 12443 + x = 123 + assert int(x + 12443) == 123 + 12443 x = -20 - assert x + 2 + 3L + True == -14L + assert x + 2 + 3 + True == -14 def test_sub(self): - x = 58543L - assert int(x - 12332L) == 58543 - 12332 - x = 237123838281233L - assert x * 12 == x * 12L + x = 58543 + assert int(x - 12332) == 58543 - 12332 + x = 237123838281233 + assert x * 12 == x * 12 def test_mul(self): - x = 363L + x = 363 assert x * 2 ** 40 == x << 40 def test_truediv(self): - a = 31415926L / 10000000L + a = 31415926 / 10000000 assert a == 3.1415926 def test_floordiv(self): - x = 31415926L - a = x // 10000000L - assert a == 3L + x = 31415926 + a = x // 10000000 + assert a == 3 def test_numerator_denominator(self): - assert (1L).numerator == 1L - assert (1L).denominator == 1L - assert (42L).numerator == 42L - assert (42L).denominator == 1L + assert (1).numerator == 1 + assert (1).denominator == 1 + assert (42).numerator == 42 + assert (42).denominator == 1 def test_compare(self): Z = 0 - ZL = 0L - for BIG in (1L, 1L << 62, 1L << 9999): + ZL = 0 + for BIG in (1, 1 << 62, 1 << 9999): assert Z == ZL assert not (Z != ZL) assert ZL == Z @@ -157,7 +157,7 @@ def test_conversion(self): class long2(int): pass - x = 1L + x = 1 x = long2(x<<100) y = int(x) assert type(y) == int @@ -173,12 +173,12 @@ assert type(long2(5) // 1) is int def test_pow(self): - x = 0L - assert pow(x, 0L, 1L) == 0L + x = 0 + assert pow(x, 0, 1) == 0 def test_getnewargs(self): - assert 0L .__getnewargs__() == (0L,) - assert (-1L) .__getnewargs__() == (-1L,) + assert 0 .__getnewargs__() == (0,) + assert (-1) .__getnewargs__() == (-1,) def test_divmod(self): def check_division(x, y): @@ -192,8 +192,8 @@ assert 0 <= r < y else: assert y < r <= 0 - for x in [-1L, 0L, 1L, 2L ** 100 - 1, -2L ** 100 - 1]: - for y in [-105566530L, -1L, 1L, 1034522340L]: + for x in [-1, 0, 1, 2 ** 100 - 1, -2 ** 100 - 1]: + for y in [-105566530, -1, 1, 1034522340]: print("checking division for %s, %s" % (x, y)) check_division(x, y) # special case from python tests: @@ -203,10 +203,10 @@ x >>= s1*16 y = 10953035502453784575 y >>= s2*16 - x = 0x3FE0003FFFFC0001FFFL - y = 0x9800FFC1L + x = 0x3FE0003FFFFC0001FFF + y = 0x9800FFC1 check_division(x, y) - raises(ZeroDivisionError, "x // 0L") + raises(ZeroDivisionError, "x // 0") def test_format(self): assert repr(12345678901234567890) == '12345678901234567890' @@ -215,31 +215,31 @@ assert oct(01234567012345670) == '01234567012345670' def test_bits(self): - x = 0xAAAAAAAAL - assert x | 0x55555555L == 0xFFFFFFFFL - assert x & 0x55555555L == 0x00000000L - assert x ^ 0x55555555L == 0xFFFFFFFFL - assert -x | 0x55555555L == -0xAAAAAAA9L - assert x | 0x555555555L == 0x5FFFFFFFFL - assert x & 0x555555555L == 0x000000000L - assert x ^ 0x555555555L == 0x5FFFFFFFFL + x = 0xAAAAAAAA + assert x | 0x55555555 == 0xFFFFFFFF + assert x & 0x55555555 == 0x00000000 + assert x ^ 0x55555555 == 0xFFFFFFFF + assert -x | 0x55555555 == -0xAAAAAAA9 + assert x | 0x555555555 == 0x5FFFFFFFF + assert x & 0x555555555 == 0x000000000 + assert x ^ 0x555555555 == 0x5FFFFFFFF def test_hash(self): # ints have the same hash as equal longs for i in range(-4, 14): assert hash(i) == hash(int(i)) # might check too much -- it's ok to change the hashing algorithm - assert hash(123456789L) == 123456789 - assert hash(1234567890123456789L) in ( + assert hash(123456789) == 123456789 + assert hash(1234567890123456789) in ( -1895067127, # with 32-bit platforms 1234567890123456789) # with 64-bit platforms def test_math_log(self): import math - raises(ValueError, math.log, 0L) - raises(ValueError, math.log, -1L) - raises(ValueError, math.log, -2L) - raises(ValueError, math.log, -(1L << 10000)) + raises(ValueError, math.log, 0) + raises(ValueError, math.log, -1) + raises(ValueError, math.log, -2) + raises(ValueError, math.log, -(1 << 10000)) #raises(ValueError, math.log, 0) raises(ValueError, math.log, -1) raises(ValueError, math.log, -2) @@ -252,11 +252,11 @@ def test_huge_longs(self): import operator - x = 1L - huge = x << 40000L + x = 1 + huge = x << 40000 raises(OverflowError, float, huge) raises(OverflowError, operator.truediv, huge, 3) - raises(OverflowError, operator.truediv, huge, 3L) + raises(OverflowError, operator.truediv, huge, 3) def test_just_trunc(self): class myint(object): @@ -267,17 +267,17 @@ def test_override___int__(self): class myint(int): def __int__(self): - return 42L - assert int(myint(21)) == 42L + return 42 + assert int(myint(21)) == 42 class myotherint(int): pass - assert int(myotherint(21)) == 21L + assert int(myotherint(21)) == 21 def test___int__(self): class A(object): def __int__(self): return 42 - assert int(A()) == 42L + assert int(A()) == 42 # but!: (blame CPython 2.7) class Integral(object): def __int__(self): @@ -288,8 +288,8 @@ assert int(TruncReturnsNonInt()) == 42 def test_conjugate(self): - assert (7L).conjugate() == 7L - assert (-7L).conjugate() == -7L + assert (7).conjugate() == 7 + assert (-7).conjugate() == -7 class L(int): pass @@ -299,10 +299,10 @@ class L(int): def __pos__(self): return 43 - assert L(7).conjugate() == 7L + assert L(7).conjugate() == 7 def test_bit_length(self): - assert 8L.bit_length() == 4 + assert (8).bit_length() == 4 assert (-1<<40).bit_length() == 41 assert ((2**31)-1).bit_length() == 31 @@ -311,8 +311,8 @@ assert int.from_bytes(b'\x01\x01', 'little') == 257 def test_negative_zero(self): - x = eval("-0L") - assert x == 0L + x = eval("-0") + assert x == 0 def test_long_real(self): class A(int): pass From noreply at buildbot.pypy.org Sat Jan 14 21:48:39 2012 From: noreply at buildbot.pypy.org (amauryfa) Date: Sat, 14 Jan 2012 21:48:39 +0100 (CET) Subject: [pypy-commit] pypy py3k: Now reject u'' literals, Message-ID: <20120114204839.4DFA082B12@wyvern.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3k Changeset: r51328:662eb2c58644 Date: 2011-12-27 00:29 +0100 http://bitbucket.org/pypy/pypy/changeset/662eb2c58644/ Log: Now reject u'' literals, expect to break many tests here and there... diff --git a/pypy/interpreter/pyparser/genpytokenize.py b/pypy/interpreter/pyparser/genpytokenize.py --- a/pypy/interpreter/pyparser/genpytokenize.py +++ b/pypy/interpreter/pyparser/genpytokenize.py @@ -141,7 +141,7 @@ # ____________________________________________________________ def makeStrPrefix (): return chain(states, - maybe(states, groupStr(states, "uUbB")), + maybe(states, groupStr(states, "bB")), maybe(states, groupStr(states, "rR"))) # ____________________________________________________________ contStr = group(states, diff --git a/pypy/interpreter/pyparser/parsestring.py b/pypy/interpreter/pyparser/parsestring.py --- a/pypy/interpreter/pyparser/parsestring.py +++ b/pypy/interpreter/pyparser/parsestring.py @@ -21,10 +21,6 @@ ps += 1 quote = s[ps] unicode = False - elif quote == 'u' or quote == 'U': - ps += 1 - quote = s[ps] - unicode = True if quote == 'r' or quote == 'R': ps += 1 quote = s[ps] diff --git a/pypy/interpreter/pyparser/pytokenize.py b/pypy/interpreter/pyparser/pytokenize.py --- a/pypy/interpreter/pyparser/pytokenize.py +++ b/pypy/interpreter/pyparser/pytokenize.py @@ -45,7 +45,7 @@ 'I': 1, 'J': 1, 'K': 1, 'L': 1, 'M': 1, 'N': 1, 'O': 1, 'P': 1, 'Q': 1, 'R': 3, 'S': 1, 'T': 1, - 'U': 2, 'V': 1, 'W': 1, 'X': 1, + 'U': 1, 'V': 1, 'W': 1, 'X': 1, 'Y': 1, 'Z': 1, '[': 13, '\\': 17, ']': 13, '^': 12, '_': 1, '`': 13, 'a': 1, 'b': 2, 'c': 1, 'd': 1, @@ -53,7 +53,7 @@ 'i': 1, 'j': 1, 'k': 1, 'l': 1, 'm': 1, 'n': 1, 'o': 1, 'p': 1, 'q': 1, 'r': 3, 's': 1, 't': 1, - 'u': 2, 'v': 1, 'w': 1, 'x': 1, + 'u': 1, 'v': 1, 'w': 1, 'x': 1, 'y': 1, 'z': 1, '{': 13, '|': 12, '}': 13, '~': 13}, # 1 @@ -311,12 +311,10 @@ '"' : doubleDFA, 'r' : None, 'R' : None, - 'u' : None, - 'U' : None, 'b' : None, 'B' : None} -for uniPrefix in ("", "u", "U", "b", "B"): +for uniPrefix in ("", "b", "B"): for rawPrefix in ("", "r", "R"): prefix = uniPrefix + rawPrefix endDFAs[prefix + "'''"] = single3DFA @@ -332,20 +330,14 @@ triple_quoted = {} for t in ("'''", '"""', "r'''", 'r"""', "R'''", 'R"""', - "u'''", 'u"""', "U'''", 'U"""', "b'''", 'b"""', "B'''", 'B"""', - "ur'''", 'ur"""', "Ur'''", 'Ur"""', - "uR'''", 'uR"""', "UR'''", 'UR"""', "br'''", 'br"""', "Br'''", 'Br"""', "bR'''", 'bR"""', "BR'''", 'BR"""'): triple_quoted[t] = t single_quoted = {} for t in ("'", '"', "r'", 'r"', "R'", 'R"', - "u'", 'u"', "U'", 'U"', "b'", 'b"', "B'", 'B"', - "ur'", 'ur"', "Ur'", 'Ur"', - "uR'", 'uR"', "UR'", 'UR"', "br'", 'br"', "Br'", 'Br"', "bR'", 'bR"', "BR'", 'BR"'): single_quoted[t] = t diff --git a/pypy/interpreter/pyparser/test/test_parsestring.py b/pypy/interpreter/pyparser/test/test_parsestring.py --- a/pypy/interpreter/pyparser/test/test_parsestring.py +++ b/pypy/interpreter/pyparser/test/test_parsestring.py @@ -6,8 +6,8 @@ space = self.space w_ret = parsestring.parsestr(space, None, literal) if isinstance(value, str): - assert space.type(w_ret) == space.w_str - assert space.str_w(w_ret) == value + assert space.type(w_ret) == space.w_bytes + assert space.bytes_w(w_ret) == value elif isinstance(value, unicode): assert space.type(w_ret) == space.w_unicode assert space.unicode_w(w_ret) == value @@ -17,49 +17,49 @@ def test_simple(self): space = self.space for s in ['hello world', 'hello\n world']: - self.parse_and_compare(repr(s), s) + self.parse_and_compare('b' + repr(s), s) - self.parse_and_compare("'''hello\\x42 world'''", 'hello\x42 world') + self.parse_and_compare("b'''hello\\x42 world'''", 'hello\x42 world') # octal - self.parse_and_compare(r'"\0"', chr(0)) - self.parse_and_compare(r'"\07"', chr(7)) - self.parse_and_compare(r'"\123"', chr(0123)) - self.parse_and_compare(r'"\400"', chr(0)) - self.parse_and_compare(r'"\9"', '\\' + '9') - self.parse_and_compare(r'"\08"', chr(0) + '8') + self.parse_and_compare(r'b"\0"', chr(0)) + self.parse_and_compare(r'b"\07"', chr(7)) + self.parse_and_compare(r'b"\123"', chr(0123)) + self.parse_and_compare(r'b"\400"', chr(0)) + self.parse_and_compare(r'b"\9"', '\\' + '9') + self.parse_and_compare(r'b"\08"', chr(0) + '8') # hexadecimal - self.parse_and_compare(r'"\xfF"', chr(0xFF)) - self.parse_and_compare(r'"\""', '"') - self.parse_and_compare(r"'\''", "'") - for s in (r'"\x"', r'"\x7"', r'"\x7g"'): + self.parse_and_compare(r'b"\xfF"', chr(0xFF)) + self.parse_and_compare(r'b"\""', '"') + self.parse_and_compare(r"b'\''", "'") + for s in (r'b"\x"', r'b"\x7"', r'b"\x7g"'): space.raises_w(space.w_ValueError, parsestring.parsestr, space, None, s) def test_unicode(self): space = self.space - for s in [u'hello world', u'hello\n world']: - self.parse_and_compare(repr(s), s) + for s in ['hello world', 'hello\n world']: + self.parse_and_compare(repr(s), unicode(s)) - self.parse_and_compare("u'''hello\\x42 world'''", + self.parse_and_compare("'''hello\\x42 world'''", u'hello\x42 world') - self.parse_and_compare("u'''hello\\u0842 world'''", + self.parse_and_compare("'''hello\\u0842 world'''", u'hello\u0842 world') s = "u'\x81'" - s = s.decode("koi8-u").encode("utf8") + s = s.decode("koi8-u").encode("utf8")[1:] w_ret = parsestring.parsestr(self.space, 'koi8-u', s) ret = space.unwrap(w_ret) assert ret == eval("# -*- coding: koi8-u -*-\nu'\x81'") def test_unicode_literals(self): space = self.space - w_ret = parsestring.parsestr(space, None, repr("hello"), True) + w_ret = parsestring.parsestr(space, None, repr("hello")) assert space.isinstance_w(w_ret, space.w_unicode) - w_ret = parsestring.parsestr(space, None, "b'hi'", True) + w_ret = parsestring.parsestr(space, None, "b'hi'") assert space.isinstance_w(w_ret, space.w_str) - w_ret = parsestring.parsestr(space, None, "r'hi'", True) + w_ret = parsestring.parsestr(space, None, "r'hi'") assert space.isinstance_w(w_ret, space.w_unicode) def test_bytes(self): @@ -77,7 +77,7 @@ s = s.decode("koi8-u").encode("utf8") w_ret = parsestring.parsestr(self.space, 'koi8-u', s) ret = space.unwrap(w_ret) - assert ret == eval("# -*- coding: koi8-u -*-\n'\x81'") + assert ret == eval("# -*- coding: koi8-u -*-\nu'\x81'") def test_multiline_unicode_strings_with_backslash(self): space = self.space From noreply at buildbot.pypy.org Sat Jan 14 21:48:40 2012 From: noreply at buildbot.pypy.org (amauryfa) Date: Sat, 14 Jan 2012 21:48:40 +0100 (CET) Subject: [pypy-commit] pypy py3k: intern() is now in the sys module Message-ID: <20120114204840.7F1CC82B12@wyvern.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3k Changeset: r51329:cb7edd67143d Date: 2011-12-27 17:41 +0100 http://bitbucket.org/pypy/pypy/changeset/cb7edd67143d/ Log: intern() is now in the sys module diff --git a/pypy/module/__builtin__/__init__.py b/pypy/module/__builtin__/__init__.py --- a/pypy/module/__builtin__/__init__.py +++ b/pypy/module/__builtin__/__init__.py @@ -70,7 +70,6 @@ 'iter' : 'operation.iter', 'next' : 'operation.next', 'id' : 'operation.id', - 'intern' : 'operation.intern', 'callable' : 'operation.callable', 'compile' : 'compiling.compile', diff --git a/pypy/module/__builtin__/operation.py b/pypy/module/__builtin__/operation.py --- a/pypy/module/__builtin__/operation.py +++ b/pypy/module/__builtin__/operation.py @@ -220,15 +220,6 @@ space.setattr(w_object, w_name, w_val) return space.w_None -def intern(space, w_str): - """``Intern'' the given string. This enters the string in the (global) -table of interned strings whose purpose is to speed up dictionary lookups. -Return the string itself or the previously interned string object with the -same value.""" - if space.is_w(space.type(w_str), space.w_unicode): - return space.new_interned_w_str(w_str) - raise OperationError(space.w_TypeError, space.wrap("intern() argument must be string.")) - def callable(space, w_object): """Check whether the object appears to be callable (i.e., some kind of function). Note that classes are callable.""" diff --git a/pypy/module/__builtin__/test/test_builtin.py b/pypy/module/__builtin__/test/test_builtin.py --- a/pypy/module/__builtin__/test/test_builtin.py +++ b/pypy/module/__builtin__/test/test_builtin.py @@ -93,20 +93,6 @@ raises(ValueError, chr, -1) raises(ValueError, chr, sys.maxunicode+1) - def test_intern(self): - raises(TypeError, intern) - raises(TypeError, intern, 1) - class S(str): - pass - raises(TypeError, intern, S("hello")) - s = "never interned before" - s2 = intern(s) - assert s == s2 - s3 = s.swapcase() - assert s3 != s2 - s4 = s3.swapcase() - assert intern(s4) is s2 - def test_globals(self): d = {"foo":"bar"} exec("def f(): return globals()", d) diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py --- a/pypy/module/sys/__init__.py +++ b/pypy/module/sys/__init__.py @@ -55,6 +55,7 @@ 'getprofile' : 'vm.getprofile', 'call_tracing' : 'vm.call_tracing', 'getsizeof' : 'vm.getsizeof', + 'intern' : 'vm.intern', 'executable' : 'space.wrap("py.py")', 'api_version' : 'version.get_api_version(space)', diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py --- a/pypy/module/sys/test/test_sysmodule.py +++ b/pypy/module/sys/test/test_sysmodule.py @@ -607,3 +607,19 @@ assert len(frames) == 1 _, other_frame = frames.popitem() assert other_frame.f_code.co_name in ('other_thread', '?') + + def test_intern(self): + from sys import intern + raises(TypeError, intern) + raises(TypeError, intern, 1) + class S(str): + pass + raises(TypeError, intern, S("hello")) + s = "never interned before" + s2 = intern(s) + assert s == s2 + s3 = s.swapcase() + assert s3 != s2 + s4 = s3.swapcase() + assert intern(s4) is s2 + diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py --- a/pypy/module/sys/vm.py +++ b/pypy/module/sys/vm.py @@ -201,3 +201,13 @@ raise OperationError(space.w_TypeError, space.wrap("sys.getsizeof() not implemented on PyPy")) return w_default + +def intern(space, w_str): + """``Intern'' the given string. This enters the string in the (global) +table of interned strings whose purpose is to speed up dictionary lookups. +Return the string itself or the previously interned string object with the +same value.""" + if space.is_w(space.type(w_str), space.w_unicode): + return space.new_interned_w_str(w_str) + raise OperationError(space.w_TypeError, space.wrap("intern() argument must be string.")) + From noreply at buildbot.pypy.org Sat Jan 14 21:48:46 2012 From: noreply at buildbot.pypy.org (amauryfa) Date: Sat, 14 Jan 2012 21:48:46 +0100 (CET) Subject: [pypy-commit] pypy py3k: hg merge default Message-ID: <20120114204846.C7B5082B12@wyvern.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3k Changeset: r51330:86189f3364a6 Date: 2012-01-14 21:45 +0100 http://bitbucket.org/pypy/pypy/changeset/86189f3364a6/ Log: hg merge default diff too long, truncating to 10000 out of 21522 lines diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -27,7 +27,7 @@ DEALINGS IN THE SOFTWARE. -PyPy Copyright holders 2003-2011 +PyPy Copyright holders 2003-2012 ----------------------------------- Except when otherwise stated (look for LICENSE files or information at @@ -37,43 +37,47 @@ Armin Rigo Maciej Fijalkowski Carl Friedrich Bolz + Amaury Forgeot d'Arc Antonio Cuni - Amaury Forgeot d'Arc Samuele Pedroni Michael Hudson Holger Krekel - Benjamin Peterson + Alex Gaynor Christian Tismer Hakan Ardo - Alex Gaynor + Benjamin Peterson + David Schneider Eric van Riet Paap Anders Chrigstrom - David Schneider Richard Emslie Dan Villiom Podlaski Christiansen Alexander Schremmer + Lukas Diekmann Aurelien Campeas Anders Lehmann Camillo Bruni Niklaus Haldimann + Sven Hager Leonardo Santagada Toon Verwaest Seo Sanghyeon + Justin Peel Lawrence Oluyede Bartosz Skowron Jakub Gustak Guido Wesdorp Daniel Roberts + Laura Creighton Adrien Di Mascio - Laura Creighton Ludovic Aubry Niko Matsakis + Wim Lavrijsen + Matti Picus Jason Creighton Jacob Hallen Alex Martelli Anders Hammarquist Jan de Mooij - Wim Lavrijsen Stephan Diehl Michael Foord Stefan Schwarzer @@ -84,34 +88,36 @@ Alexandre Fayolle Marius Gedminas Simon Burton - Justin Peel + David Edelsohn Jean-Paul Calderone John Witulski - Lukas Diekmann + Timo Paulssen holger krekel - Wim Lavrijsen Dario Bertini + Mark Pearse Andreas Stührk Jean-Philippe St. Pierre Guido van Rossum Pavel Vinogradov Valentino Volonghi Paul deGrandis + Ilya Osadchiy + Ronny Pfannschmidt Adrian Kuhn tav Georg Brandl + Philip Jenvey Gerald Klix Wanja Saatkamp - Ronny Pfannschmidt Boris Feigin Oscar Nierstrasz David Malcolm Eugene Oden Henry Mason - Sven Hager + Jeff Terrace Lukas Renggli - Ilya Osadchiy Guenter Jantzen + Ned Batchelder Bert Freudenberg Amit Regmi Ben Young @@ -142,7 +148,6 @@ Anders Qvist Beatrice During Alexander Sedov - Timo Paulssen Corbin Simpson Vincent Legoll Romain Guillebert @@ -165,9 +170,10 @@ Lucio Torre Lene Wagner Miguel de Val Borro + Artur Lisiecki + Bruno Gola Ignas Mikalajunas - Artur Lisiecki - Philip Jenvey + Stefano Rivera Joshua Gilbert Godefroid Chappelle Yusei Tahara @@ -179,17 +185,17 @@ Kristjan Valur Jonsson Bobby Impollonia Michael Hudson-Doyle + Laurence Tratt + Yasir Suhail Andrew Thompson Anders Sigfridsson Floris Bruynooghe Jacek Generowicz Dan Colish Zooko Wilcox-O Hearn - Dan Villiom Podlaski Christiansen - Anders Hammarquist + Dan Loewenherz Chris Lambacher Dinu Gherman - Dan Colish Brett Cannon Daniel Neuhäuser Michael Chermside diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py --- a/lib_pypy/_ctypes/structure.py +++ b/lib_pypy/_ctypes/structure.py @@ -73,8 +73,12 @@ class Field(object): def __init__(self, name, offset, size, ctype, num, is_bitfield): - for k in ('name', 'offset', 'size', 'ctype', 'num', 'is_bitfield'): - self.__dict__[k] = locals()[k] + self.__dict__['name'] = name + self.__dict__['offset'] = offset + self.__dict__['size'] = size + self.__dict__['ctype'] = ctype + self.__dict__['num'] = num + self.__dict__['is_bitfield'] = is_bitfield def __setattr__(self, name, value): raise AttributeError(name) diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py --- a/lib_pypy/_sqlite3.py +++ b/lib_pypy/_sqlite3.py @@ -231,8 +231,10 @@ sqlite.sqlite3_result_text.argtypes = [c_void_p, c_char_p, c_int, c_void_p] sqlite.sqlite3_result_text.restype = None -sqlite.sqlite3_enable_load_extension.argtypes = [c_void_p, c_int] -sqlite.sqlite3_enable_load_extension.restype = c_int +HAS_LOAD_EXTENSION = hasattr(sqlite, "sqlite3_enable_load_extension") +if HAS_LOAD_EXTENSION: + sqlite.sqlite3_enable_load_extension.argtypes = [c_void_p, c_int] + sqlite.sqlite3_enable_load_extension.restype = c_int ########################################## # END Wrapped SQLite C API and constants @@ -708,13 +710,14 @@ from sqlite3.dump import _iterdump return _iterdump(self) - def enable_load_extension(self, enabled): - self._check_thread() - self._check_closed() + if HAS_LOAD_EXTENSION: + def enable_load_extension(self, enabled): + self._check_thread() + self._check_closed() - rc = sqlite.sqlite3_enable_load_extension(self.db, int(enabled)) - if rc != SQLITE_OK: - raise OperationalError("Error enabling load extension") + rc = sqlite.sqlite3_enable_load_extension(self.db, int(enabled)) + if rc != SQLITE_OK: + raise OperationalError("Error enabling load extension") DML, DQL, DDL = range(3) diff --git a/lib_pypy/numpypy/__init__.py b/lib_pypy/numpypy/__init__.py new file mode 100644 --- /dev/null +++ b/lib_pypy/numpypy/__init__.py @@ -0,0 +1,2 @@ +from _numpypy import * +from .fromnumeric import * diff --git a/lib_pypy/numpypy/fromnumeric.py b/lib_pypy/numpypy/fromnumeric.py new file mode 100644 --- /dev/null +++ b/lib_pypy/numpypy/fromnumeric.py @@ -0,0 +1,2400 @@ +###################################################################### +# This is a copy of numpy/core/fromnumeric.py modified for numpypy +###################################################################### +# Each name in __all__ was a function in 'numeric' that is now +# a method in 'numpy'. +# When the corresponding method is added to numpypy BaseArray +# each function should be added as a module function +# at the applevel +# This can be as simple as doing the following +# +# def func(a, ...): +# if not hasattr(a, 'func') +# a = numpypy.array(a) +# return a.func(...) +# +###################################################################### + +import numpypy + +# Module containing non-deprecated functions borrowed from Numeric. +__docformat__ = "restructuredtext en" + +# functions that are now methods +__all__ = ['take', 'reshape', 'choose', 'repeat', 'put', + 'swapaxes', 'transpose', 'sort', 'argsort', 'argmax', 'argmin', + 'searchsorted', 'alen', + 'resize', 'diagonal', 'trace', 'ravel', 'nonzero', 'shape', + 'compress', 'clip', 'sum', 'product', 'prod', 'sometrue', 'alltrue', + 'any', 'all', 'cumsum', 'cumproduct', 'cumprod', 'ptp', 'ndim', + 'rank', 'size', 'around', 'round_', 'mean', 'std', 'var', 'squeeze', + 'amax', 'amin', + ] + +def take(a, indices, axis=None, out=None, mode='raise'): + """ + Take elements from an array along an axis. + + This function does the same thing as "fancy" indexing (indexing arrays + using arrays); however, it can be easier to use if you need elements + along a given axis. + + Parameters + ---------- + a : array_like + The source array. + indices : array_like + The indices of the values to extract. + axis : int, optional + The axis over which to select values. By default, the flattened + input array is used. + out : ndarray, optional + If provided, the result will be placed in this array. It should + be of the appropriate shape and dtype. + mode : {'raise', 'wrap', 'clip'}, optional + Specifies how out-of-bounds indices will behave. + + * 'raise' -- raise an error (default) + * 'wrap' -- wrap around + * 'clip' -- clip to the range + + 'clip' mode means that all indices that are too large are replaced + by the index that addresses the last element along that axis. Note + that this disables indexing with negative numbers. + + Returns + ------- + subarray : ndarray + The returned array has the same type as `a`. + + See Also + -------- + ndarray.take : equivalent method + + Examples + -------- + >>> a = [4, 3, 5, 7, 6, 8] + >>> indices = [0, 1, 4] + >>> np.take(a, indices) + array([4, 3, 6]) + + In this example if `a` is an ndarray, "fancy" indexing can be used. + + >>> a = np.array(a) + >>> a[indices] + array([4, 3, 6]) + + """ + raise NotImplemented('Waiting on interp level method') + + +# not deprecated --- copy if necessary, view otherwise +def reshape(a, newshape, order='C'): + """ + Gives a new shape to an array without changing its data. + + Parameters + ---------- + a : array_like + Array to be reshaped. + newshape : int or tuple of ints + The new shape should be compatible with the original shape. If + an integer, then the result will be a 1-D array of that length. + One shape dimension can be -1. In this case, the value is inferred + from the length of the array and remaining dimensions. + order : {'C', 'F', 'A'}, optional + Determines whether the array data should be viewed as in C + (row-major) order, FORTRAN (column-major) order, or the C/FORTRAN + order should be preserved. + + Returns + ------- + reshaped_array : ndarray + This will be a new view object if possible; otherwise, it will + be a copy. + + + See Also + -------- + ndarray.reshape : Equivalent method. + + Notes + ----- + + It is not always possible to change the shape of an array without + copying the data. If you want an error to be raise if the data is copied, + you should assign the new shape to the shape attribute of the array:: + + >>> a = np.zeros((10, 2)) + # A transpose make the array non-contiguous + >>> b = a.T + # Taking a view makes it possible to modify the shape without modiying the + # initial object. + >>> c = b.view() + >>> c.shape = (20) + AttributeError: incompatible shape for a non-contiguous array + + + Examples + -------- + >>> a = np.array([[1,2,3], [4,5,6]]) + >>> np.reshape(a, 6) + array([1, 2, 3, 4, 5, 6]) + >>> np.reshape(a, 6, order='F') + array([1, 4, 2, 5, 3, 6]) + + >>> np.reshape(a, (3,-1)) # the unspecified value is inferred to be 2 + array([[1, 2], + [3, 4], + [5, 6]]) + + """ + if not hasattr(a, 'reshape'): + a = numpypy.array(a) + return a.reshape(newshape) + + +def choose(a, choices, out=None, mode='raise'): + """ + Construct an array from an index array and a set of arrays to choose from. + + First of all, if confused or uncertain, definitely look at the Examples - + in its full generality, this function is less simple than it might + seem from the following code description (below ndi = + `numpy.lib.index_tricks`): + + ``np.choose(a,c) == np.array([c[a[I]][I] for I in ndi.ndindex(a.shape)])``. + + But this omits some subtleties. Here is a fully general summary: + + Given an "index" array (`a`) of integers and a sequence of `n` arrays + (`choices`), `a` and each choice array are first broadcast, as necessary, + to arrays of a common shape; calling these *Ba* and *Bchoices[i], i = + 0,...,n-1* we have that, necessarily, ``Ba.shape == Bchoices[i].shape`` + for each `i`. Then, a new array with shape ``Ba.shape`` is created as + follows: + + * if ``mode=raise`` (the default), then, first of all, each element of + `a` (and thus `Ba`) must be in the range `[0, n-1]`; now, suppose that + `i` (in that range) is the value at the `(j0, j1, ..., jm)` position + in `Ba` - then the value at the same position in the new array is the + value in `Bchoices[i]` at that same position; + + * if ``mode=wrap``, values in `a` (and thus `Ba`) may be any (signed) + integer; modular arithmetic is used to map integers outside the range + `[0, n-1]` back into that range; and then the new array is constructed + as above; + + * if ``mode=clip``, values in `a` (and thus `Ba`) may be any (signed) + integer; negative integers are mapped to 0; values greater than `n-1` + are mapped to `n-1`; and then the new array is constructed as above. + + Parameters + ---------- + a : int array + This array must contain integers in `[0, n-1]`, where `n` is the number + of choices, unless ``mode=wrap`` or ``mode=clip``, in which cases any + integers are permissible. + choices : sequence of arrays + Choice arrays. `a` and all of the choices must be broadcastable to the + same shape. If `choices` is itself an array (not recommended), then + its outermost dimension (i.e., the one corresponding to + ``choices.shape[0]``) is taken as defining the "sequence". + out : array, optional + If provided, the result will be inserted into this array. It should + be of the appropriate shape and dtype. + mode : {'raise' (default), 'wrap', 'clip'}, optional + Specifies how indices outside `[0, n-1]` will be treated: + + * 'raise' : an exception is raised + * 'wrap' : value becomes value mod `n` + * 'clip' : values < 0 are mapped to 0, values > n-1 are mapped to n-1 + + Returns + ------- + merged_array : array + The merged result. + + Raises + ------ + ValueError: shape mismatch + If `a` and each choice array are not all broadcastable to the same + shape. + + See Also + -------- + ndarray.choose : equivalent method + + Notes + ----- + To reduce the chance of misinterpretation, even though the following + "abuse" is nominally supported, `choices` should neither be, nor be + thought of as, a single array, i.e., the outermost sequence-like container + should be either a list or a tuple. + + Examples + -------- + + >>> choices = [[0, 1, 2, 3], [10, 11, 12, 13], + ... [20, 21, 22, 23], [30, 31, 32, 33]] + >>> np.choose([2, 3, 1, 0], choices + ... # the first element of the result will be the first element of the + ... # third (2+1) "array" in choices, namely, 20; the second element + ... # will be the second element of the fourth (3+1) choice array, i.e., + ... # 31, etc. + ... ) + array([20, 31, 12, 3]) + >>> np.choose([2, 4, 1, 0], choices, mode='clip') # 4 goes to 3 (4-1) + array([20, 31, 12, 3]) + >>> # because there are 4 choice arrays + >>> np.choose([2, 4, 1, 0], choices, mode='wrap') # 4 goes to (4 mod 4) + array([20, 1, 12, 3]) + >>> # i.e., 0 + + A couple examples illustrating how choose broadcasts: + + >>> a = [[1, 0, 1], [0, 1, 0], [1, 0, 1]] + >>> choices = [-10, 10] + >>> np.choose(a, choices) + array([[ 10, -10, 10], + [-10, 10, -10], + [ 10, -10, 10]]) + + >>> # With thanks to Anne Archibald + >>> a = np.array([0, 1]).reshape((2,1,1)) + >>> c1 = np.array([1, 2, 3]).reshape((1,3,1)) + >>> c2 = np.array([-1, -2, -3, -4, -5]).reshape((1,1,5)) + >>> np.choose(a, (c1, c2)) # result is 2x3x5, res[0,:,:]=c1, res[1,:,:]=c2 + array([[[ 1, 1, 1, 1, 1], + [ 2, 2, 2, 2, 2], + [ 3, 3, 3, 3, 3]], + [[-1, -2, -3, -4, -5], + [-1, -2, -3, -4, -5], + [-1, -2, -3, -4, -5]]]) + + """ + raise NotImplemented('Waiting on interp level method') + + +def repeat(a, repeats, axis=None): + """ + Repeat elements of an array. + + Parameters + ---------- + a : array_like + Input array. + repeats : {int, array of ints} + The number of repetitions for each element. `repeats` is broadcasted + to fit the shape of the given axis. + axis : int, optional + The axis along which to repeat values. By default, use the + flattened input array, and return a flat output array. + + Returns + ------- + repeated_array : ndarray + Output array which has the same shape as `a`, except along + the given axis. + + See Also + -------- + tile : Tile an array. + + Examples + -------- + >>> x = np.array([[1,2],[3,4]]) + >>> np.repeat(x, 2) + array([1, 1, 2, 2, 3, 3, 4, 4]) + >>> np.repeat(x, 3, axis=1) + array([[1, 1, 1, 2, 2, 2], + [3, 3, 3, 4, 4, 4]]) + >>> np.repeat(x, [1, 2], axis=0) + array([[1, 2], + [3, 4], + [3, 4]]) + + """ + raise NotImplemented('Waiting on interp level method') + + +def put(a, ind, v, mode='raise'): + """ + Replaces specified elements of an array with given values. + + The indexing works on the flattened target array. `put` is roughly + equivalent to: + + :: + + a.flat[ind] = v + + Parameters + ---------- + a : ndarray + Target array. + ind : array_like + Target indices, interpreted as integers. + v : array_like + Values to place in `a` at target indices. If `v` is shorter than + `ind` it will be repeated as necessary. + mode : {'raise', 'wrap', 'clip'}, optional + Specifies how out-of-bounds indices will behave. + + * 'raise' -- raise an error (default) + * 'wrap' -- wrap around + * 'clip' -- clip to the range + + 'clip' mode means that all indices that are too large are replaced + by the index that addresses the last element along that axis. Note + that this disables indexing with negative numbers. + + See Also + -------- + putmask, place + + Examples + -------- + >>> a = np.arange(5) + >>> np.put(a, [0, 2], [-44, -55]) + >>> a + array([-44, 1, -55, 3, 4]) + + >>> a = np.arange(5) + >>> np.put(a, 22, -5, mode='clip') + >>> a + array([ 0, 1, 2, 3, -5]) + + """ + raise NotImplemented('Waiting on interp level method') + + +def swapaxes(a, axis1, axis2): + """ + Interchange two axes of an array. + + Parameters + ---------- + a : array_like + Input array. + axis1 : int + First axis. + axis2 : int + Second axis. + + Returns + ------- + a_swapped : ndarray + If `a` is an ndarray, then a view of `a` is returned; otherwise + a new array is created. + + Examples + -------- + >>> x = np.array([[1,2,3]]) + >>> np.swapaxes(x,0,1) + array([[1], + [2], + [3]]) + + >>> x = np.array([[[0,1],[2,3]],[[4,5],[6,7]]]) + >>> x + array([[[0, 1], + [2, 3]], + [[4, 5], + [6, 7]]]) + + >>> np.swapaxes(x,0,2) + array([[[0, 4], + [2, 6]], + [[1, 5], + [3, 7]]]) + + """ + raise NotImplemented('Waiting on interp level method') + + +def transpose(a, axes=None): + """ + Permute the dimensions of an array. + + Parameters + ---------- + a : array_like + Input array. + axes : list of ints, optional + By default, reverse the dimensions, otherwise permute the axes + according to the values given. + + Returns + ------- + p : ndarray + `a` with its axes permuted. A view is returned whenever + possible. + + See Also + -------- + rollaxis + + Examples + -------- + >>> x = np.arange(4).reshape((2,2)) + >>> x + array([[0, 1], + [2, 3]]) + + >>> np.transpose(x) + array([[0, 2], + [1, 3]]) + + >>> x = np.ones((1, 2, 3)) + >>> np.transpose(x, (1, 0, 2)).shape + (2, 1, 3) + + """ + raise NotImplemented('Waiting on interp level method') + + +def sort(a, axis=-1, kind='quicksort', order=None): + """ + Return a sorted copy of an array. + + Parameters + ---------- + a : array_like + Array to be sorted. + axis : int or None, optional + Axis along which to sort. If None, the array is flattened before + sorting. The default is -1, which sorts along the last axis. + kind : {'quicksort', 'mergesort', 'heapsort'}, optional + Sorting algorithm. Default is 'quicksort'. + order : list, optional + When `a` is a structured array, this argument specifies which fields + to compare first, second, and so on. This list does not need to + include all of the fields. + + Returns + ------- + sorted_array : ndarray + Array of the same type and shape as `a`. + + See Also + -------- + ndarray.sort : Method to sort an array in-place. + argsort : Indirect sort. + lexsort : Indirect stable sort on multiple keys. + searchsorted : Find elements in a sorted array. + + Notes + ----- + The various sorting algorithms are characterized by their average speed, + worst case performance, work space size, and whether they are stable. A + stable sort keeps items with the same key in the same relative + order. The three available algorithms have the following + properties: + + =========== ======= ============= ============ ======= + kind speed worst case work space stable + =========== ======= ============= ============ ======= + 'quicksort' 1 O(n^2) 0 no + 'mergesort' 2 O(n*log(n)) ~n/2 yes + 'heapsort' 3 O(n*log(n)) 0 no + =========== ======= ============= ============ ======= + + All the sort algorithms make temporary copies of the data when + sorting along any but the last axis. Consequently, sorting along + the last axis is faster and uses less space than sorting along + any other axis. + + The sort order for complex numbers is lexicographic. If both the real + and imaginary parts are non-nan then the order is determined by the + real parts except when they are equal, in which case the order is + determined by the imaginary parts. + + Previous to numpy 1.4.0 sorting real and complex arrays containing nan + values led to undefined behaviour. In numpy versions >= 1.4.0 nan + values are sorted to the end. The extended sort order is: + + * Real: [R, nan] + * Complex: [R + Rj, R + nanj, nan + Rj, nan + nanj] + + where R is a non-nan real value. Complex values with the same nan + placements are sorted according to the non-nan part if it exists. + Non-nan values are sorted as before. + + Examples + -------- + >>> a = np.array([[1,4],[3,1]]) + >>> np.sort(a) # sort along the last axis + array([[1, 4], + [1, 3]]) + >>> np.sort(a, axis=None) # sort the flattened array + array([1, 1, 3, 4]) + >>> np.sort(a, axis=0) # sort along the first axis + array([[1, 1], + [3, 4]]) + + Use the `order` keyword to specify a field to use when sorting a + structured array: + + >>> dtype = [('name', 'S10'), ('height', float), ('age', int)] + >>> values = [('Arthur', 1.8, 41), ('Lancelot', 1.9, 38), + ... ('Galahad', 1.7, 38)] + >>> a = np.array(values, dtype=dtype) # create a structured array + >>> np.sort(a, order='height') # doctest: +SKIP + array([('Galahad', 1.7, 38), ('Arthur', 1.8, 41), + ('Lancelot', 1.8999999999999999, 38)], + dtype=[('name', '|S10'), ('height', '>> np.sort(a, order=['age', 'height']) # doctest: +SKIP + array([('Galahad', 1.7, 38), ('Lancelot', 1.8999999999999999, 38), + ('Arthur', 1.8, 41)], + dtype=[('name', '|S10'), ('height', '>> x = np.array([3, 1, 2]) + >>> np.argsort(x) + array([1, 2, 0]) + + Two-dimensional array: + + >>> x = np.array([[0, 3], [2, 2]]) + >>> x + array([[0, 3], + [2, 2]]) + + >>> np.argsort(x, axis=0) + array([[0, 1], + [1, 0]]) + + >>> np.argsort(x, axis=1) + array([[0, 1], + [0, 1]]) + + Sorting with keys: + + >>> x = np.array([(1, 0), (0, 1)], dtype=[('x', '>> x + array([(1, 0), (0, 1)], + dtype=[('x', '>> np.argsort(x, order=('x','y')) + array([1, 0]) + + >>> np.argsort(x, order=('y','x')) + array([0, 1]) + + """ + raise NotImplemented('Waiting on interp level method') + + +def argmax(a, axis=None): + """ + Indices of the maximum values along an axis. + + Parameters + ---------- + a : array_like + Input array. + axis : int, optional + By default, the index is into the flattened array, otherwise + along the specified axis. + + Returns + ------- + index_array : ndarray of ints + Array of indices into the array. It has the same shape as `a.shape` + with the dimension along `axis` removed. + + See Also + -------- + ndarray.argmax, argmin + amax : The maximum value along a given axis. + unravel_index : Convert a flat index into an index tuple. + + Notes + ----- + In case of multiple occurrences of the maximum values, the indices + corresponding to the first occurrence are returned. + + Examples + -------- + >>> a = np.arange(6).reshape(2,3) + >>> a + array([[0, 1, 2], + [3, 4, 5]]) + >>> np.argmax(a) + 5 + >>> np.argmax(a, axis=0) + array([1, 1, 1]) + >>> np.argmax(a, axis=1) + array([2, 2]) + + >>> b = np.arange(6) + >>> b[1] = 5 + >>> b + array([0, 5, 2, 3, 4, 5]) + >>> np.argmax(b) # Only the first occurrence is returned. + 1 + + """ + if not hasattr(a, 'argmax'): + a = numpypy.array(a) + return a.argmax() + + +def argmin(a, axis=None): + """ + Return the indices of the minimum values along an axis. + + See Also + -------- + argmax : Similar function. Please refer to `numpy.argmax` for detailed + documentation. + + """ + if not hasattr(a, 'argmin'): + a = numpypy.array(a) + return a.argmin() + + +def searchsorted(a, v, side='left'): + """ + Find indices where elements should be inserted to maintain order. + + Find the indices into a sorted array `a` such that, if the corresponding + elements in `v` were inserted before the indices, the order of `a` would + be preserved. + + Parameters + ---------- + a : 1-D array_like + Input array, sorted in ascending order. + v : array_like + Values to insert into `a`. + side : {'left', 'right'}, optional + If 'left', the index of the first suitable location found is given. If + 'right', return the last such index. If there is no suitable + index, return either 0 or N (where N is the length of `a`). + + Returns + ------- + indices : array of ints + Array of insertion points with the same shape as `v`. + + See Also + -------- + sort : Return a sorted copy of an array. + histogram : Produce histogram from 1-D data. + + Notes + ----- + Binary search is used to find the required insertion points. + + As of Numpy 1.4.0 `searchsorted` works with real/complex arrays containing + `nan` values. The enhanced sort order is documented in `sort`. + + Examples + -------- + >>> np.searchsorted([1,2,3,4,5], 3) + 2 + >>> np.searchsorted([1,2,3,4,5], 3, side='right') + 3 + >>> np.searchsorted([1,2,3,4,5], [-10, 10, 2, 3]) + array([0, 5, 1, 2]) + + """ + raise NotImplemented('Waiting on interp level method') + + +def resize(a, new_shape): + """ + Return a new array with the specified shape. + + If the new array is larger than the original array, then the new + array is filled with repeated copies of `a`. Note that this behavior + is different from a.resize(new_shape) which fills with zeros instead + of repeated copies of `a`. + + Parameters + ---------- + a : array_like + Array to be resized. + + new_shape : int or tuple of int + Shape of resized array. + + Returns + ------- + reshaped_array : ndarray + The new array is formed from the data in the old array, repeated + if necessary to fill out the required number of elements. The + data are repeated in the order that they are stored in memory. + + See Also + -------- + ndarray.resize : resize an array in-place. + + Examples + -------- + >>> a=np.array([[0,1],[2,3]]) + >>> np.resize(a,(1,4)) + array([[0, 1, 2, 3]]) + >>> np.resize(a,(2,4)) + array([[0, 1, 2, 3], + [0, 1, 2, 3]]) + + """ + raise NotImplemented('Waiting on interp level method') + + +def squeeze(a): + """ + Remove single-dimensional entries from the shape of an array. + + Parameters + ---------- + a : array_like + Input data. + + Returns + ------- + squeezed : ndarray + The input array, but with with all dimensions of length 1 + removed. Whenever possible, a view on `a` is returned. + + Examples + -------- + >>> x = np.array([[[0], [1], [2]]]) + >>> x.shape + (1, 3, 1) + >>> np.squeeze(x).shape + (3,) + + """ + raise NotImplemented('Waiting on interp level method') + + +def diagonal(a, offset=0, axis1=0, axis2=1): + """ + Return specified diagonals. + + If `a` is 2-D, returns the diagonal of `a` with the given offset, + i.e., the collection of elements of the form ``a[i, i+offset]``. If + `a` has more than two dimensions, then the axes specified by `axis1` + and `axis2` are used to determine the 2-D sub-array whose diagonal is + returned. The shape of the resulting array can be determined by + removing `axis1` and `axis2` and appending an index to the right equal + to the size of the resulting diagonals. + + Parameters + ---------- + a : array_like + Array from which the diagonals are taken. + offset : int, optional + Offset of the diagonal from the main diagonal. Can be positive or + negative. Defaults to main diagonal (0). + axis1 : int, optional + Axis to be used as the first axis of the 2-D sub-arrays from which + the diagonals should be taken. Defaults to first axis (0). + axis2 : int, optional + Axis to be used as the second axis of the 2-D sub-arrays from + which the diagonals should be taken. Defaults to second axis (1). + + Returns + ------- + array_of_diagonals : ndarray + If `a` is 2-D, a 1-D array containing the diagonal is returned. + If the dimension of `a` is larger, then an array of diagonals is + returned, "packed" from left-most dimension to right-most (e.g., + if `a` is 3-D, then the diagonals are "packed" along rows). + + Raises + ------ + ValueError + If the dimension of `a` is less than 2. + + See Also + -------- + diag : MATLAB work-a-like for 1-D and 2-D arrays. + diagflat : Create diagonal arrays. + trace : Sum along diagonals. + + Examples + -------- + >>> a = np.arange(4).reshape(2,2) + >>> a + array([[0, 1], + [2, 3]]) + >>> a.diagonal() + array([0, 3]) + >>> a.diagonal(1) + array([1]) + + A 3-D example: + + >>> a = np.arange(8).reshape(2,2,2); a + array([[[0, 1], + [2, 3]], + [[4, 5], + [6, 7]]]) + >>> a.diagonal(0, # Main diagonals of two arrays created by skipping + ... 0, # across the outer(left)-most axis last and + ... 1) # the "middle" (row) axis first. + array([[0, 6], + [1, 7]]) + + The sub-arrays whose main diagonals we just obtained; note that each + corresponds to fixing the right-most (column) axis, and that the + diagonals are "packed" in rows. + + >>> a[:,:,0] # main diagonal is [0 6] + array([[0, 2], + [4, 6]]) + >>> a[:,:,1] # main diagonal is [1 7] + array([[1, 3], + [5, 7]]) + + """ + raise NotImplemented('Waiting on interp level method') + + +def trace(a, offset=0, axis1=0, axis2=1, dtype=None, out=None): + """ + Return the sum along diagonals of the array. + + If `a` is 2-D, the sum along its diagonal with the given offset + is returned, i.e., the sum of elements ``a[i,i+offset]`` for all i. + + If `a` has more than two dimensions, then the axes specified by axis1 and + axis2 are used to determine the 2-D sub-arrays whose traces are returned. + The shape of the resulting array is the same as that of `a` with `axis1` + and `axis2` removed. + + Parameters + ---------- + a : array_like + Input array, from which the diagonals are taken. + offset : int, optional + Offset of the diagonal from the main diagonal. Can be both positive + and negative. Defaults to 0. + axis1, axis2 : int, optional + Axes to be used as the first and second axis of the 2-D sub-arrays + from which the diagonals should be taken. Defaults are the first two + axes of `a`. + dtype : dtype, optional + Determines the data-type of the returned array and of the accumulator + where the elements are summed. If dtype has the value None and `a` is + of integer type of precision less than the default integer + precision, then the default integer precision is used. Otherwise, + the precision is the same as that of `a`. + out : ndarray, optional + Array into which the output is placed. Its type is preserved and + it must be of the right shape to hold the output. + + Returns + ------- + sum_along_diagonals : ndarray + If `a` is 2-D, the sum along the diagonal is returned. If `a` has + larger dimensions, then an array of sums along diagonals is returned. + + See Also + -------- + diag, diagonal, diagflat + + Examples + -------- + >>> np.trace(np.eye(3)) + 3.0 + >>> a = np.arange(8).reshape((2,2,2)) + >>> np.trace(a) + array([6, 8]) + + >>> a = np.arange(24).reshape((2,2,2,3)) + >>> np.trace(a).shape + (2, 3) + + """ + raise NotImplemented('Waiting on interp level method') + +def ravel(a, order='C'): + """ + Return a flattened array. + + A 1-D array, containing the elements of the input, is returned. A copy is + made only if needed. + + Parameters + ---------- + a : array_like + Input array. The elements in ``a`` are read in the order specified by + `order`, and packed as a 1-D array. + order : {'C','F', 'A', 'K'}, optional + The elements of ``a`` are read in this order. 'C' means to view + the elements in C (row-major) order. 'F' means to view the elements + in Fortran (column-major) order. 'A' means to view the elements + in 'F' order if a is Fortran contiguous, 'C' order otherwise. + 'K' means to view the elements in the order they occur in memory, + except for reversing the data when strides are negative. + By default, 'C' order is used. + + Returns + ------- + 1d_array : ndarray + Output of the same dtype as `a`, and of shape ``(a.size(),)``. + + See Also + -------- + ndarray.flat : 1-D iterator over an array. + ndarray.flatten : 1-D array copy of the elements of an array + in row-major order. + + Notes + ----- + In row-major order, the row index varies the slowest, and the column + index the quickest. This can be generalized to multiple dimensions, + where row-major order implies that the index along the first axis + varies slowest, and the index along the last quickest. The opposite holds + for Fortran-, or column-major, mode. + + Examples + -------- + It is equivalent to ``reshape(-1, order=order)``. + + >>> x = np.array([[1, 2, 3], [4, 5, 6]]) + >>> print np.ravel(x) + [1 2 3 4 5 6] + + >>> print x.reshape(-1) + [1 2 3 4 5 6] + + >>> print np.ravel(x, order='F') + [1 4 2 5 3 6] + + When ``order`` is 'A', it will preserve the array's 'C' or 'F' ordering: + + >>> print np.ravel(x.T) + [1 4 2 5 3 6] + >>> print np.ravel(x.T, order='A') + [1 2 3 4 5 6] + + When ``order`` is 'K', it will preserve orderings that are neither 'C' + nor 'F', but won't reverse axes: + + >>> a = np.arange(3)[::-1]; a + array([2, 1, 0]) + >>> a.ravel(order='C') + array([2, 1, 0]) + >>> a.ravel(order='K') + array([2, 1, 0]) + + >>> a = np.arange(12).reshape(2,3,2).swapaxes(1,2); a + array([[[ 0, 2, 4], + [ 1, 3, 5]], + [[ 6, 8, 10], + [ 7, 9, 11]]]) + >>> a.ravel(order='C') + array([ 0, 2, 4, 1, 3, 5, 6, 8, 10, 7, 9, 11]) + >>> a.ravel(order='K') + array([ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]) + + """ + raise NotImplemented('Waiting on interp level method') + + +def nonzero(a): + """ + Return the indices of the elements that are non-zero. + + Returns a tuple of arrays, one for each dimension of `a`, containing + the indices of the non-zero elements in that dimension. The + corresponding non-zero values can be obtained with:: + + a[nonzero(a)] + + To group the indices by element, rather than dimension, use:: + + transpose(nonzero(a)) + + The result of this is always a 2-D array, with a row for + each non-zero element. + + Parameters + ---------- + a : array_like + Input array. + + Returns + ------- + tuple_of_arrays : tuple + Indices of elements that are non-zero. + + See Also + -------- + flatnonzero : + Return indices that are non-zero in the flattened version of the input + array. + ndarray.nonzero : + Equivalent ndarray method. + count_nonzero : + Counts the number of non-zero elements in the input array. + + Examples + -------- + >>> x = np.eye(3) + >>> x + array([[ 1., 0., 0.], + [ 0., 1., 0.], + [ 0., 0., 1.]]) + >>> np.nonzero(x) + (array([0, 1, 2]), array([0, 1, 2])) + + >>> x[np.nonzero(x)] + array([ 1., 1., 1.]) + >>> np.transpose(np.nonzero(x)) + array([[0, 0], + [1, 1], + [2, 2]]) + + A common use for ``nonzero`` is to find the indices of an array, where + a condition is True. Given an array `a`, the condition `a` > 3 is a + boolean array and since False is interpreted as 0, np.nonzero(a > 3) + yields the indices of the `a` where the condition is true. + + >>> a = np.array([[1,2,3],[4,5,6],[7,8,9]]) + >>> a > 3 + array([[False, False, False], + [ True, True, True], + [ True, True, True]], dtype=bool) + >>> np.nonzero(a > 3) + (array([1, 1, 1, 2, 2, 2]), array([0, 1, 2, 0, 1, 2])) + + The ``nonzero`` method of the boolean array can also be called. + + >>> (a > 3).nonzero() + (array([1, 1, 1, 2, 2, 2]), array([0, 1, 2, 0, 1, 2])) + + """ + raise NotImplemented('Waiting on interp level method') + + +def shape(a): + """ + Return the shape of an array. + + Parameters + ---------- + a : array_like + Input array. + + Returns + ------- + shape : tuple of ints + The elements of the shape tuple give the lengths of the + corresponding array dimensions. + + See Also + -------- + alen + ndarray.shape : Equivalent array method. + + Examples + -------- + >>> np.shape(np.eye(3)) + (3, 3) + >>> np.shape([[1, 2]]) + (1, 2) + >>> np.shape([0]) + (1,) + >>> np.shape(0) + () + + >>> a = np.array([(1, 2), (3, 4)], dtype=[('x', 'i4'), ('y', 'i4')]) + >>> np.shape(a) + (2,) + >>> a.shape + (2,) + + """ + if not hasattr(a, 'shape'): + a = numpypy.array(a) + return a.shape + + +def compress(condition, a, axis=None, out=None): + """ + Return selected slices of an array along given axis. + + When working along a given axis, a slice along that axis is returned in + `output` for each index where `condition` evaluates to True. When + working on a 1-D array, `compress` is equivalent to `extract`. + + Parameters + ---------- + condition : 1-D array of bools + Array that selects which entries to return. If len(condition) + is less than the size of `a` along the given axis, then output is + truncated to the length of the condition array. + a : array_like + Array from which to extract a part. + axis : int, optional + Axis along which to take slices. If None (default), work on the + flattened array. + out : ndarray, optional + Output array. Its type is preserved and it must be of the right + shape to hold the output. + + Returns + ------- + compressed_array : ndarray + A copy of `a` without the slices along axis for which `condition` + is false. + + See Also + -------- + take, choose, diag, diagonal, select + ndarray.compress : Equivalent method. + numpy.doc.ufuncs : Section "Output arguments" + + Examples + -------- + >>> a = np.array([[1, 2], [3, 4], [5, 6]]) + >>> a + array([[1, 2], + [3, 4], + [5, 6]]) + >>> np.compress([0, 1], a, axis=0) + array([[3, 4]]) + >>> np.compress([False, True, True], a, axis=0) + array([[3, 4], + [5, 6]]) + >>> np.compress([False, True], a, axis=1) + array([[2], + [4], + [6]]) + + Working on the flattened array does not return slices along an axis but + selects elements. + + >>> np.compress([False, True], a) + array([2]) + + """ + raise NotImplemented('Waiting on interp level method') + + +def clip(a, a_min, a_max, out=None): + """ + Clip (limit) the values in an array. + + Given an interval, values outside the interval are clipped to + the interval edges. For example, if an interval of ``[0, 1]`` + is specified, values smaller than 0 become 0, and values larger + than 1 become 1. + + Parameters + ---------- + a : array_like + Array containing elements to clip. + a_min : scalar or array_like + Minimum value. + a_max : scalar or array_like + Maximum value. If `a_min` or `a_max` are array_like, then they will + be broadcasted to the shape of `a`. + out : ndarray, optional + The results will be placed in this array. It may be the input + array for in-place clipping. `out` must be of the right shape + to hold the output. Its type is preserved. + + Returns + ------- + clipped_array : ndarray + An array with the elements of `a`, but where values + < `a_min` are replaced with `a_min`, and those > `a_max` + with `a_max`. + + See Also + -------- + numpy.doc.ufuncs : Section "Output arguments" + + Examples + -------- + >>> a = np.arange(10) + >>> np.clip(a, 1, 8) + array([1, 1, 2, 3, 4, 5, 6, 7, 8, 8]) + >>> a + array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) + >>> np.clip(a, 3, 6, out=a) + array([3, 3, 3, 3, 4, 5, 6, 6, 6, 6]) + >>> a = np.arange(10) + >>> a + array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) + >>> np.clip(a, [3,4,1,1,1,4,4,4,4,4], 8) + array([3, 4, 2, 3, 4, 5, 6, 7, 8, 8]) + + """ + raise NotImplemented('Waiting on interp level method') + + +def sum(a, axis=None, dtype=None, out=None): + """ + Sum of array elements over a given axis. + + Parameters + ---------- + a : array_like + Elements to sum. + axis : integer, optional + Axis over which the sum is taken. By default `axis` is None, + and all elements are summed. + dtype : dtype, optional + The type of the returned array and of the accumulator in which + the elements are summed. By default, the dtype of `a` is used. + An exception is when `a` has an integer type with less precision + than the default platform integer. In that case, the default + platform integer is used instead. + out : ndarray, optional + Array into which the output is placed. By default, a new array is + created. If `out` is given, it must be of the appropriate shape + (the shape of `a` with `axis` removed, i.e., + ``numpy.delete(a.shape, axis)``). Its type is preserved. See + `doc.ufuncs` (Section "Output arguments") for more details. + + Returns + ------- + sum_along_axis : ndarray + An array with the same shape as `a`, with the specified + axis removed. If `a` is a 0-d array, or if `axis` is None, a scalar + is returned. If an output array is specified, a reference to + `out` is returned. + + See Also + -------- + ndarray.sum : Equivalent method. + + cumsum : Cumulative sum of array elements. + + trapz : Integration of array values using the composite trapezoidal rule. + + mean, average + + Notes + ----- + Arithmetic is modular when using integer types, and no error is + raised on overflow. + + Examples + -------- + >>> np.sum([0.5, 1.5]) + 2.0 + >>> np.sum([0.5, 0.7, 0.2, 1.5], dtype=np.int32) + 1 + >>> np.sum([[0, 1], [0, 5]]) + 6 + >>> np.sum([[0, 1], [0, 5]], axis=0) + array([0, 6]) + >>> np.sum([[0, 1], [0, 5]], axis=1) + array([1, 5]) + + If the accumulator is too small, overflow occurs: + + >>> np.ones(128, dtype=np.int8).sum(dtype=np.int8) + -128 + + """ + if not hasattr(a, "sum"): + a = numpypy.array(a) + return a.sum() + + +def product (a, axis=None, dtype=None, out=None): + """ + Return the product of array elements over a given axis. + + See Also + -------- + prod : equivalent function; see for details. + + """ + raise NotImplemented('Waiting on interp level method') + + +def sometrue(a, axis=None, out=None): + """ + Check whether some values are true. + + Refer to `any` for full documentation. + + See Also + -------- + any : equivalent function + + """ + if not hasattr(a, 'any'): + a = numpypy.array(a) + return a.any() + + +def alltrue (a, axis=None, out=None): + """ + Check if all elements of input array are true. + + See Also + -------- + numpy.all : Equivalent function; see for details. + + """ + if not hasattr(a, 'all'): + a = numpypy.array(a) + return a.all() + +def any(a,axis=None, out=None): + """ + Test whether any array element along a given axis evaluates to True. + + Returns single boolean unless `axis` is not ``None`` + + Parameters + ---------- + a : array_like + Input array or object that can be converted to an array. + axis : int, optional + Axis along which a logical OR is performed. The default + (`axis` = `None`) is to perform a logical OR over a flattened + input array. `axis` may be negative, in which case it counts + from the last to the first axis. + out : ndarray, optional + Alternate output array in which to place the result. It must have + the same shape as the expected output and its type is preserved + (e.g., if it is of type float, then it will remain so, returning + 1.0 for True and 0.0 for False, regardless of the type of `a`). + See `doc.ufuncs` (Section "Output arguments") for details. + + Returns + ------- + any : bool or ndarray + A new boolean or `ndarray` is returned unless `out` is specified, + in which case a reference to `out` is returned. + + See Also + -------- + ndarray.any : equivalent method + + all : Test whether all elements along a given axis evaluate to True. + + Notes + ----- + Not a Number (NaN), positive infinity and negative infinity evaluate + to `True` because these are not equal to zero. + + Examples + -------- + >>> np.any([[True, False], [True, True]]) + True + + >>> np.any([[True, False], [False, False]], axis=0) + array([ True, False], dtype=bool) + + >>> np.any([-1, 0, 5]) + True + + >>> np.any(np.nan) + True + + >>> o=np.array([False]) + >>> z=np.any([-1, 4, 5], out=o) + >>> z, o + (array([ True], dtype=bool), array([ True], dtype=bool)) + >>> # Check now that z is a reference to o + >>> z is o + True + >>> id(z), id(o) # identity of z and o # doctest: +SKIP + (191614240, 191614240) + + """ + if not hasattr(a, 'any'): + a = numpypy.array(a) + return a.any() + + +def all(a,axis=None, out=None): + """ + Test whether all array elements along a given axis evaluate to True. + + Parameters + ---------- + a : array_like + Input array or object that can be converted to an array. + axis : int, optional + Axis along which a logical AND is performed. + The default (`axis` = `None`) is to perform a logical AND + over a flattened input array. `axis` may be negative, in which + case it counts from the last to the first axis. + out : ndarray, optional + Alternate output array in which to place the result. + It must have the same shape as the expected output and its + type is preserved (e.g., if ``dtype(out)`` is float, the result + will consist of 0.0's and 1.0's). See `doc.ufuncs` (Section + "Output arguments") for more details. + + Returns + ------- + all : ndarray, bool + A new boolean or array is returned unless `out` is specified, + in which case a reference to `out` is returned. + + See Also + -------- + ndarray.all : equivalent method + + any : Test whether any element along a given axis evaluates to True. + + Notes + ----- + Not a Number (NaN), positive infinity and negative infinity + evaluate to `True` because these are not equal to zero. + + Examples + -------- + >>> np.all([[True,False],[True,True]]) + False + + >>> np.all([[True,False],[True,True]], axis=0) + array([ True, False], dtype=bool) + + >>> np.all([-1, 4, 5]) + True + + >>> np.all([1.0, np.nan]) + True + + >>> o=np.array([False]) + >>> z=np.all([-1, 4, 5], out=o) + >>> id(z), id(o), z # doctest: +SKIP + (28293632, 28293632, array([ True], dtype=bool)) + + """ + if not hasattr(a, 'all'): + a = numpypy.array(a) + return a.all() + + +def cumsum (a, axis=None, dtype=None, out=None): + """ + Return the cumulative sum of the elements along a given axis. + + Parameters + ---------- + a : array_like + Input array. + axis : int, optional + Axis along which the cumulative sum is computed. The default + (None) is to compute the cumsum over the flattened array. + dtype : dtype, optional + Type of the returned array and of the accumulator in which the + elements are summed. If `dtype` is not specified, it defaults + to the dtype of `a`, unless `a` has an integer dtype with a + precision less than that of the default platform integer. In + that case, the default platform integer is used. + out : ndarray, optional + Alternative output array in which to place the result. It must + have the same shape and buffer length as the expected output + but the type will be cast if necessary. See `doc.ufuncs` + (Section "Output arguments") for more details. + + Returns + ------- + cumsum_along_axis : ndarray. + A new array holding the result is returned unless `out` is + specified, in which case a reference to `out` is returned. The + result has the same size as `a`, and the same shape as `a` if + `axis` is not None or `a` is a 1-d array. + + + See Also + -------- + sum : Sum array elements. + + trapz : Integration of array values using the composite trapezoidal rule. + + Notes + ----- + Arithmetic is modular when using integer types, and no error is + raised on overflow. + + Examples + -------- + >>> a = np.array([[1,2,3], [4,5,6]]) + >>> a + array([[1, 2, 3], + [4, 5, 6]]) + >>> np.cumsum(a) + array([ 1, 3, 6, 10, 15, 21]) + >>> np.cumsum(a, dtype=float) # specifies type of output value(s) + array([ 1., 3., 6., 10., 15., 21.]) + + >>> np.cumsum(a,axis=0) # sum over rows for each of the 3 columns + array([[1, 2, 3], + [5, 7, 9]]) + >>> np.cumsum(a,axis=1) # sum over columns for each of the 2 rows + array([[ 1, 3, 6], + [ 4, 9, 15]]) + + """ + raise NotImplemented('Waiting on interp level method') + + +def cumproduct(a, axis=None, dtype=None, out=None): + """ + Return the cumulative product over the given axis. + + + See Also + -------- + cumprod : equivalent function; see for details. + + """ + raise NotImplemented('Waiting on interp level method') + + +def ptp(a, axis=None, out=None): + """ + Range of values (maximum - minimum) along an axis. + + The name of the function comes from the acronym for 'peak to peak'. + + Parameters + ---------- + a : array_like + Input values. + axis : int, optional + Axis along which to find the peaks. By default, flatten the + array. + out : array_like + Alternative output array in which to place the result. It must + have the same shape and buffer length as the expected output, + but the type of the output values will be cast if necessary. + + Returns + ------- + ptp : ndarray + A new array holding the result, unless `out` was + specified, in which case a reference to `out` is returned. + + Examples + -------- + >>> x = np.arange(4).reshape((2,2)) + >>> x + array([[0, 1], + [2, 3]]) + + >>> np.ptp(x, axis=0) + array([2, 2]) + + >>> np.ptp(x, axis=1) + array([1, 1]) + + """ + raise NotImplemented('Waiting on interp level method') + + +def amax(a, axis=None, out=None): + """ + Return the maximum of an array or maximum along an axis. + + Parameters + ---------- + a : array_like + Input data. + axis : int, optional + Axis along which to operate. By default flattened input is used. + out : ndarray, optional + Alternate output array in which to place the result. Must be of + the same shape and buffer length as the expected output. See + `doc.ufuncs` (Section "Output arguments") for more details. + + Returns + ------- + amax : ndarray or scalar + Maximum of `a`. If `axis` is None, the result is a scalar value. + If `axis` is given, the result is an array of dimension + ``a.ndim - 1``. + + See Also + -------- + nanmax : NaN values are ignored instead of being propagated. + fmax : same behavior as the C99 fmax function. + argmax : indices of the maximum values. + + Notes + ----- + NaN values are propagated, that is if at least one item is NaN, the + corresponding max value will be NaN as well. To ignore NaN values + (MATLAB behavior), please use nanmax. + + Examples + -------- + >>> a = np.arange(4).reshape((2,2)) + >>> a + array([[0, 1], + [2, 3]]) + >>> np.amax(a) + 3 + >>> np.amax(a, axis=0) + array([2, 3]) + >>> np.amax(a, axis=1) + array([1, 3]) + + >>> b = np.arange(5, dtype=np.float) + >>> b[2] = np.NaN + >>> np.amax(b) + nan + >>> np.nanmax(b) + 4.0 + + """ + if not hasattr(a, "max"): + a = numpypy.array(a) + return a.max() + + +def amin(a, axis=None, out=None): + """ + Return the minimum of an array or minimum along an axis. + + Parameters + ---------- + a : array_like + Input data. + axis : int, optional + Axis along which to operate. By default a flattened input is used. + out : ndarray, optional + Alternative output array in which to place the result. Must + be of the same shape and buffer length as the expected output. + See `doc.ufuncs` (Section "Output arguments") for more details. + + Returns + ------- + amin : ndarray + A new array or a scalar array with the result. + + See Also + -------- + nanmin: nan values are ignored instead of being propagated + fmin: same behavior as the C99 fmin function + argmin: Return the indices of the minimum values. + + amax, nanmax, fmax + + Notes + ----- + NaN values are propagated, that is if at least one item is nan, the + corresponding min value will be nan as well. To ignore NaN values (matlab + behavior), please use nanmin. + + Examples + -------- + >>> a = np.arange(4).reshape((2,2)) + >>> a + array([[0, 1], + [2, 3]]) + >>> np.amin(a) # Minimum of the flattened array + 0 + >>> np.amin(a, axis=0) # Minima along the first axis + array([0, 1]) + >>> np.amin(a, axis=1) # Minima along the second axis + array([0, 2]) + + >>> b = np.arange(5, dtype=np.float) + >>> b[2] = np.NaN + >>> np.amin(b) + nan + >>> np.nanmin(b) + 0.0 + + """ + # amin() is equivalent to min() + if not hasattr(a, 'min'): + a = numpypy.array(a) + return a.min() + +def alen(a): + """ + Return the length of the first dimension of the input array. + + Parameters + ---------- + a : array_like + Input array. + + Returns + ------- + l : int + Length of the first dimension of `a`. + + See Also + -------- + shape, size + + Examples + -------- + >>> a = np.zeros((7,4,5)) + >>> a.shape[0] + 7 + >>> np.alen(a) + 7 + + """ + if not hasattr(a, 'shape'): + a = numpypy.array(a) + return a.shape[0] + + +def prod(a, axis=None, dtype=None, out=None): + """ + Return the product of array elements over a given axis. + + Parameters + ---------- + a : array_like + Input data. + axis : int, optional + Axis over which the product is taken. By default, the product + of all elements is calculated. + dtype : data-type, optional + The data-type of the returned array, as well as of the accumulator + in which the elements are multiplied. By default, if `a` is of + integer type, `dtype` is the default platform integer. (Note: if + the type of `a` is unsigned, then so is `dtype`.) Otherwise, + the dtype is the same as that of `a`. + out : ndarray, optional + Alternative output array in which to place the result. It must have + the same shape as the expected output, but the type of the + output values will be cast if necessary. + + Returns + ------- + product_along_axis : ndarray, see `dtype` parameter above. + An array shaped as `a` but with the specified axis removed. + Returns a reference to `out` if specified. + + See Also + -------- + ndarray.prod : equivalent method + numpy.doc.ufuncs : Section "Output arguments" + + Notes + ----- + Arithmetic is modular when using integer types, and no error is + raised on overflow. That means that, on a 32-bit platform: + + >>> x = np.array([536870910, 536870910, 536870910, 536870910]) + >>> np.prod(x) #random + 16 + + Examples + -------- + By default, calculate the product of all elements: + + >>> np.prod([1.,2.]) + 2.0 + + Even when the input array is two-dimensional: + + >>> np.prod([[1.,2.],[3.,4.]]) + 24.0 + + But we can also specify the axis over which to multiply: + + >>> np.prod([[1.,2.],[3.,4.]], axis=1) + array([ 2., 12.]) + + If the type of `x` is unsigned, then the output type is + the unsigned platform integer: + + >>> x = np.array([1, 2, 3], dtype=np.uint8) + >>> np.prod(x).dtype == np.uint + True + + If `x` is of a signed integer type, then the output type + is the default platform integer: + + >>> x = np.array([1, 2, 3], dtype=np.int8) + >>> np.prod(x).dtype == np.int + True + + """ + raise NotImplemented('Waiting on interp level method') + + +def cumprod(a, axis=None, dtype=None, out=None): + """ + Return the cumulative product of elements along a given axis. + + Parameters + ---------- + a : array_like + Input array. + axis : int, optional + Axis along which the cumulative product is computed. By default + the input is flattened. + dtype : dtype, optional + Type of the returned array, as well as of the accumulator in which + the elements are multiplied. If *dtype* is not specified, it + defaults to the dtype of `a`, unless `a` has an integer dtype with + a precision less than that of the default platform integer. In + that case, the default platform integer is used instead. + out : ndarray, optional + Alternative output array in which to place the result. It must + have the same shape and buffer length as the expected output + but the type of the resulting values will be cast if necessary. + + Returns + ------- + cumprod : ndarray + A new array holding the result is returned unless `out` is + specified, in which case a reference to out is returned. + + See Also + -------- + numpy.doc.ufuncs : Section "Output arguments" + + Notes + ----- + Arithmetic is modular when using integer types, and no error is + raised on overflow. + + Examples + -------- + >>> a = np.array([1,2,3]) + >>> np.cumprod(a) # intermediate results 1, 1*2 + ... # total product 1*2*3 = 6 + array([1, 2, 6]) + >>> a = np.array([[1, 2, 3], [4, 5, 6]]) + >>> np.cumprod(a, dtype=float) # specify type of output + array([ 1., 2., 6., 24., 120., 720.]) + + The cumulative product for each column (i.e., over the rows) of `a`: + + >>> np.cumprod(a, axis=0) + array([[ 1, 2, 3], + [ 4, 10, 18]]) + + The cumulative product for each row (i.e. over the columns) of `a`: + + >>> np.cumprod(a,axis=1) + array([[ 1, 2, 6], + [ 4, 20, 120]]) + + """ + raise NotImplemented('Waiting on interp level method') + + +def ndim(a): + """ + Return the number of dimensions of an array. + + Parameters + ---------- + a : array_like + Input array. If it is not already an ndarray, a conversion is + attempted. + + Returns + ------- + number_of_dimensions : int + The number of dimensions in `a`. Scalars are zero-dimensional. + + See Also + -------- + ndarray.ndim : equivalent method + shape : dimensions of array + ndarray.shape : dimensions of array + + Examples + -------- + >>> np.ndim([[1,2,3],[4,5,6]]) + 2 + >>> np.ndim(np.array([[1,2,3],[4,5,6]])) + 2 + >>> np.ndim(1) + 0 + + """ + if not hasattr(a, 'ndim'): + a = numpypy.array(a) + return a.ndim + + +def rank(a): + """ + Return the number of dimensions of an array. + + If `a` is not already an array, a conversion is attempted. + Scalars are zero dimensional. + + Parameters + ---------- + a : array_like + Array whose number of dimensions is desired. If `a` is not an array, + a conversion is attempted. + + Returns + ------- + number_of_dimensions : int + The number of dimensions in the array. + + See Also + -------- + ndim : equivalent function + ndarray.ndim : equivalent property + shape : dimensions of array + ndarray.shape : dimensions of array + + Notes + ----- + In the old Numeric package, `rank` was the term used for the number of + dimensions, but in Numpy `ndim` is used instead. + + Examples + -------- + >>> np.rank([1,2,3]) + 1 + >>> np.rank(np.array([[1,2,3],[4,5,6]])) + 2 + >>> np.rank(1) + 0 + + """ + if not hasattr(a, 'ndim'): + a = numpypy.array(a) + return a.ndim + + +def size(a, axis=None): + """ + Return the number of elements along a given axis. + + Parameters + ---------- + a : array_like + Input data. + axis : int, optional + Axis along which the elements are counted. By default, give + the total number of elements. + + Returns + ------- + element_count : int + Number of elements along the specified axis. + + See Also + -------- + shape : dimensions of array + ndarray.shape : dimensions of array + ndarray.size : number of elements in array + + Examples + -------- + >>> a = np.array([[1,2,3],[4,5,6]]) + >>> np.size(a) + 6 + >>> np.size(a,1) + 3 + >>> np.size(a,0) + 2 + + """ + raise NotImplemented('Waiting on interp level method') + + +def around(a, decimals=0, out=None): + """ + Evenly round to the given number of decimals. + + Parameters + ---------- + a : array_like + Input data. + decimals : int, optional + Number of decimal places to round to (default: 0). If + decimals is negative, it specifies the number of positions to + the left of the decimal point. + out : ndarray, optional + Alternative output array in which to place the result. It must have + the same shape as the expected output, but the type of the output + values will be cast if necessary. See `doc.ufuncs` (Section + "Output arguments") for details. + + Returns + ------- + rounded_array : ndarray + An array of the same type as `a`, containing the rounded values. + Unless `out` was specified, a new array is created. A reference to + the result is returned. + + The real and imaginary parts of complex numbers are rounded + separately. The result of rounding a float is a float. + + See Also + -------- + ndarray.round : equivalent method + + ceil, fix, floor, rint, trunc + + + Notes + ----- + For values exactly halfway between rounded decimal values, Numpy + rounds to the nearest even value. Thus 1.5 and 2.5 round to 2.0, + -0.5 and 0.5 round to 0.0, etc. Results may also be surprising due + to the inexact representation of decimal fractions in the IEEE + floating point standard [1]_ and errors introduced when scaling + by powers of ten. + + References + ---------- + .. [1] "Lecture Notes on the Status of IEEE 754", William Kahan, + http://www.cs.berkeley.edu/~wkahan/ieee754status/IEEE754.PDF + .. [2] "How Futile are Mindless Assessments of + Roundoff in Floating-Point Computation?", William Kahan, + http://www.cs.berkeley.edu/~wkahan/Mindless.pdf + + Examples + -------- + >>> np.around([0.37, 1.64]) + array([ 0., 2.]) + >>> np.around([0.37, 1.64], decimals=1) + array([ 0.4, 1.6]) + >>> np.around([.5, 1.5, 2.5, 3.5, 4.5]) # rounds to nearest even value + array([ 0., 2., 2., 4., 4.]) + >>> np.around([1,2,3,11], decimals=1) # ndarray of ints is returned + array([ 1, 2, 3, 11]) + >>> np.around([1,2,3,11], decimals=-1) + array([ 0, 0, 0, 10]) + + """ + raise NotImplemented('Waiting on interp level method') + + +def round_(a, decimals=0, out=None): + """ + Round an array to the given number of decimals. + + Refer to `around` for full documentation. + + See Also + -------- + around : equivalent function + + """ + raise NotImplemented('Waiting on interp level method') + + +def mean(a, axis=None, dtype=None, out=None): + """ + Compute the arithmetic mean along the specified axis. + + Returns the average of the array elements. The average is taken over + the flattened array by default, otherwise over the specified axis. + `float64` intermediate and return values are used for integer inputs. + + Parameters + ---------- + a : array_like + Array containing numbers whose mean is desired. If `a` is not an + array, a conversion is attempted. + axis : int, optional + Axis along which the means are computed. The default is to compute + the mean of the flattened array. + dtype : data-type, optional + Type to use in computing the mean. For integer inputs, the default + is `float64`; for floating point inputs, it is the same as the + input dtype. + out : ndarray, optional + Alternate output array in which to place the result. The default + is ``None``; if provided, it must have the same shape as the + expected output, but the type will be cast if necessary. + See `doc.ufuncs` for details. + + Returns + ------- + m : ndarray, see dtype parameter above + If `out=None`, returns a new array containing the mean values, + otherwise a reference to the output array is returned. + + See Also + -------- + average : Weighted average + + Notes + ----- + The arithmetic mean is the sum of the elements along the axis divided + by the number of elements. + + Note that for floating-point input, the mean is computed using the + same precision the input has. Depending on the input data, this can + cause the results to be inaccurate, especially for `float32` (see + example below). Specifying a higher-precision accumulator using the + `dtype` keyword can alleviate this issue. + + Examples + -------- + >>> a = np.array([[1, 2], [3, 4]]) + >>> np.mean(a) + 2.5 + >>> np.mean(a, axis=0) + array([ 2., 3.]) + >>> np.mean(a, axis=1) + array([ 1.5, 3.5]) + + In single precision, `mean` can be inaccurate: + + >>> a = np.zeros((2, 512*512), dtype=np.float32) + >>> a[0, :] = 1.0 + >>> a[1, :] = 0.1 + >>> np.mean(a) + 0.546875 + + Computing the mean in float64 is more accurate: + + >>> np.mean(a, dtype=np.float64) + 0.55000000074505806 + + """ + if not hasattr(a, "mean"): + a = numpypy.array(a) + return a.mean() + + +def std(a, axis=None, dtype=None, out=None, ddof=0): + """ + Compute the standard deviation along the specified axis. + + Returns the standard deviation, a measure of the spread of a distribution, + of the array elements. The standard deviation is computed for the + flattened array by default, otherwise over the specified axis. + + Parameters + ---------- + a : array_like + Calculate the standard deviation of these values. + axis : int, optional + Axis along which the standard deviation is computed. The default is + to compute the standard deviation of the flattened array. + dtype : dtype, optional + Type to use in computing the standard deviation. For arrays of + integer type the default is float64, for arrays of float types it is + the same as the array type. + out : ndarray, optional + Alternative output array in which to place the result. It must have + the same shape as the expected output but the type (of the calculated + values) will be cast if necessary. + ddof : int, optional + Means Delta Degrees of Freedom. The divisor used in calculations + is ``N - ddof``, where ``N`` represents the number of elements. + By default `ddof` is zero. + + Returns + ------- + standard_deviation : ndarray, see dtype parameter above. + If `out` is None, return a new array containing the standard deviation, + otherwise return a reference to the output array. + + See Also + -------- + var, mean + numpy.doc.ufuncs : Section "Output arguments" + + Notes + ----- + The standard deviation is the square root of the average of the squared + deviations from the mean, i.e., ``std = sqrt(mean(abs(x - x.mean())**2))``. + + The average squared deviation is normally calculated as ``x.sum() / N``, where + ``N = len(x)``. If, however, `ddof` is specified, the divisor ``N - ddof`` + is used instead. In standard statistical practice, ``ddof=1`` provides an + unbiased estimator of the variance of the infinite population. ``ddof=0`` + provides a maximum likelihood estimate of the variance for normally + distributed variables. The standard deviation computed in this function + is the square root of the estimated variance, so even with ``ddof=1``, it + will not be an unbiased estimate of the standard deviation per se. + + Note that, for complex numbers, `std` takes the absolute + value before squaring, so that the result is always real and nonnegative. + + For floating-point input, the *std* is computed using the same + precision the input has. Depending on the input data, this can cause + the results to be inaccurate, especially for float32 (see example below). + Specifying a higher-accuracy accumulator using the `dtype` keyword can + alleviate this issue. + + Examples + -------- + >>> a = np.array([[1, 2], [3, 4]]) + >>> np.std(a) + 1.1180339887498949 + >>> np.std(a, axis=0) + array([ 1., 1.]) + >>> np.std(a, axis=1) + array([ 0.5, 0.5]) + + In single precision, std() can be inaccurate: + + >>> a = np.zeros((2,512*512), dtype=np.float32) + >>> a[0,:] = 1.0 + >>> a[1,:] = 0.1 + >>> np.std(a) + 0.45172946707416706 + + Computing the standard deviation in float64 is more accurate: + + >>> np.std(a, dtype=np.float64) + 0.44999999925552653 + + """ + if not hasattr(a, "std"): + a = numpypy.array(a) + return a.std() + + +def var(a, axis=None, dtype=None, out=None, ddof=0): + """ + Compute the variance along the specified axis. + + Returns the variance of the array elements, a measure of the spread of a + distribution. The variance is computed for the flattened array by + default, otherwise over the specified axis. + + Parameters + ---------- + a : array_like + Array containing numbers whose variance is desired. If `a` is not an + array, a conversion is attempted. + axis : int, optional + Axis along which the variance is computed. The default is to compute + the variance of the flattened array. + dtype : data-type, optional + Type to use in computing the variance. For arrays of integer type + the default is `float32`; for arrays of float types it is the same as + the array type. + out : ndarray, optional + Alternate output array in which to place the result. It must have + the same shape as the expected output, but the type is cast if + necessary. + ddof : int, optional + "Delta Degrees of Freedom": the divisor used in the calculation is + ``N - ddof``, where ``N`` represents the number of elements. By + default `ddof` is zero. + + Returns + ------- + variance : ndarray, see dtype parameter above + If ``out=None``, returns a new array containing the variance; + otherwise, a reference to the output array is returned. + + See Also + -------- + std : Standard deviation + mean : Average + numpy.doc.ufuncs : Section "Output arguments" + + Notes + ----- + The variance is the average of the squared deviations from the mean, + i.e., ``var = mean(abs(x - x.mean())**2)``. + + The mean is normally calculated as ``x.sum() / N``, where ``N = len(x)``. + If, however, `ddof` is specified, the divisor ``N - ddof`` is used + instead. In standard statistical practice, ``ddof=1`` provides an + unbiased estimator of the variance of a hypothetical infinite population. + ``ddof=0`` provides a maximum likelihood estimate of the variance for + normally distributed variables. + + Note that for complex numbers, the absolute value is taken before + squaring, so that the result is always real and nonnegative. + + For floating-point input, the variance is computed using the same + precision the input has. Depending on the input data, this can cause + the results to be inaccurate, especially for `float32` (see example + below). Specifying a higher-accuracy accumulator using the ``dtype`` + keyword can alleviate this issue. + + Examples + -------- + >>> a = np.array([[1,2],[3,4]]) + >>> np.var(a) + 1.25 + >>> np.var(a,0) + array([ 1., 1.]) + >>> np.var(a,1) + array([ 0.25, 0.25]) + + In single precision, var() can be inaccurate: + + >>> a = np.zeros((2,512*512), dtype=np.float32) + >>> a[0,:] = 1.0 + >>> a[1,:] = 0.1 + >>> np.var(a) + 0.20405951142311096 + + Computing the standard deviation in float64 is more accurate: + + >>> np.var(a, dtype=np.float64) + 0.20249999932997387 + >>> ((1-0.55)**2 + (0.1-0.55)**2)/2 + 0.20250000000000001 + + """ + if not hasattr(a, "var"): + a = numpypy.array(a) + return a.var() diff --git a/lib_pypy/numpypy/test/test_fromnumeric.py b/lib_pypy/numpypy/test/test_fromnumeric.py new file mode 100644 --- /dev/null +++ b/lib_pypy/numpypy/test/test_fromnumeric.py @@ -0,0 +1,109 @@ + +from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest + +class AppTestFromNumeric(BaseNumpyAppTest): + def test_argmax(self): + # tests taken from numpy/core/fromnumeric.py docstring + from numpypy import array, arange, argmax + a = arange(6).reshape((2,3)) + assert argmax(a) == 5 + # assert (argmax(a, axis=0) == array([1, 1, 1])).all() + # assert (argmax(a, axis=1) == array([2, 2])).all() + b = arange(6) + b[1] = 5 + assert argmax(b) == 1 + + def test_argmin(self): + # tests adapted from test_argmax + from numpypy import array, arange, argmin + a = arange(6).reshape((2,3)) + assert argmin(a) == 0 + # assert (argmax(a, axis=0) == array([0, 0, 0])).all() + # assert (argmax(a, axis=1) == array([0, 0])).all() + b = arange(6) + b[1] = 0 + assert argmin(b) == 0 + + def test_shape(self): + # tests taken from numpy/core/fromnumeric.py docstring + from numpypy import array, identity, shape + assert shape(identity(3)) == (3, 3) + assert shape([[1, 2]]) == (1, 2) + assert shape([0]) == (1,) + assert shape(0) == () + # a = array([(1, 2), (3, 4)], dtype=[('x', 'i4'), ('y', 'i4')]) + # assert shape(a) == (2,) + + def test_sum(self): + # tests taken from numpy/core/fromnumeric.py docstring + from numpypy import array, sum, ones + assert sum([0.5, 1.5])== 2.0 + assert sum([[0, 1], [0, 5]]) == 6 + # assert sum([0.5, 0.7, 0.2, 1.5], dtype=int32) == 1 + # assert (sum([[0, 1], [0, 5]], axis=0) == array([0, 6])).all() + # assert (sum([[0, 1], [0, 5]], axis=1) == array([1, 5])).all() + # If the accumulator is too small, overflow occurs: + # assert ones(128, dtype=int8).sum(dtype=int8) == -128 + + def test_amin(self): + # tests taken from numpy/core/fromnumeric.py docstring + from numpypy import array, arange, amin + a = arange(4).reshape((2,2)) + assert amin(a) == 0 + # # Minima along the first axis + # assert (amin(a, axis=0) == array([0, 1])).all() + # # Minima along the second axis + # assert (amin(a, axis=1) == array([0, 2])).all() + # # NaN behaviour + # b = arange(5, dtype=float) + # b[2] = NaN + # assert amin(b) == nan + # assert nanmin(b) == 0.0 + + def test_amax(self): + # tests taken from numpy/core/fromnumeric.py docstring + from numpypy import array, arange, amax + a = arange(4).reshape((2,2)) + assert amax(a) == 3 + # assert (amax(a, axis=0) == array([2, 3])).all() + # assert (amax(a, axis=1) == array([1, 3])).all() + # # NaN behaviour + # b = arange(5, dtype=float) + # b[2] = NaN + # assert amax(b) == nan + # assert nanmax(b) == 4.0 + + def test_alen(self): + # tests taken from numpy/core/fromnumeric.py docstring + from numpypy import array, zeros, alen + a = zeros((7,4,5)) + assert a.shape[0] == 7 + assert alen(a) == 7 + + def test_ndim(self): + # tests taken from numpy/core/fromnumeric.py docstring + from numpypy import array, ndim + assert ndim([[1,2,3],[4,5,6]]) == 2 + assert ndim(array([[1,2,3],[4,5,6]])) == 2 + assert ndim(1) == 0 + + def test_rank(self): + # tests taken from numpy/core/fromnumeric.py docstring + from numpypy import array, rank + assert rank([[1,2,3],[4,5,6]]) == 2 + assert rank(array([[1,2,3],[4,5,6]])) == 2 + assert rank(1) == 0 + + def test_var(self): + from numpypy import array, var + a = array([[1,2],[3,4]]) + assert var(a) == 1.25 + # assert (np.var(a,0) == array([ 1., 1.])).all() + # assert (np.var(a,1) == array([ 0.25, 0.25])).all() + + def test_std(self): + from numpypy import array, std + a = array([[1, 2], [3, 4]]) + assert std(a) == 1.1180339887498949 + # assert (std(a, axis=0) == array([ 1., 1.])).all() + # assert (std(a, axis=1) == array([ 0.5, 0.5]).all() diff --git a/pypy/annotation/description.py b/pypy/annotation/description.py --- a/pypy/annotation/description.py +++ b/pypy/annotation/description.py @@ -180,7 +180,12 @@ if name is None: name = pyobj.func_name if signature is None: - signature = cpython_code_signature(pyobj.func_code) + if hasattr(pyobj, '_generator_next_method_of_'): + from pypy.interpreter.argument import Signature + signature = Signature(['entry']) # haaaaaack + defaults = () + else: + signature = cpython_code_signature(pyobj.func_code) if defaults is None: defaults = pyobj.func_defaults self.name = name @@ -252,7 +257,8 @@ try: inputcells = args.match_signature(signature, defs_s) except ArgErr, e: - raise TypeError, "signature mismatch: %s" % e.getmsg(self.name) + raise TypeError("signature mismatch: %s() %s" % + (self.name, e.getmsg())) return inputcells def specialize(self, inputcells, op=None): diff --git a/pypy/doc/Makefile b/pypy/doc/Makefile --- a/pypy/doc/Makefile +++ b/pypy/doc/Makefile @@ -12,7 +12,7 @@ PAPEROPT_letter = -D latex_paper_size=letter ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . -.PHONY: help clean html dirhtml pickle json htmlhelp qthelp latex changes linkcheck doctest +.PHONY: help clean html dirhtml pickle json htmlhelp qthelp latex man changes linkcheck doctest help: @echo "Please use \`make ' where is one of" @@ -23,6 +23,7 @@ @echo " htmlhelp to make HTML files and a HTML help project" @echo " qthelp to make HTML files and a qthelp project" @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " man to make manual pages" @echo " changes to make an overview of all changed/added/deprecated items" @echo " linkcheck to check all external links for integrity" @echo " doctest to run all doctests embedded in the documentation (if enabled)" @@ -79,6 +80,11 @@ @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \ "run these through (pdf)latex." +man: + $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man + @echo + @echo "Build finished. The manual pages are in $(BUILDDIR)/man" + changes: python config/generate.py $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes diff --git a/pypy/doc/coding-guide.rst b/pypy/doc/coding-guide.rst --- a/pypy/doc/coding-guide.rst +++ b/pypy/doc/coding-guide.rst @@ -175,15 +175,15 @@ RPython ================= -RPython Definition, not ------------------------ +RPython Definition +------------------ -The list and exact details of the "RPython" restrictions are a somewhat -evolving topic. In particular, we have no formal language definition -as we find it more practical to discuss and evolve the set of -restrictions while working on the whole program analysis. If you -have any questions about the restrictions below then please feel -free to mail us at pypy-dev at codespeak net. +RPython is a restricted subset of Python that is amenable to static analysis. +Although there are additions to the language and some things might surprisingly +work, this is a rough list of restrictions that should be considered. Note +that there are tons of special cased restrictions that you'll encounter +as you go. The exact definition is "RPython is everything that our translation +toolchain can accept" :) .. _`wrapped object`: coding-guide.html#wrapping-rules @@ -198,7 +198,7 @@ contain both a string and a int must be avoided. It is allowed to mix None (basically with the role of a null pointer) with many other types: `wrapped objects`, class instances, lists, dicts, strings, etc. - but *not* with int and floats. + but *not* with int, floats or tuples. **constants** @@ -209,9 +209,12 @@ have this restriction, so if you need mutable global state, store it in the attributes of some prebuilt singleton instance. + + **control structures** - all allowed but yield, ``for`` loops restricted to builtin types + all allowed, ``for`` loops restricted to builtin types, generators + very restricted. **range** @@ -226,7 +229,8 @@ **generators** - generators are not supported. + generators are supported, but their exact scope is very limited. you can't + merge two different generator in one control point. **exceptions** @@ -245,22 +249,27 @@ **strings** - a lot of, but not all string methods are supported. Indexes can be + a lot of, but not all string methods are supported and those that are + supported, not necesarilly accept all arguments. Indexes can be negative. In case they are not, then you get slightly more efficient code if the translator can prove that they are non-negative. When slicing a string it is necessary to prove that the slice start and - stop indexes are non-negative. + stop indexes are non-negative. There is no implicit str-to-unicode cast + anywhere. **tuples** no variable-length tuples; use them to store or return pairs or n-tuples of - values. Each combination of types for elements and length constitute a separate - and not mixable type. + values. Each combination of types for elements and length constitute + a separate and not mixable type. **lists** lists are used as an allocated array. Lists are over-allocated, so list.append() - is reasonably fast. Negative or out-of-bound indexes are only allowed for the + is reasonably fast. However, if you use a fixed-size list, the code + is more efficient. Annotator can figure out most of the time that your + list is fixed-size, even when you use list comprehension. + Negative or out-of-bound indexes are only allowed for the most common operations, as follows: - *indexing*: @@ -287,16 +296,14 @@ **dicts** - dicts with a unique key type only, provided it is hashable. - String keys have been the only allowed key types for a while, but this was generalized. - After some re-optimization, - the implementation could safely decide that all string dict keys should be interned. + dicts with a unique key type only, provided it is hashable. Custom + hash functions and custom equality will not be honored. + Use ``pypy.rlib.objectmodel.r_dict`` for custom hash functions. **list comprehensions** - may be used to create allocated, initialized arrays. - After list over-allocation was introduced, there is no longer any restriction. + May be used to create allocated, initialized arrays. **functions** @@ -334,9 +341,8 @@ **objects** - in PyPy, wrapped objects are borrowed from the object space. Just like - in CPython, code that needs e.g. a dictionary can use a wrapped dict - and the object space operations on it. + Normal rules apply. Special methods are not honoured, except ``__init__`` and + ``__del__``. This layout makes the number of types to take care about quite limited. diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py --- a/pypy/doc/conf.py +++ b/pypy/doc/conf.py @@ -197,3 +197,10 @@ # Example configuration for intersphinx: refer to the Python standard library. intersphinx_mapping = {'http://docs.python.org/': None} +# -- Options for manpage output------------------------------------------------- + +man_pages = [ + ('man/pypy.1', 'pypy', + u'fast, compliant alternative implementation of the Python language', + u'The PyPy Project', 1) +] diff --git a/pypy/doc/extradoc.rst b/pypy/doc/extradoc.rst --- a/pypy/doc/extradoc.rst +++ b/pypy/doc/extradoc.rst @@ -8,6 +8,9 @@ *Articles about PyPy published so far, most recent first:* (bibtex_ file) +* `Runtime Feedback in a Meta-Tracing JIT for Efficient Dynamic Languages`_, + C.F. Bolz, A. Cuni, M. Fijalkowski, M. Leuschel, S. Pedroni, A. Rigo + * `Allocation Removal by Partial Evaluation in a Tracing JIT`_, C.F. Bolz, A. Cuni, M. Fijalkowski, M. Leuschel, S. Pedroni, A. Rigo @@ -50,6 +53,9 @@ *Other research using PyPy (as far as we know it):* +* `Hardware Transactional Memory Support for Lightweight Dynamic Language Evolution`_, + N. Riley and C. Zilles + * `PyGirl: Generating Whole-System VMs from High-Level Prototypes using PyPy`_, C. Bruni and T. Verwaest @@ -65,6 +71,7 @@ .. _bibtex: https://bitbucket.org/pypy/extradoc/raw/tip/talk/bibtex.bib +.. _`Runtime Feedback in a Meta-Tracing JIT for Efficient Dynamic Languages`: https://bitbucket.org/pypy/extradoc/raw/extradoc/talk/icooolps2011/jit-hints.pdf .. _`Allocation Removal by Partial Evaluation in a Tracing JIT`: http://codespeak.net/svn/pypy/extradoc/talk/pepm2011/bolz-allocation-removal.pdf .. _`Towards a Jitting VM for Prolog Execution`: http://www.stups.uni-duesseldorf.de/publications/bolz-prolog-jit.pdf .. _`High performance implementation of Python for CLI/.NET with JIT compiler generation for dynamic languages`: http://buildbot.pypy.org/misc/antocuni-thesis.pdf @@ -74,6 +81,7 @@ .. _`Automatic JIT Compiler Generation with Runtime Partial Evaluation`: http://www.stups.uni-duesseldorf.de/thesis/final-master.pdf .. _`RPython: A Step towards Reconciling Dynamically and Statically Typed OO Languages`: http://www.disi.unige.it/person/AnconaD/papers/Recent_abstracts.html#AACM-DLS07 .. _`EU Reports`: index-report.html +.. _`Hardware Transactional Memory Support for Lightweight Dynamic Language Evolution`: http://sabi.net/nriley/pubs/dls6-riley.pdf .. _`PyGirl: Generating Whole-System VMs from High-Level Prototypes using PyPy`: http://scg.unibe.ch/archive/papers/Brun09cPyGirl.pdf .. _`Representation-Based Just-in-Time Specialization and the Psyco Prototype for Python`: http://psyco.sourceforge.net/psyco-pepm-a.ps.gz .. _`Back to the Future in One Week -- Implementing a Smalltalk VM in PyPy`: http://dx.doi.org/10.1007/978-3-540-89275-5_7 diff --git a/pypy/doc/man/pypy.1.rst b/pypy/doc/man/pypy.1.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/man/pypy.1.rst @@ -0,0 +1,90 @@ +====== + pypy +====== + +SYNOPSIS +======== + +``pypy`` [*options*] +[``-c`` *cmd*\ \|\ ``-m`` *mod*\ \|\ *file.py*\ \|\ ``-``\ ] +[*arg*\ ...] + +OPTIONS +======= + +-i + Inspect interactively after running script. + +-O + Dummy optimization flag for compatibility with C Python. + +-c *cmd* + Program passed in as CMD (terminates option list). + +-S + Do not ``import site`` on initialization. + +-u + Unbuffered binary ``stdout`` and ``stderr``. + +-h, --help + Show a help message and exit. + +-m *mod* + Library module to be run as a script (terminates option list). + +-W *arg* + Warning control (*arg* is *action*:*message*:*category*:*module*:*lineno*). + +-E + Ignore environment variables (such as ``PYTHONPATH``). + +--version + Print the PyPy version. + +--info + Print translation information about this PyPy executable. + +--jit *arg* + Low level JIT parameters. Format is + *arg*\ ``=``\ *value*\ [``,``\ *arg*\ ``=``\ *value*\ ...] + + ``off`` + Disable the JIT. + + ``threshold=``\ *value* + Number of times a loop has to run for it to become hot. + + ``function_threshold=``\ *value* + Number of times a function must run for it to become traced from + start. + + ``inlining=``\ *value* + Inline python functions or not (``1``/``0``). + + ``loop_longevity=``\ *value* + A parameter controlling how long loops will be kept before being + freed, an estimate. + + ``max_retrace_guards=``\ *value* + Number of extra guards a retrace can cause. + + ``retrace_limit=``\ *value* + How many times we can try retracing before giving up. + + ``trace_eagerness=``\ *value* + Number of times a guard has to fail before we start compiling a + bridge. + + ``trace_limit=``\ *value* + Number of recorded operations before we abort tracing with + ``ABORT_TRACE_TOO_LONG``. + + ``enable_opts=``\ *value* + Optimizations to enabled or ``all``. + Warning, this option is dangerous, and should be avoided. + +SEE ALSO +======== + +**python**\ (1) diff --git a/pypy/doc/tool/makecontributor.py b/pypy/doc/tool/makecontributor.py deleted file mode 100644 --- a/pypy/doc/tool/makecontributor.py +++ /dev/null @@ -1,47 +0,0 @@ -""" - -generates a contributor list - -""" -import py - -# this file is useless, use the following commandline instead: -# hg churn -c -t "{author}" | sed -e 's/ <.*//' - -try: - path = py.std.sys.argv[1] -except IndexError: - print "usage: %s ROOTPATH" %(py.std.sys.argv[0]) - raise SystemExit, 1 - -d = {} - -for logentry in py.path.svnwc(path).log(): - a = logentry.author - if a in d: - d[a] += 1 - else: - d[a] = 1 - -items = d.items() -items.sort(lambda x,y: -cmp(x[1], y[1])) - -import uconf # http://codespeak.net/svn/uconf/dist/uconf - -# Authors that don't want to be listed -excluded = set("anna gintas ignas".split()) -cutoff = 5 # cutoff for authors in the LICENSE file -mark = False -for author, count in items: - if author in excluded: - continue - user = uconf.system.User(author) - try: - realname = user.realname.strip() - except KeyError: - realname = author - if not mark and count < cutoff: - mark = True - print '-'*60 - print " ", realname - #print count, " ", author diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py --- a/pypy/interpreter/argument.py +++ b/pypy/interpreter/argument.py @@ -428,8 +428,8 @@ return self._match_signature(w_firstarg, scope_w, signature, defaults_w, 0) except ArgErr, e: - raise OperationError(self.space.w_TypeError, - self.space.wrap(e.getmsg(fnname))) + raise operationerrfmt(self.space.w_TypeError, + "%s() %s", fnname, e.getmsg()) def _parse(self, w_firstarg, signature, defaults_w, blindargs=0): """Parse args and kwargs according to the signature of a code object, @@ -450,8 +450,8 @@ try: return self._parse(w_firstarg, signature, defaults_w, blindargs) except ArgErr, e: - raise OperationError(self.space.w_TypeError, - self.space.wrap(e.getmsg(fnname))) + raise operationerrfmt(self.space.w_TypeError, + "%s() %s", fnname, e.getmsg()) @staticmethod def frompacked(space, w_args=None, w_kwds=None): @@ -626,7 +626,7 @@ class ArgErr(Exception): - def getmsg(self, fnname): + def getmsg(self): raise NotImplementedError class ArgErrCount(ArgErr): @@ -642,11 +642,10 @@ self.num_args = got_nargs self.num_kwds = nkwds - def getmsg(self, fnname): + def getmsg(self): n = self.expected_nargs if n == 0: - msg = "%s() takes no arguments (%d given)" % ( - fnname, + msg = "takes no arguments (%d given)" % ( self.num_args + self.num_kwds) else: defcount = self.num_defaults @@ -672,8 +671,7 @@ msg2 = " non-keyword" else: msg2 = "" - msg = "%s() takes %s %d%s argument%s (%d given)" % ( - fnname, + msg = "takes %s %d%s argument%s (%d given)" % ( msg1, n, msg2, @@ -686,9 +684,8 @@ def __init__(self, argname): self.argname = argname - def getmsg(self, fnname): - msg = "%s() got multiple values for keyword argument '%s'" % ( - fnname, + def getmsg(self): + msg = "got multiple values for keyword argument '%s'" % ( self.argname) return msg @@ -722,13 +719,11 @@ break self.kwd_name = name - def getmsg(self, fnname): + def getmsg(self): if self.num_kwds == 1: - msg = "%s() got an unexpected keyword argument '%s'" % ( - fnname, + msg = "got an unexpected keyword argument '%s'" % ( self.kwd_name) else: - msg = "%s() got %d unexpected keyword arguments" % ( - fnname, + msg = "got %d unexpected keyword arguments" % ( self.num_kwds) return msg diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -1567,12 +1567,15 @@ 'ArithmeticError', 'AssertionError', 'AttributeError', + 'BaseException', + 'DeprecationWarning', 'EOFError', 'EnvironmentError', 'Exception', 'FloatingPointError', 'IOError', 'ImportError', + 'ImportWarning', 'IndentationError', 'IndexError', 'KeyError', @@ -1593,7 +1596,10 @@ 'TabError', 'TypeError', 'UnboundLocalError', + 'UnicodeDecodeError', 'UnicodeError', + 'UnicodeEncodeError', + 'UnicodeTranslateError', 'ValueError', 'ZeroDivisionError', 'UnicodeEncodeError', diff --git a/pypy/interpreter/eval.py b/pypy/interpreter/eval.py --- a/pypy/interpreter/eval.py +++ b/pypy/interpreter/eval.py @@ -2,7 +2,6 @@ This module defines the abstract base classes that support execution: Code and Frame. """ -from pypy.rlib import jit from pypy.interpreter.error import OperationError from pypy.interpreter.baseobjspace import Wrappable @@ -98,7 +97,6 @@ "Abstract. Get the expected number of locals." raise TypeError, "abstract" - @jit.dont_look_inside def fast2locals(self): # Copy values from the fastlocals to self.w_locals if self.w_locals is None: @@ -112,7 +110,6 @@ w_name = self.space.wrap(name) self.space.setitem(self.w_locals, w_name, w_value) - @jit.dont_look_inside def locals2fast(self): # Copy values from self.w_locals to the fastlocals assert self.w_locals is not None diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py --- a/pypy/interpreter/gateway.py +++ b/pypy/interpreter/gateway.py @@ -619,7 +619,8 @@ self.descr_reqcls, args) except Exception, e: - raise self.handle_exception(space, e) + self.handle_exception(space, e) + w_result = None if w_result is None: w_result = space.w_None return w_result @@ -655,7 +656,8 @@ self.descr_reqcls, args) except Exception, e: - raise self.handle_exception(space, e) + self.handle_exception(space, e) + w_result = None if w_result is None: w_result = space.w_None return w_result @@ -674,7 +676,8 @@ self.descr_reqcls, args.prepend(w_obj)) except Exception, e: - raise self.handle_exception(space, e) + self.handle_exception(space, e) + w_result = None if w_result is None: w_result = space.w_None return w_result @@ -690,7 +693,8 @@ raise OperationError(space.w_SystemError, space.wrap("unexpected DescrMismatch error")) except Exception, e: - raise self.handle_exception(space, e) + self.handle_exception(space, e) + w_result = None if w_result is None: w_result = space.w_None return w_result @@ -708,7 +712,8 @@ self.descr_reqcls, Arguments(space, [w1])) except Exception, e: - raise self.handle_exception(space, e) + self.handle_exception(space, e) + w_result = None if w_result is None: w_result = space.w_None return w_result @@ -726,7 +731,8 @@ self.descr_reqcls, Arguments(space, [w1, w2])) except Exception, e: - raise self.handle_exception(space, e) + self.handle_exception(space, e) + w_result = None if w_result is None: w_result = space.w_None return w_result @@ -744,7 +750,8 @@ self.descr_reqcls, Arguments(space, [w1, w2, w3])) except Exception, e: - raise self.handle_exception(space, e) + self.handle_exception(space, e) + w_result = None if w_result is None: w_result = space.w_None return w_result @@ -763,7 +770,8 @@ Arguments(space, [w1, w2, w3, w4])) except Exception, e: - raise self.handle_exception(space, e) + self.handle_exception(space, e) + w_result = None if w_result is None: w_result = space.w_None return w_result diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py --- a/pypy/interpreter/generator.py +++ b/pypy/interpreter/generator.py @@ -162,7 +162,8 @@ # generate 2 versions of the function and 2 jit drivers. def _create_unpack_into(): jitdriver = jit.JitDriver(greens=['pycode'], - reds=['self', 'frame', 'results']) + reds=['self', 'frame', 'results'], + name='unpack_into') def unpack_into(self, results): """This is a hack for performance: runs the generator and collects all produced items in a list.""" @@ -196,4 +197,4 @@ self.frame = None return unpack_into unpack_into = _create_unpack_into() - unpack_into_w = _create_unpack_into() \ No newline at end of file + unpack_into_w = _create_unpack_into() diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py --- a/pypy/interpreter/test/test_argument.py +++ b/pypy/interpreter/test/test_argument.py @@ -393,8 +393,8 @@ class FakeArgErr(ArgErr): - def getmsg(self, fname): - return "msg "+fname + def getmsg(self): + return "msg" def _match_signature(*args): raise FakeArgErr() @@ -404,7 +404,7 @@ excinfo = py.test.raises(OperationError, args.parse_obj, "obj", "foo", Signature(["a", "b"], None, None)) assert excinfo.value.w_type is TypeError - assert excinfo.value._w_value == "msg foo" + assert excinfo.value.get_w_value(space) == "foo() msg" def test_args_parsing_into_scope(self): @@ -448,8 +448,8 @@ class FakeArgErr(ArgErr): - def getmsg(self, fname): - return "msg "+fname + def getmsg(self): + return "msg" def _match_signature(*args): raise FakeArgErr() @@ -460,7 +460,7 @@ "obj", [None, None], "foo", Signature(["a", "b"], None, None)) assert excinfo.value.w_type is TypeError - assert excinfo.value._w_value == "msg foo" + assert excinfo.value.get_w_value(space) == "foo() msg" def test_topacked_frompacked(self): space = DummySpace() @@ -493,35 +493,35 @@ # got_nargs, nkwds, expected_nargs, has_vararg, has_kwarg, # defaults_w, missing_args err = ArgErrCount(1, 0, 0, False, False, None, 0) - s = err.getmsg('foo') - assert s == "foo() takes no arguments (1 given)" + s = err.getmsg() + assert s == "takes no arguments (1 given)" err = ArgErrCount(0, 0, 1, False, False, [], 1) - s = err.getmsg('foo') - assert s == "foo() takes exactly 1 argument (0 given)" + s = err.getmsg() + assert s == "takes exactly 1 argument (0 given)" err = ArgErrCount(3, 0, 2, False, False, [], 0) - s = err.getmsg('foo') - assert s == "foo() takes exactly 2 arguments (3 given)" + s = err.getmsg() + assert s == "takes exactly 2 arguments (3 given)" err = ArgErrCount(3, 0, 2, False, False, ['a'], 0) - s = err.getmsg('foo') - assert s == "foo() takes at most 2 arguments (3 given)" + s = err.getmsg() + assert s == "takes at most 2 arguments (3 given)" err = ArgErrCount(1, 0, 2, True, False, [], 1) - s = err.getmsg('foo') - assert s == "foo() takes at least 2 arguments (1 given)" + s = err.getmsg() + assert s == "takes at least 2 arguments (1 given)" err = ArgErrCount(0, 1, 2, True, False, ['a'], 1) - s = err.getmsg('foo') - assert s == "foo() takes at least 1 non-keyword argument (0 given)" + s = err.getmsg() + assert s == "takes at least 1 non-keyword argument (0 given)" err = ArgErrCount(2, 1, 1, False, True, [], 0) - s = err.getmsg('foo') - assert s == "foo() takes exactly 1 non-keyword argument (2 given)" + s = err.getmsg() + assert s == "takes exactly 1 non-keyword argument (2 given)" err = ArgErrCount(0, 1, 1, False, True, [], 1) - s = err.getmsg('foo') - assert s == "foo() takes exactly 1 non-keyword argument (0 given)" + s = err.getmsg() + assert s == "takes exactly 1 non-keyword argument (0 given)" err = ArgErrCount(0, 1, 1, True, True, [], 1) - s = err.getmsg('foo') - assert s == "foo() takes at least 1 non-keyword argument (0 given)" + s = err.getmsg() + assert s == "takes at least 1 non-keyword argument (0 given)" err = ArgErrCount(2, 1, 1, False, True, ['a'], 0) - s = err.getmsg('foo') - assert s == "foo() takes at most 1 non-keyword argument (2 given)" + s = err.getmsg() + assert s == "takes at most 1 non-keyword argument (2 given)" def test_bad_type_for_star(self): space = self.space @@ -543,12 +543,12 @@ def test_unknown_keywords(self): space = DummySpace() err = ArgErrUnknownKwds(space, 1, ['a', 'b'], [True, False], None) - s = err.getmsg('foo') - assert s == "foo() got an unexpected keyword argument 'b'" + s = err.getmsg() + assert s == "got an unexpected keyword argument 'b'" err = ArgErrUnknownKwds(space, 2, ['a', 'b', 'c'], [True, False, False], None) - s = err.getmsg('foo') - assert s == "foo() got 2 unexpected keyword arguments" + s = err.getmsg() + assert s == "got 2 unexpected keyword arguments" def test_unknown_unicode_keyword(self): class DummySpaceUnicode(DummySpace): @@ -558,13 +558,13 @@ err = ArgErrUnknownKwds(space, 1, ['a', None, 'b', 'c'], [True, False, True, True], [unichr(0x1234), u'b', u'c']) - s = err.getmsg('foo') - assert s == "foo() got an unexpected keyword argument '\xe1\x88\xb4'" + s = err.getmsg() + assert s == "got an unexpected keyword argument '\xe1\x88\xb4'" def test_multiple_values(self): err = ArgErrMultipleValues('bla') - s = err.getmsg('foo') - assert s == "foo() got multiple values for keyword argument 'bla'" + s = err.getmsg() + assert s == "got multiple values for keyword argument 'bla'" class AppTestArgument: def test_error_message(self): diff --git a/pypy/jit/backend/llgraph/runner.py b/pypy/jit/backend/llgraph/runner.py --- a/pypy/jit/backend/llgraph/runner.py +++ b/pypy/jit/backend/llgraph/runner.py @@ -37,7 +37,7 @@ def get_arg_types(self): return self.arg_types - def get_return_type(self): + def get_result_type(self): return self.typeinfo def get_extra_info(self): diff --git a/pypy/jit/backend/llsupport/descr.py b/pypy/jit/backend/llsupport/descr.py --- a/pypy/jit/backend/llsupport/descr.py +++ b/pypy/jit/backend/llsupport/descr.py @@ -5,11 +5,7 @@ from pypy.jit.metainterp.history import AbstractDescr, getkind from pypy.jit.metainterp import history from pypy.jit.codewriter import heaptracker, longlong - -# The point of the class organization in this file is to make instances -# as compact as possible. This is done by not storing the field size or -# the 'is_pointer_field' flag in the instance itself but in the class -# (in methods actually) using a few classes instead of just one. +from pypy.jit.codewriter.longlong import is_longlong class GcCache(object): @@ -19,6 +15,7 @@ self._cache_size = {} self._cache_field = {} self._cache_array = {} + self._cache_arraylen = {} self._cache_call = {} self._cache_interiorfield = {} @@ -26,24 +23,15 @@ assert isinstance(STRUCT, lltype.GcStruct) def init_array_descr(self, ARRAY, arraydescr): - assert isinstance(ARRAY, lltype.GcArray) + assert (isinstance(ARRAY, lltype.GcArray) or + isinstance(ARRAY, lltype.GcStruct) and ARRAY._arrayfld) -if lltype.SignedLongLong is lltype.Signed: - def is_longlong(TYPE): - return False -else: - assert rffi.sizeof(lltype.SignedLongLong) == rffi.sizeof(lltype.Float) - def is_longlong(TYPE): - return TYPE in (lltype.SignedLongLong, lltype.UnsignedLongLong) - # ____________________________________________________________ # SizeDescrs class SizeDescr(AbstractDescr): size = 0 # help translation - is_immutable = False - tid = llop.combine_ushort(lltype.Signed, 0, 0) def __init__(self, size, count_fields_if_immut=-1): @@ -77,265 +65,247 @@ cache[STRUCT] = sizedescr return sizedescr + # ____________________________________________________________ # FieldDescrs -class BaseFieldDescr(AbstractDescr): +FLAG_POINTER = 'P' +FLAG_FLOAT = 'F' +FLAG_UNSIGNED = 'U' +FLAG_SIGNED = 'S' +FLAG_STRUCT = 'X' +FLAG_VOID = 'V' + +class FieldDescr(AbstractDescr): + name = '' offset = 0 # help translation - name = '' - _clsname = '' + field_size = 0 + flag = '\x00' - def __init__(self, name, offset): + def __init__(self, name, offset, field_size, flag): self.name = name self.offset = offset + self.field_size = field_size + self.flag = flag + + def is_pointer_field(self): + return self.flag == FLAG_POINTER + + def is_float_field(self): + return self.flag == FLAG_FLOAT + + def is_field_signed(self): + return self.flag == FLAG_SIGNED def sort_key(self): return self.offset - def get_field_size(self, translate_support_code): - raise NotImplementedError + def repr_of_descr(self): + return '' % (self.flag, self.name, self.offset) - _is_pointer_field = False # unless overridden by GcPtrFieldDescr - _is_float_field = False # unless overridden by FloatFieldDescr - _is_field_signed = False # unless overridden by XxxFieldDescr - - def is_pointer_field(self): - return self._is_pointer_field - - def is_float_field(self): - return self._is_float_field - - def is_field_signed(self): - return self._is_field_signed - - def repr_of_descr(self): - return '<%s %s %s>' % (self._clsname, self.name, self.offset) - -class DynamicFieldDescr(BaseFieldDescr): - def __init__(self, offset, fieldsize, is_pointer, is_float, is_signed): - self.offset = offset - self._fieldsize = fieldsize - self._is_pointer_field = is_pointer - self._is_float_field = is_float - self._is_field_signed = is_signed - - def get_field_size(self, translate_support_code): - return self._fieldsize - -class NonGcPtrFieldDescr(BaseFieldDescr): - _clsname = 'NonGcPtrFieldDescr' - def get_field_size(self, translate_support_code): - return symbolic.get_size_of_ptr(translate_support_code) - -class GcPtrFieldDescr(NonGcPtrFieldDescr): - _clsname = 'GcPtrFieldDescr' - _is_pointer_field = True - -def getFieldDescrClass(TYPE): - return getDescrClass(TYPE, BaseFieldDescr, GcPtrFieldDescr, - NonGcPtrFieldDescr, 'Field', 'get_field_size', - '_is_float_field', '_is_field_signed') def get_field_descr(gccache, STRUCT, fieldname): cache = gccache._cache_field try: return cache[STRUCT][fieldname] except KeyError: - offset, _ = symbolic.get_field_token(STRUCT, fieldname, - gccache.translate_support_code) + offset, size = symbolic.get_field_token(STRUCT, fieldname, + gccache.translate_support_code) FIELDTYPE = getattr(STRUCT, fieldname) + flag = get_type_flag(FIELDTYPE) name = '%s.%s' % (STRUCT._name, fieldname) - fielddescr = getFieldDescrClass(FIELDTYPE)(name, offset) + fielddescr = FieldDescr(name, offset, size, flag) cachedict = cache.setdefault(STRUCT, {}) cachedict[fieldname] = fielddescr return fielddescr +def get_type_flag(TYPE): + if isinstance(TYPE, lltype.Ptr): + if TYPE.TO._gckind == 'gc': + return FLAG_POINTER + else: + return FLAG_UNSIGNED + if isinstance(TYPE, lltype.Struct): + return FLAG_STRUCT + if TYPE is lltype.Float or is_longlong(TYPE): + return FLAG_FLOAT + if (TYPE is not lltype.Bool and isinstance(TYPE, lltype.Number) and + rffi.cast(TYPE, -1) == -1): + return FLAG_SIGNED + return FLAG_UNSIGNED + +def get_field_arraylen_descr(gccache, ARRAY_OR_STRUCT): + cache = gccache._cache_arraylen + try: + return cache[ARRAY_OR_STRUCT] + except KeyError: + tsc = gccache.translate_support_code + (_, _, ofs) = symbolic.get_array_token(ARRAY_OR_STRUCT, tsc) + size = symbolic.get_size(lltype.Signed, tsc) + result = FieldDescr("len", ofs, size, get_type_flag(lltype.Signed)) + cache[ARRAY_OR_STRUCT] = result + return result + + # ____________________________________________________________ # ArrayDescrs -_A = lltype.GcArray(lltype.Signed) # a random gcarray -_AF = lltype.GcArray(lltype.Float) # an array of C doubles +class ArrayDescr(AbstractDescr): + tid = 0 + basesize = 0 # workaround for the annotator + itemsize = 0 + lendescr = None + flag = '\x00' - -class BaseArrayDescr(AbstractDescr): - _clsname = '' - tid = llop.combine_ushort(lltype.Signed, 0, 0) - - def get_base_size(self, translate_support_code): - basesize, _, _ = symbolic.get_array_token(_A, translate_support_code) - return basesize - - def get_ofs_length(self, translate_support_code): - _, _, ofslength = symbolic.get_array_token(_A, translate_support_code) - return ofslength - - def get_item_size(self, translate_support_code): - raise NotImplementedError - - _is_array_of_pointers = False # unless overridden by GcPtrArrayDescr - _is_array_of_floats = False # unless overridden by FloatArrayDescr - _is_array_of_structs = False # unless overridden by StructArrayDescr - _is_item_signed = False # unless overridden by XxxArrayDescr + def __init__(self, basesize, itemsize, lendescr, flag): + self.basesize = basesize + self.itemsize = itemsize + self.lendescr = lendescr # or None, if no length + self.flag = flag def is_array_of_pointers(self): - return self._is_array_of_pointers + return self.flag == FLAG_POINTER def is_array_of_floats(self): - return self._is_array_of_floats + return self.flag == FLAG_FLOAT + + def is_item_signed(self): + return self.flag == FLAG_SIGNED def is_array_of_structs(self): - return self._is_array_of_structs - - def is_item_signed(self): - return self._is_item_signed + return self.flag == FLAG_STRUCT def repr_of_descr(self): - return '<%s>' % self._clsname + return '' % (self.flag, self.itemsize) -class NonGcPtrArrayDescr(BaseArrayDescr): - _clsname = 'NonGcPtrArrayDescr' - def get_item_size(self, translate_support_code): - return symbolic.get_size_of_ptr(translate_support_code) - -class GcPtrArrayDescr(NonGcPtrArrayDescr): - _clsname = 'GcPtrArrayDescr' - _is_array_of_pointers = True - -class FloatArrayDescr(BaseArrayDescr): - _clsname = 'FloatArrayDescr' - _is_array_of_floats = True - def get_base_size(self, translate_support_code): - basesize, _, _ = symbolic.get_array_token(_AF, translate_support_code) - return basesize - def get_item_size(self, translate_support_code): - return symbolic.get_size(lltype.Float, translate_support_code) - -class StructArrayDescr(BaseArrayDescr): - _clsname = 'StructArrayDescr' - _is_array_of_structs = True - -class BaseArrayNoLengthDescr(BaseArrayDescr): - def get_base_size(self, translate_support_code): - return 0 - - def get_ofs_length(self, translate_support_code): - return -1 - -class DynamicArrayNoLengthDescr(BaseArrayNoLengthDescr): - def __init__(self, itemsize): - self.itemsize = itemsize - - def get_item_size(self, translate_support_code): - return self.itemsize - -class NonGcPtrArrayNoLengthDescr(BaseArrayNoLengthDescr): - _clsname = 'NonGcPtrArrayNoLengthDescr' - def get_item_size(self, translate_support_code): - return symbolic.get_size_of_ptr(translate_support_code) - -class GcPtrArrayNoLengthDescr(NonGcPtrArrayNoLengthDescr): - _clsname = 'GcPtrArrayNoLengthDescr' - _is_array_of_pointers = True - -def getArrayDescrClass(ARRAY): - if ARRAY.OF is lltype.Float: - return FloatArrayDescr - elif isinstance(ARRAY.OF, lltype.Struct): - class Descr(StructArrayDescr): - _clsname = '%sArrayDescr' % ARRAY.OF._name - def get_item_size(self, translate_support_code): - return symbolic.get_size(ARRAY.OF, translate_support_code) - Descr.__name__ = Descr._clsname - return Descr - return getDescrClass(ARRAY.OF, BaseArrayDescr, GcPtrArrayDescr, - NonGcPtrArrayDescr, 'Array', 'get_item_size', - '_is_array_of_floats', '_is_item_signed') - -def getArrayNoLengthDescrClass(ARRAY): - return getDescrClass(ARRAY.OF, BaseArrayNoLengthDescr, GcPtrArrayNoLengthDescr, - NonGcPtrArrayNoLengthDescr, 'ArrayNoLength', 'get_item_size', - '_is_array_of_floats', '_is_item_signed') - -def get_array_descr(gccache, ARRAY): +def get_array_descr(gccache, ARRAY_OR_STRUCT): cache = gccache._cache_array try: - return cache[ARRAY] + return cache[ARRAY_OR_STRUCT] except KeyError: - # we only support Arrays that are either GcArrays, or raw no-length - # non-gc Arrays. - if ARRAY._hints.get('nolength', False): - assert not isinstance(ARRAY, lltype.GcArray) - arraydescr = getArrayNoLengthDescrClass(ARRAY)() + tsc = gccache.translate_support_code + basesize, itemsize, _ = symbolic.get_array_token(ARRAY_OR_STRUCT, tsc) + if isinstance(ARRAY_OR_STRUCT, lltype.Array): + ARRAY_INSIDE = ARRAY_OR_STRUCT else: - assert isinstance(ARRAY, lltype.GcArray) - arraydescr = getArrayDescrClass(ARRAY)() - # verify basic assumption that all arrays' basesize and ofslength - # are equal - basesize, itemsize, ofslength = symbolic.get_array_token(ARRAY, False) - assert basesize == arraydescr.get_base_size(False) - assert itemsize == arraydescr.get_item_size(False) - if not ARRAY._hints.get('nolength', False): - assert ofslength == arraydescr.get_ofs_length(False) - if isinstance(ARRAY, lltype.GcArray): - gccache.init_array_descr(ARRAY, arraydescr) - cache[ARRAY] = arraydescr + ARRAY_INSIDE = ARRAY_OR_STRUCT._flds[ARRAY_OR_STRUCT._arrayfld] + if ARRAY_INSIDE._hints.get('nolength', False): + lendescr = None + else: + lendescr = get_field_arraylen_descr(gccache, ARRAY_OR_STRUCT) + flag = get_type_flag(ARRAY_INSIDE.OF) + arraydescr = ArrayDescr(basesize, itemsize, lendescr, flag) + if ARRAY_OR_STRUCT._gckind == 'gc': + gccache.init_array_descr(ARRAY_OR_STRUCT, arraydescr) + cache[ARRAY_OR_STRUCT] = arraydescr return arraydescr + # ____________________________________________________________ # InteriorFieldDescr class InteriorFieldDescr(AbstractDescr): - arraydescr = BaseArrayDescr() # workaround for the annotator - fielddescr = BaseFieldDescr('', 0) + arraydescr = ArrayDescr(0, 0, None, '\x00') # workaround for the annotator + fielddescr = FieldDescr('', 0, 0, '\x00') def __init__(self, arraydescr, fielddescr): + assert arraydescr.flag == FLAG_STRUCT self.arraydescr = arraydescr self.fielddescr = fielddescr + def sort_key(self): + return self.fielddescr.sort_key() + def is_pointer_field(self): return self.fielddescr.is_pointer_field() def is_float_field(self): return self.fielddescr.is_float_field() - def sort_key(self): - return self.fielddescr.sort_key() - def repr_of_descr(self): return '' % self.fielddescr.repr_of_descr() -def get_interiorfield_descr(gc_ll_descr, ARRAY, FIELDTP, name): +def get_interiorfield_descr(gc_ll_descr, ARRAY, name): cache = gc_ll_descr._cache_interiorfield try: - return cache[(ARRAY, FIELDTP, name)] + return cache[(ARRAY, name)] except KeyError: arraydescr = get_array_descr(gc_ll_descr, ARRAY) - fielddescr = get_field_descr(gc_ll_descr, FIELDTP, name) + fielddescr = get_field_descr(gc_ll_descr, ARRAY.OF, name) descr = InteriorFieldDescr(arraydescr, fielddescr) - cache[(ARRAY, FIELDTP, name)] = descr + cache[(ARRAY, name)] = descr return descr +def get_dynamic_interiorfield_descr(gc_ll_descr, offset, width, fieldsize, + is_pointer, is_float, is_signed): + arraydescr = ArrayDescr(0, width, None, FLAG_STRUCT) + if is_pointer: + assert not is_float + flag = FLAG_POINTER + elif is_float: + flag = FLAG_FLOAT + elif is_signed: + flag = FLAG_SIGNED + else: + flag = FLAG_UNSIGNED + fielddescr = FieldDescr('dynamic', offset, fieldsize, flag) + return InteriorFieldDescr(arraydescr, fielddescr) + + # ____________________________________________________________ # CallDescrs -class BaseCallDescr(AbstractDescr): - _clsname = '' - loop_token = None +class CallDescr(AbstractDescr): arg_classes = '' # <-- annotation hack + result_type = '\x00' + result_flag = '\x00' ffi_flags = 1 + call_stub_i = staticmethod(lambda func, args_i, args_r, args_f: + 0) + call_stub_r = staticmethod(lambda func, args_i, args_r, args_f: + lltype.nullptr(llmemory.GCREF.TO)) + call_stub_f = staticmethod(lambda func,args_i,args_r,args_f: + longlong.ZEROF) - def __init__(self, arg_classes, extrainfo=None, ffi_flags=1): - self.arg_classes = arg_classes # string of "r" and "i" (ref/int) + def __init__(self, arg_classes, result_type, result_signed, result_size, + extrainfo=None, ffi_flags=1): + """ + 'arg_classes' is a string of characters, one per argument: + 'i', 'r', 'f', 'L', 'S' + + 'result_type' is one character from the same list or 'v' + + 'result_signed' is a boolean True/False + """ + self.arg_classes = arg_classes + self.result_type = result_type + self.result_size = result_size self.extrainfo = extrainfo self.ffi_flags = ffi_flags # NB. the default ffi_flags is 1, meaning FUNCFLAG_CDECL, which # makes sense on Windows as it's the one for all the C functions # we are compiling together with the JIT. On non-Windows platforms # it is just ignored anyway. + if result_type == 'v': + result_flag = FLAG_VOID + elif result_type == 'i': + if result_signed: + result_flag = FLAG_SIGNED + else: + result_flag = FLAG_UNSIGNED + elif result_type == history.REF: + result_flag = FLAG_POINTER + elif result_type == history.FLOAT or result_type == 'L': + result_flag = FLAG_FLOAT + elif result_type == 'S': + result_flag = FLAG_UNSIGNED + else: + raise NotImplementedError("result_type = '%s'" % (result_type,)) + self.result_flag = result_flag def __repr__(self): - res = '%s(%s)' % (self.__class__.__name__, self.arg_classes) + res = 'CallDescr(%s)' % (self.arg_classes,) extraeffect = getattr(self.extrainfo, 'extraeffect', None) if extraeffect is not None: res += ' EF=%r' % extraeffect @@ -363,14 +333,14 @@ def get_arg_types(self): return self.arg_classes - def get_return_type(self): - return self._return_type + def get_result_type(self): + return self.result_type - def get_result_size(self, translate_support_code): - raise NotImplementedError + def get_result_size(self): + return self.result_size def is_result_signed(self): - return False # unless overridden + return self.result_flag == FLAG_SIGNED def create_call_stub(self, rtyper, RESULT): from pypy.rlib.clibffi import FFI_DEFAULT_ABI @@ -408,18 +378,26 @@ seen = {'i': 0, 'r': 0, 'f': 0} args = ", ".join([process(c) for c in self.arg_classes]) - if self.get_return_type() == history.INT: + result_type = self.get_result_type() + if result_type == history.INT: result = 'rffi.cast(lltype.Signed, res)' - elif self.get_return_type() == history.REF: + category = 'i' + elif result_type == history.REF: + assert RESULT == llmemory.GCREF # should be ensured by the caller result = 'lltype.cast_opaque_ptr(llmemory.GCREF, res)' - elif self.get_return_type() == history.FLOAT: + category = 'r' + elif result_type == history.FLOAT: result = 'longlong.getfloatstorage(res)' - elif self.get_return_type() == 'L': + category = 'f' + elif result_type == 'L': result = 'rffi.cast(lltype.SignedLongLong, res)' - elif self.get_return_type() == history.VOID: - result = 'None' - elif self.get_return_type() == 'S': + category = 'f' + elif result_type == history.VOID: + result = '0' + category = 'i' + elif result_type == 'S': result = 'longlong.singlefloat2int(res)' + category = 'i' else: assert 0 source = py.code.Source(""" @@ -433,10 +411,13 @@ d = globals().copy() d.update(locals()) exec source.compile() in d - self.call_stub = d['call_stub'] + call_stub = d['call_stub'] + # store the function into one of three attributes, to preserve + # type-correctness of the return value + setattr(self, 'call_stub_%s' % category, call_stub) def verify_types(self, args_i, args_r, args_f, return_type): - assert self._return_type in return_type + assert self.result_type in return_type assert (self.arg_classes.count('i') + self.arg_classes.count('S')) == len(args_i or ()) assert self.arg_classes.count('r') == len(args_r or ()) @@ -444,161 +425,56 @@ self.arg_classes.count('L')) == len(args_f or ()) def repr_of_descr(self): - return '<%s>' % self._clsname + res = 'Call%s %d' % (self.result_type, self.result_size) + if self.arg_classes: + res += ' ' + self.arg_classes + if self.extrainfo: + res += ' EF=%d' % self.extrainfo.extraeffect + oopspecindex = self.extrainfo.oopspecindex + if oopspecindex: + res += ' OS=%d' % oopspecindex + return '<%s>' % res -class BaseIntCallDescr(BaseCallDescr): - # Base class of the various subclasses of descrs corresponding to - # calls having a return kind of 'int' (including non-gc pointers). - # The inheritance hierarchy is a bit different than with other Descr - # classes because of the 'call_stub' attribute, which is of type - # - # lambda func, args_i, args_r, args_f --> int/ref/float/void - # - # The purpose of BaseIntCallDescr is to be the parent of all classes - # in which 'call_stub' has a return kind of 'int'. - _return_type = history.INT - call_stub = staticmethod(lambda func, args_i, args_r, args_f: 0) - - _is_result_signed = False # can be overridden in XxxCallDescr - def is_result_signed(self): - return self._is_result_signed - -class DynamicIntCallDescr(BaseIntCallDescr): - """ - calldescr that works for every integer type, by explicitly passing it the - size of the result. Used only by get_call_descr_dynamic - """ - _clsname = 'DynamicIntCallDescr' - - def __init__(self, arg_classes, result_size, result_sign, extrainfo, ffi_flags): - BaseIntCallDescr.__init__(self, arg_classes, extrainfo, ffi_flags) - assert isinstance(result_sign, bool) - self._result_size = chr(result_size) - self._result_sign = result_sign - - def get_result_size(self, translate_support_code): - return ord(self._result_size) - - def is_result_signed(self): - return self._result_sign - - -class NonGcPtrCallDescr(BaseIntCallDescr): - _clsname = 'NonGcPtrCallDescr' - def get_result_size(self, translate_support_code): - return symbolic.get_size_of_ptr(translate_support_code) - -class GcPtrCallDescr(BaseCallDescr): - _clsname = 'GcPtrCallDescr' - _return_type = history.REF - call_stub = staticmethod(lambda func, args_i, args_r, args_f: - lltype.nullptr(llmemory.GCREF.TO)) - def get_result_size(self, translate_support_code): - return symbolic.get_size_of_ptr(translate_support_code) - -class FloatCallDescr(BaseCallDescr): - _clsname = 'FloatCallDescr' - _return_type = history.FLOAT - call_stub = staticmethod(lambda func,args_i,args_r,args_f: longlong.ZEROF) - def get_result_size(self, translate_support_code): - return symbolic.get_size(lltype.Float, translate_support_code) - -class LongLongCallDescr(FloatCallDescr): - _clsname = 'LongLongCallDescr' - _return_type = 'L' - -class VoidCallDescr(BaseCallDescr): - _clsname = 'VoidCallDescr' - _return_type = history.VOID - call_stub = staticmethod(lambda func, args_i, args_r, args_f: None) - def get_result_size(self, translate_support_code): - return 0 - -_SingleFloatCallDescr = None # built lazily - -def getCallDescrClass(RESULT): - if RESULT is lltype.Void: - return VoidCallDescr - if RESULT is lltype.Float: - return FloatCallDescr - if RESULT is lltype.SingleFloat: - global _SingleFloatCallDescr - if _SingleFloatCallDescr is None: - assert rffi.sizeof(rffi.UINT) == rffi.sizeof(RESULT) - class SingleFloatCallDescr(getCallDescrClass(rffi.UINT)): - _clsname = 'SingleFloatCallDescr' - _return_type = 'S' - _SingleFloatCallDescr = SingleFloatCallDescr - return _SingleFloatCallDescr - if is_longlong(RESULT): - return LongLongCallDescr - return getDescrClass(RESULT, BaseIntCallDescr, GcPtrCallDescr, - NonGcPtrCallDescr, 'Call', 'get_result_size', - Ellipsis, # <= floatattrname should not be used here - '_is_result_signed') -getCallDescrClass._annspecialcase_ = 'specialize:memo' +def map_type_to_argclass(ARG, accept_void=False): + kind = getkind(ARG) + if kind == 'int': + if ARG is lltype.SingleFloat: return 'S' + else: return 'i' + elif kind == 'ref': return 'r' + elif kind == 'float': + if is_longlong(ARG): return 'L' + else: return 'f' + elif kind == 'void': + if accept_void: return 'v' + raise NotImplementedError('ARG = %r' % (ARG,)) def get_call_descr(gccache, ARGS, RESULT, extrainfo=None): - arg_classes = [] - for ARG in ARGS: - kind = getkind(ARG) - if kind == 'int': - if ARG is lltype.SingleFloat: - arg_classes.append('S') + arg_classes = map(map_type_to_argclass, ARGS) + arg_classes = ''.join(arg_classes) + result_type = map_type_to_argclass(RESULT, accept_void=True) + RESULT_ERASED = RESULT + if RESULT is lltype.Void: + result_size = 0 + result_signed = False + else: + if isinstance(RESULT, lltype.Ptr): + # avoid too many CallDescrs + if result_type == 'r': + RESULT_ERASED = llmemory.GCREF else: - arg_classes.append('i') - elif kind == 'ref': arg_classes.append('r') - elif kind == 'float': - if is_longlong(ARG): - arg_classes.append('L') - else: - arg_classes.append('f') - else: - raise NotImplementedError('ARG = %r' % (ARG,)) - arg_classes = ''.join(arg_classes) - cls = getCallDescrClass(RESULT) - key = (cls, arg_classes, extrainfo) + RESULT_ERASED = llmemory.Address + result_size = symbolic.get_size(RESULT_ERASED, + gccache.translate_support_code) + result_signed = get_type_flag(RESULT) == FLAG_SIGNED + key = (arg_classes, result_type, result_signed, RESULT_ERASED, extrainfo) cache = gccache._cache_call try: - return cache[key] + calldescr = cache[key] except KeyError: - calldescr = cls(arg_classes, extrainfo) - calldescr.create_call_stub(gccache.rtyper, RESULT) + calldescr = CallDescr(arg_classes, result_type, result_signed, + result_size, extrainfo) + calldescr.create_call_stub(gccache.rtyper, RESULT_ERASED) cache[key] = calldescr - return calldescr - - -# ____________________________________________________________ - -def getDescrClass(TYPE, BaseDescr, GcPtrDescr, NonGcPtrDescr, - nameprefix, methodname, floatattrname, signedattrname, - _cache={}): - if isinstance(TYPE, lltype.Ptr): - if TYPE.TO._gckind == 'gc': - return GcPtrDescr - else: - return NonGcPtrDescr - if TYPE is lltype.SingleFloat: - assert rffi.sizeof(rffi.UINT) == rffi.sizeof(TYPE) - TYPE = rffi.UINT - try: - return _cache[nameprefix, TYPE] - except KeyError: - # - class Descr(BaseDescr): - _clsname = '%s%sDescr' % (TYPE._name, nameprefix) - Descr.__name__ = Descr._clsname - # - def method(self, translate_support_code): - return symbolic.get_size(TYPE, translate_support_code) - setattr(Descr, methodname, method) - # - if TYPE is lltype.Float or is_longlong(TYPE): - setattr(Descr, floatattrname, True) - elif (TYPE is not lltype.Bool and isinstance(TYPE, lltype.Number) and - rffi.cast(TYPE, -1) == -1): - setattr(Descr, signedattrname, True) - # - _cache[nameprefix, TYPE] = Descr - return Descr + assert repr(calldescr.result_size) == repr(result_size) + return calldescr diff --git a/pypy/jit/backend/llsupport/ffisupport.py b/pypy/jit/backend/llsupport/ffisupport.py --- a/pypy/jit/backend/llsupport/ffisupport.py +++ b/pypy/jit/backend/llsupport/ffisupport.py @@ -1,9 +1,7 @@ from pypy.rlib.rarithmetic import intmask from pypy.jit.metainterp import history from pypy.rpython.lltypesystem import rffi -from pypy.jit.backend.llsupport.descr import ( - DynamicIntCallDescr, NonGcPtrCallDescr, FloatCallDescr, VoidCallDescr, - LongLongCallDescr, getCallDescrClass) +from pypy.jit.backend.llsupport.descr import CallDescr class UnsupportedKind(Exception): pass @@ -16,29 +14,13 @@ argkinds = [get_ffi_type_kind(cpu, arg) for arg in ffi_args] except UnsupportedKind: return None - arg_classes = ''.join(argkinds) - if reskind == history.INT: - size = intmask(ffi_result.c_size) - signed = is_ffi_type_signed(ffi_result) - return DynamicIntCallDescr(arg_classes, size, signed, extrainfo, - ffi_flags=ffi_flags) - elif reskind == history.REF: - return NonGcPtrCallDescr(arg_classes, extrainfo, - ffi_flags=ffi_flags) - elif reskind == history.FLOAT: - return FloatCallDescr(arg_classes, extrainfo, - ffi_flags=ffi_flags) - elif reskind == history.VOID: - return VoidCallDescr(arg_classes, extrainfo, - ffi_flags=ffi_flags) - elif reskind == 'L': - return LongLongCallDescr(arg_classes, extrainfo, - ffi_flags=ffi_flags) - elif reskind == 'S': - SingleFloatCallDescr = getCallDescrClass(rffi.FLOAT) - return SingleFloatCallDescr(arg_classes, extrainfo, - ffi_flags=ffi_flags) - assert False + if reskind == history.VOID: + result_size = 0 + else: + result_size = intmask(ffi_result.c_size) + argkinds = ''.join(argkinds) + return CallDescr(argkinds, reskind, is_ffi_type_signed(ffi_result), + result_size, extrainfo, ffi_flags=ffi_flags) def get_ffi_type_kind(cpu, ffi_type): from pypy.rlib.libffi import types diff --git a/pypy/jit/backend/llsupport/gc.py b/pypy/jit/backend/llsupport/gc.py --- a/pypy/jit/backend/llsupport/gc.py +++ b/pypy/jit/backend/llsupport/gc.py @@ -1,6 +1,6 @@ import os from pypy.rlib import rgc -from pypy.rlib.objectmodel import we_are_translated +from pypy.rlib.objectmodel import we_are_translated, specialize from pypy.rlib.debug import fatalerror from pypy.rlib.rarithmetic import ovfcheck from pypy.rpython.lltypesystem import lltype, llmemory, rffi, rclass, rstr @@ -8,52 +8,93 @@ from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.annlowlevel import llhelper from pypy.translator.tool.cbuild import ExternalCompilationInfo -from pypy.jit.metainterp.history import BoxInt, BoxPtr, ConstInt, ConstPtr -from pypy.jit.metainterp.history import AbstractDescr +from pypy.jit.codewriter import heaptracker +from pypy.jit.metainterp.history import ConstPtr, AbstractDescr from pypy.jit.metainterp.resoperation import ResOperation, rop from pypy.jit.backend.llsupport import symbolic from pypy.jit.backend.llsupport.symbolic import WORD -from pypy.jit.backend.llsupport.descr import BaseSizeDescr, BaseArrayDescr +from pypy.jit.backend.llsupport.descr import SizeDescr, ArrayDescr from pypy.jit.backend.llsupport.descr import GcCache, get_field_descr -from pypy.jit.backend.llsupport.descr import GcPtrFieldDescr +from pypy.jit.backend.llsupport.descr import get_array_descr from pypy.jit.backend.llsupport.descr import get_call_descr +from pypy.jit.backend.llsupport.rewrite import GcRewriterAssembler from pypy.rpython.memory.gctransform import asmgcroot # ____________________________________________________________ class GcLLDescription(GcCache): - minimal_size_in_nursery = 0 - get_malloc_slowpath_addr = None def __init__(self, gcdescr, translator=None, rtyper=None): GcCache.__init__(self, translator is not None, rtyper) self.gcdescr = gcdescr + if translator and translator.config.translation.gcremovetypeptr: + self.fielddescr_vtable = None + else: + self.fielddescr_vtable = get_field_descr(self, rclass.OBJECT, + 'typeptr') + self._generated_functions = [] + + def _setup_str(self): + self.str_descr = get_array_descr(self, rstr.STR) + self.unicode_descr = get_array_descr(self, rstr.UNICODE) + + def generate_function(self, funcname, func, ARGS, RESULT=llmemory.GCREF): + """Generates a variant of malloc with the given name and the given + arguments. It should return NULL if out of memory. If it raises + anything, it must be an optional MemoryError. + """ + FUNCPTR = lltype.Ptr(lltype.FuncType(ARGS, RESULT)) + descr = get_call_descr(self, ARGS, RESULT) + setattr(self, funcname, func) + setattr(self, funcname + '_FUNCPTR', FUNCPTR) + setattr(self, funcname + '_descr', descr) + self._generated_functions.append(funcname) + + @specialize.arg(1) + def get_malloc_fn(self, funcname): + func = getattr(self, funcname) + FUNC = getattr(self, funcname + '_FUNCPTR') + return llhelper(FUNC, func) + + @specialize.arg(1) + def get_malloc_fn_addr(self, funcname): + ll_func = self.get_malloc_fn(funcname) + return heaptracker.adr2int(llmemory.cast_ptr_to_adr(ll_func)) + def _freeze_(self): return True def initialize(self): pass def do_write_barrier(self, gcref_struct, gcref_newptr): pass - def rewrite_assembler(self, cpu, operations, gcrefs_output_list): - return operations - def can_inline_malloc(self, descr): - return False - def can_inline_malloc_varsize(self, descr, num_elem): + def can_use_nursery_malloc(self, size): return False def has_write_barrier_class(self): return None def freeing_block(self, start, stop): pass + def get_nursery_free_addr(self): + raise NotImplementedError + def get_nursery_top_addr(self): + raise NotImplementedError - def get_funcptr_for_newarray(self): - return llhelper(self.GC_MALLOC_ARRAY, self.malloc_array) - def get_funcptr_for_newstr(self): - return llhelper(self.GC_MALLOC_STR_UNICODE, self.malloc_str) - def get_funcptr_for_newunicode(self): - return llhelper(self.GC_MALLOC_STR_UNICODE, self.malloc_unicode) + def gc_malloc(self, sizedescr): + """Blackhole: do a 'bh_new'. Also used for 'bh_new_with_vtable', + with the vtable pointer set manually afterwards.""" + assert isinstance(sizedescr, SizeDescr) + return self._bh_malloc(sizedescr) + def gc_malloc_array(self, arraydescr, num_elem): + assert isinstance(arraydescr, ArrayDescr) + return self._bh_malloc_array(arraydescr, num_elem) - def record_constptrs(self, op, gcrefs_output_list): + def gc_malloc_str(self, num_elem): + return self._bh_malloc_array(self.str_descr, num_elem) + + def gc_malloc_unicode(self, num_elem): + return self._bh_malloc_array(self.unicode_descr, num_elem) + + def _record_constptrs(self, op, gcrefs_output_list): for i in range(op.numargs()): v = op.getarg(i) if isinstance(v, ConstPtr) and bool(v.value): @@ -61,11 +102,27 @@ rgc._make_sure_does_not_move(p) gcrefs_output_list.append(p) + def rewrite_assembler(self, cpu, operations, gcrefs_output_list): + rewriter = GcRewriterAssembler(self, cpu) + newops = rewriter.rewrite(operations) + # record all GCREFs, because the GC (or Boehm) cannot see them and + # keep them alive if they end up as constants in the assembler + for op in newops: + self._record_constptrs(op, gcrefs_output_list) + return newops + # ____________________________________________________________ class GcLLDescr_boehm(GcLLDescription): - moving_gc = False - gcrootmap = None + kind = 'boehm' + moving_gc = False + round_up = False + gcrootmap = None + write_barrier_descr = None + fielddescr_tid = None + str_type_id = 0 + unicode_type_id = 0 + get_malloc_slowpath_addr = None @classmethod def configure_boehm_once(cls): @@ -76,6 +133,16 @@ from pypy.rpython.tool import rffi_platform compilation_info = rffi_platform.configure_boehm() + # on some platform GC_init is required before any other + # GC_* functions, call it here for the benefit of tests + # XXX move this to tests + init_fn_ptr = rffi.llexternal("GC_init", + [], lltype.Void, + compilation_info=compilation_info, + sandboxsafe=True, + _nowrapper=True) + init_fn_ptr() + # Versions 6.x of libgc needs to use GC_local_malloc(). # Versions 7.x of libgc removed this function; GC_malloc() has # the same behavior if libgc was compiled with @@ -95,96 +162,42 @@ sandboxsafe=True, _nowrapper=True) cls.malloc_fn_ptr = malloc_fn_ptr - cls.compilation_info = compilation_info return malloc_fn_ptr def __init__(self, gcdescr, translator, rtyper): GcLLDescription.__init__(self, gcdescr, translator, rtyper) # grab a pointer to the Boehm 'malloc' function - malloc_fn_ptr = self.configure_boehm_once() - self.funcptr_for_new = malloc_fn_ptr + self.malloc_fn_ptr = self.configure_boehm_once() + self._setup_str() + self._make_functions() - def malloc_array(basesize, itemsize, ofs_length, num_elem): + def _make_functions(self): + + def malloc_fixedsize(size): + return self.malloc_fn_ptr(size) + self.generate_function('malloc_fixedsize', malloc_fixedsize, + [lltype.Signed]) + + def malloc_array(basesize, num_elem, itemsize, ofs_length): try: - size = ovfcheck(basesize + ovfcheck(itemsize * num_elem)) + totalsize = ovfcheck(basesize + ovfcheck(itemsize * num_elem)) except OverflowError: return lltype.nullptr(llmemory.GCREF.TO) - res = self.funcptr_for_new(size) - if not res: - return res - rffi.cast(rffi.CArrayPtr(lltype.Signed), res)[ofs_length/WORD] = num_elem + res = self.malloc_fn_ptr(totalsize) + if res: + arrayptr = rffi.cast(rffi.CArrayPtr(lltype.Signed), res) + arrayptr[ofs_length/WORD] = num_elem return res - self.malloc_array = malloc_array - self.GC_MALLOC_ARRAY = lltype.Ptr(lltype.FuncType( - [lltype.Signed] * 4, llmemory.GCREF)) + self.generate_function('malloc_array', malloc_array, + [lltype.Signed] * 4) + def _bh_malloc(self, sizedescr): + return self.malloc_fixedsize(sizedescr.size) - (str_basesize, str_itemsize, str_ofs_length - ) = symbolic.get_array_token(rstr.STR, self.translate_support_code) - (unicode_basesize, unicode_itemsize, unicode_ofs_length - ) = symbolic.get_array_token(rstr.UNICODE, self.translate_support_code) - def malloc_str(length): - return self.malloc_array( - str_basesize, str_itemsize, str_ofs_length, length - ) - def malloc_unicode(length): - return self.malloc_array( - unicode_basesize, unicode_itemsize, unicode_ofs_length, length - ) - self.malloc_str = malloc_str - self.malloc_unicode = malloc_unicode - self.GC_MALLOC_STR_UNICODE = lltype.Ptr(lltype.FuncType( - [lltype.Signed], llmemory.GCREF)) - - - # on some platform GC_init is required before any other - # GC_* functions, call it here for the benefit of tests - # XXX move this to tests - init_fn_ptr = rffi.llexternal("GC_init", - [], lltype.Void, - compilation_info=self.compilation_info, - sandboxsafe=True, - _nowrapper=True) - - init_fn_ptr() - - def gc_malloc(self, sizedescr): - assert isinstance(sizedescr, BaseSizeDescr) - return self.funcptr_for_new(sizedescr.size) - - def gc_malloc_array(self, arraydescr, num_elem): - assert isinstance(arraydescr, BaseArrayDescr) - ofs_length = arraydescr.get_ofs_length(self.translate_support_code) - basesize = arraydescr.get_base_size(self.translate_support_code) - itemsize = arraydescr.get_item_size(self.translate_support_code) - return self.malloc_array(basesize, itemsize, ofs_length, num_elem) - - def gc_malloc_str(self, num_elem): - return self.malloc_str(num_elem) - - def gc_malloc_unicode(self, num_elem): - return self.malloc_unicode(num_elem) - - def args_for_new(self, sizedescr): - assert isinstance(sizedescr, BaseSizeDescr) - return [sizedescr.size] - - def args_for_new_array(self, arraydescr): - ofs_length = arraydescr.get_ofs_length(self.translate_support_code) - basesize = arraydescr.get_base_size(self.translate_support_code) - itemsize = arraydescr.get_item_size(self.translate_support_code) - return [basesize, itemsize, ofs_length] - - def get_funcptr_for_new(self): - return self.funcptr_for_new - - def rewrite_assembler(self, cpu, operations, gcrefs_output_list): - # record all GCREFs too, because Boehm cannot see them and keep them - # alive if they end up as constants in the assembler - for op in operations: - self.record_constptrs(op, gcrefs_output_list) - return GcLLDescription.rewrite_assembler(self, cpu, operations, - gcrefs_output_list) + def _bh_malloc_array(self, arraydescr, num_elem): + return self.malloc_array(arraydescr.basesize, num_elem, + arraydescr.itemsize, + arraydescr.lendescr.offset) # ____________________________________________________________ @@ -554,12 +567,14 @@ class WriteBarrierDescr(AbstractDescr): def __init__(self, gc_ll_descr): - GCClass = gc_ll_descr.GCClass self.llop1 = gc_ll_descr.llop1 self.WB_FUNCPTR = gc_ll_descr.WB_FUNCPTR self.WB_ARRAY_FUNCPTR = gc_ll_descr.WB_ARRAY_FUNCPTR - self.fielddescr_tid = get_field_descr(gc_ll_descr, GCClass.HDR, 'tid') + self.fielddescr_tid = gc_ll_descr.fielddescr_tid # + GCClass = gc_ll_descr.GCClass + if GCClass is None: # for tests + return self.jit_wb_if_flag = GCClass.JIT_WB_IF_FLAG self.jit_wb_if_flag_byteofs, self.jit_wb_if_flag_singlebyte = ( self.extract_flag_byte(self.jit_wb_if_flag)) @@ -596,48 +611,74 @@ funcaddr = llmemory.cast_ptr_to_adr(funcptr) return cpu.cast_adr_to_int(funcaddr) # this may return 0 + def has_write_barrier_from_array(self, cpu): + return self.get_write_barrier_from_array_fn(cpu) != 0 + class GcLLDescr_framework(GcLLDescription): DEBUG = False # forced to True by x86/test/test_zrpy_gc.py + kind = 'framework' + round_up = True - def __init__(self, gcdescr, translator, rtyper, llop1=llop): - from pypy.rpython.memory.gctypelayout import check_typeid - from pypy.rpython.memory.gcheader import GCHeaderBuilder - from pypy.rpython.memory.gctransform import framework + def __init__(self, gcdescr, translator, rtyper, llop1=llop, + really_not_translated=False): GcLLDescription.__init__(self, gcdescr, translator, rtyper) - assert self.translate_support_code, "required with the framework GC" self.translator = translator self.llop1 = llop1 + if really_not_translated: + assert not self.translate_support_code # but half does not work + self._initialize_for_tests() + else: + assert self.translate_support_code,"required with the framework GC" + self._check_valid_gc() + self._make_gcrootmap() + self._make_layoutbuilder() + self._setup_gcclass() + self._setup_tid() + self._setup_write_barrier() + self._setup_str() + self._make_functions(really_not_translated) + def _initialize_for_tests(self): + self.layoutbuilder = None + self.fielddescr_tid = AbstractDescr() + self.max_size_of_young_obj = 1000 + self.GCClass = None + + def _check_valid_gc(self): # we need the hybrid or minimark GC for rgc._make_sure_does_not_move() # to work - if gcdescr.config.translation.gc not in ('hybrid', 'minimark'): + if self.gcdescr.config.translation.gc not in ('hybrid', 'minimark'): raise NotImplementedError("--gc=%s not implemented with the JIT" % (gcdescr.config.translation.gc,)) + def _make_gcrootmap(self): # to find roots in the assembler, make a GcRootMap - name = gcdescr.config.translation.gcrootfinder + name = self.gcdescr.config.translation.gcrootfinder try: cls = globals()['GcRootMap_' + name] except KeyError: raise NotImplementedError("--gcrootfinder=%s not implemented" " with the JIT" % (name,)) - gcrootmap = cls(gcdescr) + gcrootmap = cls(self.gcdescr) self.gcrootmap = gcrootmap + def _make_layoutbuilder(self): # make a TransformerLayoutBuilder and save it on the translator # where it can be fished and reused by the FrameworkGCTransformer + from pypy.rpython.memory.gctransform import framework + translator = self.translator self.layoutbuilder = framework.TransformerLayoutBuilder(translator) self.layoutbuilder.delay_encoding() - self.translator._jit2gc = {'layoutbuilder': self.layoutbuilder} - gcrootmap.add_jit2gc_hooks(self.translator._jit2gc) + translator._jit2gc = {'layoutbuilder': self.layoutbuilder} + self.gcrootmap.add_jit2gc_hooks(translator._jit2gc) + def _setup_gcclass(self): + from pypy.rpython.memory.gcheader import GCHeaderBuilder self.GCClass = self.layoutbuilder.GCClass self.moving_gc = self.GCClass.moving_gc self.HDRPTR = lltype.Ptr(self.GCClass.HDR) self.gcheaderbuilder = GCHeaderBuilder(self.HDRPTR.TO) - (self.array_basesize, _, self.array_length_ofs) = \ - symbolic.get_array_token(lltype.GcArray(lltype.Signed), True) self.max_size_of_young_obj = self.GCClass.JIT_max_size_of_young_obj() self.minimal_size_in_nursery=self.GCClass.JIT_minimal_size_in_nursery() @@ -645,87 +686,124 @@ assert self.GCClass.inline_simple_malloc assert self.GCClass.inline_simple_malloc_varsize - # make a malloc function, with two arguments - def malloc_basic(size, tid): - type_id = llop.extract_ushort(llgroup.HALFWORD, tid) - check_typeid(type_id) - res = llop1.do_malloc_fixedsize_clear(llmemory.GCREF, - type_id, size, - False, False, False) - # In case the operation above failed, we are returning NULL - # from this function to assembler. There is also an RPython - # exception set, typically MemoryError; but it's easier and - # faster to check for the NULL return value, as done by - # translator/exceptiontransform.py. - #llop.debug_print(lltype.Void, "\tmalloc_basic", size, type_id, - # "-->", res) - return res - self.malloc_basic = malloc_basic - self.GC_MALLOC_BASIC = lltype.Ptr(lltype.FuncType( - [lltype.Signed, lltype.Signed], llmemory.GCREF)) + def _setup_tid(self): + self.fielddescr_tid = get_field_descr(self, self.GCClass.HDR, 'tid') + + def _setup_write_barrier(self): self.WB_FUNCPTR = lltype.Ptr(lltype.FuncType( [llmemory.Address, llmemory.Address], lltype.Void)) self.WB_ARRAY_FUNCPTR = lltype.Ptr(lltype.FuncType( [llmemory.Address, lltype.Signed, llmemory.Address], lltype.Void)) self.write_barrier_descr = WriteBarrierDescr(self) - # + + def _make_functions(self, really_not_translated): + from pypy.rpython.memory.gctypelayout import check_typeid + llop1 = self.llop1 + (self.standard_array_basesize, _, self.standard_array_length_ofs) = \ + symbolic.get_array_token(lltype.GcArray(lltype.Signed), + not really_not_translated) + + def malloc_nursery_slowpath(size): + """Allocate 'size' null bytes out of the nursery. + Note that the fast path is typically inlined by the backend.""" + if self.DEBUG: + self._random_usage_of_xmm_registers() + type_id = rffi.cast(llgroup.HALFWORD, 0) # missing here + return llop1.do_malloc_fixedsize_clear(llmemory.GCREF, + type_id, size, + False, False, False) + self.generate_function('malloc_nursery', malloc_nursery_slowpath, + [lltype.Signed]) + def malloc_array(itemsize, tid, num_elem): + """Allocate an array with a variable-size num_elem. + Only works for standard arrays.""" type_id = llop.extract_ushort(llgroup.HALFWORD, tid) check_typeid(type_id) return llop1.do_malloc_varsize_clear( llmemory.GCREF, - type_id, num_elem, self.array_basesize, itemsize, - self.array_length_ofs) - self.malloc_array = malloc_array - self.GC_MALLOC_ARRAY = lltype.Ptr(lltype.FuncType( - [lltype.Signed] * 3, llmemory.GCREF)) - # - (str_basesize, str_itemsize, str_ofs_length - ) = symbolic.get_array_token(rstr.STR, True) - (unicode_basesize, unicode_itemsize, unicode_ofs_length - ) = symbolic.get_array_token(rstr.UNICODE, True) - str_type_id = self.layoutbuilder.get_type_id(rstr.STR) - unicode_type_id = self.layoutbuilder.get_type_id(rstr.UNICODE) - # + type_id, num_elem, self.standard_array_basesize, itemsize, + self.standard_array_length_ofs) + self.generate_function('malloc_array', malloc_array, + [lltype.Signed] * 3) + + def malloc_array_nonstandard(basesize, itemsize, lengthofs, tid, + num_elem): + """For the rare case of non-standard arrays, i.e. arrays where + self.standard_array_{basesize,length_ofs} is wrong. It can + occur e.g. with arrays of floats on Win32.""" + type_id = llop.extract_ushort(llgroup.HALFWORD, tid) + check_typeid(type_id) + return llop1.do_malloc_varsize_clear( + llmemory.GCREF, + type_id, num_elem, basesize, itemsize, lengthofs) + self.generate_function('malloc_array_nonstandard', + malloc_array_nonstandard, + [lltype.Signed] * 5) + + str_type_id = self.str_descr.tid + str_basesize = self.str_descr.basesize + str_itemsize = self.str_descr.itemsize + str_ofs_length = self.str_descr.lendescr.offset + unicode_type_id = self.unicode_descr.tid + unicode_basesize = self.unicode_descr.basesize + unicode_itemsize = self.unicode_descr.itemsize + unicode_ofs_length = self.unicode_descr.lendescr.offset + def malloc_str(length): return llop1.do_malloc_varsize_clear( llmemory.GCREF, str_type_id, length, str_basesize, str_itemsize, str_ofs_length) + self.generate_function('malloc_str', malloc_str, + [lltype.Signed]) + def malloc_unicode(length): return llop1.do_malloc_varsize_clear( llmemory.GCREF, - unicode_type_id, length, unicode_basesize,unicode_itemsize, + unicode_type_id, length, unicode_basesize, unicode_itemsize, unicode_ofs_length) - self.malloc_str = malloc_str - self.malloc_unicode = malloc_unicode - self.GC_MALLOC_STR_UNICODE = lltype.Ptr(lltype.FuncType( - [lltype.Signed], llmemory.GCREF)) - # - class ForTestOnly: - pass - for_test_only = ForTestOnly() - for_test_only.x = 1.23 - def random_usage_of_xmm_registers(): - x0 = for_test_only.x - x1 = x0 * 0.1 - x2 = x0 * 0.2 - x3 = x0 * 0.3 - for_test_only.x = x0 + x1 + x2 + x3 - # - def malloc_slowpath(size): - if self.DEBUG: - random_usage_of_xmm_registers() - assert size >= self.minimal_size_in_nursery - # NB. although we call do_malloc_fixedsize_clear() here, - # it's a bit of a hack because we set tid to 0 and may - # also use it to allocate varsized objects. The tid - # and possibly the length are both set afterward. - gcref = llop1.do_malloc_fixedsize_clear(llmemory.GCREF, - 0, size, False, False, False) - return rffi.cast(lltype.Signed, gcref) - self.malloc_slowpath = malloc_slowpath - self.MALLOC_SLOWPATH = lltype.FuncType([lltype.Signed], lltype.Signed) + self.generate_function('malloc_unicode', malloc_unicode, + [lltype.Signed]) + + # Rarely called: allocate a fixed-size amount of bytes, but + # not in the nursery, because it is too big. Implemented like + # malloc_nursery_slowpath() above. + self.generate_function('malloc_fixedsize', malloc_nursery_slowpath, + [lltype.Signed]) + + def _bh_malloc(self, sizedescr): + from pypy.rpython.memory.gctypelayout import check_typeid + llop1 = self.llop1 + type_id = llop.extract_ushort(llgroup.HALFWORD, sizedescr.tid) + check_typeid(type_id) + return llop1.do_malloc_fixedsize_clear(llmemory.GCREF, + type_id, sizedescr.size, + False, False, False) + + def _bh_malloc_array(self, arraydescr, num_elem): + from pypy.rpython.memory.gctypelayout import check_typeid + llop1 = self.llop1 + type_id = llop.extract_ushort(llgroup.HALFWORD, arraydescr.tid) + check_typeid(type_id) + return llop1.do_malloc_varsize_clear(llmemory.GCREF, + type_id, num_elem, + arraydescr.basesize, + arraydescr.itemsize, + arraydescr.lendescr.offset) + + + class ForTestOnly: + pass + for_test_only = ForTestOnly() + for_test_only.x = 1.23 + + def _random_usage_of_xmm_registers(self): + x0 = self.for_test_only.x + x1 = x0 * 0.1 + x2 = x0 * 0.2 + x3 = x0 * 0.3 + self.for_test_only.x = x0 + x1 + x2 + x3 def get_nursery_free_addr(self): nurs_addr = llop.gc_adr_of_nursery_free(llmemory.Address) @@ -735,49 +813,26 @@ nurs_top_addr = llop.gc_adr_of_nursery_top(llmemory.Address) return rffi.cast(lltype.Signed, nurs_top_addr) - def get_malloc_slowpath_addr(self): - fptr = llhelper(lltype.Ptr(self.MALLOC_SLOWPATH), self.malloc_slowpath) - return rffi.cast(lltype.Signed, fptr) - def initialize(self): self.gcrootmap.initialize() def init_size_descr(self, S, descr): - type_id = self.layoutbuilder.get_type_id(S) - assert not self.layoutbuilder.is_weakref_type(S) - assert not self.layoutbuilder.has_finalizer(S) - descr.tid = llop.combine_ushort(lltype.Signed, type_id, 0) + if self.layoutbuilder is not None: + type_id = self.layoutbuilder.get_type_id(S) + assert not self.layoutbuilder.is_weakref_type(S) + assert not self.layoutbuilder.has_finalizer(S) + descr.tid = llop.combine_ushort(lltype.Signed, type_id, 0) def init_array_descr(self, A, descr): - type_id = self.layoutbuilder.get_type_id(A) - descr.tid = llop.combine_ushort(lltype.Signed, type_id, 0) + if self.layoutbuilder is not None: + type_id = self.layoutbuilder.get_type_id(A) + descr.tid = llop.combine_ushort(lltype.Signed, type_id, 0) - def gc_malloc(self, sizedescr): - assert isinstance(sizedescr, BaseSizeDescr) - return self.malloc_basic(sizedescr.size, sizedescr.tid) - - def gc_malloc_array(self, arraydescr, num_elem): - assert isinstance(arraydescr, BaseArrayDescr) - itemsize = arraydescr.get_item_size(self.translate_support_code) - return self.malloc_array(itemsize, arraydescr.tid, num_elem) - - def gc_malloc_str(self, num_elem): - return self.malloc_str(num_elem) - - def gc_malloc_unicode(self, num_elem): - return self.malloc_unicode(num_elem) - - def args_for_new(self, sizedescr): - assert isinstance(sizedescr, BaseSizeDescr) - return [sizedescr.size, sizedescr.tid] - - def args_for_new_array(self, arraydescr): - assert isinstance(arraydescr, BaseArrayDescr) - itemsize = arraydescr.get_item_size(self.translate_support_code) - return [itemsize, arraydescr.tid] - - def get_funcptr_for_new(self): - return llhelper(self.GC_MALLOC_BASIC, self.malloc_basic) + def _set_tid(self, gcptr, tid): + hdr_addr = llmemory.cast_ptr_to_adr(gcptr) + hdr_addr -= self.gcheaderbuilder.size_gc_header + hdr = llmemory.cast_adr_to_ptr(hdr_addr, self.HDRPTR) + hdr.tid = tid def do_write_barrier(self, gcref_struct, gcref_newptr): hdr_addr = llmemory.cast_ptr_to_adr(gcref_struct) @@ -791,108 +846,8 @@ funcptr(llmemory.cast_ptr_to_adr(gcref_struct), llmemory.cast_ptr_to_adr(gcref_newptr)) - def rewrite_assembler(self, cpu, operations, gcrefs_output_list): - # Perform two kinds of rewrites in parallel: - # - # - Add COND_CALLs to the write barrier before SETFIELD_GC and - # SETARRAYITEM_GC operations. - # - # - Record the ConstPtrs from the assembler. - # - newops = [] - known_lengths = {} - # we can only remember one malloc since the next malloc can possibly - # collect - last_malloc = None - for op in operations: - if op.getopnum() == rop.DEBUG_MERGE_POINT: - continue - # ---------- record the ConstPtrs ---------- - self.record_constptrs(op, gcrefs_output_list) - if op.is_malloc(): - last_malloc = op.result - elif op.can_malloc(): - last_malloc = None - # ---------- write barrier for SETFIELD_GC ---------- - if op.getopnum() == rop.SETFIELD_GC: - val = op.getarg(0) - # no need for a write barrier in the case of previous malloc - if val is not last_malloc: - v = op.getarg(1) - if isinstance(v, BoxPtr) or (isinstance(v, ConstPtr) and - bool(v.value)): # store a non-NULL - self._gen_write_barrier(newops, op.getarg(0), v) - op = op.copy_and_change(rop.SETFIELD_RAW) - # ---------- write barrier for SETINTERIORFIELD_GC ------ - if op.getopnum() == rop.SETINTERIORFIELD_GC: - val = op.getarg(0) - if val is not last_malloc: - v = op.getarg(2) - if isinstance(v, BoxPtr) or (isinstance(v, ConstPtr) and - bool(v.value)): # store a non-NULL - self._gen_write_barrier(newops, op.getarg(0), v) - op = op.copy_and_change(rop.SETINTERIORFIELD_RAW) - # ---------- write barrier for SETARRAYITEM_GC ---------- - if op.getopnum() == rop.SETARRAYITEM_GC: - val = op.getarg(0) - # no need for a write barrier in the case of previous malloc - if val is not last_malloc: - v = op.getarg(2) - if isinstance(v, BoxPtr) or (isinstance(v, ConstPtr) and - bool(v.value)): # store a non-NULL - self._gen_write_barrier_array(newops, op.getarg(0), - op.getarg(1), v, - cpu, known_lengths) - op = op.copy_and_change(rop.SETARRAYITEM_RAW) - elif op.getopnum() == rop.NEW_ARRAY: - v_length = op.getarg(0) - if isinstance(v_length, ConstInt): - known_lengths[op.result] = v_length.getint() - # ---------- - newops.append(op) - return newops - - def _gen_write_barrier(self, newops, v_base, v_value): - args = [v_base, v_value] - newops.append(ResOperation(rop.COND_CALL_GC_WB, args, None, - descr=self.write_barrier_descr)) - - def _gen_write_barrier_array(self, newops, v_base, v_index, v_value, - cpu, known_lengths): - if self.write_barrier_descr.get_write_barrier_from_array_fn(cpu) != 0: - # If we know statically the length of 'v', and it is not too - # big, then produce a regular write_barrier. If it's unknown or - # too big, produce instead a write_barrier_from_array. - LARGE = 130 - length = known_lengths.get(v_base, LARGE) - if length >= LARGE: - # unknown or too big: produce a write_barrier_from_array - args = [v_base, v_index, v_value] - newops.append(ResOperation(rop.COND_CALL_GC_WB_ARRAY, args, - None, - descr=self.write_barrier_descr)) - return - # fall-back case: produce a write_barrier - self._gen_write_barrier(newops, v_base, v_value) - - def can_inline_malloc(self, descr): - assert isinstance(descr, BaseSizeDescr) - if descr.size < self.max_size_of_young_obj: - has_finalizer = bool(descr.tid & (1<= LARGE: + # unknown or too big: produce a write_barrier_from_array + args = [v_base, v_index, v_value] + self.newops.append( + ResOperation(rop.COND_CALL_GC_WB_ARRAY, args, None, + descr=write_barrier_descr)) + return + # fall-back case: produce a write_barrier + self.gen_write_barrier(v_base, v_value) + + def round_up_for_allocation(self, size): + if not self.gc_ll_descr.round_up: + return size + if self.gc_ll_descr.translate_support_code: + from pypy.rpython.lltypesystem import llarena + return llarena.round_up_for_allocation( + size, self.gc_ll_descr.minimal_size_in_nursery) + else: + # non-translated: do it manually + # assume that "self.gc_ll_descr.minimal_size_in_nursery" is 2 WORDs + size = max(size, 2 * WORD) + return (size + WORD-1) & ~(WORD-1) # round up diff --git a/pypy/jit/backend/llsupport/test/test_descr.py b/pypy/jit/backend/llsupport/test/test_descr.py --- a/pypy/jit/backend/llsupport/test/test_descr.py +++ b/pypy/jit/backend/llsupport/test/test_descr.py @@ -1,4 +1,4 @@ -from pypy.rpython.lltypesystem import lltype, rffi +from pypy.rpython.lltypesystem import lltype, rffi, rstr from pypy.jit.backend.llsupport.descr import * from pypy.jit.backend.llsupport import symbolic from pypy.rlib.objectmodel import Symbolic @@ -53,18 +53,6 @@ ('z', lltype.Ptr(U)), ('f', lltype.Float), ('s', lltype.SingleFloat)) - assert getFieldDescrClass(lltype.Ptr(T)) is GcPtrFieldDescr - assert getFieldDescrClass(lltype.Ptr(U)) is NonGcPtrFieldDescr - cls = getFieldDescrClass(lltype.Char) - assert cls != getFieldDescrClass(lltype.Signed) - assert cls == getFieldDescrClass(lltype.Char) - clsf = getFieldDescrClass(lltype.Float) - assert clsf != cls - assert clsf == getFieldDescrClass(lltype.Float) - clss = getFieldDescrClass(lltype.SingleFloat) - assert clss not in (cls, clsf) - assert clss == getFieldDescrClass(lltype.SingleFloat) - assert clss == getFieldDescrClass(rffi.UINT) # for now # c0 = GcCache(False) c1 = GcCache(True) @@ -77,11 +65,7 @@ descr_z = get_field_descr(c2, S, 'z') descr_f = get_field_descr(c2, S, 'f') descr_s = get_field_descr(c2, S, 's') - assert descr_x.__class__ is cls - assert descr_y.__class__ is GcPtrFieldDescr - assert descr_z.__class__ is NonGcPtrFieldDescr - assert descr_f.__class__ is clsf - assert descr_s.__class__ is clss + assert isinstance(descr_x, FieldDescr) assert descr_x.name == 'S.x' assert descr_y.name == 'S.y' assert descr_z.name == 'S.z' @@ -90,33 +74,27 @@ if not tsc: assert descr_x.offset < descr_y.offset < descr_z.offset assert descr_x.sort_key() < descr_y.sort_key() < descr_z.sort_key() - assert descr_x.get_field_size(False) == rffi.sizeof(lltype.Char) - assert descr_y.get_field_size(False) == rffi.sizeof(lltype.Ptr(T)) - assert descr_z.get_field_size(False) == rffi.sizeof(lltype.Ptr(U)) - assert descr_f.get_field_size(False) == rffi.sizeof(lltype.Float) - assert descr_s.get_field_size(False) == rffi.sizeof( - lltype.SingleFloat) + assert descr_x.field_size == rffi.sizeof(lltype.Char) + assert descr_y.field_size == rffi.sizeof(lltype.Ptr(T)) + assert descr_z.field_size == rffi.sizeof(lltype.Ptr(U)) + assert descr_f.field_size == rffi.sizeof(lltype.Float) + assert descr_s.field_size == rffi.sizeof(lltype.SingleFloat) else: assert isinstance(descr_x.offset, Symbolic) assert isinstance(descr_y.offset, Symbolic) assert isinstance(descr_z.offset, Symbolic) assert isinstance(descr_f.offset, Symbolic) assert isinstance(descr_s.offset, Symbolic) - assert isinstance(descr_x.get_field_size(True), Symbolic) - assert isinstance(descr_y.get_field_size(True), Symbolic) - assert isinstance(descr_z.get_field_size(True), Symbolic) - assert isinstance(descr_f.get_field_size(True), Symbolic) - assert isinstance(descr_s.get_field_size(True), Symbolic) - assert not descr_x.is_pointer_field() - assert descr_y.is_pointer_field() - assert not descr_z.is_pointer_field() - assert not descr_f.is_pointer_field() - assert not descr_s.is_pointer_field() - assert not descr_x.is_float_field() - assert not descr_y.is_float_field() - assert not descr_z.is_float_field() - assert descr_f.is_float_field() - assert not descr_s.is_float_field() + assert isinstance(descr_x.field_size, Symbolic) + assert isinstance(descr_y.field_size, Symbolic) + assert isinstance(descr_z.field_size, Symbolic) + assert isinstance(descr_f.field_size, Symbolic) + assert isinstance(descr_s.field_size, Symbolic) + assert descr_x.flag == FLAG_UNSIGNED + assert descr_y.flag == FLAG_POINTER + assert descr_z.flag == FLAG_UNSIGNED + assert descr_f.flag == FLAG_FLOAT + assert descr_s.flag == FLAG_UNSIGNED def test_get_field_descr_sign(): @@ -128,7 +106,8 @@ for tsc in [False, True]: c2 = GcCache(tsc) descr_x = get_field_descr(c2, S, 'x') - assert descr_x.is_field_signed() == signed + assert descr_x.flag == {False: FLAG_UNSIGNED, + True: FLAG_SIGNED }[signed] def test_get_field_descr_longlong(): if sys.maxint > 2147483647: @@ -136,9 +115,8 @@ c0 = GcCache(False) S = lltype.GcStruct('S', ('y', lltype.UnsignedLongLong)) descr = get_field_descr(c0, S, 'y') - assert not descr.is_pointer_field() - assert descr.is_float_field() - assert descr.get_field_size(False) == 8 + assert descr.flag == FLAG_FLOAT + assert descr.field_size == 8 def test_get_array_descr(): @@ -149,19 +127,8 @@ A3 = lltype.GcArray(lltype.Ptr(U)) A4 = lltype.GcArray(lltype.Float) A5 = lltype.GcArray(lltype.Struct('x', ('v', lltype.Signed), - ('k', lltype.Signed))) + ('k', lltype.Signed))) A6 = lltype.GcArray(lltype.SingleFloat) - assert getArrayDescrClass(A2) is GcPtrArrayDescr - assert getArrayDescrClass(A3) is NonGcPtrArrayDescr - cls = getArrayDescrClass(A1) - assert cls != getArrayDescrClass(lltype.GcArray(lltype.Signed)) - assert cls == getArrayDescrClass(lltype.GcArray(lltype.Char)) - clsf = getArrayDescrClass(A4) - assert clsf != cls - assert clsf == getArrayDescrClass(lltype.GcArray(lltype.Float)) - clss = getArrayDescrClass(A6) - assert clss not in (clsf, cls) - assert clss == getArrayDescrClass(lltype.GcArray(rffi.UINT)) # c0 = GcCache(False) descr1 = get_array_descr(c0, A1) @@ -170,82 +137,61 @@ descr4 = get_array_descr(c0, A4) descr5 = get_array_descr(c0, A5) descr6 = get_array_descr(c0, A6) - assert descr1.__class__ is cls - assert descr2.__class__ is GcPtrArrayDescr - assert descr3.__class__ is NonGcPtrArrayDescr - assert descr4.__class__ is clsf - assert descr6.__class__ is clss + assert isinstance(descr1, ArrayDescr) assert descr1 == get_array_descr(c0, lltype.GcArray(lltype.Char)) - assert not descr1.is_array_of_pointers() - assert descr2.is_array_of_pointers() - assert not descr3.is_array_of_pointers() - assert not descr4.is_array_of_pointers() - assert not descr5.is_array_of_pointers() - assert not descr1.is_array_of_floats() - assert not descr2.is_array_of_floats() - assert not descr3.is_array_of_floats() - assert descr4.is_array_of_floats() - assert not descr5.is_array_of_floats() + assert descr1.flag == FLAG_UNSIGNED + assert descr2.flag == FLAG_POINTER + assert descr3.flag == FLAG_UNSIGNED + assert descr4.flag == FLAG_FLOAT + assert descr5.flag == FLAG_STRUCT + assert descr6.flag == FLAG_UNSIGNED # def get_alignment(code): # Retrieve default alignment for the compiler/platform return struct.calcsize('l' + code) - struct.calcsize(code) - assert descr1.get_base_size(False) == get_alignment('c') - assert descr2.get_base_size(False) == get_alignment('p') - assert descr3.get_base_size(False) == get_alignment('p') - assert descr4.get_base_size(False) == get_alignment('d') - assert descr5.get_base_size(False) == get_alignment('f') - assert descr1.get_ofs_length(False) == 0 - assert descr2.get_ofs_length(False) == 0 - assert descr3.get_ofs_length(False) == 0 - assert descr4.get_ofs_length(False) == 0 - assert descr5.get_ofs_length(False) == 0 - assert descr1.get_item_size(False) == rffi.sizeof(lltype.Char) - assert descr2.get_item_size(False) == rffi.sizeof(lltype.Ptr(T)) - assert descr3.get_item_size(False) == rffi.sizeof(lltype.Ptr(U)) - assert descr4.get_item_size(False) == rffi.sizeof(lltype.Float) - assert descr5.get_item_size(False) == rffi.sizeof(lltype.Signed) * 2 - assert descr6.get_item_size(False) == rffi.sizeof(lltype.SingleFloat) + assert descr1.basesize == get_alignment('c') + assert descr2.basesize == get_alignment('p') + assert descr3.basesize == get_alignment('p') + assert descr4.basesize == get_alignment('d') + assert descr5.basesize == get_alignment('f') + assert descr1.lendescr.offset == 0 + assert descr2.lendescr.offset == 0 + assert descr3.lendescr.offset == 0 + assert descr4.lendescr.offset == 0 + assert descr5.lendescr.offset == 0 + assert descr1.itemsize == rffi.sizeof(lltype.Char) + assert descr2.itemsize == rffi.sizeof(lltype.Ptr(T)) + assert descr3.itemsize == rffi.sizeof(lltype.Ptr(U)) + assert descr4.itemsize == rffi.sizeof(lltype.Float) + assert descr5.itemsize == rffi.sizeof(lltype.Signed) * 2 + assert descr6.itemsize == rffi.sizeof(lltype.SingleFloat) # - assert isinstance(descr1.get_base_size(True), Symbolic) - assert isinstance(descr2.get_base_size(True), Symbolic) - assert isinstance(descr3.get_base_size(True), Symbolic) - assert isinstance(descr4.get_base_size(True), Symbolic) - assert isinstance(descr5.get_base_size(True), Symbolic) - assert isinstance(descr1.get_ofs_length(True), Symbolic) - assert isinstance(descr2.get_ofs_length(True), Symbolic) - assert isinstance(descr3.get_ofs_length(True), Symbolic) - assert isinstance(descr4.get_ofs_length(True), Symbolic) - assert isinstance(descr5.get_ofs_length(True), Symbolic) - assert isinstance(descr1.get_item_size(True), Symbolic) - assert isinstance(descr2.get_item_size(True), Symbolic) - assert isinstance(descr3.get_item_size(True), Symbolic) - assert isinstance(descr4.get_item_size(True), Symbolic) - assert isinstance(descr5.get_item_size(True), Symbolic) CA = rffi.CArray(lltype.Signed) descr = get_array_descr(c0, CA) - assert not descr.is_array_of_floats() - assert descr.get_base_size(False) == 0 - assert descr.get_ofs_length(False) == -1 + assert descr.flag == FLAG_SIGNED + assert descr.basesize == 0 + assert descr.lendescr is None CA = rffi.CArray(lltype.Ptr(lltype.GcStruct('S'))) descr = get_array_descr(c0, CA) - assert descr.is_array_of_pointers() - assert descr.get_base_size(False) == 0 - assert descr.get_ofs_length(False) == -1 + assert descr.flag == FLAG_POINTER + assert descr.basesize == 0 + assert descr.lendescr is None CA = rffi.CArray(lltype.Ptr(lltype.Struct('S'))) descr = get_array_descr(c0, CA) - assert descr.get_base_size(False) == 0 - assert descr.get_ofs_length(False) == -1 + assert descr.flag == FLAG_UNSIGNED + assert descr.basesize == 0 + assert descr.lendescr is None CA = rffi.CArray(lltype.Float) descr = get_array_descr(c0, CA) - assert descr.is_array_of_floats() - assert descr.get_base_size(False) == 0 - assert descr.get_ofs_length(False) == -1 + assert descr.flag == FLAG_FLOAT + assert descr.basesize == 0 + assert descr.lendescr is None CA = rffi.CArray(rffi.FLOAT) descr = get_array_descr(c0, CA) - assert not descr.is_array_of_floats() - assert descr.get_base_size(False) == 0 - assert descr.get_ofs_length(False) == -1 + assert descr.flag == FLAG_UNSIGNED + assert descr.basesize == 0 + assert descr.itemsize == rffi.sizeof(lltype.SingleFloat) + assert descr.lendescr is None def test_get_array_descr_sign(): @@ -257,46 +203,55 @@ for tsc in [False, True]: c2 = GcCache(tsc) arraydescr = get_array_descr(c2, A) - assert arraydescr.is_item_signed() == signed + assert arraydescr.flag == {False: FLAG_UNSIGNED, + True: FLAG_SIGNED }[signed] # RA = rffi.CArray(RESTYPE) for tsc in [False, True]: c2 = GcCache(tsc) arraydescr = get_array_descr(c2, RA) - assert arraydescr.is_item_signed() == signed + assert arraydescr.flag == {False: FLAG_UNSIGNED, + True: FLAG_SIGNED }[signed] + + +def test_get_array_descr_str(): + c0 = GcCache(False) + descr1 = get_array_descr(c0, rstr.STR) + assert descr1.itemsize == rffi.sizeof(lltype.Char) + assert descr1.flag == FLAG_UNSIGNED def test_get_call_descr_not_translated(): c0 = GcCache(False) descr1 = get_call_descr(c0, [lltype.Char, lltype.Signed], lltype.Char) - assert descr1.get_result_size(False) == rffi.sizeof(lltype.Char) - assert descr1.get_return_type() == history.INT + assert descr1.get_result_size() == rffi.sizeof(lltype.Char) + assert descr1.get_result_type() == history.INT assert descr1.arg_classes == "ii" # T = lltype.GcStruct('T') descr2 = get_call_descr(c0, [lltype.Ptr(T)], lltype.Ptr(T)) - assert descr2.get_result_size(False) == rffi.sizeof(lltype.Ptr(T)) - assert descr2.get_return_type() == history.REF + assert descr2.get_result_size() == rffi.sizeof(lltype.Ptr(T)) + assert descr2.get_result_type() == history.REF assert descr2.arg_classes == "r" # U = lltype.GcStruct('U', ('x', lltype.Signed)) assert descr2 == get_call_descr(c0, [lltype.Ptr(U)], lltype.Ptr(U)) # V = lltype.Struct('V', ('x', lltype.Signed)) - assert (get_call_descr(c0, [], lltype.Ptr(V)).get_return_type() == + assert (get_call_descr(c0, [], lltype.Ptr(V)).get_result_type() == history.INT) # - assert (get_call_descr(c0, [], lltype.Void).get_return_type() == + assert (get_call_descr(c0, [], lltype.Void).get_result_type() == history.VOID) # descr4 = get_call_descr(c0, [lltype.Float, lltype.Float], lltype.Float) - assert descr4.get_result_size(False) == rffi.sizeof(lltype.Float) - assert descr4.get_return_type() == history.FLOAT + assert descr4.get_result_size() == rffi.sizeof(lltype.Float) + assert descr4.get_result_type() == history.FLOAT assert descr4.arg_classes == "ff" # descr5 = get_call_descr(c0, [lltype.SingleFloat], lltype.SingleFloat) - assert descr5.get_result_size(False) == rffi.sizeof(lltype.SingleFloat) - assert descr5.get_return_type() == "S" + assert descr5.get_result_size() == rffi.sizeof(lltype.SingleFloat) + assert descr5.get_result_type() == "S" assert descr5.arg_classes == "S" def test_get_call_descr_not_translated_longlong(): @@ -305,13 +260,13 @@ c0 = GcCache(False) # descr5 = get_call_descr(c0, [lltype.SignedLongLong], lltype.Signed) - assert descr5.get_result_size(False) == 4 - assert descr5.get_return_type() == history.INT + assert descr5.get_result_size() == 4 + assert descr5.get_result_type() == history.INT assert descr5.arg_classes == "L" # descr6 = get_call_descr(c0, [lltype.Signed], lltype.SignedLongLong) - assert descr6.get_result_size(False) == 8 - assert descr6.get_return_type() == "L" + assert descr6.get_result_size() == 8 + assert descr6.get_result_type() == "L" assert descr6.arg_classes == "i" def test_get_call_descr_translated(): @@ -319,18 +274,18 @@ T = lltype.GcStruct('T') U = lltype.GcStruct('U', ('x', lltype.Signed)) descr3 = get_call_descr(c1, [lltype.Ptr(T)], lltype.Ptr(U)) - assert isinstance(descr3.get_result_size(True), Symbolic) - assert descr3.get_return_type() == history.REF + assert isinstance(descr3.get_result_size(), Symbolic) + assert descr3.get_result_type() == history.REF assert descr3.arg_classes == "r" # descr4 = get_call_descr(c1, [lltype.Float, lltype.Float], lltype.Float) - assert isinstance(descr4.get_result_size(True), Symbolic) - assert descr4.get_return_type() == history.FLOAT + assert isinstance(descr4.get_result_size(), Symbolic) + assert descr4.get_result_type() == history.FLOAT assert descr4.arg_classes == "ff" # descr5 = get_call_descr(c1, [lltype.SingleFloat], lltype.SingleFloat) - assert isinstance(descr5.get_result_size(True), Symbolic) - assert descr5.get_return_type() == "S" + assert isinstance(descr5.get_result_size(), Symbolic) + assert descr5.get_result_type() == "S" assert descr5.arg_classes == "S" def test_call_descr_extra_info(): @@ -358,6 +313,10 @@ def test_repr_of_descr(): + def repr_of_descr(descr): + s = descr.repr_of_descr() + assert ',' not in s # makes the life easier for pypy.tool.jitlogparser + return s c0 = GcCache(False) T = lltype.GcStruct('T') S = lltype.GcStruct('S', ('x', lltype.Char), @@ -365,33 +324,34 @@ ('z', lltype.Ptr(T))) descr1 = get_size_descr(c0, S) s = symbolic.get_size(S, False) - assert descr1.repr_of_descr() == '' % s + assert repr_of_descr(descr1) == '' % s # descr2 = get_field_descr(c0, S, 'y') o, _ = symbolic.get_field_token(S, 'y', False) - assert descr2.repr_of_descr() == '' % o + assert repr_of_descr(descr2) == '' % o # descr2i = get_field_descr(c0, S, 'x') o, _ = symbolic.get_field_token(S, 'x', False) - assert descr2i.repr_of_descr() == '' % o + assert repr_of_descr(descr2i) == '' % o # descr3 = get_array_descr(c0, lltype.GcArray(lltype.Ptr(S))) - assert descr3.repr_of_descr() == '' + o = symbolic.get_size(lltype.Ptr(S), False) + assert repr_of_descr(descr3) == '' % o # descr3i = get_array_descr(c0, lltype.GcArray(lltype.Char)) - assert descr3i.repr_of_descr() == '' + assert repr_of_descr(descr3i) == '' # descr4 = get_call_descr(c0, [lltype.Char, lltype.Ptr(S)], lltype.Ptr(S)) - assert 'GcPtrCallDescr' in descr4.repr_of_descr() + assert repr_of_descr(descr4) == '' % o # descr4i = get_call_descr(c0, [lltype.Char, lltype.Ptr(S)], lltype.Char) - assert 'CharCallDescr' in descr4i.repr_of_descr() + assert repr_of_descr(descr4i) == '' # descr4f = get_call_descr(c0, [lltype.Char, lltype.Ptr(S)], lltype.Float) - assert 'FloatCallDescr' in descr4f.repr_of_descr() + assert repr_of_descr(descr4f) == '' # descr5f = get_call_descr(c0, [lltype.Char], lltype.SingleFloat) - assert 'SingleFloatCallDescr' in descr5f.repr_of_descr() + assert repr_of_descr(descr5f) == '' def test_call_stubs_1(): c0 = GcCache(False) @@ -401,10 +361,10 @@ def f(a, b): return 'c' - call_stub = descr1.call_stub fnptr = llhelper(lltype.Ptr(lltype.FuncType(ARGS, RES)), f) - res = call_stub(rffi.cast(lltype.Signed, fnptr), [1, 2], None, None) + res = descr1.call_stub_i(rffi.cast(lltype.Signed, fnptr), + [1, 2], None, None) assert res == ord('c') def test_call_stubs_2(): @@ -421,8 +381,8 @@ a = lltype.malloc(ARRAY, 3) opaquea = lltype.cast_opaque_ptr(llmemory.GCREF, a) a[0] = 1 - res = descr2.call_stub(rffi.cast(lltype.Signed, fnptr), - [], [opaquea], [longlong.getfloatstorage(3.5)]) + res = descr2.call_stub_f(rffi.cast(lltype.Signed, fnptr), + [], [opaquea], [longlong.getfloatstorage(3.5)]) assert longlong.getrealfloat(res) == 4.5 def test_call_stubs_single_float(): @@ -445,6 +405,22 @@ a = intmask(singlefloat2uint(r_singlefloat(-10.0))) b = intmask(singlefloat2uint(r_singlefloat(3.0))) c = intmask(singlefloat2uint(r_singlefloat(2.0))) - res = descr2.call_stub(rffi.cast(lltype.Signed, fnptr), - [a, b, c], [], []) + res = descr2.call_stub_i(rffi.cast(lltype.Signed, fnptr), + [a, b, c], [], []) assert float(uint2singlefloat(rffi.r_uint(res))) == -11.5 + +def test_field_arraylen_descr(): + c0 = GcCache(True) + A1 = lltype.GcArray(lltype.Signed) + fielddescr = get_field_arraylen_descr(c0, A1) + assert isinstance(fielddescr, FieldDescr) + ofs = fielddescr.offset + assert repr(ofs) == '< ArrayLengthOffset >' + # + fielddescr = get_field_arraylen_descr(c0, rstr.STR) + ofs = fielddescr.offset + assert repr(ofs) == ("< " + " 'chars'> + < ArrayLengthOffset" + " > >") + # caching: + assert fielddescr is get_field_arraylen_descr(c0, rstr.STR) diff --git a/pypy/jit/backend/llsupport/test/test_ffisupport.py b/pypy/jit/backend/llsupport/test/test_ffisupport.py --- a/pypy/jit/backend/llsupport/test/test_ffisupport.py +++ b/pypy/jit/backend/llsupport/test/test_ffisupport.py @@ -1,5 +1,6 @@ from pypy.rlib.libffi import types from pypy.jit.codewriter.longlong import is_64_bit +from pypy.jit.backend.llsupport.descr import * from pypy.jit.backend.llsupport.ffisupport import * @@ -15,7 +16,9 @@ args = [types.sint, types.pointer] descr = get_call_descr_dynamic(FakeCPU(), args, types.sint, None, ffi_flags=42) - assert isinstance(descr, DynamicIntCallDescr) + assert isinstance(descr, CallDescr) + assert descr.result_type == 'i' + assert descr.result_flag == FLAG_SIGNED assert descr.arg_classes == 'ii' assert descr.get_ffi_flags() == 42 @@ -24,18 +27,20 @@ assert descr is None # missing floats descr = get_call_descr_dynamic(FakeCPU(supports_floats=True), args, types.void, None, ffi_flags=43) - assert isinstance(descr, VoidCallDescr) + assert descr.result_type == 'v' + assert descr.result_flag == FLAG_VOID assert descr.arg_classes == 'ifi' assert descr.get_ffi_flags() == 43 descr = get_call_descr_dynamic(FakeCPU(), [], types.sint8, None, 42) - assert isinstance(descr, DynamicIntCallDescr) - assert descr.get_result_size(False) == 1 + assert descr.get_result_size() == 1 + assert descr.result_flag == FLAG_SIGNED assert descr.is_result_signed() == True descr = get_call_descr_dynamic(FakeCPU(), [], types.uint8, None, 42) - assert isinstance(descr, DynamicIntCallDescr) - assert descr.get_result_size(False) == 1 + assert isinstance(descr, CallDescr) + assert descr.get_result_size() == 1 + assert descr.result_flag == FLAG_UNSIGNED assert descr.is_result_signed() == False if not is_64_bit: @@ -44,7 +49,9 @@ assert descr is None # missing longlongs descr = get_call_descr_dynamic(FakeCPU(supports_longlong=True), [], types.slonglong, None, ffi_flags=43) - assert isinstance(descr, LongLongCallDescr) + assert isinstance(descr, CallDescr) + assert descr.result_flag == FLAG_FLOAT + assert descr.result_type == 'L' assert descr.get_ffi_flags() == 43 else: assert types.slonglong is types.slong @@ -53,6 +60,6 @@ assert descr is None # missing singlefloats descr = get_call_descr_dynamic(FakeCPU(supports_singlefloats=True), [], types.float, None, ffi_flags=44) - SingleFloatCallDescr = getCallDescrClass(rffi.FLOAT) - assert isinstance(descr, SingleFloatCallDescr) + assert descr.result_flag == FLAG_UNSIGNED + assert descr.result_type == 'S' assert descr.get_ffi_flags() == 44 diff --git a/pypy/jit/backend/llsupport/test/test_gc.py b/pypy/jit/backend/llsupport/test/test_gc.py --- a/pypy/jit/backend/llsupport/test/test_gc.py +++ b/pypy/jit/backend/llsupport/test/test_gc.py @@ -6,6 +6,7 @@ from pypy.jit.backend.llsupport.gc import * from pypy.jit.backend.llsupport import symbolic from pypy.jit.metainterp.gc import get_description +from pypy.jit.metainterp.history import BoxPtr, BoxInt, ConstPtr from pypy.jit.metainterp.resoperation import get_deep_immutable_oplist from pypy.jit.tool.oparser import parse from pypy.rpython.lltypesystem.rclass import OBJECT, OBJECT_VTABLE @@ -15,12 +16,12 @@ gc_ll_descr = GcLLDescr_boehm(None, None, None) # record = [] - prev_funcptr_for_new = gc_ll_descr.funcptr_for_new - def my_funcptr_for_new(size): - p = prev_funcptr_for_new(size) + prev_malloc_fn_ptr = gc_ll_descr.malloc_fn_ptr + def my_malloc_fn_ptr(size): + p = prev_malloc_fn_ptr(size) record.append((size, p)) return p - gc_ll_descr.funcptr_for_new = my_funcptr_for_new + gc_ll_descr.malloc_fn_ptr = my_malloc_fn_ptr # # ---------- gc_malloc ---------- S = lltype.GcStruct('S', ('x', lltype.Signed)) @@ -32,8 +33,8 @@ A = lltype.GcArray(lltype.Signed) arraydescr = get_array_descr(gc_ll_descr, A) p = gc_ll_descr.gc_malloc_array(arraydescr, 10) - assert record == [(arraydescr.get_base_size(False) + - 10 * arraydescr.get_item_size(False), p)] + assert record == [(arraydescr.basesize + + 10 * arraydescr.itemsize, p)] del record[:] # ---------- gc_malloc_str ---------- p = gc_ll_descr.gc_malloc_str(10) @@ -246,24 +247,28 @@ def __init__(self): self.record = [] + def _malloc(self, type_id, size): + tid = llop.combine_ushort(lltype.Signed, type_id, 0) + x = llmemory.raw_malloc(self.gcheaderbuilder.size_gc_header + size) + x += self.gcheaderbuilder.size_gc_header + return x, tid + def do_malloc_fixedsize_clear(self, RESTYPE, type_id, size, has_finalizer, has_light_finalizer, contains_weakptr): assert not contains_weakptr - assert not has_finalizer # in these tests - assert not has_light_finalizer # in these tests - p = llmemory.raw_malloc(size) + assert not has_finalizer + assert not has_light_finalizer + p, tid = self._malloc(type_id, size) p = llmemory.cast_adr_to_ptr(p, RESTYPE) - tid = llop.combine_ushort(lltype.Signed, type_id, 0) self.record.append(("fixedsize", repr(size), tid, p)) return p def do_malloc_varsize_clear(self, RESTYPE, type_id, length, size, itemsize, offset_to_length): - p = llmemory.raw_malloc(size + itemsize * length) + p, tid = self._malloc(type_id, size + itemsize * length) (p + offset_to_length).signed[0] = length p = llmemory.cast_adr_to_ptr(p, RESTYPE) - tid = llop.combine_ushort(lltype.Signed, type_id, 0) self.record.append(("varsize", tid, length, repr(size), repr(itemsize), repr(offset_to_length), p)) @@ -322,43 +327,40 @@ gc_ll_descr = GcLLDescr_framework(gcdescr, FakeTranslator(), None, llop1) gc_ll_descr.initialize() + llop1.gcheaderbuilder = gc_ll_descr.gcheaderbuilder self.llop1 = llop1 self.gc_ll_descr = gc_ll_descr self.fake_cpu = FakeCPU() - def test_args_for_new(self): - S = lltype.GcStruct('S', ('x', lltype.Signed)) - sizedescr = get_size_descr(self.gc_ll_descr, S) - args = self.gc_ll_descr.args_for_new(sizedescr) - for x in args: - assert lltype.typeOf(x) == lltype.Signed - A = lltype.GcArray(lltype.Signed) - arraydescr = get_array_descr(self.gc_ll_descr, A) - args = self.gc_ll_descr.args_for_new(sizedescr) - for x in args: - assert lltype.typeOf(x) == lltype.Signed +## def test_args_for_new(self): +## S = lltype.GcStruct('S', ('x', lltype.Signed)) +## sizedescr = get_size_descr(self.gc_ll_descr, S) +## args = self.gc_ll_descr.args_for_new(sizedescr) +## for x in args: +## assert lltype.typeOf(x) == lltype.Signed +## A = lltype.GcArray(lltype.Signed) +## arraydescr = get_array_descr(self.gc_ll_descr, A) +## args = self.gc_ll_descr.args_for_new(sizedescr) +## for x in args: +## assert lltype.typeOf(x) == lltype.Signed def test_gc_malloc(self): S = lltype.GcStruct('S', ('x', lltype.Signed)) sizedescr = get_size_descr(self.gc_ll_descr, S) p = self.gc_ll_descr.gc_malloc(sizedescr) - assert self.llop1.record == [("fixedsize", - repr(sizedescr.size), + assert lltype.typeOf(p) == llmemory.GCREF + assert self.llop1.record == [("fixedsize", repr(sizedescr.size), sizedescr.tid, p)] - assert repr(self.gc_ll_descr.args_for_new(sizedescr)) == repr( - [sizedescr.size, sizedescr.tid]) def test_gc_malloc_array(self): A = lltype.GcArray(lltype.Signed) arraydescr = get_array_descr(self.gc_ll_descr, A) p = self.gc_ll_descr.gc_malloc_array(arraydescr, 10) assert self.llop1.record == [("varsize", arraydescr.tid, 10, - repr(arraydescr.get_base_size(True)), - repr(arraydescr.get_item_size(True)), - repr(arraydescr.get_ofs_length(True)), + repr(arraydescr.basesize), + repr(arraydescr.itemsize), + repr(arraydescr.lendescr.offset), p)] - assert repr(self.gc_ll_descr.args_for_new_array(arraydescr)) == repr( - [arraydescr.get_item_size(True), arraydescr.tid]) def test_gc_malloc_str(self): p = self.gc_ll_descr.gc_malloc_str(10) @@ -404,10 +406,11 @@ gc_ll_descr = self.gc_ll_descr llop1 = self.llop1 # - newops = [] + rewriter = GcRewriterAssembler(gc_ll_descr, None) + newops = rewriter.newops v_base = BoxPtr() v_value = BoxPtr() - gc_ll_descr._gen_write_barrier(newops, v_base, v_value) + rewriter.gen_write_barrier(v_base, v_value) assert llop1.record == [] assert len(newops) == 1 assert newops[0].getopnum() == rop.COND_CALL_GC_WB @@ -427,8 +430,7 @@ operations = gc_ll_descr.rewrite_assembler(None, operations, []) assert len(operations) == 0 - def test_rewrite_assembler_1(self): - # check recording of ConstPtrs + def test_record_constptrs(self): class MyFakeCPU(object): def cast_adr_to_int(self, adr): assert adr == "some fake address" @@ -455,211 +457,6 @@ assert operations2 == operations assert gcrefs == [s_gcref] - def test_rewrite_assembler_2(self): - # check write barriers before SETFIELD_GC - v_base = BoxPtr() - v_value = BoxPtr() - field_descr = AbstractDescr() - operations = [ - ResOperation(rop.SETFIELD_GC, [v_base, v_value], None, - descr=field_descr), - ] - gc_ll_descr = self.gc_ll_descr - operations = get_deep_immutable_oplist(operations) - operations = gc_ll_descr.rewrite_assembler(self.fake_cpu, operations, - []) - assert len(operations) == 2 - # - assert operations[0].getopnum() == rop.COND_CALL_GC_WB - assert operations[0].getarg(0) == v_base - assert operations[0].getarg(1) == v_value - assert operations[0].result is None - # - assert operations[1].getopnum() == rop.SETFIELD_RAW - assert operations[1].getarg(0) == v_base - assert operations[1].getarg(1) == v_value - assert operations[1].getdescr() == field_descr - - def test_rewrite_assembler_3(self): - # check write barriers before SETARRAYITEM_GC - for v_new_length in (None, ConstInt(5), ConstInt(5000), BoxInt()): - v_base = BoxPtr() - v_index = BoxInt() - v_value = BoxPtr() - array_descr = AbstractDescr() - operations = [ - ResOperation(rop.SETARRAYITEM_GC, [v_base, v_index, v_value], - None, descr=array_descr), - ] - if v_new_length is not None: - operations.insert(0, ResOperation(rop.NEW_ARRAY, - [v_new_length], v_base, - descr=array_descr)) - # we need to insert another, unrelated NEW_ARRAY here - # to prevent the initialization_store optimization - operations.insert(1, ResOperation(rop.NEW_ARRAY, - [ConstInt(12)], BoxPtr(), - descr=array_descr)) - gc_ll_descr = self.gc_ll_descr - operations = get_deep_immutable_oplist(operations) - operations = gc_ll_descr.rewrite_assembler(self.fake_cpu, - operations, []) - if v_new_length is not None: - assert operations[0].getopnum() == rop.NEW_ARRAY - assert operations[1].getopnum() == rop.NEW_ARRAY - del operations[:2] - assert len(operations) == 2 - # - assert operations[0].getopnum() == rop.COND_CALL_GC_WB - assert operations[0].getarg(0) == v_base - assert operations[0].getarg(1) == v_value - assert operations[0].result is None - # - assert operations[1].getopnum() == rop.SETARRAYITEM_RAW - assert operations[1].getarg(0) == v_base - assert operations[1].getarg(1) == v_index - assert operations[1].getarg(2) == v_value - assert operations[1].getdescr() == array_descr - - def test_rewrite_assembler_4(self): - # check write barriers before SETARRAYITEM_GC, - # if we have actually a write_barrier_from_array. - self.llop1._have_wb_from_array = True - for v_new_length in (None, ConstInt(5), ConstInt(5000), BoxInt()): - v_base = BoxPtr() - v_index = BoxInt() - v_value = BoxPtr() - array_descr = AbstractDescr() - operations = [ - ResOperation(rop.SETARRAYITEM_GC, [v_base, v_index, v_value], - None, descr=array_descr), - ] - if v_new_length is not None: - operations.insert(0, ResOperation(rop.NEW_ARRAY, - [v_new_length], v_base, - descr=array_descr)) - # we need to insert another, unrelated NEW_ARRAY here - # to prevent the initialization_store optimization - operations.insert(1, ResOperation(rop.NEW_ARRAY, - [ConstInt(12)], BoxPtr(), - descr=array_descr)) - gc_ll_descr = self.gc_ll_descr - operations = get_deep_immutable_oplist(operations) - operations = gc_ll_descr.rewrite_assembler(self.fake_cpu, - operations, []) - if v_new_length is not None: - assert operations[0].getopnum() == rop.NEW_ARRAY - assert operations[1].getopnum() == rop.NEW_ARRAY - del operations[:2] - assert len(operations) == 2 - # - if isinstance(v_new_length, ConstInt) and v_new_length.value < 130: - assert operations[0].getopnum() == rop.COND_CALL_GC_WB - assert operations[0].getarg(0) == v_base - assert operations[0].getarg(1) == v_value - else: - assert operations[0].getopnum() == rop.COND_CALL_GC_WB_ARRAY - assert operations[0].getarg(0) == v_base - assert operations[0].getarg(1) == v_index - assert operations[0].getarg(2) == v_value - assert operations[0].result is None - # - assert operations[1].getopnum() == rop.SETARRAYITEM_RAW - assert operations[1].getarg(0) == v_base - assert operations[1].getarg(1) == v_index - assert operations[1].getarg(2) == v_value - assert operations[1].getdescr() == array_descr - - def test_rewrite_assembler_5(self): - S = lltype.GcStruct('S') - A = lltype.GcArray(lltype.Struct('A', ('x', lltype.Ptr(S)))) - interiordescr = get_interiorfield_descr(self.gc_ll_descr, A, - A.OF, 'x') - wbdescr = self.gc_ll_descr.write_barrier_descr - ops = parse(""" - [p1, p2] - setinteriorfield_gc(p1, 0, p2, descr=interiordescr) - jump(p1, p2) - """, namespace=locals()) - expected = parse(""" - [p1, p2] - cond_call_gc_wb(p1, p2, descr=wbdescr) - setinteriorfield_raw(p1, 0, p2, descr=interiordescr) - jump(p1, p2) - """, namespace=locals()) - operations = get_deep_immutable_oplist(ops.operations) - operations = self.gc_ll_descr.rewrite_assembler(self.fake_cpu, - operations, []) - equaloplists(operations, expected.operations) - - def test_rewrite_assembler_initialization_store(self): - S = lltype.GcStruct('S', ('parent', OBJECT), - ('x', lltype.Signed)) - s_vtable = lltype.malloc(OBJECT_VTABLE, immortal=True) - xdescr = get_field_descr(self.gc_ll_descr, S, 'x') - ops = parse(""" - [p1] - p0 = new_with_vtable(ConstClass(s_vtable)) - setfield_gc(p0, p1, descr=xdescr) - jump() - """, namespace=locals()) - expected = parse(""" - [p1] - p0 = new_with_vtable(ConstClass(s_vtable)) - # no write barrier - setfield_gc(p0, p1, descr=xdescr) - jump() - """, namespace=locals()) - operations = get_deep_immutable_oplist(ops.operations) - operations = self.gc_ll_descr.rewrite_assembler(self.fake_cpu, - operations, []) - equaloplists(operations, expected.operations) - - def test_rewrite_assembler_initialization_store_2(self): - S = lltype.GcStruct('S', ('parent', OBJECT), - ('x', lltype.Signed)) - s_vtable = lltype.malloc(OBJECT_VTABLE, immortal=True) - wbdescr = self.gc_ll_descr.write_barrier_descr - xdescr = get_field_descr(self.gc_ll_descr, S, 'x') - ops = parse(""" - [p1] - p0 = new_with_vtable(ConstClass(s_vtable)) - p3 = new_with_vtable(ConstClass(s_vtable)) - setfield_gc(p0, p1, descr=xdescr) - jump() - """, namespace=locals()) - expected = parse(""" - [p1] - p0 = new_with_vtable(ConstClass(s_vtable)) - p3 = new_with_vtable(ConstClass(s_vtable)) - cond_call_gc_wb(p0, p1, descr=wbdescr) - setfield_raw(p0, p1, descr=xdescr) - jump() - """, namespace=locals()) - operations = get_deep_immutable_oplist(ops.operations) - operations = self.gc_ll_descr.rewrite_assembler(self.fake_cpu, - operations, []) - equaloplists(operations, expected.operations) - - def test_rewrite_assembler_initialization_store_3(self): - A = lltype.GcArray(lltype.Ptr(lltype.GcStruct('S'))) - arraydescr = get_array_descr(self.gc_ll_descr, A) - ops = parse(""" - [p1] - p0 = new_array(3, descr=arraydescr) - setarrayitem_gc(p0, 0, p1, descr=arraydescr) - jump() - """, namespace=locals()) - expected = parse(""" - [p1] - p0 = new_array(3, descr=arraydescr) - setarrayitem_gc(p0, 0, p1, descr=arraydescr) - jump() - """, namespace=locals()) - operations = get_deep_immutable_oplist(ops.operations) - operations = self.gc_ll_descr.rewrite_assembler(self.fake_cpu, - operations, []) - equaloplists(operations, expected.operations) class TestFrameworkMiniMark(TestFramework): gc = 'minimark' diff --git a/pypy/jit/backend/llsupport/test/test_rewrite.py b/pypy/jit/backend/llsupport/test/test_rewrite.py new file mode 100644 --- /dev/null +++ b/pypy/jit/backend/llsupport/test/test_rewrite.py @@ -0,0 +1,668 @@ +from pypy.jit.backend.llsupport.descr import * +from pypy.jit.backend.llsupport.gc import * +from pypy.jit.metainterp.gc import get_description +from pypy.jit.tool.oparser import parse +from pypy.jit.metainterp.optimizeopt.util import equaloplists +from pypy.jit.codewriter.heaptracker import register_known_gctype + + +class Evaluator(object): + def __init__(self, scope): + self.scope = scope + def __getitem__(self, key): + return eval(key, self.scope) + + +class RewriteTests(object): + def check_rewrite(self, frm_operations, to_operations, **namespace): + S = lltype.GcStruct('S', ('x', lltype.Signed), + ('y', lltype.Signed)) + sdescr = get_size_descr(self.gc_ll_descr, S) + sdescr.tid = 1234 + # + T = lltype.GcStruct('T', ('y', lltype.Signed), + ('z', lltype.Ptr(S)), + ('t', lltype.Signed)) + tdescr = get_size_descr(self.gc_ll_descr, T) + tdescr.tid = 5678 + tzdescr = get_field_descr(self.gc_ll_descr, T, 'z') + # + A = lltype.GcArray(lltype.Signed) + adescr = get_array_descr(self.gc_ll_descr, A) + adescr.tid = 4321 + alendescr = adescr.lendescr + # + B = lltype.GcArray(lltype.Char) + bdescr = get_array_descr(self.gc_ll_descr, B) + bdescr.tid = 8765 + blendescr = bdescr.lendescr + # + C = lltype.GcArray(lltype.Ptr(S)) + cdescr = get_array_descr(self.gc_ll_descr, C) + cdescr.tid = 8111 + clendescr = cdescr.lendescr + # + E = lltype.GcStruct('Empty') + edescr = get_size_descr(self.gc_ll_descr, E) + edescr.tid = 9000 + # + vtable_descr = self.gc_ll_descr.fielddescr_vtable + O = lltype.GcStruct('O', ('parent', rclass.OBJECT), + ('x', lltype.Signed)) + o_vtable = lltype.malloc(rclass.OBJECT_VTABLE, immortal=True) + register_known_gctype(self.cpu, o_vtable, O) + # + tiddescr = self.gc_ll_descr.fielddescr_tid + wbdescr = self.gc_ll_descr.write_barrier_descr + WORD = globals()['WORD'] + # + strdescr = self.gc_ll_descr.str_descr + unicodedescr = self.gc_ll_descr.unicode_descr + strlendescr = strdescr.lendescr + unicodelendescr = unicodedescr.lendescr + # + namespace.update(locals()) + # + for funcname in self.gc_ll_descr._generated_functions: + namespace[funcname] = self.gc_ll_descr.get_malloc_fn(funcname) + namespace[funcname + '_descr'] = getattr(self.gc_ll_descr, + '%s_descr' % funcname) + # + ops = parse(frm_operations, namespace=namespace) + expected = parse(to_operations % Evaluator(namespace), + namespace=namespace) + operations = self.gc_ll_descr.rewrite_assembler(self.cpu, + ops.operations, + []) + equaloplists(operations, expected.operations) + + +class TestBoehm(RewriteTests): + def setup_method(self, meth): + class FakeCPU(object): + def sizeof(self, STRUCT): + return SizeDescrWithVTable(102) + self.cpu = FakeCPU() + self.gc_ll_descr = GcLLDescr_boehm(None, None, None) + + def test_new(self): + self.check_rewrite(""" + [] + p0 = new(descr=sdescr) + jump() + """, """ + [p1] + p0 = call_malloc_gc(ConstClass(malloc_fixedsize), %(sdescr.size)d,\ + descr=malloc_fixedsize_descr) + jump() + """) + + def test_no_collapsing(self): + self.check_rewrite(""" + [] + p0 = new(descr=sdescr) + p1 = new(descr=sdescr) + jump() + """, """ + [] + p0 = call_malloc_gc(ConstClass(malloc_fixedsize), %(sdescr.size)d,\ + descr=malloc_fixedsize_descr) + p1 = call_malloc_gc(ConstClass(malloc_fixedsize), %(sdescr.size)d,\ + descr=malloc_fixedsize_descr) + jump() + """) + + def test_new_array_fixed(self): + self.check_rewrite(""" + [] + p0 = new_array(10, descr=adescr) + jump() + """, """ + [] + p0 = call_malloc_gc(ConstClass(malloc_fixedsize), \ + %(adescr.basesize + 10 * adescr.itemsize)d, \ + descr=malloc_fixedsize_descr) + setfield_gc(p0, 10, descr=alendescr) + jump() + """) + + def test_new_array_variable(self): + self.check_rewrite(""" + [i1] + p0 = new_array(i1, descr=adescr) + jump() + """, """ + [i1] + p0 = call_malloc_gc(ConstClass(malloc_array), \ + %(adescr.basesize)d, \ + i1, \ + %(adescr.itemsize)d, \ + %(adescr.lendescr.offset)d, \ + descr=malloc_array_descr) + jump() + """) + + def test_new_with_vtable(self): + self.check_rewrite(""" + [] + p0 = new_with_vtable(ConstClass(o_vtable)) + jump() + """, """ + [p1] + p0 = call_malloc_gc(ConstClass(malloc_fixedsize), 102, \ + descr=malloc_fixedsize_descr) + setfield_gc(p0, ConstClass(o_vtable), descr=vtable_descr) + jump() + """) + + def test_newstr(self): + self.check_rewrite(""" + [i1] + p0 = newstr(i1) + jump() + """, """ + [i1] + p0 = call_malloc_gc(ConstClass(malloc_array), \ + %(strdescr.basesize)d, \ + i1, \ + %(strdescr.itemsize)d, \ + %(strlendescr.offset)d, \ + descr=malloc_array_descr) + jump() + """) + + def test_newunicode(self): + self.check_rewrite(""" + [i1] + p0 = newunicode(10) + jump() + """, """ + [i1] + p0 = call_malloc_gc(ConstClass(malloc_fixedsize), \ + %(unicodedescr.basesize + \ + 10 * unicodedescr.itemsize)d, \ + descr=malloc_fixedsize_descr) + setfield_gc(p0, 10, descr=unicodelendescr) + jump() + """) + + +class TestFramework(RewriteTests): + def setup_method(self, meth): + class config_(object): + class translation(object): + gc = 'hybrid' + gcrootfinder = 'asmgcc' + gctransformer = 'framework' + gcremovetypeptr = False + gcdescr = get_description(config_) + self.gc_ll_descr = GcLLDescr_framework(gcdescr, None, None, None, + really_not_translated=True) + self.gc_ll_descr.write_barrier_descr.has_write_barrier_from_array = ( + lambda cpu: True) + # + class FakeCPU(object): + def sizeof(self, STRUCT): + descr = SizeDescrWithVTable(102) + descr.tid = 9315 + return descr + self.cpu = FakeCPU() + + def test_rewrite_assembler_new_to_malloc(self): + self.check_rewrite(""" + [p1] + p0 = new(descr=sdescr) + jump() + """, """ + [p1] + p0 = call_malloc_nursery(%(sdescr.size)d) + setfield_gc(p0, 1234, descr=tiddescr) + jump() + """) + + def test_rewrite_assembler_new3_to_malloc(self): + self.check_rewrite(""" + [] + p0 = new(descr=sdescr) + p1 = new(descr=tdescr) + p2 = new(descr=sdescr) + jump() + """, """ + [] + p0 = call_malloc_nursery( \ + %(sdescr.size + tdescr.size + sdescr.size)d) + setfield_gc(p0, 1234, descr=tiddescr) + p1 = int_add(p0, %(sdescr.size)d) + setfield_gc(p1, 5678, descr=tiddescr) + p2 = int_add(p1, %(tdescr.size)d) + setfield_gc(p2, 1234, descr=tiddescr) + jump() + """) + + def test_rewrite_assembler_new_array_fixed_to_malloc(self): + self.check_rewrite(""" + [] + p0 = new_array(10, descr=adescr) + jump() + """, """ + [] + p0 = call_malloc_nursery( \ + %(adescr.basesize + 10 * adescr.itemsize)d) + setfield_gc(p0, 4321, descr=tiddescr) + setfield_gc(p0, 10, descr=alendescr) + jump() + """) + + def test_rewrite_assembler_new_and_new_array_fixed_to_malloc(self): + self.check_rewrite(""" + [] + p0 = new(descr=sdescr) + p1 = new_array(10, descr=adescr) + jump() + """, """ + [] + p0 = call_malloc_nursery( \ + %(sdescr.size + \ + adescr.basesize + 10 * adescr.itemsize)d) + setfield_gc(p0, 1234, descr=tiddescr) + p1 = int_add(p0, %(sdescr.size)d) + setfield_gc(p1, 4321, descr=tiddescr) + setfield_gc(p1, 10, descr=alendescr) + jump() + """) + + def test_rewrite_assembler_round_up(self): + self.check_rewrite(""" + [] + p0 = new_array(6, descr=bdescr) + jump() + """, """ + [] + p0 = call_malloc_nursery(%(bdescr.basesize + 8)d) + setfield_gc(p0, 8765, descr=tiddescr) + setfield_gc(p0, 6, descr=blendescr) + jump() + """) + + def test_rewrite_assembler_round_up_always(self): + self.check_rewrite(""" + [] + p0 = new_array(5, descr=bdescr) + p1 = new_array(5, descr=bdescr) + p2 = new_array(5, descr=bdescr) + p3 = new_array(5, descr=bdescr) + jump() + """, """ + [] + p0 = call_malloc_nursery(%(4 * (bdescr.basesize + 8))d) + setfield_gc(p0, 8765, descr=tiddescr) + setfield_gc(p0, 5, descr=blendescr) + p1 = int_add(p0, %(bdescr.basesize + 8)d) + setfield_gc(p1, 8765, descr=tiddescr) + setfield_gc(p1, 5, descr=blendescr) + p2 = int_add(p1, %(bdescr.basesize + 8)d) + setfield_gc(p2, 8765, descr=tiddescr) + setfield_gc(p2, 5, descr=blendescr) + p3 = int_add(p2, %(bdescr.basesize + 8)d) + setfield_gc(p3, 8765, descr=tiddescr) + setfield_gc(p3, 5, descr=blendescr) + jump() + """) + + def test_rewrite_assembler_minimal_size(self): + self.check_rewrite(""" + [] + p0 = new(descr=edescr) + p1 = new(descr=edescr) + jump() + """, """ + [] + p0 = call_malloc_nursery(%(4*WORD)d) + setfield_gc(p0, 9000, descr=tiddescr) + p1 = int_add(p0, %(2*WORD)d) + setfield_gc(p1, 9000, descr=tiddescr) + jump() + """) + + def test_rewrite_assembler_variable_size(self): + self.check_rewrite(""" + [i0] + p0 = new_array(i0, descr=bdescr) + jump(i0) + """, """ + [i0] + p0 = call_malloc_gc(ConstClass(malloc_array), 1, \ + %(bdescr.tid)d, i0, \ + descr=malloc_array_descr) + jump(i0) + """) + + def test_rewrite_assembler_nonstandard_array(self): + # a non-standard array is a bit hard to get; e.g. GcArray(Float) + # is like that on Win32, but not on Linux. Build one manually... + NONSTD = lltype.GcArray(lltype.Float) + nonstd_descr = get_array_descr(self.gc_ll_descr, NONSTD) + nonstd_descr.tid = 6464 + nonstd_descr.basesize = 64 # <= hacked + nonstd_descr.itemsize = 8 + nonstd_descr_gcref = 123 + self.check_rewrite(""" + [i0] + p0 = new_array(i0, descr=nonstd_descr) + jump(i0) + """, """ + [i0] + p0 = call_malloc_gc(ConstClass(malloc_array_nonstandard), \ + 64, 8, \ + %(nonstd_descr.lendescr.offset)d, \ + 6464, i0, \ + descr=malloc_array_nonstandard_descr) + jump(i0) + """, nonstd_descr=nonstd_descr) + + def test_rewrite_assembler_maximal_size_1(self): + self.gc_ll_descr.max_size_of_young_obj = 100 + self.check_rewrite(""" + [] + p0 = new_array(103, descr=bdescr) + jump() + """, """ + [] + p0 = call_malloc_gc(ConstClass(malloc_fixedsize), \ + %(bdescr.basesize + 104)d, \ + descr=malloc_fixedsize_descr) + setfield_gc(p0, 8765, descr=tiddescr) + setfield_gc(p0, 103, descr=blendescr) + jump() + """) + + def test_rewrite_assembler_maximal_size_2(self): + self.gc_ll_descr.max_size_of_young_obj = 300 + self.check_rewrite(""" + [] + p0 = new_array(101, descr=bdescr) + p1 = new_array(102, descr=bdescr) # two new_arrays can be combined + p2 = new_array(103, descr=bdescr) # but not all three + jump() + """, """ + [] + p0 = call_malloc_nursery( \ + %(2 * (bdescr.basesize + 104))d) + setfield_gc(p0, 8765, descr=tiddescr) + setfield_gc(p0, 101, descr=blendescr) + p1 = int_add(p0, %(bdescr.basesize + 104)d) + setfield_gc(p1, 8765, descr=tiddescr) + setfield_gc(p1, 102, descr=blendescr) + p2 = call_malloc_nursery( \ + %(bdescr.basesize + 104)d) + setfield_gc(p2, 8765, descr=tiddescr) + setfield_gc(p2, 103, descr=blendescr) + jump() + """) + + def test_rewrite_assembler_huge_size(self): + # "huge" is defined as "larger than 0xffffff bytes, or 16MB" + self.check_rewrite(""" + [] + p0 = new_array(20000000, descr=bdescr) + jump() + """, """ + [] + p0 = call_malloc_gc(ConstClass(malloc_array), 1, \ + %(bdescr.tid)d, 20000000, \ + descr=malloc_array_descr) + jump() + """) + + def test_new_with_vtable(self): + self.check_rewrite(""" + [] + p0 = new_with_vtable(ConstClass(o_vtable)) + jump() + """, """ + [p1] + p0 = call_malloc_nursery(104) # rounded up + setfield_gc(p0, 9315, descr=tiddescr) + setfield_gc(p0, ConstClass(o_vtable), descr=vtable_descr) + jump() + """) + + def test_new_with_vtable_too_big(self): + self.gc_ll_descr.max_size_of_young_obj = 100 + self.check_rewrite(""" + [] + p0 = new_with_vtable(ConstClass(o_vtable)) + jump() + """, """ + [p1] + p0 = call_malloc_gc(ConstClass(malloc_fixedsize), 104, \ + descr=malloc_fixedsize_descr) + setfield_gc(p0, 9315, descr=tiddescr) + setfield_gc(p0, ConstClass(o_vtable), descr=vtable_descr) + jump() + """) + + def test_rewrite_assembler_newstr_newunicode(self): + self.check_rewrite(""" + [i2] + p0 = newstr(14) + p1 = newunicode(10) + p2 = newunicode(i2) + p3 = newstr(i2) + jump() + """, """ + [i2] + p0 = call_malloc_nursery( \ + %(strdescr.basesize + 16 * strdescr.itemsize + \ + unicodedescr.basesize + 10 * unicodedescr.itemsize)d) + setfield_gc(p0, %(strdescr.tid)d, descr=tiddescr) + setfield_gc(p0, 14, descr=strlendescr) + p1 = int_add(p0, %(strdescr.basesize + 16 * strdescr.itemsize)d) + setfield_gc(p1, %(unicodedescr.tid)d, descr=tiddescr) + setfield_gc(p1, 10, descr=unicodelendescr) + p2 = call_malloc_gc(ConstClass(malloc_unicode), i2, \ + descr=malloc_unicode_descr) + p3 = call_malloc_gc(ConstClass(malloc_str), i2, \ + descr=malloc_str_descr) + jump() + """) + + def test_write_barrier_before_setfield_gc(self): + self.check_rewrite(""" + [p1, p2] + setfield_gc(p1, p2, descr=tzdescr) + jump() + """, """ + [p1, p2] + cond_call_gc_wb(p1, p2, descr=wbdescr) + setfield_raw(p1, p2, descr=tzdescr) + jump() + """) + + def test_write_barrier_before_array_without_from_array(self): + self.gc_ll_descr.write_barrier_descr.has_write_barrier_from_array = ( + lambda cpu: False) + self.check_rewrite(""" + [p1, i2, p3] + setarrayitem_gc(p1, i2, p3, descr=cdescr) + jump() + """, """ + [p1, i2, p3] + cond_call_gc_wb(p1, p3, descr=wbdescr) + setarrayitem_raw(p1, i2, p3, descr=cdescr) + jump() + """) + + def test_write_barrier_before_short_array(self): + self.gc_ll_descr.max_size_of_young_obj = 2000 + self.check_rewrite(""" + [i2, p3] + p1 = new_array(129, descr=cdescr) + call(123456) + setarrayitem_gc(p1, i2, p3, descr=cdescr) + jump() + """, """ + [i2, p3] + p1 = call_malloc_nursery( \ + %(cdescr.basesize + 129 * cdescr.itemsize)d) + setfield_gc(p1, 8111, descr=tiddescr) + setfield_gc(p1, 129, descr=clendescr) + call(123456) + cond_call_gc_wb(p1, p3, descr=wbdescr) + setarrayitem_raw(p1, i2, p3, descr=cdescr) + jump() + """) + + def test_write_barrier_before_long_array(self): + # the limit of "being too long" is fixed, arbitrarily, at 130 + self.gc_ll_descr.max_size_of_young_obj = 2000 + self.check_rewrite(""" + [i2, p3] + p1 = new_array(130, descr=cdescr) + call(123456) + setarrayitem_gc(p1, i2, p3, descr=cdescr) + jump() + """, """ + [i2, p3] + p1 = call_malloc_nursery( \ + %(cdescr.basesize + 130 * cdescr.itemsize)d) + setfield_gc(p1, 8111, descr=tiddescr) + setfield_gc(p1, 130, descr=clendescr) + call(123456) + cond_call_gc_wb_array(p1, i2, p3, descr=wbdescr) + setarrayitem_raw(p1, i2, p3, descr=cdescr) + jump() + """) + + def test_write_barrier_before_unknown_array(self): + self.check_rewrite(""" + [p1, i2, p3] + setarrayitem_gc(p1, i2, p3, descr=cdescr) + jump() + """, """ + [p1, i2, p3] + cond_call_gc_wb_array(p1, i2, p3, descr=wbdescr) + setarrayitem_raw(p1, i2, p3, descr=cdescr) + jump() + """) + + def test_label_makes_size_unknown(self): + self.check_rewrite(""" + [i2, p3] + p1 = new_array(5, descr=cdescr) + label(p1, i2, p3) + setarrayitem_gc(p1, i2, p3, descr=cdescr) + jump() + """, """ + [i2, p3] + p1 = call_malloc_nursery( \ + %(cdescr.basesize + 5 * cdescr.itemsize)d) + setfield_gc(p1, 8111, descr=tiddescr) + setfield_gc(p1, 5, descr=clendescr) + label(p1, i2, p3) + cond_call_gc_wb_array(p1, i2, p3, descr=wbdescr) + setarrayitem_raw(p1, i2, p3, descr=cdescr) + jump() + """) + + def test_write_barrier_before_setinteriorfield_gc(self): + S1 = lltype.GcStruct('S1') + INTERIOR = lltype.GcArray(('z', lltype.Ptr(S1))) + interiordescr = get_array_descr(self.gc_ll_descr, INTERIOR) + interiordescr.tid = 1291 + interiorlendescr = interiordescr.lendescr + interiorzdescr = get_interiorfield_descr(self.gc_ll_descr, + INTERIOR, 'z') + self.check_rewrite(""" + [p1, p2] + setinteriorfield_gc(p1, 0, p2, descr=interiorzdescr) + jump(p1, p2) + """, """ + [p1, p2] + cond_call_gc_wb(p1, p2, descr=wbdescr) + setinteriorfield_raw(p1, 0, p2, descr=interiorzdescr) + jump(p1, p2) + """, interiorzdescr=interiorzdescr) + + def test_initialization_store(self): + self.check_rewrite(""" + [p1] + p0 = new(descr=tdescr) + setfield_gc(p0, p1, descr=tzdescr) + jump() + """, """ + [p1] + p0 = call_malloc_nursery(%(tdescr.size)d) + setfield_gc(p0, 5678, descr=tiddescr) + setfield_gc(p0, p1, descr=tzdescr) + jump() + """) + + def test_initialization_store_2(self): + self.check_rewrite(""" + [] + p0 = new(descr=tdescr) + p1 = new(descr=sdescr) + setfield_gc(p0, p1, descr=tzdescr) + jump() + """, """ + [] + p0 = call_malloc_nursery(%(tdescr.size + sdescr.size)d) + setfield_gc(p0, 5678, descr=tiddescr) + p1 = int_add(p0, %(tdescr.size)d) + setfield_gc(p1, 1234, descr=tiddescr) + # <<>> + setfield_gc(p0, p1, descr=tzdescr) + jump() + """) + + def test_initialization_store_array(self): + self.check_rewrite(""" + [p1, i2] + p0 = new_array(5, descr=cdescr) + setarrayitem_gc(p0, i2, p1, descr=cdescr) + jump() + """, """ + [p1, i2] + p0 = call_malloc_nursery( \ + %(cdescr.basesize + 5 * cdescr.itemsize)d) + setfield_gc(p0, 8111, descr=tiddescr) + setfield_gc(p0, 5, descr=clendescr) + setarrayitem_gc(p0, i2, p1, descr=cdescr) + jump() + """) + + def test_non_initialization_store(self): + self.check_rewrite(""" + [i0] + p0 = new(descr=tdescr) + p1 = newstr(i0) + setfield_gc(p0, p1, descr=tzdescr) + jump() + """, """ + [i0] + p0 = call_malloc_nursery(%(tdescr.size)d) + setfield_gc(p0, 5678, descr=tiddescr) + p1 = call_malloc_gc(ConstClass(malloc_str), i0, \ + descr=malloc_str_descr) + cond_call_gc_wb(p0, p1, descr=wbdescr) + setfield_raw(p0, p1, descr=tzdescr) + jump() + """) + + def test_non_initialization_store_label(self): + self.check_rewrite(""" + [p1] + p0 = new(descr=tdescr) + label(p0, p1) + setfield_gc(p0, p1, descr=tzdescr) + jump() + """, """ + [p1] + p0 = call_malloc_nursery(%(tdescr.size)d) + setfield_gc(p0, 5678, descr=tiddescr) + label(p0, p1) + cond_call_gc_wb(p0, p1, descr=wbdescr) + setfield_raw(p0, p1, descr=tzdescr) + jump() + """) diff --git a/pypy/jit/backend/llsupport/test/test_runner.py b/pypy/jit/backend/llsupport/test/test_runner.py --- a/pypy/jit/backend/llsupport/test/test_runner.py +++ b/pypy/jit/backend/llsupport/test/test_runner.py @@ -8,6 +8,12 @@ class MyLLCPU(AbstractLLCPU): supports_floats = True + + class assembler(object): + @staticmethod + def set_debug(flag): + pass + def compile_loop(self, inputargs, operations, looptoken): py.test.skip("llsupport test: cannot compile operations") diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py --- a/pypy/jit/backend/test/runner_test.py +++ b/pypy/jit/backend/test/runner_test.py @@ -17,6 +17,7 @@ from pypy.rpython.llinterp import LLException from pypy.jit.codewriter import heaptracker, longlong from pypy.rlib.rarithmetic import intmask +from pypy.jit.backend.detect_cpu import autodetect_main_model_and_size def boxfloat(x): return BoxFloat(longlong.getfloatstorage(x)) @@ -27,6 +28,9 @@ class Runner(object): + add_loop_instruction = ['overload for a specific cpu'] + bridge_loop_instruction = ['overload for a specific cpu'] + def execute_operation(self, opname, valueboxes, result_type, descr=None): inputargs, operations = self._get_single_operation_list(opname, result_type, @@ -2930,6 +2934,8 @@ # overflowing value: fail = self.cpu.execute_token(looptoken, sys.maxint // 4 + 1) assert fail.identifier == excdescr.identifier + exc = self.cpu.grab_exc_value() + assert exc == "memoryerror!" def test_compile_loop_with_target(self): i0 = BoxInt() @@ -2972,6 +2978,56 @@ res = self.cpu.get_latest_value_int(0) assert res == -10 + def test_compile_asmlen(self): + from pypy.jit.backend.llsupport.llmodel import AbstractLLCPU + if not isinstance(self.cpu, AbstractLLCPU): + py.test.skip("pointless test on non-asm") + from pypy.jit.backend.x86.tool.viewcode import machine_code_dump + import ctypes + ops = """ + [i2] + i0 = same_as(i2) # but forced to be in a register + label(i0, descr=1) + i1 = int_add(i0, i0) + guard_true(i1, descr=faildesr) [i1] + jump(i1, descr=1) + """ + faildescr = BasicFailDescr(2) + loop = parse(ops, self.cpu, namespace=locals()) + faildescr = loop.operations[-2].getdescr() + jumpdescr = loop.operations[-1].getdescr() + bridge_ops = """ + [i0] + jump(i0, descr=jumpdescr) + """ + bridge = parse(bridge_ops, self.cpu, namespace=locals()) + looptoken = JitCellToken() + self.cpu.assembler.set_debug(False) + info = self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken) + bridge_info = self.cpu.compile_bridge(faildescr, bridge.inputargs, + bridge.operations, + looptoken) + self.cpu.assembler.set_debug(True) # always on untranslated + assert info.asmlen != 0 + cpuname = autodetect_main_model_and_size() + # XXX we have to check the precise assembler, otherwise + # we don't quite know if borders are correct + + def checkops(mc, ops): + assert len(mc) == len(ops) + for i in range(len(mc)): + assert mc[i].split("\t")[-1].startswith(ops[i]) + + data = ctypes.string_at(info.asmaddr, info.asmlen) + mc = list(machine_code_dump(data, info.asmaddr, cpuname)) + lines = [line for line in mc if line.count('\t') == 2] + checkops(lines, self.add_loop_instructions) + data = ctypes.string_at(bridge_info.asmaddr, bridge_info.asmlen) + mc = list(machine_code_dump(data, bridge_info.asmaddr, cpuname)) + lines = [line for line in mc if line.count('\t') == 2] + checkops(lines, self.bridge_loop_instructions) + + def test_compile_bridge_with_target(self): # This test creates a loopy piece of code in a bridge, and builds another # unrelated loop that ends in a jump directly to this loopy bit of code. diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py --- a/pypy/jit/backend/x86/assembler.py +++ b/pypy/jit/backend/x86/assembler.py @@ -7,6 +7,7 @@ from pypy.rpython.lltypesystem import lltype, rffi, rstr, llmemory from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.annlowlevel import llhelper +from pypy.rlib.jit import AsmInfo from pypy.jit.backend.model import CompiledLoopToken from pypy.jit.backend.x86.regalloc import (RegAlloc, get_ebp_ofs, _get_scale, gpr_reg_mgr_cls, _valid_addressing_size) @@ -39,6 +40,7 @@ from pypy.jit.codewriter.effectinfo import EffectInfo from pypy.jit.codewriter import longlong from pypy.rlib.rarithmetic import intmask +from pypy.rlib.objectmodel import compute_unique_id # darwin requires the stack to be 16 bytes aligned on calls. Same for gcc 4.5.0, # better safe than sorry @@ -58,7 +60,8 @@ self.is_guard_not_invalidated = is_guard_not_invalidated DEBUG_COUNTER = lltype.Struct('DEBUG_COUNTER', ('i', lltype.Signed), - ('bridge', lltype.Signed), # 0 or 1 + ('type', lltype.Char), # 'b'ridge, 'l'abel or + # 'e'ntry point ('number', lltype.Signed)) class Assembler386(object): @@ -70,10 +73,6 @@ self.cpu = cpu self.verbose = False self.rtyper = cpu.rtyper - self.malloc_func_addr = 0 - self.malloc_array_func_addr = 0 - self.malloc_str_func_addr = 0 - self.malloc_unicode_func_addr = 0 self.fail_boxes_int = values_array(lltype.Signed, failargs_limit) self.fail_boxes_ptr = values_array(llmemory.GCREF, failargs_limit) self.fail_boxes_float = values_array(longlong.FLOATSTORAGE, @@ -108,20 +107,6 @@ # the address of the function called by 'new' gc_ll_descr = self.cpu.gc_ll_descr gc_ll_descr.initialize() - ll_new = gc_ll_descr.get_funcptr_for_new() - self.malloc_func_addr = rffi.cast(lltype.Signed, ll_new) - if gc_ll_descr.get_funcptr_for_newarray is not None: - ll_new_array = gc_ll_descr.get_funcptr_for_newarray() - self.malloc_array_func_addr = rffi.cast(lltype.Signed, - ll_new_array) - if gc_ll_descr.get_funcptr_for_newstr is not None: - ll_new_str = gc_ll_descr.get_funcptr_for_newstr() - self.malloc_str_func_addr = rffi.cast(lltype.Signed, - ll_new_str) - if gc_ll_descr.get_funcptr_for_newunicode is not None: - ll_new_unicode = gc_ll_descr.get_funcptr_for_newunicode() - self.malloc_unicode_func_addr = rffi.cast(lltype.Signed, - ll_new_unicode) self.memcpy_addr = self.cpu.cast_ptr_to_int(support.memcpy_fn) self._build_failure_recovery(False) self._build_failure_recovery(True) @@ -165,12 +150,15 @@ def finish_once(self): if self._debug: debug_start('jit-backend-counts') - for struct in self.loop_run_counters: - if struct.bridge: - prefix = 'bridge ' + for i in range(len(self.loop_run_counters)): + struct = self.loop_run_counters[i] + if struct.type == 'l': + prefix = 'TargetToken(%d)' % struct.number + elif struct.type == 'b': + prefix = 'bridge ' + str(struct.number) else: - prefix = 'loop ' - debug_print(prefix + str(struct.number) + ':' + str(struct.i)) + prefix = 'entry ' + str(struct.number) + debug_print(prefix + ':' + str(struct.i)) debug_stop('jit-backend-counts') def _build_float_constants(self): @@ -275,7 +263,8 @@ # self.mc = codebuf.MachineCodeBlockWrapper() # call on_leave_jitted_save_exc() - addr = self.cpu.get_on_leave_jitted_int(save_exception=True) + addr = self.cpu.get_on_leave_jitted_int(save_exception=True, + default_to_memoryerror=True) self.mc.CALL(imm(addr)) self.mc.MOV_ri(eax.value, self.cpu.propagate_exception_v) self._call_footer() @@ -423,6 +412,7 @@ '''adds the following attributes to looptoken: _x86_function_addr (address of the generated func, as an int) _x86_loop_code (debug: addr of the start of the ResOps) + _x86_fullsize (debug: full size including failure) _x86_debug_checksum ''' # XXX this function is too longish and contains some code @@ -439,8 +429,8 @@ self.setup(looptoken) if log: - self._register_counter(False, looptoken.number) - operations = self._inject_debugging_code(looptoken, operations) + operations = self._inject_debugging_code(looptoken, operations, + 'e', looptoken.number) regalloc = RegAlloc(self, self.cpu.translate_support_code) # @@ -488,7 +478,8 @@ name = "Loop # %s: %s" % (looptoken.number, loopname) self.cpu.profile_agent.native_code_written(name, rawstart, full_size) - return ops_offset + return AsmInfo(ops_offset, rawstart + looppos, + size_excluding_failure_stuff - looppos) def assemble_bridge(self, faildescr, inputargs, operations, original_loop_token, log): @@ -497,17 +488,12 @@ assert len(set(inputargs)) == len(inputargs) descr_number = self.cpu.get_fail_descr_number(faildescr) - try: - failure_recovery = self._find_failure_recovery_bytecode(faildescr) - except ValueError: - debug_print("Bridge out of guard", descr_number, - "was already compiled!") - return + failure_recovery = self._find_failure_recovery_bytecode(faildescr) self.setup(original_loop_token) if log: - self._register_counter(True, descr_number) - operations = self._inject_debugging_code(faildescr, operations) + operations = self._inject_debugging_code(faildescr, operations, + 'b', descr_number) arglocs = self.rebuild_faillocs_from_descr(failure_recovery) if not we_are_translated(): @@ -515,6 +501,7 @@ [loc.assembler() for loc in faildescr._x86_debug_faillocs]) regalloc = RegAlloc(self, self.cpu.translate_support_code) fail_depths = faildescr._x86_current_depths + startpos = self.mc.get_relative_pos() operations = regalloc.prepare_bridge(fail_depths, inputargs, arglocs, operations, self.current_clt.allgcrefs) @@ -549,7 +536,7 @@ name = "Bridge # %s" % (descr_number,) self.cpu.profile_agent.native_code_written(name, rawstart, fullsize) - return ops_offset + return AsmInfo(ops_offset, startpos + rawstart, codeendpos - startpos) def write_pending_failure_recoveries(self): # for each pending guard, generate the code of the recovery stub @@ -614,22 +601,29 @@ return self.mc.materialize(self.cpu.asmmemmgr, allblocks, self.cpu.gc_ll_descr.gcrootmap) - def _register_counter(self, bridge, number): - if self._debug: - # YYY very minor leak -- we need the counters to stay alive - # forever, just because we want to report them at the end - # of the process - struct = lltype.malloc(DEBUG_COUNTER, flavor='raw', - track_allocation=False) - struct.i = 0 - struct.bridge = int(bridge) + def _register_counter(self, tp, number, token): + # YYY very minor leak -- we need the counters to stay alive + # forever, just because we want to report them at the end + # of the process + struct = lltype.malloc(DEBUG_COUNTER, flavor='raw', + track_allocation=False) + struct.i = 0 + struct.type = tp + if tp == 'b' or tp == 'e': struct.number = number - self.loop_run_counters.append(struct) + else: + assert token + struct.number = compute_unique_id(token) + self.loop_run_counters.append(struct) + return struct def _find_failure_recovery_bytecode(self, faildescr): adr_jump_offset = faildescr._x86_adr_jump_offset if adr_jump_offset == 0: - raise ValueError + # This case should be prevented by the logic in compile.py: + # look for CNT_BUSY_FLAG, which disables tracing from a guard + # when another tracing from the same guard is already in progress. + raise BridgeAlreadyCompiled # follow the JMP/Jcond p = rffi.cast(rffi.INTP, adr_jump_offset) adr_target = adr_jump_offset + 4 + rffi.cast(lltype.Signed, p[0]) @@ -668,27 +662,36 @@ targettoken._x86_loop_code += rawstart self.target_tokens_currently_compiling = None + def _append_debugging_code(self, operations, tp, number, token): + counter = self._register_counter(tp, number, token) + c_adr = ConstInt(rffi.cast(lltype.Signed, counter)) + box = BoxInt() + box2 = BoxInt() + ops = [ResOperation(rop.GETFIELD_RAW, [c_adr], + box, descr=self.debug_counter_descr), + ResOperation(rop.INT_ADD, [box, ConstInt(1)], box2), + ResOperation(rop.SETFIELD_RAW, [c_adr, box2], + None, descr=self.debug_counter_descr)] + operations.extend(ops) + @specialize.argtype(1) - def _inject_debugging_code(self, looptoken, operations): + def _inject_debugging_code(self, looptoken, operations, tp, number): if self._debug: # before doing anything, let's increase a counter s = 0 for op in operations: s += op.getopnum() looptoken._x86_debug_checksum = s - c_adr = ConstInt(rffi.cast(lltype.Signed, - self.loop_run_counters[-1])) - box = BoxInt() - box2 = BoxInt() - ops = [ResOperation(rop.GETFIELD_RAW, [c_adr], - box, descr=self.debug_counter_descr), - ResOperation(rop.INT_ADD, [box, ConstInt(1)], box2), - ResOperation(rop.SETFIELD_RAW, [c_adr, box2], - None, descr=self.debug_counter_descr)] - if operations[0].getopnum() == rop.LABEL: - operations = [operations[0]] + ops + operations[1:] - else: - operations = ops + operations + + newoperations = [] + self._append_debugging_code(newoperations, tp, number, + None) + for op in operations: + newoperations.append(op) + if op.getopnum() == rop.LABEL: + self._append_debugging_code(newoperations, 'l', number, + op.getdescr()) + operations = newoperations return operations def _assemble(self, regalloc, operations): @@ -809,7 +812,10 @@ target = newlooptoken._x86_function_addr mc = codebuf.MachineCodeBlockWrapper() mc.JMP(imm(target)) - assert mc.get_relative_pos() <= 13 # keep in sync with prepare_loop() + if WORD == 4: # keep in sync with prepare_loop() + assert mc.get_relative_pos() == 5 + else: + assert mc.get_relative_pos() <= 13 mc.copy_to_raw_memory(oldadr) def dump(self, text): @@ -865,8 +871,8 @@ high_part = rffi.cast(rffi.CArrayPtr(rffi.INT), from_loc.value)[1] low_part = intmask(low_part) high_part = intmask(high_part) - self.mc.MOV_bi(to_loc.value, low_part) - self.mc.MOV_bi(to_loc.value + 4, high_part) + self.mc.MOV32_bi(to_loc.value, low_part) + self.mc.MOV32_bi(to_loc.value + 4, high_part) def regalloc_perform(self, op, arglocs, resloc): genop_list[op.getopnum()](self, op, arglocs, resloc) @@ -1357,46 +1363,10 @@ self.mc.SHR_ri(resloc.value, 7) self.mc.AND_ri(resloc.value, 1) - def genop_new_with_vtable(self, op, arglocs, result_loc): - assert result_loc is eax - loc_vtable = arglocs[-1] - assert isinstance(loc_vtable, ImmedLoc) - arglocs = arglocs[:-1] - self.call(self.malloc_func_addr, arglocs, eax) - self.propagate_memoryerror_if_eax_is_null() - self.set_vtable(eax, loc_vtable) + # ---------- - def set_vtable(self, loc, loc_vtable): - if self.cpu.vtable_offset is not None: - assert isinstance(loc, RegLoc) - assert isinstance(loc_vtable, ImmedLoc) - self.mc.MOV(mem(loc, self.cpu.vtable_offset), loc_vtable) - - def set_new_array_length(self, loc, ofs_length, loc_num_elem): - assert isinstance(loc, RegLoc) - assert isinstance(loc_num_elem, ImmedLoc) - self.mc.MOV(mem(loc, ofs_length), loc_num_elem) - - # XXX genop_new is abused for all varsized mallocs with Boehm, for now - # (instead of genop_new_array, genop_newstr, genop_newunicode) - def genop_new(self, op, arglocs, result_loc): - assert result_loc is eax - self.call(self.malloc_func_addr, arglocs, eax) - self.propagate_memoryerror_if_eax_is_null() - - def genop_new_array(self, op, arglocs, result_loc): - assert result_loc is eax - self.call(self.malloc_array_func_addr, arglocs, eax) - self.propagate_memoryerror_if_eax_is_null() - - def genop_newstr(self, op, arglocs, result_loc): - assert result_loc is eax - self.call(self.malloc_str_func_addr, arglocs, eax) - self.propagate_memoryerror_if_eax_is_null() - - def genop_newunicode(self, op, arglocs, result_loc): - assert result_loc is eax - self.call(self.malloc_unicode_func_addr, arglocs, eax) + def genop_call_malloc_gc(self, op, arglocs, result_loc): + self.genop_call(op, arglocs, result_loc) self.propagate_memoryerror_if_eax_is_null() def propagate_memoryerror_if_eax_is_null(self): @@ -2065,6 +2035,8 @@ self._genop_call(op, arglocs, resloc, force_index) def _genop_call(self, op, arglocs, resloc, force_index): + from pypy.jit.backend.llsupport.descr import CallDescr + sizeloc = arglocs[0] assert isinstance(sizeloc, ImmedLoc) size = sizeloc.value @@ -2079,13 +2051,16 @@ else: tmp = eax + descr = op.getdescr() + assert isinstance(descr, CallDescr) + self._emit_call(force_index, x, arglocs, 3, tmp=tmp, - argtypes=op.getdescr().get_arg_types(), - callconv=op.getdescr().get_call_conv()) + argtypes=descr.get_arg_types(), + callconv=descr.get_call_conv()) if IS_X86_32 and isinstance(resloc, StackLoc) and resloc.type == FLOAT: # a float or a long long return - if op.getdescr().get_return_type() == 'L': + if descr.get_result_type() == 'L': self.mc.MOV_br(resloc.value, eax.value) # long long self.mc.MOV_br(resloc.value + 4, edx.value) # XXX should ideally not move the result on the stack, @@ -2094,7 +2069,7 @@ # can just be always a stack location else: self.mc.FSTPL_b(resloc.value) # float return - elif op.getdescr().get_return_type() == 'S': + elif descr.get_result_type() == 'S': # singlefloat return assert resloc is eax if IS_X86_32: @@ -2292,9 +2267,9 @@ # # Reset the vable token --- XXX really too much special logic here:-( if jd.index_of_virtualizable >= 0: - from pypy.jit.backend.llsupport.descr import BaseFieldDescr + from pypy.jit.backend.llsupport.descr import FieldDescr fielddescr = jd.vable_token_descr - assert isinstance(fielddescr, BaseFieldDescr) + assert isinstance(fielddescr, FieldDescr) ofs = fielddescr.offset self.mc.MOV(eax, arglocs[1]) self.mc.MOV_mi((eax.value, ofs), 0) @@ -2497,9 +2472,8 @@ else: self.mc.JMP(imm(target)) - def malloc_cond(self, nursery_free_adr, nursery_top_adr, size, tid): - size = max(size, self.cpu.gc_ll_descr.minimal_size_in_nursery) - size = (size + WORD-1) & ~(WORD-1) # round up + def malloc_cond(self, nursery_free_adr, nursery_top_adr, size): + assert size & (WORD-1) == 0 # must be correctly aligned self.mc.MOV(eax, heap(nursery_free_adr)) self.mc.LEA_rm(edx.value, (eax.value, size)) self.mc.CMP(edx, heap(nursery_top_adr)) @@ -2535,9 +2509,6 @@ offset = self.mc.get_relative_pos() - jmp_adr assert 0 < offset <= 127 self.mc.overwrite(jmp_adr-1, chr(offset)) - # on 64-bits, 'tid' is a value that fits in 31 bits - assert rx86.fits_in_32bits(tid) - self.mc.MOV_mi((eax.value, 0), tid) self.mc.MOV(heap(nursery_free_adr), edx) genop_discard_list = [Assembler386.not_implemented_op_discard] * rop._LAST @@ -2584,3 +2555,6 @@ def not_implemented(msg): os.write(2, '[x86/asm] %s\n' % msg) raise NotImplementedError(msg) + +class BridgeAlreadyCompiled(Exception): + pass diff --git a/pypy/jit/backend/x86/jump.py b/pypy/jit/backend/x86/jump.py --- a/pypy/jit/backend/x86/jump.py +++ b/pypy/jit/backend/x86/jump.py @@ -17,7 +17,10 @@ key = src._getregkey() if key in srccount: if key == dst_locations[i]._getregkey(): - srccount[key] = -sys.maxint # ignore a move "x = x" + # ignore a move "x = x" + # setting any "large enough" negative value is ok, but + # be careful of overflows, don't use -sys.maxint + srccount[key] = -len(dst_locations) - 1 pending_dests -= 1 else: srccount[key] += 1 diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py --- a/pypy/jit/backend/x86/regalloc.py +++ b/pypy/jit/backend/x86/regalloc.py @@ -16,8 +16,8 @@ from pypy.jit.codewriter import heaptracker, longlong from pypy.jit.codewriter.effectinfo import EffectInfo from pypy.jit.metainterp.resoperation import rop -from pypy.jit.backend.llsupport.descr import BaseFieldDescr, BaseArrayDescr -from pypy.jit.backend.llsupport.descr import BaseCallDescr, BaseSizeDescr +from pypy.jit.backend.llsupport.descr import FieldDescr, ArrayDescr +from pypy.jit.backend.llsupport.descr import CallDescr, SizeDescr from pypy.jit.backend.llsupport.descr import InteriorFieldDescr from pypy.jit.backend.llsupport.regalloc import FrameManager, RegisterManager,\ TempBox @@ -188,7 +188,10 @@ # note: we need to make a copy of inputargs because possibly_free_vars # is also used on op args, which is a non-resizable list self.possibly_free_vars(list(inputargs)) - self.min_bytes_before_label = 13 + if WORD == 4: # see redirect_call_assembler() + self.min_bytes_before_label = 5 + else: + self.min_bytes_before_label = 13 return operations def prepare_bridge(self, prev_depths, inputargs, arglocs, operations, @@ -741,7 +744,7 @@ self.xrm.possibly_free_var(op.getarg(0)) def consider_cast_int_to_float(self, op): - loc0 = self.rm.loc(op.getarg(0)) + loc0 = self.rm.make_sure_var_in_reg(op.getarg(0)) loc1 = self.xrm.force_allocate_reg(op.result) self.Perform(op, [loc0], loc1) self.rm.possibly_free_var(op.getarg(0)) @@ -870,9 +873,9 @@ def _consider_call(self, op, guard_not_forced_op=None): calldescr = op.getdescr() - assert isinstance(calldescr, BaseCallDescr) + assert isinstance(calldescr, CallDescr) assert len(calldescr.arg_classes) == op.numargs() - 1 - size = calldescr.get_result_size(self.translate_support_code) + size = calldescr.get_result_size() sign = calldescr.is_result_signed() if sign: sign_loc = imm1 @@ -917,12 +920,15 @@ consider_call_release_gil = consider_call_may_force + def consider_call_malloc_gc(self, op): + self._consider_call(op) + def consider_call_assembler(self, op, guard_op): descr = op.getdescr() assert isinstance(descr, JitCellToken) jd = descr.outermost_jitdriver_sd assert jd is not None - size = jd.portal_calldescr.get_result_size(self.translate_support_code) + size = jd.portal_calldescr.get_result_size() vable_index = jd.index_of_virtualizable if vable_index >= 0: self.rm._sync_var(op.getarg(vable_index)) @@ -957,21 +963,10 @@ consider_cond_call_gc_wb_array = consider_cond_call_gc_wb - def fastpath_malloc_fixedsize(self, op, descr): - assert isinstance(descr, BaseSizeDescr) - self._do_fastpath_malloc(op, descr.size, descr.tid) - - def fastpath_malloc_varsize(self, op, arraydescr, num_elem): - assert isinstance(arraydescr, BaseArrayDescr) - ofs_length = arraydescr.get_ofs_length(self.translate_support_code) - basesize = arraydescr.get_base_size(self.translate_support_code) - itemsize = arraydescr.get_item_size(self.translate_support_code) - size = basesize + itemsize * num_elem - self._do_fastpath_malloc(op, size, arraydescr.tid) - self.assembler.set_new_array_length(eax, ofs_length, imm(num_elem)) - - def _do_fastpath_malloc(self, op, size, tid): - gc_ll_descr = self.assembler.cpu.gc_ll_descr + def consider_call_malloc_nursery(self, op): + size_box = op.getarg(0) + assert isinstance(size_box, ConstInt) + size = size_box.getint() self.rm.force_allocate_reg(op.result, selected_reg=eax) # # We need edx as a temporary, but otherwise don't save any more @@ -980,86 +975,39 @@ self.rm.force_allocate_reg(tmp_box, selected_reg=edx) self.rm.possibly_free_var(tmp_box) # + gc_ll_descr = self.assembler.cpu.gc_ll_descr self.assembler.malloc_cond( gc_ll_descr.get_nursery_free_addr(), gc_ll_descr.get_nursery_top_addr(), - size, tid, - ) - - def consider_new(self, op): - gc_ll_descr = self.assembler.cpu.gc_ll_descr - if gc_ll_descr.can_inline_malloc(op.getdescr()): - self.fastpath_malloc_fixedsize(op, op.getdescr()) - else: - args = gc_ll_descr.args_for_new(op.getdescr()) - arglocs = [imm(x) for x in args] - return self._call(op, arglocs) - - def consider_new_with_vtable(self, op): - classint = op.getarg(0).getint() - descrsize = heaptracker.vtable2descr(self.assembler.cpu, classint) - if self.assembler.cpu.gc_ll_descr.can_inline_malloc(descrsize): - self.fastpath_malloc_fixedsize(op, descrsize) - self.assembler.set_vtable(eax, imm(classint)) - # result of fastpath malloc is in eax - else: - args = self.assembler.cpu.gc_ll_descr.args_for_new(descrsize) - arglocs = [imm(x) for x in args] - arglocs.append(self.loc(op.getarg(0))) - return self._call(op, arglocs) - - def consider_newstr(self, op): - loc = self.loc(op.getarg(0)) - return self._call(op, [loc]) - - def consider_newunicode(self, op): - loc = self.loc(op.getarg(0)) - return self._call(op, [loc]) - - def consider_new_array(self, op): - gc_ll_descr = self.assembler.cpu.gc_ll_descr - box_num_elem = op.getarg(0) - if isinstance(box_num_elem, ConstInt): - num_elem = box_num_elem.value - if gc_ll_descr.can_inline_malloc_varsize(op.getdescr(), - num_elem): - self.fastpath_malloc_varsize(op, op.getdescr(), num_elem) - return - args = self.assembler.cpu.gc_ll_descr.args_for_new_array( - op.getdescr()) - arglocs = [imm(x) for x in args] - arglocs.append(self.loc(box_num_elem)) - self._call(op, arglocs) + size) def _unpack_arraydescr(self, arraydescr): - assert isinstance(arraydescr, BaseArrayDescr) - ofs_length = arraydescr.get_ofs_length(self.translate_support_code) - ofs = arraydescr.get_base_size(self.translate_support_code) - size = arraydescr.get_item_size(self.translate_support_code) - ptr = arraydescr.is_array_of_pointers() + assert isinstance(arraydescr, ArrayDescr) + ofs = arraydescr.basesize + size = arraydescr.itemsize sign = arraydescr.is_item_signed() - return size, ofs, ofs_length, ptr, sign + return size, ofs, sign def _unpack_fielddescr(self, fielddescr): - assert isinstance(fielddescr, BaseFieldDescr) + assert isinstance(fielddescr, FieldDescr) ofs = fielddescr.offset - size = fielddescr.get_field_size(self.translate_support_code) - ptr = fielddescr.is_pointer_field() + size = fielddescr.field_size sign = fielddescr.is_field_signed() - return imm(ofs), imm(size), ptr, sign + return imm(ofs), imm(size), sign + _unpack_fielddescr._always_inline_ = True def _unpack_interiorfielddescr(self, descr): assert isinstance(descr, InteriorFieldDescr) arraydescr = descr.arraydescr - ofs = arraydescr.get_base_size(self.translate_support_code) - itemsize = arraydescr.get_item_size(self.translate_support_code) - fieldsize = descr.fielddescr.get_field_size(self.translate_support_code) + ofs = arraydescr.basesize + itemsize = arraydescr.itemsize + fieldsize = descr.fielddescr.field_size sign = descr.fielddescr.is_field_signed() ofs += descr.fielddescr.offset return imm(ofs), imm(itemsize), imm(fieldsize), sign def consider_setfield_gc(self, op): - ofs_loc, size_loc, _, _ = self._unpack_fielddescr(op.getdescr()) + ofs_loc, size_loc, _ = self._unpack_fielddescr(op.getdescr()) assert isinstance(size_loc, ImmedLoc) if size_loc.value == 1: need_lower_byte = True @@ -1117,7 +1065,7 @@ consider_unicodesetitem = consider_strsetitem def consider_setarrayitem_gc(self, op): - itemsize, ofs, _, _, _ = self._unpack_arraydescr(op.getdescr()) + itemsize, ofs, _ = self._unpack_arraydescr(op.getdescr()) args = op.getarglist() base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args) if itemsize == 1: @@ -1134,7 +1082,7 @@ consider_setarrayitem_raw = consider_setarrayitem_gc def consider_getfield_gc(self, op): - ofs_loc, size_loc, _, sign = self._unpack_fielddescr(op.getdescr()) + ofs_loc, size_loc, sign = self._unpack_fielddescr(op.getdescr()) args = op.getarglist() base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args) self.rm.possibly_free_vars(args) @@ -1150,7 +1098,7 @@ consider_getfield_gc_pure = consider_getfield_gc def consider_getarrayitem_gc(self, op): - itemsize, ofs, _, _, sign = self._unpack_arraydescr(op.getdescr()) + itemsize, ofs, sign = self._unpack_arraydescr(op.getdescr()) args = op.getarglist() base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args) ofs_loc = self.rm.make_sure_var_in_reg(op.getarg(1), args) @@ -1229,8 +1177,8 @@ def consider_arraylen_gc(self, op): arraydescr = op.getdescr() - assert isinstance(arraydescr, BaseArrayDescr) - ofs = arraydescr.get_ofs_length(self.translate_support_code) + assert isinstance(arraydescr, ArrayDescr) + ofs = arraydescr.lendescr.offset args = op.getarglist() base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args) self.rm.possibly_free_vars_for_op(op) diff --git a/pypy/jit/backend/x86/test/test_gc_integration.py b/pypy/jit/backend/x86/test/test_gc_integration.py --- a/pypy/jit/backend/x86/test/test_gc_integration.py +++ b/pypy/jit/backend/x86/test/test_gc_integration.py @@ -8,7 +8,7 @@ from pypy.jit.metainterp.resoperation import rop, ResOperation from pypy.jit.codewriter import heaptracker from pypy.jit.codewriter.effectinfo import EffectInfo -from pypy.jit.backend.llsupport.descr import GcCache +from pypy.jit.backend.llsupport.descr import GcCache, FieldDescr, FLAG_SIGNED from pypy.jit.backend.llsupport.gc import GcLLDescription from pypy.jit.backend.detect_cpu import getcpuclass from pypy.jit.backend.x86.regalloc import RegAlloc @@ -17,7 +17,7 @@ from pypy.rpython.lltypesystem import lltype, llmemory, rffi from pypy.rpython.annlowlevel import llhelper from pypy.rpython.lltypesystem import rclass, rstr -from pypy.jit.backend.llsupport.gc import GcLLDescr_framework, GcPtrFieldDescr +from pypy.jit.backend.llsupport.gc import GcLLDescr_framework from pypy.jit.backend.x86.test.test_regalloc import MockAssembler from pypy.jit.backend.x86.test.test_regalloc import BaseTestRegalloc @@ -41,20 +41,15 @@ return ['compressed'] + shape[1:] class MockGcDescr(GcCache): - def get_funcptr_for_new(self): - return 123 - get_funcptr_for_newarray = get_funcptr_for_new - get_funcptr_for_newstr = get_funcptr_for_new - get_funcptr_for_newunicode = get_funcptr_for_new get_malloc_slowpath_addr = None - + write_barrier_descr = None moving_gc = True gcrootmap = MockGcRootMap() def initialize(self): pass - record_constptrs = GcLLDescr_framework.record_constptrs.im_func + _record_constptrs = GcLLDescr_framework._record_constptrs.im_func rewrite_assembler = GcLLDescr_framework.rewrite_assembler.im_func class TestRegallocDirectGcIntegration(object): @@ -170,42 +165,32 @@ ''' self.interpret(ops, [0, 0, 0, 0, 0, 0, 0, 0, 0], run=False) +NOT_INITIALIZED = chr(0xdd) + class GCDescrFastpathMalloc(GcLLDescription): gcrootmap = None - expected_malloc_slowpath_size = WORD*2 + write_barrier_descr = None def __init__(self): - GcCache.__init__(self, False) + GcLLDescription.__init__(self, None) # create a nursery - NTP = rffi.CArray(lltype.Signed) - self.nursery = lltype.malloc(NTP, 16, flavor='raw') - self.addrs = lltype.malloc(rffi.CArray(lltype.Signed), 3, + NTP = rffi.CArray(lltype.Char) + self.nursery = lltype.malloc(NTP, 64, flavor='raw') + for i in range(64): + self.nursery[i] = NOT_INITIALIZED + self.addrs = lltype.malloc(rffi.CArray(lltype.Signed), 2, flavor='raw') self.addrs[0] = rffi.cast(lltype.Signed, self.nursery) - self.addrs[1] = self.addrs[0] + 16*WORD - self.addrs[2] = 0 - # 16 WORDs + self.addrs[1] = self.addrs[0] + 64 + self.calls = [] def malloc_slowpath(size): - assert size == self.expected_malloc_slowpath_size + self.calls.append(size) + # reset the nursery nadr = rffi.cast(lltype.Signed, self.nursery) self.addrs[0] = nadr + size - self.addrs[2] += 1 return nadr - self.malloc_slowpath = malloc_slowpath - self.MALLOC_SLOWPATH = lltype.FuncType([lltype.Signed], - lltype.Signed) - self._counter = 123000 - - def can_inline_malloc(self, descr): - return True - - def get_funcptr_for_new(self): - return 42 -# return llhelper(lltype.Ptr(self.NEW_TP), self.new) - - def init_size_descr(self, S, descr): - descr.tid = self._counter - self._counter += 1 + self.generate_function('malloc_nursery', malloc_slowpath, + [lltype.Signed], lltype.Signed) def get_nursery_free_addr(self): return rffi.cast(lltype.Signed, self.addrs) @@ -214,204 +199,61 @@ return rffi.cast(lltype.Signed, self.addrs) + WORD def get_malloc_slowpath_addr(self): - fptr = llhelper(lltype.Ptr(self.MALLOC_SLOWPATH), self.malloc_slowpath) - return rffi.cast(lltype.Signed, fptr) + return self.get_malloc_fn_addr('malloc_nursery') - get_funcptr_for_newarray = None - get_funcptr_for_newstr = None - get_funcptr_for_newunicode = None + def check_nothing_in_nursery(self): + # CALL_MALLOC_NURSERY should not write anything in the nursery + for i in range(64): + assert self.nursery[i] == NOT_INITIALIZED class TestMallocFastpath(BaseTestRegalloc): def setup_method(self, method): cpu = CPU(None, None) - cpu.vtable_offset = WORD cpu.gc_ll_descr = GCDescrFastpathMalloc() cpu.setup_once() + self.cpu = cpu - # hack: specify 'tid' explicitly, because this test is not running - # with the gc transformer - NODE = lltype.GcStruct('node', ('tid', lltype.Signed), - ('value', lltype.Signed)) - nodedescr = cpu.sizeof(NODE) - valuedescr = cpu.fielddescrof(NODE, 'value') - - self.cpu = cpu - self.nodedescr = nodedescr - vtable = lltype.malloc(rclass.OBJECT_VTABLE, immortal=True) - vtable_int = cpu.cast_adr_to_int(llmemory.cast_ptr_to_adr(vtable)) - NODE2 = lltype.GcStruct('node2', - ('parent', rclass.OBJECT), - ('tid', lltype.Signed), - ('vtable', lltype.Ptr(rclass.OBJECT_VTABLE))) - descrsize = cpu.sizeof(NODE2) - heaptracker.register_known_gctype(cpu, vtable, NODE2) - self.descrsize = descrsize - self.vtable_int = vtable_int - - self.namespace = locals().copy() - def test_malloc_fastpath(self): ops = ''' - [i0] - p0 = new(descr=nodedescr) - setfield_gc(p0, i0, descr=valuedescr) - finish(p0) + [] + p0 = call_malloc_nursery(16) + p1 = call_malloc_nursery(32) + p2 = call_malloc_nursery(16) + finish(p0, p1, p2) ''' - self.interpret(ops, [42]) - # check the nursery + self.interpret(ops, []) + # check the returned pointers gc_ll_descr = self.cpu.gc_ll_descr - assert gc_ll_descr.nursery[0] == self.nodedescr.tid - assert gc_ll_descr.nursery[1] == 42 nurs_adr = rffi.cast(lltype.Signed, gc_ll_descr.nursery) - assert gc_ll_descr.addrs[0] == nurs_adr + (WORD*2) - assert gc_ll_descr.addrs[2] == 0 # slowpath never called + ref = self.cpu.get_latest_value_ref + assert rffi.cast(lltype.Signed, ref(0)) == nurs_adr + 0 + assert rffi.cast(lltype.Signed, ref(1)) == nurs_adr + 16 + assert rffi.cast(lltype.Signed, ref(2)) == nurs_adr + 48 + # check the nursery content and state + gc_ll_descr.check_nothing_in_nursery() + assert gc_ll_descr.addrs[0] == nurs_adr + 64 + # slowpath never called + assert gc_ll_descr.calls == [] def test_malloc_slowpath(self): ops = ''' [] - p0 = new(descr=nodedescr) - p1 = new(descr=nodedescr) - p2 = new(descr=nodedescr) - p3 = new(descr=nodedescr) - p4 = new(descr=nodedescr) - p5 = new(descr=nodedescr) - p6 = new(descr=nodedescr) - p7 = new(descr=nodedescr) - p8 = new(descr=nodedescr) - finish(p0, p1, p2, p3, p4, p5, p6, p7, p8) + p0 = call_malloc_nursery(16) + p1 = call_malloc_nursery(32) + p2 = call_malloc_nursery(24) # overflow + finish(p0, p1, p2) ''' self.interpret(ops, []) + # check the returned pointers + gc_ll_descr = self.cpu.gc_ll_descr + nurs_adr = rffi.cast(lltype.Signed, gc_ll_descr.nursery) + ref = self.cpu.get_latest_value_ref + assert rffi.cast(lltype.Signed, ref(0)) == nurs_adr + 0 + assert rffi.cast(lltype.Signed, ref(1)) == nurs_adr + 16 + assert rffi.cast(lltype.Signed, ref(2)) == nurs_adr + 0 + # check the nursery content and state + gc_ll_descr.check_nothing_in_nursery() + assert gc_ll_descr.addrs[0] == nurs_adr + 24 # this should call slow path once - gc_ll_descr = self.cpu.gc_ll_descr - nadr = rffi.cast(lltype.Signed, gc_ll_descr.nursery) - assert gc_ll_descr.addrs[0] == nadr + (WORD*2) - assert gc_ll_descr.addrs[2] == 1 # slowpath called once - - def test_new_with_vtable(self): - ops = ''' - [i0, i1] - p0 = new_with_vtable(ConstClass(vtable)) - guard_class(p0, ConstClass(vtable)) [i0] - finish(i1) - ''' - self.interpret(ops, [0, 1]) - assert self.getint(0) == 1 - gc_ll_descr = self.cpu.gc_ll_descr - assert gc_ll_descr.nursery[0] == self.descrsize.tid - assert gc_ll_descr.nursery[1] == self.vtable_int - nurs_adr = rffi.cast(lltype.Signed, gc_ll_descr.nursery) - assert gc_ll_descr.addrs[0] == nurs_adr + (WORD*3) - assert gc_ll_descr.addrs[2] == 0 # slowpath never called - - -class Seen(Exception): - pass - -class GCDescrFastpathMallocVarsize(GCDescrFastpathMalloc): - def can_inline_malloc_varsize(self, arraydescr, num_elem): - return num_elem < 5 - def get_funcptr_for_newarray(self): - return 52 - def init_array_descr(self, A, descr): - descr.tid = self._counter - self._counter += 1 - def args_for_new_array(self, descr): - raise Seen("args_for_new_array") - -class TestMallocVarsizeFastpath(BaseTestRegalloc): - def setup_method(self, method): - cpu = CPU(None, None) - cpu.vtable_offset = WORD - cpu.gc_ll_descr = GCDescrFastpathMallocVarsize() - cpu.setup_once() - self.cpu = cpu - - ARRAY = lltype.GcArray(lltype.Signed) - arraydescr = cpu.arraydescrof(ARRAY) - self.arraydescr = arraydescr - ARRAYCHAR = lltype.GcArray(lltype.Char) - arraychardescr = cpu.arraydescrof(ARRAYCHAR) - - self.namespace = locals().copy() - - def test_malloc_varsize_fastpath(self): - # Hack. Running the GcLLDescr_framework without really having - # a complete GC means that we end up with both the tid and the - # length being at offset 0. In this case, so the length overwrites - # the tid. This is of course only the case in this test class. - ops = ''' - [] - p0 = new_array(4, descr=arraydescr) - setarrayitem_gc(p0, 0, 142, descr=arraydescr) - setarrayitem_gc(p0, 3, 143, descr=arraydescr) - finish(p0) - ''' - self.interpret(ops, []) - # check the nursery - gc_ll_descr = self.cpu.gc_ll_descr - assert gc_ll_descr.nursery[0] == 4 - assert gc_ll_descr.nursery[1] == 142 - assert gc_ll_descr.nursery[4] == 143 - nurs_adr = rffi.cast(lltype.Signed, gc_ll_descr.nursery) - assert gc_ll_descr.addrs[0] == nurs_adr + (WORD*5) - assert gc_ll_descr.addrs[2] == 0 # slowpath never called - - def test_malloc_varsize_slowpath(self): - ops = ''' - [] - p0 = new_array(4, descr=arraydescr) - setarrayitem_gc(p0, 0, 420, descr=arraydescr) - setarrayitem_gc(p0, 3, 430, descr=arraydescr) - p1 = new_array(4, descr=arraydescr) - setarrayitem_gc(p1, 0, 421, descr=arraydescr) - setarrayitem_gc(p1, 3, 431, descr=arraydescr) - p2 = new_array(4, descr=arraydescr) - setarrayitem_gc(p2, 0, 422, descr=arraydescr) - setarrayitem_gc(p2, 3, 432, descr=arraydescr) - p3 = new_array(4, descr=arraydescr) - setarrayitem_gc(p3, 0, 423, descr=arraydescr) - setarrayitem_gc(p3, 3, 433, descr=arraydescr) - finish(p0, p1, p2, p3) - ''' - gc_ll_descr = self.cpu.gc_ll_descr - gc_ll_descr.expected_malloc_slowpath_size = 5*WORD - self.interpret(ops, []) - assert gc_ll_descr.addrs[2] == 1 # slowpath called once - - def test_malloc_varsize_too_big(self): - ops = ''' - [] - p0 = new_array(5, descr=arraydescr) - finish(p0) - ''' - py.test.raises(Seen, self.interpret, ops, []) - - def test_malloc_varsize_variable(self): - ops = ''' - [i0] - p0 = new_array(i0, descr=arraydescr) - finish(p0) - ''' - py.test.raises(Seen, self.interpret, ops, []) - - def test_malloc_array_of_char(self): - # check that fastpath_malloc_varsize() respects the alignment - # of the pointer in the nursery - ops = ''' - [] - p1 = new_array(1, descr=arraychardescr) - p2 = new_array(2, descr=arraychardescr) - p3 = new_array(3, descr=arraychardescr) - p4 = new_array(4, descr=arraychardescr) - finish(p1, p2, p3, p4) - ''' - self.interpret(ops, []) - p1 = self.getptr(0, llmemory.GCREF) - p2 = self.getptr(1, llmemory.GCREF) - p3 = self.getptr(2, llmemory.GCREF) - p4 = self.getptr(3, llmemory.GCREF) - assert p1._obj.intval & (WORD-1) == 0 # aligned - assert p2._obj.intval & (WORD-1) == 0 # aligned - assert p3._obj.intval & (WORD-1) == 0 # aligned - assert p4._obj.intval & (WORD-1) == 0 # aligned + assert gc_ll_descr.calls == [24] diff --git a/pypy/jit/backend/x86/test/test_jump.py b/pypy/jit/backend/x86/test/test_jump.py --- a/pypy/jit/backend/x86/test/test_jump.py +++ b/pypy/jit/backend/x86/test/test_jump.py @@ -20,6 +20,11 @@ def regalloc_pop(self, loc): self.ops.append(('pop', loc)) + def regalloc_immedmem2mem(self, from_loc, to_loc): + assert isinstance(from_loc, ConstFloatLoc) + assert isinstance(to_loc, StackLoc) + self.ops.append(('immedmem2mem', from_loc, to_loc)) + def got(self, expected): print '------------------------ comparing ---------------------------' for op1, op2 in zip(self.ops, expected): @@ -244,6 +249,13 @@ else: return pick1() # + def pick2c(): + n = random.randrange(-2000, 500) + if n >= 0: + return ConstFloatLoc(n) # n is the address, not really used here + else: + return pick2() + # def pick_dst(fn, count, seen): result = [] while len(result) < count: @@ -280,12 +292,12 @@ if loc.get_width() > WORD: stack[loc.value+WORD] = 'value-hiword-%d' % i else: - assert isinstance(loc, ImmedLoc) + assert isinstance(loc, (ImmedLoc, ConstFloatLoc)) return regs1, regs2, stack # for i in range(500): seen = {} - src_locations2 = [pick2() for i in range(4)] + src_locations2 = [pick2c() for i in range(4)] dst_locations2 = pick_dst(pick2, 4, seen) src_locations1 = [pick1c() for i in range(5)] dst_locations1 = pick_dst(pick1, 5, seen) @@ -312,9 +324,15 @@ return got if isinstance(loc, ImmedLoc): return 'const-%d' % loc.value + if isinstance(loc, ConstFloatLoc): + got = 'constfloat-@%d' % loc.value + if loc.get_width() > WORD: + got = (got, 'constfloat-next-@%d' % loc.value) + return got assert 0, loc # def write(loc, newvalue): + assert (type(newvalue) is tuple) == (loc.get_width() > WORD) if isinstance(loc, RegLoc): if loc.is_xmm: regs2[loc.value] = newvalue @@ -337,10 +355,14 @@ for op in assembler.ops: if op[0] == 'mov': src, dst = op[1:] - assert isinstance(src, (RegLoc, StackLoc, ImmedLoc)) - assert isinstance(dst, (RegLoc, StackLoc)) - assert not (isinstance(src, StackLoc) and - isinstance(dst, StackLoc)) + if isinstance(src, ConstFloatLoc): + assert isinstance(dst, RegLoc) + assert dst.is_xmm + else: + assert isinstance(src, (RegLoc, StackLoc, ImmedLoc)) + assert isinstance(dst, (RegLoc, StackLoc)) + assert not (isinstance(src, StackLoc) and + isinstance(dst, StackLoc)) write(dst, read(src)) elif op[0] == 'push': src, = op[1:] @@ -350,6 +372,11 @@ dst, = op[1:] assert isinstance(dst, (RegLoc, StackLoc)) write(dst, extrapushes.pop()) + elif op[0] == 'immedmem2mem': + src, dst = op[1:] + assert isinstance(src, ConstFloatLoc) + assert isinstance(dst, StackLoc) + write(dst, read(src, 8)) else: assert 0, "unknown op: %r" % (op,) assert not extrapushes @@ -358,3 +385,32 @@ assert read(loc, WORD) == src_values1[i] for i, loc in enumerate(dst_locations2): assert read(loc, 8) == src_values2[i] + + +def test_overflow_bug(): + CASE = [ + (-144, -248), # \ cycle + (-248, -144), # / + (-488, -416), # \ two usages of -488 + (-488, -480), # / + (-488, -488), # - one self-application of -488 + ] + class FakeAssembler: + def regalloc_mov(self, src, dst): + print "mov", src, dst + def regalloc_push(self, x): + print "push", x + def regalloc_pop(self, x): + print "pop", x + def regalloc_immedmem2mem(self, x, y): + print "?????????????????????????" + def main(): + srclocs = [StackLoc(9999, x, 'i') for x,y in CASE] + dstlocs = [StackLoc(9999, y, 'i') for x,y in CASE] + remap_frame_layout(FakeAssembler(), srclocs, dstlocs, eax) + # it works when run directly + main() + # but it used to crash when translated, + # because of a -sys.maxint-2 overflowing to sys.maxint + from pypy.rpython.test.test_llinterp import interpret + interpret(main, []) diff --git a/pypy/jit/backend/x86/test/test_runner.py b/pypy/jit/backend/x86/test/test_runner.py --- a/pypy/jit/backend/x86/test/test_runner.py +++ b/pypy/jit/backend/x86/test/test_runner.py @@ -33,6 +33,13 @@ # for the individual tests see # ====> ../../test/runner_test.py + add_loop_instructions = ['mov', 'add', 'test', 'je', 'jmp'] + if WORD == 4: + bridge_loop_instructions = ['lea', 'jmp'] + else: + # the 'mov' is part of the 'jmp' so far + bridge_loop_instructions = ['lea', 'mov', 'jmp'] + def setup_method(self, meth): self.cpu = CPU(rtyper=None, stats=FakeStats()) self.cpu.setup_once() @@ -69,6 +76,7 @@ return ctypes.cast(res.value._obj.intval, ctypes.POINTER(item_tp)) def test_allocations(self): + py.test.skip("rewrite or kill") from pypy.rpython.lltypesystem import rstr allocs = [None] @@ -415,12 +423,13 @@ ] inputargs = [i0] debug._log = dlog = debug.DebugLog() - ops_offset = self.cpu.compile_loop(inputargs, operations, looptoken) + info = self.cpu.compile_loop(inputargs, operations, looptoken) + ops_offset = info.ops_offset debug._log = None # assert ops_offset is looptoken._x86_ops_offset - # getfield_raw/int_add/setfield_raw + ops + None - assert len(ops_offset) == 3 + len(operations) + 1 + # 2*(getfield_raw/int_add/setfield_raw) + ops + None + assert len(ops_offset) == 2*3 + len(operations) + 1 assert (ops_offset[operations[0]] <= ops_offset[operations[1]] <= ops_offset[operations[2]] <= @@ -518,16 +527,23 @@ from pypy.tool.logparser import parse_log_file, extract_category from pypy.rlib import debug + targettoken, preambletoken = TargetToken(), TargetToken() loop = """ [i0] - label(i0, descr=targettoken) + label(i0, descr=preambletoken) debug_merge_point('xyz', 0) i1 = int_add(i0, 1) i2 = int_ge(i1, 10) guard_false(i2) [] - jump(i1, descr=targettoken) + label(i1, descr=targettoken) + debug_merge_point('xyz', 0) + i11 = int_add(i1, 1) + i12 = int_ge(i11, 10) + guard_false(i12) [] + jump(i11, descr=targettoken) """ - ops = parse(loop, namespace={'targettoken': TargetToken()}) + ops = parse(loop, namespace={'targettoken': targettoken, + 'preambletoken': preambletoken}) debug._log = dlog = debug.DebugLog() try: self.cpu.assembler.set_debug(True) @@ -536,11 +552,18 @@ self.cpu.execute_token(looptoken, 0) # check debugging info struct = self.cpu.assembler.loop_run_counters[0] - assert struct.i == 10 + assert struct.i == 1 + struct = self.cpu.assembler.loop_run_counters[1] + assert struct.i == 1 + struct = self.cpu.assembler.loop_run_counters[2] + assert struct.i == 9 self.cpu.finish_once() finally: debug._log = None - assert ('jit-backend-counts', [('debug_print', 'loop -1:10')]) in dlog + l0 = ('debug_print', 'entry -1:1') + l1 = ('debug_print', preambletoken.repr_of_descr() + ':1') + l2 = ('debug_print', targettoken.repr_of_descr() + ':9') + assert ('jit-backend-counts', [l0, l1, l2]) in dlog def test_debugger_checksum(self): loop = """ diff --git a/pypy/jit/backend/x86/test/test_zrpy_gc.py b/pypy/jit/backend/x86/test/test_zrpy_gc.py --- a/pypy/jit/backend/x86/test/test_zrpy_gc.py +++ b/pypy/jit/backend/x86/test/test_zrpy_gc.py @@ -69,16 +69,17 @@ def get_functions_to_patch(): from pypy.jit.backend.llsupport import gc # - can_inline_malloc1 = gc.GcLLDescr_framework.can_inline_malloc - def can_inline_malloc2(*args): + can_use_nursery_malloc1 = gc.GcLLDescr_framework.can_use_nursery_malloc + def can_use_nursery_malloc2(*args): try: if os.environ['PYPY_NO_INLINE_MALLOC']: return False except KeyError: pass - return can_inline_malloc1(*args) + return can_use_nursery_malloc1(*args) # - return {(gc.GcLLDescr_framework, 'can_inline_malloc'): can_inline_malloc2} + return {(gc.GcLLDescr_framework, 'can_use_nursery_malloc'): + can_use_nursery_malloc2} def compile(f, gc, enable_opts='', **kwds): from pypy.annotation.listdef import s_list_of_strings diff --git a/pypy/jit/backend/x86/test/test_zrpy_platform.py b/pypy/jit/backend/x86/test/test_zrpy_platform.py --- a/pypy/jit/backend/x86/test/test_zrpy_platform.py +++ b/pypy/jit/backend/x86/test/test_zrpy_platform.py @@ -74,8 +74,8 @@ myjitdriver = jit.JitDriver(greens = [], reds = ['n']) def entrypoint(argv): - myjitdriver.set_param('threshold', 2) - myjitdriver.set_param('trace_eagerness', 0) + jit.set_param(myjitdriver, 'threshold', 2) + jit.set_param(myjitdriver, 'trace_eagerness', 0) n = 16 while n > 0: myjitdriver.can_enter_jit(n=n) diff --git a/pypy/jit/backend/x86/tool/viewcode.py b/pypy/jit/backend/x86/tool/viewcode.py --- a/pypy/jit/backend/x86/tool/viewcode.py +++ b/pypy/jit/backend/x86/tool/viewcode.py @@ -39,6 +39,7 @@ def machine_code_dump(data, originaddr, backend_name, label_list=None): objdump_backend_option = { 'x86': 'i386', + 'x86_32': 'i386', 'x86_64': 'x86-64', 'i386': 'i386', } diff --git a/pypy/jit/codewriter/call.py b/pypy/jit/codewriter/call.py --- a/pypy/jit/codewriter/call.py +++ b/pypy/jit/codewriter/call.py @@ -42,8 +42,7 @@ except AttributeError: pass - def is_candidate(graph): - return policy.look_inside_graph(graph) + is_candidate = policy.look_inside_graph assert len(self.jitdrivers_sd) > 0 todo = [jd.portal_graph for jd in self.jitdrivers_sd] diff --git a/pypy/jit/codewriter/policy.py b/pypy/jit/codewriter/policy.py --- a/pypy/jit/codewriter/policy.py +++ b/pypy/jit/codewriter/policy.py @@ -8,11 +8,15 @@ class JitPolicy(object): - def __init__(self): + def __init__(self, jithookiface=None): self.unsafe_loopy_graphs = set() self.supports_floats = False self.supports_longlong = False self.supports_singlefloats = False + if jithookiface is None: + from pypy.rlib.jit import JitHookInterface + jithookiface = JitHookInterface() + self.jithookiface = jithookiface def set_supports_floats(self, flag): self.supports_floats = flag diff --git a/pypy/jit/codewriter/support.py b/pypy/jit/codewriter/support.py --- a/pypy/jit/codewriter/support.py +++ b/pypy/jit/codewriter/support.py @@ -162,7 +162,6 @@ _ll_4_list_setslice = rlist.ll_listsetslice _ll_2_list_delslice_startonly = rlist.ll_listdelslice_startonly _ll_3_list_delslice_startstop = rlist.ll_listdelslice_startstop -_ll_1_list_list2fixed = lltypesystem_rlist.ll_list2fixed _ll_2_list_inplace_mul = rlist.ll_inplace_mul _ll_2_list_getitem_foldable = _ll_2_list_getitem diff --git a/pypy/jit/metainterp/compile.py b/pypy/jit/metainterp/compile.py --- a/pypy/jit/metainterp/compile.py +++ b/pypy/jit/metainterp/compile.py @@ -5,6 +5,7 @@ from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.debug import debug_start, debug_stop, debug_print from pypy.rlib import rstack +from pypy.rlib.jit import JitDebugInfo from pypy.conftest import option from pypy.tool.sourcetools import func_with_new_name @@ -38,7 +39,8 @@ else: extraprocedures = [procedure] metainterp_sd.stats.view(errmsg=errmsg, - extraprocedures=extraprocedures) + extraprocedures=extraprocedures, + metainterp_sd=metainterp_sd) def create_empty_loop(metainterp, name_prefix=''): name = metainterp.staticdata.stats.name_for_new_loop() @@ -74,7 +76,7 @@ if descr is not original_jitcell_token: original_jitcell_token.record_jump_to(descr) descr.exported_state = None - op._descr = None # clear reference, mostly for tests + op.cleardescr() # clear reference, mostly for tests elif isinstance(descr, TargetToken): # for a JUMP: record it as a potential jump. # (the following test is not enough to prevent more complicated @@ -89,8 +91,8 @@ assert descr.exported_state is None if not we_are_translated(): op._descr_wref = weakref.ref(op._descr) - op._descr = None # clear reference to prevent the history.Stats - # from keeping the loop alive during tests + op.cleardescr() # clear reference to prevent the history.Stats + # from keeping the loop alive during tests # record this looptoken on the QuasiImmut used in the code if loop.quasi_immutable_deps is not None: for qmut in loop.quasi_immutable_deps: @@ -105,38 +107,32 @@ def compile_loop(metainterp, greenkey, start, inputargs, jumpargs, - start_resumedescr, full_preamble_needed=True): + resume_at_jump_descr, full_preamble_needed=True): """Try to compile a new procedure by closing the current history back to the first operation. """ from pypy.jit.metainterp.optimizeopt import optimize_trace - history = metainterp.history metainterp_sd = metainterp.staticdata jitdriver_sd = metainterp.jitdriver_sd + history = metainterp.history - if False: - part = partial_trace - assert False - procedur_token = metainterp.get_procedure_token(greenkey) - assert procedure_token - all_target_tokens = [] - else: - jitcell_token = make_jitcell_token(jitdriver_sd) - part = create_empty_loop(metainterp) - part.inputargs = inputargs[:] - h_ops = history.operations - part.start_resumedescr = start_resumedescr - part.operations = [ResOperation(rop.LABEL, inputargs, None, descr=TargetToken(jitcell_token))] + \ - [h_ops[i].clone() for i in range(start, len(h_ops))] + \ - [ResOperation(rop.JUMP, jumpargs, None, descr=jitcell_token)] - try: - optimize_trace(metainterp_sd, part, jitdriver_sd.warmstate.enable_opts) - except InvalidLoop: - return None - target_token = part.operations[0].getdescr() - assert isinstance(target_token, TargetToken) - all_target_tokens = [target_token] + jitcell_token = make_jitcell_token(jitdriver_sd) + part = create_empty_loop(metainterp) + part.inputargs = inputargs[:] + h_ops = history.operations + part.resume_at_jump_descr = resume_at_jump_descr + part.operations = [ResOperation(rop.LABEL, inputargs, None, descr=TargetToken(jitcell_token))] + \ + [h_ops[i].clone() for i in range(start, len(h_ops))] + \ + [ResOperation(rop.LABEL, jumpargs, None, descr=jitcell_token)] + + try: + optimize_trace(metainterp_sd, part, jitdriver_sd.warmstate.enable_opts) + except InvalidLoop: + return None + target_token = part.operations[0].getdescr() + assert isinstance(target_token, TargetToken) + all_target_tokens = [target_token] loop = create_empty_loop(metainterp) loop.inputargs = part.inputargs @@ -174,17 +170,17 @@ loop.original_jitcell_token = jitcell_token for label in all_target_tokens: assert isinstance(label, TargetToken) - label.original_jitcell_token = jitcell_token if label.virtual_state and label.short_preamble: metainterp_sd.logger_ops.log_short_preamble([], label.short_preamble) jitcell_token.target_tokens = all_target_tokens + propagate_original_jitcell_token(loop) send_loop_to_backend(greenkey, jitdriver_sd, metainterp_sd, loop, "loop") record_loop_or_bridge(metainterp_sd, loop) return all_target_tokens[0] def compile_retrace(metainterp, greenkey, start, inputargs, jumpargs, - start_resumedescr, partial_trace, resumekey): + resume_at_jump_descr, partial_trace, resumekey): """Try to compile a new procedure by closing the current history back to the first operation. """ @@ -200,7 +196,7 @@ part = create_empty_loop(metainterp) part.inputargs = inputargs[:] - part.start_resumedescr = start_resumedescr + part.resume_at_jump_descr = resume_at_jump_descr h_ops = history.operations part.operations = [partial_trace.operations[-1]] + \ @@ -212,13 +208,12 @@ try: optimize_trace(metainterp_sd, part, jitdriver_sd.warmstate.enable_opts) except InvalidLoop: - #return None # XXX: Dissable for now # Fall back on jumping to preamble target_token = label.getdescr() assert isinstance(target_token, TargetToken) assert target_token.exported_state part.operations = [orignial_label] + \ - [ResOperation(rop.JUMP, target_token.exported_state.jump_args, + [ResOperation(rop.JUMP, inputargs[:], None, descr=loop_jitcell_token)] try: optimize_trace(metainterp_sd, part, jitdriver_sd.warmstate.enable_opts, @@ -246,11 +241,11 @@ for box in loop.inputargs: assert isinstance(box, Box) - target_token = loop.operations[-1].getdescr() + target_token = loop.operations[-1].getdescr() resumekey.compile_and_attach(metainterp, loop) + target_token = label.getdescr() assert isinstance(target_token, TargetToken) - target_token.original_jitcell_token = loop.original_jitcell_token record_loop_or_bridge(metainterp_sd, loop) return target_token @@ -287,14 +282,21 @@ assert i == len(inputargs) loop.operations = extra_ops + loop.operations +def propagate_original_jitcell_token(trace): + for op in trace.operations: + if op.getopnum() == rop.LABEL: + token = op.getdescr() + assert isinstance(token, TargetToken) + assert token.original_jitcell_token is None + token.original_jitcell_token = trace.original_jitcell_token + + def send_loop_to_backend(greenkey, jitdriver_sd, metainterp_sd, loop, type): vinfo = jitdriver_sd.virtualizable_info if vinfo is not None: patch_new_loop_to_load_virtualizable_fields(loop, jitdriver_sd) original_jitcell_token = loop.original_jitcell_token - jitdriver_sd.on_compile(metainterp_sd.logger_ops, original_jitcell_token, - loop.operations, type, greenkey) loopname = jitdriver_sd.warmstate.get_location_str(greenkey) globaldata = metainterp_sd.globaldata original_jitcell_token.number = n = globaldata.loopnumbering @@ -304,21 +306,41 @@ show_procedures(metainterp_sd, loop) loop.check_consistency() + if metainterp_sd.warmrunnerdesc is not None: + hooks = metainterp_sd.warmrunnerdesc.hooks + debug_info = JitDebugInfo(jitdriver_sd, metainterp_sd.logger_ops, + original_jitcell_token, loop.operations, + type, greenkey) + hooks.before_compile(debug_info) + else: + debug_info = None + hooks = None operations = get_deep_immutable_oplist(loop.operations) metainterp_sd.profiler.start_backend() debug_start("jit-backend") try: - ops_offset = metainterp_sd.cpu.compile_loop(loop.inputargs, operations, - original_jitcell_token, name=loopname) + asminfo = metainterp_sd.cpu.compile_loop(loop.inputargs, operations, + original_jitcell_token, + name=loopname) finally: debug_stop("jit-backend") metainterp_sd.profiler.end_backend() + if hooks is not None: + debug_info.asminfo = asminfo + hooks.after_compile(debug_info) metainterp_sd.stats.add_new_loop(loop) if not we_are_translated(): metainterp_sd.stats.compiled() metainterp_sd.log("compiled new " + type) # - metainterp_sd.logger_ops.log_loop(loop.inputargs, loop.operations, n, type, ops_offset) + loopname = jitdriver_sd.warmstate.get_location_str(greenkey) + if asminfo is not None: + ops_offset = asminfo.ops_offset + else: + ops_offset = None + metainterp_sd.logger_ops.log_loop(loop.inputargs, loop.operations, n, + type, ops_offset, + name=loopname) # if metainterp_sd.warmrunnerdesc is not None: # for tests metainterp_sd.warmrunnerdesc.memory_manager.keep_loop_alive(original_jitcell_token) @@ -326,25 +348,40 @@ def send_bridge_to_backend(jitdriver_sd, metainterp_sd, faildescr, inputargs, operations, original_loop_token): n = metainterp_sd.cpu.get_fail_descr_number(faildescr) - jitdriver_sd.on_compile_bridge(metainterp_sd.logger_ops, - original_loop_token, operations, n) if not we_are_translated(): show_procedures(metainterp_sd) seen = dict.fromkeys(inputargs) TreeLoop.check_consistency_of_branch(operations, seen) + if metainterp_sd.warmrunnerdesc is not None: + hooks = metainterp_sd.warmrunnerdesc.hooks + debug_info = JitDebugInfo(jitdriver_sd, metainterp_sd.logger_ops, + original_loop_token, operations, 'bridge', + fail_descr_no=n) + hooks.before_compile_bridge(debug_info) + else: + hooks = None + debug_info = None + operations = get_deep_immutable_oplist(operations) metainterp_sd.profiler.start_backend() - operations = get_deep_immutable_oplist(operations) debug_start("jit-backend") try: - ops_offset = metainterp_sd.cpu.compile_bridge(faildescr, inputargs, operations, - original_loop_token) + asminfo = metainterp_sd.cpu.compile_bridge(faildescr, inputargs, + operations, + original_loop_token) finally: debug_stop("jit-backend") metainterp_sd.profiler.end_backend() + if hooks is not None: + debug_info.asminfo = asminfo + hooks.after_compile_bridge(debug_info) if not we_are_translated(): metainterp_sd.stats.compiled() metainterp_sd.log("compiled new bridge") # + if asminfo is not None: + ops_offset = asminfo.ops_offset + else: + ops_offset = None metainterp_sd.logger_ops.log_bridge(inputargs, operations, n, ops_offset) # #if metainterp_sd.warmrunnerdesc is not None: # for tests @@ -557,6 +594,7 @@ inputargs = metainterp.history.inputargs if not we_are_translated(): self._debug_suboperations = new_loop.operations + propagate_original_jitcell_token(new_loop) send_bridge_to_backend(metainterp.jitdriver_sd, metainterp.staticdata, self, inputargs, new_loop.operations, new_loop.original_jitcell_token) @@ -743,6 +781,7 @@ jitdriver_sd = metainterp.jitdriver_sd redargs = new_loop.inputargs new_loop.original_jitcell_token = jitcell_token = make_jitcell_token(jitdriver_sd) + propagate_original_jitcell_token(new_loop) send_loop_to_backend(self.original_greenkey, metainterp.jitdriver_sd, metainterp_sd, new_loop, "entry bridge") # send the new_loop to warmspot.py, to be called directly the next time @@ -751,7 +790,7 @@ metainterp_sd.stats.add_jitcell_token(jitcell_token) -def compile_trace(metainterp, resumekey, start_resumedescr=None): +def compile_trace(metainterp, resumekey, resume_at_jump_descr=None): """Try to compile a new bridge leading from the beginning of the history to some existing place. """ @@ -767,7 +806,7 @@ # clone ops, as optimize_bridge can mutate the ops new_trace.operations = [op.clone() for op in metainterp.history.operations] - new_trace.start_resumedescr = start_resumedescr + new_trace.resume_at_jump_descr = resume_at_jump_descr metainterp_sd = metainterp.staticdata state = metainterp.jitdriver_sd.warmstate if isinstance(resumekey, ResumeAtPositionDescr): diff --git a/pypy/jit/metainterp/executor.py b/pypy/jit/metainterp/executor.py --- a/pypy/jit/metainterp/executor.py +++ b/pypy/jit/metainterp/executor.py @@ -46,7 +46,7 @@ # get the function address as an integer func = argboxes[0].getint() # do the call using the correct function from the cpu - rettype = descr.get_return_type() + rettype = descr.get_result_type() if rettype == INT or rettype == 'S': # *S*ingle float try: result = cpu.bh_call_i(func, descr, args_i, args_r, args_f) @@ -344,6 +344,8 @@ rop.SETINTERIORFIELD_RAW, rop.CALL_RELEASE_GIL, rop.QUASIIMMUT_FIELD, + rop.CALL_MALLOC_GC, + rop.CALL_MALLOC_NURSERY, rop.LABEL, ): # list of opcodes never executed by pyjitpl continue diff --git a/pypy/jit/metainterp/graphpage.py b/pypy/jit/metainterp/graphpage.py --- a/pypy/jit/metainterp/graphpage.py +++ b/pypy/jit/metainterp/graphpage.py @@ -12,7 +12,7 @@ def get_display_text(self): return None -def display_procedures(procedures, errmsg=None, highlight_procedures={}): +def display_procedures(procedures, errmsg=None, highlight_procedures={}, metainterp_sd=None): graphs = [(procedure, highlight_procedures.get(procedure, 0)) for procedure in procedures] for graph, highlight in graphs: @@ -20,7 +20,7 @@ if is_interesting_guard(op): graphs.append((SubGraph(op.getdescr()._debug_suboperations), highlight)) - graphpage = ResOpGraphPage(graphs, errmsg) + graphpage = ResOpGraphPage(graphs, errmsg, metainterp_sd) graphpage.display() def is_interesting_guard(op): @@ -36,8 +36,8 @@ class ResOpGraphPage(GraphPage): - def compute(self, graphs, errmsg=None): - resopgen = ResOpGen() + def compute(self, graphs, errmsg=None, metainterp_sd=None): + resopgen = ResOpGen(metainterp_sd) for graph, highlight in graphs: resopgen.add_graph(graph, highlight) if errmsg: @@ -50,13 +50,14 @@ CLUSTERING = True BOX_COLOR = (128, 0, 96) - def __init__(self): + def __init__(self, metainterp_sd=None): self.graphs = [] self.highlight_graphs = {} self.block_starters = {} # {graphindex: {set-of-operation-indices}} self.all_operations = {} self.errmsg = None self.target_tokens = {} + self.metainterp_sd = metainterp_sd def op_name(self, graphindex, opindex): return 'g%dop%d' % (graphindex, opindex) @@ -164,7 +165,14 @@ opindex = opstartindex while True: op = operations[opindex] - lines.append(op.repr(graytext=True)) + op_repr = op.repr(graytext=True) + if op.getopnum() == rop.DEBUG_MERGE_POINT: + jd_sd = self.metainterp_sd.jitdrivers_sd[op.getarg(0).getint()] + if jd_sd._get_printable_location_ptr: + s = jd_sd.warmstate.get_location_str(op.getarglist()[2:]) + s = s.replace(',', '.') # we use comma for argument splitting + op_repr = "debug_merge_point(%d, '%s')" % (op.getarg(1).getint(), s) + lines.append(op_repr) if is_interesting_guard(op): tgt = op.getdescr()._debug_suboperations[0] tgt_g, tgt_i = self.all_operations[tgt] diff --git a/pypy/jit/metainterp/heapcache.py b/pypy/jit/metainterp/heapcache.py --- a/pypy/jit/metainterp/heapcache.py +++ b/pypy/jit/metainterp/heapcache.py @@ -79,9 +79,9 @@ opnum == rop.COPYSTRCONTENT or opnum == rop.COPYUNICODECONTENT): return - if rop._OVF_FIRST <= opnum <= rop._OVF_LAST: - return - if rop._NOSIDEEFFECT_FIRST <= opnum <= rop._NOSIDEEFFECT_LAST: + if (rop._OVF_FIRST <= opnum <= rop._OVF_LAST or + rop._NOSIDEEFFECT_FIRST <= opnum <= rop._NOSIDEEFFECT_LAST or + rop._GUARD_FIRST <= opnum <= rop._GUARD_LAST): return if opnum == rop.CALL or opnum == rop.CALL_LOOPINVARIANT: effectinfo = descr.get_extra_info() diff --git a/pypy/jit/metainterp/history.py b/pypy/jit/metainterp/history.py --- a/pypy/jit/metainterp/history.py +++ b/pypy/jit/metainterp/history.py @@ -142,59 +142,6 @@ def repr_of_descr(self): return '%r' % (self,) - def get_arg_types(self): - """ Implement in call descr. - Must return a string of INT, REF and FLOAT ('i', 'r', 'f'). - """ - raise NotImplementedError - - def get_return_type(self): - """ Implement in call descr. - Must return INT, REF, FLOAT, or 'v' for void. - On 32-bit (hack) it can also be 'L' for longlongs. - Additionally it can be 'S' for singlefloats. - """ - raise NotImplementedError - - def get_extra_info(self): - """ Implement in call descr - """ - raise NotImplementedError - - def is_array_of_pointers(self): - """ Implement for array descr - """ - raise NotImplementedError - - def is_array_of_floats(self): - """ Implement for array descr - """ - raise NotImplementedError - - def is_array_of_structs(self): - """ Implement for array descr - """ - raise NotImplementedError - - def is_pointer_field(self): - """ Implement for field descr - """ - raise NotImplementedError - - def is_float_field(self): - """ Implement for field descr - """ - raise NotImplementedError - - def as_vtable_size_descr(self): - """ Implement for size descr representing objects with vtables. - Returns self. (it's an annotation hack) - """ - raise NotImplementedError - - def count_fields_if_immutable(self): - return -1 - def _clone_if_mutable(self): return self def clone_if_mutable(self): @@ -758,6 +705,9 @@ self.virtual_state = None self.exported_state = None + + def repr_of_descr(self): + return 'TargetToken(%d)' % compute_unique_id(self) class TreeLoop(object): inputargs = None @@ -765,7 +715,7 @@ call_pure_results = None logops = None quasi_immutable_deps = None - start_resumedescr = None + resume_at_jump_descr = None def _token(*args): raise Exception("TreeLoop.token is killed") @@ -1053,35 +1003,16 @@ return insns def check_simple_loop(self, expected=None, **check): - # Usefull in the simplest case when we have only one trace ending with - # a jump back to itself and possibly a few bridges ending with finnish. - # Only the operations within the loop formed by that single jump will - # be counted. - - # XXX hacked version, ignore and remove me when jit-targets is merged. - loops = self.get_all_loops() - loops = [loop for loop in loops if 'Preamble' not in repr(loop)] #XXX - assert len(loops) == 1 - loop, = loops - jumpop = loop.operations[-1] - assert jumpop.getopnum() == rop.JUMP - insns = {} - for op in loop.operations: - opname = op.getopname() - insns[opname] = insns.get(opname, 0) + 1 - return self._check_insns(insns, expected, check) - - def check_simple_loop(self, expected=None, **check): - # Usefull in the simplest case when we have only one trace ending with - # a jump back to itself and possibly a few bridges ending with finnish. - # Only the operations within the loop formed by that single jump will - # be counted. + """ Usefull in the simplest case when we have only one trace ending with + a jump back to itself and possibly a few bridges. + Only the operations within the loop formed by that single jump will + be counted. + """ loops = self.get_all_loops() assert len(loops) == 1 loop = loops[0] jumpop = loop.operations[-1] assert jumpop.getopnum() == rop.JUMP - assert self.check_resops(jump=1) labels = [op for op in loop.operations if op.getopnum() == rop.LABEL] targets = [op._descr_wref() for op in labels] assert None not in targets # TargetToken was freed, give up @@ -1134,7 +1065,7 @@ if option.view: self.view() - def view(self, errmsg=None, extraprocedures=[]): + def view(self, errmsg=None, extraprocedures=[], metainterp_sd=None): from pypy.jit.metainterp.graphpage import display_procedures procedures = self.get_all_loops()[:] for procedure in extraprocedures: @@ -1146,7 +1077,7 @@ if hasattr(procedure, '_looptoken_number') and ( procedure._looptoken_number in self.invalidated_token_numbers): highlight_procedures.setdefault(procedure, 2) - display_procedures(procedures, errmsg, highlight_procedures) + display_procedures(procedures, errmsg, highlight_procedures, metainterp_sd) # ---------------------------------------------------------------- diff --git a/pypy/jit/metainterp/jitdriver.py b/pypy/jit/metainterp/jitdriver.py --- a/pypy/jit/metainterp/jitdriver.py +++ b/pypy/jit/metainterp/jitdriver.py @@ -21,7 +21,6 @@ # self.portal_finishtoken... pypy.jit.metainterp.pyjitpl # self.index ... pypy.jit.codewriter.call # self.mainjitcode ... pypy.jit.codewriter.call - # self.on_compile ... pypy.jit.metainterp.warmstate # These attributes are read by the backend in CALL_ASSEMBLER: # self.assembler_helper_adr diff --git a/pypy/jit/metainterp/jitprof.py b/pypy/jit/metainterp/jitprof.py --- a/pypy/jit/metainterp/jitprof.py +++ b/pypy/jit/metainterp/jitprof.py @@ -18,8 +18,8 @@ OPT_FORCINGS ABORT_TOO_LONG ABORT_BRIDGE +ABORT_BAD_LOOP ABORT_ESCAPE -ABORT_BAD_LOOP ABORT_FORCE_QUASIIMMUT NVIRTUALS NVHOLES @@ -30,10 +30,13 @@ TOTAL_FREED_BRIDGES """ +counter_names = [] + def _setup(): names = counters.split() for i, name in enumerate(names): globals()[name] = i + counter_names.append(name) global ncounters ncounters = len(names) _setup() diff --git a/pypy/jit/metainterp/logger.py b/pypy/jit/metainterp/logger.py --- a/pypy/jit/metainterp/logger.py +++ b/pypy/jit/metainterp/logger.py @@ -5,7 +5,7 @@ from pypy.rpython.lltypesystem import lltype, llmemory, rffi from pypy.jit.metainterp.resoperation import rop from pypy.jit.metainterp.history import Const, ConstInt, Box, \ - BoxInt, ConstFloat, BoxFloat, AbstractFailDescr + BoxInt, ConstFloat, BoxFloat, AbstractFailDescr, TargetToken class Logger(object): @@ -13,14 +13,14 @@ self.metainterp_sd = metainterp_sd self.guard_number = guard_number - def log_loop(self, inputargs, operations, number=0, type=None, ops_offset=None): + def log_loop(self, inputargs, operations, number=0, type=None, ops_offset=None, name=''): if type is None: debug_start("jit-log-noopt-loop") logops = self._log_operations(inputargs, operations, ops_offset) debug_stop("jit-log-noopt-loop") else: debug_start("jit-log-opt-loop") - debug_print("# Loop", number, ":", type, + debug_print("# Loop", number, '(%s)' % name , ":", type, "with", len(operations), "ops") logops = self._log_operations(inputargs, operations, ops_offset) debug_stop("jit-log-opt-loop") @@ -135,6 +135,13 @@ fail_args = '' return s_offset + res + op.getopname() + '(' + args + ')' + fail_args + def _log_inputarg_setup_ops(self, op): + target_token = op.getdescr() + if isinstance(target_token, TargetToken): + if target_token.exported_state: + for op in target_token.exported_state.inputarg_setup_ops: + debug_print(' ' + self.repr_of_resop(op)) + def _log_operations(self, inputargs, operations, ops_offset): if not have_debug_prints(): return @@ -146,6 +153,8 @@ for i in range(len(operations)): op = operations[i] debug_print(self.repr_of_resop(operations[i], ops_offset)) + if op.getopnum() == rop.LABEL: + self._log_inputarg_setup_ops(op) if ops_offset and None in ops_offset: offset = ops_offset[None] debug_print("+%d: --end of the loop--" % offset) diff --git a/pypy/jit/metainterp/memmgr.py b/pypy/jit/metainterp/memmgr.py --- a/pypy/jit/metainterp/memmgr.py +++ b/pypy/jit/metainterp/memmgr.py @@ -1,5 +1,5 @@ import math -from pypy.rlib.rarithmetic import r_int64, r_uint +from pypy.rlib.rarithmetic import r_int64 from pypy.rlib.debug import debug_start, debug_print, debug_stop from pypy.rlib.objectmodel import we_are_translated @@ -21,7 +21,6 @@ # class MemoryManager(object): - NO_NEXT_CHECK = r_int64(2 ** 63 - 1) def __init__(self): self.check_frequency = -1 @@ -37,13 +36,12 @@ # According to my estimates it's about 5e9 years given 1000 loops # per second self.current_generation = r_int64(1) - self.next_check = self.NO_NEXT_CHECK + self.next_check = r_int64(-1) self.alive_loops = {} - self._cleanup_jitcell_dicts = lambda: None def set_max_age(self, max_age, check_frequency=0): if max_age <= 0: - self.next_check = self.NO_NEXT_CHECK + self.next_check = r_int64(-1) else: self.max_age = max_age if check_frequency <= 0: @@ -51,11 +49,10 @@ self.check_frequency = check_frequency self.next_check = self.current_generation + 1 - def next_generation(self, do_cleanups_now=True): + def next_generation(self): self.current_generation += 1 - if do_cleanups_now and self.current_generation >= self.next_check: + if self.current_generation == self.next_check: self._kill_old_loops_now() - self._cleanup_jitcell_dicts() self.next_check = self.current_generation + self.check_frequency def keep_loop_alive(self, looptoken): @@ -84,22 +81,3 @@ # a single one is not enough for all tests :-( rgc.collect(); rgc.collect(); rgc.collect() debug_stop("jit-mem-collect") - - def get_current_generation_uint(self): - """Return the current generation, possibly truncated to a uint. - To use only as an approximation for decaying counters.""" - return r_uint(self.current_generation) - - def record_jitcell_dict(self, callback): - """NOT_RPYTHON. The given jitcell_dict is a dict that needs - occasional clean-ups of old cells. A cell is old if it never - reached the threshold, and its counter decayed to a tiny value.""" - # note that the various jitcell_dicts have different RPython types, - # so we have to make a different function for each one. These - # functions are chained to each other: each calls the previous one. - def cleanup_dict(): - callback() - cleanup_previous() - # - cleanup_previous = self._cleanup_jitcell_dicts - self._cleanup_jitcell_dicts = cleanup_dict diff --git a/pypy/jit/metainterp/optimize.py b/pypy/jit/metainterp/optimize.py --- a/pypy/jit/metainterp/optimize.py +++ b/pypy/jit/metainterp/optimize.py @@ -5,58 +5,3 @@ """Raised when the optimize*.py detect that the loop that we are trying to build cannot possibly make sense as a long-running loop (e.g. it cannot run 2 complete iterations).""" - -class RetraceLoop(JitException): - """ Raised when inlining a short preamble resulted in an - InvalidLoop. This means the optimized loop is too specialized - to be useful here, so we trace it again and produced a second - copy specialized in some different way. - """ - -# ____________________________________________________________ - -def optimize_loop(metainterp_sd, old_loop_tokens, loop, enable_opts): - debug_start("jit-optimize") - try: - return _optimize_loop(metainterp_sd, old_loop_tokens, loop, - enable_opts) - finally: - debug_stop("jit-optimize") - -def _optimize_loop(metainterp_sd, old_loop_tokens, loop, enable_opts): - from pypy.jit.metainterp.optimizeopt import optimize_loop_1 - loop.logops = metainterp_sd.logger_noopt.log_loop(loop.inputargs, - loop.operations) - # XXX do we really still need a list? - if old_loop_tokens: - return old_loop_tokens[0] - optimize_loop_1(metainterp_sd, loop, enable_opts) - return None - -# ____________________________________________________________ - -def optimize_bridge(metainterp_sd, old_loop_tokens, bridge, enable_opts, - inline_short_preamble=True, retraced=False): - debug_start("jit-optimize") - try: - return _optimize_bridge(metainterp_sd, old_loop_tokens, bridge, - enable_opts, - inline_short_preamble, retraced) - finally: - debug_stop("jit-optimize") - -def _optimize_bridge(metainterp_sd, old_loop_tokens, bridge, enable_opts, - inline_short_preamble, retraced=False): - from pypy.jit.metainterp.optimizeopt import optimize_bridge_1 - bridge.logops = metainterp_sd.logger_noopt.log_loop(bridge.inputargs, - bridge.operations) - if old_loop_tokens: - old_loop_token = old_loop_tokens[0] - bridge.operations[-1].setdescr(old_loop_token) # patch jump target - optimize_bridge_1(metainterp_sd, bridge, enable_opts, - inline_short_preamble, retraced) - return old_loop_tokens[0] - #return bridge.operations[-1].getdescr() - return None - -# ____________________________________________________________ diff --git a/pypy/jit/metainterp/optimizeopt/__init__.py b/pypy/jit/metainterp/optimizeopt/__init__.py --- a/pypy/jit/metainterp/optimizeopt/__init__.py +++ b/pypy/jit/metainterp/optimizeopt/__init__.py @@ -51,34 +51,6 @@ return optimizations, unroll - -def optimize_loop_1(metainterp_sd, loop, enable_opts, - inline_short_preamble=True, retraced=False): - """Optimize loop.operations to remove internal overheadish operations. - """ - - optimizations, unroll = build_opt_chain(metainterp_sd, enable_opts, - inline_short_preamble, retraced) - if unroll: - optimize_unroll(metainterp_sd, loop, optimizations) - else: - optimizer = Optimizer(metainterp_sd, loop, optimizations) - optimizer.propagate_all_forward() - -def optimize_bridge_1(metainterp_sd, bridge, enable_opts, - inline_short_preamble=True, retraced=False): - """The same, but for a bridge. """ - enable_opts = enable_opts.copy() - try: - del enable_opts['unroll'] - except KeyError: - pass - optimize_loop_1(metainterp_sd, bridge, enable_opts, - inline_short_preamble, retraced) - -if __name__ == '__main__': - print ALL_OPTS_NAMES - def optimize_trace(metainterp_sd, loop, enable_opts, inline_short_preamble=True): """Optimize loop.operations to remove internal overheadish operations. """ @@ -96,3 +68,6 @@ finally: debug_stop("jit-optimize") +if __name__ == '__main__': + print ALL_OPTS_NAMES + diff --git a/pypy/jit/metainterp/optimizeopt/fficall.py b/pypy/jit/metainterp/optimizeopt/fficall.py --- a/pypy/jit/metainterp/optimizeopt/fficall.py +++ b/pypy/jit/metainterp/optimizeopt/fficall.py @@ -234,11 +234,11 @@ # longlongs are treated as floats, see # e.g. llsupport/descr.py:getDescrClass is_float = True - elif kind == 'u': + elif kind == 'u' or kind == 's': # they're all False pass else: - assert False, "unsupported ffitype or kind" + raise NotImplementedError("unsupported ffitype or kind: %s" % kind) # fieldsize = rffi.getintfield(ffitype, 'c_size') return self.optimizer.cpu.interiorfielddescrof_dynamic( diff --git a/pypy/jit/metainterp/optimizeopt/optimizer.py b/pypy/jit/metainterp/optimizeopt/optimizer.py --- a/pypy/jit/metainterp/optimizeopt/optimizer.py +++ b/pypy/jit/metainterp/optimizeopt/optimizer.py @@ -453,6 +453,7 @@ def clear_newoperations(self): self._newoperations = [] + self.seen_results = {} def make_equal_to(self, box, value, replace=False): assert isinstance(value, OptValue) diff --git a/pypy/jit/metainterp/optimizeopt/simplify.py b/pypy/jit/metainterp/optimizeopt/simplify.py --- a/pypy/jit/metainterp/optimizeopt/simplify.py +++ b/pypy/jit/metainterp/optimizeopt/simplify.py @@ -35,6 +35,9 @@ pass def optimize_LABEL(self, op): + descr = op.getdescr() + if isinstance(descr, JitCellToken): + return self.optimize_JUMP(op.copy_and_change(rop.JUMP)) self.last_label_descr = op.getdescr() self.emit_operation(op) diff --git a/pypy/jit/metainterp/optimizeopt/test/test_multilabel.py b/pypy/jit/metainterp/optimizeopt/test/test_multilabel.py --- a/pypy/jit/metainterp/optimizeopt/test/test_multilabel.py +++ b/pypy/jit/metainterp/optimizeopt/test/test_multilabel.py @@ -1,21 +1,25 @@ +from __future__ import with_statement from pypy.jit.metainterp.optimizeopt.test.test_util import ( - LLtypeMixin, BaseTest, Storage, _sortboxes, FakeDescrWithSnapshot) + LLtypeMixin, BaseTest, Storage, _sortboxes, FakeDescrWithSnapshot, + FakeMetaInterpStaticData) from pypy.jit.metainterp.history import TreeLoop, JitCellToken, TargetToken from pypy.jit.metainterp.resoperation import rop, opname, ResOperation from pypy.jit.metainterp.optimize import InvalidLoop from py.test import raises +from pypy.jit.metainterp.optimizeopt.optimizer import Optimization +from pypy.jit.metainterp.optimizeopt.util import make_dispatcher_method class BaseTestMultiLabel(BaseTest): enable_opts = "intbounds:rewrite:virtualize:string:earlyforce:pure:heap:unroll" - def optimize_loop(self, ops, expected): + def optimize_loop(self, ops, expected, expected_shorts=None): loop = self.parse(ops) if expected != "crash!": expected = self.parse(expected) part = TreeLoop('part') part.inputargs = loop.inputargs - part.start_resumedescr = FakeDescrWithSnapshot() + part.resume_at_jump_descr = FakeDescrWithSnapshot() token = loop.original_jitcell_token optimized = TreeLoop('optimized') @@ -32,15 +36,17 @@ if nxt < len(loop.operations): label = loop.operations[nxt] assert label.getopnum() == rop.LABEL - jumpop = ResOperation(rop.JUMP, label.getarglist(), - None, descr=token) - operations.append(jumpop) + if label.getdescr() is None: + label.setdescr(token) + operations.append(label) part.operations = operations + self._do_optimize_loop(part, None) if part.operations[-1].getopnum() == rop.LABEL: last_label = [part.operations.pop()] else: last_label = [] + optimized.operations.extend(part.operations) prv = nxt + 1 @@ -53,11 +59,36 @@ print 'Failed!' print + shorts = [op.getdescr().short_preamble + for op in optimized.operations + if op.getopnum() == rop.LABEL] + + if expected_shorts: + for short in shorts: + print + print "Short preamble:" + print '\n'.join([str(o) for o in short]) + + assert expected != "crash!", "should have raised an exception" self.assert_equal(optimized, expected) + if expected_shorts: + assert len(shorts) == len(expected_shorts) + for short, expected_short in zip(shorts, expected_shorts): + expected_short = self.parse(expected_short) + short_preamble = TreeLoop('short preamble') + assert short[0].getopnum() == rop.LABEL + short_preamble.inputargs = short[0].getarglist() + short_preamble.operations = short + self.assert_equal(short_preamble, expected_short, + text_right='expected short preamble') + + return optimized +class OptimizeoptTestMultiLabel(BaseTestMultiLabel): + def test_simple(self): ops = """ [i1] @@ -193,8 +224,244 @@ """ with raises(InvalidLoop): self.optimize_loop(ops, ops) + + def test_two_intermediate_labels_basic_1(self): + ops = """ + [p1, i1] + i2 = getfield_gc(p1, descr=valuedescr) + label(p1, i1) + i3 = getfield_gc(p1, descr=valuedescr) + i4 = int_add(i1, i3) + label(p1, i4) + i5 = int_add(i4, 1) + jump(p1, i5) + """ + expected = """ + [p1, i1] + i2 = getfield_gc(p1, descr=valuedescr) + label(p1, i1, i2) + i4 = int_add(i1, i2) + label(p1, i4) + i5 = int_add(i4, 1) + jump(p1, i5) + """ + short1 = """ + [p1, i1] + label(p1, i1) + i2 = getfield_gc(p1, descr=valuedescr) + jump(p1, i1, i2) + """ + short2 = """ + [p1, i1] + label(p1, i1) + jump(p1, i1) + """ + self.optimize_loop(ops, expected, expected_shorts=[short1, short2]) + + def test_two_intermediate_labels_basic_2(self): + ops = """ + [p1, i1] + i2 = int_add(i1, 1) + label(p1, i1) + i3 = getfield_gc(p1, descr=valuedescr) + i4 = int_add(i1, i3) + label(p1, i4) + i5 = getfield_gc(p1, descr=valuedescr) + i6 = int_add(i4, i5) + jump(p1, i6) + """ + expected = """ + [p1, i1] + i2 = int_add(i1, 1) + label(p1, i1) + i3 = getfield_gc(p1, descr=valuedescr) + i4 = int_add(i1, i3) + label(p1, i4, i3) + i6 = int_add(i4, i3) + jump(p1, i6, i3) + """ + short1 = """ + [p1, i1] + label(p1, i1) + jump(p1, i1) + """ + short2 = """ + [p1, i1] + label(p1, i1) + i2 = getfield_gc(p1, descr=valuedescr) + jump(p1, i1, i2) + """ + self.optimize_loop(ops, expected, expected_shorts=[short1, short2]) + + def test_two_intermediate_labels_both(self): + ops = """ + [p1, i1] + i2 = getfield_gc(p1, descr=valuedescr) + label(p1, i1) + i3 = getfield_gc(p1, descr=valuedescr) + i4 = int_add(i1, i3) + label(p1, i4) + i5 = getfield_gc(p1, descr=valuedescr) + i6 = int_mul(i4, i5) + jump(p1, i6) + """ + expected = """ + [p1, i1] + i2 = getfield_gc(p1, descr=valuedescr) + label(p1, i1, i2) + i4 = int_add(i1, i2) + label(p1, i4, i2) + i6 = int_mul(i4, i2) + jump(p1, i6, i2) + """ + short = """ + [p1, i1] + label(p1, i1) + i2 = getfield_gc(p1, descr=valuedescr) + jump(p1, i1, i2) + """ + self.optimize_loop(ops, expected, expected_shorts=[short, short]) + + def test_import_across_multiple_labels_basic(self): + # Not supported, juts make sure we get a functional trace + ops = """ + [p1, i1] + i2 = getfield_gc(p1, descr=valuedescr) + label(p1, i1) + i3 = int_add(i1, 1) + label(p1, i1) + i4 = getfield_gc(p1, descr=valuedescr) + i5 = int_add(i4, 1) + jump(p1, i5) + """ + self.optimize_loop(ops, ops) + + def test_import_across_multiple_labels_with_duplication(self): + # Not supported, juts make sure we get a functional trace + ops = """ + [p1, i1] + i2 = getfield_gc(p1, descr=valuedescr) + label(p1, i2) + i3 = int_add(i2, 1) + label(p1, i2) + i4 = getfield_gc(p1, descr=valuedescr) + i5 = int_add(i4, 1) + jump(p1, i5) + """ + exported = """ + [p1, i1] + i2 = getfield_gc(p1, descr=valuedescr) + i6 = same_as(i2) + label(p1, i2) + i3 = int_add(i2, 1) + label(p1, i2) + i4 = getfield_gc(p1, descr=valuedescr) + i5 = int_add(i4, 1) + jump(p1, i5) + """ + self.optimize_loop(ops, exported) + + def test_import_virtual_across_multiple_labels(self): + ops = """ + [p0, i1] + i1a = int_add(i1, 1) + pv = new_with_vtable(ConstClass(node_vtable)) + setfield_gc(pv, i1a, descr=valuedescr) + label(pv, i1) + i2 = int_mul(i1, 3) + label(pv, i2) + i3 = getfield_gc(pv, descr=valuedescr) + i4 = int_add(i3, i2) + jump(pv, i4) + """ + expected = """ + [p0, i1] + i1a = int_add(i1, 1) + i5 = same_as(i1a) + label(i1a, i1) + i2 = int_mul(i1, 3) + label(i1a, i2) + i4 = int_add(i1a, i2) + jump(i1a, i4) + """ + self.optimize_loop(ops, expected) + + def test_virtual_as_field_of_forced_box(self): + ops = """ + [p0] + pv1 = new_with_vtable(ConstClass(node_vtable)) + label(pv1, p0) + pv2 = new_with_vtable(ConstClass(node_vtable)) + setfield_gc(pv2, pv1, descr=valuedescr) + jump(pv1, pv2) + """ + with raises(InvalidLoop): + self.optimize_loop(ops, ops) + +class OptRenameStrlen(Optimization): + def propagate_forward(self, op): + dispatch_opt(self, op) + + def optimize_STRLEN(self, op): + newop = op.clone() + newop.result = op.result.clonebox() + self.emit_operation(newop) + self.make_equal_to(op.result, self.getvalue(newop.result)) + +dispatch_opt = make_dispatcher_method(OptRenameStrlen, 'optimize_', + default=OptRenameStrlen.emit_operation) + +class BaseTestOptimizerRenamingBoxes(BaseTestMultiLabel): + + def _do_optimize_loop(self, loop, call_pure_results): + from pypy.jit.metainterp.optimizeopt.unroll import optimize_unroll + from pypy.jit.metainterp.optimizeopt.util import args_dict + from pypy.jit.metainterp.optimizeopt.pure import OptPure + + self.loop = loop + loop.call_pure_results = args_dict() + metainterp_sd = FakeMetaInterpStaticData(self.cpu) + optimize_unroll(metainterp_sd, loop, [OptRenameStrlen(), OptPure()], True) + + def test_optimizer_renaming_boxes(self): + ops = """ + [p1] + i1 = strlen(p1) + label(p1) + i2 = strlen(p1) + i3 = int_add(i2, 7) + jump(p1) + """ + expected = """ + [p1] + i1 = strlen(p1) + label(p1, i1) + i11 = same_as(i1) + i2 = int_add(i11, 7) + jump(p1, i11) + """ + self.optimize_loop(ops, expected) + + def test_optimizer_renaming_boxes_not_imported(self): + ops = """ + [p1] + i1 = strlen(p1) + label(p1) + jump(p1) + """ + expected = """ + [p1] + i1 = strlen(p1) + label(p1, i1) + i11 = same_as(i1) + jump(p1, i11) + """ + self.optimize_loop(ops, expected) + - -class TestLLtype(BaseTestMultiLabel, LLtypeMixin): + +class TestLLtype(OptimizeoptTestMultiLabel, LLtypeMixin): pass +class TestOptimizerRenamingBoxesLLtype(BaseTestOptimizerRenamingBoxes, LLtypeMixin): + pass diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py --- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py +++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py @@ -117,7 +117,7 @@ def optimize_loop(self, ops, optops, call_pure_results=None): loop = self.parse(ops) - token = JitCellToken() + token = JitCellToken() loop.operations = [ResOperation(rop.LABEL, loop.inputargs, None, descr=TargetToken(token))] + \ loop.operations if loop.operations[-1].getopnum() == rop.JUMP: diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -4,7 +4,7 @@ LLtypeMixin, BaseTest, Storage, _sortboxes, convert_old_style_to_targets) import pypy.jit.metainterp.optimizeopt.optimizer as optimizeopt import pypy.jit.metainterp.optimizeopt.virtualize as virtualize -from pypy.jit.metainterp.optimizeopt import optimize_loop_1, ALL_OPTS_DICT, build_opt_chain +from pypy.jit.metainterp.optimizeopt import ALL_OPTS_DICT, build_opt_chain from pypy.jit.metainterp.optimize import InvalidLoop from pypy.jit.metainterp.history import AbstractDescr, ConstInt, BoxInt from pypy.jit.metainterp.history import TreeLoop, JitCellToken, TargetToken @@ -4211,7 +4211,6 @@ preamble = """ [p0] i0 = strlen(p0) - i3 = same_as(i0) # Should be killed by backend jump(p0) """ expected = """ @@ -5668,8 +5667,7 @@ p3 = newstr(i3) copystrcontent(p1, p3, 0, 0, i1) copystrcontent(p2, p3, 0, i1, i2) - i7 = same_as(i2) - jump(p2, p3, i7) + jump(p2, p3, i2) """ expected = """ [p1, p2, i1] @@ -5744,9 +5742,7 @@ copystrcontent(p1, p5, 0, 0, i1) copystrcontent(p2, p5, 0, i1, i2) copystrcontent(p3, p5, 0, i12, i3) - i129 = same_as(i2) - i130 = same_as(i3) - jump(p2, p3, p5, i129, i130) + jump(p2, p3, p5, i2, i3) """ expected = """ [p1, p2, p3, i1, i2] @@ -5959,8 +5955,7 @@ p4 = newstr(i5) copystrcontent(p1, p4, i1, 0, i3) copystrcontent(p2, p4, 0, i3, i4) - i9 = same_as(i4) - jump(p4, i1, i2, p2, i5, i3, i9) + jump(p4, i1, i2, p2, i5, i3, i4) """ expected = """ [p1, i1, i2, p2, i5, i3, i4] @@ -6082,9 +6077,7 @@ copystrcontent(p2, p4, 0, i1, i2) i0 = call(0, p3, p4, descr=strequaldescr) escape(i0) - i11 = same_as(i1) - i12 = same_as(i2) - jump(p1, p2, p3, i3, i11, i12) + jump(p1, p2, p3, i3, i1, i2) """ expected = """ [p1, p2, p3, i3, i1, i2] @@ -6304,7 +6297,6 @@ i1 = strlen(p1) i0 = int_eq(i1, 0) escape(i0) - i3 = same_as(i1) jump(p1, i0) """ self.optimize_strunicode_loop_extradescrs(ops, expected, preamble) @@ -6350,9 +6342,7 @@ copystrcontent(p2, p4, 0, i1, i2) i0 = call(0, s"hello world", p4, descr=streq_nonnull_descr) escape(i0) - i11 = same_as(i1) - i12 = same_as(i2) - jump(p1, p2, i3, i11, i12) + jump(p1, p2, i3, i1, i2) """ expected = """ [p1, p2, i3, i1, i2] @@ -6925,8 +6915,7 @@ [p9] i843 = strlen(p9) call(i843, descr=nonwritedescr) - i0 = same_as(i843) - jump(p9, i0) + jump(p9, i843) """ short = """ [p9] @@ -7770,7 +7759,7 @@ jump(i0, p0, i2) """ self.optimize_loop(ops, expected) - + class TestLLtype(OptimizeOptTest, LLtypeMixin): pass diff --git a/pypy/jit/metainterp/optimizeopt/test/test_util.py b/pypy/jit/metainterp/optimizeopt/test/test_util.py --- a/pypy/jit/metainterp/optimizeopt/test/test_util.py +++ b/pypy/jit/metainterp/optimizeopt/test/test_util.py @@ -430,18 +430,18 @@ preamble = TreeLoop('preamble') preamble.inputargs = inputargs - preamble.start_resumedescr = FakeDescrWithSnapshot() + preamble.resume_at_jump_descr = FakeDescrWithSnapshot() token = JitCellToken() preamble.operations = [ResOperation(rop.LABEL, inputargs, None, descr=TargetToken(token))] + \ operations + \ - [ResOperation(rop.JUMP, jump_args, None, descr=token)] + [ResOperation(rop.LABEL, jump_args, None, descr=token)] self._do_optimize_loop(preamble, call_pure_results) assert preamble.operations[-1].getopnum() == rop.LABEL inliner = Inliner(inputargs, jump_args) - loop.start_resumedescr = preamble.start_resumedescr + loop.resume_at_jump_descr = preamble.resume_at_jump_descr loop.operations = [preamble.operations[-1]] + \ [inliner.inline_op(op, clone=False) for op in cloned_operations] + \ [ResOperation(rop.JUMP, [inliner.inline_arg(a) for a in jump_args], diff --git a/pypy/jit/metainterp/optimizeopt/unroll.py b/pypy/jit/metainterp/optimizeopt/unroll.py --- a/pypy/jit/metainterp/optimizeopt/unroll.py +++ b/pypy/jit/metainterp/optimizeopt/unroll.py @@ -3,7 +3,7 @@ from pypy.jit.metainterp.compile import ResumeGuardDescr from pypy.jit.metainterp.history import TreeLoop, TargetToken, JitCellToken from pypy.jit.metainterp.jitexc import JitException -from pypy.jit.metainterp.optimize import InvalidLoop, RetraceLoop +from pypy.jit.metainterp.optimize import InvalidLoop from pypy.jit.metainterp.optimizeopt.optimizer import * from pypy.jit.metainterp.optimizeopt.generalize import KillHugeIntBounds from pypy.jit.metainterp.inliner import Inliner @@ -51,10 +51,10 @@ distinction anymore)""" inline_short_preamble = True - did_import = False def __init__(self, metainterp_sd, loop, optimizations): self.optimizer = UnrollableOptimizer(metainterp_sd, loop, optimizations) + self.boxes_created_this_iteration = None def fix_snapshot(self, jump_args, snapshot): if snapshot is None: @@ -71,7 +71,6 @@ loop = self.optimizer.loop self.optimizer.clear_newoperations() - start_label = loop.operations[0] if start_label.getopnum() == rop.LABEL: loop.operations = loop.operations[1:] @@ -82,7 +81,7 @@ start_label = None jumpop = loop.operations[-1] - if jumpop.getopnum() == rop.JUMP: + if jumpop.getopnum() == rop.JUMP or jumpop.getopnum() == rop.LABEL: loop.operations = loop.operations[:-1] else: jumpop = None @@ -91,48 +90,87 @@ self.optimizer.propagate_all_forward(clear=False) if not jumpop: - return - if self.jump_to_already_compiled_trace(jumpop): - # Found a compiled trace to jump to - if self.did_import: - - self.close_bridge(start_label) - self.finilize_short_preamble(start_label) return cell_token = jumpop.getdescr() assert isinstance(cell_token, JitCellToken) stop_label = ResOperation(rop.LABEL, jumpop.getarglist(), None, TargetToken(cell_token)) - if not self.did_import: # Enforce the previous behaviour of always peeling exactly one iteration (for now) - self.optimizer.flush() - KillHugeIntBounds(self.optimizer).apply() + From pullrequests-noreply at bitbucket.org Sat Jan 14 22:14:07 2012 From: pullrequests-noreply at bitbucket.org (Bitbucket) Date: Sat, 14 Jan 2012 21:14:07 -0000 Subject: [pypy-commit] [OPEN] Pull request #19 for pypy/pypy: Improvements to the JVM backend Message-ID: A new pull request has been opened by Micha? Bendowski. benol/pypy has changes to be pulled into pypy/pypy. https://bitbucket.org/pypy/pypy/pull-request/19/improvements-to-the-jvm-backend Title: Improvements to the JVM backend The translation process now finishes successfully, although produces invalid bytecode. Changes to be pulled: 2e1b33862b2c by benol: "Add files generated by PyCharm to .hgignore" 372628723c37 by benol: "Declare oo_primitives that should implement some rffi operations. For now the a?" 30e6d59d4333 by benol: "Add a missing cast from Unsigned to UnsignedLongLong." 0ab09a670d4a by benol: "Handle the 'jit_is_virtual' opcode by always returning False" 9031e9c4ad78 by benol: "Fix compute_unique_id to support built-ins. Otherwise the translation fails bec?" ee14653b5fa0 by benol: "Fix userspace builders in ootype Implement the ll_getlength() method of StringB?" -- This is an issue notification from bitbucket.org. You are receiving this either because you are the participating in a pull request, or you are following it. From noreply at buildbot.pypy.org Sat Jan 14 22:35:56 2012 From: noreply at buildbot.pypy.org (mattip) Date: Sat, 14 Jan 2012 22:35:56 +0100 (CET) Subject: [pypy-commit] pypy matrixmath-dot: merge from default Message-ID: <20120114213556.0B89C71025D@wyvern.cs.uni-duesseldorf.de> Author: mattip Branch: matrixmath-dot Changeset: r51331:ba2aa2febde8 Date: 2012-01-14 22:17 +0200 http://bitbucket.org/pypy/pypy/changeset/ba2aa2febde8/ Log: merge from default diff too long, truncating to 10000 out of 12004 lines diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -27,7 +27,7 @@ DEALINGS IN THE SOFTWARE. -PyPy Copyright holders 2003-2011 +PyPy Copyright holders 2003-2012 ----------------------------------- Except when otherwise stated (look for LICENSE files or information at @@ -37,43 +37,47 @@ Armin Rigo Maciej Fijalkowski Carl Friedrich Bolz + Amaury Forgeot d'Arc Antonio Cuni - Amaury Forgeot d'Arc Samuele Pedroni Michael Hudson Holger Krekel - Benjamin Peterson + Alex Gaynor Christian Tismer Hakan Ardo - Alex Gaynor + Benjamin Peterson + David Schneider Eric van Riet Paap Anders Chrigstrom - David Schneider Richard Emslie Dan Villiom Podlaski Christiansen Alexander Schremmer + Lukas Diekmann Aurelien Campeas Anders Lehmann Camillo Bruni Niklaus Haldimann + Sven Hager Leonardo Santagada Toon Verwaest Seo Sanghyeon + Justin Peel Lawrence Oluyede Bartosz Skowron Jakub Gustak Guido Wesdorp Daniel Roberts + Laura Creighton Adrien Di Mascio - Laura Creighton Ludovic Aubry Niko Matsakis + Wim Lavrijsen + Matti Picus Jason Creighton Jacob Hallen Alex Martelli Anders Hammarquist Jan de Mooij - Wim Lavrijsen Stephan Diehl Michael Foord Stefan Schwarzer @@ -84,34 +88,36 @@ Alexandre Fayolle Marius Gedminas Simon Burton - Justin Peel + David Edelsohn Jean-Paul Calderone John Witulski - Lukas Diekmann + Timo Paulssen holger krekel - Wim Lavrijsen Dario Bertini + Mark Pearse Andreas Stührk Jean-Philippe St. Pierre Guido van Rossum Pavel Vinogradov Valentino Volonghi Paul deGrandis + Ilya Osadchiy + Ronny Pfannschmidt Adrian Kuhn tav Georg Brandl + Philip Jenvey Gerald Klix Wanja Saatkamp - Ronny Pfannschmidt Boris Feigin Oscar Nierstrasz David Malcolm Eugene Oden Henry Mason - Sven Hager + Jeff Terrace Lukas Renggli - Ilya Osadchiy Guenter Jantzen + Ned Batchelder Bert Freudenberg Amit Regmi Ben Young @@ -142,7 +148,6 @@ Anders Qvist Beatrice During Alexander Sedov - Timo Paulssen Corbin Simpson Vincent Legoll Romain Guillebert @@ -165,9 +170,10 @@ Lucio Torre Lene Wagner Miguel de Val Borro + Artur Lisiecki + Bruno Gola Ignas Mikalajunas - Artur Lisiecki - Philip Jenvey + Stefano Rivera Joshua Gilbert Godefroid Chappelle Yusei Tahara @@ -179,17 +185,17 @@ Kristjan Valur Jonsson Bobby Impollonia Michael Hudson-Doyle + Laurence Tratt + Yasir Suhail Andrew Thompson Anders Sigfridsson Floris Bruynooghe Jacek Generowicz Dan Colish Zooko Wilcox-O Hearn - Dan Villiom Podlaski Christiansen - Anders Hammarquist + Dan Loewenherz Chris Lambacher Dinu Gherman - Dan Colish Brett Cannon Daniel Neuhäuser Michael Chermside diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py --- a/lib_pypy/_ctypes/structure.py +++ b/lib_pypy/_ctypes/structure.py @@ -73,8 +73,12 @@ class Field(object): def __init__(self, name, offset, size, ctype, num, is_bitfield): - for k in ('name', 'offset', 'size', 'ctype', 'num', 'is_bitfield'): - self.__dict__[k] = locals()[k] + self.__dict__['name'] = name + self.__dict__['offset'] = offset + self.__dict__['size'] = size + self.__dict__['ctype'] = ctype + self.__dict__['num'] = num + self.__dict__['is_bitfield'] = is_bitfield def __setattr__(self, name, value): raise AttributeError(name) diff --git a/lib_pypy/numpypy/__init__.py b/lib_pypy/numpypy/__init__.py new file mode 100644 --- /dev/null +++ b/lib_pypy/numpypy/__init__.py @@ -0,0 +1,2 @@ +from _numpypy import * +from .fromnumeric import * diff --git a/lib_pypy/numpypy/fromnumeric.py b/lib_pypy/numpypy/fromnumeric.py new file mode 100644 --- /dev/null +++ b/lib_pypy/numpypy/fromnumeric.py @@ -0,0 +1,2400 @@ +###################################################################### +# This is a copy of numpy/core/fromnumeric.py modified for numpypy +###################################################################### +# Each name in __all__ was a function in 'numeric' that is now +# a method in 'numpy'. +# When the corresponding method is added to numpypy BaseArray +# each function should be added as a module function +# at the applevel +# This can be as simple as doing the following +# +# def func(a, ...): +# if not hasattr(a, 'func') +# a = numpypy.array(a) +# return a.func(...) +# +###################################################################### + +import numpypy + +# Module containing non-deprecated functions borrowed from Numeric. +__docformat__ = "restructuredtext en" + +# functions that are now methods +__all__ = ['take', 'reshape', 'choose', 'repeat', 'put', + 'swapaxes', 'transpose', 'sort', 'argsort', 'argmax', 'argmin', + 'searchsorted', 'alen', + 'resize', 'diagonal', 'trace', 'ravel', 'nonzero', 'shape', + 'compress', 'clip', 'sum', 'product', 'prod', 'sometrue', 'alltrue', + 'any', 'all', 'cumsum', 'cumproduct', 'cumprod', 'ptp', 'ndim', + 'rank', 'size', 'around', 'round_', 'mean', 'std', 'var', 'squeeze', + 'amax', 'amin', + ] + +def take(a, indices, axis=None, out=None, mode='raise'): + """ + Take elements from an array along an axis. + + This function does the same thing as "fancy" indexing (indexing arrays + using arrays); however, it can be easier to use if you need elements + along a given axis. + + Parameters + ---------- + a : array_like + The source array. + indices : array_like + The indices of the values to extract. + axis : int, optional + The axis over which to select values. By default, the flattened + input array is used. + out : ndarray, optional + If provided, the result will be placed in this array. It should + be of the appropriate shape and dtype. + mode : {'raise', 'wrap', 'clip'}, optional + Specifies how out-of-bounds indices will behave. + + * 'raise' -- raise an error (default) + * 'wrap' -- wrap around + * 'clip' -- clip to the range + + 'clip' mode means that all indices that are too large are replaced + by the index that addresses the last element along that axis. Note + that this disables indexing with negative numbers. + + Returns + ------- + subarray : ndarray + The returned array has the same type as `a`. + + See Also + -------- + ndarray.take : equivalent method + + Examples + -------- + >>> a = [4, 3, 5, 7, 6, 8] + >>> indices = [0, 1, 4] + >>> np.take(a, indices) + array([4, 3, 6]) + + In this example if `a` is an ndarray, "fancy" indexing can be used. + + >>> a = np.array(a) + >>> a[indices] + array([4, 3, 6]) + + """ + raise NotImplemented('Waiting on interp level method') + + +# not deprecated --- copy if necessary, view otherwise +def reshape(a, newshape, order='C'): + """ + Gives a new shape to an array without changing its data. + + Parameters + ---------- + a : array_like + Array to be reshaped. + newshape : int or tuple of ints + The new shape should be compatible with the original shape. If + an integer, then the result will be a 1-D array of that length. + One shape dimension can be -1. In this case, the value is inferred + from the length of the array and remaining dimensions. + order : {'C', 'F', 'A'}, optional + Determines whether the array data should be viewed as in C + (row-major) order, FORTRAN (column-major) order, or the C/FORTRAN + order should be preserved. + + Returns + ------- + reshaped_array : ndarray + This will be a new view object if possible; otherwise, it will + be a copy. + + + See Also + -------- + ndarray.reshape : Equivalent method. + + Notes + ----- + + It is not always possible to change the shape of an array without + copying the data. If you want an error to be raise if the data is copied, + you should assign the new shape to the shape attribute of the array:: + + >>> a = np.zeros((10, 2)) + # A transpose make the array non-contiguous + >>> b = a.T + # Taking a view makes it possible to modify the shape without modiying the + # initial object. + >>> c = b.view() + >>> c.shape = (20) + AttributeError: incompatible shape for a non-contiguous array + + + Examples + -------- + >>> a = np.array([[1,2,3], [4,5,6]]) + >>> np.reshape(a, 6) + array([1, 2, 3, 4, 5, 6]) + >>> np.reshape(a, 6, order='F') + array([1, 4, 2, 5, 3, 6]) + + >>> np.reshape(a, (3,-1)) # the unspecified value is inferred to be 2 + array([[1, 2], + [3, 4], + [5, 6]]) + + """ + if not hasattr(a, 'reshape'): + a = numpypy.array(a) + return a.reshape(newshape) + + +def choose(a, choices, out=None, mode='raise'): + """ + Construct an array from an index array and a set of arrays to choose from. + + First of all, if confused or uncertain, definitely look at the Examples - + in its full generality, this function is less simple than it might + seem from the following code description (below ndi = + `numpy.lib.index_tricks`): + + ``np.choose(a,c) == np.array([c[a[I]][I] for I in ndi.ndindex(a.shape)])``. + + But this omits some subtleties. Here is a fully general summary: + + Given an "index" array (`a`) of integers and a sequence of `n` arrays + (`choices`), `a` and each choice array are first broadcast, as necessary, + to arrays of a common shape; calling these *Ba* and *Bchoices[i], i = + 0,...,n-1* we have that, necessarily, ``Ba.shape == Bchoices[i].shape`` + for each `i`. Then, a new array with shape ``Ba.shape`` is created as + follows: + + * if ``mode=raise`` (the default), then, first of all, each element of + `a` (and thus `Ba`) must be in the range `[0, n-1]`; now, suppose that + `i` (in that range) is the value at the `(j0, j1, ..., jm)` position + in `Ba` - then the value at the same position in the new array is the + value in `Bchoices[i]` at that same position; + + * if ``mode=wrap``, values in `a` (and thus `Ba`) may be any (signed) + integer; modular arithmetic is used to map integers outside the range + `[0, n-1]` back into that range; and then the new array is constructed + as above; + + * if ``mode=clip``, values in `a` (and thus `Ba`) may be any (signed) + integer; negative integers are mapped to 0; values greater than `n-1` + are mapped to `n-1`; and then the new array is constructed as above. + + Parameters + ---------- + a : int array + This array must contain integers in `[0, n-1]`, where `n` is the number + of choices, unless ``mode=wrap`` or ``mode=clip``, in which cases any + integers are permissible. + choices : sequence of arrays + Choice arrays. `a` and all of the choices must be broadcastable to the + same shape. If `choices` is itself an array (not recommended), then + its outermost dimension (i.e., the one corresponding to + ``choices.shape[0]``) is taken as defining the "sequence". + out : array, optional + If provided, the result will be inserted into this array. It should + be of the appropriate shape and dtype. + mode : {'raise' (default), 'wrap', 'clip'}, optional + Specifies how indices outside `[0, n-1]` will be treated: + + * 'raise' : an exception is raised + * 'wrap' : value becomes value mod `n` + * 'clip' : values < 0 are mapped to 0, values > n-1 are mapped to n-1 + + Returns + ------- + merged_array : array + The merged result. + + Raises + ------ + ValueError: shape mismatch + If `a` and each choice array are not all broadcastable to the same + shape. + + See Also + -------- + ndarray.choose : equivalent method + + Notes + ----- + To reduce the chance of misinterpretation, even though the following + "abuse" is nominally supported, `choices` should neither be, nor be + thought of as, a single array, i.e., the outermost sequence-like container + should be either a list or a tuple. + + Examples + -------- + + >>> choices = [[0, 1, 2, 3], [10, 11, 12, 13], + ... [20, 21, 22, 23], [30, 31, 32, 33]] + >>> np.choose([2, 3, 1, 0], choices + ... # the first element of the result will be the first element of the + ... # third (2+1) "array" in choices, namely, 20; the second element + ... # will be the second element of the fourth (3+1) choice array, i.e., + ... # 31, etc. + ... ) + array([20, 31, 12, 3]) + >>> np.choose([2, 4, 1, 0], choices, mode='clip') # 4 goes to 3 (4-1) + array([20, 31, 12, 3]) + >>> # because there are 4 choice arrays + >>> np.choose([2, 4, 1, 0], choices, mode='wrap') # 4 goes to (4 mod 4) + array([20, 1, 12, 3]) + >>> # i.e., 0 + + A couple examples illustrating how choose broadcasts: + + >>> a = [[1, 0, 1], [0, 1, 0], [1, 0, 1]] + >>> choices = [-10, 10] + >>> np.choose(a, choices) + array([[ 10, -10, 10], + [-10, 10, -10], + [ 10, -10, 10]]) + + >>> # With thanks to Anne Archibald + >>> a = np.array([0, 1]).reshape((2,1,1)) + >>> c1 = np.array([1, 2, 3]).reshape((1,3,1)) + >>> c2 = np.array([-1, -2, -3, -4, -5]).reshape((1,1,5)) + >>> np.choose(a, (c1, c2)) # result is 2x3x5, res[0,:,:]=c1, res[1,:,:]=c2 + array([[[ 1, 1, 1, 1, 1], + [ 2, 2, 2, 2, 2], + [ 3, 3, 3, 3, 3]], + [[-1, -2, -3, -4, -5], + [-1, -2, -3, -4, -5], + [-1, -2, -3, -4, -5]]]) + + """ + raise NotImplemented('Waiting on interp level method') + + +def repeat(a, repeats, axis=None): + """ + Repeat elements of an array. + + Parameters + ---------- + a : array_like + Input array. + repeats : {int, array of ints} + The number of repetitions for each element. `repeats` is broadcasted + to fit the shape of the given axis. + axis : int, optional + The axis along which to repeat values. By default, use the + flattened input array, and return a flat output array. + + Returns + ------- + repeated_array : ndarray + Output array which has the same shape as `a`, except along + the given axis. + + See Also + -------- + tile : Tile an array. + + Examples + -------- + >>> x = np.array([[1,2],[3,4]]) + >>> np.repeat(x, 2) + array([1, 1, 2, 2, 3, 3, 4, 4]) + >>> np.repeat(x, 3, axis=1) + array([[1, 1, 1, 2, 2, 2], + [3, 3, 3, 4, 4, 4]]) + >>> np.repeat(x, [1, 2], axis=0) + array([[1, 2], + [3, 4], + [3, 4]]) + + """ + raise NotImplemented('Waiting on interp level method') + + +def put(a, ind, v, mode='raise'): + """ + Replaces specified elements of an array with given values. + + The indexing works on the flattened target array. `put` is roughly + equivalent to: + + :: + + a.flat[ind] = v + + Parameters + ---------- + a : ndarray + Target array. + ind : array_like + Target indices, interpreted as integers. + v : array_like + Values to place in `a` at target indices. If `v` is shorter than + `ind` it will be repeated as necessary. + mode : {'raise', 'wrap', 'clip'}, optional + Specifies how out-of-bounds indices will behave. + + * 'raise' -- raise an error (default) + * 'wrap' -- wrap around + * 'clip' -- clip to the range + + 'clip' mode means that all indices that are too large are replaced + by the index that addresses the last element along that axis. Note + that this disables indexing with negative numbers. + + See Also + -------- + putmask, place + + Examples + -------- + >>> a = np.arange(5) + >>> np.put(a, [0, 2], [-44, -55]) + >>> a + array([-44, 1, -55, 3, 4]) + + >>> a = np.arange(5) + >>> np.put(a, 22, -5, mode='clip') + >>> a + array([ 0, 1, 2, 3, -5]) + + """ + raise NotImplemented('Waiting on interp level method') + + +def swapaxes(a, axis1, axis2): + """ + Interchange two axes of an array. + + Parameters + ---------- + a : array_like + Input array. + axis1 : int + First axis. + axis2 : int + Second axis. + + Returns + ------- + a_swapped : ndarray + If `a` is an ndarray, then a view of `a` is returned; otherwise + a new array is created. + + Examples + -------- + >>> x = np.array([[1,2,3]]) + >>> np.swapaxes(x,0,1) + array([[1], + [2], + [3]]) + + >>> x = np.array([[[0,1],[2,3]],[[4,5],[6,7]]]) + >>> x + array([[[0, 1], + [2, 3]], + [[4, 5], + [6, 7]]]) + + >>> np.swapaxes(x,0,2) + array([[[0, 4], + [2, 6]], + [[1, 5], + [3, 7]]]) + + """ + raise NotImplemented('Waiting on interp level method') + + +def transpose(a, axes=None): + """ + Permute the dimensions of an array. + + Parameters + ---------- + a : array_like + Input array. + axes : list of ints, optional + By default, reverse the dimensions, otherwise permute the axes + according to the values given. + + Returns + ------- + p : ndarray + `a` with its axes permuted. A view is returned whenever + possible. + + See Also + -------- + rollaxis + + Examples + -------- + >>> x = np.arange(4).reshape((2,2)) + >>> x + array([[0, 1], + [2, 3]]) + + >>> np.transpose(x) + array([[0, 2], + [1, 3]]) + + >>> x = np.ones((1, 2, 3)) + >>> np.transpose(x, (1, 0, 2)).shape + (2, 1, 3) + + """ + raise NotImplemented('Waiting on interp level method') + + +def sort(a, axis=-1, kind='quicksort', order=None): + """ + Return a sorted copy of an array. + + Parameters + ---------- + a : array_like + Array to be sorted. + axis : int or None, optional + Axis along which to sort. If None, the array is flattened before + sorting. The default is -1, which sorts along the last axis. + kind : {'quicksort', 'mergesort', 'heapsort'}, optional + Sorting algorithm. Default is 'quicksort'. + order : list, optional + When `a` is a structured array, this argument specifies which fields + to compare first, second, and so on. This list does not need to + include all of the fields. + + Returns + ------- + sorted_array : ndarray + Array of the same type and shape as `a`. + + See Also + -------- + ndarray.sort : Method to sort an array in-place. + argsort : Indirect sort. + lexsort : Indirect stable sort on multiple keys. + searchsorted : Find elements in a sorted array. + + Notes + ----- + The various sorting algorithms are characterized by their average speed, + worst case performance, work space size, and whether they are stable. A + stable sort keeps items with the same key in the same relative + order. The three available algorithms have the following + properties: + + =========== ======= ============= ============ ======= + kind speed worst case work space stable + =========== ======= ============= ============ ======= + 'quicksort' 1 O(n^2) 0 no + 'mergesort' 2 O(n*log(n)) ~n/2 yes + 'heapsort' 3 O(n*log(n)) 0 no + =========== ======= ============= ============ ======= + + All the sort algorithms make temporary copies of the data when + sorting along any but the last axis. Consequently, sorting along + the last axis is faster and uses less space than sorting along + any other axis. + + The sort order for complex numbers is lexicographic. If both the real + and imaginary parts are non-nan then the order is determined by the + real parts except when they are equal, in which case the order is + determined by the imaginary parts. + + Previous to numpy 1.4.0 sorting real and complex arrays containing nan + values led to undefined behaviour. In numpy versions >= 1.4.0 nan + values are sorted to the end. The extended sort order is: + + * Real: [R, nan] + * Complex: [R + Rj, R + nanj, nan + Rj, nan + nanj] + + where R is a non-nan real value. Complex values with the same nan + placements are sorted according to the non-nan part if it exists. + Non-nan values are sorted as before. + + Examples + -------- + >>> a = np.array([[1,4],[3,1]]) + >>> np.sort(a) # sort along the last axis + array([[1, 4], + [1, 3]]) + >>> np.sort(a, axis=None) # sort the flattened array + array([1, 1, 3, 4]) + >>> np.sort(a, axis=0) # sort along the first axis + array([[1, 1], + [3, 4]]) + + Use the `order` keyword to specify a field to use when sorting a + structured array: + + >>> dtype = [('name', 'S10'), ('height', float), ('age', int)] + >>> values = [('Arthur', 1.8, 41), ('Lancelot', 1.9, 38), + ... ('Galahad', 1.7, 38)] + >>> a = np.array(values, dtype=dtype) # create a structured array + >>> np.sort(a, order='height') # doctest: +SKIP + array([('Galahad', 1.7, 38), ('Arthur', 1.8, 41), + ('Lancelot', 1.8999999999999999, 38)], + dtype=[('name', '|S10'), ('height', '>> np.sort(a, order=['age', 'height']) # doctest: +SKIP + array([('Galahad', 1.7, 38), ('Lancelot', 1.8999999999999999, 38), + ('Arthur', 1.8, 41)], + dtype=[('name', '|S10'), ('height', '>> x = np.array([3, 1, 2]) + >>> np.argsort(x) + array([1, 2, 0]) + + Two-dimensional array: + + >>> x = np.array([[0, 3], [2, 2]]) + >>> x + array([[0, 3], + [2, 2]]) + + >>> np.argsort(x, axis=0) + array([[0, 1], + [1, 0]]) + + >>> np.argsort(x, axis=1) + array([[0, 1], + [0, 1]]) + + Sorting with keys: + + >>> x = np.array([(1, 0), (0, 1)], dtype=[('x', '>> x + array([(1, 0), (0, 1)], + dtype=[('x', '>> np.argsort(x, order=('x','y')) + array([1, 0]) + + >>> np.argsort(x, order=('y','x')) + array([0, 1]) + + """ + raise NotImplemented('Waiting on interp level method') + + +def argmax(a, axis=None): + """ + Indices of the maximum values along an axis. + + Parameters + ---------- + a : array_like + Input array. + axis : int, optional + By default, the index is into the flattened array, otherwise + along the specified axis. + + Returns + ------- + index_array : ndarray of ints + Array of indices into the array. It has the same shape as `a.shape` + with the dimension along `axis` removed. + + See Also + -------- + ndarray.argmax, argmin + amax : The maximum value along a given axis. + unravel_index : Convert a flat index into an index tuple. + + Notes + ----- + In case of multiple occurrences of the maximum values, the indices + corresponding to the first occurrence are returned. + + Examples + -------- + >>> a = np.arange(6).reshape(2,3) + >>> a + array([[0, 1, 2], + [3, 4, 5]]) + >>> np.argmax(a) + 5 + >>> np.argmax(a, axis=0) + array([1, 1, 1]) + >>> np.argmax(a, axis=1) + array([2, 2]) + + >>> b = np.arange(6) + >>> b[1] = 5 + >>> b + array([0, 5, 2, 3, 4, 5]) + >>> np.argmax(b) # Only the first occurrence is returned. + 1 + + """ + if not hasattr(a, 'argmax'): + a = numpypy.array(a) + return a.argmax() + + +def argmin(a, axis=None): + """ + Return the indices of the minimum values along an axis. + + See Also + -------- + argmax : Similar function. Please refer to `numpy.argmax` for detailed + documentation. + + """ + if not hasattr(a, 'argmin'): + a = numpypy.array(a) + return a.argmin() + + +def searchsorted(a, v, side='left'): + """ + Find indices where elements should be inserted to maintain order. + + Find the indices into a sorted array `a` such that, if the corresponding + elements in `v` were inserted before the indices, the order of `a` would + be preserved. + + Parameters + ---------- + a : 1-D array_like + Input array, sorted in ascending order. + v : array_like + Values to insert into `a`. + side : {'left', 'right'}, optional + If 'left', the index of the first suitable location found is given. If + 'right', return the last such index. If there is no suitable + index, return either 0 or N (where N is the length of `a`). + + Returns + ------- + indices : array of ints + Array of insertion points with the same shape as `v`. + + See Also + -------- + sort : Return a sorted copy of an array. + histogram : Produce histogram from 1-D data. + + Notes + ----- + Binary search is used to find the required insertion points. + + As of Numpy 1.4.0 `searchsorted` works with real/complex arrays containing + `nan` values. The enhanced sort order is documented in `sort`. + + Examples + -------- + >>> np.searchsorted([1,2,3,4,5], 3) + 2 + >>> np.searchsorted([1,2,3,4,5], 3, side='right') + 3 + >>> np.searchsorted([1,2,3,4,5], [-10, 10, 2, 3]) + array([0, 5, 1, 2]) + + """ + raise NotImplemented('Waiting on interp level method') + + +def resize(a, new_shape): + """ + Return a new array with the specified shape. + + If the new array is larger than the original array, then the new + array is filled with repeated copies of `a`. Note that this behavior + is different from a.resize(new_shape) which fills with zeros instead + of repeated copies of `a`. + + Parameters + ---------- + a : array_like + Array to be resized. + + new_shape : int or tuple of int + Shape of resized array. + + Returns + ------- + reshaped_array : ndarray + The new array is formed from the data in the old array, repeated + if necessary to fill out the required number of elements. The + data are repeated in the order that they are stored in memory. + + See Also + -------- + ndarray.resize : resize an array in-place. + + Examples + -------- + >>> a=np.array([[0,1],[2,3]]) + >>> np.resize(a,(1,4)) + array([[0, 1, 2, 3]]) + >>> np.resize(a,(2,4)) + array([[0, 1, 2, 3], + [0, 1, 2, 3]]) + + """ + raise NotImplemented('Waiting on interp level method') + + +def squeeze(a): + """ + Remove single-dimensional entries from the shape of an array. + + Parameters + ---------- + a : array_like + Input data. + + Returns + ------- + squeezed : ndarray + The input array, but with with all dimensions of length 1 + removed. Whenever possible, a view on `a` is returned. + + Examples + -------- + >>> x = np.array([[[0], [1], [2]]]) + >>> x.shape + (1, 3, 1) + >>> np.squeeze(x).shape + (3,) + + """ + raise NotImplemented('Waiting on interp level method') + + +def diagonal(a, offset=0, axis1=0, axis2=1): + """ + Return specified diagonals. + + If `a` is 2-D, returns the diagonal of `a` with the given offset, + i.e., the collection of elements of the form ``a[i, i+offset]``. If + `a` has more than two dimensions, then the axes specified by `axis1` + and `axis2` are used to determine the 2-D sub-array whose diagonal is + returned. The shape of the resulting array can be determined by + removing `axis1` and `axis2` and appending an index to the right equal + to the size of the resulting diagonals. + + Parameters + ---------- + a : array_like + Array from which the diagonals are taken. + offset : int, optional + Offset of the diagonal from the main diagonal. Can be positive or + negative. Defaults to main diagonal (0). + axis1 : int, optional + Axis to be used as the first axis of the 2-D sub-arrays from which + the diagonals should be taken. Defaults to first axis (0). + axis2 : int, optional + Axis to be used as the second axis of the 2-D sub-arrays from + which the diagonals should be taken. Defaults to second axis (1). + + Returns + ------- + array_of_diagonals : ndarray + If `a` is 2-D, a 1-D array containing the diagonal is returned. + If the dimension of `a` is larger, then an array of diagonals is + returned, "packed" from left-most dimension to right-most (e.g., + if `a` is 3-D, then the diagonals are "packed" along rows). + + Raises + ------ + ValueError + If the dimension of `a` is less than 2. + + See Also + -------- + diag : MATLAB work-a-like for 1-D and 2-D arrays. + diagflat : Create diagonal arrays. + trace : Sum along diagonals. + + Examples + -------- + >>> a = np.arange(4).reshape(2,2) + >>> a + array([[0, 1], + [2, 3]]) + >>> a.diagonal() + array([0, 3]) + >>> a.diagonal(1) + array([1]) + + A 3-D example: + + >>> a = np.arange(8).reshape(2,2,2); a + array([[[0, 1], + [2, 3]], + [[4, 5], + [6, 7]]]) + >>> a.diagonal(0, # Main diagonals of two arrays created by skipping + ... 0, # across the outer(left)-most axis last and + ... 1) # the "middle" (row) axis first. + array([[0, 6], + [1, 7]]) + + The sub-arrays whose main diagonals we just obtained; note that each + corresponds to fixing the right-most (column) axis, and that the + diagonals are "packed" in rows. + + >>> a[:,:,0] # main diagonal is [0 6] + array([[0, 2], + [4, 6]]) + >>> a[:,:,1] # main diagonal is [1 7] + array([[1, 3], + [5, 7]]) + + """ + raise NotImplemented('Waiting on interp level method') + + +def trace(a, offset=0, axis1=0, axis2=1, dtype=None, out=None): + """ + Return the sum along diagonals of the array. + + If `a` is 2-D, the sum along its diagonal with the given offset + is returned, i.e., the sum of elements ``a[i,i+offset]`` for all i. + + If `a` has more than two dimensions, then the axes specified by axis1 and + axis2 are used to determine the 2-D sub-arrays whose traces are returned. + The shape of the resulting array is the same as that of `a` with `axis1` + and `axis2` removed. + + Parameters + ---------- + a : array_like + Input array, from which the diagonals are taken. + offset : int, optional + Offset of the diagonal from the main diagonal. Can be both positive + and negative. Defaults to 0. + axis1, axis2 : int, optional + Axes to be used as the first and second axis of the 2-D sub-arrays + from which the diagonals should be taken. Defaults are the first two + axes of `a`. + dtype : dtype, optional + Determines the data-type of the returned array and of the accumulator + where the elements are summed. If dtype has the value None and `a` is + of integer type of precision less than the default integer + precision, then the default integer precision is used. Otherwise, + the precision is the same as that of `a`. + out : ndarray, optional + Array into which the output is placed. Its type is preserved and + it must be of the right shape to hold the output. + + Returns + ------- + sum_along_diagonals : ndarray + If `a` is 2-D, the sum along the diagonal is returned. If `a` has + larger dimensions, then an array of sums along diagonals is returned. + + See Also + -------- + diag, diagonal, diagflat + + Examples + -------- + >>> np.trace(np.eye(3)) + 3.0 + >>> a = np.arange(8).reshape((2,2,2)) + >>> np.trace(a) + array([6, 8]) + + >>> a = np.arange(24).reshape((2,2,2,3)) + >>> np.trace(a).shape + (2, 3) + + """ + raise NotImplemented('Waiting on interp level method') + +def ravel(a, order='C'): + """ + Return a flattened array. + + A 1-D array, containing the elements of the input, is returned. A copy is + made only if needed. + + Parameters + ---------- + a : array_like + Input array. The elements in ``a`` are read in the order specified by + `order`, and packed as a 1-D array. + order : {'C','F', 'A', 'K'}, optional + The elements of ``a`` are read in this order. 'C' means to view + the elements in C (row-major) order. 'F' means to view the elements + in Fortran (column-major) order. 'A' means to view the elements + in 'F' order if a is Fortran contiguous, 'C' order otherwise. + 'K' means to view the elements in the order they occur in memory, + except for reversing the data when strides are negative. + By default, 'C' order is used. + + Returns + ------- + 1d_array : ndarray + Output of the same dtype as `a`, and of shape ``(a.size(),)``. + + See Also + -------- + ndarray.flat : 1-D iterator over an array. + ndarray.flatten : 1-D array copy of the elements of an array + in row-major order. + + Notes + ----- + In row-major order, the row index varies the slowest, and the column + index the quickest. This can be generalized to multiple dimensions, + where row-major order implies that the index along the first axis + varies slowest, and the index along the last quickest. The opposite holds + for Fortran-, or column-major, mode. + + Examples + -------- + It is equivalent to ``reshape(-1, order=order)``. + + >>> x = np.array([[1, 2, 3], [4, 5, 6]]) + >>> print np.ravel(x) + [1 2 3 4 5 6] + + >>> print x.reshape(-1) + [1 2 3 4 5 6] + + >>> print np.ravel(x, order='F') + [1 4 2 5 3 6] + + When ``order`` is 'A', it will preserve the array's 'C' or 'F' ordering: + + >>> print np.ravel(x.T) + [1 4 2 5 3 6] + >>> print np.ravel(x.T, order='A') + [1 2 3 4 5 6] + + When ``order`` is 'K', it will preserve orderings that are neither 'C' + nor 'F', but won't reverse axes: + + >>> a = np.arange(3)[::-1]; a + array([2, 1, 0]) + >>> a.ravel(order='C') + array([2, 1, 0]) + >>> a.ravel(order='K') + array([2, 1, 0]) + + >>> a = np.arange(12).reshape(2,3,2).swapaxes(1,2); a + array([[[ 0, 2, 4], + [ 1, 3, 5]], + [[ 6, 8, 10], + [ 7, 9, 11]]]) + >>> a.ravel(order='C') + array([ 0, 2, 4, 1, 3, 5, 6, 8, 10, 7, 9, 11]) + >>> a.ravel(order='K') + array([ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]) + + """ + raise NotImplemented('Waiting on interp level method') + + +def nonzero(a): + """ + Return the indices of the elements that are non-zero. + + Returns a tuple of arrays, one for each dimension of `a`, containing + the indices of the non-zero elements in that dimension. The + corresponding non-zero values can be obtained with:: + + a[nonzero(a)] + + To group the indices by element, rather than dimension, use:: + + transpose(nonzero(a)) + + The result of this is always a 2-D array, with a row for + each non-zero element. + + Parameters + ---------- + a : array_like + Input array. + + Returns + ------- + tuple_of_arrays : tuple + Indices of elements that are non-zero. + + See Also + -------- + flatnonzero : + Return indices that are non-zero in the flattened version of the input + array. + ndarray.nonzero : + Equivalent ndarray method. + count_nonzero : + Counts the number of non-zero elements in the input array. + + Examples + -------- + >>> x = np.eye(3) + >>> x + array([[ 1., 0., 0.], + [ 0., 1., 0.], + [ 0., 0., 1.]]) + >>> np.nonzero(x) + (array([0, 1, 2]), array([0, 1, 2])) + + >>> x[np.nonzero(x)] + array([ 1., 1., 1.]) + >>> np.transpose(np.nonzero(x)) + array([[0, 0], + [1, 1], + [2, 2]]) + + A common use for ``nonzero`` is to find the indices of an array, where + a condition is True. Given an array `a`, the condition `a` > 3 is a + boolean array and since False is interpreted as 0, np.nonzero(a > 3) + yields the indices of the `a` where the condition is true. + + >>> a = np.array([[1,2,3],[4,5,6],[7,8,9]]) + >>> a > 3 + array([[False, False, False], + [ True, True, True], + [ True, True, True]], dtype=bool) + >>> np.nonzero(a > 3) + (array([1, 1, 1, 2, 2, 2]), array([0, 1, 2, 0, 1, 2])) + + The ``nonzero`` method of the boolean array can also be called. + + >>> (a > 3).nonzero() + (array([1, 1, 1, 2, 2, 2]), array([0, 1, 2, 0, 1, 2])) + + """ + raise NotImplemented('Waiting on interp level method') + + +def shape(a): + """ + Return the shape of an array. + + Parameters + ---------- + a : array_like + Input array. + + Returns + ------- + shape : tuple of ints + The elements of the shape tuple give the lengths of the + corresponding array dimensions. + + See Also + -------- + alen + ndarray.shape : Equivalent array method. + + Examples + -------- + >>> np.shape(np.eye(3)) + (3, 3) + >>> np.shape([[1, 2]]) + (1, 2) + >>> np.shape([0]) + (1,) + >>> np.shape(0) + () + + >>> a = np.array([(1, 2), (3, 4)], dtype=[('x', 'i4'), ('y', 'i4')]) + >>> np.shape(a) + (2,) + >>> a.shape + (2,) + + """ + if not hasattr(a, 'shape'): + a = numpypy.array(a) + return a.shape + + +def compress(condition, a, axis=None, out=None): + """ + Return selected slices of an array along given axis. + + When working along a given axis, a slice along that axis is returned in + `output` for each index where `condition` evaluates to True. When + working on a 1-D array, `compress` is equivalent to `extract`. + + Parameters + ---------- + condition : 1-D array of bools + Array that selects which entries to return. If len(condition) + is less than the size of `a` along the given axis, then output is + truncated to the length of the condition array. + a : array_like + Array from which to extract a part. + axis : int, optional + Axis along which to take slices. If None (default), work on the + flattened array. + out : ndarray, optional + Output array. Its type is preserved and it must be of the right + shape to hold the output. + + Returns + ------- + compressed_array : ndarray + A copy of `a` without the slices along axis for which `condition` + is false. + + See Also + -------- + take, choose, diag, diagonal, select + ndarray.compress : Equivalent method. + numpy.doc.ufuncs : Section "Output arguments" + + Examples + -------- + >>> a = np.array([[1, 2], [3, 4], [5, 6]]) + >>> a + array([[1, 2], + [3, 4], + [5, 6]]) + >>> np.compress([0, 1], a, axis=0) + array([[3, 4]]) + >>> np.compress([False, True, True], a, axis=0) + array([[3, 4], + [5, 6]]) + >>> np.compress([False, True], a, axis=1) + array([[2], + [4], + [6]]) + + Working on the flattened array does not return slices along an axis but + selects elements. + + >>> np.compress([False, True], a) + array([2]) + + """ + raise NotImplemented('Waiting on interp level method') + + +def clip(a, a_min, a_max, out=None): + """ + Clip (limit) the values in an array. + + Given an interval, values outside the interval are clipped to + the interval edges. For example, if an interval of ``[0, 1]`` + is specified, values smaller than 0 become 0, and values larger + than 1 become 1. + + Parameters + ---------- + a : array_like + Array containing elements to clip. + a_min : scalar or array_like + Minimum value. + a_max : scalar or array_like + Maximum value. If `a_min` or `a_max` are array_like, then they will + be broadcasted to the shape of `a`. + out : ndarray, optional + The results will be placed in this array. It may be the input + array for in-place clipping. `out` must be of the right shape + to hold the output. Its type is preserved. + + Returns + ------- + clipped_array : ndarray + An array with the elements of `a`, but where values + < `a_min` are replaced with `a_min`, and those > `a_max` + with `a_max`. + + See Also + -------- + numpy.doc.ufuncs : Section "Output arguments" + + Examples + -------- + >>> a = np.arange(10) + >>> np.clip(a, 1, 8) + array([1, 1, 2, 3, 4, 5, 6, 7, 8, 8]) + >>> a + array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) + >>> np.clip(a, 3, 6, out=a) + array([3, 3, 3, 3, 4, 5, 6, 6, 6, 6]) + >>> a = np.arange(10) + >>> a + array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) + >>> np.clip(a, [3,4,1,1,1,4,4,4,4,4], 8) + array([3, 4, 2, 3, 4, 5, 6, 7, 8, 8]) + + """ + raise NotImplemented('Waiting on interp level method') + + +def sum(a, axis=None, dtype=None, out=None): + """ + Sum of array elements over a given axis. + + Parameters + ---------- + a : array_like + Elements to sum. + axis : integer, optional + Axis over which the sum is taken. By default `axis` is None, + and all elements are summed. + dtype : dtype, optional + The type of the returned array and of the accumulator in which + the elements are summed. By default, the dtype of `a` is used. + An exception is when `a` has an integer type with less precision + than the default platform integer. In that case, the default + platform integer is used instead. + out : ndarray, optional + Array into which the output is placed. By default, a new array is + created. If `out` is given, it must be of the appropriate shape + (the shape of `a` with `axis` removed, i.e., + ``numpy.delete(a.shape, axis)``). Its type is preserved. See + `doc.ufuncs` (Section "Output arguments") for more details. + + Returns + ------- + sum_along_axis : ndarray + An array with the same shape as `a`, with the specified + axis removed. If `a` is a 0-d array, or if `axis` is None, a scalar + is returned. If an output array is specified, a reference to + `out` is returned. + + See Also + -------- + ndarray.sum : Equivalent method. + + cumsum : Cumulative sum of array elements. + + trapz : Integration of array values using the composite trapezoidal rule. + + mean, average + + Notes + ----- + Arithmetic is modular when using integer types, and no error is + raised on overflow. + + Examples + -------- + >>> np.sum([0.5, 1.5]) + 2.0 + >>> np.sum([0.5, 0.7, 0.2, 1.5], dtype=np.int32) + 1 + >>> np.sum([[0, 1], [0, 5]]) + 6 + >>> np.sum([[0, 1], [0, 5]], axis=0) + array([0, 6]) + >>> np.sum([[0, 1], [0, 5]], axis=1) + array([1, 5]) + + If the accumulator is too small, overflow occurs: + + >>> np.ones(128, dtype=np.int8).sum(dtype=np.int8) + -128 + + """ + if not hasattr(a, "sum"): + a = numpypy.array(a) + return a.sum() + + +def product (a, axis=None, dtype=None, out=None): + """ + Return the product of array elements over a given axis. + + See Also + -------- + prod : equivalent function; see for details. + + """ + raise NotImplemented('Waiting on interp level method') + + +def sometrue(a, axis=None, out=None): + """ + Check whether some values are true. + + Refer to `any` for full documentation. + + See Also + -------- + any : equivalent function + + """ + if not hasattr(a, 'any'): + a = numpypy.array(a) + return a.any() + + +def alltrue (a, axis=None, out=None): + """ + Check if all elements of input array are true. + + See Also + -------- + numpy.all : Equivalent function; see for details. + + """ + if not hasattr(a, 'all'): + a = numpypy.array(a) + return a.all() + +def any(a,axis=None, out=None): + """ + Test whether any array element along a given axis evaluates to True. + + Returns single boolean unless `axis` is not ``None`` + + Parameters + ---------- + a : array_like + Input array or object that can be converted to an array. + axis : int, optional + Axis along which a logical OR is performed. The default + (`axis` = `None`) is to perform a logical OR over a flattened + input array. `axis` may be negative, in which case it counts + from the last to the first axis. + out : ndarray, optional + Alternate output array in which to place the result. It must have + the same shape as the expected output and its type is preserved + (e.g., if it is of type float, then it will remain so, returning + 1.0 for True and 0.0 for False, regardless of the type of `a`). + See `doc.ufuncs` (Section "Output arguments") for details. + + Returns + ------- + any : bool or ndarray + A new boolean or `ndarray` is returned unless `out` is specified, + in which case a reference to `out` is returned. + + See Also + -------- + ndarray.any : equivalent method + + all : Test whether all elements along a given axis evaluate to True. + + Notes + ----- + Not a Number (NaN), positive infinity and negative infinity evaluate + to `True` because these are not equal to zero. + + Examples + -------- + >>> np.any([[True, False], [True, True]]) + True + + >>> np.any([[True, False], [False, False]], axis=0) + array([ True, False], dtype=bool) + + >>> np.any([-1, 0, 5]) + True + + >>> np.any(np.nan) + True + + >>> o=np.array([False]) + >>> z=np.any([-1, 4, 5], out=o) + >>> z, o + (array([ True], dtype=bool), array([ True], dtype=bool)) + >>> # Check now that z is a reference to o + >>> z is o + True + >>> id(z), id(o) # identity of z and o # doctest: +SKIP + (191614240, 191614240) + + """ + if not hasattr(a, 'any'): + a = numpypy.array(a) + return a.any() + + +def all(a,axis=None, out=None): + """ + Test whether all array elements along a given axis evaluate to True. + + Parameters + ---------- + a : array_like + Input array or object that can be converted to an array. + axis : int, optional + Axis along which a logical AND is performed. + The default (`axis` = `None`) is to perform a logical AND + over a flattened input array. `axis` may be negative, in which + case it counts from the last to the first axis. + out : ndarray, optional + Alternate output array in which to place the result. + It must have the same shape as the expected output and its + type is preserved (e.g., if ``dtype(out)`` is float, the result + will consist of 0.0's and 1.0's). See `doc.ufuncs` (Section + "Output arguments") for more details. + + Returns + ------- + all : ndarray, bool + A new boolean or array is returned unless `out` is specified, + in which case a reference to `out` is returned. + + See Also + -------- + ndarray.all : equivalent method + + any : Test whether any element along a given axis evaluates to True. + + Notes + ----- + Not a Number (NaN), positive infinity and negative infinity + evaluate to `True` because these are not equal to zero. + + Examples + -------- + >>> np.all([[True,False],[True,True]]) + False + + >>> np.all([[True,False],[True,True]], axis=0) + array([ True, False], dtype=bool) + + >>> np.all([-1, 4, 5]) + True + + >>> np.all([1.0, np.nan]) + True + + >>> o=np.array([False]) + >>> z=np.all([-1, 4, 5], out=o) + >>> id(z), id(o), z # doctest: +SKIP + (28293632, 28293632, array([ True], dtype=bool)) + + """ + if not hasattr(a, 'all'): + a = numpypy.array(a) + return a.all() + + +def cumsum (a, axis=None, dtype=None, out=None): + """ + Return the cumulative sum of the elements along a given axis. + + Parameters + ---------- + a : array_like + Input array. + axis : int, optional + Axis along which the cumulative sum is computed. The default + (None) is to compute the cumsum over the flattened array. + dtype : dtype, optional + Type of the returned array and of the accumulator in which the + elements are summed. If `dtype` is not specified, it defaults + to the dtype of `a`, unless `a` has an integer dtype with a + precision less than that of the default platform integer. In + that case, the default platform integer is used. + out : ndarray, optional + Alternative output array in which to place the result. It must + have the same shape and buffer length as the expected output + but the type will be cast if necessary. See `doc.ufuncs` + (Section "Output arguments") for more details. + + Returns + ------- + cumsum_along_axis : ndarray. + A new array holding the result is returned unless `out` is + specified, in which case a reference to `out` is returned. The + result has the same size as `a`, and the same shape as `a` if + `axis` is not None or `a` is a 1-d array. + + + See Also + -------- + sum : Sum array elements. + + trapz : Integration of array values using the composite trapezoidal rule. + + Notes + ----- + Arithmetic is modular when using integer types, and no error is + raised on overflow. + + Examples + -------- + >>> a = np.array([[1,2,3], [4,5,6]]) + >>> a + array([[1, 2, 3], + [4, 5, 6]]) + >>> np.cumsum(a) + array([ 1, 3, 6, 10, 15, 21]) + >>> np.cumsum(a, dtype=float) # specifies type of output value(s) + array([ 1., 3., 6., 10., 15., 21.]) + + >>> np.cumsum(a,axis=0) # sum over rows for each of the 3 columns + array([[1, 2, 3], + [5, 7, 9]]) + >>> np.cumsum(a,axis=1) # sum over columns for each of the 2 rows + array([[ 1, 3, 6], + [ 4, 9, 15]]) + + """ + raise NotImplemented('Waiting on interp level method') + + +def cumproduct(a, axis=None, dtype=None, out=None): + """ + Return the cumulative product over the given axis. + + + See Also + -------- + cumprod : equivalent function; see for details. + + """ + raise NotImplemented('Waiting on interp level method') + + +def ptp(a, axis=None, out=None): + """ + Range of values (maximum - minimum) along an axis. + + The name of the function comes from the acronym for 'peak to peak'. + + Parameters + ---------- + a : array_like + Input values. + axis : int, optional + Axis along which to find the peaks. By default, flatten the + array. + out : array_like + Alternative output array in which to place the result. It must + have the same shape and buffer length as the expected output, + but the type of the output values will be cast if necessary. + + Returns + ------- + ptp : ndarray + A new array holding the result, unless `out` was + specified, in which case a reference to `out` is returned. + + Examples + -------- + >>> x = np.arange(4).reshape((2,2)) + >>> x + array([[0, 1], + [2, 3]]) + + >>> np.ptp(x, axis=0) + array([2, 2]) + + >>> np.ptp(x, axis=1) + array([1, 1]) + + """ + raise NotImplemented('Waiting on interp level method') + + +def amax(a, axis=None, out=None): + """ + Return the maximum of an array or maximum along an axis. + + Parameters + ---------- + a : array_like + Input data. + axis : int, optional + Axis along which to operate. By default flattened input is used. + out : ndarray, optional + Alternate output array in which to place the result. Must be of + the same shape and buffer length as the expected output. See + `doc.ufuncs` (Section "Output arguments") for more details. + + Returns + ------- + amax : ndarray or scalar + Maximum of `a`. If `axis` is None, the result is a scalar value. + If `axis` is given, the result is an array of dimension + ``a.ndim - 1``. + + See Also + -------- + nanmax : NaN values are ignored instead of being propagated. + fmax : same behavior as the C99 fmax function. + argmax : indices of the maximum values. + + Notes + ----- + NaN values are propagated, that is if at least one item is NaN, the + corresponding max value will be NaN as well. To ignore NaN values + (MATLAB behavior), please use nanmax. + + Examples + -------- + >>> a = np.arange(4).reshape((2,2)) + >>> a + array([[0, 1], + [2, 3]]) + >>> np.amax(a) + 3 + >>> np.amax(a, axis=0) + array([2, 3]) + >>> np.amax(a, axis=1) + array([1, 3]) + + >>> b = np.arange(5, dtype=np.float) + >>> b[2] = np.NaN + >>> np.amax(b) + nan + >>> np.nanmax(b) + 4.0 + + """ + if not hasattr(a, "max"): + a = numpypy.array(a) + return a.max() + + +def amin(a, axis=None, out=None): + """ + Return the minimum of an array or minimum along an axis. + + Parameters + ---------- + a : array_like + Input data. + axis : int, optional + Axis along which to operate. By default a flattened input is used. + out : ndarray, optional + Alternative output array in which to place the result. Must + be of the same shape and buffer length as the expected output. + See `doc.ufuncs` (Section "Output arguments") for more details. + + Returns + ------- + amin : ndarray + A new array or a scalar array with the result. + + See Also + -------- + nanmin: nan values are ignored instead of being propagated + fmin: same behavior as the C99 fmin function + argmin: Return the indices of the minimum values. + + amax, nanmax, fmax + + Notes + ----- + NaN values are propagated, that is if at least one item is nan, the + corresponding min value will be nan as well. To ignore NaN values (matlab + behavior), please use nanmin. + + Examples + -------- + >>> a = np.arange(4).reshape((2,2)) + >>> a + array([[0, 1], + [2, 3]]) + >>> np.amin(a) # Minimum of the flattened array + 0 + >>> np.amin(a, axis=0) # Minima along the first axis + array([0, 1]) + >>> np.amin(a, axis=1) # Minima along the second axis + array([0, 2]) + + >>> b = np.arange(5, dtype=np.float) + >>> b[2] = np.NaN + >>> np.amin(b) + nan + >>> np.nanmin(b) + 0.0 + + """ + # amin() is equivalent to min() + if not hasattr(a, 'min'): + a = numpypy.array(a) + return a.min() + +def alen(a): + """ + Return the length of the first dimension of the input array. + + Parameters + ---------- + a : array_like + Input array. + + Returns + ------- + l : int + Length of the first dimension of `a`. + + See Also + -------- + shape, size + + Examples + -------- + >>> a = np.zeros((7,4,5)) + >>> a.shape[0] + 7 + >>> np.alen(a) + 7 + + """ + if not hasattr(a, 'shape'): + a = numpypy.array(a) + return a.shape[0] + + +def prod(a, axis=None, dtype=None, out=None): + """ + Return the product of array elements over a given axis. + + Parameters + ---------- + a : array_like + Input data. + axis : int, optional + Axis over which the product is taken. By default, the product + of all elements is calculated. + dtype : data-type, optional + The data-type of the returned array, as well as of the accumulator + in which the elements are multiplied. By default, if `a` is of + integer type, `dtype` is the default platform integer. (Note: if + the type of `a` is unsigned, then so is `dtype`.) Otherwise, + the dtype is the same as that of `a`. + out : ndarray, optional + Alternative output array in which to place the result. It must have + the same shape as the expected output, but the type of the + output values will be cast if necessary. + + Returns + ------- + product_along_axis : ndarray, see `dtype` parameter above. + An array shaped as `a` but with the specified axis removed. + Returns a reference to `out` if specified. + + See Also + -------- + ndarray.prod : equivalent method + numpy.doc.ufuncs : Section "Output arguments" + + Notes + ----- + Arithmetic is modular when using integer types, and no error is + raised on overflow. That means that, on a 32-bit platform: + + >>> x = np.array([536870910, 536870910, 536870910, 536870910]) + >>> np.prod(x) #random + 16 + + Examples + -------- + By default, calculate the product of all elements: + + >>> np.prod([1.,2.]) + 2.0 + + Even when the input array is two-dimensional: + + >>> np.prod([[1.,2.],[3.,4.]]) + 24.0 + + But we can also specify the axis over which to multiply: + + >>> np.prod([[1.,2.],[3.,4.]], axis=1) + array([ 2., 12.]) + + If the type of `x` is unsigned, then the output type is + the unsigned platform integer: + + >>> x = np.array([1, 2, 3], dtype=np.uint8) + >>> np.prod(x).dtype == np.uint + True + + If `x` is of a signed integer type, then the output type + is the default platform integer: + + >>> x = np.array([1, 2, 3], dtype=np.int8) + >>> np.prod(x).dtype == np.int + True + + """ + raise NotImplemented('Waiting on interp level method') + + +def cumprod(a, axis=None, dtype=None, out=None): + """ + Return the cumulative product of elements along a given axis. + + Parameters + ---------- + a : array_like + Input array. + axis : int, optional + Axis along which the cumulative product is computed. By default + the input is flattened. + dtype : dtype, optional + Type of the returned array, as well as of the accumulator in which + the elements are multiplied. If *dtype* is not specified, it + defaults to the dtype of `a`, unless `a` has an integer dtype with + a precision less than that of the default platform integer. In + that case, the default platform integer is used instead. + out : ndarray, optional + Alternative output array in which to place the result. It must + have the same shape and buffer length as the expected output + but the type of the resulting values will be cast if necessary. + + Returns + ------- + cumprod : ndarray + A new array holding the result is returned unless `out` is + specified, in which case a reference to out is returned. + + See Also + -------- + numpy.doc.ufuncs : Section "Output arguments" + + Notes + ----- + Arithmetic is modular when using integer types, and no error is + raised on overflow. + + Examples + -------- + >>> a = np.array([1,2,3]) + >>> np.cumprod(a) # intermediate results 1, 1*2 + ... # total product 1*2*3 = 6 + array([1, 2, 6]) + >>> a = np.array([[1, 2, 3], [4, 5, 6]]) + >>> np.cumprod(a, dtype=float) # specify type of output + array([ 1., 2., 6., 24., 120., 720.]) + + The cumulative product for each column (i.e., over the rows) of `a`: + + >>> np.cumprod(a, axis=0) + array([[ 1, 2, 3], + [ 4, 10, 18]]) + + The cumulative product for each row (i.e. over the columns) of `a`: + + >>> np.cumprod(a,axis=1) + array([[ 1, 2, 6], + [ 4, 20, 120]]) + + """ + raise NotImplemented('Waiting on interp level method') + + +def ndim(a): + """ + Return the number of dimensions of an array. + + Parameters + ---------- + a : array_like + Input array. If it is not already an ndarray, a conversion is + attempted. + + Returns + ------- + number_of_dimensions : int + The number of dimensions in `a`. Scalars are zero-dimensional. + + See Also + -------- + ndarray.ndim : equivalent method + shape : dimensions of array + ndarray.shape : dimensions of array + + Examples + -------- + >>> np.ndim([[1,2,3],[4,5,6]]) + 2 + >>> np.ndim(np.array([[1,2,3],[4,5,6]])) + 2 + >>> np.ndim(1) + 0 + + """ + if not hasattr(a, 'ndim'): + a = numpypy.array(a) + return a.ndim + + +def rank(a): + """ + Return the number of dimensions of an array. + + If `a` is not already an array, a conversion is attempted. + Scalars are zero dimensional. + + Parameters + ---------- + a : array_like + Array whose number of dimensions is desired. If `a` is not an array, + a conversion is attempted. + + Returns + ------- + number_of_dimensions : int + The number of dimensions in the array. + + See Also + -------- + ndim : equivalent function + ndarray.ndim : equivalent property + shape : dimensions of array + ndarray.shape : dimensions of array + + Notes + ----- + In the old Numeric package, `rank` was the term used for the number of + dimensions, but in Numpy `ndim` is used instead. + + Examples + -------- + >>> np.rank([1,2,3]) + 1 + >>> np.rank(np.array([[1,2,3],[4,5,6]])) + 2 + >>> np.rank(1) + 0 + + """ + if not hasattr(a, 'ndim'): + a = numpypy.array(a) + return a.ndim + + +def size(a, axis=None): + """ + Return the number of elements along a given axis. + + Parameters + ---------- + a : array_like + Input data. + axis : int, optional + Axis along which the elements are counted. By default, give + the total number of elements. + + Returns + ------- + element_count : int + Number of elements along the specified axis. + + See Also + -------- + shape : dimensions of array + ndarray.shape : dimensions of array + ndarray.size : number of elements in array + + Examples + -------- + >>> a = np.array([[1,2,3],[4,5,6]]) + >>> np.size(a) + 6 + >>> np.size(a,1) + 3 + >>> np.size(a,0) + 2 + + """ + raise NotImplemented('Waiting on interp level method') + + +def around(a, decimals=0, out=None): + """ + Evenly round to the given number of decimals. + + Parameters + ---------- + a : array_like + Input data. + decimals : int, optional + Number of decimal places to round to (default: 0). If + decimals is negative, it specifies the number of positions to + the left of the decimal point. + out : ndarray, optional + Alternative output array in which to place the result. It must have + the same shape as the expected output, but the type of the output + values will be cast if necessary. See `doc.ufuncs` (Section + "Output arguments") for details. + + Returns + ------- + rounded_array : ndarray + An array of the same type as `a`, containing the rounded values. + Unless `out` was specified, a new array is created. A reference to + the result is returned. + + The real and imaginary parts of complex numbers are rounded + separately. The result of rounding a float is a float. + + See Also + -------- + ndarray.round : equivalent method + + ceil, fix, floor, rint, trunc + + + Notes + ----- + For values exactly halfway between rounded decimal values, Numpy + rounds to the nearest even value. Thus 1.5 and 2.5 round to 2.0, + -0.5 and 0.5 round to 0.0, etc. Results may also be surprising due + to the inexact representation of decimal fractions in the IEEE + floating point standard [1]_ and errors introduced when scaling + by powers of ten. + + References + ---------- + .. [1] "Lecture Notes on the Status of IEEE 754", William Kahan, + http://www.cs.berkeley.edu/~wkahan/ieee754status/IEEE754.PDF + .. [2] "How Futile are Mindless Assessments of + Roundoff in Floating-Point Computation?", William Kahan, + http://www.cs.berkeley.edu/~wkahan/Mindless.pdf + + Examples + -------- + >>> np.around([0.37, 1.64]) + array([ 0., 2.]) + >>> np.around([0.37, 1.64], decimals=1) + array([ 0.4, 1.6]) + >>> np.around([.5, 1.5, 2.5, 3.5, 4.5]) # rounds to nearest even value + array([ 0., 2., 2., 4., 4.]) + >>> np.around([1,2,3,11], decimals=1) # ndarray of ints is returned + array([ 1, 2, 3, 11]) + >>> np.around([1,2,3,11], decimals=-1) + array([ 0, 0, 0, 10]) + + """ + raise NotImplemented('Waiting on interp level method') + + +def round_(a, decimals=0, out=None): + """ + Round an array to the given number of decimals. + + Refer to `around` for full documentation. + + See Also + -------- + around : equivalent function + + """ + raise NotImplemented('Waiting on interp level method') + + +def mean(a, axis=None, dtype=None, out=None): + """ + Compute the arithmetic mean along the specified axis. + + Returns the average of the array elements. The average is taken over + the flattened array by default, otherwise over the specified axis. + `float64` intermediate and return values are used for integer inputs. + + Parameters + ---------- + a : array_like + Array containing numbers whose mean is desired. If `a` is not an + array, a conversion is attempted. + axis : int, optional + Axis along which the means are computed. The default is to compute + the mean of the flattened array. + dtype : data-type, optional + Type to use in computing the mean. For integer inputs, the default + is `float64`; for floating point inputs, it is the same as the + input dtype. + out : ndarray, optional + Alternate output array in which to place the result. The default + is ``None``; if provided, it must have the same shape as the + expected output, but the type will be cast if necessary. + See `doc.ufuncs` for details. + + Returns + ------- + m : ndarray, see dtype parameter above + If `out=None`, returns a new array containing the mean values, + otherwise a reference to the output array is returned. + + See Also + -------- + average : Weighted average + + Notes + ----- + The arithmetic mean is the sum of the elements along the axis divided + by the number of elements. + + Note that for floating-point input, the mean is computed using the + same precision the input has. Depending on the input data, this can + cause the results to be inaccurate, especially for `float32` (see + example below). Specifying a higher-precision accumulator using the + `dtype` keyword can alleviate this issue. + + Examples + -------- + >>> a = np.array([[1, 2], [3, 4]]) + >>> np.mean(a) + 2.5 + >>> np.mean(a, axis=0) + array([ 2., 3.]) + >>> np.mean(a, axis=1) + array([ 1.5, 3.5]) + + In single precision, `mean` can be inaccurate: + + >>> a = np.zeros((2, 512*512), dtype=np.float32) + >>> a[0, :] = 1.0 + >>> a[1, :] = 0.1 + >>> np.mean(a) + 0.546875 + + Computing the mean in float64 is more accurate: + + >>> np.mean(a, dtype=np.float64) + 0.55000000074505806 + + """ + if not hasattr(a, "mean"): + a = numpypy.array(a) + return a.mean() + + +def std(a, axis=None, dtype=None, out=None, ddof=0): + """ + Compute the standard deviation along the specified axis. + + Returns the standard deviation, a measure of the spread of a distribution, + of the array elements. The standard deviation is computed for the + flattened array by default, otherwise over the specified axis. + + Parameters + ---------- + a : array_like + Calculate the standard deviation of these values. + axis : int, optional + Axis along which the standard deviation is computed. The default is + to compute the standard deviation of the flattened array. + dtype : dtype, optional + Type to use in computing the standard deviation. For arrays of + integer type the default is float64, for arrays of float types it is + the same as the array type. + out : ndarray, optional + Alternative output array in which to place the result. It must have + the same shape as the expected output but the type (of the calculated + values) will be cast if necessary. + ddof : int, optional + Means Delta Degrees of Freedom. The divisor used in calculations + is ``N - ddof``, where ``N`` represents the number of elements. + By default `ddof` is zero. + + Returns + ------- + standard_deviation : ndarray, see dtype parameter above. + If `out` is None, return a new array containing the standard deviation, + otherwise return a reference to the output array. + + See Also + -------- + var, mean + numpy.doc.ufuncs : Section "Output arguments" + + Notes + ----- + The standard deviation is the square root of the average of the squared + deviations from the mean, i.e., ``std = sqrt(mean(abs(x - x.mean())**2))``. + + The average squared deviation is normally calculated as ``x.sum() / N``, where + ``N = len(x)``. If, however, `ddof` is specified, the divisor ``N - ddof`` + is used instead. In standard statistical practice, ``ddof=1`` provides an + unbiased estimator of the variance of the infinite population. ``ddof=0`` + provides a maximum likelihood estimate of the variance for normally + distributed variables. The standard deviation computed in this function + is the square root of the estimated variance, so even with ``ddof=1``, it + will not be an unbiased estimate of the standard deviation per se. + + Note that, for complex numbers, `std` takes the absolute + value before squaring, so that the result is always real and nonnegative. + + For floating-point input, the *std* is computed using the same + precision the input has. Depending on the input data, this can cause + the results to be inaccurate, especially for float32 (see example below). + Specifying a higher-accuracy accumulator using the `dtype` keyword can + alleviate this issue. + + Examples + -------- + >>> a = np.array([[1, 2], [3, 4]]) + >>> np.std(a) + 1.1180339887498949 + >>> np.std(a, axis=0) + array([ 1., 1.]) + >>> np.std(a, axis=1) + array([ 0.5, 0.5]) + + In single precision, std() can be inaccurate: + + >>> a = np.zeros((2,512*512), dtype=np.float32) + >>> a[0,:] = 1.0 + >>> a[1,:] = 0.1 + >>> np.std(a) + 0.45172946707416706 + + Computing the standard deviation in float64 is more accurate: + + >>> np.std(a, dtype=np.float64) + 0.44999999925552653 + + """ + if not hasattr(a, "std"): + a = numpypy.array(a) + return a.std() + + +def var(a, axis=None, dtype=None, out=None, ddof=0): + """ + Compute the variance along the specified axis. + + Returns the variance of the array elements, a measure of the spread of a + distribution. The variance is computed for the flattened array by + default, otherwise over the specified axis. + + Parameters + ---------- + a : array_like + Array containing numbers whose variance is desired. If `a` is not an + array, a conversion is attempted. + axis : int, optional + Axis along which the variance is computed. The default is to compute + the variance of the flattened array. + dtype : data-type, optional + Type to use in computing the variance. For arrays of integer type + the default is `float32`; for arrays of float types it is the same as + the array type. + out : ndarray, optional + Alternate output array in which to place the result. It must have + the same shape as the expected output, but the type is cast if + necessary. + ddof : int, optional + "Delta Degrees of Freedom": the divisor used in the calculation is + ``N - ddof``, where ``N`` represents the number of elements. By + default `ddof` is zero. + + Returns + ------- + variance : ndarray, see dtype parameter above + If ``out=None``, returns a new array containing the variance; + otherwise, a reference to the output array is returned. + + See Also + -------- + std : Standard deviation + mean : Average + numpy.doc.ufuncs : Section "Output arguments" + + Notes + ----- + The variance is the average of the squared deviations from the mean, + i.e., ``var = mean(abs(x - x.mean())**2)``. + + The mean is normally calculated as ``x.sum() / N``, where ``N = len(x)``. + If, however, `ddof` is specified, the divisor ``N - ddof`` is used + instead. In standard statistical practice, ``ddof=1`` provides an + unbiased estimator of the variance of a hypothetical infinite population. + ``ddof=0`` provides a maximum likelihood estimate of the variance for + normally distributed variables. + + Note that for complex numbers, the absolute value is taken before + squaring, so that the result is always real and nonnegative. + + For floating-point input, the variance is computed using the same + precision the input has. Depending on the input data, this can cause + the results to be inaccurate, especially for `float32` (see example + below). Specifying a higher-accuracy accumulator using the ``dtype`` + keyword can alleviate this issue. + + Examples + -------- + >>> a = np.array([[1,2],[3,4]]) + >>> np.var(a) + 1.25 + >>> np.var(a,0) + array([ 1., 1.]) + >>> np.var(a,1) + array([ 0.25, 0.25]) + + In single precision, var() can be inaccurate: + + >>> a = np.zeros((2,512*512), dtype=np.float32) + >>> a[0,:] = 1.0 + >>> a[1,:] = 0.1 + >>> np.var(a) + 0.20405951142311096 + + Computing the standard deviation in float64 is more accurate: + + >>> np.var(a, dtype=np.float64) + 0.20249999932997387 + >>> ((1-0.55)**2 + (0.1-0.55)**2)/2 + 0.20250000000000001 + + """ + if not hasattr(a, "var"): + a = numpypy.array(a) + return a.var() diff --git a/lib_pypy/numpypy/test/test_fromnumeric.py b/lib_pypy/numpypy/test/test_fromnumeric.py new file mode 100644 --- /dev/null +++ b/lib_pypy/numpypy/test/test_fromnumeric.py @@ -0,0 +1,109 @@ + +from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest + +class AppTestFromNumeric(BaseNumpyAppTest): + def test_argmax(self): + # tests taken from numpy/core/fromnumeric.py docstring + from numpypy import array, arange, argmax + a = arange(6).reshape((2,3)) + assert argmax(a) == 5 + # assert (argmax(a, axis=0) == array([1, 1, 1])).all() + # assert (argmax(a, axis=1) == array([2, 2])).all() + b = arange(6) + b[1] = 5 + assert argmax(b) == 1 + + def test_argmin(self): + # tests adapted from test_argmax + from numpypy import array, arange, argmin + a = arange(6).reshape((2,3)) + assert argmin(a) == 0 + # assert (argmax(a, axis=0) == array([0, 0, 0])).all() + # assert (argmax(a, axis=1) == array([0, 0])).all() + b = arange(6) + b[1] = 0 + assert argmin(b) == 0 + + def test_shape(self): + # tests taken from numpy/core/fromnumeric.py docstring + from numpypy import array, identity, shape + assert shape(identity(3)) == (3, 3) + assert shape([[1, 2]]) == (1, 2) + assert shape([0]) == (1,) + assert shape(0) == () + # a = array([(1, 2), (3, 4)], dtype=[('x', 'i4'), ('y', 'i4')]) + # assert shape(a) == (2,) + + def test_sum(self): + # tests taken from numpy/core/fromnumeric.py docstring + from numpypy import array, sum, ones + assert sum([0.5, 1.5])== 2.0 + assert sum([[0, 1], [0, 5]]) == 6 + # assert sum([0.5, 0.7, 0.2, 1.5], dtype=int32) == 1 + # assert (sum([[0, 1], [0, 5]], axis=0) == array([0, 6])).all() + # assert (sum([[0, 1], [0, 5]], axis=1) == array([1, 5])).all() + # If the accumulator is too small, overflow occurs: + # assert ones(128, dtype=int8).sum(dtype=int8) == -128 + + def test_amin(self): + # tests taken from numpy/core/fromnumeric.py docstring + from numpypy import array, arange, amin + a = arange(4).reshape((2,2)) + assert amin(a) == 0 + # # Minima along the first axis + # assert (amin(a, axis=0) == array([0, 1])).all() + # # Minima along the second axis + # assert (amin(a, axis=1) == array([0, 2])).all() + # # NaN behaviour + # b = arange(5, dtype=float) + # b[2] = NaN + # assert amin(b) == nan + # assert nanmin(b) == 0.0 + + def test_amax(self): + # tests taken from numpy/core/fromnumeric.py docstring + from numpypy import array, arange, amax + a = arange(4).reshape((2,2)) + assert amax(a) == 3 + # assert (amax(a, axis=0) == array([2, 3])).all() + # assert (amax(a, axis=1) == array([1, 3])).all() + # # NaN behaviour + # b = arange(5, dtype=float) + # b[2] = NaN + # assert amax(b) == nan + # assert nanmax(b) == 4.0 + + def test_alen(self): + # tests taken from numpy/core/fromnumeric.py docstring + from numpypy import array, zeros, alen + a = zeros((7,4,5)) + assert a.shape[0] == 7 + assert alen(a) == 7 + + def test_ndim(self): + # tests taken from numpy/core/fromnumeric.py docstring + from numpypy import array, ndim + assert ndim([[1,2,3],[4,5,6]]) == 2 + assert ndim(array([[1,2,3],[4,5,6]])) == 2 + assert ndim(1) == 0 + + def test_rank(self): + # tests taken from numpy/core/fromnumeric.py docstring + from numpypy import array, rank + assert rank([[1,2,3],[4,5,6]]) == 2 + assert rank(array([[1,2,3],[4,5,6]])) == 2 + assert rank(1) == 0 + + def test_var(self): + from numpypy import array, var + a = array([[1,2],[3,4]]) + assert var(a) == 1.25 + # assert (np.var(a,0) == array([ 1., 1.])).all() + # assert (np.var(a,1) == array([ 0.25, 0.25])).all() + + def test_std(self): + from numpypy import array, std + a = array([[1, 2], [3, 4]]) + assert std(a) == 1.1180339887498949 + # assert (std(a, axis=0) == array([ 1., 1.])).all() + # assert (std(a, axis=1) == array([ 0.5, 0.5]).all() diff --git a/pypy/annotation/description.py b/pypy/annotation/description.py --- a/pypy/annotation/description.py +++ b/pypy/annotation/description.py @@ -180,7 +180,12 @@ if name is None: name = pyobj.func_name if signature is None: - signature = cpython_code_signature(pyobj.func_code) + if hasattr(pyobj, '_generator_next_method_of_'): + from pypy.interpreter.argument import Signature + signature = Signature(['entry']) # haaaaaack + defaults = () + else: + signature = cpython_code_signature(pyobj.func_code) if defaults is None: defaults = pyobj.func_defaults self.name = name @@ -252,7 +257,8 @@ try: inputcells = args.match_signature(signature, defs_s) except ArgErr, e: - raise TypeError, "signature mismatch: %s" % e.getmsg(self.name) + raise TypeError("signature mismatch: %s() %s" % + (self.name, e.getmsg())) return inputcells def specialize(self, inputcells, op=None): diff --git a/pypy/doc/Makefile b/pypy/doc/Makefile --- a/pypy/doc/Makefile +++ b/pypy/doc/Makefile @@ -12,7 +12,7 @@ PAPEROPT_letter = -D latex_paper_size=letter ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . -.PHONY: help clean html dirhtml pickle json htmlhelp qthelp latex changes linkcheck doctest +.PHONY: help clean html dirhtml pickle json htmlhelp qthelp latex man changes linkcheck doctest help: @echo "Please use \`make ' where is one of" @@ -23,6 +23,7 @@ @echo " htmlhelp to make HTML files and a HTML help project" @echo " qthelp to make HTML files and a qthelp project" @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " man to make manual pages" @echo " changes to make an overview of all changed/added/deprecated items" @echo " linkcheck to check all external links for integrity" @echo " doctest to run all doctests embedded in the documentation (if enabled)" @@ -79,6 +80,11 @@ @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \ "run these through (pdf)latex." +man: + $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man + @echo + @echo "Build finished. The manual pages are in $(BUILDDIR)/man" + changes: python config/generate.py $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes diff --git a/pypy/doc/coding-guide.rst b/pypy/doc/coding-guide.rst --- a/pypy/doc/coding-guide.rst +++ b/pypy/doc/coding-guide.rst @@ -175,15 +175,15 @@ RPython ================= -RPython Definition, not ------------------------ +RPython Definition +------------------ -The list and exact details of the "RPython" restrictions are a somewhat -evolving topic. In particular, we have no formal language definition -as we find it more practical to discuss and evolve the set of -restrictions while working on the whole program analysis. If you -have any questions about the restrictions below then please feel -free to mail us at pypy-dev at codespeak net. +RPython is a restricted subset of Python that is amenable to static analysis. +Although there are additions to the language and some things might surprisingly +work, this is a rough list of restrictions that should be considered. Note +that there are tons of special cased restrictions that you'll encounter +as you go. The exact definition is "RPython is everything that our translation +toolchain can accept" :) .. _`wrapped object`: coding-guide.html#wrapping-rules @@ -198,7 +198,7 @@ contain both a string and a int must be avoided. It is allowed to mix None (basically with the role of a null pointer) with many other types: `wrapped objects`, class instances, lists, dicts, strings, etc. - but *not* with int and floats. + but *not* with int, floats or tuples. **constants** @@ -209,9 +209,12 @@ have this restriction, so if you need mutable global state, store it in the attributes of some prebuilt singleton instance. + + **control structures** - all allowed but yield, ``for`` loops restricted to builtin types + all allowed, ``for`` loops restricted to builtin types, generators + very restricted. **range** @@ -226,7 +229,8 @@ **generators** - generators are not supported. + generators are supported, but their exact scope is very limited. you can't + merge two different generator in one control point. **exceptions** @@ -245,22 +249,27 @@ **strings** - a lot of, but not all string methods are supported. Indexes can be + a lot of, but not all string methods are supported and those that are + supported, not necesarilly accept all arguments. Indexes can be negative. In case they are not, then you get slightly more efficient code if the translator can prove that they are non-negative. When slicing a string it is necessary to prove that the slice start and - stop indexes are non-negative. + stop indexes are non-negative. There is no implicit str-to-unicode cast + anywhere. **tuples** no variable-length tuples; use them to store or return pairs or n-tuples of - values. Each combination of types for elements and length constitute a separate - and not mixable type. + values. Each combination of types for elements and length constitute + a separate and not mixable type. **lists** lists are used as an allocated array. Lists are over-allocated, so list.append() - is reasonably fast. Negative or out-of-bound indexes are only allowed for the + is reasonably fast. However, if you use a fixed-size list, the code + is more efficient. Annotator can figure out most of the time that your + list is fixed-size, even when you use list comprehension. + Negative or out-of-bound indexes are only allowed for the most common operations, as follows: - *indexing*: @@ -287,16 +296,14 @@ **dicts** - dicts with a unique key type only, provided it is hashable. - String keys have been the only allowed key types for a while, but this was generalized. - After some re-optimization, - the implementation could safely decide that all string dict keys should be interned. + dicts with a unique key type only, provided it is hashable. Custom + hash functions and custom equality will not be honored. + Use ``pypy.rlib.objectmodel.r_dict`` for custom hash functions. **list comprehensions** - may be used to create allocated, initialized arrays. - After list over-allocation was introduced, there is no longer any restriction. + May be used to create allocated, initialized arrays. **functions** @@ -334,9 +341,8 @@ **objects** - in PyPy, wrapped objects are borrowed from the object space. Just like - in CPython, code that needs e.g. a dictionary can use a wrapped dict - and the object space operations on it. + Normal rules apply. Special methods are not honoured, except ``__init__`` and + ``__del__``. This layout makes the number of types to take care about quite limited. diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py --- a/pypy/doc/conf.py +++ b/pypy/doc/conf.py @@ -197,3 +197,10 @@ # Example configuration for intersphinx: refer to the Python standard library. intersphinx_mapping = {'http://docs.python.org/': None} +# -- Options for manpage output------------------------------------------------- + +man_pages = [ + ('man/pypy.1', 'pypy', + u'fast, compliant alternative implementation of the Python language', + u'The PyPy Project', 1) +] diff --git a/pypy/doc/extradoc.rst b/pypy/doc/extradoc.rst --- a/pypy/doc/extradoc.rst +++ b/pypy/doc/extradoc.rst @@ -8,6 +8,9 @@ *Articles about PyPy published so far, most recent first:* (bibtex_ file) +* `Runtime Feedback in a Meta-Tracing JIT for Efficient Dynamic Languages`_, + C.F. Bolz, A. Cuni, M. Fijalkowski, M. Leuschel, S. Pedroni, A. Rigo + * `Allocation Removal by Partial Evaluation in a Tracing JIT`_, C.F. Bolz, A. Cuni, M. Fijalkowski, M. Leuschel, S. Pedroni, A. Rigo @@ -50,6 +53,9 @@ *Other research using PyPy (as far as we know it):* +* `Hardware Transactional Memory Support for Lightweight Dynamic Language Evolution`_, + N. Riley and C. Zilles + * `PyGirl: Generating Whole-System VMs from High-Level Prototypes using PyPy`_, C. Bruni and T. Verwaest @@ -65,6 +71,7 @@ .. _bibtex: https://bitbucket.org/pypy/extradoc/raw/tip/talk/bibtex.bib +.. _`Runtime Feedback in a Meta-Tracing JIT for Efficient Dynamic Languages`: https://bitbucket.org/pypy/extradoc/raw/extradoc/talk/icooolps2011/jit-hints.pdf .. _`Allocation Removal by Partial Evaluation in a Tracing JIT`: http://codespeak.net/svn/pypy/extradoc/talk/pepm2011/bolz-allocation-removal.pdf .. _`Towards a Jitting VM for Prolog Execution`: http://www.stups.uni-duesseldorf.de/publications/bolz-prolog-jit.pdf .. _`High performance implementation of Python for CLI/.NET with JIT compiler generation for dynamic languages`: http://buildbot.pypy.org/misc/antocuni-thesis.pdf @@ -74,6 +81,7 @@ .. _`Automatic JIT Compiler Generation with Runtime Partial Evaluation`: http://www.stups.uni-duesseldorf.de/thesis/final-master.pdf .. _`RPython: A Step towards Reconciling Dynamically and Statically Typed OO Languages`: http://www.disi.unige.it/person/AnconaD/papers/Recent_abstracts.html#AACM-DLS07 .. _`EU Reports`: index-report.html +.. _`Hardware Transactional Memory Support for Lightweight Dynamic Language Evolution`: http://sabi.net/nriley/pubs/dls6-riley.pdf .. _`PyGirl: Generating Whole-System VMs from High-Level Prototypes using PyPy`: http://scg.unibe.ch/archive/papers/Brun09cPyGirl.pdf .. _`Representation-Based Just-in-Time Specialization and the Psyco Prototype for Python`: http://psyco.sourceforge.net/psyco-pepm-a.ps.gz .. _`Back to the Future in One Week -- Implementing a Smalltalk VM in PyPy`: http://dx.doi.org/10.1007/978-3-540-89275-5_7 diff --git a/pypy/doc/man/pypy.1.rst b/pypy/doc/man/pypy.1.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/man/pypy.1.rst @@ -0,0 +1,90 @@ +====== + pypy +====== + +SYNOPSIS +======== + +``pypy`` [*options*] +[``-c`` *cmd*\ \|\ ``-m`` *mod*\ \|\ *file.py*\ \|\ ``-``\ ] +[*arg*\ ...] + +OPTIONS +======= + +-i + Inspect interactively after running script. + +-O + Dummy optimization flag for compatibility with C Python. + +-c *cmd* + Program passed in as CMD (terminates option list). + +-S + Do not ``import site`` on initialization. + +-u + Unbuffered binary ``stdout`` and ``stderr``. + +-h, --help + Show a help message and exit. + +-m *mod* + Library module to be run as a script (terminates option list). + +-W *arg* + Warning control (*arg* is *action*:*message*:*category*:*module*:*lineno*). + +-E + Ignore environment variables (such as ``PYTHONPATH``). + +--version + Print the PyPy version. + +--info + Print translation information about this PyPy executable. + +--jit *arg* + Low level JIT parameters. Format is + *arg*\ ``=``\ *value*\ [``,``\ *arg*\ ``=``\ *value*\ ...] + + ``off`` + Disable the JIT. + + ``threshold=``\ *value* + Number of times a loop has to run for it to become hot. + + ``function_threshold=``\ *value* + Number of times a function must run for it to become traced from + start. + + ``inlining=``\ *value* + Inline python functions or not (``1``/``0``). + + ``loop_longevity=``\ *value* + A parameter controlling how long loops will be kept before being + freed, an estimate. + + ``max_retrace_guards=``\ *value* + Number of extra guards a retrace can cause. + + ``retrace_limit=``\ *value* + How many times we can try retracing before giving up. + + ``trace_eagerness=``\ *value* + Number of times a guard has to fail before we start compiling a + bridge. + + ``trace_limit=``\ *value* + Number of recorded operations before we abort tracing with + ``ABORT_TRACE_TOO_LONG``. + + ``enable_opts=``\ *value* + Optimizations to enabled or ``all``. + Warning, this option is dangerous, and should be avoided. + +SEE ALSO +======== + +**python**\ (1) diff --git a/pypy/doc/tool/makecontributor.py b/pypy/doc/tool/makecontributor.py deleted file mode 100644 --- a/pypy/doc/tool/makecontributor.py +++ /dev/null @@ -1,47 +0,0 @@ -""" - -generates a contributor list - -""" -import py - -# this file is useless, use the following commandline instead: -# hg churn -c -t "{author}" | sed -e 's/ <.*//' - -try: - path = py.std.sys.argv[1] -except IndexError: - print "usage: %s ROOTPATH" %(py.std.sys.argv[0]) - raise SystemExit, 1 - -d = {} - -for logentry in py.path.svnwc(path).log(): - a = logentry.author - if a in d: - d[a] += 1 - else: - d[a] = 1 - -items = d.items() -items.sort(lambda x,y: -cmp(x[1], y[1])) - -import uconf # http://codespeak.net/svn/uconf/dist/uconf - -# Authors that don't want to be listed -excluded = set("anna gintas ignas".split()) -cutoff = 5 # cutoff for authors in the LICENSE file -mark = False -for author, count in items: - if author in excluded: - continue - user = uconf.system.User(author) - try: - realname = user.realname.strip() - except KeyError: - realname = author - if not mark and count < cutoff: - mark = True - print '-'*60 - print " ", realname - #print count, " ", author diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py --- a/pypy/interpreter/argument.py +++ b/pypy/interpreter/argument.py @@ -428,8 +428,8 @@ return self._match_signature(w_firstarg, scope_w, signature, defaults_w, 0) except ArgErr, e: - raise OperationError(self.space.w_TypeError, - self.space.wrap(e.getmsg(fnname))) + raise operationerrfmt(self.space.w_TypeError, + "%s() %s", fnname, e.getmsg()) def _parse(self, w_firstarg, signature, defaults_w, blindargs=0): """Parse args and kwargs according to the signature of a code object, @@ -450,8 +450,8 @@ try: return self._parse(w_firstarg, signature, defaults_w, blindargs) except ArgErr, e: - raise OperationError(self.space.w_TypeError, - self.space.wrap(e.getmsg(fnname))) + raise operationerrfmt(self.space.w_TypeError, + "%s() %s", fnname, e.getmsg()) @staticmethod def frompacked(space, w_args=None, w_kwds=None): @@ -626,7 +626,7 @@ class ArgErr(Exception): - def getmsg(self, fnname): + def getmsg(self): raise NotImplementedError class ArgErrCount(ArgErr): @@ -642,11 +642,10 @@ self.num_args = got_nargs self.num_kwds = nkwds - def getmsg(self, fnname): + def getmsg(self): n = self.expected_nargs if n == 0: - msg = "%s() takes no arguments (%d given)" % ( - fnname, + msg = "takes no arguments (%d given)" % ( self.num_args + self.num_kwds) else: defcount = self.num_defaults @@ -672,8 +671,7 @@ msg2 = " non-keyword" else: msg2 = "" - msg = "%s() takes %s %d%s argument%s (%d given)" % ( - fnname, + msg = "takes %s %d%s argument%s (%d given)" % ( msg1, n, msg2, @@ -686,9 +684,8 @@ def __init__(self, argname): self.argname = argname - def getmsg(self, fnname): - msg = "%s() got multiple values for keyword argument '%s'" % ( - fnname, + def getmsg(self): + msg = "got multiple values for keyword argument '%s'" % ( self.argname) return msg @@ -722,13 +719,11 @@ break self.kwd_name = name - def getmsg(self, fnname): + def getmsg(self): if self.num_kwds == 1: - msg = "%s() got an unexpected keyword argument '%s'" % ( - fnname, + msg = "got an unexpected keyword argument '%s'" % ( self.kwd_name) else: - msg = "%s() got %d unexpected keyword arguments" % ( - fnname, + msg = "got %d unexpected keyword arguments" % ( self.num_kwds) return msg diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -1591,12 +1591,15 @@ 'ArithmeticError', 'AssertionError', 'AttributeError', + 'BaseException', + 'DeprecationWarning', 'EOFError', 'EnvironmentError', 'Exception', 'FloatingPointError', 'IOError', 'ImportError', + 'ImportWarning', 'IndentationError', 'IndexError', 'KeyError', @@ -1617,7 +1620,10 @@ 'TabError', 'TypeError', 'UnboundLocalError', + 'UnicodeDecodeError', 'UnicodeError', + 'UnicodeEncodeError', + 'UnicodeTranslateError', 'ValueError', 'ZeroDivisionError', 'UnicodeEncodeError', diff --git a/pypy/interpreter/eval.py b/pypy/interpreter/eval.py --- a/pypy/interpreter/eval.py +++ b/pypy/interpreter/eval.py @@ -2,7 +2,6 @@ This module defines the abstract base classes that support execution: Code and Frame. """ -from pypy.rlib import jit from pypy.interpreter.error import OperationError from pypy.interpreter.baseobjspace import Wrappable @@ -98,7 +97,6 @@ "Abstract. Get the expected number of locals." raise TypeError, "abstract" - @jit.dont_look_inside def fast2locals(self): # Copy values from the fastlocals to self.w_locals if self.w_locals is None: @@ -112,7 +110,6 @@ w_name = self.space.wrap(name) self.space.setitem(self.w_locals, w_name, w_value) - @jit.dont_look_inside def locals2fast(self): # Copy values from self.w_locals to the fastlocals assert self.w_locals is not None diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py --- a/pypy/interpreter/gateway.py +++ b/pypy/interpreter/gateway.py @@ -619,7 +619,8 @@ self.descr_reqcls, args) except Exception, e: - raise self.handle_exception(space, e) + self.handle_exception(space, e) + w_result = None if w_result is None: w_result = space.w_None return w_result @@ -655,7 +656,8 @@ self.descr_reqcls, args) except Exception, e: - raise self.handle_exception(space, e) + self.handle_exception(space, e) + w_result = None if w_result is None: w_result = space.w_None return w_result @@ -674,7 +676,8 @@ self.descr_reqcls, args.prepend(w_obj)) except Exception, e: - raise self.handle_exception(space, e) + self.handle_exception(space, e) + w_result = None if w_result is None: w_result = space.w_None return w_result @@ -690,7 +693,8 @@ raise OperationError(space.w_SystemError, space.wrap("unexpected DescrMismatch error")) except Exception, e: - raise self.handle_exception(space, e) + self.handle_exception(space, e) + w_result = None if w_result is None: w_result = space.w_None return w_result @@ -708,7 +712,8 @@ self.descr_reqcls, Arguments(space, [w1])) except Exception, e: - raise self.handle_exception(space, e) + self.handle_exception(space, e) + w_result = None if w_result is None: w_result = space.w_None return w_result @@ -726,7 +731,8 @@ self.descr_reqcls, Arguments(space, [w1, w2])) except Exception, e: - raise self.handle_exception(space, e) + self.handle_exception(space, e) + w_result = None if w_result is None: w_result = space.w_None return w_result @@ -744,7 +750,8 @@ self.descr_reqcls, Arguments(space, [w1, w2, w3])) except Exception, e: - raise self.handle_exception(space, e) + self.handle_exception(space, e) + w_result = None if w_result is None: w_result = space.w_None return w_result @@ -763,7 +770,8 @@ Arguments(space, [w1, w2, w3, w4])) except Exception, e: - raise self.handle_exception(space, e) + self.handle_exception(space, e) + w_result = None if w_result is None: w_result = space.w_None return w_result diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py --- a/pypy/interpreter/generator.py +++ b/pypy/interpreter/generator.py @@ -162,7 +162,8 @@ # generate 2 versions of the function and 2 jit drivers. def _create_unpack_into(): jitdriver = jit.JitDriver(greens=['pycode'], - reds=['self', 'frame', 'results']) + reds=['self', 'frame', 'results'], + name='unpack_into') def unpack_into(self, results): """This is a hack for performance: runs the generator and collects all produced items in a list.""" @@ -196,4 +197,4 @@ self.frame = None return unpack_into unpack_into = _create_unpack_into() - unpack_into_w = _create_unpack_into() \ No newline at end of file + unpack_into_w = _create_unpack_into() diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py --- a/pypy/interpreter/test/test_argument.py +++ b/pypy/interpreter/test/test_argument.py @@ -393,8 +393,8 @@ class FakeArgErr(ArgErr): - def getmsg(self, fname): - return "msg "+fname + def getmsg(self): + return "msg" def _match_signature(*args): raise FakeArgErr() @@ -404,7 +404,7 @@ excinfo = py.test.raises(OperationError, args.parse_obj, "obj", "foo", Signature(["a", "b"], None, None)) assert excinfo.value.w_type is TypeError - assert excinfo.value._w_value == "msg foo" + assert excinfo.value.get_w_value(space) == "foo() msg" def test_args_parsing_into_scope(self): @@ -448,8 +448,8 @@ class FakeArgErr(ArgErr): - def getmsg(self, fname): - return "msg "+fname + def getmsg(self): + return "msg" def _match_signature(*args): raise FakeArgErr() @@ -460,7 +460,7 @@ "obj", [None, None], "foo", Signature(["a", "b"], None, None)) assert excinfo.value.w_type is TypeError - assert excinfo.value._w_value == "msg foo" + assert excinfo.value.get_w_value(space) == "foo() msg" def test_topacked_frompacked(self): space = DummySpace() @@ -493,35 +493,35 @@ # got_nargs, nkwds, expected_nargs, has_vararg, has_kwarg, # defaults_w, missing_args err = ArgErrCount(1, 0, 0, False, False, None, 0) - s = err.getmsg('foo') - assert s == "foo() takes no arguments (1 given)" + s = err.getmsg() + assert s == "takes no arguments (1 given)" err = ArgErrCount(0, 0, 1, False, False, [], 1) - s = err.getmsg('foo') - assert s == "foo() takes exactly 1 argument (0 given)" + s = err.getmsg() + assert s == "takes exactly 1 argument (0 given)" err = ArgErrCount(3, 0, 2, False, False, [], 0) - s = err.getmsg('foo') - assert s == "foo() takes exactly 2 arguments (3 given)" + s = err.getmsg() + assert s == "takes exactly 2 arguments (3 given)" err = ArgErrCount(3, 0, 2, False, False, ['a'], 0) - s = err.getmsg('foo') - assert s == "foo() takes at most 2 arguments (3 given)" + s = err.getmsg() + assert s == "takes at most 2 arguments (3 given)" err = ArgErrCount(1, 0, 2, True, False, [], 1) - s = err.getmsg('foo') - assert s == "foo() takes at least 2 arguments (1 given)" + s = err.getmsg() + assert s == "takes at least 2 arguments (1 given)" err = ArgErrCount(0, 1, 2, True, False, ['a'], 1) - s = err.getmsg('foo') - assert s == "foo() takes at least 1 non-keyword argument (0 given)" + s = err.getmsg() + assert s == "takes at least 1 non-keyword argument (0 given)" err = ArgErrCount(2, 1, 1, False, True, [], 0) - s = err.getmsg('foo') - assert s == "foo() takes exactly 1 non-keyword argument (2 given)" + s = err.getmsg() + assert s == "takes exactly 1 non-keyword argument (2 given)" err = ArgErrCount(0, 1, 1, False, True, [], 1) - s = err.getmsg('foo') - assert s == "foo() takes exactly 1 non-keyword argument (0 given)" + s = err.getmsg() + assert s == "takes exactly 1 non-keyword argument (0 given)" err = ArgErrCount(0, 1, 1, True, True, [], 1) - s = err.getmsg('foo') - assert s == "foo() takes at least 1 non-keyword argument (0 given)" + s = err.getmsg() + assert s == "takes at least 1 non-keyword argument (0 given)" err = ArgErrCount(2, 1, 1, False, True, ['a'], 0) - s = err.getmsg('foo') - assert s == "foo() takes at most 1 non-keyword argument (2 given)" + s = err.getmsg() + assert s == "takes at most 1 non-keyword argument (2 given)" def test_bad_type_for_star(self): space = self.space @@ -543,12 +543,12 @@ def test_unknown_keywords(self): space = DummySpace() err = ArgErrUnknownKwds(space, 1, ['a', 'b'], [True, False], None) - s = err.getmsg('foo') - assert s == "foo() got an unexpected keyword argument 'b'" + s = err.getmsg() + assert s == "got an unexpected keyword argument 'b'" err = ArgErrUnknownKwds(space, 2, ['a', 'b', 'c'], [True, False, False], None) - s = err.getmsg('foo') - assert s == "foo() got 2 unexpected keyword arguments" + s = err.getmsg() + assert s == "got 2 unexpected keyword arguments" def test_unknown_unicode_keyword(self): class DummySpaceUnicode(DummySpace): @@ -558,13 +558,13 @@ err = ArgErrUnknownKwds(space, 1, ['a', None, 'b', 'c'], [True, False, True, True], [unichr(0x1234), u'b', u'c']) - s = err.getmsg('foo') - assert s == "foo() got an unexpected keyword argument '\xe1\x88\xb4'" + s = err.getmsg() + assert s == "got an unexpected keyword argument '\xe1\x88\xb4'" def test_multiple_values(self): err = ArgErrMultipleValues('bla') - s = err.getmsg('foo') - assert s == "foo() got multiple values for keyword argument 'bla'" + s = err.getmsg() + assert s == "got multiple values for keyword argument 'bla'" class AppTestArgument: def test_error_message(self): diff --git a/pypy/jit/backend/llsupport/test/test_runner.py b/pypy/jit/backend/llsupport/test/test_runner.py --- a/pypy/jit/backend/llsupport/test/test_runner.py +++ b/pypy/jit/backend/llsupport/test/test_runner.py @@ -8,6 +8,12 @@ class MyLLCPU(AbstractLLCPU): supports_floats = True + + class assembler(object): + @staticmethod + def set_debug(flag): + pass + def compile_loop(self, inputargs, operations, looptoken): py.test.skip("llsupport test: cannot compile operations") diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py --- a/pypy/jit/backend/test/runner_test.py +++ b/pypy/jit/backend/test/runner_test.py @@ -17,6 +17,7 @@ from pypy.rpython.llinterp import LLException from pypy.jit.codewriter import heaptracker, longlong from pypy.rlib.rarithmetic import intmask +from pypy.jit.backend.detect_cpu import autodetect_main_model_and_size def boxfloat(x): return BoxFloat(longlong.getfloatstorage(x)) @@ -27,6 +28,9 @@ class Runner(object): + add_loop_instruction = ['overload for a specific cpu'] + bridge_loop_instruction = ['overload for a specific cpu'] + def execute_operation(self, opname, valueboxes, result_type, descr=None): inputargs, operations = self._get_single_operation_list(opname, result_type, @@ -2974,6 +2978,56 @@ res = self.cpu.get_latest_value_int(0) assert res == -10 + def test_compile_asmlen(self): + from pypy.jit.backend.llsupport.llmodel import AbstractLLCPU + if not isinstance(self.cpu, AbstractLLCPU): + py.test.skip("pointless test on non-asm") + from pypy.jit.backend.x86.tool.viewcode import machine_code_dump + import ctypes + ops = """ + [i2] + i0 = same_as(i2) # but forced to be in a register + label(i0, descr=1) + i1 = int_add(i0, i0) + guard_true(i1, descr=faildesr) [i1] + jump(i1, descr=1) + """ + faildescr = BasicFailDescr(2) + loop = parse(ops, self.cpu, namespace=locals()) + faildescr = loop.operations[-2].getdescr() + jumpdescr = loop.operations[-1].getdescr() + bridge_ops = """ + [i0] + jump(i0, descr=jumpdescr) + """ + bridge = parse(bridge_ops, self.cpu, namespace=locals()) + looptoken = JitCellToken() + self.cpu.assembler.set_debug(False) + info = self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken) + bridge_info = self.cpu.compile_bridge(faildescr, bridge.inputargs, + bridge.operations, + looptoken) + self.cpu.assembler.set_debug(True) # always on untranslated + assert info.asmlen != 0 + cpuname = autodetect_main_model_and_size() + # XXX we have to check the precise assembler, otherwise + # we don't quite know if borders are correct + + def checkops(mc, ops): + assert len(mc) == len(ops) + for i in range(len(mc)): + assert mc[i].split("\t")[-1].startswith(ops[i]) + + data = ctypes.string_at(info.asmaddr, info.asmlen) + mc = list(machine_code_dump(data, info.asmaddr, cpuname)) + lines = [line for line in mc if line.count('\t') == 2] + checkops(lines, self.add_loop_instructions) + data = ctypes.string_at(bridge_info.asmaddr, bridge_info.asmlen) + mc = list(machine_code_dump(data, bridge_info.asmaddr, cpuname)) + lines = [line for line in mc if line.count('\t') == 2] + checkops(lines, self.bridge_loop_instructions) + + def test_compile_bridge_with_target(self): # This test creates a loopy piece of code in a bridge, and builds another # unrelated loop that ends in a jump directly to this loopy bit of code. diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py --- a/pypy/jit/backend/x86/assembler.py +++ b/pypy/jit/backend/x86/assembler.py @@ -7,6 +7,7 @@ from pypy.rpython.lltypesystem import lltype, rffi, rstr, llmemory from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.annlowlevel import llhelper +from pypy.rlib.jit import AsmInfo from pypy.jit.backend.model import CompiledLoopToken from pypy.jit.backend.x86.regalloc import (RegAlloc, get_ebp_ofs, _get_scale, gpr_reg_mgr_cls, _valid_addressing_size) @@ -39,6 +40,7 @@ from pypy.jit.codewriter.effectinfo import EffectInfo from pypy.jit.codewriter import longlong from pypy.rlib.rarithmetic import intmask +from pypy.rlib.objectmodel import compute_unique_id # darwin requires the stack to be 16 bytes aligned on calls. Same for gcc 4.5.0, # better safe than sorry @@ -58,7 +60,8 @@ self.is_guard_not_invalidated = is_guard_not_invalidated DEBUG_COUNTER = lltype.Struct('DEBUG_COUNTER', ('i', lltype.Signed), - ('bridge', lltype.Signed), # 0 or 1 + ('type', lltype.Char), # 'b'ridge, 'l'abel or + # 'e'ntry point ('number', lltype.Signed)) class Assembler386(object): @@ -147,12 +150,15 @@ def finish_once(self): if self._debug: debug_start('jit-backend-counts') - for struct in self.loop_run_counters: - if struct.bridge: - prefix = 'bridge ' + for i in range(len(self.loop_run_counters)): + struct = self.loop_run_counters[i] + if struct.type == 'l': + prefix = 'TargetToken(%d)' % struct.number + elif struct.type == 'b': + prefix = 'bridge ' + str(struct.number) else: - prefix = 'loop ' - debug_print(prefix + str(struct.number) + ':' + str(struct.i)) + prefix = 'entry ' + str(struct.number) + debug_print(prefix + ':' + str(struct.i)) debug_stop('jit-backend-counts') def _build_float_constants(self): @@ -406,6 +412,7 @@ '''adds the following attributes to looptoken: _x86_function_addr (address of the generated func, as an int) _x86_loop_code (debug: addr of the start of the ResOps) + _x86_fullsize (debug: full size including failure) _x86_debug_checksum ''' # XXX this function is too longish and contains some code @@ -422,8 +429,8 @@ self.setup(looptoken) if log: - self._register_counter(False, looptoken.number) - operations = self._inject_debugging_code(looptoken, operations) + operations = self._inject_debugging_code(looptoken, operations, + 'e', looptoken.number) regalloc = RegAlloc(self, self.cpu.translate_support_code) # @@ -471,7 +478,8 @@ name = "Loop # %s: %s" % (looptoken.number, loopname) self.cpu.profile_agent.native_code_written(name, rawstart, full_size) - return ops_offset + return AsmInfo(ops_offset, rawstart + looppos, + size_excluding_failure_stuff - looppos) def assemble_bridge(self, faildescr, inputargs, operations, original_loop_token, log): @@ -480,17 +488,12 @@ assert len(set(inputargs)) == len(inputargs) descr_number = self.cpu.get_fail_descr_number(faildescr) - try: - failure_recovery = self._find_failure_recovery_bytecode(faildescr) - except ValueError: - debug_print("Bridge out of guard", descr_number, - "was already compiled!") - return + failure_recovery = self._find_failure_recovery_bytecode(faildescr) self.setup(original_loop_token) if log: - self._register_counter(True, descr_number) - operations = self._inject_debugging_code(faildescr, operations) + operations = self._inject_debugging_code(faildescr, operations, + 'b', descr_number) arglocs = self.rebuild_faillocs_from_descr(failure_recovery) if not we_are_translated(): @@ -498,6 +501,7 @@ [loc.assembler() for loc in faildescr._x86_debug_faillocs]) regalloc = RegAlloc(self, self.cpu.translate_support_code) fail_depths = faildescr._x86_current_depths + startpos = self.mc.get_relative_pos() operations = regalloc.prepare_bridge(fail_depths, inputargs, arglocs, operations, self.current_clt.allgcrefs) @@ -532,7 +536,7 @@ name = "Bridge # %s" % (descr_number,) self.cpu.profile_agent.native_code_written(name, rawstart, fullsize) - return ops_offset + return AsmInfo(ops_offset, startpos + rawstart, codeendpos - startpos) def write_pending_failure_recoveries(self): # for each pending guard, generate the code of the recovery stub @@ -597,22 +601,29 @@ return self.mc.materialize(self.cpu.asmmemmgr, allblocks, self.cpu.gc_ll_descr.gcrootmap) - def _register_counter(self, bridge, number): - if self._debug: - # YYY very minor leak -- we need the counters to stay alive - # forever, just because we want to report them at the end - # of the process - struct = lltype.malloc(DEBUG_COUNTER, flavor='raw', - track_allocation=False) - struct.i = 0 - struct.bridge = int(bridge) + def _register_counter(self, tp, number, token): + # YYY very minor leak -- we need the counters to stay alive + # forever, just because we want to report them at the end + # of the process + struct = lltype.malloc(DEBUG_COUNTER, flavor='raw', + track_allocation=False) + struct.i = 0 + struct.type = tp + if tp == 'b' or tp == 'e': struct.number = number - self.loop_run_counters.append(struct) + else: + assert token + struct.number = compute_unique_id(token) + self.loop_run_counters.append(struct) + return struct def _find_failure_recovery_bytecode(self, faildescr): adr_jump_offset = faildescr._x86_adr_jump_offset if adr_jump_offset == 0: - raise ValueError + # This case should be prevented by the logic in compile.py: + # look for CNT_BUSY_FLAG, which disables tracing from a guard + # when another tracing from the same guard is already in progress. + raise BridgeAlreadyCompiled # follow the JMP/Jcond p = rffi.cast(rffi.INTP, adr_jump_offset) adr_target = adr_jump_offset + 4 + rffi.cast(lltype.Signed, p[0]) @@ -651,27 +662,36 @@ targettoken._x86_loop_code += rawstart self.target_tokens_currently_compiling = None + def _append_debugging_code(self, operations, tp, number, token): + counter = self._register_counter(tp, number, token) + c_adr = ConstInt(rffi.cast(lltype.Signed, counter)) + box = BoxInt() + box2 = BoxInt() + ops = [ResOperation(rop.GETFIELD_RAW, [c_adr], + box, descr=self.debug_counter_descr), + ResOperation(rop.INT_ADD, [box, ConstInt(1)], box2), + ResOperation(rop.SETFIELD_RAW, [c_adr, box2], + None, descr=self.debug_counter_descr)] + operations.extend(ops) + @specialize.argtype(1) - def _inject_debugging_code(self, looptoken, operations): + def _inject_debugging_code(self, looptoken, operations, tp, number): if self._debug: # before doing anything, let's increase a counter s = 0 for op in operations: s += op.getopnum() looptoken._x86_debug_checksum = s - c_adr = ConstInt(rffi.cast(lltype.Signed, - self.loop_run_counters[-1])) - box = BoxInt() - box2 = BoxInt() - ops = [ResOperation(rop.GETFIELD_RAW, [c_adr], - box, descr=self.debug_counter_descr), - ResOperation(rop.INT_ADD, [box, ConstInt(1)], box2), - ResOperation(rop.SETFIELD_RAW, [c_adr, box2], - None, descr=self.debug_counter_descr)] - if operations[0].getopnum() == rop.LABEL: - operations = [operations[0]] + ops + operations[1:] - else: - operations = ops + operations + + newoperations = [] + self._append_debugging_code(newoperations, tp, number, + None) + for op in operations: + newoperations.append(op) + if op.getopnum() == rop.LABEL: + self._append_debugging_code(newoperations, 'l', number, + op.getdescr()) + operations = newoperations return operations def _assemble(self, regalloc, operations): @@ -792,7 +812,10 @@ target = newlooptoken._x86_function_addr mc = codebuf.MachineCodeBlockWrapper() mc.JMP(imm(target)) - assert mc.get_relative_pos() <= 13 # keep in sync with prepare_loop() + if WORD == 4: # keep in sync with prepare_loop() + assert mc.get_relative_pos() == 5 + else: + assert mc.get_relative_pos() <= 13 mc.copy_to_raw_memory(oldadr) def dump(self, text): @@ -2532,3 +2555,6 @@ def not_implemented(msg): os.write(2, '[x86/asm] %s\n' % msg) raise NotImplementedError(msg) + +class BridgeAlreadyCompiled(Exception): + pass diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py --- a/pypy/jit/backend/x86/regalloc.py +++ b/pypy/jit/backend/x86/regalloc.py @@ -188,7 +188,10 @@ # note: we need to make a copy of inputargs because possibly_free_vars # is also used on op args, which is a non-resizable list self.possibly_free_vars(list(inputargs)) - self.min_bytes_before_label = 13 + if WORD == 4: # see redirect_call_assembler() + self.min_bytes_before_label = 5 + else: + self.min_bytes_before_label = 13 return operations def prepare_bridge(self, prev_depths, inputargs, arglocs, operations, @@ -741,7 +744,7 @@ self.xrm.possibly_free_var(op.getarg(0)) def consider_cast_int_to_float(self, op): - loc0 = self.rm.loc(op.getarg(0)) + loc0 = self.rm.make_sure_var_in_reg(op.getarg(0)) loc1 = self.xrm.force_allocate_reg(op.result) self.Perform(op, [loc0], loc1) self.rm.possibly_free_var(op.getarg(0)) diff --git a/pypy/jit/backend/x86/test/test_runner.py b/pypy/jit/backend/x86/test/test_runner.py --- a/pypy/jit/backend/x86/test/test_runner.py +++ b/pypy/jit/backend/x86/test/test_runner.py @@ -33,6 +33,13 @@ # for the individual tests see # ====> ../../test/runner_test.py + add_loop_instructions = ['mov', 'add', 'test', 'je', 'jmp'] + if WORD == 4: + bridge_loop_instructions = ['lea', 'jmp'] + else: + # the 'mov' is part of the 'jmp' so far + bridge_loop_instructions = ['lea', 'mov', 'jmp'] + def setup_method(self, meth): self.cpu = CPU(rtyper=None, stats=FakeStats()) self.cpu.setup_once() @@ -416,12 +423,13 @@ ] inputargs = [i0] debug._log = dlog = debug.DebugLog() - ops_offset = self.cpu.compile_loop(inputargs, operations, looptoken) + info = self.cpu.compile_loop(inputargs, operations, looptoken) + ops_offset = info.ops_offset debug._log = None # assert ops_offset is looptoken._x86_ops_offset - # getfield_raw/int_add/setfield_raw + ops + None - assert len(ops_offset) == 3 + len(operations) + 1 + # 2*(getfield_raw/int_add/setfield_raw) + ops + None + assert len(ops_offset) == 2*3 + len(operations) + 1 assert (ops_offset[operations[0]] <= ops_offset[operations[1]] <= ops_offset[operations[2]] <= @@ -519,6 +527,7 @@ from pypy.tool.logparser import parse_log_file, extract_category from pypy.rlib import debug + targettoken, preambletoken = TargetToken(), TargetToken() loop = """ [i0] label(i0, descr=preambletoken) @@ -533,8 +542,8 @@ guard_false(i12) [] jump(i11, descr=targettoken) """ - ops = parse(loop, namespace={'targettoken': TargetToken(), - 'preambletoken': TargetToken()}) + ops = parse(loop, namespace={'targettoken': targettoken, + 'preambletoken': preambletoken}) debug._log = dlog = debug.DebugLog() try: self.cpu.assembler.set_debug(True) @@ -545,11 +554,16 @@ struct = self.cpu.assembler.loop_run_counters[0] assert struct.i == 1 struct = self.cpu.assembler.loop_run_counters[1] - assert struct.i == 10 + assert struct.i == 1 + struct = self.cpu.assembler.loop_run_counters[2] + assert struct.i == 9 self.cpu.finish_once() finally: debug._log = None - assert ('jit-backend-counts', [('debug_print', 'loop -1:10')]) in dlog + l0 = ('debug_print', 'entry -1:1') + l1 = ('debug_print', preambletoken.repr_of_descr() + ':1') + l2 = ('debug_print', targettoken.repr_of_descr() + ':9') + assert ('jit-backend-counts', [l0, l1, l2]) in dlog def test_debugger_checksum(self): loop = """ diff --git a/pypy/jit/backend/x86/test/test_zrpy_platform.py b/pypy/jit/backend/x86/test/test_zrpy_platform.py --- a/pypy/jit/backend/x86/test/test_zrpy_platform.py +++ b/pypy/jit/backend/x86/test/test_zrpy_platform.py @@ -74,8 +74,8 @@ myjitdriver = jit.JitDriver(greens = [], reds = ['n']) def entrypoint(argv): - myjitdriver.set_param('threshold', 2) - myjitdriver.set_param('trace_eagerness', 0) + jit.set_param(myjitdriver, 'threshold', 2) + jit.set_param(myjitdriver, 'trace_eagerness', 0) n = 16 while n > 0: myjitdriver.can_enter_jit(n=n) diff --git a/pypy/jit/backend/x86/tool/viewcode.py b/pypy/jit/backend/x86/tool/viewcode.py --- a/pypy/jit/backend/x86/tool/viewcode.py +++ b/pypy/jit/backend/x86/tool/viewcode.py @@ -39,6 +39,7 @@ def machine_code_dump(data, originaddr, backend_name, label_list=None): objdump_backend_option = { 'x86': 'i386', + 'x86_32': 'i386', 'x86_64': 'x86-64', 'i386': 'i386', } diff --git a/pypy/jit/codewriter/call.py b/pypy/jit/codewriter/call.py --- a/pypy/jit/codewriter/call.py +++ b/pypy/jit/codewriter/call.py @@ -42,8 +42,7 @@ except AttributeError: pass - def is_candidate(graph): - return policy.look_inside_graph(graph) + is_candidate = policy.look_inside_graph assert len(self.jitdrivers_sd) > 0 todo = [jd.portal_graph for jd in self.jitdrivers_sd] diff --git a/pypy/jit/codewriter/policy.py b/pypy/jit/codewriter/policy.py --- a/pypy/jit/codewriter/policy.py +++ b/pypy/jit/codewriter/policy.py @@ -8,11 +8,15 @@ class JitPolicy(object): - def __init__(self): + def __init__(self, jithookiface=None): self.unsafe_loopy_graphs = set() self.supports_floats = False self.supports_longlong = False self.supports_singlefloats = False + if jithookiface is None: + from pypy.rlib.jit import JitHookInterface + jithookiface = JitHookInterface() + self.jithookiface = jithookiface def set_supports_floats(self, flag): self.supports_floats = flag diff --git a/pypy/jit/codewriter/support.py b/pypy/jit/codewriter/support.py --- a/pypy/jit/codewriter/support.py +++ b/pypy/jit/codewriter/support.py @@ -162,7 +162,6 @@ _ll_4_list_setslice = rlist.ll_listsetslice _ll_2_list_delslice_startonly = rlist.ll_listdelslice_startonly _ll_3_list_delslice_startstop = rlist.ll_listdelslice_startstop -_ll_1_list_list2fixed = lltypesystem_rlist.ll_list2fixed _ll_2_list_inplace_mul = rlist.ll_inplace_mul _ll_2_list_getitem_foldable = _ll_2_list_getitem diff --git a/pypy/jit/metainterp/compile.py b/pypy/jit/metainterp/compile.py --- a/pypy/jit/metainterp/compile.py +++ b/pypy/jit/metainterp/compile.py @@ -5,6 +5,7 @@ from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.debug import debug_start, debug_stop, debug_print from pypy.rlib import rstack +from pypy.rlib.jit import JitDebugInfo from pypy.conftest import option from pypy.tool.sourcetools import func_with_new_name @@ -75,7 +76,7 @@ if descr is not original_jitcell_token: original_jitcell_token.record_jump_to(descr) descr.exported_state = None - op._descr = None # clear reference, mostly for tests + op.cleardescr() # clear reference, mostly for tests elif isinstance(descr, TargetToken): # for a JUMP: record it as a potential jump. # (the following test is not enough to prevent more complicated @@ -90,8 +91,8 @@ assert descr.exported_state is None if not we_are_translated(): op._descr_wref = weakref.ref(op._descr) - op._descr = None # clear reference to prevent the history.Stats - # from keeping the loop alive during tests + op.cleardescr() # clear reference to prevent the history.Stats + # from keeping the loop alive during tests # record this looptoken on the QuasiImmut used in the code if loop.quasi_immutable_deps is not None: for qmut in loop.quasi_immutable_deps: @@ -112,33 +113,26 @@ """ from pypy.jit.metainterp.optimizeopt import optimize_trace - history = metainterp.history metainterp_sd = metainterp.staticdata jitdriver_sd = metainterp.jitdriver_sd + history = metainterp.history - if False: - part = partial_trace - assert False - procedur_token = metainterp.get_procedure_token(greenkey) - assert procedure_token - all_target_tokens = [] - else: - jitcell_token = make_jitcell_token(jitdriver_sd) - part = create_empty_loop(metainterp) - part.inputargs = inputargs[:] - h_ops = history.operations - part.resume_at_jump_descr = resume_at_jump_descr - part.operations = [ResOperation(rop.LABEL, inputargs, None, descr=TargetToken(jitcell_token))] + \ - [h_ops[i].clone() for i in range(start, len(h_ops))] + \ - [ResOperation(rop.LABEL, jumpargs, None, descr=jitcell_token)] + jitcell_token = make_jitcell_token(jitdriver_sd) + part = create_empty_loop(metainterp) + part.inputargs = inputargs[:] + h_ops = history.operations + part.resume_at_jump_descr = resume_at_jump_descr + part.operations = [ResOperation(rop.LABEL, inputargs, None, descr=TargetToken(jitcell_token))] + \ + [h_ops[i].clone() for i in range(start, len(h_ops))] + \ + [ResOperation(rop.LABEL, jumpargs, None, descr=jitcell_token)] - try: - optimize_trace(metainterp_sd, part, jitdriver_sd.warmstate.enable_opts) - except InvalidLoop: - return None - target_token = part.operations[0].getdescr() - assert isinstance(target_token, TargetToken) - all_target_tokens = [target_token] + try: + optimize_trace(metainterp_sd, part, jitdriver_sd.warmstate.enable_opts) + except InvalidLoop: + return None + target_token = part.operations[0].getdescr() + assert isinstance(target_token, TargetToken) + all_target_tokens = [target_token] loop = create_empty_loop(metainterp) loop.inputargs = part.inputargs @@ -176,10 +170,10 @@ loop.original_jitcell_token = jitcell_token for label in all_target_tokens: assert isinstance(label, TargetToken) - label.original_jitcell_token = jitcell_token if label.virtual_state and label.short_preamble: metainterp_sd.logger_ops.log_short_preamble([], label.short_preamble) jitcell_token.target_tokens = all_target_tokens + propagate_original_jitcell_token(loop) send_loop_to_backend(greenkey, jitdriver_sd, metainterp_sd, loop, "loop") record_loop_or_bridge(metainterp_sd, loop) return all_target_tokens[0] @@ -247,11 +241,11 @@ for box in loop.inputargs: assert isinstance(box, Box) - target_token = loop.operations[-1].getdescr() + target_token = loop.operations[-1].getdescr() resumekey.compile_and_attach(metainterp, loop) + target_token = label.getdescr() assert isinstance(target_token, TargetToken) - target_token.original_jitcell_token = loop.original_jitcell_token record_loop_or_bridge(metainterp_sd, loop) return target_token @@ -288,14 +282,21 @@ assert i == len(inputargs) loop.operations = extra_ops + loop.operations +def propagate_original_jitcell_token(trace): + for op in trace.operations: + if op.getopnum() == rop.LABEL: + token = op.getdescr() + assert isinstance(token, TargetToken) + assert token.original_jitcell_token is None + token.original_jitcell_token = trace.original_jitcell_token + + def send_loop_to_backend(greenkey, jitdriver_sd, metainterp_sd, loop, type): vinfo = jitdriver_sd.virtualizable_info if vinfo is not None: patch_new_loop_to_load_virtualizable_fields(loop, jitdriver_sd) original_jitcell_token = loop.original_jitcell_token - jitdriver_sd.on_compile(metainterp_sd.logger_ops, original_jitcell_token, - loop.operations, type, greenkey) loopname = jitdriver_sd.warmstate.get_location_str(greenkey) globaldata = metainterp_sd.globaldata original_jitcell_token.number = n = globaldata.loopnumbering @@ -305,21 +306,41 @@ show_procedures(metainterp_sd, loop) loop.check_consistency() + if metainterp_sd.warmrunnerdesc is not None: + hooks = metainterp_sd.warmrunnerdesc.hooks + debug_info = JitDebugInfo(jitdriver_sd, metainterp_sd.logger_ops, + original_jitcell_token, loop.operations, + type, greenkey) + hooks.before_compile(debug_info) + else: + debug_info = None + hooks = None operations = get_deep_immutable_oplist(loop.operations) metainterp_sd.profiler.start_backend() debug_start("jit-backend") try: - ops_offset = metainterp_sd.cpu.compile_loop(loop.inputargs, operations, - original_jitcell_token, name=loopname) + asminfo = metainterp_sd.cpu.compile_loop(loop.inputargs, operations, + original_jitcell_token, + name=loopname) finally: debug_stop("jit-backend") metainterp_sd.profiler.end_backend() + if hooks is not None: + debug_info.asminfo = asminfo + hooks.after_compile(debug_info) metainterp_sd.stats.add_new_loop(loop) if not we_are_translated(): metainterp_sd.stats.compiled() metainterp_sd.log("compiled new " + type) # - metainterp_sd.logger_ops.log_loop(loop.inputargs, loop.operations, n, type, ops_offset) + loopname = jitdriver_sd.warmstate.get_location_str(greenkey) + if asminfo is not None: + ops_offset = asminfo.ops_offset + else: + ops_offset = None + metainterp_sd.logger_ops.log_loop(loop.inputargs, loop.operations, n, + type, ops_offset, + name=loopname) # if metainterp_sd.warmrunnerdesc is not None: # for tests metainterp_sd.warmrunnerdesc.memory_manager.keep_loop_alive(original_jitcell_token) @@ -327,25 +348,40 @@ def send_bridge_to_backend(jitdriver_sd, metainterp_sd, faildescr, inputargs, operations, original_loop_token): n = metainterp_sd.cpu.get_fail_descr_number(faildescr) - jitdriver_sd.on_compile_bridge(metainterp_sd.logger_ops, - original_loop_token, operations, n) if not we_are_translated(): show_procedures(metainterp_sd) seen = dict.fromkeys(inputargs) TreeLoop.check_consistency_of_branch(operations, seen) + if metainterp_sd.warmrunnerdesc is not None: + hooks = metainterp_sd.warmrunnerdesc.hooks + debug_info = JitDebugInfo(jitdriver_sd, metainterp_sd.logger_ops, + original_loop_token, operations, 'bridge', + fail_descr_no=n) + hooks.before_compile_bridge(debug_info) + else: + hooks = None + debug_info = None + operations = get_deep_immutable_oplist(operations) metainterp_sd.profiler.start_backend() - operations = get_deep_immutable_oplist(operations) debug_start("jit-backend") try: - ops_offset = metainterp_sd.cpu.compile_bridge(faildescr, inputargs, operations, - original_loop_token) + asminfo = metainterp_sd.cpu.compile_bridge(faildescr, inputargs, + operations, + original_loop_token) finally: debug_stop("jit-backend") metainterp_sd.profiler.end_backend() + if hooks is not None: + debug_info.asminfo = asminfo + hooks.after_compile_bridge(debug_info) if not we_are_translated(): metainterp_sd.stats.compiled() metainterp_sd.log("compiled new bridge") # + if asminfo is not None: + ops_offset = asminfo.ops_offset + else: + ops_offset = None metainterp_sd.logger_ops.log_bridge(inputargs, operations, n, ops_offset) # #if metainterp_sd.warmrunnerdesc is not None: # for tests @@ -558,6 +594,7 @@ inputargs = metainterp.history.inputargs if not we_are_translated(): self._debug_suboperations = new_loop.operations + propagate_original_jitcell_token(new_loop) send_bridge_to_backend(metainterp.jitdriver_sd, metainterp.staticdata, self, inputargs, new_loop.operations, new_loop.original_jitcell_token) @@ -744,6 +781,7 @@ jitdriver_sd = metainterp.jitdriver_sd redargs = new_loop.inputargs new_loop.original_jitcell_token = jitcell_token = make_jitcell_token(jitdriver_sd) + propagate_original_jitcell_token(new_loop) send_loop_to_backend(self.original_greenkey, metainterp.jitdriver_sd, metainterp_sd, new_loop, "entry bridge") # send the new_loop to warmspot.py, to be called directly the next time diff --git a/pypy/jit/metainterp/heapcache.py b/pypy/jit/metainterp/heapcache.py --- a/pypy/jit/metainterp/heapcache.py +++ b/pypy/jit/metainterp/heapcache.py @@ -79,9 +79,9 @@ opnum == rop.COPYSTRCONTENT or opnum == rop.COPYUNICODECONTENT): return - if rop._OVF_FIRST <= opnum <= rop._OVF_LAST: - return - if rop._NOSIDEEFFECT_FIRST <= opnum <= rop._NOSIDEEFFECT_LAST: + if (rop._OVF_FIRST <= opnum <= rop._OVF_LAST or + rop._NOSIDEEFFECT_FIRST <= opnum <= rop._NOSIDEEFFECT_LAST or + rop._GUARD_FIRST <= opnum <= rop._GUARD_LAST): return if opnum == rop.CALL or opnum == rop.CALL_LOOPINVARIANT: effectinfo = descr.get_extra_info() diff --git a/pypy/jit/metainterp/history.py b/pypy/jit/metainterp/history.py --- a/pypy/jit/metainterp/history.py +++ b/pypy/jit/metainterp/history.py @@ -1003,35 +1003,16 @@ return insns def check_simple_loop(self, expected=None, **check): - # Usefull in the simplest case when we have only one trace ending with - # a jump back to itself and possibly a few bridges ending with finnish. - # Only the operations within the loop formed by that single jump will - # be counted. - - # XXX hacked version, ignore and remove me when jit-targets is merged. - loops = self.get_all_loops() - loops = [loop for loop in loops if 'Preamble' not in repr(loop)] #XXX - assert len(loops) == 1 - loop, = loops - jumpop = loop.operations[-1] - assert jumpop.getopnum() == rop.JUMP - insns = {} - for op in loop.operations: - opname = op.getopname() - insns[opname] = insns.get(opname, 0) + 1 - return self._check_insns(insns, expected, check) - - def check_simple_loop(self, expected=None, **check): - # Usefull in the simplest case when we have only one trace ending with - # a jump back to itself and possibly a few bridges ending with finnish. - # Only the operations within the loop formed by that single jump will - # be counted. + """ Usefull in the simplest case when we have only one trace ending with + a jump back to itself and possibly a few bridges. + Only the operations within the loop formed by that single jump will + be counted. + """ loops = self.get_all_loops() assert len(loops) == 1 loop = loops[0] jumpop = loop.operations[-1] assert jumpop.getopnum() == rop.JUMP - assert self.check_resops(jump=1) labels = [op for op in loop.operations if op.getopnum() == rop.LABEL] targets = [op._descr_wref() for op in labels] assert None not in targets # TargetToken was freed, give up diff --git a/pypy/jit/metainterp/jitdriver.py b/pypy/jit/metainterp/jitdriver.py --- a/pypy/jit/metainterp/jitdriver.py +++ b/pypy/jit/metainterp/jitdriver.py @@ -21,7 +21,6 @@ # self.portal_finishtoken... pypy.jit.metainterp.pyjitpl # self.index ... pypy.jit.codewriter.call # self.mainjitcode ... pypy.jit.codewriter.call - # self.on_compile ... pypy.jit.metainterp.warmstate # These attributes are read by the backend in CALL_ASSEMBLER: # self.assembler_helper_adr diff --git a/pypy/jit/metainterp/jitprof.py b/pypy/jit/metainterp/jitprof.py --- a/pypy/jit/metainterp/jitprof.py +++ b/pypy/jit/metainterp/jitprof.py @@ -18,8 +18,8 @@ OPT_FORCINGS ABORT_TOO_LONG ABORT_BRIDGE +ABORT_BAD_LOOP ABORT_ESCAPE -ABORT_BAD_LOOP ABORT_FORCE_QUASIIMMUT NVIRTUALS NVHOLES @@ -30,10 +30,13 @@ TOTAL_FREED_BRIDGES """ +counter_names = [] + def _setup(): names = counters.split() for i, name in enumerate(names): globals()[name] = i + counter_names.append(name) global ncounters ncounters = len(names) _setup() diff --git a/pypy/jit/metainterp/logger.py b/pypy/jit/metainterp/logger.py --- a/pypy/jit/metainterp/logger.py +++ b/pypy/jit/metainterp/logger.py @@ -13,14 +13,14 @@ self.metainterp_sd = metainterp_sd self.guard_number = guard_number - def log_loop(self, inputargs, operations, number=0, type=None, ops_offset=None): + def log_loop(self, inputargs, operations, number=0, type=None, ops_offset=None, name=''): if type is None: debug_start("jit-log-noopt-loop") logops = self._log_operations(inputargs, operations, ops_offset) debug_stop("jit-log-noopt-loop") else: debug_start("jit-log-opt-loop") - debug_print("# Loop", number, ":", type, + debug_print("# Loop", number, '(%s)' % name , ":", type, "with", len(operations), "ops") logops = self._log_operations(inputargs, operations, ops_offset) debug_stop("jit-log-opt-loop") diff --git a/pypy/jit/metainterp/optimizeopt/fficall.py b/pypy/jit/metainterp/optimizeopt/fficall.py --- a/pypy/jit/metainterp/optimizeopt/fficall.py +++ b/pypy/jit/metainterp/optimizeopt/fficall.py @@ -234,11 +234,11 @@ # longlongs are treated as floats, see # e.g. llsupport/descr.py:getDescrClass is_float = True - elif kind == 'u': + elif kind == 'u' or kind == 's': # they're all False pass else: - assert False, "unsupported ffitype or kind" + raise NotImplementedError("unsupported ffitype or kind: %s" % kind) # fieldsize = rffi.getintfield(ffitype, 'c_size') return self.optimizer.cpu.interiorfielddescrof_dynamic( diff --git a/pypy/jit/metainterp/optimizeopt/test/test_multilabel.py b/pypy/jit/metainterp/optimizeopt/test/test_multilabel.py --- a/pypy/jit/metainterp/optimizeopt/test/test_multilabel.py +++ b/pypy/jit/metainterp/optimizeopt/test/test_multilabel.py @@ -1,10 +1,13 @@ from __future__ import with_statement from pypy.jit.metainterp.optimizeopt.test.test_util import ( - LLtypeMixin, BaseTest, Storage, _sortboxes, FakeDescrWithSnapshot) + LLtypeMixin, BaseTest, Storage, _sortboxes, FakeDescrWithSnapshot, + FakeMetaInterpStaticData) from pypy.jit.metainterp.history import TreeLoop, JitCellToken, TargetToken from pypy.jit.metainterp.resoperation import rop, opname, ResOperation from pypy.jit.metainterp.optimize import InvalidLoop from py.test import raises +from pypy.jit.metainterp.optimizeopt.optimizer import Optimization +from pypy.jit.metainterp.optimizeopt.util import make_dispatcher_method class BaseTestMultiLabel(BaseTest): enable_opts = "intbounds:rewrite:virtualize:string:earlyforce:pure:heap:unroll" @@ -84,6 +87,8 @@ return optimized +class OptimizeoptTestMultiLabel(BaseTestMultiLabel): + def test_simple(self): ops = """ [i1] @@ -381,6 +386,82 @@ """ self.optimize_loop(ops, expected) -class TestLLtype(BaseTestMultiLabel, LLtypeMixin): + def test_virtual_as_field_of_forced_box(self): + ops = """ + [p0] + pv1 = new_with_vtable(ConstClass(node_vtable)) + label(pv1, p0) + pv2 = new_with_vtable(ConstClass(node_vtable)) + setfield_gc(pv2, pv1, descr=valuedescr) + jump(pv1, pv2) + """ + with raises(InvalidLoop): + self.optimize_loop(ops, ops) + +class OptRenameStrlen(Optimization): + def propagate_forward(self, op): + dispatch_opt(self, op) + + def optimize_STRLEN(self, op): + newop = op.clone() + newop.result = op.result.clonebox() + self.emit_operation(newop) + self.make_equal_to(op.result, self.getvalue(newop.result)) + +dispatch_opt = make_dispatcher_method(OptRenameStrlen, 'optimize_', + default=OptRenameStrlen.emit_operation) + +class BaseTestOptimizerRenamingBoxes(BaseTestMultiLabel): + + def _do_optimize_loop(self, loop, call_pure_results): + from pypy.jit.metainterp.optimizeopt.unroll import optimize_unroll + from pypy.jit.metainterp.optimizeopt.util import args_dict + from pypy.jit.metainterp.optimizeopt.pure import OptPure + + self.loop = loop + loop.call_pure_results = args_dict() + metainterp_sd = FakeMetaInterpStaticData(self.cpu) + optimize_unroll(metainterp_sd, loop, [OptRenameStrlen(), OptPure()], True) + + def test_optimizer_renaming_boxes(self): + ops = """ + [p1] + i1 = strlen(p1) + label(p1) + i2 = strlen(p1) + i3 = int_add(i2, 7) + jump(p1) + """ + expected = """ + [p1] + i1 = strlen(p1) + label(p1, i1) + i11 = same_as(i1) + i2 = int_add(i11, 7) + jump(p1, i11) + """ + self.optimize_loop(ops, expected) + + def test_optimizer_renaming_boxes_not_imported(self): + ops = """ + [p1] + i1 = strlen(p1) + label(p1) + jump(p1) + """ + expected = """ + [p1] + i1 = strlen(p1) + label(p1, i1) + i11 = same_as(i1) + jump(p1, i11) + """ + self.optimize_loop(ops, expected) + + + +class TestLLtype(OptimizeoptTestMultiLabel, LLtypeMixin): pass +class TestOptimizerRenamingBoxesLLtype(BaseTestOptimizerRenamingBoxes, LLtypeMixin): + pass diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py --- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py +++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py @@ -117,7 +117,7 @@ def optimize_loop(self, ops, optops, call_pure_results=None): loop = self.parse(ops) - token = JitCellToken() + token = JitCellToken() loop.operations = [ResOperation(rop.LABEL, loop.inputargs, None, descr=TargetToken(token))] + \ loop.operations if loop.operations[-1].getopnum() == rop.JUMP: diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -7759,7 +7759,7 @@ jump(i0, p0, i2) """ self.optimize_loop(ops, expected) - + class TestLLtype(OptimizeOptTest, LLtypeMixin): pass diff --git a/pypy/jit/metainterp/optimizeopt/unroll.py b/pypy/jit/metainterp/optimizeopt/unroll.py --- a/pypy/jit/metainterp/optimizeopt/unroll.py +++ b/pypy/jit/metainterp/optimizeopt/unroll.py @@ -265,7 +265,16 @@ self.optimizer.importable_values[value] = imp newvalue = self.optimizer.getvalue(op.result) newresult = newvalue.get_key_box() - assert newresult is op.result or newvalue.is_constant() + # note that emitting here SAME_AS should not happen, but + # in case it does, we would prefer to be suboptimal in asm + # to a fatal RPython exception. + if newresult is not op.result and not newvalue.is_constant(): + op = ResOperation(rop.SAME_AS, [op.result], newresult) + self.optimizer._newoperations.append(op) + if self.optimizer.loop.logops: + debug_print(' Falling back to add extra: ' + + self.optimizer.loop.logops.repr_of_resop(op)) + self.optimizer.flush() self.optimizer.emitting_dissabled = False @@ -430,7 +439,13 @@ return for a in op.getarglist(): if not isinstance(a, Const) and a not in seen: - self.ensure_short_op_emitted(self.short_boxes.producer(a), optimizer, seen) + self.ensure_short_op_emitted(self.short_boxes.producer(a), optimizer, + seen) + + if self.optimizer.loop.logops: + debug_print(' Emitting short op: ' + + self.optimizer.loop.logops.repr_of_resop(op)) + optimizer.send_extra_operation(op) seen[op.result] = True if op.is_ovf(): diff --git a/pypy/jit/metainterp/optimizeopt/virtualstate.py b/pypy/jit/metainterp/optimizeopt/virtualstate.py --- a/pypy/jit/metainterp/optimizeopt/virtualstate.py +++ b/pypy/jit/metainterp/optimizeopt/virtualstate.py @@ -409,7 +409,13 @@ if self.level == LEVEL_CONSTANT: return assert 0 <= self.position_in_notvirtuals - boxes[self.position_in_notvirtuals] = value.force_box(optimizer) + if optimizer: + box = value.force_box(optimizer) + else: + if value.is_virtual(): + raise BadVirtualState + box = value.get_key_box() + boxes[self.position_in_notvirtuals] = box def _enum(self, virtual_state): if self.level == LEVEL_CONSTANT: @@ -471,8 +477,14 @@ optimizer = optimizer.optearlyforce assert len(values) == len(self.state) inputargs = [None] * len(self.notvirtuals) + + # We try twice. The first time around we allow boxes to be forced + # which might change the virtual state if the box appear in more + # than one place among the inputargs. for i in range(len(values)): self.state[i].enum_forced_boxes(inputargs, values[i], optimizer) + for i in range(len(values)): + self.state[i].enum_forced_boxes(inputargs, values[i], None) if keyboxes: for i in range(len(values)): diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py --- a/pypy/jit/metainterp/pyjitpl.py +++ b/pypy/jit/metainterp/pyjitpl.py @@ -976,10 +976,13 @@ self.verify_green_args(jitdriver_sd, greenboxes) self.debug_merge_point(jitdriver_sd, jdindex, self.metainterp.in_recursion, greenboxes) - + if self.metainterp.seen_loop_header_for_jdindex < 0: - if not jitdriver_sd.no_loop_header or not any_operation: + if not any_operation: return + if self.metainterp.in_recursion or not self.metainterp.get_procedure_token(greenboxes, True): + if not jitdriver_sd.no_loop_header: + return # automatically add a loop_header if there is none self.metainterp.seen_loop_header_for_jdindex = jdindex # @@ -1550,6 +1553,7 @@ class MetaInterp(object): in_recursion = 0 + cancel_count = 0 def __init__(self, staticdata, jitdriver_sd): self.staticdata = staticdata @@ -1790,6 +1794,15 @@ def aborted_tracing(self, reason): self.staticdata.profiler.count(reason) debug_print('~~~ ABORTING TRACING') + jd_sd = self.jitdriver_sd + if not self.current_merge_points: + greenkey = None # we're in the bridge + else: + greenkey = self.current_merge_points[0][0][:jd_sd.num_green_args] + self.staticdata.warmrunnerdesc.hooks.on_abort(reason, + jd_sd.jitdriver, + greenkey, + jd_sd.warmstate.get_location_str(greenkey)) self.staticdata.stats.aborted() def blackhole_if_trace_too_long(self): @@ -1963,9 +1976,14 @@ raise SwitchToBlackhole(ABORT_BAD_LOOP) # For now self.compile_loop(original_boxes, live_arg_boxes, start, resumedescr) # creation of the loop was cancelled! + self.cancel_count += 1 + if self.staticdata.warmrunnerdesc: + memmgr = self.staticdata.warmrunnerdesc.memory_manager + if memmgr: + if self.cancel_count > memmgr.max_unroll_loops: + self.staticdata.log('cancelled too many times!') + raise SwitchToBlackhole(ABORT_BAD_LOOP) self.staticdata.log('cancelled, tracing more...') - #self.staticdata.log('cancelled, stopping tracing') - #raise SwitchToBlackhole(ABORT_BAD_LOOP) # Otherwise, no loop found so far, so continue tracing. start = len(self.history.operations) @@ -2053,9 +2071,15 @@ from pypy.jit.metainterp.resoperation import opname raise NotImplementedError(opname[opnum]) - def get_procedure_token(self, greenkey): + def get_procedure_token(self, greenkey, with_compiled_targets=False): cell = self.jitdriver_sd.warmstate.jit_cell_at_key(greenkey) - return cell.get_procedure_token() + token = cell.get_procedure_token() + if with_compiled_targets: + if not token: + return None + if not token.target_tokens: + return None + return token def compile_loop(self, original_boxes, live_arg_boxes, start, resume_at_jump_descr): num_green_args = self.jitdriver_sd.num_green_args @@ -2088,11 +2112,9 @@ def compile_trace(self, live_arg_boxes, resume_at_jump_descr): num_green_args = self.jitdriver_sd.num_green_args greenkey = live_arg_boxes[:num_green_args] - target_jitcell_token = self.get_procedure_token(greenkey) + target_jitcell_token = self.get_procedure_token(greenkey, True) if not target_jitcell_token: return - if not target_jitcell_token.target_tokens: - return self.history.record(rop.JUMP, live_arg_boxes[num_green_args:], None, descr=target_jitcell_token) diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py --- a/pypy/jit/metainterp/resoperation.py +++ b/pypy/jit/metainterp/resoperation.py @@ -16,15 +16,15 @@ # debug name = "" pc = 0 + opnum = 0 + + _attrs_ = ('result',) def __init__(self, result): self.result = result - # methods implemented by each concrete class - # ------------------------------------------ - def getopnum(self): - raise NotImplementedError + return self.opnum # methods implemented by the arity mixins # --------------------------------------- @@ -64,6 +64,9 @@ def setdescr(self, descr): raise NotImplementedError + def cleardescr(self): + pass + # common methods # -------------- @@ -196,6 +199,9 @@ self._check_descr(descr) self._descr = descr + def cleardescr(self): + self._descr = None + def _check_descr(self, descr): if not we_are_translated() and getattr(descr, 'I_am_a_descr', False): return # needed for the mock case in oparser_model @@ -590,12 +596,9 @@ baseclass = PlainResOp mixin = arity2mixin.get(arity, N_aryOp) - def getopnum(self): - return opnum - cls_name = '%s_OP' % name bases = (get_base_class(mixin, baseclass),) - dic = {'getopnum': getopnum} + dic = {'opnum': opnum} return type(cls_name, bases, dic) setup(__name__ == '__main__') # print out the table when run directly diff --git a/pypy/jit/metainterp/test/support.py b/pypy/jit/metainterp/test/support.py --- a/pypy/jit/metainterp/test/support.py +++ b/pypy/jit/metainterp/test/support.py @@ -56,8 +56,6 @@ greenfield_info = None result_type = result_kind portal_runner_ptr = "???" - on_compile = lambda *args: None - on_compile_bridge = lambda *args: None stats = history.Stats() cpu = CPUClass(rtyper, stats, None, False) diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py --- a/pypy/jit/metainterp/test/test_ajit.py +++ b/pypy/jit/metainterp/test/test_ajit.py @@ -2629,6 +2629,38 @@ self.check_jitcell_token_count(1) self.check_target_token_count(5) + def test_max_unroll_loops(self): + from pypy.jit.metainterp.optimize import InvalidLoop + from pypy.jit.metainterp import optimizeopt + myjitdriver = JitDriver(greens = [], reds = ['n', 'i']) + # + def f(n, limit): + set_param(myjitdriver, 'threshold', 5) + set_param(myjitdriver, 'max_unroll_loops', limit) + i = 0 + while i < n: + myjitdriver.jit_merge_point(n=n, i=i) + print i + i += 1 + return i + # + def my_optimize_trace(*args, **kwds): + raise InvalidLoop + old_optimize_trace = optimizeopt.optimize_trace + optimizeopt.optimize_trace = my_optimize_trace + try: + res = self.meta_interp(f, [23, 4]) + assert res == 23 + self.check_trace_count(0) + self.check_aborted_count(3) + # + res = self.meta_interp(f, [23, 20]) + assert res == 23 + self.check_trace_count(0) + self.check_aborted_count(2) + finally: + optimizeopt.optimize_trace = old_optimize_trace + def test_retrace_limit_with_extra_guards(self): myjitdriver = JitDriver(greens = [], reds = ['n', 'i', 'sa', 'a', 'node']) @@ -2697,7 +2729,7 @@ # bridge back to the preamble of the first loop is produced. A guard in # this bridge is later traced resulting in a failed attempt of retracing # the second loop. - self.check_trace_count(8) + self.check_trace_count(9) # FIXME: Add a gloabl retrace counter and test that we are not trying more than 5 times. diff --git a/pypy/jit/metainterp/test/test_compile.py b/pypy/jit/metainterp/test/test_compile.py --- a/pypy/jit/metainterp/test/test_compile.py +++ b/pypy/jit/metainterp/test/test_compile.py @@ -18,7 +18,7 @@ self.seen.append((inputargs, operations, token)) class FakeLogger(object): - def log_loop(self, inputargs, operations, number=0, type=None, ops_offset=None): + def log_loop(self, inputargs, operations, number=0, type=None, ops_offset=None, name=''): pass def repr_of_resop(self, op): @@ -53,8 +53,6 @@ call_pure_results = {} class jitdriver_sd: warmstate = FakeState() - on_compile = staticmethod(lambda *args: None) - on_compile_bridge = staticmethod(lambda *args: None) virtualizable_info = None def test_compile_loop(): diff --git a/pypy/jit/metainterp/test/test_fficall.py b/pypy/jit/metainterp/test/test_fficall.py --- a/pypy/jit/metainterp/test/test_fficall.py +++ b/pypy/jit/metainterp/test/test_fficall.py @@ -148,28 +148,38 @@ self.check_resops({'jump': 1, 'int_lt': 2, 'setinteriorfield_raw': 4, 'getinteriorfield_raw': 8, 'int_add': 6, 'guard_true': 2}) - def test_array_getitem_uint8(self): + def _test_getitem_type(self, TYPE, ffitype, COMPUTE_TYPE): + reds = ["n", "i", "s", "data"] + if COMPUTE_TYPE is lltype.Float: + # Move the float var to the back. + reds.remove("s") + reds.append("s") myjitdriver = JitDriver( greens = [], - reds = ["n", "i", "s", "data"], + reds = reds, ) def f(data, n): - i = s = 0 + i = 0 + s = rffi.cast(COMPUTE_TYPE, 0) while i < n: myjitdriver.jit_merge_point(n=n, i=i, s=s, data=data) - s += rffi.cast(lltype.Signed, array_getitem(types.uchar, 1, data, 0, 0)) + s += rffi.cast(COMPUTE_TYPE, array_getitem(ffitype, rffi.sizeof(TYPE), data, 0, 0)) i += 1 return s + def main(n): + with lltype.scoped_alloc(rffi.CArray(TYPE), 1) as data: + data[0] = rffi.cast(TYPE, 200) + return f(data, n) + assert self.meta_interp(main, [10]) == 2000 - def main(n): - with lltype.scoped_alloc(rffi.CArray(rffi.UCHAR), 1) as data: - data[0] = rffi.cast(rffi.UCHAR, 200) - return f(data, n) - - assert self.meta_interp(main, [10]) == 2000 + def test_array_getitem_uint8(self): + self._test_getitem_type(rffi.UCHAR, types.uchar, lltype.Signed) self.check_resops({'jump': 1, 'int_lt': 2, 'getinteriorfield_raw': 2, 'guard_true': 2, 'int_add': 4}) + def test_array_getitem_float(self): + self._test_getitem_type(rffi.FLOAT, types.float, lltype.Float) + class TestFfiCall(FfiCallTests, LLJitMixin): supports_all = False diff --git a/pypy/jit/metainterp/test/test_heapcache.py b/pypy/jit/metainterp/test/test_heapcache.py --- a/pypy/jit/metainterp/test/test_heapcache.py +++ b/pypy/jit/metainterp/test/test_heapcache.py @@ -255,6 +255,11 @@ assert h.getarrayitem(box1, descr1, index1) is box2 assert h.getarrayitem(box1, descr1, index2) is box4 + h.invalidate_caches(rop.GUARD_TRUE, None, []) + assert h.getfield(box1, descr1) is box2 + assert h.getarrayitem(box1, descr1, index1) is box2 + assert h.getarrayitem(box1, descr1, index2) is box4 + h.invalidate_caches( rop.CALL_LOOPINVARIANT, FakeCallDescr(FakeEffektinfo.EF_LOOPINVARIANT), []) diff --git a/pypy/jit/metainterp/test/test_jitdriver.py b/pypy/jit/metainterp/test/test_jitdriver.py --- a/pypy/jit/metainterp/test/test_jitdriver.py +++ b/pypy/jit/metainterp/test/test_jitdriver.py @@ -10,57 +10,6 @@ def getloc2(g): return "in jitdriver2, with g=%d" % g -class JitDriverTests(object): - def test_on_compile(self): - called = {} - - class MyJitDriver(JitDriver): - def on_compile(self, logger, looptoken, operations, type, n, m): - called[(m, n, type)] = looptoken - - driver = MyJitDriver(greens = ['n', 'm'], reds = ['i']) - - def loop(n, m): - i = 0 - while i < n + m: - driver.can_enter_jit(n=n, m=m, i=i) - driver.jit_merge_point(n=n, m=m, i=i) - i += 1 - - self.meta_interp(loop, [1, 4]) - assert sorted(called.keys()) == [(4, 1, "loop")] - self.meta_interp(loop, [2, 4]) - assert sorted(called.keys()) == [(4, 1, "loop"), - (4, 2, "loop")] - - def test_on_compile_bridge(self): - called = {} - - class MyJitDriver(JitDriver): - def on_compile(self, logger, looptoken, operations, type, n, m): - called[(m, n, type)] = loop - def on_compile_bridge(self, logger, orig_token, operations, n): - assert 'bridge' not in called - called['bridge'] = orig_token - - driver = MyJitDriver(greens = ['n', 'm'], reds = ['i']) - - def loop(n, m): - i = 0 - while i < n + m: - driver.can_enter_jit(n=n, m=m, i=i) - driver.jit_merge_point(n=n, m=m, i=i) - if i >= 4: - i += 2 - i += 1 - - self.meta_interp(loop, [1, 10]) - assert sorted(called.keys()) == ['bridge', (10, 1, "loop")] - - -class TestLLtypeSingle(JitDriverTests, LLJitMixin): - pass - class MultipleJitDriversTests(object): def test_simple(self): diff --git a/pypy/jit/metainterp/test/test_jitiface.py b/pypy/jit/metainterp/test/test_jitiface.py new file mode 100644 --- /dev/null +++ b/pypy/jit/metainterp/test/test_jitiface.py @@ -0,0 +1,148 @@ + +from pypy.rlib.jit import JitDriver, JitHookInterface +from pypy.rlib import jit_hooks +from pypy.jit.metainterp.test.support import LLJitMixin +from pypy.jit.codewriter.policy import JitPolicy +from pypy.jit.metainterp.jitprof import ABORT_FORCE_QUASIIMMUT +from pypy.jit.metainterp.resoperation import rop +from pypy.rpython.annlowlevel import hlstr + +class TestJitHookInterface(LLJitMixin): + def test_abort_quasi_immut(self): + reasons = [] + + class MyJitIface(JitHookInterface): + def on_abort(self, reason, jitdriver, greenkey, greenkey_repr): + assert jitdriver is myjitdriver + assert len(greenkey) == 1 + reasons.append(reason) + assert greenkey_repr == 'blah' + + iface = MyJitIface() + + myjitdriver = JitDriver(greens=['foo'], reds=['x', 'total'], + get_printable_location=lambda *args: 'blah') + + class Foo: + _immutable_fields_ = ['a?'] + def __init__(self, a): + self.a = a + def f(a, x): + foo = Foo(a) + total = 0 + while x > 0: + myjitdriver.jit_merge_point(foo=foo, x=x, total=total) + # read a quasi-immutable field out of a Constant + total += foo.a + foo.a += 1 + x -= 1 + return total + # + assert f(100, 7) == 721 + res = self.meta_interp(f, [100, 7], policy=JitPolicy(iface)) + assert res == 721 + assert reasons == [ABORT_FORCE_QUASIIMMUT] * 2 + + def test_on_compile(self): + called = [] + + class MyJitIface(JitHookInterface): + def after_compile(self, di): + called.append(("compile", di.greenkey[1].getint(), + di.greenkey[0].getint(), di.type)) + + def before_compile(self, di): + called.append(("optimize", di.greenkey[1].getint(), + di.greenkey[0].getint(), di.type)) + + #def before_optimize(self, jitdriver, logger, looptoken, oeprations, + # type, greenkey): + # called.append(("trace", greenkey[1].getint(), + # greenkey[0].getint(), type)) + + iface = MyJitIface() + + driver = JitDriver(greens = ['n', 'm'], reds = ['i']) + + def loop(n, m): + i = 0 + while i < n + m: + driver.can_enter_jit(n=n, m=m, i=i) + driver.jit_merge_point(n=n, m=m, i=i) + i += 1 + + self.meta_interp(loop, [1, 4], policy=JitPolicy(iface)) + assert called == [#("trace", 4, 1, "loop"), + ("optimize", 4, 1, "loop"), + ("compile", 4, 1, "loop")] + self.meta_interp(loop, [2, 4], policy=JitPolicy(iface)) + assert called == [#("trace", 4, 1, "loop"), + ("optimize", 4, 1, "loop"), + ("compile", 4, 1, "loop"), + #("trace", 4, 2, "loop"), + ("optimize", 4, 2, "loop"), + ("compile", 4, 2, "loop")] + + def test_on_compile_bridge(self): + called = [] + + class MyJitIface(JitHookInterface): + def after_compile(self, di): + called.append("compile") + + def after_compile_bridge(self, di): + called.append("compile_bridge") + + def before_compile_bridge(self, di): + called.append("before_compile_bridge") + + driver = JitDriver(greens = ['n', 'm'], reds = ['i']) + + def loop(n, m): + i = 0 + while i < n + m: + driver.can_enter_jit(n=n, m=m, i=i) + driver.jit_merge_point(n=n, m=m, i=i) + if i >= 4: + i += 2 + i += 1 + + self.meta_interp(loop, [1, 10], policy=JitPolicy(MyJitIface())) + assert called == ["compile", "before_compile_bridge", "compile_bridge"] + + def test_resop_interface(self): + driver = JitDriver(greens = [], reds = ['i']) + + def loop(i): + while i > 0: + driver.jit_merge_point(i=i) + i -= 1 + + def main(): + loop(1) + op = jit_hooks.resop_new(rop.INT_ADD, + [jit_hooks.boxint_new(3), + jit_hooks.boxint_new(4)], + jit_hooks.boxint_new(1)) + assert hlstr(jit_hooks.resop_getopname(op)) == 'int_add' + assert jit_hooks.resop_getopnum(op) == rop.INT_ADD + box = jit_hooks.resop_getarg(op, 0) + assert jit_hooks.box_getint(box) == 3 + box2 = jit_hooks.box_clone(box) + assert box2 != box + assert jit_hooks.box_getint(box2) == 3 + assert not jit_hooks.box_isconst(box2) + box3 = jit_hooks.box_constbox(box) + assert jit_hooks.box_getint(box) == 3 + assert jit_hooks.box_isconst(box3) + box4 = jit_hooks.box_nonconstbox(box) + assert not jit_hooks.box_isconst(box4) + box5 = jit_hooks.boxint_new(18) + jit_hooks.resop_setarg(op, 0, box5) + assert jit_hooks.resop_getarg(op, 0) == box5 + box6 = jit_hooks.resop_getresult(op) + assert jit_hooks.box_getint(box6) == 1 + jit_hooks.resop_setresult(op, box5) + assert jit_hooks.resop_getresult(op) == box5 + + self.meta_interp(main, []) diff --git a/pypy/jit/metainterp/test/test_logger.py b/pypy/jit/metainterp/test/test_logger.py --- a/pypy/jit/metainterp/test/test_logger.py +++ b/pypy/jit/metainterp/test/test_logger.py @@ -180,7 +180,7 @@ def test_intro_loop(self): bare_logger = logger.Logger(self.make_metainterp_sd()) output = capturing(bare_logger.log_loop, [], [], 1, "foo") - assert output.splitlines()[0] == "# Loop 1 : foo with 0 ops" + assert output.splitlines()[0] == "# Loop 1 () : foo with 0 ops" pure_parse(output) def test_intro_bridge(self): diff --git a/pypy/jit/metainterp/test/test_loop.py b/pypy/jit/metainterp/test/test_loop.py --- a/pypy/jit/metainterp/test/test_loop.py +++ b/pypy/jit/metainterp/test/test_loop.py @@ -756,7 +756,7 @@ res = self.meta_interp(interpret, [1]) assert res == interpret(1) # XXX it's unsure how many loops should be there - self.check_trace_count(3) + self.check_trace_count(2) def test_path_with_operations_not_from_start(self): jitdriver = JitDriver(greens = ['k'], reds = ['n', 'z']) diff --git a/pypy/jit/metainterp/test/test_resoperation.py b/pypy/jit/metainterp/test/test_resoperation.py --- a/pypy/jit/metainterp/test/test_resoperation.py +++ b/pypy/jit/metainterp/test/test_resoperation.py @@ -30,17 +30,17 @@ cls = rop.opclasses[rop.rop.INT_ADD] assert issubclass(cls, rop.PlainResOp) assert issubclass(cls, rop.BinaryOp) - assert cls.getopnum.im_func(None) == rop.rop.INT_ADD + assert cls.getopnum.im_func(cls) == rop.rop.INT_ADD cls = rop.opclasses[rop.rop.CALL] assert issubclass(cls, rop.ResOpWithDescr) assert issubclass(cls, rop.N_aryOp) - assert cls.getopnum.im_func(None) == rop.rop.CALL + assert cls.getopnum.im_func(cls) == rop.rop.CALL cls = rop.opclasses[rop.rop.GUARD_TRUE] assert issubclass(cls, rop.GuardResOp) assert issubclass(cls, rop.UnaryOp) - assert cls.getopnum.im_func(None) == rop.rop.GUARD_TRUE + assert cls.getopnum.im_func(cls) == rop.rop.GUARD_TRUE def test_mixins_in_common_base(): INT_ADD = rop.opclasses[rop.rop.INT_ADD] diff --git a/pypy/jit/metainterp/test/test_virtual.py b/pypy/jit/metainterp/test/test_virtual.py --- a/pypy/jit/metainterp/test/test_virtual.py +++ b/pypy/jit/metainterp/test/test_virtual.py @@ -880,7 +880,7 @@ elif op == 'j': j = Int(0) elif op == '+': - sa += i.val * j.val + sa += (i.val + 2) * (j.val + 2) elif op == 'a': i = Int(i.val + 1) elif op == 'b': @@ -902,6 +902,7 @@ assert res == f(10) self.check_aborted_count(0) self.check_target_token_count(3) + self.check_resops(int_mul=2) def test_nested_loops_bridge(self): class Int(object): diff --git a/pypy/jit/metainterp/test/test_virtualstate.py b/pypy/jit/metainterp/test/test_virtualstate.py --- a/pypy/jit/metainterp/test/test_virtualstate.py +++ b/pypy/jit/metainterp/test/test_virtualstate.py @@ -5,7 +5,7 @@ VArrayStateInfo, NotVirtualStateInfo, VirtualState, ShortBoxes from pypy.jit.metainterp.optimizeopt.optimizer import OptValue from pypy.jit.metainterp.history import BoxInt, BoxFloat, BoxPtr, ConstInt, ConstPtr -from pypy.rpython.lltypesystem import lltype +from pypy.rpython.lltypesystem import lltype, llmemory from pypy.jit.metainterp.optimizeopt.test.test_util import LLtypeMixin, BaseTest, \ equaloplists, FakeDescrWithSnapshot from pypy.jit.metainterp.optimizeopt.intutils import IntBound @@ -82,6 +82,13 @@ assert isgeneral(value1, value2) assert not isgeneral(value2, value1) + assert isgeneral(OptValue(ConstInt(7)), OptValue(ConstInt(7))) + S = lltype.GcStruct('S') + foo = lltype.malloc(S) + fooref = lltype.cast_opaque_ptr(llmemory.GCREF, foo) + assert isgeneral(OptValue(ConstPtr(fooref)), + OptValue(ConstPtr(fooref))) + def test_field_matching_generalization(self): const1 = NotVirtualStateInfo(OptValue(ConstInt(1))) const2 = NotVirtualStateInfo(OptValue(ConstInt(2))) diff --git a/pypy/jit/metainterp/test/test_ztranslation.py b/pypy/jit/metainterp/test/test_ztranslation.py --- a/pypy/jit/metainterp/test/test_ztranslation.py +++ b/pypy/jit/metainterp/test/test_ztranslation.py @@ -3,7 +3,9 @@ from pypy.jit.backend.llgraph import runner from pypy.rlib.jit import JitDriver, unroll_parameters, set_param from pypy.rlib.jit import PARAMETERS, dont_look_inside, hint +from pypy.rlib.jit_hooks import boxint_new, resop_new, resop_getopnum from pypy.jit.metainterp.jitprof import Profiler +from pypy.jit.metainterp.resoperation import rop from pypy.rpython.lltypesystem import lltype, llmemory class TranslationTest: @@ -22,6 +24,7 @@ # - jitdriver hooks # - two JITs # - string concatenation, slicing and comparison + # - jit hooks interface class Frame(object): _virtualizable2_ = ['l[*]'] @@ -91,7 +94,9 @@ return f.i # def main(i, j): - return f(i) - f2(i+j, i, j) + op = resop_new(rop.INT_ADD, [boxint_new(3), boxint_new(5)], + boxint_new(8)) + return f(i) - f2(i+j, i, j) + resop_getopnum(op) res = ll_meta_interp(main, [40, 5], CPUClass=self.CPUClass, type_system=self.type_system, listops=True) diff --git a/pypy/jit/metainterp/warmspot.py b/pypy/jit/metainterp/warmspot.py --- a/pypy/jit/metainterp/warmspot.py +++ b/pypy/jit/metainterp/warmspot.py @@ -1,4 +1,5 @@ import sys, py +from pypy.tool.sourcetools import func_with_new_name from pypy.rpython.lltypesystem import lltype, llmemory from pypy.rpython.annlowlevel import llhelper, MixLevelHelperAnnotator,\ cast_base_ptr_to_instance, hlstr @@ -112,7 +113,7 @@ return ll_meta_interp(function, args, backendopt=backendopt, translate_support_code=True, **kwds) -def _find_jit_marker(graphs, marker_name): +def _find_jit_marker(graphs, marker_name, check_driver=True): results = [] for graph in graphs: for block in graph.iterblocks(): @@ -120,8 +121,8 @@ op = block.operations[i] if (op.opname == 'jit_marker' and op.args[0].value == marker_name and - (op.args[1].value is None or - op.args[1].value.active)): # the jitdriver + (not check_driver or op.args[1].value is None or + op.args[1].value.active)): # the jitdriver results.append((graph, block, i)) return results @@ -140,6 +141,9 @@ "found several jit_merge_points in the same graph") return results +def find_access_helpers(graphs): + return _find_jit_marker(graphs, 'access_helper', False) + def locate_jit_merge_point(graph): [(graph, block, pos)] = find_jit_merge_points([graph]) return block, pos, block.operations[pos] @@ -206,6 +210,7 @@ vrefinfo = VirtualRefInfo(self) self.codewriter.setup_vrefinfo(vrefinfo) # + self.hooks = policy.jithookiface self.make_virtualizable_infos() self.make_exception_classes() self.make_driverhook_graphs() @@ -213,6 +218,7 @@ self.rewrite_jit_merge_points(policy) verbose = False # not self.cpu.translate_support_code + self.rewrite_access_helpers() self.codewriter.make_jitcodes(verbose=verbose) self.rewrite_can_enter_jits() self.rewrite_set_param() @@ -619,6 +625,24 @@ graph = self.annhelper.getgraph(func, args_s, s_result) return self.annhelper.graph2delayed(graph, FUNC) + def rewrite_access_helpers(self): + ah = find_access_helpers(self.translator.graphs) + for graph, block, index in ah: + op = block.operations[index] + self.rewrite_access_helper(op) + + def rewrite_access_helper(self, op): + ARGS = [arg.concretetype for arg in op.args[2:]] + RESULT = op.result.concretetype + FUNCPTR = lltype.Ptr(lltype.FuncType(ARGS, RESULT)) + # make sure we make a copy of function so it no longer belongs + # to extregistry + func = op.args[1].value + func = func_with_new_name(func, func.func_name + '_compiled') + ptr = self.helper_func(FUNCPTR, func) + op.opname = 'direct_call' + op.args = [Constant(ptr, FUNCPTR)] + op.args[2:] + def rewrite_jit_merge_points(self, policy): for jd in self.jitdrivers_sd: self.rewrite_jit_merge_point(jd, policy) diff --git a/pypy/jit/metainterp/warmstate.py b/pypy/jit/metainterp/warmstate.py --- a/pypy/jit/metainterp/warmstate.py +++ b/pypy/jit/metainterp/warmstate.py @@ -244,6 +244,11 @@ if self.warmrunnerdesc.memory_manager: self.warmrunnerdesc.memory_manager.max_retrace_guards = value + def set_param_max_unroll_loops(self, value): + if self.warmrunnerdesc: + if self.warmrunnerdesc.memory_manager: + self.warmrunnerdesc.memory_manager.max_unroll_loops = value + def disable_noninlinable_function(self, greenkey): cell = self.jit_cell_at_key(greenkey) cell.dont_trace_here = True @@ -596,20 +601,6 @@ return fn(*greenargs) self.should_unroll_one_iteration = should_unroll_one_iteration - if hasattr(jd.jitdriver, 'on_compile'): - def on_compile(logger, token, operations, type, greenkey): - greenargs = unwrap_greenkey(greenkey) - return jd.jitdriver.on_compile(logger, token, operations, type, - *greenargs) - def on_compile_bridge(logger, orig_token, operations, n): - return jd.jitdriver.on_compile_bridge(logger, orig_token, - operations, n) - jd.on_compile = on_compile - jd.on_compile_bridge = on_compile_bridge - else: - jd.on_compile = lambda *args: None - jd.on_compile_bridge = lambda *args: None - redargtypes = ''.join([kind[0] for kind in jd.red_args_types]) def get_assembler_token(greenkey): diff --git a/pypy/jit/tool/oparser.py b/pypy/jit/tool/oparser.py --- a/pypy/jit/tool/oparser.py +++ b/pypy/jit/tool/oparser.py @@ -89,11 +89,18 @@ assert typ == 'class' return self.model.ConstObj(ootype.cast_to_object(obj)) - def get_descr(self, poss_descr): + def get_descr(self, poss_descr, allow_invent): if poss_descr.startswith('<'): return None - else: + try: return self._consts[poss_descr] + except KeyError: + if allow_invent: + int(poss_descr) + token = self.model.JitCellToken() + tt = self.model.TargetToken(token) + self._consts[poss_descr] = tt + return tt def box_for_var(self, elem): try: @@ -186,7 +193,8 @@ poss_descr = allargs[-1].strip() if poss_descr.startswith('descr='): - descr = self.get_descr(poss_descr[len('descr='):]) + descr = self.get_descr(poss_descr[len('descr='):], + opname == 'label') allargs = allargs[:-1] for arg in allargs: arg = arg.strip() diff --git a/pypy/jit/tool/oparser_model.py b/pypy/jit/tool/oparser_model.py --- a/pypy/jit/tool/oparser_model.py +++ b/pypy/jit/tool/oparser_model.py @@ -6,7 +6,7 @@ from pypy.jit.metainterp.history import TreeLoop, JitCellToken from pypy.jit.metainterp.history import Box, BoxInt, BoxFloat from pypy.jit.metainterp.history import ConstInt, ConstObj, ConstPtr, ConstFloat - from pypy.jit.metainterp.history import BasicFailDescr + from pypy.jit.metainterp.history import BasicFailDescr, TargetToken from pypy.jit.metainterp.typesystem import llhelper from pypy.jit.metainterp.history import get_const_ptr_for_string @@ -42,6 +42,10 @@ class JitCellToken(object): I_am_a_descr = True + class TargetToken(object): + def __init__(self, jct): + pass + class BasicFailDescr(object): I_am_a_descr = True diff --git a/pypy/jit/tool/pypytrace.vim b/pypy/jit/tool/pypytrace.vim --- a/pypy/jit/tool/pypytrace.vim +++ b/pypy/jit/tool/pypytrace.vim @@ -19,6 +19,7 @@ syn match pypyLoopArgs '^[[].*' syn match pypyLoopStart '^#.*' syn match pypyDebugMergePoint '^debug_merge_point(.\+)' +syn match pypyLogBoundary '[[][0-9a-f]\+[]] $[{].\+\|.\+[}]$$' hi def link pypyLoopStart Structure "hi def link pypyLoopArgs PreProc @@ -29,3 +30,4 @@ hi def link pypyNumber Number hi def link pypyDescr PreProc hi def link pypyDescrField Label +hi def link pypyLogBoundary Statement diff --git a/pypy/jit/tool/test/test_oparser.py b/pypy/jit/tool/test/test_oparser.py --- a/pypy/jit/tool/test/test_oparser.py +++ b/pypy/jit/tool/test/test_oparser.py @@ -4,7 +4,8 @@ from pypy.jit.tool.oparser import parse, OpParser from pypy.jit.metainterp.resoperation import rop -from pypy.jit.metainterp.history import AbstractDescr, BoxInt, JitCellToken +from pypy.jit.metainterp.history import AbstractDescr, BoxInt, JitCellToken,\ + TargetToken class BaseTestOparser(object): @@ -243,6 +244,16 @@ b = loop.getboxes() assert isinstance(b.sum0, BoxInt) + def test_label(self): + x = """ + [i0] + label(i0, descr=1) + jump(i0, descr=1) + """ + loop = self.parse(x) + assert loop.operations[0].getdescr() is loop.operations[1].getdescr() + assert isinstance(loop.operations[0].getdescr(), TargetToken) + class ForbiddenModule(object): def __init__(self, name, old_mod): diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py --- a/pypy/module/_codecs/test/test_codecs.py +++ b/pypy/module/_codecs/test/test_codecs.py @@ -588,10 +588,18 @@ raises(UnicodeDecodeError, '+3ADYAA-'.decode, 'utf-7') def test_utf_16_encode_decode(self): - import codecs + import codecs, sys x = u'123abc' - assert codecs.getencoder('utf-16')(x) == ('\xff\xfe1\x002\x003\x00a\x00b\x00c\x00', 6) - assert codecs.getdecoder('utf-16')('\xff\xfe1\x002\x003\x00a\x00b\x00c\x00') == (x, 14) + if sys.byteorder == 'big': + assert codecs.getencoder('utf-16')(x) == ( + '\xfe\xff\x001\x002\x003\x00a\x00b\x00c', 6) + assert codecs.getdecoder('utf-16')( + '\xfe\xff\x001\x002\x003\x00a\x00b\x00c') == (x, 14) + else: + assert codecs.getencoder('utf-16')(x) == ( + '\xff\xfe1\x002\x003\x00a\x00b\x00c\x00', 6) + assert codecs.getdecoder('utf-16')( + '\xff\xfe1\x002\x003\x00a\x00b\x00c\x00') == (x, 14) def test_unicode_escape(self): assert u'\\'.encode('unicode-escape') == '\\\\' diff --git a/pypy/module/_lsprof/interp_lsprof.py b/pypy/module/_lsprof/interp_lsprof.py --- a/pypy/module/_lsprof/interp_lsprof.py +++ b/pypy/module/_lsprof/interp_lsprof.py @@ -19,8 +19,9 @@ # cpu affinity settings srcdir = py.path.local(pypydir).join('translator', 'c', 'src') -eci = ExternalCompilationInfo(separate_module_files= - [srcdir.join('profiling.c')]) +eci = ExternalCompilationInfo( + separate_module_files=[srcdir.join('profiling.c')], + export_symbols=['pypy_setup_profiling', 'pypy_teardown_profiling']) c_setup_profiling = rffi.llexternal('pypy_setup_profiling', [], lltype.Void, diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -23,6 +23,7 @@ from pypy.interpreter.function import StaticMethod from pypy.objspace.std.sliceobject import W_SliceObject from pypy.module.__builtin__.descriptor import W_Property +from pypy.module.__builtin__.interp_memoryview import W_MemoryView from pypy.rlib.entrypoint import entrypoint from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.objectmodel import specialize @@ -387,6 +388,8 @@ "Float": "space.w_float", "Long": "space.w_long", "Complex": "space.w_complex", + "ByteArray": "space.w_bytearray", + "MemoryView": "space.gettypeobject(W_MemoryView.typedef)", "BaseObject": "space.w_object", 'None': 'space.type(space.w_None)', 'NotImplemented': 'space.type(space.w_NotImplemented)', diff --git a/pypy/module/cpyext/buffer.py b/pypy/module/cpyext/buffer.py --- a/pypy/module/cpyext/buffer.py +++ b/pypy/module/cpyext/buffer.py @@ -1,6 +1,36 @@ +from pypy.interpreter.error import OperationError from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.api import ( cpython_api, CANNOT_FAIL, Py_buffer) +from pypy.module.cpyext.pyobject import PyObject + + at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) +def PyObject_CheckBuffer(space, w_obj): + """Return 1 if obj supports the buffer interface otherwise 0.""" + return 0 # the bf_getbuffer field is never filled by cpyext + + at cpython_api([PyObject, lltype.Ptr(Py_buffer), rffi.INT_real], + rffi.INT_real, error=-1) +def PyObject_GetBuffer(space, w_obj, view, flags): + """Export obj into a Py_buffer, view. These arguments must + never be NULL. The flags argument is a bit field indicating what + kind of buffer the caller is prepared to deal with and therefore what + kind of buffer the exporter is allowed to return. The buffer interface + allows for complicated memory sharing possibilities, but some caller may + not be able to handle all the complexity but may want to see if the + exporter will let them take a simpler view to its memory. + + Some exporters may not be able to share memory in every possible way and + may need to raise errors to signal to some consumers that something is + just not possible. These errors should be a BufferError unless + there is another error that is actually causing the problem. The + exporter can use flags information to simplify how much of the + Py_buffer structure is filled in with non-default values and/or + raise an error if the object can't support a simpler view of its memory. + + 0 is returned on success and -1 on error.""" + raise OperationError(space.w_TypeError, space.wrap( + 'PyPy does not yet implement the new buffer interface')) @cpython_api([lltype.Ptr(Py_buffer), lltype.Char], rffi.INT_real, error=CANNOT_FAIL) def PyBuffer_IsContiguous(space, view, fortran): diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h --- a/pypy/module/cpyext/include/object.h +++ b/pypy/module/cpyext/include/object.h @@ -123,10 +123,6 @@ typedef Py_ssize_t (*segcountproc)(PyObject *, Py_ssize_t *); typedef Py_ssize_t (*charbufferproc)(PyObject *, Py_ssize_t, char **); -typedef int (*objobjproc)(PyObject *, PyObject *); -typedef int (*visitproc)(PyObject *, void *); -typedef int (*traverseproc)(PyObject *, visitproc, void *); - /* Py3k buffer interface */ typedef struct bufferinfo { void *buf; @@ -153,6 +149,41 @@ typedef int (*getbufferproc)(PyObject *, Py_buffer *, int); typedef void (*releasebufferproc)(PyObject *, Py_buffer *); + /* Flags for getting buffers */ +#define PyBUF_SIMPLE 0 +#define PyBUF_WRITABLE 0x0001 +/* we used to include an E, backwards compatible alias */ +#define PyBUF_WRITEABLE PyBUF_WRITABLE +#define PyBUF_FORMAT 0x0004 +#define PyBUF_ND 0x0008 +#define PyBUF_STRIDES (0x0010 | PyBUF_ND) +#define PyBUF_C_CONTIGUOUS (0x0020 | PyBUF_STRIDES) +#define PyBUF_F_CONTIGUOUS (0x0040 | PyBUF_STRIDES) +#define PyBUF_ANY_CONTIGUOUS (0x0080 | PyBUF_STRIDES) +#define PyBUF_INDIRECT (0x0100 | PyBUF_STRIDES) + +#define PyBUF_CONTIG (PyBUF_ND | PyBUF_WRITABLE) +#define PyBUF_CONTIG_RO (PyBUF_ND) + +#define PyBUF_STRIDED (PyBUF_STRIDES | PyBUF_WRITABLE) +#define PyBUF_STRIDED_RO (PyBUF_STRIDES) + +#define PyBUF_RECORDS (PyBUF_STRIDES | PyBUF_WRITABLE | PyBUF_FORMAT) +#define PyBUF_RECORDS_RO (PyBUF_STRIDES | PyBUF_FORMAT) + +#define PyBUF_FULL (PyBUF_INDIRECT | PyBUF_WRITABLE | PyBUF_FORMAT) +#define PyBUF_FULL_RO (PyBUF_INDIRECT | PyBUF_FORMAT) + + +#define PyBUF_READ 0x100 +#define PyBUF_WRITE 0x200 +#define PyBUF_SHADOW 0x400 +/* end Py3k buffer interface */ + +typedef int (*objobjproc)(PyObject *, PyObject *); +typedef int (*visitproc)(PyObject *, void *); +typedef int (*traverseproc)(PyObject *, visitproc, void *); + typedef struct { /* For numbers without flag bit Py_TPFLAGS_CHECKTYPES set, all arguments are guaranteed to be of the object's type (modulo diff --git a/pypy/module/cpyext/include/pystate.h b/pypy/module/cpyext/include/pystate.h --- a/pypy/module/cpyext/include/pystate.h +++ b/pypy/module/cpyext/include/pystate.h @@ -5,7 +5,7 @@ struct _is; /* Forward */ typedef struct _is { - int _foo; + struct _is *next; } PyInterpreterState; typedef struct _ts { diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py --- a/pypy/module/cpyext/methodobject.py +++ b/pypy/module/cpyext/methodobject.py @@ -58,6 +58,7 @@ class W_PyCFunctionObject(Wrappable): def __init__(self, space, ml, w_self, w_module=None): self.ml = ml + self.name = rffi.charp2str(self.ml.c_ml_name) self.w_self = w_self self.w_module = w_module @@ -69,7 +70,7 @@ flags &= ~(METH_CLASS | METH_STATIC | METH_COEXIST) if space.is_true(w_kw) and not flags & METH_KEYWORDS: raise OperationError(space.w_TypeError, space.wrap( - rffi.charp2str(self.ml.c_ml_name) + "() takes no keyword arguments")) + self.name + "() takes no keyword arguments")) func = rffi.cast(PyCFunction, self.ml.c_ml_meth) length = space.int_w(space.len(w_args)) @@ -80,13 +81,12 @@ if length == 0: return generic_cpy_call(space, func, w_self, None) raise OperationError(space.w_TypeError, space.wrap( - rffi.charp2str(self.ml.c_ml_name) + "() takes no arguments")) + self.name + "() takes no arguments")) elif flags & METH_O: if length != 1: raise OperationError(space.w_TypeError, space.wrap("%s() takes exactly one argument (%d given)" % ( - rffi.charp2str(self.ml.c_ml_name), - length))) + self.name, length))) w_arg = space.getitem(w_args, space.wrap(0)) return generic_cpy_call(space, func, w_self, w_arg) elif flags & METH_VARARGS: @@ -199,6 +199,7 @@ __call__ = interp2app(cfunction_descr_call), __doc__ = GetSetProperty(W_PyCFunctionObject.get_doc), __module__ = interp_attrproperty_w('w_module', cls=W_PyCFunctionObject), + __name__ = interp_attrproperty('name', cls=W_PyCFunctionObject), ) W_PyCFunctionObject.typedef.acceptable_as_base_class = False diff --git a/pypy/module/cpyext/pystate.py b/pypy/module/cpyext/pystate.py --- a/pypy/module/cpyext/pystate.py +++ b/pypy/module/cpyext/pystate.py @@ -2,7 +2,10 @@ cpython_api, generic_cpy_call, CANNOT_FAIL, CConfig, cpython_struct) from pypy.rpython.lltypesystem import rffi, lltype -PyInterpreterState = lltype.Ptr(cpython_struct("PyInterpreterState", ())) +PyInterpreterStateStruct = lltype.ForwardReference() +PyInterpreterState = lltype.Ptr(PyInterpreterStateStruct) +cpython_struct( + "PyInterpreterState", [('next', PyInterpreterState)], PyInterpreterStateStruct) PyThreadState = lltype.Ptr(cpython_struct("PyThreadState", [('interp', PyInterpreterState)])) @cpython_api([], PyThreadState, error=CANNOT_FAIL) @@ -54,7 +57,8 @@ class InterpreterState(object): def __init__(self, space): - self.interpreter_state = lltype.malloc(PyInterpreterState.TO, flavor='raw', immortal=True) + self.interpreter_state = lltype.malloc( + PyInterpreterState.TO, flavor='raw', zero=True, immortal=True) def new_thread_state(self): capsule = ThreadStateCapsule() diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py --- a/pypy/module/cpyext/stubs.py +++ b/pypy/module/cpyext/stubs.py @@ -34,141 +34,6 @@ @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) def PyObject_CheckBuffer(space, obj): - """Return 1 if obj supports the buffer interface otherwise 0.""" - raise NotImplementedError - - at cpython_api([PyObject, Py_buffer, rffi.INT_real], rffi.INT_real, error=-1) -def PyObject_GetBuffer(space, obj, view, flags): - """Export obj into a Py_buffer, view. These arguments must - never be NULL. The flags argument is a bit field indicating what - kind of buffer the caller is prepared to deal with and therefore what - kind of buffer the exporter is allowed to return. The buffer interface - allows for complicated memory sharing possibilities, but some caller may - not be able to handle all the complexity but may want to see if the - exporter will let them take a simpler view to its memory. - - Some exporters may not be able to share memory in every possible way and - may need to raise errors to signal to some consumers that something is - just not possible. These errors should be a BufferError unless - there is another error that is actually causing the problem. The - exporter can use flags information to simplify how much of the - Py_buffer structure is filled in with non-default values and/or - raise an error if the object can't support a simpler view of its memory. - - 0 is returned on success and -1 on error. - - The following table gives possible values to the flags arguments. - - Flag - - Description - - PyBUF_SIMPLE - - This is the default flag state. The returned - buffer may or may not have writable memory. The - format of the data will be assumed to be unsigned - bytes. This is a "stand-alone" flag constant. It - never needs to be '|'d to the others. The exporter - will raise an error if it cannot provide such a - contiguous buffer of bytes. - - PyBUF_WRITABLE - - The returned buffer must be writable. If it is - not writable, then raise an error. - - PyBUF_STRIDES - - This implies PyBUF_ND. The returned - buffer must provide strides information (i.e. the - strides cannot be NULL). This would be used when - the consumer can handle strided, discontiguous - arrays. Handling strides automatically assumes - you can handle shape. The exporter can raise an - error if a strided representation of the data is - not possible (i.e. without the suboffsets). - - PyBUF_ND - - The returned buffer must provide shape - information. The memory will be assumed C-style - contiguous (last dimension varies the - fastest). The exporter may raise an error if it - cannot provide this kind of contiguous buffer. If - this is not given then shape will be NULL. - - PyBUF_C_CONTIGUOUS - PyBUF_F_CONTIGUOUS - PyBUF_ANY_CONTIGUOUS - - These flags indicate that the contiguity returned - buffer must be respectively, C-contiguous (last - dimension varies the fastest), Fortran contiguous - (first dimension varies the fastest) or either - one. All of these flags imply - PyBUF_STRIDES and guarantee that the - strides buffer info structure will be filled in - correctly. - - PyBUF_INDIRECT - - This flag indicates the returned buffer must have - suboffsets information (which can be NULL if no - suboffsets are needed). This can be used when - the consumer can handle indirect array - referencing implied by these suboffsets. This - implies PyBUF_STRIDES. - - PyBUF_FORMAT - - The returned buffer must have true format - information if this flag is provided. This would - be used when the consumer is going to be checking - for what 'kind' of data is actually stored. An - exporter should always be able to provide this - information if requested. If format is not - explicitly requested then the format must be - returned as NULL (which means 'B', or - unsigned bytes) - - PyBUF_STRIDED - - This is equivalent to (PyBUF_STRIDES | - PyBUF_WRITABLE). - - PyBUF_STRIDED_RO - - This is equivalent to (PyBUF_STRIDES). - - PyBUF_RECORDS - - This is equivalent to (PyBUF_STRIDES | - PyBUF_FORMAT | PyBUF_WRITABLE). - - PyBUF_RECORDS_RO - - This is equivalent to (PyBUF_STRIDES | - PyBUF_FORMAT). - - PyBUF_FULL - - This is equivalent to (PyBUF_INDIRECT | - PyBUF_FORMAT | PyBUF_WRITABLE). - - PyBUF_FULL_RO - - This is equivalent to (PyBUF_INDIRECT | - PyBUF_FORMAT). - - PyBUF_CONTIG - - This is equivalent to (PyBUF_ND | - PyBUF_WRITABLE). - - PyBUF_CONTIG_RO - - This is equivalent to (PyBUF_ND).""" raise NotImplementedError @cpython_api([rffi.CCHARP], Py_ssize_t, error=CANNOT_FAIL) diff --git a/pypy/module/cpyext/test/test_methodobject.py b/pypy/module/cpyext/test/test_methodobject.py --- a/pypy/module/cpyext/test/test_methodobject.py +++ b/pypy/module/cpyext/test/test_methodobject.py @@ -63,6 +63,7 @@ ), ]) assert mod.getarg_O(1) == 1 + assert mod.getarg_O.__name__ == "getarg_O" raises(TypeError, mod.getarg_O) raises(TypeError, mod.getarg_O, 1, 1) diff --git a/pypy/module/cpyext/test/test_pystate.py b/pypy/module/cpyext/test/test_pystate.py --- a/pypy/module/cpyext/test/test_pystate.py +++ b/pypy/module/cpyext/test/test_pystate.py @@ -37,6 +37,7 @@ def test_thread_state_interp(self, space, api): ts = api.PyThreadState_Get() assert ts.c_interp == api.PyInterpreterState_Head() + assert ts.c_interp.c_next == nullptr(PyInterpreterState.TO) def test_basic_threadstate_dance(self, space, api): # Let extension modules call these functions, diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py --- a/pypy/module/micronumpy/__init__.py +++ b/pypy/module/micronumpy/__init__.py @@ -4,11 +4,12 @@ class PyPyModule(MixedModule): interpleveldefs = { 'debug_repr': 'interp_extras.debug_repr', + 'remove_invalidates': 'interp_extras.remove_invalidates', } appleveldefs = {} class Module(MixedModule): - applevel_name = 'numpypy' + applevel_name = '_numpypy' submodules = { 'pypy': PyPyModule @@ -47,6 +48,7 @@ 'int_': 'interp_boxes.W_LongBox', 'inexact': 'interp_boxes.W_InexactBox', 'floating': 'interp_boxes.W_FloatingBox', + 'float_': 'interp_boxes.W_Float64Box', 'float32': 'interp_boxes.W_Float32Box', 'float64': 'interp_boxes.W_Float64Box', } diff --git a/pypy/module/micronumpy/app_numpy.py b/pypy/module/micronumpy/app_numpy.py --- a/pypy/module/micronumpy/app_numpy.py +++ b/pypy/module/micronumpy/app_numpy.py @@ -1,6 +1,6 @@ import math -import numpypy +import _numpypy inf = float("inf") @@ -14,30 +14,54 @@ return mean(a) def identity(n, dtype=None): - a = numpypy.zeros((n,n), dtype=dtype) + a = _numpypy.zeros((n,n), dtype=dtype) for i in range(n): a[i][i] = 1 return a -def mean(a): +def mean(a, axis=None): if not hasattr(a, "mean"): - a = numpypy.array(a) - return a.mean() + a = _numpypy.array(a) + return a.mean(axis) -def sum(a): +def sum(a,axis=None): + '''sum(a, axis=None) + Sum of array elements over a given axis. + + Parameters + ---------- + a : array_like + Elements to sum. + axis : integer, optional + Axis over which the sum is taken. By default `axis` is None, + and all elements are summed. + + Returns + ------- + sum_along_axis : ndarray + An array with the same shape as `a`, with the specified + axis removed. If `a` is a 0-d array, or if `axis` is None, a scalar + is returned. If an output array is specified, a reference to + `out` is returned. + + See Also + -------- + ndarray.sum : Equivalent method. + ''' + # TODO: add to doc (once it's implemented): cumsum : Cumulative sum of array elements. if not hasattr(a, "sum"): - a = numpypy.array(a) - return a.sum() + a = _numpypy.array(a) + return a.sum(axis) -def min(a): +def min(a, axis=None): if not hasattr(a, "min"): - a = numpypy.array(a) - return a.min() + a = _numpypy.array(a) + return a.min(axis) -def max(a): +def max(a, axis=None): if not hasattr(a, "max"): - a = numpypy.array(a) - return a.max() + a = _numpypy.array(a) + return a.max(axis) def arange(start, stop=None, step=1, dtype=None): '''arange([start], stop[, step], dtype=None) @@ -47,9 +71,9 @@ stop = start start = 0 if dtype is None: - test = numpypy.array([start, stop, step, 0]) + test = _numpypy.array([start, stop, step, 0]) dtype = test.dtype - arr = numpypy.zeros(int(math.ceil((stop - start) / step)), dtype=dtype) + arr = _numpypy.zeros(int(math.ceil((stop - start) / step)), dtype=dtype) i = start for j in range(arr.size): arr[j] = i @@ -90,5 +114,5 @@ you should assign the new shape to the shape attribute of the array ''' if not hasattr(a, 'reshape'): - a = numpypy.array(a) + a = _numpypy.array(a) return a.reshape(shape) diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py --- a/pypy/module/micronumpy/compile.py +++ b/pypy/module/micronumpy/compile.py @@ -373,13 +373,17 @@ def execute(self, interp): if self.name in SINGLE_ARG_FUNCTIONS: - if len(self.args) != 1: + if len(self.args) != 1 and self.name != 'sum': raise ArgumentMismatch arr = self.args[0].execute(interp) if not isinstance(arr, BaseArray): raise ArgumentNotAnArray if self.name == "sum": - w_res = arr.descr_sum(interp.space) + if len(self.args)>1: + w_res = arr.descr_sum(interp.space, + self.args[1].execute(interp)) + else: + w_res = arr.descr_sum(interp.space) elif self.name == "prod": w_res = arr.descr_prod(interp.space) elif self.name == "max": @@ -430,7 +434,7 @@ ('\]', 'array_right'), ('(->)|[\+\-\*\/]', 'operator'), ('=', 'assign'), - (',', 'coma'), + (',', 'comma'), ('\|', 'pipe'), ('$', 'paren_left'), ('$', 'paren_right'), @@ -518,7 +522,7 @@ return SliceConstant(start, stop, step) - def parse_expression(self, tokens): + def parse_expression(self, tokens, accept_comma=False): stack = [] while tokens.remaining(): token = tokens.pop() @@ -538,9 +542,13 @@ stack.append(RangeConstant(tokens.pop().v)) end = tokens.pop() assert end.name == 'pipe' + elif accept_comma and token.name == 'comma': + continue else: tokens.push() break + if accept_comma: + return stack stack.reverse() lhs = stack.pop() while stack: @@ -554,10 +562,7 @@ args = [] tokens.pop() # lparen while tokens.get(0).name != 'paren_right': - if tokens.get(0).name == 'coma': - tokens.pop() - continue - args.append(self.parse_expression(tokens)) + args += self.parse_expression(tokens, accept_comma=True) return FunctionCall(name, args) def parse_array_const(self, tokens): @@ -573,7 +578,7 @@ token = tokens.pop() if token.name == 'array_right': return elems - assert token.name == 'coma' + assert token.name == 'comma' def parse_statement(self, tokens): if (tokens.get(0).name == 'identifier' and diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py --- a/pypy/module/micronumpy/interp_boxes.py +++ b/pypy/module/micronumpy/interp_boxes.py @@ -78,6 +78,7 @@ descr_sub = _binop_impl("subtract") descr_mul = _binop_impl("multiply") descr_div = _binop_impl("divide") + descr_pow = _binop_impl("power") descr_eq = _binop_impl("equal") descr_ne = _binop_impl("not_equal") descr_lt = _binop_impl("less") @@ -170,6 +171,7 @@ __sub__ = interp2app(W_GenericBox.descr_sub), __mul__ = interp2app(W_GenericBox.descr_mul), __div__ = interp2app(W_GenericBox.descr_div), + __pow__ = interp2app(W_GenericBox.descr_pow), __radd__ = interp2app(W_GenericBox.descr_radd), __rsub__ = interp2app(W_GenericBox.descr_rsub), @@ -245,6 +247,7 @@ long_name = "int64" W_LongBox.typedef = TypeDef(long_name, (W_SignedIntegerBox.typedef, int_typedef,), __module__ = "numpypy", + __new__ = interp2app(W_LongBox.descr__new__.im_func), ) W_ULongBox.typedef = TypeDef("u" + long_name, W_UnsignedIntegerBox.typedef, diff --git a/pypy/module/micronumpy/interp_extras.py b/pypy/module/micronumpy/interp_extras.py --- a/pypy/module/micronumpy/interp_extras.py +++ b/pypy/module/micronumpy/interp_extras.py @@ -5,3 +5,11 @@ @unwrap_spec(array=BaseArray) def debug_repr(space, array): return space.wrap(array.find_sig().debug_repr()) + + at unwrap_spec(array=BaseArray) +def remove_invalidates(space, array): + """ Array modification will no longer invalidate any of it's + potential children. Use only for performance debugging + """ + del array.invalidates[:] + return space.w_None diff --git a/pypy/module/micronumpy/interp_iter.py b/pypy/module/micronumpy/interp_iter.py --- a/pypy/module/micronumpy/interp_iter.py +++ b/pypy/module/micronumpy/interp_iter.py @@ -1,19 +1,20 @@ from pypy.rlib import jit from pypy.rlib.objectmodel import instantiate -from pypy.module.micronumpy.strides import calculate_broadcast_strides +from pypy.module.micronumpy.strides import calculate_broadcast_strides,\ + calculate_slice_strides -# Iterators for arrays -# -------------------- -# all those iterators with the exception of BroadcastIterator iterate over the -# entire array in C order (the last index changes the fastest). This will -# yield all elements. Views iterate over indices and look towards strides and -# backstrides to find the correct position. Notably the offset between -# x[..., i + 1] and x[..., i] will be strides[-1]. Offset between -# x[..., k + 1, 0] and x[..., k, i_max] will be backstrides[-2] etc. +class BaseTransform(object): + pass -# BroadcastIterator works like that, but for indexes that don't change source -# in the original array, strides[i] == backstrides[i] == 0 +class ViewTransform(BaseTransform): + def __init__(self, chunks): + # 4-tuple specifying slicing + self.chunks = chunks + +class BroadcastTransform(BaseTransform): + def __init__(self, res_shape): + self.res_shape = res_shape class BaseIterator(object): def next(self, shapelen): @@ -22,6 +23,15 @@ def done(self): raise NotImplementedError + def apply_transformations(self, arr, transformations): + v = self + for transform in transformations: + v = v.transform(arr, transform) + return v + + def transform(self, arr, t): + raise NotImplementedError + class ArrayIterator(BaseIterator): def __init__(self, size): self.offset = 0 @@ -36,6 +46,10 @@ def done(self): return self.offset >= self.size + def transform(self, arr, t): + return ViewIterator(arr.start, arr.strides, arr.backstrides, + arr.shape).transform(arr, t) + class OneDimIterator(BaseIterator): def __init__(self, start, step, stop): self.offset = start @@ -52,26 +66,29 @@ def done(self): return self.offset == self.size -def view_iter_from_arr(arr): - return ViewIterator(arr.start, arr.strides, arr.backstrides, arr.shape) - class ViewIterator(BaseIterator): - def __init__(self, start, strides, backstrides, shape, res_shape=None): + def __init__(self, start, strides, backstrides, shape): self.offset = start self._done = False - if res_shape is not None and res_shape != shape: - r = calculate_broadcast_strides(strides, backstrides, - shape, res_shape) - self.strides, self.backstrides = r - self.res_shape = res_shape - else: - self.strides = strides - self.backstrides = backstrides - self.res_shape = shape + self.strides = strides + self.backstrides = backstrides + self.res_shape = shape self.indices = [0] * len(self.res_shape) + def transform(self, arr, t): + if isinstance(t, BroadcastTransform): + r = calculate_broadcast_strides(self.strides, self.backstrides, + self.res_shape, t.res_shape) + return ViewIterator(self.offset, r[0], r[1], t.res_shape) + elif isinstance(t, ViewTransform): + r = calculate_slice_strides(self.res_shape, self.offset, + self.strides, + self.backstrides, t.chunks) + return ViewIterator(r[1], r[2], r[3], r[0]) + @jit.unroll_safe def next(self, shapelen): + shapelen = jit.promote(len(self.res_shape)) offset = self.offset indices = [0] * shapelen for i in range(shapelen): @@ -96,6 +113,13 @@ res._done = done return res + def apply_transformations(self, arr, transformations): + v = BaseIterator.apply_transformations(self, arr, transformations) + if len(arr.shape) == 1: + return OneDimIterator(self.offset, self.strides[0], + self.res_shape[0]) + return v + def done(self): return self._done @@ -103,11 +127,57 @@ def next(self, shapelen): return self + def transform(self, arr, t): + pass + +class AxisIterator(BaseIterator): + def __init__(self, start, dim, shape, strides, backstrides): + self.res_shape = shape[:] + self.strides = strides[:dim] + [0] + strides[dim:] + self.backstrides = backstrides[:dim] + [0] + backstrides[dim:] + self.first_line = True + self.indices = [0] * len(shape) + self._done = False + self.offset = start + self.dim = dim + + @jit.unroll_safe + def next(self, shapelen): + offset = self.offset + first_line = self.first_line + indices = [0] * shapelen + for i in range(shapelen): + indices[i] = self.indices[i] + done = False + for i in range(shapelen - 1, -1, -1): + if indices[i] < self.res_shape[i] - 1: + if i == self.dim: + first_line = False + indices[i] += 1 + offset += self.strides[i] + break + else: + indices[i] = 0 + offset -= self.backstrides[i] + else: + done = True + res = instantiate(AxisIterator) + res.offset = offset + res.indices = indices + res.strides = self.strides + res.backstrides = self.backstrides + res.res_shape = self.res_shape + res._done = done + res.first_line = first_line + res.dim = self.dim + return res + + def done(self): + return self._done + # ------ other iterators that are not part of the computation frame ---------- - -class AxisIterator(object): - """ This object will return offsets of each start of the last stride - """ + +class SkipLastAxisIterator(object): def __init__(self, arr): self.arr = arr self.indices = [0] * (len(arr.shape) - 1) @@ -125,4 +195,3 @@ self.offset -= self.arr.backstrides[i] else: self.done = True - diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -1,6 +1,6 @@ from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.error import OperationError, operationerrfmt -from pypy.interpreter.gateway import interp2app, unwrap_spec, NoneNotWrapped +from pypy.interpreter.gateway import interp2app, NoneNotWrapped from pypy.interpreter.typedef import TypeDef, GetSetProperty from pypy.module.micronumpy import interp_ufuncs, interp_dtype, signature from pypy.module.micronumpy.strides import calculate_slice_strides @@ -8,30 +8,39 @@ from pypy.rpython.lltypesystem import lltype, rffi from pypy.tool.sourcetools import func_with_new_name from pypy.rlib.rstring import StringBuilder -from pypy.module.micronumpy.interp_iter import ArrayIterator,\ - view_iter_from_arr, OneDimIterator, AxisIterator +from pypy.module.micronumpy.interp_iter import ArrayIterator, OneDimIterator,\ + SkipLastAxisIterator numpy_driver = jit.JitDriver( greens=['shapelen', 'sig'], virtualizables=['frame'], - reds=['result_size', 'frame', 'ri', 'self', 'result'] + reds=['result_size', 'frame', 'ri', 'self', 'result'], + get_printable_location=signature.new_printable_location('numpy'), + name='numpy', ) all_driver = jit.JitDriver( greens=['shapelen', 'sig'], virtualizables=['frame'], - reds=['frame', 'self', 'dtype'] + reds=['frame', 'self', 'dtype'], + get_printable_location=signature.new_printable_location('all'), + name='numpy_all', ) any_driver = jit.JitDriver( greens=['shapelen', 'sig'], virtualizables=['frame'], - reds=['frame', 'self', 'dtype'] + reds=['frame', 'self', 'dtype'], + get_printable_location=signature.new_printable_location('any'), + name='numpy_any', ) slice_driver = jit.JitDriver( greens=['shapelen', 'sig'], virtualizables=['frame'], - reds=['self', 'frame', 'source', 'res_iter'] + reds=['self', 'frame', 'arr'], + get_printable_location=signature.new_printable_location('slice'), + name='numpy_slice', ) + def _find_shape_and_elems(space, w_iterable): shape = [space.len_w(w_iterable)] batch = space.listview(w_iterable) @@ -148,9 +157,6 @@ # (meaning that the realignment of elements crosses from one step into another) # return None so that the caller can raise an exception. def calc_new_strides(new_shape, old_shape, old_strides): - # Return the proper strides for new_shape, or None if the mapping crosses - # stepping boundaries - # Assumes that prod(old_shape) == prod(new_shape), len(old_shape) > 1, and # len(new_shape) > 0 steps = [] @@ -158,6 +164,7 @@ oldI = 0 new_strides = [] if old_strides[0] < old_strides[-1]: + #Start at old_shape[0], old_stides[0] for i in range(len(old_shape)): steps.append(old_strides[i] / last_step) last_step *= old_shape[i] @@ -175,10 +182,11 @@ if n_new_elems_used == n_old_elems_to_use: oldI += 1 if oldI >= len(old_shape): - break + continue cur_step = steps[oldI] n_old_elems_to_use *= old_shape[oldI] else: + #Start at old_shape[-1], old_strides[-1] for i in range(len(old_shape) - 1, -1, -1): steps.insert(0, old_strides[i] / last_step) last_step *= old_shape[i] @@ -197,7 +205,7 @@ if n_new_elems_used == n_old_elems_to_use: oldI -= 1 if oldI < -len(old_shape): - break + continue cur_step = steps[oldI] n_old_elems_to_use *= old_shape[oldI] return new_strides @@ -277,21 +285,26 @@ descr_rpow = _binop_right_impl("power") descr_rmod = _binop_right_impl("mod") - def _reduce_ufunc_impl(ufunc_name): - def impl(self, space): + def _reduce_ufunc_impl(ufunc_name, promote_to_largest=False): + def impl(self, space, w_dim=None): + if space.is_w(w_dim, space.w_None): + w_dim = space.wrap(-1) return getattr(interp_ufuncs.get(space), ufunc_name).reduce(space, - self, multidim=True) + self, True, promote_to_largest, w_dim) return func_with_new_name(impl, "reduce_%s_impl" % ufunc_name) descr_sum = _reduce_ufunc_impl("add") - descr_prod = _reduce_ufunc_impl("multiply") + descr_sum_promote = _reduce_ufunc_impl("add", True) + descr_prod = _reduce_ufunc_impl("multiply", True) descr_max = _reduce_ufunc_impl("maximum") descr_min = _reduce_ufunc_impl("minimum") def _reduce_argmax_argmin_impl(op_name): reduce_driver = jit.JitDriver( greens=['shapelen', 'sig'], - reds=['result', 'idx', 'frame', 'self', 'cur_best', 'dtype'] + reds=['result', 'idx', 'frame', 'self', 'cur_best', 'dtype'], + get_printable_location=signature.new_printable_location(op_name), + name='numpy_' + op_name, ) def loop(self): sig = self.find_sig() @@ -377,7 +390,7 @@ elif len(self.shape) < 2 and len(w_other.shape) < 2: w_res = self.descr_mul(space, w_other) assert isinstance(w_res, BaseArray) - return w_res.descr_sum(space) + return w_res.descr_sum(space, space.wrap(-1)) dtype = interp_ufuncs.find_binop_result_dtype(space, self.find_dtype(), w_other.find_dtype()) if self.size < 1 and w_other.size < 1: @@ -438,6 +451,10 @@ def descr_get_dtype(self, space): return space.wrap(self.find_dtype()) + def descr_get_ndim(self, space): + return space.wrap(len(self.shape)) + + @jit.unroll_safe def descr_get_shape(self, space): return space.newtuple([space.wrap(i) for i in self.shape]) @@ -466,7 +483,7 @@ def descr_repr(self, space): res = StringBuilder() res.append("array(") - concrete = self.get_concrete() + concrete = self.get_concrete_or_scalar() dtype = concrete.find_dtype() if not concrete.size: res.append('[]') @@ -479,8 +496,9 @@ else: concrete.to_str(space, 1, res, indent=' ') if (dtype is not interp_dtype.get_dtype_cache(space).w_float64dtype and - dtype is not interp_dtype.get_dtype_cache(space).w_int64dtype) or \ - not self.size: + not (dtype.kind == interp_dtype.SIGNEDLTR and + dtype.itemtype.get_element_size() == rffi.sizeof(lltype.Signed)) or + not self.size): res.append(", dtype=" + dtype.name) res.append(")") return space.wrap(res.build()) @@ -613,9 +631,26 @@ ) return w_result - def descr_mean(self, space): - return space.div(self.descr_sum(space), - space.wrap(self.size)) + def descr_mean(self, space, w_dim=None): + if space.is_w(w_dim, space.w_None): + w_dim = space.wrap(-1) + w_denom = space.wrap(self.size) + else: + dim = space.int_w(w_dim) + w_denom = space.wrap(self.shape[dim]) + return space.div(self.descr_sum_promote(space, w_dim), w_denom) + + def descr_var(self, space): + # var = mean((values - mean(values)) ** 2) + w_res = self.descr_sub(space, self.descr_mean(space, space.w_None)) + assert isinstance(w_res, BaseArray) + w_res = w_res.descr_pow(space, space.wrap(2)) + assert isinstance(w_res, BaseArray) + return w_res.descr_mean(space, space.w_None) + + def descr_std(self, space): + # std(v) = sqrt(var(v)) + return interp_ufuncs.get(space).sqrt.call(space, [self.descr_var(space)]) def descr_nonzero(self, space): if self.size > 1: @@ -641,8 +676,8 @@ strides.append(concrete.strides[i]) backstrides.append(concrete.backstrides[i]) shape.append(concrete.shape[i]) - return space.wrap(W_NDimSlice(concrete.start, strides[:], - backstrides[:], shape[:], concrete)) + return space.wrap(W_NDimSlice(concrete.start, strides, + backstrides, shape, concrete)) def descr_get_flatiter(self, space): return space.wrap(W_FlatIterator(self)) @@ -650,11 +685,12 @@ def getitem(self, item): raise NotImplementedError - def find_sig(self, res_shape=None): + def find_sig(self, res_shape=None, arr=None): """ find a correct signature for the array """ res_shape = res_shape or self.shape - return signature.find_sig(self.create_sig(res_shape), self) + arr = arr or self + return signature.find_sig(self.create_sig(), arr) def descr_array_iface(self, space): if not self.shape: @@ -708,7 +744,7 @@ def copy(self, space): return Scalar(self.dtype, self.value) - def create_sig(self, res_shape): + def create_sig(self): return signature.ScalarSignature(self.dtype) def get_concrete_or_scalar(self): @@ -726,7 +762,8 @@ self.name = name def _del_sources(self): - # Function for deleting references to source arrays, to allow garbage-collecting them + # Function for deleting references to source arrays, + # to allow garbage-collecting them raise NotImplementedError def compute(self): @@ -778,11 +815,11 @@ self.size = size VirtualArray.__init__(self, 'slice', shape, child.find_dtype()) - def create_sig(self, res_shape): + def create_sig(self): if self.forced_result is not None: - return self.forced_result.create_sig(res_shape) + return self.forced_result.create_sig() return signature.VirtualSliceSignature( - self.child.create_sig(res_shape)) + self.child.create_sig()) def force_if_needed(self): if self.forced_result is None: @@ -792,6 +829,7 @@ def _del_sources(self): self.child = None + class Call1(VirtualArray): def __init__(self, ufunc, name, shape, res_dtype, values): VirtualArray.__init__(self, name, shape, res_dtype) @@ -802,16 +840,17 @@ def _del_sources(self): self.values = None - def create_sig(self, res_shape): + def create_sig(self): if self.forced_result is not None: - return self.forced_result.create_sig(res_shape) - return signature.Call1(self.ufunc, self.name, - self.values.create_sig(res_shape)) + return self.forced_result.create_sig() + return signature.Call1(self.ufunc, self.name, self.values.create_sig()) class Call2(VirtualArray): """ Intermediate class for performing binary operations. """ + _immutable_fields_ = ['left', 'right'] + def __init__(self, ufunc, name, shape, calc_dtype, res_dtype, left, right): VirtualArray.__init__(self, name, shape, res_dtype) self.ufunc = ufunc @@ -826,12 +865,55 @@ self.left = None self.right = None - def create_sig(self, res_shape): + def create_sig(self): if self.forced_result is not None: - return self.forced_result.create_sig(res_shape) + return self.forced_result.create_sig() + if self.shape != self.left.shape and self.shape != self.right.shape: + return signature.BroadcastBoth(self.ufunc, self.name, + self.calc_dtype, + self.left.create_sig(), + self.right.create_sig()) + elif self.shape != self.left.shape: + return signature.BroadcastLeft(self.ufunc, self.name, + self.calc_dtype, + self.left.create_sig(), + self.right.create_sig()) + elif self.shape != self.right.shape: + return signature.BroadcastRight(self.ufunc, self.name, + self.calc_dtype, + self.left.create_sig(), + self.right.create_sig()) return signature.Call2(self.ufunc, self.name, self.calc_dtype, - self.left.create_sig(res_shape), - self.right.create_sig(res_shape)) + self.left.create_sig(), self.right.create_sig()) + +class SliceArray(Call2): + def __init__(self, shape, dtype, left, right): + Call2.__init__(self, None, 'sliceloop', shape, dtype, dtype, left, + right) + + def create_sig(self): + lsig = self.left.create_sig() + rsig = self.right.create_sig() + if self.shape != self.right.shape: + return signature.SliceloopBroadcastSignature(self.ufunc, + self.name, + self.calc_dtype, + lsig, rsig) + return signature.SliceloopSignature(self.ufunc, self.name, + self.calc_dtype, + lsig, rsig) + +class AxisReduce(Call2): + """ NOTE: this is only used as a container, you should never + encounter such things in the wild. Remove this comment + when we'll make AxisReduce lazy + """ + _immutable_fields_ = ['left', 'right'] + + def __init__(self, ufunc, name, shape, dtype, left, right, dim): + Call2.__init__(self, ufunc, name, shape, dtype, dtype, + left, right) + self.dim = dim class ConcreteArray(BaseArray): """ An array that have actual storage, whether owned or not @@ -883,13 +965,8 @@ if self.order == 'C': strides.reverse() backstrides.reverse() - self.strides = strides[:] - self.backstrides = backstrides[:] - - def array_sig(self, res_shape): - if res_shape is not None and self.shape != res_shape: - return signature.ViewSignature(self.dtype) - return signature.ArraySignature(self.dtype) + self.strides = strides + self.backstrides = backstrides def to_str(self, space, comma, builder, indent=' ', use_ellipsis=False): '''Modifies builder with a representation of the array/slice @@ -898,76 +975,80 @@ each line will begin with indent. ''' size = self.size + ccomma = ',' * comma + ncomma = ',' * (1 - comma) + dtype = self.find_dtype() if size < 1: builder.append('[]') return + elif size == 1: + builder.append(dtype.itemtype.str_format(self.getitem(0))) + return if size > 1000: # Once this goes True it does not go back to False for recursive # calls use_ellipsis = True - dtype = self.find_dtype() ndims = len(self.shape) i = 0 - start = True builder.append('[') if ndims > 1: if use_ellipsis: - for i in range(3): - if start: - start = False - else: - builder.append(',' * comma + '\n') - if ndims == 3: + for i in range(min(3, self.shape[0])): + if i > 0: + builder.append(ccomma + '\n') + if ndims >= 3: builder.append('\n' + indent) else: builder.append(indent) view = self.create_slice([(i, 0, 0, 1)]).get_concrete() - view.to_str(space, comma, builder, indent=indent + ' ', use_ellipsis=use_ellipsis) - builder.append('\n' + indent + '..., ') - i = self.shape[0] - 3 + view.to_str(space, comma, builder, indent=indent + ' ', + use_ellipsis=use_ellipsis) + if i < self.shape[0] - 1: + builder.append(ccomma + '\n' + indent + '...' + ncomma) + i = self.shape[0] - 3 + else: + i += 1 while i < self.shape[0]: - if start: - start = False - else: - builder.append(',' * comma + '\n') - if ndims == 3: + if i > 0: + builder.append(ccomma + '\n') + if ndims >= 3: builder.append('\n' + indent) else: builder.append(indent) + # create_slice requires len(chunks) > 1 in order to reduce + # shape view = self.create_slice([(i, 0, 0, 1)]).get_concrete() - view.to_str(space, comma, builder, indent=indent + ' ', use_ellipsis=use_ellipsis) + view.to_str(space, comma, builder, indent=indent + ' ', + use_ellipsis=use_ellipsis) i += 1 elif ndims == 1: - spacer = ',' * comma + ' ' + spacer = ccomma + ' ' item = self.start # An iterator would be a nicer way to walk along the 1d array, but # how do I reset it if printing ellipsis? iterators have no # "set_offset()" i = 0 if use_ellipsis: - for i in range(3): - if start: - start = False - else: + for i in range(min(3, self.shape[0])): + if i > 0: builder.append(spacer) builder.append(dtype.itemtype.str_format(self.getitem(item))) item += self.strides[0] - # Add a comma only if comma is False - this prevents adding two - # commas - builder.append(spacer + '...' + ',' * (1 - comma)) - # Ugly, but can this be done with an iterator? - item = self.start + self.backstrides[0] - 2 * self.strides[0] - i = self.shape[0] - 3 + if i < self.shape[0] - 1: + # Add a comma only if comma is False - this prevents adding + # two commas + builder.append(spacer + '...' + ncomma) + # Ugly, but can this be done with an iterator? + item = self.start + self.backstrides[0] - 2 * self.strides[0] + i = self.shape[0] - 3 + else: + i += 1 while i < self.shape[0]: - if start: - start = False - else: + if i > 0: builder.append(spacer) builder.append(dtype.itemtype.str_format(self.getitem(item))) item += self.strides[0] i += 1 - else: - builder.append('[') builder.append(']') @jit.unroll_safe @@ -1001,20 +1082,22 @@ self.dtype is w_value.find_dtype()): self._fast_setslice(space, w_value) else: - self._sliceloop(w_value, res_shape) + arr = SliceArray(self.shape, self.dtype, self, w_value) + self._sliceloop(arr) def _fast_setslice(self, space, w_value): assert isinstance(w_value, ConcreteArray) itemsize = self.dtype.itemtype.get_element_size() - if len(self.shape) == 1: + shapelen = len(self.shape) + if shapelen == 1: rffi.c_memcpy( rffi.ptradd(self.storage, self.start * itemsize), rffi.ptradd(w_value.storage, w_value.start * itemsize), self.size * itemsize ) else: - dest = AxisIterator(self) - source = AxisIterator(w_value) + dest = SkipLastAxisIterator(self) + source = SkipLastAxisIterator(w_value) while not dest.done: rffi.c_memcpy( rffi.ptradd(self.storage, dest.offset * itemsize), @@ -1024,21 +1107,16 @@ source.next() dest.next() - def _sliceloop(self, source, res_shape): - sig = source.find_sig(res_shape) - frame = sig.create_frame(source, res_shape) - res_iter = view_iter_from_arr(self) - shapelen = len(res_shape) - while not res_iter.done(): - slice_driver.jit_merge_point(sig=sig, - frame=frame, - shapelen=shapelen, - self=self, source=source, - res_iter=res_iter) - self.setitem(res_iter.offset, sig.eval(frame, source).convert_to( - self.find_dtype())) + def _sliceloop(self, arr): + sig = arr.find_sig() + frame = sig.create_frame(arr) + shapelen = len(self.shape) + while not frame.done(): + slice_driver.jit_merge_point(sig=sig, frame=frame, self=self, + arr=arr, + shapelen=shapelen) + sig.eval(frame, arr) frame.next(shapelen) - res_iter = res_iter.next(shapelen) def copy(self, space): array = W_NDimArray(self.size, self.shape[:], self.dtype, self.order) @@ -1047,7 +1125,7 @@ class ViewArray(ConcreteArray): - def create_sig(self, res_shape): + def create_sig(self): return signature.ViewSignature(self.dtype) @@ -1084,9 +1162,9 @@ strides.reverse() backstrides.reverse() new_shape.reverse() - self.strides = strides[:] - self.backstrides = backstrides[:] - self.shape = new_shape[:] + self.strides = strides + self.backstrides = backstrides + self.shape = new_shape return new_strides = calc_new_strides(new_shape, self.shape, self.strides) if new_strides is None: @@ -1096,7 +1174,7 @@ for nd in range(len(new_shape)): new_backstrides[nd] = (new_shape[nd] - 1) * new_strides[nd] self.strides = new_strides[:] - self.backstrides = new_backstrides[:] + self.backstrides = new_backstrides self.shape = new_shape[:] class W_NDimArray(ConcreteArray): @@ -1111,8 +1189,8 @@ self.shape = new_shape self.calc_strides(new_shape) - def create_sig(self, res_shape): - return self.array_sig(res_shape) + def create_sig(self): + return signature.ArraySignature(self.dtype) def __del__(self): lltype.free(self.storage, flavor='raw', track_allocation=False) @@ -1239,6 +1317,7 @@ shape = GetSetProperty(BaseArray.descr_get_shape, BaseArray.descr_set_shape), size = GetSetProperty(BaseArray.descr_get_size), + ndim = GetSetProperty(BaseArray.descr_get_ndim), T = GetSetProperty(BaseArray.descr_get_transpose), flat = GetSetProperty(BaseArray.descr_get_flatiter), @@ -1253,6 +1332,8 @@ all = interp2app(BaseArray.descr_all), any = interp2app(BaseArray.descr_any), dot = interp2app(BaseArray.descr_dot), + var = interp2app(BaseArray.descr_var), + std = interp2app(BaseArray.descr_std), copy = interp2app(BaseArray.descr_copy), reshape = interp2app(BaseArray.descr_reshape), @@ -1279,7 +1360,7 @@ def descr_next(self, space): if self.iter.done(): raise OperationError(space.w_StopIteration, space.w_None) - result = self.eval(self.iter) + result = self.getitem(self.iter.offset) self.iter = self.iter.next(self.shapelen) return result diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py --- a/pypy/module/micronumpy/interp_ufuncs.py +++ b/pypy/module/micronumpy/interp_ufuncs.py @@ -1,19 +1,31 @@ from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.error import OperationError, operationerrfmt -from pypy.interpreter.gateway import interp2app, unwrap_spec +from pypy.interpreter.gateway import interp2app from pypy.interpreter.typedef import TypeDef, GetSetProperty, interp_attrproperty -from pypy.module.micronumpy import interp_boxes, interp_dtype, types -from pypy.module.micronumpy.signature import ReduceSignature, ScalarSignature, find_sig +from pypy.module.micronumpy import interp_boxes, interp_dtype +from pypy.module.micronumpy.signature import ReduceSignature,\ + find_sig, new_printable_location, AxisReduceSignature, ScalarSignature from pypy.rlib import jit from pypy.rlib.rarithmetic import LONG_BIT from pypy.tool.sourcetools import func_with_new_name reduce_driver = jit.JitDriver( - greens = ['shapelen', "sig"], - virtualizables = ["frame"], - reds = ["frame", "self", "dtype", "value", "obj"] + greens=['shapelen', "sig"], + virtualizables=["frame"], + reds=["frame", "self", "dtype", "value", "obj"], + get_printable_location=new_printable_location('reduce'), + name='numpy_reduce', ) +axisreduce_driver = jit.JitDriver( + greens=['shapelen', 'sig'], + virtualizables=['frame'], + reds=['self','arr', 'identity', 'frame'], + name='numpy_axisreduce', + get_printable_location=new_printable_location('axisreduce'), +) + + class W_Ufunc(Wrappable): _attrs_ = ["name", "promote_to_float", "promote_bools", "identity"] _immutable_fields_ = ["promote_to_float", "promote_bools", "name"] @@ -46,18 +58,72 @@ ) return self.call(space, __args__.arguments_w) - def descr_reduce(self, space, w_obj): - return self.reduce(space, w_obj, multidim=False) + def descr_reduce(self, space, w_obj, w_dim=0): + """reduce(...) + reduce(a, axis=0) - def reduce(self, space, w_obj, multidim): - from pypy.module.micronumpy.interp_numarray import convert_to_array, Scalar - + Reduces `a`'s dimension by one, by applying ufunc along one axis. + + Let :math:`a.shape = (N_0, ..., N_i, ..., N_{M-1})`. Then + :math:`ufunc.reduce(a, axis=i)[k_0, ..,k_{i-1}, k_{i+1}, .., k_{M-1}]` = + the result of iterating `j` over :math:`range(N_i)`, cumulatively applying + ufunc to each :math:`a[k_0, ..,k_{i-1}, j, k_{i+1}, .., k_{M-1}]`. + For a one-dimensional array, reduce produces results equivalent to: + :: + + r = op.identity # op = ufunc + for i in xrange(len(A)): + r = op(r, A[i]) + return r + + For example, add.reduce() is equivalent to sum(). + + Parameters + ---------- + a : array_like + The array to act on. + axis : int, optional + The axis along which to apply the reduction. + + Examples + -------- + >>> np.multiply.reduce([2,3,5]) + 30 + + A multi-dimensional array example: + + >>> X = np.arange(8).reshape((2,2,2)) + >>> X + array([[[0, 1], + [2, 3]], + [[4, 5], + [6, 7]]]) + >>> np.add.reduce(X, 0) + array([[ 4, 6], + [ 8, 10]]) + >>> np.add.reduce(X) # confirm: default axis value is 0 + array([[ 4, 6], + [ 8, 10]]) + >>> np.add.reduce(X, 1) + array([[ 2, 4], + [10, 12]]) + >>> np.add.reduce(X, 2) + array([[ 1, 5], + [ 9, 13]]) + """ + return self.reduce(space, w_obj, False, False, w_dim) + + def reduce(self, space, w_obj, multidim, promote_to_largest, w_dim): + from pypy.module.micronumpy.interp_numarray import convert_to_array, \ + Scalar if self.argcount != 2: raise OperationError(space.w_ValueError, space.wrap("reduce only " "supported for binary functions")) - + dim = space.int_w(w_dim) assert isinstance(self, W_Ufunc2) obj = convert_to_array(space, w_obj) + if dim >= len(obj.shape): + raise OperationError(space.w_ValueError, space.wrap("axis(=%d) out of bounds" % dim)) if isinstance(obj, Scalar): raise OperationError(space.w_TypeError, space.wrap("cannot reduce " "on a scalar")) @@ -65,26 +131,80 @@ size = obj.size dtype = find_unaryop_result_dtype( space, obj.find_dtype(), - promote_to_largest=True + promote_to_float=self.promote_to_float, + promote_to_largest=promote_to_largest, + promote_bools=True ) shapelen = len(obj.shape) + if self.identity is None and size == 0: + raise operationerrfmt(space.w_ValueError, "zero-size array to " + "%s.reduce without identity", self.name) + if shapelen > 1 and dim >= 0: + res = self.do_axis_reduce(obj, dtype, dim) + return space.wrap(res) + scalarsig = ScalarSignature(dtype) sig = find_sig(ReduceSignature(self.func, self.name, dtype, - ScalarSignature(dtype), - obj.create_sig(obj.shape)), obj) + scalarsig, + obj.create_sig()), obj) frame = sig.create_frame(obj) - if shapelen > 1 and not multidim: - raise OperationError(space.w_NotImplementedError, - space.wrap("not implemented yet")) if self.identity is None: - if size == 0: - raise operationerrfmt(space.w_ValueError, "zero-size array to " - "%s.reduce without identity", self.name) value = sig.eval(frame, obj).convert_to(dtype) frame.next(shapelen) else: value = self.identity.convert_to(dtype) return self.reduce_loop(shapelen, sig, frame, value, obj, dtype) + def do_axis_reduce(self, obj, dtype, dim): + from pypy.module.micronumpy.interp_numarray import AxisReduce,\ + W_NDimArray + + shape = obj.shape[0:dim] + obj.shape[dim + 1:len(obj.shape)] + size = 1 + for s in shape: + size *= s + result = W_NDimArray(size, shape, dtype) + rightsig = obj.create_sig() + # note - this is just a wrapper so signature can fetch + # both left and right, nothing more, especially + # this is not a true virtual array, because shapes + # don't quite match + arr = AxisReduce(self.func, self.name, obj.shape, dtype, + result, obj, dim) + scalarsig = ScalarSignature(dtype) + sig = find_sig(AxisReduceSignature(self.func, self.name, dtype, + scalarsig, rightsig), arr) + assert isinstance(sig, AxisReduceSignature) + frame = sig.create_frame(arr) + shapelen = len(obj.shape) + if self.identity is not None: + identity = self.identity.convert_to(dtype) + else: + identity = None + self.reduce_axis_loop(frame, sig, shapelen, arr, identity) + return result + + def reduce_axis_loop(self, frame, sig, shapelen, arr, identity): + # note - we can be advanterous here, depending on the exact field + # layout. For now let's say we iterate the original way and + # simply follow the original iteration order + while not frame.done(): + axisreduce_driver.jit_merge_point(frame=frame, self=self, + sig=sig, + identity=identity, + shapelen=shapelen, arr=arr) + iter = frame.get_final_iter() + v = sig.eval(frame, arr).convert_to(sig.calc_dtype) + if iter.first_line: + if identity is not None: + value = self.func(sig.calc_dtype, identity, v) + else: + value = v + else: + cur = arr.left.getitem(iter.offset) + value = self.func(sig.calc_dtype, cur, v) + arr.left.setitem(iter.offset, value) + frame.next(shapelen) + def reduce_loop(self, shapelen, sig, frame, value, obj, dtype): while not frame.done(): reduce_driver.jit_merge_point(sig=sig, @@ -92,10 +212,12 @@ value=value, obj=obj, frame=frame, dtype=dtype) assert isinstance(sig, ReduceSignature) - value = sig.binfunc(dtype, value, sig.eval(frame, obj).convert_to(dtype)) + value = sig.binfunc(dtype, value, + sig.eval(frame, obj).convert_to(dtype)) frame.next(shapelen) return value + class W_Ufunc1(W_Ufunc): argcount = 1 @@ -180,6 +302,7 @@ reduce = interp2app(W_Ufunc.descr_reduce), ) + def find_binop_result_dtype(space, dt1, dt2, promote_to_float=False, promote_bools=False): # dt1.num should be <= dt2.num @@ -228,6 +351,7 @@ dtypenum += 3 return interp_dtype.get_dtype_cache(space).builtin_dtypes[dtypenum] + def find_unaryop_result_dtype(space, dt, promote_to_float=False, promote_bools=False, promote_to_largest=False): if promote_bools and (dt.kind == interp_dtype.BOOLLTR): @@ -252,6 +376,7 @@ assert False return dt + def find_dtype_for_scalar(space, w_obj, current_guess=None): bool_dtype = interp_dtype.get_dtype_cache(space).w_booldtype long_dtype = interp_dtype.get_dtype_cache(space).w_longdtype @@ -345,7 +470,8 @@ identity = extra_kwargs.get("identity") if identity is not None: - identity = interp_dtype.get_dtype_cache(space).w_longdtype.box(identity) + identity = \ + interp_dtype.get_dtype_cache(space).w_longdtype.box(identity) extra_kwargs["identity"] = identity func = ufunc_dtype_caller(space, ufunc_name, op_name, argcount, diff --git a/pypy/module/micronumpy/signature.py b/pypy/module/micronumpy/signature.py --- a/pypy/module/micronumpy/signature.py +++ b/pypy/module/micronumpy/signature.py @@ -1,10 +1,37 @@ from pypy.rlib.objectmodel import r_dict, compute_identity_hash, compute_hash from pypy.rlib.rarithmetic import intmask from pypy.module.micronumpy.interp_iter import ViewIterator, ArrayIterator, \ - OneDimIterator, ConstantIterator -from pypy.module.micronumpy.strides import calculate_slice_strides + ConstantIterator, AxisIterator, ViewTransform,\ + BroadcastTransform from pypy.rlib.jit import hint, unroll_safe, promote +""" Signature specifies both the numpy expression that has been constructed +and the assembler to be compiled. This is a very important observation - +Two expressions will be using the same assembler if and only if they are +compiled to the same signature. + +This is also a very convinient tool for specializations. For example +a + a and a + b (where a != b) will compile to different assembler because +we specialize on the same array access. + +When evaluating, signatures will create iterators per signature node, +potentially sharing some of them. Iterators depend also on the actual +expression, they're not only dependant on the array itself. For example +a + b where a is dim 2 and b is dim 1 would create a broadcasted iterator for +the array b. + +Such iterator changes are called Transformations. An actual iterator would +be a combination of array and various transformation, like view, broadcast, +dimension swapping etc. + +See interp_iter for transformations +""" + +def new_printable_location(driver_name): + def get_printable_location(shapelen, sig): + return 'numpy ' + sig.debug_repr() + ' [%d dims,%s]' % (shapelen, driver_name) + return get_printable_location + def sigeq(one, two): return one.eq(two) @@ -28,7 +55,8 @@ return sig class NumpyEvalFrame(object): - _virtualizable2_ = ['iterators[*]', 'final_iter', 'arraylist[*]'] + _virtualizable2_ = ['iterators[*]', 'final_iter', 'arraylist[*]', + 'value', 'identity'] @unroll_safe def __init__(self, iterators, arrays): @@ -46,7 +74,7 @@ def done(self): final_iter = promote(self.final_iter) if final_iter < 0: - return False + assert False return self.iterators[final_iter].done() @unroll_safe @@ -54,6 +82,12 @@ for i in range(len(self.iterators)): self.iterators[i] = self.iterators[i].next(shapelen) + def get_final_iter(self): + final_iter = promote(self.final_iter) + if final_iter < 0: + assert False + return self.iterators[final_iter] + def _add_ptr_to_cache(ptr, cache): i = 0 for p in cache: @@ -65,6 +99,9 @@ cache.append(ptr) return res +def new_cache(): + return r_dict(sigeq_no_numbering, sighash) + class Signature(object): _attrs_ = ['iter_no', 'array_no'] _immutable_fields_ = ['iter_no', 'array_no'] @@ -73,7 +110,7 @@ iter_no = 0 def invent_numbering(self): - cache = r_dict(sigeq_no_numbering, sighash) + cache = new_cache() allnumbers = [] self._invent_numbering(cache, allnumbers) @@ -90,13 +127,13 @@ allnumbers.append(no) self.iter_no = no - def create_frame(self, arr, res_shape=None): - res_shape = res_shape or arr.shape + def create_frame(self, arr): iterlist = [] arraylist = [] - self._create_iter(iterlist, arraylist, arr, res_shape, []) + self._create_iter(iterlist, arraylist, arr, []) return NumpyEvalFrame(iterlist, arraylist) + class ConcreteSignature(Signature): _immutable_fields_ = ['dtype'] @@ -115,16 +152,6 @@ def hash(self): return compute_identity_hash(self.dtype) - def allocate_view_iter(self, arr, res_shape, chunklist): - r = arr.shape, arr.start, arr.strides, arr.backstrides - if chunklist: - for chunkelem in chunklist: - r = calculate_slice_strides(r[0], r[1], r[2], r[3], chunkelem) - shape, start, strides, backstrides = r - if len(res_shape) == 1: - return OneDimIterator(start, strides[0], res_shape[0]) - return ViewIterator(start, strides, backstrides, shape, res_shape) - class ArraySignature(ConcreteSignature): def debug_repr(self): return 'Array' @@ -132,23 +159,25 @@ def _invent_array_numbering(self, arr, cache): from pypy.module.micronumpy.interp_numarray import ConcreteArray concr = arr.get_concrete() + # this get_concrete never forces assembler. If we're here and array + # is not of a concrete class it means that we have a _forced_result, + # otherwise the signature would not match assert isinstance(concr, ConcreteArray) + assert concr.dtype is self.dtype self.array_no = _add_ptr_to_cache(concr.storage, cache) - def _create_iter(self, iterlist, arraylist, arr, res_shape, chunklist): + def _create_iter(self, iterlist, arraylist, arr, transforms): from pypy.module.micronumpy.interp_numarray import ConcreteArray concr = arr.get_concrete() assert isinstance(concr, ConcreteArray) storage = concr.storage if self.iter_no >= len(iterlist): - iterlist.append(self.allocate_iter(concr, res_shape, chunklist)) + iterlist.append(self.allocate_iter(concr, transforms)) if self.array_no >= len(arraylist): arraylist.append(storage) - def allocate_iter(self, arr, res_shape, chunklist): - if chunklist: - return self.allocate_view_iter(arr, res_shape, chunklist) - return ArrayIterator(arr.size) + def allocate_iter(self, arr, transforms): + return ArrayIterator(arr.size).apply_transformations(arr, transforms) def eval(self, frame, arr): iter = frame.iterators[self.iter_no] @@ -161,7 +190,7 @@ def _invent_array_numbering(self, arr, cache): pass - def _create_iter(self, iterlist, arraylist, arr, res_shape, chunklist): + def _create_iter(self, iterlist, arraylist, arr, transforms): if self.iter_no >= len(iterlist): iter = ConstantIterator() iterlist.append(iter) @@ -181,8 +210,9 @@ allnumbers.append(no) self.iter_no = no - def allocate_iter(self, arr, res_shape, chunklist): - return self.allocate_view_iter(arr, res_shape, chunklist) + def allocate_iter(self, arr, transforms): + return ViewIterator(arr.start, arr.strides, arr.backstrides, + arr.shape).apply_transformations(arr, transforms) class VirtualSliceSignature(Signature): def __init__(self, child): @@ -193,6 +223,9 @@ assert isinstance(arr, VirtualSlice) self.child._invent_array_numbering(arr.child, cache) + def _invent_numbering(self, cache, allnumbers): + self.child._invent_numbering(new_cache(), allnumbers) + def hash(self): return intmask(self.child.hash() ^ 1234) @@ -202,12 +235,11 @@ assert isinstance(other, VirtualSliceSignature) return self.child.eq(other.child, compare_array_no) - def _create_iter(self, iterlist, arraylist, arr, res_shape, chunklist): + def _create_iter(self, iterlist, arraylist, arr, transforms): from pypy.module.micronumpy.interp_numarray import VirtualSlice assert isinstance(arr, VirtualSlice) - chunklist.append(arr.chunks) - self.child._create_iter(iterlist, arraylist, arr.child, res_shape, - chunklist) + transforms = transforms + [ViewTransform(arr.chunks)] + self.child._create_iter(iterlist, arraylist, arr.child, transforms) def eval(self, frame, arr): from pypy.module.micronumpy.interp_numarray import VirtualSlice @@ -243,11 +275,10 @@ assert isinstance(arr, Call1) self.child._invent_array_numbering(arr.values, cache) - def _create_iter(self, iterlist, arraylist, arr, res_shape, chunklist): + def _create_iter(self, iterlist, arraylist, arr, transforms): from pypy.module.micronumpy.interp_numarray import Call1 assert isinstance(arr, Call1) - self.child._create_iter(iterlist, arraylist, arr.values, res_shape, - chunklist) + self.child._create_iter(iterlist, arraylist, arr.values, transforms) def eval(self, frame, arr): from pypy.module.micronumpy.interp_numarray import Call1 @@ -288,29 +319,68 @@ self.left._invent_numbering(cache, allnumbers) self.right._invent_numbering(cache, allnumbers) - def _create_iter(self, iterlist, arraylist, arr, res_shape, chunklist): + def _create_iter(self, iterlist, arraylist, arr, transforms): from pypy.module.micronumpy.interp_numarray import Call2 assert isinstance(arr, Call2) - self.left._create_iter(iterlist, arraylist, arr.left, res_shape, - chunklist) - self.right._create_iter(iterlist, arraylist, arr.right, res_shape, - chunklist) + self.left._create_iter(iterlist, arraylist, arr.left, transforms) + self.right._create_iter(iterlist, arraylist, arr.right, transforms) def eval(self, frame, arr): from pypy.module.micronumpy.interp_numarray import Call2 assert isinstance(arr, Call2) lhs = self.left.eval(frame, arr.left).convert_to(self.calc_dtype) rhs = self.right.eval(frame, arr.right).convert_to(self.calc_dtype) + return self.binfunc(self.calc_dtype, lhs, rhs) def debug_repr(self): return 'Call2(%s, %s, %s)' % (self.name, self.left.debug_repr(), self.right.debug_repr()) +class BroadcastLeft(Call2): + def _invent_numbering(self, cache, allnumbers): + self.left._invent_numbering(new_cache(), allnumbers) + self.right._invent_numbering(cache, allnumbers) + + def _create_iter(self, iterlist, arraylist, arr, transforms): + from pypy.module.micronumpy.interp_numarray import Call2 + + assert isinstance(arr, Call2) + ltransforms = transforms + [BroadcastTransform(arr.shape)] + self.left._create_iter(iterlist, arraylist, arr.left, ltransforms) + self.right._create_iter(iterlist, arraylist, arr.right, transforms) + +class BroadcastRight(Call2): + def _invent_numbering(self, cache, allnumbers): + self.left._invent_numbering(cache, allnumbers) + self.right._invent_numbering(new_cache(), allnumbers) + + def _create_iter(self, iterlist, arraylist, arr, transforms): + from pypy.module.micronumpy.interp_numarray import Call2 + + assert isinstance(arr, Call2) + rtransforms = transforms + [BroadcastTransform(arr.shape)] + self.left._create_iter(iterlist, arraylist, arr.left, transforms) + self.right._create_iter(iterlist, arraylist, arr.right, rtransforms) + +class BroadcastBoth(Call2): + def _invent_numbering(self, cache, allnumbers): + self.left._invent_numbering(new_cache(), allnumbers) + self.right._invent_numbering(new_cache(), allnumbers) + + def _create_iter(self, iterlist, arraylist, arr, transforms): + from pypy.module.micronumpy.interp_numarray import Call2 + + assert isinstance(arr, Call2) + rtransforms = transforms + [BroadcastTransform(arr.shape)] + ltransforms = transforms + [BroadcastTransform(arr.shape)] + self.left._create_iter(iterlist, arraylist, arr.left, ltransforms) + self.right._create_iter(iterlist, arraylist, arr.right, rtransforms) + class ReduceSignature(Call2): - def _create_iter(self, iterlist, arraylist, arr, res_shape, chunklist): - self.right._create_iter(iterlist, arraylist, arr, res_shape, chunklist) + def _create_iter(self, iterlist, arraylist, arr, transforms): + self.right._create_iter(iterlist, arraylist, arr, transforms) def _invent_numbering(self, cache, allnumbers): self.right._invent_numbering(cache, allnumbers) @@ -320,3 +390,63 @@ def eval(self, frame, arr): return self.right.eval(frame, arr) + + def debug_repr(self): + return 'ReduceSig(%s, %s)' % (self.name, self.right.debug_repr()) + +class SliceloopSignature(Call2): + def eval(self, frame, arr): + from pypy.module.micronumpy.interp_numarray import Call2 + + assert isinstance(arr, Call2) + ofs = frame.iterators[0].offset + arr.left.setitem(ofs, self.right.eval(frame, arr.right).convert_to( + self.calc_dtype)) + + def debug_repr(self): + return 'SliceLoop(%s, %s, %s)' % (self.name, self.left.debug_repr(), + self.right.debug_repr()) + +class SliceloopBroadcastSignature(SliceloopSignature): + def _invent_numbering(self, cache, allnumbers): + self.left._invent_numbering(new_cache(), allnumbers) + self.right._invent_numbering(cache, allnumbers) + + def _create_iter(self, iterlist, arraylist, arr, transforms): + from pypy.module.micronumpy.interp_numarray import SliceArray + + assert isinstance(arr, SliceArray) + rtransforms = transforms + [BroadcastTransform(arr.shape)] + self.left._create_iter(iterlist, arraylist, arr.left, transforms) + self.right._create_iter(iterlist, arraylist, arr.right, rtransforms) + +class AxisReduceSignature(Call2): + def _create_iter(self, iterlist, arraylist, arr, transforms): + from pypy.module.micronumpy.interp_numarray import AxisReduce,\ + ConcreteArray + + assert isinstance(arr, AxisReduce) + left = arr.left + assert isinstance(left, ConcreteArray) + iterlist.append(AxisIterator(left.start, arr.dim, arr.shape, + left.strides, left.backstrides)) + self.right._create_iter(iterlist, arraylist, arr.right, transforms) + + def _invent_numbering(self, cache, allnumbers): + allnumbers.append(0) + self.right._invent_numbering(cache, allnumbers) + + def _invent_array_numbering(self, arr, cache): + from pypy.module.micronumpy.interp_numarray import AxisReduce + + assert isinstance(arr, AxisReduce) + self.right._invent_array_numbering(arr.right, cache) + + def eval(self, frame, arr): + from pypy.module.micronumpy.interp_numarray import AxisReduce + + assert isinstance(arr, AxisReduce) + return self.right.eval(frame, arr.right).convert_to(self.calc_dtype) + + def debug_repr(self): + return 'AxisReduceSig(%s, %s)' % (self.name, self.right.debug_repr()) diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py --- a/pypy/module/micronumpy/strides.py +++ b/pypy/module/micronumpy/strides.py @@ -1,4 +1,9 @@ +from pypy.rlib import jit + + at jit.look_inside_iff(lambda shape, start, strides, backstrides, chunks: + jit.isconstant(len(chunks)) +) def calculate_slice_strides(shape, start, strides, backstrides, chunks): rstrides = [] rbackstrides = [] diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py --- a/pypy/module/micronumpy/test/test_dtypes.py +++ b/pypy/module/micronumpy/test/test_dtypes.py @@ -3,7 +3,7 @@ class AppTestDtypes(BaseNumpyAppTest): def test_dtype(self): - from numpypy import dtype + from _numpypy import dtype d = dtype('?') assert d.num == 0 @@ -14,7 +14,7 @@ raises(TypeError, dtype, 1042) def test_dtype_with_types(self): - from numpypy import dtype + from _numpypy import dtype assert dtype(bool).num == 0 assert dtype(int).num == 7 @@ -22,13 +22,13 @@ assert dtype(float).num == 12 def test_array_dtype_attr(self): - from numpypy import array, dtype + from _numpypy import array, dtype a = array(range(5), long) assert a.dtype is dtype(long) def test_repr_str(self): - from numpypy import dtype + from _numpypy import dtype assert repr(dtype) == "" d = dtype('?') @@ -36,7 +36,7 @@ assert str(d) == "bool" def test_bool_array(self): - from numpypy import array, False_, True_ + from _numpypy import array, False_, True_ a = array([0, 1, 2, 2.5], dtype='?') assert a[0] is False_ @@ -44,7 +44,7 @@ assert a[i] is True_ def test_copy_array_with_dtype(self): - from numpypy import array, False_, True_, int64 + from _numpypy import array, False_, True_, int64 a = array([0, 1, 2, 3], dtype=long) # int on 64-bit, long in 32-bit @@ -58,35 +58,35 @@ assert b[0] is False_ def test_zeros_bool(self): - from numpypy import zeros, False_ + from _numpypy import zeros, False_ a = zeros(10, dtype=bool) for i in range(10): assert a[i] is False_ def test_ones_bool(self): - from numpypy import ones, True_ + from _numpypy import ones, True_ a = ones(10, dtype=bool) for i in range(10): assert a[i] is True_ def test_zeros_long(self): - from numpypy import zeros, int64 + from _numpypy import zeros, int64 a = zeros(10, dtype=long) for i in range(10): assert isinstance(a[i], int64) assert a[1] == 0 def test_ones_long(self): - from numpypy import ones, int64 + from _numpypy import ones, int64 a = ones(10, dtype=long) for i in range(10): assert isinstance(a[i], int64) assert a[1] == 1 def test_overflow(self): - from numpypy import array, dtype + from _numpypy import array, dtype assert array([128], 'b')[0] == -128 assert array([256], 'B')[0] == 0 assert array([32768], 'h')[0] == -32768 @@ -98,7 +98,7 @@ raises(OverflowError, "array([2**64], 'Q')") def test_bool_binop_types(self): - from numpypy import array, dtype + from _numpypy import array, dtype types = [ '?', 'b', 'B', 'h', 'H', 'i', 'I', 'l', 'L', 'q', 'Q', 'f', 'd' ] @@ -107,7 +107,7 @@ assert (a + array([0], t)).dtype is dtype(t) def test_binop_types(self): - from numpypy import array, dtype + from _numpypy import array, dtype tests = [('b','B','h'), ('b','h','h'), ('b','H','i'), ('b','i','i'), ('b','l','l'), ('b','q','q'), ('b','Q','d'), ('B','h','h'), ('B','H','H'), ('B','i','i'), ('B','I','I'), ('B','l','l'), @@ -129,7 +129,7 @@ assert (array([1], d1) + array([1], d2)).dtype is dtype(dout) def test_add_int8(self): - from numpypy import array, dtype + from _numpypy import array, dtype a = array(range(5), dtype="int8") b = a + a @@ -138,7 +138,7 @@ assert b[i] == i * 2 def test_add_int16(self): - from numpypy import array, dtype + from _numpypy import array, dtype a = array(range(5), dtype="int16") b = a + a @@ -147,7 +147,7 @@ assert b[i] == i * 2 def test_add_uint32(self): - from numpypy import array, dtype + from _numpypy import array, dtype a = array(range(5), dtype="I") b = a + a @@ -156,19 +156,28 @@ assert b[i] == i * 2 def test_shape(self): - from numpypy import dtype + from _numpypy import dtype assert dtype(long).shape == () def test_cant_subclass(self): - from numpypy import dtype + from _numpypy import dtype # You can't subclass dtype raises(TypeError, type, "Foo", (dtype,), {}) + def test_new(self): + import _numpypy as np + assert np.int_(4) == 4 + assert np.float_(3.4) == 3.4 + + def test_pow(self): + from _numpypy import int_ + assert int_(4) ** 2 == 16 + class AppTestTypes(BaseNumpyAppTest): def test_abstract_types(self): - import numpypy as numpy + import _numpypy as numpy raises(TypeError, numpy.generic, 0) raises(TypeError, numpy.number, 0) raises(TypeError, numpy.integer, 0) @@ -181,7 +190,7 @@ raises(TypeError, numpy.inexact, 0) def test_bool(self): - import numpypy as numpy + import _numpypy as numpy assert numpy.bool_.mro() == [numpy.bool_, numpy.generic, object] assert numpy.bool_(3) is numpy.True_ @@ -196,7 +205,7 @@ assert numpy.bool_("False") is numpy.True_ def test_int8(self): - import numpypy as numpy + import _numpypy as numpy assert numpy.int8.mro() == [numpy.int8, numpy.signedinteger, numpy.integer, numpy.number, numpy.generic, object] @@ -218,7 +227,7 @@ assert numpy.int8('128') == -128 def test_uint8(self): - import numpypy as numpy + import _numpypy as numpy assert numpy.uint8.mro() == [numpy.uint8, numpy.unsignedinteger, numpy.integer, numpy.number, numpy.generic, object] @@ -241,7 +250,7 @@ assert numpy.uint8('256') == 0 def test_int16(self): - import numpypy as numpy + import _numpypy as numpy x = numpy.int16(3) assert x == 3 @@ -251,7 +260,7 @@ assert numpy.int16('32768') == -32768 def test_uint16(self): - import numpypy as numpy + import _numpypy as numpy assert numpy.uint16(65535) == 65535 assert numpy.uint16(65536) == 0 @@ -260,7 +269,7 @@ def test_int32(self): import sys - import numpypy as numpy + import _numpypy as numpy x = numpy.int32(23) assert x == 23 @@ -275,7 +284,7 @@ def test_uint32(self): import sys - import numpypy as numpy + import _numpypy as numpy assert numpy.uint32(10) == 10 @@ -286,14 +295,14 @@ assert numpy.uint32('4294967296') == 0 def test_int_(self): - import numpypy as numpy + import _numpypy as numpy assert numpy.int_ is numpy.dtype(int).type assert numpy.int_.mro() == [numpy.int_, numpy.signedinteger, numpy.integer, numpy.number, numpy.generic, int, object] def test_int64(self): import sys - import numpypy as numpy + import _numpypy as numpy if sys.maxint == 2 ** 63 -1: assert numpy.int64.mro() == [numpy.int64, numpy.signedinteger, numpy.integer, numpy.number, numpy.generic, int, object] @@ -315,7 +324,7 @@ def test_uint64(self): import sys - import numpypy as numpy + import _numpypy as numpy assert numpy.uint64.mro() == [numpy.uint64, numpy.unsignedinteger, numpy.integer, numpy.number, numpy.generic, object] @@ -330,7 +339,7 @@ raises(OverflowError, numpy.uint64(18446744073709551616)) def test_float32(self): - import numpypy as numpy + import _numpypy as numpy assert numpy.float32.mro() == [numpy.float32, numpy.floating, numpy.inexact, numpy.number, numpy.generic, object] @@ -339,7 +348,7 @@ raises(ValueError, numpy.float32, '23.2df') def test_float64(self): - import numpypy as numpy + import _numpypy as numpy assert numpy.float64.mro() == [numpy.float64, numpy.floating, numpy.inexact, numpy.number, numpy.generic, float, object] @@ -352,7 +361,7 @@ raises(ValueError, numpy.float64, '23.2df') def test_subclass_type(self): - import numpypy as numpy + import _numpypy as numpy class X(numpy.float64): def m(self): diff --git a/pypy/module/micronumpy/test/test_module.py b/pypy/module/micronumpy/test/test_module.py --- a/pypy/module/micronumpy/test/test_module.py +++ b/pypy/module/micronumpy/test/test_module.py @@ -3,33 +3,33 @@ class AppTestNumPyModule(BaseNumpyAppTest): def test_mean(self): - from numpypy import array, mean + from _numpypy import array, mean assert mean(array(range(5))) == 2.0 assert mean(range(5)) == 2.0 def test_average(self): - from numpypy import array, average + from _numpypy import array, average assert average(range(10)) == 4.5 assert average(array(range(10))) == 4.5 def test_sum(self): - from numpypy import array, sum + from _numpypy import array, sum assert sum(range(10)) == 45 assert sum(array(range(10))) == 45 def test_min(self): - from numpypy import array, min + from _numpypy import array, min assert min(range(10)) == 0 assert min(array(range(10))) == 0 def test_max(self): - from numpypy import array, max + from _numpypy import array, max assert max(range(10)) == 9 assert max(array(range(10))) == 9 def test_constants(self): import math - from numpypy import inf, e, pi + from _numpypy import inf, e, pi assert type(inf) is float assert inf == float("inf") assert e == math.e diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -157,10 +157,13 @@ assert calc_new_strides([2, 3, 4], [8, 3], [1, 16]) is None assert calc_new_strides([24], [2, 4, 3], [48, 6, 1]) is None assert calc_new_strides([24], [2, 4, 3], [24, 6, 2]) == [2] + assert calc_new_strides([105, 1], [3, 5, 7], [35, 7, 1]) == [1, 1] + assert calc_new_strides([1, 105], [3, 5, 7], [35, 7, 1]) == [105, 1] + class AppTestNumArray(BaseNumpyAppTest): def test_ndarray(self): - from numpypy import ndarray, array, dtype + from _numpypy import ndarray, array, dtype assert type(ndarray) is type assert type(array) is not type @@ -175,12 +178,26 @@ assert a.dtype is dtype(int) def test_type(self): - from numpypy import array + from _numpypy import array ar = array(range(5)) assert type(ar) is type(ar + ar) + def test_ndim(self): + from _numpypy import array + x = array(0.2) + assert x.ndim == 0 + x = array([1, 2]) + assert x.ndim == 1 + x = array([[1, 2], [3, 4]]) + assert x.ndim == 2 + x = array([[[1, 2], [3, 4]], [[5, 6], [7, 8]]]) + assert x.ndim == 3 + # numpy actually raises an AttributeError, but _numpypy raises an + # TypeError + raises(TypeError, 'x.ndim = 3') + def test_init(self): - from numpypy import zeros + from _numpypy import zeros a = zeros(15) # Check that storage was actually zero'd. assert a[10] == 0.0 @@ -189,7 +206,7 @@ assert a[13] == 5.3 def test_size(self): - from numpypy import array + from _numpypy import array assert array(3).size == 1 a = array([1, 2, 3]) assert a.size == 3 @@ -200,13 +217,13 @@ Test that empty() works. """ - from numpypy import empty + from _numpypy import empty a = empty(2) a[1] = 1.0 assert a[1] == 1.0 def test_ones(self): - from numpypy import ones + from _numpypy import ones a = ones(3) assert len(a) == 3 assert a[0] == 1 @@ -215,7 +232,7 @@ assert a[2] == 4 def test_copy(self): - from numpypy import arange, array + from _numpypy import arange, array a = arange(5) b = a.copy() for i in xrange(5): @@ -231,13 +248,17 @@ c = b.copy() assert (c == b).all() + a = arange(15).reshape(5,3) + b = a.copy() + assert (b == a).all() + def test_iterator_init(self): - from numpypy import array + from _numpypy import array a = array(range(5)) assert a[3] == 3 def test_getitem(self): - from numpypy import array + from _numpypy import array a = array(range(5)) raises(IndexError, "a[5]") a = a + a @@ -246,7 +267,7 @@ raises(IndexError, "a[-6]") def test_getitem_tuple(self): - from numpypy import array + from _numpypy import array a = array(range(5)) raises(IndexError, "a[(1,2)]") for i in xrange(5): @@ -256,7 +277,7 @@ assert a[i] == b[i] def test_setitem(self): - from numpypy import array + from _numpypy import array a = array(range(5)) a[-1] = 5.0 assert a[4] == 5.0 @@ -264,7 +285,7 @@ raises(IndexError, "a[-6] = 3.0") def test_setitem_tuple(self): - from numpypy import array + from _numpypy import array a = array(range(5)) raises(IndexError, "a[(1,2)] = [0,1]") for i in xrange(5): @@ -275,7 +296,7 @@ assert a[i] == i def test_setslice_array(self): - from numpypy import array + from _numpypy import array a = array(range(5)) b = array(range(2)) a[1:4:2] = b @@ -286,7 +307,7 @@ assert b[1] == 0. def test_setslice_of_slice_array(self): - from numpypy import array, zeros + from _numpypy import array, zeros a = zeros(5) a[::2] = array([9., 10., 11.]) assert a[0] == 9. @@ -305,7 +326,7 @@ assert a[0] == 3. def test_setslice_list(self): - from numpypy import array + from _numpypy import array a = array(range(5), float) b = [0., 1.] a[1:4:2] = b @@ -313,14 +334,14 @@ assert a[3] == 1. def test_setslice_constant(self): - from numpypy import array + from _numpypy import array a = array(range(5), float) a[1:4:2] = 0. assert a[1] == 0. assert a[3] == 0. def test_scalar(self): - from numpypy import array, dtype + from _numpypy import array, dtype a = array(3) raises(IndexError, "a[0]") raises(IndexError, "a[0] = 5") @@ -329,13 +350,13 @@ assert a.dtype is dtype(int) def test_len(self): - from numpypy import array + from _numpypy import array a = array(range(5)) assert len(a) == 5 assert len(a + a) == 5 def test_shape(self): - from numpypy import array + from _numpypy import array a = array(range(5)) assert a.shape == (5,) b = a + a @@ -344,7 +365,7 @@ assert c.shape == (3,) def test_set_shape(self): - from numpypy import array, zeros + from _numpypy import array, zeros a = array([]) a.shape = [] a = array(range(12)) @@ -364,7 +385,7 @@ a.shape = (1,) def test_reshape(self): - from numpypy import array, zeros + from _numpypy import array, zeros a = array(range(12)) exc = raises(ValueError, "b = a.reshape((3, 10))") assert str(exc.value) == "total size of new array must be unchanged" @@ -377,7 +398,7 @@ a.shape = (12, 2) def test_slice_reshape(self): - from numpypy import zeros, arange + from _numpypy import zeros, arange a = zeros((4, 2, 3)) b = a[::2, :, :] b.shape = (2, 6) @@ -413,13 +434,13 @@ raises(ValueError, arange(10).reshape, (5, -1, -1)) def test_reshape_varargs(self): - from numpypy import arange + from _numpypy import arange z = arange(96).reshape(12, -1) y = z.reshape(4, 3, 8) assert y.shape == (4, 3, 8) def test_add(self): - from numpypy import array + from _numpypy import array a = array(range(5)) b = a + a for i in range(5): @@ -432,7 +453,7 @@ assert c[i] == bool(a[i] + b[i]) def test_add_other(self): - from numpypy import array + from _numpypy import array a = array(range(5)) b = array([i for i in reversed(range(5))]) c = a + b @@ -440,20 +461,20 @@ assert c[i] == 4 def test_add_constant(self): - from numpypy import array + from _numpypy import array a = array(range(5)) b = a + 5 for i in range(5): assert b[i] == i + 5 def test_radd(self): - from numpypy import array + from _numpypy import array r = 3 + array(range(3)) for i in range(3): assert r[i] == i + 3 def test_add_list(self): - from numpypy import array, ndarray + from _numpypy import array, ndarray a = array(range(5)) b = list(reversed(range(5))) c = a + b @@ -462,14 +483,14 @@ assert c[i] == 4 def test_subtract(self): - from numpypy import array + from _numpypy import array a = array(range(5)) b = a - a for i in range(5): assert b[i] == 0 def test_subtract_other(self): - from numpypy import array + from _numpypy import array a = array(range(5)) b = array([1, 1, 1, 1, 1]) c = a - b @@ -477,35 +498,35 @@ assert c[i] == i - 1 def test_subtract_constant(self): - from numpypy import array + from _numpypy import array a = array(range(5)) b = a - 5 for i in range(5): assert b[i] == i - 5 def test_scalar_subtract(self): - from numpypy import int32 + from _numpypy import int32 assert int32(2) - 1 == 1 assert 1 - int32(2) == -1 def test_mul(self): - import numpypy + import _numpypy - a = numpypy.array(range(5)) + a = _numpypy.array(range(5)) b = a * a for i in range(5): assert b[i] == i * i assert b.dtype is numpypy.dtype(int) - a = numpypy.array(range(5), dtype=bool) + a = _numpypy.array(range(5), dtype=bool) b = a * a - assert b.dtype is numpypy.dtype(bool) - assert b[0] is numpypy.False_ + assert b.dtype is _numpypy.dtype(bool) + assert b[0] is _numpypy.False_ for i in range(1, 5): - assert b[i] is numpypy.True_ + assert b[i] is _numpypy.True_ def test_mul_constant(self): - from numpypy import array + from _numpypy import array a = array(range(5)) b = a * 5 for i in range(5): @@ -513,7 +534,7 @@ def test_div(self): from math import isnan - from numpypy import array, dtype, inf + from _numpypy import array, dtype, inf a = array(range(1, 6)) b = a / a @@ -545,7 +566,7 @@ assert c[2] == -inf def test_div_other(self): - from numpypy import array + from _numpypy import array a = array(range(5)) b = array([2, 2, 2, 2, 2], float) c = a / b @@ -553,14 +574,14 @@ assert c[i] == i / 2.0 def test_div_constant(self): - from numpypy import array + from _numpypy import array a = array(range(5)) b = a / 5.0 for i in range(5): assert b[i] == i / 5.0 def test_pow(self): - from numpypy import array + from _numpypy import array a = array(range(5), float) b = a ** a for i in range(5): @@ -570,7 +591,7 @@ assert (a ** 2 == a * a).all() def test_pow_other(self): - from numpypy import array + from _numpypy import array a = array(range(5), float) b = array([2, 2, 2, 2, 2]) c = a ** b @@ -578,14 +599,14 @@ assert c[i] == i ** 2 def test_pow_constant(self): - from numpypy import array + from _numpypy import array a = array(range(5), float) b = a ** 2 for i in range(5): assert b[i] == i ** 2 def test_mod(self): - from numpypy import array + from _numpypy import array a = array(range(1, 6)) b = a % a for i in range(5): @@ -598,7 +619,7 @@ assert b[i] == 1 def test_mod_other(self): - from numpypy import array + from _numpypy import array a = array(range(5)) b = array([2, 2, 2, 2, 2]) c = a % b @@ -606,14 +627,14 @@ assert c[i] == i % 2 def test_mod_constant(self): - from numpypy import array + from _numpypy import array a = array(range(5)) b = a % 2 for i in range(5): assert b[i] == i % 2 def test_pos(self): - from numpypy import array + from _numpypy import array a = array([1., -2., 3., -4., -5.]) b = +a for i in range(5): @@ -624,7 +645,7 @@ assert a[i] == i def test_neg(self): - from numpypy import array + from _numpypy import array a = array([1., -2., 3., -4., -5.]) b = -a for i in range(5): @@ -635,7 +656,7 @@ assert a[i] == -i def test_abs(self): - from numpypy import array + from _numpypy import array a = array([1., -2., 3., -4., -5.]) b = abs(a) for i in range(5): @@ -646,7 +667,7 @@ assert a[i + 5] == abs(i) def test_auto_force(self): - from numpypy import array + from _numpypy import array a = array(range(5)) b = a - 1 a[2] = 3 @@ -660,7 +681,7 @@ assert c[1] == 4 def test_getslice(self): - from numpypy import array + from _numpypy import array a = array(range(5)) s = a[1:5] assert len(s) == 4 @@ -674,7 +695,7 @@ assert s[0] == 5 def test_getslice_step(self): - from numpypy import array + from _numpypy import array a = array(range(10)) s = a[1:9:2] assert len(s) == 4 @@ -682,7 +703,7 @@ assert s[i] == a[2 * i + 1] def test_slice_update(self): - from numpypy import array + from _numpypy import array a = array(range(5)) s = a[0:3] s[1] = 10 @@ -692,7 +713,7 @@ def test_slice_invaidate(self): # check that slice shares invalidation list with - from numpypy import array + from _numpypy import array a = array(range(5)) s = a[0:2] b = array([10, 11]) @@ -706,13 +727,18 @@ assert d[1] == 12 def test_mean(self): - from numpypy import array + from _numpypy import array, mean a = array(range(5)) assert a.mean() == 2.0 assert a[:4].mean() == 1.5 + a = array(range(105)).reshape(3, 5, 7) + b = mean(a, axis=0) + b[0,0]==35. + assert (b == array(range(35, 70), dtype=float).reshape(5, 7)).all() + assert (mean(a, 2) == array(range(0, 15), dtype=float).reshape(3, 5) * 7 + 3).all() def test_sum(self): - from numpypy import array + from _numpypy import array a = array(range(5)) b = a.sum() assert b == 10 @@ -721,53 +747,78 @@ a = array([True] * 5, bool) assert a.sum() == 5 + raises(TypeError, 'a.sum(2, 3)') + + def test_reduce_nd(self): + from numpypy import arange, array, multiply + a = arange(15).reshape(5, 3) + assert a.sum() == 105 + assert a.max() == 14 + assert array([]).sum() == 0.0 + raises(ValueError, 'array([]).max()') + assert (a.sum(0) == [30, 35, 40]).all() + assert (a.sum(1) == [3, 12, 21, 30, 39]).all() + assert (a.max(0) == [12, 13, 14]).all() + assert (a.max(1) == [2, 5, 8, 11, 14]).all() + assert ((a + a).max() == 28) + assert ((a + a).max(0) == [24, 26, 28]).all() + assert ((a + a).sum(1) == [6, 24, 42, 60, 78]).all() + assert (multiply.reduce(a) == array([0, 3640, 12320])).all() + a = array(range(105)).reshape(3, 5, 7) + assert (a[:, 1, :].sum(0) == [126, 129, 132, 135, 138, 141, 144]).all() + assert (a[:, 1, :].sum(1) == [70, 315, 560]).all() + raises (ValueError, 'a[:, 1, :].sum(2)') + assert ((a + a).T.sum(2).T == (a + a).sum(0)).all() + assert (a.reshape(1,-1).sum(0) == range(105)).all() + assert (a.reshape(1,-1).sum(1) == 5460) + def test_identity(self): - from numpypy import identity, array - from numpypy import int32, float64, dtype + from _numpypy import identity, array + from _numpypy import int32, float64, dtype a = identity(0) assert len(a) == 0 assert a.dtype == dtype('float64') - assert a.shape == (0,0) + assert a.shape == (0, 0) b = identity(1, dtype=int32) assert len(b) == 1 assert b[0][0] == 1 - assert b.shape == (1,1) + assert b.shape == (1, 1) assert b.dtype == dtype('int32') c = identity(2) - assert c.shape == (2,2) - assert (c == [[1,0],[0,1]]).all() + assert c.shape == (2, 2) + assert (c == [[1, 0], [0, 1]]).all() d = identity(3, dtype='int32') - assert d.shape == (3,3) + assert d.shape == (3, 3) assert d.dtype == dtype('int32') - assert (d == [[1,0,0],[0,1,0],[0,0,1]]).all() + assert (d == [[1, 0, 0], [0, 1, 0], [0, 0, 1]]).all() def test_prod(self): - from numpypy import array + from _numpypy import array a = array(range(1, 6)) assert a.prod() == 120.0 assert a[:4].prod() == 24.0 def test_max(self): - from numpypy import array + from _numpypy import array a = array([-1.2, 3.4, 5.7, -3.0, 2.7]) assert a.max() == 5.7 b = array([]) raises(ValueError, "b.max()") def test_max_add(self): - from numpypy import array + from _numpypy import array a = array([-1.2, 3.4, 5.7, -3.0, 2.7]) assert (a + a).max() == 11.4 def test_min(self): - from numpypy import array + from _numpypy import array a = array([-1.2, 3.4, 5.7, -3.0, 2.7]) assert a.min() == -3.0 b = array([]) raises(ValueError, "b.min()") def test_argmax(self): - from numpypy import array + from _numpypy import array a = array([-1.2, 3.4, 5.7, -3.0, 2.7]) r = a.argmax() assert r == 2 @@ -788,14 +839,14 @@ assert a.argmax() == 2 def test_argmin(self): - from numpypy import array + from _numpypy import array a = array([-1.2, 3.4, 5.7, -3.0, 2.7]) assert a.argmin() == 3 b = array([]) raises(ValueError, "b.argmin()") def test_all(self): - from numpypy import array + from _numpypy import array a = array(range(5)) assert a.all() == False a[0] = 3.0 @@ -804,7 +855,7 @@ assert b.all() == True def test_any(self): - from numpypy import array, zeros + from _numpypy import array, zeros a = array(range(5)) assert a.any() == True b = zeros(5) @@ -813,7 +864,7 @@ assert c.any() == False def test_dot(self): - from numpypy import array, dot + from _numpypy import array, dot a = array(range(5)) assert a.dot(a) == 30.0 @@ -839,14 +890,14 @@ [[86, 302, 518], [110, 390, 670], [134, 478, 822]]]).all() def test_dot_constant(self): - from numpypy import array + from _numpypy import array a = array(range(5)) b = a.dot(2.5) for i in xrange(5): assert b[i] == 2.5 * a[i] def test_dtype_guessing(self): - from numpypy import array, dtype, float64, int8, bool_ + from _numpypy import array, dtype, float64, int8, bool_ assert array([True]).dtype is dtype(bool) assert array([True, False]).dtype is dtype(bool) @@ -863,7 +914,7 @@ def test_comparison(self): import operator - from numpypy import array, dtype + from _numpypy import array, dtype a = array(range(5)) b = array(range(5), float) @@ -882,7 +933,7 @@ assert c[i] == func(b[i], 3) def test_nonzero(self): - from numpypy import array + from _numpypy import array a = array([1, 2]) raises(ValueError, bool, a) raises(ValueError, bool, a == a) @@ -892,7 +943,7 @@ assert not bool(array([0])) def test_slice_assignment(self): - from numpypy import array + from _numpypy import array a = array(range(5)) a[::-1] = a assert (a == [0, 1, 2, 1, 0]).all() @@ -902,8 +953,8 @@ assert (a == [8, 6, 4, 2, 0]).all() def test_debug_repr(self): - from numpypy import zeros, sin - from numpypy.pypy import debug_repr + from _numpypy import zeros, sin + from _numpypy.pypy import debug_repr a = zeros(1) assert debug_repr(a) == 'Array' assert debug_repr(a + a) == 'Call2(add, Array, Array)' @@ -916,8 +967,17 @@ b[0] = 3 assert debug_repr(b) == 'Array' + def test_remove_invalidates(self): + from _numpypy import array + from _numpypy.pypy import remove_invalidates + a = array([1, 2, 3]) + b = a + a + remove_invalidates(a) + a[0] = 14 + assert b[0] == 28 + def test_virtual_views(self): - from numpypy import arange + from _numpypy import arange a = arange(15) c = (a + a) d = c[::2] @@ -935,7 +995,7 @@ assert b[1] == 2 def test_tolist_scalar(self): - from numpypy import int32, bool_ + from _numpypy import int32, bool_ x = int32(23) assert x.tolist() == 23 assert type(x.tolist()) is int @@ -943,13 +1003,13 @@ assert y.tolist() is True def test_tolist_zerodim(self): - from numpypy import array + from _numpypy import array x = array(3) assert x.tolist() == 3 assert type(x.tolist()) is int def test_tolist_singledim(self): - from numpypy import array + from _numpypy import array a = array(range(5)) assert a.tolist() == [0, 1, 2, 3, 4] assert type(a.tolist()[0]) is int @@ -957,41 +1017,55 @@ assert b.tolist() == [0.2, 0.4, 0.6] def test_tolist_multidim(self): - from numpypy import array + from _numpypy import array a = array([[1, 2], [3, 4]]) assert a.tolist() == [[1, 2], [3, 4]] def test_tolist_view(self): - from numpypy import array - a = array([[1,2],[3,4]]) + from _numpypy import array + a = array([[1, 2], [3, 4]]) assert (a + a).tolist() == [[2, 4], [6, 8]] def test_tolist_slice(self): - from numpypy import array + from _numpypy import array a = array([[17.1, 27.2], [40.3, 50.3]]) - assert a[:,0].tolist() == [17.1, 40.3] + assert a[:, 0].tolist() == [17.1, 40.3] assert a[0].tolist() == [17.1, 27.2] + def test_var(self): + from _numpypy import array + a = array(range(10)) + assert a.var() == 8.25 + a = array([5.0]) + assert a.var() == 0.0 + + def test_std(self): + from _numpypy import array + a = array(range(10)) + assert a.std() == 2.8722813232690143 + a = array([5.0]) + assert a.std() == 0.0 + class AppTestMultiDim(BaseNumpyAppTest): def test_init(self): - import numpypy - a = numpypy.zeros((2, 2)) + import _numpypy + a = _numpypy.zeros((2, 2)) assert len(a) == 2 def test_shape(self): - import numpypy - assert numpypy.zeros(1).shape == (1,) - assert numpypy.zeros((2, 2)).shape == (2, 2) - assert numpypy.zeros((3, 1, 2)).shape == (3, 1, 2) - assert numpypy.array([[1], [2], [3]]).shape == (3, 1) - assert len(numpypy.zeros((3, 1, 2))) == 3 - raises(TypeError, len, numpypy.zeros(())) - raises(ValueError, numpypy.array, [[1, 2], 3]) + import _numpypy + assert _numpypy.zeros(1).shape == (1,) + assert _numpypy.zeros((2, 2)).shape == (2, 2) + assert _numpypy.zeros((3, 1, 2)).shape == (3, 1, 2) + assert _numpypy.array([[1], [2], [3]]).shape == (3, 1) + assert len(_numpypy.zeros((3, 1, 2))) == 3 + raises(TypeError, len, _numpypy.zeros(())) + raises(ValueError, _numpypy.array, [[1, 2], 3]) def test_getsetitem(self): - import numpypy - a = numpypy.zeros((2, 3, 1)) + import _numpypy + a = _numpypy.zeros((2, 3, 1)) raises(IndexError, a.__getitem__, (2, 0, 0)) raises(IndexError, a.__getitem__, (0, 3, 0)) raises(IndexError, a.__getitem__, (0, 0, 1)) @@ -1002,8 +1076,8 @@ assert a[1, -1, 0] == 3 def test_slices(self): - import numpypy - a = numpypy.zeros((4, 3, 2)) + import _numpypy + a = _numpypy.zeros((4, 3, 2)) raises(IndexError, a.__getitem__, (4,)) raises(IndexError, a.__getitem__, (3, 3)) raises(IndexError, a.__getitem__, (slice(None), 3)) @@ -1036,51 +1110,51 @@ assert a[1][2][1] == 15 def test_init_2(self): - import numpypy - raises(ValueError, numpypy.array, [[1], 2]) - raises(ValueError, numpypy.array, [[1, 2], [3]]) - raises(ValueError, numpypy.array, [[[1, 2], [3, 4], 5]]) - raises(ValueError, numpypy.array, [[[1, 2], [3, 4], [5]]]) - a = numpypy.array([[1, 2], [4, 5]]) + import _numpypy + raises(ValueError, _numpypy.array, [[1], 2]) + raises(ValueError, _numpypy.array, [[1, 2], [3]]) + raises(ValueError, _numpypy.array, [[[1, 2], [3, 4], 5]]) + raises(ValueError, _numpypy.array, [[[1, 2], [3, 4], [5]]]) + a = _numpypy.array([[1, 2], [4, 5]]) assert a[0, 1] == 2 assert a[0][1] == 2 - a = numpypy.array(([[[1, 2], [3, 4], [5, 6]]])) + a = _numpypy.array(([[[1, 2], [3, 4], [5, 6]]])) assert (a[0, 1] == [3, 4]).all() def test_setitem_slice(self): - import numpypy - a = numpypy.zeros((3, 4)) + import _numpypy + a = _numpypy.zeros((3, 4)) a[1] = [1, 2, 3, 4] assert a[1, 2] == 3 raises(TypeError, a[1].__setitem__, [1, 2, 3]) - a = numpypy.array([[1, 2], [3, 4]]) + a = _numpypy.array([[1, 2], [3, 4]]) assert (a == [[1, 2], [3, 4]]).all() - a[1] = numpypy.array([5, 6]) + a[1] = _numpypy.array([5, 6]) assert (a == [[1, 2], [5, 6]]).all() - a[:, 1] = numpypy.array([8, 10]) + a[:, 1] = _numpypy.array([8, 10]) assert (a == [[1, 8], [5, 10]]).all() - a[0, :: -1] = numpypy.array([11, 12]) + a[0, :: -1] = _numpypy.array([11, 12]) assert (a == [[12, 11], [5, 10]]).all() def test_ufunc(self): - from numpypy import array + from _numpypy import array a = array([[1, 2], [3, 4], [5, 6]]) assert ((a + a) == \ array([[1 + 1, 2 + 2], [3 + 3, 4 + 4], [5 + 5, 6 + 6]])).all() def test_getitem_add(self): - from numpypy import array + from _numpypy import array a = array([[1, 2], [3, 4], [5, 6], [7, 8], [9, 10]]) assert (a + a)[1, 1] == 8 def test_ufunc_negative(self): - from numpypy import array, negative + from _numpypy import array, negative a = array([[1, 2], [3, 4]]) b = negative(a + a) assert (b == [[-2, -4], [-6, -8]]).all() def test_getitem_3(self): - from numpypy import array + from _numpypy import array a = array([[1, 2], [3, 4], [5, 6], [7, 8], [9, 10], [11, 12], [13, 14]]) b = a[::2] @@ -1091,37 +1165,37 @@ assert c[1][1] == 12 def test_multidim_ones(self): - from numpypy import ones + from _numpypy import ones a = ones((1, 2, 3)) assert a[0, 1, 2] == 1.0 def test_multidim_setslice(self): - from numpypy import zeros, ones + from _numpypy import zeros, ones a = zeros((3, 3)) b = ones((3, 3)) - a[:,1:3] = b[:,1:3] + a[:, 1:3] = b[:, 1:3] assert (a == [[0, 1, 1], [0, 1, 1], [0, 1, 1]]).all() a = zeros((3, 3)) b = ones((3, 3)) - a[:,::2] = b[:,::2] + a[:, ::2] = b[:, ::2] assert (a == [[1, 0, 1], [1, 0, 1], [1, 0, 1]]).all() def test_broadcast_ufunc(self): - from numpypy import array + from _numpypy import array a = array([[1, 2], [3, 4], [5, 6]]) b = array([5, 6]) c = ((a + b) == [[1 + 5, 2 + 6], [3 + 5, 4 + 6], [5 + 5, 6 + 6]]) assert c.all() def test_broadcast_setslice(self): - from numpypy import zeros, ones + from _numpypy import zeros, ones a = zeros((10, 10)) b = ones(10) a[:, :] = b assert a[3, 5] == 1 def test_broadcast_shape_agreement(self): - from numpypy import zeros, array + from _numpypy import zeros, array a = zeros((3, 1, 3)) b = array(((10, 11, 12), (20, 21, 22), (30, 31, 32))) c = ((a + b) == [b, b, b]) @@ -1135,7 +1209,7 @@ assert c.all() def test_broadcast_scalar(self): - from numpypy import zeros + from _numpypy import zeros a = zeros((4, 5), 'd') a[:, 1] = 3 assert a[2, 1] == 3 @@ -1146,14 +1220,14 @@ assert a[3, 2] == 0 def test_broadcast_call2(self): - from numpypy import zeros, ones + from _numpypy import zeros, ones a = zeros((4, 1, 5)) b = ones((4, 3, 5)) b[:] = (a + a) assert (b == zeros((4, 3, 5))).all() def test_broadcast_virtualview(self): - from numpypy import arange, zeros + from _numpypy import arange, zeros a = arange(8).reshape([2, 2, 2]) b = (a + a)[1, 1] c = zeros((2, 2, 2)) @@ -1161,13 +1235,13 @@ assert (c == [[[12, 14], [12, 14]], [[12, 14], [12, 14]]]).all() def test_argmax(self): - from numpypy import array + from _numpypy import array a = array([[1, 2], [3, 4], [5, 6]]) assert a.argmax() == 5 assert a[:2, ].argmax() == 3 def test_broadcast_wrong_shapes(self): - from numpypy import zeros + from _numpypy import zeros a = zeros((4, 3, 2)) b = zeros((4, 2)) exc = raises(ValueError, lambda: a + b) @@ -1175,7 +1249,7 @@ " together with shapes (4,3,2) (4,2)" def test_reduce(self): - from numpypy import array + from _numpypy import array a = array([[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]]) assert a.sum() == (13 * 12) / 2 b = a[1:, 1::2] @@ -1183,7 +1257,7 @@ assert c.sum() == (6 + 8 + 10 + 12) * 2 def test_transpose(self): - from numpypy import array + from _numpypy import array a = array(((range(3), range(3, 6)), (range(6, 9), range(9, 12)), (range(12, 15), range(15, 18)), @@ -1202,7 +1276,7 @@ assert(b[:, 0] == a[0, :]).all() def test_flatiter(self): - from numpypy import array, flatiter + from _numpypy import array, flatiter a = array([[10, 30], [40, 60]]) f_iter = a.flat assert f_iter.next() == 10 @@ -1217,23 +1291,23 @@ assert s == 140 def test_flatiter_array_conv(self): - from numpypy import array, dot + from _numpypy import array, dot a = array([1, 2, 3]) assert dot(a.flat, a.flat) == 14 def test_flatiter_varray(self): - from numpypy import ones + from _numpypy import ones a = ones((2, 2)) assert list(((a + a).flat)) == [2, 2, 2, 2] def test_slice_copy(self): - from numpypy import zeros + from _numpypy import zeros a = zeros((10, 10)) b = a[0].copy() assert (b == zeros(10)).all() def test_array_interface(self): - from numpypy import array + from _numpypy import array a = array([1, 2, 3]) i = a.__array_interface__ assert isinstance(i['data'][0], int) @@ -1242,6 +1316,7 @@ assert isinstance(i['data'][0], int) raises(TypeError, getattr, array(3), '__array_interface__') + class AppTestSupport(BaseNumpyAppTest): def setup_class(cls): import struct @@ -1254,7 +1329,7 @@ def test_fromstring(self): import sys - from numpypy import fromstring, array, uint8, float32, int32 + from _numpypy import fromstring, array, uint8, float32, int32 a = fromstring(self.data) for i in range(4): @@ -1284,17 +1359,17 @@ assert g[1] == 2 assert g[2] == 3 h = fromstring("1, , 2, 3", dtype=uint8, sep=",") - assert (h == [1,0,2,3]).all() + assert (h == [1, 0, 2, 3]).all() i = fromstring("1 2 3", dtype=uint8, sep=" ") - assert (i == [1,2,3]).all() + assert (i == [1, 2, 3]).all() j = fromstring("1\t\t\t\t2\t3", dtype=uint8, sep="\t") - assert (j == [1,2,3]).all() + assert (j == [1, 2, 3]).all() k = fromstring("1,x,2,3", dtype=uint8, sep=",") - assert (k == [1,0]).all() + assert (k == [1, 0]).all() l = fromstring("1,x,2,3", dtype='float32', sep=",") - assert (l == [1.0,-1.0]).all() + assert (l == [1.0, -1.0]).all() m = fromstring("1,,2,3", sep=",") - assert (m == [1.0,-1.0,2.0,3.0]).all() + assert (m == [1.0, -1.0, 2.0, 3.0]).all() n = fromstring("3.4 2.0 3.8 2.2", dtype=int32, sep=" ") assert (n == [3]).all() o = fromstring("1.0 2f.0f 3.8 2.2", dtype=float32, sep=" ") @@ -1318,7 +1393,7 @@ assert (u == [1, 0]).all() def test_fromstring_types(self): - from numpypy import (fromstring, int8, int16, int32, int64, uint8, + from _numpypy import (fromstring, int8, int16, int32, int64, uint8, uint16, uint32, float32, float64) a = fromstring('\xFF', dtype=int8) @@ -1342,9 +1417,8 @@ j = fromstring(self.ulongval, dtype='L') assert j[0] == 12 - def test_fromstring_invalid(self): - from numpypy import fromstring, uint16, uint8, int32 + from _numpypy import fromstring, uint16, uint8, int32 #default dtype is 64-bit float, so 3 bytes should fail raises(ValueError, fromstring, "\x01\x02\x03") #3 bytes is not modulo 2 bytes (int16) @@ -1355,7 +1429,8 @@ class AppTestRepr(BaseNumpyAppTest): def test_repr(self): - from numpypy import array, zeros + from _numpypy import array, zeros + int_size = array(5).dtype.itemsize a = array(range(5), float) assert repr(a) == "array([0.0, 1.0, 2.0, 3.0, 4.0])" a = array([], float) @@ -1363,14 +1438,26 @@ a = zeros(1001) assert repr(a) == "array([0.0, 0.0, 0.0, ..., 0.0, 0.0, 0.0])" a = array(range(5), long) - assert repr(a) == "array([0, 1, 2, 3, 4])" + if a.dtype.itemsize == int_size: + assert repr(a) == "array([0, 1, 2, 3, 4])" + else: + assert repr(a) == "array([0, 1, 2, 3, 4], dtype=int64)" + a = array(range(5), 'int32') + if a.dtype.itemsize == int_size: + assert repr(a) == "array([0, 1, 2, 3, 4])" + else: + assert repr(a) == "array([0, 1, 2, 3, 4], dtype=int32)" a = array([], long) assert repr(a) == "array([], dtype=int64)" a = array([True, False, True, False], "?") assert repr(a) == "array([True, False, True, False], dtype=bool)" + a = zeros([]) + assert repr(a) == "array(0.0)" + a = array(0.2) + assert repr(a) == "array(0.2)" def test_repr_multi(self): - from numpypy import array, zeros + from _numpypy import arange, zeros a = zeros((3, 4)) assert repr(a) == '''array([[0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0], @@ -1383,9 +1470,19 @@ [[0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0]]])''' + a = arange(1002).reshape((2, 501)) + assert repr(a) == '''array([[0, 1, 2, ..., 498, 499, 500], + [501, 502, 503, ..., 999, 1000, 1001]])''' + assert repr(a.T) == '''array([[0, 501], + [1, 502], + [2, 503], + ..., + [498, 999], + [499, 1000], + [500, 1001]])''' def test_repr_slice(self): - from numpypy import array, zeros + from _numpypy import array, zeros a = array(range(5), float) b = a[1::2] assert repr(b) == "array([1.0, 3.0])" @@ -1400,7 +1497,7 @@ assert repr(b) == "array([], shape=(0, 5), dtype=int16)" def test_str(self): - from numpypy import array, zeros + from _numpypy import array, zeros a = array(range(5), float) assert str(a) == "[0.0 1.0 2.0 3.0 4.0]" assert str((2 * a)[:]) == "[0.0 2.0 4.0 6.0 8.0]" @@ -1426,14 +1523,14 @@ a = zeros((400, 400), dtype=int) assert str(a) == "[[0 0 0 ..., 0 0 0]\n [0 0 0 ..., 0 0 0]\n" \ - " [0 0 0 ..., 0 0 0]\n ..., \n [0 0 0 ..., 0 0 0]\n" \ + " [0 0 0 ..., 0 0 0]\n ...,\n [0 0 0 ..., 0 0 0]\n" \ " [0 0 0 ..., 0 0 0]\n [0 0 0 ..., 0 0 0]]" a = zeros((2, 2, 2)) r = str(a) assert r == '[[[0.0 0.0]\n [0.0 0.0]]\n\n [[0.0 0.0]\n [0.0 0.0]]]' def test_str_slice(self): - from numpypy import array, zeros + from _numpypy import array, zeros a = array(range(5), float) b = a[1::2] assert str(b) == "[1.0 3.0]" @@ -1449,7 +1546,7 @@ class AppTestRanges(BaseNumpyAppTest): def test_arange(self): - from numpypy import arange, array, dtype + from _numpypy import arange, array, dtype a = arange(3) assert (a == [0, 1, 2]).all() assert a.dtype is dtype(int) @@ -1471,10 +1568,14 @@ class AppTestRanges(BaseNumpyAppTest): def test_app_reshape(self): - from numpypy import arange, array, dtype, reshape + from _numpypy import arange, array, dtype, reshape a = arange(12) b = reshape(a, (3, 4)) assert b.shape == (3, 4) a = range(12) b = reshape(a, (3, 4)) assert b.shape == (3, 4) + a = array(range(105)).reshape(3, 5, 7) + assert a.reshape(1, -1).shape == (1, 105) + assert a.reshape(1, 1, -1).shape == (1, 1, 105) + assert a.reshape(-1, 1, 1).shape == (105, 1, 1) diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py --- a/pypy/module/micronumpy/test/test_ufuncs.py +++ b/pypy/module/micronumpy/test/test_ufuncs.py @@ -4,14 +4,14 @@ class AppTestUfuncs(BaseNumpyAppTest): def test_ufunc_instance(self): - from numpypy import add, ufunc + from _numpypy import add, ufunc assert isinstance(add, ufunc) assert repr(add) == "" assert repr(ufunc) == "" def test_ufunc_attrs(self): - from numpypy import add, multiply, sin + from _numpypy import add, multiply, sin assert add.identity == 0 assert multiply.identity == 1 @@ -22,7 +22,7 @@ assert sin.nin == 1 def test_wrong_arguments(self): - from numpypy import add, sin + from _numpypy import add, sin raises(ValueError, add, 1) raises(TypeError, add, 1, 2, 3) @@ -30,14 +30,14 @@ raises(ValueError, sin) def test_single_item(self): - from numpypy import negative, sign, minimum + from _numpypy import negative, sign, minimum assert negative(5.0) == -5.0 assert sign(-0.0) == 0.0 assert minimum(2.0, 3.0) == 2.0 def test_sequence(self): - from numpypy import array, ndarray, negative, minimum + from _numpypy import array, ndarray, negative, minimum a = array(range(3)) b = [2.0, 1.0, 0.0] c = 1.0 @@ -71,7 +71,7 @@ assert min_c_b[i] == min(b[i], c) def test_negative(self): - from numpypy import array, negative + from _numpypy import array, negative a = array([-5.0, 0.0, 1.0]) b = negative(a) @@ -86,7 +86,7 @@ assert negative(a + a)[3] == -6 def test_abs(self): - from numpypy import array, absolute + from _numpypy import array, absolute a = array([-5.0, -0.0, 1.0]) b = absolute(a) @@ -94,7 +94,7 @@ assert b[i] == abs(a[i]) def test_add(self): - from numpypy import array, add + from _numpypy import array, add a = array([-5.0, -0.0, 1.0]) b = array([ 3.0, -2.0,-3.0]) @@ -103,7 +103,7 @@ assert c[i] == a[i] + b[i] def test_divide(self): - from numpypy import array, divide + from _numpypy import array, divide a = array([-5.0, -0.0, 1.0]) b = array([ 3.0, -2.0,-3.0]) @@ -114,7 +114,7 @@ assert (divide(array([-10]), array([2])) == array([-5])).all() def test_fabs(self): - from numpypy import array, fabs + from _numpypy import array, fabs from math import fabs as math_fabs a = array([-5.0, -0.0, 1.0]) @@ -123,7 +123,7 @@ assert b[i] == math_fabs(a[i]) def test_minimum(self): - from numpypy import array, minimum + from _numpypy import array, minimum a = array([-5.0, -0.0, 1.0]) b = array([ 3.0, -2.0,-3.0]) @@ -132,7 +132,7 @@ assert c[i] == min(a[i], b[i]) def test_maximum(self): - from numpypy import array, maximum + from _numpypy import array, maximum a = array([-5.0, -0.0, 1.0]) b = array([ 3.0, -2.0,-3.0]) @@ -145,7 +145,7 @@ assert isinstance(x, (int, long)) def test_multiply(self): - from numpypy import array, multiply + from _numpypy import array, multiply a = array([-5.0, -0.0, 1.0]) b = array([ 3.0, -2.0,-3.0]) @@ -154,7 +154,7 @@ assert c[i] == a[i] * b[i] def test_sign(self): - from numpypy import array, sign, dtype + from _numpypy import array, sign, dtype reference = [-1.0, 0.0, 0.0, 1.0] a = array([-5.0, -0.0, 0.0, 6.0]) @@ -173,7 +173,7 @@ assert a[1] == 0 def test_reciporocal(self): - from numpypy import array, reciprocal + from _numpypy import array, reciprocal reference = [-0.2, float("inf"), float("-inf"), 2.0] a = array([-5.0, 0.0, -0.0, 0.5]) @@ -182,7 +182,7 @@ assert b[i] == reference[i] def test_subtract(self): - from numpypy import array, subtract + from _numpypy import array, subtract a = array([-5.0, -0.0, 1.0]) b = array([ 3.0, -2.0,-3.0]) @@ -191,7 +191,7 @@ assert c[i] == a[i] - b[i] def test_floor(self): - from numpypy import array, floor + from _numpypy import array, floor reference = [-2.0, -1.0, 0.0, 1.0, 1.0] a = array([-1.4, -1.0, 0.0, 1.0, 1.4]) @@ -200,7 +200,7 @@ assert b[i] == reference[i] def test_copysign(self): - from numpypy import array, copysign + from _numpypy import array, copysign reference = [5.0, -0.0, 0.0, -6.0] a = array([-5.0, 0.0, 0.0, 6.0]) @@ -216,7 +216,7 @@ def test_exp(self): import math - from numpypy import array, exp + from _numpypy import array, exp a = array([-5.0, -0.0, 0.0, 12345678.0, float("inf"), -float('inf'), -12343424.0]) @@ -230,7 +230,7 @@ def test_sin(self): import math - from numpypy import array, sin + from _numpypy import array, sin a = array([0, 1, 2, 3, math.pi, math.pi*1.5, math.pi*2]) b = sin(a) @@ -243,7 +243,7 @@ def test_cos(self): import math - from numpypy import array, cos + from _numpypy import array, cos a = array([0, 1, 2, 3, math.pi, math.pi*1.5, math.pi*2]) b = cos(a) @@ -252,7 +252,7 @@ def test_tan(self): import math - from numpypy import array, tan + from _numpypy import array, tan a = array([0, 1, 2, 3, math.pi, math.pi*1.5, math.pi*2]) b = tan(a) @@ -262,7 +262,7 @@ def test_arcsin(self): import math - from numpypy import array, arcsin + from _numpypy import array, arcsin a = array([-1, -0.5, -0.33, 0, 0.33, 0.5, 1]) b = arcsin(a) @@ -276,7 +276,7 @@ def test_arccos(self): import math - from numpypy import array, arccos + from _numpypy import array, arccos a = array([-1, -0.5, -0.33, 0, 0.33, 0.5, 1]) b = arccos(a) @@ -291,20 +291,20 @@ def test_arctan(self): import math - from numpypy import array, arctan + from _numpypy import array, arctan a = array([-3, -2, -1, 0, 1, 2, 3, float('inf'), float('-inf')]) b = arctan(a) for i in range(len(a)): assert b[i] == math.atan(a[i]) - a = array([float('nan')]) + a = array([float('nan')]) b = arctan(a) assert math.isnan(b[0]) def test_arcsinh(self): import math - from numpypy import arcsinh, inf + from _numpypy import arcsinh, inf for v in [inf, -inf, 1.0, math.e]: assert math.asinh(v) == arcsinh(v) @@ -312,7 +312,7 @@ def test_arctanh(self): import math - from numpypy import arctanh + from _numpypy import arctanh for v in [.99, .5, 0, -.5, -.99]: assert math.atanh(v) == arctanh(v) @@ -323,7 +323,7 @@ def test_sqrt(self): import math - from numpypy import sqrt + from _numpypy import sqrt nan, inf = float("nan"), float("inf") data = [1, 2, 3, inf] @@ -333,22 +333,28 @@ assert math.isnan(sqrt(nan)) def test_reduce_errors(self): - from numpypy import sin, add + from _numpypy import sin, add raises(ValueError, sin.reduce, [1, 2, 3]) - raises(TypeError, add.reduce, 1) + raises(ValueError, add.reduce, 1) - def test_reduce(self): - from numpypy import add, maximum + def test_reduce_1d(self): + from _numpypy import add, maximum assert add.reduce([1, 2, 3]) == 6 assert maximum.reduce([1]) == 1 assert maximum.reduce([1, 2, 3]) == 3 raises(ValueError, maximum.reduce, []) + def test_reduceND(self): + from numpypy import add, arange + a = arange(12).reshape(3, 4) + assert (add.reduce(a, 0) == [12, 15, 18, 21]).all() + assert (add.reduce(a, 1) == [6.0, 22.0, 38.0]).all() + def test_comparisons(self): import operator - from numpypy import equal, not_equal, less, less_equal, greater, greater_equal + from _numpypy import equal, not_equal, less, less_equal, greater, greater_equal for ufunc, func in [ (equal, operator.eq), diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py --- a/pypy/module/micronumpy/test/test_zjit.py +++ b/pypy/module/micronumpy/test/test_zjit.py @@ -47,6 +47,8 @@ def f(i): interp = InterpreterState(codes[i]) interp.run(space) + if not len(interp.results): + raise Exception("need results") w_res = interp.results[-1] if isinstance(w_res, BaseArray): concr = w_res.get_concrete_or_scalar() @@ -115,6 +117,28 @@ "int_add": 1, "int_ge": 1, "guard_false": 1, "jump": 1, 'arraylen_gc': 1}) + def define_axissum(): + return """ + a = [[1, 2], [3, 4], [5, 6], [7, 8], [9, 10]] + b = sum(a,0) + b -> 1 + """ + + def test_axissum(self): + result = self.run("axissum") + assert result == 30 + # XXX note - the bridge here is fairly crucial and yet it's pretty + # bogus. We need to improve the situation somehow. + self.check_simple_loop({'getinteriorfield_raw': 2, + 'setinteriorfield_raw': 1, + 'arraylen_gc': 1, + 'guard_true': 1, + 'int_lt': 1, + 'jump': 1, + 'float_add': 1, + 'int_add': 3, + }) + def define_prod(): return """ a = |30| @@ -193,9 +217,9 @@ # This is the sum of the ops for both loops, however if you remove the # optimization then you end up with 2 float_adds, so we can still be # sure it was optimized correctly. - self.check_resops({'setinteriorfield_raw': 4, 'getfield_gc': 26, + self.check_resops({'setinteriorfield_raw': 4, 'getfield_gc': 22, 'getarrayitem_gc': 4, 'getarrayitem_gc_pure': 2, - 'getfield_gc_pure': 4, + 'getfield_gc_pure': 8, 'guard_class': 8, 'int_add': 8, 'float_mul': 2, 'jump': 2, 'int_ge': 4, 'getinteriorfield_raw': 4, 'float_add': 2, @@ -212,7 +236,8 @@ def test_ufunc(self): result = self.run("ufunc") assert result == -6 - self.check_simple_loop({"getinteriorfield_raw": 2, "float_add": 1, "float_neg": 1, + self.check_simple_loop({"getinteriorfield_raw": 2, "float_add": 1, + "float_neg": 1, "setinteriorfield_raw": 1, "int_add": 2, "int_ge": 1, "guard_false": 1, "jump": 1, 'arraylen_gc': 1}) @@ -322,10 +347,9 @@ result = self.run("setslice") assert result == 11.0 self.check_trace_count(1) - self.check_simple_loop({'getinteriorfield_raw': 2, 'float_add' : 1, - 'setinteriorfield_raw': 1, 'int_add': 3, - 'int_lt': 1, 'guard_true': 1, 'jump': 1, - 'arraylen_gc': 3}) + self.check_simple_loop({'getinteriorfield_raw': 2, 'float_add': 1, + 'setinteriorfield_raw': 1, 'int_add': 2, + 'int_eq': 1, 'guard_false': 1, 'jump': 1}) def define_virtual_slice(): return """ @@ -339,11 +363,12 @@ result = self.run("virtual_slice") assert result == 4 self.check_trace_count(1) - self.check_simple_loop({'getinteriorfield_raw': 2, 'float_add' : 1, + self.check_simple_loop({'getinteriorfield_raw': 2, 'float_add': 1, 'setinteriorfield_raw': 1, 'int_add': 2, 'int_ge': 1, 'guard_false': 1, 'jump': 1, 'arraylen_gc': 1}) + class TestNumpyOld(LLJitMixin): def setup_class(cls): py.test.skip("old") @@ -377,4 +402,3 @@ result = self.meta_interp(f, [5], listops=True, backendopt=True) assert result == f(5) - diff --git a/pypy/module/pypyjit/__init__.py b/pypy/module/pypyjit/__init__.py --- a/pypy/module/pypyjit/__init__.py +++ b/pypy/module/pypyjit/__init__.py @@ -7,16 +7,21 @@ interpleveldefs = { 'set_param': 'interp_jit.set_param', 'residual_call': 'interp_jit.residual_call', - 'set_compile_hook': 'interp_jit.set_compile_hook', - 'DebugMergePoint': 'interp_resop.W_DebugMergePoint', + 'set_compile_hook': 'interp_resop.set_compile_hook', + 'set_optimize_hook': 'interp_resop.set_optimize_hook', + 'set_abort_hook': 'interp_resop.set_abort_hook', + 'ResOperation': 'interp_resop.WrappedOp', + 'Box': 'interp_resop.WrappedBox', } def setup_after_space_initialization(self): # force the __extend__ hacks to occur early from pypy.module.pypyjit.interp_jit import pypyjitdriver + from pypy.module.pypyjit.policy import pypy_hooks # add the 'defaults' attribute from pypy.rlib.jit import PARAMETERS space = self.space pypyjitdriver.space = space w_obj = space.wrap(PARAMETERS) space.setattr(space.wrap(self), space.wrap('defaults'), w_obj) + pypy_hooks.space = space diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py --- a/pypy/module/pypyjit/interp_jit.py +++ b/pypy/module/pypyjit/interp_jit.py @@ -13,11 +13,7 @@ from pypy.interpreter.pycode import PyCode, CO_GENERATOR from pypy.interpreter.pyframe import PyFrame from pypy.interpreter.pyopcode import ExitFrame -from pypy.interpreter.gateway import unwrap_spec from opcode import opmap -from pypy.rlib.nonconst import NonConstant -from pypy.jit.metainterp.resoperation import rop -from pypy.module.pypyjit.interp_resop import debug_merge_point_from_boxes PyFrame._virtualizable2_ = ['last_instr', 'pycode', 'valuestackdepth', 'locals_stack_w[*]', @@ -51,72 +47,19 @@ def should_unroll_one_iteration(next_instr, is_being_profiled, bytecode): return (bytecode.co_flags & CO_GENERATOR) != 0 -def wrap_oplist(space, logops, operations): - list_w = [] - for op in operations: - if op.getopnum() == rop.DEBUG_MERGE_POINT: - list_w.append(space.wrap(debug_merge_point_from_boxes( - op.getarglist()))) - else: - list_w.append(space.wrap(logops.repr_of_resop(op))) - return list_w - class PyPyJitDriver(JitDriver): reds = ['frame', 'ec'] greens = ['next_instr', 'is_being_profiled', 'pycode'] virtualizables = ['frame'] - def on_compile(self, logger, looptoken, operations, type, next_instr, - is_being_profiled, ll_pycode): - from pypy.rpython.annlowlevel import cast_base_ptr_to_instance - - space = self.space - cache = space.fromcache(Cache) - if cache.in_recursion: - return - if space.is_true(cache.w_compile_hook): - logops = logger._make_log_operations() - list_w = wrap_oplist(space, logops, operations) - pycode = cast_base_ptr_to_instance(PyCode, ll_pycode) - cache.in_recursion = True - try: - space.call_function(cache.w_compile_hook, - space.wrap('main'), - space.wrap(type), - space.newtuple([pycode, - space.wrap(next_instr), - space.wrap(is_being_profiled)]), - space.newlist(list_w)) - except OperationError, e: - e.write_unraisable(space, "jit hook ", cache.w_compile_hook) - cache.in_recursion = False - - def on_compile_bridge(self, logger, orig_looptoken, operations, n): - space = self.space - cache = space.fromcache(Cache) - if cache.in_recursion: - return - if space.is_true(cache.w_compile_hook): - logops = logger._make_log_operations() - list_w = wrap_oplist(space, logops, operations) - cache.in_recursion = True - try: - space.call_function(cache.w_compile_hook, - space.wrap('main'), - space.wrap('bridge'), - space.wrap(n), - space.newlist(list_w)) - except OperationError, e: - e.write_unraisable(space, "jit hook ", cache.w_compile_hook) - cache.in_recursion = False - pypyjitdriver = PyPyJitDriver(get_printable_location = get_printable_location, get_jitcell_at = get_jitcell_at, set_jitcell_at = set_jitcell_at, confirm_enter_jit = confirm_enter_jit, can_never_inline = can_never_inline, should_unroll_one_iteration = - should_unroll_one_iteration) + should_unroll_one_iteration, + name='pypyjit') class __extend__(PyFrame): @@ -223,34 +166,3 @@ '''For testing. Invokes callable(...), but without letting the JIT follow the call.''' return space.call_args(w_callable, __args__) - -class Cache(object): - in_recursion = False - - def __init__(self, space): - self.w_compile_hook = space.w_None - -def set_compile_hook(space, w_hook): - """ set_compile_hook(hook) - - Set a compiling hook that will be called each time a loop is compiled. - The hook will be called with the following signature: - hook(merge_point_type, loop_type, greenkey or guard_number, operations) - - for now merge point type is always `main` - - loop_type can be either `loop` `entry_bridge` or `bridge` - in case loop is not `bridge`, greenkey will be a set of constants - for jit merge point. in case it's `main` it'll be a tuple - (code, offset, is_being_profiled) - - Note that jit hook is not reentrant. It means that if the code - inside the jit hook is itself jitted, it will get compiled, but the - jit hook won't be called for that. - - XXX write down what else - """ - cache = space.fromcache(Cache) - cache.w_compile_hook = w_hook - cache.in_recursion = NonConstant(False) - return space.w_None diff --git a/pypy/module/pypyjit/interp_resop.py b/pypy/module/pypyjit/interp_resop.py --- a/pypy/module/pypyjit/interp_resop.py +++ b/pypy/module/pypyjit/interp_resop.py @@ -1,41 +1,197 @@ -from pypy.interpreter.typedef import TypeDef, interp_attrproperty +from pypy.interpreter.typedef import TypeDef, GetSetProperty from pypy.interpreter.baseobjspace import Wrappable -from pypy.interpreter.gateway import unwrap_spec, interp2app +from pypy.interpreter.gateway import unwrap_spec, interp2app, NoneNotWrapped from pypy.interpreter.pycode import PyCode -from pypy.rpython.lltypesystem import lltype -from pypy.rpython.annlowlevel import cast_base_ptr_to_instance +from pypy.interpreter.error import OperationError +from pypy.rpython.lltypesystem import lltype, llmemory +from pypy.rpython.annlowlevel import cast_base_ptr_to_instance, hlstr from pypy.rpython.lltypesystem.rclass import OBJECT +from pypy.jit.metainterp.resoperation import rop, AbstractResOp +from pypy.rlib.nonconst import NonConstant +from pypy.rlib import jit_hooks -class W_DebugMergePoint(Wrappable): - """ A class representing debug_merge_point JIT operation +class Cache(object): + in_recursion = False + + def __init__(self, space): + self.w_compile_hook = space.w_None + self.w_abort_hook = space.w_None + self.w_optimize_hook = space.w_None + +def wrap_greenkey(space, jitdriver, greenkey, greenkey_repr): + if greenkey is None: + return space.w_None + jitdriver_name = jitdriver.name + if jitdriver_name == 'pypyjit': + next_instr = greenkey[0].getint() + is_being_profiled = greenkey[1].getint() + ll_code = lltype.cast_opaque_ptr(lltype.Ptr(OBJECT), + greenkey[2].getref_base()) + pycode = cast_base_ptr_to_instance(PyCode, ll_code) + return space.newtuple([space.wrap(pycode), space.wrap(next_instr), + space.newbool(bool(is_being_profiled))]) + else: + return space.wrap(greenkey_repr) + +def set_compile_hook(space, w_hook): + """ set_compile_hook(hook) + + Set a compiling hook that will be called each time a loop is compiled. + The hook will be called with the following signature: + hook(jitdriver_name, loop_type, greenkey or guard_number, operations, + assembler_addr, assembler_length) + + jitdriver_name is the name of this particular jitdriver, 'pypyjit' is + the main interpreter loop + + loop_type can be either `loop` `entry_bridge` or `bridge` + in case loop is not `bridge`, greenkey will be a tuple of constants + or a string describing it. + + for the interpreter loop` it'll be a tuple + (code, offset, is_being_profiled) + + assembler_addr is an integer describing where assembler starts, + can be accessed via ctypes, assembler_lenght is the lenght of compiled + asm + + Note that jit hook is not reentrant. It means that if the code + inside the jit hook is itself jitted, it will get compiled, but the + jit hook won't be called for that. """ + cache = space.fromcache(Cache) + cache.w_compile_hook = w_hook + cache.in_recursion = NonConstant(False) - def __init__(self, mp_no, offset, pycode): - self.mp_no = mp_no +def set_optimize_hook(space, w_hook): + """ set_optimize_hook(hook) + + Set a compiling hook that will be called each time a loop is optimized, + but before assembler compilation. This allows to add additional + optimizations on Python level. + + The hook will be called with the following signature: + hook(jitdriver_name, loop_type, greenkey or guard_number, operations) + + jitdriver_name is the name of this particular jitdriver, 'pypyjit' is + the main interpreter loop + + loop_type can be either `loop` `entry_bridge` or `bridge` + in case loop is not `bridge`, greenkey will be a tuple of constants + or a string describing it. + + for the interpreter loop` it'll be a tuple + (code, offset, is_being_profiled) + + Note that jit hook is not reentrant. It means that if the code + inside the jit hook is itself jitted, it will get compiled, but the + jit hook won't be called for that. + + Result value will be the resulting list of operations, or None + """ + cache = space.fromcache(Cache) + cache.w_optimize_hook = w_hook + cache.in_recursion = NonConstant(False) + +def set_abort_hook(space, w_hook): + """ set_abort_hook(hook) + + Set a hook (callable) that will be called each time there is tracing + aborted due to some reason. + + The hook will be called as in: hook(jitdriver_name, greenkey, reason) + + Where reason is the reason for abort, see documentation for set_compile_hook + for descriptions of other arguments. + """ + cache = space.fromcache(Cache) + cache.w_abort_hook = w_hook + cache.in_recursion = NonConstant(False) + +def wrap_oplist(space, logops, operations, ops_offset=None): + l_w = [] + for op in operations: + if ops_offset is None: + ofs = -1 + else: + ofs = ops_offset.get(op, 0) + l_w.append(WrappedOp(jit_hooks._cast_to_gcref(op), ofs, + logops.repr_of_resop(op))) + return l_w + +class WrappedBox(Wrappable): + """ A class representing a single box + """ + def __init__(self, llbox): + self.llbox = llbox + + def descr_getint(self, space): + return space.wrap(jit_hooks.box_getint(self.llbox)) + + at unwrap_spec(no=int) +def descr_new_box(space, w_tp, no): + return WrappedBox(jit_hooks.boxint_new(no)) + +WrappedBox.typedef = TypeDef( + 'Box', + __new__ = interp2app(descr_new_box), + getint = interp2app(WrappedBox.descr_getint), +) + + at unwrap_spec(num=int, offset=int, repr=str, res=WrappedBox) +def descr_new_resop(space, w_tp, num, w_args, res, offset=-1, + repr=''): + args = [space.interp_w(WrappedBox, w_arg).llbox for w_arg in + space.listview(w_args)] + if res is None: + llres = jit_hooks.emptyval() + else: + llres = res.llbox + return WrappedOp(jit_hooks.resop_new(num, args, llres), offset, repr) + +class WrappedOp(Wrappable): + """ A class representing a single ResOperation, wrapped nicely + """ + def __init__(self, op, offset, repr_of_resop): + self.op = op self.offset = offset - self.pycode = pycode + self.repr_of_resop = repr_of_resop def descr_repr(self, space): - return space.wrap('DebugMergePoint()') + return space.wrap(self.repr_of_resop) - at unwrap_spec(mp_no=int, offset=int, pycode=PyCode) -def new_debug_merge_point(space, w_tp, mp_no, offset, pycode): - return W_DebugMergePoint(mp_no, offset, pycode) + def descr_num(self, space): + return space.wrap(jit_hooks.resop_getopnum(self.op)) -def debug_merge_point_from_boxes(boxes): - mp_no = boxes[0].getint() - offset = boxes[2].getint() - llcode = lltype.cast_opaque_ptr(lltype.Ptr(OBJECT), - boxes[4].getref_base()) - pycode = cast_base_ptr_to_instance(PyCode, llcode) - assert pycode is not None - return W_DebugMergePoint(mp_no, offset, pycode) + def descr_name(self, space): + return space.wrap(hlstr(jit_hooks.resop_getopname(self.op))) -W_DebugMergePoint.typedef = TypeDef( - 'DebugMergePoint', - __new__ = interp2app(new_debug_merge_point), - __doc__ = W_DebugMergePoint.__doc__, - __repr__ = interp2app(W_DebugMergePoint.descr_repr), - code = interp_attrproperty('pycode', W_DebugMergePoint), + @unwrap_spec(no=int) + def descr_getarg(self, space, no): + return WrappedBox(jit_hooks.resop_getarg(self.op, no)) + + @unwrap_spec(no=int, box=WrappedBox) + def descr_setarg(self, space, no, box): + jit_hooks.resop_setarg(self.op, no, box.llbox) + + def descr_getresult(self, space): + return WrappedBox(jit_hooks.resop_getresult(self.op)) + + def descr_setresult(self, space, w_box): + box = space.interp_w(WrappedBox, w_box) + jit_hooks.resop_setresult(self.op, box.llbox) + +WrappedOp.typedef = TypeDef( + 'ResOperation', + __doc__ = WrappedOp.__doc__, + __new__ = interp2app(descr_new_resop), + __repr__ = interp2app(WrappedOp.descr_repr), + num = GetSetProperty(WrappedOp.descr_num), + name = GetSetProperty(WrappedOp.descr_name), + getarg = interp2app(WrappedOp.descr_getarg), + setarg = interp2app(WrappedOp.descr_setarg), + result = GetSetProperty(WrappedOp.descr_getresult, + WrappedOp.descr_setresult) ) +WrappedOp.acceptable_as_base_class = False diff --git a/pypy/module/pypyjit/policy.py b/pypy/module/pypyjit/policy.py --- a/pypy/module/pypyjit/policy.py +++ b/pypy/module/pypyjit/policy.py @@ -1,4 +1,112 @@ from pypy.jit.codewriter.policy import JitPolicy +from pypy.rlib.jit import JitHookInterface +from pypy.rlib import jit_hooks +from pypy.interpreter.error import OperationError +from pypy.jit.metainterp.jitprof import counter_names +from pypy.module.pypyjit.interp_resop import wrap_oplist, Cache, wrap_greenkey,\ + WrappedOp + +class PyPyJitIface(JitHookInterface): + def on_abort(self, reason, jitdriver, greenkey, greenkey_repr): + space = self.space + cache = space.fromcache(Cache) + if cache.in_recursion: + return + if space.is_true(cache.w_abort_hook): + cache.in_recursion = True + try: + try: + space.call_function(cache.w_abort_hook, + space.wrap(jitdriver.name), + wrap_greenkey(space, jitdriver, + greenkey, greenkey_repr), + space.wrap(counter_names[reason])) + except OperationError, e: + e.write_unraisable(space, "jit hook ", cache.w_abort_hook) + finally: + cache.in_recursion = False + + def after_compile(self, debug_info): + w_greenkey = wrap_greenkey(self.space, debug_info.get_jitdriver(), + debug_info.greenkey, + debug_info.get_greenkey_repr()) + self._compile_hook(debug_info, w_greenkey) + + def after_compile_bridge(self, debug_info): + self._compile_hook(debug_info, + self.space.wrap(debug_info.fail_descr_no)) + + def before_compile(self, debug_info): + w_greenkey = wrap_greenkey(self.space, debug_info.get_jitdriver(), + debug_info.greenkey, + debug_info.get_greenkey_repr()) + self._optimize_hook(debug_info, w_greenkey) + + def before_compile_bridge(self, debug_info): + self._optimize_hook(debug_info, + self.space.wrap(debug_info.fail_descr_no)) + + def _compile_hook(self, debug_info, w_arg): + space = self.space + cache = space.fromcache(Cache) + if cache.in_recursion: + return + if space.is_true(cache.w_compile_hook): + logops = debug_info.logger._make_log_operations() + list_w = wrap_oplist(space, logops, debug_info.operations, + debug_info.asminfo.ops_offset) + cache.in_recursion = True + try: + try: + jd_name = debug_info.get_jitdriver().name + asminfo = debug_info.asminfo + space.call_function(cache.w_compile_hook, + space.wrap(jd_name), + space.wrap(debug_info.type), + w_arg, + space.newlist(list_w), + space.wrap(asminfo.asmaddr), + space.wrap(asminfo.asmlen)) + except OperationError, e: + e.write_unraisable(space, "jit hook ", cache.w_compile_hook) + finally: + cache.in_recursion = False + + def _optimize_hook(self, debug_info, w_arg): + space = self.space + cache = space.fromcache(Cache) + if cache.in_recursion: + return + if space.is_true(cache.w_optimize_hook): + logops = debug_info.logger._make_log_operations() + list_w = wrap_oplist(space, logops, debug_info.operations) + cache.in_recursion = True + try: + try: + jd_name = debug_info.get_jitdriver().name + w_res = space.call_function(cache.w_optimize_hook, + space.wrap(jd_name), + space.wrap(debug_info.type), + w_arg, + space.newlist(list_w)) + if space.is_w(w_res, space.w_None): + return + l = [] + for w_item in space.listview(w_res): + item = space.interp_w(WrappedOp, w_item) + l.append(jit_hooks._cast_to_resop(item.op)) + del debug_info.operations[:] # modifying operations above is + # probably not a great idea since types may not work + # and we'll end up with half-working list and + # a segfault/fatal RPython error + for elem in l: + debug_info.operations.append(elem) + except OperationError, e: + e.write_unraisable(space, "jit hook ", cache.w_compile_hook) + finally: + cache.in_recursion = False + +pypy_hooks = PyPyJitIface() class PyPyJitPolicy(JitPolicy): @@ -12,12 +120,16 @@ modname == 'thread.os_thread'): return True if '.' in modname: - modname, _ = modname.split('.', 1) + modname, rest = modname.split('.', 1) + else: + rest = '' if modname in ['pypyjit', 'signal', 'micronumpy', 'math', 'exceptions', 'imp', 'sys', 'array', '_ffi', 'itertools', 'operator', 'posix', '_socket', '_sre', '_lsprof', '_weakref', '__pypy__', 'cStringIO', '_collections', 'struct', 'mmap', 'marshal']: + if modname == 'pypyjit' and 'interp_resop' in rest: + return False return True return False diff --git a/pypy/module/pypyjit/test/test_jit_hook.py b/pypy/module/pypyjit/test/test_jit_hook.py --- a/pypy/module/pypyjit/test/test_jit_hook.py +++ b/pypy/module/pypyjit/test/test_jit_hook.py @@ -1,22 +1,40 @@ import py from pypy.conftest import gettestobjspace, option +from pypy.interpreter.gateway import interp2app from pypy.interpreter.pycode import PyCode -from pypy.interpreter.gateway import interp2app -from pypy.jit.metainterp.history import JitCellToken -from pypy.jit.metainterp.resoperation import ResOperation, rop +from pypy.jit.metainterp.history import JitCellToken, ConstInt, ConstPtr +from pypy.jit.metainterp.resoperation import rop from pypy.jit.metainterp.logger import Logger from pypy.rpython.annlowlevel import (cast_instance_to_base_ptr, cast_base_ptr_to_instance) from pypy.rpython.lltypesystem import lltype, llmemory +from pypy.rpython.lltypesystem.rclass import OBJECT from pypy.module.pypyjit.interp_jit import pypyjitdriver +from pypy.module.pypyjit.policy import pypy_hooks from pypy.jit.tool.oparser import parse from pypy.jit.metainterp.typesystem import llhelper +from pypy.jit.metainterp.jitprof import ABORT_TOO_LONG +from pypy.rlib.jit import JitDebugInfo, AsmInfo + +class MockJitDriverSD(object): + class warmstate(object): + @staticmethod + def get_location_str(boxes): + ll_code = lltype.cast_opaque_ptr(lltype.Ptr(OBJECT), + boxes[2].getref_base()) + pycode = cast_base_ptr_to_instance(PyCode, ll_code) + return pycode.co_name + + jitdriver = pypyjitdriver + class MockSD(object): class cpu(object): ts = llhelper + jitdrivers_sd = [MockJitDriverSD] + class AppTestJitHook(object): def setup_class(cls): if option.runappdirect: @@ -24,9 +42,9 @@ space = gettestobjspace(usemodules=('pypyjit',)) cls.space = space w_f = space.appexec([], """(): - def f(): + def function(): pass - return f + return function """) cls.w_f = w_f ll_code = cast_instance_to_base_ptr(w_f.code) @@ -34,41 +52,73 @@ logger = Logger(MockSD()) oplist = parse(""" - [i1, i2] + [i1, i2, p2] i3 = int_add(i1, i2) debug_merge_point(0, 0, 0, 0, ConstPtr(ptr0)) + guard_nonnull(p2) [] guard_true(i3) [] """, namespace={'ptr0': code_gcref}).operations + greenkey = [ConstInt(0), ConstInt(0), ConstPtr(code_gcref)] + offset = {} + for i, op in enumerate(oplist): + if i != 1: + offset[op] = i + + di_loop = JitDebugInfo(MockJitDriverSD, logger, JitCellToken(), + oplist, 'loop', greenkey) + di_loop_optimize = JitDebugInfo(MockJitDriverSD, logger, JitCellToken(), + oplist, 'loop', greenkey) + di_loop.asminfo = AsmInfo(offset, 0, 0) + di_bridge = JitDebugInfo(MockJitDriverSD, logger, JitCellToken(), + oplist, 'bridge', fail_descr_no=0) + di_bridge.asminfo = AsmInfo(offset, 0, 0) def interp_on_compile(): - pypyjitdriver.on_compile(logger, JitCellToken(), oplist, 'loop', - 0, False, ll_code) + di_loop.oplist = cls.oplist + pypy_hooks.after_compile(di_loop) def interp_on_compile_bridge(): - pypyjitdriver.on_compile_bridge(logger, JitCellToken(), oplist, 0) + pypy_hooks.after_compile_bridge(di_bridge) + + def interp_on_optimize(): + di_loop_optimize.oplist = cls.oplist + pypy_hooks.before_compile(di_loop_optimize) + + def interp_on_abort(): + pypy_hooks.on_abort(ABORT_TOO_LONG, pypyjitdriver, greenkey, + 'blah') cls.w_on_compile = space.wrap(interp2app(interp_on_compile)) cls.w_on_compile_bridge = space.wrap(interp2app(interp_on_compile_bridge)) + cls.w_on_abort = space.wrap(interp2app(interp_on_abort)) + cls.w_int_add_num = space.wrap(rop.INT_ADD) + cls.w_on_optimize = space.wrap(interp2app(interp_on_optimize)) + cls.orig_oplist = oplist + + def setup_method(self, meth): + self.__class__.oplist = self.orig_oplist[:] def test_on_compile(self): import pypyjit all = [] - def hook(*args): - assert args[0] == 'main' - assert args[1] in ['loop', 'bridge'] - all.append(args[2:]) + def hook(name, looptype, tuple_or_guard_no, ops, asmstart, asmlen): + all.append((name, looptype, tuple_or_guard_no, ops)) From noreply at buildbot.pypy.org Sat Jan 14 22:35:57 2012 From: noreply at buildbot.pypy.org (mattip) Date: Sat, 14 Jan 2012 22:35:57 +0100 (CET) Subject: [pypy-commit] pypy matrixmath-dot: fix merge Message-ID: <20120114213557.4AADB71025D@wyvern.cs.uni-duesseldorf.de> Author: mattip Branch: matrixmath-dot Changeset: r51332:cdee39d4ae23 Date: 2012-01-14 23:05 +0200 http://bitbucket.org/pypy/pypy/changeset/cdee39d4ae23/ Log: fix merge diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -194,7 +194,8 @@ n_new_elems_used = 1 oldI = -1 n_old_elems_to_use = old_shape[-1] - for s in new_shape[::-1]: + for i in range(len(new_shape) - 1, -1, -1): + s = new_shape[i] new_strides.insert(0, cur_step * n_new_elems_used) n_new_elems_used *= s while n_new_elems_used > n_old_elems_to_use: @@ -383,7 +384,6 @@ the second-to-last of `b`:: dot(a, b)[i,j,k,m] = sum(a[i,j,:] * b[k,:,m])''' - #numpy's doc string :) w_other = convert_to_array(space, w_other) if isinstance(w_other, Scalar): return self.descr_mul(space, w_other) @@ -420,10 +420,13 @@ for os in out_shape: out_size *= os out_ndims = len(out_shape) - #TODO: what should the order be? C or F? + # TODO: what should the order be? C or F? arr = W_NDimArray(out_size, out_shape, dtype=dtype) - out_iter = view_iter_from_arr(arr) - #TODO: invalidate self, w_other with arr ? + # TODO: this is all a bogus mess of previous work, + # rework within the context of transformations + ''' + out_iter = ViewIterator(arr.start, arr.strides, arr.backstrides, arr.shape) + # TODO: invalidate self, w_other with arr ? while not out_iter.done(): my_index = self.start other_index = w_other.start @@ -443,6 +446,7 @@ value = w_res.descr_sum(space) arr.setitem(out_iter.get_offset(), value) out_iter = out_iter.next(out_ndims) + ''' return arr def get_concrete(self): diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -516,7 +516,7 @@ b = a * a for i in range(5): assert b[i] == i * i - assert b.dtype is numpypy.dtype(int) + assert b.dtype is a.dtype a = _numpypy.array(range(5), dtype=bool) b = a * a @@ -740,8 +740,7 @@ def test_sum(self): from _numpypy import array a = array(range(5)) - b = a.sum() - assert b == 10 + assert a.sum() == 10 assert a[:4].sum() == 6 a = array([True] * 5, bool) From noreply at buildbot.pypy.org Sat Jan 14 22:35:58 2012 From: noreply at buildbot.pypy.org (mattip) Date: Sat, 14 Jan 2012 22:35:58 +0100 (CET) Subject: [pypy-commit] pypy default: numpypy: rename kwarg 'dim' to 'axis' Message-ID: <20120114213558.78E7671025D@wyvern.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r51333:eb0269c21eec Date: 2012-01-14 23:25 +0200 http://bitbucket.org/pypy/pypy/changeset/eb0269c21eec/ Log: numpypy: rename kwarg 'dim' to 'axis' diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -287,11 +287,11 @@ descr_rmod = _binop_right_impl("mod") def _reduce_ufunc_impl(ufunc_name, promote_to_largest=False): - def impl(self, space, w_dim=None): - if space.is_w(w_dim, space.w_None): - w_dim = space.wrap(-1) + def impl(self, space, w_axis=None): + if space.is_w(w_axis, space.w_None): + w_axis = space.wrap(-1) return getattr(interp_ufuncs.get(space), ufunc_name).reduce(space, - self, True, promote_to_largest, w_dim) + self, True, promote_to_largest, w_axis) return func_with_new_name(impl, "reduce_%s_impl" % ufunc_name) descr_sum = _reduce_ufunc_impl("add") @@ -569,14 +569,14 @@ ) return w_result - def descr_mean(self, space, w_dim=None): - if space.is_w(w_dim, space.w_None): - w_dim = space.wrap(-1) + def descr_mean(self, space, w_axis=None): + if space.is_w(w_axis, space.w_None): + w_axis = space.wrap(-1) w_denom = space.wrap(self.size) else: - dim = space.int_w(w_dim) + dim = space.int_w(w_axis) w_denom = space.wrap(self.shape[dim]) - return space.div(self.descr_sum_promote(space, w_dim), w_denom) + return space.div(self.descr_sum_promote(space, w_axis), w_denom) def descr_var(self, space): # var = mean((values - mean(values)) ** 2) diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -733,6 +733,7 @@ a = array(range(105)).reshape(3, 5, 7) b = mean(a, axis=0) b[0,0]==35. + assert a.mean(axis=0)[0, 0] == 35 assert (b == array(range(35, 70), dtype=float).reshape(5, 7)).all() assert (mean(a, 2) == array(range(0, 15), dtype=float).reshape(3, 5) * 7 + 3).all() @@ -755,6 +756,7 @@ assert array([]).sum() == 0.0 raises(ValueError, 'array([]).max()') assert (a.sum(0) == [30, 35, 40]).all() + assert (a.sum(axis=0) == [30, 35, 40]).all() assert (a.sum(1) == [3, 12, 21, 30, 39]).all() assert (a.max(0) == [12, 13, 14]).all() assert (a.max(1) == [2, 5, 8, 11, 14]).all() From noreply at buildbot.pypy.org Sun Jan 15 13:40:31 2012 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 15 Jan 2012 13:40:31 +0100 (CET) Subject: [pypy-commit] pypy numpy-indexing-by-arrays: close old branch will be redone Message-ID: <20120115124031.1D535820D8@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: numpy-indexing-by-arrays Changeset: r51334:08302909741e Date: 2012-01-14 22:59 +0200 http://bitbucket.org/pypy/pypy/changeset/08302909741e/ Log: close old branch will be redone From noreply at buildbot.pypy.org Sun Jan 15 13:40:32 2012 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 15 Jan 2012 13:40:32 +0100 (CET) Subject: [pypy-commit] pypy numpy-indexing-by-arrays-2: a branch to work on indexing by arrays - start by introducing new way of Message-ID: <20120115124032.7F4E382C03@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: numpy-indexing-by-arrays-2 Changeset: r51335:ad840d9d2a35 Date: 2012-01-15 14:36 +0200 http://bitbucket.org/pypy/pypy/changeset/ad840d9d2a35/ Log: a branch to work on indexing by arrays - start by introducing new way of describing shapes diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py --- a/pypy/module/micronumpy/interp_dtype.py +++ b/pypy/module/micronumpy/interp_dtype.py @@ -84,6 +84,12 @@ def descr_get_shape(self, space): return space.newtuple([]) + def is_int_type(self): + return self.kind == SIGNEDLTR or self.kind == UNSIGNEDLTR + + def is_bool_type(self): + return self.kind == BOOLLTR + W_Dtype.typedef = TypeDef("dtype", __module__ = "numpypy", __new__ = interp2app(W_Dtype.descr__new__.im_func), diff --git a/pypy/module/micronumpy/interp_iter.py b/pypy/module/micronumpy/interp_iter.py --- a/pypy/module/micronumpy/interp_iter.py +++ b/pypy/module/micronumpy/interp_iter.py @@ -4,6 +4,30 @@ from pypy.module.micronumpy.strides import calculate_broadcast_strides,\ calculate_slice_strides +# structures to describe slicing + +class BaseChunk(object): + pass + +class Chunk(BaseChunk): + def __init__(self, start, stop, step, lgt): + self.start = start + self.stop = stop + self.step = step + self.lgt = lgt + + def extend_shape(self, shape): + if self.step != 0: + shape.append(self.lgt) + +class IntArrayChunk(BaseChunk): + def __init__(self, arr): + self.arr = arr.get_concrete() + +class BoolArrayChunk(BaseChunk): + def __init__(self, arr): + self.arr = arr.get_concrete() + class BaseTransform(object): pass diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -2,14 +2,15 @@ from pypy.interpreter.error import OperationError, operationerrfmt from pypy.interpreter.gateway import interp2app, NoneNotWrapped from pypy.interpreter.typedef import TypeDef, GetSetProperty -from pypy.module.micronumpy import interp_ufuncs, interp_dtype, signature +from pypy.module.micronumpy import interp_ufuncs, interp_dtype, signature,\ + interp_boxes from pypy.module.micronumpy.strides import calculate_slice_strides from pypy.rlib import jit from pypy.rpython.lltypesystem import lltype, rffi from pypy.tool.sourcetools import func_with_new_name from pypy.rlib.rstring import StringBuilder from pypy.module.micronumpy.interp_iter import ArrayIterator, OneDimIterator,\ - SkipLastAxisIterator + SkipLastAxisIterator, Chunk numpy_driver = jit.JitDriver( greens=['shapelen', 'sig'], @@ -40,7 +41,6 @@ name='numpy_slice', ) - def _find_shape_and_elems(space, w_iterable): shape = [space.len_w(w_iterable)] batch = space.listview(w_iterable) @@ -479,8 +479,8 @@ def _prepare_slice_args(self, space, w_idx): if (space.isinstance_w(w_idx, space.w_int) or space.isinstance_w(w_idx, space.w_slice)): - return [space.decode_index4(w_idx, self.shape[0])] - return [space.decode_index4(w_item, self.shape[i]) for i, w_item in + return [Chunk(*space.decode_index4(w_idx, self.shape[0]))] + return [Chunk(*space.decode_index4(w_item, self.shape[i])) for i, w_item in enumerate(space.fixedview(w_idx))] def descr_getitem(self, space, w_idx): @@ -509,9 +509,8 @@ def create_slice(self, chunks): shape = [] i = -1 - for i, (start_, stop, step, lgt) in enumerate(chunks): - if step != 0: - shape.append(lgt) + for i, chunk in enumerate(chunks): + chunk.extend_shape(shape) s = i + 1 assert s >= 0 shape += self.shape[s:] @@ -938,7 +937,7 @@ builder.append('\n' + indent) else: builder.append(indent) - view = self.create_slice([(i, 0, 0, 1)]).get_concrete() + view = self.create_slice([Chunk(i, 0, 0, 1)]).get_concrete() view.to_str(space, comma, builder, indent=indent + ' ', use_ellipsis=use_ellipsis) if i < self.shape[0] - 1: @@ -955,7 +954,7 @@ builder.append(indent) # create_slice requires len(chunks) > 1 in order to reduce # shape - view = self.create_slice([(i, 0, 0, 1)]).get_concrete() + view = self.create_slice([Chunk(i, 0, 0, 1)]).get_concrete() view.to_str(space, comma, builder, indent=indent + ' ', use_ellipsis=use_ellipsis) i += 1 diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py --- a/pypy/module/micronumpy/strides.py +++ b/pypy/module/micronumpy/strides.py @@ -10,12 +10,12 @@ rstart = start rshape = [] i = -1 - for i, (start_, stop, step, lgt) in enumerate(chunks): - if step != 0: - rstrides.append(strides[i] * step) - rbackstrides.append(strides[i] * (lgt - 1) * step) - rshape.append(lgt) - rstart += strides[i] * start_ + for i, chunk in enumerate(chunks): + if chunk.step != 0: + rstrides.append(strides[i] * chunk.step) + rbackstrides.append(strides[i] * (chunk.lgt - 1) * chunk.step) + rshape.append(chunk.lgt) + rstart += strides[i] * chunk.start # add a reminder s = i + 1 assert s >= 0 diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -2,6 +2,7 @@ import py from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest from pypy.module.micronumpy.interp_numarray import W_NDimArray, shape_agreement +from pypy.module.micronumpy.interp_iter import Chunk from pypy.module.micronumpy import signature from pypy.interpreter.error import OperationError from pypy.conftest import gettestobjspace @@ -37,53 +38,54 @@ def test_create_slice_f(self): a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'F') - s = a.create_slice([(3, 0, 0, 1)]) + s = a.create_slice([Chunk(3, 0, 0, 1)]) assert s.start == 3 assert s.strides == [10, 50] assert s.backstrides == [40, 100] - s = a.create_slice([(1, 9, 2, 4)]) + s = a.create_slice([Chunk(1, 9, 2, 4)]) assert s.start == 1 assert s.strides == [2, 10, 50] assert s.backstrides == [6, 40, 100] - s = a.create_slice([(1, 5, 3, 2), (1, 2, 1, 1), (1, 0, 0, 1)]) + s = a.create_slice([Chunk(1, 5, 3, 2), Chunk(1, 2, 1, 1), Chunk(1, 0, 0, 1)]) assert s.shape == [2, 1] assert s.strides == [3, 10] assert s.backstrides == [3, 0] - s = a.create_slice([(0, 10, 1, 10), (2, 0, 0, 1)]) + s = a.create_slice([Chunk(0, 10, 1, 10), Chunk(2, 0, 0, 1)]) assert s.start == 20 assert s.shape == [10, 3] def test_create_slice_c(self): a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'C') - s = a.create_slice([(3, 0, 0, 1)]) + s = a.create_slice([Chunk(3, 0, 0, 1)]) assert s.start == 45 assert s.strides == [3, 1] assert s.backstrides == [12, 2] - s = a.create_slice([(1, 9, 2, 4)]) + s = a.create_slice([Chunk(1, 9, 2, 4)]) assert s.start == 15 assert s.strides == [30, 3, 1] assert s.backstrides == [90, 12, 2] - s = a.create_slice([(1, 5, 3, 2), (1, 2, 1, 1), (1, 0, 0, 1)]) + s = a.create_slice([Chunk(1, 5, 3, 2), Chunk(1, 2, 1, 1), + Chunk(1, 0, 0, 1)]) assert s.start == 19 assert s.shape == [2, 1] assert s.strides == [45, 3] assert s.backstrides == [45, 0] - s = a.create_slice([(0, 10, 1, 10), (2, 0, 0, 1)]) + s = a.create_slice([Chunk(0, 10, 1, 10), Chunk(2, 0, 0, 1)]) assert s.start == 6 assert s.shape == [10, 3] def test_slice_of_slice_f(self): a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'F') - s = a.create_slice([(5, 0, 0, 1)]) + s = a.create_slice([Chunk(5, 0, 0, 1)]) assert s.start == 5 - s2 = s.create_slice([(3, 0, 0, 1)]) + s2 = s.create_slice([Chunk(3, 0, 0, 1)]) assert s2.shape == [3] assert s2.strides == [50] assert s2.parent is a assert s2.backstrides == [100] assert s2.start == 35 - s = a.create_slice([(1, 5, 3, 2)]) - s2 = s.create_slice([(0, 2, 1, 2), (2, 0, 0, 1)]) + s = a.create_slice([Chunk(1, 5, 3, 2)]) + s2 = s.create_slice([Chunk(0, 2, 1, 2), Chunk(2, 0, 0, 1)]) assert s2.shape == [2, 3] assert s2.strides == [3, 50] assert s2.backstrides == [3, 100] @@ -91,16 +93,16 @@ def test_slice_of_slice_c(self): a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), order='C') - s = a.create_slice([(5, 0, 0, 1)]) + s = a.create_slice([Chunk(5, 0, 0, 1)]) assert s.start == 15 * 5 - s2 = s.create_slice([(3, 0, 0, 1)]) + s2 = s.create_slice([Chunk(3, 0, 0, 1)]) assert s2.shape == [3] assert s2.strides == [1] assert s2.parent is a assert s2.backstrides == [2] assert s2.start == 5 * 15 + 3 * 3 - s = a.create_slice([(1, 5, 3, 2)]) - s2 = s.create_slice([(0, 2, 1, 2), (2, 0, 0, 1)]) + s = a.create_slice([Chunk(1, 5, 3, 2)]) + s2 = s.create_slice([Chunk(0, 2, 1, 2), Chunk(2, 0, 0, 1)]) assert s2.shape == [2, 3] assert s2.strides == [45, 1] assert s2.backstrides == [45, 2] @@ -108,14 +110,14 @@ def test_negative_step_f(self): a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'F') - s = a.create_slice([(9, -1, -2, 5)]) + s = a.create_slice([Chunk(9, -1, -2, 5)]) assert s.start == 9 assert s.strides == [-2, 10, 50] assert s.backstrides == [-8, 40, 100] def test_negative_step_c(self): a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), order='C') - s = a.create_slice([(9, -1, -2, 5)]) + s = a.create_slice([Chunk(9, -1, -2, 5)]) assert s.start == 135 assert s.strides == [-30, 3, 1] assert s.backstrides == [-120, 12, 2] @@ -124,7 +126,7 @@ a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'F') r = a._index_of_single_item(self.space, self.newtuple(1, 2, 2)) assert r == 1 + 2 * 10 + 2 * 50 - s = a.create_slice([(0, 10, 1, 10), (2, 0, 0, 1)]) + s = a.create_slice([Chunk(0, 10, 1, 10), Chunk(2, 0, 0, 1)]) r = s._index_of_single_item(self.space, self.newtuple(1, 0)) assert r == a._index_of_single_item(self.space, self.newtuple(1, 2, 0)) r = s._index_of_single_item(self.space, self.newtuple(1, 1)) @@ -134,7 +136,7 @@ a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'C') r = a._index_of_single_item(self.space, self.newtuple(1, 2, 2)) assert r == 1 * 3 * 5 + 2 * 3 + 2 - s = a.create_slice([(0, 10, 1, 10), (2, 0, 0, 1)]) + s = a.create_slice([Chunk(0, 10, 1, 10), Chunk(2, 0, 0, 1)]) r = s._index_of_single_item(self.space, self.newtuple(1, 0)) assert r == a._index_of_single_item(self.space, self.newtuple(1, 2, 0)) r = s._index_of_single_item(self.space, self.newtuple(1, 1)) @@ -1298,6 +1300,17 @@ assert isinstance(i['data'][0], int) raises(TypeError, getattr, array(3), '__array_interface__') + def test_array_indexing_one_elem(self): + skip("not yet") + from _numpypy import array, arange + raises(IndexError, 'arange(3)[array([3.5])]') + a = arange(3)[array([1])] + assert a == 1 + assert a[0] == 1 + raises(IndexError,'arange(3)[array([15])]') + assert arange(3)[array([-3])] == 0 + raises(IndexError,'arange(3)[array([-15])]') + assert arange(3)[array(1)] == 1 class AppTestSupport(BaseNumpyAppTest): def setup_class(cls): From pullrequests-noreply at bitbucket.org Sun Jan 15 22:25:55 2012 From: pullrequests-noreply at bitbucket.org (Bitbucket) Date: Sun, 15 Jan 2012 21:25:55 -0000 Subject: [pypy-commit] [OPEN] Pull request #20 for pypy/pypy: Improvements to the JVM backend, this time with tests Message-ID: A new pull request has been opened by Micha? Bendowski. benol/pypy has changes to be pulled into pypy/pypy. https://bitbucket.org/pypy/pypy/pull-request/20/improvements-to-the-jvm-backend-this-time Title: Improvements to the JVM backend, this time with tests Changes to be pulled: 8efdb1515b2c by Micha? Bendowski: "Handle the 'jit_is_virtual' opcode by always returning False" f1d52bd62e15 by Micha? Bendowski: "Add a missing cast from Unsigned to UnsignedLongLong in the JVM backend." 8ff039cadd19 by Micha? Bendowski: "Declare oo_primitives that should implement some rffi operations. For now the a?" 25d4d323cb5f by Micha? Bendowski: "Fix compute_unique_id to support built-ins in ootype. Otherwise the translation?" 7f09531dd6a9 by Micha? Bendowski: "Fix userspace builders in ootype Implement the getlength() method of StringBuil?" 578e69f273f9 by Micha? Bendowski: "Add files generated by PyCharm to .hgignore" -- This is an issue notification from bitbucket.org. You are receiving this either because you are the participating in a pull request, or you are following it. From noreply at buildbot.pypy.org Sun Jan 15 23:03:30 2012 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 15 Jan 2012 23:03:30 +0100 (CET) Subject: [pypy-commit] extradoc extradoc: add a blog draft Message-ID: <20120115220330.1A988820D8@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: extradoc Changeset: r4034:0d6b3b5ff0f7 Date: 2012-01-16 00:03 +0200 http://bitbucket.org/pypy/extradoc/changeset/0d6b3b5ff0f7/ Log: add a blog draft diff --git a/blog/draft/numpy-internship.rst b/blog/draft/numpy-internship.rst new file mode 100644 --- /dev/null +++ b/blog/draft/numpy-internship.rst @@ -0,0 +1,39 @@ + +Hello everyone. + +I would like to inform everyone that there is a very interesting opportunity +for doing an internship at `NCAR`_ in the lovely town of `Boulder`_, situated +on the foothils of Rocky Mountains. Before you read on, make sure you are: + +* A student of a US University, who is legally eglibale to work in the US. + +* At least finishing second year this year. + +* Apply before February 3rd. + +The internship itself is to focus on using PyPy (in some way) to provide +a high performance numeric kernel for an atmospheric model and measuring how +fast we can go. This is very well in line with what the current effort on +NumPy in PyPy is about. The internship will be mentored by `Davide del Vento`_ +and I hope to have some influence over where it goes myself :-) + +Few interesting links: + +* `program website`_ + +* `internship proposal`_ - note that the actual roadmap is very flexible, as + long as it's numeric kernel of an atmospheric model using PyPy. + +Fell free to contact Davide for details about the proposal and `pypy-dev`_ or +me directly for details about pypy + +.. _`Davide del Vento`: http://www.linkedin.com/in/delvento +.. _`NCAR`: http://ncar.ucar.edu/ +.. _`Boulder`: http://en.wikipedia.org/wiki/Boulder,_Colorado +.. _`program website`: http://www2.cisl.ucar.edu/siparcs/ +.. _`internship proposal`: http://www2.cisl.ucar.edu/siparcs/opportunities/ad +.. _`pypy-dev`: http://mail.python.org/mailman/listinfo/pypy-dev + +Cheers, +fijal + From noreply at buildbot.pypy.org Sun Jan 15 23:32:26 2012 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 15 Jan 2012 23:32:26 +0100 (CET) Subject: [pypy-commit] extradoc extradoc: (confluecne) typos Message-ID: <20120115223226.C8E96820D8@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: extradoc Changeset: r4035:15ad7db9d445 Date: 2012-01-16 00:32 +0200 http://bitbucket.org/pypy/extradoc/changeset/15ad7db9d445/ Log: (confluecne) typos diff --git a/blog/draft/numpy-internship.rst b/blog/draft/numpy-internship.rst --- a/blog/draft/numpy-internship.rst +++ b/blog/draft/numpy-internship.rst @@ -1,31 +1,31 @@ -Hello everyone. +Hello, everyone -I would like to inform everyone that there is a very interesting opportunity +I would like to inform you that there is a very interesting opportunity for doing an internship at `NCAR`_ in the lovely town of `Boulder`_, situated -on the foothils of Rocky Mountains. Before you read on, make sure you are: +on the foothils of Rocky Mountains. Before you read on, make sure: -* A student of a US University, who is legally eglibale to work in the US. +* are a student of a US University, who is legally egligible to work in the US -* At least finishing second year this year. +* are at least finishing second year this year, -* Apply before February 3rd. +* apply before February 3rd. -The internship itself is to focus on using PyPy (in some way) to provide -a high performance numeric kernel for an atmospheric model and measuring how -fast we can go. This is very well in line with what the current effort on +The internship itself will focus on using PyPy (in some way) to provide +a high performance numeric kernel for an atmospheric model, and measuring how +fast we can go. This is very much in line with what the current effort on NumPy in PyPy is about. The internship will be mentored by `Davide del Vento`_ and I hope to have some influence over where it goes myself :-) -Few interesting links: +A few interesting links: * `program website`_ * `internship proposal`_ - note that the actual roadmap is very flexible, as long as it's numeric kernel of an atmospheric model using PyPy. -Fell free to contact Davide for details about the proposal and `pypy-dev`_ or -me directly for details about pypy +Feel free to contact Davide for details about the proposal and `pypy-dev`_ or +me directly for details about PyPy. .. _`Davide del Vento`: http://www.linkedin.com/in/delvento .. _`NCAR`: http://ncar.ucar.edu/ From noreply at buildbot.pypy.org Sun Jan 15 23:36:19 2012 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 15 Jan 2012 23:36:19 +0100 (CET) Subject: [pypy-commit] extradoc extradoc: (confluence) more typos Message-ID: <20120115223619.01E45820D8@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: extradoc Changeset: r4036:469f9b257694 Date: 2012-01-16 00:35 +0200 http://bitbucket.org/pypy/extradoc/changeset/469f9b257694/ Log: (confluence) more typos diff --git a/blog/draft/numpy-internship.rst b/blog/draft/numpy-internship.rst --- a/blog/draft/numpy-internship.rst +++ b/blog/draft/numpy-internship.rst @@ -3,11 +3,11 @@ I would like to inform you that there is a very interesting opportunity for doing an internship at `NCAR`_ in the lovely town of `Boulder`_, situated -on the foothils of Rocky Mountains. Before you read on, make sure: +on the foothils of Rocky Mountains. Before you read on, make sure you: -* are a student of a US University, who is legally egligible to work in the US +* are a student of a US University, who is legally eligible to work in the US -* are at least finishing second year this year, +* are at least finishing second year this year * apply before February 3rd. @@ -22,7 +22,7 @@ * `program website`_ * `internship proposal`_ - note that the actual roadmap is very flexible, as - long as it's numeric kernel of an atmospheric model using PyPy. + long as it's a numeric kernel of an atmospheric model using PyPy. Feel free to contact Davide for details about the proposal and `pypy-dev`_ or me directly for details about PyPy. From noreply at buildbot.pypy.org Mon Jan 16 09:48:54 2012 From: noreply at buildbot.pypy.org (bivab) Date: Mon, 16 Jan 2012 09:48:54 +0100 (CET) Subject: [pypy-commit] pypy arm-backend-2: add (set|get)_interiorfield_raw methods Message-ID: <20120116084854.7A170820D8@wyvern.cs.uni-duesseldorf.de> Author: David Schneider Branch: arm-backend-2 Changeset: r51336:6b5ffc5a9e85 Date: 2012-01-16 09:45 +0100 http://bitbucket.org/pypy/pypy/changeset/6b5ffc5a9e85/ Log: add (set|get)_interiorfield_raw methods diff --git a/pypy/jit/backend/arm/opassembler.py b/pypy/jit/backend/arm/opassembler.py --- a/pypy/jit/backend/arm/opassembler.py +++ b/pypy/jit/backend/arm/opassembler.py @@ -688,6 +688,7 @@ signed = op.getdescr().fielddescr.is_field_signed() self._ensure_result_bit_extension(res_loc, fieldsize.value, signed) return fcond + emit_op_getinteriorfield_raw = emit_op_getinteriorfield_gc def emit_op_setinteriorfield_gc(self, op, arglocs, regalloc, fcond): (base_loc, index_loc, value_loc, @@ -715,6 +716,7 @@ else: assert 0 return fcond + emit_op_setinteriorfield_raw = emit_op_setinteriorfield_gc class ArrayOpAssember(object): diff --git a/pypy/jit/backend/arm/regalloc.py b/pypy/jit/backend/arm/regalloc.py --- a/pypy/jit/backend/arm/regalloc.py +++ b/pypy/jit/backend/arm/regalloc.py @@ -772,6 +772,7 @@ result_loc = self.force_allocate_reg(op.result) return [base_loc, index_loc, result_loc, ofs_loc, imm(ofs), imm(itemsize), imm(fieldsize)] + prepare_op_getinteriorfield_raw = prepare_op_getinteriorfield_gc def prepare_op_setinteriorfield_gc(self, op, fcond): t = unpack_interiorfielddescr(op.getdescr()) @@ -788,6 +789,7 @@ self.assembler.load(ofs_loc, immofs) return [base_loc, index_loc, value_loc, ofs_loc, imm(ofs), imm(itemsize), imm(fieldsize)] + prepare_op_setinteriorfield_raw = prepare_op_setinteriorfield_gc def prepare_op_arraylen_gc(self, op, fcond): arraydescr = op.getdescr() From noreply at buildbot.pypy.org Mon Jan 16 09:48:55 2012 From: noreply at buildbot.pypy.org (bivab) Date: Mon, 16 Jan 2012 09:48:55 +0100 (CET) Subject: [pypy-commit] pypy arm-backend-2: print some information when hitting a missing operation in the backend Message-ID: <20120116084855.AC102820D8@wyvern.cs.uni-duesseldorf.de> Author: David Schneider Branch: arm-backend-2 Changeset: r51337:a317879ec998 Date: 2012-01-16 09:48 +0100 http://bitbucket.org/pypy/pypy/changeset/a317879ec998/ Log: print some information when hitting a missing operation in the backend diff --git a/pypy/jit/backend/arm/assembler.py b/pypy/jit/backend/arm/assembler.py --- a/pypy/jit/backend/arm/assembler.py +++ b/pypy/jit/backend/arm/assembler.py @@ -1071,10 +1071,13 @@ def notimplemented_op(self, op, arglocs, regalloc, fcond): + print "[ARM/asm] %s not implemented" % op.getopname() raise NotImplementedError(op) def notimplemented_op_with_guard(self, op, guard_op, arglocs, regalloc, fcond): + print "[ARM/asm] %s with guard %s not implemented" % \ + (op.getopname(), guard_op.getopname()) raise NotImplementedError(op) asm_operations = [notimplemented_op] * (rop._LAST + 1) diff --git a/pypy/jit/backend/arm/regalloc.py b/pypy/jit/backend/arm/regalloc.py --- a/pypy/jit/backend/arm/regalloc.py +++ b/pypy/jit/backend/arm/regalloc.py @@ -1166,10 +1166,13 @@ def notimplemented(self, op, fcond): + print "[ARM/regalloc] %s not implemented" % op.getopname() raise NotImplementedError(op) def notimplemented_with_guard(self, op, guard_op, fcond): + print "[ARM/regalloc] %s with guard %s not implemented" % \ + (op.getopname(), guard_op.getopname()) raise NotImplementedError(op) operations = [notimplemented] * (rop._LAST + 1) From noreply at buildbot.pypy.org Mon Jan 16 11:21:59 2012 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 16 Jan 2012 11:21:59 +0100 (CET) Subject: [pypy-commit] extradoc extradoc: Planning session for today Message-ID: <20120116102159.2ECCE820D8@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: extradoc Changeset: r4037:c36311962789 Date: 2012-01-16 11:21 +0100 http://bitbucket.org/pypy/extradoc/changeset/c36311962789/ Log: Planning session for today diff --git a/sprintinfo/leysin-winter-2012/planning.txt b/sprintinfo/leysin-winter-2012/planning.txt new file mode 100644 --- /dev/null +++ b/sprintinfo/leysin-winter-2012/planning.txt @@ -0,0 +1,25 @@ + +People present +-------------- + +* Antonio Cuni +* Armin Rigo +* Romain Guillebert + +Things we want to do +-------------------- + +* some skiing + +* review the JVM backend pull request (anto) + +* py3k (romain) + +* ffistruct + +* Cython backend (romain) + +* STM + start work on the GC (anto, arigo) + +* concurrent-marksweep GC From noreply at buildbot.pypy.org Mon Jan 16 11:22:00 2012 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 16 Jan 2012 11:22:00 +0100 (CET) Subject: [pypy-commit] extradoc extradoc: merge heads Message-ID: <20120116102200.5F3DC820D8@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: extradoc Changeset: r4038:588db0e8396b Date: 2012-01-16 11:21 +0100 http://bitbucket.org/pypy/extradoc/changeset/588db0e8396b/ Log: merge heads diff --git a/blog/draft/numpy-internship.rst b/blog/draft/numpy-internship.rst new file mode 100644 --- /dev/null +++ b/blog/draft/numpy-internship.rst @@ -0,0 +1,39 @@ + +Hello, everyone + +I would like to inform you that there is a very interesting opportunity +for doing an internship at `NCAR`_ in the lovely town of `Boulder`_, situated +on the foothils of Rocky Mountains. Before you read on, make sure you: + +* are a student of a US University, who is legally eligible to work in the US + +* are at least finishing second year this year + +* apply before February 3rd. + +The internship itself will focus on using PyPy (in some way) to provide +a high performance numeric kernel for an atmospheric model, and measuring how +fast we can go. This is very much in line with what the current effort on +NumPy in PyPy is about. The internship will be mentored by `Davide del Vento`_ +and I hope to have some influence over where it goes myself :-) + +A few interesting links: + +* `program website`_ + +* `internship proposal`_ - note that the actual roadmap is very flexible, as + long as it's a numeric kernel of an atmospheric model using PyPy. + +Feel free to contact Davide for details about the proposal and `pypy-dev`_ or +me directly for details about PyPy. + +.. _`Davide del Vento`: http://www.linkedin.com/in/delvento +.. _`NCAR`: http://ncar.ucar.edu/ +.. _`Boulder`: http://en.wikipedia.org/wiki/Boulder,_Colorado +.. _`program website`: http://www2.cisl.ucar.edu/siparcs/ +.. _`internship proposal`: http://www2.cisl.ucar.edu/siparcs/opportunities/ad +.. _`pypy-dev`: http://mail.python.org/mailman/listinfo/pypy-dev + +Cheers, +fijal + diff --git a/planning/micronumpy.txt b/planning/micronumpy.txt --- a/planning/micronumpy.txt +++ b/planning/micronumpy.txt @@ -17,8 +17,6 @@ - more attributes/methods on numpy.flatiter -- axis= parameter to various methods - - expose ndarray.ctypes - subclassing ndarray (instantiating subcalsses curently returns the wrong type) From noreply at buildbot.pypy.org Mon Jan 16 14:05:02 2012 From: noreply at buildbot.pypy.org (rguillebert) Date: Mon, 16 Jan 2012 14:05:02 +0100 (CET) Subject: [pypy-commit] pypy py3k: Add a failing test for extended attribute unpacking Message-ID: <20120116130502.A0BC9820D8@wyvern.cs.uni-duesseldorf.de> Author: Romain Guillebert Branch: py3k Changeset: r51338:5500c9127aab Date: 2012-01-16 14:04 +0100 http://bitbucket.org/pypy/pypy/changeset/5500c9127aab/ Log: Add a failing test for extended attribute unpacking diff --git a/pypy/interpreter/pyparser/test/test_pyparse.py b/pypy/interpreter/pyparser/test/test_pyparse.py --- a/pypy/interpreter/pyparser/test/test_pyparse.py +++ b/pypy/interpreter/pyparser/test/test_pyparse.py @@ -153,3 +153,6 @@ self.parse('0b1101') self.parse('0b0l') py.test.raises(SyntaxError, self.parse, "0b112") + + def test_new_extended_unpacking(self): + self.parse('(a, *rest, b) = 1, 2, 3, 4, 5') From noreply at buildbot.pypy.org Mon Jan 16 14:18:02 2012 From: noreply at buildbot.pypy.org (rguillebert) Date: Mon, 16 Jan 2012 14:18:02 +0100 (CET) Subject: [pypy-commit] pypy py3k: Rename the test commited earlier, rewrite tests that should pass. Message-ID: <20120116131802.3EEEF820D8@wyvern.cs.uni-duesseldorf.de> Author: Romain Guillebert Branch: py3k Changeset: r51339:34ff01570492 Date: 2012-01-16 14:17 +0100 http://bitbucket.org/pypy/pypy/changeset/34ff01570492/ Log: Rename the test commited earlier, rewrite tests that should pass. diff --git a/pypy/interpreter/pyparser/test/test_pyparse.py b/pypy/interpreter/pyparser/test/test_pyparse.py --- a/pypy/interpreter/pyparser/test/test_pyparse.py +++ b/pypy/interpreter/pyparser/test/test_pyparse.py @@ -146,13 +146,13 @@ def test_new_octal_literal(self): self.parse('0777') self.parse('0o777') - self.parse('0o777L') + py.test.raises(SyntaxError, self.parse, '0o777L') py.test.raises(SyntaxError, self.parse, "0o778") def test_new_binary_literal(self): self.parse('0b1101') - self.parse('0b0l') + py.test.raises(SyntaxError, self.parse, '0b0l') py.test.raises(SyntaxError, self.parse, "0b112") - def test_new_extended_unpacking(self): + def test_py3k_extended_unpacking(self): self.parse('(a, *rest, b) = 1, 2, 3, 4, 5') From noreply at buildbot.pypy.org Mon Jan 16 14:25:15 2012 From: noreply at buildbot.pypy.org (rguillebert) Date: Mon, 16 Jan 2012 14:25:15 +0100 (CET) Subject: [pypy-commit] pypy py3k: Parsing 0777 should fail on py3k Message-ID: <20120116132515.E2CD4820D8@wyvern.cs.uni-duesseldorf.de> Author: Romain Guillebert Branch: py3k Changeset: r51340:851536b685a6 Date: 2012-01-16 14:24 +0100 http://bitbucket.org/pypy/pypy/changeset/851536b685a6/ Log: Parsing 0777 should fail on py3k diff --git a/pypy/interpreter/pyparser/test/test_pyparse.py b/pypy/interpreter/pyparser/test/test_pyparse.py --- a/pypy/interpreter/pyparser/test/test_pyparse.py +++ b/pypy/interpreter/pyparser/test/test_pyparse.py @@ -144,7 +144,6 @@ py.test.raises(SyntaxError, self.parse, "b'a\\n") def test_new_octal_literal(self): - self.parse('0777') self.parse('0o777') py.test.raises(SyntaxError, self.parse, '0o777L') py.test.raises(SyntaxError, self.parse, "0o778") @@ -154,5 +153,8 @@ py.test.raises(SyntaxError, self.parse, '0b0l') py.test.raises(SyntaxError, self.parse, "0b112") + def test_py3k_reject_old_binary_literal(self): + py.test.raises(SyntaxError, self.parse, '0777') + def test_py3k_extended_unpacking(self): self.parse('(a, *rest, b) = 1, 2, 3, 4, 5') From noreply at buildbot.pypy.org Mon Jan 16 14:59:34 2012 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 16 Jan 2012 14:59:34 +0100 (CET) Subject: [pypy-commit] pypy stm: Add a (skipped) test about using the minimark GC. Message-ID: <20120116135934.7DC8A820D8@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stm Changeset: r51341:91a0ee8fc4ab Date: 2012-01-16 11:53 +0100 http://bitbucket.org/pypy/pypy/changeset/91a0ee8fc4ab/ Log: Add a (skipped) test about using the minimark GC. diff --git a/pypy/translator/stm/test/targetdemo.py b/pypy/translator/stm/test/targetdemo.py --- a/pypy/translator/stm/test/targetdemo.py +++ b/pypy/translator/stm/test/targetdemo.py @@ -3,29 +3,35 @@ from pypy.translator.stm import rstm -NUM_THREADS = 4 -LENGTH = 5000 - - class Node: def __init__(self, value): self.value = value self.next = None +class Global: + NUM_THREADS = 4 + LENGTH = 5000 + USE_MEMORY = False + anchor = Node(-1) +glob = Global() + def add_at_end_of_chained_list(node, value): + x = Node(value) while node.next: node = node.next - newnode = Node(value) + if glob.USE_MEMORY: + x = Node(value) + newnode = x node.next = newnode def check_chained_list(node): - seen = [0] * (LENGTH+1) - seen[-1] = NUM_THREADS + seen = [0] * (glob.LENGTH+1) + seen[-1] = glob.NUM_THREADS while node is not None: value = node.value #print value - if not (0 <= value < LENGTH): + if not (0 <= value < glob.LENGTH): print "node.value out of bounds:", value raise AssertionError seen[value] += 1 @@ -34,19 +40,15 @@ value, seen[value]) raise AssertionError node = node.next - if seen[LENGTH-1] != NUM_THREADS: + if seen[glob.LENGTH-1] != glob.NUM_THREADS: print "seen[LENGTH-1] != NUM_THREADS" raise AssertionError print "check ok!" -class Global: - anchor = Node(-1) -glob = Global() - def run_me(): print "thread starting..." - for i in range(LENGTH): + for i in range(glob.LENGTH): add_at_end_of_chained_list(glob.anchor, i) rstm.transaction_boundary() print "thread done." @@ -57,11 +59,17 @@ def entry_point(argv): print "hello world" + if len(argv) > 1: + glob.NUM_THREADS = int(argv[1]) + if len(argv) > 2: + glob.LENGTH = int(argv[2]) + if len(argv) > 3: + glob.USE_MEMORY = bool(int(argv[3])) glob.done = 0 - for i in range(NUM_THREADS): + for i in range(glob.NUM_THREADS): ll_thread.start_new_thread(run_me, ()) print "sleeping..." - while glob.done < NUM_THREADS: # poor man's lock + while glob.done < glob.NUM_THREADS: # poor man's lock time.sleep(1) print "done sleeping." check_chained_list(glob.anchor.next) diff --git a/pypy/translator/stm/test/test_transform.py b/pypy/translator/stm/test/test_transform.py --- a/pypy/translator/stm/test/test_transform.py +++ b/pypy/translator/stm/test/test_transform.py @@ -183,12 +183,13 @@ # ____________________________________________________________ class CompiledSTMTests(StandaloneTests): + gc = "none" def compile(self, entry_point): from pypy.config.pypyoption import get_pypy_config self.config = get_pypy_config(translating=True) self.config.translation.stm = True - self.config.translation.gc = "none" + self.config.translation.gc = self.gc # # Prevent the RaiseAnalyzer from just emitting "WARNING: Unknown # operation". We want instead it to crash. diff --git a/pypy/translator/stm/test/test_ztranslated.py b/pypy/translator/stm/test/test_ztranslated.py --- a/pypy/translator/stm/test/test_ztranslated.py +++ b/pypy/translator/stm/test/test_ztranslated.py @@ -1,3 +1,4 @@ +import py from pypy.translator.stm.test.test_transform import CompiledSTMTests from pypy.translator.stm.test import targetdemo @@ -6,6 +7,18 @@ def test_hello_world(self): t, cbuilder = self.compile(targetdemo.entry_point) - data = cbuilder.cmdexec('') + data = cbuilder.cmdexec('4 5000') assert 'done sleeping.' in data assert 'check ok!' in data + + +class TestSTMFramework(CompiledSTMTests): + gc = "minimark" + + def test_hello_world(self): + py.test.skip("in-progress") + t, cbuilder = self.compile(targetdemo.entry_point) + data = cbuilder.cmdexec('4 5000 1') + # ^^^ should check that it doesn't take 1G of RAM + assert 'done sleeping.' in data + assert 'check ok!' in data From noreply at buildbot.pypy.org Mon Jan 16 14:59:35 2012 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 16 Jan 2012 14:59:35 +0100 (CET) Subject: [pypy-commit] pypy stm: Revert 0782958b144f. No longer needed. Message-ID: <20120116135935.AB3AF820D8@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stm Changeset: r51342:382a5969beda Date: 2012-01-16 14:58 +0100 http://bitbucket.org/pypy/pypy/changeset/382a5969beda/ Log: Revert 0782958b144f. No longer needed. diff --git a/pypy/doc/discussion/stm_todo.txt b/pypy/doc/discussion/stm_todo.txt --- a/pypy/doc/discussion/stm_todo.txt +++ b/pypy/doc/discussion/stm_todo.txt @@ -7,4 +7,3 @@ e23ab2c195c1 Added a number of "# XXX --- custom version for STM ---" 31f2ed861176 One more - 0782958b144f Hard-coded the STM logic in rffi.aroundstate diff --git a/pypy/rpython/lltypesystem/rffi.py b/pypy/rpython/lltypesystem/rffi.py --- a/pypy/rpython/lltypesystem/rffi.py +++ b/pypy/rpython/lltypesystem/rffi.py @@ -313,21 +313,11 @@ AroundFnPtr = lltype.Ptr(lltype.FuncType([], lltype.Void)) class AroundState: - # XXX for stm with need to comment out this, and use a custom logic -## def _freeze_(self): -## self.before = None # or a regular RPython function -## self.after = None # or a regular RPython function -## return False - @staticmethod - def before(): - from pypy.translator.stm import rstm - rstm.commit_transaction() - @staticmethod - def after(): - from pypy.translator.stm import rstm - rstm.begin_inevitable_transaction() + _alloc_flavor_ = "raw" def _freeze_(self): - return True + self.before = None # or a regular RPython function + self.after = None # or a regular RPython function + return False aroundstate = AroundState() aroundstate._freeze_() From noreply at buildbot.pypy.org Mon Jan 16 14:59:36 2012 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 16 Jan 2012 14:59:36 +0100 (CET) Subject: [pypy-commit] pypy stm: (antocuni, arigo) Message-ID: <20120116135936.F32A6820D8@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stm Changeset: r51343:2965c13c2427 Date: 2012-01-16 14:59 +0100 http://bitbucket.org/pypy/pypy/changeset/2965c13c2427/ Log: (antocuni, arigo) Start to refactor the world. The idea is not to build on RPython threads any more. diff --git a/pypy/rpython/lltypesystem/lloperation.py b/pypy/rpython/lltypesystem/lloperation.py --- a/pypy/rpython/lltypesystem/lloperation.py +++ b/pypy/rpython/lltypesystem/lloperation.py @@ -401,13 +401,7 @@ 'stm_setarrayitem': LLOp(), 'stm_getinteriorfield': LLOp(sideeffects=False, canrun=True), 'stm_setinteriorfield': LLOp(), - - 'stm_begin_transaction': LLOp(), - 'stm_commit_transaction': LLOp(), - 'stm_begin_inevitable_transaction': LLOp(), - 'stm_transaction_boundary': LLOp(), - 'stm_declare_variable': LLOp(), - 'stm_try_inevitable': LLOp(), + 'stm_become_inevitable':LLOp(), # __________ address operations __________ diff --git a/pypy/translator/c/funcgen.py b/pypy/translator/c/funcgen.py --- a/pypy/translator/c/funcgen.py +++ b/pypy/translator/c/funcgen.py @@ -227,6 +227,7 @@ yield '\treturn NULL;' yield '}' if self.exception_policy == "stm": + xxxx yield 'STM_MAKE_INEVITABLE();' retval = self.expr(block.inputargs[0]) if self.exception_policy != "exc_helper": @@ -608,12 +609,6 @@ OP_STM_SETARRAYITEM = _OP_STM OP_STM_GETINTERIORFIELD = _OP_STM OP_STM_SETINTERIORFIELD = _OP_STM - OP_STM_BEGIN_TRANSACTION = _OP_STM - OP_STM_COMMIT_TRANSACTION = _OP_STM - OP_STM_BEGIN_INEVITABLE_TRANSACTION = _OP_STM - OP_STM_TRANSACTION_BOUNDARY = _OP_STM - OP_STM_DECLARE_VARIABLE = _OP_STM - OP_STM_TRY_INEVITABLE = _OP_STM def OP_PTR_NONZERO(self, op): diff --git a/pypy/translator/c/genc.py b/pypy/translator/c/genc.py --- a/pypy/translator/c/genc.py +++ b/pypy/translator/c/genc.py @@ -134,7 +134,7 @@ if self.config.translation.stm: from pypy.translator.stm import transform transformer = transform.STMTransformer(self.translator) - transformer.transform(self.getentrypointptr()) + transformer.transform() log.info("Software Transactional Memory transformation applied") gcpolicyclass = self.get_gcpolicyclass() diff --git a/pypy/translator/stm/_rffi_stm.py b/pypy/translator/stm/_rffi_stm.py --- a/pypy/translator/stm/_rffi_stm.py +++ b/pypy/translator/stm/_rffi_stm.py @@ -26,16 +26,16 @@ descriptor_init = llexternal('stm_descriptor_init', [], lltype.Void) descriptor_done = llexternal('stm_descriptor_done', [], lltype.Void) -begin_transaction = llexternal('STM_begin_transaction', [], lltype.Void) -begin_inevitable_transaction = llexternal('stm_begin_inevitable_transaction', - [], lltype.Void) -commit_transaction = llexternal('stm_commit_transaction', [], lltype.Signed) +##begin_transaction = llexternal('STM_begin_transaction', [], lltype.Void) +##begin_inevitable_transaction = llexternal('stm_begin_inevitable_transaction', +## [], lltype.Void) +##commit_transaction = llexternal('stm_commit_transaction', [], lltype.Signed) try_inevitable = llexternal('stm_try_inevitable', [], lltype.Void) -descriptor_init_and_being_inevitable_transaction = llexternal( - 'stm_descriptor_init_and_being_inevitable_transaction', [], lltype.Void) -commit_transaction_and_descriptor_done = llexternal( - 'stm_commit_transaction_and_descriptor_done', [], lltype.Void) +##descriptor_init_and_being_inevitable_transaction = llexternal( +## 'stm_descriptor_init_and_being_inevitable_transaction', [], lltype.Void) +##commit_transaction_and_descriptor_done = llexternal( +## 'stm_commit_transaction_and_descriptor_done', [], lltype.Void) stm_read_word = llexternal('stm_read_word', [SignedP], lltype.Signed) stm_write_word = llexternal('stm_write_word', [SignedP, lltype.Signed], diff --git a/pypy/translator/stm/funcgen.py b/pypy/translator/stm/funcgen.py --- a/pypy/translator/stm/funcgen.py +++ b/pypy/translator/stm/funcgen.py @@ -1,7 +1,7 @@ from pypy.rpython.lltypesystem import lltype, rffi from pypy.objspace.flow.model import Constant from pypy.translator.c.support import cdecl, c_string_constant -from pypy.translator.stm.rstm import size_of_voidp +from pypy.translator.stm.llstm import size_of_voidp def _stm_generic_get(funcgen, op, expr, simple_struct=False): diff --git a/pypy/translator/stm/rstm.py b/pypy/translator/stm/llstm.py rename from pypy/translator/stm/rstm.py rename to pypy/translator/stm/llstm.py --- a/pypy/translator/stm/rstm.py +++ b/pypy/translator/stm/llstm.py @@ -1,11 +1,20 @@ +""" +This file is mostly here for testing. Usually, transform.py will transform +the getfields (etc) that occur in the graphs directly into stm_getfields, +which are operations that are recognized by the C backend. See funcgen.py +which contains similar logic, which is run by the C backend. +""" import sys -from pypy.rpython.lltypesystem import lltype, rffi +from pypy.rpython.lltypesystem import lltype, rffi, rclass from pypy.rpython.extregistry import ExtRegistryEntry from pypy.translator.stm import _rffi_stm from pypy.annotation import model as annmodel from pypy.objspace.flow.model import Constant from pypy.rlib.rarithmetic import r_uint, r_ulonglong from pypy.rlib import longlong2float +from pypy.rlib.objectmodel import specialize +from pypy.rpython.annlowlevel import cast_instance_to_base_ptr +from pypy.rpython.annlowlevel import cast_base_ptr_to_instance size_of_voidp = rffi.sizeof(rffi.VOIDP) assert size_of_voidp & (size_of_voidp - 1) == 0 @@ -83,21 +92,9 @@ "NOT_RPYTHON" raise NotImplementedError("sorry") -def begin_transaction(): - "NOT_RPYTHON. For tests only" - raise NotImplementedError("hard to really emulate") - -def commit_transaction(): - "NOT_RPYTHON. For tests only" - raise NotImplementedError("hard to really emulate") - -def begin_inevitable_transaction(): - "NOT_RPYTHON. For tests only, and at start up, but not in normal code." - raise NotImplementedError("hard to really emulate") - -def transaction_boundary(): - "NOT_RPYTHON. This is the one normally used" - raise NotImplementedError("hard to really emulate") +##def stm_setarrayitem(arrayptr, index, value): +## "NOT_RPYTHON" +## raise NotImplementedError("sorry") # ____________________________________________________________ @@ -149,13 +146,13 @@ resulttype = hop.r_result) -class ExtEntry(ExtRegistryEntry): - _about_ = (begin_transaction, commit_transaction, - begin_inevitable_transaction, transaction_boundary) +##class ExtEntry(ExtRegistryEntry): +## _about_ = (begin_transaction, commit_transaction, +## begin_inevitable_transaction, transaction_boundary) - def compute_result_annotation(self): - return None +## def compute_result_annotation(self): +## return None - def specialize_call(self, hop): - hop.exception_cannot_occur() - hop.genop("stm_" + self.instance.__name__, []) +## def specialize_call(self, hop): +## hop.exception_cannot_occur() +## hop.genop("stm_" + self.instance.__name__, []) diff --git a/pypy/translator/stm/llstminterp.py b/pypy/translator/stm/llstminterp.py --- a/pypy/translator/stm/llstminterp.py +++ b/pypy/translator/stm/llstminterp.py @@ -1,6 +1,6 @@ from pypy.rpython.lltypesystem import lltype from pypy.rpython.llinterp import LLFrame, LLException -from pypy.translator.stm import rstm +##from pypy.translator.stm import rstm from pypy.translator.stm.transform import op_in_set, ALWAYS_ALLOW_OPERATIONS diff --git a/pypy/translator/stm/src_stm/et.c b/pypy/translator/stm/src_stm/et.c --- a/pypy/translator/stm/src_stm/et.c +++ b/pypy/translator/stm/src_stm/et.c @@ -515,6 +515,8 @@ long stm_read_word(long* addr) { struct tx_descriptor *d = thread_descriptor; + if (!d) + return *addr; #ifdef RPY_STM_ASSERT assert(d->transaction_active); #endif @@ -579,6 +581,11 @@ void stm_write_word(long* addr, long val) { struct tx_descriptor *d = thread_descriptor; + if (!d) + { + *addr = val; + return; + } #ifdef RPY_STM_ASSERT assert(d->transaction_active); #endif diff --git a/pypy/translator/stm/test/test_funcgen.py b/pypy/translator/stm/test/test_funcgen.py --- a/pypy/translator/stm/test/test_funcgen.py +++ b/pypy/translator/stm/test/test_funcgen.py @@ -1,9 +1,55 @@ from pypy.rpython.lltypesystem import lltype, rffi from pypy.rlib.rarithmetic import r_longlong, r_singlefloat from pypy.translator.stm.test.test_transform import CompiledSTMTests -from pypy.translator.stm import rstm +#from pypy.rlib import rstm +from pypy.translator.c.test.test_standalone import StandaloneTests +from pypy.rlib.debug import debug_print +class TestRStm(object): + + def compile(self, entry_point): + from pypy.translator.translator import TranslationContext + from pypy.annotation.listdef import s_list_of_strings + from pypy.translator.c.genc import CStandaloneBuilder + from pypy.translator.tool.cbuild import ExternalCompilationInfo + t = TranslationContext() + t.config.translation.gc = 'boehm' + t.buildannotator().build_types(entry_point, [s_list_of_strings]) + t.buildrtyper().specialize() + t.stm_transformation_applied = True # not really, but for these tests + cbuilder = CStandaloneBuilder(t, entry_point, t.config) + force_debug = ExternalCompilationInfo(pre_include_bits=[ + "#define RPY_ASSERT 1\n" + "#define RPY_LL_ASSERT 1\n" + ]) + cbuilder.eci = cbuilder.eci.merge(force_debug) + cbuilder.generate_source() + cbuilder.compile() + return t, cbuilder + + def test_compiled_stm_getfield(self): + from pypy.translator.stm.test import test_llstm + def entry_point(argv): + test_llstm.test_stm_getfield() + debug_print('ok!') + return 0 + t, cbuilder = self.compile(entry_point) + _, data = cbuilder.cmdexec('', err=True) + assert data.endswith('ok!\n') + + def test_compiled_stm_setfield(self): + from pypy.translator.stm.test import test_llstm + def entry_point(argv): + test_llstm.test_stm_setfield() + debug_print('ok!') + return 0 + t, cbuilder = self.compile(entry_point) + _, data = cbuilder.cmdexec('', err=True) + assert data.endswith('ok!\n') + +# ____________________________________________________________ + A = lltype.GcStruct('A', ('x', lltype.Signed), ('y', lltype.Signed), ('c1', lltype.Char), ('c2', lltype.Char), ('c3', lltype.Char), ('l', lltype.SignedLongLong), diff --git a/pypy/translator/stm/test/test_rstm.py b/pypy/translator/stm/test/test_llstm.py rename from pypy/translator/stm/test/test_rstm.py rename to pypy/translator/stm/test/test_llstm.py --- a/pypy/translator/stm/test/test_rstm.py +++ b/pypy/translator/stm/test/test_llstm.py @@ -1,9 +1,8 @@ import py from pypy.translator.stm._rffi_stm import * -from pypy.translator.stm.rstm import * +from pypy.translator.stm.llstm import * from pypy.rpython.annlowlevel import llhelper from pypy.rlib.rarithmetic import r_longlong, r_singlefloat -from pypy.rlib.debug import debug_print A = lltype.Struct('A', ('x', lltype.Signed), ('y', lltype.Signed), @@ -146,50 +145,3 @@ assert float(a.sb) == float(rs2b) assert a.y == 10 lltype.free(a, flavor='raw') - -# ____________________________________________________________ - -from pypy.translator.translator import TranslationContext -from pypy.annotation.listdef import s_list_of_strings -from pypy.translator.c.genc import CStandaloneBuilder -from pypy.translator.tool.cbuild import ExternalCompilationInfo - -class StmTests(object): - config = None - - def compile(self, entry_point): - t = TranslationContext(self.config) - t.config.translation.gc = 'boehm' - t.buildannotator().build_types(entry_point, [s_list_of_strings]) - t.buildrtyper().specialize() - t.stm_transformation_applied = True # not really, but for these tests - cbuilder = CStandaloneBuilder(t, entry_point, t.config) - force_debug = ExternalCompilationInfo(pre_include_bits=[ - "#define RPY_ASSERT 1\n" - "#define RPY_LL_ASSERT 1\n" - ]) - cbuilder.eci = cbuilder.eci.merge(force_debug) - cbuilder.generate_source() - cbuilder.compile() - return t, cbuilder - - -class TestRStm(StmTests): - - def test_compiled_stm_getfield(self): - def entry_point(argv): - test_stm_getfield() - debug_print('ok!') - return 0 - t, cbuilder = self.compile(entry_point) - _, data = cbuilder.cmdexec('', err=True) - assert data.endswith('ok!\n') - - def test_compiled_stm_setfield(self): - def entry_point(argv): - test_stm_setfield() - debug_print('ok!') - return 0 - t, cbuilder = self.compile(entry_point) - _, data = cbuilder.cmdexec('', err=True) - assert data.endswith('ok!\n') diff --git a/pypy/translator/stm/test/test_transform.py b/pypy/translator/stm/test/test_transform.py --- a/pypy/translator/stm/test/test_transform.py +++ b/pypy/translator/stm/test/test_transform.py @@ -3,7 +3,7 @@ from pypy.objspace.flow.model import summary from pypy.translator.stm.llstminterp import eval_stm_graph from pypy.translator.stm.transform import transform_graph -from pypy.translator.stm import rstm +##from pypy.translator.stm import rstm from pypy.translator.c.test.test_standalone import StandaloneTests from pypy.rlib.debug import debug_print from pypy.conftest import option @@ -198,7 +198,7 @@ try: res = StandaloneTests.compile(self, entry_point, debug=True) finally: - del RaiseAnalyzer.fail_on_unknown_operation + RaiseAnalyzer.fail_on_unknown_operation = False return res diff --git a/pypy/translator/stm/transform.py b/pypy/translator/stm/transform.py --- a/pypy/translator/stm/transform.py +++ b/pypy/translator/stm/transform.py @@ -23,18 +23,18 @@ def __init__(self, translator=None): self.translator = translator - def transform(self, entrypointptr): + def transform(self): ##, entrypointptr): assert not hasattr(self.translator, 'stm_transformation_applied') - entrypointgraph = entrypointptr._obj.graph +## entrypointgraph = entrypointptr._obj.graph for graph in self.translator.graphs: - self.seen_transaction_boundary = False - self.seen_gc_stack_bottom = False +## self.seen_transaction_boundary = False +## self.seen_gc_stack_bottom = False self.transform_graph(graph) - if self.seen_transaction_boundary: - self.add_stm_declare_variable(graph) - if self.seen_gc_stack_bottom: - self.add_descriptor_init_stuff(graph) - self.add_descriptor_init_stuff(entrypointgraph, main=True) +## if self.seen_transaction_boundary: +## self.add_stm_declare_variable(graph) +## if self.seen_gc_stack_bottom: +## self.add_descriptor_init_stuff(graph) +## self.add_descriptor_init_stuff(entrypointgraph, main=True) self.translator.stm_transformation_applied = True def transform_block(self, block): @@ -68,42 +68,42 @@ for block in graph.iterblocks(): self.transform_block(block) - def add_descriptor_init_stuff(self, graph, main=False): - if main: - self._add_calls_around(graph, - _rffi_stm.begin_inevitable_transaction, - _rffi_stm.commit_transaction) - self._add_calls_around(graph, - _rffi_stm.descriptor_init, - _rffi_stm.descriptor_done) +## def add_descriptor_init_stuff(self, graph, main=False): +## if main: +## self._add_calls_around(graph, +## _rffi_stm.begin_inevitable_transaction, +## _rffi_stm.commit_transaction) +## self._add_calls_around(graph, +## _rffi_stm.descriptor_init, +## _rffi_stm.descriptor_done) - def _add_calls_around(self, graph, f_init, f_done): - c_init = Constant(f_init, lltype.typeOf(f_init)) - c_done = Constant(f_done, lltype.typeOf(f_done)) - # - block = graph.startblock - v = varoftype(lltype.Void) - op = SpaceOperation('direct_call', [c_init], v) - block.operations.insert(0, op) - # - v = copyvar(self.translator.annotator, graph.getreturnvar()) - extrablock = Block([v]) - v_none = varoftype(lltype.Void) - newop = SpaceOperation('direct_call', [c_done], v_none) - extrablock.operations = [newop] - extrablock.closeblock(Link([v], graph.returnblock)) - for block in graph.iterblocks(): - if block is not extrablock: - for link in block.exits: - if link.target is graph.returnblock: - link.target = extrablock - checkgraph(graph) +## def _add_calls_around(self, graph, f_init, f_done): +## c_init = Constant(f_init, lltype.typeOf(f_init)) +## c_done = Constant(f_done, lltype.typeOf(f_done)) +## # +## block = graph.startblock +## v = varoftype(lltype.Void) +## op = SpaceOperation('direct_call', [c_init], v) +## block.operations.insert(0, op) +## # +## v = copyvar(self.translator.annotator, graph.getreturnvar()) +## extrablock = Block([v]) +## v_none = varoftype(lltype.Void) +## newop = SpaceOperation('direct_call', [c_done], v_none) +## extrablock.operations = [newop] +## extrablock.closeblock(Link([v], graph.returnblock)) +## for block in graph.iterblocks(): +## if block is not extrablock: +## for link in block.exits: +## if link.target is graph.returnblock: +## link.target = extrablock +## checkgraph(graph) - def add_stm_declare_variable(self, graph): - block = graph.startblock - v = varoftype(lltype.Void) - op = SpaceOperation('stm_declare_variable', [], v) - block.operations.insert(0, op) +## def add_stm_declare_variable(self, graph): +## block = graph.startblock +## v = varoftype(lltype.Void) +## op = SpaceOperation('stm_declare_variable', [], v) +## block.operations.insert(0, op) # ---------- @@ -173,22 +173,22 @@ op1 = SpaceOperation('stm_setinteriorfield', op.args, op.result) newoperations.append(op1) - def stt_stm_transaction_boundary(self, newoperations, op): - self.seen_transaction_boundary = True - v_result = op.result - # record in op.args the list of variables that are alive across - # this call - block = self.current_block - vars = set() - for op in block.operations[:self.current_op_index:-1]: - vars.discard(op.result) - vars.update(op.args) - for link in block.exits: - vars.update(link.args) - vars.update(link.getextravars()) - livevars = [v for v in vars if isinstance(v, Variable)] - newop = SpaceOperation('stm_transaction_boundary', livevars, v_result) - newoperations.append(newop) +## def stt_stm_transaction_boundary(self, newoperations, op): +## self.seen_transaction_boundary = True +## v_result = op.result +## # record in op.args the list of variables that are alive across +## # this call +## block = self.current_block +## vars = set() +## for op in block.operations[:self.current_op_index:-1]: +## vars.discard(op.result) +## vars.update(op.args) +## for link in block.exits: +## vars.update(link.args) +## vars.update(link.getextravars()) +## livevars = [v for v in vars if isinstance(v, Variable)] +## newop = SpaceOperation('stm_transaction_boundary', livevars, v_result) +## newoperations.append(newop) def stt_malloc(self, newoperations, op): flags = op.args[1].value @@ -199,7 +199,7 @@ return flags['flavor'] == 'gc' def stt_gc_stack_bottom(self, newoperations, op): - self.seen_gc_stack_bottom = True +## self.seen_gc_stack_bottom = True newoperations.append(op) @@ -210,7 +210,7 @@ def turn_inevitable(newoperations, info): c_info = Constant(info, lltype.Void) - op1 = SpaceOperation('stm_try_inevitable', [c_info], + op1 = SpaceOperation('stm_become_inevitable', [c_info], varoftype(lltype.Void)) newoperations.append(op1) From noreply at buildbot.pypy.org Mon Jan 16 15:43:48 2012 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 16 Jan 2012 15:43:48 +0100 (CET) Subject: [pypy-commit] pypy stm: (antocuni, arigo) Message-ID: <20120116144348.2D288820D8@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stm Changeset: r51344:7dc69a93d5be Date: 2012-01-16 15:43 +0100 http://bitbucket.org/pypy/pypy/changeset/7dc69a93d5be/ Log: (antocuni, arigo) Fix the llstminterp. Wondering a bit what is its purpose... diff --git a/pypy/translator/stm/TODO.txt b/pypy/translator/stm/TODO.txt new file mode 100644 --- /dev/null +++ b/pypy/translator/stm/TODO.txt @@ -0,0 +1,9 @@ + + +* turn calls to stm_read_word() to: + if (d) x = stm_read_word(a); else x = *a; + + this way, gcc will optimize: + if (d) x = stm_read_word(a); else x = *a; + if (d) y = stm_read_word(b); else y = *b; + to check !d only once if it is null diff --git a/pypy/translator/stm/llstm.py b/pypy/translator/stm/llstm.py --- a/pypy/translator/stm/llstm.py +++ b/pypy/translator/stm/llstm.py @@ -92,9 +92,13 @@ "NOT_RPYTHON" raise NotImplementedError("sorry") -##def stm_setarrayitem(arrayptr, index, value): -## "NOT_RPYTHON" -## raise NotImplementedError("sorry") +def stm_setarrayitem(arrayptr, index, value): + "NOT_RPYTHON" + raise NotImplementedError("sorry") + +def stm_become_inevitable(why): + "NOT_RPYTHON" + raise NotImplementedError("sorry") # ____________________________________________________________ @@ -146,13 +150,28 @@ resulttype = hop.r_result) -##class ExtEntry(ExtRegistryEntry): -## _about_ = (begin_transaction, commit_transaction, -## begin_inevitable_transaction, transaction_boundary) +class ExtEntry(ExtRegistryEntry): + _about_ = stm_setarrayitem -## def compute_result_annotation(self): -## return None + def compute_result_annotation(self, s_arrayptr, s_index, s_newvalue): + return None -## def specialize_call(self, hop): -## hop.exception_cannot_occur() -## hop.genop("stm_" + self.instance.__name__, []) + def specialize_call(self, hop): + r_arrayptr = hop.args_r[0] + v_arrayptr, v_index, v_newvalue = hop.inputargs(r_arrayptr, + lltype.Signed, + hop.args_r[2]) + hop.exception_cannot_occur() + hop.genop('stm_setarrayitem', [v_arrayptr, v_index, v_newvalue]) + + +class ExtEntry(ExtRegistryEntry): + _about_ = stm_become_inevitable + + def compute_result_annotation(self, s_why): + return None + + def specialize_call(self, hop): + hop.exception_cannot_occur() + c_why = hop.inputconst(lltype.Signed, 42) # XXX + hop.genop("stm_become_inevitable", [c_why]) diff --git a/pypy/translator/stm/llstminterp.py b/pypy/translator/stm/llstminterp.py --- a/pypy/translator/stm/llstminterp.py +++ b/pypy/translator/stm/llstminterp.py @@ -146,44 +146,38 @@ raise ForbiddenInstructionInSTMMode(stm_mode, self.graph) def opstm_stm_getfield(self, struct, fieldname): - self.check_stm_mode(lambda m: m != "not_in_transaction") return LLFrame.op_getfield(self, struct, fieldname) def opstm_stm_setfield(self, struct, fieldname, value): - self.check_stm_mode(lambda m: m != "not_in_transaction") LLFrame.op_setfield(self, struct, fieldname, value) def opstm_stm_getarrayitem(self, array, index): - self.check_stm_mode(lambda m: m != "not_in_transaction") return LLFrame.op_getarrayitem(self, array, index) def opstm_stm_setarrayitem(self, array, index, value): - self.check_stm_mode(lambda m: m != "not_in_transaction") LLFrame.op_setarrayitem(self, array, index, value) def opstm_stm_getinteriorfield(self, obj, *offsets): - self.check_stm_mode(lambda m: m != "not_in_transaction") return LLFrame.op_getinteriorfield(self, obj, *offsets) def opstm_stm_setinteriorfield(self, obj, *fieldnamesval): - self.check_stm_mode(lambda m: m != "not_in_transaction") LLFrame.op_setinteriorfield(self, obj, *fieldnamesval) - def opstm_stm_begin_transaction(self): - self.check_stm_mode(lambda m: m == "not_in_transaction") - self.llinterpreter.stm_mode = "regular_transaction" - self.llinterpreter.last_transaction_started_in_frame = self +## def opstm_stm_begin_transaction(self): +## self.check_stm_mode(lambda m: m == "not_in_transaction") +## self.llinterpreter.stm_mode = "regular_transaction" +## self.llinterpreter.last_transaction_started_in_frame = self - def opstm_stm_commit_transaction(self): - self.check_stm_mode(lambda m: m != "not_in_transaction") - self.llinterpreter.stm_mode = "not_in_transaction" +## def opstm_stm_commit_transaction(self): +## self.check_stm_mode(lambda m: m != "not_in_transaction") +## self.llinterpreter.stm_mode = "not_in_transaction" - def opstm_stm_transaction_boundary(self): - self.check_stm_mode(lambda m: m != "not_in_transaction") - self.llinterpreter.stm_mode = "regular_transaction" - self.llinterpreter.last_transaction_started_in_frame = self +## def opstm_stm_transaction_boundary(self): +## self.check_stm_mode(lambda m: m != "not_in_transaction") +## self.llinterpreter.stm_mode = "regular_transaction" +## self.llinterpreter.last_transaction_started_in_frame = self - def opstm_stm_try_inevitable(self, why): + def opstm_stm_become_inevitable(self, why): self.check_stm_mode(lambda m: m != "not_in_transaction") self.llinterpreter.stm_mode = "inevitable_transaction" print why diff --git a/pypy/translator/stm/test/test_llstminterp.py b/pypy/translator/stm/test/test_llstminterp.py --- a/pypy/translator/stm/test/test_llstminterp.py +++ b/pypy/translator/stm/test/test_llstminterp.py @@ -4,7 +4,7 @@ from pypy.translator.stm.llstminterp import eval_stm_graph from pypy.translator.stm.llstminterp import ForbiddenInstructionInSTMMode from pypy.translator.stm.llstminterp import ReturnWithTransactionActive -from pypy.translator.stm import rstm +from pypy.translator.stm import llstm ALL_STM_MODES = ["not_in_transaction", "regular_transaction", @@ -14,55 +14,59 @@ def func(n): return (n+1) * (n+2) interp, graph = get_interpreter(func, [5]) - res = eval_stm_graph(interp, graph, [5]) + res = eval_stm_graph(interp, graph, [5], + stm_mode="not_in_transaction", + final_stm_mode="not_in_transaction") assert res == 42 def test_forbidden(): S = lltype.GcStruct('S', ('x', lltype.Signed)) p = lltype.malloc(S, immortal=True) p.x = 42 - def func(p): + # + def funcget(p): return p.x - interp, graph = get_interpreter(func, [p]) + interp, graph = get_interpreter(funcget, [p]) + py.test.raises(ForbiddenInstructionInSTMMode, + eval_stm_graph, interp, graph, [p], + stm_mode="regular_transaction") + # + def funcset(p): + p.x = 43 + interp, graph = get_interpreter(funcset, [p]) py.test.raises(ForbiddenInstructionInSTMMode, eval_stm_graph, interp, graph, [p], stm_mode="regular_transaction") -def test_stm_getfield(): - S = lltype.GcStruct('S', ('x', lltype.Signed)) +def test_stm_getfield_stm_setfield(): + S = lltype.GcStruct('S', ('x', lltype.Signed), ('y', lltype.Signed)) p = lltype.malloc(S, immortal=True) p.x = 42 def func(p): - return rstm.stm_getfield(p, 'x') + llstm.stm_setfield(p, 'y', 43) + return llstm.stm_getfield(p, 'x') interp, graph = get_interpreter(func, [p]) - # forbidden in "not_in_transaction" mode - py.test.raises(ForbiddenInstructionInSTMMode, - eval_stm_graph, interp, graph, [p], - stm_mode="not_in_transaction") - # works in "regular_transaction" mode - res = eval_stm_graph(interp, graph, [p], stm_mode="regular_transaction") - assert res == 42 - # works in "inevitable_transaction" mode - res = eval_stm_graph(interp, graph, [p], stm_mode="inevitable_transaction") - assert res == 42 + # works in all modes + for mode in ALL_STM_MODES: + p.y = 0 + res = eval_stm_graph(interp, graph, [p], stm_mode=mode) + assert res == 42 + assert p.y == 43 -def test_stm_getarrayitem(): +def test_stm_getarrayitem_stm_setarrayitem(): A = lltype.GcArray(lltype.Signed) p = lltype.malloc(A, 5, immortal=True) p[3] = 42 def func(p): - return rstm.stm_getarrayitem(p, 3) + llstm.stm_setarrayitem(p, 2, 43) + return llstm.stm_getarrayitem(p, 3) interp, graph = get_interpreter(func, [p]) - # forbidden in "not_in_transaction" mode - py.test.raises(ForbiddenInstructionInSTMMode, - eval_stm_graph, interp, graph, [p], - stm_mode="not_in_transaction") - # works in "regular_transaction" mode - res = eval_stm_graph(interp, graph, [p], stm_mode="regular_transaction") - assert res == 42 - # works in "inevitable_transaction" mode - res = eval_stm_graph(interp, graph, [p], stm_mode="inevitable_transaction") - assert res == 42 + # works in all modes + for mode in ALL_STM_MODES: + p[2] = 0 + res = eval_stm_graph(interp, graph, [p], stm_mode=mode) + assert res == 42 + assert p[2] == 43 def test_getfield_immutable(): S = lltype.GcStruct('S', ('x', lltype.Signed), hints = {'immutable': True}) @@ -76,66 +80,13 @@ res = eval_stm_graph(interp, graph, [p], stm_mode=mode) assert res == 42 -def test_begin_commit_transaction(): - S = lltype.GcStruct('S', ('x', lltype.Signed)) - p = lltype.malloc(S, immortal=True) - p.x = 42 - def func(p): - rstm.begin_transaction() - res = rstm.stm_getfield(p, 'x') - rstm.commit_transaction() - return res - interp, graph = get_interpreter(func, [p]) - res = eval_stm_graph(interp, graph, [p]) - assert res == 42 - -def test_call_and_return_with_regular_transaction(): - def g(): - pass - g._dont_inline_ = True +def test_become_inevitable(): def func(): - rstm.begin_transaction() - g() - rstm.commit_transaction() + llstm.stm_become_inevitable("foobar!") interp, graph = get_interpreter(func, []) - eval_stm_graph(interp, graph, []) - -def test_cannot_return_with_regular_transaction(): - def g(): - rstm.begin_transaction() - g._dont_inline_ = True - def func(): - g() - rstm.commit_transaction() - interp, graph = get_interpreter(func, []) - py.test.raises(ReturnWithTransactionActive, - eval_stm_graph, interp, graph, []) - -def test_cannot_raise_with_regular_transaction(): - def g(): - rstm.begin_transaction() - raise ValueError - g._dont_inline_ = True - def func(): - try: - g() - except ValueError: - pass - rstm.commit_transaction() - interp, graph = get_interpreter(func, []) - py.test.raises(ReturnWithTransactionActive, - eval_stm_graph, interp, graph, []) - -def test_transaction_boundary(): - def func(n): - if n > 5: - rstm.transaction_boundary() - interp, graph = get_interpreter(func, [2]) - eval_stm_graph(interp, graph, [10], - stm_mode="regular_transaction", - final_stm_mode="inevitable_transaction", - automatic_promotion=True) - eval_stm_graph(interp, graph, [1], - stm_mode="regular_transaction", - final_stm_mode="regular_transaction", - automatic_promotion=True) + py.test.raises(ForbiddenInstructionInSTMMode, + eval_stm_graph, interp, graph, [], + stm_mode="not_in_transaction") + eval_stm_graph(interp, graph, [], stm_mode="regular_transaction", + final_stm_mode="inevitable_transaction") + eval_stm_graph(interp, graph, [], stm_mode="inevitable_transaction") From noreply at buildbot.pypy.org Mon Jan 16 16:30:58 2012 From: noreply at buildbot.pypy.org (antocuni) Date: Mon, 16 Jan 2012 16:30:58 +0100 (CET) Subject: [pypy-commit] pypy stm: (arigo, antocuni): fix test_getfield_all_sizes, and make sure we test both the cases of stm_getfield inside and outside a transaction Message-ID: <20120116153058.1C89D820D8@wyvern.cs.uni-duesseldorf.de> Author: Antonio Cuni Branch: stm Changeset: r51345:a81e51b21e06 Date: 2012-01-16 16:06 +0100 http://bitbucket.org/pypy/pypy/changeset/a81e51b21e06/ Log: (arigo, antocuni): fix test_getfield_all_sizes, and make sure we test both the cases of stm_getfield inside and outside a transaction diff --git a/pypy/translator/stm/src_stm/et.c b/pypy/translator/stm/src_stm/et.c --- a/pypy/translator/stm/src_stm/et.c +++ b/pypy/translator/stm/src_stm/et.c @@ -676,6 +676,10 @@ void* stm_perform_transaction(void*(*callback)(void*), void *arg) { void *result; +#ifdef RPY_STM_ASSERT + /* you need to call descriptor_init() before calling stm_perform_transaction */ + assert(thread_descriptor != NULL); +#endif STM_begin_transaction(); result = callback(arg); stm_commit_transaction(); diff --git a/pypy/translator/stm/test/targetdemo.py b/pypy/translator/stm/test/targetdemo.py --- a/pypy/translator/stm/test/targetdemo.py +++ b/pypy/translator/stm/test/targetdemo.py @@ -1,6 +1,6 @@ import time from pypy.module.thread import ll_thread -from pypy.translator.stm import rstm +#from pypy.translator.stm import rstm class Node: diff --git a/pypy/translator/stm/test/test_funcgen.py b/pypy/translator/stm/test/test_funcgen.py --- a/pypy/translator/stm/test/test_funcgen.py +++ b/pypy/translator/stm/test/test_funcgen.py @@ -2,8 +2,11 @@ from pypy.rlib.rarithmetic import r_longlong, r_singlefloat from pypy.translator.stm.test.test_transform import CompiledSTMTests #from pypy.rlib import rstm +from pypy.translator.stm._rffi_stm import (CALLBACK, perform_transaction, + descriptor_init, descriptor_done) from pypy.translator.c.test.test_standalone import StandaloneTests from pypy.rlib.debug import debug_print +from pypy.rpython.annlowlevel import llhelper class TestRStm(object): @@ -78,7 +81,7 @@ return a a_prebuilt = make_a_1() -def do_stm_getfield(argv): +def _play_with_getfield(dummy_arg): a = a_prebuilt assert a.x == -611 assert a.c1 == '/' @@ -89,7 +92,7 @@ assert a.f == rf1 assert float(a.sa) == float(rs1a) assert float(a.sb) == float(rs1b) - return 0 + return lltype.nullptr(rffi.VOIDP.TO) def do_stm_setfield(argv): a = a_prebuilt @@ -241,6 +244,19 @@ class TestFuncGen(CompiledSTMTests): def test_getfield_all_sizes(self): + def do_stm_getfield(argv): + _play_with_getfield(None) + return 0 + t, cbuilder = self.compile(do_stm_getfield) + cbuilder.cmdexec('') + + def test_getfield_all_sizes_inside_transaction(self): + def do_stm_getfield(argv): + callback = llhelper(CALLBACK, _play_with_getfield) + descriptor_init() + perform_transaction(callback, lltype.nullptr(rffi.VOIDP.TO)) + descriptor_done() + return 0 t, cbuilder = self.compile(do_stm_getfield) cbuilder.cmdexec('') From noreply at buildbot.pypy.org Mon Jan 16 16:30:59 2012 From: noreply at buildbot.pypy.org (antocuni) Date: Mon, 16 Jan 2012 16:30:59 +0100 (CET) Subject: [pypy-commit] pypy stm: (arigo, antocuni): fix test_setfield_all_sizes, and make sure we test both the cases of stm_setfield inside and outside a transaction Message-ID: <20120116153059.41A47820D8@wyvern.cs.uni-duesseldorf.de> Author: Antonio Cuni Branch: stm Changeset: r51346:6b3914441db1 Date: 2012-01-16 16:18 +0100 http://bitbucket.org/pypy/pypy/changeset/6b3914441db1/ Log: (arigo, antocuni): fix test_setfield_all_sizes, and make sure we test both the cases of stm_setfield inside and outside a transaction diff --git a/pypy/translator/stm/test/test_funcgen.py b/pypy/translator/stm/test/test_funcgen.py --- a/pypy/translator/stm/test/test_funcgen.py +++ b/pypy/translator/stm/test/test_funcgen.py @@ -67,6 +67,8 @@ rs1b = r_singlefloat(40.121) rs2b = r_singlefloat(-9e9) +NULL = lltype.nullptr(rffi.VOIDP.TO) + def make_a_1(): a = lltype.malloc(A, immortal=True) a.x = -611 @@ -92,9 +94,9 @@ assert a.f == rf1 assert float(a.sa) == float(rs1a) assert float(a.sb) == float(rs1b) - return lltype.nullptr(rffi.VOIDP.TO) - -def do_stm_setfield(argv): + return NULL + +def _play_with_setfields(dummy_arg): a = a_prebuilt # a.x = 12871981 @@ -111,7 +113,13 @@ a.f = rf2 a.sa = rs2a a.sb = rs2b - # + # read the values which have not been commited yet, but are local to the + # transaction + _check_values_of_fields(dummy_arg) + return NULL + +def _check_values_of_fields(dummy_arg): + a = a_prebuilt assert a.x == 12871981 assert a.c1 == '(' assert a.c2 == '?' @@ -120,19 +128,7 @@ assert a.f == rf2 assert float(a.sa) == float(rs2a) assert float(a.sb) == float(rs2b) - # - rstm.transaction_boundary() - # - assert a.x == 12871981 - assert a.c1 == '(' - assert a.c2 == '?' - assert a.c3 == ')' - assert a.l == rll2 - assert a.f == rf2 - assert float(a.sa) == float(rs2a) - assert float(a.sb) == float(rs2b) - # - return 0 + return NULL def make_array(OF): @@ -254,13 +250,29 @@ def do_stm_getfield(argv): callback = llhelper(CALLBACK, _play_with_getfield) descriptor_init() - perform_transaction(callback, lltype.nullptr(rffi.VOIDP.TO)) + perform_transaction(callback, NULL) descriptor_done() return 0 t, cbuilder = self.compile(do_stm_getfield) cbuilder.cmdexec('') def test_setfield_all_sizes(self): + def do_stm_setfield(argv): + _play_with_setfields(None) + return 0 + t, cbuilder = self.compile(do_stm_setfield) + cbuilder.cmdexec('') + + def test_setfield_all_sizes_inside_transaction(self): + def do_stm_setfield(argv): + callback1 = llhelper(CALLBACK, _play_with_setfields) + callback2 = llhelper(CALLBACK, _check_values_of_fields) + descriptor_init() + perform_transaction(callback1, NULL) + # read values which aren't local to the transaction + perform_transaction(callback2, NULL) + descriptor_done() + return 0 t, cbuilder = self.compile(do_stm_setfield) cbuilder.cmdexec('') From noreply at buildbot.pypy.org Mon Jan 16 16:31:00 2012 From: noreply at buildbot.pypy.org (antocuni) Date: Mon, 16 Jan 2012 16:31:00 +0100 (CET) Subject: [pypy-commit] pypy stm: (arigo, antocuni): fix test_getarrayitem_all_sizes, and make sure we test both the cases of stm_getarrayitem inside and outside a transaction Message-ID: <20120116153100.65F91820D8@wyvern.cs.uni-duesseldorf.de> Author: Antonio Cuni Branch: stm Changeset: r51347:08f6815aba37 Date: 2012-01-16 16:20 +0100 http://bitbucket.org/pypy/pypy/changeset/08f6815aba37/ Log: (arigo, antocuni): fix test_getarrayitem_all_sizes, and make sure we test both the cases of stm_getarrayitem inside and outside a transaction diff --git a/pypy/translator/stm/test/test_funcgen.py b/pypy/translator/stm/test/test_funcgen.py --- a/pypy/translator/stm/test/test_funcgen.py +++ b/pypy/translator/stm/test/test_funcgen.py @@ -152,11 +152,12 @@ array[i] = rffi.cast(lltype.typeOf(array).TO.OF, newvalues[i]) change._annspecialcase_ = 'specialize:ll' -def do_stm_getarrayitem(argv): +def _play_with_getarrayitem(dummy_arg): check(prebuilt_array_signed, [1, 10, -1, -10, 42]) check(prebuilt_array_char, [chr(1), chr(10), chr(255), chr(246), chr(42)]) - return 0 + return NULL + def do_stm_setarrayitem(argv): change(prebuilt_array_signed, [500000, -10000000, 3]) @@ -277,9 +278,23 @@ cbuilder.cmdexec('') def test_getarrayitem_all_sizes(self): + def do_stm_getarrayitem(argv): + _play_with_getarrayitem(None) + return 0 t, cbuilder = self.compile(do_stm_getarrayitem) cbuilder.cmdexec('') + def test_getarrayitem_all_sizes_inside_transaction(self): + def do_stm_getarrayitem(argv): + callback = llhelper(CALLBACK, _play_with_getarrayitem) + descriptor_init() + perform_transaction(callback, NULL) + descriptor_done() + return 0 + t, cbuilder = self.compile(do_stm_getarrayitem) + cbuilder.cmdexec('') + + def test_setarrayitem_all_sizes(self): t, cbuilder = self.compile(do_stm_setarrayitem) cbuilder.cmdexec('') From noreply at buildbot.pypy.org Mon Jan 16 16:31:01 2012 From: noreply at buildbot.pypy.org (antocuni) Date: Mon, 16 Jan 2012 16:31:01 +0100 (CET) Subject: [pypy-commit] pypy stm: (arigo, antocuni): fix test_setarrayitem_all_sizes, and make sure we test both the cases of stm_setarrayitem inside and outside a transaction Message-ID: <20120116153101.88CF1820D8@wyvern.cs.uni-duesseldorf.de> Author: Antonio Cuni Branch: stm Changeset: r51348:da7457ea38cd Date: 2012-01-16 16:25 +0100 http://bitbucket.org/pypy/pypy/changeset/da7457ea38cd/ Log: (arigo, antocuni): fix test_setarrayitem_all_sizes, and make sure we test both the cases of stm_setarrayitem inside and outside a transaction diff --git a/pypy/translator/stm/test/test_funcgen.py b/pypy/translator/stm/test/test_funcgen.py --- a/pypy/translator/stm/test/test_funcgen.py +++ b/pypy/translator/stm/test/test_funcgen.py @@ -159,7 +159,7 @@ return NULL -def do_stm_setarrayitem(argv): +def _play_with_setarrayitem_1(dummy_arg): change(prebuilt_array_signed, [500000, -10000000, 3]) check(prebuilt_array_signed, [500000, -10000000, 3, -10, 42]) prebuilt_array_char[0] = 'A' @@ -168,19 +168,19 @@ check(prebuilt_array_char, ['A', chr(10), chr(255), 'B', chr(42)]) prebuilt_array_char[4] = 'C' check(prebuilt_array_char, ['A', chr(10), chr(255), 'B', 'C']) - # - rstm.transaction_boundary() - # + return NULL + +def _play_with_setarrayitem_2(dummy_arg): check(prebuilt_array_char, ['A', chr(10), chr(255), 'B', 'C']) prebuilt_array_char[1] = 'D' check(prebuilt_array_char, ['A', 'D', chr(255), 'B', 'C']) prebuilt_array_char[2] = 'E' check(prebuilt_array_char, ['A', 'D', 'E', 'B', 'C']) - # - rstm.transaction_boundary() - # + return NULL + +def _play_with_setarrayitem_3(dummy_arg): check(prebuilt_array_char, ['A', 'D', 'E', 'B', 'C']) - return 0 + return NULL def make_array_of_structs(T1, T2): @@ -296,6 +296,26 @@ def test_setarrayitem_all_sizes(self): + def do_stm_setarrayitem(argv): + _play_with_setarrayitem_1(None) + _play_with_setarrayitem_2(None) + _play_with_setarrayitem_3(None) + return 0 + t, cbuilder = self.compile(do_stm_setarrayitem) + cbuilder.cmdexec('') + + def test_setarrayitem_all_sizes_inside_transaction(self): + def do_stm_setarrayitem(argv): + callback1 = llhelper(CALLBACK, _play_with_setarrayitem_1) + callback2 = llhelper(CALLBACK, _play_with_setarrayitem_2) + callback3 = llhelper(CALLBACK, _play_with_setarrayitem_3) + # + descriptor_init() + perform_transaction(callback1, NULL) + perform_transaction(callback2, NULL) + perform_transaction(callback3, NULL) + descriptor_done() + return 0 t, cbuilder = self.compile(do_stm_setarrayitem) cbuilder.cmdexec('') From noreply at buildbot.pypy.org Mon Jan 16 16:31:02 2012 From: noreply at buildbot.pypy.org (antocuni) Date: Mon, 16 Jan 2012 16:31:02 +0100 (CET) Subject: [pypy-commit] pypy stm: (arigo, antocuni): fix test_getinteriorfield_all_sizes, and make sure we test both the cases of stm_getinteriorfield inside and outside a transaction Message-ID: <20120116153102.CE994820D8@wyvern.cs.uni-duesseldorf.de> Author: Antonio Cuni Branch: stm Changeset: r51349:e7847828c029 Date: 2012-01-16 16:27 +0100 http://bitbucket.org/pypy/pypy/changeset/e7847828c029/ Log: (arigo, antocuni): fix test_getinteriorfield_all_sizes, and make sure we test both the cases of stm_getinteriorfield inside and outside a transaction diff --git a/pypy/translator/stm/test/test_funcgen.py b/pypy/translator/stm/test/test_funcgen.py --- a/pypy/translator/stm/test/test_funcgen.py +++ b/pypy/translator/stm/test/test_funcgen.py @@ -212,11 +212,12 @@ array[i].y = rffi.cast(lltype.typeOf(array).TO.OF.y, newvalues2[i]) change2._annspecialcase_ = 'specialize:ll' -def do_stm_getinteriorfield(argv): +def _play_with_getinteriorfield(dummy_arg): check2(prebuilt_array_signed_signed, [1, -1, -50], [10, 20, -30]) check2(prebuilt_array_char_char, [chr(1), chr(255), chr(206)], [chr(10), chr(20), chr(226)]) - return 0 + return NULL + def do_stm_setinteriorfield(argv): change2(prebuilt_array_signed_signed, [500000, -10000000], [102101202]) @@ -320,9 +321,23 @@ cbuilder.cmdexec('') def test_getinteriorfield_all_sizes(self): + def do_stm_getinteriorfield(argv): + _play_with_getinteriorfield(None) + return 0 t, cbuilder = self.compile(do_stm_getinteriorfield) cbuilder.cmdexec('') + def test_getinteriorfield_all_sizes_inside_transaction(self): + def do_stm_getinteriorfield(argv): + callback = llhelper(CALLBACK, _play_with_getinteriorfield) + descriptor_init() + perform_transaction(callback, NULL) + descriptor_done() + return 0 + t, cbuilder = self.compile(do_stm_getinteriorfield) + cbuilder.cmdexec('') + + def test_setinteriorfield_all_sizes(self): t, cbuilder = self.compile(do_stm_setinteriorfield) cbuilder.cmdexec('') From noreply at buildbot.pypy.org Mon Jan 16 16:31:04 2012 From: noreply at buildbot.pypy.org (antocuni) Date: Mon, 16 Jan 2012 16:31:04 +0100 (CET) Subject: [pypy-commit] pypy stm: (arigo, antocuni): fix test_setinteriorfield_all_sizes, and make sure we test both the cases of stm_setinteriorfield inside and outside a transaction Message-ID: <20120116153104.00B64820D8@wyvern.cs.uni-duesseldorf.de> Author: Antonio Cuni Branch: stm Changeset: r51350:a2b3f2c9ea45 Date: 2012-01-16 16:29 +0100 http://bitbucket.org/pypy/pypy/changeset/a2b3f2c9ea45/ Log: (arigo, antocuni): fix test_setinteriorfield_all_sizes, and make sure we test both the cases of stm_setinteriorfield inside and outside a transaction diff --git a/pypy/translator/stm/test/test_funcgen.py b/pypy/translator/stm/test/test_funcgen.py --- a/pypy/translator/stm/test/test_funcgen.py +++ b/pypy/translator/stm/test/test_funcgen.py @@ -219,21 +219,21 @@ return NULL -def do_stm_setinteriorfield(argv): +def _play_with_setinteriorfield_1(dummy_arg): change2(prebuilt_array_signed_signed, [500000, -10000000], [102101202]) check2(prebuilt_array_signed_signed, [500000, -10000000, -50], [102101202, 20, -30]) change2(prebuilt_array_char_char, ['a'], ['b']) check2(prebuilt_array_char_char, ['a', chr(255), chr(206)], ['b', chr(20), chr(226)]) - # - rstm.transaction_boundary() - # + return NULL + +def _play_with_setinteriorfield_2(dummy_arg): check2(prebuilt_array_signed_signed, [500000, -10000000, -50], [102101202, 20, -30]) check2(prebuilt_array_char_char, ['a', chr(255), chr(206)], ['b', chr(20), chr(226)]) - return 0 + return NULL # ____________________________________________________________ @@ -339,5 +339,22 @@ def test_setinteriorfield_all_sizes(self): + def do_stm_setinteriorfield(argv): + _play_with_setinteriorfield_1(None) + _play_with_setinteriorfield_2(None) + return 0 t, cbuilder = self.compile(do_stm_setinteriorfield) cbuilder.cmdexec('') + + def test_setinteriorfield_all_sizes_inside_transaction(self): + def do_stm_setinteriorfield(argv): + callback1 = llhelper(CALLBACK, _play_with_setinteriorfield_1) + callback2 = llhelper(CALLBACK, _play_with_setinteriorfield_2) + # + descriptor_init() + perform_transaction(callback1, NULL) + perform_transaction(callback2, NULL) + descriptor_done() + return 0 + t, cbuilder = self.compile(do_stm_setinteriorfield) + cbuilder.cmdexec('') From noreply at buildbot.pypy.org Mon Jan 16 17:51:11 2012 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 16 Jan 2012 17:51:11 +0100 (CET) Subject: [pypy-commit] pypy default: (antocuni, arigo) Message-ID: <20120116165111.349D2820D8@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r51351:e244681d320e Date: 2012-01-16 17:49 +0100 http://bitbucket.org/pypy/pypy/changeset/e244681d320e/ Log: (antocuni, arigo) Add support for malloc'ing RPython instances non-movable, just by specifying "_alloc_nonmovable_ = True" on the class. diff --git a/pypy/rpython/lltypesystem/rclass.py b/pypy/rpython/lltypesystem/rclass.py --- a/pypy/rpython/lltypesystem/rclass.py +++ b/pypy/rpython/lltypesystem/rclass.py @@ -510,7 +510,13 @@ ctype = inputconst(Void, self.object_type) cflags = inputconst(Void, flags) vlist = [ctype, cflags] - vptr = llops.genop('malloc', vlist, + cnonmovable = self.classdef.classdesc.read_attribute( + '_alloc_nonmovable_', Constant(False)) + if cnonmovable.value: + opname = 'malloc_nonmovable' + else: + opname = 'malloc' + vptr = llops.genop(opname, vlist, resulttype = Ptr(self.object_type)) ctypeptr = inputconst(CLASSTYPE, self.rclass.getvtable()) self.setfield(vptr, '__class__', ctypeptr, llops) diff --git a/pypy/rpython/test/test_rclass.py b/pypy/rpython/test/test_rclass.py --- a/pypy/rpython/test/test_rclass.py +++ b/pypy/rpython/test/test_rclass.py @@ -1130,6 +1130,18 @@ assert sorted([u]) == [6] # 32-bit types assert sorted([i, r, d, l]) == [2, 3, 4, 5] # 64-bit types + def test_nonmovable(self): + for (nonmovable, opname) in [(True, 'malloc_nonmovable'), + (False, 'malloc')]: + class A(object): + _alloc_nonmovable_ = nonmovable + def f(): + return A() + t, typer, graph = self.gengraph(f, []) + assert summary(graph) == {opname: 1, + 'cast_pointer': 1, + 'setfield': 1} + class TestOOtype(BaseTestRclass, OORtypeMixin): From noreply at buildbot.pypy.org Mon Jan 16 17:51:23 2012 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 16 Jan 2012 17:51:23 +0100 (CET) Subject: [pypy-commit] pypy stm: hg merge default Message-ID: <20120116165123.872CB820D8@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stm Changeset: r51352:30f56b4fb5ea Date: 2012-01-16 17:50 +0100 http://bitbucket.org/pypy/pypy/changeset/30f56b4fb5ea/ Log: hg merge default diff too long, truncating to 10000 out of 44586 lines diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -27,7 +27,7 @@ DEALINGS IN THE SOFTWARE. -PyPy Copyright holders 2003-2011 +PyPy Copyright holders 2003-2012 ----------------------------------- Except when otherwise stated (look for LICENSE files or information at @@ -37,43 +37,47 @@ Armin Rigo Maciej Fijalkowski Carl Friedrich Bolz + Amaury Forgeot d'Arc Antonio Cuni - Amaury Forgeot d'Arc Samuele Pedroni Michael Hudson Holger Krekel - Benjamin Peterson + Alex Gaynor Christian Tismer Hakan Ardo - Alex Gaynor + Benjamin Peterson + David Schneider Eric van Riet Paap Anders Chrigstrom - David Schneider Richard Emslie Dan Villiom Podlaski Christiansen Alexander Schremmer + Lukas Diekmann Aurelien Campeas Anders Lehmann Camillo Bruni Niklaus Haldimann + Sven Hager Leonardo Santagada Toon Verwaest Seo Sanghyeon + Justin Peel Lawrence Oluyede Bartosz Skowron Jakub Gustak Guido Wesdorp Daniel Roberts + Laura Creighton Adrien Di Mascio - Laura Creighton Ludovic Aubry Niko Matsakis + Wim Lavrijsen + Matti Picus Jason Creighton Jacob Hallen Alex Martelli Anders Hammarquist Jan de Mooij - Wim Lavrijsen Stephan Diehl Michael Foord Stefan Schwarzer @@ -84,34 +88,36 @@ Alexandre Fayolle Marius Gedminas Simon Burton - Justin Peel + David Edelsohn Jean-Paul Calderone John Witulski - Lukas Diekmann + Timo Paulssen holger krekel - Wim Lavrijsen Dario Bertini + Mark Pearse Andreas Stührk Jean-Philippe St. Pierre Guido van Rossum Pavel Vinogradov Valentino Volonghi Paul deGrandis + Ilya Osadchiy + Ronny Pfannschmidt Adrian Kuhn tav Georg Brandl + Philip Jenvey Gerald Klix Wanja Saatkamp - Ronny Pfannschmidt Boris Feigin Oscar Nierstrasz David Malcolm Eugene Oden Henry Mason - Sven Hager + Jeff Terrace Lukas Renggli - Ilya Osadchiy Guenter Jantzen + Ned Batchelder Bert Freudenberg Amit Regmi Ben Young @@ -142,7 +148,6 @@ Anders Qvist Beatrice During Alexander Sedov - Timo Paulssen Corbin Simpson Vincent Legoll Romain Guillebert @@ -165,9 +170,10 @@ Lucio Torre Lene Wagner Miguel de Val Borro + Artur Lisiecki + Bruno Gola Ignas Mikalajunas - Artur Lisiecki - Philip Jenvey + Stefano Rivera Joshua Gilbert Godefroid Chappelle Yusei Tahara @@ -179,17 +185,17 @@ Kristjan Valur Jonsson Bobby Impollonia Michael Hudson-Doyle + Laurence Tratt + Yasir Suhail Andrew Thompson Anders Sigfridsson Floris Bruynooghe Jacek Generowicz Dan Colish Zooko Wilcox-O Hearn - Dan Villiom Podlaski Christiansen - Anders Hammarquist + Dan Loewenherz Chris Lambacher Dinu Gherman - Dan Colish Brett Cannon Daniel Neuhäuser Michael Chermside diff --git a/lib-python/modified-2.7/ctypes/__init__.py b/lib-python/modified-2.7/ctypes/__init__.py --- a/lib-python/modified-2.7/ctypes/__init__.py +++ b/lib-python/modified-2.7/ctypes/__init__.py @@ -351,7 +351,7 @@ self._FuncPtr = _FuncPtr if handle is None: - self._handle = _ffi.CDLL(name) + self._handle = _ffi.CDLL(name, mode) else: self._handle = handle diff --git a/lib-python/modified-2.7/ctypes/test/test_callbacks.py b/lib-python/modified-2.7/ctypes/test/test_callbacks.py --- a/lib-python/modified-2.7/ctypes/test/test_callbacks.py +++ b/lib-python/modified-2.7/ctypes/test/test_callbacks.py @@ -1,5 +1,6 @@ import unittest from ctypes import * +from ctypes.test import xfail import _ctypes_test class Callbacks(unittest.TestCase): @@ -98,6 +99,7 @@ ## self.check_type(c_char_p, "abc") ## self.check_type(c_char_p, "def") + @xfail def test_pyobject(self): o = () from sys import getrefcount as grc diff --git a/lib-python/modified-2.7/ctypes/test/test_libc.py b/lib-python/modified-2.7/ctypes/test/test_libc.py --- a/lib-python/modified-2.7/ctypes/test/test_libc.py +++ b/lib-python/modified-2.7/ctypes/test/test_libc.py @@ -25,7 +25,10 @@ lib.my_qsort(chars, len(chars)-1, sizeof(c_char), comparefunc(sort)) self.assertEqual(chars.raw, " ,,aaaadmmmnpppsss\x00") - def test_no_more_xfail(self): + def SKIPPED_test_no_more_xfail(self): + # We decided to not explicitly support the whole ctypes-2.7 + # and instead go for a case-by-case, demand-driven approach. + # So this test is skipped instead of failing. import socket import ctypes.test self.assertTrue(not hasattr(ctypes.test, 'xfail'), diff --git a/lib_pypy/_collections.py b/lib_pypy/_collections.py --- a/lib_pypy/_collections.py +++ b/lib_pypy/_collections.py @@ -379,12 +379,14 @@ class defaultdict(dict): def __init__(self, *args, **kwds): - self.default_factory = None - if 'default_factory' in kwds: - self.default_factory = kwds.pop('default_factory') - elif len(args) > 0 and (callable(args[0]) or args[0] is None): - self.default_factory = args[0] + if len(args) > 0: + default_factory = args[0] args = args[1:] + if not callable(default_factory) and default_factory is not None: + raise TypeError("first argument must be callable") + else: + default_factory = None + self.default_factory = default_factory super(defaultdict, self).__init__(*args, **kwds) def __missing__(self, key): @@ -404,7 +406,7 @@ recurse.remove(id(self)) def copy(self): - return type(self)(self, default_factory=self.default_factory) + return type(self)(self.default_factory, self) def __copy__(self): return self.copy() diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py --- a/lib_pypy/_ctypes/structure.py +++ b/lib_pypy/_ctypes/structure.py @@ -73,8 +73,12 @@ class Field(object): def __init__(self, name, offset, size, ctype, num, is_bitfield): - for k in ('name', 'offset', 'size', 'ctype', 'num', 'is_bitfield'): - self.__dict__[k] = locals()[k] + self.__dict__['name'] = name + self.__dict__['offset'] = offset + self.__dict__['size'] = size + self.__dict__['ctype'] = ctype + self.__dict__['num'] = num + self.__dict__['is_bitfield'] = is_bitfield def __setattr__(self, name, value): raise AttributeError(name) diff --git a/lib_pypy/_sha.py b/lib_pypy/_sha.py --- a/lib_pypy/_sha.py +++ b/lib_pypy/_sha.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# -*- coding: iso-8859-1 +# -*- coding: iso-8859-1 -*- # Note that PyPy contains also a built-in module 'sha' which will hide # this one if compiled in. diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py --- a/lib_pypy/_sqlite3.py +++ b/lib_pypy/_sqlite3.py @@ -231,8 +231,10 @@ sqlite.sqlite3_result_text.argtypes = [c_void_p, c_char_p, c_int, c_void_p] sqlite.sqlite3_result_text.restype = None -sqlite.sqlite3_enable_load_extension.argtypes = [c_void_p, c_int] -sqlite.sqlite3_enable_load_extension.restype = c_int +HAS_LOAD_EXTENSION = hasattr(sqlite, "sqlite3_enable_load_extension") +if HAS_LOAD_EXTENSION: + sqlite.sqlite3_enable_load_extension.argtypes = [c_void_p, c_int] + sqlite.sqlite3_enable_load_extension.restype = c_int ########################################## # END Wrapped SQLite C API and constants @@ -708,13 +710,14 @@ from sqlite3.dump import _iterdump return _iterdump(self) - def enable_load_extension(self, enabled): - self._check_thread() - self._check_closed() + if HAS_LOAD_EXTENSION: + def enable_load_extension(self, enabled): + self._check_thread() + self._check_closed() - rc = sqlite.sqlite3_enable_load_extension(self.db, int(enabled)) - if rc != SQLITE_OK: - raise OperationalError("Error enabling load extension") + rc = sqlite.sqlite3_enable_load_extension(self.db, int(enabled)) + if rc != SQLITE_OK: + raise OperationalError("Error enabling load extension") DML, DQL, DDL = range(3) diff --git a/lib_pypy/distributed/socklayer.py b/lib_pypy/distributed/socklayer.py --- a/lib_pypy/distributed/socklayer.py +++ b/lib_pypy/distributed/socklayer.py @@ -2,7 +2,7 @@ import py from socket import socket -XXX needs import adaptation as 'green' is removed from py lib for years +raise ImportError("XXX needs import adaptation as 'green' is removed from py lib for years") from py.impl.green.msgstruct import decodemessage, message from socket import socket, AF_INET, SOCK_STREAM import marshal diff --git a/lib_pypy/itertools.py b/lib_pypy/itertools.py --- a/lib_pypy/itertools.py +++ b/lib_pypy/itertools.py @@ -25,7 +25,7 @@ __all__ = ['chain', 'count', 'cycle', 'dropwhile', 'groupby', 'ifilter', 'ifilterfalse', 'imap', 'islice', 'izip', 'repeat', 'starmap', - 'takewhile', 'tee'] + 'takewhile', 'tee', 'compress', 'product'] try: from __pypy__ import builtinify except ImportError: builtinify = lambda f: f diff --git a/lib_pypy/numpypy/__init__.py b/lib_pypy/numpypy/__init__.py new file mode 100644 --- /dev/null +++ b/lib_pypy/numpypy/__init__.py @@ -0,0 +1,2 @@ +from _numpypy import * +from .fromnumeric import * diff --git a/lib_pypy/numpypy/fromnumeric.py b/lib_pypy/numpypy/fromnumeric.py new file mode 100644 --- /dev/null +++ b/lib_pypy/numpypy/fromnumeric.py @@ -0,0 +1,2400 @@ +###################################################################### +# This is a copy of numpy/core/fromnumeric.py modified for numpypy +###################################################################### +# Each name in __all__ was a function in 'numeric' that is now +# a method in 'numpy'. +# When the corresponding method is added to numpypy BaseArray +# each function should be added as a module function +# at the applevel +# This can be as simple as doing the following +# +# def func(a, ...): +# if not hasattr(a, 'func') +# a = numpypy.array(a) +# return a.func(...) +# +###################################################################### + +import numpypy + +# Module containing non-deprecated functions borrowed from Numeric. +__docformat__ = "restructuredtext en" + +# functions that are now methods +__all__ = ['take', 'reshape', 'choose', 'repeat', 'put', + 'swapaxes', 'transpose', 'sort', 'argsort', 'argmax', 'argmin', + 'searchsorted', 'alen', + 'resize', 'diagonal', 'trace', 'ravel', 'nonzero', 'shape', + 'compress', 'clip', 'sum', 'product', 'prod', 'sometrue', 'alltrue', + 'any', 'all', 'cumsum', 'cumproduct', 'cumprod', 'ptp', 'ndim', + 'rank', 'size', 'around', 'round_', 'mean', 'std', 'var', 'squeeze', + 'amax', 'amin', + ] + +def take(a, indices, axis=None, out=None, mode='raise'): + """ + Take elements from an array along an axis. + + This function does the same thing as "fancy" indexing (indexing arrays + using arrays); however, it can be easier to use if you need elements + along a given axis. + + Parameters + ---------- + a : array_like + The source array. + indices : array_like + The indices of the values to extract. + axis : int, optional + The axis over which to select values. By default, the flattened + input array is used. + out : ndarray, optional + If provided, the result will be placed in this array. It should + be of the appropriate shape and dtype. + mode : {'raise', 'wrap', 'clip'}, optional + Specifies how out-of-bounds indices will behave. + + * 'raise' -- raise an error (default) + * 'wrap' -- wrap around + * 'clip' -- clip to the range + + 'clip' mode means that all indices that are too large are replaced + by the index that addresses the last element along that axis. Note + that this disables indexing with negative numbers. + + Returns + ------- + subarray : ndarray + The returned array has the same type as `a`. + + See Also + -------- + ndarray.take : equivalent method + + Examples + -------- + >>> a = [4, 3, 5, 7, 6, 8] + >>> indices = [0, 1, 4] + >>> np.take(a, indices) + array([4, 3, 6]) + + In this example if `a` is an ndarray, "fancy" indexing can be used. + + >>> a = np.array(a) + >>> a[indices] + array([4, 3, 6]) + + """ + raise NotImplemented('Waiting on interp level method') + + +# not deprecated --- copy if necessary, view otherwise +def reshape(a, newshape, order='C'): + """ + Gives a new shape to an array without changing its data. + + Parameters + ---------- + a : array_like + Array to be reshaped. + newshape : int or tuple of ints + The new shape should be compatible with the original shape. If + an integer, then the result will be a 1-D array of that length. + One shape dimension can be -1. In this case, the value is inferred + from the length of the array and remaining dimensions. + order : {'C', 'F', 'A'}, optional + Determines whether the array data should be viewed as in C + (row-major) order, FORTRAN (column-major) order, or the C/FORTRAN + order should be preserved. + + Returns + ------- + reshaped_array : ndarray + This will be a new view object if possible; otherwise, it will + be a copy. + + + See Also + -------- + ndarray.reshape : Equivalent method. + + Notes + ----- + + It is not always possible to change the shape of an array without + copying the data. If you want an error to be raise if the data is copied, + you should assign the new shape to the shape attribute of the array:: + + >>> a = np.zeros((10, 2)) + # A transpose make the array non-contiguous + >>> b = a.T + # Taking a view makes it possible to modify the shape without modiying the + # initial object. + >>> c = b.view() + >>> c.shape = (20) + AttributeError: incompatible shape for a non-contiguous array + + + Examples + -------- + >>> a = np.array([[1,2,3], [4,5,6]]) + >>> np.reshape(a, 6) + array([1, 2, 3, 4, 5, 6]) + >>> np.reshape(a, 6, order='F') + array([1, 4, 2, 5, 3, 6]) + + >>> np.reshape(a, (3,-1)) # the unspecified value is inferred to be 2 + array([[1, 2], + [3, 4], + [5, 6]]) + + """ + if not hasattr(a, 'reshape'): + a = numpypy.array(a) + return a.reshape(newshape) + + +def choose(a, choices, out=None, mode='raise'): + """ + Construct an array from an index array and a set of arrays to choose from. + + First of all, if confused or uncertain, definitely look at the Examples - + in its full generality, this function is less simple than it might + seem from the following code description (below ndi = + `numpy.lib.index_tricks`): + + ``np.choose(a,c) == np.array([c[a[I]][I] for I in ndi.ndindex(a.shape)])``. + + But this omits some subtleties. Here is a fully general summary: + + Given an "index" array (`a`) of integers and a sequence of `n` arrays + (`choices`), `a` and each choice array are first broadcast, as necessary, + to arrays of a common shape; calling these *Ba* and *Bchoices[i], i = + 0,...,n-1* we have that, necessarily, ``Ba.shape == Bchoices[i].shape`` + for each `i`. Then, a new array with shape ``Ba.shape`` is created as + follows: + + * if ``mode=raise`` (the default), then, first of all, each element of + `a` (and thus `Ba`) must be in the range `[0, n-1]`; now, suppose that + `i` (in that range) is the value at the `(j0, j1, ..., jm)` position + in `Ba` - then the value at the same position in the new array is the + value in `Bchoices[i]` at that same position; + + * if ``mode=wrap``, values in `a` (and thus `Ba`) may be any (signed) + integer; modular arithmetic is used to map integers outside the range + `[0, n-1]` back into that range; and then the new array is constructed + as above; + + * if ``mode=clip``, values in `a` (and thus `Ba`) may be any (signed) + integer; negative integers are mapped to 0; values greater than `n-1` + are mapped to `n-1`; and then the new array is constructed as above. + + Parameters + ---------- + a : int array + This array must contain integers in `[0, n-1]`, where `n` is the number + of choices, unless ``mode=wrap`` or ``mode=clip``, in which cases any + integers are permissible. + choices : sequence of arrays + Choice arrays. `a` and all of the choices must be broadcastable to the + same shape. If `choices` is itself an array (not recommended), then + its outermost dimension (i.e., the one corresponding to + ``choices.shape[0]``) is taken as defining the "sequence". + out : array, optional + If provided, the result will be inserted into this array. It should + be of the appropriate shape and dtype. + mode : {'raise' (default), 'wrap', 'clip'}, optional + Specifies how indices outside `[0, n-1]` will be treated: + + * 'raise' : an exception is raised + * 'wrap' : value becomes value mod `n` + * 'clip' : values < 0 are mapped to 0, values > n-1 are mapped to n-1 + + Returns + ------- + merged_array : array + The merged result. + + Raises + ------ + ValueError: shape mismatch + If `a` and each choice array are not all broadcastable to the same + shape. + + See Also + -------- + ndarray.choose : equivalent method + + Notes + ----- + To reduce the chance of misinterpretation, even though the following + "abuse" is nominally supported, `choices` should neither be, nor be + thought of as, a single array, i.e., the outermost sequence-like container + should be either a list or a tuple. + + Examples + -------- + + >>> choices = [[0, 1, 2, 3], [10, 11, 12, 13], + ... [20, 21, 22, 23], [30, 31, 32, 33]] + >>> np.choose([2, 3, 1, 0], choices + ... # the first element of the result will be the first element of the + ... # third (2+1) "array" in choices, namely, 20; the second element + ... # will be the second element of the fourth (3+1) choice array, i.e., + ... # 31, etc. + ... ) + array([20, 31, 12, 3]) + >>> np.choose([2, 4, 1, 0], choices, mode='clip') # 4 goes to 3 (4-1) + array([20, 31, 12, 3]) + >>> # because there are 4 choice arrays + >>> np.choose([2, 4, 1, 0], choices, mode='wrap') # 4 goes to (4 mod 4) + array([20, 1, 12, 3]) + >>> # i.e., 0 + + A couple examples illustrating how choose broadcasts: + + >>> a = [[1, 0, 1], [0, 1, 0], [1, 0, 1]] + >>> choices = [-10, 10] + >>> np.choose(a, choices) + array([[ 10, -10, 10], + [-10, 10, -10], + [ 10, -10, 10]]) + + >>> # With thanks to Anne Archibald + >>> a = np.array([0, 1]).reshape((2,1,1)) + >>> c1 = np.array([1, 2, 3]).reshape((1,3,1)) + >>> c2 = np.array([-1, -2, -3, -4, -5]).reshape((1,1,5)) + >>> np.choose(a, (c1, c2)) # result is 2x3x5, res[0,:,:]=c1, res[1,:,:]=c2 + array([[[ 1, 1, 1, 1, 1], + [ 2, 2, 2, 2, 2], + [ 3, 3, 3, 3, 3]], + [[-1, -2, -3, -4, -5], + [-1, -2, -3, -4, -5], + [-1, -2, -3, -4, -5]]]) + + """ + raise NotImplemented('Waiting on interp level method') + + +def repeat(a, repeats, axis=None): + """ + Repeat elements of an array. + + Parameters + ---------- + a : array_like + Input array. + repeats : {int, array of ints} + The number of repetitions for each element. `repeats` is broadcasted + to fit the shape of the given axis. + axis : int, optional + The axis along which to repeat values. By default, use the + flattened input array, and return a flat output array. + + Returns + ------- + repeated_array : ndarray + Output array which has the same shape as `a`, except along + the given axis. + + See Also + -------- + tile : Tile an array. + + Examples + -------- + >>> x = np.array([[1,2],[3,4]]) + >>> np.repeat(x, 2) + array([1, 1, 2, 2, 3, 3, 4, 4]) + >>> np.repeat(x, 3, axis=1) + array([[1, 1, 1, 2, 2, 2], + [3, 3, 3, 4, 4, 4]]) + >>> np.repeat(x, [1, 2], axis=0) + array([[1, 2], + [3, 4], + [3, 4]]) + + """ + raise NotImplemented('Waiting on interp level method') + + +def put(a, ind, v, mode='raise'): + """ + Replaces specified elements of an array with given values. + + The indexing works on the flattened target array. `put` is roughly + equivalent to: + + :: + + a.flat[ind] = v + + Parameters + ---------- + a : ndarray + Target array. + ind : array_like + Target indices, interpreted as integers. + v : array_like + Values to place in `a` at target indices. If `v` is shorter than + `ind` it will be repeated as necessary. + mode : {'raise', 'wrap', 'clip'}, optional + Specifies how out-of-bounds indices will behave. + + * 'raise' -- raise an error (default) + * 'wrap' -- wrap around + * 'clip' -- clip to the range + + 'clip' mode means that all indices that are too large are replaced + by the index that addresses the last element along that axis. Note + that this disables indexing with negative numbers. + + See Also + -------- + putmask, place + + Examples + -------- + >>> a = np.arange(5) + >>> np.put(a, [0, 2], [-44, -55]) + >>> a + array([-44, 1, -55, 3, 4]) + + >>> a = np.arange(5) + >>> np.put(a, 22, -5, mode='clip') + >>> a + array([ 0, 1, 2, 3, -5]) + + """ + raise NotImplemented('Waiting on interp level method') + + +def swapaxes(a, axis1, axis2): + """ + Interchange two axes of an array. + + Parameters + ---------- + a : array_like + Input array. + axis1 : int + First axis. + axis2 : int + Second axis. + + Returns + ------- + a_swapped : ndarray + If `a` is an ndarray, then a view of `a` is returned; otherwise + a new array is created. + + Examples + -------- + >>> x = np.array([[1,2,3]]) + >>> np.swapaxes(x,0,1) + array([[1], + [2], + [3]]) + + >>> x = np.array([[[0,1],[2,3]],[[4,5],[6,7]]]) + >>> x + array([[[0, 1], + [2, 3]], + [[4, 5], + [6, 7]]]) + + >>> np.swapaxes(x,0,2) + array([[[0, 4], + [2, 6]], + [[1, 5], + [3, 7]]]) + + """ + raise NotImplemented('Waiting on interp level method') + + +def transpose(a, axes=None): + """ + Permute the dimensions of an array. + + Parameters + ---------- + a : array_like + Input array. + axes : list of ints, optional + By default, reverse the dimensions, otherwise permute the axes + according to the values given. + + Returns + ------- + p : ndarray + `a` with its axes permuted. A view is returned whenever + possible. + + See Also + -------- + rollaxis + + Examples + -------- + >>> x = np.arange(4).reshape((2,2)) + >>> x + array([[0, 1], + [2, 3]]) + + >>> np.transpose(x) + array([[0, 2], + [1, 3]]) + + >>> x = np.ones((1, 2, 3)) + >>> np.transpose(x, (1, 0, 2)).shape + (2, 1, 3) + + """ + raise NotImplemented('Waiting on interp level method') + + +def sort(a, axis=-1, kind='quicksort', order=None): + """ + Return a sorted copy of an array. + + Parameters + ---------- + a : array_like + Array to be sorted. + axis : int or None, optional + Axis along which to sort. If None, the array is flattened before + sorting. The default is -1, which sorts along the last axis. + kind : {'quicksort', 'mergesort', 'heapsort'}, optional + Sorting algorithm. Default is 'quicksort'. + order : list, optional + When `a` is a structured array, this argument specifies which fields + to compare first, second, and so on. This list does not need to + include all of the fields. + + Returns + ------- + sorted_array : ndarray + Array of the same type and shape as `a`. + + See Also + -------- + ndarray.sort : Method to sort an array in-place. + argsort : Indirect sort. + lexsort : Indirect stable sort on multiple keys. + searchsorted : Find elements in a sorted array. + + Notes + ----- + The various sorting algorithms are characterized by their average speed, + worst case performance, work space size, and whether they are stable. A + stable sort keeps items with the same key in the same relative + order. The three available algorithms have the following + properties: + + =========== ======= ============= ============ ======= + kind speed worst case work space stable + =========== ======= ============= ============ ======= + 'quicksort' 1 O(n^2) 0 no + 'mergesort' 2 O(n*log(n)) ~n/2 yes + 'heapsort' 3 O(n*log(n)) 0 no + =========== ======= ============= ============ ======= + + All the sort algorithms make temporary copies of the data when + sorting along any but the last axis. Consequently, sorting along + the last axis is faster and uses less space than sorting along + any other axis. + + The sort order for complex numbers is lexicographic. If both the real + and imaginary parts are non-nan then the order is determined by the + real parts except when they are equal, in which case the order is + determined by the imaginary parts. + + Previous to numpy 1.4.0 sorting real and complex arrays containing nan + values led to undefined behaviour. In numpy versions >= 1.4.0 nan + values are sorted to the end. The extended sort order is: + + * Real: [R, nan] + * Complex: [R + Rj, R + nanj, nan + Rj, nan + nanj] + + where R is a non-nan real value. Complex values with the same nan + placements are sorted according to the non-nan part if it exists. + Non-nan values are sorted as before. + + Examples + -------- + >>> a = np.array([[1,4],[3,1]]) + >>> np.sort(a) # sort along the last axis + array([[1, 4], + [1, 3]]) + >>> np.sort(a, axis=None) # sort the flattened array + array([1, 1, 3, 4]) + >>> np.sort(a, axis=0) # sort along the first axis + array([[1, 1], + [3, 4]]) + + Use the `order` keyword to specify a field to use when sorting a + structured array: + + >>> dtype = [('name', 'S10'), ('height', float), ('age', int)] + >>> values = [('Arthur', 1.8, 41), ('Lancelot', 1.9, 38), + ... ('Galahad', 1.7, 38)] + >>> a = np.array(values, dtype=dtype) # create a structured array + >>> np.sort(a, order='height') # doctest: +SKIP + array([('Galahad', 1.7, 38), ('Arthur', 1.8, 41), + ('Lancelot', 1.8999999999999999, 38)], + dtype=[('name', '|S10'), ('height', '>> np.sort(a, order=['age', 'height']) # doctest: +SKIP + array([('Galahad', 1.7, 38), ('Lancelot', 1.8999999999999999, 38), + ('Arthur', 1.8, 41)], + dtype=[('name', '|S10'), ('height', '>> x = np.array([3, 1, 2]) + >>> np.argsort(x) + array([1, 2, 0]) + + Two-dimensional array: + + >>> x = np.array([[0, 3], [2, 2]]) + >>> x + array([[0, 3], + [2, 2]]) + + >>> np.argsort(x, axis=0) + array([[0, 1], + [1, 0]]) + + >>> np.argsort(x, axis=1) + array([[0, 1], + [0, 1]]) + + Sorting with keys: + + >>> x = np.array([(1, 0), (0, 1)], dtype=[('x', '>> x + array([(1, 0), (0, 1)], + dtype=[('x', '>> np.argsort(x, order=('x','y')) + array([1, 0]) + + >>> np.argsort(x, order=('y','x')) + array([0, 1]) + + """ + raise NotImplemented('Waiting on interp level method') + + +def argmax(a, axis=None): + """ + Indices of the maximum values along an axis. + + Parameters + ---------- + a : array_like + Input array. + axis : int, optional + By default, the index is into the flattened array, otherwise + along the specified axis. + + Returns + ------- + index_array : ndarray of ints + Array of indices into the array. It has the same shape as `a.shape` + with the dimension along `axis` removed. + + See Also + -------- + ndarray.argmax, argmin + amax : The maximum value along a given axis. + unravel_index : Convert a flat index into an index tuple. + + Notes + ----- + In case of multiple occurrences of the maximum values, the indices + corresponding to the first occurrence are returned. + + Examples + -------- + >>> a = np.arange(6).reshape(2,3) + >>> a + array([[0, 1, 2], + [3, 4, 5]]) + >>> np.argmax(a) + 5 + >>> np.argmax(a, axis=0) + array([1, 1, 1]) + >>> np.argmax(a, axis=1) + array([2, 2]) + + >>> b = np.arange(6) + >>> b[1] = 5 + >>> b + array([0, 5, 2, 3, 4, 5]) + >>> np.argmax(b) # Only the first occurrence is returned. + 1 + + """ + if not hasattr(a, 'argmax'): + a = numpypy.array(a) + return a.argmax() + + +def argmin(a, axis=None): + """ + Return the indices of the minimum values along an axis. + + See Also + -------- + argmax : Similar function. Please refer to `numpy.argmax` for detailed + documentation. + + """ + if not hasattr(a, 'argmin'): + a = numpypy.array(a) + return a.argmin() + + +def searchsorted(a, v, side='left'): + """ + Find indices where elements should be inserted to maintain order. + + Find the indices into a sorted array `a` such that, if the corresponding + elements in `v` were inserted before the indices, the order of `a` would + be preserved. + + Parameters + ---------- + a : 1-D array_like + Input array, sorted in ascending order. + v : array_like + Values to insert into `a`. + side : {'left', 'right'}, optional + If 'left', the index of the first suitable location found is given. If + 'right', return the last such index. If there is no suitable + index, return either 0 or N (where N is the length of `a`). + + Returns + ------- + indices : array of ints + Array of insertion points with the same shape as `v`. + + See Also + -------- + sort : Return a sorted copy of an array. + histogram : Produce histogram from 1-D data. + + Notes + ----- + Binary search is used to find the required insertion points. + + As of Numpy 1.4.0 `searchsorted` works with real/complex arrays containing + `nan` values. The enhanced sort order is documented in `sort`. + + Examples + -------- + >>> np.searchsorted([1,2,3,4,5], 3) + 2 + >>> np.searchsorted([1,2,3,4,5], 3, side='right') + 3 + >>> np.searchsorted([1,2,3,4,5], [-10, 10, 2, 3]) + array([0, 5, 1, 2]) + + """ + raise NotImplemented('Waiting on interp level method') + + +def resize(a, new_shape): + """ + Return a new array with the specified shape. + + If the new array is larger than the original array, then the new + array is filled with repeated copies of `a`. Note that this behavior + is different from a.resize(new_shape) which fills with zeros instead + of repeated copies of `a`. + + Parameters + ---------- + a : array_like + Array to be resized. + + new_shape : int or tuple of int + Shape of resized array. + + Returns + ------- + reshaped_array : ndarray + The new array is formed from the data in the old array, repeated + if necessary to fill out the required number of elements. The + data are repeated in the order that they are stored in memory. + + See Also + -------- + ndarray.resize : resize an array in-place. + + Examples + -------- + >>> a=np.array([[0,1],[2,3]]) + >>> np.resize(a,(1,4)) + array([[0, 1, 2, 3]]) + >>> np.resize(a,(2,4)) + array([[0, 1, 2, 3], + [0, 1, 2, 3]]) + + """ + raise NotImplemented('Waiting on interp level method') + + +def squeeze(a): + """ + Remove single-dimensional entries from the shape of an array. + + Parameters + ---------- + a : array_like + Input data. + + Returns + ------- + squeezed : ndarray + The input array, but with with all dimensions of length 1 + removed. Whenever possible, a view on `a` is returned. + + Examples + -------- + >>> x = np.array([[[0], [1], [2]]]) + >>> x.shape + (1, 3, 1) + >>> np.squeeze(x).shape + (3,) + + """ + raise NotImplemented('Waiting on interp level method') + + +def diagonal(a, offset=0, axis1=0, axis2=1): + """ + Return specified diagonals. + + If `a` is 2-D, returns the diagonal of `a` with the given offset, + i.e., the collection of elements of the form ``a[i, i+offset]``. If + `a` has more than two dimensions, then the axes specified by `axis1` + and `axis2` are used to determine the 2-D sub-array whose diagonal is + returned. The shape of the resulting array can be determined by + removing `axis1` and `axis2` and appending an index to the right equal + to the size of the resulting diagonals. + + Parameters + ---------- + a : array_like + Array from which the diagonals are taken. + offset : int, optional + Offset of the diagonal from the main diagonal. Can be positive or + negative. Defaults to main diagonal (0). + axis1 : int, optional + Axis to be used as the first axis of the 2-D sub-arrays from which + the diagonals should be taken. Defaults to first axis (0). + axis2 : int, optional + Axis to be used as the second axis of the 2-D sub-arrays from + which the diagonals should be taken. Defaults to second axis (1). + + Returns + ------- + array_of_diagonals : ndarray + If `a` is 2-D, a 1-D array containing the diagonal is returned. + If the dimension of `a` is larger, then an array of diagonals is + returned, "packed" from left-most dimension to right-most (e.g., + if `a` is 3-D, then the diagonals are "packed" along rows). + + Raises + ------ + ValueError + If the dimension of `a` is less than 2. + + See Also + -------- + diag : MATLAB work-a-like for 1-D and 2-D arrays. + diagflat : Create diagonal arrays. + trace : Sum along diagonals. + + Examples + -------- + >>> a = np.arange(4).reshape(2,2) + >>> a + array([[0, 1], + [2, 3]]) + >>> a.diagonal() + array([0, 3]) + >>> a.diagonal(1) + array([1]) + + A 3-D example: + + >>> a = np.arange(8).reshape(2,2,2); a + array([[[0, 1], + [2, 3]], + [[4, 5], + [6, 7]]]) + >>> a.diagonal(0, # Main diagonals of two arrays created by skipping + ... 0, # across the outer(left)-most axis last and + ... 1) # the "middle" (row) axis first. + array([[0, 6], + [1, 7]]) + + The sub-arrays whose main diagonals we just obtained; note that each + corresponds to fixing the right-most (column) axis, and that the + diagonals are "packed" in rows. + + >>> a[:,:,0] # main diagonal is [0 6] + array([[0, 2], + [4, 6]]) + >>> a[:,:,1] # main diagonal is [1 7] + array([[1, 3], + [5, 7]]) + + """ + raise NotImplemented('Waiting on interp level method') + + +def trace(a, offset=0, axis1=0, axis2=1, dtype=None, out=None): + """ + Return the sum along diagonals of the array. + + If `a` is 2-D, the sum along its diagonal with the given offset + is returned, i.e., the sum of elements ``a[i,i+offset]`` for all i. + + If `a` has more than two dimensions, then the axes specified by axis1 and + axis2 are used to determine the 2-D sub-arrays whose traces are returned. + The shape of the resulting array is the same as that of `a` with `axis1` + and `axis2` removed. + + Parameters + ---------- + a : array_like + Input array, from which the diagonals are taken. + offset : int, optional + Offset of the diagonal from the main diagonal. Can be both positive + and negative. Defaults to 0. + axis1, axis2 : int, optional + Axes to be used as the first and second axis of the 2-D sub-arrays + from which the diagonals should be taken. Defaults are the first two + axes of `a`. + dtype : dtype, optional + Determines the data-type of the returned array and of the accumulator + where the elements are summed. If dtype has the value None and `a` is + of integer type of precision less than the default integer + precision, then the default integer precision is used. Otherwise, + the precision is the same as that of `a`. + out : ndarray, optional + Array into which the output is placed. Its type is preserved and + it must be of the right shape to hold the output. + + Returns + ------- + sum_along_diagonals : ndarray + If `a` is 2-D, the sum along the diagonal is returned. If `a` has + larger dimensions, then an array of sums along diagonals is returned. + + See Also + -------- + diag, diagonal, diagflat + + Examples + -------- + >>> np.trace(np.eye(3)) + 3.0 + >>> a = np.arange(8).reshape((2,2,2)) + >>> np.trace(a) + array([6, 8]) + + >>> a = np.arange(24).reshape((2,2,2,3)) + >>> np.trace(a).shape + (2, 3) + + """ + raise NotImplemented('Waiting on interp level method') + +def ravel(a, order='C'): + """ + Return a flattened array. + + A 1-D array, containing the elements of the input, is returned. A copy is + made only if needed. + + Parameters + ---------- + a : array_like + Input array. The elements in ``a`` are read in the order specified by + `order`, and packed as a 1-D array. + order : {'C','F', 'A', 'K'}, optional + The elements of ``a`` are read in this order. 'C' means to view + the elements in C (row-major) order. 'F' means to view the elements + in Fortran (column-major) order. 'A' means to view the elements + in 'F' order if a is Fortran contiguous, 'C' order otherwise. + 'K' means to view the elements in the order they occur in memory, + except for reversing the data when strides are negative. + By default, 'C' order is used. + + Returns + ------- + 1d_array : ndarray + Output of the same dtype as `a`, and of shape ``(a.size(),)``. + + See Also + -------- + ndarray.flat : 1-D iterator over an array. + ndarray.flatten : 1-D array copy of the elements of an array + in row-major order. + + Notes + ----- + In row-major order, the row index varies the slowest, and the column + index the quickest. This can be generalized to multiple dimensions, + where row-major order implies that the index along the first axis + varies slowest, and the index along the last quickest. The opposite holds + for Fortran-, or column-major, mode. + + Examples + -------- + It is equivalent to ``reshape(-1, order=order)``. + + >>> x = np.array([[1, 2, 3], [4, 5, 6]]) + >>> print np.ravel(x) + [1 2 3 4 5 6] + + >>> print x.reshape(-1) + [1 2 3 4 5 6] + + >>> print np.ravel(x, order='F') + [1 4 2 5 3 6] + + When ``order`` is 'A', it will preserve the array's 'C' or 'F' ordering: + + >>> print np.ravel(x.T) + [1 4 2 5 3 6] + >>> print np.ravel(x.T, order='A') + [1 2 3 4 5 6] + + When ``order`` is 'K', it will preserve orderings that are neither 'C' + nor 'F', but won't reverse axes: + + >>> a = np.arange(3)[::-1]; a + array([2, 1, 0]) + >>> a.ravel(order='C') + array([2, 1, 0]) + >>> a.ravel(order='K') + array([2, 1, 0]) + + >>> a = np.arange(12).reshape(2,3,2).swapaxes(1,2); a + array([[[ 0, 2, 4], + [ 1, 3, 5]], + [[ 6, 8, 10], + [ 7, 9, 11]]]) + >>> a.ravel(order='C') + array([ 0, 2, 4, 1, 3, 5, 6, 8, 10, 7, 9, 11]) + >>> a.ravel(order='K') + array([ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]) + + """ + raise NotImplemented('Waiting on interp level method') + + +def nonzero(a): + """ + Return the indices of the elements that are non-zero. + + Returns a tuple of arrays, one for each dimension of `a`, containing + the indices of the non-zero elements in that dimension. The + corresponding non-zero values can be obtained with:: + + a[nonzero(a)] + + To group the indices by element, rather than dimension, use:: + + transpose(nonzero(a)) + + The result of this is always a 2-D array, with a row for + each non-zero element. + + Parameters + ---------- + a : array_like + Input array. + + Returns + ------- + tuple_of_arrays : tuple + Indices of elements that are non-zero. + + See Also + -------- + flatnonzero : + Return indices that are non-zero in the flattened version of the input + array. + ndarray.nonzero : + Equivalent ndarray method. + count_nonzero : + Counts the number of non-zero elements in the input array. + + Examples + -------- + >>> x = np.eye(3) + >>> x + array([[ 1., 0., 0.], + [ 0., 1., 0.], + [ 0., 0., 1.]]) + >>> np.nonzero(x) + (array([0, 1, 2]), array([0, 1, 2])) + + >>> x[np.nonzero(x)] + array([ 1., 1., 1.]) + >>> np.transpose(np.nonzero(x)) + array([[0, 0], + [1, 1], + [2, 2]]) + + A common use for ``nonzero`` is to find the indices of an array, where + a condition is True. Given an array `a`, the condition `a` > 3 is a + boolean array and since False is interpreted as 0, np.nonzero(a > 3) + yields the indices of the `a` where the condition is true. + + >>> a = np.array([[1,2,3],[4,5,6],[7,8,9]]) + >>> a > 3 + array([[False, False, False], + [ True, True, True], + [ True, True, True]], dtype=bool) + >>> np.nonzero(a > 3) + (array([1, 1, 1, 2, 2, 2]), array([0, 1, 2, 0, 1, 2])) + + The ``nonzero`` method of the boolean array can also be called. + + >>> (a > 3).nonzero() + (array([1, 1, 1, 2, 2, 2]), array([0, 1, 2, 0, 1, 2])) + + """ + raise NotImplemented('Waiting on interp level method') + + +def shape(a): + """ + Return the shape of an array. + + Parameters + ---------- + a : array_like + Input array. + + Returns + ------- + shape : tuple of ints + The elements of the shape tuple give the lengths of the + corresponding array dimensions. + + See Also + -------- + alen + ndarray.shape : Equivalent array method. + + Examples + -------- + >>> np.shape(np.eye(3)) + (3, 3) + >>> np.shape([[1, 2]]) + (1, 2) + >>> np.shape([0]) + (1,) + >>> np.shape(0) + () + + >>> a = np.array([(1, 2), (3, 4)], dtype=[('x', 'i4'), ('y', 'i4')]) + >>> np.shape(a) + (2,) + >>> a.shape + (2,) + + """ + if not hasattr(a, 'shape'): + a = numpypy.array(a) + return a.shape + + +def compress(condition, a, axis=None, out=None): + """ + Return selected slices of an array along given axis. + + When working along a given axis, a slice along that axis is returned in + `output` for each index where `condition` evaluates to True. When + working on a 1-D array, `compress` is equivalent to `extract`. + + Parameters + ---------- + condition : 1-D array of bools + Array that selects which entries to return. If len(condition) + is less than the size of `a` along the given axis, then output is + truncated to the length of the condition array. + a : array_like + Array from which to extract a part. + axis : int, optional + Axis along which to take slices. If None (default), work on the + flattened array. + out : ndarray, optional + Output array. Its type is preserved and it must be of the right + shape to hold the output. + + Returns + ------- + compressed_array : ndarray + A copy of `a` without the slices along axis for which `condition` + is false. + + See Also + -------- + take, choose, diag, diagonal, select + ndarray.compress : Equivalent method. + numpy.doc.ufuncs : Section "Output arguments" + + Examples + -------- + >>> a = np.array([[1, 2], [3, 4], [5, 6]]) + >>> a + array([[1, 2], + [3, 4], + [5, 6]]) + >>> np.compress([0, 1], a, axis=0) + array([[3, 4]]) + >>> np.compress([False, True, True], a, axis=0) + array([[3, 4], + [5, 6]]) + >>> np.compress([False, True], a, axis=1) + array([[2], + [4], + [6]]) + + Working on the flattened array does not return slices along an axis but + selects elements. + + >>> np.compress([False, True], a) + array([2]) + + """ + raise NotImplemented('Waiting on interp level method') + + +def clip(a, a_min, a_max, out=None): + """ + Clip (limit) the values in an array. + + Given an interval, values outside the interval are clipped to + the interval edges. For example, if an interval of ``[0, 1]`` + is specified, values smaller than 0 become 0, and values larger + than 1 become 1. + + Parameters + ---------- + a : array_like + Array containing elements to clip. + a_min : scalar or array_like + Minimum value. + a_max : scalar or array_like + Maximum value. If `a_min` or `a_max` are array_like, then they will + be broadcasted to the shape of `a`. + out : ndarray, optional + The results will be placed in this array. It may be the input + array for in-place clipping. `out` must be of the right shape + to hold the output. Its type is preserved. + + Returns + ------- + clipped_array : ndarray + An array with the elements of `a`, but where values + < `a_min` are replaced with `a_min`, and those > `a_max` + with `a_max`. + + See Also + -------- + numpy.doc.ufuncs : Section "Output arguments" + + Examples + -------- + >>> a = np.arange(10) + >>> np.clip(a, 1, 8) + array([1, 1, 2, 3, 4, 5, 6, 7, 8, 8]) + >>> a + array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) + >>> np.clip(a, 3, 6, out=a) + array([3, 3, 3, 3, 4, 5, 6, 6, 6, 6]) + >>> a = np.arange(10) + >>> a + array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) + >>> np.clip(a, [3,4,1,1,1,4,4,4,4,4], 8) + array([3, 4, 2, 3, 4, 5, 6, 7, 8, 8]) + + """ + raise NotImplemented('Waiting on interp level method') + + +def sum(a, axis=None, dtype=None, out=None): + """ + Sum of array elements over a given axis. + + Parameters + ---------- + a : array_like + Elements to sum. + axis : integer, optional + Axis over which the sum is taken. By default `axis` is None, + and all elements are summed. + dtype : dtype, optional + The type of the returned array and of the accumulator in which + the elements are summed. By default, the dtype of `a` is used. + An exception is when `a` has an integer type with less precision + than the default platform integer. In that case, the default + platform integer is used instead. + out : ndarray, optional + Array into which the output is placed. By default, a new array is + created. If `out` is given, it must be of the appropriate shape + (the shape of `a` with `axis` removed, i.e., + ``numpy.delete(a.shape, axis)``). Its type is preserved. See + `doc.ufuncs` (Section "Output arguments") for more details. + + Returns + ------- + sum_along_axis : ndarray + An array with the same shape as `a`, with the specified + axis removed. If `a` is a 0-d array, or if `axis` is None, a scalar + is returned. If an output array is specified, a reference to + `out` is returned. + + See Also + -------- + ndarray.sum : Equivalent method. + + cumsum : Cumulative sum of array elements. + + trapz : Integration of array values using the composite trapezoidal rule. + + mean, average + + Notes + ----- + Arithmetic is modular when using integer types, and no error is + raised on overflow. + + Examples + -------- + >>> np.sum([0.5, 1.5]) + 2.0 + >>> np.sum([0.5, 0.7, 0.2, 1.5], dtype=np.int32) + 1 + >>> np.sum([[0, 1], [0, 5]]) + 6 + >>> np.sum([[0, 1], [0, 5]], axis=0) + array([0, 6]) + >>> np.sum([[0, 1], [0, 5]], axis=1) + array([1, 5]) + + If the accumulator is too small, overflow occurs: + + >>> np.ones(128, dtype=np.int8).sum(dtype=np.int8) + -128 + + """ + if not hasattr(a, "sum"): + a = numpypy.array(a) + return a.sum() + + +def product (a, axis=None, dtype=None, out=None): + """ + Return the product of array elements over a given axis. + + See Also + -------- + prod : equivalent function; see for details. + + """ + raise NotImplemented('Waiting on interp level method') + + +def sometrue(a, axis=None, out=None): + """ + Check whether some values are true. + + Refer to `any` for full documentation. + + See Also + -------- + any : equivalent function + + """ + if not hasattr(a, 'any'): + a = numpypy.array(a) + return a.any() + + +def alltrue (a, axis=None, out=None): + """ + Check if all elements of input array are true. + + See Also + -------- + numpy.all : Equivalent function; see for details. + + """ + if not hasattr(a, 'all'): + a = numpypy.array(a) + return a.all() + +def any(a,axis=None, out=None): + """ + Test whether any array element along a given axis evaluates to True. + + Returns single boolean unless `axis` is not ``None`` + + Parameters + ---------- + a : array_like + Input array or object that can be converted to an array. + axis : int, optional + Axis along which a logical OR is performed. The default + (`axis` = `None`) is to perform a logical OR over a flattened + input array. `axis` may be negative, in which case it counts + from the last to the first axis. + out : ndarray, optional + Alternate output array in which to place the result. It must have + the same shape as the expected output and its type is preserved + (e.g., if it is of type float, then it will remain so, returning + 1.0 for True and 0.0 for False, regardless of the type of `a`). + See `doc.ufuncs` (Section "Output arguments") for details. + + Returns + ------- + any : bool or ndarray + A new boolean or `ndarray` is returned unless `out` is specified, + in which case a reference to `out` is returned. + + See Also + -------- + ndarray.any : equivalent method + + all : Test whether all elements along a given axis evaluate to True. + + Notes + ----- + Not a Number (NaN), positive infinity and negative infinity evaluate + to `True` because these are not equal to zero. + + Examples + -------- + >>> np.any([[True, False], [True, True]]) + True + + >>> np.any([[True, False], [False, False]], axis=0) + array([ True, False], dtype=bool) + + >>> np.any([-1, 0, 5]) + True + + >>> np.any(np.nan) + True + + >>> o=np.array([False]) + >>> z=np.any([-1, 4, 5], out=o) + >>> z, o + (array([ True], dtype=bool), array([ True], dtype=bool)) + >>> # Check now that z is a reference to o + >>> z is o + True + >>> id(z), id(o) # identity of z and o # doctest: +SKIP + (191614240, 191614240) + + """ + if not hasattr(a, 'any'): + a = numpypy.array(a) + return a.any() + + +def all(a,axis=None, out=None): + """ + Test whether all array elements along a given axis evaluate to True. + + Parameters + ---------- + a : array_like + Input array or object that can be converted to an array. + axis : int, optional + Axis along which a logical AND is performed. + The default (`axis` = `None`) is to perform a logical AND + over a flattened input array. `axis` may be negative, in which + case it counts from the last to the first axis. + out : ndarray, optional + Alternate output array in which to place the result. + It must have the same shape as the expected output and its + type is preserved (e.g., if ``dtype(out)`` is float, the result + will consist of 0.0's and 1.0's). See `doc.ufuncs` (Section + "Output arguments") for more details. + + Returns + ------- + all : ndarray, bool + A new boolean or array is returned unless `out` is specified, + in which case a reference to `out` is returned. + + See Also + -------- + ndarray.all : equivalent method + + any : Test whether any element along a given axis evaluates to True. + + Notes + ----- + Not a Number (NaN), positive infinity and negative infinity + evaluate to `True` because these are not equal to zero. + + Examples + -------- + >>> np.all([[True,False],[True,True]]) + False + + >>> np.all([[True,False],[True,True]], axis=0) + array([ True, False], dtype=bool) + + >>> np.all([-1, 4, 5]) + True + + >>> np.all([1.0, np.nan]) + True + + >>> o=np.array([False]) + >>> z=np.all([-1, 4, 5], out=o) + >>> id(z), id(o), z # doctest: +SKIP + (28293632, 28293632, array([ True], dtype=bool)) + + """ + if not hasattr(a, 'all'): + a = numpypy.array(a) + return a.all() + + +def cumsum (a, axis=None, dtype=None, out=None): + """ + Return the cumulative sum of the elements along a given axis. + + Parameters + ---------- + a : array_like + Input array. + axis : int, optional + Axis along which the cumulative sum is computed. The default + (None) is to compute the cumsum over the flattened array. + dtype : dtype, optional + Type of the returned array and of the accumulator in which the + elements are summed. If `dtype` is not specified, it defaults + to the dtype of `a`, unless `a` has an integer dtype with a + precision less than that of the default platform integer. In + that case, the default platform integer is used. + out : ndarray, optional + Alternative output array in which to place the result. It must + have the same shape and buffer length as the expected output + but the type will be cast if necessary. See `doc.ufuncs` + (Section "Output arguments") for more details. + + Returns + ------- + cumsum_along_axis : ndarray. + A new array holding the result is returned unless `out` is + specified, in which case a reference to `out` is returned. The + result has the same size as `a`, and the same shape as `a` if + `axis` is not None or `a` is a 1-d array. + + + See Also + -------- + sum : Sum array elements. + + trapz : Integration of array values using the composite trapezoidal rule. + + Notes + ----- + Arithmetic is modular when using integer types, and no error is + raised on overflow. + + Examples + -------- + >>> a = np.array([[1,2,3], [4,5,6]]) + >>> a + array([[1, 2, 3], + [4, 5, 6]]) + >>> np.cumsum(a) + array([ 1, 3, 6, 10, 15, 21]) + >>> np.cumsum(a, dtype=float) # specifies type of output value(s) + array([ 1., 3., 6., 10., 15., 21.]) + + >>> np.cumsum(a,axis=0) # sum over rows for each of the 3 columns + array([[1, 2, 3], + [5, 7, 9]]) + >>> np.cumsum(a,axis=1) # sum over columns for each of the 2 rows + array([[ 1, 3, 6], + [ 4, 9, 15]]) + + """ + raise NotImplemented('Waiting on interp level method') + + +def cumproduct(a, axis=None, dtype=None, out=None): + """ + Return the cumulative product over the given axis. + + + See Also + -------- + cumprod : equivalent function; see for details. + + """ + raise NotImplemented('Waiting on interp level method') + + +def ptp(a, axis=None, out=None): + """ + Range of values (maximum - minimum) along an axis. + + The name of the function comes from the acronym for 'peak to peak'. + + Parameters + ---------- + a : array_like + Input values. + axis : int, optional + Axis along which to find the peaks. By default, flatten the + array. + out : array_like + Alternative output array in which to place the result. It must + have the same shape and buffer length as the expected output, + but the type of the output values will be cast if necessary. + + Returns + ------- + ptp : ndarray + A new array holding the result, unless `out` was + specified, in which case a reference to `out` is returned. + + Examples + -------- + >>> x = np.arange(4).reshape((2,2)) + >>> x + array([[0, 1], + [2, 3]]) + + >>> np.ptp(x, axis=0) + array([2, 2]) + + >>> np.ptp(x, axis=1) + array([1, 1]) + + """ + raise NotImplemented('Waiting on interp level method') + + +def amax(a, axis=None, out=None): + """ + Return the maximum of an array or maximum along an axis. + + Parameters + ---------- + a : array_like + Input data. + axis : int, optional + Axis along which to operate. By default flattened input is used. + out : ndarray, optional + Alternate output array in which to place the result. Must be of + the same shape and buffer length as the expected output. See + `doc.ufuncs` (Section "Output arguments") for more details. + + Returns + ------- + amax : ndarray or scalar + Maximum of `a`. If `axis` is None, the result is a scalar value. + If `axis` is given, the result is an array of dimension + ``a.ndim - 1``. + + See Also + -------- + nanmax : NaN values are ignored instead of being propagated. + fmax : same behavior as the C99 fmax function. + argmax : indices of the maximum values. + + Notes + ----- + NaN values are propagated, that is if at least one item is NaN, the + corresponding max value will be NaN as well. To ignore NaN values + (MATLAB behavior), please use nanmax. + + Examples + -------- + >>> a = np.arange(4).reshape((2,2)) + >>> a + array([[0, 1], + [2, 3]]) + >>> np.amax(a) + 3 + >>> np.amax(a, axis=0) + array([2, 3]) + >>> np.amax(a, axis=1) + array([1, 3]) + + >>> b = np.arange(5, dtype=np.float) + >>> b[2] = np.NaN + >>> np.amax(b) + nan + >>> np.nanmax(b) + 4.0 + + """ + if not hasattr(a, "max"): + a = numpypy.array(a) + return a.max() + + +def amin(a, axis=None, out=None): + """ + Return the minimum of an array or minimum along an axis. + + Parameters + ---------- + a : array_like + Input data. + axis : int, optional + Axis along which to operate. By default a flattened input is used. + out : ndarray, optional + Alternative output array in which to place the result. Must + be of the same shape and buffer length as the expected output. + See `doc.ufuncs` (Section "Output arguments") for more details. + + Returns + ------- + amin : ndarray + A new array or a scalar array with the result. + + See Also + -------- + nanmin: nan values are ignored instead of being propagated + fmin: same behavior as the C99 fmin function + argmin: Return the indices of the minimum values. + + amax, nanmax, fmax + + Notes + ----- + NaN values are propagated, that is if at least one item is nan, the + corresponding min value will be nan as well. To ignore NaN values (matlab + behavior), please use nanmin. + + Examples + -------- + >>> a = np.arange(4).reshape((2,2)) + >>> a + array([[0, 1], + [2, 3]]) + >>> np.amin(a) # Minimum of the flattened array + 0 + >>> np.amin(a, axis=0) # Minima along the first axis + array([0, 1]) + >>> np.amin(a, axis=1) # Minima along the second axis + array([0, 2]) + + >>> b = np.arange(5, dtype=np.float) + >>> b[2] = np.NaN + >>> np.amin(b) + nan + >>> np.nanmin(b) + 0.0 + + """ + # amin() is equivalent to min() + if not hasattr(a, 'min'): + a = numpypy.array(a) + return a.min() + +def alen(a): + """ + Return the length of the first dimension of the input array. + + Parameters + ---------- + a : array_like + Input array. + + Returns + ------- + l : int + Length of the first dimension of `a`. + + See Also + -------- + shape, size + + Examples + -------- + >>> a = np.zeros((7,4,5)) + >>> a.shape[0] + 7 + >>> np.alen(a) + 7 + + """ + if not hasattr(a, 'shape'): + a = numpypy.array(a) + return a.shape[0] + + +def prod(a, axis=None, dtype=None, out=None): + """ + Return the product of array elements over a given axis. + + Parameters + ---------- + a : array_like + Input data. + axis : int, optional + Axis over which the product is taken. By default, the product + of all elements is calculated. + dtype : data-type, optional + The data-type of the returned array, as well as of the accumulator + in which the elements are multiplied. By default, if `a` is of + integer type, `dtype` is the default platform integer. (Note: if + the type of `a` is unsigned, then so is `dtype`.) Otherwise, + the dtype is the same as that of `a`. + out : ndarray, optional + Alternative output array in which to place the result. It must have + the same shape as the expected output, but the type of the + output values will be cast if necessary. + + Returns + ------- + product_along_axis : ndarray, see `dtype` parameter above. + An array shaped as `a` but with the specified axis removed. + Returns a reference to `out` if specified. + + See Also + -------- + ndarray.prod : equivalent method + numpy.doc.ufuncs : Section "Output arguments" + + Notes + ----- + Arithmetic is modular when using integer types, and no error is + raised on overflow. That means that, on a 32-bit platform: + + >>> x = np.array([536870910, 536870910, 536870910, 536870910]) + >>> np.prod(x) #random + 16 + + Examples + -------- + By default, calculate the product of all elements: + + >>> np.prod([1.,2.]) + 2.0 + + Even when the input array is two-dimensional: + + >>> np.prod([[1.,2.],[3.,4.]]) + 24.0 + + But we can also specify the axis over which to multiply: + + >>> np.prod([[1.,2.],[3.,4.]], axis=1) + array([ 2., 12.]) + + If the type of `x` is unsigned, then the output type is + the unsigned platform integer: + + >>> x = np.array([1, 2, 3], dtype=np.uint8) + >>> np.prod(x).dtype == np.uint + True + + If `x` is of a signed integer type, then the output type + is the default platform integer: + + >>> x = np.array([1, 2, 3], dtype=np.int8) + >>> np.prod(x).dtype == np.int + True + + """ + raise NotImplemented('Waiting on interp level method') + + +def cumprod(a, axis=None, dtype=None, out=None): + """ + Return the cumulative product of elements along a given axis. + + Parameters + ---------- + a : array_like + Input array. + axis : int, optional + Axis along which the cumulative product is computed. By default + the input is flattened. + dtype : dtype, optional + Type of the returned array, as well as of the accumulator in which + the elements are multiplied. If *dtype* is not specified, it + defaults to the dtype of `a`, unless `a` has an integer dtype with + a precision less than that of the default platform integer. In + that case, the default platform integer is used instead. + out : ndarray, optional + Alternative output array in which to place the result. It must + have the same shape and buffer length as the expected output + but the type of the resulting values will be cast if necessary. + + Returns + ------- + cumprod : ndarray + A new array holding the result is returned unless `out` is + specified, in which case a reference to out is returned. + + See Also + -------- + numpy.doc.ufuncs : Section "Output arguments" + + Notes + ----- + Arithmetic is modular when using integer types, and no error is + raised on overflow. + + Examples + -------- + >>> a = np.array([1,2,3]) + >>> np.cumprod(a) # intermediate results 1, 1*2 + ... # total product 1*2*3 = 6 + array([1, 2, 6]) + >>> a = np.array([[1, 2, 3], [4, 5, 6]]) + >>> np.cumprod(a, dtype=float) # specify type of output + array([ 1., 2., 6., 24., 120., 720.]) + + The cumulative product for each column (i.e., over the rows) of `a`: + + >>> np.cumprod(a, axis=0) + array([[ 1, 2, 3], + [ 4, 10, 18]]) + + The cumulative product for each row (i.e. over the columns) of `a`: + + >>> np.cumprod(a,axis=1) + array([[ 1, 2, 6], + [ 4, 20, 120]]) + + """ + raise NotImplemented('Waiting on interp level method') + + +def ndim(a): + """ + Return the number of dimensions of an array. + + Parameters + ---------- + a : array_like + Input array. If it is not already an ndarray, a conversion is + attempted. + + Returns + ------- + number_of_dimensions : int + The number of dimensions in `a`. Scalars are zero-dimensional. + + See Also + -------- + ndarray.ndim : equivalent method + shape : dimensions of array + ndarray.shape : dimensions of array + + Examples + -------- + >>> np.ndim([[1,2,3],[4,5,6]]) + 2 + >>> np.ndim(np.array([[1,2,3],[4,5,6]])) + 2 + >>> np.ndim(1) + 0 + + """ + if not hasattr(a, 'ndim'): + a = numpypy.array(a) + return a.ndim + + +def rank(a): + """ + Return the number of dimensions of an array. + + If `a` is not already an array, a conversion is attempted. + Scalars are zero dimensional. + + Parameters + ---------- + a : array_like + Array whose number of dimensions is desired. If `a` is not an array, + a conversion is attempted. + + Returns + ------- + number_of_dimensions : int + The number of dimensions in the array. + + See Also + -------- + ndim : equivalent function + ndarray.ndim : equivalent property + shape : dimensions of array + ndarray.shape : dimensions of array + + Notes + ----- + In the old Numeric package, `rank` was the term used for the number of + dimensions, but in Numpy `ndim` is used instead. + + Examples + -------- + >>> np.rank([1,2,3]) + 1 + >>> np.rank(np.array([[1,2,3],[4,5,6]])) + 2 + >>> np.rank(1) + 0 + + """ + if not hasattr(a, 'ndim'): + a = numpypy.array(a) + return a.ndim + + +def size(a, axis=None): + """ + Return the number of elements along a given axis. + + Parameters + ---------- + a : array_like + Input data. + axis : int, optional + Axis along which the elements are counted. By default, give + the total number of elements. + + Returns + ------- + element_count : int + Number of elements along the specified axis. + + See Also + -------- + shape : dimensions of array + ndarray.shape : dimensions of array + ndarray.size : number of elements in array + + Examples + -------- + >>> a = np.array([[1,2,3],[4,5,6]]) + >>> np.size(a) + 6 + >>> np.size(a,1) + 3 + >>> np.size(a,0) + 2 + + """ + raise NotImplemented('Waiting on interp level method') + + +def around(a, decimals=0, out=None): + """ + Evenly round to the given number of decimals. + + Parameters + ---------- + a : array_like + Input data. + decimals : int, optional + Number of decimal places to round to (default: 0). If + decimals is negative, it specifies the number of positions to + the left of the decimal point. + out : ndarray, optional + Alternative output array in which to place the result. It must have + the same shape as the expected output, but the type of the output + values will be cast if necessary. See `doc.ufuncs` (Section + "Output arguments") for details. + + Returns + ------- + rounded_array : ndarray + An array of the same type as `a`, containing the rounded values. + Unless `out` was specified, a new array is created. A reference to + the result is returned. + + The real and imaginary parts of complex numbers are rounded + separately. The result of rounding a float is a float. + + See Also + -------- + ndarray.round : equivalent method + + ceil, fix, floor, rint, trunc + + + Notes + ----- + For values exactly halfway between rounded decimal values, Numpy + rounds to the nearest even value. Thus 1.5 and 2.5 round to 2.0, + -0.5 and 0.5 round to 0.0, etc. Results may also be surprising due + to the inexact representation of decimal fractions in the IEEE + floating point standard [1]_ and errors introduced when scaling + by powers of ten. + + References + ---------- + .. [1] "Lecture Notes on the Status of IEEE 754", William Kahan, + http://www.cs.berkeley.edu/~wkahan/ieee754status/IEEE754.PDF + .. [2] "How Futile are Mindless Assessments of + Roundoff in Floating-Point Computation?", William Kahan, + http://www.cs.berkeley.edu/~wkahan/Mindless.pdf + + Examples + -------- + >>> np.around([0.37, 1.64]) + array([ 0., 2.]) + >>> np.around([0.37, 1.64], decimals=1) + array([ 0.4, 1.6]) + >>> np.around([.5, 1.5, 2.5, 3.5, 4.5]) # rounds to nearest even value + array([ 0., 2., 2., 4., 4.]) + >>> np.around([1,2,3,11], decimals=1) # ndarray of ints is returned + array([ 1, 2, 3, 11]) + >>> np.around([1,2,3,11], decimals=-1) + array([ 0, 0, 0, 10]) + + """ + raise NotImplemented('Waiting on interp level method') + + +def round_(a, decimals=0, out=None): + """ + Round an array to the given number of decimals. + + Refer to `around` for full documentation. + + See Also + -------- + around : equivalent function + + """ + raise NotImplemented('Waiting on interp level method') + + +def mean(a, axis=None, dtype=None, out=None): + """ + Compute the arithmetic mean along the specified axis. + + Returns the average of the array elements. The average is taken over + the flattened array by default, otherwise over the specified axis. + `float64` intermediate and return values are used for integer inputs. + + Parameters + ---------- + a : array_like + Array containing numbers whose mean is desired. If `a` is not an + array, a conversion is attempted. + axis : int, optional + Axis along which the means are computed. The default is to compute + the mean of the flattened array. + dtype : data-type, optional + Type to use in computing the mean. For integer inputs, the default + is `float64`; for floating point inputs, it is the same as the + input dtype. + out : ndarray, optional + Alternate output array in which to place the result. The default + is ``None``; if provided, it must have the same shape as the + expected output, but the type will be cast if necessary. + See `doc.ufuncs` for details. + + Returns + ------- + m : ndarray, see dtype parameter above + If `out=None`, returns a new array containing the mean values, + otherwise a reference to the output array is returned. + + See Also + -------- + average : Weighted average + + Notes + ----- + The arithmetic mean is the sum of the elements along the axis divided + by the number of elements. + + Note that for floating-point input, the mean is computed using the + same precision the input has. Depending on the input data, this can + cause the results to be inaccurate, especially for `float32` (see + example below). Specifying a higher-precision accumulator using the + `dtype` keyword can alleviate this issue. + + Examples + -------- + >>> a = np.array([[1, 2], [3, 4]]) + >>> np.mean(a) + 2.5 + >>> np.mean(a, axis=0) + array([ 2., 3.]) + >>> np.mean(a, axis=1) + array([ 1.5, 3.5]) + + In single precision, `mean` can be inaccurate: + + >>> a = np.zeros((2, 512*512), dtype=np.float32) + >>> a[0, :] = 1.0 + >>> a[1, :] = 0.1 + >>> np.mean(a) + 0.546875 + + Computing the mean in float64 is more accurate: + + >>> np.mean(a, dtype=np.float64) + 0.55000000074505806 + + """ + if not hasattr(a, "mean"): + a = numpypy.array(a) + return a.mean() + + +def std(a, axis=None, dtype=None, out=None, ddof=0): + """ + Compute the standard deviation along the specified axis. + + Returns the standard deviation, a measure of the spread of a distribution, + of the array elements. The standard deviation is computed for the + flattened array by default, otherwise over the specified axis. + + Parameters + ---------- + a : array_like + Calculate the standard deviation of these values. + axis : int, optional + Axis along which the standard deviation is computed. The default is + to compute the standard deviation of the flattened array. + dtype : dtype, optional + Type to use in computing the standard deviation. For arrays of + integer type the default is float64, for arrays of float types it is + the same as the array type. + out : ndarray, optional + Alternative output array in which to place the result. It must have + the same shape as the expected output but the type (of the calculated + values) will be cast if necessary. + ddof : int, optional + Means Delta Degrees of Freedom. The divisor used in calculations + is ``N - ddof``, where ``N`` represents the number of elements. + By default `ddof` is zero. + + Returns + ------- + standard_deviation : ndarray, see dtype parameter above. + If `out` is None, return a new array containing the standard deviation, + otherwise return a reference to the output array. + + See Also + -------- + var, mean + numpy.doc.ufuncs : Section "Output arguments" + + Notes + ----- + The standard deviation is the square root of the average of the squared + deviations from the mean, i.e., ``std = sqrt(mean(abs(x - x.mean())**2))``. + + The average squared deviation is normally calculated as ``x.sum() / N``, where + ``N = len(x)``. If, however, `ddof` is specified, the divisor ``N - ddof`` + is used instead. In standard statistical practice, ``ddof=1`` provides an + unbiased estimator of the variance of the infinite population. ``ddof=0`` + provides a maximum likelihood estimate of the variance for normally + distributed variables. The standard deviation computed in this function + is the square root of the estimated variance, so even with ``ddof=1``, it + will not be an unbiased estimate of the standard deviation per se. + + Note that, for complex numbers, `std` takes the absolute + value before squaring, so that the result is always real and nonnegative. + + For floating-point input, the *std* is computed using the same + precision the input has. Depending on the input data, this can cause + the results to be inaccurate, especially for float32 (see example below). + Specifying a higher-accuracy accumulator using the `dtype` keyword can + alleviate this issue. + + Examples + -------- + >>> a = np.array([[1, 2], [3, 4]]) + >>> np.std(a) + 1.1180339887498949 + >>> np.std(a, axis=0) + array([ 1., 1.]) + >>> np.std(a, axis=1) + array([ 0.5, 0.5]) + + In single precision, std() can be inaccurate: + + >>> a = np.zeros((2,512*512), dtype=np.float32) + >>> a[0,:] = 1.0 + >>> a[1,:] = 0.1 + >>> np.std(a) + 0.45172946707416706 + + Computing the standard deviation in float64 is more accurate: + + >>> np.std(a, dtype=np.float64) + 0.44999999925552653 + + """ + if not hasattr(a, "std"): + a = numpypy.array(a) + return a.std() + + +def var(a, axis=None, dtype=None, out=None, ddof=0): + """ + Compute the variance along the specified axis. + + Returns the variance of the array elements, a measure of the spread of a + distribution. The variance is computed for the flattened array by + default, otherwise over the specified axis. + + Parameters + ---------- + a : array_like + Array containing numbers whose variance is desired. If `a` is not an + array, a conversion is attempted. + axis : int, optional + Axis along which the variance is computed. The default is to compute + the variance of the flattened array. + dtype : data-type, optional + Type to use in computing the variance. For arrays of integer type + the default is `float32`; for arrays of float types it is the same as + the array type. + out : ndarray, optional + Alternate output array in which to place the result. It must have + the same shape as the expected output, but the type is cast if + necessary. + ddof : int, optional + "Delta Degrees of Freedom": the divisor used in the calculation is + ``N - ddof``, where ``N`` represents the number of elements. By + default `ddof` is zero. + + Returns + ------- + variance : ndarray, see dtype parameter above + If ``out=None``, returns a new array containing the variance; + otherwise, a reference to the output array is returned. + + See Also + -------- + std : Standard deviation + mean : Average + numpy.doc.ufuncs : Section "Output arguments" + + Notes + ----- + The variance is the average of the squared deviations from the mean, + i.e., ``var = mean(abs(x - x.mean())**2)``. + + The mean is normally calculated as ``x.sum() / N``, where ``N = len(x)``. + If, however, `ddof` is specified, the divisor ``N - ddof`` is used + instead. In standard statistical practice, ``ddof=1`` provides an + unbiased estimator of the variance of a hypothetical infinite population. + ``ddof=0`` provides a maximum likelihood estimate of the variance for + normally distributed variables. + + Note that for complex numbers, the absolute value is taken before + squaring, so that the result is always real and nonnegative. + + For floating-point input, the variance is computed using the same + precision the input has. Depending on the input data, this can cause + the results to be inaccurate, especially for `float32` (see example + below). Specifying a higher-accuracy accumulator using the ``dtype`` + keyword can alleviate this issue. + + Examples + -------- + >>> a = np.array([[1,2],[3,4]]) + >>> np.var(a) + 1.25 + >>> np.var(a,0) + array([ 1., 1.]) + >>> np.var(a,1) + array([ 0.25, 0.25]) + + In single precision, var() can be inaccurate: + + >>> a = np.zeros((2,512*512), dtype=np.float32) + >>> a[0,:] = 1.0 + >>> a[1,:] = 0.1 + >>> np.var(a) + 0.20405951142311096 + + Computing the standard deviation in float64 is more accurate: + + >>> np.var(a, dtype=np.float64) + 0.20249999932997387 + >>> ((1-0.55)**2 + (0.1-0.55)**2)/2 + 0.20250000000000001 + + """ + if not hasattr(a, "var"): + a = numpypy.array(a) + return a.var() diff --git a/lib_pypy/numpypy/test/test_fromnumeric.py b/lib_pypy/numpypy/test/test_fromnumeric.py new file mode 100644 --- /dev/null +++ b/lib_pypy/numpypy/test/test_fromnumeric.py @@ -0,0 +1,109 @@ + +from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest + +class AppTestFromNumeric(BaseNumpyAppTest): + def test_argmax(self): + # tests taken from numpy/core/fromnumeric.py docstring + from numpypy import array, arange, argmax + a = arange(6).reshape((2,3)) + assert argmax(a) == 5 + # assert (argmax(a, axis=0) == array([1, 1, 1])).all() + # assert (argmax(a, axis=1) == array([2, 2])).all() + b = arange(6) + b[1] = 5 + assert argmax(b) == 1 + + def test_argmin(self): + # tests adapted from test_argmax + from numpypy import array, arange, argmin + a = arange(6).reshape((2,3)) + assert argmin(a) == 0 + # assert (argmax(a, axis=0) == array([0, 0, 0])).all() + # assert (argmax(a, axis=1) == array([0, 0])).all() + b = arange(6) + b[1] = 0 + assert argmin(b) == 0 + + def test_shape(self): + # tests taken from numpy/core/fromnumeric.py docstring + from numpypy import array, identity, shape + assert shape(identity(3)) == (3, 3) + assert shape([[1, 2]]) == (1, 2) + assert shape([0]) == (1,) + assert shape(0) == () + # a = array([(1, 2), (3, 4)], dtype=[('x', 'i4'), ('y', 'i4')]) + # assert shape(a) == (2,) + + def test_sum(self): + # tests taken from numpy/core/fromnumeric.py docstring + from numpypy import array, sum, ones + assert sum([0.5, 1.5])== 2.0 + assert sum([[0, 1], [0, 5]]) == 6 + # assert sum([0.5, 0.7, 0.2, 1.5], dtype=int32) == 1 + # assert (sum([[0, 1], [0, 5]], axis=0) == array([0, 6])).all() + # assert (sum([[0, 1], [0, 5]], axis=1) == array([1, 5])).all() + # If the accumulator is too small, overflow occurs: + # assert ones(128, dtype=int8).sum(dtype=int8) == -128 + + def test_amin(self): + # tests taken from numpy/core/fromnumeric.py docstring + from numpypy import array, arange, amin + a = arange(4).reshape((2,2)) + assert amin(a) == 0 + # # Minima along the first axis + # assert (amin(a, axis=0) == array([0, 1])).all() + # # Minima along the second axis + # assert (amin(a, axis=1) == array([0, 2])).all() + # # NaN behaviour + # b = arange(5, dtype=float) + # b[2] = NaN + # assert amin(b) == nan + # assert nanmin(b) == 0.0 + + def test_amax(self): + # tests taken from numpy/core/fromnumeric.py docstring + from numpypy import array, arange, amax + a = arange(4).reshape((2,2)) + assert amax(a) == 3 + # assert (amax(a, axis=0) == array([2, 3])).all() + # assert (amax(a, axis=1) == array([1, 3])).all() + # # NaN behaviour + # b = arange(5, dtype=float) + # b[2] = NaN + # assert amax(b) == nan + # assert nanmax(b) == 4.0 + + def test_alen(self): + # tests taken from numpy/core/fromnumeric.py docstring + from numpypy import array, zeros, alen + a = zeros((7,4,5)) + assert a.shape[0] == 7 + assert alen(a) == 7 + + def test_ndim(self): + # tests taken from numpy/core/fromnumeric.py docstring + from numpypy import array, ndim + assert ndim([[1,2,3],[4,5,6]]) == 2 + assert ndim(array([[1,2,3],[4,5,6]])) == 2 + assert ndim(1) == 0 + + def test_rank(self): + # tests taken from numpy/core/fromnumeric.py docstring + from numpypy import array, rank + assert rank([[1,2,3],[4,5,6]]) == 2 + assert rank(array([[1,2,3],[4,5,6]])) == 2 + assert rank(1) == 0 + + def test_var(self): + from numpypy import array, var + a = array([[1,2],[3,4]]) + assert var(a) == 1.25 + # assert (np.var(a,0) == array([ 1., 1.])).all() + # assert (np.var(a,1) == array([ 0.25, 0.25])).all() + + def test_std(self): + from numpypy import array, std + a = array([[1, 2], [3, 4]]) + assert std(a) == 1.1180339887498949 + # assert (std(a, axis=0) == array([ 1., 1.])).all() + # assert (std(a, axis=1) == array([ 0.5, 0.5]).all() diff --git a/py/_code/code.py b/py/_code/code.py --- a/py/_code/code.py +++ b/py/_code/code.py @@ -164,6 +164,7 @@ # if something: # assume this causes a NameError # # _this_ lines and the one # below we don't want from entry.getsource() + end = min(end, len(source)) for i in range(self.lineno, end): if source[i].rstrip().endswith(':'): end = i + 1 diff --git a/pypy/annotation/binaryop.py b/pypy/annotation/binaryop.py --- a/pypy/annotation/binaryop.py +++ b/pypy/annotation/binaryop.py @@ -252,7 +252,26 @@ # unsignedness is considered a rare and contagious disease def union((int1, int2)): - knowntype = rarithmetic.compute_restype(int1.knowntype, int2.knowntype) + if int1.unsigned == int2.unsigned: + knowntype = rarithmetic.compute_restype(int1.knowntype, int2.knowntype) + else: + t1 = int1.knowntype + if t1 is bool: + t1 = int + t2 = int2.knowntype + if t2 is bool: + t2 = int + + if t2 is int: + if int2.nonneg == False: + raise UnionError, "Merging %s and a possibly negative int is not allowed" % t1 + knowntype = t1 + elif t1 is int: + if int1.nonneg == False: + raise UnionError, "Merging %s and a possibly negative int is not allowed" % t2 + knowntype = t2 + else: + raise UnionError, "Merging these types (%s, %s) is not supported" % (t1, t2) return SomeInteger(nonneg=int1.nonneg and int2.nonneg, knowntype=knowntype) diff --git a/pypy/annotation/description.py b/pypy/annotation/description.py --- a/pypy/annotation/description.py +++ b/pypy/annotation/description.py @@ -180,7 +180,12 @@ if name is None: name = pyobj.func_name if signature is None: - signature = cpython_code_signature(pyobj.func_code) + if hasattr(pyobj, '_generator_next_method_of_'): + from pypy.interpreter.argument import Signature + signature = Signature(['entry']) # haaaaaack + defaults = () + else: + signature = cpython_code_signature(pyobj.func_code) if defaults is None: defaults = pyobj.func_defaults self.name = name @@ -252,7 +257,8 @@ try: inputcells = args.match_signature(signature, defs_s) except ArgErr, e: - raise TypeError, "signature mismatch: %s" % e.getmsg(self.name) + raise TypeError("signature mismatch: %s() %s" % + (self.name, e.getmsg())) return inputcells def specialize(self, inputcells, op=None): diff --git a/pypy/annotation/model.py b/pypy/annotation/model.py --- a/pypy/annotation/model.py +++ b/pypy/annotation/model.py @@ -591,13 +591,11 @@ immutable = True def __init__(self, method): self.method = method - -NUMBER = object() + annotation_to_ll_map = [ (SomeSingleFloat(), lltype.SingleFloat), (s_None, lltype.Void), # also matches SomeImpossibleValue() (s_Bool, lltype.Bool), - (SomeInteger(knowntype=r_ulonglong), NUMBER), (SomeFloat(), lltype.Float), (SomeLongFloat(), lltype.LongFloat), (SomeChar(), lltype.Char), @@ -623,10 +621,11 @@ return lltype.Ptr(p.PARENTTYPE) if isinstance(s_val, SomePtr): return s_val.ll_ptrtype + if type(s_val) is SomeInteger: + return lltype.build_number(None, s_val.knowntype) + for witness, T in annotation_to_ll_map: if witness.contains(s_val): - if T is NUMBER: - return lltype.build_number(None, s_val.knowntype) return T if info is None: info = '' @@ -635,7 +634,7 @@ raise ValueError("%sshould return a low-level type,\ngot instead %r" % ( info, s_val)) -ll_to_annotation_map = dict([(ll, ann) for ann, ll in annotation_to_ll_map if ll is not NUMBER]) +ll_to_annotation_map = dict([(ll, ann) for ann, ll in annotation_to_ll_map]) def lltype_to_annotation(T): try: diff --git a/pypy/annotation/specialize.py b/pypy/annotation/specialize.py --- a/pypy/annotation/specialize.py +++ b/pypy/annotation/specialize.py @@ -36,9 +36,7 @@ newtup = SpaceOperation('newtuple', starargs, argscopy[-1]) newstartblock.operations.append(newtup) newstartblock.closeblock(Link(argscopy, graph.startblock)) - graph.startblock.isstartblock = False graph.startblock = newstartblock - newstartblock.isstartblock = True argnames = argnames + ['.star%d' % i for i in range(nb_extra_args)] graph.signature = Signature(argnames) # note that we can mostly ignore defaults: if nb_extra_args > 0, diff --git a/pypy/annotation/test/test_annrpython.py b/pypy/annotation/test/test_annrpython.py --- a/pypy/annotation/test/test_annrpython.py +++ b/pypy/annotation/test/test_annrpython.py @@ -856,6 +856,46 @@ py.test.raises(Exception, a.build_types, f, []) # if you want to get a r_uint, you have to be explicit about it + def test_add_different_ints(self): + def f(a, b): + return a + b + a = self.RPythonAnnotator() + py.test.raises(Exception, a.build_types, f, [r_uint, int]) + + def test_merge_different_ints(self): + def f(a, b): + if a: + c = a + else: + c = b + return c + a = self.RPythonAnnotator() + py.test.raises(Exception, a.build_types, f, [r_uint, int]) + + def test_merge_ruint_zero(self): + def f(a): + if a: + c = a + else: + c = 0 + return c + a = self.RPythonAnnotator() + s = a.build_types(f, [r_uint]) + assert s == annmodel.SomeInteger(nonneg = True, unsigned = True) + + def test_merge_ruint_nonneg_signed(self): + def f(a, b): + if a: + c = a + else: + assert b >= 0 + c = b + return c + a = self.RPythonAnnotator() + s = a.build_types(f, [r_uint, int]) + assert s == annmodel.SomeInteger(nonneg = True, unsigned = True) + + def test_prebuilt_long_that_is_not_too_long(self): small_constant = 12L def f(): @@ -3029,7 +3069,7 @@ if g(x, y): g(x, r_uint(y)) a = self.RPythonAnnotator() - a.build_types(f, [int, int]) + py.test.raises(Exception, a.build_types, f, [int, int]) def test_compare_with_zero(self): def g(): diff --git a/pypy/bin/checkmodule.py b/pypy/bin/checkmodule.py --- a/pypy/bin/checkmodule.py +++ b/pypy/bin/checkmodule.py @@ -1,43 +1,45 @@ #! /usr/bin/env python """ -Usage: checkmodule.py [-b backend] +Usage: checkmodule.py -Compiles the PyPy extension module from pypy/module// -into a fake program which does nothing. Useful for testing whether a -modules compiles without doing a full translation. Default backend is cli. - -WARNING: this is still incomplete: there are chances that the -compilation fails with strange errors not due to the module. If a -module is known to compile during a translation but don't pass -checkmodule.py, please report the bug (or, better, correct it :-). +Check annotation and rtyping of the PyPy extension module from +pypy/module//. Useful for testing whether a +modules compiles without doing a full translation. """ import autopath -import sys +import sys, os from pypy.objspace.fake.checkmodule import checkmodule def main(argv): - try: - assert len(argv) in (2, 4) - if len(argv) == 2: - backend = 'cli' - modname = argv[1] - if modname in ('-h', '--help'): - print >> sys.stderr, __doc__ - sys.exit(0) - if modname.startswith('-'): - print >> sys.stderr, "Bad command line" - print >> sys.stderr, __doc__ - sys.exit(1) - else: - _, b, backend, modname = argv - assert b == '-b' - except AssertionError: + if len(argv) != 2: print >> sys.stderr, __doc__ sys.exit(2) + modname = argv[1] + if modname in ('-h', '--help'): + print >> sys.stderr, __doc__ + sys.exit(0) + if modname.startswith('-'): + print >> sys.stderr, "Bad command line" + print >> sys.stderr, __doc__ + sys.exit(1) + if os.path.sep in modname: + if os.path.basename(modname) == '': + modname = os.path.dirname(modname) + if os.path.basename(os.path.dirname(modname)) != 'module': + print >> sys.stderr, "Must give '../module/xxx', or just 'xxx'." + sys.exit(1) + modname = os.path.basename(modname) + try: + checkmodule(modname) + except Exception, e: + import traceback, pdb + traceback.print_exc() + pdb.post_mortem(sys.exc_info()[2]) + return 1 else: - checkmodule(modname, backend, interactive=True) - print 'Module compiled succesfully' + print 'Passed.' + return 0 if __name__ == '__main__': - main(sys.argv) + sys.exit(main(sys.argv)) diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py --- a/pypy/config/pypyoption.py +++ b/pypy/config/pypyoption.py @@ -252,6 +252,10 @@ "use small tuples", default=False), + BoolOption("withspecialisedtuple", + "use specialised tuples", + default=False), + BoolOption("withrope", "use ropes as the string implementation", default=False, requires=[("objspace.std.withstrslice", False), @@ -366,6 +370,7 @@ config.objspace.std.suggest(getattributeshortcut=True) config.objspace.std.suggest(newshortcut=True) config.objspace.std.suggest(withidentitydict=True) + config.objspace.std.suggest(withspecialisedtuple=True) #if not IS_64_BITS: # config.objspace.std.suggest(withsmalllong=True) diff --git a/pypy/conftest.py b/pypy/conftest.py --- a/pypy/conftest.py +++ b/pypy/conftest.py @@ -496,6 +496,17 @@ def setup(self): super(AppClassCollector, self).setup() cls = self.obj + # + # + for name in dir(cls): + if name.startswith('test_'): + func = getattr(cls, name, None) + code = getattr(func, 'func_code', None) + if code and code.co_flags & 32: + raise AssertionError("unsupported: %r is a generator " + "app-level test method" % (name,)) + # + # space = cls.space clsname = cls.__name__ if self.config.option.runappdirect: diff --git a/pypy/doc/Makefile b/pypy/doc/Makefile --- a/pypy/doc/Makefile +++ b/pypy/doc/Makefile @@ -12,7 +12,7 @@ PAPEROPT_letter = -D latex_paper_size=letter ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . -.PHONY: help clean html dirhtml pickle json htmlhelp qthelp latex changes linkcheck doctest +.PHONY: help clean html dirhtml pickle json htmlhelp qthelp latex man changes linkcheck doctest help: @echo "Please use \`make ' where is one of" @@ -23,6 +23,7 @@ @echo " htmlhelp to make HTML files and a HTML help project" @echo " qthelp to make HTML files and a qthelp project" @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " man to make manual pages" @echo " changes to make an overview of all changed/added/deprecated items" @echo " linkcheck to check all external links for integrity" @echo " doctest to run all doctests embedded in the documentation (if enabled)" @@ -79,6 +80,11 @@ @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \ "run these through (pdf)latex." +man: + $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man + @echo + @echo "Build finished. The manual pages are in $(BUILDDIR)/man" + changes: python config/generate.py $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes diff --git a/pypy/doc/coding-guide.rst b/pypy/doc/coding-guide.rst --- a/pypy/doc/coding-guide.rst +++ b/pypy/doc/coding-guide.rst @@ -175,15 +175,15 @@ RPython ================= -RPython Definition, not ------------------------ +RPython Definition +------------------ -The list and exact details of the "RPython" restrictions are a somewhat -evolving topic. In particular, we have no formal language definition -as we find it more practical to discuss and evolve the set of -restrictions while working on the whole program analysis. If you -have any questions about the restrictions below then please feel -free to mail us at pypy-dev at codespeak net. +RPython is a restricted subset of Python that is amenable to static analysis. +Although there are additions to the language and some things might surprisingly +work, this is a rough list of restrictions that should be considered. Note +that there are tons of special cased restrictions that you'll encounter +as you go. The exact definition is "RPython is everything that our translation +toolchain can accept" :) .. _`wrapped object`: coding-guide.html#wrapping-rules @@ -198,7 +198,7 @@ contain both a string and a int must be avoided. It is allowed to mix None (basically with the role of a null pointer) with many other types: `wrapped objects`, class instances, lists, dicts, strings, etc. - but *not* with int and floats. + but *not* with int, floats or tuples. **constants** @@ -209,9 +209,12 @@ have this restriction, so if you need mutable global state, store it in the attributes of some prebuilt singleton instance. + + **control structures** - all allowed but yield, ``for`` loops restricted to builtin types + all allowed, ``for`` loops restricted to builtin types, generators + very restricted. **range** @@ -226,7 +229,8 @@ **generators** - generators are not supported. + generators are supported, but their exact scope is very limited. you can't + merge two different generator in one control point. **exceptions** @@ -245,22 +249,27 @@ **strings** - a lot of, but not all string methods are supported. Indexes can be + a lot of, but not all string methods are supported and those that are + supported, not necesarilly accept all arguments. Indexes can be negative. In case they are not, then you get slightly more efficient code if the translator can prove that they are non-negative. When slicing a string it is necessary to prove that the slice start and - stop indexes are non-negative. + stop indexes are non-negative. There is no implicit str-to-unicode cast + anywhere. **tuples** no variable-length tuples; use them to store or return pairs or n-tuples of - values. Each combination of types for elements and length constitute a separate - and not mixable type. + values. Each combination of types for elements and length constitute + a separate and not mixable type. **lists** lists are used as an allocated array. Lists are over-allocated, so list.append() - is reasonably fast. Negative or out-of-bound indexes are only allowed for the + is reasonably fast. However, if you use a fixed-size list, the code + is more efficient. Annotator can figure out most of the time that your + list is fixed-size, even when you use list comprehension. + Negative or out-of-bound indexes are only allowed for the most common operations, as follows: - *indexing*: @@ -287,16 +296,14 @@ **dicts** - dicts with a unique key type only, provided it is hashable. - String keys have been the only allowed key types for a while, but this was generalized. - After some re-optimization, - the implementation could safely decide that all string dict keys should be interned. + dicts with a unique key type only, provided it is hashable. Custom + hash functions and custom equality will not be honored. + Use ``pypy.rlib.objectmodel.r_dict`` for custom hash functions. **list comprehensions** - may be used to create allocated, initialized arrays. - After list over-allocation was introduced, there is no longer any restriction. + May be used to create allocated, initialized arrays. **functions** @@ -334,9 +341,8 @@ **objects** - in PyPy, wrapped objects are borrowed from the object space. Just like - in CPython, code that needs e.g. a dictionary can use a wrapped dict - and the object space operations on it. + Normal rules apply. Special methods are not honoured, except ``__init__`` and + ``__del__``. This layout makes the number of types to take care about quite limited. diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py --- a/pypy/doc/conf.py +++ b/pypy/doc/conf.py @@ -45,9 +45,9 @@ # built documents. # # The short X.Y version. -version = '1.6' +version = '1.7' # The full version, including alpha/beta/rc tags. -release = '1.6' +release = '1.7' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -197,3 +197,10 @@ # Example configuration for intersphinx: refer to the Python standard library. intersphinx_mapping = {'http://docs.python.org/': None} +# -- Options for manpage output------------------------------------------------- + +man_pages = [ + ('man/pypy.1', 'pypy', + u'fast, compliant alternative implementation of the Python language', + u'The PyPy Project', 1) +] diff --git a/pypy/doc/config/objspace.std.withspecialisedtuple.txt b/pypy/doc/config/objspace.std.withspecialisedtuple.txt new file mode 100644 --- /dev/null +++ b/pypy/doc/config/objspace.std.withspecialisedtuple.txt @@ -0,0 +1,3 @@ +Use "specialized tuples", a custom implementation for some common kinds +of tuples. Currently limited to tuples of length 2, in three variants: +(int, int), (float, float), and a generic (object, object). diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst --- a/pypy/doc/cpython_differences.rst +++ b/pypy/doc/cpython_differences.rst @@ -304,5 +304,14 @@ never a dictionary as it sometimes is in CPython. Assigning to ``__builtins__`` has no effect. +* directly calling the internal magic methods of a few built-in types + with invalid arguments may have a slightly different result. For + example, ``[].__add__(None)`` and ``(2).__add__(None)`` both return + ``NotImplemented`` on PyPy; on CPython, only the later does, and the + former raises ``TypeError``. (Of course, ``[]+None`` and ``2+None`` + both raise ``TypeError`` everywhere.) This difference is an + implementation detail that shows up because of internal C-level slots + that PyPy does not have. + .. include:: _ref.txt diff --git a/pypy/doc/extradoc.rst b/pypy/doc/extradoc.rst --- a/pypy/doc/extradoc.rst +++ b/pypy/doc/extradoc.rst @@ -8,6 +8,9 @@ *Articles about PyPy published so far, most recent first:* (bibtex_ file) +* `Runtime Feedback in a Meta-Tracing JIT for Efficient Dynamic Languages`_, + C.F. Bolz, A. Cuni, M. Fijalkowski, M. Leuschel, S. Pedroni, A. Rigo + * `Allocation Removal by Partial Evaluation in a Tracing JIT`_, C.F. Bolz, A. Cuni, M. Fijalkowski, M. Leuschel, S. Pedroni, A. Rigo @@ -50,6 +53,9 @@ *Other research using PyPy (as far as we know it):* +* `Hardware Transactional Memory Support for Lightweight Dynamic Language Evolution`_, + N. Riley and C. Zilles + * `PyGirl: Generating Whole-System VMs from High-Level Prototypes using PyPy`_, C. Bruni and T. Verwaest @@ -65,6 +71,7 @@ .. _bibtex: https://bitbucket.org/pypy/extradoc/raw/tip/talk/bibtex.bib +.. _`Runtime Feedback in a Meta-Tracing JIT for Efficient Dynamic Languages`: https://bitbucket.org/pypy/extradoc/raw/extradoc/talk/icooolps2011/jit-hints.pdf .. _`Allocation Removal by Partial Evaluation in a Tracing JIT`: http://codespeak.net/svn/pypy/extradoc/talk/pepm2011/bolz-allocation-removal.pdf .. _`Towards a Jitting VM for Prolog Execution`: http://www.stups.uni-duesseldorf.de/publications/bolz-prolog-jit.pdf .. _`High performance implementation of Python for CLI/.NET with JIT compiler generation for dynamic languages`: http://buildbot.pypy.org/misc/antocuni-thesis.pdf @@ -74,6 +81,7 @@ .. _`Automatic JIT Compiler Generation with Runtime Partial Evaluation`: http://www.stups.uni-duesseldorf.de/thesis/final-master.pdf .. _`RPython: A Step towards Reconciling Dynamically and Statically Typed OO Languages`: http://www.disi.unige.it/person/AnconaD/papers/Recent_abstracts.html#AACM-DLS07 .. _`EU Reports`: index-report.html +.. _`Hardware Transactional Memory Support for Lightweight Dynamic Language Evolution`: http://sabi.net/nriley/pubs/dls6-riley.pdf .. _`PyGirl: Generating Whole-System VMs from High-Level Prototypes using PyPy`: http://scg.unibe.ch/archive/papers/Brun09cPyGirl.pdf .. _`Representation-Based Just-in-Time Specialization and the Psyco Prototype for Python`: http://psyco.sourceforge.net/psyco-pepm-a.ps.gz .. _`Back to the Future in One Week -- Implementing a Smalltalk VM in PyPy`: http://dx.doi.org/10.1007/978-3-540-89275-5_7 diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst --- a/pypy/doc/faq.rst +++ b/pypy/doc/faq.rst @@ -112,10 +112,32 @@ You might be interested in our `benchmarking site`_ and our `jit documentation`_. +Note that the JIT has a very high warm-up cost, meaning that the +programs are slow at the beginning. If you want to compare the timings +with CPython, even relatively simple programs need to run *at least* one +second, preferrably at least a few seconds. Large, complicated programs +need even more time to warm-up the JIT. + .. _`benchmarking site`: http://speed.pypy.org .. _`jit documentation`: jit/index.html +--------------------------------------------------------------- +Couldn't the JIT dump and reload already-compiled machine code? +--------------------------------------------------------------- + +No, we found no way of doing that. The JIT generates machine code +containing a large number of constant addresses --- constant at the time +the machine code is written. The vast majority is probably not at all +constants that you find in the executable, with a nice link name. E.g. +the addresses of Python classes are used all the time, but Python +classes don't come statically from the executable; they are created anew +every time you restart your program. This makes saving and reloading +machine code completely impossible without some very advanced way of +mapping addresses in the old (now-dead) process to addresses in the new +process, including checking that all the previous assumptions about the +(now-dead) object are still true about the new object. + .. _`prolog and javascript`: diff --git a/pypy/doc/index.rst b/pypy/doc/index.rst --- a/pypy/doc/index.rst +++ b/pypy/doc/index.rst @@ -15,7 +15,7 @@ * `FAQ`_: some frequently asked questions. -* `Release 1.6`_: the latest official release +* `Release 1.7`_: the latest official release * `PyPy Blog`_: news and status info about PyPy @@ -75,7 +75,7 @@ .. _`Getting Started`: getting-started.html .. _`Papers`: extradoc.html .. _`Videos`: video-index.html -.. _`Release 1.6`: http://pypy.org/download.html +.. _`Release 1.7`: http://pypy.org/download.html .. _`speed.pypy.org`: http://speed.pypy.org .. _`RPython toolchain`: translation.html .. _`potential project ideas`: project-ideas.html @@ -120,9 +120,9 @@ Windows, on top of .NET, and on top of Java. To dig into PyPy it is recommended to try out the current Mercurial default branch, which is always working or mostly working, -instead of the latest release, which is `1.6`__. +instead of the latest release, which is `1.7`__. -.. __: release-1.6.0.html +.. __: release-1.7.0.html PyPy is mainly developed on Linux and Mac OS X. Windows is supported, but platform-specific bugs tend to take longer before we notice and fix diff --git a/pypy/doc/man/pypy.1.rst b/pypy/doc/man/pypy.1.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/man/pypy.1.rst @@ -0,0 +1,90 @@ +====== + pypy +====== + +SYNOPSIS +======== + +``pypy`` [*options*] +[``-c`` *cmd*\ \|\ ``-m`` *mod*\ \|\ *file.py*\ \|\ ``-``\ ] +[*arg*\ ...] + +OPTIONS +======= + +-i + Inspect interactively after running script. + +-O + Dummy optimization flag for compatibility with C Python. + +-c *cmd* + Program passed in as CMD (terminates option list). + +-S + Do not ``import site`` on initialization. + +-u + Unbuffered binary ``stdout`` and ``stderr``. + +-h, --help + Show a help message and exit. + +-m *mod* + Library module to be run as a script (terminates option list). + +-W *arg* + Warning control (*arg* is *action*:*message*:*category*:*module*:*lineno*). + +-E + Ignore environment variables (such as ``PYTHONPATH``). + +--version + Print the PyPy version. + +--info + Print translation information about this PyPy executable. + +--jit *arg* + Low level JIT parameters. Format is + *arg*\ ``=``\ *value*\ [``,``\ *arg*\ ``=``\ *value*\ ...] + + ``off`` + Disable the JIT. + + ``threshold=``\ *value* + Number of times a loop has to run for it to become hot. + + ``function_threshold=``\ *value* + Number of times a function must run for it to become traced from + start. + + ``inlining=``\ *value* + Inline python functions or not (``1``/``0``). + + ``loop_longevity=``\ *value* + A parameter controlling how long loops will be kept before being + freed, an estimate. + + ``max_retrace_guards=``\ *value* + Number of extra guards a retrace can cause. + + ``retrace_limit=``\ *value* + How many times we can try retracing before giving up. + + ``trace_eagerness=``\ *value* + Number of times a guard has to fail before we start compiling a + bridge. + + ``trace_limit=``\ *value* + Number of recorded operations before we abort tracing with + ``ABORT_TRACE_TOO_LONG``. + + ``enable_opts=``\ *value* + Optimizations to enabled or ``all``. + Warning, this option is dangerous, and should be avoided. + +SEE ALSO +======== + +**python**\ (1) diff --git a/pypy/doc/tool/makecontributor.py b/pypy/doc/tool/makecontributor.py deleted file mode 100644 --- a/pypy/doc/tool/makecontributor.py +++ /dev/null @@ -1,47 +0,0 @@ -""" - -generates a contributor list - -""" -import py - -# this file is useless, use the following commandline instead: -# hg churn -c -t "{author}" | sed -e 's/ <.*//' - -try: - path = py.std.sys.argv[1] -except IndexError: - print "usage: %s ROOTPATH" %(py.std.sys.argv[0]) - raise SystemExit, 1 - -d = {} - -for logentry in py.path.svnwc(path).log(): - a = logentry.author - if a in d: - d[a] += 1 - else: - d[a] = 1 - -items = d.items() -items.sort(lambda x,y: -cmp(x[1], y[1])) - -import uconf # http://codespeak.net/svn/uconf/dist/uconf - -# Authors that don't want to be listed -excluded = set("anna gintas ignas".split()) -cutoff = 5 # cutoff for authors in the LICENSE file -mark = False -for author, count in items: - if author in excluded: - continue - user = uconf.system.User(author) - try: - realname = user.realname.strip() - except KeyError: - realname = author - if not mark and count < cutoff: - mark = True - print '-'*60 - print " ", realname - #print count, " ", author diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py --- a/pypy/interpreter/argument.py +++ b/pypy/interpreter/argument.py @@ -428,8 +428,8 @@ return self._match_signature(w_firstarg, scope_w, signature, defaults_w, 0) except ArgErr, e: - raise OperationError(self.space.w_TypeError, - self.space.wrap(e.getmsg(fnname))) + raise operationerrfmt(self.space.w_TypeError, + "%s() %s", fnname, e.getmsg()) def _parse(self, w_firstarg, signature, defaults_w, blindargs=0): """Parse args and kwargs according to the signature of a code object, @@ -450,8 +450,8 @@ try: return self._parse(w_firstarg, signature, defaults_w, blindargs) except ArgErr, e: - raise OperationError(self.space.w_TypeError, - self.space.wrap(e.getmsg(fnname))) + raise operationerrfmt(self.space.w_TypeError, + "%s() %s", fnname, e.getmsg()) @staticmethod def frompacked(space, w_args=None, w_kwds=None): @@ -626,7 +626,7 @@ class ArgErr(Exception): - def getmsg(self, fnname): + def getmsg(self): raise NotImplementedError class ArgErrCount(ArgErr): @@ -642,11 +642,10 @@ self.num_args = got_nargs self.num_kwds = nkwds - def getmsg(self, fnname): + def getmsg(self): n = self.expected_nargs if n == 0: - msg = "%s() takes no arguments (%d given)" % ( - fnname, + msg = "takes no arguments (%d given)" % ( self.num_args + self.num_kwds) else: defcount = self.num_defaults @@ -672,8 +671,7 @@ msg2 = " non-keyword" else: msg2 = "" - msg = "%s() takes %s %d%s argument%s (%d given)" % ( - fnname, + msg = "takes %s %d%s argument%s (%d given)" % ( msg1, n, msg2, @@ -686,9 +684,8 @@ def __init__(self, argname): self.argname = argname - def getmsg(self, fnname): - msg = "%s() got multiple values for keyword argument '%s'" % ( - fnname, + def getmsg(self): + msg = "got multiple values for keyword argument '%s'" % ( self.argname) return msg @@ -722,13 +719,11 @@ break self.kwd_name = name - def getmsg(self, fnname): + def getmsg(self): if self.num_kwds == 1: - msg = "%s() got an unexpected keyword argument '%s'" % ( - fnname, + msg = "got an unexpected keyword argument '%s'" % ( self.kwd_name) else: - msg = "%s() got %d unexpected keyword arguments" % ( - fnname, + msg = "got %d unexpected keyword arguments" % ( self.num_kwds) return msg diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py --- a/pypy/interpreter/astcompiler/ast.py +++ b/pypy/interpreter/astcompiler/ast.py @@ -51,6 +51,24 @@ space.setattr(self, w_name, space.getitem(w_state, w_name)) + def missing_field(self, space, required, host): + "Find which required field is missing." + state = self.initialization_state + for i in range(len(required)): + if (state >> i) & 1: + continue # field is present + missing = required[i] + if missing is None: + continue # field is optional + w_obj = self.getdictvalue(space, missing) + if w_obj is None: + err = "required field \"%s\" missing from %s" + raise operationerrfmt(space.w_TypeError, err, missing, host) + else: + err = "incorrect type for field \"%s\" in %s" + raise operationerrfmt(space.w_TypeError, err, missing, host) + raise AssertionError("should not reach here") + class NodeVisitorNotImplemented(Exception): pass @@ -94,17 +112,6 @@ ) -def missing_field(space, state, required, host): - "Find which required field is missing." - for i in range(len(required)): - if not (state >> i) & 1: - missing = required[i] - if missing is not None: - err = "required field \"%s\" missing from %s" - err = err % (missing, host) - w_err = space.wrap(err) - raise OperationError(space.w_TypeError, w_err) - raise AssertionError("should not reach here") class mod(AST): @@ -112,7 +119,6 @@ class Module(mod): - def __init__(self, body): self.body = body self.w_body = None @@ -128,7 +134,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 1: - missing_field(space, self.initialization_state, ['body'], 'Module') + self.missing_field(space, ['body'], 'Module') else: pass w_list = self.w_body @@ -145,7 +151,6 @@ class Interactive(mod): - def __init__(self, body): self.body = body self.w_body = None @@ -161,7 +166,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 1: - missing_field(space, self.initialization_state, ['body'], 'Interactive') + self.missing_field(space, ['body'], 'Interactive') else: pass w_list = self.w_body @@ -178,7 +183,6 @@ class Expression(mod): - def __init__(self, body): self.body = body self.initialization_state = 1 @@ -192,7 +196,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 1: - missing_field(space, self.initialization_state, ['body'], 'Expression') + self.missing_field(space, ['body'], 'Expression') else: pass self.body.sync_app_attrs(space) @@ -200,7 +204,6 @@ class Suite(mod): - def __init__(self, body): self.body = body self.w_body = None @@ -216,7 +219,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 1: - missing_field(space, self.initialization_state, ['body'], 'Suite') + self.missing_field(space, ['body'], 'Suite') else: pass w_list = self.w_body @@ -232,15 +235,13 @@ class stmt(AST): + def __init__(self, lineno, col_offset): self.lineno = lineno self.col_offset = col_offset class FunctionDef(stmt): - _lineno_mask = 16 - _col_offset_mask = 32 - def __init__(self, name, args, body, decorator_list, lineno, col_offset): self.name = name self.args = args @@ -264,7 +265,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 63: - missing_field(space, self.initialization_state, ['name', 'args', 'body', 'decorator_list', 'lineno', 'col_offset'], 'FunctionDef') + self.missing_field(space, ['lineno', 'col_offset', 'name', 'args', 'body', 'decorator_list'], 'FunctionDef') else: pass self.args.sync_app_attrs(space) @@ -292,9 +293,6 @@ class ClassDef(stmt): - _lineno_mask = 16 - _col_offset_mask = 32 - def __init__(self, name, bases, body, decorator_list, lineno, col_offset): self.name = name self.bases = bases @@ -320,7 +318,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 63: - missing_field(space, self.initialization_state, ['name', 'bases', 'body', 'decorator_list', 'lineno', 'col_offset'], 'ClassDef') + self.missing_field(space, ['lineno', 'col_offset', 'name', 'bases', 'body', 'decorator_list'], 'ClassDef') else: pass w_list = self.w_bases @@ -357,9 +355,6 @@ class Return(stmt): - _lineno_mask = 2 - _col_offset_mask = 4 - def __init__(self, value, lineno, col_offset): self.value = value stmt.__init__(self, lineno, col_offset) @@ -374,10 +369,10 @@ return visitor.visit_Return(self) def sync_app_attrs(self, space): - if (self.initialization_state & ~1) ^ 6: - missing_field(space, self.initialization_state, [None, 'lineno', 'col_offset'], 'Return') + if (self.initialization_state & ~4) ^ 3: + self.missing_field(space, ['lineno', 'col_offset', None], 'Return') else: - if not self.initialization_state & 1: + if not self.initialization_state & 4: self.value = None if self.value: self.value.sync_app_attrs(space) @@ -385,9 +380,6 @@ class Delete(stmt): - _lineno_mask = 2 - _col_offset_mask = 4 - def __init__(self, targets, lineno, col_offset): self.targets = targets self.w_targets = None @@ -404,7 +396,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 7: - missing_field(space, self.initialization_state, ['targets', 'lineno', 'col_offset'], 'Delete') + self.missing_field(space, ['lineno', 'col_offset', 'targets'], 'Delete') else: pass w_list = self.w_targets @@ -421,9 +413,6 @@ class Assign(stmt): - _lineno_mask = 4 - _col_offset_mask = 8 - def __init__(self, targets, value, lineno, col_offset): self.targets = targets self.w_targets = None @@ -442,7 +431,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 15: - missing_field(space, self.initialization_state, ['targets', 'value', 'lineno', 'col_offset'], 'Assign') + self.missing_field(space, ['lineno', 'col_offset', 'targets', 'value'], 'Assign') else: pass w_list = self.w_targets @@ -460,9 +449,6 @@ class AugAssign(stmt): - _lineno_mask = 8 - _col_offset_mask = 16 - def __init__(self, target, op, value, lineno, col_offset): self.target = target self.op = op @@ -480,7 +466,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 31: - missing_field(space, self.initialization_state, ['target', 'op', 'value', 'lineno', 'col_offset'], 'AugAssign') + self.missing_field(space, ['lineno', 'col_offset', 'target', 'op', 'value'], 'AugAssign') else: pass self.target.sync_app_attrs(space) @@ -489,9 +475,6 @@ class Print(stmt): - _lineno_mask = 8 - _col_offset_mask = 16 - def __init__(self, dest, values, nl, lineno, col_offset): self.dest = dest self.values = values @@ -511,10 +494,10 @@ return visitor.visit_Print(self) def sync_app_attrs(self, space): - if (self.initialization_state & ~1) ^ 30: - missing_field(space, self.initialization_state, [None, 'values', 'nl', 'lineno', 'col_offset'], 'Print') + if (self.initialization_state & ~4) ^ 27: + self.missing_field(space, ['lineno', 'col_offset', None, 'values', 'nl'], 'Print') else: - if not self.initialization_state & 1: + if not self.initialization_state & 4: self.dest = None if self.dest: self.dest.sync_app_attrs(space) @@ -532,9 +515,6 @@ class For(stmt): - _lineno_mask = 16 - _col_offset_mask = 32 - def __init__(self, target, iter, body, orelse, lineno, col_offset): self.target = target self.iter = iter @@ -559,7 +539,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 63: - missing_field(space, self.initialization_state, ['target', 'iter', 'body', 'orelse', 'lineno', 'col_offset'], 'For') + self.missing_field(space, ['lineno', 'col_offset', 'target', 'iter', 'body', 'orelse'], 'For') else: pass self.target.sync_app_attrs(space) @@ -588,9 +568,6 @@ class While(stmt): - _lineno_mask = 8 - _col_offset_mask = 16 - def __init__(self, test, body, orelse, lineno, col_offset): self.test = test self.body = body @@ -613,7 +590,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 31: - missing_field(space, self.initialization_state, ['test', 'body', 'orelse', 'lineno', 'col_offset'], 'While') + self.missing_field(space, ['lineno', 'col_offset', 'test', 'body', 'orelse'], 'While') else: pass self.test.sync_app_attrs(space) @@ -641,9 +618,6 @@ class If(stmt): - _lineno_mask = 8 - _col_offset_mask = 16 - def __init__(self, test, body, orelse, lineno, col_offset): self.test = test self.body = body @@ -666,7 +640,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 31: - missing_field(space, self.initialization_state, ['test', 'body', 'orelse', 'lineno', 'col_offset'], 'If') + self.missing_field(space, ['lineno', 'col_offset', 'test', 'body', 'orelse'], 'If') else: pass self.test.sync_app_attrs(space) @@ -694,9 +668,6 @@ class With(stmt): - _lineno_mask = 8 - _col_offset_mask = 16 - def __init__(self, context_expr, optional_vars, body, lineno, col_offset): self.context_expr = context_expr self.optional_vars = optional_vars @@ -717,10 +688,10 @@ return visitor.visit_With(self) def sync_app_attrs(self, space): - if (self.initialization_state & ~2) ^ 29: - missing_field(space, self.initialization_state, ['context_expr', None, 'body', 'lineno', 'col_offset'], 'With') + if (self.initialization_state & ~8) ^ 23: + self.missing_field(space, ['lineno', 'col_offset', 'context_expr', None, 'body'], 'With') else: - if not self.initialization_state & 2: + if not self.initialization_state & 8: self.optional_vars = None self.context_expr.sync_app_attrs(space) if self.optional_vars: @@ -739,9 +710,6 @@ class Raise(stmt): - _lineno_mask = 8 - _col_offset_mask = 16 - def __init__(self, type, inst, tback, lineno, col_offset): self.type = type self.inst = inst @@ -762,14 +730,14 @@ return visitor.visit_Raise(self) def sync_app_attrs(self, space): - if (self.initialization_state & ~7) ^ 24: - missing_field(space, self.initialization_state, [None, None, None, 'lineno', 'col_offset'], 'Raise') + if (self.initialization_state & ~28) ^ 3: + self.missing_field(space, ['lineno', 'col_offset', None, None, None], 'Raise') else: - if not self.initialization_state & 1: + if not self.initialization_state & 4: self.type = None - if not self.initialization_state & 2: + if not self.initialization_state & 8: self.inst = None - if not self.initialization_state & 4: + if not self.initialization_state & 16: self.tback = None if self.type: self.type.sync_app_attrs(space) @@ -781,9 +749,6 @@ class TryExcept(stmt): - _lineno_mask = 8 - _col_offset_mask = 16 - def __init__(self, body, handlers, orelse, lineno, col_offset): self.body = body self.w_body = None @@ -808,7 +773,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 31: - missing_field(space, self.initialization_state, ['body', 'handlers', 'orelse', 'lineno', 'col_offset'], 'TryExcept') + self.missing_field(space, ['lineno', 'col_offset', 'body', 'handlers', 'orelse'], 'TryExcept') else: pass w_list = self.w_body @@ -845,9 +810,6 @@ class TryFinally(stmt): - _lineno_mask = 4 - _col_offset_mask = 8 - def __init__(self, body, finalbody, lineno, col_offset): self.body = body self.w_body = None @@ -868,7 +830,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 15: - missing_field(space, self.initialization_state, ['body', 'finalbody', 'lineno', 'col_offset'], 'TryFinally') + self.missing_field(space, ['lineno', 'col_offset', 'body', 'finalbody'], 'TryFinally') else: pass w_list = self.w_body @@ -895,9 +857,6 @@ class Assert(stmt): - _lineno_mask = 4 - _col_offset_mask = 8 - def __init__(self, test, msg, lineno, col_offset): self.test = test self.msg = msg @@ -914,10 +873,10 @@ return visitor.visit_Assert(self) def sync_app_attrs(self, space): - if (self.initialization_state & ~2) ^ 13: - missing_field(space, self.initialization_state, ['test', None, 'lineno', 'col_offset'], 'Assert') + if (self.initialization_state & ~8) ^ 7: + self.missing_field(space, ['lineno', 'col_offset', 'test', None], 'Assert') else: - if not self.initialization_state & 2: + if not self.initialization_state & 8: self.msg = None self.test.sync_app_attrs(space) if self.msg: @@ -926,9 +885,6 @@ class Import(stmt): - _lineno_mask = 2 - _col_offset_mask = 4 - def __init__(self, names, lineno, col_offset): self.names = names self.w_names = None @@ -945,7 +901,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 7: - missing_field(space, self.initialization_state, ['names', 'lineno', 'col_offset'], 'Import') + self.missing_field(space, ['lineno', 'col_offset', 'names'], 'Import') else: pass w_list = self.w_names @@ -962,9 +918,6 @@ class ImportFrom(stmt): - _lineno_mask = 8 - _col_offset_mask = 16 - def __init__(self, module, names, level, lineno, col_offset): self.module = module self.names = names @@ -982,12 +935,12 @@ return visitor.visit_ImportFrom(self) def sync_app_attrs(self, space): - if (self.initialization_state & ~5) ^ 26: - missing_field(space, self.initialization_state, [None, 'names', None, 'lineno', 'col_offset'], 'ImportFrom') + if (self.initialization_state & ~20) ^ 11: + self.missing_field(space, ['lineno', 'col_offset', None, 'names', None], 'ImportFrom') else: - if not self.initialization_state & 1: + if not self.initialization_state & 4: self.module = None - if not self.initialization_state & 4: + if not self.initialization_state & 16: self.level = 0 w_list = self.w_names if w_list is not None: @@ -1003,9 +956,6 @@ class Exec(stmt): - _lineno_mask = 8 - _col_offset_mask = 16 - def __init__(self, body, globals, locals, lineno, col_offset): self.body = body self.globals = globals @@ -1025,12 +975,12 @@ return visitor.visit_Exec(self) def sync_app_attrs(self, space): - if (self.initialization_state & ~6) ^ 25: - missing_field(space, self.initialization_state, ['body', None, None, 'lineno', 'col_offset'], 'Exec') + if (self.initialization_state & ~24) ^ 7: + self.missing_field(space, ['lineno', 'col_offset', 'body', None, None], 'Exec') else: - if not self.initialization_state & 2: + if not self.initialization_state & 8: self.globals = None - if not self.initialization_state & 4: + if not self.initialization_state & 16: self.locals = None self.body.sync_app_attrs(space) if self.globals: @@ -1041,9 +991,6 @@ class Global(stmt): - _lineno_mask = 2 - _col_offset_mask = 4 - def __init__(self, names, lineno, col_offset): self.names = names self.w_names = None @@ -1058,7 +1005,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 7: - missing_field(space, self.initialization_state, ['names', 'lineno', 'col_offset'], 'Global') + self.missing_field(space, ['lineno', 'col_offset', 'names'], 'Global') else: pass w_list = self.w_names @@ -1072,9 +1019,6 @@ class Expr(stmt): - _lineno_mask = 2 - _col_offset_mask = 4 - def __init__(self, value, lineno, col_offset): self.value = value stmt.__init__(self, lineno, col_offset) @@ -1089,7 +1033,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 7: - missing_field(space, self.initialization_state, ['value', 'lineno', 'col_offset'], 'Expr') + self.missing_field(space, ['lineno', 'col_offset', 'value'], 'Expr') else: pass self.value.sync_app_attrs(space) @@ -1097,9 +1041,6 @@ class Pass(stmt): - _lineno_mask = 1 - _col_offset_mask = 2 - def __init__(self, lineno, col_offset): stmt.__init__(self, lineno, col_offset) self.initialization_state = 3 @@ -1112,16 +1053,13 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 3: - missing_field(space, self.initialization_state, ['lineno', 'col_offset'], 'Pass') + self.missing_field(space, ['lineno', 'col_offset'], 'Pass') else: pass class Break(stmt): - _lineno_mask = 1 - _col_offset_mask = 2 - def __init__(self, lineno, col_offset): stmt.__init__(self, lineno, col_offset) self.initialization_state = 3 @@ -1134,16 +1072,13 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 3: - missing_field(space, self.initialization_state, ['lineno', 'col_offset'], 'Break') + self.missing_field(space, ['lineno', 'col_offset'], 'Break') else: pass class Continue(stmt): - _lineno_mask = 1 - _col_offset_mask = 2 - def __init__(self, lineno, col_offset): stmt.__init__(self, lineno, col_offset) self.initialization_state = 3 @@ -1156,21 +1091,19 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 3: - missing_field(space, self.initialization_state, ['lineno', 'col_offset'], 'Continue') + self.missing_field(space, ['lineno', 'col_offset'], 'Continue') else: pass class expr(AST): + def __init__(self, lineno, col_offset): self.lineno = lineno self.col_offset = col_offset class BoolOp(expr): - _lineno_mask = 4 - _col_offset_mask = 8 - def __init__(self, op, values, lineno, col_offset): self.op = op self.values = values @@ -1188,7 +1121,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 15: - missing_field(space, self.initialization_state, ['op', 'values', 'lineno', 'col_offset'], 'BoolOp') + self.missing_field(space, ['lineno', 'col_offset', 'op', 'values'], 'BoolOp') else: pass w_list = self.w_values @@ -1205,9 +1138,6 @@ class BinOp(expr): - _lineno_mask = 8 - _col_offset_mask = 16 - def __init__(self, left, op, right, lineno, col_offset): self.left = left self.op = op @@ -1225,7 +1155,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 31: - missing_field(space, self.initialization_state, ['left', 'op', 'right', 'lineno', 'col_offset'], 'BinOp') + self.missing_field(space, ['lineno', 'col_offset', 'left', 'op', 'right'], 'BinOp') else: pass self.left.sync_app_attrs(space) @@ -1234,9 +1164,6 @@ class UnaryOp(expr): - _lineno_mask = 4 - _col_offset_mask = 8 - def __init__(self, op, operand, lineno, col_offset): self.op = op self.operand = operand @@ -1252,7 +1179,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 15: - missing_field(space, self.initialization_state, ['op', 'operand', 'lineno', 'col_offset'], 'UnaryOp') + self.missing_field(space, ['lineno', 'col_offset', 'op', 'operand'], 'UnaryOp') else: pass self.operand.sync_app_attrs(space) @@ -1260,9 +1187,6 @@ class Lambda(expr): - _lineno_mask = 4 - _col_offset_mask = 8 - def __init__(self, args, body, lineno, col_offset): self.args = args self.body = body @@ -1279,7 +1203,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 15: - missing_field(space, self.initialization_state, ['args', 'body', 'lineno', 'col_offset'], 'Lambda') + self.missing_field(space, ['lineno', 'col_offset', 'args', 'body'], 'Lambda') else: pass self.args.sync_app_attrs(space) @@ -1288,9 +1212,6 @@ class IfExp(expr): - _lineno_mask = 8 - _col_offset_mask = 16 - def __init__(self, test, body, orelse, lineno, col_offset): self.test = test self.body = body @@ -1309,7 +1230,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 31: - missing_field(space, self.initialization_state, ['test', 'body', 'orelse', 'lineno', 'col_offset'], 'IfExp') + self.missing_field(space, ['lineno', 'col_offset', 'test', 'body', 'orelse'], 'IfExp') else: pass self.test.sync_app_attrs(space) @@ -1319,9 +1240,6 @@ class Dict(expr): - _lineno_mask = 4 - _col_offset_mask = 8 - def __init__(self, keys, values, lineno, col_offset): self.keys = keys self.w_keys = None @@ -1342,7 +1260,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 15: - missing_field(space, self.initialization_state, ['keys', 'values', 'lineno', 'col_offset'], 'Dict') + self.missing_field(space, ['lineno', 'col_offset', 'keys', 'values'], 'Dict') else: pass w_list = self.w_keys @@ -1369,9 +1287,6 @@ class Set(expr): - _lineno_mask = 2 - _col_offset_mask = 4 - def __init__(self, elts, lineno, col_offset): self.elts = elts self.w_elts = None @@ -1388,7 +1303,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 7: - missing_field(space, self.initialization_state, ['elts', 'lineno', 'col_offset'], 'Set') + self.missing_field(space, ['lineno', 'col_offset', 'elts'], 'Set') else: pass w_list = self.w_elts @@ -1405,9 +1320,6 @@ class ListComp(expr): - _lineno_mask = 4 - _col_offset_mask = 8 - def __init__(self, elt, generators, lineno, col_offset): self.elt = elt self.generators = generators @@ -1426,7 +1338,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 15: - missing_field(space, self.initialization_state, ['elt', 'generators', 'lineno', 'col_offset'], 'ListComp') + self.missing_field(space, ['lineno', 'col_offset', 'elt', 'generators'], 'ListComp') else: pass self.elt.sync_app_attrs(space) @@ -1444,9 +1356,6 @@ class SetComp(expr): - _lineno_mask = 4 - _col_offset_mask = 8 - def __init__(self, elt, generators, lineno, col_offset): self.elt = elt self.generators = generators @@ -1465,7 +1374,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 15: - missing_field(space, self.initialization_state, ['elt', 'generators', 'lineno', 'col_offset'], 'SetComp') + self.missing_field(space, ['lineno', 'col_offset', 'elt', 'generators'], 'SetComp') else: pass self.elt.sync_app_attrs(space) @@ -1483,9 +1392,6 @@ class DictComp(expr): - _lineno_mask = 8 - _col_offset_mask = 16 - def __init__(self, key, value, generators, lineno, col_offset): self.key = key self.value = value @@ -1506,7 +1412,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 31: - missing_field(space, self.initialization_state, ['key', 'value', 'generators', 'lineno', 'col_offset'], 'DictComp') + self.missing_field(space, ['lineno', 'col_offset', 'key', 'value', 'generators'], 'DictComp') else: pass self.key.sync_app_attrs(space) @@ -1525,9 +1431,6 @@ class GeneratorExp(expr): - _lineno_mask = 4 - _col_offset_mask = 8 - def __init__(self, elt, generators, lineno, col_offset): self.elt = elt self.generators = generators @@ -1546,7 +1449,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 15: - missing_field(space, self.initialization_state, ['elt', 'generators', 'lineno', 'col_offset'], 'GeneratorExp') + self.missing_field(space, ['lineno', 'col_offset', 'elt', 'generators'], 'GeneratorExp') else: pass self.elt.sync_app_attrs(space) @@ -1564,9 +1467,6 @@ class Yield(expr): - _lineno_mask = 2 - _col_offset_mask = 4 - def __init__(self, value, lineno, col_offset): self.value = value expr.__init__(self, lineno, col_offset) @@ -1581,10 +1481,10 @@ return visitor.visit_Yield(self) def sync_app_attrs(self, space): - if (self.initialization_state & ~1) ^ 6: - missing_field(space, self.initialization_state, [None, 'lineno', 'col_offset'], 'Yield') + if (self.initialization_state & ~4) ^ 3: + self.missing_field(space, ['lineno', 'col_offset', None], 'Yield') else: - if not self.initialization_state & 1: + if not self.initialization_state & 4: self.value = None if self.value: self.value.sync_app_attrs(space) @@ -1592,9 +1492,6 @@ class Compare(expr): - _lineno_mask = 8 - _col_offset_mask = 16 - def __init__(self, left, ops, comparators, lineno, col_offset): self.left = left self.ops = ops @@ -1615,7 +1512,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 31: - missing_field(space, self.initialization_state, ['left', 'ops', 'comparators', 'lineno', 'col_offset'], 'Compare') + self.missing_field(space, ['lineno', 'col_offset', 'left', 'ops', 'comparators'], 'Compare') else: pass self.left.sync_app_attrs(space) @@ -1640,9 +1537,6 @@ class Call(expr): - _lineno_mask = 32 - _col_offset_mask = 64 - def __init__(self, func, args, keywords, starargs, kwargs, lineno, col_offset): self.func = func self.args = args @@ -1670,12 +1564,12 @@ return visitor.visit_Call(self) def sync_app_attrs(self, space): - if (self.initialization_state & ~24) ^ 103: - missing_field(space, self.initialization_state, ['func', 'args', 'keywords', None, None, 'lineno', 'col_offset'], 'Call') + if (self.initialization_state & ~96) ^ 31: + self.missing_field(space, ['lineno', 'col_offset', 'func', 'args', 'keywords', None, None], 'Call') else: - if not self.initialization_state & 8: + if not self.initialization_state & 32: self.starargs = None - if not self.initialization_state & 16: + if not self.initialization_state & 64: self.kwargs = None self.func.sync_app_attrs(space) w_list = self.w_args @@ -1706,9 +1600,6 @@ class Repr(expr): - _lineno_mask = 2 - _col_offset_mask = 4 - def __init__(self, value, lineno, col_offset): self.value = value expr.__init__(self, lineno, col_offset) @@ -1723,7 +1614,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 7: - missing_field(space, self.initialization_state, ['value', 'lineno', 'col_offset'], 'Repr') + self.missing_field(space, ['lineno', 'col_offset', 'value'], 'Repr') else: pass self.value.sync_app_attrs(space) @@ -1731,9 +1622,6 @@ class Num(expr): - _lineno_mask = 2 - _col_offset_mask = 4 - def __init__(self, n, lineno, col_offset): self.n = n expr.__init__(self, lineno, col_offset) @@ -1747,16 +1635,13 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 7: - missing_field(space, self.initialization_state, ['n', 'lineno', 'col_offset'], 'Num') + self.missing_field(space, ['lineno', 'col_offset', 'n'], 'Num') else: pass class Str(expr): - _lineno_mask = 2 - _col_offset_mask = 4 - def __init__(self, s, lineno, col_offset): self.s = s expr.__init__(self, lineno, col_offset) @@ -1770,16 +1655,13 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 7: - missing_field(space, self.initialization_state, ['s', 'lineno', 'col_offset'], 'Str') + self.missing_field(space, ['lineno', 'col_offset', 's'], 'Str') else: pass class Attribute(expr): - _lineno_mask = 8 - _col_offset_mask = 16 - def __init__(self, value, attr, ctx, lineno, col_offset): self.value = value self.attr = attr @@ -1796,7 +1678,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 31: - missing_field(space, self.initialization_state, ['value', 'attr', 'ctx', 'lineno', 'col_offset'], 'Attribute') + self.missing_field(space, ['lineno', 'col_offset', 'value', 'attr', 'ctx'], 'Attribute') else: pass self.value.sync_app_attrs(space) @@ -1804,9 +1686,6 @@ class Subscript(expr): - _lineno_mask = 8 - _col_offset_mask = 16 - def __init__(self, value, slice, ctx, lineno, col_offset): self.value = value self.slice = slice @@ -1824,7 +1703,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 31: - missing_field(space, self.initialization_state, ['value', 'slice', 'ctx', 'lineno', 'col_offset'], 'Subscript') + self.missing_field(space, ['lineno', 'col_offset', 'value', 'slice', 'ctx'], 'Subscript') else: pass self.value.sync_app_attrs(space) @@ -1833,9 +1712,6 @@ class Name(expr): - _lineno_mask = 4 - _col_offset_mask = 8 - def __init__(self, id, ctx, lineno, col_offset): self.id = id self.ctx = ctx @@ -1850,16 +1726,13 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 15: - missing_field(space, self.initialization_state, ['id', 'ctx', 'lineno', 'col_offset'], 'Name') + self.missing_field(space, ['lineno', 'col_offset', 'id', 'ctx'], 'Name') else: pass class List(expr): - _lineno_mask = 4 - _col_offset_mask = 8 - def __init__(self, elts, ctx, lineno, col_offset): self.elts = elts self.w_elts = None @@ -1877,7 +1750,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 15: - missing_field(space, self.initialization_state, ['elts', 'ctx', 'lineno', 'col_offset'], 'List') + self.missing_field(space, ['lineno', 'col_offset', 'elts', 'ctx'], 'List') else: pass w_list = self.w_elts @@ -1894,9 +1767,6 @@ class Tuple(expr): - _lineno_mask = 4 - _col_offset_mask = 8 - def __init__(self, elts, ctx, lineno, col_offset): self.elts = elts self.w_elts = None @@ -1914,7 +1784,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 15: - missing_field(space, self.initialization_state, ['elts', 'ctx', 'lineno', 'col_offset'], 'Tuple') + self.missing_field(space, ['lineno', 'col_offset', 'elts', 'ctx'], 'Tuple') else: pass w_list = self.w_elts @@ -1931,9 +1801,6 @@ class Const(expr): - _lineno_mask = 2 - _col_offset_mask = 4 - def __init__(self, value, lineno, col_offset): self.value = value expr.__init__(self, lineno, col_offset) @@ -1947,7 +1814,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 7: - missing_field(space, self.initialization_state, ['value', 'lineno', 'col_offset'], 'Const') + self.missing_field(space, ['lineno', 'col_offset', 'value'], 'Const') else: pass @@ -2009,7 +1876,6 @@ class Ellipsis(slice): - def __init__(self): self.initialization_state = 0 @@ -2021,14 +1887,13 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 0: - missing_field(space, self.initialization_state, [], 'Ellipsis') + self.missing_field(space, [], 'Ellipsis') else: pass class Slice(slice): - def __init__(self, lower, upper, step): self.lower = lower self.upper = upper @@ -2049,7 +1914,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~7) ^ 0: - missing_field(space, self.initialization_state, [None, None, None], 'Slice') + self.missing_field(space, [None, None, None], 'Slice') else: if not self.initialization_state & 1: self.lower = None @@ -2067,7 +1932,6 @@ class ExtSlice(slice): - def __init__(self, dims): self.dims = dims self.w_dims = None @@ -2083,7 +1947,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 1: - missing_field(space, self.initialization_state, ['dims'], 'ExtSlice') + self.missing_field(space, ['dims'], 'ExtSlice') else: pass w_list = self.w_dims @@ -2100,7 +1964,6 @@ class Index(slice): - def __init__(self, value): self.value = value self.initialization_state = 1 @@ -2114,7 +1977,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 1: - missing_field(space, self.initialization_state, ['value'], 'Index') + self.missing_field(space, ['value'], 'Index') else: pass self.value.sync_app_attrs(space) @@ -2377,7 +2240,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 7: - missing_field(space, self.initialization_state, ['target', 'iter', 'ifs'], 'comprehension') + self.missing_field(space, ['target', 'iter', 'ifs'], 'comprehension') else: pass self.target.sync_app_attrs(space) @@ -2394,15 +2257,13 @@ node.sync_app_attrs(space) class excepthandler(AST): + def __init__(self, lineno, col_offset): self.lineno = lineno self.col_offset = col_offset class ExceptHandler(excepthandler): - _lineno_mask = 8 - _col_offset_mask = 16 - def __init__(self, type, name, body, lineno, col_offset): self.type = type self.name = name @@ -2424,12 +2285,12 @@ return visitor.visit_ExceptHandler(self) def sync_app_attrs(self, space): - if (self.initialization_state & ~3) ^ 28: - missing_field(space, self.initialization_state, [None, None, 'body', 'lineno', 'col_offset'], 'ExceptHandler') + if (self.initialization_state & ~12) ^ 19: + self.missing_field(space, ['lineno', 'col_offset', None, None, 'body'], 'ExceptHandler') else: - if not self.initialization_state & 1: + if not self.initialization_state & 4: self.type = None - if not self.initialization_state & 2: + if not self.initialization_state & 8: self.name = None if self.type: self.type.sync_app_attrs(space) @@ -2470,7 +2331,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~6) ^ 9: - missing_field(space, self.initialization_state, ['args', None, None, 'defaults'], 'arguments') + self.missing_field(space, ['args', None, None, 'defaults'], 'arguments') else: if not self.initialization_state & 2: self.vararg = None @@ -2513,7 +2374,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~0) ^ 3: - missing_field(space, self.initialization_state, ['arg', 'value'], 'keyword') + self.missing_field(space, ['arg', 'value'], 'keyword') else: pass self.value.sync_app_attrs(space) @@ -2533,7 +2394,7 @@ def sync_app_attrs(self, space): if (self.initialization_state & ~2) ^ 1: - missing_field(space, self.initialization_state, ['name', None], 'alias') + self.missing_field(space, ['name', None], 'alias') else: if not self.initialization_state & 2: self.asname = None @@ -3019,6 +2880,8 @@ def Expression_set_body(space, w_self, w_new_value): try: w_self.body = space.interp_w(expr, w_new_value, False) + if type(w_self.body) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise @@ -3098,7 +2961,7 @@ w_obj = w_self.getdictvalue(space, 'lineno') if w_obj is not None: return w_obj - if not w_self.initialization_state & w_self._lineno_mask: + if not w_self.initialization_state & 1: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'lineno') return space.wrap(w_self.lineno) @@ -3112,14 +2975,14 @@ w_self.setdictvalue(space, 'lineno', w_new_value) return w_self.deldictvalue(space, 'lineno') - w_self.initialization_state |= w_self._lineno_mask + w_self.initialization_state |= 1 def stmt_get_col_offset(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'col_offset') if w_obj is not None: return w_obj - if not w_self.initialization_state & w_self._col_offset_mask: + if not w_self.initialization_state & 2: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'col_offset') return space.wrap(w_self.col_offset) @@ -3133,7 +2996,7 @@ w_self.setdictvalue(space, 'col_offset', w_new_value) return w_self.deldictvalue(space, 'col_offset') - w_self.initialization_state |= w_self._col_offset_mask + w_self.initialization_state |= 2 stmt.typedef = typedef.TypeDef("stmt", AST.typedef, @@ -3149,7 +3012,7 @@ w_obj = w_self.getdictvalue(space, 'name') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'name') return space.wrap(w_self.name) @@ -3163,14 +3026,14 @@ w_self.setdictvalue(space, 'name', w_new_value) return w_self.deldictvalue(space, 'name') - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 def FunctionDef_get_args(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'args') if w_obj is not None: return w_obj - if not w_self.initialization_state & 2: + if not w_self.initialization_state & 8: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'args') return space.wrap(w_self.args) @@ -3184,10 +3047,10 @@ w_self.setdictvalue(space, 'args', w_new_value) return w_self.deldictvalue(space, 'args') - w_self.initialization_state |= 2 + w_self.initialization_state |= 8 def FunctionDef_get_body(space, w_self): - if not w_self.initialization_state & 4: + if not w_self.initialization_state & 16: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body') if w_self.w_body is None: @@ -3201,10 +3064,10 @@ def FunctionDef_set_body(space, w_self, w_new_value): w_self.w_body = w_new_value - w_self.initialization_state |= 4 + w_self.initialization_state |= 16 def FunctionDef_get_decorator_list(space, w_self): - if not w_self.initialization_state & 8: + if not w_self.initialization_state & 32: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'decorator_list') if w_self.w_decorator_list is None: @@ -3218,7 +3081,7 @@ def FunctionDef_set_decorator_list(space, w_self, w_new_value): w_self.w_decorator_list = w_new_value - w_self.initialization_state |= 8 + w_self.initialization_state |= 32 _FunctionDef_field_unroller = unrolling_iterable(['name', 'args', 'body', 'decorator_list']) def FunctionDef_init(space, w_self, __args__): @@ -3254,7 +3117,7 @@ w_obj = w_self.getdictvalue(space, 'name') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'name') return space.wrap(w_self.name) @@ -3268,10 +3131,10 @@ w_self.setdictvalue(space, 'name', w_new_value) return w_self.deldictvalue(space, 'name') - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 def ClassDef_get_bases(space, w_self): - if not w_self.initialization_state & 2: + if not w_self.initialization_state & 8: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'bases') if w_self.w_bases is None: @@ -3285,10 +3148,10 @@ def ClassDef_set_bases(space, w_self, w_new_value): w_self.w_bases = w_new_value - w_self.initialization_state |= 2 + w_self.initialization_state |= 8 def ClassDef_get_body(space, w_self): - if not w_self.initialization_state & 4: + if not w_self.initialization_state & 16: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body') if w_self.w_body is None: @@ -3302,10 +3165,10 @@ def ClassDef_set_body(space, w_self, w_new_value): w_self.w_body = w_new_value - w_self.initialization_state |= 4 + w_self.initialization_state |= 16 def ClassDef_get_decorator_list(space, w_self): - if not w_self.initialization_state & 8: + if not w_self.initialization_state & 32: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'decorator_list') if w_self.w_decorator_list is None: @@ -3319,7 +3182,7 @@ def ClassDef_set_decorator_list(space, w_self, w_new_value): w_self.w_decorator_list = w_new_value - w_self.initialization_state |= 8 + w_self.initialization_state |= 32 _ClassDef_field_unroller = unrolling_iterable(['name', 'bases', 'body', 'decorator_list']) def ClassDef_init(space, w_self, __args__): @@ -3356,7 +3219,7 @@ w_obj = w_self.getdictvalue(space, 'value') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value') return space.wrap(w_self.value) @@ -3364,13 +3227,15 @@ def Return_set_value(space, w_self, w_new_value): try: w_self.value = space.interp_w(expr, w_new_value, True) + if type(w_self.value) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'value', w_new_value) return w_self.deldictvalue(space, 'value') - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 _Return_field_unroller = unrolling_iterable(['value']) def Return_init(space, w_self, __args__): @@ -3397,7 +3262,7 @@ ) def Delete_get_targets(space, w_self): - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'targets') if w_self.w_targets is None: @@ -3411,7 +3276,7 @@ def Delete_set_targets(space, w_self, w_new_value): w_self.w_targets = w_new_value - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 _Delete_field_unroller = unrolling_iterable(['targets']) def Delete_init(space, w_self, __args__): @@ -3439,7 +3304,7 @@ ) def Assign_get_targets(space, w_self): - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'targets') if w_self.w_targets is None: @@ -3453,14 +3318,14 @@ def Assign_set_targets(space, w_self, w_new_value): w_self.w_targets = w_new_value - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 def Assign_get_value(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'value') if w_obj is not None: return w_obj - if not w_self.initialization_state & 2: + if not w_self.initialization_state & 8: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value') return space.wrap(w_self.value) @@ -3468,13 +3333,15 @@ def Assign_set_value(space, w_self, w_new_value): try: w_self.value = space.interp_w(expr, w_new_value, False) + if type(w_self.value) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'value', w_new_value) return w_self.deldictvalue(space, 'value') - w_self.initialization_state |= 2 + w_self.initialization_state |= 8 _Assign_field_unroller = unrolling_iterable(['targets', 'value']) def Assign_init(space, w_self, __args__): @@ -3507,7 +3374,7 @@ w_obj = w_self.getdictvalue(space, 'target') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'target') return space.wrap(w_self.target) @@ -3515,20 +3382,22 @@ def AugAssign_set_target(space, w_self, w_new_value): try: w_self.target = space.interp_w(expr, w_new_value, False) + if type(w_self.target) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'target', w_new_value) return w_self.deldictvalue(space, 'target') - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 def AugAssign_get_op(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'op') if w_obj is not None: return w_obj - if not w_self.initialization_state & 2: + if not w_self.initialization_state & 8: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'op') return operator_to_class[w_self.op - 1]() @@ -3544,14 +3413,14 @@ return # need to save the original object too w_self.setdictvalue(space, 'op', w_new_value) - w_self.initialization_state |= 2 + w_self.initialization_state |= 8 def AugAssign_get_value(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'value') if w_obj is not None: return w_obj - if not w_self.initialization_state & 4: + if not w_self.initialization_state & 16: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value') return space.wrap(w_self.value) @@ -3559,13 +3428,15 @@ def AugAssign_set_value(space, w_self, w_new_value): try: w_self.value = space.interp_w(expr, w_new_value, False) + if type(w_self.value) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'value', w_new_value) return w_self.deldictvalue(space, 'value') - w_self.initialization_state |= 4 + w_self.initialization_state |= 16 _AugAssign_field_unroller = unrolling_iterable(['target', 'op', 'value']) def AugAssign_init(space, w_self, __args__): @@ -3598,7 +3469,7 @@ w_obj = w_self.getdictvalue(space, 'dest') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'dest') return space.wrap(w_self.dest) @@ -3606,16 +3477,18 @@ def Print_set_dest(space, w_self, w_new_value): try: w_self.dest = space.interp_w(expr, w_new_value, True) + if type(w_self.dest) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'dest', w_new_value) return w_self.deldictvalue(space, 'dest') - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 def Print_get_values(space, w_self): - if not w_self.initialization_state & 2: + if not w_self.initialization_state & 8: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'values') if w_self.w_values is None: @@ -3629,14 +3502,14 @@ def Print_set_values(space, w_self, w_new_value): w_self.w_values = w_new_value - w_self.initialization_state |= 2 + w_self.initialization_state |= 8 def Print_get_nl(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'nl') if w_obj is not None: return w_obj - if not w_self.initialization_state & 4: + if not w_self.initialization_state & 16: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'nl') return space.wrap(w_self.nl) @@ -3650,7 +3523,7 @@ w_self.setdictvalue(space, 'nl', w_new_value) return w_self.deldictvalue(space, 'nl') - w_self.initialization_state |= 4 + w_self.initialization_state |= 16 _Print_field_unroller = unrolling_iterable(['dest', 'values', 'nl']) def Print_init(space, w_self, __args__): @@ -3684,7 +3557,7 @@ w_obj = w_self.getdictvalue(space, 'target') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'target') return space.wrap(w_self.target) @@ -3692,20 +3565,22 @@ def For_set_target(space, w_self, w_new_value): try: w_self.target = space.interp_w(expr, w_new_value, False) + if type(w_self.target) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'target', w_new_value) return w_self.deldictvalue(space, 'target') - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 def For_get_iter(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'iter') if w_obj is not None: return w_obj - if not w_self.initialization_state & 2: + if not w_self.initialization_state & 8: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'iter') return space.wrap(w_self.iter) @@ -3713,16 +3588,18 @@ def For_set_iter(space, w_self, w_new_value): try: w_self.iter = space.interp_w(expr, w_new_value, False) + if type(w_self.iter) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'iter', w_new_value) return w_self.deldictvalue(space, 'iter') - w_self.initialization_state |= 2 + w_self.initialization_state |= 8 def For_get_body(space, w_self): - if not w_self.initialization_state & 4: + if not w_self.initialization_state & 16: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body') if w_self.w_body is None: @@ -3736,10 +3613,10 @@ def For_set_body(space, w_self, w_new_value): w_self.w_body = w_new_value - w_self.initialization_state |= 4 + w_self.initialization_state |= 16 def For_get_orelse(space, w_self): - if not w_self.initialization_state & 8: + if not w_self.initialization_state & 32: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse') if w_self.w_orelse is None: @@ -3753,7 +3630,7 @@ def For_set_orelse(space, w_self, w_new_value): w_self.w_orelse = w_new_value - w_self.initialization_state |= 8 + w_self.initialization_state |= 32 _For_field_unroller = unrolling_iterable(['target', 'iter', 'body', 'orelse']) def For_init(space, w_self, __args__): @@ -3789,7 +3666,7 @@ w_obj = w_self.getdictvalue(space, 'test') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'test') return space.wrap(w_self.test) @@ -3797,16 +3674,18 @@ def While_set_test(space, w_self, w_new_value): try: w_self.test = space.interp_w(expr, w_new_value, False) + if type(w_self.test) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'test', w_new_value) return w_self.deldictvalue(space, 'test') - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 def While_get_body(space, w_self): - if not w_self.initialization_state & 2: + if not w_self.initialization_state & 8: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body') if w_self.w_body is None: @@ -3820,10 +3699,10 @@ def While_set_body(space, w_self, w_new_value): w_self.w_body = w_new_value - w_self.initialization_state |= 2 + w_self.initialization_state |= 8 def While_get_orelse(space, w_self): - if not w_self.initialization_state & 4: + if not w_self.initialization_state & 16: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse') if w_self.w_orelse is None: @@ -3837,7 +3716,7 @@ def While_set_orelse(space, w_self, w_new_value): w_self.w_orelse = w_new_value - w_self.initialization_state |= 4 + w_self.initialization_state |= 16 _While_field_unroller = unrolling_iterable(['test', 'body', 'orelse']) def While_init(space, w_self, __args__): @@ -3872,7 +3751,7 @@ w_obj = w_self.getdictvalue(space, 'test') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'test') return space.wrap(w_self.test) @@ -3880,16 +3759,18 @@ def If_set_test(space, w_self, w_new_value): try: w_self.test = space.interp_w(expr, w_new_value, False) + if type(w_self.test) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'test', w_new_value) return w_self.deldictvalue(space, 'test') - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 def If_get_body(space, w_self): - if not w_self.initialization_state & 2: + if not w_self.initialization_state & 8: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body') if w_self.w_body is None: @@ -3903,10 +3784,10 @@ def If_set_body(space, w_self, w_new_value): w_self.w_body = w_new_value - w_self.initialization_state |= 2 + w_self.initialization_state |= 8 def If_get_orelse(space, w_self): - if not w_self.initialization_state & 4: + if not w_self.initialization_state & 16: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse') if w_self.w_orelse is None: @@ -3920,7 +3801,7 @@ def If_set_orelse(space, w_self, w_new_value): w_self.w_orelse = w_new_value - w_self.initialization_state |= 4 + w_self.initialization_state |= 16 _If_field_unroller = unrolling_iterable(['test', 'body', 'orelse']) def If_init(space, w_self, __args__): @@ -3955,7 +3836,7 @@ w_obj = w_self.getdictvalue(space, 'context_expr') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'context_expr') return space.wrap(w_self.context_expr) @@ -3963,20 +3844,22 @@ def With_set_context_expr(space, w_self, w_new_value): try: w_self.context_expr = space.interp_w(expr, w_new_value, False) + if type(w_self.context_expr) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'context_expr', w_new_value) return w_self.deldictvalue(space, 'context_expr') - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 def With_get_optional_vars(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'optional_vars') if w_obj is not None: return w_obj - if not w_self.initialization_state & 2: + if not w_self.initialization_state & 8: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'optional_vars') return space.wrap(w_self.optional_vars) @@ -3984,16 +3867,18 @@ def With_set_optional_vars(space, w_self, w_new_value): try: w_self.optional_vars = space.interp_w(expr, w_new_value, True) + if type(w_self.optional_vars) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'optional_vars', w_new_value) return w_self.deldictvalue(space, 'optional_vars') - w_self.initialization_state |= 2 + w_self.initialization_state |= 8 def With_get_body(space, w_self): - if not w_self.initialization_state & 4: + if not w_self.initialization_state & 16: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body') if w_self.w_body is None: @@ -4007,7 +3892,7 @@ def With_set_body(space, w_self, w_new_value): w_self.w_body = w_new_value - w_self.initialization_state |= 4 + w_self.initialization_state |= 16 _With_field_unroller = unrolling_iterable(['context_expr', 'optional_vars', 'body']) def With_init(space, w_self, __args__): @@ -4041,7 +3926,7 @@ w_obj = w_self.getdictvalue(space, 'type') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'type') return space.wrap(w_self.type) @@ -4049,20 +3934,22 @@ def Raise_set_type(space, w_self, w_new_value): try: w_self.type = space.interp_w(expr, w_new_value, True) + if type(w_self.type) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'type', w_new_value) return w_self.deldictvalue(space, 'type') - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 def Raise_get_inst(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'inst') if w_obj is not None: return w_obj - if not w_self.initialization_state & 2: + if not w_self.initialization_state & 8: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'inst') return space.wrap(w_self.inst) @@ -4070,20 +3957,22 @@ def Raise_set_inst(space, w_self, w_new_value): try: w_self.inst = space.interp_w(expr, w_new_value, True) + if type(w_self.inst) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'inst', w_new_value) return w_self.deldictvalue(space, 'inst') - w_self.initialization_state |= 2 + w_self.initialization_state |= 8 def Raise_get_tback(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'tback') if w_obj is not None: return w_obj - if not w_self.initialization_state & 4: + if not w_self.initialization_state & 16: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'tback') return space.wrap(w_self.tback) @@ -4091,13 +3980,15 @@ def Raise_set_tback(space, w_self, w_new_value): try: w_self.tback = space.interp_w(expr, w_new_value, True) + if type(w_self.tback) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'tback', w_new_value) return w_self.deldictvalue(space, 'tback') - w_self.initialization_state |= 4 + w_self.initialization_state |= 16 _Raise_field_unroller = unrolling_iterable(['type', 'inst', 'tback']) def Raise_init(space, w_self, __args__): @@ -4126,7 +4017,7 @@ ) def TryExcept_get_body(space, w_self): - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body') if w_self.w_body is None: @@ -4140,10 +4031,10 @@ def TryExcept_set_body(space, w_self, w_new_value): w_self.w_body = w_new_value - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 def TryExcept_get_handlers(space, w_self): - if not w_self.initialization_state & 2: + if not w_self.initialization_state & 8: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'handlers') if w_self.w_handlers is None: @@ -4157,10 +4048,10 @@ def TryExcept_set_handlers(space, w_self, w_new_value): w_self.w_handlers = w_new_value - w_self.initialization_state |= 2 + w_self.initialization_state |= 8 def TryExcept_get_orelse(space, w_self): - if not w_self.initialization_state & 4: + if not w_self.initialization_state & 16: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse') if w_self.w_orelse is None: @@ -4174,7 +4065,7 @@ def TryExcept_set_orelse(space, w_self, w_new_value): w_self.w_orelse = w_new_value - w_self.initialization_state |= 4 + w_self.initialization_state |= 16 _TryExcept_field_unroller = unrolling_iterable(['body', 'handlers', 'orelse']) def TryExcept_init(space, w_self, __args__): @@ -4206,7 +4097,7 @@ ) def TryFinally_get_body(space, w_self): - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body') if w_self.w_body is None: @@ -4220,10 +4111,10 @@ def TryFinally_set_body(space, w_self, w_new_value): w_self.w_body = w_new_value - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 def TryFinally_get_finalbody(space, w_self): - if not w_self.initialization_state & 2: + if not w_self.initialization_state & 8: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'finalbody') if w_self.w_finalbody is None: @@ -4237,7 +4128,7 @@ def TryFinally_set_finalbody(space, w_self, w_new_value): w_self.w_finalbody = w_new_value - w_self.initialization_state |= 2 + w_self.initialization_state |= 8 _TryFinally_field_unroller = unrolling_iterable(['body', 'finalbody']) def TryFinally_init(space, w_self, __args__): @@ -4271,7 +4162,7 @@ w_obj = w_self.getdictvalue(space, 'test') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'test') return space.wrap(w_self.test) @@ -4279,20 +4170,22 @@ def Assert_set_test(space, w_self, w_new_value): try: w_self.test = space.interp_w(expr, w_new_value, False) + if type(w_self.test) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'test', w_new_value) return w_self.deldictvalue(space, 'test') - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 def Assert_get_msg(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'msg') if w_obj is not None: return w_obj - if not w_self.initialization_state & 2: + if not w_self.initialization_state & 8: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'msg') return space.wrap(w_self.msg) @@ -4300,13 +4193,15 @@ def Assert_set_msg(space, w_self, w_new_value): try: w_self.msg = space.interp_w(expr, w_new_value, True) + if type(w_self.msg) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'msg', w_new_value) return w_self.deldictvalue(space, 'msg') - w_self.initialization_state |= 2 + w_self.initialization_state |= 8 _Assert_field_unroller = unrolling_iterable(['test', 'msg']) def Assert_init(space, w_self, __args__): @@ -4334,7 +4229,7 @@ ) def Import_get_names(space, w_self): - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'names') if w_self.w_names is None: @@ -4348,7 +4243,7 @@ def Import_set_names(space, w_self, w_new_value): w_self.w_names = w_new_value - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 _Import_field_unroller = unrolling_iterable(['names']) def Import_init(space, w_self, __args__): @@ -4380,7 +4275,7 @@ w_obj = w_self.getdictvalue(space, 'module') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'module') return space.wrap(w_self.module) @@ -4397,10 +4292,10 @@ w_self.setdictvalue(space, 'module', w_new_value) return w_self.deldictvalue(space, 'module') - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 def ImportFrom_get_names(space, w_self): - if not w_self.initialization_state & 2: + if not w_self.initialization_state & 8: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'names') if w_self.w_names is None: @@ -4414,14 +4309,14 @@ def ImportFrom_set_names(space, w_self, w_new_value): w_self.w_names = w_new_value - w_self.initialization_state |= 2 + w_self.initialization_state |= 8 def ImportFrom_get_level(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'level') if w_obj is not None: return w_obj - if not w_self.initialization_state & 4: + if not w_self.initialization_state & 16: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'level') return space.wrap(w_self.level) @@ -4435,7 +4330,7 @@ w_self.setdictvalue(space, 'level', w_new_value) return w_self.deldictvalue(space, 'level') - w_self.initialization_state |= 4 + w_self.initialization_state |= 16 _ImportFrom_field_unroller = unrolling_iterable(['module', 'names', 'level']) def ImportFrom_init(space, w_self, __args__): @@ -4469,7 +4364,7 @@ w_obj = w_self.getdictvalue(space, 'body') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body') return space.wrap(w_self.body) @@ -4477,20 +4372,22 @@ def Exec_set_body(space, w_self, w_new_value): try: w_self.body = space.interp_w(expr, w_new_value, False) + if type(w_self.body) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'body', w_new_value) return w_self.deldictvalue(space, 'body') - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 def Exec_get_globals(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'globals') if w_obj is not None: return w_obj - if not w_self.initialization_state & 2: + if not w_self.initialization_state & 8: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'globals') return space.wrap(w_self.globals) @@ -4498,20 +4395,22 @@ def Exec_set_globals(space, w_self, w_new_value): try: w_self.globals = space.interp_w(expr, w_new_value, True) + if type(w_self.globals) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'globals', w_new_value) return w_self.deldictvalue(space, 'globals') - w_self.initialization_state |= 2 + w_self.initialization_state |= 8 def Exec_get_locals(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'locals') if w_obj is not None: return w_obj - if not w_self.initialization_state & 4: + if not w_self.initialization_state & 16: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'locals') return space.wrap(w_self.locals) @@ -4519,13 +4418,15 @@ def Exec_set_locals(space, w_self, w_new_value): try: w_self.locals = space.interp_w(expr, w_new_value, True) + if type(w_self.locals) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'locals', w_new_value) return w_self.deldictvalue(space, 'locals') - w_self.initialization_state |= 4 + w_self.initialization_state |= 16 _Exec_field_unroller = unrolling_iterable(['body', 'globals', 'locals']) def Exec_init(space, w_self, __args__): @@ -4554,7 +4455,7 @@ ) def Global_get_names(space, w_self): - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'names') if w_self.w_names is None: @@ -4568,7 +4469,7 @@ def Global_set_names(space, w_self, w_new_value): w_self.w_names = w_new_value - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 _Global_field_unroller = unrolling_iterable(['names']) def Global_init(space, w_self, __args__): @@ -4600,7 +4501,7 @@ w_obj = w_self.getdictvalue(space, 'value') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value') return space.wrap(w_self.value) @@ -4608,13 +4509,15 @@ def Expr_set_value(space, w_self, w_new_value): try: w_self.value = space.interp_w(expr, w_new_value, False) + if type(w_self.value) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'value', w_new_value) return w_self.deldictvalue(space, 'value') - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 _Expr_field_unroller = unrolling_iterable(['value']) def Expr_init(space, w_self, __args__): @@ -4696,7 +4599,7 @@ w_obj = w_self.getdictvalue(space, 'lineno') if w_obj is not None: return w_obj - if not w_self.initialization_state & w_self._lineno_mask: + if not w_self.initialization_state & 1: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'lineno') return space.wrap(w_self.lineno) @@ -4710,14 +4613,14 @@ w_self.setdictvalue(space, 'lineno', w_new_value) return w_self.deldictvalue(space, 'lineno') - w_self.initialization_state |= w_self._lineno_mask + w_self.initialization_state |= 1 def expr_get_col_offset(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'col_offset') if w_obj is not None: return w_obj - if not w_self.initialization_state & w_self._col_offset_mask: + if not w_self.initialization_state & 2: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'col_offset') return space.wrap(w_self.col_offset) @@ -4731,7 +4634,7 @@ w_self.setdictvalue(space, 'col_offset', w_new_value) return w_self.deldictvalue(space, 'col_offset') - w_self.initialization_state |= w_self._col_offset_mask + w_self.initialization_state |= 2 expr.typedef = typedef.TypeDef("expr", AST.typedef, @@ -4747,7 +4650,7 @@ w_obj = w_self.getdictvalue(space, 'op') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'op') return boolop_to_class[w_self.op - 1]() @@ -4763,10 +4666,10 @@ return # need to save the original object too w_self.setdictvalue(space, 'op', w_new_value) - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 def BoolOp_get_values(space, w_self): - if not w_self.initialization_state & 2: + if not w_self.initialization_state & 8: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'values') if w_self.w_values is None: @@ -4780,7 +4683,7 @@ def BoolOp_set_values(space, w_self, w_new_value): w_self.w_values = w_new_value - w_self.initialization_state |= 2 + w_self.initialization_state |= 8 _BoolOp_field_unroller = unrolling_iterable(['op', 'values']) def BoolOp_init(space, w_self, __args__): @@ -4813,7 +4716,7 @@ w_obj = w_self.getdictvalue(space, 'left') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'left') return space.wrap(w_self.left) @@ -4821,20 +4724,22 @@ def BinOp_set_left(space, w_self, w_new_value): try: w_self.left = space.interp_w(expr, w_new_value, False) + if type(w_self.left) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'left', w_new_value) return w_self.deldictvalue(space, 'left') - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 def BinOp_get_op(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'op') if w_obj is not None: return w_obj - if not w_self.initialization_state & 2: + if not w_self.initialization_state & 8: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'op') return operator_to_class[w_self.op - 1]() @@ -4850,14 +4755,14 @@ return # need to save the original object too w_self.setdictvalue(space, 'op', w_new_value) - w_self.initialization_state |= 2 + w_self.initialization_state |= 8 def BinOp_get_right(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'right') if w_obj is not None: return w_obj - if not w_self.initialization_state & 4: + if not w_self.initialization_state & 16: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'right') return space.wrap(w_self.right) @@ -4865,13 +4770,15 @@ def BinOp_set_right(space, w_self, w_new_value): try: w_self.right = space.interp_w(expr, w_new_value, False) + if type(w_self.right) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'right', w_new_value) return w_self.deldictvalue(space, 'right') - w_self.initialization_state |= 4 + w_self.initialization_state |= 16 _BinOp_field_unroller = unrolling_iterable(['left', 'op', 'right']) def BinOp_init(space, w_self, __args__): @@ -4904,7 +4811,7 @@ w_obj = w_self.getdictvalue(space, 'op') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'op') return unaryop_to_class[w_self.op - 1]() @@ -4920,14 +4827,14 @@ return # need to save the original object too w_self.setdictvalue(space, 'op', w_new_value) - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 def UnaryOp_get_operand(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'operand') if w_obj is not None: return w_obj - if not w_self.initialization_state & 2: + if not w_self.initialization_state & 8: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'operand') return space.wrap(w_self.operand) @@ -4935,13 +4842,15 @@ def UnaryOp_set_operand(space, w_self, w_new_value): try: w_self.operand = space.interp_w(expr, w_new_value, False) + if type(w_self.operand) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'operand', w_new_value) return w_self.deldictvalue(space, 'operand') - w_self.initialization_state |= 2 + w_self.initialization_state |= 8 _UnaryOp_field_unroller = unrolling_iterable(['op', 'operand']) def UnaryOp_init(space, w_self, __args__): @@ -4973,7 +4882,7 @@ w_obj = w_self.getdictvalue(space, 'args') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'args') return space.wrap(w_self.args) @@ -4987,14 +4896,14 @@ w_self.setdictvalue(space, 'args', w_new_value) return w_self.deldictvalue(space, 'args') - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 def Lambda_get_body(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'body') if w_obj is not None: return w_obj - if not w_self.initialization_state & 2: + if not w_self.initialization_state & 8: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body') return space.wrap(w_self.body) @@ -5002,13 +4911,15 @@ def Lambda_set_body(space, w_self, w_new_value): try: w_self.body = space.interp_w(expr, w_new_value, False) + if type(w_self.body) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'body', w_new_value) return w_self.deldictvalue(space, 'body') - w_self.initialization_state |= 2 + w_self.initialization_state |= 8 _Lambda_field_unroller = unrolling_iterable(['args', 'body']) def Lambda_init(space, w_self, __args__): @@ -5040,7 +4951,7 @@ w_obj = w_self.getdictvalue(space, 'test') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'test') return space.wrap(w_self.test) @@ -5048,20 +4959,22 @@ def IfExp_set_test(space, w_self, w_new_value): try: w_self.test = space.interp_w(expr, w_new_value, False) + if type(w_self.test) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'test', w_new_value) return w_self.deldictvalue(space, 'test') - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 def IfExp_get_body(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'body') if w_obj is not None: return w_obj - if not w_self.initialization_state & 2: + if not w_self.initialization_state & 8: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body') return space.wrap(w_self.body) @@ -5069,20 +4982,22 @@ def IfExp_set_body(space, w_self, w_new_value): try: w_self.body = space.interp_w(expr, w_new_value, False) + if type(w_self.body) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'body', w_new_value) return w_self.deldictvalue(space, 'body') - w_self.initialization_state |= 2 + w_self.initialization_state |= 8 def IfExp_get_orelse(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'orelse') if w_obj is not None: return w_obj - if not w_self.initialization_state & 4: + if not w_self.initialization_state & 16: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse') return space.wrap(w_self.orelse) @@ -5090,13 +5005,15 @@ def IfExp_set_orelse(space, w_self, w_new_value): try: w_self.orelse = space.interp_w(expr, w_new_value, False) + if type(w_self.orelse) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'orelse', w_new_value) return w_self.deldictvalue(space, 'orelse') - w_self.initialization_state |= 4 + w_self.initialization_state |= 16 _IfExp_field_unroller = unrolling_iterable(['test', 'body', 'orelse']) def IfExp_init(space, w_self, __args__): @@ -5125,7 +5042,7 @@ ) def Dict_get_keys(space, w_self): - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'keys') if w_self.w_keys is None: @@ -5139,10 +5056,10 @@ def Dict_set_keys(space, w_self, w_new_value): w_self.w_keys = w_new_value - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 def Dict_get_values(space, w_self): - if not w_self.initialization_state & 2: + if not w_self.initialization_state & 8: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'values') if w_self.w_values is None: @@ -5156,7 +5073,7 @@ def Dict_set_values(space, w_self, w_new_value): w_self.w_values = w_new_value - w_self.initialization_state |= 2 + w_self.initialization_state |= 8 _Dict_field_unroller = unrolling_iterable(['keys', 'values']) def Dict_init(space, w_self, __args__): @@ -5186,7 +5103,7 @@ ) def Set_get_elts(space, w_self): - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elts') if w_self.w_elts is None: @@ -5200,7 +5117,7 @@ def Set_set_elts(space, w_self, w_new_value): w_self.w_elts = w_new_value - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 _Set_field_unroller = unrolling_iterable(['elts']) def Set_init(space, w_self, __args__): @@ -5232,7 +5149,7 @@ w_obj = w_self.getdictvalue(space, 'elt') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elt') return space.wrap(w_self.elt) @@ -5240,16 +5157,18 @@ def ListComp_set_elt(space, w_self, w_new_value): try: w_self.elt = space.interp_w(expr, w_new_value, False) + if type(w_self.elt) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'elt', w_new_value) return w_self.deldictvalue(space, 'elt') - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 def ListComp_get_generators(space, w_self): - if not w_self.initialization_state & 2: + if not w_self.initialization_state & 8: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'generators') if w_self.w_generators is None: @@ -5263,7 +5182,7 @@ def ListComp_set_generators(space, w_self, w_new_value): w_self.w_generators = w_new_value - w_self.initialization_state |= 2 + w_self.initialization_state |= 8 _ListComp_field_unroller = unrolling_iterable(['elt', 'generators']) def ListComp_init(space, w_self, __args__): @@ -5296,7 +5215,7 @@ w_obj = w_self.getdictvalue(space, 'elt') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elt') return space.wrap(w_self.elt) @@ -5304,16 +5223,18 @@ def SetComp_set_elt(space, w_self, w_new_value): try: w_self.elt = space.interp_w(expr, w_new_value, False) + if type(w_self.elt) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'elt', w_new_value) return w_self.deldictvalue(space, 'elt') - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 def SetComp_get_generators(space, w_self): - if not w_self.initialization_state & 2: + if not w_self.initialization_state & 8: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'generators') if w_self.w_generators is None: @@ -5327,7 +5248,7 @@ def SetComp_set_generators(space, w_self, w_new_value): w_self.w_generators = w_new_value - w_self.initialization_state |= 2 + w_self.initialization_state |= 8 _SetComp_field_unroller = unrolling_iterable(['elt', 'generators']) def SetComp_init(space, w_self, __args__): @@ -5360,7 +5281,7 @@ w_obj = w_self.getdictvalue(space, 'key') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'key') return space.wrap(w_self.key) @@ -5368,20 +5289,22 @@ def DictComp_set_key(space, w_self, w_new_value): try: w_self.key = space.interp_w(expr, w_new_value, False) + if type(w_self.key) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'key', w_new_value) return w_self.deldictvalue(space, 'key') - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 def DictComp_get_value(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'value') if w_obj is not None: return w_obj - if not w_self.initialization_state & 2: + if not w_self.initialization_state & 8: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value') return space.wrap(w_self.value) @@ -5389,16 +5312,18 @@ def DictComp_set_value(space, w_self, w_new_value): try: w_self.value = space.interp_w(expr, w_new_value, False) + if type(w_self.value) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'value', w_new_value) return w_self.deldictvalue(space, 'value') - w_self.initialization_state |= 2 + w_self.initialization_state |= 8 def DictComp_get_generators(space, w_self): - if not w_self.initialization_state & 4: + if not w_self.initialization_state & 16: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'generators') if w_self.w_generators is None: @@ -5412,7 +5337,7 @@ def DictComp_set_generators(space, w_self, w_new_value): w_self.w_generators = w_new_value - w_self.initialization_state |= 4 + w_self.initialization_state |= 16 _DictComp_field_unroller = unrolling_iterable(['key', 'value', 'generators']) def DictComp_init(space, w_self, __args__): @@ -5446,7 +5371,7 @@ w_obj = w_self.getdictvalue(space, 'elt') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elt') return space.wrap(w_self.elt) @@ -5454,16 +5379,18 @@ def GeneratorExp_set_elt(space, w_self, w_new_value): try: w_self.elt = space.interp_w(expr, w_new_value, False) + if type(w_self.elt) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'elt', w_new_value) return w_self.deldictvalue(space, 'elt') - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 def GeneratorExp_get_generators(space, w_self): - if not w_self.initialization_state & 2: + if not w_self.initialization_state & 8: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'generators') if w_self.w_generators is None: @@ -5477,7 +5404,7 @@ def GeneratorExp_set_generators(space, w_self, w_new_value): w_self.w_generators = w_new_value - w_self.initialization_state |= 2 + w_self.initialization_state |= 8 _GeneratorExp_field_unroller = unrolling_iterable(['elt', 'generators']) def GeneratorExp_init(space, w_self, __args__): @@ -5510,7 +5437,7 @@ w_obj = w_self.getdictvalue(space, 'value') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value') return space.wrap(w_self.value) @@ -5518,13 +5445,15 @@ def Yield_set_value(space, w_self, w_new_value): try: w_self.value = space.interp_w(expr, w_new_value, True) + if type(w_self.value) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'value', w_new_value) return w_self.deldictvalue(space, 'value') - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 _Yield_field_unroller = unrolling_iterable(['value']) def Yield_init(space, w_self, __args__): @@ -5555,7 +5484,7 @@ w_obj = w_self.getdictvalue(space, 'left') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'left') return space.wrap(w_self.left) @@ -5563,16 +5492,18 @@ def Compare_set_left(space, w_self, w_new_value): try: w_self.left = space.interp_w(expr, w_new_value, False) + if type(w_self.left) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'left', w_new_value) return w_self.deldictvalue(space, 'left') - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 def Compare_get_ops(space, w_self): - if not w_self.initialization_state & 2: + if not w_self.initialization_state & 8: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ops') if w_self.w_ops is None: @@ -5586,10 +5517,10 @@ def Compare_set_ops(space, w_self, w_new_value): w_self.w_ops = w_new_value - w_self.initialization_state |= 2 + w_self.initialization_state |= 8 def Compare_get_comparators(space, w_self): - if not w_self.initialization_state & 4: + if not w_self.initialization_state & 16: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'comparators') if w_self.w_comparators is None: @@ -5603,7 +5534,7 @@ def Compare_set_comparators(space, w_self, w_new_value): w_self.w_comparators = w_new_value - w_self.initialization_state |= 4 + w_self.initialization_state |= 16 _Compare_field_unroller = unrolling_iterable(['left', 'ops', 'comparators']) def Compare_init(space, w_self, __args__): @@ -5638,7 +5569,7 @@ w_obj = w_self.getdictvalue(space, 'func') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'func') return space.wrap(w_self.func) @@ -5646,16 +5577,18 @@ def Call_set_func(space, w_self, w_new_value): try: w_self.func = space.interp_w(expr, w_new_value, False) + if type(w_self.func) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'func', w_new_value) return w_self.deldictvalue(space, 'func') - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 def Call_get_args(space, w_self): - if not w_self.initialization_state & 2: + if not w_self.initialization_state & 8: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'args') if w_self.w_args is None: @@ -5669,10 +5602,10 @@ def Call_set_args(space, w_self, w_new_value): w_self.w_args = w_new_value - w_self.initialization_state |= 2 + w_self.initialization_state |= 8 def Call_get_keywords(space, w_self): - if not w_self.initialization_state & 4: + if not w_self.initialization_state & 16: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'keywords') if w_self.w_keywords is None: @@ -5686,14 +5619,14 @@ def Call_set_keywords(space, w_self, w_new_value): w_self.w_keywords = w_new_value - w_self.initialization_state |= 4 + w_self.initialization_state |= 16 def Call_get_starargs(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'starargs') if w_obj is not None: return w_obj - if not w_self.initialization_state & 8: + if not w_self.initialization_state & 32: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'starargs') return space.wrap(w_self.starargs) @@ -5701,20 +5634,22 @@ def Call_set_starargs(space, w_self, w_new_value): try: w_self.starargs = space.interp_w(expr, w_new_value, True) + if type(w_self.starargs) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'starargs', w_new_value) return w_self.deldictvalue(space, 'starargs') - w_self.initialization_state |= 8 + w_self.initialization_state |= 32 def Call_get_kwargs(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'kwargs') if w_obj is not None: return w_obj - if not w_self.initialization_state & 16: + if not w_self.initialization_state & 64: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'kwargs') return space.wrap(w_self.kwargs) @@ -5722,13 +5657,15 @@ def Call_set_kwargs(space, w_self, w_new_value): try: w_self.kwargs = space.interp_w(expr, w_new_value, True) + if type(w_self.kwargs) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'kwargs', w_new_value) return w_self.deldictvalue(space, 'kwargs') - w_self.initialization_state |= 16 + w_self.initialization_state |= 64 _Call_field_unroller = unrolling_iterable(['func', 'args', 'keywords', 'starargs', 'kwargs']) def Call_init(space, w_self, __args__): @@ -5765,7 +5702,7 @@ w_obj = w_self.getdictvalue(space, 'value') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value') return space.wrap(w_self.value) @@ -5773,13 +5710,15 @@ def Repr_set_value(space, w_self, w_new_value): try: w_self.value = space.interp_w(expr, w_new_value, False) + if type(w_self.value) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'value', w_new_value) return w_self.deldictvalue(space, 'value') - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 _Repr_field_unroller = unrolling_iterable(['value']) def Repr_init(space, w_self, __args__): @@ -5810,7 +5749,7 @@ w_obj = w_self.getdictvalue(space, 'n') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'n') return w_self.n @@ -5824,7 +5763,7 @@ w_self.setdictvalue(space, 'n', w_new_value) return w_self.deldictvalue(space, 'n') - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 _Num_field_unroller = unrolling_iterable(['n']) def Num_init(space, w_self, __args__): @@ -5855,7 +5794,7 @@ w_obj = w_self.getdictvalue(space, 's') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 's') return w_self.s @@ -5869,7 +5808,7 @@ w_self.setdictvalue(space, 's', w_new_value) return w_self.deldictvalue(space, 's') - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 _Str_field_unroller = unrolling_iterable(['s']) def Str_init(space, w_self, __args__): @@ -5900,7 +5839,7 @@ w_obj = w_self.getdictvalue(space, 'value') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value') return space.wrap(w_self.value) @@ -5908,20 +5847,22 @@ def Attribute_set_value(space, w_self, w_new_value): try: w_self.value = space.interp_w(expr, w_new_value, False) + if type(w_self.value) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'value', w_new_value) return w_self.deldictvalue(space, 'value') - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 def Attribute_get_attr(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'attr') if w_obj is not None: return w_obj - if not w_self.initialization_state & 2: + if not w_self.initialization_state & 8: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'attr') return space.wrap(w_self.attr) @@ -5935,14 +5876,14 @@ w_self.setdictvalue(space, 'attr', w_new_value) return w_self.deldictvalue(space, 'attr') - w_self.initialization_state |= 2 + w_self.initialization_state |= 8 def Attribute_get_ctx(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'ctx') if w_obj is not None: return w_obj - if not w_self.initialization_state & 4: + if not w_self.initialization_state & 16: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx') return expr_context_to_class[w_self.ctx - 1]() @@ -5958,7 +5899,7 @@ return # need to save the original object too w_self.setdictvalue(space, 'ctx', w_new_value) - w_self.initialization_state |= 4 + w_self.initialization_state |= 16 _Attribute_field_unroller = unrolling_iterable(['value', 'attr', 'ctx']) def Attribute_init(space, w_self, __args__): @@ -5991,7 +5932,7 @@ w_obj = w_self.getdictvalue(space, 'value') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value') return space.wrap(w_self.value) @@ -5999,20 +5940,22 @@ def Subscript_set_value(space, w_self, w_new_value): try: w_self.value = space.interp_w(expr, w_new_value, False) + if type(w_self.value) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'value', w_new_value) return w_self.deldictvalue(space, 'value') - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 def Subscript_get_slice(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'slice') if w_obj is not None: return w_obj - if not w_self.initialization_state & 2: + if not w_self.initialization_state & 8: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'slice') return space.wrap(w_self.slice) @@ -6020,20 +5963,22 @@ def Subscript_set_slice(space, w_self, w_new_value): try: w_self.slice = space.interp_w(slice, w_new_value, False) + if type(w_self.slice) is slice: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'slice', w_new_value) return w_self.deldictvalue(space, 'slice') - w_self.initialization_state |= 2 + w_self.initialization_state |= 8 def Subscript_get_ctx(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'ctx') if w_obj is not None: return w_obj - if not w_self.initialization_state & 4: + if not w_self.initialization_state & 16: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx') return expr_context_to_class[w_self.ctx - 1]() @@ -6049,7 +5994,7 @@ return # need to save the original object too w_self.setdictvalue(space, 'ctx', w_new_value) - w_self.initialization_state |= 4 + w_self.initialization_state |= 16 _Subscript_field_unroller = unrolling_iterable(['value', 'slice', 'ctx']) def Subscript_init(space, w_self, __args__): @@ -6082,7 +6027,7 @@ w_obj = w_self.getdictvalue(space, 'id') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'id') return space.wrap(w_self.id) @@ -6096,14 +6041,14 @@ w_self.setdictvalue(space, 'id', w_new_value) return w_self.deldictvalue(space, 'id') - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 def Name_get_ctx(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'ctx') if w_obj is not None: return w_obj - if not w_self.initialization_state & 2: + if not w_self.initialization_state & 8: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx') return expr_context_to_class[w_self.ctx - 1]() @@ -6119,7 +6064,7 @@ return # need to save the original object too w_self.setdictvalue(space, 'ctx', w_new_value) - w_self.initialization_state |= 2 + w_self.initialization_state |= 8 _Name_field_unroller = unrolling_iterable(['id', 'ctx']) def Name_init(space, w_self, __args__): @@ -6147,7 +6092,7 @@ ) def List_get_elts(space, w_self): - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elts') if w_self.w_elts is None: @@ -6161,14 +6106,14 @@ def List_set_elts(space, w_self, w_new_value): w_self.w_elts = w_new_value - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 def List_get_ctx(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'ctx') if w_obj is not None: return w_obj - if not w_self.initialization_state & 2: + if not w_self.initialization_state & 8: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx') return expr_context_to_class[w_self.ctx - 1]() @@ -6184,7 +6129,7 @@ return # need to save the original object too w_self.setdictvalue(space, 'ctx', w_new_value) - w_self.initialization_state |= 2 + w_self.initialization_state |= 8 _List_field_unroller = unrolling_iterable(['elts', 'ctx']) def List_init(space, w_self, __args__): @@ -6213,7 +6158,7 @@ ) def Tuple_get_elts(space, w_self): - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elts') if w_self.w_elts is None: @@ -6227,14 +6172,14 @@ def Tuple_set_elts(space, w_self, w_new_value): w_self.w_elts = w_new_value - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 def Tuple_get_ctx(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'ctx') if w_obj is not None: return w_obj - if not w_self.initialization_state & 2: + if not w_self.initialization_state & 8: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx') return expr_context_to_class[w_self.ctx - 1]() @@ -6250,7 +6195,7 @@ return # need to save the original object too w_self.setdictvalue(space, 'ctx', w_new_value) - w_self.initialization_state |= 2 + w_self.initialization_state |= 8 _Tuple_field_unroller = unrolling_iterable(['elts', 'ctx']) def Tuple_init(space, w_self, __args__): @@ -6283,7 +6228,7 @@ w_obj = w_self.getdictvalue(space, 'value') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value') return w_self.value @@ -6297,7 +6242,7 @@ w_self.setdictvalue(space, 'value', w_new_value) return w_self.deldictvalue(space, 'value') - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 _Const_field_unroller = unrolling_iterable(['value']) def Const_init(space, w_self, __args__): @@ -6409,6 +6354,8 @@ def Slice_set_lower(space, w_self, w_new_value): try: w_self.lower = space.interp_w(expr, w_new_value, True) + if type(w_self.lower) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise @@ -6430,6 +6377,8 @@ def Slice_set_upper(space, w_self, w_new_value): try: w_self.upper = space.interp_w(expr, w_new_value, True) + if type(w_self.upper) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise @@ -6451,6 +6400,8 @@ def Slice_set_step(space, w_self, w_new_value): try: w_self.step = space.interp_w(expr, w_new_value, True) + if type(w_self.step) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise @@ -6540,6 +6491,8 @@ def Index_set_value(space, w_self, w_new_value): try: w_self.value = space.interp_w(expr, w_new_value, False) + if type(w_self.value) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise @@ -6809,6 +6762,8 @@ def comprehension_set_target(space, w_self, w_new_value): try: w_self.target = space.interp_w(expr, w_new_value, False) + if type(w_self.target) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise @@ -6830,6 +6785,8 @@ def comprehension_set_iter(space, w_self, w_new_value): try: w_self.iter = space.interp_w(expr, w_new_value, False) + if type(w_self.iter) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise @@ -6887,7 +6844,7 @@ w_obj = w_self.getdictvalue(space, 'lineno') if w_obj is not None: return w_obj - if not w_self.initialization_state & w_self._lineno_mask: + if not w_self.initialization_state & 1: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'lineno') return space.wrap(w_self.lineno) @@ -6901,14 +6858,14 @@ w_self.setdictvalue(space, 'lineno', w_new_value) return w_self.deldictvalue(space, 'lineno') - w_self.initialization_state |= w_self._lineno_mask + w_self.initialization_state |= 1 def excepthandler_get_col_offset(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'col_offset') if w_obj is not None: return w_obj - if not w_self.initialization_state & w_self._col_offset_mask: + if not w_self.initialization_state & 2: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'col_offset') return space.wrap(w_self.col_offset) @@ -6922,7 +6879,7 @@ w_self.setdictvalue(space, 'col_offset', w_new_value) return w_self.deldictvalue(space, 'col_offset') - w_self.initialization_state |= w_self._col_offset_mask + w_self.initialization_state |= 2 excepthandler.typedef = typedef.TypeDef("excepthandler", AST.typedef, @@ -6938,7 +6895,7 @@ w_obj = w_self.getdictvalue(space, 'type') if w_obj is not None: return w_obj - if not w_self.initialization_state & 1: + if not w_self.initialization_state & 4: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'type') return space.wrap(w_self.type) @@ -6946,20 +6903,22 @@ def ExceptHandler_set_type(space, w_self, w_new_value): try: w_self.type = space.interp_w(expr, w_new_value, True) + if type(w_self.type) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'type', w_new_value) return w_self.deldictvalue(space, 'type') - w_self.initialization_state |= 1 + w_self.initialization_state |= 4 def ExceptHandler_get_name(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'name') if w_obj is not None: return w_obj - if not w_self.initialization_state & 2: + if not w_self.initialization_state & 8: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'name') return space.wrap(w_self.name) @@ -6967,16 +6926,18 @@ def ExceptHandler_set_name(space, w_self, w_new_value): try: w_self.name = space.interp_w(expr, w_new_value, True) + if type(w_self.name) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise w_self.setdictvalue(space, 'name', w_new_value) return w_self.deldictvalue(space, 'name') - w_self.initialization_state |= 2 + w_self.initialization_state |= 8 def ExceptHandler_get_body(space, w_self): - if not w_self.initialization_state & 4: + if not w_self.initialization_state & 16: typename = space.type(w_self).getname(space) raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body') if w_self.w_body is None: @@ -6990,7 +6951,7 @@ def ExceptHandler_set_body(space, w_self, w_new_value): w_self.w_body = w_new_value - w_self.initialization_state |= 4 + w_self.initialization_state |= 16 _ExceptHandler_field_unroller = unrolling_iterable(['type', 'name', 'body']) def ExceptHandler_init(space, w_self, __args__): @@ -7164,6 +7125,8 @@ def keyword_set_value(space, w_self, w_new_value): try: w_self.value = space.interp_w(expr, w_new_value, False) + if type(w_self.value) is expr: + raise OperationError(space.w_TypeError, space.w_None) except OperationError, e: if not e.match(space, space.w_TypeError): raise diff --git a/pypy/interpreter/astcompiler/optimize.py b/pypy/interpreter/astcompiler/optimize.py --- a/pypy/interpreter/astcompiler/optimize.py +++ b/pypy/interpreter/astcompiler/optimize.py @@ -1,6 +1,5 @@ """codegen helpers and AST constant folding.""" import sys -import itertools from pypy.interpreter.astcompiler import ast, consts, misc from pypy.tool import stdlib_opcode as ops @@ -146,8 +145,7 @@ } unrolling_unary_folders = unrolling_iterable(unary_folders.items()) -for folder in itertools.chain(binary_folders.itervalues(), - unary_folders.itervalues()): +for folder in binary_folders.values() + unary_folders.values(): folder._always_inline_ = True del folder diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py --- a/pypy/interpreter/astcompiler/tools/asdl_py.py +++ b/pypy/interpreter/astcompiler/tools/asdl_py.py @@ -79,6 +79,7 @@ else: self.emit("class %s(AST):" % (base,)) if sum.attributes: + self.emit("") args = ", ".join(attr.name.value for attr in sum.attributes) self.emit("def __init__(self, %s):" % (args,), 1) for attr in sum.attributes: @@ -114,7 +115,7 @@ else: names.append(repr(field.name.value)) sub = (", ".join(names), name.value) - self.emit("missing_field(space, self.initialization_state, [%s], %r)" + self.emit("self.missing_field(space, [%s], %r)" % sub, 3) self.emit("else:", 2) # Fill in all the default fields. @@ -195,17 +196,13 @@ def visitConstructor(self, cons, base, extra_attributes): self.emit("class %s(%s):" % (cons.name, base)) self.emit("") - for field in self.data.cons_attributes[cons]: - subst = (field.name, self.data.field_masks[field]) - self.emit("_%s_mask = %i" % subst, 1) - self.emit("") self.make_constructor(cons.fields, cons, extra_attributes, base) self.emit("") self.emit("def walkabout(self, visitor):", 1) self.emit("visitor.visit_%s(self)" % (cons.name,), 2) self.emit("") self.make_mutate_over(cons, cons.name) - self.make_var_syncer(cons.fields + self.data.cons_attributes[cons], + self.make_var_syncer(self.data.cons_attributes[cons] + cons.fields, cons, cons.name) def visitField(self, field): @@ -324,7 +321,7 @@ def visitSum(self, sum, name): for field in sum.attributes: - self.make_property(field, name, True) + self.make_property(field, name) self.make_typedef(name, "AST", sum.attributes, fields_name="_attributes") if not is_simple_sum(sum): @@ -400,13 +397,10 @@ def visitField(self, field, name): self.make_property(field, name) - def make_property(self, field, name, different_masks=False): + def make_property(self, field, name): func = "def %s_get_%s(space, w_self):" % (name, field.name) self.emit(func) - if different_masks: - flag = "w_self._%s_mask" % (field.name,) - else: - flag = self.data.field_masks[field] + flag = self.data.field_masks[field] if not field.seq: self.emit("if w_self.w_dict is not None:", 1) self.emit(" w_obj = w_self.getdictvalue(space, '%s')" % (field.name,), 1) @@ -458,6 +452,11 @@ config = (field.name, field.type, repr(field.opt)) self.emit("w_self.%s = space.interp_w(%s, w_new_value, %s)" % config, 2) + if field.type.value not in self.data.prod_simple: + self.emit("if type(w_self.%s) is %s:" % ( + field.name, field.type), 2) + self.emit("raise OperationError(space.w_TypeError, " + "space.w_None)", 3) else: level = 2 if field.opt and field.type.value != "int": @@ -505,7 +504,10 @@ optional_mask = 0 for i, field in enumerate(fields): flag = 1 << i - field_masks[field] = flag + if field not in field_masks: + field_masks[field] = flag + else: + assert field_masks[field] == flag if field.opt: optional_mask |= flag else: @@ -518,9 +520,9 @@ if is_simple_sum(sum): simple_types.add(tp.name.value) else: + attrs = [field for field in sum.attributes] for cons in sum.types: - attrs = [copy_field(field) for field in sum.attributes] - add_masks(cons.fields + attrs, cons) + add_masks(attrs + cons.fields, cons) cons_attributes[cons] = attrs else: prod = tp.value @@ -588,6 +590,24 @@ space.setattr(self, w_name, space.getitem(w_state, w_name)) + def missing_field(self, space, required, host): + "Find which required field is missing." + state = self.initialization_state + for i in range(len(required)): + if (state >> i) & 1: + continue # field is present + missing = required[i] + if missing is None: + continue # field is optional + w_obj = self.getdictvalue(space, missing) + if w_obj is None: + err = "required field \\"%s\\" missing from %s" + raise operationerrfmt(space.w_TypeError, err, missing, host) + else: + err = "incorrect type for field \\"%s\\" in %s" + raise operationerrfmt(space.w_TypeError, err, missing, host) + raise AssertionError("should not reach here") + class NodeVisitorNotImplemented(Exception): pass @@ -631,15 +651,6 @@ ) -def missing_field(space, state, required, host): - "Find which required field is missing." - for i in range(len(required)): - if not (state >> i) & 1: - missing = required[i] - if missing is not None: - err = "required field \\"%s\\" missing from %s" - raise operationerrfmt(space.w_TypeError, err, missing, host) - raise AssertionError("should not reach here") """ diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -1,4 +1,3 @@ -import itertools import pypy from pypy.interpreter.executioncontext import ExecutionContext, ActionFlag from pypy.interpreter.executioncontext import UserDelAction, FrameTraceAction @@ -191,8 +190,8 @@ def is_w(self, space, w_other): return self is w_other - def unique_id(self, space): - return space.wrap(compute_unique_id(self)) + def immutable_unique_id(self, space): + return None def str_w(self, space): w_msg = typed_unwrap_error_msg(space, "string", self) @@ -488,6 +487,16 @@ 'parser', 'fcntl', '_codecs', 'binascii' ] + # These modules are treated like CPython treats built-in modules, + # i.e. they always shadow any xx.py. The other modules are treated + # like CPython treats extension modules, and are loaded in sys.path + # order by the fake entry '.../lib_pypy/__extensions__'. + MODULES_THAT_ALWAYS_SHADOW = dict.fromkeys([ + '__builtin__', '__pypy__', '_ast', '_codecs', '_sre', '_warnings', + '_weakref', 'errno', 'exceptions', 'gc', 'imp', 'marshal', + 'posix', 'nt', 'pwd', 'signal', 'sys', 'thread', 'zipimport', + ], None) + def make_builtins(self): "NOT_RPYTHON: only for initializing the space." @@ -519,8 +528,8 @@ exception_types_w = self.export_builtin_exceptions() # initialize with "bootstrap types" from objspace (e.g. w_None) - types_w = itertools.chain(self.get_builtin_types().iteritems(), - exception_types_w.iteritems()) + types_w = (self.get_builtin_types().items() + + exception_types_w.items()) for name, w_type in types_w: self.setitem(self.builtin.w_dict, self.wrap(name), w_type) @@ -697,7 +706,10 @@ return w_two.is_w(self, w_one) def id(self, w_obj): - return w_obj.unique_id(self) + w_result = w_obj.immutable_unique_id(self) + if w_result is None: + w_result = self.wrap(compute_unique_id(w_obj)) + return w_result def hash_w(self, w_obj): """shortcut for space.int_w(space.hash(w_obj))""" @@ -1579,12 +1591,15 @@ 'ArithmeticError', 'AssertionError', 'AttributeError', + 'BaseException', + 'DeprecationWarning', 'EOFError', 'EnvironmentError', 'Exception', 'FloatingPointError', 'IOError', 'ImportError', + 'ImportWarning', 'IndentationError', 'IndexError', 'KeyError', @@ -1605,9 +1620,14 @@ 'TabError', 'TypeError', 'UnboundLocalError', + 'UnicodeDecodeError', 'UnicodeError', + 'UnicodeEncodeError', + 'UnicodeTranslateError', 'ValueError', 'ZeroDivisionError', + 'UnicodeEncodeError', + 'UnicodeDecodeError', ] ## Irregular part of the interface: diff --git a/pypy/interpreter/eval.py b/pypy/interpreter/eval.py --- a/pypy/interpreter/eval.py +++ b/pypy/interpreter/eval.py @@ -2,7 +2,6 @@ This module defines the abstract base classes that support execution: Code and Frame. """ -from pypy.rlib import jit from pypy.interpreter.error import OperationError from pypy.interpreter.baseobjspace import Wrappable @@ -98,7 +97,6 @@ "Abstract. Get the expected number of locals." raise TypeError, "abstract" - @jit.dont_look_inside def fast2locals(self): # Copy values from the fastlocals to self.w_locals if self.w_locals is None: @@ -112,7 +110,6 @@ w_name = self.space.wrap(name) self.space.setitem(self.w_locals, w_name, w_value) - @jit.dont_look_inside def locals2fast(self): # Copy values from self.w_locals to the fastlocals assert self.w_locals is not None diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py --- a/pypy/interpreter/gateway.py +++ b/pypy/interpreter/gateway.py @@ -619,7 +619,8 @@ self.descr_reqcls, args) except Exception, e: - raise self.handle_exception(space, e) + self.handle_exception(space, e) + w_result = None if w_result is None: w_result = space.w_None return w_result @@ -655,7 +656,8 @@ self.descr_reqcls, args) except Exception, e: - raise self.handle_exception(space, e) + self.handle_exception(space, e) + w_result = None if w_result is None: w_result = space.w_None return w_result @@ -674,7 +676,8 @@ self.descr_reqcls, args.prepend(w_obj)) except Exception, e: - raise self.handle_exception(space, e) + self.handle_exception(space, e) + w_result = None if w_result is None: w_result = space.w_None return w_result @@ -690,7 +693,8 @@ raise OperationError(space.w_SystemError, space.wrap("unexpected DescrMismatch error")) except Exception, e: - raise self.handle_exception(space, e) + self.handle_exception(space, e) + w_result = None if w_result is None: w_result = space.w_None return w_result @@ -708,7 +712,8 @@ self.descr_reqcls, Arguments(space, [w1])) except Exception, e: - raise self.handle_exception(space, e) + self.handle_exception(space, e) + w_result = None if w_result is None: w_result = space.w_None return w_result @@ -726,7 +731,8 @@ self.descr_reqcls, Arguments(space, [w1, w2])) except Exception, e: - raise self.handle_exception(space, e) + self.handle_exception(space, e) + w_result = None if w_result is None: w_result = space.w_None return w_result @@ -744,7 +750,8 @@ self.descr_reqcls, Arguments(space, [w1, w2, w3])) except Exception, e: - raise self.handle_exception(space, e) + self.handle_exception(space, e) + w_result = None if w_result is None: w_result = space.w_None return w_result @@ -763,7 +770,8 @@ Arguments(space, [w1, w2, w3, w4])) except Exception, e: - raise self.handle_exception(space, e) + self.handle_exception(space, e) + w_result = None if w_result is None: w_result = space.w_None return w_result diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py --- a/pypy/interpreter/generator.py +++ b/pypy/interpreter/generator.py @@ -162,7 +162,8 @@ # generate 2 versions of the function and 2 jit drivers. def _create_unpack_into(): jitdriver = jit.JitDriver(greens=['pycode'], - reds=['self', 'frame', 'results']) + reds=['self', 'frame', 'results'], + name='unpack_into') def unpack_into(self, results): """This is a hack for performance: runs the generator and collects all produced items in a list.""" @@ -196,4 +197,4 @@ self.frame = None return unpack_into unpack_into = _create_unpack_into() - unpack_into_w = _create_unpack_into() \ No newline at end of file + unpack_into_w = _create_unpack_into() diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py --- a/pypy/interpreter/pyframe.py +++ b/pypy/interpreter/pyframe.py @@ -10,7 +10,7 @@ from pypy.rlib.objectmodel import we_are_translated, instantiate from pypy.rlib.jit import hint from pypy.rlib.debug import make_sure_not_resized, check_nonneg -from pypy.rlib.rarithmetic import intmask +from pypy.rlib.rarithmetic import intmask, r_uint from pypy.rlib import jit from pypy.tool import stdlib_opcode from pypy.tool.stdlib_opcode import host_bytecode_spec @@ -167,7 +167,7 @@ # Execution starts just after the last_instr. Initially, # last_instr is -1. After a generator suspends it points to # the YIELD_VALUE instruction. - next_instr = self.last_instr + 1 + next_instr = r_uint(self.last_instr + 1) if next_instr != 0: self.pushvalue(w_inputvalue) # @@ -691,6 +691,7 @@ handlerposition = space.int_w(w_handlerposition) valuestackdepth = space.int_w(w_valuestackdepth) assert valuestackdepth >= 0 + assert handlerposition >= 0 blk = instantiate(get_block_class(opname)) blk.handlerposition = handlerposition blk.valuestackdepth = valuestackdepth diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -837,6 +837,7 @@ raise Yield def jump_absolute(self, jumpto, next_instr, ec): + check_nonneg(jumpto) return jumpto def JUMP_FORWARD(self, jumpby, next_instr): @@ -1278,7 +1279,7 @@ def handle(self, frame, unroller): next_instr = self.really_handle(frame, unroller) # JIT hack - return next_instr + return r_uint(next_instr) def really_handle(self, frame, unroller): """ Purely abstract method diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py --- a/pypy/interpreter/test/test_argument.py +++ b/pypy/interpreter/test/test_argument.py @@ -393,8 +393,8 @@ class FakeArgErr(ArgErr): - def getmsg(self, fname): - return "msg "+fname + def getmsg(self): + return "msg" def _match_signature(*args): raise FakeArgErr() @@ -404,7 +404,7 @@ excinfo = py.test.raises(OperationError, args.parse_obj, "obj", "foo", Signature(["a", "b"], None, None)) assert excinfo.value.w_type is TypeError - assert excinfo.value._w_value == "msg foo" + assert excinfo.value.get_w_value(space) == "foo() msg" def test_args_parsing_into_scope(self): @@ -448,8 +448,8 @@ class FakeArgErr(ArgErr): - def getmsg(self, fname): - return "msg "+fname + def getmsg(self): + return "msg" def _match_signature(*args): raise FakeArgErr() @@ -460,7 +460,7 @@ "obj", [None, None], "foo", Signature(["a", "b"], None, None)) assert excinfo.value.w_type is TypeError - assert excinfo.value._w_value == "msg foo" + assert excinfo.value.get_w_value(space) == "foo() msg" def test_topacked_frompacked(self): space = DummySpace() @@ -493,35 +493,35 @@ # got_nargs, nkwds, expected_nargs, has_vararg, has_kwarg, # defaults_w, missing_args err = ArgErrCount(1, 0, 0, False, False, None, 0) - s = err.getmsg('foo') - assert s == "foo() takes no arguments (1 given)" + s = err.getmsg() + assert s == "takes no arguments (1 given)" err = ArgErrCount(0, 0, 1, False, False, [], 1) - s = err.getmsg('foo') - assert s == "foo() takes exactly 1 argument (0 given)" + s = err.getmsg() + assert s == "takes exactly 1 argument (0 given)" err = ArgErrCount(3, 0, 2, False, False, [], 0) - s = err.getmsg('foo') - assert s == "foo() takes exactly 2 arguments (3 given)" + s = err.getmsg() + assert s == "takes exactly 2 arguments (3 given)" err = ArgErrCount(3, 0, 2, False, False, ['a'], 0) - s = err.getmsg('foo') - assert s == "foo() takes at most 2 arguments (3 given)" + s = err.getmsg() + assert s == "takes at most 2 arguments (3 given)" err = ArgErrCount(1, 0, 2, True, False, [], 1) - s = err.getmsg('foo') - assert s == "foo() takes at least 2 arguments (1 given)" + s = err.getmsg() + assert s == "takes at least 2 arguments (1 given)" err = ArgErrCount(0, 1, 2, True, False, ['a'], 1) - s = err.getmsg('foo') - assert s == "foo() takes at least 1 non-keyword argument (0 given)" + s = err.getmsg() + assert s == "takes at least 1 non-keyword argument (0 given)" err = ArgErrCount(2, 1, 1, False, True, [], 0) - s = err.getmsg('foo') - assert s == "foo() takes exactly 1 non-keyword argument (2 given)" + s = err.getmsg() + assert s == "takes exactly 1 non-keyword argument (2 given)" err = ArgErrCount(0, 1, 1, False, True, [], 1) - s = err.getmsg('foo') - assert s == "foo() takes exactly 1 non-keyword argument (0 given)" + s = err.getmsg() + assert s == "takes exactly 1 non-keyword argument (0 given)" err = ArgErrCount(0, 1, 1, True, True, [], 1) - s = err.getmsg('foo') - assert s == "foo() takes at least 1 non-keyword argument (0 given)" + s = err.getmsg() + assert s == "takes at least 1 non-keyword argument (0 given)" err = ArgErrCount(2, 1, 1, False, True, ['a'], 0) - s = err.getmsg('foo') - assert s == "foo() takes at most 1 non-keyword argument (2 given)" + s = err.getmsg() + assert s == "takes at most 1 non-keyword argument (2 given)" def test_bad_type_for_star(self): space = self.space @@ -543,12 +543,12 @@ def test_unknown_keywords(self): space = DummySpace() err = ArgErrUnknownKwds(space, 1, ['a', 'b'], [True, False], None) - s = err.getmsg('foo') - assert s == "foo() got an unexpected keyword argument 'b'" + s = err.getmsg() + assert s == "got an unexpected keyword argument 'b'" err = ArgErrUnknownKwds(space, 2, ['a', 'b', 'c'], [True, False, False], None) - s = err.getmsg('foo') - assert s == "foo() got 2 unexpected keyword arguments" + s = err.getmsg() + assert s == "got 2 unexpected keyword arguments" def test_unknown_unicode_keyword(self): class DummySpaceUnicode(DummySpace): @@ -558,13 +558,13 @@ err = ArgErrUnknownKwds(space, 1, ['a', None, 'b', 'c'], [True, False, True, True], [unichr(0x1234), u'b', u'c']) - s = err.getmsg('foo') - assert s == "foo() got an unexpected keyword argument '\xe1\x88\xb4'" + s = err.getmsg() + assert s == "got an unexpected keyword argument '\xe1\x88\xb4'" def test_multiple_values(self): err = ArgErrMultipleValues('bla') - s = err.getmsg('foo') - assert s == "foo() got multiple values for keyword argument 'bla'" + s = err.getmsg() + assert s == "got multiple values for keyword argument 'bla'" class AppTestArgument: def test_error_message(self): diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py --- a/pypy/interpreter/typedef.py +++ b/pypy/interpreter/typedef.py @@ -54,7 +54,11 @@ # Hash support def default_identity_hash(space, w_obj): - return space.wrap(compute_identity_hash(w_obj)) + w_unique_id = w_obj.immutable_unique_id(space) + if w_unique_id is None: # common case + return space.wrap(compute_identity_hash(w_obj)) + else: + return space.hash(w_unique_id) # ____________________________________________________________ # diff --git a/pypy/jit/backend/llgraph/llimpl.py b/pypy/jit/backend/llgraph/llimpl.py --- a/pypy/jit/backend/llgraph/llimpl.py +++ b/pypy/jit/backend/llgraph/llimpl.py @@ -8,6 +8,7 @@ from pypy.objspace.flow.model import Variable, Constant from pypy.annotation import model as annmodel from pypy.jit.metainterp.history import REF, INT, FLOAT +from pypy.jit.metainterp import history from pypy.jit.codewriter import heaptracker from pypy.rpython.lltypesystem import lltype, llmemory, rclass, rstr, rffi from pypy.rpython.ootypesystem import ootype @@ -20,7 +21,7 @@ from pypy.jit.backend.llgraph import symbolic from pypy.jit.codewriter import longlong -from pypy.rlib import libffi +from pypy.rlib import libffi, clibffi from pypy.rlib.objectmodel import ComputedIntSymbolic, we_are_translated from pypy.rlib.rarithmetic import ovfcheck from pypy.rlib.rarithmetic import r_longlong, r_ulonglong, r_uint @@ -48,6 +49,11 @@ value._the_opaque_pointer = op return op +def _normalize(value): + if isinstance(value, lltype._ptr): + value = lltype.top_container(value._obj) + return value + def from_opaque_string(s): if isinstance(s, str): return s @@ -322,6 +328,14 @@ _variables.append(v) return r +def compile_started_vars(clt): + if not hasattr(clt, '_debug_argtypes'): # only when compiling the loop + argtypes = [v.concretetype for v in _variables] + try: + clt._debug_argtypes = argtypes + except AttributeError: # when 'clt' is actually a translated + pass # GcStruct + def compile_add(loop, opnum): loop = _from_opaque(loop) loop.operations.append(Operation(opnum)) @@ -347,6 +361,16 @@ op = loop.operations[-1] op.descr = weakref.ref(descr) +TARGET_TOKENS = weakref.WeakKeyDictionary() + +def compile_add_target_token(loop, descr, clt): + # here, 'clt' is the compiled_loop_token of the original loop that + # we are compiling + loop = _from_opaque(loop) + op = loop.operations[-1] + descrobj = _normalize(descr) + TARGET_TOKENS[descrobj] = loop, len(loop.operations), op.args, clt + def compile_add_var(loop, intvar): loop = _from_opaque(loop) op = loop.operations[-1] @@ -381,13 +405,25 @@ _variables.append(v) return r -def compile_add_jump_target(loop, loop_target): +def compile_add_jump_target(loop, targettoken, source_clt): loop = _from_opaque(loop) - loop_target = _from_opaque(loop_target) + descrobj = _normalize(targettoken) + (loop_target, target_opindex, target_inputargs, target_clt + ) = TARGET_TOKENS[descrobj] + # + try: + assert source_clt._debug_argtypes == target_clt._debug_argtypes + except AttributeError: # when translated + pass + # op = loop.operations[-1] op.jump_target = loop_target + op.jump_target_opindex = target_opindex + op.jump_target_inputargs = target_inputargs assert op.opnum == rop.JUMP - assert len(op.args) == len(loop_target.inputargs) + assert [v.concretetype for v in op.args] == ( + [v.concretetype for v in target_inputargs]) + # if loop_target == loop: log.info("compiling new loop") else: @@ -521,10 +557,11 @@ self.opindex += 1 continue if op.opnum == rop.JUMP: - assert len(op.jump_target.inputargs) == len(args) - self.env = dict(zip(op.jump_target.inputargs, args)) + inputargs = op.jump_target_inputargs + assert len(inputargs) == len(args) + self.env = dict(zip(inputargs, args)) self.loop = op.jump_target - self.opindex = 0 + self.opindex = op.jump_target_opindex _stats.exec_jumps += 1 elif op.opnum == rop.FINISH: if self.verbose: @@ -617,6 +654,15 @@ # return _op_default_implementation + def op_label(self, _, *args): + op = self.loop.operations[self.opindex] + assert op.opnum == rop.LABEL + assert len(op.args) == len(args) + newenv = {} + for v, value in zip(op.args, args): + newenv[v] = value + self.env = newenv + def op_debug_merge_point(self, _, *args): from pypy.jit.metainterp.warmspot import get_stats try: @@ -959,6 +1005,7 @@ self._may_force = self.opindex try: inpargs = _from_opaque(ctl.compiled_version).inputargs + assert len(inpargs) == len(args) for i, inparg in enumerate(inpargs): TYPE = inparg.concretetype if TYPE is lltype.Signed: @@ -1432,6 +1479,10 @@ res = _getinteriorfield_raw(libffi.types.slong, array, index, width, ofs) return res +def do_getinteriorfield_raw_float(array, index, width, ofs): + res = _getinteriorfield_raw(libffi.types.double, array, index, width, ofs) + return res + def _getfield_raw(struct, fieldnum): STRUCT, fieldname = symbolic.TokenToField[fieldnum] ptr = cast_from_int(lltype.Ptr(STRUCT), struct) @@ -1510,12 +1561,17 @@ do_setinteriorfield_gc_float = new_setinteriorfield_gc(cast_from_floatstorage) do_setinteriorfield_gc_ptr = new_setinteriorfield_gc(cast_from_ptr) -def new_setinteriorfield_raw(ffitype): +def new_setinteriorfield_raw(cast_func, ffitype): def do_setinteriorfield_raw(array, index, newvalue, width, ofs): addr = rffi.cast(rffi.VOIDP, array) + for TYPE, ffitype2 in clibffi.ffitype_map: + if ffitype2 is ffitype: + newvalue = cast_func(TYPE, newvalue) + break return libffi.array_setitem(ffitype, width, addr, index, ofs, newvalue) return do_setinteriorfield_raw -do_setinteriorfield_raw_int = new_setinteriorfield_raw(libffi.types.slong) +do_setinteriorfield_raw_int = new_setinteriorfield_raw(cast_from_int, libffi.types.slong) +do_setinteriorfield_raw_float = new_setinteriorfield_raw(cast_from_floatstorage, libffi.types.double) def do_setfield_raw_int(struct, fieldnum, newvalue): STRUCT, fieldname = symbolic.TokenToField[fieldnum] @@ -1779,9 +1835,11 @@ setannotation(compile_start_int_var, annmodel.SomeInteger()) setannotation(compile_start_ref_var, annmodel.SomeInteger()) setannotation(compile_start_float_var, annmodel.SomeInteger()) +setannotation(compile_started_vars, annmodel.s_None) setannotation(compile_add, annmodel.s_None) setannotation(compile_add_descr, annmodel.s_None) setannotation(compile_add_descr_arg, annmodel.s_None) +setannotation(compile_add_target_token, annmodel.s_None) setannotation(compile_add_var, annmodel.s_None) setannotation(compile_add_int_const, annmodel.s_None) setannotation(compile_add_ref_const, annmodel.s_None) diff --git a/pypy/jit/backend/llgraph/runner.py b/pypy/jit/backend/llgraph/runner.py --- a/pypy/jit/backend/llgraph/runner.py +++ b/pypy/jit/backend/llgraph/runner.py @@ -37,7 +37,7 @@ def get_arg_types(self): return self.arg_types - def get_return_type(self): + def get_result_type(self): return self.typeinfo def get_extra_info(self): @@ -138,29 +138,30 @@ clt = original_loop_token.compiled_loop_token clt.loop_and_bridges.append(c) clt.compiling_a_bridge() - self._compile_loop_or_bridge(c, inputargs, operations) + self._compile_loop_or_bridge(c, inputargs, operations, clt) old, oldindex = faildescr._compiled_fail llimpl.compile_redirect_fail(old, oldindex, c) - def compile_loop(self, inputargs, operations, looptoken, log=True, name=''): + def compile_loop(self, inputargs, operations, jitcell_token, + log=True, name=''): """In a real assembler backend, this should assemble the given list of operations. Here we just generate a similar CompiledLoop instance. The code here is RPython, whereas the code in llimpl is not. """ c = llimpl.compile_start() - clt = model.CompiledLoopToken(self, looptoken.number) + clt = model.CompiledLoopToken(self, jitcell_token.number) clt.loop_and_bridges = [c] clt.compiled_version = c - looptoken.compiled_loop_token = clt - self._compile_loop_or_bridge(c, inputargs, operations) + jitcell_token.compiled_loop_token = clt + self._compile_loop_or_bridge(c, inputargs, operations, clt) def free_loop_and_bridges(self, compiled_loop_token): for c in compiled_loop_token.loop_and_bridges: llimpl.mark_as_free(c) model.AbstractCPU.free_loop_and_bridges(self, compiled_loop_token) - def _compile_loop_or_bridge(self, c, inputargs, operations): + def _compile_loop_or_bridge(self, c, inputargs, operations, clt): var2index = {} for box in inputargs: if isinstance(box, history.BoxInt): @@ -172,10 +173,11 @@ var2index[box] = llimpl.compile_start_float_var(c) else: raise Exception("box is: %r" % (box,)) - self._compile_operations(c, operations, var2index) + llimpl.compile_started_vars(clt) + self._compile_operations(c, operations, var2index, clt) return c - def _compile_operations(self, c, operations, var2index): + def _compile_operations(self, c, operations, var2index, clt): for op in operations: llimpl.compile_add(c, op.getopnum()) descr = op.getdescr() @@ -183,9 +185,11 @@ llimpl.compile_add_descr(c, descr.ofs, descr.typeinfo, descr.arg_types, descr.extrainfo, descr.width) - if (isinstance(descr, history.LoopToken) and - op.getopnum() != rop.JUMP): + if isinstance(descr, history.JitCellToken): + assert op.getopnum() != rop.JUMP llimpl.compile_add_loop_token(c, descr) + if isinstance(descr, history.TargetToken) and op.getopnum() == rop.LABEL: + llimpl.compile_add_target_token(c, descr, clt) if self.is_oo and isinstance(descr, (OODescr, MethDescr)): # hack hack, not rpython c._obj.externalobj.operations[-1].setdescr(descr) @@ -239,9 +243,7 @@ assert op.is_final() if op.getopnum() == rop.JUMP: targettoken = op.getdescr() - assert isinstance(targettoken, history.LoopToken) - compiled_version = targettoken.compiled_loop_token.compiled_version - llimpl.compile_add_jump_target(c, compiled_version) + llimpl.compile_add_jump_target(c, targettoken, clt) elif op.getopnum() == rop.FINISH: faildescr = op.getdescr() index = self.get_fail_descr_number(faildescr) @@ -260,21 +262,28 @@ self.latest_frame = frame return fail_index - def execute_token(self, loop_token): - """Calls the assembler generated for the given loop. - Returns the ResOperation that failed, of type rop.FAIL. - """ - fail_index = self._execute_token(loop_token) - return self.get_fail_descr_from_number(fail_index) - - def set_future_value_int(self, index, intvalue): - llimpl.set_future_value_int(index, intvalue) - - def set_future_value_ref(self, index, objvalue): - llimpl.set_future_value_ref(index, objvalue) - - def set_future_value_float(self, index, floatvalue): - llimpl.set_future_value_float(index, floatvalue) + def make_execute_token(self, *argtypes): + nb_args = len(argtypes) + unroll_argtypes = unrolling_iterable(list(enumerate(argtypes))) + # + def execute_token(loop_token, *args): + assert len(args) == nb_args + for index, TYPE in unroll_argtypes: + x = args[index] + assert TYPE == lltype.typeOf(x) + if TYPE == lltype.Signed: + llimpl.set_future_value_int(index, x) + elif TYPE == llmemory.GCREF: + llimpl.set_future_value_ref(index, x) + elif TYPE == longlong.FLOATSTORAGE: + llimpl.set_future_value_float(index, x) + else: + assert 0 + # + fail_index = self._execute_token(loop_token) + return self.get_fail_descr_from_number(fail_index) + # + return execute_token def get_latest_value_int(self, index): return llimpl.frame_int_getvalue(self.latest_frame, index) diff --git a/pypy/jit/backend/llsupport/asmmemmgr.py b/pypy/jit/backend/llsupport/asmmemmgr.py --- a/pypy/jit/backend/llsupport/asmmemmgr.py +++ b/pypy/jit/backend/llsupport/asmmemmgr.py @@ -37,25 +37,25 @@ self._add_free_block(smaller_stop, stop) stop = smaller_stop result = (start, stop) - self.total_mallocs += stop - start + self.total_mallocs += r_uint(stop - start) return result # pair (start, stop) def free(self, start, stop): """Free a block (start, stop) returned by a previous malloc().""" - self.total_mallocs -= (stop - start) + self.total_mallocs -= r_uint(stop - start) self._add_free_block(start, stop) def open_malloc(self, minsize): """Allocate at least minsize bytes. Returns (start, stop).""" result = self._allocate_block(minsize) (start, stop) = result - self.total_mallocs += stop - start + self.total_mallocs += r_uint(stop - start) return result def open_free(self, middle, stop): """Used for freeing the end of an open-allocated block of memory.""" if stop - middle >= self.min_fragment: - self.total_mallocs -= (stop - middle) + self.total_mallocs -= r_uint(stop - middle) self._add_free_block(middle, stop) return True else: @@ -77,7 +77,7 @@ # Hack to make sure that mcs are not within 32-bits of one # another for testing purposes rmmap.hint.pos += 0x80000000 - size - self.total_memory_allocated += size + self.total_memory_allocated += r_uint(size) data = rffi.cast(lltype.Signed, data) return self._add_free_block(data, data + size) diff --git a/pypy/jit/backend/llsupport/descr.py b/pypy/jit/backend/llsupport/descr.py --- a/pypy/jit/backend/llsupport/descr.py +++ b/pypy/jit/backend/llsupport/descr.py @@ -5,11 +5,7 @@ from pypy.jit.metainterp.history import AbstractDescr, getkind from pypy.jit.metainterp import history from pypy.jit.codewriter import heaptracker, longlong - -# The point of the class organization in this file is to make instances -# as compact as possible. This is done by not storing the field size or -# the 'is_pointer_field' flag in the instance itself but in the class -# (in methods actually) using a few classes instead of just one. +from pypy.jit.codewriter.longlong import is_longlong class GcCache(object): @@ -19,6 +15,7 @@ self._cache_size = {} self._cache_field = {} self._cache_array = {} + self._cache_arraylen = {} self._cache_call = {} self._cache_interiorfield = {} @@ -26,24 +23,15 @@ assert isinstance(STRUCT, lltype.GcStruct) def init_array_descr(self, ARRAY, arraydescr): - assert isinstance(ARRAY, lltype.GcArray) + assert (isinstance(ARRAY, lltype.GcArray) or + isinstance(ARRAY, lltype.GcStruct) and ARRAY._arrayfld) -if lltype.SignedLongLong is lltype.Signed: - def is_longlong(TYPE): - return False -else: - assert rffi.sizeof(lltype.SignedLongLong) == rffi.sizeof(lltype.Float) - def is_longlong(TYPE): - return TYPE in (lltype.SignedLongLong, lltype.UnsignedLongLong) - # ____________________________________________________________ # SizeDescrs class SizeDescr(AbstractDescr): size = 0 # help translation - is_immutable = False - tid = llop.combine_ushort(lltype.Signed, 0, 0) def __init__(self, size, count_fields_if_immut=-1): @@ -77,265 +65,247 @@ cache[STRUCT] = sizedescr return sizedescr + # ____________________________________________________________ # FieldDescrs -class BaseFieldDescr(AbstractDescr): +FLAG_POINTER = 'P' +FLAG_FLOAT = 'F' +FLAG_UNSIGNED = 'U' +FLAG_SIGNED = 'S' +FLAG_STRUCT = 'X' +FLAG_VOID = 'V' + +class FieldDescr(AbstractDescr): + name = '' offset = 0 # help translation - name = '' - _clsname = '' + field_size = 0 + flag = '\x00' - def __init__(self, name, offset): + def __init__(self, name, offset, field_size, flag): self.name = name self.offset = offset + self.field_size = field_size + self.flag = flag + + def is_pointer_field(self): + return self.flag == FLAG_POINTER + + def is_float_field(self): + return self.flag == FLAG_FLOAT + + def is_field_signed(self): + return self.flag == FLAG_SIGNED def sort_key(self): return self.offset - def get_field_size(self, translate_support_code): - raise NotImplementedError + def repr_of_descr(self): + return '' % (self.flag, self.name, self.offset) - _is_pointer_field = False # unless overridden by GcPtrFieldDescr - _is_float_field = False # unless overridden by FloatFieldDescr - _is_field_signed = False # unless overridden by XxxFieldDescr - - def is_pointer_field(self): - return self._is_pointer_field - - def is_float_field(self): - return self._is_float_field - - def is_field_signed(self): - return self._is_field_signed - - def repr_of_descr(self): - return '<%s %s %s>' % (self._clsname, self.name, self.offset) - -class DynamicFieldDescr(BaseFieldDescr): - def __init__(self, offset, fieldsize, is_pointer, is_float, is_signed): - self.offset = offset - self._fieldsize = fieldsize - self._is_pointer_field = is_pointer - self._is_float_field = is_float - self._is_field_signed = is_signed - - def get_field_size(self, translate_support_code): - return self._fieldsize - -class NonGcPtrFieldDescr(BaseFieldDescr): - _clsname = 'NonGcPtrFieldDescr' - def get_field_size(self, translate_support_code): - return symbolic.get_size_of_ptr(translate_support_code) - -class GcPtrFieldDescr(NonGcPtrFieldDescr): - _clsname = 'GcPtrFieldDescr' - _is_pointer_field = True - -def getFieldDescrClass(TYPE): - return getDescrClass(TYPE, BaseFieldDescr, GcPtrFieldDescr, - NonGcPtrFieldDescr, 'Field', 'get_field_size', - '_is_float_field', '_is_field_signed') def get_field_descr(gccache, STRUCT, fieldname): cache = gccache._cache_field try: return cache[STRUCT][fieldname] except KeyError: - offset, _ = symbolic.get_field_token(STRUCT, fieldname, - gccache.translate_support_code) + offset, size = symbolic.get_field_token(STRUCT, fieldname, + gccache.translate_support_code) FIELDTYPE = getattr(STRUCT, fieldname) + flag = get_type_flag(FIELDTYPE) name = '%s.%s' % (STRUCT._name, fieldname) - fielddescr = getFieldDescrClass(FIELDTYPE)(name, offset) + fielddescr = FieldDescr(name, offset, size, flag) cachedict = cache.setdefault(STRUCT, {}) cachedict[fieldname] = fielddescr return fielddescr +def get_type_flag(TYPE): + if isinstance(TYPE, lltype.Ptr): + if TYPE.TO._gckind == 'gc': + return FLAG_POINTER + else: + return FLAG_UNSIGNED + if isinstance(TYPE, lltype.Struct): + return FLAG_STRUCT + if TYPE is lltype.Float or is_longlong(TYPE): + return FLAG_FLOAT + if (TYPE is not lltype.Bool and isinstance(TYPE, lltype.Number) and + rffi.cast(TYPE, -1) == -1): + return FLAG_SIGNED + return FLAG_UNSIGNED + +def get_field_arraylen_descr(gccache, ARRAY_OR_STRUCT): + cache = gccache._cache_arraylen + try: + return cache[ARRAY_OR_STRUCT] + except KeyError: + tsc = gccache.translate_support_code + (_, _, ofs) = symbolic.get_array_token(ARRAY_OR_STRUCT, tsc) + size = symbolic.get_size(lltype.Signed, tsc) + result = FieldDescr("len", ofs, size, get_type_flag(lltype.Signed)) + cache[ARRAY_OR_STRUCT] = result + return result + + # ____________________________________________________________ # ArrayDescrs -_A = lltype.GcArray(lltype.Signed) # a random gcarray -_AF = lltype.GcArray(lltype.Float) # an array of C doubles +class ArrayDescr(AbstractDescr): + tid = 0 + basesize = 0 # workaround for the annotator + itemsize = 0 + lendescr = None + flag = '\x00' - -class BaseArrayDescr(AbstractDescr): - _clsname = '' - tid = llop.combine_ushort(lltype.Signed, 0, 0) - - def get_base_size(self, translate_support_code): - basesize, _, _ = symbolic.get_array_token(_A, translate_support_code) - return basesize - - def get_ofs_length(self, translate_support_code): - _, _, ofslength = symbolic.get_array_token(_A, translate_support_code) - return ofslength - - def get_item_size(self, translate_support_code): - raise NotImplementedError - - _is_array_of_pointers = False # unless overridden by GcPtrArrayDescr - _is_array_of_floats = False # unless overridden by FloatArrayDescr - _is_array_of_structs = False # unless overridden by StructArrayDescr - _is_item_signed = False # unless overridden by XxxArrayDescr + def __init__(self, basesize, itemsize, lendescr, flag): + self.basesize = basesize + self.itemsize = itemsize + self.lendescr = lendescr # or None, if no length + self.flag = flag def is_array_of_pointers(self): - return self._is_array_of_pointers + return self.flag == FLAG_POINTER def is_array_of_floats(self): - return self._is_array_of_floats + return self.flag == FLAG_FLOAT + + def is_item_signed(self): + return self.flag == FLAG_SIGNED def is_array_of_structs(self): - return self._is_array_of_structs - - def is_item_signed(self): - return self._is_item_signed + return self.flag == FLAG_STRUCT def repr_of_descr(self): - return '<%s>' % self._clsname + return '' % (self.flag, self.itemsize) -class NonGcPtrArrayDescr(BaseArrayDescr): - _clsname = 'NonGcPtrArrayDescr' - def get_item_size(self, translate_support_code): - return symbolic.get_size_of_ptr(translate_support_code) - -class GcPtrArrayDescr(NonGcPtrArrayDescr): - _clsname = 'GcPtrArrayDescr' - _is_array_of_pointers = True - -class FloatArrayDescr(BaseArrayDescr): - _clsname = 'FloatArrayDescr' - _is_array_of_floats = True - def get_base_size(self, translate_support_code): - basesize, _, _ = symbolic.get_array_token(_AF, translate_support_code) - return basesize - def get_item_size(self, translate_support_code): - return symbolic.get_size(lltype.Float, translate_support_code) - -class StructArrayDescr(BaseArrayDescr): - _clsname = 'StructArrayDescr' - _is_array_of_structs = True - -class BaseArrayNoLengthDescr(BaseArrayDescr): - def get_base_size(self, translate_support_code): - return 0 - - def get_ofs_length(self, translate_support_code): - return -1 - -class DynamicArrayNoLengthDescr(BaseArrayNoLengthDescr): - def __init__(self, itemsize): - self.itemsize = itemsize - - def get_item_size(self, translate_support_code): - return self.itemsize - -class NonGcPtrArrayNoLengthDescr(BaseArrayNoLengthDescr): - _clsname = 'NonGcPtrArrayNoLengthDescr' - def get_item_size(self, translate_support_code): - return symbolic.get_size_of_ptr(translate_support_code) - -class GcPtrArrayNoLengthDescr(NonGcPtrArrayNoLengthDescr): - _clsname = 'GcPtrArrayNoLengthDescr' - _is_array_of_pointers = True - -def getArrayDescrClass(ARRAY): - if ARRAY.OF is lltype.Float: - return FloatArrayDescr - elif isinstance(ARRAY.OF, lltype.Struct): - class Descr(StructArrayDescr): - _clsname = '%sArrayDescr' % ARRAY.OF._name - def get_item_size(self, translate_support_code): - return symbolic.get_size(ARRAY.OF, translate_support_code) - Descr.__name__ = Descr._clsname - return Descr - return getDescrClass(ARRAY.OF, BaseArrayDescr, GcPtrArrayDescr, - NonGcPtrArrayDescr, 'Array', 'get_item_size', - '_is_array_of_floats', '_is_item_signed') - -def getArrayNoLengthDescrClass(ARRAY): - return getDescrClass(ARRAY.OF, BaseArrayNoLengthDescr, GcPtrArrayNoLengthDescr, - NonGcPtrArrayNoLengthDescr, 'ArrayNoLength', 'get_item_size', - '_is_array_of_floats', '_is_item_signed') - -def get_array_descr(gccache, ARRAY): +def get_array_descr(gccache, ARRAY_OR_STRUCT): cache = gccache._cache_array try: - return cache[ARRAY] + return cache[ARRAY_OR_STRUCT] except KeyError: - # we only support Arrays that are either GcArrays, or raw no-length - # non-gc Arrays. - if ARRAY._hints.get('nolength', False): - assert not isinstance(ARRAY, lltype.GcArray) - arraydescr = getArrayNoLengthDescrClass(ARRAY)() + tsc = gccache.translate_support_code + basesize, itemsize, _ = symbolic.get_array_token(ARRAY_OR_STRUCT, tsc) + if isinstance(ARRAY_OR_STRUCT, lltype.Array): + ARRAY_INSIDE = ARRAY_OR_STRUCT else: - assert isinstance(ARRAY, lltype.GcArray) - arraydescr = getArrayDescrClass(ARRAY)() - # verify basic assumption that all arrays' basesize and ofslength - # are equal - basesize, itemsize, ofslength = symbolic.get_array_token(ARRAY, False) - assert basesize == arraydescr.get_base_size(False) - assert itemsize == arraydescr.get_item_size(False) - if not ARRAY._hints.get('nolength', False): - assert ofslength == arraydescr.get_ofs_length(False) - if isinstance(ARRAY, lltype.GcArray): - gccache.init_array_descr(ARRAY, arraydescr) - cache[ARRAY] = arraydescr + ARRAY_INSIDE = ARRAY_OR_STRUCT._flds[ARRAY_OR_STRUCT._arrayfld] + if ARRAY_INSIDE._hints.get('nolength', False): + lendescr = None + else: + lendescr = get_field_arraylen_descr(gccache, ARRAY_OR_STRUCT) + flag = get_type_flag(ARRAY_INSIDE.OF) + arraydescr = ArrayDescr(basesize, itemsize, lendescr, flag) + if ARRAY_OR_STRUCT._gckind == 'gc': + gccache.init_array_descr(ARRAY_OR_STRUCT, arraydescr) + cache[ARRAY_OR_STRUCT] = arraydescr return arraydescr + # ____________________________________________________________ # InteriorFieldDescr class InteriorFieldDescr(AbstractDescr): - arraydescr = BaseArrayDescr() # workaround for the annotator - fielddescr = BaseFieldDescr('', 0) + arraydescr = ArrayDescr(0, 0, None, '\x00') # workaround for the annotator + fielddescr = FieldDescr('', 0, 0, '\x00') def __init__(self, arraydescr, fielddescr): + assert arraydescr.flag == FLAG_STRUCT self.arraydescr = arraydescr self.fielddescr = fielddescr + def sort_key(self): + return self.fielddescr.sort_key() + def is_pointer_field(self): return self.fielddescr.is_pointer_field() def is_float_field(self): return self.fielddescr.is_float_field() - def sort_key(self): - return self.fielddescr.sort_key() - def repr_of_descr(self): return '' % self.fielddescr.repr_of_descr() -def get_interiorfield_descr(gc_ll_descr, ARRAY, FIELDTP, name): +def get_interiorfield_descr(gc_ll_descr, ARRAY, name): cache = gc_ll_descr._cache_interiorfield try: - return cache[(ARRAY, FIELDTP, name)] + return cache[(ARRAY, name)] except KeyError: arraydescr = get_array_descr(gc_ll_descr, ARRAY) - fielddescr = get_field_descr(gc_ll_descr, FIELDTP, name) + fielddescr = get_field_descr(gc_ll_descr, ARRAY.OF, name) descr = InteriorFieldDescr(arraydescr, fielddescr) - cache[(ARRAY, FIELDTP, name)] = descr + cache[(ARRAY, name)] = descr return descr +def get_dynamic_interiorfield_descr(gc_ll_descr, offset, width, fieldsize, + is_pointer, is_float, is_signed): + arraydescr = ArrayDescr(0, width, None, FLAG_STRUCT) + if is_pointer: + assert not is_float + flag = FLAG_POINTER + elif is_float: + flag = FLAG_FLOAT + elif is_signed: + flag = FLAG_SIGNED + else: + flag = FLAG_UNSIGNED + fielddescr = FieldDescr('dynamic', offset, fieldsize, flag) + return InteriorFieldDescr(arraydescr, fielddescr) + + # ____________________________________________________________ # CallDescrs -class BaseCallDescr(AbstractDescr): - _clsname = '' - loop_token = None +class CallDescr(AbstractDescr): arg_classes = '' # <-- annotation hack + result_type = '\x00' + result_flag = '\x00' ffi_flags = 1 + call_stub_i = staticmethod(lambda func, args_i, args_r, args_f: + 0) + call_stub_r = staticmethod(lambda func, args_i, args_r, args_f: + lltype.nullptr(llmemory.GCREF.TO)) + call_stub_f = staticmethod(lambda func,args_i,args_r,args_f: + longlong.ZEROF) - def __init__(self, arg_classes, extrainfo=None, ffi_flags=1): - self.arg_classes = arg_classes # string of "r" and "i" (ref/int) + def __init__(self, arg_classes, result_type, result_signed, result_size, + extrainfo=None, ffi_flags=1): + """ + 'arg_classes' is a string of characters, one per argument: + 'i', 'r', 'f', 'L', 'S' + + 'result_type' is one character from the same list or 'v' + + 'result_signed' is a boolean True/False + """ + self.arg_classes = arg_classes + self.result_type = result_type + self.result_size = result_size self.extrainfo = extrainfo self.ffi_flags = ffi_flags # NB. the default ffi_flags is 1, meaning FUNCFLAG_CDECL, which # makes sense on Windows as it's the one for all the C functions # we are compiling together with the JIT. On non-Windows platforms # it is just ignored anyway. + if result_type == 'v': + result_flag = FLAG_VOID + elif result_type == 'i': + if result_signed: + result_flag = FLAG_SIGNED + else: + result_flag = FLAG_UNSIGNED + elif result_type == history.REF: + result_flag = FLAG_POINTER + elif result_type == history.FLOAT or result_type == 'L': + result_flag = FLAG_FLOAT + elif result_type == 'S': + result_flag = FLAG_UNSIGNED + else: + raise NotImplementedError("result_type = '%s'" % (result_type,)) + self.result_flag = result_flag def __repr__(self): - res = '%s(%s)' % (self.__class__.__name__, self.arg_classes) + res = 'CallDescr(%s)' % (self.arg_classes,) extraeffect = getattr(self.extrainfo, 'extraeffect', None) if extraeffect is not None: res += ' EF=%r' % extraeffect @@ -363,14 +333,14 @@ def get_arg_types(self): return self.arg_classes - def get_return_type(self): - return self._return_type + def get_result_type(self): + return self.result_type - def get_result_size(self, translate_support_code): - raise NotImplementedError + def get_result_size(self): + return self.result_size def is_result_signed(self): - return False # unless overridden + return self.result_flag == FLAG_SIGNED def create_call_stub(self, rtyper, RESULT): from pypy.rlib.clibffi import FFI_DEFAULT_ABI @@ -408,18 +378,26 @@ seen = {'i': 0, 'r': 0, 'f': 0} args = ", ".join([process(c) for c in self.arg_classes]) - if self.get_return_type() == history.INT: + result_type = self.get_result_type() + if result_type == history.INT: result = 'rffi.cast(lltype.Signed, res)' - elif self.get_return_type() == history.REF: + category = 'i' + elif result_type == history.REF: + assert RESULT == llmemory.GCREF # should be ensured by the caller result = 'lltype.cast_opaque_ptr(llmemory.GCREF, res)' - elif self.get_return_type() == history.FLOAT: + category = 'r' + elif result_type == history.FLOAT: result = 'longlong.getfloatstorage(res)' - elif self.get_return_type() == 'L': + category = 'f' + elif result_type == 'L': result = 'rffi.cast(lltype.SignedLongLong, res)' - elif self.get_return_type() == history.VOID: - result = 'None' - elif self.get_return_type() == 'S': + category = 'f' + elif result_type == history.VOID: + result = '0' + category = 'i' + elif result_type == 'S': result = 'longlong.singlefloat2int(res)' + category = 'i' else: assert 0 source = py.code.Source(""" @@ -433,10 +411,13 @@ d = globals().copy() d.update(locals()) exec source.compile() in d - self.call_stub = d['call_stub'] + call_stub = d['call_stub'] + # store the function into one of three attributes, to preserve + # type-correctness of the return value + setattr(self, 'call_stub_%s' % category, call_stub) def verify_types(self, args_i, args_r, args_f, return_type): - assert self._return_type in return_type + assert self.result_type in return_type assert (self.arg_classes.count('i') + self.arg_classes.count('S')) == len(args_i or ()) assert self.arg_classes.count('r') == len(args_r or ()) @@ -444,161 +425,56 @@ self.arg_classes.count('L')) == len(args_f or ()) def repr_of_descr(self): - return '<%s>' % self._clsname + res = 'Call%s %d' % (self.result_type, self.result_size) + if self.arg_classes: + res += ' ' + self.arg_classes + if self.extrainfo: + res += ' EF=%d' % self.extrainfo.extraeffect + oopspecindex = self.extrainfo.oopspecindex + if oopspecindex: + res += ' OS=%d' % oopspecindex + return '<%s>' % res -class BaseIntCallDescr(BaseCallDescr): - # Base class of the various subclasses of descrs corresponding to - # calls having a return kind of 'int' (including non-gc pointers). - # The inheritance hierarchy is a bit different than with other Descr - # classes because of the 'call_stub' attribute, which is of type - # - # lambda func, args_i, args_r, args_f --> int/ref/float/void - # - # The purpose of BaseIntCallDescr is to be the parent of all classes - # in which 'call_stub' has a return kind of 'int'. - _return_type = history.INT - call_stub = staticmethod(lambda func, args_i, args_r, args_f: 0) - - _is_result_signed = False # can be overridden in XxxCallDescr - def is_result_signed(self): - return self._is_result_signed - -class DynamicIntCallDescr(BaseIntCallDescr): - """ - calldescr that works for every integer type, by explicitly passing it the - size of the result. Used only by get_call_descr_dynamic - """ - _clsname = 'DynamicIntCallDescr' - - def __init__(self, arg_classes, result_size, result_sign, extrainfo, ffi_flags): - BaseIntCallDescr.__init__(self, arg_classes, extrainfo, ffi_flags) - assert isinstance(result_sign, bool) - self._result_size = chr(result_size) - self._result_sign = result_sign - - def get_result_size(self, translate_support_code): - return ord(self._result_size) - - def is_result_signed(self): - return self._result_sign - - -class NonGcPtrCallDescr(BaseIntCallDescr): - _clsname = 'NonGcPtrCallDescr' - def get_result_size(self, translate_support_code): - return symbolic.get_size_of_ptr(translate_support_code) - -class GcPtrCallDescr(BaseCallDescr): - _clsname = 'GcPtrCallDescr' - _return_type = history.REF - call_stub = staticmethod(lambda func, args_i, args_r, args_f: - lltype.nullptr(llmemory.GCREF.TO)) - def get_result_size(self, translate_support_code): - return symbolic.get_size_of_ptr(translate_support_code) - -class FloatCallDescr(BaseCallDescr): - _clsname = 'FloatCallDescr' - _return_type = history.FLOAT - call_stub = staticmethod(lambda func,args_i,args_r,args_f: longlong.ZEROF) - def get_result_size(self, translate_support_code): - return symbolic.get_size(lltype.Float, translate_support_code) - -class LongLongCallDescr(FloatCallDescr): - _clsname = 'LongLongCallDescr' - _return_type = 'L' - -class VoidCallDescr(BaseCallDescr): - _clsname = 'VoidCallDescr' - _return_type = history.VOID - call_stub = staticmethod(lambda func, args_i, args_r, args_f: None) - def get_result_size(self, translate_support_code): - return 0 - -_SingleFloatCallDescr = None # built lazily - -def getCallDescrClass(RESULT): - if RESULT is lltype.Void: - return VoidCallDescr - if RESULT is lltype.Float: - return FloatCallDescr - if RESULT is lltype.SingleFloat: - global _SingleFloatCallDescr - if _SingleFloatCallDescr is None: - assert rffi.sizeof(rffi.UINT) == rffi.sizeof(RESULT) - class SingleFloatCallDescr(getCallDescrClass(rffi.UINT)): - _clsname = 'SingleFloatCallDescr' - _return_type = 'S' - _SingleFloatCallDescr = SingleFloatCallDescr - return _SingleFloatCallDescr - if is_longlong(RESULT): - return LongLongCallDescr - return getDescrClass(RESULT, BaseIntCallDescr, GcPtrCallDescr, - NonGcPtrCallDescr, 'Call', 'get_result_size', - Ellipsis, # <= floatattrname should not be used here - '_is_result_signed') -getCallDescrClass._annspecialcase_ = 'specialize:memo' +def map_type_to_argclass(ARG, accept_void=False): + kind = getkind(ARG) + if kind == 'int': + if ARG is lltype.SingleFloat: return 'S' + else: return 'i' + elif kind == 'ref': return 'r' + elif kind == 'float': + if is_longlong(ARG): return 'L' + else: return 'f' + elif kind == 'void': + if accept_void: return 'v' + raise NotImplementedError('ARG = %r' % (ARG,)) def get_call_descr(gccache, ARGS, RESULT, extrainfo=None): - arg_classes = [] - for ARG in ARGS: - kind = getkind(ARG) - if kind == 'int': - if ARG is lltype.SingleFloat: - arg_classes.append('S') + arg_classes = map(map_type_to_argclass, ARGS) + arg_classes = ''.join(arg_classes) + result_type = map_type_to_argclass(RESULT, accept_void=True) + RESULT_ERASED = RESULT + if RESULT is lltype.Void: + result_size = 0 + result_signed = False + else: + if isinstance(RESULT, lltype.Ptr): + # avoid too many CallDescrs + if result_type == 'r': + RESULT_ERASED = llmemory.GCREF else: - arg_classes.append('i') - elif kind == 'ref': arg_classes.append('r') - elif kind == 'float': - if is_longlong(ARG): - arg_classes.append('L') - else: - arg_classes.append('f') - else: - raise NotImplementedError('ARG = %r' % (ARG,)) - arg_classes = ''.join(arg_classes) - cls = getCallDescrClass(RESULT) - key = (cls, arg_classes, extrainfo) + RESULT_ERASED = llmemory.Address + result_size = symbolic.get_size(RESULT_ERASED, + gccache.translate_support_code) + result_signed = get_type_flag(RESULT) == FLAG_SIGNED + key = (arg_classes, result_type, result_signed, RESULT_ERASED, extrainfo) cache = gccache._cache_call try: - return cache[key] + calldescr = cache[key] except KeyError: - calldescr = cls(arg_classes, extrainfo) - calldescr.create_call_stub(gccache.rtyper, RESULT) + calldescr = CallDescr(arg_classes, result_type, result_signed, + result_size, extrainfo) + calldescr.create_call_stub(gccache.rtyper, RESULT_ERASED) cache[key] = calldescr - return calldescr - - -# ____________________________________________________________ - -def getDescrClass(TYPE, BaseDescr, GcPtrDescr, NonGcPtrDescr, - nameprefix, methodname, floatattrname, signedattrname, - _cache={}): - if isinstance(TYPE, lltype.Ptr): - if TYPE.TO._gckind == 'gc': - return GcPtrDescr - else: - return NonGcPtrDescr - if TYPE is lltype.SingleFloat: - assert rffi.sizeof(rffi.UINT) == rffi.sizeof(TYPE) - TYPE = rffi.UINT - try: - return _cache[nameprefix, TYPE] - except KeyError: - # - class Descr(BaseDescr): - _clsname = '%s%sDescr' % (TYPE._name, nameprefix) - Descr.__name__ = Descr._clsname - # - def method(self, translate_support_code): - return symbolic.get_size(TYPE, translate_support_code) - setattr(Descr, methodname, method) - # - if TYPE is lltype.Float or is_longlong(TYPE): - setattr(Descr, floatattrname, True) - elif (TYPE is not lltype.Bool and isinstance(TYPE, lltype.Number) and - rffi.cast(TYPE, -1) == -1): - setattr(Descr, signedattrname, True) - # - _cache[nameprefix, TYPE] = Descr - return Descr + assert repr(calldescr.result_size) == repr(result_size) + return calldescr diff --git a/pypy/jit/backend/llsupport/ffisupport.py b/pypy/jit/backend/llsupport/ffisupport.py --- a/pypy/jit/backend/llsupport/ffisupport.py +++ b/pypy/jit/backend/llsupport/ffisupport.py @@ -1,9 +1,7 @@ from pypy.rlib.rarithmetic import intmask from pypy.jit.metainterp import history from pypy.rpython.lltypesystem import rffi -from pypy.jit.backend.llsupport.descr import ( - DynamicIntCallDescr, NonGcPtrCallDescr, FloatCallDescr, VoidCallDescr, - LongLongCallDescr, getCallDescrClass) +from pypy.jit.backend.llsupport.descr import CallDescr class UnsupportedKind(Exception): pass @@ -16,29 +14,13 @@ argkinds = [get_ffi_type_kind(cpu, arg) for arg in ffi_args] except UnsupportedKind: return None - arg_classes = ''.join(argkinds) - if reskind == history.INT: - size = intmask(ffi_result.c_size) - signed = is_ffi_type_signed(ffi_result) - return DynamicIntCallDescr(arg_classes, size, signed, extrainfo, - ffi_flags=ffi_flags) - elif reskind == history.REF: - return NonGcPtrCallDescr(arg_classes, extrainfo, - ffi_flags=ffi_flags) - elif reskind == history.FLOAT: - return FloatCallDescr(arg_classes, extrainfo, - ffi_flags=ffi_flags) - elif reskind == history.VOID: - return VoidCallDescr(arg_classes, extrainfo, - ffi_flags=ffi_flags) - elif reskind == 'L': - return LongLongCallDescr(arg_classes, extrainfo, - ffi_flags=ffi_flags) - elif reskind == 'S': - SingleFloatCallDescr = getCallDescrClass(rffi.FLOAT) - return SingleFloatCallDescr(arg_classes, extrainfo, - ffi_flags=ffi_flags) - assert False + if reskind == history.VOID: + result_size = 0 + else: + result_size = intmask(ffi_result.c_size) + argkinds = ''.join(argkinds) + return CallDescr(argkinds, reskind, is_ffi_type_signed(ffi_result), + result_size, extrainfo, ffi_flags=ffi_flags) def get_ffi_type_kind(cpu, ffi_type): from pypy.rlib.libffi import types diff --git a/pypy/jit/backend/llsupport/gc.py b/pypy/jit/backend/llsupport/gc.py --- a/pypy/jit/backend/llsupport/gc.py +++ b/pypy/jit/backend/llsupport/gc.py @@ -1,6 +1,6 @@ import os from pypy.rlib import rgc -from pypy.rlib.objectmodel import we_are_translated +from pypy.rlib.objectmodel import we_are_translated, specialize from pypy.rlib.debug import fatalerror from pypy.rlib.rarithmetic import ovfcheck from pypy.rpython.lltypesystem import lltype, llmemory, rffi, rclass, rstr @@ -8,52 +8,93 @@ from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.annlowlevel import llhelper from pypy.translator.tool.cbuild import ExternalCompilationInfo -from pypy.jit.metainterp.history import BoxInt, BoxPtr, ConstInt, ConstPtr -from pypy.jit.metainterp.history import AbstractDescr +from pypy.jit.codewriter import heaptracker +from pypy.jit.metainterp.history import ConstPtr, AbstractDescr from pypy.jit.metainterp.resoperation import ResOperation, rop from pypy.jit.backend.llsupport import symbolic from pypy.jit.backend.llsupport.symbolic import WORD -from pypy.jit.backend.llsupport.descr import BaseSizeDescr, BaseArrayDescr +from pypy.jit.backend.llsupport.descr import SizeDescr, ArrayDescr from pypy.jit.backend.llsupport.descr import GcCache, get_field_descr -from pypy.jit.backend.llsupport.descr import GcPtrFieldDescr +from pypy.jit.backend.llsupport.descr import get_array_descr from pypy.jit.backend.llsupport.descr import get_call_descr +from pypy.jit.backend.llsupport.rewrite import GcRewriterAssembler from pypy.rpython.memory.gctransform import asmgcroot # ____________________________________________________________ class GcLLDescription(GcCache): - minimal_size_in_nursery = 0 - get_malloc_slowpath_addr = None def __init__(self, gcdescr, translator=None, rtyper=None): GcCache.__init__(self, translator is not None, rtyper) self.gcdescr = gcdescr + if translator and translator.config.translation.gcremovetypeptr: + self.fielddescr_vtable = None + else: + self.fielddescr_vtable = get_field_descr(self, rclass.OBJECT, + 'typeptr') + self._generated_functions = [] + + def _setup_str(self): + self.str_descr = get_array_descr(self, rstr.STR) + self.unicode_descr = get_array_descr(self, rstr.UNICODE) + + def generate_function(self, funcname, func, ARGS, RESULT=llmemory.GCREF): + """Generates a variant of malloc with the given name and the given + arguments. It should return NULL if out of memory. If it raises + anything, it must be an optional MemoryError. + """ + FUNCPTR = lltype.Ptr(lltype.FuncType(ARGS, RESULT)) + descr = get_call_descr(self, ARGS, RESULT) + setattr(self, funcname, func) + setattr(self, funcname + '_FUNCPTR', FUNCPTR) + setattr(self, funcname + '_descr', descr) + self._generated_functions.append(funcname) + + @specialize.arg(1) + def get_malloc_fn(self, funcname): + func = getattr(self, funcname) + FUNC = getattr(self, funcname + '_FUNCPTR') + return llhelper(FUNC, func) + + @specialize.arg(1) + def get_malloc_fn_addr(self, funcname): + ll_func = self.get_malloc_fn(funcname) + return heaptracker.adr2int(llmemory.cast_ptr_to_adr(ll_func)) + def _freeze_(self): return True def initialize(self): pass def do_write_barrier(self, gcref_struct, gcref_newptr): pass - def rewrite_assembler(self, cpu, operations, gcrefs_output_list): - return operations - def can_inline_malloc(self, descr): - return False - def can_inline_malloc_varsize(self, descr, num_elem): + def can_use_nursery_malloc(self, size): return False def has_write_barrier_class(self): return None def freeing_block(self, start, stop): pass + def get_nursery_free_addr(self): + raise NotImplementedError + def get_nursery_top_addr(self): + raise NotImplementedError - def get_funcptr_for_newarray(self): - return llhelper(self.GC_MALLOC_ARRAY, self.malloc_array) - def get_funcptr_for_newstr(self): - return llhelper(self.GC_MALLOC_STR_UNICODE, self.malloc_str) - def get_funcptr_for_newunicode(self): - return llhelper(self.GC_MALLOC_STR_UNICODE, self.malloc_unicode) + def gc_malloc(self, sizedescr): + """Blackhole: do a 'bh_new'. Also used for 'bh_new_with_vtable', + with the vtable pointer set manually afterwards.""" + assert isinstance(sizedescr, SizeDescr) + return self._bh_malloc(sizedescr) + def gc_malloc_array(self, arraydescr, num_elem): + assert isinstance(arraydescr, ArrayDescr) + return self._bh_malloc_array(arraydescr, num_elem) - def record_constptrs(self, op, gcrefs_output_list): + def gc_malloc_str(self, num_elem): + return self._bh_malloc_array(self.str_descr, num_elem) + + def gc_malloc_unicode(self, num_elem): + return self._bh_malloc_array(self.unicode_descr, num_elem) + + def _record_constptrs(self, op, gcrefs_output_list): for i in range(op.numargs()): v = op.getarg(i) if isinstance(v, ConstPtr) and bool(v.value): @@ -61,11 +102,27 @@ rgc._make_sure_does_not_move(p) gcrefs_output_list.append(p) + def rewrite_assembler(self, cpu, operations, gcrefs_output_list): + rewriter = GcRewriterAssembler(self, cpu) + newops = rewriter.rewrite(operations) + # record all GCREFs, because the GC (or Boehm) cannot see them and + # keep them alive if they end up as constants in the assembler + for op in newops: + self._record_constptrs(op, gcrefs_output_list) + return newops + # ____________________________________________________________ class GcLLDescr_boehm(GcLLDescription): - moving_gc = False - gcrootmap = None + kind = 'boehm' + moving_gc = False + round_up = False + gcrootmap = None + write_barrier_descr = None + fielddescr_tid = None + str_type_id = 0 + unicode_type_id = 0 + get_malloc_slowpath_addr = None @classmethod def configure_boehm_once(cls): @@ -76,6 +133,16 @@ from pypy.rpython.tool import rffi_platform compilation_info = rffi_platform.configure_boehm() + # on some platform GC_init is required before any other + # GC_* functions, call it here for the benefit of tests + # XXX move this to tests + init_fn_ptr = rffi.llexternal("GC_init", + [], lltype.Void, + compilation_info=compilation_info, + sandboxsafe=True, + _nowrapper=True) + init_fn_ptr() + # Versions 6.x of libgc needs to use GC_local_malloc(). # Versions 7.x of libgc removed this function; GC_malloc() has # the same behavior if libgc was compiled with @@ -95,96 +162,42 @@ sandboxsafe=True, _nowrapper=True) cls.malloc_fn_ptr = malloc_fn_ptr - cls.compilation_info = compilation_info return malloc_fn_ptr def __init__(self, gcdescr, translator, rtyper): GcLLDescription.__init__(self, gcdescr, translator, rtyper) # grab a pointer to the Boehm 'malloc' function - malloc_fn_ptr = self.configure_boehm_once() - self.funcptr_for_new = malloc_fn_ptr + self.malloc_fn_ptr = self.configure_boehm_once() + self._setup_str() + self._make_functions() - def malloc_array(basesize, itemsize, ofs_length, num_elem): + def _make_functions(self): + + def malloc_fixedsize(size): + return self.malloc_fn_ptr(size) + self.generate_function('malloc_fixedsize', malloc_fixedsize, + [lltype.Signed]) + + def malloc_array(basesize, num_elem, itemsize, ofs_length): try: - size = ovfcheck(basesize + ovfcheck(itemsize * num_elem)) + totalsize = ovfcheck(basesize + ovfcheck(itemsize * num_elem)) except OverflowError: return lltype.nullptr(llmemory.GCREF.TO) - res = self.funcptr_for_new(size) - if not res: - return res - rffi.cast(rffi.CArrayPtr(lltype.Signed), res)[ofs_length/WORD] = num_elem + res = self.malloc_fn_ptr(totalsize) + if res: + arrayptr = rffi.cast(rffi.CArrayPtr(lltype.Signed), res) + arrayptr[ofs_length/WORD] = num_elem return res - self.malloc_array = malloc_array - self.GC_MALLOC_ARRAY = lltype.Ptr(lltype.FuncType( - [lltype.Signed] * 4, llmemory.GCREF)) + self.generate_function('malloc_array', malloc_array, + [lltype.Signed] * 4) + def _bh_malloc(self, sizedescr): + return self.malloc_fixedsize(sizedescr.size) - (str_basesize, str_itemsize, str_ofs_length - ) = symbolic.get_array_token(rstr.STR, self.translate_support_code) - (unicode_basesize, unicode_itemsize, unicode_ofs_length - ) = symbolic.get_array_token(rstr.UNICODE, self.translate_support_code) - def malloc_str(length): - return self.malloc_array( - str_basesize, str_itemsize, str_ofs_length, length - ) - def malloc_unicode(length): - return self.malloc_array( - unicode_basesize, unicode_itemsize, unicode_ofs_length, length - ) - self.malloc_str = malloc_str - self.malloc_unicode = malloc_unicode - self.GC_MALLOC_STR_UNICODE = lltype.Ptr(lltype.FuncType( - [lltype.Signed], llmemory.GCREF)) - - - # on some platform GC_init is required before any other - # GC_* functions, call it here for the benefit of tests - # XXX move this to tests - init_fn_ptr = rffi.llexternal("GC_init", - [], lltype.Void, - compilation_info=self.compilation_info, - sandboxsafe=True, - _nowrapper=True) - - init_fn_ptr() - - def gc_malloc(self, sizedescr): - assert isinstance(sizedescr, BaseSizeDescr) - return self.funcptr_for_new(sizedescr.size) - - def gc_malloc_array(self, arraydescr, num_elem): - assert isinstance(arraydescr, BaseArrayDescr) - ofs_length = arraydescr.get_ofs_length(self.translate_support_code) - basesize = arraydescr.get_base_size(self.translate_support_code) - itemsize = arraydescr.get_item_size(self.translate_support_code) - return self.malloc_array(basesize, itemsize, ofs_length, num_elem) - - def gc_malloc_str(self, num_elem): - return self.malloc_str(num_elem) - - def gc_malloc_unicode(self, num_elem): - return self.malloc_unicode(num_elem) - - def args_for_new(self, sizedescr): - assert isinstance(sizedescr, BaseSizeDescr) - return [sizedescr.size] - - def args_for_new_array(self, arraydescr): - ofs_length = arraydescr.get_ofs_length(self.translate_support_code) - basesize = arraydescr.get_base_size(self.translate_support_code) - itemsize = arraydescr.get_item_size(self.translate_support_code) - return [basesize, itemsize, ofs_length] - - def get_funcptr_for_new(self): - return self.funcptr_for_new - - def rewrite_assembler(self, cpu, operations, gcrefs_output_list): - # record all GCREFs too, because Boehm cannot see them and keep them - # alive if they end up as constants in the assembler - for op in operations: - self.record_constptrs(op, gcrefs_output_list) - return GcLLDescription.rewrite_assembler(self, cpu, operations, - gcrefs_output_list) + def _bh_malloc_array(self, arraydescr, num_elem): + return self.malloc_array(arraydescr.basesize, num_elem, + arraydescr.itemsize, + arraydescr.lendescr.offset) # ____________________________________________________________ @@ -554,12 +567,14 @@ class WriteBarrierDescr(AbstractDescr): def __init__(self, gc_ll_descr): - GCClass = gc_ll_descr.GCClass self.llop1 = gc_ll_descr.llop1 self.WB_FUNCPTR = gc_ll_descr.WB_FUNCPTR self.WB_ARRAY_FUNCPTR = gc_ll_descr.WB_ARRAY_FUNCPTR - self.fielddescr_tid = get_field_descr(gc_ll_descr, GCClass.HDR, 'tid') + self.fielddescr_tid = gc_ll_descr.fielddescr_tid # + GCClass = gc_ll_descr.GCClass + if GCClass is None: # for tests + return self.jit_wb_if_flag = GCClass.JIT_WB_IF_FLAG self.jit_wb_if_flag_byteofs, self.jit_wb_if_flag_singlebyte = ( self.extract_flag_byte(self.jit_wb_if_flag)) @@ -596,48 +611,74 @@ funcaddr = llmemory.cast_ptr_to_adr(funcptr) return cpu.cast_adr_to_int(funcaddr) # this may return 0 + def has_write_barrier_from_array(self, cpu): + return self.get_write_barrier_from_array_fn(cpu) != 0 + class GcLLDescr_framework(GcLLDescription): DEBUG = False # forced to True by x86/test/test_zrpy_gc.py + kind = 'framework' + round_up = True - def __init__(self, gcdescr, translator, rtyper, llop1=llop): - from pypy.rpython.memory.gctypelayout import check_typeid - from pypy.rpython.memory.gcheader import GCHeaderBuilder - from pypy.rpython.memory.gctransform import framework + def __init__(self, gcdescr, translator, rtyper, llop1=llop, + really_not_translated=False): GcLLDescription.__init__(self, gcdescr, translator, rtyper) - assert self.translate_support_code, "required with the framework GC" self.translator = translator self.llop1 = llop1 + if really_not_translated: + assert not self.translate_support_code # but half does not work + self._initialize_for_tests() + else: + assert self.translate_support_code,"required with the framework GC" + self._check_valid_gc() + self._make_gcrootmap() + self._make_layoutbuilder() + self._setup_gcclass() + self._setup_tid() + self._setup_write_barrier() + self._setup_str() + self._make_functions(really_not_translated) + def _initialize_for_tests(self): + self.layoutbuilder = None + self.fielddescr_tid = AbstractDescr() + self.max_size_of_young_obj = 1000 + self.GCClass = None + + def _check_valid_gc(self): # we need the hybrid or minimark GC for rgc._make_sure_does_not_move() # to work - if gcdescr.config.translation.gc not in ('hybrid', 'minimark'): + if self.gcdescr.config.translation.gc not in ('hybrid', 'minimark'): raise NotImplementedError("--gc=%s not implemented with the JIT" % (gcdescr.config.translation.gc,)) + def _make_gcrootmap(self): # to find roots in the assembler, make a GcRootMap - name = gcdescr.config.translation.gcrootfinder + name = self.gcdescr.config.translation.gcrootfinder try: cls = globals()['GcRootMap_' + name] except KeyError: raise NotImplementedError("--gcrootfinder=%s not implemented" " with the JIT" % (name,)) - gcrootmap = cls(gcdescr) + gcrootmap = cls(self.gcdescr) self.gcrootmap = gcrootmap + def _make_layoutbuilder(self): # make a TransformerLayoutBuilder and save it on the translator # where it can be fished and reused by the FrameworkGCTransformer + from pypy.rpython.memory.gctransform import framework + translator = self.translator self.layoutbuilder = framework.TransformerLayoutBuilder(translator) self.layoutbuilder.delay_encoding() - self.translator._jit2gc = {'layoutbuilder': self.layoutbuilder} - gcrootmap.add_jit2gc_hooks(self.translator._jit2gc) + translator._jit2gc = {'layoutbuilder': self.layoutbuilder} + self.gcrootmap.add_jit2gc_hooks(translator._jit2gc) + def _setup_gcclass(self): + from pypy.rpython.memory.gcheader import GCHeaderBuilder self.GCClass = self.layoutbuilder.GCClass self.moving_gc = self.GCClass.moving_gc self.HDRPTR = lltype.Ptr(self.GCClass.HDR) self.gcheaderbuilder = GCHeaderBuilder(self.HDRPTR.TO) - (self.array_basesize, _, self.array_length_ofs) = \ - symbolic.get_array_token(lltype.GcArray(lltype.Signed), True) self.max_size_of_young_obj = self.GCClass.JIT_max_size_of_young_obj() self.minimal_size_in_nursery=self.GCClass.JIT_minimal_size_in_nursery() @@ -645,87 +686,124 @@ assert self.GCClass.inline_simple_malloc assert self.GCClass.inline_simple_malloc_varsize - # make a malloc function, with two arguments - def malloc_basic(size, tid): - type_id = llop.extract_ushort(llgroup.HALFWORD, tid) - check_typeid(type_id) - res = llop1.do_malloc_fixedsize_clear(llmemory.GCREF, - type_id, size, - False, False, False) - # In case the operation above failed, we are returning NULL - # from this function to assembler. There is also an RPython - # exception set, typically MemoryError; but it's easier and - # faster to check for the NULL return value, as done by - # translator/exceptiontransform.py. - #llop.debug_print(lltype.Void, "\tmalloc_basic", size, type_id, - # "-->", res) - return res - self.malloc_basic = malloc_basic - self.GC_MALLOC_BASIC = lltype.Ptr(lltype.FuncType( - [lltype.Signed, lltype.Signed], llmemory.GCREF)) + def _setup_tid(self): + self.fielddescr_tid = get_field_descr(self, self.GCClass.HDR, 'tid') + + def _setup_write_barrier(self): self.WB_FUNCPTR = lltype.Ptr(lltype.FuncType( [llmemory.Address, llmemory.Address], lltype.Void)) self.WB_ARRAY_FUNCPTR = lltype.Ptr(lltype.FuncType( [llmemory.Address, lltype.Signed, llmemory.Address], lltype.Void)) self.write_barrier_descr = WriteBarrierDescr(self) - # + + def _make_functions(self, really_not_translated): + from pypy.rpython.memory.gctypelayout import check_typeid + llop1 = self.llop1 + (self.standard_array_basesize, _, self.standard_array_length_ofs) = \ + symbolic.get_array_token(lltype.GcArray(lltype.Signed), + not really_not_translated) + + def malloc_nursery_slowpath(size): + """Allocate 'size' null bytes out of the nursery. + Note that the fast path is typically inlined by the backend.""" + if self.DEBUG: + self._random_usage_of_xmm_registers() + type_id = rffi.cast(llgroup.HALFWORD, 0) # missing here + return llop1.do_malloc_fixedsize_clear(llmemory.GCREF, + type_id, size, + False, False, False) + self.generate_function('malloc_nursery', malloc_nursery_slowpath, + [lltype.Signed]) + def malloc_array(itemsize, tid, num_elem): + """Allocate an array with a variable-size num_elem. + Only works for standard arrays.""" type_id = llop.extract_ushort(llgroup.HALFWORD, tid) check_typeid(type_id) return llop1.do_malloc_varsize_clear( llmemory.GCREF, - type_id, num_elem, self.array_basesize, itemsize, - self.array_length_ofs) - self.malloc_array = malloc_array - self.GC_MALLOC_ARRAY = lltype.Ptr(lltype.FuncType( - [lltype.Signed] * 3, llmemory.GCREF)) - # - (str_basesize, str_itemsize, str_ofs_length - ) = symbolic.get_array_token(rstr.STR, True) - (unicode_basesize, unicode_itemsize, unicode_ofs_length - ) = symbolic.get_array_token(rstr.UNICODE, True) - str_type_id = self.layoutbuilder.get_type_id(rstr.STR) - unicode_type_id = self.layoutbuilder.get_type_id(rstr.UNICODE) - # + type_id, num_elem, self.standard_array_basesize, itemsize, + self.standard_array_length_ofs) + self.generate_function('malloc_array', malloc_array, + [lltype.Signed] * 3) + + def malloc_array_nonstandard(basesize, itemsize, lengthofs, tid, + num_elem): + """For the rare case of non-standard arrays, i.e. arrays where + self.standard_array_{basesize,length_ofs} is wrong. It can + occur e.g. with arrays of floats on Win32.""" + type_id = llop.extract_ushort(llgroup.HALFWORD, tid) + check_typeid(type_id) + return llop1.do_malloc_varsize_clear( + llmemory.GCREF, + type_id, num_elem, basesize, itemsize, lengthofs) + self.generate_function('malloc_array_nonstandard', + malloc_array_nonstandard, + [lltype.Signed] * 5) + + str_type_id = self.str_descr.tid + str_basesize = self.str_descr.basesize + str_itemsize = self.str_descr.itemsize + str_ofs_length = self.str_descr.lendescr.offset + unicode_type_id = self.unicode_descr.tid + unicode_basesize = self.unicode_descr.basesize + unicode_itemsize = self.unicode_descr.itemsize + unicode_ofs_length = self.unicode_descr.lendescr.offset + def malloc_str(length): return llop1.do_malloc_varsize_clear( llmemory.GCREF, str_type_id, length, str_basesize, str_itemsize, str_ofs_length) + self.generate_function('malloc_str', malloc_str, + [lltype.Signed]) + def malloc_unicode(length): return llop1.do_malloc_varsize_clear( llmemory.GCREF, - unicode_type_id, length, unicode_basesize,unicode_itemsize, + unicode_type_id, length, unicode_basesize, unicode_itemsize, unicode_ofs_length) - self.malloc_str = malloc_str - self.malloc_unicode = malloc_unicode - self.GC_MALLOC_STR_UNICODE = lltype.Ptr(lltype.FuncType( - [lltype.Signed], llmemory.GCREF)) - # - class ForTestOnly: - pass - for_test_only = ForTestOnly() - for_test_only.x = 1.23 - def random_usage_of_xmm_registers(): - x0 = for_test_only.x - x1 = x0 * 0.1 - x2 = x0 * 0.2 - x3 = x0 * 0.3 - for_test_only.x = x0 + x1 + x2 + x3 - # - def malloc_slowpath(size): - if self.DEBUG: - random_usage_of_xmm_registers() - assert size >= self.minimal_size_in_nursery - # NB. although we call do_malloc_fixedsize_clear() here, - # it's a bit of a hack because we set tid to 0 and may - # also use it to allocate varsized objects. The tid - # and possibly the length are both set afterward. - gcref = llop1.do_malloc_fixedsize_clear(llmemory.GCREF, - 0, size, False, False, False) - return rffi.cast(lltype.Signed, gcref) - self.malloc_slowpath = malloc_slowpath - self.MALLOC_SLOWPATH = lltype.FuncType([lltype.Signed], lltype.Signed) + self.generate_function('malloc_unicode', malloc_unicode, + [lltype.Signed]) + + # Rarely called: allocate a fixed-size amount of bytes, but + # not in the nursery, because it is too big. Implemented like + # malloc_nursery_slowpath() above. + self.generate_function('malloc_fixedsize', malloc_nursery_slowpath, + [lltype.Signed]) + + def _bh_malloc(self, sizedescr): + from pypy.rpython.memory.gctypelayout import check_typeid + llop1 = self.llop1 + type_id = llop.extract_ushort(llgroup.HALFWORD, sizedescr.tid) + check_typeid(type_id) + return llop1.do_malloc_fixedsize_clear(llmemory.GCREF, + type_id, sizedescr.size, + False, False, False) + + def _bh_malloc_array(self, arraydescr, num_elem): + from pypy.rpython.memory.gctypelayout import check_typeid + llop1 = self.llop1 + type_id = llop.extract_ushort(llgroup.HALFWORD, arraydescr.tid) + check_typeid(type_id) + return llop1.do_malloc_varsize_clear(llmemory.GCREF, + type_id, num_elem, + arraydescr.basesize, + arraydescr.itemsize, + arraydescr.lendescr.offset) + + + class ForTestOnly: + pass + for_test_only = ForTestOnly() + for_test_only.x = 1.23 + + def _random_usage_of_xmm_registers(self): + x0 = self.for_test_only.x + x1 = x0 * 0.1 + x2 = x0 * 0.2 + x3 = x0 * 0.3 + self.for_test_only.x = x0 + x1 + x2 + x3 def get_nursery_free_addr(self): nurs_addr = llop.gc_adr_of_nursery_free(llmemory.Address) @@ -735,49 +813,26 @@ nurs_top_addr = llop.gc_adr_of_nursery_top(llmemory.Address) return rffi.cast(lltype.Signed, nurs_top_addr) - def get_malloc_slowpath_addr(self): - fptr = llhelper(lltype.Ptr(self.MALLOC_SLOWPATH), self.malloc_slowpath) - return rffi.cast(lltype.Signed, fptr) - def initialize(self): self.gcrootmap.initialize() def init_size_descr(self, S, descr): - type_id = self.layoutbuilder.get_type_id(S) - assert not self.layoutbuilder.is_weakref_type(S) - assert not self.layoutbuilder.has_finalizer(S) - descr.tid = llop.combine_ushort(lltype.Signed, type_id, 0) + if self.layoutbuilder is not None: + type_id = self.layoutbuilder.get_type_id(S) + assert not self.layoutbuilder.is_weakref_type(S) + assert not self.layoutbuilder.has_finalizer(S) + descr.tid = llop.combine_ushort(lltype.Signed, type_id, 0) def init_array_descr(self, A, descr): - type_id = self.layoutbuilder.get_type_id(A) - descr.tid = llop.combine_ushort(lltype.Signed, type_id, 0) + if self.layoutbuilder is not None: + type_id = self.layoutbuilder.get_type_id(A) + descr.tid = llop.combine_ushort(lltype.Signed, type_id, 0) - def gc_malloc(self, sizedescr): - assert isinstance(sizedescr, BaseSizeDescr) - return self.malloc_basic(sizedescr.size, sizedescr.tid) - - def gc_malloc_array(self, arraydescr, num_elem): - assert isinstance(arraydescr, BaseArrayDescr) - itemsize = arraydescr.get_item_size(self.translate_support_code) - return self.malloc_array(itemsize, arraydescr.tid, num_elem) - - def gc_malloc_str(self, num_elem): - return self.malloc_str(num_elem) - - def gc_malloc_unicode(self, num_elem): - return self.malloc_unicode(num_elem) - - def args_for_new(self, sizedescr): - assert isinstance(sizedescr, BaseSizeDescr) - return [sizedescr.size, sizedescr.tid] - - def args_for_new_array(self, arraydescr): - assert isinstance(arraydescr, BaseArrayDescr) - itemsize = arraydescr.get_item_size(self.translate_support_code) - return [itemsize, arraydescr.tid] - - def get_funcptr_for_new(self): - return llhelper(self.GC_MALLOC_BASIC, self.malloc_basic) + def _set_tid(self, gcptr, tid): + hdr_addr = llmemory.cast_ptr_to_adr(gcptr) + hdr_addr -= self.gcheaderbuilder.size_gc_header + hdr = llmemory.cast_adr_to_ptr(hdr_addr, self.HDRPTR) + hdr.tid = tid def do_write_barrier(self, gcref_struct, gcref_newptr): hdr_addr = llmemory.cast_ptr_to_adr(gcref_struct) @@ -791,99 +846,8 @@ funcptr(llmemory.cast_ptr_to_adr(gcref_struct), llmemory.cast_ptr_to_adr(gcref_newptr)) - def rewrite_assembler(self, cpu, operations, gcrefs_output_list): - # Perform two kinds of rewrites in parallel: - # - # - Add COND_CALLs to the write barrier before SETFIELD_GC and - # SETARRAYITEM_GC operations. - # - # - Record the ConstPtrs from the assembler. - # - newops = [] - known_lengths = {} - # we can only remember one malloc since the next malloc can possibly - # collect - last_malloc = None - for op in operations: - if op.getopnum() == rop.DEBUG_MERGE_POINT: - continue - # ---------- record the ConstPtrs ---------- - self.record_constptrs(op, gcrefs_output_list) - if op.is_malloc(): - last_malloc = op.result - elif op.can_malloc(): - last_malloc = None - # ---------- write barrier for SETFIELD_GC ---------- - if op.getopnum() == rop.SETFIELD_GC: - val = op.getarg(0) - # no need for a write barrier in the case of previous malloc - if val is not last_malloc: - v = op.getarg(1) - if isinstance(v, BoxPtr) or (isinstance(v, ConstPtr) and - bool(v.value)): # store a non-NULL - self._gen_write_barrier(newops, op.getarg(0), v) - op = op.copy_and_change(rop.SETFIELD_RAW) - # ---------- write barrier for SETARRAYITEM_GC ---------- - if op.getopnum() == rop.SETARRAYITEM_GC: - val = op.getarg(0) - # no need for a write barrier in the case of previous malloc - if val is not last_malloc: - v = op.getarg(2) - if isinstance(v, BoxPtr) or (isinstance(v, ConstPtr) and - bool(v.value)): # store a non-NULL - self._gen_write_barrier_array(newops, op.getarg(0), - op.getarg(1), v, - cpu, known_lengths) - op = op.copy_and_change(rop.SETARRAYITEM_RAW) - elif op.getopnum() == rop.NEW_ARRAY: - v_length = op.getarg(0) - if isinstance(v_length, ConstInt): - known_lengths[op.result] = v_length.getint() - # ---------- - newops.append(op) - return newops - - def _gen_write_barrier(self, newops, v_base, v_value): - args = [v_base, v_value] - newops.append(ResOperation(rop.COND_CALL_GC_WB, args, None, - descr=self.write_barrier_descr)) - - def _gen_write_barrier_array(self, newops, v_base, v_index, v_value, - cpu, known_lengths): - if self.write_barrier_descr.get_write_barrier_from_array_fn(cpu) != 0: - # If we know statically the length of 'v', and it is not too - # big, then produce a regular write_barrier. If it's unknown or - # too big, produce instead a write_barrier_from_array. - LARGE = 130 - length = known_lengths.get(v_base, LARGE) - if length >= LARGE: - # unknown or too big: produce a write_barrier_from_array - args = [v_base, v_index, v_value] - newops.append(ResOperation(rop.COND_CALL_GC_WB_ARRAY, args, - None, - descr=self.write_barrier_descr)) - return - # fall-back case: produce a write_barrier - self._gen_write_barrier(newops, v_base, v_value) - - def can_inline_malloc(self, descr): - assert isinstance(descr, BaseSizeDescr) - if descr.size < self.max_size_of_young_obj: - has_finalizer = bool(descr.tid & (1<= len(self.used): + self.used.append(False) + if self.used[index + i]: + return False # already in use + # good, we can reuse the location + for i in range(size): + self.used[index + i] = True + self.bindings[box] = loc + return True + # abstract methods that need to be overwritten for specific assemblers @staticmethod def frame_pos(loc, type): @@ -49,6 +123,10 @@ @staticmethod def frame_size(type): return 1 + @staticmethod + def get_loc_index(loc): + raise NotImplementedError("Purely abstract") + class RegisterManager(object): """ Class that keeps track of register allocations @@ -68,7 +146,14 @@ self.frame_manager = frame_manager self.assembler = assembler + def is_still_alive(self, v): + # Check if 'v' is alive at the current position. + # Return False if the last usage is strictly before. + return self.longevity[v][1] >= self.position + def stays_alive(self, v): + # Check if 'v' stays alive after the current position. + # Return False if the last usage is before or at position. return self.longevity[v][1] > self.position def next_instruction(self, incr=1): @@ -84,11 +169,14 @@ point for all variables that might be in registers. """ self._check_type(v) - if isinstance(v, Const) or v not in self.reg_bindings: + if isinstance(v, Const): return if v not in self.longevity or self.longevity[v][1] <= self.position: - self.free_regs.append(self.reg_bindings[v]) - del self.reg_bindings[v] + if v in self.reg_bindings: + self.free_regs.append(self.reg_bindings[v]) + del self.reg_bindings[v] + if self.frame_manager is not None: + self.frame_manager.mark_as_free(v) def possibly_free_vars(self, vars): """ Same as 'possibly_free_var', but for all v in vars. diff --git a/pypy/jit/backend/llsupport/rewrite.py b/pypy/jit/backend/llsupport/rewrite.py new file mode 100644 --- /dev/null +++ b/pypy/jit/backend/llsupport/rewrite.py @@ -0,0 +1,328 @@ +import sys +from pypy.rlib.rarithmetic import ovfcheck +from pypy.jit.metainterp.history import ConstInt, BoxPtr, ConstPtr +from pypy.jit.metainterp.resoperation import ResOperation, rop +from pypy.jit.codewriter import heaptracker +from pypy.jit.backend.llsupport.symbolic import WORD +from pypy.jit.backend.llsupport.descr import SizeDescr, ArrayDescr + + +class GcRewriterAssembler(object): + # This class performs the following rewrites on the list of operations: + # + # - Remove the DEBUG_MERGE_POINTs. + # + # - Turn all NEW_xxx to either a CALL_MALLOC_GC, or a CALL_MALLOC_NURSERY + # followed by SETFIELDs in order to initialize their GC fields. The + # two advantages of CALL_MALLOC_NURSERY is that it inlines the common + # path, and we need only one such operation to allocate several blocks + # of memory at once. + # + # - Add COND_CALLs to the write barrier before SETFIELD_GC and + # SETARRAYITEM_GC operations. + + _previous_size = -1 + _op_malloc_nursery = None + _v_last_malloced_nursery = None + c_zero = ConstInt(0) + + def __init__(self, gc_ll_descr, cpu): + self.gc_ll_descr = gc_ll_descr + self.cpu = cpu + self.newops = [] + self.known_lengths = {} + self.recent_mallocs = {} # set of variables + + def rewrite(self, operations): + # we can only remember one malloc since the next malloc can possibly + # collect; but we can try to collapse several known-size mallocs into + # one, both for performance and to reduce the number of write + # barriers. We do this on each "basic block" of operations, which in + # this case means between CALLs or unknown-size mallocs. + # + for op in operations: + if op.getopnum() == rop.DEBUG_MERGE_POINT: + continue + # ---------- turn NEWxxx into CALL_MALLOC_xxx ---------- + if op.is_malloc(): + self.handle_malloc_operation(op) + continue From noreply at buildbot.pypy.org Mon Jan 16 17:53:42 2012 From: noreply at buildbot.pypy.org (antocuni) Date: Mon, 16 Jan 2012 17:53:42 +0100 (CET) Subject: [pypy-commit] pypy stm: (arigo, antocuni) fix test which failed after the renaming of stm_become_inevitable Message-ID: <20120116165342.AA6E3820D8@wyvern.cs.uni-duesseldorf.de> Author: Antonio Cuni Branch: stm Changeset: r51353:c3ebb4f01089 Date: 2012-01-16 16:33 +0100 http://bitbucket.org/pypy/pypy/changeset/c3ebb4f01089/ Log: (arigo, antocuni) fix test which failed after the renaming of stm_become_inevitable diff --git a/pypy/translator/stm/test/test_transform.py b/pypy/translator/stm/test/test_transform.py --- a/pypy/translator/stm/test/test_transform.py +++ b/pypy/translator/stm/test/test_transform.py @@ -150,7 +150,7 @@ return p.x interp, graph = get_interpreter(func, [p]) transform_graph(graph) - assert summary(graph) == {'stm_try_inevitable': 1, 'getfield': 1} + assert summary(graph) == {'stm_become_inevitable': 1, 'getfield': 1} res = eval_stm_graph(interp, graph, [p], stm_mode="regular_transaction", final_stm_mode="inevitable_transaction") assert res == 42 @@ -163,7 +163,7 @@ p.x = 43 interp, graph = get_interpreter(func, [p]) transform_graph(graph) - assert summary(graph) == {'stm_try_inevitable': 1, 'setfield': 1} + assert summary(graph) == {'stm_become_inevitable': 1, 'setfield': 1} eval_stm_graph(interp, graph, [p], stm_mode="regular_transaction", final_stm_mode="inevitable_transaction") @@ -175,7 +175,7 @@ return p[3] interp, graph = get_interpreter(func, [p]) transform_graph(graph) - assert summary(graph) == {'stm_try_inevitable': 1, 'getarrayitem': 1} + assert summary(graph) == {'stm_become_inevitable': 1, 'getarrayitem': 1} res = eval_stm_graph(interp, graph, [p], stm_mode="regular_transaction", final_stm_mode="inevitable_transaction") assert res == 42 From noreply at buildbot.pypy.org Mon Jan 16 17:53:43 2012 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 16 Jan 2012 17:53:43 +0100 (CET) Subject: [pypy-commit] pypy stm: hg merge Message-ID: <20120116165343.CA6F0820D8@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stm Changeset: r51354:a14e0f148aad Date: 2012-01-16 17:53 +0100 http://bitbucket.org/pypy/pypy/changeset/a14e0f148aad/ Log: hg merge diff --git a/pypy/translator/stm/test/test_transform.py b/pypy/translator/stm/test/test_transform.py --- a/pypy/translator/stm/test/test_transform.py +++ b/pypy/translator/stm/test/test_transform.py @@ -150,7 +150,7 @@ return p.x interp, graph = get_interpreter(func, [p]) transform_graph(graph) - assert summary(graph) == {'stm_try_inevitable': 1, 'getfield': 1} + assert summary(graph) == {'stm_become_inevitable': 1, 'getfield': 1} res = eval_stm_graph(interp, graph, [p], stm_mode="regular_transaction", final_stm_mode="inevitable_transaction") assert res == 42 @@ -163,7 +163,7 @@ p.x = 43 interp, graph = get_interpreter(func, [p]) transform_graph(graph) - assert summary(graph) == {'stm_try_inevitable': 1, 'setfield': 1} + assert summary(graph) == {'stm_become_inevitable': 1, 'setfield': 1} eval_stm_graph(interp, graph, [p], stm_mode="regular_transaction", final_stm_mode="inevitable_transaction") @@ -175,7 +175,7 @@ return p[3] interp, graph = get_interpreter(func, [p]) transform_graph(graph) - assert summary(graph) == {'stm_try_inevitable': 1, 'getarrayitem': 1} + assert summary(graph) == {'stm_become_inevitable': 1, 'getarrayitem': 1} res = eval_stm_graph(interp, graph, [p], stm_mode="regular_transaction", final_stm_mode="inevitable_transaction") assert res == 42 From noreply at buildbot.pypy.org Mon Jan 16 18:15:39 2012 From: noreply at buildbot.pypy.org (antocuni) Date: Mon, 16 Jan 2012 18:15:39 +0100 (CET) Subject: [pypy-commit] pypy stm: (arigo, antocuni): start to write the RPython level interface for using transactions in rlib/rstm.py. Move the compiled tests from translator/stm/test_transform.py to rlib/test/test_rstm.py Message-ID: <20120116171539.5FE89820D8@wyvern.cs.uni-duesseldorf.de> Author: Antonio Cuni Branch: stm Changeset: r51355:1aa99f2f035f Date: 2012-01-16 18:06 +0100 http://bitbucket.org/pypy/pypy/changeset/1aa99f2f035f/ Log: (arigo, antocuni): start to write the RPython level interface for using transactions in rlib/rstm.py. Move the compiled tests from translator/stm/test_transform.py to rlib/test/test_rstm.py diff --git a/pypy/rlib/rstm.py b/pypy/rlib/rstm.py new file mode 100644 --- /dev/null +++ b/pypy/rlib/rstm.py @@ -0,0 +1,38 @@ +from pypy.rlib.objectmodel import specialize, we_are_translated, keepalive_until_here +from pypy.rpython.lltypesystem import rffi, lltype +from pypy.rpython.annlowlevel import (cast_base_ptr_to_instance, + cast_instance_to_base_ptr, + llhelper) +from pypy.translator.stm import _rffi_stm + + at specialize.memo() +def _get_stm_callback(func, argcls): + def _stm_callback(llarg): + if we_are_translated(): + llarg = rffi.cast(rclass.OBJECTPTR, llarg) + arg = cast_base_ptr_to_instance(argcls, llarg) + else: + arg = lltype.TLS.stm_callback_arg + res = func(arg) + assert res is None + return lltype.nullptr(rffi.VOIDP.TO) + return _stm_callback + + at specialize.arg(0, 1) +def stm_perform_transaction(func, argcls, arg): + assert isinstance(arg, argcls) + assert argcls._alloc_nonmovable_ + if we_are_translated(): + llarg = cast_instance_to_base_ptr(arg) + llarg = rffi.cast(rffi.VOIDP, llarg) + else: + # only for tests + lltype.TLS.stm_callback_arg = arg + llarg = lltype.nullptr(rffi.VOIDP.TO) + callback = _get_stm_callback(func, argcls) + llcallback = llhelper(_rffi_stm.CALLBACK, callback) + _rffi_stm.perform_transaction(llcallback, llarg) + keepalive_until_here(arg) + +stm_descriptor_init = _rffi_stm.descriptor_init +stm_descriptor_done = _rffi_stm.descriptor_done diff --git a/pypy/rlib/test/test_rstm.py b/pypy/rlib/test/test_rstm.py new file mode 100644 --- /dev/null +++ b/pypy/rlib/test/test_rstm.py @@ -0,0 +1,51 @@ +from pypy.rlib.debug import debug_print +from pypy.rlib import rstm +from pypy.translator.c.test.test_standalone import StandaloneTests + +def test_stm_perform_transaction(): + class Arg(object): + _alloc_nonmovable_ = True + + def setx(arg): + arg.x = 42 + + arg = Arg() + rstm.stm_descriptor_init() + rstm.stm_perform_transaction(setx, Arg, arg) + rstm.stm_descriptor_done() + assert arg.x == 42 + + +class CompiledSTMTests(StandaloneTests): + gc = "none" + + def compile(self, entry_point): + from pypy.config.pypyoption import get_pypy_config + self.config = get_pypy_config(translating=True) + self.config.translation.stm = True + self.config.translation.gc = self.gc + # + # Prevent the RaiseAnalyzer from just emitting "WARNING: Unknown + # operation". We want instead it to crash. + from pypy.translator.backendopt.canraise import RaiseAnalyzer + RaiseAnalyzer.fail_on_unknown_operation = True + try: + res = StandaloneTests.compile(self, entry_point, debug=True) + finally: + RaiseAnalyzer.fail_on_unknown_operation = False + return res + + +class TestTransformSingleThread(CompiledSTMTests): + + def test_no_pointer_operations(self): + def simplefunc(argv): + i = 0 + while i < 100: + i += 3 + debug_print(i) + return 0 + t, cbuilder = self.compile(simplefunc) + dataout, dataerr = cbuilder.cmdexec('', err=True) + assert dataout == '' + assert '102' in dataerr.splitlines() diff --git a/pypy/translator/stm/src_stm/et.c b/pypy/translator/stm/src_stm/et.c --- a/pypy/translator/stm/src_stm/et.c +++ b/pypy/translator/stm/src_stm/et.c @@ -676,10 +676,8 @@ void* stm_perform_transaction(void*(*callback)(void*), void *arg) { void *result; -#ifdef RPY_STM_ASSERT /* you need to call descriptor_init() before calling stm_perform_transaction */ assert(thread_descriptor != NULL); -#endif STM_begin_transaction(); result = callback(arg); stm_commit_transaction(); diff --git a/pypy/translator/stm/test/test_transform.py b/pypy/translator/stm/test/test_transform.py --- a/pypy/translator/stm/test/test_transform.py +++ b/pypy/translator/stm/test/test_transform.py @@ -3,9 +3,6 @@ from pypy.objspace.flow.model import summary from pypy.translator.stm.llstminterp import eval_stm_graph from pypy.translator.stm.transform import transform_graph -##from pypy.translator.stm import rstm -from pypy.translator.c.test.test_standalone import StandaloneTests -from pypy.rlib.debug import debug_print from pypy.conftest import option @@ -179,96 +176,3 @@ res = eval_stm_graph(interp, graph, [p], stm_mode="regular_transaction", final_stm_mode="inevitable_transaction") assert res == 42 - -# ____________________________________________________________ - -class CompiledSTMTests(StandaloneTests): - gc = "none" - - def compile(self, entry_point): - from pypy.config.pypyoption import get_pypy_config - self.config = get_pypy_config(translating=True) - self.config.translation.stm = True - self.config.translation.gc = self.gc - # - # Prevent the RaiseAnalyzer from just emitting "WARNING: Unknown - # operation". We want instead it to crash. - from pypy.translator.backendopt.canraise import RaiseAnalyzer - RaiseAnalyzer.fail_on_unknown_operation = True - try: - res = StandaloneTests.compile(self, entry_point, debug=True) - finally: - RaiseAnalyzer.fail_on_unknown_operation = False - return res - - -class TestTransformSingleThread(CompiledSTMTests): - - def test_no_pointer_operations(self): - def simplefunc(argv): - i = 0 - while i < 100: - i += 3 - debug_print(i) - return 0 - t, cbuilder = self.compile(simplefunc) - dataout, dataerr = cbuilder.cmdexec('', err=True) - assert dataout == '' - assert '102' in dataerr.splitlines() - - def test_fails_when_nonbalanced_begin(self): - def simplefunc(argv): - rstm.begin_transaction() - return 0 - t, cbuilder = self.compile(simplefunc) - cbuilder.cmdexec('', expect_crash=True) - - def test_fails_when_nonbalanced_commit(self): - def simplefunc(argv): - rstm.commit_transaction() - rstm.commit_transaction() - return 0 - t, cbuilder = self.compile(simplefunc) - cbuilder.cmdexec('', expect_crash=True) - - def test_begin_inevitable_transaction(self): - def simplefunc(argv): - rstm.commit_transaction() - rstm.begin_inevitable_transaction() - return 0 - t, cbuilder = self.compile(simplefunc) - cbuilder.cmdexec('') - - def test_transaction_boundary_1(self): - def simplefunc(argv): - rstm.transaction_boundary() - return 0 - t, cbuilder = self.compile(simplefunc) - cbuilder.cmdexec('') - - def test_transaction_boundary_2(self): - def simplefunc(argv): - rstm.transaction_boundary() - rstm.transaction_boundary() - rstm.transaction_boundary() - return 0 - t, cbuilder = self.compile(simplefunc) - cbuilder.cmdexec('') - - def test_transaction_boundary_3(self): - def simplefunc(argv): - s1 = argv[0] - debug_print('STEP1:', len(s1)) - rstm.transaction_boundary() - rstm.transaction_boundary() - rstm.transaction_boundary() - debug_print('STEP2:', len(s1)) - return 0 - t, cbuilder = self.compile(simplefunc) - data, err = cbuilder.cmdexec('', err=True) - lines = err.splitlines() - steps = [(line[:6], line[6:]) - for line in lines if line.startswith('STEP')] - steps = zip(*steps) - assert steps[0] == ('STEP1:', 'STEP2:') - assert steps[1][0] == steps[1][1] From noreply at buildbot.pypy.org Mon Jan 16 18:15:40 2012 From: noreply at buildbot.pypy.org (antocuni) Date: Mon, 16 Jan 2012 18:15:40 +0100 (CET) Subject: [pypy-commit] pypy stm: (arigo, antocuni): rename _rffi_stm.* to _rffi_stm.stm_* and rstm.stm_* to rstm.* Message-ID: <20120116171540.926FE820D8@wyvern.cs.uni-duesseldorf.de> Author: Antonio Cuni Branch: stm Changeset: r51356:f19ffa0b7bb3 Date: 2012-01-16 18:14 +0100 http://bitbucket.org/pypy/pypy/changeset/f19ffa0b7bb3/ Log: (arigo, antocuni): rename _rffi_stm.* to _rffi_stm.stm_* and rstm.stm_* to rstm.* diff --git a/pypy/rlib/rstm.py b/pypy/rlib/rstm.py --- a/pypy/rlib/rstm.py +++ b/pypy/rlib/rstm.py @@ -19,7 +19,7 @@ return _stm_callback @specialize.arg(0, 1) -def stm_perform_transaction(func, argcls, arg): +def perform_transaction(func, argcls, arg): assert isinstance(arg, argcls) assert argcls._alloc_nonmovable_ if we_are_translated(): @@ -31,8 +31,8 @@ llarg = lltype.nullptr(rffi.VOIDP.TO) callback = _get_stm_callback(func, argcls) llcallback = llhelper(_rffi_stm.CALLBACK, callback) - _rffi_stm.perform_transaction(llcallback, llarg) + _rffi_stm.stm_perform_transaction(llcallback, llarg) keepalive_until_here(arg) -stm_descriptor_init = _rffi_stm.descriptor_init -stm_descriptor_done = _rffi_stm.descriptor_done +descriptor_init = _rffi_stm.stm_descriptor_init +descriptor_done = _rffi_stm.stm_descriptor_done diff --git a/pypy/rlib/test/test_rstm.py b/pypy/rlib/test/test_rstm.py --- a/pypy/rlib/test/test_rstm.py +++ b/pypy/rlib/test/test_rstm.py @@ -10,9 +10,9 @@ arg.x = 42 arg = Arg() - rstm.stm_descriptor_init() - rstm.stm_perform_transaction(setx, Arg, arg) - rstm.stm_descriptor_done() + rstm.descriptor_init() + rstm.perform_transaction(setx, Arg, arg) + rstm.descriptor_done() assert arg.x == 42 diff --git a/pypy/translator/stm/_rffi_stm.py b/pypy/translator/stm/_rffi_stm.py --- a/pypy/translator/stm/_rffi_stm.py +++ b/pypy/translator/stm/_rffi_stm.py @@ -23,14 +23,14 @@ SignedP = rffi.CArrayPtr(lltype.Signed) -descriptor_init = llexternal('stm_descriptor_init', [], lltype.Void) -descriptor_done = llexternal('stm_descriptor_done', [], lltype.Void) +stm_descriptor_init = llexternal('stm_descriptor_init', [], lltype.Void) +stm_descriptor_done = llexternal('stm_descriptor_done', [], lltype.Void) ##begin_transaction = llexternal('STM_begin_transaction', [], lltype.Void) ##begin_inevitable_transaction = llexternal('stm_begin_inevitable_transaction', ## [], lltype.Void) ##commit_transaction = llexternal('stm_commit_transaction', [], lltype.Signed) -try_inevitable = llexternal('stm_try_inevitable', [], lltype.Void) +stm_try_inevitable = llexternal('stm_try_inevitable', [], lltype.Void) ##descriptor_init_and_being_inevitable_transaction = llexternal( ## 'stm_descriptor_init_and_being_inevitable_transaction', [], lltype.Void) @@ -42,7 +42,7 @@ lltype.Void) CALLBACK = lltype.Ptr(lltype.FuncType([rffi.VOIDP], rffi.VOIDP)) -perform_transaction = llexternal('stm_perform_transaction', - [CALLBACK, rffi.VOIDP], rffi.VOIDP) +stm_perform_transaction = llexternal('stm_perform_transaction', + [CALLBACK, rffi.VOIDP], rffi.VOIDP) -abort_and_retry = llexternal('stm_abort_and_retry', [], lltype.Void) +stm_abort_and_retry = llexternal('stm_abort_and_retry', [], lltype.Void) diff --git a/pypy/translator/stm/test/test_funcgen.py b/pypy/translator/stm/test/test_funcgen.py --- a/pypy/translator/stm/test/test_funcgen.py +++ b/pypy/translator/stm/test/test_funcgen.py @@ -1,9 +1,9 @@ from pypy.rpython.lltypesystem import lltype, rffi from pypy.rlib.rarithmetic import r_longlong, r_singlefloat -from pypy.translator.stm.test.test_transform import CompiledSTMTests +from pypy.rlib.test.test_rstm import CompiledSTMTests #from pypy.rlib import rstm -from pypy.translator.stm._rffi_stm import (CALLBACK, perform_transaction, - descriptor_init, descriptor_done) +from pypy.translator.stm._rffi_stm import (CALLBACK, stm_perform_transaction, + stm_descriptor_init, stm_descriptor_done) from pypy.translator.c.test.test_standalone import StandaloneTests from pypy.rlib.debug import debug_print from pypy.rpython.annlowlevel import llhelper @@ -251,9 +251,9 @@ def test_getfield_all_sizes_inside_transaction(self): def do_stm_getfield(argv): callback = llhelper(CALLBACK, _play_with_getfield) - descriptor_init() - perform_transaction(callback, NULL) - descriptor_done() + stm_descriptor_init() + stm_perform_transaction(callback, NULL) + stm_descriptor_done() return 0 t, cbuilder = self.compile(do_stm_getfield) cbuilder.cmdexec('') @@ -269,11 +269,11 @@ def do_stm_setfield(argv): callback1 = llhelper(CALLBACK, _play_with_setfields) callback2 = llhelper(CALLBACK, _check_values_of_fields) - descriptor_init() - perform_transaction(callback1, NULL) + stm_descriptor_init() + stm_perform_transaction(callback1, NULL) # read values which aren't local to the transaction - perform_transaction(callback2, NULL) - descriptor_done() + stm_perform_transaction(callback2, NULL) + stm_descriptor_done() return 0 t, cbuilder = self.compile(do_stm_setfield) cbuilder.cmdexec('') @@ -288,9 +288,9 @@ def test_getarrayitem_all_sizes_inside_transaction(self): def do_stm_getarrayitem(argv): callback = llhelper(CALLBACK, _play_with_getarrayitem) - descriptor_init() - perform_transaction(callback, NULL) - descriptor_done() + stm_descriptor_init() + stm_perform_transaction(callback, NULL) + stm_descriptor_done() return 0 t, cbuilder = self.compile(do_stm_getarrayitem) cbuilder.cmdexec('') @@ -311,11 +311,11 @@ callback2 = llhelper(CALLBACK, _play_with_setarrayitem_2) callback3 = llhelper(CALLBACK, _play_with_setarrayitem_3) # - descriptor_init() - perform_transaction(callback1, NULL) - perform_transaction(callback2, NULL) - perform_transaction(callback3, NULL) - descriptor_done() + stm_descriptor_init() + stm_perform_transaction(callback1, NULL) + stm_perform_transaction(callback2, NULL) + stm_perform_transaction(callback3, NULL) + stm_descriptor_done() return 0 t, cbuilder = self.compile(do_stm_setarrayitem) cbuilder.cmdexec('') @@ -330,9 +330,9 @@ def test_getinteriorfield_all_sizes_inside_transaction(self): def do_stm_getinteriorfield(argv): callback = llhelper(CALLBACK, _play_with_getinteriorfield) - descriptor_init() - perform_transaction(callback, NULL) - descriptor_done() + stm_descriptor_init() + stm_perform_transaction(callback, NULL) + stm_descriptor_done() return 0 t, cbuilder = self.compile(do_stm_getinteriorfield) cbuilder.cmdexec('') @@ -351,10 +351,10 @@ callback1 = llhelper(CALLBACK, _play_with_setinteriorfield_1) callback2 = llhelper(CALLBACK, _play_with_setinteriorfield_2) # - descriptor_init() - perform_transaction(callback1, NULL) - perform_transaction(callback2, NULL) - descriptor_done() + stm_descriptor_init() + stm_perform_transaction(callback1, NULL) + stm_perform_transaction(callback2, NULL) + stm_descriptor_done() return 0 t, cbuilder = self.compile(do_stm_setinteriorfield) cbuilder.cmdexec('') diff --git a/pypy/translator/stm/test/test_llstm.py b/pypy/translator/stm/test/test_llstm.py --- a/pypy/translator/stm/test/test_llstm.py +++ b/pypy/translator/stm/test/test_llstm.py @@ -44,7 +44,7 @@ assert a.x == -611 # xxx still the old value when reading non-transact. if a.y < 10: a.y += 1 # non-transactionally - abort_and_retry() + stm_abort_and_retry() return lltype.nullptr(rffi.VOIDP.TO) def test_stm_getfield(): @@ -58,10 +58,10 @@ a.f = rf1 a.sa = rs1a a.sb = rs1b - descriptor_init() - perform_transaction(llhelper(CALLBACK, callback1), + stm_descriptor_init() + stm_perform_transaction(llhelper(CALLBACK, callback1), rffi.cast(rffi.VOIDP, a)) - descriptor_done() + stm_descriptor_done() assert a.x == 420 assert a.c1 == '/' assert a.c2 == '\\' @@ -117,7 +117,7 @@ assert float(a.sb) == float(rs1b) if a.y < 10: a.y += 1 # non-transactionally - abort_and_retry() + stm_abort_and_retry() return lltype.nullptr(rffi.VOIDP.TO) def test_stm_setfield(): @@ -131,10 +131,10 @@ a.f = rf1 a.sa = rs1a a.sb = rs1b - descriptor_init() - perform_transaction(llhelper(CALLBACK, callback2), + stm_descriptor_init() + stm_perform_transaction(llhelper(CALLBACK, callback2), rffi.cast(rffi.VOIDP, a)) - descriptor_done() + stm_descriptor_done() assert a.x == 420 assert a.c1 == '(' assert a.c2 == '?' diff --git a/pypy/translator/stm/test/test_rffi_stm.py b/pypy/translator/stm/test/test_rffi_stm.py --- a/pypy/translator/stm/test/test_rffi_stm.py +++ b/pypy/translator/stm/test/test_rffi_stm.py @@ -2,35 +2,35 @@ from pypy.rpython.annlowlevel import llhelper def test_descriptor(): - descriptor_init() - descriptor_done() + stm_descriptor_init() + stm_descriptor_done() -def test_perform_transaction(): +def test_stm_perform_transaction(): def callback1(x): return lltype.nullptr(rffi.VOIDP.TO) - descriptor_init() - perform_transaction(llhelper(CALLBACK, callback1), + stm_descriptor_init() + stm_perform_transaction(llhelper(CALLBACK, callback1), lltype.nullptr(rffi.VOIDP.TO)) - descriptor_done() + stm_descriptor_done() -def test_abort_and_retry(): +def test_stm_abort_and_retry(): A = lltype.Struct('A', ('x', lltype.Signed), ('y', lltype.Signed)) a = lltype.malloc(A, immortal=True, flavor='raw') a.y = 0 def callback1(x): if a.y < 10: a.y += 1 # non-transactionally - abort_and_retry() + stm_abort_and_retry() else: a.x = 42 * a.y return lltype.nullptr(rffi.VOIDP.TO) - descriptor_init() - perform_transaction(llhelper(CALLBACK, callback1), + stm_descriptor_init() + stm_perform_transaction(llhelper(CALLBACK, callback1), lltype.nullptr(rffi.VOIDP.TO)) - descriptor_done() + stm_descriptor_done() assert a.x == 420 -def test_abort_and_retry_transactionally(): +def test_stm_abort_and_retry_transactionally(): A = lltype.Struct('A', ('x', lltype.Signed), ('y', lltype.Signed)) a = lltype.malloc(A, immortal=True, flavor='raw') a.x = -611 @@ -45,11 +45,11 @@ assert a.x == -611 # xxx still the old value when reading non-transact. if a.y < 10: a.y += 1 # non-transactionally - abort_and_retry() + stm_abort_and_retry() else: return lltype.nullptr(rffi.VOIDP.TO) - descriptor_init() - perform_transaction(llhelper(CALLBACK, callback1), + stm_descriptor_init() + stm_perform_transaction(llhelper(CALLBACK, callback1), lltype.nullptr(rffi.VOIDP.TO)) - descriptor_done() + stm_descriptor_done() assert a.x == 420 diff --git a/pypy/translator/stm/test/test_ztranslated.py b/pypy/translator/stm/test/test_ztranslated.py --- a/pypy/translator/stm/test/test_ztranslated.py +++ b/pypy/translator/stm/test/test_ztranslated.py @@ -1,5 +1,5 @@ import py -from pypy.translator.stm.test.test_transform import CompiledSTMTests +from pypy.rlib.test.test_rstm import CompiledSTMTests from pypy.translator.stm.test import targetdemo From noreply at buildbot.pypy.org Mon Jan 16 18:16:24 2012 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Mon, 16 Jan 2012 18:16:24 +0100 (CET) Subject: [pypy-commit] pypy default: allow inlining into _codecs, makes simple decoding ~3x faster Message-ID: <20120116171624.357B0820D8@wyvern.cs.uni-duesseldorf.de> Author: Alex Gaynor Branch: Changeset: r51357:af6b237eaecf Date: 2012-01-16 18:15 +0100 http://bitbucket.org/pypy/pypy/changeset/af6b237eaecf/ Log: allow inlining into _codecs, makes simple decoding ~3x faster diff --git a/pypy/module/pypyjit/policy.py b/pypy/module/pypyjit/policy.py --- a/pypy/module/pypyjit/policy.py +++ b/pypy/module/pypyjit/policy.py @@ -127,7 +127,7 @@ 'imp', 'sys', 'array', '_ffi', 'itertools', 'operator', 'posix', '_socket', '_sre', '_lsprof', '_weakref', '__pypy__', 'cStringIO', '_collections', 'struct', - 'mmap', 'marshal']: + 'mmap', 'marshal', '_codecs']: if modname == 'pypyjit' and 'interp_resop' in rest: return False return True From noreply at buildbot.pypy.org Mon Jan 16 18:16:25 2012 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Mon, 16 Jan 2012 18:16:25 +0100 (CET) Subject: [pypy-commit] pypy default: merged upstream Message-ID: <20120116171625.5B8DD820D8@wyvern.cs.uni-duesseldorf.de> Author: Alex Gaynor Branch: Changeset: r51358:8688ce42472e Date: 2012-01-16 18:16 +0100 http://bitbucket.org/pypy/pypy/changeset/8688ce42472e/ Log: merged upstream diff --git a/pypy/rpython/lltypesystem/rclass.py b/pypy/rpython/lltypesystem/rclass.py --- a/pypy/rpython/lltypesystem/rclass.py +++ b/pypy/rpython/lltypesystem/rclass.py @@ -510,7 +510,13 @@ ctype = inputconst(Void, self.object_type) cflags = inputconst(Void, flags) vlist = [ctype, cflags] - vptr = llops.genop('malloc', vlist, + cnonmovable = self.classdef.classdesc.read_attribute( + '_alloc_nonmovable_', Constant(False)) + if cnonmovable.value: + opname = 'malloc_nonmovable' + else: + opname = 'malloc' + vptr = llops.genop(opname, vlist, resulttype = Ptr(self.object_type)) ctypeptr = inputconst(CLASSTYPE, self.rclass.getvtable()) self.setfield(vptr, '__class__', ctypeptr, llops) diff --git a/pypy/rpython/test/test_rclass.py b/pypy/rpython/test/test_rclass.py --- a/pypy/rpython/test/test_rclass.py +++ b/pypy/rpython/test/test_rclass.py @@ -1130,6 +1130,18 @@ assert sorted([u]) == [6] # 32-bit types assert sorted([i, r, d, l]) == [2, 3, 4, 5] # 64-bit types + def test_nonmovable(self): + for (nonmovable, opname) in [(True, 'malloc_nonmovable'), + (False, 'malloc')]: + class A(object): + _alloc_nonmovable_ = nonmovable + def f(): + return A() + t, typer, graph = self.gengraph(f, []) + assert summary(graph) == {opname: 1, + 'cast_pointer': 1, + 'setfield': 1} + class TestOOtype(BaseTestRclass, OORtypeMixin): From noreply at buildbot.pypy.org Mon Jan 16 18:19:04 2012 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 16 Jan 2012 18:19:04 +0100 (CET) Subject: [pypy-commit] pypy stm: (antocuni, arigo) Message-ID: <20120116171904.34B72820D8@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stm Changeset: r51359:e2645247a9c2 Date: 2012-01-16 18:18 +0100 http://bitbucket.org/pypy/pypy/changeset/e2645247a9c2/ Log: (antocuni, arigo) Move CompiledSTMTests to its own file in translator/stm/test/support.py. diff --git a/pypy/rlib/test/test_rstm.py b/pypy/rlib/test/test_rstm.py --- a/pypy/rlib/test/test_rstm.py +++ b/pypy/rlib/test/test_rstm.py @@ -1,6 +1,6 @@ from pypy.rlib.debug import debug_print from pypy.rlib import rstm -from pypy.translator.c.test.test_standalone import StandaloneTests +from pypy.translator.stm.test.support import CompiledSTMTests def test_stm_perform_transaction(): class Arg(object): @@ -16,26 +16,6 @@ assert arg.x == 42 -class CompiledSTMTests(StandaloneTests): - gc = "none" - - def compile(self, entry_point): - from pypy.config.pypyoption import get_pypy_config - self.config = get_pypy_config(translating=True) - self.config.translation.stm = True - self.config.translation.gc = self.gc - # - # Prevent the RaiseAnalyzer from just emitting "WARNING: Unknown - # operation". We want instead it to crash. - from pypy.translator.backendopt.canraise import RaiseAnalyzer - RaiseAnalyzer.fail_on_unknown_operation = True - try: - res = StandaloneTests.compile(self, entry_point, debug=True) - finally: - RaiseAnalyzer.fail_on_unknown_operation = False - return res - - class TestTransformSingleThread(CompiledSTMTests): def test_no_pointer_operations(self): diff --git a/pypy/translator/stm/test/support.py b/pypy/translator/stm/test/support.py new file mode 100644 --- /dev/null +++ b/pypy/translator/stm/test/support.py @@ -0,0 +1,23 @@ +"""CompiledSTMTests, a support class for translated tests with STM""" + +from pypy.translator.c.test.test_standalone import StandaloneTests + + +class CompiledSTMTests(StandaloneTests): + gc = "none" + + def compile(self, entry_point): + from pypy.config.pypyoption import get_pypy_config + self.config = get_pypy_config(translating=True) + self.config.translation.stm = True + self.config.translation.gc = self.gc + # + # Prevent the RaiseAnalyzer from just emitting "WARNING: Unknown + # operation". We want instead it to crash. + from pypy.translator.backendopt.canraise import RaiseAnalyzer + RaiseAnalyzer.fail_on_unknown_operation = True + try: + res = StandaloneTests.compile(self, entry_point, debug=True) + finally: + RaiseAnalyzer.fail_on_unknown_operation = False + return res diff --git a/pypy/translator/stm/test/test_funcgen.py b/pypy/translator/stm/test/test_funcgen.py --- a/pypy/translator/stm/test/test_funcgen.py +++ b/pypy/translator/stm/test/test_funcgen.py @@ -1,7 +1,6 @@ from pypy.rpython.lltypesystem import lltype, rffi from pypy.rlib.rarithmetic import r_longlong, r_singlefloat -from pypy.rlib.test.test_rstm import CompiledSTMTests -#from pypy.rlib import rstm +from pypy.translator.stm.test.support import CompiledSTMTests from pypy.translator.stm._rffi_stm import (CALLBACK, stm_perform_transaction, stm_descriptor_init, stm_descriptor_done) from pypy.translator.c.test.test_standalone import StandaloneTests diff --git a/pypy/translator/stm/test/test_ztranslated.py b/pypy/translator/stm/test/test_ztranslated.py --- a/pypy/translator/stm/test/test_ztranslated.py +++ b/pypy/translator/stm/test/test_ztranslated.py @@ -1,5 +1,5 @@ import py -from pypy.rlib.test.test_rstm import CompiledSTMTests +from pypy.translator.stm.test.support import CompiledSTMTests from pypy.translator.stm.test import targetdemo From noreply at buildbot.pypy.org Mon Jan 16 18:29:45 2012 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 16 Jan 2012 18:29:45 +0100 (CET) Subject: [pypy-commit] pypy stm: (antocuni, arigo) Message-ID: <20120116172945.04325820D8@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stm Changeset: r51360:dbadc138e9b5 Date: 2012-01-16 18:29 +0100 http://bitbucket.org/pypy/pypy/changeset/dbadc138e9b5/ Log: (antocuni, arigo) Add an integration test to test_rstm.py: really call rstm.perform_transaction() in a C-compiled test. diff --git a/pypy/rlib/rstm.py b/pypy/rlib/rstm.py --- a/pypy/rlib/rstm.py +++ b/pypy/rlib/rstm.py @@ -1,5 +1,5 @@ from pypy.rlib.objectmodel import specialize, we_are_translated, keepalive_until_here -from pypy.rpython.lltypesystem import rffi, lltype +from pypy.rpython.lltypesystem import rffi, lltype, rclass from pypy.rpython.annlowlevel import (cast_base_ptr_to_instance, cast_instance_to_base_ptr, llhelper) diff --git a/pypy/rlib/test/test_rstm.py b/pypy/rlib/test/test_rstm.py --- a/pypy/rlib/test/test_rstm.py +++ b/pypy/rlib/test/test_rstm.py @@ -2,14 +2,18 @@ from pypy.rlib import rstm from pypy.translator.stm.test.support import CompiledSTMTests + +class Arg(object): + _alloc_nonmovable_ = True + +def setx(arg): + debug_print(arg.x) + arg.x = 42 + + def test_stm_perform_transaction(): - class Arg(object): - _alloc_nonmovable_ = True - - def setx(arg): - arg.x = 42 - arg = Arg() + arg.x = 202 rstm.descriptor_init() rstm.perform_transaction(setx, Arg, arg) rstm.descriptor_done() @@ -29,3 +33,12 @@ dataout, dataerr = cbuilder.cmdexec('', err=True) assert dataout == '' assert '102' in dataerr.splitlines() + + def test_perform_transaction(self): + def f(argv): + test_stm_perform_transaction() + return 0 + t, cbuilder = self.compile(f) + dataout, dataerr = cbuilder.cmdexec('', err=True) + assert dataout == '' + assert '202' in dataerr.splitlines() diff --git a/pypy/translator/c/funcgen.py b/pypy/translator/c/funcgen.py --- a/pypy/translator/c/funcgen.py +++ b/pypy/translator/c/funcgen.py @@ -226,9 +226,7 @@ yield '\tRPyConvertExceptionToCPython();' yield '\treturn NULL;' yield '}' - if self.exception_policy == "stm": - xxxx - yield 'STM_MAKE_INEVITABLE();' + assert self.exception_policy != "stm", "old code" retval = self.expr(block.inputargs[0]) if self.exception_policy != "exc_helper": yield 'RPY_DEBUG_RETURN();' @@ -609,6 +607,7 @@ OP_STM_SETARRAYITEM = _OP_STM OP_STM_GETINTERIORFIELD = _OP_STM OP_STM_SETINTERIORFIELD = _OP_STM + OP_STM_BECOME_INEVITABLE = _OP_STM def OP_PTR_NONZERO(self, op): diff --git a/pypy/translator/stm/funcgen.py b/pypy/translator/stm/funcgen.py --- a/pypy/translator/stm/funcgen.py +++ b/pypy/translator/stm/funcgen.py @@ -118,49 +118,7 @@ return _stm_generic_set(funcgen, op, expr, T) -def stm_begin_transaction(funcgen, op): - return 'STM_begin_transaction();' - -def stm_commit_transaction(funcgen, op): - return 'stm_commit_transaction();' - -def stm_begin_inevitable_transaction(funcgen, op): - return 'stm_begin_inevitable_transaction();' - -def stm_declare_variable(funcgen, op): - # this operation occurs only once at the start of a function if - # it uses stm_transaction_boundary - assert funcgen.exception_policy is None - funcgen.exception_policy = 'stm' - return 'STM_DECLARE_VARIABLE();' - -def stm_transaction_boundary(funcgen, op): - assert funcgen.exception_policy == 'stm' - # make code looking like this: - # - # stm_commit_transaction(); - # { - # volatile long tmp_123 = l_123; - # setjmp(jmpbuf); - # l_123 = tmp_123; - # } - # stm_begin_transaction(&jmpbuf); - # - lines = ['\tsetjmp(jmpbuf);'] - TMPVAR = 'tmp_%s' - for v in op.args: - tmpname = TMPVAR % v.name - cdeclname = cdecl(funcgen.lltypename(v), 'volatile ' + tmpname) - realname = funcgen.expr(v) - lines.insert(0, '\t%s = %s;' % (cdeclname, realname)) - lines.append('\t%s = %s;' % (realname, tmpname)) - lines.insert(0, '{') - lines.insert(0, 'stm_commit_transaction();') - lines.append('}') - lines.append('stm_begin_transaction(&jmpbuf);') - return '\n'.join(lines) - -def stm_try_inevitable(funcgen, op): +def stm_become_inevitable(funcgen, op): info = op.args[0].value string_literal = c_string_constant(info) return 'stm_try_inevitable(STM_EXPLAIN1(%s));' % (string_literal,) diff --git a/pypy/translator/stm/src_stm/et.c b/pypy/translator/stm/src_stm/et.c --- a/pypy/translator/stm/src_stm/et.c +++ b/pypy/translator/stm/src_stm/et.c @@ -775,6 +775,8 @@ by another thread. We set the lowest bit in global_timestamp to 1. */ struct tx_descriptor *d = thread_descriptor; + if (!d) + return; #ifdef RPY_STM_ASSERT PYPY_DEBUG_START("stm-inevitable"); From noreply at buildbot.pypy.org Mon Jan 16 18:56:27 2012 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 16 Jan 2012 18:56:27 +0100 (CET) Subject: [pypy-commit] pypy stm: (antocuni, arigo) Message-ID: <20120116175627.1DA9B820D8@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stm Changeset: r51361:1e52821ea7bc Date: 2012-01-16 18:44 +0100 http://bitbucket.org/pypy/pypy/changeset/1e52821ea7bc/ Log: (antocuni, arigo) Test (maybe) for inevitable transactions. diff --git a/pypy/rlib/test/test_rstm.py b/pypy/rlib/test/test_rstm.py --- a/pypy/rlib/test/test_rstm.py +++ b/pypy/rlib/test/test_rstm.py @@ -1,3 +1,4 @@ +import os from pypy.rlib.debug import debug_print from pypy.rlib import rstm from pypy.translator.stm.test.support import CompiledSTMTests @@ -8,12 +9,15 @@ def setx(arg): debug_print(arg.x) + if arg.x == 303: + # this will trigger stm_become_inevitable() + os.write(1, "hello\n") arg.x = 42 -def test_stm_perform_transaction(): +def test_stm_perform_transaction(initial_x=202): arg = Arg() - arg.x = 202 + arg.x = initial_x rstm.descriptor_init() rstm.perform_transaction(setx, Arg, arg) rstm.descriptor_done() @@ -42,3 +46,12 @@ dataout, dataerr = cbuilder.cmdexec('', err=True) assert dataout == '' assert '202' in dataerr.splitlines() + + def test_perform_transaction_inevitable(self): + def f(argv): + test_stm_perform_transaction(303) + return 0 + t, cbuilder = self.compile(f) + dataout, dataerr = cbuilder.cmdexec('', err=True) + assert 'hello' in dataout.splitlines() + assert '303' in dataerr.splitlines() From noreply at buildbot.pypy.org Mon Jan 16 18:56:28 2012 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 16 Jan 2012 18:56:28 +0100 (CET) Subject: [pypy-commit] pypy stm: (antocuni, arigo) Message-ID: <20120116175628.49899820D8@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stm Changeset: r51362:f565b7971e14 Date: 2012-01-16 18:55 +0100 http://bitbucket.org/pypy/pypy/changeset/f565b7971e14/ Log: (antocuni, arigo) A way to get and test the current transaction mode, for debugging. diff --git a/pypy/rlib/rstm.py b/pypy/rlib/rstm.py --- a/pypy/rlib/rstm.py +++ b/pypy/rlib/rstm.py @@ -36,3 +36,4 @@ descriptor_init = _rffi_stm.stm_descriptor_init descriptor_done = _rffi_stm.stm_descriptor_done +debug_get_state = _rffi_stm.stm_debug_get_state diff --git a/pypy/rlib/test/test_rstm.py b/pypy/rlib/test/test_rstm.py --- a/pypy/rlib/test/test_rstm.py +++ b/pypy/rlib/test/test_rstm.py @@ -9,18 +9,24 @@ def setx(arg): debug_print(arg.x) + assert rstm.debug_get_state() == 1 if arg.x == 303: # this will trigger stm_become_inevitable() os.write(1, "hello\n") + assert rstm.debug_get_state() == 2 arg.x = 42 def test_stm_perform_transaction(initial_x=202): arg = Arg() arg.x = initial_x + assert rstm.debug_get_state() == -1 rstm.descriptor_init() + assert rstm.debug_get_state() == 0 rstm.perform_transaction(setx, Arg, arg) + assert rstm.debug_get_state() == 0 rstm.descriptor_done() + assert rstm.debug_get_state() == -1 assert arg.x == 42 diff --git a/pypy/translator/stm/_rffi_stm.py b/pypy/translator/stm/_rffi_stm.py --- a/pypy/translator/stm/_rffi_stm.py +++ b/pypy/translator/stm/_rffi_stm.py @@ -46,3 +46,4 @@ [CALLBACK, rffi.VOIDP], rffi.VOIDP) stm_abort_and_retry = llexternal('stm_abort_and_retry', [], lltype.Void) +stm_debug_get_state = llexternal('stm_debug_get_state', [], lltype.Signed) diff --git a/pypy/translator/stm/src_stm/et.c b/pypy/translator/stm/src_stm/et.c --- a/pypy/translator/stm/src_stm/et.c +++ b/pypy/translator/stm/src_stm/et.c @@ -82,9 +82,7 @@ owner_version_t my_lock_word; unsigned init_counter; struct RedoLog redolog; /* last item, because it's the biggest one */ -#ifdef RPY_STM_ASSERT int transaction_active; -#endif }; /* global_timestamp contains in its lowest bit a flag equal to 1 @@ -148,9 +146,7 @@ static _Bool is_inevitable(struct tx_descriptor *d) { -#ifdef RPY_STM_ASSERT assert(d->transaction_active); -#endif return is_inevitable_or_inactive(d); } @@ -254,10 +250,8 @@ { d->reads.size = 0; redolog_clear(&d->redolog); -#ifdef RPY_STM_ASSERT assert(d->transaction_active); d->transaction_active = 0; -#endif d->setjmp_buf = NULL; } @@ -687,10 +681,8 @@ void stm_begin_transaction(jmp_buf* buf) { struct tx_descriptor *d = thread_descriptor; -#ifdef RPY_STM_ASSERT assert(!d->transaction_active); d->transaction_active = 1; -#endif d->setjmp_buf = buf; d->start_time = d->last_known_global_timestamp & ~1; } @@ -842,9 +834,7 @@ struct tx_descriptor *d = thread_descriptor; unsigned long curtime; -#ifdef RPY_STM_ASSERT assert(!d->transaction_active); -#endif retry: mutex_lock(); /* possibly waiting here */ @@ -861,10 +851,8 @@ if (bool_cas(&global_timestamp, curtime, curtime + 1)) break; } -#ifdef RPY_STM_ASSERT assert(!d->transaction_active); d->transaction_active = 1; -#endif d->setjmp_buf = NULL; d->start_time = curtime; #ifdef COMMIT_OTHER_INEV @@ -974,4 +962,17 @@ #endif } +long stm_debug_get_state(void) +{ + struct tx_descriptor *d = thread_descriptor; + if (!d) + return -1; + if (!d->transaction_active) + return 0; + if (!is_inevitable(d)) + return 1; + else + return 2; +} + #endif /* PYPY_NOT_MAIN_FILE */ diff --git a/pypy/translator/stm/src_stm/et.h b/pypy/translator/stm/src_stm/et.h --- a/pypy/translator/stm/src_stm/et.h +++ b/pypy/translator/stm/src_stm/et.h @@ -37,6 +37,10 @@ void stm_abort_and_retry(void); void stm_descriptor_init_and_being_inevitable_transaction(void); void stm_commit_transaction_and_descriptor_done(void); +long stm_debug_get_state(void); /* -1: descriptor_init() was not called + 0: not in a transaction + 1: in a regular transaction + 2: in an inevitable transaction */ /* for testing only: */ #define STM_begin_transaction() ; \ diff --git a/pypy/translator/stm/test/test_rffi_stm.py b/pypy/translator/stm/test/test_rffi_stm.py --- a/pypy/translator/stm/test/test_rffi_stm.py +++ b/pypy/translator/stm/test/test_rffi_stm.py @@ -50,6 +50,19 @@ return lltype.nullptr(rffi.VOIDP.TO) stm_descriptor_init() stm_perform_transaction(llhelper(CALLBACK, callback1), - lltype.nullptr(rffi.VOIDP.TO)) + lltype.nullptr(rffi.VOIDP.TO)) stm_descriptor_done() assert a.x == 420 + +def test_stm_debug_get_state(): + def callback1(x): + assert stm_debug_get_state() == 1 + stm_try_inevitable() + assert stm_debug_get_state() == 2 + return lltype.nullptr(rffi.VOIDP.TO) + assert stm_debug_get_state() == -1 + stm_descriptor_init() + assert stm_debug_get_state() == 0 + stm_perform_transaction(llhelper(CALLBACK, callback1), + lltype.nullptr(rffi.VOIDP.TO)) + stm_descriptor_done() From noreply at buildbot.pypy.org Mon Jan 16 20:11:35 2012 From: noreply at buildbot.pypy.org (mattip) Date: Mon, 16 Jan 2012 20:11:35 +0100 (CET) Subject: [pypy-commit] pypy default: add failing test, add fix Message-ID: <20120116191135.CFC46820D8@wyvern.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r51363:ee1b4ea67ed9 Date: 2012-01-16 21:10 +0200 http://bitbucket.org/pypy/pypy/changeset/ee1b4ea67ed9/ Log: add failing test, add fix diff --git a/pypy/module/micronumpy/interp_iter.py b/pypy/module/micronumpy/interp_iter.py --- a/pypy/module/micronumpy/interp_iter.py +++ b/pypy/module/micronumpy/interp_iter.py @@ -157,6 +157,8 @@ offset += self.strides[i] break else: + if i == self.dim: + first_line = True indices[i] = 0 offset -= self.backstrides[i] else: diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -771,6 +771,8 @@ assert ((a + a).T.sum(2).T == (a + a).sum(0)).all() assert (a.reshape(1,-1).sum(0) == range(105)).all() assert (a.reshape(1,-1).sum(1) == 5460) + assert (array([[1,2],[3,4]]).prod(0) == [3, 8]).all() + assert (array([[1,2],[3,4]]).prod(1) == [2, 12]).all() def test_identity(self): from _numpypy import identity, array From noreply at buildbot.pypy.org Mon Jan 16 21:16:09 2012 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 16 Jan 2012 21:16:09 +0100 (CET) Subject: [pypy-commit] pypy default: A bit experimental - try to preallocate the size of unicode join and remove Message-ID: <20120116201610.009C0820D8@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r51364:f6b8525d8a10 Date: 2012-01-16 22:14 +0200 http://bitbucket.org/pypy/pypy/changeset/f6b8525d8a10/ Log: A bit experimental - try to preallocate the size of unicode join and remove a pointless performance hack (the general optimization should work already) diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -201,7 +201,7 @@ return space.newbool(container.find(item) != -1) def unicode_join__Unicode_ANY(space, w_self, w_list): - list_w = space.unpackiterable(w_list) + list_w = space.listview(w_list) size = len(list_w) if size == 0: @@ -216,22 +216,21 @@ def _unicode_join_many_items(space, w_self, list_w, size): self = w_self._value - sb = UnicodeBuilder() + prealloc_size = 0 + for i in range(size): + prealloc_size += len(space.unicode_w(list_w[i])) + sb = UnicodeBuilder(prealloc_size) for i in range(size): if self and i != 0: sb.append(self) w_s = list_w[i] - if isinstance(w_s, W_UnicodeObject): - # shortcut for performance - sb.append(w_s._value) - else: - try: - sb.append(space.unicode_w(w_s)) - except OperationError, e: - if not e.match(space, space.w_TypeError): - raise - raise operationerrfmt(space.w_TypeError, - "sequence item %d: expected string or Unicode", i) + try: + sb.append(space.unicode_w(w_s)) + except OperationError, e: + if not e.match(space, space.w_TypeError): + raise + raise operationerrfmt(space.w_TypeError, + "sequence item %d: expected string or Unicode", i) return space.wrap(sb.build()) def hash__Unicode(space, w_uni): From noreply at buildbot.pypy.org Mon Jan 16 21:16:11 2012 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 16 Jan 2012 21:16:11 +0100 (CET) Subject: [pypy-commit] pypy default: mere Message-ID: <20120116201611.60CE7820D8@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r51365:5e43d79c76a7 Date: 2012-01-16 22:15 +0200 http://bitbucket.org/pypy/pypy/changeset/5e43d79c76a7/ Log: mere diff --git a/pypy/module/micronumpy/interp_iter.py b/pypy/module/micronumpy/interp_iter.py --- a/pypy/module/micronumpy/interp_iter.py +++ b/pypy/module/micronumpy/interp_iter.py @@ -157,6 +157,8 @@ offset += self.strides[i] break else: + if i == self.dim: + first_line = True indices[i] = 0 offset -= self.backstrides[i] else: diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -287,11 +287,11 @@ descr_rmod = _binop_right_impl("mod") def _reduce_ufunc_impl(ufunc_name, promote_to_largest=False): - def impl(self, space, w_dim=None): - if space.is_w(w_dim, space.w_None): - w_dim = space.wrap(-1) + def impl(self, space, w_axis=None): + if space.is_w(w_axis, space.w_None): + w_axis = space.wrap(-1) return getattr(interp_ufuncs.get(space), ufunc_name).reduce(space, - self, True, promote_to_largest, w_dim) + self, True, promote_to_largest, w_axis) return func_with_new_name(impl, "reduce_%s_impl" % ufunc_name) descr_sum = _reduce_ufunc_impl("add") @@ -569,14 +569,14 @@ ) return w_result - def descr_mean(self, space, w_dim=None): - if space.is_w(w_dim, space.w_None): - w_dim = space.wrap(-1) + def descr_mean(self, space, w_axis=None): + if space.is_w(w_axis, space.w_None): + w_axis = space.wrap(-1) w_denom = space.wrap(self.size) else: - dim = space.int_w(w_dim) + dim = space.int_w(w_axis) w_denom = space.wrap(self.shape[dim]) - return space.div(self.descr_sum_promote(space, w_dim), w_denom) + return space.div(self.descr_sum_promote(space, w_axis), w_denom) def descr_var(self, space): # var = mean((values - mean(values)) ** 2) diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -733,6 +733,7 @@ a = array(range(105)).reshape(3, 5, 7) b = mean(a, axis=0) b[0,0]==35. + assert a.mean(axis=0)[0, 0] == 35 assert (b == array(range(35, 70), dtype=float).reshape(5, 7)).all() assert (mean(a, 2) == array(range(0, 15), dtype=float).reshape(3, 5) * 7 + 3).all() @@ -755,6 +756,7 @@ assert array([]).sum() == 0.0 raises(ValueError, 'array([]).max()') assert (a.sum(0) == [30, 35, 40]).all() + assert (a.sum(axis=0) == [30, 35, 40]).all() assert (a.sum(1) == [3, 12, 21, 30, 39]).all() assert (a.max(0) == [12, 13, 14]).all() assert (a.max(1) == [2, 5, 8, 11, 14]).all() @@ -769,6 +771,8 @@ assert ((a + a).T.sum(2).T == (a + a).sum(0)).all() assert (a.reshape(1,-1).sum(0) == range(105)).all() assert (a.reshape(1,-1).sum(1) == 5460) + assert (array([[1,2],[3,4]]).prod(0) == [3, 8]).all() + assert (array([[1,2],[3,4]]).prod(1) == [2, 12]).all() def test_identity(self): from _numpypy import identity, array diff --git a/pypy/module/pypyjit/policy.py b/pypy/module/pypyjit/policy.py --- a/pypy/module/pypyjit/policy.py +++ b/pypy/module/pypyjit/policy.py @@ -127,7 +127,7 @@ 'imp', 'sys', 'array', '_ffi', 'itertools', 'operator', 'posix', '_socket', '_sre', '_lsprof', '_weakref', '__pypy__', 'cStringIO', '_collections', 'struct', - 'mmap', 'marshal']: + 'mmap', 'marshal', '_codecs']: if modname == 'pypyjit' and 'interp_resop' in rest: return False return True diff --git a/pypy/rpython/lltypesystem/rclass.py b/pypy/rpython/lltypesystem/rclass.py --- a/pypy/rpython/lltypesystem/rclass.py +++ b/pypy/rpython/lltypesystem/rclass.py @@ -510,7 +510,13 @@ ctype = inputconst(Void, self.object_type) cflags = inputconst(Void, flags) vlist = [ctype, cflags] - vptr = llops.genop('malloc', vlist, + cnonmovable = self.classdef.classdesc.read_attribute( + '_alloc_nonmovable_', Constant(False)) + if cnonmovable.value: + opname = 'malloc_nonmovable' + else: + opname = 'malloc' + vptr = llops.genop(opname, vlist, resulttype = Ptr(self.object_type)) ctypeptr = inputconst(CLASSTYPE, self.rclass.getvtable()) self.setfield(vptr, '__class__', ctypeptr, llops) diff --git a/pypy/rpython/test/test_rclass.py b/pypy/rpython/test/test_rclass.py --- a/pypy/rpython/test/test_rclass.py +++ b/pypy/rpython/test/test_rclass.py @@ -1130,6 +1130,18 @@ assert sorted([u]) == [6] # 32-bit types assert sorted([i, r, d, l]) == [2, 3, 4, 5] # 64-bit types + def test_nonmovable(self): + for (nonmovable, opname) in [(True, 'malloc_nonmovable'), + (False, 'malloc')]: + class A(object): + _alloc_nonmovable_ = nonmovable + def f(): + return A() + t, typer, graph = self.gengraph(f, []) + assert summary(graph) == {opname: 1, + 'cast_pointer': 1, + 'setfield': 1} + class TestOOtype(BaseTestRclass, OORtypeMixin): From notifications-noreply at bitbucket.org Mon Jan 16 23:03:37 2012 From: notifications-noreply at bitbucket.org (Bitbucket) Date: Mon, 16 Jan 2012 22:03:37 -0000 Subject: [pypy-commit] Notification: pypy Message-ID: <20120116220337.18609.7903@bitbucket05.managed.contegix.com> You have received a notification from Romain Guillebert. Hi, I forked pypy. My fork is at https://bitbucket.org/rguillebert/pypy. -- Disable notifications at https://bitbucket.org/account/notifications/ From noreply at buildbot.pypy.org Tue Jan 17 00:17:03 2012 From: noreply at buildbot.pypy.org (rguillebert) Date: Tue, 17 Jan 2012 00:17:03 +0100 (CET) Subject: [pypy-commit] pypy py3k: Reject old octal literals Message-ID: <20120116231703.0C4B5820D8@wyvern.cs.uni-duesseldorf.de> Author: Romain Guillebert Branch: py3k Changeset: r51366:d745dbc657b4 Date: 2012-01-16 18:25 +0100 http://bitbucket.org/pypy/pypy/changeset/d745dbc657b4/ Log: Reject old octal literals diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py --- a/pypy/interpreter/pycode.py +++ b/pypy/interpreter/pycode.py @@ -143,7 +143,7 @@ elif opcode == opcodedesc.LOAD_NAME.index: self.co_flags |= CO_CONTAINSGLOBALS - co_names = property(lambda self: [self.space.unwrap(w_name) for w_name in self.co_names_w]) # for trace + co_names = property(lambda self: [self.space.str_w(w_name) for w_name in self.co_names_w]) # for trace def signature(self): return self._signature diff --git a/pypy/interpreter/pyparser/genpytokenize.py b/pypy/interpreter/pyparser/genpytokenize.py --- a/pypy/interpreter/pyparser/genpytokenize.py +++ b/pypy/interpreter/pyparser/genpytokenize.py @@ -63,10 +63,8 @@ groupStr(states, "0123456789abcdefABCDEF"))) octNumber = chain(states, newArcPair(states, "0"), - maybe(states, - chain(states, - groupStr(states, "oO"), - groupStr(states, "01234567"))), + groupStr(states, "oO"), + groupStr(states, "01234567"), any(states, groupStr(states, "01234567"))) binNumber = chain(states, newArcPair(states, "0"), @@ -75,7 +73,8 @@ decNumber = chain(states, groupStr(states, "123456789"), any(states, makeDigits())) - intNumber = group(states, hexNumber, octNumber, binNumber, decNumber) + zero = newArcPair(states, "0") + intNumber = group(states, hexNumber, octNumber, binNumber, decNumber, zero) # ____________________________________________________________ # Exponents def makeExp (): diff --git a/pypy/interpreter/pyparser/pytokenize.py b/pypy/interpreter/pyparser/pytokenize.py --- a/pypy/interpreter/pyparser/pytokenize.py +++ b/pypy/interpreter/pyparser/pytokenize.py @@ -25,10 +25,10 @@ accepts = [True, True, True, True, True, True, True, True, True, True, False, True, True, True, True, False, - False, False, True, False, False, True, False, - False, True, False, True, False, True, False, - False, True, False, False, True, True, True, - False, False, True, False, False, False, True] + False, False, True, False, False, False, False, + True, False, True, False, True, False, False, + True, False, False, True, True, True, False, + False, True, False, False, False, True] states = [ # 0 {'\t': 0, '\n': 13, '\x0c': 0, @@ -110,21 +110,21 @@ 'v': 1, 'w': 1, 'x': 1, 'y': 1, 'z': 1}, # 4 - {'.': 24, '0': 21, '1': 21, '2': 21, - '3': 21, '4': 21, '5': 21, '6': 21, - '7': 21, '8': 23, '9': 23, 'B': 22, - 'E': 25, 'J': 13, 'O': 20, 'X': 19, - 'b': 22, 'e': 25, 'j': 13, 'o': 20, + {'.': 23, '0': 22, '1': 22, '2': 22, + '3': 22, '4': 22, '5': 22, '6': 22, + '7': 22, '8': 22, '9': 22, 'B': 21, + 'E': 24, 'J': 13, 'O': 20, 'X': 19, + 'b': 21, 'e': 24, 'j': 13, 'o': 20, 'x': 19}, # 5 - {'.': 24, '0': 5, '1': 5, '2': 5, + {'.': 23, '0': 5, '1': 5, '2': 5, '3': 5, '4': 5, '5': 5, '6': 5, - '7': 5, '8': 5, '9': 5, 'E': 25, - 'J': 13, 'e': 25, 'j': 13}, + '7': 5, '8': 5, '9': 5, 'E': 24, + 'J': 13, 'e': 24, 'j': 13}, # 6 - {'0': 26, '1': 26, '2': 26, '3': 26, - '4': 26, '5': 26, '6': 26, '7': 26, - '8': 26, '9': 26}, + {'0': 25, '1': 25, '2': 25, '3': 25, + '4': 25, '5': 25, '6': 25, '7': 25, + '8': 25, '9': 25}, # 7 {'*': 12, '=': 13}, # 8 @@ -142,105 +142,100 @@ # 14 {'\n': 13}, # 15 - {automata.DEFAULT: 30, '\n': 27, - '\r': 27, "'": 28, '\\': 29}, + {automata.DEFAULT: 29, '\n': 26, + '\r': 26, "'": 27, '\\': 28}, # 16 - {automata.DEFAULT: 33, '\n': 27, - '\r': 27, '"': 31, '\\': 32}, + {automata.DEFAULT: 32, '\n': 26, + '\r': 26, '"': 30, '\\': 31}, # 17 {'\n': 13, '\r': 14}, # 18 - {automata.DEFAULT: 18, '\n': 27, '\r': 27}, + {automata.DEFAULT: 18, '\n': 26, '\r': 26}, # 19 + {'0': 33, '1': 33, '2': 33, '3': 33, + '4': 33, '5': 33, '6': 33, '7': 33, + '8': 33, '9': 33, 'A': 33, 'B': 33, + 'C': 33, 'D': 33, 'E': 33, 'F': 33, + 'a': 33, 'b': 33, 'c': 33, 'd': 33, + 'e': 33, 'f': 33}, + # 20 {'0': 34, '1': 34, '2': 34, '3': 34, - '4': 34, '5': 34, '6': 34, '7': 34, - '8': 34, '9': 34, 'A': 34, 'B': 34, - 'C': 34, 'D': 34, 'E': 34, 'F': 34, - 'a': 34, 'b': 34, 'c': 34, 'd': 34, - 'e': 34, 'f': 34}, - # 20 - {'0': 35, '1': 35, '2': 35, '3': 35, - '4': 35, '5': 35, '6': 35, '7': 35}, + '4': 34, '5': 34, '6': 34, '7': 34}, # 21 - {'.': 24, '0': 21, '1': 21, '2': 21, - '3': 21, '4': 21, '5': 21, '6': 21, - '7': 21, '8': 23, '9': 23, 'E': 25, - 'J': 13, 'e': 25, 'j': 13}, + {'0': 35, '1': 35}, # 22 - {'0': 36, '1': 36}, + {'.': 23, '0': 22, '1': 22, '2': 22, + '3': 22, '4': 22, '5': 22, '6': 22, + '7': 22, '8': 22, '9': 22, 'E': 24, + 'J': 13, 'e': 24, 'j': 13}, # 23 - {'.': 24, '0': 23, '1': 23, '2': 23, - '3': 23, '4': 23, '5': 23, '6': 23, - '7': 23, '8': 23, '9': 23, 'E': 25, - 'J': 13, 'e': 25, 'j': 13}, + {'0': 23, '1': 23, '2': 23, '3': 23, + '4': 23, '5': 23, '6': 23, '7': 23, + '8': 23, '9': 23, 'E': 36, 'J': 13, + 'e': 36, 'j': 13}, # 24 - {'0': 24, '1': 24, '2': 24, '3': 24, - '4': 24, '5': 24, '6': 24, '7': 24, - '8': 24, '9': 24, 'E': 37, 'J': 13, - 'e': 37, 'j': 13}, + {'+': 37, '-': 37, '0': 38, '1': 38, + '2': 38, '3': 38, '4': 38, '5': 38, + '6': 38, '7': 38, '8': 38, '9': 38}, # 25 - {'+': 38, '-': 38, '0': 39, '1': 39, - '2': 39, '3': 39, '4': 39, '5': 39, - '6': 39, '7': 39, '8': 39, '9': 39}, + {'0': 25, '1': 25, '2': 25, '3': 25, + '4': 25, '5': 25, '6': 25, '7': 25, + '8': 25, '9': 25, 'E': 36, 'J': 13, + 'e': 36, 'j': 13}, # 26 - {'0': 26, '1': 26, '2': 26, '3': 26, - '4': 26, '5': 26, '6': 26, '7': 26, - '8': 26, '9': 26, 'E': 37, 'J': 13, - 'e': 37, 'j': 13}, + {}, # 27 - {}, + {"'": 13}, # 28 - {"'": 13}, + {automata.DEFAULT: 39, '\n': 13, '\r': 14}, # 29 + {automata.DEFAULT: 29, '\n': 26, + '\r': 26, "'": 13, '\\': 28}, + # 30 + {'"': 13}, + # 31 {automata.DEFAULT: 40, '\n': 13, '\r': 14}, - # 30 - {automata.DEFAULT: 30, '\n': 27, - '\r': 27, "'": 13, '\\': 29}, - # 31 - {'"': 13}, # 32 - {automata.DEFAULT: 41, '\n': 13, '\r': 14}, + {automata.DEFAULT: 32, '\n': 26, + '\r': 26, '"': 13, '\\': 31}, # 33 - {automata.DEFAULT: 33, '\n': 27, - '\r': 27, '"': 13, '\\': 32}, + {'0': 33, '1': 33, '2': 33, '3': 33, + '4': 33, '5': 33, '6': 33, '7': 33, + '8': 33, '9': 33, 'A': 33, 'B': 33, + 'C': 33, 'D': 33, 'E': 33, 'F': 33, + 'a': 33, 'b': 33, 'c': 33, 'd': 33, + 'e': 33, 'f': 33}, # 34 {'0': 34, '1': 34, '2': 34, '3': 34, - '4': 34, '5': 34, '6': 34, '7': 34, - '8': 34, '9': 34, 'A': 34, 'B': 34, - 'C': 34, 'D': 34, 'E': 34, 'F': 34, - 'a': 34, 'b': 34, 'c': 34, 'd': 34, - 'e': 34, 'f': 34}, + '4': 34, '5': 34, '6': 34, '7': 34}, # 35 - {'0': 35, '1': 35, '2': 35, '3': 35, - '4': 35, '5': 35, '6': 35, '7': 35}, + {'0': 35, '1': 35}, # 36 - {'0': 36, '1': 36}, + {'+': 41, '-': 41, '0': 42, '1': 42, + '2': 42, '3': 42, '4': 42, '5': 42, + '6': 42, '7': 42, '8': 42, '9': 42}, # 37 - {'+': 42, '-': 42, '0': 43, '1': 43, - '2': 43, '3': 43, '4': 43, '5': 43, - '6': 43, '7': 43, '8': 43, '9': 43}, + {'0': 38, '1': 38, '2': 38, '3': 38, + '4': 38, '5': 38, '6': 38, '7': 38, + '8': 38, '9': 38}, # 38 - {'0': 39, '1': 39, '2': 39, '3': 39, - '4': 39, '5': 39, '6': 39, '7': 39, - '8': 39, '9': 39}, + {'0': 38, '1': 38, '2': 38, '3': 38, + '4': 38, '5': 38, '6': 38, '7': 38, + '8': 38, '9': 38, 'J': 13, 'j': 13}, # 39 - {'0': 39, '1': 39, '2': 39, '3': 39, - '4': 39, '5': 39, '6': 39, '7': 39, - '8': 39, '9': 39, 'J': 13, 'j': 13}, + {automata.DEFAULT: 39, '\n': 26, + '\r': 26, "'": 13, '\\': 28}, # 40 - {automata.DEFAULT: 40, '\n': 27, - '\r': 27, "'": 13, '\\': 29}, + {automata.DEFAULT: 40, '\n': 26, + '\r': 26, '"': 13, '\\': 31}, # 41 - {automata.DEFAULT: 41, '\n': 27, - '\r': 27, '"': 13, '\\': 32}, + {'0': 42, '1': 42, '2': 42, '3': 42, + '4': 42, '5': 42, '6': 42, '7': 42, + '8': 42, '9': 42}, # 42 - {'0': 43, '1': 43, '2': 43, '3': 43, - '4': 43, '5': 43, '6': 43, '7': 43, - '8': 43, '9': 43}, - # 43 - {'0': 43, '1': 43, '2': 43, '3': 43, - '4': 43, '5': 43, '6': 43, '7': 43, - '8': 43, '9': 43, 'J': 13, 'j': 13}, + {'0': 42, '1': 42, '2': 42, '3': 42, + '4': 42, '5': 42, '6': 42, '7': 42, + '8': 42, '9': 42, 'J': 13, 'j': 13}, ] pseudoDFA = automata.DFA(states, accepts) @@ -304,6 +299,8 @@ ] doubleDFA = automata.DFA(states, accepts) + + #_______________________________________________________________________ # End of automatically generated DFA's From noreply at buildbot.pypy.org Tue Jan 17 00:17:04 2012 From: noreply at buildbot.pypy.org (rguillebert) Date: Tue, 17 Jan 2012 00:17:04 +0100 (CET) Subject: [pypy-commit] pypy py3k: Use new octal literals Message-ID: <20120116231704.33DCC820D8@wyvern.cs.uni-duesseldorf.de> Author: Romain Guillebert Branch: py3k Changeset: r51367:4714f110a1ae Date: 2012-01-16 18:58 +0100 http://bitbucket.org/pypy/pypy/changeset/4714f110a1ae/ Log: Use new octal literals diff --git a/pypy/translator/goal/nanos.py b/pypy/translator/goal/nanos.py --- a/pypy/translator/goal/nanos.py +++ b/pypy/translator/goal/nanos.py @@ -85,7 +85,7 @@ st = posix.stat(path) except posix.error: return False - return (st.st_mode & 0170000) == 0100000 # S_ISREG + return (st.st_mode & 0o170000) == 0o100000 # S_ISREG def islink(path): import posix @@ -93,7 +93,7 @@ st = posix.lstat(path) except posix.error: return False - return (st.st_mode & 0170000) == 0120000 # S_ISLNK + return (st.st_mode & 0o170000) == 0o120000 # S_ISLNK """, filename=__file__) @@ -249,7 +249,7 @@ st = nt.stat(path) except nt.error: return False - return (st.st_mode & 0170000) == 0100000 # S_ISREG + return (st.st_mode & 0o170000) == 0o100000 # S_ISREG def islink(path): return False From noreply at buildbot.pypy.org Tue Jan 17 00:26:32 2012 From: noreply at buildbot.pypy.org (amauryfa) Date: Tue, 17 Jan 2012 00:26:32 +0100 (CET) Subject: [pypy-commit] pypy py3k: PEP3147: .pyc files are now named ./__pycache__/foo.pypy-17.pyc Message-ID: <20120116232632.6BF12820D8@wyvern.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3k Changeset: r51368:a20df1bb1bb8 Date: 2012-01-16 23:53 +0100 http://bitbucket.org/pypy/pypy/changeset/a20df1bb1bb8/ Log: PEP3147: .pyc files are now named ./__pycache__/foo.pypy-17.pyc (Note: the "nolonepycfile" option makes less sense now) diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py --- a/pypy/module/imp/importing.py +++ b/pypy/module/imp/importing.py @@ -34,6 +34,8 @@ DEFAULT_SOABI = 'pypy-%d%d' % PYPY_VERSION[:2] CHECK_FOR_PYW = sys.platform == 'win32' +PYC_TAG = 'pypy-%d%d' % PYPY_VERSION[:2] + @specialize.memo() def get_so_extension(space): if space.config.objspace.soabi is not None: @@ -91,10 +93,7 @@ else: # XXX that's slow def case_ok(filename): - index = filename.rfind(os.sep) - if os.altsep is not None: - index2 = filename.rfind(os.altsep) - index = max(index, index2) + index = rightmost_sep(filename) if index < 0: directory = os.curdir else: @@ -862,17 +861,63 @@ space.wrap(space.builtin)) code_w.exec_code(space, w_dict, w_dict) +def rightmost_sep(filename): + "Like filename.rfind('/'), but also search for \\." + index = filename.rfind(os.sep) + if os.altsep is not None: + index2 = filename.rfind(os.altsep) + index = max(index, index2) + return index + def make_compiled_pathname(pathname): "Given the path to a .py file, return the path to its .pyc file." - return pathname + 'c' + # foo.py -> __pycache__/foo..pyc + + lastpos = rightmost_sep(pathname) + 1 + assert lastpos >= 0 # zero when slash, takes the full name + fname = pathname[lastpos:] + if lastpos > 0: + # Windows: re-use the last separator character (/ or \\) when + # appending the __pycache__ path. + lastsep = pathname[lastpos-1] + else: + lastsep = os.sep + ext = fname + for i in range(len(fname)): + if fname[i] == '.': + ext = fname[:i + 1] + + result = (pathname[:lastpos] + "__pycache__" + lastsep + + ext + PYC_TAG + '.pyc') + return result def make_source_pathname(pathname): - pos_extension = len(pathname) - 4 # len('.pyc') - if pos_extension < 0: - raise ValueError("path is too short") - if pathname[pos_extension:] != '.pyc': - raise ValueError("not a .pyc path name") - return pathname[:pos_extension + 3] + "Given the path to a .pyc file, return the path to its .py file." + # (...)/__pycache__/foo..pyc -> (...)/foo.py + + right = rightmost_sep(pathname) + if right < 0: + raise ValueError() + left = rightmost_sep(pathname[:right]) + 1 + assert left >= 0 + if pathname[left:right] != '__pycache__': + raise ValueError() + + # Now verify that the path component to the right of the last + # slash has two dots in it. + rightpart = pathname[right + 1:] + dot0 = rightpart.find('.') + 1 + if dot0 <= 0: + raise ValueError + dot1 = rightpart[dot0:].find('.') + 1 + if dot1 <= 0: + raise ValueError + # Too many dots? + if rightpart[dot0 + dot1:].find('.') >= 0: + raise ValueError + + result = pathname[:left] + rightpart[:dot0] + 'py' + return result @jit.dont_look_inside def load_source_module(space, w_modulename, w_mod, pathname, source, @@ -1027,6 +1072,17 @@ Errors are ignored, if a write error occurs an attempt is made to remove the file. """ + # Ensure that the __pycache__ directory exists + dirsep = rightmost_sep(cpathname) + if dirsep < 0: + return + dirname = cpathname[:dirsep] + mode = src_mode | 0333 # +wx + try: + os.mkdir(dirname, mode) + except OSError: + pass + w_marshal = space.getbuiltinmodule('marshal') try: w_bytes = space.call_method(w_marshal, 'dumps', space.wrap(co), diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py --- a/pypy/module/imp/test/test_import.py +++ b/pypy/module/imp/test/test_import.py @@ -38,7 +38,7 @@ test_reload = "def test():\n raise ValueError\n", infinite_reload = "import infinite_reload; reload(infinite_reload)", del_sys_module = "import sys\ndel sys.modules['del_sys_module']\n", - itertools = "hello_world = 42\n", + queue = "hello_world = 42\n", gc = "should_never_be_seen = 42\n", ) root.ensure("notapackage", dir=1) # empty, no __init__.py @@ -75,7 +75,7 @@ ) setuppkg("pkg_substituting", __init__ = "import sys, pkg_substituted\n" - "print 'TOTO', __name__\n" + "print('TOTO', __name__)\n" "sys.modules[__name__] = pkg_substituted") setuppkg("pkg_substituted", mod='') setuppkg("evil_pkg", @@ -102,6 +102,7 @@ w = space.wrap w_modname = w("compiled.x") filename = str(p.join("x.py")) + pycname = importing.make_compiled_pathname("x.py") stream = streamio.open_file_as_stream(filename, "r") try: importing.load_source_module(space, @@ -113,7 +114,7 @@ stream.close() if space.config.objspace.usepycfiles: # also create a lone .pyc file - p.join('lone.pyc').write(p.join('x.pyc').read(mode='rb'), + p.join('lone.pyc').write(p.join(pycname).read(mode='rb'), mode='wb') # create a .pyw file @@ -150,7 +151,7 @@ class AppTestImport: def setup_class(cls): # interpreter-level - cls.space = gettestobjspace(usemodules=['itertools']) + #cls.space = gettestobjspace(usemodules=['itertools']) cls.w_runappdirect = cls.space.wrap(conftest.option.runappdirect) cls.saved_modules = _setup(cls.space) #XXX Compile class @@ -159,6 +160,9 @@ def teardown_class(cls): # interpreter-level _teardown(cls.space, cls.saved_modules) + def w_exec_(self, cmd, ns): + exec(cmd, ns) + def test_set_sys_modules_during_import(self): from evil_pkg import evil assert evil.a == 42 @@ -373,7 +377,7 @@ def test_invalid__name__(self): glob = {} - exec "__name__ = None; import sys" in glob + exec("__name__ = None; import sys", glob) import sys assert glob['sys'] is sys @@ -418,16 +422,18 @@ assert pkg.pkg1.__package__ == 'pkg.pkg1' def test_future_relative_import_error_when_in_non_package(self): - exec """def imp(): + ns = {} + exec("""def imp(): from .string import inpackage - """.rstrip() - raises(ValueError, imp) + """.rstrip(), ns) + raises(ValueError, ns['imp']) def test_future_relative_import_error_when_in_non_package2(self): - exec """def imp(): + ns = {} + exec("""def imp(): from .. import inpackage - """.rstrip() - raises(ValueError, imp) + """.rstrip(), ns) + raises(ValueError, ns['imp']) def test_relative_import_with___name__(self): import sys @@ -456,9 +462,9 @@ def test__package__(self): # Regression test for http://bugs.python.org/issue3221. def check_absolute(): - exec "from os import path" in ns + self.exec_("from os import path", ns) def check_relative(): - exec "from . import a" in ns + self.exec_("from . import a", ns) # Check both OK with __package__ and __name__ correct ns = dict(__package__='pkg', __name__='pkg.notarealmodule') @@ -578,8 +584,11 @@ def test_cache_from_source(self): import imp - assert imp.cache_from_source('a/b/c.py') == 'a/b/c.pyc' - assert imp.source_from_cache('a/b/c.pyc') == 'a/b/c.py' + pycfile = imp.cache_from_source('a/b/c.py') + assert pycfile.startswith('a/b/__pycache__/c.pypy-') + assert pycfile.endswith('.pyc') + assert imp.source_from_cache('a/b/__pycache__/c.pypy-17.pyc' + ) == 'a/b/c.py' raises(ValueError, imp.source_from_cache, 'a/b/c.py') def test_shadow_builtin(self): @@ -596,34 +605,32 @@ def test_shadow_extension_1(self): if self.runappdirect: skip("hard to test: module is already imported") - # 'import itertools' is supposed to find itertools.py if there is + # 'import queue' is supposed to find queue.py if there is # one in sys.path. import sys - assert 'itertools' not in sys.modules - import itertools - assert hasattr(itertools, 'hello_world') - assert not hasattr(itertools, 'count') - assert '(built-in)' not in repr(itertools) - del sys.modules['itertools'] + assert 'queue' not in sys.modules + import queue + assert hasattr(queue, 'hello_world') + assert not hasattr(queue, 'count') + assert '(built-in)' not in repr(queue) + del sys.modules['queue'] def test_shadow_extension_2(self): if self.runappdirect: skip("hard to test: module is already imported") - # 'import itertools' is supposed to find the built-in module even + # 'import queue' is supposed to find the built-in module even # if there is also one in sys.path as long as it is *after* the - # special entry '.../lib_pypy/__extensions__'. (Note that for now - # there is one in lib_pypy/itertools.py, which should not be seen - # either; hence the (built-in) test below.) + # special entry '.../lib_pypy/__extensions__'. import sys - assert 'itertools' not in sys.modules + assert 'queue' not in sys.modules sys.path.append(sys.path.pop(0)) try: - import itertools - assert not hasattr(itertools, 'hello_world') - assert hasattr(itertools, 'izip') - assert '(built-in)' in repr(itertools) + import queue + assert not hasattr(queue, 'hello_world') + assert hasattr(queue, 'izip') + assert '(built-in)' in repr(queue) finally: sys.path.insert(0, sys.path.pop()) - del sys.modules['itertools'] + del sys.modules['queue'] class TestAbi: @@ -885,7 +892,7 @@ stream.close() # And the .pyc has been generated - cpathname = udir.join('test.pyc') + cpathname = udir.join(importing.make_compiled_pathname('test.py')) assert cpathname.check() def test_write_compiled_module(self): @@ -979,7 +986,7 @@ py.test.skip("unresolved issues with win32 shell quoting rules") from pypy.interpreter.test.test_zpy import pypypath extrapath = udir.ensure("pythonpath", dir=1) - extrapath.join("urllib.py").write("print 42\n") + extrapath.join("urllib.py").write("print(42)\n") old = os.environ.get('PYTHONPATH', None) oldlang = os.environ.pop('LANG', None) try: @@ -1024,7 +1031,7 @@ if fullname in self.namestoblock: return self def load_module(self, fullname): - raise ImportError, "blocked" + raise ImportError("blocked") import sys, imp modname = "errno" # an arbitrary harmless builtin module @@ -1094,7 +1101,7 @@ path = [self.path] try: file, filename, stuff = imp.find_module(subname, path) - except ImportError, e: + except ImportError: return None return ImpLoader(file, filename, stuff) @@ -1139,8 +1146,8 @@ def test_run_compiled_module(self): # XXX minimal test only - import imp, new - module = new.module('foobar') + import imp, types + module = types.ModuleType('foobar') raises(IOError, imp._run_compiled_module, 'foobar', 'this_file_does_not_exist', None, module) @@ -1161,14 +1168,14 @@ # a mostly-empty zip file path = os.path.join(self.udir, 'test_getimporter.zip') f = open(path, 'wb') - f.write('PK\x03\x04\n\x00\x00\x00\x00\x00P\x9eN>\x00\x00\x00\x00\x00' - '\x00\x00\x00\x00\x00\x00\x00\x05\x00\x15\x00emptyUT\t\x00' - '\x03wyYMwyYMUx\x04\x00\xf4\x01d\x00PK\x01\x02\x17\x03\n\x00' - '\x00\x00\x00\x00P\x9eN>\x00\x00\x00\x00\x00\x00\x00\x00\x00' - '\x00\x00\x00\x05\x00\r\x00\x00\x00\x00\x00\x00\x00\x00\x00' - '\xa4\x81\x00\x00\x00\x00emptyUT\x05\x00\x03wyYMUx\x00\x00PK' - '\x05\x06\x00\x00\x00\x00\x01\x00\x01\x00@\x00\x00\x008\x00' - '\x00\x00\x00\x00') + f.write(b'PK\x03\x04\n\x00\x00\x00\x00\x00P\x9eN>\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x05\x00\x15\x00emptyUT\t\x00' + b'\x03wyYMwyYMUx\x04\x00\xf4\x01d\x00PK\x01\x02\x17\x03\n\x00' + b'\x00\x00\x00\x00P\x9eN>\x00\x00\x00\x00\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x05\x00\r\x00\x00\x00\x00\x00\x00\x00\x00\x00' + b'\xa4\x81\x00\x00\x00\x00emptyUT\x05\x00\x03wyYMUx\x00\x00PK' + b'\x05\x06\x00\x00\x00\x00\x01\x00\x01\x00@\x00\x00\x008\x00' + b'\x00\x00\x00\x00') f.close() importer = imp._getimporter(path) import zipimport @@ -1193,16 +1200,16 @@ def test_import_possibly_from_pyc(self): from compiled import x if self.usepycfiles: - assert x.__file__.endswith('x.pyc') + assert x.__file__.endswith('.pyc') else: - assert x.__file__.endswith('x.py') + assert x.__file__.endswith('.py') try: from compiled import lone except ImportError: assert not self.lonepycfiles, "should have found 'lone.pyc'" else: assert self.lonepycfiles, "should not have found 'lone.pyc'" - assert lone.__file__.endswith('lone.pyc') + assert lone.__file__.endswith('.pyc') class AppTestNoLonePycFile(AppTestNoPycFile): spaceconfig = { From noreply at buildbot.pypy.org Tue Jan 17 00:26:33 2012 From: noreply at buildbot.pypy.org (amauryfa) Date: Tue, 17 Jan 2012 00:26:33 +0100 (CET) Subject: [pypy-commit] pypy py3k: Remove long() and L suffix from binascii.py Message-ID: <20120116232633.92827820D8@wyvern.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3k Changeset: r51369:708194fb76ef Date: 2012-01-16 20:32 +0100 http://bitbucket.org/pypy/pypy/changeset/708194fb76ef/ Log: Remove long() and L suffix from binascii.py diff --git a/lib_pypy/binascii.py b/lib_pypy/binascii.py --- a/lib_pypy/binascii.py +++ b/lib_pypy/binascii.py @@ -598,68 +598,68 @@ return ''.join(result) crc_32_tab = [ - 0x00000000L, 0x77073096L, 0xee0e612cL, 0x990951baL, 0x076dc419L, - 0x706af48fL, 0xe963a535L, 0x9e6495a3L, 0x0edb8832L, 0x79dcb8a4L, - 0xe0d5e91eL, 0x97d2d988L, 0x09b64c2bL, 0x7eb17cbdL, 0xe7b82d07L, - 0x90bf1d91L, 0x1db71064L, 0x6ab020f2L, 0xf3b97148L, 0x84be41deL, - 0x1adad47dL, 0x6ddde4ebL, 0xf4d4b551L, 0x83d385c7L, 0x136c9856L, - 0x646ba8c0L, 0xfd62f97aL, 0x8a65c9ecL, 0x14015c4fL, 0x63066cd9L, - 0xfa0f3d63L, 0x8d080df5L, 0x3b6e20c8L, 0x4c69105eL, 0xd56041e4L, - 0xa2677172L, 0x3c03e4d1L, 0x4b04d447L, 0xd20d85fdL, 0xa50ab56bL, - 0x35b5a8faL, 0x42b2986cL, 0xdbbbc9d6L, 0xacbcf940L, 0x32d86ce3L, - 0x45df5c75L, 0xdcd60dcfL, 0xabd13d59L, 0x26d930acL, 0x51de003aL, - 0xc8d75180L, 0xbfd06116L, 0x21b4f4b5L, 0x56b3c423L, 0xcfba9599L, - 0xb8bda50fL, 0x2802b89eL, 0x5f058808L, 0xc60cd9b2L, 0xb10be924L, - 0x2f6f7c87L, 0x58684c11L, 0xc1611dabL, 0xb6662d3dL, 0x76dc4190L, - 0x01db7106L, 0x98d220bcL, 0xefd5102aL, 0x71b18589L, 0x06b6b51fL, - 0x9fbfe4a5L, 0xe8b8d433L, 0x7807c9a2L, 0x0f00f934L, 0x9609a88eL, - 0xe10e9818L, 0x7f6a0dbbL, 0x086d3d2dL, 0x91646c97L, 0xe6635c01L, - 0x6b6b51f4L, 0x1c6c6162L, 0x856530d8L, 0xf262004eL, 0x6c0695edL, - 0x1b01a57bL, 0x8208f4c1L, 0xf50fc457L, 0x65b0d9c6L, 0x12b7e950L, - 0x8bbeb8eaL, 0xfcb9887cL, 0x62dd1ddfL, 0x15da2d49L, 0x8cd37cf3L, - 0xfbd44c65L, 0x4db26158L, 0x3ab551ceL, 0xa3bc0074L, 0xd4bb30e2L, - 0x4adfa541L, 0x3dd895d7L, 0xa4d1c46dL, 0xd3d6f4fbL, 0x4369e96aL, - 0x346ed9fcL, 0xad678846L, 0xda60b8d0L, 0x44042d73L, 0x33031de5L, - 0xaa0a4c5fL, 0xdd0d7cc9L, 0x5005713cL, 0x270241aaL, 0xbe0b1010L, - 0xc90c2086L, 0x5768b525L, 0x206f85b3L, 0xb966d409L, 0xce61e49fL, - 0x5edef90eL, 0x29d9c998L, 0xb0d09822L, 0xc7d7a8b4L, 0x59b33d17L, - 0x2eb40d81L, 0xb7bd5c3bL, 0xc0ba6cadL, 0xedb88320L, 0x9abfb3b6L, - 0x03b6e20cL, 0x74b1d29aL, 0xead54739L, 0x9dd277afL, 0x04db2615L, - 0x73dc1683L, 0xe3630b12L, 0x94643b84L, 0x0d6d6a3eL, 0x7a6a5aa8L, - 0xe40ecf0bL, 0x9309ff9dL, 0x0a00ae27L, 0x7d079eb1L, 0xf00f9344L, - 0x8708a3d2L, 0x1e01f268L, 0x6906c2feL, 0xf762575dL, 0x806567cbL, - 0x196c3671L, 0x6e6b06e7L, 0xfed41b76L, 0x89d32be0L, 0x10da7a5aL, - 0x67dd4accL, 0xf9b9df6fL, 0x8ebeeff9L, 0x17b7be43L, 0x60b08ed5L, - 0xd6d6a3e8L, 0xa1d1937eL, 0x38d8c2c4L, 0x4fdff252L, 0xd1bb67f1L, - 0xa6bc5767L, 0x3fb506ddL, 0x48b2364bL, 0xd80d2bdaL, 0xaf0a1b4cL, - 0x36034af6L, 0x41047a60L, 0xdf60efc3L, 0xa867df55L, 0x316e8eefL, - 0x4669be79L, 0xcb61b38cL, 0xbc66831aL, 0x256fd2a0L, 0x5268e236L, - 0xcc0c7795L, 0xbb0b4703L, 0x220216b9L, 0x5505262fL, 0xc5ba3bbeL, - 0xb2bd0b28L, 0x2bb45a92L, 0x5cb36a04L, 0xc2d7ffa7L, 0xb5d0cf31L, - 0x2cd99e8bL, 0x5bdeae1dL, 0x9b64c2b0L, 0xec63f226L, 0x756aa39cL, - 0x026d930aL, 0x9c0906a9L, 0xeb0e363fL, 0x72076785L, 0x05005713L, - 0x95bf4a82L, 0xe2b87a14L, 0x7bb12baeL, 0x0cb61b38L, 0x92d28e9bL, - 0xe5d5be0dL, 0x7cdcefb7L, 0x0bdbdf21L, 0x86d3d2d4L, 0xf1d4e242L, - 0x68ddb3f8L, 0x1fda836eL, 0x81be16cdL, 0xf6b9265bL, 0x6fb077e1L, - 0x18b74777L, 0x88085ae6L, 0xff0f6a70L, 0x66063bcaL, 0x11010b5cL, - 0x8f659effL, 0xf862ae69L, 0x616bffd3L, 0x166ccf45L, 0xa00ae278L, - 0xd70dd2eeL, 0x4e048354L, 0x3903b3c2L, 0xa7672661L, 0xd06016f7L, - 0x4969474dL, 0x3e6e77dbL, 0xaed16a4aL, 0xd9d65adcL, 0x40df0b66L, - 0x37d83bf0L, 0xa9bcae53L, 0xdebb9ec5L, 0x47b2cf7fL, 0x30b5ffe9L, - 0xbdbdf21cL, 0xcabac28aL, 0x53b39330L, 0x24b4a3a6L, 0xbad03605L, - 0xcdd70693L, 0x54de5729L, 0x23d967bfL, 0xb3667a2eL, 0xc4614ab8L, - 0x5d681b02L, 0x2a6f2b94L, 0xb40bbe37L, 0xc30c8ea1L, 0x5a05df1bL, - 0x2d02ef8dL + 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, + 0x706af48f, 0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4, + 0xe0d5e91e, 0x97d2d988, 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, + 0x90bf1d91, 0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de, + 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, 0x136c9856, + 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9, + 0xfa0f3d63, 0x8d080df5, 0x3b6e20c8, 0x4c69105e, 0xd56041e4, + 0xa2677172, 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, + 0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, + 0x45df5c75, 0xdcd60dcf, 0xabd13d59, 0x26d930ac, 0x51de003a, + 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, 0xcfba9599, + 0xb8bda50f, 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, + 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, 0x76dc4190, + 0x01db7106, 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, + 0x9fbfe4a5, 0xe8b8d433, 0x7807c9a2, 0x0f00f934, 0x9609a88e, + 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01, + 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, 0x6c0695ed, + 0x1b01a57b, 0x8208f4c1, 0xf50fc457, 0x65b0d9c6, 0x12b7e950, + 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, + 0xfbd44c65, 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, + 0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb, 0x4369e96a, + 0x346ed9fc, 0xad678846, 0xda60b8d0, 0x44042d73, 0x33031de5, + 0xaa0a4c5f, 0xdd0d7cc9, 0x5005713c, 0x270241aa, 0xbe0b1010, + 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f, + 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, + 0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad, 0xedb88320, 0x9abfb3b6, + 0x03b6e20c, 0x74b1d29a, 0xead54739, 0x9dd277af, 0x04db2615, + 0x73dc1683, 0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8, + 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, 0xf00f9344, + 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, + 0x196c3671, 0x6e6b06e7, 0xfed41b76, 0x89d32be0, 0x10da7a5a, + 0x67dd4acc, 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, + 0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, + 0xa6bc5767, 0x3fb506dd, 0x48b2364b, 0xd80d2bda, 0xaf0a1b4c, + 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, 0x316e8eef, + 0x4669be79, 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, + 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, 0xc5ba3bbe, + 0xb2bd0b28, 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, + 0x2cd99e8b, 0x5bdeae1d, 0x9b64c2b0, 0xec63f226, 0x756aa39c, + 0x026d930a, 0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713, + 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, 0x92d28e9b, + 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, 0x86d3d2d4, 0xf1d4e242, + 0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, + 0x18b74777, 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, + 0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45, 0xa00ae278, + 0xd70dd2ee, 0x4e048354, 0x3903b3c2, 0xa7672661, 0xd06016f7, + 0x4969474d, 0x3e6e77db, 0xaed16a4a, 0xd9d65adc, 0x40df0b66, + 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9, + 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, + 0xcdd70693, 0x54de5729, 0x23d967bf, 0xb3667a2e, 0xc4614ab8, + 0x5d681b02, 0x2a6f2b94, 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, + 0x2d02ef8d ] def crc32(s, crc=0): result = 0 - crc = ~long(crc) & 0xffffffffL + crc = ~int(crc) & 0xffffffff for c in s: - crc = crc_32_tab[(crc ^ c) & 0xffL] ^ (crc >> 8) + crc = crc_32_tab[(crc ^ c) & 0xff] ^ (crc >> 8) #/* Note: (crc >> 8) MUST zero fill on left - result = crc ^ 0xffffffffL + result = crc ^ 0xffffffff if result > 2**31: result = ((result + 2**31) % 2**32) - 2**31 From noreply at buildbot.pypy.org Tue Jan 17 00:26:34 2012 From: noreply at buildbot.pypy.org (amauryfa) Date: Tue, 17 Jan 2012 00:26:34 +0100 (CET) Subject: [pypy-commit] pypy py3k: Fix a test in test_longobject.py Message-ID: <20120116232634.B9A30820D8@wyvern.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3k Changeset: r51370:dcae009093d8 Date: 2012-01-16 20:54 +0100 http://bitbucket.org/pypy/pypy/changeset/dcae009093d8/ Log: Fix a test in test_longobject.py diff --git a/pypy/objspace/std/longtype.py b/pypy/objspace/std/longtype.py --- a/pypy/objspace/std/longtype.py +++ b/pypy/objspace/std/longtype.py @@ -40,6 +40,7 @@ w_obj = space.int(w_obj) else: w_obj = space.trunc(w_obj) + w_obj = space.int(w_obj) return w_obj else: base = space.int_w(w_base) From noreply at buildbot.pypy.org Tue Jan 17 00:26:35 2012 From: noreply at buildbot.pypy.org (amauryfa) Date: Tue, 17 Jan 2012 00:26:35 +0100 (CET) Subject: [pypy-commit] pypy py3k: Fixes in termios module Message-ID: <20120116232635.E70AC820D8@wyvern.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3k Changeset: r51371:4a6fb1817feb Date: 2012-01-16 23:53 +0100 http://bitbucket.org/pypy/pypy/changeset/4a6fb1817feb/ Log: Fixes in termios module diff --git a/pypy/module/termios/interp_termios.py b/pypy/module/termios/interp_termios.py --- a/pypy/module/termios/interp_termios.py +++ b/pypy/module/termios/interp_termios.py @@ -24,15 +24,11 @@ fd = space.c_filedescriptor_w(w_fd) w_iflag, w_oflag, w_cflag, w_lflag, w_ispeed, w_ospeed, w_cc = \ space.unpackiterable(w_attributes, expected_length=7) - w_builtin = space.getbuiltinmodule('__builtin__') cc = [] for w_c in space.unpackiterable(w_cc): if space.is_true(space.isinstance(w_c, space.w_int)): - ch = space.call_function(space.getattr(w_builtin, - space.wrap('chr')), w_c) - cc.append(space.str_w(ch)) - else: - cc.append(space.str_w(w_c)) + w_c = space.call(space.w_bytes, space.newlist([w_c])) + cc.append(space.bytes_w(w_c)) tup = (space.int_w(w_iflag), space.int_w(w_oflag), space.int_w(w_cflag), space.int_w(w_lflag), space.int_w(w_ispeed), space.int_w(w_ospeed), cc) diff --git a/pypy/module/termios/test/test_termios.py b/pypy/module/termios/test/test_termios.py --- a/pypy/module/termios/test/test_termios.py +++ b/pypy/module/termios/test/test_termios.py @@ -23,7 +23,7 @@ def _spawn(self, *args, **kwds): print 'SPAWN:', args, kwds - child = self.pexpect.spawn(*args, **kwds) + child = self.pexpect.spawn(timeout=600, *args, **kwds) child.logfile = sys.stdout return child @@ -53,7 +53,7 @@ termios.tcdrain(2) termios.tcflush(2, termios.TCIOFLUSH) termios.tcflow(2, termios.TCOON) - print 'ok!' + print('ok!') """) f = udir.join("test_tcall.py") f.write(source) @@ -65,7 +65,7 @@ import sys import termios termios.tcsetattr(sys.stdin, 1, [16640, 4, 191, 2608, 15, 15, ['\x03', '\x1c', '\x7f', '\x15', '\x04', 0, 1, '\x00', '\x11', '\x13', '\x1a', '\x00', '\x12', '\x0f', '\x17', '\x16', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00']]) - print 'ok!' + print('ok!') """) f = udir.join("test_tcsetattr.py") f.write(source) @@ -76,9 +76,9 @@ source = py.code.Source(""" import termios import fcntl - lgt = len(fcntl.ioctl(2, termios.TIOCGWINSZ, '\000'*8)) + lgt = len(fcntl.ioctl(2, termios.TIOCGWINSZ, b'\000'*8)) assert lgt == 8 - print 'ok!' + print('ok!') """) f = udir.join("test_ioctl_termios.py") f.write(source) @@ -97,7 +97,7 @@ assert len([i for i in f[-1] if isinstance(i, int)]) == 2 assert isinstance(f[-1][termios.VMIN], int) assert isinstance(f[-1][termios.VTIME], int) - print 'ok!' + print('ok!') """) f = udir.join("test_ioctl_termios.py") f.write(source) From noreply at buildbot.pypy.org Tue Jan 17 00:26:37 2012 From: noreply at buildbot.pypy.org (amauryfa) Date: Tue, 17 Jan 2012 00:26:37 +0100 (CET) Subject: [pypy-commit] pypy py3k: bin/checkmodule.py: Add support for space.wrabytes Message-ID: <20120116232637.19DEB820D8@wyvern.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3k Changeset: r51372:3843c6835175 Date: 2012-01-16 23:54 +0100 http://bitbucket.org/pypy/pypy/changeset/3843c6835175/ Log: bin/checkmodule.py: Add support for space.wrabytes diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py --- a/pypy/objspace/fake/objspace.py +++ b/pypy/objspace/fake/objspace.py @@ -157,6 +157,10 @@ "NOT_RPYTHON" raise NotImplementedError + def wrapbytes(self, x): + assert isinstance(x, str) + return w_some_obj() + def _see_interp2app(self, interp2app): "NOT_RPYTHON" activation = interp2app._code.activation @@ -278,7 +282,7 @@ for name in (ObjSpace.ConstantTable + ObjSpace.ExceptionTable + ['int', 'str', 'float', 'long', 'tuple', 'list', - 'dict', 'unicode', 'complex', 'slice', 'bool', + 'dict', 'bytes', 'complex', 'slice', 'bool', 'type', 'text', 'object']): setattr(FakeObjSpace, 'w_' + name, w_some_obj()) # From noreply at buildbot.pypy.org Tue Jan 17 09:56:37 2012 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 17 Jan 2012 09:56:37 +0100 (CET) Subject: [pypy-commit] pypy numpy-indexing-by-arrays-2: indexing by bool arrays of the same shape, step 1 Message-ID: <20120117085637.7C341820D8@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: numpy-indexing-by-arrays-2 Changeset: r51373:bec89443f75e Date: 2012-01-17 10:55 +0200 http://bitbucket.org/pypy/pypy/changeset/bec89443f75e/ Log: indexing by bool arrays of the same shape, step 1 diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py --- a/pypy/module/micronumpy/interp_dtype.py +++ b/pypy/module/micronumpy/interp_dtype.py @@ -46,6 +46,10 @@ def getitem(self, storage, i): return self.itemtype.read(storage, self.itemtype.get_element_size(), i, 0) + def getitem_bool(self, storage, i): + isize = self.itemtype.get_element_size() + return self.itemtype.read_raw(storage, isize, i, 0, bool) + def setitem(self, storage, i, box): self.itemtype.store(storage, self.itemtype.get_element_size(), i, 0, box) diff --git a/pypy/module/micronumpy/interp_iter.py b/pypy/module/micronumpy/interp_iter.py --- a/pypy/module/micronumpy/interp_iter.py +++ b/pypy/module/micronumpy/interp_iter.py @@ -62,11 +62,18 @@ self.size = size def next(self, shapelen): + return self._next(1) + + def _next(self, ofs): arr = instantiate(ArrayIterator) arr.size = self.size - arr.offset = self.offset + 1 + arr.offset = self.offset + ofs return arr + def next_no_increase(self, shapelen): + # a hack to make JIT believe this is always virtual + return self._next(0) + def done(self): return self.offset >= self.size diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -483,7 +483,41 @@ return [Chunk(*space.decode_index4(w_item, self.shape[i])) for i, w_item in enumerate(space.fixedview(w_idx))] + def count_all_true(self, arr): + sig = arr.find_sig() + frame = sig.create_frame(self) + shapelen = len(arr.shape) + s = 0 + while not frame.done(): + iter = frame.get_final_iter() + s += arr.dtype.getitem_bool(arr.storage, iter.offset) + frame.next(shapelen) + return s + + def getitem_filter(self, space, arr): + concr = arr.get_concrete() + size = self.count_all_true(concr) + res = W_NDimArray(size, [size], self.find_dtype()) + ri = ArrayIterator(size) + shapelen = len(self.shape) + argi = ArrayIterator(concr.size) + sig = self.find_sig() + frame = sig.create_frame(self) + while not frame.done(): + if concr.dtype.getitem_bool(concr.storage, argi.offset): + v = sig.eval(frame, self) + res.setitem(ri.offset, v) + ri = ri.next(1) + else: + ri = ri.next_no_increase(1) + argi = argi.next(shapelen) + frame.next(shapelen) + return res + def descr_getitem(self, space, w_idx): + if (isinstance(w_idx, BaseArray) and w_idx.shape == self.shape and + w_idx.find_dtype().is_bool_type()): + return self.getitem_filter(space, w_idx) if self._single_item_result(space, w_idx): concrete = self.get_concrete() item = concrete._index_of_single_item(space, w_idx) @@ -716,8 +750,7 @@ frame=frame, ri=ri, self=self, result=result) - result.dtype.setitem(result.storage, ri.offset, - sig.eval(frame, self)) + result.setitem(ri.offset, sig.eval(frame, self)) frame.next(shapelen) ri = ri.next(shapelen) return result diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -1312,6 +1312,13 @@ raises(IndexError,'arange(3)[array([-15])]') assert arange(3)[array(1)] == 1 + def test_array_indexing_bool(self): + from _numpypy import arange + a = arange(10) + assert (a[a > 3] == [4, 5, 6, 7, 8, 9]).all() + a = arange(10).reshape(5, 2) + assert (a[a > 3] == [4, 5, 6, 7, 8, 9]).all() + class AppTestSupport(BaseNumpyAppTest): def setup_class(cls): import struct diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py --- a/pypy/module/micronumpy/types.py +++ b/pypy/module/micronumpy/types.py @@ -94,6 +94,11 @@ width, storage, i, offset )) + @specialize.arg(5) + def read_raw(self, storage, width, i, offset, tp): + return libffi.array_getitem(clibffi.cast_type_to_ffitype(self.T), + width, storage, i, offset) + def store(self, storage, width, i, offset, box): value = self.unbox(box) libffi.array_setitem(clibffi.cast_type_to_ffitype(self.T), @@ -436,4 +441,4 @@ class Float64(BaseType, Float): T = rffi.DOUBLE BoxType = interp_boxes.W_Float64Box - format_code = "d" \ No newline at end of file + format_code = "d" From noreply at buildbot.pypy.org Tue Jan 17 10:10:20 2012 From: noreply at buildbot.pypy.org (benol) Date: Tue, 17 Jan 2012 10:10:20 +0100 (CET) Subject: [pypy-commit] pypy jvm-improvements: Add files generated by PyCharm to .hgignore Message-ID: <20120117091020.03C69820D8@wyvern.cs.uni-duesseldorf.de> Author: Michal Bendowski Branch: jvm-improvements Changeset: r51374:1afe14470fa3 Date: 2012-01-14 20:18 +0100 http://bitbucket.org/pypy/pypy/changeset/1afe14470fa3/ Log: Add files generated by PyCharm to .hgignore diff --git a/.hgignore b/.hgignore --- a/.hgignore +++ b/.hgignore @@ -2,6 +2,9 @@ *.py[co] *~ .*.swp +.idea +.project +.pydevproject syntax: regexp ^testresult$ From noreply at buildbot.pypy.org Tue Jan 17 10:10:21 2012 From: noreply at buildbot.pypy.org (benol) Date: Tue, 17 Jan 2012 10:10:21 +0100 (CET) Subject: [pypy-commit] pypy jvm-improvements: Fix userspace builders in ootype Message-ID: <20120117091021.31C8A820D8@wyvern.cs.uni-duesseldorf.de> Author: Michal Bendowski Branch: jvm-improvements Changeset: r51375:90cb8d420991 Date: 2012-01-11 22:26 +0100 http://bitbucket.org/pypy/pypy/changeset/90cb8d420991/ Log: Fix userspace builders in ootype Implement the getlength() method of StringBuilders in ootype. diff --git a/pypy/rpython/ootypesystem/ootype.py b/pypy/rpython/ootypesystem/ootype.py --- a/pypy/rpython/ootypesystem/ootype.py +++ b/pypy/rpython/ootypesystem/ootype.py @@ -512,6 +512,7 @@ "ll_append_char": Meth([CHARTP], Void), "ll_append": Meth([STRINGTP], Void), "ll_build": Meth([], STRINGTP), + "ll_getlength": Meth([], Signed), }) self._setup_methods({}) @@ -1543,6 +1544,9 @@ else: return make_unicode(u''.join(self._buf)) + def ll_getlength(self): + return self.ll_build().ll_strlen() + class _null_string_builder(_null_mixin(_string_builder), _string_builder): def __init__(self, STRING_BUILDER): self.__dict__["_TYPE"] = STRING_BUILDER diff --git a/pypy/rpython/ootypesystem/rbuilder.py b/pypy/rpython/ootypesystem/rbuilder.py --- a/pypy/rpython/ootypesystem/rbuilder.py +++ b/pypy/rpython/ootypesystem/rbuilder.py @@ -21,6 +21,10 @@ builder.ll_append_char(char) @staticmethod + def ll_getlength(builder): + return builder.ll_getlength() + + @staticmethod def ll_append(builder, string): builder.ll_append(string) diff --git a/pypy/rpython/test/test_rbuilder.py b/pypy/rpython/test/test_rbuilder.py --- a/pypy/rpython/test/test_rbuilder.py +++ b/pypy/rpython/test/test_rbuilder.py @@ -124,9 +124,5 @@ pass class TestOOtype(BaseTestStringBuilder, OORtypeMixin): - def test_string_getlength(self): - py.test.skip("getlength(): not implemented on ootype") - def test_unicode_getlength(self): - py.test.skip("getlength(): not implemented on ootype") def test_append_charpsize(self): py.test.skip("append_charpsize(): not implemented on ootype") diff --git a/pypy/translator/jvm/builtin.py b/pypy/translator/jvm/builtin.py --- a/pypy/translator/jvm/builtin.py +++ b/pypy/translator/jvm/builtin.py @@ -84,6 +84,9 @@ (ootype.StringBuilder.__class__, "ll_build"): jvm.Method.v(jStringBuilder, "toString", (), jString), + (ootype.StringBuilder.__class__, "ll_getlength"): + jvm.Method.v(jStringBuilder, "length", (), jInt), + (ootype.String.__class__, "ll_hash"): jvm.Method.v(jString, "hashCode", (), jInt), diff --git a/pypy/translator/jvm/test/test_builder.py b/pypy/translator/jvm/test/test_builder.py new file mode 100644 --- /dev/null +++ b/pypy/translator/jvm/test/test_builder.py @@ -0,0 +1,7 @@ +from pypy.translator.jvm.test.runtest import JvmTest +from pypy.rpython.test.test_rbuilder import BaseTestStringBuilder +import py + +class TestJvmStringBuilder(JvmTest, BaseTestStringBuilder): + def test_append_charpsize(self): + py.test.skip("append_charpsize(): not implemented on ootype") From noreply at buildbot.pypy.org Tue Jan 17 10:10:22 2012 From: noreply at buildbot.pypy.org (benol) Date: Tue, 17 Jan 2012 10:10:22 +0100 (CET) Subject: [pypy-commit] pypy jvm-improvements: Fix compute_unique_id to support built-ins in ootype. Message-ID: <20120117091022.5D2F1820D8@wyvern.cs.uni-duesseldorf.de> Author: Michal Bendowski Branch: jvm-improvements Changeset: r51376:143a2edf9601 Date: 2012-01-11 22:29 +0100 http://bitbucket.org/pypy/pypy/changeset/143a2edf9601/ Log: Fix compute_unique_id to support built-ins in ootype. Otherwise the translation fails because it doesn't know how to apply compute_unique_id to a String. In the jvm backend this is implemented by System.identityHashCode() which can be applied to our representations of built-ins equally well as for instances. diff --git a/pypy/rlib/objectmodel.py b/pypy/rlib/objectmodel.py --- a/pypy/rlib/objectmodel.py +++ b/pypy/rlib/objectmodel.py @@ -420,7 +420,7 @@ vobj.concretetype.TO._gckind == 'gc') else: from pypy.rpython.ootypesystem import ootype - ok = isinstance(vobj.concretetype, ootype.Instance) + ok = isinstance(vobj.concretetype, (ootype.Instance, ootype.BuiltinType)) if not ok: from pypy.rpython.error import TyperError raise TyperError("compute_unique_id() cannot be applied to" diff --git a/pypy/rpython/ootypesystem/ootype.py b/pypy/rpython/ootypesystem/ootype.py --- a/pypy/rpython/ootypesystem/ootype.py +++ b/pypy/rpython/ootypesystem/ootype.py @@ -1377,6 +1377,9 @@ def _cast_to_object(self): return make_object(self) + def _identityhash(self): + return hash(self) + class _string(_builtin_type): def __init__(self, STRING, value = ''): diff --git a/pypy/rpython/test/test_rbuiltin.py b/pypy/rpython/test/test_rbuiltin.py --- a/pypy/rpython/test/test_rbuiltin.py +++ b/pypy/rpython/test/test_rbuiltin.py @@ -463,6 +463,20 @@ assert x1 == intmask(x0) assert x3 == intmask(x2) + def test_id_on_builtins(self): + from pypy.rlib.objectmodel import compute_unique_id + from pypy.rlib.rstring import StringBuilder, UnicodeBuilder + def fn(): + return (compute_unique_id("foo"), + compute_unique_id(u"bar"), + compute_unique_id([1]), + compute_unique_id({"foo": 3}), + compute_unique_id(StringBuilder()), + compute_unique_id(UnicodeBuilder())) + res = self.interpret(fn, []) + for id in self.ll_unpack_tuple(res, 6): + assert isinstance(id, (int, r_longlong)) + def test_cast_primitive(self): from pypy.rpython.annlowlevel import LowLevelAnnotatorPolicy def llf(u): From noreply at buildbot.pypy.org Tue Jan 17 10:10:23 2012 From: noreply at buildbot.pypy.org (benol) Date: Tue, 17 Jan 2012 10:10:23 +0100 (CET) Subject: [pypy-commit] pypy jvm-improvements: Declare oo_primitives that should implement some rffi operations. Message-ID: <20120117091023.831F3820D8@wyvern.cs.uni-duesseldorf.de> Author: Michal Bendowski Branch: jvm-improvements Changeset: r51377:6b650a500d68 Date: 2012-01-14 19:04 +0100 http://bitbucket.org/pypy/pypy/changeset/6b650a500d68/ Log: Declare oo_primitives that should implement some rffi operations. For now the actual implementations are missing, but once we get the JVM backend to work in some way, this will have to be revisited. diff --git a/pypy/rlib/longlong2float.py b/pypy/rlib/longlong2float.py --- a/pypy/rlib/longlong2float.py +++ b/pypy/rlib/longlong2float.py @@ -79,19 +79,23 @@ longlong2float = rffi.llexternal( "pypy__longlong2float", [rffi.LONGLONG], rffi.DOUBLE, _callable=longlong2float_emulator, compilation_info=eci, - _nowrapper=True, elidable_function=True, sandboxsafe=True) + _nowrapper=True, elidable_function=True, sandboxsafe=True, + oo_primitive="pypy__longlong2float") float2longlong = rffi.llexternal( "pypy__float2longlong", [rffi.DOUBLE], rffi.LONGLONG, _callable=float2longlong_emulator, compilation_info=eci, - _nowrapper=True, elidable_function=True, sandboxsafe=True) + _nowrapper=True, elidable_function=True, sandboxsafe=True, + oo_primitive="pypy__float2longlong") uint2singlefloat = rffi.llexternal( "pypy__uint2singlefloat", [rffi.UINT], rffi.FLOAT, _callable=uint2singlefloat_emulator, compilation_info=eci, - _nowrapper=True, elidable_function=True, sandboxsafe=True) + _nowrapper=True, elidable_function=True, sandboxsafe=True, + oo_primitive="pypy__uint2singlefloat") singlefloat2uint = rffi.llexternal( "pypy__singlefloat2uint", [rffi.FLOAT], rffi.UINT, _callable=singlefloat2uint_emulator, compilation_info=eci, - _nowrapper=True, elidable_function=True, sandboxsafe=True) + _nowrapper=True, elidable_function=True, sandboxsafe=True, + oo_primitive="pypy__singlefloat2uint") diff --git a/pypy/rlib/rmd5.py b/pypy/rlib/rmd5.py --- a/pypy/rlib/rmd5.py +++ b/pypy/rlib/rmd5.py @@ -51,7 +51,7 @@ _rotateLeft = rffi.llexternal( "pypy__rotateLeft", [lltype.Unsigned, lltype.Signed], lltype.Unsigned, _callable=_rotateLeft_emulator, compilation_info=eci, - _nowrapper=True, elidable_function=True) + _nowrapper=True, elidable_function=True, oo_primitive='pypy__rotateLeft') # TODO implement the oo_primitive # we expect the function _rotateLeft to be actually inlined From noreply at buildbot.pypy.org Tue Jan 17 10:10:24 2012 From: noreply at buildbot.pypy.org (benol) Date: Tue, 17 Jan 2012 10:10:24 +0100 (CET) Subject: [pypy-commit] pypy jvm-improvements: Add a missing cast from Unsigned to UnsignedLongLong in the JVM Message-ID: <20120117091024.AA136820D8@wyvern.cs.uni-duesseldorf.de> Author: Michal Bendowski Branch: jvm-improvements Changeset: r51378:c59ec9806ac6 Date: 2012-01-14 20:15 +0100 http://bitbucket.org/pypy/pypy/changeset/c59ec9806ac6/ Log: Add a missing cast from Unsigned to UnsignedLongLong in the JVM backend. diff --git a/pypy/translator/jvm/metavm.py b/pypy/translator/jvm/metavm.py --- a/pypy/translator/jvm/metavm.py +++ b/pypy/translator/jvm/metavm.py @@ -92,6 +92,7 @@ CASTS = { # FROM TO (ootype.Signed, ootype.UnsignedLongLong): jvm.I2L, + (ootype.Unsigned, ootype.UnsignedLongLong): jvm.I2L, (ootype.SignedLongLong, ootype.Signed): jvm.L2I, (ootype.UnsignedLongLong, ootype.Unsigned): jvm.L2I, (ootype.UnsignedLongLong, ootype.Signed): jvm.L2I, diff --git a/pypy/translator/oosupport/test_template/cast.py b/pypy/translator/oosupport/test_template/cast.py --- a/pypy/translator/oosupport/test_template/cast.py +++ b/pypy/translator/oosupport/test_template/cast.py @@ -13,6 +13,9 @@ def to_longlong(x): return r_longlong(x) +def to_ulonglong(x): + return r_ulonglong(x) + def uint_to_int(x): return intmask(x) @@ -56,6 +59,9 @@ def test_unsignedlonglong_to_unsigned4(self): self.check(to_uint, [r_ulonglong(18446744073709551615l)]) # max 64 bit num + def test_unsigned_to_usignedlonglong(self): + self.check(to_ulonglong, [r_uint(42)]) + def test_uint_to_int(self): self.check(uint_to_int, [r_uint(sys.maxint+1)]) From noreply at buildbot.pypy.org Tue Jan 17 10:10:25 2012 From: noreply at buildbot.pypy.org (benol) Date: Tue, 17 Jan 2012 10:10:25 +0100 (CET) Subject: [pypy-commit] pypy jvm-improvements: Handle the 'jit_is_virtual' opcode by always returning False Message-ID: <20120117091025.D29F0820D8@wyvern.cs.uni-duesseldorf.de> Author: Michal Bendowski Branch: jvm-improvements Changeset: r51379:e31e85a8d333 Date: 2012-01-14 19:01 +0100 http://bitbucket.org/pypy/pypy/changeset/e31e85a8d333/ Log: Handle the 'jit_is_virtual' opcode by always returning False diff --git a/pypy/translator/jvm/opcodes.py b/pypy/translator/jvm/opcodes.py --- a/pypy/translator/jvm/opcodes.py +++ b/pypy/translator/jvm/opcodes.py @@ -101,6 +101,7 @@ 'jit_force_virtualizable': Ignore, 'jit_force_virtual': DoNothing, 'jit_force_quasi_immutable': Ignore, + 'jit_is_virtual': PushPrimitive(ootype.Bool, False), 'debug_assert': [], # TODO: implement? 'debug_start_traceback': Ignore, From noreply at buildbot.pypy.org Tue Jan 17 10:10:27 2012 From: noreply at buildbot.pypy.org (benol) Date: Tue, 17 Jan 2012 10:10:27 +0100 (CET) Subject: [pypy-commit] pypy jvm-improvements: Implemented float2longlong and longlong2float for the JVM. Message-ID: <20120117091027.0EEFE820D8@wyvern.cs.uni-duesseldorf.de> Author: Michal Bendowski Branch: jvm-improvements Changeset: r51380:5ba62496112e Date: 2012-01-16 00:56 +0100 http://bitbucket.org/pypy/pypy/changeset/5ba62496112e/ Log: Implemented float2longlong and longlong2float for the JVM. Also removed the oo_primitive for pypy__rotateLeft - it's not needed on 32 bit architecture (and JVM backend doesn't support 64 bit anyway). diff --git a/pypy/rlib/rmd5.py b/pypy/rlib/rmd5.py --- a/pypy/rlib/rmd5.py +++ b/pypy/rlib/rmd5.py @@ -51,7 +51,7 @@ _rotateLeft = rffi.llexternal( "pypy__rotateLeft", [lltype.Unsigned, lltype.Signed], lltype.Unsigned, _callable=_rotateLeft_emulator, compilation_info=eci, - _nowrapper=True, elidable_function=True, oo_primitive='pypy__rotateLeft') # TODO implement the oo_primitive + _nowrapper=True, elidable_function=True) # we expect the function _rotateLeft to be actually inlined diff --git a/pypy/translator/jvm/database.py b/pypy/translator/jvm/database.py --- a/pypy/translator/jvm/database.py +++ b/pypy/translator/jvm/database.py @@ -358,7 +358,7 @@ ootype.Unsigned:jvm.PYPYSERIALIZEUINT, ootype.SignedLongLong:jvm.LONGTOSTRINGL, ootype.UnsignedLongLong: jvm.PYPYSERIALIZEULONG, - ootype.Float:jvm.DOUBLETOSTRINGD, + ootype.Float:jvm.PYPYSERIALIZEDOUBLE, ootype.Bool:jvm.PYPYSERIALIZEBOOLEAN, ootype.Void:jvm.PYPYSERIALIZEVOID, ootype.Char:jvm.PYPYESCAPEDCHAR, diff --git a/pypy/translator/jvm/src/pypy/PyPy.java b/pypy/translator/jvm/src/pypy/PyPy.java --- a/pypy/translator/jvm/src/pypy/PyPy.java +++ b/pypy/translator/jvm/src/pypy/PyPy.java @@ -8,6 +8,7 @@ import java.util.Map; import java.text.DecimalFormat; import java.lang.reflect.Array; +import java.nio.ByteBuffer; /** * Class with a number of utility routines. One instance of this is @@ -283,6 +284,20 @@ } } + public double pypy__longlong2float(long l) { + ByteBuffer buf = ByteBuffer.allocate(8); + buf.putLong(l); + buf.flip(); + return buf.getDouble(); + } + + public long pypy__float2longlong(double d) { + ByteBuffer buf = ByteBuffer.allocate(8); + buf.putDouble(d); + buf.flip(); + return buf.getLong(); + } + public double ooparse_float(String s) { try { return Double.parseDouble(s); @@ -353,6 +368,19 @@ return "False"; } + public static String serialize_double(double d) { + if (Double.isNaN(d)) { + return "float(\"nan\")"; + } else if (Double.isInfinite(d)) { + if (d > 0) + return "float(\"inf\")"; + else + return "float(\"-inf\")"; + } else { + return Double.toString(d); + } + } + private static String format_char(char c) { String res = "\\x"; if (c <= 0x0F) res = res + "0"; diff --git a/pypy/translator/jvm/test/runtest.py b/pypy/translator/jvm/test/runtest.py --- a/pypy/translator/jvm/test/runtest.py +++ b/pypy/translator/jvm/test/runtest.py @@ -56,6 +56,7 @@ # CLI could-be duplicate class JvmGeneratedSourceWrapper(object): + def __init__(self, gensrc): """ gensrc is an instance of JvmGeneratedSource """ self.gensrc = gensrc diff --git a/pypy/translator/jvm/test/test_longlong2float.py b/pypy/translator/jvm/test/test_longlong2float.py new file mode 100644 --- /dev/null +++ b/pypy/translator/jvm/test/test_longlong2float.py @@ -0,0 +1,20 @@ +from pypy.translator.jvm.test.runtest import JvmTest +from pypy.rlib.longlong2float import * +from pypy.rlib.test.test_longlong2float import enum_floats +from pypy.rlib.test.test_longlong2float import fn as float2longlong2float +import py + +class TestLongLong2Float(JvmTest): + + def test_float2longlong_and_longlong2float(self): + def func(f): + return float2longlong2float(f) + + for f in enum_floats(): + assert repr(f) == repr(self.interpret(func, [f])) + + def test_uint2singlefloat(self): + py.test.skip("uint2singlefloat is not implemented in ootype") + + def test_singlefloat2uint(self): + py.test.skip("singlefloat2uint is not implemented in ootype") diff --git a/pypy/translator/jvm/typesystem.py b/pypy/translator/jvm/typesystem.py --- a/pypy/translator/jvm/typesystem.py +++ b/pypy/translator/jvm/typesystem.py @@ -955,6 +955,7 @@ PYPYSERIALIZEUINT = Method.s(jPyPy, 'serialize_uint', (jInt,), jString) PYPYSERIALIZEULONG = Method.s(jPyPy, 'serialize_ulonglong', (jLong,),jString) PYPYSERIALIZEVOID = Method.s(jPyPy, 'serialize_void', (), jString) +PYPYSERIALIZEDOUBLE = Method.s(jPyPy, 'serialize_double', (jDouble,), jString) PYPYESCAPEDCHAR = Method.s(jPyPy, 'escaped_char', (jChar,), jString) PYPYESCAPEDUNICHAR = Method.s(jPyPy, 'escaped_unichar', (jChar,), jString) PYPYESCAPEDSTRING = Method.s(jPyPy, 'escaped_string', (jString,), jString) From noreply at buildbot.pypy.org Tue Jan 17 10:10:28 2012 From: noreply at buildbot.pypy.org (benol) Date: Tue, 17 Jan 2012 10:10:28 +0100 (CET) Subject: [pypy-commit] pypy jvm-improvements: Simpler implementations of float2longlong and longlong2float. Message-ID: <20120117091028.3F46C820D8@wyvern.cs.uni-duesseldorf.de> Author: Michal Bendowski Branch: jvm-improvements Changeset: r51381:79b211ffff3c Date: 2012-01-16 21:19 +0100 http://bitbucket.org/pypy/pypy/changeset/79b211ffff3c/ Log: Simpler implementations of float2longlong and longlong2float. diff --git a/pypy/translator/jvm/src/pypy/PyPy.java b/pypy/translator/jvm/src/pypy/PyPy.java --- a/pypy/translator/jvm/src/pypy/PyPy.java +++ b/pypy/translator/jvm/src/pypy/PyPy.java @@ -8,7 +8,6 @@ import java.util.Map; import java.text.DecimalFormat; import java.lang.reflect.Array; -import java.nio.ByteBuffer; /** * Class with a number of utility routines. One instance of this is @@ -285,17 +284,11 @@ } public double pypy__longlong2float(long l) { - ByteBuffer buf = ByteBuffer.allocate(8); - buf.putLong(l); - buf.flip(); - return buf.getDouble(); + return Double.longBitsToDouble(l); } public long pypy__float2longlong(double d) { - ByteBuffer buf = ByteBuffer.allocate(8); - buf.putDouble(d); - buf.flip(); - return buf.getLong(); + return Double.doubleToRawLongBits(d); } public double ooparse_float(String s) { From noreply at buildbot.pypy.org Tue Jan 17 10:10:29 2012 From: noreply at buildbot.pypy.org (benol) Date: Tue, 17 Jan 2012 10:10:29 +0100 (CET) Subject: [pypy-commit] pypy jvm-improvements: Fix the implementation of compute_unique_id for _builtin_type. Message-ID: <20120117091029.6BEFE820D8@wyvern.cs.uni-duesseldorf.de> Author: Michal Bendowski Branch: jvm-improvements Changeset: r51382:259f2ab80ae4 Date: 2012-01-16 22:26 +0100 http://bitbucket.org/pypy/pypy/changeset/259f2ab80ae4/ Log: Fix the implementation of compute_unique_id for _builtin_type. diff --git a/pypy/rpython/ootypesystem/ootype.py b/pypy/rpython/ootypesystem/ootype.py --- a/pypy/rpython/ootypesystem/ootype.py +++ b/pypy/rpython/ootypesystem/ootype.py @@ -1378,7 +1378,7 @@ return make_object(self) def _identityhash(self): - return hash(self) + return object.__hash__(self) class _string(_builtin_type): diff --git a/pypy/rpython/test/test_rbuiltin.py b/pypy/rpython/test/test_rbuiltin.py --- a/pypy/rpython/test/test_rbuiltin.py +++ b/pypy/rpython/test/test_rbuiltin.py @@ -477,6 +477,17 @@ for id in self.ll_unpack_tuple(res, 6): assert isinstance(id, (int, r_longlong)) + def test_uniqueness_of_id_on_strings(self): + from pypy.rlib.objectmodel import compute_unique_id + def fn(s1, s2): + return (compute_unique_id(s1), compute_unique_id(s2)) + + s1 = "foo" + s2 = ''.join(['f','oo']) + res = self.interpret(fn, [self.string_to_ll(s1), self.string_to_ll(s2)]) + i1, i2 = self.ll_unpack_tuple(res, 2) + assert i1 != i2 + def test_cast_primitive(self): from pypy.rpython.annlowlevel import LowLevelAnnotatorPolicy def llf(u): From noreply at buildbot.pypy.org Tue Jan 17 10:29:48 2012 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 17 Jan 2012 10:29:48 +0100 (CET) Subject: [pypy-commit] pypy numpy-indexing-by-arrays-2: add & and | Message-ID: <20120117092948.C88A5820D8@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: numpy-indexing-by-arrays-2 Changeset: r51383:35596eb33a6f Date: 2012-01-17 11:29 +0200 http://bitbucket.org/pypy/pypy/changeset/35596eb33a6f/ Log: add & and | diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py --- a/pypy/module/micronumpy/__init__.py +++ b/pypy/module/micronumpy/__init__.py @@ -85,6 +85,8 @@ ("subtract", "subtract"), ('sqrt', 'sqrt'), ("tan", "tan"), + ('bitwise_and', 'bitwise_and'), + ('bitwise_or', 'bitwise_or'), ]: interpleveldefs[exposed] = "interp_ufuncs.get(space).%s" % impl diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -270,6 +270,9 @@ descr_gt = _binop_impl("greater") descr_ge = _binop_impl("greater_equal") + descr_and = _binop_impl("bitwise_and") + descr_or = _binop_impl("bitwise_or") + def _binop_right_impl(ufunc_name): def impl(self, space, w_other): w_other = scalar_w(space, @@ -1279,6 +1282,9 @@ __gt__ = interp2app(BaseArray.descr_gt), __ge__ = interp2app(BaseArray.descr_ge), + __and__ = interp2app(BaseArray.descr_and), + __or__ = interp2app(BaseArray.descr_or), + __repr__ = interp2app(BaseArray.descr_repr), __str__ = interp2app(BaseArray.descr_str), __array_interface__ = GetSetProperty(BaseArray.descr_array_iface), diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py --- a/pypy/module/micronumpy/interp_ufuncs.py +++ b/pypy/module/micronumpy/interp_ufuncs.py @@ -425,6 +425,8 @@ ("add", "add", 2, {"identity": 0}), ("subtract", "sub", 2), ("multiply", "mul", 2, {"identity": 1}), + ("bitwise_and", "bitwise_and", 2, {"identity": 1}), + ("bitwise_or", "bitwise_or", 2, {"identity": 0}), ("divide", "div", 2, {"promote_bools": True}), ("mod", "mod", 2, {"promote_bools": True}), ("power", "pow", 2, {"promote_bools": True}), diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -1318,6 +1318,7 @@ assert (a[a > 3] == [4, 5, 6, 7, 8, 9]).all() a = arange(10).reshape(5, 2) assert (a[a > 3] == [4, 5, 6, 7, 8, 9]).all() + assert (a[a & 1] == [0, 2, 4, 6, 8]).all() class AppTestSupport(BaseNumpyAppTest): def setup_class(cls): diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py --- a/pypy/module/micronumpy/test/test_ufuncs.py +++ b/pypy/module/micronumpy/test/test_ufuncs.py @@ -347,11 +347,19 @@ raises(ValueError, maximum.reduce, []) def test_reduceND(self): - from numpypy import add, arange + from _numpypy import add, arange a = arange(12).reshape(3, 4) assert (add.reduce(a, 0) == [12, 15, 18, 21]).all() assert (add.reduce(a, 1) == [6.0, 22.0, 38.0]).all() + def test_bitwise(self): + from _numpypy import bitwise_and, bitwise_or, arange + a = arange(6).reshape(2, 3) + assert (a & 1 == [[0, 1, 0], [1, 0, 1]]).all() + assert (a & 1 == bitwise_and(a, 1)).all() + assert (a | 1 == [[1, 1, 3], [3, 5, 5]]).all() + assert (a | 1 == bitwise_or(a, 1)).all() + def test_comparisons(self): import operator from _numpypy import equal, not_equal, less, less_equal, greater, greater_equal diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py --- a/pypy/module/micronumpy/types.py +++ b/pypy/module/micronumpy/types.py @@ -174,6 +174,15 @@ def min(self, v1, v2): return min(v1, v2) + @simple_binary_op + def bitwise_and(self, v1, v2): + return v1 & v2 + + @simple_binary_op + def bitwise_or(self, v1, v2): + return v1 | v2 + + class Bool(BaseType, Primitive): T = lltype.Bool BoxType = interp_boxes.W_BoolBox From noreply at buildbot.pypy.org Tue Jan 17 10:53:32 2012 From: noreply at buildbot.pypy.org (hager) Date: Tue, 17 Jan 2012 10:53:32 +0100 (CET) Subject: [pypy-commit] pypy ppc-jit-backend: check var.type in make_sure_var_in_reg Message-ID: <20120117095332.BE5C3820D8@wyvern.cs.uni-duesseldorf.de> Author: hager Branch: ppc-jit-backend Changeset: r51384:909126039943 Date: 2012-01-17 10:51 +0100 http://bitbucket.org/pypy/pypy/changeset/909126039943/ Log: check var.type in make_sure_var_in_reg diff --git a/pypy/jit/backend/ppc/ppcgen/regalloc.py b/pypy/jit/backend/ppc/ppcgen/regalloc.py --- a/pypy/jit/backend/ppc/ppcgen/regalloc.py +++ b/pypy/jit/backend/ppc/ppcgen/regalloc.py @@ -306,8 +306,11 @@ def make_sure_var_in_reg(self, var, forbidden_vars=[], selected_reg=None, need_lower_byte=False): - return self.rm.make_sure_var_in_reg(var, forbidden_vars, - selected_reg, need_lower_byte) + if var.type == FLOAT: + assert 0, "not implemented yet" + else: + return self.rm.make_sure_var_in_reg(var, forbidden_vars, + selected_reg, need_lower_byte) def _sync_var(self, v): if v.type == FLOAT: From noreply at buildbot.pypy.org Tue Jan 17 11:10:54 2012 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 17 Jan 2012 11:10:54 +0100 (CET) Subject: [pypy-commit] pypy numpy-indexing-by-arrays-2: good, fix the test Message-ID: <20120117101054.E80EE820D8@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: numpy-indexing-by-arrays-2 Changeset: r51385:23d3ca4e7881 Date: 2012-01-17 12:10 +0200 http://bitbucket.org/pypy/pypy/changeset/23d3ca4e7881/ Log: good, fix the test diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -1318,7 +1318,7 @@ assert (a[a > 3] == [4, 5, 6, 7, 8, 9]).all() a = arange(10).reshape(5, 2) assert (a[a > 3] == [4, 5, 6, 7, 8, 9]).all() - assert (a[a & 1] == [0, 2, 4, 6, 8]).all() + assert (a[a & 1 == 1] == [1, 3, 5, 7, 9]).all() class AppTestSupport(BaseNumpyAppTest): def setup_class(cls): From cfbolz at gmx.de Tue Jan 17 11:11:05 2012 From: cfbolz at gmx.de (Carl Friedrich Bolz) Date: Tue, 17 Jan 2012 11:11:05 +0100 Subject: [pypy-commit] pypy default: A bit experimental - try to preallocate the size of unicode join and remove In-Reply-To: <20120116201610.009C0820D8@wyvern.cs.uni-duesseldorf.de> References: <20120116201610.009C0820D8@wyvern.cs.uni-duesseldorf.de> Message-ID: <4F154939.4030304@gmx.de> Comments inline. On 01/16/2012 09:16 PM, fijal wrote: > Author: Maciej Fijalkowski > Branch: > Changeset: r51364:f6b8525d8a10 > Date: 2012-01-16 22:14 +0200 > http://bitbucket.org/pypy/pypy/changeset/f6b8525d8a10/ > > Log: A bit experimental - try to preallocate the size of unicode join and > remove a pointless performance hack (the general optimization should > work already) > > diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py > --- a/pypy/objspace/std/unicodeobject.py > +++ b/pypy/objspace/std/unicodeobject.py > @@ -201,7 +201,7 @@ > return space.newbool(container.find(item) != -1) > > def unicode_join__Unicode_ANY(space, w_self, w_list): > - list_w = space.unpackiterable(w_list) > + list_w = space.listview(w_list) > size = len(list_w) > > if size == 0: > @@ -216,22 +216,21 @@ > > def _unicode_join_many_items(space, w_self, list_w, size): > self = w_self._value > - sb = UnicodeBuilder() > + prealloc_size = 0 > + for i in range(size): > + prealloc_size += len(space.unicode_w(list_w[i])) > + sb = UnicodeBuilder(prealloc_size) Doesn't this produce the wrong error message if something in the list is not a unicode object? I guess no tests fails because nobody checks the error message. Cheers, Carl Friedrich > for i in range(size): > if self and i != 0: > sb.append(self) > w_s = list_w[i] > - if isinstance(w_s, W_UnicodeObject): > - # shortcut for performance > - sb.append(w_s._value) > - else: > - try: > - sb.append(space.unicode_w(w_s)) > - except OperationError, e: > - if not e.match(space, space.w_TypeError): > - raise > - raise operationerrfmt(space.w_TypeError, > - "sequence item %d: expected string or Unicode", i) > + try: > + sb.append(space.unicode_w(w_s)) > + except OperationError, e: > + if not e.match(space, space.w_TypeError): > + raise > + raise operationerrfmt(space.w_TypeError, > + "sequence item %d: expected string or Unicode", i) > return space.wrap(sb.build()) > > def hash__Unicode(space, w_uni): > _______________________________________________ > pypy-commit mailing list > pypy-commit at python.org > http://mail.python.org/mailman/listinfo/pypy-commit From noreply at buildbot.pypy.org Tue Jan 17 11:11:51 2012 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 17 Jan 2012 11:11:51 +0100 (CET) Subject: [pypy-commit] extradoc extradoc: (all) planning for today Message-ID: <20120117101151.E2DED820D8@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: extradoc Changeset: r4039:eb50533e45ea Date: 2012-01-17 11:11 +0100 http://bitbucket.org/pypy/extradoc/changeset/eb50533e45ea/ Log: (all) planning for today diff --git a/sprintinfo/leysin-winter-2012/planning.txt b/sprintinfo/leysin-winter-2012/planning.txt --- a/sprintinfo/leysin-winter-2012/planning.txt +++ b/sprintinfo/leysin-winter-2012/planning.txt @@ -9,17 +9,19 @@ Things we want to do -------------------- -* some skiing +* some skiing: wednesday (last day of sun :-/ ) -* review the JVM backend pull request (anto) +* review the JVM backend pull request (DONE) -* py3k (romain) +* py3k (romain, anto later) * ffistruct -* Cython backend (romain) +* Cython backend * STM - start work on the GC (anto, arigo) + - refactored the RPython API: mostly done, must adapt targetdemo*.py + (anto, arigo) + - start work on the GC (arigo later?) * concurrent-marksweep GC From noreply at buildbot.pypy.org Tue Jan 17 11:16:33 2012 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 17 Jan 2012 11:16:33 +0100 (CET) Subject: [pypy-commit] pypy numpy-indexing-by-arrays-2: jit merge points Message-ID: <20120117101633.A8E49820D8@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: numpy-indexing-by-arrays-2 Changeset: r51386:fed67b9508a6 Date: 2012-01-17 12:15 +0200 http://bitbucket.org/pypy/pypy/changeset/fed67b9508a6/ Log: jit merge points diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -40,6 +40,18 @@ get_printable_location=signature.new_printable_location('slice'), name='numpy_slice', ) +count_driver = jit.JitDriver( + greens=['shapelen'], + virtualizables=['frame'], + reds=['frame', 's', 'iter', 'arr'], + name='numpy_count' +) +filter_driver = jit.JitDriver( + greens=['sig', 'shapelen'], + virtualizables=['frame'], + reds=['concr', 'argi', 'ri', 'frame', 'v', 'res'], + name='numpy_filter', +) def _find_shape_and_elems(space, w_iterable): shape = [space.len_w(w_iterable)] @@ -491,7 +503,10 @@ frame = sig.create_frame(self) shapelen = len(arr.shape) s = 0 + iter = None while not frame.done(): + count_driver.jit_merge_point(arr=arr, frame=frame, iter=iter, s=s, + shapelen=shapelen) iter = frame.get_final_iter() s += arr.dtype.getitem_bool(arr.storage, iter.offset) frame.next(shapelen) @@ -506,7 +521,11 @@ argi = ArrayIterator(concr.size) sig = self.find_sig() frame = sig.create_frame(self) + v = None while not frame.done(): + filter_driver.jit_merge_point(concr=concr, argi=argi, ri=ri, + frame=frame, v=v, res=res, sig=sig, + shapelen=shapelen) if concr.dtype.getitem_bool(concr.storage, argi.offset): v = sig.eval(frame, self) res.setitem(ri.offset, v) From noreply at buildbot.pypy.org Tue Jan 17 11:27:20 2012 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 17 Jan 2012 11:27:20 +0100 (CET) Subject: [pypy-commit] pypy numpy-indexing-by-arrays-2: fix test_zjit. Skip the test that fails every time someone touches the world. Message-ID: <20120117102720.ACD76820D8@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: numpy-indexing-by-arrays-2 Changeset: r51387:92eaf5297003 Date: 2012-01-17 12:26 +0200 http://bitbucket.org/pypy/pypy/changeset/92eaf5297003/ Log: fix test_zjit. Skip the test that fails every time someone touches the world. diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py --- a/pypy/module/micronumpy/interp_dtype.py +++ b/pypy/module/micronumpy/interp_dtype.py @@ -48,7 +48,7 @@ def getitem_bool(self, storage, i): isize = self.itemtype.get_element_size() - return self.itemtype.read_raw(storage, isize, i, 0, bool) + return self.itemtype.read_bool(storage, isize, i, 0) def setitem(self, storage, i, box): self.itemtype.store(storage, self.itemtype.get_element_size(), i, 0, box) diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -43,13 +43,13 @@ count_driver = jit.JitDriver( greens=['shapelen'], virtualizables=['frame'], - reds=['frame', 's', 'iter', 'arr'], + reds=['s', 'frame', 'iter', 'arr'], name='numpy_count' ) filter_driver = jit.JitDriver( - greens=['sig', 'shapelen'], + greens=['shapelen', 'sig'], virtualizables=['frame'], - reds=['concr', 'argi', 'ri', 'frame', 'v', 'res'], + reds=['concr', 'argi', 'ri', 'frame', 'v', 'res', 'self'], name='numpy_filter', ) @@ -525,7 +525,7 @@ while not frame.done(): filter_driver.jit_merge_point(concr=concr, argi=argi, ri=ri, frame=frame, v=v, res=res, sig=sig, - shapelen=shapelen) + shapelen=shapelen, self=self) if concr.dtype.getitem_bool(concr.storage, argi.offset): v = sig.eval(frame, self) res.setitem(ri.offset, v) diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py --- a/pypy/module/micronumpy/test/test_zjit.py +++ b/pypy/module/micronumpy/test/test_zjit.py @@ -217,6 +217,7 @@ # This is the sum of the ops for both loops, however if you remove the # optimization then you end up with 2 float_adds, so we can still be # sure it was optimized correctly. + py.test.skip("too fragile") self.check_resops({'setinteriorfield_raw': 4, 'getfield_gc': 22, 'getarrayitem_gc': 4, 'getarrayitem_gc_pure': 2, 'getfield_gc_pure': 8, diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py --- a/pypy/module/micronumpy/types.py +++ b/pypy/module/micronumpy/types.py @@ -94,10 +94,8 @@ width, storage, i, offset )) - @specialize.arg(5) - def read_raw(self, storage, width, i, offset, tp): - return libffi.array_getitem(clibffi.cast_type_to_ffitype(self.T), - width, storage, i, offset) + def read_bool(self, storage, width, i, offset): + raise NotImplementedError def store(self, storage, width, i, offset, box): value = self.unbox(box) @@ -199,6 +197,11 @@ else: return self.False + + def read_bool(self, storage, width, i, offset): + return libffi.array_getitem(clibffi.cast_type_to_ffitype(self.T), + width, storage, i, offset) + def coerce_subtype(self, space, w_subtype, w_item): # Doesn't return subclasses so it can return the constants. return self._coerce(space, w_item) From noreply at buildbot.pypy.org Tue Jan 17 11:29:46 2012 From: noreply at buildbot.pypy.org (antocuni) Date: Tue, 17 Jan 2012 11:29:46 +0100 (CET) Subject: [pypy-commit] pypy jvm-improvements: close about-to-be-merged branch Message-ID: <20120117102946.DC461820D8@wyvern.cs.uni-duesseldorf.de> Author: Antonio Cuni Branch: jvm-improvements Changeset: r51388:88e6de55b13e Date: 2012-01-17 11:28 +0100 http://bitbucket.org/pypy/pypy/changeset/88e6de55b13e/ Log: close about-to-be-merged branch From noreply at buildbot.pypy.org Tue Jan 17 11:29:48 2012 From: noreply at buildbot.pypy.org (antocuni) Date: Tue, 17 Jan 2012 11:29:48 +0100 (CET) Subject: [pypy-commit] pypy default: (benol) merge the jvm-improvements branch, which fixes (again :-)) the translation for the JVM backend Message-ID: <20120117102948.1EF92820D8@wyvern.cs.uni-duesseldorf.de> Author: Antonio Cuni Branch: Changeset: r51389:c1b66ebfb441 Date: 2012-01-17 11:29 +0100 http://bitbucket.org/pypy/pypy/changeset/c1b66ebfb441/ Log: (benol) merge the jvm-improvements branch, which fixes (again :-)) the translation for the JVM backend diff --git a/.hgignore b/.hgignore --- a/.hgignore +++ b/.hgignore @@ -2,6 +2,9 @@ *.py[co] *~ .*.swp +.idea +.project +.pydevproject syntax: regexp ^testresult$ diff --git a/pypy/rlib/longlong2float.py b/pypy/rlib/longlong2float.py --- a/pypy/rlib/longlong2float.py +++ b/pypy/rlib/longlong2float.py @@ -79,19 +79,23 @@ longlong2float = rffi.llexternal( "pypy__longlong2float", [rffi.LONGLONG], rffi.DOUBLE, _callable=longlong2float_emulator, compilation_info=eci, - _nowrapper=True, elidable_function=True, sandboxsafe=True) + _nowrapper=True, elidable_function=True, sandboxsafe=True, + oo_primitive="pypy__longlong2float") float2longlong = rffi.llexternal( "pypy__float2longlong", [rffi.DOUBLE], rffi.LONGLONG, _callable=float2longlong_emulator, compilation_info=eci, - _nowrapper=True, elidable_function=True, sandboxsafe=True) + _nowrapper=True, elidable_function=True, sandboxsafe=True, + oo_primitive="pypy__float2longlong") uint2singlefloat = rffi.llexternal( "pypy__uint2singlefloat", [rffi.UINT], rffi.FLOAT, _callable=uint2singlefloat_emulator, compilation_info=eci, - _nowrapper=True, elidable_function=True, sandboxsafe=True) + _nowrapper=True, elidable_function=True, sandboxsafe=True, + oo_primitive="pypy__uint2singlefloat") singlefloat2uint = rffi.llexternal( "pypy__singlefloat2uint", [rffi.FLOAT], rffi.UINT, _callable=singlefloat2uint_emulator, compilation_info=eci, - _nowrapper=True, elidable_function=True, sandboxsafe=True) + _nowrapper=True, elidable_function=True, sandboxsafe=True, + oo_primitive="pypy__singlefloat2uint") diff --git a/pypy/rlib/objectmodel.py b/pypy/rlib/objectmodel.py --- a/pypy/rlib/objectmodel.py +++ b/pypy/rlib/objectmodel.py @@ -420,7 +420,7 @@ vobj.concretetype.TO._gckind == 'gc') else: from pypy.rpython.ootypesystem import ootype - ok = isinstance(vobj.concretetype, ootype.Instance) + ok = isinstance(vobj.concretetype, (ootype.Instance, ootype.BuiltinType)) if not ok: from pypy.rpython.error import TyperError raise TyperError("compute_unique_id() cannot be applied to" diff --git a/pypy/rpython/ootypesystem/ootype.py b/pypy/rpython/ootypesystem/ootype.py --- a/pypy/rpython/ootypesystem/ootype.py +++ b/pypy/rpython/ootypesystem/ootype.py @@ -512,6 +512,7 @@ "ll_append_char": Meth([CHARTP], Void), "ll_append": Meth([STRINGTP], Void), "ll_build": Meth([], STRINGTP), + "ll_getlength": Meth([], Signed), }) self._setup_methods({}) @@ -1376,6 +1377,9 @@ def _cast_to_object(self): return make_object(self) + def _identityhash(self): + return object.__hash__(self) + class _string(_builtin_type): def __init__(self, STRING, value = ''): @@ -1543,6 +1547,9 @@ else: return make_unicode(u''.join(self._buf)) + def ll_getlength(self): + return self.ll_build().ll_strlen() + class _null_string_builder(_null_mixin(_string_builder), _string_builder): def __init__(self, STRING_BUILDER): self.__dict__["_TYPE"] = STRING_BUILDER diff --git a/pypy/rpython/ootypesystem/rbuilder.py b/pypy/rpython/ootypesystem/rbuilder.py --- a/pypy/rpython/ootypesystem/rbuilder.py +++ b/pypy/rpython/ootypesystem/rbuilder.py @@ -21,6 +21,10 @@ builder.ll_append_char(char) @staticmethod + def ll_getlength(builder): + return builder.ll_getlength() + + @staticmethod def ll_append(builder, string): builder.ll_append(string) diff --git a/pypy/rpython/test/test_rbuilder.py b/pypy/rpython/test/test_rbuilder.py --- a/pypy/rpython/test/test_rbuilder.py +++ b/pypy/rpython/test/test_rbuilder.py @@ -124,9 +124,5 @@ pass class TestOOtype(BaseTestStringBuilder, OORtypeMixin): - def test_string_getlength(self): - py.test.skip("getlength(): not implemented on ootype") - def test_unicode_getlength(self): - py.test.skip("getlength(): not implemented on ootype") def test_append_charpsize(self): py.test.skip("append_charpsize(): not implemented on ootype") diff --git a/pypy/rpython/test/test_rbuiltin.py b/pypy/rpython/test/test_rbuiltin.py --- a/pypy/rpython/test/test_rbuiltin.py +++ b/pypy/rpython/test/test_rbuiltin.py @@ -463,6 +463,31 @@ assert x1 == intmask(x0) assert x3 == intmask(x2) + def test_id_on_builtins(self): + from pypy.rlib.objectmodel import compute_unique_id + from pypy.rlib.rstring import StringBuilder, UnicodeBuilder + def fn(): + return (compute_unique_id("foo"), + compute_unique_id(u"bar"), + compute_unique_id([1]), + compute_unique_id({"foo": 3}), + compute_unique_id(StringBuilder()), + compute_unique_id(UnicodeBuilder())) + res = self.interpret(fn, []) + for id in self.ll_unpack_tuple(res, 6): + assert isinstance(id, (int, r_longlong)) + + def test_uniqueness_of_id_on_strings(self): + from pypy.rlib.objectmodel import compute_unique_id + def fn(s1, s2): + return (compute_unique_id(s1), compute_unique_id(s2)) + + s1 = "foo" + s2 = ''.join(['f','oo']) + res = self.interpret(fn, [self.string_to_ll(s1), self.string_to_ll(s2)]) + i1, i2 = self.ll_unpack_tuple(res, 2) + assert i1 != i2 + def test_cast_primitive(self): from pypy.rpython.annlowlevel import LowLevelAnnotatorPolicy def llf(u): diff --git a/pypy/translator/jvm/builtin.py b/pypy/translator/jvm/builtin.py --- a/pypy/translator/jvm/builtin.py +++ b/pypy/translator/jvm/builtin.py @@ -84,6 +84,9 @@ (ootype.StringBuilder.__class__, "ll_build"): jvm.Method.v(jStringBuilder, "toString", (), jString), + (ootype.StringBuilder.__class__, "ll_getlength"): + jvm.Method.v(jStringBuilder, "length", (), jInt), + (ootype.String.__class__, "ll_hash"): jvm.Method.v(jString, "hashCode", (), jInt), diff --git a/pypy/translator/jvm/database.py b/pypy/translator/jvm/database.py --- a/pypy/translator/jvm/database.py +++ b/pypy/translator/jvm/database.py @@ -358,7 +358,7 @@ ootype.Unsigned:jvm.PYPYSERIALIZEUINT, ootype.SignedLongLong:jvm.LONGTOSTRINGL, ootype.UnsignedLongLong: jvm.PYPYSERIALIZEULONG, - ootype.Float:jvm.DOUBLETOSTRINGD, + ootype.Float:jvm.PYPYSERIALIZEDOUBLE, ootype.Bool:jvm.PYPYSERIALIZEBOOLEAN, ootype.Void:jvm.PYPYSERIALIZEVOID, ootype.Char:jvm.PYPYESCAPEDCHAR, diff --git a/pypy/translator/jvm/metavm.py b/pypy/translator/jvm/metavm.py --- a/pypy/translator/jvm/metavm.py +++ b/pypy/translator/jvm/metavm.py @@ -92,6 +92,7 @@ CASTS = { # FROM TO (ootype.Signed, ootype.UnsignedLongLong): jvm.I2L, + (ootype.Unsigned, ootype.UnsignedLongLong): jvm.I2L, (ootype.SignedLongLong, ootype.Signed): jvm.L2I, (ootype.UnsignedLongLong, ootype.Unsigned): jvm.L2I, (ootype.UnsignedLongLong, ootype.Signed): jvm.L2I, diff --git a/pypy/translator/jvm/opcodes.py b/pypy/translator/jvm/opcodes.py --- a/pypy/translator/jvm/opcodes.py +++ b/pypy/translator/jvm/opcodes.py @@ -101,6 +101,7 @@ 'jit_force_virtualizable': Ignore, 'jit_force_virtual': DoNothing, 'jit_force_quasi_immutable': Ignore, + 'jit_is_virtual': PushPrimitive(ootype.Bool, False), 'debug_assert': [], # TODO: implement? 'debug_start_traceback': Ignore, diff --git a/pypy/translator/jvm/src/pypy/PyPy.java b/pypy/translator/jvm/src/pypy/PyPy.java --- a/pypy/translator/jvm/src/pypy/PyPy.java +++ b/pypy/translator/jvm/src/pypy/PyPy.java @@ -283,6 +283,14 @@ } } + public double pypy__longlong2float(long l) { + return Double.longBitsToDouble(l); + } + + public long pypy__float2longlong(double d) { + return Double.doubleToRawLongBits(d); + } + public double ooparse_float(String s) { try { return Double.parseDouble(s); @@ -353,6 +361,19 @@ return "False"; } + public static String serialize_double(double d) { + if (Double.isNaN(d)) { + return "float(\"nan\")"; + } else if (Double.isInfinite(d)) { + if (d > 0) + return "float(\"inf\")"; + else + return "float(\"-inf\")"; + } else { + return Double.toString(d); + } + } + private static String format_char(char c) { String res = "\\x"; if (c <= 0x0F) res = res + "0"; diff --git a/pypy/translator/jvm/test/runtest.py b/pypy/translator/jvm/test/runtest.py --- a/pypy/translator/jvm/test/runtest.py +++ b/pypy/translator/jvm/test/runtest.py @@ -56,6 +56,7 @@ # CLI could-be duplicate class JvmGeneratedSourceWrapper(object): + def __init__(self, gensrc): """ gensrc is an instance of JvmGeneratedSource """ self.gensrc = gensrc diff --git a/pypy/translator/jvm/test/test_builder.py b/pypy/translator/jvm/test/test_builder.py new file mode 100644 --- /dev/null +++ b/pypy/translator/jvm/test/test_builder.py @@ -0,0 +1,7 @@ +from pypy.translator.jvm.test.runtest import JvmTest +from pypy.rpython.test.test_rbuilder import BaseTestStringBuilder +import py + +class TestJvmStringBuilder(JvmTest, BaseTestStringBuilder): + def test_append_charpsize(self): + py.test.skip("append_charpsize(): not implemented on ootype") diff --git a/pypy/translator/jvm/test/test_longlong2float.py b/pypy/translator/jvm/test/test_longlong2float.py new file mode 100644 --- /dev/null +++ b/pypy/translator/jvm/test/test_longlong2float.py @@ -0,0 +1,20 @@ +from pypy.translator.jvm.test.runtest import JvmTest +from pypy.rlib.longlong2float import * +from pypy.rlib.test.test_longlong2float import enum_floats +from pypy.rlib.test.test_longlong2float import fn as float2longlong2float +import py + +class TestLongLong2Float(JvmTest): + + def test_float2longlong_and_longlong2float(self): + def func(f): + return float2longlong2float(f) + + for f in enum_floats(): + assert repr(f) == repr(self.interpret(func, [f])) + + def test_uint2singlefloat(self): + py.test.skip("uint2singlefloat is not implemented in ootype") + + def test_singlefloat2uint(self): + py.test.skip("singlefloat2uint is not implemented in ootype") diff --git a/pypy/translator/jvm/typesystem.py b/pypy/translator/jvm/typesystem.py --- a/pypy/translator/jvm/typesystem.py +++ b/pypy/translator/jvm/typesystem.py @@ -955,6 +955,7 @@ PYPYSERIALIZEUINT = Method.s(jPyPy, 'serialize_uint', (jInt,), jString) PYPYSERIALIZEULONG = Method.s(jPyPy, 'serialize_ulonglong', (jLong,),jString) PYPYSERIALIZEVOID = Method.s(jPyPy, 'serialize_void', (), jString) +PYPYSERIALIZEDOUBLE = Method.s(jPyPy, 'serialize_double', (jDouble,), jString) PYPYESCAPEDCHAR = Method.s(jPyPy, 'escaped_char', (jChar,), jString) PYPYESCAPEDUNICHAR = Method.s(jPyPy, 'escaped_unichar', (jChar,), jString) PYPYESCAPEDSTRING = Method.s(jPyPy, 'escaped_string', (jString,), jString) diff --git a/pypy/translator/oosupport/test_template/cast.py b/pypy/translator/oosupport/test_template/cast.py --- a/pypy/translator/oosupport/test_template/cast.py +++ b/pypy/translator/oosupport/test_template/cast.py @@ -13,6 +13,9 @@ def to_longlong(x): return r_longlong(x) +def to_ulonglong(x): + return r_ulonglong(x) + def uint_to_int(x): return intmask(x) @@ -56,6 +59,9 @@ def test_unsignedlonglong_to_unsigned4(self): self.check(to_uint, [r_ulonglong(18446744073709551615l)]) # max 64 bit num + def test_unsigned_to_usignedlonglong(self): + self.check(to_ulonglong, [r_uint(42)]) + def test_uint_to_int(self): self.check(uint_to_int, [r_uint(sys.maxint+1)]) From noreply at buildbot.pypy.org Tue Jan 17 12:16:17 2012 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 17 Jan 2012 12:16:17 +0100 (CET) Subject: [pypy-commit] pypy default: a minor improvement Message-ID: <20120117111617.82677820D8@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r51390:89436d06f0ae Date: 2012-01-17 12:36 +0200 http://bitbucket.org/pypy/pypy/changeset/89436d06f0ae/ Log: a minor improvement diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -216,7 +216,7 @@ def _unicode_join_many_items(space, w_self, list_w, size): self = w_self._value - prealloc_size = 0 + prealloc_size = len(self) * (size - 1) for i in range(size): prealloc_size += len(space.unicode_w(list_w[i])) sb = UnicodeBuilder(prealloc_size) From noreply at buildbot.pypy.org Tue Jan 17 12:16:18 2012 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 17 Jan 2012 12:16:18 +0100 (CET) Subject: [pypy-commit] pypy numpy-indexing-by-arrays-2: progress on arr[arr_of_bools] = arr Message-ID: <20120117111618.AE2B2820D8@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: numpy-indexing-by-arrays-2 Changeset: r51391:fe9bca1da2b6 Date: 2012-01-17 13:15 +0200 http://bitbucket.org/pypy/pypy/changeset/fe9bca1da2b6/ Log: progress on arr[arr_of_bools] = arr diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -10,7 +10,7 @@ from pypy.tool.sourcetools import func_with_new_name from pypy.rlib.rstring import StringBuilder from pypy.module.micronumpy.interp_iter import ArrayIterator, OneDimIterator,\ - SkipLastAxisIterator, Chunk + SkipLastAxisIterator, Chunk, ViewIterator numpy_driver = jit.JitDriver( greens=['shapelen', 'sig'], @@ -518,7 +518,7 @@ res = W_NDimArray(size, [size], self.find_dtype()) ri = ArrayIterator(size) shapelen = len(self.shape) - argi = ArrayIterator(concr.size) + argi = concr.create_iter() sig = self.find_sig() frame = sig.create_frame(self) v = None @@ -536,6 +536,18 @@ frame.next(shapelen) return res + def setitem_filter(self, space, idx, val): + arr = SliceArray(self.shape, self.dtype, self, val) + shapelen = len(arr.shape) + sig = arr.find_sig() + frame = sig.create_frame(arr) + idxi = idx.create_iter() + while not frame.done(): + if idx.dtype.getitem_bool(idx.storage, idxi.offset): + sig.eval(frame, arr) + idxi = idxi.next(shapelen) + frame.next(shapelen) + def descr_getitem(self, space, w_idx): if (isinstance(w_idx, BaseArray) and w_idx.shape == self.shape and w_idx.find_dtype().is_bool_type()): @@ -549,6 +561,11 @@ def descr_setitem(self, space, w_idx, w_value): self.invalidated() + if (isinstance(w_idx, BaseArray) and w_idx.shape == self.shape and + w_idx.find_dtype().is_bool_type()): + return self.get_concrete().setitem_filter(space, + w_idx.get_concrete(), + convert_to_array(space, w_value)) if self._single_item_result(space, w_idx): concrete = self.get_concrete() item = concrete._index_of_single_item(space, w_idx) @@ -1135,6 +1152,10 @@ parent) self.start = start + def create_iter(self): + return ViewIterator(self.start, self.strides, self.backstrides, + self.shape) + def setshape(self, space, new_shape): if len(self.shape) < 1: return @@ -1181,6 +1202,9 @@ self.shape = new_shape self.calc_strides(new_shape) + def create_iter(self): + return ArrayIterator(self.size) + def create_sig(self): return signature.ArraySignature(self.dtype) @@ -1235,6 +1259,7 @@ arr = W_NDimArray(size, shape[:], dtype=dtype, order=order) shapelen = len(shape) arr_iter = ArrayIterator(arr.size) + # XXX we might want to have a jitdriver here for i in range(len(elems_w)): w_elem = elems_w[i] dtype.setitem(arr.storage, arr_iter.offset, diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -1320,6 +1320,15 @@ assert (a[a > 3] == [4, 5, 6, 7, 8, 9]).all() assert (a[a & 1 == 1] == [1, 3, 5, 7, 9]).all() + def test_array_indexing_bool_setitem(self): + from _numpypy import arange, array + a = arange(6) + a[a > 3] = 15 + assert (a == [0, 1, 2, 3, 15, 15]).all() + a = arange(6).reshape(3, 2) + a[a & 1 == 1] = array([8, 9, 10]) + assert (a == [[0, 8], [3, 9], [5, 10]]).all() + class AppTestSupport(BaseNumpyAppTest): def setup_class(cls): import struct From noreply at buildbot.pypy.org Tue Jan 17 12:16:19 2012 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 17 Jan 2012 12:16:19 +0100 (CET) Subject: [pypy-commit] pypy default: merge Message-ID: <20120117111619.D552C820D8@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r51392:81aaf2060a57 Date: 2012-01-17 13:15 +0200 http://bitbucket.org/pypy/pypy/changeset/81aaf2060a57/ Log: merge diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -216,7 +216,7 @@ def _unicode_join_many_items(space, w_self, list_w, size): self = w_self._value - prealloc_size = 0 + prealloc_size = len(self) * (size - 1) for i in range(size): prealloc_size += len(space.unicode_w(list_w[i])) sb = UnicodeBuilder(prealloc_size) From noreply at buildbot.pypy.org Tue Jan 17 12:25:26 2012 From: noreply at buildbot.pypy.org (rguillebert) Date: Tue, 17 Jan 2012 12:25:26 +0100 (CET) Subject: [pypy-commit] pypy py3k: On linux, set ascii as the default locale if none can be found Message-ID: <20120117112526.02E53820D8@wyvern.cs.uni-duesseldorf.de> Author: Romain Guillebert Branch: py3k Changeset: r51393:16202ef48d39 Date: 2012-01-17 12:23 +0100 http://bitbucket.org/pypy/pypy/changeset/16202ef48d39/ Log: On linux, set ascii as the default locale if none can be found diff --git a/pypy/module/sys/interp_encoding.py b/pypy/module/sys/interp_encoding.py --- a/pypy/module/sys/interp_encoding.py +++ b/pypy/module/sys/interp_encoding.py @@ -33,6 +33,8 @@ base_encoding = "mbcs" elif sys.platform == "darwin": base_encoding = "utf-8" +elif sys.platform == "linux2": + base_encoding = "ascii" else: base_encoding = None From noreply at buildbot.pypy.org Tue Jan 17 12:25:27 2012 From: noreply at buildbot.pypy.org (rguillebert) Date: Tue, 17 Jan 2012 12:25:27 +0100 (CET) Subject: [pypy-commit] pypy py3k: Merge heads Message-ID: <20120117112527.61741820D8@wyvern.cs.uni-duesseldorf.de> Author: Romain Guillebert Branch: py3k Changeset: r51394:7948fef3ed5a Date: 2012-01-17 12:25 +0100 http://bitbucket.org/pypy/pypy/changeset/7948fef3ed5a/ Log: Merge heads diff --git a/lib_pypy/binascii.py b/lib_pypy/binascii.py --- a/lib_pypy/binascii.py +++ b/lib_pypy/binascii.py @@ -598,68 +598,68 @@ return ''.join(result) crc_32_tab = [ - 0x00000000L, 0x77073096L, 0xee0e612cL, 0x990951baL, 0x076dc419L, - 0x706af48fL, 0xe963a535L, 0x9e6495a3L, 0x0edb8832L, 0x79dcb8a4L, - 0xe0d5e91eL, 0x97d2d988L, 0x09b64c2bL, 0x7eb17cbdL, 0xe7b82d07L, - 0x90bf1d91L, 0x1db71064L, 0x6ab020f2L, 0xf3b97148L, 0x84be41deL, - 0x1adad47dL, 0x6ddde4ebL, 0xf4d4b551L, 0x83d385c7L, 0x136c9856L, - 0x646ba8c0L, 0xfd62f97aL, 0x8a65c9ecL, 0x14015c4fL, 0x63066cd9L, - 0xfa0f3d63L, 0x8d080df5L, 0x3b6e20c8L, 0x4c69105eL, 0xd56041e4L, - 0xa2677172L, 0x3c03e4d1L, 0x4b04d447L, 0xd20d85fdL, 0xa50ab56bL, - 0x35b5a8faL, 0x42b2986cL, 0xdbbbc9d6L, 0xacbcf940L, 0x32d86ce3L, - 0x45df5c75L, 0xdcd60dcfL, 0xabd13d59L, 0x26d930acL, 0x51de003aL, - 0xc8d75180L, 0xbfd06116L, 0x21b4f4b5L, 0x56b3c423L, 0xcfba9599L, - 0xb8bda50fL, 0x2802b89eL, 0x5f058808L, 0xc60cd9b2L, 0xb10be924L, - 0x2f6f7c87L, 0x58684c11L, 0xc1611dabL, 0xb6662d3dL, 0x76dc4190L, - 0x01db7106L, 0x98d220bcL, 0xefd5102aL, 0x71b18589L, 0x06b6b51fL, - 0x9fbfe4a5L, 0xe8b8d433L, 0x7807c9a2L, 0x0f00f934L, 0x9609a88eL, - 0xe10e9818L, 0x7f6a0dbbL, 0x086d3d2dL, 0x91646c97L, 0xe6635c01L, - 0x6b6b51f4L, 0x1c6c6162L, 0x856530d8L, 0xf262004eL, 0x6c0695edL, - 0x1b01a57bL, 0x8208f4c1L, 0xf50fc457L, 0x65b0d9c6L, 0x12b7e950L, - 0x8bbeb8eaL, 0xfcb9887cL, 0x62dd1ddfL, 0x15da2d49L, 0x8cd37cf3L, - 0xfbd44c65L, 0x4db26158L, 0x3ab551ceL, 0xa3bc0074L, 0xd4bb30e2L, - 0x4adfa541L, 0x3dd895d7L, 0xa4d1c46dL, 0xd3d6f4fbL, 0x4369e96aL, - 0x346ed9fcL, 0xad678846L, 0xda60b8d0L, 0x44042d73L, 0x33031de5L, - 0xaa0a4c5fL, 0xdd0d7cc9L, 0x5005713cL, 0x270241aaL, 0xbe0b1010L, - 0xc90c2086L, 0x5768b525L, 0x206f85b3L, 0xb966d409L, 0xce61e49fL, - 0x5edef90eL, 0x29d9c998L, 0xb0d09822L, 0xc7d7a8b4L, 0x59b33d17L, - 0x2eb40d81L, 0xb7bd5c3bL, 0xc0ba6cadL, 0xedb88320L, 0x9abfb3b6L, - 0x03b6e20cL, 0x74b1d29aL, 0xead54739L, 0x9dd277afL, 0x04db2615L, - 0x73dc1683L, 0xe3630b12L, 0x94643b84L, 0x0d6d6a3eL, 0x7a6a5aa8L, - 0xe40ecf0bL, 0x9309ff9dL, 0x0a00ae27L, 0x7d079eb1L, 0xf00f9344L, - 0x8708a3d2L, 0x1e01f268L, 0x6906c2feL, 0xf762575dL, 0x806567cbL, - 0x196c3671L, 0x6e6b06e7L, 0xfed41b76L, 0x89d32be0L, 0x10da7a5aL, - 0x67dd4accL, 0xf9b9df6fL, 0x8ebeeff9L, 0x17b7be43L, 0x60b08ed5L, - 0xd6d6a3e8L, 0xa1d1937eL, 0x38d8c2c4L, 0x4fdff252L, 0xd1bb67f1L, - 0xa6bc5767L, 0x3fb506ddL, 0x48b2364bL, 0xd80d2bdaL, 0xaf0a1b4cL, - 0x36034af6L, 0x41047a60L, 0xdf60efc3L, 0xa867df55L, 0x316e8eefL, - 0x4669be79L, 0xcb61b38cL, 0xbc66831aL, 0x256fd2a0L, 0x5268e236L, - 0xcc0c7795L, 0xbb0b4703L, 0x220216b9L, 0x5505262fL, 0xc5ba3bbeL, - 0xb2bd0b28L, 0x2bb45a92L, 0x5cb36a04L, 0xc2d7ffa7L, 0xb5d0cf31L, - 0x2cd99e8bL, 0x5bdeae1dL, 0x9b64c2b0L, 0xec63f226L, 0x756aa39cL, - 0x026d930aL, 0x9c0906a9L, 0xeb0e363fL, 0x72076785L, 0x05005713L, - 0x95bf4a82L, 0xe2b87a14L, 0x7bb12baeL, 0x0cb61b38L, 0x92d28e9bL, - 0xe5d5be0dL, 0x7cdcefb7L, 0x0bdbdf21L, 0x86d3d2d4L, 0xf1d4e242L, - 0x68ddb3f8L, 0x1fda836eL, 0x81be16cdL, 0xf6b9265bL, 0x6fb077e1L, - 0x18b74777L, 0x88085ae6L, 0xff0f6a70L, 0x66063bcaL, 0x11010b5cL, - 0x8f659effL, 0xf862ae69L, 0x616bffd3L, 0x166ccf45L, 0xa00ae278L, - 0xd70dd2eeL, 0x4e048354L, 0x3903b3c2L, 0xa7672661L, 0xd06016f7L, - 0x4969474dL, 0x3e6e77dbL, 0xaed16a4aL, 0xd9d65adcL, 0x40df0b66L, - 0x37d83bf0L, 0xa9bcae53L, 0xdebb9ec5L, 0x47b2cf7fL, 0x30b5ffe9L, - 0xbdbdf21cL, 0xcabac28aL, 0x53b39330L, 0x24b4a3a6L, 0xbad03605L, - 0xcdd70693L, 0x54de5729L, 0x23d967bfL, 0xb3667a2eL, 0xc4614ab8L, - 0x5d681b02L, 0x2a6f2b94L, 0xb40bbe37L, 0xc30c8ea1L, 0x5a05df1bL, - 0x2d02ef8dL + 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, + 0x706af48f, 0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4, + 0xe0d5e91e, 0x97d2d988, 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, + 0x90bf1d91, 0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de, + 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, 0x136c9856, + 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9, + 0xfa0f3d63, 0x8d080df5, 0x3b6e20c8, 0x4c69105e, 0xd56041e4, + 0xa2677172, 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, + 0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, + 0x45df5c75, 0xdcd60dcf, 0xabd13d59, 0x26d930ac, 0x51de003a, + 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, 0xcfba9599, + 0xb8bda50f, 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, + 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, 0x76dc4190, + 0x01db7106, 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, + 0x9fbfe4a5, 0xe8b8d433, 0x7807c9a2, 0x0f00f934, 0x9609a88e, + 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01, + 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, 0x6c0695ed, + 0x1b01a57b, 0x8208f4c1, 0xf50fc457, 0x65b0d9c6, 0x12b7e950, + 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, + 0xfbd44c65, 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, + 0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb, 0x4369e96a, + 0x346ed9fc, 0xad678846, 0xda60b8d0, 0x44042d73, 0x33031de5, + 0xaa0a4c5f, 0xdd0d7cc9, 0x5005713c, 0x270241aa, 0xbe0b1010, + 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f, + 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, + 0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad, 0xedb88320, 0x9abfb3b6, + 0x03b6e20c, 0x74b1d29a, 0xead54739, 0x9dd277af, 0x04db2615, + 0x73dc1683, 0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8, + 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, 0xf00f9344, + 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, + 0x196c3671, 0x6e6b06e7, 0xfed41b76, 0x89d32be0, 0x10da7a5a, + 0x67dd4acc, 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, + 0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, + 0xa6bc5767, 0x3fb506dd, 0x48b2364b, 0xd80d2bda, 0xaf0a1b4c, + 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, 0x316e8eef, + 0x4669be79, 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, + 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, 0xc5ba3bbe, + 0xb2bd0b28, 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, + 0x2cd99e8b, 0x5bdeae1d, 0x9b64c2b0, 0xec63f226, 0x756aa39c, + 0x026d930a, 0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713, + 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, 0x92d28e9b, + 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, 0x86d3d2d4, 0xf1d4e242, + 0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, + 0x18b74777, 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, + 0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45, 0xa00ae278, + 0xd70dd2ee, 0x4e048354, 0x3903b3c2, 0xa7672661, 0xd06016f7, + 0x4969474d, 0x3e6e77db, 0xaed16a4a, 0xd9d65adc, 0x40df0b66, + 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9, + 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, + 0xcdd70693, 0x54de5729, 0x23d967bf, 0xb3667a2e, 0xc4614ab8, + 0x5d681b02, 0x2a6f2b94, 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, + 0x2d02ef8d ] def crc32(s, crc=0): result = 0 - crc = ~long(crc) & 0xffffffffL + crc = ~int(crc) & 0xffffffff for c in s: - crc = crc_32_tab[(crc ^ c) & 0xffL] ^ (crc >> 8) + crc = crc_32_tab[(crc ^ c) & 0xff] ^ (crc >> 8) #/* Note: (crc >> 8) MUST zero fill on left - result = crc ^ 0xffffffffL + result = crc ^ 0xffffffff if result > 2**31: result = ((result + 2**31) % 2**32) - 2**31 diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py --- a/pypy/module/imp/importing.py +++ b/pypy/module/imp/importing.py @@ -34,6 +34,8 @@ DEFAULT_SOABI = 'pypy-%d%d' % PYPY_VERSION[:2] CHECK_FOR_PYW = sys.platform == 'win32' +PYC_TAG = 'pypy-%d%d' % PYPY_VERSION[:2] + @specialize.memo() def get_so_extension(space): if space.config.objspace.soabi is not None: @@ -91,10 +93,7 @@ else: # XXX that's slow def case_ok(filename): - index = filename.rfind(os.sep) - if os.altsep is not None: - index2 = filename.rfind(os.altsep) - index = max(index, index2) + index = rightmost_sep(filename) if index < 0: directory = os.curdir else: @@ -862,17 +861,63 @@ space.wrap(space.builtin)) code_w.exec_code(space, w_dict, w_dict) +def rightmost_sep(filename): + "Like filename.rfind('/'), but also search for \\." + index = filename.rfind(os.sep) + if os.altsep is not None: + index2 = filename.rfind(os.altsep) + index = max(index, index2) + return index + def make_compiled_pathname(pathname): "Given the path to a .py file, return the path to its .pyc file." - return pathname + 'c' + # foo.py -> __pycache__/foo..pyc + + lastpos = rightmost_sep(pathname) + 1 + assert lastpos >= 0 # zero when slash, takes the full name + fname = pathname[lastpos:] + if lastpos > 0: + # Windows: re-use the last separator character (/ or \\) when + # appending the __pycache__ path. + lastsep = pathname[lastpos-1] + else: + lastsep = os.sep + ext = fname + for i in range(len(fname)): + if fname[i] == '.': + ext = fname[:i + 1] + + result = (pathname[:lastpos] + "__pycache__" + lastsep + + ext + PYC_TAG + '.pyc') + return result def make_source_pathname(pathname): - pos_extension = len(pathname) - 4 # len('.pyc') - if pos_extension < 0: - raise ValueError("path is too short") - if pathname[pos_extension:] != '.pyc': - raise ValueError("not a .pyc path name") - return pathname[:pos_extension + 3] + "Given the path to a .pyc file, return the path to its .py file." + # (...)/__pycache__/foo..pyc -> (...)/foo.py + + right = rightmost_sep(pathname) + if right < 0: + raise ValueError() + left = rightmost_sep(pathname[:right]) + 1 + assert left >= 0 + if pathname[left:right] != '__pycache__': + raise ValueError() + + # Now verify that the path component to the right of the last + # slash has two dots in it. + rightpart = pathname[right + 1:] + dot0 = rightpart.find('.') + 1 + if dot0 <= 0: + raise ValueError + dot1 = rightpart[dot0:].find('.') + 1 + if dot1 <= 0: + raise ValueError + # Too many dots? + if rightpart[dot0 + dot1:].find('.') >= 0: + raise ValueError + + result = pathname[:left] + rightpart[:dot0] + 'py' + return result @jit.dont_look_inside def load_source_module(space, w_modulename, w_mod, pathname, source, @@ -1027,6 +1072,17 @@ Errors are ignored, if a write error occurs an attempt is made to remove the file. """ + # Ensure that the __pycache__ directory exists + dirsep = rightmost_sep(cpathname) + if dirsep < 0: + return + dirname = cpathname[:dirsep] + mode = src_mode | 0333 # +wx + try: + os.mkdir(dirname, mode) + except OSError: + pass + w_marshal = space.getbuiltinmodule('marshal') try: w_bytes = space.call_method(w_marshal, 'dumps', space.wrap(co), diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py --- a/pypy/module/imp/test/test_import.py +++ b/pypy/module/imp/test/test_import.py @@ -38,7 +38,7 @@ test_reload = "def test():\n raise ValueError\n", infinite_reload = "import infinite_reload; reload(infinite_reload)", del_sys_module = "import sys\ndel sys.modules['del_sys_module']\n", - itertools = "hello_world = 42\n", + queue = "hello_world = 42\n", gc = "should_never_be_seen = 42\n", ) root.ensure("notapackage", dir=1) # empty, no __init__.py @@ -75,7 +75,7 @@ ) setuppkg("pkg_substituting", __init__ = "import sys, pkg_substituted\n" - "print 'TOTO', __name__\n" + "print('TOTO', __name__)\n" "sys.modules[__name__] = pkg_substituted") setuppkg("pkg_substituted", mod='') setuppkg("evil_pkg", @@ -102,6 +102,7 @@ w = space.wrap w_modname = w("compiled.x") filename = str(p.join("x.py")) + pycname = importing.make_compiled_pathname("x.py") stream = streamio.open_file_as_stream(filename, "r") try: importing.load_source_module(space, @@ -113,7 +114,7 @@ stream.close() if space.config.objspace.usepycfiles: # also create a lone .pyc file - p.join('lone.pyc').write(p.join('x.pyc').read(mode='rb'), + p.join('lone.pyc').write(p.join(pycname).read(mode='rb'), mode='wb') # create a .pyw file @@ -150,7 +151,7 @@ class AppTestImport: def setup_class(cls): # interpreter-level - cls.space = gettestobjspace(usemodules=['itertools']) + #cls.space = gettestobjspace(usemodules=['itertools']) cls.w_runappdirect = cls.space.wrap(conftest.option.runappdirect) cls.saved_modules = _setup(cls.space) #XXX Compile class @@ -159,6 +160,9 @@ def teardown_class(cls): # interpreter-level _teardown(cls.space, cls.saved_modules) + def w_exec_(self, cmd, ns): + exec(cmd, ns) + def test_set_sys_modules_during_import(self): from evil_pkg import evil assert evil.a == 42 @@ -373,7 +377,7 @@ def test_invalid__name__(self): glob = {} - exec "__name__ = None; import sys" in glob + exec("__name__ = None; import sys", glob) import sys assert glob['sys'] is sys @@ -418,16 +422,18 @@ assert pkg.pkg1.__package__ == 'pkg.pkg1' def test_future_relative_import_error_when_in_non_package(self): - exec """def imp(): + ns = {} + exec("""def imp(): from .string import inpackage - """.rstrip() - raises(ValueError, imp) + """.rstrip(), ns) + raises(ValueError, ns['imp']) def test_future_relative_import_error_when_in_non_package2(self): - exec """def imp(): + ns = {} + exec("""def imp(): from .. import inpackage - """.rstrip() - raises(ValueError, imp) + """.rstrip(), ns) + raises(ValueError, ns['imp']) def test_relative_import_with___name__(self): import sys @@ -456,9 +462,9 @@ def test__package__(self): # Regression test for http://bugs.python.org/issue3221. def check_absolute(): - exec "from os import path" in ns + self.exec_("from os import path", ns) def check_relative(): - exec "from . import a" in ns + self.exec_("from . import a", ns) # Check both OK with __package__ and __name__ correct ns = dict(__package__='pkg', __name__='pkg.notarealmodule') @@ -578,8 +584,11 @@ def test_cache_from_source(self): import imp - assert imp.cache_from_source('a/b/c.py') == 'a/b/c.pyc' - assert imp.source_from_cache('a/b/c.pyc') == 'a/b/c.py' + pycfile = imp.cache_from_source('a/b/c.py') + assert pycfile.startswith('a/b/__pycache__/c.pypy-') + assert pycfile.endswith('.pyc') + assert imp.source_from_cache('a/b/__pycache__/c.pypy-17.pyc' + ) == 'a/b/c.py' raises(ValueError, imp.source_from_cache, 'a/b/c.py') def test_shadow_builtin(self): @@ -596,34 +605,32 @@ def test_shadow_extension_1(self): if self.runappdirect: skip("hard to test: module is already imported") - # 'import itertools' is supposed to find itertools.py if there is + # 'import queue' is supposed to find queue.py if there is # one in sys.path. import sys - assert 'itertools' not in sys.modules - import itertools - assert hasattr(itertools, 'hello_world') - assert not hasattr(itertools, 'count') - assert '(built-in)' not in repr(itertools) - del sys.modules['itertools'] + assert 'queue' not in sys.modules + import queue + assert hasattr(queue, 'hello_world') + assert not hasattr(queue, 'count') + assert '(built-in)' not in repr(queue) + del sys.modules['queue'] def test_shadow_extension_2(self): if self.runappdirect: skip("hard to test: module is already imported") - # 'import itertools' is supposed to find the built-in module even + # 'import queue' is supposed to find the built-in module even # if there is also one in sys.path as long as it is *after* the - # special entry '.../lib_pypy/__extensions__'. (Note that for now - # there is one in lib_pypy/itertools.py, which should not be seen - # either; hence the (built-in) test below.) + # special entry '.../lib_pypy/__extensions__'. import sys - assert 'itertools' not in sys.modules + assert 'queue' not in sys.modules sys.path.append(sys.path.pop(0)) try: - import itertools - assert not hasattr(itertools, 'hello_world') - assert hasattr(itertools, 'izip') - assert '(built-in)' in repr(itertools) + import queue + assert not hasattr(queue, 'hello_world') + assert hasattr(queue, 'izip') + assert '(built-in)' in repr(queue) finally: sys.path.insert(0, sys.path.pop()) - del sys.modules['itertools'] + del sys.modules['queue'] class TestAbi: @@ -885,7 +892,7 @@ stream.close() # And the .pyc has been generated - cpathname = udir.join('test.pyc') + cpathname = udir.join(importing.make_compiled_pathname('test.py')) assert cpathname.check() def test_write_compiled_module(self): @@ -979,7 +986,7 @@ py.test.skip("unresolved issues with win32 shell quoting rules") from pypy.interpreter.test.test_zpy import pypypath extrapath = udir.ensure("pythonpath", dir=1) - extrapath.join("urllib.py").write("print 42\n") + extrapath.join("urllib.py").write("print(42)\n") old = os.environ.get('PYTHONPATH', None) oldlang = os.environ.pop('LANG', None) try: @@ -1024,7 +1031,7 @@ if fullname in self.namestoblock: return self def load_module(self, fullname): - raise ImportError, "blocked" + raise ImportError("blocked") import sys, imp modname = "errno" # an arbitrary harmless builtin module @@ -1094,7 +1101,7 @@ path = [self.path] try: file, filename, stuff = imp.find_module(subname, path) - except ImportError, e: + except ImportError: return None return ImpLoader(file, filename, stuff) @@ -1139,8 +1146,8 @@ def test_run_compiled_module(self): # XXX minimal test only - import imp, new - module = new.module('foobar') + import imp, types + module = types.ModuleType('foobar') raises(IOError, imp._run_compiled_module, 'foobar', 'this_file_does_not_exist', None, module) @@ -1161,14 +1168,14 @@ # a mostly-empty zip file path = os.path.join(self.udir, 'test_getimporter.zip') f = open(path, 'wb') - f.write('PK\x03\x04\n\x00\x00\x00\x00\x00P\x9eN>\x00\x00\x00\x00\x00' - '\x00\x00\x00\x00\x00\x00\x00\x05\x00\x15\x00emptyUT\t\x00' - '\x03wyYMwyYMUx\x04\x00\xf4\x01d\x00PK\x01\x02\x17\x03\n\x00' - '\x00\x00\x00\x00P\x9eN>\x00\x00\x00\x00\x00\x00\x00\x00\x00' - '\x00\x00\x00\x05\x00\r\x00\x00\x00\x00\x00\x00\x00\x00\x00' - '\xa4\x81\x00\x00\x00\x00emptyUT\x05\x00\x03wyYMUx\x00\x00PK' - '\x05\x06\x00\x00\x00\x00\x01\x00\x01\x00@\x00\x00\x008\x00' - '\x00\x00\x00\x00') + f.write(b'PK\x03\x04\n\x00\x00\x00\x00\x00P\x9eN>\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x05\x00\x15\x00emptyUT\t\x00' + b'\x03wyYMwyYMUx\x04\x00\xf4\x01d\x00PK\x01\x02\x17\x03\n\x00' + b'\x00\x00\x00\x00P\x9eN>\x00\x00\x00\x00\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x05\x00\r\x00\x00\x00\x00\x00\x00\x00\x00\x00' + b'\xa4\x81\x00\x00\x00\x00emptyUT\x05\x00\x03wyYMUx\x00\x00PK' + b'\x05\x06\x00\x00\x00\x00\x01\x00\x01\x00@\x00\x00\x008\x00' + b'\x00\x00\x00\x00') f.close() importer = imp._getimporter(path) import zipimport @@ -1193,16 +1200,16 @@ def test_import_possibly_from_pyc(self): from compiled import x if self.usepycfiles: - assert x.__file__.endswith('x.pyc') + assert x.__file__.endswith('.pyc') else: - assert x.__file__.endswith('x.py') + assert x.__file__.endswith('.py') try: from compiled import lone except ImportError: assert not self.lonepycfiles, "should have found 'lone.pyc'" else: assert self.lonepycfiles, "should not have found 'lone.pyc'" - assert lone.__file__.endswith('lone.pyc') + assert lone.__file__.endswith('.pyc') class AppTestNoLonePycFile(AppTestNoPycFile): spaceconfig = { diff --git a/pypy/module/termios/interp_termios.py b/pypy/module/termios/interp_termios.py --- a/pypy/module/termios/interp_termios.py +++ b/pypy/module/termios/interp_termios.py @@ -24,15 +24,11 @@ fd = space.c_filedescriptor_w(w_fd) w_iflag, w_oflag, w_cflag, w_lflag, w_ispeed, w_ospeed, w_cc = \ space.unpackiterable(w_attributes, expected_length=7) - w_builtin = space.getbuiltinmodule('__builtin__') cc = [] for w_c in space.unpackiterable(w_cc): if space.is_true(space.isinstance(w_c, space.w_int)): - ch = space.call_function(space.getattr(w_builtin, - space.wrap('chr')), w_c) - cc.append(space.str_w(ch)) - else: - cc.append(space.str_w(w_c)) + w_c = space.call(space.w_bytes, space.newlist([w_c])) + cc.append(space.bytes_w(w_c)) tup = (space.int_w(w_iflag), space.int_w(w_oflag), space.int_w(w_cflag), space.int_w(w_lflag), space.int_w(w_ispeed), space.int_w(w_ospeed), cc) diff --git a/pypy/module/termios/test/test_termios.py b/pypy/module/termios/test/test_termios.py --- a/pypy/module/termios/test/test_termios.py +++ b/pypy/module/termios/test/test_termios.py @@ -23,7 +23,7 @@ def _spawn(self, *args, **kwds): print 'SPAWN:', args, kwds - child = self.pexpect.spawn(*args, **kwds) + child = self.pexpect.spawn(timeout=600, *args, **kwds) child.logfile = sys.stdout return child @@ -53,7 +53,7 @@ termios.tcdrain(2) termios.tcflush(2, termios.TCIOFLUSH) termios.tcflow(2, termios.TCOON) - print 'ok!' + print('ok!') """) f = udir.join("test_tcall.py") f.write(source) @@ -65,7 +65,7 @@ import sys import termios termios.tcsetattr(sys.stdin, 1, [16640, 4, 191, 2608, 15, 15, ['\x03', '\x1c', '\x7f', '\x15', '\x04', 0, 1, '\x00', '\x11', '\x13', '\x1a', '\x00', '\x12', '\x0f', '\x17', '\x16', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00']]) - print 'ok!' + print('ok!') """) f = udir.join("test_tcsetattr.py") f.write(source) @@ -76,9 +76,9 @@ source = py.code.Source(""" import termios import fcntl - lgt = len(fcntl.ioctl(2, termios.TIOCGWINSZ, '\000'*8)) + lgt = len(fcntl.ioctl(2, termios.TIOCGWINSZ, b'\000'*8)) assert lgt == 8 - print 'ok!' + print('ok!') """) f = udir.join("test_ioctl_termios.py") f.write(source) @@ -97,7 +97,7 @@ assert len([i for i in f[-1] if isinstance(i, int)]) == 2 assert isinstance(f[-1][termios.VMIN], int) assert isinstance(f[-1][termios.VTIME], int) - print 'ok!' + print('ok!') """) f = udir.join("test_ioctl_termios.py") f.write(source) diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py --- a/pypy/objspace/fake/objspace.py +++ b/pypy/objspace/fake/objspace.py @@ -157,6 +157,10 @@ "NOT_RPYTHON" raise NotImplementedError + def wrapbytes(self, x): + assert isinstance(x, str) + return w_some_obj() + def _see_interp2app(self, interp2app): "NOT_RPYTHON" activation = interp2app._code.activation @@ -278,7 +282,7 @@ for name in (ObjSpace.ConstantTable + ObjSpace.ExceptionTable + ['int', 'str', 'float', 'long', 'tuple', 'list', - 'dict', 'unicode', 'complex', 'slice', 'bool', + 'dict', 'bytes', 'complex', 'slice', 'bool', 'type', 'text', 'object']): setattr(FakeObjSpace, 'w_' + name, w_some_obj()) # diff --git a/pypy/objspace/std/longtype.py b/pypy/objspace/std/longtype.py --- a/pypy/objspace/std/longtype.py +++ b/pypy/objspace/std/longtype.py @@ -40,6 +40,7 @@ w_obj = space.int(w_obj) else: w_obj = space.trunc(w_obj) + w_obj = space.int(w_obj) return w_obj else: base = space.int_w(w_base) From noreply at buildbot.pypy.org Tue Jan 17 13:20:59 2012 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 17 Jan 2012 13:20:59 +0100 (CET) Subject: [pypy-commit] extradoc extradoc: add a section Message-ID: <20120117122059.8DFC5820D8@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: extradoc Changeset: r4040:ebe7ed17d1d5 Date: 2012-01-17 14:20 +0200 http://bitbucket.org/pypy/extradoc/changeset/ebe7ed17d1d5/ Log: add a section diff --git a/planning/micronumpy.txt b/planning/micronumpy.txt --- a/planning/micronumpy.txt +++ b/planning/micronumpy.txt @@ -24,3 +24,10 @@ * keep subclass type when slicing, __array_finalize__ * ndarray.view + +OPTIMIZATIONS +------------- + +- SSE + +- count number of True's for bool arrays, so we don't have to recompute From noreply at buildbot.pypy.org Tue Jan 17 13:28:44 2012 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 17 Jan 2012 13:28:44 +0100 (CET) Subject: [pypy-commit] pypy numpy-indexing-by-arrays-2: fix setitem with bool index Message-ID: <20120117122844.5CDF9820D8@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: numpy-indexing-by-arrays-2 Changeset: r51395:6fe5770303bb Date: 2012-01-17 14:28 +0200 http://bitbucket.org/pypy/pypy/changeset/6fe5770303bb/ Log: fix setitem with bool index diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -537,16 +537,18 @@ return res def setitem_filter(self, space, idx, val): - arr = SliceArray(self.shape, self.dtype, self, val) - shapelen = len(arr.shape) + size = self.count_all_true(idx) + arr = SliceArray([size], self.dtype, self, val) sig = arr.find_sig() + shapelen = len(self.shape) frame = sig.create_frame(arr) idxi = idx.create_iter() while not frame.done(): if idx.dtype.getitem_bool(idx.storage, idxi.offset): sig.eval(frame, arr) + frame.next_from_second(1) + frame.next_first(shapelen) idxi = idxi.next(shapelen) - frame.next(shapelen) def descr_getitem(self, space, w_idx): if (isinstance(w_idx, BaseArray) and w_idx.shape == self.shape and diff --git a/pypy/module/micronumpy/signature.py b/pypy/module/micronumpy/signature.py --- a/pypy/module/micronumpy/signature.py +++ b/pypy/module/micronumpy/signature.py @@ -82,6 +82,16 @@ for i in range(len(self.iterators)): self.iterators[i] = self.iterators[i].next(shapelen) + @unroll_safe + def next_from_second(self, shapelen): + """ Don't increase the first iterator + """ + for i in range(1, len(self.iterators)): + self.iterators[i] = self.iterators[i].next(shapelen) + + def next_first(self, shapelen): + self.iterators[0] = self.iterators[0].next(shapelen) + def get_final_iter(self): final_iter = promote(self.final_iter) if final_iter < 0: diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -1327,7 +1327,7 @@ assert (a == [0, 1, 2, 3, 15, 15]).all() a = arange(6).reshape(3, 2) a[a & 1 == 1] = array([8, 9, 10]) - assert (a == [[0, 8], [3, 9], [5, 10]]).all() + assert (a == [[0, 8], [2, 9], [4, 10]]).all() class AppTestSupport(BaseNumpyAppTest): def setup_class(cls): From noreply at buildbot.pypy.org Tue Jan 17 13:32:02 2012 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 17 Jan 2012 13:32:02 +0100 (CET) Subject: [pypy-commit] pypy numpy-indexing-by-arrays-2: introduce a jit driver, although it's a bit pointless right now Message-ID: <20120117123202.DA524820D8@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: numpy-indexing-by-arrays-2 Changeset: r51396:b48aeb33bb41 Date: 2012-01-17 14:31 +0200 http://bitbucket.org/pypy/pypy/changeset/b48aeb33bb41/ Log: introduce a jit driver, although it's a bit pointless right now diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -52,6 +52,12 @@ reds=['concr', 'argi', 'ri', 'frame', 'v', 'res', 'self'], name='numpy_filter', ) +filter_set_driver = jit.JitDriver( + greens=['shapelen', 'sig'], + virtualizables=['frame'], + reds=['idx', 'idxi', 'frame', 'arr'], + name='numpy_filterset', +) def _find_shape_and_elems(space, w_iterable): shape = [space.len_w(w_iterable)] @@ -544,6 +550,9 @@ frame = sig.create_frame(arr) idxi = idx.create_iter() while not frame.done(): + filter_set_driver.jit_merge_point(idx=idx, idxi=idxi, sig=sig, + frame=frame, arr=arr, + shapelen=shapelen) if idx.dtype.getitem_bool(idx.storage, idxi.offset): sig.eval(frame, arr) frame.next_from_second(1) From noreply at buildbot.pypy.org Tue Jan 17 13:33:31 2012 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 17 Jan 2012 13:33:31 +0100 (CET) Subject: [pypy-commit] extradoc extradoc: add two more items Message-ID: <20120117123331.16CC7820D8@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: extradoc Changeset: r4041:daaef6427a03 Date: 2012-01-17 14:33 +0200 http://bitbucket.org/pypy/extradoc/changeset/daaef6427a03/ Log: add two more items diff --git a/planning/micronumpy.txt b/planning/micronumpy.txt --- a/planning/micronumpy.txt +++ b/planning/micronumpy.txt @@ -31,3 +31,8 @@ - SSE - count number of True's for bool arrays, so we don't have to recompute + +- bridges are a tad too complicated - we need to store optimizer info + in order to improve though + +- iterators should possibly not require reinstantiating them. From noreply at buildbot.pypy.org Tue Jan 17 13:48:46 2012 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 17 Jan 2012 13:48:46 +0100 (CET) Subject: [pypy-commit] pypy default: (fijal, cfbolz) improve error reporting Message-ID: <20120117124846.D6538820D8@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r51397:865005356c2f Date: 2012-01-17 14:48 +0200 http://bitbucket.org/pypy/pypy/changeset/865005356c2f/ Log: (fijal, cfbolz) improve error reporting diff --git a/pypy/objspace/std/test/test_unicodeobject.py b/pypy/objspace/std/test/test_unicodeobject.py --- a/pypy/objspace/std/test/test_unicodeobject.py +++ b/pypy/objspace/std/test/test_unicodeobject.py @@ -64,6 +64,12 @@ check(', '.join([u'a']), u'a') check(', '.join(['a', u'b']), u'a, b') check(u', '.join(['a', 'b']), u'a, b') + try: + u''.join([u'a', 2, 3]) + except TypeError, e: + assert 'sequence item 1' in str(e) + else: + raise Exception("DID NOT RAISE") if sys.version_info >= (2,3): def test_contains_ex(self): diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -218,19 +218,19 @@ self = w_self._value prealloc_size = len(self) * (size - 1) for i in range(size): - prealloc_size += len(space.unicode_w(list_w[i])) + try: + prealloc_size += len(space.unicode_w(list_w[i])) + except OperationError, e: + if not e.match(space, space.w_TypeError): + raise + raise operationerrfmt(space.w_TypeError, + "sequence item %d: expected string or Unicode", i) sb = UnicodeBuilder(prealloc_size) for i in range(size): if self and i != 0: sb.append(self) w_s = list_w[i] - try: - sb.append(space.unicode_w(w_s)) - except OperationError, e: - if not e.match(space, space.w_TypeError): - raise - raise operationerrfmt(space.w_TypeError, - "sequence item %d: expected string or Unicode", i) + sb.append(space.unicode_w(w_s)) return space.wrap(sb.build()) def hash__Unicode(space, w_uni): From noreply at buildbot.pypy.org Tue Jan 17 14:52:50 2012 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 17 Jan 2012 14:52:50 +0100 (CET) Subject: [pypy-commit] pypy stm: (antocuni, arigo) Message-ID: <20120117135250.CEFF4820D8@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stm Changeset: r51398:29e9345db0e6 Date: 2012-01-17 11:57 +0100 http://bitbucket.org/pypy/pypy/changeset/29e9345db0e6/ Log: (antocuni, arigo) Test and fix: handle the case of stm_{get,set}field() called after descriptor_init() but outside a transaction. diff --git a/pypy/translator/stm/src_stm/et.c b/pypy/translator/stm/src_stm/et.c --- a/pypy/translator/stm/src_stm/et.c +++ b/pypy/translator/stm/src_stm/et.c @@ -80,15 +80,19 @@ #endif unsigned int spinloop_counter; owner_version_t my_lock_word; - unsigned init_counter; struct RedoLog redolog; /* last item, because it's the biggest one */ int transaction_active; }; +static const struct tx_descriptor null_tx = { + .transaction_active = 0 +}; +#define NULL_TX ((struct tx_descriptor *)(&null_tx)) + /* global_timestamp contains in its lowest bit a flag equal to 1 if there is an inevitable transaction running */ static volatile unsigned long global_timestamp = 2; -static __thread struct tx_descriptor *thread_descriptor = NULL; +static __thread struct tx_descriptor *thread_descriptor = NULL_TX; #ifdef COMMIT_OTHER_INEV static struct tx_descriptor *volatile thread_descriptor_inev; static volatile unsigned long d_inev_checking = 0; @@ -509,11 +513,8 @@ long stm_read_word(long* addr) { struct tx_descriptor *d = thread_descriptor; - if (!d) + if (!d->transaction_active) return *addr; -#ifdef RPY_STM_ASSERT - assert(d->transaction_active); -#endif // check writeset first wlog_t* found; @@ -575,23 +576,18 @@ void stm_write_word(long* addr, long val) { struct tx_descriptor *d = thread_descriptor; - if (!d) - { - *addr = val; - return; - } -#ifdef RPY_STM_ASSERT - assert(d->transaction_active); -#endif + if (!d->transaction_active) { + *addr = val; + return; + } redolog_insert(&d->redolog, addr, val); } void stm_descriptor_init(void) { - if (thread_descriptor != NULL) - thread_descriptor->init_counter++; - else + assert(thread_descriptor == NULL_TX); + if (1) /* for hg diff */ { struct tx_descriptor *d = malloc(sizeof(struct tx_descriptor)); memset(d, 0, sizeof(struct tx_descriptor)); @@ -606,7 +602,6 @@ d->my_lock_word = ~d->my_lock_word; assert(IS_LOCKED(d->my_lock_word)); d->spinloop_counter = (unsigned int)(d->my_lock_word | 1); - d->init_counter = 1; thread_descriptor = d; @@ -621,11 +616,9 @@ void stm_descriptor_done(void) { struct tx_descriptor *d = thread_descriptor; - d->init_counter--; - if (d->init_counter > 0) - return; + assert(d != NULL_TX); - thread_descriptor = NULL; + thread_descriptor = NULL_TX; #ifdef RPY_STM_DEBUG_PRINT PYPY_DEBUG_START("stm-done"); @@ -671,7 +664,7 @@ { void *result; /* you need to call descriptor_init() before calling stm_perform_transaction */ - assert(thread_descriptor != NULL); + assert(thread_descriptor != NULL_TX); STM_begin_transaction(); result = callback(arg); stm_commit_transaction(); @@ -767,7 +760,7 @@ by another thread. We set the lowest bit in global_timestamp to 1. */ struct tx_descriptor *d = thread_descriptor; - if (!d) + if (!d->transaction_active) return; #ifdef RPY_STM_ASSERT @@ -965,7 +958,7 @@ long stm_debug_get_state(void) { struct tx_descriptor *d = thread_descriptor; - if (!d) + if (d == NULL_TX) return -1; if (!d->transaction_active) return 0; diff --git a/pypy/translator/stm/test/test_funcgen.py b/pypy/translator/stm/test/test_funcgen.py --- a/pypy/translator/stm/test/test_funcgen.py +++ b/pypy/translator/stm/test/test_funcgen.py @@ -257,6 +257,16 @@ t, cbuilder = self.compile(do_stm_getfield) cbuilder.cmdexec('') + def test_getfield_all_sizes_outside_transaction(self): + def do_stm_getfield(argv): + stm_descriptor_init() + # we have a descriptor, but we don't call it in a transaction + _play_with_getfield(None) + stm_descriptor_done() + return 0 + t, cbuilder = self.compile(do_stm_getfield) + cbuilder.cmdexec('') + def test_setfield_all_sizes(self): def do_stm_setfield(argv): _play_with_setfields(None) @@ -277,6 +287,15 @@ t, cbuilder = self.compile(do_stm_setfield) cbuilder.cmdexec('') + def test_setfield_all_sizes_outside_transaction(self): + def do_stm_setfield(argv): + stm_descriptor_init() + _play_with_setfields(None) + stm_descriptor_done() + return 0 + t, cbuilder = self.compile(do_stm_setfield) + cbuilder.cmdexec('') + def test_getarrayitem_all_sizes(self): def do_stm_getarrayitem(argv): _play_with_getarrayitem(None) From noreply at buildbot.pypy.org Tue Jan 17 14:52:52 2012 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 17 Jan 2012 14:52:52 +0100 (CET) Subject: [pypy-commit] pypy stm: (antocuni, arigo) Message-ID: <20120117135252.033EA82110@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stm Changeset: r51399:4ef670f3a925 Date: 2012-01-17 12:39 +0100 http://bitbucket.org/pypy/pypy/changeset/4ef670f3a925/ Log: (antocuni, arigo) Refactored targetdemo.py to use the new interface. Allow debug_start/debug_print/debug_stop to work (as long as you don't say PYPYLOG=..:file) by making the flag thread-local. diff --git a/pypy/translator/c/src/debug_print.c b/pypy/translator/c/src/debug_print.c --- a/pypy/translator/c/src/debug_print.c +++ b/pypy/translator/c/src/debug_print.c @@ -15,8 +15,8 @@ #include "src/profiling.h" #include "src/debug_print.h" -long pypy_have_debug_prints = -1; -FILE *pypy_debug_file = NULL; +__thread long pypy_have_debug_prints = -1; +FILE *pypy_debug_file = NULL; /* XXX make it thread-local too? */ static unsigned char debug_ready = 0; static unsigned char debug_profile = 0; static char *debug_start_colors_1 = ""; diff --git a/pypy/translator/c/src/debug_print.h b/pypy/translator/c/src/debug_print.h --- a/pypy/translator/c/src/debug_print.h +++ b/pypy/translator/c/src/debug_print.h @@ -38,7 +38,7 @@ void pypy_debug_stop(const char *category); long pypy_debug_offset(void); -extern long pypy_have_debug_prints; +extern __thread long pypy_have_debug_prints; extern FILE *pypy_debug_file; #define OP_LL_READ_TIMESTAMP(val) READ_TIMESTAMP(val) diff --git a/pypy/translator/stm/test/targetdemo.py b/pypy/translator/stm/test/targetdemo.py --- a/pypy/translator/stm/test/targetdemo.py +++ b/pypy/translator/stm/test/targetdemo.py @@ -1,6 +1,7 @@ import time from pypy.module.thread import ll_thread -#from pypy.translator.stm import rstm +from pypy.rlib import rstm +from pypy.rlib.debug import debug_print class Node: @@ -15,8 +16,13 @@ anchor = Node(-1) glob = Global() +class Arg: + _alloc_nonmovable_ = True -def add_at_end_of_chained_list(node, value): + +def add_at_end_of_chained_list(arg): + node = arg.anchor + value = arg.value x = Node(value) while node.next: node = node.next @@ -46,14 +52,23 @@ print "check ok!" -def run_me(): - print "thread starting..." - for i in range(glob.LENGTH): - add_at_end_of_chained_list(glob.anchor, i) - rstm.transaction_boundary() +def increment_done(arg): print "thread done." glob.done += 1 +def run_me(): + debug_print("thread starting...") + arg = Arg() + rstm.descriptor_init() + try: + for i in range(glob.LENGTH): + arg.anchor = glob.anchor + arg.value = i + rstm.perform_transaction(add_at_end_of_chained_list, Arg, arg) + rstm.perform_transaction(increment_done, Arg, arg) + finally: + rstm.descriptor_done() + # __________ Entry point __________ From noreply at buildbot.pypy.org Tue Jan 17 14:52:53 2012 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 17 Jan 2012 14:52:53 +0100 (CET) Subject: [pypy-commit] pypy default: Use default=False, and enable it only in -O2/O3/Ojit, like the Message-ID: <20120117135253.29A5382C03@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r51400:3d6e00235d87 Date: 2011-11-05 19:36 +0100 http://bitbucket.org/pypy/pypy/changeset/3d6e00235d87/ Log: Use default=False, and enable it only in -O2/O3/Ojit, like the other optimizations. Fixes an issue if weakrefs are disabled. (transplanted from 5eb3fc96c89290e270737f0652388f29154d3b2c) diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py --- a/pypy/config/pypyoption.py +++ b/pypy/config/pypyoption.py @@ -340,7 +340,7 @@ requires=[("objspace.std.builtinshortcut", True)]), BoolOption("withidentitydict", "track types that override __hash__, __eq__ or __cmp__ and use a special dict strategy for those which do not", - default=True), + default=False), ]), ]) @@ -370,6 +370,7 @@ config.objspace.std.suggest(getattributeshortcut=True) config.objspace.std.suggest(newshortcut=True) config.objspace.std.suggest(withspecialisedtuple=True) + config.objspace.std.suggest(withidentitydict=True) #if not IS_64_BITS: # config.objspace.std.suggest(withsmalllong=True) From noreply at buildbot.pypy.org Tue Jan 17 14:52:54 2012 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 17 Jan 2012 14:52:54 +0100 (CET) Subject: [pypy-commit] pypy default: merge heads Message-ID: <20120117135254.6308382CF4@wyvern.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r51401:f34f0c11299f Date: 2012-01-17 14:52 +0100 http://bitbucket.org/pypy/pypy/changeset/f34f0c11299f/ Log: merge heads diff --git a/.hgignore b/.hgignore --- a/.hgignore +++ b/.hgignore @@ -2,6 +2,9 @@ *.py[co] *~ .*.swp +.idea +.project +.pydevproject syntax: regexp ^testresult$ diff --git a/pypy/objspace/std/test/test_unicodeobject.py b/pypy/objspace/std/test/test_unicodeobject.py --- a/pypy/objspace/std/test/test_unicodeobject.py +++ b/pypy/objspace/std/test/test_unicodeobject.py @@ -64,6 +64,12 @@ check(', '.join([u'a']), u'a') check(', '.join(['a', u'b']), u'a, b') check(u', '.join(['a', 'b']), u'a, b') + try: + u''.join([u'a', 2, 3]) + except TypeError, e: + assert 'sequence item 1' in str(e) + else: + raise Exception("DID NOT RAISE") if sys.version_info >= (2,3): def test_contains_ex(self): diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -216,21 +216,21 @@ def _unicode_join_many_items(space, w_self, list_w, size): self = w_self._value - prealloc_size = 0 + prealloc_size = len(self) * (size - 1) for i in range(size): - prealloc_size += len(space.unicode_w(list_w[i])) + try: + prealloc_size += len(space.unicode_w(list_w[i])) + except OperationError, e: + if not e.match(space, space.w_TypeError): + raise + raise operationerrfmt(space.w_TypeError, + "sequence item %d: expected string or Unicode", i) sb = UnicodeBuilder(prealloc_size) for i in range(size): if self and i != 0: sb.append(self) w_s = list_w[i] - try: - sb.append(space.unicode_w(w_s)) - except OperationError, e: - if not e.match(space, space.w_TypeError): - raise - raise operationerrfmt(space.w_TypeError, - "sequence item %d: expected string or Unicode", i) + sb.append(space.unicode_w(w_s)) return space.wrap(sb.build()) def hash__Unicode(space, w_uni): diff --git a/pypy/rlib/longlong2float.py b/pypy/rlib/longlong2float.py --- a/pypy/rlib/longlong2float.py +++ b/pypy/rlib/longlong2float.py @@ -79,19 +79,23 @@ longlong2float = rffi.llexternal( "pypy__longlong2float", [rffi.LONGLONG], rffi.DOUBLE, _callable=longlong2float_emulator, compilation_info=eci, - _nowrapper=True, elidable_function=True, sandboxsafe=True) + _nowrapper=True, elidable_function=True, sandboxsafe=True, + oo_primitive="pypy__longlong2float") float2longlong = rffi.llexternal( "pypy__float2longlong", [rffi.DOUBLE], rffi.LONGLONG, _callable=float2longlong_emulator, compilation_info=eci, - _nowrapper=True, elidable_function=True, sandboxsafe=True) + _nowrapper=True, elidable_function=True, sandboxsafe=True, + oo_primitive="pypy__float2longlong") uint2singlefloat = rffi.llexternal( "pypy__uint2singlefloat", [rffi.UINT], rffi.FLOAT, _callable=uint2singlefloat_emulator, compilation_info=eci, - _nowrapper=True, elidable_function=True, sandboxsafe=True) + _nowrapper=True, elidable_function=True, sandboxsafe=True, + oo_primitive="pypy__uint2singlefloat") singlefloat2uint = rffi.llexternal( "pypy__singlefloat2uint", [rffi.FLOAT], rffi.UINT, _callable=singlefloat2uint_emulator, compilation_info=eci, - _nowrapper=True, elidable_function=True, sandboxsafe=True) + _nowrapper=True, elidable_function=True, sandboxsafe=True, + oo_primitive="pypy__singlefloat2uint") diff --git a/pypy/rlib/objectmodel.py b/pypy/rlib/objectmodel.py --- a/pypy/rlib/objectmodel.py +++ b/pypy/rlib/objectmodel.py @@ -420,7 +420,7 @@ vobj.concretetype.TO._gckind == 'gc') else: from pypy.rpython.ootypesystem import ootype - ok = isinstance(vobj.concretetype, ootype.Instance) + ok = isinstance(vobj.concretetype, (ootype.Instance, ootype.BuiltinType)) if not ok: from pypy.rpython.error import TyperError raise TyperError("compute_unique_id() cannot be applied to" diff --git a/pypy/rpython/ootypesystem/ootype.py b/pypy/rpython/ootypesystem/ootype.py --- a/pypy/rpython/ootypesystem/ootype.py +++ b/pypy/rpython/ootypesystem/ootype.py @@ -512,6 +512,7 @@ "ll_append_char": Meth([CHARTP], Void), "ll_append": Meth([STRINGTP], Void), "ll_build": Meth([], STRINGTP), + "ll_getlength": Meth([], Signed), }) self._setup_methods({}) @@ -1376,6 +1377,9 @@ def _cast_to_object(self): return make_object(self) + def _identityhash(self): + return object.__hash__(self) + class _string(_builtin_type): def __init__(self, STRING, value = ''): @@ -1543,6 +1547,9 @@ else: return make_unicode(u''.join(self._buf)) + def ll_getlength(self): + return self.ll_build().ll_strlen() + class _null_string_builder(_null_mixin(_string_builder), _string_builder): def __init__(self, STRING_BUILDER): self.__dict__["_TYPE"] = STRING_BUILDER diff --git a/pypy/rpython/ootypesystem/rbuilder.py b/pypy/rpython/ootypesystem/rbuilder.py --- a/pypy/rpython/ootypesystem/rbuilder.py +++ b/pypy/rpython/ootypesystem/rbuilder.py @@ -21,6 +21,10 @@ builder.ll_append_char(char) @staticmethod + def ll_getlength(builder): + return builder.ll_getlength() + + @staticmethod def ll_append(builder, string): builder.ll_append(string) diff --git a/pypy/rpython/test/test_rbuilder.py b/pypy/rpython/test/test_rbuilder.py --- a/pypy/rpython/test/test_rbuilder.py +++ b/pypy/rpython/test/test_rbuilder.py @@ -124,9 +124,5 @@ pass class TestOOtype(BaseTestStringBuilder, OORtypeMixin): - def test_string_getlength(self): - py.test.skip("getlength(): not implemented on ootype") - def test_unicode_getlength(self): - py.test.skip("getlength(): not implemented on ootype") def test_append_charpsize(self): py.test.skip("append_charpsize(): not implemented on ootype") diff --git a/pypy/rpython/test/test_rbuiltin.py b/pypy/rpython/test/test_rbuiltin.py --- a/pypy/rpython/test/test_rbuiltin.py +++ b/pypy/rpython/test/test_rbuiltin.py @@ -463,6 +463,31 @@ assert x1 == intmask(x0) assert x3 == intmask(x2) + def test_id_on_builtins(self): + from pypy.rlib.objectmodel import compute_unique_id + from pypy.rlib.rstring import StringBuilder, UnicodeBuilder + def fn(): + return (compute_unique_id("foo"), + compute_unique_id(u"bar"), + compute_unique_id([1]), + compute_unique_id({"foo": 3}), + compute_unique_id(StringBuilder()), + compute_unique_id(UnicodeBuilder())) + res = self.interpret(fn, []) + for id in self.ll_unpack_tuple(res, 6): + assert isinstance(id, (int, r_longlong)) + + def test_uniqueness_of_id_on_strings(self): + from pypy.rlib.objectmodel import compute_unique_id + def fn(s1, s2): + return (compute_unique_id(s1), compute_unique_id(s2)) + + s1 = "foo" + s2 = ''.join(['f','oo']) + res = self.interpret(fn, [self.string_to_ll(s1), self.string_to_ll(s2)]) + i1, i2 = self.ll_unpack_tuple(res, 2) + assert i1 != i2 + def test_cast_primitive(self): from pypy.rpython.annlowlevel import LowLevelAnnotatorPolicy def llf(u): diff --git a/pypy/translator/jvm/builtin.py b/pypy/translator/jvm/builtin.py --- a/pypy/translator/jvm/builtin.py +++ b/pypy/translator/jvm/builtin.py @@ -84,6 +84,9 @@ (ootype.StringBuilder.__class__, "ll_build"): jvm.Method.v(jStringBuilder, "toString", (), jString), + (ootype.StringBuilder.__class__, "ll_getlength"): + jvm.Method.v(jStringBuilder, "length", (), jInt), + (ootype.String.__class__, "ll_hash"): jvm.Method.v(jString, "hashCode", (), jInt), diff --git a/pypy/translator/jvm/database.py b/pypy/translator/jvm/database.py --- a/pypy/translator/jvm/database.py +++ b/pypy/translator/jvm/database.py @@ -358,7 +358,7 @@ ootype.Unsigned:jvm.PYPYSERIALIZEUINT, ootype.SignedLongLong:jvm.LONGTOSTRINGL, ootype.UnsignedLongLong: jvm.PYPYSERIALIZEULONG, - ootype.Float:jvm.DOUBLETOSTRINGD, + ootype.Float:jvm.PYPYSERIALIZEDOUBLE, ootype.Bool:jvm.PYPYSERIALIZEBOOLEAN, ootype.Void:jvm.PYPYSERIALIZEVOID, ootype.Char:jvm.PYPYESCAPEDCHAR, diff --git a/pypy/translator/jvm/metavm.py b/pypy/translator/jvm/metavm.py --- a/pypy/translator/jvm/metavm.py +++ b/pypy/translator/jvm/metavm.py @@ -92,6 +92,7 @@ CASTS = { # FROM TO (ootype.Signed, ootype.UnsignedLongLong): jvm.I2L, + (ootype.Unsigned, ootype.UnsignedLongLong): jvm.I2L, (ootype.SignedLongLong, ootype.Signed): jvm.L2I, (ootype.UnsignedLongLong, ootype.Unsigned): jvm.L2I, (ootype.UnsignedLongLong, ootype.Signed): jvm.L2I, diff --git a/pypy/translator/jvm/opcodes.py b/pypy/translator/jvm/opcodes.py --- a/pypy/translator/jvm/opcodes.py +++ b/pypy/translator/jvm/opcodes.py @@ -101,6 +101,7 @@ 'jit_force_virtualizable': Ignore, 'jit_force_virtual': DoNothing, 'jit_force_quasi_immutable': Ignore, + 'jit_is_virtual': PushPrimitive(ootype.Bool, False), 'debug_assert': [], # TODO: implement? 'debug_start_traceback': Ignore, diff --git a/pypy/translator/jvm/src/pypy/PyPy.java b/pypy/translator/jvm/src/pypy/PyPy.java --- a/pypy/translator/jvm/src/pypy/PyPy.java +++ b/pypy/translator/jvm/src/pypy/PyPy.java @@ -283,6 +283,14 @@ } } + public double pypy__longlong2float(long l) { + return Double.longBitsToDouble(l); + } + + public long pypy__float2longlong(double d) { + return Double.doubleToRawLongBits(d); + } + public double ooparse_float(String s) { try { return Double.parseDouble(s); @@ -353,6 +361,19 @@ return "False"; } + public static String serialize_double(double d) { + if (Double.isNaN(d)) { + return "float(\"nan\")"; + } else if (Double.isInfinite(d)) { + if (d > 0) + return "float(\"inf\")"; + else + return "float(\"-inf\")"; + } else { + return Double.toString(d); + } + } + private static String format_char(char c) { String res = "\\x"; if (c <= 0x0F) res = res + "0"; diff --git a/pypy/translator/jvm/test/runtest.py b/pypy/translator/jvm/test/runtest.py --- a/pypy/translator/jvm/test/runtest.py +++ b/pypy/translator/jvm/test/runtest.py @@ -56,6 +56,7 @@ # CLI could-be duplicate class JvmGeneratedSourceWrapper(object): + def __init__(self, gensrc): """ gensrc is an instance of JvmGeneratedSource """ self.gensrc = gensrc diff --git a/pypy/translator/jvm/test/test_builder.py b/pypy/translator/jvm/test/test_builder.py new file mode 100644 --- /dev/null +++ b/pypy/translator/jvm/test/test_builder.py @@ -0,0 +1,7 @@ +from pypy.translator.jvm.test.runtest import JvmTest +from pypy.rpython.test.test_rbuilder import BaseTestStringBuilder +import py + +class TestJvmStringBuilder(JvmTest, BaseTestStringBuilder): + def test_append_charpsize(self): + py.test.skip("append_charpsize(): not implemented on ootype") diff --git a/pypy/translator/jvm/test/test_longlong2float.py b/pypy/translator/jvm/test/test_longlong2float.py new file mode 100644 --- /dev/null +++ b/pypy/translator/jvm/test/test_longlong2float.py @@ -0,0 +1,20 @@ +from pypy.translator.jvm.test.runtest import JvmTest +from pypy.rlib.longlong2float import * +from pypy.rlib.test.test_longlong2float import enum_floats +from pypy.rlib.test.test_longlong2float import fn as float2longlong2float +import py + +class TestLongLong2Float(JvmTest): + + def test_float2longlong_and_longlong2float(self): + def func(f): + return float2longlong2float(f) + + for f in enum_floats(): + assert repr(f) == repr(self.interpret(func, [f])) + + def test_uint2singlefloat(self): + py.test.skip("uint2singlefloat is not implemented in ootype") + + def test_singlefloat2uint(self): + py.test.skip("singlefloat2uint is not implemented in ootype") diff --git a/pypy/translator/jvm/typesystem.py b/pypy/translator/jvm/typesystem.py --- a/pypy/translator/jvm/typesystem.py +++ b/pypy/translator/jvm/typesystem.py @@ -955,6 +955,7 @@ PYPYSERIALIZEUINT = Method.s(jPyPy, 'serialize_uint', (jInt,), jString) PYPYSERIALIZEULONG = Method.s(jPyPy, 'serialize_ulonglong', (jLong,),jString) PYPYSERIALIZEVOID = Method.s(jPyPy, 'serialize_void', (), jString) +PYPYSERIALIZEDOUBLE = Method.s(jPyPy, 'serialize_double', (jDouble,), jString) PYPYESCAPEDCHAR = Method.s(jPyPy, 'escaped_char', (jChar,), jString) PYPYESCAPEDUNICHAR = Method.s(jPyPy, 'escaped_unichar', (jChar,), jString) PYPYESCAPEDSTRING = Method.s(jPyPy, 'escaped_string', (jString,), jString) diff --git a/pypy/translator/oosupport/test_template/cast.py b/pypy/translator/oosupport/test_template/cast.py --- a/pypy/translator/oosupport/test_template/cast.py +++ b/pypy/translator/oosupport/test_template/cast.py @@ -13,6 +13,9 @@ def to_longlong(x): return r_longlong(x) +def to_ulonglong(x): + return r_ulonglong(x) + def uint_to_int(x): return intmask(x) @@ -56,6 +59,9 @@ def test_unsignedlonglong_to_unsigned4(self): self.check(to_uint, [r_ulonglong(18446744073709551615l)]) # max 64 bit num + def test_unsigned_to_usignedlonglong(self): + self.check(to_ulonglong, [r_uint(42)]) + def test_uint_to_int(self): self.check(uint_to_int, [r_uint(sys.maxint+1)]) From noreply at buildbot.pypy.org Tue Jan 17 16:03:11 2012 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 17 Jan 2012 16:03:11 +0100 (CET) Subject: [pypy-commit] pypy release-1.8.x: Create a release branch Message-ID: <20120117150311.9E208820D8@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: release-1.8.x Changeset: r51402:2b32f9c1533d Date: 2012-01-17 17:02 +0200 http://bitbucket.org/pypy/pypy/changeset/2b32f9c1533d/ Log: Create a release branch diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py --- a/pypy/module/sys/version.py +++ b/pypy/module/sys/version.py @@ -10,7 +10,7 @@ CPYTHON_VERSION = (2, 7, 1, "final", 42) #XXX # sync patchlevel.h CPYTHON_API_VERSION = 1013 #XXX # sync with include/modsupport.h -PYPY_VERSION = (1, 7, 1, "dev", 0) #XXX # sync patchlevel.h +PYPY_VERSION = (1, 8, 0, "final", 0) #XXX # sync patchlevel.h if platform.name == 'msvc': COMPILER_INFO = 'MSC v.%d 32 bit' % (platform.version * 10 + 600) From noreply at buildbot.pypy.org Tue Jan 17 16:04:43 2012 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 17 Jan 2012 16:04:43 +0100 (CET) Subject: [pypy-commit] pypy default: mark patchlevel as well Message-ID: <20120117150443.D8473820D8@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r51403:a4261375b359 Date: 2012-01-17 17:03 +0200 http://bitbucket.org/pypy/pypy/changeset/a4261375b359/ Log: mark patchlevel as well diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h --- a/pypy/module/cpyext/include/patchlevel.h +++ b/pypy/module/cpyext/include/patchlevel.h @@ -29,7 +29,7 @@ #define PY_VERSION "2.7.1" /* PyPy version as a string */ -#define PYPY_VERSION "1.7.1" +#define PYPY_VERSION "1.8.0" /* Subversion Revision number of this file (not of the repository). * Empty since Mercurial migration. */ From noreply at buildbot.pypy.org Tue Jan 17 16:04:45 2012 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 17 Jan 2012 16:04:45 +0100 (CET) Subject: [pypy-commit] pypy default: update it here as well Message-ID: <20120117150445.0BC30820D8@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r51404:96334e1f84f3 Date: 2012-01-17 17:04 +0200 http://bitbucket.org/pypy/pypy/changeset/96334e1f84f3/ Log: update it here as well diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h --- a/pypy/module/cpyext/include/patchlevel.h +++ b/pypy/module/cpyext/include/patchlevel.h @@ -29,7 +29,7 @@ #define PY_VERSION "2.7.1" /* PyPy version as a string */ -#define PYPY_VERSION "1.8.0" +#define PYPY_VERSION "1.8.1" /* Subversion Revision number of this file (not of the repository). * Empty since Mercurial migration. */ diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py --- a/pypy/module/sys/version.py +++ b/pypy/module/sys/version.py @@ -10,7 +10,7 @@ CPYTHON_VERSION = (2, 7, 1, "final", 42) #XXX # sync patchlevel.h CPYTHON_API_VERSION = 1013 #XXX # sync with include/modsupport.h -PYPY_VERSION = (1, 7, 1, "dev", 0) #XXX # sync patchlevel.h +PYPY_VERSION = (1, 8, 1, "dev", 0) #XXX # sync patchlevel.h if platform.name == 'msvc': COMPILER_INFO = 'MSC v.%d 32 bit' % (platform.version * 10 + 600) From noreply at buildbot.pypy.org Tue Jan 17 16:22:17 2012 From: noreply at buildbot.pypy.org (rguillebert) Date: Tue, 17 Jan 2012 16:22:17 +0100 (CET) Subject: [pypy-commit] pypy py3k: (antocuni,romain) Adapt the test to py3k Message-ID: <20120117152217.68F60820D8@wyvern.cs.uni-duesseldorf.de> Author: Romain Guillebert Branch: py3k Changeset: r51405:f3ca10db7f5e Date: 2012-01-17 14:44 +0100 http://bitbucket.org/pypy/pypy/changeset/f3ca10db7f5e/ Log: (antocuni,romain) Adapt the test to py3k diff --git a/pypy/interpreter/pyparser/test/test_pyparse.py b/pypy/interpreter/pyparser/test/test_pyparse.py --- a/pypy/interpreter/pyparser/test/test_pyparse.py +++ b/pypy/interpreter/pyparser/test/test_pyparse.py @@ -38,7 +38,7 @@ """, info=info) assert tree.type == syms.file_input assert info.encoding == "iso-8859-1" - sentence = u"u'Die Männer ärgen sich!'" + sentence = u"'Die Männer ärgen sich!'" input = (u"# coding: utf-7\nstuff = %s" % (sentence,)).encode("utf-7") tree = self.parse(input, info=info) assert info.encoding == "utf-7" From noreply at buildbot.pypy.org Tue Jan 17 16:22:18 2012 From: noreply at buildbot.pypy.org (rguillebert) Date: Tue, 17 Jan 2012 16:22:18 +0100 (CET) Subject: [pypy-commit] pypy py3k: (antocuni, romain) started to parse the new tuple unpacking (only the case with no parenthesis is implemented so far) Message-ID: <20120117152218.94201820D8@wyvern.cs.uni-duesseldorf.de> Author: Romain Guillebert Branch: py3k Changeset: r51406:5dd563b2360f Date: 2012-01-17 16:21 +0100 http://bitbucket.org/pypy/pypy/changeset/5dd563b2360f/ Log: (antocuni, romain) started to parse the new tuple unpacking (only the case with no parenthesis is implemented so far) diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py --- a/pypy/interpreter/astcompiler/astbuilder.py +++ b/pypy/interpreter/astcompiler/astbuilder.py @@ -680,7 +680,7 @@ self.set_context(target_expr, ast.Store) targets.append(target_expr) value_child = stmt.children[-1] - if value_child.type == syms.testlist: + if value_child.type == syms.testlist or value_child.type == syms.testlist_star_expr: value_expr = self.handle_testlist(value_child) else: value_expr = self.handle_expr(value_child) diff --git a/pypy/interpreter/pyparser/data/Grammar3.2 b/pypy/interpreter/pyparser/data/Grammar3.2 --- a/pypy/interpreter/pyparser/data/Grammar3.2 +++ b/pypy/interpreter/pyparser/data/Grammar3.2 @@ -37,8 +37,9 @@ simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE small_stmt: (expr_stmt | del_stmt | pass_stmt | flow_stmt | import_stmt | global_stmt | nonlocal_stmt | assert_stmt) -expr_stmt: testlist (augassign (yield_expr|testlist) | - ('=' (yield_expr|testlist))*) +expr_stmt: testlist_star_expr (augassign (yield_expr|testlist) | + ('=' (yield_expr|testlist_star_expr))*) +testlist_star_expr: (test|star_expr) (',' (test|star_expr))* [','] augassign: ('+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' | '<<=' | '>>=' | '**=' | '//=') # For normal assignments, additional restrictions enforced by the interpreter @@ -93,6 +94,7 @@ not_test: 'not' not_test | comparison comparison: expr (comp_op expr)* comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not' +star_expr: '*' expr expr: xor_expr ('|' xor_expr)* xor_expr: and_expr ('^' and_expr)* and_expr: shift_expr ('&' shift_expr)* diff --git a/pypy/interpreter/pyparser/test/test_pyparse.py b/pypy/interpreter/pyparser/test/test_pyparse.py --- a/pypy/interpreter/pyparser/test/test_pyparse.py +++ b/pypy/interpreter/pyparser/test/test_pyparse.py @@ -157,4 +157,5 @@ py.test.raises(SyntaxError, self.parse, '0777') def test_py3k_extended_unpacking(self): + self.parse('a, *rest, b = 1, 2, 3, 4, 5') self.parse('(a, *rest, b) = 1, 2, 3, 4, 5') From noreply at buildbot.pypy.org Tue Jan 17 17:20:32 2012 From: noreply at buildbot.pypy.org (antocuni) Date: Tue, 17 Jan 2012 17:20:32 +0100 (CET) Subject: [pypy-commit] pypy py3k: (antocuni, romain): most of the time the parser does not need a space, thus we instantiate it only for the tests (only one) which require it. This is useful because if we change the grammar, we might break the applevel code which is run during the initialization phase, thus resulting in an annoying and unrelated failure Message-ID: <20120117162032.C8CDE820D8@wyvern.cs.uni-duesseldorf.de> Author: Antonio Cuni Branch: py3k Changeset: r51407:dc00842db627 Date: 2012-01-17 17:20 +0100 http://bitbucket.org/pypy/pypy/changeset/dc00842db627/ Log: (antocuni, romain): most of the time the parser does not need a space, thus we instantiate it only for the tests (only one) which require it. This is useful because if we change the grammar, we might break the applevel code which is run during the initialization phase, thus resulting in an annoying and unrelated failure diff --git a/pypy/interpreter/pyparser/test/test_pyparse.py b/pypy/interpreter/pyparser/test/test_pyparse.py --- a/pypy/interpreter/pyparser/test/test_pyparse.py +++ b/pypy/interpreter/pyparser/test/test_pyparse.py @@ -6,10 +6,10 @@ from pypy.interpreter.astcompiler import consts -class TestPythonParser: +class TestPythonParserWithoutSpace: def setup_class(self): - self.parser = pyparse.PythonParser(self.space) + self.parser = pyparse.PythonParser(None) def parse(self, source, mode="exec", info=None): if info is None: @@ -31,39 +31,6 @@ tree = self.parse("name = 32") assert self.parser.root is None - def test_encoding(self): - info = pyparse.CompileInfo("", "exec") - tree = self.parse("""# coding: latin-1 -stuff = "nothing" -""", info=info) - assert tree.type == syms.file_input - assert info.encoding == "iso-8859-1" - sentence = u"'Die Männer ärgen sich!'" - input = (u"# coding: utf-7\nstuff = %s" % (sentence,)).encode("utf-7") - tree = self.parse(input, info=info) - assert info.encoding == "utf-7" - input = "# coding: iso-8859-15\nx" - self.parse(input, info=info) - assert info.encoding == "iso-8859-15" - input = "\xEF\xBB\xBF# coding: utf-8\nx" - self.parse(input, info=info) - assert info.encoding == "utf-8" - input = "# coding: utf-8\nx" - info.flags |= consts.PyCF_SOURCE_IS_UTF8 - exc = py.test.raises(SyntaxError, self.parse, input, info=info).value - info.flags &= ~consts.PyCF_SOURCE_IS_UTF8 - assert exc.msg == "coding declaration in unicode string" - input = "\xEF\xBB\xBF# coding: latin-1\nx" - exc = py.test.raises(SyntaxError, self.parse, input).value - assert exc.msg == "UTF-8 BOM with non-utf8 coding cookie" - input = "# coding: not-here" - exc = py.test.raises(SyntaxError, self.parse, input).value - assert exc.msg == "Unknown encoding: not-here" - input = u"# coding: ascii\n\xe2".encode('utf-8') - exc = py.test.raises(SyntaxError, self.parse, input).value - assert exc.msg == ("'ascii' codec can't decode byte 0xc3 " - "in position 16: ordinal not in range(128)") - def test_encoding_pep3120(self): info = pyparse.CompileInfo("", "exec") tree = self.parse("""foo = '日本'""", info=info) @@ -159,3 +126,43 @@ def test_py3k_extended_unpacking(self): self.parse('a, *rest, b = 1, 2, 3, 4, 5') self.parse('(a, *rest, b) = 1, 2, 3, 4, 5') + + +class TestPythonParserWithSpace(TestPythonParserWithoutSpace): + + def setup_class(self): + self.parser = pyparse.PythonParser(self.space) + + def test_encoding(self): + info = pyparse.CompileInfo("", "exec") + tree = self.parse("""# coding: latin-1 +stuff = "nothing" +""", info=info) + assert tree.type == syms.file_input + assert info.encoding == "iso-8859-1" + sentence = u"'Die Männer ärgen sich!'" + input = (u"# coding: utf-7\nstuff = %s" % (sentence,)).encode("utf-7") + tree = self.parse(input, info=info) + assert info.encoding == "utf-7" + input = "# coding: iso-8859-15\nx" + self.parse(input, info=info) + assert info.encoding == "iso-8859-15" + input = "\xEF\xBB\xBF# coding: utf-8\nx" + self.parse(input, info=info) + assert info.encoding == "utf-8" + input = "# coding: utf-8\nx" + info.flags |= consts.PyCF_SOURCE_IS_UTF8 + exc = py.test.raises(SyntaxError, self.parse, input, info=info).value + info.flags &= ~consts.PyCF_SOURCE_IS_UTF8 + assert exc.msg == "coding declaration in unicode string" + input = "\xEF\xBB\xBF# coding: latin-1\nx" + exc = py.test.raises(SyntaxError, self.parse, input).value + assert exc.msg == "UTF-8 BOM with non-utf8 coding cookie" + input = "# coding: not-here" + exc = py.test.raises(SyntaxError, self.parse, input).value + assert exc.msg == "Unknown encoding: not-here" + input = u"# coding: ascii\n\xe2".encode('utf-8') + exc = py.test.raises(SyntaxError, self.parse, input).value + assert exc.msg == ("'ascii' codec can't decode byte 0xc3 " + "in position 16: ordinal not in range(128)") + From noreply at buildbot.pypy.org Tue Jan 17 17:21:51 2012 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 17 Jan 2012 17:21:51 +0100 (CET) Subject: [pypy-commit] benchmarks default: add cpython documentation generation by sphinx Message-ID: <20120117162151.C3D65710260@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r157:70c28be43161 Date: 2012-01-17 18:07 +0200 http://bitbucket.org/pypy/benchmarks/changeset/70c28be43161/ Log: add cpython documentation generation by sphinx diff too long, truncating to 10000 out of 486679 lines diff --git a/benchmarks.py b/benchmarks.py --- a/benchmarks.py +++ b/benchmarks.py @@ -144,3 +144,31 @@ result.append((name, data)) return result BM_translate.benchmark_name = 'trans' + +def BM_cpython_doc(base_python, changed_python, options): + from unladen_swallow.perf import RawResult + import subprocess, shutil + t = [] + + for python in [base_python, changed_python]: + maindir = relative('lib/cpython-doc') + builddir = os.path.join(os.path.join(maindir, 'tools'), 'build') + shutil.rmtree(builddir) + build = relative('lib/cpython-doc/tools/sphinx-build.py') + os.mkdir(builddir) + docdir = os.path.join(builddir, 'doctrees') + os.mkdir(docdir) + htmldir = os.path.join(builddir, 'html') + os.mkdir(htmldir) + args = base_python + [build, '-b', 'html', '-d', docdir, maindir, htmldir] + proc = subprocess.Popen(args, stderr=subprocess.PIPE) + out, err = proc.communicate() + retcode = proc.poll() + if retcode != 0: + print out + print err + raise Exception("sphinx-build.py failed") + t.append(float(out.splitlines(-1).strip())) + return RawResult([t[0]], [t[1]]) + +BM_cpython_doc.benchmark_name = 'sphinx' diff --git a/lib/cpython-doc/ACKS.txt b/lib/cpython-doc/ACKS.txt new file mode 100644 --- /dev/null +++ b/lib/cpython-doc/ACKS.txt @@ -0,0 +1,237 @@ +Contributors to the Python Documentation +---------------------------------------- + +This section lists people who have contributed in some way to the Python +documentation. It is probably not complete -- if you feel that you or +anyone else should be on this list, please let us know (send email to +docs at python.org), and we'll be glad to correct the problem. + +.. acks:: + + * Aahz + * Michael Abbott + * Steve Alexander + * Jim Ahlstrom + * Fred Allen + * A. Amoroso + * Pehr Anderson + * Oliver Andrich + * Heidi Annexstad + * Jesús Cea Avión + * Manuel Balsera + * Daniel Barclay + * Chris Barker + * Don Bashford + * Anthony Baxter + * Alexander Belopolsky + * Bennett Benson + * Jonathan Black + * Robin Boerdijk + * Michal Bozon + * Aaron Brancotti + * Georg Brandl + * Keith Briggs + * Ian Bruntlett + * Lee Busby + * Arnaud Calmettes + * Lorenzo M. Catucci + * Carl Cerecke + * Mauro Cicognini + * Gilles Civario + * Mike Clarkson + * Steve Clift + * Dave Cole + * Matthew Cowles + * Jeremy Craven + * Andrew Dalke + * Ben Darnell + * L. Peter Deutsch + * Robert Donohue + * Fred L. Drake, Jr. + * Jacques Ducasse + * Josip Dzolonga + * Jeff Epler + * Michael Ernst + * Blame Andy Eskilsson + * Carey Evans + * Martijn Faassen + * Carl Feynman + * Dan Finnie + * Hernán Martínez Foffani + * Michael Foord + * Stefan Franke + * Jim Fulton + * Peter Funk + * Lele Gaifax + * Matthew Gallagher + * Gabriel Genellina + * Ben Gertzfield + * Nadim Ghaznavi + * Jonathan Giddy + * Matt Giuca + * Shelley Gooch + * Nathaniel Gray + * Grant Griffin + * Thomas Guettler + * Anders Hammarquist + * Mark Hammond + * Harald Hanche-Olsen + * Manus Hand + * Gerhard Häring + * Travis B. Hartwell + * Tim Hatch + * Janko Hauser + * Ben Hayden + * Thomas Heller + * Bernhard Herzog + * Magnus L. Hetland + * Konrad Hinsen + * Stefan Hoffmeister + * Albert Hofkamp + * Gregor Hoffleit + * Steve Holden + * Thomas Holenstein + * Gerrit Holl + * Rob Hooft + * Brian Hooper + * Randall Hopper + * Michael Hudson + * Eric Huss + * Jeremy Hylton + * Roger Irwin + * Jack Jansen + * Philip H. Jensen + * Pedro Diaz Jimenez + * Kent Johnson + * Lucas de Jonge + * Andreas Jung + * Robert Kern + * Jim Kerr + * Jan Kim + * Kamil Kisiel + * Greg Kochanski + * Guido Kollerie + * Peter A. Koren + * Daniel Kozan + * Andrew M. Kuchling + * Dave Kuhlman + * Erno Kuusela + * Ross Lagerwall + * Thomas Lamb + * Detlef Lannert + * Piers Lauder + * Glyph Lefkowitz + * Robert Lehmann + * Marc-André Lemburg + * Ross Light + * Gediminas Liktaras + * Ulf A. Lindgren + * Everett Lipman + * Mirko Liss + * Martin von Löwis + * Fredrik Lundh + * Jeff MacDonald + * John Machin + * Andrew MacIntyre + * Vladimir Marangozov + * Vincent Marchetti + * Westley Martínez + * Laura Matson + * Daniel May + * Rebecca McCreary + * Doug Mennella + * Paolo Milani + * Skip Montanaro + * Paul Moore + * Ross Moore + * Sjoerd Mullender + * Dale Nagata + * Trent Nelson + * Michal Nowikowski + * Steffen Daode Nurpmeso + * Ng Pheng Siong + * Koray Oner + * Tomas Oppelstrup + * Denis S. Otkidach + * Zooko O'Whielacronx + * Shriphani Palakodety + * William Park + * Joonas Paalasmaa + * Harri Pasanen + * Bo Peng + * Tim Peters + * Benjamin Peterson + * Christopher Petrilli + * Justin D. Pettit + * Chris Phoenix + * François Pinard + * Paul Prescod + * Eric S. Raymond + * Edward K. Ream + * Terry J. Reedy + * Sean Reifschneider + * Bernhard Reiter + * Armin Rigo + * Wes Rishel + * Armin Ronacher + * Jim Roskind + * Guido van Rossum + * Donald Wallace Rouse II + * Mark Russell + * Nick Russo + * Chris Ryland + * Constantina S. + * Hugh Sasse + * Bob Savage + * Scott Schram + * Neil Schemenauer + * Barry Scott + * Joakim Sernbrant + * Justin Sheehy + * Charlie Shepherd + * Yue Shuaijie + * SilentGhost + * Michael Simcich + * Ionel Simionescu + * Michael Sloan + * Gregory P. Smith + * Roy Smith + * Clay Spence + * Nicholas Spies + * Tage Stabell-Kulo + * Frank Stajano + * Anthony Starks + * Greg Stein + * Peter Stoehr + * Mark Summerfield + * Reuben Sumner + * Kalle Svensson + * Jim Tittsler + * David Turner + * Sandro Tosi + * Ville Vainio + * Nadeem Vawda + * Martijn Vries + * Charles G. Waldman + * Greg Ward + * Barry Warsaw + * Corran Webster + * Glyn Webster + * Bob Weiner + * Eddy Welbourne + * Jeff Wheeler + * Mats Wichmann + * Gerry Wiener + * Timothy Wild + * Paul Winkler + * Collin Winter + * Blake Winton + * Dan Wolfe + * Adam Woodbeck + * Steven Work + * Thomas Wouters + * Ka-Ping Yee + * Rory Yorke + * Moshe Zadka + * Milan Zamazal + * Cheng Zhang diff --git a/lib/cpython-doc/Makefile b/lib/cpython-doc/Makefile new file mode 100644 --- /dev/null +++ b/lib/cpython-doc/Makefile @@ -0,0 +1,193 @@ +# +# Makefile for Python documentation +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# + +# You can set these variables from the command line. +PYTHON = python +SVNROOT = http://svn.python.org/projects +SPHINXOPTS = +PAPER = +SOURCES = +DISTVERSION = $(shell $(PYTHON) tools/sphinxext/patchlevel.py) + +ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees -D latex_paper_size=$(PAPER) \ + $(SPHINXOPTS) . build/$(BUILDER) $(SOURCES) + +.PHONY: help checkout update build html htmlhelp latex text changes linkcheck \ + suspicious coverage doctest pydoc-topics htmlview clean dist check serve \ + autobuild-dev autobuild-stable + +help: + @echo "Please use \`make ' where is one of" + @echo " clean to remove build files" + @echo " update to update build tools" + @echo " html to make standalone HTML files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " text to make plain text files" + @echo " epub to make EPUB files" + @echo " changes to make an overview over all changed/added/deprecated items" + @echo " linkcheck to check all external links for integrity" + @echo " coverage to check documentation coverage for library and C API" + @echo " doctest to run doctests in the documentation" + @echo " pydoc-topics to regenerate the pydoc topics file" + @echo " dist to create a \"dist\" directory with archived docs for download" + @echo " suspicious to check for suspicious markup in output text" + @echo " check to run a check for frequent markup errors" + @echo " serve to serve the documentation on the localhost (8000)" + +# Note: if you update versions here, do the same in make.bat and README.txt +checkout: + @if [ ! -d tools/sphinx ]; then \ + echo "Checking out Sphinx..."; \ + svn checkout $(SVNROOT)/external/Sphinx-1.0.7/sphinx tools/sphinx; \ + fi + @if [ ! -d tools/docutils ]; then \ + echo "Checking out Docutils..."; \ + svn checkout $(SVNROOT)/external/docutils-0.6/docutils tools/docutils; \ + fi + @if [ ! -d tools/jinja2 ]; then \ + echo "Checking out Jinja..."; \ + svn checkout $(SVNROOT)/external/Jinja-2.3.1/jinja2 tools/jinja2; \ + fi + @if [ ! -d tools/pygments ]; then \ + echo "Checking out Pygments..."; \ + svn checkout $(SVNROOT)/external/Pygments-1.3.1/pygments tools/pygments; \ + fi + +update: clean checkout + +build: checkout + mkdir -p build/$(BUILDER) build/doctrees + echo $(ALLSPHINXOPTS) + $(PYTHON) tools/sphinx-build.py $(ALLSPHINXOPTS) + @echo + +html: BUILDER = html +html: build + @echo "Build finished. The HTML pages are in build/html." + +htmlhelp: BUILDER = htmlhelp +htmlhelp: build + @echo "Build finished; now you can run HTML Help Workshop with the" \ + "build/htmlhelp/pydoc.hhp project file." + +latex: BUILDER = latex +latex: build + @echo "Build finished; the LaTeX files are in build/latex." + @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \ + "run these through (pdf)latex." + +text: BUILDER = text +text: build + @echo "Build finished; the text files are in build/text." + +epub: BUILDER = epub +epub: build + @echo "Build finished; the epub files are in build/epub." + +changes: BUILDER = changes +changes: build + @echo "The overview file is in build/changes." + +linkcheck: BUILDER = linkcheck +linkcheck: build + @echo "Link check complete; look for any errors in the above output" \ + "or in build/$(BUILDER)/output.txt" + +suspicious: BUILDER = suspicious +suspicious: build + @echo "Suspicious check complete; look for any errors in the above output" \ + "or in build/$(BUILDER)/suspicious.csv. If all issues are false" \ + "positives, append that file to tools/sphinxext/susp-ignored.csv." + +coverage: BUILDER = coverage +coverage: build + @echo "Coverage finished; see c.txt and python.txt in build/coverage" + +doctest: BUILDER = doctest +doctest: build + @echo "Testing of doctests in the sources finished, look at the" \ + "results in build/doctest/output.txt" + +pydoc-topics: BUILDER = pydoc-topics +pydoc-topics: build + @echo "Building finished; now copy build/pydoc-topics/topics.py" \ + "to ../Lib/pydoc_data/topics.py" + +htmlview: html + $(PYTHON) -c "import webbrowser; webbrowser.open('build/html/index.html')" + +clean: + -rm -rf build/* + +dist: + rm -rf dist + mkdir -p dist + + # archive the HTML + make html + cp -pPR build/html dist/python-$(DISTVERSION)-docs-html + tar -C dist -cf dist/python-$(DISTVERSION)-docs-html.tar python-$(DISTVERSION)-docs-html + bzip2 -9 -k dist/python-$(DISTVERSION)-docs-html.tar + (cd dist; zip -q -r -9 python-$(DISTVERSION)-docs-html.zip python-$(DISTVERSION)-docs-html) + rm -r dist/python-$(DISTVERSION)-docs-html + rm dist/python-$(DISTVERSION)-docs-html.tar + + # archive the text build + make text + cp -pPR build/text dist/python-$(DISTVERSION)-docs-text + tar -C dist -cf dist/python-$(DISTVERSION)-docs-text.tar python-$(DISTVERSION)-docs-text + bzip2 -9 -k dist/python-$(DISTVERSION)-docs-text.tar + (cd dist; zip -q -r -9 python-$(DISTVERSION)-docs-text.zip python-$(DISTVERSION)-docs-text) + rm -r dist/python-$(DISTVERSION)-docs-text + rm dist/python-$(DISTVERSION)-docs-text.tar + + # archive the A4 latex + rm -rf build/latex + make latex PAPER=a4 + -sed -i 's/makeindex/makeindex -q/' build/latex/Makefile + (cd build/latex; make clean && make all-pdf && make FMT=pdf zip bz2) + cp build/latex/docs-pdf.zip dist/python-$(DISTVERSION)-docs-pdf-a4.zip + cp build/latex/docs-pdf.tar.bz2 dist/python-$(DISTVERSION)-docs-pdf-a4.tar.bz2 + + # archive the letter latex + rm -rf build/latex + make latex PAPER=letter + -sed -i 's/makeindex/makeindex -q/' build/latex/Makefile + (cd build/latex; make clean && make all-pdf && make FMT=pdf zip bz2) + cp build/latex/docs-pdf.zip dist/python-$(DISTVERSION)-docs-pdf-letter.zip + cp build/latex/docs-pdf.tar.bz2 dist/python-$(DISTVERSION)-docs-pdf-letter.tar.bz2 + + # archive the epub build + rm -rf build/epub + make epub + mkdir -p dist/python-$(DISTVERSION)-docs-epub + cp -pPR build/epub/*.epub dist/python-$(DISTVERSION)-docs-epub/ + tar -C dist -cf dist/python-$(DISTVERSION)-docs-epub.tar python-$(DISTVERSION)-docs-epub + bzip2 -9 -k dist/python-$(DISTVERSION)-docs-epub.tar + (cd dist; zip -q -r -9 python-$(DISTVERSION)-docs-epub.zip python-$(DISTVERSION)-docs-epub) + rm -r dist/python-$(DISTVERSION)-docs-epub + rm dist/python-$(DISTVERSION)-docs-epub.tar + +check: + $(PYTHON) tools/rstlint.py -i tools + +serve: + ../Tools/scripts/serve.py build/html + +# Targets for daily automated doc build + +# for development releases: always build +autobuild-dev: + make update + make dist SPHINXOPTS='-A daily=1' + +# for stable releases: only build if not in pre-release stage (alpha, beta, rc) +autobuild-stable: + @case $(DISTVERSION) in *[abc]*) \ + echo "Not building; $(DISTVERSION) is not a release version."; \ + exit 1;; \ + esac + @make autobuild-dev diff --git a/lib/cpython-doc/README.txt b/lib/cpython-doc/README.txt new file mode 100644 --- /dev/null +++ b/lib/cpython-doc/README.txt @@ -0,0 +1,152 @@ +Python Documentation README +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This directory contains the reStructuredText (reST) sources to the Python +documentation. You don't need to build them yourself, prebuilt versions are +available at http://docs.python.org/download/. + +Documentation on the authoring Python documentation, including information about +both style and markup, is available in the "Documenting Python" chapter of the +documentation. There's also a chapter intended to point out differences to +those familiar with the previous docs written in LaTeX. + + +Building the docs +================= + +You need to have Python 2.4 or higher installed; the toolset used to build the +docs is written in Python. It is called *Sphinx*, it is not included in this +tree, but maintained separately. Also needed are the docutils, supplying the +base markup that Sphinx uses, Jinja, a templating engine, and optionally +Pygments, a code highlighter. + + +Using make +---------- + +Luckily, a Makefile has been prepared so that on Unix, provided you have +installed Python and Subversion, you can just run :: + + make html + +to check out the necessary toolset in the `tools/` subdirectory and build the +HTML output files. To view the generated HTML, point your favorite browser at +the top-level index `build/html/index.html` after running "make". + +To use a Python interpreter that's not called ``python``, use the standard +way to set Makefile variables, using e.g. :: + + make html PYTHON=/usr/bin/python2.5 + +Available make targets are: + + * "html", which builds standalone HTML files for offline viewing. + + * "htmlhelp", which builds HTML files and a HTML Help project file usable to + convert them into a single Compiled HTML (.chm) file -- these are popular + under Microsoft Windows, but very handy on every platform. + + To create the CHM file, you need to run the Microsoft HTML Help Workshop over + the generated project (.hhp) file. + + * "latex", which builds LaTeX source files as input to "pdflatex" to produce + PDF documents. + + * "text", which builds a plain text file for each source file. + + * "epub", which builds an EPUB document, suitable to be viewed on e-book + readers. + + * "linkcheck", which checks all external references to see whether they are + broken, redirected or malformed, and outputs this information to stdout as + well as a plain-text (.txt) file. + + * "changes", which builds an overview over all versionadded/versionchanged/ + deprecated items in the current version. This is meant as a help for the + writer of the "What's New" document. + + * "coverage", which builds a coverage overview for standard library modules and + C API. + + * "pydoc-topics", which builds a Python module containing a dictionary with + plain text documentation for the labels defined in + `tools/sphinxext/pyspecific.py` -- pydoc needs these to show topic and + keyword help. + +A "make update" updates the Subversion checkouts in `tools/`. + + +Without make +------------ + +You'll need to install the Sphinx package, either by checking it out via :: + + svn co http://svn.python.org/projects/external/Sphinx-1.0.7/sphinx tools/sphinx + +or by installing it from PyPI. + +Then, you need to install Docutils, either by checking it out via :: + + svn co http://svn.python.org/projects/external/docutils-0.6/docutils tools/docutils + +or by installing it from http://docutils.sf.net/. + +You also need Jinja2, either by checking it out via :: + + svn co http://svn.python.org/projects/external/Jinja-2.3.1/jinja2 tools/jinja2 + +or by installing it from PyPI. + +You can optionally also install Pygments, either as a checkout via :: + + svn co http://svn.python.org/projects/external/Pygments-1.3.1/pygments tools/pygments + +or from PyPI at http://pypi.python.org/pypi/Pygments. + + +Then, make an output directory, e.g. under `build/`, and run :: + + python tools/sphinx-build.py -b . build/ + +where `` is one of html, text, latex, or htmlhelp (for explanations see +the make targets above). + + +Contributing +============ + +Bugs in the content should be reported to the Python bug tracker at +http://bugs.python.org. + +Bugs in the toolset should be reported in the Sphinx bug tracker at +http://www.bitbucket.org/birkenfeld/sphinx/issues/. + +You can also send a mail to the Python Documentation Team at docs at python.org, +and we will process your request as soon as possible. + +If you want to help the Documentation Team, you are always welcome. Just send +a mail to docs at python.org. + + +Copyright notice +================ + +The Python source is copyrighted, but you can freely use and copy it +as long as you don't change or remove the copyright notice: + +---------------------------------------------------------------------- +Copyright (c) 2000-2012 Python Software Foundation. +All rights reserved. + +Copyright (c) 2000 BeOpen.com. +All rights reserved. + +Copyright (c) 1995-2000 Corporation for National Research Initiatives. +All rights reserved. + +Copyright (c) 1991-1995 Stichting Mathematisch Centrum. +All rights reserved. + +See the file "license.rst" for information on usage and redistribution +of this file, and for a DISCLAIMER OF ALL WARRANTIES. +---------------------------------------------------------------------- diff --git a/lib/cpython-doc/about.rst b/lib/cpython-doc/about.rst new file mode 100644 --- /dev/null +++ b/lib/cpython-doc/about.rst @@ -0,0 +1,36 @@ +===================== +About these documents +===================== + + +These documents are generated from `reStructuredText`_ sources by `Sphinx`_, a +document processor specifically written for the Python documentation. + +.. _reStructuredText: http://docutils.sf.net/rst.html +.. _Sphinx: http://sphinx.pocoo.org/ + +.. In the online version of these documents, you can submit comments and suggest + changes directly on the documentation pages. + +Development of the documentation and its toolchain takes place on the +docs at python.org mailing list. We're always looking for volunteers wanting +to help with the docs, so feel free to send a mail there! + +Many thanks go to: + +* Fred L. Drake, Jr., the creator of the original Python documentation toolset + and writer of much of the content; +* the `Docutils `_ project for creating + reStructuredText and the Docutils suite; +* Fredrik Lundh for his `Alternative Python Reference + `_ project from which Sphinx got many good + ideas. + +See :ref:`reporting-bugs` for information how to report bugs in this +documentation, or Python itself. + +.. including the ACKS file here so that it can be maintained separately +.. include:: ACKS.txt + +It is only with the input and contributions of the Python community +that Python has such wonderful documentation -- Thank You! diff --git a/lib/cpython-doc/bugs.rst b/lib/cpython-doc/bugs.rst new file mode 100644 --- /dev/null +++ b/lib/cpython-doc/bugs.rst @@ -0,0 +1,75 @@ +.. _reporting-bugs: + +************** +Reporting Bugs +************** + +Python is a mature programming language which has established a reputation for +stability. In order to maintain this reputation, the developers would like to +know of any deficiencies you find in Python. + + +Documentation bugs +================== + +If you find a bug in this documentation or would like to propose an improvement, +please send an e-mail to docs at python.org describing the bug and where you found +it. If you have a suggestion how to fix it, include that as well. + +docs at python.org is a mailing list run by volunteers; your request will be +noticed, even if it takes a while to be processed. + +Of course, if you want a more persistent record of your issue, you can use the +issue tracker for documentation bugs as well. + + +Using the Python issue tracker +============================== + +Bug reports for Python itself should be submitted via the Python Bug Tracker +(http://bugs.python.org/). The bug tracker offers a Web form which allows +pertinent information to be entered and submitted to the developers. + +The first step in filing a report is to determine whether the problem has +already been reported. The advantage in doing so, aside from saving the +developers time, is that you learn what has been done to fix it; it may be that +the problem has already been fixed for the next release, or additional +information is needed (in which case you are welcome to provide it if you can!). +To do this, search the bug database using the search box on the top of the page. + +If the problem you're reporting is not already in the bug tracker, go back to +the Python Bug Tracker and log in. If you don't already have a tracker account, +select the "Register" link or, if you use OpenID, one of the OpenID provider +logos in the sidebar. It is not possible to submit a bug report anonymously. + +Being now logged in, you can submit a bug. Select the "Create New" link in the +sidebar to open the bug reporting form. + +The submission form has a number of fields. For the "Title" field, enter a +*very* short description of the problem; less than ten words is good. In the +"Type" field, select the type of your problem; also select the "Component" and +"Versions" to which the bug relates. + +In the "Comment" field, describe the problem in detail, including what you +expected to happen and what did happen. Be sure to include whether any +extension modules were involved, and what hardware and software platform you +were using (including version information as appropriate). + +Each bug report will be assigned to a developer who will determine what needs to +be done to correct the problem. You will receive an update each time action is +taken on the bug. + + +.. seealso:: + + `Python Developer's Guide `_ + Detailed description of the issue workflow and developers tools. + + `How to Report Bugs Effectively `_ + Article which goes into some detail about how to create a useful bug report. + This describes what kind of information is useful and why it is useful. + + `Bug Writing Guidelines `_ + Information about writing a good bug report. Some of this is specific to the + Mozilla project, but describes general good practices. + diff --git a/lib/cpython-doc/c-api/abstract.rst b/lib/cpython-doc/c-api/abstract.rst new file mode 100644 --- /dev/null +++ b/lib/cpython-doc/c-api/abstract.rst @@ -0,0 +1,26 @@ +.. highlightlang:: c + +.. _abstract: + +********************** +Abstract Objects Layer +********************** + +The functions in this chapter interact with Python objects regardless of their +type, or with wide classes of object types (e.g. all numerical types, or all +sequence types). When used on object types for which they do not apply, they +will raise a Python exception. + +It is not possible to use these functions on objects that are not properly +initialized, such as a list object that has been created by :c:func:`PyList_New`, +but whose items have not been set to some non-\ ``NULL`` value yet. + +.. toctree:: + + object.rst + number.rst + sequence.rst + mapping.rst + iter.rst + buffer.rst + objbuffer.rst diff --git a/lib/cpython-doc/c-api/allocation.rst b/lib/cpython-doc/c-api/allocation.rst new file mode 100644 --- /dev/null +++ b/lib/cpython-doc/c-api/allocation.rst @@ -0,0 +1,71 @@ +.. highlightlang:: c + +.. _allocating-objects: + +Allocating Objects on the Heap +============================== + + +.. c:function:: PyObject* _PyObject_New(PyTypeObject *type) + + +.. c:function:: PyVarObject* _PyObject_NewVar(PyTypeObject *type, Py_ssize_t size) + + +.. c:function:: PyObject* PyObject_Init(PyObject *op, PyTypeObject *type) + + Initialize a newly-allocated object *op* with its type and initial + reference. Returns the initialized object. If *type* indicates that the + object participates in the cyclic garbage detector, it is added to the + detector's set of observed objects. Other fields of the object are not + affected. + + +.. c:function:: PyVarObject* PyObject_InitVar(PyVarObject *op, PyTypeObject *type, Py_ssize_t size) + + This does everything :c:func:`PyObject_Init` does, and also initializes the + length information for a variable-size object. + + +.. c:function:: TYPE* PyObject_New(TYPE, PyTypeObject *type) + + Allocate a new Python object using the C structure type *TYPE* and the + Python type object *type*. Fields not defined by the Python object header + are not initialized; the object's reference count will be one. The size of + the memory allocation is determined from the :attr:`tp_basicsize` field of + the type object. + + +.. c:function:: TYPE* PyObject_NewVar(TYPE, PyTypeObject *type, Py_ssize_t size) + + Allocate a new Python object using the C structure type *TYPE* and the + Python type object *type*. Fields not defined by the Python object header + are not initialized. The allocated memory allows for the *TYPE* structure + plus *size* fields of the size given by the :attr:`tp_itemsize` field of + *type*. This is useful for implementing objects like tuples, which are + able to determine their size at construction time. Embedding the array of + fields into the same allocation decreases the number of allocations, + improving the memory management efficiency. + + +.. c:function:: void PyObject_Del(PyObject *op) + + Releases memory allocated to an object using :c:func:`PyObject_New` or + :c:func:`PyObject_NewVar`. This is normally called from the + :attr:`tp_dealloc` handler specified in the object's type. The fields of + the object should not be accessed after this call as the memory is no + longer a valid Python object. + + +.. c:var:: PyObject _Py_NoneStruct + + Object which is visible in Python as ``None``. This should only be accessed + using the :c:macro:`Py_None` macro, which evaluates to a pointer to this + object. + + +.. seealso:: + + :c:func:`PyModule_Create` + To allocate and create extension modules. + diff --git a/lib/cpython-doc/c-api/arg.rst b/lib/cpython-doc/c-api/arg.rst new file mode 100644 --- /dev/null +++ b/lib/cpython-doc/c-api/arg.rst @@ -0,0 +1,619 @@ +.. highlightlang:: c + +.. _arg-parsing: + +Parsing arguments and building values +===================================== + +These functions are useful when creating your own extensions functions and +methods. Additional information and examples are available in +:ref:`extending-index`. + +The first three of these functions described, :c:func:`PyArg_ParseTuple`, +:c:func:`PyArg_ParseTupleAndKeywords`, and :c:func:`PyArg_Parse`, all use *format +strings* which are used to tell the function about the expected arguments. The +format strings use the same syntax for each of these functions. + +----------------- +Parsing arguments +----------------- + +A format string consists of zero or more "format units." A format unit +describes one Python object; it is usually a single character or a parenthesized +sequence of format units. With a few exceptions, a format unit that is not a +parenthesized sequence normally corresponds to a single address argument to +these functions. In the following description, the quoted form is the format +unit; the entry in (round) parentheses is the Python object type that matches +the format unit; and the entry in [square] brackets is the type of the C +variable(s) whose address should be passed. + +Strings and buffers +------------------- + +These formats allow to access an object as a contiguous chunk of memory. +You don't have to provide raw storage for the returned unicode or bytes +area. Also, you won't have to release any memory yourself, except with the +``es``, ``es#``, ``et`` and ``et#`` formats. + +However, when a :c:type:`Py_buffer` structure gets filled, the underlying +buffer is locked so that the caller can subsequently use the buffer even +inside a :c:type:`Py_BEGIN_ALLOW_THREADS` block without the risk of mutable data +being resized or destroyed. As a result, **you have to call** +:c:func:`PyBuffer_Release` after you have finished processing the data (or +in any early abort case). + +Unless otherwise stated, buffers are not NUL-terminated. + +.. note:: + For all ``#`` variants of formats (``s#``, ``y#``, etc.), the type of + the length argument (int or :c:type:`Py_ssize_t`) is controlled by + defining the macro :c:macro:`PY_SSIZE_T_CLEAN` before including + :file:`Python.h`. If the macro was defined, length is a + :c:type:`Py_ssize_t` rather than an :c:type:`int`. This behavior will change + in a future Python version to only support :c:type:`Py_ssize_t` and + drop :c:type:`int` support. It is best to always define :c:macro:`PY_SSIZE_T_CLEAN`. + + +``s`` (:class:`str`) [const char \*] + Convert a Unicode object to a C pointer to a character string. + A pointer to an existing string is stored in the character pointer + variable whose address you pass. The C string is NUL-terminated. + The Python string must not contain embedded NUL bytes; if it does, + a :exc:`TypeError` exception is raised. Unicode objects are converted + to C strings using ``'utf-8'`` encoding. If this conversion fails, a + :exc:`UnicodeError` is raised. + + .. note:: + This format does not accept bytes-like objects. If you want to accept + filesystem paths and convert them to C character strings, it is + preferable to use the ``O&`` format with :c:func:`PyUnicode_FSConverter` + as *converter*. + +``s*`` (:class:`str`, :class:`bytes`, :class:`bytearray` or buffer compatible object) [Py_buffer] + This format accepts Unicode objects as well as objects supporting the + buffer protocol. + It fills a :c:type:`Py_buffer` structure provided by the caller. + In this case the resulting C string may contain embedded NUL bytes. + Unicode objects are converted to C strings using ``'utf-8'`` encoding. + +``s#`` (:class:`str`, :class:`bytes` or read-only buffer compatible object) [const char \*, int or :c:type:`Py_ssize_t`] + Like ``s*``, except that it doesn't accept mutable buffer-like objects + such as :class:`bytearray`. The result is stored into two C variables, + the first one a pointer to a C string, the second one its length. + The string may contain embedded null bytes. Unicode objects are converted + to C strings using ``'utf-8'`` encoding. + +``z`` (:class:`str` or ``None``) [const char \*] + Like ``s``, but the Python object may also be ``None``, in which case the C + pointer is set to *NULL*. + +``z*`` (:class:`str`, :class:`bytes`, :class:`bytearray`, buffer compatible object or ``None``) [Py_buffer] + Like ``s*``, but the Python object may also be ``None``, in which case the + ``buf`` member of the :c:type:`Py_buffer` structure is set to *NULL*. + +``z#`` (:class:`str`, :class:`bytes`, read-only buffer compatible object or ``None``) [const char \*, int] + Like ``s#``, but the Python object may also be ``None``, in which case the C + pointer is set to *NULL*. + +``y`` (:class:`bytes`) [const char \*] + This format converts a bytes-like object to a C pointer to a character + string; it does not accept Unicode objects. The bytes buffer must not + contain embedded NUL bytes; if it does, a :exc:`TypeError` + exception is raised. + +``y*`` (:class:`bytes`, :class:`bytearray` or buffer compatible object) [Py_buffer] + This variant on ``s*`` doesn't accept Unicode objects, only objects + supporting the buffer protocol. **This is the recommended way to accept + binary data.** + +``y#`` (:class:`bytes`) [const char \*, int] + This variant on ``s#`` doesn't accept Unicode objects, only bytes-like + objects. + +``S`` (:class:`bytes`) [PyBytesObject \*] + Requires that the Python object is a :class:`bytes` object, without + attempting any conversion. Raises :exc:`TypeError` if the object is not + a bytes object. The C variable may also be declared as :c:type:`PyObject\*`. + +``Y`` (:class:`bytearray`) [PyByteArrayObject \*] + Requires that the Python object is a :class:`bytearray` object, without + attempting any conversion. Raises :exc:`TypeError` if the object is not + a :class:`bytearray` object. The C variable may also be declared as :c:type:`PyObject\*`. + +``u`` (:class:`str`) [Py_UNICODE \*] + Convert a Python Unicode object to a C pointer to a NUL-terminated buffer of + Unicode characters. You must pass the address of a :c:type:`Py_UNICODE` + pointer variable, which will be filled with the pointer to an existing + Unicode buffer. Please note that the width of a :c:type:`Py_UNICODE` + character depends on compilation options (it is either 16 or 32 bits). + The Python string must not contain embedded NUL characters; if it does, + a :exc:`TypeError` exception is raised. + + .. note:: + Since ``u`` doesn't give you back the length of the string, and it + may contain embedded NUL characters, it is recommended to use ``u#`` + or ``U`` instead. + +``u#`` (:class:`str`) [Py_UNICODE \*, int] + This variant on ``u`` stores into two C variables, the first one a pointer to a + Unicode data buffer, the second one its length. + +``Z`` (:class:`str` or ``None``) [Py_UNICODE \*] + Like ``u``, but the Python object may also be ``None``, in which case the + :c:type:`Py_UNICODE` pointer is set to *NULL*. + +``Z#`` (:class:`str` or ``None``) [Py_UNICODE \*, int] + Like ``u#``, but the Python object may also be ``None``, in which case the + :c:type:`Py_UNICODE` pointer is set to *NULL*. + +``U`` (:class:`str`) [PyObject \*] + Requires that the Python object is a Unicode object, without attempting + any conversion. Raises :exc:`TypeError` if the object is not a Unicode + object. The C variable may also be declared as :c:type:`PyObject\*`. + +``w*`` (:class:`bytearray` or read-write byte-oriented buffer) [Py_buffer] + This format accepts any object which implements the read-write buffer + interface. It fills a :c:type:`Py_buffer` structure provided by the caller. + The buffer may contain embedded null bytes. The caller have to call + :c:func:`PyBuffer_Release` when it is done with the buffer. + +``es`` (:class:`str`) [const char \*encoding, char \*\*buffer] + This variant on ``s`` is used for encoding Unicode into a character buffer. + It only works for encoded data without embedded NUL bytes. + + This format requires two arguments. The first is only used as input, and + must be a :c:type:`const char\*` which points to the name of an encoding as a + NUL-terminated string, or *NULL*, in which case ``'utf-8'`` encoding is used. + An exception is raised if the named encoding is not known to Python. The + second argument must be a :c:type:`char\*\*`; the value of the pointer it + references will be set to a buffer with the contents of the argument text. + The text will be encoded in the encoding specified by the first argument. + + :c:func:`PyArg_ParseTuple` will allocate a buffer of the needed size, copy the + encoded data into this buffer and adjust *\*buffer* to reference the newly + allocated storage. The caller is responsible for calling :c:func:`PyMem_Free` to + free the allocated buffer after use. + +``et`` (:class:`str`, :class:`bytes` or :class:`bytearray`) [const char \*encoding, char \*\*buffer] + Same as ``es`` except that byte string objects are passed through without + recoding them. Instead, the implementation assumes that the byte string object uses + the encoding passed in as parameter. + +``es#`` (:class:`str`) [const char \*encoding, char \*\*buffer, int \*buffer_length] + This variant on ``s#`` is used for encoding Unicode into a character buffer. + Unlike the ``es`` format, this variant allows input data which contains NUL + characters. + + It requires three arguments. The first is only used as input, and must be a + :c:type:`const char\*` which points to the name of an encoding as a + NUL-terminated string, or *NULL*, in which case ``'utf-8'`` encoding is used. + An exception is raised if the named encoding is not known to Python. The + second argument must be a :c:type:`char\*\*`; the value of the pointer it + references will be set to a buffer with the contents of the argument text. + The text will be encoded in the encoding specified by the first argument. + The third argument must be a pointer to an integer; the referenced integer + will be set to the number of bytes in the output buffer. + + There are two modes of operation: + + If *\*buffer* points a *NULL* pointer, the function will allocate a buffer of + the needed size, copy the encoded data into this buffer and set *\*buffer* to + reference the newly allocated storage. The caller is responsible for calling + :c:func:`PyMem_Free` to free the allocated buffer after usage. + + If *\*buffer* points to a non-*NULL* pointer (an already allocated buffer), + :c:func:`PyArg_ParseTuple` will use this location as the buffer and interpret the + initial value of *\*buffer_length* as the buffer size. It will then copy the + encoded data into the buffer and NUL-terminate it. If the buffer is not large + enough, a :exc:`ValueError` will be set. + + In both cases, *\*buffer_length* is set to the length of the encoded data + without the trailing NUL byte. + +``et#`` (:class:`str`, :class:`bytes` or :class:`bytearray`) [const char \*encoding, char \*\*buffer, int \*buffer_length] + Same as ``es#`` except that byte string objects are passed through without recoding + them. Instead, the implementation assumes that the byte string object uses the + encoding passed in as parameter. + +Numbers +------- + +``b`` (:class:`int`) [unsigned char] + Convert a nonnegative Python integer to an unsigned tiny int, stored in a C + :c:type:`unsigned char`. + +``B`` (:class:`int`) [unsigned char] + Convert a Python integer to a tiny int without overflow checking, stored in a C + :c:type:`unsigned char`. + +``h`` (:class:`int`) [short int] + Convert a Python integer to a C :c:type:`short int`. + +``H`` (:class:`int`) [unsigned short int] + Convert a Python integer to a C :c:type:`unsigned short int`, without overflow + checking. + +``i`` (:class:`int`) [int] + Convert a Python integer to a plain C :c:type:`int`. + +``I`` (:class:`int`) [unsigned int] + Convert a Python integer to a C :c:type:`unsigned int`, without overflow + checking. + +``l`` (:class:`int`) [long int] + Convert a Python integer to a C :c:type:`long int`. + +``k`` (:class:`int`) [unsigned long] + Convert a Python integer to a C :c:type:`unsigned long` without + overflow checking. + +``L`` (:class:`int`) [PY_LONG_LONG] + Convert a Python integer to a C :c:type:`long long`. This format is only + available on platforms that support :c:type:`long long` (or :c:type:`_int64` on + Windows). + +``K`` (:class:`int`) [unsigned PY_LONG_LONG] + Convert a Python integer to a C :c:type:`unsigned long long` + without overflow checking. This format is only available on platforms that + support :c:type:`unsigned long long` (or :c:type:`unsigned _int64` on Windows). + +``n`` (:class:`int`) [Py_ssize_t] + Convert a Python integer to a C :c:type:`Py_ssize_t`. + +``c`` (:class:`bytes` or :class:`bytearray` of length 1) [char] + Convert a Python byte, represented as a :class:`bytes` or + :class:`bytearray` object of length 1, to a C :c:type:`char`. + + .. versionchanged:: 3.3 Allow :class:`bytearray` objects + +``C`` (:class:`str` of length 1) [int] + Convert a Python character, represented as a :class:`str` object of + length 1, to a C :c:type:`int`. + +``f`` (:class:`float`) [float] + Convert a Python floating point number to a C :c:type:`float`. + +``d`` (:class:`float`) [double] + Convert a Python floating point number to a C :c:type:`double`. + +``D`` (:class:`complex`) [Py_complex] + Convert a Python complex number to a C :c:type:`Py_complex` structure. + +Other objects +------------- + +``O`` (object) [PyObject \*] + Store a Python object (without any conversion) in a C object pointer. The C + program thus receives the actual object that was passed. The object's reference + count is not increased. The pointer stored is not *NULL*. + +``O!`` (object) [*typeobject*, PyObject \*] + Store a Python object in a C object pointer. This is similar to ``O``, but + takes two C arguments: the first is the address of a Python type object, the + second is the address of the C variable (of type :c:type:`PyObject\*`) into which + the object pointer is stored. If the Python object does not have the required + type, :exc:`TypeError` is raised. + +``O&`` (object) [*converter*, *anything*] + Convert a Python object to a C variable through a *converter* function. This + takes two arguments: the first is a function, the second is the address of a C + variable (of arbitrary type), converted to :c:type:`void \*`. The *converter* + function in turn is called as follows:: + + status = converter(object, address); + + where *object* is the Python object to be converted and *address* is the + :c:type:`void\*` argument that was passed to the :c:func:`PyArg_Parse\*` function. + The returned *status* should be ``1`` for a successful conversion and ``0`` if + the conversion has failed. When the conversion fails, the *converter* function + should raise an exception and leave the content of *address* unmodified. + + If the *converter* returns ``Py_CLEANUP_SUPPORTED``, it may get called a + second time if the argument parsing eventually fails, giving the converter a + chance to release any memory that it had already allocated. In this second + call, the *object* parameter will be NULL; *address* will have the same value + as in the original call. + + .. versionchanged:: 3.1 + ``Py_CLEANUP_SUPPORTED`` was added. + +``(items)`` (:class:`tuple`) [*matching-items*] + The object must be a Python sequence whose length is the number of format units + in *items*. The C arguments must correspond to the individual format units in + *items*. Format units for sequences may be nested. + +It is possible to pass "long" integers (integers whose value exceeds the +platform's :const:`LONG_MAX`) however no proper range checking is done --- the +most significant bits are silently truncated when the receiving field is too +small to receive the value (actually, the semantics are inherited from downcasts +in C --- your mileage may vary). + +A few other characters have a meaning in a format string. These may not occur +inside nested parentheses. They are: + +``|`` + Indicates that the remaining arguments in the Python argument list are optional. + The C variables corresponding to optional arguments should be initialized to + their default value --- when an optional argument is not specified, + :c:func:`PyArg_ParseTuple` does not touch the contents of the corresponding C + variable(s). + +``:`` + The list of format units ends here; the string after the colon is used as the + function name in error messages (the "associated value" of the exception that + :c:func:`PyArg_ParseTuple` raises). + +``;`` + The list of format units ends here; the string after the semicolon is used as + the error message *instead* of the default error message. ``:`` and ``;`` + mutually exclude each other. + +Note that any Python object references which are provided to the caller are +*borrowed* references; do not decrement their reference count! + +Additional arguments passed to these functions must be addresses of variables +whose type is determined by the format string; these are used to store values +from the input tuple. There are a few cases, as described in the list of format +units above, where these parameters are used as input values; they should match +what is specified for the corresponding format unit in that case. + +For the conversion to succeed, the *arg* object must match the format +and the format must be exhausted. On success, the +:c:func:`PyArg_Parse\*` functions return true, otherwise they return +false and raise an appropriate exception. When the +:c:func:`PyArg_Parse\*` functions fail due to conversion failure in one +of the format units, the variables at the addresses corresponding to that +and the following format units are left untouched. + +API Functions +------------- + +.. c:function:: int PyArg_ParseTuple(PyObject *args, const char *format, ...) + + Parse the parameters of a function that takes only positional parameters into + local variables. Returns true on success; on failure, it returns false and + raises the appropriate exception. + + +.. c:function:: int PyArg_VaParse(PyObject *args, const char *format, va_list vargs) + + Identical to :c:func:`PyArg_ParseTuple`, except that it accepts a va_list rather + than a variable number of arguments. + + +.. c:function:: int PyArg_ParseTupleAndKeywords(PyObject *args, PyObject *kw, const char *format, char *keywords[], ...) + + Parse the parameters of a function that takes both positional and keyword + parameters into local variables. Returns true on success; on failure, it + returns false and raises the appropriate exception. + + +.. c:function:: int PyArg_VaParseTupleAndKeywords(PyObject *args, PyObject *kw, const char *format, char *keywords[], va_list vargs) + + Identical to :c:func:`PyArg_ParseTupleAndKeywords`, except that it accepts a + va_list rather than a variable number of arguments. + + +.. c:function:: int PyArg_ValidateKeywordArguments(PyObject *) + + Ensure that the keys in the keywords argument dictionary are strings. This + is only needed if :c:func:`PyArg_ParseTupleAndKeywords` is not used, since the + latter already does this check. + + .. versionadded:: 3.2 + + +.. XXX deprecated, will be removed +.. c:function:: int PyArg_Parse(PyObject *args, const char *format, ...) + + Function used to deconstruct the argument lists of "old-style" functions --- + these are functions which use the :const:`METH_OLDARGS` parameter parsing + method. This is not recommended for use in parameter parsing in new code, and + most code in the standard interpreter has been modified to no longer use this + for that purpose. It does remain a convenient way to decompose other tuples, + however, and may continue to be used for that purpose. + + +.. c:function:: int PyArg_UnpackTuple(PyObject *args, const char *name, Py_ssize_t min, Py_ssize_t max, ...) + + A simpler form of parameter retrieval which does not use a format string to + specify the types of the arguments. Functions which use this method to retrieve + their parameters should be declared as :const:`METH_VARARGS` in function or + method tables. The tuple containing the actual parameters should be passed as + *args*; it must actually be a tuple. The length of the tuple must be at least + *min* and no more than *max*; *min* and *max* may be equal. Additional + arguments must be passed to the function, each of which should be a pointer to a + :c:type:`PyObject\*` variable; these will be filled in with the values from + *args*; they will contain borrowed references. The variables which correspond + to optional parameters not given by *args* will not be filled in; these should + be initialized by the caller. This function returns true on success and false if + *args* is not a tuple or contains the wrong number of elements; an exception + will be set if there was a failure. + + This is an example of the use of this function, taken from the sources for the + :mod:`_weakref` helper module for weak references:: + + static PyObject * + weakref_ref(PyObject *self, PyObject *args) + { + PyObject *object; + PyObject *callback = NULL; + PyObject *result = NULL; + + if (PyArg_UnpackTuple(args, "ref", 1, 2, &object, &callback)) { + result = PyWeakref_NewRef(object, callback); + } + return result; + } + + The call to :c:func:`PyArg_UnpackTuple` in this example is entirely equivalent to + this call to :c:func:`PyArg_ParseTuple`:: + + PyArg_ParseTuple(args, "O|O:ref", &object, &callback) + + +--------------- +Building values +--------------- + +.. c:function:: PyObject* Py_BuildValue(const char *format, ...) + + Create a new value based on a format string similar to those accepted by the + :c:func:`PyArg_Parse\*` family of functions and a sequence of values. Returns + the value or *NULL* in the case of an error; an exception will be raised if + *NULL* is returned. + + :c:func:`Py_BuildValue` does not always build a tuple. It builds a tuple only if + its format string contains two or more format units. If the format string is + empty, it returns ``None``; if it contains exactly one format unit, it returns + whatever object is described by that format unit. To force it to return a tuple + of size 0 or one, parenthesize the format string. + + When memory buffers are passed as parameters to supply data to build objects, as + for the ``s`` and ``s#`` formats, the required data is copied. Buffers provided + by the caller are never referenced by the objects created by + :c:func:`Py_BuildValue`. In other words, if your code invokes :c:func:`malloc` + and passes the allocated memory to :c:func:`Py_BuildValue`, your code is + responsible for calling :c:func:`free` for that memory once + :c:func:`Py_BuildValue` returns. + + In the following description, the quoted form is the format unit; the entry in + (round) parentheses is the Python object type that the format unit will return; + and the entry in [square] brackets is the type of the C value(s) to be passed. + + The characters space, tab, colon and comma are ignored in format strings (but + not within format units such as ``s#``). This can be used to make long format + strings a tad more readable. + + ``s`` (:class:`str` or ``None``) [char \*] + Convert a null-terminated C string to a Python :class:`str` object using ``'utf-8'`` + encoding. If the C string pointer is *NULL*, ``None`` is used. + + ``s#`` (:class:`str` or ``None``) [char \*, int] + Convert a C string and its length to a Python :class:`str` object using ``'utf-8'`` + encoding. If the C string pointer is *NULL*, the length is ignored and + ``None`` is returned. + + ``y`` (:class:`bytes`) [char \*] + This converts a C string to a Python :func:`bytes` object. If the C + string pointer is *NULL*, ``None`` is returned. + + ``y#`` (:class:`bytes`) [char \*, int] + This converts a C string and its lengths to a Python object. If the C + string pointer is *NULL*, ``None`` is returned. + + ``z`` (:class:`str` or ``None``) [char \*] + Same as ``s``. + + ``z#`` (:class:`str` or ``None``) [char \*, int] + Same as ``s#``. + + ``u`` (:class:`str`) [Py_UNICODE \*] + Convert a null-terminated buffer of Unicode (UCS-2 or UCS-4) data to a Python + Unicode object. If the Unicode buffer pointer is *NULL*, ``None`` is returned. + + ``u#`` (:class:`str`) [Py_UNICODE \*, int] + Convert a Unicode (UCS-2 or UCS-4) data buffer and its length to a Python + Unicode object. If the Unicode buffer pointer is *NULL*, the length is ignored + and ``None`` is returned. + + ``U`` (:class:`str` or ``None``) [char \*] + Same as ``s``. + + ``U#`` (:class:`str` or ``None``) [char \*, int] + Same as ``s#``. + + ``i`` (:class:`int`) [int] + Convert a plain C :c:type:`int` to a Python integer object. + + ``b`` (:class:`int`) [char] + Convert a plain C :c:type:`char` to a Python integer object. + + ``h`` (:class:`int`) [short int] + Convert a plain C :c:type:`short int` to a Python integer object. + + ``l`` (:class:`int`) [long int] + Convert a C :c:type:`long int` to a Python integer object. + + ``B`` (:class:`int`) [unsigned char] + Convert a C :c:type:`unsigned char` to a Python integer object. + + ``H`` (:class:`int`) [unsigned short int] + Convert a C :c:type:`unsigned short int` to a Python integer object. + + ``I`` (:class:`int`) [unsigned int] + Convert a C :c:type:`unsigned int` to a Python integer object. + + ``k`` (:class:`int`) [unsigned long] + Convert a C :c:type:`unsigned long` to a Python integer object. + + ``L`` (:class:`int`) [PY_LONG_LONG] + Convert a C :c:type:`long long` to a Python integer object. Only available + on platforms that support :c:type:`long long` (or :c:type:`_int64` on + Windows). + + ``K`` (:class:`int`) [unsigned PY_LONG_LONG] + Convert a C :c:type:`unsigned long long` to a Python integer object. Only + available on platforms that support :c:type:`unsigned long long` (or + :c:type:`unsigned _int64` on Windows). + + ``n`` (:class:`int`) [Py_ssize_t] + Convert a C :c:type:`Py_ssize_t` to a Python integer. + + ``c`` (:class:`bytes` of length 1) [char] + Convert a C :c:type:`int` representing a byte to a Python :class:`bytes` object of + length 1. + + ``C`` (:class:`str` of length 1) [int] + Convert a C :c:type:`int` representing a character to Python :class:`str` + object of length 1. + + ``d`` (:class:`float`) [double] + Convert a C :c:type:`double` to a Python floating point number. + + ``f`` (:class:`float`) [float] + Convert a C :c:type:`float` to a Python floating point number. + + ``D`` (:class:`complex`) [Py_complex \*] + Convert a C :c:type:`Py_complex` structure to a Python complex number. + + ``O`` (object) [PyObject \*] + Pass a Python object untouched (except for its reference count, which is + incremented by one). If the object passed in is a *NULL* pointer, it is assumed + that this was caused because the call producing the argument found an error and + set an exception. Therefore, :c:func:`Py_BuildValue` will return *NULL* but won't + raise an exception. If no exception has been raised yet, :exc:`SystemError` is + set. + + ``S`` (object) [PyObject \*] + Same as ``O``. + + ``N`` (object) [PyObject \*] + Same as ``O``, except it doesn't increment the reference count on the object. + Useful when the object is created by a call to an object constructor in the + argument list. + + ``O&`` (object) [*converter*, *anything*] + Convert *anything* to a Python object through a *converter* function. The + function is called with *anything* (which should be compatible with :c:type:`void + \*`) as its argument and should return a "new" Python object, or *NULL* if an + error occurred. + + ``(items)`` (:class:`tuple`) [*matching-items*] + Convert a sequence of C values to a Python tuple with the same number of items. + + ``[items]`` (:class:`list`) [*matching-items*] + Convert a sequence of C values to a Python list with the same number of items. + + ``{items}`` (:class:`dict`) [*matching-items*] + Convert a sequence of C values to a Python dictionary. Each pair of consecutive + C values adds one item to the dictionary, serving as key and value, + respectively. + + If there is an error in the format string, the :exc:`SystemError` exception is + set and *NULL* returned. + +.. c:function:: PyObject* Py_VaBuildValue(const char *format, va_list vargs) + + Identical to :c:func:`Py_BuildValue`, except that it accepts a va_list + rather than a variable number of arguments. diff --git a/lib/cpython-doc/c-api/bool.rst b/lib/cpython-doc/c-api/bool.rst new file mode 100644 --- /dev/null +++ b/lib/cpython-doc/c-api/bool.rst @@ -0,0 +1,46 @@ +.. highlightlang:: c + +.. _boolobjects: + +Boolean Objects +--------------- + +Booleans in Python are implemented as a subclass of integers. There are only +two booleans, :const:`Py_False` and :const:`Py_True`. As such, the normal +creation and deletion functions don't apply to booleans. The following macros +are available, however. + + +.. c:function:: int PyBool_Check(PyObject *o) + + Return true if *o* is of type :c:data:`PyBool_Type`. + + +.. c:var:: PyObject* Py_False + + The Python ``False`` object. This object has no methods. It needs to be + treated just like any other object with respect to reference counts. + + +.. c:var:: PyObject* Py_True + + The Python ``True`` object. This object has no methods. It needs to be treated + just like any other object with respect to reference counts. + + +.. c:macro:: Py_RETURN_FALSE + + Return :const:`Py_False` from a function, properly incrementing its reference + count. + + +.. c:macro:: Py_RETURN_TRUE + + Return :const:`Py_True` from a function, properly incrementing its reference + count. + + +.. c:function:: PyObject* PyBool_FromLong(long v) + + Return a new reference to :const:`Py_True` or :const:`Py_False` depending on the + truth value of *v*. diff --git a/lib/cpython-doc/c-api/buffer.rst b/lib/cpython-doc/c-api/buffer.rst new file mode 100644 --- /dev/null +++ b/lib/cpython-doc/c-api/buffer.rst @@ -0,0 +1,326 @@ +.. highlightlang:: c + +.. _bufferobjects: + +Buffer Protocol +--------------- + +.. sectionauthor:: Greg Stein +.. sectionauthor:: Benjamin Peterson + + +.. index:: + single: buffer interface + +Certain objects available in Python wrap access to an underlying memory +array or *buffer*. Such objects include the built-in :class:`bytes` and +:class:`bytearray`, and some extension types like :class:`array.array`. +Third-party libraries may define their own types for special purposes, such +as image processing or numeric analysis. + +While each of these types have their own semantics, they share the common +characteristic of being backed by a possibly large memory buffer. It is +then desireable, in some situations, to access that buffer directly and +without intermediate copying. + +Python provides such a facility at the C level in the form of the *buffer +protocol*. This protocol has two sides: + +.. index:: single: PyBufferProcs + +- on the producer side, a type can export a "buffer interface" which allows + objects of that type to expose information about their underlying buffer. + This interface is described in the section :ref:`buffer-structs`; + +- on the consumer side, several means are available to obtain a pointer to + the raw underlying data of an object (for example a method parameter). + +Simple objects such as :class:`bytes` and :class:`bytearray` expose their +underlying buffer in byte-oriented form. Other forms are possible; for example, +the elements exposed by a :class:`array.array` can be multi-byte values. + +An example consumer of the buffer interface is the :meth:`~io.BufferedIOBase.write` +method of file objects: any object that can export a series of bytes through +the buffer interface can be written to a file. While :meth:`write` only +needs read-only access to the internal contents of the object passed to it, +other methods such as :meth:`~io.BufferedIOBase.readinto` need write access +to the contents of their argument. The buffer interface allows objects to +selectively allow or reject exporting of read-write and read-only buffers. + +There are two ways for a consumer of the buffer interface to acquire a buffer +over a target object: + +* call :c:func:`PyObject_GetBuffer` with the right parameters; + +* call :c:func:`PyArg_ParseTuple` (or one of its siblings) with one of the + ``y*``, ``w*`` or ``s*`` :ref:`format codes `. + +In both cases, :c:func:`PyBuffer_Release` must be called when the buffer +isn't needed anymore. Failure to do so could lead to various issues such as +resource leaks. + + +The buffer structure +==================== + +Buffer structures (or simply "buffers") are useful as a way to expose the +binary data from another object to the Python programmer. They can also be +used as a zero-copy slicing mechanism. Using their ability to reference a +block of memory, it is possible to expose any data to the Python programmer +quite easily. The memory could be a large, constant array in a C extension, +it could be a raw block of memory for manipulation before passing to an +operating system library, or it could be used to pass around structured data +in its native, in-memory format. + +Contrary to most data types exposed by the Python interpreter, buffers +are not :c:type:`PyObject` pointers but rather simple C structures. This +allows them to be created and copied very simply. When a generic wrapper +around a buffer is needed, a :ref:`memoryview ` object +can be created. + + +.. c:type:: Py_buffer + + .. c:member:: void *buf + + A pointer to the start of the memory for the object. + + .. c:member:: Py_ssize_t len + :noindex: + + The total length of the memory in bytes. + + .. c:member:: int readonly + + An indicator of whether the buffer is read only. + + .. c:member:: const char *format + :noindex: + + A *NULL* terminated string in :mod:`struct` module style syntax giving + the contents of the elements available through the buffer. If this is + *NULL*, ``"B"`` (unsigned bytes) is assumed. + + .. c:member:: int ndim + + The number of dimensions the memory represents as a multi-dimensional + array. If it is 0, :c:data:`strides` and :c:data:`suboffsets` must be + *NULL*. + + .. c:member:: Py_ssize_t *shape + + An array of :c:type:`Py_ssize_t`\s the length of :c:data:`ndim` giving the + shape of the memory as a multi-dimensional array. Note that + ``((*shape)[0] * ... * (*shape)[ndims-1])*itemsize`` should be equal to + :c:data:`len`. + + .. c:member:: Py_ssize_t *strides + + An array of :c:type:`Py_ssize_t`\s the length of :c:data:`ndim` giving the + number of bytes to skip to get to a new element in each dimension. + + .. c:member:: Py_ssize_t *suboffsets + + An array of :c:type:`Py_ssize_t`\s the length of :c:data:`ndim`. If these + suboffset numbers are greater than or equal to 0, then the value stored + along the indicated dimension is a pointer and the suboffset value + dictates how many bytes to add to the pointer after de-referencing. A + suboffset value that it negative indicates that no de-referencing should + occur (striding in a contiguous memory block). + + Here is a function that returns a pointer to the element in an N-D array + pointed to by an N-dimensional index when there are both non-NULL strides + and suboffsets:: + + void *get_item_pointer(int ndim, void *buf, Py_ssize_t *strides, + Py_ssize_t *suboffsets, Py_ssize_t *indices) { + char *pointer = (char*)buf; + int i; + for (i = 0; i < ndim; i++) { + pointer += strides[i] * indices[i]; + if (suboffsets[i] >=0 ) { + pointer = *((char**)pointer) + suboffsets[i]; + } + } + return (void*)pointer; + } + + + .. c:member:: Py_ssize_t itemsize + + This is a storage for the itemsize (in bytes) of each element of the + shared memory. It is technically un-necessary as it can be obtained + using :c:func:`PyBuffer_SizeFromFormat`, however an exporter may know + this information without parsing the format string and it is necessary + to know the itemsize for proper interpretation of striding. Therefore, + storing it is more convenient and faster. + + .. c:member:: void *internal + + This is for use internally by the exporting object. For example, this + might be re-cast as an integer by the exporter and used to store flags + about whether or not the shape, strides, and suboffsets arrays must be + freed when the buffer is released. The consumer should never alter this + value. + + +Buffer-related functions +======================== + + +.. c:function:: int PyObject_CheckBuffer(PyObject *obj) + + Return 1 if *obj* supports the buffer interface otherwise 0. When 1 is + returned, it doesn't guarantee that :c:func:`PyObject_GetBuffer` will + succeed. + + +.. c:function:: int PyObject_GetBuffer(PyObject *obj, Py_buffer *view, int flags) + + Export a view over some internal data from the target object *obj*. + *obj* must not be NULL, and *view* must point to an existing + :c:type:`Py_buffer` structure allocated by the caller (most uses of + this function will simply declare a local variable of type + :c:type:`Py_buffer`). The *flags* argument is a bit field indicating + what kind of buffer is requested. The buffer interface allows + for complicated memory layout possibilities; however, some callers + won't want to handle all the complexity and instead request a simple + view of the target object (using :c:macro:`PyBUF_SIMPLE` for a read-only + view and :c:macro:`PyBUF_WRITABLE` for a read-write view). + + Some exporters may not be able to share memory in every possible way and + may need to raise errors to signal to some consumers that something is + just not possible. These errors should be a :exc:`BufferError` unless + there is another error that is actually causing the problem. The + exporter can use flags information to simplify how much of the + :c:data:`Py_buffer` structure is filled in with non-default values and/or + raise an error if the object can't support a simpler view of its memory. + + On success, 0 is returned and the *view* structure is filled with useful + values. On error, -1 is returned and an exception is raised; the *view* + is left in an undefined state. + + The following are the possible values to the *flags* arguments. + + .. c:macro:: PyBUF_SIMPLE + + This is the default flag. The returned buffer exposes a read-only + memory area. The format of data is assumed to be raw unsigned bytes, + without any particular structure. This is a "stand-alone" flag + constant. It never needs to be '|'d to the others. The exporter will + raise an error if it cannot provide such a contiguous buffer of bytes. + + .. c:macro:: PyBUF_WRITABLE + + Like :c:macro:`PyBUF_SIMPLE`, but the returned buffer is writable. If + the exporter doesn't support writable buffers, an error is raised. + + .. c:macro:: PyBUF_STRIDES + + This implies :c:macro:`PyBUF_ND`. The returned buffer must provide + strides information (i.e. the strides cannot be NULL). This would be + used when the consumer can handle strided, discontiguous arrays. + Handling strides automatically assumes you can handle shape. The + exporter can raise an error if a strided representation of the data is + not possible (i.e. without the suboffsets). + + .. c:macro:: PyBUF_ND + + The returned buffer must provide shape information. The memory will be + assumed C-style contiguous (last dimension varies the fastest). The + exporter may raise an error if it cannot provide this kind of + contiguous buffer. If this is not given then shape will be *NULL*. + + .. c:macro:: PyBUF_C_CONTIGUOUS + PyBUF_F_CONTIGUOUS + PyBUF_ANY_CONTIGUOUS + + These flags indicate that the contiguity returned buffer must be + respectively, C-contiguous (last dimension varies the fastest), Fortran + contiguous (first dimension varies the fastest) or either one. All of + these flags imply :c:macro:`PyBUF_STRIDES` and guarantee that the + strides buffer info structure will be filled in correctly. + + .. c:macro:: PyBUF_INDIRECT + + This flag indicates the returned buffer must have suboffsets + information (which can be NULL if no suboffsets are needed). This can + be used when the consumer can handle indirect array referencing implied + by these suboffsets. This implies :c:macro:`PyBUF_STRIDES`. + + .. c:macro:: PyBUF_FORMAT + + The returned buffer must have true format information if this flag is + provided. This would be used when the consumer is going to be checking + for what 'kind' of data is actually stored. An exporter should always + be able to provide this information if requested. If format is not + explicitly requested then the format must be returned as *NULL* (which + means ``'B'``, or unsigned bytes). + + .. c:macro:: PyBUF_STRIDED + + This is equivalent to ``(PyBUF_STRIDES | PyBUF_WRITABLE)``. + + .. c:macro:: PyBUF_STRIDED_RO + + This is equivalent to ``(PyBUF_STRIDES)``. + + .. c:macro:: PyBUF_RECORDS + + This is equivalent to ``(PyBUF_STRIDES | PyBUF_FORMAT | + PyBUF_WRITABLE)``. + + .. c:macro:: PyBUF_RECORDS_RO + + This is equivalent to ``(PyBUF_STRIDES | PyBUF_FORMAT)``. + + .. c:macro:: PyBUF_FULL + + This is equivalent to ``(PyBUF_INDIRECT | PyBUF_FORMAT | + PyBUF_WRITABLE)``. + + .. c:macro:: PyBUF_FULL_RO + + This is equivalent to ``(PyBUF_INDIRECT | PyBUF_FORMAT)``. + + .. c:macro:: PyBUF_CONTIG + + This is equivalent to ``(PyBUF_ND | PyBUF_WRITABLE)``. + + .. c:macro:: PyBUF_CONTIG_RO + + This is equivalent to ``(PyBUF_ND)``. + + +.. c:function:: void PyBuffer_Release(Py_buffer *view) + + Release the buffer *view*. This should be called when the buffer is no + longer being used as it may free memory from it. + + +.. c:function:: Py_ssize_t PyBuffer_SizeFromFormat(const char *) + + Return the implied :c:data:`~Py_buffer.itemsize` from the struct-stype + :c:data:`~Py_buffer.format`. + + +.. c:function:: int PyBuffer_IsContiguous(Py_buffer *view, char fortran) + + Return 1 if the memory defined by the *view* is C-style (*fortran* is + ``'C'``) or Fortran-style (*fortran* is ``'F'``) contiguous or either one + (*fortran* is ``'A'``). Return 0 otherwise. + + +.. c:function:: void PyBuffer_FillContiguousStrides(int ndim, Py_ssize_t *shape, Py_ssize_t *strides, Py_ssize_t itemsize, char fortran) + + Fill the *strides* array with byte-strides of a contiguous (C-style if + *fortran* is ``'C'`` or Fortran-style if *fortran* is ``'F'``) array of the + given shape with the given number of bytes per element. + + +.. c:function:: int PyBuffer_FillInfo(Py_buffer *view, PyObject *obj, void *buf, Py_ssize_t len, int readonly, int infoflags) + + Fill in a buffer-info structure, *view*, correctly for an exporter that can + only share a contiguous chunk of memory of "unsigned bytes" of the given + length. Return 0 on success and -1 (with raising an error) on error. + diff --git a/lib/cpython-doc/c-api/bytearray.rst b/lib/cpython-doc/c-api/bytearray.rst new file mode 100644 --- /dev/null +++ b/lib/cpython-doc/c-api/bytearray.rst @@ -0,0 +1,86 @@ +.. highlightlang:: c + +.. _bytearrayobjects: + +Byte Array Objects +------------------ + +.. index:: object: bytearray + + +.. c:type:: PyByteArrayObject + + This subtype of :c:type:`PyObject` represents a Python bytearray object. + + +.. c:var:: PyTypeObject PyByteArray_Type + + This instance of :c:type:`PyTypeObject` represents the Python bytearray type; + it is the same object as :class:`bytearray` in the Python layer. + + +Type check macros +^^^^^^^^^^^^^^^^^ + +.. c:function:: int PyByteArray_Check(PyObject *o) + + Return true if the object *o* is a bytearray object or an instance of a + subtype of the bytearray type. + + +.. c:function:: int PyByteArray_CheckExact(PyObject *o) + + Return true if the object *o* is a bytearray object, but not an instance of a + subtype of the bytearray type. + + +Direct API functions +^^^^^^^^^^^^^^^^^^^^ + +.. c:function:: PyObject* PyByteArray_FromObject(PyObject *o) + + Return a new bytearray object from any object, *o*, that implements the + buffer protocol. + + .. XXX expand about the buffer protocol, at least somewhere + + +.. c:function:: PyObject* PyByteArray_FromStringAndSize(const char *string, Py_ssize_t len) + + Create a new bytearray object from *string* and its length, *len*. On + failure, *NULL* is returned. + + +.. c:function:: PyObject* PyByteArray_Concat(PyObject *a, PyObject *b) + + Concat bytearrays *a* and *b* and return a new bytearray with the result. + + +.. c:function:: Py_ssize_t PyByteArray_Size(PyObject *bytearray) + + Return the size of *bytearray* after checking for a *NULL* pointer. + + +.. c:function:: char* PyByteArray_AsString(PyObject *bytearray) + + Return the contents of *bytearray* as a char array after checking for a + *NULL* pointer. + + +.. c:function:: int PyByteArray_Resize(PyObject *bytearray, Py_ssize_t len) + + Resize the internal buffer of *bytearray* to *len*. + +Macros +^^^^^^ + +These macros trade safety for speed and they don't check pointers. + +.. c:function:: char* PyByteArray_AS_STRING(PyObject *bytearray) + + Macro version of :c:func:`PyByteArray_AsString`. + + +.. c:function:: Py_ssize_t PyByteArray_GET_SIZE(PyObject *bytearray) + + Macro version of :c:func:`PyByteArray_Size`. diff --git a/lib/cpython-doc/c-api/bytes.rst b/lib/cpython-doc/c-api/bytes.rst new file mode 100644 --- /dev/null +++ b/lib/cpython-doc/c-api/bytes.rst @@ -0,0 +1,192 @@ +.. highlightlang:: c + +.. _bytesobjects: + +Bytes Objects +------------- + +These functions raise :exc:`TypeError` when expecting a bytes parameter and are +called with a non-bytes parameter. + +.. index:: object: bytes + + +.. c:type:: PyBytesObject + + This subtype of :c:type:`PyObject` represents a Python bytes object. + + +.. c:var:: PyTypeObject PyBytes_Type + + This instance of :c:type:`PyTypeObject` represents the Python bytes type; it + is the same object as :class:`bytes` in the Python layer. + + +.. c:function:: int PyBytes_Check(PyObject *o) + + Return true if the object *o* is a bytes object or an instance of a subtype + of the bytes type. + + +.. c:function:: int PyBytes_CheckExact(PyObject *o) + + Return true if the object *o* is a bytes object, but not an instance of a + subtype of the bytes type. + + +.. c:function:: PyObject* PyBytes_FromString(const char *v) + + Return a new bytes object with a copy of the string *v* as value on success, + and *NULL* on failure. The parameter *v* must not be *NULL*; it will not be + checked. + + +.. c:function:: PyObject* PyBytes_FromStringAndSize(const char *v, Py_ssize_t len) + + Return a new bytes object with a copy of the string *v* as value and length + *len* on success, and *NULL* on failure. If *v* is *NULL*, the contents of + the bytes object are uninitialized. + + +.. c:function:: PyObject* PyBytes_FromFormat(const char *format, ...) + + Take a C :c:func:`printf`\ -style *format* string and a variable number of + arguments, calculate the size of the resulting Python bytes object and return + a bytes object with the values formatted into it. The variable arguments + must be C types and must correspond exactly to the format characters in the + *format* string. The following format characters are allowed: + + .. % XXX: This should be exactly the same as the table in PyErr_Format. + .. % One should just refer to the other. + .. % XXX: The descriptions for %zd and %zu are wrong, but the truth is complicated + .. % because not all compilers support the %z width modifier -- we fake it + .. % when necessary via interpolating PY_FORMAT_SIZE_T. + + +-------------------+---------------+--------------------------------+ + | Format Characters | Type | Comment | + +===================+===============+================================+ + | :attr:`%%` | *n/a* | The literal % character. | + +-------------------+---------------+--------------------------------+ + | :attr:`%c` | int | A single character, | + | | | represented as an C int. | + +-------------------+---------------+--------------------------------+ + | :attr:`%d` | int | Exactly equivalent to | + | | | ``printf("%d")``. | + +-------------------+---------------+--------------------------------+ + | :attr:`%u` | unsigned int | Exactly equivalent to | + | | | ``printf("%u")``. | + +-------------------+---------------+--------------------------------+ + | :attr:`%ld` | long | Exactly equivalent to | + | | | ``printf("%ld")``. | + +-------------------+---------------+--------------------------------+ + | :attr:`%lu` | unsigned long | Exactly equivalent to | + | | | ``printf("%lu")``. | + +-------------------+---------------+--------------------------------+ + | :attr:`%zd` | Py_ssize_t | Exactly equivalent to | + | | | ``printf("%zd")``. | + +-------------------+---------------+--------------------------------+ + | :attr:`%zu` | size_t | Exactly equivalent to | + | | | ``printf("%zu")``. | + +-------------------+---------------+--------------------------------+ + | :attr:`%i` | int | Exactly equivalent to | + | | | ``printf("%i")``. | + +-------------------+---------------+--------------------------------+ + | :attr:`%x` | int | Exactly equivalent to | + | | | ``printf("%x")``. | + +-------------------+---------------+--------------------------------+ + | :attr:`%s` | char\* | A null-terminated C character | + | | | array. | + +-------------------+---------------+--------------------------------+ + | :attr:`%p` | void\* | The hex representation of a C | + | | | pointer. Mostly equivalent to | + | | | ``printf("%p")`` except that | + | | | it is guaranteed to start with | + | | | the literal ``0x`` regardless | + | | | of what the platform's | + | | | ``printf`` yields. | + +-------------------+---------------+--------------------------------+ + + An unrecognized format character causes all the rest of the format string to be + copied as-is to the result string, and any extra arguments discarded. + + +.. c:function:: PyObject* PyBytes_FromFormatV(const char *format, va_list vargs) + + Identical to :c:func:`PyBytes_FromFormat` except that it takes exactly two + arguments. + + +.. c:function:: PyObject* PyBytes_FromObject(PyObject *o) + + Return the bytes representation of object *o* that implements the buffer + protocol. + + +.. c:function:: Py_ssize_t PyBytes_Size(PyObject *o) + + Return the length of the bytes in bytes object *o*. + + +.. c:function:: Py_ssize_t PyBytes_GET_SIZE(PyObject *o) + + Macro form of :c:func:`PyBytes_Size` but without error checking. + + +.. c:function:: char* PyBytes_AsString(PyObject *o) + + Return a NUL-terminated representation of the contents of *o*. The pointer + refers to the internal buffer of *o*, not a copy. The data must not be + modified in any way, unless the string was just created using + ``PyBytes_FromStringAndSize(NULL, size)``. It must not be deallocated. If + *o* is not a string object at all, :c:func:`PyBytes_AsString` returns *NULL* + and raises :exc:`TypeError`. + + +.. c:function:: char* PyBytes_AS_STRING(PyObject *string) + + Macro form of :c:func:`PyBytes_AsString` but without error checking. + + +.. c:function:: int PyBytes_AsStringAndSize(PyObject *obj, char **buffer, Py_ssize_t *length) + + Return a NUL-terminated representation of the contents of the object *obj* + through the output variables *buffer* and *length*. + + If *length* is *NULL*, the resulting buffer may not contain NUL characters; + if it does, the function returns ``-1`` and a :exc:`TypeError` is raised. + + The buffer refers to an internal string buffer of *obj*, not a copy. The data + must not be modified in any way, unless the string was just created using + ``PyBytes_FromStringAndSize(NULL, size)``. It must not be deallocated. If + *string* is not a string object at all, :c:func:`PyBytes_AsStringAndSize` + returns ``-1`` and raises :exc:`TypeError`. + + +.. c:function:: void PyBytes_Concat(PyObject **bytes, PyObject *newpart) + + Create a new bytes object in *\*bytes* containing the contents of *newpart* + appended to *bytes*; the caller will own the new reference. The reference to + the old value of *bytes* will be stolen. If the new string cannot be + created, the old reference to *bytes* will still be discarded and the value + of *\*bytes* will be set to *NULL*; the appropriate exception will be set. + + +.. c:function:: void PyBytes_ConcatAndDel(PyObject **bytes, PyObject *newpart) + + Create a new string object in *\*bytes* containing the contents of *newpart* + appended to *bytes*. This version decrements the reference count of + *newpart*. + + +.. c:function:: int _PyBytes_Resize(PyObject **bytes, Py_ssize_t newsize) + + A way to resize a bytes object even though it is "immutable". Only use this + to build up a brand new bytes object; don't use this if the bytes may already + be known in other parts of the code. It is an error to call this function if + the refcount on the input bytes object is not one. Pass the address of an + existing bytes object as an lvalue (it may be written into), and the new size + desired. On success, *\*bytes* holds the resized bytes object and ``0`` is + returned; the address in *\*bytes* may differ from its input value. If the + reallocation fails, the original bytes object at *\*bytes* is deallocated, + *\*bytes* is set to *NULL*, a memory exception is set, and ``-1`` is + returned. diff --git a/lib/cpython-doc/c-api/capsule.rst b/lib/cpython-doc/c-api/capsule.rst new file mode 100644 --- /dev/null +++ b/lib/cpython-doc/c-api/capsule.rst @@ -0,0 +1,150 @@ +.. highlightlang:: c + +.. _capsules: + +Capsules +-------- + +.. index:: object: Capsule + +Refer to :ref:`using-capsules` for more information on using these objects. + + +.. c:type:: PyCapsule + + This subtype of :c:type:`PyObject` represents an opaque value, useful for C + extension modules who need to pass an opaque value (as a :c:type:`void\*` + pointer) through Python code to other C code. It is often used to make a C + function pointer defined in one module available to other modules, so the + regular import mechanism can be used to access C APIs defined in dynamically + loaded modules. + +.. c:type:: PyCapsule_Destructor + + The type of a destructor callback for a capsule. Defined as:: + + typedef void (*PyCapsule_Destructor)(PyObject *); + + See :c:func:`PyCapsule_New` for the semantics of PyCapsule_Destructor + callbacks. + + +.. c:function:: int PyCapsule_CheckExact(PyObject *p) + + Return true if its argument is a :c:type:`PyCapsule`. + + +.. c:function:: PyObject* PyCapsule_New(void *pointer, const char *name, PyCapsule_Destructor destructor) + + Create a :c:type:`PyCapsule` encapsulating the *pointer*. The *pointer* + argument may not be *NULL*. + + On failure, set an exception and return *NULL*. + + The *name* string may either be *NULL* or a pointer to a valid C string. If + non-*NULL*, this string must outlive the capsule. (Though it is permitted to + free it inside the *destructor*.) + + If the *destructor* argument is not *NULL*, it will be called with the + capsule as its argument when it is destroyed. + + If this capsule will be stored as an attribute of a module, the *name* should + be specified as ``modulename.attributename``. This will enable other modules + to import the capsule using :c:func:`PyCapsule_Import`. + + +.. c:function:: void* PyCapsule_GetPointer(PyObject *capsule, const char *name) + + Retrieve the *pointer* stored in the capsule. On failure, set an exception + and return *NULL*. + + The *name* parameter must compare exactly to the name stored in the capsule. + If the name stored in the capsule is *NULL*, the *name* passed in must also + be *NULL*. Python uses the C function :c:func:`strcmp` to compare capsule + names. + + +.. c:function:: PyCapsule_Destructor PyCapsule_GetDestructor(PyObject *capsule) + + Return the current destructor stored in the capsule. On failure, set an + exception and return *NULL*. + + It is legal for a capsule to have a *NULL* destructor. This makes a *NULL* + return code somewhat ambiguous; use :c:func:`PyCapsule_IsValid` or + :c:func:`PyErr_Occurred` to disambiguate. + + +.. c:function:: void* PyCapsule_GetContext(PyObject *capsule) + + Return the current context stored in the capsule. On failure, set an + exception and return *NULL*. + + It is legal for a capsule to have a *NULL* context. This makes a *NULL* + return code somewhat ambiguous; use :c:func:`PyCapsule_IsValid` or + :c:func:`PyErr_Occurred` to disambiguate. + + +.. c:function:: const char* PyCapsule_GetName(PyObject *capsule) + + Return the current name stored in the capsule. On failure, set an exception + and return *NULL*. + + It is legal for a capsule to have a *NULL* name. This makes a *NULL* return + code somewhat ambiguous; use :c:func:`PyCapsule_IsValid` or + :c:func:`PyErr_Occurred` to disambiguate. + + +.. c:function:: void* PyCapsule_Import(const char *name, int no_block) + + Import a pointer to a C object from a capsule attribute in a module. The + *name* parameter should specify the full name to the attribute, as in + ``module.attribute``. The *name* stored in the capsule must match this + string exactly. If *no_block* is true, import the module without blocking + (using :c:func:`PyImport_ImportModuleNoBlock`). If *no_block* is false, + import the module conventionally (using :c:func:`PyImport_ImportModule`). + + Return the capsule's internal *pointer* on success. On failure, set an + exception and return *NULL*. However, if :c:func:`PyCapsule_Import` failed to + import the module, and *no_block* was true, no exception is set. + +.. c:function:: int PyCapsule_IsValid(PyObject *capsule, const char *name) + + Determines whether or not *capsule* is a valid capsule. A valid capsule is + non-*NULL*, passes :c:func:`PyCapsule_CheckExact`, has a non-*NULL* pointer + stored in it, and its internal name matches the *name* parameter. (See + :c:func:`PyCapsule_GetPointer` for information on how capsule names are + compared.) + + In other words, if :c:func:`PyCapsule_IsValid` returns a true value, calls to + any of the accessors (any function starting with :c:func:`PyCapsule_Get`) are + guaranteed to succeed. + + Return a nonzero value if the object is valid and matches the name passed in. + Return 0 otherwise. This function will not fail. + +.. c:function:: int PyCapsule_SetContext(PyObject *capsule, void *context) + + Set the context pointer inside *capsule* to *context*. + + Return 0 on success. Return nonzero and set an exception on failure. + +.. c:function:: int PyCapsule_SetDestructor(PyObject *capsule, PyCapsule_Destructor destructor) + + Set the destructor inside *capsule* to *destructor*. + + Return 0 on success. Return nonzero and set an exception on failure. + +.. c:function:: int PyCapsule_SetName(PyObject *capsule, const char *name) + + Set the name inside *capsule* to *name*. If non-*NULL*, the name must + outlive the capsule. If the previous *name* stored in the capsule was not + *NULL*, no attempt is made to free it. + + Return 0 on success. Return nonzero and set an exception on failure. + +.. c:function:: int PyCapsule_SetPointer(PyObject *capsule, void *pointer) + + Set the void pointer inside *capsule* to *pointer*. The pointer may not be + *NULL*. + + Return 0 on success. Return nonzero and set an exception on failure. diff --git a/lib/cpython-doc/c-api/cell.rst b/lib/cpython-doc/c-api/cell.rst new file mode 100644 --- /dev/null +++ b/lib/cpython-doc/c-api/cell.rst @@ -0,0 +1,62 @@ +.. highlightlang:: c + +.. _cell-objects: + +Cell Objects +------------ + +"Cell" objects are used to implement variables referenced by multiple scopes. +For each such variable, a cell object is created to store the value; the local +variables of each stack frame that references the value contains a reference to +the cells from outer scopes which also use that variable. When the value is +accessed, the value contained in the cell is used instead of the cell object +itself. This de-referencing of the cell object requires support from the +generated byte-code; these are not automatically de-referenced when accessed. +Cell objects are not likely to be useful elsewhere. + + +.. c:type:: PyCellObject + + The C structure used for cell objects. + + +.. c:var:: PyTypeObject PyCell_Type + + The type object corresponding to cell objects. + + +.. c:function:: int PyCell_Check(ob) + + Return true if *ob* is a cell object; *ob* must not be *NULL*. + + +.. c:function:: PyObject* PyCell_New(PyObject *ob) + + Create and return a new cell object containing the value *ob*. The parameter may + be *NULL*. + + +.. c:function:: PyObject* PyCell_Get(PyObject *cell) + + Return the contents of the cell *cell*. + + +.. c:function:: PyObject* PyCell_GET(PyObject *cell) + + Return the contents of the cell *cell*, but without checking that *cell* is + non-*NULL* and a cell object. + + +.. c:function:: int PyCell_Set(PyObject *cell, PyObject *value) + + Set the contents of the cell object *cell* to *value*. This releases the + reference to any current content of the cell. *value* may be *NULL*. *cell* + must be non-*NULL*; if it is not a cell object, ``-1`` will be returned. On + success, ``0`` will be returned. + + +.. c:function:: void PyCell_SET(PyObject *cell, PyObject *value) + + Sets the value of the cell object *cell* to *value*. No reference counts are + adjusted, and no checks are made for safety; *cell* must be non-*NULL* and must + be a cell object. diff --git a/lib/cpython-doc/c-api/code.rst b/lib/cpython-doc/c-api/code.rst new file mode 100644 --- /dev/null +++ b/lib/cpython-doc/c-api/code.rst @@ -0,0 +1,50 @@ +.. highlightlang:: c + +.. _codeobjects: + +Code Objects +------------ + +.. sectionauthor:: Jeffrey Yasskin + + +.. index:: + object: code + +Code objects are a low-level detail of the CPython implementation. +Each one represents a chunk of executable code that hasn't yet been +bound into a function. + +.. c:type:: PyCodeObject + + The C structure of the objects used to describe code objects. The + fields of this type are subject to change at any time. + + +.. c:var:: PyTypeObject PyCode_Type + + This is an instance of :c:type:`PyTypeObject` representing the Python + :class:`code` type. + + +.. c:function:: int PyCode_Check(PyObject *co) + + Return true if *co* is a :class:`code` object + +.. c:function:: int PyCode_GetNumFree(PyObject *co) + + Return the number of free variables in *co*. + +.. c:function:: PyCodeObject *PyCode_New(int argcount, int nlocals, int stacksize, int flags, PyObject *code, PyObject *consts, PyObject *names, PyObject *varnames, PyObject *freevars, PyObject *cellvars, PyObject *filename, PyObject *name, int firstlineno, PyObject *lnotab) + + Return a new code object. If you need a dummy code object to + create a frame, use :c:func:`PyCode_NewEmpty` instead. Calling + :c:func:`PyCode_New` directly can bind you to a precise Python + version since the definition of the bytecode changes often. + + +.. c:function:: int PyCode_NewEmpty(const char *filename, const char *funcname, int firstlineno) + + Return a new empty code object with the specified filename, + function name, and first line number. It is illegal to + :func:`exec` or :func:`eval` the resulting code object. diff --git a/lib/cpython-doc/c-api/codec.rst b/lib/cpython-doc/c-api/codec.rst new file mode 100644 --- /dev/null +++ b/lib/cpython-doc/c-api/codec.rst @@ -0,0 +1,118 @@ +.. _codec-registry: + +Codec registry and support functions +==================================== + +.. c:function:: int PyCodec_Register(PyObject *search_function) + + Register a new codec search function. + + As side effect, this tries to load the :mod:`encodings` package, if not yet + done, to make sure that it is always first in the list of search functions. + +.. c:function:: int PyCodec_KnownEncoding(const char *encoding) + + Return ``1`` or ``0`` depending on whether there is a registered codec for + the given *encoding*. + +.. c:function:: PyObject* PyCodec_Encode(PyObject *object, const char *encoding, const char *errors) + + Generic codec based encoding API. + + *object* is passed through the encoder function found for the given + *encoding* using the error handling method defined by *errors*. *errors* may + be *NULL* to use the default method defined for the codec. Raises a + :exc:`LookupError` if no encoder can be found. + +.. c:function:: PyObject* PyCodec_Decode(PyObject *object, const char *encoding, const char *errors) + + Generic codec based decoding API. + + *object* is passed through the decoder function found for the given + *encoding* using the error handling method defined by *errors*. *errors* may + be *NULL* to use the default method defined for the codec. Raises a + :exc:`LookupError` if no encoder can be found. + + +Codec lookup API +---------------- + +In the following functions, the *encoding* string is looked up converted to all +lower-case characters, which makes encodings looked up through this mechanism +effectively case-insensitive. If no codec is found, a :exc:`KeyError` is set +and *NULL* returned. + +.. c:function:: PyObject* PyCodec_Encoder(const char *encoding) + + Get an encoder function for the given *encoding*. + +.. c:function:: PyObject* PyCodec_Decoder(const char *encoding) + + Get a decoder function for the given *encoding*. + +.. c:function:: PyObject* PyCodec_IncrementalEncoder(const char *encoding, const char *errors) + + Get an :class:`IncrementalEncoder` object for the given *encoding*. + +.. c:function:: PyObject* PyCodec_IncrementalDecoder(const char *encoding, const char *errors) + + Get an :class:`IncrementalDecoder` object for the given *encoding*. + +.. c:function:: PyObject* PyCodec_StreamReader(const char *encoding, PyObject *stream, const char *errors) + + Get a :class:`StreamReader` factory function for the given *encoding*. + +.. c:function:: PyObject* PyCodec_StreamWriter(const char *encoding, PyObject *stream, const char *errors) + + Get a :class:`StreamWriter` factory function for the given *encoding*. + + +Registry API for Unicode encoding error handlers +------------------------------------------------ + +.. c:function:: int PyCodec_RegisterError(const char *name, PyObject *error) + + Register the error handling callback function *error* under the given *name*. + This callback function will be called by a codec when it encounters + unencodable characters/undecodable bytes and *name* is specified as the error + parameter in the call to the encode/decode function. + + The callback gets a single argument, an instance of + :exc:`UnicodeEncodeError`, :exc:`UnicodeDecodeError` or + :exc:`UnicodeTranslateError` that holds information about the problematic + sequence of characters or bytes and their offset in the original string (see + :ref:`unicodeexceptions` for functions to extract this information). The + callback must either raise the given exception, or return a two-item tuple + containing the replacement for the problematic sequence, and an integer + giving the offset in the original string at which encoding/decoding should be + resumed. + + Return ``0`` on success, ``-1`` on error. + +.. c:function:: PyObject* PyCodec_LookupError(const char *name) + + Lookup the error handling callback function registered under *name*. As a + special case *NULL* can be passed, in which case the error handling callback + for "strict" will be returned. + +.. c:function:: PyObject* PyCodec_StrictErrors(PyObject *exc) + + Raise *exc* as an exception. + +.. c:function:: PyObject* PyCodec_IgnoreErrors(PyObject *exc) + + Ignore the unicode error, skipping the faulty input. + +.. c:function:: PyObject* PyCodec_ReplaceErrors(PyObject *exc) + + Replace the unicode encode error with ``?`` or ``U+FFFD``. + +.. c:function:: PyObject* PyCodec_XMLCharRefReplaceErrors(PyObject *exc) + + Replace the unicode encode error with XML character references. + +.. c:function:: PyObject* PyCodec_BackslashReplaceErrors(PyObject *exc) + + Replace the unicode encode error with backslash escapes (``\x``, ``\u`` and + ``\U``). + diff --git a/lib/cpython-doc/c-api/complex.rst b/lib/cpython-doc/c-api/complex.rst new file mode 100644 --- /dev/null +++ b/lib/cpython-doc/c-api/complex.rst @@ -0,0 +1,132 @@ +.. highlightlang:: c + +.. _complexobjects: + +Complex Number Objects +---------------------- + +.. index:: object: complex number + +Python's complex number objects are implemented as two distinct types when +viewed from the C API: one is the Python object exposed to Python programs, and +the other is a C structure which represents the actual complex number value. +The API provides functions for working with both. + + +Complex Numbers as C Structures +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Note that the functions which accept these structures as parameters and return +them as results do so *by value* rather than dereferencing them through +pointers. This is consistent throughout the API. + + +.. c:type:: Py_complex + + The C structure which corresponds to the value portion of a Python complex + number object. Most of the functions for dealing with complex number objects + use structures of this type as input or output values, as appropriate. It is + defined as:: + + typedef struct { + double real; + double imag; + } Py_complex; + + +.. c:function:: Py_complex _Py_c_sum(Py_complex left, Py_complex right) + + Return the sum of two complex numbers, using the C :c:type:`Py_complex` + representation. + + +.. c:function:: Py_complex _Py_c_diff(Py_complex left, Py_complex right) + + Return the difference between two complex numbers, using the C + :c:type:`Py_complex` representation. + + +.. c:function:: Py_complex _Py_c_neg(Py_complex complex) + + Return the negation of the complex number *complex*, using the C + :c:type:`Py_complex` representation. + + +.. c:function:: Py_complex _Py_c_prod(Py_complex left, Py_complex right) + + Return the product of two complex numbers, using the C :c:type:`Py_complex` + representation. + + +.. c:function:: Py_complex _Py_c_quot(Py_complex dividend, Py_complex divisor) + + Return the quotient of two complex numbers, using the C :c:type:`Py_complex` + representation. + + If *divisor* is null, this method returns zero and sets + :c:data:`errno` to :c:data:`EDOM`. + + +.. c:function:: Py_complex _Py_c_pow(Py_complex num, Py_complex exp) + + Return the exponentiation of *num* by *exp*, using the C :c:type:`Py_complex` + representation. + + If *num* is null and *exp* is not a positive real number, + this method returns zero and sets :c:data:`errno` to :c:data:`EDOM`. + + +Complex Numbers as Python Objects +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + + +.. c:type:: PyComplexObject + + This subtype of :c:type:`PyObject` represents a Python complex number object. + + +.. c:var:: PyTypeObject PyComplex_Type + + This instance of :c:type:`PyTypeObject` represents the Python complex number + type. It is the same object as :class:`complex` in the Python layer. + + +.. c:function:: int PyComplex_Check(PyObject *p) + + Return true if its argument is a :c:type:`PyComplexObject` or a subtype of + :c:type:`PyComplexObject`. + + +.. c:function:: int PyComplex_CheckExact(PyObject *p) + + Return true if its argument is a :c:type:`PyComplexObject`, but not a subtype of + :c:type:`PyComplexObject`. + + +.. c:function:: PyObject* PyComplex_FromCComplex(Py_complex v) + + Create a new Python complex number object from a C :c:type:`Py_complex` value. + + +.. c:function:: PyObject* PyComplex_FromDoubles(double real, double imag) + + Return a new :c:type:`PyComplexObject` object from *real* and *imag*. + + +.. c:function:: double PyComplex_RealAsDouble(PyObject *op) + + Return the real part of *op* as a C :c:type:`double`. + + +.. c:function:: double PyComplex_ImagAsDouble(PyObject *op) + + Return the imaginary part of *op* as a C :c:type:`double`. + + +.. c:function:: Py_complex PyComplex_AsCComplex(PyObject *op) + + Return the :c:type:`Py_complex` value of the complex number *op*. + + If *op* is not a Python complex number object but has a :meth:`__complex__` + method, this method will first be called to convert *op* to a Python complex + number object. Upon failure, this method returns ``-1.0`` as a real value. diff --git a/lib/cpython-doc/c-api/concrete.rst b/lib/cpython-doc/c-api/concrete.rst new file mode 100644 --- /dev/null +++ b/lib/cpython-doc/c-api/concrete.rst @@ -0,0 +1,108 @@ +.. highlightlang:: c + + +.. _concrete: + +********************** +Concrete Objects Layer +********************** + +The functions in this chapter are specific to certain Python object types. +Passing them an object of the wrong type is not a good idea; if you receive an +object from a Python program and you are not sure that it has the right type, +you must perform a type check first; for example, to check that an object is a +dictionary, use :c:func:`PyDict_Check`. The chapter is structured like the +"family tree" of Python object types. + +.. warning:: + + While the functions described in this chapter carefully check the type of the + objects which are passed in, many of them do not check for *NULL* being passed + instead of a valid object. Allowing *NULL* to be passed in can cause memory + access violations and immediate termination of the interpreter. + + +.. _fundamental: + +Fundamental Objects +=================== + +This section describes Python type objects and the singleton object ``None``. + +.. toctree:: + + type.rst + none.rst + + +.. _numericobjects: + +Numeric Objects +=============== + +.. index:: object: numeric + +.. toctree:: + + long.rst + bool.rst + float.rst + complex.rst + + +.. _sequenceobjects: + +Sequence Objects +================ + +.. index:: object: sequence + +Generic operations on sequence objects were discussed in the previous chapter; +this section deals with the specific kinds of sequence objects that are +intrinsic to the Python language. + +.. XXX sort out unicode, str, bytes and bytearray + +.. toctree:: + + bytes.rst + bytearray.rst + unicode.rst + tuple.rst + list.rst + + +.. _mapobjects: + +Mapping Objects +=============== + +.. index:: object: mapping + +.. toctree:: + + dict.rst + + +.. _otherobjects: + +Other Objects +============= + +.. toctree:: + + set.rst + function.rst + method.rst + file.rst + module.rst + iterator.rst + descriptor.rst + slice.rst + memoryview.rst + weakref.rst + capsule.rst + cell.rst + gen.rst + datetime.rst + code.rst diff --git a/lib/cpython-doc/c-api/conversion.rst b/lib/cpython-doc/c-api/conversion.rst new file mode 100644 --- /dev/null +++ b/lib/cpython-doc/c-api/conversion.rst @@ -0,0 +1,131 @@ +.. highlightlang:: c + +.. _string-conversion: + +String conversion and formatting +================================ + +Functions for number conversion and formatted string output. + + +.. c:function:: int PyOS_snprintf(char *str, size_t size, const char *format, ...) + + Output not more than *size* bytes to *str* according to the format string + *format* and the extra arguments. See the Unix man page :manpage:`snprintf(2)`. + + +.. c:function:: int PyOS_vsnprintf(char *str, size_t size, const char *format, va_list va) + + Output not more than *size* bytes to *str* according to the format string + *format* and the variable argument list *va*. Unix man page + :manpage:`vsnprintf(2)`. + +:c:func:`PyOS_snprintf` and :c:func:`PyOS_vsnprintf` wrap the Standard C library +functions :c:func:`snprintf` and :c:func:`vsnprintf`. Their purpose is to +guarantee consistent behavior in corner cases, which the Standard C functions do +not. + +The wrappers ensure that *str*[*size*-1] is always ``'\0'`` upon return. They +never write more than *size* bytes (including the trailing ``'\0'``) into str. +Both functions require that ``str != NULL``, ``size > 0`` and ``format != +NULL``. + +If the platform doesn't have :c:func:`vsnprintf` and the buffer size needed to +avoid truncation exceeds *size* by more than 512 bytes, Python aborts with a +*Py_FatalError*. + +The return value (*rv*) for these functions should be interpreted as follows: + +* When ``0 <= rv < size``, the output conversion was successful and *rv* + characters were written to *str* (excluding the trailing ``'\0'`` byte at + *str*[*rv*]). + +* When ``rv >= size``, the output conversion was truncated and a buffer with + ``rv + 1`` bytes would have been needed to succeed. *str*[*size*-1] is ``'\0'`` + in this case. + +* When ``rv < 0``, "something bad happened." *str*[*size*-1] is ``'\0'`` in + this case too, but the rest of *str* is undefined. The exact cause of the error + depends on the underlying platform. + +The following functions provide locale-independent string to number conversions. + + +.. c:function:: double PyOS_string_to_double(const char *s, char **endptr, PyObject *overflow_exception) + + Convert a string ``s`` to a :c:type:`double`, raising a Python + exception on failure. The set of accepted strings corresponds to + the set of strings accepted by Python's :func:`float` constructor, + except that ``s`` must not have leading or trailing whitespace. + The conversion is independent of the current locale. + + If ``endptr`` is ``NULL``, convert the whole string. Raise + ValueError and return ``-1.0`` if the string is not a valid + representation of a floating-point number. + + If endptr is not ``NULL``, convert as much of the string as + possible and set ``*endptr`` to point to the first unconverted + character. If no initial segment of the string is the valid + representation of a floating-point number, set ``*endptr`` to point + to the beginning of the string, raise ValueError, and return + ``-1.0``. + + If ``s`` represents a value that is too large to store in a float + (for example, ``"1e500"`` is such a string on many platforms) then + if ``overflow_exception`` is ``NULL`` return ``Py_HUGE_VAL`` (with + an appropriate sign) and don't set any exception. Otherwise, + ``overflow_exception`` must point to a Python exception object; + raise that exception and return ``-1.0``. In both cases, set + ``*endptr`` to point to the first character after the converted value. + + If any other error occurs during the conversion (for example an + out-of-memory error), set the appropriate Python exception and + return ``-1.0``. + + .. versionadded:: 3.1 + + +.. c:function:: char* PyOS_double_to_string(double val, char format_code, int precision, int flags, int *ptype) + + Convert a :c:type:`double` *val* to a string using supplied + *format_code*, *precision*, and *flags*. + + *format_code* must be one of ``'e'``, ``'E'``, ``'f'``, ``'F'``, + ``'g'``, ``'G'`` or ``'r'``. For ``'r'``, the supplied *precision* + must be 0 and is ignored. The ``'r'`` format code specifies the + standard :func:`repr` format. + + *flags* can be zero or more of the values *Py_DTSF_SIGN*, + *Py_DTSF_ADD_DOT_0*, or *Py_DTSF_ALT*, or-ed together: + + * *Py_DTSF_SIGN* means to always precede the returned string with a sign + character, even if *val* is non-negative. + + * *Py_DTSF_ADD_DOT_0* means to ensure that the returned string will not look + like an integer. + + * *Py_DTSF_ALT* means to apply "alternate" formatting rules. See the + documentation for the :c:func:`PyOS_snprintf` ``'#'`` specifier for + details. + + If *ptype* is non-NULL, then the value it points to will be set to one of + *Py_DTST_FINITE*, *Py_DTST_INFINITE*, or *Py_DTST_NAN*, signifying that + *val* is a finite number, an infinite number, or not a number, respectively. + + The return value is a pointer to *buffer* with the converted string or + *NULL* if the conversion failed. The caller is responsible for freeing the + returned string by calling :c:func:`PyMem_Free`. + + .. versionadded:: 3.1 + + +.. c:function:: char* PyOS_stricmp(char *s1, char *s2) + + Case insensitive comparison of strings. The function works almost + identically to :c:func:`strcmp` except that it ignores the case. + + +.. c:function:: char* PyOS_strnicmp(char *s1, char *s2, Py_ssize_t size) + + Case insensitive comparison of strings. The function works almost + identically to :c:func:`strncmp` except that it ignores the case. diff --git a/lib/cpython-doc/c-api/datetime.rst b/lib/cpython-doc/c-api/datetime.rst new file mode 100644 --- /dev/null +++ b/lib/cpython-doc/c-api/datetime.rst @@ -0,0 +1,184 @@ +.. highlightlang:: c + +.. _datetimeobjects: + +DateTime Objects +---------------- + +Various date and time objects are supplied by the :mod:`datetime` module. +Before using any of these functions, the header file :file:`datetime.h` must be +included in your source (note that this is not included by :file:`Python.h`), +and the macro :c:macro:`PyDateTime_IMPORT` must be invoked, usually as part of +the module initialisation function. The macro puts a pointer to a C structure +into a static variable, :c:data:`PyDateTimeAPI`, that is used by the following +macros. + +Type-check macros: + +.. c:function:: int PyDate_Check(PyObject *ob) + + Return true if *ob* is of type :c:data:`PyDateTime_DateType` or a subtype of + :c:data:`PyDateTime_DateType`. *ob* must not be *NULL*. + + +.. c:function:: int PyDate_CheckExact(PyObject *ob) + + Return true if *ob* is of type :c:data:`PyDateTime_DateType`. *ob* must not be + *NULL*. + + +.. c:function:: int PyDateTime_Check(PyObject *ob) + + Return true if *ob* is of type :c:data:`PyDateTime_DateTimeType` or a subtype of + :c:data:`PyDateTime_DateTimeType`. *ob* must not be *NULL*. + + +.. c:function:: int PyDateTime_CheckExact(PyObject *ob) + + Return true if *ob* is of type :c:data:`PyDateTime_DateTimeType`. *ob* must not + be *NULL*. + + +.. c:function:: int PyTime_Check(PyObject *ob) + + Return true if *ob* is of type :c:data:`PyDateTime_TimeType` or a subtype of + :c:data:`PyDateTime_TimeType`. *ob* must not be *NULL*. + + +.. c:function:: int PyTime_CheckExact(PyObject *ob) + + Return true if *ob* is of type :c:data:`PyDateTime_TimeType`. *ob* must not be + *NULL*. + + +.. c:function:: int PyDelta_Check(PyObject *ob) + + Return true if *ob* is of type :c:data:`PyDateTime_DeltaType` or a subtype of + :c:data:`PyDateTime_DeltaType`. *ob* must not be *NULL*. + + +.. c:function:: int PyDelta_CheckExact(PyObject *ob) + + Return true if *ob* is of type :c:data:`PyDateTime_DeltaType`. *ob* must not be + *NULL*. + + +.. c:function:: int PyTZInfo_Check(PyObject *ob) + + Return true if *ob* is of type :c:data:`PyDateTime_TZInfoType` or a subtype of + :c:data:`PyDateTime_TZInfoType`. *ob* must not be *NULL*. + + +.. c:function:: int PyTZInfo_CheckExact(PyObject *ob) + + Return true if *ob* is of type :c:data:`PyDateTime_TZInfoType`. *ob* must not be + *NULL*. + + +Macros to create objects: + +.. c:function:: PyObject* PyDate_FromDate(int year, int month, int day) + + Return a ``datetime.date`` object with the specified year, month and day. + + +.. c:function:: PyObject* PyDateTime_FromDateAndTime(int year, int month, int day, int hour, int minute, int second, int usecond) + + Return a ``datetime.datetime`` object with the specified year, month, day, hour, + minute, second and microsecond. + + +.. c:function:: PyObject* PyTime_FromTime(int hour, int minute, int second, int usecond) + + Return a ``datetime.time`` object with the specified hour, minute, second and + microsecond. + + +.. c:function:: PyObject* PyDelta_FromDSU(int days, int seconds, int useconds) + + Return a ``datetime.timedelta`` object representing the given number of days, + seconds and microseconds. Normalization is performed so that the resulting + number of microseconds and seconds lie in the ranges documented for + ``datetime.timedelta`` objects. + + +Macros to extract fields from date objects. The argument must be an instance of +:c:data:`PyDateTime_Date`, including subclasses (such as +:c:data:`PyDateTime_DateTime`). The argument must not be *NULL*, and the type is +not checked: + +.. c:function:: int PyDateTime_GET_YEAR(PyDateTime_Date *o) + + Return the year, as a positive int. + + +.. c:function:: int PyDateTime_GET_MONTH(PyDateTime_Date *o) + + Return the month, as an int from 1 through 12. + + +.. c:function:: int PyDateTime_GET_DAY(PyDateTime_Date *o) + + Return the day, as an int from 1 through 31. + + +Macros to extract fields from datetime objects. The argument must be an +instance of :c:data:`PyDateTime_DateTime`, including subclasses. The argument +must not be *NULL*, and the type is not checked: + +.. c:function:: int PyDateTime_DATE_GET_HOUR(PyDateTime_DateTime *o) + + Return the hour, as an int from 0 through 23. + + +.. c:function:: int PyDateTime_DATE_GET_MINUTE(PyDateTime_DateTime *o) + + Return the minute, as an int from 0 through 59. + + +.. c:function:: int PyDateTime_DATE_GET_SECOND(PyDateTime_DateTime *o) + + Return the second, as an int from 0 through 59. + + +.. c:function:: int PyDateTime_DATE_GET_MICROSECOND(PyDateTime_DateTime *o) + + Return the microsecond, as an int from 0 through 999999. + + +Macros to extract fields from time objects. The argument must be an instance of +:c:data:`PyDateTime_Time`, including subclasses. The argument must not be *NULL*, +and the type is not checked: + +.. c:function:: int PyDateTime_TIME_GET_HOUR(PyDateTime_Time *o) + + Return the hour, as an int from 0 through 23. + + +.. c:function:: int PyDateTime_TIME_GET_MINUTE(PyDateTime_Time *o) + + Return the minute, as an int from 0 through 59. + + +.. c:function:: int PyDateTime_TIME_GET_SECOND(PyDateTime_Time *o) + + Return the second, as an int from 0 through 59. + + +.. c:function:: int PyDateTime_TIME_GET_MICROSECOND(PyDateTime_Time *o) + + Return the microsecond, as an int from 0 through 999999. + + +Macros for the convenience of modules implementing the DB API: + +.. c:function:: PyObject* PyDateTime_FromTimestamp(PyObject *args) + + Create and return a new ``datetime.datetime`` object given an argument tuple + suitable for passing to ``datetime.datetime.fromtimestamp()``. + + +.. c:function:: PyObject* PyDate_FromTimestamp(PyObject *args) + + Create and return a new ``datetime.date`` object given an argument tuple + suitable for passing to ``datetime.date.fromtimestamp()``. diff --git a/lib/cpython-doc/c-api/descriptor.rst b/lib/cpython-doc/c-api/descriptor.rst new file mode 100644 --- /dev/null +++ b/lib/cpython-doc/c-api/descriptor.rst @@ -0,0 +1,40 @@ +.. highlightlang:: c + +.. _descriptor-objects: + +Descriptor Objects +------------------ + +"Descriptors" are objects that describe some attribute of an object. They are +found in the dictionary of type objects. + +.. XXX document these! + +.. c:var:: PyTypeObject PyProperty_Type + + The type object for the built-in descriptor types. + + +.. c:function:: PyObject* PyDescr_NewGetSet(PyTypeObject *type, struct PyGetSetDef *getset) + + +.. c:function:: PyObject* PyDescr_NewMember(PyTypeObject *type, struct PyMemberDef *meth) + + +.. c:function:: PyObject* PyDescr_NewMethod(PyTypeObject *type, struct PyMethodDef *meth) + + +.. c:function:: PyObject* PyDescr_NewWrapper(PyTypeObject *type, struct wrapperbase *wrapper, void *wrapped) + + +.. c:function:: PyObject* PyDescr_NewClassMethod(PyTypeObject *type, PyMethodDef *method) + + +.. c:function:: int PyDescr_IsData(PyObject *descr) + + Return true if the descriptor objects *descr* describes a data attribute, or + false if it describes a method. *descr* must be a descriptor object; there is + no error checking. + + +.. c:function:: PyObject* PyWrapper_New(PyObject *, PyObject *) diff --git a/lib/cpython-doc/c-api/dict.rst b/lib/cpython-doc/c-api/dict.rst new file mode 100644 --- /dev/null +++ b/lib/cpython-doc/c-api/dict.rst @@ -0,0 +1,218 @@ +.. highlightlang:: c + +.. _dictobjects: + +Dictionary Objects +------------------ + +.. index:: object: dictionary + + +.. c:type:: PyDictObject + + This subtype of :c:type:`PyObject` represents a Python dictionary object. + + +.. c:var:: PyTypeObject PyDict_Type + + This instance of :c:type:`PyTypeObject` represents the Python dictionary + type. This is the same object as :class:`dict` in the Python layer. + + +.. c:function:: int PyDict_Check(PyObject *p) + + Return true if *p* is a dict object or an instance of a subtype of the dict + type. + + +.. c:function:: int PyDict_CheckExact(PyObject *p) + + Return true if *p* is a dict object, but not an instance of a subtype of + the dict type. + + +.. c:function:: PyObject* PyDict_New() + + Return a new empty dictionary, or *NULL* on failure. + + +.. c:function:: PyObject* PyDictProxy_New(PyObject *dict) + + Return a proxy object for a mapping which enforces read-only behavior. + This is normally used to create a proxy to prevent modification of the + dictionary for non-dynamic class types. + + +.. c:function:: void PyDict_Clear(PyObject *p) + + Empty an existing dictionary of all key-value pairs. + + +.. c:function:: int PyDict_Contains(PyObject *p, PyObject *key) + + Determine if dictionary *p* contains *key*. If an item in *p* is matches + *key*, return ``1``, otherwise return ``0``. On error, return ``-1``. + This is equivalent to the Python expression ``key in p``. + + +.. c:function:: PyObject* PyDict_Copy(PyObject *p) + + Return a new dictionary that contains the same key-value pairs as *p*. + + +.. c:function:: int PyDict_SetItem(PyObject *p, PyObject *key, PyObject *val) + + Insert *value* into the dictionary *p* with a key of *key*. *key* must be + :term:`hashable`; if it isn't, :exc:`TypeError` will be raised. Return + ``0`` on success or ``-1`` on failure. + + +.. c:function:: int PyDict_SetItemString(PyObject *p, const char *key, PyObject *val) + + .. index:: single: PyUnicode_FromString() + + Insert *value* into the dictionary *p* using *key* as a key. *key* should + be a :c:type:`char\*`. The key object is created using + ``PyUnicode_FromString(key)``. Return ``0`` on success or ``-1`` on + failure. + + +.. c:function:: int PyDict_DelItem(PyObject *p, PyObject *key) + + Remove the entry in dictionary *p* with key *key*. *key* must be hashable; + if it isn't, :exc:`TypeError` is raised. Return ``0`` on success or ``-1`` + on failure. + + +.. c:function:: int PyDict_DelItemString(PyObject *p, char *key) + + Remove the entry in dictionary *p* which has a key specified by the string + *key*. Return ``0`` on success or ``-1`` on failure. + + +.. c:function:: PyObject* PyDict_GetItem(PyObject *p, PyObject *key) + + Return the object from dictionary *p* which has a key *key*. Return *NULL* + if the key *key* is not present, but *without* setting an exception. + + +.. c:function:: PyObject* PyDict_GetItemWithError(PyObject *p, PyObject *key) + + Variant of :c:func:`PyDict_GetItem` that does not suppress + exceptions. Return *NULL* **with** an exception set if an exception + occurred. Return *NULL* **without** an exception set if the key + wasn't present. + + +.. c:function:: PyObject* PyDict_GetItemString(PyObject *p, const char *key) + + This is the same as :c:func:`PyDict_GetItem`, but *key* is specified as a + :c:type:`char\*`, rather than a :c:type:`PyObject\*`. + + +.. c:function:: PyObject* PyDict_Items(PyObject *p) + + Return a :c:type:`PyListObject` containing all the items from the dictionary. + + +.. c:function:: PyObject* PyDict_Keys(PyObject *p) + + Return a :c:type:`PyListObject` containing all the keys from the dictionary. + + +.. c:function:: PyObject* PyDict_Values(PyObject *p) + + Return a :c:type:`PyListObject` containing all the values from the dictionary + *p*. + + +.. c:function:: Py_ssize_t PyDict_Size(PyObject *p) + + .. index:: builtin: len + + Return the number of items in the dictionary. This is equivalent to + ``len(p)`` on a dictionary. + + +.. c:function:: int PyDict_Next(PyObject *p, Py_ssize_t *ppos, PyObject **pkey, PyObject **pvalue) + + Iterate over all key-value pairs in the dictionary *p*. The + :c:type:`Py_ssize_t` referred to by *ppos* must be initialized to ``0`` + prior to the first call to this function to start the iteration; the + function returns true for each pair in the dictionary, and false once all + pairs have been reported. The parameters *pkey* and *pvalue* should either + point to :c:type:`PyObject\*` variables that will be filled in with each key + and value, respectively, or may be *NULL*. Any references returned through + them are borrowed. *ppos* should not be altered during iteration. Its + value represents offsets within the internal dictionary structure, and + since the structure is sparse, the offsets are not consecutive. + + For example:: + + PyObject *key, *value; + Py_ssize_t pos = 0; + + while (PyDict_Next(self->dict, &pos, &key, &value)) { + /* do something interesting with the values... */ + ... + } + + The dictionary *p* should not be mutated during iteration. It is safe to + modify the values of the keys as you iterate over the dictionary, but only + so long as the set of keys does not change. For example:: + + PyObject *key, *value; + Py_ssize_t pos = 0; + + while (PyDict_Next(self->dict, &pos, &key, &value)) { + long i = PyLong_AsLong(value); + if (i == -1 && PyErr_Occurred()) { + return -1; + } + PyObject *o = PyLong_FromLong(i + 1); + if (o == NULL) + return -1; + if (PyDict_SetItem(self->dict, key, o) < 0) { + Py_DECREF(o); + return -1; + } + Py_DECREF(o); + } + + +.. c:function:: int PyDict_Merge(PyObject *a, PyObject *b, int override) + + Iterate over mapping object *b* adding key-value pairs to dictionary *a*. + *b* may be a dictionary, or any object supporting :c:func:`PyMapping_Keys` + and :c:func:`PyObject_GetItem`. If *override* is true, existing pairs in *a* + will be replaced if a matching key is found in *b*, otherwise pairs will + only be added if there is not a matching key in *a*. Return ``0`` on + success or ``-1`` if an exception was raised. + + +.. c:function:: int PyDict_Update(PyObject *a, PyObject *b) + + This is the same as ``PyDict_Merge(a, b, 1)`` in C, or ``a.update(b)`` in + Python. Return ``0`` on success or ``-1`` if an exception was raised. + + +.. c:function:: int PyDict_MergeFromSeq2(PyObject *a, PyObject *seq2, int override) + + Update or merge into dictionary *a*, from the key-value pairs in *seq2*. + *seq2* must be an iterable object producing iterable objects of length 2, + viewed as key-value pairs. In case of duplicate keys, the last wins if + *override* is true, else the first wins. Return ``0`` on success or ``-1`` + if an exception was raised. Equivalent Python (except for the return + value):: + + def PyDict_MergeFromSeq2(a, seq2, override): + for key, value in seq2: + if override or key not in a: + a[key] = value + + +.. c:function:: int PyDict_ClearFreeList() + + Clear the free list. Return the total number of freed items. + + .. versionadded:: 3.3 diff --git a/lib/cpython-doc/c-api/exceptions.rst b/lib/cpython-doc/c-api/exceptions.rst new file mode 100644 --- /dev/null +++ b/lib/cpython-doc/c-api/exceptions.rst @@ -0,0 +1,750 @@ +.. highlightlang:: c + + +.. _exceptionhandling: + +****************** +Exception Handling +****************** + +The functions described in this chapter will let you handle and raise Python +exceptions. It is important to understand some of the basics of Python +exception handling. It works somewhat like the Unix :c:data:`errno` variable: +there is a global indicator (per thread) of the last error that occurred. Most +functions don't clear this on success, but will set it to indicate the cause of +the error on failure. Most functions also return an error indicator, usually +*NULL* if they are supposed to return a pointer, or ``-1`` if they return an +integer (exception: the :c:func:`PyArg_\*` functions return ``1`` for success and +``0`` for failure). + +When a function must fail because some function it called failed, it generally +doesn't set the error indicator; the function it called already set it. It is +responsible for either handling the error and clearing the exception or +returning after cleaning up any resources it holds (such as object references or +memory allocations); it should *not* continue normally if it is not prepared to +handle the error. If returning due to an error, it is important to indicate to +the caller that an error has been set. If the error is not handled or carefully +propagated, additional calls into the Python/C API may not behave as intended +and may fail in mysterious ways. + +The error indicator consists of three Python objects corresponding to the result +of ``sys.exc_info()``. API functions exist to interact with the error indicator +in various ways. There is a separate error indicator for each thread. + +.. XXX Order of these should be more thoughtful. + Either alphabetical or some kind of structure. + + +.. c:function:: void PyErr_PrintEx(int set_sys_last_vars) + + Print a standard traceback to ``sys.stderr`` and clear the error indicator. + Call this function only when the error indicator is set. (Otherwise it will + cause a fatal error!) + + If *set_sys_last_vars* is nonzero, the variables :data:`sys.last_type`, + :data:`sys.last_value` and :data:`sys.last_traceback` will be set to the + type, value and traceback of the printed exception, respectively. + + +.. c:function:: void PyErr_Print() + + Alias for ``PyErr_PrintEx(1)``. + + +.. c:function:: PyObject* PyErr_Occurred() + + Test whether the error indicator is set. If set, return the exception *type* + (the first argument to the last call to one of the :c:func:`PyErr_Set\*` + functions or to :c:func:`PyErr_Restore`). If not set, return *NULL*. You do not + own a reference to the return value, so you do not need to :c:func:`Py_DECREF` + it. + + .. note:: + + Do not compare the return value to a specific exception; use + :c:func:`PyErr_ExceptionMatches` instead, shown below. (The comparison could + easily fail since the exception may be an instance instead of a class, in the + case of a class exception, or it may the a subclass of the expected exception.) + + +.. c:function:: int PyErr_ExceptionMatches(PyObject *exc) + + Equivalent to ``PyErr_GivenExceptionMatches(PyErr_Occurred(), exc)``. This + should only be called when an exception is actually set; a memory access + violation will occur if no exception has been raised. + + +.. c:function:: int PyErr_GivenExceptionMatches(PyObject *given, PyObject *exc) + + Return true if the *given* exception matches the exception in *exc*. If + *exc* is a class object, this also returns true when *given* is an instance + of a subclass. If *exc* is a tuple, all exceptions in the tuple (and + recursively in subtuples) are searched for a match. + + +.. c:function:: void PyErr_NormalizeException(PyObject**exc, PyObject**val, PyObject**tb) + + Under certain circumstances, the values returned by :c:func:`PyErr_Fetch` below + can be "unnormalized", meaning that ``*exc`` is a class object but ``*val`` is + not an instance of the same class. This function can be used to instantiate + the class in that case. If the values are already normalized, nothing happens. + The delayed normalization is implemented to improve performance. + + +.. c:function:: void PyErr_Clear() + + Clear the error indicator. If the error indicator is not set, there is no + effect. + + +.. c:function:: void PyErr_Fetch(PyObject **ptype, PyObject **pvalue, PyObject **ptraceback) + + Retrieve the error indicator into three variables whose addresses are passed. + If the error indicator is not set, set all three variables to *NULL*. If it is + set, it will be cleared and you own a reference to each object retrieved. The + value and traceback object may be *NULL* even when the type object is not. + + .. note:: + + This function is normally only used by code that needs to handle exceptions or + by code that needs to save and restore the error indicator temporarily. + + +.. c:function:: void PyErr_Restore(PyObject *type, PyObject *value, PyObject *traceback) + + Set the error indicator from the three objects. If the error indicator is + already set, it is cleared first. If the objects are *NULL*, the error + indicator is cleared. Do not pass a *NULL* type and non-*NULL* value or + traceback. The exception type should be a class. Do not pass an invalid + exception type or value. (Violating these rules will cause subtle problems + later.) This call takes away a reference to each object: you must own a + reference to each object before the call and after the call you no longer own + these references. (If you don't understand this, don't use this function. I + warned you.) + + .. note:: + + This function is normally only used by code that needs to save and restore the + error indicator temporarily; use :c:func:`PyErr_Fetch` to save the current + exception state. + + +.. c:function:: void PyErr_SetString(PyObject *type, const char *message) + + This is the most common way to set the error indicator. The first argument + specifies the exception type; it is normally one of the standard exceptions, + e.g. :c:data:`PyExc_RuntimeError`. You need not increment its reference count. + The second argument is an error message; it is decoded from ``'utf-8``'. + + +.. c:function:: void PyErr_SetObject(PyObject *type, PyObject *value) + + This function is similar to :c:func:`PyErr_SetString` but lets you specify an + arbitrary Python object for the "value" of the exception. + + +.. c:function:: PyObject* PyErr_Format(PyObject *exception, const char *format, ...) + + This function sets the error indicator and returns *NULL*. *exception* + should be a Python exception class. The *format* and subsequent + parameters help format the error message; they have the same meaning and + values as in :c:func:`PyUnicode_FromFormat`. *format* is an ASCII-encoded + string. + + +.. c:function:: void PyErr_SetNone(PyObject *type) + + This is a shorthand for ``PyErr_SetObject(type, Py_None)``. + + +.. c:function:: int PyErr_BadArgument() + + This is a shorthand for ``PyErr_SetString(PyExc_TypeError, message)``, where + *message* indicates that a built-in operation was invoked with an illegal + argument. It is mostly for internal use. + + +.. c:function:: PyObject* PyErr_NoMemory() + + This is a shorthand for ``PyErr_SetNone(PyExc_MemoryError)``; it returns *NULL* + so an object allocation function can write ``return PyErr_NoMemory();`` when it + runs out of memory. + + +.. c:function:: PyObject* PyErr_SetFromErrno(PyObject *type) + + .. index:: single: strerror() + + This is a convenience function to raise an exception when a C library function + has returned an error and set the C variable :c:data:`errno`. It constructs a + tuple object whose first item is the integer :c:data:`errno` value and whose + second item is the corresponding error message (gotten from :c:func:`strerror`), + and then calls ``PyErr_SetObject(type, object)``. On Unix, when the + :c:data:`errno` value is :const:`EINTR`, indicating an interrupted system call, + this calls :c:func:`PyErr_CheckSignals`, and if that set the error indicator, + leaves it set to that. The function always returns *NULL*, so a wrapper + function around a system call can write ``return PyErr_SetFromErrno(type);`` + when the system call returns an error. + + +.. c:function:: PyObject* PyErr_SetFromErrnoWithFilename(PyObject *type, const char *filename) + + Similar to :c:func:`PyErr_SetFromErrno`, with the additional behavior that if + *filename* is not *NULL*, it is passed to the constructor of *type* as a third + parameter. In the case of exceptions such as :exc:`IOError` and :exc:`OSError`, + this is used to define the :attr:`filename` attribute of the exception instance. + *filename* is decoded from the filesystem encoding + (:func:`sys.getfilesystemencoding`). + + +.. c:function:: PyObject* PyErr_SetFromWindowsErr(int ierr) + + This is a convenience function to raise :exc:`WindowsError`. If called with + *ierr* of :c:data:`0`, the error code returned by a call to :c:func:`GetLastError` + is used instead. It calls the Win32 function :c:func:`FormatMessage` to retrieve + the Windows description of error code given by *ierr* or :c:func:`GetLastError`, + then it constructs a tuple object whose first item is the *ierr* value and whose + second item is the corresponding error message (gotten from + :c:func:`FormatMessage`), and then calls ``PyErr_SetObject(PyExc_WindowsError, + object)``. This function always returns *NULL*. Availability: Windows. + + +.. c:function:: PyObject* PyErr_SetExcFromWindowsErr(PyObject *type, int ierr) + + Similar to :c:func:`PyErr_SetFromWindowsErr`, with an additional parameter + specifying the exception type to be raised. Availability: Windows. + + +.. c:function:: PyObject* PyErr_SetFromWindowsErrWithFilename(int ierr, const char *filename) + + Similar to :c:func:`PyErr_SetFromWindowsErr`, with the additional behavior that + if *filename* is not *NULL*, it is passed to the constructor of + :exc:`WindowsError` as a third parameter. *filename* is decoded from the + filesystem encoding (:func:`sys.getfilesystemencoding`). Availability: + Windows. + + +.. c:function:: PyObject* PyErr_SetExcFromWindowsErrWithFilename(PyObject *type, int ierr, char *filename) + + Similar to :c:func:`PyErr_SetFromWindowsErrWithFilename`, with an additional + parameter specifying the exception type to be raised. Availability: Windows. + + +.. c:function:: void PyErr_SyntaxLocationEx(char *filename, int lineno, int col_offset) + + Set file, line, and offset information for the current exception. If the + current exception is not a :exc:`SyntaxError`, then it sets additional + attributes, which make the exception printing subsystem think the exception + is a :exc:`SyntaxError`. *filename* is decoded from the filesystem encoding + (:func:`sys.getfilesystemencoding`). + +.. versionadded:: 3.2 + + +.. c:function:: void PyErr_SyntaxLocation(char *filename, int lineno) + + Like :c:func:`PyErr_SyntaxLocationExc`, but the col_offset parameter is + omitted. + + +.. c:function:: void PyErr_BadInternalCall() + + This is a shorthand for ``PyErr_SetString(PyExc_SystemError, message)``, + where *message* indicates that an internal operation (e.g. a Python/C API + function) was invoked with an illegal argument. It is mostly for internal + use. + + +.. c:function:: int PyErr_WarnEx(PyObject *category, char *message, int stack_level) + + Issue a warning message. The *category* argument is a warning category (see + below) or *NULL*; the *message* argument is an UTF-8 encoded string. *stack_level* is a + positive number giving a number of stack frames; the warning will be issued from + the currently executing line of code in that stack frame. A *stack_level* of 1 + is the function calling :c:func:`PyErr_WarnEx`, 2 is the function above that, + and so forth. + + This function normally prints a warning message to *sys.stderr*; however, it is + also possible that the user has specified that warnings are to be turned into + errors, and in that case this will raise an exception. It is also possible that + the function raises an exception because of a problem with the warning machinery + (the implementation imports the :mod:`warnings` module to do the heavy lifting). + The return value is ``0`` if no exception is raised, or ``-1`` if an exception + is raised. (It is not possible to determine whether a warning message is + actually printed, nor what the reason is for the exception; this is + intentional.) If an exception is raised, the caller should do its normal + exception handling (for example, :c:func:`Py_DECREF` owned references and return + an error value). + + Warning categories must be subclasses of :c:data:`Warning`; the default warning + category is :c:data:`RuntimeWarning`. The standard Python warning categories are + available as global variables whose names are ``PyExc_`` followed by the Python + exception name. These have the type :c:type:`PyObject\*`; they are all class + objects. Their names are :c:data:`PyExc_Warning`, :c:data:`PyExc_UserWarning`, + :c:data:`PyExc_UnicodeWarning`, :c:data:`PyExc_DeprecationWarning`, + :c:data:`PyExc_SyntaxWarning`, :c:data:`PyExc_RuntimeWarning`, and + :c:data:`PyExc_FutureWarning`. :c:data:`PyExc_Warning` is a subclass of + :c:data:`PyExc_Exception`; the other warning categories are subclasses of + :c:data:`PyExc_Warning`. + + For information about warning control, see the documentation for the + :mod:`warnings` module and the :option:`-W` option in the command line + documentation. There is no C API for warning control. + + +.. c:function:: int PyErr_WarnExplicit(PyObject *category, const char *message, const char *filename, int lineno, const char *module, PyObject *registry) + + Issue a warning message with explicit control over all warning attributes. This + is a straightforward wrapper around the Python function + :func:`warnings.warn_explicit`, see there for more information. The *module* + and *registry* arguments may be set to *NULL* to get the default effect + described there. *message* and *module* are UTF-8 encoded strings, + *filename* is decoded from the filesystem encoding + (:func:`sys.getfilesystemencoding`). + + +.. c:function:: int PyErr_WarnFormat(PyObject *category, Py_ssize_t stack_level, const char *format, ...) + + Function similar to :c:func:`PyErr_WarnEx`, but use + :c:func:`PyUnicode_FromFormat` to format the warning message. *format* is + an ASCII-encoded string. + + .. versionadded:: 3.2 + +.. c:function:: int PyErr_CheckSignals() + + .. index:: + module: signal + single: SIGINT + single: KeyboardInterrupt (built-in exception) + + This function interacts with Python's signal handling. It checks whether a + signal has been sent to the processes and if so, invokes the corresponding + signal handler. If the :mod:`signal` module is supported, this can invoke a + signal handler written in Python. In all cases, the default effect for + :const:`SIGINT` is to raise the :exc:`KeyboardInterrupt` exception. If an + exception is raised the error indicator is set and the function returns ``-1``; + otherwise the function returns ``0``. The error indicator may or may not be + cleared if it was previously set. + + +.. c:function:: void PyErr_SetInterrupt() + + .. index:: + single: SIGINT + single: KeyboardInterrupt (built-in exception) + + This function simulates the effect of a :const:`SIGINT` signal arriving --- the + next time :c:func:`PyErr_CheckSignals` is called, :exc:`KeyboardInterrupt` will + be raised. It may be called without holding the interpreter lock. + + .. % XXX This was described as obsolete, but is used in + .. % _thread.interrupt_main() (used from IDLE), so it's still needed. + + +.. c:function:: int PySignal_SetWakeupFd(int fd) + + This utility function specifies a file descriptor to which a ``'\0'`` byte will + be written whenever a signal is received. It returns the previous such file + descriptor. The value ``-1`` disables the feature; this is the initial state. + This is equivalent to :func:`signal.set_wakeup_fd` in Python, but without any + error checking. *fd* should be a valid file descriptor. The function should + only be called from the main thread. + + +.. c:function:: PyObject* PyErr_NewException(char *name, PyObject *base, PyObject *dict) + + This utility function creates and returns a new exception class. The *name* + argument must be the name of the new exception, a C string of the form + ``module.classname``. The *base* and *dict* arguments are normally *NULL*. + This creates a class object derived from :exc:`Exception` (accessible in C as + :c:data:`PyExc_Exception`). + + The :attr:`__module__` attribute of the new class is set to the first part (up + to the last dot) of the *name* argument, and the class name is set to the last + part (after the last dot). The *base* argument can be used to specify alternate + base classes; it can either be only one class or a tuple of classes. The *dict* + argument can be used to specify a dictionary of class variables and methods. + + +.. c:function:: PyObject* PyErr_NewExceptionWithDoc(char *name, char *doc, PyObject *base, PyObject *dict) + + Same as :c:func:`PyErr_NewException`, except that the new exception class can + easily be given a docstring: If *doc* is non-*NULL*, it will be used as the + docstring for the exception class. + + .. versionadded:: 3.2 + + +.. c:function:: void PyErr_WriteUnraisable(PyObject *obj) + + This utility function prints a warning message to ``sys.stderr`` when an + exception has been set but it is impossible for the interpreter to actually + raise the exception. It is used, for example, when an exception occurs in an + :meth:`__del__` method. + + The function is called with a single argument *obj* that identifies the context + in which the unraisable exception occurred. The repr of *obj* will be printed in + the warning message. + + +Exception Objects +================= + +.. c:function:: PyObject* PyException_GetTraceback(PyObject *ex) + + Return the traceback associated with the exception as a new reference, as + accessible from Python through :attr:`__traceback__`. If there is no + traceback associated, this returns *NULL*. + + +.. c:function:: int PyException_SetTraceback(PyObject *ex, PyObject *tb) + + Set the traceback associated with the exception to *tb*. Use ``Py_None`` to + clear it. + + +.. c:function:: PyObject* PyException_GetContext(PyObject *ex) + + Return the context (another exception instance during whose handling *ex* was + raised) associated with the exception as a new reference, as accessible from + Python through :attr:`__context__`. If there is no context associated, this + returns *NULL*. + + +.. c:function:: void PyException_SetContext(PyObject *ex, PyObject *ctx) + + Set the context associated with the exception to *ctx*. Use *NULL* to clear + it. There is no type check to make sure that *ctx* is an exception instance. + This steals a reference to *ctx*. + + +.. c:function:: PyObject* PyException_GetCause(PyObject *ex) + + Return the cause (another exception instance set by ``raise ... from ...``) + associated with the exception as a new reference, as accessible from Python + through :attr:`__cause__`. If there is no cause associated, this returns + *NULL*. + + +.. c:function:: void PyException_SetCause(PyObject *ex, PyObject *ctx) + + Set the cause associated with the exception to *ctx*. Use *NULL* to clear + it. There is no type check to make sure that *ctx* is an exception instance. + This steals a reference to *ctx*. + + +.. _unicodeexceptions: + +Unicode Exception Objects +========================= + +The following functions are used to create and modify Unicode exceptions from C. + +.. c:function:: PyObject* PyUnicodeDecodeError_Create(const char *encoding, const char *object, Py_ssize_t length, Py_ssize_t start, Py_ssize_t end, const char *reason) + + Create a :class:`UnicodeDecodeError` object with the attributes *encoding*, + *object*, *length*, *start*, *end* and *reason*. *encoding* and *reason* are + UTF-8 encoded strings. + +.. c:function:: PyObject* PyUnicodeEncodeError_Create(const char *encoding, const Py_UNICODE *object, Py_ssize_t length, Py_ssize_t start, Py_ssize_t end, const char *reason) + + Create a :class:`UnicodeEncodeError` object with the attributes *encoding*, + *object*, *length*, *start*, *end* and *reason*. *encoding* and *reason* are + UTF-8 encoded strings. + +.. c:function:: PyObject* PyUnicodeTranslateError_Create(const Py_UNICODE *object, Py_ssize_t length, Py_ssize_t start, Py_ssize_t end, const char *reason) + + Create a :class:`UnicodeTranslateError` object with the attributes *object*, + *length*, *start*, *end* and *reason*. *reason* is an UTF-8 encoded string. + +.. c:function:: PyObject* PyUnicodeDecodeError_GetEncoding(PyObject *exc) + PyObject* PyUnicodeEncodeError_GetEncoding(PyObject *exc) + + Return the *encoding* attribute of the given exception object. + +.. c:function:: PyObject* PyUnicodeDecodeError_GetObject(PyObject *exc) + PyObject* PyUnicodeEncodeError_GetObject(PyObject *exc) + PyObject* PyUnicodeTranslateError_GetObject(PyObject *exc) + + Return the *object* attribute of the given exception object. + +.. c:function:: int PyUnicodeDecodeError_GetStart(PyObject *exc, Py_ssize_t *start) + int PyUnicodeEncodeError_GetStart(PyObject *exc, Py_ssize_t *start) + int PyUnicodeTranslateError_GetStart(PyObject *exc, Py_ssize_t *start) + + Get the *start* attribute of the given exception object and place it into + *\*start*. *start* must not be *NULL*. Return ``0`` on success, ``-1`` on + failure. + +.. c:function:: int PyUnicodeDecodeError_SetStart(PyObject *exc, Py_ssize_t start) + int PyUnicodeEncodeError_SetStart(PyObject *exc, Py_ssize_t start) + int PyUnicodeTranslateError_SetStart(PyObject *exc, Py_ssize_t start) + + Set the *start* attribute of the given exception object to *start*. Return + ``0`` on success, ``-1`` on failure. + +.. c:function:: int PyUnicodeDecodeError_GetEnd(PyObject *exc, Py_ssize_t *end) + int PyUnicodeEncodeError_GetEnd(PyObject *exc, Py_ssize_t *end) + int PyUnicodeTranslateError_GetEnd(PyObject *exc, Py_ssize_t *end) + + Get the *end* attribute of the given exception object and place it into + *\*end*. *end* must not be *NULL*. Return ``0`` on success, ``-1`` on + failure. + +.. c:function:: int PyUnicodeDecodeError_SetEnd(PyObject *exc, Py_ssize_t end) + int PyUnicodeEncodeError_SetEnd(PyObject *exc, Py_ssize_t end) + int PyUnicodeTranslateError_SetEnd(PyObject *exc, Py_ssize_t end) + + Set the *end* attribute of the given exception object to *end*. Return ``0`` + on success, ``-1`` on failure. + +.. c:function:: PyObject* PyUnicodeDecodeError_GetReason(PyObject *exc) + PyObject* PyUnicodeEncodeError_GetReason(PyObject *exc) + PyObject* PyUnicodeTranslateError_GetReason(PyObject *exc) + + Return the *reason* attribute of the given exception object. + +.. c:function:: int PyUnicodeDecodeError_SetReason(PyObject *exc, const char *reason) + int PyUnicodeEncodeError_SetReason(PyObject *exc, const char *reason) + int PyUnicodeTranslateError_SetReason(PyObject *exc, const char *reason) + + Set the *reason* attribute of the given exception object to *reason*. Return + ``0`` on success, ``-1`` on failure. + + +Recursion Control +================= + +These two functions provide a way to perform safe recursive calls at the C +level, both in the core and in extension modules. They are needed if the +recursive code does not necessarily invoke Python code (which tracks its +recursion depth automatically). + +.. c:function:: int Py_EnterRecursiveCall(char *where) + + Marks a point where a recursive C-level call is about to be performed. + + If :const:`USE_STACKCHECK` is defined, this function checks if the OS + stack overflowed using :c:func:`PyOS_CheckStack`. In this is the case, it + sets a :exc:`MemoryError` and returns a nonzero value. + + The function then checks if the recursion limit is reached. If this is the + case, a :exc:`RuntimeError` is set and a nonzero value is returned. + Otherwise, zero is returned. + + *where* should be a string such as ``" in instance check"`` to be + concatenated to the :exc:`RuntimeError` message caused by the recursion depth + limit. + +.. c:function:: void Py_LeaveRecursiveCall() + + Ends a :c:func:`Py_EnterRecursiveCall`. Must be called once for each + *successful* invocation of :c:func:`Py_EnterRecursiveCall`. + +Properly implementing :attr:`tp_repr` for container types requires +special recursion handling. In addition to protecting the stack, +:attr:`tp_repr` also needs to track objects to prevent cycles. The +following two functions facilitate this functionality. Effectively, +these are the C equivalent to :func:`reprlib.recursive_repr`. + +.. c:function:: int Py_ReprEnter(PyObject *object) + + Called at the beginning of the :attr:`tp_repr` implementation to + detect cycles. + + If the object has already been processed, the function returns a + positive integer. In that case the :attr:`tp_repr` implementation + should return a string object indicating a cycle. As examples, + :class:`dict` objects return ``{...}`` and :class:`list` objects + return ``[...]``. + + The function will return a negative integer if the recursion limit + is reached. In that case the :attr:`tp_repr` implementation should + typically return ``NULL``. + + Otherwise, the function returns zero and the :attr:`tp_repr` + implementation can continue normally. + +.. c:function:: void Py_ReprLeave(PyObject *object) + + Ends a :c:func:`Py_ReprEnter`. Must be called once for each + invocation of :c:func:`Py_ReprEnter` that returns zero. + + +.. _standardexceptions: + +Standard Exceptions +=================== + +All standard Python exceptions are available as global variables whose names are +``PyExc_`` followed by the Python exception name. These have the type +:c:type:`PyObject\*`; they are all class objects. For completeness, here are all +the variables: + ++-----------------------------------------+---------------------------------+----------+ +| C Name | Python Name | Notes | ++=========================================+=================================+==========+ +| :c:data:`PyExc_BaseException` | :exc:`BaseException` | \(1) | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_Exception` | :exc:`Exception` | \(1) | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ArithmeticError` | :exc:`ArithmeticError` | \(1) | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_LookupError` | :exc:`LookupError` | \(1) | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_AssertionError` | :exc:`AssertionError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_AttributeError` | :exc:`AttributeError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_BlockingIOError` | :exc:`BlockingIOError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_BrokenPipeError` | :exc:`BrokenPipeError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ChildProcessError` | :exc:`ChildProcessError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ConnectionError` | :exc:`ConnectionError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ConnectionAbortedError` | :exc:`ConnectionAbortedError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ConnectionRefusedError` | :exc:`ConnectionRefusedError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ConnectionResetError` | :exc:`ConnectionResetError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_FileExistsError` | :exc:`FileExistsError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_FileNotFoundError` | :exc:`FileNotFoundError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_EOFError` | :exc:`EOFError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_FloatingPointError` | :exc:`FloatingPointError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ImportError` | :exc:`ImportError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_IndexError` | :exc:`IndexError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_InterruptedError` | :exc:`InterruptedError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_IsADirectoryError` | :exc:`IsADirectoryError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_KeyError` | :exc:`KeyError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_KeyboardInterrupt` | :exc:`KeyboardInterrupt` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_MemoryError` | :exc:`MemoryError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_NameError` | :exc:`NameError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_NotADirectoryError` | :exc:`NotADirectoryError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_NotImplementedError` | :exc:`NotImplementedError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_OSError` | :exc:`OSError` | \(1) | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_OverflowError` | :exc:`OverflowError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_PermissionError` | :exc:`PermissionError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ProcessLookupError` | :exc:`ProcessLookupError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ReferenceError` | :exc:`ReferenceError` | \(2) | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_RuntimeError` | :exc:`RuntimeError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_SyntaxError` | :exc:`SyntaxError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_SystemError` | :exc:`SystemError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_TimeoutError` | :exc:`TimeoutError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_SystemExit` | :exc:`SystemExit` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_TypeError` | :exc:`TypeError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ValueError` | :exc:`ValueError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ZeroDivisionError` | :exc:`ZeroDivisionError` | | ++-----------------------------------------+---------------------------------+----------+ + +.. versionadded:: 3.3 + :c:data:`PyExc_BlockingIOError`, :c:data:`PyExc_BrokenPipeError`, + :c:data:`PyExc_ChildProcessError`, :c:data:`PyExc_ConnectionError`, + :c:data:`PyExc_ConnectionAbortedError`, :c:data:`PyExc_ConnectionRefusedError`, + :c:data:`PyExc_ConnectionResetError`, :c:data:`PyExc_FileExistsError`, + :c:data:`PyExc_FileNotFoundError`, :c:data:`PyExc_InterruptedError`, + :c:data:`PyExc_IsADirectoryError`, :c:data:`PyExc_NotADirectoryError`, + :c:data:`PyExc_PermissionError`, :c:data:`PyExc_ProcessLookupError` + and :c:data:`PyExc_TimeoutError` were introduced following :pep:`3151`. + + +These are compatibility aliases to :c:data:`PyExc_OSError`: + ++-------------------------------------+----------+ +| C Name | Notes | ++=====================================+==========+ +| :c:data:`PyExc_EnvironmentError` | | ++-------------------------------------+----------+ +| :c:data:`PyExc_IOError` | | ++-------------------------------------+----------+ +| :c:data:`PyExc_WindowsError` | \(3) | ++-------------------------------------+----------+ + +.. versionchanged:: 3.3 + These aliases used to be separate exception types. + + +.. index:: + single: PyExc_BaseException + single: PyExc_Exception + single: PyExc_ArithmeticError + single: PyExc_LookupError + single: PyExc_AssertionError + single: PyExc_AttributeError + single: PyExc_BlockingIOError + single: PyExc_BrokenPipeError + single: PyExc_ConnectionError + single: PyExc_ConnectionAbortedError + single: PyExc_ConnectionRefusedError + single: PyExc_ConnectionResetError + single: PyExc_EOFError + single: PyExc_FileExistsError + single: PyExc_FileNotFoundError + single: PyExc_FloatingPointError + single: PyExc_ImportError + single: PyExc_IndexError + single: PyExc_InterruptedError + single: PyExc_IsADirectoryError + single: PyExc_KeyError + single: PyExc_KeyboardInterrupt + single: PyExc_MemoryError + single: PyExc_NameError + single: PyExc_NotADirectoryError + single: PyExc_NotImplementedError + single: PyExc_OSError + single: PyExc_OverflowError + single: PyExc_PermissionError + single: PyExc_ProcessLookupError + single: PyExc_ReferenceError + single: PyExc_RuntimeError + single: PyExc_SyntaxError + single: PyExc_SystemError + single: PyExc_SystemExit + single: PyExc_TimeoutError + single: PyExc_TypeError + single: PyExc_ValueError + single: PyExc_ZeroDivisionError + single: PyExc_EnvironmentError + single: PyExc_IOError + single: PyExc_WindowsError + +Notes: + +(1) + This is a base class for other standard exceptions. + +(2) + This is the same as :exc:`weakref.ReferenceError`. + +(3) + Only defined on Windows; protect code that uses this by testing that the + preprocessor macro ``MS_WINDOWS`` is defined. diff --git a/lib/cpython-doc/c-api/file.rst b/lib/cpython-doc/c-api/file.rst new file mode 100644 --- /dev/null +++ b/lib/cpython-doc/c-api/file.rst @@ -0,0 +1,75 @@ +.. highlightlang:: c + +.. _fileobjects: + +File Objects +------------ + +.. index:: object: file + +These APIs are a minimal emulation of the Python 2 C API for built-in file +objects, which used to rely on the buffered I/O (:c:type:`FILE\*`) support +from the C standard library. In Python 3, files and streams use the new +:mod:`io` module, which defines several layers over the low-level unbuffered +I/O of the operating system. The functions described below are +convenience C wrappers over these new APIs, and meant mostly for internal +error reporting in the interpreter; third-party code is advised to access +the :mod:`io` APIs instead. + + +.. c:function:: PyFile_FromFd(int fd, char *name, char *mode, int buffering, char *encoding, char *errors, char *newline, int closefd) + + Create a Python file object from the file descriptor of an already + opened file *fd*. The arguments *name*, *encoding*, *errors* and *newline* + can be *NULL* to use the defaults; *buffering* can be *-1* to use the + default. *name* is ignored and kept for backward compatibility. Return + *NULL* on failure. For a more comprehensive description of the arguments, + please refer to the :func:`io.open` function documentation. + + .. warning:: + + Since Python streams have their own buffering layer, mixing them with + OS-level file descriptors can produce various issues (such as unexpected + ordering of data). + + .. versionchanged:: 3.2 + Ignore *name* attribute. + + +.. c:function:: int PyObject_AsFileDescriptor(PyObject *p) + + Return the file descriptor associated with *p* as an :c:type:`int`. If the + object is an integer, its value is returned. If not, the + object's :meth:`fileno` method is called if it exists; the method must return + an integer, which is returned as the file descriptor value. Sets an + exception and returns ``-1`` on failure. + + +.. c:function:: PyObject* PyFile_GetLine(PyObject *p, int n) + + .. index:: single: EOFError (built-in exception) + + Equivalent to ``p.readline([n])``, this function reads one line from the + object *p*. *p* may be a file object or any object with a :meth:`readline` + method. If *n* is ``0``, exactly one line is read, regardless of the length of + the line. If *n* is greater than ``0``, no more than *n* bytes will be read + from the file; a partial line can be returned. In both cases, an empty string + is returned if the end of the file is reached immediately. If *n* is less than + ``0``, however, one line is read regardless of length, but :exc:`EOFError` is + raised if the end of the file is reached immediately. + + +.. c:function:: int PyFile_WriteObject(PyObject *obj, PyObject *p, int flags) + + .. index:: single: Py_PRINT_RAW + + Write object *obj* to file object *p*. The only supported flag for *flags* is + :const:`Py_PRINT_RAW`; if given, the :func:`str` of the object is written + instead of the :func:`repr`. Return ``0`` on success or ``-1`` on failure; the + appropriate exception will be set. + + +.. c:function:: int PyFile_WriteString(const char *s, PyObject *p) + + Write string *s* to file object *p*. Return ``0`` on success or ``-1`` on + failure; the appropriate exception will be set. diff --git a/lib/cpython-doc/c-api/float.rst b/lib/cpython-doc/c-api/float.rst new file mode 100644 --- /dev/null +++ b/lib/cpython-doc/c-api/float.rst @@ -0,0 +1,79 @@ +.. highlightlang:: c + +.. _floatobjects: + +Floating Point Objects +---------------------- + +.. index:: object: floating point + + +.. c:type:: PyFloatObject + + This subtype of :c:type:`PyObject` represents a Python floating point object. + + +.. c:var:: PyTypeObject PyFloat_Type + + This instance of :c:type:`PyTypeObject` represents the Python floating point + type. This is the same object as :class:`float` in the Python layer. + + +.. c:function:: int PyFloat_Check(PyObject *p) + + Return true if its argument is a :c:type:`PyFloatObject` or a subtype of + :c:type:`PyFloatObject`. + + +.. c:function:: int PyFloat_CheckExact(PyObject *p) + + Return true if its argument is a :c:type:`PyFloatObject`, but not a subtype of + :c:type:`PyFloatObject`. + + +.. c:function:: PyObject* PyFloat_FromString(PyObject *str) + + Create a :c:type:`PyFloatObject` object based on the string value in *str*, or + *NULL* on failure. + + +.. c:function:: PyObject* PyFloat_FromDouble(double v) + + Create a :c:type:`PyFloatObject` object from *v*, or *NULL* on failure. + + +.. c:function:: double PyFloat_AsDouble(PyObject *pyfloat) + + Return a C :c:type:`double` representation of the contents of *pyfloat*. If + *pyfloat* is not a Python floating point object but has a :meth:`__float__` + method, this method will first be called to convert *pyfloat* into a float. + This method returns ``-1.0`` upon failure, so one should call + :c:func:`PyErr_Occurred` to check for errors. + + +.. c:function:: double PyFloat_AS_DOUBLE(PyObject *pyfloat) + + Return a C :c:type:`double` representation of the contents of *pyfloat*, but + without error checking. + + +.. c:function:: PyObject* PyFloat_GetInfo(void) + + Return a structseq instance which contains information about the + precision, minimum and maximum values of a float. It's a thin wrapper + around the header file :file:`float.h`. + + +.. c:function:: double PyFloat_GetMax() + + Return the maximum representable finite float *DBL_MAX* as C :c:type:`double`. + + +.. c:function:: double PyFloat_GetMin() + + Return the minimum normalized positive float *DBL_MIN* as C :c:type:`double`. + +.. c:function:: int PyFloat_ClearFreeList() + + Clear the float free list. Return the number of items that could not + be freed. diff --git a/lib/cpython-doc/c-api/function.rst b/lib/cpython-doc/c-api/function.rst new file mode 100644 --- /dev/null +++ b/lib/cpython-doc/c-api/function.rst @@ -0,0 +1,107 @@ +.. highlightlang:: c + +.. _function-objects: + +Function Objects +---------------- + +.. index:: object: function + +There are a few functions specific to Python functions. + + +.. c:type:: PyFunctionObject + + The C structure used for functions. + + +.. c:var:: PyTypeObject PyFunction_Type + + .. index:: single: MethodType (in module types) + + This is an instance of :c:type:`PyTypeObject` and represents the Python function + type. It is exposed to Python programmers as ``types.FunctionType``. + + +.. c:function:: int PyFunction_Check(PyObject *o) + + Return true if *o* is a function object (has type :c:data:`PyFunction_Type`). + The parameter must not be *NULL*. + + +.. c:function:: PyObject* PyFunction_New(PyObject *code, PyObject *globals) + + Return a new function object associated with the code object *code*. *globals* + must be a dictionary with the global variables accessible to the function. + + The function's docstring, name and *__module__* are retrieved from the code + object, the argument defaults and closure are set to *NULL*. + + +.. c:function:: PyObject* PyFunction_NewWithQualName(PyObject *code, PyObject *globals, PyObject *qualname) + + As :c:func:`PyFunction_New`, but also allows to set the function object's + ``__qualname__`` attribute. *qualname* should be a unicode object or NULL; + if NULL, the ``__qualname__`` attribute is set to the same value as its + ``__name__`` attribute. + + .. versionadded:: 3.3 + + +.. c:function:: PyObject* PyFunction_GetCode(PyObject *op) + + Return the code object associated with the function object *op*. + + +.. c:function:: PyObject* PyFunction_GetGlobals(PyObject *op) + + Return the globals dictionary associated with the function object *op*. + + +.. c:function:: PyObject* PyFunction_GetModule(PyObject *op) + + Return the *__module__* attribute of the function object *op*. This is normally + a string containing the module name, but can be set to any other object by + Python code. + + +.. c:function:: PyObject* PyFunction_GetDefaults(PyObject *op) + + Return the argument default values of the function object *op*. This can be a + tuple of arguments or *NULL*. + + +.. c:function:: int PyFunction_SetDefaults(PyObject *op, PyObject *defaults) + + Set the argument default values for the function object *op*. *defaults* must be + *Py_None* or a tuple. + + Raises :exc:`SystemError` and returns ``-1`` on failure. + + +.. c:function:: PyObject* PyFunction_GetClosure(PyObject *op) + + Return the closure associated with the function object *op*. This can be *NULL* + or a tuple of cell objects. + + +.. c:function:: int PyFunction_SetClosure(PyObject *op, PyObject *closure) + + Set the closure associated with the function object *op*. *closure* must be + *Py_None* or a tuple of cell objects. + + Raises :exc:`SystemError` and returns ``-1`` on failure. + + +.. c:function:: PyObject *PyFunction_GetAnnotations(PyObject *op) + + Return the annotations of the function object *op*. This can be a + mutable dictionary or *NULL*. + + +.. c:function:: int PyFunction_SetAnnotations(PyObject *op, PyObject *annotations) + + Set the annotations for the function object *op*. *annotations* + must be a dictionary or *Py_None*. + + Raises :exc:`SystemError` and returns ``-1`` on failure. diff --git a/lib/cpython-doc/c-api/gcsupport.rst b/lib/cpython-doc/c-api/gcsupport.rst new file mode 100644 --- /dev/null +++ b/lib/cpython-doc/c-api/gcsupport.rst @@ -0,0 +1,152 @@ +.. highlightlang:: c + +.. _supporting-cycle-detection: + +Supporting Cyclic Garbage Collection +==================================== + +Python's support for detecting and collecting garbage which involves circular +references requires support from object types which are "containers" for other +objects which may also be containers. Types which do not store references to +other objects, or which only store references to atomic types (such as numbers +or strings), do not need to provide any explicit support for garbage +collection. + +To create a container type, the :attr:`tp_flags` field of the type object must +include the :const:`Py_TPFLAGS_HAVE_GC` and provide an implementation of the +:attr:`tp_traverse` handler. If instances of the type are mutable, a +:attr:`tp_clear` implementation must also be provided. + + +.. data:: Py_TPFLAGS_HAVE_GC + :noindex: + + Objects with a type with this flag set must conform with the rules + documented here. For convenience these objects will be referred to as + container objects. + +Constructors for container types must conform to two rules: + +#. The memory for the object must be allocated using :c:func:`PyObject_GC_New` + or :c:func:`PyObject_GC_NewVar`. + +#. Once all the fields which may contain references to other containers are + initialized, it must call :c:func:`PyObject_GC_Track`. + + +.. c:function:: TYPE* PyObject_GC_New(TYPE, PyTypeObject *type) + + Analogous to :c:func:`PyObject_New` but for container objects with the + :const:`Py_TPFLAGS_HAVE_GC` flag set. + + +.. c:function:: TYPE* PyObject_GC_NewVar(TYPE, PyTypeObject *type, Py_ssize_t size) + + Analogous to :c:func:`PyObject_NewVar` but for container objects with the + :const:`Py_TPFLAGS_HAVE_GC` flag set. + + +.. c:function:: TYPE* PyObject_GC_Resize(TYPE, PyVarObject *op, Py_ssize_t newsize) + + Resize an object allocated by :c:func:`PyObject_NewVar`. Returns the + resized object or *NULL* on failure. + + +.. c:function:: void PyObject_GC_Track(PyObject *op) + + Adds the object *op* to the set of container objects tracked by the + collector. The collector can run at unexpected times so objects must be + valid while being tracked. This should be called once all the fields + followed by the :attr:`tp_traverse` handler become valid, usually near the + end of the constructor. + + +.. c:function:: void _PyObject_GC_TRACK(PyObject *op) + + A macro version of :c:func:`PyObject_GC_Track`. It should not be used for + extension modules. + +Similarly, the deallocator for the object must conform to a similar pair of +rules: + +#. Before fields which refer to other containers are invalidated, + :c:func:`PyObject_GC_UnTrack` must be called. + +#. The object's memory must be deallocated using :c:func:`PyObject_GC_Del`. + + +.. c:function:: void PyObject_GC_Del(void *op) + + Releases memory allocated to an object using :c:func:`PyObject_GC_New` or + :c:func:`PyObject_GC_NewVar`. + + +.. c:function:: void PyObject_GC_UnTrack(void *op) + + Remove the object *op* from the set of container objects tracked by the + collector. Note that :c:func:`PyObject_GC_Track` can be called again on + this object to add it back to the set of tracked objects. The deallocator + (:attr:`tp_dealloc` handler) should call this for the object before any of + the fields used by the :attr:`tp_traverse` handler become invalid. + + +.. c:function:: void _PyObject_GC_UNTRACK(PyObject *op) + + A macro version of :c:func:`PyObject_GC_UnTrack`. It should not be used for + extension modules. + +The :attr:`tp_traverse` handler accepts a function parameter of this type: + + +.. c:type:: int (*visitproc)(PyObject *object, void *arg) + + Type of the visitor function passed to the :attr:`tp_traverse` handler. + The function should be called with an object to traverse as *object* and + the third parameter to the :attr:`tp_traverse` handler as *arg*. The + Python core uses several visitor functions to implement cyclic garbage + detection; it's not expected that users will need to write their own + visitor functions. + +The :attr:`tp_traverse` handler must have the following type: + + +.. c:type:: int (*traverseproc)(PyObject *self, visitproc visit, void *arg) + + Traversal function for a container object. Implementations must call the + *visit* function for each object directly contained by *self*, with the + parameters to *visit* being the contained object and the *arg* value passed + to the handler. The *visit* function must not be called with a *NULL* + object argument. If *visit* returns a non-zero value that value should be + returned immediately. + +To simplify writing :attr:`tp_traverse` handlers, a :c:func:`Py_VISIT` macro is +provided. In order to use this macro, the :attr:`tp_traverse` implementation +must name its arguments exactly *visit* and *arg*: + + +.. c:function:: void Py_VISIT(PyObject *o) + + Call the *visit* callback, with arguments *o* and *arg*. If *visit* returns + a non-zero value, then return it. Using this macro, :attr:`tp_traverse` + handlers look like:: + + static int + my_traverse(Noddy *self, visitproc visit, void *arg) + { + Py_VISIT(self->foo); + Py_VISIT(self->bar); + return 0; + } + +The :attr:`tp_clear` handler must be of the :c:type:`inquiry` type, or *NULL* +if the object is immutable. + + +.. c:type:: int (*inquiry)(PyObject *self) + + Drop references that may have created reference cycles. Immutable objects + do not have to define this method since they can never directly create + reference cycles. Note that the object must still be valid after calling + this method (don't just call :c:func:`Py_DECREF` on a reference). The + collector will call this method if it detects that this object is involved + in a reference cycle. diff --git a/lib/cpython-doc/c-api/gen.rst b/lib/cpython-doc/c-api/gen.rst new file mode 100644 --- /dev/null +++ b/lib/cpython-doc/c-api/gen.rst @@ -0,0 +1,38 @@ +.. highlightlang:: c + +.. _gen-objects: + +Generator Objects +----------------- + +Generator objects are what Python uses to implement generator iterators. They +are normally created by iterating over a function that yields values, rather +than explicitly calling :c:func:`PyGen_New`. + + +.. c:type:: PyGenObject + + The C structure used for generator objects. + + +.. c:var:: PyTypeObject PyGen_Type + + The type object corresponding to generator objects + + +.. c:function:: int PyGen_Check(ob) + + Return true if *ob* is a generator object; *ob* must not be *NULL*. + + +.. c:function:: int PyGen_CheckExact(ob) + + Return true if *ob*'s type is *PyGen_Type* is a generator object; *ob* must not + be *NULL*. + + +.. c:function:: PyObject* PyGen_New(PyFrameObject *frame) + + Create and return a new generator object based on the *frame* object. A + reference to *frame* is stolen by this function. The parameter must not be + *NULL*. diff --git a/lib/cpython-doc/c-api/import.rst b/lib/cpython-doc/c-api/import.rst new file mode 100644 --- /dev/null +++ b/lib/cpython-doc/c-api/import.rst @@ -0,0 +1,300 @@ +.. highlightlang:: c + +.. _importing: + +Importing Modules +================= + + +.. c:function:: PyObject* PyImport_ImportModule(const char *name) + + .. index:: + single: package variable; __all__ + single: __all__ (package variable) + single: modules (in module sys) + + This is a simplified interface to :c:func:`PyImport_ImportModuleEx` below, + leaving the *globals* and *locals* arguments set to *NULL* and *level* set + to 0. When the *name* + argument contains a dot (when it specifies a submodule of a package), the + *fromlist* argument is set to the list ``['*']`` so that the return value is the + named module rather than the top-level package containing it as would otherwise + be the case. (Unfortunately, this has an additional side effect when *name* in + fact specifies a subpackage instead of a submodule: the submodules specified in + the package's ``__all__`` variable are loaded.) Return a new reference to the + imported module, or *NULL* with an exception set on failure. A failing + import of a module doesn't leave the module in :data:`sys.modules`. + + This function always uses absolute imports. + + +.. c:function:: PyObject* PyImport_ImportModuleNoBlock(const char *name) + + This version of :c:func:`PyImport_ImportModule` does not block. It's intended + to be used in C functions that import other modules to execute a function. + The import may block if another thread holds the import lock. The function + :c:func:`PyImport_ImportModuleNoBlock` never blocks. It first tries to fetch + the module from sys.modules and falls back to :c:func:`PyImport_ImportModule` + unless the lock is held, in which case the function will raise an + :exc:`ImportError`. + + +.. c:function:: PyObject* PyImport_ImportModuleEx(char *name, PyObject *globals, PyObject *locals, PyObject *fromlist) + + .. index:: builtin: __import__ + + Import a module. This is best described by referring to the built-in Python + function :func:`__import__`, as the standard :func:`__import__` function calls + this function directly. + + The return value is a new reference to the imported module or top-level + package, or *NULL* with an exception set on failure. Like for + :func:`__import__`, the return value when a submodule of a package was + requested is normally the top-level package, unless a non-empty *fromlist* + was given. + + Failing imports remove incomplete module objects, like with + :c:func:`PyImport_ImportModule`. + + +.. c:function:: PyObject* PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, PyObject *locals, PyObject *fromlist, int level) + + Import a module. This is best described by referring to the built-in Python + function :func:`__import__`, as the standard :func:`__import__` function calls + this function directly. + + The return value is a new reference to the imported module or top-level package, + or *NULL* with an exception set on failure. Like for :func:`__import__`, + the return value when a submodule of a package was requested is normally the + top-level package, unless a non-empty *fromlist* was given. + + .. versionadded:: 3.3 + + +.. c:function:: PyObject* PyImport_ImportModuleLevel(char *name, PyObject *globals, PyObject *locals, PyObject *fromlist, int level) + + Similar to :c:func:`PyImport_ImportModuleLevelObject`, but the name is an + UTF-8 encoded string instead of a Unicode object. + +.. c:function:: PyObject* PyImport_Import(PyObject *name) + + This is a higher-level interface that calls the current "import hook + function" (with an explicit *level* of 0, meaning absolute import). It + invokes the :func:`__import__` function from the ``__builtins__`` of the + current globals. This means that the import is done using whatever import + hooks are installed in the current environment. + + This function always uses absolute imports. + + +.. c:function:: PyObject* PyImport_ReloadModule(PyObject *m) + + Reload a module. Return a new reference to the reloaded module, or *NULL* with + an exception set on failure (the module still exists in this case). + + +.. c:function:: PyObject* PyImport_AddModuleObject(PyObject *name) + + Return the module object corresponding to a module name. The *name* argument + may be of the form ``package.module``. First check the modules dictionary if + there's one there, and if not, create a new one and insert it in the modules + dictionary. Return *NULL* with an exception set on failure. + + .. note:: + + This function does not load or import the module; if the module wasn't already + loaded, you will get an empty module object. Use :c:func:`PyImport_ImportModule` + or one of its variants to import a module. Package structures implied by a + dotted name for *name* are not created if not already present. + + .. versionadded:: 3.3 + + +.. c:function:: PyObject* PyImport_AddModule(const char *name) + + Similar to :c:func:`PyImport_AddModuleObject`, but the name is a UTF-8 + encoded string instead of a Unicode object. + + +.. c:function:: PyObject* PyImport_ExecCodeModule(char *name, PyObject *co) + + .. index:: builtin: compile + + Given a module name (possibly of the form ``package.module``) and a code object + read from a Python bytecode file or obtained from the built-in function + :func:`compile`, load the module. Return a new reference to the module object, + or *NULL* with an exception set if an error occurred. *name* + is removed from :attr:`sys.modules` in error cases, even if *name* was already + in :attr:`sys.modules` on entry to :c:func:`PyImport_ExecCodeModule`. Leaving + incompletely initialized modules in :attr:`sys.modules` is dangerous, as imports of + such modules have no way to know that the module object is an unknown (and + probably damaged with respect to the module author's intents) state. + + The module's :attr:`__file__` attribute will be set to the code object's + :c:member:`co_filename`. + + This function will reload the module if it was already imported. See + :c:func:`PyImport_ReloadModule` for the intended way to reload a module. + + If *name* points to a dotted name of the form ``package.module``, any package + structures not already created will still not be created. + + See also :c:func:`PyImport_ExecCodeModuleEx` and + :c:func:`PyImport_ExecCodeModuleWithPathnames`. + + +.. c:function:: PyObject* PyImport_ExecCodeModuleEx(char *name, PyObject *co, char *pathname) + + Like :c:func:`PyImport_ExecCodeModule`, but the :attr:`__file__` attribute of + the module object is set to *pathname* if it is non-``NULL``. + + See also :c:func:`PyImport_ExecCodeModuleWithPathnames`. + + +.. c:function:: PyObject* PyImport_ExecCodeModuleObject(PyObject *name, PyObject *co, PyObject *pathname, PyObject *cpathname) + + Like :c:func:`PyImport_ExecCodeModuleEx`, but the :attr:`__cached__` + attribute of the module object is set to *cpathname* if it is + non-``NULL``. Of the three functions, this is the preferred one to use. + + .. versionadded:: 3.3 + + +.. c:function:: PyObject* PyImport_ExecCodeModuleWithPathnames(char *name, PyObject *co, char *pathname, char *cpathname) + + Like :c:func:`PyImport_ExecCodeModuleObject`, but *name*, *pathname* and + *cpathname* are UTF-8 encoded strings. + + .. versionadded:: 3.2 + + +.. c:function:: long PyImport_GetMagicNumber() + + Return the magic number for Python bytecode files (a.k.a. :file:`.pyc` and + :file:`.pyo` files). The magic number should be present in the first four bytes + of the bytecode file, in little-endian byte order. + + +.. c:function:: const char * PyImport_GetMagicTag() + + Return the magic tag string for :pep:`3147` format Python bytecode file + names. + + .. versionadded:: 3.2 + +.. c:function:: PyObject* PyImport_GetModuleDict() + + Return the dictionary used for the module administration (a.k.a. + ``sys.modules``). Note that this is a per-interpreter variable. + + +.. c:function:: PyObject* PyImport_GetImporter(PyObject *path) + + Return an importer object for a :data:`sys.path`/:attr:`pkg.__path__` item + *path*, possibly by fetching it from the :data:`sys.path_importer_cache` + dict. If it wasn't yet cached, traverse :data:`sys.path_hooks` until a hook + is found that can handle the path item. Return ``None`` if no hook could; + this tells our caller it should fall back to the built-in import mechanism. + Cache the result in :data:`sys.path_importer_cache`. Return a new reference + to the importer object. + + +.. c:function:: void _PyImport_Init() + + Initialize the import mechanism. For internal use only. + + +.. c:function:: void PyImport_Cleanup() + + Empty the module table. For internal use only. + + +.. c:function:: void _PyImport_Fini() + + Finalize the import mechanism. For internal use only. + + +.. c:function:: PyObject* _PyImport_FindExtension(char *, char *) + + For internal use only. + + +.. c:function:: PyObject* _PyImport_FixupExtension(char *, char *) + + For internal use only. + + +.. c:function:: int PyImport_ImportFrozenModuleObject(PyObject *name) + + Load a frozen module named *name*. Return ``1`` for success, ``0`` if the + module is not found, and ``-1`` with an exception set if the initialization + failed. To access the imported module on a successful load, use + :c:func:`PyImport_ImportModule`. (Note the misnomer --- this function would + reload the module if it was already imported.) + + .. versionadded:: 3.3 + + +.. c:function:: int PyImport_ImportFrozenModule(char *name) + + Similar to :c:func:`PyImport_ImportFrozenModuleObject`, but the name is a + UTF-8 encoded string instead of a Unicode object. + + +.. c:type:: struct _frozen + + .. index:: single: freeze utility + + This is the structure type definition for frozen module descriptors, as + generated by the :program:`freeze` utility (see :file:`Tools/freeze/` in the + Python source distribution). Its definition, found in :file:`Include/import.h`, + is:: + + struct _frozen { + char *name; + unsigned char *code; + int size; + }; + + +.. c:var:: struct _frozen* PyImport_FrozenModules + + This pointer is initialized to point to an array of :c:type:`struct _frozen` + records, terminated by one whose members are all *NULL* or zero. When a frozen + module is imported, it is searched in this table. Third-party code could play + tricks with this to provide a dynamically created collection of frozen modules. + + +.. c:function:: int PyImport_AppendInittab(const char *name, PyObject* (*initfunc)(void)) + + Add a single module to the existing table of built-in modules. This is a + convenience wrapper around :c:func:`PyImport_ExtendInittab`, returning ``-1`` if + the table could not be extended. The new module can be imported by the name + *name*, and uses the function *initfunc* as the initialization function called + on the first attempted import. This should be called before + :c:func:`Py_Initialize`. + + +.. c:type:: struct _inittab + + Structure describing a single entry in the list of built-in modules. Each of + these structures gives the name and initialization function for a module built + into the interpreter. The name is an ASCII encoded string. Programs which + embed Python may use an array of these structures in conjunction with + :c:func:`PyImport_ExtendInittab` to provide additional built-in modules. + The structure is defined in :file:`Include/import.h` as:: + + struct _inittab { + char *name; /* ASCII encoded string */ + PyObject* (*initfunc)(void); + }; + + +.. c:function:: int PyImport_ExtendInittab(struct _inittab *newtab) + + Add a collection of modules to the table of built-in modules. The *newtab* + array must end with a sentinel entry which contains *NULL* for the :attr:`name` + field; failure to provide the sentinel value can result in a memory fault. + Returns ``0`` on success or ``-1`` if insufficient memory could be allocated to + extend the internal table. In the event of failure, no modules are added to the + internal table. This should be called before :c:func:`Py_Initialize`. diff --git a/lib/cpython-doc/c-api/index.rst b/lib/cpython-doc/c-api/index.rst new file mode 100644 --- /dev/null +++ b/lib/cpython-doc/c-api/index.rst @@ -0,0 +1,27 @@ +.. _c-api-index: + +################################## + Python/C API Reference Manual +################################## + +:Release: |version| +:Date: |today| + +This manual documents the API used by C and C++ programmers who want to write +extension modules or embed Python. It is a companion to :ref:`extending-index`, +which describes the general principles of extension writing but does not +document the API functions in detail. + +.. toctree:: + :maxdepth: 2 + + intro.rst + veryhigh.rst + refcounting.rst + exceptions.rst + utilities.rst + abstract.rst + concrete.rst + init.rst + memory.rst + objimpl.rst diff --git a/lib/cpython-doc/c-api/init.rst b/lib/cpython-doc/c-api/init.rst new file mode 100644 --- /dev/null +++ b/lib/cpython-doc/c-api/init.rst @@ -0,0 +1,1134 @@ +.. highlightlang:: c + + +.. _initialization: + +***************************************** +Initialization, Finalization, and Threads +***************************************** + + +Initializing and finalizing the interpreter +=========================================== + + +.. c:function:: void Py_Initialize() + + .. index:: + single: Py_SetProgramName() + single: PyEval_InitThreads() + single: modules (in module sys) + single: path (in module sys) + module: builtins + module: __main__ + module: sys + triple: module; search; path + single: PySys_SetArgv() + single: PySys_SetArgvEx() + single: Py_Finalize() + + Initialize the Python interpreter. In an application embedding Python, this + should be called before using any other Python/C API functions; with the + exception of :c:func:`Py_SetProgramName`, :c:func:`Py_SetPythonHome` and :c:func:`Py_SetPath`. This initializes + the table of loaded modules (``sys.modules``), and creates the fundamental + modules :mod:`builtins`, :mod:`__main__` and :mod:`sys`. It also initializes + the module search path (``sys.path``). It does not set ``sys.argv``; use + :c:func:`PySys_SetArgvEx` for that. This is a no-op when called for a second time + (without calling :c:func:`Py_Finalize` first). There is no return value; it is a + fatal error if the initialization fails. + + +.. c:function:: void Py_InitializeEx(int initsigs) + + This function works like :c:func:`Py_Initialize` if *initsigs* is 1. If + *initsigs* is 0, it skips initialization registration of signal handlers, which + might be useful when Python is embedded. + + +.. c:function:: int Py_IsInitialized() + + Return true (nonzero) when the Python interpreter has been initialized, false + (zero) if not. After :c:func:`Py_Finalize` is called, this returns false until + :c:func:`Py_Initialize` is called again. + + +.. c:function:: void Py_Finalize() + + Undo all initializations made by :c:func:`Py_Initialize` and subsequent use of + Python/C API functions, and destroy all sub-interpreters (see + :c:func:`Py_NewInterpreter` below) that were created and not yet destroyed since + the last call to :c:func:`Py_Initialize`. Ideally, this frees all memory + allocated by the Python interpreter. This is a no-op when called for a second + time (without calling :c:func:`Py_Initialize` again first). There is no return + value; errors during finalization are ignored. + + This function is provided for a number of reasons. An embedding application + might want to restart Python without having to restart the application itself. + An application that has loaded the Python interpreter from a dynamically + loadable library (or DLL) might want to free all memory allocated by Python + before unloading the DLL. During a hunt for memory leaks in an application a + developer might want to free all memory allocated by Python before exiting from + the application. + + **Bugs and caveats:** The destruction of modules and objects in modules is done + in random order; this may cause destructors (:meth:`__del__` methods) to fail + when they depend on other objects (even functions) or modules. Dynamically + loaded extension modules loaded by Python are not unloaded. Small amounts of + memory allocated by the Python interpreter may not be freed (if you find a leak, + please report it). Memory tied up in circular references between objects is not + freed. Some memory allocated by extension modules may not be freed. Some + extensions may not work properly if their initialization routine is called more + than once; this can happen if an application calls :c:func:`Py_Initialize` and + :c:func:`Py_Finalize` more than once. + + +Process-wide parameters +======================= + + +.. c:function:: void Py_SetProgramName(wchar_t *name) + + .. index:: + single: Py_Initialize() + single: main() + single: Py_GetPath() + + This function should be called before :c:func:`Py_Initialize` is called for + the first time, if it is called at all. It tells the interpreter the value + of the ``argv[0]`` argument to the :c:func:`main` function of the program + (converted to wide characters). + This is used by :c:func:`Py_GetPath` and some other functions below to find + the Python run-time libraries relative to the interpreter executable. The + default value is ``'python'``. The argument should point to a + zero-terminated wide character string in static storage whose contents will not + change for the duration of the program's execution. No code in the Python + interpreter will change the contents of this storage. + + +.. c:function:: wchar* Py_GetProgramName() + + .. index:: single: Py_SetProgramName() + + Return the program name set with :c:func:`Py_SetProgramName`, or the default. + The returned string points into static storage; the caller should not modify its + value. + + +.. c:function:: wchar_t* Py_GetPrefix() + + Return the *prefix* for installed platform-independent files. This is derived + through a number of complicated rules from the program name set with + :c:func:`Py_SetProgramName` and some environment variables; for example, if the + program name is ``'/usr/local/bin/python'``, the prefix is ``'/usr/local'``. The + returned string points into static storage; the caller should not modify its + value. This corresponds to the :makevar:`prefix` variable in the top-level + :file:`Makefile` and the ``--prefix`` argument to the :program:`configure` + script at build time. The value is available to Python code as ``sys.prefix``. + It is only useful on Unix. See also the next function. + + +.. c:function:: wchar_t* Py_GetExecPrefix() + + Return the *exec-prefix* for installed platform-*dependent* files. This is + derived through a number of complicated rules from the program name set with + :c:func:`Py_SetProgramName` and some environment variables; for example, if the + program name is ``'/usr/local/bin/python'``, the exec-prefix is + ``'/usr/local'``. The returned string points into static storage; the caller + should not modify its value. This corresponds to the :makevar:`exec_prefix` + variable in the top-level :file:`Makefile` and the ``--exec-prefix`` + argument to the :program:`configure` script at build time. The value is + available to Python code as ``sys.exec_prefix``. It is only useful on Unix. + + Background: The exec-prefix differs from the prefix when platform dependent + files (such as executables and shared libraries) are installed in a different + directory tree. In a typical installation, platform dependent files may be + installed in the :file:`/usr/local/plat` subtree while platform independent may + be installed in :file:`/usr/local`. + + Generally speaking, a platform is a combination of hardware and software + families, e.g. Sparc machines running the Solaris 2.x operating system are + considered the same platform, but Intel machines running Solaris 2.x are another + platform, and Intel machines running Linux are yet another platform. Different + major revisions of the same operating system generally also form different + platforms. Non-Unix operating systems are a different story; the installation + strategies on those systems are so different that the prefix and exec-prefix are + meaningless, and set to the empty string. Note that compiled Python bytecode + files are platform independent (but not independent from the Python version by + which they were compiled!). + + System administrators will know how to configure the :program:`mount` or + :program:`automount` programs to share :file:`/usr/local` between platforms + while having :file:`/usr/local/plat` be a different filesystem for each + platform. + + +.. c:function:: wchar_t* Py_GetProgramFullPath() + + .. index:: + single: Py_SetProgramName() + single: executable (in module sys) + + Return the full program name of the Python executable; this is computed as a + side-effect of deriving the default module search path from the program name + (set by :c:func:`Py_SetProgramName` above). The returned string points into + static storage; the caller should not modify its value. The value is available + to Python code as ``sys.executable``. + + +.. c:function:: wchar_t* Py_GetPath() + + .. index:: + triple: module; search; path + single: path (in module sys) + single: Py_SetPath() + + Return the default module search path; this is computed from the program name + (set by :c:func:`Py_SetProgramName` above) and some environment variables. + The returned string consists of a series of directory names separated by a + platform dependent delimiter character. The delimiter character is ``':'`` + on Unix and Mac OS X, ``';'`` on Windows. The returned string points into + static storage; the caller should not modify its value. The list + :data:`sys.path` is initialized with this value on interpreter startup; it + can be (and usually is) modified later to change the search path for loading + modules. + + .. XXX should give the exact rules + + +.. c:function:: void Py_SetPath(const wchar_t *) + + .. index:: + triple: module; search; path + single: path (in module sys) + single: Py_GetPath() + + Set the default module search path. If this function is called before + :c:func:`Py_Initialize`, then :c:func:`Py_GetPath` won't attempt to compute a + default search path but uses the one provided instead. This is useful if + Python is embedded by an application that has full knowledge of the location + of all modules. The path components should be separated by semicolons. + + This also causes :data:`sys.executable` to be set only to the raw program + name (see :c:func:`Py_SetProgramName`) and for :data:`sys.prefix` and + :data:`sys.exec_prefix` to be empty. It is up to the caller to modify these + if required after calling :c:func:`Py_Initialize`. + + +.. c:function:: const char* Py_GetVersion() + + Return the version of this Python interpreter. This is a string that looks + something like :: + + "3.0a5+ (py3k:63103M, May 12 2008, 00:53:55) \n[GCC 4.2.3]" + + .. index:: single: version (in module sys) + + The first word (up to the first space character) is the current Python version; + the first three characters are the major and minor version separated by a + period. The returned string points into static storage; the caller should not + modify its value. The value is available to Python code as :data:`sys.version`. + + +.. c:function:: const char* Py_GetPlatform() + + .. index:: single: platform (in module sys) + + Return the platform identifier for the current platform. On Unix, this is + formed from the "official" name of the operating system, converted to lower + case, followed by the major revision number; e.g., for Solaris 2.x, which is + also known as SunOS 5.x, the value is ``'sunos5'``. On Mac OS X, it is + ``'darwin'``. On Windows, it is ``'win'``. The returned string points into + static storage; the caller should not modify its value. The value is available + to Python code as ``sys.platform``. + + +.. c:function:: const char* Py_GetCopyright() + + Return the official copyright string for the current Python version, for example + + ``'Copyright 1991-1995 Stichting Mathematisch Centrum, Amsterdam'`` + + .. index:: single: copyright (in module sys) + + The returned string points into static storage; the caller should not modify its + value. The value is available to Python code as ``sys.copyright``. + + +.. c:function:: const char* Py_GetCompiler() + + Return an indication of the compiler used to build the current Python version, + in square brackets, for example:: + + "[GCC 2.7.2.2]" + + .. index:: single: version (in module sys) + + The returned string points into static storage; the caller should not modify its + value. The value is available to Python code as part of the variable + ``sys.version``. + + +.. c:function:: const char* Py_GetBuildInfo() + + Return information about the sequence number and build date and time of the + current Python interpreter instance, for example :: + + "#67, Aug 1 1997, 22:34:28" + + .. index:: single: version (in module sys) + + The returned string points into static storage; the caller should not modify its + value. The value is available to Python code as part of the variable + ``sys.version``. + + +.. c:function:: void PySys_SetArgvEx(int argc, wchar_t **argv, int updatepath) + + .. index:: + single: main() + single: Py_FatalError() + single: argv (in module sys) + + Set :data:`sys.argv` based on *argc* and *argv*. These parameters are + similar to those passed to the program's :c:func:`main` function with the + difference that the first entry should refer to the script file to be + executed rather than the executable hosting the Python interpreter. If there + isn't a script that will be run, the first entry in *argv* can be an empty + string. If this function fails to initialize :data:`sys.argv`, a fatal + condition is signalled using :c:func:`Py_FatalError`. + + If *updatepath* is zero, this is all the function does. If *updatepath* + is non-zero, the function also modifies :data:`sys.path` according to the + following algorithm: + + - If the name of an existing script is passed in ``argv[0]``, the absolute + path of the directory where the script is located is prepended to + :data:`sys.path`. + - Otherwise (that is, if *argc* is 0 or ``argv[0]`` doesn't point + to an existing file name), an empty string is prepended to + :data:`sys.path`, which is the same as prepending the current working + directory (``"."``). + + .. note:: + It is recommended that applications embedding the Python interpreter + for purposes other than executing a single script pass 0 as *updatepath*, + and update :data:`sys.path` themselves if desired. + See `CVE-2008-5983 `_. + + On versions before 3.1.3, you can achieve the same effect by manually + popping the first :data:`sys.path` element after having called + :c:func:`PySys_SetArgv`, for example using:: + + PyRun_SimpleString("import sys; sys.path.pop(0)\n"); + + .. versionadded:: 3.1.3 + + .. XXX impl. doesn't seem consistent in allowing 0/NULL for the params; + check w/ Guido. + + +.. c:function:: void PySys_SetArgv(int argc, wchar_t **argv) + + This function works like :c:func:`PySys_SetArgvEx` with *updatepath* set to 1. + + +.. c:function:: void Py_SetPythonHome(wchar_t *home) + + Set the default "home" directory, that is, the location of the standard + Python libraries. See :envvar:`PYTHONHOME` for the meaning of the + argument string. + + The argument should point to a zero-terminated character string in static + storage whose contents will not change for the duration of the program's + execution. No code in the Python interpreter will change the contents of + this storage. + + +.. c:function:: w_char* Py_GetPythonHome() + + Return the default "home", that is, the value set by a previous call to + :c:func:`Py_SetPythonHome`, or the value of the :envvar:`PYTHONHOME` + environment variable if it is set. + + +.. _threads: + +Thread State and the Global Interpreter Lock +============================================ + +.. index:: + single: global interpreter lock + single: interpreter lock + single: lock, interpreter + +The Python interpreter is not fully thread-safe. In order to support +multi-threaded Python programs, there's a global lock, called the :term:`global +interpreter lock` or :term:`GIL`, that must be held by the current thread before +it can safely access Python objects. Without the lock, even the simplest +operations could cause problems in a multi-threaded program: for example, when +two threads simultaneously increment the reference count of the same object, the +reference count could end up being incremented only once instead of twice. + +.. index:: single: setswitchinterval() (in module sys) + +Therefore, the rule exists that only the thread that has acquired the +:term:`GIL` may operate on Python objects or call Python/C API functions. +In order to emulate concurrency of execution, the interpreter regularly +tries to switch threads (see :func:`sys.setswitchinterval`). The lock is also +released around potentially blocking I/O operations like reading or writing +a file, so that other Python threads can run in the meantime. + +.. index:: + single: PyThreadState + single: PyThreadState + +The Python interpreter keeps some thread-specific bookkeeping information +inside a data structure called :c:type:`PyThreadState`. There's also one +global variable pointing to the current :c:type:`PyThreadState`: it can +be retrieved using :c:func:`PyThreadState_Get`. + +Releasing the GIL from extension code +------------------------------------- + +Most extension code manipulating the :term:`GIL` has the following simple +structure:: + + Save the thread state in a local variable. + Release the global interpreter lock. + ... Do some blocking I/O operation ... + Reacquire the global interpreter lock. + Restore the thread state from the local variable. + +This is so common that a pair of macros exists to simplify it:: + + Py_BEGIN_ALLOW_THREADS + ... Do some blocking I/O operation ... + Py_END_ALLOW_THREADS + +.. index:: + single: Py_BEGIN_ALLOW_THREADS + single: Py_END_ALLOW_THREADS + +The :c:macro:`Py_BEGIN_ALLOW_THREADS` macro opens a new block and declares a +hidden local variable; the :c:macro:`Py_END_ALLOW_THREADS` macro closes the +block. These two macros are still available when Python is compiled without +thread support (they simply have an empty expansion). + +When thread support is enabled, the block above expands to the following code:: + + PyThreadState *_save; + + _save = PyEval_SaveThread(); + ...Do some blocking I/O operation... + PyEval_RestoreThread(_save); + +.. index:: + single: PyEval_RestoreThread() + single: PyEval_SaveThread() + +Here is how these functions work: the global interpreter lock is used to protect the pointer to the +current thread state. When releasing the lock and saving the thread state, +the current thread state pointer must be retrieved before the lock is released +(since another thread could immediately acquire the lock and store its own thread +state in the global variable). Conversely, when acquiring the lock and restoring +the thread state, the lock must be acquired before storing the thread state +pointer. + +.. note:: + Calling system I/O functions is the most common use case for releasing + the GIL, but it can also be useful before calling long-running computations + which don't need access to Python objects, such as compression or + cryptographic functions operating over memory buffers. For example, the + standard :mod:`zlib` and :mod:`hashlib` modules release the GIL when + compressing or hashing data. + +Non-Python created threads +-------------------------- + +When threads are created using the dedicated Python APIs (such as the +:mod:`threading` module), a thread state is automatically associated to them +and the code showed above is therefore correct. However, when threads are +created from C (for example by a third-party library with its own thread +management), they don't hold the GIL, nor is there a thread state structure +for them. + +If you need to call Python code from these threads (often this will be part +of a callback API provided by the aforementioned third-party library), +you must first register these threads with the interpreter by +creating a thread state data structure, then acquiring the GIL, and finally +storing their thread state pointer, before you can start using the Python/C +API. When you are done, you should reset the thread state pointer, release +the GIL, and finally free the thread state data structure. + +The :c:func:`PyGILState_Ensure` and :c:func:`PyGILState_Release` functions do +all of the above automatically. The typical idiom for calling into Python +from a C thread is:: + + PyGILState_STATE gstate; + gstate = PyGILState_Ensure(); + + /* Perform Python actions here. */ + result = CallSomeFunction(); + /* evaluate result or handle exception */ + + /* Release the thread. No Python API allowed beyond this point. */ + PyGILState_Release(gstate); + +Note that the :c:func:`PyGILState_\*` functions assume there is only one global +interpreter (created automatically by :c:func:`Py_Initialize`). Python +supports the creation of additional interpreters (using +:c:func:`Py_NewInterpreter`), but mixing multiple interpreters and the +:c:func:`PyGILState_\*` API is unsupported. + +Another important thing to note about threads is their behaviour in the face +of the C :c:func:`fork` call. On most systems with :c:func:`fork`, after a +process forks only the thread that issued the fork will exist. That also +means any locks held by other threads will never be released. Python solves +this for :func:`os.fork` by acquiring the locks it uses internally before +the fork, and releasing them afterwards. In addition, it resets any +:ref:`lock-objects` in the child. When extending or embedding Python, there +is no way to inform Python of additional (non-Python) locks that need to be +acquired before or reset after a fork. OS facilities such as +:c:func:`pthread_atfork` would need to be used to accomplish the same thing. +Additionally, when extending or embedding Python, calling :c:func:`fork` +directly rather than through :func:`os.fork` (and returning to or calling +into Python) may result in a deadlock by one of Python's internal locks +being held by a thread that is defunct after the fork. +:c:func:`PyOS_AfterFork` tries to reset the necessary locks, but is not +always able to. + + +High-level API +-------------- + +These are the most commonly used types and functions when writing C extension +code, or when embedding the Python interpreter: + +.. c:type:: PyInterpreterState + + This data structure represents the state shared by a number of cooperating + threads. Threads belonging to the same interpreter share their module + administration and a few other internal items. There are no public members in + this structure. + + Threads belonging to different interpreters initially share nothing, except + process state like available memory, open file descriptors and such. The global + interpreter lock is also shared by all threads, regardless of to which + interpreter they belong. + + +.. c:type:: PyThreadState + + This data structure represents the state of a single thread. The only public + data member is :c:type:`PyInterpreterState \*`:attr:`interp`, which points to + this thread's interpreter state. + + +.. c:function:: void PyEval_InitThreads() + + .. index:: + single: PyEval_AcquireThread() + single: PyEval_ReleaseThread() + single: PyEval_SaveThread() + single: PyEval_RestoreThread() + + Initialize and acquire the global interpreter lock. It should be called in the + main thread before creating a second thread or engaging in any other thread + operations such as ``PyEval_ReleaseThread(tstate)``. It is not needed before + calling :c:func:`PyEval_SaveThread` or :c:func:`PyEval_RestoreThread`. + + This is a no-op when called for a second time. + + .. versionchanged:: 3.2 + This function cannot be called before :c:func:`Py_Initialize()` anymore. + + .. index:: module: _thread + + .. note:: + When only the main thread exists, no GIL operations are needed. This is a + common situation (most Python programs do not use threads), and the lock + operations slow the interpreter down a bit. Therefore, the lock is not + created initially. This situation is equivalent to having acquired the lock: + when there is only a single thread, all object accesses are safe. Therefore, + when this function initializes the global interpreter lock, it also acquires + it. Before the Python :mod:`_thread` module creates a new thread, knowing + that either it has the lock or the lock hasn't been created yet, it calls + :c:func:`PyEval_InitThreads`. When this call returns, it is guaranteed that + the lock has been created and that the calling thread has acquired it. + + It is **not** safe to call this function when it is unknown which thread (if + any) currently has the global interpreter lock. + + This function is not available when thread support is disabled at compile time. + + +.. c:function:: int PyEval_ThreadsInitialized() + + Returns a non-zero value if :c:func:`PyEval_InitThreads` has been called. This + function can be called without holding the GIL, and therefore can be used to + avoid calls to the locking API when running single-threaded. This function is + not available when thread support is disabled at compile time. + + +.. c:function:: PyThreadState* PyEval_SaveThread() + + Release the global interpreter lock (if it has been created and thread + support is enabled) and reset the thread state to *NULL*, returning the + previous thread state (which is not *NULL*). If the lock has been created, + the current thread must have acquired it. (This function is available even + when thread support is disabled at compile time.) + + +.. c:function:: void PyEval_RestoreThread(PyThreadState *tstate) + + Acquire the global interpreter lock (if it has been created and thread + support is enabled) and set the thread state to *tstate*, which must not be + *NULL*. If the lock has been created, the current thread must not have + acquired it, otherwise deadlock ensues. (This function is available even + when thread support is disabled at compile time.) + + +.. c:function:: PyThreadState* PyThreadState_Get() + + Return the current thread state. The global interpreter lock must be held. + When the current thread state is *NULL*, this issues a fatal error (so that + the caller needn't check for *NULL*). + + +.. c:function:: PyThreadState* PyThreadState_Swap(PyThreadState *tstate) + + Swap the current thread state with the thread state given by the argument + *tstate*, which may be *NULL*. The global interpreter lock must be held + and is not released. + + +.. c:function:: void PyEval_ReInitThreads() + + This function is called from :c:func:`PyOS_AfterFork` to ensure that newly + created child processes don't hold locks referring to threads which + are not running in the child process. + + +The following functions use thread-local storage, and are not compatible +with sub-interpreters: + +.. c:function:: PyGILState_STATE PyGILState_Ensure() + + Ensure that the current thread is ready to call the Python C API regardless + of the current state of Python, or of the global interpreter lock. This may + be called as many times as desired by a thread as long as each call is + matched with a call to :c:func:`PyGILState_Release`. In general, other + thread-related APIs may be used between :c:func:`PyGILState_Ensure` and + :c:func:`PyGILState_Release` calls as long as the thread state is restored to + its previous state before the Release(). For example, normal usage of the + :c:macro:`Py_BEGIN_ALLOW_THREADS` and :c:macro:`Py_END_ALLOW_THREADS` macros is + acceptable. + + The return value is an opaque "handle" to the thread state when + :c:func:`PyGILState_Ensure` was called, and must be passed to + :c:func:`PyGILState_Release` to ensure Python is left in the same state. Even + though recursive calls are allowed, these handles *cannot* be shared - each + unique call to :c:func:`PyGILState_Ensure` must save the handle for its call + to :c:func:`PyGILState_Release`. + + When the function returns, the current thread will hold the GIL and be able + to call arbitrary Python code. Failure is a fatal error. + + +.. c:function:: void PyGILState_Release(PyGILState_STATE) + + Release any resources previously acquired. After this call, Python's state will + be the same as it was prior to the corresponding :c:func:`PyGILState_Ensure` call + (but generally this state will be unknown to the caller, hence the use of the + GILState API). + + Every call to :c:func:`PyGILState_Ensure` must be matched by a call to + :c:func:`PyGILState_Release` on the same thread. + + +.. c:function:: PyThreadState PyGILState_GetThisThreadState() + + Get the current thread state for this thread. May return ``NULL`` if no + GILState API has been used on the current thread. Note that the main thread + always has such a thread-state, even if no auto-thread-state call has been + made on the main thread. This is mainly a helper/diagnostic function. + + +The following macros are normally used without a trailing semicolon; look for +example usage in the Python source distribution. + + +.. c:macro:: Py_BEGIN_ALLOW_THREADS + + This macro expands to ``{ PyThreadState *_save; _save = PyEval_SaveThread();``. + Note that it contains an opening brace; it must be matched with a following + :c:macro:`Py_END_ALLOW_THREADS` macro. See above for further discussion of this + macro. It is a no-op when thread support is disabled at compile time. + + +.. c:macro:: Py_END_ALLOW_THREADS + + This macro expands to ``PyEval_RestoreThread(_save); }``. Note that it contains + a closing brace; it must be matched with an earlier + :c:macro:`Py_BEGIN_ALLOW_THREADS` macro. See above for further discussion of + this macro. It is a no-op when thread support is disabled at compile time. + + +.. c:macro:: Py_BLOCK_THREADS + + This macro expands to ``PyEval_RestoreThread(_save);``: it is equivalent to + :c:macro:`Py_END_ALLOW_THREADS` without the closing brace. It is a no-op when + thread support is disabled at compile time. + + +.. c:macro:: Py_UNBLOCK_THREADS + + This macro expands to ``_save = PyEval_SaveThread();``: it is equivalent to + :c:macro:`Py_BEGIN_ALLOW_THREADS` without the opening brace and variable + declaration. It is a no-op when thread support is disabled at compile time. + + +Low-level API +------------- + +All of the following functions are only available when thread support is enabled +at compile time, and must be called only when the global interpreter lock has +been created. + + +.. c:function:: PyInterpreterState* PyInterpreterState_New() + + Create a new interpreter state object. The global interpreter lock need not + be held, but may be held if it is necessary to serialize calls to this + function. + + +.. c:function:: void PyInterpreterState_Clear(PyInterpreterState *interp) + + Reset all information in an interpreter state object. The global interpreter + lock must be held. + + +.. c:function:: void PyInterpreterState_Delete(PyInterpreterState *interp) + + Destroy an interpreter state object. The global interpreter lock need not be + held. The interpreter state must have been reset with a previous call to + :c:func:`PyInterpreterState_Clear`. + + +.. c:function:: PyThreadState* PyThreadState_New(PyInterpreterState *interp) + + Create a new thread state object belonging to the given interpreter object. + The global interpreter lock need not be held, but may be held if it is + necessary to serialize calls to this function. + + +.. c:function:: void PyThreadState_Clear(PyThreadState *tstate) + + Reset all information in a thread state object. The global interpreter lock + must be held. + + +.. c:function:: void PyThreadState_Delete(PyThreadState *tstate) + + Destroy a thread state object. The global interpreter lock need not be held. + The thread state must have been reset with a previous call to + :c:func:`PyThreadState_Clear`. + + +.. c:function:: PyObject* PyThreadState_GetDict() + + Return a dictionary in which extensions can store thread-specific state + information. Each extension should use a unique key to use to store state in + the dictionary. It is okay to call this function when no current thread state + is available. If this function returns *NULL*, no exception has been raised and + the caller should assume no current thread state is available. + + +.. c:function:: int PyThreadState_SetAsyncExc(long id, PyObject *exc) + + Asynchronously raise an exception in a thread. The *id* argument is the thread + id of the target thread; *exc* is the exception object to be raised. This + function does not steal any references to *exc*. To prevent naive misuse, you + must write your own C extension to call this. Must be called with the GIL held. + Returns the number of thread states modified; this is normally one, but will be + zero if the thread id isn't found. If *exc* is :const:`NULL`, the pending + exception (if any) for the thread is cleared. This raises no exceptions. + + +.. c:function:: void PyEval_AcquireThread(PyThreadState *tstate) + + Acquire the global interpreter lock and set the current thread state to + *tstate*, which should not be *NULL*. The lock must have been created earlier. + If this thread already has the lock, deadlock ensues. + + :c:func:`PyEval_RestoreThread` is a higher-level function which is always + available (even when thread support isn't enabled or when threads have + not been initialized). + + +.. c:function:: void PyEval_ReleaseThread(PyThreadState *tstate) + + Reset the current thread state to *NULL* and release the global interpreter + lock. The lock must have been created earlier and must be held by the current + thread. The *tstate* argument, which must not be *NULL*, is only used to check + that it represents the current thread state --- if it isn't, a fatal error is + reported. + + :c:func:`PyEval_SaveThread` is a higher-level function which is always + available (even when thread support isn't enabled or when threads have + not been initialized). + + +.. c:function:: void PyEval_AcquireLock() + + Acquire the global interpreter lock. The lock must have been created earlier. + If this thread already has the lock, a deadlock ensues. + + .. deprecated:: 3.2 + This function does not update the current thread state. Please use + :c:func:`PyEval_RestoreThread` or :c:func:`PyEval_AcquireThread` + instead. + + +.. c:function:: void PyEval_ReleaseLock() + + Release the global interpreter lock. The lock must have been created earlier. + + .. deprecated:: 3.2 + This function does not update the current thread state. Please use + :c:func:`PyEval_SaveThread` or :c:func:`PyEval_ReleaseThread` + instead. + + +Sub-interpreter support +======================= + +While in most uses, you will only embed a single Python interpreter, there +are cases where you need to create several independent interpreters in the +same process and perhaps even in the same thread. Sub-interpreters allow +you to do that. You can switch between sub-interpreters using the +:c:func:`PyThreadState_Swap` function. You can create and destroy them +using the following functions: + + +.. c:function:: PyThreadState* Py_NewInterpreter() + + .. index:: + module: builtins + module: __main__ + module: sys + single: stdout (in module sys) + single: stderr (in module sys) + single: stdin (in module sys) + + Create a new sub-interpreter. This is an (almost) totally separate environment + for the execution of Python code. In particular, the new interpreter has + separate, independent versions of all imported modules, including the + fundamental modules :mod:`builtins`, :mod:`__main__` and :mod:`sys`. The + table of loaded modules (``sys.modules``) and the module search path + (``sys.path``) are also separate. The new environment has no ``sys.argv`` + variable. It has new standard I/O stream file objects ``sys.stdin``, + ``sys.stdout`` and ``sys.stderr`` (however these refer to the same underlying + file descriptors). + + The return value points to the first thread state created in the new + sub-interpreter. This thread state is made in the current thread state. + Note that no actual thread is created; see the discussion of thread states + below. If creation of the new interpreter is unsuccessful, *NULL* is + returned; no exception is set since the exception state is stored in the + current thread state and there may not be a current thread state. (Like all + other Python/C API functions, the global interpreter lock must be held before + calling this function and is still held when it returns; however, unlike most + other Python/C API functions, there needn't be a current thread state on + entry.) + + .. index:: + single: Py_Finalize() + single: Py_Initialize() + + Extension modules are shared between (sub-)interpreters as follows: the first + time a particular extension is imported, it is initialized normally, and a + (shallow) copy of its module's dictionary is squirreled away. When the same + extension is imported by another (sub-)interpreter, a new module is initialized + and filled with the contents of this copy; the extension's ``init`` function is + not called. Note that this is different from what happens when an extension is + imported after the interpreter has been completely re-initialized by calling + :c:func:`Py_Finalize` and :c:func:`Py_Initialize`; in that case, the extension's + ``initmodule`` function *is* called again. + + .. index:: single: close() (in module os) + + +.. c:function:: void Py_EndInterpreter(PyThreadState *tstate) + + .. index:: single: Py_Finalize() + + Destroy the (sub-)interpreter represented by the given thread state. The given + thread state must be the current thread state. See the discussion of thread + states below. When the call returns, the current thread state is *NULL*. All + thread states associated with this interpreter are destroyed. (The global + interpreter lock must be held before calling this function and is still held + when it returns.) :c:func:`Py_Finalize` will destroy all sub-interpreters that + haven't been explicitly destroyed at that point. + + +Bugs and caveats +---------------- + +Because sub-interpreters (and the main interpreter) are part of the same +process, the insulation between them isn't perfect --- for example, using +low-level file operations like :func:`os.close` they can +(accidentally or maliciously) affect each other's open files. Because of the +way extensions are shared between (sub-)interpreters, some extensions may not +work properly; this is especially likely when the extension makes use of +(static) global variables, or when the extension manipulates its module's +dictionary after its initialization. It is possible to insert objects created +in one sub-interpreter into a namespace of another sub-interpreter; this should +be done with great care to avoid sharing user-defined functions, methods, +instances or classes between sub-interpreters, since import operations executed +by such objects may affect the wrong (sub-)interpreter's dictionary of loaded +modules. + +Also note that combining this functionality with :c:func:`PyGILState_\*` APIs +is delicate, because these APIs assume a bijection between Python thread states +and OS-level threads, an assumption broken by the presence of sub-interpreters. +It is highly recommended that you don't switch sub-interpreters between a pair +of matching :c:func:`PyGILState_Ensure` and :c:func:`PyGILState_Release` calls. +Furthermore, extensions (such as :mod:`ctypes`) using these APIs to allow calling +of Python code from non-Python created threads will probably be broken when using +sub-interpreters. + + +Asynchronous Notifications +========================== + +A mechanism is provided to make asynchronous notifications to the main +interpreter thread. These notifications take the form of a function +pointer and a void argument. + +.. index:: single: setcheckinterval() (in module sys) + +Every check interval, when the global interpreter lock is released and +reacquired, Python will also call any such provided functions. This can be used +for example by asynchronous IO handlers. The notification can be scheduled from +a worker thread and the actual call than made at the earliest convenience by the +main thread where it has possession of the global interpreter lock and can +perform any Python API calls. + +.. c:function:: int Py_AddPendingCall(int (*func)(void *), void *arg) + + .. index:: single: Py_AddPendingCall() + + Post a notification to the Python main thread. If successful, *func* will be + called with the argument *arg* at the earliest convenience. *func* will be + called having the global interpreter lock held and can thus use the full + Python API and can take any action such as setting object attributes to + signal IO completion. It must return 0 on success, or -1 signalling an + exception. The notification function won't be interrupted to perform another + asynchronous notification recursively, but it can still be interrupted to + switch threads if the global interpreter lock is released, for example, if it + calls back into Python code. + + This function returns 0 on success in which case the notification has been + scheduled. Otherwise, for example if the notification buffer is full, it + returns -1 without setting any exception. + + This function can be called on any thread, be it a Python thread or some + other system thread. If it is a Python thread, it doesn't matter if it holds + the global interpreter lock or not. + + .. versionadded:: 3.1 + + +.. _profiling: + +Profiling and Tracing +===================== + +.. sectionauthor:: Fred L. Drake, Jr. + + +The Python interpreter provides some low-level support for attaching profiling +and execution tracing facilities. These are used for profiling, debugging, and +coverage analysis tools. + +This C interface allows the profiling or tracing code to avoid the overhead of +calling through Python-level callable objects, making a direct C function call +instead. The essential attributes of the facility have not changed; the +interface allows trace functions to be installed per-thread, and the basic +events reported to the trace function are the same as had been reported to the +Python-level trace functions in previous versions. + + +.. c:type:: int (*Py_tracefunc)(PyObject *obj, PyFrameObject *frame, int what, PyObject *arg) + + The type of the trace function registered using :c:func:`PyEval_SetProfile` and + :c:func:`PyEval_SetTrace`. The first parameter is the object passed to the + registration function as *obj*, *frame* is the frame object to which the event + pertains, *what* is one of the constants :const:`PyTrace_CALL`, + :const:`PyTrace_EXCEPTION`, :const:`PyTrace_LINE`, :const:`PyTrace_RETURN`, + :const:`PyTrace_C_CALL`, :const:`PyTrace_C_EXCEPTION`, or + :const:`PyTrace_C_RETURN`, and *arg* depends on the value of *what*: + + +------------------------------+--------------------------------------+ + | Value of *what* | Meaning of *arg* | + +==============================+======================================+ + | :const:`PyTrace_CALL` | Always *NULL*. | + +------------------------------+--------------------------------------+ + | :const:`PyTrace_EXCEPTION` | Exception information as returned by | + | | :func:`sys.exc_info`. | + +------------------------------+--------------------------------------+ + | :const:`PyTrace_LINE` | Always *NULL*. | + +------------------------------+--------------------------------------+ + | :const:`PyTrace_RETURN` | Value being returned to the caller, | + | | or *NULL* if caused by an exception. | + +------------------------------+--------------------------------------+ + | :const:`PyTrace_C_CALL` | Function object being called. | + +------------------------------+--------------------------------------+ + | :const:`PyTrace_C_EXCEPTION` | Function object being called. | + +------------------------------+--------------------------------------+ + | :const:`PyTrace_C_RETURN` | Function object being called. | + +------------------------------+--------------------------------------+ + + +.. c:var:: int PyTrace_CALL + + The value of the *what* parameter to a :c:type:`Py_tracefunc` function when a new + call to a function or method is being reported, or a new entry into a generator. + Note that the creation of the iterator for a generator function is not reported + as there is no control transfer to the Python bytecode in the corresponding + frame. + + +.. c:var:: int PyTrace_EXCEPTION + + The value of the *what* parameter to a :c:type:`Py_tracefunc` function when an + exception has been raised. The callback function is called with this value for + *what* when after any bytecode is processed after which the exception becomes + set within the frame being executed. The effect of this is that as exception + propagation causes the Python stack to unwind, the callback is called upon + return to each frame as the exception propagates. Only trace functions receives + these events; they are not needed by the profiler. + + +.. c:var:: int PyTrace_LINE + + The value passed as the *what* parameter to a trace function (but not a + profiling function) when a line-number event is being reported. + + +.. c:var:: int PyTrace_RETURN + + The value for the *what* parameter to :c:type:`Py_tracefunc` functions when a + call is returning without propagating an exception. + + +.. c:var:: int PyTrace_C_CALL + + The value for the *what* parameter to :c:type:`Py_tracefunc` functions when a C + function is about to be called. + + +.. c:var:: int PyTrace_C_EXCEPTION + + The value for the *what* parameter to :c:type:`Py_tracefunc` functions when a C + function has raised an exception. + + +.. c:var:: int PyTrace_C_RETURN + + The value for the *what* parameter to :c:type:`Py_tracefunc` functions when a C + function has returned. + + +.. c:function:: void PyEval_SetProfile(Py_tracefunc func, PyObject *obj) + + Set the profiler function to *func*. The *obj* parameter is passed to the + function as its first parameter, and may be any Python object, or *NULL*. If + the profile function needs to maintain state, using a different value for *obj* + for each thread provides a convenient and thread-safe place to store it. The + profile function is called for all monitored events except the line-number + events. + + +.. c:function:: void PyEval_SetTrace(Py_tracefunc func, PyObject *obj) + + Set the tracing function to *func*. This is similar to + :c:func:`PyEval_SetProfile`, except the tracing function does receive line-number + events. + +.. c:function:: PyObject* PyEval_GetCallStats(PyObject *self) + + Return a tuple of function call counts. There are constants defined for the + positions within the tuple: + + +-------------------------------+-------+ + | Name | Value | + +===============================+=======+ + | :const:`PCALL_ALL` | 0 | + +-------------------------------+-------+ + | :const:`PCALL_FUNCTION` | 1 | + +-------------------------------+-------+ + | :const:`PCALL_FAST_FUNCTION` | 2 | + +-------------------------------+-------+ + | :const:`PCALL_FASTER_FUNCTION`| 3 | + +-------------------------------+-------+ + | :const:`PCALL_METHOD` | 4 | + +-------------------------------+-------+ + | :const:`PCALL_BOUND_METHOD` | 5 | + +-------------------------------+-------+ + | :const:`PCALL_CFUNCTION` | 6 | + +-------------------------------+-------+ + | :const:`PCALL_TYPE` | 7 | + +-------------------------------+-------+ + | :const:`PCALL_GENERATOR` | 8 | + +-------------------------------+-------+ + | :const:`PCALL_OTHER` | 9 | + +-------------------------------+-------+ + | :const:`PCALL_POP` | 10 | + +-------------------------------+-------+ + + :const:`PCALL_FAST_FUNCTION` means no argument tuple needs to be created. + :const:`PCALL_FASTER_FUNCTION` means that the fast-path frame setup code is used. + + If there is a method call where the call can be optimized by changing + the argument tuple and calling the function directly, it gets recorded + twice. + + This function is only present if Python is compiled with :const:`CALL_PROFILE` + defined. + +.. _advanced-debugging: + +Advanced Debugger Support +========================= + +.. sectionauthor:: Fred L. Drake, Jr. + + +These functions are only intended to be used by advanced debugging tools. + + +.. c:function:: PyInterpreterState* PyInterpreterState_Head() + + Return the interpreter state object at the head of the list of all such objects. + + +.. c:function:: PyInterpreterState* PyInterpreterState_Next(PyInterpreterState *interp) + + Return the next interpreter state object after *interp* from the list of all + such objects. + + +.. c:function:: PyThreadState * PyInterpreterState_ThreadHead(PyInterpreterState *interp) + + Return the a pointer to the first :c:type:`PyThreadState` object in the list of + threads associated with the interpreter *interp*. + + +.. c:function:: PyThreadState* PyThreadState_Next(PyThreadState *tstate) + + Return the next thread state object after *tstate* from the list of all such + objects belonging to the same :c:type:`PyInterpreterState` object. + diff --git a/lib/cpython-doc/c-api/intro.rst b/lib/cpython-doc/c-api/intro.rst new file mode 100644 --- /dev/null +++ b/lib/cpython-doc/c-api/intro.rst @@ -0,0 +1,630 @@ +.. highlightlang:: c + + +.. _api-intro: + +************ +Introduction +************ + +The Application Programmer's Interface to Python gives C and C++ programmers +access to the Python interpreter at a variety of levels. The API is equally +usable from C++, but for brevity it is generally referred to as the Python/C +API. There are two fundamentally different reasons for using the Python/C API. +The first reason is to write *extension modules* for specific purposes; these +are C modules that extend the Python interpreter. This is probably the most +common use. The second reason is to use Python as a component in a larger +application; this technique is generally referred to as :dfn:`embedding` Python +in an application. + +Writing an extension module is a relatively well-understood process, where a +"cookbook" approach works well. There are several tools that automate the +process to some extent. While people have embedded Python in other +applications since its early existence, the process of embedding Python is less +straightforward than writing an extension. + +Many API functions are useful independent of whether you're embedding or +extending Python; moreover, most applications that embed Python will need to +provide a custom extension as well, so it's probably a good idea to become +familiar with writing an extension before attempting to embed Python in a real +application. + + +.. _api-includes: + +Include Files +============= + +All function, type and macro definitions needed to use the Python/C API are +included in your code by the following line:: + + #include "Python.h" + +This implies inclusion of the following standard headers: ````, +````, ````, ````, ```` and ```` +(if available). + +.. note:: + + Since Python may define some pre-processor definitions which affect the standard + headers on some systems, you *must* include :file:`Python.h` before any standard + headers are included. + +All user visible names defined by Python.h (except those defined by the included +standard headers) have one of the prefixes ``Py`` or ``_Py``. Names beginning +with ``_Py`` are for internal use by the Python implementation and should not be +used by extension writers. Structure member names do not have a reserved prefix. + +**Important:** user code should never define names that begin with ``Py`` or +``_Py``. This confuses the reader, and jeopardizes the portability of the user +code to future Python versions, which may define additional names beginning with +one of these prefixes. + +The header files are typically installed with Python. On Unix, these are +located in the directories :file:`{prefix}/include/pythonversion/` and +:file:`{exec_prefix}/include/pythonversion/`, where :envvar:`prefix` and +:envvar:`exec_prefix` are defined by the corresponding parameters to Python's +:program:`configure` script and *version* is ``sys.version[:3]``. On Windows, +the headers are installed in :file:`{prefix}/include`, where :envvar:`prefix` is +the installation directory specified to the installer. + +To include the headers, place both directories (if different) on your compiler's +search path for includes. Do *not* place the parent directories on the search +path and then use ``#include ``; this will break on +multi-platform builds since the platform independent headers under +:envvar:`prefix` include the platform specific headers from +:envvar:`exec_prefix`. + +C++ users should note that though the API is defined entirely using C, the +header files do properly declare the entry points to be ``extern "C"``, so there +is no need to do anything special to use the API from C++. + + +.. _api-objects: + +Objects, Types and Reference Counts +=================================== + +.. index:: object: type + +Most Python/C API functions have one or more arguments as well as a return value +of type :c:type:`PyObject\*`. This type is a pointer to an opaque data type +representing an arbitrary Python object. Since all Python object types are +treated the same way by the Python language in most situations (e.g., +assignments, scope rules, and argument passing), it is only fitting that they +should be represented by a single C type. Almost all Python objects live on the +heap: you never declare an automatic or static variable of type +:c:type:`PyObject`, only pointer variables of type :c:type:`PyObject\*` can be +declared. The sole exception are the type objects; since these must never be +deallocated, they are typically static :c:type:`PyTypeObject` objects. + +All Python objects (even Python integers) have a :dfn:`type` and a +:dfn:`reference count`. An object's type determines what kind of object it is +(e.g., an integer, a list, or a user-defined function; there are many more as +explained in :ref:`types`). For each of the well-known types there is a macro +to check whether an object is of that type; for instance, ``PyList_Check(a)`` is +true if (and only if) the object pointed to by *a* is a Python list. + + +.. _api-refcounts: + +Reference Counts +---------------- + +The reference count is important because today's computers have a finite (and +often severely limited) memory size; it counts how many different places there +are that have a reference to an object. Such a place could be another object, +or a global (or static) C variable, or a local variable in some C function. +When an object's reference count becomes zero, the object is deallocated. If +it contains references to other objects, their reference count is decremented. +Those other objects may be deallocated in turn, if this decrement makes their +reference count become zero, and so on. (There's an obvious problem with +objects that reference each other here; for now, the solution is "don't do +that.") + +.. index:: + single: Py_INCREF() + single: Py_DECREF() + +Reference counts are always manipulated explicitly. The normal way is to use +the macro :c:func:`Py_INCREF` to increment an object's reference count by one, +and :c:func:`Py_DECREF` to decrement it by one. The :c:func:`Py_DECREF` macro +is considerably more complex than the incref one, since it must check whether +the reference count becomes zero and then cause the object's deallocator to be +called. The deallocator is a function pointer contained in the object's type +structure. The type-specific deallocator takes care of decrementing the +reference counts for other objects contained in the object if this is a compound +object type, such as a list, as well as performing any additional finalization +that's needed. There's no chance that the reference count can overflow; at +least as many bits are used to hold the reference count as there are distinct +memory locations in virtual memory (assuming ``sizeof(Py_ssize_t) >= sizeof(void*)``). +Thus, the reference count increment is a simple operation. + +It is not necessary to increment an object's reference count for every local +variable that contains a pointer to an object. In theory, the object's +reference count goes up by one when the variable is made to point to it and it +goes down by one when the variable goes out of scope. However, these two +cancel each other out, so at the end the reference count hasn't changed. The +only real reason to use the reference count is to prevent the object from being +deallocated as long as our variable is pointing to it. If we know that there +is at least one other reference to the object that lives at least as long as +our variable, there is no need to increment the reference count temporarily. +An important situation where this arises is in objects that are passed as +arguments to C functions in an extension module that are called from Python; +the call mechanism guarantees to hold a reference to every argument for the +duration of the call. + +However, a common pitfall is to extract an object from a list and hold on to it +for a while without incrementing its reference count. Some other operation might +conceivably remove the object from the list, decrementing its reference count +and possible deallocating it. The real danger is that innocent-looking +operations may invoke arbitrary Python code which could do this; there is a code +path which allows control to flow back to the user from a :c:func:`Py_DECREF`, so +almost any operation is potentially dangerous. + +A safe approach is to always use the generic operations (functions whose name +begins with ``PyObject_``, ``PyNumber_``, ``PySequence_`` or ``PyMapping_``). +These operations always increment the reference count of the object they return. +This leaves the caller with the responsibility to call :c:func:`Py_DECREF` when +they are done with the result; this soon becomes second nature. + + +.. _api-refcountdetails: + +Reference Count Details +^^^^^^^^^^^^^^^^^^^^^^^ + +The reference count behavior of functions in the Python/C API is best explained +in terms of *ownership of references*. Ownership pertains to references, never +to objects (objects are not owned: they are always shared). "Owning a +reference" means being responsible for calling Py_DECREF on it when the +reference is no longer needed. Ownership can also be transferred, meaning that +the code that receives ownership of the reference then becomes responsible for +eventually decref'ing it by calling :c:func:`Py_DECREF` or :c:func:`Py_XDECREF` +when it's no longer needed---or passing on this responsibility (usually to its +caller). When a function passes ownership of a reference on to its caller, the +caller is said to receive a *new* reference. When no ownership is transferred, +the caller is said to *borrow* the reference. Nothing needs to be done for a +borrowed reference. + +Conversely, when a calling function passes in a reference to an object, there +are two possibilities: the function *steals* a reference to the object, or it +does not. *Stealing a reference* means that when you pass a reference to a +function, that function assumes that it now owns that reference, and you are not +responsible for it any longer. + +.. index:: + single: PyList_SetItem() + single: PyTuple_SetItem() + +Few functions steal references; the two notable exceptions are +:c:func:`PyList_SetItem` and :c:func:`PyTuple_SetItem`, which steal a reference +to the item (but not to the tuple or list into which the item is put!). These +functions were designed to steal a reference because of a common idiom for +populating a tuple or list with newly created objects; for example, the code to +create the tuple ``(1, 2, "three")`` could look like this (forgetting about +error handling for the moment; a better way to code this is shown below):: + + PyObject *t; + + t = PyTuple_New(3); + PyTuple_SetItem(t, 0, PyLong_FromLong(1L)); + PyTuple_SetItem(t, 1, PyLong_FromLong(2L)); + PyTuple_SetItem(t, 2, PyString_FromString("three")); + +Here, :c:func:`PyLong_FromLong` returns a new reference which is immediately +stolen by :c:func:`PyTuple_SetItem`. When you want to keep using an object +although the reference to it will be stolen, use :c:func:`Py_INCREF` to grab +another reference before calling the reference-stealing function. + +Incidentally, :c:func:`PyTuple_SetItem` is the *only* way to set tuple items; +:c:func:`PySequence_SetItem` and :c:func:`PyObject_SetItem` refuse to do this +since tuples are an immutable data type. You should only use +:c:func:`PyTuple_SetItem` for tuples that you are creating yourself. + +Equivalent code for populating a list can be written using :c:func:`PyList_New` +and :c:func:`PyList_SetItem`. + +However, in practice, you will rarely use these ways of creating and populating +a tuple or list. There's a generic function, :c:func:`Py_BuildValue`, that can +create most common objects from C values, directed by a :dfn:`format string`. +For example, the above two blocks of code could be replaced by the following +(which also takes care of the error checking):: + + PyObject *tuple, *list; + + tuple = Py_BuildValue("(iis)", 1, 2, "three"); + list = Py_BuildValue("[iis]", 1, 2, "three"); + +It is much more common to use :c:func:`PyObject_SetItem` and friends with items +whose references you are only borrowing, like arguments that were passed in to +the function you are writing. In that case, their behaviour regarding reference +counts is much saner, since you don't have to increment a reference count so you +can give a reference away ("have it be stolen"). For example, this function +sets all items of a list (actually, any mutable sequence) to a given item:: + + int + set_all(PyObject *target, PyObject *item) + { + int i, n; + + n = PyObject_Length(target); + if (n < 0) + return -1; + for (i = 0; i < n; i++) { + PyObject *index = PyLong_FromLong(i); + if (!index) + return -1; + if (PyObject_SetItem(target, index, item) < 0) + return -1; + Py_DECREF(index); + } + return 0; + } + +.. index:: single: set_all() + +The situation is slightly different for function return values. While passing +a reference to most functions does not change your ownership responsibilities +for that reference, many functions that return a reference to an object give +you ownership of the reference. The reason is simple: in many cases, the +returned object is created on the fly, and the reference you get is the only +reference to the object. Therefore, the generic functions that return object +references, like :c:func:`PyObject_GetItem` and :c:func:`PySequence_GetItem`, +always return a new reference (the caller becomes the owner of the reference). + +It is important to realize that whether you own a reference returned by a +function depends on which function you call only --- *the plumage* (the type of +the object passed as an argument to the function) *doesn't enter into it!* +Thus, if you extract an item from a list using :c:func:`PyList_GetItem`, you +don't own the reference --- but if you obtain the same item from the same list +using :c:func:`PySequence_GetItem` (which happens to take exactly the same +arguments), you do own a reference to the returned object. + +.. index:: + single: PyList_GetItem() + single: PySequence_GetItem() + +Here is an example of how you could write a function that computes the sum of +the items in a list of integers; once using :c:func:`PyList_GetItem`, and once +using :c:func:`PySequence_GetItem`. :: + + long + sum_list(PyObject *list) + { + int i, n; + long total = 0; + PyObject *item; + + n = PyList_Size(list); + if (n < 0) + return -1; /* Not a list */ + for (i = 0; i < n; i++) { + item = PyList_GetItem(list, i); /* Can't fail */ + if (!PyLong_Check(item)) continue; /* Skip non-integers */ + total += PyLong_AsLong(item); + } + return total; + } + +.. index:: single: sum_list() + +:: + + long + sum_sequence(PyObject *sequence) + { + int i, n; + long total = 0; + PyObject *item; + n = PySequence_Length(sequence); + if (n < 0) + return -1; /* Has no length */ + for (i = 0; i < n; i++) { + item = PySequence_GetItem(sequence, i); + if (item == NULL) + return -1; /* Not a sequence, or other failure */ + if (PyLong_Check(item)) + total += PyLong_AsLong(item); + Py_DECREF(item); /* Discard reference ownership */ + } + return total; + } + +.. index:: single: sum_sequence() + + +.. _api-types: + +Types +----- + +There are few other data types that play a significant role in the Python/C +API; most are simple C types such as :c:type:`int`, :c:type:`long`, +:c:type:`double` and :c:type:`char\*`. A few structure types are used to +describe static tables used to list the functions exported by a module or the +data attributes of a new object type, and another is used to describe the value +of a complex number. These will be discussed together with the functions that +use them. + + +.. _api-exceptions: + +Exceptions +========== + +The Python programmer only needs to deal with exceptions if specific error +handling is required; unhandled exceptions are automatically propagated to the +caller, then to the caller's caller, and so on, until they reach the top-level +interpreter, where they are reported to the user accompanied by a stack +traceback. + +.. index:: single: PyErr_Occurred() + +For C programmers, however, error checking always has to be explicit. All +functions in the Python/C API can raise exceptions, unless an explicit claim is +made otherwise in a function's documentation. In general, when a function +encounters an error, it sets an exception, discards any object references that +it owns, and returns an error indicator. If not documented otherwise, this +indicator is either *NULL* or ``-1``, depending on the function's return type. +A few functions return a Boolean true/false result, with false indicating an +error. Very few functions return no explicit error indicator or have an +ambiguous return value, and require explicit testing for errors with +:c:func:`PyErr_Occurred`. These exceptions are always explicitly documented. + +.. index:: + single: PyErr_SetString() + single: PyErr_Clear() + +Exception state is maintained in per-thread storage (this is equivalent to +using global storage in an unthreaded application). A thread can be in one of +two states: an exception has occurred, or not. The function +:c:func:`PyErr_Occurred` can be used to check for this: it returns a borrowed +reference to the exception type object when an exception has occurred, and +*NULL* otherwise. There are a number of functions to set the exception state: +:c:func:`PyErr_SetString` is the most common (though not the most general) +function to set the exception state, and :c:func:`PyErr_Clear` clears the +exception state. + +The full exception state consists of three objects (all of which can be +*NULL*): the exception type, the corresponding exception value, and the +traceback. These have the same meanings as the Python result of +``sys.exc_info()``; however, they are not the same: the Python objects represent +the last exception being handled by a Python :keyword:`try` ... +:keyword:`except` statement, while the C level exception state only exists while +an exception is being passed on between C functions until it reaches the Python +bytecode interpreter's main loop, which takes care of transferring it to +``sys.exc_info()`` and friends. + +.. index:: single: exc_info() (in module sys) + +Note that starting with Python 1.5, the preferred, thread-safe way to access the +exception state from Python code is to call the function :func:`sys.exc_info`, +which returns the per-thread exception state for Python code. Also, the +semantics of both ways to access the exception state have changed so that a +function which catches an exception will save and restore its thread's exception +state so as to preserve the exception state of its caller. This prevents common +bugs in exception handling code caused by an innocent-looking function +overwriting the exception being handled; it also reduces the often unwanted +lifetime extension for objects that are referenced by the stack frames in the +traceback. + +As a general principle, a function that calls another function to perform some +task should check whether the called function raised an exception, and if so, +pass the exception state on to its caller. It should discard any object +references that it owns, and return an error indicator, but it should *not* set +another exception --- that would overwrite the exception that was just raised, +and lose important information about the exact cause of the error. + +.. index:: single: sum_sequence() + +A simple example of detecting exceptions and passing them on is shown in the +:c:func:`sum_sequence` example above. It so happens that that example doesn't +need to clean up any owned references when it detects an error. The following +example function shows some error cleanup. First, to remind you why you like +Python, we show the equivalent Python code:: + + def incr_item(dict, key): + try: + item = dict[key] + except KeyError: + item = 0 + dict[key] = item + 1 + +.. index:: single: incr_item() + +Here is the corresponding C code, in all its glory:: + + int + incr_item(PyObject *dict, PyObject *key) + { + /* Objects all initialized to NULL for Py_XDECREF */ + PyObject *item = NULL, *const_one = NULL, *incremented_item = NULL; + int rv = -1; /* Return value initialized to -1 (failure) */ + + item = PyObject_GetItem(dict, key); + if (item == NULL) { + /* Handle KeyError only: */ + if (!PyErr_ExceptionMatches(PyExc_KeyError)) + goto error; + + /* Clear the error and use zero: */ + PyErr_Clear(); + item = PyLong_FromLong(0L); + if (item == NULL) + goto error; + } + const_one = PyLong_FromLong(1L); + if (const_one == NULL) + goto error; + + incremented_item = PyNumber_Add(item, const_one); + if (incremented_item == NULL) + goto error; + + if (PyObject_SetItem(dict, key, incremented_item) < 0) + goto error; + rv = 0; /* Success */ + /* Continue with cleanup code */ + + error: + /* Cleanup code, shared by success and failure path */ + + /* Use Py_XDECREF() to ignore NULL references */ + Py_XDECREF(item); + Py_XDECREF(const_one); + Py_XDECREF(incremented_item); + + return rv; /* -1 for error, 0 for success */ + } + +.. index:: single: incr_item() + +.. index:: + single: PyErr_ExceptionMatches() + single: PyErr_Clear() + single: Py_XDECREF() + +This example represents an endorsed use of the ``goto`` statement in C! +It illustrates the use of :c:func:`PyErr_ExceptionMatches` and +:c:func:`PyErr_Clear` to handle specific exceptions, and the use of +:c:func:`Py_XDECREF` to dispose of owned references that may be *NULL* (note the +``'X'`` in the name; :c:func:`Py_DECREF` would crash when confronted with a +*NULL* reference). It is important that the variables used to hold owned +references are initialized to *NULL* for this to work; likewise, the proposed +return value is initialized to ``-1`` (failure) and only set to success after +the final call made is successful. + + +.. _api-embedding: + +Embedding Python +================ + +The one important task that only embedders (as opposed to extension writers) of +the Python interpreter have to worry about is the initialization, and possibly +the finalization, of the Python interpreter. Most functionality of the +interpreter can only be used after the interpreter has been initialized. + +.. index:: + single: Py_Initialize() + module: builtins + module: __main__ + module: sys + triple: module; search; path + single: path (in module sys) + +The basic initialization function is :c:func:`Py_Initialize`. This initializes +the table of loaded modules, and creates the fundamental modules +:mod:`builtins`, :mod:`__main__`, and :mod:`sys`. It also +initializes the module search path (``sys.path``). + +.. index:: single: PySys_SetArgvEx() + +:c:func:`Py_Initialize` does not set the "script argument list" (``sys.argv``). +If this variable is needed by Python code that will be executed later, it must +be set explicitly with a call to ``PySys_SetArgvEx(argc, argv, updatepath)`` +after the call to :c:func:`Py_Initialize`. + +On most systems (in particular, on Unix and Windows, although the details are +slightly different), :c:func:`Py_Initialize` calculates the module search path +based upon its best guess for the location of the standard Python interpreter +executable, assuming that the Python library is found in a fixed location +relative to the Python interpreter executable. In particular, it looks for a +directory named :file:`lib/python{X.Y}` relative to the parent directory +where the executable named :file:`python` is found on the shell command search +path (the environment variable :envvar:`PATH`). + +For instance, if the Python executable is found in +:file:`/usr/local/bin/python`, it will assume that the libraries are in +:file:`/usr/local/lib/python{X.Y}`. (In fact, this particular path is also +the "fallback" location, used when no executable file named :file:`python` is +found along :envvar:`PATH`.) The user can override this behavior by setting the +environment variable :envvar:`PYTHONHOME`, or insert additional directories in +front of the standard path by setting :envvar:`PYTHONPATH`. + +.. index:: + single: Py_SetProgramName() + single: Py_GetPath() + single: Py_GetPrefix() + single: Py_GetExecPrefix() + single: Py_GetProgramFullPath() + +The embedding application can steer the search by calling +``Py_SetProgramName(file)`` *before* calling :c:func:`Py_Initialize`. Note that +:envvar:`PYTHONHOME` still overrides this and :envvar:`PYTHONPATH` is still +inserted in front of the standard path. An application that requires total +control has to provide its own implementation of :c:func:`Py_GetPath`, +:c:func:`Py_GetPrefix`, :c:func:`Py_GetExecPrefix`, and +:c:func:`Py_GetProgramFullPath` (all defined in :file:`Modules/getpath.c`). + +.. index:: single: Py_IsInitialized() + +Sometimes, it is desirable to "uninitialize" Python. For instance, the +application may want to start over (make another call to +:c:func:`Py_Initialize`) or the application is simply done with its use of +Python and wants to free memory allocated by Python. This can be accomplished +by calling :c:func:`Py_Finalize`. The function :c:func:`Py_IsInitialized` returns +true if Python is currently in the initialized state. More information about +these functions is given in a later chapter. Notice that :c:func:`Py_Finalize` +does *not* free all memory allocated by the Python interpreter, e.g. memory +allocated by extension modules currently cannot be released. + + +.. _api-debugging: + +Debugging Builds +================ + +Python can be built with several macros to enable extra checks of the +interpreter and extension modules. These checks tend to add a large amount of +overhead to the runtime so they are not enabled by default. + +A full list of the various types of debugging builds is in the file +:file:`Misc/SpecialBuilds.txt` in the Python source distribution. Builds are +available that support tracing of reference counts, debugging the memory +allocator, or low-level profiling of the main interpreter loop. Only the most +frequently-used builds will be described in the remainder of this section. + +Compiling the interpreter with the :c:macro:`Py_DEBUG` macro defined produces +what is generally meant by "a debug build" of Python. :c:macro:`Py_DEBUG` is +enabled in the Unix build by adding ``--with-pydebug`` to the +:file:`./configure` command. It is also implied by the presence of the +not-Python-specific :c:macro:`_DEBUG` macro. When :c:macro:`Py_DEBUG` is enabled +in the Unix build, compiler optimization is disabled. + +In addition to the reference count debugging described below, the following +extra checks are performed: + +* Extra checks are added to the object allocator. + +* Extra checks are added to the parser and compiler. + +* Downcasts from wide types to narrow types are checked for loss of information. + +* A number of assertions are added to the dictionary and set implementations. + In addition, the set object acquires a :meth:`test_c_api` method. + +* Sanity checks of the input arguments are added to frame creation. + +* The storage for ints is initialized with a known invalid pattern to catch + reference to uninitialized digits. + +* Low-level tracing and extra exception checking are added to the runtime + virtual machine. + +* Extra checks are added to the memory arena implementation. + +* Extra debugging is added to the thread module. + +There may be additional checks not mentioned here. + +Defining :c:macro:`Py_TRACE_REFS` enables reference tracing. When defined, a +circular doubly linked list of active objects is maintained by adding two extra +fields to every :c:type:`PyObject`. Total allocations are tracked as well. Upon +exit, all existing references are printed. (In interactive mode this happens +after every statement run by the interpreter.) Implied by :c:macro:`Py_DEBUG`. + +Please refer to :file:`Misc/SpecialBuilds.txt` in the Python source distribution +for more detailed information. + diff --git a/lib/cpython-doc/c-api/iter.rst b/lib/cpython-doc/c-api/iter.rst new file mode 100644 --- /dev/null +++ b/lib/cpython-doc/c-api/iter.rst @@ -0,0 +1,47 @@ +.. highlightlang:: c + +.. _iterator: + +Iterator Protocol +================= + +There are only a couple of functions specifically for working with iterators. + +.. c:function:: int PyIter_Check(PyObject *o) + + Return true if the object *o* supports the iterator protocol. + + +.. c:function:: PyObject* PyIter_Next(PyObject *o) + + Return the next value from the iteration *o*. If the object is an iterator, + this retrieves the next value from the iteration, and returns *NULL* with no + exception set if there are no remaining items. If the object is not an + iterator, :exc:`TypeError` is raised, or if there is an error in retrieving the + item, returns *NULL* and passes along the exception. + +To write a loop which iterates over an iterator, the C code should look +something like this:: + + PyObject *iterator = PyObject_GetIter(obj); + PyObject *item; + + if (iterator == NULL) { + /* propagate error */ + } + + while (item = PyIter_Next(iterator)) { + /* do something with item */ + ... + /* release reference when done */ + Py_DECREF(item); + } + + Py_DECREF(iterator); + + if (PyErr_Occurred()) { + /* propagate error */ + } + else { + /* continue doing useful work */ + } diff --git a/lib/cpython-doc/c-api/iterator.rst b/lib/cpython-doc/c-api/iterator.rst new file mode 100644 --- /dev/null +++ b/lib/cpython-doc/c-api/iterator.rst @@ -0,0 +1,50 @@ +.. highlightlang:: c + +.. _iterator-objects: + +Iterator Objects +---------------- + +Python provides two general-purpose iterator objects. The first, a sequence +iterator, works with an arbitrary sequence supporting the :meth:`__getitem__` +method. The second works with a callable object and a sentinel value, calling +the callable for each item in the sequence, and ending the iteration when the +sentinel value is returned. + + +.. c:var:: PyTypeObject PySeqIter_Type + + Type object for iterator objects returned by :c:func:`PySeqIter_New` and the + one-argument form of the :func:`iter` built-in function for built-in sequence + types. + + +.. c:function:: int PySeqIter_Check(op) + + Return true if the type of *op* is :c:data:`PySeqIter_Type`. + + +.. c:function:: PyObject* PySeqIter_New(PyObject *seq) + + Return an iterator that works with a general sequence object, *seq*. The + iteration ends when the sequence raises :exc:`IndexError` for the subscripting + operation. + + +.. c:var:: PyTypeObject PyCallIter_Type + + Type object for iterator objects returned by :c:func:`PyCallIter_New` and the + two-argument form of the :func:`iter` built-in function. + + +.. c:function:: int PyCallIter_Check(op) + + Return true if the type of *op* is :c:data:`PyCallIter_Type`. + + +.. c:function:: PyObject* PyCallIter_New(PyObject *callable, PyObject *sentinel) + + Return a new iterator. The first parameter, *callable*, can be any Python + callable object that can be called with no parameters; each call to it should + return the next item in the iteration. When *callable* returns a value equal to + *sentinel*, the iteration will be terminated. diff --git a/lib/cpython-doc/c-api/list.rst b/lib/cpython-doc/c-api/list.rst new file mode 100644 --- /dev/null +++ b/lib/cpython-doc/c-api/list.rst @@ -0,0 +1,151 @@ +.. highlightlang:: c + +.. _listobjects: + +List Objects +------------ + +.. index:: object: list + + +.. c:type:: PyListObject + + This subtype of :c:type:`PyObject` represents a Python list object. + + +.. c:var:: PyTypeObject PyList_Type + + This instance of :c:type:`PyTypeObject` represents the Python list type. + This is the same object as :class:`list` in the Python layer. + + +.. c:function:: int PyList_Check(PyObject *p) + + Return true if *p* is a list object or an instance of a subtype of the list + type. + + +.. c:function:: int PyList_CheckExact(PyObject *p) + + Return true if *p* is a list object, but not an instance of a subtype of + the list type. + + +.. c:function:: PyObject* PyList_New(Py_ssize_t len) + + Return a new list of length *len* on success, or *NULL* on failure. + + .. note:: + + If *len* is greater than zero, the returned list object's items are + set to ``NULL``. Thus you cannot use abstract API functions such as + :c:func:`PySequence_SetItem` or expose the object to Python code before + setting all items to a real object with :c:func:`PyList_SetItem`. + + +.. c:function:: Py_ssize_t PyList_Size(PyObject *list) + + .. index:: builtin: len + + Return the length of the list object in *list*; this is equivalent to + ``len(list)`` on a list object. + + +.. c:function:: Py_ssize_t PyList_GET_SIZE(PyObject *list) + + Macro form of :c:func:`PyList_Size` without error checking. + + +.. c:function:: PyObject* PyList_GetItem(PyObject *list, Py_ssize_t index) + + Return the object at position *index* in the list pointed to by *list*. The + position must be positive, indexing from the end of the list is not + supported. If *index* is out of bounds, return *NULL* and set an + :exc:`IndexError` exception. + + +.. c:function:: PyObject* PyList_GET_ITEM(PyObject *list, Py_ssize_t i) + + Macro form of :c:func:`PyList_GetItem` without error checking. + + +.. c:function:: int PyList_SetItem(PyObject *list, Py_ssize_t index, PyObject *item) + + Set the item at index *index* in list to *item*. Return ``0`` on success + or ``-1`` on failure. + + .. note:: + + This function "steals" a reference to *item* and discards a reference to + an item already in the list at the affected position. + + +.. c:function:: void PyList_SET_ITEM(PyObject *list, Py_ssize_t i, PyObject *o) + + Macro form of :c:func:`PyList_SetItem` without error checking. This is + normally only used to fill in new lists where there is no previous content. + + .. note:: + + This macro "steals" a reference to *item*, and, unlike + :c:func:`PyList_SetItem`, does *not* discard a reference to any item that + is being replaced; any reference in *list* at position *i* will be + leaked. + + +.. c:function:: int PyList_Insert(PyObject *list, Py_ssize_t index, PyObject *item) + + Insert the item *item* into list *list* in front of index *index*. Return + ``0`` if successful; return ``-1`` and set an exception if unsuccessful. + Analogous to ``list.insert(index, item)``. + + +.. c:function:: int PyList_Append(PyObject *list, PyObject *item) + + Append the object *item* at the end of list *list*. Return ``0`` if + successful; return ``-1`` and set an exception if unsuccessful. Analogous + to ``list.append(item)``. + + +.. c:function:: PyObject* PyList_GetSlice(PyObject *list, Py_ssize_t low, Py_ssize_t high) + + Return a list of the objects in *list* containing the objects *between* *low* + and *high*. Return *NULL* and set an exception if unsuccessful. Analogous + to ``list[low:high]``. Negative indices, as when slicing from Python, are not + supported. + + +.. c:function:: int PyList_SetSlice(PyObject *list, Py_ssize_t low, Py_ssize_t high, PyObject *itemlist) + + Set the slice of *list* between *low* and *high* to the contents of + *itemlist*. Analogous to ``list[low:high] = itemlist``. The *itemlist* may + be *NULL*, indicating the assignment of an empty list (slice deletion). + Return ``0`` on success, ``-1`` on failure. Negative indices, as when + slicing from Python, are not supported. + + +.. c:function:: int PyList_Sort(PyObject *list) + + Sort the items of *list* in place. Return ``0`` on success, ``-1`` on + failure. This is equivalent to ``list.sort()``. + + +.. c:function:: int PyList_Reverse(PyObject *list) + + Reverse the items of *list* in place. Return ``0`` on success, ``-1`` on + failure. This is the equivalent of ``list.reverse()``. + + +.. c:function:: PyObject* PyList_AsTuple(PyObject *list) + + .. index:: builtin: tuple + + Return a new tuple object containing the contents of *list*; equivalent to + ``tuple(list)``. + + +.. c:function:: int PyList_ClearFreeList() + + Clear the free list. Return the total number of freed items. + + .. versionadded:: 3.3 diff --git a/lib/cpython-doc/c-api/long.rst b/lib/cpython-doc/c-api/long.rst new file mode 100644 --- /dev/null +++ b/lib/cpython-doc/c-api/long.rst @@ -0,0 +1,237 @@ +.. highlightlang:: c + +.. _longobjects: + +Integer Objects +--------------- + +.. index:: object: long integer + object: integer + +All integers are implemented as "long" integer objects of arbitrary size. + +.. c:type:: PyLongObject + + This subtype of :c:type:`PyObject` represents a Python integer object. + + +.. c:var:: PyTypeObject PyLong_Type + + This instance of :c:type:`PyTypeObject` represents the Python integer type. + This is the same object as :class:`int` in the Python layer. + + +.. c:function:: int PyLong_Check(PyObject *p) + + Return true if its argument is a :c:type:`PyLongObject` or a subtype of + :c:type:`PyLongObject`. + + +.. c:function:: int PyLong_CheckExact(PyObject *p) + + Return true if its argument is a :c:type:`PyLongObject`, but not a subtype of + :c:type:`PyLongObject`. + + +.. c:function:: PyObject* PyLong_FromLong(long v) + + Return a new :c:type:`PyLongObject` object from *v*, or *NULL* on failure. + + The current implementation keeps an array of integer objects for all integers + between ``-5`` and ``256``, when you create an int in that range you actually + just get back a reference to the existing object. So it should be possible to + change the value of ``1``. I suspect the behaviour of Python in this case is + undefined. :-) + + +.. c:function:: PyObject* PyLong_FromUnsignedLong(unsigned long v) + + Return a new :c:type:`PyLongObject` object from a C :c:type:`unsigned long`, or + *NULL* on failure. + + +.. c:function:: PyObject* PyLong_FromSsize_t(Py_ssize_t v) + + Return a new :c:type:`PyLongObject` object from a C :c:type:`Py_ssize_t`, or + *NULL* on failure. + + +.. c:function:: PyObject* PyLong_FromSize_t(size_t v) + + Return a new :c:type:`PyLongObject` object from a C :c:type:`size_t`, or + *NULL* on failure. + + +.. c:function:: PyObject* PyLong_FromLongLong(PY_LONG_LONG v) + + Return a new :c:type:`PyLongObject` object from a C :c:type:`long long`, or *NULL* + on failure. + + +.. c:function:: PyObject* PyLong_FromUnsignedLongLong(unsigned PY_LONG_LONG v) + + Return a new :c:type:`PyLongObject` object from a C :c:type:`unsigned long long`, + or *NULL* on failure. + + +.. c:function:: PyObject* PyLong_FromDouble(double v) + + Return a new :c:type:`PyLongObject` object from the integer part of *v*, or + *NULL* on failure. + + +.. c:function:: PyObject* PyLong_FromString(char *str, char **pend, int base) + + Return a new :c:type:`PyLongObject` based on the string value in *str*, which + is interpreted according to the radix in *base*. If *pend* is non-*NULL*, + *\*pend* will point to the first character in *str* which follows the + representation of the number. If *base* is ``0``, the radix will be + determined based on the leading characters of *str*: if *str* starts with + ``'0x'`` or ``'0X'``, radix 16 will be used; if *str* starts with ``'0o'`` or + ``'0O'``, radix 8 will be used; if *str* starts with ``'0b'`` or ``'0B'``, + radix 2 will be used; otherwise radix 10 will be used. If *base* is not + ``0``, it must be between ``2`` and ``36``, inclusive. Leading spaces are + ignored. If there are no digits, :exc:`ValueError` will be raised. + + +.. c:function:: PyObject* PyLong_FromUnicode(Py_UNICODE *u, Py_ssize_t length, int base) + + Convert a sequence of Unicode digits to a Python integer value. The Unicode + string is first encoded to a byte string using :c:func:`PyUnicode_EncodeDecimal` + and then converted using :c:func:`PyLong_FromString`. + + .. deprecated-removed:: 3.3 4.0 + Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using + :c:func:`PyLong_FromUnicodeObject`. + + +.. c:function:: PyObject* PyLong_FromUnicodeObject(PyObject *u, int base) + + Convert a sequence of Unicode digits in the string *u* to a Python integer + value. The Unicode string is first encoded to a byte string using + :c:func:`PyUnicode_EncodeDecimal` and then converted using + :c:func:`PyLong_FromString`. + + .. versionadded:: 3.3 + + +.. c:function:: PyObject* PyLong_FromVoidPtr(void *p) + + Create a Python integer from the pointer *p*. The pointer value can be + retrieved from the resulting value using :c:func:`PyLong_AsVoidPtr`. + + +.. XXX alias PyLong_AS_LONG (for now) +.. c:function:: long PyLong_AsLong(PyObject *pylong) + + .. index:: + single: LONG_MAX + single: OverflowError (built-in exception) + + Return a C :c:type:`long` representation of the contents of *pylong*. If + *pylong* is greater than :const:`LONG_MAX`, raise an :exc:`OverflowError`, + and return -1. Convert non-long objects automatically to long first, + and return -1 if that raises exceptions. + +.. c:function:: long PyLong_AsLongAndOverflow(PyObject *pylong, int *overflow) + + Return a C :c:type:`long` representation of the contents of + *pylong*. If *pylong* is greater than :const:`LONG_MAX` or less + than :const:`LONG_MIN`, set *\*overflow* to ``1`` or ``-1``, + respectively, and return ``-1``; otherwise, set *\*overflow* to + ``0``. If any other exception occurs (for example a TypeError or + MemoryError), then ``-1`` will be returned and *\*overflow* will + be ``0``. + + +.. c:function:: PY_LONG_LONG PyLong_AsLongLongAndOverflow(PyObject *pylong, int *overflow) + + Return a C :c:type:`long long` representation of the contents of + *pylong*. If *pylong* is greater than :const:`PY_LLONG_MAX` or less + than :const:`PY_LLONG_MIN`, set *\*overflow* to ``1`` or ``-1``, + respectively, and return ``-1``; otherwise, set *\*overflow* to + ``0``. If any other exception occurs (for example a TypeError or + MemoryError), then ``-1`` will be returned and *\*overflow* will + be ``0``. + + .. versionadded:: 3.2 + + +.. c:function:: Py_ssize_t PyLong_AsSsize_t(PyObject *pylong) + + .. index:: + single: PY_SSIZE_T_MAX + single: OverflowError (built-in exception) + + Return a C :c:type:`Py_ssize_t` representation of the contents of *pylong*. + If *pylong* is greater than :const:`PY_SSIZE_T_MAX`, an :exc:`OverflowError` + is raised and ``-1`` will be returned. + + +.. c:function:: unsigned long PyLong_AsUnsignedLong(PyObject *pylong) + + .. index:: + single: ULONG_MAX + single: OverflowError (built-in exception) + + Return a C :c:type:`unsigned long` representation of the contents of *pylong*. + If *pylong* is greater than :const:`ULONG_MAX`, an :exc:`OverflowError` is + raised. + + +.. c:function:: size_t PyLong_AsSize_t(PyObject *pylong) + + Return a :c:type:`size_t` representation of the contents of *pylong*. If + *pylong* is greater than the maximum value for a :c:type:`size_t`, an + :exc:`OverflowError` is raised. + + +.. c:function:: PY_LONG_LONG PyLong_AsLongLong(PyObject *pylong) + + .. index:: + single: OverflowError (built-in exception) + + Return a C :c:type:`long long` from a Python integer. If *pylong* + cannot be represented as a :c:type:`long long`, an + :exc:`OverflowError` is raised and ``-1`` is returned. + + +.. c:function:: unsigned PY_LONG_LONG PyLong_AsUnsignedLongLong(PyObject *pylong) + + .. index:: + single: OverflowError (built-in exception) + + Return a C :c:type:`unsigned long long` from a Python integer. If + *pylong* cannot be represented as an :c:type:`unsigned long long`, + an :exc:`OverflowError` is raised and ``(unsigned long long)-1`` is + returned. + + .. versionchanged:: 3.1 + A negative *pylong* now raises :exc:`OverflowError`, not :exc:`TypeError`. + + +.. c:function:: unsigned long PyLong_AsUnsignedLongMask(PyObject *io) + + Return a C :c:type:`unsigned long` from a Python integer, without checking for + overflow. + + +.. c:function:: unsigned PY_LONG_LONG PyLong_AsUnsignedLongLongMask(PyObject *io) + + Return a C :c:type:`unsigned long long` from a Python integer, without + checking for overflow. + + +.. c:function:: double PyLong_AsDouble(PyObject *pylong) + + Return a C :c:type:`double` representation of the contents of *pylong*. If + *pylong* cannot be approximately represented as a :c:type:`double`, an + :exc:`OverflowError` exception is raised and ``-1.0`` will be returned. + + +.. c:function:: void* PyLong_AsVoidPtr(PyObject *pylong) + + Convert a Python integer *pylong* to a C :c:type:`void` pointer. + If *pylong* cannot be converted, an :exc:`OverflowError` will be raised. This + is only assured to produce a usable :c:type:`void` pointer for values created + with :c:func:`PyLong_FromVoidPtr`. diff --git a/lib/cpython-doc/c-api/mapping.rst b/lib/cpython-doc/c-api/mapping.rst new file mode 100644 --- /dev/null +++ b/lib/cpython-doc/c-api/mapping.rst @@ -0,0 +1,79 @@ +.. highlightlang:: c + +.. _mapping: + +Mapping Protocol +================ + + +.. c:function:: int PyMapping_Check(PyObject *o) + + Return ``1`` if the object provides mapping protocol, and ``0`` otherwise. This + function always succeeds. + + +.. c:function:: Py_ssize_t PyMapping_Size(PyObject *o) + Py_ssize_t PyMapping_Length(PyObject *o) + + .. index:: builtin: len + + Returns the number of keys in object *o* on success, and ``-1`` on failure. For + objects that do not provide mapping protocol, this is equivalent to the Python + expression ``len(o)``. + + +.. c:function:: int PyMapping_DelItemString(PyObject *o, char *key) + + Remove the mapping for object *key* from the object *o*. Return ``-1`` on + failure. This is equivalent to the Python statement ``del o[key]``. + + +.. c:function:: int PyMapping_DelItem(PyObject *o, PyObject *key) + + Remove the mapping for object *key* from the object *o*. Return ``-1`` on + failure. This is equivalent to the Python statement ``del o[key]``. + + +.. c:function:: int PyMapping_HasKeyString(PyObject *o, char *key) + + On success, return ``1`` if the mapping object has the key *key* and ``0`` + otherwise. This is equivalent to the Python expression ``key in o``. + This function always succeeds. + + +.. c:function:: int PyMapping_HasKey(PyObject *o, PyObject *key) + + Return ``1`` if the mapping object has the key *key* and ``0`` otherwise. This + is equivalent to the Python expression ``key in o``. This function always + succeeds. + + +.. c:function:: PyObject* PyMapping_Keys(PyObject *o) + + On success, return a list of the keys in object *o*. On failure, return *NULL*. + This is equivalent to the Python expression ``list(o.keys())``. + + +.. c:function:: PyObject* PyMapping_Values(PyObject *o) + + On success, return a list of the values in object *o*. On failure, return + *NULL*. This is equivalent to the Python expression ``list(o.values())``. + + +.. c:function:: PyObject* PyMapping_Items(PyObject *o) + + On success, return a list of the items in object *o*, where each item is a tuple + containing a key-value pair. On failure, return *NULL*. This is equivalent to + the Python expression ``list(o.items())``. + + +.. c:function:: PyObject* PyMapping_GetItemString(PyObject *o, char *key) + + Return element of *o* corresponding to the object *key* or *NULL* on failure. + This is the equivalent of the Python expression ``o[key]``. + + +.. c:function:: int PyMapping_SetItemString(PyObject *o, char *key, PyObject *v) + + Map the object *key* to the value *v* in object *o*. Returns ``-1`` on failure. + This is the equivalent of the Python statement ``o[key] = v``. diff --git a/lib/cpython-doc/c-api/marshal.rst b/lib/cpython-doc/c-api/marshal.rst new file mode 100644 --- /dev/null +++ b/lib/cpython-doc/c-api/marshal.rst @@ -0,0 +1,89 @@ +.. highlightlang:: c + +.. _marshalling-utils: + +Data marshalling support +======================== + +These routines allow C code to work with serialized objects using the same +data format as the :mod:`marshal` module. There are functions to write data +into the serialization format, and additional functions that can be used to +read the data back. Files used to store marshalled data must be opened in +binary mode. + +Numeric values are stored with the least significant byte first. + +The module supports two versions of the data format: version 0 is the +historical version, version 1 shares interned strings in the file, and upon +unmarshalling. Version 2 uses a binary format for floating point numbers. +*Py_MARSHAL_VERSION* indicates the current file format (currently 2). + + +.. c:function:: void PyMarshal_WriteLongToFile(long value, FILE *file, int version) + + Marshal a :c:type:`long` integer, *value*, to *file*. This will only write + the least-significant 32 bits of *value*; regardless of the size of the + native :c:type:`long` type. *version* indicates the file format. + + +.. c:function:: void PyMarshal_WriteObjectToFile(PyObject *value, FILE *file, int version) + + Marshal a Python object, *value*, to *file*. + *version* indicates the file format. + + +.. c:function:: PyObject* PyMarshal_WriteObjectToString(PyObject *value, int version) + + Return a string object containing the marshalled representation of *value*. + *version* indicates the file format. + + +The following functions allow marshalled values to be read back in. + +XXX What about error detection? It appears that reading past the end of the +file will always result in a negative numeric value (where that's relevant), +but it's not clear that negative values won't be handled properly when there's +no error. What's the right way to tell? Should only non-negative values be +written using these routines? + + +.. c:function:: long PyMarshal_ReadLongFromFile(FILE *file) + + Return a C :c:type:`long` from the data stream in a :c:type:`FILE\*` opened + for reading. Only a 32-bit value can be read in using this function, + regardless of the native size of :c:type:`long`. + + +.. c:function:: int PyMarshal_ReadShortFromFile(FILE *file) + + Return a C :c:type:`short` from the data stream in a :c:type:`FILE\*` opened + for reading. Only a 16-bit value can be read in using this function, + regardless of the native size of :c:type:`short`. + + +.. c:function:: PyObject* PyMarshal_ReadObjectFromFile(FILE *file) + + Return a Python object from the data stream in a :c:type:`FILE\*` opened for + reading. On error, sets the appropriate exception (:exc:`EOFError` or + :exc:`TypeError`) and returns *NULL*. + + +.. c:function:: PyObject* PyMarshal_ReadLastObjectFromFile(FILE *file) + + Return a Python object from the data stream in a :c:type:`FILE\*` opened for + reading. Unlike :c:func:`PyMarshal_ReadObjectFromFile`, this function + assumes that no further objects will be read from the file, allowing it to + aggressively load file data into memory so that the de-serialization can + operate from data in memory rather than reading a byte at a time from the + file. Only use these variant if you are certain that you won't be reading + anything else from the file. On error, sets the appropriate exception + (:exc:`EOFError` or :exc:`TypeError`) and returns *NULL*. + + +.. c:function:: PyObject* PyMarshal_ReadObjectFromString(char *string, Py_ssize_t len) + + Return a Python object from the data stream in a character buffer + containing *len* bytes pointed to by *string*. On error, sets the + appropriate exception (:exc:`EOFError` or :exc:`TypeError`) and returns + *NULL*. + diff --git a/lib/cpython-doc/c-api/memory.rst b/lib/cpython-doc/c-api/memory.rst new file mode 100644 --- /dev/null +++ b/lib/cpython-doc/c-api/memory.rst @@ -0,0 +1,209 @@ +.. highlightlang:: c + + +.. _memory: + +***************** +Memory Management +***************** + +.. sectionauthor:: Vladimir Marangozov + + + +.. _memoryoverview: + +Overview +======== + +Memory management in Python involves a private heap containing all Python +objects and data structures. The management of this private heap is ensured +internally by the *Python memory manager*. The Python memory manager has +different components which deal with various dynamic storage management aspects, +like sharing, segmentation, preallocation or caching. + +At the lowest level, a raw memory allocator ensures that there is enough room in +the private heap for storing all Python-related data by interacting with the +memory manager of the operating system. On top of the raw memory allocator, +several object-specific allocators operate on the same heap and implement +distinct memory management policies adapted to the peculiarities of every object +type. For example, integer objects are managed differently within the heap than +strings, tuples or dictionaries because integers imply different storage +requirements and speed/space tradeoffs. The Python memory manager thus delegates +some of the work to the object-specific allocators, but ensures that the latter +operate within the bounds of the private heap. + +It is important to understand that the management of the Python heap is +performed by the interpreter itself and that the user has no control over it, +even if she regularly manipulates object pointers to memory blocks inside that +heap. The allocation of heap space for Python objects and other internal +buffers is performed on demand by the Python memory manager through the Python/C +API functions listed in this document. + +.. index:: + single: malloc() + single: calloc() + single: realloc() + single: free() + +To avoid memory corruption, extension writers should never try to operate on +Python objects with the functions exported by the C library: :c:func:`malloc`, +:c:func:`calloc`, :c:func:`realloc` and :c:func:`free`. This will result in mixed +calls between the C allocator and the Python memory manager with fatal +consequences, because they implement different algorithms and operate on +different heaps. However, one may safely allocate and release memory blocks +with the C library allocator for individual purposes, as shown in the following +example:: + + PyObject *res; + char *buf = (char *) malloc(BUFSIZ); /* for I/O */ + + if (buf == NULL) + return PyErr_NoMemory(); + ...Do some I/O operation involving buf... + res = PyString_FromString(buf); + free(buf); /* malloc'ed */ + return res; + +In this example, the memory request for the I/O buffer is handled by the C +library allocator. The Python memory manager is involved only in the allocation +of the string object returned as a result. + +In most situations, however, it is recommended to allocate memory from the +Python heap specifically because the latter is under control of the Python +memory manager. For example, this is required when the interpreter is extended +with new object types written in C. Another reason for using the Python heap is +the desire to *inform* the Python memory manager about the memory needs of the +extension module. Even when the requested memory is used exclusively for +internal, highly-specific purposes, delegating all memory requests to the Python +memory manager causes the interpreter to have a more accurate image of its +memory footprint as a whole. Consequently, under certain circumstances, the +Python memory manager may or may not trigger appropriate actions, like garbage +collection, memory compaction or other preventive procedures. Note that by using +the C library allocator as shown in the previous example, the allocated memory +for the I/O buffer escapes completely the Python memory manager. + + +.. _memoryinterface: + +Memory Interface +================ + +The following function sets, modeled after the ANSI C standard, but specifying +behavior when requesting zero bytes, are available for allocating and releasing +memory from the Python heap: + + +.. c:function:: void* PyMem_Malloc(size_t n) + + Allocates *n* bytes and returns a pointer of type :c:type:`void\*` to the + allocated memory, or *NULL* if the request fails. Requesting zero bytes returns + a distinct non-*NULL* pointer if possible, as if :c:func:`PyMem_Malloc(1)` had + been called instead. The memory will not have been initialized in any way. + + +.. c:function:: void* PyMem_Realloc(void *p, size_t n) + + Resizes the memory block pointed to by *p* to *n* bytes. The contents will be + unchanged to the minimum of the old and the new sizes. If *p* is *NULL*, the + call is equivalent to :c:func:`PyMem_Malloc(n)`; else if *n* is equal to zero, + the memory block is resized but is not freed, and the returned pointer is + non-*NULL*. Unless *p* is *NULL*, it must have been returned by a previous call + to :c:func:`PyMem_Malloc` or :c:func:`PyMem_Realloc`. If the request fails, + :c:func:`PyMem_Realloc` returns *NULL* and *p* remains a valid pointer to the + previous memory area. + + +.. c:function:: void PyMem_Free(void *p) + + Frees the memory block pointed to by *p*, which must have been returned by a + previous call to :c:func:`PyMem_Malloc` or :c:func:`PyMem_Realloc`. Otherwise, or + if :c:func:`PyMem_Free(p)` has been called before, undefined behavior occurs. If + *p* is *NULL*, no operation is performed. + +The following type-oriented macros are provided for convenience. Note that +*TYPE* refers to any C type. + + +.. c:function:: TYPE* PyMem_New(TYPE, size_t n) + + Same as :c:func:`PyMem_Malloc`, but allocates ``(n * sizeof(TYPE))`` bytes of + memory. Returns a pointer cast to :c:type:`TYPE\*`. The memory will not have + been initialized in any way. + + +.. c:function:: TYPE* PyMem_Resize(void *p, TYPE, size_t n) + + Same as :c:func:`PyMem_Realloc`, but the memory block is resized to ``(n * + sizeof(TYPE))`` bytes. Returns a pointer cast to :c:type:`TYPE\*`. On return, + *p* will be a pointer to the new memory area, or *NULL* in the event of + failure. This is a C preprocessor macro; p is always reassigned. Save + the original value of p to avoid losing memory when handling errors. + + +.. c:function:: void PyMem_Del(void *p) + + Same as :c:func:`PyMem_Free`. + +In addition, the following macro sets are provided for calling the Python memory +allocator directly, without involving the C API functions listed above. However, +note that their use does not preserve binary compatibility across Python +versions and is therefore deprecated in extension modules. + +:c:func:`PyMem_MALLOC`, :c:func:`PyMem_REALLOC`, :c:func:`PyMem_FREE`. + +:c:func:`PyMem_NEW`, :c:func:`PyMem_RESIZE`, :c:func:`PyMem_DEL`. + + +.. _memoryexamples: + +Examples +======== + +Here is the example from section :ref:`memoryoverview`, rewritten so that the +I/O buffer is allocated from the Python heap by using the first function set:: + + PyObject *res; + char *buf = (char *) PyMem_Malloc(BUFSIZ); /* for I/O */ + + if (buf == NULL) + return PyErr_NoMemory(); + /* ...Do some I/O operation involving buf... */ + res = PyString_FromString(buf); + PyMem_Free(buf); /* allocated with PyMem_Malloc */ + return res; + +The same code using the type-oriented function set:: + + PyObject *res; + char *buf = PyMem_New(char, BUFSIZ); /* for I/O */ + + if (buf == NULL) + return PyErr_NoMemory(); + /* ...Do some I/O operation involving buf... */ + res = PyString_FromString(buf); + PyMem_Del(buf); /* allocated with PyMem_New */ + return res; + +Note that in the two examples above, the buffer is always manipulated via +functions belonging to the same set. Indeed, it is required to use the same +memory API family for a given memory block, so that the risk of mixing different +allocators is reduced to a minimum. The following code sequence contains two +errors, one of which is labeled as *fatal* because it mixes two different +allocators operating on different heaps. :: + + char *buf1 = PyMem_New(char, BUFSIZ); + char *buf2 = (char *) malloc(BUFSIZ); + char *buf3 = (char *) PyMem_Malloc(BUFSIZ); + ... + PyMem_Del(buf3); /* Wrong -- should be PyMem_Free() */ + free(buf2); /* Right -- allocated via malloc() */ + free(buf1); /* Fatal -- should be PyMem_Del() */ + +In addition to the functions aimed at handling raw memory blocks from the Python +heap, objects in Python are allocated and released with :c:func:`PyObject_New`, +:c:func:`PyObject_NewVar` and :c:func:`PyObject_Del`. + +These will be explained in the next chapter on defining and implementing new +object types in C. + diff --git a/lib/cpython-doc/c-api/memoryview.rst b/lib/cpython-doc/c-api/memoryview.rst new file mode 100644 --- /dev/null +++ b/lib/cpython-doc/c-api/memoryview.rst @@ -0,0 +1,52 @@ +.. highlightlang:: c + +.. _memoryview-objects: + +.. index:: + object: memoryview + +MemoryView objects +------------------ + +A :class:`memoryview` object exposes the C level :ref:`buffer interface +` as a Python object which can then be passed around like +any other object. + + +.. c:function:: PyObject *PyMemoryView_FromObject(PyObject *obj) + + Create a memoryview object from an object that provides the buffer interface. + If *obj* supports writable buffer exports, the memoryview object will be + readable and writable, otherwise it will be read-only. + + +.. c:function:: PyObject *PyMemoryView_FromBuffer(Py_buffer *view) + + Create a memoryview object wrapping the given buffer structure *view*. + The memoryview object then owns the buffer represented by *view*, which + means you shouldn't try to call :c:func:`PyBuffer_Release` yourself: it + will be done on deallocation of the memoryview object. + + +.. c:function:: PyObject *PyMemoryView_GetContiguous(PyObject *obj, int buffertype, char order) + + Create a memoryview object to a contiguous chunk of memory (in either + 'C' or 'F'ortran *order*) from an object that defines the buffer + interface. If memory is contiguous, the memoryview object points to the + original memory. Otherwise, a copy is made and the memoryview points to a + new bytes object. + + +.. c:function:: int PyMemoryView_Check(PyObject *obj) + + Return true if the object *obj* is a memoryview object. It is not + currently allowed to create subclasses of :class:`memoryview`. + + +.. c:function:: Py_buffer *PyMemoryView_GET_BUFFER(PyObject *obj) + + Return a pointer to the buffer structure wrapped by the given + memoryview object. The object **must** be a memoryview instance; + this macro doesn't check its type, you must do it yourself or you + will risk crashes. + diff --git a/lib/cpython-doc/c-api/method.rst b/lib/cpython-doc/c-api/method.rst new file mode 100644 --- /dev/null +++ b/lib/cpython-doc/c-api/method.rst @@ -0,0 +1,100 @@ +.. highlightlang:: c + +.. _instancemethod-objects: + +Instance Method Objects +----------------------- + +.. index:: object: instancemethod + +An instance method is a wrapper for a :c:data:`PyCFunction` and the new way +to bind a :c:data:`PyCFunction` to a class object. It replaces the former call +``PyMethod_New(func, NULL, class)``. + + +.. c:var:: PyTypeObject PyInstanceMethod_Type + + This instance of :c:type:`PyTypeObject` represents the Python instance + method type. It is not exposed to Python programs. + + +.. c:function:: int PyInstanceMethod_Check(PyObject *o) + + Return true if *o* is an instance method object (has type + :c:data:`PyInstanceMethod_Type`). The parameter must not be *NULL*. + + +.. c:function:: PyObject* PyInstanceMethod_New(PyObject *func) + + Return a new instance method object, with *func* being any callable object + *func* is the function that will be called when the instance method is + called. + + +.. c:function:: PyObject* PyInstanceMethod_Function(PyObject *im) + + Return the function object associated with the instance method *im*. + + +.. c:function:: PyObject* PyInstanceMethod_GET_FUNCTION(PyObject *im) + + Macro version of :c:func:`PyInstanceMethod_Function` which avoids error checking. + + +.. _method-objects: + +Method Objects +-------------- + +.. index:: object: method + +Methods are bound function objects. Methods are always bound to an instance of +an user-defined class. Unbound methods (methods bound to a class object) are +no longer available. + + +.. c:var:: PyTypeObject PyMethod_Type + + .. index:: single: MethodType (in module types) + + This instance of :c:type:`PyTypeObject` represents the Python method type. This + is exposed to Python programs as ``types.MethodType``. + + +.. c:function:: int PyMethod_Check(PyObject *o) + + Return true if *o* is a method object (has type :c:data:`PyMethod_Type`). The + parameter must not be *NULL*. + + +.. c:function:: PyObject* PyMethod_New(PyObject *func, PyObject *self) + + Return a new method object, with *func* being any callable object and *self* + the instance the method should be bound. *func* is the function that will + be called when the method is called. *self* must not be *NULL*. + + +.. c:function:: PyObject* PyMethod_Function(PyObject *meth) + + Return the function object associated with the method *meth*. + + +.. c:function:: PyObject* PyMethod_GET_FUNCTION(PyObject *meth) + + Macro version of :c:func:`PyMethod_Function` which avoids error checking. + + +.. c:function:: PyObject* PyMethod_Self(PyObject *meth) + + Return the instance associated with the method *meth*. + + +.. c:function:: PyObject* PyMethod_GET_SELF(PyObject *meth) + + Macro version of :c:func:`PyMethod_Self` which avoids error checking. + + +.. c:function:: int PyMethod_ClearFreeList() + + Clear the free list. Return the total number of freed items. + diff --git a/lib/cpython-doc/c-api/module.rst b/lib/cpython-doc/c-api/module.rst new file mode 100644 --- /dev/null +++ b/lib/cpython-doc/c-api/module.rst @@ -0,0 +1,231 @@ +.. highlightlang:: c + +.. _moduleobjects: + +Module Objects +-------------- + +.. index:: object: module + +There are only a few functions special to module objects. + + +.. c:var:: PyTypeObject PyModule_Type + + .. index:: single: ModuleType (in module types) + + This instance of :c:type:`PyTypeObject` represents the Python module type. This + is exposed to Python programs as ``types.ModuleType``. + + +.. c:function:: int PyModule_Check(PyObject *p) + + Return true if *p* is a module object, or a subtype of a module object. + + +.. c:function:: int PyModule_CheckExact(PyObject *p) + + Return true if *p* is a module object, but not a subtype of + :c:data:`PyModule_Type`. + + +.. c:function:: PyObject* PyModule_NewObject(PyObject *name) + + .. index:: + single: __name__ (module attribute) + single: __doc__ (module attribute) + single: __file__ (module attribute) + + Return a new module object with the :attr:`__name__` attribute set to *name*. + Only the module's :attr:`__doc__` and :attr:`__name__` attributes are filled in; + the caller is responsible for providing a :attr:`__file__` attribute. + + .. versionadded:: 3.3 + + +.. c:function:: PyObject* PyModule_New(const char *name) + + Similar to :c:func:`PyImport_NewObject`, but the name is an UTF-8 encoded + string instead of a Unicode object. + + +.. c:function:: PyObject* PyModule_GetDict(PyObject *module) + + .. index:: single: __dict__ (module attribute) + + Return the dictionary object that implements *module*'s namespace; this object + is the same as the :attr:`__dict__` attribute of the module object. This + function never fails. It is recommended extensions use other + :c:func:`PyModule_\*` and :c:func:`PyObject_\*` functions rather than directly + manipulate a module's :attr:`__dict__`. + + +.. c:function:: PyObject* PyModule_GetNameObject(PyObject *module) + + .. index:: + single: __name__ (module attribute) + single: SystemError (built-in exception) + + Return *module*'s :attr:`__name__` value. If the module does not provide one, + or if it is not a string, :exc:`SystemError` is raised and *NULL* is returned. + + .. versionadded:: 3.3 + + +.. c:function:: char* PyModule_GetName(PyObject *module) + + Similar to :c:func:`PyModule_GetNameObject` but return the name encoded to + ``'utf-8'``. + + +.. c:function:: PyObject* PyModule_GetFilenameObject(PyObject *module) + + .. index:: + single: __file__ (module attribute) + single: SystemError (built-in exception) + + Return the name of the file from which *module* was loaded using *module*'s + :attr:`__file__` attribute. If this is not defined, or if it is not a + unicode string, raise :exc:`SystemError` and return *NULL*; otherwise return + a reference to a Unicode object. + + .. versionadded:: 3.2 + + +.. c:function:: char* PyModule_GetFilename(PyObject *module) + + Similar to :c:func:`PyModule_GetFilenameObject` but return the filename + encoded to 'utf-8'. + + .. deprecated:: 3.2 + :c:func:`PyModule_GetFilename` raises :c:type:`UnicodeEncodeError` on + unencodable filenames, use :c:func:`PyModule_GetFilenameObject` instead. + + +.. c:function:: void* PyModule_GetState(PyObject *module) + + Return the "state" of the module, that is, a pointer to the block of memory + allocated at module creation time, or *NULL*. See + :c:member:`PyModuleDef.m_size`. + + +.. c:function:: PyModuleDef* PyModule_GetDef(PyObject *module) + + Return a pointer to the :c:type:`PyModuleDef` struct from which the module was + created, or *NULL* if the module wasn't created with + :c:func:`PyModule_Create`. + + +Initializing C modules +^^^^^^^^^^^^^^^^^^^^^^ + +These functions are usually used in the module initialization function. + +.. c:function:: PyObject* PyModule_Create(PyModuleDef *module) + + Create a new module object, given the definition in *module*. This behaves + like :c:func:`PyModule_Create2` with *module_api_version* set to + :const:`PYTHON_API_VERSION`. + + +.. c:function:: PyObject* PyModule_Create2(PyModuleDef *module, int module_api_version) + + Create a new module object, given the definition in *module*, assuming the + API version *module_api_version*. If that version does not match the version + of the running interpreter, a :exc:`RuntimeWarning` is emitted. + + .. note:: + + Most uses of this function should be using :c:func:`PyModule_Create` + instead; only use this if you are sure you need it. + + +.. c:type:: PyModuleDef + + This struct holds all information that is needed to create a module object. + There is usually only one static variable of that type for each module, which + is statically initialized and then passed to :c:func:`PyModule_Create` in the + module initialization function. + + .. c:member:: PyModuleDef_Base m_base + + Always initialize this member to :const:`PyModuleDef_HEAD_INIT`. + + .. c:member:: char* m_name + + Name for the new module. + + .. c:member:: char* m_doc + + Docstring for the module; usually a docstring variable created with + :c:func:`PyDoc_STRVAR` is used. + + .. c:member:: Py_ssize_t m_size + + If the module object needs additional memory, this should be set to the + number of bytes to allocate; a pointer to the block of memory can be + retrieved with :c:func:`PyModule_GetState`. If no memory is needed, set + this to ``-1``. + + This memory should be used, rather than static globals, to hold per-module + state, since it is then safe for use in multiple sub-interpreters. It is + freed when the module object is deallocated, after the :c:member:`m_free` + function has been called, if present. + + .. c:member:: PyMethodDef* m_methods + + A pointer to a table of module-level functions, described by + :c:type:`PyMethodDef` values. Can be *NULL* if no functions are present. + + .. c:member:: inquiry m_reload + + Currently unused, should be *NULL*. + + .. c:member:: traverseproc m_traverse + + A traversal function to call during GC traversal of the module object, or + *NULL* if not needed. + + .. c:member:: inquiry m_clear + + A clear function to call during GC clearing of the module object, or + *NULL* if not needed. + + .. c:member:: freefunc m_free + + A function to call during deallocation of the module object, or *NULL* if + not needed. + + +.. c:function:: int PyModule_AddObject(PyObject *module, const char *name, PyObject *value) + + Add an object to *module* as *name*. This is a convenience function which can + be used from the module's initialization function. This steals a reference to + *value*. Return ``-1`` on error, ``0`` on success. + + +.. c:function:: int PyModule_AddIntConstant(PyObject *module, const char *name, long value) + + Add an integer constant to *module* as *name*. This convenience function can be + used from the module's initialization function. Return ``-1`` on error, ``0`` on + success. + + +.. c:function:: int PyModule_AddStringConstant(PyObject *module, const char *name, const char *value) + + Add a string constant to *module* as *name*. This convenience function can be + used from the module's initialization function. The string *value* must be + null-terminated. Return ``-1`` on error, ``0`` on success. + + +.. c:function:: int PyModule_AddIntMacro(PyObject *module, macro) + + Add an int constant to *module*. The name and the value are taken from + *macro*. For example ``PyModule_AddIntMacro(module, AF_INET)`` adds the int + constant *AF_INET* with the value of *AF_INET* to *module*. + Return ``-1`` on error, ``0`` on success. + + +.. c:function:: int PyModule_AddStringMacro(PyObject *module, macro) + + Add a string constant to *module*. diff --git a/lib/cpython-doc/c-api/none.rst b/lib/cpython-doc/c-api/none.rst new file mode 100644 --- /dev/null +++ b/lib/cpython-doc/c-api/none.rst @@ -0,0 +1,26 @@ +.. highlightlang:: c + +.. _noneobject: + +The None Object +--------------- + +.. index:: object: None + +Note that the :c:type:`PyTypeObject` for ``None`` is not directly exposed in the +Python/C API. Since ``None`` is a singleton, testing for object identity (using +``==`` in C) is sufficient. There is no :c:func:`PyNone_Check` function for the +same reason. + + +.. c:var:: PyObject* Py_None + + The Python ``None`` object, denoting lack of value. This object has no methods. + It needs to be treated just like any other object with respect to reference + counts. + + +.. c:macro:: Py_RETURN_NONE + + Properly handle returning :c:data:`Py_None` from within a C function (that is, + increment the reference count of None and return it.) diff --git a/lib/cpython-doc/c-api/number.rst b/lib/cpython-doc/c-api/number.rst new file mode 100644 --- /dev/null +++ b/lib/cpython-doc/c-api/number.rst @@ -0,0 +1,265 @@ +.. highlightlang:: c + +.. _number: + +Number Protocol +=============== + + +.. c:function:: int PyNumber_Check(PyObject *o) + + Returns ``1`` if the object *o* provides numeric protocols, and false otherwise. + This function always succeeds. + + +.. c:function:: PyObject* PyNumber_Add(PyObject *o1, PyObject *o2) + + Returns the result of adding *o1* and *o2*, or *NULL* on failure. This is the + equivalent of the Python expression ``o1 + o2``. + + +.. c:function:: PyObject* PyNumber_Subtract(PyObject *o1, PyObject *o2) + + Returns the result of subtracting *o2* from *o1*, or *NULL* on failure. This is + the equivalent of the Python expression ``o1 - o2``. + + +.. c:function:: PyObject* PyNumber_Multiply(PyObject *o1, PyObject *o2) + + Returns the result of multiplying *o1* and *o2*, or *NULL* on failure. This is + the equivalent of the Python expression ``o1 * o2``. + + +.. c:function:: PyObject* PyNumber_FloorDivide(PyObject *o1, PyObject *o2) + + Return the floor of *o1* divided by *o2*, or *NULL* on failure. This is + equivalent to the "classic" division of integers. + + +.. c:function:: PyObject* PyNumber_TrueDivide(PyObject *o1, PyObject *o2) + + Return a reasonable approximation for the mathematical value of *o1* divided by + *o2*, or *NULL* on failure. The return value is "approximate" because binary + floating point numbers are approximate; it is not possible to represent all real + numbers in base two. This function can return a floating point value when + passed two integers. + + +.. c:function:: PyObject* PyNumber_Remainder(PyObject *o1, PyObject *o2) + + Returns the remainder of dividing *o1* by *o2*, or *NULL* on failure. This is + the equivalent of the Python expression ``o1 % o2``. + + +.. c:function:: PyObject* PyNumber_Divmod(PyObject *o1, PyObject *o2) + + .. index:: builtin: divmod + + See the built-in function :func:`divmod`. Returns *NULL* on failure. This is + the equivalent of the Python expression ``divmod(o1, o2)``. + + +.. c:function:: PyObject* PyNumber_Power(PyObject *o1, PyObject *o2, PyObject *o3) + + .. index:: builtin: pow + + See the built-in function :func:`pow`. Returns *NULL* on failure. This is the + equivalent of the Python expression ``pow(o1, o2, o3)``, where *o3* is optional. + If *o3* is to be ignored, pass :c:data:`Py_None` in its place (passing *NULL* for + *o3* would cause an illegal memory access). + + +.. c:function:: PyObject* PyNumber_Negative(PyObject *o) + + Returns the negation of *o* on success, or *NULL* on failure. This is the + equivalent of the Python expression ``-o``. + + +.. c:function:: PyObject* PyNumber_Positive(PyObject *o) + + Returns *o* on success, or *NULL* on failure. This is the equivalent of the + Python expression ``+o``. + + +.. c:function:: PyObject* PyNumber_Absolute(PyObject *o) + + .. index:: builtin: abs + + Returns the absolute value of *o*, or *NULL* on failure. This is the equivalent + of the Python expression ``abs(o)``. + + +.. c:function:: PyObject* PyNumber_Invert(PyObject *o) + + Returns the bitwise negation of *o* on success, or *NULL* on failure. This is + the equivalent of the Python expression ``~o``. + + +.. c:function:: PyObject* PyNumber_Lshift(PyObject *o1, PyObject *o2) + + Returns the result of left shifting *o1* by *o2* on success, or *NULL* on + failure. This is the equivalent of the Python expression ``o1 << o2``. + + +.. c:function:: PyObject* PyNumber_Rshift(PyObject *o1, PyObject *o2) + + Returns the result of right shifting *o1* by *o2* on success, or *NULL* on + failure. This is the equivalent of the Python expression ``o1 >> o2``. + + +.. c:function:: PyObject* PyNumber_And(PyObject *o1, PyObject *o2) + + Returns the "bitwise and" of *o1* and *o2* on success and *NULL* on failure. + This is the equivalent of the Python expression ``o1 & o2``. + + +.. c:function:: PyObject* PyNumber_Xor(PyObject *o1, PyObject *o2) + + Returns the "bitwise exclusive or" of *o1* by *o2* on success, or *NULL* on + failure. This is the equivalent of the Python expression ``o1 ^ o2``. + + +.. c:function:: PyObject* PyNumber_Or(PyObject *o1, PyObject *o2) + + Returns the "bitwise or" of *o1* and *o2* on success, or *NULL* on failure. + This is the equivalent of the Python expression ``o1 | o2``. + + +.. c:function:: PyObject* PyNumber_InPlaceAdd(PyObject *o1, PyObject *o2) + + Returns the result of adding *o1* and *o2*, or *NULL* on failure. The operation + is done *in-place* when *o1* supports it. This is the equivalent of the Python + statement ``o1 += o2``. + + +.. c:function:: PyObject* PyNumber_InPlaceSubtract(PyObject *o1, PyObject *o2) + + Returns the result of subtracting *o2* from *o1*, or *NULL* on failure. The + operation is done *in-place* when *o1* supports it. This is the equivalent of + the Python statement ``o1 -= o2``. + + +.. c:function:: PyObject* PyNumber_InPlaceMultiply(PyObject *o1, PyObject *o2) + + Returns the result of multiplying *o1* and *o2*, or *NULL* on failure. The + operation is done *in-place* when *o1* supports it. This is the equivalent of + the Python statement ``o1 *= o2``. + + +.. c:function:: PyObject* PyNumber_InPlaceFloorDivide(PyObject *o1, PyObject *o2) + + Returns the mathematical floor of dividing *o1* by *o2*, or *NULL* on failure. + The operation is done *in-place* when *o1* supports it. This is the equivalent + of the Python statement ``o1 //= o2``. + + +.. c:function:: PyObject* PyNumber_InPlaceTrueDivide(PyObject *o1, PyObject *o2) + + Return a reasonable approximation for the mathematical value of *o1* divided by + *o2*, or *NULL* on failure. The return value is "approximate" because binary + floating point numbers are approximate; it is not possible to represent all real + numbers in base two. This function can return a floating point value when + passed two integers. The operation is done *in-place* when *o1* supports it. + + +.. c:function:: PyObject* PyNumber_InPlaceRemainder(PyObject *o1, PyObject *o2) + + Returns the remainder of dividing *o1* by *o2*, or *NULL* on failure. The + operation is done *in-place* when *o1* supports it. This is the equivalent of + the Python statement ``o1 %= o2``. + + +.. c:function:: PyObject* PyNumber_InPlacePower(PyObject *o1, PyObject *o2, PyObject *o3) + + .. index:: builtin: pow + + See the built-in function :func:`pow`. Returns *NULL* on failure. The operation + is done *in-place* when *o1* supports it. This is the equivalent of the Python + statement ``o1 **= o2`` when o3 is :c:data:`Py_None`, or an in-place variant of + ``pow(o1, o2, o3)`` otherwise. If *o3* is to be ignored, pass :c:data:`Py_None` + in its place (passing *NULL* for *o3* would cause an illegal memory access). + + +.. c:function:: PyObject* PyNumber_InPlaceLshift(PyObject *o1, PyObject *o2) + + Returns the result of left shifting *o1* by *o2* on success, or *NULL* on + failure. The operation is done *in-place* when *o1* supports it. This is the + equivalent of the Python statement ``o1 <<= o2``. + + +.. c:function:: PyObject* PyNumber_InPlaceRshift(PyObject *o1, PyObject *o2) + + Returns the result of right shifting *o1* by *o2* on success, or *NULL* on + failure. The operation is done *in-place* when *o1* supports it. This is the + equivalent of the Python statement ``o1 >>= o2``. + + +.. c:function:: PyObject* PyNumber_InPlaceAnd(PyObject *o1, PyObject *o2) + + Returns the "bitwise and" of *o1* and *o2* on success and *NULL* on failure. The + operation is done *in-place* when *o1* supports it. This is the equivalent of + the Python statement ``o1 &= o2``. + + +.. c:function:: PyObject* PyNumber_InPlaceXor(PyObject *o1, PyObject *o2) + + Returns the "bitwise exclusive or" of *o1* by *o2* on success, or *NULL* on + failure. The operation is done *in-place* when *o1* supports it. This is the + equivalent of the Python statement ``o1 ^= o2``. + + +.. c:function:: PyObject* PyNumber_InPlaceOr(PyObject *o1, PyObject *o2) + + Returns the "bitwise or" of *o1* and *o2* on success, or *NULL* on failure. The + operation is done *in-place* when *o1* supports it. This is the equivalent of + the Python statement ``o1 |= o2``. + + +.. c:function:: PyObject* PyNumber_Long(PyObject *o) + + .. index:: builtin: int + + Returns the *o* converted to an integer object on success, or *NULL* on + failure. This is the equivalent of the Python expression ``int(o)``. + + +.. c:function:: PyObject* PyNumber_Float(PyObject *o) + + .. index:: builtin: float + + Returns the *o* converted to a float object on success, or *NULL* on failure. + This is the equivalent of the Python expression ``float(o)``. + + +.. c:function:: PyObject* PyNumber_Index(PyObject *o) + + Returns the *o* converted to a Python int on success or *NULL* with a + :exc:`TypeError` exception raised on failure. + + +.. c:function:: PyObject* PyNumber_ToBase(PyObject *n, int base) + + Returns the integer *n* converted to base *base* as a string. The *base* + argument must be one of 2, 8, 10, or 16. For base 2, 8, or 16, the + returned string is prefixed with a base marker of ``'0b'``, ``'0o'``, or + ``'0x'``, respectively. If *n* is not a Python int, it is converted with + :c:func:`PyNumber_Index` first. + + +.. c:function:: Py_ssize_t PyNumber_AsSsize_t(PyObject *o, PyObject *exc) + + Returns *o* converted to a Py_ssize_t value if *o* can be interpreted as an + integer. If the call fails, an exception is raised and -1 is returned. + + If *o* can be converted to a Python int but the attempt to + convert to a Py_ssize_t value would raise an :exc:`OverflowError`, then the + *exc* argument is the type of exception that will be raised (usually + :exc:`IndexError` or :exc:`OverflowError`). If *exc* is *NULL*, then the + exception is cleared and the value is clipped to *PY_SSIZE_T_MIN* for a negative + integer or *PY_SSIZE_T_MAX* for a positive integer. + + +.. c:function:: int PyIndex_Check(PyObject *o) + + Returns True if *o* is an index integer (has the nb_index slot of the + tp_as_number structure filled in). diff --git a/lib/cpython-doc/c-api/objbuffer.rst b/lib/cpython-doc/c-api/objbuffer.rst new file mode 100644 --- /dev/null +++ b/lib/cpython-doc/c-api/objbuffer.rst @@ -0,0 +1,51 @@ +.. highlightlang:: c + +Old Buffer Protocol +------------------- + +.. deprecated:: 3.0 + +These functions were part of the "old buffer protocol" API in Python 2. +In Python 3, this protocol doesn't exist anymore but the functions are still +exposed to ease porting 2.x code. They act as a compatibility wrapper +around the :ref:`new buffer protocol `, but they don't give +you control over the lifetime of the resources acquired when a buffer is +exported. + +Therefore, it is recommended that you call :c:func:`PyObject_GetBuffer` +(or the ``y*`` or ``w*`` :ref:`format codes ` with the +:c:func:`PyArg_ParseTuple` family of functions) to get a buffer view over +an object, and :c:func:`PyBuffer_Release` when the buffer view can be released. + + +.. c:function:: int PyObject_AsCharBuffer(PyObject *obj, const char **buffer, Py_ssize_t *buffer_len) + + Returns a pointer to a read-only memory location usable as character-based + input. The *obj* argument must support the single-segment character buffer + interface. On success, returns ``0``, sets *buffer* to the memory location + and *buffer_len* to the buffer length. Returns ``-1`` and sets a + :exc:`TypeError` on error. + + +.. c:function:: int PyObject_AsReadBuffer(PyObject *obj, const void **buffer, Py_ssize_t *buffer_len) + + Returns a pointer to a read-only memory location containing arbitrary data. + The *obj* argument must support the single-segment readable buffer + interface. On success, returns ``0``, sets *buffer* to the memory location + and *buffer_len* to the buffer length. Returns ``-1`` and sets a + :exc:`TypeError` on error. + + +.. c:function:: int PyObject_CheckReadBuffer(PyObject *o) + + Returns ``1`` if *o* supports the single-segment readable buffer interface. + Otherwise returns ``0``. + + +.. c:function:: int PyObject_AsWriteBuffer(PyObject *obj, void **buffer, Py_ssize_t *buffer_len) + + Returns a pointer to a writable memory location. The *obj* argument must + support the single-segment, character buffer interface. On success, + returns ``0``, sets *buffer* to the memory location and *buffer_len* to the + buffer length. Returns ``-1`` and sets a :exc:`TypeError` on error. + diff --git a/lib/cpython-doc/c-api/object.rst b/lib/cpython-doc/c-api/object.rst new file mode 100644 --- /dev/null +++ b/lib/cpython-doc/c-api/object.rst @@ -0,0 +1,361 @@ +.. highlightlang:: c + +.. _object: + +Object Protocol +=============== + + +.. c:var:: PyObject* Py_NotImplemented + + The ``NotImplemented`` singleton, used to signal that an operation is + not implemented for the given type combination. + + +.. c:macro:: Py_RETURN_NOTIMPLEMENTED + + Properly handle returning :c:data:`Py_NotImplemented` from within a C + function (that is, increment the reference count of NotImplemented and + return it). + + +.. c:function:: int PyObject_Print(PyObject *o, FILE *fp, int flags) + + Print an object *o*, on file *fp*. Returns ``-1`` on error. The flags argument + is used to enable certain printing options. The only option currently supported + is :const:`Py_PRINT_RAW`; if given, the :func:`str` of the object is written + instead of the :func:`repr`. + + +.. c:function:: int PyObject_HasAttr(PyObject *o, PyObject *attr_name) + + Returns ``1`` if *o* has the attribute *attr_name*, and ``0`` otherwise. This + is equivalent to the Python expression ``hasattr(o, attr_name)``. This function + always succeeds. + + +.. c:function:: int PyObject_HasAttrString(PyObject *o, const char *attr_name) + + Returns ``1`` if *o* has the attribute *attr_name*, and ``0`` otherwise. This + is equivalent to the Python expression ``hasattr(o, attr_name)``. This function + always succeeds. + + +.. c:function:: PyObject* PyObject_GetAttr(PyObject *o, PyObject *attr_name) + + Retrieve an attribute named *attr_name* from object *o*. Returns the attribute + value on success, or *NULL* on failure. This is the equivalent of the Python + expression ``o.attr_name``. + + +.. c:function:: PyObject* PyObject_GetAttrString(PyObject *o, const char *attr_name) + + Retrieve an attribute named *attr_name* from object *o*. Returns the attribute + value on success, or *NULL* on failure. This is the equivalent of the Python + expression ``o.attr_name``. + + +.. c:function:: PyObject* PyObject_GenericGetAttr(PyObject *o, PyObject *name) + + Generic attribute getter function that is meant to be put into a type + object's ``tp_getattro`` slot. It looks for a descriptor in the dictionary + of classes in the object's MRO as well as an attribute in the object's + :attr:`__dict__` (if present). As outlined in :ref:`descriptors`, data + descriptors take preference over instance attributes, while non-data + descriptors don't. Otherwise, an :exc:`AttributeError` is raised. + + +.. c:function:: int PyObject_SetAttr(PyObject *o, PyObject *attr_name, PyObject *v) + + Set the value of the attribute named *attr_name*, for object *o*, to the value + *v*. Returns ``-1`` on failure. This is the equivalent of the Python statement + ``o.attr_name = v``. + + +.. c:function:: int PyObject_SetAttrString(PyObject *o, const char *attr_name, PyObject *v) + + Set the value of the attribute named *attr_name*, for object *o*, to the value + *v*. Returns ``-1`` on failure. This is the equivalent of the Python statement + ``o.attr_name = v``. + + +.. c:function:: int PyObject_GenericSetAttr(PyObject *o, PyObject *name, PyObject *value) + + Generic attribute setter function that is meant to be put into a type + object's ``tp_setattro`` slot. It looks for a data descriptor in the + dictionary of classes in the object's MRO, and if found it takes preference + over setting the attribute in the instance dictionary. Otherwise, the + attribute is set in the object's :attr:`__dict__` (if present). Otherwise, + an :exc:`AttributeError` is raised and ``-1`` is returned. + + +.. c:function:: int PyObject_DelAttr(PyObject *o, PyObject *attr_name) + + Delete attribute named *attr_name*, for object *o*. Returns ``-1`` on failure. + This is the equivalent of the Python statement ``del o.attr_name``. + + +.. c:function:: int PyObject_DelAttrString(PyObject *o, const char *attr_name) + + Delete attribute named *attr_name*, for object *o*. Returns ``-1`` on failure. + This is the equivalent of the Python statement ``del o.attr_name``. + + +.. c:function:: PyObject* PyObject_RichCompare(PyObject *o1, PyObject *o2, int opid) + + Compare the values of *o1* and *o2* using the operation specified by *opid*, + which must be one of :const:`Py_LT`, :const:`Py_LE`, :const:`Py_EQ`, + :const:`Py_NE`, :const:`Py_GT`, or :const:`Py_GE`, corresponding to ``<``, + ``<=``, ``==``, ``!=``, ``>``, or ``>=`` respectively. This is the equivalent of + the Python expression ``o1 op o2``, where ``op`` is the operator corresponding + to *opid*. Returns the value of the comparison on success, or *NULL* on failure. + + +.. c:function:: int PyObject_RichCompareBool(PyObject *o1, PyObject *o2, int opid) + + Compare the values of *o1* and *o2* using the operation specified by *opid*, + which must be one of :const:`Py_LT`, :const:`Py_LE`, :const:`Py_EQ`, + :const:`Py_NE`, :const:`Py_GT`, or :const:`Py_GE`, corresponding to ``<``, + ``<=``, ``==``, ``!=``, ``>``, or ``>=`` respectively. Returns ``-1`` on error, + ``0`` if the result is false, ``1`` otherwise. This is the equivalent of the + Python expression ``o1 op o2``, where ``op`` is the operator corresponding to + *opid*. + +.. note:: + If *o1* and *o2* are the same object, :c:func:`PyObject_RichCompareBool` + will always return ``1`` for :const:`Py_EQ` and ``0`` for :const:`Py_NE`. + +.. c:function:: PyObject* PyObject_Repr(PyObject *o) + + .. index:: builtin: repr + + Compute a string representation of object *o*. Returns the string + representation on success, *NULL* on failure. This is the equivalent of the + Python expression ``repr(o)``. Called by the :func:`repr` built-in function. + + +.. c:function:: PyObject* PyObject_ASCII(PyObject *o) + + .. index:: builtin: ascii + + As :c:func:`PyObject_Repr`, compute a string representation of object *o*, but + escape the non-ASCII characters in the string returned by + :c:func:`PyObject_Repr` with ``\x``, ``\u`` or ``\U`` escapes. This generates + a string similar to that returned by :c:func:`PyObject_Repr` in Python 2. + Called by the :func:`ascii` built-in function. + + +.. c:function:: PyObject* PyObject_Str(PyObject *o) + + .. index:: builtin: str + + Compute a string representation of object *o*. Returns the string + representation on success, *NULL* on failure. This is the equivalent of the + Python expression ``str(o)``. Called by the :func:`str` built-in function + and, therefore, by the :func:`print` function. + +.. c:function:: PyObject* PyObject_Bytes(PyObject *o) + + .. index:: builtin: bytes + + Compute a bytes representation of object *o*. *NULL* is returned on + failure and a bytes object on success. This is equivalent to the Python + expression ``bytes(o)``, when *o* is not an integer. Unlike ``bytes(o)``, + a TypeError is raised when *o* is an integer instead of a zero-initialized + bytes object. + +.. c:function:: int PyObject_IsInstance(PyObject *inst, PyObject *cls) + + Returns ``1`` if *inst* is an instance of the class *cls* or a subclass of + *cls*, or ``0`` if not. On error, returns ``-1`` and sets an exception. If + *cls* is a type object rather than a class object, :c:func:`PyObject_IsInstance` + returns ``1`` if *inst* is of type *cls*. If *cls* is a tuple, the check will + be done against every entry in *cls*. The result will be ``1`` when at least one + of the checks returns ``1``, otherwise it will be ``0``. If *inst* is not a + class instance and *cls* is neither a type object, nor a class object, nor a + tuple, *inst* must have a :attr:`__class__` attribute --- the class relationship + of the value of that attribute with *cls* will be used to determine the result + of this function. + + +Subclass determination is done in a fairly straightforward way, but includes a +wrinkle that implementors of extensions to the class system may want to be aware +of. If :class:`A` and :class:`B` are class objects, :class:`B` is a subclass of +:class:`A` if it inherits from :class:`A` either directly or indirectly. If +either is not a class object, a more general mechanism is used to determine the +class relationship of the two objects. When testing if *B* is a subclass of +*A*, if *A* is *B*, :c:func:`PyObject_IsSubclass` returns true. If *A* and *B* +are different objects, *B*'s :attr:`__bases__` attribute is searched in a +depth-first fashion for *A* --- the presence of the :attr:`__bases__` attribute +is considered sufficient for this determination. + + +.. c:function:: int PyObject_IsSubclass(PyObject *derived, PyObject *cls) + + Returns ``1`` if the class *derived* is identical to or derived from the class + *cls*, otherwise returns ``0``. In case of an error, returns ``-1``. If *cls* + is a tuple, the check will be done against every entry in *cls*. The result will + be ``1`` when at least one of the checks returns ``1``, otherwise it will be + ``0``. If either *derived* or *cls* is not an actual class object (or tuple), + this function uses the generic algorithm described above. + + +.. c:function:: int PyCallable_Check(PyObject *o) + + Determine if the object *o* is callable. Return ``1`` if the object is callable + and ``0`` otherwise. This function always succeeds. + + +.. c:function:: PyObject* PyObject_Call(PyObject *callable_object, PyObject *args, PyObject *kw) + + Call a callable Python object *callable_object*, with arguments given by the + tuple *args*, and named arguments given by the dictionary *kw*. If no named + arguments are needed, *kw* may be *NULL*. *args* must not be *NULL*, use an + empty tuple if no arguments are needed. Returns the result of the call on + success, or *NULL* on failure. This is the equivalent of the Python expression + ``callable_object(*args, **kw)``. + + +.. c:function:: PyObject* PyObject_CallObject(PyObject *callable_object, PyObject *args) + + Call a callable Python object *callable_object*, with arguments given by the + tuple *args*. If no arguments are needed, then *args* may be *NULL*. Returns + the result of the call on success, or *NULL* on failure. This is the equivalent + of the Python expression ``callable_object(*args)``. + + +.. c:function:: PyObject* PyObject_CallFunction(PyObject *callable, char *format, ...) + + Call a callable Python object *callable*, with a variable number of C arguments. + The C arguments are described using a :c:func:`Py_BuildValue` style format + string. The format may be *NULL*, indicating that no arguments are provided. + Returns the result of the call on success, or *NULL* on failure. This is the + equivalent of the Python expression ``callable(*args)``. Note that if you only + pass :c:type:`PyObject \*` args, :c:func:`PyObject_CallFunctionObjArgs` is a + faster alternative. + + +.. c:function:: PyObject* PyObject_CallMethod(PyObject *o, char *method, char *format, ...) + + Call the method named *method* of object *o* with a variable number of C + arguments. The C arguments are described by a :c:func:`Py_BuildValue` format + string that should produce a tuple. The format may be *NULL*, indicating that + no arguments are provided. Returns the result of the call on success, or *NULL* + on failure. This is the equivalent of the Python expression ``o.method(args)``. + Note that if you only pass :c:type:`PyObject \*` args, + :c:func:`PyObject_CallMethodObjArgs` is a faster alternative. + + +.. c:function:: PyObject* PyObject_CallFunctionObjArgs(PyObject *callable, ..., NULL) + + Call a callable Python object *callable*, with a variable number of + :c:type:`PyObject\*` arguments. The arguments are provided as a variable number + of parameters followed by *NULL*. Returns the result of the call on success, or + *NULL* on failure. + + +.. c:function:: PyObject* PyObject_CallMethodObjArgs(PyObject *o, PyObject *name, ..., NULL) + + Calls a method of the object *o*, where the name of the method is given as a + Python string object in *name*. It is called with a variable number of + :c:type:`PyObject\*` arguments. The arguments are provided as a variable number + of parameters followed by *NULL*. Returns the result of the call on success, or + *NULL* on failure. + + +.. c:function:: Py_hash_t PyObject_Hash(PyObject *o) + + .. index:: builtin: hash + + Compute and return the hash value of an object *o*. On failure, return ``-1``. + This is the equivalent of the Python expression ``hash(o)``. + + .. versionchanged:: 3.2 + The return type is now Py_hash_t. This is a signed integer the same size + as Py_ssize_t. + + +.. c:function:: Py_hash_t PyObject_HashNotImplemented(PyObject *o) + + Set a :exc:`TypeError` indicating that ``type(o)`` is not hashable and return ``-1``. + This function receives special treatment when stored in a ``tp_hash`` slot, + allowing a type to explicitly indicate to the interpreter that it is not + hashable. + + +.. c:function:: int PyObject_IsTrue(PyObject *o) + + Returns ``1`` if the object *o* is considered to be true, and ``0`` otherwise. + This is equivalent to the Python expression ``not not o``. On failure, return + ``-1``. + + +.. c:function:: int PyObject_Not(PyObject *o) + + Returns ``0`` if the object *o* is considered to be true, and ``1`` otherwise. + This is equivalent to the Python expression ``not o``. On failure, return + ``-1``. + + +.. c:function:: PyObject* PyObject_Type(PyObject *o) + + .. index:: builtin: type + + When *o* is non-*NULL*, returns a type object corresponding to the object type + of object *o*. On failure, raises :exc:`SystemError` and returns *NULL*. This + is equivalent to the Python expression ``type(o)``. This function increments the + reference count of the return value. There's really no reason to use this + function instead of the common expression ``o->ob_type``, which returns a + pointer of type :c:type:`PyTypeObject\*`, except when the incremented reference + count is needed. + + +.. c:function:: int PyObject_TypeCheck(PyObject *o, PyTypeObject *type) + + Return true if the object *o* is of type *type* or a subtype of *type*. Both + parameters must be non-*NULL*. + + +.. c:function:: Py_ssize_t PyObject_Length(PyObject *o) + Py_ssize_t PyObject_Size(PyObject *o) + + .. index:: builtin: len + + Return the length of object *o*. If the object *o* provides either the sequence + and mapping protocols, the sequence length is returned. On error, ``-1`` is + returned. This is the equivalent to the Python expression ``len(o)``. + + +.. c:function:: PyObject* PyObject_GetItem(PyObject *o, PyObject *key) + + Return element of *o* corresponding to the object *key* or *NULL* on failure. + This is the equivalent of the Python expression ``o[key]``. + + +.. c:function:: int PyObject_SetItem(PyObject *o, PyObject *key, PyObject *v) + + Map the object *key* to the value *v*. Returns ``-1`` on failure. This is the + equivalent of the Python statement ``o[key] = v``. + + +.. c:function:: int PyObject_DelItem(PyObject *o, PyObject *key) + + Delete the mapping for *key* from *o*. Returns ``-1`` on failure. This is the + equivalent of the Python statement ``del o[key]``. + + +.. c:function:: PyObject* PyObject_Dir(PyObject *o) + + This is equivalent to the Python expression ``dir(o)``, returning a (possibly + empty) list of strings appropriate for the object argument, or *NULL* if there + was an error. If the argument is *NULL*, this is like the Python ``dir()``, + returning the names of the current locals; in this case, if no execution frame + is active then *NULL* is returned but :c:func:`PyErr_Occurred` will return false. + + +.. c:function:: PyObject* PyObject_GetIter(PyObject *o) + + This is equivalent to the Python expression ``iter(o)``. It returns a new + iterator for the object argument, or the object itself if the object is already + an iterator. Raises :exc:`TypeError` and returns *NULL* if the object cannot be + iterated. diff --git a/lib/cpython-doc/c-api/objimpl.rst b/lib/cpython-doc/c-api/objimpl.rst new file mode 100644 --- /dev/null +++ b/lib/cpython-doc/c-api/objimpl.rst @@ -0,0 +1,17 @@ +.. highlightlang:: c + +.. _newtypes: + +***************************** +Object Implementation Support +***************************** + +This chapter describes the functions, types, and macros used when defining new +object types. + +.. toctree:: + + allocation.rst + structures.rst + typeobj.rst + gcsupport.rst diff --git a/lib/cpython-doc/c-api/refcounting.rst b/lib/cpython-doc/c-api/refcounting.rst new file mode 100644 --- /dev/null +++ b/lib/cpython-doc/c-api/refcounting.rst @@ -0,0 +1,73 @@ +.. highlightlang:: c + + +.. _countingrefs: + +****************** +Reference Counting +****************** + +The macros in this section are used for managing reference counts of Python +objects. + + +.. c:function:: void Py_INCREF(PyObject *o) + + Increment the reference count for object *o*. The object must not be *NULL*; if + you aren't sure that it isn't *NULL*, use :c:func:`Py_XINCREF`. + + +.. c:function:: void Py_XINCREF(PyObject *o) + + Increment the reference count for object *o*. The object may be *NULL*, in + which case the macro has no effect. + + +.. c:function:: void Py_DECREF(PyObject *o) + + Decrement the reference count for object *o*. The object must not be *NULL*; if + you aren't sure that it isn't *NULL*, use :c:func:`Py_XDECREF`. If the reference + count reaches zero, the object's type's deallocation function (which must not be + *NULL*) is invoked. + + .. warning:: + + The deallocation function can cause arbitrary Python code to be invoked (e.g. + when a class instance with a :meth:`__del__` method is deallocated). While + exceptions in such code are not propagated, the executed code has free access to + all Python global variables. This means that any object that is reachable from + a global variable should be in a consistent state before :c:func:`Py_DECREF` is + invoked. For example, code to delete an object from a list should copy a + reference to the deleted object in a temporary variable, update the list data + structure, and then call :c:func:`Py_DECREF` for the temporary variable. + + +.. c:function:: void Py_XDECREF(PyObject *o) + + Decrement the reference count for object *o*. The object may be *NULL*, in + which case the macro has no effect; otherwise the effect is the same as for + :c:func:`Py_DECREF`, and the same warning applies. + + +.. c:function:: void Py_CLEAR(PyObject *o) + + Decrement the reference count for object *o*. The object may be *NULL*, in + which case the macro has no effect; otherwise the effect is the same as for + :c:func:`Py_DECREF`, except that the argument is also set to *NULL*. The warning + for :c:func:`Py_DECREF` does not apply with respect to the object passed because + the macro carefully uses a temporary variable and sets the argument to *NULL* + before decrementing its reference count. + + It is a good idea to use this macro whenever decrementing the value of a + variable that might be traversed during garbage collection. + + +The following functions are for runtime dynamic embedding of Python: +``Py_IncRef(PyObject *o)``, ``Py_DecRef(PyObject *o)``. They are +simply exported function versions of :c:func:`Py_XINCREF` and +:c:func:`Py_XDECREF`, respectively. + +The following functions or macros are only for use within the interpreter core: +:c:func:`_Py_Dealloc`, :c:func:`_Py_ForgetReference`, :c:func:`_Py_NewReference`, +as well as the global variable :c:data:`_Py_RefTotal`. + diff --git a/lib/cpython-doc/c-api/reflection.rst b/lib/cpython-doc/c-api/reflection.rst new file mode 100644 --- /dev/null +++ b/lib/cpython-doc/c-api/reflection.rst @@ -0,0 +1,49 @@ +.. highlightlang:: c + +.. _reflection: + +Reflection +========== + +.. c:function:: PyObject* PyEval_GetBuiltins() + + Return a dictionary of the builtins in the current execution frame, + or the interpreter of the thread state if no frame is currently executing. + + +.. c:function:: PyObject* PyEval_GetLocals() + + Return a dictionary of the local variables in the current execution frame, + or *NULL* if no frame is currently executing. + + +.. c:function:: PyObject* PyEval_GetGlobals() + + Return a dictionary of the global variables in the current execution frame, + or *NULL* if no frame is currently executing. + + +.. c:function:: PyFrameObject* PyEval_GetFrame() + + Return the current thread state's frame, which is *NULL* if no frame is + currently executing. + + +.. c:function:: int PyFrame_GetLineNumber(PyFrameObject *frame) + + Return the line number that *frame* is currently executing. + + +.. c:function:: const char* PyEval_GetFuncName(PyObject *func) + + Return the name of *func* if it is a function, class or instance object, else the + name of *func*\s type. + + +.. c:function:: const char* PyEval_GetFuncDesc(PyObject *func) + + Return a description string, depending on the type of *func*. + Return values include "()" for functions and methods, " constructor", + " instance", and " object". Concatenated with the result of + :c:func:`PyEval_GetFuncName`, the result will be a description of + *func*. diff --git a/lib/cpython-doc/c-api/sequence.rst b/lib/cpython-doc/c-api/sequence.rst new file mode 100644 --- /dev/null +++ b/lib/cpython-doc/c-api/sequence.rst @@ -0,0 +1,162 @@ +.. highlightlang:: c + +.. _sequence: + +Sequence Protocol +================= + + +.. c:function:: int PySequence_Check(PyObject *o) + + Return ``1`` if the object provides sequence protocol, and ``0`` otherwise. + This function always succeeds. + + +.. c:function:: Py_ssize_t PySequence_Size(PyObject *o) + Py_ssize_t PySequence_Length(PyObject *o) + + .. index:: builtin: len + + Returns the number of objects in sequence *o* on success, and ``-1`` on failure. + For objects that do not provide sequence protocol, this is equivalent to the + Python expression ``len(o)``. + + +.. c:function:: PyObject* PySequence_Concat(PyObject *o1, PyObject *o2) + + Return the concatenation of *o1* and *o2* on success, and *NULL* on failure. + This is the equivalent of the Python expression ``o1 + o2``. + + +.. c:function:: PyObject* PySequence_Repeat(PyObject *o, Py_ssize_t count) + + Return the result of repeating sequence object *o* *count* times, or *NULL* on + failure. This is the equivalent of the Python expression ``o * count``. + + +.. c:function:: PyObject* PySequence_InPlaceConcat(PyObject *o1, PyObject *o2) + + Return the concatenation of *o1* and *o2* on success, and *NULL* on failure. + The operation is done *in-place* when *o1* supports it. This is the equivalent + of the Python expression ``o1 += o2``. + + +.. c:function:: PyObject* PySequence_InPlaceRepeat(PyObject *o, Py_ssize_t count) + + Return the result of repeating sequence object *o* *count* times, or *NULL* on + failure. The operation is done *in-place* when *o* supports it. This is the + equivalent of the Python expression ``o *= count``. + + +.. c:function:: PyObject* PySequence_GetItem(PyObject *o, Py_ssize_t i) + + Return the *i*\ th element of *o*, or *NULL* on failure. This is the equivalent of + the Python expression ``o[i]``. + + +.. c:function:: PyObject* PySequence_GetSlice(PyObject *o, Py_ssize_t i1, Py_ssize_t i2) + + Return the slice of sequence object *o* between *i1* and *i2*, or *NULL* on + failure. This is the equivalent of the Python expression ``o[i1:i2]``. + + +.. c:function:: int PySequence_SetItem(PyObject *o, Py_ssize_t i, PyObject *v) + + Assign object *v* to the *i*\ th element of *o*. Returns ``-1`` on failure. This + is the equivalent of the Python statement ``o[i] = v``. This function *does + not* steal a reference to *v*. + + +.. c:function:: int PySequence_DelItem(PyObject *o, Py_ssize_t i) + + Delete the *i*\ th element of object *o*. Returns ``-1`` on failure. This is the + equivalent of the Python statement ``del o[i]``. + + +.. c:function:: int PySequence_SetSlice(PyObject *o, Py_ssize_t i1, Py_ssize_t i2, PyObject *v) + + Assign the sequence object *v* to the slice in sequence object *o* from *i1* to + *i2*. This is the equivalent of the Python statement ``o[i1:i2] = v``. + + +.. c:function:: int PySequence_DelSlice(PyObject *o, Py_ssize_t i1, Py_ssize_t i2) + + Delete the slice in sequence object *o* from *i1* to *i2*. Returns ``-1`` on + failure. This is the equivalent of the Python statement ``del o[i1:i2]``. + + +.. c:function:: Py_ssize_t PySequence_Count(PyObject *o, PyObject *value) + + Return the number of occurrences of *value* in *o*, that is, return the number + of keys for which ``o[key] == value``. On failure, return ``-1``. This is + equivalent to the Python expression ``o.count(value)``. + + +.. c:function:: int PySequence_Contains(PyObject *o, PyObject *value) + + Determine if *o* contains *value*. If an item in *o* is equal to *value*, + return ``1``, otherwise return ``0``. On error, return ``-1``. This is + equivalent to the Python expression ``value in o``. + + +.. c:function:: Py_ssize_t PySequence_Index(PyObject *o, PyObject *value) + + Return the first index *i* for which ``o[i] == value``. On error, return + ``-1``. This is equivalent to the Python expression ``o.index(value)``. + + +.. c:function:: PyObject* PySequence_List(PyObject *o) + + Return a list object with the same contents as the arbitrary sequence *o*. The + returned list is guaranteed to be new. + + +.. c:function:: PyObject* PySequence_Tuple(PyObject *o) + + .. index:: builtin: tuple + + Return a tuple object with the same contents as the arbitrary sequence *o* or + *NULL* on failure. If *o* is a tuple, a new reference will be returned, + otherwise a tuple will be constructed with the appropriate contents. This is + equivalent to the Python expression ``tuple(o)``. + + +.. c:function:: PyObject* PySequence_Fast(PyObject *o, const char *m) + + Returns the sequence *o* as a tuple, unless it is already a tuple or list, in + which case *o* is returned. Use :c:func:`PySequence_Fast_GET_ITEM` to access the + members of the result. Returns *NULL* on failure. If the object is not a + sequence, raises :exc:`TypeError` with *m* as the message text. + + +.. c:function:: PyObject* PySequence_Fast_GET_ITEM(PyObject *o, Py_ssize_t i) + + Return the *i*\ th element of *o*, assuming that *o* was returned by + :c:func:`PySequence_Fast`, *o* is not *NULL*, and that *i* is within bounds. + + +.. c:function:: PyObject** PySequence_Fast_ITEMS(PyObject *o) + + Return the underlying array of PyObject pointers. Assumes that *o* was returned + by :c:func:`PySequence_Fast` and *o* is not *NULL*. + + Note, if a list gets resized, the reallocation may relocate the items array. + So, only use the underlying array pointer in contexts where the sequence + cannot change. + + +.. c:function:: PyObject* PySequence_ITEM(PyObject *o, Py_ssize_t i) + + Return the *i*\ th element of *o* or *NULL* on failure. Macro form of + :c:func:`PySequence_GetItem` but without checking that + :c:func:`PySequence_Check` on *o* is true and without adjustment for negative + indices. + + +.. c:function:: Py_ssize_t PySequence_Fast_GET_SIZE(PyObject *o) + + Returns the length of *o*, assuming that *o* was returned by + :c:func:`PySequence_Fast` and that *o* is not *NULL*. The size can also be + gotten by calling :c:func:`PySequence_Size` on *o*, but + :c:func:`PySequence_Fast_GET_SIZE` is faster because it can assume *o* is a list + or tuple. diff --git a/lib/cpython-doc/c-api/set.rst b/lib/cpython-doc/c-api/set.rst new file mode 100644 --- /dev/null +++ b/lib/cpython-doc/c-api/set.rst @@ -0,0 +1,166 @@ +.. highlightlang:: c + +.. _setobjects: + +Set Objects +----------- + +.. sectionauthor:: Raymond D. Hettinger + + +.. index:: + object: set + object: frozenset + +This section details the public API for :class:`set` and :class:`frozenset` +objects. Any functionality not listed below is best accessed using the either +the abstract object protocol (including :c:func:`PyObject_CallMethod`, +:c:func:`PyObject_RichCompareBool`, :c:func:`PyObject_Hash`, +:c:func:`PyObject_Repr`, :c:func:`PyObject_IsTrue`, :c:func:`PyObject_Print`, and +:c:func:`PyObject_GetIter`) or the abstract number protocol (including +:c:func:`PyNumber_And`, :c:func:`PyNumber_Subtract`, :c:func:`PyNumber_Or`, +:c:func:`PyNumber_Xor`, :c:func:`PyNumber_InPlaceAnd`, +:c:func:`PyNumber_InPlaceSubtract`, :c:func:`PyNumber_InPlaceOr`, and +:c:func:`PyNumber_InPlaceXor`). + + +.. c:type:: PySetObject + + This subtype of :c:type:`PyObject` is used to hold the internal data for both + :class:`set` and :class:`frozenset` objects. It is like a :c:type:`PyDictObject` + in that it is a fixed size for small sets (much like tuple storage) and will + point to a separate, variable sized block of memory for medium and large sized + sets (much like list storage). None of the fields of this structure should be + considered public and are subject to change. All access should be done through + the documented API rather than by manipulating the values in the structure. + + +.. c:var:: PyTypeObject PySet_Type + + This is an instance of :c:type:`PyTypeObject` representing the Python + :class:`set` type. + + +.. c:var:: PyTypeObject PyFrozenSet_Type + + This is an instance of :c:type:`PyTypeObject` representing the Python + :class:`frozenset` type. + +The following type check macros work on pointers to any Python object. Likewise, +the constructor functions work with any iterable Python object. + + +.. c:function:: int PySet_Check(PyObject *p) + + Return true if *p* is a :class:`set` object or an instance of a subtype. + +.. c:function:: int PyFrozenSet_Check(PyObject *p) + + Return true if *p* is a :class:`frozenset` object or an instance of a + subtype. + +.. c:function:: int PyAnySet_Check(PyObject *p) + + Return true if *p* is a :class:`set` object, a :class:`frozenset` object, or an + instance of a subtype. + + +.. c:function:: int PyAnySet_CheckExact(PyObject *p) + + Return true if *p* is a :class:`set` object or a :class:`frozenset` object but + not an instance of a subtype. + + +.. c:function:: int PyFrozenSet_CheckExact(PyObject *p) + + Return true if *p* is a :class:`frozenset` object but not an instance of a + subtype. + + +.. c:function:: PyObject* PySet_New(PyObject *iterable) + + Return a new :class:`set` containing objects returned by the *iterable*. The + *iterable* may be *NULL* to create a new empty set. Return the new set on + success or *NULL* on failure. Raise :exc:`TypeError` if *iterable* is not + actually iterable. The constructor is also useful for copying a set + (``c=set(s)``). + + +.. c:function:: PyObject* PyFrozenSet_New(PyObject *iterable) + + Return a new :class:`frozenset` containing objects returned by the *iterable*. + The *iterable* may be *NULL* to create a new empty frozenset. Return the new + set on success or *NULL* on failure. Raise :exc:`TypeError` if *iterable* is + not actually iterable. + + +The following functions and macros are available for instances of :class:`set` +or :class:`frozenset` or instances of their subtypes. + + +.. c:function:: Py_ssize_t PySet_Size(PyObject *anyset) + + .. index:: builtin: len + + Return the length of a :class:`set` or :class:`frozenset` object. Equivalent to + ``len(anyset)``. Raises a :exc:`PyExc_SystemError` if *anyset* is not a + :class:`set`, :class:`frozenset`, or an instance of a subtype. + + +.. c:function:: Py_ssize_t PySet_GET_SIZE(PyObject *anyset) + + Macro form of :c:func:`PySet_Size` without error checking. + + +.. c:function:: int PySet_Contains(PyObject *anyset, PyObject *key) + + Return 1 if found, 0 if not found, and -1 if an error is encountered. Unlike + the Python :meth:`__contains__` method, this function does not automatically + convert unhashable sets into temporary frozensets. Raise a :exc:`TypeError` if + the *key* is unhashable. Raise :exc:`PyExc_SystemError` if *anyset* is not a + :class:`set`, :class:`frozenset`, or an instance of a subtype. + + +.. c:function:: int PySet_Add(PyObject *set, PyObject *key) + + Add *key* to a :class:`set` instance. Also works with :class:`frozenset` + instances (like :c:func:`PyTuple_SetItem` it can be used to fill-in the values + of brand new frozensets before they are exposed to other code). Return 0 on + success or -1 on failure. Raise a :exc:`TypeError` if the *key* is + unhashable. Raise a :exc:`MemoryError` if there is no room to grow. Raise a + :exc:`SystemError` if *set* is an not an instance of :class:`set` or its + subtype. + + +The following functions are available for instances of :class:`set` or its +subtypes but not for instances of :class:`frozenset` or its subtypes. + + +.. c:function:: int PySet_Discard(PyObject *set, PyObject *key) + + Return 1 if found and removed, 0 if not found (no action taken), and -1 if an + error is encountered. Does not raise :exc:`KeyError` for missing keys. Raise a + :exc:`TypeError` if the *key* is unhashable. Unlike the Python :meth:`discard` + method, this function does not automatically convert unhashable sets into + temporary frozensets. Raise :exc:`PyExc_SystemError` if *set* is an not an + instance of :class:`set` or its subtype. + + +.. c:function:: PyObject* PySet_Pop(PyObject *set) + + Return a new reference to an arbitrary object in the *set*, and removes the + object from the *set*. Return *NULL* on failure. Raise :exc:`KeyError` if the + set is empty. Raise a :exc:`SystemError` if *set* is an not an instance of + :class:`set` or its subtype. + + +.. c:function:: int PySet_Clear(PyObject *set) + + Empty an existing set of all elements. + + +.. c:function:: int PySet_ClearFreeList() + + Clear the free list. Return the total number of freed items. + + .. versionadded:: 3.3 diff --git a/lib/cpython-doc/c-api/slice.rst b/lib/cpython-doc/c-api/slice.rst new file mode 100644 --- /dev/null +++ b/lib/cpython-doc/c-api/slice.rst @@ -0,0 +1,58 @@ +.. highlightlang:: c + +.. _slice-objects: + +Slice Objects +------------- + + +.. c:var:: PyTypeObject PySlice_Type + + The type object for slice objects. This is the same as :class:`slice` in the + Python layer. + + +.. c:function:: int PySlice_Check(PyObject *ob) + + Return true if *ob* is a slice object; *ob* must not be *NULL*. + + +.. c:function:: PyObject* PySlice_New(PyObject *start, PyObject *stop, PyObject *step) + + Return a new slice object with the given values. The *start*, *stop*, and + *step* parameters are used as the values of the slice object attributes of + the same names. Any of the values may be *NULL*, in which case the + ``None`` will be used for the corresponding attribute. Return *NULL* if + the new object could not be allocated. + + +.. c:function:: int PySlice_GetIndices(PyObject *slice, Py_ssize_t length, Py_ssize_t *start, Py_ssize_t *stop, Py_ssize_t *step) + + Retrieve the start, stop and step indices from the slice object *slice*, + assuming a sequence of length *length*. Treats indices greater than + *length* as errors. + + Returns 0 on success and -1 on error with no exception set (unless one of + the indices was not :const:`None` and failed to be converted to an integer, + in which case -1 is returned with an exception set). + + You probably do not want to use this function. + + .. versionchanged:: 3.2 + The parameter type for the *slice* parameter was ``PySliceObject*`` + before. + + +.. c:function:: int PySlice_GetIndicesEx(PyObject *slice, Py_ssize_t length, Py_ssize_t *start, Py_ssize_t *stop, Py_ssize_t *step, Py_ssize_t *slicelength) + + Usable replacement for :c:func:`PySlice_GetIndices`. Retrieve the start, + stop, and step indices from the slice object *slice* assuming a sequence of + length *length*, and store the length of the slice in *slicelength*. Out + of bounds indices are clipped in a manner consistent with the handling of + normal slices. + + Returns 0 on success and -1 on error with exception set. + + .. versionchanged:: 3.2 + The parameter type for the *slice* parameter was ``PySliceObject*`` + before. diff --git a/lib/cpython-doc/c-api/structures.rst b/lib/cpython-doc/c-api/structures.rst new file mode 100644 --- /dev/null +++ b/lib/cpython-doc/c-api/structures.rst @@ -0,0 +1,284 @@ +.. highlightlang:: c + +.. _common-structs: + +Common Object Structures +======================== + +There are a large number of structures which are used in the definition of +object types for Python. This section describes these structures and how they +are used. + +All Python objects ultimately share a small number of fields at the beginning +of the object's representation in memory. These are represented by the +:c:type:`PyObject` and :c:type:`PyVarObject` types, which are defined, in turn, +by the expansions of some macros also used, whether directly or indirectly, in +the definition of all other Python objects. + + +.. c:type:: PyObject + + All object types are extensions of this type. This is a type which + contains the information Python needs to treat a pointer to an object as an + object. In a normal "release" build, it contains only the object's + reference count and a pointer to the corresponding type object. It + corresponds to the fields defined by the expansion of the ``PyObject_HEAD`` + macro. + + +.. c:type:: PyVarObject + + This is an extension of :c:type:`PyObject` that adds the :attr:`ob_size` + field. This is only used for objects that have some notion of *length*. + This type does not often appear in the Python/C API. It corresponds to the + fields defined by the expansion of the ``PyObject_VAR_HEAD`` macro. + +These macros are used in the definition of :c:type:`PyObject` and +:c:type:`PyVarObject`: + +.. XXX need to document PEP 3123 changes here + +.. c:macro:: PyObject_HEAD + + This is a macro which expands to the declarations of the fields of the + :c:type:`PyObject` type; it is used when declaring new types which represent + objects without a varying length. The specific fields it expands to depend + on the definition of :c:macro:`Py_TRACE_REFS`. By default, that macro is + not defined, and :c:macro:`PyObject_HEAD` expands to:: + + Py_ssize_t ob_refcnt; + PyTypeObject *ob_type; + + When :c:macro:`Py_TRACE_REFS` is defined, it expands to:: + + PyObject *_ob_next, *_ob_prev; + Py_ssize_t ob_refcnt; + PyTypeObject *ob_type; + + +.. c:macro:: PyObject_VAR_HEAD + + This is a macro which expands to the declarations of the fields of the + :c:type:`PyVarObject` type; it is used when declaring new types which + represent objects with a length that varies from instance to instance. + This macro always expands to:: + + PyObject_HEAD + Py_ssize_t ob_size; + + Note that :c:macro:`PyObject_HEAD` is part of the expansion, and that its own + expansion varies depending on the definition of :c:macro:`Py_TRACE_REFS`. + + +.. c:macro:: PyObject_HEAD_INIT(type) + + This is a macro which expands to initialization values for a new + :c:type:`PyObject` type. This macro expands to:: + + _PyObject_EXTRA_INIT + 1, type, + + +.. c:macro:: PyVarObject_HEAD_INIT(type, size) + + This is a macro which expands to initialization values for a new + :c:type:`PyVarObject` type, including the :attr:`ob_size` field. + This macro expands to:: + + _PyObject_EXTRA_INIT + 1, type, size, + + +.. c:type:: PyCFunction + + Type of the functions used to implement most Python callables in C. + Functions of this type take two :c:type:`PyObject\*` parameters and return + one such value. If the return value is *NULL*, an exception shall have + been set. If not *NULL*, the return value is interpreted as the return + value of the function as exposed in Python. The function must return a new + reference. + + +.. c:type:: PyCFunctionWithKeywords + + Type of the functions used to implement Python callables in C that take + keyword arguments: they take three :c:type:`PyObject\*` parameters and return + one such value. See :c:type:`PyCFunction` above for the meaning of the return + value. + + +.. c:type:: PyMethodDef + + Structure used to describe a method of an extension type. This structure has + four fields: + + +------------------+-------------+-------------------------------+ + | Field | C Type | Meaning | + +==================+=============+===============================+ + | :attr:`ml_name` | char \* | name of the method | + +------------------+-------------+-------------------------------+ + | :attr:`ml_meth` | PyCFunction | pointer to the C | + | | | implementation | + +------------------+-------------+-------------------------------+ + | :attr:`ml_flags` | int | flag bits indicating how the | + | | | call should be constructed | + +------------------+-------------+-------------------------------+ + | :attr:`ml_doc` | char \* | points to the contents of the | + | | | docstring | + +------------------+-------------+-------------------------------+ + +The :attr:`ml_meth` is a C function pointer. The functions may be of different +types, but they always return :c:type:`PyObject\*`. If the function is not of +the :c:type:`PyCFunction`, the compiler will require a cast in the method table. +Even though :c:type:`PyCFunction` defines the first parameter as +:c:type:`PyObject\*`, it is common that the method implementation uses a the +specific C type of the *self* object. + +The :attr:`ml_flags` field is a bitfield which can include the following flags. +The individual flags indicate either a calling convention or a binding +convention. Of the calling convention flags, only :const:`METH_VARARGS` and +:const:`METH_KEYWORDS` can be combined (but note that :const:`METH_KEYWORDS` +alone is equivalent to ``METH_VARARGS | METH_KEYWORDS``). Any of the calling +convention flags can be combined with a binding flag. + + +.. data:: METH_VARARGS + + This is the typical calling convention, where the methods have the type + :c:type:`PyCFunction`. The function expects two :c:type:`PyObject\*` values. + The first one is the *self* object for methods; for module functions, it is + the module object. The second parameter (often called *args*) is a tuple + object representing all arguments. This parameter is typically processed + using :c:func:`PyArg_ParseTuple` or :c:func:`PyArg_UnpackTuple`. + + +.. data:: METH_KEYWORDS + + Methods with these flags must be of type :c:type:`PyCFunctionWithKeywords`. + The function expects three parameters: *self*, *args*, and a dictionary of + all the keyword arguments. The flag is typically combined with + :const:`METH_VARARGS`, and the parameters are typically processed using + :c:func:`PyArg_ParseTupleAndKeywords`. + + +.. data:: METH_NOARGS + + Methods without parameters don't need to check whether arguments are given if + they are listed with the :const:`METH_NOARGS` flag. They need to be of type + :c:type:`PyCFunction`. The first parameter is typically named *self* and will + hold a reference to the module or object instance. In all cases the second + parameter will be *NULL*. + + +.. data:: METH_O + + Methods with a single object argument can be listed with the :const:`METH_O` + flag, instead of invoking :c:func:`PyArg_ParseTuple` with a ``"O"`` argument. + They have the type :c:type:`PyCFunction`, with the *self* parameter, and a + :c:type:`PyObject\*` parameter representing the single argument. + + +These two constants are not used to indicate the calling convention but the +binding when use with methods of classes. These may not be used for functions +defined for modules. At most one of these flags may be set for any given +method. + + +.. data:: METH_CLASS + + .. index:: builtin: classmethod + + The method will be passed the type object as the first parameter rather + than an instance of the type. This is used to create *class methods*, + similar to what is created when using the :func:`classmethod` built-in + function. + + +.. data:: METH_STATIC + + .. index:: builtin: staticmethod + + The method will be passed *NULL* as the first parameter rather than an + instance of the type. This is used to create *static methods*, similar to + what is created when using the :func:`staticmethod` built-in function. + +One other constant controls whether a method is loaded in place of another +definition with the same method name. + + +.. data:: METH_COEXIST + + The method will be loaded in place of existing definitions. Without + *METH_COEXIST*, the default is to skip repeated definitions. Since slot + wrappers are loaded before the method table, the existence of a + *sq_contains* slot, for example, would generate a wrapped method named + :meth:`__contains__` and preclude the loading of a corresponding + PyCFunction with the same name. With the flag defined, the PyCFunction + will be loaded in place of the wrapper object and will co-exist with the + slot. This is helpful because calls to PyCFunctions are optimized more + than wrapper object calls. + + +.. c:type:: PyMemberDef + + Structure which describes an attribute of a type which corresponds to a C + struct member. Its fields are: + + +------------------+-------------+-------------------------------+ + | Field | C Type | Meaning | + +==================+=============+===============================+ + | :attr:`name` | char \* | name of the member | + +------------------+-------------+-------------------------------+ + | :attr:`type` | int | the type of the member in the | + | | | C struct | + +------------------+-------------+-------------------------------+ + | :attr:`offset` | Py_ssize_t | the offset in bytes that the | + | | | member is located on the | + | | | type's object struct | + +------------------+-------------+-------------------------------+ + | :attr:`flags` | int | flag bits indicating if the | + | | | field should be read-only or | + | | | writable | + +------------------+-------------+-------------------------------+ + | :attr:`doc` | char \* | points to the contents of the | + | | | docstring | + +------------------+-------------+-------------------------------+ + + :attr:`type` can be one of many ``T_`` macros corresponding to various C + types. When the member is accessed in Python, it will be converted to the + equivalent Python type. + + =============== ================== + Macro name C type + =============== ================== + T_SHORT short + T_INT int + T_LONG long + T_FLOAT float + T_DOUBLE double + T_STRING char \* + T_OBJECT PyObject \* + T_OBJECT_EX PyObject \* + T_CHAR char + T_BYTE char + T_UBYTE unsigned char + T_UINT unsigned int + T_USHORT unsigned short + T_ULONG unsigned long + T_BOOL char + T_LONGLONG long long + T_ULONGLONG unsigned long long + T_PYSSIZET Py_ssize_t + =============== ================== + + :c:macro:`T_OBJECT` and :c:macro:`T_OBJECT_EX` differ in that + :c:macro:`T_OBJECT` returns ``None`` if the member is *NULL* and + :c:macro:`T_OBJECT_EX` raises an :exc:`AttributeError`. Try to use + :c:macro:`T_OBJECT_EX` over :c:macro:`T_OBJECT` because :c:macro:`T_OBJECT_EX` + handles use of the :keyword:`del` statement on that attribute more correctly + than :c:macro:`T_OBJECT`. + + :attr:`flags` can be 0 for write and read access or :c:macro:`READONLY` for + read-only access. Using :c:macro:`T_STRING` for :attr:`type` implies + :c:macro:`READONLY`. Only :c:macro:`T_OBJECT` and :c:macro:`T_OBJECT_EX` + members can be deleted. (They are set to *NULL*). diff --git a/lib/cpython-doc/c-api/sys.rst b/lib/cpython-doc/c-api/sys.rst new file mode 100644 --- /dev/null +++ b/lib/cpython-doc/c-api/sys.rst @@ -0,0 +1,186 @@ +.. highlightlang:: c + +.. _os: + +Operating System Utilities +========================== + + +.. c:function:: int Py_FdIsInteractive(FILE *fp, const char *filename) + + Return true (nonzero) if the standard I/O file *fp* with name *filename* is + deemed interactive. This is the case for files for which ``isatty(fileno(fp))`` + is true. If the global flag :c:data:`Py_InteractiveFlag` is true, this function + also returns true if the *filename* pointer is *NULL* or if the name is equal to + one of the strings ``''`` or ``'???'``. + + +.. c:function:: void PyOS_AfterFork() + + Function to update some internal state after a process fork; this should be + called in the new process if the Python interpreter will continue to be used. + If a new executable is loaded into the new process, this function does not need + to be called. + + +.. c:function:: int PyOS_CheckStack() + + Return true when the interpreter runs out of stack space. This is a reliable + check, but is only available when :const:`USE_STACKCHECK` is defined (currently + on Windows using the Microsoft Visual C++ compiler). :const:`USE_STACKCHECK` + will be defined automatically; you should never change the definition in your + own code. + + +.. c:function:: PyOS_sighandler_t PyOS_getsig(int i) + + Return the current signal handler for signal *i*. This is a thin wrapper around + either :c:func:`sigaction` or :c:func:`signal`. Do not call those functions + directly! :c:type:`PyOS_sighandler_t` is a typedef alias for :c:type:`void + (\*)(int)`. + + +.. c:function:: PyOS_sighandler_t PyOS_setsig(int i, PyOS_sighandler_t h) + + Set the signal handler for signal *i* to be *h*; return the old signal handler. + This is a thin wrapper around either :c:func:`sigaction` or :c:func:`signal`. Do + not call those functions directly! :c:type:`PyOS_sighandler_t` is a typedef + alias for :c:type:`void (\*)(int)`. + +.. _systemfunctions: + +System Functions +================ + +These are utility functions that make functionality from the :mod:`sys` module +accessible to C code. They all work with the current interpreter thread's +:mod:`sys` module's dict, which is contained in the internal thread state structure. + +.. c:function:: PyObject *PySys_GetObject(char *name) + + Return the object *name* from the :mod:`sys` module or *NULL* if it does + not exist, without setting an exception. + +.. c:function:: FILE *PySys_GetFile(char *name, FILE *def) + + Return the :c:type:`FILE*` associated with the object *name* in the + :mod:`sys` module, or *def* if *name* is not in the module or is not associated + with a :c:type:`FILE*`. + +.. c:function:: int PySys_SetObject(char *name, PyObject *v) + + Set *name* in the :mod:`sys` module to *v* unless *v* is *NULL*, in which + case *name* is deleted from the sys module. Returns ``0`` on success, ``-1`` + on error. + +.. c:function:: void PySys_ResetWarnOptions() + + Reset :data:`sys.warnoptions` to an empty list. + +.. c:function:: void PySys_AddWarnOption(wchar_t *s) + + Append *s* to :data:`sys.warnoptions`. + +.. c:function:: void PySys_AddWarnOptionUnicode(PyObject *unicode) + + Append *unicode* to :data:`sys.warnoptions`. + +.. c:function:: void PySys_SetPath(wchar_t *path) + + Set :data:`sys.path` to a list object of paths found in *path* which should + be a list of paths separated with the platform's search path delimiter + (``:`` on Unix, ``;`` on Windows). + +.. c:function:: void PySys_WriteStdout(const char *format, ...) + + Write the output string described by *format* to :data:`sys.stdout`. No + exceptions are raised, even if truncation occurs (see below). + + *format* should limit the total size of the formatted output string to + 1000 bytes or less -- after 1000 bytes, the output string is truncated. + In particular, this means that no unrestricted "%s" formats should occur; + these should be limited using "%.s" where is a decimal number + calculated so that plus the maximum size of other formatted text does not + exceed 1000 bytes. Also watch out for "%f", which can print hundreds of + digits for very large numbers. + + If a problem occurs, or :data:`sys.stdout` is unset, the formatted message + is written to the real (C level) *stdout*. + +.. c:function:: void PySys_WriteStderr(const char *format, ...) + + As :c:func:`PySys_WriteStdout`, but write to :data:`sys.stderr` or *stderr* + instead. + +.. c:function:: void PySys_FormatStdout(const char *format, ...) + + Function similar to PySys_WriteStdout() but format the message using + :c:func:`PyUnicode_FromFormatV` and don't truncate the message to an + arbitrary length. + + .. versionadded:: 3.2 + +.. c:function:: void PySys_FormatStderr(const char *format, ...) + + As :c:func:`PySys_FormatStdout`, but write to :data:`sys.stderr` or *stderr* + instead. + + .. versionadded:: 3.2 + +.. c:function:: void PySys_AddXOption(const wchar_t *s) + + Parse *s* as a set of :option:`-X` options and add them to the current + options mapping as returned by :c:func:`PySys_GetXOptions`. + + .. versionadded:: 3.2 + +.. c:function:: PyObject *PySys_GetXOptions() + + Return the current dictionary of :option:`-X` options, similarly to + :data:`sys._xoptions`. On error, *NULL* is returned and an exception is + set. + + .. versionadded:: 3.2 + + +.. _processcontrol: + +Process Control +=============== + + +.. c:function:: void Py_FatalError(const char *message) + + .. index:: single: abort() + + Print a fatal error message and kill the process. No cleanup is performed. + This function should only be invoked when a condition is detected that would + make it dangerous to continue using the Python interpreter; e.g., when the + object administration appears to be corrupted. On Unix, the standard C library + function :c:func:`abort` is called which will attempt to produce a :file:`core` + file. + + +.. c:function:: void Py_Exit(int status) + + .. index:: + single: Py_Finalize() + single: exit() + + Exit the current process. This calls :c:func:`Py_Finalize` and then calls the + standard C library function ``exit(status)``. + + +.. c:function:: int Py_AtExit(void (*func) ()) + + .. index:: + single: Py_Finalize() + single: cleanup functions + + Register a cleanup function to be called by :c:func:`Py_Finalize`. The cleanup + function will be called with no arguments and should return no value. At most + 32 cleanup functions can be registered. When the registration is successful, + :c:func:`Py_AtExit` returns ``0``; on failure, it returns ``-1``. The cleanup + function registered last is called first. Each cleanup function will be called + at most once. Since Python's internal finalization will have completed before + the cleanup function, no Python APIs should be called by *func*. diff --git a/lib/cpython-doc/c-api/tuple.rst b/lib/cpython-doc/c-api/tuple.rst new file mode 100644 --- /dev/null +++ b/lib/cpython-doc/c-api/tuple.rst @@ -0,0 +1,110 @@ +.. highlightlang:: c + +.. _tupleobjects: + +Tuple Objects +------------- + +.. index:: object: tuple + + +.. c:type:: PyTupleObject + + This subtype of :c:type:`PyObject` represents a Python tuple object. + + +.. c:var:: PyTypeObject PyTuple_Type + + This instance of :c:type:`PyTypeObject` represents the Python tuple type; it + is the same object as :class:`tuple` in the Python layer. + + +.. c:function:: int PyTuple_Check(PyObject *p) + + Return true if *p* is a tuple object or an instance of a subtype of the tuple + type. + + +.. c:function:: int PyTuple_CheckExact(PyObject *p) + + Return true if *p* is a tuple object, but not an instance of a subtype of the + tuple type. + + +.. c:function:: PyObject* PyTuple_New(Py_ssize_t len) + + Return a new tuple object of size *len*, or *NULL* on failure. + + +.. c:function:: PyObject* PyTuple_Pack(Py_ssize_t n, ...) + + Return a new tuple object of size *n*, or *NULL* on failure. The tuple values + are initialized to the subsequent *n* C arguments pointing to Python objects. + ``PyTuple_Pack(2, a, b)`` is equivalent to ``Py_BuildValue("(OO)", a, b)``. + + +.. c:function:: Py_ssize_t PyTuple_Size(PyObject *p) + + Take a pointer to a tuple object, and return the size of that tuple. + + +.. c:function:: Py_ssize_t PyTuple_GET_SIZE(PyObject *p) + + Return the size of the tuple *p*, which must be non-*NULL* and point to a tuple; + no error checking is performed. + + +.. c:function:: PyObject* PyTuple_GetItem(PyObject *p, Py_ssize_t pos) + + Return the object at position *pos* in the tuple pointed to by *p*. If *pos* is + out of bounds, return *NULL* and sets an :exc:`IndexError` exception. + + +.. c:function:: PyObject* PyTuple_GET_ITEM(PyObject *p, Py_ssize_t pos) + + Like :c:func:`PyTuple_GetItem`, but does no checking of its arguments. + + +.. c:function:: PyObject* PyTuple_GetSlice(PyObject *p, Py_ssize_t low, Py_ssize_t high) + + Take a slice of the tuple pointed to by *p* from *low* to *high* and return it + as a new tuple. + + +.. c:function:: int PyTuple_SetItem(PyObject *p, Py_ssize_t pos, PyObject *o) + + Insert a reference to object *o* at position *pos* of the tuple pointed to by + *p*. Return ``0`` on success. + + .. note:: + + This function "steals" a reference to *o*. + + +.. c:function:: void PyTuple_SET_ITEM(PyObject *p, Py_ssize_t pos, PyObject *o) + + Like :c:func:`PyTuple_SetItem`, but does no error checking, and should *only* be + used to fill in brand new tuples. + + .. note:: + + This function "steals" a reference to *o*. + + +.. c:function:: int _PyTuple_Resize(PyObject **p, Py_ssize_t newsize) + + Can be used to resize a tuple. *newsize* will be the new length of the tuple. + Because tuples are *supposed* to be immutable, this should only be used if there + is only one reference to the object. Do *not* use this if the tuple may already + be known to some other part of the code. The tuple will always grow or shrink + at the end. Think of this as destroying the old tuple and creating a new one, + only more efficiently. Returns ``0`` on success. Client code should never + assume that the resulting value of ``*p`` will be the same as before calling + this function. If the object referenced by ``*p`` is replaced, the original + ``*p`` is destroyed. On failure, returns ``-1`` and sets ``*p`` to *NULL*, and + raises :exc:`MemoryError` or :exc:`SystemError`. + + +.. c:function:: int PyTuple_ClearFreeList() + + Clear the free list. Return the total number of freed items. diff --git a/lib/cpython-doc/c-api/type.rst b/lib/cpython-doc/c-api/type.rst new file mode 100644 --- /dev/null +++ b/lib/cpython-doc/c-api/type.rst @@ -0,0 +1,86 @@ +.. highlightlang:: c + +.. _typeobjects: + +Type Objects +------------ + +.. index:: object: type + + +.. c:type:: PyTypeObject + + The C structure of the objects used to describe built-in types. + + +.. c:var:: PyObject* PyType_Type + + This is the type object for type objects; it is the same object as + :class:`type` in the Python layer. + + +.. c:function:: int PyType_Check(PyObject *o) + + Return true if the object *o* is a type object, including instances of types + derived from the standard type object. Return false in all other cases. + + +.. c:function:: int PyType_CheckExact(PyObject *o) + + Return true if the object *o* is a type object, but not a subtype of the + standard type object. Return false in all other cases. + + +.. c:function:: unsigned int PyType_ClearCache() + + Clear the internal lookup cache. Return the current version tag. + +.. c:function:: long PyType_GetFlags(PyTypeObject* type) + + Return the :attr:`tp_flags` member of *type*. This function is primarily + meant for use with `Py_LIMITED_API`; the individual flag bits are + guaranteed to be stable across Python releases, but access to + :attr:`tp_flags` itself is not part of the limited API. + + .. versionadded:: 3.2 + +.. c:function:: void PyType_Modified(PyTypeObject *type) + + Invalidate the internal lookup cache for the type and all of its + subtypes. This function must be called after any manual + modification of the attributes or base classes of the type. + + +.. c:function:: int PyType_HasFeature(PyObject *o, int feature) + + Return true if the type object *o* sets the feature *feature*. Type features + are denoted by single bit flags. + + +.. c:function:: int PyType_IS_GC(PyObject *o) + + Return true if the type object includes support for the cycle detector; this + tests the type flag :const:`Py_TPFLAGS_HAVE_GC`. + + +.. c:function:: int PyType_IsSubtype(PyTypeObject *a, PyTypeObject *b) + + Return true if *a* is a subtype of *b*. + + +.. c:function:: PyObject* PyType_GenericAlloc(PyTypeObject *type, Py_ssize_t nitems) + + XXX: Document. + + +.. c:function:: PyObject* PyType_GenericNew(PyTypeObject *type, PyObject *args, PyObject *kwds) + + XXX: Document. + + +.. c:function:: int PyType_Ready(PyTypeObject *type) + + Finalize a type object. This should be called on all type objects to finish + their initialization. This function is responsible for adding inherited slots + from a type's base class. Return ``0`` on success, or return ``-1`` and sets an + exception on error. diff --git a/lib/cpython-doc/c-api/typeobj.rst b/lib/cpython-doc/c-api/typeobj.rst new file mode 100644 From noreply at buildbot.pypy.org Tue Jan 17 17:21:53 2012 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 17 Jan 2012 17:21:53 +0100 (CET) Subject: [pypy-commit] benchmarks default: merge Message-ID: <20120117162153.39E61710260@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r158:77aa6fd8e4ec Date: 2012-01-17 18:08 +0200 http://bitbucket.org/pypy/benchmarks/changeset/77aa6fd8e4ec/ Log: merge diff --git a/own/bm_gzip.py b/own/bm_gzip.py new file mode 100644 --- /dev/null +++ b/own/bm_gzip.py @@ -0,0 +1,45 @@ +import os +import time +import shutil +import tempfile +import tarfile + +DJANGO_DIR = os.path.join(os.path.dirname(__file__), os.pardir, + 'unladen_swallow', 'lib', 'django') + +def _bootstrap(): + fd, archive = tempfile.mkstemp() + os.close(fd) + with tarfile.open(archive, 'w:gz') as targz: + targz.add(DJANGO_DIR) + return archive + +def bench(archive): + dest = tempfile.mkdtemp() + try: + with tarfile.open(archive) as targz: + targz.extractall(dest) + finally: + shutil.rmtree(dest) + +def main(n): + archive = _bootstrap() + try: + times = [] + for k in range(n): + t0 = time.time() + bench(archive) + times.append(time.time() - t0) + return times + finally: + os.remove(archive) + +if __name__ == '__main__': + import util, optparse + parser = optparse.OptionParser( + usage="%prog [options]", + description="Test the performance of the GZip decompression benchmark") + util.add_standard_options_to(parser) + options, args = parser.parse_args() + + util.run_benchmark(options, options.num_runs, main) diff --git a/runner.py b/runner.py --- a/runner.py +++ b/runner.py @@ -80,7 +80,7 @@ default=','.join(BENCHMARK_SET), help=("Comma-separated list of benchmarks to run" " Valid benchmarks are: " + - ", ".join(BENCHMARK_SET))) + ", ".join(sorted(BENCHMARK_SET)))) parser.add_option('-p', '--pypy-c', default=sys.executable, help='pypy-c or other modified python to run against') parser.add_option('-r', '--revision', default=0, action="store", From noreply at buildbot.pypy.org Tue Jan 17 17:28:24 2012 From: noreply at buildbot.pypy.org (antocuni) Date: Tue, 17 Jan 2012 17:28:24 +0100 (CET) Subject: [pypy-commit] pypy py3k: bah, if we inherit we run all the test twice Message-ID: <20120117162824.BF011710260@wyvern.cs.uni-duesseldorf.de> Author: Antonio Cuni Branch: py3k Changeset: r51408:46e3b0574cc8 Date: 2012-01-17 17:28 +0100 http://bitbucket.org/pypy/pypy/changeset/46e3b0574cc8/ Log: bah, if we inherit we run all the test twice diff --git a/pypy/interpreter/pyparser/test/test_pyparse.py b/pypy/interpreter/pyparser/test/test_pyparse.py --- a/pypy/interpreter/pyparser/test/test_pyparse.py +++ b/pypy/interpreter/pyparser/test/test_pyparse.py @@ -128,11 +128,16 @@ self.parse('(a, *rest, b) = 1, 2, 3, 4, 5') -class TestPythonParserWithSpace(TestPythonParserWithoutSpace): +class TestPythonParserWithSpace: def setup_class(self): self.parser = pyparse.PythonParser(self.space) + def parse(self, source, mode="exec", info=None): + if info is None: + info = pyparse.CompileInfo("", mode) + return self.parser.parse_source(source, info) + def test_encoding(self): info = pyparse.CompileInfo("", "exec") tree = self.parse("""# coding: latin-1 From noreply at buildbot.pypy.org Tue Jan 17 17:30:36 2012 From: noreply at buildbot.pypy.org (rguillebert) Date: Tue, 17 Jan 2012 17:30:36 +0100 (CET) Subject: [pypy-commit] pypy py3k: (antocuni, romain) port the py3k grammar for the new tuple unpacking (with parenthesis) Message-ID: <20120117163036.B5F58710260@wyvern.cs.uni-duesseldorf.de> Author: Romain Guillebert Branch: py3k Changeset: r51409:e110ee00c88b Date: 2012-01-17 17:30 +0100 http://bitbucket.org/pypy/pypy/changeset/e110ee00c88b/ Log: (antocuni, romain) port the py3k grammar for the new tuple unpacking (with parenthesis) diff --git a/pypy/interpreter/pyparser/data/Grammar3.2 b/pypy/interpreter/pyparser/data/Grammar3.2 --- a/pypy/interpreter/pyparser/data/Grammar3.2 +++ b/pypy/interpreter/pyparser/data/Grammar3.2 @@ -104,11 +104,10 @@ factor: ('+'|'-'|'~') factor | power power: atom trailer* ['**' factor] atom: ('(' [yield_expr|testlist_comp] ')' | - '[' [listmaker] ']' | + '[' [testlist_comp] ']' | '{' [dictorsetmaker] '}' | NAME | NUMBER | STRING+) -listmaker: test ( list_for | (',' test)* [','] ) -testlist_comp: test ( comp_for | (',' test)* [','] ) +testlist_comp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] ) lambdef: 'lambda' [varargslist] ':' test trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME subscriptlist: subscript (',' subscript)* [','] From noreply at buildbot.pypy.org Tue Jan 17 17:37:52 2012 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 17 Jan 2012 17:37:52 +0100 (CET) Subject: [pypy-commit] benchmarks default: fix the benchmark Message-ID: <20120117163752.DC12D820D8@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r159:c01ab59c2a28 Date: 2012-01-17 17:37 +0100 http://bitbucket.org/pypy/benchmarks/changeset/c01ab59c2a28/ Log: fix the benchmark diff --git a/.hgignore b/.hgignore --- a/.hgignore +++ b/.hgignore @@ -1,2 +1,3 @@ .*\.py[co] -.*~ \ No newline at end of file +.*~ +lib/cpython-doc/tools/build diff --git a/benchmarks.py b/benchmarks.py --- a/benchmarks.py +++ b/benchmarks.py @@ -161,14 +161,14 @@ htmldir = os.path.join(builddir, 'html') os.mkdir(htmldir) args = base_python + [build, '-b', 'html', '-d', docdir, maindir, htmldir] - proc = subprocess.Popen(args, stderr=subprocess.PIPE) + proc = subprocess.Popen(args, stderr=subprocess.PIPE, stdout=subprocess.PIPE) out, err = proc.communicate() retcode = proc.poll() if retcode != 0: print out print err raise Exception("sphinx-build.py failed") - t.append(float(out.splitlines(-1).strip())) + t.append(float(out.splitlines()[-1])) return RawResult([t[0]], [t[1]]) BM_cpython_doc.benchmark_name = 'sphinx' diff --git a/lib/cpython-doc/tools/build/doctrees/about.doctree b/lib/cpython-doc/tools/build/doctrees/about.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/about.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/bugs.doctree b/lib/cpython-doc/tools/build/doctrees/bugs.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/bugs.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/c-api/abstract.doctree b/lib/cpython-doc/tools/build/doctrees/c-api/abstract.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/c-api/abstract.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/c-api/allocation.doctree b/lib/cpython-doc/tools/build/doctrees/c-api/allocation.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/c-api/allocation.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/c-api/arg.doctree b/lib/cpython-doc/tools/build/doctrees/c-api/arg.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/c-api/arg.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/c-api/bool.doctree b/lib/cpython-doc/tools/build/doctrees/c-api/bool.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/c-api/bool.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/c-api/buffer.doctree b/lib/cpython-doc/tools/build/doctrees/c-api/buffer.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/c-api/buffer.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/c-api/bytearray.doctree b/lib/cpython-doc/tools/build/doctrees/c-api/bytearray.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/c-api/bytearray.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/c-api/bytes.doctree b/lib/cpython-doc/tools/build/doctrees/c-api/bytes.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/c-api/bytes.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/c-api/capsule.doctree b/lib/cpython-doc/tools/build/doctrees/c-api/capsule.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/c-api/capsule.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/c-api/cell.doctree b/lib/cpython-doc/tools/build/doctrees/c-api/cell.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/c-api/cell.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/c-api/code.doctree b/lib/cpython-doc/tools/build/doctrees/c-api/code.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/c-api/code.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/c-api/codec.doctree b/lib/cpython-doc/tools/build/doctrees/c-api/codec.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/c-api/codec.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/c-api/complex.doctree b/lib/cpython-doc/tools/build/doctrees/c-api/complex.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/c-api/complex.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/c-api/concrete.doctree b/lib/cpython-doc/tools/build/doctrees/c-api/concrete.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/c-api/concrete.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/c-api/conversion.doctree b/lib/cpython-doc/tools/build/doctrees/c-api/conversion.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/c-api/conversion.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/c-api/datetime.doctree b/lib/cpython-doc/tools/build/doctrees/c-api/datetime.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/c-api/datetime.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/c-api/descriptor.doctree b/lib/cpython-doc/tools/build/doctrees/c-api/descriptor.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/c-api/descriptor.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/c-api/dict.doctree b/lib/cpython-doc/tools/build/doctrees/c-api/dict.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/c-api/dict.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/c-api/exceptions.doctree b/lib/cpython-doc/tools/build/doctrees/c-api/exceptions.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/c-api/exceptions.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/c-api/file.doctree b/lib/cpython-doc/tools/build/doctrees/c-api/file.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/c-api/file.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/c-api/float.doctree b/lib/cpython-doc/tools/build/doctrees/c-api/float.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/c-api/float.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/c-api/function.doctree b/lib/cpython-doc/tools/build/doctrees/c-api/function.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/c-api/function.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/c-api/gcsupport.doctree b/lib/cpython-doc/tools/build/doctrees/c-api/gcsupport.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/c-api/gcsupport.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/c-api/gen.doctree b/lib/cpython-doc/tools/build/doctrees/c-api/gen.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/c-api/gen.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/c-api/import.doctree b/lib/cpython-doc/tools/build/doctrees/c-api/import.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/c-api/import.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/c-api/index.doctree b/lib/cpython-doc/tools/build/doctrees/c-api/index.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/c-api/index.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/c-api/init.doctree b/lib/cpython-doc/tools/build/doctrees/c-api/init.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/c-api/init.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/c-api/intro.doctree b/lib/cpython-doc/tools/build/doctrees/c-api/intro.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/c-api/intro.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/c-api/iter.doctree b/lib/cpython-doc/tools/build/doctrees/c-api/iter.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/c-api/iter.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/c-api/iterator.doctree b/lib/cpython-doc/tools/build/doctrees/c-api/iterator.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/c-api/iterator.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/c-api/list.doctree b/lib/cpython-doc/tools/build/doctrees/c-api/list.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/c-api/list.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/c-api/long.doctree b/lib/cpython-doc/tools/build/doctrees/c-api/long.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/c-api/long.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/c-api/mapping.doctree b/lib/cpython-doc/tools/build/doctrees/c-api/mapping.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/c-api/mapping.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/c-api/marshal.doctree b/lib/cpython-doc/tools/build/doctrees/c-api/marshal.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/c-api/marshal.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/c-api/memory.doctree b/lib/cpython-doc/tools/build/doctrees/c-api/memory.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/c-api/memory.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/c-api/memoryview.doctree b/lib/cpython-doc/tools/build/doctrees/c-api/memoryview.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/c-api/memoryview.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/c-api/method.doctree b/lib/cpython-doc/tools/build/doctrees/c-api/method.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/c-api/method.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/c-api/module.doctree b/lib/cpython-doc/tools/build/doctrees/c-api/module.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/c-api/module.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/c-api/none.doctree b/lib/cpython-doc/tools/build/doctrees/c-api/none.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/c-api/none.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/c-api/number.doctree b/lib/cpython-doc/tools/build/doctrees/c-api/number.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/c-api/number.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/c-api/objbuffer.doctree b/lib/cpython-doc/tools/build/doctrees/c-api/objbuffer.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/c-api/objbuffer.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/c-api/object.doctree b/lib/cpython-doc/tools/build/doctrees/c-api/object.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/c-api/object.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/c-api/objimpl.doctree b/lib/cpython-doc/tools/build/doctrees/c-api/objimpl.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/c-api/objimpl.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/c-api/refcounting.doctree b/lib/cpython-doc/tools/build/doctrees/c-api/refcounting.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/c-api/refcounting.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/c-api/reflection.doctree b/lib/cpython-doc/tools/build/doctrees/c-api/reflection.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/c-api/reflection.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/c-api/sequence.doctree b/lib/cpython-doc/tools/build/doctrees/c-api/sequence.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/c-api/sequence.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/c-api/set.doctree b/lib/cpython-doc/tools/build/doctrees/c-api/set.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/c-api/set.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/c-api/slice.doctree b/lib/cpython-doc/tools/build/doctrees/c-api/slice.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/c-api/slice.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/c-api/structures.doctree b/lib/cpython-doc/tools/build/doctrees/c-api/structures.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/c-api/structures.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/c-api/sys.doctree b/lib/cpython-doc/tools/build/doctrees/c-api/sys.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/c-api/sys.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/c-api/tuple.doctree b/lib/cpython-doc/tools/build/doctrees/c-api/tuple.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/c-api/tuple.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/c-api/type.doctree b/lib/cpython-doc/tools/build/doctrees/c-api/type.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/c-api/type.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/c-api/typeobj.doctree b/lib/cpython-doc/tools/build/doctrees/c-api/typeobj.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/c-api/typeobj.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/c-api/unicode.doctree b/lib/cpython-doc/tools/build/doctrees/c-api/unicode.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/c-api/unicode.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/c-api/utilities.doctree b/lib/cpython-doc/tools/build/doctrees/c-api/utilities.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/c-api/utilities.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/c-api/veryhigh.doctree b/lib/cpython-doc/tools/build/doctrees/c-api/veryhigh.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/c-api/veryhigh.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/c-api/weakref.doctree b/lib/cpython-doc/tools/build/doctrees/c-api/weakref.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/c-api/weakref.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/contents.doctree b/lib/cpython-doc/tools/build/doctrees/contents.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/contents.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/copyright.doctree b/lib/cpython-doc/tools/build/doctrees/copyright.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/copyright.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/distutils/apiref.doctree b/lib/cpython-doc/tools/build/doctrees/distutils/apiref.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/distutils/apiref.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/distutils/builtdist.doctree b/lib/cpython-doc/tools/build/doctrees/distutils/builtdist.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/distutils/builtdist.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/distutils/commandref.doctree b/lib/cpython-doc/tools/build/doctrees/distutils/commandref.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/distutils/commandref.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/distutils/configfile.doctree b/lib/cpython-doc/tools/build/doctrees/distutils/configfile.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/distutils/configfile.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/distutils/examples.doctree b/lib/cpython-doc/tools/build/doctrees/distutils/examples.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/distutils/examples.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/distutils/extending.doctree b/lib/cpython-doc/tools/build/doctrees/distutils/extending.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/distutils/extending.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/distutils/index.doctree b/lib/cpython-doc/tools/build/doctrees/distutils/index.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/distutils/index.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/distutils/install.doctree b/lib/cpython-doc/tools/build/doctrees/distutils/install.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/distutils/install.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/distutils/introduction.doctree b/lib/cpython-doc/tools/build/doctrees/distutils/introduction.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/distutils/introduction.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/distutils/packageindex.doctree b/lib/cpython-doc/tools/build/doctrees/distutils/packageindex.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/distutils/packageindex.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/distutils/setupscript.doctree b/lib/cpython-doc/tools/build/doctrees/distutils/setupscript.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/distutils/setupscript.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/distutils/sourcedist.doctree b/lib/cpython-doc/tools/build/doctrees/distutils/sourcedist.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/distutils/sourcedist.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/distutils/uploading.doctree b/lib/cpython-doc/tools/build/doctrees/distutils/uploading.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/distutils/uploading.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/extending/building.doctree b/lib/cpython-doc/tools/build/doctrees/extending/building.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/extending/building.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/extending/embedding.doctree b/lib/cpython-doc/tools/build/doctrees/extending/embedding.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/extending/embedding.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/extending/extending.doctree b/lib/cpython-doc/tools/build/doctrees/extending/extending.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/extending/extending.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/extending/index.doctree b/lib/cpython-doc/tools/build/doctrees/extending/index.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/extending/index.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/extending/newtypes.doctree b/lib/cpython-doc/tools/build/doctrees/extending/newtypes.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/extending/newtypes.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/extending/windows.doctree b/lib/cpython-doc/tools/build/doctrees/extending/windows.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/extending/windows.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/faq/design.doctree b/lib/cpython-doc/tools/build/doctrees/faq/design.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/faq/design.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/faq/extending.doctree b/lib/cpython-doc/tools/build/doctrees/faq/extending.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/faq/extending.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/faq/general.doctree b/lib/cpython-doc/tools/build/doctrees/faq/general.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/faq/general.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/faq/gui.doctree b/lib/cpython-doc/tools/build/doctrees/faq/gui.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/faq/gui.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/faq/index.doctree b/lib/cpython-doc/tools/build/doctrees/faq/index.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/faq/index.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/faq/installed.doctree b/lib/cpython-doc/tools/build/doctrees/faq/installed.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/faq/installed.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/faq/library.doctree b/lib/cpython-doc/tools/build/doctrees/faq/library.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/faq/library.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/faq/programming.doctree b/lib/cpython-doc/tools/build/doctrees/faq/programming.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/faq/programming.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/faq/windows.doctree b/lib/cpython-doc/tools/build/doctrees/faq/windows.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/faq/windows.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/glossary.doctree b/lib/cpython-doc/tools/build/doctrees/glossary.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/glossary.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/howto/advocacy.doctree b/lib/cpython-doc/tools/build/doctrees/howto/advocacy.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/howto/advocacy.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/howto/cporting.doctree b/lib/cpython-doc/tools/build/doctrees/howto/cporting.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/howto/cporting.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/howto/curses.doctree b/lib/cpython-doc/tools/build/doctrees/howto/curses.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/howto/curses.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/howto/descriptor.doctree b/lib/cpython-doc/tools/build/doctrees/howto/descriptor.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/howto/descriptor.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/howto/functional.doctree b/lib/cpython-doc/tools/build/doctrees/howto/functional.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/howto/functional.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/howto/index.doctree b/lib/cpython-doc/tools/build/doctrees/howto/index.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/howto/index.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/howto/logging-cookbook.doctree b/lib/cpython-doc/tools/build/doctrees/howto/logging-cookbook.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/howto/logging-cookbook.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/howto/logging.doctree b/lib/cpython-doc/tools/build/doctrees/howto/logging.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/howto/logging.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/howto/pyporting.doctree b/lib/cpython-doc/tools/build/doctrees/howto/pyporting.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/howto/pyporting.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/howto/regex.doctree b/lib/cpython-doc/tools/build/doctrees/howto/regex.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/howto/regex.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/howto/sockets.doctree b/lib/cpython-doc/tools/build/doctrees/howto/sockets.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/howto/sockets.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/howto/sorting.doctree b/lib/cpython-doc/tools/build/doctrees/howto/sorting.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/howto/sorting.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/howto/unicode.doctree b/lib/cpython-doc/tools/build/doctrees/howto/unicode.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/howto/unicode.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/howto/urllib2.doctree b/lib/cpython-doc/tools/build/doctrees/howto/urllib2.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/howto/urllib2.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/howto/webservers.doctree b/lib/cpython-doc/tools/build/doctrees/howto/webservers.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/howto/webservers.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/install/index.doctree b/lib/cpython-doc/tools/build/doctrees/install/index.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/install/index.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/install/install.doctree b/lib/cpython-doc/tools/build/doctrees/install/install.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/install/install.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/install/pysetup-config.doctree b/lib/cpython-doc/tools/build/doctrees/install/pysetup-config.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/install/pysetup-config.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/install/pysetup-servers.doctree b/lib/cpython-doc/tools/build/doctrees/install/pysetup-servers.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/install/pysetup-servers.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/install/pysetup.doctree b/lib/cpython-doc/tools/build/doctrees/install/pysetup.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/install/pysetup.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/2to3.doctree b/lib/cpython-doc/tools/build/doctrees/library/2to3.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/2to3.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/__future__.doctree b/lib/cpython-doc/tools/build/doctrees/library/__future__.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/__future__.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/__main__.doctree b/lib/cpython-doc/tools/build/doctrees/library/__main__.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/__main__.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/_dummy_thread.doctree b/lib/cpython-doc/tools/build/doctrees/library/_dummy_thread.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/_dummy_thread.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/_thread.doctree b/lib/cpython-doc/tools/build/doctrees/library/_thread.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/_thread.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/abc.doctree b/lib/cpython-doc/tools/build/doctrees/library/abc.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/abc.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/aifc.doctree b/lib/cpython-doc/tools/build/doctrees/library/aifc.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/aifc.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/allos.doctree b/lib/cpython-doc/tools/build/doctrees/library/allos.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/allos.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/archiving.doctree b/lib/cpython-doc/tools/build/doctrees/library/archiving.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/archiving.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/argparse.doctree b/lib/cpython-doc/tools/build/doctrees/library/argparse.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/argparse.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/array.doctree b/lib/cpython-doc/tools/build/doctrees/library/array.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/array.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/ast.doctree b/lib/cpython-doc/tools/build/doctrees/library/ast.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/ast.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/asynchat.doctree b/lib/cpython-doc/tools/build/doctrees/library/asynchat.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/asynchat.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/asyncore.doctree b/lib/cpython-doc/tools/build/doctrees/library/asyncore.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/asyncore.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/atexit.doctree b/lib/cpython-doc/tools/build/doctrees/library/atexit.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/atexit.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/audioop.doctree b/lib/cpython-doc/tools/build/doctrees/library/audioop.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/audioop.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/base64.doctree b/lib/cpython-doc/tools/build/doctrees/library/base64.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/base64.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/bdb.doctree b/lib/cpython-doc/tools/build/doctrees/library/bdb.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/bdb.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/binascii.doctree b/lib/cpython-doc/tools/build/doctrees/library/binascii.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/binascii.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/binhex.doctree b/lib/cpython-doc/tools/build/doctrees/library/binhex.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/binhex.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/bisect.doctree b/lib/cpython-doc/tools/build/doctrees/library/bisect.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/bisect.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/builtins.doctree b/lib/cpython-doc/tools/build/doctrees/library/builtins.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/builtins.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/bz2.doctree b/lib/cpython-doc/tools/build/doctrees/library/bz2.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/bz2.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/calendar.doctree b/lib/cpython-doc/tools/build/doctrees/library/calendar.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/calendar.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/cgi.doctree b/lib/cpython-doc/tools/build/doctrees/library/cgi.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/cgi.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/cgitb.doctree b/lib/cpython-doc/tools/build/doctrees/library/cgitb.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/cgitb.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/chunk.doctree b/lib/cpython-doc/tools/build/doctrees/library/chunk.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/chunk.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/cmath.doctree b/lib/cpython-doc/tools/build/doctrees/library/cmath.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/cmath.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/cmd.doctree b/lib/cpython-doc/tools/build/doctrees/library/cmd.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/cmd.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/code.doctree b/lib/cpython-doc/tools/build/doctrees/library/code.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/code.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/codecs.doctree b/lib/cpython-doc/tools/build/doctrees/library/codecs.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/codecs.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/codeop.doctree b/lib/cpython-doc/tools/build/doctrees/library/codeop.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/codeop.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/collections.abc.doctree b/lib/cpython-doc/tools/build/doctrees/library/collections.abc.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/collections.abc.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/collections.doctree b/lib/cpython-doc/tools/build/doctrees/library/collections.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/collections.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/colorsys.doctree b/lib/cpython-doc/tools/build/doctrees/library/colorsys.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/colorsys.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/compileall.doctree b/lib/cpython-doc/tools/build/doctrees/library/compileall.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/compileall.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/concurrent.futures.doctree b/lib/cpython-doc/tools/build/doctrees/library/concurrent.futures.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/concurrent.futures.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/configparser.doctree b/lib/cpython-doc/tools/build/doctrees/library/configparser.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/configparser.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/constants.doctree b/lib/cpython-doc/tools/build/doctrees/library/constants.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/constants.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/contextlib.doctree b/lib/cpython-doc/tools/build/doctrees/library/contextlib.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/contextlib.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/copy.doctree b/lib/cpython-doc/tools/build/doctrees/library/copy.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/copy.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/copyreg.doctree b/lib/cpython-doc/tools/build/doctrees/library/copyreg.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/copyreg.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/crypt.doctree b/lib/cpython-doc/tools/build/doctrees/library/crypt.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/crypt.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/crypto.doctree b/lib/cpython-doc/tools/build/doctrees/library/crypto.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/crypto.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/csv.doctree b/lib/cpython-doc/tools/build/doctrees/library/csv.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/csv.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/ctypes.doctree b/lib/cpython-doc/tools/build/doctrees/library/ctypes.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/ctypes.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/curses.ascii.doctree b/lib/cpython-doc/tools/build/doctrees/library/curses.ascii.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/curses.ascii.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/curses.doctree b/lib/cpython-doc/tools/build/doctrees/library/curses.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/curses.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/curses.panel.doctree b/lib/cpython-doc/tools/build/doctrees/library/curses.panel.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/curses.panel.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/custominterp.doctree b/lib/cpython-doc/tools/build/doctrees/library/custominterp.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/custominterp.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/datatypes.doctree b/lib/cpython-doc/tools/build/doctrees/library/datatypes.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/datatypes.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/datetime.doctree b/lib/cpython-doc/tools/build/doctrees/library/datetime.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/datetime.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/dbm.doctree b/lib/cpython-doc/tools/build/doctrees/library/dbm.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/dbm.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/debug.doctree b/lib/cpython-doc/tools/build/doctrees/library/debug.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/debug.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/decimal.doctree b/lib/cpython-doc/tools/build/doctrees/library/decimal.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/decimal.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/development.doctree b/lib/cpython-doc/tools/build/doctrees/library/development.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/development.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/difflib.doctree b/lib/cpython-doc/tools/build/doctrees/library/difflib.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/difflib.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/dis.doctree b/lib/cpython-doc/tools/build/doctrees/library/dis.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/dis.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/distutils.doctree b/lib/cpython-doc/tools/build/doctrees/library/distutils.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/distutils.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/doctest.doctree b/lib/cpython-doc/tools/build/doctrees/library/doctest.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/doctest.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/dummy_threading.doctree b/lib/cpython-doc/tools/build/doctrees/library/dummy_threading.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/dummy_threading.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/email-examples.doctree b/lib/cpython-doc/tools/build/doctrees/library/email-examples.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/email-examples.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/email.charset.doctree b/lib/cpython-doc/tools/build/doctrees/library/email.charset.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/email.charset.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/email.doctree b/lib/cpython-doc/tools/build/doctrees/library/email.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/email.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/email.encoders.doctree b/lib/cpython-doc/tools/build/doctrees/library/email.encoders.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/email.encoders.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/email.errors.doctree b/lib/cpython-doc/tools/build/doctrees/library/email.errors.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/email.errors.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/email.generator.doctree b/lib/cpython-doc/tools/build/doctrees/library/email.generator.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/email.generator.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/email.header.doctree b/lib/cpython-doc/tools/build/doctrees/library/email.header.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/email.header.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/email.iterators.doctree b/lib/cpython-doc/tools/build/doctrees/library/email.iterators.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/email.iterators.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/email.message.doctree b/lib/cpython-doc/tools/build/doctrees/library/email.message.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/email.message.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/email.mime.doctree b/lib/cpython-doc/tools/build/doctrees/library/email.mime.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/email.mime.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/email.parser.doctree b/lib/cpython-doc/tools/build/doctrees/library/email.parser.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/email.parser.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/email.policy.doctree b/lib/cpython-doc/tools/build/doctrees/library/email.policy.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/email.policy.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/email.util.doctree b/lib/cpython-doc/tools/build/doctrees/library/email.util.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/email.util.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/errno.doctree b/lib/cpython-doc/tools/build/doctrees/library/errno.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/errno.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/exceptions.doctree b/lib/cpython-doc/tools/build/doctrees/library/exceptions.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/exceptions.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/faulthandler.doctree b/lib/cpython-doc/tools/build/doctrees/library/faulthandler.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/faulthandler.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/fcntl.doctree b/lib/cpython-doc/tools/build/doctrees/library/fcntl.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/fcntl.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/filecmp.doctree b/lib/cpython-doc/tools/build/doctrees/library/filecmp.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/filecmp.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/fileformats.doctree b/lib/cpython-doc/tools/build/doctrees/library/fileformats.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/fileformats.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/fileinput.doctree b/lib/cpython-doc/tools/build/doctrees/library/fileinput.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/fileinput.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/filesys.doctree b/lib/cpython-doc/tools/build/doctrees/library/filesys.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/filesys.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/fnmatch.doctree b/lib/cpython-doc/tools/build/doctrees/library/fnmatch.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/fnmatch.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/formatter.doctree b/lib/cpython-doc/tools/build/doctrees/library/formatter.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/formatter.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/fpectl.doctree b/lib/cpython-doc/tools/build/doctrees/library/fpectl.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/fpectl.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/fractions.doctree b/lib/cpython-doc/tools/build/doctrees/library/fractions.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/fractions.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/frameworks.doctree b/lib/cpython-doc/tools/build/doctrees/library/frameworks.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/frameworks.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/ftplib.doctree b/lib/cpython-doc/tools/build/doctrees/library/ftplib.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/ftplib.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/functional.doctree b/lib/cpython-doc/tools/build/doctrees/library/functional.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/functional.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/functions.doctree b/lib/cpython-doc/tools/build/doctrees/library/functions.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/functions.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/functools.doctree b/lib/cpython-doc/tools/build/doctrees/library/functools.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/functools.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/gc.doctree b/lib/cpython-doc/tools/build/doctrees/library/gc.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/gc.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/getopt.doctree b/lib/cpython-doc/tools/build/doctrees/library/getopt.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/getopt.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/getpass.doctree b/lib/cpython-doc/tools/build/doctrees/library/getpass.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/getpass.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/gettext.doctree b/lib/cpython-doc/tools/build/doctrees/library/gettext.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/gettext.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/glob.doctree b/lib/cpython-doc/tools/build/doctrees/library/glob.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/glob.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/grp.doctree b/lib/cpython-doc/tools/build/doctrees/library/grp.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/grp.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/gzip.doctree b/lib/cpython-doc/tools/build/doctrees/library/gzip.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/gzip.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/hashlib.doctree b/lib/cpython-doc/tools/build/doctrees/library/hashlib.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/hashlib.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/heapq.doctree b/lib/cpython-doc/tools/build/doctrees/library/heapq.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/heapq.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/hmac.doctree b/lib/cpython-doc/tools/build/doctrees/library/hmac.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/hmac.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/html.doctree b/lib/cpython-doc/tools/build/doctrees/library/html.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/html.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/html.entities.doctree b/lib/cpython-doc/tools/build/doctrees/library/html.entities.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/html.entities.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/html.parser.doctree b/lib/cpython-doc/tools/build/doctrees/library/html.parser.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/html.parser.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/http.client.doctree b/lib/cpython-doc/tools/build/doctrees/library/http.client.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/http.client.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/http.cookiejar.doctree b/lib/cpython-doc/tools/build/doctrees/library/http.cookiejar.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/http.cookiejar.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/http.cookies.doctree b/lib/cpython-doc/tools/build/doctrees/library/http.cookies.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/http.cookies.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/http.server.doctree b/lib/cpython-doc/tools/build/doctrees/library/http.server.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/http.server.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/i18n.doctree b/lib/cpython-doc/tools/build/doctrees/library/i18n.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/i18n.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/idle.doctree b/lib/cpython-doc/tools/build/doctrees/library/idle.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/idle.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/imaplib.doctree b/lib/cpython-doc/tools/build/doctrees/library/imaplib.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/imaplib.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/imghdr.doctree b/lib/cpython-doc/tools/build/doctrees/library/imghdr.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/imghdr.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/imp.doctree b/lib/cpython-doc/tools/build/doctrees/library/imp.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/imp.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/importlib.doctree b/lib/cpython-doc/tools/build/doctrees/library/importlib.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/importlib.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/index.doctree b/lib/cpython-doc/tools/build/doctrees/library/index.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/index.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/inspect.doctree b/lib/cpython-doc/tools/build/doctrees/library/inspect.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/inspect.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/internet.doctree b/lib/cpython-doc/tools/build/doctrees/library/internet.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/internet.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/intro.doctree b/lib/cpython-doc/tools/build/doctrees/library/intro.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/intro.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/io.doctree b/lib/cpython-doc/tools/build/doctrees/library/io.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/io.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/ipc.doctree b/lib/cpython-doc/tools/build/doctrees/library/ipc.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/ipc.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/itertools.doctree b/lib/cpython-doc/tools/build/doctrees/library/itertools.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/itertools.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/json.doctree b/lib/cpython-doc/tools/build/doctrees/library/json.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/json.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/keyword.doctree b/lib/cpython-doc/tools/build/doctrees/library/keyword.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/keyword.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/language.doctree b/lib/cpython-doc/tools/build/doctrees/library/language.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/language.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/linecache.doctree b/lib/cpython-doc/tools/build/doctrees/library/linecache.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/linecache.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/locale.doctree b/lib/cpython-doc/tools/build/doctrees/library/locale.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/locale.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/logging.config.doctree b/lib/cpython-doc/tools/build/doctrees/library/logging.config.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/logging.config.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/logging.doctree b/lib/cpython-doc/tools/build/doctrees/library/logging.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/logging.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/logging.handlers.doctree b/lib/cpython-doc/tools/build/doctrees/library/logging.handlers.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/logging.handlers.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/lzma.doctree b/lib/cpython-doc/tools/build/doctrees/library/lzma.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/lzma.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/macpath.doctree b/lib/cpython-doc/tools/build/doctrees/library/macpath.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/macpath.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/mailbox.doctree b/lib/cpython-doc/tools/build/doctrees/library/mailbox.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/mailbox.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/mailcap.doctree b/lib/cpython-doc/tools/build/doctrees/library/mailcap.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/mailcap.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/markup.doctree b/lib/cpython-doc/tools/build/doctrees/library/markup.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/markup.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/marshal.doctree b/lib/cpython-doc/tools/build/doctrees/library/marshal.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/marshal.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/math.doctree b/lib/cpython-doc/tools/build/doctrees/library/math.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/math.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/mimetypes.doctree b/lib/cpython-doc/tools/build/doctrees/library/mimetypes.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/mimetypes.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/misc.doctree b/lib/cpython-doc/tools/build/doctrees/library/misc.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/misc.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/mm.doctree b/lib/cpython-doc/tools/build/doctrees/library/mm.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/mm.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/mmap.doctree b/lib/cpython-doc/tools/build/doctrees/library/mmap.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/mmap.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/modulefinder.doctree b/lib/cpython-doc/tools/build/doctrees/library/modulefinder.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/modulefinder.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/modules.doctree b/lib/cpython-doc/tools/build/doctrees/library/modules.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/modules.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/msilib.doctree b/lib/cpython-doc/tools/build/doctrees/library/msilib.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/msilib.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/msvcrt.doctree b/lib/cpython-doc/tools/build/doctrees/library/msvcrt.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/msvcrt.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/multiprocessing.doctree b/lib/cpython-doc/tools/build/doctrees/library/multiprocessing.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/multiprocessing.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/netdata.doctree b/lib/cpython-doc/tools/build/doctrees/library/netdata.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/netdata.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/netrc.doctree b/lib/cpython-doc/tools/build/doctrees/library/netrc.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/netrc.doctree has changed diff --git a/lib/cpython-doc/tools/build/doctrees/library/nis.doctree b/lib/cpython-doc/tools/build/doctrees/library/nis.doctree deleted file mode 100644 Binary file lib/cpython-doc/tools/build/doctrees/library/nis.doctree has changed From noreply at buildbot.pypy.org Tue Jan 17 18:51:41 2012 From: noreply at buildbot.pypy.org (rguillebert) Date: Tue, 17 Jan 2012 18:51:41 +0100 (CET) Subject: [pypy-commit] pypy py3k: (antocuni, romain) Add support for the new unpacking at the ast level Message-ID: <20120117175141.BBC7D820D8@wyvern.cs.uni-duesseldorf.de> Author: Romain Guillebert Branch: py3k Changeset: r51410:fbf25d09b601 Date: 2012-01-17 18:51 +0100 http://bitbucket.org/pypy/pypy/changeset/fbf25d09b601/ Log: (antocuni,romain) Add support for the new unpacking at the ast level fixed list comprehension diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py --- a/pypy/interpreter/astcompiler/ast.py +++ b/pypy/interpreter/astcompiler/ast.py @@ -1507,6 +1507,11 @@ for node in self.comparators: node.sync_app_attrs(space) +class Starred(expr): + def __init__(self, value, ctx, lineno, col_offset): + self.value = value + self.ctx = ctx + expr.__init__(self, lineno, col_offset) class Call(expr): diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py --- a/pypy/interpreter/astcompiler/astbuilder.py +++ b/pypy/interpreter/astcompiler/astbuilder.py @@ -680,7 +680,7 @@ self.set_context(target_expr, ast.Store) targets.append(target_expr) value_child = stmt.children[-1] - if value_child.type == syms.testlist or value_child.type == syms.testlist_star_expr: + if value_child.type == syms.testlist_star_expr: value_expr = self.handle_testlist(value_child) else: value_expr = self.handle_expr(value_child) @@ -740,6 +740,8 @@ operands.append(self.handle_expr(expr_node.children[i + 1])) return ast.Compare(expr, operators, operands, expr_node.lineno, expr_node.column) + elif expr_node_type == syms.star_expr: + return self.handle_star_expr(expr_node) elif expr_node_type == syms.expr or \ expr_node_type == syms.xor_expr or \ expr_node_type == syms.and_expr or \ @@ -766,6 +768,10 @@ else: raise AssertionError("unknown expr") + def handle_star_expr(self, star_expr_node): + expr = self.handle_expr(star_expr_node.children[1]) + return ast.Starred(expr, ast.Load, star_expr_node.lineno, star_expr_node.column) + def handle_lambdef(self, lambdef_node): expr = self.handle_expr(lambdef_node.children[-1]) if len(lambdef_node.children) == 3: @@ -1229,8 +1235,8 @@ elt = self.handle_expr(listcomp_node.children[0]) comps = self.comprehension_helper(listcomp_node.children[1], "handle_testlist", - syms.list_for, syms.list_if, - syms.list_iter, + syms.comp_for, syms.comp_if, + syms.comp_iter, comp_fix_unamed_tuple_location=True) return ast.ListComp(elt, comps, listcomp_node.lineno, listcomp_node.column) diff --git a/pypy/interpreter/astcompiler/asthelpers.py b/pypy/interpreter/astcompiler/asthelpers.py --- a/pypy/interpreter/astcompiler/asthelpers.py +++ b/pypy/interpreter/astcompiler/asthelpers.py @@ -133,6 +133,13 @@ _description = "comparison" +class __extend__(ast.Starred): + + _description = "starred expression" + + def set_context(self, ctx): + self.ctx = ctx + self.value.set_context(ctx) class __extend__(ast.IfExp): diff --git a/pypy/interpreter/astcompiler/test/test_astbuilder.py b/pypy/interpreter/astcompiler/test/test_astbuilder.py --- a/pypy/interpreter/astcompiler/test/test_astbuilder.py +++ b/pypy/interpreter/astcompiler/test/test_astbuilder.py @@ -600,6 +600,17 @@ assert isinstance(tup.elts[0], ast.Name) assert tup.elts[0].ctx == ast.Store + def test_assign_starred(self): + assign = self.get_first_stmt("*a, b = x") + assert isinstance(assign, ast.Assign) + assert len(assign.targets) == 1 + names = assign.targets[0] + assert len(names.elts) == 2 + assert isinstance(names.elts[0], ast.Starred) + assert isinstance(names.elts[1], ast.Name) + assert isinstance(names.elts[0].value, ast.Name) + assert names.elts[0].value.id == "a" + def test_name(self): name = self.get_first_expr("hi") assert isinstance(name, ast.Name) diff --git a/pypy/interpreter/pyparser/data/Grammar3.2 b/pypy/interpreter/pyparser/data/Grammar3.2 --- a/pypy/interpreter/pyparser/data/Grammar3.2 +++ b/pypy/interpreter/pyparser/data/Grammar3.2 @@ -127,11 +127,6 @@ # The reason that keywords are test nodes instead of NAME is that using NAME # results in an ambiguity. ast.c makes sure it's a NAME. argument: test [comp_for] | test '=' test - -list_iter: list_for | list_if -list_for: 'for' exprlist 'in' testlist_safe [list_iter] -list_if: 'if' old_test [list_iter] - comp_iter: comp_for | comp_if comp_for: 'for' exprlist 'in' or_test [comp_iter] comp_if: 'if' old_test [comp_iter] From noreply at buildbot.pypy.org Tue Jan 17 19:23:24 2012 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Tue, 17 Jan 2012 19:23:24 +0100 (CET) Subject: [pypy-commit] pypy default: move some stuff that's in fromnumeric out of the appnumpy file. Message-ID: <20120117182324.E4D7D820D8@wyvern.cs.uni-duesseldorf.de> Author: Alex Gaynor Branch: Changeset: r51411:8dea12b8117a Date: 2012-01-17 12:23 -0600 http://bitbucket.org/pypy/pypy/changeset/8dea12b8117a/ Log: move some stuff that's in fromnumeric out of the appnumpy file. diff --git a/lib_pypy/numpypy/test/test_fromnumeric.py b/lib_pypy/numpypy/test/test_fromnumeric.py --- a/lib_pypy/numpypy/test/test_fromnumeric.py +++ b/lib_pypy/numpypy/test/test_fromnumeric.py @@ -1,7 +1,7 @@ - from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest -class AppTestFromNumeric(BaseNumpyAppTest): + +class AppTestFromNumeric(BaseNumpyAppTest): def test_argmax(self): # tests taken from numpy/core/fromnumeric.py docstring from numpypy import array, arange, argmax @@ -23,7 +23,7 @@ b = arange(6) b[1] = 0 assert argmin(b) == 0 - + def test_shape(self): # tests taken from numpy/core/fromnumeric.py docstring from numpypy import array, identity, shape @@ -44,7 +44,7 @@ # assert (sum([[0, 1], [0, 5]], axis=1) == array([1, 5])).all() # If the accumulator is too small, overflow occurs: # assert ones(128, dtype=int8).sum(dtype=int8) == -128 - + def test_amin(self): # tests taken from numpy/core/fromnumeric.py docstring from numpypy import array, arange, amin @@ -86,14 +86,14 @@ assert ndim([[1,2,3],[4,5,6]]) == 2 assert ndim(array([[1,2,3],[4,5,6]])) == 2 assert ndim(1) == 0 - + def test_rank(self): # tests taken from numpy/core/fromnumeric.py docstring from numpypy import array, rank assert rank([[1,2,3],[4,5,6]]) == 2 assert rank(array([[1,2,3],[4,5,6]])) == 2 assert rank(1) == 0 - + def test_var(self): from numpypy import array, var a = array([[1,2],[3,4]]) @@ -107,3 +107,21 @@ assert std(a) == 1.1180339887498949 # assert (std(a, axis=0) == array([ 1., 1.])).all() # assert (std(a, axis=1) == array([ 0.5, 0.5]).all() + + def test_mean(self): + from numpypy import array, mean + assert mean(array(range(5))) == 2.0 + assert mean(range(5)) == 2.0 + + def test_reshape(self): + from numpypy import arange, array, dtype, reshape + a = arange(12) + b = reshape(a, (3, 4)) + assert b.shape == (3, 4) + a = range(12) + b = reshape(a, (3, 4)) + assert b.shape == (3, 4) + a = array(range(105)).reshape(3, 5, 7) + assert reshape(a, 1, -1).shape == (1, 105) + assert reshape(a, 1, 1, -1).shape == (1, 1, 105) + assert reshape(a, -1, 1, 1).shape == (105, 1, 1) diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py --- a/pypy/module/micronumpy/__init__.py +++ b/pypy/module/micronumpy/__init__.py @@ -90,7 +90,6 @@ appleveldefs = { 'average': 'app_numpy.average', - 'mean': 'app_numpy.mean', 'sum': 'app_numpy.sum', 'min': 'app_numpy.min', 'identity': 'app_numpy.identity', @@ -99,5 +98,4 @@ 'e': 'app_numpy.e', 'pi': 'app_numpy.pi', 'arange': 'app_numpy.arange', - 'reshape': 'app_numpy.reshape', } diff --git a/pypy/module/micronumpy/app_numpy.py b/pypy/module/micronumpy/app_numpy.py --- a/pypy/module/micronumpy/app_numpy.py +++ b/pypy/module/micronumpy/app_numpy.py @@ -11,23 +11,20 @@ def average(a): # This implements a weighted average, for now we don't implement the # weighting, just the average part! - return mean(a) + if not hasattr(a, "mean"): + a = _numpypy.array(a) + return a.mean() def identity(n, dtype=None): - a = _numpypy.zeros((n,n), dtype=dtype) + a = _numpypy.zeros((n, n), dtype=dtype) for i in range(n): a[i][i] = 1 return a -def mean(a, axis=None): - if not hasattr(a, "mean"): - a = _numpypy.array(a) - return a.mean(axis) - def sum(a,axis=None): '''sum(a, axis=None) Sum of array elements over a given axis. - + Parameters ---------- a : array_like @@ -35,7 +32,7 @@ axis : integer, optional Axis over which the sum is taken. By default `axis` is None, and all elements are summed. - + Returns ------- sum_along_axis : ndarray @@ -43,7 +40,7 @@ axis removed. If `a` is a 0-d array, or if `axis` is None, a scalar is returned. If an output array is specified, a reference to `out` is returned. - + See Also -------- ndarray.sum : Equivalent method. @@ -79,40 +76,3 @@ arr[j] = i i += step return arr - - -def reshape(a, shape): - '''reshape(a, newshape) - Gives a new shape to an array without changing its data. - - Parameters - ---------- - a : array_like - Array to be reshaped. - newshape : int or tuple of ints - The new shape should be compatible with the original shape. If - an integer, then the result will be a 1-D array of that length. - One shape dimension can be -1. In this case, the value is inferred - from the length of the array and remaining dimensions. - - Returns - ------- - reshaped_array : ndarray - This will be a new view object if possible; otherwise, it will - be a copy. - - - See Also - -------- - ndarray.reshape : Equivalent method. - - Notes - ----- - - It is not always possible to change the shape of an array without - copying the data. If you want an error to be raise if the data is copied, - you should assign the new shape to the shape attribute of the array -''' - if not hasattr(a, 'reshape'): - a = _numpypy.array(a) - return a.reshape(shape) diff --git a/pypy/module/micronumpy/test/test_module.py b/pypy/module/micronumpy/test/test_module.py --- a/pypy/module/micronumpy/test/test_module.py +++ b/pypy/module/micronumpy/test/test_module.py @@ -2,16 +2,11 @@ class AppTestNumPyModule(BaseNumpyAppTest): - def test_mean(self): - from _numpypy import array, mean - assert mean(array(range(5))) == 2.0 - assert mean(range(5)) == 2.0 - def test_average(self): from _numpypy import array, average assert average(range(10)) == 4.5 assert average(array(range(10))) == 4.5 - + def test_sum(self): from _numpypy import array, sum assert sum(range(10)) == 45 @@ -21,7 +16,7 @@ from _numpypy import array, min assert min(range(10)) == 0 assert min(array(range(10))) == 0 - + def test_max(self): from _numpypy import array, max assert max(range(10)) == 9 diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -726,16 +726,16 @@ assert d[1] == 12 def test_mean(self): - from _numpypy import array, mean + from _numpypy import array a = array(range(5)) assert a.mean() == 2.0 assert a[:4].mean() == 1.5 a = array(range(105)).reshape(3, 5, 7) - b = mean(a, axis=0) - b[0,0]==35. + b = a.mean(axis=0) + b[0, 0]==35. assert a.mean(axis=0)[0, 0] == 35 assert (b == array(range(35, 70), dtype=float).reshape(5, 7)).all() - assert (mean(a, 2) == array(range(0, 15), dtype=float).reshape(3, 5) * 7 + 3).all() + assert (a.mean(2) == array(range(0, 15), dtype=float).reshape(3, 5) * 7 + 3).all() def test_sum(self): from _numpypy import array @@ -1550,18 +1550,3 @@ a = arange(0, 0.8, 0.1) assert len(a) == 8 assert arange(False, True, True).dtype is dtype(int) - - -class AppTestRanges(BaseNumpyAppTest): - def test_app_reshape(self): - from _numpypy import arange, array, dtype, reshape - a = arange(12) - b = reshape(a, (3, 4)) - assert b.shape == (3, 4) - a = range(12) - b = reshape(a, (3, 4)) - assert b.shape == (3, 4) - a = array(range(105)).reshape(3, 5, 7) - assert a.reshape(1, -1).shape == (1, 105) - assert a.reshape(1, 1, -1).shape == (1, 1, 105) - assert a.reshape(-1, 1, 1).shape == (105, 1, 1) From noreply at buildbot.pypy.org Tue Jan 17 19:25:39 2012 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Tue, 17 Jan 2012 19:25:39 +0100 (CET) Subject: [pypy-commit] pypy default: oops, fix tests Message-ID: <20120117182539.6D275820D8@wyvern.cs.uni-duesseldorf.de> Author: Alex Gaynor Branch: Changeset: r51412:593bcb6b2e6f Date: 2012-01-17 12:25 -0600 http://bitbucket.org/pypy/pypy/changeset/593bcb6b2e6f/ Log: oops, fix tests diff --git a/lib_pypy/numpypy/test/test_fromnumeric.py b/lib_pypy/numpypy/test/test_fromnumeric.py --- a/lib_pypy/numpypy/test/test_fromnumeric.py +++ b/lib_pypy/numpypy/test/test_fromnumeric.py @@ -122,6 +122,6 @@ b = reshape(a, (3, 4)) assert b.shape == (3, 4) a = array(range(105)).reshape(3, 5, 7) - assert reshape(a, 1, -1).shape == (1, 105) - assert reshape(a, 1, 1, -1).shape == (1, 1, 105) - assert reshape(a, -1, 1, 1).shape == (105, 1, 1) + assert reshape(a, (1, -1)).shape == (1, 105) + assert reshape(a, (1, 1, -1)).shape == (1, 1, 105) + assert reshape(a, (-1, 1, 1)).shape == (105, 1, 1) From noreply at buildbot.pypy.org Tue Jan 17 19:28:04 2012 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Tue, 17 Jan 2012 19:28:04 +0100 (CET) Subject: [pypy-commit] pypy default: uncomment a test Message-ID: <20120117182804.4371C820D8@wyvern.cs.uni-duesseldorf.de> Author: Alex Gaynor Branch: Changeset: r51413:f602adef8b38 Date: 2012-01-17 12:27 -0600 http://bitbucket.org/pypy/pypy/changeset/f602adef8b38/ Log: uncomment a test diff --git a/lib_pypy/numpypy/test/test_fromnumeric.py b/lib_pypy/numpypy/test/test_fromnumeric.py --- a/lib_pypy/numpypy/test/test_fromnumeric.py +++ b/lib_pypy/numpypy/test/test_fromnumeric.py @@ -18,8 +18,8 @@ from numpypy import array, arange, argmin a = arange(6).reshape((2,3)) assert argmin(a) == 0 - # assert (argmax(a, axis=0) == array([0, 0, 0])).all() - # assert (argmax(a, axis=1) == array([0, 0])).all() + assert (argmin(a, axis=0) == array([0, 0, 0])).all() + assert (argmin(a, axis=1) == array([0, 0])).all() b = arange(6) b[1] = 0 assert argmin(b) == 0 From noreply at buildbot.pypy.org Tue Jan 17 19:51:58 2012 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Tue, 17 Jan 2012 19:51:58 +0100 (CET) Subject: [pypy-commit] pypy default: split _codecs.lookup_codec into parts with and without a loop, so the JIT can inline most of lookup_codec, eventually it might even make sense to make this codec_search_cache thing be a purefunction, but I'm not sure what it's sematnics are ATM Message-ID: <20120117185158.35F89820D8@wyvern.cs.uni-duesseldorf.de> Author: Alex Gaynor Branch: Changeset: r51414:33bd443587ba Date: 2012-01-17 12:51 -0600 http://bitbucket.org/pypy/pypy/changeset/33bd443587ba/ Log: split _codecs.lookup_codec into parts with and without a loop, so the JIT can inline most of lookup_codec, eventually it might even make sense to make this codec_search_cache thing be a purefunction, but I'm not sure what it's sematnics are ATM diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py --- a/pypy/module/_codecs/interp_codecs.py +++ b/pypy/module/_codecs/interp_codecs.py @@ -108,6 +108,10 @@ w_result = state.codec_search_cache.get(normalized_encoding, None) if w_result is not None: return w_result + return _lookup_codec_loop(space, encoding, normalized_encoding) + +def _lookup_codec_loop(space, encoding, normalized_encoding): + state = space.fromcache(CodecState) if state.codec_need_encodings: w_import = space.getattr(space.builtin, space.wrap("__import__")) # registers new codecs From noreply at buildbot.pypy.org Tue Jan 17 20:27:29 2012 From: noreply at buildbot.pypy.org (amauryfa) Date: Tue, 17 Jan 2012 20:27:29 +0100 (CET) Subject: [pypy-commit] pypy py3k: Remove more L suffixes and long() calls Message-ID: <20120117192729.D461E820D8@wyvern.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3k Changeset: r51415:d682184291a6 Date: 2012-01-17 20:25 +0100 http://bitbucket.org/pypy/pypy/changeset/d682184291a6/ Log: Remove more L suffixes and long() calls diff --git a/lib_pypy/_md5.py b/lib_pypy/_md5.py --- a/lib_pypy/_md5.py +++ b/lib_pypy/_md5.py @@ -48,7 +48,7 @@ "Transform a list of characters into a list of longs." imax = len(list) // 4 - hl = [0L] * imax + hl = [0] * imax j = 0 i = 0 @@ -100,16 +100,16 @@ (now summed-up in one function). """ - res = 0L + res = 0 res = res + a + func(b, c, d) res = res + x res = res + ac - res = res & 0xffffffffL + res = res & 0xffffffff res = _rotateLeft(res, s) - res = res & 0xffffffffL + res = res & 0xffffffff res = res + b - return res & 0xffffffffL + return res & 0xffffffff class md5: @@ -122,7 +122,7 @@ "Initialisation." # Initial message length in bits(!). - self.length = 0L + self.length = 0 self.count = [0, 0] # Initial empty message as a sequence of bytes (8 bit characters). @@ -139,15 +139,15 @@ def init(self): "Initialize the message-digest and set all fields to zero." - self.length = 0L + self.length = 0 self.count = [0, 0] self.input = [] # Load magic initialization constants. - self.A = 0x67452301L - self.B = 0xefcdab89L - self.C = 0x98badcfeL - self.D = 0x10325476L + self.A = 0x67452301 + self.B = 0xefcdab89 + self.C = 0x98badcfe + self.D = 0x10325476 def _transform(self, inp): @@ -164,90 +164,90 @@ S11, S12, S13, S14 = 7, 12, 17, 22 - a = XX(F, a, b, c, d, inp[ 0], S11, 0xD76AA478L) # 1 - d = XX(F, d, a, b, c, inp[ 1], S12, 0xE8C7B756L) # 2 - c = XX(F, c, d, a, b, inp[ 2], S13, 0x242070DBL) # 3 - b = XX(F, b, c, d, a, inp[ 3], S14, 0xC1BDCEEEL) # 4 - a = XX(F, a, b, c, d, inp[ 4], S11, 0xF57C0FAFL) # 5 - d = XX(F, d, a, b, c, inp[ 5], S12, 0x4787C62AL) # 6 - c = XX(F, c, d, a, b, inp[ 6], S13, 0xA8304613L) # 7 - b = XX(F, b, c, d, a, inp[ 7], S14, 0xFD469501L) # 8 - a = XX(F, a, b, c, d, inp[ 8], S11, 0x698098D8L) # 9 - d = XX(F, d, a, b, c, inp[ 9], S12, 0x8B44F7AFL) # 10 - c = XX(F, c, d, a, b, inp[10], S13, 0xFFFF5BB1L) # 11 - b = XX(F, b, c, d, a, inp[11], S14, 0x895CD7BEL) # 12 - a = XX(F, a, b, c, d, inp[12], S11, 0x6B901122L) # 13 - d = XX(F, d, a, b, c, inp[13], S12, 0xFD987193L) # 14 - c = XX(F, c, d, a, b, inp[14], S13, 0xA679438EL) # 15 - b = XX(F, b, c, d, a, inp[15], S14, 0x49B40821L) # 16 + a = XX(F, a, b, c, d, inp[ 0], S11, 0xD76AA478) # 1 + d = XX(F, d, a, b, c, inp[ 1], S12, 0xE8C7B756) # 2 + c = XX(F, c, d, a, b, inp[ 2], S13, 0x242070DB) # 3 + b = XX(F, b, c, d, a, inp[ 3], S14, 0xC1BDCEEE) # 4 + a = XX(F, a, b, c, d, inp[ 4], S11, 0xF57C0FAF) # 5 + d = XX(F, d, a, b, c, inp[ 5], S12, 0x4787C62A) # 6 + c = XX(F, c, d, a, b, inp[ 6], S13, 0xA8304613) # 7 + b = XX(F, b, c, d, a, inp[ 7], S14, 0xFD469501) # 8 + a = XX(F, a, b, c, d, inp[ 8], S11, 0x698098D8) # 9 + d = XX(F, d, a, b, c, inp[ 9], S12, 0x8B44F7AF) # 10 + c = XX(F, c, d, a, b, inp[10], S13, 0xFFFF5BB1) # 11 + b = XX(F, b, c, d, a, inp[11], S14, 0x895CD7BE) # 12 + a = XX(F, a, b, c, d, inp[12], S11, 0x6B901122) # 13 + d = XX(F, d, a, b, c, inp[13], S12, 0xFD987193) # 14 + c = XX(F, c, d, a, b, inp[14], S13, 0xA679438E) # 15 + b = XX(F, b, c, d, a, inp[15], S14, 0x49B40821) # 16 # Round 2. S21, S22, S23, S24 = 5, 9, 14, 20 - a = XX(G, a, b, c, d, inp[ 1], S21, 0xF61E2562L) # 17 - d = XX(G, d, a, b, c, inp[ 6], S22, 0xC040B340L) # 18 - c = XX(G, c, d, a, b, inp[11], S23, 0x265E5A51L) # 19 - b = XX(G, b, c, d, a, inp[ 0], S24, 0xE9B6C7AAL) # 20 - a = XX(G, a, b, c, d, inp[ 5], S21, 0xD62F105DL) # 21 - d = XX(G, d, a, b, c, inp[10], S22, 0x02441453L) # 22 - c = XX(G, c, d, a, b, inp[15], S23, 0xD8A1E681L) # 23 - b = XX(G, b, c, d, a, inp[ 4], S24, 0xE7D3FBC8L) # 24 - a = XX(G, a, b, c, d, inp[ 9], S21, 0x21E1CDE6L) # 25 - d = XX(G, d, a, b, c, inp[14], S22, 0xC33707D6L) # 26 - c = XX(G, c, d, a, b, inp[ 3], S23, 0xF4D50D87L) # 27 - b = XX(G, b, c, d, a, inp[ 8], S24, 0x455A14EDL) # 28 - a = XX(G, a, b, c, d, inp[13], S21, 0xA9E3E905L) # 29 - d = XX(G, d, a, b, c, inp[ 2], S22, 0xFCEFA3F8L) # 30 - c = XX(G, c, d, a, b, inp[ 7], S23, 0x676F02D9L) # 31 - b = XX(G, b, c, d, a, inp[12], S24, 0x8D2A4C8AL) # 32 + a = XX(G, a, b, c, d, inp[ 1], S21, 0xF61E2562) # 17 + d = XX(G, d, a, b, c, inp[ 6], S22, 0xC040B340) # 18 + c = XX(G, c, d, a, b, inp[11], S23, 0x265E5A51) # 19 + b = XX(G, b, c, d, a, inp[ 0], S24, 0xE9B6C7AA) # 20 + a = XX(G, a, b, c, d, inp[ 5], S21, 0xD62F105D) # 21 + d = XX(G, d, a, b, c, inp[10], S22, 0x02441453) # 22 + c = XX(G, c, d, a, b, inp[15], S23, 0xD8A1E681) # 23 + b = XX(G, b, c, d, a, inp[ 4], S24, 0xE7D3FBC8) # 24 + a = XX(G, a, b, c, d, inp[ 9], S21, 0x21E1CDE6) # 25 + d = XX(G, d, a, b, c, inp[14], S22, 0xC33707D6) # 26 + c = XX(G, c, d, a, b, inp[ 3], S23, 0xF4D50D87) # 27 + b = XX(G, b, c, d, a, inp[ 8], S24, 0x455A14ED) # 28 + a = XX(G, a, b, c, d, inp[13], S21, 0xA9E3E905) # 29 + d = XX(G, d, a, b, c, inp[ 2], S22, 0xFCEFA3F8) # 30 + c = XX(G, c, d, a, b, inp[ 7], S23, 0x676F02D9) # 31 + b = XX(G, b, c, d, a, inp[12], S24, 0x8D2A4C8A) # 32 # Round 3. S31, S32, S33, S34 = 4, 11, 16, 23 - a = XX(H, a, b, c, d, inp[ 5], S31, 0xFFFA3942L) # 33 - d = XX(H, d, a, b, c, inp[ 8], S32, 0x8771F681L) # 34 - c = XX(H, c, d, a, b, inp[11], S33, 0x6D9D6122L) # 35 - b = XX(H, b, c, d, a, inp[14], S34, 0xFDE5380CL) # 36 - a = XX(H, a, b, c, d, inp[ 1], S31, 0xA4BEEA44L) # 37 - d = XX(H, d, a, b, c, inp[ 4], S32, 0x4BDECFA9L) # 38 - c = XX(H, c, d, a, b, inp[ 7], S33, 0xF6BB4B60L) # 39 - b = XX(H, b, c, d, a, inp[10], S34, 0xBEBFBC70L) # 40 - a = XX(H, a, b, c, d, inp[13], S31, 0x289B7EC6L) # 41 - d = XX(H, d, a, b, c, inp[ 0], S32, 0xEAA127FAL) # 42 - c = XX(H, c, d, a, b, inp[ 3], S33, 0xD4EF3085L) # 43 - b = XX(H, b, c, d, a, inp[ 6], S34, 0x04881D05L) # 44 - a = XX(H, a, b, c, d, inp[ 9], S31, 0xD9D4D039L) # 45 - d = XX(H, d, a, b, c, inp[12], S32, 0xE6DB99E5L) # 46 - c = XX(H, c, d, a, b, inp[15], S33, 0x1FA27CF8L) # 47 - b = XX(H, b, c, d, a, inp[ 2], S34, 0xC4AC5665L) # 48 + a = XX(H, a, b, c, d, inp[ 5], S31, 0xFFFA3942) # 33 + d = XX(H, d, a, b, c, inp[ 8], S32, 0x8771F681) # 34 + c = XX(H, c, d, a, b, inp[11], S33, 0x6D9D6122) # 35 + b = XX(H, b, c, d, a, inp[14], S34, 0xFDE5380C) # 36 + a = XX(H, a, b, c, d, inp[ 1], S31, 0xA4BEEA44) # 37 + d = XX(H, d, a, b, c, inp[ 4], S32, 0x4BDECFA9) # 38 + c = XX(H, c, d, a, b, inp[ 7], S33, 0xF6BB4B60) # 39 + b = XX(H, b, c, d, a, inp[10], S34, 0xBEBFBC70) # 40 + a = XX(H, a, b, c, d, inp[13], S31, 0x289B7EC6) # 41 + d = XX(H, d, a, b, c, inp[ 0], S32, 0xEAA127FA) # 42 + c = XX(H, c, d, a, b, inp[ 3], S33, 0xD4EF3085) # 43 + b = XX(H, b, c, d, a, inp[ 6], S34, 0x04881D05) # 44 + a = XX(H, a, b, c, d, inp[ 9], S31, 0xD9D4D039) # 45 + d = XX(H, d, a, b, c, inp[12], S32, 0xE6DB99E5) # 46 + c = XX(H, c, d, a, b, inp[15], S33, 0x1FA27CF8) # 47 + b = XX(H, b, c, d, a, inp[ 2], S34, 0xC4AC5665) # 48 # Round 4. S41, S42, S43, S44 = 6, 10, 15, 21 - a = XX(I, a, b, c, d, inp[ 0], S41, 0xF4292244L) # 49 - d = XX(I, d, a, b, c, inp[ 7], S42, 0x432AFF97L) # 50 - c = XX(I, c, d, a, b, inp[14], S43, 0xAB9423A7L) # 51 - b = XX(I, b, c, d, a, inp[ 5], S44, 0xFC93A039L) # 52 - a = XX(I, a, b, c, d, inp[12], S41, 0x655B59C3L) # 53 - d = XX(I, d, a, b, c, inp[ 3], S42, 0x8F0CCC92L) # 54 - c = XX(I, c, d, a, b, inp[10], S43, 0xFFEFF47DL) # 55 - b = XX(I, b, c, d, a, inp[ 1], S44, 0x85845DD1L) # 56 - a = XX(I, a, b, c, d, inp[ 8], S41, 0x6FA87E4FL) # 57 - d = XX(I, d, a, b, c, inp[15], S42, 0xFE2CE6E0L) # 58 - c = XX(I, c, d, a, b, inp[ 6], S43, 0xA3014314L) # 59 - b = XX(I, b, c, d, a, inp[13], S44, 0x4E0811A1L) # 60 - a = XX(I, a, b, c, d, inp[ 4], S41, 0xF7537E82L) # 61 - d = XX(I, d, a, b, c, inp[11], S42, 0xBD3AF235L) # 62 - c = XX(I, c, d, a, b, inp[ 2], S43, 0x2AD7D2BBL) # 63 - b = XX(I, b, c, d, a, inp[ 9], S44, 0xEB86D391L) # 64 + a = XX(I, a, b, c, d, inp[ 0], S41, 0xF4292244) # 49 + d = XX(I, d, a, b, c, inp[ 7], S42, 0x432AFF97) # 50 + c = XX(I, c, d, a, b, inp[14], S43, 0xAB9423A7) # 51 + b = XX(I, b, c, d, a, inp[ 5], S44, 0xFC93A039) # 52 + a = XX(I, a, b, c, d, inp[12], S41, 0x655B59C3) # 53 + d = XX(I, d, a, b, c, inp[ 3], S42, 0x8F0CCC92) # 54 + c = XX(I, c, d, a, b, inp[10], S43, 0xFFEFF47D) # 55 + b = XX(I, b, c, d, a, inp[ 1], S44, 0x85845DD1) # 56 + a = XX(I, a, b, c, d, inp[ 8], S41, 0x6FA87E4F) # 57 + d = XX(I, d, a, b, c, inp[15], S42, 0xFE2CE6E0) # 58 + c = XX(I, c, d, a, b, inp[ 6], S43, 0xA3014314) # 59 + b = XX(I, b, c, d, a, inp[13], S44, 0x4E0811A1) # 60 + a = XX(I, a, b, c, d, inp[ 4], S41, 0xF7537E82) # 61 + d = XX(I, d, a, b, c, inp[11], S42, 0xBD3AF235) # 62 + c = XX(I, c, d, a, b, inp[ 2], S43, 0x2AD7D2BB) # 63 + b = XX(I, b, c, d, a, inp[ 9], S44, 0xEB86D391) # 64 - A = (A + a) & 0xffffffffL - B = (B + b) & 0xffffffffL - C = (C + c) & 0xffffffffL - D = (D + d) & 0xffffffffL + A = (A + a) & 0xffffffff + B = (B + b) & 0xffffffff + C = (C + c) & 0xffffffff + D = (D + d) & 0xffffffff self.A, self.B, self.C, self.D = A, B, C, D @@ -273,7 +273,7 @@ leninBuf = long(len(inBuf)) # Compute number of bytes mod 64. - index = (self.count[0] >> 3) & 0x3FL + index = (self.count[0] >> 3) & 0x3F # Update number of bits. self.count[0] = self.count[0] + (leninBuf << 3) @@ -312,7 +312,7 @@ input = [] + self.input count = [] + self.count - index = (self.count[0] >> 3) & 0x3fL + index = (self.count[0] >> 3) & 0x3f if index < 56: padLen = 56 - index diff --git a/lib_pypy/_sha1.py b/lib_pypy/_sha1.py --- a/lib_pypy/_sha1.py +++ b/lib_pypy/_sha1.py @@ -38,7 +38,7 @@ s = b'' pack = struct.pack while n > 0: - s = pack('>I', n & 0xffffffffL) + s + s = pack('>I', n & 0xffffffff) + s n = n >> 32 # Strip off leading zeros. @@ -64,7 +64,7 @@ "Transform a list of characters into a list of longs." imax = len(list) // 4 - hl = [0L] * imax + hl = [0] * imax j = 0 i = 0 @@ -108,10 +108,10 @@ # Constants to be used K = [ - 0x5A827999L, # ( 0 <= t <= 19) - 0x6ED9EBA1L, # (20 <= t <= 39) - 0x8F1BBCDCL, # (40 <= t <= 59) - 0xCA62C1D6L # (60 <= t <= 79) + 0x5A827999, # ( 0 <= t <= 19) + 0x6ED9EBA1, # (20 <= t <= 39) + 0x8F1BBCDC, # (40 <= t <= 59) + 0xCA62C1D6 # (60 <= t <= 79) ] class sha: @@ -124,7 +124,7 @@ "Initialisation." # Initial message length in bits(!). - self.length = 0L + self.length = 0 self.count = [0, 0] # Initial empty message as a sequence of bytes (8 bit characters). @@ -138,21 +138,21 @@ def init(self): "Initialize the message-digest and set all fields to zero." - self.length = 0L + self.length = 0 self.input = [] # Initial 160 bit message digest (5 times 32 bit). - self.H0 = 0x67452301L - self.H1 = 0xEFCDAB89L - self.H2 = 0x98BADCFEL - self.H3 = 0x10325476L - self.H4 = 0xC3D2E1F0L + self.H0 = 0x67452301 + self.H1 = 0xEFCDAB89 + self.H2 = 0x98BADCFE + self.H3 = 0x10325476 + self.H4 = 0xC3D2E1F0 def _transform(self, W): for t in range(16, 80): W.append(_rotateLeft( - W[t-3] ^ W[t-8] ^ W[t-14] ^ W[t-16], 1) & 0xffffffffL) + W[t-3] ^ W[t-8] ^ W[t-14] ^ W[t-16], 1) & 0xffffffff) A = self.H0 B = self.H1 @@ -166,49 +166,49 @@ TEMP = _rotateLeft(A, 5) + f[t/20] + E + W[t] + K[t/20] E = D D = C - C = _rotateLeft(B, 30) & 0xffffffffL + C = _rotateLeft(B, 30) & 0xffffffff B = A - A = TEMP & 0xffffffffL + A = TEMP & 0xffffffff """ for t in range(0, 20): TEMP = _rotateLeft(A, 5) + ((B & C) | ((~ B) & D)) + E + W[t] + K[0] E = D D = C - C = _rotateLeft(B, 30) & 0xffffffffL + C = _rotateLeft(B, 30) & 0xffffffff B = A - A = TEMP & 0xffffffffL + A = TEMP & 0xffffffff for t in range(20, 40): TEMP = _rotateLeft(A, 5) + (B ^ C ^ D) + E + W[t] + K[1] E = D D = C - C = _rotateLeft(B, 30) & 0xffffffffL + C = _rotateLeft(B, 30) & 0xffffffff B = A - A = TEMP & 0xffffffffL + A = TEMP & 0xffffffff for t in range(40, 60): TEMP = _rotateLeft(A, 5) + ((B & C) | (B & D) | (C & D)) + E + W[t] + K[2] E = D D = C - C = _rotateLeft(B, 30) & 0xffffffffL + C = _rotateLeft(B, 30) & 0xffffffff B = A - A = TEMP & 0xffffffffL + A = TEMP & 0xffffffff for t in range(60, 80): TEMP = _rotateLeft(A, 5) + (B ^ C ^ D) + E + W[t] + K[3] E = D D = C - C = _rotateLeft(B, 30) & 0xffffffffL + C = _rotateLeft(B, 30) & 0xffffffff B = A - A = TEMP & 0xffffffffL + A = TEMP & 0xffffffff - self.H0 = (self.H0 + A) & 0xffffffffL - self.H1 = (self.H1 + B) & 0xffffffffL - self.H2 = (self.H2 + C) & 0xffffffffL - self.H3 = (self.H3 + D) & 0xffffffffL - self.H4 = (self.H4 + E) & 0xffffffffL + self.H0 = (self.H0 + A) & 0xffffffff + self.H1 = (self.H1 + B) & 0xffffffff + self.H2 = (self.H2 + C) & 0xffffffff + self.H3 = (self.H3 + D) & 0xffffffff + self.H4 = (self.H4 + E) & 0xffffffff # Down from here all methods follow the Python Standard Library @@ -230,10 +230,10 @@ to the hashed string. """ - leninBuf = long(len(inBuf)) + leninBuf = len(inBuf) # Compute number of bytes mod 64. - index = (self.count[1] >> 3) & 0x3FL + index = (self.count[1] >> 3) & 0x3F # Update number of bits. self.count[1] = self.count[1] + (leninBuf << 3) @@ -273,7 +273,7 @@ input = [] + self.input count = [] + self.count - index = (self.count[1] >> 3) & 0x3fL + index = (self.count[1] >> 3) & 0x3f if index < 56: padLen = 56 - index diff --git a/lib_pypy/_sha512.py b/lib_pypy/_sha512.py --- a/lib_pypy/_sha512.py +++ b/lib_pypy/_sha512.py @@ -10,7 +10,7 @@ def new_shaobject(): return { - 'digest': [0L]*8, + 'digest': [0]*8, 'count_lo': 0, 'count_hi': 0, 'data': [0]* SHA_BLOCKSIZE, From noreply at buildbot.pypy.org Tue Jan 17 20:27:31 2012 From: noreply at buildbot.pypy.org (amauryfa) Date: Tue, 17 Jan 2012 20:27:31 +0100 (CET) Subject: [pypy-commit] pypy py3k: Another fix in termios Message-ID: <20120117192731.1757E820D8@wyvern.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3k Changeset: r51416:03901e509c25 Date: 2012-01-17 20:25 +0100 http://bitbucket.org/pypy/pypy/changeset/03901e509c25/ Log: Another fix in termios diff --git a/pypy/module/termios/interp_termios.py b/pypy/module/termios/interp_termios.py --- a/pypy/module/termios/interp_termios.py +++ b/pypy/module/termios/interp_termios.py @@ -46,7 +46,7 @@ iflag, oflag, cflag, lflag, ispeed, ospeed, cc = tup l_w = [space.wrap(i) for i in [iflag, oflag, cflag, lflag, ispeed, ospeed]] # last one need to be chosen carefully - cc_w = [space.wrap(i) for i in cc] + cc_w = [space.wrapbytes(i) for i in cc] if lflag & termios.ICANON: cc_w[termios.VMIN] = space.wrap(ord(cc[termios.VMIN][0])) cc_w[termios.VTIME] = space.wrap(ord(cc[termios.VTIME][0])) From noreply at buildbot.pypy.org Tue Jan 17 20:27:32 2012 From: noreply at buildbot.pypy.org (amauryfa) Date: Tue, 17 Jan 2012 20:27:32 +0100 (CET) Subject: [pypy-commit] pypy py3k: Add a reminder about the broken pyrepl. Message-ID: <20120117192732.4CE39820D8@wyvern.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3k Changeset: r51417:2c6715a45957 Date: 2012-01-17 20:26 +0100 http://bitbucket.org/pypy/pypy/changeset/2c6715a45957/ Log: Add a reminder about the broken pyrepl. diff --git a/lib_pypy/_pypy_interact.py b/lib_pypy/_pypy_interact.py --- a/lib_pypy/_pypy_interact.py +++ b/lib_pypy/_pypy_interact.py @@ -36,6 +36,9 @@ from pyrepl.simple_interact import run_multiline_interactive_console except ImportError: run_simple_interactive_console(mainmodule) + except SyntaxError: + print("Warning: 'import pyrepl' failed with SyntaxError") + run_simple_interactive_console(mainmodule) else: run_multiline_interactive_console(mainmodule) From noreply at buildbot.pypy.org Tue Jan 17 20:27:33 2012 From: noreply at buildbot.pypy.org (amauryfa) Date: Tue, 17 Jan 2012 20:27:33 +0100 (CET) Subject: [pypy-commit] pypy py3k: ast.py is generated. Add the Starred in Python.asdl, and regenerate Message-ID: <20120117192733.8DDA7820D8@wyvern.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3k Changeset: r51418:5b195e604794 Date: 2012-01-17 20:26 +0100 http://bitbucket.org/pypy/pypy/changeset/5b195e604794/ Log: ast.py is generated. Add the Starred in Python.asdl, and regenerate with "python interpreter/astcompiler/tools/asdl_py.py" diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py --- a/pypy/interpreter/astcompiler/ast.py +++ b/pypy/interpreter/astcompiler/ast.py @@ -1507,11 +1507,6 @@ for node in self.comparators: node.sync_app_attrs(space) -class Starred(expr): - def __init__(self, value, ctx, lineno, col_offset): - self.value = value - self.ctx = ctx - expr.__init__(self, lineno, col_offset) class Call(expr): @@ -1666,6 +1661,29 @@ self.slice.sync_app_attrs(space) +class Starred(expr): + + def __init__(self, value, ctx, lineno, col_offset): + self.value = value + self.ctx = ctx + expr.__init__(self, lineno, col_offset) + self.initialization_state = 15 + + def walkabout(self, visitor): + visitor.visit_Starred(self) + + def mutate_over(self, visitor): + self.value = self.value.mutate_over(visitor) + return visitor.visit_Starred(self) + + def sync_app_attrs(self, space): + if (self.initialization_state & ~0) ^ 15: + self.missing_field(space, ['lineno', 'col_offset', 'value', 'ctx'], 'Starred') + else: + pass + self.value.sync_app_attrs(space) + + class Name(expr): def __init__(self, id, ctx, lineno, col_offset): @@ -2453,6 +2471,8 @@ return self.default_visitor(node) def visit_Subscript(self, node): return self.default_visitor(node) + def visit_Starred(self, node): + return self.default_visitor(node) def visit_Name(self, node): return self.default_visitor(node) def visit_List(self, node): @@ -2663,6 +2683,9 @@ node.value.walkabout(self) node.slice.walkabout(self) + def visit_Starred(self, node): + node.value.walkabout(self) + def visit_Name(self, node): pass @@ -5814,6 +5837,77 @@ __init__=interp2app(Subscript_init), ) +def Starred_get_value(space, w_self): + if w_self.w_dict is not None: + w_obj = w_self.getdictvalue(space, 'value') + if w_obj is not None: + return w_obj + if not w_self.initialization_state & 4: + typename = space.type(w_self).getname(space) + raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value') + return space.wrap(w_self.value) + +def Starred_set_value(space, w_self, w_new_value): + try: + w_self.value = space.interp_w(expr, w_new_value, False) + if type(w_self.value) is expr: + raise OperationError(space.w_TypeError, space.w_None) + except OperationError, e: + if not e.match(space, space.w_TypeError): + raise + w_self.setdictvalue(space, 'value', w_new_value) + return + w_self.deldictvalue(space, 'value') + w_self.initialization_state |= 4 + +def Starred_get_ctx(space, w_self): + if w_self.w_dict is not None: + w_obj = w_self.getdictvalue(space, 'ctx') + if w_obj is not None: + return w_obj + if not w_self.initialization_state & 8: + typename = space.type(w_self).getname(space) + raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx') + return expr_context_to_class[w_self.ctx - 1]() + +def Starred_set_ctx(space, w_self, w_new_value): + try: + obj = space.interp_w(expr_context, w_new_value) + w_self.ctx = obj.to_simple_int(space) + except OperationError, e: + if not e.match(space, space.w_TypeError): + raise + w_self.setdictvalue(space, 'ctx', w_new_value) + return + # need to save the original object too + w_self.setdictvalue(space, 'ctx', w_new_value) + w_self.initialization_state |= 8 + +_Starred_field_unroller = unrolling_iterable(['value', 'ctx']) +def Starred_init(space, w_self, __args__): + w_self = space.descr_self_interp_w(Starred, w_self) + args_w, kwargs_w = __args__.unpack() + if args_w: + if len(args_w) != 2: + w_err = space.wrap("Starred constructor takes either 0 or 2 positional arguments") + raise OperationError(space.w_TypeError, w_err) + i = 0 + for field in _Starred_field_unroller: + space.setattr(w_self, space.wrap(field), args_w[i]) + i += 1 + for field, w_value in kwargs_w.iteritems(): + space.setattr(w_self, space.wrap(field), w_value) + +Starred.typedef = typedef.TypeDef("Starred", + expr.typedef, + __module__='_ast', + _fields=_FieldsWrapper(['value', 'ctx']), + value=typedef.GetSetProperty(Starred_get_value, Starred_set_value, cls=Starred), + ctx=typedef.GetSetProperty(Starred_get_ctx, Starred_set_ctx, cls=Starred), + __new__=interp2app(get_AST_new(Starred)), + __init__=interp2app(Starred_init), +) + def Name_get_id(space, w_self): if w_self.w_dict is not None: w_obj = w_self.getdictvalue(space, 'id') diff --git a/pypy/interpreter/astcompiler/tools/Python.asdl b/pypy/interpreter/astcompiler/tools/Python.asdl --- a/pypy/interpreter/astcompiler/tools/Python.asdl +++ b/pypy/interpreter/astcompiler/tools/Python.asdl @@ -73,6 +73,7 @@ -- the following expression can appear in assignment context | Attribute(expr value, identifier attr, expr_context ctx) | Subscript(expr value, slice slice, expr_context ctx) + | Starred(expr value, expr_context ctx) | Name(identifier id, expr_context ctx) | List(expr* elts, expr_context ctx) | Tuple(expr* elts, expr_context ctx) From noreply at buildbot.pypy.org Tue Jan 17 20:46:31 2012 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 17 Jan 2012 20:46:31 +0100 (CET) Subject: [pypy-commit] pypy default: a test and a fix Message-ID: <20120117194631.DDDB4820D8@wyvern.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r51419:80570f9631d7 Date: 2012-01-17 21:29 +0200 http://bitbucket.org/pypy/pypy/changeset/80570f9631d7/ Log: a test and a fix diff --git a/pypy/tool/jitlogparser/parser.py b/pypy/tool/jitlogparser/parser.py --- a/pypy/tool/jitlogparser/parser.py +++ b/pypy/tool/jitlogparser/parser.py @@ -140,13 +140,15 @@ bytecode_name = None is_bytecode = True inline_level = None + has_dmp = False def parse_code_data(self, arg): m = re.search('

\w]+)[\.,] file \'(.+?)\'[\.,] line (\d+)> #(\d+) (\w+)',
                       arg)
         if m is None:
             # a non-code loop, like StrLiteralSearch or something
-            self.bytecode_name = arg
+            if arg:
+                self.bytecode_name = arg
         else:
             self.name, self.filename, lineno, bytecode_no, self.bytecode_name = m.groups()
             self.startlineno = int(lineno)
@@ -218,7 +220,7 @@
         self.inputargs = inputargs
         self.chunks = chunks
         for chunk in self.chunks:
-            if chunk.filename is not None:
+            if chunk.bytecode_name is not None:
                 self.startlineno = chunk.startlineno
                 self.filename = chunk.filename
                 self.name = chunk.name
diff --git a/pypy/tool/jitlogparser/test/test_parser.py b/pypy/tool/jitlogparser/test/test_parser.py
--- a/pypy/tool/jitlogparser/test/test_parser.py
+++ b/pypy/tool/jitlogparser/test/test_parser.py
@@ -283,3 +283,13 @@
     assert loops[-1].count == 1234
     assert loops[1].count == 123
     assert loops[2].count == 12
+
+def test_parse_nonpython():
+    loop = parse("""
+    []
+    debug_merge_point(0, 'random')
+    debug_merge_point(0, ' #15 COMPARE_OP')
+    """)
+    f = Function.from_operations(loop.operations, LoopStorage())
+    assert f.chunks[-1].filename == 'x.py'
+    assert f.filename is None

From noreply at buildbot.pypy.org  Tue Jan 17 20:46:33 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 17 Jan 2012 20:46:33 +0100 (CET)
Subject: [pypy-commit] pypy default: merge
Message-ID: <20120117194633.23413820D8@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: 
Changeset: r51420:31b1dbaa7024
Date: 2012-01-17 21:45 +0200
http://bitbucket.org/pypy/pypy/changeset/31b1dbaa7024/

Log:	merge

diff --git a/lib_pypy/numpypy/test/test_fromnumeric.py b/lib_pypy/numpypy/test/test_fromnumeric.py
--- a/lib_pypy/numpypy/test/test_fromnumeric.py
+++ b/lib_pypy/numpypy/test/test_fromnumeric.py
@@ -1,7 +1,7 @@
-
 from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
 
-class AppTestFromNumeric(BaseNumpyAppTest):     
+
+class AppTestFromNumeric(BaseNumpyAppTest):
     def test_argmax(self):
         # tests taken from numpy/core/fromnumeric.py docstring
         from numpypy import array, arange, argmax
@@ -18,12 +18,12 @@
         from numpypy import array, arange, argmin
         a = arange(6).reshape((2,3))
         assert argmin(a) == 0
-        # assert (argmax(a, axis=0) == array([0, 0, 0])).all()
-        # assert (argmax(a, axis=1) == array([0, 0])).all()
+        assert (argmin(a, axis=0) == array([0, 0, 0])).all()
+        assert (argmin(a, axis=1) == array([0, 0])).all()
         b = arange(6)
         b[1] = 0
         assert argmin(b) == 0
-   
+
     def test_shape(self):
         # tests taken from numpy/core/fromnumeric.py docstring
         from numpypy import array, identity, shape
@@ -44,7 +44,7 @@
         # assert (sum([[0, 1], [0, 5]], axis=1) == array([1, 5])).all()
         # If the accumulator is too small, overflow occurs:
         # assert ones(128, dtype=int8).sum(dtype=int8) == -128
-                                 
+
     def test_amin(self):
         # tests taken from numpy/core/fromnumeric.py docstring
         from numpypy import array, arange, amin
@@ -86,14 +86,14 @@
         assert ndim([[1,2,3],[4,5,6]]) == 2
         assert ndim(array([[1,2,3],[4,5,6]])) == 2
         assert ndim(1) == 0
-    
+
     def test_rank(self):
         # tests taken from numpy/core/fromnumeric.py docstring
         from numpypy import array, rank
         assert rank([[1,2,3],[4,5,6]]) == 2
         assert rank(array([[1,2,3],[4,5,6]])) == 2
         assert rank(1) == 0
-    
+
     def test_var(self):
         from numpypy import array, var
         a = array([[1,2],[3,4]])
@@ -107,3 +107,21 @@
         assert std(a) ==  1.1180339887498949
         # assert (std(a, axis=0) == array([ 1.,  1.])).all()
         # assert (std(a, axis=1) == array([ 0.5,  0.5]).all()
+
+    def test_mean(self):
+        from numpypy import array, mean
+        assert mean(array(range(5))) == 2.0
+        assert mean(range(5)) == 2.0
+
+    def test_reshape(self):
+        from numpypy import arange, array, dtype, reshape
+        a = arange(12)
+        b = reshape(a, (3, 4))
+        assert b.shape == (3, 4)
+        a = range(12)
+        b = reshape(a, (3, 4))
+        assert b.shape == (3, 4)
+        a = array(range(105)).reshape(3, 5, 7)
+        assert reshape(a, (1, -1)).shape == (1, 105)
+        assert reshape(a, (1, 1, -1)).shape == (1, 1, 105)
+        assert reshape(a, (-1, 1, 1)).shape == (105, 1, 1)
diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -108,6 +108,10 @@
     w_result = state.codec_search_cache.get(normalized_encoding, None)
     if w_result is not None:
         return w_result
+    return _lookup_codec_loop(space, encoding, normalized_encoding)
+
+def _lookup_codec_loop(space, encoding, normalized_encoding):
+    state = space.fromcache(CodecState)
     if state.codec_need_encodings:
         w_import = space.getattr(space.builtin, space.wrap("__import__"))
         # registers new codecs
diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -90,7 +90,6 @@
 
     appleveldefs = {
         'average': 'app_numpy.average',
-        'mean': 'app_numpy.mean',
         'sum': 'app_numpy.sum',
         'min': 'app_numpy.min',
         'identity': 'app_numpy.identity',
@@ -99,5 +98,4 @@
         'e': 'app_numpy.e',
         'pi': 'app_numpy.pi',
         'arange': 'app_numpy.arange',
-        'reshape': 'app_numpy.reshape',
     }
diff --git a/pypy/module/micronumpy/app_numpy.py b/pypy/module/micronumpy/app_numpy.py
--- a/pypy/module/micronumpy/app_numpy.py
+++ b/pypy/module/micronumpy/app_numpy.py
@@ -11,23 +11,20 @@
 def average(a):
     # This implements a weighted average, for now we don't implement the
     # weighting, just the average part!
-    return mean(a)
+    if not hasattr(a, "mean"):
+        a = _numpypy.array(a)
+    return a.mean()
 
 def identity(n, dtype=None):
-    a = _numpypy.zeros((n,n), dtype=dtype)
+    a = _numpypy.zeros((n, n), dtype=dtype)
     for i in range(n):
         a[i][i] = 1
     return a
 
-def mean(a, axis=None):
-    if not hasattr(a, "mean"):
-        a = _numpypy.array(a)
-    return a.mean(axis)
-
 def sum(a,axis=None):
     '''sum(a, axis=None)
     Sum of array elements over a given axis.
-    
+
     Parameters
     ----------
     a : array_like
@@ -35,7 +32,7 @@
     axis : integer, optional
         Axis over which the sum is taken. By default `axis` is None,
         and all elements are summed.
-    
+
     Returns
     -------
     sum_along_axis : ndarray
@@ -43,7 +40,7 @@
         axis removed.   If `a` is a 0-d array, or if `axis` is None, a scalar
         is returned.  If an output array is specified, a reference to
         `out` is returned.
-    
+
     See Also
     --------
     ndarray.sum : Equivalent method.
@@ -79,40 +76,3 @@
         arr[j] = i
         i += step
     return arr
-
-
-def reshape(a, shape):
-    '''reshape(a, newshape)
-    Gives a new shape to an array without changing its data.
-
-    Parameters
-    ----------
-    a : array_like
-        Array to be reshaped.
-    newshape : int or tuple of ints
-        The new shape should be compatible with the original shape. If
-        an integer, then the result will be a 1-D array of that length.
-        One shape dimension can be -1. In this case, the value is inferred
-        from the length of the array and remaining dimensions.
-
-    Returns
-    -------
-    reshaped_array : ndarray
-        This will be a new view object if possible; otherwise, it will
-        be a copy.
-
-
-    See Also
-    --------
-    ndarray.reshape : Equivalent method.
-
-    Notes
-    -----
-
-    It is not always possible to change the shape of an array without
-    copying the data. If you want an error to be raise if the data is copied,
-    you should assign the new shape to the shape attribute of the array
-'''
-    if not hasattr(a, 'reshape'):
-        a = _numpypy.array(a)
-    return a.reshape(shape)
diff --git a/pypy/module/micronumpy/test/test_module.py b/pypy/module/micronumpy/test/test_module.py
--- a/pypy/module/micronumpy/test/test_module.py
+++ b/pypy/module/micronumpy/test/test_module.py
@@ -2,16 +2,11 @@
 
 
 class AppTestNumPyModule(BaseNumpyAppTest):
-    def test_mean(self):
-        from _numpypy import array, mean
-        assert mean(array(range(5))) == 2.0
-        assert mean(range(5)) == 2.0
-
     def test_average(self):
         from _numpypy import array, average
         assert average(range(10)) == 4.5
         assert average(array(range(10))) == 4.5
-        
+
     def test_sum(self):
         from _numpypy import array, sum
         assert sum(range(10)) == 45
@@ -21,7 +16,7 @@
         from _numpypy import array, min
         assert min(range(10)) == 0
         assert min(array(range(10))) == 0
-        
+
     def test_max(self):
         from _numpypy import array, max
         assert max(range(10)) == 9
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -726,16 +726,16 @@
         assert d[1] == 12
 
     def test_mean(self):
-        from _numpypy import array, mean
+        from _numpypy import array
         a = array(range(5))
         assert a.mean() == 2.0
         assert a[:4].mean() == 1.5
         a = array(range(105)).reshape(3, 5, 7)
-        b = mean(a, axis=0)
-        b[0,0]==35.
+        b = a.mean(axis=0)
+        b[0, 0]==35.
         assert a.mean(axis=0)[0, 0] == 35
         assert (b == array(range(35, 70), dtype=float).reshape(5, 7)).all()
-        assert (mean(a, 2) == array(range(0, 15), dtype=float).reshape(3, 5) * 7 + 3).all()
+        assert (a.mean(2) == array(range(0, 15), dtype=float).reshape(3, 5) * 7 + 3).all()
 
     def test_sum(self):
         from _numpypy import array
@@ -1550,18 +1550,3 @@
         a = arange(0, 0.8, 0.1)
         assert len(a) == 8
         assert arange(False, True, True).dtype is dtype(int)
-
-
-class AppTestRanges(BaseNumpyAppTest):
-    def test_app_reshape(self):
-        from _numpypy import arange, array, dtype, reshape
-        a = arange(12)
-        b = reshape(a, (3, 4))
-        assert b.shape == (3, 4)
-        a = range(12)
-        b = reshape(a, (3, 4))
-        assert b.shape == (3, 4)
-        a = array(range(105)).reshape(3, 5, 7)
-        assert a.reshape(1, -1).shape == (1, 105)
-        assert a.reshape(1, 1, -1).shape == (1, 1, 105)
-        assert a.reshape(-1, 1, 1).shape == (105, 1, 1)

From noreply at buildbot.pypy.org  Tue Jan 17 21:46:23 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 17 Jan 2012 21:46:23 +0100 (CET)
Subject: [pypy-commit] pypy better-jit-hooks-2: dmp creation
Message-ID: <20120117204623.16E8A820D8@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: better-jit-hooks-2
Changeset: r51421:0513eb237097
Date: 2012-01-17 22:25 +0200
http://bitbucket.org/pypy/pypy/changeset/0513eb237097/

Log:	dmp creation

diff --git a/pypy/module/pypyjit/__init__.py b/pypy/module/pypyjit/__init__.py
--- a/pypy/module/pypyjit/__init__.py
+++ b/pypy/module/pypyjit/__init__.py
@@ -11,6 +11,7 @@
         'set_optimize_hook': 'interp_resop.set_optimize_hook',
         'set_abort_hook': 'interp_resop.set_abort_hook',
         'ResOperation': 'interp_resop.WrappedOp',
+        'DebugMergePoint': 'interp_resop.DebugMergePoint',
         'Box': 'interp_resop.WrappedBox',
     }
 
diff --git a/pypy/module/pypyjit/interp_resop.py b/pypy/module/pypyjit/interp_resop.py
--- a/pypy/module/pypyjit/interp_resop.py
+++ b/pypy/module/pypyjit/interp_resop.py
@@ -1,5 +1,6 @@
 
-from pypy.interpreter.typedef import TypeDef, GetSetProperty
+from pypy.interpreter.typedef import TypeDef, GetSetProperty,\
+     interp_attrproperty
 from pypy.interpreter.baseobjspace import Wrappable
 from pypy.interpreter.gateway import unwrap_spec, interp2app, NoneNotWrapped
 from pypy.interpreter.pycode import PyCode
@@ -150,6 +151,14 @@
         llres = res.llbox
     return WrappedOp(jit_hooks.resop_new(num, args, llres), offset, repr)
 
+ at unwrap_spec(repr=str, pycode=PyCode, bytecode_no=int)
+def descr_new_dmp(space, w_tp, w_args, repr, pycode, bytecode_no):
+    args = [space.interp_w(WrappedBox, w_arg).llbox for w_arg in
+            space.listview(w_args)]
+    num = rop.DEBUG_MERGE_POINT
+    return DebugMergePoint(jit_hooks.resop_new(num, args, jit_hooks.emptyval()),
+                           repr, pycode, bytecode_no)
+
 class WrappedOp(Wrappable):
     """ A class representing a single ResOperation, wrapped nicely
     """
@@ -182,6 +191,17 @@
         box = space.interp_w(WrappedBox, w_box)
         jit_hooks.resop_setresult(self.op, box.llbox)
 
+class DebugMergePoint(WrappedOp):
+    def __init__(self, op, repr_of_resop, pycode, bytecode_no):
+        WrappedOp.__init__(self, op, -1, repr_of_resop)
+        self.pycode = pycode
+        self.bytecode_no = bytecode_no
+
+    def get_bytecode_no(self, space):
+        if self.pycode is None:
+            return space.w_None
+        return space.wrap(self.bytecode_no)
+
 WrappedOp.typedef = TypeDef(
     'ResOperation',
     __doc__ = WrappedOp.__doc__,
@@ -195,3 +215,13 @@
                             WrappedOp.descr_setresult)
 )
 WrappedOp.acceptable_as_base_class = False
+
+DebugMergePoint.typedef = TypeDef(
+    'DebugMergePoint', WrappedOp.typedef,
+    __new__ = interp2app(descr_new_dmp),
+    pycode = interp_attrproperty('pycode', cls=DebugMergePoint),
+    bytecode_no = GetSetProperty(DebugMergePoint.get_bytecode_no),
+)
+DebugMergePoint.acceptable_as_base_class = False
+
+                   
diff --git a/pypy/module/pypyjit/test/test_jit_hook.py b/pypy/module/pypyjit/test/test_jit_hook.py
--- a/pypy/module/pypyjit/test/test_jit_hook.py
+++ b/pypy/module/pypyjit/test/test_jit_hook.py
@@ -92,6 +92,7 @@
         cls.w_on_compile_bridge = space.wrap(interp2app(interp_on_compile_bridge))
         cls.w_on_abort = space.wrap(interp2app(interp_on_abort))
         cls.w_int_add_num = space.wrap(rop.INT_ADD)
+        cls.w_dmp_num = space.wrap(rop.DEBUG_MERGE_POINT)
         cls.w_on_optimize = space.wrap(interp2app(interp_on_optimize))
         cls.orig_oplist = oplist
 
@@ -211,3 +212,14 @@
         assert op.getarg(0).getint() == 4
         op.result = box
         assert op.result.getint() == 1
+
+    def test_creation_dmp(self):
+        from pypyjit import DebugMergePoint, Box
+
+        def f():
+            pass
+
+        op = DebugMergePoint([Box(0)], '', f.func_code, 0)
+        assert op.bytecode_no == 0
+        assert op.pycode is f.func_code
+        assert op.num == self.dmp_num
diff --git a/pypy/rlib/jit_hooks.py b/pypy/rlib/jit_hooks.py
--- a/pypy/rlib/jit_hooks.py
+++ b/pypy/rlib/jit_hooks.py
@@ -52,7 +52,10 @@
     from pypy.jit.metainterp.history import ResOperation
 
     args = [_cast_to_box(llargs[i]) for i in range(len(llargs))]
-    res = _cast_to_box(llres)
+    if llres:
+        res = _cast_to_box(llres)
+    else:
+        res = None
     return _cast_to_gcref(ResOperation(no, args, res))
 
 @register_helper(annmodel.SomePtr(llmemory.GCREF))

From noreply at buildbot.pypy.org  Tue Jan 17 21:46:24 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 17 Jan 2012 21:46:24 +0100 (CET)
Subject: [pypy-commit] pypy better-jit-hooks-2: implement jithooks
Message-ID: <20120117204624.4BD1A820D8@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: better-jit-hooks-2
Changeset: r51422:1ef43f689d9b
Date: 2012-01-17 22:45 +0200
http://bitbucket.org/pypy/pypy/changeset/1ef43f689d9b/

Log:	implement jithooks

diff --git a/pypy/module/pypyjit/interp_resop.py b/pypy/module/pypyjit/interp_resop.py
--- a/pypy/module/pypyjit/interp_resop.py
+++ b/pypy/module/pypyjit/interp_resop.py
@@ -11,6 +11,7 @@
 from pypy.jit.metainterp.resoperation import rop, AbstractResOp
 from pypy.rlib.nonconst import NonConstant
 from pypy.rlib import jit_hooks
+from pypy.module.pypyjit.interp_jit import pypyjitdriver
 
 class Cache(object):
     in_recursion = False
@@ -112,13 +113,24 @@
 
 def wrap_oplist(space, logops, operations, ops_offset=None):
     l_w = []
+    jitdrivers_sd = logops.metainterp_sd.jitdrivers_sd
     for op in operations:
         if ops_offset is None:
             ofs = -1
         else:
             ofs = ops_offset.get(op, 0)
-        l_w.append(WrappedOp(jit_hooks._cast_to_gcref(op), ofs,
-                             logops.repr_of_resop(op)))
+        if op.opnum == rop.DEBUG_MERGE_POINT:
+            jd_sd = jitdrivers_sd[op.getarg(0).getint()]
+            greenkey = op.getarglist()[2:]
+            repr = jd_sd.warmstate.get_location_str(greenkey)
+            w_descr = wrap_greenkey(space, jd_sd.jitdriver, greenkey, repr)
+            l_w.append(DebugMergePoint(space, jit_hooks._cast_to_gcref(op),
+                                       logops.repr_of_resop(op),
+                                       jd_sd.jitdriver.name,
+                                       w_descr))
+        else:
+            l_w.append(WrappedOp(jit_hooks._cast_to_gcref(op), ofs,
+                                 logops.repr_of_resop(op)))
     return l_w
 
 class WrappedBox(Wrappable):
@@ -151,13 +163,14 @@
         llres = res.llbox
     return WrappedOp(jit_hooks.resop_new(num, args, llres), offset, repr)
 
- at unwrap_spec(repr=str, pycode=PyCode, bytecode_no=int)
-def descr_new_dmp(space, w_tp, w_args, repr, pycode, bytecode_no):
+ at unwrap_spec(repr=str, jd_name=str)
+def descr_new_dmp(space, w_tp, w_args, repr, jd_name, w_descr):
     args = [space.interp_w(WrappedBox, w_arg).llbox for w_arg in
             space.listview(w_args)]
     num = rop.DEBUG_MERGE_POINT
-    return DebugMergePoint(jit_hooks.resop_new(num, args, jit_hooks.emptyval()),
-                           repr, pycode, bytecode_no)
+    return DebugMergePoint(space,
+                           jit_hooks.resop_new(num, args, jit_hooks.emptyval()),
+                           repr, jd_name, w_descr)
 
 class WrappedOp(Wrappable):
     """ A class representing a single ResOperation, wrapped nicely
@@ -192,15 +205,26 @@
         jit_hooks.resop_setresult(self.op, box.llbox)
 
 class DebugMergePoint(WrappedOp):
-    def __init__(self, op, repr_of_resop, pycode, bytecode_no):
+    def __init__(self, space, op, repr_of_resop, jd_name, w_descr):
         WrappedOp.__init__(self, op, -1, repr_of_resop)
-        self.pycode = pycode
-        self.bytecode_no = bytecode_no
+        self.w_descr = w_descr
+        self.jd_name = jd_name
+
+    def get_descr(self, space):
+        return self.w_descr
+
+    def get_pycode(self, space):
+        if self.jd_name == pypyjitdriver.name:
+            return space.getitem(self.w_descr, space.wrap(0))
+        raise OperationError(space.w_AttributeError, space.wrap('DebugMergePoint is not the main jitdriver DMP'))
 
     def get_bytecode_no(self, space):
-        if self.pycode is None:
-            return space.w_None
-        return space.wrap(self.bytecode_no)
+        if self.jd_name == pypyjitdriver.name:
+            return space.getitem(self.w_descr, space.wrap(1))
+        raise OperationError(space.w_AttributeError, space.wrap('DebugMergePoint is not the main jitdriver DMP'))
+
+    def get_jitdriver(self, space):
+        return space.wrap(self.jd_name)
 
 WrappedOp.typedef = TypeDef(
     'ResOperation',
@@ -219,8 +243,10 @@
 DebugMergePoint.typedef = TypeDef(
     'DebugMergePoint', WrappedOp.typedef,
     __new__ = interp2app(descr_new_dmp),
-    pycode = interp_attrproperty('pycode', cls=DebugMergePoint),
+    descr = GetSetProperty(DebugMergePoint.get_descr),
+    pycode = GetSetProperty(DebugMergePoint.get_pycode),
     bytecode_no = GetSetProperty(DebugMergePoint.get_bytecode_no),
+    jitdriver_name = GetSetProperty(DebugMergePoint.get_jitdriver),
 )
 DebugMergePoint.acceptable_as_base_class = False
 
diff --git a/pypy/module/pypyjit/test/test_jit_hook.py b/pypy/module/pypyjit/test/test_jit_hook.py
--- a/pypy/module/pypyjit/test/test_jit_hook.py
+++ b/pypy/module/pypyjit/test/test_jit_hook.py
@@ -118,6 +118,9 @@
         assert elem[2][2] == False
         assert len(elem[3]) == 4
         int_add = elem[3][0]
+        dmp = elem[3][1]
+        assert isinstance(dmp, pypyjit.DebugMergePoint)
+        assert dmp.pycode is self.f.func_code
         #assert int_add.name == 'int_add'
         assert int_add.num == self.int_add_num
         self.on_compile_bridge()
@@ -219,7 +222,11 @@
         def f():
             pass
 
-        op = DebugMergePoint([Box(0)], '', f.func_code, 0)
+        op = DebugMergePoint([Box(0)], 'repr', 'pypyjit', (f.func_code, 0, 0))
         assert op.bytecode_no == 0
         assert op.pycode is f.func_code
+        assert repr(op) == 'repr'
+        assert op.jitdriver_name == 'pypyjit'
         assert op.num == self.dmp_num
+        op = DebugMergePoint([Box(0)], 'repr', 'notmain', ('str',))
+        raises(AttributeError, 'op.pycode')

From noreply at buildbot.pypy.org  Tue Jan 17 22:17:22 2012
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Tue, 17 Jan 2012 22:17:22 +0100 (CET)
Subject: [pypy-commit] pypy better-jit-hooks-2: Some naming changes.
Message-ID: <20120117211722.7FBBB820D8@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor 
Branch: better-jit-hooks-2
Changeset: r51423:b906e1141138
Date: 2012-01-17 15:17 -0600
http://bitbucket.org/pypy/pypy/changeset/b906e1141138/

Log:	Some naming changes.

diff --git a/pypy/module/pypyjit/interp_resop.py b/pypy/module/pypyjit/interp_resop.py
--- a/pypy/module/pypyjit/interp_resop.py
+++ b/pypy/module/pypyjit/interp_resop.py
@@ -216,14 +216,14 @@
     def get_pycode(self, space):
         if self.jd_name == pypyjitdriver.name:
             return space.getitem(self.w_descr, space.wrap(0))
-        raise OperationError(space.w_AttributeError, space.wrap('DebugMergePoint is not the main jitdriver DMP'))
+        raise OperationError(space.w_AttributeError, space.wrap("This DebugMergePoint doesn't belong to the main Python JitDriver"))
 
     def get_bytecode_no(self, space):
         if self.jd_name == pypyjitdriver.name:
             return space.getitem(self.w_descr, space.wrap(1))
-        raise OperationError(space.w_AttributeError, space.wrap('DebugMergePoint is not the main jitdriver DMP'))
+        raise OperationError(space.w_AttributeError, space.wrap("This DebugMergePoint doesn't belong to the main Python JitDriver"))
 
-    def get_jitdriver(self, space):
+    def get_jitdriver_name(self, space):
         return space.wrap(self.jd_name)
 
 WrappedOp.typedef = TypeDef(
@@ -246,7 +246,7 @@
     descr = GetSetProperty(DebugMergePoint.get_descr),
     pycode = GetSetProperty(DebugMergePoint.get_pycode),
     bytecode_no = GetSetProperty(DebugMergePoint.get_bytecode_no),
-    jitdriver_name = GetSetProperty(DebugMergePoint.get_jitdriver),
+    jitdriver_name = GetSetProperty(DebugMergePoint.get_jitdriver_name),
 )
 DebugMergePoint.acceptable_as_base_class = False
 

From noreply at buildbot.pypy.org  Tue Jan 17 22:27:44 2012
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Tue, 17 Jan 2012 22:27:44 +0100 (CET)
Subject: [pypy-commit] pypy better-jit-hooks-2: change DMP.descr to
	DMP.greenkey
Message-ID: <20120117212744.5C04A820D8@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor 
Branch: better-jit-hooks-2
Changeset: r51424:6354270bb576
Date: 2012-01-17 15:27 -0600
http://bitbucket.org/pypy/pypy/changeset/6354270bb576/

Log:	change DMP.descr to DMP.greenkey

diff --git a/pypy/module/pypyjit/interp_resop.py b/pypy/module/pypyjit/interp_resop.py
--- a/pypy/module/pypyjit/interp_resop.py
+++ b/pypy/module/pypyjit/interp_resop.py
@@ -1,6 +1,6 @@
 
-from pypy.interpreter.typedef import TypeDef, GetSetProperty,\
-     interp_attrproperty
+from pypy.interpreter.typedef import (TypeDef, GetSetProperty,
+     interp_attrproperty, interp_attrproperty_w)
 from pypy.interpreter.baseobjspace import Wrappable
 from pypy.interpreter.gateway import unwrap_spec, interp2app, NoneNotWrapped
 from pypy.interpreter.pycode import PyCode
@@ -123,11 +123,11 @@
             jd_sd = jitdrivers_sd[op.getarg(0).getint()]
             greenkey = op.getarglist()[2:]
             repr = jd_sd.warmstate.get_location_str(greenkey)
-            w_descr = wrap_greenkey(space, jd_sd.jitdriver, greenkey, repr)
+            w_greenkey = wrap_greenkey(space, jd_sd.jitdriver, greenkey, repr)
             l_w.append(DebugMergePoint(space, jit_hooks._cast_to_gcref(op),
                                        logops.repr_of_resop(op),
                                        jd_sd.jitdriver.name,
-                                       w_descr))
+                                       w_greenkey))
         else:
             l_w.append(WrappedOp(jit_hooks._cast_to_gcref(op), ofs,
                                  logops.repr_of_resop(op)))
@@ -164,13 +164,13 @@
     return WrappedOp(jit_hooks.resop_new(num, args, llres), offset, repr)
 
 @unwrap_spec(repr=str, jd_name=str)
-def descr_new_dmp(space, w_tp, w_args, repr, jd_name, w_descr):
+def descr_new_dmp(space, w_tp, w_args, repr, jd_name, w_greenkey):
     args = [space.interp_w(WrappedBox, w_arg).llbox for w_arg in
             space.listview(w_args)]
     num = rop.DEBUG_MERGE_POINT
     return DebugMergePoint(space,
                            jit_hooks.resop_new(num, args, jit_hooks.emptyval()),
-                           repr, jd_name, w_descr)
+                           repr, jd_name, w_greenkey)
 
 class WrappedOp(Wrappable):
     """ A class representing a single ResOperation, wrapped nicely
@@ -205,22 +205,19 @@
         jit_hooks.resop_setresult(self.op, box.llbox)
 
 class DebugMergePoint(WrappedOp):
-    def __init__(self, space, op, repr_of_resop, jd_name, w_descr):
+    def __init__(self, space, op, repr_of_resop, jd_name, w_greenkey):
         WrappedOp.__init__(self, op, -1, repr_of_resop)
-        self.w_descr = w_descr
+        self.w_greenkey = w_greenkey
         self.jd_name = jd_name
 
-    def get_descr(self, space):
-        return self.w_descr
-
     def get_pycode(self, space):
         if self.jd_name == pypyjitdriver.name:
-            return space.getitem(self.w_descr, space.wrap(0))
+            return space.getitem(self.w_greenkey, space.wrap(0))
         raise OperationError(space.w_AttributeError, space.wrap("This DebugMergePoint doesn't belong to the main Python JitDriver"))
 
     def get_bytecode_no(self, space):
         if self.jd_name == pypyjitdriver.name:
-            return space.getitem(self.w_descr, space.wrap(1))
+            return space.getitem(self.w_greenkey, space.wrap(1))
         raise OperationError(space.w_AttributeError, space.wrap("This DebugMergePoint doesn't belong to the main Python JitDriver"))
 
     def get_jitdriver_name(self, space):
@@ -243,11 +240,11 @@
 DebugMergePoint.typedef = TypeDef(
     'DebugMergePoint', WrappedOp.typedef,
     __new__ = interp2app(descr_new_dmp),
-    descr = GetSetProperty(DebugMergePoint.get_descr),
+    greenkey = interp_attrproperty_w("w_greenkey", cls=DebugMergePoint),
     pycode = GetSetProperty(DebugMergePoint.get_pycode),
     bytecode_no = GetSetProperty(DebugMergePoint.get_bytecode_no),
     jitdriver_name = GetSetProperty(DebugMergePoint.get_jitdriver_name),
 )
 DebugMergePoint.acceptable_as_base_class = False
 
-                   
+
diff --git a/pypy/module/pypyjit/test/test_jit_hook.py b/pypy/module/pypyjit/test/test_jit_hook.py
--- a/pypy/module/pypyjit/test/test_jit_hook.py
+++ b/pypy/module/pypyjit/test/test_jit_hook.py
@@ -121,6 +121,7 @@
         dmp = elem[3][1]
         assert isinstance(dmp, pypyjit.DebugMergePoint)
         assert dmp.pycode is self.f.func_code
+        assert dmp.greenkey == (self.f.func_code, 0, False)
         #assert int_add.name == 'int_add'
         assert int_add.num == self.int_add_num
         self.on_compile_bridge()

From noreply at buildbot.pypy.org  Tue Jan 17 22:30:52 2012
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Tue, 17 Jan 2012 22:30:52 +0100 (CET)
Subject: [pypy-commit] pypy better-jit-hooks-2: Close branch for merge
Message-ID: <20120117213052.8BC1C820D8@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor 
Branch: better-jit-hooks-2
Changeset: r51425:886f6fbce8ef
Date: 2012-01-17 15:29 -0600
http://bitbucket.org/pypy/pypy/changeset/886f6fbce8ef/

Log:	Close branch for merge


From noreply at buildbot.pypy.org  Tue Jan 17 22:30:53 2012
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Tue, 17 Jan 2012 22:30:53 +0100 (CET)
Subject: [pypy-commit] pypy default: (fijal mostly) Merged
	better-jit-hooks-2.
Message-ID: <20120117213053.C276B820D8@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor 
Branch: 
Changeset: r51426:c6ea8ef23064
Date: 2012-01-17 15:30 -0600
http://bitbucket.org/pypy/pypy/changeset/c6ea8ef23064/

Log:	(fijal mostly) Merged better-jit-hooks-2.

	This exposes metadata about DebugMergePoints in jit hooks,
	specifically the current bytecode and the pycode object.

diff --git a/pypy/module/pypyjit/__init__.py b/pypy/module/pypyjit/__init__.py
--- a/pypy/module/pypyjit/__init__.py
+++ b/pypy/module/pypyjit/__init__.py
@@ -11,6 +11,7 @@
         'set_optimize_hook': 'interp_resop.set_optimize_hook',
         'set_abort_hook': 'interp_resop.set_abort_hook',
         'ResOperation': 'interp_resop.WrappedOp',
+        'DebugMergePoint': 'interp_resop.DebugMergePoint',
         'Box': 'interp_resop.WrappedBox',
     }
 
diff --git a/pypy/module/pypyjit/interp_resop.py b/pypy/module/pypyjit/interp_resop.py
--- a/pypy/module/pypyjit/interp_resop.py
+++ b/pypy/module/pypyjit/interp_resop.py
@@ -1,5 +1,6 @@
 
-from pypy.interpreter.typedef import TypeDef, GetSetProperty
+from pypy.interpreter.typedef import (TypeDef, GetSetProperty,
+     interp_attrproperty, interp_attrproperty_w)
 from pypy.interpreter.baseobjspace import Wrappable
 from pypy.interpreter.gateway import unwrap_spec, interp2app, NoneNotWrapped
 from pypy.interpreter.pycode import PyCode
@@ -10,6 +11,7 @@
 from pypy.jit.metainterp.resoperation import rop, AbstractResOp
 from pypy.rlib.nonconst import NonConstant
 from pypy.rlib import jit_hooks
+from pypy.module.pypyjit.interp_jit import pypyjitdriver
 
 class Cache(object):
     in_recursion = False
@@ -111,13 +113,24 @@
 
 def wrap_oplist(space, logops, operations, ops_offset=None):
     l_w = []
+    jitdrivers_sd = logops.metainterp_sd.jitdrivers_sd
     for op in operations:
         if ops_offset is None:
             ofs = -1
         else:
             ofs = ops_offset.get(op, 0)
-        l_w.append(WrappedOp(jit_hooks._cast_to_gcref(op), ofs,
-                             logops.repr_of_resop(op)))
+        if op.opnum == rop.DEBUG_MERGE_POINT:
+            jd_sd = jitdrivers_sd[op.getarg(0).getint()]
+            greenkey = op.getarglist()[2:]
+            repr = jd_sd.warmstate.get_location_str(greenkey)
+            w_greenkey = wrap_greenkey(space, jd_sd.jitdriver, greenkey, repr)
+            l_w.append(DebugMergePoint(space, jit_hooks._cast_to_gcref(op),
+                                       logops.repr_of_resop(op),
+                                       jd_sd.jitdriver.name,
+                                       w_greenkey))
+        else:
+            l_w.append(WrappedOp(jit_hooks._cast_to_gcref(op), ofs,
+                                 logops.repr_of_resop(op)))
     return l_w
 
 class WrappedBox(Wrappable):
@@ -150,6 +163,15 @@
         llres = res.llbox
     return WrappedOp(jit_hooks.resop_new(num, args, llres), offset, repr)
 
+ at unwrap_spec(repr=str, jd_name=str)
+def descr_new_dmp(space, w_tp, w_args, repr, jd_name, w_greenkey):
+    args = [space.interp_w(WrappedBox, w_arg).llbox for w_arg in
+            space.listview(w_args)]
+    num = rop.DEBUG_MERGE_POINT
+    return DebugMergePoint(space,
+                           jit_hooks.resop_new(num, args, jit_hooks.emptyval()),
+                           repr, jd_name, w_greenkey)
+
 class WrappedOp(Wrappable):
     """ A class representing a single ResOperation, wrapped nicely
     """
@@ -182,6 +204,25 @@
         box = space.interp_w(WrappedBox, w_box)
         jit_hooks.resop_setresult(self.op, box.llbox)
 
+class DebugMergePoint(WrappedOp):
+    def __init__(self, space, op, repr_of_resop, jd_name, w_greenkey):
+        WrappedOp.__init__(self, op, -1, repr_of_resop)
+        self.w_greenkey = w_greenkey
+        self.jd_name = jd_name
+
+    def get_pycode(self, space):
+        if self.jd_name == pypyjitdriver.name:
+            return space.getitem(self.w_greenkey, space.wrap(0))
+        raise OperationError(space.w_AttributeError, space.wrap("This DebugMergePoint doesn't belong to the main Python JitDriver"))
+
+    def get_bytecode_no(self, space):
+        if self.jd_name == pypyjitdriver.name:
+            return space.getitem(self.w_greenkey, space.wrap(1))
+        raise OperationError(space.w_AttributeError, space.wrap("This DebugMergePoint doesn't belong to the main Python JitDriver"))
+
+    def get_jitdriver_name(self, space):
+        return space.wrap(self.jd_name)
+
 WrappedOp.typedef = TypeDef(
     'ResOperation',
     __doc__ = WrappedOp.__doc__,
@@ -195,3 +236,15 @@
                             WrappedOp.descr_setresult)
 )
 WrappedOp.acceptable_as_base_class = False
+
+DebugMergePoint.typedef = TypeDef(
+    'DebugMergePoint', WrappedOp.typedef,
+    __new__ = interp2app(descr_new_dmp),
+    greenkey = interp_attrproperty_w("w_greenkey", cls=DebugMergePoint),
+    pycode = GetSetProperty(DebugMergePoint.get_pycode),
+    bytecode_no = GetSetProperty(DebugMergePoint.get_bytecode_no),
+    jitdriver_name = GetSetProperty(DebugMergePoint.get_jitdriver_name),
+)
+DebugMergePoint.acceptable_as_base_class = False
+
+
diff --git a/pypy/module/pypyjit/test/test_jit_hook.py b/pypy/module/pypyjit/test/test_jit_hook.py
--- a/pypy/module/pypyjit/test/test_jit_hook.py
+++ b/pypy/module/pypyjit/test/test_jit_hook.py
@@ -92,6 +92,7 @@
         cls.w_on_compile_bridge = space.wrap(interp2app(interp_on_compile_bridge))
         cls.w_on_abort = space.wrap(interp2app(interp_on_abort))
         cls.w_int_add_num = space.wrap(rop.INT_ADD)
+        cls.w_dmp_num = space.wrap(rop.DEBUG_MERGE_POINT)
         cls.w_on_optimize = space.wrap(interp2app(interp_on_optimize))
         cls.orig_oplist = oplist
 
@@ -117,6 +118,10 @@
         assert elem[2][2] == False
         assert len(elem[3]) == 4
         int_add = elem[3][0]
+        dmp = elem[3][1]
+        assert isinstance(dmp, pypyjit.DebugMergePoint)
+        assert dmp.pycode is self.f.func_code
+        assert dmp.greenkey == (self.f.func_code, 0, False)
         #assert int_add.name == 'int_add'
         assert int_add.num == self.int_add_num
         self.on_compile_bridge()
@@ -211,3 +216,18 @@
         assert op.getarg(0).getint() == 4
         op.result = box
         assert op.result.getint() == 1
+
+    def test_creation_dmp(self):
+        from pypyjit import DebugMergePoint, Box
+
+        def f():
+            pass
+
+        op = DebugMergePoint([Box(0)], 'repr', 'pypyjit', (f.func_code, 0, 0))
+        assert op.bytecode_no == 0
+        assert op.pycode is f.func_code
+        assert repr(op) == 'repr'
+        assert op.jitdriver_name == 'pypyjit'
+        assert op.num == self.dmp_num
+        op = DebugMergePoint([Box(0)], 'repr', 'notmain', ('str',))
+        raises(AttributeError, 'op.pycode')
diff --git a/pypy/rlib/jit_hooks.py b/pypy/rlib/jit_hooks.py
--- a/pypy/rlib/jit_hooks.py
+++ b/pypy/rlib/jit_hooks.py
@@ -52,7 +52,10 @@
     from pypy.jit.metainterp.history import ResOperation
 
     args = [_cast_to_box(llargs[i]) for i in range(len(llargs))]
-    res = _cast_to_box(llres)
+    if llres:
+        res = _cast_to_box(llres)
+    else:
+        res = None
     return _cast_to_gcref(ResOperation(no, args, res))
 
 @register_helper(annmodel.SomePtr(llmemory.GCREF))

From noreply at buildbot.pypy.org  Wed Jan 18 00:11:10 2012
From: noreply at buildbot.pypy.org (mikefc)
Date: Wed, 18 Jan 2012 00:11:10 +0100 (CET)
Subject: [pypy-commit] pypy default: Move fromnumeric into core/
 subdirectory to follow numpy's layout
Message-ID: <20120117231110.20F63710260@wyvern.cs.uni-duesseldorf.de>

Author: mikefc
Branch: 
Changeset: r51428:e4314d54ea7f
Date: 2012-01-18 09:10 +1000
http://bitbucket.org/pypy/pypy/changeset/e4314d54ea7f/

Log:	Move fromnumeric into core/ subdirectory to follow numpy's layout

diff --git a/lib_pypy/numpypy/__init__.py b/lib_pypy/numpypy/__init__.py
--- a/lib_pypy/numpypy/__init__.py
+++ b/lib_pypy/numpypy/__init__.py
@@ -1,2 +1,2 @@
 from _numpypy import *
-from .fromnumeric import *
+from .core import *
diff --git a/lib_pypy/numpypy/core/__init__.py b/lib_pypy/numpypy/core/__init__.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/numpypy/core/__init__.py
@@ -0,0 +1,1 @@
+from .fromnumeric import *
diff --git a/lib_pypy/numpypy/fromnumeric.py b/lib_pypy/numpypy/core/fromnumeric.py
rename from lib_pypy/numpypy/fromnumeric.py
rename to lib_pypy/numpypy/core/fromnumeric.py

From noreply at buildbot.pypy.org  Wed Jan 18 00:29:52 2012
From: noreply at buildbot.pypy.org (mikefc)
Date: Wed, 18 Jan 2012 00:29:52 +0100 (CET)
Subject: [pypy-commit] pypy default: Add numpypy.transpose() function to
 call the array method
Message-ID: <20120117232952.5E205710260@wyvern.cs.uni-duesseldorf.de>

Author: mikefc
Branch: 
Changeset: r51429:bc30fb8eed3e
Date: 2012-01-18 09:29 +1000
http://bitbucket.org/pypy/pypy/changeset/bc30fb8eed3e/

Log:	Add numpypy.transpose() function to call the array method

diff --git a/lib_pypy/numpypy/core/fromnumeric.py b/lib_pypy/numpypy/core/fromnumeric.py
--- a/lib_pypy/numpypy/core/fromnumeric.py
+++ b/lib_pypy/numpypy/core/fromnumeric.py
@@ -451,8 +451,11 @@
     (2, 1, 3)
 
     """
-    raise NotImplemented('Waiting on interp level method')
-
+    if axes is not None:
+        raise NotImplementedError('No "axes" arg yet.')
+    if not hasattr(a, 'T'):
+        a = numpypy.array(a)
+    return a.T
 
 def sort(a, axis=-1, kind='quicksort', order=None):
     """
diff --git a/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
--- a/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
+++ b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
@@ -125,3 +125,13 @@
         assert reshape(a, (1, -1)).shape == (1, 105)
         assert reshape(a, (1, 1, -1)).shape == (1, 1, 105)
         assert reshape(a, (-1, 1, 1)).shape == (105, 1, 1)
+    
+    def test_transpose(self):   
+        from numpypy import arange, array, transpose, ones
+        x = arange(4).reshape((2,2))
+        assert (transpose(x) == array([[0, 2],[1, 3]])).all()
+        # Once axes argument is implemented, add more tests
+        raises(NotImplementedError, "transpose(x, axes=(1, 0, 2))")
+        # x = ones((1, 2, 3))
+        # assert transpose(x, (1, 0, 2)).shape == (2, 1, 3)
+

From noreply at buildbot.pypy.org  Wed Jan 18 00:32:39 2012
From: noreply at buildbot.pypy.org (mikefc)
Date: Wed, 18 Jan 2012 00:32:39 +0100 (CET)
Subject: [pypy-commit] pypy default: Use the proper exception for
	NotImplementedError
Message-ID: <20120117233239.C8C0B710260@wyvern.cs.uni-duesseldorf.de>

Author: mikefc
Branch: 
Changeset: r51430:07bd76d923fd
Date: 2012-01-18 09:32 +1000
http://bitbucket.org/pypy/pypy/changeset/07bd76d923fd/

Log:	Use the proper exception for NotImplementedError

diff --git a/lib_pypy/numpypy/core/fromnumeric.py b/lib_pypy/numpypy/core/fromnumeric.py
--- a/lib_pypy/numpypy/core/fromnumeric.py
+++ b/lib_pypy/numpypy/core/fromnumeric.py
@@ -85,7 +85,7 @@
     array([4, 3, 6])
 
     """
-    raise NotImplemented('Waiting on interp level method')
+    raise NotImplementedError('Waiting on interp level method')
 
 
 # not deprecated --- copy if necessary, view otherwise
@@ -273,7 +273,7 @@
             [-1, -2, -3, -4, -5]]])
 
     """
-    raise NotImplemented('Waiting on interp level method')
+    raise NotImplementedError('Waiting on interp level method')
 
 
 def repeat(a, repeats, axis=None):
@@ -315,7 +315,7 @@
            [3, 4]])
 
     """
-    raise NotImplemented('Waiting on interp level method')
+    raise NotImplementedError('Waiting on interp level method')
 
 
 def put(a, ind, v, mode='raise'):
@@ -366,7 +366,7 @@
     array([ 0,  1,  2,  3, -5])
 
     """
-    raise NotImplemented('Waiting on interp level method')
+    raise NotImplementedError('Waiting on interp level method')
 
 
 def swapaxes(a, axis1, axis2):
@@ -410,7 +410,7 @@
             [3, 7]]])
 
     """
-    raise NotImplemented('Waiting on interp level method')
+    raise NotImplementedError('Waiting on interp level method')
 
 
 def transpose(a, axes=None):
@@ -556,7 +556,7 @@
           dtype=[('name', '|S10'), ('height', '

Author: Maciej Fijalkowski 
Branch: extradoc
Changeset: r4043:5ddb005ca8e2
Date: 2012-01-18 01:40 +0200
http://bitbucket.org/pypy/extradoc/changeset/5ddb005ca8e2/

Log:	some made up claims

diff --git a/blog/draft/pypy-2011.rst b/blog/draft/pypy-2011.rst
--- a/blog/draft/pypy-2011.rst
+++ b/blog/draft/pypy-2011.rst
@@ -24,7 +24,10 @@
 * We made PyPy over 2x faster (on a select set of benchmarks),
   `1.4 and nightly`_ comparison
 
+* We made PyPy 17x more compatible (of course this is unmeasurable, so why
+  not just claim it)
 
-
+* We probably have [XXX insert unicode for infinity] infinitely many times more users now, since noone used
+  1.4 in production
 
 .. _`1.4 and nightly` http://speed.pypy.org/comparison/?exe=1%2B172%2C1%2BL%2Bdefault&ben=1%2C34%2C27%2C2%2C25%2C3%2C4%2C5%2C22%2C6%2C39%2C7%2C8%2C23%2C24%2C9%2C10%2C11%2C12%2C13%2C14%2C15%2C35%2C36%2C37%2C38%2C16%2C28%2C30%2C32%2C29%2C33%2C17%2C18%2C19%2C20&env=1%2C2&hor=true&bas=1%2B172&chart=normal+bars

From noreply at buildbot.pypy.org  Wed Jan 18 00:53:54 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 18 Jan 2012 00:53:54 +0100 (CET)
Subject: [pypy-commit] benchmarks default: unbreak benchmarks
Message-ID: <20120117235354.3C98E71063B@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: 
Changeset: r160:b6376749f6e5
Date: 2012-01-18 01:53 +0200
http://bitbucket.org/pypy/benchmarks/changeset/b6376749f6e5/

Log:	unbreak benchmarks

diff --git a/benchmarks.py b/benchmarks.py
--- a/benchmarks.py
+++ b/benchmarks.py
@@ -153,7 +153,10 @@
     for python in [base_python, changed_python]:
         maindir = relative('lib/cpython-doc')
         builddir = os.path.join(os.path.join(maindir, 'tools'), 'build')
-        shutil.rmtree(builddir)
+        try:
+            shutil.rmtree(builddir)
+        except OSError:
+            pass
         build = relative('lib/cpython-doc/tools/sphinx-build.py')
         os.mkdir(builddir)
         docdir = os.path.join(builddir, 'doctrees')

From noreply at buildbot.pypy.org  Wed Jan 18 00:38:35 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 18 Jan 2012 00:38:35 +0100 (CET)
Subject: [pypy-commit] extradoc extradoc: start drafting blog post
Message-ID: <20120117233835.398AD710260@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: extradoc
Changeset: r4042:7288adc71948
Date: 2012-01-18 01:38 +0200
http://bitbucket.org/pypy/extradoc/changeset/7288adc71948/

Log:	start drafting blog post

diff --git a/blog/draft/pypy-2011.rst b/blog/draft/pypy-2011.rst
new file mode 100644
--- /dev/null
+++ b/blog/draft/pypy-2011.rst
@@ -0,0 +1,30 @@
+PyPy development in 2011
+========================
+
+Hello everyone.
+
+PyPy development is fast, sometimes very fast. We try to do 3-4 releases a year
+and yet a lot of time when someone complains about the last release being slow
+we usually say "oh, but it's the release, that was AGES ago". That makes
+something as big as summarizing a year next to impossible, but using my
+external IQ amplifiers like internet and hg log, I will try to provide you
+with some semi-serious statistics:
+
+* We made 3 releases last year, boringly, 1.5, 1.6 and 1.7
+
+* Number of visitors on pypy.org grew from about 200 to 800 per day (averaged
+  weekly)
+
+* We spoke at 15 conferences/meetups
+
+* We made 10660 commits, or 29 per day (mostly typos ;-)
+
+* We published 43 blog posts, keeping you entertained almost weekly
+
+* We made PyPy over 2x faster (on a select set of benchmarks),
+  `1.4 and nightly`_ comparison
+
+
+
+
+.. _`1.4 and nightly` http://speed.pypy.org/comparison/?exe=1%2B172%2C1%2BL%2Bdefault&ben=1%2C34%2C27%2C2%2C25%2C3%2C4%2C5%2C22%2C6%2C39%2C7%2C8%2C23%2C24%2C9%2C10%2C11%2C12%2C13%2C14%2C15%2C35%2C36%2C37%2C38%2C16%2C28%2C30%2C32%2C29%2C33%2C17%2C18%2C19%2C20&env=1%2C2&hor=true&bas=1%2B172&chart=normal+bars

From noreply at buildbot.pypy.org  Wed Jan 18 01:22:47 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Wed, 18 Jan 2012 01:22:47 +0100 (CET)
Subject: [pypy-commit] pypy matrixmath-dot: merge from default
Message-ID: <20120118002247.8B9A5710260@wyvern.cs.uni-duesseldorf.de>

Author: mattip
Branch: matrixmath-dot
Changeset: r51431:d512cc3a2cf4
Date: 2012-01-17 21:06 +0200
http://bitbucket.org/pypy/pypy/changeset/d512cc3a2cf4/

Log:	merge from default

diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -2,6 +2,9 @@
 *.py[co]
 *~
 .*.swp
+.idea
+.project
+.pydevproject
 
 syntax: regexp
 ^testresult$
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -340,7 +340,7 @@
                    requires=[("objspace.std.builtinshortcut", True)]),
         BoolOption("withidentitydict",
                    "track types that override __hash__, __eq__ or __cmp__ and use a special dict strategy for those which do not",
-                   default=True),
+                   default=False),
      ]),
 ])
 
@@ -370,6 +370,7 @@
         config.objspace.std.suggest(getattributeshortcut=True)
         config.objspace.std.suggest(newshortcut=True)
         config.objspace.std.suggest(withspecialisedtuple=True)
+        config.objspace.std.suggest(withidentitydict=True)
         #if not IS_64_BITS:
         #    config.objspace.std.suggest(withsmalllong=True)
 
diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -29,7 +29,7 @@
 #define PY_VERSION		"2.7.1"
 
 /* PyPy version as a string */
-#define PYPY_VERSION "1.7.1"
+#define PYPY_VERSION "1.8.1"
 
 /* Subversion Revision number of this file (not of the repository).
  * Empty since Mercurial migration. */
diff --git a/pypy/module/micronumpy/interp_iter.py b/pypy/module/micronumpy/interp_iter.py
--- a/pypy/module/micronumpy/interp_iter.py
+++ b/pypy/module/micronumpy/interp_iter.py
@@ -157,6 +157,8 @@
                 offset += self.strides[i]
                 break
             else:
+                if i == self.dim:
+                    first_line = True
                 indices[i] = 0
                 offset -= self.backstrides[i]
         else:
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -287,11 +287,11 @@
     descr_rmod = _binop_right_impl("mod")
 
     def _reduce_ufunc_impl(ufunc_name, promote_to_largest=False):
-        def impl(self, space, w_dim=None):
-            if space.is_w(w_dim, space.w_None):
-                w_dim = space.wrap(-1)
+        def impl(self, space, w_axis=None):
+            if space.is_w(w_axis, space.w_None):
+                w_axis = space.wrap(-1)
             return getattr(interp_ufuncs.get(space), ufunc_name).reduce(space,
-                                        self, True, promote_to_largest, w_dim)
+                                        self, True, promote_to_largest, w_axis)
         return func_with_new_name(impl, "reduce_%s_impl" % ufunc_name)
 
     descr_sum = _reduce_ufunc_impl("add")
@@ -635,14 +635,14 @@
             )
         return w_result
 
-    def descr_mean(self, space, w_dim=None):
-        if space.is_w(w_dim, space.w_None):
-            w_dim = space.wrap(-1)
+    def descr_mean(self, space, w_axis=None):
+        if space.is_w(w_axis, space.w_None):
+            w_axis = space.wrap(-1)
             w_denom = space.wrap(self.size)
         else:
-            dim = space.int_w(w_dim)
+            dim = space.int_w(w_axis)
             w_denom = space.wrap(self.shape[dim])
-        return space.div(self.descr_sum_promote(space, w_dim), w_denom)
+        return space.div(self.descr_sum_promote(space, w_axis), w_denom)
 
     def descr_var(self, space):
         # var = mean((values - mean(values)) ** 2)
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -734,6 +734,7 @@
         a = array(range(105)).reshape(3, 5, 7)
         b = mean(a, axis=0)
         b[0,0]==35.
+        assert a.mean(axis=0)[0, 0] == 35
         assert (b == array(range(35, 70), dtype=float).reshape(5, 7)).all()
         assert (mean(a, 2) == array(range(0, 15), dtype=float).reshape(3, 5) * 7 + 3).all()
 
@@ -756,6 +757,7 @@
         assert array([]).sum() == 0.0
         raises(ValueError, 'array([]).max()')
         assert (a.sum(0) == [30, 35, 40]).all()
+        assert (a.sum(axis=0) == [30, 35, 40]).all()
         assert (a.sum(1) == [3, 12, 21, 30, 39]).all()
         assert (a.max(0) == [12, 13, 14]).all()
         assert (a.max(1) == [2, 5, 8, 11, 14]).all()
@@ -770,6 +772,8 @@
         assert ((a + a).T.sum(2).T == (a + a).sum(0)).all()
         assert (a.reshape(1,-1).sum(0) == range(105)).all()
         assert (a.reshape(1,-1).sum(1) == 5460)
+        assert (array([[1,2],[3,4]]).prod(0) == [3, 8]).all()
+        assert (array([[1,2],[3,4]]).prod(1) == [2, 12]).all()
 
     def test_identity(self):
         from _numpypy import identity, array
diff --git a/pypy/module/pypyjit/policy.py b/pypy/module/pypyjit/policy.py
--- a/pypy/module/pypyjit/policy.py
+++ b/pypy/module/pypyjit/policy.py
@@ -127,7 +127,7 @@
                        'imp', 'sys', 'array', '_ffi', 'itertools', 'operator',
                        'posix', '_socket', '_sre', '_lsprof', '_weakref',
                        '__pypy__', 'cStringIO', '_collections', 'struct',
-                       'mmap', 'marshal']:
+                       'mmap', 'marshal', '_codecs']:
             if modname == 'pypyjit' and 'interp_resop' in rest:
                 return False
             return True
diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py
--- a/pypy/module/sys/version.py
+++ b/pypy/module/sys/version.py
@@ -10,7 +10,7 @@
 CPYTHON_VERSION            = (2, 7, 1, "final", 42)   #XXX # sync patchlevel.h
 CPYTHON_API_VERSION        = 1013   #XXX # sync with include/modsupport.h
 
-PYPY_VERSION               = (1, 7, 1, "dev", 0)    #XXX # sync patchlevel.h
+PYPY_VERSION               = (1, 8, 1, "dev", 0)    #XXX # sync patchlevel.h
 
 if platform.name == 'msvc':
     COMPILER_INFO = 'MSC v.%d 32 bit' % (platform.version * 10 + 600)
diff --git a/pypy/objspace/std/test/test_unicodeobject.py b/pypy/objspace/std/test/test_unicodeobject.py
--- a/pypy/objspace/std/test/test_unicodeobject.py
+++ b/pypy/objspace/std/test/test_unicodeobject.py
@@ -64,6 +64,12 @@
         check(', '.join([u'a']), u'a')
         check(', '.join(['a', u'b']), u'a, b')
         check(u', '.join(['a', 'b']), u'a, b')
+        try:
+            u''.join([u'a', 2, 3])
+        except TypeError, e:
+            assert 'sequence item 1' in str(e)
+        else:
+            raise Exception("DID NOT RAISE")
 
     if sys.version_info >= (2,3):
         def test_contains_ex(self):
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -201,7 +201,7 @@
     return space.newbool(container.find(item) != -1)
 
 def unicode_join__Unicode_ANY(space, w_self, w_list):
-    list_w = space.unpackiterable(w_list)
+    list_w = space.listview(w_list)
     size = len(list_w)
 
     if size == 0:
@@ -216,22 +216,21 @@
 
 def _unicode_join_many_items(space, w_self, list_w, size):
     self = w_self._value
-    sb = UnicodeBuilder()
+    prealloc_size = len(self) * (size - 1)
+    for i in range(size):
+        try:
+            prealloc_size += len(space.unicode_w(list_w[i]))
+        except OperationError, e:
+            if not e.match(space, space.w_TypeError):
+                raise
+            raise operationerrfmt(space.w_TypeError,
+                        "sequence item %d: expected string or Unicode", i)
+    sb = UnicodeBuilder(prealloc_size)
     for i in range(size):
         if self and i != 0:
             sb.append(self)
         w_s = list_w[i]
-        if isinstance(w_s, W_UnicodeObject):
-            # shortcut for performance
-            sb.append(w_s._value)
-        else:
-            try:
-                sb.append(space.unicode_w(w_s))
-            except OperationError, e:
-                if not e.match(space, space.w_TypeError):
-                    raise
-                raise operationerrfmt(space.w_TypeError,
-                    "sequence item %d: expected string or Unicode", i)
+        sb.append(space.unicode_w(w_s))
     return space.wrap(sb.build())
 
 def hash__Unicode(space, w_uni):
diff --git a/pypy/rlib/longlong2float.py b/pypy/rlib/longlong2float.py
--- a/pypy/rlib/longlong2float.py
+++ b/pypy/rlib/longlong2float.py
@@ -79,19 +79,23 @@
 longlong2float = rffi.llexternal(
     "pypy__longlong2float", [rffi.LONGLONG], rffi.DOUBLE,
     _callable=longlong2float_emulator, compilation_info=eci,
-    _nowrapper=True, elidable_function=True, sandboxsafe=True)
+    _nowrapper=True, elidable_function=True, sandboxsafe=True,
+    oo_primitive="pypy__longlong2float")
 
 float2longlong = rffi.llexternal(
     "pypy__float2longlong", [rffi.DOUBLE], rffi.LONGLONG,
     _callable=float2longlong_emulator, compilation_info=eci,
-    _nowrapper=True, elidable_function=True, sandboxsafe=True)
+    _nowrapper=True, elidable_function=True, sandboxsafe=True,
+    oo_primitive="pypy__float2longlong")
 
 uint2singlefloat = rffi.llexternal(
     "pypy__uint2singlefloat", [rffi.UINT], rffi.FLOAT,
     _callable=uint2singlefloat_emulator, compilation_info=eci,
-    _nowrapper=True, elidable_function=True, sandboxsafe=True)
+    _nowrapper=True, elidable_function=True, sandboxsafe=True,
+    oo_primitive="pypy__uint2singlefloat")
 
 singlefloat2uint = rffi.llexternal(
     "pypy__singlefloat2uint", [rffi.FLOAT], rffi.UINT,
     _callable=singlefloat2uint_emulator, compilation_info=eci,
-    _nowrapper=True, elidable_function=True, sandboxsafe=True)
+    _nowrapper=True, elidable_function=True, sandboxsafe=True,
+    oo_primitive="pypy__singlefloat2uint")
diff --git a/pypy/rlib/objectmodel.py b/pypy/rlib/objectmodel.py
--- a/pypy/rlib/objectmodel.py
+++ b/pypy/rlib/objectmodel.py
@@ -420,7 +420,7 @@
                   vobj.concretetype.TO._gckind == 'gc')
         else:
             from pypy.rpython.ootypesystem import ootype
-            ok = isinstance(vobj.concretetype, ootype.Instance)
+            ok = isinstance(vobj.concretetype, (ootype.Instance, ootype.BuiltinType))
         if not ok:
             from pypy.rpython.error import TyperError
             raise TyperError("compute_unique_id() cannot be applied to"
diff --git a/pypy/rpython/lltypesystem/rclass.py b/pypy/rpython/lltypesystem/rclass.py
--- a/pypy/rpython/lltypesystem/rclass.py
+++ b/pypy/rpython/lltypesystem/rclass.py
@@ -510,7 +510,13 @@
         ctype = inputconst(Void, self.object_type)
         cflags = inputconst(Void, flags)
         vlist = [ctype, cflags]
-        vptr = llops.genop('malloc', vlist,
+        cnonmovable = self.classdef.classdesc.read_attribute(
+            '_alloc_nonmovable_', Constant(False))
+        if cnonmovable.value:
+            opname = 'malloc_nonmovable'
+        else:
+            opname = 'malloc'
+        vptr = llops.genop(opname, vlist,
                            resulttype = Ptr(self.object_type))
         ctypeptr = inputconst(CLASSTYPE, self.rclass.getvtable())
         self.setfield(vptr, '__class__', ctypeptr, llops)
diff --git a/pypy/rpython/ootypesystem/ootype.py b/pypy/rpython/ootypesystem/ootype.py
--- a/pypy/rpython/ootypesystem/ootype.py
+++ b/pypy/rpython/ootypesystem/ootype.py
@@ -512,6 +512,7 @@
             "ll_append_char": Meth([CHARTP], Void),
             "ll_append": Meth([STRINGTP], Void),
             "ll_build": Meth([], STRINGTP),
+            "ll_getlength": Meth([], Signed),
             })
         self._setup_methods({})
 
@@ -1376,6 +1377,9 @@
     def _cast_to_object(self):
         return make_object(self)
 
+    def _identityhash(self):
+        return object.__hash__(self)
+
 class _string(_builtin_type):
 
     def __init__(self, STRING, value = ''):
@@ -1543,6 +1547,9 @@
         else:
             return make_unicode(u''.join(self._buf))
 
+    def ll_getlength(self):
+        return self.ll_build().ll_strlen()
+
 class _null_string_builder(_null_mixin(_string_builder), _string_builder):
     def __init__(self, STRING_BUILDER):
         self.__dict__["_TYPE"] = STRING_BUILDER
diff --git a/pypy/rpython/ootypesystem/rbuilder.py b/pypy/rpython/ootypesystem/rbuilder.py
--- a/pypy/rpython/ootypesystem/rbuilder.py
+++ b/pypy/rpython/ootypesystem/rbuilder.py
@@ -21,6 +21,10 @@
         builder.ll_append_char(char)
 
     @staticmethod
+    def ll_getlength(builder):
+        return builder.ll_getlength()
+
+    @staticmethod
     def ll_append(builder, string):
         builder.ll_append(string)
 
diff --git a/pypy/rpython/test/test_rbuilder.py b/pypy/rpython/test/test_rbuilder.py
--- a/pypy/rpython/test/test_rbuilder.py
+++ b/pypy/rpython/test/test_rbuilder.py
@@ -124,9 +124,5 @@
     pass
 
 class TestOOtype(BaseTestStringBuilder, OORtypeMixin):
-    def test_string_getlength(self):
-        py.test.skip("getlength(): not implemented on ootype")
-    def test_unicode_getlength(self):
-        py.test.skip("getlength(): not implemented on ootype")
     def test_append_charpsize(self):
         py.test.skip("append_charpsize(): not implemented on ootype")
diff --git a/pypy/rpython/test/test_rbuiltin.py b/pypy/rpython/test/test_rbuiltin.py
--- a/pypy/rpython/test/test_rbuiltin.py
+++ b/pypy/rpython/test/test_rbuiltin.py
@@ -463,6 +463,31 @@
             assert x1 == intmask(x0)
             assert x3 == intmask(x2)
 
+    def test_id_on_builtins(self):
+        from pypy.rlib.objectmodel import compute_unique_id
+        from pypy.rlib.rstring import StringBuilder, UnicodeBuilder
+        def fn():
+            return (compute_unique_id("foo"),
+                    compute_unique_id(u"bar"),
+                    compute_unique_id([1]),
+                    compute_unique_id({"foo": 3}),
+                    compute_unique_id(StringBuilder()),
+                    compute_unique_id(UnicodeBuilder()))
+        res = self.interpret(fn, [])
+        for id in self.ll_unpack_tuple(res, 6):
+            assert isinstance(id, (int, r_longlong))
+
+    def test_uniqueness_of_id_on_strings(self):
+        from pypy.rlib.objectmodel import compute_unique_id
+        def fn(s1, s2):
+            return (compute_unique_id(s1), compute_unique_id(s2))
+
+        s1 = "foo"
+        s2 = ''.join(['f','oo'])
+        res = self.interpret(fn, [self.string_to_ll(s1), self.string_to_ll(s2)])
+        i1, i2 = self.ll_unpack_tuple(res, 2)
+        assert i1 != i2
+
     def test_cast_primitive(self):
         from pypy.rpython.annlowlevel import LowLevelAnnotatorPolicy
         def llf(u):
diff --git a/pypy/rpython/test/test_rclass.py b/pypy/rpython/test/test_rclass.py
--- a/pypy/rpython/test/test_rclass.py
+++ b/pypy/rpython/test/test_rclass.py
@@ -1130,6 +1130,18 @@
             assert sorted([u]) == [6]                    # 32-bit types
             assert sorted([i, r, d, l]) == [2, 3, 4, 5]  # 64-bit types
 
+    def test_nonmovable(self):
+        for (nonmovable, opname) in [(True, 'malloc_nonmovable'),
+                                     (False, 'malloc')]:
+            class A(object):
+                _alloc_nonmovable_ = nonmovable
+            def f():
+                return A()
+            t, typer, graph = self.gengraph(f, [])
+            assert summary(graph) == {opname: 1,
+                                      'cast_pointer': 1,
+                                      'setfield': 1}
+
 
 class TestOOtype(BaseTestRclass, OORtypeMixin):
 
diff --git a/pypy/translator/jvm/builtin.py b/pypy/translator/jvm/builtin.py
--- a/pypy/translator/jvm/builtin.py
+++ b/pypy/translator/jvm/builtin.py
@@ -84,6 +84,9 @@
     (ootype.StringBuilder.__class__, "ll_build"):
     jvm.Method.v(jStringBuilder, "toString", (), jString),
 
+    (ootype.StringBuilder.__class__, "ll_getlength"):
+    jvm.Method.v(jStringBuilder, "length", (), jInt),
+
     (ootype.String.__class__, "ll_hash"):
     jvm.Method.v(jString, "hashCode", (), jInt),
 
diff --git a/pypy/translator/jvm/database.py b/pypy/translator/jvm/database.py
--- a/pypy/translator/jvm/database.py
+++ b/pypy/translator/jvm/database.py
@@ -358,7 +358,7 @@
         ootype.Unsigned:jvm.PYPYSERIALIZEUINT,
         ootype.SignedLongLong:jvm.LONGTOSTRINGL,
         ootype.UnsignedLongLong: jvm.PYPYSERIALIZEULONG,
-        ootype.Float:jvm.DOUBLETOSTRINGD,
+        ootype.Float:jvm.PYPYSERIALIZEDOUBLE,
         ootype.Bool:jvm.PYPYSERIALIZEBOOLEAN,
         ootype.Void:jvm.PYPYSERIALIZEVOID,
         ootype.Char:jvm.PYPYESCAPEDCHAR,
diff --git a/pypy/translator/jvm/metavm.py b/pypy/translator/jvm/metavm.py
--- a/pypy/translator/jvm/metavm.py
+++ b/pypy/translator/jvm/metavm.py
@@ -92,6 +92,7 @@
 CASTS = {
 #   FROM                      TO
     (ootype.Signed,           ootype.UnsignedLongLong): jvm.I2L,
+    (ootype.Unsigned,         ootype.UnsignedLongLong): jvm.I2L,
     (ootype.SignedLongLong,   ootype.Signed):           jvm.L2I,
     (ootype.UnsignedLongLong, ootype.Unsigned):         jvm.L2I,
     (ootype.UnsignedLongLong, ootype.Signed):           jvm.L2I,
diff --git a/pypy/translator/jvm/opcodes.py b/pypy/translator/jvm/opcodes.py
--- a/pypy/translator/jvm/opcodes.py
+++ b/pypy/translator/jvm/opcodes.py
@@ -101,6 +101,7 @@
     'jit_force_virtualizable':  Ignore,
     'jit_force_virtual':        DoNothing,
     'jit_force_quasi_immutable': Ignore,
+    'jit_is_virtual':           PushPrimitive(ootype.Bool, False),
 
     'debug_assert':              [], # TODO: implement?
     'debug_start_traceback':    Ignore,
diff --git a/pypy/translator/jvm/src/pypy/PyPy.java b/pypy/translator/jvm/src/pypy/PyPy.java
--- a/pypy/translator/jvm/src/pypy/PyPy.java
+++ b/pypy/translator/jvm/src/pypy/PyPy.java
@@ -283,6 +283,14 @@
         }
     }
 
+    public double pypy__longlong2float(long l) {
+        return Double.longBitsToDouble(l);
+    }
+
+    public long pypy__float2longlong(double d) {
+        return Double.doubleToRawLongBits(d);
+    }
+
     public double ooparse_float(String s) {
         try {
             return Double.parseDouble(s);
@@ -353,6 +361,19 @@
             return "False";
     }
 
+    public static String serialize_double(double d) {
+        if (Double.isNaN(d)) {
+            return "float(\"nan\")";
+        } else if (Double.isInfinite(d)) {
+            if (d > 0)
+                return "float(\"inf\")";
+            else
+                return "float(\"-inf\")";
+        } else {
+            return Double.toString(d);
+        }
+    }
+
     private static String format_char(char c) {
         String res = "\\x";
         if (c <= 0x0F) res = res + "0";
diff --git a/pypy/translator/jvm/test/runtest.py b/pypy/translator/jvm/test/runtest.py
--- a/pypy/translator/jvm/test/runtest.py
+++ b/pypy/translator/jvm/test/runtest.py
@@ -56,6 +56,7 @@
 
 # CLI could-be duplicate
 class JvmGeneratedSourceWrapper(object):
+
     def __init__(self, gensrc):
         """ gensrc is an instance of JvmGeneratedSource """
         self.gensrc = gensrc
diff --git a/pypy/translator/jvm/test/test_builder.py b/pypy/translator/jvm/test/test_builder.py
new file mode 100644
--- /dev/null
+++ b/pypy/translator/jvm/test/test_builder.py
@@ -0,0 +1,7 @@
+from pypy.translator.jvm.test.runtest import JvmTest
+from pypy.rpython.test.test_rbuilder import BaseTestStringBuilder
+import py
+
+class TestJvmStringBuilder(JvmTest, BaseTestStringBuilder):
+    def test_append_charpsize(self):
+        py.test.skip("append_charpsize(): not implemented on ootype")
diff --git a/pypy/translator/jvm/test/test_longlong2float.py b/pypy/translator/jvm/test/test_longlong2float.py
new file mode 100644
--- /dev/null
+++ b/pypy/translator/jvm/test/test_longlong2float.py
@@ -0,0 +1,20 @@
+from pypy.translator.jvm.test.runtest import JvmTest
+from pypy.rlib.longlong2float import *
+from pypy.rlib.test.test_longlong2float import enum_floats
+from pypy.rlib.test.test_longlong2float import fn as float2longlong2float
+import py
+
+class TestLongLong2Float(JvmTest):
+
+    def test_float2longlong_and_longlong2float(self):
+        def func(f):
+            return float2longlong2float(f)
+
+        for f in enum_floats():
+            assert repr(f) == repr(self.interpret(func, [f]))
+
+    def test_uint2singlefloat(self):
+        py.test.skip("uint2singlefloat is not implemented in ootype")
+
+    def test_singlefloat2uint(self):
+        py.test.skip("singlefloat2uint is not implemented in ootype")
diff --git a/pypy/translator/jvm/typesystem.py b/pypy/translator/jvm/typesystem.py
--- a/pypy/translator/jvm/typesystem.py
+++ b/pypy/translator/jvm/typesystem.py
@@ -955,6 +955,7 @@
 PYPYSERIALIZEUINT  =    Method.s(jPyPy, 'serialize_uint', (jInt,), jString)
 PYPYSERIALIZEULONG =    Method.s(jPyPy, 'serialize_ulonglong', (jLong,),jString)
 PYPYSERIALIZEVOID =     Method.s(jPyPy, 'serialize_void', (), jString)
+PYPYSERIALIZEDOUBLE =   Method.s(jPyPy, 'serialize_double', (jDouble,), jString)
 PYPYESCAPEDCHAR =       Method.s(jPyPy, 'escaped_char', (jChar,), jString)
 PYPYESCAPEDUNICHAR =    Method.s(jPyPy, 'escaped_unichar', (jChar,), jString)
 PYPYESCAPEDSTRING =     Method.s(jPyPy, 'escaped_string', (jString,), jString)
diff --git a/pypy/translator/oosupport/test_template/cast.py b/pypy/translator/oosupport/test_template/cast.py
--- a/pypy/translator/oosupport/test_template/cast.py
+++ b/pypy/translator/oosupport/test_template/cast.py
@@ -13,6 +13,9 @@
 def to_longlong(x):
     return r_longlong(x)
 
+def to_ulonglong(x):
+    return r_ulonglong(x)
+
 def uint_to_int(x):
     return intmask(x)
 
@@ -56,6 +59,9 @@
     def test_unsignedlonglong_to_unsigned4(self):
         self.check(to_uint, [r_ulonglong(18446744073709551615l)]) # max 64 bit num
 
+    def test_unsigned_to_usignedlonglong(self):
+        self.check(to_ulonglong, [r_uint(42)])
+
     def test_uint_to_int(self):
         self.check(uint_to_int, [r_uint(sys.maxint+1)])
 

From noreply at buildbot.pypy.org  Wed Jan 18 01:22:48 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Wed, 18 Jan 2012 01:22:48 +0100 (CET)
Subject: [pypy-commit] pypy matrixmath-dot: passes a test, needs cleanup
Message-ID: <20120118002248.C74A6710260@wyvern.cs.uni-duesseldorf.de>

Author: mattip
Branch: matrixmath-dot
Changeset: r51432:f62709780578
Date: 2012-01-18 02:22 +0200
http://bitbucket.org/pypy/pypy/changeset/f62709780578/

Log:	passes a test, needs cleanup

diff --git a/pypy/module/micronumpy/interp_iter.py b/pypy/module/micronumpy/interp_iter.py
--- a/pypy/module/micronumpy/interp_iter.py
+++ b/pypy/module/micronumpy/interp_iter.py
@@ -2,7 +2,7 @@
 from pypy.rlib import jit
 from pypy.rlib.objectmodel import instantiate
 from pypy.module.micronumpy.strides import calculate_broadcast_strides,\
-     calculate_slice_strides
+     calculate_slice_strides, calculate_dot_strides
 
 class BaseTransform(object):
     pass
@@ -16,6 +16,11 @@
     def __init__(self, res_shape):
         self.res_shape = res_shape
 
+class DotTransform(BaseTransform):
+    def __init__(self, res_shape, skip_dims):
+        self.res_shape = res_shape
+        self.skip_dims = skip_dims
+
 class BaseIterator(object):
     def next(self, shapelen):
         raise NotImplementedError
@@ -85,6 +90,10 @@
                                         self.strides,
                                         self.backstrides, t.chunks)
             return ViewIterator(r[1], r[2], r[3], r[0])
+        elif isinstance(t, DotTransform):
+            r = calculate_dot_strides(self.strides, self.backstrides,
+                                     t.res_shape, t.skip_dims)
+            return ViewIterator(self.offset, r[0], r[1], t.res_shape)
 
     @jit.unroll_safe
     def next(self, shapelen):
@@ -130,6 +139,7 @@
     def transform(self, arr, t):
         pass
 
+
 class AxisIterator(BaseIterator):
     def __init__(self, start, dim, shape, strides, backstrides):
         self.res_shape = shape[:]
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -3,13 +3,14 @@
 from pypy.interpreter.gateway import interp2app, NoneNotWrapped
 from pypy.interpreter.typedef import TypeDef, GetSetProperty
 from pypy.module.micronumpy import interp_ufuncs, interp_dtype, signature
-from pypy.module.micronumpy.strides import calculate_slice_strides
+from pypy.module.micronumpy.strides import calculate_slice_strides,\
+                                           calculate_dot_strides
 from pypy.rlib import jit
 from pypy.rpython.lltypesystem import lltype, rffi
 from pypy.tool.sourcetools import func_with_new_name
 from pypy.rlib.rstring import StringBuilder
 from pypy.module.micronumpy.interp_iter import ArrayIterator, OneDimIterator,\
-     SkipLastAxisIterator
+     SkipLastAxisIterator, ViewIterator
 
 numpy_driver = jit.JitDriver(
     greens=['shapelen', 'sig'],
@@ -211,6 +212,28 @@
                 n_old_elems_to_use *= old_shape[oldI]
     return new_strides
 
+def match_dot_shapes(space, self, other):
+    my_critical_dim_size = self.shape[-1]
+    other_critical_dim_size = other.shape[0]
+    other_critical_dim = 0
+    other_critical_dim_stride = other.strides[0]
+    out_shape = []
+    if len(other.shape) > 1:
+        other_critical_dim = len(other.shape) - 2
+        other_critical_dim_size = other.shape[other_critical_dim]
+        other_critical_dim_stride = other.strides[other_critical_dim]
+        assert other_critical_dim >= 0
+        out_shape += self.shape[:-1] + \
+                     other.shape[0:other_critical_dim] + \
+                     other.shape[other_critical_dim + 1:]
+    elif len(other.shape) > 0:
+        #dot does not reduce for scalars
+        out_shape += self.shape[:-1]
+    if my_critical_dim_size != other_critical_dim_size:
+        raise OperationError(space.w_ValueError, space.wrap(
+                                        "objects are not aligned"))
+    return out_shape, other_critical_dim
+
 class BaseArray(Wrappable):
     _attrs_ = ["invalidates", "shape", 'size']
 
@@ -384,70 +407,62 @@
     the second-to-last of `b`::
 
         dot(a, b)[i,j,k,m] = sum(a[i,j,:] * b[k,:,m])'''
-        w_other = convert_to_array(space, w_other)
-        if isinstance(w_other, Scalar):
-            return self.descr_mul(space, w_other)
-        elif len(self.shape) < 2 and len(w_other.shape) < 2:
-            w_res = self.descr_mul(space, w_other)
+        other = convert_to_array(space, w_other)
+        if isinstance(other, Scalar):
+            return self.descr_mul(space, other)
+        elif len(self.shape) < 2 and len(other.shape) < 2:
+            w_res = self.descr_mul(space, other)
             assert isinstance(w_res, BaseArray)
             return w_res.descr_sum(space, space.wrap(-1))
         dtype = interp_ufuncs.find_binop_result_dtype(space,
-                                     self.find_dtype(), w_other.find_dtype())
-        if self.size < 1 and w_other.size < 1:
+                                     self.find_dtype(), other.find_dtype())
+        if self.size < 1 and other.size < 1:
             #numpy compatability
             return scalar_w(space, dtype, space.wrap(0))
         #Do the dims match?
-        my_critical_dim_size = self.shape[-1]
-        other_critical_dim_size = w_other.shape[0]
-        other_critical_dim = 0
-        other_critical_dim_stride = w_other.strides[0]
-        out_shape = []
-        if len(w_other.shape) > 1:
-            other_critical_dim = len(w_other.shape) - 2
-            other_critical_dim_size = w_other.shape[other_critical_dim]
-            other_critical_dim_stride = w_other.strides[other_critical_dim]
-            assert other_critical_dim >= 0
-            out_shape += self.shape[:-1] + \
-                         w_other.shape[0:other_critical_dim] + \
-                         w_other.shape[other_critical_dim + 1:]
-        elif len(w_other.shape) > 0:
-            #dot does not reduce for scalars
-            out_shape += self.shape[:-1]
-        if my_critical_dim_size != other_critical_dim_size:
-            raise OperationError(space.w_ValueError, space.wrap(
-                                            "objects are not aligned"))
+        out_shape, other_critical_dim = match_dot_shapes(space, self, other)
         out_size = 1
-        for os in out_shape:
-            out_size *= os
-        out_ndims = len(out_shape)
-        # TODO: what should the order be? C or F?
-        arr = W_NDimArray(out_size, out_shape, dtype=dtype)
-        # TODO: this is all a bogus mess of previous work, 
-        # rework within the context of transformations
-        '''
-        out_iter = ViewIterator(arr.start, arr.strides, arr.backstrides, arr.shape)
-        # TODO: invalidate self, w_other with arr ?
-        while not out_iter.done():
-            my_index = self.start
-            other_index = w_other.start
-            i = 0
-            while i < len(self.shape) - 1:
-                my_index += out_iter.indices[i] * self.strides[i]
-                i += 1
-            for j in range(len(w_other.shape) - 2):
-                other_index += out_iter.indices[i] * w_other.strides[j]
-            other_index += out_iter.indices[-1] * w_other.strides[-1]
-            w_ssd = space.newlist([space.wrap(my_index),
-                                   space.wrap(len(self.shape) - 1)])
-            w_osd = space.newlist([space.wrap(other_index),
-                                   space.wrap(other_critical_dim)])
-            w_res = self.descr_mul(space, w_other)
-            assert isinstance(w_res, BaseArray)
-            value = w_res.descr_sum(space)
-            arr.setitem(out_iter.get_offset(), value)
-            out_iter = out_iter.next(out_ndims)
-        '''
-        return arr
+        for o in out_shape:
+            out_size *= o
+        result = W_NDimArray(out_size, out_shape, dtype)
+        # given a.shape == [3, 5, 7],
+        #       b.shape == [2, 7, 4]
+        #  result.shape == [3, 5, 2, 4]
+        # all iterators shapes should be [3, 5, 2, 7, 4]
+        # result should skip dims 3 which is results.ndims - 1
+        # a should skip 2, 4 which is a.ndims-1 + range(b.ndims) 
+        #       except where it==(b.ndims-2)
+        # b should skip 0, 1
+        mul = interp_ufuncs.get(space).multiply.func
+        add = interp_ufuncs.get(space).add.func
+        broadcast_shape = self.shape[:-1] + other.shape
+        #Aww, cmon, this is the product of a warped mind.
+        left_skip = [len(self.shape) - 1 + i for i in range(len(other.shape)) if i != other_critical_dim]
+        right_skip = range(len(self.shape) - 1)
+        arr = DotArray(mul, 'DotName', out_shape, dtype, self, other,
+                                        left_skip, right_skip)
+        arr.broadcast_shape = broadcast_shape
+        arr.result_skip = [len(out_shape) - 1]
+        #Make this lazy someday...
+        sig = signature.find_sig(signature.DotSignature(mul, 'dot', dtype,
+                                  self.create_sig(), other.create_sig()), arr)
+        assert isinstance(sig, signature.DotSignature)
+        self.do_dot_loop(sig, result, arr, add)
+        return result
+
+    def do_dot_loop(self, sig, result, arr, add):
+        frame = sig.create_frame(arr)
+        shapelen = len(arr.broadcast_shape)
+        _r = calculate_dot_strides(result.strides, result.backstrides,
+                                      arr.broadcast_shape, arr.result_skip)
+        ri = ViewIterator(0, _r[0], _r[1], arr.broadcast_shape)
+        while not frame.done():
+            v = sig.eval(frame, arr).convert_to(sig.calc_dtype)
+            z = result.getitem(ri.offset)
+            value = add(sig.calc_dtype, v, result.getitem(ri.offset))
+            result.setitem(ri.offset, value)
+            frame.next(shapelen)
+            ri = ri.next(shapelen)
 
     def get_concrete(self):
         raise NotImplementedError
@@ -919,6 +934,23 @@
                        left, right)
         self.dim = dim
 
+class DotArray(Call2):
+    """ NOTE: this is only used as a container, you should never
+    encounter such things in the wild. Remove this comment
+    when we'll make Dot lazy
+    """
+    _immutable_fields_ = ['left', 'right']
+    
+    def __init__(self, ufunc, name, shape, dtype, left, right, left_skip, right_skip):
+        Call2.__init__(self, ufunc, name, shape, dtype, dtype,
+                       left, right)
+        self.left_skip = left_skip
+        self.right_skip = right_skip
+    def create_sig(self):
+        #if self.forced_result is not None:
+        #    return self.forced_result.create_sig()
+        assert NotImplementedError 
+
 class ConcreteArray(BaseArray):
     """ An array that have actual storage, whether owned or not
     """
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -192,17 +192,17 @@
                                               sig=sig,
                                               identity=identity,
                                               shapelen=shapelen, arr=arr)
-            iter = frame.get_final_iter()
+            iterator = frame.get_final_iter()
             v = sig.eval(frame, arr).convert_to(sig.calc_dtype)
-            if iter.first_line:
+            if iterator.first_line:
                 if identity is not None:
                     value = self.func(sig.calc_dtype, identity, v)
                 else:
                     value = v
             else:
-                cur = arr.left.getitem(iter.offset)
+                cur = arr.left.getitem(iterator.offset)
                 value = self.func(sig.calc_dtype, cur, v)
-            arr.left.setitem(iter.offset, value)
+            arr.left.setitem(iterator.offset, value)
             frame.next(shapelen)
 
     def reduce_loop(self, shapelen, sig, frame, value, obj, dtype):
diff --git a/pypy/module/micronumpy/signature.py b/pypy/module/micronumpy/signature.py
--- a/pypy/module/micronumpy/signature.py
+++ b/pypy/module/micronumpy/signature.py
@@ -2,7 +2,7 @@
 from pypy.rlib.rarithmetic import intmask
 from pypy.module.micronumpy.interp_iter import ViewIterator, ArrayIterator, \
      ConstantIterator, AxisIterator, ViewTransform,\
-     BroadcastTransform
+     BroadcastTransform, DotTransform
 from pypy.rlib.jit import hint, unroll_safe, promote
 
 """ Signature specifies both the numpy expression that has been constructed
@@ -331,7 +331,6 @@
         assert isinstance(arr, Call2)
         lhs = self.left.eval(frame, arr.left).convert_to(self.calc_dtype)
         rhs = self.right.eval(frame, arr.right).convert_to(self.calc_dtype)
-        
         return self.binfunc(self.calc_dtype, lhs, rhs)
 
     def debug_repr(self):
@@ -450,3 +449,21 @@
     
     def debug_repr(self):
         return 'AxisReduceSig(%s, %s)' % (self.name, self.right.debug_repr())
+
+class DotSignature(Call2):
+    def _invent_numbering(self, cache, allnumbers):
+        self.left._invent_numbering(new_cache(), allnumbers)
+        self.right._invent_numbering(new_cache(), allnumbers)
+
+    def _create_iter(self, iterlist, arraylist, arr, transforms):
+        from pypy.module.micronumpy.interp_numarray import DotArray
+
+        assert isinstance(arr, DotArray)
+        rtransforms = transforms + [DotTransform(arr.broadcast_shape, arr.right_skip)]
+        ltransforms = transforms + [DotTransform(arr.broadcast_shape, arr.left_skip)]
+        self.left._create_iter(iterlist, arraylist, arr.left, ltransforms)
+        self.right._create_iter(iterlist, arraylist, arr.right, rtransforms)
+
+    def debug_repr(self):
+        return 'DotSig(%s, %s %s)' % (self.name, self.right.debug_repr(),
+						 self.left.debug_repr())
diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py
--- a/pypy/module/micronumpy/strides.py
+++ b/pypy/module/micronumpy/strides.py
@@ -37,3 +37,17 @@
     rstrides = [0] * (len(res_shape) - len(orig_shape)) + rstrides
     rbackstrides = [0] * (len(res_shape) - len(orig_shape)) + rbackstrides
     return rstrides, rbackstrides
+
+def calculate_dot_strides(strides, backstrides, res_shape, skip_dims):
+    rstrides = []
+    rbackstrides = []
+    j=0
+    for i in range(len(res_shape)):
+        if i in skip_dims:
+            rstrides.append(0)
+            rbackstrides.append(0)
+        else:
+            rstrides.append(strides[j])
+            rbackstrides.append(backstrides[j])
+            j += 1
+    return rstrides, rbackstrides
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -867,7 +867,7 @@
         assert c.any() == False
 
     def test_dot(self):
-        from _numpypy import array, dot
+        from _numpypy import array, dot, arange
         a = array(range(5))
         assert a.dot(a) == 30.0
 
@@ -876,13 +876,12 @@
         assert dot(range(5), range(5)) == 30
         assert (dot(5, [1, 2, 3]) == [5, 10, 15]).all()
 
-        a = array([range(4), range(4, 8), range(8, 12)])
-        b = array([range(3), range(3, 6), range(6, 9), range(9, 12)])
+        a = arange(12).reshape(3, 4)
+        b = arange(12).reshape(4, 3)
         c = a.dot(b)
         assert (c == [[ 42, 48, 54], [114, 136, 158], [186, 224, 262]]).all()
 
-        a = array([[range(4), range(4, 8), range(8, 12)],
-                   [range(12, 16), range(16, 20), range(20, 24)]])
+        a = arange(24).reshape(2, 3, 4)
         raises(ValueError, "a.dot(a)")
         b = a[0, :, :].T
         #Superfluous shape test makes the intention of the test clearer

From noreply at buildbot.pypy.org  Wed Jan 18 09:50:45 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Wed, 18 Jan 2012 09:50:45 +0100 (CET)
Subject: [pypy-commit] extradoc extradoc: add a new point
Message-ID: <20120118085045.D4ECE820D8@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni 
Branch: extradoc
Changeset: r4044:f07d98b31047
Date: 2012-01-18 09:44 +0100
http://bitbucket.org/pypy/extradoc/changeset/f07d98b31047/

Log:	add a new point

diff --git a/blog/draft/pypy-2011.rst b/blog/draft/pypy-2011.rst
--- a/blog/draft/pypy-2011.rst
+++ b/blog/draft/pypy-2011.rst
@@ -24,6 +24,10 @@
 * We made PyPy over 2x faster (on a select set of benchmarks),
   `1.4 and nightly`_ comparison
 
+* We got 71 new people which did at least one commit in the PyPy repository
+  (XXX: this is what I got by doing wc on contributors.txt at 2011-01-01 and
+  the current contributors.rst, but we still need to update the latter)
+
 * We made PyPy 17x more compatible (of course this is unmeasurable, so why
   not just claim it)
 

From noreply at buildbot.pypy.org  Wed Jan 18 10:46:27 2012
From: noreply at buildbot.pypy.org (rguillebert)
Date: Wed, 18 Jan 2012 10:46:27 +0100 (CET)
Subject: [pypy-commit] extradoc extradoc: Planning for today
Message-ID: <20120118094627.E56D0820D8@wyvern.cs.uni-duesseldorf.de>

Author: Romain Guillebert 
Branch: extradoc
Changeset: r4045:3fb931e40174
Date: 2012-01-18 10:46 +0100
http://bitbucket.org/pypy/extradoc/changeset/3fb931e40174/

Log:	Planning for today

diff --git a/sprintinfo/leysin-winter-2012/planning.txt b/sprintinfo/leysin-winter-2012/planning.txt
--- a/sprintinfo/leysin-winter-2012/planning.txt
+++ b/sprintinfo/leysin-winter-2012/planning.txt
@@ -5,23 +5,25 @@
 * Antonio Cuni
 * Armin Rigo
 * Romain Guillebert
+* David Schneider
 
 Things we want to do
 --------------------
 
-* some skiing: wednesday (last day of sun :-/ )
+* some skiing (anto, armin)
 
 * review the JVM backend pull request (DONE)
 
-* py3k (romain, anto later)
+* py3k (romain)
 
 * ffistruct
 
 * Cython backend
 
+* Debug the ARM backend (bivab)
+
 * STM
   - refactored the RPython API: mostly done, must adapt targetdemo*.py
-    (anto, arigo)
-  - start work on the GC (arigo later?)
+  - start work on the GC
 
 * concurrent-marksweep GC

From noreply at buildbot.pypy.org  Wed Jan 18 11:04:56 2012
From: noreply at buildbot.pypy.org (hager)
Date: Wed, 18 Jan 2012 11:04:56 +0100 (CET)
Subject: [pypy-commit] pypy ppc-jit-backend: refactored allocation of boxes
Message-ID: <20120118100456.A5F88820D8@wyvern.cs.uni-duesseldorf.de>

Author: hager 
Branch: ppc-jit-backend
Changeset: r51433:4c659707392a
Date: 2012-01-18 11:04 +0100
http://bitbucket.org/pypy/pypy/changeset/4c659707392a/

Log:	refactored allocation of boxes

diff --git a/pypy/jit/backend/ppc/ppcgen/helper/regalloc.py b/pypy/jit/backend/ppc/ppcgen/helper/regalloc.py
--- a/pypy/jit/backend/ppc/ppcgen/helper/regalloc.py
+++ b/pypy/jit/backend/ppc/ppcgen/helper/regalloc.py
@@ -1,5 +1,12 @@
 from pypy.jit.metainterp.history import ConstInt
 from pypy.rlib.objectmodel import we_are_translated
+from pypy.jit.metainterp.history import Box
+
+
+def check_imm_box(arg, size=0xFF, allow_zero=True):
+    if isinstance(arg, ConstInt):
+        return _check_imm_arg(arg.getint(), size, allow_zero)
+    return False
 
 def _check_imm_arg(arg, size=0xFF, allow_zero=True):
     #assert not isinstance(arg, ConstInt)
@@ -19,34 +26,36 @@
         arg0, arg1 = boxes
         imm_a0 = _check_imm_arg(arg0)
         imm_a1 = _check_imm_arg(arg1)
-        l0, box = self._ensure_value_is_boxed(arg0, forbidden_vars=boxes)
-        boxes.append(box)
+        l0 = self._ensure_value_is_boxed(arg0, forbidden_vars=boxes)
+
         if imm_a1 and not imm_a0:
             l1 = self.make_sure_var_in_reg(arg1, boxes)
         else:
-            l1, box = self._ensure_value_is_boxed(arg1, forbidden_vars=boxes)
-            boxes.append(box)
-        self.possibly_free_vars(boxes)
+            l1 = self._ensure_value_is_boxed(arg1, forbidden_vars=boxes)
+
+        self.possibly_free_vars_for_op(op)
+        self.free_temp_vars()
         res = self.force_allocate_reg(op.result)
-        self.possibly_free_var(op.result)
+        #self.possibly_free_var(op.result)
         return [l0, l1, res]
     return f
 
 def prepare_unary_cmp():
     def f(self, op):
         a0 = op.getarg(0)
-        reg, box = self._ensure_value_is_boxed(a0)
-        res = self.force_allocate_reg(op.result, [box])
-        self.possibly_free_vars([a0, box, op.result])
+        assert isinstance(a0, Box)
+        reg = self.make_sure_var_in_reg(a0)
+        self.possibly_free_vars_for_op(op)
+        res = self.force_allocate_reg(op.result, [a0])
         return [reg, res]
     return f
 
 def prepare_unary_int_op():
     def f(self, op):
-        l0, box = self._ensure_value_is_boxed(op.getarg(0))
-        self.possibly_free_var(box)
+        l0 = self._ensure_value_is_boxed(op.getarg(0))
+        self.possibly_free_vars_for_op(op)
+        self.free_temp_vars()
         res = self.force_allocate_reg(op.result)
-        self.possibly_free_var(op.result)
         return [l0, res]
     return f
 
@@ -57,20 +66,17 @@
         imm_b0 = _check_imm_arg(b0)
         imm_b1 = _check_imm_arg(b1)
         if not imm_b0 and imm_b1:
-            l0, box = self._ensure_value_is_boxed(b0)
-            l1 = self.make_sure_var_in_reg(b1, [b0])
-            boxes.append(box)
+            l0 = self._ensure_value_is_boxed(b0)
+            l1 = self.make_sure_var_in_reg(b1, boxes)
         elif imm_b0 and not imm_b1:
             l0 = self.make_sure_var_in_reg(b0)
-            l1, box = self._ensure_value_is_boxed(b1, [b0])
-            boxes.append(box)
+            l1 = self._ensure_value_is_boxed(b1, boxes)
         else:
-            l0, box = self._ensure_value_is_boxed(b0)
-            boxes.append(box)
-            l1, box = self._ensure_value_is_boxed(b1, [box])
-            boxes.append(box)
+            l0 = self._ensure_value_is_boxed(b0)
+            l1 = self._ensure_value_is_boxed(b1, boxes)
         locs = [l0, l1]
-        self.possibly_free_vars(boxes)
+        self.possibly_free_vars_for_op(op)
+        self.free_temp_vars()
         res = self.force_allocate_reg(op.result)
         return locs + [res]
     return f
@@ -80,12 +86,9 @@
         boxes = list(op.getarglist())
         b0, b1 = boxes
 
-        reg1, box = self._ensure_value_is_boxed(b0, forbidden_vars=boxes)
-        boxes.append(box)
-        reg2, box = self._ensure_value_is_boxed(b1, forbidden_vars=boxes)
-        boxes.append(box)
+        reg1 = self._ensure_value_is_boxed(b0, forbidden_vars=boxes)
+        reg2 = self._ensure_value_is_boxed(b1, forbidden_vars=boxes)
 
-        self.possibly_free_vars(boxes)
         self.possibly_free_vars_for_op(op)
         res = self.force_allocate_reg(op.result)
         self.possibly_free_var(op.result)
diff --git a/pypy/jit/backend/ppc/ppcgen/opassembler.py b/pypy/jit/backend/ppc/ppcgen/opassembler.py
--- a/pypy/jit/backend/ppc/ppcgen/opassembler.py
+++ b/pypy/jit/backend/ppc/ppcgen/opassembler.py
@@ -6,7 +6,7 @@
                                               GPR_SAVE_AREA, BACKCHAIN_SIZE,
                                               MAX_REG_PARAMS)
 
-from pypy.jit.metainterp.history import (JitCellToken, TargetToken, 
+from pypy.jit.metainterp.history import (JitCellToken, TargetToken, Box,
                                          AbstractFailDescr, FLOAT, INT, REF)
 from pypy.rlib.objectmodel import we_are_translated
 from pypy.jit.backend.ppc.ppcgen.helper.assembler import (count_reg_args,
@@ -704,11 +704,9 @@
 
     def _emit_copystrcontent(self, op, regalloc, is_unicode):
         # compute the source address
-        args = list(op.getarglist())
-        base_loc, box = regalloc._ensure_value_is_boxed(args[0], args)
-        args.append(box)
-        ofs_loc, box = regalloc._ensure_value_is_boxed(args[2], args)
-        args.append(box)
+        args = op.getarglist()
+        base_loc = regalloc._ensure_value_is_boxed(args[0], args)
+        ofs_loc = regalloc._ensure_value_is_boxed(args[2], args)
         assert args[0] is not args[1]    # forbidden case of aliasing
         regalloc.possibly_free_var(args[0])
         if args[3] is not args[2] is not args[4]:  # MESS MESS MESS: don't free
@@ -724,41 +722,44 @@
         dstaddr_box = TempPtr()
         dstaddr_loc = regalloc.force_allocate_reg(dstaddr_box)
         forbidden_vars.append(dstaddr_box)
-        base_loc, box = regalloc._ensure_value_is_boxed(args[1], forbidden_vars)
-        args.append(box)
-        forbidden_vars.append(box)
-        ofs_loc, box = regalloc._ensure_value_is_boxed(args[3], forbidden_vars)
-        args.append(box)
+        base_loc = regalloc._ensure_value_is_boxed(args[1], forbidden_vars)
+        ofs_loc = regalloc._ensure_value_is_boxed(args[3], forbidden_vars)
         assert base_loc.is_reg()
         assert ofs_loc.is_reg()
         regalloc.possibly_free_var(args[1])
         if args[3] is not args[4]:     # more of the MESS described above
             regalloc.possibly_free_var(args[3])
+        regalloc.free_temp_vars()
         self._gen_address_inside_string(base_loc, ofs_loc, dstaddr_loc,
                                         is_unicode=is_unicode)
 
         # compute the length in bytes
         forbidden_vars = [srcaddr_box, dstaddr_box]
-        length_loc, length_box = regalloc._ensure_value_is_boxed(args[4], forbidden_vars)
-        args.append(length_box)
+        if isinstance(args[4], Box):
+            length_box = args[4]
+            length_loc = regalloc.make_sure_var_in_reg(args[4], forbidden_vars)
+        else:
+            length_box = TempInt()
+            length_loc = regalloc.force_allocate_reg(length_box, forbidden_vars)
+            imm = regalloc.convert_to_imm(args[4])
+            self.load(length_loc, imm)
         if is_unicode:
-            forbidden_vars = [srcaddr_box, dstaddr_box]
             bytes_box = TempPtr()
             bytes_loc = regalloc.force_allocate_reg(bytes_box, forbidden_vars)
             scale = self._get_unicode_item_scale()
             assert length_loc.is_reg()
-            self.mc.li(r.r0.value, 1<

Author: Romain Guillebert 
Branch: py3k
Changeset: r51434:68762432f02d
Date: 2012-01-18 12:07 +0100
http://bitbucket.org/pypy/pypy/changeset/68762432f02d/

Log:	Add a codegen test for py3k's new tuple unpacking

diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -798,6 +798,13 @@
         yield self.st, test, "f()", 42
     # This line is needed for py.code to find the source.
 
+    def test_tuple_unpacking(self):
+        func = """def f():
+            (a, *b, c) = 1, 2, 3, 4, 5
+            return a, b, c
+        """
+        yield self.st, func, "f()", (1, [2, 3, 4], 5)
+
 
 class AppTestCompiler:
 

From noreply at buildbot.pypy.org  Wed Jan 18 12:35:38 2012
From: noreply at buildbot.pypy.org (bivab)
Date: Wed, 18 Jan 2012 12:35:38 +0100 (CET)
Subject: [pypy-commit] pypy arm-backend-2: merge default
Message-ID: <20120118113538.02036820D8@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider 
Branch: arm-backend-2
Changeset: r51435:280cca48c156
Date: 2012-01-18 09:57 +0100
http://bitbucket.org/pypy/pypy/changeset/280cca48c156/

Log:	merge default

diff too long, truncating to 10000 out of 10623 lines

diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -3,6 +3,9 @@
 *.sw[po]
 *~
 .*.swp
+.idea
+.project
+.pydevproject
 
 syntax: regexp
 ^testresult$
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -27,7 +27,7 @@
     DEALINGS IN THE SOFTWARE.
 
 
-PyPy Copyright holders 2003-2011
+PyPy Copyright holders 2003-2012
 ----------------------------------- 
 
 Except when otherwise stated (look for LICENSE files or information at
@@ -37,43 +37,47 @@
     Armin Rigo
     Maciej Fijalkowski
     Carl Friedrich Bolz
+    Amaury Forgeot d'Arc
     Antonio Cuni
-    Amaury Forgeot d'Arc
     Samuele Pedroni
     Michael Hudson
     Holger Krekel
-    Benjamin Peterson
+    Alex Gaynor
     Christian Tismer
     Hakan Ardo
-    Alex Gaynor
+    Benjamin Peterson
+    David Schneider
     Eric van Riet Paap
     Anders Chrigstrom
-    David Schneider
     Richard Emslie
     Dan Villiom Podlaski Christiansen
     Alexander Schremmer
+    Lukas Diekmann
     Aurelien Campeas
     Anders Lehmann
     Camillo Bruni
     Niklaus Haldimann
+    Sven Hager
     Leonardo Santagada
     Toon Verwaest
     Seo Sanghyeon
+    Justin Peel
     Lawrence Oluyede
     Bartosz Skowron
     Jakub Gustak
     Guido Wesdorp
     Daniel Roberts
+    Laura Creighton
     Adrien Di Mascio
-    Laura Creighton
     Ludovic Aubry
     Niko Matsakis
+    Wim Lavrijsen
+    Matti Picus
     Jason Creighton
     Jacob Hallen
     Alex Martelli
     Anders Hammarquist
     Jan de Mooij
-    Wim Lavrijsen
     Stephan Diehl
     Michael Foord
     Stefan Schwarzer
@@ -84,34 +88,36 @@
     Alexandre Fayolle
     Marius Gedminas
     Simon Burton
-    Justin Peel
+    David Edelsohn
     Jean-Paul Calderone
     John Witulski
-    Lukas Diekmann
+    Timo Paulssen
     holger krekel
-    Wim Lavrijsen
     Dario Bertini
+    Mark Pearse
     Andreas Stührk
     Jean-Philippe St. Pierre
     Guido van Rossum
     Pavel Vinogradov
     Valentino Volonghi
     Paul deGrandis
+    Ilya Osadchiy
+    Ronny Pfannschmidt
     Adrian Kuhn
     tav
     Georg Brandl
+    Philip Jenvey
     Gerald Klix
     Wanja Saatkamp
-    Ronny Pfannschmidt
     Boris Feigin
     Oscar Nierstrasz
     David Malcolm
     Eugene Oden
     Henry Mason
-    Sven Hager
+    Jeff Terrace
     Lukas Renggli
-    Ilya Osadchiy
     Guenter Jantzen
+    Ned Batchelder
     Bert Freudenberg
     Amit Regmi
     Ben Young
@@ -142,7 +148,6 @@
     Anders Qvist
     Beatrice During
     Alexander Sedov
-    Timo Paulssen
     Corbin Simpson
     Vincent Legoll
     Romain Guillebert
@@ -165,9 +170,10 @@
     Lucio Torre
     Lene Wagner
     Miguel de Val Borro
+    Artur Lisiecki
+    Bruno Gola
     Ignas Mikalajunas
-    Artur Lisiecki
-    Philip Jenvey
+    Stefano Rivera
     Joshua Gilbert
     Godefroid Chappelle
     Yusei Tahara
@@ -179,17 +185,17 @@
     Kristjan Valur Jonsson
     Bobby Impollonia
     Michael Hudson-Doyle
+    Laurence Tratt
+    Yasir Suhail
     Andrew Thompson
     Anders Sigfridsson
     Floris Bruynooghe
     Jacek Generowicz
     Dan Colish
     Zooko Wilcox-O Hearn
-    Dan Villiom Podlaski Christiansen
-    Anders Hammarquist
+    Dan Loewenherz
     Chris Lambacher
     Dinu Gherman
-    Dan Colish
     Brett Cannon
     Daniel Neuhäuser
     Michael Chermside
diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py
--- a/lib_pypy/_ctypes/structure.py
+++ b/lib_pypy/_ctypes/structure.py
@@ -73,8 +73,12 @@
 
 class Field(object):
     def __init__(self, name, offset, size, ctype, num, is_bitfield):
-        for k in ('name', 'offset', 'size', 'ctype', 'num', 'is_bitfield'):
-            self.__dict__[k] = locals()[k]
+        self.__dict__['name'] = name
+        self.__dict__['offset'] = offset
+        self.__dict__['size'] = size
+        self.__dict__['ctype'] = ctype
+        self.__dict__['num'] = num
+        self.__dict__['is_bitfield'] = is_bitfield
 
     def __setattr__(self, name, value):
         raise AttributeError(name)
diff --git a/lib_pypy/numpypy/__init__.py b/lib_pypy/numpypy/__init__.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/numpypy/__init__.py
@@ -0,0 +1,2 @@
+from _numpypy import *
+from .core import *
diff --git a/lib_pypy/numpypy/core/__init__.py b/lib_pypy/numpypy/core/__init__.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/numpypy/core/__init__.py
@@ -0,0 +1,1 @@
+from .fromnumeric import *
diff --git a/lib_pypy/numpypy/core/fromnumeric.py b/lib_pypy/numpypy/core/fromnumeric.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/numpypy/core/fromnumeric.py
@@ -0,0 +1,2403 @@
+######################################################################    
+# This is a copy of numpy/core/fromnumeric.py modified for numpypy
+######################################################################
+# Each name in __all__ was a function in  'numeric' that is now 
+# a method in 'numpy'.
+# When the corresponding method is added to numpypy BaseArray
+# each function should be added as a module function 
+# at the applevel 
+# This can be as simple as doing the following
+#
+# def func(a, ...):
+#     if not hasattr(a, 'func')
+#         a = numpypy.array(a)
+#     return a.func(...)
+#
+######################################################################
+
+import numpypy
+
+# Module containing non-deprecated functions borrowed from Numeric.
+__docformat__ = "restructuredtext en"
+
+# functions that are now methods
+__all__ = ['take', 'reshape', 'choose', 'repeat', 'put',
+           'swapaxes', 'transpose', 'sort', 'argsort', 'argmax', 'argmin',
+           'searchsorted', 'alen',
+           'resize', 'diagonal', 'trace', 'ravel', 'nonzero', 'shape',
+           'compress', 'clip', 'sum', 'product', 'prod', 'sometrue', 'alltrue',
+           'any', 'all', 'cumsum', 'cumproduct', 'cumprod', 'ptp', 'ndim',
+           'rank', 'size', 'around', 'round_', 'mean', 'std', 'var', 'squeeze',
+           'amax', 'amin',
+          ]
+          
+def take(a, indices, axis=None, out=None, mode='raise'):
+    """
+    Take elements from an array along an axis.
+
+    This function does the same thing as "fancy" indexing (indexing arrays
+    using arrays); however, it can be easier to use if you need elements
+    along a given axis.
+
+    Parameters
+    ----------
+    a : array_like
+        The source array.
+    indices : array_like
+        The indices of the values to extract.
+    axis : int, optional
+        The axis over which to select values. By default, the flattened
+        input array is used.
+    out : ndarray, optional
+        If provided, the result will be placed in this array. It should
+        be of the appropriate shape and dtype.
+    mode : {'raise', 'wrap', 'clip'}, optional
+        Specifies how out-of-bounds indices will behave.
+
+        * 'raise' -- raise an error (default)
+        * 'wrap' -- wrap around
+        * 'clip' -- clip to the range
+
+        'clip' mode means that all indices that are too large are replaced
+        by the index that addresses the last element along that axis. Note
+        that this disables indexing with negative numbers.
+
+    Returns
+    -------
+    subarray : ndarray
+        The returned array has the same type as `a`.
+
+    See Also
+    --------
+    ndarray.take : equivalent method
+
+    Examples
+    --------
+    >>> a = [4, 3, 5, 7, 6, 8]
+    >>> indices = [0, 1, 4]
+    >>> np.take(a, indices)
+    array([4, 3, 6])
+
+    In this example if `a` is an ndarray, "fancy" indexing can be used.
+
+    >>> a = np.array(a)
+    >>> a[indices]
+    array([4, 3, 6])
+
+    """
+    raise NotImplementedError('Waiting on interp level method')
+
+
+# not deprecated --- copy if necessary, view otherwise
+def reshape(a, newshape, order='C'):
+    """
+    Gives a new shape to an array without changing its data.
+
+    Parameters
+    ----------
+    a : array_like
+        Array to be reshaped.
+    newshape : int or tuple of ints
+        The new shape should be compatible with the original shape. If
+        an integer, then the result will be a 1-D array of that length.
+        One shape dimension can be -1. In this case, the value is inferred
+        from the length of the array and remaining dimensions.
+    order : {'C', 'F', 'A'}, optional
+        Determines whether the array data should be viewed as in C
+        (row-major) order, FORTRAN (column-major) order, or the C/FORTRAN
+        order should be preserved.
+
+    Returns
+    -------
+    reshaped_array : ndarray
+        This will be a new view object if possible; otherwise, it will
+        be a copy.
+
+
+    See Also
+    --------
+    ndarray.reshape : Equivalent method.
+
+    Notes
+    -----
+
+    It is not always possible to change the shape of an array without
+    copying the data. If you want an error to be raise if the data is copied,
+    you should assign the new shape to the shape attribute of the array::
+
+     >>> a = np.zeros((10, 2))
+     # A transpose make the array non-contiguous
+     >>> b = a.T
+     # Taking a view makes it possible to modify the shape without modiying the
+     # initial object.
+     >>> c = b.view()
+     >>> c.shape = (20)
+     AttributeError: incompatible shape for a non-contiguous array
+
+
+    Examples
+    --------
+    >>> a = np.array([[1,2,3], [4,5,6]])
+    >>> np.reshape(a, 6)
+    array([1, 2, 3, 4, 5, 6])
+    >>> np.reshape(a, 6, order='F')
+    array([1, 4, 2, 5, 3, 6])
+
+    >>> np.reshape(a, (3,-1))       # the unspecified value is inferred to be 2
+    array([[1, 2],
+           [3, 4],
+           [5, 6]])
+
+    """
+    if not hasattr(a, 'reshape'):
+       a = numpypy.array(a)
+    return a.reshape(newshape)
+
+
+def choose(a, choices, out=None, mode='raise'):
+    """
+    Construct an array from an index array and a set of arrays to choose from.
+
+    First of all, if confused or uncertain, definitely look at the Examples -
+    in its full generality, this function is less simple than it might
+    seem from the following code description (below ndi =
+    `numpy.lib.index_tricks`):
+
+    ``np.choose(a,c) == np.array([c[a[I]][I] for I in ndi.ndindex(a.shape)])``.
+
+    But this omits some subtleties.  Here is a fully general summary:
+
+    Given an "index" array (`a`) of integers and a sequence of `n` arrays
+    (`choices`), `a` and each choice array are first broadcast, as necessary,
+    to arrays of a common shape; calling these *Ba* and *Bchoices[i], i =
+    0,...,n-1* we have that, necessarily, ``Ba.shape == Bchoices[i].shape``
+    for each `i`.  Then, a new array with shape ``Ba.shape`` is created as
+    follows:
+
+    * if ``mode=raise`` (the default), then, first of all, each element of
+      `a` (and thus `Ba`) must be in the range `[0, n-1]`; now, suppose that
+      `i` (in that range) is the value at the `(j0, j1, ..., jm)` position
+      in `Ba` - then the value at the same position in the new array is the
+      value in `Bchoices[i]` at that same position;
+
+    * if ``mode=wrap``, values in `a` (and thus `Ba`) may be any (signed)
+      integer; modular arithmetic is used to map integers outside the range
+      `[0, n-1]` back into that range; and then the new array is constructed
+      as above;
+
+    * if ``mode=clip``, values in `a` (and thus `Ba`) may be any (signed)
+      integer; negative integers are mapped to 0; values greater than `n-1`
+      are mapped to `n-1`; and then the new array is constructed as above.
+
+    Parameters
+    ----------
+    a : int array
+        This array must contain integers in `[0, n-1]`, where `n` is the number
+        of choices, unless ``mode=wrap`` or ``mode=clip``, in which cases any
+        integers are permissible.
+    choices : sequence of arrays
+        Choice arrays. `a` and all of the choices must be broadcastable to the
+        same shape.  If `choices` is itself an array (not recommended), then
+        its outermost dimension (i.e., the one corresponding to
+        ``choices.shape[0]``) is taken as defining the "sequence".
+    out : array, optional
+        If provided, the result will be inserted into this array. It should
+        be of the appropriate shape and dtype.
+    mode : {'raise' (default), 'wrap', 'clip'}, optional
+        Specifies how indices outside `[0, n-1]` will be treated:
+
+          * 'raise' : an exception is raised
+          * 'wrap' : value becomes value mod `n`
+          * 'clip' : values < 0 are mapped to 0, values > n-1 are mapped to n-1
+
+    Returns
+    -------
+    merged_array : array
+        The merged result.
+
+    Raises
+    ------
+    ValueError: shape mismatch
+        If `a` and each choice array are not all broadcastable to the same
+        shape.
+
+    See Also
+    --------
+    ndarray.choose : equivalent method
+
+    Notes
+    -----
+    To reduce the chance of misinterpretation, even though the following
+    "abuse" is nominally supported, `choices` should neither be, nor be
+    thought of as, a single array, i.e., the outermost sequence-like container
+    should be either a list or a tuple.
+
+    Examples
+    --------
+
+    >>> choices = [[0, 1, 2, 3], [10, 11, 12, 13],
+    ...   [20, 21, 22, 23], [30, 31, 32, 33]]
+    >>> np.choose([2, 3, 1, 0], choices
+    ... # the first element of the result will be the first element of the
+    ... # third (2+1) "array" in choices, namely, 20; the second element
+    ... # will be the second element of the fourth (3+1) choice array, i.e.,
+    ... # 31, etc.
+    ... )
+    array([20, 31, 12,  3])
+    >>> np.choose([2, 4, 1, 0], choices, mode='clip') # 4 goes to 3 (4-1)
+    array([20, 31, 12,  3])
+    >>> # because there are 4 choice arrays
+    >>> np.choose([2, 4, 1, 0], choices, mode='wrap') # 4 goes to (4 mod 4)
+    array([20,  1, 12,  3])
+    >>> # i.e., 0
+
+    A couple examples illustrating how choose broadcasts:
+
+    >>> a = [[1, 0, 1], [0, 1, 0], [1, 0, 1]]
+    >>> choices = [-10, 10]
+    >>> np.choose(a, choices)
+    array([[ 10, -10,  10],
+           [-10,  10, -10],
+           [ 10, -10,  10]])
+
+    >>> # With thanks to Anne Archibald
+    >>> a = np.array([0, 1]).reshape((2,1,1))
+    >>> c1 = np.array([1, 2, 3]).reshape((1,3,1))
+    >>> c2 = np.array([-1, -2, -3, -4, -5]).reshape((1,1,5))
+    >>> np.choose(a, (c1, c2)) # result is 2x3x5, res[0,:,:]=c1, res[1,:,:]=c2
+    array([[[ 1,  1,  1,  1,  1],
+            [ 2,  2,  2,  2,  2],
+            [ 3,  3,  3,  3,  3]],
+           [[-1, -2, -3, -4, -5],
+            [-1, -2, -3, -4, -5],
+            [-1, -2, -3, -4, -5]]])
+
+    """
+    raise NotImplementedError('Waiting on interp level method')
+
+
+def repeat(a, repeats, axis=None):
+    """
+    Repeat elements of an array.
+
+    Parameters
+    ----------
+    a : array_like
+        Input array.
+    repeats : {int, array of ints}
+        The number of repetitions for each element.  `repeats` is broadcasted
+        to fit the shape of the given axis.
+    axis : int, optional
+        The axis along which to repeat values.  By default, use the
+        flattened input array, and return a flat output array.
+
+    Returns
+    -------
+    repeated_array : ndarray
+        Output array which has the same shape as `a`, except along
+        the given axis.
+
+    See Also
+    --------
+    tile : Tile an array.
+
+    Examples
+    --------
+    >>> x = np.array([[1,2],[3,4]])
+    >>> np.repeat(x, 2)
+    array([1, 1, 2, 2, 3, 3, 4, 4])
+    >>> np.repeat(x, 3, axis=1)
+    array([[1, 1, 1, 2, 2, 2],
+           [3, 3, 3, 4, 4, 4]])
+    >>> np.repeat(x, [1, 2], axis=0)
+    array([[1, 2],
+           [3, 4],
+           [3, 4]])
+
+    """
+    raise NotImplementedError('Waiting on interp level method')
+
+
+def put(a, ind, v, mode='raise'):
+    """
+    Replaces specified elements of an array with given values.
+
+    The indexing works on the flattened target array. `put` is roughly
+    equivalent to:
+
+    ::
+
+        a.flat[ind] = v
+
+    Parameters
+    ----------
+    a : ndarray
+        Target array.
+    ind : array_like
+        Target indices, interpreted as integers.
+    v : array_like
+        Values to place in `a` at target indices. If `v` is shorter than
+        `ind` it will be repeated as necessary.
+    mode : {'raise', 'wrap', 'clip'}, optional
+        Specifies how out-of-bounds indices will behave.
+
+        * 'raise' -- raise an error (default)
+        * 'wrap' -- wrap around
+        * 'clip' -- clip to the range
+
+        'clip' mode means that all indices that are too large are replaced
+        by the index that addresses the last element along that axis. Note
+        that this disables indexing with negative numbers.
+
+    See Also
+    --------
+    putmask, place
+
+    Examples
+    --------
+    >>> a = np.arange(5)
+    >>> np.put(a, [0, 2], [-44, -55])
+    >>> a
+    array([-44,   1, -55,   3,   4])
+
+    >>> a = np.arange(5)
+    >>> np.put(a, 22, -5, mode='clip')
+    >>> a
+    array([ 0,  1,  2,  3, -5])
+
+    """
+    raise NotImplementedError('Waiting on interp level method')
+
+
+def swapaxes(a, axis1, axis2):
+    """
+    Interchange two axes of an array.
+
+    Parameters
+    ----------
+    a : array_like
+        Input array.
+    axis1 : int
+        First axis.
+    axis2 : int
+        Second axis.
+
+    Returns
+    -------
+    a_swapped : ndarray
+        If `a` is an ndarray, then a view of `a` is returned; otherwise
+        a new array is created.
+
+    Examples
+    --------
+    >>> x = np.array([[1,2,3]])
+    >>> np.swapaxes(x,0,1)
+    array([[1],
+           [2],
+           [3]])
+
+    >>> x = np.array([[[0,1],[2,3]],[[4,5],[6,7]]])
+    >>> x
+    array([[[0, 1],
+            [2, 3]],
+           [[4, 5],
+            [6, 7]]])
+
+    >>> np.swapaxes(x,0,2)
+    array([[[0, 4],
+            [2, 6]],
+           [[1, 5],
+            [3, 7]]])
+
+    """
+    raise NotImplementedError('Waiting on interp level method')
+
+
+def transpose(a, axes=None):
+    """
+    Permute the dimensions of an array.
+
+    Parameters
+    ----------
+    a : array_like
+        Input array.
+    axes : list of ints, optional
+        By default, reverse the dimensions, otherwise permute the axes
+        according to the values given.
+
+    Returns
+    -------
+    p : ndarray
+        `a` with its axes permuted.  A view is returned whenever
+        possible.
+
+    See Also
+    --------
+    rollaxis
+
+    Examples
+    --------
+    >>> x = np.arange(4).reshape((2,2))
+    >>> x
+    array([[0, 1],
+           [2, 3]])
+
+    >>> np.transpose(x)
+    array([[0, 2],
+           [1, 3]])
+
+    >>> x = np.ones((1, 2, 3))
+    >>> np.transpose(x, (1, 0, 2)).shape
+    (2, 1, 3)
+
+    """
+    if axes is not None:
+        raise NotImplementedError('No "axes" arg yet.')
+    if not hasattr(a, 'T'):
+        a = numpypy.array(a)
+    return a.T
+
+def sort(a, axis=-1, kind='quicksort', order=None):
+    """
+    Return a sorted copy of an array.
+
+    Parameters
+    ----------
+    a : array_like
+        Array to be sorted.
+    axis : int or None, optional
+        Axis along which to sort. If None, the array is flattened before
+        sorting. The default is -1, which sorts along the last axis.
+    kind : {'quicksort', 'mergesort', 'heapsort'}, optional
+        Sorting algorithm. Default is 'quicksort'.
+    order : list, optional
+        When `a` is a structured array, this argument specifies which fields
+        to compare first, second, and so on.  This list does not need to
+        include all of the fields.
+
+    Returns
+    -------
+    sorted_array : ndarray
+        Array of the same type and shape as `a`.
+
+    See Also
+    --------
+    ndarray.sort : Method to sort an array in-place.
+    argsort : Indirect sort.
+    lexsort : Indirect stable sort on multiple keys.
+    searchsorted : Find elements in a sorted array.
+
+    Notes
+    -----
+    The various sorting algorithms are characterized by their average speed,
+    worst case performance, work space size, and whether they are stable. A
+    stable sort keeps items with the same key in the same relative
+    order. The three available algorithms have the following
+    properties:
+
+    =========== ======= ============= ============ =======
+       kind      speed   worst case    work space  stable
+    =========== ======= ============= ============ =======
+    'quicksort'    1     O(n^2)            0          no
+    'mergesort'    2     O(n*log(n))      ~n/2        yes
+    'heapsort'     3     O(n*log(n))       0          no
+    =========== ======= ============= ============ =======
+
+    All the sort algorithms make temporary copies of the data when
+    sorting along any but the last axis.  Consequently, sorting along
+    the last axis is faster and uses less space than sorting along
+    any other axis.
+
+    The sort order for complex numbers is lexicographic. If both the real
+    and imaginary parts are non-nan then the order is determined by the
+    real parts except when they are equal, in which case the order is
+    determined by the imaginary parts.
+
+    Previous to numpy 1.4.0 sorting real and complex arrays containing nan
+    values led to undefined behaviour. In numpy versions >= 1.4.0 nan
+    values are sorted to the end. The extended sort order is:
+
+      * Real: [R, nan]
+      * Complex: [R + Rj, R + nanj, nan + Rj, nan + nanj]
+
+    where R is a non-nan real value. Complex values with the same nan
+    placements are sorted according to the non-nan part if it exists.
+    Non-nan values are sorted as before.
+
+    Examples
+    --------
+    >>> a = np.array([[1,4],[3,1]])
+    >>> np.sort(a)                # sort along the last axis
+    array([[1, 4],
+           [1, 3]])
+    >>> np.sort(a, axis=None)     # sort the flattened array
+    array([1, 1, 3, 4])
+    >>> np.sort(a, axis=0)        # sort along the first axis
+    array([[1, 1],
+           [3, 4]])
+
+    Use the `order` keyword to specify a field to use when sorting a
+    structured array:
+
+    >>> dtype = [('name', 'S10'), ('height', float), ('age', int)]
+    >>> values = [('Arthur', 1.8, 41), ('Lancelot', 1.9, 38),
+    ...           ('Galahad', 1.7, 38)]
+    >>> a = np.array(values, dtype=dtype)       # create a structured array
+    >>> np.sort(a, order='height')                        # doctest: +SKIP
+    array([('Galahad', 1.7, 38), ('Arthur', 1.8, 41),
+           ('Lancelot', 1.8999999999999999, 38)],
+          dtype=[('name', '|S10'), ('height', '>> np.sort(a, order=['age', 'height'])               # doctest: +SKIP
+    array([('Galahad', 1.7, 38), ('Lancelot', 1.8999999999999999, 38),
+           ('Arthur', 1.8, 41)],
+          dtype=[('name', '|S10'), ('height', '>> x = np.array([3, 1, 2])
+    >>> np.argsort(x)
+    array([1, 2, 0])
+
+    Two-dimensional array:
+
+    >>> x = np.array([[0, 3], [2, 2]])
+    >>> x
+    array([[0, 3],
+           [2, 2]])
+
+    >>> np.argsort(x, axis=0)
+    array([[0, 1],
+           [1, 0]])
+
+    >>> np.argsort(x, axis=1)
+    array([[0, 1],
+           [0, 1]])
+
+    Sorting with keys:
+
+    >>> x = np.array([(1, 0), (0, 1)], dtype=[('x', '>> x
+    array([(1, 0), (0, 1)],
+          dtype=[('x', '>> np.argsort(x, order=('x','y'))
+    array([1, 0])
+
+    >>> np.argsort(x, order=('y','x'))
+    array([0, 1])
+
+    """
+    raise NotImplementedError('Waiting on interp level method')
+
+
+def argmax(a, axis=None):
+    """
+    Indices of the maximum values along an axis.
+
+    Parameters
+    ----------
+    a : array_like
+        Input array.
+    axis : int, optional
+        By default, the index is into the flattened array, otherwise
+        along the specified axis.
+
+    Returns
+    -------
+    index_array : ndarray of ints
+        Array of indices into the array. It has the same shape as `a.shape`
+        with the dimension along `axis` removed.
+
+    See Also
+    --------
+    ndarray.argmax, argmin
+    amax : The maximum value along a given axis.
+    unravel_index : Convert a flat index into an index tuple.
+
+    Notes
+    -----
+    In case of multiple occurrences of the maximum values, the indices
+    corresponding to the first occurrence are returned.
+
+    Examples
+    --------
+    >>> a = np.arange(6).reshape(2,3)
+    >>> a
+    array([[0, 1, 2],
+           [3, 4, 5]])
+    >>> np.argmax(a)
+    5
+    >>> np.argmax(a, axis=0)
+    array([1, 1, 1])
+    >>> np.argmax(a, axis=1)
+    array([2, 2])
+
+    >>> b = np.arange(6)
+    >>> b[1] = 5
+    >>> b
+    array([0, 5, 2, 3, 4, 5])
+    >>> np.argmax(b) # Only the first occurrence is returned.
+    1
+
+    """
+    if not hasattr(a, 'argmax'):
+        a = numpypy.array(a)
+    return a.argmax()
+
+
+def argmin(a, axis=None):
+    """
+    Return the indices of the minimum values along an axis.
+
+    See Also
+    --------
+    argmax : Similar function.  Please refer to `numpy.argmax` for detailed
+        documentation.
+
+    """
+    if not hasattr(a, 'argmin'):
+        a = numpypy.array(a)
+    return a.argmin()
+
+
+def searchsorted(a, v, side='left'):
+    """
+    Find indices where elements should be inserted to maintain order.
+
+    Find the indices into a sorted array `a` such that, if the corresponding
+    elements in `v` were inserted before the indices, the order of `a` would
+    be preserved.
+
+    Parameters
+    ----------
+    a : 1-D array_like
+        Input array, sorted in ascending order.
+    v : array_like
+        Values to insert into `a`.
+    side : {'left', 'right'}, optional
+        If 'left', the index of the first suitable location found is given.  If
+        'right', return the last such index.  If there is no suitable
+        index, return either 0 or N (where N is the length of `a`).
+
+    Returns
+    -------
+    indices : array of ints
+        Array of insertion points with the same shape as `v`.
+
+    See Also
+    --------
+    sort : Return a sorted copy of an array.
+    histogram : Produce histogram from 1-D data.
+
+    Notes
+    -----
+    Binary search is used to find the required insertion points.
+
+    As of Numpy 1.4.0 `searchsorted` works with real/complex arrays containing
+    `nan` values. The enhanced sort order is documented in `sort`.
+
+    Examples
+    --------
+    >>> np.searchsorted([1,2,3,4,5], 3)
+    2
+    >>> np.searchsorted([1,2,3,4,5], 3, side='right')
+    3
+    >>> np.searchsorted([1,2,3,4,5], [-10, 10, 2, 3])
+    array([0, 5, 1, 2])
+
+    """
+    raise NotImplementedError('Waiting on interp level method')
+
+
+def resize(a, new_shape):
+    """
+    Return a new array with the specified shape.
+
+    If the new array is larger than the original array, then the new
+    array is filled with repeated copies of `a`.  Note that this behavior
+    is different from a.resize(new_shape) which fills with zeros instead
+    of repeated copies of `a`.
+
+    Parameters
+    ----------
+    a : array_like
+        Array to be resized.
+
+    new_shape : int or tuple of int
+        Shape of resized array.
+
+    Returns
+    -------
+    reshaped_array : ndarray
+        The new array is formed from the data in the old array, repeated
+        if necessary to fill out the required number of elements.  The
+        data are repeated in the order that they are stored in memory.
+
+    See Also
+    --------
+    ndarray.resize : resize an array in-place.
+
+    Examples
+    --------
+    >>> a=np.array([[0,1],[2,3]])
+    >>> np.resize(a,(1,4))
+    array([[0, 1, 2, 3]])
+    >>> np.resize(a,(2,4))
+    array([[0, 1, 2, 3],
+           [0, 1, 2, 3]])
+
+    """
+    raise NotImplementedError('Waiting on interp level method')
+
+
+def squeeze(a):
+    """
+    Remove single-dimensional entries from the shape of an array.
+
+    Parameters
+    ----------
+    a : array_like
+        Input data.
+
+    Returns
+    -------
+    squeezed : ndarray
+        The input array, but with with all dimensions of length 1
+        removed.  Whenever possible, a view on `a` is returned.
+
+    Examples
+    --------
+    >>> x = np.array([[[0], [1], [2]]])
+    >>> x.shape
+    (1, 3, 1)
+    >>> np.squeeze(x).shape
+    (3,)
+
+    """
+    raise NotImplementedError('Waiting on interp level method')
+
+
+def diagonal(a, offset=0, axis1=0, axis2=1):
+    """
+    Return specified diagonals.
+
+    If `a` is 2-D, returns the diagonal of `a` with the given offset,
+    i.e., the collection of elements of the form ``a[i, i+offset]``.  If
+    `a` has more than two dimensions, then the axes specified by `axis1`
+    and `axis2` are used to determine the 2-D sub-array whose diagonal is
+    returned.  The shape of the resulting array can be determined by
+    removing `axis1` and `axis2` and appending an index to the right equal
+    to the size of the resulting diagonals.
+
+    Parameters
+    ----------
+    a : array_like
+        Array from which the diagonals are taken.
+    offset : int, optional
+        Offset of the diagonal from the main diagonal.  Can be positive or
+        negative.  Defaults to main diagonal (0).
+    axis1 : int, optional
+        Axis to be used as the first axis of the 2-D sub-arrays from which
+        the diagonals should be taken.  Defaults to first axis (0).
+    axis2 : int, optional
+        Axis to be used as the second axis of the 2-D sub-arrays from
+        which the diagonals should be taken. Defaults to second axis (1).
+
+    Returns
+    -------
+    array_of_diagonals : ndarray
+        If `a` is 2-D, a 1-D array containing the diagonal is returned.
+        If the dimension of `a` is larger, then an array of diagonals is
+        returned, "packed" from left-most dimension to right-most (e.g.,
+        if `a` is 3-D, then the diagonals are "packed" along rows).
+
+    Raises
+    ------
+    ValueError
+        If the dimension of `a` is less than 2.
+
+    See Also
+    --------
+    diag : MATLAB work-a-like for 1-D and 2-D arrays.
+    diagflat : Create diagonal arrays.
+    trace : Sum along diagonals.
+
+    Examples
+    --------
+    >>> a = np.arange(4).reshape(2,2)
+    >>> a
+    array([[0, 1],
+           [2, 3]])
+    >>> a.diagonal()
+    array([0, 3])
+    >>> a.diagonal(1)
+    array([1])
+
+    A 3-D example:
+
+    >>> a = np.arange(8).reshape(2,2,2); a
+    array([[[0, 1],
+            [2, 3]],
+           [[4, 5],
+            [6, 7]]])
+    >>> a.diagonal(0, # Main diagonals of two arrays created by skipping
+    ...            0, # across the outer(left)-most axis last and
+    ...            1) # the "middle" (row) axis first.
+    array([[0, 6],
+           [1, 7]])
+
+    The sub-arrays whose main diagonals we just obtained; note that each
+    corresponds to fixing the right-most (column) axis, and that the
+    diagonals are "packed" in rows.
+
+    >>> a[:,:,0] # main diagonal is [0 6]
+    array([[0, 2],
+           [4, 6]])
+    >>> a[:,:,1] # main diagonal is [1 7]
+    array([[1, 3],
+           [5, 7]])
+
+    """
+    raise NotImplementedError('Waiting on interp level method')
+
+
+def trace(a, offset=0, axis1=0, axis2=1, dtype=None, out=None):
+    """
+    Return the sum along diagonals of the array.
+
+    If `a` is 2-D, the sum along its diagonal with the given offset
+    is returned, i.e., the sum of elements ``a[i,i+offset]`` for all i.
+
+    If `a` has more than two dimensions, then the axes specified by axis1 and
+    axis2 are used to determine the 2-D sub-arrays whose traces are returned.
+    The shape of the resulting array is the same as that of `a` with `axis1`
+    and `axis2` removed.
+
+    Parameters
+    ----------
+    a : array_like
+        Input array, from which the diagonals are taken.
+    offset : int, optional
+        Offset of the diagonal from the main diagonal. Can be both positive
+        and negative. Defaults to 0.
+    axis1, axis2 : int, optional
+        Axes to be used as the first and second axis of the 2-D sub-arrays
+        from which the diagonals should be taken. Defaults are the first two
+        axes of `a`.
+    dtype : dtype, optional
+        Determines the data-type of the returned array and of the accumulator
+        where the elements are summed. If dtype has the value None and `a` is
+        of integer type of precision less than the default integer
+        precision, then the default integer precision is used. Otherwise,
+        the precision is the same as that of `a`.
+    out : ndarray, optional
+        Array into which the output is placed. Its type is preserved and
+        it must be of the right shape to hold the output.
+
+    Returns
+    -------
+    sum_along_diagonals : ndarray
+        If `a` is 2-D, the sum along the diagonal is returned.  If `a` has
+        larger dimensions, then an array of sums along diagonals is returned.
+
+    See Also
+    --------
+    diag, diagonal, diagflat
+
+    Examples
+    --------
+    >>> np.trace(np.eye(3))
+    3.0
+    >>> a = np.arange(8).reshape((2,2,2))
+    >>> np.trace(a)
+    array([6, 8])
+
+    >>> a = np.arange(24).reshape((2,2,2,3))
+    >>> np.trace(a).shape
+    (2, 3)
+
+    """
+    raise NotImplementedError('Waiting on interp level method')
+
+def ravel(a, order='C'):
+    """
+    Return a flattened array.
+
+    A 1-D array, containing the elements of the input, is returned.  A copy is
+    made only if needed.
+
+    Parameters
+    ----------
+    a : array_like
+        Input array.  The elements in ``a`` are read in the order specified by
+        `order`, and packed as a 1-D array.
+    order : {'C','F', 'A', 'K'}, optional
+        The elements of ``a`` are read in this order. 'C' means to view
+        the elements in C (row-major) order. 'F' means to view the elements
+        in Fortran (column-major) order. 'A' means to view the elements
+        in 'F' order if a is Fortran contiguous, 'C' order otherwise.
+        'K' means to view the elements in the order they occur in memory,
+        except for reversing the data when strides are negative.
+        By default, 'C' order is used.
+
+    Returns
+    -------
+    1d_array : ndarray
+        Output of the same dtype as `a`, and of shape ``(a.size(),)``.
+
+    See Also
+    --------
+    ndarray.flat : 1-D iterator over an array.
+    ndarray.flatten : 1-D array copy of the elements of an array
+                      in row-major order.
+
+    Notes
+    -----
+    In row-major order, the row index varies the slowest, and the column
+    index the quickest.  This can be generalized to multiple dimensions,
+    where row-major order implies that the index along the first axis
+    varies slowest, and the index along the last quickest.  The opposite holds
+    for Fortran-, or column-major, mode.
+
+    Examples
+    --------
+    It is equivalent to ``reshape(-1, order=order)``.
+
+    >>> x = np.array([[1, 2, 3], [4, 5, 6]])
+    >>> print np.ravel(x)
+    [1 2 3 4 5 6]
+
+    >>> print x.reshape(-1)
+    [1 2 3 4 5 6]
+
+    >>> print np.ravel(x, order='F')
+    [1 4 2 5 3 6]
+
+    When ``order`` is 'A', it will preserve the array's 'C' or 'F' ordering:
+
+    >>> print np.ravel(x.T)
+    [1 4 2 5 3 6]
+    >>> print np.ravel(x.T, order='A')
+    [1 2 3 4 5 6]
+
+    When ``order`` is 'K', it will preserve orderings that are neither 'C'
+    nor 'F', but won't reverse axes:
+
+    >>> a = np.arange(3)[::-1]; a
+    array([2, 1, 0])
+    >>> a.ravel(order='C')
+    array([2, 1, 0])
+    >>> a.ravel(order='K')
+    array([2, 1, 0])
+
+    >>> a = np.arange(12).reshape(2,3,2).swapaxes(1,2); a
+    array([[[ 0,  2,  4],
+            [ 1,  3,  5]],
+           [[ 6,  8, 10],
+            [ 7,  9, 11]]])
+    >>> a.ravel(order='C')
+    array([ 0,  2,  4,  1,  3,  5,  6,  8, 10,  7,  9, 11])
+    >>> a.ravel(order='K')
+    array([ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11])
+
+    """
+    raise NotImplementedError('Waiting on interp level method')
+
+
+def nonzero(a):
+    """
+    Return the indices of the elements that are non-zero.
+
+    Returns a tuple of arrays, one for each dimension of `a`, containing
+    the indices of the non-zero elements in that dimension. The
+    corresponding non-zero values can be obtained with::
+
+        a[nonzero(a)]
+
+    To group the indices by element, rather than dimension, use::
+
+        transpose(nonzero(a))
+
+    The result of this is always a 2-D array, with a row for
+    each non-zero element.
+
+    Parameters
+    ----------
+    a : array_like
+        Input array.
+
+    Returns
+    -------
+    tuple_of_arrays : tuple
+        Indices of elements that are non-zero.
+
+    See Also
+    --------
+    flatnonzero :
+        Return indices that are non-zero in the flattened version of the input
+        array.
+    ndarray.nonzero :
+        Equivalent ndarray method.
+    count_nonzero :
+        Counts the number of non-zero elements in the input array.
+
+    Examples
+    --------
+    >>> x = np.eye(3)
+    >>> x
+    array([[ 1.,  0.,  0.],
+           [ 0.,  1.,  0.],
+           [ 0.,  0.,  1.]])
+    >>> np.nonzero(x)
+    (array([0, 1, 2]), array([0, 1, 2]))
+
+    >>> x[np.nonzero(x)]
+    array([ 1.,  1.,  1.])
+    >>> np.transpose(np.nonzero(x))
+    array([[0, 0],
+           [1, 1],
+           [2, 2]])
+
+    A common use for ``nonzero`` is to find the indices of an array, where
+    a condition is True.  Given an array `a`, the condition `a` > 3 is a
+    boolean array and since False is interpreted as 0, np.nonzero(a > 3)
+    yields the indices of the `a` where the condition is true.
+
+    >>> a = np.array([[1,2,3],[4,5,6],[7,8,9]])
+    >>> a > 3
+    array([[False, False, False],
+           [ True,  True,  True],
+           [ True,  True,  True]], dtype=bool)
+    >>> np.nonzero(a > 3)
+    (array([1, 1, 1, 2, 2, 2]), array([0, 1, 2, 0, 1, 2]))
+
+    The ``nonzero`` method of the boolean array can also be called.
+
+    >>> (a > 3).nonzero()
+    (array([1, 1, 1, 2, 2, 2]), array([0, 1, 2, 0, 1, 2]))
+
+    """
+    raise NotImplementedError('Waiting on interp level method')
+
+
+def shape(a):
+    """
+    Return the shape of an array.
+
+    Parameters
+    ----------
+    a : array_like
+        Input array.
+
+    Returns
+    -------
+    shape : tuple of ints
+        The elements of the shape tuple give the lengths of the
+        corresponding array dimensions.
+
+    See Also
+    --------
+    alen
+    ndarray.shape : Equivalent array method.
+
+    Examples
+    --------
+    >>> np.shape(np.eye(3))
+    (3, 3)
+    >>> np.shape([[1, 2]])
+    (1, 2)
+    >>> np.shape([0])
+    (1,)
+    >>> np.shape(0)
+    ()
+
+    >>> a = np.array([(1, 2), (3, 4)], dtype=[('x', 'i4'), ('y', 'i4')])
+    >>> np.shape(a)
+    (2,)
+    >>> a.shape
+    (2,)
+
+    """
+    if not hasattr(a, 'shape'):
+        a = numpypy.array(a)
+    return a.shape
+
+
+def compress(condition, a, axis=None, out=None):
+    """
+    Return selected slices of an array along given axis.
+
+    When working along a given axis, a slice along that axis is returned in
+    `output` for each index where `condition` evaluates to True. When
+    working on a 1-D array, `compress` is equivalent to `extract`.
+
+    Parameters
+    ----------
+    condition : 1-D array of bools
+        Array that selects which entries to return. If len(condition)
+        is less than the size of `a` along the given axis, then output is
+        truncated to the length of the condition array.
+    a : array_like
+        Array from which to extract a part.
+    axis : int, optional
+        Axis along which to take slices. If None (default), work on the
+        flattened array.
+    out : ndarray, optional
+        Output array.  Its type is preserved and it must be of the right
+        shape to hold the output.
+
+    Returns
+    -------
+    compressed_array : ndarray
+        A copy of `a` without the slices along axis for which `condition`
+        is false.
+
+    See Also
+    --------
+    take, choose, diag, diagonal, select
+    ndarray.compress : Equivalent method.
+    numpy.doc.ufuncs : Section "Output arguments"
+
+    Examples
+    --------
+    >>> a = np.array([[1, 2], [3, 4], [5, 6]])
+    >>> a
+    array([[1, 2],
+           [3, 4],
+           [5, 6]])
+    >>> np.compress([0, 1], a, axis=0)
+    array([[3, 4]])
+    >>> np.compress([False, True, True], a, axis=0)
+    array([[3, 4],
+           [5, 6]])
+    >>> np.compress([False, True], a, axis=1)
+    array([[2],
+           [4],
+           [6]])
+
+    Working on the flattened array does not return slices along an axis but
+    selects elements.
+
+    >>> np.compress([False, True], a)
+    array([2])
+
+    """
+    raise NotImplementedError('Waiting on interp level method')
+
+
+def clip(a, a_min, a_max, out=None):
+    """
+    Clip (limit) the values in an array.
+
+    Given an interval, values outside the interval are clipped to
+    the interval edges.  For example, if an interval of ``[0, 1]``
+    is specified, values smaller than 0 become 0, and values larger
+    than 1 become 1.
+
+    Parameters
+    ----------
+    a : array_like
+        Array containing elements to clip.
+    a_min : scalar or array_like
+        Minimum value.
+    a_max : scalar or array_like
+        Maximum value.  If `a_min` or `a_max` are array_like, then they will
+        be broadcasted to the shape of `a`.
+    out : ndarray, optional
+        The results will be placed in this array. It may be the input
+        array for in-place clipping.  `out` must be of the right shape
+        to hold the output.  Its type is preserved.
+
+    Returns
+    -------
+    clipped_array : ndarray
+        An array with the elements of `a`, but where values
+        < `a_min` are replaced with `a_min`, and those > `a_max`
+        with `a_max`.
+
+    See Also
+    --------
+    numpy.doc.ufuncs : Section "Output arguments"
+
+    Examples
+    --------
+    >>> a = np.arange(10)
+    >>> np.clip(a, 1, 8)
+    array([1, 1, 2, 3, 4, 5, 6, 7, 8, 8])
+    >>> a
+    array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])
+    >>> np.clip(a, 3, 6, out=a)
+    array([3, 3, 3, 3, 4, 5, 6, 6, 6, 6])
+    >>> a = np.arange(10)
+    >>> a
+    array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])
+    >>> np.clip(a, [3,4,1,1,1,4,4,4,4,4], 8)
+    array([3, 4, 2, 3, 4, 5, 6, 7, 8, 8])
+
+    """
+    raise NotImplementedError('Waiting on interp level method')
+
+
+def sum(a, axis=None, dtype=None, out=None):
+    """
+    Sum of array elements over a given axis.
+
+    Parameters
+    ----------
+    a : array_like
+        Elements to sum.
+    axis : integer, optional
+        Axis over which the sum is taken. By default `axis` is None,
+        and all elements are summed.
+    dtype : dtype, optional
+        The type of the returned array and of the accumulator in which
+        the elements are summed.  By default, the dtype of `a` is used.
+        An exception is when `a` has an integer type with less precision
+        than the default platform integer.  In that case, the default
+        platform integer is used instead.
+    out : ndarray, optional
+        Array into which the output is placed.  By default, a new array is
+        created.  If `out` is given, it must be of the appropriate shape
+        (the shape of `a` with `axis` removed, i.e.,
+        ``numpy.delete(a.shape, axis)``).  Its type is preserved. See
+        `doc.ufuncs` (Section "Output arguments") for more details.
+
+    Returns
+    -------
+    sum_along_axis : ndarray
+        An array with the same shape as `a`, with the specified
+        axis removed.   If `a` is a 0-d array, or if `axis` is None, a scalar
+        is returned.  If an output array is specified, a reference to
+        `out` is returned.
+
+    See Also
+    --------
+    ndarray.sum : Equivalent method.
+
+    cumsum : Cumulative sum of array elements.
+
+    trapz : Integration of array values using the composite trapezoidal rule.
+
+    mean, average
+
+    Notes
+    -----
+    Arithmetic is modular when using integer types, and no error is
+    raised on overflow.
+
+    Examples
+    --------
+    >>> np.sum([0.5, 1.5])
+    2.0
+    >>> np.sum([0.5, 0.7, 0.2, 1.5], dtype=np.int32)
+    1
+    >>> np.sum([[0, 1], [0, 5]])
+    6
+    >>> np.sum([[0, 1], [0, 5]], axis=0)
+    array([0, 6])
+    >>> np.sum([[0, 1], [0, 5]], axis=1)
+    array([1, 5])
+
+    If the accumulator is too small, overflow occurs:
+
+    >>> np.ones(128, dtype=np.int8).sum(dtype=np.int8)
+    -128
+
+    """
+    if not hasattr(a, "sum"):
+        a = numpypy.array(a)
+    return a.sum()
+
+
+def product (a, axis=None, dtype=None, out=None):
+    """
+    Return the product of array elements over a given axis.
+
+    See Also
+    --------
+    prod : equivalent function; see for details.
+
+    """
+    raise NotImplementedError('Waiting on interp level method')
+
+
+def sometrue(a, axis=None, out=None):
+    """
+    Check whether some values are true.
+
+    Refer to `any` for full documentation.
+
+    See Also
+    --------
+    any : equivalent function
+
+    """
+    if not hasattr(a, 'any'):
+        a = numpypy.array(a)
+    return a.any()
+
+
+def alltrue (a, axis=None, out=None):
+    """
+    Check if all elements of input array are true.
+
+    See Also
+    --------
+    numpy.all : Equivalent function; see for details.
+
+    """
+    if not hasattr(a, 'all'):
+        a = numpypy.array(a)
+    return a.all()
+
+def any(a,axis=None, out=None):
+    """
+    Test whether any array element along a given axis evaluates to True.
+
+    Returns single boolean unless `axis` is not ``None``
+
+    Parameters
+    ----------
+    a : array_like
+        Input array or object that can be converted to an array.
+    axis : int, optional
+        Axis along which a logical OR is performed.  The default
+        (`axis` = `None`) is to perform a logical OR over a flattened
+        input array. `axis` may be negative, in which case it counts
+        from the last to the first axis.
+    out : ndarray, optional
+        Alternate output array in which to place the result.  It must have
+        the same shape as the expected output and its type is preserved
+        (e.g., if it is of type float, then it will remain so, returning
+        1.0 for True and 0.0 for False, regardless of the type of `a`).
+        See `doc.ufuncs` (Section "Output arguments") for details.
+
+    Returns
+    -------
+    any : bool or ndarray
+        A new boolean or `ndarray` is returned unless `out` is specified,
+        in which case a reference to `out` is returned.
+
+    See Also
+    --------
+    ndarray.any : equivalent method
+
+    all : Test whether all elements along a given axis evaluate to True.
+
+    Notes
+    -----
+    Not a Number (NaN), positive infinity and negative infinity evaluate
+    to `True` because these are not equal to zero.
+
+    Examples
+    --------
+    >>> np.any([[True, False], [True, True]])
+    True
+
+    >>> np.any([[True, False], [False, False]], axis=0)
+    array([ True, False], dtype=bool)
+
+    >>> np.any([-1, 0, 5])
+    True
+
+    >>> np.any(np.nan)
+    True
+
+    >>> o=np.array([False])
+    >>> z=np.any([-1, 4, 5], out=o)
+    >>> z, o
+    (array([ True], dtype=bool), array([ True], dtype=bool))
+    >>> # Check now that z is a reference to o
+    >>> z is o
+    True
+    >>> id(z), id(o) # identity of z and o              # doctest: +SKIP
+    (191614240, 191614240)
+
+    """
+    if not hasattr(a, 'any'):
+        a = numpypy.array(a)
+    return a.any()
+
+
+def all(a,axis=None, out=None):
+    """
+    Test whether all array elements along a given axis evaluate to True.
+
+    Parameters
+    ----------
+    a : array_like
+        Input array or object that can be converted to an array.
+    axis : int, optional
+        Axis along which a logical AND is performed.
+        The default (`axis` = `None`) is to perform a logical AND
+        over a flattened input array.  `axis` may be negative, in which
+        case it counts from the last to the first axis.
+    out : ndarray, optional
+        Alternate output array in which to place the result.
+        It must have the same shape as the expected output and its
+        type is preserved (e.g., if ``dtype(out)`` is float, the result
+        will consist of 0.0's and 1.0's).  See `doc.ufuncs` (Section
+        "Output arguments") for more details.
+
+    Returns
+    -------
+    all : ndarray, bool
+        A new boolean or array is returned unless `out` is specified,
+        in which case a reference to `out` is returned.
+
+    See Also
+    --------
+    ndarray.all : equivalent method
+
+    any : Test whether any element along a given axis evaluates to True.
+
+    Notes
+    -----
+    Not a Number (NaN), positive infinity and negative infinity
+    evaluate to `True` because these are not equal to zero.
+
+    Examples
+    --------
+    >>> np.all([[True,False],[True,True]])
+    False
+
+    >>> np.all([[True,False],[True,True]], axis=0)
+    array([ True, False], dtype=bool)
+
+    >>> np.all([-1, 4, 5])
+    True
+
+    >>> np.all([1.0, np.nan])
+    True
+
+    >>> o=np.array([False])
+    >>> z=np.all([-1, 4, 5], out=o)
+    >>> id(z), id(o), z                             # doctest: +SKIP
+    (28293632, 28293632, array([ True], dtype=bool))
+
+    """
+    if not hasattr(a, 'all'):
+        a = numpypy.array(a)
+    return a.all()
+
+
+def cumsum (a, axis=None, dtype=None, out=None):
+    """
+    Return the cumulative sum of the elements along a given axis.
+
+    Parameters
+    ----------
+    a : array_like
+        Input array.
+    axis : int, optional
+        Axis along which the cumulative sum is computed. The default
+        (None) is to compute the cumsum over the flattened array.
+    dtype : dtype, optional
+        Type of the returned array and of the accumulator in which the
+        elements are summed.  If `dtype` is not specified, it defaults
+        to the dtype of `a`, unless `a` has an integer dtype with a
+        precision less than that of the default platform integer.  In
+        that case, the default platform integer is used.
+    out : ndarray, optional
+        Alternative output array in which to place the result. It must
+        have the same shape and buffer length as the expected output
+        but the type will be cast if necessary. See `doc.ufuncs`
+        (Section "Output arguments") for more details.
+
+    Returns
+    -------
+    cumsum_along_axis : ndarray.
+        A new array holding the result is returned unless `out` is
+        specified, in which case a reference to `out` is returned. The
+        result has the same size as `a`, and the same shape as `a` if
+        `axis` is not None or `a` is a 1-d array.
+
+
+    See Also
+    --------
+    sum : Sum array elements.
+
+    trapz : Integration of array values using the composite trapezoidal rule.
+
+    Notes
+    -----
+    Arithmetic is modular when using integer types, and no error is
+    raised on overflow.
+
+    Examples
+    --------
+    >>> a = np.array([[1,2,3], [4,5,6]])
+    >>> a
+    array([[1, 2, 3],
+           [4, 5, 6]])
+    >>> np.cumsum(a)
+    array([ 1,  3,  6, 10, 15, 21])
+    >>> np.cumsum(a, dtype=float)     # specifies type of output value(s)
+    array([  1.,   3.,   6.,  10.,  15.,  21.])
+
+    >>> np.cumsum(a,axis=0)      # sum over rows for each of the 3 columns
+    array([[1, 2, 3],
+           [5, 7, 9]])
+    >>> np.cumsum(a,axis=1)      # sum over columns for each of the 2 rows
+    array([[ 1,  3,  6],
+           [ 4,  9, 15]])
+
+    """
+    raise NotImplementedError('Waiting on interp level method')
+
+
+def cumproduct(a, axis=None, dtype=None, out=None):
+    """
+    Return the cumulative product over the given axis.
+
+
+    See Also
+    --------
+    cumprod : equivalent function; see for details.
+
+    """
+    raise NotImplementedError('Waiting on interp level method')
+
+
+def ptp(a, axis=None, out=None):
+    """
+    Range of values (maximum - minimum) along an axis.
+
+    The name of the function comes from the acronym for 'peak to peak'.
+
+    Parameters
+    ----------
+    a : array_like
+        Input values.
+    axis : int, optional
+        Axis along which to find the peaks.  By default, flatten the
+        array.
+    out : array_like
+        Alternative output array in which to place the result. It must
+        have the same shape and buffer length as the expected output,
+        but the type of the output values will be cast if necessary.
+
+    Returns
+    -------
+    ptp : ndarray
+        A new array holding the result, unless `out` was
+        specified, in which case a reference to `out` is returned.
+
+    Examples
+    --------
+    >>> x = np.arange(4).reshape((2,2))
+    >>> x
+    array([[0, 1],
+           [2, 3]])
+
+    >>> np.ptp(x, axis=0)
+    array([2, 2])
+
+    >>> np.ptp(x, axis=1)
+    array([1, 1])
+
+    """
+    raise NotImplementedError('Waiting on interp level method')
+
+
+def amax(a, axis=None, out=None):
+    """
+    Return the maximum of an array or maximum along an axis.
+
+    Parameters
+    ----------
+    a : array_like
+        Input data.
+    axis : int, optional
+        Axis along which to operate.  By default flattened input is used.
+    out : ndarray, optional
+        Alternate output array in which to place the result.  Must be of
+        the same shape and buffer length as the expected output.  See
+        `doc.ufuncs` (Section "Output arguments") for more details.
+
+    Returns
+    -------
+    amax : ndarray or scalar
+        Maximum of `a`. If `axis` is None, the result is a scalar value.
+        If `axis` is given, the result is an array of dimension
+        ``a.ndim - 1``.
+
+    See Also
+    --------
+    nanmax : NaN values are ignored instead of being propagated.
+    fmax : same behavior as the C99 fmax function.
+    argmax : indices of the maximum values.
+
+    Notes
+    -----
+    NaN values are propagated, that is if at least one item is NaN, the
+    corresponding max value will be NaN as well.  To ignore NaN values
+    (MATLAB behavior), please use nanmax.
+
+    Examples
+    --------
+    >>> a = np.arange(4).reshape((2,2))
+    >>> a
+    array([[0, 1],
+           [2, 3]])
+    >>> np.amax(a)
+    3
+    >>> np.amax(a, axis=0)
+    array([2, 3])
+    >>> np.amax(a, axis=1)
+    array([1, 3])
+
+    >>> b = np.arange(5, dtype=np.float)
+    >>> b[2] = np.NaN
+    >>> np.amax(b)
+    nan
+    >>> np.nanmax(b)
+    4.0
+
+    """
+    if not hasattr(a, "max"):
+        a = numpypy.array(a)
+    return a.max()
+
+
+def amin(a, axis=None, out=None):
+    """
+    Return the minimum of an array or minimum along an axis.
+
+    Parameters
+    ----------
+    a : array_like
+        Input data.
+    axis : int, optional
+        Axis along which to operate.  By default a flattened input is used.
+    out : ndarray, optional
+        Alternative output array in which to place the result.  Must
+        be of the same shape and buffer length as the expected output.
+        See `doc.ufuncs` (Section "Output arguments") for more details.
+
+    Returns
+    -------
+    amin : ndarray
+        A new array or a scalar array with the result.
+
+    See Also
+    --------
+    nanmin: nan values are ignored instead of being propagated
+    fmin: same behavior as the C99 fmin function
+    argmin: Return the indices of the minimum values.
+
+    amax, nanmax, fmax
+
+    Notes
+    -----
+    NaN values are propagated, that is if at least one item is nan, the
+    corresponding min value will be nan as well. To ignore NaN values (matlab
+    behavior), please use nanmin.
+
+    Examples
+    --------
+    >>> a = np.arange(4).reshape((2,2))
+    >>> a
+    array([[0, 1],
+           [2, 3]])
+    >>> np.amin(a)           # Minimum of the flattened array
+    0
+    >>> np.amin(a, axis=0)         # Minima along the first axis
+    array([0, 1])
+    >>> np.amin(a, axis=1)         # Minima along the second axis
+    array([0, 2])
+
+    >>> b = np.arange(5, dtype=np.float)
+    >>> b[2] = np.NaN
+    >>> np.amin(b)
+    nan
+    >>> np.nanmin(b)
+    0.0
+
+    """
+    # amin() is equivalent to min()
+    if not hasattr(a, 'min'):
+        a = numpypy.array(a)
+    return a.min()
+
+def alen(a):
+    """
+    Return the length of the first dimension of the input array.
+
+    Parameters
+    ----------
+    a : array_like
+       Input array.
+
+    Returns
+    -------
+    l : int
+       Length of the first dimension of `a`.
+
+    See Also
+    --------
+    shape, size
+
+    Examples
+    --------
+    >>> a = np.zeros((7,4,5))
+    >>> a.shape[0]
+    7
+    >>> np.alen(a)
+    7
+
+    """
+    if not hasattr(a, 'shape'):
+        a = numpypy.array(a)
+    return a.shape[0]
+
+
+def prod(a, axis=None, dtype=None, out=None):
+    """
+    Return the product of array elements over a given axis.
+
+    Parameters
+    ----------
+    a : array_like
+        Input data.
+    axis : int, optional
+        Axis over which the product is taken.  By default, the product
+        of all elements is calculated.
+    dtype : data-type, optional
+        The data-type of the returned array, as well as of the accumulator
+        in which the elements are multiplied.  By default, if `a` is of
+        integer type, `dtype` is the default platform integer. (Note: if
+        the type of `a` is unsigned, then so is `dtype`.)  Otherwise,
+        the dtype is the same as that of `a`.
+    out : ndarray, optional
+        Alternative output array in which to place the result. It must have
+        the same shape as the expected output, but the type of the
+        output values will be cast if necessary.
+
+    Returns
+    -------
+    product_along_axis : ndarray, see `dtype` parameter above.
+        An array shaped as `a` but with the specified axis removed.
+        Returns a reference to `out` if specified.
+
+    See Also
+    --------
+    ndarray.prod : equivalent method
+    numpy.doc.ufuncs : Section "Output arguments"
+
+    Notes
+    -----
+    Arithmetic is modular when using integer types, and no error is
+    raised on overflow.  That means that, on a 32-bit platform:
+
+    >>> x = np.array([536870910, 536870910, 536870910, 536870910])
+    >>> np.prod(x) #random
+    16
+
+    Examples
+    --------
+    By default, calculate the product of all elements:
+
+    >>> np.prod([1.,2.])
+    2.0
+
+    Even when the input array is two-dimensional:
+
+    >>> np.prod([[1.,2.],[3.,4.]])
+    24.0
+
+    But we can also specify the axis over which to multiply:
+
+    >>> np.prod([[1.,2.],[3.,4.]], axis=1)
+    array([  2.,  12.])
+
+    If the type of `x` is unsigned, then the output type is
+    the unsigned platform integer:
+
+    >>> x = np.array([1, 2, 3], dtype=np.uint8)
+    >>> np.prod(x).dtype == np.uint
+    True
+
+    If `x` is of a signed integer type, then the output type
+    is the default platform integer:
+
+    >>> x = np.array([1, 2, 3], dtype=np.int8)
+    >>> np.prod(x).dtype == np.int
+    True
+
+    """
+    raise NotImplementedError('Waiting on interp level method')
+
+
+def cumprod(a, axis=None, dtype=None, out=None):
+    """
+    Return the cumulative product of elements along a given axis.
+
+    Parameters
+    ----------
+    a : array_like
+        Input array.
+    axis : int, optional
+        Axis along which the cumulative product is computed.  By default
+        the input is flattened.
+    dtype : dtype, optional
+        Type of the returned array, as well as of the accumulator in which
+        the elements are multiplied.  If *dtype* is not specified, it
+        defaults to the dtype of `a`, unless `a` has an integer dtype with
+        a precision less than that of the default platform integer.  In
+        that case, the default platform integer is used instead.
+    out : ndarray, optional
+        Alternative output array in which to place the result. It must
+        have the same shape and buffer length as the expected output
+        but the type of the resulting values will be cast if necessary.
+
+    Returns
+    -------
+    cumprod : ndarray
+        A new array holding the result is returned unless `out` is
+        specified, in which case a reference to out is returned.
+
+    See Also
+    --------
+    numpy.doc.ufuncs : Section "Output arguments"
+
+    Notes
+    -----
+    Arithmetic is modular when using integer types, and no error is
+    raised on overflow.
+
+    Examples
+    --------
+    >>> a = np.array([1,2,3])
+    >>> np.cumprod(a) # intermediate results 1, 1*2
+    ...               # total product 1*2*3 = 6
+    array([1, 2, 6])
+    >>> a = np.array([[1, 2, 3], [4, 5, 6]])
+    >>> np.cumprod(a, dtype=float) # specify type of output
+    array([   1.,    2.,    6.,   24.,  120.,  720.])
+
+    The cumulative product for each column (i.e., over the rows) of `a`:
+
+    >>> np.cumprod(a, axis=0)
+    array([[ 1,  2,  3],
+           [ 4, 10, 18]])
+
+    The cumulative product for each row (i.e. over the columns) of `a`:
+
+    >>> np.cumprod(a,axis=1)
+    array([[  1,   2,   6],
+           [  4,  20, 120]])
+
+    """
+    raise NotImplementedError('Waiting on interp level method')
+
+
+def ndim(a):
+    """
+    Return the number of dimensions of an array.
+
+    Parameters
+    ----------
+    a : array_like
+        Input array.  If it is not already an ndarray, a conversion is
+        attempted.
+
+    Returns
+    -------
+    number_of_dimensions : int
+        The number of dimensions in `a`.  Scalars are zero-dimensional.
+
+    See Also
+    --------
+    ndarray.ndim : equivalent method
+    shape : dimensions of array
+    ndarray.shape : dimensions of array
+
+    Examples
+    --------
+    >>> np.ndim([[1,2,3],[4,5,6]])
+    2
+    >>> np.ndim(np.array([[1,2,3],[4,5,6]]))
+    2
+    >>> np.ndim(1)
+    0
+
+    """
+    if not hasattr(a, 'ndim'):
+        a = numpypy.array(a)
+    return a.ndim
+
+
+def rank(a):
+    """
+    Return the number of dimensions of an array.
+
+    If `a` is not already an array, a conversion is attempted.
+    Scalars are zero dimensional.
+
+    Parameters
+    ----------
+    a : array_like
+        Array whose number of dimensions is desired. If `a` is not an array,
+        a conversion is attempted.
+
+    Returns
+    -------
+    number_of_dimensions : int
+        The number of dimensions in the array.
+
+    See Also
+    --------
+    ndim : equivalent function
+    ndarray.ndim : equivalent property
+    shape : dimensions of array
+    ndarray.shape : dimensions of array
+
+    Notes
+    -----
+    In the old Numeric package, `rank` was the term used for the number of
+    dimensions, but in Numpy `ndim` is used instead.
+
+    Examples
+    --------
+    >>> np.rank([1,2,3])
+    1
+    >>> np.rank(np.array([[1,2,3],[4,5,6]]))
+    2
+    >>> np.rank(1)
+    0
+
+    """
+    if not hasattr(a, 'ndim'):
+        a = numpypy.array(a)
+    return a.ndim
+
+
+def size(a, axis=None):
+    """
+    Return the number of elements along a given axis.
+
+    Parameters
+    ----------
+    a : array_like
+        Input data.
+    axis : int, optional
+        Axis along which the elements are counted.  By default, give
+        the total number of elements.
+
+    Returns
+    -------
+    element_count : int
+        Number of elements along the specified axis.
+
+    See Also
+    --------
+    shape : dimensions of array
+    ndarray.shape : dimensions of array
+    ndarray.size : number of elements in array
+
+    Examples
+    --------
+    >>> a = np.array([[1,2,3],[4,5,6]])
+    >>> np.size(a)
+    6
+    >>> np.size(a,1)
+    3
+    >>> np.size(a,0)
+    2
+
+    """
+    raise NotImplementedError('Waiting on interp level method')
+
+
+def around(a, decimals=0, out=None):
+    """
+    Evenly round to the given number of decimals.
+
+    Parameters
+    ----------
+    a : array_like
+        Input data.
+    decimals : int, optional
+        Number of decimal places to round to (default: 0).  If
+        decimals is negative, it specifies the number of positions to
+        the left of the decimal point.
+    out : ndarray, optional
+        Alternative output array in which to place the result. It must have
+        the same shape as the expected output, but the type of the output
+        values will be cast if necessary. See `doc.ufuncs` (Section
+        "Output arguments") for details.
+
+    Returns
+    -------
+    rounded_array : ndarray
+        An array of the same type as `a`, containing the rounded values.
+        Unless `out` was specified, a new array is created.  A reference to
+        the result is returned.
+
+        The real and imaginary parts of complex numbers are rounded
+        separately.  The result of rounding a float is a float.
+
+    See Also
+    --------
+    ndarray.round : equivalent method
+
+    ceil, fix, floor, rint, trunc
+
+
+    Notes
+    -----
+    For values exactly halfway between rounded decimal values, Numpy
+    rounds to the nearest even value. Thus 1.5 and 2.5 round to 2.0,
+    -0.5 and 0.5 round to 0.0, etc. Results may also be surprising due
+    to the inexact representation of decimal fractions in the IEEE
+    floating point standard [1]_ and errors introduced when scaling
+    by powers of ten.
+
+    References
+    ----------
+    .. [1] "Lecture Notes on the Status of  IEEE 754", William Kahan,
+           http://www.cs.berkeley.edu/~wkahan/ieee754status/IEEE754.PDF
+    .. [2] "How Futile are Mindless Assessments of
+           Roundoff in Floating-Point Computation?", William Kahan,
+           http://www.cs.berkeley.edu/~wkahan/Mindless.pdf
+
+    Examples
+    --------
+    >>> np.around([0.37, 1.64])
+    array([ 0.,  2.])
+    >>> np.around([0.37, 1.64], decimals=1)
+    array([ 0.4,  1.6])
+    >>> np.around([.5, 1.5, 2.5, 3.5, 4.5]) # rounds to nearest even value
+    array([ 0.,  2.,  2.,  4.,  4.])
+    >>> np.around([1,2,3,11], decimals=1) # ndarray of ints is returned
+    array([ 1,  2,  3, 11])
+    >>> np.around([1,2,3,11], decimals=-1)
+    array([ 0,  0,  0, 10])
+
+    """
+    raise NotImplementedError('Waiting on interp level method')
+
+
+def round_(a, decimals=0, out=None):
+    """
+    Round an array to the given number of decimals.
+
+    Refer to `around` for full documentation.
+
+    See Also
+    --------
+    around : equivalent function
+
+    """
+    raise NotImplementedError('Waiting on interp level method')
+
+
+def mean(a, axis=None, dtype=None, out=None):
+    """
+    Compute the arithmetic mean along the specified axis.
+
+    Returns the average of the array elements.  The average is taken over
+    the flattened array by default, otherwise over the specified axis.
+    `float64` intermediate and return values are used for integer inputs.
+
+    Parameters
+    ----------
+    a : array_like
+        Array containing numbers whose mean is desired. If `a` is not an
+        array, a conversion is attempted.
+    axis : int, optional
+        Axis along which the means are computed. The default is to compute
+        the mean of the flattened array.
+    dtype : data-type, optional
+        Type to use in computing the mean.  For integer inputs, the default
+        is `float64`; for floating point inputs, it is the same as the
+        input dtype.
+    out : ndarray, optional
+        Alternate output array in which to place the result.  The default
+        is ``None``; if provided, it must have the same shape as the
+        expected output, but the type will be cast if necessary.
+        See `doc.ufuncs` for details.
+
+    Returns
+    -------
+    m : ndarray, see dtype parameter above
+        If `out=None`, returns a new array containing the mean values,
+        otherwise a reference to the output array is returned.
+
+    See Also
+    --------
+    average : Weighted average
+
+    Notes
+    -----
+    The arithmetic mean is the sum of the elements along the axis divided
+    by the number of elements.
+
+    Note that for floating-point input, the mean is computed using the
+    same precision the input has.  Depending on the input data, this can
+    cause the results to be inaccurate, especially for `float32` (see
+    example below).  Specifying a higher-precision accumulator using the
+    `dtype` keyword can alleviate this issue.
+
+    Examples
+    --------
+    >>> a = np.array([[1, 2], [3, 4]])
+    >>> np.mean(a)
+    2.5
+    >>> np.mean(a, axis=0)
+    array([ 2.,  3.])
+    >>> np.mean(a, axis=1)
+    array([ 1.5,  3.5])
+
+    In single precision, `mean` can be inaccurate:
+
+    >>> a = np.zeros((2, 512*512), dtype=np.float32)
+    >>> a[0, :] = 1.0
+    >>> a[1, :] = 0.1
+    >>> np.mean(a)
+    0.546875
+
+    Computing the mean in float64 is more accurate:
+
+    >>> np.mean(a, dtype=np.float64)
+    0.55000000074505806
+
+    """
+    if not hasattr(a, "mean"):
+        a = numpypy.array(a)
+    return a.mean()
+
+
+def std(a, axis=None, dtype=None, out=None, ddof=0):
+    """
+    Compute the standard deviation along the specified axis.
+
+    Returns the standard deviation, a measure of the spread of a distribution,
+    of the array elements. The standard deviation is computed for the
+    flattened array by default, otherwise over the specified axis.
+
+    Parameters
+    ----------
+    a : array_like
+        Calculate the standard deviation of these values.
+    axis : int, optional
+        Axis along which the standard deviation is computed. The default is
+        to compute the standard deviation of the flattened array.
+    dtype : dtype, optional
+        Type to use in computing the standard deviation. For arrays of
+        integer type the default is float64, for arrays of float types it is
+        the same as the array type.
+    out : ndarray, optional
+        Alternative output array in which to place the result. It must have
+        the same shape as the expected output but the type (of the calculated
+        values) will be cast if necessary.
+    ddof : int, optional
+        Means Delta Degrees of Freedom.  The divisor used in calculations
+        is ``N - ddof``, where ``N`` represents the number of elements.
+        By default `ddof` is zero.
+
+    Returns
+    -------
+    standard_deviation : ndarray, see dtype parameter above.
+        If `out` is None, return a new array containing the standard deviation,
+        otherwise return a reference to the output array.
+
+    See Also
+    --------
+    var, mean
+    numpy.doc.ufuncs : Section "Output arguments"
+
+    Notes
+    -----
+    The standard deviation is the square root of the average of the squared
+    deviations from the mean, i.e., ``std = sqrt(mean(abs(x - x.mean())**2))``.
+
+    The average squared deviation is normally calculated as ``x.sum() / N``, where
+    ``N = len(x)``.  If, however, `ddof` is specified, the divisor ``N - ddof``
+    is used instead. In standard statistical practice, ``ddof=1`` provides an
+    unbiased estimator of the variance of the infinite population. ``ddof=0``
+    provides a maximum likelihood estimate of the variance for normally
+    distributed variables. The standard deviation computed in this function
+    is the square root of the estimated variance, so even with ``ddof=1``, it
+    will not be an unbiased estimate of the standard deviation per se.
+
+    Note that, for complex numbers, `std` takes the absolute
+    value before squaring, so that the result is always real and nonnegative.
+
+    For floating-point input, the *std* is computed using the same
+    precision the input has. Depending on the input data, this can cause
+    the results to be inaccurate, especially for float32 (see example below).
+    Specifying a higher-accuracy accumulator using the `dtype` keyword can
+    alleviate this issue.
+
+    Examples
+    --------
+    >>> a = np.array([[1, 2], [3, 4]])
+    >>> np.std(a)
+    1.1180339887498949
+    >>> np.std(a, axis=0)
+    array([ 1.,  1.])
+    >>> np.std(a, axis=1)
+    array([ 0.5,  0.5])
+
+    In single precision, std() can be inaccurate:
+
+    >>> a = np.zeros((2,512*512), dtype=np.float32)
+    >>> a[0,:] = 1.0
+    >>> a[1,:] = 0.1
+    >>> np.std(a)
+    0.45172946707416706
+
+    Computing the standard deviation in float64 is more accurate:
+
+    >>> np.std(a, dtype=np.float64)
+    0.44999999925552653
+
+    """
+    if not hasattr(a, "std"):
+        a = numpypy.array(a)
+    return a.std()
+
+
+def var(a, axis=None, dtype=None, out=None, ddof=0):
+    """
+    Compute the variance along the specified axis.
+
+    Returns the variance of the array elements, a measure of the spread of a
+    distribution.  The variance is computed for the flattened array by
+    default, otherwise over the specified axis.
+
+    Parameters
+    ----------
+    a : array_like
+        Array containing numbers whose variance is desired.  If `a` is not an
+        array, a conversion is attempted.
+    axis : int, optional
+        Axis along which the variance is computed.  The default is to compute
+        the variance of the flattened array.
+    dtype : data-type, optional
+        Type to use in computing the variance.  For arrays of integer type
+        the default is `float32`; for arrays of float types it is the same as
+        the array type.
+    out : ndarray, optional
+        Alternate output array in which to place the result.  It must have
+        the same shape as the expected output, but the type is cast if
+        necessary.
+    ddof : int, optional
+        "Delta Degrees of Freedom": the divisor used in the calculation is
+        ``N - ddof``, where ``N`` represents the number of elements. By
+        default `ddof` is zero.
+
+    Returns
+    -------
+    variance : ndarray, see dtype parameter above
+        If ``out=None``, returns a new array containing the variance;
+        otherwise, a reference to the output array is returned.
+
+    See Also
+    --------
+    std : Standard deviation
+    mean : Average
+    numpy.doc.ufuncs : Section "Output arguments"
+
+    Notes
+    -----
+    The variance is the average of the squared deviations from the mean,
+    i.e.,  ``var = mean(abs(x - x.mean())**2)``.
+
+    The mean is normally calculated as ``x.sum() / N``, where ``N = len(x)``.
+    If, however, `ddof` is specified, the divisor ``N - ddof`` is used
+    instead.  In standard statistical practice, ``ddof=1`` provides an
+    unbiased estimator of the variance of a hypothetical infinite population.
+    ``ddof=0`` provides a maximum likelihood estimate of the variance for
+    normally distributed variables.
+
+    Note that for complex numbers, the absolute value is taken before
+    squaring, so that the result is always real and nonnegative.
+
+    For floating-point input, the variance is computed using the same
+    precision the input has.  Depending on the input data, this can cause
+    the results to be inaccurate, especially for `float32` (see example
+    below).  Specifying a higher-accuracy accumulator using the ``dtype``
+    keyword can alleviate this issue.
+
+    Examples
+    --------
+    >>> a = np.array([[1,2],[3,4]])
+    >>> np.var(a)
+    1.25
+    >>> np.var(a,0)
+    array([ 1.,  1.])
+    >>> np.var(a,1)
+    array([ 0.25,  0.25])
+
+    In single precision, var() can be inaccurate:
+
+    >>> a = np.zeros((2,512*512), dtype=np.float32)
+    >>> a[0,:] = 1.0
+    >>> a[1,:] = 0.1
+    >>> np.var(a)
+    0.20405951142311096
+
+    Computing the standard deviation in float64 is more accurate:
+
+    >>> np.var(a, dtype=np.float64)
+    0.20249999932997387
+    >>> ((1-0.55)**2 + (0.1-0.55)**2)/2
+    0.20250000000000001
+
+    """
+    if not hasattr(a, "var"):
+        a = numpypy.array(a)
+    return a.var()
diff --git a/pypy/annotation/description.py b/pypy/annotation/description.py
--- a/pypy/annotation/description.py
+++ b/pypy/annotation/description.py
@@ -257,7 +257,8 @@
         try:
             inputcells = args.match_signature(signature, defs_s)
         except ArgErr, e:
-            raise TypeError, "signature mismatch: %s" % e.getmsg(self.name)
+            raise TypeError("signature mismatch: %s() %s" % 
+                            (self.name, e.getmsg()))
         return inputcells
 
     def specialize(self, inputcells, op=None):
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -340,7 +340,7 @@
                    requires=[("objspace.std.builtinshortcut", True)]),
         BoolOption("withidentitydict",
                    "track types that override __hash__, __eq__ or __cmp__ and use a special dict strategy for those which do not",
-                   default=True),
+                   default=False),
      ]),
 ])
 
@@ -370,6 +370,7 @@
         config.objspace.std.suggest(getattributeshortcut=True)
         config.objspace.std.suggest(newshortcut=True)
         config.objspace.std.suggest(withspecialisedtuple=True)
+        config.objspace.std.suggest(withidentitydict=True)
         #if not IS_64_BITS:
         #    config.objspace.std.suggest(withsmalllong=True)
 
diff --git a/pypy/doc/Makefile b/pypy/doc/Makefile
--- a/pypy/doc/Makefile
+++ b/pypy/doc/Makefile
@@ -12,7 +12,7 @@
 PAPEROPT_letter = -D latex_paper_size=letter
 ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
 
-.PHONY: help clean html dirhtml pickle json htmlhelp qthelp latex changes linkcheck doctest
+.PHONY: help clean html dirhtml pickle json htmlhelp qthelp latex man changes linkcheck doctest
 
 help:
 	@echo "Please use \`make ' where  is one of"
@@ -23,6 +23,7 @@
 	@echo "  htmlhelp  to make HTML files and a HTML help project"
 	@echo "  qthelp    to make HTML files and a qthelp project"
 	@echo "  latex     to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
+	@echo "  man       to make manual pages"
 	@echo "  changes   to make an overview of all changed/added/deprecated items"
 	@echo "  linkcheck to check all external links for integrity"
 	@echo "  doctest   to run all doctests embedded in the documentation (if enabled)"
@@ -79,6 +80,11 @@
 	@echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \
 	      "run these through (pdf)latex."
 
+man:
+	$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
+	@echo
+	@echo "Build finished. The manual pages are in $(BUILDDIR)/man"
+
 changes:
 	python config/generate.py
 	$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
diff --git a/pypy/doc/coding-guide.rst b/pypy/doc/coding-guide.rst
--- a/pypy/doc/coding-guide.rst
+++ b/pypy/doc/coding-guide.rst
@@ -175,15 +175,15 @@
 RPython
 =================
 
-RPython Definition, not
------------------------
+RPython Definition
+------------------
 
-The list and exact details of the "RPython" restrictions are a somewhat
-evolving topic.  In particular, we have no formal language definition
-as we find it more practical to discuss and evolve the set of
-restrictions while working on the whole program analysis.  If you
-have any questions about the restrictions below then please feel
-free to mail us at pypy-dev at codespeak net.
+RPython is a restricted subset of Python that is amenable to static analysis.
+Although there are additions to the language and some things might surprisingly
+work, this is a rough list of restrictions that should be considered. Note
+that there are tons of special cased restrictions that you'll encounter
+as you go. The exact definition is "RPython is everything that our translation
+toolchain can accept" :)
 
 .. _`wrapped object`: coding-guide.html#wrapping-rules
 
@@ -198,7 +198,7 @@
   contain both a string and a int must be avoided.  It is allowed to
   mix None (basically with the role of a null pointer) with many other
   types: `wrapped objects`, class instances, lists, dicts, strings, etc.
-  but *not* with int and floats.
+  but *not* with int, floats or tuples.
 
 **constants**
 
@@ -209,9 +209,12 @@
   have this restriction, so if you need mutable global state, store it
   in the attributes of some prebuilt singleton instance.
 
+
+
 **control structures**
 
-  all allowed but yield, ``for`` loops restricted to builtin types
+  all allowed, ``for`` loops restricted to builtin types, generators
+  very restricted.
 
 **range**
 
@@ -226,7 +229,8 @@
 
 **generators**
 
-  generators are not supported.
+  generators are supported, but their exact scope is very limited. you can't
+  merge two different generator in one control point.
 
 **exceptions**
 
@@ -245,22 +249,27 @@
 
 **strings**
 
-  a lot of, but not all string methods are supported.  Indexes can be
+  a lot of, but not all string methods are supported and those that are
+  supported, not necesarilly accept all arguments.  Indexes can be
   negative.  In case they are not, then you get slightly more efficient
   code if the translator can prove that they are non-negative.  When
   slicing a string it is necessary to prove that the slice start and
-  stop indexes are non-negative.
+  stop indexes are non-negative. There is no implicit str-to-unicode cast
+  anywhere.
 
 **tuples**
 
   no variable-length tuples; use them to store or return pairs or n-tuples of
-  values. Each combination of types for elements and length constitute a separate
-  and not mixable type.
+  values. Each combination of types for elements and length constitute
+  a separate and not mixable type.
 
 **lists**
 
   lists are used as an allocated array.  Lists are over-allocated, so list.append()
-  is reasonably fast.  Negative or out-of-bound indexes are only allowed for the
+  is reasonably fast. However, if you use a fixed-size list, the code
+  is more efficient. Annotator can figure out most of the time that your
+  list is fixed-size, even when you use list comprehension.
+  Negative or out-of-bound indexes are only allowed for the
   most common operations, as follows:
 
   - *indexing*:
@@ -287,16 +296,14 @@
 
 **dicts**
 
-  dicts with a unique key type only, provided it is hashable. 
-  String keys have been the only allowed key types for a while, but this was generalized. 
-  After some re-optimization,
-  the implementation could safely decide that all string dict keys should be interned.
+  dicts with a unique key type only, provided it is hashable. Custom
+  hash functions and custom equality will not be honored.
+  Use ``pypy.rlib.objectmodel.r_dict`` for custom hash functions.
 
 
 **list comprehensions**
 
-  may be used to create allocated, initialized arrays.
-  After list over-allocation was introduced, there is no longer any restriction.
+  May be used to create allocated, initialized arrays.
 
 **functions**
 
@@ -334,9 +341,8 @@
 
 **objects**
 
-  in PyPy, wrapped objects are borrowed from the object space. Just like
-  in CPython, code that needs e.g. a dictionary can use a wrapped dict
-  and the object space operations on it.
+  Normal rules apply. Special methods are not honoured, except ``__init__`` and
+  ``__del__``.
 
 This layout makes the number of types to take care about quite limited.
 
diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py
--- a/pypy/doc/conf.py
+++ b/pypy/doc/conf.py
@@ -197,3 +197,10 @@
 # Example configuration for intersphinx: refer to the Python standard library.
 intersphinx_mapping = {'http://docs.python.org/': None}
 
+# -- Options for manpage output-------------------------------------------------
+
+man_pages = [
+  ('man/pypy.1', 'pypy',
+   u'fast, compliant alternative implementation of the Python language',
+   u'The PyPy Project', 1)
+]
diff --git a/pypy/doc/extradoc.rst b/pypy/doc/extradoc.rst
--- a/pypy/doc/extradoc.rst
+++ b/pypy/doc/extradoc.rst
@@ -8,6 +8,9 @@
 *Articles about PyPy published so far, most recent first:* (bibtex_ file)
 
 
+* `Runtime Feedback in a Meta-Tracing JIT for Efficient Dynamic Languages`_,
+  C.F. Bolz, A. Cuni, M. Fijalkowski, M. Leuschel, S. Pedroni, A. Rigo
+
 * `Allocation Removal by Partial Evaluation in a Tracing JIT`_,
   C.F. Bolz, A. Cuni, M. Fijalkowski, M. Leuschel, S. Pedroni, A. Rigo
 
@@ -50,6 +53,9 @@
 
 *Other research using PyPy (as far as we know it):*
 
+* `Hardware Transactional Memory Support for Lightweight Dynamic Language Evolution`_,
+  N. Riley and C. Zilles
+
 * `PyGirl: Generating Whole-System VMs from High-Level Prototypes using PyPy`_,
   C. Bruni and T. Verwaest
 
@@ -65,6 +71,7 @@
 
 
 .. _bibtex: https://bitbucket.org/pypy/extradoc/raw/tip/talk/bibtex.bib
+.. _`Runtime Feedback in a Meta-Tracing JIT for Efficient Dynamic Languages`: https://bitbucket.org/pypy/extradoc/raw/extradoc/talk/icooolps2011/jit-hints.pdf
 .. _`Allocation Removal by Partial Evaluation in a Tracing JIT`: http://codespeak.net/svn/pypy/extradoc/talk/pepm2011/bolz-allocation-removal.pdf
 .. _`Towards a Jitting VM for Prolog Execution`: http://www.stups.uni-duesseldorf.de/publications/bolz-prolog-jit.pdf
 .. _`High performance implementation of Python for CLI/.NET with JIT compiler generation for dynamic languages`: http://buildbot.pypy.org/misc/antocuni-thesis.pdf
@@ -74,6 +81,7 @@
 .. _`Automatic JIT Compiler Generation with Runtime Partial Evaluation`:  http://www.stups.uni-duesseldorf.de/thesis/final-master.pdf
 .. _`RPython: A Step towards Reconciling Dynamically and Statically Typed OO Languages`: http://www.disi.unige.it/person/AnconaD/papers/Recent_abstracts.html#AACM-DLS07
 .. _`EU Reports`: index-report.html
+.. _`Hardware Transactional Memory Support for Lightweight Dynamic Language Evolution`: http://sabi.net/nriley/pubs/dls6-riley.pdf
 .. _`PyGirl: Generating Whole-System VMs from High-Level Prototypes using PyPy`: http://scg.unibe.ch/archive/papers/Brun09cPyGirl.pdf
 .. _`Representation-Based Just-in-Time Specialization and the Psyco Prototype for Python`: http://psyco.sourceforge.net/psyco-pepm-a.ps.gz
 .. _`Back to the Future in One Week -- Implementing a Smalltalk VM in PyPy`: http://dx.doi.org/10.1007/978-3-540-89275-5_7
diff --git a/pypy/doc/man/pypy.1.rst b/pypy/doc/man/pypy.1.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/man/pypy.1.rst
@@ -0,0 +1,90 @@
+======
+ pypy
+======
+
+SYNOPSIS
+========
+
+``pypy`` [*options*]
+[``-c`` *cmd*\ \|\ ``-m`` *mod*\ \|\ *file.py*\ \|\ ``-``\ ]
+[*arg*\ ...]
+
+OPTIONS
+=======
+
+-i
+    Inspect interactively after running script.
+
+-O
+    Dummy optimization flag for compatibility with C Python.
+
+-c *cmd*
+    Program passed in as CMD (terminates option list).
+
+-S
+    Do not ``import site`` on initialization.
+
+-u
+    Unbuffered binary ``stdout`` and ``stderr``.
+
+-h, --help
+    Show a help message and exit.
+
+-m *mod*
+    Library module to be run as a script (terminates option list).
+
+-W *arg*
+    Warning control (*arg* is *action*:*message*:*category*:*module*:*lineno*).
+
+-E
+    Ignore environment variables (such as ``PYTHONPATH``).
+
+--version
+    Print the PyPy version.
+
+--info
+    Print translation information about this PyPy executable.
+
+--jit *arg*
+    Low level JIT parameters. Format is
+    *arg*\ ``=``\ *value*\ [``,``\ *arg*\ ``=``\ *value*\ ...]
+
+    ``off``
+        Disable the JIT.
+
+    ``threshold=``\ *value*
+        Number of times a loop has to run for it to become hot.
+
+    ``function_threshold=``\ *value*
+        Number of times a function must run for it to become traced from
+        start.
+
+    ``inlining=``\ *value*
+        Inline python functions or not (``1``/``0``).
+
+    ``loop_longevity=``\ *value*
+        A parameter controlling how long loops will be kept before being
+        freed, an estimate.
+
+    ``max_retrace_guards=``\ *value*
+        Number of extra guards a retrace can cause.
+
+    ``retrace_limit=``\ *value*
+        How many times we can try retracing before giving up.
+
+    ``trace_eagerness=``\ *value*
+        Number of times a guard has to fail before we start compiling a
+        bridge.
+
+    ``trace_limit=``\ *value*
+        Number of recorded operations before we abort tracing with
+        ``ABORT_TRACE_TOO_LONG``.
+
+    ``enable_opts=``\ *value*
+        Optimizations to enabled or ``all``.
+        Warning, this option is dangerous, and should be avoided.
+
+SEE ALSO
+========
+
+**python**\ (1)
diff --git a/pypy/doc/tool/makecontributor.py b/pypy/doc/tool/makecontributor.py
deleted file mode 100644
--- a/pypy/doc/tool/makecontributor.py
+++ /dev/null
@@ -1,47 +0,0 @@
-"""
-
-generates a contributor list
-
-"""
-import py
-
-# this file is useless, use the following commandline instead:
-# hg churn -c -t "{author}" | sed -e 's/ <.*//'
-
-try: 
-    path = py.std.sys.argv[1]
-except IndexError: 
-    print "usage: %s ROOTPATH" %(py.std.sys.argv[0])
-    raise SystemExit, 1
-
-d = {}
-
-for logentry in py.path.svnwc(path).log(): 
-    a = logentry.author 
-    if a in d: 
-        d[a] += 1
-    else: 
-        d[a] = 1
-
-items = d.items()
-items.sort(lambda x,y: -cmp(x[1], y[1]))
-
-import uconf # http://codespeak.net/svn/uconf/dist/uconf 
-
-# Authors that don't want to be listed
-excluded = set("anna gintas ignas".split())
-cutoff = 5 # cutoff for authors in the LICENSE file
-mark = False
-for author, count in items: 
-    if author in excluded:
-        continue
-    user = uconf.system.User(author)
-    try:
-        realname = user.realname.strip()
-    except KeyError:
-        realname = author
-    if not mark and count < cutoff:
-        mark = True
-        print '-'*60
-    print "   ", realname
-    #print count, "   ", author 
diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -428,8 +428,8 @@
             return self._match_signature(w_firstarg,
                                          scope_w, signature, defaults_w, 0)
         except ArgErr, e:
-            raise OperationError(self.space.w_TypeError,
-                                 self.space.wrap(e.getmsg(fnname)))
+            raise operationerrfmt(self.space.w_TypeError,
+                                  "%s() %s", fnname, e.getmsg())
 
     def _parse(self, w_firstarg, signature, defaults_w, blindargs=0):
         """Parse args and kwargs according to the signature of a code object,
@@ -450,8 +450,8 @@
         try:
             return self._parse(w_firstarg, signature, defaults_w, blindargs)
         except ArgErr, e:
-            raise OperationError(self.space.w_TypeError,
-                                 self.space.wrap(e.getmsg(fnname)))
+            raise operationerrfmt(self.space.w_TypeError,
+                                  "%s() %s", fnname, e.getmsg())
 
     @staticmethod
     def frompacked(space, w_args=None, w_kwds=None):
@@ -626,7 +626,7 @@
 
 class ArgErr(Exception):
 
-    def getmsg(self, fnname):
+    def getmsg(self):
         raise NotImplementedError
 
 class ArgErrCount(ArgErr):
@@ -642,11 +642,10 @@
         self.num_args = got_nargs
         self.num_kwds = nkwds
 
-    def getmsg(self, fnname):
+    def getmsg(self):
         n = self.expected_nargs
         if n == 0:
-            msg = "%s() takes no arguments (%d given)" % (
-                fnname,
+            msg = "takes no arguments (%d given)" % (
                 self.num_args + self.num_kwds)
         else:
             defcount = self.num_defaults
@@ -672,8 +671,7 @@
                 msg2 = " non-keyword"
             else:
                 msg2 = ""
-            msg = "%s() takes %s %d%s argument%s (%d given)" % (
-                fnname,
+            msg = "takes %s %d%s argument%s (%d given)" % (
                 msg1,
                 n,
                 msg2,
@@ -686,9 +684,8 @@
     def __init__(self, argname):
         self.argname = argname
 
-    def getmsg(self, fnname):
-        msg = "%s() got multiple values for keyword argument '%s'" % (
-            fnname,
+    def getmsg(self):
+        msg = "got multiple values for keyword argument '%s'" % (
             self.argname)
         return msg
 
@@ -722,13 +719,11 @@
                     break
         self.kwd_name = name
 
-    def getmsg(self, fnname):
+    def getmsg(self):
         if self.num_kwds == 1:
-            msg = "%s() got an unexpected keyword argument '%s'" % (
-                fnname,
+            msg = "got an unexpected keyword argument '%s'" % (
                 self.kwd_name)
         else:
-            msg = "%s() got %d unexpected keyword arguments" % (
-                fnname,
+            msg = "got %d unexpected keyword arguments" % (
                 self.num_kwds)
         return msg
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1591,12 +1591,15 @@
     'ArithmeticError',
     'AssertionError',
     'AttributeError',
+    'BaseException',
+    'DeprecationWarning',
     'EOFError',
     'EnvironmentError',
     'Exception',
     'FloatingPointError',
     'IOError',
     'ImportError',
+    'ImportWarning',
     'IndentationError',
     'IndexError',
     'KeyError',
@@ -1617,7 +1620,10 @@
     'TabError',
     'TypeError',
     'UnboundLocalError',
+    'UnicodeDecodeError',
     'UnicodeError',
+    'UnicodeEncodeError',
+    'UnicodeTranslateError',
     'ValueError',
     'ZeroDivisionError',
     'UnicodeEncodeError',
diff --git a/pypy/interpreter/eval.py b/pypy/interpreter/eval.py
--- a/pypy/interpreter/eval.py
+++ b/pypy/interpreter/eval.py
@@ -2,7 +2,6 @@
 This module defines the abstract base classes that support execution:
 Code and Frame.
 """
-from pypy.rlib import jit
 from pypy.interpreter.error import OperationError
 from pypy.interpreter.baseobjspace import Wrappable
 
diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -162,7 +162,8 @@
     # generate 2 versions of the function and 2 jit drivers.
     def _create_unpack_into():
         jitdriver = jit.JitDriver(greens=['pycode'],
-                                  reds=['self', 'frame', 'results'])
+                                  reds=['self', 'frame', 'results'],
+                                  name='unpack_into')
         def unpack_into(self, results):
             """This is a hack for performance: runs the generator and collects
             all produced items in a list."""
@@ -196,4 +197,4 @@
                 self.frame = None
         return unpack_into
     unpack_into = _create_unpack_into()
-    unpack_into_w = _create_unpack_into()
\ No newline at end of file
+    unpack_into_w = _create_unpack_into()
diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py
--- a/pypy/interpreter/test/test_argument.py
+++ b/pypy/interpreter/test/test_argument.py
@@ -393,8 +393,8 @@
 
         class FakeArgErr(ArgErr):
 
-            def getmsg(self, fname):
-                return "msg "+fname
+            def getmsg(self):
+                return "msg"
 
         def _match_signature(*args):
             raise FakeArgErr()
@@ -404,7 +404,7 @@
         excinfo = py.test.raises(OperationError, args.parse_obj, "obj", "foo",
                        Signature(["a", "b"], None, None))
         assert excinfo.value.w_type is TypeError
-        assert excinfo.value._w_value == "msg foo"
+        assert excinfo.value.get_w_value(space) == "foo() msg"
 
 
     def test_args_parsing_into_scope(self):
@@ -448,8 +448,8 @@
 
         class FakeArgErr(ArgErr):
 
-            def getmsg(self, fname):
-                return "msg "+fname
+            def getmsg(self):
+                return "msg"
 
         def _match_signature(*args):
             raise FakeArgErr()
@@ -460,7 +460,7 @@
                                  "obj", [None, None], "foo",
                                  Signature(["a", "b"], None, None))
         assert excinfo.value.w_type is TypeError
-        assert excinfo.value._w_value == "msg foo"
+        assert excinfo.value.get_w_value(space) == "foo() msg"
 
     def test_topacked_frompacked(self):
         space = DummySpace()
@@ -493,35 +493,35 @@
         # got_nargs, nkwds, expected_nargs, has_vararg, has_kwarg,
         # defaults_w, missing_args
         err = ArgErrCount(1, 0, 0, False, False, None, 0)
-        s = err.getmsg('foo')
-        assert s == "foo() takes no arguments (1 given)"
+        s = err.getmsg()
+        assert s == "takes no arguments (1 given)"
         err = ArgErrCount(0, 0, 1, False, False, [], 1)
-        s = err.getmsg('foo')
-        assert s == "foo() takes exactly 1 argument (0 given)"
+        s = err.getmsg()
+        assert s == "takes exactly 1 argument (0 given)"
         err = ArgErrCount(3, 0, 2, False, False, [], 0)
-        s = err.getmsg('foo')
-        assert s == "foo() takes exactly 2 arguments (3 given)"
+        s = err.getmsg()
+        assert s == "takes exactly 2 arguments (3 given)"
         err = ArgErrCount(3, 0, 2, False, False, ['a'], 0)
-        s = err.getmsg('foo')
-        assert s == "foo() takes at most 2 arguments (3 given)"
+        s = err.getmsg()
+        assert s == "takes at most 2 arguments (3 given)"
         err = ArgErrCount(1, 0, 2, True, False, [], 1)
-        s = err.getmsg('foo')
-        assert s == "foo() takes at least 2 arguments (1 given)"
+        s = err.getmsg()
+        assert s == "takes at least 2 arguments (1 given)"
         err = ArgErrCount(0, 1, 2, True, False, ['a'], 1)
-        s = err.getmsg('foo')
-        assert s == "foo() takes at least 1 non-keyword argument (0 given)"
+        s = err.getmsg()
+        assert s == "takes at least 1 non-keyword argument (0 given)"
         err = ArgErrCount(2, 1, 1, False, True, [], 0)
-        s = err.getmsg('foo')
-        assert s == "foo() takes exactly 1 non-keyword argument (2 given)"
+        s = err.getmsg()
+        assert s == "takes exactly 1 non-keyword argument (2 given)"
         err = ArgErrCount(0, 1, 1, False, True, [], 1)
-        s = err.getmsg('foo')
-        assert s == "foo() takes exactly 1 non-keyword argument (0 given)"
+        s = err.getmsg()
+        assert s == "takes exactly 1 non-keyword argument (0 given)"
         err = ArgErrCount(0, 1, 1, True, True, [], 1)
-        s = err.getmsg('foo')
-        assert s == "foo() takes at least 1 non-keyword argument (0 given)"
+        s = err.getmsg()
+        assert s == "takes at least 1 non-keyword argument (0 given)"
         err = ArgErrCount(2, 1, 1, False, True, ['a'], 0)
-        s = err.getmsg('foo')
-        assert s == "foo() takes at most 1 non-keyword argument (2 given)"
+        s = err.getmsg()
+        assert s == "takes at most 1 non-keyword argument (2 given)"
 
     def test_bad_type_for_star(self):
         space = self.space
@@ -543,12 +543,12 @@
     def test_unknown_keywords(self):
         space = DummySpace()
         err = ArgErrUnknownKwds(space, 1, ['a', 'b'], [True, False], None)
-        s = err.getmsg('foo')
-        assert s == "foo() got an unexpected keyword argument 'b'"
+        s = err.getmsg()
+        assert s == "got an unexpected keyword argument 'b'"
         err = ArgErrUnknownKwds(space, 2, ['a', 'b', 'c'],
                                 [True, False, False], None)
-        s = err.getmsg('foo')
-        assert s == "foo() got 2 unexpected keyword arguments"
+        s = err.getmsg()
+        assert s == "got 2 unexpected keyword arguments"
 
     def test_unknown_unicode_keyword(self):
         class DummySpaceUnicode(DummySpace):
@@ -558,13 +558,13 @@
         err = ArgErrUnknownKwds(space, 1, ['a', None, 'b', 'c'],
                                 [True, False, True, True],
                                 [unichr(0x1234), u'b', u'c'])
-        s = err.getmsg('foo')
-        assert s == "foo() got an unexpected keyword argument '\xe1\x88\xb4'"
+        s = err.getmsg()
+        assert s == "got an unexpected keyword argument '\xe1\x88\xb4'"
 
     def test_multiple_values(self):
         err = ArgErrMultipleValues('bla')
-        s = err.getmsg('foo')
-        assert s == "foo() got multiple values for keyword argument 'bla'"
+        s = err.getmsg()
+        assert s == "got multiple values for keyword argument 'bla'"
 
 class AppTestArgument:
     def test_error_message(self):
diff --git a/pypy/jit/backend/llsupport/test/test_runner.py b/pypy/jit/backend/llsupport/test/test_runner.py
--- a/pypy/jit/backend/llsupport/test/test_runner.py
+++ b/pypy/jit/backend/llsupport/test/test_runner.py
@@ -8,6 +8,12 @@
 
 class MyLLCPU(AbstractLLCPU):
     supports_floats = True
+
+    class assembler(object):
+        @staticmethod
+        def set_debug(flag):
+            pass
+    
     def compile_loop(self, inputargs, operations, looptoken):
         py.test.skip("llsupport test: cannot compile operations")
 
diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py
--- a/pypy/jit/backend/test/runner_test.py
+++ b/pypy/jit/backend/test/runner_test.py
@@ -17,6 +17,7 @@
 from pypy.rpython.llinterp import LLException
 from pypy.jit.codewriter import heaptracker, longlong
 from pypy.rlib.rarithmetic import intmask
+from pypy.jit.backend.detect_cpu import autodetect_main_model_and_size
 
 def boxfloat(x):
     return BoxFloat(longlong.getfloatstorage(x))
@@ -27,6 +28,9 @@
 
 class Runner(object):
 
+    add_loop_instruction = ['overload for a specific cpu']
+    bridge_loop_instruction = ['overload for a specific cpu']
+
     def execute_operation(self, opname, valueboxes, result_type, descr=None):
         inputargs, operations = self._get_single_operation_list(opname,
                                                                 result_type,
@@ -3184,6 +3188,56 @@
         res = self.cpu.get_latest_value_int(0)
         assert res == -10
 
+    def test_compile_asmlen(self):
+        from pypy.jit.backend.llsupport.llmodel import AbstractLLCPU
+        if not isinstance(self.cpu, AbstractLLCPU):
+            py.test.skip("pointless test on non-asm")
+        from pypy.jit.backend.x86.tool.viewcode import machine_code_dump
+        import ctypes
+        ops = """
+        [i2]
+        i0 = same_as(i2)    # but forced to be in a register
+        label(i0, descr=1)
+        i1 = int_add(i0, i0)
+        guard_true(i1, descr=faildesr) [i1]
+        jump(i1, descr=1)
+        """
+        faildescr = BasicFailDescr(2)
+        loop = parse(ops, self.cpu, namespace=locals())
+        faildescr = loop.operations[-2].getdescr()
+        jumpdescr = loop.operations[-1].getdescr()
+        bridge_ops = """
+        [i0]
+        jump(i0, descr=jumpdescr)
+        """
+        bridge = parse(bridge_ops, self.cpu, namespace=locals())
+        looptoken = JitCellToken()
+        self.cpu.assembler.set_debug(False)
+        info = self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken)
+        bridge_info = self.cpu.compile_bridge(faildescr, bridge.inputargs,
+                                              bridge.operations,
+                                              looptoken)
+        self.cpu.assembler.set_debug(True) # always on untranslated
+        assert info.asmlen != 0
+        cpuname = autodetect_main_model_and_size()
+        # XXX we have to check the precise assembler, otherwise
+        # we don't quite know if borders are correct
+
+        def checkops(mc, ops):
+            assert len(mc) == len(ops)
+            for i in range(len(mc)):
+                assert mc[i].split("\t")[-1].startswith(ops[i])
+            
+        data = ctypes.string_at(info.asmaddr, info.asmlen)
+        mc = list(machine_code_dump(data, info.asmaddr, cpuname))
+        lines = [line for line in mc if line.count('\t') == 2]
+        checkops(lines, self.add_loop_instructions)
+        data = ctypes.string_at(bridge_info.asmaddr, bridge_info.asmlen)
+        mc = list(machine_code_dump(data, bridge_info.asmaddr, cpuname))
+        lines = [line for line in mc if line.count('\t') == 2]
+        checkops(lines, self.bridge_loop_instructions)
+
+
     def test_compile_bridge_with_target(self):
         # This test creates a loopy piece of code in a bridge, and builds another
         # unrelated loop that ends in a jump directly to this loopy bit of code.
diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -7,6 +7,7 @@
 from pypy.rpython.lltypesystem import lltype, rffi, rstr, llmemory
 from pypy.rpython.lltypesystem.lloperation import llop
 from pypy.rpython.annlowlevel import llhelper
+from pypy.rlib.jit import AsmInfo
 from pypy.jit.backend.model import CompiledLoopToken
 from pypy.jit.backend.x86.regalloc import (RegAlloc, get_ebp_ofs, _get_scale,
     gpr_reg_mgr_cls, _valid_addressing_size)
@@ -411,6 +412,7 @@
         '''adds the following attributes to looptoken:
                _x86_function_addr   (address of the generated func, as an int)
                _x86_loop_code       (debug: addr of the start of the ResOps)
+               _x86_fullsize        (debug: full size including failure)
                _x86_debug_checksum
         '''
         # XXX this function is too longish and contains some code
@@ -476,7 +478,8 @@
             name = "Loop # %s: %s" % (looptoken.number, loopname)
             self.cpu.profile_agent.native_code_written(name,
                                                        rawstart, full_size)
-        return ops_offset
+        return AsmInfo(ops_offset, rawstart + looppos,
+                       size_excluding_failure_stuff - looppos)
 
     def assemble_bridge(self, faildescr, inputargs, operations,
                         original_loop_token, log):
@@ -485,12 +488,7 @@
             assert len(set(inputargs)) == len(inputargs)
 
         descr_number = self.cpu.get_fail_descr_number(faildescr)
-        try:
-            failure_recovery = self._find_failure_recovery_bytecode(faildescr)
-        except ValueError:
-            debug_print("Bridge out of guard", descr_number,
-                        "was already compiled!")
-            return
+        failure_recovery = self._find_failure_recovery_bytecode(faildescr)
 
         self.setup(original_loop_token)
         if log:
@@ -503,6 +501,7 @@
                     [loc.assembler() for loc in faildescr._x86_debug_faillocs])
         regalloc = RegAlloc(self, self.cpu.translate_support_code)
         fail_depths = faildescr._x86_current_depths
+        startpos = self.mc.get_relative_pos()
         operations = regalloc.prepare_bridge(fail_depths, inputargs, arglocs,
                                              operations,
                                              self.current_clt.allgcrefs)
@@ -537,7 +536,7 @@
             name = "Bridge # %s" % (descr_number,)
             self.cpu.profile_agent.native_code_written(name,
                                                        rawstart, fullsize)
-        return ops_offset
+        return AsmInfo(ops_offset, startpos + rawstart, codeendpos - startpos)
 
     def write_pending_failure_recoveries(self):
         # for each pending guard, generate the code of the recovery stub
@@ -621,7 +620,10 @@
     def _find_failure_recovery_bytecode(self, faildescr):
         adr_jump_offset = faildescr._x86_adr_jump_offset
         if adr_jump_offset == 0:
-            raise ValueError
+            # This case should be prevented by the logic in compile.py:
+            # look for CNT_BUSY_FLAG, which disables tracing from a guard
+            # when another tracing from the same guard is already in progress.
+            raise BridgeAlreadyCompiled
         # follow the JMP/Jcond
         p = rffi.cast(rffi.INTP, adr_jump_offset)
         adr_target = adr_jump_offset + 4 + rffi.cast(lltype.Signed, p[0])
@@ -810,7 +812,10 @@
         target = newlooptoken._x86_function_addr
         mc = codebuf.MachineCodeBlockWrapper()
         mc.JMP(imm(target))
-        assert mc.get_relative_pos() <= 13  # keep in sync with prepare_loop()
+        if WORD == 4:         # keep in sync with prepare_loop()
+            assert mc.get_relative_pos() == 5
+        else:
+            assert mc.get_relative_pos() <= 13
         mc.copy_to_raw_memory(oldadr)
 
     def dump(self, text):
@@ -2550,3 +2555,6 @@
 def not_implemented(msg):
     os.write(2, '[x86/asm] %s\n' % msg)
     raise NotImplementedError(msg)
+
+class BridgeAlreadyCompiled(Exception):
+    pass
diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py
--- a/pypy/jit/backend/x86/regalloc.py
+++ b/pypy/jit/backend/x86/regalloc.py
@@ -194,7 +194,10 @@
         # note: we need to make a copy of inputargs because possibly_free_vars
         # is also used on op args, which is a non-resizable list
         self.possibly_free_vars(list(inputargs))
-        self.min_bytes_before_label = 13
+        if WORD == 4:       # see redirect_call_assembler()
+            self.min_bytes_before_label = 5
+        else:
+            self.min_bytes_before_label = 13
         return operations
 
     def prepare_bridge(self, prev_depths, inputargs, arglocs, operations,
@@ -683,7 +686,7 @@
         self.xrm.possibly_free_var(op.getarg(0))
 
     def consider_cast_int_to_float(self, op):
-        loc0 = self.rm.loc(op.getarg(0))
+        loc0 = self.rm.make_sure_var_in_reg(op.getarg(0))
         loc1 = self.xrm.force_allocate_reg(op.result)
         self.Perform(op, [loc0], loc1)
         self.rm.possibly_free_var(op.getarg(0))
diff --git a/pypy/jit/backend/x86/test/test_runner.py b/pypy/jit/backend/x86/test/test_runner.py
--- a/pypy/jit/backend/x86/test/test_runner.py
+++ b/pypy/jit/backend/x86/test/test_runner.py
@@ -33,6 +33,13 @@
     # for the individual tests see
     # ====> ../../test/runner_test.py
 
+    add_loop_instructions = ['mov', 'add', 'test', 'je', 'jmp']
+    if WORD == 4:
+        bridge_loop_instructions = ['lea', 'jmp']
+    else:
+        # the 'mov' is part of the 'jmp' so far
+        bridge_loop_instructions = ['lea', 'mov', 'jmp']
+
     def setup_method(self, meth):
         self.cpu = CPU(rtyper=None, stats=FakeStats())
         self.cpu.setup_once()
@@ -416,12 +423,13 @@
             ]
         inputargs = [i0]
         debug._log = dlog = debug.DebugLog()
-        ops_offset = self.cpu.compile_loop(inputargs, operations, looptoken)
+        info = self.cpu.compile_loop(inputargs, operations, looptoken)
+        ops_offset = info.ops_offset
         debug._log = None
         #
         assert ops_offset is looptoken._x86_ops_offset
-        # getfield_raw/int_add/setfield_raw + ops + None
-        assert len(ops_offset) == 3 + len(operations) + 1
+        # 2*(getfield_raw/int_add/setfield_raw) + ops + None
+        assert len(ops_offset) == 2*3 + len(operations) + 1
         assert (ops_offset[operations[0]] <=
                 ops_offset[operations[1]] <=
                 ops_offset[operations[2]] <=
diff --git a/pypy/jit/backend/x86/tool/viewcode.py b/pypy/jit/backend/x86/tool/viewcode.py
--- a/pypy/jit/backend/x86/tool/viewcode.py
+++ b/pypy/jit/backend/x86/tool/viewcode.py
@@ -39,6 +39,7 @@
 def machine_code_dump(data, originaddr, backend_name, label_list=None):
     objdump_backend_option = {
         'x86': 'i386',
+        'x86_32': 'i386',
         'x86_64': 'x86-64',
         'i386': 'i386',
     }
diff --git a/pypy/jit/codewriter/policy.py b/pypy/jit/codewriter/policy.py
--- a/pypy/jit/codewriter/policy.py
+++ b/pypy/jit/codewriter/policy.py
@@ -8,11 +8,15 @@
 
 
 class JitPolicy(object):
-    def __init__(self):
+    def __init__(self, jithookiface=None):
         self.unsafe_loopy_graphs = set()
         self.supports_floats = False
         self.supports_longlong = False
         self.supports_singlefloats = False
+        if jithookiface is None:
+            from pypy.rlib.jit import JitHookInterface
+            jithookiface = JitHookInterface()
+        self.jithookiface = jithookiface
 
     def set_supports_floats(self, flag):
         self.supports_floats = flag
diff --git a/pypy/jit/metainterp/compile.py b/pypy/jit/metainterp/compile.py
--- a/pypy/jit/metainterp/compile.py
+++ b/pypy/jit/metainterp/compile.py
@@ -5,6 +5,7 @@
 from pypy.rlib.objectmodel import we_are_translated
 from pypy.rlib.debug import debug_start, debug_stop, debug_print
 from pypy.rlib import rstack
+from pypy.rlib.jit import JitDebugInfo
 from pypy.conftest import option
 from pypy.tool.sourcetools import func_with_new_name
 
@@ -75,7 +76,7 @@
             if descr is not original_jitcell_token:
                 original_jitcell_token.record_jump_to(descr)
             descr.exported_state = None
-            op._descr = None    # clear reference, mostly for tests
+            op.cleardescr()    # clear reference, mostly for tests
         elif isinstance(descr, TargetToken):
             # for a JUMP: record it as a potential jump.
             # (the following test is not enough to prevent more complicated
@@ -90,8 +91,8 @@
             assert descr.exported_state is None 
             if not we_are_translated():
                 op._descr_wref = weakref.ref(op._descr)
-            op._descr = None    # clear reference to prevent the history.Stats
-                                # from keeping the loop alive during tests
+            op.cleardescr()    # clear reference to prevent the history.Stats
+                               # from keeping the loop alive during tests
     # record this looptoken on the QuasiImmut used in the code
     if loop.quasi_immutable_deps is not None:
         for qmut in loop.quasi_immutable_deps:
@@ -296,8 +297,6 @@
         patch_new_loop_to_load_virtualizable_fields(loop, jitdriver_sd)
 
     original_jitcell_token = loop.original_jitcell_token
-    jitdriver_sd.on_compile(metainterp_sd.logger_ops, original_jitcell_token,
-                            loop.operations, type, greenkey)
     loopname = jitdriver_sd.warmstate.get_location_str(greenkey)
     globaldata = metainterp_sd.globaldata
     original_jitcell_token.number = n = globaldata.loopnumbering
@@ -307,21 +306,38 @@
         show_procedures(metainterp_sd, loop)
         loop.check_consistency()
 
+    if metainterp_sd.warmrunnerdesc is not None:
+        hooks = metainterp_sd.warmrunnerdesc.hooks
+        debug_info = JitDebugInfo(jitdriver_sd, metainterp_sd.logger_ops,
+                                  original_jitcell_token, loop.operations,
+                                  type, greenkey)
+        hooks.before_compile(debug_info)
+    else:
+        debug_info = None
+        hooks = None
     operations = get_deep_immutable_oplist(loop.operations)
     metainterp_sd.profiler.start_backend()
     debug_start("jit-backend")
     try:
-        ops_offset = metainterp_sd.cpu.compile_loop(loop.inputargs, operations,
-                                                    original_jitcell_token, name=loopname)
+        asminfo = metainterp_sd.cpu.compile_loop(loop.inputargs, operations,
+                                                  original_jitcell_token,
+                                                  name=loopname)
     finally:
         debug_stop("jit-backend")
     metainterp_sd.profiler.end_backend()
+    if hooks is not None:
+        debug_info.asminfo = asminfo
+        hooks.after_compile(debug_info)
     metainterp_sd.stats.add_new_loop(loop)
     if not we_are_translated():
         metainterp_sd.stats.compiled()
     metainterp_sd.log("compiled new " + type)
     #
     loopname = jitdriver_sd.warmstate.get_location_str(greenkey)
+    if asminfo is not None:
+        ops_offset = asminfo.ops_offset
+    else:
+        ops_offset = None
     metainterp_sd.logger_ops.log_loop(loop.inputargs, loop.operations, n,
                                       type, ops_offset,
                                       name=loopname)
@@ -332,25 +348,40 @@
 def send_bridge_to_backend(jitdriver_sd, metainterp_sd, faildescr, inputargs,
                            operations, original_loop_token):
     n = metainterp_sd.cpu.get_fail_descr_number(faildescr)
-    jitdriver_sd.on_compile_bridge(metainterp_sd.logger_ops,
-                                   original_loop_token, operations, n)
     if not we_are_translated():
         show_procedures(metainterp_sd)
         seen = dict.fromkeys(inputargs)
         TreeLoop.check_consistency_of_branch(operations, seen)
+    if metainterp_sd.warmrunnerdesc is not None:
+        hooks = metainterp_sd.warmrunnerdesc.hooks
+        debug_info = JitDebugInfo(jitdriver_sd, metainterp_sd.logger_ops,
+                                  original_loop_token, operations, 'bridge',
+                                  fail_descr_no=n)
+        hooks.before_compile_bridge(debug_info)
+    else:
+        hooks = None
+        debug_info = None
+    operations = get_deep_immutable_oplist(operations)
     metainterp_sd.profiler.start_backend()
-    operations = get_deep_immutable_oplist(operations)
     debug_start("jit-backend")
     try:
-        ops_offset = metainterp_sd.cpu.compile_bridge(faildescr, inputargs, operations,
-                                                      original_loop_token)
+        asminfo = metainterp_sd.cpu.compile_bridge(faildescr, inputargs,
+                                                   operations,
+                                                   original_loop_token)
     finally:
         debug_stop("jit-backend")
     metainterp_sd.profiler.end_backend()
+    if hooks is not None:
+        debug_info.asminfo = asminfo
+        hooks.after_compile_bridge(debug_info)
     if not we_are_translated():
         metainterp_sd.stats.compiled()
     metainterp_sd.log("compiled new bridge")
     #
+    if asminfo is not None:
+        ops_offset = asminfo.ops_offset
+    else:
+        ops_offset = None
     metainterp_sd.logger_ops.log_bridge(inputargs, operations, n, ops_offset)
     #
     #if metainterp_sd.warmrunnerdesc is not None:    # for tests
diff --git a/pypy/jit/metainterp/history.py b/pypy/jit/metainterp/history.py
--- a/pypy/jit/metainterp/history.py
+++ b/pypy/jit/metainterp/history.py
@@ -1003,16 +1003,16 @@
         return insns
 
     def check_simple_loop(self, expected=None, **check):
-        # Usefull in the simplest case when we have only one trace ending with
-        # a jump back to itself and possibly a few bridges ending with finnish.
-        # Only the operations within the loop formed by that single jump will
-        # be counted.
+        """ Usefull in the simplest case when we have only one trace ending with
+        a jump back to itself and possibly a few bridges.
+        Only the operations within the loop formed by that single jump will
+        be counted.
+        """
         loops = self.get_all_loops()
         assert len(loops) == 1
         loop = loops[0]
         jumpop = loop.operations[-1]
         assert jumpop.getopnum() == rop.JUMP
-        assert self.check_resops(jump=1)
         labels = [op for op in loop.operations if op.getopnum() == rop.LABEL]
         targets = [op._descr_wref() for op in labels]
         assert None not in targets # TargetToken was freed, give up
diff --git a/pypy/jit/metainterp/jitdriver.py b/pypy/jit/metainterp/jitdriver.py
--- a/pypy/jit/metainterp/jitdriver.py
+++ b/pypy/jit/metainterp/jitdriver.py
@@ -21,7 +21,6 @@
     #    self.portal_finishtoken... pypy.jit.metainterp.pyjitpl
     #    self.index             ... pypy.jit.codewriter.call
     #    self.mainjitcode       ... pypy.jit.codewriter.call
-    #    self.on_compile        ... pypy.jit.metainterp.warmstate
 
     # These attributes are read by the backend in CALL_ASSEMBLER:
     #    self.assembler_helper_adr
diff --git a/pypy/jit/metainterp/jitprof.py b/pypy/jit/metainterp/jitprof.py
--- a/pypy/jit/metainterp/jitprof.py
+++ b/pypy/jit/metainterp/jitprof.py
@@ -18,8 +18,8 @@
 OPT_FORCINGS
 ABORT_TOO_LONG
 ABORT_BRIDGE
+ABORT_BAD_LOOP
 ABORT_ESCAPE
-ABORT_BAD_LOOP
 ABORT_FORCE_QUASIIMMUT
 NVIRTUALS
 NVHOLES
@@ -30,10 +30,13 @@
 TOTAL_FREED_BRIDGES
 """
 
+counter_names = []
+
 def _setup():
     names = counters.split()
     for i, name in enumerate(names):
         globals()[name] = i
+        counter_names.append(name)
     global ncounters
     ncounters = len(names)
 _setup()
diff --git a/pypy/jit/metainterp/optimizeopt/fficall.py b/pypy/jit/metainterp/optimizeopt/fficall.py
--- a/pypy/jit/metainterp/optimizeopt/fficall.py
+++ b/pypy/jit/metainterp/optimizeopt/fficall.py
@@ -234,11 +234,11 @@
             # longlongs are treated as floats, see
             # e.g. llsupport/descr.py:getDescrClass
             is_float = True
-        elif kind == 'u':
+        elif kind == 'u' or kind == 's':
             # they're all False
             pass
         else:
-            assert False, "unsupported ffitype or kind"
+            raise NotImplementedError("unsupported ffitype or kind: %s" % kind)
         #
         fieldsize = rffi.getintfield(ffitype, 'c_size')
         return self.optimizer.cpu.interiorfielddescrof_dynamic(
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_multilabel.py b/pypy/jit/metainterp/optimizeopt/test/test_multilabel.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_multilabel.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_multilabel.py
@@ -442,6 +442,22 @@
         """
         self.optimize_loop(ops, expected)
 
+    def test_optimizer_renaming_boxes_not_imported(self):
+        ops = """
+        [p1]
+        i1 = strlen(p1)
+        label(p1)
+        jump(p1)
+        """
+        expected = """
+        [p1]
+        i1 = strlen(p1)
+        label(p1, i1)
+        i11 = same_as(i1)
+        jump(p1, i11)
+        """
+        self.optimize_loop(ops, expected)
+
         
 
 class TestLLtype(OptimizeoptTestMultiLabel, LLtypeMixin):
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -117,7 +117,7 @@
 
     def optimize_loop(self, ops, optops, call_pure_results=None):
         loop = self.parse(ops)
-        token = JitCellToken() 
+        token = JitCellToken()
         loop.operations = [ResOperation(rop.LABEL, loop.inputargs, None, descr=TargetToken(token))] + \
                           loop.operations
         if loop.operations[-1].getopnum() == rop.JUMP:
diff --git a/pypy/jit/metainterp/optimizeopt/unroll.py b/pypy/jit/metainterp/optimizeopt/unroll.py
--- a/pypy/jit/metainterp/optimizeopt/unroll.py
+++ b/pypy/jit/metainterp/optimizeopt/unroll.py
@@ -271,6 +271,10 @@
                 if newresult is not op.result and not newvalue.is_constant():
                     op = ResOperation(rop.SAME_AS, [op.result], newresult)
                     self.optimizer._newoperations.append(op)
+                    if self.optimizer.loop.logops:
+                        debug_print('  Falling back to add extra: ' +
+                                    self.optimizer.loop.logops.repr_of_resop(op))
+                    
         self.optimizer.flush()
         self.optimizer.emitting_dissabled = False
 
@@ -435,7 +439,13 @@
             return
         for a in op.getarglist():
             if not isinstance(a, Const) and a not in seen:
-                self.ensure_short_op_emitted(self.short_boxes.producer(a), optimizer, seen)
+                self.ensure_short_op_emitted(self.short_boxes.producer(a), optimizer,
+                                             seen)
+
+        if self.optimizer.loop.logops:
+            debug_print('  Emitting short op: ' +
+                        self.optimizer.loop.logops.repr_of_resop(op))
+
         optimizer.send_extra_operation(op)
         seen[op.result] = True
         if op.is_ovf():
diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -1553,6 +1553,7 @@
 
 class MetaInterp(object):
     in_recursion = 0
+    cancel_count = 0
 
     def __init__(self, staticdata, jitdriver_sd):
         self.staticdata = staticdata
@@ -1793,6 +1794,15 @@
     def aborted_tracing(self, reason):
         self.staticdata.profiler.count(reason)
         debug_print('~~~ ABORTING TRACING')
+        jd_sd = self.jitdriver_sd
+        if not self.current_merge_points:
+            greenkey = None # we're in the bridge
+        else:
+            greenkey = self.current_merge_points[0][0][:jd_sd.num_green_args]
+            self.staticdata.warmrunnerdesc.hooks.on_abort(reason,
+                                                          jd_sd.jitdriver,
+                                                          greenkey,
+                                                          jd_sd.warmstate.get_location_str(greenkey))
         self.staticdata.stats.aborted()
 
     def blackhole_if_trace_too_long(self):
@@ -1966,9 +1976,14 @@
                         raise SwitchToBlackhole(ABORT_BAD_LOOP) # For now
                 self.compile_loop(original_boxes, live_arg_boxes, start, resumedescr)
                 # creation of the loop was cancelled!
+                self.cancel_count += 1
+                if self.staticdata.warmrunnerdesc:
+                    memmgr = self.staticdata.warmrunnerdesc.memory_manager
+                    if memmgr:
+                        if self.cancel_count > memmgr.max_unroll_loops:
+                            self.staticdata.log('cancelled too many times!')
+                            raise SwitchToBlackhole(ABORT_BAD_LOOP)
                 self.staticdata.log('cancelled, tracing more...')
-                #self.staticdata.log('cancelled, stopping tracing')
-                #raise SwitchToBlackhole(ABORT_BAD_LOOP)
 
         # Otherwise, no loop found so far, so continue tracing.
         start = len(self.history.operations)
diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py
--- a/pypy/jit/metainterp/resoperation.py
+++ b/pypy/jit/metainterp/resoperation.py
@@ -16,15 +16,15 @@
     # debug
     name = ""
     pc = 0
+    opnum = 0
+
+    _attrs_ = ('result',)
 
     def __init__(self, result):
         self.result = result
 
-    # methods implemented by each concrete class
-    # ------------------------------------------
-
     def getopnum(self):
-        raise NotImplementedError
+        return self.opnum
 
     # methods implemented by the arity mixins
     # ---------------------------------------
@@ -64,6 +64,9 @@
     def setdescr(self, descr):
         raise NotImplementedError
 
+    def cleardescr(self):
+        pass
+
     # common methods
     # --------------
 
@@ -196,6 +199,9 @@
         self._check_descr(descr)
         self._descr = descr
 
+    def cleardescr(self):
+        self._descr = None
+
     def _check_descr(self, descr):
         if not we_are_translated() and getattr(descr, 'I_am_a_descr', False):
             return # needed for the mock case in oparser_model
@@ -590,12 +596,9 @@
         baseclass = PlainResOp
     mixin = arity2mixin.get(arity, N_aryOp)
 
-    def getopnum(self):
-        return opnum
-
     cls_name = '%s_OP' % name
     bases = (get_base_class(mixin, baseclass),)
-    dic = {'getopnum': getopnum}
+    dic = {'opnum': opnum}
     return type(cls_name, bases, dic)
 
 setup(__name__ == '__main__')   # print out the table when run directly
diff --git a/pypy/jit/metainterp/test/support.py b/pypy/jit/metainterp/test/support.py
--- a/pypy/jit/metainterp/test/support.py
+++ b/pypy/jit/metainterp/test/support.py
@@ -56,8 +56,6 @@
         greenfield_info = None
         result_type = result_kind
         portal_runner_ptr = "???"
-        on_compile = lambda *args: None
-        on_compile_bridge = lambda *args: None
 
     stats = history.Stats()
     cpu = CPUClass(rtyper, stats, None, False)
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -2629,6 +2629,38 @@
         self.check_jitcell_token_count(1)
         self.check_target_token_count(5)
 
+    def test_max_unroll_loops(self):
+        from pypy.jit.metainterp.optimize import InvalidLoop
+        from pypy.jit.metainterp import optimizeopt
+        myjitdriver = JitDriver(greens = [], reds = ['n', 'i'])
+        #
+        def f(n, limit):
+            set_param(myjitdriver, 'threshold', 5)
+            set_param(myjitdriver, 'max_unroll_loops', limit)
+            i = 0
+            while i < n:
+                myjitdriver.jit_merge_point(n=n, i=i)
+                print i
+                i += 1
+            return i
+        #
+        def my_optimize_trace(*args, **kwds):
+            raise InvalidLoop
+        old_optimize_trace = optimizeopt.optimize_trace
+        optimizeopt.optimize_trace = my_optimize_trace
+        try:
+            res = self.meta_interp(f, [23, 4])
+            assert res == 23
+            self.check_trace_count(0)
+            self.check_aborted_count(3)
+            #
+            res = self.meta_interp(f, [23, 20])
+            assert res == 23
+            self.check_trace_count(0)
+            self.check_aborted_count(2)
+        finally:
+            optimizeopt.optimize_trace = old_optimize_trace
+
     def test_retrace_limit_with_extra_guards(self):
         myjitdriver = JitDriver(greens = [], reds = ['n', 'i', 'sa', 'a',
                                                      'node'])
diff --git a/pypy/jit/metainterp/test/test_compile.py b/pypy/jit/metainterp/test/test_compile.py
--- a/pypy/jit/metainterp/test/test_compile.py
+++ b/pypy/jit/metainterp/test/test_compile.py
@@ -53,8 +53,6 @@
     call_pure_results = {}
     class jitdriver_sd:
         warmstate = FakeState()
-        on_compile = staticmethod(lambda *args: None)
-        on_compile_bridge = staticmethod(lambda *args: None)
         virtualizable_info = None
 
 def test_compile_loop():
diff --git a/pypy/jit/metainterp/test/test_fficall.py b/pypy/jit/metainterp/test/test_fficall.py
--- a/pypy/jit/metainterp/test/test_fficall.py
+++ b/pypy/jit/metainterp/test/test_fficall.py
@@ -148,28 +148,38 @@
         self.check_resops({'jump': 1, 'int_lt': 2, 'setinteriorfield_raw': 4,
                            'getinteriorfield_raw': 8, 'int_add': 6, 'guard_true': 2})
 
-    def test_array_getitem_uint8(self):
+    def _test_getitem_type(self, TYPE, ffitype, COMPUTE_TYPE):
+        reds = ["n", "i", "s", "data"]
+        if COMPUTE_TYPE is lltype.Float:
+            # Move the float var to the back.
+            reds.remove("s")
+            reds.append("s")
         myjitdriver = JitDriver(
             greens = [],
-            reds = ["n", "i", "s", "data"],
+            reds = reds,
         )
         def f(data, n):
-            i = s = 0
+            i = 0
+            s = rffi.cast(COMPUTE_TYPE, 0)
             while i < n:
                 myjitdriver.jit_merge_point(n=n, i=i, s=s, data=data)
-                s += rffi.cast(lltype.Signed, array_getitem(types.uchar, 1, data, 0, 0))
+                s += rffi.cast(COMPUTE_TYPE, array_getitem(ffitype, rffi.sizeof(TYPE), data, 0, 0))
                 i += 1
             return s
+        def main(n):
+            with lltype.scoped_alloc(rffi.CArray(TYPE), 1) as data:
+                data[0] = rffi.cast(TYPE, 200)
+                return f(data, n)
+        assert self.meta_interp(main, [10]) == 2000
 
-        def main(n):
-            with lltype.scoped_alloc(rffi.CArray(rffi.UCHAR), 1) as data:
-                data[0] = rffi.cast(rffi.UCHAR, 200)
-                return f(data, n)
-
-        assert self.meta_interp(main, [10]) == 2000
+    def test_array_getitem_uint8(self):
+        self._test_getitem_type(rffi.UCHAR, types.uchar, lltype.Signed)
         self.check_resops({'jump': 1, 'int_lt': 2, 'getinteriorfield_raw': 2,
                            'guard_true': 2, 'int_add': 4})
 
+    def test_array_getitem_float(self):
+        self._test_getitem_type(rffi.FLOAT, types.float, lltype.Float)
+
 
 class TestFfiCall(FfiCallTests, LLJitMixin):
     supports_all = False
diff --git a/pypy/jit/metainterp/test/test_jitdriver.py b/pypy/jit/metainterp/test/test_jitdriver.py
--- a/pypy/jit/metainterp/test/test_jitdriver.py
+++ b/pypy/jit/metainterp/test/test_jitdriver.py
@@ -10,57 +10,6 @@
 def getloc2(g):
     return "in jitdriver2, with g=%d" % g
 
-class JitDriverTests(object):
-    def test_on_compile(self):
-        called = {}
-        
-        class MyJitDriver(JitDriver):
-            def on_compile(self, logger, looptoken, operations, type, n, m):
-                called[(m, n, type)] = looptoken
-
-        driver = MyJitDriver(greens = ['n', 'm'], reds = ['i'])
-
-        def loop(n, m):
-            i = 0
-            while i < n + m:
-                driver.can_enter_jit(n=n, m=m, i=i)
-                driver.jit_merge_point(n=n, m=m, i=i)
-                i += 1
-
-        self.meta_interp(loop, [1, 4])
-        assert sorted(called.keys()) == [(4, 1, "loop")]
-        self.meta_interp(loop, [2, 4])
-        assert sorted(called.keys()) == [(4, 1, "loop"),
-                                         (4, 2, "loop")]
-
-    def test_on_compile_bridge(self):
-        called = {}
-        
-        class MyJitDriver(JitDriver):
-            def on_compile(self, logger, looptoken, operations, type, n, m):
-                called[(m, n, type)] = loop
-            def on_compile_bridge(self, logger, orig_token, operations, n):
-                assert 'bridge' not in called
-                called['bridge'] = orig_token
-
-        driver = MyJitDriver(greens = ['n', 'm'], reds = ['i'])
-
-        def loop(n, m):
-            i = 0
-            while i < n + m:
-                driver.can_enter_jit(n=n, m=m, i=i)
-                driver.jit_merge_point(n=n, m=m, i=i)
-                if i >= 4:
-                    i += 2
-                i += 1
-
-        self.meta_interp(loop, [1, 10])
-        assert sorted(called.keys()) == ['bridge', (10, 1, "loop")]
-
-
-class TestLLtypeSingle(JitDriverTests, LLJitMixin):
-    pass
-
 class MultipleJitDriversTests(object):
 
     def test_simple(self):
diff --git a/pypy/jit/metainterp/test/test_jitiface.py b/pypy/jit/metainterp/test/test_jitiface.py
new file mode 100644
--- /dev/null
+++ b/pypy/jit/metainterp/test/test_jitiface.py
@@ -0,0 +1,148 @@
+
+from pypy.rlib.jit import JitDriver, JitHookInterface
+from pypy.rlib import jit_hooks
+from pypy.jit.metainterp.test.support import LLJitMixin
+from pypy.jit.codewriter.policy import JitPolicy
+from pypy.jit.metainterp.jitprof import ABORT_FORCE_QUASIIMMUT
+from pypy.jit.metainterp.resoperation import rop
+from pypy.rpython.annlowlevel import hlstr
+
+class TestJitHookInterface(LLJitMixin):
+    def test_abort_quasi_immut(self):
+        reasons = []
+        
+        class MyJitIface(JitHookInterface):
+            def on_abort(self, reason, jitdriver, greenkey, greenkey_repr):
+                assert jitdriver is myjitdriver
+                assert len(greenkey) == 1
+                reasons.append(reason)
+                assert greenkey_repr == 'blah'
+
+        iface = MyJitIface()
+
+        myjitdriver = JitDriver(greens=['foo'], reds=['x', 'total'],
+                                get_printable_location=lambda *args: 'blah')
+
+        class Foo:
+            _immutable_fields_ = ['a?']
+            def __init__(self, a):
+                self.a = a
+        def f(a, x):
+            foo = Foo(a)
+            total = 0
+            while x > 0:
+                myjitdriver.jit_merge_point(foo=foo, x=x, total=total)
+                # read a quasi-immutable field out of a Constant
+                total += foo.a
+                foo.a += 1
+                x -= 1
+            return total
+        #
+        assert f(100, 7) == 721
+        res = self.meta_interp(f, [100, 7], policy=JitPolicy(iface))
+        assert res == 721
+        assert reasons == [ABORT_FORCE_QUASIIMMUT] * 2
+
+    def test_on_compile(self):
+        called = []
+        
+        class MyJitIface(JitHookInterface):
+            def after_compile(self, di):
+                called.append(("compile", di.greenkey[1].getint(),
+                               di.greenkey[0].getint(), di.type))
+
+            def before_compile(self, di):
+                called.append(("optimize", di.greenkey[1].getint(),
+                               di.greenkey[0].getint(), di.type))
+
+            #def before_optimize(self, jitdriver, logger, looptoken, oeprations,
+            #                   type, greenkey):
+            #    called.append(("trace", greenkey[1].getint(),
+            #                   greenkey[0].getint(), type))
+
+        iface = MyJitIface()
+
+        driver = JitDriver(greens = ['n', 'm'], reds = ['i'])
+
+        def loop(n, m):
+            i = 0
+            while i < n + m:
+                driver.can_enter_jit(n=n, m=m, i=i)
+                driver.jit_merge_point(n=n, m=m, i=i)
+                i += 1
+
+        self.meta_interp(loop, [1, 4], policy=JitPolicy(iface))
+        assert called == [#("trace", 4, 1, "loop"),
+                          ("optimize", 4, 1, "loop"),
+                          ("compile", 4, 1, "loop")]
+        self.meta_interp(loop, [2, 4], policy=JitPolicy(iface))
+        assert called == [#("trace", 4, 1, "loop"),
+                          ("optimize", 4, 1, "loop"),
+                          ("compile", 4, 1, "loop"),
+                          #("trace", 4, 2, "loop"),
+                          ("optimize", 4, 2, "loop"),
+                          ("compile", 4, 2, "loop")]
+
+    def test_on_compile_bridge(self):
+        called = []
+        
+        class MyJitIface(JitHookInterface):
+            def after_compile(self, di):
+                called.append("compile")
+
+            def after_compile_bridge(self, di):
+                called.append("compile_bridge")
+
+            def before_compile_bridge(self, di):
+                called.append("before_compile_bridge")
+            
+        driver = JitDriver(greens = ['n', 'm'], reds = ['i'])
+
+        def loop(n, m):
+            i = 0
+            while i < n + m:
+                driver.can_enter_jit(n=n, m=m, i=i)
+                driver.jit_merge_point(n=n, m=m, i=i)
+                if i >= 4:
+                    i += 2
+                i += 1
+
+        self.meta_interp(loop, [1, 10], policy=JitPolicy(MyJitIface()))
+        assert called == ["compile", "before_compile_bridge", "compile_bridge"]
+
+    def test_resop_interface(self):
+        driver = JitDriver(greens = [], reds = ['i'])
+
+        def loop(i):
+            while i > 0:
+                driver.jit_merge_point(i=i)
+                i -= 1
+
+        def main():
+            loop(1)
+            op = jit_hooks.resop_new(rop.INT_ADD,
+                                     [jit_hooks.boxint_new(3),
+                                      jit_hooks.boxint_new(4)],
+                                     jit_hooks.boxint_new(1))
+            assert hlstr(jit_hooks.resop_getopname(op)) == 'int_add'
+            assert jit_hooks.resop_getopnum(op) == rop.INT_ADD
+            box = jit_hooks.resop_getarg(op, 0)
+            assert jit_hooks.box_getint(box) == 3
+            box2 = jit_hooks.box_clone(box)
+            assert box2 != box
+            assert jit_hooks.box_getint(box2) == 3
+            assert not jit_hooks.box_isconst(box2)
+            box3 = jit_hooks.box_constbox(box)
+            assert jit_hooks.box_getint(box) == 3
+            assert jit_hooks.box_isconst(box3)
+            box4 = jit_hooks.box_nonconstbox(box)
+            assert not jit_hooks.box_isconst(box4)
+            box5 = jit_hooks.boxint_new(18)
+            jit_hooks.resop_setarg(op, 0, box5)
+            assert jit_hooks.resop_getarg(op, 0) == box5
+            box6 = jit_hooks.resop_getresult(op)
+            assert jit_hooks.box_getint(box6) == 1
+            jit_hooks.resop_setresult(op, box5)
+            assert jit_hooks.resop_getresult(op) == box5
+
+        self.meta_interp(main, [])
diff --git a/pypy/jit/metainterp/test/test_resoperation.py b/pypy/jit/metainterp/test/test_resoperation.py
--- a/pypy/jit/metainterp/test/test_resoperation.py
+++ b/pypy/jit/metainterp/test/test_resoperation.py
@@ -30,17 +30,17 @@
     cls = rop.opclasses[rop.rop.INT_ADD]
     assert issubclass(cls, rop.PlainResOp)
     assert issubclass(cls, rop.BinaryOp)
-    assert cls.getopnum.im_func(None) == rop.rop.INT_ADD
+    assert cls.getopnum.im_func(cls) == rop.rop.INT_ADD
 
     cls = rop.opclasses[rop.rop.CALL]
     assert issubclass(cls, rop.ResOpWithDescr)
     assert issubclass(cls, rop.N_aryOp)
-    assert cls.getopnum.im_func(None) == rop.rop.CALL
+    assert cls.getopnum.im_func(cls) == rop.rop.CALL
 
     cls = rop.opclasses[rop.rop.GUARD_TRUE]
     assert issubclass(cls, rop.GuardResOp)
     assert issubclass(cls, rop.UnaryOp)
-    assert cls.getopnum.im_func(None) == rop.rop.GUARD_TRUE
+    assert cls.getopnum.im_func(cls) == rop.rop.GUARD_TRUE
 
 def test_mixins_in_common_base():
     INT_ADD = rop.opclasses[rop.rop.INT_ADD]
diff --git a/pypy/jit/metainterp/test/test_virtualstate.py b/pypy/jit/metainterp/test/test_virtualstate.py
--- a/pypy/jit/metainterp/test/test_virtualstate.py
+++ b/pypy/jit/metainterp/test/test_virtualstate.py
@@ -5,7 +5,7 @@
      VArrayStateInfo, NotVirtualStateInfo, VirtualState, ShortBoxes
 from pypy.jit.metainterp.optimizeopt.optimizer import OptValue
 from pypy.jit.metainterp.history import BoxInt, BoxFloat, BoxPtr, ConstInt, ConstPtr
-from pypy.rpython.lltypesystem import lltype
+from pypy.rpython.lltypesystem import lltype, llmemory
 from pypy.jit.metainterp.optimizeopt.test.test_util import LLtypeMixin, BaseTest, \
                                                            equaloplists, FakeDescrWithSnapshot
 from pypy.jit.metainterp.optimizeopt.intutils import IntBound
@@ -82,6 +82,13 @@
         assert isgeneral(value1, value2)
         assert not isgeneral(value2, value1)
 
+        assert isgeneral(OptValue(ConstInt(7)), OptValue(ConstInt(7)))
+        S = lltype.GcStruct('S')
+        foo = lltype.malloc(S)
+        fooref = lltype.cast_opaque_ptr(llmemory.GCREF, foo)
+        assert isgeneral(OptValue(ConstPtr(fooref)),
+                         OptValue(ConstPtr(fooref)))
+
     def test_field_matching_generalization(self):
         const1 = NotVirtualStateInfo(OptValue(ConstInt(1)))
         const2 = NotVirtualStateInfo(OptValue(ConstInt(2)))
diff --git a/pypy/jit/metainterp/test/test_ztranslation.py b/pypy/jit/metainterp/test/test_ztranslation.py
--- a/pypy/jit/metainterp/test/test_ztranslation.py
+++ b/pypy/jit/metainterp/test/test_ztranslation.py
@@ -3,7 +3,9 @@
 from pypy.jit.backend.llgraph import runner
 from pypy.rlib.jit import JitDriver, unroll_parameters, set_param
 from pypy.rlib.jit import PARAMETERS, dont_look_inside, hint
+from pypy.rlib.jit_hooks import boxint_new, resop_new, resop_getopnum
 from pypy.jit.metainterp.jitprof import Profiler
+from pypy.jit.metainterp.resoperation import rop
 from pypy.rpython.lltypesystem import lltype, llmemory
 
 class TranslationTest:
@@ -22,6 +24,7 @@
         # - jitdriver hooks
         # - two JITs
         # - string concatenation, slicing and comparison
+        # - jit hooks interface
 
         class Frame(object):
             _virtualizable2_ = ['l[*]']
@@ -91,7 +94,9 @@
             return f.i
         #
         def main(i, j):
-            return f(i) - f2(i+j, i, j)
+            op = resop_new(rop.INT_ADD, [boxint_new(3), boxint_new(5)],
+                           boxint_new(8))
+            return f(i) - f2(i+j, i, j) + resop_getopnum(op)
         res = ll_meta_interp(main, [40, 5], CPUClass=self.CPUClass,
                              type_system=self.type_system,
                              listops=True)
diff --git a/pypy/jit/metainterp/warmspot.py b/pypy/jit/metainterp/warmspot.py
--- a/pypy/jit/metainterp/warmspot.py
+++ b/pypy/jit/metainterp/warmspot.py
@@ -1,4 +1,5 @@
 import sys, py
+from pypy.tool.sourcetools import func_with_new_name
 from pypy.rpython.lltypesystem import lltype, llmemory
 from pypy.rpython.annlowlevel import llhelper, MixLevelHelperAnnotator,\
      cast_base_ptr_to_instance, hlstr
@@ -112,7 +113,7 @@
     return ll_meta_interp(function, args, backendopt=backendopt,
                           translate_support_code=True, **kwds)
 
-def _find_jit_marker(graphs, marker_name):
+def _find_jit_marker(graphs, marker_name, check_driver=True):
     results = []
     for graph in graphs:
         for block in graph.iterblocks():
@@ -120,8 +121,8 @@
                 op = block.operations[i]
                 if (op.opname == 'jit_marker' and
                     op.args[0].value == marker_name and
-                    (op.args[1].value is None or
-                    op.args[1].value.active)):   # the jitdriver
+                    (not check_driver or op.args[1].value is None or
+                     op.args[1].value.active)):   # the jitdriver
                     results.append((graph, block, i))
     return results
 
@@ -140,6 +141,9 @@
         "found several jit_merge_points in the same graph")
     return results
 
+def find_access_helpers(graphs):
+    return _find_jit_marker(graphs, 'access_helper', False)
+
 def locate_jit_merge_point(graph):
     [(graph, block, pos)] = find_jit_merge_points([graph])
     return block, pos, block.operations[pos]
@@ -206,6 +210,7 @@
         vrefinfo = VirtualRefInfo(self)
         self.codewriter.setup_vrefinfo(vrefinfo)
         #
+        self.hooks = policy.jithookiface
         self.make_virtualizable_infos()
         self.make_exception_classes()
         self.make_driverhook_graphs()
@@ -213,6 +218,7 @@
         self.rewrite_jit_merge_points(policy)
 
         verbose = False # not self.cpu.translate_support_code
+        self.rewrite_access_helpers()
         self.codewriter.make_jitcodes(verbose=verbose)
         self.rewrite_can_enter_jits()
         self.rewrite_set_param()
@@ -619,6 +625,24 @@
         graph = self.annhelper.getgraph(func, args_s, s_result)
         return self.annhelper.graph2delayed(graph, FUNC)
 
+    def rewrite_access_helpers(self):
+        ah = find_access_helpers(self.translator.graphs)
+        for graph, block, index in ah:
+            op = block.operations[index]
+            self.rewrite_access_helper(op)
+
+    def rewrite_access_helper(self, op):
+        ARGS = [arg.concretetype for arg in op.args[2:]]
+        RESULT = op.result.concretetype
+        FUNCPTR = lltype.Ptr(lltype.FuncType(ARGS, RESULT))
+        # make sure we make a copy of function so it no longer belongs
+        # to extregistry
+        func = op.args[1].value
+        func = func_with_new_name(func, func.func_name + '_compiled')
+        ptr = self.helper_func(FUNCPTR, func)
+        op.opname = 'direct_call'
+        op.args = [Constant(ptr, FUNCPTR)] + op.args[2:]
+
     def rewrite_jit_merge_points(self, policy):
         for jd in self.jitdrivers_sd:
             self.rewrite_jit_merge_point(jd, policy)
diff --git a/pypy/jit/metainterp/warmstate.py b/pypy/jit/metainterp/warmstate.py
--- a/pypy/jit/metainterp/warmstate.py
+++ b/pypy/jit/metainterp/warmstate.py
@@ -244,6 +244,11 @@
             if self.warmrunnerdesc.memory_manager:
                 self.warmrunnerdesc.memory_manager.max_retrace_guards = value
 
+    def set_param_max_unroll_loops(self, value):
+        if self.warmrunnerdesc:
+            if self.warmrunnerdesc.memory_manager:
+                self.warmrunnerdesc.memory_manager.max_unroll_loops = value
+
     def disable_noninlinable_function(self, greenkey):
         cell = self.jit_cell_at_key(greenkey)
         cell.dont_trace_here = True
@@ -596,20 +601,6 @@
                 return fn(*greenargs)
         self.should_unroll_one_iteration = should_unroll_one_iteration
         
-        if hasattr(jd.jitdriver, 'on_compile'):
-            def on_compile(logger, token, operations, type, greenkey):
-                greenargs = unwrap_greenkey(greenkey)
-                return jd.jitdriver.on_compile(logger, token, operations, type,
-                                               *greenargs)
-            def on_compile_bridge(logger, orig_token, operations, n):
-                return jd.jitdriver.on_compile_bridge(logger, orig_token,
-                                                      operations, n)
-            jd.on_compile = on_compile
-            jd.on_compile_bridge = on_compile_bridge
-        else:
-            jd.on_compile = lambda *args: None
-            jd.on_compile_bridge = lambda *args: None
-
         redargtypes = ''.join([kind[0] for kind in jd.red_args_types])
 
         def get_assembler_token(greenkey):
diff --git a/pypy/jit/tool/oparser.py b/pypy/jit/tool/oparser.py
--- a/pypy/jit/tool/oparser.py
+++ b/pypy/jit/tool/oparser.py
@@ -89,11 +89,18 @@
                 assert typ == 'class'
                 return self.model.ConstObj(ootype.cast_to_object(obj))
 
-    def get_descr(self, poss_descr):
+    def get_descr(self, poss_descr, allow_invent):
         if poss_descr.startswith('<'):
             return None
-        else:
+        try:
             return self._consts[poss_descr]
+        except KeyError:
+            if allow_invent:
+                int(poss_descr)
+                token = self.model.JitCellToken()
+                tt = self.model.TargetToken(token)
+                self._consts[poss_descr] = tt
+                return tt
 
     def box_for_var(self, elem):
         try:
@@ -186,7 +193,8 @@
 
             poss_descr = allargs[-1].strip()
             if poss_descr.startswith('descr='):
-                descr = self.get_descr(poss_descr[len('descr='):])
+                descr = self.get_descr(poss_descr[len('descr='):],
+                                       opname == 'label')
                 allargs = allargs[:-1]
             for arg in allargs:
                 arg = arg.strip()
diff --git a/pypy/jit/tool/oparser_model.py b/pypy/jit/tool/oparser_model.py
--- a/pypy/jit/tool/oparser_model.py
+++ b/pypy/jit/tool/oparser_model.py
@@ -6,7 +6,7 @@
         from pypy.jit.metainterp.history import TreeLoop, JitCellToken
         from pypy.jit.metainterp.history import Box, BoxInt, BoxFloat
         from pypy.jit.metainterp.history import ConstInt, ConstObj, ConstPtr, ConstFloat
-        from pypy.jit.metainterp.history import BasicFailDescr
+        from pypy.jit.metainterp.history import BasicFailDescr, TargetToken
         from pypy.jit.metainterp.typesystem import llhelper
 
         from pypy.jit.metainterp.history import get_const_ptr_for_string
@@ -42,6 +42,10 @@
         class JitCellToken(object):
             I_am_a_descr = True
 
+        class TargetToken(object):
+            def __init__(self, jct):
+                pass
+
         class BasicFailDescr(object):
             I_am_a_descr = True
 
diff --git a/pypy/jit/tool/pypytrace.vim b/pypy/jit/tool/pypytrace.vim
--- a/pypy/jit/tool/pypytrace.vim
+++ b/pypy/jit/tool/pypytrace.vim
@@ -19,6 +19,7 @@
 syn match pypyLoopArgs    '^[[].*'
 syn match pypyLoopStart   '^#.*'
 syn match pypyDebugMergePoint  '^debug_merge_point(.\+)'
+syn match pypyLogBoundary '[[][0-9a-f]\+[]] \([{].\+\|.\+[}]\)$'
 
 hi def link pypyLoopStart   Structure
 "hi def link pypyLoopArgs    PreProc
@@ -29,3 +30,4 @@
 hi def link pypyNumber      Number
 hi def link pypyDescr       PreProc
 hi def link pypyDescrField  Label
+hi def link pypyLogBoundary Statement
diff --git a/pypy/jit/tool/test/test_oparser.py b/pypy/jit/tool/test/test_oparser.py
--- a/pypy/jit/tool/test/test_oparser.py
+++ b/pypy/jit/tool/test/test_oparser.py
@@ -4,7 +4,8 @@
 
 from pypy.jit.tool.oparser import parse, OpParser
 from pypy.jit.metainterp.resoperation import rop
-from pypy.jit.metainterp.history import AbstractDescr, BoxInt, JitCellToken
+from pypy.jit.metainterp.history import AbstractDescr, BoxInt, JitCellToken,\
+     TargetToken
 
 class BaseTestOparser(object):
 
@@ -243,6 +244,16 @@
         b = loop.getboxes()
         assert isinstance(b.sum0, BoxInt)
 
+    def test_label(self):
+        x = """
+        [i0]
+        label(i0, descr=1)
+        jump(i0, descr=1)
+        """
+        loop = self.parse(x)
+        assert loop.operations[0].getdescr() is loop.operations[1].getdescr()
+        assert isinstance(loop.operations[0].getdescr(), TargetToken)
+
 
 class ForbiddenModule(object):
     def __init__(self, name, old_mod):
diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -108,6 +108,10 @@
     w_result = state.codec_search_cache.get(normalized_encoding, None)
     if w_result is not None:
         return w_result
+    return _lookup_codec_loop(space, encoding, normalized_encoding)
+
+def _lookup_codec_loop(space, encoding, normalized_encoding):
+    state = space.fromcache(CodecState)
     if state.codec_need_encodings:
         w_import = space.getattr(space.builtin, space.wrap("__import__"))
         # registers new codecs
diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py
--- a/pypy/module/_codecs/test/test_codecs.py
+++ b/pypy/module/_codecs/test/test_codecs.py
@@ -588,10 +588,18 @@
         raises(UnicodeDecodeError, '+3ADYAA-'.decode, 'utf-7')
 
     def test_utf_16_encode_decode(self):
-        import codecs
+        import codecs, sys
         x = u'123abc'
-        assert codecs.getencoder('utf-16')(x) == ('\xff\xfe1\x002\x003\x00a\x00b\x00c\x00', 6)
-        assert codecs.getdecoder('utf-16')('\xff\xfe1\x002\x003\x00a\x00b\x00c\x00') == (x, 14)
+        if sys.byteorder == 'big':
+            assert codecs.getencoder('utf-16')(x) == (
+                    '\xfe\xff\x001\x002\x003\x00a\x00b\x00c', 6)
+            assert codecs.getdecoder('utf-16')(
+                    '\xfe\xff\x001\x002\x003\x00a\x00b\x00c') == (x, 14)
+        else:
+            assert codecs.getencoder('utf-16')(x) == (
+                    '\xff\xfe1\x002\x003\x00a\x00b\x00c\x00', 6)
+            assert codecs.getdecoder('utf-16')(
+                    '\xff\xfe1\x002\x003\x00a\x00b\x00c\x00') == (x, 14)
 
     def test_unicode_escape(self):        
         assert u'\\'.encode('unicode-escape') == '\\\\'
diff --git a/pypy/module/_lsprof/interp_lsprof.py b/pypy/module/_lsprof/interp_lsprof.py
--- a/pypy/module/_lsprof/interp_lsprof.py
+++ b/pypy/module/_lsprof/interp_lsprof.py
@@ -19,8 +19,9 @@
 # cpu affinity settings
 
 srcdir = py.path.local(pypydir).join('translator', 'c', 'src')
-eci = ExternalCompilationInfo(separate_module_files=
-                              [srcdir.join('profiling.c')])
+eci = ExternalCompilationInfo(
+    separate_module_files=[srcdir.join('profiling.c')],
+    export_symbols=['pypy_setup_profiling', 'pypy_teardown_profiling'])
                                                      
 c_setup_profiling = rffi.llexternal('pypy_setup_profiling',
                                   [], lltype.Void,
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -23,6 +23,7 @@
 from pypy.interpreter.function import StaticMethod
 from pypy.objspace.std.sliceobject import W_SliceObject
 from pypy.module.__builtin__.descriptor import W_Property
+from pypy.module.__builtin__.interp_memoryview import W_MemoryView
 from pypy.rlib.entrypoint import entrypoint
 from pypy.rlib.unroll import unrolling_iterable
 from pypy.rlib.objectmodel import specialize
@@ -387,6 +388,8 @@
         "Float": "space.w_float",
         "Long": "space.w_long",
         "Complex": "space.w_complex",
+        "ByteArray": "space.w_bytearray",
+        "MemoryView": "space.gettypeobject(W_MemoryView.typedef)",
         "BaseObject": "space.w_object",
         'None': 'space.type(space.w_None)',
         'NotImplemented': 'space.type(space.w_NotImplemented)',
diff --git a/pypy/module/cpyext/buffer.py b/pypy/module/cpyext/buffer.py
--- a/pypy/module/cpyext/buffer.py
+++ b/pypy/module/cpyext/buffer.py
@@ -1,6 +1,36 @@
+from pypy.interpreter.error import OperationError
 from pypy.rpython.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
     cpython_api, CANNOT_FAIL, Py_buffer)
+from pypy.module.cpyext.pyobject import PyObject
+
+ at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
+def PyObject_CheckBuffer(space, w_obj):
+    """Return 1 if obj supports the buffer interface otherwise 0."""
+    return 0  # the bf_getbuffer field is never filled by cpyext
+
+ at cpython_api([PyObject, lltype.Ptr(Py_buffer), rffi.INT_real],
+             rffi.INT_real, error=-1)
+def PyObject_GetBuffer(space, w_obj, view, flags):
+    """Export obj into a Py_buffer, view.  These arguments must
+    never be NULL.  The flags argument is a bit field indicating what
+    kind of buffer the caller is prepared to deal with and therefore what
+    kind of buffer the exporter is allowed to return.  The buffer interface
+    allows for complicated memory sharing possibilities, but some caller may
+    not be able to handle all the complexity but may want to see if the
+    exporter will let them take a simpler view to its memory.
+
+    Some exporters may not be able to share memory in every possible way and
+    may need to raise errors to signal to some consumers that something is
+    just not possible. These errors should be a BufferError unless
+    there is another error that is actually causing the problem. The
+    exporter can use flags information to simplify how much of the
+    Py_buffer structure is filled in with non-default values and/or
+    raise an error if the object can't support a simpler view of its memory.
+
+    0 is returned on success and -1 on error."""
+    raise OperationError(space.w_TypeError, space.wrap(
+            'PyPy does not yet implement the new buffer interface'))
 
 @cpython_api([lltype.Ptr(Py_buffer), lltype.Char], rffi.INT_real, error=CANNOT_FAIL)
 def PyBuffer_IsContiguous(space, view, fortran):
diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -123,10 +123,6 @@
 typedef Py_ssize_t (*segcountproc)(PyObject *, Py_ssize_t *);
 typedef Py_ssize_t (*charbufferproc)(PyObject *, Py_ssize_t, char **);
 
-typedef int (*objobjproc)(PyObject *, PyObject *);
-typedef int (*visitproc)(PyObject *, void *);
-typedef int (*traverseproc)(PyObject *, visitproc, void *);
-
 /* Py3k buffer interface */
 typedef struct bufferinfo {
     void *buf;
@@ -153,6 +149,41 @@
 typedef int (*getbufferproc)(PyObject *, Py_buffer *, int);
 typedef void (*releasebufferproc)(PyObject *, Py_buffer *);
 
+    /* Flags for getting buffers */
+#define PyBUF_SIMPLE 0
+#define PyBUF_WRITABLE 0x0001
+/*  we used to include an E, backwards compatible alias  */
+#define PyBUF_WRITEABLE PyBUF_WRITABLE
+#define PyBUF_FORMAT 0x0004
+#define PyBUF_ND 0x0008
+#define PyBUF_STRIDES (0x0010 | PyBUF_ND)
+#define PyBUF_C_CONTIGUOUS (0x0020 | PyBUF_STRIDES)
+#define PyBUF_F_CONTIGUOUS (0x0040 | PyBUF_STRIDES)
+#define PyBUF_ANY_CONTIGUOUS (0x0080 | PyBUF_STRIDES)
+#define PyBUF_INDIRECT (0x0100 | PyBUF_STRIDES)
+
+#define PyBUF_CONTIG (PyBUF_ND | PyBUF_WRITABLE)
+#define PyBUF_CONTIG_RO (PyBUF_ND)
+
+#define PyBUF_STRIDED (PyBUF_STRIDES | PyBUF_WRITABLE)
+#define PyBUF_STRIDED_RO (PyBUF_STRIDES)
+
+#define PyBUF_RECORDS (PyBUF_STRIDES | PyBUF_WRITABLE | PyBUF_FORMAT)
+#define PyBUF_RECORDS_RO (PyBUF_STRIDES | PyBUF_FORMAT)
+
+#define PyBUF_FULL (PyBUF_INDIRECT | PyBUF_WRITABLE | PyBUF_FORMAT)
+#define PyBUF_FULL_RO (PyBUF_INDIRECT | PyBUF_FORMAT)
+
+
+#define PyBUF_READ  0x100
+#define PyBUF_WRITE 0x200
+#define PyBUF_SHADOW 0x400
+/* end Py3k buffer interface */
+
+typedef int (*objobjproc)(PyObject *, PyObject *);
+typedef int (*visitproc)(PyObject *, void *);
+typedef int (*traverseproc)(PyObject *, visitproc, void *);
+
 typedef struct {
 	/* For numbers without flag bit Py_TPFLAGS_CHECKTYPES set, all
 	   arguments are guaranteed to be of the object's type (modulo
diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -29,7 +29,7 @@
 #define PY_VERSION		"2.7.1"
 
 /* PyPy version as a string */
-#define PYPY_VERSION "1.7.1"
+#define PYPY_VERSION "1.8.1"
 
 /* Subversion Revision number of this file (not of the repository).
  * Empty since Mercurial migration. */
diff --git a/pypy/module/cpyext/include/pystate.h b/pypy/module/cpyext/include/pystate.h
--- a/pypy/module/cpyext/include/pystate.h
+++ b/pypy/module/cpyext/include/pystate.h
@@ -5,7 +5,7 @@
 struct _is; /* Forward */
 
 typedef struct _is {
-    int _foo;
+    struct _is *next;
 } PyInterpreterState;
 
 typedef struct _ts {
diff --git a/pypy/module/cpyext/pystate.py b/pypy/module/cpyext/pystate.py
--- a/pypy/module/cpyext/pystate.py
+++ b/pypy/module/cpyext/pystate.py
@@ -2,7 +2,10 @@
     cpython_api, generic_cpy_call, CANNOT_FAIL, CConfig, cpython_struct)
 from pypy.rpython.lltypesystem import rffi, lltype
 
-PyInterpreterState = lltype.Ptr(cpython_struct("PyInterpreterState", ()))
+PyInterpreterStateStruct = lltype.ForwardReference()
+PyInterpreterState = lltype.Ptr(PyInterpreterStateStruct)
+cpython_struct(
+    "PyInterpreterState", [('next', PyInterpreterState)], PyInterpreterStateStruct)
 PyThreadState = lltype.Ptr(cpython_struct("PyThreadState", [('interp', PyInterpreterState)]))
 
 @cpython_api([], PyThreadState, error=CANNOT_FAIL)
@@ -54,7 +57,8 @@
 
 class InterpreterState(object):
     def __init__(self, space):
-        self.interpreter_state = lltype.malloc(PyInterpreterState.TO, flavor='raw', immortal=True)
+        self.interpreter_state = lltype.malloc(
+            PyInterpreterState.TO, flavor='raw', zero=True, immortal=True)
 
     def new_thread_state(self):
         capsule = ThreadStateCapsule()
diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py
--- a/pypy/module/cpyext/stubs.py
+++ b/pypy/module/cpyext/stubs.py
@@ -34,141 +34,6 @@
 
 @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
 def PyObject_CheckBuffer(space, obj):
-    """Return 1 if obj supports the buffer interface otherwise 0."""
-    raise NotImplementedError
-
- at cpython_api([PyObject, Py_buffer, rffi.INT_real], rffi.INT_real, error=-1)
-def PyObject_GetBuffer(space, obj, view, flags):
-    """Export obj into a Py_buffer, view.  These arguments must
-    never be NULL.  The flags argument is a bit field indicating what
-    kind of buffer the caller is prepared to deal with and therefore what
-    kind of buffer the exporter is allowed to return.  The buffer interface
-    allows for complicated memory sharing possibilities, but some caller may
-    not be able to handle all the complexity but may want to see if the
-    exporter will let them take a simpler view to its memory.
-
-    Some exporters may not be able to share memory in every possible way and
-    may need to raise errors to signal to some consumers that something is
-    just not possible. These errors should be a BufferError unless
-    there is another error that is actually causing the problem. The
-    exporter can use flags information to simplify how much of the
-    Py_buffer structure is filled in with non-default values and/or
-    raise an error if the object can't support a simpler view of its memory.
-
-    0 is returned on success and -1 on error.
-
-    The following table gives possible values to the flags arguments.
-
-    Flag
-
-    Description
-
-    PyBUF_SIMPLE
-
-    This is the default flag state.  The returned
-    buffer may or may not have writable memory.  The
-    format of the data will be assumed to be unsigned
-    bytes.  This is a "stand-alone" flag constant. It
-    never needs to be '|'d to the others. The exporter
-    will raise an error if it cannot provide such a
-    contiguous buffer of bytes.
-
-    PyBUF_WRITABLE
-
-    The returned buffer must be writable.  If it is
-    not writable, then raise an error.
-
-    PyBUF_STRIDES
-
-    This implies PyBUF_ND. The returned
-    buffer must provide strides information (i.e. the
-    strides cannot be NULL). This would be used when
-    the consumer can handle strided, discontiguous
-    arrays.  Handling strides automatically assumes
-    you can handle shape.  The exporter can raise an
-    error if a strided representation of the data is
-    not possible (i.e. without the suboffsets).
-
-    PyBUF_ND
-
-    The returned buffer must provide shape
-    information. The memory will be assumed C-style
-    contiguous (last dimension varies the
-    fastest). The exporter may raise an error if it
-    cannot provide this kind of contiguous buffer. If
-    this is not given then shape will be NULL.
-
-    PyBUF_C_CONTIGUOUS
-    PyBUF_F_CONTIGUOUS
-    PyBUF_ANY_CONTIGUOUS
-
-    These flags indicate that the contiguity returned
-    buffer must be respectively, C-contiguous (last
-    dimension varies the fastest), Fortran contiguous
-    (first dimension varies the fastest) or either
-    one.  All of these flags imply
-    PyBUF_STRIDES and guarantee that the
-    strides buffer info structure will be filled in
-    correctly.
-
-    PyBUF_INDIRECT
-
-    This flag indicates the returned buffer must have
-    suboffsets information (which can be NULL if no
-    suboffsets are needed).  This can be used when
-    the consumer can handle indirect array
-    referencing implied by these suboffsets. This
-    implies PyBUF_STRIDES.
-
-    PyBUF_FORMAT
-
-    The returned buffer must have true format
-    information if this flag is provided. This would
-    be used when the consumer is going to be checking
-    for what 'kind' of data is actually stored. An
-    exporter should always be able to provide this
-    information if requested. If format is not
-    explicitly requested then the format must be
-    returned as NULL (which means 'B', or
-    unsigned bytes)
-
-    PyBUF_STRIDED
-
-    This is equivalent to (PyBUF_STRIDES |
-    PyBUF_WRITABLE).
-
-    PyBUF_STRIDED_RO
-
-    This is equivalent to (PyBUF_STRIDES).
-
-    PyBUF_RECORDS
-
-    This is equivalent to (PyBUF_STRIDES |
-    PyBUF_FORMAT | PyBUF_WRITABLE).
-
-    PyBUF_RECORDS_RO
-
-    This is equivalent to (PyBUF_STRIDES |
-    PyBUF_FORMAT).
-
-    PyBUF_FULL
-
-    This is equivalent to (PyBUF_INDIRECT |
-    PyBUF_FORMAT | PyBUF_WRITABLE).
-
-    PyBUF_FULL_RO
-
-    This is equivalent to (PyBUF_INDIRECT |
-    PyBUF_FORMAT).
-
-    PyBUF_CONTIG
-
-    This is equivalent to (PyBUF_ND |
-    PyBUF_WRITABLE).
-
-    PyBUF_CONTIG_RO
-
-    This is equivalent to (PyBUF_ND)."""
     raise NotImplementedError
 
 @cpython_api([rffi.CCHARP], Py_ssize_t, error=CANNOT_FAIL)
diff --git a/pypy/module/cpyext/test/test_pystate.py b/pypy/module/cpyext/test/test_pystate.py
--- a/pypy/module/cpyext/test/test_pystate.py
+++ b/pypy/module/cpyext/test/test_pystate.py
@@ -37,6 +37,7 @@
     def test_thread_state_interp(self, space, api):
         ts = api.PyThreadState_Get()
         assert ts.c_interp == api.PyInterpreterState_Head()
+        assert ts.c_interp.c_next == nullptr(PyInterpreterState.TO)
 
     def test_basic_threadstate_dance(self, space, api):
         # Let extension modules call these functions,
diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -9,7 +9,7 @@
     appleveldefs = {}
 
 class Module(MixedModule):
-    applevel_name = 'numpypy'
+    applevel_name = '_numpypy'
 
     submodules = {
         'pypy': PyPyModule
@@ -48,6 +48,7 @@
         'int_': 'interp_boxes.W_LongBox',
         'inexact': 'interp_boxes.W_InexactBox',
         'floating': 'interp_boxes.W_FloatingBox',
+        'float_': 'interp_boxes.W_Float64Box',
         'float32': 'interp_boxes.W_Float32Box',
         'float64': 'interp_boxes.W_Float64Box',
     }
@@ -89,7 +90,6 @@
 
     appleveldefs = {
         'average': 'app_numpy.average',
-        'mean': 'app_numpy.mean',
         'sum': 'app_numpy.sum',
         'min': 'app_numpy.min',
         'identity': 'app_numpy.identity',
@@ -98,5 +98,4 @@
         'e': 'app_numpy.e',
         'pi': 'app_numpy.pi',
         'arange': 'app_numpy.arange',
-        'reshape': 'app_numpy.reshape',
     }
diff --git a/pypy/module/micronumpy/app_numpy.py b/pypy/module/micronumpy/app_numpy.py
--- a/pypy/module/micronumpy/app_numpy.py
+++ b/pypy/module/micronumpy/app_numpy.py
@@ -1,6 +1,6 @@
 import math
 
-import numpypy
+import _numpypy
 
 
 inf = float("inf")
@@ -11,33 +11,54 @@
 def average(a):
     # This implements a weighted average, for now we don't implement the
     # weighting, just the average part!
-    return mean(a)
+    if not hasattr(a, "mean"):
+        a = _numpypy.array(a)
+    return a.mean()
 
 def identity(n, dtype=None):
-    a = numpypy.zeros((n,n), dtype=dtype)
+    a = _numpypy.zeros((n, n), dtype=dtype)
     for i in range(n):
         a[i][i] = 1
     return a
 
-def mean(a):
-    if not hasattr(a, "mean"):
-        a = numpypy.array(a)
-    return a.mean()
+def sum(a,axis=None):
+    '''sum(a, axis=None)
+    Sum of array elements over a given axis.
 
-def sum(a):
+    Parameters
+    ----------
+    a : array_like
+        Elements to sum.
+    axis : integer, optional
+        Axis over which the sum is taken. By default `axis` is None,
+        and all elements are summed.
+
+    Returns
+    -------
+    sum_along_axis : ndarray
+        An array with the same shape as `a`, with the specified
+        axis removed.   If `a` is a 0-d array, or if `axis` is None, a scalar
+        is returned.  If an output array is specified, a reference to
+        `out` is returned.
+
+    See Also
+    --------
+    ndarray.sum : Equivalent method.
+    '''
+    # TODO: add to doc (once it's implemented): cumsum : Cumulative sum of array elements.
     if not hasattr(a, "sum"):
-        a = numpypy.array(a)
-    return a.sum()
+        a = _numpypy.array(a)
+    return a.sum(axis)
 
-def min(a):
+def min(a, axis=None):
     if not hasattr(a, "min"):
-        a = numpypy.array(a)
-    return a.min()
+        a = _numpypy.array(a)
+    return a.min(axis)
 
-def max(a):
+def max(a, axis=None):
     if not hasattr(a, "max"):
-        a = numpypy.array(a)
-    return a.max()
+        a = _numpypy.array(a)
+    return a.max(axis)
 
 def arange(start, stop=None, step=1, dtype=None):
     '''arange([start], stop[, step], dtype=None)
@@ -47,48 +68,11 @@
         stop = start
         start = 0
     if dtype is None:
-        test = numpypy.array([start, stop, step, 0])
+        test = _numpypy.array([start, stop, step, 0])
         dtype = test.dtype
-    arr = numpypy.zeros(int(math.ceil((stop - start) / step)), dtype=dtype)
+    arr = _numpypy.zeros(int(math.ceil((stop - start) / step)), dtype=dtype)
     i = start
     for j in range(arr.size):
         arr[j] = i
         i += step
     return arr
-
-
-def reshape(a, shape):
-    '''reshape(a, newshape)
-    Gives a new shape to an array without changing its data.
-
-    Parameters
-    ----------
-    a : array_like
-        Array to be reshaped.
-    newshape : int or tuple of ints
-        The new shape should be compatible with the original shape. If
-        an integer, then the result will be a 1-D array of that length.
-        One shape dimension can be -1. In this case, the value is inferred
-        from the length of the array and remaining dimensions.
-
-    Returns
-    -------
-    reshaped_array : ndarray
-        This will be a new view object if possible; otherwise, it will
-        be a copy.
-
-
-    See Also
-    --------
-    ndarray.reshape : Equivalent method.
-
-    Notes
-    -----
-
-    It is not always possible to change the shape of an array without
-    copying the data. If you want an error to be raise if the data is copied,
-    you should assign the new shape to the shape attribute of the array
-'''
-    if not hasattr(a, 'reshape'):
-        a = numpypy.array(a)
-    return a.reshape(shape)
diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -372,13 +372,17 @@
 
     def execute(self, interp):
         if self.name in SINGLE_ARG_FUNCTIONS:
-            if len(self.args) != 1:
+            if len(self.args) != 1 and self.name != 'sum':
                 raise ArgumentMismatch
             arr = self.args[0].execute(interp)
             if not isinstance(arr, BaseArray):
                 raise ArgumentNotAnArray
             if self.name == "sum":
-                w_res = arr.descr_sum(interp.space)
+                if len(self.args)>1:
+                    w_res = arr.descr_sum(interp.space,
+                                          self.args[1].execute(interp))
+                else:
+                    w_res = arr.descr_sum(interp.space)
             elif self.name == "prod":
                 w_res = arr.descr_prod(interp.space)
             elif self.name == "max":
@@ -416,7 +420,7 @@
     ('\]', 'array_right'),
     ('(->)|[\+\-\*\/]', 'operator'),
     ('=', 'assign'),
-    (',', 'coma'),
+    (',', 'comma'),
     ('\|', 'pipe'),
     ('\(', 'paren_left'),
     ('\)', 'paren_right'),
@@ -504,7 +508,7 @@
         return SliceConstant(start, stop, step)
 
 
-    def parse_expression(self, tokens):
+    def parse_expression(self, tokens, accept_comma=False):
         stack = []
         while tokens.remaining():
             token = tokens.pop()
@@ -524,9 +528,13 @@
                 stack.append(RangeConstant(tokens.pop().v))
                 end = tokens.pop()
                 assert end.name == 'pipe'
+            elif accept_comma and token.name == 'comma':
+                continue
             else:
                 tokens.push()
                 break
+        if accept_comma:
+            return stack
         stack.reverse()
         lhs = stack.pop()
         while stack:
@@ -540,7 +548,7 @@
         args = []
         tokens.pop() # lparen
         while tokens.get(0).name != 'paren_right':
-            args.append(self.parse_expression(tokens))
+            args += self.parse_expression(tokens, accept_comma=True)
         return FunctionCall(name, args)
 
     def parse_array_const(self, tokens):
@@ -556,7 +564,7 @@
             token = tokens.pop()
             if token.name == 'array_right':
                 return elems
-            assert token.name == 'coma'
+            assert token.name == 'comma'
 
     def parse_statement(self, tokens):
         if (tokens.get(0).name == 'identifier' and
diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py
--- a/pypy/module/micronumpy/interp_boxes.py
+++ b/pypy/module/micronumpy/interp_boxes.py
@@ -78,6 +78,7 @@
     descr_sub = _binop_impl("subtract")
     descr_mul = _binop_impl("multiply")
     descr_div = _binop_impl("divide")
+    descr_pow = _binop_impl("power")
     descr_eq = _binop_impl("equal")
     descr_ne = _binop_impl("not_equal")
     descr_lt = _binop_impl("less")
@@ -170,6 +171,7 @@
     __sub__ = interp2app(W_GenericBox.descr_sub),
     __mul__ = interp2app(W_GenericBox.descr_mul),
     __div__ = interp2app(W_GenericBox.descr_div),
+    __pow__ = interp2app(W_GenericBox.descr_pow),
 
     __radd__ = interp2app(W_GenericBox.descr_radd),
     __rsub__ = interp2app(W_GenericBox.descr_rsub),
@@ -245,6 +247,7 @@
     long_name = "int64"
 W_LongBox.typedef = TypeDef(long_name, (W_SignedIntegerBox.typedef, int_typedef,),
     __module__ = "numpypy",
+   __new__ = interp2app(W_LongBox.descr__new__.im_func),
 )
 
 W_ULongBox.typedef = TypeDef("u" + long_name, W_UnsignedIntegerBox.typedef,
diff --git a/pypy/module/micronumpy/interp_iter.py b/pypy/module/micronumpy/interp_iter.py
--- a/pypy/module/micronumpy/interp_iter.py
+++ b/pypy/module/micronumpy/interp_iter.py
@@ -1,19 +1,20 @@
 
 from pypy.rlib import jit
 from pypy.rlib.objectmodel import instantiate
-from pypy.module.micronumpy.strides import calculate_broadcast_strides
+from pypy.module.micronumpy.strides import calculate_broadcast_strides,\
+     calculate_slice_strides
 
-# Iterators for arrays
-# --------------------
-# all those iterators with the exception of BroadcastIterator iterate over the
-# entire array in C order (the last index changes the fastest). This will
-# yield all elements. Views iterate over indices and look towards strides and
-# backstrides to find the correct position. Notably the offset between
-# x[..., i + 1] and x[..., i] will be strides[-1]. Offset between
-# x[..., k + 1, 0] and x[..., k, i_max] will be backstrides[-2] etc.
+class BaseTransform(object):
+    pass
 
-# BroadcastIterator works like that, but for indexes that don't change source
-# in the original array, strides[i] == backstrides[i] == 0
+class ViewTransform(BaseTransform):
+    def __init__(self, chunks):
+        # 4-tuple specifying slicing
+        self.chunks = chunks
+
+class BroadcastTransform(BaseTransform):
+    def __init__(self, res_shape):
+        self.res_shape = res_shape
 
 class BaseIterator(object):
     def next(self, shapelen):
@@ -22,6 +23,15 @@
     def done(self):
         raise NotImplementedError
 
+    def apply_transformations(self, arr, transformations):
+        v = self
+        for transform in transformations:
+            v = v.transform(arr, transform)
+        return v
+
+    def transform(self, arr, t):
+        raise NotImplementedError
+
 class ArrayIterator(BaseIterator):
     def __init__(self, size):
         self.offset = 0
@@ -36,6 +46,10 @@
     def done(self):
         return self.offset >= self.size
 
+    def transform(self, arr, t):
+        return ViewIterator(arr.start, arr.strides, arr.backstrides,
+                            arr.shape).transform(arr, t)
+
 class OneDimIterator(BaseIterator):
     def __init__(self, start, step, stop):
         self.offset = start
@@ -52,26 +66,29 @@
     def done(self):
         return self.offset == self.size
 
-def view_iter_from_arr(arr):
-    return ViewIterator(arr.start, arr.strides, arr.backstrides, arr.shape)
-
 class ViewIterator(BaseIterator):
-    def __init__(self, start, strides, backstrides, shape, res_shape=None):
+    def __init__(self, start, strides, backstrides, shape):
         self.offset  = start
         self._done   = False
-        if res_shape is not None and res_shape != shape:
-            r = calculate_broadcast_strides(strides, backstrides,
-                                            shape, res_shape)
-            self.strides, self.backstrides = r
-            self.res_shape = res_shape
-        else:
-            self.strides = strides
-            self.backstrides = backstrides
-            self.res_shape = shape
+        self.strides = strides
+        self.backstrides = backstrides
+        self.res_shape = shape
         self.indices = [0] * len(self.res_shape)
 
+    def transform(self, arr, t):
+        if isinstance(t, BroadcastTransform):
+            r = calculate_broadcast_strides(self.strides, self.backstrides,
+                                            self.res_shape, t.res_shape)
+            return ViewIterator(self.offset, r[0], r[1], t.res_shape)
+        elif isinstance(t, ViewTransform):
+            r = calculate_slice_strides(self.res_shape, self.offset,
+                                        self.strides,
+                                        self.backstrides, t.chunks)
+            return ViewIterator(r[1], r[2], r[3], r[0])
+
     @jit.unroll_safe
     def next(self, shapelen):
+        shapelen = jit.promote(len(self.res_shape))
         offset = self.offset
         indices = [0] * shapelen
         for i in range(shapelen):
@@ -96,6 +113,13 @@
         res._done = done
         return res
 
+    def apply_transformations(self, arr, transformations):
+        v = BaseIterator.apply_transformations(self, arr, transformations)
+        if len(arr.shape) == 1:
+            return OneDimIterator(self.offset, self.strides[0],
+                                  self.res_shape[0])
+        return v
+
     def done(self):
         return self._done
 
@@ -103,11 +127,59 @@
     def next(self, shapelen):
         return self
 
+    def transform(self, arr, t):
+        pass
+
+class AxisIterator(BaseIterator):
+    def __init__(self, start, dim, shape, strides, backstrides):
+        self.res_shape = shape[:]
+        self.strides = strides[:dim] + [0] + strides[dim:]
+        self.backstrides = backstrides[:dim] + [0] + backstrides[dim:]
+        self.first_line = True
+        self.indices = [0] * len(shape)
+        self._done = False
+        self.offset = start
+        self.dim = dim
+
+    @jit.unroll_safe
+    def next(self, shapelen):
+        offset = self.offset
+        first_line = self.first_line
+        indices = [0] * shapelen
+        for i in range(shapelen):
+            indices[i] = self.indices[i]
+        done = False
+        for i in range(shapelen - 1, -1, -1):
+            if indices[i] < self.res_shape[i] - 1:
+                if i == self.dim:
+                    first_line = False
+                indices[i] += 1
+                offset += self.strides[i]
+                break
+            else:
+                if i == self.dim:
+                    first_line = True
+                indices[i] = 0
+                offset -= self.backstrides[i]
+        else:
+            done = True
+        res = instantiate(AxisIterator)
+        res.offset = offset
+        res.indices = indices
+        res.strides = self.strides
+        res.backstrides = self.backstrides
+        res.res_shape = self.res_shape
+        res._done = done
+        res.first_line = first_line
+        res.dim = self.dim
+        return res        
+
+    def done(self):
+        return self._done
+
 # ------ other iterators that are not part of the computation frame ----------
-
-class AxisIterator(object):
-    """ This object will return offsets of each start of the last stride
-    """
+    
+class SkipLastAxisIterator(object):
     def __init__(self, arr):
         self.arr = arr
         self.indices = [0] * (len(arr.shape) - 1)
@@ -125,4 +197,3 @@
                 self.offset -= self.arr.backstrides[i]
         else:
             self.done = True
-        
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -8,34 +8,39 @@
 from pypy.rpython.lltypesystem import lltype, rffi
 from pypy.tool.sourcetools import func_with_new_name
 from pypy.rlib.rstring import StringBuilder
-from pypy.module.micronumpy.interp_iter import ArrayIterator,\
-     view_iter_from_arr, OneDimIterator, AxisIterator
+from pypy.module.micronumpy.interp_iter import ArrayIterator, OneDimIterator,\
+     SkipLastAxisIterator
 
 numpy_driver = jit.JitDriver(
     greens=['shapelen', 'sig'],
     virtualizables=['frame'],
     reds=['result_size', 'frame', 'ri', 'self', 'result'],
     get_printable_location=signature.new_printable_location('numpy'),
+    name='numpy',
 )
 all_driver = jit.JitDriver(
     greens=['shapelen', 'sig'],
     virtualizables=['frame'],
     reds=['frame', 'self', 'dtype'],
     get_printable_location=signature.new_printable_location('all'),
+    name='numpy_all',
 )
 any_driver = jit.JitDriver(
     greens=['shapelen', 'sig'],
     virtualizables=['frame'],
     reds=['frame', 'self', 'dtype'],
     get_printable_location=signature.new_printable_location('any'),
+    name='numpy_any',
 )
 slice_driver = jit.JitDriver(
     greens=['shapelen', 'sig'],
     virtualizables=['frame'],
-    reds=['self', 'frame', 'source', 'res_iter'],
+    reds=['self', 'frame', 'arr'],
     get_printable_location=signature.new_printable_location('slice'),
+    name='numpy_slice',
 )
 
+
 def _find_shape_and_elems(space, w_iterable):
     shape = [space.len_w(w_iterable)]
     batch = space.listview(w_iterable)
@@ -152,9 +157,6 @@
 # (meaning that the realignment of elements crosses from one step into another)
 # return None so that the caller can raise an exception.
 def calc_new_strides(new_shape, old_shape, old_strides):
-    # Return the proper strides for new_shape, or None if the mapping crosses
-    # stepping boundaries
-
     # Assumes that prod(old_shape) == prod(new_shape), len(old_shape) > 1, and
     # len(new_shape) > 0
     steps = []
@@ -162,6 +164,7 @@
     oldI = 0
     new_strides = []
     if old_strides[0] < old_strides[-1]:
+        #Start at old_shape[0], old_stides[0]
         for i in range(len(old_shape)):
             steps.append(old_strides[i] / last_step)
             last_step *= old_shape[i]
@@ -179,10 +182,11 @@
             if n_new_elems_used == n_old_elems_to_use:
                 oldI += 1
                 if oldI >= len(old_shape):
-                    break
+                    continue
                 cur_step = steps[oldI]
                 n_old_elems_to_use *= old_shape[oldI]
     else:
+        #Start at old_shape[-1], old_strides[-1]
         for i in range(len(old_shape) - 1, -1, -1):
             steps.insert(0, old_strides[i] / last_step)
             last_step *= old_shape[i]
@@ -202,7 +206,7 @@
             if n_new_elems_used == n_old_elems_to_use:
                 oldI -= 1
                 if oldI < -len(old_shape):
-                    break
+                    continue
                 cur_step = steps[oldI]
                 n_old_elems_to_use *= old_shape[oldI]
     return new_strides
@@ -282,13 +286,17 @@
     descr_rpow = _binop_right_impl("power")
     descr_rmod = _binop_right_impl("mod")
 
-    def _reduce_ufunc_impl(ufunc_name):
-        def impl(self, space):
-            return getattr(interp_ufuncs.get(space), ufunc_name).reduce(space, self, multidim=True)
+    def _reduce_ufunc_impl(ufunc_name, promote_to_largest=False):
+        def impl(self, space, w_axis=None):
+            if space.is_w(w_axis, space.w_None):
+                w_axis = space.wrap(-1)
+            return getattr(interp_ufuncs.get(space), ufunc_name).reduce(space,
+                                        self, True, promote_to_largest, w_axis)
         return func_with_new_name(impl, "reduce_%s_impl" % ufunc_name)
 
     descr_sum = _reduce_ufunc_impl("add")
-    descr_prod = _reduce_ufunc_impl("multiply")
+    descr_sum_promote = _reduce_ufunc_impl("add", True)
+    descr_prod = _reduce_ufunc_impl("multiply", True)
     descr_max = _reduce_ufunc_impl("maximum")
     descr_min = _reduce_ufunc_impl("minimum")
 
@@ -297,6 +305,7 @@
             greens=['shapelen', 'sig'],
             reds=['result', 'idx', 'frame', 'self', 'cur_best', 'dtype'],
             get_printable_location=signature.new_printable_location(op_name),
+            name='numpy_' + op_name,
         )
         def loop(self):
             sig = self.find_sig()
@@ -372,7 +381,7 @@
         else:
             w_res = self.descr_mul(space, w_other)
             assert isinstance(w_res, BaseArray)
-            return w_res.descr_sum(space)
+            return w_res.descr_sum(space, space.wrap(-1))
 
     def get_concrete(self):
         raise NotImplementedError
@@ -380,6 +389,9 @@
     def descr_get_dtype(self, space):
         return space.wrap(self.find_dtype())
 
+    def descr_get_ndim(self, space):
+        return space.wrap(len(self.shape))
+
     @jit.unroll_safe
     def descr_get_shape(self, space):
         return space.newtuple([space.wrap(i) for i in self.shape])
@@ -409,7 +421,7 @@
     def descr_repr(self, space):
         res = StringBuilder()
         res.append("array(")
-        concrete = self.get_concrete()
+        concrete = self.get_concrete_or_scalar()
         dtype = concrete.find_dtype()
         if not concrete.size:
             res.append('[]')
@@ -422,8 +434,9 @@
         else:
             concrete.to_str(space, 1, res, indent='       ')
         if (dtype is not interp_dtype.get_dtype_cache(space).w_float64dtype and
-            dtype is not interp_dtype.get_dtype_cache(space).w_int64dtype) or \
-            not self.size:
+            not (dtype.kind == interp_dtype.SIGNEDLTR and
+            dtype.itemtype.get_element_size() == rffi.sizeof(lltype.Signed)) or
+            not self.size):
             res.append(", dtype=" + dtype.name)
         res.append(")")
         return space.wrap(res.build())
@@ -556,8 +569,26 @@
             )
         return w_result
 
-    def descr_mean(self, space):
-        return space.div(self.descr_sum(space), space.wrap(self.size))
+    def descr_mean(self, space, w_axis=None):
+        if space.is_w(w_axis, space.w_None):
+            w_axis = space.wrap(-1)
+            w_denom = space.wrap(self.size)
+        else:
+            dim = space.int_w(w_axis)
+            w_denom = space.wrap(self.shape[dim])
+        return space.div(self.descr_sum_promote(space, w_axis), w_denom)
+
+    def descr_var(self, space):
+        # var = mean((values - mean(values)) ** 2)
+        w_res = self.descr_sub(space, self.descr_mean(space, space.w_None))
+        assert isinstance(w_res, BaseArray) 
+        w_res = w_res.descr_pow(space, space.wrap(2))
+        assert isinstance(w_res, BaseArray)
+        return w_res.descr_mean(space, space.w_None)
+
+    def descr_std(self, space):
+        # std(v) = sqrt(var(v))
+        return interp_ufuncs.get(space).sqrt.call(space, [self.descr_var(space)])
 
     def descr_nonzero(self, space):
         if self.size > 1:
@@ -592,11 +623,12 @@
     def getitem(self, item):
         raise NotImplementedError
 
-    def find_sig(self, res_shape=None):
+    def find_sig(self, res_shape=None, arr=None):
         """ find a correct signature for the array
         """
         res_shape = res_shape or self.shape
-        return signature.find_sig(self.create_sig(res_shape), self)
+        arr = arr or self
+        return signature.find_sig(self.create_sig(), arr)
 
     def descr_array_iface(self, space):
         if not self.shape:
@@ -650,7 +682,7 @@
     def copy(self, space):
         return Scalar(self.dtype, self.value)
 
-    def create_sig(self, res_shape):
+    def create_sig(self):
         return signature.ScalarSignature(self.dtype)
 
     def get_concrete_or_scalar(self):
@@ -668,7 +700,8 @@
         self.name = name
 
     def _del_sources(self):
-        # Function for deleting references to source arrays, to allow garbage-collecting them
+        # Function for deleting references to source arrays,
+        # to allow garbage-collecting them
         raise NotImplementedError
 
     def compute(self):
@@ -720,11 +753,11 @@
         self.size = size
         VirtualArray.__init__(self, 'slice', shape, child.find_dtype())
 
-    def create_sig(self, res_shape):
+    def create_sig(self):
         if self.forced_result is not None:
-            return self.forced_result.create_sig(res_shape)
+            return self.forced_result.create_sig()
         return signature.VirtualSliceSignature(
-            self.child.create_sig(res_shape))
+            self.child.create_sig())
 
     def force_if_needed(self):
         if self.forced_result is None:
@@ -734,6 +767,7 @@
     def _del_sources(self):
         self.child = None
 
+
 class Call1(VirtualArray):
     def __init__(self, ufunc, name, shape, res_dtype, values):
         VirtualArray.__init__(self, name, shape, res_dtype)
@@ -744,16 +778,17 @@
     def _del_sources(self):
         self.values = None
 
-    def create_sig(self, res_shape):
+    def create_sig(self):
         if self.forced_result is not None:
-            return self.forced_result.create_sig(res_shape)
-        return signature.Call1(self.ufunc, self.name,
-                               self.values.create_sig(res_shape))
+            return self.forced_result.create_sig()
+        return signature.Call1(self.ufunc, self.name, self.values.create_sig())
 
 class Call2(VirtualArray):
     """
     Intermediate class for performing binary operations.
     """
+    _immutable_fields_ = ['left', 'right']
+    
     def __init__(self, ufunc, name, shape, calc_dtype, res_dtype, left, right):
         VirtualArray.__init__(self, name, shape, res_dtype)
         self.ufunc = ufunc
@@ -768,12 +803,55 @@
         self.left = None
         self.right = None
 
-    def create_sig(self, res_shape):
+    def create_sig(self):
         if self.forced_result is not None:
-            return self.forced_result.create_sig(res_shape)
+            return self.forced_result.create_sig()
+        if self.shape != self.left.shape and self.shape != self.right.shape:
+            return signature.BroadcastBoth(self.ufunc, self.name,
+                                           self.calc_dtype,
+                                           self.left.create_sig(),
+                                           self.right.create_sig())
+        elif self.shape != self.left.shape:
+            return signature.BroadcastLeft(self.ufunc, self.name,
+                                           self.calc_dtype,
+                                           self.left.create_sig(),
+                                           self.right.create_sig())
+        elif self.shape != self.right.shape:
+            return signature.BroadcastRight(self.ufunc, self.name,
+                                            self.calc_dtype,
+                                            self.left.create_sig(),
+                                            self.right.create_sig())
         return signature.Call2(self.ufunc, self.name, self.calc_dtype,
-                               self.left.create_sig(res_shape),
-                               self.right.create_sig(res_shape))
+                               self.left.create_sig(), self.right.create_sig())
+
+class SliceArray(Call2):
+    def __init__(self, shape, dtype, left, right):
+        Call2.__init__(self, None, 'sliceloop', shape, dtype, dtype, left,
+                       right)
+    
+    def create_sig(self):
+        lsig = self.left.create_sig()
+        rsig = self.right.create_sig()
+        if self.shape != self.right.shape:
+            return signature.SliceloopBroadcastSignature(self.ufunc,
+                                                         self.name,
+                                                         self.calc_dtype,
+                                                         lsig, rsig)
+        return signature.SliceloopSignature(self.ufunc, self.name,
+                                            self.calc_dtype,
+                                            lsig, rsig)
+
+class AxisReduce(Call2):
+    """ NOTE: this is only used as a container, you should never
+    encounter such things in the wild. Remove this comment
+    when we'll make AxisReduce lazy
+    """
+    _immutable_fields_ = ['left', 'right']
+    
+    def __init__(self, ufunc, name, shape, dtype, left, right, dim):
+        Call2.__init__(self, ufunc, name, shape, dtype, dtype,
+                       left, right)
+        self.dim = dim
 
 class ConcreteArray(BaseArray):
     """ An array that have actual storage, whether owned or not
@@ -828,11 +906,6 @@
         self.strides = strides
         self.backstrides = backstrides
 
-    def array_sig(self, res_shape):
-        if res_shape is not None and self.shape != res_shape:
-            return signature.ViewSignature(self.dtype)
-        return signature.ArraySignature(self.dtype)
-
     def to_str(self, space, comma, builder, indent=' ', use_ellipsis=False):
         '''Modifies builder with a representation of the array/slice
         The items will be seperated by a comma if comma is 1
@@ -840,80 +913,80 @@
         each line will begin with indent.
         '''
         size = self.size
+        ccomma = ',' * comma
+        ncomma = ',' * (1 - comma)
+        dtype = self.find_dtype()
         if size < 1:
             builder.append('[]')
             return
+        elif size == 1:
+            builder.append(dtype.itemtype.str_format(self.getitem(0)))
+            return
         if size > 1000:
             # Once this goes True it does not go back to False for recursive
             # calls
             use_ellipsis = True
-        dtype = self.find_dtype()
         ndims = len(self.shape)
         i = 0
-        start = True
         builder.append('[')
         if ndims > 1:
             if use_ellipsis:
-                for i in range(3):
-                    if start:
-                        start = False
-                    else:
-                        builder.append(',' * comma + '\n')
-                        if ndims == 3:
+                for i in range(min(3, self.shape[0])):
+                    if i > 0:
+                        builder.append(ccomma + '\n')
+                        if ndims >= 3:
                             builder.append('\n' + indent)
                         else:
                             builder.append(indent)
-                    # create_slice requires len(chunks) > 1 in order to reduce
-                    # shape
-                    view = self.create_slice([(i, 0, 0, 1), (0, self.shape[1], 1, self.shape[1])]).get_concrete()
-                    view.to_str(space, comma, builder, indent=indent + ' ', use_ellipsis=use_ellipsis)
-                builder.append('\n' + indent + '..., ')
-                i = self.shape[0] - 3
+                    view = self.create_slice([(i, 0, 0, 1)]).get_concrete()
+                    view.to_str(space, comma, builder, indent=indent + ' ',
+                                                    use_ellipsis=use_ellipsis)
+                if i < self.shape[0] - 1:
+                    builder.append(ccomma + '\n' + indent + '...' + ncomma)
+                    i = self.shape[0] - 3
+                else:
+                    i += 1
             while i < self.shape[0]:
-                if start:
-                    start = False
-                else:
-                    builder.append(',' * comma + '\n')
-                    if ndims == 3:
+                if i > 0:
+                    builder.append(ccomma + '\n')
+                    if ndims >= 3:
                         builder.append('\n' + indent)
                     else:
                         builder.append(indent)
                 # create_slice requires len(chunks) > 1 in order to reduce
                 # shape
-                view = self.create_slice([(i, 0, 0, 1), (0, self.shape[1], 1, self.shape[1])]).get_concrete()
-                view.to_str(space, comma, builder, indent=indent + ' ', use_ellipsis=use_ellipsis)
+                view = self.create_slice([(i, 0, 0, 1)]).get_concrete()
+                view.to_str(space, comma, builder, indent=indent + ' ',
+                                                    use_ellipsis=use_ellipsis)
                 i += 1
         elif ndims == 1:
-            spacer = ',' * comma + ' '
+            spacer = ccomma + ' '
             item = self.start
             # An iterator would be a nicer way to walk along the 1d array, but
             # how do I reset it if printing ellipsis? iterators have no
             # "set_offset()"
             i = 0
             if use_ellipsis:
-                for i in range(3):
-                    if start:
-                        start = False
-                    else:
+                for i in range(min(3, self.shape[0])):
+                    if i > 0:
                         builder.append(spacer)
                     builder.append(dtype.itemtype.str_format(self.getitem(item)))
                     item += self.strides[0]
-                # Add a comma only if comma is False - this prevents adding two
-                # commas
-                builder.append(spacer + '...' + ',' * (1 - comma))
-                # Ugly, but can this be done with an iterator?
-                item = self.start + self.backstrides[0] - 2 * self.strides[0]
-                i = self.shape[0] - 3
+                if i < self.shape[0] - 1:
+                    # Add a comma only if comma is False - this prevents adding
+                    # two commas
+                    builder.append(spacer + '...' + ncomma)
+                    # Ugly, but can this be done with an iterator?
+                    item = self.start + self.backstrides[0] - 2 * self.strides[0]
+                    i = self.shape[0] - 3
+                else:
+                    i += 1
             while i < self.shape[0]:
-                if start:
-                    start = False
-                else:
+                if i > 0:
                     builder.append(spacer)
                 builder.append(dtype.itemtype.str_format(self.getitem(item)))
                 item += self.strides[0]
                 i += 1
-        else:
-            builder.append('[')
         builder.append(']')
 
     @jit.unroll_safe
@@ -947,20 +1020,22 @@
             self.dtype is w_value.find_dtype()):
             self._fast_setslice(space, w_value)
         else:
-            self._sliceloop(w_value, res_shape)
+            arr = SliceArray(self.shape, self.dtype, self, w_value)
+            self._sliceloop(arr)
 
     def _fast_setslice(self, space, w_value):
         assert isinstance(w_value, ConcreteArray)
         itemsize = self.dtype.itemtype.get_element_size()
-        if len(self.shape) == 1:
+        shapelen = len(self.shape)
+        if shapelen == 1:
             rffi.c_memcpy(
                 rffi.ptradd(self.storage, self.start * itemsize),
                 rffi.ptradd(w_value.storage, w_value.start * itemsize),
                 self.size * itemsize
             )
         else:
-            dest = AxisIterator(self)
-            source = AxisIterator(w_value)
+            dest = SkipLastAxisIterator(self)
+            source = SkipLastAxisIterator(w_value)
             while not dest.done:
                 rffi.c_memcpy(
                     rffi.ptradd(self.storage, dest.offset * itemsize),
@@ -970,21 +1045,16 @@
                 source.next()
                 dest.next()
 
-    def _sliceloop(self, source, res_shape):
-        sig = source.find_sig(res_shape)
-        frame = sig.create_frame(source, res_shape)
-        res_iter = view_iter_from_arr(self)
-        shapelen = len(res_shape)
-        while not res_iter.done():
-            slice_driver.jit_merge_point(sig=sig,
-                                         frame=frame,
-                                         shapelen=shapelen,
-                                         self=self, source=source,
-                                         res_iter=res_iter)
-            self.setitem(res_iter.offset, sig.eval(frame, source).convert_to(
-                self.find_dtype()))
+    def _sliceloop(self, arr):
+        sig = arr.find_sig()
+        frame = sig.create_frame(arr)
+        shapelen = len(self.shape)
+        while not frame.done():
+            slice_driver.jit_merge_point(sig=sig, frame=frame, self=self,
+                                         arr=arr,
+                                         shapelen=shapelen)
+            sig.eval(frame, arr)
             frame.next(shapelen)
-            res_iter = res_iter.next(shapelen)
 
     def copy(self, space):
         array = W_NDimArray(self.size, self.shape[:], self.dtype, self.order)
@@ -993,7 +1063,7 @@
 
 
 class ViewArray(ConcreteArray):
-    def create_sig(self, res_shape):
+    def create_sig(self):
         return signature.ViewSignature(self.dtype)
 
 
@@ -1057,8 +1127,8 @@
         self.shape = new_shape
         self.calc_strides(new_shape)
 
-    def create_sig(self, res_shape):
-        return self.array_sig(res_shape)
+    def create_sig(self):
+        return signature.ArraySignature(self.dtype)
 
     def __del__(self):
         lltype.free(self.storage, flavor='raw', track_allocation=False)
@@ -1185,6 +1255,7 @@
     shape = GetSetProperty(BaseArray.descr_get_shape,
                            BaseArray.descr_set_shape),
     size = GetSetProperty(BaseArray.descr_get_size),
+    ndim = GetSetProperty(BaseArray.descr_get_ndim),
 
     T = GetSetProperty(BaseArray.descr_get_transpose),
     flat = GetSetProperty(BaseArray.descr_get_flatiter),
@@ -1199,6 +1270,8 @@
     all = interp2app(BaseArray.descr_all),
     any = interp2app(BaseArray.descr_any),
     dot = interp2app(BaseArray.descr_dot),
+    var = interp2app(BaseArray.descr_var),
+    std = interp2app(BaseArray.descr_std),
 
     copy = interp2app(BaseArray.descr_copy),
     reshape = interp2app(BaseArray.descr_reshape),
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -3,19 +3,29 @@
 from pypy.interpreter.gateway import interp2app
 from pypy.interpreter.typedef import TypeDef, GetSetProperty, interp_attrproperty
 from pypy.module.micronumpy import interp_boxes, interp_dtype
-from pypy.module.micronumpy.signature import ReduceSignature, ScalarSignature,\
-     find_sig, new_printable_location
+from pypy.module.micronumpy.signature import ReduceSignature,\
+     find_sig, new_printable_location, AxisReduceSignature, ScalarSignature
 from pypy.rlib import jit
 from pypy.rlib.rarithmetic import LONG_BIT
 from pypy.tool.sourcetools import func_with_new_name
 
 reduce_driver = jit.JitDriver(
-    greens = ['shapelen', "sig"],
-    virtualizables = ["frame"],
-    reds = ["frame", "self", "dtype", "value", "obj"],
+    greens=['shapelen', "sig"],
+    virtualizables=["frame"],
+    reds=["frame", "self", "dtype", "value", "obj"],
     get_printable_location=new_printable_location('reduce'),
+    name='numpy_reduce',
 )
 
+axisreduce_driver = jit.JitDriver(
+    greens=['shapelen', 'sig'],
+    virtualizables=['frame'],
+    reds=['self','arr', 'identity', 'frame'],
+    name='numpy_axisreduce',
+    get_printable_location=new_printable_location('axisreduce'),
+)
+
+
 class W_Ufunc(Wrappable):
     _attrs_ = ["name", "promote_to_float", "promote_bools", "identity"]
     _immutable_fields_ = ["promote_to_float", "promote_bools", "name"]
@@ -48,18 +58,72 @@
             )
         return self.call(space, __args__.arguments_w)
 
-    def descr_reduce(self, space, w_obj):
-        return self.reduce(space, w_obj, multidim=False)
+    def descr_reduce(self, space, w_obj, w_dim=0):
+        """reduce(...)
+        reduce(a, axis=0)
 
-    def reduce(self, space, w_obj, multidim):
-        from pypy.module.micronumpy.interp_numarray import convert_to_array, Scalar
-        
+        Reduces `a`'s dimension by one, by applying ufunc along one axis.
+
+        Let :math:`a.shape = (N_0, ..., N_i, ..., N_{M-1})`.  Then
+        :math:`ufunc.reduce(a, axis=i)[k_0, ..,k_{i-1}, k_{i+1}, .., k_{M-1}]` =
+        the result of iterating `j` over :math:`range(N_i)`, cumulatively applying
+        ufunc to each :math:`a[k_0, ..,k_{i-1}, j, k_{i+1}, .., k_{M-1}]`.
+        For a one-dimensional array, reduce produces results equivalent to:
+        ::
+
+         r = op.identity # op = ufunc
+         for i in xrange(len(A)):
+           r = op(r, A[i])
+         return r
+
+        For example, add.reduce() is equivalent to sum().
+
+        Parameters
+        ----------
+        a : array_like
+            The array to act on.
+        axis : int, optional
+            The axis along which to apply the reduction.
+
+        Examples
+        --------
+        >>> np.multiply.reduce([2,3,5])
+        30
+
+        A multi-dimensional array example:
+
+        >>> X = np.arange(8).reshape((2,2,2))
+        >>> X
+        array([[[0, 1],
+                [2, 3]],
+               [[4, 5],
+                [6, 7]]])
+        >>> np.add.reduce(X, 0)
+        array([[ 4,  6],
+               [ 8, 10]])
+        >>> np.add.reduce(X) # confirm: default axis value is 0
+        array([[ 4,  6],
+               [ 8, 10]])
+        >>> np.add.reduce(X, 1)
+        array([[ 2,  4],
+               [10, 12]])
+        >>> np.add.reduce(X, 2)
+        array([[ 1,  5],
+               [ 9, 13]])
+        """
+        return self.reduce(space, w_obj, False, False, w_dim)
+
+    def reduce(self, space, w_obj, multidim, promote_to_largest, w_dim):
+        from pypy.module.micronumpy.interp_numarray import convert_to_array, \
+                                                           Scalar
         if self.argcount != 2:
             raise OperationError(space.w_ValueError, space.wrap("reduce only "
                 "supported for binary functions"))
-
+        dim = space.int_w(w_dim)
         assert isinstance(self, W_Ufunc2)
         obj = convert_to_array(space, w_obj)
+        if dim >= len(obj.shape):
+            raise OperationError(space.w_ValueError, space.wrap("axis(=%d) out of bounds" % dim))
         if isinstance(obj, Scalar):
             raise OperationError(space.w_TypeError, space.wrap("cannot reduce "
                 "on a scalar"))
@@ -67,26 +131,80 @@
         size = obj.size
         dtype = find_unaryop_result_dtype(
             space, obj.find_dtype(),
-            promote_to_largest=True
+            promote_to_float=self.promote_to_float,
+            promote_to_largest=promote_to_largest,
+            promote_bools=True
         )
         shapelen = len(obj.shape)
+        if self.identity is None and size == 0:
+            raise operationerrfmt(space.w_ValueError, "zero-size array to "
+                    "%s.reduce without identity", self.name)
+        if shapelen > 1 and dim >= 0:
+            res = self.do_axis_reduce(obj, dtype, dim)
+            return space.wrap(res)
+        scalarsig = ScalarSignature(dtype)
         sig = find_sig(ReduceSignature(self.func, self.name, dtype,
-                                       ScalarSignature(dtype),
-                                       obj.create_sig(obj.shape)), obj)
+                                       scalarsig,
+                                       obj.create_sig()), obj)
         frame = sig.create_frame(obj)
-        if shapelen > 1 and not multidim:
-            raise OperationError(space.w_NotImplementedError,
-                space.wrap("not implemented yet"))
         if self.identity is None:
-            if size == 0:
-                raise operationerrfmt(space.w_ValueError, "zero-size array to "
-                    "%s.reduce without identity", self.name)
             value = sig.eval(frame, obj).convert_to(dtype)
             frame.next(shapelen)
         else:
             value = self.identity.convert_to(dtype)
         return self.reduce_loop(shapelen, sig, frame, value, obj, dtype)
 
+    def do_axis_reduce(self, obj, dtype, dim):
+        from pypy.module.micronumpy.interp_numarray import AxisReduce,\
+             W_NDimArray
+        
+        shape = obj.shape[0:dim] + obj.shape[dim + 1:len(obj.shape)]
+        size = 1
+        for s in shape:
+            size *= s
+        result = W_NDimArray(size, shape, dtype)
+        rightsig = obj.create_sig()
+        # note - this is just a wrapper so signature can fetch
+        #        both left and right, nothing more, especially
+        #        this is not a true virtual array, because shapes
+        #        don't quite match
+        arr = AxisReduce(self.func, self.name, obj.shape, dtype,
+                         result, obj, dim)
+        scalarsig = ScalarSignature(dtype)
+        sig = find_sig(AxisReduceSignature(self.func, self.name, dtype,
+                                           scalarsig, rightsig), arr)
+        assert isinstance(sig, AxisReduceSignature)
+        frame = sig.create_frame(arr)
+        shapelen = len(obj.shape)
+        if self.identity is not None:
+            identity = self.identity.convert_to(dtype)
+        else:
+            identity = None
+        self.reduce_axis_loop(frame, sig, shapelen, arr, identity)
+        return result
+
+    def reduce_axis_loop(self, frame, sig, shapelen, arr, identity):
+        # note - we can be advanterous here, depending on the exact field
+        # layout. For now let's say we iterate the original way and
+        # simply follow the original iteration order
+        while not frame.done():
+            axisreduce_driver.jit_merge_point(frame=frame, self=self,
+                                              sig=sig,
+                                              identity=identity,
+                                              shapelen=shapelen, arr=arr)
+            iter = frame.get_final_iter()
+            v = sig.eval(frame, arr).convert_to(sig.calc_dtype)
+            if iter.first_line:
+                if identity is not None:
+                    value = self.func(sig.calc_dtype, identity, v)
+                else:
+                    value = v
+            else:
+                cur = arr.left.getitem(iter.offset)
+                value = self.func(sig.calc_dtype, cur, v)
+            arr.left.setitem(iter.offset, value)
+            frame.next(shapelen)
+
     def reduce_loop(self, shapelen, sig, frame, value, obj, dtype):
         while not frame.done():
             reduce_driver.jit_merge_point(sig=sig,
@@ -94,10 +212,12 @@
                                           value=value, obj=obj, frame=frame,
                                           dtype=dtype)
             assert isinstance(sig, ReduceSignature)
-            value = sig.binfunc(dtype, value, sig.eval(frame, obj).convert_to(dtype))
+            value = sig.binfunc(dtype, value,
+                                sig.eval(frame, obj).convert_to(dtype))
             frame.next(shapelen)
         return value
 
+
 class W_Ufunc1(W_Ufunc):
     argcount = 1
 
@@ -182,6 +302,7 @@
     reduce = interp2app(W_Ufunc.descr_reduce),
 )
 
+
 def find_binop_result_dtype(space, dt1, dt2, promote_to_float=False,
     promote_bools=False):
     # dt1.num should be <= dt2.num
@@ -230,6 +351,7 @@
             dtypenum += 3
         return interp_dtype.get_dtype_cache(space).builtin_dtypes[dtypenum]
 
+
 def find_unaryop_result_dtype(space, dt, promote_to_float=False,
     promote_bools=False, promote_to_largest=False):
     if promote_bools and (dt.kind == interp_dtype.BOOLLTR):
@@ -254,6 +376,7 @@
             assert False
     return dt
 
+
 def find_dtype_for_scalar(space, w_obj, current_guess=None):
     bool_dtype = interp_dtype.get_dtype_cache(space).w_booldtype
     long_dtype = interp_dtype.get_dtype_cache(space).w_longdtype
@@ -347,7 +470,8 @@
 
         identity = extra_kwargs.get("identity")
         if identity is not None:
-            identity = interp_dtype.get_dtype_cache(space).w_longdtype.box(identity)
+            identity = \
+                 interp_dtype.get_dtype_cache(space).w_longdtype.box(identity)
         extra_kwargs["identity"] = identity
 
         func = ufunc_dtype_caller(space, ufunc_name, op_name, argcount,
diff --git a/pypy/module/micronumpy/signature.py b/pypy/module/micronumpy/signature.py
--- a/pypy/module/micronumpy/signature.py
+++ b/pypy/module/micronumpy/signature.py
@@ -1,10 +1,32 @@
 from pypy.rlib.objectmodel import r_dict, compute_identity_hash, compute_hash
 from pypy.rlib.rarithmetic import intmask
 from pypy.module.micronumpy.interp_iter import ViewIterator, ArrayIterator, \
-     OneDimIterator, ConstantIterator
-from pypy.module.micronumpy.strides import calculate_slice_strides
+     ConstantIterator, AxisIterator, ViewTransform,\
+     BroadcastTransform
 from pypy.rlib.jit import hint, unroll_safe, promote
 
+""" Signature specifies both the numpy expression that has been constructed
+and the assembler to be compiled. This is a very important observation -
+Two expressions will be using the same assembler if and only if they are
+compiled to the same signature.
+
+This is also a very convinient tool for specializations. For example
+a + a and a + b (where a != b) will compile to different assembler because
+we specialize on the same array access.
+
+When evaluating, signatures will create iterators per signature node,
+potentially sharing some of them. Iterators depend also on the actual
+expression, they're not only dependant on the array itself. For example
+a + b where a is dim 2 and b is dim 1 would create a broadcasted iterator for
+the array b.
+
+Such iterator changes are called Transformations. An actual iterator would
+be a combination of array and various transformation, like view, broadcast,
+dimension swapping etc.
+
+See interp_iter for transformations
+"""
+
 def new_printable_location(driver_name):
     def get_printable_location(shapelen, sig):
         return 'numpy ' + sig.debug_repr() + ' [%d dims,%s]' % (shapelen, driver_name)
@@ -33,7 +55,8 @@
         return sig
 
 class NumpyEvalFrame(object):
-    _virtualizable2_ = ['iterators[*]', 'final_iter', 'arraylist[*]']
+    _virtualizable2_ = ['iterators[*]', 'final_iter', 'arraylist[*]',
+                        'value', 'identity']
 
     @unroll_safe
     def __init__(self, iterators, arrays):
@@ -51,7 +74,7 @@
     def done(self):
         final_iter = promote(self.final_iter)
         if final_iter < 0:
-            return False
+            assert False
         return self.iterators[final_iter].done()
 
     @unroll_safe
@@ -59,6 +82,12 @@
         for i in range(len(self.iterators)):
             self.iterators[i] = self.iterators[i].next(shapelen)
 
+    def get_final_iter(self):
+        final_iter = promote(self.final_iter)
+        if final_iter < 0:
+            assert False
+        return self.iterators[final_iter]
+
 def _add_ptr_to_cache(ptr, cache):
     i = 0
     for p in cache:
@@ -70,6 +99,9 @@
         cache.append(ptr)
         return res
 
+def new_cache():
+    return r_dict(sigeq_no_numbering, sighash)
+
 class Signature(object):
     _attrs_ = ['iter_no', 'array_no']
     _immutable_fields_ = ['iter_no', 'array_no']
@@ -78,7 +110,7 @@
     iter_no = 0
 
     def invent_numbering(self):
-        cache = r_dict(sigeq_no_numbering, sighash)
+        cache = new_cache()
         allnumbers = []
         self._invent_numbering(cache, allnumbers)
 
@@ -95,13 +127,13 @@
             allnumbers.append(no)
         self.iter_no = no
 
-    def create_frame(self, arr, res_shape=None):
-        res_shape = res_shape or arr.shape
+    def create_frame(self, arr):
         iterlist = []
         arraylist = []
-        self._create_iter(iterlist, arraylist, arr, res_shape, [])
+        self._create_iter(iterlist, arraylist, arr, [])
         return NumpyEvalFrame(iterlist, arraylist)
 
+
 class ConcreteSignature(Signature):
     _immutable_fields_ = ['dtype']
 
@@ -120,16 +152,6 @@
     def hash(self):
         return compute_identity_hash(self.dtype)
 
-    def allocate_view_iter(self, arr, res_shape, chunklist):
-        r = arr.shape, arr.start, arr.strides, arr.backstrides
-        if chunklist:
-            for chunkelem in chunklist:
-                r = calculate_slice_strides(r[0], r[1], r[2], r[3], chunkelem)
-        shape, start, strides, backstrides = r
-        if len(res_shape) == 1:
-            return OneDimIterator(start, strides[0], res_shape[0])
-        return ViewIterator(start, strides, backstrides, shape, res_shape)
-
 class ArraySignature(ConcreteSignature):
     def debug_repr(self):
         return 'Array'
@@ -137,23 +159,25 @@
     def _invent_array_numbering(self, arr, cache):
         from pypy.module.micronumpy.interp_numarray import ConcreteArray
         concr = arr.get_concrete()
+        # this get_concrete never forces assembler. If we're here and array
+        # is not of a concrete class it means that we have a _forced_result,
+        # otherwise the signature would not match
         assert isinstance(concr, ConcreteArray)
+        assert concr.dtype is self.dtype
         self.array_no = _add_ptr_to_cache(concr.storage, cache)
 
-    def _create_iter(self, iterlist, arraylist, arr, res_shape, chunklist):
+    def _create_iter(self, iterlist, arraylist, arr, transforms):
         from pypy.module.micronumpy.interp_numarray import ConcreteArray
         concr = arr.get_concrete()
         assert isinstance(concr, ConcreteArray)
         storage = concr.storage
         if self.iter_no >= len(iterlist):
-            iterlist.append(self.allocate_iter(concr, res_shape, chunklist))
+            iterlist.append(self.allocate_iter(concr, transforms))
         if self.array_no >= len(arraylist):
             arraylist.append(storage)
 
-    def allocate_iter(self, arr, res_shape, chunklist):
-        if chunklist:
-            return self.allocate_view_iter(arr, res_shape, chunklist)
-        return ArrayIterator(arr.size)
+    def allocate_iter(self, arr, transforms):
+        return ArrayIterator(arr.size).apply_transformations(arr, transforms)
 
     def eval(self, frame, arr):
         iter = frame.iterators[self.iter_no]
@@ -166,7 +190,7 @@
     def _invent_array_numbering(self, arr, cache):
         pass
 
-    def _create_iter(self, iterlist, arraylist, arr, res_shape, chunklist):
+    def _create_iter(self, iterlist, arraylist, arr, transforms):
         if self.iter_no >= len(iterlist):
             iter = ConstantIterator()
             iterlist.append(iter)
@@ -186,8 +210,9 @@
         allnumbers.append(no)
         self.iter_no = no
 
-    def allocate_iter(self, arr, res_shape, chunklist):
-        return self.allocate_view_iter(arr, res_shape, chunklist)
+    def allocate_iter(self, arr, transforms):
+        return ViewIterator(arr.start, arr.strides, arr.backstrides,
+                            arr.shape).apply_transformations(arr, transforms)
 
 class VirtualSliceSignature(Signature):
     def __init__(self, child):
@@ -198,6 +223,9 @@
         assert isinstance(arr, VirtualSlice)
         self.child._invent_array_numbering(arr.child, cache)
 
+    def _invent_numbering(self, cache, allnumbers):
+        self.child._invent_numbering(new_cache(), allnumbers)
+
     def hash(self):
         return intmask(self.child.hash() ^ 1234)
 
@@ -207,12 +235,11 @@
         assert isinstance(other, VirtualSliceSignature)
         return self.child.eq(other.child, compare_array_no)
 
-    def _create_iter(self, iterlist, arraylist, arr, res_shape, chunklist):
+    def _create_iter(self, iterlist, arraylist, arr, transforms):
         from pypy.module.micronumpy.interp_numarray import VirtualSlice
         assert isinstance(arr, VirtualSlice)
-        chunklist.append(arr.chunks)
-        self.child._create_iter(iterlist, arraylist, arr.child, res_shape,
-                                chunklist)
+        transforms = transforms + [ViewTransform(arr.chunks)]
+        self.child._create_iter(iterlist, arraylist, arr.child, transforms)
 
     def eval(self, frame, arr):
         from pypy.module.micronumpy.interp_numarray import VirtualSlice
@@ -248,11 +275,10 @@
         assert isinstance(arr, Call1)
         self.child._invent_array_numbering(arr.values, cache)
 
-    def _create_iter(self, iterlist, arraylist, arr, res_shape, chunklist):
+    def _create_iter(self, iterlist, arraylist, arr, transforms):
         from pypy.module.micronumpy.interp_numarray import Call1
         assert isinstance(arr, Call1)
-        self.child._create_iter(iterlist, arraylist, arr.values, res_shape,
-                                chunklist)
+        self.child._create_iter(iterlist, arraylist, arr.values, transforms)
 
     def eval(self, frame, arr):
         from pypy.module.micronumpy.interp_numarray import Call1
@@ -293,29 +319,68 @@
         self.left._invent_numbering(cache, allnumbers)
         self.right._invent_numbering(cache, allnumbers)
 
-    def _create_iter(self, iterlist, arraylist, arr, res_shape, chunklist):
+    def _create_iter(self, iterlist, arraylist, arr, transforms):
         from pypy.module.micronumpy.interp_numarray import Call2
 
         assert isinstance(arr, Call2)
-        self.left._create_iter(iterlist, arraylist, arr.left, res_shape,
-                               chunklist)
-        self.right._create_iter(iterlist, arraylist, arr.right, res_shape,
-                                chunklist)
+        self.left._create_iter(iterlist, arraylist, arr.left, transforms)
+        self.right._create_iter(iterlist, arraylist, arr.right, transforms)
 
     def eval(self, frame, arr):
         from pypy.module.micronumpy.interp_numarray import Call2
         assert isinstance(arr, Call2)
         lhs = self.left.eval(frame, arr.left).convert_to(self.calc_dtype)
         rhs = self.right.eval(frame, arr.right).convert_to(self.calc_dtype)
+        
         return self.binfunc(self.calc_dtype, lhs, rhs)
 
     def debug_repr(self):
         return 'Call2(%s, %s, %s)' % (self.name, self.left.debug_repr(),
                                       self.right.debug_repr())
 
+class BroadcastLeft(Call2):
+    def _invent_numbering(self, cache, allnumbers):
+        self.left._invent_numbering(new_cache(), allnumbers)
+        self.right._invent_numbering(cache, allnumbers)
+    
+    def _create_iter(self, iterlist, arraylist, arr, transforms):
+        from pypy.module.micronumpy.interp_numarray import Call2
+
+        assert isinstance(arr, Call2)
+        ltransforms = transforms + [BroadcastTransform(arr.shape)]
+        self.left._create_iter(iterlist, arraylist, arr.left, ltransforms)
+        self.right._create_iter(iterlist, arraylist, arr.right, transforms)
+
+class BroadcastRight(Call2):
+    def _invent_numbering(self, cache, allnumbers):
+        self.left._invent_numbering(cache, allnumbers)
+        self.right._invent_numbering(new_cache(), allnumbers)
+
+    def _create_iter(self, iterlist, arraylist, arr, transforms):
+        from pypy.module.micronumpy.interp_numarray import Call2
+
+        assert isinstance(arr, Call2)
+        rtransforms = transforms + [BroadcastTransform(arr.shape)]
+        self.left._create_iter(iterlist, arraylist, arr.left, transforms)
+        self.right._create_iter(iterlist, arraylist, arr.right, rtransforms)
+
+class BroadcastBoth(Call2):
+    def _invent_numbering(self, cache, allnumbers):
+        self.left._invent_numbering(new_cache(), allnumbers)
+        self.right._invent_numbering(new_cache(), allnumbers)
+
+    def _create_iter(self, iterlist, arraylist, arr, transforms):
+        from pypy.module.micronumpy.interp_numarray import Call2
+
+        assert isinstance(arr, Call2)
+        rtransforms = transforms + [BroadcastTransform(arr.shape)]
+        ltransforms = transforms + [BroadcastTransform(arr.shape)]
+        self.left._create_iter(iterlist, arraylist, arr.left, ltransforms)
+        self.right._create_iter(iterlist, arraylist, arr.right, rtransforms)
+
 class ReduceSignature(Call2):
-    def _create_iter(self, iterlist, arraylist, arr, res_shape, chunklist):
-        self.right._create_iter(iterlist, arraylist, arr, res_shape, chunklist)
+    def _create_iter(self, iterlist, arraylist, arr, transforms):
+        self.right._create_iter(iterlist, arraylist, arr, transforms)
 
     def _invent_numbering(self, cache, allnumbers):
         self.right._invent_numbering(cache, allnumbers)
@@ -325,3 +390,63 @@
 
     def eval(self, frame, arr):
         return self.right.eval(frame, arr)
+
+    def debug_repr(self):
+        return 'ReduceSig(%s, %s)' % (self.name, self.right.debug_repr())
+
+class SliceloopSignature(Call2):
+    def eval(self, frame, arr):
+        from pypy.module.micronumpy.interp_numarray import Call2
+        
+        assert isinstance(arr, Call2)
+        ofs = frame.iterators[0].offset
+        arr.left.setitem(ofs, self.right.eval(frame, arr.right).convert_to(
+            self.calc_dtype))
+    
+    def debug_repr(self):
+        return 'SliceLoop(%s, %s, %s)' % (self.name, self.left.debug_repr(),
+                                          self.right.debug_repr())
+
+class SliceloopBroadcastSignature(SliceloopSignature):
+    def _invent_numbering(self, cache, allnumbers):
+        self.left._invent_numbering(new_cache(), allnumbers)
+        self.right._invent_numbering(cache, allnumbers)
+
+    def _create_iter(self, iterlist, arraylist, arr, transforms):
+        from pypy.module.micronumpy.interp_numarray import SliceArray
+
+        assert isinstance(arr, SliceArray)
+        rtransforms = transforms + [BroadcastTransform(arr.shape)]
+        self.left._create_iter(iterlist, arraylist, arr.left, transforms)
+        self.right._create_iter(iterlist, arraylist, arr.right, rtransforms)
+
+class AxisReduceSignature(Call2):
+    def _create_iter(self, iterlist, arraylist, arr, transforms):
+        from pypy.module.micronumpy.interp_numarray import AxisReduce,\
+             ConcreteArray
+
+        assert isinstance(arr, AxisReduce)
+        left = arr.left
+        assert isinstance(left, ConcreteArray)
+        iterlist.append(AxisIterator(left.start, arr.dim, arr.shape,
+                                     left.strides, left.backstrides))
+        self.right._create_iter(iterlist, arraylist, arr.right, transforms)
+
+    def _invent_numbering(self, cache, allnumbers):
+        allnumbers.append(0)
+        self.right._invent_numbering(cache, allnumbers)
+
+    def _invent_array_numbering(self, arr, cache):
+        from pypy.module.micronumpy.interp_numarray import AxisReduce
+
+        assert isinstance(arr, AxisReduce)
+        self.right._invent_array_numbering(arr.right, cache)
+
+    def eval(self, frame, arr):
+        from pypy.module.micronumpy.interp_numarray import AxisReduce
+
+        assert isinstance(arr, AxisReduce)
+        return self.right.eval(frame, arr.right).convert_to(self.calc_dtype)
+    
+    def debug_repr(self):
+        return 'AxisReduceSig(%s, %s)' % (self.name, self.right.debug_repr())
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -3,7 +3,7 @@
 
 class AppTestDtypes(BaseNumpyAppTest):
     def test_dtype(self):
-        from numpypy import dtype
+        from _numpypy import dtype
 
         d = dtype('?')
         assert d.num == 0
@@ -14,7 +14,7 @@
         raises(TypeError, dtype, 1042)
 
     def test_dtype_with_types(self):
-        from numpypy import dtype
+        from _numpypy import dtype
 
         assert dtype(bool).num == 0
         assert dtype(int).num == 7
@@ -22,13 +22,13 @@
         assert dtype(float).num == 12
 
     def test_array_dtype_attr(self):
-        from numpypy import array, dtype
+        from _numpypy import array, dtype
 
         a = array(range(5), long)
         assert a.dtype is dtype(long)
 
     def test_repr_str(self):
-        from numpypy import dtype
+        from _numpypy import dtype
 
         assert repr(dtype) == ""
         d = dtype('?')
@@ -36,7 +36,7 @@
         assert str(d) == "bool"
 
     def test_bool_array(self):
-        from numpypy import array, False_, True_
+        from _numpypy import array, False_, True_
 
         a = array([0, 1, 2, 2.5], dtype='?')
         assert a[0] is False_
@@ -44,7 +44,7 @@
             assert a[i] is True_
 
     def test_copy_array_with_dtype(self):
-        from numpypy import array, False_, True_, int64
+        from _numpypy import array, False_, True_, int64
 
         a = array([0, 1, 2, 3], dtype=long)
         # int on 64-bit, long in 32-bit
@@ -58,35 +58,35 @@
         assert b[0] is False_
 
     def test_zeros_bool(self):
-        from numpypy import zeros, False_
+        from _numpypy import zeros, False_
 
         a = zeros(10, dtype=bool)
         for i in range(10):
             assert a[i] is False_
 
     def test_ones_bool(self):
-        from numpypy import ones, True_
+        from _numpypy import ones, True_
 
         a = ones(10, dtype=bool)
         for i in range(10):
             assert a[i] is True_
 
     def test_zeros_long(self):
-        from numpypy import zeros, int64
+        from _numpypy import zeros, int64
         a = zeros(10, dtype=long)
         for i in range(10):
             assert isinstance(a[i], int64)
             assert a[1] == 0
 
     def test_ones_long(self):
-        from numpypy import ones, int64
+        from _numpypy import ones, int64
         a = ones(10, dtype=long)
         for i in range(10):
             assert isinstance(a[i], int64)
             assert a[1] == 1
 
     def test_overflow(self):
-        from numpypy import array, dtype
+        from _numpypy import array, dtype
         assert array([128], 'b')[0] == -128
         assert array([256], 'B')[0] == 0
         assert array([32768], 'h')[0] == -32768
@@ -98,7 +98,7 @@
         raises(OverflowError, "array([2**64], 'Q')")
 
     def test_bool_binop_types(self):
-        from numpypy import array, dtype
+        from _numpypy import array, dtype
         types = [
             '?', 'b', 'B', 'h', 'H', 'i', 'I', 'l', 'L', 'q', 'Q', 'f', 'd'
         ]
@@ -107,7 +107,7 @@
             assert (a + array([0], t)).dtype is dtype(t)
 
     def test_binop_types(self):
-        from numpypy import array, dtype
+        from _numpypy import array, dtype
         tests = [('b','B','h'), ('b','h','h'), ('b','H','i'), ('b','i','i'),
                  ('b','l','l'), ('b','q','q'), ('b','Q','d'), ('B','h','h'),
                  ('B','H','H'), ('B','i','i'), ('B','I','I'), ('B','l','l'),
@@ -129,7 +129,7 @@
             assert (array([1], d1) + array([1], d2)).dtype is dtype(dout)
 
     def test_add_int8(self):
-        from numpypy import array, dtype
+        from _numpypy import array, dtype
 
         a = array(range(5), dtype="int8")
         b = a + a
@@ -138,7 +138,7 @@
             assert b[i] == i * 2
 
     def test_add_int16(self):
-        from numpypy import array, dtype
+        from _numpypy import array, dtype
 
         a = array(range(5), dtype="int16")
         b = a + a
@@ -147,7 +147,7 @@
             assert b[i] == i * 2
 
     def test_add_uint32(self):
-        from numpypy import array, dtype
+        from _numpypy import array, dtype
 
         a = array(range(5), dtype="I")
         b = a + a
@@ -156,19 +156,28 @@
             assert b[i] == i * 2
 
     def test_shape(self):
-        from numpypy import dtype
+        from _numpypy import dtype
 
         assert dtype(long).shape == ()
 
     def test_cant_subclass(self):
-        from numpypy import dtype
+        from _numpypy import dtype
 
         # You can't subclass dtype
         raises(TypeError, type, "Foo", (dtype,), {})
 
+    def test_new(self):
+        import _numpypy as np
+        assert np.int_(4) == 4
+        assert np.float_(3.4) == 3.4
+
+    def test_pow(self):
+        from _numpypy import int_
+        assert int_(4) ** 2 == 16
+
 class AppTestTypes(BaseNumpyAppTest):
     def test_abstract_types(self):
-        import numpypy as numpy
+        import _numpypy as numpy
         raises(TypeError, numpy.generic, 0)
         raises(TypeError, numpy.number, 0)
         raises(TypeError, numpy.integer, 0)
@@ -181,7 +190,7 @@
         raises(TypeError, numpy.inexact, 0)
 
     def test_bool(self):
-        import numpypy as numpy
+        import _numpypy as numpy
 
         assert numpy.bool_.mro() == [numpy.bool_, numpy.generic, object]
         assert numpy.bool_(3) is numpy.True_
@@ -196,7 +205,7 @@
         assert numpy.bool_("False") is numpy.True_
 
     def test_int8(self):
-        import numpypy as numpy
+        import _numpypy as numpy
 
         assert numpy.int8.mro() == [numpy.int8, numpy.signedinteger, numpy.integer, numpy.number, numpy.generic, object]
 
@@ -218,7 +227,7 @@
         assert numpy.int8('128') == -128
 
     def test_uint8(self):
-        import numpypy as numpy
+        import _numpypy as numpy
 
         assert numpy.uint8.mro() == [numpy.uint8, numpy.unsignedinteger, numpy.integer, numpy.number, numpy.generic, object]
 
@@ -241,7 +250,7 @@
         assert numpy.uint8('256') == 0
 
     def test_int16(self):
-        import numpypy as numpy
+        import _numpypy as numpy
 
         x = numpy.int16(3)
         assert x == 3
@@ -251,7 +260,7 @@
         assert numpy.int16('32768') == -32768
 
     def test_uint16(self):
-        import numpypy as numpy
+        import _numpypy as numpy
 
         assert numpy.uint16(65535) == 65535
         assert numpy.uint16(65536) == 0
@@ -260,7 +269,7 @@
 
     def test_int32(self):
         import sys
-        import numpypy as numpy
+        import _numpypy as numpy
 
         x = numpy.int32(23)
         assert x == 23
@@ -275,7 +284,7 @@
 
     def test_uint32(self):
         import sys
-        import numpypy as numpy
+        import _numpypy as numpy
 
         assert numpy.uint32(10) == 10
 
@@ -286,14 +295,14 @@
             assert numpy.uint32('4294967296') == 0
 
     def test_int_(self):
-        import numpypy as numpy
+        import _numpypy as numpy
 
         assert numpy.int_ is numpy.dtype(int).type
         assert numpy.int_.mro() == [numpy.int_, numpy.signedinteger, numpy.integer, numpy.number, numpy.generic, int, object]
 
     def test_int64(self):
         import sys
-        import numpypy as numpy
+        import _numpypy as numpy
 
         if sys.maxint == 2 ** 63 -1:
             assert numpy.int64.mro() == [numpy.int64, numpy.signedinteger, numpy.integer, numpy.number, numpy.generic, int, object]
@@ -315,7 +324,7 @@
 
     def test_uint64(self):
         import sys
-        import numpypy as numpy
+        import _numpypy as numpy
 
         assert numpy.uint64.mro() == [numpy.uint64, numpy.unsignedinteger, numpy.integer, numpy.number, numpy.generic, object]
 
@@ -330,7 +339,7 @@
         raises(OverflowError, numpy.uint64(18446744073709551616))
 
     def test_float32(self):
-        import numpypy as numpy
+        import _numpypy as numpy
 
         assert numpy.float32.mro() == [numpy.float32, numpy.floating, numpy.inexact, numpy.number, numpy.generic, object]
 
@@ -339,7 +348,7 @@
         raises(ValueError, numpy.float32, '23.2df')
 
     def test_float64(self):
-        import numpypy as numpy
+        import _numpypy as numpy
 
         assert numpy.float64.mro() == [numpy.float64, numpy.floating, numpy.inexact, numpy.number, numpy.generic, float, object]
 
@@ -352,7 +361,7 @@
         raises(ValueError, numpy.float64, '23.2df')
 
     def test_subclass_type(self):
-        import numpypy as numpy
+        import _numpypy as numpy
 
         class X(numpy.float64):
             def m(self):
diff --git a/pypy/module/micronumpy/test/test_module.py b/pypy/module/micronumpy/test/test_module.py
--- a/pypy/module/micronumpy/test/test_module.py
+++ b/pypy/module/micronumpy/test/test_module.py
@@ -2,34 +2,29 @@
 
 
 class AppTestNumPyModule(BaseNumpyAppTest):
-    def test_mean(self):
-        from numpypy import array, mean
-        assert mean(array(range(5))) == 2.0
-        assert mean(range(5)) == 2.0
-
     def test_average(self):
-        from numpypy import array, average
+        from _numpypy import array, average
         assert average(range(10)) == 4.5
         assert average(array(range(10))) == 4.5
-        
+
     def test_sum(self):
-        from numpypy import array, sum
+        from _numpypy import array, sum
         assert sum(range(10)) == 45
         assert sum(array(range(10))) == 45
 
     def test_min(self):
-        from numpypy import array, min
+        from _numpypy import array, min
         assert min(range(10)) == 0
         assert min(array(range(10))) == 0
-        
+
     def test_max(self):
-        from numpypy import array, max
+        from _numpypy import array, max
         assert max(range(10)) == 9
         assert max(array(range(10))) == 9
 
     def test_constants(self):
         import math
-        from numpypy import inf, e, pi
+        from _numpypy import inf, e, pi
         assert type(inf) is float
         assert inf == float("inf")
         assert e == math.e
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -157,10 +157,13 @@
         assert calc_new_strides([2, 3, 4], [8, 3], [1, 16]) is None
         assert calc_new_strides([24], [2, 4, 3], [48, 6, 1]) is None
         assert calc_new_strides([24], [2, 4, 3], [24, 6, 2]) == [2]
+        assert calc_new_strides([105, 1], [3, 5, 7], [35, 7, 1]) == [1, 1]
+        assert calc_new_strides([1, 105], [3, 5, 7], [35, 7, 1]) == [105, 1]
+
 
 class AppTestNumArray(BaseNumpyAppTest):
     def test_ndarray(self):
-        from numpypy import ndarray, array, dtype
+        from _numpypy import ndarray, array, dtype
 
         assert type(ndarray) is type
         assert type(array) is not type
@@ -175,12 +178,26 @@
         assert a.dtype is dtype(int)
 
     def test_type(self):
-        from numpypy import array
+        from _numpypy import array
         ar = array(range(5))
         assert type(ar) is type(ar + ar)
 
+    def test_ndim(self):
+        from _numpypy import array
+        x = array(0.2)
+        assert x.ndim == 0
+        x = array([1, 2])
+        assert x.ndim == 1
+        x = array([[1, 2], [3, 4]])
+        assert x.ndim == 2
+        x = array([[[1, 2], [3, 4]], [[5, 6], [7, 8]]])
+        assert x.ndim == 3
+        # numpy actually raises an AttributeError, but _numpypy raises an
+        # TypeError
+        raises(TypeError, 'x.ndim = 3')
+
     def test_init(self):
-        from numpypy import zeros
+        from _numpypy import zeros
         a = zeros(15)
         # Check that storage was actually zero'd.
         assert a[10] == 0.0
@@ -189,7 +206,7 @@
         assert a[13] == 5.3
 
     def test_size(self):
-        from numpypy import array
+        from _numpypy import array
         assert array(3).size == 1
         a = array([1, 2, 3])
         assert a.size == 3
@@ -200,13 +217,13 @@
         Test that empty() works.
         """
 
-        from numpypy import empty
+        from _numpypy import empty
         a = empty(2)
         a[1] = 1.0
         assert a[1] == 1.0
 
     def test_ones(self):
-        from numpypy import ones
+        from _numpypy import ones
         a = ones(3)
         assert len(a) == 3
         assert a[0] == 1
@@ -215,7 +232,7 @@
         assert a[2] == 4
 
     def test_copy(self):
-        from numpypy import arange, array
+        from _numpypy import arange, array
         a = arange(5)
         b = a.copy()
         for i in xrange(5):
@@ -231,13 +248,17 @@
         c = b.copy()
         assert (c == b).all()
 
+        a = arange(15).reshape(5,3)
+        b = a.copy()
+        assert (b == a).all()
+
     def test_iterator_init(self):
-        from numpypy import array
+        from _numpypy import array
         a = array(range(5))
         assert a[3] == 3
 
     def test_getitem(self):
-        from numpypy import array
+        from _numpypy import array
         a = array(range(5))
         raises(IndexError, "a[5]")
         a = a + a
@@ -246,7 +267,7 @@
         raises(IndexError, "a[-6]")
 
     def test_getitem_tuple(self):
-        from numpypy import array
+        from _numpypy import array
         a = array(range(5))
         raises(IndexError, "a[(1,2)]")
         for i in xrange(5):
@@ -256,7 +277,7 @@
             assert a[i] == b[i]
 
     def test_setitem(self):
-        from numpypy import array
+        from _numpypy import array
         a = array(range(5))
         a[-1] = 5.0
         assert a[4] == 5.0
@@ -264,7 +285,7 @@
         raises(IndexError, "a[-6] = 3.0")
 
     def test_setitem_tuple(self):
-        from numpypy import array
+        from _numpypy import array
         a = array(range(5))
         raises(IndexError, "a[(1,2)] = [0,1]")
         for i in xrange(5):
@@ -275,7 +296,7 @@
             assert a[i] == i
 
     def test_setslice_array(self):
-        from numpypy import array
+        from _numpypy import array
         a = array(range(5))
         b = array(range(2))
         a[1:4:2] = b
@@ -286,7 +307,7 @@
         assert b[1] == 0.
 
     def test_setslice_of_slice_array(self):
-        from numpypy import array, zeros
+        from _numpypy import array, zeros
         a = zeros(5)
         a[::2] = array([9., 10., 11.])
         assert a[0] == 9.
@@ -305,7 +326,7 @@
         assert a[0] == 3.
 
     def test_setslice_list(self):
-        from numpypy import array
+        from _numpypy import array
         a = array(range(5), float)
         b = [0., 1.]
         a[1:4:2] = b
@@ -313,14 +334,14 @@
         assert a[3] == 1.
 
     def test_setslice_constant(self):
-        from numpypy import array
+        from _numpypy import array
         a = array(range(5), float)
         a[1:4:2] = 0.
         assert a[1] == 0.
         assert a[3] == 0.
 
     def test_scalar(self):
-        from numpypy import array, dtype
+        from _numpypy import array, dtype
         a = array(3)
         raises(IndexError, "a[0]")
         raises(IndexError, "a[0] = 5")
@@ -329,13 +350,13 @@
         assert a.dtype is dtype(int)
 
     def test_len(self):
-        from numpypy import array
+        from _numpypy import array
         a = array(range(5))
         assert len(a) == 5
         assert len(a + a) == 5
 
     def test_shape(self):
-        from numpypy import array
+        from _numpypy import array
         a = array(range(5))
         assert a.shape == (5,)
         b = a + a
@@ -344,7 +365,7 @@
         assert c.shape == (3,)
 
     def test_set_shape(self):
-        from numpypy import array, zeros
+        from _numpypy import array, zeros
         a = array([])
         a.shape = []
         a = array(range(12))
@@ -364,7 +385,7 @@
         a.shape = (1,)
 
     def test_reshape(self):
-        from numpypy import array, zeros
+        from _numpypy import array, zeros
         a = array(range(12))
         exc = raises(ValueError, "b = a.reshape((3, 10))")
         assert str(exc.value) == "total size of new array must be unchanged"
@@ -377,7 +398,7 @@
         a.shape = (12, 2)
 
     def test_slice_reshape(self):
-        from numpypy import zeros, arange
+        from _numpypy import zeros, arange
         a = zeros((4, 2, 3))
         b = a[::2, :, :]
         b.shape = (2, 6)
@@ -413,13 +434,13 @@
         raises(ValueError, arange(10).reshape, (5, -1, -1))
 
     def test_reshape_varargs(self):
-        from numpypy import arange
+        from _numpypy import arange
         z = arange(96).reshape(12, -1)
         y = z.reshape(4, 3, 8)
         assert y.shape == (4, 3, 8)
 
     def test_add(self):
-        from numpypy import array
+        from _numpypy import array
         a = array(range(5))
         b = a + a
         for i in range(5):
@@ -432,7 +453,7 @@
             assert c[i] == bool(a[i] + b[i])
 
     def test_add_other(self):
-        from numpypy import array
+        from _numpypy import array
         a = array(range(5))
         b = array([i for i in reversed(range(5))])
         c = a + b
@@ -440,20 +461,20 @@
             assert c[i] == 4
 
     def test_add_constant(self):
-        from numpypy import array
+        from _numpypy import array
         a = array(range(5))
         b = a + 5
         for i in range(5):
             assert b[i] == i + 5
 
     def test_radd(self):
-        from numpypy import array
+        from _numpypy import array
         r = 3 + array(range(3))
         for i in range(3):
             assert r[i] == i + 3
 
     def test_add_list(self):
-        from numpypy import array, ndarray
+        from _numpypy import array, ndarray
         a = array(range(5))
         b = list(reversed(range(5)))
         c = a + b
@@ -462,14 +483,14 @@
             assert c[i] == 4
 
     def test_subtract(self):
-        from numpypy import array
+        from _numpypy import array
         a = array(range(5))
         b = a - a
         for i in range(5):
             assert b[i] == 0
 
     def test_subtract_other(self):
-        from numpypy import array
+        from _numpypy import array
         a = array(range(5))
         b = array([1, 1, 1, 1, 1])
         c = a - b
@@ -477,34 +498,34 @@
             assert c[i] == i - 1
 
     def test_subtract_constant(self):
-        from numpypy import array
+        from _numpypy import array
         a = array(range(5))
         b = a - 5
         for i in range(5):
             assert b[i] == i - 5
 
     def test_scalar_subtract(self):
-        from numpypy import int32
+        from _numpypy import int32
         assert int32(2) - 1 == 1
         assert 1 - int32(2) == -1
 
     def test_mul(self):
-        import numpypy
+        import _numpypy
 
-        a = numpypy.array(range(5))
+        a = _numpypy.array(range(5))
         b = a * a
         for i in range(5):
             assert b[i] == i * i
 
-        a = numpypy.array(range(5), dtype=bool)
+        a = _numpypy.array(range(5), dtype=bool)
         b = a * a
-        assert b.dtype is numpypy.dtype(bool)
-        assert b[0] is numpypy.False_
+        assert b.dtype is _numpypy.dtype(bool)
+        assert b[0] is _numpypy.False_
         for i in range(1, 5):
-            assert b[i] is numpypy.True_
+            assert b[i] is _numpypy.True_
 
     def test_mul_constant(self):
-        from numpypy import array
+        from _numpypy import array
         a = array(range(5))
         b = a * 5
         for i in range(5):
@@ -512,7 +533,7 @@
 
     def test_div(self):
         from math import isnan
-        from numpypy import array, dtype, inf
+        from _numpypy import array, dtype, inf
 
         a = array(range(1, 6))
         b = a / a
@@ -544,7 +565,7 @@
         assert c[2] == -inf
 
     def test_div_other(self):
-        from numpypy import array
+        from _numpypy import array
         a = array(range(5))
         b = array([2, 2, 2, 2, 2], float)
         c = a / b
@@ -552,14 +573,14 @@
             assert c[i] == i / 2.0
 
     def test_div_constant(self):
-        from numpypy import array
+        from _numpypy import array
         a = array(range(5))
         b = a / 5.0
         for i in range(5):
             assert b[i] == i / 5.0
 
     def test_pow(self):
-        from numpypy import array
+        from _numpypy import array
         a = array(range(5), float)
         b = a ** a
         for i in range(5):
@@ -569,7 +590,7 @@
         assert (a ** 2 == a * a).all()
 
     def test_pow_other(self):
-        from numpypy import array
+        from _numpypy import array
         a = array(range(5), float)
         b = array([2, 2, 2, 2, 2])
         c = a ** b
@@ -577,14 +598,14 @@
             assert c[i] == i ** 2
 
     def test_pow_constant(self):
-        from numpypy import array
+        from _numpypy import array
         a = array(range(5), float)
         b = a ** 2
         for i in range(5):
             assert b[i] == i ** 2
 
     def test_mod(self):
-        from numpypy import array
+        from _numpypy import array
         a = array(range(1, 6))
         b = a % a
         for i in range(5):
@@ -597,7 +618,7 @@
             assert b[i] == 1
 
     def test_mod_other(self):
-        from numpypy import array
+        from _numpypy import array
         a = array(range(5))
         b = array([2, 2, 2, 2, 2])
         c = a % b
@@ -605,14 +626,14 @@
             assert c[i] == i % 2
 
     def test_mod_constant(self):
-        from numpypy import array
+        from _numpypy import array
         a = array(range(5))
         b = a % 2
         for i in range(5):
             assert b[i] == i % 2
 
     def test_pos(self):
-        from numpypy import array
+        from _numpypy import array
         a = array([1., -2., 3., -4., -5.])
         b = +a
         for i in range(5):
@@ -623,7 +644,7 @@
             assert a[i] == i
 
     def test_neg(self):
-        from numpypy import array
+        from _numpypy import array
         a = array([1., -2., 3., -4., -5.])
         b = -a
         for i in range(5):
@@ -634,7 +655,7 @@
             assert a[i] == -i
 
     def test_abs(self):
-        from numpypy import array
+        from _numpypy import array
         a = array([1., -2., 3., -4., -5.])
         b = abs(a)
         for i in range(5):
@@ -645,7 +666,7 @@
             assert a[i + 5] == abs(i)
 
     def test_auto_force(self):
-        from numpypy import array
+        from _numpypy import array
         a = array(range(5))
         b = a - 1
         a[2] = 3
@@ -659,7 +680,7 @@
         assert c[1] == 4
 
     def test_getslice(self):
-        from numpypy import array
+        from _numpypy import array
         a = array(range(5))
         s = a[1:5]
         assert len(s) == 4
@@ -673,7 +694,7 @@
         assert s[0] == 5
 
     def test_getslice_step(self):
-        from numpypy import array
+        from _numpypy import array
         a = array(range(10))
         s = a[1:9:2]
         assert len(s) == 4
@@ -681,7 +702,7 @@
             assert s[i] == a[2 * i + 1]
 
     def test_slice_update(self):
-        from numpypy import array
+        from _numpypy import array
         a = array(range(5))
         s = a[0:3]
         s[1] = 10
@@ -691,7 +712,7 @@
 
     def test_slice_invaidate(self):
         # check that slice shares invalidation list with
-        from numpypy import array
+        from _numpypy import array
         a = array(range(5))
         s = a[0:2]
         b = array([10, 11])
@@ -705,13 +726,19 @@
         assert d[1] == 12
 
     def test_mean(self):
-        from numpypy import array
+        from _numpypy import array
         a = array(range(5))
         assert a.mean() == 2.0
         assert a[:4].mean() == 1.5
+        a = array(range(105)).reshape(3, 5, 7)
+        b = a.mean(axis=0)
+        b[0, 0]==35.
+        assert a.mean(axis=0)[0, 0] == 35
+        assert (b == array(range(35, 70), dtype=float).reshape(5, 7)).all()
+        assert (a.mean(2) == array(range(0, 15), dtype=float).reshape(3, 5) * 7 + 3).all()
 
     def test_sum(self):
-        from numpypy import array
+        from _numpypy import array
         a = array(range(5))
         assert a.sum() == 10.0
         assert a[:4].sum() == 6.0
@@ -719,53 +746,81 @@
         a = array([True] * 5, bool)
         assert a.sum() == 5
 
+        raises(TypeError, 'a.sum(2, 3)')
+
+    def test_reduce_nd(self):
+        from numpypy import arange, array, multiply
+        a = arange(15).reshape(5, 3)
+        assert a.sum() == 105
+        assert a.max() == 14
+        assert array([]).sum() == 0.0
+        raises(ValueError, 'array([]).max()')
+        assert (a.sum(0) == [30, 35, 40]).all()
+        assert (a.sum(axis=0) == [30, 35, 40]).all()
+        assert (a.sum(1) == [3, 12, 21, 30, 39]).all()
+        assert (a.max(0) == [12, 13, 14]).all()
+        assert (a.max(1) == [2, 5, 8, 11, 14]).all()
+        assert ((a + a).max() == 28)
+        assert ((a + a).max(0) == [24, 26, 28]).all()
+        assert ((a + a).sum(1) == [6, 24, 42, 60, 78]).all()
+        assert (multiply.reduce(a) == array([0, 3640, 12320])).all()
+        a = array(range(105)).reshape(3, 5, 7)
+        assert (a[:, 1, :].sum(0) == [126, 129, 132, 135, 138, 141, 144]).all()
+        assert (a[:, 1, :].sum(1) == [70, 315, 560]).all()
+        raises (ValueError, 'a[:, 1, :].sum(2)')
+        assert ((a + a).T.sum(2).T == (a + a).sum(0)).all()
+        assert (a.reshape(1,-1).sum(0) == range(105)).all()
+        assert (a.reshape(1,-1).sum(1) == 5460)
+        assert (array([[1,2],[3,4]]).prod(0) == [3, 8]).all()
+        assert (array([[1,2],[3,4]]).prod(1) == [2, 12]).all()
+
     def test_identity(self):
-        from numpypy import identity, array
-        from numpypy import int32, float64, dtype
+        from _numpypy import identity, array
+        from _numpypy import int32, float64, dtype
         a = identity(0)
         assert len(a) == 0
         assert a.dtype == dtype('float64')
-        assert a.shape == (0,0)
+        assert a.shape == (0, 0)
         b = identity(1, dtype=int32)
         assert len(b) == 1
         assert b[0][0] == 1
-        assert b.shape == (1,1)
+        assert b.shape == (1, 1)
         assert b.dtype == dtype('int32')
         c = identity(2)
-        assert c.shape == (2,2)
-        assert (c == [[1,0],[0,1]]).all()
+        assert c.shape == (2, 2)
+        assert (c == [[1, 0], [0, 1]]).all()
         d = identity(3, dtype='int32')
-        assert d.shape == (3,3)
+        assert d.shape == (3, 3)
         assert d.dtype == dtype('int32')
-        assert (d == [[1,0,0],[0,1,0],[0,0,1]]).all()
+        assert (d == [[1, 0, 0], [0, 1, 0], [0, 0, 1]]).all()
 
     def test_prod(self):
-        from numpypy import array
+        from _numpypy import array
         a = array(range(1, 6))
         assert a.prod() == 120.0
         assert a[:4].prod() == 24.0
 
     def test_max(self):
-        from numpypy import array
+        from _numpypy import array
         a = array([-1.2, 3.4, 5.7, -3.0, 2.7])
         assert a.max() == 5.7
         b = array([])
         raises(ValueError, "b.max()")
 
     def test_max_add(self):
-        from numpypy import array
+        from _numpypy import array
         a = array([-1.2, 3.4, 5.7, -3.0, 2.7])
         assert (a + a).max() == 11.4
 
     def test_min(self):
-        from numpypy import array
+        from _numpypy import array
         a = array([-1.2, 3.4, 5.7, -3.0, 2.7])
         assert a.min() == -3.0
         b = array([])
         raises(ValueError, "b.min()")
 
     def test_argmax(self):
-        from numpypy import array
+        from _numpypy import array
         a = array([-1.2, 3.4, 5.7, -3.0, 2.7])
         r = a.argmax()
         assert r == 2
@@ -786,14 +841,14 @@
         assert a.argmax() == 2
 
     def test_argmin(self):
-        from numpypy import array
+        from _numpypy import array
         a = array([-1.2, 3.4, 5.7, -3.0, 2.7])
         assert a.argmin() == 3
         b = array([])
         raises(ValueError, "b.argmin()")
 
     def test_all(self):
-        from numpypy import array
+        from _numpypy import array
         a = array(range(5))
         assert a.all() == False
         a[0] = 3.0
@@ -802,7 +857,7 @@
         assert b.all() == True
 
     def test_any(self):
-        from numpypy import array, zeros
+        from _numpypy import array, zeros
         a = array(range(5))
         assert a.any() == True
         b = zeros(5)
@@ -811,7 +866,7 @@
         assert c.any() == False
 
     def test_dot(self):
-        from numpypy import array, dot
+        from _numpypy import array, dot
         a = array(range(5))
         assert a.dot(a) == 30.0
 
@@ -821,14 +876,14 @@
         assert (dot(5, [1, 2, 3]) == [5, 10, 15]).all()
 
     def test_dot_constant(self):
-        from numpypy import array
+        from _numpypy import array
         a = array(range(5))
         b = a.dot(2.5)
         for i in xrange(5):
             assert b[i] == 2.5 * a[i]
 
     def test_dtype_guessing(self):
-        from numpypy import array, dtype, float64, int8, bool_
+        from _numpypy import array, dtype, float64, int8, bool_
 
         assert array([True]).dtype is dtype(bool)
         assert array([True, False]).dtype is dtype(bool)
@@ -845,7 +900,7 @@
 
     def test_comparison(self):
         import operator
-        from numpypy import array, dtype
+        from _numpypy import array, dtype
 
         a = array(range(5))
         b = array(range(5), float)
@@ -864,7 +919,7 @@
                 assert c[i] == func(b[i], 3)
 
     def test_nonzero(self):
-        from numpypy import array
+        from _numpypy import array
         a = array([1, 2])
         raises(ValueError, bool, a)
         raises(ValueError, bool, a == a)
@@ -874,7 +929,7 @@
         assert not bool(array([0]))
 
     def test_slice_assignment(self):
-        from numpypy import array
+        from _numpypy import array
         a = array(range(5))
         a[::-1] = a
         assert (a == [0, 1, 2, 1, 0]).all()
@@ -884,8 +939,8 @@
         assert (a == [8, 6, 4, 2, 0]).all()
 
     def test_debug_repr(self):
-        from numpypy import zeros, sin
-        from numpypy.pypy import debug_repr
+        from _numpypy import zeros, sin
+        from _numpypy.pypy import debug_repr
         a = zeros(1)
         assert debug_repr(a) == 'Array'
         assert debug_repr(a + a) == 'Call2(add, Array, Array)'
@@ -899,8 +954,8 @@
         assert debug_repr(b) == 'Array'
 
     def test_remove_invalidates(self):
-        from numpypy import array
-        from numpypy.pypy import remove_invalidates
+        from _numpypy import array
+        from _numpypy.pypy import remove_invalidates
         a = array([1, 2, 3])
         b = a + a
         remove_invalidates(a)
@@ -908,7 +963,7 @@
         assert b[0] == 28
 
     def test_virtual_views(self):
-        from numpypy import arange
+        from _numpypy import arange
         a = arange(15)
         c = (a + a)
         d = c[::2]
@@ -926,7 +981,7 @@
         assert b[1] == 2
 
     def test_tolist_scalar(self):
-        from numpypy import int32, bool_
+        from _numpypy import int32, bool_
         x = int32(23)
         assert x.tolist() == 23
         assert type(x.tolist()) is int
@@ -934,13 +989,13 @@
         assert y.tolist() is True
 
     def test_tolist_zerodim(self):
-        from numpypy import array
+        from _numpypy import array
         x = array(3)
         assert x.tolist() == 3
         assert type(x.tolist()) is int
 
     def test_tolist_singledim(self):
-        from numpypy import array
+        from _numpypy import array
         a = array(range(5))
         assert a.tolist() == [0, 1, 2, 3, 4]
         assert type(a.tolist()[0]) is int
@@ -948,41 +1003,55 @@
         assert b.tolist() == [0.2, 0.4, 0.6]
 
     def test_tolist_multidim(self):
-        from numpypy import array
+        from _numpypy import array
         a = array([[1, 2], [3, 4]])
         assert a.tolist() == [[1, 2], [3, 4]]
 
     def test_tolist_view(self):
-        from numpypy import array
-        a = array([[1,2],[3,4]])
+        from _numpypy import array
+        a = array([[1, 2], [3, 4]])
         assert (a + a).tolist() == [[2, 4], [6, 8]]
 
     def test_tolist_slice(self):
-        from numpypy import array
+        from _numpypy import array
         a = array([[17.1, 27.2], [40.3, 50.3]])
-        assert a[:,0].tolist() == [17.1, 40.3]
+        assert a[:, 0].tolist() == [17.1, 40.3]
         assert a[0].tolist() == [17.1, 27.2]
 
+    def test_var(self):
+        from _numpypy import array
+        a = array(range(10))
+        assert a.var() == 8.25
+        a = array([5.0])
+        assert a.var() == 0.0
+
+    def test_std(self):
+        from _numpypy import array
+        a = array(range(10))
+        assert a.std() == 2.8722813232690143
+        a = array([5.0])
+        assert a.std() == 0.0
+
 
 class AppTestMultiDim(BaseNumpyAppTest):
     def test_init(self):
-        import numpypy
-        a = numpypy.zeros((2, 2))
+        import _numpypy
+        a = _numpypy.zeros((2, 2))
         assert len(a) == 2
 
     def test_shape(self):
-        import numpypy
-        assert numpypy.zeros(1).shape == (1,)
-        assert numpypy.zeros((2, 2)).shape == (2, 2)
-        assert numpypy.zeros((3, 1, 2)).shape == (3, 1, 2)
-        assert numpypy.array([[1], [2], [3]]).shape == (3, 1)
-        assert len(numpypy.zeros((3, 1, 2))) == 3
-        raises(TypeError, len, numpypy.zeros(()))
-        raises(ValueError, numpypy.array, [[1, 2], 3])
+        import _numpypy
+        assert _numpypy.zeros(1).shape == (1,)
+        assert _numpypy.zeros((2, 2)).shape == (2, 2)
+        assert _numpypy.zeros((3, 1, 2)).shape == (3, 1, 2)
+        assert _numpypy.array([[1], [2], [3]]).shape == (3, 1)
+        assert len(_numpypy.zeros((3, 1, 2))) == 3
+        raises(TypeError, len, _numpypy.zeros(()))
+        raises(ValueError, _numpypy.array, [[1, 2], 3])
 
     def test_getsetitem(self):
-        import numpypy
-        a = numpypy.zeros((2, 3, 1))
+        import _numpypy
+        a = _numpypy.zeros((2, 3, 1))
         raises(IndexError, a.__getitem__, (2, 0, 0))
         raises(IndexError, a.__getitem__, (0, 3, 0))
         raises(IndexError, a.__getitem__, (0, 0, 1))
@@ -993,8 +1062,8 @@
         assert a[1, -1, 0] == 3
 
     def test_slices(self):
-        import numpypy
-        a = numpypy.zeros((4, 3, 2))
+        import _numpypy
+        a = _numpypy.zeros((4, 3, 2))
         raises(IndexError, a.__getitem__, (4,))
         raises(IndexError, a.__getitem__, (3, 3))
         raises(IndexError, a.__getitem__, (slice(None), 3))
@@ -1027,51 +1096,51 @@
         assert a[1][2][1] == 15
 
     def test_init_2(self):
-        import numpypy
-        raises(ValueError, numpypy.array, [[1], 2])
-        raises(ValueError, numpypy.array, [[1, 2], [3]])
-        raises(ValueError, numpypy.array, [[[1, 2], [3, 4], 5]])
-        raises(ValueError, numpypy.array, [[[1, 2], [3, 4], [5]]])
-        a = numpypy.array([[1, 2], [4, 5]])
+        import _numpypy
+        raises(ValueError, _numpypy.array, [[1], 2])
+        raises(ValueError, _numpypy.array, [[1, 2], [3]])
+        raises(ValueError, _numpypy.array, [[[1, 2], [3, 4], 5]])
+        raises(ValueError, _numpypy.array, [[[1, 2], [3, 4], [5]]])
+        a = _numpypy.array([[1, 2], [4, 5]])
         assert a[0, 1] == 2
         assert a[0][1] == 2
-        a = numpypy.array(([[[1, 2], [3, 4], [5, 6]]]))
+        a = _numpypy.array(([[[1, 2], [3, 4], [5, 6]]]))
         assert (a[0, 1] == [3, 4]).all()
 
     def test_setitem_slice(self):
-        import numpypy
-        a = numpypy.zeros((3, 4))
+        import _numpypy
+        a = _numpypy.zeros((3, 4))
         a[1] = [1, 2, 3, 4]
         assert a[1, 2] == 3
         raises(TypeError, a[1].__setitem__, [1, 2, 3])
-        a = numpypy.array([[1, 2], [3, 4]])
+        a = _numpypy.array([[1, 2], [3, 4]])
         assert (a == [[1, 2], [3, 4]]).all()
-        a[1] = numpypy.array([5, 6])
+        a[1] = _numpypy.array([5, 6])
         assert (a == [[1, 2], [5, 6]]).all()
-        a[:, 1] = numpypy.array([8, 10])
+        a[:, 1] = _numpypy.array([8, 10])
         assert (a == [[1, 8], [5, 10]]).all()
-        a[0, :: -1] = numpypy.array([11, 12])
+        a[0, :: -1] = _numpypy.array([11, 12])
         assert (a == [[12, 11], [5, 10]]).all()
 
     def test_ufunc(self):
-        from numpypy import array
+        from _numpypy import array
         a = array([[1, 2], [3, 4], [5, 6]])
         assert ((a + a) == \
             array([[1 + 1, 2 + 2], [3 + 3, 4 + 4], [5 + 5, 6 + 6]])).all()
 
     def test_getitem_add(self):
-        from numpypy import array
+        from _numpypy import array
         a = array([[1, 2], [3, 4], [5, 6], [7, 8], [9, 10]])
         assert (a + a)[1, 1] == 8
 
     def test_ufunc_negative(self):
-        from numpypy import array, negative
+        from _numpypy import array, negative
         a = array([[1, 2], [3, 4]])
         b = negative(a + a)
         assert (b == [[-2, -4], [-6, -8]]).all()
 
     def test_getitem_3(self):
-        from numpypy import array
+        from _numpypy import array
         a = array([[1, 2], [3, 4], [5, 6], [7, 8],
                    [9, 10], [11, 12], [13, 14]])
         b = a[::2]
@@ -1082,37 +1151,37 @@
         assert c[1][1] == 12
 
     def test_multidim_ones(self):
-        from numpypy import ones
+        from _numpypy import ones
         a = ones((1, 2, 3))
         assert a[0, 1, 2] == 1.0
 
     def test_multidim_setslice(self):
-        from numpypy import zeros, ones
+        from _numpypy import zeros, ones
         a = zeros((3, 3))
         b = ones((3, 3))
-        a[:,1:3] = b[:,1:3]
+        a[:, 1:3] = b[:, 1:3]
         assert (a == [[0, 1, 1], [0, 1, 1], [0, 1, 1]]).all()
         a = zeros((3, 3))
         b = ones((3, 3))
-        a[:,::2] = b[:,::2]
+        a[:, ::2] = b[:, ::2]
         assert (a == [[1, 0, 1], [1, 0, 1], [1, 0, 1]]).all()
 
     def test_broadcast_ufunc(self):
-        from numpypy import array
+        from _numpypy import array
         a = array([[1, 2], [3, 4], [5, 6]])
         b = array([5, 6])
         c = ((a + b) == [[1 + 5, 2 + 6], [3 + 5, 4 + 6], [5 + 5, 6 + 6]])
         assert c.all()
 
     def test_broadcast_setslice(self):
-        from numpypy import zeros, ones
+        from _numpypy import zeros, ones
         a = zeros((10, 10))
         b = ones(10)
         a[:, :] = b
         assert a[3, 5] == 1
 
     def test_broadcast_shape_agreement(self):
-        from numpypy import zeros, array
+        from _numpypy import zeros, array
         a = zeros((3, 1, 3))
         b = array(((10, 11, 12), (20, 21, 22), (30, 31, 32)))
         c = ((a + b) == [b, b, b])
@@ -1126,7 +1195,7 @@
         assert c.all()
 
     def test_broadcast_scalar(self):
-        from numpypy import zeros
+        from _numpypy import zeros
         a = zeros((4, 5), 'd')
         a[:, 1] = 3
         assert a[2, 1] == 3
@@ -1137,14 +1206,14 @@
         assert a[3, 2] == 0
 
     def test_broadcast_call2(self):
-        from numpypy import zeros, ones
+        from _numpypy import zeros, ones
         a = zeros((4, 1, 5))
         b = ones((4, 3, 5))
         b[:] = (a + a)
         assert (b == zeros((4, 3, 5))).all()
 
     def test_broadcast_virtualview(self):
-        from numpypy import arange, zeros
+        from _numpypy import arange, zeros
         a = arange(8).reshape([2, 2, 2])
         b = (a + a)[1, 1]
         c = zeros((2, 2, 2))
@@ -1152,13 +1221,13 @@
         assert (c == [[[12, 14], [12, 14]], [[12, 14], [12, 14]]]).all()
 
     def test_argmax(self):
-        from numpypy import array
+        from _numpypy import array
         a = array([[1, 2], [3, 4], [5, 6]])
         assert a.argmax() == 5
         assert a[:2, ].argmax() == 3
 
     def test_broadcast_wrong_shapes(self):
-        from numpypy import zeros
+        from _numpypy import zeros
         a = zeros((4, 3, 2))
         b = zeros((4, 2))
         exc = raises(ValueError, lambda: a + b)
@@ -1166,7 +1235,7 @@
             " together with shapes (4,3,2) (4,2)"
 
     def test_reduce(self):
-        from numpypy import array
+        from _numpypy import array
         a = array([[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]])
         assert a.sum() == (13 * 12) / 2
         b = a[1:, 1::2]
@@ -1174,7 +1243,7 @@
         assert c.sum() == (6 + 8 + 10 + 12) * 2
 
     def test_transpose(self):
-        from numpypy import array
+        from _numpypy import array
         a = array(((range(3), range(3, 6)),
                    (range(6, 9), range(9, 12)),
                    (range(12, 15), range(15, 18)),
@@ -1193,7 +1262,7 @@
         assert(b[:, 0] == a[0, :]).all()
 
     def test_flatiter(self):
-        from numpypy import array, flatiter
+        from _numpypy import array, flatiter
         a = array([[10, 30], [40, 60]])
         f_iter = a.flat
         assert f_iter.next() == 10
@@ -1208,23 +1277,23 @@
         assert s == 140
 
     def test_flatiter_array_conv(self):
-        from numpypy import array, dot
+        from _numpypy import array, dot
         a = array([1, 2, 3])
         assert dot(a.flat, a.flat) == 14
 
     def test_flatiter_varray(self):
-        from numpypy import ones
+        from _numpypy import ones
         a = ones((2, 2))
         assert list(((a + a).flat)) == [2, 2, 2, 2]
 
     def test_slice_copy(self):
-        from numpypy import zeros
+        from _numpypy import zeros
         a = zeros((10, 10))
         b = a[0].copy()
         assert (b == zeros(10)).all()
 
     def test_array_interface(self):
-        from numpypy import array
+        from _numpypy import array
         a = array([1, 2, 3])
         i = a.__array_interface__
         assert isinstance(i['data'][0], int)
@@ -1233,6 +1302,7 @@
         assert isinstance(i['data'][0], int)
         raises(TypeError, getattr, array(3), '__array_interface__')
 
+
 class AppTestSupport(BaseNumpyAppTest):
     def setup_class(cls):
         import struct
@@ -1245,7 +1315,7 @@
 
     def test_fromstring(self):
         import sys
-        from numpypy import fromstring, array, uint8, float32, int32
+        from _numpypy import fromstring, array, uint8, float32, int32
 
         a = fromstring(self.data)
         for i in range(4):
@@ -1275,17 +1345,17 @@
         assert g[1] == 2
         assert g[2] == 3
         h = fromstring("1, , 2, 3", dtype=uint8, sep=",")
-        assert (h == [1,0,2,3]).all()
+        assert (h == [1, 0, 2, 3]).all()
         i = fromstring("1    2 3", dtype=uint8, sep=" ")
-        assert (i == [1,2,3]).all()
+        assert (i == [1, 2, 3]).all()
         j = fromstring("1\t\t\t\t2\t3", dtype=uint8, sep="\t")
-        assert (j == [1,2,3]).all()
+        assert (j == [1, 2, 3]).all()
         k = fromstring("1,x,2,3", dtype=uint8, sep=",")
-        assert (k == [1,0]).all()
+        assert (k == [1, 0]).all()
         l = fromstring("1,x,2,3", dtype='float32', sep=",")
-        assert (l == [1.0,-1.0]).all()
+        assert (l == [1.0, -1.0]).all()
         m = fromstring("1,,2,3", sep=",")
-        assert (m == [1.0,-1.0,2.0,3.0]).all()
+        assert (m == [1.0, -1.0, 2.0, 3.0]).all()
         n = fromstring("3.4 2.0 3.8 2.2", dtype=int32, sep=" ")
         assert (n == [3]).all()
         o = fromstring("1.0 2f.0f 3.8 2.2", dtype=float32, sep=" ")
@@ -1309,7 +1379,7 @@
             assert (u == [1, 0]).all()
 
     def test_fromstring_types(self):
-        from numpypy import (fromstring, int8, int16, int32, int64, uint8,
+        from _numpypy import (fromstring, int8, int16, int32, int64, uint8,
             uint16, uint32, float32, float64)
 
         a = fromstring('\xFF', dtype=int8)
@@ -1333,9 +1403,8 @@
         j = fromstring(self.ulongval, dtype='L')
         assert j[0] == 12
 
-
     def test_fromstring_invalid(self):
-        from numpypy import fromstring, uint16, uint8, int32
+        from _numpypy import fromstring, uint16, uint8, int32
         #default dtype is 64-bit float, so 3 bytes should fail
         raises(ValueError, fromstring, "\x01\x02\x03")
         #3 bytes is not modulo 2 bytes (int16)
@@ -1346,7 +1415,8 @@
 
 class AppTestRepr(BaseNumpyAppTest):
     def test_repr(self):
-        from numpypy import array, zeros
+        from _numpypy import array, zeros
+        int_size = array(5).dtype.itemsize
         a = array(range(5), float)
         assert repr(a) == "array([0.0, 1.0, 2.0, 3.0, 4.0])"
         a = array([], float)
@@ -1354,14 +1424,26 @@
         a = zeros(1001)
         assert repr(a) == "array([0.0, 0.0, 0.0, ..., 0.0, 0.0, 0.0])"
         a = array(range(5), long)
-        assert repr(a) == "array([0, 1, 2, 3, 4])"
+        if a.dtype.itemsize == int_size:
+            assert repr(a) == "array([0, 1, 2, 3, 4])"
+        else:
+            assert repr(a) == "array([0, 1, 2, 3, 4], dtype=int64)"
+        a = array(range(5), 'int32')
+        if a.dtype.itemsize == int_size:
+            assert repr(a) == "array([0, 1, 2, 3, 4])"
+        else:
+            assert repr(a) == "array([0, 1, 2, 3, 4], dtype=int32)"
         a = array([], long)
         assert repr(a) == "array([], dtype=int64)"
         a = array([True, False, True, False], "?")
         assert repr(a) == "array([True, False, True, False], dtype=bool)"
+        a = zeros([])
+        assert repr(a) == "array(0.0)"
+        a = array(0.2)
+        assert repr(a) == "array(0.2)"
 
     def test_repr_multi(self):
-        from numpypy import array, zeros
+        from _numpypy import arange, zeros
         a = zeros((3, 4))
         assert repr(a) == '''array([[0.0, 0.0, 0.0, 0.0],
        [0.0, 0.0, 0.0, 0.0],
@@ -1374,9 +1456,19 @@
        [[0.0, 0.0, 0.0, 0.0],
         [0.0, 0.0, 0.0, 0.0],
         [0.0, 0.0, 0.0, 0.0]]])'''
+        a = arange(1002).reshape((2, 501))
+        assert repr(a) == '''array([[0, 1, 2, ..., 498, 499, 500],
+       [501, 502, 503, ..., 999, 1000, 1001]])'''
+        assert repr(a.T) == '''array([[0, 501],
+       [1, 502],
+       [2, 503],
+       ...,
+       [498, 999],
+       [499, 1000],
+       [500, 1001]])'''
 
     def test_repr_slice(self):
-        from numpypy import array, zeros
+        from _numpypy import array, zeros
         a = array(range(5), float)
         b = a[1::2]
         assert repr(b) == "array([1.0, 3.0])"
@@ -1391,7 +1483,7 @@
         assert repr(b) == "array([], shape=(0, 5), dtype=int16)"
 
     def test_str(self):
-        from numpypy import array, zeros
+        from _numpypy import array, zeros
         a = array(range(5), float)
         assert str(a) == "[0.0 1.0 2.0 3.0 4.0]"
         assert str((2 * a)[:]) == "[0.0 2.0 4.0 6.0 8.0]"
@@ -1417,14 +1509,14 @@
 
         a = zeros((400, 400), dtype=int)
         assert str(a) == "[[0 0 0 ..., 0 0 0]\n [0 0 0 ..., 0 0 0]\n" \
-           " [0 0 0 ..., 0 0 0]\n ..., \n [0 0 0 ..., 0 0 0]\n" \
+           " [0 0 0 ..., 0 0 0]\n ...,\n [0 0 0 ..., 0 0 0]\n" \
            " [0 0 0 ..., 0 0 0]\n [0 0 0 ..., 0 0 0]]"
         a = zeros((2, 2, 2))
         r = str(a)
         assert r == '[[[0.0 0.0]\n  [0.0 0.0]]\n\n [[0.0 0.0]\n  [0.0 0.0]]]'
 
     def test_str_slice(self):
-        from numpypy import array, zeros
+        from _numpypy import array, zeros
         a = array(range(5), float)
         b = a[1::2]
         assert str(b) == "[1.0 3.0]"
@@ -1440,7 +1532,7 @@
 
 class AppTestRanges(BaseNumpyAppTest):
     def test_arange(self):
-        from numpypy import arange, array, dtype
+        from _numpypy import arange, array, dtype
         a = arange(3)
         assert (a == [0, 1, 2]).all()
         assert a.dtype is dtype(int)
@@ -1458,14 +1550,3 @@
         a = arange(0, 0.8, 0.1)
         assert len(a) == 8
         assert arange(False, True, True).dtype is dtype(int)
-
-
-class AppTestRanges(BaseNumpyAppTest):
-    def test_app_reshape(self):
-        from numpypy import arange, array, dtype, reshape
-        a = arange(12)
-        b = reshape(a, (3, 4))
-        assert b.shape == (3, 4)
-        a = range(12)
-        b = reshape(a, (3, 4))
-        assert b.shape == (3, 4)
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -4,14 +4,14 @@
 
 class AppTestUfuncs(BaseNumpyAppTest):
     def test_ufunc_instance(self):
-        from numpypy import add, ufunc
+        from _numpypy import add, ufunc
 
         assert isinstance(add, ufunc)
         assert repr(add) == ""
         assert repr(ufunc) == ""
 
     def test_ufunc_attrs(self):
-        from numpypy import add, multiply, sin
+        from _numpypy import add, multiply, sin
 
         assert add.identity == 0
         assert multiply.identity == 1
@@ -22,7 +22,7 @@
         assert sin.nin == 1
 
     def test_wrong_arguments(self):
-        from numpypy import add, sin
+        from _numpypy import add, sin
 
         raises(ValueError, add, 1)
         raises(TypeError, add, 1, 2, 3)
@@ -30,14 +30,14 @@
         raises(ValueError, sin)
 
     def test_single_item(self):
-        from numpypy import negative, sign, minimum
+        from _numpypy import negative, sign, minimum
 
         assert negative(5.0) == -5.0
         assert sign(-0.0) == 0.0
         assert minimum(2.0, 3.0) == 2.0
 
     def test_sequence(self):
-        from numpypy import array, ndarray, negative, minimum
+        from _numpypy import array, ndarray, negative, minimum
         a = array(range(3))
         b = [2.0, 1.0, 0.0]
         c = 1.0
@@ -71,7 +71,7 @@
             assert min_c_b[i] == min(b[i], c)
 
     def test_negative(self):
-        from numpypy import array, negative
+        from _numpypy import array, negative
 
         a = array([-5.0, 0.0, 1.0])
         b = negative(a)
@@ -86,7 +86,7 @@
         assert negative(a + a)[3] == -6
 
     def test_abs(self):
-        from numpypy import array, absolute
+        from _numpypy import array, absolute
 
         a = array([-5.0, -0.0, 1.0])
         b = absolute(a)
@@ -94,7 +94,7 @@
             assert b[i] == abs(a[i])
 
     def test_add(self):
-        from numpypy import array, add
+        from _numpypy import array, add
 
         a = array([-5.0, -0.0, 1.0])
         b = array([ 3.0, -2.0,-3.0])
@@ -103,7 +103,7 @@
             assert c[i] == a[i] + b[i]
 
     def test_divide(self):
-        from numpypy import array, divide
+        from _numpypy import array, divide
 
         a = array([-5.0, -0.0, 1.0])
         b = array([ 3.0, -2.0,-3.0])
@@ -114,7 +114,7 @@
         assert (divide(array([-10]), array([2])) == array([-5])).all()
 
     def test_fabs(self):
-        from numpypy import array, fabs
+        from _numpypy import array, fabs
         from math import fabs as math_fabs
 
         a = array([-5.0, -0.0, 1.0])
@@ -123,7 +123,7 @@
             assert b[i] == math_fabs(a[i])
 
     def test_minimum(self):
-        from numpypy import array, minimum
+        from _numpypy import array, minimum
 
         a = array([-5.0, -0.0, 1.0])
         b = array([ 3.0, -2.0,-3.0])
@@ -132,7 +132,7 @@
             assert c[i] == min(a[i], b[i])
 
     def test_maximum(self):
-        from numpypy import array, maximum
+        from _numpypy import array, maximum
 
         a = array([-5.0, -0.0, 1.0])
         b = array([ 3.0, -2.0,-3.0])
@@ -145,7 +145,7 @@
         assert isinstance(x, (int, long))
 
     def test_multiply(self):
-        from numpypy import array, multiply
+        from _numpypy import array, multiply
 
         a = array([-5.0, -0.0, 1.0])
         b = array([ 3.0, -2.0,-3.0])
@@ -154,7 +154,7 @@
             assert c[i] == a[i] * b[i]
 
     def test_sign(self):
-        from numpypy import array, sign, dtype
+        from _numpypy import array, sign, dtype
 
         reference = [-1.0, 0.0, 0.0, 1.0]
         a = array([-5.0, -0.0, 0.0, 6.0])
@@ -173,7 +173,7 @@
         assert a[1] == 0
 
     def test_reciporocal(self):
-        from numpypy import array, reciprocal
+        from _numpypy import array, reciprocal
 
         reference = [-0.2, float("inf"), float("-inf"), 2.0]
         a = array([-5.0, 0.0, -0.0, 0.5])
@@ -182,7 +182,7 @@
             assert b[i] == reference[i]
 
     def test_subtract(self):
-        from numpypy import array, subtract
+        from _numpypy import array, subtract
 
         a = array([-5.0, -0.0, 1.0])
         b = array([ 3.0, -2.0,-3.0])
@@ -191,7 +191,7 @@
             assert c[i] == a[i] - b[i]
 
     def test_floor(self):
-        from numpypy import array, floor
+        from _numpypy import array, floor
 
         reference = [-2.0, -1.0, 0.0, 1.0, 1.0]
         a = array([-1.4, -1.0, 0.0, 1.0, 1.4])
@@ -200,7 +200,7 @@
             assert b[i] == reference[i]
 
     def test_copysign(self):
-        from numpypy import array, copysign
+        from _numpypy import array, copysign
 
         reference = [5.0, -0.0, 0.0, -6.0]
         a = array([-5.0, 0.0, 0.0, 6.0])
@@ -216,7 +216,7 @@
 
     def test_exp(self):
         import math
-        from numpypy import array, exp
+        from _numpypy import array, exp
 
         a = array([-5.0, -0.0, 0.0, 12345678.0, float("inf"),
                    -float('inf'), -12343424.0])
@@ -230,7 +230,7 @@
 
     def test_sin(self):
         import math
-        from numpypy import array, sin
+        from _numpypy import array, sin
 
         a = array([0, 1, 2, 3, math.pi, math.pi*1.5, math.pi*2])
         b = sin(a)
@@ -243,7 +243,7 @@
 
     def test_cos(self):
         import math
-        from numpypy import array, cos
+        from _numpypy import array, cos
 
         a = array([0, 1, 2, 3, math.pi, math.pi*1.5, math.pi*2])
         b = cos(a)
@@ -252,7 +252,7 @@
 
     def test_tan(self):
         import math
-        from numpypy import array, tan
+        from _numpypy import array, tan
 
         a = array([0, 1, 2, 3, math.pi, math.pi*1.5, math.pi*2])
         b = tan(a)
@@ -262,7 +262,7 @@
 
     def test_arcsin(self):
         import math
-        from numpypy import array, arcsin
+        from _numpypy import array, arcsin
 
         a = array([-1, -0.5, -0.33, 0, 0.33, 0.5, 1])
         b = arcsin(a)
@@ -276,7 +276,7 @@
 
     def test_arccos(self):
         import math
-        from numpypy import array, arccos
+        from _numpypy import array, arccos
 
         a = array([-1, -0.5, -0.33, 0, 0.33, 0.5, 1])
         b = arccos(a)
@@ -291,20 +291,20 @@
 
     def test_arctan(self):
         import math
-        from numpypy import array, arctan
+        from _numpypy import array, arctan
 
         a = array([-3, -2, -1, 0, 1, 2, 3, float('inf'), float('-inf')])
         b = arctan(a)
         for i in range(len(a)):
             assert b[i] == math.atan(a[i])
 
-        a  = array([float('nan')])
+        a = array([float('nan')])
         b = arctan(a)
         assert math.isnan(b[0])
 
     def test_arcsinh(self):
         import math
-        from numpypy import arcsinh, inf
+        from _numpypy import arcsinh, inf
 
         for v in [inf, -inf, 1.0, math.e]:
             assert math.asinh(v) == arcsinh(v)
@@ -312,7 +312,7 @@
 
     def test_arctanh(self):
         import math
-        from numpypy import arctanh
+        from _numpypy import arctanh
 
         for v in [.99, .5, 0, -.5, -.99]:
             assert math.atanh(v) == arctanh(v)
@@ -323,7 +323,7 @@
 
     def test_sqrt(self):
         import math
-        from numpypy import sqrt
+        from _numpypy import sqrt
 
         nan, inf = float("nan"), float("inf")
         data = [1, 2, 3, inf]
@@ -333,22 +333,28 @@
         assert math.isnan(sqrt(nan))
 
     def test_reduce_errors(self):
-        from numpypy import sin, add
+        from _numpypy import sin, add
 
         raises(ValueError, sin.reduce, [1, 2, 3])
-        raises(TypeError, add.reduce, 1)
+        raises(ValueError, add.reduce, 1)
 
-    def test_reduce(self):
-        from numpypy import add, maximum
+    def test_reduce_1d(self):
+        from _numpypy import add, maximum
 
         assert add.reduce([1, 2, 3]) == 6
         assert maximum.reduce([1]) == 1
         assert maximum.reduce([1, 2, 3]) == 3
         raises(ValueError, maximum.reduce, [])
 
+    def test_reduceND(self):
+        from numpypy import add, arange
+        a = arange(12).reshape(3, 4)
+        assert (add.reduce(a, 0) == [12, 15, 18, 21]).all()
+        assert (add.reduce(a, 1) == [6.0, 22.0, 38.0]).all()
+
     def test_comparisons(self):
         import operator
-        from numpypy import equal, not_equal, less, less_equal, greater, greater_equal
+        from _numpypy import equal, not_equal, less, less_equal, greater, greater_equal
 
         for ufunc, func in [
             (equal, operator.eq),
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -47,6 +47,8 @@
         def f(i):
             interp = InterpreterState(codes[i])
             interp.run(space)
+            if not len(interp.results):
+                raise Exception("need results")
             w_res = interp.results[-1]
             if isinstance(w_res, BaseArray):
                 concr = w_res.get_concrete_or_scalar()
@@ -115,6 +117,28 @@
                                 "int_add": 1, "int_ge": 1, "guard_false": 1,
                                 "jump": 1, 'arraylen_gc': 1})
 
+    def define_axissum():
+        return """
+        a = [[1, 2], [3, 4], [5, 6], [7, 8], [9, 10]]
+        b = sum(a,0)
+        b -> 1
+        """
+
+    def test_axissum(self):
+        result = self.run("axissum")
+        assert result == 30
+        # XXX note - the bridge here is fairly crucial and yet it's pretty
+        #            bogus. We need to improve the situation somehow.
+        self.check_simple_loop({'getinteriorfield_raw': 2,
+                                'setinteriorfield_raw': 1,
+                                'arraylen_gc': 1,
+                                'guard_true': 1,
+                                'int_lt': 1,
+                                'jump': 1,
+                                'float_add': 1,
+                                'int_add': 3,
+                                })
+
     def define_prod():
         return """
         a = |30|
@@ -193,9 +217,9 @@
         # This is the sum of the ops for both loops, however if you remove the
         # optimization then you end up with 2 float_adds, so we can still be
         # sure it was optimized correctly.
-        self.check_resops({'setinteriorfield_raw': 4, 'getfield_gc': 26,
+        self.check_resops({'setinteriorfield_raw': 4, 'getfield_gc': 22,
                            'getarrayitem_gc': 4, 'getarrayitem_gc_pure': 2,
-                           'getfield_gc_pure': 4,
+                           'getfield_gc_pure': 8,
                            'guard_class': 8, 'int_add': 8, 'float_mul': 2,
                            'jump': 2, 'int_ge': 4,
                            'getinteriorfield_raw': 4, 'float_add': 2,
@@ -212,7 +236,8 @@
     def test_ufunc(self):
         result = self.run("ufunc")
         assert result == -6
-        self.check_simple_loop({"getinteriorfield_raw": 2, "float_add": 1, "float_neg": 1,
+        self.check_simple_loop({"getinteriorfield_raw": 2, "float_add": 1,
+                                "float_neg": 1,
                                 "setinteriorfield_raw": 1, "int_add": 2,
                                 "int_ge": 1, "guard_false": 1, "jump": 1,
                                 'arraylen_gc': 1})
@@ -322,10 +347,9 @@
         result = self.run("setslice")
         assert result == 11.0
         self.check_trace_count(1)
-        self.check_simple_loop({'getinteriorfield_raw': 2, 'float_add' : 1,
-                                'setinteriorfield_raw': 1, 'int_add': 3,
-                                'int_lt': 1, 'guard_true': 1, 'jump': 1,
-                                'arraylen_gc': 3})
+        self.check_simple_loop({'getinteriorfield_raw': 2, 'float_add': 1,
+                                'setinteriorfield_raw': 1, 'int_add': 2,
+                                'int_eq': 1, 'guard_false': 1, 'jump': 1})
 
     def define_virtual_slice():
         return """
@@ -339,11 +363,12 @@
         result = self.run("virtual_slice")
         assert result == 4
         self.check_trace_count(1)
-        self.check_simple_loop({'getinteriorfield_raw': 2, 'float_add' : 1,
+        self.check_simple_loop({'getinteriorfield_raw': 2, 'float_add': 1,
                                 'setinteriorfield_raw': 1, 'int_add': 2,
                                 'int_ge': 1, 'guard_false': 1, 'jump': 1,
                                 'arraylen_gc': 1})
 
+
 class TestNumpyOld(LLJitMixin):
     def setup_class(cls):
         py.test.skip("old")
@@ -377,4 +402,3 @@
 
         result = self.meta_interp(f, [5], listops=True, backendopt=True)
         assert result == f(5)
-
diff --git a/pypy/module/pypyjit/__init__.py b/pypy/module/pypyjit/__init__.py
--- a/pypy/module/pypyjit/__init__.py
+++ b/pypy/module/pypyjit/__init__.py
@@ -7,16 +7,22 @@
     interpleveldefs = {
         'set_param':    'interp_jit.set_param',
         'residual_call': 'interp_jit.residual_call',
-        'set_compile_hook': 'interp_jit.set_compile_hook',
-        'DebugMergePoint': 'interp_resop.W_DebugMergePoint',
+        'set_compile_hook': 'interp_resop.set_compile_hook',
+        'set_optimize_hook': 'interp_resop.set_optimize_hook',
+        'set_abort_hook': 'interp_resop.set_abort_hook',
+        'ResOperation': 'interp_resop.WrappedOp',
+        'DebugMergePoint': 'interp_resop.DebugMergePoint',
+        'Box': 'interp_resop.WrappedBox',
     }
 
     def setup_after_space_initialization(self):
         # force the __extend__ hacks to occur early
         from pypy.module.pypyjit.interp_jit import pypyjitdriver
+        from pypy.module.pypyjit.policy import pypy_hooks
         # add the 'defaults' attribute
         from pypy.rlib.jit import PARAMETERS
         space = self.space
         pypyjitdriver.space = space
         w_obj = space.wrap(PARAMETERS)
         space.setattr(space.wrap(self), space.wrap('defaults'), w_obj)
+        pypy_hooks.space = space
diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py
--- a/pypy/module/pypyjit/interp_jit.py
+++ b/pypy/module/pypyjit/interp_jit.py
@@ -13,11 +13,7 @@
 from pypy.interpreter.pycode import PyCode, CO_GENERATOR
 from pypy.interpreter.pyframe import PyFrame
 from pypy.interpreter.pyopcode import ExitFrame
-from pypy.interpreter.gateway import unwrap_spec
 from opcode import opmap
-from pypy.rlib.nonconst import NonConstant
-from pypy.jit.metainterp.resoperation import rop
-from pypy.module.pypyjit.interp_resop import debug_merge_point_from_boxes
 
 PyFrame._virtualizable2_ = ['last_instr', 'pycode',
                             'valuestackdepth', 'locals_stack_w[*]',
@@ -51,72 +47,19 @@
 def should_unroll_one_iteration(next_instr, is_being_profiled, bytecode):
     return (bytecode.co_flags & CO_GENERATOR) != 0
 
-def wrap_oplist(space, logops, operations):
-    list_w = []
-    for op in operations:
-        if op.getopnum() == rop.DEBUG_MERGE_POINT:
-            list_w.append(space.wrap(debug_merge_point_from_boxes(
-                op.getarglist())))
-        else:
-            list_w.append(space.wrap(logops.repr_of_resop(op)))
-    return list_w
-
 class PyPyJitDriver(JitDriver):
     reds = ['frame', 'ec']
     greens = ['next_instr', 'is_being_profiled', 'pycode']
     virtualizables = ['frame']
 
-    def on_compile(self, logger, looptoken, operations, type, next_instr,
-                   is_being_profiled, ll_pycode):
-        from pypy.rpython.annlowlevel import cast_base_ptr_to_instance
-
-        space = self.space
-        cache = space.fromcache(Cache)
-        if cache.in_recursion:
-            return
-        if space.is_true(cache.w_compile_hook):
-            logops = logger._make_log_operations()
-            list_w = wrap_oplist(space, logops, operations)
-            pycode = cast_base_ptr_to_instance(PyCode, ll_pycode)
-            cache.in_recursion = True
-            try:
-                space.call_function(cache.w_compile_hook,
-                                    space.wrap('main'),
-                                    space.wrap(type),
-                                    space.newtuple([pycode,
-                                    space.wrap(next_instr),
-                                    space.wrap(is_being_profiled)]),
-                                    space.newlist(list_w))
-            except OperationError, e:
-                e.write_unraisable(space, "jit hook ", cache.w_compile_hook)
-            cache.in_recursion = False
-
-    def on_compile_bridge(self, logger, orig_looptoken, operations, n):
-        space = self.space
-        cache = space.fromcache(Cache)
-        if cache.in_recursion:
-            return
-        if space.is_true(cache.w_compile_hook):
-            logops = logger._make_log_operations()
-            list_w = wrap_oplist(space, logops, operations)
-            cache.in_recursion = True
-            try:
-                space.call_function(cache.w_compile_hook,
-                                    space.wrap('main'),
-                                    space.wrap('bridge'),
-                                    space.wrap(n),
-                                    space.newlist(list_w))
-            except OperationError, e:
-                e.write_unraisable(space, "jit hook ", cache.w_compile_hook)
-            cache.in_recursion = False
-
 pypyjitdriver = PyPyJitDriver(get_printable_location = get_printable_location,
                               get_jitcell_at = get_jitcell_at,
                               set_jitcell_at = set_jitcell_at,
                               confirm_enter_jit = confirm_enter_jit,
                               can_never_inline = can_never_inline,
                               should_unroll_one_iteration =
-                              should_unroll_one_iteration)
+                              should_unroll_one_iteration,
+                              name='pypyjit')
 
 class __extend__(PyFrame):
 
@@ -223,34 +166,3 @@
     '''For testing.  Invokes callable(...), but without letting
     the JIT follow the call.'''
     return space.call_args(w_callable, __args__)
-
-class Cache(object):
-    in_recursion = False
-
-    def __init__(self, space):
-        self.w_compile_hook = space.w_None
-
-def set_compile_hook(space, w_hook):
-    """ set_compile_hook(hook)
-
-    Set a compiling hook that will be called each time a loop is compiled.
-    The hook will be called with the following signature:
-    hook(merge_point_type, loop_type, greenkey or guard_number, operations)
-
-    for now merge point type is always `main`
-
-    loop_type can be either `loop` `entry_bridge` or `bridge`
-    in case loop is not `bridge`, greenkey will be a set of constants
-    for jit merge point. in case it's `main` it'll be a tuple
-    (code, offset, is_being_profiled)
-
-    Note that jit hook is not reentrant. It means that if the code
-    inside the jit hook is itself jitted, it will get compiled, but the
-    jit hook won't be called for that.
-
-    XXX write down what else
-    """
-    cache = space.fromcache(Cache)
-    cache.w_compile_hook = w_hook
-    cache.in_recursion = NonConstant(False)
-    return space.w_None
diff --git a/pypy/module/pypyjit/interp_resop.py b/pypy/module/pypyjit/interp_resop.py
--- a/pypy/module/pypyjit/interp_resop.py
+++ b/pypy/module/pypyjit/interp_resop.py
@@ -1,41 +1,250 @@
 
-from pypy.interpreter.typedef import TypeDef, interp_attrproperty
+from pypy.interpreter.typedef import (TypeDef, GetSetProperty,
+     interp_attrproperty, interp_attrproperty_w)
 from pypy.interpreter.baseobjspace import Wrappable
-from pypy.interpreter.gateway import unwrap_spec, interp2app
+from pypy.interpreter.gateway import unwrap_spec, interp2app, NoneNotWrapped
 from pypy.interpreter.pycode import PyCode
-from pypy.rpython.lltypesystem import lltype
-from pypy.rpython.annlowlevel import cast_base_ptr_to_instance
+from pypy.interpreter.error import OperationError
+from pypy.rpython.lltypesystem import lltype, llmemory
+from pypy.rpython.annlowlevel import cast_base_ptr_to_instance, hlstr
 from pypy.rpython.lltypesystem.rclass import OBJECT
+from pypy.jit.metainterp.resoperation import rop, AbstractResOp
+from pypy.rlib.nonconst import NonConstant
+from pypy.rlib import jit_hooks
+from pypy.module.pypyjit.interp_jit import pypyjitdriver
 
-class W_DebugMergePoint(Wrappable):
-    """ A class representing debug_merge_point JIT operation
+class Cache(object):
+    in_recursion = False
+
+    def __init__(self, space):
+        self.w_compile_hook = space.w_None
+        self.w_abort_hook = space.w_None
+        self.w_optimize_hook = space.w_None
+
+def wrap_greenkey(space, jitdriver, greenkey, greenkey_repr):
+    if greenkey is None:
+        return space.w_None
+    jitdriver_name = jitdriver.name
+    if jitdriver_name == 'pypyjit':
+        next_instr = greenkey[0].getint()
+        is_being_profiled = greenkey[1].getint()
+        ll_code = lltype.cast_opaque_ptr(lltype.Ptr(OBJECT),
+                                         greenkey[2].getref_base())
+        pycode = cast_base_ptr_to_instance(PyCode, ll_code)
+        return space.newtuple([space.wrap(pycode), space.wrap(next_instr),
+                               space.newbool(bool(is_being_profiled))])
+    else:
+        return space.wrap(greenkey_repr)
+
+def set_compile_hook(space, w_hook):
+    """ set_compile_hook(hook)
+
+    Set a compiling hook that will be called each time a loop is compiled.
+    The hook will be called with the following signature:
+    hook(jitdriver_name, loop_type, greenkey or guard_number, operations,
+         assembler_addr, assembler_length)
+
+    jitdriver_name is the name of this particular jitdriver, 'pypyjit' is
+    the main interpreter loop
+
+    loop_type can be either `loop` `entry_bridge` or `bridge`
+    in case loop is not `bridge`, greenkey will be a tuple of constants
+    or a string describing it.
+
+    for the interpreter loop` it'll be a tuple
+    (code, offset, is_being_profiled)
+
+    assembler_addr is an integer describing where assembler starts,
+    can be accessed via ctypes, assembler_lenght is the lenght of compiled
+    asm
+
+    Note that jit hook is not reentrant. It means that if the code
+    inside the jit hook is itself jitted, it will get compiled, but the
+    jit hook won't be called for that.
     """
+    cache = space.fromcache(Cache)
+    cache.w_compile_hook = w_hook
+    cache.in_recursion = NonConstant(False)
 
-    def __init__(self, mp_no, offset, pycode):
-        self.mp_no = mp_no
+def set_optimize_hook(space, w_hook):
+    """ set_optimize_hook(hook)
+
+    Set a compiling hook that will be called each time a loop is optimized,
+    but before assembler compilation. This allows to add additional
+    optimizations on Python level.
+    
+    The hook will be called with the following signature:
+    hook(jitdriver_name, loop_type, greenkey or guard_number, operations)
+
+    jitdriver_name is the name of this particular jitdriver, 'pypyjit' is
+    the main interpreter loop
+
+    loop_type can be either `loop` `entry_bridge` or `bridge`
+    in case loop is not `bridge`, greenkey will be a tuple of constants
+    or a string describing it.
+
+    for the interpreter loop` it'll be a tuple
+    (code, offset, is_being_profiled)
+
+    Note that jit hook is not reentrant. It means that if the code
+    inside the jit hook is itself jitted, it will get compiled, but the
+    jit hook won't be called for that.
+
+    Result value will be the resulting list of operations, or None
+    """
+    cache = space.fromcache(Cache)
+    cache.w_optimize_hook = w_hook
+    cache.in_recursion = NonConstant(False)
+
+def set_abort_hook(space, w_hook):
+    """ set_abort_hook(hook)
+
+    Set a hook (callable) that will be called each time there is tracing
+    aborted due to some reason.
+
+    The hook will be called as in: hook(jitdriver_name, greenkey, reason)
+
+    Where reason is the reason for abort, see documentation for set_compile_hook
+    for descriptions of other arguments.
+    """
+    cache = space.fromcache(Cache)
+    cache.w_abort_hook = w_hook
+    cache.in_recursion = NonConstant(False)
+
+def wrap_oplist(space, logops, operations, ops_offset=None):
+    l_w = []
+    jitdrivers_sd = logops.metainterp_sd.jitdrivers_sd
+    for op in operations:
+        if ops_offset is None:
+            ofs = -1
+        else:
+            ofs = ops_offset.get(op, 0)
+        if op.opnum == rop.DEBUG_MERGE_POINT:
+            jd_sd = jitdrivers_sd[op.getarg(0).getint()]
+            greenkey = op.getarglist()[2:]
+            repr = jd_sd.warmstate.get_location_str(greenkey)
+            w_greenkey = wrap_greenkey(space, jd_sd.jitdriver, greenkey, repr)
+            l_w.append(DebugMergePoint(space, jit_hooks._cast_to_gcref(op),
+                                       logops.repr_of_resop(op),
+                                       jd_sd.jitdriver.name,
+                                       w_greenkey))
+        else:
+            l_w.append(WrappedOp(jit_hooks._cast_to_gcref(op), ofs,
+                                 logops.repr_of_resop(op)))
+    return l_w
+
+class WrappedBox(Wrappable):
+    """ A class representing a single box
+    """
+    def __init__(self, llbox):
+        self.llbox = llbox
+
+    def descr_getint(self, space):
+        return space.wrap(jit_hooks.box_getint(self.llbox))
+
+ at unwrap_spec(no=int)
+def descr_new_box(space, w_tp, no):
+    return WrappedBox(jit_hooks.boxint_new(no))
+
+WrappedBox.typedef = TypeDef(
+    'Box',
+    __new__ = interp2app(descr_new_box),
+    getint = interp2app(WrappedBox.descr_getint),
+)
+
+ at unwrap_spec(num=int, offset=int, repr=str, res=WrappedBox)
+def descr_new_resop(space, w_tp, num, w_args, res, offset=-1,
+                    repr=''):
+    args = [space.interp_w(WrappedBox, w_arg).llbox for w_arg in
+            space.listview(w_args)]
+    if res is None:
+        llres = jit_hooks.emptyval()
+    else:
+        llres = res.llbox
+    return WrappedOp(jit_hooks.resop_new(num, args, llres), offset, repr)
+
+ at unwrap_spec(repr=str, jd_name=str)
+def descr_new_dmp(space, w_tp, w_args, repr, jd_name, w_greenkey):
+    args = [space.interp_w(WrappedBox, w_arg).llbox for w_arg in
+            space.listview(w_args)]
+    num = rop.DEBUG_MERGE_POINT
+    return DebugMergePoint(space,
+                           jit_hooks.resop_new(num, args, jit_hooks.emptyval()),
+                           repr, jd_name, w_greenkey)
+
+class WrappedOp(Wrappable):
+    """ A class representing a single ResOperation, wrapped nicely
+    """
+    def __init__(self, op, offset, repr_of_resop):
+        self.op = op
         self.offset = offset
-        self.pycode = pycode
+        self.repr_of_resop = repr_of_resop
 
     def descr_repr(self, space):
-        return space.wrap('DebugMergePoint()')
+        return space.wrap(self.repr_of_resop)
 
- at unwrap_spec(mp_no=int, offset=int, pycode=PyCode)
-def new_debug_merge_point(space, w_tp, mp_no, offset, pycode):
-    return W_DebugMergePoint(mp_no, offset, pycode)
+    def descr_num(self, space):
+        return space.wrap(jit_hooks.resop_getopnum(self.op))
 
-def debug_merge_point_from_boxes(boxes):
-    mp_no = boxes[0].getint()
-    offset = boxes[2].getint()
-    llcode = lltype.cast_opaque_ptr(lltype.Ptr(OBJECT),
-                                    boxes[4].getref_base())
-    pycode = cast_base_ptr_to_instance(PyCode, llcode)
-    assert pycode is not None
-    return W_DebugMergePoint(mp_no, offset, pycode)
+    def descr_name(self, space):
+        return space.wrap(hlstr(jit_hooks.resop_getopname(self.op)))
 
-W_DebugMergePoint.typedef = TypeDef(
-    'DebugMergePoint',
-    __new__ = interp2app(new_debug_merge_point),
-    __doc__ = W_DebugMergePoint.__doc__,
-    __repr__ = interp2app(W_DebugMergePoint.descr_repr),
-    code = interp_attrproperty('pycode', W_DebugMergePoint),
+    @unwrap_spec(no=int)
+    def descr_getarg(self, space, no):
+        return WrappedBox(jit_hooks.resop_getarg(self.op, no))
+
+    @unwrap_spec(no=int, box=WrappedBox)
+    def descr_setarg(self, space, no, box):
+        jit_hooks.resop_setarg(self.op, no, box.llbox)
+
+    def descr_getresult(self, space):
+        return WrappedBox(jit_hooks.resop_getresult(self.op))
+
+    def descr_setresult(self, space, w_box):
+        box = space.interp_w(WrappedBox, w_box)
+        jit_hooks.resop_setresult(self.op, box.llbox)
+
+class DebugMergePoint(WrappedOp):
+    def __init__(self, space, op, repr_of_resop, jd_name, w_greenkey):
+        WrappedOp.__init__(self, op, -1, repr_of_resop)
+        self.w_greenkey = w_greenkey
+        self.jd_name = jd_name
+
+    def get_pycode(self, space):
+        if self.jd_name == pypyjitdriver.name:
+            return space.getitem(self.w_greenkey, space.wrap(0))
+        raise OperationError(space.w_AttributeError, space.wrap("This DebugMergePoint doesn't belong to the main Python JitDriver"))
+
+    def get_bytecode_no(self, space):
+        if self.jd_name == pypyjitdriver.name:
+            return space.getitem(self.w_greenkey, space.wrap(1))
+        raise OperationError(space.w_AttributeError, space.wrap("This DebugMergePoint doesn't belong to the main Python JitDriver"))
+
+    def get_jitdriver_name(self, space):
+        return space.wrap(self.jd_name)
+
+WrappedOp.typedef = TypeDef(
+    'ResOperation',
+    __doc__ = WrappedOp.__doc__,
+    __new__ = interp2app(descr_new_resop),
+    __repr__ = interp2app(WrappedOp.descr_repr),
+    num = GetSetProperty(WrappedOp.descr_num),
+    name = GetSetProperty(WrappedOp.descr_name),
+    getarg = interp2app(WrappedOp.descr_getarg),
+    setarg = interp2app(WrappedOp.descr_setarg),
+    result = GetSetProperty(WrappedOp.descr_getresult,
+                            WrappedOp.descr_setresult)
 )
+WrappedOp.acceptable_as_base_class = False
+
+DebugMergePoint.typedef = TypeDef(
+    'DebugMergePoint', WrappedOp.typedef,
+    __new__ = interp2app(descr_new_dmp),
+    greenkey = interp_attrproperty_w("w_greenkey", cls=DebugMergePoint),
+    pycode = GetSetProperty(DebugMergePoint.get_pycode),
+    bytecode_no = GetSetProperty(DebugMergePoint.get_bytecode_no),
+    jitdriver_name = GetSetProperty(DebugMergePoint.get_jitdriver_name),
+)
+DebugMergePoint.acceptable_as_base_class = False
+
+
diff --git a/pypy/module/pypyjit/policy.py b/pypy/module/pypyjit/policy.py
--- a/pypy/module/pypyjit/policy.py
+++ b/pypy/module/pypyjit/policy.py
@@ -1,4 +1,112 @@
 from pypy.jit.codewriter.policy import JitPolicy
+from pypy.rlib.jit import JitHookInterface
+from pypy.rlib import jit_hooks
+from pypy.interpreter.error import OperationError
+from pypy.jit.metainterp.jitprof import counter_names
+from pypy.module.pypyjit.interp_resop import wrap_oplist, Cache, wrap_greenkey,\
+     WrappedOp
+
+class PyPyJitIface(JitHookInterface):
+    def on_abort(self, reason, jitdriver, greenkey, greenkey_repr):
+        space = self.space
+        cache = space.fromcache(Cache)
+        if cache.in_recursion:
+            return
+        if space.is_true(cache.w_abort_hook):
+            cache.in_recursion = True
+            try:
+                try:
+                    space.call_function(cache.w_abort_hook,
+                                        space.wrap(jitdriver.name),
+                                        wrap_greenkey(space, jitdriver,
+                                                      greenkey, greenkey_repr),
+                                        space.wrap(counter_names[reason]))
+                except OperationError, e:
+                    e.write_unraisable(space, "jit hook ", cache.w_abort_hook)
+            finally:
+                cache.in_recursion = False
+
+    def after_compile(self, debug_info):
+        w_greenkey = wrap_greenkey(self.space, debug_info.get_jitdriver(),
+                                   debug_info.greenkey,
+                                   debug_info.get_greenkey_repr())
+        self._compile_hook(debug_info, w_greenkey)
+
+    def after_compile_bridge(self, debug_info):
+        self._compile_hook(debug_info,
+                           self.space.wrap(debug_info.fail_descr_no))
+
+    def before_compile(self, debug_info):
+        w_greenkey = wrap_greenkey(self.space, debug_info.get_jitdriver(),
+                                   debug_info.greenkey,
+                                   debug_info.get_greenkey_repr())
+        self._optimize_hook(debug_info, w_greenkey)
+
+    def before_compile_bridge(self, debug_info):
+        self._optimize_hook(debug_info,
+                            self.space.wrap(debug_info.fail_descr_no))
+
+    def _compile_hook(self, debug_info, w_arg):
+        space = self.space
+        cache = space.fromcache(Cache)
+        if cache.in_recursion:
+            return
+        if space.is_true(cache.w_compile_hook):
+            logops = debug_info.logger._make_log_operations()
+            list_w = wrap_oplist(space, logops, debug_info.operations,
+                                 debug_info.asminfo.ops_offset)
+            cache.in_recursion = True
+            try:
+                try:
+                    jd_name = debug_info.get_jitdriver().name
+                    asminfo = debug_info.asminfo
+                    space.call_function(cache.w_compile_hook,
+                                        space.wrap(jd_name),
+                                        space.wrap(debug_info.type),
+                                        w_arg,
+                                        space.newlist(list_w),
+                                        space.wrap(asminfo.asmaddr),
+                                        space.wrap(asminfo.asmlen))
+                except OperationError, e:
+                    e.write_unraisable(space, "jit hook ", cache.w_compile_hook)
+            finally:
+                cache.in_recursion = False
+
+    def _optimize_hook(self, debug_info, w_arg):
+        space = self.space
+        cache = space.fromcache(Cache)
+        if cache.in_recursion:
+            return
+        if space.is_true(cache.w_optimize_hook):
+            logops = debug_info.logger._make_log_operations()
+            list_w = wrap_oplist(space, logops, debug_info.operations)
+            cache.in_recursion = True
+            try:
+                try:
+                    jd_name = debug_info.get_jitdriver().name
+                    w_res = space.call_function(cache.w_optimize_hook,
+                                                space.wrap(jd_name),
+                                                space.wrap(debug_info.type),
+                                                w_arg,
+                                                space.newlist(list_w))
+                    if space.is_w(w_res, space.w_None):
+                        return
+                    l = []
+                    for w_item in space.listview(w_res):
+                        item = space.interp_w(WrappedOp, w_item)
+                        l.append(jit_hooks._cast_to_resop(item.op))
+                    del debug_info.operations[:] # modifying operations above is
+                    # probably not a great idea since types may not work
+                    # and we'll end up with half-working list and
+                    # a segfault/fatal RPython error
+                    for elem in l:
+                        debug_info.operations.append(elem)
+                except OperationError, e:
+                    e.write_unraisable(space, "jit hook ", cache.w_compile_hook)
+            finally:
+                cache.in_recursion = False
+
+pypy_hooks = PyPyJitIface()
 
 class PyPyJitPolicy(JitPolicy):
 
@@ -12,12 +120,16 @@
                 modname == 'thread.os_thread'):
             return True
         if '.' in modname:
-            modname, _ = modname.split('.', 1)
+            modname, rest = modname.split('.', 1)
+        else:
+            rest = ''
         if modname in ['pypyjit', 'signal', 'micronumpy', 'math', 'exceptions',
                        'imp', 'sys', 'array', '_ffi', 'itertools', 'operator',
                        'posix', '_socket', '_sre', '_lsprof', '_weakref',
                        '__pypy__', 'cStringIO', '_collections', 'struct',
-                       'mmap', 'marshal']:
+                       'mmap', 'marshal', '_codecs']:
+            if modname == 'pypyjit' and 'interp_resop' in rest:
+                return False
             return True
         return False
 
diff --git a/pypy/module/pypyjit/test/test_jit_hook.py b/pypy/module/pypyjit/test/test_jit_hook.py
--- a/pypy/module/pypyjit/test/test_jit_hook.py
+++ b/pypy/module/pypyjit/test/test_jit_hook.py
@@ -1,22 +1,40 @@
 
 import py
 from pypy.conftest import gettestobjspace, option
+from pypy.interpreter.gateway import interp2app
 from pypy.interpreter.pycode import PyCode
-from pypy.interpreter.gateway import interp2app
-from pypy.jit.metainterp.history import JitCellToken
-from pypy.jit.metainterp.resoperation import ResOperation, rop
+from pypy.jit.metainterp.history import JitCellToken, ConstInt, ConstPtr
+from pypy.jit.metainterp.resoperation import rop
 from pypy.jit.metainterp.logger import Logger
 from pypy.rpython.annlowlevel import (cast_instance_to_base_ptr,
                                       cast_base_ptr_to_instance)
 from pypy.rpython.lltypesystem import lltype, llmemory
+from pypy.rpython.lltypesystem.rclass import OBJECT
 from pypy.module.pypyjit.interp_jit import pypyjitdriver
+from pypy.module.pypyjit.policy import pypy_hooks
 from pypy.jit.tool.oparser import parse
 from pypy.jit.metainterp.typesystem import llhelper
+from pypy.jit.metainterp.jitprof import ABORT_TOO_LONG
+from pypy.rlib.jit import JitDebugInfo, AsmInfo
+
+class MockJitDriverSD(object):
+    class warmstate(object):
+        @staticmethod
+        def get_location_str(boxes):
+            ll_code = lltype.cast_opaque_ptr(lltype.Ptr(OBJECT),
+                                             boxes[2].getref_base())
+            pycode = cast_base_ptr_to_instance(PyCode, ll_code)
+            return pycode.co_name
+
+    jitdriver = pypyjitdriver
+
 
 class MockSD(object):
     class cpu(object):
         ts = llhelper
 
+    jitdrivers_sd = [MockJitDriverSD]
+
 class AppTestJitHook(object):
     def setup_class(cls):
         if option.runappdirect:
@@ -24,9 +42,9 @@
         space = gettestobjspace(usemodules=('pypyjit',))
         cls.space = space
         w_f = space.appexec([], """():
-        def f():
+        def function():
             pass
-        return f
+        return function
         """)
         cls.w_f = w_f
         ll_code = cast_instance_to_base_ptr(w_f.code)
@@ -34,41 +52,78 @@
         logger = Logger(MockSD())
 
         oplist = parse("""
-        [i1, i2]
+        [i1, i2, p2]
         i3 = int_add(i1, i2)
         debug_merge_point(0, 0, 0, 0, ConstPtr(ptr0))
+        guard_nonnull(p2) []
         guard_true(i3) []
         """, namespace={'ptr0': code_gcref}).operations
+        greenkey = [ConstInt(0), ConstInt(0), ConstPtr(code_gcref)]
+        offset = {}
+        for i, op in enumerate(oplist):
+            if i != 1:
+               offset[op] = i
+
+        di_loop = JitDebugInfo(MockJitDriverSD, logger, JitCellToken(),
+                               oplist, 'loop', greenkey)
+        di_loop_optimize = JitDebugInfo(MockJitDriverSD, logger, JitCellToken(),
+                                        oplist, 'loop', greenkey)
+        di_loop.asminfo = AsmInfo(offset, 0, 0)
+        di_bridge = JitDebugInfo(MockJitDriverSD, logger, JitCellToken(),
+                                 oplist, 'bridge', fail_descr_no=0)
+        di_bridge.asminfo = AsmInfo(offset, 0, 0)
 
         def interp_on_compile():
-            pypyjitdriver.on_compile(logger, JitCellToken(), oplist, 'loop',
-                                     0, False, ll_code)
+            di_loop.oplist = cls.oplist
+            pypy_hooks.after_compile(di_loop)
 
         def interp_on_compile_bridge():
-            pypyjitdriver.on_compile_bridge(logger, JitCellToken(), oplist, 0)
+            pypy_hooks.after_compile_bridge(di_bridge)
+
+        def interp_on_optimize():
+            di_loop_optimize.oplist = cls.oplist
+            pypy_hooks.before_compile(di_loop_optimize)
+
+        def interp_on_abort():
+            pypy_hooks.on_abort(ABORT_TOO_LONG, pypyjitdriver, greenkey,
+                                'blah')
         
         cls.w_on_compile = space.wrap(interp2app(interp_on_compile))
         cls.w_on_compile_bridge = space.wrap(interp2app(interp_on_compile_bridge))
+        cls.w_on_abort = space.wrap(interp2app(interp_on_abort))
+        cls.w_int_add_num = space.wrap(rop.INT_ADD)
+        cls.w_dmp_num = space.wrap(rop.DEBUG_MERGE_POINT)
+        cls.w_on_optimize = space.wrap(interp2app(interp_on_optimize))
+        cls.orig_oplist = oplist
+
+    def setup_method(self, meth):
+        self.__class__.oplist = self.orig_oplist[:]
 
     def test_on_compile(self):
         import pypyjit
         all = []
 
-        def hook(*args):
-            assert args[0] == 'main'
-            assert args[1] in ['loop', 'bridge']
-            all.append(args[2:])
+        def hook(name, looptype, tuple_or_guard_no, ops, asmstart, asmlen):
+            all.append((name, looptype, tuple_or_guard_no, ops))
         
         self.on_compile()
         pypyjit.set_compile_hook(hook)
         assert not all
         self.on_compile()
         assert len(all) == 1
-        assert all[0][0][0].co_name == 'f'
-        assert all[0][0][1] == 0
-        assert all[0][0][2] == False
-        assert len(all[0][1]) == 3
-        assert 'int_add' in all[0][1][0]
+        elem = all[0]
+        assert elem[0] == 'pypyjit'
+        assert elem[2][0].co_name == 'function'
+        assert elem[2][1] == 0
+        assert elem[2][2] == False
+        assert len(elem[3]) == 4
+        int_add = elem[3][0]
+        dmp = elem[3][1]
+        assert isinstance(dmp, pypyjit.DebugMergePoint)
+        assert dmp.pycode is self.f.func_code
+        assert dmp.greenkey == (self.f.func_code, 0, False)
+        #assert int_add.name == 'int_add'
+        assert int_add.num == self.int_add_num
         self.on_compile_bridge()
         assert len(all) == 2
         pypyjit.set_compile_hook(None)
@@ -116,11 +171,63 @@
 
         pypyjit.set_compile_hook(hook)
         self.on_compile()
-        dmp = l[0][3][1]
-        assert isinstance(dmp, pypyjit.DebugMergePoint)
-        assert dmp.code is self.f.func_code
+        op = l[0][3][1]
+        assert isinstance(op, pypyjit.ResOperation)
+        assert 'function' in repr(op)
+
+    def test_on_abort(self):
+        import pypyjit
+        l = []
+
+        def hook(jitdriver_name, greenkey, reason):
+            l.append((jitdriver_name, reason))
+        
+        pypyjit.set_abort_hook(hook)
+        self.on_abort()
+        assert l == [('pypyjit', 'ABORT_TOO_LONG')]
+
+    def test_on_optimize(self):
+        import pypyjit
+        l = []
+
+        def hook(name, looptype, tuple_or_guard_no, ops, *args):
+            l.append(ops)
+
+        def optimize_hook(name, looptype, tuple_or_guard_no, ops):
+            return []
+
+        pypyjit.set_compile_hook(hook)
+        pypyjit.set_optimize_hook(optimize_hook)
+        self.on_optimize()
+        self.on_compile()
+        assert l == [[]]
 
     def test_creation(self):
-        import pypyjit
-        dmp = pypyjit.DebugMergePoint(0, 0, self.f.func_code)
-        assert dmp.code is self.f.func_code 
+        from pypyjit import Box, ResOperation
+
+        op = ResOperation(self.int_add_num, [Box(1), Box(3)], Box(4))
+        assert op.num == self.int_add_num
+        assert op.name == 'int_add'
+        box = op.getarg(0)
+        assert box.getint() == 1
+        box2 = op.result
+        assert box2.getint() == 4
+        op.setarg(0, box2)
+        assert op.getarg(0).getint() == 4
+        op.result = box
+        assert op.result.getint() == 1
+
+    def test_creation_dmp(self):
+        from pypyjit import DebugMergePoint, Box
+
+        def f():
+            pass
+
+        op = DebugMergePoint([Box(0)], 'repr', 'pypyjit', (f.func_code, 0, 0))
+        assert op.bytecode_no == 0
+        assert op.pycode is f.func_code
+        assert repr(op) == 'repr'
+        assert op.jitdriver_name == 'pypyjit'
+        assert op.num == self.dmp_num
+        op = DebugMergePoint([Box(0)], 'repr', 'notmain', ('str',))
+        raises(AttributeError, 'op.pycode')
diff --git a/pypy/module/pypyjit/test/test_policy.py b/pypy/module/pypyjit/test/test_policy.py
--- a/pypy/module/pypyjit/test/test_policy.py
+++ b/pypy/module/pypyjit/test/test_policy.py
@@ -52,6 +52,7 @@
     for modname in 'pypyjit', 'signal', 'micronumpy', 'math', 'imp':
         assert pypypolicy.look_inside_pypy_module(modname)
         assert pypypolicy.look_inside_pypy_module(modname + '.foo')
+    assert not pypypolicy.look_inside_pypy_module('pypyjit.interp_resop')
 
 def test_see_jit_module():
     assert pypypolicy.look_inside_pypy_module('pypyjit.interp_jit')
diff --git a/pypy/module/pypyjit/test/test_ztranslation.py b/pypy/module/pypyjit/test/test_ztranslation.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/pypyjit/test/test_ztranslation.py
@@ -0,0 +1,5 @@
+
+from pypy.objspace.fake.checkmodule import checkmodule
+
+def test_pypyjit_translates():
+    checkmodule('pypyjit')
diff --git a/pypy/module/pypyjit/test_pypy_c/test_00_model.py b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
--- a/pypy/module/pypyjit/test_pypy_c/test_00_model.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
@@ -8,10 +8,12 @@
 from pypy.tool import logparser
 from pypy.jit.tool.jitoutput import parse_prof
 from pypy.module.pypyjit.test_pypy_c.model import (Log, find_ids_range,
-                                                   find_ids, TraceWithIds,
+                                                   find_ids,
                                                    OpMatcher, InvalidMatch)
 
 class BaseTestPyPyC(object):
+    log_string = 'jit-log-opt,jit-log-noopt,jit-log-virtualstate,jit-summary'
+    
     def setup_class(cls):
         if '__pypy__' not in sys.builtin_module_names:
             py.test.skip("must run this test with pypy")
@@ -52,8 +54,7 @@
             cmdline += ['--jit', ','.join(jitcmdline)]
         cmdline.append(str(self.filepath))
         #
-        print cmdline, logfile
-        env={'PYPYLOG': 'jit-log-opt,jit-log-noopt,jit-log-virtualstate,jit-summary:' + str(logfile)}
+        env={'PYPYLOG': self.log_string + ':' + str(logfile)}
         pipe = subprocess.Popen(cmdline,
                                 env=env,
                                 stdout=subprocess.PIPE,
diff --git a/pypy/module/pypyjit/test_pypy_c/test__ffi.py b/pypy/module/pypyjit/test_pypy_c/test__ffi.py
--- a/pypy/module/pypyjit/test_pypy_c/test__ffi.py
+++ b/pypy/module/pypyjit/test_pypy_c/test__ffi.py
@@ -98,7 +98,8 @@
             end = time.time()
             return end - start
         #
-        log = self.run(main, [get_libc_name(), 200], threshold=150)
+        log = self.run(main, [get_libc_name(), 200], threshold=150,
+                       import_site=True)
         assert 1 <= log.result <= 1.5 # at most 0.5 seconds of overhead
         loops = log.loops_by_id('sleep')
         assert len(loops) == 1 # make sure that we actually JITted the loop
@@ -121,7 +122,7 @@
             return fabs._ptr.getaddr(), x
 
         libm_name = get_libm_name(sys.platform)
-        log = self.run(main, [libm_name])
+        log = self.run(main, [libm_name], import_site=True)
         fabs_addr, res = log.result
         assert res == -4.0
         loop, = log.loops_by_filename(self.filepath)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -15,7 +15,7 @@
                 i += letters[i % len(letters)] == uletters[i % len(letters)]
             return i
 
-        log = self.run(main, [300])
+        log = self.run(main, [300], import_site=True)
         assert log.result == 300
         loop, = log.loops_by_filename(self.filepath)
         assert loop.match("""
@@ -55,7 +55,7 @@
                 i += int(long(string.digits[i % len(string.digits)], 16))
             return i
 
-        log = self.run(main, [1100])
+        log = self.run(main, [1100], import_site=True)
         assert log.result == main(1100)
         loop, = log.loops_by_filename(self.filepath)
         assert loop.match("""
diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -42,7 +42,7 @@
         'argv'                  : 'state.get(space).w_argv',
         'py3kwarning'           : 'space.w_False',
         'warnoptions'           : 'state.get(space).w_warnoptions', 
-        'builtin_module_names'  : 'state.w_None',
+        'builtin_module_names'  : 'space.w_None',
         'pypy_getudir'          : 'state.pypy_getudir',    # not translated
         'pypy_initial_path'     : 'state.pypy_initial_path',
 
diff --git a/pypy/module/sys/app.py b/pypy/module/sys/app.py
--- a/pypy/module/sys/app.py
+++ b/pypy/module/sys/app.py
@@ -66,11 +66,11 @@
     return None
 
 copyright_str = """
-Copyright 2003-2011 PyPy development team.
+Copyright 2003-2012 PyPy development team.
 All Rights Reserved.
 For further information, see 
 
-Portions Copyright (c) 2001-2008 Python Software Foundation.
+Portions Copyright (c) 2001-2012 Python Software Foundation.
 All Rights Reserved.
 
 Portions Copyright (c) 2000 BeOpen.com.
diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py
--- a/pypy/module/sys/version.py
+++ b/pypy/module/sys/version.py
@@ -10,7 +10,7 @@
 CPYTHON_VERSION            = (2, 7, 1, "final", 42)   #XXX # sync patchlevel.h
 CPYTHON_API_VERSION        = 1013   #XXX # sync with include/modsupport.h
 
-PYPY_VERSION               = (1, 7, 1, "dev", 0)    #XXX # sync patchlevel.h
+PYPY_VERSION               = (1, 8, 1, "dev", 0)    #XXX # sync patchlevel.h
 
 if platform.name == 'msvc':
     COMPILER_INFO = 'MSC v.%d 32 bit' % (platform.version * 10 + 600)
diff --git a/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
@@ -0,0 +1,137 @@
+from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
+
+
+class AppTestFromNumeric(BaseNumpyAppTest):
+    def test_argmax(self):
+        # tests taken from numpy/core/fromnumeric.py docstring
+        from numpypy import array, arange, argmax
+        a = arange(6).reshape((2,3))
+        assert argmax(a) == 5
+        # assert (argmax(a, axis=0) == array([1, 1, 1])).all()
+        # assert (argmax(a, axis=1) == array([2, 2])).all()
+        b = arange(6)
+        b[1] = 5
+        assert argmax(b) == 1
+
+    def test_argmin(self):
+        # tests adapted from test_argmax
+        from numpypy import array, arange, argmin
+        a = arange(6).reshape((2,3))
+        assert argmin(a) == 0
+        assert (argmin(a, axis=0) == array([0, 0, 0])).all()
+        assert (argmin(a, axis=1) == array([0, 0])).all()
+        b = arange(6)
+        b[1] = 0
+        assert argmin(b) == 0
+
+    def test_shape(self):
+        # tests taken from numpy/core/fromnumeric.py docstring
+        from numpypy import array, identity, shape
+        assert shape(identity(3)) == (3, 3)
+        assert shape([[1, 2]]) == (1, 2)
+        assert shape([0]) ==  (1,)
+        assert shape(0) == ()
+        # a = array([(1, 2), (3, 4)], dtype=[('x', 'i4'), ('y', 'i4')])
+        # assert shape(a) == (2,)
+
+    def test_sum(self):
+        # tests taken from numpy/core/fromnumeric.py docstring
+        from numpypy import array, sum, ones
+        assert sum([0.5, 1.5])== 2.0
+        assert sum([[0, 1], [0, 5]]) == 6
+        # assert sum([0.5, 0.7, 0.2, 1.5], dtype=int32) == 1
+        # assert (sum([[0, 1], [0, 5]], axis=0) == array([0, 6])).all()
+        # assert (sum([[0, 1], [0, 5]], axis=1) == array([1, 5])).all()
+        # If the accumulator is too small, overflow occurs:
+        # assert ones(128, dtype=int8).sum(dtype=int8) == -128
+
+    def test_amin(self):
+        # tests taken from numpy/core/fromnumeric.py docstring
+        from numpypy import array, arange, amin
+        a = arange(4).reshape((2,2))
+        assert amin(a) == 0
+        # # Minima along the first axis
+        # assert (amin(a, axis=0) == array([0, 1])).all()
+        # # Minima along the second axis
+        # assert (amin(a, axis=1) == array([0, 2])).all()
+        # # NaN behaviour
+        # b = arange(5, dtype=float)
+        # b[2] = NaN
+        # assert amin(b) == nan
+        # assert nanmin(b) == 0.0
+
+    def test_amax(self):
+        # tests taken from numpy/core/fromnumeric.py docstring
+        from numpypy import array, arange, amax
+        a = arange(4).reshape((2,2))
+        assert amax(a) == 3
+        # assert (amax(a, axis=0) == array([2, 3])).all()
+        # assert (amax(a, axis=1) == array([1, 3])).all()
+        # # NaN behaviour
+        # b = arange(5, dtype=float)
+        # b[2] = NaN
+        # assert amax(b) == nan
+        # assert nanmax(b) == 4.0
+
+    def test_alen(self):
+        # tests taken from numpy/core/fromnumeric.py docstring
+        from numpypy import array, zeros, alen
+        a = zeros((7,4,5))
+        assert a.shape[0] == 7
+        assert alen(a)    == 7
+
+    def test_ndim(self):
+        # tests taken from numpy/core/fromnumeric.py docstring
+        from numpypy import array, ndim
+        assert ndim([[1,2,3],[4,5,6]]) == 2
+        assert ndim(array([[1,2,3],[4,5,6]])) == 2
+        assert ndim(1) == 0
+
+    def test_rank(self):
+        # tests taken from numpy/core/fromnumeric.py docstring
+        from numpypy import array, rank
+        assert rank([[1,2,3],[4,5,6]]) == 2
+        assert rank(array([[1,2,3],[4,5,6]])) == 2
+        assert rank(1) == 0
+
+    def test_var(self):
+        from numpypy import array, var
+        a = array([[1,2],[3,4]])
+        assert var(a) == 1.25
+        # assert (np.var(a,0) == array([ 1.,  1.])).all()
+        # assert (np.var(a,1) == array([ 0.25,  0.25])).all()
+
+    def test_std(self):
+        from numpypy import array, std
+        a = array([[1, 2], [3, 4]])
+        assert std(a) ==  1.1180339887498949
+        # assert (std(a, axis=0) == array([ 1.,  1.])).all()
+        # assert (std(a, axis=1) == array([ 0.5,  0.5]).all()
+
+    def test_mean(self):
+        from numpypy import array, mean
+        assert mean(array(range(5))) == 2.0
+        assert mean(range(5)) == 2.0
+
+    def test_reshape(self):
+        from numpypy import arange, array, dtype, reshape
+        a = arange(12)
+        b = reshape(a, (3, 4))
+        assert b.shape == (3, 4)
+        a = range(12)
+        b = reshape(a, (3, 4))
+        assert b.shape == (3, 4)
+        a = array(range(105)).reshape(3, 5, 7)
+        assert reshape(a, (1, -1)).shape == (1, 105)
+        assert reshape(a, (1, 1, -1)).shape == (1, 1, 105)
+        assert reshape(a, (-1, 1, 1)).shape == (105, 1, 1)
+    
+    def test_transpose(self):   
+        from numpypy import arange, array, transpose, ones
+        x = arange(4).reshape((2,2))
+        assert (transpose(x) == array([[0, 2],[1, 3]])).all()
+        # Once axes argument is implemented, add more tests
+        raises(NotImplementedError, "transpose(x, axes=(1, 0, 2))")
+        # x = ones((1, 2, 3))
+        # assert transpose(x, (1, 0, 2)).shape == (2, 1, 3)
+
diff --git a/pypy/objspace/fake/checkmodule.py b/pypy/objspace/fake/checkmodule.py
--- a/pypy/objspace/fake/checkmodule.py
+++ b/pypy/objspace/fake/checkmodule.py
@@ -1,8 +1,10 @@
 from pypy.objspace.fake.objspace import FakeObjSpace, W_Root
+from pypy.config.pypyoption import get_pypy_config
 
 
 def checkmodule(modname):
-    space = FakeObjSpace()
+    config = get_pypy_config(translating=True)
+    space = FakeObjSpace(config)
     mod = __import__('pypy.module.%s' % modname, None, None, ['__doc__'])
     # force computation and record what we wrap
     module = mod.Module(space, W_Root())
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -93,9 +93,9 @@
 
 class FakeObjSpace(ObjSpace):
 
-    def __init__(self):
+    def __init__(self, config=None):
         self._seen_extras = []
-        ObjSpace.__init__(self)
+        ObjSpace.__init__(self, config=config)
 
     def float_w(self, w_obj):
         is_root(w_obj)
@@ -135,6 +135,9 @@
     def newfloat(self, x):
         return w_some_obj()
 
+    def newcomplex(self, x, y):
+        return w_some_obj()
+
     def marshal_w(self, w_obj):
         "NOT_RPYTHON"
         raise NotImplementedError
@@ -215,6 +218,10 @@
             expected_length = 3
         return [w_some_obj()] * expected_length
 
+    def unpackcomplex(self, w_complex):
+        is_root(w_complex)
+        return 1.1, 2.2
+
     def allocate_instance(self, cls, w_subtype):
         is_root(w_subtype)
         return instantiate(cls)
@@ -232,6 +239,11 @@
     def exec_(self, *args, **kwds):
         pass
 
+    def createexecutioncontext(self):
+        ec = ObjSpace.createexecutioncontext(self)
+        ec._py_repr = None
+        return ec
+
     # ----------
 
     def translates(self, func=None, argtypes=None, **kwds):
@@ -267,18 +279,21 @@
                  ObjSpace.ExceptionTable +
                  ['int', 'str', 'float', 'long', 'tuple', 'list',
                   'dict', 'unicode', 'complex', 'slice', 'bool',
-                  'type', 'basestring']):
+                  'type', 'basestring', 'object']):
         setattr(FakeObjSpace, 'w_' + name, w_some_obj())
     #
     for (name, _, arity, _) in ObjSpace.MethodTable:
         args = ['w_%d' % i for i in range(arity)]
+        params = args[:]
         d = {'is_root': is_root,
              'w_some_obj': w_some_obj}
+        if name in ('get',):
+            params[-1] += '=None'
         exec compile2("""\
             def meth(self, %s):
                 %s
                 return w_some_obj()
-        """ % (', '.join(args),
+        """ % (', '.join(params),
                '; '.join(['is_root(%s)' % arg for arg in args]))) in d
         meth = func_with_new_name(d['meth'], name)
         setattr(FakeObjSpace, name, meth)
@@ -301,9 +316,12 @@
     pass
 FakeObjSpace.default_compiler = FakeCompiler()
 
-class FakeModule(object):
+class FakeModule(Wrappable):
+    def __init__(self):
+        self.w_dict = w_some_obj()
     def get(self, name):
         name + "xx"   # check that it's a string
         return w_some_obj()
 FakeObjSpace.sys = FakeModule()
 FakeObjSpace.sys.filesystemencoding = 'foobar'
+FakeObjSpace.builtin = FakeModule()
diff --git a/pypy/objspace/fake/test/test_objspace.py b/pypy/objspace/fake/test/test_objspace.py
--- a/pypy/objspace/fake/test/test_objspace.py
+++ b/pypy/objspace/fake/test/test_objspace.py
@@ -40,7 +40,7 @@
     def test_constants(self):
         space = self.space
         space.translates(lambda: (space.w_None, space.w_True, space.w_False,
-                                  space.w_int, space.w_str,
+                                  space.w_int, space.w_str, space.w_object,
                                   space.w_TypeError))
 
     def test_wrap(self):
@@ -72,3 +72,9 @@
 
     def test_newlist(self):
         self.space.newlist([W_Root(), W_Root()])
+
+    def test_default_values(self):
+        # the __get__ method takes either 2 or 3 arguments
+        space = self.space
+        space.translates(lambda: (space.get(W_Root(), W_Root()),
+                                  space.get(W_Root(), W_Root(), W_Root())))
diff --git a/pypy/objspace/std/test/test_unicodeobject.py b/pypy/objspace/std/test/test_unicodeobject.py
--- a/pypy/objspace/std/test/test_unicodeobject.py
+++ b/pypy/objspace/std/test/test_unicodeobject.py
@@ -64,6 +64,12 @@
         check(', '.join([u'a']), u'a')
         check(', '.join(['a', u'b']), u'a, b')
         check(u', '.join(['a', 'b']), u'a, b')
+        try:
+            u''.join([u'a', 2, 3])
+        except TypeError, e:
+            assert 'sequence item 1' in str(e)
+        else:
+            raise Exception("DID NOT RAISE")
 
     if sys.version_info >= (2,3):
         def test_contains_ex(self):
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -201,7 +201,7 @@
     return space.newbool(container.find(item) != -1)
 
 def unicode_join__Unicode_ANY(space, w_self, w_list):
-    list_w = space.unpackiterable(w_list)
+    list_w = space.listview(w_list)
     size = len(list_w)
 
     if size == 0:
@@ -216,22 +216,21 @@
 
 def _unicode_join_many_items(space, w_self, list_w, size):
     self = w_self._value
-    sb = UnicodeBuilder()
+    prealloc_size = len(self) * (size - 1)
+    for i in range(size):
+        try:
+            prealloc_size += len(space.unicode_w(list_w[i]))
+        except OperationError, e:
+            if not e.match(space, space.w_TypeError):
+                raise
+            raise operationerrfmt(space.w_TypeError,
+                        "sequence item %d: expected string or Unicode", i)
+    sb = UnicodeBuilder(prealloc_size)
     for i in range(size):
         if self and i != 0:
             sb.append(self)
         w_s = list_w[i]
-        if isinstance(w_s, W_UnicodeObject):
-            # shortcut for performance
-            sb.append(w_s._value)
-        else:
-            try:
-                sb.append(space.unicode_w(w_s))
-            except OperationError, e:
-                if not e.match(space, space.w_TypeError):
-                    raise
-                raise operationerrfmt(space.w_TypeError,
-                    "sequence item %d: expected string or Unicode", i)
+        sb.append(space.unicode_w(w_s))
     return space.wrap(sb.build())
 
 def hash__Unicode(space, w_uni):
diff --git a/pypy/rlib/clibffi.py b/pypy/rlib/clibffi.py
--- a/pypy/rlib/clibffi.py
+++ b/pypy/rlib/clibffi.py
@@ -30,6 +30,9 @@
 _MAC_OS = platform.name == "darwin"
 _FREEBSD_7 = platform.name == "freebsd7"
 
+_LITTLE_ENDIAN = sys.byteorder == 'little'
+_BIG_ENDIAN = sys.byteorder == 'big'
+
 if _WIN32:
     from pypy.rlib import rwin32
 
@@ -360,12 +363,36 @@
 cast_type_to_ffitype._annspecialcase_ = 'specialize:memo'
 
 def push_arg_as_ffiptr(ffitp, arg, ll_buf):
-    # this is for primitive types. For structures and arrays
-    # would be something different (more dynamic)
+    # This is for primitive types.  Note that the exact type of 'arg' may be
+    # different from the expected 'c_size'.  To cope with that, we fall back
+    # to a byte-by-byte copy.
     TP = lltype.typeOf(arg)
     TP_P = lltype.Ptr(rffi.CArray(TP))
-    buf = rffi.cast(TP_P, ll_buf)
-    buf[0] = arg
+    TP_size = rffi.sizeof(TP)
+    c_size = intmask(ffitp.c_size)
+    # if both types have the same size, we can directly write the
+    # value to the buffer
+    if c_size == TP_size:
+        buf = rffi.cast(TP_P, ll_buf)
+        buf[0] = arg
+    else:
+        # needs byte-by-byte copying.  Make sure 'arg' is an integer type.
+        # Note that this won't work for rffi.FLOAT/rffi.DOUBLE.
+        assert TP is not rffi.FLOAT and TP is not rffi.DOUBLE
+        if TP_size <= rffi.sizeof(lltype.Signed):
+            arg = rffi.cast(lltype.Unsigned, arg)
+        else:
+            arg = rffi.cast(lltype.UnsignedLongLong, arg)
+        if _LITTLE_ENDIAN:
+            for i in range(c_size):
+                ll_buf[i] = chr(arg & 0xFF)
+                arg >>= 8
+        elif _BIG_ENDIAN:
+            for i in range(c_size-1, -1, -1):
+                ll_buf[i] = chr(arg & 0xFF)
+                arg >>= 8
+        else:
+            raise AssertionError
 push_arg_as_ffiptr._annspecialcase_ = 'specialize:argtype(1)'
 
 
diff --git a/pypy/rlib/jit.py b/pypy/rlib/jit.py
--- a/pypy/rlib/jit.py
+++ b/pypy/rlib/jit.py
@@ -6,18 +6,24 @@
 from pypy.rlib.objectmodel import CDefinedIntSymbolic, keepalive_until_here, specialize
 from pypy.rlib.unroll import unrolling_iterable
 from pypy.rpython.extregistry import ExtRegistryEntry
-from pypy.tool.sourcetools import func_with_new_name
 
 DEBUG_ELIDABLE_FUNCTIONS = False
 
 
 def elidable(func):
-    """ Decorate a function as "trace-elidable". This means precisely that:
+    """ Decorate a function as "trace-elidable". Usually this means simply that
+    the function is constant-foldable, i.e. is pure and has no side-effects.
+
+    In some situations it is ok to use this decorator if the function *has*
+    side effects, as long as these side-effects are idempotent. A typical
+    example for this would be a cache.
+
+    To be totally precise:
 
     (1) the result of the call should not change if the arguments are
         the same (same numbers or same pointers)
     (2) it's fine to remove the call completely if we can guess the result
-    according to rule 1
+        according to rule 1
     (3) the function call can be moved around by optimizer,
         but only so it'll be called earlier and not later.
 
@@ -386,6 +392,19 @@
 class JitHintError(Exception):
     """Inconsistency in the JIT hints."""
 
+PARAMETER_DOCS = {
+    'threshold': 'number of times a loop has to run for it to become hot',
+    'function_threshold': 'number of times a function must run for it to become traced from start',
+    'trace_eagerness': 'number of times a guard has to fail before we start compiling a bridge',
+    'trace_limit': 'number of recorded operations before we abort tracing with ABORT_TOO_LONG',
+    'inlining': 'inline python functions or not (1/0)',
+    'loop_longevity': 'a parameter controlling how long loops will be kept before being freed, an estimate',
+    'retrace_limit': 'how many times we can try retracing before giving up',
+    'max_retrace_guards': 'number of extra guards a retrace can cause',
+    'max_unroll_loops': 'number of extra unrollings a loop can cause',
+    'enable_opts': 'optimizations to enable or all, INTERNAL USE ONLY'
+    }
+
 PARAMETERS = {'threshold': 1039, # just above 1024, prime
               'function_threshold': 1619, # slightly more than one above, also prime
               'trace_eagerness': 200,
@@ -394,6 +413,7 @@
               'loop_longevity': 1000,
               'retrace_limit': 5,
               'max_retrace_guards': 15,
+              'max_unroll_loops': 4,
               'enable_opts': 'all',
               }
 unroll_parameters = unrolling_iterable(PARAMETERS.items())
@@ -410,13 +430,16 @@
 
     active = True          # if set to False, this JitDriver is ignored
     virtualizables = []
+    name = 'jitdriver'
 
     def __init__(self, greens=None, reds=None, virtualizables=None,
                  get_jitcell_at=None, set_jitcell_at=None,
                  get_printable_location=None, confirm_enter_jit=None,
-                 can_never_inline=None, should_unroll_one_iteration=None):
+                 can_never_inline=None, should_unroll_one_iteration=None,
+                 name='jitdriver'):
         if greens is not None:
             self.greens = greens
+        self.name = name
         if reds is not None:
             self.reds = reds
         if not hasattr(self, 'greens') or not hasattr(self, 'reds'):
@@ -450,23 +473,6 @@
         # special-cased by ExtRegistryEntry
         pass
 
-    def on_compile(self, logger, looptoken, operations, type, *greenargs):
-        """ A hook called when loop is compiled. Overwrite
-        for your own jitdriver if you want to do something special, like
-        call applevel code
-        """
-
-    def on_compile_bridge(self, logger, orig_looptoken, operations, n):
-        """ A hook called when a bridge is compiled. Overwrite
-        for your own jitdriver if you want to do something special
-        """
-
-    # note: if you overwrite this functions with the above signature it'll
-    #       work, but the *greenargs is different for each jitdriver, so we
-    #       can't share the same methods
-    del on_compile
-    del on_compile_bridge
-
     def _make_extregistryentries(self):
         # workaround: we cannot declare ExtRegistryEntries for functions
         # used as methods of a frozen object, but we can attach the
@@ -628,7 +634,6 @@
     def specialize_call(self, hop, **kwds_i):
         # XXX to be complete, this could also check that the concretetype
         # of the variables are the same for each of the calls.
-        from pypy.rpython.error import TyperError
         from pypy.rpython.lltypesystem import lltype
         driver = self.instance.im_self
         greens_v = []
@@ -741,6 +746,105 @@
         return hop.genop('jit_marker', vlist,
                          resulttype=lltype.Void)
 
+class AsmInfo(object):
+    """ An addition to JitDebugInfo concerning assembler. Attributes:
+    
+    ops_offset - dict of offsets of operations or None
+    asmaddr - (int) raw address of assembler block
+    asmlen - assembler block length
+    """
+    def __init__(self, ops_offset, asmaddr, asmlen):
+        self.ops_offset = ops_offset
+        self.asmaddr = asmaddr
+        self.asmlen = asmlen
+
+class JitDebugInfo(object):
+    """ An object representing debug info. Attributes meanings:
+
+    greenkey - a list of green boxes or None for bridge
+    logger - an instance of jit.metainterp.logger.LogOperations
+    type - either 'loop', 'entry bridge' or 'bridge'
+    looptoken - description of a loop
+    fail_descr_no - number of failing descr for bridges, -1 otherwise
+    asminfo - extra assembler information
+    """
+
+    asminfo = None
+    def __init__(self, jitdriver_sd, logger, looptoken, operations, type,
+                 greenkey=None, fail_descr_no=-1):
+        self.jitdriver_sd = jitdriver_sd
+        self.logger = logger
+        self.looptoken = looptoken
+        self.operations = operations
+        self.type = type
+        if type == 'bridge':
+            assert fail_descr_no != -1
+        else:
+            assert greenkey is not None
+        self.greenkey = greenkey
+        self.fail_descr_no = fail_descr_no
+
+    def get_jitdriver(self):
+        """ Return where the jitdriver on which the jitting started
+        """
+        return self.jitdriver_sd.jitdriver
+
+    def get_greenkey_repr(self):
+        """ Return the string repr of a greenkey
+        """
+        return self.jitdriver_sd.warmstate.get_location_str(self.greenkey)
+
+class JitHookInterface(object):
+    """ This is the main connector between the JIT and the interpreter.
+    Several methods on this class will be invoked at various stages
+    of JIT running like JIT loops compiled, aborts etc.
+    An instance of this class will be available as policy.jithookiface.
+    """
+    def on_abort(self, reason, jitdriver, greenkey, greenkey_repr):
+        """ A hook called each time a loop is aborted with jitdriver and
+        greenkey where it started, reason is a string why it got aborted
+        """
+
+    #def before_optimize(self, debug_info):
+    #    """ A hook called before optimizer is run, called with instance of
+    #    JitDebugInfo. Overwrite for custom behavior
+    #    """
+    # DISABLED
+
+    def before_compile(self, debug_info):
+        """ A hook called after a loop is optimized, before compiling assembler,
+        called with JitDebugInfo instance. Overwrite for custom behavior
+        """
+
+    def after_compile(self, debug_info):
+        """ A hook called after a loop has compiled assembler,
+        called with JitDebugInfo instance. Overwrite for custom behavior
+        """
+
+    #def before_optimize_bridge(self, debug_info):
+    #                           operations, fail_descr_no):
+    #    """ A hook called before a bridge is optimized.
+    #    Called with JitDebugInfo instance, overwrite for
+    #    custom behavior
+    #    """
+    # DISABLED
+
+    def before_compile_bridge(self, debug_info):
+        """ A hook called before a bridge is compiled, but after optimizations
+        are performed. Called with instance of debug_info, overwrite for
+        custom behavior
+        """
+
+    def after_compile_bridge(self, debug_info):
+        """ A hook called after a bridge is compiled, called with JitDebugInfo
+        instance, overwrite for custom behavior
+        """
+
+    def get_stats(self):
+        """ Returns various statistics
+        """
+        raise NotImplementedError
+
 def record_known_class(value, cls):
     """
     Assure the JIT that value is an instance of cls. This is not a precise
@@ -748,7 +852,6 @@
     """
     assert isinstance(value, cls)
 
-
 class Entry(ExtRegistryEntry):
     _about_ = record_known_class
 
@@ -759,7 +862,8 @@
         assert isinstance(s_inst, annmodel.SomeInstance)
 
     def specialize_call(self, hop):
-        from pypy.rpython.lltypesystem import lltype, rclass
+        from pypy.rpython.lltypesystem import rclass, lltype
+        
         classrepr = rclass.get_type_repr(hop.rtyper)
 
         hop.exception_cannot_occur()
diff --git a/pypy/rlib/jit_hooks.py b/pypy/rlib/jit_hooks.py

From noreply at buildbot.pypy.org  Wed Jan 18 12:35:39 2012
From: noreply at buildbot.pypy.org (bivab)
Date: Wed, 18 Jan 2012 12:35:39 +0100 (CET)
Subject: [pypy-commit] pypy arm-backend-2: move this test to the x86 backend
Message-ID: <20120118113539.40E12820D8@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider 
Branch: arm-backend-2
Changeset: r51436:b2fff2c474ff
Date: 2012-01-18 12:11 +0100
http://bitbucket.org/pypy/pypy/changeset/b2fff2c474ff/

Log:	move this test to the x86 backend

diff --git a/pypy/jit/backend/x86/test/test_runner.py b/pypy/jit/backend/x86/test/test_runner.py
--- a/pypy/jit/backend/x86/test/test_runner.py
+++ b/pypy/jit/backend/x86/test/test_runner.py
@@ -582,3 +582,52 @@
         self.cpu.execute_token(looptoken, 0)
         assert looptoken._x86_debug_checksum == sum([op.getopnum()
                                                      for op in ops.operations])
+
+    def test_compile_asmlen(self):
+        from pypy.jit.backend.llsupport.llmodel import AbstractLLCPU
+        if not isinstance(self.cpu, AbstractLLCPU):
+            py.test.skip("pointless test on non-asm")
+        from pypy.jit.backend.x86.tool.viewcode import machine_code_dump
+        import ctypes
+        ops = """
+        [i2]
+        i0 = same_as(i2)    # but forced to be in a register
+        label(i0, descr=1)
+        i1 = int_add(i0, i0)
+        guard_true(i1, descr=faildesr) [i1]
+        jump(i1, descr=1)
+        """
+        faildescr = BasicFailDescr(2)
+        loop = parse(ops, self.cpu, namespace=locals())
+        faildescr = loop.operations[-2].getdescr()
+        jumpdescr = loop.operations[-1].getdescr()
+        bridge_ops = """
+        [i0]
+        jump(i0, descr=jumpdescr)
+        """
+        bridge = parse(bridge_ops, self.cpu, namespace=locals())
+        looptoken = JitCellToken()
+        self.cpu.assembler.set_debug(False)
+        info = self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken)
+        bridge_info = self.cpu.compile_bridge(faildescr, bridge.inputargs,
+                                              bridge.operations,
+                                              looptoken)
+        self.cpu.assembler.set_debug(True) # always on untranslated
+        assert info.asmlen != 0
+        cpuname = autodetect_main_model_and_size()
+        # XXX we have to check the precise assembler, otherwise
+        # we don't quite know if borders are correct
+
+        def checkops(mc, ops):
+            assert len(mc) == len(ops)
+            for i in range(len(mc)):
+                assert mc[i].split("\t")[-1].startswith(ops[i])
+            
+        data = ctypes.string_at(info.asmaddr, info.asmlen)
+        mc = list(machine_code_dump(data, info.asmaddr, cpuname))
+        lines = [line for line in mc if line.count('\t') == 2]
+        checkops(lines, self.add_loop_instructions)
+        data = ctypes.string_at(bridge_info.asmaddr, bridge_info.asmlen)
+        mc = list(machine_code_dump(data, bridge_info.asmaddr, cpuname))
+        lines = [line for line in mc if line.count('\t') == 2]
+        checkops(lines, self.bridge_loop_instructions)

From noreply at buildbot.pypy.org  Wed Jan 18 12:35:40 2012
From: noreply at buildbot.pypy.org (bivab)
Date: Wed, 18 Jan 2012 12:35:40 +0100 (CET)
Subject: [pypy-commit] pypy arm-backend-2: Create a CPU instance for each
	test
Message-ID: <20120118113540.7BA1D820D8@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider 
Branch: arm-backend-2
Changeset: r51437:1c132ebb86eb
Date: 2012-01-18 12:13 +0100
http://bitbucket.org/pypy/pypy/changeset/1c132ebb86eb/

Log:	Create a CPU instance for each test

diff --git a/pypy/jit/backend/arm/test/test_runner.py b/pypy/jit/backend/arm/test/test_runner.py
--- a/pypy/jit/backend/arm/test/test_runner.py
+++ b/pypy/jit/backend/arm/test/test_runner.py
@@ -23,12 +23,9 @@
 
 class TestARM(LLtypeBackendTest):
 
-    def setup_class(cls):
-        cls.cpu = ArmCPU(rtyper=None, stats=FakeStats())
-        cls.cpu.setup_once()
-
-    def teardown_method(self, method):
-        self.cpu.assembler.teardown()
+    def setup_method(self, meth):
+        self.cpu = ArmCPU(rtyper=None, stats=FakeStats())
+        self.cpu.setup_once()
 
     # for the individual tests see
     # ====> ../../test/runner_test.py

From noreply at buildbot.pypy.org  Wed Jan 18 12:35:41 2012
From: noreply at buildbot.pypy.org (bivab)
Date: Wed, 18 Jan 2012 12:35:41 +0100 (CET)
Subject: [pypy-commit] pypy arm-backend-2: kill unused code
Message-ID: <20120118113541.B7916820D8@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider 
Branch: arm-backend-2
Changeset: r51438:d0ed9d2beb0e
Date: 2012-01-18 12:14 +0100
http://bitbucket.org/pypy/pypy/changeset/d0ed9d2beb0e/

Log:	kill unused code

diff --git a/pypy/jit/backend/arm/assembler.py b/pypy/jit/backend/arm/assembler.py
--- a/pypy/jit/backend/arm/assembler.py
+++ b/pypy/jit/backend/arm/assembler.py
@@ -1,12 +1,10 @@
 from __future__ import with_statement
 import os
-from pypy.jit.backend.arm.helper.assembler import saved_registers, \
-                                                    decode32, encode32, \
-                                                    decode64
+from pypy.jit.backend.arm.helper.assembler import saved_registers
 from pypy.jit.backend.arm import conditions as c
 from pypy.jit.backend.arm import registers as r
 from pypy.jit.backend.arm.arch import WORD, DOUBLE_WORD, FUNC_ALIGN, \
-                                    PC_OFFSET, N_REGISTERS_SAVED_BY_MALLOC
+                                    N_REGISTERS_SAVED_BY_MALLOC
 from pypy.jit.backend.arm.codebuilder import ARMv7Builder, OverwritingBuilder
 from pypy.jit.backend.arm.regalloc import (Regalloc, ARMFrameManager,
                     ARMv7RegisterManager, check_imm_arg,
diff --git a/pypy/jit/backend/arm/helper/assembler.py b/pypy/jit/backend/arm/helper/assembler.py
--- a/pypy/jit/backend/arm/helper/assembler.py
+++ b/pypy/jit/backend/arm/helper/assembler.py
@@ -195,25 +195,3 @@
             reg_args = x
             break
     return reg_args
-
-def decode32(mem, index):
-    return intmask(ord(mem[index])
-            | ord(mem[index+1]) << 8
-            | ord(mem[index+2]) << 16
-            | ord(mem[index+3]) << 24)
-
-def decode64(mem, index):
-    low = decode32(mem, index)
-    index += 4
-    high = decode32(mem, index)
-    return (r_longlong(high) << 32) | r_longlong(r_uint(low))
-
-def encode32(mem, i, n):
-    mem[i] = chr(n & 0xFF)
-    mem[i+1] = chr((n >> 8) & 0xFF)
-    mem[i+2] = chr((n >> 16) & 0xFF)
-    mem[i+3] = chr((n >> 24) & 0xFF)
-
-def encode64(mem, i, n):
-    for x in range(8):
-        mem[i+x] = chr((n >> (x*8)) & 0xFF)
diff --git a/pypy/jit/backend/arm/test/test_helper.py b/pypy/jit/backend/arm/test/test_helper.py
--- a/pypy/jit/backend/arm/test/test_helper.py
+++ b/pypy/jit/backend/arm/test/test_helper.py
@@ -6,6 +6,7 @@
 from pypy.jit.backend.arm.test.support import skip_unless_arm
 skip_unless_arm()
 
+
 def test_count_reg_args():
     assert count_reg_args([BoxPtr()]) == 1
     assert count_reg_args([BoxPtr()] * 2) == 2
@@ -21,26 +22,3 @@
 
     assert count_reg_args([BoxInt(), BoxFloat(), BoxInt()]) == 2
     assert count_reg_args([BoxInt(), BoxInt(), BoxInt(), BoxFloat()]) == 3
-
-def test_encode32():
-    mem = [None]*4
-    encode32(mem, 0, 1234567)
-    assert ''.join(mem) == '\x87\xd6\x12\x00'
-    mem = [None]*4
-    encode32(mem, 0, 983040)
-    assert ''.join(mem) == '\x00\x00\x0F\x00'
-
-def test_decode32():
-    mem = list('\x87\xd6\x12\x00')
-    assert decode32(mem, 0) ==  1234567
-    mem = list('\x00\x00\x0F\x00')
-    assert decode32(mem, 0) == 983040
-
-def test_decode64():
-    mem = list('\x87\xd6\x12\x00\x00\x00\x0F\x00')
-    assert decode64(mem, 0) == 4222124651894407L
-
-def test_encode64():
-    mem = [None] * 8
-    encode64(mem, 0, 4222124651894407L)
-    assert ''.join(mem) ==  '\x87\xd6\x12\x00\x00\x00\x0F\x00'

From noreply at buildbot.pypy.org  Wed Jan 18 12:35:42 2012
From: noreply at buildbot.pypy.org (bivab)
Date: Wed, 18 Jan 2012 12:35:42 +0100 (CET)
Subject: [pypy-commit] pypy arm-backend-2: Generate more debugging
 information (taken from the x86 backend)
Message-ID: <20120118113542.F23D8820D8@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider 
Branch: arm-backend-2
Changeset: r51439:fceb632073b8
Date: 2012-01-18 12:16 +0100
http://bitbucket.org/pypy/pypy/changeset/fceb632073b8/

Log:	Generate more debugging information (taken from the x86 backend)

diff --git a/pypy/jit/backend/arm/assembler.py b/pypy/jit/backend/arm/assembler.py
--- a/pypy/jit/backend/arm/assembler.py
+++ b/pypy/jit/backend/arm/assembler.py
@@ -14,19 +14,28 @@
 from pypy.jit.backend.llsupport.asmmemmgr import MachineDataBlockWrapper
 from pypy.jit.backend.model import CompiledLoopToken
 from pypy.jit.codewriter import longlong
-from pypy.jit.metainterp.history import (AbstractFailDescr, INT, REF, FLOAT)
-from pypy.jit.metainterp.resoperation import rop
+from pypy.jit.metainterp.history import AbstractFailDescr, INT, REF, FLOAT
+from pypy.jit.metainterp.history import BoxInt, ConstInt
+from pypy.jit.metainterp.resoperation import rop, ResOperation
 from pypy.rlib import rgc
-from pypy.rlib.objectmodel import we_are_translated
+from pypy.rlib.objectmodel import we_are_translated, specialize
 from pypy.rpython.annlowlevel import llhelper
 from pypy.rpython.lltypesystem import lltype, rffi, llmemory
 from pypy.rpython.lltypesystem.lloperation import llop
 from pypy.jit.backend.arm.opassembler import ResOpAssembler
-from pypy.rlib.debug import debug_print, debug_start, debug_stop
+from pypy.rlib.debug import (debug_print, debug_start, debug_stop,
+                             have_debug_prints)
+from pypy.rlib.jit import AsmInfo
+from pypy.rlib.objectmodel import compute_unique_id
 
 # XXX Move to llsupport
 from pypy.jit.backend.x86.support import values_array, memcpy_fn
 
+DEBUG_COUNTER = lltype.Struct('DEBUG_COUNTER', ('i', lltype.Signed),
+                              ('type', lltype.Char), # 'b'ridge, 'l'abel or
+                                                     # 'e'ntry point
+                              ('number', lltype.Signed))
+
 
 class AssemblerARM(ResOpAssembler):
 
@@ -53,6 +62,12 @@
         self.datablockwrapper = None
         self.propagate_exception_path = 0
         self._compute_stack_size()
+        self._debug = False
+        self.loop_run_counters = []
+        self.debug_counter_descr = cpu.fielddescrof(DEBUG_COUNTER, 'i')
+
+    def set_debug(self, v):
+        self._debug = v
 
     def _compute_stack_size(self):
         self.STACK_FIXED_AREA = len(r.callee_saved_registers) * WORD
@@ -100,6 +115,72 @@
         self._leave_jitted_hook_save_exc = \
                                     self._gen_leave_jitted_hook_code(True)
         self._leave_jitted_hook = self._gen_leave_jitted_hook_code(False)
+        debug_start('jit-backend-counts')
+        self.set_debug(have_debug_prints())
+        debug_stop('jit-backend-counts')
+
+    def finish_once(self):
+        if self._debug:
+            debug_start('jit-backend-counts')
+            for i in range(len(self.loop_run_counters)):
+                struct = self.loop_run_counters[i]
+                if struct.type == 'l':
+                    prefix = 'TargetToken(%d)' % struct.number
+                elif struct.type == 'b':
+                    prefix = 'bridge ' + str(struct.number)
+                else:
+                    prefix = 'entry ' + str(struct.number)
+                debug_print(prefix + ':' + str(struct.i))
+            debug_stop('jit-backend-counts')
+
+    # XXX: merge with x86
+    def _register_counter(self, tp, number, token):
+        # YYY very minor leak -- we need the counters to stay alive
+        # forever, just because we want to report them at the end
+        # of the process
+        struct = lltype.malloc(DEBUG_COUNTER, flavor='raw',
+                               track_allocation=False)
+        struct.i = 0
+        struct.type = tp
+        if tp == 'b' or tp == 'e':
+            struct.number = number
+        else:
+            assert token
+            struct.number = compute_unique_id(token)
+        self.loop_run_counters.append(struct)
+        return struct
+
+    def _append_debugging_code(self, operations, tp, number, token):
+        counter = self._register_counter(tp, number, token)
+        c_adr = ConstInt(rffi.cast(lltype.Signed, counter))
+        box = BoxInt()
+        box2 = BoxInt()
+        ops = [ResOperation(rop.GETFIELD_RAW, [c_adr],
+                            box, descr=self.debug_counter_descr),
+               ResOperation(rop.INT_ADD, [box, ConstInt(1)], box2),
+               ResOperation(rop.SETFIELD_RAW, [c_adr, box2],
+                            None, descr=self.debug_counter_descr)]
+        operations.extend(ops)
+
+    @specialize.argtype(1)
+    def _inject_debugging_code(self, looptoken, operations, tp, number):
+        if self._debug:
+            # before doing anything, let's increase a counter
+            s = 0
+            for op in operations:
+                s += op.getopnum()
+            looptoken._arm_debug_checksum = s
+
+            newoperations = []
+            self._append_debugging_code(newoperations, tp, number,
+                                        None)
+            for op in operations:
+                newoperations.append(op)
+                if op.getopnum() == rop.LABEL:
+                    self._append_debugging_code(newoperations, 'l', number,
+                                                op.getdescr())
+            operations = newoperations
+        return operations
 
     @staticmethod
     def _release_gil_shadowstack():
@@ -491,7 +572,10 @@
             assert len(set(inputargs)) == len(inputargs)
 
         operations = self.setup(looptoken, operations)
-        self._dump(operations)
+        if log:
+            operations = self._inject_debugging_code(looptoken, operations,
+                                                     'e', looptoken.number)
+            self._dump(operations)
 
         self._call_header()
         sp_patch_location = self._prepare_sp_patch_position()
@@ -536,7 +620,11 @@
     def assemble_bridge(self, faildescr, inputargs, operations,
                                                     original_loop_token, log):
         operations = self.setup(original_loop_token, operations)
-        self._dump(operations, 'bridge')
+        descr_number = self.cpu.get_fail_descr_number(faildescr)
+        if log:
+            operations = self._inject_debugging_code(faildescr, operations,
+                                                     'b', descr_number)
+            self._dump(operations, 'bridge')
         assert isinstance(faildescr, AbstractFailDescr)
         code = self._find_failure_recovery_bytecode(faildescr)
         frame_depth = faildescr._arm_current_frame_depth
@@ -736,7 +824,7 @@
         return True
 
     def _insert_checks(self, mc=None):
-        if not we_are_translated():
+        if self._debug:
             if mc is None:
                 mc = self.mc
             mc.CMP_rr(r.fp.value, r.sp.value)
diff --git a/pypy/jit/backend/arm/test/test_runner.py b/pypy/jit/backend/arm/test/test_runner.py
--- a/pypy/jit/backend/arm/test/test_runner.py
+++ b/pypy/jit/backend/arm/test/test_runner.py
@@ -202,3 +202,44 @@
             args = [i+1 for i in range(numargs)]
             res = self.cpu.execute_token(looptoken, *args)
             assert self.cpu.get_latest_value_int(0) == sum(args)
+
+    def test_debugger_on(self):
+        from pypy.rlib import debug
+
+        targettoken, preambletoken = TargetToken(), TargetToken()
+        loop = """
+        [i0]
+        label(i0, descr=preambletoken)
+        debug_merge_point('xyz', 0)
+        i1 = int_add(i0, 1)
+        i2 = int_ge(i1, 10)
+        guard_false(i2) []
+        label(i1, descr=targettoken)
+        debug_merge_point('xyz', 0)
+        i11 = int_add(i1, 1)
+        i12 = int_ge(i11, 10)
+        guard_false(i12) []
+        jump(i11, descr=targettoken)
+        """
+        ops = parse(loop, namespace={'targettoken': targettoken,
+                                     'preambletoken': preambletoken})
+        debug._log = dlog = debug.DebugLog()
+        try:
+            self.cpu.assembler.set_debug(True)
+            looptoken = JitCellToken()
+            self.cpu.compile_loop(ops.inputargs, ops.operations, looptoken)
+            self.cpu.execute_token(looptoken, 0)
+            # check debugging info
+            struct = self.cpu.assembler.loop_run_counters[0]
+            assert struct.i == 1
+            struct = self.cpu.assembler.loop_run_counters[1]
+            assert struct.i == 1
+            struct = self.cpu.assembler.loop_run_counters[2]
+            assert struct.i == 9
+            self.cpu.finish_once()
+        finally:
+            debug._log = None
+        l0 = ('debug_print', 'entry -1:1')
+        l1 = ('debug_print', preambletoken.repr_of_descr() + ':1')
+        l2 = ('debug_print', targettoken.repr_of_descr() + ':9')
+        assert ('jit-backend-counts', [l0, l1, l2]) in dlog

From noreply at buildbot.pypy.org  Wed Jan 18 12:35:44 2012
From: noreply at buildbot.pypy.org (bivab)
Date: Wed, 18 Jan 2012 12:35:44 +0100 (CET)
Subject: [pypy-commit] pypy arm-backend-2: Port changes since last merge
Message-ID: <20120118113544.3FB31820D8@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider 
Branch: arm-backend-2
Changeset: r51440:be46980ab0b2
Date: 2012-01-18 12:16 +0100
http://bitbucket.org/pypy/pypy/changeset/be46980ab0b2/

Log:	Port changes since last merge

diff --git a/pypy/jit/backend/arm/assembler.py b/pypy/jit/backend/arm/assembler.py
--- a/pypy/jit/backend/arm/assembler.py
+++ b/pypy/jit/backend/arm/assembler.py
@@ -596,6 +596,7 @@
         rawstart = self.materialize_loop(looptoken)
         looptoken._arm_func_addr = rawstart
 
+        size_excluding_failure_stuff = self.mc.get_relative_pos()
         self.process_pending_guards(rawstart)
         self.fixup_target_tokens(rawstart)
 
@@ -604,8 +605,13 @@
             self.mc._dump_trace(rawstart,
                     'loop_%s.asm' % self.cpu.total_compiled_loops)
             print 'Done assembling loop with token %r' % looptoken
+
+        ops_offset = self.mc.ops_offset
         self.teardown()
 
+        return AsmInfo(ops_offset, rawstart + loop_head,
+                       size_excluding_failure_stuff - loop_head)
+
     def _assemble(self, operations, regalloc):
         regalloc.compute_hint_frame_locations(operations)
         #self.mc.BKPT()
@@ -632,6 +638,7 @@
         if not we_are_translated():
             assert len(inputargs) == len(arglocs)
 
+        startpos = self.mc.get_relative_pos()
         regalloc = Regalloc(assembler=self, frame_manager=ARMFrameManager())
         regalloc.prepare_bridge(inputargs, arglocs, operations)
 
@@ -639,15 +646,19 @@
 
         frame_depth = self._assemble(operations, regalloc)
 
+        codeendpos = self.mc.get_relative_pos()
+
         self._patch_sp_offset(sp_patch_location, frame_depth)
 
         self.write_pending_failure_recoveries()
+
         rawstart = self.materialize_loop(original_loop_token)
+
         self.process_pending_guards(rawstart)
+        self.fixup_target_tokens(rawstart)
 
         self.patch_trace(faildescr, original_loop_token,
                                     rawstart, regalloc)
-        self.fixup_target_tokens(rawstart)
 
         if not we_are_translated():
             # for the benefit of tests
@@ -658,12 +669,19 @@
                 self.cpu.total_compiled_bridges)
         self.current_clt.frame_depth = max(self.current_clt.frame_depth,
                                                                 frame_depth)
+        ops_offset = self.mc.ops_offset
         self.teardown()
+        return AsmInfo(ops_offset, startpos + rawstart, codeendpos - startpos)
 
     def _find_failure_recovery_bytecode(self, faildescr):
-        guard_addr = faildescr._arm_block_start + faildescr._arm_guard_pos
+        guard_stub_addr = faildescr._arm_recovery_stub_offset
+        if guard_stub_addr == 0:
+            # This case should be prevented by the logic in compile.py:
+            # look for CNT_BUSY_FLAG, which disables tracing from a guard
+            # when another tracing from the same guard is already in progress.
+            raise BridgeAlreadyCompiled
         # a guard requires 3 words to encode the jump to the exit code.
-        return guard_addr + 3 * WORD
+        return guard_stub_addr + 3 * WORD
 
     def fixup_target_tokens(self, rawstart):
         for targettoken in self.target_tokens_currently_compiling:
@@ -698,21 +716,27 @@
         for tok in self.pending_guards:
             descr = tok.descr
             assert isinstance(descr, AbstractFailDescr)
+            jump_target = tok.pos_recovery_stub
+            relative_target = jump_target - tok.offset
 
-            #XXX _arm_block_start should go in the looptoken
-            descr._arm_block_start = block_start
+            addr = block_start + tok.offset
+            stub_addr = block_start + jump_target
+
+            descr._arm_recovery_stub_offset = stub_addr
 
             if not tok.is_invalidate:
                 #patch the guard jumpt to the stub
                 # overwrite the generate NOP with a B_offs to the pos of the
                 # stub
                 mc = ARMv7Builder()
-                mc.B_offs(descr._arm_guard_pos - tok.offset,
-                                    c.get_opposite_of(tok.fcond))
-                mc.copy_to_raw_memory(block_start + tok.offset)
+                mc.B_offs(relative_target, c.get_opposite_of(tok.fcond))
+                mc.copy_to_raw_memory(addr)
             else:
-                clt.invalidate_positions.append(
-                (block_start + tok.offset, descr._arm_guard_pos - tok.offset))
+                # GUARD_NOT_INVALIDATED, record an entry in
+                # clt.invalidate_positions of the form:
+                #     (addr-in-the-code-of-the-not-yet-written-jump-target,
+                #      relative-target-to-use)
+                clt.invalidate_positions.append((addr, relative_target))
 
     def get_asmmemmgr_blocks(self, looptoken):
         clt = looptoken.compiled_loop_token
@@ -770,6 +794,7 @@
             regalloc.next_instruction()
             i = regalloc.position()
             op = operations[i]
+            self.mc.mark_op(op)
             opnum = op.getopnum()
             if op.has_no_side_effect() and op.result not in regalloc.longevity:
                 regalloc.possibly_free_vars_for_op(op)
@@ -797,6 +822,7 @@
             regalloc.possibly_free_vars_for_op(op)
             regalloc.free_temp_vars()
             regalloc._check_invariants()
+        self.mc.mark_op(None)  # end of the loop
 
     # from ../x86/regalloc.py
     def can_merge_with_next_guard(self, op, i, operations):
@@ -852,9 +878,11 @@
         # The first instruction (word) is not overwritten, because it is the
         # one that actually checks the condition
         b = ARMv7Builder()
-        patch_addr = faildescr._arm_block_start + faildescr._arm_guard_pos
+        adr_jump_offset = faildescr._arm_recovery_stub_offset
+        assert adr_jump_offset != 0
         b.B(bridge_addr)
-        b.copy_to_raw_memory(patch_addr)
+        b.copy_to_raw_memory(adr_jump_offset)
+        faildescr._arm_recovery_stub_offset = 0
 
     # regalloc support
     def load(self, loc, value):
@@ -1186,3 +1214,7 @@
     if hasattr(AssemblerARM, methname):
         func = getattr(AssemblerARM, methname).im_func
         asm_operations_with_guard[value] = func
+
+
+class BridgeAlreadyCompiled(Exception):
+    pass
diff --git a/pypy/jit/backend/arm/codebuilder.py b/pypy/jit/backend/arm/codebuilder.py
--- a/pypy/jit/backend/arm/codebuilder.py
+++ b/pypy/jit/backend/arm/codebuilder.py
@@ -265,6 +265,15 @@
     def __init__(self):
         AbstractARMv7Builder.__init__(self)
         self.init_block_builder()
+        #
+        # ResOperation --> offset in the assembly.
+        # ops_offset[None] represents the beginning of the code after the last op
+        # (i.e., the tail of the loop)
+        self.ops_offset = {}
+
+    def mark_op(self, op):
+        pos = self.get_relative_pos()
+        self.ops_offset[op] = pos
 
     def _dump_trace(self, addr, name, formatter=-1):
         if not we_are_translated():
diff --git a/pypy/jit/backend/arm/runner.py b/pypy/jit/backend/arm/runner.py
--- a/pypy/jit/backend/arm/runner.py
+++ b/pypy/jit/backend/arm/runner.py
@@ -32,19 +32,19 @@
         self.assembler.setup_once()
 
     def finish_once(self):
-        pass
+        self.assembler.finish_once()
 
     def compile_loop(self, inputargs, operations, looptoken,
                                                     log=True, name=''):
-        self.assembler.assemble_loop(inputargs, operations,
+        return self.assembler.assemble_loop(inputargs, operations,
                                                     looptoken, log=log)
 
     def compile_bridge(self, faildescr, inputargs, operations,
                                        original_loop_token, log=True):
         clt = original_loop_token.compiled_loop_token
         clt.compiling_a_bridge()
-        self.assembler.assemble_bridge(faildescr, inputargs, operations,
-                                       original_loop_token, log=log)
+        return self.assembler.assemble_bridge(faildescr, inputargs, operations,
+                                                original_loop_token, log=log)
 
     def get_latest_value_float(self, index):
         return self.assembler.fail_boxes_float.getitem(index)

From noreply at buildbot.pypy.org  Wed Jan 18 12:35:45 2012
From: noreply at buildbot.pypy.org (bivab)
Date: Wed, 18 Jan 2012 12:35:45 +0100 (CET)
Subject: [pypy-commit] pypy arm-backend-2: also remove test_compile_asmlen
 from runner_test after it was moved to the x86 backend
Message-ID: <20120118113545.7B141820D8@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider 
Branch: arm-backend-2
Changeset: r51441:839659291f03
Date: 2012-01-18 12:33 +0100
http://bitbucket.org/pypy/pypy/changeset/839659291f03/

Log:	also remove test_compile_asmlen from runner_test after it was moved
	to the x86 backend

diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py
--- a/pypy/jit/backend/test/runner_test.py
+++ b/pypy/jit/backend/test/runner_test.py
@@ -3188,55 +3188,6 @@
         res = self.cpu.get_latest_value_int(0)
         assert res == -10
 
-    def test_compile_asmlen(self):
-        from pypy.jit.backend.llsupport.llmodel import AbstractLLCPU
-        if not isinstance(self.cpu, AbstractLLCPU):
-            py.test.skip("pointless test on non-asm")
-        from pypy.jit.backend.x86.tool.viewcode import machine_code_dump
-        import ctypes
-        ops = """
-        [i2]
-        i0 = same_as(i2)    # but forced to be in a register
-        label(i0, descr=1)
-        i1 = int_add(i0, i0)
-        guard_true(i1, descr=faildesr) [i1]
-        jump(i1, descr=1)
-        """
-        faildescr = BasicFailDescr(2)
-        loop = parse(ops, self.cpu, namespace=locals())
-        faildescr = loop.operations[-2].getdescr()
-        jumpdescr = loop.operations[-1].getdescr()
-        bridge_ops = """
-        [i0]
-        jump(i0, descr=jumpdescr)
-        """
-        bridge = parse(bridge_ops, self.cpu, namespace=locals())
-        looptoken = JitCellToken()
-        self.cpu.assembler.set_debug(False)
-        info = self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken)
-        bridge_info = self.cpu.compile_bridge(faildescr, bridge.inputargs,
-                                              bridge.operations,
-                                              looptoken)
-        self.cpu.assembler.set_debug(True) # always on untranslated
-        assert info.asmlen != 0
-        cpuname = autodetect_main_model_and_size()
-        # XXX we have to check the precise assembler, otherwise
-        # we don't quite know if borders are correct
-
-        def checkops(mc, ops):
-            assert len(mc) == len(ops)
-            for i in range(len(mc)):
-                assert mc[i].split("\t")[-1].startswith(ops[i])
-            
-        data = ctypes.string_at(info.asmaddr, info.asmlen)
-        mc = list(machine_code_dump(data, info.asmaddr, cpuname))
-        lines = [line for line in mc if line.count('\t') == 2]
-        checkops(lines, self.add_loop_instructions)
-        data = ctypes.string_at(bridge_info.asmaddr, bridge_info.asmlen)
-        mc = list(machine_code_dump(data, bridge_info.asmaddr, cpuname))
-        lines = [line for line in mc if line.count('\t') == 2]
-        checkops(lines, self.bridge_loop_instructions)
-
 
     def test_compile_bridge_with_target(self):
         # This test creates a loopy piece of code in a bridge, and builds another

From noreply at buildbot.pypy.org  Wed Jan 18 13:44:43 2012
From: noreply at buildbot.pypy.org (hager)
Date: Wed, 18 Jan 2012 13:44:43 +0100 (CET)
Subject: [pypy-commit] pypy ppc-jit-backend: adjust size parameter in
 _check_im_arg and check_im_box
Message-ID: <20120118124443.07F14820D8@wyvern.cs.uni-duesseldorf.de>

Author: hager 
Branch: ppc-jit-backend
Changeset: r51442:d0d21bbde762
Date: 2012-01-18 13:43 +0100
http://bitbucket.org/pypy/pypy/changeset/d0d21bbde762/

Log:	adjust size parameter in _check_im_arg and check_im_box

diff --git a/pypy/jit/backend/ppc/ppcgen/helper/regalloc.py b/pypy/jit/backend/ppc/ppcgen/helper/regalloc.py
--- a/pypy/jit/backend/ppc/ppcgen/helper/regalloc.py
+++ b/pypy/jit/backend/ppc/ppcgen/helper/regalloc.py
@@ -2,13 +2,14 @@
 from pypy.rlib.objectmodel import we_are_translated
 from pypy.jit.metainterp.history import Box
 
+IMM_SIZE = 2 ** 15 - 1
 
-def check_imm_box(arg, size=0xFF, allow_zero=True):
+def check_imm_box(arg, size=IMM_SIZE, allow_zero=True):
     if isinstance(arg, ConstInt):
         return _check_imm_arg(arg.getint(), size, allow_zero)
     return False
 
-def _check_imm_arg(arg, size=0xFF, allow_zero=True):
+def _check_imm_arg(arg, size=IMM_SIZE, allow_zero=True):
     #assert not isinstance(arg, ConstInt)
     #if not we_are_translated():
     #    if not isinstance(arg, int):

From noreply at buildbot.pypy.org  Wed Jan 18 13:44:44 2012
From: noreply at buildbot.pypy.org (hager)
Date: Wed, 18 Jan 2012 13:44:44 +0100 (CET)
Subject: [pypy-commit] pypy ppc-jit-backend: remove comment
Message-ID: <20120118124444.3F367820D8@wyvern.cs.uni-duesseldorf.de>

Author: hager 
Branch: ppc-jit-backend
Changeset: r51443:35aaf3680c92
Date: 2012-01-18 13:44 +0100
http://bitbucket.org/pypy/pypy/changeset/35aaf3680c92/

Log:	remove comment

diff --git a/pypy/jit/backend/ppc/ppcgen/helper/regalloc.py b/pypy/jit/backend/ppc/ppcgen/helper/regalloc.py
--- a/pypy/jit/backend/ppc/ppcgen/helper/regalloc.py
+++ b/pypy/jit/backend/ppc/ppcgen/helper/regalloc.py
@@ -37,7 +37,6 @@
         self.possibly_free_vars_for_op(op)
         self.free_temp_vars()
         res = self.force_allocate_reg(op.result)
-        #self.possibly_free_var(op.result)
         return [l0, l1, res]
     return f
 

From noreply at buildbot.pypy.org  Wed Jan 18 15:00:00 2012
From: noreply at buildbot.pypy.org (justinpeel)
Date: Wed, 18 Jan 2012 15:00:00 +0100 (CET)
Subject: [pypy-commit] pypy default: fix a bug in to_str for ndim=1,
 size=1 arrays. also affected any multi-dim array where the last
 dimension is 1. tests included.
Message-ID: <20120118140000.E44F2820D8@wyvern.cs.uni-duesseldorf.de>

Author: Justin Peel 
Branch: 
Changeset: r51444:db7b9b0aa08d
Date: 2012-01-17 22:13 -0700
http://bitbucket.org/pypy/pypy/changeset/db7b9b0aa08d/

Log:	fix a bug in to_str for ndim=1, size=1 arrays. also affected any
	multi-dim array where the last dimension is 1. tests included.

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -919,14 +919,14 @@
         if size < 1:
             builder.append('[]')
             return
-        elif size == 1:
-            builder.append(dtype.itemtype.str_format(self.getitem(0)))
-            return
         if size > 1000:
             # Once this goes True it does not go back to False for recursive
             # calls
             use_ellipsis = True
         ndims = len(self.shape)
+        if ndims == 0:
+            builder.append(dtype.itemtype.str_format(self.getitem(0)))
+            return
         i = 0
         builder.append('[')
         if ndims > 1:
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1441,9 +1441,11 @@
         assert repr(a) == "array(0.0)"
         a = array(0.2)
         assert repr(a) == "array(0.2)"
+        a = array([2])
+        assert repr(a) == "array([2])"
 
     def test_repr_multi(self):
-        from _numpypy import arange, zeros
+        from _numpypy import arange, zeros, array
         a = zeros((3, 4))
         assert repr(a) == '''array([[0.0, 0.0, 0.0, 0.0],
        [0.0, 0.0, 0.0, 0.0],
@@ -1466,6 +1468,9 @@
        [498, 999],
        [499, 1000],
        [500, 1001]])'''
+        a = arange(2).reshape((2,1))
+        assert repr(a) == '''array([[0],
+       [1]])'''
 
     def test_repr_slice(self):
         from _numpypy import array, zeros

From noreply at buildbot.pypy.org  Wed Jan 18 15:53:42 2012
From: noreply at buildbot.pypy.org (edelsohn)
Date: Wed, 18 Jan 2012 15:53:42 +0100 (CET)
Subject: [pypy-commit] pypy ppc-jit-backend: Remove boxes from
 getinteriorfield_gc, setinteriorfield_gc.
Message-ID: <20120118145342.E2078820D8@wyvern.cs.uni-duesseldorf.de>

Author: edelsohn
Branch: ppc-jit-backend
Changeset: r51445:db20886769cb
Date: 2012-01-18 09:53 -0500
http://bitbucket.org/pypy/pypy/changeset/db20886769cb/

Log:	Remove boxes from getinteriorfield_gc, setinteriorfield_gc.

diff --git a/pypy/jit/backend/ppc/ppcgen/regalloc.py b/pypy/jit/backend/ppc/ppcgen/regalloc.py
--- a/pypy/jit/backend/ppc/ppcgen/regalloc.py
+++ b/pypy/jit/backend/ppc/ppcgen/regalloc.py
@@ -597,17 +597,14 @@
         t = unpack_interiorfielddescr(op.getdescr())
         ofs, itemsize, fieldsize, sign = t
         args = op.getarglist()
-        base_loc, base_box = self._ensure_value_is_boxed(op.getarg(0), args)
-        index_loc, index_box = self._ensure_value_is_boxed(op.getarg(1), args)
+        base_loc = self._ensure_value_is_boxed(op.getarg(0), args)
+        index_loc = self._ensure_value_is_boxed(op.getarg(1), args)
         c_ofs = ConstInt(ofs)
         if _check_imm_arg(c_ofs):
             ofs_loc = imm(ofs)
         else:
-            ofs_loc, ofs_box = self._ensure_value_is_boxed(c_ofs, [base_box])
-            self.possibly_free_var(ofs_box)
+            ofs_loc = self._ensure_value_is_boxed(c_ofs, args)
         self.possibly_free_vars_for_op(op)
-        self.possibly_free_var(base_box)
-        self.possibly_free_var(index_box)
         self.free_temp_vars()
         result_loc = self.force_allocate_reg(op.result)
         self.possibly_free_var(op.result)
@@ -618,18 +615,14 @@
         t = unpack_interiorfielddescr(op.getdescr())
         ofs, itemsize, fieldsize, sign = t
         args = op.getarglist()
-        base_loc, base_box = self._ensure_value_is_boxed(op.getarg(0), args)
-        index_loc, index_box  = self._ensure_value_is_boxed(op.getarg(1), args)
-        value_loc, value_box = self._ensure_value_is_boxed(op.getarg(2), args)
+        base_loc = self._ensure_value_is_boxed(op.getarg(0), args)
+        index_loc = self._ensure_value_is_boxed(op.getarg(1), args)
+        value_loc = self._ensure_value_is_boxed(op.getarg(2), args)
         c_ofs = ConstInt(ofs)
         if _check_imm_arg(c_ofs):
             ofs_loc = imm(ofs)
         else:
-            ofs_loc, ofs_box = self._ensure_value_is_boxed(c_ofs, [base_box])
-            self.possibly_free_var(ofs_box)
-        self.possibly_free_var(base_box)
-        self.possibly_free_var(index_box)
-        self.possibly_free_var(value_box)
+            ofs_loc = self._ensure_value_is_boxed(c_ofs, args)
         return [base_loc, index_loc, value_loc, ofs_loc, imm(ofs),
                                         imm(itemsize), imm(fieldsize)]
 

From noreply at buildbot.pypy.org  Wed Jan 18 17:09:18 2012
From: noreply at buildbot.pypy.org (rguillebert)
Date: Wed, 18 Jan 2012 17:09:18 +0100 (CET)
Subject: [pypy-commit] pypy py3k: Implement py3k's tuple unpacking in the
	code generator
Message-ID: <20120118160918.58F07820D8@wyvern.cs.uni-duesseldorf.de>

Author: Romain Guillebert 
Branch: py3k
Changeset: r51446:f4cb22c63136
Date: 2012-01-18 13:55 +0100
http://bitbucket.org/pypy/pypy/changeset/f4cb22c63136/

Log:	Implement py3k's tuple unpacking in the code generator

diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -833,11 +833,22 @@
         self.update_position(tup.lineno)
         elt_count = len(tup.elts) if tup.elts is not None else 0
         if tup.ctx == ast.Store:
-            self.emit_op_arg(ops.UNPACK_SEQUENCE, elt_count)
+            star_pos = -1
+            if elt_count > 0:
+                for i, elt in enumerate(tup.elts):
+                    if isinstance(elt, ast.Starred):
+                        star_pos = i
+            if star_pos > -1:
+                self.emit_op_arg(ops.UNPACK_EX, star_pos | (elt_count-star_pos-1)<<8)
+            else:
+                self.emit_op_arg(ops.UNPACK_SEQUENCE, elt_count)
         self.visit_sequence(tup.elts)
         if tup.ctx == ast.Load:
             self.emit_op_arg(ops.BUILD_TUPLE, elt_count)
 
+    def visit_Starred(self, star):
+        star.value.walkabout(self)
+
     def visit_List(self, l):
         self.update_position(l.lineno)
         elt_count = len(l.elts) if l.elts is not None else 0

From noreply at buildbot.pypy.org  Wed Jan 18 17:09:19 2012
From: noreply at buildbot.pypy.org (rguillebert)
Date: Wed, 18 Jan 2012 17:09:19 +0100 (CET)
Subject: [pypy-commit] pypy py3k: Implement the UNPACK_EX bytecode which is
 used by py3k's new tuple unpacking
Message-ID: <20120118160919.A0AAA820D8@wyvern.cs.uni-duesseldorf.de>

Author: Romain Guillebert 
Branch: py3k
Changeset: r51447:6face22d8492
Date: 2012-01-18 14:11 +0100
http://bitbucket.org/pypy/pypy/changeset/6face22d8492/

Log:	Implement the UNPACK_EX bytecode which is used by py3k's new tuple
	unpacking

diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -582,7 +582,25 @@
         self.pushrevvalues(itemcount, items)
 
     def UNPACK_EX(self, oparg, next_instr):
-        raise NotImplementedError
+        left = oparg & 0xFF
+        right = (oparg & 0xFF00) >> 8
+        w_iterable = self.popvalue()
+
+        items = self.space.fixedview(w_iterable)
+        itemcount = len(items)
+
+        i = itemcount - 1
+        while i >= itemcount-right:
+            self.pushvalue(items[i])
+            i -= 1
+
+        self.pushvalue(self.space.newlist(items[left:itemcount-right]))
+
+        i = left - 1
+        while i >= 0:
+            self.pushvalue(items[i])
+            i -= 1
+
 
     def STORE_ATTR(self, nameindex, next_instr):
         "obj.attributename = newvalue"

From noreply at buildbot.pypy.org  Wed Jan 18 17:09:20 2012
From: noreply at buildbot.pypy.org (rguillebert)
Date: Wed, 18 Jan 2012 17:09:20 +0100 (CET)
Subject: [pypy-commit] pypy py3k: Check if the destination doesn't contain
	multiple stars
Message-ID: <20120118160920.DD12F820D8@wyvern.cs.uni-duesseldorf.de>

Author: Romain Guillebert 
Branch: py3k
Changeset: r51448:d0b94548df3b
Date: 2012-01-18 14:22 +0100
http://bitbucket.org/pypy/pypy/changeset/d0b94548df3b/

Log:	Check if the destination doesn't contain multiple stars

diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -837,6 +837,8 @@
             if elt_count > 0:
                 for i, elt in enumerate(tup.elts):
                     if isinstance(elt, ast.Starred):
+                        if star_pos != -1:
+                            self.error("two starred expressions in assignment", tup)
                         star_pos = i
             if star_pos > -1:
                 self.emit_op_arg(ops.UNPACK_EX, star_pos | (elt_count-star_pos-1)<<8)

From noreply at buildbot.pypy.org  Wed Jan 18 17:53:46 2012
From: noreply at buildbot.pypy.org (bivab)
Date: Wed, 18 Jan 2012 17:53:46 +0100 (CET)
Subject: [pypy-commit] pypy arm-backend-2: (arigo,
 bivab) fix for guard_nonnull_class which was emitting two guards.
Message-ID: <20120118165346.E6965820D8@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider 
Branch: arm-backend-2
Changeset: r51449:48cc79e4f981
Date: 2012-01-18 17:32 +0100
http://bitbucket.org/pypy/pypy/changeset/48cc79e4f981/

Log:	(arigo, bivab) fix for guard_nonnull_class which was emitting two
	guards. Additionally make sure that the offset is a small enough imm
	value.

diff --git a/pypy/jit/backend/arm/opassembler.py b/pypy/jit/backend/arm/opassembler.py
--- a/pypy/jit/backend/arm/opassembler.py
+++ b/pypy/jit/backend/arm/opassembler.py
@@ -267,38 +267,28 @@
     # from ../x86/assembler.py:1265
     def emit_op_guard_class(self, op, arglocs, regalloc, fcond):
         self._cmp_guard_class(op, arglocs, regalloc, fcond)
+        self._emit_guard(op, arglocs[3:], c.EQ, save_exc=False)
         return fcond
 
     def emit_op_guard_nonnull_class(self, op, arglocs, regalloc, fcond):
-        offset = self.cpu.vtable_offset
+        self.mc.CMP_ri(arglocs[0].value, 1)
+        self._cmp_guard_class(op, arglocs, regalloc, c.HS)
+        self._emit_guard(op, arglocs[3:], c.EQ, save_exc=False)
+        return fcond
 
-        self.mc.CMP_ri(arglocs[0].value, 0)
+    def _cmp_guard_class(self, op, locs, regalloc, fcond):
+        offset = locs[2]
         if offset is not None:
-            self._emit_guard(op, arglocs[3:], c.NE, save_exc=False)
+            self.mc.LDR_ri(r.ip.value, locs[0].value, offset.value, cond=fcond)
+            self.mc.CMP_rr(r.ip.value, locs[1].value, cond=fcond)
         else:
             raise NotImplementedError
-        self._cmp_guard_class(op, arglocs, regalloc, fcond)
-        return fcond
+            # XXX port from x86 backend once gc support is in place
 
     def emit_op_guard_not_invalidated(self, op, locs, regalloc, fcond):
         return self._emit_guard(op, locs, fcond, save_exc=False,
                                             is_guard_not_invalidated=True)
 
-    def _cmp_guard_class(self, op, locs, regalloc, fcond):
-        offset = locs[2]
-        if offset is not None:
-            if offset.is_imm():
-                self.mc.LDR_ri(r.ip.value, locs[0].value, offset.value)
-            else:
-                assert offset.is_reg()
-                self.mc.LDR_rr(r.ip.value, locs[0].value, offset.value)
-            self.mc.CMP_rr(r.ip.value, locs[1].value)
-        else:
-            raise NotImplementedError
-            # XXX port from x86 backend once gc support is in place
-
-        return self._emit_guard(op, locs[3:], c.EQ, save_exc=False)
-
 
 class OpAssembler(object):
 
diff --git a/pypy/jit/backend/arm/regalloc.py b/pypy/jit/backend/arm/regalloc.py
--- a/pypy/jit/backend/arm/regalloc.py
+++ b/pypy/jit/backend/arm/regalloc.py
@@ -646,13 +646,14 @@
         boxes = op.getarglist()
 
         x = self._ensure_value_is_boxed(boxes[0], boxes)
-        y = self.get_scratch_reg(REF, forbidden_vars=boxes)
+        y = self.get_scratch_reg(INT, forbidden_vars=boxes)
         y_val = rffi.cast(lltype.Signed, op.getarg(1).getint())
         self.assembler.load(y, imm(y_val))
 
         offset = self.cpu.vtable_offset
         assert offset is not None
-        offset_loc = self._ensure_value_is_boxed(ConstInt(offset), boxes)
+        assert check_imm_arg(offset)
+        offset_loc = imm(offset)
         arglocs = self._prepare_guard(op, [x, y, offset_loc])
 
         return arglocs

From noreply at buildbot.pypy.org  Wed Jan 18 17:53:48 2012
From: noreply at buildbot.pypy.org (bivab)
Date: Wed, 18 Jan 2012 17:53:48 +0100 (CET)
Subject: [pypy-commit] pypy arm-backend-2: (arigo,
 bivab) refactor guard and guard_token handling a bit and get rid of
 _arm_guard_pos
Message-ID: <20120118165348.29CF0820D8@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider 
Branch: arm-backend-2
Changeset: r51450:1caec07bb02e
Date: 2012-01-18 17:33 +0100
http://bitbucket.org/pypy/pypy/changeset/1caec07bb02e/

Log:	(arigo, bivab) refactor guard and guard_token handling a bit and get
	rid of _arm_guard_pos

diff --git a/pypy/jit/backend/arm/assembler.py b/pypy/jit/backend/arm/assembler.py
--- a/pypy/jit/backend/arm/assembler.py
+++ b/pypy/jit/backend/arm/assembler.py
@@ -674,7 +674,7 @@
         return AsmInfo(ops_offset, startpos + rawstart, codeendpos - startpos)
 
     def _find_failure_recovery_bytecode(self, faildescr):
-        guard_stub_addr = faildescr._arm_recovery_stub_offset
+        guard_stub_addr = faildescr._arm_failure_recovery_block
         if guard_stub_addr == 0:
             # This case should be prevented by the logic in compile.py:
             # look for CNT_BUSY_FLAG, which disables tracing from a guard
@@ -709,34 +709,25 @@
                                         tok.faillocs, save_exc=tok.save_exc)
             # store info on the descr
             descr._arm_current_frame_depth = tok.faillocs[0].getint()
-            descr._arm_guard_pos = pos
 
     def process_pending_guards(self, block_start):
         clt = self.current_clt
         for tok in self.pending_guards:
             descr = tok.descr
             assert isinstance(descr, AbstractFailDescr)
-            jump_target = tok.pos_recovery_stub
-            relative_target = jump_target - tok.offset
-
-            addr = block_start + tok.offset
-            stub_addr = block_start + jump_target
-
-            descr._arm_recovery_stub_offset = stub_addr
-
+            failure_recovery_pos = block_start + tok.pos_recovery_stub
+            descr._arm_failure_recovery_block = failure_recovery_pos
+            relative_offset = tok.pos_recovery_stub - tok.offset
+            guard_pos = block_start + tok.offset
             if not tok.is_invalidate:
-                #patch the guard jumpt to the stub
+                # patch the guard jumpt to the stub
                 # overwrite the generate NOP with a B_offs to the pos of the
                 # stub
                 mc = ARMv7Builder()
-                mc.B_offs(relative_target, c.get_opposite_of(tok.fcond))
-                mc.copy_to_raw_memory(addr)
+                mc.B_offs(relative_offset, c.get_opposite_of(tok.fcond))
+                mc.copy_to_raw_memory(guard_pos)
             else:
-                # GUARD_NOT_INVALIDATED, record an entry in
-                # clt.invalidate_positions of the form:
-                #     (addr-in-the-code-of-the-not-yet-written-jump-target,
-                #      relative-target-to-use)
-                clt.invalidate_positions.append((addr, relative_target))
+                clt.invalidate_positions.append((guard_pos, relative_offset))
 
     def get_asmmemmgr_blocks(self, looptoken):
         clt = looptoken.compiled_loop_token
@@ -875,14 +866,12 @@
                 self.mc.ASR_ri(resloc.value, resloc.value, 16)
 
     def patch_trace(self, faildescr, looptoken, bridge_addr, regalloc):
-        # The first instruction (word) is not overwritten, because it is the
-        # one that actually checks the condition
         b = ARMv7Builder()
-        adr_jump_offset = faildescr._arm_recovery_stub_offset
-        assert adr_jump_offset != 0
+        patch_addr = faildescr._arm_failure_recovery_block
+        assert patch_addr != 0
         b.B(bridge_addr)
-        b.copy_to_raw_memory(adr_jump_offset)
-        faildescr._arm_recovery_stub_offset = 0
+        b.copy_to_raw_memory(patch_addr)
+        faildescr._arm_failure_recovery_block = 0
 
     # regalloc support
     def load(self, loc, value):

From noreply at buildbot.pypy.org  Wed Jan 18 17:53:49 2012
From: noreply at buildbot.pypy.org (bivab)
Date: Wed, 18 Jan 2012 17:53:49 +0100 (CET)
Subject: [pypy-commit] pypy arm-backend-2: (bivab,
 arigo) Add test for an operation that does not correctly emit the
 code for the guard, i.e. emitting two guards for the same operation
Message-ID: <20120118165349.61A5C820D8@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider 
Branch: arm-backend-2
Changeset: r51451:8dd9c2555f29
Date: 2012-01-18 17:46 +0100
http://bitbucket.org/pypy/pypy/changeset/8dd9c2555f29/

Log:	(bivab, arigo) Add test for an operation that does not correctly
	emit the code for the guard, i.e. emitting two guards for the same
	operation

diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py
--- a/pypy/jit/backend/test/runner_test.py
+++ b/pypy/jit/backend/test/runner_test.py
@@ -3273,6 +3273,24 @@
         fail = self.cpu.execute_token(looptoken2, -9)
         assert fail.identifier == 42
 
+    def test_some_issue(self):
+        t_box, T_box = self.alloc_instance(self.T)
+        null_box = self.null_instance()
+        faildescr = BasicFailDescr(42)
+        operations = [
+            ResOperation(rop.GUARD_NONNULL_CLASS, [t_box, T_box], None,
+                                                        descr=faildescr),
+            ResOperation(rop.FINISH, [], None, descr=BasicFailDescr(1))]
+        operations[0].setfailargs([])
+        looptoken = JitCellToken()
+        inputargs = [t_box]
+        self.cpu.compile_loop(inputargs, operations, looptoken)
+        operations = [
+            ResOperation(rop.FINISH, [], None, descr=BasicFailDescr(99))
+        ]
+        self.cpu.compile_bridge(faildescr, [], operations, looptoken)
+        fail = self.cpu.execute_token(looptoken, null_box.getref_base())
+        assert fail.identifier == 99
 
 class OOtypeBackendTest(BaseBackendTest):
 

From noreply at buildbot.pypy.org  Wed Jan 18 17:53:50 2012
From: noreply at buildbot.pypy.org (bivab)
Date: Wed, 18 Jan 2012 17:53:50 +0100 (CET)
Subject: [pypy-commit] pypy arm-backend-2: merge heads
Message-ID: <20120118165350.9EFFB820D8@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider 
Branch: arm-backend-2
Changeset: r51452:aee28928eeae
Date: 2012-01-18 17:46 +0100
http://bitbucket.org/pypy/pypy/changeset/aee28928eeae/

Log:	merge heads

diff --git a/pypy/jit/backend/arm/assembler.py b/pypy/jit/backend/arm/assembler.py
--- a/pypy/jit/backend/arm/assembler.py
+++ b/pypy/jit/backend/arm/assembler.py
@@ -674,7 +674,7 @@
         return AsmInfo(ops_offset, startpos + rawstart, codeendpos - startpos)
 
     def _find_failure_recovery_bytecode(self, faildescr):
-        guard_stub_addr = faildescr._arm_recovery_stub_offset
+        guard_stub_addr = faildescr._arm_failure_recovery_block
         if guard_stub_addr == 0:
             # This case should be prevented by the logic in compile.py:
             # look for CNT_BUSY_FLAG, which disables tracing from a guard
@@ -709,34 +709,25 @@
                                         tok.faillocs, save_exc=tok.save_exc)
             # store info on the descr
             descr._arm_current_frame_depth = tok.faillocs[0].getint()
-            descr._arm_guard_pos = pos
 
     def process_pending_guards(self, block_start):
         clt = self.current_clt
         for tok in self.pending_guards:
             descr = tok.descr
             assert isinstance(descr, AbstractFailDescr)
-            jump_target = tok.pos_recovery_stub
-            relative_target = jump_target - tok.offset
-
-            addr = block_start + tok.offset
-            stub_addr = block_start + jump_target
-
-            descr._arm_recovery_stub_offset = stub_addr
-
+            failure_recovery_pos = block_start + tok.pos_recovery_stub
+            descr._arm_failure_recovery_block = failure_recovery_pos
+            relative_offset = tok.pos_recovery_stub - tok.offset
+            guard_pos = block_start + tok.offset
             if not tok.is_invalidate:
-                #patch the guard jumpt to the stub
+                # patch the guard jumpt to the stub
                 # overwrite the generate NOP with a B_offs to the pos of the
                 # stub
                 mc = ARMv7Builder()
-                mc.B_offs(relative_target, c.get_opposite_of(tok.fcond))
-                mc.copy_to_raw_memory(addr)
+                mc.B_offs(relative_offset, c.get_opposite_of(tok.fcond))
+                mc.copy_to_raw_memory(guard_pos)
             else:
-                # GUARD_NOT_INVALIDATED, record an entry in
-                # clt.invalidate_positions of the form:
-                #     (addr-in-the-code-of-the-not-yet-written-jump-target,
-                #      relative-target-to-use)
-                clt.invalidate_positions.append((addr, relative_target))
+                clt.invalidate_positions.append((guard_pos, relative_offset))
 
     def get_asmmemmgr_blocks(self, looptoken):
         clt = looptoken.compiled_loop_token
@@ -875,14 +866,12 @@
                 self.mc.ASR_ri(resloc.value, resloc.value, 16)
 
     def patch_trace(self, faildescr, looptoken, bridge_addr, regalloc):
-        # The first instruction (word) is not overwritten, because it is the
-        # one that actually checks the condition
         b = ARMv7Builder()
-        adr_jump_offset = faildescr._arm_recovery_stub_offset
-        assert adr_jump_offset != 0
+        patch_addr = faildescr._arm_failure_recovery_block
+        assert patch_addr != 0
         b.B(bridge_addr)
-        b.copy_to_raw_memory(adr_jump_offset)
-        faildescr._arm_recovery_stub_offset = 0
+        b.copy_to_raw_memory(patch_addr)
+        faildescr._arm_failure_recovery_block = 0
 
     # regalloc support
     def load(self, loc, value):
diff --git a/pypy/jit/backend/arm/opassembler.py b/pypy/jit/backend/arm/opassembler.py
--- a/pypy/jit/backend/arm/opassembler.py
+++ b/pypy/jit/backend/arm/opassembler.py
@@ -267,38 +267,28 @@
     # from ../x86/assembler.py:1265
     def emit_op_guard_class(self, op, arglocs, regalloc, fcond):
         self._cmp_guard_class(op, arglocs, regalloc, fcond)
+        self._emit_guard(op, arglocs[3:], c.EQ, save_exc=False)
         return fcond
 
     def emit_op_guard_nonnull_class(self, op, arglocs, regalloc, fcond):
-        offset = self.cpu.vtable_offset
+        self.mc.CMP_ri(arglocs[0].value, 1)
+        self._cmp_guard_class(op, arglocs, regalloc, c.HS)
+        self._emit_guard(op, arglocs[3:], c.EQ, save_exc=False)
+        return fcond
 
-        self.mc.CMP_ri(arglocs[0].value, 0)
+    def _cmp_guard_class(self, op, locs, regalloc, fcond):
+        offset = locs[2]
         if offset is not None:
-            self._emit_guard(op, arglocs[3:], c.NE, save_exc=False)
+            self.mc.LDR_ri(r.ip.value, locs[0].value, offset.value, cond=fcond)
+            self.mc.CMP_rr(r.ip.value, locs[1].value, cond=fcond)
         else:
             raise NotImplementedError
-        self._cmp_guard_class(op, arglocs, regalloc, fcond)
-        return fcond
+            # XXX port from x86 backend once gc support is in place
 
     def emit_op_guard_not_invalidated(self, op, locs, regalloc, fcond):
         return self._emit_guard(op, locs, fcond, save_exc=False,
                                             is_guard_not_invalidated=True)
 
-    def _cmp_guard_class(self, op, locs, regalloc, fcond):
-        offset = locs[2]
-        if offset is not None:
-            if offset.is_imm():
-                self.mc.LDR_ri(r.ip.value, locs[0].value, offset.value)
-            else:
-                assert offset.is_reg()
-                self.mc.LDR_rr(r.ip.value, locs[0].value, offset.value)
-            self.mc.CMP_rr(r.ip.value, locs[1].value)
-        else:
-            raise NotImplementedError
-            # XXX port from x86 backend once gc support is in place
-
-        return self._emit_guard(op, locs[3:], c.EQ, save_exc=False)
-
 
 class OpAssembler(object):
 
diff --git a/pypy/jit/backend/arm/regalloc.py b/pypy/jit/backend/arm/regalloc.py
--- a/pypy/jit/backend/arm/regalloc.py
+++ b/pypy/jit/backend/arm/regalloc.py
@@ -646,13 +646,14 @@
         boxes = op.getarglist()
 
         x = self._ensure_value_is_boxed(boxes[0], boxes)
-        y = self.get_scratch_reg(REF, forbidden_vars=boxes)
+        y = self.get_scratch_reg(INT, forbidden_vars=boxes)
         y_val = rffi.cast(lltype.Signed, op.getarg(1).getint())
         self.assembler.load(y, imm(y_val))
 
         offset = self.cpu.vtable_offset
         assert offset is not None
-        offset_loc = self._ensure_value_is_boxed(ConstInt(offset), boxes)
+        assert check_imm_arg(offset)
+        offset_loc = imm(offset)
         arglocs = self._prepare_guard(op, [x, y, offset_loc])
 
         return arglocs

From noreply at buildbot.pypy.org  Wed Jan 18 17:53:51 2012
From: noreply at buildbot.pypy.org (bivab)
Date: Wed, 18 Jan 2012 17:53:51 +0100 (CET)
Subject: [pypy-commit] pypy arm-backend-2: give test a proper name
Message-ID: <20120118165351.D76AD820D8@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider 
Branch: arm-backend-2
Changeset: r51453:0e325c9172fb
Date: 2012-01-18 17:49 +0100
http://bitbucket.org/pypy/pypy/changeset/0e325c9172fb/

Log:	give test a proper name

diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py
--- a/pypy/jit/backend/test/runner_test.py
+++ b/pypy/jit/backend/test/runner_test.py
@@ -3273,7 +3273,7 @@
         fail = self.cpu.execute_token(looptoken2, -9)
         assert fail.identifier == 42
 
-    def test_some_issue(self):
+    def test_wrong_guard_nonnull_class(self):
         t_box, T_box = self.alloc_instance(self.T)
         null_box = self.null_instance()
         faildescr = BasicFailDescr(42)

From noreply at buildbot.pypy.org  Wed Jan 18 18:15:27 2012
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Wed, 18 Jan 2012 18:15:27 +0100 (CET)
Subject: [pypy-commit] pypy default: issue #1005 -- accept 'float' as a dtype
Message-ID: <20120118171527.E4F70820D8@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor 
Branch: 
Changeset: r51454:923df65f2286
Date: 2012-01-18 11:15 -0600
http://bitbucket.org/pypy/pypy/changeset/923df65f2286/

Log:	issue #1005 -- accept 'float' as a dtype

diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -20,7 +20,7 @@
 class W_Dtype(Wrappable):
     _immutable_fields_ = ["itemtype", "num", "kind"]
 
-    def __init__(self, itemtype, num, kind, name, char, w_box_type, alternate_constructors=[]):
+    def __init__(self, itemtype, num, kind, name, char, w_box_type, alternate_constructors=[], aliases=[]):
         self.itemtype = itemtype
         self.num = num
         self.kind = kind
@@ -28,6 +28,7 @@
         self.char = char
         self.w_box_type = w_box_type
         self.alternate_constructors = alternate_constructors
+        self.aliases = aliases
 
     def malloc(self, length):
         # XXX find out why test_zjit explodes with tracking of allocations
@@ -62,7 +63,7 @@
         elif space.isinstance_w(w_dtype, space.w_str):
             name = space.str_w(w_dtype)
             for dtype in cache.builtin_dtypes:
-                if dtype.name == name or dtype.char == name:
+                if dtype.name == name or dtype.char == name or name in dtype.aliases:
                     return dtype
         else:
             for dtype in cache.builtin_dtypes:
@@ -107,7 +108,7 @@
             kind=BOOLLTR,
             name="bool",
             char="?",
-            w_box_type = space.gettypefor(interp_boxes.W_BoolBox),
+            w_box_type=space.gettypefor(interp_boxes.W_BoolBox),
             alternate_constructors=[space.w_bool],
         )
         self.w_int8dtype = W_Dtype(
@@ -116,7 +117,7 @@
             kind=SIGNEDLTR,
             name="int8",
             char="b",
-            w_box_type = space.gettypefor(interp_boxes.W_Int8Box)
+            w_box_type=space.gettypefor(interp_boxes.W_Int8Box)
         )
         self.w_uint8dtype = W_Dtype(
             types.UInt8(),
@@ -124,7 +125,7 @@
             kind=UNSIGNEDLTR,
             name="uint8",
             char="B",
-            w_box_type = space.gettypefor(interp_boxes.W_UInt8Box),
+            w_box_type=space.gettypefor(interp_boxes.W_UInt8Box),
         )
         self.w_int16dtype = W_Dtype(
             types.Int16(),
@@ -132,7 +133,7 @@
             kind=SIGNEDLTR,
             name="int16",
             char="h",
-            w_box_type = space.gettypefor(interp_boxes.W_Int16Box),
+            w_box_type=space.gettypefor(interp_boxes.W_Int16Box),
         )
         self.w_uint16dtype = W_Dtype(
             types.UInt16(),
@@ -140,7 +141,7 @@
             kind=UNSIGNEDLTR,
             name="uint16",
             char="H",
-            w_box_type = space.gettypefor(interp_boxes.W_UInt16Box),
+            w_box_type=space.gettypefor(interp_boxes.W_UInt16Box),
         )
         self.w_int32dtype = W_Dtype(
             types.Int32(),
@@ -148,7 +149,7 @@
             kind=SIGNEDLTR,
             name="int32",
             char="i",
-             w_box_type = space.gettypefor(interp_boxes.W_Int32Box),
+            w_box_type=space.gettypefor(interp_boxes.W_Int32Box),
        )
         self.w_uint32dtype = W_Dtype(
             types.UInt32(),
@@ -156,7 +157,7 @@
             kind=UNSIGNEDLTR,
             name="uint32",
             char="I",
-            w_box_type = space.gettypefor(interp_boxes.W_UInt32Box),
+            w_box_type=space.gettypefor(interp_boxes.W_UInt32Box),
         )
         if LONG_BIT == 32:
             name = "int32"
@@ -168,7 +169,7 @@
             kind=SIGNEDLTR,
             name=name,
             char="l",
-            w_box_type = space.gettypefor(interp_boxes.W_LongBox),
+            w_box_type=space.gettypefor(interp_boxes.W_LongBox),
             alternate_constructors=[space.w_int],
         )
         self.w_ulongdtype = W_Dtype(
@@ -177,7 +178,7 @@
             kind=UNSIGNEDLTR,
             name="u" + name,
             char="L",
-            w_box_type = space.gettypefor(interp_boxes.W_ULongBox),
+            w_box_type=space.gettypefor(interp_boxes.W_ULongBox),
         )
         self.w_int64dtype = W_Dtype(
             types.Int64(),
@@ -185,7 +186,7 @@
             kind=SIGNEDLTR,
             name="int64",
             char="q",
-            w_box_type = space.gettypefor(interp_boxes.W_Int64Box),
+            w_box_type=space.gettypefor(interp_boxes.W_Int64Box),
             alternate_constructors=[space.w_long],
         )
         self.w_uint64dtype = W_Dtype(
@@ -194,7 +195,7 @@
             kind=UNSIGNEDLTR,
             name="uint64",
             char="Q",
-            w_box_type = space.gettypefor(interp_boxes.W_UInt64Box),
+            w_box_type=space.gettypefor(interp_boxes.W_UInt64Box),
         )
         self.w_float32dtype = W_Dtype(
             types.Float32(),
@@ -202,7 +203,7 @@
             kind=FLOATINGLTR,
             name="float32",
             char="f",
-            w_box_type = space.gettypefor(interp_boxes.W_Float32Box),
+            w_box_type=space.gettypefor(interp_boxes.W_Float32Box),
         )
         self.w_float64dtype = W_Dtype(
             types.Float64(),
@@ -212,6 +213,7 @@
             char="d",
             w_box_type = space.gettypefor(interp_boxes.W_Float64Box),
             alternate_constructors=[space.w_float],
+            aliases=["float"],
         )
 
         self.builtin_dtypes = [
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -166,14 +166,11 @@
         # You can't subclass dtype
         raises(TypeError, type, "Foo", (dtype,), {})
 
-    def test_new(self):
-        import _numpypy as np
-        assert np.int_(4) == 4
-        assert np.float_(3.4) == 3.4
+    def test_aliases(self):
+        from _numpypy import dtype
 
-    def test_pow(self):
-        from _numpypy import int_
-        assert int_(4) ** 2 == 16
+        assert dtype("float") is dtype(float)
+
 
 class AppTestTypes(BaseNumpyAppTest):
     def test_abstract_types(self):
@@ -189,6 +186,15 @@
         raises(TypeError, numpy.floating, 0)
         raises(TypeError, numpy.inexact, 0)
 
+    def test_new(self):
+        import _numpypy as np
+        assert np.int_(4) == 4
+        assert np.float_(3.4) == 3.4
+
+    def test_pow(self):
+        from _numpypy import int_
+        assert int_(4) ** 2 == 16
+
     def test_bool(self):
         import _numpypy as numpy
 
@@ -318,7 +324,7 @@
         else:
             raises(OverflowError, numpy.int64, 9223372036854775807)
             raises(OverflowError, numpy.int64, '9223372036854775807')
-        
+
         raises(OverflowError, numpy.int64, 9223372036854775808)
         raises(OverflowError, numpy.int64, '9223372036854775808')
 

From noreply at buildbot.pypy.org  Wed Jan 18 18:51:22 2012
From: noreply at buildbot.pypy.org (bivab)
Date: Wed, 18 Jan 2012 18:51:22 +0100 (CET)
Subject: [pypy-commit] pypy ppc-jit-backend: (arigo,
 bivab) use get_spp_offset to correctly compute the offset which is
 currently differnt for positive and negative stack locations, arg!
Message-ID: <20120118175122.9020C820D8@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider 
Branch: ppc-jit-backend
Changeset: r51455:d9303a92441b
Date: 2012-01-18 09:42 -0800
http://bitbucket.org/pypy/pypy/changeset/d9303a92441b/

Log:	(arigo, bivab) use get_spp_offset to correctly compute the offset
	which is currently differnt for positive and negative stack
	locations, arg!

diff --git a/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py b/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py
--- a/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py
+++ b/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py
@@ -40,7 +40,7 @@
 from pypy.rpython.annlowlevel import llhelper
 from pypy.rlib.objectmodel import we_are_translated
 from pypy.rpython.lltypesystem.lloperation import llop
-from pypy.jit.backend.ppc.ppcgen.locations import StackLocation
+from pypy.jit.backend.ppc.ppcgen.locations import StackLocation, get_spp_offset
 
 memcpy_fn = rffi.llexternal('memcpy', [llmemory.Address, llmemory.Address,
                                        rffi.SIZE_T], lltype.Void,
@@ -212,7 +212,7 @@
                 if group == self.FLOAT_TYPE:
                     assert 0, "not implemented yet"
                 else:
-                    start = spp_loc - (stack_location + 1) * WORD
+                    start = spp_loc + get_spp_offset(stack_location)
                     value = rffi.cast(rffi.LONGP, start)[0]
             else: # REG_LOC
                 reg = ord(enc[i])

From noreply at buildbot.pypy.org  Wed Jan 18 18:51:23 2012
From: noreply at buildbot.pypy.org (bivab)
Date: Wed, 18 Jan 2012 18:51:23 +0100 (CET)
Subject: [pypy-commit] pypy ppc-jit-backend: (arigo,
 bivab) Add tests for finish and guard that check that fail/finish
 args pased to the loop on the stack are decoded as they should
Message-ID: <20120118175123.C957B820D8@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider 
Branch: ppc-jit-backend
Changeset: r51456:7a455997891c
Date: 2012-01-18 09:48 -0800
http://bitbucket.org/pypy/pypy/changeset/7a455997891c/

Log:	(arigo, bivab) Add tests for finish and guard that check that
	fail/finish args pased to the loop on the stack are decoded as they
	should

diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py
--- a/pypy/jit/backend/test/runner_test.py
+++ b/pypy/jit/backend/test/runner_test.py
@@ -3359,6 +3359,27 @@
         fail = self.cpu.execute_token(looptoken2, -9)
         assert fail.identifier == 42
 
+    def test_finish_with_long_arglist(self):
+        boxes = [BoxInt(i) for i in range(30)]
+        ops = [ResOperation(rop.FINISH, boxes, None, descr=BasicFailDescr(1))]
+        looptoken = JitCellToken()
+        self.cpu.compile_loop(boxes, ops, looptoken)
+        fail = self.cpu.execute_token(looptoken, *range(30))
+        assert fail.identifier == 1
+        for i in range(30):
+            assert self.cpu.get_latest_value_int(i) == i
+
+    def test_guard_with_long_arglist(self):
+        boxes = [BoxInt(i) for i in range(30)]
+        ops = [ResOperation(rop.GUARD_FALSE, [boxes[1]], None, descr=BasicFailDescr(1)),
+		ResOperation(rop.FINISH, [], None, descr=BasicFailDescr(2))]
+	ops[0].setfailargs(boxes)
+        looptoken = JitCellToken()
+        self.cpu.compile_loop(boxes, ops, looptoken)
+        fail = self.cpu.execute_token(looptoken, *range(30))
+        assert fail.identifier == 1
+        for i in range(30):
+            assert self.cpu.get_latest_value_int(i) == i
 
 class OOtypeBackendTest(BaseBackendTest):
 

From noreply at buildbot.pypy.org  Wed Jan 18 18:51:25 2012
From: noreply at buildbot.pypy.org (bivab)
Date: Wed, 18 Jan 2012 18:51:25 +0100 (CET)
Subject: [pypy-commit] pypy ppc-jit-backend: merge heads
Message-ID: <20120118175125.10319820D8@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider 
Branch: ppc-jit-backend
Changeset: r51457:326dc978de77
Date: 2012-01-18 09:49 -0800
http://bitbucket.org/pypy/pypy/changeset/326dc978de77/

Log:	merge heads

diff --git a/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py b/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py
--- a/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py
+++ b/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py
@@ -40,7 +40,7 @@
 from pypy.rpython.annlowlevel import llhelper
 from pypy.rlib.objectmodel import we_are_translated
 from pypy.rpython.lltypesystem.lloperation import llop
-from pypy.jit.backend.ppc.ppcgen.locations import StackLocation
+from pypy.jit.backend.ppc.ppcgen.locations import StackLocation, get_spp_offset
 
 memcpy_fn = rffi.llexternal('memcpy', [llmemory.Address, llmemory.Address,
                                        rffi.SIZE_T], lltype.Void,
@@ -212,7 +212,7 @@
                 if group == self.FLOAT_TYPE:
                     assert 0, "not implemented yet"
                 else:
-                    start = spp_loc - (stack_location + 1) * WORD
+                    start = spp_loc + get_spp_offset(stack_location)
                     value = rffi.cast(rffi.LONGP, start)[0]
             else: # REG_LOC
                 reg = ord(enc[i])

From noreply at buildbot.pypy.org  Wed Jan 18 18:52:17 2012
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Wed, 18 Jan 2012 18:52:17 +0100 (CET)
Subject: [pypy-commit] pypy default: Issue #1007 -- added fill() method to
	numpypy arrays
Message-ID: <20120118175217.459AF820D8@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor 
Branch: 
Changeset: r51458:36de78269626
Date: 2012-01-18 11:52 -0600
http://bitbucket.org/pypy/pypy/changeset/36de78269626/

Log:	Issue #1007 -- added fill() method to numpypy arrays

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -581,7 +581,7 @@
     def descr_var(self, space):
         # var = mean((values - mean(values)) ** 2)
         w_res = self.descr_sub(space, self.descr_mean(space, space.w_None))
-        assert isinstance(w_res, BaseArray) 
+        assert isinstance(w_res, BaseArray)
         w_res = w_res.descr_pow(space, space.wrap(2))
         assert isinstance(w_res, BaseArray)
         return w_res.descr_mean(space, space.w_None)
@@ -590,6 +590,10 @@
         # std(v) = sqrt(var(v))
         return interp_ufuncs.get(space).sqrt.call(space, [self.descr_var(space)])
 
+    def descr_fill(self, space, w_value):
+        concr = self.get_concrete_or_scalar()
+        concr.fill(space, w_value)
+
     def descr_nonzero(self, space):
         if self.size > 1:
             raise OperationError(space.w_ValueError, space.wrap(
@@ -682,6 +686,9 @@
     def copy(self, space):
         return Scalar(self.dtype, self.value)
 
+    def fill(self, space, w_value):
+        self.value = self.dtype.coerce(space, w_value)
+
     def create_sig(self):
         return signature.ScalarSignature(self.dtype)
 
@@ -788,7 +795,7 @@
     Intermediate class for performing binary operations.
     """
     _immutable_fields_ = ['left', 'right']
-    
+
     def __init__(self, ufunc, name, shape, calc_dtype, res_dtype, left, right):
         VirtualArray.__init__(self, name, shape, res_dtype)
         self.ufunc = ufunc
@@ -828,7 +835,7 @@
     def __init__(self, shape, dtype, left, right):
         Call2.__init__(self, None, 'sliceloop', shape, dtype, dtype, left,
                        right)
-    
+
     def create_sig(self):
         lsig = self.left.create_sig()
         rsig = self.right.create_sig()
@@ -847,7 +854,7 @@
     when we'll make AxisReduce lazy
     """
     _immutable_fields_ = ['left', 'right']
-    
+
     def __init__(self, ufunc, name, shape, dtype, left, right, dim):
         Call2.__init__(self, ufunc, name, shape, dtype, dtype,
                        left, right)
@@ -1061,6 +1068,9 @@
         array.setslice(space, self)
         return array
 
+    def fill(self, space, w_value):
+        self.setslice(space, scalar_w(space, self.dtype, w_value))
+
 
 class ViewArray(ConcreteArray):
     def create_sig(self):
@@ -1273,6 +1283,8 @@
     var = interp2app(BaseArray.descr_var),
     std = interp2app(BaseArray.descr_std),
 
+    fill = interp2app(BaseArray.descr_fill),
+
     copy = interp2app(BaseArray.descr_copy),
     reshape = interp2app(BaseArray.descr_reshape),
     tolist = interp2app(BaseArray.descr_tolist),
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1302,6 +1302,28 @@
         assert isinstance(i['data'][0], int)
         raises(TypeError, getattr, array(3), '__array_interface__')
 
+    def test_fill(self):
+        from _numpypy import array
+
+        a = array([1, 2, 3])
+        a.fill(10)
+        assert (a == [10, 10, 10]).all()
+        a.fill(False)
+        assert (a == [0, 0, 0]).all()
+
+        b = a[:1]
+        b.fill(4)
+        assert (b == [4]).all()
+        assert (a == [4, 0, 0]).all()
+
+        c = b + b
+        c.fill(27)
+        assert (c == [27]).all()
+
+        d = array(10)
+        d.fill(100)
+        assert d == 100
+
 
 class AppTestSupport(BaseNumpyAppTest):
     def setup_class(cls):

From noreply at buildbot.pypy.org  Wed Jan 18 18:55:37 2012
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Wed, 18 Jan 2012 18:55:37 +0100 (CET)
Subject: [pypy-commit] pypy default: add an arrylen_gc to this so the test
 passes, it's killed by the backend (I wish we did DCE earlier)
Message-ID: <20120118175537.EE994820D8@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor 
Branch: 
Changeset: r51459:8f0e9fcd8de3
Date: 2012-01-18 11:55 -0600
http://bitbucket.org/pypy/pypy/changeset/8f0e9fcd8de3/

Log:	add an arrylen_gc to this so the test passes, it's killed by the
	backend (I wish we did DCE earlier)

diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -349,7 +349,8 @@
         self.check_trace_count(1)
         self.check_simple_loop({'getinteriorfield_raw': 2, 'float_add': 1,
                                 'setinteriorfield_raw': 1, 'int_add': 2,
-                                'int_eq': 1, 'guard_false': 1, 'jump': 1})
+                                'int_eq': 1, 'guard_false': 1, 'jump': 1,
+                                'arraylen_gc': 1})
 
     def define_virtual_slice():
         return """

From noreply at buildbot.pypy.org  Wed Jan 18 20:05:00 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Wed, 18 Jan 2012 20:05:00 +0100 (CET)
Subject: [pypy-commit] pypy py3k: (arigo, romain,
 antocuni) aaargh. enumerate did not convert the repr of the items,
 resulting in a lot of confusion if you enumerate a list of instances,
 leading to incorrect C code. Fix it,
 and add a sanity check inside rtuple.newtuple
Message-ID: <20120118190500.F33C7820D8@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni 
Branch: py3k
Changeset: r51460:8df1c2074547
Date: 2012-01-18 20:03 +0100
http://bitbucket.org/pypy/pypy/changeset/8df1c2074547/

Log:	(arigo, romain, antocuni) aaargh. enumerate did not convert the repr
	of the items, resulting in a lot of confusion if you enumerate a
	list of instances, leading to incorrect C code. Fix it, and add a
	sanity check inside rtuple.newtuple

diff --git a/pypy/rpython/lltypesystem/rtuple.py b/pypy/rpython/lltypesystem/rtuple.py
--- a/pypy/rpython/lltypesystem/rtuple.py
+++ b/pypy/rpython/lltypesystem/rtuple.py
@@ -27,6 +27,10 @@
 
     def newtuple(cls, llops, r_tuple, items_v):
         # items_v should have the lowleveltype of the internal reprs
+        assert len(r_tuple.items_r) == len(items_v)
+        for r_item, v_item in zip(r_tuple.items_r, items_v):
+            assert r_item.lowleveltype == v_item.concretetype
+        #
         if len(r_tuple.items_r) == 0:
             return inputconst(Void, ())    # a Void empty tuple
         c1 = inputconst(Void, r_tuple.lowleveltype.TO)
diff --git a/pypy/rpython/rrange.py b/pypy/rpython/rrange.py
--- a/pypy/rpython/rrange.py
+++ b/pypy/rpython/rrange.py
@@ -204,7 +204,10 @@
         v_index = hop.gendirectcall(self.ll_getnextindex, v_enumerate)
         hop2 = hop.copy()
         hop2.args_r = [self.r_baseiter]
+        r_item_src = self.r_baseiter.r_list.external_item_repr
+        r_item_dst = hop.r_result.items_r[1]
         v_item = self.r_baseiter.rtype_next(hop2)
+        v_item = hop.llops.convertvar(v_item, r_item_src, r_item_dst)
         return hop.r_result.newtuple(hop.llops, hop.r_result,
                                      [v_index, v_item])
 
diff --git a/pypy/rpython/test/test_rrange.py b/pypy/rpython/test/test_rrange.py
--- a/pypy/rpython/test/test_rrange.py
+++ b/pypy/rpython/test/test_rrange.py
@@ -169,6 +169,22 @@
         res = self.interpret(fn, [2])
         assert res == 789
 
+    def test_enumerate_instances(self):
+        class A:
+            pass
+        def fn(n):
+            a = A()
+            b = A()
+            a.k = 10
+            b.k = 20
+            for i, x in enumerate([a, b]):
+                if i == n:
+                    return x.k
+            return 5
+        res = self.interpret(fn, [1])
+        assert res == 20
+
+
 
 class TestLLtype(BaseTestRrange, LLRtypeMixin):
     from pypy.rpython.lltypesystem import rrange 

From noreply at buildbot.pypy.org  Wed Jan 18 20:05:02 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Wed, 18 Jan 2012 20:05:02 +0100 (CET)
Subject: [pypy-commit] pypy default: (arigo, romain,
 antocuni) aaargh. enumerate did not convert the repr of the items,
 resulting in a lot of confusion if you enumerate a list of instances,
 leading to incorrect C code. Fix it,
 and add a sanity check inside rtuple.newtuple
Message-ID: <20120118190502.38F42820D8@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni 
Branch: 
Changeset: r51461:a5b5795a3edd
Date: 2012-01-18 20:03 +0100
http://bitbucket.org/pypy/pypy/changeset/a5b5795a3edd/

Log:	(arigo, romain, antocuni) aaargh. enumerate did not convert the repr
	of the items, resulting in a lot of confusion if you enumerate a
	list of instances, leading to incorrect C code. Fix it, and add a
	sanity check inside rtuple.newtuple

diff --git a/pypy/rpython/lltypesystem/rtuple.py b/pypy/rpython/lltypesystem/rtuple.py
--- a/pypy/rpython/lltypesystem/rtuple.py
+++ b/pypy/rpython/lltypesystem/rtuple.py
@@ -27,6 +27,10 @@
 
     def newtuple(cls, llops, r_tuple, items_v):
         # items_v should have the lowleveltype of the internal reprs
+        assert len(r_tuple.items_r) == len(items_v)
+        for r_item, v_item in zip(r_tuple.items_r, items_v):
+            assert r_item.lowleveltype == v_item.concretetype
+        #
         if len(r_tuple.items_r) == 0:
             return inputconst(Void, ())    # a Void empty tuple
         c1 = inputconst(Void, r_tuple.lowleveltype.TO)
diff --git a/pypy/rpython/rrange.py b/pypy/rpython/rrange.py
--- a/pypy/rpython/rrange.py
+++ b/pypy/rpython/rrange.py
@@ -204,7 +204,10 @@
         v_index = hop.gendirectcall(self.ll_getnextindex, v_enumerate)
         hop2 = hop.copy()
         hop2.args_r = [self.r_baseiter]
+        r_item_src = self.r_baseiter.r_list.external_item_repr
+        r_item_dst = hop.r_result.items_r[1]
         v_item = self.r_baseiter.rtype_next(hop2)
+        v_item = hop.llops.convertvar(v_item, r_item_src, r_item_dst)
         return hop.r_result.newtuple(hop.llops, hop.r_result,
                                      [v_index, v_item])
 
diff --git a/pypy/rpython/test/test_rrange.py b/pypy/rpython/test/test_rrange.py
--- a/pypy/rpython/test/test_rrange.py
+++ b/pypy/rpython/test/test_rrange.py
@@ -169,6 +169,22 @@
         res = self.interpret(fn, [2])
         assert res == 789
 
+    def test_enumerate_instances(self):
+        class A:
+            pass
+        def fn(n):
+            a = A()
+            b = A()
+            a.k = 10
+            b.k = 20
+            for i, x in enumerate([a, b]):
+                if i == n:
+                    return x.k
+            return 5
+        res = self.interpret(fn, [1])
+        assert res == 20
+
+
 
 class TestLLtype(BaseTestRrange, LLRtypeMixin):
     from pypy.rpython.lltypesystem import rrange 

From noreply at buildbot.pypy.org  Wed Jan 18 20:05:03 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Wed, 18 Jan 2012 20:05:03 +0100 (CET)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <20120118190503.6FCE8820D8@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni 
Branch: 
Changeset: r51462:c85a96246d2f
Date: 2012-01-18 20:04 +0100
http://bitbucket.org/pypy/pypy/changeset/c85a96246d2f/

Log:	merge heads

diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -20,7 +20,7 @@
 class W_Dtype(Wrappable):
     _immutable_fields_ = ["itemtype", "num", "kind"]
 
-    def __init__(self, itemtype, num, kind, name, char, w_box_type, alternate_constructors=[]):
+    def __init__(self, itemtype, num, kind, name, char, w_box_type, alternate_constructors=[], aliases=[]):
         self.itemtype = itemtype
         self.num = num
         self.kind = kind
@@ -28,6 +28,7 @@
         self.char = char
         self.w_box_type = w_box_type
         self.alternate_constructors = alternate_constructors
+        self.aliases = aliases
 
     def malloc(self, length):
         # XXX find out why test_zjit explodes with tracking of allocations
@@ -62,7 +63,7 @@
         elif space.isinstance_w(w_dtype, space.w_str):
             name = space.str_w(w_dtype)
             for dtype in cache.builtin_dtypes:
-                if dtype.name == name or dtype.char == name:
+                if dtype.name == name or dtype.char == name or name in dtype.aliases:
                     return dtype
         else:
             for dtype in cache.builtin_dtypes:
@@ -107,7 +108,7 @@
             kind=BOOLLTR,
             name="bool",
             char="?",
-            w_box_type = space.gettypefor(interp_boxes.W_BoolBox),
+            w_box_type=space.gettypefor(interp_boxes.W_BoolBox),
             alternate_constructors=[space.w_bool],
         )
         self.w_int8dtype = W_Dtype(
@@ -116,7 +117,7 @@
             kind=SIGNEDLTR,
             name="int8",
             char="b",
-            w_box_type = space.gettypefor(interp_boxes.W_Int8Box)
+            w_box_type=space.gettypefor(interp_boxes.W_Int8Box)
         )
         self.w_uint8dtype = W_Dtype(
             types.UInt8(),
@@ -124,7 +125,7 @@
             kind=UNSIGNEDLTR,
             name="uint8",
             char="B",
-            w_box_type = space.gettypefor(interp_boxes.W_UInt8Box),
+            w_box_type=space.gettypefor(interp_boxes.W_UInt8Box),
         )
         self.w_int16dtype = W_Dtype(
             types.Int16(),
@@ -132,7 +133,7 @@
             kind=SIGNEDLTR,
             name="int16",
             char="h",
-            w_box_type = space.gettypefor(interp_boxes.W_Int16Box),
+            w_box_type=space.gettypefor(interp_boxes.W_Int16Box),
         )
         self.w_uint16dtype = W_Dtype(
             types.UInt16(),
@@ -140,7 +141,7 @@
             kind=UNSIGNEDLTR,
             name="uint16",
             char="H",
-            w_box_type = space.gettypefor(interp_boxes.W_UInt16Box),
+            w_box_type=space.gettypefor(interp_boxes.W_UInt16Box),
         )
         self.w_int32dtype = W_Dtype(
             types.Int32(),
@@ -148,7 +149,7 @@
             kind=SIGNEDLTR,
             name="int32",
             char="i",
-             w_box_type = space.gettypefor(interp_boxes.W_Int32Box),
+            w_box_type=space.gettypefor(interp_boxes.W_Int32Box),
        )
         self.w_uint32dtype = W_Dtype(
             types.UInt32(),
@@ -156,7 +157,7 @@
             kind=UNSIGNEDLTR,
             name="uint32",
             char="I",
-            w_box_type = space.gettypefor(interp_boxes.W_UInt32Box),
+            w_box_type=space.gettypefor(interp_boxes.W_UInt32Box),
         )
         if LONG_BIT == 32:
             name = "int32"
@@ -168,7 +169,7 @@
             kind=SIGNEDLTR,
             name=name,
             char="l",
-            w_box_type = space.gettypefor(interp_boxes.W_LongBox),
+            w_box_type=space.gettypefor(interp_boxes.W_LongBox),
             alternate_constructors=[space.w_int],
         )
         self.w_ulongdtype = W_Dtype(
@@ -177,7 +178,7 @@
             kind=UNSIGNEDLTR,
             name="u" + name,
             char="L",
-            w_box_type = space.gettypefor(interp_boxes.W_ULongBox),
+            w_box_type=space.gettypefor(interp_boxes.W_ULongBox),
         )
         self.w_int64dtype = W_Dtype(
             types.Int64(),
@@ -185,7 +186,7 @@
             kind=SIGNEDLTR,
             name="int64",
             char="q",
-            w_box_type = space.gettypefor(interp_boxes.W_Int64Box),
+            w_box_type=space.gettypefor(interp_boxes.W_Int64Box),
             alternate_constructors=[space.w_long],
         )
         self.w_uint64dtype = W_Dtype(
@@ -194,7 +195,7 @@
             kind=UNSIGNEDLTR,
             name="uint64",
             char="Q",
-            w_box_type = space.gettypefor(interp_boxes.W_UInt64Box),
+            w_box_type=space.gettypefor(interp_boxes.W_UInt64Box),
         )
         self.w_float32dtype = W_Dtype(
             types.Float32(),
@@ -202,7 +203,7 @@
             kind=FLOATINGLTR,
             name="float32",
             char="f",
-            w_box_type = space.gettypefor(interp_boxes.W_Float32Box),
+            w_box_type=space.gettypefor(interp_boxes.W_Float32Box),
         )
         self.w_float64dtype = W_Dtype(
             types.Float64(),
@@ -212,6 +213,7 @@
             char="d",
             w_box_type = space.gettypefor(interp_boxes.W_Float64Box),
             alternate_constructors=[space.w_float],
+            aliases=["float"],
         )
 
         self.builtin_dtypes = [
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -581,7 +581,7 @@
     def descr_var(self, space):
         # var = mean((values - mean(values)) ** 2)
         w_res = self.descr_sub(space, self.descr_mean(space, space.w_None))
-        assert isinstance(w_res, BaseArray) 
+        assert isinstance(w_res, BaseArray)
         w_res = w_res.descr_pow(space, space.wrap(2))
         assert isinstance(w_res, BaseArray)
         return w_res.descr_mean(space, space.w_None)
@@ -590,6 +590,10 @@
         # std(v) = sqrt(var(v))
         return interp_ufuncs.get(space).sqrt.call(space, [self.descr_var(space)])
 
+    def descr_fill(self, space, w_value):
+        concr = self.get_concrete_or_scalar()
+        concr.fill(space, w_value)
+
     def descr_nonzero(self, space):
         if self.size > 1:
             raise OperationError(space.w_ValueError, space.wrap(
@@ -682,6 +686,9 @@
     def copy(self, space):
         return Scalar(self.dtype, self.value)
 
+    def fill(self, space, w_value):
+        self.value = self.dtype.coerce(space, w_value)
+
     def create_sig(self):
         return signature.ScalarSignature(self.dtype)
 
@@ -788,7 +795,7 @@
     Intermediate class for performing binary operations.
     """
     _immutable_fields_ = ['left', 'right']
-    
+
     def __init__(self, ufunc, name, shape, calc_dtype, res_dtype, left, right):
         VirtualArray.__init__(self, name, shape, res_dtype)
         self.ufunc = ufunc
@@ -828,7 +835,7 @@
     def __init__(self, shape, dtype, left, right):
         Call2.__init__(self, None, 'sliceloop', shape, dtype, dtype, left,
                        right)
-    
+
     def create_sig(self):
         lsig = self.left.create_sig()
         rsig = self.right.create_sig()
@@ -847,7 +854,7 @@
     when we'll make AxisReduce lazy
     """
     _immutable_fields_ = ['left', 'right']
-    
+
     def __init__(self, ufunc, name, shape, dtype, left, right, dim):
         Call2.__init__(self, ufunc, name, shape, dtype, dtype,
                        left, right)
@@ -1061,6 +1068,9 @@
         array.setslice(space, self)
         return array
 
+    def fill(self, space, w_value):
+        self.setslice(space, scalar_w(space, self.dtype, w_value))
+
 
 class ViewArray(ConcreteArray):
     def create_sig(self):
@@ -1273,6 +1283,8 @@
     var = interp2app(BaseArray.descr_var),
     std = interp2app(BaseArray.descr_std),
 
+    fill = interp2app(BaseArray.descr_fill),
+
     copy = interp2app(BaseArray.descr_copy),
     reshape = interp2app(BaseArray.descr_reshape),
     tolist = interp2app(BaseArray.descr_tolist),
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -166,14 +166,11 @@
         # You can't subclass dtype
         raises(TypeError, type, "Foo", (dtype,), {})
 
-    def test_new(self):
-        import _numpypy as np
-        assert np.int_(4) == 4
-        assert np.float_(3.4) == 3.4
+    def test_aliases(self):
+        from _numpypy import dtype
 
-    def test_pow(self):
-        from _numpypy import int_
-        assert int_(4) ** 2 == 16
+        assert dtype("float") is dtype(float)
+
 
 class AppTestTypes(BaseNumpyAppTest):
     def test_abstract_types(self):
@@ -189,6 +186,15 @@
         raises(TypeError, numpy.floating, 0)
         raises(TypeError, numpy.inexact, 0)
 
+    def test_new(self):
+        import _numpypy as np
+        assert np.int_(4) == 4
+        assert np.float_(3.4) == 3.4
+
+    def test_pow(self):
+        from _numpypy import int_
+        assert int_(4) ** 2 == 16
+
     def test_bool(self):
         import _numpypy as numpy
 
@@ -318,7 +324,7 @@
         else:
             raises(OverflowError, numpy.int64, 9223372036854775807)
             raises(OverflowError, numpy.int64, '9223372036854775807')
-        
+
         raises(OverflowError, numpy.int64, 9223372036854775808)
         raises(OverflowError, numpy.int64, '9223372036854775808')
 
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1302,6 +1302,28 @@
         assert isinstance(i['data'][0], int)
         raises(TypeError, getattr, array(3), '__array_interface__')
 
+    def test_fill(self):
+        from _numpypy import array
+
+        a = array([1, 2, 3])
+        a.fill(10)
+        assert (a == [10, 10, 10]).all()
+        a.fill(False)
+        assert (a == [0, 0, 0]).all()
+
+        b = a[:1]
+        b.fill(4)
+        assert (b == [4]).all()
+        assert (a == [4, 0, 0]).all()
+
+        c = b + b
+        c.fill(27)
+        assert (c == [27]).all()
+
+        d = array(10)
+        d.fill(100)
+        assert d == 100
+
 
 class AppTestSupport(BaseNumpyAppTest):
     def setup_class(cls):
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -349,7 +349,8 @@
         self.check_trace_count(1)
         self.check_simple_loop({'getinteriorfield_raw': 2, 'float_add': 1,
                                 'setinteriorfield_raw': 1, 'int_add': 2,
-                                'int_eq': 1, 'guard_false': 1, 'jump': 1})
+                                'int_eq': 1, 'guard_false': 1, 'jump': 1,
+                                'arraylen_gc': 1})
 
     def define_virtual_slice():
         return """

From pullrequests-noreply at bitbucket.org  Wed Jan 18 21:40:07 2012
From: pullrequests-noreply at bitbucket.org (Bitbucket)
Date: Wed, 18 Jan 2012 20:40:07 -0000
Subject: [pypy-commit] [REJECTED] Pull request #20 for pypy/pypy:
 Improvements to the JVM backend, this time with tests
In-Reply-To: 
References: 
Message-ID: <20120118204007.24052.8366@bitbucket01.managed.contegix.com>

Pull request #20 has been rejected by Antonio Cuni.

benol/pypy had changes to be pulled into pypy/pypy.

https://bitbucket.org/pypy/pypy/pull-request/20/improvements-to-the-jvm-backend-this-time

manually merged

Rejected changes:

8efdb1515b2c by Micha? Bendowski: "Handle the 'jit_is_virtual' opcode by always returning False"
f1d52bd62e15 by Micha? Bendowski: "Add a missing cast from Unsigned to UnsignedLongLong in the JVM
backend."
8ff039cadd19 by Micha? Bendowski: "Declare oo_primitives that should implement some rffi operations.

For now the a?"
25d4d323cb5f by Micha? Bendowski: "Fix compute_unique_id to support built-ins in ootype.

Otherwise the translation?"
7f09531dd6a9 by Micha? Bendowski: "Fix userspace builders in ootype

Implement the getlength() method of StringBuil?"
578e69f273f9 by Micha? Bendowski: "Add files generated by PyCharm to .hgignore"


The pull request has been closed.

--
This is an issue notification from bitbucket.org.
You are receiving this either because you are the participating
in a pull request, or you are following it.


From pullrequests-noreply at bitbucket.org  Wed Jan 18 21:42:39 2012
From: pullrequests-noreply at bitbucket.org (Bitbucket)
Date: Wed, 18 Jan 2012 20:42:39 -0000
Subject: [pypy-commit] [REJECTED] Pull request #19 for pypy/pypy:
	Improvements to the JVM backend
In-Reply-To: 
References: 
Message-ID: <20120118204239.8312.39133@bitbucket01.managed.contegix.com>

Pull request #19 has been rejected by Antonio Cuni.

benol/pypy had changes to be pulled into pypy/pypy.

https://bitbucket.org/pypy/pypy/pull-request/19/improvements-to-the-jvm-backend

manually merged

Rejected changes:

2e1b33862b2c by Micha? Bendowski: "Add files generated by PyCharm to .hgignore"
372628723c37 by Micha? Bendowski: "Declare oo_primitives that should implement some rffi operations.

For now the a?"
30e6d59d4333 by Micha? Bendowski: "Add a missing cast from Unsigned to UnsignedLongLong."
0ab09a670d4a by Micha? Bendowski: "Handle the 'jit_is_virtual' opcode by always returning False"
9031e9c4ad78 by Micha? Bendowski: "Fix compute_unique_id to support built-ins.

Otherwise the translation fails bec?"
ee14653b5fa0 by Micha? Bendowski: "Fix userspace builders in ootype

Implement the ll_getlength() method of StringB?"


The pull request has been closed.

--
This is an issue notification from bitbucket.org.
You are receiving this either because you are the participating
in a pull request, or you are following it.


From noreply at buildbot.pypy.org  Wed Jan 18 21:57:43 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Wed, 18 Jan 2012 21:57:43 +0100 (CET)
Subject: [pypy-commit] pypy numpypy-reshape: add (still not enough) failing
	tests
Message-ID: <20120118205743.179DE820D8@wyvern.cs.uni-duesseldorf.de>

Author: mattip
Branch: numpypy-reshape
Changeset: r51463:95362a60e6b8
Date: 2012-01-18 22:57 +0200
http://bitbucket.org/pypy/pypy/changeset/95362a60e6b8/

Log:	add (still not enough) failing tests

diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -396,6 +396,11 @@
         assert (a == [1000, 1, 2, 3, 1000, 5, 6, 7, 1000, 9, 10, 11]).all()
         a = zeros((4, 2, 3))
         a.shape = (12, 2)
+        assert array(1).reshape((1, 1, 1, 1)).shape == (1, 1, 1, 1)
+        assert array([2, 2, 2]).reshape((1, 1, 1, -1)).shape == (1, 1, 1, 3)
+        a=numpypy.array(1)
+        a.shape = (1, 1, 1)
+        assert a.shape == (1, 1, 1)
 
     def test_slice_reshape(self):
         from _numpypy import zeros, arange

From noreply at buildbot.pypy.org  Wed Jan 18 23:43:34 2012
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Wed, 18 Jan 2012 23:43:34 +0100 (CET)
Subject: [pypy-commit] pypy default: Added ztranslation tests for _codecs
Message-ID: <20120118224334.7BEE5820D8@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor 
Branch: 
Changeset: r51464:0b188f082267
Date: 2012-01-18 15:42 -0600
http://bitbucket.org/pypy/pypy/changeset/0b188f082267/

Log:	Added ztranslation tests for _codecs

diff --git a/pypy/module/_codecs/test/test_ztranslation.py b/pypy/module/_codecs/test/test_ztranslation.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_codecs/test/test_ztranslation.py
@@ -0,0 +1,5 @@
+from pypy.objspace.fake.checkmodule import checkmodule
+
+
+def test__codecs_translates():
+    checkmodule('_codecs')

From noreply at buildbot.pypy.org  Wed Jan 18 23:43:35 2012
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Wed, 18 Jan 2012 23:43:35 +0100 (CET)
Subject: [pypy-commit] pypy numpypy-remove-scalar: Remove scalar. It needs
 some new support in signatures though.
Message-ID: <20120118224335.B919982110@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor 
Branch: numpypy-remove-scalar
Changeset: r51465:6cb068520d24
Date: 2012-01-18 16:40 -0600
http://bitbucket.org/pypy/pypy/changeset/6cb068520d24/

Log:	Remove scalar. It needs some new support in signatures though.

diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -6,11 +6,10 @@
 import re
 
 from pypy.interpreter.baseobjspace import InternalSpaceCache, W_Root
-from pypy.module.micronumpy import interp_boxes
+from pypy.module.micronumpy import interp_boxes, interp_ufuncs
 from pypy.module.micronumpy.interp_dtype import get_dtype_cache
-from pypy.module.micronumpy.interp_numarray import (Scalar, BaseArray,
-     scalar_w, W_NDimArray, array)
-from pypy.module.micronumpy import interp_ufuncs
+from pypy.module.micronumpy.interp_numarray import (BaseArray, W_NDimArray,
+    convert_to_array, array)
 from pypy.rlib.objectmodel import specialize, instantiate
 
 
@@ -54,10 +53,10 @@
         self.fromcache = InternalSpaceCache(self).getorbuild
 
     def issequence_w(self, w_obj):
-        return isinstance(w_obj, ListObject) or isinstance(w_obj, W_NDimArray)
+        return isinstance(w_obj, ListObject) or isinstance(w_obj, BaseArray)
 
     def isinstance_w(self, w_obj, w_tp):
-        return w_obj.tp == w_tp
+        return not isinstance(w_obj, BaseArray) and w_obj.tp == w_tp
 
     def decode_index4(self, w_idx, size):
         if isinstance(w_idx, IntObject):
@@ -260,8 +259,7 @@
             w_rhs = self.rhs.execute(interp)
         if not isinstance(w_lhs, BaseArray):
             # scalar
-            dtype = get_dtype_cache(interp.space).w_float64dtype
-            w_lhs = scalar_w(interp.space, dtype, w_lhs)
+            w_lhs = convert_to_array(interp.space, w_lhs)
         assert isinstance(w_lhs, BaseArray)
         if self.name == '+':
             w_res = w_lhs.descr_add(interp.space, w_rhs)
@@ -270,7 +268,6 @@
         elif self.name == '-':
             w_res = w_lhs.descr_sub(interp.space, w_rhs)
         elif self.name == '->':
-            assert not isinstance(w_rhs, Scalar)
             if isinstance(w_rhs, FloatObject):
                 w_rhs = IntObject(int(w_rhs.floatval))
             assert isinstance(w_lhs, BaseArray)
@@ -280,7 +277,7 @@
         if (not isinstance(w_res, BaseArray) and
             not isinstance(w_res, interp_boxes.W_GenericBox)):
             dtype = get_dtype_cache(interp.space).w_float64dtype
-            w_res = scalar_w(interp.space, dtype, w_res)
+            w_res = convert_to_array(interp.space, w_res, dtype=dtype)
         return w_res
 
     def __repr__(self):
@@ -398,17 +395,7 @@
                 w_res = neg.call(interp.space, [arr])
             else:
                 assert False # unreachable code
-            if isinstance(w_res, BaseArray):
-                return w_res
-            if isinstance(w_res, FloatObject):
-                dtype = get_dtype_cache(interp.space).w_float64dtype
-            elif isinstance(w_res, BoolObject):
-                dtype = get_dtype_cache(interp.space).w_booldtype
-            elif isinstance(w_res, interp_boxes.W_GenericBox):
-                dtype = w_res.get_dtype(interp.space)
-            else:
-                dtype = None
-            return scalar_w(interp.space, dtype, w_res)
+            return w_res
         else:
             raise WrongFunctionName
 
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -1,15 +1,16 @@
 from pypy.interpreter.baseobjspace import Wrappable
 from pypy.interpreter.error import OperationError, operationerrfmt
-from pypy.interpreter.gateway import interp2app, NoneNotWrapped
+from pypy.interpreter.gateway import interp2app, NoneNotWrapped, unwrap_spec
 from pypy.interpreter.typedef import TypeDef, GetSetProperty
 from pypy.module.micronumpy import interp_ufuncs, interp_dtype, signature
+from pypy.module.micronumpy.interp_iter import (ArrayIterator, OneDimIterator,
+     SkipLastAxisIterator)
 from pypy.module.micronumpy.strides import calculate_slice_strides
 from pypy.rlib import jit
+from pypy.rlib.rstring import StringBuilder
 from pypy.rpython.lltypesystem import lltype, rffi
 from pypy.tool.sourcetools import func_with_new_name
-from pypy.rlib.rstring import StringBuilder
-from pypy.module.micronumpy.interp_iter import ArrayIterator, OneDimIterator,\
-     SkipLastAxisIterator
+
 
 numpy_driver = jit.JitDriver(
     greens=['shapelen', 'sig'],
@@ -272,10 +273,7 @@
 
     def _binop_right_impl(ufunc_name):
         def impl(self, space, w_other):
-            w_other = scalar_w(space,
-                interp_ufuncs.find_dtype_for_scalar(space, w_other, self.find_dtype()),
-                w_other
-            )
+            w_other = convert_to_array(space, w_other, dtype=self.find_dtype())
             return getattr(interp_ufuncs.get(space), ufunc_name).call(space, [w_other, self])
         return func_with_new_name(impl, "binop_right_%s_impl" % ufunc_name)
 
@@ -375,8 +373,7 @@
     descr_argmin = _reduce_argmax_argmin_impl("min")
 
     def descr_dot(self, space, w_other):
-        w_other = convert_to_array(space, w_other)
-        if isinstance(w_other, Scalar):
+        if is_scalar(space, w_other):
             return self.descr_mul(space, w_other)
         else:
             w_res = self.descr_mul(space, w_other)
@@ -399,8 +396,6 @@
     def descr_set_shape(self, space, w_iterable):
         new_shape = get_shape_from_iterable(space,
                             self.size, w_iterable)
-        if isinstance(self, Scalar):
-            return
         self.get_concrete().setshape(space, new_shape)
 
     def descr_get_size(self, space):
@@ -560,8 +555,7 @@
 
     def descr_tolist(self, space):
         if len(self.shape) == 0:
-            assert isinstance(self, Scalar)
-            return self.value.descr_tolist(space)
+            return self.getitem(0).descr_tolist(space)
         w_result = space.newlist([])
         for i in range(self.shape[0]):
             space.call_method(w_result, "append",
@@ -650,50 +644,23 @@
     def supports_fast_slicing(self):
         return False
 
-def convert_to_array(space, w_obj):
+def convert_to_array(space, w_obj, dtype=None):
     if isinstance(w_obj, BaseArray):
+        assert dtype is None
         return w_obj
     elif space.issequence_w(w_obj):
         # Convert to array.
-        return array(space, w_obj, w_order=None)
+        return array(space, w_obj, w_order=None, w_dtype=dtype)
     else:
         # If it's a scalar
-        dtype = interp_ufuncs.find_dtype_for_scalar(space, w_obj)
-        return scalar_w(space, dtype, w_obj)
+        if dtype is None:
+            dtype = interp_ufuncs.find_dtype_for_scalar(space, w_obj)
+        arr = W_NDimArray(1, [], dtype)
+        arr.setitem(0, dtype.coerce(space, w_obj))
+        return arr
 
-def scalar_w(space, dtype, w_obj):
-    return Scalar(dtype, dtype.coerce(space, w_obj))
-
-class Scalar(BaseArray):
-    """
-    Intermediate class representing a literal.
-    """
-    size = 1
-    _attrs_ = ["dtype", "value", "shape"]
-
-    def __init__(self, dtype, value):
-        self.shape = []
-        BaseArray.__init__(self, [])
-        self.dtype = dtype
-        self.value = value
-
-    def find_dtype(self):
-        return self.dtype
-
-    def to_str(self, space, comma, builder, indent=' ', use_ellipsis=False):
-        builder.append(self.dtype.itemtype.str_format(self.value))
-
-    def copy(self, space):
-        return Scalar(self.dtype, self.value)
-
-    def fill(self, space, w_value):
-        self.value = self.dtype.coerce(space, w_value)
-
-    def create_sig(self):
-        return signature.ScalarSignature(self.dtype)
-
-    def get_concrete_or_scalar(self):
-        return self
+def is_scalar(space, w_obj):
+    return not isinstance(w_obj, BaseArray) and not space.issequence_w(w_obj)
 
 
 class VirtualArray(BaseArray):
@@ -884,7 +851,7 @@
         return self
 
     def supports_fast_slicing(self):
-        return self.order == 'C' and self.strides[-1] == 1
+        return self.order == 'C' and bool(self.strides) and self.strides[-1] == 1
 
     def find_dtype(self):
         return self.dtype
@@ -1069,7 +1036,7 @@
         return array
 
     def fill(self, space, w_value):
-        self.setslice(space, scalar_w(space, self.dtype, w_value))
+        self.setslice(space, convert_to_array(space, w_value))
 
 
 class ViewArray(ConcreteArray):
@@ -1129,9 +1096,9 @@
     """ A class representing contiguous array. We know that each iteration
     by say ufunc will increase the data index by one
     """
-    def setitem(self, item, value):
+    def setitem(self, idx, value):
         self.invalidated()
-        self.dtype.setitem(self.storage, item, value)
+        self.dtype.setitem(self.storage, idx, value)
 
     def setshape(self, space, new_shape):
         self.shape = new_shape
@@ -1165,7 +1132,7 @@
         dtype = space.interp_w(interp_dtype.W_Dtype,
             space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype)
         )
-        return scalar_w(space, dtype, w_item_or_iterable)
+        return convert_to_array(space, w_item_or_iterable, dtype=dtype)
     if w_order is None:
         order = 'C'
     else:
@@ -1217,9 +1184,9 @@
     return space.wrap(arr)
 
 def dot(space, w_obj, w_obj2):
+    if is_scalar(space, w_obj):
+        return convert_to_array(space, w_obj2).descr_dot(space, w_obj)
     w_arr = convert_to_array(space, w_obj)
-    if isinstance(w_arr, Scalar):
-        return convert_to_array(space, w_obj2).descr_dot(space, w_arr)
     return w_arr.descr_dot(space, w_obj2)
 
 BaseArray.typedef = TypeDef(
@@ -1307,6 +1274,9 @@
         self.iter = OneDimIterator(arr.start, self.strides[0],
                                    self.shape[0])
 
+    def descr_iter(self):
+        return self
+
     def descr_next(self, space):
         if self.iter.done():
             raise OperationError(space.w_StopIteration, space.w_None)
@@ -1314,12 +1284,10 @@
         self.iter = self.iter.next(self.shapelen)
         return result
 
-    def descr_iter(self):
-        return self
 
 W_FlatIterator.typedef = TypeDef(
     'flatiter',
+    __iter__ = interp2app(W_FlatIterator.descr_iter),
     next = interp2app(W_FlatIterator.descr_next),
-    __iter__ = interp2app(W_FlatIterator.descr_iter),
 )
 W_FlatIterator.acceptable_as_base_class = False
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -3,12 +3,13 @@
 from pypy.interpreter.gateway import interp2app
 from pypy.interpreter.typedef import TypeDef, GetSetProperty, interp_attrproperty
 from pypy.module.micronumpy import interp_boxes, interp_dtype
-from pypy.module.micronumpy.signature import ReduceSignature,\
-     find_sig, new_printable_location, AxisReduceSignature, ScalarSignature
+from pypy.module.micronumpy.signature import (ReduceSignature,
+    AxisReduceSignature, ScalarSignature, new_printable_location, find_sig)
 from pypy.rlib import jit
 from pypy.rlib.rarithmetic import LONG_BIT
 from pypy.tool.sourcetools import func_with_new_name
 
+
 reduce_driver = jit.JitDriver(
     greens=['shapelen', "sig"],
     virtualizables=["frame"],
@@ -114,8 +115,8 @@
         return self.reduce(space, w_obj, False, False, w_dim)
 
     def reduce(self, space, w_obj, multidim, promote_to_largest, w_dim):
-        from pypy.module.micronumpy.interp_numarray import convert_to_array, \
-                                                           Scalar
+        from pypy.module.micronumpy.interp_numarray import convert_to_array
+
         if self.argcount != 2:
             raise OperationError(space.w_ValueError, space.wrap("reduce only "
                 "supported for binary functions"))
@@ -124,7 +125,7 @@
         obj = convert_to_array(space, w_obj)
         if dim >= len(obj.shape):
             raise OperationError(space.w_ValueError, space.wrap("axis(=%d) out of bounds" % dim))
-        if isinstance(obj, Scalar):
+        if not obj.shape:
             raise OperationError(space.w_TypeError, space.wrap("cannot reduce "
                 "on a scalar"))
 
@@ -155,9 +156,9 @@
         return self.reduce_loop(shapelen, sig, frame, value, obj, dtype)
 
     def do_axis_reduce(self, obj, dtype, dim):
-        from pypy.module.micronumpy.interp_numarray import AxisReduce,\
-             W_NDimArray
-        
+        from pypy.module.micronumpy.interp_numarray import (AxisReduce,
+             W_NDimArray)
+
         shape = obj.shape[0:dim] + obj.shape[dim + 1:len(obj.shape)]
         size = 1
         for s in shape:
@@ -231,17 +232,22 @@
 
     def call(self, space, args_w):
         from pypy.module.micronumpy.interp_numarray import (Call1,
-            convert_to_array, Scalar)
+            convert_to_array, is_scalar)
 
         [w_obj] = args_w
-        w_obj = convert_to_array(space, w_obj)
+        scalar = is_scalar(space, w_obj)
+        if scalar:
+            w_obj_dtype = find_dtype_for_scalar(space, w_obj)
+        else:
+            w_obj = convert_to_array(space, w_obj)
+            w_obj_dtype = w_obj.find_dtype()
         res_dtype = find_unaryop_result_dtype(space,
-            w_obj.find_dtype(),
+            w_obj_dtype,
             promote_to_float=self.promote_to_float,
             promote_bools=self.promote_bools,
         )
-        if isinstance(w_obj, Scalar):
-            return self.func(res_dtype, w_obj.value.convert_to(res_dtype))
+        if scalar:
+            return self.func(res_dtype, res_dtype.coerce(space, w_obj))
 
         w_res = Call1(self.func, self.name, w_obj.shape, res_dtype, w_obj)
         w_obj.add_invalidates(w_res)
@@ -260,14 +266,22 @@
         self.comparison_func = comparison_func
 
     def call(self, space, args_w):
-        from pypy.module.micronumpy.interp_numarray import (Call2,
-            convert_to_array, Scalar, shape_agreement)
+        from pypy.module.micronumpy.interp_numarray import (BaseArray, Call2,
+            convert_to_array, is_scalar, shape_agreement)
 
         [w_lhs, w_rhs] = args_w
-        w_lhs = convert_to_array(space, w_lhs)
-        w_rhs = convert_to_array(space, w_rhs)
+        scalar = is_scalar(space, w_lhs) and is_scalar(space, w_rhs)
+        if scalar:
+            w_lhs_dtype = find_dtype_for_scalar(space, w_lhs)
+            w_rhs_dtype = find_dtype_for_scalar(space, w_rhs)
+        else:
+            w_lhs = convert_to_array(space, w_lhs)
+            w_rhs = convert_to_array(space, w_rhs)
+            w_lhs_dtype = w_lhs.find_dtype()
+            w_rhs_dtype = w_rhs.find_dtype()
+
         calc_dtype = find_binop_result_dtype(space,
-            w_lhs.find_dtype(), w_rhs.find_dtype(),
+            w_lhs_dtype, w_rhs_dtype,
             promote_to_float=self.promote_to_float,
             promote_bools=self.promote_bools,
         )
@@ -275,12 +289,14 @@
             res_dtype = interp_dtype.get_dtype_cache(space).w_booldtype
         else:
             res_dtype = calc_dtype
-        if isinstance(w_lhs, Scalar) and isinstance(w_rhs, Scalar):
+        if scalar:
             return self.func(calc_dtype,
-                w_lhs.value.convert_to(calc_dtype),
-                w_rhs.value.convert_to(calc_dtype)
+                calc_dtype.coerce(space, w_lhs),
+                calc_dtype.coerce(space, w_rhs),
             )
 
+        assert isinstance(w_lhs, BaseArray)
+        assert isinstance(w_rhs, BaseArray)
         new_shape = shape_agreement(space, w_lhs.shape, w_rhs.shape)
         w_res = Call2(self.func, self.name,
                       new_shape, calc_dtype,
diff --git a/pypy/module/micronumpy/signature.py b/pypy/module/micronumpy/signature.py
--- a/pypy/module/micronumpy/signature.py
+++ b/pypy/module/micronumpy/signature.py
@@ -195,11 +195,6 @@
             iter = ConstantIterator()
             iterlist.append(iter)
 
-    def eval(self, frame, arr):
-        from pypy.module.micronumpy.interp_numarray import Scalar
-        assert isinstance(arr, Scalar)
-        return arr.value
-
 class ViewSignature(ArraySignature):
     def debug_repr(self):
         return 'Slice'
@@ -331,7 +326,7 @@
         assert isinstance(arr, Call2)
         lhs = self.left.eval(frame, arr.left).convert_to(self.calc_dtype)
         rhs = self.right.eval(frame, arr.right).convert_to(self.calc_dtype)
-        
+
         return self.binfunc(self.calc_dtype, lhs, rhs)
 
     def debug_repr(self):
@@ -342,7 +337,7 @@
     def _invent_numbering(self, cache, allnumbers):
         self.left._invent_numbering(new_cache(), allnumbers)
         self.right._invent_numbering(cache, allnumbers)
-    
+
     def _create_iter(self, iterlist, arraylist, arr, transforms):
         from pypy.module.micronumpy.interp_numarray import Call2
 
@@ -397,12 +392,12 @@
 class SliceloopSignature(Call2):
     def eval(self, frame, arr):
         from pypy.module.micronumpy.interp_numarray import Call2
-        
+
         assert isinstance(arr, Call2)
         ofs = frame.iterators[0].offset
         arr.left.setitem(ofs, self.right.eval(frame, arr.right).convert_to(
             self.calc_dtype))
-    
+
     def debug_repr(self):
         return 'SliceLoop(%s, %s, %s)' % (self.name, self.left.debug_repr(),
                                           self.right.debug_repr())
@@ -447,6 +442,6 @@
 
         assert isinstance(arr, AxisReduce)
         return self.right.eval(frame, arr.right).convert_to(self.calc_dtype)
-    
+
     def debug_repr(self):
         return 'AxisReduceSig(%s, %s)' % (self.name, self.right.debug_repr())
diff --git a/pypy/module/micronumpy/test/test_base.py b/pypy/module/micronumpy/test/test_base.py
--- a/pypy/module/micronumpy/test/test_base.py
+++ b/pypy/module/micronumpy/test/test_base.py
@@ -1,6 +1,6 @@
 from pypy.conftest import gettestobjspace
 from pypy.module.micronumpy.interp_dtype import get_dtype_cache
-from pypy.module.micronumpy.interp_numarray import W_NDimArray, Scalar
+from pypy.module.micronumpy.interp_numarray import W_NDimArray
 from pypy.module.micronumpy.interp_ufuncs import (find_binop_result_dtype,
         find_unaryop_result_dtype)
 
@@ -16,7 +16,7 @@
         ar = W_NDimArray(10, [10], dtype=float64_dtype)
         ar2 = W_NDimArray(10, [10], dtype=float64_dtype)
         v1 = ar.descr_add(space, ar)
-        v2 = ar.descr_add(space, Scalar(float64_dtype, 2.0))
+        v2 = ar.descr_add(space, space.wrap(2.0))
         sig1 = v1.find_sig()
         sig2 = v2.find_sig()
         assert v1 is not v2
@@ -26,7 +26,7 @@
         sig1b = ar2.descr_add(space, ar).find_sig()
         assert sig1b.left.array_no != sig1b.right.array_no
         assert sig1b is not sig1
-        v3 = ar.descr_add(space, Scalar(float64_dtype, 1.0))
+        v3 = ar.descr_add(space, space.wrap(1.0))
         sig3 = v3.find_sig()
         assert sig2 is sig3
         v4 = ar.descr_add(space, ar)
diff --git a/pypy/module/micronumpy/test/test_compile.py b/pypy/module/micronumpy/test/test_compile.py
--- a/pypy/module/micronumpy/test/test_compile.py
+++ b/pypy/module/micronumpy/test/test_compile.py
@@ -135,7 +135,7 @@
         r
         """
         interp = self.run(code)
-        assert interp.results[0].value.value == 15
+        assert interp.results[0].value == 15
 
     def test_sum2(self):
         code = """
@@ -144,8 +144,7 @@
         sum(b)
         """
         interp = self.run(code)
-        assert interp.results[0].value.value == 30 * (30 - 1)
-
+        assert interp.results[0].value == 30 * (30 - 1)
 
     def test_array_write(self):
         code = """
@@ -163,7 +162,7 @@
         b = a + a
         min(b)
         """)
-        assert interp.results[0].value.value == -24
+        assert interp.results[0].value == -24
 
     def test_max(self):
         interp = self.run("""
@@ -172,7 +171,7 @@
         b = a + a
         max(b)
         """)
-        assert interp.results[0].value.value == 256
+        assert interp.results[0].value == 256
 
     def test_slice(self):
         interp = self.run("""
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -945,7 +945,7 @@
         assert debug_repr(a) == 'Array'
         assert debug_repr(a + a) == 'Call2(add, Array, Array)'
         assert debug_repr(a[::2]) == 'Slice'
-        assert debug_repr(a + 2) == 'Call2(add, Array, Scalar)'
+        assert debug_repr(a + 2) == 'Call2(add, Array, Array)'
         assert debug_repr(a + a.flat) == 'Call2(add, Array, Slice)'
         assert debug_repr(sin(a)) == 'Call1(sin, Array)'
 

From noreply at buildbot.pypy.org  Wed Jan 18 23:43:36 2012
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Wed, 18 Jan 2012 23:43:36 +0100 (CET)
Subject: [pypy-commit] pypy default: merged default
Message-ID: <20120118224336.E8E67820D8@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor 
Branch: 
Changeset: r51466:f254dc780358
Date: 2012-01-18 16:43 -0600
http://bitbucket.org/pypy/pypy/changeset/f254dc780358/

Log:	merged default

diff --git a/pypy/module/_codecs/test/test_ztranslation.py b/pypy/module/_codecs/test/test_ztranslation.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_codecs/test/test_ztranslation.py
@@ -0,0 +1,5 @@
+from pypy.objspace.fake.checkmodule import checkmodule
+
+
+def test__codecs_translates():
+    checkmodule('_codecs')

From noreply at buildbot.pypy.org  Thu Jan 19 00:11:38 2012
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu, 19 Jan 2012 00:11:38 +0100 (CET)
Subject: [pypy-commit] pypy py3k: Test and fix RLock context manager.
Message-ID: <20120118231138.C1A15820D8@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: py3k
Changeset: r51467:fc8fcd7a4f56
Date: 2012-01-17 21:21 +0100
http://bitbucket.org/pypy/pypy/changeset/fc8fcd7a4f56/

Log:	Test and fix RLock context manager.

diff --git a/pypy/module/thread/os_lock.py b/pypy/module/thread/os_lock.py
--- a/pypy/module/thread/os_lock.py
+++ b/pypy/module/thread/os_lock.py
@@ -234,7 +234,7 @@
         self.acquire_w(space)
         return self
 
-    def descr__exit__(self, space, *args):
+    def descr__exit__(self, space, __args__):
         self.release_w(space)
 
 W_RLock.typedef = TypeDef(
diff --git a/pypy/module/thread/test/test_lock.py b/pypy/module/thread/test/test_lock.py
--- a/pypy/module/thread/test/test_lock.py
+++ b/pypy/module/thread/test/test_lock.py
@@ -122,3 +122,9 @@
         lock.release()
         assert lock._is_owned() is False
 
+    def test_context_manager(self):
+        import _thread
+        lock = _thread.RLock()
+        with lock:
+            assert lock._is_owned() is True
+

From noreply at buildbot.pypy.org  Thu Jan 19 00:11:40 2012
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu, 19 Jan 2012 00:11:40 +0100 (CET)
Subject: [pypy-commit] pypy py3k: Fix octal notation in binascii.py
Message-ID: <20120118231140.0511D820D8@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: py3k
Changeset: r51468:a218f270838c
Date: 2012-01-17 22:18 +0100
http://bitbucket.org/pypy/pypy/changeset/a218f270838c/

Log:	Fix octal notation in binascii.py

diff --git a/lib_pypy/binascii.py b/lib_pypy/binascii.py
--- a/lib_pypy/binascii.py
+++ b/lib_pypy/binascii.py
@@ -68,7 +68,7 @@
          chr(0x20 + (((B << 2) | ((C >> 6) & 0x3)) & 0x3F)),
          chr(0x20 + (( C                         ) & 0x3F))])
               for A, B, C in triples_gen(s)]
-    return chr(ord(' ') + (length & 077)) + ''.join(result) + '\n'
+    return chr(ord(' ') + (length & 0o77)) + ''.join(result) + '\n'
 
 
 table_a2b_base64 = {

From noreply at buildbot.pypy.org  Thu Jan 19 00:11:41 2012
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu, 19 Jan 2012 00:11:41 +0100 (CET)
Subject: [pypy-commit] pypy py3k: Fixes for the struct module: the builtin
 part is now named _struct.
Message-ID: <20120118231141.4322B820D8@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: py3k
Changeset: r51469:8b47ef4c3162
Date: 2012-01-18 22:09 +0100
http://bitbucket.org/pypy/pypy/changeset/8b47ef4c3162/

Log:	Fixes for the struct module: the builtin part is now named _struct.

diff --git a/lib_pypy/struct.py b/lib_pypy/_struct.py
rename from lib_pypy/struct.py
rename to lib_pypy/_struct.py
--- a/lib_pypy/struct.py
+++ b/lib_pypy/_struct.py
@@ -51,7 +51,7 @@
     bytes = [b for b in data[index:index+size]]
     if le == 'little':
         bytes.reverse()
-    number = 0L
+    number = 0
     for b in bytes:
         number = number << 8 | b
     return int(number)
@@ -415,3 +415,7 @@
         raise error("unpack_from requires a buffer of at least %d bytes"
                     % (size,))
     return unpack(fmt, data)
+
+def _clearcache():
+    "Clear the internal cache."
+    # No cache in this implementation
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1280,11 +1280,14 @@
         return self.str_w(w_obj)
 
     def str_w(self, w_obj):
-        try:
-            return self.unicode_w(w_obj).encode('ascii')
-        except UnicodeEncodeError:
-            w_bytes = self.call_method(w_obj, 'encode', self.wrap('utf-8'))
-            return self.bytes_w(w_bytes)
+        if self.isinstance_w(w_obj, self.w_unicode):
+            try:
+                return self.unicode_w(w_obj).encode('ascii')
+            except UnicodeEncodeError:
+                w_bytes = self.call_method(w_obj, 'encode', self.wrap('utf-8'))
+                return self.bytes_w(w_bytes)
+        else:
+            return w_obj.bytes_w(self)
 
     def bytes_w(self, w_obj):
         return w_obj.bytes_w(self)
diff --git a/pypy/module/struct/__init__.py b/pypy/module/struct/__init__.py
--- a/pypy/module/struct/__init__.py
+++ b/pypy/module/struct/__init__.py
@@ -45,10 +45,13 @@
 
 The variable struct.error is an exception raised on errors."""
 
+    applevel_name = '_struct'
+
     interpleveldefs = {
         'calcsize': 'interp_struct.calcsize',
         'pack': 'interp_struct.pack',
         'unpack': 'interp_struct.unpack',
+        '_clearcache': 'interp_struct.clearcache',
         }
 
     appleveldefs = {
diff --git a/pypy/module/struct/app_struct.py b/pypy/module/struct/app_struct.py
--- a/pypy/module/struct/app_struct.py
+++ b/pypy/module/struct/app_struct.py
@@ -2,7 +2,7 @@
 """
 Application-level definitions for the struct module.
 """
-import struct
+import _struct as struct
 
 class error(Exception):
     """Exception raised on various occasions; argument is a string
@@ -15,7 +15,7 @@
 
 # XXX inefficient
 def unpack_from(fmt, buf, offset=0):
-    size = struct.calcsize(fmt)
+    size = _struct.calcsize(fmt)
     data = buffer(buf)[offset:offset+size]
     if len(data) != size:
         raise error("unpack_from requires a buffer of at least %d bytes"
diff --git a/pypy/module/struct/interp_struct.py b/pypy/module/struct/interp_struct.py
--- a/pypy/module/struct/interp_struct.py
+++ b/pypy/module/struct/interp_struct.py
@@ -33,3 +33,7 @@
     except StructError, e:
         raise e.at_applevel(space)
     return space.newtuple(fmtiter.result_w[:])
+
+def clearcache(space):
+    "Clear the internal cache."
+    # No cache in this implementation

From noreply at buildbot.pypy.org  Thu Jan 19 00:11:42 2012
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu, 19 Jan 2012 00:11:42 +0100 (CET)
Subject: [pypy-commit] pypy py3k: Fix translation
Message-ID: <20120118231142.7C7BB820D8@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: py3k
Changeset: r51470:af6e253e47d6
Date: 2012-01-18 22:09 +0100
http://bitbucket.org/pypy/pypy/changeset/af6e253e47d6/

Log:	Fix translation

diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -582,26 +582,34 @@
         self.pushrevvalues(itemcount, items)
 
     def UNPACK_EX(self, oparg, next_instr):
+        "a, *b, c = range(10)"
         left = oparg & 0xFF
         right = (oparg & 0xFF00) >> 8
         w_iterable = self.popvalue()
-
         items = self.space.fixedview(w_iterable)
         itemcount = len(items)
-
+        if right < itemcount:
+            count = left + right
+            if count == 1:
+                plural = ''
+            else:
+                plural = 's'
+            raise operationerrfmt(self.space.w_ValueError,
+                                  "need more than %d value%s to unpack",
+                                  left + right, plural)
+        right = itemcount - right
+        assert right >= 0
+        # push values in reverse order
         i = itemcount - 1
-        while i >= itemcount-right:
+        while i >= right:
             self.pushvalue(items[i])
             i -= 1
-
-        self.pushvalue(self.space.newlist(items[left:itemcount-right]))
-
+        self.pushvalue(self.space.newlist(items[left:right]))
         i = left - 1
         while i >= 0:
             self.pushvalue(items[i])
             i -= 1
 
-
     def STORE_ATTR(self, nameindex, next_instr):
         "obj.attributename = newvalue"
         w_attributename = self.getname_w(nameindex)

From noreply at buildbot.pypy.org  Thu Jan 19 01:05:05 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Thu, 19 Jan 2012 01:05:05 +0100 (CET)
Subject: [pypy-commit] pypy numppy-flatitter: add setitem,
	getitem to flatitter
Message-ID: <20120119000505.55A85820D8@wyvern.cs.uni-duesseldorf.de>

Author: mattip
Branch: numppy-flatitter
Changeset: r51471:60b724406de5
Date: 2012-01-19 01:46 +0200
http://bitbucket.org/pypy/pypy/changeset/60b724406de5/

Log:	add setitem, getitem to flatitter

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -1299,13 +1299,18 @@
         size = 1
         for sh in arr.shape:
             size *= sh
-        self.strides = [arr.strides[-1]]
-        self.backstrides = [arr.backstrides[-1]]
-        ViewArray.__init__(self, size, [size], arr.dtype, arr.order,
-                               arr)
+        if arr.strides[-1] < arr.strides[0]:
+            self.strides = [arr.strides[-1]]
+            self.backstrides = [arr.backstrides[-1]]
+        else:
+            self.strides = [arr.strides[0]]
+            self.backstrides = [arr.backstrides[0]]
+        ViewArray.__init__(self, size, [size], arr.dtype, order=arr.order,
+                               parent=arr)
         self.shapelen = len(arr.shape)
         self.iter = OneDimIterator(arr.start, self.strides[0],
                                    self.shape[0])
+        self.base = arr
 
     def descr_next(self, space):
         if self.iter.done():
@@ -1317,9 +1322,42 @@
     def descr_iter(self):
         return self
 
+    def descr_getitem(self, space, w_idx):
+        if not space.isinstance_w(w_idx, space.w_int):
+            raise OperationError(space.w_ValueError, space.wrap(
+                        "non-integer indexing not supported yet"))
+        _i = space.int_w(w_idx)
+        if _i<0:
+            i = self.size + _i
+        else:
+            i = _i
+        if i >= self.size or i < 0:
+            raise operationerrfmt(space.w_IndexError,
+                            "index (%d) out of range (%d<=index<%d", 
+                                _i, -self.size, self.size)
+        result = self.getitem(self.base.start + i * self.strides[0])
+        return result
+
+    def descr_setitem(self, space, w_idx, w_value):
+        if not space.isinstance_w(w_idx, space.w_int):
+            raise OperationError(space.w_ValueError, space.wrap(
+                        "non-integer indexing not supported yet"))
+        _i = space.int_w(w_idx)
+        if _i<0:
+            i = self.size + _i
+        else:
+            i = _i
+        if i >= self.size or i < 0:
+            raise operationerrfmt(space.w_IndexError,
+                            "index (%d) out of range (%d<=index<%d", 
+                                _i, -self.size, self.size)
+        self.setitem(self.base.start + i * self.strides[0], w_value)
+
 W_FlatIterator.typedef = TypeDef(
     'flatiter',
     next = interp2app(W_FlatIterator.descr_next),
     __iter__ = interp2app(W_FlatIterator.descr_iter),
+    __getitem__ = interp2app(W_FlatIterator.descr_getitem),
+    __setitem__ = interp2app(W_FlatIterator.descr_setitem),
 )
 W_FlatIterator.acceptable_as_base_class = False
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -276,6 +276,12 @@
         for i in xrange(5):
             assert a[i] == b[i]
 
+    def test_getitem_nd(self):
+        from _numpypy import arange
+        a = arange(15).reshape(3, 5)
+        assert a[1, 3] == 8
+        assert a.T[1, 2] == 11 
+
     def test_setitem(self):
         from _numpypy import array
         a = array(range(5))
@@ -1286,6 +1292,29 @@
         a = ones((2, 2))
         assert list(((a + a).flat)) == [2, 2, 2, 2]
 
+    def test_flatiter_getitem(self):
+        from _numpypy import arange
+        a = arange(10)
+        assert a.flat[3] == 3
+        assert a[2:].flat[3] == 5
+        assert (a + a).flat[3] == 6
+        assert a[::2].flat[3] == 6
+        assert a.reshape(2,5).flat[3] == 3
+        b = a.flat
+        b.next()
+        b.next()
+        b.next()
+        assert b[3] == 3
+        assert b[-2] == 8
+        raises(IndexError, "b[11]")
+        raises(IndexError, "b[-11]")
+
+    def test_flatiter_transpose(self):
+        from _numpypy import arange
+        a = arange(10)
+        skip('out-of-order transformations do not work yet')
+        assert a.reshape(2,5).T.flat[3] == 6
+
     def test_slice_copy(self):
         from _numpypy import zeros
         a = zeros((10, 10))

From notifications-noreply at bitbucket.org  Thu Jan 19 01:54:02 2012
From: notifications-noreply at bitbucket.org (Bitbucket)
Date: Thu, 19 Jan 2012 00:54:02 -0000
Subject: [pypy-commit] Notification: pypy
Message-ID: <20120119005402.27163.74454@bitbucket05.managed.contegix.com>

You have received a notification from David Ripton.

Hi, I forked pypy. My fork is at https://bitbucket.org/dripton/pypy.

--
Disable notifications at https://bitbucket.org/account/notifications/

From noreply at buildbot.pypy.org  Thu Jan 19 10:57:46 2012
From: noreply at buildbot.pypy.org (rguillebert)
Date: Thu, 19 Jan 2012 10:57:46 +0100 (CET)
Subject: [pypy-commit] pypy py3k: Import the Grammar for the new yield from
	construct
Message-ID: <20120119095746.84751820D8@wyvern.cs.uni-duesseldorf.de>

Author: Romain Guillebert 
Branch: py3k
Changeset: r51472:1233713137f2
Date: 2012-01-19 10:52 +0100
http://bitbucket.org/pypy/pypy/changeset/1233713137f2/

Log:	Import the Grammar for the new yield from construct

diff --git a/pypy/interpreter/pyparser/data/Grammar3.2 b/pypy/interpreter/pyparser/data/Grammar3.2
--- a/pypy/interpreter/pyparser/data/Grammar3.2
+++ b/pypy/interpreter/pyparser/data/Grammar3.2
@@ -136,4 +136,5 @@
 # not used in grammar, but may appear in "node" passed from Parser to Compiler
 encoding_decl: NAME
 
-yield_expr: 'yield' [testlist]
+yield_expr: 'yield' [yield_arg]
+yield_arg: 'from' test | testlist

From noreply at buildbot.pypy.org  Thu Jan 19 10:57:47 2012
From: noreply at buildbot.pypy.org (rguillebert)
Date: Thu, 19 Jan 2012 10:57:47 +0100 (CET)
Subject: [pypy-commit] pypy py3k: Change the Grammar's indentation,
 it's easier to diff it with cpython's Grammar now
Message-ID: <20120119095747.C8990820D8@wyvern.cs.uni-duesseldorf.de>

Author: Romain Guillebert 
Branch: py3k
Changeset: r51473:2e923cfc9e5e
Date: 2012-01-19 10:57 +0100
http://bitbucket.org/pypy/pypy/changeset/2e923cfc9e5e/

Log:	Change the Grammar's indentation, it's easier to diff it with
	cpython's Grammar now

diff --git a/pypy/interpreter/pyparser/data/Grammar3.2 b/pypy/interpreter/pyparser/data/Grammar3.2
--- a/pypy/interpreter/pyparser/data/Grammar3.2
+++ b/pypy/interpreter/pyparser/data/Grammar3.2
@@ -11,9 +11,9 @@
 # "How to Change Python's Grammar"
 
 # Start symbols for the grammar:
-#	single_input is a single interactive statement;
-#	file_input is a module or sequence of commands read from an input file;
-#	eval_input is the input for the eval() and input() functions.
+#       single_input is a single interactive statement;
+#       file_input is a module or sequence of commands read from an input file;
+#       eval_input is the input for the eval() and input() functions.
 # NB: compound_stmt in single_input is followed by extra NEWLINE!
 single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
 file_input: (NEWLINE | stmt)* ENDMARKER
@@ -70,9 +70,9 @@
 for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite]
 try_stmt: ('try' ':' suite
            ((except_clause ':' suite)+
-	    ['else' ':' suite]
-	    ['finally' ':' suite] |
-	   'finally' ':' suite))
+            ['else' ':' suite]
+            ['finally' ':' suite] |
+           'finally' ':' suite))
 with_stmt: 'with' with_item (',' with_item)*  ':' suite
 with_item: test ['as' expr]
 # NB compile.c makes sure that the default except clause is last

From noreply at buildbot.pypy.org  Thu Jan 19 11:07:07 2012
From: noreply at buildbot.pypy.org (rguillebert)
Date: Thu, 19 Jan 2012 11:07:07 +0100 (CET)
Subject: [pypy-commit] pypy py3k: yield from is a python 3.3 construct
Message-ID: <20120119100707.C31B8820D8@wyvern.cs.uni-duesseldorf.de>

Author: Romain Guillebert 
Branch: py3k
Changeset: r51474:8339f8cc6ae1
Date: 2012-01-19 11:06 +0100
http://bitbucket.org/pypy/pypy/changeset/8339f8cc6ae1/

Log:	yield from is a python 3.3 construct

diff --git a/pypy/interpreter/pyparser/data/Grammar3.2 b/pypy/interpreter/pyparser/data/Grammar3.2
--- a/pypy/interpreter/pyparser/data/Grammar3.2
+++ b/pypy/interpreter/pyparser/data/Grammar3.2
@@ -136,5 +136,4 @@
 # not used in grammar, but may appear in "node" passed from Parser to Compiler
 encoding_decl: NAME
 
-yield_expr: 'yield' [yield_arg]
-yield_arg: 'from' test | testlist
+yield_expr: 'yield' [testlist]

From noreply at buildbot.pypy.org  Thu Jan 19 11:24:54 2012
From: noreply at buildbot.pypy.org (bivab)
Date: Thu, 19 Jan 2012 11:24:54 +0100 (CET)
Subject: [pypy-commit] extradoc extradoc: Planning for today
Message-ID: <20120119102454.EA4EC820D8@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider 
Branch: extradoc
Changeset: r4046:079d771c58af
Date: 2012-01-19 11:24 +0100
http://bitbucket.org/pypy/extradoc/changeset/079d771c58af/

Log:	Planning for today

diff --git a/sprintinfo/leysin-winter-2012/planning.txt b/sprintinfo/leysin-winter-2012/planning.txt
--- a/sprintinfo/leysin-winter-2012/planning.txt
+++ b/sprintinfo/leysin-winter-2012/planning.txt
@@ -10,20 +10,21 @@
 Things we want to do
 --------------------
 
-* some skiing (anto, armin)
+* some skiing (DONE)
 
 * review the JVM backend pull request (DONE)
 
-* py3k (romain)
+* py3k (romain, anto)
 
 * ffistruct
 
 * Cython backend
 
-* Debug the ARM backend (bivab)
+* Debug the ARM backend (bivab, armin around)
 
 * STM
-  - refactored the RPython API: mostly done, must adapt targetdemo*.py
+  - refactored the RPython API (DONE)
+  - app-level transaction module (armin, bivab around)
   - start work on the GC
 
 * concurrent-marksweep GC

From noreply at buildbot.pypy.org  Thu Jan 19 12:05:19 2012
From: noreply at buildbot.pypy.org (rguillebert)
Date: Thu, 19 Jan 2012 12:05:19 +0100 (CET)
Subject: [pypy-commit] pypy py3k: (antocuni,
 romain) implement keyword only args in the astbuilder
Message-ID: <20120119110519.3186E820D8@wyvern.cs.uni-duesseldorf.de>

Author: Romain Guillebert 
Branch: py3k
Changeset: r51475:27f5a4ae0d7d
Date: 2012-01-19 12:04 +0100
http://bitbucket.org/pypy/pypy/changeset/27f5a4ae0d7d/

Log:	(antocuni, romain) implement keyword only args in the astbuilder

diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -2280,18 +2280,22 @@
 
 class arguments(AST):
 
-    def __init__(self, args, vararg, kwarg, defaults):
+    def __init__(self, args, vararg, kwonlyargs, kwarg, defaults):
         self.args = args
         self.w_args = None
         self.vararg = vararg
+        self.kwonlyargs = kwonlyargs
+        self.w_kwonlyargs = None
         self.kwarg = kwarg
         self.defaults = defaults
         self.w_defaults = None
-        self.initialization_state = 15
+        self.initialization_state = 31
 
     def mutate_over(self, visitor):
         if self.args:
             visitor._mutate_sequence(self.args)
+        if self.kwonlyargs:
+            visitor._mutate_sequence(self.kwonlyargs)
         if self.defaults:
             visitor._mutate_sequence(self.defaults)
         return visitor.visit_arguments(self)
@@ -2300,12 +2304,12 @@
         visitor.visit_arguments(self)
 
     def sync_app_attrs(self, space):
-        if (self.initialization_state & ~6) ^ 9:
-            self.missing_field(space, ['args', None, None, 'defaults'], 'arguments')
+        if (self.initialization_state & ~10) ^ 21:
+            self.missing_field(space, ['args', None, 'kwonlyargs', None, 'defaults'], 'arguments')
         else:
             if not self.initialization_state & 2:
                 self.vararg = None
-            if not self.initialization_state & 4:
+            if not self.initialization_state & 8:
                 self.kwarg = None
         w_list = self.w_args
         if w_list is not None:
@@ -2317,6 +2321,16 @@
         if self.args is not None:
             for node in self.args:
                 node.sync_app_attrs(space)
+        w_list = self.w_kwonlyargs
+        if w_list is not None:
+            list_w = space.listview(w_list)
+            if list_w:
+                self.kwonlyargs = [space.interp_w(expr, w_obj) for w_obj in list_w]
+            else:
+                self.kwonlyargs = None
+        if self.kwonlyargs is not None:
+            for node in self.kwonlyargs:
+                node.sync_app_attrs(space)
         w_list = self.w_defaults
         if w_list is not None:
             list_w = space.listview(w_list)
@@ -2727,6 +2741,7 @@
 
     def visit_arguments(self, node):
         self.visit_sequence(node.args)
+        self.visit_sequence(node.kwonlyargs)
         self.visit_sequence(node.defaults)
 
     def visit_keyword(self, node):
@@ -6908,12 +6923,29 @@
     w_self.deldictvalue(space, 'vararg')
     w_self.initialization_state |= 2
 
+def arguments_get_kwonlyargs(space, w_self):
+    if not w_self.initialization_state & 4:
+        typename = space.type(w_self).getname(space)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'kwonlyargs')
+    if w_self.w_kwonlyargs is None:
+        if w_self.kwonlyargs is None:
+            list_w = []
+        else:
+            list_w = [space.wrap(node) for node in w_self.kwonlyargs]
+        w_list = space.newlist(list_w)
+        w_self.w_kwonlyargs = w_list
+    return w_self.w_kwonlyargs
+
+def arguments_set_kwonlyargs(space, w_self, w_new_value):
+    w_self.w_kwonlyargs = w_new_value
+    w_self.initialization_state |= 4
+
 def arguments_get_kwarg(space, w_self):
     if w_self.w_dict is not None:
         w_obj = w_self.getdictvalue(space, 'kwarg')
         if w_obj is not None:
             return w_obj
-    if not w_self.initialization_state & 4:
+    if not w_self.initialization_state & 8:
         typename = space.type(w_self).getname(space)
         raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'kwarg')
     return space.wrap(w_self.kwarg)
@@ -6930,10 +6962,10 @@
         w_self.setdictvalue(space, 'kwarg', w_new_value)
         return
     w_self.deldictvalue(space, 'kwarg')
-    w_self.initialization_state |= 4
+    w_self.initialization_state |= 8
 
 def arguments_get_defaults(space, w_self):
-    if not w_self.initialization_state & 8:
+    if not w_self.initialization_state & 16:
         typename = space.type(w_self).getname(space)
         raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'defaults')
     if w_self.w_defaults is None:
@@ -6947,17 +6979,18 @@
 
 def arguments_set_defaults(space, w_self, w_new_value):
     w_self.w_defaults = w_new_value
-    w_self.initialization_state |= 8
-
-_arguments_field_unroller = unrolling_iterable(['args', 'vararg', 'kwarg', 'defaults'])
+    w_self.initialization_state |= 16
+
+_arguments_field_unroller = unrolling_iterable(['args', 'vararg', 'kwonlyargs', 'kwarg', 'defaults'])
 def arguments_init(space, w_self, __args__):
     w_self = space.descr_self_interp_w(arguments, w_self)
     w_self.w_args = None
+    w_self.w_kwonlyargs = None
     w_self.w_defaults = None
     args_w, kwargs_w = __args__.unpack()
     if args_w:
-        if len(args_w) != 4:
-            w_err = space.wrap("arguments constructor takes either 0 or 4 positional arguments")
+        if len(args_w) != 5:
+            w_err = space.wrap("arguments constructor takes either 0 or 5 positional arguments")
             raise OperationError(space.w_TypeError, w_err)
         i = 0
         for field in _arguments_field_unroller:
@@ -6969,9 +7002,10 @@
 arguments.typedef = typedef.TypeDef("arguments",
     AST.typedef,
     __module__='_ast',
-    _fields=_FieldsWrapper(['args', 'vararg', 'kwarg', 'defaults']),
+    _fields=_FieldsWrapper(['args', 'vararg', 'kwonlyargs', 'kwarg', 'defaults']),
     args=typedef.GetSetProperty(arguments_get_args, arguments_set_args, cls=arguments),
     vararg=typedef.GetSetProperty(arguments_get_vararg, arguments_set_vararg, cls=arguments),
+    kwonlyargs=typedef.GetSetProperty(arguments_get_kwonlyargs, arguments_set_kwonlyargs, cls=arguments),
     kwarg=typedef.GetSetProperty(arguments_get_kwarg, arguments_set_kwarg, cls=arguments),
     defaults=typedef.GetSetProperty(arguments_get_defaults, arguments_set_defaults, cls=arguments),
     __new__=interp2app(get_AST_new(arguments)),
diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py
--- a/pypy/interpreter/astcompiler/astbuilder.py
+++ b/pypy/interpreter/astcompiler/astbuilder.py
@@ -508,13 +508,14 @@
         # and varargslist (lambda definition).
         if arguments_node.type == syms.parameters:
             if len(arguments_node.children) == 2:
-                return ast.arguments(None, None, None, None)
+                return ast.arguments(None, None, None, None, None)
             arguments_node = arguments_node.children[1]
         i = 0
         child_count = len(arguments_node.children)
         defaults = []
         args = []
         variable_arg = None
+        keywordonly_args = None
         keywords_arg = None
         have_default = False
         while i < child_count:
@@ -552,13 +553,16 @@
                         self.check_forbidden_name(arg_name, name_node)
                         name = ast.Name(arg_name, ast.Param, name_node.lineno,
                                         name_node.column)
-                        args.append(name)
+                        if keywordonly_args is None:
+                            args.append(name)
+                        else:
+                            keywordonly_args.append(name)
                     i += 2
                     break
             elif arg_type == tokens.STAR:
                 name_node = arguments_node.children[i + 1]
+                keywordonly_args = []
                 if name_node.type == tokens.COMMA:
-                    # XXX for now
                     i += 2
                 else:
                     variable_arg = name_node.children[0].value
@@ -575,7 +579,7 @@
             defaults = None
         if not args:
             args = None
-        return ast.arguments(args, variable_arg, keywords_arg, defaults)
+        return ast.arguments(args, variable_arg, keywordonly_args, keywords_arg, defaults)
 
     def handle_arg_unpacking(self, fplist_node):
         args = []
@@ -775,7 +779,7 @@
     def handle_lambdef(self, lambdef_node):
         expr = self.handle_expr(lambdef_node.children[-1])
         if len(lambdef_node.children) == 3:
-            args = ast.arguments(None, None, None, None)
+            args = ast.arguments(None, None, None, None, None)
         else:
             args = self.handle_arguments(lambdef_node.children[1])
         return ast.Lambda(args, expr, lambdef_node.lineno, lambdef_node.column)
diff --git a/pypy/interpreter/astcompiler/test/test_astbuilder.py b/pypy/interpreter/astcompiler/test/test_astbuilder.py
--- a/pypy/interpreter/astcompiler/test/test_astbuilder.py
+++ b/pypy/interpreter/astcompiler/test/test_astbuilder.py
@@ -1185,3 +1185,17 @@
         if1, if2 = comps[0].ifs
         assert isinstance(if1, ast.Name)
         assert isinstance(if2, ast.Name)
+
+    def test_kwonly_arguments(self):
+        fn = self.get_first_stmt("def f(a, b, c, *, kwarg): pass")
+        assert isinstance(fn, ast.FunctionDef)
+        assert len(fn.args.kwonlyargs) == 1
+        assert isinstance(fn.args.kwonlyargs[0], ast.expr)
+        assert fn.args.kwonlyargs[0].id == "kwarg"
+
+    def test_kwonly_arguments_2(self):
+        fn = self.get_first_stmt("def f(a, b, c, *args, kwarg): pass")
+        assert isinstance(fn, ast.FunctionDef)
+        assert len(fn.args.kwonlyargs) == 1
+        assert isinstance(fn.args.kwonlyargs[0], ast.expr)
+        assert fn.args.kwonlyargs[0].id == "kwarg"
diff --git a/pypy/interpreter/astcompiler/tools/Python.asdl b/pypy/interpreter/astcompiler/tools/Python.asdl
--- a/pypy/interpreter/astcompiler/tools/Python.asdl
+++ b/pypy/interpreter/astcompiler/tools/Python.asdl
@@ -105,7 +105,7 @@
 	excepthandler = ExceptHandler(expr? type, identifier? name, stmt* body)
 	                attributes(int lineno, int col_offset)
 
-	arguments = (expr* args, identifier? vararg, 
+	arguments = (expr* args, identifier? vararg, expr* kwonlyargs, 
 		     identifier? kwarg, expr* defaults)
 
         -- keyword arguments supplied to call

From noreply at buildbot.pypy.org  Thu Jan 19 12:19:37 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 19 Jan 2012 12:19:37 +0100 (CET)
Subject: [pypy-commit] pypy default: (bivab, arigo)
Message-ID: <20120119111937.3D45C820D8@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r51476:798d7b72c373
Date: 2012-01-19 12:19 +0100
http://bitbucket.org/pypy/pypy/changeset/798d7b72c373/

Log:	(bivab, arigo)

	Test cpu.get_latest_force_token(). It was not tested. Argh.

diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py
--- a/pypy/jit/backend/test/runner_test.py
+++ b/pypy/jit/backend/test/runner_test.py
@@ -1872,6 +1872,7 @@
                 values.append(descr)
                 values.append(self.cpu.get_latest_value_int(0))
                 values.append(self.cpu.get_latest_value_int(1))
+                values.append(token)
 
         FUNC = self.FuncType([lltype.Signed, lltype.Signed], lltype.Void)
         func_ptr = llhelper(lltype.Ptr(FUNC), maybe_force)
@@ -1902,7 +1903,8 @@
         assert fail.identifier == 1
         assert self.cpu.get_latest_value_int(0) == 1
         assert self.cpu.get_latest_value_int(1) == 10
-        assert values == [faildescr, 1, 10]
+        token = self.cpu.get_latest_force_token()
+        assert values == [faildescr, 1, 10, token]
 
     def test_force_operations_returning_int(self):
         values = []
@@ -1911,6 +1913,7 @@
                self.cpu.force(token)
                values.append(self.cpu.get_latest_value_int(0))
                values.append(self.cpu.get_latest_value_int(2))
+               values.append(token)
             return 42
 
         FUNC = self.FuncType([lltype.Signed, lltype.Signed], lltype.Signed)
@@ -1944,7 +1947,8 @@
         assert self.cpu.get_latest_value_int(0) == 1
         assert self.cpu.get_latest_value_int(1) == 42
         assert self.cpu.get_latest_value_int(2) == 10
-        assert values == [1, 10]
+        token = self.cpu.get_latest_force_token()
+        assert values == [1, 10, token]
 
     def test_force_operations_returning_float(self):
         values = []
@@ -1953,6 +1957,7 @@
                self.cpu.force(token)
                values.append(self.cpu.get_latest_value_int(0))
                values.append(self.cpu.get_latest_value_int(2))
+               values.append(token)
             return 42.5
 
         FUNC = self.FuncType([lltype.Signed, lltype.Signed], lltype.Float)
@@ -1988,7 +1993,8 @@
         x = self.cpu.get_latest_value_float(1)
         assert longlong.getrealfloat(x) == 42.5
         assert self.cpu.get_latest_value_int(2) == 10
-        assert values == [1, 10]
+        token = self.cpu.get_latest_force_token()
+        assert values == [1, 10, token]
 
     def test_call_to_c_function(self):
         from pypy.rlib.libffi import CDLL, types, ArgChain, FUNCFLAG_CDECL

From noreply at buildbot.pypy.org  Thu Jan 19 12:28:54 2012
From: noreply at buildbot.pypy.org (bivab)
Date: Thu, 19 Jan 2012 12:28:54 +0100 (CET)
Subject: [pypy-commit] pypy arm-backend-2: (arigo,
 bivab) update the fail_force_index when failing a guard
Message-ID: <20120119112854.B92DB820D8@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider 
Branch: arm-backend-2
Changeset: r51477:4c73e2cd39cf
Date: 2012-01-19 12:21 +0100
http://bitbucket.org/pypy/pypy/changeset/4c73e2cd39cf/

Log:	(arigo, bivab) update the fail_force_index when failing a guard

diff --git a/pypy/jit/backend/arm/assembler.py b/pypy/jit/backend/arm/assembler.py
--- a/pypy/jit/backend/arm/assembler.py
+++ b/pypy/jit/backend/arm/assembler.py
@@ -257,6 +257,7 @@
         to the failboxes.  Values for spilled vars and registers are stored on
         stack at frame_loc """
         assert frame_pointer & 1 == 0
+        self.fail_force_index = frame_pointer
         bytecode = rffi.cast(rffi.UCHARP, mem_loc)
         num = 0
         value = 0

From noreply at buildbot.pypy.org  Thu Jan 19 12:28:56 2012
From: noreply at buildbot.pypy.org (bivab)
Date: Thu, 19 Jan 2012 12:28:56 +0100 (CET)
Subject: [pypy-commit] pypy arm-backend-2: merge default
Message-ID: <20120119112856.E6A52820D8@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider 
Branch: arm-backend-2
Changeset: r51478:faf485aa5523
Date: 2012-01-19 12:26 +0100
http://bitbucket.org/pypy/pypy/changeset/faf485aa5523/

Log:	merge default

diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py
--- a/pypy/jit/backend/test/runner_test.py
+++ b/pypy/jit/backend/test/runner_test.py
@@ -2032,6 +2032,7 @@
                 values.append(descr)
                 values.append(self.cpu.get_latest_value_int(0))
                 values.append(self.cpu.get_latest_value_int(1))
+                values.append(token)
 
         FUNC = self.FuncType([lltype.Signed, lltype.Signed], lltype.Void)
         func_ptr = llhelper(lltype.Ptr(FUNC), maybe_force)
@@ -2062,7 +2063,8 @@
         assert fail.identifier == 1
         assert self.cpu.get_latest_value_int(0) == 1
         assert self.cpu.get_latest_value_int(1) == 10
-        assert values == [faildescr, 1, 10]
+        token = self.cpu.get_latest_force_token()
+        assert values == [faildescr, 1, 10, token]
 
     def test_force_operations_returning_int(self):
         values = []
@@ -2071,6 +2073,7 @@
                self.cpu.force(token)
                values.append(self.cpu.get_latest_value_int(0))
                values.append(self.cpu.get_latest_value_int(2))
+               values.append(token)
             return 42
 
         FUNC = self.FuncType([lltype.Signed, lltype.Signed], lltype.Signed)
@@ -2104,7 +2107,8 @@
         assert self.cpu.get_latest_value_int(0) == 1
         assert self.cpu.get_latest_value_int(1) == 42
         assert self.cpu.get_latest_value_int(2) == 10
-        assert values == [1, 10]
+        token = self.cpu.get_latest_force_token()
+        assert values == [1, 10, token]
 
     def test_force_operations_returning_float(self):
         if not self.cpu.supports_floats:
@@ -2115,6 +2119,7 @@
                self.cpu.force(token)
                values.append(self.cpu.get_latest_value_int(0))
                values.append(self.cpu.get_latest_value_int(2))
+               values.append(token)
             return 42.5
 
         FUNC = self.FuncType([lltype.Signed, lltype.Signed], lltype.Float)
@@ -2150,7 +2155,8 @@
         x = self.cpu.get_latest_value_float(1)
         assert longlong.getrealfloat(x) == 42.5
         assert self.cpu.get_latest_value_int(2) == 10
-        assert values == [1, 10]
+        token = self.cpu.get_latest_force_token()
+        assert values == [1, 10, token]
 
     def test_call_to_c_function(self):
         from pypy.rlib.libffi import CDLL, types, ArgChain, FUNCFLAG_CDECL
diff --git a/pypy/module/_codecs/test/test_ztranslation.py b/pypy/module/_codecs/test/test_ztranslation.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_codecs/test/test_ztranslation.py
@@ -0,0 +1,5 @@
+from pypy.objspace.fake.checkmodule import checkmodule
+
+
+def test__codecs_translates():
+    checkmodule('_codecs')
diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -20,7 +20,7 @@
 class W_Dtype(Wrappable):
     _immutable_fields_ = ["itemtype", "num", "kind"]
 
-    def __init__(self, itemtype, num, kind, name, char, w_box_type, alternate_constructors=[]):
+    def __init__(self, itemtype, num, kind, name, char, w_box_type, alternate_constructors=[], aliases=[]):
         self.itemtype = itemtype
         self.num = num
         self.kind = kind
@@ -28,6 +28,7 @@
         self.char = char
         self.w_box_type = w_box_type
         self.alternate_constructors = alternate_constructors
+        self.aliases = aliases
 
     def malloc(self, length):
         # XXX find out why test_zjit explodes with tracking of allocations
@@ -62,7 +63,7 @@
         elif space.isinstance_w(w_dtype, space.w_str):
             name = space.str_w(w_dtype)
             for dtype in cache.builtin_dtypes:
-                if dtype.name == name or dtype.char == name:
+                if dtype.name == name or dtype.char == name or name in dtype.aliases:
                     return dtype
         else:
             for dtype in cache.builtin_dtypes:
@@ -107,7 +108,7 @@
             kind=BOOLLTR,
             name="bool",
             char="?",
-            w_box_type = space.gettypefor(interp_boxes.W_BoolBox),
+            w_box_type=space.gettypefor(interp_boxes.W_BoolBox),
             alternate_constructors=[space.w_bool],
         )
         self.w_int8dtype = W_Dtype(
@@ -116,7 +117,7 @@
             kind=SIGNEDLTR,
             name="int8",
             char="b",
-            w_box_type = space.gettypefor(interp_boxes.W_Int8Box)
+            w_box_type=space.gettypefor(interp_boxes.W_Int8Box)
         )
         self.w_uint8dtype = W_Dtype(
             types.UInt8(),
@@ -124,7 +125,7 @@
             kind=UNSIGNEDLTR,
             name="uint8",
             char="B",
-            w_box_type = space.gettypefor(interp_boxes.W_UInt8Box),
+            w_box_type=space.gettypefor(interp_boxes.W_UInt8Box),
         )
         self.w_int16dtype = W_Dtype(
             types.Int16(),
@@ -132,7 +133,7 @@
             kind=SIGNEDLTR,
             name="int16",
             char="h",
-            w_box_type = space.gettypefor(interp_boxes.W_Int16Box),
+            w_box_type=space.gettypefor(interp_boxes.W_Int16Box),
         )
         self.w_uint16dtype = W_Dtype(
             types.UInt16(),
@@ -140,7 +141,7 @@
             kind=UNSIGNEDLTR,
             name="uint16",
             char="H",
-            w_box_type = space.gettypefor(interp_boxes.W_UInt16Box),
+            w_box_type=space.gettypefor(interp_boxes.W_UInt16Box),
         )
         self.w_int32dtype = W_Dtype(
             types.Int32(),
@@ -148,7 +149,7 @@
             kind=SIGNEDLTR,
             name="int32",
             char="i",
-             w_box_type = space.gettypefor(interp_boxes.W_Int32Box),
+            w_box_type=space.gettypefor(interp_boxes.W_Int32Box),
        )
         self.w_uint32dtype = W_Dtype(
             types.UInt32(),
@@ -156,7 +157,7 @@
             kind=UNSIGNEDLTR,
             name="uint32",
             char="I",
-            w_box_type = space.gettypefor(interp_boxes.W_UInt32Box),
+            w_box_type=space.gettypefor(interp_boxes.W_UInt32Box),
         )
         if LONG_BIT == 32:
             name = "int32"
@@ -168,7 +169,7 @@
             kind=SIGNEDLTR,
             name=name,
             char="l",
-            w_box_type = space.gettypefor(interp_boxes.W_LongBox),
+            w_box_type=space.gettypefor(interp_boxes.W_LongBox),
             alternate_constructors=[space.w_int],
         )
         self.w_ulongdtype = W_Dtype(
@@ -177,7 +178,7 @@
             kind=UNSIGNEDLTR,
             name="u" + name,
             char="L",
-            w_box_type = space.gettypefor(interp_boxes.W_ULongBox),
+            w_box_type=space.gettypefor(interp_boxes.W_ULongBox),
         )
         self.w_int64dtype = W_Dtype(
             types.Int64(),
@@ -185,7 +186,7 @@
             kind=SIGNEDLTR,
             name="int64",
             char="q",
-            w_box_type = space.gettypefor(interp_boxes.W_Int64Box),
+            w_box_type=space.gettypefor(interp_boxes.W_Int64Box),
             alternate_constructors=[space.w_long],
         )
         self.w_uint64dtype = W_Dtype(
@@ -194,7 +195,7 @@
             kind=UNSIGNEDLTR,
             name="uint64",
             char="Q",
-            w_box_type = space.gettypefor(interp_boxes.W_UInt64Box),
+            w_box_type=space.gettypefor(interp_boxes.W_UInt64Box),
         )
         self.w_float32dtype = W_Dtype(
             types.Float32(),
@@ -202,7 +203,7 @@
             kind=FLOATINGLTR,
             name="float32",
             char="f",
-            w_box_type = space.gettypefor(interp_boxes.W_Float32Box),
+            w_box_type=space.gettypefor(interp_boxes.W_Float32Box),
         )
         self.w_float64dtype = W_Dtype(
             types.Float64(),
@@ -212,6 +213,7 @@
             char="d",
             w_box_type = space.gettypefor(interp_boxes.W_Float64Box),
             alternate_constructors=[space.w_float],
+            aliases=["float"],
         )
 
         self.builtin_dtypes = [
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -581,7 +581,7 @@
     def descr_var(self, space):
         # var = mean((values - mean(values)) ** 2)
         w_res = self.descr_sub(space, self.descr_mean(space, space.w_None))
-        assert isinstance(w_res, BaseArray) 
+        assert isinstance(w_res, BaseArray)
         w_res = w_res.descr_pow(space, space.wrap(2))
         assert isinstance(w_res, BaseArray)
         return w_res.descr_mean(space, space.w_None)
@@ -590,6 +590,10 @@
         # std(v) = sqrt(var(v))
         return interp_ufuncs.get(space).sqrt.call(space, [self.descr_var(space)])
 
+    def descr_fill(self, space, w_value):
+        concr = self.get_concrete_or_scalar()
+        concr.fill(space, w_value)
+
     def descr_nonzero(self, space):
         if self.size > 1:
             raise OperationError(space.w_ValueError, space.wrap(
@@ -682,6 +686,9 @@
     def copy(self, space):
         return Scalar(self.dtype, self.value)
 
+    def fill(self, space, w_value):
+        self.value = self.dtype.coerce(space, w_value)
+
     def create_sig(self):
         return signature.ScalarSignature(self.dtype)
 
@@ -788,7 +795,7 @@
     Intermediate class for performing binary operations.
     """
     _immutable_fields_ = ['left', 'right']
-    
+
     def __init__(self, ufunc, name, shape, calc_dtype, res_dtype, left, right):
         VirtualArray.__init__(self, name, shape, res_dtype)
         self.ufunc = ufunc
@@ -828,7 +835,7 @@
     def __init__(self, shape, dtype, left, right):
         Call2.__init__(self, None, 'sliceloop', shape, dtype, dtype, left,
                        right)
-    
+
     def create_sig(self):
         lsig = self.left.create_sig()
         rsig = self.right.create_sig()
@@ -847,7 +854,7 @@
     when we'll make AxisReduce lazy
     """
     _immutable_fields_ = ['left', 'right']
-    
+
     def __init__(self, ufunc, name, shape, dtype, left, right, dim):
         Call2.__init__(self, ufunc, name, shape, dtype, dtype,
                        left, right)
@@ -919,14 +926,14 @@
         if size < 1:
             builder.append('[]')
             return
-        elif size == 1:
-            builder.append(dtype.itemtype.str_format(self.getitem(0)))
-            return
         if size > 1000:
             # Once this goes True it does not go back to False for recursive
             # calls
             use_ellipsis = True
         ndims = len(self.shape)
+        if ndims == 0:
+            builder.append(dtype.itemtype.str_format(self.getitem(0)))
+            return
         i = 0
         builder.append('[')
         if ndims > 1:
@@ -1061,6 +1068,9 @@
         array.setslice(space, self)
         return array
 
+    def fill(self, space, w_value):
+        self.setslice(space, scalar_w(space, self.dtype, w_value))
+
 
 class ViewArray(ConcreteArray):
     def create_sig(self):
@@ -1273,6 +1283,8 @@
     var = interp2app(BaseArray.descr_var),
     std = interp2app(BaseArray.descr_std),
 
+    fill = interp2app(BaseArray.descr_fill),
+
     copy = interp2app(BaseArray.descr_copy),
     reshape = interp2app(BaseArray.descr_reshape),
     tolist = interp2app(BaseArray.descr_tolist),
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -166,14 +166,11 @@
         # You can't subclass dtype
         raises(TypeError, type, "Foo", (dtype,), {})
 
-    def test_new(self):
-        import _numpypy as np
-        assert np.int_(4) == 4
-        assert np.float_(3.4) == 3.4
+    def test_aliases(self):
+        from _numpypy import dtype
 
-    def test_pow(self):
-        from _numpypy import int_
-        assert int_(4) ** 2 == 16
+        assert dtype("float") is dtype(float)
+
 
 class AppTestTypes(BaseNumpyAppTest):
     def test_abstract_types(self):
@@ -189,6 +186,15 @@
         raises(TypeError, numpy.floating, 0)
         raises(TypeError, numpy.inexact, 0)
 
+    def test_new(self):
+        import _numpypy as np
+        assert np.int_(4) == 4
+        assert np.float_(3.4) == 3.4
+
+    def test_pow(self):
+        from _numpypy import int_
+        assert int_(4) ** 2 == 16
+
     def test_bool(self):
         import _numpypy as numpy
 
@@ -318,7 +324,7 @@
         else:
             raises(OverflowError, numpy.int64, 9223372036854775807)
             raises(OverflowError, numpy.int64, '9223372036854775807')
-        
+
         raises(OverflowError, numpy.int64, 9223372036854775808)
         raises(OverflowError, numpy.int64, '9223372036854775808')
 
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1302,6 +1302,28 @@
         assert isinstance(i['data'][0], int)
         raises(TypeError, getattr, array(3), '__array_interface__')
 
+    def test_fill(self):
+        from _numpypy import array
+
+        a = array([1, 2, 3])
+        a.fill(10)
+        assert (a == [10, 10, 10]).all()
+        a.fill(False)
+        assert (a == [0, 0, 0]).all()
+
+        b = a[:1]
+        b.fill(4)
+        assert (b == [4]).all()
+        assert (a == [4, 0, 0]).all()
+
+        c = b + b
+        c.fill(27)
+        assert (c == [27]).all()
+
+        d = array(10)
+        d.fill(100)
+        assert d == 100
+
 
 class AppTestSupport(BaseNumpyAppTest):
     def setup_class(cls):
@@ -1441,9 +1463,11 @@
         assert repr(a) == "array(0.0)"
         a = array(0.2)
         assert repr(a) == "array(0.2)"
+        a = array([2])
+        assert repr(a) == "array([2])"
 
     def test_repr_multi(self):
-        from _numpypy import arange, zeros
+        from _numpypy import arange, zeros, array
         a = zeros((3, 4))
         assert repr(a) == '''array([[0.0, 0.0, 0.0, 0.0],
        [0.0, 0.0, 0.0, 0.0],
@@ -1466,6 +1490,9 @@
        [498, 999],
        [499, 1000],
        [500, 1001]])'''
+        a = arange(2).reshape((2,1))
+        assert repr(a) == '''array([[0],
+       [1]])'''
 
     def test_repr_slice(self):
         from _numpypy import array, zeros
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -349,7 +349,8 @@
         self.check_trace_count(1)
         self.check_simple_loop({'getinteriorfield_raw': 2, 'float_add': 1,
                                 'setinteriorfield_raw': 1, 'int_add': 2,
-                                'int_eq': 1, 'guard_false': 1, 'jump': 1})
+                                'int_eq': 1, 'guard_false': 1, 'jump': 1,
+                                'arraylen_gc': 1})
 
     def define_virtual_slice():
         return """
diff --git a/pypy/rpython/lltypesystem/rtuple.py b/pypy/rpython/lltypesystem/rtuple.py
--- a/pypy/rpython/lltypesystem/rtuple.py
+++ b/pypy/rpython/lltypesystem/rtuple.py
@@ -27,6 +27,10 @@
 
     def newtuple(cls, llops, r_tuple, items_v):
         # items_v should have the lowleveltype of the internal reprs
+        assert len(r_tuple.items_r) == len(items_v)
+        for r_item, v_item in zip(r_tuple.items_r, items_v):
+            assert r_item.lowleveltype == v_item.concretetype
+        #
         if len(r_tuple.items_r) == 0:
             return inputconst(Void, ())    # a Void empty tuple
         c1 = inputconst(Void, r_tuple.lowleveltype.TO)
diff --git a/pypy/rpython/rrange.py b/pypy/rpython/rrange.py
--- a/pypy/rpython/rrange.py
+++ b/pypy/rpython/rrange.py
@@ -204,7 +204,10 @@
         v_index = hop.gendirectcall(self.ll_getnextindex, v_enumerate)
         hop2 = hop.copy()
         hop2.args_r = [self.r_baseiter]
+        r_item_src = self.r_baseiter.r_list.external_item_repr
+        r_item_dst = hop.r_result.items_r[1]
         v_item = self.r_baseiter.rtype_next(hop2)
+        v_item = hop.llops.convertvar(v_item, r_item_src, r_item_dst)
         return hop.r_result.newtuple(hop.llops, hop.r_result,
                                      [v_index, v_item])
 
diff --git a/pypy/rpython/test/test_rrange.py b/pypy/rpython/test/test_rrange.py
--- a/pypy/rpython/test/test_rrange.py
+++ b/pypy/rpython/test/test_rrange.py
@@ -169,6 +169,22 @@
         res = self.interpret(fn, [2])
         assert res == 789
 
+    def test_enumerate_instances(self):
+        class A:
+            pass
+        def fn(n):
+            a = A()
+            b = A()
+            a.k = 10
+            b.k = 20
+            for i, x in enumerate([a, b]):
+                if i == n:
+                    return x.k
+            return 5
+        res = self.interpret(fn, [1])
+        assert res == 20
+
+
 
 class TestLLtype(BaseTestRrange, LLRtypeMixin):
     from pypy.rpython.lltypesystem import rrange 

From noreply at buildbot.pypy.org  Thu Jan 19 12:28:58 2012
From: noreply at buildbot.pypy.org (bivab)
Date: Thu, 19 Jan 2012 12:28:58 +0100 (CET)
Subject: [pypy-commit] pypy arm-backend-2: merge heads
Message-ID: <20120119112858.87BD8820D8@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider 
Branch: arm-backend-2
Changeset: r51479:75e7ec3f604f
Date: 2012-01-19 12:28 +0100
http://bitbucket.org/pypy/pypy/changeset/75e7ec3f604f/

Log:	merge heads

diff --git a/pypy/jit/backend/arm/assembler.py b/pypy/jit/backend/arm/assembler.py
--- a/pypy/jit/backend/arm/assembler.py
+++ b/pypy/jit/backend/arm/assembler.py
@@ -257,6 +257,7 @@
         to the failboxes.  Values for spilled vars and registers are stored on
         stack at frame_loc """
         assert frame_pointer & 1 == 0
+        self.fail_force_index = frame_pointer
         bytecode = rffi.cast(rffi.UCHARP, mem_loc)
         num = 0
         value = 0

From noreply at buildbot.pypy.org  Thu Jan 19 12:55:31 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Thu, 19 Jan 2012 12:55:31 +0100 (CET)
Subject: [pypy-commit] pypy py3k: (antocuni,
 romain): fix test_try to match the new python3 syntax and the
 relative ast structure
Message-ID: <20120119115531.29553820D8@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni 
Branch: py3k
Changeset: r51480:cafc48ed3976
Date: 2012-01-19 12:18 +0100
http://bitbucket.org/pypy/pypy/changeset/cafc48ed3976/

Log:	(antocuni, romain): fix test_try to match the new python3 syntax and
	the relative ast structure

diff --git a/pypy/interpreter/astcompiler/test/test_astbuilder.py b/pypy/interpreter/astcompiler/test/test_astbuilder.py
--- a/pypy/interpreter/astcompiler/test/test_astbuilder.py
+++ b/pypy/interpreter/astcompiler/test/test_astbuilder.py
@@ -311,13 +311,15 @@
         assert isinstance(fr.orelse[0].value, ast.Num)
 
     def test_try(self):
-        tr = self.get_first_stmt("try: x\nfinally: pass")
+        tr = self.get_first_stmt("try: x" + "\n" +
+                                 "finally: pass")
         assert isinstance(tr, ast.TryFinally)
         assert len(tr.body) == 1
         assert isinstance(tr.body[0].value, ast.Name)
         assert len(tr.finalbody) == 1
         assert isinstance(tr.finalbody[0], ast.Pass)
-        tr = self.get_first_stmt("try: x\nexcept: pass")
+        tr = self.get_first_stmt("try: x" + "\n" +
+                                 "except: pass")
         assert isinstance(tr, ast.TryExcept)
         assert len(tr.body) == 1
         assert isinstance(tr.body[0].value, ast.Name)
@@ -329,7 +331,8 @@
         assert len(handler.body) == 1
         assert isinstance(handler.body[0], ast.Pass)
         assert tr.orelse is None
-        tr = self.get_first_stmt("try: x\nexcept Exception: pass")
+        tr = self.get_first_stmt("try: x" + "\n" +
+                                 "except Exception: pass")
         assert len(tr.handlers) == 1
         handler = tr.handlers[0]
         assert isinstance(handler.type, ast.Name)
@@ -337,40 +340,48 @@
         assert handler.name is None
         assert len(handler.body) == 1
         assert tr.orelse is None
-        tr = self.get_first_stmt("try: x\nexcept Exception, e: pass")
+        tr = self.get_first_stmt("try: x" + "\n" +
+                                 "except Exception as e: pass")
         assert len(tr.handlers) == 1
         handler = tr.handlers[0]
         assert isinstance(handler.type, ast.Name)
-        assert isinstance(handler.name, ast.Name)
-        assert handler.name.ctx == ast.Store
-        assert handler.name.id == "e"
+        assert handler.type.id == "Exception"
+        assert handler.name == "e"
         assert len(handler.body) == 1
-        tr = self.get_first_stmt("try: x\nexcept: pass\nelse: 4")
+        tr = self.get_first_stmt("try: x" + "\n" +
+                                 "except: pass" + "\n" +
+                                 "else: 4")
         assert len(tr.body) == 1
         assert isinstance(tr.body[0].value, ast.Name)
         assert len(tr.handlers) == 1
         assert isinstance(tr.handlers[0].body[0], ast.Pass)
         assert len(tr.orelse) == 1
         assert isinstance(tr.orelse[0].value, ast.Num)
-        tr = self.get_first_stmt("try: x\nexcept Exc, a: 5\nexcept F: pass")
+        tr = self.get_first_stmt("try: x" + "\n" +
+                                 "except Exc as a: 5" + "\n" +
+                                 "except F: pass")
         assert len(tr.handlers) == 2
         h1, h2 = tr.handlers
         assert isinstance(h1.type, ast.Name)
-        assert isinstance(h1.name, ast.Name)
+        assert h1.name == "a"
         assert isinstance(h1.body[0].value, ast.Num)
         assert isinstance(h2.type, ast.Name)
         assert h2.name is None
         assert isinstance(h2.body[0], ast.Pass)
-        tr = self.get_first_stmt("try: x\nexcept Exc as a: 5\nexcept F: pass")
+        tr = self.get_first_stmt("try: x" + "\n" +
+                                 "except Exc as a: 5" + "\n" +
+                                 "except F: pass")
         assert len(tr.handlers) == 2
         h1, h2 = tr.handlers
         assert isinstance(h1.type, ast.Name)
-        assert isinstance(h1.name, ast.Name)
+        assert h1.name == "a"
         assert isinstance(h1.body[0].value, ast.Num)
         assert isinstance(h2.type, ast.Name)
         assert h2.name is None
         assert isinstance(h2.body[0], ast.Pass)
-        tr = self.get_first_stmt("try: x\nexcept: 4\nfinally: pass")
+        tr = self.get_first_stmt("try: x" + "\n" +
+                                 "except: 4" + "\n" +
+                                 "finally: pass")
         assert isinstance(tr, ast.TryFinally)
         assert len(tr.finalbody) == 1
         assert isinstance(tr.finalbody[0], ast.Pass)
@@ -382,7 +393,10 @@
         assert isinstance(exc.handlers[0].body[0].value, ast.Num)
         assert len(exc.body) == 1
         assert isinstance(exc.body[0].value, ast.Name)
-        tr = self.get_first_stmt("try: x\nexcept: 4\nelse: 'hi'\nfinally: pass")
+        tr = self.get_first_stmt("try: x" + "\n" +
+                                 "except: 4" + "\n" +
+                                 "else: 'hi'" + "\n" +
+                                 "finally: pass")
         assert isinstance(tr, ast.TryFinally)
         assert len(tr.finalbody) == 1
         assert isinstance(tr.finalbody[0], ast.Pass)

From noreply at buildbot.pypy.org  Thu Jan 19 12:55:32 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Thu, 19 Jan 2012 12:55:32 +0100 (CET)
Subject: [pypy-commit] pypy py3k: (antocuni,
 romain): fix test_set_context by removing the case u'str' which is
 no longer syntactically correct
Message-ID: <20120119115532.6568D820D8@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni 
Branch: py3k
Changeset: r51481:5efd72235ac9
Date: 2012-01-19 12:21 +0100
http://bitbucket.org/pypy/pypy/changeset/5efd72235ac9/

Log:	(antocuni, romain): fix test_set_context by removing the case u'str'
	which is no longer syntactically correct

diff --git a/pypy/interpreter/astcompiler/test/test_astbuilder.py b/pypy/interpreter/astcompiler/test/test_astbuilder.py
--- a/pypy/interpreter/astcompiler/test/test_astbuilder.py
+++ b/pypy/interpreter/astcompiler/test/test_astbuilder.py
@@ -705,7 +705,6 @@
             ("{x for x in z}", "set comprehension"),
             ("{x : x for x in z}", "dict comprehension"),
             ("'str'", "literal"),
-            ("u'str'", "literal"),
             ("b'bytes'", "literal"),
             ("()", "()"),
             ("23", "literal"),

From noreply at buildbot.pypy.org  Thu Jan 19 12:55:33 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Thu, 19 Jan 2012 12:55:33 +0100 (CET)
Subject: [pypy-commit] pypy py3k: (antocuni,
 romain): fix test_string by using a plain string literal instead of
 the no-longer valid u''
Message-ID: <20120119115533.9DDE1820D8@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni 
Branch: py3k
Changeset: r51482:d39b62c9983e
Date: 2012-01-19 12:22 +0100
http://bitbucket.org/pypy/pypy/changeset/d39b62c9983e/

Log:	(antocuni, romain): fix test_string by using a plain string literal
	instead of the no-longer valid u''

diff --git a/pypy/interpreter/astcompiler/test/test_astbuilder.py b/pypy/interpreter/astcompiler/test/test_astbuilder.py
--- a/pypy/interpreter/astcompiler/test/test_astbuilder.py
+++ b/pypy/interpreter/astcompiler/test/test_astbuilder.py
@@ -1055,7 +1055,7 @@
         assert space.eq_w(s.s, space.wrapbytes("hi implicitly extra"))
         raises(SyntaxError, self.get_first_expr, "b'hello' 'world'")
         sentence = u"Die Männer ärgen sich!"
-        source = u"# coding: utf-7\nstuff = u'%s'" % (sentence,)
+        source = u"# coding: utf-7\nstuff = '%s'" % (sentence,)
         info = pyparse.CompileInfo("", "exec")
         tree = self.parser.parse_source(source.encode("utf-7"), info)
         assert info.encoding == "utf-7"

From noreply at buildbot.pypy.org  Thu Jan 19 12:55:34 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Thu, 19 Jan 2012 12:55:34 +0100 (CET)
Subject: [pypy-commit] pypy py3k: (antocuni,
 romain): 'mostly' fix test_number,
 by adapting it to the new py3 syntax for number literals. The only failure
 left is '0000' which still raises a syntax error, fixing it soon
Message-ID: <20120119115534.D64ED820D8@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni 
Branch: py3k
Changeset: r51483:d2ccbdd80be0
Date: 2012-01-19 12:30 +0100
http://bitbucket.org/pypy/pypy/changeset/d2ccbdd80be0/

Log:	(antocuni, romain): 'mostly' fix test_number, by adapting it to the
	new py3 syntax for number literals. The only failure left is '0000'
	which still raises a syntax error, fixing it soon

diff --git a/pypy/interpreter/astcompiler/test/test_astbuilder.py b/pypy/interpreter/astcompiler/test/test_astbuilder.py
--- a/pypy/interpreter/astcompiler/test/test_astbuilder.py
+++ b/pypy/interpreter/astcompiler/test/test_astbuilder.py
@@ -1084,14 +1084,11 @@
         space = self.space
         assert space.eq_w(get_num("32"), space.wrap(32))
         assert space.eq_w(get_num("32.5"), space.wrap(32.5))
-        assert space.eq_w(get_num("32L"), space.newlong(32))
-        assert space.eq_w(get_num("32l"), space.newlong(32))
-        assert space.eq_w(get_num("0L"), space.newlong(0))
         assert space.eq_w(get_num("2"), space.wrap(2))
         assert space.eq_w(get_num("13j"), space.wrap(13j))
         assert space.eq_w(get_num("13J"), space.wrap(13J))
-        assert space.eq_w(get_num("053"), space.wrap(053))
-        assert space.eq_w(get_num("00053"), space.wrap(053))
+        assert space.eq_w(get_num("0o53"), space.wrap(053))
+        assert space.eq_w(get_num("0o0053"), space.wrap(053))
         for num in ("0x53", "0X53", "0x0000053", "0X00053"):
             assert space.eq_w(get_num(num), space.wrap(0x53))
         assert space.eq_w(get_num("0Xb0d2"), space.wrap(0xb0d2))
@@ -1100,7 +1097,7 @@
         assert space.eq_w(get_num("00000"), space.wrap(0))
         assert space.eq_w(get_num("-3"), space.wrap(-3))
         assert space.eq_w(get_num("-0"), space.wrap(0))
-        assert space.eq_w(get_num("-0xAAAAAAL"), space.wrap(-0xAAAAAAL))
+        assert space.eq_w(get_num("-0xAAAAAA"), space.wrap(-0xAAAAAAL))
         n = get_num(str(-sys.maxint - 1))
         assert space.is_true(space.isinstance(n, space.w_int))
         for num in ("0o53", "0O53", "0o0000053", "0O00053"):
@@ -1111,6 +1108,12 @@
         py.test.raises(SyntaxError, self.get_ast, "0x")
         py.test.raises(SyntaxError, self.get_ast, "0b")
         py.test.raises(SyntaxError, self.get_ast, "0o")
+        py.test.raises(SyntaxError, self.get_ast, "32L")
+        py.test.raises(SyntaxError, self.get_ast, "32l")
+        py.test.raises(SyntaxError, self.get_ast, "0L")
+        py.test.raises(SyntaxError, self.get_ast, "-0xAAAAAAL")
+        py.test.raises(SyntaxError, self.get_ast, "053")
+        py.test.raises(SyntaxError, self.get_ast, "00053")
 
     def check_comprehension(self, brackets, ast_type):
         def brack(s):

From noreply at buildbot.pypy.org  Thu Jan 19 12:55:36 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Thu, 19 Jan 2012 12:55:36 +0100 (CET)
Subject: [pypy-commit] pypy py3k: (antocuni,
 romain): update the tokenizer to accept 0000* as a literal
Message-ID: <20120119115536.1C0FA820D8@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni 
Branch: py3k
Changeset: r51484:ac227cc49124
Date: 2012-01-19 12:44 +0100
http://bitbucket.org/pypy/pypy/changeset/ac227cc49124/

Log:	(antocuni, romain): update the tokenizer to accept 0000* as a
	literal

diff --git a/pypy/interpreter/pyparser/genpytokenize.py b/pypy/interpreter/pyparser/genpytokenize.py
--- a/pypy/interpreter/pyparser/genpytokenize.py
+++ b/pypy/interpreter/pyparser/genpytokenize.py
@@ -73,7 +73,9 @@
     decNumber = chain(states,
                       groupStr(states, "123456789"),
                       any(states, makeDigits()))
-    zero = newArcPair(states, "0")
+    zero = chain(states,
+                 newArcPair(states, "0"),
+                 any(states, newArcPair(states, "0")))
     intNumber = group(states, hexNumber, octNumber, binNumber, decNumber, zero)
     # ____________________________________________________________
     # Exponents
diff --git a/pypy/interpreter/pyparser/pytokenize.py b/pypy/interpreter/pyparser/pytokenize.py
--- a/pypy/interpreter/pyparser/pytokenize.py
+++ b/pypy/interpreter/pyparser/pytokenize.py
@@ -25,10 +25,10 @@
 
 accepts = [True, True, True, True, True, True, True, True,
            True, True, False, True, True, True, True, False,
-           False, False, True, False, False, False, False,
-           True, False, True, False, True, False, False,
-           True, False, False, True, True, True, False,
-           False, True, False, False, False, True]
+           False, False, True, False, False, False, True,
+           False, True, False, True, False, True, False,
+           False, True, False, False, True, True, True,
+           False, False, True, False, False, False, True]
 states = [
     # 0
     {'\t': 0, '\n': 13, '\x0c': 0,
@@ -110,21 +110,21 @@
      'v': 1, 'w': 1, 'x': 1, 'y': 1,
      'z': 1},
     # 4
-    {'.': 23, '0': 22, '1': 22, '2': 22,
-     '3': 22, '4': 22, '5': 22, '6': 22,
-     '7': 22, '8': 22, '9': 22, 'B': 21,
-     'E': 24, 'J': 13, 'O': 20, 'X': 19,
-     'b': 21, 'e': 24, 'j': 13, 'o': 20,
+    {'.': 24, '0': 22, '1': 23, '2': 23,
+     '3': 23, '4': 23, '5': 23, '6': 23,
+     '7': 23, '8': 23, '9': 23, 'B': 21,
+     'E': 25, 'J': 13, 'O': 20, 'X': 19,
+     'b': 21, 'e': 25, 'j': 13, 'o': 20,
      'x': 19},
     # 5
-    {'.': 23, '0': 5, '1': 5, '2': 5,
+    {'.': 24, '0': 5, '1': 5, '2': 5,
      '3': 5, '4': 5, '5': 5, '6': 5,
-     '7': 5, '8': 5, '9': 5, 'E': 24,
-     'J': 13, 'e': 24, 'j': 13},
+     '7': 5, '8': 5, '9': 5, 'E': 25,
+     'J': 13, 'e': 25, 'j': 13},
     # 6
-    {'0': 25, '1': 25, '2': 25, '3': 25,
-     '4': 25, '5': 25, '6': 25, '7': 25,
-     '8': 25, '9': 25},
+    {'0': 26, '1': 26, '2': 26, '3': 26,
+     '4': 26, '5': 26, '6': 26, '7': 26,
+     '8': 26, '9': 26},
     # 7
     {'*': 12, '=': 13},
     # 8
@@ -142,100 +142,105 @@
     # 14
     {'\n': 13},
     # 15
-    {automata.DEFAULT: 29, '\n': 26,
-     '\r': 26, "'": 27, '\\': 28},
+    {automata.DEFAULT: 30, '\n': 27,
+     '\r': 27, "'": 28, '\\': 29},
     # 16
-    {automata.DEFAULT: 32, '\n': 26,
-     '\r': 26, '"': 30, '\\': 31},
+    {automata.DEFAULT: 33, '\n': 27,
+     '\r': 27, '"': 31, '\\': 32},
     # 17
     {'\n': 13, '\r': 14},
     # 18
-    {automata.DEFAULT: 18, '\n': 26, '\r': 26},
+    {automata.DEFAULT: 18, '\n': 27, '\r': 27},
     # 19
-    {'0': 33, '1': 33, '2': 33, '3': 33,
-     '4': 33, '5': 33, '6': 33, '7': 33,
-     '8': 33, '9': 33, 'A': 33, 'B': 33,
-     'C': 33, 'D': 33, 'E': 33, 'F': 33,
-     'a': 33, 'b': 33, 'c': 33, 'd': 33,
-     'e': 33, 'f': 33},
+    {'0': 34, '1': 34, '2': 34, '3': 34,
+     '4': 34, '5': 34, '6': 34, '7': 34,
+     '8': 34, '9': 34, 'A': 34, 'B': 34,
+     'C': 34, 'D': 34, 'E': 34, 'F': 34,
+     'a': 34, 'b': 34, 'c': 34, 'd': 34,
+     'e': 34, 'f': 34},
     # 20
-    {'0': 34, '1': 34, '2': 34, '3': 34,
-     '4': 34, '5': 34, '6': 34, '7': 34},
+    {'0': 35, '1': 35, '2': 35, '3': 35,
+     '4': 35, '5': 35, '6': 35, '7': 35},
     # 21
-    {'0': 35, '1': 35},
+    {'0': 36, '1': 36},
     # 22
-    {'.': 23, '0': 22, '1': 22, '2': 22,
-     '3': 22, '4': 22, '5': 22, '6': 22,
-     '7': 22, '8': 22, '9': 22, 'E': 24,
-     'J': 13, 'e': 24, 'j': 13},
+    {'.': 24, '0': 22, '1': 23, '2': 23,
+     '3': 23, '4': 23, '5': 23, '6': 23,
+     '7': 23, '8': 23, '9': 23, 'E': 25,
+     'J': 13, 'e': 25, 'j': 13},
     # 23
-    {'0': 23, '1': 23, '2': 23, '3': 23,
-     '4': 23, '5': 23, '6': 23, '7': 23,
-     '8': 23, '9': 23, 'E': 36, 'J': 13,
-     'e': 36, 'j': 13},
+    {'.': 24, '0': 23, '1': 23, '2': 23,
+     '3': 23, '4': 23, '5': 23, '6': 23,
+     '7': 23, '8': 23, '9': 23, 'E': 25,
+     'J': 13, 'e': 25, 'j': 13},
     # 24
-    {'+': 37, '-': 37, '0': 38, '1': 38,
-     '2': 38, '3': 38, '4': 38, '5': 38,
-     '6': 38, '7': 38, '8': 38, '9': 38},
+    {'0': 24, '1': 24, '2': 24, '3': 24,
+     '4': 24, '5': 24, '6': 24, '7': 24,
+     '8': 24, '9': 24, 'E': 37, 'J': 13,
+     'e': 37, 'j': 13},
     # 25
-    {'0': 25, '1': 25, '2': 25, '3': 25,
-     '4': 25, '5': 25, '6': 25, '7': 25,
-     '8': 25, '9': 25, 'E': 36, 'J': 13,
-     'e': 36, 'j': 13},
+    {'+': 38, '-': 38, '0': 39, '1': 39,
+     '2': 39, '3': 39, '4': 39, '5': 39,
+     '6': 39, '7': 39, '8': 39, '9': 39},
     # 26
+    {'0': 26, '1': 26, '2': 26, '3': 26,
+     '4': 26, '5': 26, '6': 26, '7': 26,
+     '8': 26, '9': 26, 'E': 37, 'J': 13,
+     'e': 37, 'j': 13},
+    # 27
     {},
-    # 27
+    # 28
     {"'": 13},
-    # 28
-    {automata.DEFAULT: 39, '\n': 13, '\r': 14},
     # 29
-    {automata.DEFAULT: 29, '\n': 26,
-     '\r': 26, "'": 13, '\\': 28},
+    {automata.DEFAULT: 40, '\n': 13, '\r': 14},
     # 30
+    {automata.DEFAULT: 30, '\n': 27,
+     '\r': 27, "'": 13, '\\': 29},
+    # 31
     {'"': 13},
-    # 31
-    {automata.DEFAULT: 40, '\n': 13, '\r': 14},
     # 32
-    {automata.DEFAULT: 32, '\n': 26,
-     '\r': 26, '"': 13, '\\': 31},
+    {automata.DEFAULT: 41, '\n': 13, '\r': 14},
     # 33
-    {'0': 33, '1': 33, '2': 33, '3': 33,
-     '4': 33, '5': 33, '6': 33, '7': 33,
-     '8': 33, '9': 33, 'A': 33, 'B': 33,
-     'C': 33, 'D': 33, 'E': 33, 'F': 33,
-     'a': 33, 'b': 33, 'c': 33, 'd': 33,
-     'e': 33, 'f': 33},
+    {automata.DEFAULT: 33, '\n': 27,
+     '\r': 27, '"': 13, '\\': 32},
     # 34
     {'0': 34, '1': 34, '2': 34, '3': 34,
-     '4': 34, '5': 34, '6': 34, '7': 34},
+     '4': 34, '5': 34, '6': 34, '7': 34,
+     '8': 34, '9': 34, 'A': 34, 'B': 34,
+     'C': 34, 'D': 34, 'E': 34, 'F': 34,
+     'a': 34, 'b': 34, 'c': 34, 'd': 34,
+     'e': 34, 'f': 34},
     # 35
-    {'0': 35, '1': 35},
+    {'0': 35, '1': 35, '2': 35, '3': 35,
+     '4': 35, '5': 35, '6': 35, '7': 35},
     # 36
-    {'+': 41, '-': 41, '0': 42, '1': 42,
-     '2': 42, '3': 42, '4': 42, '5': 42,
-     '6': 42, '7': 42, '8': 42, '9': 42},
+    {'0': 36, '1': 36},
     # 37
-    {'0': 38, '1': 38, '2': 38, '3': 38,
-     '4': 38, '5': 38, '6': 38, '7': 38,
-     '8': 38, '9': 38},
+    {'+': 42, '-': 42, '0': 43, '1': 43,
+     '2': 43, '3': 43, '4': 43, '5': 43,
+     '6': 43, '7': 43, '8': 43, '9': 43},
     # 38
-    {'0': 38, '1': 38, '2': 38, '3': 38,
-     '4': 38, '5': 38, '6': 38, '7': 38,
-     '8': 38, '9': 38, 'J': 13, 'j': 13},
+    {'0': 39, '1': 39, '2': 39, '3': 39,
+     '4': 39, '5': 39, '6': 39, '7': 39,
+     '8': 39, '9': 39},
     # 39
-    {automata.DEFAULT: 39, '\n': 26,
-     '\r': 26, "'": 13, '\\': 28},
+    {'0': 39, '1': 39, '2': 39, '3': 39,
+     '4': 39, '5': 39, '6': 39, '7': 39,
+     '8': 39, '9': 39, 'J': 13, 'j': 13},
     # 40
-    {automata.DEFAULT: 40, '\n': 26,
-     '\r': 26, '"': 13, '\\': 31},
+    {automata.DEFAULT: 40, '\n': 27,
+     '\r': 27, "'": 13, '\\': 29},
     # 41
-    {'0': 42, '1': 42, '2': 42, '3': 42,
-     '4': 42, '5': 42, '6': 42, '7': 42,
-     '8': 42, '9': 42},
+    {automata.DEFAULT: 41, '\n': 27,
+     '\r': 27, '"': 13, '\\': 32},
     # 42
-    {'0': 42, '1': 42, '2': 42, '3': 42,
-     '4': 42, '5': 42, '6': 42, '7': 42,
-     '8': 42, '9': 42, 'J': 13, 'j': 13},
+    {'0': 43, '1': 43, '2': 43, '3': 43,
+     '4': 43, '5': 43, '6': 43, '7': 43,
+     '8': 43, '9': 43},
+    # 43
+    {'0': 43, '1': 43, '2': 43, '3': 43,
+     '4': 43, '5': 43, '6': 43, '7': 43,
+     '8': 43, '9': 43, 'J': 13, 'j': 13},
     ]
 pseudoDFA = automata.DFA(states, accepts)
 

From noreply at buildbot.pypy.org  Thu Jan 19 12:55:37 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Thu, 19 Jan 2012 12:55:37 +0100 (CET)
Subject: [pypy-commit] pypy py3k: (antocuni,
 romain): use a better way to regenerate the DFA machinery: earlier
 you had to manually copy&paste lines into pytokenize.py,
 now you can just redirect the output to dfa_generated.py
Message-ID: <20120119115537.582EC820D8@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni 
Branch: py3k
Changeset: r51485:1e634f696054
Date: 2012-01-19 12:54 +0100
http://bitbucket.org/pypy/pypy/changeset/1e634f696054/

Log:	(antocuni, romain): use a better way to regenerate the DFA
	machinery: earlier you had to manually copy&paste lines into
	pytokenize.py, now you can just redirect the output to
	dfa_generated.py

diff --git a/pypy/interpreter/pyparser/dfa_generated.py b/pypy/interpreter/pyparser/dfa_generated.py
new file mode 100644
--- /dev/null
+++ b/pypy/interpreter/pyparser/dfa_generated.py
@@ -0,0 +1,287 @@
+# THIS FILE IS AUTOMATICALLY GENERATED BY gendfa.py
+# DO NOT EDIT
+# TO REGENERATE THE FILE, RUN:
+#     python gendfa.py > dfa_generated.py
+
+from pypy.interpreter.pyparser import automata
+accepts = [True, True, True, True, True, True, True, True,
+           True, True, False, True, True, True, True, False,
+           False, False, True, False, False, False, True,
+           False, True, False, True, False, True, False,
+           False, True, False, False, True, True, True,
+           False, False, True, False, False, False, True]
+states = [
+    # 0
+    {'\t': 0, '\n': 13, '\x0c': 0,
+     '\r': 14, ' ': 0, '!': 10, '"': 16,
+     '#': 18, '%': 12, '&': 12, "'": 15,
+     '(': 13, ')': 13, '*': 7, '+': 12,
+     ',': 13, '-': 12, '.': 6, '/': 11,
+     '0': 4, '1': 5, '2': 5, '3': 5,
+     '4': 5, '5': 5, '6': 5, '7': 5,
+     '8': 5, '9': 5, ':': 13, ';': 13,
+     '<': 9, '=': 12, '>': 8, '@': 13,
+     'A': 1, 'B': 2, 'C': 1, 'D': 1,
+     'E': 1, 'F': 1, 'G': 1, 'H': 1,
+     'I': 1, 'J': 1, 'K': 1, 'L': 1,
+     'M': 1, 'N': 1, 'O': 1, 'P': 1,
+     'Q': 1, 'R': 3, 'S': 1, 'T': 1,
+     'U': 1, 'V': 1, 'W': 1, 'X': 1,
+     'Y': 1, 'Z': 1, '[': 13, '\\': 17,
+     ']': 13, '^': 12, '_': 1, '`': 13,
+     'a': 1, 'b': 2, 'c': 1, 'd': 1,
+     'e': 1, 'f': 1, 'g': 1, 'h': 1,
+     'i': 1, 'j': 1, 'k': 1, 'l': 1,
+     'm': 1, 'n': 1, 'o': 1, 'p': 1,
+     'q': 1, 'r': 3, 's': 1, 't': 1,
+     'u': 1, 'v': 1, 'w': 1, 'x': 1,
+     'y': 1, 'z': 1, '{': 13, '|': 12,
+     '}': 13, '~': 13},
+    # 1
+    {'0': 1, '1': 1, '2': 1, '3': 1,
+     '4': 1, '5': 1, '6': 1, '7': 1,
+     '8': 1, '9': 1, 'A': 1, 'B': 1,
+     'C': 1, 'D': 1, 'E': 1, 'F': 1,
+     'G': 1, 'H': 1, 'I': 1, 'J': 1,
+     'K': 1, 'L': 1, 'M': 1, 'N': 1,
+     'O': 1, 'P': 1, 'Q': 1, 'R': 1,
+     'S': 1, 'T': 1, 'U': 1, 'V': 1,
+     'W': 1, 'X': 1, 'Y': 1, 'Z': 1,
+     '_': 1, 'a': 1, 'b': 1, 'c': 1,
+     'd': 1, 'e': 1, 'f': 1, 'g': 1,
+     'h': 1, 'i': 1, 'j': 1, 'k': 1,
+     'l': 1, 'm': 1, 'n': 1, 'o': 1,
+     'p': 1, 'q': 1, 'r': 1, 's': 1,
+     't': 1, 'u': 1, 'v': 1, 'w': 1,
+     'x': 1, 'y': 1, 'z': 1},
+    # 2
+    {'"': 16, "'": 15, '0': 1, '1': 1,
+     '2': 1, '3': 1, '4': 1, '5': 1,
+     '6': 1, '7': 1, '8': 1, '9': 1,
+     'A': 1, 'B': 1, 'C': 1, 'D': 1,
+     'E': 1, 'F': 1, 'G': 1, 'H': 1,
+     'I': 1, 'J': 1, 'K': 1, 'L': 1,
+     'M': 1, 'N': 1, 'O': 1, 'P': 1,
+     'Q': 1, 'R': 3, 'S': 1, 'T': 1,
+     'U': 1, 'V': 1, 'W': 1, 'X': 1,
+     'Y': 1, 'Z': 1, '_': 1, 'a': 1,
+     'b': 1, 'c': 1, 'd': 1, 'e': 1,
+     'f': 1, 'g': 1, 'h': 1, 'i': 1,
+     'j': 1, 'k': 1, 'l': 1, 'm': 1,
+     'n': 1, 'o': 1, 'p': 1, 'q': 1,
+     'r': 3, 's': 1, 't': 1, 'u': 1,
+     'v': 1, 'w': 1, 'x': 1, 'y': 1,
+     'z': 1},
+    # 3
+    {'"': 16, "'": 15, '0': 1, '1': 1,
+     '2': 1, '3': 1, '4': 1, '5': 1,
+     '6': 1, '7': 1, '8': 1, '9': 1,
+     'A': 1, 'B': 1, 'C': 1, 'D': 1,
+     'E': 1, 'F': 1, 'G': 1, 'H': 1,
+     'I': 1, 'J': 1, 'K': 1, 'L': 1,
+     'M': 1, 'N': 1, 'O': 1, 'P': 1,
+     'Q': 1, 'R': 1, 'S': 1, 'T': 1,
+     'U': 1, 'V': 1, 'W': 1, 'X': 1,
+     'Y': 1, 'Z': 1, '_': 1, 'a': 1,
+     'b': 1, 'c': 1, 'd': 1, 'e': 1,
+     'f': 1, 'g': 1, 'h': 1, 'i': 1,
+     'j': 1, 'k': 1, 'l': 1, 'm': 1,
+     'n': 1, 'o': 1, 'p': 1, 'q': 1,
+     'r': 1, 's': 1, 't': 1, 'u': 1,
+     'v': 1, 'w': 1, 'x': 1, 'y': 1,
+     'z': 1},
+    # 4
+    {'.': 24, '0': 22, '1': 23, '2': 23,
+     '3': 23, '4': 23, '5': 23, '6': 23,
+     '7': 23, '8': 23, '9': 23, 'B': 21,
+     'E': 25, 'J': 13, 'O': 20, 'X': 19,
+     'b': 21, 'e': 25, 'j': 13, 'o': 20,
+     'x': 19},
+    # 5
+    {'.': 24, '0': 5, '1': 5, '2': 5,
+     '3': 5, '4': 5, '5': 5, '6': 5,
+     '7': 5, '8': 5, '9': 5, 'E': 25,
+     'J': 13, 'e': 25, 'j': 13},
+    # 6
+    {'0': 26, '1': 26, '2': 26, '3': 26,
+     '4': 26, '5': 26, '6': 26, '7': 26,
+     '8': 26, '9': 26},
+    # 7
+    {'*': 12, '=': 13},
+    # 8
+    {'=': 13, '>': 12},
+    # 9
+    {'<': 12, '=': 13, '>': 13},
+    # 10
+    {'=': 13},
+    # 11
+    {'/': 12, '=': 13},
+    # 12
+    {'=': 13},
+    # 13
+    {},
+    # 14
+    {'\n': 13},
+    # 15
+    {automata.DEFAULT: 30, '\n': 27,
+     '\r': 27, "'": 28, '\\': 29},
+    # 16
+    {automata.DEFAULT: 33, '\n': 27,
+     '\r': 27, '"': 31, '\\': 32},
+    # 17
+    {'\n': 13, '\r': 14},
+    # 18
+    {automata.DEFAULT: 18, '\n': 27, '\r': 27},
+    # 19
+    {'0': 34, '1': 34, '2': 34, '3': 34,
+     '4': 34, '5': 34, '6': 34, '7': 34,
+     '8': 34, '9': 34, 'A': 34, 'B': 34,
+     'C': 34, 'D': 34, 'E': 34, 'F': 34,
+     'a': 34, 'b': 34, 'c': 34, 'd': 34,
+     'e': 34, 'f': 34},
+    # 20
+    {'0': 35, '1': 35, '2': 35, '3': 35,
+     '4': 35, '5': 35, '6': 35, '7': 35},
+    # 21
+    {'0': 36, '1': 36},
+    # 22
+    {'.': 24, '0': 22, '1': 23, '2': 23,
+     '3': 23, '4': 23, '5': 23, '6': 23,
+     '7': 23, '8': 23, '9': 23, 'E': 25,
+     'J': 13, 'e': 25, 'j': 13},
+    # 23
+    {'.': 24, '0': 23, '1': 23, '2': 23,
+     '3': 23, '4': 23, '5': 23, '6': 23,
+     '7': 23, '8': 23, '9': 23, 'E': 25,
+     'J': 13, 'e': 25, 'j': 13},
+    # 24
+    {'0': 24, '1': 24, '2': 24, '3': 24,
+     '4': 24, '5': 24, '6': 24, '7': 24,
+     '8': 24, '9': 24, 'E': 37, 'J': 13,
+     'e': 37, 'j': 13},
+    # 25
+    {'+': 38, '-': 38, '0': 39, '1': 39,
+     '2': 39, '3': 39, '4': 39, '5': 39,
+     '6': 39, '7': 39, '8': 39, '9': 39},
+    # 26
+    {'0': 26, '1': 26, '2': 26, '3': 26,
+     '4': 26, '5': 26, '6': 26, '7': 26,
+     '8': 26, '9': 26, 'E': 37, 'J': 13,
+     'e': 37, 'j': 13},
+    # 27
+    {},
+    # 28
+    {"'": 13},
+    # 29
+    {automata.DEFAULT: 40, '\n': 13, '\r': 14},
+    # 30
+    {automata.DEFAULT: 30, '\n': 27,
+     '\r': 27, "'": 13, '\\': 29},
+    # 31
+    {'"': 13},
+    # 32
+    {automata.DEFAULT: 41, '\n': 13, '\r': 14},
+    # 33
+    {automata.DEFAULT: 33, '\n': 27,
+     '\r': 27, '"': 13, '\\': 32},
+    # 34
+    {'0': 34, '1': 34, '2': 34, '3': 34,
+     '4': 34, '5': 34, '6': 34, '7': 34,
+     '8': 34, '9': 34, 'A': 34, 'B': 34,
+     'C': 34, 'D': 34, 'E': 34, 'F': 34,
+     'a': 34, 'b': 34, 'c': 34, 'd': 34,
+     'e': 34, 'f': 34},
+    # 35
+    {'0': 35, '1': 35, '2': 35, '3': 35,
+     '4': 35, '5': 35, '6': 35, '7': 35},
+    # 36
+    {'0': 36, '1': 36},
+    # 37
+    {'+': 42, '-': 42, '0': 43, '1': 43,
+     '2': 43, '3': 43, '4': 43, '5': 43,
+     '6': 43, '7': 43, '8': 43, '9': 43},
+    # 38
+    {'0': 39, '1': 39, '2': 39, '3': 39,
+     '4': 39, '5': 39, '6': 39, '7': 39,
+     '8': 39, '9': 39},
+    # 39
+    {'0': 39, '1': 39, '2': 39, '3': 39,
+     '4': 39, '5': 39, '6': 39, '7': 39,
+     '8': 39, '9': 39, 'J': 13, 'j': 13},
+    # 40
+    {automata.DEFAULT: 40, '\n': 27,
+     '\r': 27, "'": 13, '\\': 29},
+    # 41
+    {automata.DEFAULT: 41, '\n': 27,
+     '\r': 27, '"': 13, '\\': 32},
+    # 42
+    {'0': 43, '1': 43, '2': 43, '3': 43,
+     '4': 43, '5': 43, '6': 43, '7': 43,
+     '8': 43, '9': 43},
+    # 43
+    {'0': 43, '1': 43, '2': 43, '3': 43,
+     '4': 43, '5': 43, '6': 43, '7': 43,
+     '8': 43, '9': 43, 'J': 13, 'j': 13},
+    ]
+pseudoDFA = automata.DFA(states, accepts)
+
+accepts = [False, False, False, False, False, True]
+states = [
+    # 0
+    {automata.DEFAULT: 0, '"': 1, '\\': 2},
+    # 1
+    {automata.DEFAULT: 4, '"': 3, '\\': 2},
+    # 2
+    {automata.DEFAULT: 4},
+    # 3
+    {automata.DEFAULT: 4, '"': 5, '\\': 2},
+    # 4
+    {automata.DEFAULT: 4, '"': 1, '\\': 2},
+    # 5
+    {automata.DEFAULT: 4, '"': 5, '\\': 2},
+    ]
+double3DFA = automata.NonGreedyDFA(states, accepts)
+
+accepts = [False, False, False, False, False, True]
+states = [
+    # 0
+    {automata.DEFAULT: 0, "'": 1, '\\': 2},
+    # 1
+    {automata.DEFAULT: 4, "'": 3, '\\': 2},
+    # 2
+    {automata.DEFAULT: 4},
+    # 3
+    {automata.DEFAULT: 4, "'": 5, '\\': 2},
+    # 4
+    {automata.DEFAULT: 4, "'": 1, '\\': 2},
+    # 5
+    {automata.DEFAULT: 4, "'": 5, '\\': 2},
+    ]
+single3DFA = automata.NonGreedyDFA(states, accepts)
+
+accepts = [False, True, False, False]
+states = [
+    # 0
+    {automata.DEFAULT: 0, "'": 1, '\\': 2},
+    # 1
+    {},
+    # 2
+    {automata.DEFAULT: 3},
+    # 3
+    {automata.DEFAULT: 3, "'": 1, '\\': 2},
+    ]
+singleDFA = automata.DFA(states, accepts)
+
+accepts = [False, True, False, False]
+states = [
+    # 0
+    {automata.DEFAULT: 0, '"': 1, '\\': 2},
+    # 1
+    {},
+    # 2
+    {automata.DEFAULT: 3},
+    # 3
+    {automata.DEFAULT: 3, '"': 1, '\\': 2},
+    ]
+doubleDFA = automata.DFA(states, accepts)
+
diff --git a/pypy/interpreter/pyparser/genpytokenize.py b/pypy/interpreter/pyparser/gendfa.py
rename from pypy/interpreter/pyparser/genpytokenize.py
rename to pypy/interpreter/pyparser/gendfa.py
--- a/pypy/interpreter/pyparser/genpytokenize.py
+++ b/pypy/interpreter/pyparser/gendfa.py
@@ -1,5 +1,5 @@
 #! /usr/bin/env python
-"""Module genPytokenize
+"""Module gendfa
 
 Generates finite state automata for recognizing Python tokens.  These are hand
 coded versions of the regular expressions originally appearing in Ping's
@@ -7,6 +7,10 @@
 
 When run from the command line, this should pretty print the DFA machinery.
 
+To regenerate the dfa, run::
+
+    $ python gendfa.py > dfa_generated.py
+
 $Id: genPytokenize.py,v 1.1 2003/10/02 17:37:17 jriehl Exp $
 """
 
@@ -305,6 +309,12 @@
     print
 
 def main ():
+    print "# THIS FILE IS AUTOMATICALLY GENERATED BY gendfa.py"
+    print "# DO NOT EDIT"
+    print "# TO REGENERATE THE FILE, RUN:"
+    print "#     python gendfa.py > dfa_generated.py"
+    print
+    print "from pypy.interpreter.pyparser import automata"
     pseudoDFA = makePyPseudoDFA()
     output("pseudoDFA", "DFA", pseudoDFA)
     endDFAMap = makePyEndDFAMap()
diff --git a/pypy/interpreter/pyparser/pytokenize.py b/pypy/interpreter/pyparser/pytokenize.py
--- a/pypy/interpreter/pyparser/pytokenize.py
+++ b/pypy/interpreter/pyparser/pytokenize.py
@@ -17,298 +17,10 @@
 # ______________________________________________________________________
 
 from pypy.interpreter.pyparser import automata
+from pypy.interpreter.pyparser.dfa_generated import *
 
 __all__ = [ "tokenize" ]
 
-# ______________________________________________________________________
-# Automatically generated DFA's
-
-accepts = [True, True, True, True, True, True, True, True,
-           True, True, False, True, True, True, True, False,
-           False, False, True, False, False, False, True,
-           False, True, False, True, False, True, False,
-           False, True, False, False, True, True, True,
-           False, False, True, False, False, False, True]
-states = [
-    # 0
-    {'\t': 0, '\n': 13, '\x0c': 0,
-     '\r': 14, ' ': 0, '!': 10, '"': 16,
-     '#': 18, '%': 12, '&': 12, "'": 15,
-     '(': 13, ')': 13, '*': 7, '+': 12,
-     ',': 13, '-': 12, '.': 6, '/': 11,
-     '0': 4, '1': 5, '2': 5, '3': 5,
-     '4': 5, '5': 5, '6': 5, '7': 5,
-     '8': 5, '9': 5, ':': 13, ';': 13,
-     '<': 9, '=': 12, '>': 8, '@': 13,
-     'A': 1, 'B': 2, 'C': 1, 'D': 1,
-     'E': 1, 'F': 1, 'G': 1, 'H': 1,
-     'I': 1, 'J': 1, 'K': 1, 'L': 1,
-     'M': 1, 'N': 1, 'O': 1, 'P': 1,
-     'Q': 1, 'R': 3, 'S': 1, 'T': 1,
-     'U': 1, 'V': 1, 'W': 1, 'X': 1,
-     'Y': 1, 'Z': 1, '[': 13, '\\': 17,
-     ']': 13, '^': 12, '_': 1, '`': 13,
-     'a': 1, 'b': 2, 'c': 1, 'd': 1,
-     'e': 1, 'f': 1, 'g': 1, 'h': 1,
-     'i': 1, 'j': 1, 'k': 1, 'l': 1,
-     'm': 1, 'n': 1, 'o': 1, 'p': 1,
-     'q': 1, 'r': 3, 's': 1, 't': 1,
-     'u': 1, 'v': 1, 'w': 1, 'x': 1,
-     'y': 1, 'z': 1, '{': 13, '|': 12,
-     '}': 13, '~': 13},
-    # 1
-    {'0': 1, '1': 1, '2': 1, '3': 1,
-     '4': 1, '5': 1, '6': 1, '7': 1,
-     '8': 1, '9': 1, 'A': 1, 'B': 1,
-     'C': 1, 'D': 1, 'E': 1, 'F': 1,
-     'G': 1, 'H': 1, 'I': 1, 'J': 1,
-     'K': 1, 'L': 1, 'M': 1, 'N': 1,
-     'O': 1, 'P': 1, 'Q': 1, 'R': 1,
-     'S': 1, 'T': 1, 'U': 1, 'V': 1,
-     'W': 1, 'X': 1, 'Y': 1, 'Z': 1,
-     '_': 1, 'a': 1, 'b': 1, 'c': 1,
-     'd': 1, 'e': 1, 'f': 1, 'g': 1,
-     'h': 1, 'i': 1, 'j': 1, 'k': 1,
-     'l': 1, 'm': 1, 'n': 1, 'o': 1,
-     'p': 1, 'q': 1, 'r': 1, 's': 1,
-     't': 1, 'u': 1, 'v': 1, 'w': 1,
-     'x': 1, 'y': 1, 'z': 1},
-    # 2
-    {'"': 16, "'": 15, '0': 1, '1': 1,
-     '2': 1, '3': 1, '4': 1, '5': 1,
-     '6': 1, '7': 1, '8': 1, '9': 1,
-     'A': 1, 'B': 1, 'C': 1, 'D': 1,
-     'E': 1, 'F': 1, 'G': 1, 'H': 1,
-     'I': 1, 'J': 1, 'K': 1, 'L': 1,
-     'M': 1, 'N': 1, 'O': 1, 'P': 1,
-     'Q': 1, 'R': 3, 'S': 1, 'T': 1,
-     'U': 1, 'V': 1, 'W': 1, 'X': 1,
-     'Y': 1, 'Z': 1, '_': 1, 'a': 1,
-     'b': 1, 'c': 1, 'd': 1, 'e': 1,
-     'f': 1, 'g': 1, 'h': 1, 'i': 1,
-     'j': 1, 'k': 1, 'l': 1, 'm': 1,
-     'n': 1, 'o': 1, 'p': 1, 'q': 1,
-     'r': 3, 's': 1, 't': 1, 'u': 1,
-     'v': 1, 'w': 1, 'x': 1, 'y': 1,
-     'z': 1},
-    # 3
-    {'"': 16, "'": 15, '0': 1, '1': 1,
-     '2': 1, '3': 1, '4': 1, '5': 1,
-     '6': 1, '7': 1, '8': 1, '9': 1,
-     'A': 1, 'B': 1, 'C': 1, 'D': 1,
-     'E': 1, 'F': 1, 'G': 1, 'H': 1,
-     'I': 1, 'J': 1, 'K': 1, 'L': 1,
-     'M': 1, 'N': 1, 'O': 1, 'P': 1,
-     'Q': 1, 'R': 1, 'S': 1, 'T': 1,
-     'U': 1, 'V': 1, 'W': 1, 'X': 1,
-     'Y': 1, 'Z': 1, '_': 1, 'a': 1,
-     'b': 1, 'c': 1, 'd': 1, 'e': 1,
-     'f': 1, 'g': 1, 'h': 1, 'i': 1,
-     'j': 1, 'k': 1, 'l': 1, 'm': 1,
-     'n': 1, 'o': 1, 'p': 1, 'q': 1,
-     'r': 1, 's': 1, 't': 1, 'u': 1,
-     'v': 1, 'w': 1, 'x': 1, 'y': 1,
-     'z': 1},
-    # 4
-    {'.': 24, '0': 22, '1': 23, '2': 23,
-     '3': 23, '4': 23, '5': 23, '6': 23,
-     '7': 23, '8': 23, '9': 23, 'B': 21,
-     'E': 25, 'J': 13, 'O': 20, 'X': 19,
-     'b': 21, 'e': 25, 'j': 13, 'o': 20,
-     'x': 19},
-    # 5
-    {'.': 24, '0': 5, '1': 5, '2': 5,
-     '3': 5, '4': 5, '5': 5, '6': 5,
-     '7': 5, '8': 5, '9': 5, 'E': 25,
-     'J': 13, 'e': 25, 'j': 13},
-    # 6
-    {'0': 26, '1': 26, '2': 26, '3': 26,
-     '4': 26, '5': 26, '6': 26, '7': 26,
-     '8': 26, '9': 26},
-    # 7
-    {'*': 12, '=': 13},
-    # 8
-    {'=': 13, '>': 12},
-    # 9
-    {'<': 12, '=': 13, '>': 13},
-    # 10
-    {'=': 13},
-    # 11
-    {'/': 12, '=': 13},
-    # 12
-    {'=': 13},
-    # 13
-    {},
-    # 14
-    {'\n': 13},
-    # 15
-    {automata.DEFAULT: 30, '\n': 27,
-     '\r': 27, "'": 28, '\\': 29},
-    # 16
-    {automata.DEFAULT: 33, '\n': 27,
-     '\r': 27, '"': 31, '\\': 32},
-    # 17
-    {'\n': 13, '\r': 14},
-    # 18
-    {automata.DEFAULT: 18, '\n': 27, '\r': 27},
-    # 19
-    {'0': 34, '1': 34, '2': 34, '3': 34,
-     '4': 34, '5': 34, '6': 34, '7': 34,
-     '8': 34, '9': 34, 'A': 34, 'B': 34,
-     'C': 34, 'D': 34, 'E': 34, 'F': 34,
-     'a': 34, 'b': 34, 'c': 34, 'd': 34,
-     'e': 34, 'f': 34},
-    # 20
-    {'0': 35, '1': 35, '2': 35, '3': 35,
-     '4': 35, '5': 35, '6': 35, '7': 35},
-    # 21
-    {'0': 36, '1': 36},
-    # 22
-    {'.': 24, '0': 22, '1': 23, '2': 23,
-     '3': 23, '4': 23, '5': 23, '6': 23,
-     '7': 23, '8': 23, '9': 23, 'E': 25,
-     'J': 13, 'e': 25, 'j': 13},
-    # 23
-    {'.': 24, '0': 23, '1': 23, '2': 23,
-     '3': 23, '4': 23, '5': 23, '6': 23,
-     '7': 23, '8': 23, '9': 23, 'E': 25,
-     'J': 13, 'e': 25, 'j': 13},
-    # 24
-    {'0': 24, '1': 24, '2': 24, '3': 24,
-     '4': 24, '5': 24, '6': 24, '7': 24,
-     '8': 24, '9': 24, 'E': 37, 'J': 13,
-     'e': 37, 'j': 13},
-    # 25
-    {'+': 38, '-': 38, '0': 39, '1': 39,
-     '2': 39, '3': 39, '4': 39, '5': 39,
-     '6': 39, '7': 39, '8': 39, '9': 39},
-    # 26
-    {'0': 26, '1': 26, '2': 26, '3': 26,
-     '4': 26, '5': 26, '6': 26, '7': 26,
-     '8': 26, '9': 26, 'E': 37, 'J': 13,
-     'e': 37, 'j': 13},
-    # 27
-    {},
-    # 28
-    {"'": 13},
-    # 29
-    {automata.DEFAULT: 40, '\n': 13, '\r': 14},
-    # 30
-    {automata.DEFAULT: 30, '\n': 27,
-     '\r': 27, "'": 13, '\\': 29},
-    # 31
-    {'"': 13},
-    # 32
-    {automata.DEFAULT: 41, '\n': 13, '\r': 14},
-    # 33
-    {automata.DEFAULT: 33, '\n': 27,
-     '\r': 27, '"': 13, '\\': 32},
-    # 34
-    {'0': 34, '1': 34, '2': 34, '3': 34,
-     '4': 34, '5': 34, '6': 34, '7': 34,
-     '8': 34, '9': 34, 'A': 34, 'B': 34,
-     'C': 34, 'D': 34, 'E': 34, 'F': 34,
-     'a': 34, 'b': 34, 'c': 34, 'd': 34,
-     'e': 34, 'f': 34},
-    # 35
-    {'0': 35, '1': 35, '2': 35, '3': 35,
-     '4': 35, '5': 35, '6': 35, '7': 35},
-    # 36
-    {'0': 36, '1': 36},
-    # 37
-    {'+': 42, '-': 42, '0': 43, '1': 43,
-     '2': 43, '3': 43, '4': 43, '5': 43,
-     '6': 43, '7': 43, '8': 43, '9': 43},
-    # 38
-    {'0': 39, '1': 39, '2': 39, '3': 39,
-     '4': 39, '5': 39, '6': 39, '7': 39,
-     '8': 39, '9': 39},
-    # 39
-    {'0': 39, '1': 39, '2': 39, '3': 39,
-     '4': 39, '5': 39, '6': 39, '7': 39,
-     '8': 39, '9': 39, 'J': 13, 'j': 13},
-    # 40
-    {automata.DEFAULT: 40, '\n': 27,
-     '\r': 27, "'": 13, '\\': 29},
-    # 41
-    {automata.DEFAULT: 41, '\n': 27,
-     '\r': 27, '"': 13, '\\': 32},
-    # 42
-    {'0': 43, '1': 43, '2': 43, '3': 43,
-     '4': 43, '5': 43, '6': 43, '7': 43,
-     '8': 43, '9': 43},
-    # 43
-    {'0': 43, '1': 43, '2': 43, '3': 43,
-     '4': 43, '5': 43, '6': 43, '7': 43,
-     '8': 43, '9': 43, 'J': 13, 'j': 13},
-    ]
-pseudoDFA = automata.DFA(states, accepts)
-
-accepts = [False, False, False, False, False, True]
-states = [
-    # 0
-    {automata.DEFAULT: 0, '"': 1, '\\': 2},
-    # 1
-    {automata.DEFAULT: 4, '"': 3, '\\': 2},
-    # 2
-    {automata.DEFAULT: 4},
-    # 3
-    {automata.DEFAULT: 4, '"': 5, '\\': 2},
-    # 4
-    {automata.DEFAULT: 4, '"': 1, '\\': 2},
-    # 5
-    {automata.DEFAULT: 4, '"': 5, '\\': 2},
-    ]
-double3DFA = automata.NonGreedyDFA(states, accepts)
-
-accepts = [False, False, False, False, False, True]
-states = [
-    # 0
-    {automata.DEFAULT: 0, "'": 1, '\\': 2},
-    # 1
-    {automata.DEFAULT: 4, "'": 3, '\\': 2},
-    # 2
-    {automata.DEFAULT: 4},
-    # 3
-    {automata.DEFAULT: 4, "'": 5, '\\': 2},
-    # 4
-    {automata.DEFAULT: 4, "'": 1, '\\': 2},
-    # 5
-    {automata.DEFAULT: 4, "'": 5, '\\': 2},
-    ]
-single3DFA = automata.NonGreedyDFA(states, accepts)
-
-accepts = [False, True, False, False]
-states = [
-    # 0
-    {automata.DEFAULT: 0, "'": 1, '\\': 2},
-    # 1
-    {},
-    # 2
-    {automata.DEFAULT: 3},
-    # 3
-    {automata.DEFAULT: 3, "'": 1, '\\': 2},
-    ]
-singleDFA = automata.DFA(states, accepts)
-
-accepts = [False, True, False, False]
-states = [
-    # 0
-    {automata.DEFAULT: 0, '"': 1, '\\': 2},
-    # 1
-    {},
-    # 2
-    {automata.DEFAULT: 3},
-    # 3
-    {automata.DEFAULT: 3, '"': 1, '\\': 2},
-    ]
-doubleDFA = automata.DFA(states, accepts)
-
-
-
-#_______________________________________________________________________
-# End of automatically generated DFA's
-
 endDFAs = {"'" : singleDFA,
            '"' : doubleDFA,
            'r' : None,

From noreply at buildbot.pypy.org  Thu Jan 19 15:50:37 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Thu, 19 Jan 2012 15:50:37 +0100 (CET)
Subject: [pypy-commit] pypy py3k: (antocuni,
 romain) make this test at least runnable by using a py3k compatible
 syntax. Tons of failures, of course
Message-ID: <20120119145037.934CD820D8@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni 
Branch: py3k
Changeset: r51486:155402d74dfc
Date: 2012-01-19 15:50 +0100
http://bitbucket.org/pypy/pypy/changeset/155402d74dfc/

Log:	(antocuni, romain) make this test at least runnable by using a py3k
	compatible syntax. Tons of failures, of course

diff --git a/pypy/module/marshal/test/test_marshal.py b/pypy/module/marshal/test/test_marshal.py
--- a/pypy/module/marshal/test/test_marshal.py
+++ b/pypy/module/marshal/test/test_marshal.py
@@ -7,12 +7,13 @@
         cls.w_tmpfile = cls.space.wrap(str(tmpfile))
 
     def w_marshal_check(self, case):
-        import marshal, StringIO
+        import marshal
+        from io import StringIO
         s = marshal.dumps(case)
-        print repr(s)
+        print(repr(s))
         x = marshal.loads(s)
         assert x == case and type(x) is type(case)
-        f = StringIO.StringIO()
+        f = StringIO()
         marshal.dump(case, f)
         f.seek(0)
         x = marshal.load(f)

From noreply at buildbot.pypy.org  Thu Jan 19 16:15:05 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 19 Jan 2012 16:15:05 +0100 (CET)
Subject: [pypy-commit] pypy stm: (bivab, arigo)
Message-ID: <20120119151505.B4F61820D8@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stm
Changeset: r51487:b52f0e138410
Date: 2012-01-19 16:14 +0100
http://bitbucket.org/pypy/pypy/changeset/b52f0e138410/

Log:	(bivab, arigo)

	Wrote the app-level interface for 'transaction', with careful(?)
	messy locking.

diff --git a/pypy/module/transaction/__init__.py b/pypy/module/transaction/__init__.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/transaction/__init__.py
@@ -0,0 +1,20 @@
+
+from pypy.interpreter.mixedmodule import MixedModule 
+
+class Module(MixedModule):
+    """Transaction module. XXX document me
+    """
+
+    interpleveldefs = {
+        'set_num_threads': 'interp_transaction.set_num_threads',
+        'add': 'interp_transaction.add',
+        'run': 'interp_transaction.run',
+        'TransactionError': 'interp_transaction.state.w_error',
+    }
+
+    appleveldefs = {
+    }
+
+    def startup(self, space):
+        from pypy.module.transaction import interp_transaction
+        interp_transaction.state.startup(space)
diff --git a/pypy/module/transaction/interp_transaction.py b/pypy/module/transaction/interp_transaction.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/transaction/interp_transaction.py
@@ -0,0 +1,138 @@
+from pypy.interpreter.error import OperationError
+from pypy.interpreter.gateway import unwrap_spec
+from pypy.module.transaction import threadintf
+
+
+NUM_THREADS_DEFAULT = 4     # by default
+
+
+class State(object):
+
+    def _freeze_(self):
+        self.__dict__.clear()
+        self.running = False
+        self.num_threads = NUM_THREADS_DEFAULT
+
+    def startup(self, space):
+        self.space = space
+        self.pending = []
+        self.ll_lock = threadintf.allocate_lock()
+        self.ll_no_tasks_pending_lock = threadintf.allocate_lock()
+        self.ll_unfinished_lock = threadintf.allocate_lock()
+        self.w_error = space.new_exception_class(
+            "transaction.TransactionError")
+        self.lock_no_tasks_pending()
+        self.lock_unfinished()
+
+    def set_num_threads(self, num):
+        if self.running:
+            space = self.space
+            raise OperationError(space.w_ValueError,
+                                 space.wrap("cannot change the number of "
+                                            "threads when transaction.run() "
+                                            "is active"))
+        self.num_threads = num_threads
+
+    def lock(self):
+        # XXX think about the interaction between locks and the GC
+        threadintf.acquire(self.ll_lock, True)
+
+    def unlock(self):
+        threadintf.release(self.ll_lock)
+
+    def lock_no_tasks_pending(self):
+        threadintf.acquire(self.ll_no_tasks_pending_lock, True)
+
+    def unlock_no_tasks_pending(self):
+        threadintf.release(self.ll_no_tasks_pending_lock)
+
+    def assert_locked_no_tasks_pending(self):
+        just_locked = threadintf.acquire(self.ll_no_tasks_pending_lock, False)
+        assert not just_locked
+
+    def lock_unfinished(self):
+        threadintf.acquire(self.ll_unfinished_lock, True)
+
+    def unlock_unfinished(self):
+        threadintf.release(self.ll_unfinished_lock)
+
+
+state = State()
+state._freeze_()
+
+
+ at unwrap_spec(num=int)
+def set_num_threads(space, num):
+    if num < 1:
+        num = 1
+    state.set_num_threads(num)
+
+
+class Pending:
+    def __init__(self, w_callback, args):
+        self.w_callback = w_callback
+        self.args = args
+
+    def run(self):
+        space = state.space
+        space.call_args(self.w_callback, self.args)
+        # xxx exceptions?
+
+
+def add(space, w_callback, __args__):
+    state.lock()
+    was_empty = len(state.pending) == 0
+    state.pending.append(Pending(w_callback, __args__))
+    if was_empty:
+        state.unlock_no_tasks_pending()
+    state.unlock()
+
+
+def _run_thread():
+    state.lock()
+    #
+    while True:
+        if len(state.pending) == 0:
+            state.assert_locked_no_tasks_pending()
+            state.num_waiting_threads += 1
+            if state.num_waiting_threads == state.num_threads:
+                state.finished = True
+                state.unlock_unfinished()
+                state.unlock_no_tasks_pending()
+            state.unlock()
+            #
+            state.lock_no_tasks_pending()
+            state.unlock_no_tasks_pending()
+            #
+            state.lock()
+            if state.finished:
+                break
+            state.num_waiting_threads -= 1
+        else:
+            pending = state.pending.pop(0)
+            if len(state.pending) == 0:
+                state.lock_no_tasks_pending()
+            state.unlock()
+            pending.run()
+            state.lock()
+    #
+    state.unlock()
+
+
+def run(space):
+    if state.running:
+        raise OperationError(
+            state.w_error,
+            space.wrap("recursive invocation of transaction.run()"))
+    state.num_waiting_threads = 0
+    state.finished = False
+    state.running = True
+    #
+    for i in range(state.num_threads):
+        threadintf.start_new_thread(_run_thread, ())
+    #
+    state.lock_unfinished()
+    assert state.num_waiting_threads == state.num_threads
+    assert len(state.pending) == 0
+    state.lock_no_tasks_pending()
+    state.running = False
diff --git a/pypy/module/transaction/test/__init__.py b/pypy/module/transaction/test/__init__.py
new file mode 100644
diff --git a/pypy/module/transaction/test/test_interp_transaction.py b/pypy/module/transaction/test/test_interp_transaction.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/transaction/test/test_interp_transaction.py
@@ -0,0 +1,24 @@
+from pypy.module.transaction import interp_transaction
+
+
+class FakeSpace:
+    def new_exception_class(self, name):
+        return "some error class"
+    def call_args(self, w_callback, args):
+        w_callback(*args)
+
+
+def test_linear_list():
+    space = FakeSpace()
+    interp_transaction.state.startup(space)
+    seen = []
+    #
+    def do(n):
+        seen.append(n)
+        if n < 200:
+            interp_transaction.add(space, do, (n+1,))
+    #
+    interp_transaction.add(space, do, (0,))
+    assert seen == []
+    interp_transaction.run(space)
+    assert seen == range(201)
diff --git a/pypy/module/transaction/test/test_transaction.py b/pypy/module/transaction/test/test_transaction.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/transaction/test/test_transaction.py
@@ -0,0 +1,16 @@
+import py
+from pypy.conftest import gettestobjspace
+
+
+class AppTestTransaction: 
+    def setup_class(cls):
+        cls.space = gettestobjspace(usemodules=['transaction'])
+
+    def test_simple(self):
+        import transaction
+        lst = []
+        transaction.add(lst.append, 5)
+        transaction.add(lst.append, 6)
+        transaction.add(lst.append, 7)
+        transaction.run()
+        assert sorted(lst) == [5, 6, 7]
diff --git a/pypy/module/transaction/threadintf.py b/pypy/module/transaction/threadintf.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/transaction/threadintf.py
@@ -0,0 +1,19 @@
+import thread
+
+
+def allocate_lock():
+    "NOT_RPYTHON"
+    return thread.allocate_lock()
+
+def acquire(lock, wait):
+    "NOT_RPYTHON"
+    lock.acquire(wait)
+
+def release(lock):
+    "NOT_RPYTHON"
+    lock.release()
+
+def start_new_thread(callback, args):
+    "NOT_RPYTHON"
+    thread.start_new_thread(callback, args)
+

From noreply at buildbot.pypy.org  Thu Jan 19 17:17:22 2012
From: noreply at buildbot.pypy.org (rguillebert)
Date: Thu, 19 Jan 2012 17:17:22 +0100 (CET)
Subject: [pypy-commit] pypy py3k: (antocuni,
 rguillebert) Add a marshalling test of pycode objects
Message-ID: <20120119161722.7E228820D8@wyvern.cs.uni-duesseldorf.de>

Author: Romain Guillebert 
Branch: py3k
Changeset: r51488:176a25c48305
Date: 2012-01-19 17:17 +0100
http://bitbucket.org/pypy/pypy/changeset/176a25c48305/

Log:	(antocuni, rguillebert) Add a marshalling test of pycode objects

diff --git a/pypy/module/marshal/test/test_marshalimpl.py b/pypy/module/marshal/test/test_marshalimpl.py
--- a/pypy/module/marshal/test/test_marshalimpl.py
+++ b/pypy/module/marshal/test/test_marshalimpl.py
@@ -55,6 +55,17 @@
         z = marshal.loads('I\x00\x1c\xf4\xab\xfd\xff\xff\xff')
         assert z == -10000000000
 
+    def test_marshal_code_object(self):
+        def foo(a, b):
+            pass
+
+        import marshal
+        s = marshal.dumps(foo.__code__)
+        code2 = marshal.loads(s)
+        for attr_name in dir(code2):
+            if attr_name.startswith("co_"):
+                assert getattr(code2, attr_name) == getattr(foo.__code__, attr_name)
+
 
 class AppTestMarshalSmallLong(AppTestMarshalMore):
     def setup_class(cls):

From noreply at buildbot.pypy.org  Thu Jan 19 17:45:00 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 19 Jan 2012 17:45:00 +0100 (CET)
Subject: [pypy-commit] pypy numppy-flatitter: just reuse what we have (evil
	laugh)
Message-ID: <20120119164500.6A835820D8@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: numppy-flatitter
Changeset: r51489:e7ccfba90d71
Date: 2012-01-19 18:44 +0200
http://bitbucket.org/pypy/pypy/changeset/e7ccfba90d71/

Log:	just reuse what we have (evil laugh)

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -1310,6 +1310,7 @@
         self.shapelen = len(arr.shape)
         self.iter = OneDimIterator(arr.start, self.strides[0],
                                    self.shape[0])
+        self.start = arr.start
         self.base = arr
 
     def descr_next(self, space):
@@ -1322,42 +1323,11 @@
     def descr_iter(self):
         return self
 
-    def descr_getitem(self, space, w_idx):
-        if not space.isinstance_w(w_idx, space.w_int):
-            raise OperationError(space.w_ValueError, space.wrap(
-                        "non-integer indexing not supported yet"))
-        _i = space.int_w(w_idx)
-        if _i<0:
-            i = self.size + _i
-        else:
-            i = _i
-        if i >= self.size or i < 0:
-            raise operationerrfmt(space.w_IndexError,
-                            "index (%d) out of range (%d<=index<%d", 
-                                _i, -self.size, self.size)
-        result = self.getitem(self.base.start + i * self.strides[0])
-        return result
-
-    def descr_setitem(self, space, w_idx, w_value):
-        if not space.isinstance_w(w_idx, space.w_int):
-            raise OperationError(space.w_ValueError, space.wrap(
-                        "non-integer indexing not supported yet"))
-        _i = space.int_w(w_idx)
-        if _i<0:
-            i = self.size + _i
-        else:
-            i = _i
-        if i >= self.size or i < 0:
-            raise operationerrfmt(space.w_IndexError,
-                            "index (%d) out of range (%d<=index<%d", 
-                                _i, -self.size, self.size)
-        self.setitem(self.base.start + i * self.strides[0], w_value)
-
 W_FlatIterator.typedef = TypeDef(
     'flatiter',
     next = interp2app(W_FlatIterator.descr_next),
     __iter__ = interp2app(W_FlatIterator.descr_iter),
-    __getitem__ = interp2app(W_FlatIterator.descr_getitem),
-    __setitem__ = interp2app(W_FlatIterator.descr_setitem),
+    __getitem__ = interp2app(BaseArray.descr_getitem),
+    __setitem__ = interp2app(BaseArray.descr_setitem),
 )
 W_FlatIterator.acceptable_as_base_class = False

From noreply at buildbot.pypy.org  Thu Jan 19 18:00:14 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 19 Jan 2012 18:00:14 +0100 (CET)
Subject: [pypy-commit] pypy stm: (bivab, arigo)
Message-ID: <20120119170014.E6C34820D8@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stm
Changeset: r51490:601a7695b552
Date: 2012-01-19 16:41 +0100
http://bitbucket.org/pypy/pypy/changeset/601a7695b552/

Log:	(bivab, arigo)

	A failing test.

diff --git a/pypy/module/transaction/interp_transaction.py b/pypy/module/transaction/interp_transaction.py
--- a/pypy/module/transaction/interp_transaction.py
+++ b/pypy/module/transaction/interp_transaction.py
@@ -97,7 +97,6 @@
             state.num_waiting_threads += 1
             if state.num_waiting_threads == state.num_threads:
                 state.finished = True
-                state.unlock_unfinished()
                 state.unlock_no_tasks_pending()
             state.unlock()
             #
@@ -105,9 +104,9 @@
             state.unlock_no_tasks_pending()
             #
             state.lock()
+            state.num_waiting_threads -= 1
             if state.finished:
                 break
-            state.num_waiting_threads -= 1
         else:
             pending = state.pending.pop(0)
             if len(state.pending) == 0:
@@ -116,6 +115,8 @@
             pending.run()
             state.lock()
     #
+    if state.num_waiting_threads == 0:    # only the last thread to leave
+        state.unlock_unfinished()
     state.unlock()
 
 
@@ -132,7 +133,7 @@
         threadintf.start_new_thread(_run_thread, ())
     #
     state.lock_unfinished()
-    assert state.num_waiting_threads == state.num_threads
+    assert state.num_waiting_threads == 0
     assert len(state.pending) == 0
     state.lock_no_tasks_pending()
     state.running = False
diff --git a/pypy/module/transaction/test/test_interp_transaction.py b/pypy/module/transaction/test/test_interp_transaction.py
--- a/pypy/module/transaction/test/test_interp_transaction.py
+++ b/pypy/module/transaction/test/test_interp_transaction.py
@@ -1,3 +1,4 @@
+import time
 from pypy.module.transaction import interp_transaction
 
 
@@ -22,3 +23,49 @@
     assert seen == []
     interp_transaction.run(space)
     assert seen == range(201)
+
+
+def test_tree_of_transactions():
+    space = FakeSpace()
+    interp_transaction.state.startup(space)
+    seen = []
+    #
+    def do(level):
+        seen.append(level)
+        if level < 11:
+            interp_transaction.add(space, do, (level+1,))
+            interp_transaction.add(space, do, (level+1,))
+    #
+    interp_transaction.add(space, do, (0,))
+    assert seen == []
+    interp_transaction.run(space)
+    for i in range(12):
+        assert seen.count(i) == 2 ** i
+    assert len(seen) == 2 ** 12 - 1
+
+
+def test_transactional_simple():
+    space = FakeSpace()
+    interp_transaction.state.startup(space)
+    lst = []
+    def f(n):
+        lst.append(n+0)
+        lst.append(n+1)
+        time.sleep(0.05)
+        lst.append(n+2)
+        lst.append(n+3)
+        lst.append(n+4)
+        time.sleep(0.25)
+        lst.append(n+5)
+        lst.append(n+6)
+    interp_transaction.add(space, f, (10,))
+    interp_transaction.add(space, f, (20,))
+    interp_transaction.add(space, f, (30,))
+    interp_transaction.run(space)
+    assert len(lst) == 7 * 3
+    seen = set()
+    for start in range(0, 21, 7):
+        seen.add(lst[start])
+        for index in range(7):
+            assert lst[start + index] == lst[start] + index
+    assert seen == set([10, 20, 30])
diff --git a/pypy/module/transaction/test/test_transaction.py b/pypy/module/transaction/test/test_transaction.py
--- a/pypy/module/transaction/test/test_transaction.py
+++ b/pypy/module/transaction/test/test_transaction.py
@@ -4,6 +4,7 @@
 
 class AppTestTransaction: 
     def setup_class(cls):
+        py.test.skip("XXX not transactional!")
         cls.space = gettestobjspace(usemodules=['transaction'])
 
     def test_simple(self):
@@ -14,3 +15,26 @@
         transaction.add(lst.append, 7)
         transaction.run()
         assert sorted(lst) == [5, 6, 7]
+
+    def test_almost_as_simple(self):
+        import transaction
+        lst = []
+        def f(n):
+            lst.append(n+0)
+            lst.append(n+1)
+            lst.append(n+2)
+            lst.append(n+3)
+            lst.append(n+4)
+            lst.append(n+5)
+            lst.append(n+6)
+        transaction.add(f, 10)
+        transaction.add(f, 20)
+        transaction.add(f, 30)
+        transaction.run()
+        assert len(lst) == 7 * 3
+        seen = set()
+        for start in range(0, 21, 7):
+            seen.append(lst[start])
+            for index in range(7):
+                assert lst[start + index] == lst[start] + index
+        assert seen == set([10, 20, 30])

From noreply at buildbot.pypy.org  Thu Jan 19 18:00:16 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 19 Jan 2012 18:00:16 +0100 (CET)
Subject: [pypy-commit] pypy stm: (bivab, arigo)
Message-ID: <20120119170016.27211820D8@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stm
Changeset: r51491:fd78c427e13b
Date: 2012-01-19 17:51 +0100
http://bitbucket.org/pypy/pypy/changeset/fd78c427e13b/

Log:	(bivab, arigo)

	Fixed rlib.rstm to run correctly on top of CPython with multiple
	threads.

diff --git a/pypy/rlib/rstm.py b/pypy/rlib/rstm.py
--- a/pypy/rlib/rstm.py
+++ b/pypy/rlib/rstm.py
@@ -1,3 +1,4 @@
+import thread
 from pypy.rlib.objectmodel import specialize, we_are_translated, keepalive_until_here
 from pypy.rpython.lltypesystem import rffi, lltype, rclass
 from pypy.rpython.annlowlevel import (cast_base_ptr_to_instance,
@@ -5,6 +6,8 @@
                                       llhelper)
 from pypy.translator.stm import _rffi_stm
 
+_global_lock = thread.allocate_lock()
+
 @specialize.memo()
 def _get_stm_callback(func, argcls):
     def _stm_callback(llarg):
@@ -26,14 +29,29 @@
         llarg = cast_instance_to_base_ptr(arg)
         llarg = rffi.cast(rffi.VOIDP, llarg)
     else:
-        # only for tests
+        # only for tests: we want (1) to test the calls to the C library,
+        # but also (2) to work with multiple Python threads, so we acquire
+        # and release some custom GIL here --- even though it doesn't make
+        # sense from an STM point of view :-/
+        _global_lock.acquire()
         lltype.TLS.stm_callback_arg = arg
         llarg = lltype.nullptr(rffi.VOIDP.TO)
     callback = _get_stm_callback(func, argcls)
     llcallback = llhelper(_rffi_stm.CALLBACK, callback)
     _rffi_stm.stm_perform_transaction(llcallback, llarg)
     keepalive_until_here(arg)
+    if not we_are_translated():
+        _global_lock.release()
 
-descriptor_init = _rffi_stm.stm_descriptor_init
-descriptor_done = _rffi_stm.stm_descriptor_done
-debug_get_state = _rffi_stm.stm_debug_get_state
+def descriptor_init():
+    if not we_are_translated(): _global_lock.acquire()
+    _rffi_stm.stm_descriptor_init()
+    if not we_are_translated(): _global_lock.release()
+
+def descriptor_done():
+    if not we_are_translated(): _global_lock.acquire()
+    _rffi_stm.stm_descriptor_done()
+    if not we_are_translated(): _global_lock.release()
+
+def debug_get_state():
+    return _rffi_stm.stm_debug_get_state()
diff --git a/pypy/rlib/test/test_rstm.py b/pypy/rlib/test/test_rstm.py
--- a/pypy/rlib/test/test_rstm.py
+++ b/pypy/rlib/test/test_rstm.py
@@ -1,4 +1,4 @@
-import os
+import os, thread, time
 from pypy.rlib.debug import debug_print
 from pypy.rlib import rstm
 from pypy.translator.stm.test.support import CompiledSTMTests
@@ -29,6 +29,20 @@
     assert rstm.debug_get_state() == -1
     assert arg.x == 42
 
+def test_stm_multiple_threads():
+    ok = []
+    def f(i):
+        test_stm_perform_transaction()
+        ok.append(i)
+    for i in range(10):
+        thread.start_new_thread(f, (i,))
+    timeout = 10
+    while len(ok) < 10:
+        time.sleep(0.1)
+        timeout -= 0.1
+        assert timeout >= 0.0, "timeout!"
+    assert sorted(ok) == range(10)
+
 
 class TestTransformSingleThread(CompiledSTMTests):
 

From noreply at buildbot.pypy.org  Thu Jan 19 18:00:17 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 19 Jan 2012 18:00:17 +0100 (CET)
Subject: [pypy-commit] pypy stm: (bivab, arigo)
Message-ID: <20120119170017.5B2B0820D8@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stm
Changeset: r51492:bc50696a4294
Date: 2012-01-19 17:59 +0100
http://bitbucket.org/pypy/pypy/changeset/bc50696a4294/

Log:	(bivab, arigo)

	- really create a low-level transaction for running each transaction
	- changed the addition of new transactions to go into a per-thread
	list until the transaction really commits
	- enable some more tests

diff --git a/pypy/module/transaction/interp_transaction.py b/pypy/module/transaction/interp_transaction.py
--- a/pypy/module/transaction/interp_transaction.py
+++ b/pypy/module/transaction/interp_transaction.py
@@ -1,6 +1,7 @@
 from pypy.interpreter.error import OperationError
 from pypy.interpreter.gateway import unwrap_spec
 from pypy.module.transaction import threadintf
+from pypy.rlib import rstm
 
 
 NUM_THREADS_DEFAULT = 4     # by default
@@ -21,8 +22,9 @@
         self.ll_unfinished_lock = threadintf.allocate_lock()
         self.w_error = space.new_exception_class(
             "transaction.TransactionError")
-        self.lock_no_tasks_pending()
         self.lock_unfinished()
+        self.main_thread_id = threadintf.thread_id()
+        self.pending_lists = {self.main_thread_id: self.pending}
 
     def set_num_threads(self, num):
         if self.running:
@@ -46,9 +48,11 @@
     def unlock_no_tasks_pending(self):
         threadintf.release(self.ll_no_tasks_pending_lock)
 
-    def assert_locked_no_tasks_pending(self):
+    def is_locked_no_tasks_pending(self):
         just_locked = threadintf.acquire(self.ll_no_tasks_pending_lock, False)
-        assert not just_locked
+        if just_locked:
+            threadintf.release(self.ll_no_tasks_pending_lock)
+        return not just_locked
 
     def lock_unfinished(self):
         threadintf.acquire(self.ll_unfinished_lock, True)
@@ -69,31 +73,47 @@
 
 
 class Pending:
+    _alloc_nonmovable_ = True
+
     def __init__(self, w_callback, args):
         self.w_callback = w_callback
         self.args = args
 
     def run(self):
+        rstm.perform_transaction(Pending._run_in_transaction, Pending, self)
+
+    @staticmethod
+    def _run_in_transaction(pending):
         space = state.space
-        space.call_args(self.w_callback, self.args)
+        space.call_args(pending.w_callback, pending.args)
         # xxx exceptions?
 
 
 def add(space, w_callback, __args__):
-    state.lock()
+    id = threadintf.thread_id()
+    state.pending_lists[id].append(Pending(w_callback, __args__))
+
+
+def add_list(new_pending_list):
+    if len(new_pending_list) == 0:
+        return
     was_empty = len(state.pending) == 0
-    state.pending.append(Pending(w_callback, __args__))
+    state.pending += new_pending_list
+    del new_pending_list[:]
     if was_empty:
         state.unlock_no_tasks_pending()
-    state.unlock()
 
 
 def _run_thread():
     state.lock()
+    my_pending_list = []
+    my_thread_id = threadintf.thread_id()
+    state.pending_lists[my_thread_id] = my_pending_list
+    rstm.descriptor_init()
     #
     while True:
         if len(state.pending) == 0:
-            state.assert_locked_no_tasks_pending()
+            assert state.is_locked_no_tasks_pending()
             state.num_waiting_threads += 1
             if state.num_waiting_threads == state.num_threads:
                 state.finished = True
@@ -114,7 +134,10 @@
             state.unlock()
             pending.run()
             state.lock()
+            add_list(my_pending_list)
     #
+    rstm.descriptor_done()
+    del state.pending_lists[my_thread_id]
     if state.num_waiting_threads == 0:    # only the last thread to leave
         state.unlock_unfinished()
     state.unlock()
@@ -125,6 +148,9 @@
         raise OperationError(
             state.w_error,
             space.wrap("recursive invocation of transaction.run()"))
+    assert not state.is_locked_no_tasks_pending()
+    if len(state.pending) == 0:
+        return
     state.num_waiting_threads = 0
     state.finished = False
     state.running = True
@@ -132,8 +158,10 @@
     for i in range(state.num_threads):
         threadintf.start_new_thread(_run_thread, ())
     #
-    state.lock_unfinished()
+    state.lock_unfinished()  # wait for all threads to finish
+    #
     assert state.num_waiting_threads == 0
     assert len(state.pending) == 0
-    state.lock_no_tasks_pending()
+    assert state.pending_lists.keys() == [state.main_thread_id]
+    assert not state.is_locked_no_tasks_pending()
     state.running = False
diff --git a/pypy/module/transaction/test/test_transaction.py b/pypy/module/transaction/test/test_transaction.py
--- a/pypy/module/transaction/test/test_transaction.py
+++ b/pypy/module/transaction/test/test_transaction.py
@@ -4,7 +4,6 @@
 
 class AppTestTransaction: 
     def setup_class(cls):
-        py.test.skip("XXX not transactional!")
         cls.space = gettestobjspace(usemodules=['transaction'])
 
     def test_simple(self):
@@ -34,7 +33,7 @@
         assert len(lst) == 7 * 3
         seen = set()
         for start in range(0, 21, 7):
-            seen.append(lst[start])
+            seen.add(lst[start])
             for index in range(7):
                 assert lst[start + index] == lst[start] + index
         assert seen == set([10, 20, 30])
diff --git a/pypy/module/transaction/threadintf.py b/pypy/module/transaction/threadintf.py
--- a/pypy/module/transaction/threadintf.py
+++ b/pypy/module/transaction/threadintf.py
@@ -7,7 +7,7 @@
 
 def acquire(lock, wait):
     "NOT_RPYTHON"
-    lock.acquire(wait)
+    return lock.acquire(wait)
 
 def release(lock):
     "NOT_RPYTHON"
@@ -17,3 +17,6 @@
     "NOT_RPYTHON"
     thread.start_new_thread(callback, args)
 
+def thread_id():
+    "NOT_RPYTHON"
+    return thread.get_ident()

From noreply at buildbot.pypy.org  Thu Jan 19 18:08:54 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 19 Jan 2012 18:08:54 +0100 (CET)
Subject: [pypy-commit] pypy numppy-flatitter: an example of failing test
Message-ID: <20120119170854.BEEEE820D8@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: numppy-flatitter
Changeset: r51493:d0f36182d3eb
Date: 2012-01-19 19:08 +0200
http://bitbucket.org/pypy/pypy/changeset/d0f36182d3eb/

Log:	an example of failing test

diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1309,6 +1309,11 @@
         raises(IndexError, "b[11]")
         raises(IndexError, "b[-11]")
 
+    def test_flatiter_view(self):
+        from _numpypy import arange
+        a = arange(10).reshape(5, 2)
+        assert (a[::2].flat == [0, 1, 4, 5, 8, 9])
+
     def test_flatiter_transpose(self):
         from _numpypy import arange
         a = arange(10)

From noreply at buildbot.pypy.org  Thu Jan 19 18:11:24 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 19 Jan 2012 18:11:24 +0100 (CET)
Subject: [pypy-commit] pypy numppy-flatitter: add REVIEW
Message-ID: <20120119171124.160B5820D8@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: numppy-flatitter
Changeset: r51494:3a1f1a964ab4
Date: 2012-01-19 19:10 +0200
http://bitbucket.org/pypy/pypy/changeset/3a1f1a964ab4/

Log:	add REVIEW

diff --git a/pypy/module/micronumpy/REVIEW b/pypy/module/micronumpy/REVIEW
new file mode 100644
--- /dev/null
+++ b/pypy/module/micronumpy/REVIEW
@@ -0,0 +1,10 @@
+* I think we should wait for indexing-by-arrays-2, since this would clean up
+  the iterator interface
+
+* I commited a failing test, it seems the entire approach is pretty doomed.
+  Instead we should keep the parent iterator (and not create a OneDim one)
+
+* For getitem, we need to reuse parent getitem, but with some hook
+  that recomputes indexes. That hook would be used for any sort of access,
+  be it slices or be it integers, but in general I would like to avoid code
+  duplication, since the indexing is getting slowly fairly complex.
\ No newline at end of file

From noreply at buildbot.pypy.org  Thu Jan 19 18:21:21 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 19 Jan 2012 18:21:21 +0100 (CET)
Subject: [pypy-commit] pypy stm: (bivab, arigo)
Message-ID: <20120119172121.91244820D8@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stm
Changeset: r51495:5d6f1589889c
Date: 2012-01-19 18:19 +0100
http://bitbucket.org/pypy/pypy/changeset/5d6f1589889c/

Log:	(bivab, arigo) Translation fixes.

diff --git a/pypy/module/transaction/__init__.py b/pypy/module/transaction/__init__.py
--- a/pypy/module/transaction/__init__.py
+++ b/pypy/module/transaction/__init__.py
@@ -9,10 +9,10 @@
         'set_num_threads': 'interp_transaction.set_num_threads',
         'add': 'interp_transaction.add',
         'run': 'interp_transaction.run',
-        'TransactionError': 'interp_transaction.state.w_error',
     }
 
     appleveldefs = {
+        'TransactionError': 'app_transaction.TransactionError',
     }
 
     def startup(self, space):
diff --git a/pypy/module/transaction/interp_transaction.py b/pypy/module/transaction/interp_transaction.py
--- a/pypy/module/transaction/interp_transaction.py
+++ b/pypy/module/transaction/interp_transaction.py
@@ -13,15 +13,20 @@
         self.__dict__.clear()
         self.running = False
         self.num_threads = NUM_THREADS_DEFAULT
+        self.pending = []
+        self.pending_lists = {0: self.pending}
+        self.ll_lock = threadintf.null_ll_lock
+        self.ll_no_tasks_pending_lock = threadintf.null_ll_lock
+        self.ll_unfinished_lock = threadintf.null_ll_lock
 
     def startup(self, space):
         self.space = space
-        self.pending = []
+        w_module = space.getbuiltinmodule('transaction')
+        self.w_error = space.getattr(w_module, space.wrap('TransactionError'))
+        #
         self.ll_lock = threadintf.allocate_lock()
         self.ll_no_tasks_pending_lock = threadintf.allocate_lock()
         self.ll_unfinished_lock = threadintf.allocate_lock()
-        self.w_error = space.new_exception_class(
-            "transaction.TransactionError")
         self.lock_unfinished()
         self.main_thread_id = threadintf.thread_id()
         self.pending_lists = {self.main_thread_id: self.pending}
@@ -33,7 +38,7 @@
                                  space.wrap("cannot change the number of "
                                             "threads when transaction.run() "
                                             "is active"))
-        self.num_threads = num_threads
+        self.num_threads = num
 
     def lock(self):
         # XXX think about the interaction between locks and the GC
diff --git a/pypy/module/transaction/test/test_transaction.py b/pypy/module/transaction/test/test_transaction.py
--- a/pypy/module/transaction/test/test_transaction.py
+++ b/pypy/module/transaction/test/test_transaction.py
@@ -6,6 +6,10 @@
     def setup_class(cls):
         cls.space = gettestobjspace(usemodules=['transaction'])
 
+    def test_set_num_threads(self):
+        import transaction
+        transaction.set_num_threads(4)
+
     def test_simple(self):
         import transaction
         lst = []
diff --git a/pypy/module/transaction/threadintf.py b/pypy/module/transaction/threadintf.py
--- a/pypy/module/transaction/threadintf.py
+++ b/pypy/module/transaction/threadintf.py
@@ -1,22 +1,36 @@
 import thread
+from pypy.module.thread import ll_thread
+from pypy.rlib.objectmodel import we_are_translated
 
 
+null_ll_lock = ll_thread.null_ll_lock
+
 def allocate_lock():
-    "NOT_RPYTHON"
-    return thread.allocate_lock()
+    if we_are_translated():
+        return ll_thread.allocate_ll_lock()
+    else:
+        return thread.allocate_lock()
 
 def acquire(lock, wait):
-    "NOT_RPYTHON"
-    return lock.acquire(wait)
+    if we_are_translated():
+        return ll_thread.acquire_NOAUTO(lock, wait)
+    else:
+        return lock.acquire(wait)
 
 def release(lock):
-    "NOT_RPYTHON"
-    lock.release()
+    if we_are_translated():
+        ll_thread.release_NOAUTO(lock)
+    else:
+        lock.release()
 
 def start_new_thread(callback, args):
-    "NOT_RPYTHON"
-    thread.start_new_thread(callback, args)
+    if we_are_translated():
+        ll_thread.start_new_thread(callback, args)
+    else:
+        thread.start_new_thread(callback, args)
 
 def thread_id():
-    "NOT_RPYTHON"
-    return thread.get_ident()
+    if we_are_translated():
+        return ll_thread.get_ident()
+    else:
+        return thread.get_ident()

From noreply at buildbot.pypy.org  Thu Jan 19 18:21:22 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 19 Jan 2012 18:21:22 +0100 (CET)
Subject: [pypy-commit] pypy stm: Forgot to add two files.
Message-ID: <20120119172122.C7EFA820D8@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stm
Changeset: r51496:4692940c82e4
Date: 2012-01-19 18:21 +0100
http://bitbucket.org/pypy/pypy/changeset/4692940c82e4/

Log:	Forgot to add two files.

diff --git a/pypy/module/transaction/app_transaction.py b/pypy/module/transaction/app_transaction.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/transaction/app_transaction.py
@@ -0,0 +1,3 @@
+
+class TransactionError(Exception):
+    pass
diff --git a/pypy/module/transaction/test/test_ztranslation.py b/pypy/module/transaction/test/test_ztranslation.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/transaction/test/test_ztranslation.py
@@ -0,0 +1,4 @@
+from pypy.objspace.fake.checkmodule import checkmodule
+
+def test_checkmodule():
+    checkmodule('transaction')

From noreply at buildbot.pypy.org  Thu Jan 19 18:22:56 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 19 Jan 2012 18:22:56 +0100 (CET)
Subject: [pypy-commit] pypy numpypy-ufuncs: improve the test
Message-ID: <20120119172256.301BF820D8@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: numpypy-ufuncs
Changeset: r51497:77ac35b7fd12
Date: 2012-01-19 19:22 +0200
http://bitbucket.org/pypy/pypy/changeset/77ac35b7fd12/

Log:	improve the test

diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -193,11 +193,15 @@
     def test_floorceil(self):
         from _numpypy import array, floor, ceil
         import math
-        reference = [-2.0, -1.0, 0.0, 1.0, 1.0]
-        a = array([-1.4, -1.0, 0.0, 1.0, 1.4])
+        reference = [-2.0, -2.0, -1.0, 0.0, 1.0, 1.0, 0]
+        a = array([-1.4, -1.5, -1.0, 0.0, 1.0, 1.4, 0.5])
         b = floor(a)
         for i in range(5):
             assert b[i] == reference[i]
+        reference = [-1.0, -1.0, -1.0, 0.0, 1.0, 2.0, 1.0]
+        a = array([-1.4, -1.5, -1.0, 0.0, 1.0, 1.4, 0.5])
+        b = ceil(a)
+        assert (reference == b).all()
         inf = float("inf")
         data = [1.5, 2.9999, -1.999, inf]
         results = [math.floor(x) for x in data]

From noreply at buildbot.pypy.org  Thu Jan 19 18:24:24 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 19 Jan 2012 18:24:24 +0100 (CET)
Subject: [pypy-commit] pypy default: (mattip) merge numpypy-ufuncs,
	adding ceil
Message-ID: <20120119172424.B4602820D8@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: 
Changeset: r51498:7745b3fcec92
Date: 2012-01-19 19:23 +0200
http://bitbucket.org/pypy/pypy/changeset/7745b3fcec92/

Log:	(mattip) merge numpypy-ufuncs, adding ceil

diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -70,6 +70,7 @@
         ("exp", "exp"),
         ("fabs", "fabs"),
         ("floor", "floor"),
+        ("ceil", "ceil"),
         ("greater", "greater"),
         ("greater_equal", "greater_equal"),
         ("less", "less"),
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -449,6 +449,7 @@
 
             ("fabs", "fabs", 1, {"promote_to_float": True}),
             ("floor", "floor", 1, {"promote_to_float": True}),
+            ("ceil", "ceil", 1, {"promote_to_float": True}),
             ("exp", "exp", 1, {"promote_to_float": True}),
 
             ('sqrt', 'sqrt', 1, {'promote_to_float': True}),
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -190,14 +190,24 @@
         for i in range(3):
             assert c[i] == a[i] - b[i]
 
-    def test_floor(self):
-        from _numpypy import array, floor
-
-        reference = [-2.0, -1.0, 0.0, 1.0, 1.0]
-        a = array([-1.4, -1.0, 0.0, 1.0, 1.4])
+    def test_floorceil(self):
+        from _numpypy import array, floor, ceil
+        import math
+        reference = [-2.0, -2.0, -1.0, 0.0, 1.0, 1.0, 0]
+        a = array([-1.4, -1.5, -1.0, 0.0, 1.0, 1.4, 0.5])
         b = floor(a)
         for i in range(5):
             assert b[i] == reference[i]
+        reference = [-1.0, -1.0, -1.0, 0.0, 1.0, 2.0, 1.0]
+        a = array([-1.4, -1.5, -1.0, 0.0, 1.0, 1.4, 0.5])
+        b = ceil(a)
+        assert (reference == b).all()
+        inf = float("inf")
+        data = [1.5, 2.9999, -1.999, inf]
+        results = [math.floor(x) for x in data]
+        assert (floor(data) == results).all()
+        results = [math.ceil(x) for x in data]
+        assert (ceil(data) == results).all()
 
     def test_copysign(self):
         from _numpypy import array, copysign
@@ -238,7 +248,7 @@
             assert b[i] == math.sin(a[i])
 
         a = sin(array([True, False], dtype=bool))
-        assert abs(a[0] - sin(1)) < 1e-7 # a[0] will be less precise
+        assert abs(a[0] - sin(1)) < 1e-7  # a[0] will be less precise
         assert a[1] == 0.0
 
     def test_cos(self):
@@ -259,7 +269,6 @@
         for i in range(len(a)):
             assert b[i] == math.tan(a[i])
 
-
     def test_arcsin(self):
         import math
         from _numpypy import array, arcsin
@@ -283,7 +292,6 @@
         for i in range(len(a)):
             assert b[i] == math.acos(a[i])
 
-
         a = array([-10, -1.5, -1.01, 1.01, 1.5, 10, float('nan'), float('inf'), float('-inf')])
         b = arccos(a)
         for f in b:
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -374,6 +374,10 @@
         return math.floor(v)
 
     @simple_unary_op
+    def ceil(self, v):
+        return math.ceil(v)
+
+    @simple_unary_op
     def exp(self, v):
         try:
             return math.exp(v)
@@ -436,4 +440,4 @@
 class Float64(BaseType, Float):
     T = rffi.DOUBLE
     BoxType = interp_boxes.W_Float64Box
-    format_code = "d"
\ No newline at end of file
+    format_code = "d"

From noreply at buildbot.pypy.org  Thu Jan 19 18:24:25 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 19 Jan 2012 18:24:25 +0100 (CET)
Subject: [pypy-commit] pypy numpypy-ufuncs: close merged branch
Message-ID: <20120119172425.E5981820D8@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: numpypy-ufuncs
Changeset: r51499:8e57f6c0502b
Date: 2012-01-19 19:23 +0200
http://bitbucket.org/pypy/pypy/changeset/8e57f6c0502b/

Log:	close merged branch


From noreply at buildbot.pypy.org  Thu Jan 19 18:28:47 2012
From: noreply at buildbot.pypy.org (rguillebert)
Date: Thu, 19 Jan 2012 18:28:47 +0100 (CET)
Subject: [pypy-commit] pypy py3k: (antocuni,
 romain) killed the code that checked for version numbers of Python
Message-ID: <20120119172847.838DE820D8@wyvern.cs.uni-duesseldorf.de>

Author: Romain Guillebert 
Branch: py3k
Changeset: r51500:b3f6b7f219a0
Date: 2012-01-19 18:28 +0100
http://bitbucket.org/pypy/pypy/changeset/b3f6b7f219a0/

Log:	(antocuni, romain) killed the code that checked for version numbers
	of Python 2, changed the way we compute the magic number for pyc
	files

diff --git a/pypy/interpreter/nestedscope.py b/pypy/interpreter/nestedscope.py
--- a/pypy/interpreter/nestedscope.py
+++ b/pypy/interpreter/nestedscope.py
@@ -220,18 +220,9 @@
     def MAKE_CLOSURE(self, numdefaults, next_instr):
         w_codeobj = self.popvalue()
         codeobj = self.space.interp_w(pycode.PyCode, w_codeobj)
-        if codeobj.magic >= 0xa0df281:    # CPython 2.5 AST branch merge
-            w_freevarstuple = self.popvalue()
-            freevars = [self.space.interp_w(Cell, cell)
-                        for cell in self.space.fixedview(w_freevarstuple)]
-        else:
-            n = len(codeobj.co_freevars)
-            freevars = [None] * n
-            while True:
-                n -= 1
-                if n < 0:
-                    break
-                freevars[n] = self.space.interp_w(Cell, self.popvalue())
+        w_freevarstuple = self.popvalue()
+        freevars = [self.space.interp_w(Cell, cell)
+                    for cell in self.space.fixedview(w_freevarstuple)]
         defaultarguments = self.popvalues(numdefaults)
         fn = function.Function(self.space, codeobj, self.w_globals,
                                defaultarguments, freevars)
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -24,12 +24,19 @@
 def unpack_str_tuple(space,w_str_tuple):
     return [space.str_w(w_el) for w_el in space.unpackiterable(w_str_tuple)]
 
-
 # Magic numbers for the bytecode version in code objects.
 # See comments in pypy/module/imp/importing.
 cpython_magic, = struct.unpack("= 0xa0df2ef
-            # Implementation since 2.7a0: 62191 (introduce SETUP_WITH)
-            or self.pycode.magic >= 0xa0df2d1):
-            # implementation since 2.6a1: 62161 (WITH_CLEANUP optimization)
-            self.popvalue()
-            self.popvalue()
-            w_unroller = self.popvalue()
-            w_exitfunc = self.popvalue()
-            self.pushvalue(w_unroller)
-            self.pushvalue(self.space.w_None)
-            self.pushvalue(self.space.w_None)
-        elif self.pycode.magic >= 0xa0df28c:
-            # Implementation since 2.5a0: 62092 (changed WITH_CLEANUP opcode)
-            w_exitfunc = self.popvalue()
-            w_unroller = self.peekvalue(2)
-        else:
-            raise NotImplementedError("WITH_CLEANUP for CPython <= 2.4")
+        self.popvalue()
+        self.popvalue()
+        w_unroller = self.popvalue()
+        w_exitfunc = self.popvalue()
+        self.pushvalue(w_unroller)
+        self.pushvalue(self.space.w_None)
+        self.pushvalue(self.space.w_None)
 
         unroller = self.space.interpclass_w(w_unroller)
         is_app_exc = (unroller is not None and
diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -810,16 +810,14 @@
    a .pyc file in text mode the magic number will be wrong; also, the
    Apple MPW compiler swaps their values, botching string constants.
 
-   CPython uses values between 20121 - 62xxx
+   CPython 2 uses values between 20121 - 62xxx
+   CPython 3 uses values greater than 3000
+   PyPy uses values under 3000
 
 """
 
-# picking a magic number is a mess.  So far it works because we
-# have only one extra opcode, which bumps the magic number by +2, and CPython
-# leaves a gap of 10 when it increases
-# its own magic number.  To avoid assigning exactly the same numbers
-# as CPython we always add a +2.  We'll have to think again when we
-# get three more new opcodes
+# Depending on which opcodes are enabled, eg. CALL_METHOD we bump the version
+# number by some constant
 #
 #  * CALL_METHOD            +2
 #

From noreply at buildbot.pypy.org  Thu Jan 19 20:34:15 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 19 Jan 2012 20:34:15 +0100 (CET)
Subject: [pypy-commit] pypy numpy-indexing-by-arrays-2: progress;
Message-ID: <20120119193415.B5737820D8@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: numpy-indexing-by-arrays-2
Changeset: r51501:c5926e6a02bc
Date: 2012-01-19 21:31 +0200
http://bitbucket.org/pypy/pypy/changeset/c5926e6a02bc/

Log:	progress;

diff --git a/pypy/module/micronumpy/interp_iter.py b/pypy/module/micronumpy/interp_iter.py
--- a/pypy/module/micronumpy/interp_iter.py
+++ b/pypy/module/micronumpy/interp_iter.py
@@ -24,6 +24,9 @@
     def __init__(self, arr):
         self.arr = arr.get_concrete()
 
+    def extend_shape(self, shape):
+        shape.extend(self.arr.shape)
+
 class BoolArrayChunk(BaseChunk):
     def __init__(self, arr):
         self.arr = arr.get_concrete()
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -10,7 +10,7 @@
 from pypy.tool.sourcetools import func_with_new_name
 from pypy.rlib.rstring import StringBuilder
 from pypy.module.micronumpy.interp_iter import ArrayIterator, OneDimIterator,\
-     SkipLastAxisIterator, Chunk, ViewIterator
+     SkipLastAxisIterator, Chunk, ViewIterator, BoolArrayChunk, IntArrayChunk
 
 numpy_driver = jit.JitDriver(
     greens=['shapelen', 'sig'],
@@ -229,6 +229,17 @@
                 n_old_elems_to_use *= old_shape[oldI]
     return new_strides
 
+def wrap_chunk(space, w_idx, size):
+    if (space.isinstance_w(w_idx, space.w_int) or
+        space.isinstance_w(w_idx, space.w_slice)):
+        return Chunk(*space.decode_index4(w_idx, size))
+    arr = convert_to_array(space, w_idx)
+    if arr.find_dtype().is_bool_type():
+        return BoolArrayChunk(arr)
+    elif arr.find_dtype().is_int_type():
+        return IntArrayChunk(arr)
+    raise OperationError(space.w_IndexError, space.wrap("arrays used as indices must be of integer (or boolean) type"))
+
 class BaseArray(Wrappable):
     _attrs_ = ["invalidates", "shape", 'size']
 
@@ -485,6 +496,8 @@
         elif (space.isinstance_w(w_idx, space.w_slice) or
               space.isinstance_w(w_idx, space.w_int)):
             return False
+        if isinstance(w_idx, BaseArray):
+            return False
         lgt = space.len_w(w_idx)
         if lgt > shape_len:
             raise OperationError(space.w_IndexError,
@@ -494,14 +507,15 @@
         for w_item in space.fixedview(w_idx):
             if space.isinstance_w(w_item, space.w_slice):
                 return False
+            if isinstance(w_item, BaseArray):
+                return False
         return True
 
     @jit.unroll_safe
     def _prepare_slice_args(self, space, w_idx):
-        if (space.isinstance_w(w_idx, space.w_int) or
-            space.isinstance_w(w_idx, space.w_slice)):
-            return [Chunk(*space.decode_index4(w_idx, self.shape[0]))]
-        return [Chunk(*space.decode_index4(w_item, self.shape[i])) for i, w_item in
+        if not space.isinstance_w(w_idx, space.w_tuple):
+            return [wrap_chunk(space, w_idx, self.shape[0])]
+        return [wrap_chunk(space, w_item, self.shape[i]) for i, w_item in
                 enumerate(space.fixedview(w_idx))]
 
     def count_all_true(self, arr):
@@ -563,6 +577,7 @@
         if (isinstance(w_idx, BaseArray) and w_idx.shape == self.shape and
             w_idx.find_dtype().is_bool_type()):
             return self.getitem_filter(space, w_idx)
+        # XXX deal with a scalar
         if self._single_item_result(space, w_idx):
             concrete = self.get_concrete()
             item = concrete._index_of_single_item(space, w_idx)
@@ -589,6 +604,13 @@
         view = self.create_slice(chunks).get_concrete()
         view.setslice(space, w_value)
 
+    def force_slice(self, shape, chunks):
+        size = 1
+        for elem in shape:
+            size *= elem
+        res = W_NDimArray(size, shape, self.find_dtype())
+        xxx
+
     @jit.unroll_safe
     def create_slice(self, chunks):
         shape = []
@@ -598,6 +620,9 @@
         s = i + 1
         assert s >= 0
         shape += self.shape[s:]
+        for chunk in chunks:
+            if not isinstance(chunk, Chunk):
+                return self.force_slice(shape, chunks)
         if not isinstance(self, ConcreteArray):
             return VirtualSlice(self, chunks, shape)
         r = calculate_slice_strides(self.shape, self.start, self.strides,
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1301,16 +1301,13 @@
         raises(TypeError, getattr, array(3), '__array_interface__')
 
     def test_array_indexing_one_elem(self):
-        skip("not yet")
         from _numpypy import array, arange
         raises(IndexError, 'arange(3)[array([3.5])]')
         a = arange(3)[array([1])]
-        assert a == 1
-        assert a[0] == 1
+        assert a == [1]
         raises(IndexError,'arange(3)[array([15])]')
         assert arange(3)[array([-3])] == 0
         raises(IndexError,'arange(3)[array([-15])]')
-        assert arange(3)[array(1)] == 1
 
     def test_array_indexing_bool(self):
         from _numpypy import arange

From noreply at buildbot.pypy.org  Thu Jan 19 20:34:16 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 19 Jan 2012 20:34:16 +0100 (CET)
Subject: [pypy-commit] pypy matrixmath-dot: mistake?
Message-ID: <20120119193416.EFC7D820D8@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: matrixmath-dot
Changeset: r51502:275370ca0170
Date: 2012-01-19 21:33 +0200
http://bitbucket.org/pypy/pypy/changeset/275370ca0170/

Log:	mistake?

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -458,7 +458,6 @@
         ri = ViewIterator(0, _r[0], _r[1], arr.broadcast_shape)
         while not frame.done():
             v = sig.eval(frame, arr).convert_to(sig.calc_dtype)
-            z = result.getitem(ri.offset)
             value = add(sig.calc_dtype, v, result.getitem(ri.offset))
             result.setitem(ri.offset, value)
             frame.next(shapelen)

From noreply at buildbot.pypy.org  Thu Jan 19 23:20:22 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Thu, 19 Jan 2012 23:20:22 +0100 (CET)
Subject: [pypy-commit] pypy py3k: (antocuni,
 romain): implement keyword-only arguments in the astcompiler and in
 the interpreter
Message-ID: <20120119222022.9273E82CF8@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni 
Branch: py3k
Changeset: r51503:243e484588fb
Date: 2012-01-19 23:20 +0100
http://bitbucket.org/pypy/pypy/changeset/243e484588fb/

Log:	(antocuni, romain): implement keyword-only arguments in the
	astcompiler and in the interpreter

diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -10,19 +10,27 @@
 class Signature(object):
     _immutable_ = True
     _immutable_fields_ = ["argnames[*]"]
-    __slots__ = ("argnames", "varargname", "kwargname")
+    __slots__ = ("argnames", "kwonlyargnames", "varargname", "kwargname")
 
-    def __init__(self, argnames, varargname=None, kwargname=None):
+    def __init__(self, argnames, varargname=None, kwargname=None, kwonlyargnames=None):
         self.argnames = argnames
         self.varargname = varargname
         self.kwargname = kwargname
+        if kwonlyargnames is None:
+            kwonlyargnames = []
+        self.kwonlyargnames = kwonlyargnames
 
     @jit.elidable
     def find_argname(self, name):
         try:
             return self.argnames.index(name)
         except ValueError:
-            return -1
+            pass
+        try:
+            return len(self.argnames) + self.kwonlyargnames.index(name)
+        except ValueError:
+            pass
+        return -1
 
     def num_argnames(self):
         return len(self.argnames)
@@ -43,20 +51,22 @@
         argnames = self.argnames
         if self.varargname is not None:
             argnames = argnames + [self.varargname]
+        argnames += self.kwonlyargnames
         if self.kwargname is not None:
             argnames = argnames + [self.kwargname]
         return argnames
 
     def __repr__(self):
-        return "Signature(%r, %r, %r)" % (
-                self.argnames, self.varargname, self.kwargname)
+        return "Signature(%r, %r, %r, %r)" % (
+                self.argnames, self.varargname, self.kwargname, self.kwonlyargnames)
 
     def __eq__(self, other):
         if not isinstance(other, Signature):
             return NotImplemented
         return (self.argnames == other.argnames and
                 self.varargname == other.varargname and
-                self.kwargname == other.kwargname)
+                self.kwargname == other.kwargname and
+                self.kwonlyargnames == other.kwonlyargnames)
 
     def __ne__(self, other):
         if not isinstance(other, Signature):
diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -147,6 +147,7 @@
         self.free_vars = _list_to_dict(scope.free_vars, len(self.cell_vars))
         self.w_consts = space.newdict()
         self.argcount = 0
+        self.kwonlyargcount = 0
         self.lineno_set = False
         self.lineno = 0
         self.add_none_to_final_return = True
@@ -459,6 +460,7 @@
         bytecode = ''.join([block.get_code() for block in blocks])
         return pycode.PyCode(self.space,
                              self.argcount,
+                             self.kwonlyargcount,
                              len(self.var_names),
                              stack_depth,
                              flags,
diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -1173,6 +1173,9 @@
         if args.args:
             self._handle_nested_args(args.args)
             self.argcount = len(args.args)
+        if args.kwonlyargs:
+            self._handle_nested_args(args.kwonlyargs)
+            self.kwonlyargcount = len(args.kwonlyargs)
         if func.body:
             for i in range(start, len(func.body)):
                 func.body[i].walkabout(self)
diff --git a/pypy/interpreter/astcompiler/symtable.py b/pypy/interpreter/astcompiler/symtable.py
--- a/pypy/interpreter/astcompiler/symtable.py
+++ b/pypy/interpreter/astcompiler/symtable.py
@@ -495,6 +495,8 @@
         assert isinstance(scope, FunctionScope) # Annotator hint.
         if arguments.args:
             self._handle_params(arguments.args, True)
+        if arguments.kwonlyargs:
+            self._handle_params(arguments.kwonlyargs, True)
         if arguments.vararg:
             self.note_symbol(arguments.vararg, SYM_PARAM)
             scope.note_variable_arg(arguments.vararg)
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -5,6 +5,7 @@
 from pypy.interpreter.pyparser.test import expressions
 from pypy.interpreter.pycode import PyCode
 from pypy.interpreter.pyparser.error import SyntaxError, IndentationError
+from pypy.interpreter.error import OperationError
 from pypy.tool import stdlib_opcode as ops
 
 def compile_with_astcompiler(expr, mode, space):
@@ -41,9 +42,11 @@
         source = str(py.code.Source(source))
         space = self.space
         code = compile_with_astcompiler(source, 'exec', space)
-        # 2.7 bytecode is too different, the standard `dis` module crashes
+        # 3.2 bytecode is too different, the standard `dis` module crashes
         # on older cpython versions
-        if sys.version_info >= (2, 7):
+        if sys.version_info >= (3, 2):
+            # this will only (maybe) work in the far future, when we run pypy
+            # on top of Python 3. For now, it's just disabled
             print
             code.dump()
         w_dict = space.newdict()
@@ -242,7 +245,9 @@
                 return a, b
         """)
         decl = str(decl) + '\n'
-        yield self.st, decl + "x=f(1, b=2)", "x", (1, 2)
+        self.st(decl + "x=f(1, b=2)", "x", (1, 2))
+        operr = py.test.raises(OperationError, 'self.st(decl + "x=f(1, 2)", "x", (1, 2))')
+        assert operr.value.w_type is self.space.w_TypeError
 
     def test_listmakers(self):
         yield (self.st,
diff --git a/pypy/interpreter/astcompiler/test/test_symtable.py b/pypy/interpreter/astcompiler/test/test_symtable.py
--- a/pypy/interpreter/astcompiler/test/test_symtable.py
+++ b/pypy/interpreter/astcompiler/test/test_symtable.py
@@ -131,6 +131,11 @@
         for name in ("a", "b", "c"):
             assert scp.lookup(name) == symtable.SCOPE_LOCAL
 
+    def test_arguments_kwonly(self):
+        scp = self.func_scope("def f(a, *, b): pass")
+        for name in ("a", "b"):
+            assert scp.lookup(name) == symtable.SCOPE_LOCAL
+
     def test_function(self):
         scp = self.func_scope("def f(): x = 4")
         assert scp.lookup("x") == symtable.SCOPE_LOCAL
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -16,7 +16,7 @@
 from pypy.rlib.rarithmetic import intmask
 from pypy.rlib.debug import make_sure_not_resized
 from pypy.rlib import jit
-from pypy.rlib.objectmodel import compute_hash
+from pypy.rlib.objectmodel import compute_hash, we_are_translated
 from pypy.tool.stdlib_opcode import opcodedesc, HAVE_ARGUMENT
 
 # helper
@@ -42,8 +42,16 @@
 def cpython_code_signature(code):
     "([list-of-arg-names], vararg-name-or-None, kwarg-name-or-None)."
     argcount = code.co_argcount
+    if we_are_translated():
+        kwonlyargcount = code.co_kwonlyargcount
+    else:
+        # for compatibility with CPython 2.7 code objects
+        kwonlyargcount = getattr(code, 'co_kwonlyargcount', 0)
     assert argcount >= 0     # annotator hint
+    assert kwonlyargcount >= 0
     argnames = list(code.co_varnames[:argcount])
+    n = argcount + kwonlyargcount
+    kwonlyargs = list(code.co_varnames[argcount:n])
     if code.co_flags & CO_VARARGS:
         varargname = code.co_varnames[argcount]
         argcount += 1
@@ -54,7 +62,7 @@
         argcount += 1
     else:
         kwargname = None
-    return Signature(argnames, varargname, kwargname)
+    return Signature(argnames, varargname, kwargname, kwonlyargs)
 
 class PyCode(eval.Code):
     "CPython-style code objects."
@@ -62,7 +70,7 @@
     _immutable_fields_ = ["co_consts_w[*]", "co_names_w[*]", "co_varnames[*]",
                           "co_freevars[*]", "co_cellvars[*]"]
 
-    def __init__(self, space,  argcount, nlocals, stacksize, flags,
+    def __init__(self, space,  argcount, kwonlyargcount, nlocals, stacksize, flags,
                      code, consts, names, varnames, filename,
                      name, firstlineno, lnotab, freevars, cellvars,
                      hidden_applevel=False, magic = default_magic):
@@ -72,6 +80,7 @@
         eval.Code.__init__(self, name)
         assert nlocals >= 0
         self.co_argcount = argcount
+        self.co_kwonlyargcount = kwonlyargcount
         self.co_nlocals = nlocals
         self.co_stacksize = stacksize
         self.co_flags = flags
@@ -175,6 +184,7 @@
         # stick the underlying CPython magic value, if the code object
         # comes from there
         return cls(space, code.co_argcount,
+                      getattr(code, 'co_kwonlyargcount', 0),
                       code.co_nlocals,
                       code.co_stacksize,
                       code.co_flags,
@@ -250,6 +260,7 @@
                 consts[num] = self.space.unwrap(w)
             num += 1
         return new.code( self.co_argcount,
+                         self.co_kwonlyargcount,
                          self.co_nlocals,
                          self.co_stacksize,
                          self.co_flags,
@@ -297,6 +308,7 @@
             return space.w_False
         areEqual = (self.co_name == other.co_name and
                     self.co_argcount == other.co_argcount and
+                    self.co_kwonlyargcount == other.co_kwonlyargcount and
                     self.co_nlocals == other.co_nlocals and
                     self.co_flags == other.co_flags and
                     self.co_firstlineno == other.co_firstlineno and
@@ -323,6 +335,7 @@
         space = self.space
         result =  compute_hash(self.co_name)
         result ^= self.co_argcount
+        result ^= self.co_kwonlyargcount
         result ^= self.co_nlocals
         result ^= self.co_flags
         result ^= self.co_firstlineno
@@ -337,12 +350,12 @@
             w_result = space.xor(w_result, space.hash(w_const))
         return w_result
 
-    @unwrap_spec(argcount=int, nlocals=int, stacksize=int, flags=int,
+    @unwrap_spec(argcount=int, kwonlyargcount=int, nlocals=int, stacksize=int, flags=int,
                  codestring=str,
                  filename=str, name=str, firstlineno=int,
                  lnotab=str, magic=int)
     def descr_code__new__(space, w_subtype,
-                          argcount, nlocals, stacksize, flags,
+                          argcount, kwonlyargcount, nlocals, stacksize, flags,
                           codestring, w_constants, w_names,
                           w_varnames, filename, name, firstlineno,
                           lnotab, w_freevars=NoneNotWrapped,
@@ -351,6 +364,9 @@
         if argcount < 0:
             raise OperationError(space.w_ValueError,
                                  space.wrap("code: argcount must not be negative"))
+        if kwonlyargcount < 0:
+            raise OperationError(space.w_ValueError,
+                                 space.wrap("code: kwonlyargcount must not be negative"))
         if nlocals < 0:
             raise OperationError(space.w_ValueError,
                                  space.wrap("code: nlocals must not be negative"))
@@ -369,7 +385,7 @@
         else:
             cellvars = []
         code = space.allocate_instance(PyCode, w_subtype)
-        PyCode.__init__(code, space, argcount, nlocals, stacksize, flags, codestring, consts_w[:], names,
+        PyCode.__init__(code, space, argcount, kwonlyargcount, nlocals, stacksize, flags, codestring, consts_w[:], names,
                       varnames, filename, name, firstlineno, lnotab, freevars, cellvars, magic=magic)
         return space.wrap(code)
 
@@ -381,6 +397,7 @@
         w        = space.wrap
         tup      = [
             w(self.co_argcount),
+            w(self.co_kwonlyargcount),
             w(self.co_nlocals),
             w(self.co_stacksize),
             w(self.co_flags),
diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py
--- a/pypy/interpreter/test/test_argument.py
+++ b/pypy/interpreter/test/test_argument.py
@@ -26,12 +26,12 @@
         assert sig.has_kwarg()
         assert sig.scope_length() == 4
         assert sig.getallvarnames() == ["a", "b", "c", "c"]
-        sig = Signature(["a", "b", "c"], "d", "c")
+        sig = Signature(["a", "b", "c"], "d", "c", ["kwonly"])
         assert sig.num_argnames() == 3
         assert sig.has_vararg()
         assert sig.has_kwarg()
         assert sig.scope_length() == 5
-        assert sig.getallvarnames() == ["a", "b", "c", "d", "c"]
+        assert sig.getallvarnames() == ["a", "b", "c", "d", "kwonly", "c"]
 
     def test_eq(self):
         sig1 = Signature(["a", "b", "c"], "d", "c")
@@ -40,11 +40,12 @@
 
 
     def test_find_argname(self):
-        sig = Signature(["a", "b", "c"], None, None)
+        sig = Signature(["a", "b", "c"], None, None, ["kwonly"])
         assert sig.find_argname("a") == 0
         assert sig.find_argname("b") == 1
         assert sig.find_argname("c") == 2
         assert sig.find_argname("d") == -1
+        assert sig.find_argname("kwonly") == 3
 
     def test_tuply(self):
         sig = Signature(["a", "b", "c"], "d", "e")
diff --git a/pypy/objspace/std/marshal_impl.py b/pypy/objspace/std/marshal_impl.py
--- a/pypy/objspace/std/marshal_impl.py
+++ b/pypy/objspace/std/marshal_impl.py
@@ -313,6 +313,7 @@
     # see pypy.interpreter.pycode for the layout
     x = space.interp_w(PyCode, w_pycode)
     m.put_int(x.co_argcount)
+    m.put_int(x.co_kwonlyargcount)
     m.put_int(x.co_nlocals)
     m.put_int(x.co_stacksize)
     m.put_int(x.co_flags)
@@ -355,6 +356,7 @@
 
 def unmarshal_pycode(space, u, tc):
     argcount    = u.get_int()
+    kwonlyargcount = u.get_int()
     nlocals     = u.get_int()
     stacksize   = u.get_int()
     flags       = u.get_int()
@@ -370,7 +372,7 @@
     name        = unmarshal_str(u)
     firstlineno = u.get_int()
     lnotab      = unmarshal_str(u)
-    code = PyCode(space, argcount, nlocals, stacksize, flags,
+    code = PyCode(space, argcount, kwonlyargcount, nlocals, stacksize, flags,
                   code, consts_w[:], names, varnames, filename,
                   name, firstlineno, lnotab, freevars, cellvars)
     return space.wrap(code)

From noreply at buildbot.pypy.org  Fri Jan 20 08:42:10 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 20 Jan 2012 08:42:10 +0100 (CET)
Subject: [pypy-commit] pypy numppy-flatitter: use the parent iterator
Message-ID: <20120120074210.1987382CF8@wyvern.cs.uni-duesseldorf.de>

Author: mattip
Branch: numppy-flatitter
Changeset: r51504:47faffcb9ec3
Date: 2012-01-19 21:30 +0200
http://bitbucket.org/pypy/pypy/changeset/47faffcb9ec3/

Log:	use the parent iterator

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -1308,8 +1308,10 @@
         ViewArray.__init__(self, size, [size], arr.dtype, order=arr.order,
                                parent=arr)
         self.shapelen = len(arr.shape)
-        self.iter = OneDimIterator(arr.start, self.strides[0],
-                                   self.shape[0])
+        sig = arr.find_sig()
+        #self.iter = OneDimIterator(arr.start, self.strides[0],
+        #                           self.shape[0])
+        self.iter = sig.create_frame(arr).get_final_iter()
         self.start = arr.start
         self.base = arr
 

From noreply at buildbot.pypy.org  Fri Jan 20 08:42:11 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 20 Jan 2012 08:42:11 +0100 (CET)
Subject: [pypy-commit] pypy numppy-flatitter: correct test for missing ==
 operator, un-skip failing test
Message-ID: <20120120074211.557B382CF8@wyvern.cs.uni-duesseldorf.de>

Author: mattip
Branch: numppy-flatitter
Changeset: r51505:23731c70eb07
Date: 2012-01-19 22:44 +0200
http://bitbucket.org/pypy/pypy/changeset/23731c70eb07/

Log:	correct test for missing == operator, un-skip failing test

diff --git a/pypy/module/micronumpy/REVIEW b/pypy/module/micronumpy/REVIEW
--- a/pypy/module/micronumpy/REVIEW
+++ b/pypy/module/micronumpy/REVIEW
@@ -1,10 +1,9 @@
 * I think we should wait for indexing-by-arrays-2, since this would clean up
   the iterator interface
 
-* I commited a failing test, it seems the entire approach is pretty doomed.
-  Instead we should keep the parent iterator (and not create a OneDim one)
-
 * For getitem, we need to reuse parent getitem, but with some hook
   that recomputes indexes. That hook would be used for any sort of access,
   be it slices or be it integers, but in general I would like to avoid code
-  duplication, since the indexing is getting slowly fairly complex.
\ No newline at end of file
+  duplication, since the indexing is getting slowly fairly complex.
+
+* iterating over a transposed array still fails.
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -1325,11 +1325,15 @@
     def descr_iter(self):
         return self
 
+    def descr_index(self, space):
+        return space.wrap(self.iter.offset)
+
 W_FlatIterator.typedef = TypeDef(
     'flatiter',
     next = interp2app(W_FlatIterator.descr_next),
     __iter__ = interp2app(W_FlatIterator.descr_iter),
     __getitem__ = interp2app(BaseArray.descr_getitem),
     __setitem__ = interp2app(BaseArray.descr_setitem),
+    index = GetSetProperty(W_FlatIterator.descr_index),
 )
 W_FlatIterator.acceptable_as_base_class = False
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1308,16 +1308,22 @@
         assert b[-2] == 8
         raises(IndexError, "b[11]")
         raises(IndexError, "b[-11]")
+        assert b.index == 3
 
     def test_flatiter_view(self):
         from _numpypy import arange
         a = arange(10).reshape(5, 2)
-        assert (a[::2].flat == [0, 1, 4, 5, 8, 9])
+        #no == yet.
+        # a[::2].flat == [0, 1, 4, 5, 8, 9]
+        isequal = True
+        for y,z in zip(a[::2].flat, [0, 1, 4, 5, 8, 9]):
+            if y != z:
+                isequal = False
+        assert isequal == True
 
     def test_flatiter_transpose(self):
         from _numpypy import arange
         a = arange(10)
-        skip('out-of-order transformations do not work yet')
         assert a.reshape(2,5).T.flat[3] == 6
 
     def test_slice_copy(self):

From noreply at buildbot.pypy.org  Fri Jan 20 08:42:12 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 20 Jan 2012 08:42:12 +0100 (CET)
Subject: [pypy-commit] pypy matrixmath-dot: refactor and rework,
	still need more tests
Message-ID: <20120120074212.93C4982CF8@wyvern.cs.uni-duesseldorf.de>

Author: mattip
Branch: matrixmath-dot
Changeset: r51506:2bcfa95fe92a
Date: 2012-01-20 09:41 +0200
http://bitbucket.org/pypy/pypy/changeset/2bcfa95fe92a/

Log:	refactor and rework, still need more tests

diff --git a/pypy/module/micronumpy/dot.py b/pypy/module/micronumpy/dot.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/micronumpy/dot.py
@@ -0,0 +1,68 @@
+from pypy.module.micronumpy import interp_ufuncs
+from pypy.module.micronumpy.strides import calculate_dot_strides
+from pypy.interpreter.error import OperationError, operationerrfmt
+from pypy.module.micronumpy.interp_iter import ViewIterator
+
+
+def match_dot_shapes(space, left, right):
+    my_critical_dim_size = left.shape[-1]
+    right_critical_dim_size = right.shape[0]
+    right_critical_dim = 0
+    right_critical_dim_stride = right.strides[0]
+    out_shape = []
+    if len(right.shape) > 1:
+        right_critical_dim = len(right.shape) - 2
+        right_critical_dim_size = right.shape[right_critical_dim]
+        right_critical_dim_stride = right.strides[right_critical_dim]
+        assert right_critical_dim >= 0
+        out_shape += left.shape[:-1] + \
+                     right.shape[0:right_critical_dim] + \
+                     right.shape[right_critical_dim + 1:]
+    elif len(right.shape) > 0:
+        #dot does not reduce for scalars
+        out_shape += left.shape[:-1]
+    if my_critical_dim_size != right_critical_dim_size:
+        raise OperationError(space.w_ValueError, space.wrap(
+                                        "objects are not aligned"))
+    return out_shape, right_critical_dim
+
+
+def multidim_dot(space, left, right, result, dtype, right_critical_dim):
+    ''' assumes left, right are concrete arrays
+    given left.shape == [3, 5, 7],
+          right.shape == [2, 7, 4]
+     result.shape == [3, 5, 2, 4]
+    broadcast shape should be [3, 5, 2, 7, 4]
+    result should skip dims 3 which is results.ndims - 1
+    left should skip 2, 4 which is a.ndims-1 + range(right.ndims)
+          except where it==(right.ndims-2)
+    right should skip 0, 1
+    '''
+    mul = interp_ufuncs.get(space).multiply.func
+    add = interp_ufuncs.get(space).add.func
+    broadcast_shape = left.shape[:-1] + right.shape
+    left_skip = [len(left.shape) - 1 + i for i in range(len(right.shape))
+                                         if i != right_critical_dim]
+    right_skip = range(len(left.shape) - 1)
+    result_skip = [len(result.shape) - 1]
+    shapelen = len(broadcast_shape)
+    _r = calculate_dot_strides(result.strides, result.backstrides,
+                                  broadcast_shape, result_skip)
+    outi = ViewIterator(0, _r[0], _r[1], broadcast_shape)
+    _r = calculate_dot_strides(left.strides, left.backstrides,
+                                  broadcast_shape, left_skip)
+    lefti = ViewIterator(0, _r[0], _r[1], broadcast_shape)
+    _r = calculate_dot_strides(right.strides, right.backstrides,
+                                  broadcast_shape, right_skip)
+    righti = ViewIterator(0, _r[0], _r[1], broadcast_shape)
+    while not outi.done():
+        v = mul(dtype, left.getitem(lefti.offset),
+                       right.getitem(righti.offset))
+        value = add(dtype, v, result.getitem(outi.offset))
+        result.setitem(outi.offset, value)
+        outi = outi.next(shapelen)
+        righti = righti.next(shapelen)
+        lefti = lefti.next(shapelen)
+    assert lefti.done()
+    assert righti.done()
+    return result
diff --git a/pypy/module/micronumpy/interp_iter.py b/pypy/module/micronumpy/interp_iter.py
--- a/pypy/module/micronumpy/interp_iter.py
+++ b/pypy/module/micronumpy/interp_iter.py
@@ -16,10 +16,6 @@
     def __init__(self, res_shape):
         self.res_shape = res_shape
 
-class DotTransform(BaseTransform):
-    def __init__(self, res_shape, skip_dims):
-        self.res_shape = res_shape
-        self.skip_dims = skip_dims
 
 class BaseIterator(object):
     def next(self, shapelen):
@@ -90,10 +86,6 @@
                                         self.strides,
                                         self.backstrides, t.chunks)
             return ViewIterator(r[1], r[2], r[3], r[0])
-        elif isinstance(t, DotTransform):
-            r = calculate_dot_strides(self.strides, self.backstrides,
-                                     t.res_shape, t.skip_dims)
-            return ViewIterator(self.offset, r[0], r[1], t.res_shape)
 
     @jit.unroll_safe
     def next(self, shapelen):
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -3,14 +3,15 @@
 from pypy.interpreter.gateway import interp2app, NoneNotWrapped
 from pypy.interpreter.typedef import TypeDef, GetSetProperty
 from pypy.module.micronumpy import interp_ufuncs, interp_dtype, signature
-from pypy.module.micronumpy.strides import calculate_slice_strides,\
-                                           calculate_dot_strides
+from pypy.module.micronumpy.strides import calculate_slice_strides
 from pypy.rlib import jit
 from pypy.rpython.lltypesystem import lltype, rffi
 from pypy.tool.sourcetools import func_with_new_name
 from pypy.rlib.rstring import StringBuilder
 from pypy.module.micronumpy.interp_iter import ArrayIterator, OneDimIterator,\
-     SkipLastAxisIterator, ViewIterator
+     SkipLastAxisIterator
+from pypy.module.micronumpy.dot import multidim_dot, match_dot_shapes, dot_docstring
+
 
 numpy_driver = jit.JitDriver(
     greens=['shapelen', 'sig'],
@@ -212,28 +213,6 @@
                 n_old_elems_to_use *= old_shape[oldI]
     return new_strides
 
-def match_dot_shapes(space, self, other):
-    my_critical_dim_size = self.shape[-1]
-    other_critical_dim_size = other.shape[0]
-    other_critical_dim = 0
-    other_critical_dim_stride = other.strides[0]
-    out_shape = []
-    if len(other.shape) > 1:
-        other_critical_dim = len(other.shape) - 2
-        other_critical_dim_size = other.shape[other_critical_dim]
-        other_critical_dim_stride = other.strides[other_critical_dim]
-        assert other_critical_dim >= 0
-        out_shape += self.shape[:-1] + \
-                     other.shape[0:other_critical_dim] + \
-                     other.shape[other_critical_dim + 1:]
-    elif len(other.shape) > 0:
-        #dot does not reduce for scalars
-        out_shape += self.shape[:-1]
-    if my_critical_dim_size != other_critical_dim_size:
-        raise OperationError(space.w_ValueError, space.wrap(
-                                        "objects are not aligned"))
-    return out_shape, other_critical_dim
-
 class BaseArray(Wrappable):
     _attrs_ = ["invalidates", "shape", 'size']
 
@@ -399,14 +378,6 @@
     descr_argmin = _reduce_argmax_argmin_impl("min")
 
     def descr_dot(self, space, w_other):
-        '''Dot product of two arrays.
-
-    For 2-D arrays it is equivalent to matrix multiplication, and for 1-D
-    arrays to inner product of vectors (without complex conjugation). For
-    N dimensions it is a sum product over the last axis of `a` and
-    the second-to-last of `b`::
-
-        dot(a, b)[i,j,k,m] = sum(a[i,j,:] * b[k,:,m])'''
         other = convert_to_array(space, w_other)
         if isinstance(other, Scalar):
             return self.descr_mul(space, other)
@@ -425,43 +396,10 @@
         for o in out_shape:
             out_size *= o
         result = W_NDimArray(out_size, out_shape, dtype)
-        # given a.shape == [3, 5, 7],
-        #       b.shape == [2, 7, 4]
-        #  result.shape == [3, 5, 2, 4]
-        # all iterators shapes should be [3, 5, 2, 7, 4]
-        # result should skip dims 3 which is results.ndims - 1
-        # a should skip 2, 4 which is a.ndims-1 + range(b.ndims) 
-        #       except where it==(b.ndims-2)
-        # b should skip 0, 1
-        mul = interp_ufuncs.get(space).multiply.func
-        add = interp_ufuncs.get(space).add.func
-        broadcast_shape = self.shape[:-1] + other.shape
-        #Aww, cmon, this is the product of a warped mind.
-        left_skip = [len(self.shape) - 1 + i for i in range(len(other.shape)) if i != other_critical_dim]
-        right_skip = range(len(self.shape) - 1)
-        arr = DotArray(mul, 'DotName', out_shape, dtype, self, other,
-                                        left_skip, right_skip)
-        arr.broadcast_shape = broadcast_shape
-        arr.result_skip = [len(out_shape) - 1]
-        #Make this lazy someday...
-        sig = signature.find_sig(signature.DotSignature(mul, 'dot', dtype,
-                                  self.create_sig(), other.create_sig()), arr)
-        assert isinstance(sig, signature.DotSignature)
-        self.do_dot_loop(sig, result, arr, add)
-        return result
-
-    def do_dot_loop(self, sig, result, arr, add):
-        frame = sig.create_frame(arr)
-        shapelen = len(arr.broadcast_shape)
-        _r = calculate_dot_strides(result.strides, result.backstrides,
-                                      arr.broadcast_shape, arr.result_skip)
-        ri = ViewIterator(0, _r[0], _r[1], arr.broadcast_shape)
-        while not frame.done():
-            v = sig.eval(frame, arr).convert_to(sig.calc_dtype)
-            value = add(sig.calc_dtype, v, result.getitem(ri.offset))
-            result.setitem(ri.offset, value)
-            frame.next(shapelen)
-            ri = ri.next(shapelen)
+        # This is the place to add fpypy and blas
+        return multidim_dot(space, self.get_concrete(), 
+                            other.get_concrete(), result, dtype,
+                            other_critical_dim)
 
     def get_concrete(self):
         raise NotImplementedError
@@ -933,23 +871,6 @@
                        left, right)
         self.dim = dim
 
-class DotArray(Call2):
-    """ NOTE: this is only used as a container, you should never
-    encounter such things in the wild. Remove this comment
-    when we'll make Dot lazy
-    """
-    _immutable_fields_ = ['left', 'right']
-    
-    def __init__(self, ufunc, name, shape, dtype, left, right, left_skip, right_skip):
-        Call2.__init__(self, ufunc, name, shape, dtype, dtype,
-                       left, right)
-        self.left_skip = left_skip
-        self.right_skip = right_skip
-    def create_sig(self):
-        #if self.forced_result is not None:
-        #    return self.forced_result.create_sig()
-        assert NotImplementedError 
-
 class ConcreteArray(BaseArray):
     """ An array that have actual storage, whether owned or not
     """
@@ -1304,6 +1225,8 @@
     return space.wrap(arr)
 
 def dot(space, w_obj, w_obj2):
+    '''see numpypy.dot. Does not exist as an ndarray method in numpy.
+    '''
     w_arr = convert_to_array(space, w_obj)
     if isinstance(w_arr, Scalar):
         return convert_to_array(space, w_obj2).descr_dot(space, w_arr)
diff --git a/pypy/module/micronumpy/signature.py b/pypy/module/micronumpy/signature.py
--- a/pypy/module/micronumpy/signature.py
+++ b/pypy/module/micronumpy/signature.py
@@ -2,7 +2,7 @@
 from pypy.rlib.rarithmetic import intmask
 from pypy.module.micronumpy.interp_iter import ViewIterator, ArrayIterator, \
      ConstantIterator, AxisIterator, ViewTransform,\
-     BroadcastTransform, DotTransform
+     BroadcastTransform
 from pypy.rlib.jit import hint, unroll_safe, promote
 
 """ Signature specifies both the numpy expression that has been constructed
@@ -449,21 +449,3 @@
     
     def debug_repr(self):
         return 'AxisReduceSig(%s, %s)' % (self.name, self.right.debug_repr())
-
-class DotSignature(Call2):
-    def _invent_numbering(self, cache, allnumbers):
-        self.left._invent_numbering(new_cache(), allnumbers)
-        self.right._invent_numbering(new_cache(), allnumbers)
-
-    def _create_iter(self, iterlist, arraylist, arr, transforms):
-        from pypy.module.micronumpy.interp_numarray import DotArray
-
-        assert isinstance(arr, DotArray)
-        rtransforms = transforms + [DotTransform(arr.broadcast_shape, arr.right_skip)]
-        ltransforms = transforms + [DotTransform(arr.broadcast_shape, arr.left_skip)]
-        self.left._create_iter(iterlist, arraylist, arr.left, ltransforms)
-        self.right._create_iter(iterlist, arraylist, arr.right, rtransforms)
-
-    def debug_repr(self):
-        return 'DotSig(%s, %s %s)' % (self.name, self.right.debug_repr(),
-						 self.left.debug_repr())
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -869,7 +869,7 @@
     def test_dot(self):
         from _numpypy import array, dot, arange
         a = array(range(5))
-        assert a.dot(a) == 30.0
+        assert dot(a, a) == 30.0
 
         a = array(range(5))
         assert a.dot(range(5)) == 30
@@ -887,9 +887,11 @@
         #Superfluous shape test makes the intention of the test clearer
         assert a.shape == (2, 3, 4)
         assert b.shape == (4, 3)
-        c = a.dot(b)
+        c = dot(a, b)
         assert (c == [[[14, 38, 62], [38, 126, 214], [62, 214, 366]],
                    [[86, 302, 518], [110, 390, 670], [134, 478, 822]]]).all()
+        c = dot(a, b[:, :, 2])
+        assert (c == [[38, 126, 214], [302, 390, 478]]).all()
 
     def test_dot_constant(self):
         from _numpypy import array

From noreply at buildbot.pypy.org  Fri Jan 20 09:23:07 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 20 Jan 2012 09:23:07 +0100 (CET)
Subject: [pypy-commit] pypy matrixmath-dot: whoops
Message-ID: <20120120082307.E327782CF8@wyvern.cs.uni-duesseldorf.de>

Author: mattip
Branch: matrixmath-dot
Changeset: r51507:600fcfb76aab
Date: 2012-01-20 10:22 +0200
http://bitbucket.org/pypy/pypy/changeset/600fcfb76aab/

Log:	whoops

diff --git a/pypy/module/micronumpy/dot.py b/pypy/module/micronumpy/dot.py
--- a/pypy/module/micronumpy/dot.py
+++ b/pypy/module/micronumpy/dot.py
@@ -2,8 +2,17 @@
 from pypy.module.micronumpy.strides import calculate_dot_strides
 from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.module.micronumpy.interp_iter import ViewIterator
+from pypy.module.micronumpy.signature import new_printable_location
+from pypy.rlib import jit
 
 
+dot_driver = jit.JitDriver(
+    greens=['shapelen', 'left', 'right'],
+    reds=['lefti', 'righti', 'outi', 'result'],
+    get_printable_location=new_printable_location('dot'),
+    name='dot',
+)
+
 def match_dot_shapes(space, left, right):
     my_critical_dim_size = left.shape[-1]
     right_critical_dim_size = right.shape[0]
@@ -27,6 +36,7 @@
     return out_shape, right_critical_dim
 
 
+ at jit.unroll_safe
 def multidim_dot(space, left, right, result, dtype, right_critical_dim):
     ''' assumes left, right are concrete arrays
     given left.shape == [3, 5, 7],
@@ -56,6 +66,14 @@
                                   broadcast_shape, right_skip)
     righti = ViewIterator(0, _r[0], _r[1], broadcast_shape)
     while not outi.done():
+        dot_driver.jit_merge_point(left=left,
+                                   right=right,
+                                   shape_len=shape_len,
+                                   lefti=lefti,
+                                   righti=righti,
+                                   outi=outi,
+                                   result=result,
+                                  )
         v = mul(dtype, left.getitem(lefti.offset),
                        right.getitem(righti.offset))
         value = add(dtype, v, result.getitem(outi.offset))
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -10,7 +10,7 @@
 from pypy.rlib.rstring import StringBuilder
 from pypy.module.micronumpy.interp_iter import ArrayIterator, OneDimIterator,\
      SkipLastAxisIterator
-from pypy.module.micronumpy.dot import multidim_dot, match_dot_shapes, dot_docstring
+from pypy.module.micronumpy.dot import multidim_dot, match_dot_shapes
 
 
 numpy_driver = jit.JitDriver(
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -368,6 +368,20 @@
                                 'int_ge': 1, 'guard_false': 1, 'jump': 1,
                                 'arraylen_gc': 1})
 
+    def ddefine_dot():
+        return """
+        a = [[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]]
+        b=[[0, 1, 2], [3, 4, 5], [6, 7, 8], [9, 10, 11]]
+        c = a.dot(b)
+        c -> 1 -> 2
+        """
+
+    def test_dot(self):
+        py.test.skip("not yet")
+        result = self.run("dot")
+        assert result == 184
+        self.check_simple_loop({})
+
 
 class TestNumpyOld(LLJitMixin):
     def setup_class(cls):

From noreply at buildbot.pypy.org  Fri Jan 20 10:03:05 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 20 Jan 2012 10:03:05 +0100 (CET)
Subject: [pypy-commit] pypy stm: void fields.
Message-ID: <20120120090305.127E082CF8@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stm
Changeset: r51508:5c885d90cfd1
Date: 2012-01-19 22:20 +0100
http://bitbucket.org/pypy/pypy/changeset/5c885d90cfd1/

Log:	void fields.

diff --git a/pypy/translator/stm/test/test_llstm.py b/pypy/translator/stm/test/test_llstm.py
--- a/pypy/translator/stm/test/test_llstm.py
+++ b/pypy/translator/stm/test/test_llstm.py
@@ -9,7 +9,7 @@
                        ('c1', lltype.Char), ('c2', lltype.Char),
                        ('c3', lltype.Char), ('l', lltype.SignedLongLong),
                        ('f', lltype.Float), ('sa', lltype.SingleFloat),
-                       ('sb', lltype.SingleFloat))
+                       ('sb', lltype.SingleFloat), ('v', lltype.Void))
 rll1 = r_longlong(-10000000000003)
 rll2 = r_longlong(-300400500600700)
 rf1 = -12.38976129
@@ -29,6 +29,7 @@
     assert a.f == rf1
     assert float(a.sa) == float(rs1a)
     assert float(a.sb) == float(rs1b)
+    assert a.v == None
     assert stm_getfield(a, 'x') == -611
     assert stm_getfield(a, 'c2') == '\\'
     assert stm_getfield(a, 'c1') == '/'
@@ -58,6 +59,7 @@
     a.f = rf1
     a.sa = rs1a
     a.sb = rs1b
+    a.v = None
     stm_descriptor_init()
     stm_perform_transaction(llhelper(CALLBACK, callback1),
                         rffi.cast(rffi.VOIDP, a))
@@ -70,6 +72,7 @@
     assert a.f == rf1
     assert float(a.sa) == float(rs1a)
     assert float(a.sb) == float(rs1b)
+    assert a.v == None
     assert a.y == 10
     lltype.free(a, flavor='raw')
 
@@ -83,6 +86,7 @@
     assert a.f == rf1
     assert float(a.sa) == float(rs1a)
     assert float(a.sb) == float(rs1b)
+    assert a.v == None
     assert stm_getfield(a, 'x') == -611
     assert stm_getfield(a, 'c1') == '&'
     assert stm_getfield(a, 'c2') == '*'
@@ -115,6 +119,7 @@
     assert a.f == rf1
     assert float(a.sa) == float(rs1a)
     assert float(a.sb) == float(rs1b)
+    assert a.v == None
     if a.y < 10:
         a.y += 1    # non-transactionally
         stm_abort_and_retry()
@@ -131,6 +136,7 @@
     a.f = rf1
     a.sa = rs1a
     a.sb = rs1b
+    a.v = None
     stm_descriptor_init()
     stm_perform_transaction(llhelper(CALLBACK, callback2),
                         rffi.cast(rffi.VOIDP, a))
@@ -143,5 +149,6 @@
     assert a.f == rf2
     assert float(a.sa) == float(rs2a)
     assert float(a.sb) == float(rs2b)
+    assert a.v == None
     assert a.y == 10
     lltype.free(a, flavor='raw')
diff --git a/pypy/translator/stm/test/test_transform.py b/pypy/translator/stm/test/test_transform.py
--- a/pypy/translator/stm/test/test_transform.py
+++ b/pypy/translator/stm/test/test_transform.py
@@ -53,6 +53,16 @@
     res = eval_stm_graph(interp, graph, [p], stm_mode="regular_transaction")
     assert res == 42
 
+def test_void_field():
+    S = lltype.GcStruct('S', ('v', lltype.Void))
+    p = lltype.malloc(S, immortal=True)
+    def func(p):
+        p.v = None
+        return p.v
+    interp, graph = get_interpreter(func, [p])
+    transform_graph(graph)
+    assert summary(graph) == {'getfield': 1, 'setfield': 1}
+
 def test_getarraysize():
     A = lltype.GcArray(lltype.Signed)
     p = lltype.malloc(A, 100, immortal=True)
diff --git a/pypy/translator/stm/transform.py b/pypy/translator/stm/transform.py
--- a/pypy/translator/stm/transform.py
+++ b/pypy/translator/stm/transform.py
@@ -109,7 +109,9 @@
 
     def stt_getfield(self, newoperations, op):
         STRUCT = op.args[0].concretetype.TO
-        if STRUCT._immutable_field(op.args[1].value):
+        if op.result.concretetype is lltype.Void:
+            op1 = op
+        elif STRUCT._immutable_field(op.args[1].value):
             op1 = op
         elif STRUCT._gckind == 'raw':
             turn_inevitable(newoperations, "getfield-raw")
@@ -120,7 +122,9 @@
 
     def stt_setfield(self, newoperations, op):
         STRUCT = op.args[0].concretetype.TO
-        if STRUCT._immutable_field(op.args[1].value):
+        if op.args[2].concretetype is lltype.Void:
+            op1 = op
+        elif STRUCT._immutable_field(op.args[1].value):
             op1 = op
         elif STRUCT._gckind == 'raw':
             turn_inevitable(newoperations, "setfield-raw")
@@ -131,7 +135,9 @@
 
     def stt_getarrayitem(self, newoperations, op):
         ARRAY = op.args[0].concretetype.TO
-        if ARRAY._immutable_field():
+        if op.result.concretetype is lltype.Void:
+            op1 = op
+        elif ARRAY._immutable_field():
             op1 = op
         elif ARRAY._gckind == 'raw':
             turn_inevitable(newoperations, "getarrayitem-raw")
@@ -142,7 +148,9 @@
 
     def stt_setarrayitem(self, newoperations, op):
         ARRAY = op.args[0].concretetype.TO
-        if ARRAY._immutable_field():
+        if op.args[2].concretetype is lltype.Void:
+            op1 = op
+        elif ARRAY._immutable_field():
             op1 = op
         elif ARRAY._gckind == 'raw':
             turn_inevitable(newoperations, "setarrayitem-raw")
@@ -153,7 +161,9 @@
 
     def stt_getinteriorfield(self, newoperations, op):
         OUTER = op.args[0].concretetype.TO
-        if OUTER._hints.get('immutable'):
+        if op.result.concretetype is lltype.Void:
+            op1 = op
+        elif OUTER._hints.get('immutable'):
             op1 = op
         elif OUTER._gckind == 'raw':
             turn_inevitable(newoperations, "getinteriorfield-raw")
@@ -164,7 +174,9 @@
 
     def stt_setinteriorfield(self, newoperations, op):
         OUTER = op.args[0].concretetype.TO
-        if OUTER._hints.get('immutable'):
+        if op.args[-1].concretetype is lltype.Void:
+            op1 = op
+        elif OUTER._hints.get('immutable'):
             op1 = op
         elif OUTER._gckind == 'raw':
             turn_inevitable(newoperations, "setinteriorfield-raw")

From noreply at buildbot.pypy.org  Fri Jan 20 10:03:06 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 20 Jan 2012 10:03:06 +0100 (CET)
Subject: [pypy-commit] pypy stm: Re-enable cast_ptr_to_adr for now.
Message-ID: <20120120090306.4C28582CF8@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stm
Changeset: r51509:f1643c0913ad
Date: 2012-01-19 23:02 +0100
http://bitbucket.org/pypy/pypy/changeset/f1643c0913ad/

Log:	Re-enable cast_ptr_to_adr for now.

diff --git a/pypy/translator/c/funcgen.py b/pypy/translator/c/funcgen.py
--- a/pypy/translator/c/funcgen.py
+++ b/pypy/translator/c/funcgen.py
@@ -686,9 +686,9 @@
 
     def OP_CAST_PTR_TO_ADR(self, op):
         if self.lltypemap(op.args[0]).TO._gckind == 'gc' and self._is_stm():
-            raise AssertionError("cast_ptr_to_adr(gcref) is a bad idea "
-                                 "with STM.  Consider checking config.stm "
-                                 "in %r" % (self.graph,))
+            from pypy.translator.c.support import log
+            log.WARNING("cast_ptr_to_adr(gcref) might be a bad idea with STM:")
+            log.WARNING("  %r" % (self.graph,))
         return self.OP_CAST_POINTER(op)
 
     def OP_CAST_INT_TO_PTR(self, op):
diff --git a/pypy/translator/stm/test/test_funcgen.py b/pypy/translator/stm/test/test_funcgen.py
--- a/pypy/translator/stm/test/test_funcgen.py
+++ b/pypy/translator/stm/test/test_funcgen.py
@@ -1,5 +1,6 @@
 from pypy.rpython.lltypesystem import lltype, rffi
 from pypy.rlib.rarithmetic import r_longlong, r_singlefloat
+from pypy.rlib.objectmodel import compute_identity_hash
 from pypy.translator.stm.test.support import CompiledSTMTests
 from pypy.translator.stm._rffi_stm import (CALLBACK, stm_perform_transaction,
                                            stm_descriptor_init, stm_descriptor_done)
@@ -50,6 +51,16 @@
         _, data = cbuilder.cmdexec('', err=True)
         assert data.endswith('ok!\n')
 
+    def test_compile_identity_hash(self):
+        class A:
+            pass
+        def entry_point(argv):
+            a = A()
+            debug_print(compute_identity_hash(a))
+            return 0
+        t, cbuilder = self.compile(entry_point)
+        _, data = cbuilder.cmdexec('', err=True)
+
 # ____________________________________________________________
 
 A = lltype.GcStruct('A', ('x', lltype.Signed), ('y', lltype.Signed),

From noreply at buildbot.pypy.org  Fri Jan 20 10:03:07 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 20 Jan 2012 10:03:07 +0100 (CET)
Subject: [pypy-commit] pypy stm: Fix for --gc=none.
Message-ID: <20120120090307.7CCDB82CF8@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stm
Changeset: r51510:a0831447f4d7
Date: 2012-01-19 23:12 +0100
http://bitbucket.org/pypy/pypy/changeset/a0831447f4d7/

Log:	Fix for --gc=none.

diff --git a/pypy/translator/c/src/mem.h b/pypy/translator/c/src/mem.h
--- a/pypy/translator/c/src/mem.h
+++ b/pypy/translator/c/src/mem.h
@@ -230,6 +230,9 @@
 #define OP_BOEHM_DISAPPEARING_LINK(link, obj, r)  /* nothing */
 #define OP_GC__DISABLE_FINALIZERS(r)  /* nothing */
 #define OP_GC__ENABLE_FINALIZERS(r)  /* nothing */
+#define GC_REGISTER_FINALIZER(a, b, c, d, e)  /* nothing */
+#define GC_gcollect()  /* nothing */
+#define GC_set_max_heap_size(a)  /* nothing */
 #endif
 
 /************************************************************/

From noreply at buildbot.pypy.org  Fri Jan 20 10:03:08 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 20 Jan 2012 10:03:08 +0100 (CET)
Subject: [pypy-commit] pypy stm: Shut off spurious warnings.
Message-ID: <20120120090308.AEAB982CF8@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stm
Changeset: r51511:08154d958352
Date: 2012-01-19 23:17 +0100
http://bitbucket.org/pypy/pypy/changeset/08154d958352/

Log:	Shut off spurious warnings.

diff --git a/pypy/translator/stm/src_stm/et.c b/pypy/translator/stm/src_stm/et.c
--- a/pypy/translator/stm/src_stm/et.c
+++ b/pypy/translator/stm/src_stm/et.c
@@ -861,19 +861,19 @@
 }
 
 // XXX little-endian only!
-unsigned long stm_read_partial_word(int fieldsize, char *addr)
+unsigned long stm_read_partial_word(int fieldsize, void *addr)
 {
   int misalignment = ((long)addr) & (sizeof(void*)-1);
-  long *p = (long*)(addr - misalignment);
+  long *p = (long*)((char *)addr - misalignment);
   unsigned long word = stm_read_word(p);
   return word >> (misalignment * 8);
 }
 
 // XXX little-endian only!
-void stm_write_partial_word(int fieldsize, char *addr, unsigned long nval)
+void stm_write_partial_word(int fieldsize, void *addr, unsigned long nval)
 {
   int misalignment = ((long)addr) & (sizeof(void*)-1);
-  long *p = (long*)(addr - misalignment);
+  long *p = (long*)((char *)addr - misalignment);
   long val = nval << (misalignment * 8);
   long word = stm_read_word(p);
   long mask = ((1L << (fieldsize * 8)) - 1) << (misalignment * 8);
@@ -951,7 +951,7 @@
 #if PYPY_LONG_BIT == 32
   stm_write_word(addr, ii);         /* 32 bits */
 #else
-  stm_write_partial_word(4, (char *)addr, ii);   /* 64 bits */
+  stm_write_partial_word(4, addr, ii);   /* 64 bits */
 #endif
 }
 
diff --git a/pypy/translator/stm/src_stm/et.h b/pypy/translator/stm/src_stm/et.h
--- a/pypy/translator/stm/src_stm/et.h
+++ b/pypy/translator/stm/src_stm/et.h
@@ -58,8 +58,8 @@
            (long*)(((char*)(base)) + ((offset) & ~(sizeof(void*)-1))))  \
         >> (8 * ((offset) & (sizeof(void*)-1))))
 
-unsigned long stm_read_partial_word(int fieldsize, char *addr);
-void stm_write_partial_word(int fieldsize, char *addr, unsigned long nval);
+unsigned long stm_read_partial_word(int fieldsize, void *addr);
+void stm_write_partial_word(int fieldsize, void *addr, unsigned long nval);
 
 double stm_read_double(long *addr);
 void stm_write_double(long *addr, double val);

From noreply at buildbot.pypy.org  Fri Jan 20 10:03:09 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 20 Jan 2012 10:03:09 +0100 (CET)
Subject: [pypy-commit] pypy stm: Bah. Temporary workaround: can't use bool_t
 because casting to
Message-ID: <20120120090309.E1FFC82CF8@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stm
Changeset: r51512:3bb490226684
Date: 2012-01-20 00:16 +0100
http://bitbucket.org/pypy/pypy/changeset/3bb490226684/

Log:	Bah. Temporary workaround: can't use bool_t because casting to a
	bool_t has unexpected results for stm_*_partial_word()

diff --git a/pypy/translator/c/src/g_prerequisite.h b/pypy/translator/c/src/g_prerequisite.h
--- a/pypy/translator/c/src/g_prerequisite.h
+++ b/pypy/translator/c/src/g_prerequisite.h
@@ -19,7 +19,7 @@
 #include 
 
 
-#ifdef __GNUC__       /* other platforms too, probably */
+#if 0 //def __GNUC__       /* other platforms too, probably */  XXX FIX ME
 typedef _Bool bool_t;
 #else
 typedef unsigned char bool_t;
diff --git a/pypy/translator/stm/src_stm/et.c b/pypy/translator/stm/src_stm/et.c
--- a/pypy/translator/stm/src_stm/et.c
+++ b/pypy/translator/stm/src_stm/et.c
@@ -513,6 +513,9 @@
 long stm_read_word(long* addr)
 {
   struct tx_descriptor *d = thread_descriptor;
+#ifdef RPY_STM_ASSERT
+  assert((((long)addr) & (sizeof(void*)-1)) == 0);
+#endif
   if (!d->transaction_active)
     return *addr;
 
@@ -576,6 +579,7 @@
 void stm_write_word(long* addr, long val)
 {
   struct tx_descriptor *d = thread_descriptor;
+  assert((((long)addr) & (sizeof(void*)-1)) == 0);
   if (!d->transaction_active) {
     *addr = val;
     return;
@@ -864,7 +868,7 @@
 unsigned long stm_read_partial_word(int fieldsize, void *addr)
 {
   int misalignment = ((long)addr) & (sizeof(void*)-1);
-  long *p = (long*)((char *)addr - misalignment);
+  long *p = (long*)(((char *)addr) - misalignment);
   unsigned long word = stm_read_word(p);
   return word >> (misalignment * 8);
 }
@@ -873,7 +877,7 @@
 void stm_write_partial_word(int fieldsize, void *addr, unsigned long nval)
 {
   int misalignment = ((long)addr) & (sizeof(void*)-1);
-  long *p = (long*)((char *)addr - misalignment);
+  long *p = (long*)(((char *)addr) - misalignment);
   long val = nval << (misalignment * 8);
   long word = stm_read_word(p);
   long mask = ((1L << (fieldsize * 8)) - 1) << (misalignment * 8);
diff --git a/pypy/translator/stm/test/test_llstm.py b/pypy/translator/stm/test/test_llstm.py
--- a/pypy/translator/stm/test/test_llstm.py
+++ b/pypy/translator/stm/test/test_llstm.py
@@ -97,7 +97,13 @@
     assert float(stm_getfield(a, 'sb')) == float(rs1b)
     stm_setfield(a, 'x', 42 * a.y)
     stm_setfield(a, 'c1', '(')
+    assert stm_getfield(a, 'c1') == '('
+    assert stm_getfield(a, 'c2') == '*'
+    assert stm_getfield(a, 'c3') == '#'
     stm_setfield(a, 'c2', '?')
+    assert stm_getfield(a, 'c1') == '('
+    assert stm_getfield(a, 'c2') == '?'
+    assert stm_getfield(a, 'c3') == '#'
     stm_setfield(a, 'c3', ')')
     stm_setfield(a, 'l', rll2)
     stm_setfield(a, 'f', rf2)

From noreply at buildbot.pypy.org  Fri Jan 20 10:12:28 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 20 Jan 2012 10:12:28 +0100 (CET)
Subject: [pypy-commit] pypy stm: Skip this import if it fails because of
	_weakref
Message-ID: <20120120091228.A8D4F82CF8@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stm
Changeset: r51513:ffedd17ff570
Date: 2012-01-20 10:12 +0100
http://bitbucket.org/pypy/pypy/changeset/ffedd17ff570/

Log:	Skip this import if it fails because of _weakref

diff --git a/lib-python/modified-2.7/UserDict.py b/lib-python/modified-2.7/UserDict.py
--- a/lib-python/modified-2.7/UserDict.py
+++ b/lib-python/modified-2.7/UserDict.py
@@ -85,8 +85,12 @@
     def __iter__(self):
         return iter(self.data)
 
-import _abcoll
-_abcoll.MutableMapping.register(IterableUserDict)
+try:
+    import _abcoll
+except ImportError:
+    pass    # e.g. no '_weakref' module on this pypy
+else:
+    _abcoll.MutableMapping.register(IterableUserDict)
 
 
 class DictMixin:

From noreply at buildbot.pypy.org  Fri Jan 20 10:15:32 2012
From: noreply at buildbot.pypy.org (bivab)
Date: Fri, 20 Jan 2012 10:15:32 +0100 (CET)
Subject: [pypy-commit] extradoc extradoc: Planning for today
Message-ID: <20120120091532.6359482CF8@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider 
Branch: extradoc
Changeset: r4047:586ccee3a515
Date: 2012-01-20 10:14 +0100
http://bitbucket.org/pypy/extradoc/changeset/586ccee3a515/

Log:	Planning for today

diff --git a/sprintinfo/leysin-winter-2012/planning.txt b/sprintinfo/leysin-winter-2012/planning.txt
--- a/sprintinfo/leysin-winter-2012/planning.txt
+++ b/sprintinfo/leysin-winter-2012/planning.txt
@@ -14,17 +14,19 @@
 
 * review the JVM backend pull request (DONE)
 
-* py3k (romain, anto)
+* py3k (romain, anto) (some progress)
 
 * ffistruct
 
 * Cython backend
 
-* Debug the ARM backend (bivab, armin around)
+* Debug the ARM backend (bivab, armin around) (some progress)
 
 * STM
   - refactored the RPython API (DONE)
-  - app-level transaction module (armin, bivab around)
+  - app-level transaction module (DONE)
   - start work on the GC
 
 * concurrent-marksweep GC
+
+* lightweight tracing experiment (everyone)

From noreply at buildbot.pypy.org  Fri Jan 20 10:54:44 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 20 Jan 2012 10:54:44 +0100 (CET)
Subject: [pypy-commit] pypy stm: add 5 operations that cause a pypy-stm to
 go into inevitable mode
Message-ID: <20120120095444.19D1D82CF8@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stm
Changeset: r51514:ceeb6b131e29
Date: 2012-01-20 10:54 +0100
http://bitbucket.org/pypy/pypy/changeset/ceeb6b131e29/

Log:	add 5 operations that cause a pypy-stm to go into inevitable mode

diff --git a/pypy/translator/stm/transform.py b/pypy/translator/stm/transform.py
--- a/pypy/translator/stm/transform.py
+++ b/pypy/translator/stm/transform.py
@@ -8,7 +8,8 @@
 
 ALWAYS_ALLOW_OPERATIONS = set([
     'direct_call', 'force_cast', 'keepalive', 'cast_ptr_to_adr',
-    'debug_print', 'debug_assert',
+    'debug_print', 'debug_assert', 'cast_opaque_ptr', 'hint',
+    'indirect_call', 'stack_current',
     ])
 ALWAYS_ALLOW_OPERATIONS |= set(lloperation.enum_tryfold_ops())
 
@@ -210,6 +211,8 @@
         flags = op.args[1].value
         return flags['flavor'] == 'gc'
 
+    stt_malloc_nonmovable = stt_malloc
+
     def stt_gc_stack_bottom(self, newoperations, op):
 ##        self.seen_gc_stack_bottom = True
         newoperations.append(op)

From noreply at buildbot.pypy.org  Fri Jan 20 11:21:12 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 20 Jan 2012 11:21:12 +0100 (CET)
Subject: [pypy-commit] pypy numpy-indexing-by-arrays-2: merge default
Message-ID: <20120120102112.5902F82CF8@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: numpy-indexing-by-arrays-2
Changeset: r51515:54c7d0197d66
Date: 2012-01-20 11:15 +0200
http://bitbucket.org/pypy/pypy/changeset/54c7d0197d66/

Log:	merge default

diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -2,6 +2,9 @@
 *.py[co]
 *~
 .*.swp
+.idea
+.project
+.pydevproject
 
 syntax: regexp
 ^testresult$
diff --git a/lib_pypy/numpypy/__init__.py b/lib_pypy/numpypy/__init__.py
--- a/lib_pypy/numpypy/__init__.py
+++ b/lib_pypy/numpypy/__init__.py
@@ -1,2 +1,2 @@
 from _numpypy import *
-from .fromnumeric import *
+from .core import *
diff --git a/lib_pypy/numpypy/core/__init__.py b/lib_pypy/numpypy/core/__init__.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/numpypy/core/__init__.py
@@ -0,0 +1,1 @@
+from .fromnumeric import *
diff --git a/lib_pypy/numpypy/fromnumeric.py b/lib_pypy/numpypy/core/fromnumeric.py
rename from lib_pypy/numpypy/fromnumeric.py
rename to lib_pypy/numpypy/core/fromnumeric.py
--- a/lib_pypy/numpypy/fromnumeric.py
+++ b/lib_pypy/numpypy/core/fromnumeric.py
@@ -85,7 +85,7 @@
     array([4, 3, 6])
 
     """
-    raise NotImplemented('Waiting on interp level method')
+    raise NotImplementedError('Waiting on interp level method')
 
 
 # not deprecated --- copy if necessary, view otherwise
@@ -273,7 +273,7 @@
             [-1, -2, -3, -4, -5]]])
 
     """
-    raise NotImplemented('Waiting on interp level method')
+    raise NotImplementedError('Waiting on interp level method')
 
 
 def repeat(a, repeats, axis=None):
@@ -315,7 +315,7 @@
            [3, 4]])
 
     """
-    raise NotImplemented('Waiting on interp level method')
+    raise NotImplementedError('Waiting on interp level method')
 
 
 def put(a, ind, v, mode='raise'):
@@ -366,7 +366,7 @@
     array([ 0,  1,  2,  3, -5])
 
     """
-    raise NotImplemented('Waiting on interp level method')
+    raise NotImplementedError('Waiting on interp level method')
 
 
 def swapaxes(a, axis1, axis2):
@@ -410,7 +410,7 @@
             [3, 7]]])
 
     """
-    raise NotImplemented('Waiting on interp level method')
+    raise NotImplementedError('Waiting on interp level method')
 
 
 def transpose(a, axes=None):
@@ -451,8 +451,11 @@
     (2, 1, 3)
 
     """
-    raise NotImplemented('Waiting on interp level method')
-
+    if axes is not None:
+        raise NotImplementedError('No "axes" arg yet.')
+    if not hasattr(a, 'T'):
+        a = numpypy.array(a)
+    return a.T
 
 def sort(a, axis=-1, kind='quicksort', order=None):
     """
@@ -553,7 +556,7 @@
           dtype=[('name', '|S10'), ('height', ' 1:
             raise OperationError(space.w_ValueError, space.wrap(
@@ -790,6 +794,9 @@
     def copy(self, space):
         return Scalar(self.dtype, self.value)
 
+    def fill(self, space, w_value):
+        self.value = self.dtype.coerce(space, w_value)
+
     def create_sig(self):
         return signature.ScalarSignature(self.dtype)
 
@@ -895,7 +902,7 @@
     Intermediate class for performing binary operations.
     """
     _immutable_fields_ = ['left', 'right']
-    
+
     def __init__(self, ufunc, name, shape, calc_dtype, res_dtype, left, right):
         VirtualArray.__init__(self, name, shape, res_dtype)
         self.ufunc = ufunc
@@ -935,7 +942,7 @@
     def __init__(self, shape, dtype, left, right):
         Call2.__init__(self, None, 'sliceloop', shape, dtype, dtype, left,
                        right)
-    
+
     def create_sig(self):
         lsig = self.left.create_sig()
         rsig = self.right.create_sig()
@@ -954,7 +961,7 @@
     when we'll make AxisReduce lazy
     """
     _immutable_fields_ = ['left', 'right']
-    
+
     def __init__(self, ufunc, name, shape, dtype, left, right, dim):
         Call2.__init__(self, ufunc, name, shape, dtype, dtype,
                        left, right)
@@ -1026,14 +1033,14 @@
         if size < 1:
             builder.append('[]')
             return
-        elif size == 1:
-            builder.append(dtype.itemtype.str_format(self.getitem(0)))
-            return
         if size > 1000:
             # Once this goes True it does not go back to False for recursive
             # calls
             use_ellipsis = True
         ndims = len(self.shape)
+        if ndims == 0:
+            builder.append(dtype.itemtype.str_format(self.getitem(0)))
+            return
         i = 0
         builder.append('[')
         if ndims > 1:
@@ -1168,6 +1175,9 @@
         array.setslice(space, self)
         return array
 
+    def fill(self, space, w_value):
+        self.setslice(space, scalar_w(space, self.dtype, w_value))
+
 
 class ViewArray(ConcreteArray):
     def create_sig(self):
@@ -1391,6 +1401,8 @@
     var = interp2app(BaseArray.descr_var),
     std = interp2app(BaseArray.descr_std),
 
+    fill = interp2app(BaseArray.descr_fill),
+
     copy = interp2app(BaseArray.descr_copy),
     reshape = interp2app(BaseArray.descr_reshape),
     tolist = interp2app(BaseArray.descr_tolist),
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -451,6 +451,7 @@
 
             ("fabs", "fabs", 1, {"promote_to_float": True}),
             ("floor", "floor", 1, {"promote_to_float": True}),
+            ("ceil", "ceil", 1, {"promote_to_float": True}),
             ("exp", "exp", 1, {"promote_to_float": True}),
 
             ('sqrt', 'sqrt', 1, {'promote_to_float': True}),
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -166,14 +166,11 @@
         # You can't subclass dtype
         raises(TypeError, type, "Foo", (dtype,), {})
 
-    def test_new(self):
-        import _numpypy as np
-        assert np.int_(4) == 4
-        assert np.float_(3.4) == 3.4
+    def test_aliases(self):
+        from _numpypy import dtype
 
-    def test_pow(self):
-        from _numpypy import int_
-        assert int_(4) ** 2 == 16
+        assert dtype("float") is dtype(float)
+
 
 class AppTestTypes(BaseNumpyAppTest):
     def test_abstract_types(self):
@@ -189,6 +186,15 @@
         raises(TypeError, numpy.floating, 0)
         raises(TypeError, numpy.inexact, 0)
 
+    def test_new(self):
+        import _numpypy as np
+        assert np.int_(4) == 4
+        assert np.float_(3.4) == 3.4
+
+    def test_pow(self):
+        from _numpypy import int_
+        assert int_(4) ** 2 == 16
+
     def test_bool(self):
         import _numpypy as numpy
 
@@ -318,7 +324,7 @@
         else:
             raises(OverflowError, numpy.int64, 9223372036854775807)
             raises(OverflowError, numpy.int64, '9223372036854775807')
-        
+
         raises(OverflowError, numpy.int64, 9223372036854775808)
         raises(OverflowError, numpy.int64, '9223372036854775808')
 
diff --git a/pypy/module/micronumpy/test/test_module.py b/pypy/module/micronumpy/test/test_module.py
--- a/pypy/module/micronumpy/test/test_module.py
+++ b/pypy/module/micronumpy/test/test_module.py
@@ -2,16 +2,11 @@
 
 
 class AppTestNumPyModule(BaseNumpyAppTest):
-    def test_mean(self):
-        from _numpypy import array, mean
-        assert mean(array(range(5))) == 2.0
-        assert mean(range(5)) == 2.0
-
     def test_average(self):
         from _numpypy import array, average
         assert average(range(10)) == 4.5
         assert average(array(range(10))) == 4.5
-        
+
     def test_sum(self):
         from _numpypy import array, sum
         assert sum(range(10)) == 45
@@ -21,7 +16,7 @@
         from _numpypy import array, min
         assert min(range(10)) == 0
         assert min(array(range(10))) == 0
-        
+
     def test_max(self):
         from _numpypy import array, max
         assert max(range(10)) == 9
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -728,15 +728,16 @@
         assert d[1] == 12
 
     def test_mean(self):
-        from _numpypy import array, mean
+        from _numpypy import array
         a = array(range(5))
         assert a.mean() == 2.0
         assert a[:4].mean() == 1.5
         a = array(range(105)).reshape(3, 5, 7)
-        b = mean(a, axis=0)
-        b[0,0]==35.
+        b = a.mean(axis=0)
+        b[0, 0]==35.
+        assert a.mean(axis=0)[0, 0] == 35
         assert (b == array(range(35, 70), dtype=float).reshape(5, 7)).all()
-        assert (mean(a, 2) == array(range(0, 15), dtype=float).reshape(3, 5) * 7 + 3).all()
+        assert (a.mean(2) == array(range(0, 15), dtype=float).reshape(3, 5) * 7 + 3).all()
 
     def test_sum(self):
         from _numpypy import array
@@ -757,6 +758,7 @@
         assert array([]).sum() == 0.0
         raises(ValueError, 'array([]).max()')
         assert (a.sum(0) == [30, 35, 40]).all()
+        assert (a.sum(axis=0) == [30, 35, 40]).all()
         assert (a.sum(1) == [3, 12, 21, 30, 39]).all()
         assert (a.max(0) == [12, 13, 14]).all()
         assert (a.max(1) == [2, 5, 8, 11, 14]).all()
@@ -771,6 +773,8 @@
         assert ((a + a).T.sum(2).T == (a + a).sum(0)).all()
         assert (a.reshape(1,-1).sum(0) == range(105)).all()
         assert (a.reshape(1,-1).sum(1) == 5460)
+        assert (array([[1,2],[3,4]]).prod(0) == [3, 8]).all()
+        assert (array([[1,2],[3,4]]).prod(1) == [2, 12]).all()
 
     def test_identity(self):
         from _numpypy import identity, array
@@ -1309,6 +1313,26 @@
         assert arange(3)[array([-3])] == 0
         raises(IndexError,'arange(3)[array([-15])]')
 
+    def test_fill(self):
+        from _numpypy import array
+        a = array([1, 2, 3])
+        a.fill(10)
+        assert (a == [10, 10, 10]).all()
+        a.fill(False)
+        assert (a == [0, 0, 0]).all()
+        b = a[:1]
+        b.fill(4)
+        assert (b == [4]).all()
+        assert (a == [4, 0, 0]).all()
+
+        c = b + b
+        c.fill(27)
+        assert (c == [27]).all()
+
+        d = array(10)
+        d.fill(100)
+        assert d == 100
+
     def test_array_indexing_bool(self):
         from _numpypy import arange
         a = arange(10)
@@ -1464,9 +1488,11 @@
         assert repr(a) == "array(0.0)"
         a = array(0.2)
         assert repr(a) == "array(0.2)"
+        a = array([2])
+        assert repr(a) == "array([2])"
 
     def test_repr_multi(self):
-        from _numpypy import arange, zeros
+        from _numpypy import arange, zeros, array
         a = zeros((3, 4))
         assert repr(a) == '''array([[0.0, 0.0, 0.0, 0.0],
        [0.0, 0.0, 0.0, 0.0],
@@ -1489,6 +1515,9 @@
        [498, 999],
        [499, 1000],
        [500, 1001]])'''
+        a = arange(2).reshape((2,1))
+        assert repr(a) == '''array([[0],
+       [1]])'''
 
     def test_repr_slice(self):
         from _numpypy import array, zeros
@@ -1573,18 +1602,3 @@
         a = arange(0, 0.8, 0.1)
         assert len(a) == 8
         assert arange(False, True, True).dtype is dtype(int)
-
-
-class AppTestRanges(BaseNumpyAppTest):
-    def test_app_reshape(self):
-        from _numpypy import arange, array, dtype, reshape
-        a = arange(12)
-        b = reshape(a, (3, 4))
-        assert b.shape == (3, 4)
-        a = range(12)
-        b = reshape(a, (3, 4))
-        assert b.shape == (3, 4)
-        a = array(range(105)).reshape(3, 5, 7)
-        assert a.reshape(1, -1).shape == (1, 105)
-        assert a.reshape(1, 1, -1).shape == (1, 1, 105)
-        assert a.reshape(-1, 1, 1).shape == (105, 1, 1)
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -190,14 +190,24 @@
         for i in range(3):
             assert c[i] == a[i] - b[i]
 
-    def test_floor(self):
-        from _numpypy import array, floor
-
-        reference = [-2.0, -1.0, 0.0, 1.0, 1.0]
-        a = array([-1.4, -1.0, 0.0, 1.0, 1.4])
+    def test_floorceil(self):
+        from _numpypy import array, floor, ceil
+        import math
+        reference = [-2.0, -2.0, -1.0, 0.0, 1.0, 1.0, 0]
+        a = array([-1.4, -1.5, -1.0, 0.0, 1.0, 1.4, 0.5])
         b = floor(a)
         for i in range(5):
             assert b[i] == reference[i]
+        reference = [-1.0, -1.0, -1.0, 0.0, 1.0, 2.0, 1.0]
+        a = array([-1.4, -1.5, -1.0, 0.0, 1.0, 1.4, 0.5])
+        b = ceil(a)
+        assert (reference == b).all()
+        inf = float("inf")
+        data = [1.5, 2.9999, -1.999, inf]
+        results = [math.floor(x) for x in data]
+        assert (floor(data) == results).all()
+        results = [math.ceil(x) for x in data]
+        assert (ceil(data) == results).all()
 
     def test_copysign(self):
         from _numpypy import array, copysign
@@ -238,7 +248,7 @@
             assert b[i] == math.sin(a[i])
 
         a = sin(array([True, False], dtype=bool))
-        assert abs(a[0] - sin(1)) < 1e-7 # a[0] will be less precise
+        assert abs(a[0] - sin(1)) < 1e-7  # a[0] will be less precise
         assert a[1] == 0.0
 
     def test_cos(self):
@@ -259,7 +269,6 @@
         for i in range(len(a)):
             assert b[i] == math.tan(a[i])
 
-
     def test_arcsin(self):
         import math
         from _numpypy import array, arcsin
@@ -283,7 +292,6 @@
         for i in range(len(a)):
             assert b[i] == math.acos(a[i])
 
-
         a = array([-10, -1.5, -1.01, 1.01, 1.5, 10, float('nan'), float('inf'), float('-inf')])
         b = arccos(a)
         for f in b:
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -350,7 +350,8 @@
         self.check_trace_count(1)
         self.check_simple_loop({'getinteriorfield_raw': 2, 'float_add': 1,
                                 'setinteriorfield_raw': 1, 'int_add': 2,
-                                'int_eq': 1, 'guard_false': 1, 'jump': 1})
+                                'int_eq': 1, 'guard_false': 1, 'jump': 1,
+                                'arraylen_gc': 1})
 
     def define_virtual_slice():
         return """
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -391,6 +391,10 @@
         return math.floor(v)
 
     @simple_unary_op
+    def ceil(self, v):
+        return math.ceil(v)
+
+    @simple_unary_op
     def exp(self, v):
         try:
             return math.exp(v)
diff --git a/pypy/module/pypyjit/__init__.py b/pypy/module/pypyjit/__init__.py
--- a/pypy/module/pypyjit/__init__.py
+++ b/pypy/module/pypyjit/__init__.py
@@ -11,6 +11,7 @@
         'set_optimize_hook': 'interp_resop.set_optimize_hook',
         'set_abort_hook': 'interp_resop.set_abort_hook',
         'ResOperation': 'interp_resop.WrappedOp',
+        'DebugMergePoint': 'interp_resop.DebugMergePoint',
         'Box': 'interp_resop.WrappedBox',
     }
 
diff --git a/pypy/module/pypyjit/interp_resop.py b/pypy/module/pypyjit/interp_resop.py
--- a/pypy/module/pypyjit/interp_resop.py
+++ b/pypy/module/pypyjit/interp_resop.py
@@ -1,5 +1,6 @@
 
-from pypy.interpreter.typedef import TypeDef, GetSetProperty
+from pypy.interpreter.typedef import (TypeDef, GetSetProperty,
+     interp_attrproperty, interp_attrproperty_w)
 from pypy.interpreter.baseobjspace import Wrappable
 from pypy.interpreter.gateway import unwrap_spec, interp2app, NoneNotWrapped
 from pypy.interpreter.pycode import PyCode
@@ -10,6 +11,7 @@
 from pypy.jit.metainterp.resoperation import rop, AbstractResOp
 from pypy.rlib.nonconst import NonConstant
 from pypy.rlib import jit_hooks
+from pypy.module.pypyjit.interp_jit import pypyjitdriver
 
 class Cache(object):
     in_recursion = False
@@ -111,13 +113,24 @@
 
 def wrap_oplist(space, logops, operations, ops_offset=None):
     l_w = []
+    jitdrivers_sd = logops.metainterp_sd.jitdrivers_sd
     for op in operations:
         if ops_offset is None:
             ofs = -1
         else:
             ofs = ops_offset.get(op, 0)
-        l_w.append(WrappedOp(jit_hooks._cast_to_gcref(op), ofs,
-                             logops.repr_of_resop(op)))
+        if op.opnum == rop.DEBUG_MERGE_POINT:
+            jd_sd = jitdrivers_sd[op.getarg(0).getint()]
+            greenkey = op.getarglist()[2:]
+            repr = jd_sd.warmstate.get_location_str(greenkey)
+            w_greenkey = wrap_greenkey(space, jd_sd.jitdriver, greenkey, repr)
+            l_w.append(DebugMergePoint(space, jit_hooks._cast_to_gcref(op),
+                                       logops.repr_of_resop(op),
+                                       jd_sd.jitdriver.name,
+                                       w_greenkey))
+        else:
+            l_w.append(WrappedOp(jit_hooks._cast_to_gcref(op), ofs,
+                                 logops.repr_of_resop(op)))
     return l_w
 
 class WrappedBox(Wrappable):
@@ -150,6 +163,15 @@
         llres = res.llbox
     return WrappedOp(jit_hooks.resop_new(num, args, llres), offset, repr)
 
+ at unwrap_spec(repr=str, jd_name=str)
+def descr_new_dmp(space, w_tp, w_args, repr, jd_name, w_greenkey):
+    args = [space.interp_w(WrappedBox, w_arg).llbox for w_arg in
+            space.listview(w_args)]
+    num = rop.DEBUG_MERGE_POINT
+    return DebugMergePoint(space,
+                           jit_hooks.resop_new(num, args, jit_hooks.emptyval()),
+                           repr, jd_name, w_greenkey)
+
 class WrappedOp(Wrappable):
     """ A class representing a single ResOperation, wrapped nicely
     """
@@ -182,6 +204,25 @@
         box = space.interp_w(WrappedBox, w_box)
         jit_hooks.resop_setresult(self.op, box.llbox)
 
+class DebugMergePoint(WrappedOp):
+    def __init__(self, space, op, repr_of_resop, jd_name, w_greenkey):
+        WrappedOp.__init__(self, op, -1, repr_of_resop)
+        self.w_greenkey = w_greenkey
+        self.jd_name = jd_name
+
+    def get_pycode(self, space):
+        if self.jd_name == pypyjitdriver.name:
+            return space.getitem(self.w_greenkey, space.wrap(0))
+        raise OperationError(space.w_AttributeError, space.wrap("This DebugMergePoint doesn't belong to the main Python JitDriver"))
+
+    def get_bytecode_no(self, space):
+        if self.jd_name == pypyjitdriver.name:
+            return space.getitem(self.w_greenkey, space.wrap(1))
+        raise OperationError(space.w_AttributeError, space.wrap("This DebugMergePoint doesn't belong to the main Python JitDriver"))
+
+    def get_jitdriver_name(self, space):
+        return space.wrap(self.jd_name)
+
 WrappedOp.typedef = TypeDef(
     'ResOperation',
     __doc__ = WrappedOp.__doc__,
@@ -195,3 +236,15 @@
                             WrappedOp.descr_setresult)
 )
 WrappedOp.acceptable_as_base_class = False
+
+DebugMergePoint.typedef = TypeDef(
+    'DebugMergePoint', WrappedOp.typedef,
+    __new__ = interp2app(descr_new_dmp),
+    greenkey = interp_attrproperty_w("w_greenkey", cls=DebugMergePoint),
+    pycode = GetSetProperty(DebugMergePoint.get_pycode),
+    bytecode_no = GetSetProperty(DebugMergePoint.get_bytecode_no),
+    jitdriver_name = GetSetProperty(DebugMergePoint.get_jitdriver_name),
+)
+DebugMergePoint.acceptable_as_base_class = False
+
+
diff --git a/pypy/module/pypyjit/policy.py b/pypy/module/pypyjit/policy.py
--- a/pypy/module/pypyjit/policy.py
+++ b/pypy/module/pypyjit/policy.py
@@ -127,7 +127,7 @@
                        'imp', 'sys', 'array', '_ffi', 'itertools', 'operator',
                        'posix', '_socket', '_sre', '_lsprof', '_weakref',
                        '__pypy__', 'cStringIO', '_collections', 'struct',
-                       'mmap', 'marshal']:
+                       'mmap', 'marshal', '_codecs']:
             if modname == 'pypyjit' and 'interp_resop' in rest:
                 return False
             return True
diff --git a/pypy/module/pypyjit/test/test_jit_hook.py b/pypy/module/pypyjit/test/test_jit_hook.py
--- a/pypy/module/pypyjit/test/test_jit_hook.py
+++ b/pypy/module/pypyjit/test/test_jit_hook.py
@@ -92,6 +92,7 @@
         cls.w_on_compile_bridge = space.wrap(interp2app(interp_on_compile_bridge))
         cls.w_on_abort = space.wrap(interp2app(interp_on_abort))
         cls.w_int_add_num = space.wrap(rop.INT_ADD)
+        cls.w_dmp_num = space.wrap(rop.DEBUG_MERGE_POINT)
         cls.w_on_optimize = space.wrap(interp2app(interp_on_optimize))
         cls.orig_oplist = oplist
 
@@ -117,6 +118,10 @@
         assert elem[2][2] == False
         assert len(elem[3]) == 4
         int_add = elem[3][0]
+        dmp = elem[3][1]
+        assert isinstance(dmp, pypyjit.DebugMergePoint)
+        assert dmp.pycode is self.f.func_code
+        assert dmp.greenkey == (self.f.func_code, 0, False)
         #assert int_add.name == 'int_add'
         assert int_add.num == self.int_add_num
         self.on_compile_bridge()
@@ -211,3 +216,18 @@
         assert op.getarg(0).getint() == 4
         op.result = box
         assert op.result.getint() == 1
+
+    def test_creation_dmp(self):
+        from pypyjit import DebugMergePoint, Box
+
+        def f():
+            pass
+
+        op = DebugMergePoint([Box(0)], 'repr', 'pypyjit', (f.func_code, 0, 0))
+        assert op.bytecode_no == 0
+        assert op.pycode is f.func_code
+        assert repr(op) == 'repr'
+        assert op.jitdriver_name == 'pypyjit'
+        assert op.num == self.dmp_num
+        op = DebugMergePoint([Box(0)], 'repr', 'notmain', ('str',))
+        raises(AttributeError, 'op.pycode')
diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py
--- a/pypy/module/sys/version.py
+++ b/pypy/module/sys/version.py
@@ -10,7 +10,7 @@
 CPYTHON_VERSION            = (2, 7, 1, "final", 42)   #XXX # sync patchlevel.h
 CPYTHON_API_VERSION        = 1013   #XXX # sync with include/modsupport.h
 
-PYPY_VERSION               = (1, 7, 1, "dev", 0)    #XXX # sync patchlevel.h
+PYPY_VERSION               = (1, 8, 1, "dev", 0)    #XXX # sync patchlevel.h
 
 if platform.name == 'msvc':
     COMPILER_INFO = 'MSC v.%d 32 bit' % (platform.version * 10 + 600)
diff --git a/lib_pypy/numpypy/test/test_fromnumeric.py b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
rename from lib_pypy/numpypy/test/test_fromnumeric.py
rename to pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
--- a/lib_pypy/numpypy/test/test_fromnumeric.py
+++ b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
@@ -1,7 +1,7 @@
-
 from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
 
-class AppTestFromNumeric(BaseNumpyAppTest):     
+
+class AppTestFromNumeric(BaseNumpyAppTest):
     def test_argmax(self):
         # tests taken from numpy/core/fromnumeric.py docstring
         from numpypy import array, arange, argmax
@@ -18,12 +18,12 @@
         from numpypy import array, arange, argmin
         a = arange(6).reshape((2,3))
         assert argmin(a) == 0
-        # assert (argmax(a, axis=0) == array([0, 0, 0])).all()
-        # assert (argmax(a, axis=1) == array([0, 0])).all()
+        assert (argmin(a, axis=0) == array([0, 0, 0])).all()
+        assert (argmin(a, axis=1) == array([0, 0])).all()
         b = arange(6)
         b[1] = 0
         assert argmin(b) == 0
-   
+
     def test_shape(self):
         # tests taken from numpy/core/fromnumeric.py docstring
         from numpypy import array, identity, shape
@@ -44,7 +44,7 @@
         # assert (sum([[0, 1], [0, 5]], axis=1) == array([1, 5])).all()
         # If the accumulator is too small, overflow occurs:
         # assert ones(128, dtype=int8).sum(dtype=int8) == -128
-                                 
+
     def test_amin(self):
         # tests taken from numpy/core/fromnumeric.py docstring
         from numpypy import array, arange, amin
@@ -86,14 +86,14 @@
         assert ndim([[1,2,3],[4,5,6]]) == 2
         assert ndim(array([[1,2,3],[4,5,6]])) == 2
         assert ndim(1) == 0
-    
+
     def test_rank(self):
         # tests taken from numpy/core/fromnumeric.py docstring
         from numpypy import array, rank
         assert rank([[1,2,3],[4,5,6]]) == 2
         assert rank(array([[1,2,3],[4,5,6]])) == 2
         assert rank(1) == 0
-    
+
     def test_var(self):
         from numpypy import array, var
         a = array([[1,2],[3,4]])
@@ -107,3 +107,31 @@
         assert std(a) ==  1.1180339887498949
         # assert (std(a, axis=0) == array([ 1.,  1.])).all()
         # assert (std(a, axis=1) == array([ 0.5,  0.5]).all()
+
+    def test_mean(self):
+        from numpypy import array, mean
+        assert mean(array(range(5))) == 2.0
+        assert mean(range(5)) == 2.0
+
+    def test_reshape(self):
+        from numpypy import arange, array, dtype, reshape
+        a = arange(12)
+        b = reshape(a, (3, 4))
+        assert b.shape == (3, 4)
+        a = range(12)
+        b = reshape(a, (3, 4))
+        assert b.shape == (3, 4)
+        a = array(range(105)).reshape(3, 5, 7)
+        assert reshape(a, (1, -1)).shape == (1, 105)
+        assert reshape(a, (1, 1, -1)).shape == (1, 1, 105)
+        assert reshape(a, (-1, 1, 1)).shape == (105, 1, 1)
+    
+    def test_transpose(self):   
+        from numpypy import arange, array, transpose, ones
+        x = arange(4).reshape((2,2))
+        assert (transpose(x) == array([[0, 2],[1, 3]])).all()
+        # Once axes argument is implemented, add more tests
+        raises(NotImplementedError, "transpose(x, axes=(1, 0, 2))")
+        # x = ones((1, 2, 3))
+        # assert transpose(x, (1, 0, 2)).shape == (2, 1, 3)
+
diff --git a/pypy/objspace/std/test/test_unicodeobject.py b/pypy/objspace/std/test/test_unicodeobject.py
--- a/pypy/objspace/std/test/test_unicodeobject.py
+++ b/pypy/objspace/std/test/test_unicodeobject.py
@@ -64,6 +64,12 @@
         check(', '.join([u'a']), u'a')
         check(', '.join(['a', u'b']), u'a, b')
         check(u', '.join(['a', 'b']), u'a, b')
+        try:
+            u''.join([u'a', 2, 3])
+        except TypeError, e:
+            assert 'sequence item 1' in str(e)
+        else:
+            raise Exception("DID NOT RAISE")
 
     if sys.version_info >= (2,3):
         def test_contains_ex(self):
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -201,7 +201,7 @@
     return space.newbool(container.find(item) != -1)
 
 def unicode_join__Unicode_ANY(space, w_self, w_list):
-    list_w = space.unpackiterable(w_list)
+    list_w = space.listview(w_list)
     size = len(list_w)
 
     if size == 0:
@@ -216,22 +216,21 @@
 
 def _unicode_join_many_items(space, w_self, list_w, size):
     self = w_self._value
-    sb = UnicodeBuilder()
+    prealloc_size = len(self) * (size - 1)
+    for i in range(size):
+        try:
+            prealloc_size += len(space.unicode_w(list_w[i]))
+        except OperationError, e:
+            if not e.match(space, space.w_TypeError):
+                raise
+            raise operationerrfmt(space.w_TypeError,
+                        "sequence item %d: expected string or Unicode", i)
+    sb = UnicodeBuilder(prealloc_size)
     for i in range(size):
         if self and i != 0:
             sb.append(self)
         w_s = list_w[i]
-        if isinstance(w_s, W_UnicodeObject):
-            # shortcut for performance
-            sb.append(w_s._value)
-        else:
-            try:
-                sb.append(space.unicode_w(w_s))
-            except OperationError, e:
-                if not e.match(space, space.w_TypeError):
-                    raise
-                raise operationerrfmt(space.w_TypeError,
-                    "sequence item %d: expected string or Unicode", i)
+        sb.append(space.unicode_w(w_s))
     return space.wrap(sb.build())
 
 def hash__Unicode(space, w_uni):
diff --git a/pypy/rlib/jit_hooks.py b/pypy/rlib/jit_hooks.py
--- a/pypy/rlib/jit_hooks.py
+++ b/pypy/rlib/jit_hooks.py
@@ -52,7 +52,10 @@
     from pypy.jit.metainterp.history import ResOperation
 
     args = [_cast_to_box(llargs[i]) for i in range(len(llargs))]
-    res = _cast_to_box(llres)
+    if llres:
+        res = _cast_to_box(llres)
+    else:
+        res = None
     return _cast_to_gcref(ResOperation(no, args, res))
 
 @register_helper(annmodel.SomePtr(llmemory.GCREF))
diff --git a/pypy/rlib/longlong2float.py b/pypy/rlib/longlong2float.py
--- a/pypy/rlib/longlong2float.py
+++ b/pypy/rlib/longlong2float.py
@@ -79,19 +79,23 @@
 longlong2float = rffi.llexternal(
     "pypy__longlong2float", [rffi.LONGLONG], rffi.DOUBLE,
     _callable=longlong2float_emulator, compilation_info=eci,
-    _nowrapper=True, elidable_function=True, sandboxsafe=True)
+    _nowrapper=True, elidable_function=True, sandboxsafe=True,
+    oo_primitive="pypy__longlong2float")
 
 float2longlong = rffi.llexternal(
     "pypy__float2longlong", [rffi.DOUBLE], rffi.LONGLONG,
     _callable=float2longlong_emulator, compilation_info=eci,
-    _nowrapper=True, elidable_function=True, sandboxsafe=True)
+    _nowrapper=True, elidable_function=True, sandboxsafe=True,
+    oo_primitive="pypy__float2longlong")
 
 uint2singlefloat = rffi.llexternal(
     "pypy__uint2singlefloat", [rffi.UINT], rffi.FLOAT,
     _callable=uint2singlefloat_emulator, compilation_info=eci,
-    _nowrapper=True, elidable_function=True, sandboxsafe=True)
+    _nowrapper=True, elidable_function=True, sandboxsafe=True,
+    oo_primitive="pypy__uint2singlefloat")
 
 singlefloat2uint = rffi.llexternal(
     "pypy__singlefloat2uint", [rffi.FLOAT], rffi.UINT,
     _callable=singlefloat2uint_emulator, compilation_info=eci,
-    _nowrapper=True, elidable_function=True, sandboxsafe=True)
+    _nowrapper=True, elidable_function=True, sandboxsafe=True,
+    oo_primitive="pypy__singlefloat2uint")
diff --git a/pypy/rlib/objectmodel.py b/pypy/rlib/objectmodel.py
--- a/pypy/rlib/objectmodel.py
+++ b/pypy/rlib/objectmodel.py
@@ -420,7 +420,7 @@
                   vobj.concretetype.TO._gckind == 'gc')
         else:
             from pypy.rpython.ootypesystem import ootype
-            ok = isinstance(vobj.concretetype, ootype.Instance)
+            ok = isinstance(vobj.concretetype, (ootype.Instance, ootype.BuiltinType))
         if not ok:
             from pypy.rpython.error import TyperError
             raise TyperError("compute_unique_id() cannot be applied to"
diff --git a/pypy/rpython/lltypesystem/rclass.py b/pypy/rpython/lltypesystem/rclass.py
--- a/pypy/rpython/lltypesystem/rclass.py
+++ b/pypy/rpython/lltypesystem/rclass.py
@@ -510,7 +510,13 @@
         ctype = inputconst(Void, self.object_type)
         cflags = inputconst(Void, flags)
         vlist = [ctype, cflags]
-        vptr = llops.genop('malloc', vlist,
+        cnonmovable = self.classdef.classdesc.read_attribute(
+            '_alloc_nonmovable_', Constant(False))
+        if cnonmovable.value:
+            opname = 'malloc_nonmovable'
+        else:
+            opname = 'malloc'
+        vptr = llops.genop(opname, vlist,
                            resulttype = Ptr(self.object_type))
         ctypeptr = inputconst(CLASSTYPE, self.rclass.getvtable())
         self.setfield(vptr, '__class__', ctypeptr, llops)
diff --git a/pypy/rpython/lltypesystem/rtuple.py b/pypy/rpython/lltypesystem/rtuple.py
--- a/pypy/rpython/lltypesystem/rtuple.py
+++ b/pypy/rpython/lltypesystem/rtuple.py
@@ -27,6 +27,10 @@
 
     def newtuple(cls, llops, r_tuple, items_v):
         # items_v should have the lowleveltype of the internal reprs
+        assert len(r_tuple.items_r) == len(items_v)
+        for r_item, v_item in zip(r_tuple.items_r, items_v):
+            assert r_item.lowleveltype == v_item.concretetype
+        #
         if len(r_tuple.items_r) == 0:
             return inputconst(Void, ())    # a Void empty tuple
         c1 = inputconst(Void, r_tuple.lowleveltype.TO)
diff --git a/pypy/rpython/ootypesystem/ootype.py b/pypy/rpython/ootypesystem/ootype.py
--- a/pypy/rpython/ootypesystem/ootype.py
+++ b/pypy/rpython/ootypesystem/ootype.py
@@ -512,6 +512,7 @@
             "ll_append_char": Meth([CHARTP], Void),
             "ll_append": Meth([STRINGTP], Void),
             "ll_build": Meth([], STRINGTP),
+            "ll_getlength": Meth([], Signed),
             })
         self._setup_methods({})
 
@@ -1376,6 +1377,9 @@
     def _cast_to_object(self):
         return make_object(self)
 
+    def _identityhash(self):
+        return object.__hash__(self)
+
 class _string(_builtin_type):
 
     def __init__(self, STRING, value = ''):
@@ -1543,6 +1547,9 @@
         else:
             return make_unicode(u''.join(self._buf))
 
+    def ll_getlength(self):
+        return self.ll_build().ll_strlen()
+
 class _null_string_builder(_null_mixin(_string_builder), _string_builder):
     def __init__(self, STRING_BUILDER):
         self.__dict__["_TYPE"] = STRING_BUILDER
diff --git a/pypy/rpython/ootypesystem/rbuilder.py b/pypy/rpython/ootypesystem/rbuilder.py
--- a/pypy/rpython/ootypesystem/rbuilder.py
+++ b/pypy/rpython/ootypesystem/rbuilder.py
@@ -21,6 +21,10 @@
         builder.ll_append_char(char)
 
     @staticmethod
+    def ll_getlength(builder):
+        return builder.ll_getlength()
+
+    @staticmethod
     def ll_append(builder, string):
         builder.ll_append(string)
 
diff --git a/pypy/rpython/rrange.py b/pypy/rpython/rrange.py
--- a/pypy/rpython/rrange.py
+++ b/pypy/rpython/rrange.py
@@ -204,7 +204,10 @@
         v_index = hop.gendirectcall(self.ll_getnextindex, v_enumerate)
         hop2 = hop.copy()
         hop2.args_r = [self.r_baseiter]
+        r_item_src = self.r_baseiter.r_list.external_item_repr
+        r_item_dst = hop.r_result.items_r[1]
         v_item = self.r_baseiter.rtype_next(hop2)
+        v_item = hop.llops.convertvar(v_item, r_item_src, r_item_dst)
         return hop.r_result.newtuple(hop.llops, hop.r_result,
                                      [v_index, v_item])
 
diff --git a/pypy/rpython/test/test_rbuilder.py b/pypy/rpython/test/test_rbuilder.py
--- a/pypy/rpython/test/test_rbuilder.py
+++ b/pypy/rpython/test/test_rbuilder.py
@@ -124,9 +124,5 @@
     pass
 
 class TestOOtype(BaseTestStringBuilder, OORtypeMixin):
-    def test_string_getlength(self):
-        py.test.skip("getlength(): not implemented on ootype")
-    def test_unicode_getlength(self):
-        py.test.skip("getlength(): not implemented on ootype")
     def test_append_charpsize(self):
         py.test.skip("append_charpsize(): not implemented on ootype")
diff --git a/pypy/rpython/test/test_rbuiltin.py b/pypy/rpython/test/test_rbuiltin.py
--- a/pypy/rpython/test/test_rbuiltin.py
+++ b/pypy/rpython/test/test_rbuiltin.py
@@ -463,6 +463,31 @@
             assert x1 == intmask(x0)
             assert x3 == intmask(x2)
 
+    def test_id_on_builtins(self):
+        from pypy.rlib.objectmodel import compute_unique_id
+        from pypy.rlib.rstring import StringBuilder, UnicodeBuilder
+        def fn():
+            return (compute_unique_id("foo"),
+                    compute_unique_id(u"bar"),
+                    compute_unique_id([1]),
+                    compute_unique_id({"foo": 3}),
+                    compute_unique_id(StringBuilder()),
+                    compute_unique_id(UnicodeBuilder()))
+        res = self.interpret(fn, [])
+        for id in self.ll_unpack_tuple(res, 6):
+            assert isinstance(id, (int, r_longlong))
+
+    def test_uniqueness_of_id_on_strings(self):
+        from pypy.rlib.objectmodel import compute_unique_id
+        def fn(s1, s2):
+            return (compute_unique_id(s1), compute_unique_id(s2))
+
+        s1 = "foo"
+        s2 = ''.join(['f','oo'])
+        res = self.interpret(fn, [self.string_to_ll(s1), self.string_to_ll(s2)])
+        i1, i2 = self.ll_unpack_tuple(res, 2)
+        assert i1 != i2
+
     def test_cast_primitive(self):
         from pypy.rpython.annlowlevel import LowLevelAnnotatorPolicy
         def llf(u):
diff --git a/pypy/rpython/test/test_rclass.py b/pypy/rpython/test/test_rclass.py
--- a/pypy/rpython/test/test_rclass.py
+++ b/pypy/rpython/test/test_rclass.py
@@ -1130,6 +1130,18 @@
             assert sorted([u]) == [6]                    # 32-bit types
             assert sorted([i, r, d, l]) == [2, 3, 4, 5]  # 64-bit types
 
+    def test_nonmovable(self):
+        for (nonmovable, opname) in [(True, 'malloc_nonmovable'),
+                                     (False, 'malloc')]:
+            class A(object):
+                _alloc_nonmovable_ = nonmovable
+            def f():
+                return A()
+            t, typer, graph = self.gengraph(f, [])
+            assert summary(graph) == {opname: 1,
+                                      'cast_pointer': 1,
+                                      'setfield': 1}
+
 
 class TestOOtype(BaseTestRclass, OORtypeMixin):
 
diff --git a/pypy/rpython/test/test_rrange.py b/pypy/rpython/test/test_rrange.py
--- a/pypy/rpython/test/test_rrange.py
+++ b/pypy/rpython/test/test_rrange.py
@@ -169,6 +169,22 @@
         res = self.interpret(fn, [2])
         assert res == 789
 
+    def test_enumerate_instances(self):
+        class A:
+            pass
+        def fn(n):
+            a = A()
+            b = A()
+            a.k = 10
+            b.k = 20
+            for i, x in enumerate([a, b]):
+                if i == n:
+                    return x.k
+            return 5
+        res = self.interpret(fn, [1])
+        assert res == 20
+
+
 
 class TestLLtype(BaseTestRrange, LLRtypeMixin):
     from pypy.rpython.lltypesystem import rrange 
diff --git a/pypy/tool/jitlogparser/parser.py b/pypy/tool/jitlogparser/parser.py
--- a/pypy/tool/jitlogparser/parser.py
+++ b/pypy/tool/jitlogparser/parser.py
@@ -140,13 +140,15 @@
     bytecode_name = None
     is_bytecode = True
     inline_level = None
+    has_dmp = False
 
     def parse_code_data(self, arg):
         m = re.search('\w]+)[\.,] file \'(.+?)\'[\.,] line (\d+)> #(\d+) (\w+)',
                       arg)
         if m is None:
             # a non-code loop, like StrLiteralSearch or something
-            self.bytecode_name = arg
+            if arg:
+                self.bytecode_name = arg
         else:
             self.name, self.filename, lineno, bytecode_no, self.bytecode_name = m.groups()
             self.startlineno = int(lineno)
@@ -218,7 +220,7 @@
         self.inputargs = inputargs
         self.chunks = chunks
         for chunk in self.chunks:
-            if chunk.filename is not None:
+            if chunk.bytecode_name is not None:
                 self.startlineno = chunk.startlineno
                 self.filename = chunk.filename
                 self.name = chunk.name
diff --git a/pypy/tool/jitlogparser/test/test_parser.py b/pypy/tool/jitlogparser/test/test_parser.py
--- a/pypy/tool/jitlogparser/test/test_parser.py
+++ b/pypy/tool/jitlogparser/test/test_parser.py
@@ -283,3 +283,13 @@
     assert loops[-1].count == 1234
     assert loops[1].count == 123
     assert loops[2].count == 12
+
+def test_parse_nonpython():
+    loop = parse("""
+    []
+    debug_merge_point(0, 'random')
+    debug_merge_point(0, ' #15 COMPARE_OP')
+    """)
+    f = Function.from_operations(loop.operations, LoopStorage())
+    assert f.chunks[-1].filename == 'x.py'
+    assert f.filename is None
diff --git a/pypy/translator/jvm/builtin.py b/pypy/translator/jvm/builtin.py
--- a/pypy/translator/jvm/builtin.py
+++ b/pypy/translator/jvm/builtin.py
@@ -84,6 +84,9 @@
     (ootype.StringBuilder.__class__, "ll_build"):
     jvm.Method.v(jStringBuilder, "toString", (), jString),
 
+    (ootype.StringBuilder.__class__, "ll_getlength"):
+    jvm.Method.v(jStringBuilder, "length", (), jInt),
+
     (ootype.String.__class__, "ll_hash"):
     jvm.Method.v(jString, "hashCode", (), jInt),
 
diff --git a/pypy/translator/jvm/database.py b/pypy/translator/jvm/database.py
--- a/pypy/translator/jvm/database.py
+++ b/pypy/translator/jvm/database.py
@@ -358,7 +358,7 @@
         ootype.Unsigned:jvm.PYPYSERIALIZEUINT,
         ootype.SignedLongLong:jvm.LONGTOSTRINGL,
         ootype.UnsignedLongLong: jvm.PYPYSERIALIZEULONG,
-        ootype.Float:jvm.DOUBLETOSTRINGD,
+        ootype.Float:jvm.PYPYSERIALIZEDOUBLE,
         ootype.Bool:jvm.PYPYSERIALIZEBOOLEAN,
         ootype.Void:jvm.PYPYSERIALIZEVOID,
         ootype.Char:jvm.PYPYESCAPEDCHAR,
diff --git a/pypy/translator/jvm/metavm.py b/pypy/translator/jvm/metavm.py
--- a/pypy/translator/jvm/metavm.py
+++ b/pypy/translator/jvm/metavm.py
@@ -92,6 +92,7 @@
 CASTS = {
 #   FROM                      TO
     (ootype.Signed,           ootype.UnsignedLongLong): jvm.I2L,
+    (ootype.Unsigned,         ootype.UnsignedLongLong): jvm.I2L,
     (ootype.SignedLongLong,   ootype.Signed):           jvm.L2I,
     (ootype.UnsignedLongLong, ootype.Unsigned):         jvm.L2I,
     (ootype.UnsignedLongLong, ootype.Signed):           jvm.L2I,
diff --git a/pypy/translator/jvm/opcodes.py b/pypy/translator/jvm/opcodes.py
--- a/pypy/translator/jvm/opcodes.py
+++ b/pypy/translator/jvm/opcodes.py
@@ -101,6 +101,7 @@
     'jit_force_virtualizable':  Ignore,
     'jit_force_virtual':        DoNothing,
     'jit_force_quasi_immutable': Ignore,
+    'jit_is_virtual':           PushPrimitive(ootype.Bool, False),
 
     'debug_assert':              [], # TODO: implement?
     'debug_start_traceback':    Ignore,
diff --git a/pypy/translator/jvm/src/pypy/PyPy.java b/pypy/translator/jvm/src/pypy/PyPy.java
--- a/pypy/translator/jvm/src/pypy/PyPy.java
+++ b/pypy/translator/jvm/src/pypy/PyPy.java
@@ -283,6 +283,14 @@
         }
     }
 
+    public double pypy__longlong2float(long l) {
+        return Double.longBitsToDouble(l);
+    }
+
+    public long pypy__float2longlong(double d) {
+        return Double.doubleToRawLongBits(d);
+    }
+
     public double ooparse_float(String s) {
         try {
             return Double.parseDouble(s);
@@ -353,6 +361,19 @@
             return "False";
     }
 
+    public static String serialize_double(double d) {
+        if (Double.isNaN(d)) {
+            return "float(\"nan\")";
+        } else if (Double.isInfinite(d)) {
+            if (d > 0)
+                return "float(\"inf\")";
+            else
+                return "float(\"-inf\")";
+        } else {
+            return Double.toString(d);
+        }
+    }
+
     private static String format_char(char c) {
         String res = "\\x";
         if (c <= 0x0F) res = res + "0";
diff --git a/pypy/translator/jvm/test/runtest.py b/pypy/translator/jvm/test/runtest.py
--- a/pypy/translator/jvm/test/runtest.py
+++ b/pypy/translator/jvm/test/runtest.py
@@ -56,6 +56,7 @@
 
 # CLI could-be duplicate
 class JvmGeneratedSourceWrapper(object):
+
     def __init__(self, gensrc):
         """ gensrc is an instance of JvmGeneratedSource """
         self.gensrc = gensrc
diff --git a/pypy/translator/jvm/test/test_builder.py b/pypy/translator/jvm/test/test_builder.py
new file mode 100644
--- /dev/null
+++ b/pypy/translator/jvm/test/test_builder.py
@@ -0,0 +1,7 @@
+from pypy.translator.jvm.test.runtest import JvmTest
+from pypy.rpython.test.test_rbuilder import BaseTestStringBuilder
+import py
+
+class TestJvmStringBuilder(JvmTest, BaseTestStringBuilder):
+    def test_append_charpsize(self):
+        py.test.skip("append_charpsize(): not implemented on ootype")
diff --git a/pypy/translator/jvm/test/test_longlong2float.py b/pypy/translator/jvm/test/test_longlong2float.py
new file mode 100644
--- /dev/null
+++ b/pypy/translator/jvm/test/test_longlong2float.py
@@ -0,0 +1,20 @@
+from pypy.translator.jvm.test.runtest import JvmTest
+from pypy.rlib.longlong2float import *
+from pypy.rlib.test.test_longlong2float import enum_floats
+from pypy.rlib.test.test_longlong2float import fn as float2longlong2float
+import py
+
+class TestLongLong2Float(JvmTest):
+
+    def test_float2longlong_and_longlong2float(self):
+        def func(f):
+            return float2longlong2float(f)
+
+        for f in enum_floats():
+            assert repr(f) == repr(self.interpret(func, [f]))
+
+    def test_uint2singlefloat(self):
+        py.test.skip("uint2singlefloat is not implemented in ootype")
+
+    def test_singlefloat2uint(self):
+        py.test.skip("singlefloat2uint is not implemented in ootype")
diff --git a/pypy/translator/jvm/typesystem.py b/pypy/translator/jvm/typesystem.py
--- a/pypy/translator/jvm/typesystem.py
+++ b/pypy/translator/jvm/typesystem.py
@@ -955,6 +955,7 @@
 PYPYSERIALIZEUINT  =    Method.s(jPyPy, 'serialize_uint', (jInt,), jString)
 PYPYSERIALIZEULONG =    Method.s(jPyPy, 'serialize_ulonglong', (jLong,),jString)
 PYPYSERIALIZEVOID =     Method.s(jPyPy, 'serialize_void', (), jString)
+PYPYSERIALIZEDOUBLE =   Method.s(jPyPy, 'serialize_double', (jDouble,), jString)
 PYPYESCAPEDCHAR =       Method.s(jPyPy, 'escaped_char', (jChar,), jString)
 PYPYESCAPEDUNICHAR =    Method.s(jPyPy, 'escaped_unichar', (jChar,), jString)
 PYPYESCAPEDSTRING =     Method.s(jPyPy, 'escaped_string', (jString,), jString)
diff --git a/pypy/translator/oosupport/test_template/cast.py b/pypy/translator/oosupport/test_template/cast.py
--- a/pypy/translator/oosupport/test_template/cast.py
+++ b/pypy/translator/oosupport/test_template/cast.py
@@ -13,6 +13,9 @@
 def to_longlong(x):
     return r_longlong(x)
 
+def to_ulonglong(x):
+    return r_ulonglong(x)
+
 def uint_to_int(x):
     return intmask(x)
 
@@ -56,6 +59,9 @@
     def test_unsignedlonglong_to_unsigned4(self):
         self.check(to_uint, [r_ulonglong(18446744073709551615l)]) # max 64 bit num
 
+    def test_unsigned_to_usignedlonglong(self):
+        self.check(to_ulonglong, [r_uint(42)])
+
     def test_uint_to_int(self):
         self.check(uint_to_int, [r_uint(sys.maxint+1)])
 

From noreply at buildbot.pypy.org  Fri Jan 20 11:21:13 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 20 Jan 2012 11:21:13 +0100 (CET)
Subject: [pypy-commit] pypy numpy-indexing-by-arrays-2: first approximation
 of array[array-of-int]
Message-ID: <20120120102113.91D5582CF8@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: numpy-indexing-by-arrays-2
Changeset: r51516:d1cbcc34848d
Date: 2012-01-20 12:20 +0200
http://bitbucket.org/pypy/pypy/changeset/d1cbcc34848d/

Log:	first approximation of array[array-of-int]

diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py
--- a/pypy/module/micronumpy/interp_boxes.py
+++ b/pypy/module/micronumpy/interp_boxes.py
@@ -104,7 +104,10 @@
     _attrs_ = ()
 
 class W_IntegerBox(W_NumberBox):
-    pass
+    def convert_to_int(self):
+        from pypy.rpython.lltypesystem import rffi, lltype
+        
+        return rffi.cast(lltype.Signed, self.value)
 
 class W_SignedIntegerBox(W_IntegerBox):
     pass
diff --git a/pypy/module/micronumpy/interp_iter.py b/pypy/module/micronumpy/interp_iter.py
--- a/pypy/module/micronumpy/interp_iter.py
+++ b/pypy/module/micronumpy/interp_iter.py
@@ -20,6 +20,9 @@
         if self.step != 0:
             shape.append(self.lgt)
 
+    def get_iter(self):
+        xxx
+
 class IntArrayChunk(BaseChunk):
     def __init__(self, arr):
         self.arr = arr.get_concrete()
@@ -27,10 +30,22 @@
     def extend_shape(self, shape):
         shape.extend(self.arr.shape)
 
+    def get_iter(self):
+        return self.arr.create_iter()
+
+    def get_index(self, iter):
+        return self.arr.getitem(iter.offset).convert_to_int()
+
 class BoolArrayChunk(BaseChunk):
     def __init__(self, arr):
         self.arr = arr.get_concrete()
 
+    def extend_shape(self, shape):
+        xxx
+
+    def get_iter(self):
+        xxx
+
 class BaseTransform(object):
     pass
 
@@ -212,6 +227,30 @@
         return self._done
 
 # ------ other iterators that are not part of the computation frame ----------
+
+class ChunkIterator(object):
+    def __init__(self, shape, chunks):
+        self.chunks = chunks
+        self.indices = [0] * len(shape)
+        self.shape = shape
+        self.chunk_iters = [chunk.get_iter() for chunk in self.chunks]
+
+    def next(self, shapelen):
+        for i in range(shapelen - 1, -1, -1):
+            if self.indices[i] < self.shape[i] - 1:
+                self.indices[i] += 1
+                self.chunk_iters[i] = self.chunk_iters[i].next()
+                break
+            else:
+                self.indices[i] = 0
+                # XXX reset one dim iter probably
+        return self
+
+    def get_index(self, shapelen):
+        l = []
+        for i in range(shapelen):
+            l.append(self.chunks[i].get_index(self.chunk_iters[i]))
+        return l
     
 class SkipLastAxisIterator(object):
     def __init__(self, arr):
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -10,7 +10,8 @@
 from pypy.tool.sourcetools import func_with_new_name
 from pypy.rlib.rstring import StringBuilder
 from pypy.module.micronumpy.interp_iter import ArrayIterator, OneDimIterator,\
-     SkipLastAxisIterator, Chunk, ViewIterator, BoolArrayChunk, IntArrayChunk
+     SkipLastAxisIterator, Chunk, ViewIterator, BoolArrayChunk, IntArrayChunk,\
+     ChunkIterator
 
 numpy_driver = jit.JitDriver(
     greens=['shapelen', 'sig'],
@@ -583,6 +584,9 @@
             item = concrete._index_of_single_item(space, w_idx)
             return concrete.getitem(item)
         chunks = self._prepare_slice_args(space, w_idx)
+        for chunk in chunks:
+            if not isinstance(chunk, Chunk):
+                return self.get_concrete().force_slice_getitem(space, chunks)
         return space.wrap(self.create_slice(chunks))
 
     def descr_setitem(self, space, w_idx, w_value):
@@ -604,13 +608,6 @@
         view = self.create_slice(chunks).get_concrete()
         view.setslice(space, w_value)
 
-    def force_slice(self, shape, chunks):
-        size = 1
-        for elem in shape:
-            size *= elem
-        res = W_NDimArray(size, shape, self.find_dtype())
-        xxx
-
     @jit.unroll_safe
     def create_slice(self, chunks):
         shape = []
@@ -620,9 +617,6 @@
         s = i + 1
         assert s >= 0
         shape += self.shape[s:]
-        for chunk in chunks:
-            if not isinstance(chunk, Chunk):
-                return self.force_slice(shape, chunks)
         if not isinstance(self, ConcreteArray):
             return VirtualSlice(self, chunks, shape)
         r = calculate_slice_strides(self.shape, self.start, self.strides,
@@ -1104,6 +1098,20 @@
         builder.append(']')
 
     @jit.unroll_safe
+    def _index_of_single_item_int(self, space, index):
+        item = self.start
+        for i in range(len(index)):
+            v = index[i]
+            if v < 0:
+                v += self.shape[i]
+            if v < 0 or v >= self.shape[i]:
+                raise operationerrfmt(space.w_IndexError,
+                    "index (%d) out of range (0<=index<%d", i, self.shape[i],
+                )
+            item += v * self.strides[i]
+        return item
+
+    @jit.unroll_safe
     def _index_of_single_item(self, space, w_idx):
         if space.isinstance_w(w_idx, space.w_int):
             idx = space.int_w(w_idx)
@@ -1115,17 +1123,7 @@
             return self.start + idx * self.strides[0]
         index = [space.int_w(w_item)
                  for w_item in space.fixedview(w_idx)]
-        item = self.start
-        for i in range(len(index)):
-            v = index[i]
-            if v < 0:
-                v += self.shape[i]
-            if v < 0 or v >= self.shape[i]:
-                raise operationerrfmt(space.w_IndexError,
-                    "index (%d) out of range (0<=index<%d", i, self.shape[i],
-                )
-            item += v * self.strides[i]
-        return item
+        return self._index_of_single_item_int(space, index)
 
     def setslice(self, space, w_value):
         res_shape = shape_agreement(space, self.shape, w_value.shape)
@@ -1178,6 +1176,28 @@
     def fill(self, space, w_value):
         self.setslice(space, scalar_w(space, self.dtype, w_value))
 
+    def force_slice_getitem(self, space, chunks):
+        shape = []
+        i = -1
+        for i, chunk in enumerate(chunks):
+            chunk.extend_shape(shape)
+        s = i + 1
+        assert s >= 0
+        shape += self.shape[s:]
+        size = 1
+        for elem in shape:
+            size *= elem
+        res = W_NDimArray(size, shape, self.find_dtype())
+        ri = res.create_iter()
+        ci = ChunkIterator(shape, chunks)
+        shapelen = len(shape)
+        while not ri.done():
+            index = ci.get_index(shapelen)
+            v = self.getitem(self._index_of_single_item_int(space, index))
+            res.setitem(ri.offset, v)
+            ri = ri.next(shapelen)
+            ci = ci.next(shapelen)
+        return res
 
 class ViewArray(ConcreteArray):
     def create_sig(self):

From noreply at buildbot.pypy.org  Fri Jan 20 11:24:28 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 20 Jan 2012 11:24:28 +0100 (CET)
Subject: [pypy-commit] pypy numpy-indexing-by-arrays-2: more tests and a fix
Message-ID: <20120120102428.2B10F82CF8@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: numpy-indexing-by-arrays-2
Changeset: r51517:b5668c7a1c53
Date: 2012-01-20 12:24 +0200
http://bitbucket.org/pypy/pypy/changeset/b5668c7a1c53/

Log:	more tests and a fix

diff --git a/pypy/module/micronumpy/interp_iter.py b/pypy/module/micronumpy/interp_iter.py
--- a/pypy/module/micronumpy/interp_iter.py
+++ b/pypy/module/micronumpy/interp_iter.py
@@ -239,7 +239,7 @@
         for i in range(shapelen - 1, -1, -1):
             if self.indices[i] < self.shape[i] - 1:
                 self.indices[i] += 1
-                self.chunk_iters[i] = self.chunk_iters[i].next()
+                self.chunk_iters[i] = self.chunk_iters[i].next(shapelen)
                 break
             else:
                 self.indices[i] = 0
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1312,6 +1312,10 @@
         raises(IndexError,'arange(3)[array([15])]')
         assert arange(3)[array([-3])] == 0
         raises(IndexError,'arange(3)[array([-15])]')
+        a = arange(10)
+        assert (a[array([3, 2, 1])] == [3, 2, 1]).all()
+        raises(IndexError, 'a[array([3, 2, 1, 15])]')
+        assert (a[array([3, 2, 5, 3])] == [3, 2, 5, 3]).all()
 
     def test_fill(self):
         from _numpypy import array

From noreply at buildbot.pypy.org  Fri Jan 20 12:56:36 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 20 Jan 2012 12:56:36 +0100 (CET)
Subject: [pypy-commit] pypy stm: Kill parts of the code that are outdated.
Message-ID: <20120120115636.55FBB82CF8@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stm
Changeset: r51518:83ad741c9a27
Date: 2012-01-20 12:56 +0100
http://bitbucket.org/pypy/pypy/changeset/83ad741c9a27/

Log:	Kill parts of the code that are outdated.

diff --git a/pypy/translator/stm/_rffi_stm.py b/pypy/translator/stm/_rffi_stm.py
--- a/pypy/translator/stm/_rffi_stm.py
+++ b/pypy/translator/stm/_rffi_stm.py
@@ -26,17 +26,8 @@
 stm_descriptor_init = llexternal('stm_descriptor_init', [], lltype.Void)
 stm_descriptor_done = llexternal('stm_descriptor_done', [], lltype.Void)
 
-##begin_transaction = llexternal('STM_begin_transaction', [], lltype.Void)
-##begin_inevitable_transaction = llexternal('stm_begin_inevitable_transaction',
-##                                          [], lltype.Void)
-##commit_transaction = llexternal('stm_commit_transaction', [], lltype.Signed)
 stm_try_inevitable = llexternal('stm_try_inevitable', [], lltype.Void)
 
-##descriptor_init_and_being_inevitable_transaction = llexternal(
-##    'stm_descriptor_init_and_being_inevitable_transaction', [], lltype.Void)
-##commit_transaction_and_descriptor_done = llexternal(
-##    'stm_commit_transaction_and_descriptor_done', [], lltype.Void)
-
 stm_read_word = llexternal('stm_read_word', [SignedP], lltype.Signed)
 stm_write_word = llexternal('stm_write_word', [SignedP, lltype.Signed],
                             lltype.Void)
diff --git a/pypy/translator/stm/src_stm/et.c b/pypy/translator/stm/src_stm/et.c
--- a/pypy/translator/stm/src_stm/et.c
+++ b/pypy/translator/stm/src_stm/et.c
@@ -59,9 +59,6 @@
 
 /************************************************************/
 
-/* Uncomment the line to try this extra code.  Doesn't work reliably so far */
-/*#define COMMIT_OTHER_INEV*/
-
 #define ABORT_REASONS 8
 #define SPINLOOP_REASONS 10
 #define OTHERINEV_REASONS 5
@@ -75,9 +72,6 @@
   unsigned num_commits;
   unsigned num_aborts[ABORT_REASONS];
   unsigned num_spinloops[SPINLOOP_REASONS];
-#ifdef COMMIT_OTHER_INEV
-  unsigned num_otherinev[OTHERINEV_REASONS];
-#endif
   unsigned int spinloop_counter;
   owner_version_t my_lock_word;
   struct RedoLog redolog;   /* last item, because it's the biggest one */
@@ -93,10 +87,6 @@
    if there is an inevitable transaction running */
 static volatile unsigned long global_timestamp = 2;
 static __thread struct tx_descriptor *thread_descriptor = NULL_TX;
-#ifdef COMMIT_OTHER_INEV
-static struct tx_descriptor *volatile thread_descriptor_inev;
-static volatile unsigned long d_inev_checking = 0;
-#endif
 
 /************************************************************/
 
@@ -352,7 +342,7 @@
 static pthread_mutex_t mutex_inevitable = PTHREAD_MUTEX_INITIALIZER;
 # ifdef RPY_STM_ASSERT
 unsigned long locked_by = 0;
-void mutex_lock(void)
+static void mutex_lock(void)
 {
   unsigned long pself = (unsigned long)pthread_self();
   if (PYPY_HAVE_DEBUG_PRINTS) fprintf(PYPY_DEBUG_FILE,
@@ -363,7 +353,7 @@
   if (PYPY_HAVE_DEBUG_PRINTS) fprintf(PYPY_DEBUG_FILE,
                                       "%lx: mutex inev locked\n", pself);
 }
-void mutex_unlock(void)
+static void mutex_unlock(void)
 {
   unsigned long pself = (unsigned long)pthread_self();
   locked_by = 0;
@@ -380,85 +370,7 @@
 # define mutex_unlock()   /* nothing */
 #endif
 
-#ifdef COMMIT_OTHER_INEV
-unsigned long can_commit_with_other_inevitable(struct tx_descriptor *d,
-                                               unsigned long expected)
-{
-  int i;
-  owner_version_t ovt;
-  unsigned long result = 0;
-  struct tx_descriptor *d_inev;
-
-  // 'd_inev_checking' is 1 or 2 when an inevitable transaction is running
-  // and didn't start committing yet; otherwise it is 0.  It is normally 1
-  // except in this function.
-  if (!bool_cas(&d_inev_checking, 1, 2))
-    {
-      d->num_otherinev[4]++;
-      return 0;
-    }
-
-  // optimization only: did the inevitable thread 'd_inev' read any data
-  // that we are about to commit?  If we are sure that the answer is
-  // negative, then commit anyway, because it cannot make the inevitable
-  // thread fail.  We can safely check an approximation of this, because
-  // we hold a lock on all orecs that we would like to write.  So if all
-  // orecs read by d_inev are not locked now, then no conflict.  This
-  // function is allowed to "fail" and give up rather than spinloop
-  // waiting for a condition to be true, which is potentially dangerous
-  // here, because we acquired all the locks.
-
-  // Note that if the inevitable thread itself adds in parallel an extra
-  // orec to d_inev->reads, *and* if this new orec is locked, then we
-  // will miss it here; but the d_inev thread will spinloop waiting for
-  // us to be done.  So even if we commit, the d_inev thread will just
-  // wait and load the new committed value.
-
-  // while we are in this function, the d_inev thread is prevented from
-  // going too far with the commitTransaction() code because d_inev_checking
-  // is greater than 1; it will just tx_spinloop(9).  (And of course it
-  // cannot abort.)
-
-  d_inev = thread_descriptor_inev;
-  if (!bool_cas(&d_inev->reads.locked, 0, 1))
-    {
-      d->num_otherinev[1]++;
-      goto give_up_1;
-    }
-
-  for (i=d_inev->reads.size; i--; )
-    {
-      ovt = d_inev->reads.items[i]->v;     // read this orec
-      if (ovt == d->my_lock_word)
-        {
-          d->num_otherinev[2]++;
-          goto give_up_2;
-        }
-    }
-  assert(expected & 1);
-  if (!change_global_timestamp(d, expected, expected + 2))
-    {
-      d->num_otherinev[3]++;
-      goto give_up_2;
-    }
-
-  /* success: scale d_inet forward */
-  d->num_otherinev[0]++;
-  result = expected + 1;
-  assert(d_inev->start_time == result - 2);
-  d_inev->start_time = result;
-  CFENCE;
-
- give_up_2:
-  d_inev->reads.locked = 0;
-
- give_up_1:
-  d_inev_checking = 1;
-  return result;
-}
-#endif
-
-void wait_end_inevitability(struct tx_descriptor *d)
+static void wait_end_inevitability(struct tx_descriptor *d)
 {
   unsigned long curts;
   releaseLocksForRetry(d);
@@ -485,16 +397,11 @@
   acquireLocks(d);
 }
 
-void commitInevitableTransaction(struct tx_descriptor *d)
+static void commitInevitableTransaction(struct tx_descriptor *d)
 {
   unsigned long ts;
   _Bool ok;
 
-#ifdef COMMIT_OTHER_INEV
-  // reset d_inev_checking back from 1 to 0
-  while (!bool_cas(&d_inev_checking, 1, 0))
-    tx_spinloop(9);
-#endif
   // no-one else can modify global_timestamp if I'm inevitable
   // and d_inev_checking is 0
   ts = get_global_timestamp(d);
@@ -529,12 +436,6 @@
   volatile orec_t* o = get_orec((void*)addr);
   owner_version_t ovt;
 
-#ifdef COMMIT_OTHER_INEV
-  // log orec BEFORE we spinloop waiting for the orec lock to be released,
-  // for can_commit_with_other_inevitable()
-  oreclist_insert(&d->reads, (orec_t*)o);
-#endif
-
  retry:
   // read the orec BEFORE we read anything else
   ovt = o->v;
@@ -551,13 +452,7 @@
       }
       // else this location is too new, scale forward
       owner_version_t newts = get_global_timestamp(d) & ~1;
-#ifdef COMMIT_OTHER_INEV
-      d->reads.size--;   // ignore the newly logged orec
-#endif
       validate_fast(d, 1);
-#ifdef COMMIT_OTHER_INEV
-      d->reads.size++;
-#endif
       d->start_time = newts;
     }
 
@@ -569,9 +464,7 @@
   if (o->v != ovt)
     goto retry;       /* oups, try again */
 
-#ifndef COMMIT_OTHER_INEV
   oreclist_insert(&d->reads, (orec_t*)o);
-#endif
 
   return tmp;
 }
@@ -649,12 +542,6 @@
       p += sprintf(p, "%c%d", i == 1 ? '|' : ',',
                    d->num_spinloops[i]);
 
-#ifdef COMMIT_OTHER_INEV
-    for (i=0; inum_otherinev[i]);
-#endif
-
     p += sprintf(p, "]\n");
     fwrite(line, 1, p - line, PYPY_DEBUG_FILE);
   }
@@ -664,18 +551,7 @@
   free(d);
 }
 
-void* stm_perform_transaction(void*(*callback)(void*), void *arg)
-{
-  void *result;
-  /* you need to call descriptor_init() before calling stm_perform_transaction */
-  assert(thread_descriptor != NULL_TX);
-  STM_begin_transaction();
-  result = callback(arg);
-  stm_commit_transaction();
-  return result;
-}
-
-void stm_begin_transaction(jmp_buf* buf)
+static void begin_transaction(jmp_buf* buf)
 {
   struct tx_descriptor *d = thread_descriptor;
   assert(!d->transaction_active);
@@ -684,7 +560,7 @@
   d->start_time = d->last_known_global_timestamp & ~1;
 }
 
-long stm_commit_transaction(void)
+static long commit_transaction(void)
 {
   struct tx_descriptor *d = thread_descriptor;
 
@@ -720,15 +596,6 @@
           unsigned long expected = get_global_timestamp(d);
           if (expected & 1)
             {
-#ifdef COMMIT_OTHER_INEV
-              // there is another inevitable transaction running.
-              expected = can_commit_with_other_inevitable(d, expected);
-              if (expected != 0)
-                {
-                  d->end_time = expected;
-                  break;
-                }
-#endif
               // wait until it is done.  hopefully we can then proceed
               // without conflicts.
               wait_end_inevitability(d);
@@ -757,6 +624,20 @@
   return d->end_time;
 }
 
+void* stm_perform_transaction(void*(*callback)(void*), void *arg)
+{
+  void *result;
+  jmp_buf _jmpbuf;
+  /* you need to call descriptor_init() before calling
+     stm_perform_transaction() */
+  assert(thread_descriptor != NULL_TX);
+  setjmp(_jmpbuf);
+  begin_transaction(&_jmpbuf);
+  result = callback(arg);
+  commit_transaction();
+  return result;
+}
+
 void stm_try_inevitable(STM_CCHARP1(why))
 {
   /* when a transaction is inevitable, its start_time is equal to
@@ -809,56 +690,11 @@
       mutex_unlock();
     }
   d->setjmp_buf = NULL;   /* inevitable from now on */
-#ifdef COMMIT_OTHER_INEV
-  thread_descriptor_inev = d;
-  CFENCE;
-  d_inev_checking = 1;
-#endif
 #ifdef RPY_STM_ASSERT
   PYPY_DEBUG_STOP("stm-inevitable");
 #endif
 }
 
-void stm_try_inevitable_if(jmp_buf *buf  STM_CCHARP(why))
-{
-  struct tx_descriptor *d = thread_descriptor;
-  if (d->setjmp_buf == buf)
-    stm_try_inevitable(STM_EXPLAIN1(why));
-}
-
-void stm_begin_inevitable_transaction(void)
-{
-  struct tx_descriptor *d = thread_descriptor;
-  unsigned long curtime;
-
-  assert(!d->transaction_active);
-
- retry:
-  mutex_lock();   /* possibly waiting here */
-
-  while (1)
-    {
-      curtime = global_timestamp;
-      if (curtime & 1)
-        {
-          mutex_unlock();
-          tx_spinloop(5);
-          goto retry;
-        }
-      if (bool_cas(&global_timestamp, curtime, curtime + 1))
-        break;
-    }
-  assert(!d->transaction_active);
-  d->transaction_active = 1;
-  d->setjmp_buf = NULL;
-  d->start_time = curtime;
-#ifdef COMMIT_OTHER_INEV
-  thread_descriptor_inev = d;
-  CFENCE;
-  d_inev_checking = 1;
-#endif
-}
-
 void stm_abort_and_retry(void)
 {
   tx_abort(7);     /* manual abort */
diff --git a/pypy/translator/stm/src_stm/et.h b/pypy/translator/stm/src_stm/et.h
--- a/pypy/translator/stm/src_stm/et.h
+++ b/pypy/translator/stm/src_stm/et.h
@@ -12,14 +12,10 @@
 #include "src/commondefs.h"
 
 #ifdef RPY_STM_ASSERT
-#  define STM_CCHARP(arg)     , char* arg
 #  define STM_CCHARP1(arg)    char* arg
-#  define STM_EXPLAIN(info)   , info
 #  define STM_EXPLAIN1(info)  info
 #else
-#  define STM_CCHARP(arg)     /* nothing */
 #  define STM_CCHARP1(arg)    void
-#  define STM_EXPLAIN(info)   /* nothing */
 #  define STM_EXPLAIN1(info)  /* nothing */
 #endif
 
@@ -27,31 +23,15 @@
 void stm_descriptor_init(void);
 void stm_descriptor_done(void);
 void* stm_perform_transaction(void*(*)(void*), void*);
-void stm_begin_transaction(jmp_buf* buf);
-long stm_commit_transaction(void);
 long stm_read_word(long* addr);
 void stm_write_word(long* addr, long val);
 void stm_try_inevitable(STM_CCHARP1(why));
-void stm_try_inevitable_if(jmp_buf* buf  STM_CCHARP(why));
-void stm_begin_inevitable_transaction(void);
 void stm_abort_and_retry(void);
-void stm_descriptor_init_and_being_inevitable_transaction(void);
-void stm_commit_transaction_and_descriptor_done(void);
 long stm_debug_get_state(void);  /* -1: descriptor_init() was not called
                                      0: not in a transaction
                                      1: in a regular transaction
                                      2: in an inevitable transaction */
 
-/* for testing only: */
-#define STM_begin_transaction()         ; \
-       jmp_buf _jmpbuf;                   \
-       setjmp(_jmpbuf);                   \
-       stm_begin_transaction(&_jmpbuf)
-
-#define STM_DECLARE_VARIABLE()          ; jmp_buf jmpbuf
-#define STM_MAKE_INEVITABLE()           stm_try_inevitable_if(&jmpbuf  \
-                                                        STM_EXPLAIN("return"))
-
 // XXX little-endian only!
 #define STM_read_partial_word(T, base, offset)                          \
     (T)(stm_read_word(                                                  \

From noreply at buildbot.pypy.org  Fri Jan 20 14:25:05 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 20 Jan 2012 14:25:05 +0100 (CET)
Subject: [pypy-commit] pypy stm: Clean up the implementation of the reads
 and writes of less than one word.
Message-ID: <20120120132505.D0C9A82CF8@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stm
Changeset: r51519:fe658ba14686
Date: 2012-01-20 13:24 +0100
http://bitbucket.org/pypy/pypy/changeset/fe658ba14686/

Log:	Clean up the implementation of the reads and writes of less than one
	word.

diff --git a/pypy/translator/c/src/g_prerequisite.h b/pypy/translator/c/src/g_prerequisite.h
--- a/pypy/translator/c/src/g_prerequisite.h
+++ b/pypy/translator/c/src/g_prerequisite.h
@@ -19,7 +19,7 @@
 #include 
 
 
-#if 0 //def __GNUC__       /* other platforms too, probably */  XXX FIX ME
+#ifdef __GNUC__       /* other platforms too, probably */
 typedef _Bool bool_t;
 #else
 typedef unsigned char bool_t;
diff --git a/pypy/translator/stm/funcgen.py b/pypy/translator/stm/funcgen.py
--- a/pypy/translator/stm/funcgen.py
+++ b/pypy/translator/stm/funcgen.py
@@ -27,6 +27,7 @@
         return '%s = (%s)%s((long*)&%s);' % (
             newvalue, cresulttypename, funcname, expr)
     else:
+        assert fieldsize in (1, 2, 4)
         if simple_struct:
             # assume that the object is aligned, and any possible misalignment
             # comes from the field offset, so that it can be resolved at
@@ -35,14 +36,20 @@
             structdef = funcgen.db.gettypedefnode(STRUCT)
             basename = funcgen.expr(op.args[0])
             fieldname = op.args[1].value
-            return '%s = STM_read_partial_word(%s, %s, offsetof(%s, %s));' % (
+            trailing = ''
+            if T == lltype.Bool:
+                trailing = ' & 1'    # needed in this case, otherwise casting
+                                     # a several-bytes value to bool_t would
+                                     # take into account all the several bytes
+            return '%s = (%s)(stm_fx_read_partial(%s, offsetof(%s, %s))%s);'% (
                 newvalue, cresulttypename, basename,
                 cdecl(funcgen.db.gettype(STRUCT), ''),
-                structdef.c_struct_field_name(fieldname))
+                structdef.c_struct_field_name(fieldname),
+                trailing)
         #
         else:
-            return '%s = stm_read_partial_word(sizeof(%s), &%s);' % (
-                newvalue, cresulttypename, expr)
+            return '%s = (%s)stm_read_partial_%d(&%s);' % (
+                newvalue, cresulttypename, fieldsize, expr)
 
 def _stm_generic_set(funcgen, op, targetexpr, T):
     basename = funcgen.expr(op.args[0])
@@ -69,10 +76,9 @@
         return '%s((long*)&%s, (%s)%s);' % (
             funcname, targetexpr, newtype, newvalue)
     else:
-        itemtypename = funcgen.db.gettype(T)
-        citemtypename = cdecl(itemtypename, '')
-        return ('stm_write_partial_word(sizeof(%s), &%s, %s);' % (
-            citemtypename, targetexpr, newvalue))
+        assert fieldsize in (1, 2, 4)
+        return ('stm_write_partial_%d(&%s, (unsigned long)%s);' % (
+            fieldsize, targetexpr, newvalue))
 
 
 def field_expr(funcgen, args):
diff --git a/pypy/translator/stm/src_stm/et.c b/pypy/translator/stm/src_stm/et.c
--- a/pypy/translator/stm/src_stm/et.c
+++ b/pypy/translator/stm/src_stm/et.c
@@ -701,25 +701,47 @@
 }
 
 // XXX little-endian only!
-unsigned long stm_read_partial_word(int fieldsize, void *addr)
-{
-  int misalignment = ((long)addr) & (sizeof(void*)-1);
-  long *p = (long*)(((char *)addr) - misalignment);
-  unsigned long word = stm_read_word(p);
-  return word >> (misalignment * 8);
+#define READ_PARTIAL_WORD(T, fieldsize, addr)           \
+  int misalignment = ((long)addr) & (sizeof(void*)-1);  \
+  long *p = (long*)(((char *)addr) - misalignment);     \
+  unsigned long word = stm_read_word(p);                \
+  assert(sizeof(T) == fieldsize);                       \
+  return (T)(word >> (misalignment * 8));
+
+unsigned char stm_read_partial_1(void *addr) {
+  READ_PARTIAL_WORD(unsigned char, 1, addr)
 }
+unsigned short stm_read_partial_2(void *addr) {
+  READ_PARTIAL_WORD(unsigned short, 2, addr)
+}
+#if PYPY_LONG_BIT == 64
+unsigned int stm_read_partial_4(void *addr) {
+  READ_PARTIAL_WORD(unsigned int, 4, addr)
+}
+#endif
 
 // XXX little-endian only!
-void stm_write_partial_word(int fieldsize, void *addr, unsigned long nval)
-{
-  int misalignment = ((long)addr) & (sizeof(void*)-1);
-  long *p = (long*)(((char *)addr) - misalignment);
-  long val = nval << (misalignment * 8);
-  long word = stm_read_word(p);
-  long mask = ((1L << (fieldsize * 8)) - 1) << (misalignment * 8);
-  val = (val & mask) | (word & ~mask);
+#define WRITE_PARTIAL_WORD(fieldsize, addr, nval)                       \
+  int misalignment = ((long)addr) & (sizeof(void*)-1);                  \
+  long *p = (long*)(((char *)addr) - misalignment);                     \
+  long val = ((long)nval) << (misalignment * 8);                        \
+  long word = stm_read_word(p);                                         \
+  long mask = ((1L << (fieldsize * 8)) - 1) << (misalignment * 8);      \
+  val = (val & mask) | (word & ~mask);                                  \
   stm_write_word(p, val);
+
+void stm_write_partial_1(void *addr, unsigned char nval) {
+  WRITE_PARTIAL_WORD(1, addr, nval)
 }
+void stm_write_partial_2(void *addr, unsigned short nval) {
+  WRITE_PARTIAL_WORD(2, addr, nval)
+}
+#if PYPY_LONG_BIT == 64
+void stm_write_partial_4(void *addr, unsigned int nval) {
+  WRITE_PARTIAL_WORD(4, addr, nval)
+}
+#endif
+
 
 #if PYPY_LONG_BIT == 32
 long long stm_read_doubleword(long *addr)
@@ -791,7 +813,7 @@
 #if PYPY_LONG_BIT == 32
   stm_write_word(addr, ii);         /* 32 bits */
 #else
-  stm_write_partial_word(4, addr, ii);   /* 64 bits */
+  stm_write_partial_4(addr, ii);    /* 64 bits */
 #endif
 }
 
diff --git a/pypy/translator/stm/src_stm/et.h b/pypy/translator/stm/src_stm/et.h
--- a/pypy/translator/stm/src_stm/et.h
+++ b/pypy/translator/stm/src_stm/et.h
@@ -33,13 +33,21 @@
                                      2: in an inevitable transaction */
 
 // XXX little-endian only!
-#define STM_read_partial_word(T, base, offset)                          \
-    (T)(stm_read_word(                                                  \
+/* this macro is used if 'base' is a word-aligned pointer and 'offset'
+   is a compile-time constant */
+#define stm_fx_read_partial(base, offset)                               \
+       (stm_read_word(                                                  \
            (long*)(((char*)(base)) + ((offset) & ~(sizeof(void*)-1))))  \
         >> (8 * ((offset) & (sizeof(void*)-1))))
 
-unsigned long stm_read_partial_word(int fieldsize, void *addr);
-void stm_write_partial_word(int fieldsize, void *addr, unsigned long nval);
+unsigned char stm_read_partial_1(void *addr);
+unsigned short stm_read_partial_2(void *addr);
+void stm_write_partial_1(void *addr, unsigned char nval);
+void stm_write_partial_2(void *addr, unsigned short nval);
+#if PYPY_LONG_BIT == 64
+unsigned int stm_read_partial_4(void *addr);
+void stm_write_partial_4(void *addr, unsigned int nval);
+#endif
 
 double stm_read_double(long *addr);
 void stm_write_double(long *addr, double val);

From noreply at buildbot.pypy.org  Fri Jan 20 14:25:07 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 20 Jan 2012 14:25:07 +0100 (CET)
Subject: [pypy-commit] pypy stm: (bivab, romain, arigo)
Message-ID: <20120120132507.11EC282CF8@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stm
Changeset: r51520:3b0b849eed24
Date: 2012-01-20 14:24 +0100
http://bitbucket.org/pypy/pypy/changeset/3b0b849eed24/

Log:	(bivab, romain, arigo)

	Found out that a "volatile struct { int x; }" is not the same thing
	as a "struct { volatile int x; }". In fact the "volatile" in the
	first example seems to have no effect. Bah. Fixed by removing the
	struct completely, as nowadays it contains only one field.

diff --git a/pypy/translator/stm/src_stm/et.c b/pypy/translator/stm/src_stm/et.c
--- a/pypy/translator/stm/src_stm/et.c
+++ b/pypy/translator/stm/src_stm/et.c
@@ -34,9 +34,7 @@
     (((unsigned long)(num)) > ((unsigned long)(max_age)))
 typedef long owner_version_t;
 
-typedef struct {
-  owner_version_t v;   // the current version number
-} orec_t;
+typedef volatile owner_version_t orec_t;
 
 /*** Specify the number of orecs in the global array. */
 #define NUM_STRIPES  1048576
@@ -45,14 +43,14 @@
 static char orecs[NUM_STRIPES * sizeof(orec_t)];
 
 /*** map addresses to orec table entries */
-inline static volatile orec_t* get_orec(void* addr)
+inline static orec_t *get_orec(void* addr)
 {
   unsigned long index = (unsigned long)addr;
 #ifdef RPY_STM_ASSERT
   assert(!(index & (sizeof(orec_t)-1)));
 #endif
   char *p = orecs + (index & ((NUM_STRIPES-1) * sizeof(orec_t)));
-  return (volatile orec_t *)p;
+  return (orec_t *)p;
 }
 
 #include "src_stm/lists.c"
@@ -158,9 +156,9 @@
          appears in the redolog list.  If it's not, then p == -1. */
       if (item->p != -1)
         {
-          volatile orec_t* o = get_orec(item->addr);
+          orec_t* o = get_orec(item->addr);
           CFENCE;
-          o->v = newver;
+          *o = newver;
         }
     } REDOLOG_LOOP_END;
 }
@@ -173,8 +171,8 @@
     {
       if (item->p != -1)
         {
-          volatile orec_t* o = get_orec(item->addr);
-          o->v = item->p;
+          orec_t* o = get_orec(item->addr);
+          *o = item->p;
         }
     } REDOLOG_LOOP_END;
 }
@@ -187,8 +185,8 @@
     {
       if (item->p != -1)
         {
-          volatile orec_t* o = get_orec(item->addr);
-          o->v = item->p;
+          orec_t* o = get_orec(item->addr);
+          *o = item->p;
           item->p = -1;
         }
     } REDOLOG_LOOP_END;
@@ -202,11 +200,11 @@
   REDOLOG_LOOP_BACKWARD(d->redolog, item)
     {
       // get orec, read its version#
-      volatile orec_t* o = get_orec(item->addr);
+      orec_t* o = get_orec(item->addr);
       owner_version_t ovt;
 
     retry:
-      ovt = o->v;
+      ovt = *o;
 
       // if orec not locked, lock it
       //
@@ -214,7 +212,7 @@
       // reads.  Since most writes are also reads, we'll just abort under this
       // condition.  This can introduce false conflicts
       if (!IS_LOCKED_OR_NEWER(ovt, d->start_time)) {
-        if (!bool_cas(&o->v, ovt, d->my_lock_word))
+        if (!bool_cas(o, ovt, d->my_lock_word))
           goto retry;
         // save old version to item->p.  Now we hold the lock.
         // in case of duplicate orecs, only the last one has p != -1.
@@ -291,7 +289,7 @@
   for (i=0; ireads.size; i++)
     {
     retry:
-      ovt = d->reads.items[i]->v;
+      ovt = *(d->reads.items[i]);
       if (IS_LOCKED_OR_NEWER(ovt, d->start_time))
         {
           // If locked, we wait until it becomes unlocked.  The chances are
@@ -321,7 +319,7 @@
   assert(!is_inevitable(d));
   for (i=0; ireads.size; i++)
     {
-      ovt = d->reads.items[i]->v;      // read this orec
+      ovt = *(d->reads.items[i]);      // read this orec
       if (IS_LOCKED_OR_NEWER(ovt, d->start_time))
         {
           if (!IS_LOCKED(ovt))
@@ -433,12 +431,12 @@
 
  not_found:;
   // get the orec addr
-  volatile orec_t* o = get_orec((void*)addr);
+  orec_t* o = get_orec((void*)addr);
   owner_version_t ovt;
 
  retry:
   // read the orec BEFORE we read anything else
-  ovt = o->v;
+  ovt = *o;
   CFENCE;
 
   // this tx doesn't hold any locks, so if the lock for this addr is held,
@@ -461,10 +459,10 @@
 
   // postvalidate AFTER reading addr:
   CFENCE;
-  if (o->v != ovt)
+  if (*o != ovt)
     goto retry;       /* oups, try again */
 
-  oreclist_insert(&d->reads, (orec_t*)o);
+  oreclist_insert(&d->reads, o);
 
   return tmp;
 }

From noreply at buildbot.pypy.org  Fri Jan 20 15:00:20 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 20 Jan 2012 15:00:20 +0100 (CET)
Subject: [pypy-commit] pypy stm: Add two __builtin_expect() to optimize the
 order of the assembler, maybe.
Message-ID: <20120120140020.D974582CF8@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stm
Changeset: r51521:4f9f97138a8c
Date: 2012-01-20 15:00 +0100
http://bitbucket.org/pypy/pypy/changeset/4f9f97138a8c/

Log:	Add two __builtin_expect() to optimize the order of the assembler,
	maybe.

diff --git a/pypy/translator/stm/src_stm/et.c b/pypy/translator/stm/src_stm/et.c
--- a/pypy/translator/stm/src_stm/et.c
+++ b/pypy/translator/stm/src_stm/et.c
@@ -31,7 +31,7 @@
 
 #define IS_LOCKED(num)  ((num) < 0)
 #define IS_LOCKED_OR_NEWER(num, max_age) \
-    (((unsigned long)(num)) > ((unsigned long)(max_age)))
+  __builtin_expect(((unsigned long)(num)) > ((unsigned long)(max_age)), 0)
 typedef long owner_version_t;
 
 typedef volatile owner_version_t orec_t;
diff --git a/pypy/translator/stm/src_stm/lists.c b/pypy/translator/stm/src_stm/lists.c
--- a/pypy/translator/stm/src_stm/lists.c
+++ b/pypy/translator/stm/src_stm/lists.c
@@ -102,7 +102,7 @@
   unsigned long _key = (unsigned long)(addr1);                  \
   char *_p = (char *)((redolog).toplevel.items);                \
   char *_entry = *(char **)(_p + (_key & TREE_MASK));           \
-  if (_entry == NULL)                                           \
+  if (__builtin_expect(_entry == NULL, 1))                      \
     goto_not_found;    /* common case, hopefully */             \
   result = _redolog_find(_entry, addr1);                        \
   if (result == NULL || result->addr != (addr1))                \

From noreply at buildbot.pypy.org  Fri Jan 20 15:17:23 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 20 Jan 2012 15:17:23 +0100 (CET)
Subject: [pypy-commit] pypy numpy-indexing-by-arrays-bool: A branch to merge
 what's there in indexing by arrays, the rest left for the
Message-ID: <20120120141723.926E382CF8@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: numpy-indexing-by-arrays-bool
Changeset: r51522:8cc13fd6cd81
Date: 2012-01-20 16:16 +0200
http://bitbucket.org/pypy/pypy/changeset/8cc13fd6cd81/

Log:	A branch to merge what's there in indexing by arrays, the rest left
	for the future when I figure out.

	Fix translation

diff --git a/pypy/module/micronumpy/interp_iter.py b/pypy/module/micronumpy/interp_iter.py
--- a/pypy/module/micronumpy/interp_iter.py
+++ b/pypy/module/micronumpy/interp_iter.py
@@ -6,10 +6,7 @@
 
 # structures to describe slicing
 
-class BaseChunk(object):
-    pass
-
-class Chunk(BaseChunk):
+class Chunk(object):
     def __init__(self, start, stop, step, lgt):
         self.start = start
         self.stop = stop
@@ -20,14 +17,6 @@
         if self.step != 0:
             shape.append(self.lgt)
 
-class IntArrayChunk(BaseChunk):
-    def __init__(self, arr):
-        self.arr = arr.get_concrete()
-
-class BoolArrayChunk(BaseChunk):
-    def __init__(self, arr):
-        self.arr = arr.get_concrete()
-
 class BaseTransform(object):
     pass
 
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -249,15 +249,16 @@
 
 
 class W_Ufunc2(W_Ufunc):
-    _immutable_fields_ = ["comparison_func", "func", "name"]
+    _immutable_fields_ = ["comparison_func", "func", "name", "int_only"]
     argcount = 2
 
     def __init__(self, func, name, promote_to_float=False, promote_bools=False,
-        identity=None, comparison_func=False):
+        identity=None, comparison_func=False, int_only=False):
 
         W_Ufunc.__init__(self, name, promote_to_float, promote_bools, identity)
         self.func = func
         self.comparison_func = comparison_func
+        self.int_only = int_only
 
     def call(self, space, args_w):
         from pypy.module.micronumpy.interp_numarray import (Call2,
@@ -268,6 +269,7 @@
         w_rhs = convert_to_array(space, w_rhs)
         calc_dtype = find_binop_result_dtype(space,
             w_lhs.find_dtype(), w_rhs.find_dtype(),
+            int_only=self.int_only,
             promote_to_float=self.promote_to_float,
             promote_bools=self.promote_bools,
         )
@@ -304,10 +306,12 @@
 
 
 def find_binop_result_dtype(space, dt1, dt2, promote_to_float=False,
-    promote_bools=False):
+    promote_bools=False, int_only=False):
     # dt1.num should be <= dt2.num
     if dt1.num > dt2.num:
         dt1, dt2 = dt2, dt1
+    if int_only and (not dt1.is_int_type() or not dt2.is_int_type()):
+        raise OperationError(space.w_TypeError, space.wrap("Unsupported types"))
     # Some operations promote op(bool, bool) to return int8, rather than bool
     if promote_bools and (dt1.kind == dt2.kind == interp_dtype.BOOLLTR):
         return interp_dtype.get_dtype_cache(space).w_int8dtype
@@ -425,8 +429,10 @@
             ("add", "add", 2, {"identity": 0}),
             ("subtract", "sub", 2),
             ("multiply", "mul", 2, {"identity": 1}),
-            ("bitwise_and", "bitwise_and", 2, {"identity": 1}),
-            ("bitwise_or", "bitwise_or", 2, {"identity": 0}),
+            ("bitwise_and", "bitwise_and", 2, {"identity": 1,
+                                               'int_only': True}),
+            ("bitwise_or", "bitwise_or", 2, {"identity": 0,
+                                             'int_only': True}),
             ("divide", "div", 2, {"promote_bools": True}),
             ("mod", "mod", 2, {"promote_bools": True}),
             ("power", "pow", 2, {"promote_bools": True}),
@@ -477,7 +483,7 @@
         extra_kwargs["identity"] = identity
 
         func = ufunc_dtype_caller(space, ufunc_name, op_name, argcount,
-            comparison_func=extra_kwargs.get("comparison_func", False)
+            comparison_func=extra_kwargs.get("comparison_func", False),
         )
         if argcount == 1:
             ufunc = W_Ufunc1(func, ufunc_name, **extra_kwargs)
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -353,12 +353,13 @@
         assert (add.reduce(a, 1) == [6.0, 22.0, 38.0]).all()
 
     def test_bitwise(self):
-        from _numpypy import bitwise_and, bitwise_or, arange
+        from _numpypy import bitwise_and, bitwise_or, arange, array
         a = arange(6).reshape(2, 3)
         assert (a & 1 == [[0, 1, 0], [1, 0, 1]]).all()
         assert (a & 1 == bitwise_and(a, 1)).all()
         assert (a | 1 == [[1, 1, 3], [3, 5, 5]]).all()
         assert (a | 1 == bitwise_or(a, 1)).all()
+        raises(TypeError, 'array([1.0]) & 1')
 
     def test_comparisons(self):
         import operator
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -171,14 +171,6 @@
     @simple_binary_op
     def min(self, v1, v2):
         return min(v1, v2)
-
-    @simple_binary_op
-    def bitwise_and(self, v1, v2):
-        return v1 & v2
-
-    @simple_binary_op
-    def bitwise_or(self, v1, v2):
-        return v1 | v2
     
 
 class Bool(BaseType, Primitive):
@@ -270,6 +262,14 @@
             assert v == 0
             return 0
 
+    @simple_binary_op
+    def bitwise_and(self, v1, v2):
+        return v1 & v2
+
+    @simple_binary_op
+    def bitwise_or(self, v1, v2):
+        return v1 | v2
+
 class Int8(BaseType, Integer):
     T = rffi.SIGNEDCHAR
     BoxType = interp_boxes.W_Int8Box

From noreply at buildbot.pypy.org  Fri Jan 20 15:19:33 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 20 Jan 2012 15:19:33 +0100 (CET)
Subject: [pypy-commit] pypy numpy-indexing-by-arrays-bool: merge default
Message-ID: <20120120141933.CB48082CF8@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: numpy-indexing-by-arrays-bool
Changeset: r51523:3f86800ad9ba
Date: 2012-01-20 16:18 +0200
http://bitbucket.org/pypy/pypy/changeset/3f86800ad9ba/

Log:	merge default

diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -2,6 +2,9 @@
 *.py[co]
 *~
 .*.swp
+.idea
+.project
+.pydevproject
 
 syntax: regexp
 ^testresult$
diff --git a/lib_pypy/numpypy/__init__.py b/lib_pypy/numpypy/__init__.py
--- a/lib_pypy/numpypy/__init__.py
+++ b/lib_pypy/numpypy/__init__.py
@@ -1,2 +1,2 @@
 from _numpypy import *
-from .fromnumeric import *
+from .core import *
diff --git a/lib_pypy/numpypy/core/__init__.py b/lib_pypy/numpypy/core/__init__.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/numpypy/core/__init__.py
@@ -0,0 +1,1 @@
+from .fromnumeric import *
diff --git a/lib_pypy/numpypy/fromnumeric.py b/lib_pypy/numpypy/core/fromnumeric.py
rename from lib_pypy/numpypy/fromnumeric.py
rename to lib_pypy/numpypy/core/fromnumeric.py
--- a/lib_pypy/numpypy/fromnumeric.py
+++ b/lib_pypy/numpypy/core/fromnumeric.py
@@ -85,7 +85,7 @@
     array([4, 3, 6])
 
     """
-    raise NotImplemented('Waiting on interp level method')
+    raise NotImplementedError('Waiting on interp level method')
 
 
 # not deprecated --- copy if necessary, view otherwise
@@ -273,7 +273,7 @@
             [-1, -2, -3, -4, -5]]])
 
     """
-    raise NotImplemented('Waiting on interp level method')
+    raise NotImplementedError('Waiting on interp level method')
 
 
 def repeat(a, repeats, axis=None):
@@ -315,7 +315,7 @@
            [3, 4]])
 
     """
-    raise NotImplemented('Waiting on interp level method')
+    raise NotImplementedError('Waiting on interp level method')
 
 
 def put(a, ind, v, mode='raise'):
@@ -366,7 +366,7 @@
     array([ 0,  1,  2,  3, -5])
 
     """
-    raise NotImplemented('Waiting on interp level method')
+    raise NotImplementedError('Waiting on interp level method')
 
 
 def swapaxes(a, axis1, axis2):
@@ -410,7 +410,7 @@
             [3, 7]]])
 
     """
-    raise NotImplemented('Waiting on interp level method')
+    raise NotImplementedError('Waiting on interp level method')
 
 
 def transpose(a, axes=None):
@@ -451,8 +451,11 @@
     (2, 1, 3)
 
     """
-    raise NotImplemented('Waiting on interp level method')
-
+    if axes is not None:
+        raise NotImplementedError('No "axes" arg yet.')
+    if not hasattr(a, 'T'):
+        a = numpypy.array(a)
+    return a.T
 
 def sort(a, axis=-1, kind='quicksort', order=None):
     """
@@ -553,7 +556,7 @@
           dtype=[('name', '|S10'), ('height', ' 1:
             raise OperationError(space.w_ValueError, space.wrap(
@@ -765,6 +769,9 @@
     def copy(self, space):
         return Scalar(self.dtype, self.value)
 
+    def fill(self, space, w_value):
+        self.value = self.dtype.coerce(space, w_value)
+
     def create_sig(self):
         return signature.ScalarSignature(self.dtype)
 
@@ -870,7 +877,7 @@
     Intermediate class for performing binary operations.
     """
     _immutable_fields_ = ['left', 'right']
-    
+
     def __init__(self, ufunc, name, shape, calc_dtype, res_dtype, left, right):
         VirtualArray.__init__(self, name, shape, res_dtype)
         self.ufunc = ufunc
@@ -910,7 +917,7 @@
     def __init__(self, shape, dtype, left, right):
         Call2.__init__(self, None, 'sliceloop', shape, dtype, dtype, left,
                        right)
-    
+
     def create_sig(self):
         lsig = self.left.create_sig()
         rsig = self.right.create_sig()
@@ -929,7 +936,7 @@
     when we'll make AxisReduce lazy
     """
     _immutable_fields_ = ['left', 'right']
-    
+
     def __init__(self, ufunc, name, shape, dtype, left, right, dim):
         Call2.__init__(self, ufunc, name, shape, dtype, dtype,
                        left, right)
@@ -1001,14 +1008,14 @@
         if size < 1:
             builder.append('[]')
             return
-        elif size == 1:
-            builder.append(dtype.itemtype.str_format(self.getitem(0)))
-            return
         if size > 1000:
             # Once this goes True it does not go back to False for recursive
             # calls
             use_ellipsis = True
         ndims = len(self.shape)
+        if ndims == 0:
+            builder.append(dtype.itemtype.str_format(self.getitem(0)))
+            return
         i = 0
         builder.append('[')
         if ndims > 1:
@@ -1143,6 +1150,9 @@
         array.setslice(space, self)
         return array
 
+    def fill(self, space, w_value):
+        self.setslice(space, scalar_w(space, self.dtype, w_value))
+
 
 class ViewArray(ConcreteArray):
     def create_sig(self):
@@ -1366,6 +1376,8 @@
     var = interp2app(BaseArray.descr_var),
     std = interp2app(BaseArray.descr_std),
 
+    fill = interp2app(BaseArray.descr_fill),
+
     copy = interp2app(BaseArray.descr_copy),
     reshape = interp2app(BaseArray.descr_reshape),
     tolist = interp2app(BaseArray.descr_tolist),
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -457,6 +457,7 @@
 
             ("fabs", "fabs", 1, {"promote_to_float": True}),
             ("floor", "floor", 1, {"promote_to_float": True}),
+            ("ceil", "ceil", 1, {"promote_to_float": True}),
             ("exp", "exp", 1, {"promote_to_float": True}),
 
             ('sqrt', 'sqrt', 1, {'promote_to_float': True}),
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -166,14 +166,11 @@
         # You can't subclass dtype
         raises(TypeError, type, "Foo", (dtype,), {})
 
-    def test_new(self):
-        import _numpypy as np
-        assert np.int_(4) == 4
-        assert np.float_(3.4) == 3.4
+    def test_aliases(self):
+        from _numpypy import dtype
 
-    def test_pow(self):
-        from _numpypy import int_
-        assert int_(4) ** 2 == 16
+        assert dtype("float") is dtype(float)
+
 
 class AppTestTypes(BaseNumpyAppTest):
     def test_abstract_types(self):
@@ -189,6 +186,15 @@
         raises(TypeError, numpy.floating, 0)
         raises(TypeError, numpy.inexact, 0)
 
+    def test_new(self):
+        import _numpypy as np
+        assert np.int_(4) == 4
+        assert np.float_(3.4) == 3.4
+
+    def test_pow(self):
+        from _numpypy import int_
+        assert int_(4) ** 2 == 16
+
     def test_bool(self):
         import _numpypy as numpy
 
@@ -318,7 +324,7 @@
         else:
             raises(OverflowError, numpy.int64, 9223372036854775807)
             raises(OverflowError, numpy.int64, '9223372036854775807')
-        
+
         raises(OverflowError, numpy.int64, 9223372036854775808)
         raises(OverflowError, numpy.int64, '9223372036854775808')
 
diff --git a/pypy/module/micronumpy/test/test_module.py b/pypy/module/micronumpy/test/test_module.py
--- a/pypy/module/micronumpy/test/test_module.py
+++ b/pypy/module/micronumpy/test/test_module.py
@@ -2,16 +2,11 @@
 
 
 class AppTestNumPyModule(BaseNumpyAppTest):
-    def test_mean(self):
-        from _numpypy import array, mean
-        assert mean(array(range(5))) == 2.0
-        assert mean(range(5)) == 2.0
-
     def test_average(self):
         from _numpypy import array, average
         assert average(range(10)) == 4.5
         assert average(array(range(10))) == 4.5
-        
+
     def test_sum(self):
         from _numpypy import array, sum
         assert sum(range(10)) == 45
@@ -21,7 +16,7 @@
         from _numpypy import array, min
         assert min(range(10)) == 0
         assert min(array(range(10))) == 0
-        
+
     def test_max(self):
         from _numpypy import array, max
         assert max(range(10)) == 9
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -728,15 +728,16 @@
         assert d[1] == 12
 
     def test_mean(self):
-        from _numpypy import array, mean
+        from _numpypy import array
         a = array(range(5))
         assert a.mean() == 2.0
         assert a[:4].mean() == 1.5
         a = array(range(105)).reshape(3, 5, 7)
-        b = mean(a, axis=0)
-        b[0,0]==35.
+        b = a.mean(axis=0)
+        b[0, 0]==35.
+        assert a.mean(axis=0)[0, 0] == 35
         assert (b == array(range(35, 70), dtype=float).reshape(5, 7)).all()
-        assert (mean(a, 2) == array(range(0, 15), dtype=float).reshape(3, 5) * 7 + 3).all()
+        assert (a.mean(2) == array(range(0, 15), dtype=float).reshape(3, 5) * 7 + 3).all()
 
     def test_sum(self):
         from _numpypy import array
@@ -757,6 +758,7 @@
         assert array([]).sum() == 0.0
         raises(ValueError, 'array([]).max()')
         assert (a.sum(0) == [30, 35, 40]).all()
+        assert (a.sum(axis=0) == [30, 35, 40]).all()
         assert (a.sum(1) == [3, 12, 21, 30, 39]).all()
         assert (a.max(0) == [12, 13, 14]).all()
         assert (a.max(1) == [2, 5, 8, 11, 14]).all()
@@ -771,6 +773,8 @@
         assert ((a + a).T.sum(2).T == (a + a).sum(0)).all()
         assert (a.reshape(1,-1).sum(0) == range(105)).all()
         assert (a.reshape(1,-1).sum(1) == 5460)
+        assert (array([[1,2],[3,4]]).prod(0) == [3, 8]).all()
+        assert (array([[1,2],[3,4]]).prod(1) == [2, 12]).all()
 
     def test_identity(self):
         from _numpypy import identity, array
@@ -1312,6 +1316,26 @@
         raises(IndexError,'arange(3)[array([-15])]')
         assert arange(3)[array(1)] == 1
 
+    def test_fill(self):
+        from _numpypy import array
+        a = array([1, 2, 3])
+        a.fill(10)
+        assert (a == [10, 10, 10]).all()
+        a.fill(False)
+        assert (a == [0, 0, 0]).all()
+        b = a[:1]
+        b.fill(4)
+        assert (b == [4]).all()
+        assert (a == [4, 0, 0]).all()
+
+        c = b + b
+        c.fill(27)
+        assert (c == [27]).all()
+
+        d = array(10)
+        d.fill(100)
+        assert d == 100
+
     def test_array_indexing_bool(self):
         from _numpypy import arange
         a = arange(10)
@@ -1329,6 +1353,8 @@
         a[a & 1 == 1] = array([8, 9, 10])
         assert (a == [[0, 8], [2, 9], [4, 10]]).all()
 
+
+
 class AppTestSupport(BaseNumpyAppTest):
     def setup_class(cls):
         import struct
@@ -1467,9 +1493,11 @@
         assert repr(a) == "array(0.0)"
         a = array(0.2)
         assert repr(a) == "array(0.2)"
+        a = array([2])
+        assert repr(a) == "array([2])"
 
     def test_repr_multi(self):
-        from _numpypy import arange, zeros
+        from _numpypy import arange, zeros, array
         a = zeros((3, 4))
         assert repr(a) == '''array([[0.0, 0.0, 0.0, 0.0],
        [0.0, 0.0, 0.0, 0.0],
@@ -1492,6 +1520,9 @@
        [498, 999],
        [499, 1000],
        [500, 1001]])'''
+        a = arange(2).reshape((2,1))
+        assert repr(a) == '''array([[0],
+       [1]])'''
 
     def test_repr_slice(self):
         from _numpypy import array, zeros
@@ -1576,18 +1607,3 @@
         a = arange(0, 0.8, 0.1)
         assert len(a) == 8
         assert arange(False, True, True).dtype is dtype(int)
-
-
-class AppTestRanges(BaseNumpyAppTest):
-    def test_app_reshape(self):
-        from _numpypy import arange, array, dtype, reshape
-        a = arange(12)
-        b = reshape(a, (3, 4))
-        assert b.shape == (3, 4)
-        a = range(12)
-        b = reshape(a, (3, 4))
-        assert b.shape == (3, 4)
-        a = array(range(105)).reshape(3, 5, 7)
-        assert a.reshape(1, -1).shape == (1, 105)
-        assert a.reshape(1, 1, -1).shape == (1, 1, 105)
-        assert a.reshape(-1, 1, 1).shape == (105, 1, 1)
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -190,14 +190,24 @@
         for i in range(3):
             assert c[i] == a[i] - b[i]
 
-    def test_floor(self):
-        from _numpypy import array, floor
-
-        reference = [-2.0, -1.0, 0.0, 1.0, 1.0]
-        a = array([-1.4, -1.0, 0.0, 1.0, 1.4])
+    def test_floorceil(self):
+        from _numpypy import array, floor, ceil
+        import math
+        reference = [-2.0, -2.0, -1.0, 0.0, 1.0, 1.0, 0]
+        a = array([-1.4, -1.5, -1.0, 0.0, 1.0, 1.4, 0.5])
         b = floor(a)
         for i in range(5):
             assert b[i] == reference[i]
+        reference = [-1.0, -1.0, -1.0, 0.0, 1.0, 2.0, 1.0]
+        a = array([-1.4, -1.5, -1.0, 0.0, 1.0, 1.4, 0.5])
+        b = ceil(a)
+        assert (reference == b).all()
+        inf = float("inf")
+        data = [1.5, 2.9999, -1.999, inf]
+        results = [math.floor(x) for x in data]
+        assert (floor(data) == results).all()
+        results = [math.ceil(x) for x in data]
+        assert (ceil(data) == results).all()
 
     def test_copysign(self):
         from _numpypy import array, copysign
@@ -238,7 +248,7 @@
             assert b[i] == math.sin(a[i])
 
         a = sin(array([True, False], dtype=bool))
-        assert abs(a[0] - sin(1)) < 1e-7 # a[0] will be less precise
+        assert abs(a[0] - sin(1)) < 1e-7  # a[0] will be less precise
         assert a[1] == 0.0
 
     def test_cos(self):
@@ -259,7 +269,6 @@
         for i in range(len(a)):
             assert b[i] == math.tan(a[i])
 
-
     def test_arcsin(self):
         import math
         from _numpypy import array, arcsin
@@ -283,7 +292,6 @@
         for i in range(len(a)):
             assert b[i] == math.acos(a[i])
 
-
         a = array([-10, -1.5, -1.01, 1.01, 1.5, 10, float('nan'), float('inf'), float('-inf')])
         b = arccos(a)
         for f in b:
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -350,7 +350,8 @@
         self.check_trace_count(1)
         self.check_simple_loop({'getinteriorfield_raw': 2, 'float_add': 1,
                                 'setinteriorfield_raw': 1, 'int_add': 2,
-                                'int_eq': 1, 'guard_false': 1, 'jump': 1})
+                                'int_eq': 1, 'guard_false': 1, 'jump': 1,
+                                'arraylen_gc': 1})
 
     def define_virtual_slice():
         return """
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -391,6 +391,10 @@
         return math.floor(v)
 
     @simple_unary_op
+    def ceil(self, v):
+        return math.ceil(v)
+
+    @simple_unary_op
     def exp(self, v):
         try:
             return math.exp(v)
diff --git a/pypy/module/pypyjit/__init__.py b/pypy/module/pypyjit/__init__.py
--- a/pypy/module/pypyjit/__init__.py
+++ b/pypy/module/pypyjit/__init__.py
@@ -11,6 +11,7 @@
         'set_optimize_hook': 'interp_resop.set_optimize_hook',
         'set_abort_hook': 'interp_resop.set_abort_hook',
         'ResOperation': 'interp_resop.WrappedOp',
+        'DebugMergePoint': 'interp_resop.DebugMergePoint',
         'Box': 'interp_resop.WrappedBox',
     }
 
diff --git a/pypy/module/pypyjit/interp_resop.py b/pypy/module/pypyjit/interp_resop.py
--- a/pypy/module/pypyjit/interp_resop.py
+++ b/pypy/module/pypyjit/interp_resop.py
@@ -1,5 +1,6 @@
 
-from pypy.interpreter.typedef import TypeDef, GetSetProperty
+from pypy.interpreter.typedef import (TypeDef, GetSetProperty,
+     interp_attrproperty, interp_attrproperty_w)
 from pypy.interpreter.baseobjspace import Wrappable
 from pypy.interpreter.gateway import unwrap_spec, interp2app, NoneNotWrapped
 from pypy.interpreter.pycode import PyCode
@@ -10,6 +11,7 @@
 from pypy.jit.metainterp.resoperation import rop, AbstractResOp
 from pypy.rlib.nonconst import NonConstant
 from pypy.rlib import jit_hooks
+from pypy.module.pypyjit.interp_jit import pypyjitdriver
 
 class Cache(object):
     in_recursion = False
@@ -111,13 +113,24 @@
 
 def wrap_oplist(space, logops, operations, ops_offset=None):
     l_w = []
+    jitdrivers_sd = logops.metainterp_sd.jitdrivers_sd
     for op in operations:
         if ops_offset is None:
             ofs = -1
         else:
             ofs = ops_offset.get(op, 0)
-        l_w.append(WrappedOp(jit_hooks._cast_to_gcref(op), ofs,
-                             logops.repr_of_resop(op)))
+        if op.opnum == rop.DEBUG_MERGE_POINT:
+            jd_sd = jitdrivers_sd[op.getarg(0).getint()]
+            greenkey = op.getarglist()[2:]
+            repr = jd_sd.warmstate.get_location_str(greenkey)
+            w_greenkey = wrap_greenkey(space, jd_sd.jitdriver, greenkey, repr)
+            l_w.append(DebugMergePoint(space, jit_hooks._cast_to_gcref(op),
+                                       logops.repr_of_resop(op),
+                                       jd_sd.jitdriver.name,
+                                       w_greenkey))
+        else:
+            l_w.append(WrappedOp(jit_hooks._cast_to_gcref(op), ofs,
+                                 logops.repr_of_resop(op)))
     return l_w
 
 class WrappedBox(Wrappable):
@@ -150,6 +163,15 @@
         llres = res.llbox
     return WrappedOp(jit_hooks.resop_new(num, args, llres), offset, repr)
 
+ at unwrap_spec(repr=str, jd_name=str)
+def descr_new_dmp(space, w_tp, w_args, repr, jd_name, w_greenkey):
+    args = [space.interp_w(WrappedBox, w_arg).llbox for w_arg in
+            space.listview(w_args)]
+    num = rop.DEBUG_MERGE_POINT
+    return DebugMergePoint(space,
+                           jit_hooks.resop_new(num, args, jit_hooks.emptyval()),
+                           repr, jd_name, w_greenkey)
+
 class WrappedOp(Wrappable):
     """ A class representing a single ResOperation, wrapped nicely
     """
@@ -182,6 +204,25 @@
         box = space.interp_w(WrappedBox, w_box)
         jit_hooks.resop_setresult(self.op, box.llbox)
 
+class DebugMergePoint(WrappedOp):
+    def __init__(self, space, op, repr_of_resop, jd_name, w_greenkey):
+        WrappedOp.__init__(self, op, -1, repr_of_resop)
+        self.w_greenkey = w_greenkey
+        self.jd_name = jd_name
+
+    def get_pycode(self, space):
+        if self.jd_name == pypyjitdriver.name:
+            return space.getitem(self.w_greenkey, space.wrap(0))
+        raise OperationError(space.w_AttributeError, space.wrap("This DebugMergePoint doesn't belong to the main Python JitDriver"))
+
+    def get_bytecode_no(self, space):
+        if self.jd_name == pypyjitdriver.name:
+            return space.getitem(self.w_greenkey, space.wrap(1))
+        raise OperationError(space.w_AttributeError, space.wrap("This DebugMergePoint doesn't belong to the main Python JitDriver"))
+
+    def get_jitdriver_name(self, space):
+        return space.wrap(self.jd_name)
+
 WrappedOp.typedef = TypeDef(
     'ResOperation',
     __doc__ = WrappedOp.__doc__,
@@ -195,3 +236,15 @@
                             WrappedOp.descr_setresult)
 )
 WrappedOp.acceptable_as_base_class = False
+
+DebugMergePoint.typedef = TypeDef(
+    'DebugMergePoint', WrappedOp.typedef,
+    __new__ = interp2app(descr_new_dmp),
+    greenkey = interp_attrproperty_w("w_greenkey", cls=DebugMergePoint),
+    pycode = GetSetProperty(DebugMergePoint.get_pycode),
+    bytecode_no = GetSetProperty(DebugMergePoint.get_bytecode_no),
+    jitdriver_name = GetSetProperty(DebugMergePoint.get_jitdriver_name),
+)
+DebugMergePoint.acceptable_as_base_class = False
+
+
diff --git a/pypy/module/pypyjit/policy.py b/pypy/module/pypyjit/policy.py
--- a/pypy/module/pypyjit/policy.py
+++ b/pypy/module/pypyjit/policy.py
@@ -127,7 +127,7 @@
                        'imp', 'sys', 'array', '_ffi', 'itertools', 'operator',
                        'posix', '_socket', '_sre', '_lsprof', '_weakref',
                        '__pypy__', 'cStringIO', '_collections', 'struct',
-                       'mmap', 'marshal']:
+                       'mmap', 'marshal', '_codecs']:
             if modname == 'pypyjit' and 'interp_resop' in rest:
                 return False
             return True
diff --git a/pypy/module/pypyjit/test/test_jit_hook.py b/pypy/module/pypyjit/test/test_jit_hook.py
--- a/pypy/module/pypyjit/test/test_jit_hook.py
+++ b/pypy/module/pypyjit/test/test_jit_hook.py
@@ -92,6 +92,7 @@
         cls.w_on_compile_bridge = space.wrap(interp2app(interp_on_compile_bridge))
         cls.w_on_abort = space.wrap(interp2app(interp_on_abort))
         cls.w_int_add_num = space.wrap(rop.INT_ADD)
+        cls.w_dmp_num = space.wrap(rop.DEBUG_MERGE_POINT)
         cls.w_on_optimize = space.wrap(interp2app(interp_on_optimize))
         cls.orig_oplist = oplist
 
@@ -117,6 +118,10 @@
         assert elem[2][2] == False
         assert len(elem[3]) == 4
         int_add = elem[3][0]
+        dmp = elem[3][1]
+        assert isinstance(dmp, pypyjit.DebugMergePoint)
+        assert dmp.pycode is self.f.func_code
+        assert dmp.greenkey == (self.f.func_code, 0, False)
         #assert int_add.name == 'int_add'
         assert int_add.num == self.int_add_num
         self.on_compile_bridge()
@@ -211,3 +216,18 @@
         assert op.getarg(0).getint() == 4
         op.result = box
         assert op.result.getint() == 1
+
+    def test_creation_dmp(self):
+        from pypyjit import DebugMergePoint, Box
+
+        def f():
+            pass
+
+        op = DebugMergePoint([Box(0)], 'repr', 'pypyjit', (f.func_code, 0, 0))
+        assert op.bytecode_no == 0
+        assert op.pycode is f.func_code
+        assert repr(op) == 'repr'
+        assert op.jitdriver_name == 'pypyjit'
+        assert op.num == self.dmp_num
+        op = DebugMergePoint([Box(0)], 'repr', 'notmain', ('str',))
+        raises(AttributeError, 'op.pycode')
diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py
--- a/pypy/module/sys/version.py
+++ b/pypy/module/sys/version.py
@@ -10,7 +10,7 @@
 CPYTHON_VERSION            = (2, 7, 1, "final", 42)   #XXX # sync patchlevel.h
 CPYTHON_API_VERSION        = 1013   #XXX # sync with include/modsupport.h
 
-PYPY_VERSION               = (1, 7, 1, "dev", 0)    #XXX # sync patchlevel.h
+PYPY_VERSION               = (1, 8, 1, "dev", 0)    #XXX # sync patchlevel.h
 
 if platform.name == 'msvc':
     COMPILER_INFO = 'MSC v.%d 32 bit' % (platform.version * 10 + 600)
diff --git a/lib_pypy/numpypy/test/test_fromnumeric.py b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
rename from lib_pypy/numpypy/test/test_fromnumeric.py
rename to pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
--- a/lib_pypy/numpypy/test/test_fromnumeric.py
+++ b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
@@ -1,7 +1,7 @@
-
 from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
 
-class AppTestFromNumeric(BaseNumpyAppTest):     
+
+class AppTestFromNumeric(BaseNumpyAppTest):
     def test_argmax(self):
         # tests taken from numpy/core/fromnumeric.py docstring
         from numpypy import array, arange, argmax
@@ -18,12 +18,12 @@
         from numpypy import array, arange, argmin
         a = arange(6).reshape((2,3))
         assert argmin(a) == 0
-        # assert (argmax(a, axis=0) == array([0, 0, 0])).all()
-        # assert (argmax(a, axis=1) == array([0, 0])).all()
+        assert (argmin(a, axis=0) == array([0, 0, 0])).all()
+        assert (argmin(a, axis=1) == array([0, 0])).all()
         b = arange(6)
         b[1] = 0
         assert argmin(b) == 0
-   
+
     def test_shape(self):
         # tests taken from numpy/core/fromnumeric.py docstring
         from numpypy import array, identity, shape
@@ -44,7 +44,7 @@
         # assert (sum([[0, 1], [0, 5]], axis=1) == array([1, 5])).all()
         # If the accumulator is too small, overflow occurs:
         # assert ones(128, dtype=int8).sum(dtype=int8) == -128
-                                 
+
     def test_amin(self):
         # tests taken from numpy/core/fromnumeric.py docstring
         from numpypy import array, arange, amin
@@ -86,14 +86,14 @@
         assert ndim([[1,2,3],[4,5,6]]) == 2
         assert ndim(array([[1,2,3],[4,5,6]])) == 2
         assert ndim(1) == 0
-    
+
     def test_rank(self):
         # tests taken from numpy/core/fromnumeric.py docstring
         from numpypy import array, rank
         assert rank([[1,2,3],[4,5,6]]) == 2
         assert rank(array([[1,2,3],[4,5,6]])) == 2
         assert rank(1) == 0
-    
+
     def test_var(self):
         from numpypy import array, var
         a = array([[1,2],[3,4]])
@@ -107,3 +107,31 @@
         assert std(a) ==  1.1180339887498949
         # assert (std(a, axis=0) == array([ 1.,  1.])).all()
         # assert (std(a, axis=1) == array([ 0.5,  0.5]).all()
+
+    def test_mean(self):
+        from numpypy import array, mean
+        assert mean(array(range(5))) == 2.0
+        assert mean(range(5)) == 2.0
+
+    def test_reshape(self):
+        from numpypy import arange, array, dtype, reshape
+        a = arange(12)
+        b = reshape(a, (3, 4))
+        assert b.shape == (3, 4)
+        a = range(12)
+        b = reshape(a, (3, 4))
+        assert b.shape == (3, 4)
+        a = array(range(105)).reshape(3, 5, 7)
+        assert reshape(a, (1, -1)).shape == (1, 105)
+        assert reshape(a, (1, 1, -1)).shape == (1, 1, 105)
+        assert reshape(a, (-1, 1, 1)).shape == (105, 1, 1)
+    
+    def test_transpose(self):   
+        from numpypy import arange, array, transpose, ones
+        x = arange(4).reshape((2,2))
+        assert (transpose(x) == array([[0, 2],[1, 3]])).all()
+        # Once axes argument is implemented, add more tests
+        raises(NotImplementedError, "transpose(x, axes=(1, 0, 2))")
+        # x = ones((1, 2, 3))
+        # assert transpose(x, (1, 0, 2)).shape == (2, 1, 3)
+
diff --git a/pypy/objspace/std/test/test_unicodeobject.py b/pypy/objspace/std/test/test_unicodeobject.py
--- a/pypy/objspace/std/test/test_unicodeobject.py
+++ b/pypy/objspace/std/test/test_unicodeobject.py
@@ -64,6 +64,12 @@
         check(', '.join([u'a']), u'a')
         check(', '.join(['a', u'b']), u'a, b')
         check(u', '.join(['a', 'b']), u'a, b')
+        try:
+            u''.join([u'a', 2, 3])
+        except TypeError, e:
+            assert 'sequence item 1' in str(e)
+        else:
+            raise Exception("DID NOT RAISE")
 
     if sys.version_info >= (2,3):
         def test_contains_ex(self):
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -201,7 +201,7 @@
     return space.newbool(container.find(item) != -1)
 
 def unicode_join__Unicode_ANY(space, w_self, w_list):
-    list_w = space.unpackiterable(w_list)
+    list_w = space.listview(w_list)
     size = len(list_w)
 
     if size == 0:
@@ -216,22 +216,21 @@
 
 def _unicode_join_many_items(space, w_self, list_w, size):
     self = w_self._value
-    sb = UnicodeBuilder()
+    prealloc_size = len(self) * (size - 1)
+    for i in range(size):
+        try:
+            prealloc_size += len(space.unicode_w(list_w[i]))
+        except OperationError, e:
+            if not e.match(space, space.w_TypeError):
+                raise
+            raise operationerrfmt(space.w_TypeError,
+                        "sequence item %d: expected string or Unicode", i)
+    sb = UnicodeBuilder(prealloc_size)
     for i in range(size):
         if self and i != 0:
             sb.append(self)
         w_s = list_w[i]
-        if isinstance(w_s, W_UnicodeObject):
-            # shortcut for performance
-            sb.append(w_s._value)
-        else:
-            try:
-                sb.append(space.unicode_w(w_s))
-            except OperationError, e:
-                if not e.match(space, space.w_TypeError):
-                    raise
-                raise operationerrfmt(space.w_TypeError,
-                    "sequence item %d: expected string or Unicode", i)
+        sb.append(space.unicode_w(w_s))
     return space.wrap(sb.build())
 
 def hash__Unicode(space, w_uni):
diff --git a/pypy/rlib/jit_hooks.py b/pypy/rlib/jit_hooks.py
--- a/pypy/rlib/jit_hooks.py
+++ b/pypy/rlib/jit_hooks.py
@@ -52,7 +52,10 @@
     from pypy.jit.metainterp.history import ResOperation
 
     args = [_cast_to_box(llargs[i]) for i in range(len(llargs))]
-    res = _cast_to_box(llres)
+    if llres:
+        res = _cast_to_box(llres)
+    else:
+        res = None
     return _cast_to_gcref(ResOperation(no, args, res))
 
 @register_helper(annmodel.SomePtr(llmemory.GCREF))
diff --git a/pypy/rlib/longlong2float.py b/pypy/rlib/longlong2float.py
--- a/pypy/rlib/longlong2float.py
+++ b/pypy/rlib/longlong2float.py
@@ -79,19 +79,23 @@
 longlong2float = rffi.llexternal(
     "pypy__longlong2float", [rffi.LONGLONG], rffi.DOUBLE,
     _callable=longlong2float_emulator, compilation_info=eci,
-    _nowrapper=True, elidable_function=True, sandboxsafe=True)
+    _nowrapper=True, elidable_function=True, sandboxsafe=True,
+    oo_primitive="pypy__longlong2float")
 
 float2longlong = rffi.llexternal(
     "pypy__float2longlong", [rffi.DOUBLE], rffi.LONGLONG,
     _callable=float2longlong_emulator, compilation_info=eci,
-    _nowrapper=True, elidable_function=True, sandboxsafe=True)
+    _nowrapper=True, elidable_function=True, sandboxsafe=True,
+    oo_primitive="pypy__float2longlong")
 
 uint2singlefloat = rffi.llexternal(
     "pypy__uint2singlefloat", [rffi.UINT], rffi.FLOAT,
     _callable=uint2singlefloat_emulator, compilation_info=eci,
-    _nowrapper=True, elidable_function=True, sandboxsafe=True)
+    _nowrapper=True, elidable_function=True, sandboxsafe=True,
+    oo_primitive="pypy__uint2singlefloat")
 
 singlefloat2uint = rffi.llexternal(
     "pypy__singlefloat2uint", [rffi.FLOAT], rffi.UINT,
     _callable=singlefloat2uint_emulator, compilation_info=eci,
-    _nowrapper=True, elidable_function=True, sandboxsafe=True)
+    _nowrapper=True, elidable_function=True, sandboxsafe=True,
+    oo_primitive="pypy__singlefloat2uint")
diff --git a/pypy/rlib/objectmodel.py b/pypy/rlib/objectmodel.py
--- a/pypy/rlib/objectmodel.py
+++ b/pypy/rlib/objectmodel.py
@@ -420,7 +420,7 @@
                   vobj.concretetype.TO._gckind == 'gc')
         else:
             from pypy.rpython.ootypesystem import ootype
-            ok = isinstance(vobj.concretetype, ootype.Instance)
+            ok = isinstance(vobj.concretetype, (ootype.Instance, ootype.BuiltinType))
         if not ok:
             from pypy.rpython.error import TyperError
             raise TyperError("compute_unique_id() cannot be applied to"
diff --git a/pypy/rpython/lltypesystem/rclass.py b/pypy/rpython/lltypesystem/rclass.py
--- a/pypy/rpython/lltypesystem/rclass.py
+++ b/pypy/rpython/lltypesystem/rclass.py
@@ -510,7 +510,13 @@
         ctype = inputconst(Void, self.object_type)
         cflags = inputconst(Void, flags)
         vlist = [ctype, cflags]
-        vptr = llops.genop('malloc', vlist,
+        cnonmovable = self.classdef.classdesc.read_attribute(
+            '_alloc_nonmovable_', Constant(False))
+        if cnonmovable.value:
+            opname = 'malloc_nonmovable'
+        else:
+            opname = 'malloc'
+        vptr = llops.genop(opname, vlist,
                            resulttype = Ptr(self.object_type))
         ctypeptr = inputconst(CLASSTYPE, self.rclass.getvtable())
         self.setfield(vptr, '__class__', ctypeptr, llops)
diff --git a/pypy/rpython/lltypesystem/rtuple.py b/pypy/rpython/lltypesystem/rtuple.py
--- a/pypy/rpython/lltypesystem/rtuple.py
+++ b/pypy/rpython/lltypesystem/rtuple.py
@@ -27,6 +27,10 @@
 
     def newtuple(cls, llops, r_tuple, items_v):
         # items_v should have the lowleveltype of the internal reprs
+        assert len(r_tuple.items_r) == len(items_v)
+        for r_item, v_item in zip(r_tuple.items_r, items_v):
+            assert r_item.lowleveltype == v_item.concretetype
+        #
         if len(r_tuple.items_r) == 0:
             return inputconst(Void, ())    # a Void empty tuple
         c1 = inputconst(Void, r_tuple.lowleveltype.TO)
diff --git a/pypy/rpython/ootypesystem/ootype.py b/pypy/rpython/ootypesystem/ootype.py
--- a/pypy/rpython/ootypesystem/ootype.py
+++ b/pypy/rpython/ootypesystem/ootype.py
@@ -512,6 +512,7 @@
             "ll_append_char": Meth([CHARTP], Void),
             "ll_append": Meth([STRINGTP], Void),
             "ll_build": Meth([], STRINGTP),
+            "ll_getlength": Meth([], Signed),
             })
         self._setup_methods({})
 
@@ -1376,6 +1377,9 @@
     def _cast_to_object(self):
         return make_object(self)
 
+    def _identityhash(self):
+        return object.__hash__(self)
+
 class _string(_builtin_type):
 
     def __init__(self, STRING, value = ''):
@@ -1543,6 +1547,9 @@
         else:
             return make_unicode(u''.join(self._buf))
 
+    def ll_getlength(self):
+        return self.ll_build().ll_strlen()
+
 class _null_string_builder(_null_mixin(_string_builder), _string_builder):
     def __init__(self, STRING_BUILDER):
         self.__dict__["_TYPE"] = STRING_BUILDER
diff --git a/pypy/rpython/ootypesystem/rbuilder.py b/pypy/rpython/ootypesystem/rbuilder.py
--- a/pypy/rpython/ootypesystem/rbuilder.py
+++ b/pypy/rpython/ootypesystem/rbuilder.py
@@ -21,6 +21,10 @@
         builder.ll_append_char(char)
 
     @staticmethod
+    def ll_getlength(builder):
+        return builder.ll_getlength()
+
+    @staticmethod
     def ll_append(builder, string):
         builder.ll_append(string)
 
diff --git a/pypy/rpython/rrange.py b/pypy/rpython/rrange.py
--- a/pypy/rpython/rrange.py
+++ b/pypy/rpython/rrange.py
@@ -204,7 +204,10 @@
         v_index = hop.gendirectcall(self.ll_getnextindex, v_enumerate)
         hop2 = hop.copy()
         hop2.args_r = [self.r_baseiter]
+        r_item_src = self.r_baseiter.r_list.external_item_repr
+        r_item_dst = hop.r_result.items_r[1]
         v_item = self.r_baseiter.rtype_next(hop2)
+        v_item = hop.llops.convertvar(v_item, r_item_src, r_item_dst)
         return hop.r_result.newtuple(hop.llops, hop.r_result,
                                      [v_index, v_item])
 
diff --git a/pypy/rpython/test/test_rbuilder.py b/pypy/rpython/test/test_rbuilder.py
--- a/pypy/rpython/test/test_rbuilder.py
+++ b/pypy/rpython/test/test_rbuilder.py
@@ -124,9 +124,5 @@
     pass
 
 class TestOOtype(BaseTestStringBuilder, OORtypeMixin):
-    def test_string_getlength(self):
-        py.test.skip("getlength(): not implemented on ootype")
-    def test_unicode_getlength(self):
-        py.test.skip("getlength(): not implemented on ootype")
     def test_append_charpsize(self):
         py.test.skip("append_charpsize(): not implemented on ootype")
diff --git a/pypy/rpython/test/test_rbuiltin.py b/pypy/rpython/test/test_rbuiltin.py
--- a/pypy/rpython/test/test_rbuiltin.py
+++ b/pypy/rpython/test/test_rbuiltin.py
@@ -463,6 +463,31 @@
             assert x1 == intmask(x0)
             assert x3 == intmask(x2)
 
+    def test_id_on_builtins(self):
+        from pypy.rlib.objectmodel import compute_unique_id
+        from pypy.rlib.rstring import StringBuilder, UnicodeBuilder
+        def fn():
+            return (compute_unique_id("foo"),
+                    compute_unique_id(u"bar"),
+                    compute_unique_id([1]),
+                    compute_unique_id({"foo": 3}),
+                    compute_unique_id(StringBuilder()),
+                    compute_unique_id(UnicodeBuilder()))
+        res = self.interpret(fn, [])
+        for id in self.ll_unpack_tuple(res, 6):
+            assert isinstance(id, (int, r_longlong))
+
+    def test_uniqueness_of_id_on_strings(self):
+        from pypy.rlib.objectmodel import compute_unique_id
+        def fn(s1, s2):
+            return (compute_unique_id(s1), compute_unique_id(s2))
+
+        s1 = "foo"
+        s2 = ''.join(['f','oo'])
+        res = self.interpret(fn, [self.string_to_ll(s1), self.string_to_ll(s2)])
+        i1, i2 = self.ll_unpack_tuple(res, 2)
+        assert i1 != i2
+
     def test_cast_primitive(self):
         from pypy.rpython.annlowlevel import LowLevelAnnotatorPolicy
         def llf(u):
diff --git a/pypy/rpython/test/test_rclass.py b/pypy/rpython/test/test_rclass.py
--- a/pypy/rpython/test/test_rclass.py
+++ b/pypy/rpython/test/test_rclass.py
@@ -1130,6 +1130,18 @@
             assert sorted([u]) == [6]                    # 32-bit types
             assert sorted([i, r, d, l]) == [2, 3, 4, 5]  # 64-bit types
 
+    def test_nonmovable(self):
+        for (nonmovable, opname) in [(True, 'malloc_nonmovable'),
+                                     (False, 'malloc')]:
+            class A(object):
+                _alloc_nonmovable_ = nonmovable
+            def f():
+                return A()
+            t, typer, graph = self.gengraph(f, [])
+            assert summary(graph) == {opname: 1,
+                                      'cast_pointer': 1,
+                                      'setfield': 1}
+
 
 class TestOOtype(BaseTestRclass, OORtypeMixin):
 
diff --git a/pypy/rpython/test/test_rrange.py b/pypy/rpython/test/test_rrange.py
--- a/pypy/rpython/test/test_rrange.py
+++ b/pypy/rpython/test/test_rrange.py
@@ -169,6 +169,22 @@
         res = self.interpret(fn, [2])
         assert res == 789
 
+    def test_enumerate_instances(self):
+        class A:
+            pass
+        def fn(n):
+            a = A()
+            b = A()
+            a.k = 10
+            b.k = 20
+            for i, x in enumerate([a, b]):
+                if i == n:
+                    return x.k
+            return 5
+        res = self.interpret(fn, [1])
+        assert res == 20
+
+
 
 class TestLLtype(BaseTestRrange, LLRtypeMixin):
     from pypy.rpython.lltypesystem import rrange 
diff --git a/pypy/tool/jitlogparser/parser.py b/pypy/tool/jitlogparser/parser.py
--- a/pypy/tool/jitlogparser/parser.py
+++ b/pypy/tool/jitlogparser/parser.py
@@ -140,13 +140,15 @@
     bytecode_name = None
     is_bytecode = True
     inline_level = None
+    has_dmp = False
 
     def parse_code_data(self, arg):
         m = re.search('\w]+)[\.,] file \'(.+?)\'[\.,] line (\d+)> #(\d+) (\w+)',
                       arg)
         if m is None:
             # a non-code loop, like StrLiteralSearch or something
-            self.bytecode_name = arg
+            if arg:
+                self.bytecode_name = arg
         else:
             self.name, self.filename, lineno, bytecode_no, self.bytecode_name = m.groups()
             self.startlineno = int(lineno)
@@ -218,7 +220,7 @@
         self.inputargs = inputargs
         self.chunks = chunks
         for chunk in self.chunks:
-            if chunk.filename is not None:
+            if chunk.bytecode_name is not None:
                 self.startlineno = chunk.startlineno
                 self.filename = chunk.filename
                 self.name = chunk.name
diff --git a/pypy/tool/jitlogparser/test/test_parser.py b/pypy/tool/jitlogparser/test/test_parser.py
--- a/pypy/tool/jitlogparser/test/test_parser.py
+++ b/pypy/tool/jitlogparser/test/test_parser.py
@@ -283,3 +283,13 @@
     assert loops[-1].count == 1234
     assert loops[1].count == 123
     assert loops[2].count == 12
+
+def test_parse_nonpython():
+    loop = parse("""
+    []
+    debug_merge_point(0, 'random')
+    debug_merge_point(0, ' #15 COMPARE_OP')
+    """)
+    f = Function.from_operations(loop.operations, LoopStorage())
+    assert f.chunks[-1].filename == 'x.py'
+    assert f.filename is None
diff --git a/pypy/translator/jvm/builtin.py b/pypy/translator/jvm/builtin.py
--- a/pypy/translator/jvm/builtin.py
+++ b/pypy/translator/jvm/builtin.py
@@ -84,6 +84,9 @@
     (ootype.StringBuilder.__class__, "ll_build"):
     jvm.Method.v(jStringBuilder, "toString", (), jString),
 
+    (ootype.StringBuilder.__class__, "ll_getlength"):
+    jvm.Method.v(jStringBuilder, "length", (), jInt),
+
     (ootype.String.__class__, "ll_hash"):
     jvm.Method.v(jString, "hashCode", (), jInt),
 
diff --git a/pypy/translator/jvm/database.py b/pypy/translator/jvm/database.py
--- a/pypy/translator/jvm/database.py
+++ b/pypy/translator/jvm/database.py
@@ -358,7 +358,7 @@
         ootype.Unsigned:jvm.PYPYSERIALIZEUINT,
         ootype.SignedLongLong:jvm.LONGTOSTRINGL,
         ootype.UnsignedLongLong: jvm.PYPYSERIALIZEULONG,
-        ootype.Float:jvm.DOUBLETOSTRINGD,
+        ootype.Float:jvm.PYPYSERIALIZEDOUBLE,
         ootype.Bool:jvm.PYPYSERIALIZEBOOLEAN,
         ootype.Void:jvm.PYPYSERIALIZEVOID,
         ootype.Char:jvm.PYPYESCAPEDCHAR,
diff --git a/pypy/translator/jvm/metavm.py b/pypy/translator/jvm/metavm.py
--- a/pypy/translator/jvm/metavm.py
+++ b/pypy/translator/jvm/metavm.py
@@ -92,6 +92,7 @@
 CASTS = {
 #   FROM                      TO
     (ootype.Signed,           ootype.UnsignedLongLong): jvm.I2L,
+    (ootype.Unsigned,         ootype.UnsignedLongLong): jvm.I2L,
     (ootype.SignedLongLong,   ootype.Signed):           jvm.L2I,
     (ootype.UnsignedLongLong, ootype.Unsigned):         jvm.L2I,
     (ootype.UnsignedLongLong, ootype.Signed):           jvm.L2I,
diff --git a/pypy/translator/jvm/opcodes.py b/pypy/translator/jvm/opcodes.py
--- a/pypy/translator/jvm/opcodes.py
+++ b/pypy/translator/jvm/opcodes.py
@@ -101,6 +101,7 @@
     'jit_force_virtualizable':  Ignore,
     'jit_force_virtual':        DoNothing,
     'jit_force_quasi_immutable': Ignore,
+    'jit_is_virtual':           PushPrimitive(ootype.Bool, False),
 
     'debug_assert':              [], # TODO: implement?
     'debug_start_traceback':    Ignore,
diff --git a/pypy/translator/jvm/src/pypy/PyPy.java b/pypy/translator/jvm/src/pypy/PyPy.java
--- a/pypy/translator/jvm/src/pypy/PyPy.java
+++ b/pypy/translator/jvm/src/pypy/PyPy.java
@@ -283,6 +283,14 @@
         }
     }
 
+    public double pypy__longlong2float(long l) {
+        return Double.longBitsToDouble(l);
+    }
+
+    public long pypy__float2longlong(double d) {
+        return Double.doubleToRawLongBits(d);
+    }
+
     public double ooparse_float(String s) {
         try {
             return Double.parseDouble(s);
@@ -353,6 +361,19 @@
             return "False";
     }
 
+    public static String serialize_double(double d) {
+        if (Double.isNaN(d)) {
+            return "float(\"nan\")";
+        } else if (Double.isInfinite(d)) {
+            if (d > 0)
+                return "float(\"inf\")";
+            else
+                return "float(\"-inf\")";
+        } else {
+            return Double.toString(d);
+        }
+    }
+
     private static String format_char(char c) {
         String res = "\\x";
         if (c <= 0x0F) res = res + "0";
diff --git a/pypy/translator/jvm/test/runtest.py b/pypy/translator/jvm/test/runtest.py
--- a/pypy/translator/jvm/test/runtest.py
+++ b/pypy/translator/jvm/test/runtest.py
@@ -56,6 +56,7 @@
 
 # CLI could-be duplicate
 class JvmGeneratedSourceWrapper(object):
+
     def __init__(self, gensrc):
         """ gensrc is an instance of JvmGeneratedSource """
         self.gensrc = gensrc
diff --git a/pypy/translator/jvm/test/test_builder.py b/pypy/translator/jvm/test/test_builder.py
new file mode 100644
--- /dev/null
+++ b/pypy/translator/jvm/test/test_builder.py
@@ -0,0 +1,7 @@
+from pypy.translator.jvm.test.runtest import JvmTest
+from pypy.rpython.test.test_rbuilder import BaseTestStringBuilder
+import py
+
+class TestJvmStringBuilder(JvmTest, BaseTestStringBuilder):
+    def test_append_charpsize(self):
+        py.test.skip("append_charpsize(): not implemented on ootype")
diff --git a/pypy/translator/jvm/test/test_longlong2float.py b/pypy/translator/jvm/test/test_longlong2float.py
new file mode 100644
--- /dev/null
+++ b/pypy/translator/jvm/test/test_longlong2float.py
@@ -0,0 +1,20 @@
+from pypy.translator.jvm.test.runtest import JvmTest
+from pypy.rlib.longlong2float import *
+from pypy.rlib.test.test_longlong2float import enum_floats
+from pypy.rlib.test.test_longlong2float import fn as float2longlong2float
+import py
+
+class TestLongLong2Float(JvmTest):
+
+    def test_float2longlong_and_longlong2float(self):
+        def func(f):
+            return float2longlong2float(f)
+
+        for f in enum_floats():
+            assert repr(f) == repr(self.interpret(func, [f]))
+
+    def test_uint2singlefloat(self):
+        py.test.skip("uint2singlefloat is not implemented in ootype")
+
+    def test_singlefloat2uint(self):
+        py.test.skip("singlefloat2uint is not implemented in ootype")
diff --git a/pypy/translator/jvm/typesystem.py b/pypy/translator/jvm/typesystem.py
--- a/pypy/translator/jvm/typesystem.py
+++ b/pypy/translator/jvm/typesystem.py
@@ -955,6 +955,7 @@
 PYPYSERIALIZEUINT  =    Method.s(jPyPy, 'serialize_uint', (jInt,), jString)
 PYPYSERIALIZEULONG =    Method.s(jPyPy, 'serialize_ulonglong', (jLong,),jString)
 PYPYSERIALIZEVOID =     Method.s(jPyPy, 'serialize_void', (), jString)
+PYPYSERIALIZEDOUBLE =   Method.s(jPyPy, 'serialize_double', (jDouble,), jString)
 PYPYESCAPEDCHAR =       Method.s(jPyPy, 'escaped_char', (jChar,), jString)
 PYPYESCAPEDUNICHAR =    Method.s(jPyPy, 'escaped_unichar', (jChar,), jString)
 PYPYESCAPEDSTRING =     Method.s(jPyPy, 'escaped_string', (jString,), jString)
diff --git a/pypy/translator/oosupport/test_template/cast.py b/pypy/translator/oosupport/test_template/cast.py
--- a/pypy/translator/oosupport/test_template/cast.py
+++ b/pypy/translator/oosupport/test_template/cast.py
@@ -13,6 +13,9 @@
 def to_longlong(x):
     return r_longlong(x)
 
+def to_ulonglong(x):
+    return r_ulonglong(x)
+
 def uint_to_int(x):
     return intmask(x)
 
@@ -56,6 +59,9 @@
     def test_unsignedlonglong_to_unsigned4(self):
         self.check(to_uint, [r_ulonglong(18446744073709551615l)]) # max 64 bit num
 
+    def test_unsigned_to_usignedlonglong(self):
+        self.check(to_ulonglong, [r_uint(42)])
+
     def test_uint_to_int(self):
         self.check(uint_to_int, [r_uint(sys.maxint+1)])
 

From noreply at buildbot.pypy.org  Fri Jan 20 15:21:49 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 20 Jan 2012 15:21:49 +0100 (CET)
Subject: [pypy-commit] pypy default: merge numpy-indexing-by-arrays-bool,
 this adds some basic indexing by bool
Message-ID: <20120120142149.4095B82CF8@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: 
Changeset: r51524:ab36137540d5
Date: 2012-01-20 16:21 +0200
http://bitbucket.org/pypy/pypy/changeset/ab36137540d5/

Log:	merge numpy-indexing-by-arrays-bool, this adds some basic indexing
	by bool arrays (only when shapes match) as well as bitwise_and/or
	and a few helper functions. More work on indexing by arrays to be
	done in the future

diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -86,6 +86,8 @@
         ("subtract", "subtract"),
         ('sqrt', 'sqrt'),
         ("tan", "tan"),
+        ('bitwise_and', 'bitwise_and'),
+        ('bitwise_or', 'bitwise_or'),
     ]:
         interpleveldefs[exposed] = "interp_ufuncs.get(space).%s" % impl
 
diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -47,6 +47,10 @@
     def getitem(self, storage, i):
         return self.itemtype.read(storage, self.itemtype.get_element_size(), i, 0)
 
+    def getitem_bool(self, storage, i):
+        isize = self.itemtype.get_element_size()
+        return self.itemtype.read_bool(storage, isize, i, 0)
+
     def setitem(self, storage, i, box):
         self.itemtype.store(storage, self.itemtype.get_element_size(), i, 0, box)
 
@@ -85,6 +89,12 @@
     def descr_get_shape(self, space):
         return space.newtuple([])
 
+    def is_int_type(self):
+        return self.kind == SIGNEDLTR or self.kind == UNSIGNEDLTR
+
+    def is_bool_type(self):
+        return self.kind == BOOLLTR
+
 W_Dtype.typedef = TypeDef("dtype",
     __module__ = "numpypy",
     __new__ = interp2app(W_Dtype.descr__new__.im_func),
diff --git a/pypy/module/micronumpy/interp_iter.py b/pypy/module/micronumpy/interp_iter.py
--- a/pypy/module/micronumpy/interp_iter.py
+++ b/pypy/module/micronumpy/interp_iter.py
@@ -4,6 +4,19 @@
 from pypy.module.micronumpy.strides import calculate_broadcast_strides,\
      calculate_slice_strides
 
+# structures to describe slicing
+
+class Chunk(object):
+    def __init__(self, start, stop, step, lgt):
+        self.start = start
+        self.stop = stop
+        self.step = step
+        self.lgt = lgt
+
+    def extend_shape(self, shape):
+        if self.step != 0:
+            shape.append(self.lgt)
+
 class BaseTransform(object):
     pass
 
@@ -38,11 +51,18 @@
         self.size = size
 
     def next(self, shapelen):
+        return self._next(1)
+
+    def _next(self, ofs):
         arr = instantiate(ArrayIterator)
         arr.size = self.size
-        arr.offset = self.offset + 1
+        arr.offset = self.offset + ofs
         return arr
 
+    def next_no_increase(self, shapelen):
+        # a hack to make JIT believe this is always virtual
+        return self._next(0)
+
     def done(self):
         return self.offset >= self.size
 
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -2,14 +2,15 @@
 from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.interpreter.gateway import interp2app, NoneNotWrapped
 from pypy.interpreter.typedef import TypeDef, GetSetProperty
-from pypy.module.micronumpy import interp_ufuncs, interp_dtype, signature
+from pypy.module.micronumpy import interp_ufuncs, interp_dtype, signature,\
+     interp_boxes
 from pypy.module.micronumpy.strides import calculate_slice_strides
 from pypy.rlib import jit
 from pypy.rpython.lltypesystem import lltype, rffi
 from pypy.tool.sourcetools import func_with_new_name
 from pypy.rlib.rstring import StringBuilder
 from pypy.module.micronumpy.interp_iter import ArrayIterator, OneDimIterator,\
-     SkipLastAxisIterator
+     SkipLastAxisIterator, Chunk, ViewIterator
 
 numpy_driver = jit.JitDriver(
     greens=['shapelen', 'sig'],
@@ -39,7 +40,24 @@
     get_printable_location=signature.new_printable_location('slice'),
     name='numpy_slice',
 )
-
+count_driver = jit.JitDriver(
+    greens=['shapelen'],
+    virtualizables=['frame'],
+    reds=['s', 'frame', 'iter', 'arr'],
+    name='numpy_count'
+)
+filter_driver = jit.JitDriver(
+    greens=['shapelen', 'sig'],
+    virtualizables=['frame'],
+    reds=['concr', 'argi', 'ri', 'frame', 'v', 'res', 'self'],
+    name='numpy_filter',
+)
+filter_set_driver = jit.JitDriver(
+    greens=['shapelen', 'sig'],
+    virtualizables=['frame'],
+    reds=['idx', 'idxi', 'frame', 'arr'],
+    name='numpy_filterset',
+)
 
 def _find_shape_and_elems(space, w_iterable):
     shape = [space.len_w(w_iterable)]
@@ -270,6 +288,9 @@
     descr_gt = _binop_impl("greater")
     descr_ge = _binop_impl("greater_equal")
 
+    descr_and = _binop_impl("bitwise_and")
+    descr_or = _binop_impl("bitwise_or")
+
     def _binop_right_impl(ufunc_name):
         def impl(self, space, w_other):
             w_other = scalar_w(space,
@@ -479,11 +500,69 @@
     def _prepare_slice_args(self, space, w_idx):
         if (space.isinstance_w(w_idx, space.w_int) or
             space.isinstance_w(w_idx, space.w_slice)):
-            return [space.decode_index4(w_idx, self.shape[0])]
-        return [space.decode_index4(w_item, self.shape[i]) for i, w_item in
+            return [Chunk(*space.decode_index4(w_idx, self.shape[0]))]
+        return [Chunk(*space.decode_index4(w_item, self.shape[i])) for i, w_item in
                 enumerate(space.fixedview(w_idx))]
 
+    def count_all_true(self, arr):
+        sig = arr.find_sig()
+        frame = sig.create_frame(self)
+        shapelen = len(arr.shape)
+        s = 0
+        iter = None
+        while not frame.done():
+            count_driver.jit_merge_point(arr=arr, frame=frame, iter=iter, s=s,
+                                         shapelen=shapelen)
+            iter = frame.get_final_iter()
+            s += arr.dtype.getitem_bool(arr.storage, iter.offset)
+            frame.next(shapelen)
+        return s
+
+    def getitem_filter(self, space, arr):
+        concr = arr.get_concrete()
+        size = self.count_all_true(concr)
+        res = W_NDimArray(size, [size], self.find_dtype())
+        ri = ArrayIterator(size)
+        shapelen = len(self.shape)
+        argi = concr.create_iter()
+        sig = self.find_sig()
+        frame = sig.create_frame(self)
+        v = None
+        while not frame.done():
+            filter_driver.jit_merge_point(concr=concr, argi=argi, ri=ri,
+                                          frame=frame, v=v, res=res, sig=sig,
+                                          shapelen=shapelen, self=self)
+            if concr.dtype.getitem_bool(concr.storage, argi.offset):
+                v = sig.eval(frame, self)
+                res.setitem(ri.offset, v)
+                ri = ri.next(1)
+            else:
+                ri = ri.next_no_increase(1)
+            argi = argi.next(shapelen)
+            frame.next(shapelen)
+        return res
+
+    def setitem_filter(self, space, idx, val):
+        size = self.count_all_true(idx)
+        arr = SliceArray([size], self.dtype, self, val)
+        sig = arr.find_sig()
+        shapelen = len(self.shape)
+        frame = sig.create_frame(arr)
+        idxi = idx.create_iter()
+        while not frame.done():
+            filter_set_driver.jit_merge_point(idx=idx, idxi=idxi, sig=sig,
+                                              frame=frame, arr=arr,
+                                              shapelen=shapelen)
+            if idx.dtype.getitem_bool(idx.storage, idxi.offset):
+                sig.eval(frame, arr)
+                frame.next_from_second(1)
+            frame.next_first(shapelen)
+            idxi = idxi.next(shapelen)
+
     def descr_getitem(self, space, w_idx):
+        if (isinstance(w_idx, BaseArray) and w_idx.shape == self.shape and
+            w_idx.find_dtype().is_bool_type()):
+            return self.getitem_filter(space, w_idx)
         if self._single_item_result(space, w_idx):
             concrete = self.get_concrete()
             item = concrete._index_of_single_item(space, w_idx)
@@ -493,6 +572,11 @@
 
     def descr_setitem(self, space, w_idx, w_value):
         self.invalidated()
+        if (isinstance(w_idx, BaseArray) and w_idx.shape == self.shape and
+            w_idx.find_dtype().is_bool_type()):
+            return self.get_concrete().setitem_filter(space,
+                                                      w_idx.get_concrete(),
+                                             convert_to_array(space, w_value))
         if self._single_item_result(space, w_idx):
             concrete = self.get_concrete()
             item = concrete._index_of_single_item(space, w_idx)
@@ -509,9 +593,8 @@
     def create_slice(self, chunks):
         shape = []
         i = -1
-        for i, (start_, stop, step, lgt) in enumerate(chunks):
-            if step != 0:
-                shape.append(lgt)
+        for i, chunk in enumerate(chunks):
+            chunk.extend_shape(shape)
         s = i + 1
         assert s >= 0
         shape += self.shape[s:]
@@ -724,8 +807,7 @@
                                          frame=frame,
                                          ri=ri,
                                          self=self, result=result)
-            result.dtype.setitem(result.storage, ri.offset,
-                                 sig.eval(frame, self))
+            result.setitem(ri.offset, sig.eval(frame, self))
             frame.next(shapelen)
             ri = ri.next(shapelen)
         return result
@@ -945,7 +1027,7 @@
                             builder.append('\n' + indent)
                         else:
                             builder.append(indent)
-                    view = self.create_slice([(i, 0, 0, 1)]).get_concrete()
+                    view = self.create_slice([Chunk(i, 0, 0, 1)]).get_concrete()
                     view.to_str(space, comma, builder, indent=indent + ' ',
                                                     use_ellipsis=use_ellipsis)
                 if i < self.shape[0] - 1:
@@ -962,7 +1044,7 @@
                         builder.append(indent)
                 # create_slice requires len(chunks) > 1 in order to reduce
                 # shape
-                view = self.create_slice([(i, 0, 0, 1)]).get_concrete()
+                view = self.create_slice([Chunk(i, 0, 0, 1)]).get_concrete()
                 view.to_str(space, comma, builder, indent=indent + ' ',
                                                     use_ellipsis=use_ellipsis)
                 i += 1
@@ -1091,6 +1173,10 @@
                                parent)
         self.start = start
 
+    def create_iter(self):
+        return ViewIterator(self.start, self.strides, self.backstrides,
+                            self.shape)
+
     def setshape(self, space, new_shape):
         if len(self.shape) < 1:
             return
@@ -1137,6 +1223,9 @@
         self.shape = new_shape
         self.calc_strides(new_shape)
 
+    def create_iter(self):
+        return ArrayIterator(self.size)
+
     def create_sig(self):
         return signature.ArraySignature(self.dtype)
 
@@ -1191,6 +1280,7 @@
     arr = W_NDimArray(size, shape[:], dtype=dtype, order=order)
     shapelen = len(shape)
     arr_iter = ArrayIterator(arr.size)
+    # XXX we might want to have a jitdriver here
     for i in range(len(elems_w)):
         w_elem = elems_w[i]
         dtype.setitem(arr.storage, arr_iter.offset,
@@ -1257,6 +1347,9 @@
     __gt__ = interp2app(BaseArray.descr_gt),
     __ge__ = interp2app(BaseArray.descr_ge),
 
+    __and__ = interp2app(BaseArray.descr_and),
+    __or__ = interp2app(BaseArray.descr_or),
+
     __repr__ = interp2app(BaseArray.descr_repr),
     __str__ = interp2app(BaseArray.descr_str),
     __array_interface__ = GetSetProperty(BaseArray.descr_array_iface),
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -249,15 +249,16 @@
 
 
 class W_Ufunc2(W_Ufunc):
-    _immutable_fields_ = ["comparison_func", "func", "name"]
+    _immutable_fields_ = ["comparison_func", "func", "name", "int_only"]
     argcount = 2
 
     def __init__(self, func, name, promote_to_float=False, promote_bools=False,
-        identity=None, comparison_func=False):
+        identity=None, comparison_func=False, int_only=False):
 
         W_Ufunc.__init__(self, name, promote_to_float, promote_bools, identity)
         self.func = func
         self.comparison_func = comparison_func
+        self.int_only = int_only
 
     def call(self, space, args_w):
         from pypy.module.micronumpy.interp_numarray import (Call2,
@@ -268,6 +269,7 @@
         w_rhs = convert_to_array(space, w_rhs)
         calc_dtype = find_binop_result_dtype(space,
             w_lhs.find_dtype(), w_rhs.find_dtype(),
+            int_only=self.int_only,
             promote_to_float=self.promote_to_float,
             promote_bools=self.promote_bools,
         )
@@ -304,10 +306,12 @@
 
 
 def find_binop_result_dtype(space, dt1, dt2, promote_to_float=False,
-    promote_bools=False):
+    promote_bools=False, int_only=False):
     # dt1.num should be <= dt2.num
     if dt1.num > dt2.num:
         dt1, dt2 = dt2, dt1
+    if int_only and (not dt1.is_int_type() or not dt2.is_int_type()):
+        raise OperationError(space.w_TypeError, space.wrap("Unsupported types"))
     # Some operations promote op(bool, bool) to return int8, rather than bool
     if promote_bools and (dt1.kind == dt2.kind == interp_dtype.BOOLLTR):
         return interp_dtype.get_dtype_cache(space).w_int8dtype
@@ -425,6 +429,10 @@
             ("add", "add", 2, {"identity": 0}),
             ("subtract", "sub", 2),
             ("multiply", "mul", 2, {"identity": 1}),
+            ("bitwise_and", "bitwise_and", 2, {"identity": 1,
+                                               'int_only': True}),
+            ("bitwise_or", "bitwise_or", 2, {"identity": 0,
+                                             'int_only': True}),
             ("divide", "div", 2, {"promote_bools": True}),
             ("mod", "mod", 2, {"promote_bools": True}),
             ("power", "pow", 2, {"promote_bools": True}),
@@ -476,7 +484,7 @@
         extra_kwargs["identity"] = identity
 
         func = ufunc_dtype_caller(space, ufunc_name, op_name, argcount,
-            comparison_func=extra_kwargs.get("comparison_func", False)
+            comparison_func=extra_kwargs.get("comparison_func", False),
         )
         if argcount == 1:
             ufunc = W_Ufunc1(func, ufunc_name, **extra_kwargs)
diff --git a/pypy/module/micronumpy/signature.py b/pypy/module/micronumpy/signature.py
--- a/pypy/module/micronumpy/signature.py
+++ b/pypy/module/micronumpy/signature.py
@@ -82,6 +82,16 @@
         for i in range(len(self.iterators)):
             self.iterators[i] = self.iterators[i].next(shapelen)
 
+    @unroll_safe
+    def next_from_second(self, shapelen):
+        """ Don't increase the first iterator
+        """
+        for i in range(1, len(self.iterators)):
+            self.iterators[i] = self.iterators[i].next(shapelen)
+
+    def next_first(self, shapelen):
+        self.iterators[0] = self.iterators[0].next(shapelen)
+
     def get_final_iter(self):
         final_iter = promote(self.final_iter)
         if final_iter < 0:
diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py
--- a/pypy/module/micronumpy/strides.py
+++ b/pypy/module/micronumpy/strides.py
@@ -10,12 +10,12 @@
     rstart = start
     rshape = []
     i = -1
-    for i, (start_, stop, step, lgt) in enumerate(chunks):
-        if step != 0:
-            rstrides.append(strides[i] * step)
-            rbackstrides.append(strides[i] * (lgt - 1) * step)
-            rshape.append(lgt)
-        rstart += strides[i] * start_
+    for i, chunk in enumerate(chunks):
+        if chunk.step != 0:
+            rstrides.append(strides[i] * chunk.step)
+            rbackstrides.append(strides[i] * (chunk.lgt - 1) * chunk.step)
+            rshape.append(chunk.lgt)
+        rstart += strides[i] * chunk.start
     # add a reminder
     s = i + 1
     assert s >= 0
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -2,6 +2,7 @@
 import py
 from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
 from pypy.module.micronumpy.interp_numarray import W_NDimArray, shape_agreement
+from pypy.module.micronumpy.interp_iter import Chunk
 from pypy.module.micronumpy import signature
 from pypy.interpreter.error import OperationError
 from pypy.conftest import gettestobjspace
@@ -37,53 +38,54 @@
 
     def test_create_slice_f(self):
         a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'F')
-        s = a.create_slice([(3, 0, 0, 1)])
+        s = a.create_slice([Chunk(3, 0, 0, 1)])
         assert s.start == 3
         assert s.strides == [10, 50]
         assert s.backstrides == [40, 100]
-        s = a.create_slice([(1, 9, 2, 4)])
+        s = a.create_slice([Chunk(1, 9, 2, 4)])
         assert s.start == 1
         assert s.strides == [2, 10, 50]
         assert s.backstrides == [6, 40, 100]
-        s = a.create_slice([(1, 5, 3, 2), (1, 2, 1, 1), (1, 0, 0, 1)])
+        s = a.create_slice([Chunk(1, 5, 3, 2), Chunk(1, 2, 1, 1), Chunk(1, 0, 0, 1)])
         assert s.shape == [2, 1]
         assert s.strides == [3, 10]
         assert s.backstrides == [3, 0]
-        s = a.create_slice([(0, 10, 1, 10), (2, 0, 0, 1)])
+        s = a.create_slice([Chunk(0, 10, 1, 10), Chunk(2, 0, 0, 1)])
         assert s.start == 20
         assert s.shape == [10, 3]
 
     def test_create_slice_c(self):
         a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'C')
-        s = a.create_slice([(3, 0, 0, 1)])
+        s = a.create_slice([Chunk(3, 0, 0, 1)])
         assert s.start == 45
         assert s.strides == [3, 1]
         assert s.backstrides == [12, 2]
-        s = a.create_slice([(1, 9, 2, 4)])
+        s = a.create_slice([Chunk(1, 9, 2, 4)])
         assert s.start == 15
         assert s.strides == [30, 3, 1]
         assert s.backstrides == [90, 12, 2]
-        s = a.create_slice([(1, 5, 3, 2), (1, 2, 1, 1), (1, 0, 0, 1)])
+        s = a.create_slice([Chunk(1, 5, 3, 2), Chunk(1, 2, 1, 1),
+                            Chunk(1, 0, 0, 1)])
         assert s.start == 19
         assert s.shape == [2, 1]
         assert s.strides == [45, 3]
         assert s.backstrides == [45, 0]
-        s = a.create_slice([(0, 10, 1, 10), (2, 0, 0, 1)])
+        s = a.create_slice([Chunk(0, 10, 1, 10), Chunk(2, 0, 0, 1)])
         assert s.start == 6
         assert s.shape == [10, 3]
 
     def test_slice_of_slice_f(self):
         a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'F')
-        s = a.create_slice([(5, 0, 0, 1)])
+        s = a.create_slice([Chunk(5, 0, 0, 1)])
         assert s.start == 5
-        s2 = s.create_slice([(3, 0, 0, 1)])
+        s2 = s.create_slice([Chunk(3, 0, 0, 1)])
         assert s2.shape == [3]
         assert s2.strides == [50]
         assert s2.parent is a
         assert s2.backstrides == [100]
         assert s2.start == 35
-        s = a.create_slice([(1, 5, 3, 2)])
-        s2 = s.create_slice([(0, 2, 1, 2), (2, 0, 0, 1)])
+        s = a.create_slice([Chunk(1, 5, 3, 2)])
+        s2 = s.create_slice([Chunk(0, 2, 1, 2), Chunk(2, 0, 0, 1)])
         assert s2.shape == [2, 3]
         assert s2.strides == [3, 50]
         assert s2.backstrides == [3, 100]
@@ -91,16 +93,16 @@
 
     def test_slice_of_slice_c(self):
         a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), order='C')
-        s = a.create_slice([(5, 0, 0, 1)])
+        s = a.create_slice([Chunk(5, 0, 0, 1)])
         assert s.start == 15 * 5
-        s2 = s.create_slice([(3, 0, 0, 1)])
+        s2 = s.create_slice([Chunk(3, 0, 0, 1)])
         assert s2.shape == [3]
         assert s2.strides == [1]
         assert s2.parent is a
         assert s2.backstrides == [2]
         assert s2.start == 5 * 15 + 3 * 3
-        s = a.create_slice([(1, 5, 3, 2)])
-        s2 = s.create_slice([(0, 2, 1, 2), (2, 0, 0, 1)])
+        s = a.create_slice([Chunk(1, 5, 3, 2)])
+        s2 = s.create_slice([Chunk(0, 2, 1, 2), Chunk(2, 0, 0, 1)])
         assert s2.shape == [2, 3]
         assert s2.strides == [45, 1]
         assert s2.backstrides == [45, 2]
@@ -108,14 +110,14 @@
 
     def test_negative_step_f(self):
         a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'F')
-        s = a.create_slice([(9, -1, -2, 5)])
+        s = a.create_slice([Chunk(9, -1, -2, 5)])
         assert s.start == 9
         assert s.strides == [-2, 10, 50]
         assert s.backstrides == [-8, 40, 100]
 
     def test_negative_step_c(self):
         a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), order='C')
-        s = a.create_slice([(9, -1, -2, 5)])
+        s = a.create_slice([Chunk(9, -1, -2, 5)])
         assert s.start == 135
         assert s.strides == [-30, 3, 1]
         assert s.backstrides == [-120, 12, 2]
@@ -124,7 +126,7 @@
         a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'F')
         r = a._index_of_single_item(self.space, self.newtuple(1, 2, 2))
         assert r == 1 + 2 * 10 + 2 * 50
-        s = a.create_slice([(0, 10, 1, 10), (2, 0, 0, 1)])
+        s = a.create_slice([Chunk(0, 10, 1, 10), Chunk(2, 0, 0, 1)])
         r = s._index_of_single_item(self.space, self.newtuple(1, 0))
         assert r == a._index_of_single_item(self.space, self.newtuple(1, 2, 0))
         r = s._index_of_single_item(self.space, self.newtuple(1, 1))
@@ -134,7 +136,7 @@
         a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'C')
         r = a._index_of_single_item(self.space, self.newtuple(1, 2, 2))
         assert r == 1 * 3 * 5 + 2 * 3 + 2
-        s = a.create_slice([(0, 10, 1, 10), (2, 0, 0, 1)])
+        s = a.create_slice([Chunk(0, 10, 1, 10), Chunk(2, 0, 0, 1)])
         r = s._index_of_single_item(self.space, self.newtuple(1, 0))
         assert r == a._index_of_single_item(self.space, self.newtuple(1, 2, 0))
         r = s._index_of_single_item(self.space, self.newtuple(1, 1))
@@ -1302,15 +1304,25 @@
         assert isinstance(i['data'][0], int)
         raises(TypeError, getattr, array(3), '__array_interface__')
 
+    def test_array_indexing_one_elem(self):
+        skip("not yet")
+        from _numpypy import array, arange
+        raises(IndexError, 'arange(3)[array([3.5])]')
+        a = arange(3)[array([1])]
+        assert a == 1
+        assert a[0] == 1
+        raises(IndexError,'arange(3)[array([15])]')
+        assert arange(3)[array([-3])] == 0
+        raises(IndexError,'arange(3)[array([-15])]')
+        assert arange(3)[array(1)] == 1
+
     def test_fill(self):
         from _numpypy import array
-
         a = array([1, 2, 3])
         a.fill(10)
         assert (a == [10, 10, 10]).all()
         a.fill(False)
         assert (a == [0, 0, 0]).all()
-
         b = a[:1]
         b.fill(4)
         assert (b == [4]).all()
@@ -1324,6 +1336,24 @@
         d.fill(100)
         assert d == 100
 
+    def test_array_indexing_bool(self):
+        from _numpypy import arange
+        a = arange(10)
+        assert (a[a > 3] == [4, 5, 6, 7, 8, 9]).all()
+        a = arange(10).reshape(5, 2)
+        assert (a[a > 3] == [4, 5, 6, 7, 8, 9]).all()
+        assert (a[a & 1 == 1] == [1, 3, 5, 7, 9]).all()
+
+    def test_array_indexing_bool_setitem(self):
+        from _numpypy import arange, array
+        a = arange(6)
+        a[a > 3] = 15
+        assert (a == [0, 1, 2, 3, 15, 15]).all()
+        a = arange(6).reshape(3, 2)
+        a[a & 1 == 1] = array([8, 9, 10])
+        assert (a == [[0, 8], [2, 9], [4, 10]]).all()
+
+
 
 class AppTestSupport(BaseNumpyAppTest):
     def setup_class(cls):
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -355,11 +355,20 @@
         raises(ValueError, maximum.reduce, [])
 
     def test_reduceND(self):
-        from numpypy import add, arange
+        from _numpypy import add, arange
         a = arange(12).reshape(3, 4)
         assert (add.reduce(a, 0) == [12, 15, 18, 21]).all()
         assert (add.reduce(a, 1) == [6.0, 22.0, 38.0]).all()
 
+    def test_bitwise(self):
+        from _numpypy import bitwise_and, bitwise_or, arange, array
+        a = arange(6).reshape(2, 3)
+        assert (a & 1 == [[0, 1, 0], [1, 0, 1]]).all()
+        assert (a & 1 == bitwise_and(a, 1)).all()
+        assert (a | 1 == [[1, 1, 3], [3, 5, 5]]).all()
+        assert (a | 1 == bitwise_or(a, 1)).all()
+        raises(TypeError, 'array([1.0]) & 1')
+
     def test_comparisons(self):
         import operator
         from _numpypy import equal, not_equal, less, less_equal, greater, greater_equal
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -217,6 +217,7 @@
         # This is the sum of the ops for both loops, however if you remove the
         # optimization then you end up with 2 float_adds, so we can still be
         # sure it was optimized correctly.
+        py.test.skip("too fragile")
         self.check_resops({'setinteriorfield_raw': 4, 'getfield_gc': 22,
                            'getarrayitem_gc': 4, 'getarrayitem_gc_pure': 2,
                            'getfield_gc_pure': 8,
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -94,6 +94,9 @@
             width, storage, i, offset
         ))
 
+    def read_bool(self, storage, width, i, offset):
+        raise NotImplementedError
+
     def store(self, storage, width, i, offset, box):
         value = self.unbox(box)
         libffi.array_setitem(clibffi.cast_type_to_ffitype(self.T),
@@ -168,6 +171,7 @@
     @simple_binary_op
     def min(self, v1, v2):
         return min(v1, v2)
+    
 
 class Bool(BaseType, Primitive):
     T = lltype.Bool
@@ -185,6 +189,11 @@
         else:
             return self.False
 
+
+    def read_bool(self, storage, width, i, offset):
+        return libffi.array_getitem(clibffi.cast_type_to_ffitype(self.T),
+                                    width, storage, i, offset)
+
     def coerce_subtype(self, space, w_subtype, w_item):
         # Doesn't return subclasses so it can return the constants.
         return self._coerce(space, w_item)
@@ -253,6 +262,14 @@
             assert v == 0
             return 0
 
+    @simple_binary_op
+    def bitwise_and(self, v1, v2):
+        return v1 & v2
+
+    @simple_binary_op
+    def bitwise_or(self, v1, v2):
+        return v1 | v2
+
 class Int8(BaseType, Integer):
     T = rffi.SIGNEDCHAR
     BoxType = interp_boxes.W_Int8Box

From noreply at buildbot.pypy.org  Fri Jan 20 15:32:23 2012
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Fri, 20 Jan 2012 15:32:23 +0100 (CET)
Subject: [pypy-commit] pypy default: resuffle pypy.tool.version and add
 support for .hg_archival.txt, fixes issue952
Message-ID: <20120120143223.5C1FB82CF8@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt 
Branch: 
Changeset: r51525:79770e0c2f93
Date: 2012-01-20 15:31 +0100
http://bitbucket.org/pypy/pypy/changeset/79770e0c2f93/

Log:	resuffle pypy.tool.version and add support for .hg_archival.txt,
	fixes issue952

diff --git a/pypy/tool/test/test_version.py b/pypy/tool/test/test_version.py
--- a/pypy/tool/test/test_version.py
+++ b/pypy/tool/test/test_version.py
@@ -1,6 +1,22 @@
 import os, sys
 import py
-from pypy.tool.version import get_repo_version_info
+from pypy.tool.version import get_repo_version_info, _get_hg_archive_version
+
+def test_hg_archival_version(tmpdir):
+    def version_for(name, **kw):
+        path = tmpdir.join(name)
+        path.write('\n'.join('%s: %s' % x for x in kw.items()))
+        return _get_hg_archive_version(str(path))
+
+    assert version_for('release',
+                       tag='release-123',
+                       node='000',
+                      ) == ('PyPy', 'release-123', '000')
+    assert version_for('somebranch',
+                       node='000',
+                       branch='something',
+                      ) == ('PyPy', 'something', '000')
+
 
 def test_get_repo_version_info():
     assert get_repo_version_info(None)
diff --git a/pypy/tool/version.py b/pypy/tool/version.py
--- a/pypy/tool/version.py
+++ b/pypy/tool/version.py
@@ -3,111 +3,139 @@
 from subprocess import Popen, PIPE
 import pypy
 pypydir = os.path.dirname(os.path.abspath(pypy.__file__))
+pypyroot = os.path.dirname(pypydir)
+default_retval = 'PyPy', '?', '?'
+
+def maywarn(err, repo_type='Mercurial'):
+    if not err:
+        return
+
+    from pypy.tool.ansi_print import ansi_log
+    log = py.log.Producer("version")
+    py.log.setconsumer("version", ansi_log)
+    log.WARNING('Errors getting %s information: %s' % (repo_type, err))
 
 def get_repo_version_info(hgexe=None):
     '''Obtain version information by invoking the 'hg' or 'git' commands.'''
-    # TODO: support extracting from .hg_archival.txt
-
-    default_retval = 'PyPy', '?', '?'
-    pypyroot = os.path.abspath(os.path.join(pypydir, '..'))
-
-    def maywarn(err, repo_type='Mercurial'):
-        if not err:
-            return
-
-        from pypy.tool.ansi_print import ansi_log
-        log = py.log.Producer("version")
-        py.log.setconsumer("version", ansi_log)
-        log.WARNING('Errors getting %s information: %s' % (repo_type, err))
 
     # Try to see if we can get info from Git if hgexe is not specified.
     if not hgexe:
         if os.path.isdir(os.path.join(pypyroot, '.git')):
-            gitexe = py.path.local.sysfind('git')
-            if gitexe:
-                try:
-                    p = Popen(
-                        [str(gitexe), 'rev-parse', 'HEAD'],
-                        stdout=PIPE, stderr=PIPE, cwd=pypyroot
-                        )
-                except OSError, e:
-                    maywarn(e, 'Git')
-                    return default_retval
-                if p.wait() != 0:
-                    maywarn(p.stderr.read(), 'Git')
-                    return default_retval
-                revision_id = p.stdout.read().strip()[:12]
-                p = Popen(
-                    [str(gitexe), 'describe', '--tags', '--exact-match'],
-                    stdout=PIPE, stderr=PIPE, cwd=pypyroot
-                    )
-                if p.wait() != 0:
-                    p = Popen(
-                        [str(gitexe), 'branch'], stdout=PIPE, stderr=PIPE,
-                        cwd=pypyroot
-                        )
-                    if p.wait() != 0:
-                        maywarn(p.stderr.read(), 'Git')
-                        return 'PyPy', '?', revision_id
-                    branch = '?'
-                    for line in p.stdout.read().strip().split('\n'):
-                        if line.startswith('* '):
-                            branch = line[1:].strip()
-                            if branch == '(no branch)':
-                                branch = '?'
-                            break
-                    return 'PyPy', branch, revision_id
-                return 'PyPy', p.stdout.read().strip(), revision_id
+            return _get_git_version()
 
     # Fallback to trying Mercurial.
     if hgexe is None:
         hgexe = py.path.local.sysfind('hg')
 
-    if not os.path.isdir(os.path.join(pypyroot, '.hg')):
+    if os.path.isfile(os.path.join(pypyroot, '.hg_archival.txt')):
+        return _get_hg_archive_version(os.path.join(pypyroot, '.hg_archival.txt'))
+    elif not os.path.isdir(os.path.join(pypyroot, '.hg')):
         maywarn('Not running from a Mercurial repository!')
         return default_retval
     elif not hgexe:
         maywarn('Cannot find Mercurial command!')
         return default_retval
     else:
-        env = dict(os.environ)
-        # get Mercurial into scripting mode
-        env['HGPLAIN'] = '1'
-        # disable user configuration, extensions, etc.
-        env['HGRCPATH'] = os.devnull
+        return _get_hg_version(hgexe)
 
-        try:
-            p = Popen([str(hgexe), 'version', '-q'],
-                      stdout=PIPE, stderr=PIPE, env=env)
-        except OSError, e:
-            maywarn(e)
-            return default_retval
 
-        if not p.stdout.read().startswith('Mercurial Distributed SCM'):
-            maywarn('command does not identify itself as Mercurial')
-            return default_retval
+def _get_hg_version(hgexe):
+    env = dict(os.environ)
+    # get Mercurial into scripting mode
+    env['HGPLAIN'] = '1'
+    # disable user configuration, extensions, etc.
+    env['HGRCPATH'] = os.devnull
 
-        p = Popen([str(hgexe), 'id', '-i', pypyroot],
+    try:
+        p = Popen([str(hgexe), 'version', '-q'],
                   stdout=PIPE, stderr=PIPE, env=env)
-        hgid = p.stdout.read().strip()
+    except OSError, e:
+        maywarn(e)
+        return default_retval
+
+    if not p.stdout.read().startswith('Mercurial Distributed SCM'):
+        maywarn('command does not identify itself as Mercurial')
+        return default_retval
+
+    p = Popen([str(hgexe), 'id', '-i', pypyroot],
+              stdout=PIPE, stderr=PIPE, env=env)
+    hgid = p.stdout.read().strip()
+    maywarn(p.stderr.read())
+    if p.wait() != 0:
+        hgid = '?'
+
+    p = Popen([str(hgexe), 'id', '-t', pypyroot],
+              stdout=PIPE, stderr=PIPE, env=env)
+    hgtags = [t for t in p.stdout.read().strip().split() if t != 'tip']
+    maywarn(p.stderr.read())
+    if p.wait() != 0:
+        hgtags = ['?']
+
+    if hgtags:
+        return 'PyPy', hgtags[0], hgid
+    else:
+        # use the branch instead
+        p = Popen([str(hgexe), 'id', '-b', pypyroot],
+                  stdout=PIPE, stderr=PIPE, env=env)
+        hgbranch = p.stdout.read().strip()
         maywarn(p.stderr.read())
+
+        return 'PyPy', hgbranch, hgid
+
+
+def _get_hg_archive_version(path):
+    fp = open(path)
+    try:
+        data = dict(x.split(': ', 1) for x in fp.read().splitlines())
+    finally:
+        fp.close()
+    if 'tag' in data:
+        return 'PyPy', data['tag'], data['node']
+    else:
+        return 'PyPy', data['branch'], data['node']
+
+
+def _get_git_version():
+    #XXX: this function is a untested hack,
+    #     so the git mirror tav made will work
+    gitexe = py.path.local.sysfind('git')
+    if not gitexe:
+        return default_retval
+
+    try:
+        p = Popen(
+            [str(gitexe), 'rev-parse', 'HEAD'],
+            stdout=PIPE, stderr=PIPE, cwd=pypyroot
+            )
+    except OSError, e:
+        maywarn(e, 'Git')
+        return default_retval
+    if p.wait() != 0:
+        maywarn(p.stderr.read(), 'Git')
+        return default_retval
+    revision_id = p.stdout.read().strip()[:12]
+    p = Popen(
+        [str(gitexe), 'describe', '--tags', '--exact-match'],
+        stdout=PIPE, stderr=PIPE, cwd=pypyroot
+        )
+    if p.wait() != 0:
+        p = Popen(
+            [str(gitexe), 'branch'], stdout=PIPE, stderr=PIPE,
+            cwd=pypyroot
+            )
         if p.wait() != 0:
-            hgid = '?'
+            maywarn(p.stderr.read(), 'Git')
+            return 'PyPy', '?', revision_id
+        branch = '?'
+        for line in p.stdout.read().strip().split('\n'):
+            if line.startswith('* '):
+                branch = line[1:].strip()
+                if branch == '(no branch)':
+                    branch = '?'
+                break
+        return 'PyPy', branch, revision_id
+    return 'PyPy', p.stdout.read().strip(), revision_id
 
-        p = Popen([str(hgexe), 'id', '-t', pypyroot],
-                  stdout=PIPE, stderr=PIPE, env=env)
-        hgtags = [t for t in p.stdout.read().strip().split() if t != 'tip']
-        maywarn(p.stderr.read())
-        if p.wait() != 0:
-            hgtags = ['?']
 
-        if hgtags:
-            return 'PyPy', hgtags[0], hgid
-        else:
-            # use the branch instead
-            p = Popen([str(hgexe), 'id', '-b', pypyroot],
-                      stdout=PIPE, stderr=PIPE, env=env)
-            hgbranch = p.stdout.read().strip()
-            maywarn(p.stderr.read())
-
-            return 'PyPy', hgbranch, hgid
+if __name__ == '__main__':
+    print get_repo_version_info()

From noreply at buildbot.pypy.org  Fri Jan 20 15:35:10 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Fri, 20 Jan 2012 15:35:10 +0100 (CET)
Subject: [pypy-commit] pypy core-only-tracing: (antocuni, arigo, bivab,
 romain): a branch where to experiment with 'lightweight tracing',
 in which we inline only the opcode implementations and nothing else
Message-ID: <20120120143510.7848E82CF8@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni 
Branch: core-only-tracing
Changeset: r51526:08855ad04675
Date: 2012-01-20 11:10 +0100
http://bitbucket.org/pypy/pypy/changeset/08855ad04675/

Log:	(antocuni, arigo, bivab, romain): a branch where to experiment with
	'lightweight tracing', in which we inline only the opcode
	implementations and nothing else


From noreply at buildbot.pypy.org  Fri Jan 20 15:35:11 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Fri, 20 Jan 2012 15:35:11 +0100 (CET)
Subject: [pypy-commit] pypy core-only-tracing: (antocuni, arigo,
 romain): introduce the is_core flag on JitCodes,
 which will be used to select which graphs to inline when tracing in
 core-only mode
Message-ID: <20120120143511.B43E282CF8@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni 
Branch: core-only-tracing
Changeset: r51527:655088dbaa64
Date: 2012-01-20 12:01 +0100
http://bitbucket.org/pypy/pypy/changeset/655088dbaa64/

Log:	(antocuni, arigo, romain): introduce the is_core flag on JitCodes,
	which will be used to select which graphs to inline when tracing in
	core-only mode

diff --git a/pypy/jit/codewriter/call.py b/pypy/jit/codewriter/call.py
--- a/pypy/jit/codewriter/call.py
+++ b/pypy/jit/codewriter/call.py
@@ -53,6 +53,7 @@
                                                           ll_args, ll_res)
                 todo.append(c_func.value._obj.graph)
         candidate_graphs = set(todo)
+        core_candidate_graphs = set(todo)
 
         def callers():
             graph = top_graph
@@ -77,8 +78,11 @@
                     assert is_candidate(graph)
                     todo.append(graph)
                     candidate_graphs.add(graph)
+                    if policy.is_core_graph(graph):
+                        core_candidate_graphs.add(graph)
                     coming_from[graph] = top_graph
         self.candidate_graphs = candidate_graphs
+        self.core_candidate_graphs = core_candidate_graphs
         return candidate_graphs
 
     def graphs_from(self, op, is_candidate=None):
@@ -148,6 +152,9 @@
         # used only after find_all_graphs()
         return graph in self.candidate_graphs
 
+    def is_core(self, graph):
+        return graph in self.core_candidate_graphs
+
     def grab_initial_jitcodes(self):
         for jd in self.jitdrivers_sd:
             jd.mainjitcode = self.get_jitcode(jd.portal_graph)
@@ -164,8 +171,9 @@
             return self.jitcodes[graph]
         except KeyError:
             fnaddr, calldescr = self.get_jitcode_calldescr(graph)
+            is_core = self.is_core(graph)
             jitcode = JitCode(graph.name, fnaddr, calldescr,
-                              called_from=called_from)
+                              called_from=called_from, is_core=is_core)
             self.jitcodes[graph] = jitcode
             self.unfinished_graphs.append(graph)
             return jitcode
diff --git a/pypy/jit/codewriter/jitcode.py b/pypy/jit/codewriter/jitcode.py
--- a/pypy/jit/codewriter/jitcode.py
+++ b/pypy/jit/codewriter/jitcode.py
@@ -8,11 +8,12 @@
     _empty_r = []
     _empty_f = []
 
-    def __init__(self, name, fnaddr=None, calldescr=None, called_from=None):
+    def __init__(self, name, fnaddr=None, calldescr=None, called_from=None, is_core=False):
         self.name = name
         self.fnaddr = fnaddr
         self.calldescr = calldescr
         self.is_portal = False
+        self.is_core = is_core
         self._called_from = called_from   # debugging
         self._ssarepr     = None          # debugging
 
diff --git a/pypy/jit/codewriter/policy.py b/pypy/jit/codewriter/policy.py
--- a/pypy/jit/codewriter/policy.py
+++ b/pypy/jit/codewriter/policy.py
@@ -38,6 +38,9 @@
     def look_inside_function(self, func):
         return True # look into everything by default
 
+    def is_core_graph(self, graph):
+        return True
+
     def _reject_function(self, func):
         if hasattr(func, '_jit_look_inside_'):
             return not func._jit_look_inside_
diff --git a/pypy/jit/codewriter/test/test_call.py b/pypy/jit/codewriter/test/test_call.py
--- a/pypy/jit/codewriter/test/test_call.py
+++ b/pypy/jit/codewriter/test/test_call.py
@@ -10,6 +10,9 @@
     def look_inside_graph(self, graph):
         return True
 
+    def is_core_graph(self, graph):
+        return True
+
 
 def test_graphs_from_direct_call():
     cc = CallControl()
@@ -159,6 +162,7 @@
                 return lltype.functionptr(F, 'bar')
     #
     cc = CallControl(FakeCPU(FakeRTyper()))
+    cc.core_candidate_graphs = set()
     class somegraph:
         name = "foo"
     jitcode = cc.get_jitcode(somegraph)
@@ -210,3 +214,33 @@
     op = block.operations[-1]
     call_descr = cc.getcalldescr(op)
     assert call_descr.extrainfo.has_random_effects()
+
+
+def test_mark_jitcode_as_core():
+    from pypy.jit.codewriter.test.test_flatten import FakeCPU
+
+    class MyPolicy:
+        def look_inside_graph(self, graph):
+            return graph.name in ('f', 'g')
+        
+        def is_core_graph(self, graph):
+            if graph.name == 'f':
+                return True
+            return False
+
+    def g(x):
+        return x + 2
+    def f(x):
+        return g(x) + 1
+    rtyper = support.annotate(f, [7])
+    jitdriver_sd = FakeJitDriverSD(rtyper.annotator.translator.graphs[0])
+    cc = CallControl(jitdrivers_sd=[jitdriver_sd])
+    res = cc.find_all_graphs(MyPolicy())
+    # hack hack hack
+    cc.cpu = FakeCPU(rtyper)
+    cc.rtyper = rtyper
+    graphs = dict([(graph.name, graph) for graph in res])
+    jitcode_f = cc.get_jitcode(graphs['f'])
+    jitcode_g = cc.get_jitcode(graphs['g'])
+    assert jitcode_f.is_core
+    assert not jitcode_g.is_core
diff --git a/pypy/jit/codewriter/test/test_codewriter.py b/pypy/jit/codewriter/test/test_codewriter.py
--- a/pypy/jit/codewriter/test/test_codewriter.py
+++ b/pypy/jit/codewriter/test/test_codewriter.py
@@ -40,6 +40,9 @@
     def look_inside_graph(self, graph):
         return graph.name != 'dont_look'
 
+    def is_core_graph(self, graph):
+        return True
+
 class FakeJitDriverSD:
     def __init__(self, portal_graph):
         self.portal_graph = portal_graph
@@ -162,6 +165,9 @@
             name = graph.name
             return not (name.startswith('instantiate_') and
                         name.endswith('A2'))
+
+        def is_core_graph(self, graph):
+            return True
     class A1:
         pass
     class A2(A1):

From noreply at buildbot.pypy.org  Fri Jan 20 15:35:12 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Fri, 20 Jan 2012 15:35:12 +0100 (CET)
Subject: [pypy-commit] pypy core-only-tracing: (antocuni, arigo,
 romain) complain if a graph is marked as access_direct but not core,
 because in this case we would have a wrong behaviour when tracing in
 core-only mode
Message-ID: <20120120143512.E8F2582CF8@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni 
Branch: core-only-tracing
Changeset: r51528:da475d18764e
Date: 2012-01-20 12:11 +0100
http://bitbucket.org/pypy/pypy/changeset/da475d18764e/

Log:	(antocuni, arigo, romain) complain if a graph is marked as
	access_direct but not core, because in this case we would have a
	wrong behaviour when tracing in core-only mode

diff --git a/pypy/jit/codewriter/policy.py b/pypy/jit/codewriter/policy.py
--- a/pypy/jit/codewriter/policy.py
+++ b/pypy/jit/codewriter/policy.py
@@ -76,19 +76,22 @@
         if res and contains_loop:
             self.unsafe_loopy_graphs.add(graph)
         res = res and not contains_loop
-        if (see_function and not res and
-            getattr(graph, "access_directly", False)):
-            # This happens when we have a function which has an argument with
-            # the access_directly flag, and the annotator has determined we will
-            # see the function. (See
-            # pypy/annotation/specialize.py:default_specialize) However,
-            # look_inside_graph just decided that we will not see it. (It has a
-            # loop or unsupported variables.) If we return False, the call will
-            # be turned into a residual call, but the graph is access_directly!
-            # If such a function is called and accesses a virtualizable, the JIT
-            # will not notice, and the virtualizable will fall out of sync. So,
-            # we fail loudly now.
-            raise ValueError("access_directly on a function which we don't see %s" % graph)
+        if getattr(graph, "access_directly", False):
+            if (see_function and not res):
+                # This happens when we have a function which has an argument with
+                # the access_directly flag, and the annotator has determined we will
+                # see the function. (See
+                # pypy/annotation/specialize.py:default_specialize) However,
+                # look_inside_graph just decided that we will not see it. (It has a
+                # loop or unsupported variables.) If we return False, the call will
+                # be turned into a residual call, but the graph is access_directly!
+                # If such a function is called and accesses a virtualizable, the JIT
+                # will not notice, and the virtualizable will fall out of sync. So,
+                # we fail loudly now.
+                raise ValueError("access_directly on a function which we don't see %s" % graph)
+            if not self.is_core_graph(graph):
+                # the same comment as above applies when we run in core-only tracing mode
+                raise ValueError("access_directly on a function which is not core: %s" % graph)
         return res
 
 def contains_unsupported_variable_type(graph, supports_floats,
diff --git a/pypy/jit/codewriter/test/test_policy.py b/pypy/jit/codewriter/test/test_policy.py
--- a/pypy/jit/codewriter/test/test_policy.py
+++ b/pypy/jit/codewriter/test/test_policy.py
@@ -130,3 +130,23 @@
     h_graph = rtyper.annotator.translator.graphs[1]
     assert h_graph.func is h
     py.test.raises(ValueError, JitPolicy().look_inside_graph, h_graph)
+
+
+def test_access_directly_but_not_core():
+    class MyPolicy(JitPolicy):
+        def is_core_graph(self, graph):
+            assert graph.name.startswith('h__AccessDirect')
+            return False
+    
+    class X:
+        _virtualizable2_ = ["a"]
+    def h(x, y):
+        return x.a + y
+    def f(y):
+        x = jit.hint(X(), access_directly=True)
+        x.a = 4
+        h(x, y)
+    rtyper = support.annotate(f, [3])
+    h_graph = rtyper.annotator.translator.graphs[1]
+    assert h_graph.func is h
+    py.test.raises(ValueError, MyPolicy().look_inside_graph, h_graph)

From noreply at buildbot.pypy.org  Fri Jan 20 15:35:14 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Fri, 20 Jan 2012 15:35:14 +0100 (CET)
Subject: [pypy-commit] pypy core-only-tracing: implement the 'fast' jit
 mode, in which we trace only inside the 'core' graphs, and do residual
 calls to everything else;
 still in-progress, at least one case is missing, see next checkin
Message-ID: <20120120143514.466E982CF8@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni 
Branch: core-only-tracing
Changeset: r51529:927230d64a20
Date: 2012-01-20 14:12 +0100
http://bitbucket.org/pypy/pypy/changeset/927230d64a20/

Log:	implement the 'fast' jit mode, in which we trace only inside the
	'core' graphs, and do residual calls to everything else; still in-
	progress, at least one case is missing, see next checkin

diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -778,15 +778,30 @@
         result = vinfo.get_array_length(virtualizable, arrayindex)
         return ConstInt(result)
 
+    def perform_call_maybe(self, jitcode, argboxes):
+        core_only_mode = (self.metainterp.jitdriver_sd.warmstate.jitmode == 'fast')
+        if core_only_mode and self.jitcode.is_core:
+            # never inline in this mode
+            funcbox = ConstInt(jitcode.get_fnaddr_as_int())
+            # jitcode always has a calldescr, but it might not have the
+            # correct effectinfo. The result is that we might generate a
+            # call_may_force when a call might suffice; however, we don't care
+            # too much because for the PyPy interpreter most calls are to
+            # space.*, so they would be call_may_force anyway.
+            calldescr = jitcode.calldescr
+            return self.do_residual_call(funcbox, calldescr, argboxes)
+        # normal mode
+        return self.metainterp.perform_call(jitcode, argboxes)
+
     @arguments("jitcode", "boxes")
     def _opimpl_inline_call1(self, jitcode, argboxes):
-        return self.metainterp.perform_call(jitcode, argboxes)
+        return self.perform_call_maybe(jitcode, argboxes)
     @arguments("jitcode", "boxes2")
     def _opimpl_inline_call2(self, jitcode, argboxes):
-        return self.metainterp.perform_call(jitcode, argboxes)
+        return self.perform_call_maybe(jitcode, argboxes)
     @arguments("jitcode", "boxes3")
     def _opimpl_inline_call3(self, jitcode, argboxes):
-        return self.metainterp.perform_call(jitcode, argboxes)
+        return self.perform_call_maybe(jitcode, argboxes)
 
     opimpl_inline_call_r_i = _opimpl_inline_call1
     opimpl_inline_call_r_r = _opimpl_inline_call1
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -2942,6 +2942,39 @@
         res = self.meta_interp(f, [32])
         assert res == f(32)
 
+    def test_residual_call_from_core_graph(self):
+        class MyPolicy(JitPolicy):
+            def is_core_graph(self, graph):
+                return graph.name == 'f'
+        myjitdriver = JitDriver(greens = [], reds = ['x', 'y', 'res'])
+
+        def a(x):
+            return x
+        def b(x, y):
+            return x+y
+        def c(x, y, z):
+            return x+y+z
+        def f(x, y):
+            res = 0
+            while y > 0:
+                myjitdriver.can_enter_jit(x=x, y=y, res=res)
+                myjitdriver.jit_merge_point(x=x, y=y, res=res)
+                res = a(x) + b(x, res) + c(x, -x, -x) # at the end, it's like doing x+res :-)
+                y -= 1
+            return res
+        res = self.meta_interp(f, [6, 7], policy=MyPolicy(), jitmode='fast') # fast == trace only core graphs
+        assert res == 42
+        self.check_trace_count(1)
+        # this is suboptimal because we get a call_may_force instead of a
+        # call. Look at the comment inside pyjitpl...perform_call_maybe for
+        # details
+        self.check_resops({'jump': 1, 'int_gt': 2, 'guard_true': 2, 'int_sub': 2,
+                           'int_neg': 1, 'int_add': 4,
+                           'call_may_force': 6,
+                           'guard_no_exception': 6,
+                           'guard_not_forced': 6})
+
+
 
 class TestOOtype(BasicTests, OOJitMixin):
 
diff --git a/pypy/jit/metainterp/warmspot.py b/pypy/jit/metainterp/warmspot.py
--- a/pypy/jit/metainterp/warmspot.py
+++ b/pypy/jit/metainterp/warmspot.py
@@ -67,7 +67,8 @@
                     backendopt=False, trace_limit=sys.maxint,
                     inline=False, loop_longevity=0, retrace_limit=5,
                     function_threshold=4,
-                    enable_opts=ALL_OPTS_NAMES, max_retrace_guards=15, **kwds):
+                    enable_opts=ALL_OPTS_NAMES, max_retrace_guards=15,
+                    jitmode='full', **kwds):
     from pypy.config.config import ConfigError
     translator = interp.typer.annotator.translator
     try:
@@ -92,6 +93,7 @@
         jd.warmstate.set_param_loop_longevity(loop_longevity)
         jd.warmstate.set_param_retrace_limit(retrace_limit)
         jd.warmstate.set_param_max_retrace_guards(max_retrace_guards)
+        jd.warmstate.set_param_jitmode(jitmode)
         jd.warmstate.set_param_enable_opts(enable_opts)
     warmrunnerdesc.finish()
     if graph_and_interp_only:
diff --git a/pypy/jit/metainterp/warmstate.py b/pypy/jit/metainterp/warmstate.py
--- a/pypy/jit/metainterp/warmstate.py
+++ b/pypy/jit/metainterp/warmstate.py
@@ -213,6 +213,9 @@
     def set_param_inlining(self, value):
         self.inlining = value
 
+    def set_param_jitmode(self, value):
+        self.jitmode = value
+
     def set_param_enable_opts(self, value):
         from pypy.jit.metainterp.optimizeopt import ALL_OPTS_DICT, ALL_OPTS_NAMES
 
diff --git a/pypy/rlib/jit.py b/pypy/rlib/jit.py
--- a/pypy/rlib/jit.py
+++ b/pypy/rlib/jit.py
@@ -402,6 +402,7 @@
     'retrace_limit': 'how many times we can try retracing before giving up',
     'max_retrace_guards': 'number of extra guards a retrace can cause',
     'max_unroll_loops': 'number of extra unrollings a loop can cause',
+    'jitmode': '"full" (default) or "fast"',
     'enable_opts': 'optimizations to enable or all, INTERNAL USE ONLY'
     }
 
@@ -414,6 +415,7 @@
               'retrace_limit': 5,
               'max_retrace_guards': 15,
               'max_unroll_loops': 4,
+              'jitmode': 'full',
               'enable_opts': 'all',
               }
 unroll_parameters = unrolling_iterable(PARAMETERS.items())

From noreply at buildbot.pypy.org  Fri Jan 20 15:35:15 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Fri, 20 Jan 2012 15:35:15 +0100 (CET)
Subject: [pypy-commit] pypy core-only-tracing: fix tests
Message-ID: <20120120143515.7E32A82CF8@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni 
Branch: core-only-tracing
Changeset: r51530:04c7e6ed45c9
Date: 2012-01-20 14:18 +0100
http://bitbucket.org/pypy/pypy/changeset/04c7e6ed45c9/

Log:	fix tests

diff --git a/pypy/jit/metainterp/test/support.py b/pypy/jit/metainterp/test/support.py
--- a/pypy/jit/metainterp/test/support.py
+++ b/pypy/jit/metainterp/test/support.py
@@ -40,6 +40,7 @@
         _cell = FakeJitCell()
 
         trace_limit = sys.maxint
+        jitmode = 'full'
         enable_opts = ALL_OPTS_DICT
 
     func._jit_unroll_safe_ = True

From noreply at buildbot.pypy.org  Fri Jan 20 15:35:16 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Fri, 20 Jan 2012 15:35:16 +0100 (CET)
Subject: [pypy-commit] pypy core-only-tracing: make sure not to inline
 indirect calls when in core-only mode
Message-ID: <20120120143516.BE01882CF8@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni 
Branch: core-only-tracing
Changeset: r51531:03e571309f26
Date: 2012-01-20 14:24 +0100
http://bitbucket.org/pypy/pypy/changeset/03e571309f26/

Log:	make sure not to inline indirect calls when in core-only mode

diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -1388,7 +1388,7 @@
         jitcode = sd.bytecode_for_address(key)
         if jitcode is not None:
             # we should follow calls to this graph
-            return self.metainterp.perform_call(jitcode, argboxes)
+            return self.perform_call_maybe(jitcode, argboxes)
         else:
             # but we should not follow calls to that graph
             return self.do_residual_call(funcbox, calldescr, argboxes)
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -2974,6 +2974,37 @@
                            'guard_no_exception': 6,
                            'guard_not_forced': 6})
 
+    def test_dont_inline_residual_call_from_core_graph(self):
+        class MyPolicy(JitPolicy):
+            def is_core_graph(self, graph):
+                return graph.name == 'f'
+        myjitdriver = JitDriver(greens = [], reds = ['x', 'y', 'res'])
+
+        def a(x):
+            return x+2
+        def b(x):
+            return x+1
+        def f(x, y):
+            res = 0
+            while y > 0:
+                myjitdriver.can_enter_jit(x=x, y=y, res=res)
+                myjitdriver.jit_merge_point(x=x, y=y, res=res)
+                if y == 5:
+                    f = a
+                else:
+                    f = b
+                y -= 1
+                res += f(x)
+            return res
+        res = self.meta_interp(f, [5, 7], policy=MyPolicy(), jitmode='fast') # fast == trace only core graphs
+        assert res == 43
+        self.check_trace_count(1)
+        self.check_resops({'jump': 1, 'int_gt': 2, 'guard_true': 2, 'int_sub': 2,
+                           'int_eq': 2, 'int_add': 2, 'guard_false': 2,
+                           'call_may_force': 2,
+                           'guard_no_exception': 2,
+                           'guard_not_forced': 2})
+
 
 
 class TestOOtype(BasicTests, OOJitMixin):

From noreply at buildbot.pypy.org  Fri Jan 20 15:35:18 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Fri, 20 Jan 2012 15:35:18 +0100 (CET)
Subject: [pypy-commit] pypy core-only-tracing: make sure to inline
	core-to-core calls
Message-ID: <20120120143518.087D282CF8@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni 
Branch: core-only-tracing
Changeset: r51532:4a040e6b95e4
Date: 2012-01-20 15:01 +0100
http://bitbucket.org/pypy/pypy/changeset/4a040e6b95e4/

Log:	make sure to inline core-to-core calls

diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -780,8 +780,8 @@
 
     def perform_call_maybe(self, jitcode, argboxes):
         core_only_mode = (self.metainterp.jitdriver_sd.warmstate.jitmode == 'fast')
-        if core_only_mode and self.jitcode.is_core:
-            # never inline in this mode
+        # in core_only_mode, don't inline calls from core to non-core graphs
+        if core_only_mode and self.jitcode.is_core and not jitcode.is_core:
             funcbox = ConstInt(jitcode.get_fnaddr_as_int())
             # jitcode always has a calldescr, but it might not have the
             # correct effectinfo. The result is that we might generate a
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -3005,6 +3005,27 @@
                            'guard_no_exception': 2,
                            'guard_not_forced': 2})
 
+    def test_inline_core_to_core_calls(self):
+        class MyPolicy(JitPolicy):
+            def is_core_graph(self, graph):
+                return graph.name in ('f', 'a')
+        myjitdriver = JitDriver(greens = [], reds = ['x', 'y', 'res'])
+
+        def a(x, y):
+            return x+y
+        def f(x, y):
+            res = 0
+            while y > 0:
+                myjitdriver.can_enter_jit(x=x, y=y, res=res)
+                myjitdriver.jit_merge_point(x=x, y=y, res=res)
+                res = a(res, x)
+                y -= 1
+            return res
+        res = self.meta_interp(f, [6, 7], policy=MyPolicy(), jitmode='fast') # fast == trace only core graphs
+        assert res == 42
+        self.check_trace_count(1)
+        self.check_resops({'jump': 1, 'int_gt': 2, 'guard_true': 2, 'int_sub': 2,
+                           'int_add': 2})
 
 
 class TestOOtype(BasicTests, OOJitMixin):

From noreply at buildbot.pypy.org  Fri Jan 20 15:35:19 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Fri, 20 Jan 2012 15:35:19 +0100 (CET)
Subject: [pypy-commit] pypy core-only-tracing: rename 'fast' mode to
	'core-only'
Message-ID: <20120120143519.4934A82CF8@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni 
Branch: core-only-tracing
Changeset: r51533:94130878552c
Date: 2012-01-20 15:05 +0100
http://bitbucket.org/pypy/pypy/changeset/94130878552c/

Log:	rename 'fast' mode to 'core-only'

diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -779,7 +779,7 @@
         return ConstInt(result)
 
     def perform_call_maybe(self, jitcode, argboxes):
-        core_only_mode = (self.metainterp.jitdriver_sd.warmstate.jitmode == 'fast')
+        core_only_mode = (self.metainterp.jitdriver_sd.warmstate.jitmode == 'core-only')
         # in core_only_mode, don't inline calls from core to non-core graphs
         if core_only_mode and self.jitcode.is_core and not jitcode.is_core:
             funcbox = ConstInt(jitcode.get_fnaddr_as_int())
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -2962,7 +2962,7 @@
                 res = a(x) + b(x, res) + c(x, -x, -x) # at the end, it's like doing x+res :-)
                 y -= 1
             return res
-        res = self.meta_interp(f, [6, 7], policy=MyPolicy(), jitmode='fast') # fast == trace only core graphs
+        res = self.meta_interp(f, [6, 7], policy=MyPolicy(), jitmode='core-only')
         assert res == 42
         self.check_trace_count(1)
         # this is suboptimal because we get a call_may_force instead of a
@@ -2996,7 +2996,7 @@
                 y -= 1
                 res += f(x)
             return res
-        res = self.meta_interp(f, [5, 7], policy=MyPolicy(), jitmode='fast') # fast == trace only core graphs
+        res = self.meta_interp(f, [5, 7], policy=MyPolicy(), jitmode='core-only')
         assert res == 43
         self.check_trace_count(1)
         self.check_resops({'jump': 1, 'int_gt': 2, 'guard_true': 2, 'int_sub': 2,
@@ -3021,7 +3021,7 @@
                 res = a(res, x)
                 y -= 1
             return res
-        res = self.meta_interp(f, [6, 7], policy=MyPolicy(), jitmode='fast') # fast == trace only core graphs
+        res = self.meta_interp(f, [6, 7], policy=MyPolicy(), jitmode='core-only')
         assert res == 42
         self.check_trace_count(1)
         self.check_resops({'jump': 1, 'int_gt': 2, 'guard_true': 2, 'int_sub': 2,
diff --git a/pypy/rlib/jit.py b/pypy/rlib/jit.py
--- a/pypy/rlib/jit.py
+++ b/pypy/rlib/jit.py
@@ -402,7 +402,7 @@
     'retrace_limit': 'how many times we can try retracing before giving up',
     'max_retrace_guards': 'number of extra guards a retrace can cause',
     'max_unroll_loops': 'number of extra unrollings a loop can cause',
-    'jitmode': '"full" (default) or "fast"',
+    'jitmode': '"full" (default) or "core-only"',
     'enable_opts': 'optimizations to enable or all, INTERNAL USE ONLY'
     }
 

From noreply at buildbot.pypy.org  Fri Jan 20 15:35:20 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Fri, 20 Jan 2012 15:35:20 +0100 (CET)
Subject: [pypy-commit] pypy core-only-tracing: always disable inlining when
 setting the jitmode to core-only
Message-ID: <20120120143520.7EB4B82CF8@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni 
Branch: core-only-tracing
Changeset: r51534:730dac0eff41
Date: 2012-01-20 15:08 +0100
http://bitbucket.org/pypy/pypy/changeset/730dac0eff41/

Log:	always disable inlining when setting the jitmode to core-only

diff --git a/pypy/jit/metainterp/test/test_warmstate.py b/pypy/jit/metainterp/test/test_warmstate.py
--- a/pypy/jit/metainterp/test/test_warmstate.py
+++ b/pypy/jit/metainterp/test/test_warmstate.py
@@ -324,3 +324,12 @@
         cell = get_jitcell(True, i)
         cell.counter = -2
         assert len(warmstate._jitcell_dict) == i + 1
+
+def test_set_params():
+    warmstate = WarmEnterState(None, None)
+    assert warmstate.inlining == 1
+    assert warmstate.jitmode == 'full'
+    warmstate.set_param_jitmode('core-only')
+    assert warmstate.inlining == 0
+    assert warmstate.jitmode == 'core-only'
+    
diff --git a/pypy/jit/metainterp/warmstate.py b/pypy/jit/metainterp/warmstate.py
--- a/pypy/jit/metainterp/warmstate.py
+++ b/pypy/jit/metainterp/warmstate.py
@@ -215,6 +215,8 @@
 
     def set_param_jitmode(self, value):
         self.jitmode = value
+        if value == 'core-only':
+            self.set_param_inlining(0)
 
     def set_param_enable_opts(self, value):
         from pypy.jit.metainterp.optimizeopt import ALL_OPTS_DICT, ALL_OPTS_NAMES

From noreply at buildbot.pypy.org  Fri Jan 20 15:35:21 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Fri, 20 Jan 2012 15:35:21 +0100 (CET)
Subject: [pypy-commit] pypy core-only-tracing: (antocuni,
 arigo): if we are in core mode,
 we know for sure that the jitcode we are tracing is_core
Message-ID: <20120120143521.B7E2A82CF8@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni 
Branch: core-only-tracing
Changeset: r51535:d82aeadcfb55
Date: 2012-01-20 15:11 +0100
http://bitbucket.org/pypy/pypy/changeset/d82aeadcfb55/

Log:	(antocuni, arigo): if we are in core mode, we know for sure that the
	jitcode we are tracing is_core

diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -781,7 +781,8 @@
     def perform_call_maybe(self, jitcode, argboxes):
         core_only_mode = (self.metainterp.jitdriver_sd.warmstate.jitmode == 'core-only')
         # in core_only_mode, don't inline calls from core to non-core graphs
-        if core_only_mode and self.jitcode.is_core and not jitcode.is_core:
+        if core_only_mode and not jitcode.is_core:
+            assert self.jitcode.is_core
             funcbox = ConstInt(jitcode.get_fnaddr_as_int())
             # jitcode always has a calldescr, but it might not have the
             # correct effectinfo. The result is that we might generate a

From noreply at buildbot.pypy.org  Fri Jan 20 15:35:23 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Fri, 20 Jan 2012 15:35:23 +0100 (CET)
Subject: [pypy-commit] pypy core-only-tracing: (antocuni,
 arigo): use a boolean instead of a string to store the value of
 core-only-mode, and don't disable inlining automatically (for now at least)
Message-ID: <20120120143523.0041A82CF8@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni 
Branch: core-only-tracing
Changeset: r51536:54d6ea8b9249
Date: 2012-01-20 15:16 +0100
http://bitbucket.org/pypy/pypy/changeset/54d6ea8b9249/

Log:	(antocuni, arigo): use a boolean instead of a string to store the
	value of core-only-mode, and don't disable inlining automatically
	(for now at least)

diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -779,7 +779,7 @@
         return ConstInt(result)
 
     def perform_call_maybe(self, jitcode, argboxes):
-        core_only_mode = (self.metainterp.jitdriver_sd.warmstate.jitmode == 'core-only')
+        core_only_mode = (self.metainterp.jitdriver_sd.warmstate.jitmode_core_only)
         # in core_only_mode, don't inline calls from core to non-core graphs
         if core_only_mode and not jitcode.is_core:
             assert self.jitcode.is_core
diff --git a/pypy/jit/metainterp/test/support.py b/pypy/jit/metainterp/test/support.py
--- a/pypy/jit/metainterp/test/support.py
+++ b/pypy/jit/metainterp/test/support.py
@@ -40,7 +40,7 @@
         _cell = FakeJitCell()
 
         trace_limit = sys.maxint
-        jitmode = 'full'
+        jitmode_core_only = False
         enable_opts = ALL_OPTS_DICT
 
     func._jit_unroll_safe_ = True
diff --git a/pypy/jit/metainterp/test/test_warmstate.py b/pypy/jit/metainterp/test/test_warmstate.py
--- a/pypy/jit/metainterp/test/test_warmstate.py
+++ b/pypy/jit/metainterp/test/test_warmstate.py
@@ -324,12 +324,3 @@
         cell = get_jitcell(True, i)
         cell.counter = -2
         assert len(warmstate._jitcell_dict) == i + 1
-
-def test_set_params():
-    warmstate = WarmEnterState(None, None)
-    assert warmstate.inlining == 1
-    assert warmstate.jitmode == 'full'
-    warmstate.set_param_jitmode('core-only')
-    assert warmstate.inlining == 0
-    assert warmstate.jitmode == 'core-only'
-    
diff --git a/pypy/jit/metainterp/warmstate.py b/pypy/jit/metainterp/warmstate.py
--- a/pypy/jit/metainterp/warmstate.py
+++ b/pypy/jit/metainterp/warmstate.py
@@ -214,9 +214,7 @@
         self.inlining = value
 
     def set_param_jitmode(self, value):
-        self.jitmode = value
-        if value == 'core-only':
-            self.set_param_inlining(0)
+        self.jitmode_core_only = (value == 'core-only')
 
     def set_param_enable_opts(self, value):
         from pypy.jit.metainterp.optimizeopt import ALL_OPTS_DICT, ALL_OPTS_NAMES

From noreply at buildbot.pypy.org  Fri Jan 20 15:35:24 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Fri, 20 Jan 2012 15:35:24 +0100 (CET)
Subject: [pypy-commit] pypy core-only-tracing: (antocuni,
	arigo): improve this test
Message-ID: <20120120143524.39E5D82CF8@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni 
Branch: core-only-tracing
Changeset: r51537:4cb941b199d3
Date: 2012-01-20 15:28 +0100
http://bitbucket.org/pypy/pypy/changeset/4cb941b199d3/

Log:	(antocuni, arigo): improve this test

diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -2974,36 +2974,45 @@
                            'guard_no_exception': 6,
                            'guard_not_forced': 6})
 
-    def test_dont_inline_residual_call_from_core_graph(self):
+    def test_dont_inline_indirect_call_from_core_graph_to_non_core_graph(self):
         class MyPolicy(JitPolicy):
             def is_core_graph(self, graph):
-                return graph.name == 'f'
-        myjitdriver = JitDriver(greens = [], reds = ['x', 'y', 'res'])
+                return graph.name in ('f', 'a')
+        myjitdriver = JitDriver(greens = [], reds = ['x', 'y', 'z', 'res'])
 
         def a(x):
             return x+2
         def b(x):
             return x+1
-        def f(x, y):
+        def f(x, y, z):
             res = 0
             while y > 0:
-                myjitdriver.can_enter_jit(x=x, y=y, res=res)
-                myjitdriver.jit_merge_point(x=x, y=y, res=res)
-                if y == 5:
+                myjitdriver.can_enter_jit(x=x, y=y, z=z, res=res)
+                myjitdriver.jit_merge_point(x=x, y=y, z=z, res=res)
+                if z:
                     f = a
                 else:
                     f = b
                 y -= 1
                 res += f(x)
             return res
-        res = self.meta_interp(f, [5, 7], policy=MyPolicy(), jitmode='core-only')
-        assert res == 43
+        # indirect call to b (non-core)
+        res = self.meta_interp(f, [5, 7, 0], policy=MyPolicy(), jitmode='core-only')
+        assert res == 42
         self.check_trace_count(1)
         self.check_resops({'jump': 1, 'int_gt': 2, 'guard_true': 2, 'int_sub': 2,
-                           'int_eq': 2, 'int_add': 2, 'guard_false': 2,
+                           'int_is_true': 1, 'int_add': 2, 'guard_false': 1,
                            'call_may_force': 2,
                            'guard_no_exception': 2,
                            'guard_not_forced': 2})
+        #
+        # indirect call to a (core)
+        res = self.meta_interp(f, [5, 7, 1], policy=MyPolicy(), jitmode='core-only')
+        assert res == 49
+        self.check_trace_count(1)
+        self.check_resops({'jump': 1, 'int_gt': 2, 'guard_true': 3, 'int_sub': 2,
+                           'int_is_true': 1, 'int_add': 3})
+
 
     def test_inline_core_to_core_calls(self):
         class MyPolicy(JitPolicy):

From noreply at buildbot.pypy.org  Fri Jan 20 15:35:25 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Fri, 20 Jan 2012 15:35:25 +0100 (CET)
Subject: [pypy-commit] pypy core-only-tracing: (antocuni, arigo,
 romain around): try to implement a reasonable is_core_function
 policy for the pypy interpreter
Message-ID: <20120120143525.6D95E82CF8@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni 
Branch: core-only-tracing
Changeset: r51538:c19f30c468d8
Date: 2012-01-20 15:34 +0100
http://bitbucket.org/pypy/pypy/changeset/c19f30c468d8/

Log:	(antocuni, arigo, romain around): try to implement a reasonable
	is_core_function policy for the pypy interpreter

diff --git a/pypy/jit/codewriter/policy.py b/pypy/jit/codewriter/policy.py
--- a/pypy/jit/codewriter/policy.py
+++ b/pypy/jit/codewriter/policy.py
@@ -39,6 +39,14 @@
         return True # look into everything by default
 
     def is_core_graph(self, graph):
+        try:
+            func = graph.func
+        except AttributeError:
+            return True
+        else:
+            return self.is_core_function(func)
+
+    def is_core_function(self, func):
         return True
 
     def _reject_function(self, func):
diff --git a/pypy/module/pypyjit/policy.py b/pypy/module/pypyjit/policy.py
--- a/pypy/module/pypyjit/policy.py
+++ b/pypy/module/pypyjit/policy.py
@@ -150,3 +150,8 @@
                 return False
 
         return True
+
+    def is_core_function(self, func):
+        mod = func.__module__ or '?'
+        is_interpreter = mod.startswith('pypy.interpreter.')
+        return is_interpreter or mod.startswith('pypy.module.pypyjit.')

From noreply at buildbot.pypy.org  Fri Jan 20 15:41:51 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 20 Jan 2012 15:41:51 +0100 (CET)
Subject: [pypy-commit] pypy matrixmath-dot: progress
Message-ID: <20120120144151.1A22C82CF8@wyvern.cs.uni-duesseldorf.de>

Author: mattip
Branch: matrixmath-dot
Changeset: r51539:a2e2a35726cc
Date: 2012-01-20 11:12 +0200
http://bitbucket.org/pypy/pypy/changeset/a2e2a35726cc/

Log:	progress

diff --git a/pypy/module/micronumpy/dot.py b/pypy/module/micronumpy/dot.py
--- a/pypy/module/micronumpy/dot.py
+++ b/pypy/module/micronumpy/dot.py
@@ -7,8 +7,8 @@
 
 
 dot_driver = jit.JitDriver(
-    greens=['shapelen', 'left', 'right'],
-    reds=['lefti', 'righti', 'outi', 'result'],
+    greens=['shape_len', 'left'],
+    reds=['lefti', 'righti', 'outi', 'result', 'right'],
     get_printable_location=new_printable_location('dot'),
     name='dot',
 )
@@ -55,7 +55,7 @@
                                          if i != right_critical_dim]
     right_skip = range(len(left.shape) - 1)
     result_skip = [len(result.shape) - 1]
-    shapelen = len(broadcast_shape)
+    shape_len = len(broadcast_shape)
     _r = calculate_dot_strides(result.strides, result.backstrides,
                                   broadcast_shape, result_skip)
     outi = ViewIterator(0, _r[0], _r[1], broadcast_shape)
@@ -78,9 +78,9 @@
                        right.getitem(righti.offset))
         value = add(dtype, v, result.getitem(outi.offset))
         result.setitem(outi.offset, value)
-        outi = outi.next(shapelen)
-        righti = righti.next(shapelen)
-        lefti = lefti.next(shapelen)
+        outi = outi.next(shape_len)
+        righti = righti.next(shape_len)
+        lefti = lefti.next(shape_len)
     assert lefti.done()
     assert righti.done()
     return result
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -368,16 +368,15 @@
                                 'int_ge': 1, 'guard_false': 1, 'jump': 1,
                                 'arraylen_gc': 1})
 
-    def ddefine_dot():
+    def define_dot():
         return """
         a = [[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]]
         b=[[0, 1, 2], [3, 4, 5], [6, 7, 8], [9, 10, 11]]
-        c = a.dot(b)
+        c = dot(a, b)
         c -> 1 -> 2
         """
 
     def test_dot(self):
-        py.test.skip("not yet")
         result = self.run("dot")
         assert result == 184
         self.check_simple_loop({})

From noreply at buildbot.pypy.org  Fri Jan 20 15:41:52 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 20 Jan 2012 15:41:52 +0100 (CET)
Subject: [pypy-commit] pypy matrixmath-dot: test for slice of transposed
	array fails
Message-ID: <20120120144152.4F68E82CF8@wyvern.cs.uni-duesseldorf.de>

Author: mattip
Branch: matrixmath-dot
Changeset: r51540:bb1838ae2306
Date: 2012-01-20 16:40 +0200
http://bitbucket.org/pypy/pypy/changeset/bb1838ae2306/

Log:	test for slice of transposed array fails

diff --git a/pypy/module/micronumpy/dot.py b/pypy/module/micronumpy/dot.py
--- a/pypy/module/micronumpy/dot.py
+++ b/pypy/module/micronumpy/dot.py
@@ -6,10 +6,13 @@
 from pypy.rlib import jit
 
 
+def dot_printable_location(shapelen, sig):
+    return 'numpy dot [%d dims]' % (shapelen)
+
 dot_driver = jit.JitDriver(
     greens=['shape_len', 'left'],
-    reds=['lefti', 'righti', 'outi', 'result', 'right'],
-    get_printable_location=new_printable_location('dot'),
+    reds=['lefti', 'righti', 'outi', 'result', 'right','sig','dtype'],
+    get_printable_location=dot_printable_location,
     name='dot',
 )
 
@@ -41,31 +44,35 @@
     ''' assumes left, right are concrete arrays
     given left.shape == [3, 5, 7],
           right.shape == [2, 7, 4]
+    then
      result.shape == [3, 5, 2, 4]
-    broadcast shape should be [3, 5, 2, 7, 4]
-    result should skip dims 3 which is results.ndims - 1
-    left should skip 2, 4 which is a.ndims-1 + range(right.ndims)
+     broadcast shape should be [3, 5, 2, 7, 4]
+     result should skip dims 3 which is len(result_shape) - 1
+        (note that if right is 1d, result should 
+                  skip len(result_shape))
+     left should skip 2, 4 which is a.ndims-1 + range(right.ndims)
           except where it==(right.ndims-2)
-    right should skip 0, 1
+     right should skip 0, 1
     '''
-    mul = interp_ufuncs.get(space).multiply.func
-    add = interp_ufuncs.get(space).add.func
     broadcast_shape = left.shape[:-1] + right.shape
+    shape_len = len(broadcast_shape)
     left_skip = [len(left.shape) - 1 + i for i in range(len(right.shape))
                                          if i != right_critical_dim]
     right_skip = range(len(left.shape) - 1)
-    result_skip = [len(result.shape) - 1]
-    shape_len = len(broadcast_shape)
+    result_skip = [len(result.shape) - (len(right.shape) > 1)]
     _r = calculate_dot_strides(result.strides, result.backstrides,
                                   broadcast_shape, result_skip)
-    outi = ViewIterator(0, _r[0], _r[1], broadcast_shape)
+    outi = ViewIterator(result.start, _r[0], _r[1], broadcast_shape)
     _r = calculate_dot_strides(left.strides, left.backstrides,
                                   broadcast_shape, left_skip)
-    lefti = ViewIterator(0, _r[0], _r[1], broadcast_shape)
+    lefti = ViewIterator(left.start, _r[0], _r[1], broadcast_shape)
     _r = calculate_dot_strides(right.strides, right.backstrides,
                                   broadcast_shape, right_skip)
-    righti = ViewIterator(0, _r[0], _r[1], broadcast_shape)
+    righti = ViewIterator(right.start, _r[0], _r[1], broadcast_shape)
+    if right.size==4:
+        xxx
     while not outi.done():
+        '''
         dot_driver.jit_merge_point(left=left,
                                    right=right,
                                    shape_len=shape_len,
@@ -73,10 +80,17 @@
                                    righti=righti,
                                    outi=outi,
                                    result=result,
+                                   dtype=dtype,
+                                   sig=None, #For get_printable_location
                                   )
-        v = mul(dtype, left.getitem(lefti.offset),
-                       right.getitem(righti.offset))
-        value = add(dtype, v, result.getitem(outi.offset))
+        '''
+        lval = left.getitem(lefti.offset).convert_to(dtype) 
+        rval = right.getitem(righti.offset).convert_to(dtype) 
+        outval = result.getitem(outi.offset).convert_to(dtype) 
+        v = dtype.itemtype.mul(lval, rval)
+        value = dtype.itemtype.add(v, outval)
+        #Do I need to convert it to result.dtype or does settiem do that?
+        assert outi.offset < result.size
         result.setitem(outi.offset, value)
         outi = outi.next(shape_len)
         righti = righti.next(shape_len)
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -890,7 +890,7 @@
         c = dot(a, b)
         assert (c == [[[14, 38, 62], [38, 126, 214], [62, 214, 366]],
                    [[86, 302, 518], [110, 390, 670], [134, 478, 822]]]).all()
-        c = dot(a, b[:, :, 2])
+        c = dot(a, b[:, 2])
         assert (c == [[38, 126, 214], [302, 390, 478]]).all()
 
     def test_dot_constant(self):

From noreply at buildbot.pypy.org  Fri Jan 20 15:52:34 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Fri, 20 Jan 2012 15:52:34 +0100 (CET)
Subject: [pypy-commit] pypy core-only-tracing: tentative rpython fix
Message-ID: <20120120145234.399C282CF8@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni 
Branch: core-only-tracing
Changeset: r51541:8789a7c425dc
Date: 2012-01-20 15:52 +0100
http://bitbucket.org/pypy/pypy/changeset/8789a7c425dc/

Log:	tentative rpython fix

diff --git a/pypy/rlib/jit.py b/pypy/rlib/jit.py
--- a/pypy/rlib/jit.py
+++ b/pypy/rlib/jit.py
@@ -527,8 +527,8 @@
             raise ValueError
         name = parts[0]
         value = parts[1]
-        if name == 'enable_opts':
-            set_param(driver, 'enable_opts', value)
+        if name == 'enable_opts' or name == 'jitmode':
+            set_param(driver, name, value)
         else:
             for name1, _ in unroll_parameters:
                 if name1 == name and name1 != 'enable_opts':

From pullrequests-noreply at bitbucket.org  Fri Jan 20 17:07:55 2012
From: pullrequests-noreply at bitbucket.org (Bitbucket)
Date: Fri, 20 Jan 2012 16:07:55 -0000
Subject: [pypy-commit] [OPEN] Pull request #22 for pypy/pypy: fix
	documentation bugs
Message-ID: 

A new pull request has been opened by tomo cocoa.

cocoatomo/pypydoc has changes to be pulled into pypy/pypy.

https://bitbucket.org/pypy/pypy/pull-request/22/fix-documentation-bugs

Title: fix documentation bugs

some miscellaneous typo were fixed

Changes to be pulled:



--
This is an issue notification from bitbucket.org.
You are receiving this either because you are the participating
in a pull request, or you are following it.


From noreply at buildbot.pypy.org  Fri Jan 20 17:14:39 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 20 Jan 2012 17:14:39 +0100 (CET)
Subject: [pypy-commit] pypy default: For clarity. It used to work before too
 with a too-short array on 64-bits, 
Message-ID: <20120120161439.924D482CF8@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r51542:437f207e58f0
Date: 2012-01-20 17:13 +0100
http://bitbucket.org/pypy/pypy/changeset/437f207e58f0/

Log:	For clarity. It used to work before too with a too-short array on
	64-bits, but that's only because it would only need to read 'rax' or
	'xmm0', which were both in the array.

diff --git a/pypy/jit/backend/x86/runner.py b/pypy/jit/backend/x86/runner.py
--- a/pypy/jit/backend/x86/runner.py
+++ b/pypy/jit/backend/x86/runner.py
@@ -6,7 +6,7 @@
 from pypy.jit.codewriter import longlong
 from pypy.jit.metainterp import history, compile
 from pypy.jit.backend.x86.assembler import Assembler386
-from pypy.jit.backend.x86.arch import FORCE_INDEX_OFS
+from pypy.jit.backend.x86.arch import FORCE_INDEX_OFS, IS_X86_32
 from pypy.jit.backend.x86.profagent import ProfileAgent
 from pypy.jit.backend.llsupport.llmodel import AbstractLLCPU
 from pypy.jit.backend.x86 import regloc
@@ -142,7 +142,9 @@
     cast_ptr_to_int._annspecialcase_ = 'specialize:arglltype(0)'
     cast_ptr_to_int = staticmethod(cast_ptr_to_int)
 
-    all_null_registers = lltype.malloc(rffi.LONGP.TO, 24,
+    all_null_registers = lltype.malloc(rffi.LONGP.TO,
+                                       IS_X86_32 and (16+8)  # 16 + 8 regs
+                                                 or (16+16), # 16 + 16 regs
                                        flavor='raw', zero=True,
                                        immortal=True)
 

From noreply at buildbot.pypy.org  Fri Jan 20 17:14:40 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 20 Jan 2012 17:14:40 +0100 (CET)
Subject: [pypy-commit] pypy default: I think that this "if" doesn't make
 sense. Kill tentatively.
Message-ID: <20120120161440.C8FC582CF8@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r51543:a80226d0b3cd
Date: 2012-01-20 17:14 +0100
http://bitbucket.org/pypy/pypy/changeset/a80226d0b3cd/

Log:	I think that this "if" doesn't make sense. Kill tentatively.

diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -2042,10 +2042,7 @@
         size = sizeloc.value
         signloc = arglocs[1]
 
-        if isinstance(op.getarg(0), Const):
-            x = imm(op.getarg(0).getint())
-        else:
-            x = arglocs[2]
+        x = arglocs[2]     # the function address
         if x is eax:
             tmp = ecx
         else:

From noreply at buildbot.pypy.org  Fri Jan 20 17:17:16 2012
From: noreply at buildbot.pypy.org (bivab)
Date: Fri, 20 Jan 2012 17:17:16 +0100 (CET)
Subject: [pypy-commit] pypy arm-backend-2: (arigo,
 bivab) make sure to pass a zeroed pice of memory to
 failure_recovery_func when forcing for the area where registers whould be
 stored.
Message-ID: <20120120161716.EF1F882CF8@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider 
Branch: arm-backend-2
Changeset: r51544:58888d2298ed
Date: 2012-01-20 13:21 +0100
http://bitbucket.org/pypy/pypy/changeset/58888d2298ed/

Log:	(arigo, bivab) make sure to pass a zeroed pice of memory to
	failure_recovery_func when forcing for the area where registers
	whould be stored.

diff --git a/pypy/jit/backend/arm/runner.py b/pypy/jit/backend/arm/runner.py
--- a/pypy/jit/backend/arm/runner.py
+++ b/pypy/jit/backend/arm/runner.py
@@ -1,5 +1,4 @@
 from pypy.jit.backend.arm.assembler import AssemblerARM
-from pypy.jit.backend.arm.arch import WORD, DOUBLE_WORD
 from pypy.jit.backend.arm.registers import all_regs, all_vfp_regs
 from pypy.jit.backend.llsupport.llmodel import AbstractLLCPU
 from pypy.rpython.llinterp import LLInterpreter
@@ -100,6 +99,10 @@
     cast_ptr_to_int._annspecialcase_ = 'specialize:arglltype(0)'
     cast_ptr_to_int = staticmethod(cast_ptr_to_int)
 
+    all_null_registers = lltype.malloc(rffi.LONGP.TO,
+                        len(all_vfp_regs) * 2 + len(all_regs),
+                        flavor='raw', zero=True, immortal=True)
+
     def force(self, addr_of_force_index):
         TP = rffi.CArrayPtr(lltype.Signed)
         fail_index = rffi.cast(TP, addr_of_force_index)[0]
@@ -107,16 +110,12 @@
         faildescr = self.get_fail_descr_from_number(fail_index)
         rffi.cast(TP, addr_of_force_index)[0] = ~fail_index
         bytecode = self.assembler._find_failure_recovery_bytecode(faildescr)
+        addr_all_null_regsiters = rffi.cast(rffi.LONG, self.all_null_registers)
         # start of "no gc operation!" block
-        frame_depth = faildescr._arm_current_frame_depth * WORD
-        addr_end_of_frame = (addr_of_force_index -
-                            (frame_depth +
-                            len(all_regs) * WORD +
-                            len(all_vfp_regs) * DOUBLE_WORD))
         fail_index_2 = self.assembler.failure_recovery_func(
             bytecode,
             addr_of_force_index,
-            addr_end_of_frame)
+            addr_all_null_regsiters)
         self.assembler.leave_jitted_hook()
         # end of "no gc operation!" block
         assert fail_index == fail_index_2

From noreply at buildbot.pypy.org  Fri Jan 20 17:17:18 2012
From: noreply at buildbot.pypy.org (bivab)
Date: Fri, 20 Jan 2012 17:17:18 +0100 (CET)
Subject: [pypy-commit] pypy arm-backend-2: Add a test to make sure that
 cpu.force uses a zeroed piece of memory as the area where the registers
 should be read when restoring the values to the failboxes
Message-ID: <20120120161718.354A382CF8@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider 
Branch: arm-backend-2
Changeset: r51545:f523474c7b3e
Date: 2012-01-20 13:54 +0100
http://bitbucket.org/pypy/pypy/changeset/f523474c7b3e/

Log:	Add a test to make sure that cpu.force uses a zeroed piece of memory
	as the area where the registers should be read when restoring the
	values to the failboxes

diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py
--- a/pypy/jit/backend/test/runner_test.py
+++ b/pypy/jit/backend/test/runner_test.py
@@ -3298,6 +3298,43 @@
         fail = self.cpu.execute_token(looptoken, null_box.getref_base())
         assert fail.identifier == 99
 
+    def test_forcing_op_with_fail_arg_in_reg(self):
+        values = []
+        def maybe_force(token, flag):
+            self.cpu.force(token)
+            values.append(self.cpu.get_latest_value_int(0))
+            values.append(token)
+            return 42
+
+        FUNC = self.FuncType([lltype.Signed, lltype.Signed], lltype.Signed)
+        func_ptr = llhelper(lltype.Ptr(FUNC), maybe_force)
+        funcbox = self.get_funcbox(self.cpu, func_ptr).constbox()
+        calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT,
+                                         EffectInfo.MOST_GENERAL)
+        i0 = BoxInt()
+        i1 = BoxInt()
+        i2 = BoxInt()
+        tok = BoxInt()
+        faildescr = BasicFailDescr(23)
+        ops = [
+        ResOperation(rop.FORCE_TOKEN, [], tok),
+        ResOperation(rop.CALL_MAY_FORCE, [funcbox, tok, i1], i2,
+                     descr=calldescr),
+        ResOperation(rop.GUARD_NOT_FORCED, [], None, descr=faildescr),
+        ResOperation(rop.FINISH, [i2], None, descr=BasicFailDescr(0))
+        ]
+        ops[2].setfailargs([i2])
+        looptoken = JitCellToken()
+        self.cpu.compile_loop([i0, i1], ops, looptoken)
+        fail = self.cpu.execute_token(looptoken, 20, 0)
+        assert fail.identifier == 23
+        assert self.cpu.get_latest_value_int(0) == 42
+        # make sure that force reads the registers from a zeroed piece of
+        # memory
+        assert values[0] == 0
+        token = self.cpu.get_latest_force_token()
+        assert values[1] == token
+
 class OOtypeBackendTest(BaseBackendTest):
 
     type_system = 'ootype'

From noreply at buildbot.pypy.org  Fri Jan 20 17:17:19 2012
From: noreply at buildbot.pypy.org (bivab)
Date: Fri, 20 Jan 2012 17:17:19 +0100 (CET)
Subject: [pypy-commit] pypy arm-backend-2: merge heads
Message-ID: <20120120161719.6917682CF8@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider 
Branch: arm-backend-2
Changeset: r51546:b5daa363d5b9
Date: 2012-01-20 13:56 +0100
http://bitbucket.org/pypy/pypy/changeset/b5daa363d5b9/

Log:	merge heads

diff --git a/pypy/jit/backend/arm/runner.py b/pypy/jit/backend/arm/runner.py
--- a/pypy/jit/backend/arm/runner.py
+++ b/pypy/jit/backend/arm/runner.py
@@ -1,5 +1,4 @@
 from pypy.jit.backend.arm.assembler import AssemblerARM
-from pypy.jit.backend.arm.arch import WORD, DOUBLE_WORD
 from pypy.jit.backend.arm.registers import all_regs, all_vfp_regs
 from pypy.jit.backend.llsupport.llmodel import AbstractLLCPU
 from pypy.rpython.llinterp import LLInterpreter
@@ -100,6 +99,10 @@
     cast_ptr_to_int._annspecialcase_ = 'specialize:arglltype(0)'
     cast_ptr_to_int = staticmethod(cast_ptr_to_int)
 
+    all_null_registers = lltype.malloc(rffi.LONGP.TO,
+                        len(all_vfp_regs) * 2 + len(all_regs),
+                        flavor='raw', zero=True, immortal=True)
+
     def force(self, addr_of_force_index):
         TP = rffi.CArrayPtr(lltype.Signed)
         fail_index = rffi.cast(TP, addr_of_force_index)[0]
@@ -107,16 +110,12 @@
         faildescr = self.get_fail_descr_from_number(fail_index)
         rffi.cast(TP, addr_of_force_index)[0] = ~fail_index
         bytecode = self.assembler._find_failure_recovery_bytecode(faildescr)
+        addr_all_null_regsiters = rffi.cast(rffi.LONG, self.all_null_registers)
         # start of "no gc operation!" block
-        frame_depth = faildescr._arm_current_frame_depth * WORD
-        addr_end_of_frame = (addr_of_force_index -
-                            (frame_depth +
-                            len(all_regs) * WORD +
-                            len(all_vfp_regs) * DOUBLE_WORD))
         fail_index_2 = self.assembler.failure_recovery_func(
             bytecode,
             addr_of_force_index,
-            addr_end_of_frame)
+            addr_all_null_regsiters)
         self.assembler.leave_jitted_hook()
         # end of "no gc operation!" block
         assert fail_index == fail_index_2

From noreply at buildbot.pypy.org  Fri Jan 20 17:17:20 2012
From: noreply at buildbot.pypy.org (bivab)
Date: Fri, 20 Jan 2012 17:17:20 +0100 (CET)
Subject: [pypy-commit] pypy arm-backend-2: (arigo,
 bivab) implement the BLX ARM instruction to branch to an address
 stored in a register
Message-ID: <20120120161720.9D23382CF8@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider 
Branch: arm-backend-2
Changeset: r51547:95557903da83
Date: 2012-01-20 17:11 +0100
http://bitbucket.org/pypy/pypy/changeset/95557903da83/

Log:	(arigo, bivab) implement the BLX ARM instruction to branch to an
	address stored in a register

diff --git a/pypy/jit/backend/arm/codebuilder.py b/pypy/jit/backend/arm/codebuilder.py
--- a/pypy/jit/backend/arm/codebuilder.py
+++ b/pypy/jit/backend/arm/codebuilder.py
@@ -178,14 +178,11 @@
 
     def BL(self, addr, c=cond.AL):
         target = rffi.cast(rffi.INT, addr)
-        if c == cond.AL:
-            self.ADD_ri(reg.lr.value, reg.pc.value, arch.PC_OFFSET / 2)
-            self.LDR_ri(reg.pc.value, reg.pc.value, imm=-arch.PC_OFFSET / 2)
-            self.write32(target)
-        else:
-            self.gen_load_int(reg.ip.value, target, cond=c)
-            self.MOV_rr(reg.lr.value, reg.pc.value, cond=c)
-            self.MOV_rr(reg.pc.value, reg.ip.value, cond=c)
+        self.gen_load_int(reg.ip.value, target, cond=c)
+        self.BLX(reg.ip.value)
+
+    def BLX(self, reg, c=cond.AL):
+        self.write32(c << 28 | 0x12FFF3 << 4 | (reg & 0xF))
 
     def MOVT_ri(self, rd, imm16, c=cond.AL):
         """Move Top writes an immediate value to the top halfword of the

From noreply at buildbot.pypy.org  Fri Jan 20 17:17:21 2012
From: noreply at buildbot.pypy.org (bivab)
Date: Fri, 20 Jan 2012 17:17:21 +0100 (CET)
Subject: [pypy-commit] pypy arm-backend-2: (arigo,
 bivab) refactor the code used to make calls to handle more work in
 the register allocator and perform calls only using locations.
Message-ID: <20120120161721.D843A82CF8@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider 
Branch: arm-backend-2
Changeset: r51548:d06bbcb1c9fb
Date: 2012-01-20 17:13 +0100
http://bitbucket.org/pypy/pypy/changeset/d06bbcb1c9fb/

Log:	(arigo, bivab) refactor the code used to make calls to handle more
	work in the register allocator and perform calls only using
	locations.

diff --git a/pypy/jit/backend/arm/opassembler.py b/pypy/jit/backend/arm/opassembler.py
--- a/pypy/jit/backend/arm/opassembler.py
+++ b/pypy/jit/backend/arm/opassembler.py
@@ -361,14 +361,13 @@
         self.gen_func_epilog()
         return fcond
 
-    def emit_op_call(self, op, args, regalloc, fcond,
-                                force_index=NO_FORCE_INDEX):
-        adr = args[0].value
-        arglist = op.getarglist()[1:]
+    def emit_op_call(self, op, arglocs, regalloc, fcond, force_index=NO_FORCE_INDEX):
         if force_index == NO_FORCE_INDEX:
             force_index = self.write_new_force_index()
-        cond = self._emit_call(force_index, adr, arglist,
-                                    regalloc, fcond, op.result)
+        resloc = arglocs[0]
+        adr = arglocs[1]
+        arglist = arglocs[2:]
+        cond = self._emit_call(force_index, adr, arglist, fcond, resloc)
         descr = op.getdescr()
         #XXX Hack, Hack, Hack
         if (op.result and not we_are_translated()):
@@ -379,15 +378,10 @@
             self._ensure_result_bit_extension(loc, size, signed)
         return cond
 
-    # XXX improve this interface
-    # emit_op_call_may_force
-    # XXX improve freeing of stuff here
-    # XXX add an interface that takes locations instead of boxes
-    def _emit_call(self, force_index, adr, args, regalloc, fcond=c.AL,
-                                                            result=None):
-        n_args = len(args)
-        reg_args = count_reg_args(args)
-
+    def _emit_call(self, force_index, adr, arglocs, fcond=c.AL, resloc=None):
+        assert self._regalloc.before_call_called
+        n_args = len(arglocs)
+        reg_args = count_reg_args(arglocs)
         # all arguments past the 4th go on the stack
         n = 0   # used to count the number of words pushed on the stack, so we
                 #can later modify the SP back to its original value
@@ -396,7 +390,7 @@
             stack_args = []
             count = 0
             for i in range(reg_args, n_args):
-                arg = args[i]
+                arg = arglocs[i]
                 if arg.type != FLOAT:
                     count += 1
                     n += WORD
@@ -417,8 +411,7 @@
                 if arg is None:
                     self.mc.PUSH([r.ip.value])
                 else:
-                    self.regalloc_push(regalloc.loc(arg))
-
+                    self.regalloc_push(arg)
         # collect variables that need to go in registers and the registers they
         # will be stored in
         num = 0
@@ -427,16 +420,16 @@
         non_float_regs = []
         float_locs = []
         for i in range(reg_args):
-            arg = args[i]
+            arg = arglocs[i]
             if arg.type == FLOAT and count % 2 != 0:
                     num += 1
                     count = 0
             reg = r.caller_resp[num]
 
             if arg.type == FLOAT:
-                float_locs.append((regalloc.loc(arg), reg))
+                float_locs.append((arg, reg))
             else:
-                non_float_locs.append(regalloc.loc(arg))
+                non_float_locs.append(arg)
                 non_float_regs.append(reg)
 
             if arg.type == FLOAT:
@@ -457,14 +450,12 @@
         #the actual call
         self.mc.BL(adr)
         self.mark_gc_roots(force_index)
-        regalloc.possibly_free_vars(args)
         # readjust the sp in case we passed some args on the stack
         if n > 0:
             self._adjust_sp(-n, fcond=fcond)
 
         # restore the argumets stored on the stack
-        if result is not None:
-            resloc = regalloc.after_call(result)
+        if resloc is not None:
             if resloc.is_vfp_reg():
                 # move result to the allocated register
                 self.mov_to_vfp_loc(r.r0, r.r1, resloc)
@@ -889,8 +880,8 @@
             length_box = TempInt()
             length_loc = regalloc.force_allocate_reg(length_box,
                                         forbidden_vars, selected_reg=r.r2)
-            imm = regalloc.convert_to_imm(args[4])
-            self.load(length_loc, imm)
+            immloc = regalloc.convert_to_imm(args[4])
+            self.load(length_loc, immloc)
         if is_unicode:
             bytes_box = TempPtr()
             bytes_loc = regalloc.force_allocate_reg(bytes_box,
@@ -902,8 +893,9 @@
             length_box = bytes_box
             length_loc = bytes_loc
         # call memcpy()
-        self._emit_call(NO_FORCE_INDEX, self.memcpy_addr,
-                            [dstaddr_box, srcaddr_box, length_box], regalloc)
+        regalloc.before_call()
+        self._emit_call(NO_FORCE_INDEX, imm(self.memcpy_addr),
+                            [dstaddr_loc, srcaddr_loc, length_loc])
 
         regalloc.possibly_free_var(length_box)
         regalloc.possibly_free_var(dstaddr_box)
@@ -993,17 +985,19 @@
     # XXX Split into some helper methods
     def emit_guard_call_assembler(self, op, guard_op, arglocs, regalloc,
                                                                     fcond):
+        tmploc = arglocs[1]
+        resloc = arglocs[2]
+        callargs = arglocs[3:]
+
         faildescr = guard_op.getdescr()
         fail_index = self.cpu.get_fail_descr_number(faildescr)
         self._write_fail_index(fail_index)
-
         descr = op.getdescr()
         assert isinstance(descr, JitCellToken)
-        # XXX check this
-        # assert len(arglocs) - 2 == descr.compiled_loop_token._debug_nbargs
-        resbox = TempInt()
-        self._emit_call(fail_index, descr._arm_func_addr,
-                        op.getarglist(), regalloc, fcond, result=resbox)
+        # check value
+        assert tmploc is r.r0
+        self._emit_call(fail_index, imm(descr._arm_func_addr),
+                                callargs, fcond, resloc=tmploc)
         if op.result is None:
             value = self.cpu.done_with_this_frame_void_v
         else:
@@ -1016,12 +1010,8 @@
                 value = self.cpu.done_with_this_frame_float_v
             else:
                 raise AssertionError(kind)
-        # check value
-        resloc = regalloc.try_allocate_reg(resbox)
-        assert resloc is r.r0
         self.mc.gen_load_int(r.ip.value, value)
-        self.mc.CMP_rr(resloc.value, r.ip.value)
-        regalloc.possibly_free_var(resbox)
+        self.mc.CMP_rr(tmploc.value, r.ip.value)
 
         fast_jmp_pos = self.mc.currpos()
         self.mc.BKPT()
@@ -1035,14 +1025,12 @@
         asm_helper_adr = self.cpu.cast_adr_to_int(jd.assembler_helper_adr)
         with saved_registers(self.mc, r.caller_resp[1:] + [r.ip],
                                     r.caller_vfp_resp):
-            # resbox is allready in r0
-            self.mov_loc_loc(arglocs[1], r.r1)
+            # result of previous call is in r0
+            self.mov_loc_loc(arglocs[0], r.r1)
             self.mc.BL(asm_helper_adr)
-            if op.result:
-                resloc = regalloc.after_call(op.result)
-                if resloc.is_vfp_reg():
-                    # move result to the allocated register
-                    self.mov_to_vfp_loc(r.r0, r.r1, resloc)
+            if op.result and resloc.is_vfp_reg():
+                # move result to the allocated register
+                self.mov_to_vfp_loc(r.r0, r.r1, resloc)
 
         # jump to merge point
         jmp_pos = self.mc.currpos()
@@ -1063,11 +1051,10 @@
             fielddescr = jd.vable_token_descr
             assert isinstance(fielddescr, FieldDescr)
             ofs = fielddescr.offset
-            resloc = regalloc.force_allocate_reg(resbox)
-            self.mov_loc_loc(arglocs[1], r.ip)
-            self.mc.MOV_ri(resloc.value, 0)
-            self.mc.STR_ri(resloc.value, r.ip.value, ofs)
-            regalloc.possibly_free_var(resbox)
+            tmploc = regalloc.get_scratch_reg(INT)
+            self.mov_loc_loc(arglocs[0], r.ip)
+            self.mc.MOV_ri(tmploc.value, 0)
+            self.mc.STR_ri(tmploc.value, r.ip.value, ofs)
 
         if op.result is not None:
             # load the return value from fail_boxes_xxx[0]
@@ -1080,8 +1067,6 @@
                 adr = self.fail_boxes_float.get_addr_for_num(0)
             else:
                 raise AssertionError(kind)
-            resloc = regalloc.force_allocate_reg(op.result)
-            regalloc.possibly_free_var(resbox)
             self.mc.gen_load_int(r.ip.value, adr)
             if op.result.type == FLOAT:
                 self.mc.VLDR(resloc.value, r.ip.value)
@@ -1118,14 +1103,48 @@
 
     def emit_guard_call_may_force(self, op, guard_op, arglocs, regalloc,
                                                                     fcond):
+        faildescr = guard_op.getdescr()
+        fail_index = self.cpu.get_fail_descr_number(faildescr)
+        self._write_fail_index(fail_index)
+        numargs = op.numargs()
+        callargs = arglocs[2:numargs]
+        adr = arglocs[1]
+        resloc = arglocs[0]
+        self._emit_call(fail_index, adr, callargs, fcond, resloc)
+
+        self.mc.LDR_ri(r.ip.value, r.fp.value)
+        self.mc.CMP_ri(r.ip.value, 0)
+        self._emit_guard(guard_op, arglocs[1 + numargs:], c.GE, save_exc=True)
+        return fcond
+
+    def emit_guard_call_release_gil(self, op, guard_op, arglocs, regalloc,
+                                                                    fcond):
+
+        # first, close the stack in the sense of the asmgcc GC root tracker
+        gcrootmap = self.cpu.gc_ll_descr.gcrootmap
+        numargs = op.numargs()
+        resloc = arglocs[0]
+        adr = arglocs[1]
+        callargs = arglocs[2:numargs]
+
+        if gcrootmap:
+            self.call_release_gil(gcrootmap, arglocs, fcond)
+        # do the call
+        faildescr = guard_op.getdescr()
+        fail_index = self.cpu.get_fail_descr_number(faildescr)
+        self._write_fail_index(fail_index)
+
+        self._emit_call(fail_index, adr, callargs, fcond, resloc)
+        # then reopen the stack
+        if gcrootmap:
+            self.call_reacquire_gil(gcrootmap, resloc, fcond)
+
         self.mc.LDR_ri(r.ip.value, r.fp.value)
         self.mc.CMP_ri(r.ip.value, 0)
 
-        self._emit_guard(guard_op, arglocs, c.GE, save_exc=True)
+        self._emit_guard(guard_op, arglocs[1 + numargs:], c.GE, save_exc=True)
         return fcond
 
-    emit_guard_call_release_gil = emit_guard_call_may_force
-
     def call_release_gil(self, gcrootmap, save_registers, fcond):
         # First, we need to save away the registers listed in
         # 'save_registers' that are not callee-save.  XXX We assume that
@@ -1136,8 +1155,7 @@
                 regs_to_save.append(reg)
         assert gcrootmap.is_shadow_stack
         with saved_registers(self.mc, regs_to_save):
-            self._emit_call(NO_FORCE_INDEX, self.releasegil_addr, [],
-                                                    self._regalloc, fcond)
+            self._emit_call(NO_FORCE_INDEX, imm(self.releasegil_addr), [], fcond)
 
     def call_reacquire_gil(self, gcrootmap, save_loc, fcond):
         # save the previous result into the stack temporarily.
@@ -1154,8 +1172,7 @@
             regs_to_save.append(r.ip)  # for alingment
         assert gcrootmap.is_shadow_stack
         with saved_registers(self.mc, regs_to_save, vfp_regs_to_save):
-            self._emit_call(NO_FORCE_INDEX, self.reacqgil_addr, [],
-                                                    self._regalloc, fcond)
+            self._emit_call(NO_FORCE_INDEX, imm(self.reacqgil_addr), [], fcond)
 
     def write_new_force_index(self):
         # for shadowstack only: get a new, unused force_index number and
diff --git a/pypy/jit/backend/arm/regalloc.py b/pypy/jit/backend/arm/regalloc.py
--- a/pypy/jit/backend/arm/regalloc.py
+++ b/pypy/jit/backend/arm/regalloc.py
@@ -553,12 +553,28 @@
                 args = self.prepare_op_math_sqrt(op, fcond)
                 self.assembler.emit_op_math_sqrt(op, args, self, fcond)
                 return
-        args = [imm(rffi.cast(lltype.Signed, op.getarg(0).getint()))]
+        return self._prepare_call(op)
+
+    def _prepare_call(self, op, force_store=[], save_all_regs=False):
+        args = []
+        args.append(None)
+        for i in range(op.numargs()):
+            args.append(self.loc(op.getarg(i)))
+        # spill variables that need to be saved around calls
+        self.vfprm.before_call(save_all_regs=save_all_regs)
+        if not save_all_regs:
+            gcrootmap = self.assembler.cpu.gc_ll_descr.gcrootmap
+            if gcrootmap and gcrootmap.is_shadow_stack:
+                save_all_regs = 2
+        self.rm.before_call(save_all_regs=save_all_regs)
+        if op.result:
+            resloc = self.after_call(op.result)
+            args[0] = resloc
+        self.before_call_called = True
         return args
 
     def prepare_op_call_malloc_gc(self, op, fcond):
-        args = [imm(rffi.cast(lltype.Signed, op.getarg(0).getint()))]
-        return args
+        return self._prepare_call(op)
 
     def _prepare_guard(self, op, args=None):
         if args is None:
@@ -1033,58 +1049,25 @@
             self._compute_hint_frame_locations_from_descr(descr)
 
     def prepare_guard_call_may_force(self, op, guard_op, fcond):
-        faildescr = guard_op.getdescr()
-        fail_index = self.cpu.get_fail_descr_number(faildescr)
-        self.assembler._write_fail_index(fail_index)
-        args = [imm(rffi.cast(lltype.Signed, op.getarg(0).getint()))]
-        for v in guard_op.getfailargs():
-            if v in self.rm.reg_bindings or v in self.vfprm.reg_bindings:
-                self.force_spill_var(v)
-        self.assembler.emit_op_call(op, args, self, fcond, fail_index)
-        locs = self._prepare_guard(guard_op)
-        self.possibly_free_vars(guard_op.getfailargs())
-        return locs
-
-    def prepare_guard_call_release_gil(self, op, guard_op, fcond):
-        # first, close the stack in the sense of the asmgcc GC root tracker
-        gcrootmap = self.cpu.gc_ll_descr.gcrootmap
-        if gcrootmap:
-            arglocs = []
-            args = op.getarglist()
-            for i in range(op.numargs()):
-                loc = self._ensure_value_is_boxed(op.getarg(i), args)
-                arglocs.append(loc)
-            self.assembler.call_release_gil(gcrootmap, arglocs, fcond)
-        # do the call
-        faildescr = guard_op.getdescr()
-        fail_index = self.cpu.get_fail_descr_number(faildescr)
-        self.assembler._write_fail_index(fail_index)
-        args = [imm(rffi.cast(lltype.Signed, op.getarg(0).getint()))]
-        self.assembler.emit_op_call(op, args, self, fcond, fail_index)
-        # then reopen the stack
-        if gcrootmap:
-            if op.result:
-                result_loc = self.call_result_location(op.result)
-            else:
-                result_loc = None
-            self.assembler.call_reacquire_gil(gcrootmap, result_loc, fcond)
-        locs = self._prepare_guard(guard_op)
-        return locs
+        args = self._prepare_call(op, save_all_regs=True)
+        return self._prepare_guard(guard_op, args)
+    prepare_guard_call_release_gil = prepare_guard_call_may_force
 
     def prepare_guard_call_assembler(self, op, guard_op, fcond):
         descr = op.getdescr()
         assert isinstance(descr, JitCellToken)
         jd = descr.outermost_jitdriver_sd
         assert jd is not None
-        size = jd.portal_calldescr.get_result_size()
         vable_index = jd.index_of_virtualizable
         if vable_index >= 0:
             self._sync_var(op.getarg(vable_index))
             vable = self.frame_manager.loc(op.getarg(vable_index))
         else:
             vable = imm(0)
+        # make sure the call result location is free
+        tmploc = self.get_scratch_reg(INT, selected_reg=r.r0)
         self.possibly_free_vars(guard_op.getfailargs())
-        return [imm(size), vable]
+        return [vable, tmploc] + self._prepare_call(op, save_all_regs=True)
 
     def _prepare_args_for_new_op(self, new_args):
         gc_ll_descr = self.cpu.gc_ll_descr

From noreply at buildbot.pypy.org  Fri Jan 20 17:17:23 2012
From: noreply at buildbot.pypy.org (bivab)
Date: Fri, 20 Jan 2012 17:17:23 +0100 (CET)
Subject: [pypy-commit] pypy arm-backend-2: (arigo,
 bivab) Add a test that checks the behaviour of calling functions
 stored in boxes
Message-ID: <20120120161723.1BD0D82CF8@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider 
Branch: arm-backend-2
Changeset: r51549:9bd04064df9c
Date: 2012-01-20 17:14 +0100
http://bitbucket.org/pypy/pypy/changeset/9bd04064df9c/

Log:	(arigo, bivab) Add a test that checks the behaviour of calling
	functions stored in boxes

diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py
--- a/pypy/jit/backend/test/runner_test.py
+++ b/pypy/jit/backend/test/runner_test.py
@@ -573,6 +573,28 @@
         res = self.execute_operation(rop.CALL, [funcbox] + map(BoxInt, args), 'int', descr=calldescr)
         assert res.value == func(*args)
 
+    def test_call_box_func(self):
+        def a(a1, a2):
+            return a1 + a2
+        def b(b1, b2):
+            return b1 * b2
+
+        arg1 = 40
+        arg2 = 2
+        for f in [a, b]:
+            TP = lltype.Signed
+            FPTR = self.Ptr(self.FuncType([TP, TP], TP))
+            func_ptr = llhelper(FPTR, f)
+            FUNC = deref(FPTR)
+            funcconst = self.get_funcbox(self.cpu, func_ptr)
+            funcbox = funcconst.clonebox()
+            calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT,
+                                        EffectInfo.MOST_GENERAL)
+            res = self.execute_operation(rop.CALL,
+                                         [funcbox, BoxInt(arg1), BoxInt(arg2)],
+                                         'int', descr=calldescr)
+            assert res.getint() == f(arg1, arg2)
+        
     def test_call_stack_alignment(self):
         # test stack alignment issues, notably for Mac OS/X.
         # also test the ordering of the arguments.

From noreply at buildbot.pypy.org  Fri Jan 20 17:17:24 2012
From: noreply at buildbot.pypy.org (bivab)
Date: Fri, 20 Jan 2012 17:17:24 +0100 (CET)
Subject: [pypy-commit] pypy arm-backend-2: (arigo,
 bivab) add support for calling functions using indirect calls
Message-ID: <20120120161724.538ED82CF8@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider 
Branch: arm-backend-2
Changeset: r51550:ec71ed6721d4
Date: 2012-01-20 17:15 +0100
http://bitbucket.org/pypy/pypy/changeset/ec71ed6721d4/

Log:	(arigo, bivab) add support for calling functions using indirect
	calls

diff --git a/pypy/jit/backend/arm/opassembler.py b/pypy/jit/backend/arm/opassembler.py
--- a/pypy/jit/backend/arm/opassembler.py
+++ b/pypy/jit/backend/arm/opassembler.py
@@ -437,10 +437,14 @@
             else:
                 num += 1
                 count += 1
-
-        # spill variables that need to be saved around calls
-        regalloc.before_call(save_all_regs=2)
-
+        # Check that the address of the function we want to call is not
+        # currently stored in one of the registers used to pass the arguments.
+        # If this happens to be the case we remap the register to r4 and use r4
+        # to call the function
+        if adr in non_float_regs:
+            non_float_locs.append(adr)
+            non_float_regs.append(r.r4)
+            adr = r.r4
         # remap values stored in core registers
         remap_frame_layout(self, non_float_locs, non_float_regs, r.ip)
 
@@ -448,7 +452,15 @@
             self.mov_from_vfp_loc(loc, reg, r.all_regs[reg.value + 1])
 
         #the actual call
-        self.mc.BL(adr)
+        if adr.is_imm():
+            self.mc.BL(adr.value)
+        elif adr.is_stack():
+            self.mov_loc_loc(adr, r.ip)
+            adr = r.ip
+        else:
+            assert adr.is_reg()
+        if adr.is_reg():
+            self.mc.BLX(adr.value)
         self.mark_gc_roots(force_index)
         # readjust the sp in case we passed some args on the stack
         if n > 0:

From noreply at buildbot.pypy.org  Fri Jan 20 17:17:25 2012
From: noreply at buildbot.pypy.org (bivab)
Date: Fri, 20 Jan 2012 17:17:25 +0100 (CET)
Subject: [pypy-commit] pypy default: (arigo,
 bivab) Add a test that checks the behaviour of calling functions
 stored in boxes
Message-ID: <20120120161725.8912482CF8@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider 
Branch: 
Changeset: r51551:509859199b7d
Date: 2012-01-20 17:14 +0100
http://bitbucket.org/pypy/pypy/changeset/509859199b7d/

Log:	(arigo, bivab) Add a test that checks the behaviour of calling
	functions stored in boxes

diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py
--- a/pypy/jit/backend/test/runner_test.py
+++ b/pypy/jit/backend/test/runner_test.py
@@ -551,6 +551,28 @@
         res = self.execute_operation(rop.CALL, [funcbox] + map(BoxInt, args), 'int', descr=calldescr)
         assert res.value == func(*args)
 
+    def test_call_box_func(self):
+        def a(a1, a2):
+            return a1 + a2
+        def b(b1, b2):
+            return b1 * b2
+
+        arg1 = 40
+        arg2 = 2
+        for f in [a, b]:
+            TP = lltype.Signed
+            FPTR = self.Ptr(self.FuncType([TP, TP], TP))
+            func_ptr = llhelper(FPTR, f)
+            FUNC = deref(FPTR)
+            funcconst = self.get_funcbox(self.cpu, func_ptr)
+            funcbox = funcconst.clonebox()
+            calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT,
+                                        EffectInfo.MOST_GENERAL)
+            res = self.execute_operation(rop.CALL,
+                                         [funcbox, BoxInt(arg1), BoxInt(arg2)],
+                                         'int', descr=calldescr)
+            assert res.getint() == f(arg1, arg2)
+        
     def test_call_stack_alignment(self):
         # test stack alignment issues, notably for Mac OS/X.
         # also test the ordering of the arguments.

From noreply at buildbot.pypy.org  Fri Jan 20 17:23:11 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 20 Jan 2012 17:23:11 +0100 (CET)
Subject: [pypy-commit] pypy default: Fix the new test_call_box_func() on
	64-bit.
Message-ID: <20120120162311.19CFC82CF8@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r51552:245cb20ecdc0
Date: 2012-01-20 17:22 +0100
http://bitbucket.org/pypy/pypy/changeset/245cb20ecdc0/

Log:	Fix the new test_call_box_func() on 64-bit.

diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -1118,6 +1118,12 @@
             for src, dst in singlefloats:
                 self.mc.MOVD(dst, src)
         # Finally remap the arguments in the main regs
+        # If x is a register and is in dst_locs, then oups, it needs to
+        # be moved away:
+        if x in dst_locs:
+            src_locs.append(x)
+            dst_locs.append(r10)
+            x = r10
         remap_frame_layout(self, src_locs, dst_locs, X86_64_SCRATCH_REG)
 
         self._regalloc.reserve_param(len(pass_on_stack))

From noreply at buildbot.pypy.org  Fri Jan 20 17:24:18 2012
From: noreply at buildbot.pypy.org (bivab)
Date: Fri, 20 Jan 2012 17:24:18 +0100 (CET)
Subject: [pypy-commit] pypy arm-backend-2: forgot to kill this line
Message-ID: <20120120162418.581D682CF8@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider 
Branch: arm-backend-2
Changeset: r51553:5172f0c3f717
Date: 2012-01-20 17:24 +0100
http://bitbucket.org/pypy/pypy/changeset/5172f0c3f717/

Log:	forgot to kill this line

diff --git a/pypy/jit/backend/arm/opassembler.py b/pypy/jit/backend/arm/opassembler.py
--- a/pypy/jit/backend/arm/opassembler.py
+++ b/pypy/jit/backend/arm/opassembler.py
@@ -379,7 +379,6 @@
         return cond
 
     def _emit_call(self, force_index, adr, arglocs, fcond=c.AL, resloc=None):
-        assert self._regalloc.before_call_called
         n_args = len(arglocs)
         reg_args = count_reg_args(arglocs)
         # all arguments past the 4th go on the stack

From noreply at buildbot.pypy.org  Fri Jan 20 17:26:32 2012
From: noreply at buildbot.pypy.org (cocoatomo)
Date: Fri, 20 Jan 2012 17:26:32 +0100 (CET)
Subject: [pypy-commit] pypy default: fix typo
Message-ID: <20120120162632.E75C382CF8@wyvern.cs.uni-duesseldorf.de>

Author: cocoatomo 
Branch: 
Changeset: r51554:ec4009b69fc3
Date: 2012-01-20 11:14 +0900
http://bitbucket.org/pypy/pypy/changeset/ec4009b69fc3/

Log:	fix typo

diff --git a/pypy/doc/translation.rst b/pypy/doc/translation.rst
--- a/pypy/doc/translation.rst
+++ b/pypy/doc/translation.rst
@@ -155,7 +155,7 @@
                    function.  The two input variables are the exception class
                    and the exception value, respectively.  (No other block will
                    actually link to the exceptblock if the function does not
-                   explicitely raise exceptions.)
+                   explicitly raise exceptions.)
 
 
 ``Block``
@@ -325,7 +325,7 @@
 Mutable objects need special treatment during annotation, because
 the annotation of contained values needs to be possibly updated to account
 for mutation operations, and consequently the annotation information
-reflown through the relevant parts of the flow the graphs.
+reflown through the relevant parts of the flow graphs.
 
 * ``SomeList`` stands for a list of homogeneous type (i.e. all the
   elements of the list are represented by a single common ``SomeXxx``
@@ -503,8 +503,8 @@
 
 Since RPython is a garbage collected language there is a lot of heap memory
 allocation going on all the time, which would either not occur at all in a more
-traditional explicitely managed language or results in an object which dies at
-a time known in advance and can thus be explicitely deallocated. For example a
+traditional explicitly managed language or results in an object which dies at
+a time known in advance and can thus be explicitly deallocated. For example a
 loop of the following form::
 
     for i in range(n):
@@ -696,7 +696,7 @@
 
 So far it is the second most mature high level backend after GenCLI:
 it still can't translate the full Standard Interpreter, but after the
-Leysin sprint we were able to compile and run the rpytstone and
+Leysin sprint we were able to compile and run the rpystone and
 richards benchmarks.
 
 GenJVM is almost entirely the work of Niko Matsakis, who worked on it

From noreply at buildbot.pypy.org  Fri Jan 20 17:26:34 2012
From: noreply at buildbot.pypy.org (cocoatomo)
Date: Fri, 20 Jan 2012 17:26:34 +0100 (CET)
Subject: [pypy-commit] pypy default: merge
Message-ID: <20120120162634.96EC582CF8@wyvern.cs.uni-duesseldorf.de>

Author: cocoatomo 
Branch: 
Changeset: r51555:3f7ae53430c4
Date: 2012-01-20 14:06 +0900
http://bitbucket.org/pypy/pypy/changeset/3f7ae53430c4/

Log:	merge

diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -2,6 +2,9 @@
 *.py[co]
 *~
 .*.swp
+.idea
+.project
+.pydevproject
 
 syntax: regexp
 ^testresult$
diff --git a/lib_pypy/numpypy/__init__.py b/lib_pypy/numpypy/__init__.py
--- a/lib_pypy/numpypy/__init__.py
+++ b/lib_pypy/numpypy/__init__.py
@@ -1,2 +1,2 @@
 from _numpypy import *
-from .fromnumeric import *
+from .core import *
diff --git a/lib_pypy/numpypy/core/__init__.py b/lib_pypy/numpypy/core/__init__.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/numpypy/core/__init__.py
@@ -0,0 +1,1 @@
+from .fromnumeric import *
diff --git a/lib_pypy/numpypy/fromnumeric.py b/lib_pypy/numpypy/core/fromnumeric.py
rename from lib_pypy/numpypy/fromnumeric.py
rename to lib_pypy/numpypy/core/fromnumeric.py
--- a/lib_pypy/numpypy/fromnumeric.py
+++ b/lib_pypy/numpypy/core/fromnumeric.py
@@ -85,7 +85,7 @@
     array([4, 3, 6])
 
     """
-    raise NotImplemented('Waiting on interp level method')
+    raise NotImplementedError('Waiting on interp level method')
 
 
 # not deprecated --- copy if necessary, view otherwise
@@ -273,7 +273,7 @@
             [-1, -2, -3, -4, -5]]])
 
     """
-    raise NotImplemented('Waiting on interp level method')
+    raise NotImplementedError('Waiting on interp level method')
 
 
 def repeat(a, repeats, axis=None):
@@ -315,7 +315,7 @@
            [3, 4]])
 
     """
-    raise NotImplemented('Waiting on interp level method')
+    raise NotImplementedError('Waiting on interp level method')
 
 
 def put(a, ind, v, mode='raise'):
@@ -366,7 +366,7 @@
     array([ 0,  1,  2,  3, -5])
 
     """
-    raise NotImplemented('Waiting on interp level method')
+    raise NotImplementedError('Waiting on interp level method')
 
 
 def swapaxes(a, axis1, axis2):
@@ -410,7 +410,7 @@
             [3, 7]]])
 
     """
-    raise NotImplemented('Waiting on interp level method')
+    raise NotImplementedError('Waiting on interp level method')
 
 
 def transpose(a, axes=None):
@@ -451,8 +451,11 @@
     (2, 1, 3)
 
     """
-    raise NotImplemented('Waiting on interp level method')
-
+    if axes is not None:
+        raise NotImplementedError('No "axes" arg yet.')
+    if not hasattr(a, 'T'):
+        a = numpypy.array(a)
+    return a.T
 
 def sort(a, axis=-1, kind='quicksort', order=None):
     """
@@ -553,7 +556,7 @@
           dtype=[('name', '|S10'), ('height', '1:
+                    w_res = arr.descr_sum(interp.space,
+                                          self.args[1].execute(interp))
+                else:
+                    w_res = arr.descr_sum(interp.space)
             elif self.name == "prod":
                 w_res = arr.descr_prod(interp.space)
             elif self.name == "max":
@@ -416,7 +420,7 @@
     ('\]', 'array_right'),
     ('(->)|[\+\-\*\/]', 'operator'),
     ('=', 'assign'),
-    (',', 'coma'),
+    (',', 'comma'),
     ('\|', 'pipe'),
     ('\(', 'paren_left'),
     ('\)', 'paren_right'),
@@ -504,7 +508,7 @@
         return SliceConstant(start, stop, step)
 
 
-    def parse_expression(self, tokens):
+    def parse_expression(self, tokens, accept_comma=False):
         stack = []
         while tokens.remaining():
             token = tokens.pop()
@@ -524,9 +528,13 @@
                 stack.append(RangeConstant(tokens.pop().v))
                 end = tokens.pop()
                 assert end.name == 'pipe'
+            elif accept_comma and token.name == 'comma':
+                continue
             else:
                 tokens.push()
                 break
+        if accept_comma:
+            return stack
         stack.reverse()
         lhs = stack.pop()
         while stack:
@@ -540,7 +548,7 @@
         args = []
         tokens.pop() # lparen
         while tokens.get(0).name != 'paren_right':
-            args.append(self.parse_expression(tokens))
+            args += self.parse_expression(tokens, accept_comma=True)
         return FunctionCall(name, args)
 
     def parse_array_const(self, tokens):
@@ -556,7 +564,7 @@
             token = tokens.pop()
             if token.name == 'array_right':
                 return elems
-            assert token.name == 'coma'
+            assert token.name == 'comma'
 
     def parse_statement(self, tokens):
         if (tokens.get(0).name == 'identifier' and
diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -20,7 +20,7 @@
 class W_Dtype(Wrappable):
     _immutable_fields_ = ["itemtype", "num", "kind"]
 
-    def __init__(self, itemtype, num, kind, name, char, w_box_type, alternate_constructors=[]):
+    def __init__(self, itemtype, num, kind, name, char, w_box_type, alternate_constructors=[], aliases=[]):
         self.itemtype = itemtype
         self.num = num
         self.kind = kind
@@ -28,6 +28,7 @@
         self.char = char
         self.w_box_type = w_box_type
         self.alternate_constructors = alternate_constructors
+        self.aliases = aliases
 
     def malloc(self, length):
         # XXX find out why test_zjit explodes with tracking of allocations
@@ -62,7 +63,7 @@
         elif space.isinstance_w(w_dtype, space.w_str):
             name = space.str_w(w_dtype)
             for dtype in cache.builtin_dtypes:
-                if dtype.name == name or dtype.char == name:
+                if dtype.name == name or dtype.char == name or name in dtype.aliases:
                     return dtype
         else:
             for dtype in cache.builtin_dtypes:
@@ -107,7 +108,7 @@
             kind=BOOLLTR,
             name="bool",
             char="?",
-            w_box_type = space.gettypefor(interp_boxes.W_BoolBox),
+            w_box_type=space.gettypefor(interp_boxes.W_BoolBox),
             alternate_constructors=[space.w_bool],
         )
         self.w_int8dtype = W_Dtype(
@@ -116,7 +117,7 @@
             kind=SIGNEDLTR,
             name="int8",
             char="b",
-            w_box_type = space.gettypefor(interp_boxes.W_Int8Box)
+            w_box_type=space.gettypefor(interp_boxes.W_Int8Box)
         )
         self.w_uint8dtype = W_Dtype(
             types.UInt8(),
@@ -124,7 +125,7 @@
             kind=UNSIGNEDLTR,
             name="uint8",
             char="B",
-            w_box_type = space.gettypefor(interp_boxes.W_UInt8Box),
+            w_box_type=space.gettypefor(interp_boxes.W_UInt8Box),
         )
         self.w_int16dtype = W_Dtype(
             types.Int16(),
@@ -132,7 +133,7 @@
             kind=SIGNEDLTR,
             name="int16",
             char="h",
-            w_box_type = space.gettypefor(interp_boxes.W_Int16Box),
+            w_box_type=space.gettypefor(interp_boxes.W_Int16Box),
         )
         self.w_uint16dtype = W_Dtype(
             types.UInt16(),
@@ -140,7 +141,7 @@
             kind=UNSIGNEDLTR,
             name="uint16",
             char="H",
-            w_box_type = space.gettypefor(interp_boxes.W_UInt16Box),
+            w_box_type=space.gettypefor(interp_boxes.W_UInt16Box),
         )
         self.w_int32dtype = W_Dtype(
             types.Int32(),
@@ -148,7 +149,7 @@
             kind=SIGNEDLTR,
             name="int32",
             char="i",
-             w_box_type = space.gettypefor(interp_boxes.W_Int32Box),
+            w_box_type=space.gettypefor(interp_boxes.W_Int32Box),
        )
         self.w_uint32dtype = W_Dtype(
             types.UInt32(),
@@ -156,7 +157,7 @@
             kind=UNSIGNEDLTR,
             name="uint32",
             char="I",
-            w_box_type = space.gettypefor(interp_boxes.W_UInt32Box),
+            w_box_type=space.gettypefor(interp_boxes.W_UInt32Box),
         )
         if LONG_BIT == 32:
             name = "int32"
@@ -168,7 +169,7 @@
             kind=SIGNEDLTR,
             name=name,
             char="l",
-            w_box_type = space.gettypefor(interp_boxes.W_LongBox),
+            w_box_type=space.gettypefor(interp_boxes.W_LongBox),
             alternate_constructors=[space.w_int],
         )
         self.w_ulongdtype = W_Dtype(
@@ -177,7 +178,7 @@
             kind=UNSIGNEDLTR,
             name="u" + name,
             char="L",
-            w_box_type = space.gettypefor(interp_boxes.W_ULongBox),
+            w_box_type=space.gettypefor(interp_boxes.W_ULongBox),
         )
         self.w_int64dtype = W_Dtype(
             types.Int64(),
@@ -185,7 +186,7 @@
             kind=SIGNEDLTR,
             name="int64",
             char="q",
-            w_box_type = space.gettypefor(interp_boxes.W_Int64Box),
+            w_box_type=space.gettypefor(interp_boxes.W_Int64Box),
             alternate_constructors=[space.w_long],
         )
         self.w_uint64dtype = W_Dtype(
@@ -194,7 +195,7 @@
             kind=UNSIGNEDLTR,
             name="uint64",
             char="Q",
-            w_box_type = space.gettypefor(interp_boxes.W_UInt64Box),
+            w_box_type=space.gettypefor(interp_boxes.W_UInt64Box),
         )
         self.w_float32dtype = W_Dtype(
             types.Float32(),
@@ -202,7 +203,7 @@
             kind=FLOATINGLTR,
             name="float32",
             char="f",
-            w_box_type = space.gettypefor(interp_boxes.W_Float32Box),
+            w_box_type=space.gettypefor(interp_boxes.W_Float32Box),
         )
         self.w_float64dtype = W_Dtype(
             types.Float64(),
@@ -212,6 +213,7 @@
             char="d",
             w_box_type = space.gettypefor(interp_boxes.W_Float64Box),
             alternate_constructors=[space.w_float],
+            aliases=["float"],
         )
 
         self.builtin_dtypes = [
diff --git a/pypy/module/micronumpy/interp_iter.py b/pypy/module/micronumpy/interp_iter.py
--- a/pypy/module/micronumpy/interp_iter.py
+++ b/pypy/module/micronumpy/interp_iter.py
@@ -1,19 +1,20 @@
 
 from pypy.rlib import jit
 from pypy.rlib.objectmodel import instantiate
-from pypy.module.micronumpy.strides import calculate_broadcast_strides
+from pypy.module.micronumpy.strides import calculate_broadcast_strides,\
+     calculate_slice_strides
 
-# Iterators for arrays
-# --------------------
-# all those iterators with the exception of BroadcastIterator iterate over the
-# entire array in C order (the last index changes the fastest). This will
-# yield all elements. Views iterate over indices and look towards strides and
-# backstrides to find the correct position. Notably the offset between
-# x[..., i + 1] and x[..., i] will be strides[-1]. Offset between
-# x[..., k + 1, 0] and x[..., k, i_max] will be backstrides[-2] etc.
+class BaseTransform(object):
+    pass
 
-# BroadcastIterator works like that, but for indexes that don't change source
-# in the original array, strides[i] == backstrides[i] == 0
+class ViewTransform(BaseTransform):
+    def __init__(self, chunks):
+        # 4-tuple specifying slicing
+        self.chunks = chunks
+
+class BroadcastTransform(BaseTransform):
+    def __init__(self, res_shape):
+        self.res_shape = res_shape
 
 class BaseIterator(object):
     def next(self, shapelen):
@@ -22,6 +23,15 @@
     def done(self):
         raise NotImplementedError
 
+    def apply_transformations(self, arr, transformations):
+        v = self
+        for transform in transformations:
+            v = v.transform(arr, transform)
+        return v
+
+    def transform(self, arr, t):
+        raise NotImplementedError
+
 class ArrayIterator(BaseIterator):
     def __init__(self, size):
         self.offset = 0
@@ -36,6 +46,10 @@
     def done(self):
         return self.offset >= self.size
 
+    def transform(self, arr, t):
+        return ViewIterator(arr.start, arr.strides, arr.backstrides,
+                            arr.shape).transform(arr, t)
+
 class OneDimIterator(BaseIterator):
     def __init__(self, start, step, stop):
         self.offset = start
@@ -52,26 +66,29 @@
     def done(self):
         return self.offset == self.size
 
-def view_iter_from_arr(arr):
-    return ViewIterator(arr.start, arr.strides, arr.backstrides, arr.shape)
-
 class ViewIterator(BaseIterator):
-    def __init__(self, start, strides, backstrides, shape, res_shape=None):
+    def __init__(self, start, strides, backstrides, shape):
         self.offset  = start
         self._done   = False
-        if res_shape is not None and res_shape != shape:
-            r = calculate_broadcast_strides(strides, backstrides,
-                                            shape, res_shape)
-            self.strides, self.backstrides = r
-            self.res_shape = res_shape
-        else:
-            self.strides = strides
-            self.backstrides = backstrides
-            self.res_shape = shape
+        self.strides = strides
+        self.backstrides = backstrides
+        self.res_shape = shape
         self.indices = [0] * len(self.res_shape)
 
+    def transform(self, arr, t):
+        if isinstance(t, BroadcastTransform):
+            r = calculate_broadcast_strides(self.strides, self.backstrides,
+                                            self.res_shape, t.res_shape)
+            return ViewIterator(self.offset, r[0], r[1], t.res_shape)
+        elif isinstance(t, ViewTransform):
+            r = calculate_slice_strides(self.res_shape, self.offset,
+                                        self.strides,
+                                        self.backstrides, t.chunks)
+            return ViewIterator(r[1], r[2], r[3], r[0])
+
     @jit.unroll_safe
     def next(self, shapelen):
+        shapelen = jit.promote(len(self.res_shape))
         offset = self.offset
         indices = [0] * shapelen
         for i in range(shapelen):
@@ -96,6 +113,13 @@
         res._done = done
         return res
 
+    def apply_transformations(self, arr, transformations):
+        v = BaseIterator.apply_transformations(self, arr, transformations)
+        if len(arr.shape) == 1:
+            return OneDimIterator(self.offset, self.strides[0],
+                                  self.res_shape[0])
+        return v
+
     def done(self):
         return self._done
 
@@ -103,11 +127,59 @@
     def next(self, shapelen):
         return self
 
+    def transform(self, arr, t):
+        pass
+
+class AxisIterator(BaseIterator):
+    def __init__(self, start, dim, shape, strides, backstrides):
+        self.res_shape = shape[:]
+        self.strides = strides[:dim] + [0] + strides[dim:]
+        self.backstrides = backstrides[:dim] + [0] + backstrides[dim:]
+        self.first_line = True
+        self.indices = [0] * len(shape)
+        self._done = False
+        self.offset = start
+        self.dim = dim
+
+    @jit.unroll_safe
+    def next(self, shapelen):
+        offset = self.offset
+        first_line = self.first_line
+        indices = [0] * shapelen
+        for i in range(shapelen):
+            indices[i] = self.indices[i]
+        done = False
+        for i in range(shapelen - 1, -1, -1):
+            if indices[i] < self.res_shape[i] - 1:
+                if i == self.dim:
+                    first_line = False
+                indices[i] += 1
+                offset += self.strides[i]
+                break
+            else:
+                if i == self.dim:
+                    first_line = True
+                indices[i] = 0
+                offset -= self.backstrides[i]
+        else:
+            done = True
+        res = instantiate(AxisIterator)
+        res.offset = offset
+        res.indices = indices
+        res.strides = self.strides
+        res.backstrides = self.backstrides
+        res.res_shape = self.res_shape
+        res._done = done
+        res.first_line = first_line
+        res.dim = self.dim
+        return res        
+
+    def done(self):
+        return self._done
+
 # ------ other iterators that are not part of the computation frame ----------
-
-class AxisIterator(object):
-    """ This object will return offsets of each start of the last stride
-    """
+    
+class SkipLastAxisIterator(object):
     def __init__(self, arr):
         self.arr = arr
         self.indices = [0] * (len(arr.shape) - 1)
@@ -125,4 +197,3 @@
                 self.offset -= self.arr.backstrides[i]
         else:
             self.done = True
-        
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -8,8 +8,8 @@
 from pypy.rpython.lltypesystem import lltype, rffi
 from pypy.tool.sourcetools import func_with_new_name
 from pypy.rlib.rstring import StringBuilder
-from pypy.module.micronumpy.interp_iter import ArrayIterator,\
-     view_iter_from_arr, OneDimIterator, AxisIterator
+from pypy.module.micronumpy.interp_iter import ArrayIterator, OneDimIterator,\
+     SkipLastAxisIterator
 
 numpy_driver = jit.JitDriver(
     greens=['shapelen', 'sig'],
@@ -35,11 +35,12 @@
 slice_driver = jit.JitDriver(
     greens=['shapelen', 'sig'],
     virtualizables=['frame'],
-    reds=['self', 'frame', 'source', 'res_iter'],
+    reds=['self', 'frame', 'arr'],
     get_printable_location=signature.new_printable_location('slice'),
     name='numpy_slice',
 )
 
+
 def _find_shape_and_elems(space, w_iterable):
     shape = [space.len_w(w_iterable)]
     batch = space.listview(w_iterable)
@@ -156,9 +157,6 @@
 # (meaning that the realignment of elements crosses from one step into another)
 # return None so that the caller can raise an exception.
 def calc_new_strides(new_shape, old_shape, old_strides):
-    # Return the proper strides for new_shape, or None if the mapping crosses
-    # stepping boundaries
-
     # Assumes that prod(old_shape) == prod(new_shape), len(old_shape) > 1, and
     # len(new_shape) > 0
     steps = []
@@ -166,6 +164,7 @@
     oldI = 0
     new_strides = []
     if old_strides[0] < old_strides[-1]:
+        #Start at old_shape[0], old_stides[0]
         for i in range(len(old_shape)):
             steps.append(old_strides[i] / last_step)
             last_step *= old_shape[i]
@@ -183,10 +182,11 @@
             if n_new_elems_used == n_old_elems_to_use:
                 oldI += 1
                 if oldI >= len(old_shape):
-                    break
+                    continue
                 cur_step = steps[oldI]
                 n_old_elems_to_use *= old_shape[oldI]
     else:
+        #Start at old_shape[-1], old_strides[-1]
         for i in range(len(old_shape) - 1, -1, -1):
             steps.insert(0, old_strides[i] / last_step)
             last_step *= old_shape[i]
@@ -206,7 +206,7 @@
             if n_new_elems_used == n_old_elems_to_use:
                 oldI -= 1
                 if oldI < -len(old_shape):
-                    break
+                    continue
                 cur_step = steps[oldI]
                 n_old_elems_to_use *= old_shape[oldI]
     return new_strides
@@ -286,13 +286,17 @@
     descr_rpow = _binop_right_impl("power")
     descr_rmod = _binop_right_impl("mod")
 
-    def _reduce_ufunc_impl(ufunc_name):
-        def impl(self, space):
-            return getattr(interp_ufuncs.get(space), ufunc_name).reduce(space, self, multidim=True)
+    def _reduce_ufunc_impl(ufunc_name, promote_to_largest=False):
+        def impl(self, space, w_axis=None):
+            if space.is_w(w_axis, space.w_None):
+                w_axis = space.wrap(-1)
+            return getattr(interp_ufuncs.get(space), ufunc_name).reduce(space,
+                                        self, True, promote_to_largest, w_axis)
         return func_with_new_name(impl, "reduce_%s_impl" % ufunc_name)
 
     descr_sum = _reduce_ufunc_impl("add")
-    descr_prod = _reduce_ufunc_impl("multiply")
+    descr_sum_promote = _reduce_ufunc_impl("add", True)
+    descr_prod = _reduce_ufunc_impl("multiply", True)
     descr_max = _reduce_ufunc_impl("maximum")
     descr_min = _reduce_ufunc_impl("minimum")
 
@@ -377,7 +381,7 @@
         else:
             w_res = self.descr_mul(space, w_other)
             assert isinstance(w_res, BaseArray)
-            return w_res.descr_sum(space)
+            return w_res.descr_sum(space, space.wrap(-1))
 
     def get_concrete(self):
         raise NotImplementedError
@@ -565,21 +569,31 @@
             )
         return w_result
 
-    def descr_mean(self, space):
-        return space.div(self.descr_sum(space), space.wrap(self.size))
+    def descr_mean(self, space, w_axis=None):
+        if space.is_w(w_axis, space.w_None):
+            w_axis = space.wrap(-1)
+            w_denom = space.wrap(self.size)
+        else:
+            dim = space.int_w(w_axis)
+            w_denom = space.wrap(self.shape[dim])
+        return space.div(self.descr_sum_promote(space, w_axis), w_denom)
 
     def descr_var(self, space):
         # var = mean((values - mean(values)) ** 2)
-        w_res = self.descr_sub(space, self.descr_mean(space))
-        assert isinstance(w_res, BaseArray) 
+        w_res = self.descr_sub(space, self.descr_mean(space, space.w_None))
+        assert isinstance(w_res, BaseArray)
         w_res = w_res.descr_pow(space, space.wrap(2))
         assert isinstance(w_res, BaseArray)
-        return w_res.descr_mean(space)
+        return w_res.descr_mean(space, space.w_None)
 
     def descr_std(self, space):
         # std(v) = sqrt(var(v))
         return interp_ufuncs.get(space).sqrt.call(space, [self.descr_var(space)])
 
+    def descr_fill(self, space, w_value):
+        concr = self.get_concrete_or_scalar()
+        concr.fill(space, w_value)
+
     def descr_nonzero(self, space):
         if self.size > 1:
             raise OperationError(space.w_ValueError, space.wrap(
@@ -613,11 +627,12 @@
     def getitem(self, item):
         raise NotImplementedError
 
-    def find_sig(self, res_shape=None):
+    def find_sig(self, res_shape=None, arr=None):
         """ find a correct signature for the array
         """
         res_shape = res_shape or self.shape
-        return signature.find_sig(self.create_sig(res_shape), self)
+        arr = arr or self
+        return signature.find_sig(self.create_sig(), arr)
 
     def descr_array_iface(self, space):
         if not self.shape:
@@ -671,7 +686,10 @@
     def copy(self, space):
         return Scalar(self.dtype, self.value)
 
-    def create_sig(self, res_shape):
+    def fill(self, space, w_value):
+        self.value = self.dtype.coerce(space, w_value)
+
+    def create_sig(self):
         return signature.ScalarSignature(self.dtype)
 
     def get_concrete_or_scalar(self):
@@ -689,7 +707,8 @@
         self.name = name
 
     def _del_sources(self):
-        # Function for deleting references to source arrays, to allow garbage-collecting them
+        # Function for deleting references to source arrays,
+        # to allow garbage-collecting them
         raise NotImplementedError
 
     def compute(self):
@@ -741,11 +760,11 @@
         self.size = size
         VirtualArray.__init__(self, 'slice', shape, child.find_dtype())
 
-    def create_sig(self, res_shape):
+    def create_sig(self):
         if self.forced_result is not None:
-            return self.forced_result.create_sig(res_shape)
+            return self.forced_result.create_sig()
         return signature.VirtualSliceSignature(
-            self.child.create_sig(res_shape))
+            self.child.create_sig())
 
     def force_if_needed(self):
         if self.forced_result is None:
@@ -755,6 +774,7 @@
     def _del_sources(self):
         self.child = None
 
+
 class Call1(VirtualArray):
     def __init__(self, ufunc, name, shape, res_dtype, values):
         VirtualArray.__init__(self, name, shape, res_dtype)
@@ -765,16 +785,17 @@
     def _del_sources(self):
         self.values = None
 
-    def create_sig(self, res_shape):
+    def create_sig(self):
         if self.forced_result is not None:
-            return self.forced_result.create_sig(res_shape)
-        return signature.Call1(self.ufunc, self.name,
-                               self.values.create_sig(res_shape))
+            return self.forced_result.create_sig()
+        return signature.Call1(self.ufunc, self.name, self.values.create_sig())
 
 class Call2(VirtualArray):
     """
     Intermediate class for performing binary operations.
     """
+    _immutable_fields_ = ['left', 'right']
+
     def __init__(self, ufunc, name, shape, calc_dtype, res_dtype, left, right):
         VirtualArray.__init__(self, name, shape, res_dtype)
         self.ufunc = ufunc
@@ -789,12 +810,55 @@
         self.left = None
         self.right = None
 
-    def create_sig(self, res_shape):
+    def create_sig(self):
         if self.forced_result is not None:
-            return self.forced_result.create_sig(res_shape)
+            return self.forced_result.create_sig()
+        if self.shape != self.left.shape and self.shape != self.right.shape:
+            return signature.BroadcastBoth(self.ufunc, self.name,
+                                           self.calc_dtype,
+                                           self.left.create_sig(),
+                                           self.right.create_sig())
+        elif self.shape != self.left.shape:
+            return signature.BroadcastLeft(self.ufunc, self.name,
+                                           self.calc_dtype,
+                                           self.left.create_sig(),
+                                           self.right.create_sig())
+        elif self.shape != self.right.shape:
+            return signature.BroadcastRight(self.ufunc, self.name,
+                                            self.calc_dtype,
+                                            self.left.create_sig(),
+                                            self.right.create_sig())
         return signature.Call2(self.ufunc, self.name, self.calc_dtype,
-                               self.left.create_sig(res_shape),
-                               self.right.create_sig(res_shape))
+                               self.left.create_sig(), self.right.create_sig())
+
+class SliceArray(Call2):
+    def __init__(self, shape, dtype, left, right):
+        Call2.__init__(self, None, 'sliceloop', shape, dtype, dtype, left,
+                       right)
+
+    def create_sig(self):
+        lsig = self.left.create_sig()
+        rsig = self.right.create_sig()
+        if self.shape != self.right.shape:
+            return signature.SliceloopBroadcastSignature(self.ufunc,
+                                                         self.name,
+                                                         self.calc_dtype,
+                                                         lsig, rsig)
+        return signature.SliceloopSignature(self.ufunc, self.name,
+                                            self.calc_dtype,
+                                            lsig, rsig)
+
+class AxisReduce(Call2):
+    """ NOTE: this is only used as a container, you should never
+    encounter such things in the wild. Remove this comment
+    when we'll make AxisReduce lazy
+    """
+    _immutable_fields_ = ['left', 'right']
+
+    def __init__(self, ufunc, name, shape, dtype, left, right, dim):
+        Call2.__init__(self, ufunc, name, shape, dtype, dtype,
+                       left, right)
+        self.dim = dim
 
 class ConcreteArray(BaseArray):
     """ An array that have actual storage, whether owned or not
@@ -849,11 +913,6 @@
         self.strides = strides
         self.backstrides = backstrides
 
-    def array_sig(self, res_shape):
-        if res_shape is not None and self.shape != res_shape:
-            return signature.ViewSignature(self.dtype)
-        return signature.ArraySignature(self.dtype)
-
     def to_str(self, space, comma, builder, indent=' ', use_ellipsis=False):
         '''Modifies builder with a representation of the array/slice
         The items will be seperated by a comma if comma is 1
@@ -867,14 +926,14 @@
         if size < 1:
             builder.append('[]')
             return
-        elif size == 1:
-            builder.append(dtype.itemtype.str_format(self.getitem(0)))
-            return
         if size > 1000:
             # Once this goes True it does not go back to False for recursive
             # calls
             use_ellipsis = True
         ndims = len(self.shape)
+        if ndims == 0:
+            builder.append(dtype.itemtype.str_format(self.getitem(0)))
+            return
         i = 0
         builder.append('[')
         if ndims > 1:
@@ -890,7 +949,7 @@
                     view.to_str(space, comma, builder, indent=indent + ' ',
                                                     use_ellipsis=use_ellipsis)
                 if i < self.shape[0] - 1:
-                    builder.append(ccomma +'\n' + indent + '...' + ncomma)
+                    builder.append(ccomma + '\n' + indent + '...' + ncomma)
                     i = self.shape[0] - 3
                 else:
                     i += 1
@@ -968,20 +1027,22 @@
             self.dtype is w_value.find_dtype()):
             self._fast_setslice(space, w_value)
         else:
-            self._sliceloop(w_value, res_shape)
+            arr = SliceArray(self.shape, self.dtype, self, w_value)
+            self._sliceloop(arr)
 
     def _fast_setslice(self, space, w_value):
         assert isinstance(w_value, ConcreteArray)
         itemsize = self.dtype.itemtype.get_element_size()
-        if len(self.shape) == 1:
+        shapelen = len(self.shape)
+        if shapelen == 1:
             rffi.c_memcpy(
                 rffi.ptradd(self.storage, self.start * itemsize),
                 rffi.ptradd(w_value.storage, w_value.start * itemsize),
                 self.size * itemsize
             )
         else:
-            dest = AxisIterator(self)
-            source = AxisIterator(w_value)
+            dest = SkipLastAxisIterator(self)
+            source = SkipLastAxisIterator(w_value)
             while not dest.done:
                 rffi.c_memcpy(
                     rffi.ptradd(self.storage, dest.offset * itemsize),
@@ -991,30 +1052,28 @@
                 source.next()
                 dest.next()
 
-    def _sliceloop(self, source, res_shape):
-        sig = source.find_sig(res_shape)
-        frame = sig.create_frame(source, res_shape)
-        res_iter = view_iter_from_arr(self)
-        shapelen = len(res_shape)
-        while not res_iter.done():
-            slice_driver.jit_merge_point(sig=sig,
-                                         frame=frame,
-                                         shapelen=shapelen,
-                                         self=self, source=source,
-                                         res_iter=res_iter)
-            self.setitem(res_iter.offset, sig.eval(frame, source).convert_to(
-                self.find_dtype()))
+    def _sliceloop(self, arr):
+        sig = arr.find_sig()
+        frame = sig.create_frame(arr)
+        shapelen = len(self.shape)
+        while not frame.done():
+            slice_driver.jit_merge_point(sig=sig, frame=frame, self=self,
+                                         arr=arr,
+                                         shapelen=shapelen)
+            sig.eval(frame, arr)
             frame.next(shapelen)
-            res_iter = res_iter.next(shapelen)
 
     def copy(self, space):
         array = W_NDimArray(self.size, self.shape[:], self.dtype, self.order)
         array.setslice(space, self)
         return array
 
+    def fill(self, space, w_value):
+        self.setslice(space, scalar_w(space, self.dtype, w_value))
+
 
 class ViewArray(ConcreteArray):
-    def create_sig(self, res_shape):
+    def create_sig(self):
         return signature.ViewSignature(self.dtype)
 
 
@@ -1078,8 +1137,8 @@
         self.shape = new_shape
         self.calc_strides(new_shape)
 
-    def create_sig(self, res_shape):
-        return self.array_sig(res_shape)
+    def create_sig(self):
+        return signature.ArraySignature(self.dtype)
 
     def __del__(self):
         lltype.free(self.storage, flavor='raw', track_allocation=False)
@@ -1224,6 +1283,8 @@
     var = interp2app(BaseArray.descr_var),
     std = interp2app(BaseArray.descr_std),
 
+    fill = interp2app(BaseArray.descr_fill),
+
     copy = interp2app(BaseArray.descr_copy),
     reshape = interp2app(BaseArray.descr_reshape),
     tolist = interp2app(BaseArray.descr_tolist),
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -3,20 +3,29 @@
 from pypy.interpreter.gateway import interp2app
 from pypy.interpreter.typedef import TypeDef, GetSetProperty, interp_attrproperty
 from pypy.module.micronumpy import interp_boxes, interp_dtype
-from pypy.module.micronumpy.signature import ReduceSignature, ScalarSignature,\
-     find_sig, new_printable_location
+from pypy.module.micronumpy.signature import ReduceSignature,\
+     find_sig, new_printable_location, AxisReduceSignature, ScalarSignature
 from pypy.rlib import jit
 from pypy.rlib.rarithmetic import LONG_BIT
 from pypy.tool.sourcetools import func_with_new_name
 
 reduce_driver = jit.JitDriver(
-    greens = ['shapelen', "sig"],
-    virtualizables = ["frame"],
-    reds = ["frame", "self", "dtype", "value", "obj"],
+    greens=['shapelen', "sig"],
+    virtualizables=["frame"],
+    reds=["frame", "self", "dtype", "value", "obj"],
     get_printable_location=new_printable_location('reduce'),
     name='numpy_reduce',
 )
 
+axisreduce_driver = jit.JitDriver(
+    greens=['shapelen', 'sig'],
+    virtualizables=['frame'],
+    reds=['self','arr', 'identity', 'frame'],
+    name='numpy_axisreduce',
+    get_printable_location=new_printable_location('axisreduce'),
+)
+
+
 class W_Ufunc(Wrappable):
     _attrs_ = ["name", "promote_to_float", "promote_bools", "identity"]
     _immutable_fields_ = ["promote_to_float", "promote_bools", "name"]
@@ -49,18 +58,72 @@
             )
         return self.call(space, __args__.arguments_w)
 
-    def descr_reduce(self, space, w_obj):
-        return self.reduce(space, w_obj, multidim=False)
+    def descr_reduce(self, space, w_obj, w_dim=0):
+        """reduce(...)
+        reduce(a, axis=0)
 
-    def reduce(self, space, w_obj, multidim):
-        from pypy.module.micronumpy.interp_numarray import convert_to_array, Scalar
-        
+        Reduces `a`'s dimension by one, by applying ufunc along one axis.
+
+        Let :math:`a.shape = (N_0, ..., N_i, ..., N_{M-1})`.  Then
+        :math:`ufunc.reduce(a, axis=i)[k_0, ..,k_{i-1}, k_{i+1}, .., k_{M-1}]` =
+        the result of iterating `j` over :math:`range(N_i)`, cumulatively applying
+        ufunc to each :math:`a[k_0, ..,k_{i-1}, j, k_{i+1}, .., k_{M-1}]`.
+        For a one-dimensional array, reduce produces results equivalent to:
+        ::
+
+         r = op.identity # op = ufunc
+         for i in xrange(len(A)):
+           r = op(r, A[i])
+         return r
+
+        For example, add.reduce() is equivalent to sum().
+
+        Parameters
+        ----------
+        a : array_like
+            The array to act on.
+        axis : int, optional
+            The axis along which to apply the reduction.
+
+        Examples
+        --------
+        >>> np.multiply.reduce([2,3,5])
+        30
+
+        A multi-dimensional array example:
+
+        >>> X = np.arange(8).reshape((2,2,2))
+        >>> X
+        array([[[0, 1],
+                [2, 3]],
+               [[4, 5],
+                [6, 7]]])
+        >>> np.add.reduce(X, 0)
+        array([[ 4,  6],
+               [ 8, 10]])
+        >>> np.add.reduce(X) # confirm: default axis value is 0
+        array([[ 4,  6],
+               [ 8, 10]])
+        >>> np.add.reduce(X, 1)
+        array([[ 2,  4],
+               [10, 12]])
+        >>> np.add.reduce(X, 2)
+        array([[ 1,  5],
+               [ 9, 13]])
+        """
+        return self.reduce(space, w_obj, False, False, w_dim)
+
+    def reduce(self, space, w_obj, multidim, promote_to_largest, w_dim):
+        from pypy.module.micronumpy.interp_numarray import convert_to_array, \
+                                                           Scalar
         if self.argcount != 2:
             raise OperationError(space.w_ValueError, space.wrap("reduce only "
                 "supported for binary functions"))
-
+        dim = space.int_w(w_dim)
         assert isinstance(self, W_Ufunc2)
         obj = convert_to_array(space, w_obj)
+        if dim >= len(obj.shape):
+            raise OperationError(space.w_ValueError, space.wrap("axis(=%d) out of bounds" % dim))
         if isinstance(obj, Scalar):
             raise OperationError(space.w_TypeError, space.wrap("cannot reduce "
                 "on a scalar"))
@@ -68,26 +131,80 @@
         size = obj.size
         dtype = find_unaryop_result_dtype(
             space, obj.find_dtype(),
-            promote_to_largest=True
+            promote_to_float=self.promote_to_float,
+            promote_to_largest=promote_to_largest,
+            promote_bools=True
         )
         shapelen = len(obj.shape)
+        if self.identity is None and size == 0:
+            raise operationerrfmt(space.w_ValueError, "zero-size array to "
+                    "%s.reduce without identity", self.name)
+        if shapelen > 1 and dim >= 0:
+            res = self.do_axis_reduce(obj, dtype, dim)
+            return space.wrap(res)
+        scalarsig = ScalarSignature(dtype)
         sig = find_sig(ReduceSignature(self.func, self.name, dtype,
-                                       ScalarSignature(dtype),
-                                       obj.create_sig(obj.shape)), obj)
+                                       scalarsig,
+                                       obj.create_sig()), obj)
         frame = sig.create_frame(obj)
-        if shapelen > 1 and not multidim:
-            raise OperationError(space.w_NotImplementedError,
-                space.wrap("not implemented yet"))
         if self.identity is None:
-            if size == 0:
-                raise operationerrfmt(space.w_ValueError, "zero-size array to "
-                    "%s.reduce without identity", self.name)
             value = sig.eval(frame, obj).convert_to(dtype)
             frame.next(shapelen)
         else:
             value = self.identity.convert_to(dtype)
         return self.reduce_loop(shapelen, sig, frame, value, obj, dtype)
 
+    def do_axis_reduce(self, obj, dtype, dim):
+        from pypy.module.micronumpy.interp_numarray import AxisReduce,\
+             W_NDimArray
+        
+        shape = obj.shape[0:dim] + obj.shape[dim + 1:len(obj.shape)]
+        size = 1
+        for s in shape:
+            size *= s
+        result = W_NDimArray(size, shape, dtype)
+        rightsig = obj.create_sig()
+        # note - this is just a wrapper so signature can fetch
+        #        both left and right, nothing more, especially
+        #        this is not a true virtual array, because shapes
+        #        don't quite match
+        arr = AxisReduce(self.func, self.name, obj.shape, dtype,
+                         result, obj, dim)
+        scalarsig = ScalarSignature(dtype)
+        sig = find_sig(AxisReduceSignature(self.func, self.name, dtype,
+                                           scalarsig, rightsig), arr)
+        assert isinstance(sig, AxisReduceSignature)
+        frame = sig.create_frame(arr)
+        shapelen = len(obj.shape)
+        if self.identity is not None:
+            identity = self.identity.convert_to(dtype)
+        else:
+            identity = None
+        self.reduce_axis_loop(frame, sig, shapelen, arr, identity)
+        return result
+
+    def reduce_axis_loop(self, frame, sig, shapelen, arr, identity):
+        # note - we can be advanterous here, depending on the exact field
+        # layout. For now let's say we iterate the original way and
+        # simply follow the original iteration order
+        while not frame.done():
+            axisreduce_driver.jit_merge_point(frame=frame, self=self,
+                                              sig=sig,
+                                              identity=identity,
+                                              shapelen=shapelen, arr=arr)
+            iter = frame.get_final_iter()
+            v = sig.eval(frame, arr).convert_to(sig.calc_dtype)
+            if iter.first_line:
+                if identity is not None:
+                    value = self.func(sig.calc_dtype, identity, v)
+                else:
+                    value = v
+            else:
+                cur = arr.left.getitem(iter.offset)
+                value = self.func(sig.calc_dtype, cur, v)
+            arr.left.setitem(iter.offset, value)
+            frame.next(shapelen)
+
     def reduce_loop(self, shapelen, sig, frame, value, obj, dtype):
         while not frame.done():
             reduce_driver.jit_merge_point(sig=sig,
@@ -95,10 +212,12 @@
                                           value=value, obj=obj, frame=frame,
                                           dtype=dtype)
             assert isinstance(sig, ReduceSignature)
-            value = sig.binfunc(dtype, value, sig.eval(frame, obj).convert_to(dtype))
+            value = sig.binfunc(dtype, value,
+                                sig.eval(frame, obj).convert_to(dtype))
             frame.next(shapelen)
         return value
 
+
 class W_Ufunc1(W_Ufunc):
     argcount = 1
 
@@ -183,6 +302,7 @@
     reduce = interp2app(W_Ufunc.descr_reduce),
 )
 
+
 def find_binop_result_dtype(space, dt1, dt2, promote_to_float=False,
     promote_bools=False):
     # dt1.num should be <= dt2.num
@@ -231,6 +351,7 @@
             dtypenum += 3
         return interp_dtype.get_dtype_cache(space).builtin_dtypes[dtypenum]
 
+
 def find_unaryop_result_dtype(space, dt, promote_to_float=False,
     promote_bools=False, promote_to_largest=False):
     if promote_bools and (dt.kind == interp_dtype.BOOLLTR):
@@ -255,6 +376,7 @@
             assert False
     return dt
 
+
 def find_dtype_for_scalar(space, w_obj, current_guess=None):
     bool_dtype = interp_dtype.get_dtype_cache(space).w_booldtype
     long_dtype = interp_dtype.get_dtype_cache(space).w_longdtype
@@ -327,6 +449,7 @@
 
             ("fabs", "fabs", 1, {"promote_to_float": True}),
             ("floor", "floor", 1, {"promote_to_float": True}),
+            ("ceil", "ceil", 1, {"promote_to_float": True}),
             ("exp", "exp", 1, {"promote_to_float": True}),
 
             ('sqrt', 'sqrt', 1, {'promote_to_float': True}),
@@ -348,7 +471,8 @@
 
         identity = extra_kwargs.get("identity")
         if identity is not None:
-            identity = interp_dtype.get_dtype_cache(space).w_longdtype.box(identity)
+            identity = \
+                 interp_dtype.get_dtype_cache(space).w_longdtype.box(identity)
         extra_kwargs["identity"] = identity
 
         func = ufunc_dtype_caller(space, ufunc_name, op_name, argcount,
diff --git a/pypy/module/micronumpy/signature.py b/pypy/module/micronumpy/signature.py
--- a/pypy/module/micronumpy/signature.py
+++ b/pypy/module/micronumpy/signature.py
@@ -1,10 +1,32 @@
 from pypy.rlib.objectmodel import r_dict, compute_identity_hash, compute_hash
 from pypy.rlib.rarithmetic import intmask
 from pypy.module.micronumpy.interp_iter import ViewIterator, ArrayIterator, \
-     OneDimIterator, ConstantIterator
-from pypy.module.micronumpy.strides import calculate_slice_strides
+     ConstantIterator, AxisIterator, ViewTransform,\
+     BroadcastTransform
 from pypy.rlib.jit import hint, unroll_safe, promote
 
+""" Signature specifies both the numpy expression that has been constructed
+and the assembler to be compiled. This is a very important observation -
+Two expressions will be using the same assembler if and only if they are
+compiled to the same signature.
+
+This is also a very convinient tool for specializations. For example
+a + a and a + b (where a != b) will compile to different assembler because
+we specialize on the same array access.
+
+When evaluating, signatures will create iterators per signature node,
+potentially sharing some of them. Iterators depend also on the actual
+expression, they're not only dependant on the array itself. For example
+a + b where a is dim 2 and b is dim 1 would create a broadcasted iterator for
+the array b.
+
+Such iterator changes are called Transformations. An actual iterator would
+be a combination of array and various transformation, like view, broadcast,
+dimension swapping etc.
+
+See interp_iter for transformations
+"""
+
 def new_printable_location(driver_name):
     def get_printable_location(shapelen, sig):
         return 'numpy ' + sig.debug_repr() + ' [%d dims,%s]' % (shapelen, driver_name)
@@ -33,7 +55,8 @@
         return sig
 
 class NumpyEvalFrame(object):
-    _virtualizable2_ = ['iterators[*]', 'final_iter', 'arraylist[*]']
+    _virtualizable2_ = ['iterators[*]', 'final_iter', 'arraylist[*]',
+                        'value', 'identity']
 
     @unroll_safe
     def __init__(self, iterators, arrays):
@@ -51,7 +74,7 @@
     def done(self):
         final_iter = promote(self.final_iter)
         if final_iter < 0:
-            return False
+            assert False
         return self.iterators[final_iter].done()
 
     @unroll_safe
@@ -59,6 +82,12 @@
         for i in range(len(self.iterators)):
             self.iterators[i] = self.iterators[i].next(shapelen)
 
+    def get_final_iter(self):
+        final_iter = promote(self.final_iter)
+        if final_iter < 0:
+            assert False
+        return self.iterators[final_iter]
+
 def _add_ptr_to_cache(ptr, cache):
     i = 0
     for p in cache:
@@ -70,6 +99,9 @@
         cache.append(ptr)
         return res
 
+def new_cache():
+    return r_dict(sigeq_no_numbering, sighash)
+
 class Signature(object):
     _attrs_ = ['iter_no', 'array_no']
     _immutable_fields_ = ['iter_no', 'array_no']
@@ -78,7 +110,7 @@
     iter_no = 0
 
     def invent_numbering(self):
-        cache = r_dict(sigeq_no_numbering, sighash)
+        cache = new_cache()
         allnumbers = []
         self._invent_numbering(cache, allnumbers)
 
@@ -95,13 +127,13 @@
             allnumbers.append(no)
         self.iter_no = no
 
-    def create_frame(self, arr, res_shape=None):
-        res_shape = res_shape or arr.shape
+    def create_frame(self, arr):
         iterlist = []
         arraylist = []
-        self._create_iter(iterlist, arraylist, arr, res_shape, [])
+        self._create_iter(iterlist, arraylist, arr, [])
         return NumpyEvalFrame(iterlist, arraylist)
 
+
 class ConcreteSignature(Signature):
     _immutable_fields_ = ['dtype']
 
@@ -120,16 +152,6 @@
     def hash(self):
         return compute_identity_hash(self.dtype)
 
-    def allocate_view_iter(self, arr, res_shape, chunklist):
-        r = arr.shape, arr.start, arr.strides, arr.backstrides
-        if chunklist:
-            for chunkelem in chunklist:
-                r = calculate_slice_strides(r[0], r[1], r[2], r[3], chunkelem)
-        shape, start, strides, backstrides = r
-        if len(res_shape) == 1:
-            return OneDimIterator(start, strides[0], res_shape[0])
-        return ViewIterator(start, strides, backstrides, shape, res_shape)
-
 class ArraySignature(ConcreteSignature):
     def debug_repr(self):
         return 'Array'
@@ -141,22 +163,21 @@
         # is not of a concrete class it means that we have a _forced_result,
         # otherwise the signature would not match
         assert isinstance(concr, ConcreteArray)
+        assert concr.dtype is self.dtype
         self.array_no = _add_ptr_to_cache(concr.storage, cache)
 
-    def _create_iter(self, iterlist, arraylist, arr, res_shape, chunklist):
+    def _create_iter(self, iterlist, arraylist, arr, transforms):
         from pypy.module.micronumpy.interp_numarray import ConcreteArray
         concr = arr.get_concrete()
         assert isinstance(concr, ConcreteArray)
         storage = concr.storage
         if self.iter_no >= len(iterlist):
-            iterlist.append(self.allocate_iter(concr, res_shape, chunklist))
+            iterlist.append(self.allocate_iter(concr, transforms))
         if self.array_no >= len(arraylist):
             arraylist.append(storage)
 
-    def allocate_iter(self, arr, res_shape, chunklist):
-        if chunklist:
-            return self.allocate_view_iter(arr, res_shape, chunklist)
-        return ArrayIterator(arr.size)
+    def allocate_iter(self, arr, transforms):
+        return ArrayIterator(arr.size).apply_transformations(arr, transforms)
 
     def eval(self, frame, arr):
         iter = frame.iterators[self.iter_no]
@@ -169,7 +190,7 @@
     def _invent_array_numbering(self, arr, cache):
         pass
 
-    def _create_iter(self, iterlist, arraylist, arr, res_shape, chunklist):
+    def _create_iter(self, iterlist, arraylist, arr, transforms):
         if self.iter_no >= len(iterlist):
             iter = ConstantIterator()
             iterlist.append(iter)
@@ -189,8 +210,9 @@
         allnumbers.append(no)
         self.iter_no = no
 
-    def allocate_iter(self, arr, res_shape, chunklist):
-        return self.allocate_view_iter(arr, res_shape, chunklist)
+    def allocate_iter(self, arr, transforms):
+        return ViewIterator(arr.start, arr.strides, arr.backstrides,
+                            arr.shape).apply_transformations(arr, transforms)
 
 class VirtualSliceSignature(Signature):
     def __init__(self, child):
@@ -201,6 +223,9 @@
         assert isinstance(arr, VirtualSlice)
         self.child._invent_array_numbering(arr.child, cache)
 
+    def _invent_numbering(self, cache, allnumbers):
+        self.child._invent_numbering(new_cache(), allnumbers)
+
     def hash(self):
         return intmask(self.child.hash() ^ 1234)
 
@@ -210,12 +235,11 @@
         assert isinstance(other, VirtualSliceSignature)
         return self.child.eq(other.child, compare_array_no)
 
-    def _create_iter(self, iterlist, arraylist, arr, res_shape, chunklist):
+    def _create_iter(self, iterlist, arraylist, arr, transforms):
         from pypy.module.micronumpy.interp_numarray import VirtualSlice
         assert isinstance(arr, VirtualSlice)
-        chunklist.append(arr.chunks)
-        self.child._create_iter(iterlist, arraylist, arr.child, res_shape,
-                                chunklist)
+        transforms = transforms + [ViewTransform(arr.chunks)]
+        self.child._create_iter(iterlist, arraylist, arr.child, transforms)
 
     def eval(self, frame, arr):
         from pypy.module.micronumpy.interp_numarray import VirtualSlice
@@ -251,11 +275,10 @@
         assert isinstance(arr, Call1)
         self.child._invent_array_numbering(arr.values, cache)
 
-    def _create_iter(self, iterlist, arraylist, arr, res_shape, chunklist):
+    def _create_iter(self, iterlist, arraylist, arr, transforms):
         from pypy.module.micronumpy.interp_numarray import Call1
         assert isinstance(arr, Call1)
-        self.child._create_iter(iterlist, arraylist, arr.values, res_shape,
-                                chunklist)
+        self.child._create_iter(iterlist, arraylist, arr.values, transforms)
 
     def eval(self, frame, arr):
         from pypy.module.micronumpy.interp_numarray import Call1
@@ -296,29 +319,68 @@
         self.left._invent_numbering(cache, allnumbers)
         self.right._invent_numbering(cache, allnumbers)
 
-    def _create_iter(self, iterlist, arraylist, arr, res_shape, chunklist):
+    def _create_iter(self, iterlist, arraylist, arr, transforms):
         from pypy.module.micronumpy.interp_numarray import Call2
 
         assert isinstance(arr, Call2)
-        self.left._create_iter(iterlist, arraylist, arr.left, res_shape,
-                               chunklist)
-        self.right._create_iter(iterlist, arraylist, arr.right, res_shape,
-                                chunklist)
+        self.left._create_iter(iterlist, arraylist, arr.left, transforms)
+        self.right._create_iter(iterlist, arraylist, arr.right, transforms)
 
     def eval(self, frame, arr):
         from pypy.module.micronumpy.interp_numarray import Call2
         assert isinstance(arr, Call2)
         lhs = self.left.eval(frame, arr.left).convert_to(self.calc_dtype)
         rhs = self.right.eval(frame, arr.right).convert_to(self.calc_dtype)
+        
         return self.binfunc(self.calc_dtype, lhs, rhs)
 
     def debug_repr(self):
         return 'Call2(%s, %s, %s)' % (self.name, self.left.debug_repr(),
                                       self.right.debug_repr())
 
+class BroadcastLeft(Call2):
+    def _invent_numbering(self, cache, allnumbers):
+        self.left._invent_numbering(new_cache(), allnumbers)
+        self.right._invent_numbering(cache, allnumbers)
+    
+    def _create_iter(self, iterlist, arraylist, arr, transforms):
+        from pypy.module.micronumpy.interp_numarray import Call2
+
+        assert isinstance(arr, Call2)
+        ltransforms = transforms + [BroadcastTransform(arr.shape)]
+        self.left._create_iter(iterlist, arraylist, arr.left, ltransforms)
+        self.right._create_iter(iterlist, arraylist, arr.right, transforms)
+
+class BroadcastRight(Call2):
+    def _invent_numbering(self, cache, allnumbers):
+        self.left._invent_numbering(cache, allnumbers)
+        self.right._invent_numbering(new_cache(), allnumbers)
+
+    def _create_iter(self, iterlist, arraylist, arr, transforms):
+        from pypy.module.micronumpy.interp_numarray import Call2
+
+        assert isinstance(arr, Call2)
+        rtransforms = transforms + [BroadcastTransform(arr.shape)]
+        self.left._create_iter(iterlist, arraylist, arr.left, transforms)
+        self.right._create_iter(iterlist, arraylist, arr.right, rtransforms)
+
+class BroadcastBoth(Call2):
+    def _invent_numbering(self, cache, allnumbers):
+        self.left._invent_numbering(new_cache(), allnumbers)
+        self.right._invent_numbering(new_cache(), allnumbers)
+
+    def _create_iter(self, iterlist, arraylist, arr, transforms):
+        from pypy.module.micronumpy.interp_numarray import Call2
+
+        assert isinstance(arr, Call2)
+        rtransforms = transforms + [BroadcastTransform(arr.shape)]
+        ltransforms = transforms + [BroadcastTransform(arr.shape)]
+        self.left._create_iter(iterlist, arraylist, arr.left, ltransforms)
+        self.right._create_iter(iterlist, arraylist, arr.right, rtransforms)
+
 class ReduceSignature(Call2):
-    def _create_iter(self, iterlist, arraylist, arr, res_shape, chunklist):
-        self.right._create_iter(iterlist, arraylist, arr, res_shape, chunklist)
+    def _create_iter(self, iterlist, arraylist, arr, transforms):
+        self.right._create_iter(iterlist, arraylist, arr, transforms)
 
     def _invent_numbering(self, cache, allnumbers):
         self.right._invent_numbering(cache, allnumbers)
@@ -328,3 +390,63 @@
 
     def eval(self, frame, arr):
         return self.right.eval(frame, arr)
+
+    def debug_repr(self):
+        return 'ReduceSig(%s, %s)' % (self.name, self.right.debug_repr())
+
+class SliceloopSignature(Call2):
+    def eval(self, frame, arr):
+        from pypy.module.micronumpy.interp_numarray import Call2
+        
+        assert isinstance(arr, Call2)
+        ofs = frame.iterators[0].offset
+        arr.left.setitem(ofs, self.right.eval(frame, arr.right).convert_to(
+            self.calc_dtype))
+    
+    def debug_repr(self):
+        return 'SliceLoop(%s, %s, %s)' % (self.name, self.left.debug_repr(),
+                                          self.right.debug_repr())
+
+class SliceloopBroadcastSignature(SliceloopSignature):
+    def _invent_numbering(self, cache, allnumbers):
+        self.left._invent_numbering(new_cache(), allnumbers)
+        self.right._invent_numbering(cache, allnumbers)
+
+    def _create_iter(self, iterlist, arraylist, arr, transforms):
+        from pypy.module.micronumpy.interp_numarray import SliceArray
+
+        assert isinstance(arr, SliceArray)
+        rtransforms = transforms + [BroadcastTransform(arr.shape)]
+        self.left._create_iter(iterlist, arraylist, arr.left, transforms)
+        self.right._create_iter(iterlist, arraylist, arr.right, rtransforms)
+
+class AxisReduceSignature(Call2):
+    def _create_iter(self, iterlist, arraylist, arr, transforms):
+        from pypy.module.micronumpy.interp_numarray import AxisReduce,\
+             ConcreteArray
+
+        assert isinstance(arr, AxisReduce)
+        left = arr.left
+        assert isinstance(left, ConcreteArray)
+        iterlist.append(AxisIterator(left.start, arr.dim, arr.shape,
+                                     left.strides, left.backstrides))
+        self.right._create_iter(iterlist, arraylist, arr.right, transforms)
+
+    def _invent_numbering(self, cache, allnumbers):
+        allnumbers.append(0)
+        self.right._invent_numbering(cache, allnumbers)
+
+    def _invent_array_numbering(self, arr, cache):
+        from pypy.module.micronumpy.interp_numarray import AxisReduce
+
+        assert isinstance(arr, AxisReduce)
+        self.right._invent_array_numbering(arr.right, cache)
+
+    def eval(self, frame, arr):
+        from pypy.module.micronumpy.interp_numarray import AxisReduce
+
+        assert isinstance(arr, AxisReduce)
+        return self.right.eval(frame, arr.right).convert_to(self.calc_dtype)
+    
+    def debug_repr(self):
+        return 'AxisReduceSig(%s, %s)' % (self.name, self.right.debug_repr())
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -166,14 +166,11 @@
         # You can't subclass dtype
         raises(TypeError, type, "Foo", (dtype,), {})
 
-    def test_new(self):
-        import _numpypy as np
-        assert np.int_(4) == 4
-        assert np.float_(3.4) == 3.4
+    def test_aliases(self):
+        from _numpypy import dtype
 
-    def test_pow(self):
-        from _numpypy import int_
-        assert int_(4) ** 2 == 16
+        assert dtype("float") is dtype(float)
+
 
 class AppTestTypes(BaseNumpyAppTest):
     def test_abstract_types(self):
@@ -189,6 +186,15 @@
         raises(TypeError, numpy.floating, 0)
         raises(TypeError, numpy.inexact, 0)
 
+    def test_new(self):
+        import _numpypy as np
+        assert np.int_(4) == 4
+        assert np.float_(3.4) == 3.4
+
+    def test_pow(self):
+        from _numpypy import int_
+        assert int_(4) ** 2 == 16
+
     def test_bool(self):
         import _numpypy as numpy
 
@@ -318,7 +324,7 @@
         else:
             raises(OverflowError, numpy.int64, 9223372036854775807)
             raises(OverflowError, numpy.int64, '9223372036854775807')
-        
+
         raises(OverflowError, numpy.int64, 9223372036854775808)
         raises(OverflowError, numpy.int64, '9223372036854775808')
 
diff --git a/pypy/module/micronumpy/test/test_module.py b/pypy/module/micronumpy/test/test_module.py
--- a/pypy/module/micronumpy/test/test_module.py
+++ b/pypy/module/micronumpy/test/test_module.py
@@ -2,16 +2,11 @@
 
 
 class AppTestNumPyModule(BaseNumpyAppTest):
-    def test_mean(self):
-        from _numpypy import array, mean
-        assert mean(array(range(5))) == 2.0
-        assert mean(range(5)) == 2.0
-
     def test_average(self):
         from _numpypy import array, average
         assert average(range(10)) == 4.5
         assert average(array(range(10))) == 4.5
-        
+
     def test_sum(self):
         from _numpypy import array, sum
         assert sum(range(10)) == 45
@@ -21,7 +16,7 @@
         from _numpypy import array, min
         assert min(range(10)) == 0
         assert min(array(range(10))) == 0
-        
+
     def test_max(self):
         from _numpypy import array, max
         assert max(range(10)) == 9
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -157,6 +157,8 @@
         assert calc_new_strides([2, 3, 4], [8, 3], [1, 16]) is None
         assert calc_new_strides([24], [2, 4, 3], [48, 6, 1]) is None
         assert calc_new_strides([24], [2, 4, 3], [24, 6, 2]) == [2]
+        assert calc_new_strides([105, 1], [3, 5, 7], [35, 7, 1]) == [1, 1]
+        assert calc_new_strides([1, 105], [3, 5, 7], [35, 7, 1]) == [105, 1]
 
 
 class AppTestNumArray(BaseNumpyAppTest):
@@ -246,6 +248,10 @@
         c = b.copy()
         assert (c == b).all()
 
+        a = arange(15).reshape(5,3)
+        b = a.copy()
+        assert (b == a).all()
+
     def test_iterator_init(self):
         from _numpypy import array
         a = array(range(5))
@@ -724,6 +730,12 @@
         a = array(range(5))
         assert a.mean() == 2.0
         assert a[:4].mean() == 1.5
+        a = array(range(105)).reshape(3, 5, 7)
+        b = a.mean(axis=0)
+        b[0, 0]==35.
+        assert a.mean(axis=0)[0, 0] == 35
+        assert (b == array(range(35, 70), dtype=float).reshape(5, 7)).all()
+        assert (a.mean(2) == array(range(0, 15), dtype=float).reshape(3, 5) * 7 + 3).all()
 
     def test_sum(self):
         from _numpypy import array
@@ -734,6 +746,34 @@
         a = array([True] * 5, bool)
         assert a.sum() == 5
 
+        raises(TypeError, 'a.sum(2, 3)')
+
+    def test_reduce_nd(self):
+        from numpypy import arange, array, multiply
+        a = arange(15).reshape(5, 3)
+        assert a.sum() == 105
+        assert a.max() == 14
+        assert array([]).sum() == 0.0
+        raises(ValueError, 'array([]).max()')
+        assert (a.sum(0) == [30, 35, 40]).all()
+        assert (a.sum(axis=0) == [30, 35, 40]).all()
+        assert (a.sum(1) == [3, 12, 21, 30, 39]).all()
+        assert (a.max(0) == [12, 13, 14]).all()
+        assert (a.max(1) == [2, 5, 8, 11, 14]).all()
+        assert ((a + a).max() == 28)
+        assert ((a + a).max(0) == [24, 26, 28]).all()
+        assert ((a + a).sum(1) == [6, 24, 42, 60, 78]).all()
+        assert (multiply.reduce(a) == array([0, 3640, 12320])).all()
+        a = array(range(105)).reshape(3, 5, 7)
+        assert (a[:, 1, :].sum(0) == [126, 129, 132, 135, 138, 141, 144]).all()
+        assert (a[:, 1, :].sum(1) == [70, 315, 560]).all()
+        raises (ValueError, 'a[:, 1, :].sum(2)')
+        assert ((a + a).T.sum(2).T == (a + a).sum(0)).all()
+        assert (a.reshape(1,-1).sum(0) == range(105)).all()
+        assert (a.reshape(1,-1).sum(1) == 5460)
+        assert (array([[1,2],[3,4]]).prod(0) == [3, 8]).all()
+        assert (array([[1,2],[3,4]]).prod(1) == [2, 12]).all()
+
     def test_identity(self):
         from _numpypy import identity, array
         from _numpypy import int32, float64, dtype
@@ -1262,6 +1302,28 @@
         assert isinstance(i['data'][0], int)
         raises(TypeError, getattr, array(3), '__array_interface__')
 
+    def test_fill(self):
+        from _numpypy import array
+
+        a = array([1, 2, 3])
+        a.fill(10)
+        assert (a == [10, 10, 10]).all()
+        a.fill(False)
+        assert (a == [0, 0, 0]).all()
+
+        b = a[:1]
+        b.fill(4)
+        assert (b == [4]).all()
+        assert (a == [4, 0, 0]).all()
+
+        c = b + b
+        c.fill(27)
+        assert (c == [27]).all()
+
+        d = array(10)
+        d.fill(100)
+        assert d == 100
+
 
 class AppTestSupport(BaseNumpyAppTest):
     def setup_class(cls):
@@ -1401,9 +1463,11 @@
         assert repr(a) == "array(0.0)"
         a = array(0.2)
         assert repr(a) == "array(0.2)"
+        a = array([2])
+        assert repr(a) == "array([2])"
 
     def test_repr_multi(self):
-        from _numpypy import arange, zeros
+        from _numpypy import arange, zeros, array
         a = zeros((3, 4))
         assert repr(a) == '''array([[0.0, 0.0, 0.0, 0.0],
        [0.0, 0.0, 0.0, 0.0],
@@ -1426,6 +1490,9 @@
        [498, 999],
        [499, 1000],
        [500, 1001]])'''
+        a = arange(2).reshape((2,1))
+        assert repr(a) == '''array([[0],
+       [1]])'''
 
     def test_repr_slice(self):
         from _numpypy import array, zeros
@@ -1510,14 +1577,3 @@
         a = arange(0, 0.8, 0.1)
         assert len(a) == 8
         assert arange(False, True, True).dtype is dtype(int)
-
-
-class AppTestRanges(BaseNumpyAppTest):
-    def test_app_reshape(self):
-        from _numpypy import arange, array, dtype, reshape
-        a = arange(12)
-        b = reshape(a, (3, 4))
-        assert b.shape == (3, 4)
-        a = range(12)
-        b = reshape(a, (3, 4))
-        assert b.shape == (3, 4)
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -190,14 +190,24 @@
         for i in range(3):
             assert c[i] == a[i] - b[i]
 
-    def test_floor(self):
-        from _numpypy import array, floor
-
-        reference = [-2.0, -1.0, 0.0, 1.0, 1.0]
-        a = array([-1.4, -1.0, 0.0, 1.0, 1.4])
+    def test_floorceil(self):
+        from _numpypy import array, floor, ceil
+        import math
+        reference = [-2.0, -2.0, -1.0, 0.0, 1.0, 1.0, 0]
+        a = array([-1.4, -1.5, -1.0, 0.0, 1.0, 1.4, 0.5])
         b = floor(a)
         for i in range(5):
             assert b[i] == reference[i]
+        reference = [-1.0, -1.0, -1.0, 0.0, 1.0, 2.0, 1.0]
+        a = array([-1.4, -1.5, -1.0, 0.0, 1.0, 1.4, 0.5])
+        b = ceil(a)
+        assert (reference == b).all()
+        inf = float("inf")
+        data = [1.5, 2.9999, -1.999, inf]
+        results = [math.floor(x) for x in data]
+        assert (floor(data) == results).all()
+        results = [math.ceil(x) for x in data]
+        assert (ceil(data) == results).all()
 
     def test_copysign(self):
         from _numpypy import array, copysign
@@ -238,7 +248,7 @@
             assert b[i] == math.sin(a[i])
 
         a = sin(array([True, False], dtype=bool))
-        assert abs(a[0] - sin(1)) < 1e-7 # a[0] will be less precise
+        assert abs(a[0] - sin(1)) < 1e-7  # a[0] will be less precise
         assert a[1] == 0.0
 
     def test_cos(self):
@@ -259,7 +269,6 @@
         for i in range(len(a)):
             assert b[i] == math.tan(a[i])
 
-
     def test_arcsin(self):
         import math
         from _numpypy import array, arcsin
@@ -283,7 +292,6 @@
         for i in range(len(a)):
             assert b[i] == math.acos(a[i])
 
-
         a = array([-10, -1.5, -1.01, 1.01, 1.5, 10, float('nan'), float('inf'), float('-inf')])
         b = arccos(a)
         for f in b:
@@ -298,7 +306,7 @@
         for i in range(len(a)):
             assert b[i] == math.atan(a[i])
 
-        a  = array([float('nan')])
+        a = array([float('nan')])
         b = arctan(a)
         assert math.isnan(b[0])
 
@@ -336,9 +344,9 @@
         from _numpypy import sin, add
 
         raises(ValueError, sin.reduce, [1, 2, 3])
-        raises(TypeError, add.reduce, 1)
+        raises(ValueError, add.reduce, 1)
 
-    def test_reduce(self):
+    def test_reduce_1d(self):
         from _numpypy import add, maximum
 
         assert add.reduce([1, 2, 3]) == 6
@@ -346,6 +354,12 @@
         assert maximum.reduce([1, 2, 3]) == 3
         raises(ValueError, maximum.reduce, [])
 
+    def test_reduceND(self):
+        from numpypy import add, arange
+        a = arange(12).reshape(3, 4)
+        assert (add.reduce(a, 0) == [12, 15, 18, 21]).all()
+        assert (add.reduce(a, 1) == [6.0, 22.0, 38.0]).all()
+
     def test_comparisons(self):
         import operator
         from _numpypy import equal, not_equal, less, less_equal, greater, greater_equal
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -47,6 +47,8 @@
         def f(i):
             interp = InterpreterState(codes[i])
             interp.run(space)
+            if not len(interp.results):
+                raise Exception("need results")
             w_res = interp.results[-1]
             if isinstance(w_res, BaseArray):
                 concr = w_res.get_concrete_or_scalar()
@@ -115,6 +117,28 @@
                                 "int_add": 1, "int_ge": 1, "guard_false": 1,
                                 "jump": 1, 'arraylen_gc': 1})
 
+    def define_axissum():
+        return """
+        a = [[1, 2], [3, 4], [5, 6], [7, 8], [9, 10]]
+        b = sum(a,0)
+        b -> 1
+        """
+
+    def test_axissum(self):
+        result = self.run("axissum")
+        assert result == 30
+        # XXX note - the bridge here is fairly crucial and yet it's pretty
+        #            bogus. We need to improve the situation somehow.
+        self.check_simple_loop({'getinteriorfield_raw': 2,
+                                'setinteriorfield_raw': 1,
+                                'arraylen_gc': 1,
+                                'guard_true': 1,
+                                'int_lt': 1,
+                                'jump': 1,
+                                'float_add': 1,
+                                'int_add': 3,
+                                })
+
     def define_prod():
         return """
         a = |30|
@@ -193,9 +217,9 @@
         # This is the sum of the ops for both loops, however if you remove the
         # optimization then you end up with 2 float_adds, so we can still be
         # sure it was optimized correctly.
-        self.check_resops({'setinteriorfield_raw': 4, 'getfield_gc': 26,
+        self.check_resops({'setinteriorfield_raw': 4, 'getfield_gc': 22,
                            'getarrayitem_gc': 4, 'getarrayitem_gc_pure': 2,
-                           'getfield_gc_pure': 4,
+                           'getfield_gc_pure': 8,
                            'guard_class': 8, 'int_add': 8, 'float_mul': 2,
                            'jump': 2, 'int_ge': 4,
                            'getinteriorfield_raw': 4, 'float_add': 2,
@@ -212,7 +236,8 @@
     def test_ufunc(self):
         result = self.run("ufunc")
         assert result == -6
-        self.check_simple_loop({"getinteriorfield_raw": 2, "float_add": 1, "float_neg": 1,
+        self.check_simple_loop({"getinteriorfield_raw": 2, "float_add": 1,
+                                "float_neg": 1,
                                 "setinteriorfield_raw": 1, "int_add": 2,
                                 "int_ge": 1, "guard_false": 1, "jump": 1,
                                 'arraylen_gc': 1})
@@ -322,10 +347,10 @@
         result = self.run("setslice")
         assert result == 11.0
         self.check_trace_count(1)
-        self.check_simple_loop({'getinteriorfield_raw': 2, 'float_add' : 1,
-                                'setinteriorfield_raw': 1, 'int_add': 3,
-                                'int_lt': 1, 'guard_true': 1, 'jump': 1,
-                                'arraylen_gc': 3})
+        self.check_simple_loop({'getinteriorfield_raw': 2, 'float_add': 1,
+                                'setinteriorfield_raw': 1, 'int_add': 2,
+                                'int_eq': 1, 'guard_false': 1, 'jump': 1,
+                                'arraylen_gc': 1})
 
     def define_virtual_slice():
         return """
@@ -339,11 +364,12 @@
         result = self.run("virtual_slice")
         assert result == 4
         self.check_trace_count(1)
-        self.check_simple_loop({'getinteriorfield_raw': 2, 'float_add' : 1,
+        self.check_simple_loop({'getinteriorfield_raw': 2, 'float_add': 1,
                                 'setinteriorfield_raw': 1, 'int_add': 2,
                                 'int_ge': 1, 'guard_false': 1, 'jump': 1,
                                 'arraylen_gc': 1})
 
+
 class TestNumpyOld(LLJitMixin):
     def setup_class(cls):
         py.test.skip("old")
@@ -377,4 +403,3 @@
 
         result = self.meta_interp(f, [5], listops=True, backendopt=True)
         assert result == f(5)
-
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -374,6 +374,10 @@
         return math.floor(v)
 
     @simple_unary_op
+    def ceil(self, v):
+        return math.ceil(v)
+
+    @simple_unary_op
     def exp(self, v):
         try:
             return math.exp(v)
@@ -436,4 +440,4 @@
 class Float64(BaseType, Float):
     T = rffi.DOUBLE
     BoxType = interp_boxes.W_Float64Box
-    format_code = "d"
\ No newline at end of file
+    format_code = "d"
diff --git a/pypy/module/pypyjit/__init__.py b/pypy/module/pypyjit/__init__.py
--- a/pypy/module/pypyjit/__init__.py
+++ b/pypy/module/pypyjit/__init__.py
@@ -11,6 +11,7 @@
         'set_optimize_hook': 'interp_resop.set_optimize_hook',
         'set_abort_hook': 'interp_resop.set_abort_hook',
         'ResOperation': 'interp_resop.WrappedOp',
+        'DebugMergePoint': 'interp_resop.DebugMergePoint',
         'Box': 'interp_resop.WrappedBox',
     }
 
diff --git a/pypy/module/pypyjit/interp_resop.py b/pypy/module/pypyjit/interp_resop.py
--- a/pypy/module/pypyjit/interp_resop.py
+++ b/pypy/module/pypyjit/interp_resop.py
@@ -1,5 +1,6 @@
 
-from pypy.interpreter.typedef import TypeDef, GetSetProperty
+from pypy.interpreter.typedef import (TypeDef, GetSetProperty,
+     interp_attrproperty, interp_attrproperty_w)
 from pypy.interpreter.baseobjspace import Wrappable
 from pypy.interpreter.gateway import unwrap_spec, interp2app, NoneNotWrapped
 from pypy.interpreter.pycode import PyCode
@@ -10,6 +11,7 @@
 from pypy.jit.metainterp.resoperation import rop, AbstractResOp
 from pypy.rlib.nonconst import NonConstant
 from pypy.rlib import jit_hooks
+from pypy.module.pypyjit.interp_jit import pypyjitdriver
 
 class Cache(object):
     in_recursion = False
@@ -111,13 +113,24 @@
 
 def wrap_oplist(space, logops, operations, ops_offset=None):
     l_w = []
+    jitdrivers_sd = logops.metainterp_sd.jitdrivers_sd
     for op in operations:
         if ops_offset is None:
             ofs = -1
         else:
             ofs = ops_offset.get(op, 0)
-        l_w.append(WrappedOp(jit_hooks._cast_to_gcref(op), ofs,
-                             logops.repr_of_resop(op)))
+        if op.opnum == rop.DEBUG_MERGE_POINT:
+            jd_sd = jitdrivers_sd[op.getarg(0).getint()]
+            greenkey = op.getarglist()[2:]
+            repr = jd_sd.warmstate.get_location_str(greenkey)
+            w_greenkey = wrap_greenkey(space, jd_sd.jitdriver, greenkey, repr)
+            l_w.append(DebugMergePoint(space, jit_hooks._cast_to_gcref(op),
+                                       logops.repr_of_resop(op),
+                                       jd_sd.jitdriver.name,
+                                       w_greenkey))
+        else:
+            l_w.append(WrappedOp(jit_hooks._cast_to_gcref(op), ofs,
+                                 logops.repr_of_resop(op)))
     return l_w
 
 class WrappedBox(Wrappable):
@@ -150,6 +163,15 @@
         llres = res.llbox
     return WrappedOp(jit_hooks.resop_new(num, args, llres), offset, repr)
 
+ at unwrap_spec(repr=str, jd_name=str)
+def descr_new_dmp(space, w_tp, w_args, repr, jd_name, w_greenkey):
+    args = [space.interp_w(WrappedBox, w_arg).llbox for w_arg in
+            space.listview(w_args)]
+    num = rop.DEBUG_MERGE_POINT
+    return DebugMergePoint(space,
+                           jit_hooks.resop_new(num, args, jit_hooks.emptyval()),
+                           repr, jd_name, w_greenkey)
+
 class WrappedOp(Wrappable):
     """ A class representing a single ResOperation, wrapped nicely
     """
@@ -182,6 +204,25 @@
         box = space.interp_w(WrappedBox, w_box)
         jit_hooks.resop_setresult(self.op, box.llbox)
 
+class DebugMergePoint(WrappedOp):
+    def __init__(self, space, op, repr_of_resop, jd_name, w_greenkey):
+        WrappedOp.__init__(self, op, -1, repr_of_resop)
+        self.w_greenkey = w_greenkey
+        self.jd_name = jd_name
+
+    def get_pycode(self, space):
+        if self.jd_name == pypyjitdriver.name:
+            return space.getitem(self.w_greenkey, space.wrap(0))
+        raise OperationError(space.w_AttributeError, space.wrap("This DebugMergePoint doesn't belong to the main Python JitDriver"))
+
+    def get_bytecode_no(self, space):
+        if self.jd_name == pypyjitdriver.name:
+            return space.getitem(self.w_greenkey, space.wrap(1))
+        raise OperationError(space.w_AttributeError, space.wrap("This DebugMergePoint doesn't belong to the main Python JitDriver"))
+
+    def get_jitdriver_name(self, space):
+        return space.wrap(self.jd_name)
+
 WrappedOp.typedef = TypeDef(
     'ResOperation',
     __doc__ = WrappedOp.__doc__,
@@ -195,3 +236,15 @@
                             WrappedOp.descr_setresult)
 )
 WrappedOp.acceptable_as_base_class = False
+
+DebugMergePoint.typedef = TypeDef(
+    'DebugMergePoint', WrappedOp.typedef,
+    __new__ = interp2app(descr_new_dmp),
+    greenkey = interp_attrproperty_w("w_greenkey", cls=DebugMergePoint),
+    pycode = GetSetProperty(DebugMergePoint.get_pycode),
+    bytecode_no = GetSetProperty(DebugMergePoint.get_bytecode_no),
+    jitdriver_name = GetSetProperty(DebugMergePoint.get_jitdriver_name),
+)
+DebugMergePoint.acceptable_as_base_class = False
+
+
diff --git a/pypy/module/pypyjit/policy.py b/pypy/module/pypyjit/policy.py
--- a/pypy/module/pypyjit/policy.py
+++ b/pypy/module/pypyjit/policy.py
@@ -127,7 +127,7 @@
                        'imp', 'sys', 'array', '_ffi', 'itertools', 'operator',
                        'posix', '_socket', '_sre', '_lsprof', '_weakref',
                        '__pypy__', 'cStringIO', '_collections', 'struct',
-                       'mmap', 'marshal']:
+                       'mmap', 'marshal', '_codecs']:
             if modname == 'pypyjit' and 'interp_resop' in rest:
                 return False
             return True
diff --git a/pypy/module/pypyjit/test/test_jit_hook.py b/pypy/module/pypyjit/test/test_jit_hook.py
--- a/pypy/module/pypyjit/test/test_jit_hook.py
+++ b/pypy/module/pypyjit/test/test_jit_hook.py
@@ -92,6 +92,7 @@
         cls.w_on_compile_bridge = space.wrap(interp2app(interp_on_compile_bridge))
         cls.w_on_abort = space.wrap(interp2app(interp_on_abort))
         cls.w_int_add_num = space.wrap(rop.INT_ADD)
+        cls.w_dmp_num = space.wrap(rop.DEBUG_MERGE_POINT)
         cls.w_on_optimize = space.wrap(interp2app(interp_on_optimize))
         cls.orig_oplist = oplist
 
@@ -117,6 +118,10 @@
         assert elem[2][2] == False
         assert len(elem[3]) == 4
         int_add = elem[3][0]
+        dmp = elem[3][1]
+        assert isinstance(dmp, pypyjit.DebugMergePoint)
+        assert dmp.pycode is self.f.func_code
+        assert dmp.greenkey == (self.f.func_code, 0, False)
         #assert int_add.name == 'int_add'
         assert int_add.num == self.int_add_num
         self.on_compile_bridge()
@@ -211,3 +216,18 @@
         assert op.getarg(0).getint() == 4
         op.result = box
         assert op.result.getint() == 1
+
+    def test_creation_dmp(self):
+        from pypyjit import DebugMergePoint, Box
+
+        def f():
+            pass
+
+        op = DebugMergePoint([Box(0)], 'repr', 'pypyjit', (f.func_code, 0, 0))
+        assert op.bytecode_no == 0
+        assert op.pycode is f.func_code
+        assert repr(op) == 'repr'
+        assert op.jitdriver_name == 'pypyjit'
+        assert op.num == self.dmp_num
+        op = DebugMergePoint([Box(0)], 'repr', 'notmain', ('str',))
+        raises(AttributeError, 'op.pycode')
diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py
--- a/pypy/module/sys/version.py
+++ b/pypy/module/sys/version.py
@@ -10,7 +10,7 @@
 CPYTHON_VERSION            = (2, 7, 1, "final", 42)   #XXX # sync patchlevel.h
 CPYTHON_API_VERSION        = 1013   #XXX # sync with include/modsupport.h
 
-PYPY_VERSION               = (1, 7, 1, "dev", 0)    #XXX # sync patchlevel.h
+PYPY_VERSION               = (1, 8, 1, "dev", 0)    #XXX # sync patchlevel.h
 
 if platform.name == 'msvc':
     COMPILER_INFO = 'MSC v.%d 32 bit' % (platform.version * 10 + 600)
diff --git a/lib_pypy/numpypy/test/test_fromnumeric.py b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
rename from lib_pypy/numpypy/test/test_fromnumeric.py
rename to pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
--- a/lib_pypy/numpypy/test/test_fromnumeric.py
+++ b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
@@ -1,7 +1,7 @@
-
 from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
 
-class AppTestFromNumeric(BaseNumpyAppTest):     
+
+class AppTestFromNumeric(BaseNumpyAppTest):
     def test_argmax(self):
         # tests taken from numpy/core/fromnumeric.py docstring
         from numpypy import array, arange, argmax
@@ -18,12 +18,12 @@
         from numpypy import array, arange, argmin
         a = arange(6).reshape((2,3))
         assert argmin(a) == 0
-        # assert (argmax(a, axis=0) == array([0, 0, 0])).all()
-        # assert (argmax(a, axis=1) == array([0, 0])).all()
+        assert (argmin(a, axis=0) == array([0, 0, 0])).all()
+        assert (argmin(a, axis=1) == array([0, 0])).all()
         b = arange(6)
         b[1] = 0
         assert argmin(b) == 0
-   
+
     def test_shape(self):
         # tests taken from numpy/core/fromnumeric.py docstring
         from numpypy import array, identity, shape
@@ -44,7 +44,7 @@
         # assert (sum([[0, 1], [0, 5]], axis=1) == array([1, 5])).all()
         # If the accumulator is too small, overflow occurs:
         # assert ones(128, dtype=int8).sum(dtype=int8) == -128
-                                 
+
     def test_amin(self):
         # tests taken from numpy/core/fromnumeric.py docstring
         from numpypy import array, arange, amin
@@ -86,14 +86,14 @@
         assert ndim([[1,2,3],[4,5,6]]) == 2
         assert ndim(array([[1,2,3],[4,5,6]])) == 2
         assert ndim(1) == 0
-    
+
     def test_rank(self):
         # tests taken from numpy/core/fromnumeric.py docstring
         from numpypy import array, rank
         assert rank([[1,2,3],[4,5,6]]) == 2
         assert rank(array([[1,2,3],[4,5,6]])) == 2
         assert rank(1) == 0
-    
+
     def test_var(self):
         from numpypy import array, var
         a = array([[1,2],[3,4]])
@@ -107,3 +107,31 @@
         assert std(a) ==  1.1180339887498949
         # assert (std(a, axis=0) == array([ 1.,  1.])).all()
         # assert (std(a, axis=1) == array([ 0.5,  0.5]).all()
+
+    def test_mean(self):
+        from numpypy import array, mean
+        assert mean(array(range(5))) == 2.0
+        assert mean(range(5)) == 2.0
+
+    def test_reshape(self):
+        from numpypy import arange, array, dtype, reshape
+        a = arange(12)
+        b = reshape(a, (3, 4))
+        assert b.shape == (3, 4)
+        a = range(12)
+        b = reshape(a, (3, 4))
+        assert b.shape == (3, 4)
+        a = array(range(105)).reshape(3, 5, 7)
+        assert reshape(a, (1, -1)).shape == (1, 105)
+        assert reshape(a, (1, 1, -1)).shape == (1, 1, 105)
+        assert reshape(a, (-1, 1, 1)).shape == (105, 1, 1)
+    
+    def test_transpose(self):   
+        from numpypy import arange, array, transpose, ones
+        x = arange(4).reshape((2,2))
+        assert (transpose(x) == array([[0, 2],[1, 3]])).all()
+        # Once axes argument is implemented, add more tests
+        raises(NotImplementedError, "transpose(x, axes=(1, 0, 2))")
+        # x = ones((1, 2, 3))
+        # assert transpose(x, (1, 0, 2)).shape == (2, 1, 3)
+
diff --git a/pypy/objspace/std/test/test_unicodeobject.py b/pypy/objspace/std/test/test_unicodeobject.py
--- a/pypy/objspace/std/test/test_unicodeobject.py
+++ b/pypy/objspace/std/test/test_unicodeobject.py
@@ -64,6 +64,12 @@
         check(', '.join([u'a']), u'a')
         check(', '.join(['a', u'b']), u'a, b')
         check(u', '.join(['a', 'b']), u'a, b')
+        try:
+            u''.join([u'a', 2, 3])
+        except TypeError, e:
+            assert 'sequence item 1' in str(e)
+        else:
+            raise Exception("DID NOT RAISE")
 
     if sys.version_info >= (2,3):
         def test_contains_ex(self):
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -201,7 +201,7 @@
     return space.newbool(container.find(item) != -1)
 
 def unicode_join__Unicode_ANY(space, w_self, w_list):
-    list_w = space.unpackiterable(w_list)
+    list_w = space.listview(w_list)
     size = len(list_w)
 
     if size == 0:
@@ -216,22 +216,21 @@
 
 def _unicode_join_many_items(space, w_self, list_w, size):
     self = w_self._value
-    sb = UnicodeBuilder()
+    prealloc_size = len(self) * (size - 1)
+    for i in range(size):
+        try:
+            prealloc_size += len(space.unicode_w(list_w[i]))
+        except OperationError, e:
+            if not e.match(space, space.w_TypeError):
+                raise
+            raise operationerrfmt(space.w_TypeError,
+                        "sequence item %d: expected string or Unicode", i)
+    sb = UnicodeBuilder(prealloc_size)
     for i in range(size):
         if self and i != 0:
             sb.append(self)
         w_s = list_w[i]
-        if isinstance(w_s, W_UnicodeObject):
-            # shortcut for performance
-            sb.append(w_s._value)
-        else:
-            try:
-                sb.append(space.unicode_w(w_s))
-            except OperationError, e:
-                if not e.match(space, space.w_TypeError):
-                    raise
-                raise operationerrfmt(space.w_TypeError,
-                    "sequence item %d: expected string or Unicode", i)
+        sb.append(space.unicode_w(w_s))
     return space.wrap(sb.build())
 
 def hash__Unicode(space, w_uni):
diff --git a/pypy/rlib/jit_hooks.py b/pypy/rlib/jit_hooks.py
--- a/pypy/rlib/jit_hooks.py
+++ b/pypy/rlib/jit_hooks.py
@@ -52,7 +52,10 @@
     from pypy.jit.metainterp.history import ResOperation
 
     args = [_cast_to_box(llargs[i]) for i in range(len(llargs))]
-    res = _cast_to_box(llres)
+    if llres:
+        res = _cast_to_box(llres)
+    else:
+        res = None
     return _cast_to_gcref(ResOperation(no, args, res))
 
 @register_helper(annmodel.SomePtr(llmemory.GCREF))
diff --git a/pypy/rlib/longlong2float.py b/pypy/rlib/longlong2float.py
--- a/pypy/rlib/longlong2float.py
+++ b/pypy/rlib/longlong2float.py
@@ -79,19 +79,23 @@
 longlong2float = rffi.llexternal(
     "pypy__longlong2float", [rffi.LONGLONG], rffi.DOUBLE,
     _callable=longlong2float_emulator, compilation_info=eci,
-    _nowrapper=True, elidable_function=True, sandboxsafe=True)
+    _nowrapper=True, elidable_function=True, sandboxsafe=True,
+    oo_primitive="pypy__longlong2float")
 
 float2longlong = rffi.llexternal(
     "pypy__float2longlong", [rffi.DOUBLE], rffi.LONGLONG,
     _callable=float2longlong_emulator, compilation_info=eci,
-    _nowrapper=True, elidable_function=True, sandboxsafe=True)
+    _nowrapper=True, elidable_function=True, sandboxsafe=True,
+    oo_primitive="pypy__float2longlong")
 
 uint2singlefloat = rffi.llexternal(
     "pypy__uint2singlefloat", [rffi.UINT], rffi.FLOAT,
     _callable=uint2singlefloat_emulator, compilation_info=eci,
-    _nowrapper=True, elidable_function=True, sandboxsafe=True)
+    _nowrapper=True, elidable_function=True, sandboxsafe=True,
+    oo_primitive="pypy__uint2singlefloat")
 
 singlefloat2uint = rffi.llexternal(
     "pypy__singlefloat2uint", [rffi.FLOAT], rffi.UINT,
     _callable=singlefloat2uint_emulator, compilation_info=eci,
-    _nowrapper=True, elidable_function=True, sandboxsafe=True)
+    _nowrapper=True, elidable_function=True, sandboxsafe=True,
+    oo_primitive="pypy__singlefloat2uint")
diff --git a/pypy/rlib/objectmodel.py b/pypy/rlib/objectmodel.py
--- a/pypy/rlib/objectmodel.py
+++ b/pypy/rlib/objectmodel.py
@@ -420,7 +420,7 @@
                   vobj.concretetype.TO._gckind == 'gc')
         else:
             from pypy.rpython.ootypesystem import ootype
-            ok = isinstance(vobj.concretetype, ootype.Instance)
+            ok = isinstance(vobj.concretetype, (ootype.Instance, ootype.BuiltinType))
         if not ok:
             from pypy.rpython.error import TyperError
             raise TyperError("compute_unique_id() cannot be applied to"
diff --git a/pypy/rpython/lltypesystem/rclass.py b/pypy/rpython/lltypesystem/rclass.py
--- a/pypy/rpython/lltypesystem/rclass.py
+++ b/pypy/rpython/lltypesystem/rclass.py
@@ -510,7 +510,13 @@
         ctype = inputconst(Void, self.object_type)
         cflags = inputconst(Void, flags)
         vlist = [ctype, cflags]
-        vptr = llops.genop('malloc', vlist,
+        cnonmovable = self.classdef.classdesc.read_attribute(
+            '_alloc_nonmovable_', Constant(False))
+        if cnonmovable.value:
+            opname = 'malloc_nonmovable'
+        else:
+            opname = 'malloc'
+        vptr = llops.genop(opname, vlist,
                            resulttype = Ptr(self.object_type))
         ctypeptr = inputconst(CLASSTYPE, self.rclass.getvtable())
         self.setfield(vptr, '__class__', ctypeptr, llops)
diff --git a/pypy/rpython/lltypesystem/rtuple.py b/pypy/rpython/lltypesystem/rtuple.py
--- a/pypy/rpython/lltypesystem/rtuple.py
+++ b/pypy/rpython/lltypesystem/rtuple.py
@@ -27,6 +27,10 @@
 
     def newtuple(cls, llops, r_tuple, items_v):
         # items_v should have the lowleveltype of the internal reprs
+        assert len(r_tuple.items_r) == len(items_v)
+        for r_item, v_item in zip(r_tuple.items_r, items_v):
+            assert r_item.lowleveltype == v_item.concretetype
+        #
         if len(r_tuple.items_r) == 0:
             return inputconst(Void, ())    # a Void empty tuple
         c1 = inputconst(Void, r_tuple.lowleveltype.TO)
diff --git a/pypy/rpython/ootypesystem/ootype.py b/pypy/rpython/ootypesystem/ootype.py
--- a/pypy/rpython/ootypesystem/ootype.py
+++ b/pypy/rpython/ootypesystem/ootype.py
@@ -512,6 +512,7 @@
             "ll_append_char": Meth([CHARTP], Void),
             "ll_append": Meth([STRINGTP], Void),
             "ll_build": Meth([], STRINGTP),
+            "ll_getlength": Meth([], Signed),
             })
         self._setup_methods({})
 
@@ -1376,6 +1377,9 @@
     def _cast_to_object(self):
         return make_object(self)
 
+    def _identityhash(self):
+        return object.__hash__(self)
+
 class _string(_builtin_type):
 
     def __init__(self, STRING, value = ''):
@@ -1543,6 +1547,9 @@
         else:
             return make_unicode(u''.join(self._buf))
 
+    def ll_getlength(self):
+        return self.ll_build().ll_strlen()
+
 class _null_string_builder(_null_mixin(_string_builder), _string_builder):
     def __init__(self, STRING_BUILDER):
         self.__dict__["_TYPE"] = STRING_BUILDER
diff --git a/pypy/rpython/ootypesystem/rbuilder.py b/pypy/rpython/ootypesystem/rbuilder.py
--- a/pypy/rpython/ootypesystem/rbuilder.py
+++ b/pypy/rpython/ootypesystem/rbuilder.py
@@ -21,6 +21,10 @@
         builder.ll_append_char(char)
 
     @staticmethod
+    def ll_getlength(builder):
+        return builder.ll_getlength()
+
+    @staticmethod
     def ll_append(builder, string):
         builder.ll_append(string)
 
diff --git a/pypy/rpython/rrange.py b/pypy/rpython/rrange.py
--- a/pypy/rpython/rrange.py
+++ b/pypy/rpython/rrange.py
@@ -204,7 +204,10 @@
         v_index = hop.gendirectcall(self.ll_getnextindex, v_enumerate)
         hop2 = hop.copy()
         hop2.args_r = [self.r_baseiter]
+        r_item_src = self.r_baseiter.r_list.external_item_repr
+        r_item_dst = hop.r_result.items_r[1]
         v_item = self.r_baseiter.rtype_next(hop2)
+        v_item = hop.llops.convertvar(v_item, r_item_src, r_item_dst)
         return hop.r_result.newtuple(hop.llops, hop.r_result,
                                      [v_index, v_item])
 
diff --git a/pypy/rpython/test/test_rbuilder.py b/pypy/rpython/test/test_rbuilder.py
--- a/pypy/rpython/test/test_rbuilder.py
+++ b/pypy/rpython/test/test_rbuilder.py
@@ -124,9 +124,5 @@
     pass
 
 class TestOOtype(BaseTestStringBuilder, OORtypeMixin):
-    def test_string_getlength(self):
-        py.test.skip("getlength(): not implemented on ootype")
-    def test_unicode_getlength(self):
-        py.test.skip("getlength(): not implemented on ootype")
     def test_append_charpsize(self):
         py.test.skip("append_charpsize(): not implemented on ootype")
diff --git a/pypy/rpython/test/test_rbuiltin.py b/pypy/rpython/test/test_rbuiltin.py
--- a/pypy/rpython/test/test_rbuiltin.py
+++ b/pypy/rpython/test/test_rbuiltin.py
@@ -463,6 +463,31 @@
             assert x1 == intmask(x0)
             assert x3 == intmask(x2)
 
+    def test_id_on_builtins(self):
+        from pypy.rlib.objectmodel import compute_unique_id
+        from pypy.rlib.rstring import StringBuilder, UnicodeBuilder
+        def fn():
+            return (compute_unique_id("foo"),
+                    compute_unique_id(u"bar"),
+                    compute_unique_id([1]),
+                    compute_unique_id({"foo": 3}),
+                    compute_unique_id(StringBuilder()),
+                    compute_unique_id(UnicodeBuilder()))
+        res = self.interpret(fn, [])
+        for id in self.ll_unpack_tuple(res, 6):
+            assert isinstance(id, (int, r_longlong))
+
+    def test_uniqueness_of_id_on_strings(self):
+        from pypy.rlib.objectmodel import compute_unique_id
+        def fn(s1, s2):
+            return (compute_unique_id(s1), compute_unique_id(s2))
+
+        s1 = "foo"
+        s2 = ''.join(['f','oo'])
+        res = self.interpret(fn, [self.string_to_ll(s1), self.string_to_ll(s2)])
+        i1, i2 = self.ll_unpack_tuple(res, 2)
+        assert i1 != i2
+
     def test_cast_primitive(self):
         from pypy.rpython.annlowlevel import LowLevelAnnotatorPolicy
         def llf(u):
diff --git a/pypy/rpython/test/test_rclass.py b/pypy/rpython/test/test_rclass.py
--- a/pypy/rpython/test/test_rclass.py
+++ b/pypy/rpython/test/test_rclass.py
@@ -1130,6 +1130,18 @@
             assert sorted([u]) == [6]                    # 32-bit types
             assert sorted([i, r, d, l]) == [2, 3, 4, 5]  # 64-bit types
 
+    def test_nonmovable(self):
+        for (nonmovable, opname) in [(True, 'malloc_nonmovable'),
+                                     (False, 'malloc')]:
+            class A(object):
+                _alloc_nonmovable_ = nonmovable
+            def f():
+                return A()
+            t, typer, graph = self.gengraph(f, [])
+            assert summary(graph) == {opname: 1,
+                                      'cast_pointer': 1,
+                                      'setfield': 1}
+
 
 class TestOOtype(BaseTestRclass, OORtypeMixin):
 
diff --git a/pypy/rpython/test/test_rrange.py b/pypy/rpython/test/test_rrange.py
--- a/pypy/rpython/test/test_rrange.py
+++ b/pypy/rpython/test/test_rrange.py
@@ -169,6 +169,22 @@
         res = self.interpret(fn, [2])
         assert res == 789
 
+    def test_enumerate_instances(self):
+        class A:
+            pass
+        def fn(n):
+            a = A()
+            b = A()
+            a.k = 10
+            b.k = 20
+            for i, x in enumerate([a, b]):
+                if i == n:
+                    return x.k
+            return 5
+        res = self.interpret(fn, [1])
+        assert res == 20
+
+
 
 class TestLLtype(BaseTestRrange, LLRtypeMixin):
     from pypy.rpython.lltypesystem import rrange 
diff --git a/pypy/tool/jitlogparser/parser.py b/pypy/tool/jitlogparser/parser.py
--- a/pypy/tool/jitlogparser/parser.py
+++ b/pypy/tool/jitlogparser/parser.py
@@ -140,13 +140,15 @@
     bytecode_name = None
     is_bytecode = True
     inline_level = None
+    has_dmp = False
 
     def parse_code_data(self, arg):
         m = re.search('\w]+)[\.,] file \'(.+?)\'[\.,] line (\d+)> #(\d+) (\w+)',
                       arg)
         if m is None:
             # a non-code loop, like StrLiteralSearch or something
-            self.bytecode_name = arg
+            if arg:
+                self.bytecode_name = arg
         else:
             self.name, self.filename, lineno, bytecode_no, self.bytecode_name = m.groups()
             self.startlineno = int(lineno)
@@ -218,7 +220,7 @@
         self.inputargs = inputargs
         self.chunks = chunks
         for chunk in self.chunks:
-            if chunk.filename is not None:
+            if chunk.bytecode_name is not None:
                 self.startlineno = chunk.startlineno
                 self.filename = chunk.filename
                 self.name = chunk.name
diff --git a/pypy/tool/jitlogparser/test/test_parser.py b/pypy/tool/jitlogparser/test/test_parser.py
--- a/pypy/tool/jitlogparser/test/test_parser.py
+++ b/pypy/tool/jitlogparser/test/test_parser.py
@@ -283,3 +283,13 @@
     assert loops[-1].count == 1234
     assert loops[1].count == 123
     assert loops[2].count == 12
+
+def test_parse_nonpython():
+    loop = parse("""
+    []
+    debug_merge_point(0, 'random')
+    debug_merge_point(0, ' #15 COMPARE_OP')
+    """)
+    f = Function.from_operations(loop.operations, LoopStorage())
+    assert f.chunks[-1].filename == 'x.py'
+    assert f.filename is None
diff --git a/pypy/translator/jvm/builtin.py b/pypy/translator/jvm/builtin.py
--- a/pypy/translator/jvm/builtin.py
+++ b/pypy/translator/jvm/builtin.py
@@ -84,6 +84,9 @@
     (ootype.StringBuilder.__class__, "ll_build"):
     jvm.Method.v(jStringBuilder, "toString", (), jString),
 
+    (ootype.StringBuilder.__class__, "ll_getlength"):
+    jvm.Method.v(jStringBuilder, "length", (), jInt),
+
     (ootype.String.__class__, "ll_hash"):
     jvm.Method.v(jString, "hashCode", (), jInt),
 
diff --git a/pypy/translator/jvm/database.py b/pypy/translator/jvm/database.py
--- a/pypy/translator/jvm/database.py
+++ b/pypy/translator/jvm/database.py
@@ -358,7 +358,7 @@
         ootype.Unsigned:jvm.PYPYSERIALIZEUINT,
         ootype.SignedLongLong:jvm.LONGTOSTRINGL,
         ootype.UnsignedLongLong: jvm.PYPYSERIALIZEULONG,
-        ootype.Float:jvm.DOUBLETOSTRINGD,
+        ootype.Float:jvm.PYPYSERIALIZEDOUBLE,
         ootype.Bool:jvm.PYPYSERIALIZEBOOLEAN,
         ootype.Void:jvm.PYPYSERIALIZEVOID,
         ootype.Char:jvm.PYPYESCAPEDCHAR,
diff --git a/pypy/translator/jvm/metavm.py b/pypy/translator/jvm/metavm.py
--- a/pypy/translator/jvm/metavm.py
+++ b/pypy/translator/jvm/metavm.py
@@ -92,6 +92,7 @@
 CASTS = {
 #   FROM                      TO
     (ootype.Signed,           ootype.UnsignedLongLong): jvm.I2L,
+    (ootype.Unsigned,         ootype.UnsignedLongLong): jvm.I2L,
     (ootype.SignedLongLong,   ootype.Signed):           jvm.L2I,
     (ootype.UnsignedLongLong, ootype.Unsigned):         jvm.L2I,
     (ootype.UnsignedLongLong, ootype.Signed):           jvm.L2I,
diff --git a/pypy/translator/jvm/opcodes.py b/pypy/translator/jvm/opcodes.py
--- a/pypy/translator/jvm/opcodes.py
+++ b/pypy/translator/jvm/opcodes.py
@@ -101,6 +101,7 @@
     'jit_force_virtualizable':  Ignore,
     'jit_force_virtual':        DoNothing,
     'jit_force_quasi_immutable': Ignore,
+    'jit_is_virtual':           PushPrimitive(ootype.Bool, False),
 
     'debug_assert':              [], # TODO: implement?
     'debug_start_traceback':    Ignore,
diff --git a/pypy/translator/jvm/src/pypy/PyPy.java b/pypy/translator/jvm/src/pypy/PyPy.java
--- a/pypy/translator/jvm/src/pypy/PyPy.java
+++ b/pypy/translator/jvm/src/pypy/PyPy.java
@@ -283,6 +283,14 @@
         }
     }
 
+    public double pypy__longlong2float(long l) {
+        return Double.longBitsToDouble(l);
+    }
+
+    public long pypy__float2longlong(double d) {
+        return Double.doubleToRawLongBits(d);
+    }
+
     public double ooparse_float(String s) {
         try {
             return Double.parseDouble(s);
@@ -353,6 +361,19 @@
             return "False";
     }
 
+    public static String serialize_double(double d) {
+        if (Double.isNaN(d)) {
+            return "float(\"nan\")";
+        } else if (Double.isInfinite(d)) {
+            if (d > 0)
+                return "float(\"inf\")";
+            else
+                return "float(\"-inf\")";
+        } else {
+            return Double.toString(d);
+        }
+    }
+
     private static String format_char(char c) {
         String res = "\\x";
         if (c <= 0x0F) res = res + "0";
diff --git a/pypy/translator/jvm/test/runtest.py b/pypy/translator/jvm/test/runtest.py
--- a/pypy/translator/jvm/test/runtest.py
+++ b/pypy/translator/jvm/test/runtest.py
@@ -56,6 +56,7 @@
 
 # CLI could-be duplicate
 class JvmGeneratedSourceWrapper(object):
+
     def __init__(self, gensrc):
         """ gensrc is an instance of JvmGeneratedSource """
         self.gensrc = gensrc
diff --git a/pypy/translator/jvm/test/test_builder.py b/pypy/translator/jvm/test/test_builder.py
new file mode 100644
--- /dev/null
+++ b/pypy/translator/jvm/test/test_builder.py
@@ -0,0 +1,7 @@
+from pypy.translator.jvm.test.runtest import JvmTest
+from pypy.rpython.test.test_rbuilder import BaseTestStringBuilder
+import py
+
+class TestJvmStringBuilder(JvmTest, BaseTestStringBuilder):
+    def test_append_charpsize(self):
+        py.test.skip("append_charpsize(): not implemented on ootype")
diff --git a/pypy/translator/jvm/test/test_longlong2float.py b/pypy/translator/jvm/test/test_longlong2float.py
new file mode 100644
--- /dev/null
+++ b/pypy/translator/jvm/test/test_longlong2float.py
@@ -0,0 +1,20 @@
+from pypy.translator.jvm.test.runtest import JvmTest
+from pypy.rlib.longlong2float import *
+from pypy.rlib.test.test_longlong2float import enum_floats
+from pypy.rlib.test.test_longlong2float import fn as float2longlong2float
+import py
+
+class TestLongLong2Float(JvmTest):
+
+    def test_float2longlong_and_longlong2float(self):
+        def func(f):
+            return float2longlong2float(f)
+
+        for f in enum_floats():
+            assert repr(f) == repr(self.interpret(func, [f]))
+
+    def test_uint2singlefloat(self):
+        py.test.skip("uint2singlefloat is not implemented in ootype")
+
+    def test_singlefloat2uint(self):
+        py.test.skip("singlefloat2uint is not implemented in ootype")
diff --git a/pypy/translator/jvm/typesystem.py b/pypy/translator/jvm/typesystem.py
--- a/pypy/translator/jvm/typesystem.py
+++ b/pypy/translator/jvm/typesystem.py
@@ -955,6 +955,7 @@
 PYPYSERIALIZEUINT  =    Method.s(jPyPy, 'serialize_uint', (jInt,), jString)
 PYPYSERIALIZEULONG =    Method.s(jPyPy, 'serialize_ulonglong', (jLong,),jString)
 PYPYSERIALIZEVOID =     Method.s(jPyPy, 'serialize_void', (), jString)
+PYPYSERIALIZEDOUBLE =   Method.s(jPyPy, 'serialize_double', (jDouble,), jString)
 PYPYESCAPEDCHAR =       Method.s(jPyPy, 'escaped_char', (jChar,), jString)
 PYPYESCAPEDUNICHAR =    Method.s(jPyPy, 'escaped_unichar', (jChar,), jString)
 PYPYESCAPEDSTRING =     Method.s(jPyPy, 'escaped_string', (jString,), jString)
diff --git a/pypy/translator/oosupport/test_template/cast.py b/pypy/translator/oosupport/test_template/cast.py
--- a/pypy/translator/oosupport/test_template/cast.py
+++ b/pypy/translator/oosupport/test_template/cast.py
@@ -13,6 +13,9 @@
 def to_longlong(x):
     return r_longlong(x)
 
+def to_ulonglong(x):
+    return r_ulonglong(x)
+
 def uint_to_int(x):
     return intmask(x)
 
@@ -56,6 +59,9 @@
     def test_unsignedlonglong_to_unsigned4(self):
         self.check(to_uint, [r_ulonglong(18446744073709551615l)]) # max 64 bit num
 
+    def test_unsigned_to_usignedlonglong(self):
+        self.check(to_ulonglong, [r_uint(42)])
+
     def test_uint_to_int(self):
         self.check(uint_to_int, [r_uint(sys.maxint+1)])
 

From noreply at buildbot.pypy.org  Fri Jan 20 17:26:35 2012
From: noreply at buildbot.pypy.org (cocoatomo)
Date: Fri, 20 Jan 2012 17:26:35 +0100 (CET)
Subject: [pypy-commit] pypy default: merge
Message-ID: <20120120162635.DE0A082CF8@wyvern.cs.uni-duesseldorf.de>

Author: cocoatomo 
Branch: 
Changeset: r51556:8853197b01a6
Date: 2012-01-21 01:04 +0900
http://bitbucket.org/pypy/pypy/changeset/8853197b01a6/

Log:	merge

diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -86,6 +86,8 @@
         ("subtract", "subtract"),
         ('sqrt', 'sqrt'),
         ("tan", "tan"),
+        ('bitwise_and', 'bitwise_and'),
+        ('bitwise_or', 'bitwise_or'),
     ]:
         interpleveldefs[exposed] = "interp_ufuncs.get(space).%s" % impl
 
diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -47,6 +47,10 @@
     def getitem(self, storage, i):
         return self.itemtype.read(storage, self.itemtype.get_element_size(), i, 0)
 
+    def getitem_bool(self, storage, i):
+        isize = self.itemtype.get_element_size()
+        return self.itemtype.read_bool(storage, isize, i, 0)
+
     def setitem(self, storage, i, box):
         self.itemtype.store(storage, self.itemtype.get_element_size(), i, 0, box)
 
@@ -85,6 +89,12 @@
     def descr_get_shape(self, space):
         return space.newtuple([])
 
+    def is_int_type(self):
+        return self.kind == SIGNEDLTR or self.kind == UNSIGNEDLTR
+
+    def is_bool_type(self):
+        return self.kind == BOOLLTR
+
 W_Dtype.typedef = TypeDef("dtype",
     __module__ = "numpypy",
     __new__ = interp2app(W_Dtype.descr__new__.im_func),
diff --git a/pypy/module/micronumpy/interp_iter.py b/pypy/module/micronumpy/interp_iter.py
--- a/pypy/module/micronumpy/interp_iter.py
+++ b/pypy/module/micronumpy/interp_iter.py
@@ -4,6 +4,19 @@
 from pypy.module.micronumpy.strides import calculate_broadcast_strides,\
      calculate_slice_strides
 
+# structures to describe slicing
+
+class Chunk(object):
+    def __init__(self, start, stop, step, lgt):
+        self.start = start
+        self.stop = stop
+        self.step = step
+        self.lgt = lgt
+
+    def extend_shape(self, shape):
+        if self.step != 0:
+            shape.append(self.lgt)
+
 class BaseTransform(object):
     pass
 
@@ -38,11 +51,18 @@
         self.size = size
 
     def next(self, shapelen):
+        return self._next(1)
+
+    def _next(self, ofs):
         arr = instantiate(ArrayIterator)
         arr.size = self.size
-        arr.offset = self.offset + 1
+        arr.offset = self.offset + ofs
         return arr
 
+    def next_no_increase(self, shapelen):
+        # a hack to make JIT believe this is always virtual
+        return self._next(0)
+
     def done(self):
         return self.offset >= self.size
 
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -2,14 +2,15 @@
 from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.interpreter.gateway import interp2app, NoneNotWrapped
 from pypy.interpreter.typedef import TypeDef, GetSetProperty
-from pypy.module.micronumpy import interp_ufuncs, interp_dtype, signature
+from pypy.module.micronumpy import interp_ufuncs, interp_dtype, signature,\
+     interp_boxes
 from pypy.module.micronumpy.strides import calculate_slice_strides
 from pypy.rlib import jit
 from pypy.rpython.lltypesystem import lltype, rffi
 from pypy.tool.sourcetools import func_with_new_name
 from pypy.rlib.rstring import StringBuilder
 from pypy.module.micronumpy.interp_iter import ArrayIterator, OneDimIterator,\
-     SkipLastAxisIterator
+     SkipLastAxisIterator, Chunk, ViewIterator
 
 numpy_driver = jit.JitDriver(
     greens=['shapelen', 'sig'],
@@ -39,7 +40,24 @@
     get_printable_location=signature.new_printable_location('slice'),
     name='numpy_slice',
 )
-
+count_driver = jit.JitDriver(
+    greens=['shapelen'],
+    virtualizables=['frame'],
+    reds=['s', 'frame', 'iter', 'arr'],
+    name='numpy_count'
+)
+filter_driver = jit.JitDriver(
+    greens=['shapelen', 'sig'],
+    virtualizables=['frame'],
+    reds=['concr', 'argi', 'ri', 'frame', 'v', 'res', 'self'],
+    name='numpy_filter',
+)
+filter_set_driver = jit.JitDriver(
+    greens=['shapelen', 'sig'],
+    virtualizables=['frame'],
+    reds=['idx', 'idxi', 'frame', 'arr'],
+    name='numpy_filterset',
+)
 
 def _find_shape_and_elems(space, w_iterable):
     shape = [space.len_w(w_iterable)]
@@ -270,6 +288,9 @@
     descr_gt = _binop_impl("greater")
     descr_ge = _binop_impl("greater_equal")
 
+    descr_and = _binop_impl("bitwise_and")
+    descr_or = _binop_impl("bitwise_or")
+
     def _binop_right_impl(ufunc_name):
         def impl(self, space, w_other):
             w_other = scalar_w(space,
@@ -479,11 +500,69 @@
     def _prepare_slice_args(self, space, w_idx):
         if (space.isinstance_w(w_idx, space.w_int) or
             space.isinstance_w(w_idx, space.w_slice)):
-            return [space.decode_index4(w_idx, self.shape[0])]
-        return [space.decode_index4(w_item, self.shape[i]) for i, w_item in
+            return [Chunk(*space.decode_index4(w_idx, self.shape[0]))]
+        return [Chunk(*space.decode_index4(w_item, self.shape[i])) for i, w_item in
                 enumerate(space.fixedview(w_idx))]
 
+    def count_all_true(self, arr):
+        sig = arr.find_sig()
+        frame = sig.create_frame(self)
+        shapelen = len(arr.shape)
+        s = 0
+        iter = None
+        while not frame.done():
+            count_driver.jit_merge_point(arr=arr, frame=frame, iter=iter, s=s,
+                                         shapelen=shapelen)
+            iter = frame.get_final_iter()
+            s += arr.dtype.getitem_bool(arr.storage, iter.offset)
+            frame.next(shapelen)
+        return s
+
+    def getitem_filter(self, space, arr):
+        concr = arr.get_concrete()
+        size = self.count_all_true(concr)
+        res = W_NDimArray(size, [size], self.find_dtype())
+        ri = ArrayIterator(size)
+        shapelen = len(self.shape)
+        argi = concr.create_iter()
+        sig = self.find_sig()
+        frame = sig.create_frame(self)
+        v = None
+        while not frame.done():
+            filter_driver.jit_merge_point(concr=concr, argi=argi, ri=ri,
+                                          frame=frame, v=v, res=res, sig=sig,
+                                          shapelen=shapelen, self=self)
+            if concr.dtype.getitem_bool(concr.storage, argi.offset):
+                v = sig.eval(frame, self)
+                res.setitem(ri.offset, v)
+                ri = ri.next(1)
+            else:
+                ri = ri.next_no_increase(1)
+            argi = argi.next(shapelen)
+            frame.next(shapelen)
+        return res
+
+    def setitem_filter(self, space, idx, val):
+        size = self.count_all_true(idx)
+        arr = SliceArray([size], self.dtype, self, val)
+        sig = arr.find_sig()
+        shapelen = len(self.shape)
+        frame = sig.create_frame(arr)
+        idxi = idx.create_iter()
+        while not frame.done():
+            filter_set_driver.jit_merge_point(idx=idx, idxi=idxi, sig=sig,
+                                              frame=frame, arr=arr,
+                                              shapelen=shapelen)
+            if idx.dtype.getitem_bool(idx.storage, idxi.offset):
+                sig.eval(frame, arr)
+                frame.next_from_second(1)
+            frame.next_first(shapelen)
+            idxi = idxi.next(shapelen)
+
     def descr_getitem(self, space, w_idx):
+        if (isinstance(w_idx, BaseArray) and w_idx.shape == self.shape and
+            w_idx.find_dtype().is_bool_type()):
+            return self.getitem_filter(space, w_idx)
         if self._single_item_result(space, w_idx):
             concrete = self.get_concrete()
             item = concrete._index_of_single_item(space, w_idx)
@@ -493,6 +572,11 @@
 
     def descr_setitem(self, space, w_idx, w_value):
         self.invalidated()
+        if (isinstance(w_idx, BaseArray) and w_idx.shape == self.shape and
+            w_idx.find_dtype().is_bool_type()):
+            return self.get_concrete().setitem_filter(space,
+                                                      w_idx.get_concrete(),
+                                             convert_to_array(space, w_value))
         if self._single_item_result(space, w_idx):
             concrete = self.get_concrete()
             item = concrete._index_of_single_item(space, w_idx)
@@ -509,9 +593,8 @@
     def create_slice(self, chunks):
         shape = []
         i = -1
-        for i, (start_, stop, step, lgt) in enumerate(chunks):
-            if step != 0:
-                shape.append(lgt)
+        for i, chunk in enumerate(chunks):
+            chunk.extend_shape(shape)
         s = i + 1
         assert s >= 0
         shape += self.shape[s:]
@@ -724,8 +807,7 @@
                                          frame=frame,
                                          ri=ri,
                                          self=self, result=result)
-            result.dtype.setitem(result.storage, ri.offset,
-                                 sig.eval(frame, self))
+            result.setitem(ri.offset, sig.eval(frame, self))
             frame.next(shapelen)
             ri = ri.next(shapelen)
         return result
@@ -945,7 +1027,7 @@
                             builder.append('\n' + indent)
                         else:
                             builder.append(indent)
-                    view = self.create_slice([(i, 0, 0, 1)]).get_concrete()
+                    view = self.create_slice([Chunk(i, 0, 0, 1)]).get_concrete()
                     view.to_str(space, comma, builder, indent=indent + ' ',
                                                     use_ellipsis=use_ellipsis)
                 if i < self.shape[0] - 1:
@@ -962,7 +1044,7 @@
                         builder.append(indent)
                 # create_slice requires len(chunks) > 1 in order to reduce
                 # shape
-                view = self.create_slice([(i, 0, 0, 1)]).get_concrete()
+                view = self.create_slice([Chunk(i, 0, 0, 1)]).get_concrete()
                 view.to_str(space, comma, builder, indent=indent + ' ',
                                                     use_ellipsis=use_ellipsis)
                 i += 1
@@ -1091,6 +1173,10 @@
                                parent)
         self.start = start
 
+    def create_iter(self):
+        return ViewIterator(self.start, self.strides, self.backstrides,
+                            self.shape)
+
     def setshape(self, space, new_shape):
         if len(self.shape) < 1:
             return
@@ -1137,6 +1223,9 @@
         self.shape = new_shape
         self.calc_strides(new_shape)
 
+    def create_iter(self):
+        return ArrayIterator(self.size)
+
     def create_sig(self):
         return signature.ArraySignature(self.dtype)
 
@@ -1191,6 +1280,7 @@
     arr = W_NDimArray(size, shape[:], dtype=dtype, order=order)
     shapelen = len(shape)
     arr_iter = ArrayIterator(arr.size)
+    # XXX we might want to have a jitdriver here
     for i in range(len(elems_w)):
         w_elem = elems_w[i]
         dtype.setitem(arr.storage, arr_iter.offset,
@@ -1257,6 +1347,9 @@
     __gt__ = interp2app(BaseArray.descr_gt),
     __ge__ = interp2app(BaseArray.descr_ge),
 
+    __and__ = interp2app(BaseArray.descr_and),
+    __or__ = interp2app(BaseArray.descr_or),
+
     __repr__ = interp2app(BaseArray.descr_repr),
     __str__ = interp2app(BaseArray.descr_str),
     __array_interface__ = GetSetProperty(BaseArray.descr_array_iface),
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -249,15 +249,16 @@
 
 
 class W_Ufunc2(W_Ufunc):
-    _immutable_fields_ = ["comparison_func", "func", "name"]
+    _immutable_fields_ = ["comparison_func", "func", "name", "int_only"]
     argcount = 2
 
     def __init__(self, func, name, promote_to_float=False, promote_bools=False,
-        identity=None, comparison_func=False):
+        identity=None, comparison_func=False, int_only=False):
 
         W_Ufunc.__init__(self, name, promote_to_float, promote_bools, identity)
         self.func = func
         self.comparison_func = comparison_func
+        self.int_only = int_only
 
     def call(self, space, args_w):
         from pypy.module.micronumpy.interp_numarray import (Call2,
@@ -268,6 +269,7 @@
         w_rhs = convert_to_array(space, w_rhs)
         calc_dtype = find_binop_result_dtype(space,
             w_lhs.find_dtype(), w_rhs.find_dtype(),
+            int_only=self.int_only,
             promote_to_float=self.promote_to_float,
             promote_bools=self.promote_bools,
         )
@@ -304,10 +306,12 @@
 
 
 def find_binop_result_dtype(space, dt1, dt2, promote_to_float=False,
-    promote_bools=False):
+    promote_bools=False, int_only=False):
     # dt1.num should be <= dt2.num
     if dt1.num > dt2.num:
         dt1, dt2 = dt2, dt1
+    if int_only and (not dt1.is_int_type() or not dt2.is_int_type()):
+        raise OperationError(space.w_TypeError, space.wrap("Unsupported types"))
     # Some operations promote op(bool, bool) to return int8, rather than bool
     if promote_bools and (dt1.kind == dt2.kind == interp_dtype.BOOLLTR):
         return interp_dtype.get_dtype_cache(space).w_int8dtype
@@ -425,6 +429,10 @@
             ("add", "add", 2, {"identity": 0}),
             ("subtract", "sub", 2),
             ("multiply", "mul", 2, {"identity": 1}),
+            ("bitwise_and", "bitwise_and", 2, {"identity": 1,
+                                               'int_only': True}),
+            ("bitwise_or", "bitwise_or", 2, {"identity": 0,
+                                             'int_only': True}),
             ("divide", "div", 2, {"promote_bools": True}),
             ("mod", "mod", 2, {"promote_bools": True}),
             ("power", "pow", 2, {"promote_bools": True}),
@@ -476,7 +484,7 @@
         extra_kwargs["identity"] = identity
 
         func = ufunc_dtype_caller(space, ufunc_name, op_name, argcount,
-            comparison_func=extra_kwargs.get("comparison_func", False)
+            comparison_func=extra_kwargs.get("comparison_func", False),
         )
         if argcount == 1:
             ufunc = W_Ufunc1(func, ufunc_name, **extra_kwargs)
diff --git a/pypy/module/micronumpy/signature.py b/pypy/module/micronumpy/signature.py
--- a/pypy/module/micronumpy/signature.py
+++ b/pypy/module/micronumpy/signature.py
@@ -82,6 +82,16 @@
         for i in range(len(self.iterators)):
             self.iterators[i] = self.iterators[i].next(shapelen)
 
+    @unroll_safe
+    def next_from_second(self, shapelen):
+        """ Don't increase the first iterator
+        """
+        for i in range(1, len(self.iterators)):
+            self.iterators[i] = self.iterators[i].next(shapelen)
+
+    def next_first(self, shapelen):
+        self.iterators[0] = self.iterators[0].next(shapelen)
+
     def get_final_iter(self):
         final_iter = promote(self.final_iter)
         if final_iter < 0:
diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py
--- a/pypy/module/micronumpy/strides.py
+++ b/pypy/module/micronumpy/strides.py
@@ -10,12 +10,12 @@
     rstart = start
     rshape = []
     i = -1
-    for i, (start_, stop, step, lgt) in enumerate(chunks):
-        if step != 0:
-            rstrides.append(strides[i] * step)
-            rbackstrides.append(strides[i] * (lgt - 1) * step)
-            rshape.append(lgt)
-        rstart += strides[i] * start_
+    for i, chunk in enumerate(chunks):
+        if chunk.step != 0:
+            rstrides.append(strides[i] * chunk.step)
+            rbackstrides.append(strides[i] * (chunk.lgt - 1) * chunk.step)
+            rshape.append(chunk.lgt)
+        rstart += strides[i] * chunk.start
     # add a reminder
     s = i + 1
     assert s >= 0
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -2,6 +2,7 @@
 import py
 from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
 from pypy.module.micronumpy.interp_numarray import W_NDimArray, shape_agreement
+from pypy.module.micronumpy.interp_iter import Chunk
 from pypy.module.micronumpy import signature
 from pypy.interpreter.error import OperationError
 from pypy.conftest import gettestobjspace
@@ -37,53 +38,54 @@
 
     def test_create_slice_f(self):
         a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'F')
-        s = a.create_slice([(3, 0, 0, 1)])
+        s = a.create_slice([Chunk(3, 0, 0, 1)])
         assert s.start == 3
         assert s.strides == [10, 50]
         assert s.backstrides == [40, 100]
-        s = a.create_slice([(1, 9, 2, 4)])
+        s = a.create_slice([Chunk(1, 9, 2, 4)])
         assert s.start == 1
         assert s.strides == [2, 10, 50]
         assert s.backstrides == [6, 40, 100]
-        s = a.create_slice([(1, 5, 3, 2), (1, 2, 1, 1), (1, 0, 0, 1)])
+        s = a.create_slice([Chunk(1, 5, 3, 2), Chunk(1, 2, 1, 1), Chunk(1, 0, 0, 1)])
         assert s.shape == [2, 1]
         assert s.strides == [3, 10]
         assert s.backstrides == [3, 0]
-        s = a.create_slice([(0, 10, 1, 10), (2, 0, 0, 1)])
+        s = a.create_slice([Chunk(0, 10, 1, 10), Chunk(2, 0, 0, 1)])
         assert s.start == 20
         assert s.shape == [10, 3]
 
     def test_create_slice_c(self):
         a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'C')
-        s = a.create_slice([(3, 0, 0, 1)])
+        s = a.create_slice([Chunk(3, 0, 0, 1)])
         assert s.start == 45
         assert s.strides == [3, 1]
         assert s.backstrides == [12, 2]
-        s = a.create_slice([(1, 9, 2, 4)])
+        s = a.create_slice([Chunk(1, 9, 2, 4)])
         assert s.start == 15
         assert s.strides == [30, 3, 1]
         assert s.backstrides == [90, 12, 2]
-        s = a.create_slice([(1, 5, 3, 2), (1, 2, 1, 1), (1, 0, 0, 1)])
+        s = a.create_slice([Chunk(1, 5, 3, 2), Chunk(1, 2, 1, 1),
+                            Chunk(1, 0, 0, 1)])
         assert s.start == 19
         assert s.shape == [2, 1]
         assert s.strides == [45, 3]
         assert s.backstrides == [45, 0]
-        s = a.create_slice([(0, 10, 1, 10), (2, 0, 0, 1)])
+        s = a.create_slice([Chunk(0, 10, 1, 10), Chunk(2, 0, 0, 1)])
         assert s.start == 6
         assert s.shape == [10, 3]
 
     def test_slice_of_slice_f(self):
         a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'F')
-        s = a.create_slice([(5, 0, 0, 1)])
+        s = a.create_slice([Chunk(5, 0, 0, 1)])
         assert s.start == 5
-        s2 = s.create_slice([(3, 0, 0, 1)])
+        s2 = s.create_slice([Chunk(3, 0, 0, 1)])
         assert s2.shape == [3]
         assert s2.strides == [50]
         assert s2.parent is a
         assert s2.backstrides == [100]
         assert s2.start == 35
-        s = a.create_slice([(1, 5, 3, 2)])
-        s2 = s.create_slice([(0, 2, 1, 2), (2, 0, 0, 1)])
+        s = a.create_slice([Chunk(1, 5, 3, 2)])
+        s2 = s.create_slice([Chunk(0, 2, 1, 2), Chunk(2, 0, 0, 1)])
         assert s2.shape == [2, 3]
         assert s2.strides == [3, 50]
         assert s2.backstrides == [3, 100]
@@ -91,16 +93,16 @@
 
     def test_slice_of_slice_c(self):
         a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), order='C')
-        s = a.create_slice([(5, 0, 0, 1)])
+        s = a.create_slice([Chunk(5, 0, 0, 1)])
         assert s.start == 15 * 5
-        s2 = s.create_slice([(3, 0, 0, 1)])
+        s2 = s.create_slice([Chunk(3, 0, 0, 1)])
         assert s2.shape == [3]
         assert s2.strides == [1]
         assert s2.parent is a
         assert s2.backstrides == [2]
         assert s2.start == 5 * 15 + 3 * 3
-        s = a.create_slice([(1, 5, 3, 2)])
-        s2 = s.create_slice([(0, 2, 1, 2), (2, 0, 0, 1)])
+        s = a.create_slice([Chunk(1, 5, 3, 2)])
+        s2 = s.create_slice([Chunk(0, 2, 1, 2), Chunk(2, 0, 0, 1)])
         assert s2.shape == [2, 3]
         assert s2.strides == [45, 1]
         assert s2.backstrides == [45, 2]
@@ -108,14 +110,14 @@
 
     def test_negative_step_f(self):
         a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'F')
-        s = a.create_slice([(9, -1, -2, 5)])
+        s = a.create_slice([Chunk(9, -1, -2, 5)])
         assert s.start == 9
         assert s.strides == [-2, 10, 50]
         assert s.backstrides == [-8, 40, 100]
 
     def test_negative_step_c(self):
         a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), order='C')
-        s = a.create_slice([(9, -1, -2, 5)])
+        s = a.create_slice([Chunk(9, -1, -2, 5)])
         assert s.start == 135
         assert s.strides == [-30, 3, 1]
         assert s.backstrides == [-120, 12, 2]
@@ -124,7 +126,7 @@
         a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'F')
         r = a._index_of_single_item(self.space, self.newtuple(1, 2, 2))
         assert r == 1 + 2 * 10 + 2 * 50
-        s = a.create_slice([(0, 10, 1, 10), (2, 0, 0, 1)])
+        s = a.create_slice([Chunk(0, 10, 1, 10), Chunk(2, 0, 0, 1)])
         r = s._index_of_single_item(self.space, self.newtuple(1, 0))
         assert r == a._index_of_single_item(self.space, self.newtuple(1, 2, 0))
         r = s._index_of_single_item(self.space, self.newtuple(1, 1))
@@ -134,7 +136,7 @@
         a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'C')
         r = a._index_of_single_item(self.space, self.newtuple(1, 2, 2))
         assert r == 1 * 3 * 5 + 2 * 3 + 2
-        s = a.create_slice([(0, 10, 1, 10), (2, 0, 0, 1)])
+        s = a.create_slice([Chunk(0, 10, 1, 10), Chunk(2, 0, 0, 1)])
         r = s._index_of_single_item(self.space, self.newtuple(1, 0))
         assert r == a._index_of_single_item(self.space, self.newtuple(1, 2, 0))
         r = s._index_of_single_item(self.space, self.newtuple(1, 1))
@@ -1302,15 +1304,25 @@
         assert isinstance(i['data'][0], int)
         raises(TypeError, getattr, array(3), '__array_interface__')
 
+    def test_array_indexing_one_elem(self):
+        skip("not yet")
+        from _numpypy import array, arange
+        raises(IndexError, 'arange(3)[array([3.5])]')
+        a = arange(3)[array([1])]
+        assert a == 1
+        assert a[0] == 1
+        raises(IndexError,'arange(3)[array([15])]')
+        assert arange(3)[array([-3])] == 0
+        raises(IndexError,'arange(3)[array([-15])]')
+        assert arange(3)[array(1)] == 1
+
     def test_fill(self):
         from _numpypy import array
-
         a = array([1, 2, 3])
         a.fill(10)
         assert (a == [10, 10, 10]).all()
         a.fill(False)
         assert (a == [0, 0, 0]).all()
-
         b = a[:1]
         b.fill(4)
         assert (b == [4]).all()
@@ -1324,6 +1336,24 @@
         d.fill(100)
         assert d == 100
 
+    def test_array_indexing_bool(self):
+        from _numpypy import arange
+        a = arange(10)
+        assert (a[a > 3] == [4, 5, 6, 7, 8, 9]).all()
+        a = arange(10).reshape(5, 2)
+        assert (a[a > 3] == [4, 5, 6, 7, 8, 9]).all()
+        assert (a[a & 1 == 1] == [1, 3, 5, 7, 9]).all()
+
+    def test_array_indexing_bool_setitem(self):
+        from _numpypy import arange, array
+        a = arange(6)
+        a[a > 3] = 15
+        assert (a == [0, 1, 2, 3, 15, 15]).all()
+        a = arange(6).reshape(3, 2)
+        a[a & 1 == 1] = array([8, 9, 10])
+        assert (a == [[0, 8], [2, 9], [4, 10]]).all()
+
+
 
 class AppTestSupport(BaseNumpyAppTest):
     def setup_class(cls):
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -355,11 +355,20 @@
         raises(ValueError, maximum.reduce, [])
 
     def test_reduceND(self):
-        from numpypy import add, arange
+        from _numpypy import add, arange
         a = arange(12).reshape(3, 4)
         assert (add.reduce(a, 0) == [12, 15, 18, 21]).all()
         assert (add.reduce(a, 1) == [6.0, 22.0, 38.0]).all()
 
+    def test_bitwise(self):
+        from _numpypy import bitwise_and, bitwise_or, arange, array
+        a = arange(6).reshape(2, 3)
+        assert (a & 1 == [[0, 1, 0], [1, 0, 1]]).all()
+        assert (a & 1 == bitwise_and(a, 1)).all()
+        assert (a | 1 == [[1, 1, 3], [3, 5, 5]]).all()
+        assert (a | 1 == bitwise_or(a, 1)).all()
+        raises(TypeError, 'array([1.0]) & 1')
+
     def test_comparisons(self):
         import operator
         from _numpypy import equal, not_equal, less, less_equal, greater, greater_equal
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -217,6 +217,7 @@
         # This is the sum of the ops for both loops, however if you remove the
         # optimization then you end up with 2 float_adds, so we can still be
         # sure it was optimized correctly.
+        py.test.skip("too fragile")
         self.check_resops({'setinteriorfield_raw': 4, 'getfield_gc': 22,
                            'getarrayitem_gc': 4, 'getarrayitem_gc_pure': 2,
                            'getfield_gc_pure': 8,
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -94,6 +94,9 @@
             width, storage, i, offset
         ))
 
+    def read_bool(self, storage, width, i, offset):
+        raise NotImplementedError
+
     def store(self, storage, width, i, offset, box):
         value = self.unbox(box)
         libffi.array_setitem(clibffi.cast_type_to_ffitype(self.T),
@@ -168,6 +171,7 @@
     @simple_binary_op
     def min(self, v1, v2):
         return min(v1, v2)
+    
 
 class Bool(BaseType, Primitive):
     T = lltype.Bool
@@ -185,6 +189,11 @@
         else:
             return self.False
 
+
+    def read_bool(self, storage, width, i, offset):
+        return libffi.array_getitem(clibffi.cast_type_to_ffitype(self.T),
+                                    width, storage, i, offset)
+
     def coerce_subtype(self, space, w_subtype, w_item):
         # Doesn't return subclasses so it can return the constants.
         return self._coerce(space, w_item)
@@ -253,6 +262,14 @@
             assert v == 0
             return 0
 
+    @simple_binary_op
+    def bitwise_and(self, v1, v2):
+        return v1 & v2
+
+    @simple_binary_op
+    def bitwise_or(self, v1, v2):
+        return v1 | v2
+
 class Int8(BaseType, Integer):
     T = rffi.SIGNEDCHAR
     BoxType = interp_boxes.W_Int8Box
diff --git a/pypy/tool/test/test_version.py b/pypy/tool/test/test_version.py
--- a/pypy/tool/test/test_version.py
+++ b/pypy/tool/test/test_version.py
@@ -1,6 +1,22 @@
 import os, sys
 import py
-from pypy.tool.version import get_repo_version_info
+from pypy.tool.version import get_repo_version_info, _get_hg_archive_version
+
+def test_hg_archival_version(tmpdir):
+    def version_for(name, **kw):
+        path = tmpdir.join(name)
+        path.write('\n'.join('%s: %s' % x for x in kw.items()))
+        return _get_hg_archive_version(str(path))
+
+    assert version_for('release',
+                       tag='release-123',
+                       node='000',
+                      ) == ('PyPy', 'release-123', '000')
+    assert version_for('somebranch',
+                       node='000',
+                       branch='something',
+                      ) == ('PyPy', 'something', '000')
+
 
 def test_get_repo_version_info():
     assert get_repo_version_info(None)
diff --git a/pypy/tool/version.py b/pypy/tool/version.py
--- a/pypy/tool/version.py
+++ b/pypy/tool/version.py
@@ -3,111 +3,139 @@
 from subprocess import Popen, PIPE
 import pypy
 pypydir = os.path.dirname(os.path.abspath(pypy.__file__))
+pypyroot = os.path.dirname(pypydir)
+default_retval = 'PyPy', '?', '?'
+
+def maywarn(err, repo_type='Mercurial'):
+    if not err:
+        return
+
+    from pypy.tool.ansi_print import ansi_log
+    log = py.log.Producer("version")
+    py.log.setconsumer("version", ansi_log)
+    log.WARNING('Errors getting %s information: %s' % (repo_type, err))
 
 def get_repo_version_info(hgexe=None):
     '''Obtain version information by invoking the 'hg' or 'git' commands.'''
-    # TODO: support extracting from .hg_archival.txt
-
-    default_retval = 'PyPy', '?', '?'
-    pypyroot = os.path.abspath(os.path.join(pypydir, '..'))
-
-    def maywarn(err, repo_type='Mercurial'):
-        if not err:
-            return
-
-        from pypy.tool.ansi_print import ansi_log
-        log = py.log.Producer("version")
-        py.log.setconsumer("version", ansi_log)
-        log.WARNING('Errors getting %s information: %s' % (repo_type, err))
 
     # Try to see if we can get info from Git if hgexe is not specified.
     if not hgexe:
         if os.path.isdir(os.path.join(pypyroot, '.git')):
-            gitexe = py.path.local.sysfind('git')
-            if gitexe:
-                try:
-                    p = Popen(
-                        [str(gitexe), 'rev-parse', 'HEAD'],
-                        stdout=PIPE, stderr=PIPE, cwd=pypyroot
-                        )
-                except OSError, e:
-                    maywarn(e, 'Git')
-                    return default_retval
-                if p.wait() != 0:
-                    maywarn(p.stderr.read(), 'Git')
-                    return default_retval
-                revision_id = p.stdout.read().strip()[:12]
-                p = Popen(
-                    [str(gitexe), 'describe', '--tags', '--exact-match'],
-                    stdout=PIPE, stderr=PIPE, cwd=pypyroot
-                    )
-                if p.wait() != 0:
-                    p = Popen(
-                        [str(gitexe), 'branch'], stdout=PIPE, stderr=PIPE,
-                        cwd=pypyroot
-                        )
-                    if p.wait() != 0:
-                        maywarn(p.stderr.read(), 'Git')
-                        return 'PyPy', '?', revision_id
-                    branch = '?'
-                    for line in p.stdout.read().strip().split('\n'):
-                        if line.startswith('* '):
-                            branch = line[1:].strip()
-                            if branch == '(no branch)':
-                                branch = '?'
-                            break
-                    return 'PyPy', branch, revision_id
-                return 'PyPy', p.stdout.read().strip(), revision_id
+            return _get_git_version()
 
     # Fallback to trying Mercurial.
     if hgexe is None:
         hgexe = py.path.local.sysfind('hg')
 
-    if not os.path.isdir(os.path.join(pypyroot, '.hg')):
+    if os.path.isfile(os.path.join(pypyroot, '.hg_archival.txt')):
+        return _get_hg_archive_version(os.path.join(pypyroot, '.hg_archival.txt'))
+    elif not os.path.isdir(os.path.join(pypyroot, '.hg')):
         maywarn('Not running from a Mercurial repository!')
         return default_retval
     elif not hgexe:
         maywarn('Cannot find Mercurial command!')
         return default_retval
     else:
-        env = dict(os.environ)
-        # get Mercurial into scripting mode
-        env['HGPLAIN'] = '1'
-        # disable user configuration, extensions, etc.
-        env['HGRCPATH'] = os.devnull
+        return _get_hg_version(hgexe)
 
-        try:
-            p = Popen([str(hgexe), 'version', '-q'],
-                      stdout=PIPE, stderr=PIPE, env=env)
-        except OSError, e:
-            maywarn(e)
-            return default_retval
 
-        if not p.stdout.read().startswith('Mercurial Distributed SCM'):
-            maywarn('command does not identify itself as Mercurial')
-            return default_retval
+def _get_hg_version(hgexe):
+    env = dict(os.environ)
+    # get Mercurial into scripting mode
+    env['HGPLAIN'] = '1'
+    # disable user configuration, extensions, etc.
+    env['HGRCPATH'] = os.devnull
 
-        p = Popen([str(hgexe), 'id', '-i', pypyroot],
+    try:
+        p = Popen([str(hgexe), 'version', '-q'],
                   stdout=PIPE, stderr=PIPE, env=env)
-        hgid = p.stdout.read().strip()
+    except OSError, e:
+        maywarn(e)
+        return default_retval
+
+    if not p.stdout.read().startswith('Mercurial Distributed SCM'):
+        maywarn('command does not identify itself as Mercurial')
+        return default_retval
+
+    p = Popen([str(hgexe), 'id', '-i', pypyroot],
+              stdout=PIPE, stderr=PIPE, env=env)
+    hgid = p.stdout.read().strip()
+    maywarn(p.stderr.read())
+    if p.wait() != 0:
+        hgid = '?'
+
+    p = Popen([str(hgexe), 'id', '-t', pypyroot],
+              stdout=PIPE, stderr=PIPE, env=env)
+    hgtags = [t for t in p.stdout.read().strip().split() if t != 'tip']
+    maywarn(p.stderr.read())
+    if p.wait() != 0:
+        hgtags = ['?']
+
+    if hgtags:
+        return 'PyPy', hgtags[0], hgid
+    else:
+        # use the branch instead
+        p = Popen([str(hgexe), 'id', '-b', pypyroot],
+                  stdout=PIPE, stderr=PIPE, env=env)
+        hgbranch = p.stdout.read().strip()
         maywarn(p.stderr.read())
+
+        return 'PyPy', hgbranch, hgid
+
+
+def _get_hg_archive_version(path):
+    fp = open(path)
+    try:
+        data = dict(x.split(': ', 1) for x in fp.read().splitlines())
+    finally:
+        fp.close()
+    if 'tag' in data:
+        return 'PyPy', data['tag'], data['node']
+    else:
+        return 'PyPy', data['branch'], data['node']
+
+
+def _get_git_version():
+    #XXX: this function is a untested hack,
+    #     so the git mirror tav made will work
+    gitexe = py.path.local.sysfind('git')
+    if not gitexe:
+        return default_retval
+
+    try:
+        p = Popen(
+            [str(gitexe), 'rev-parse', 'HEAD'],
+            stdout=PIPE, stderr=PIPE, cwd=pypyroot
+            )
+    except OSError, e:
+        maywarn(e, 'Git')
+        return default_retval
+    if p.wait() != 0:
+        maywarn(p.stderr.read(), 'Git')
+        return default_retval
+    revision_id = p.stdout.read().strip()[:12]
+    p = Popen(
+        [str(gitexe), 'describe', '--tags', '--exact-match'],
+        stdout=PIPE, stderr=PIPE, cwd=pypyroot
+        )
+    if p.wait() != 0:
+        p = Popen(
+            [str(gitexe), 'branch'], stdout=PIPE, stderr=PIPE,
+            cwd=pypyroot
+            )
         if p.wait() != 0:
-            hgid = '?'
+            maywarn(p.stderr.read(), 'Git')
+            return 'PyPy', '?', revision_id
+        branch = '?'
+        for line in p.stdout.read().strip().split('\n'):
+            if line.startswith('* '):
+                branch = line[1:].strip()
+                if branch == '(no branch)':
+                    branch = '?'
+                break
+        return 'PyPy', branch, revision_id
+    return 'PyPy', p.stdout.read().strip(), revision_id
 
-        p = Popen([str(hgexe), 'id', '-t', pypyroot],
-                  stdout=PIPE, stderr=PIPE, env=env)
-        hgtags = [t for t in p.stdout.read().strip().split() if t != 'tip']
-        maywarn(p.stderr.read())
-        if p.wait() != 0:
-            hgtags = ['?']
 
-        if hgtags:
-            return 'PyPy', hgtags[0], hgid
-        else:
-            # use the branch instead
-            p = Popen([str(hgexe), 'id', '-b', pypyroot],
-                      stdout=PIPE, stderr=PIPE, env=env)
-            hgbranch = p.stdout.read().strip()
-            maywarn(p.stderr.read())
-
-            return 'PyPy', hgbranch, hgid
+if __name__ == '__main__':
+    print get_repo_version_info()

From noreply at buildbot.pypy.org  Fri Jan 20 18:01:54 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 20 Jan 2012 18:01:54 +0100 (CET)
Subject: [pypy-commit] pypy stm: Propagate the exception that occurs in a
	transaction.
Message-ID: <20120120170154.2B71A82CF8@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stm
Changeset: r51558:b4fd35482b14
Date: 2012-01-20 18:01 +0100
http://bitbucket.org/pypy/pypy/changeset/b4fd35482b14/

Log:	Propagate the exception that occurs in a transaction.

diff --git a/pypy/module/transaction/interp_transaction.py b/pypy/module/transaction/interp_transaction.py
--- a/pypy/module/transaction/interp_transaction.py
+++ b/pypy/module/transaction/interp_transaction.py
@@ -89,9 +89,13 @@
 
     @staticmethod
     def _run_in_transaction(pending):
-        space = state.space
-        space.call_args(pending.w_callback, pending.args)
-        # xxx exceptions?
+        if state.got_exception is not None:
+            return   # return early if there is already a 'got_exception'
+        try:
+            space = state.space
+            space.call_args(pending.w_callback, pending.args)
+        except Exception, e:
+            state.got_exception = e
 
 
 def add(space, w_callback, __args__):
@@ -159,6 +163,7 @@
     state.num_waiting_threads = 0
     state.finished = False
     state.running = True
+    state.got_exception = None
     #
     for i in range(state.num_threads):
         threadintf.start_new_thread(_run_thread, ())
@@ -170,3 +175,9 @@
     assert state.pending_lists.keys() == [state.main_thread_id]
     assert not state.is_locked_no_tasks_pending()
     state.running = False
+    #
+    # now re-raise the exception that we got in a transaction
+    if state.got_exception is not None:
+        e = state.got_exception
+        state.got_exception = None
+        raise e
diff --git a/pypy/module/transaction/test/test_transaction.py b/pypy/module/transaction/test/test_transaction.py
--- a/pypy/module/transaction/test/test_transaction.py
+++ b/pypy/module/transaction/test/test_transaction.py
@@ -41,3 +41,21 @@
             for index in range(7):
                 assert lst[start + index] == lst[start] + index
         assert seen == set([10, 20, 30])
+
+    def test_propagate_exception(self):
+        import transaction, time
+        lst = []
+        def f(n):
+            lst.append(n)
+            time.sleep(0.5)
+            raise ValueError(n)
+        transaction.add(f, 10)
+        transaction.add(f, 20)
+        transaction.add(f, 30)
+        try:
+            transaction.run()
+            assert 0, "should have raised ValueError"
+        except ValueError, e:
+            pass
+        assert len(lst) == 1
+        assert lst[0] == e.args[0]

From noreply at buildbot.pypy.org  Fri Jan 20 18:46:33 2012
From: noreply at buildbot.pypy.org (bivab)
Date: Fri, 20 Jan 2012 18:46:33 +0100 (CET)
Subject: [pypy-commit] pypy arm-backend-2: Forgot to pass the condition flag
	here. Argh!!
Message-ID: <20120120174633.C8A3682CF8@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider 
Branch: arm-backend-2
Changeset: r51559:0109b2cfd41f
Date: 2012-01-20 18:46 +0100
http://bitbucket.org/pypy/pypy/changeset/0109b2cfd41f/

Log:	Forgot to pass the condition flag here. Argh!!

diff --git a/pypy/jit/backend/arm/codebuilder.py b/pypy/jit/backend/arm/codebuilder.py
--- a/pypy/jit/backend/arm/codebuilder.py
+++ b/pypy/jit/backend/arm/codebuilder.py
@@ -179,7 +179,7 @@
     def BL(self, addr, c=cond.AL):
         target = rffi.cast(rffi.INT, addr)
         self.gen_load_int(reg.ip.value, target, cond=c)
-        self.BLX(reg.ip.value)
+        self.BLX(reg.ip.value, c)
 
     def BLX(self, reg, c=cond.AL):
         self.write32(c << 28 | 0x12FFF3 << 4 | (reg & 0xF))

From noreply at buildbot.pypy.org  Sat Jan 21 10:25:39 2012
From: noreply at buildbot.pypy.org (rguillebert)
Date: Sat, 21 Jan 2012 10:25:39 +0100 (CET)
Subject: [pypy-commit] pypy py3k: Port the generator applevel tests to py3k
Message-ID: <20120121092539.26C3282CB2@wyvern.cs.uni-duesseldorf.de>

Author: Romain Guillebert 
Branch: py3k
Changeset: r51560:328a6033f00a
Date: 2012-01-21 10:25 +0100
http://bitbucket.org/pypy/pypy/changeset/328a6033f00a/

Log:	Port the generator applevel tests to py3k

diff --git a/pypy/interpreter/test/test_generator.py b/pypy/interpreter/test/test_generator.py
--- a/pypy/interpreter/test/test_generator.py
+++ b/pypy/interpreter/test/test_generator.py
@@ -3,14 +3,14 @@
     def test_generator(self):
         def f():
             yield 1
-        assert f().next() == 1
+        assert next(f()) == 1
 
     def test_generator2(self):
         def f():
             yield 1
         g = f()
-        assert g.next() == 1
-        raises(StopIteration, g.next)
+        assert next(g) == 1
+        raises(StopIteration, next, g)
 
     def test_attributes(self):
         def f():
@@ -21,9 +21,9 @@
         assert g.__name__ == 'f'
         assert g.gi_frame is not None
         assert not g.gi_running
-        g.next()
+        next(g)
         assert not g.gi_running
-        raises(StopIteration, g.next)
+        raises(StopIteration, next, g)
         assert not g.gi_running
         assert g.gi_frame is None
         assert g.gi_code is f.func_code
@@ -43,13 +43,13 @@
 
     def test_generator5(self):
         d = {}
-        exec """if 1:
+        exec("""if 1:
         def f():
             v = (yield )
             yield v
         g = f()
-        g.next()
-        """ in d
+        next(g)
+        """, d, d)
         g = d['g']
         assert g.send(42) == 42
 
@@ -73,13 +73,13 @@
             except:
                 yield 3
         g = f()
-        assert g.next() == 1
+        assert next(g) == 1
         assert g.throw(NameError("Error")) == 3
-        raises(StopIteration, g.next)
+        raises(StopIteration, next, g)
 
     def test_throw4(self):
         d = {}
-        exec """if 1:
+        exec("""if 1:
         def f():
             try:
                 yield 1
@@ -87,12 +87,12 @@
             except:
                 yield 3
         g = f()
-        """ in d
+        """, d, d)
         g = d['g']
-        assert g.next() == 1
-        assert g.next() == 2
+        assert next(g) == 1
+        assert next(g) == 2
         assert g.throw(NameError("Error")) == 3
-        raises(StopIteration, g.next)
+        raises(StopIteration, next, g)
 
     def test_throw5(self):
         def f():
@@ -105,7 +105,7 @@
             except:
                 pass
         g = f()
-        g.next()
+        next(g)
         # String exceptions are not allowed anymore
         raises(TypeError, g.throw, "Error")
         assert g.throw(Exception) == 3
@@ -158,9 +158,9 @@
         def f():
             yield 1
         g = f()
-        res = g.next()
+        res = next(g)
         assert res == 1
-        raises(StopIteration, g.next)
+        raises(StopIteration, next, g)
         raises(NameError, g.throw, NameError)
 
     def test_close(self):
@@ -176,7 +176,7 @@
             except GeneratorExit:
                 raise StopIteration
         g = f()
-        g.next()
+        next(g)
         assert g.close() is None
 
     def test_close3(self):
@@ -186,7 +186,7 @@
             except GeneratorExit:
                 raise NameError
         g = f()
-        g.next()
+        next(g)
         raises(NameError, g.close)
 
     def test_close_fail(self):
@@ -196,22 +196,22 @@
             except GeneratorExit:
                 yield 2
         g = f()
-        g.next()
+        next(g)
         raises(RuntimeError, g.close)
 
     def test_close_on_collect(self):
         ## we need to exec it, else it won't run on python2.4
         d = {}
-        exec """
+        exec("""
         def f():
             try:
                 yield
             finally:
                 f.x = 42
-        """.strip() in d
+        """.strip(), d, d)
 
         g = d['f']()
-        g.next()
+        next(g)
         del g
         import gc
         gc.collect()
@@ -233,30 +233,31 @@
     def test_generator_propagate_stopiteration(self):
         def f():
             it = iter([1])
-            while 1: yield it.next()
+            while 1: yield next(it)
         g = f()
         assert [x for x in g] == [1]
 
     def test_generator_restart(self):
         def g():
-            i = me.next()
+            i = next(me)
             yield i
         me = g()
-        raises(ValueError, me.next)
+        raises(ValueError, next, me)
 
     def test_generator_expression(self):
-        exec "res = sum(i*i for i in range(5))"
-        assert res == 30
+        d = {}
+        exec("res = sum(i*i for i in range(5))", d, d)
+        assert d['res'] == 30
 
     def test_generator_expression_2(self):
         d = {}
-        exec """
+        exec("""
 def f():
     total = sum(i for i in [x for x in z])
     return total, x
 z = [1, 2, 7]
 res = f()
-""" in d
+""", d, d)
         assert d['res'] == (10, 7)
 
     def test_repr(self):
@@ -278,4 +279,4 @@
         def f():
             yield 1
             raise StopIteration
-        assert tuple(f()) == (1,)
\ No newline at end of file
+        assert tuple(f()) == (1,)

From noreply at buildbot.pypy.org  Sat Jan 21 10:42:02 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 21 Jan 2012 10:42:02 +0100 (CET)
Subject: [pypy-commit] pypy default: Fix a complexity bug: if 'b' is a large
	bytearray, 
Message-ID: <20120121094202.F3BC382CB2@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r51561:a9118db9d0bc
Date: 2012-01-21 10:41 +0100
http://bitbucket.org/pypy/pypy/changeset/a9118db9d0bc/

Log:	Fix a complexity bug: if 'b' is a large bytearray,

	 b[small_slice] = small_string

	used to take a time proportional to how long the large bytearray is.

diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -624,8 +624,8 @@
 
     if step == 1:
         assert start >= 0
-        assert slicelength >= 0
-        del items[start:start+slicelength]
+        if slicelength > 0:
+            del items[start:start+slicelength]
     else:
         n = len(items)
         i = start
@@ -662,10 +662,11 @@
             while i >= lim:
                 items[i] = items[i-delta]
                 i -= 1
-        elif start >= 0:
+        elif delta == 0:
+            pass
+        else:
+            assert start >= 0   # start<0 is only possible with slicelength==0
             del items[start:start+delta]
-        else:
-            assert delta==0   # start<0 is only possible with slicelength==0
     elif len2 != slicelength:  # No resize for extended slices
         raise operationerrfmt(space.w_ValueError, "attempt to "
               "assign sequence of size %d to extended slice of size %d",
diff --git a/pypy/objspace/std/test/test_bytearrayobject.py b/pypy/objspace/std/test/test_bytearrayobject.py
--- a/pypy/objspace/std/test/test_bytearrayobject.py
+++ b/pypy/objspace/std/test/test_bytearrayobject.py
@@ -1,5 +1,9 @@
+from pypy import conftest
 
 class AppTestBytesArray:
+    def setup_class(cls):
+        cls.w_runappdirect = cls.space.wrap(conftest.option.runappdirect)
+
     def test_basics(self):
         b = bytearray()
         assert type(b) is bytearray
@@ -439,3 +443,15 @@
     def test_reduce(self):
         assert bytearray('caf\xe9').__reduce__() == (
             bytearray, (u'caf\xe9', 'latin-1'), None)
+
+    def test_setitem_slice_performance(self):
+        # because of a complexity bug, this used to take forever on a
+        # translated pypy.  On CPython2.6 -A, it takes around 8 seconds.
+        if self.runappdirect:
+            count = 16*1024*1024
+        else:
+            count = 1024
+        b = bytearray(count)
+        for i in range(count):
+            b[i:i+1] = 'y'
+        assert str(b) == 'y' * count

From noreply at buildbot.pypy.org  Sat Jan 21 10:49:06 2012
From: noreply at buildbot.pypy.org (rguillebert)
Date: Sat, 21 Jan 2012 10:49:06 +0100 (CET)
Subject: [pypy-commit] pypy py3k: Some applevel code porting for
	test_interpreter
Message-ID: <20120121094906.6F92C82CB2@wyvern.cs.uni-duesseldorf.de>

Author: Romain Guillebert 
Branch: py3k
Changeset: r51562:76974f372468
Date: 2012-01-21 10:48 +0100
http://bitbucket.org/pypy/pypy/changeset/76974f372468/

Log:	Some applevel code porting for test_interpreter

diff --git a/pypy/interpreter/test/test_interpreter.py b/pypy/interpreter/test/test_interpreter.py
--- a/pypy/interpreter/test/test_interpreter.py
+++ b/pypy/interpreter/test/test_interpreter.py
@@ -265,7 +265,7 @@
         out = Out()
         try:
             sys.stdout = out
-            print 10
+            print(10)
             assert out.args == ['10','\n']
         finally:
             sys.stdout = save
@@ -281,15 +281,15 @@
                 self.data.append((type(x), x))
         sys.stdout = out = Out()
         try:
-            print unichr(0xa2)
+            print(unichr(0xa2))
             assert out.data == [(unicode, unichr(0xa2)), (str, "\n")]
             out.data = []
             out.encoding = "cp424"     # ignored!
-            print unichr(0xa2)
+            print(unichr(0xa2))
             assert out.data == [(unicode, unichr(0xa2)), (str, "\n")]
             del out.data[:]
             del out.encoding
-            print u"foo\t", u"bar\n", u"trick", u"baz\n"  # softspace handling
+            print "foo\t", "bar\n", "trick", "baz\n"  # softspace handling
             assert out.data == [(unicode, "foo\t"),
                                 (unicode, "bar\n"),
                                 (unicode, "trick"),
@@ -307,7 +307,7 @@
         def f(): f()
         try:
             f()
-        except RuntimeError, e:
+        except RuntimeError as e:
             assert str(e) == "maximum recursion depth exceeded"
         else:
             assert 0, "should have raised!"

From noreply at buildbot.pypy.org  Sat Jan 21 11:14:33 2012
From: noreply at buildbot.pypy.org (rguillebert)
Date: Sat, 21 Jan 2012 11:14:33 +0100 (CET)
Subject: [pypy-commit] pypy py3k: Port test_signal's applevel tests to py3k
Message-ID: <20120121101433.2616582CB2@wyvern.cs.uni-duesseldorf.de>

Author: Romain Guillebert 
Branch: py3k
Changeset: r51563:538110d792c0
Date: 2012-01-21 11:13 +0100
http://bitbucket.org/pypy/pypy/changeset/538110d792c0/

Log:	Port test_signal's applevel tests to py3k

diff --git a/pypy/module/signal/test/test_signal.py b/pypy/module/signal/test/test_signal.py
--- a/pypy/module/signal/test/test_signal.py
+++ b/pypy/module/signal/test/test_signal.py
@@ -174,7 +174,7 @@
             except OSError:
                 pass
             else:
-                raise AssertionError, "os.read(fd_read, 1) succeeded?"
+                raise AssertionError("os.read(fd_read, 1) succeeded?")
         #
         fd_read, fd_write = posix.pipe()
         flags = fcntl.fcntl(fd_write, fcntl.F_GETFL, 0)
@@ -189,7 +189,7 @@
             cannot_read()
             posix.kill(posix.getpid(), signal.SIGUSR1)
             res = posix.read(fd_read, 1)
-            assert res == '\x00'
+            assert res == b'\x00'
             cannot_read()
         finally:
             old_wakeup = signal.set_wakeup_fd(old_wakeup)
@@ -252,7 +252,7 @@
             signal(SIGALRM, handler)
             alarm(1)
             try:
-                s.accept()
+                s._accept()
             except Alarm:
                 pass
             else:

From noreply at buildbot.pypy.org  Sat Jan 21 11:18:18 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 21 Jan 2012 11:18:18 +0100 (CET)
Subject: [pypy-commit] pypy default: lists have the same performance issue
 as bytearray objects. Fix.
Message-ID: <20120121101818.9BC2A82CB2@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r51564:5c3a4d558079
Date: 2012-01-21 11:18 +0100
http://bitbucket.org/pypy/pypy/changeset/5c3a4d558079/

Log:	lists have the same performance issue as bytearray objects. Fix.

diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -804,10 +804,11 @@
                 while i >= lim:
                     items[i] = items[i-delta]
                     i -= 1
-            elif start >= 0:
+            elif delta == 0:
+                pass
+            else:
+                assert start >= 0 # start<0 is only possible with slicelength==0
                 del items[start:start+delta]
-            else:
-                assert delta==0   # start<0 is only possible with slicelength==0
         elif len2 != slicelength:  # No resize for extended slices
             raise operationerrfmt(self.space.w_ValueError, "attempt to "
                   "assign sequence of size %d to extended slice of size %d",
@@ -851,8 +852,8 @@
 
         if step == 1:
             assert start >= 0
-            assert slicelength >= 0
-            del items[start:start+slicelength]
+            if slicelength > 0:
+                del items[start:start+slicelength]
         else:
             n = len(items)
             i = start
diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py
--- a/pypy/objspace/std/test/test_listobject.py
+++ b/pypy/objspace/std/test/test_listobject.py
@@ -397,6 +397,7 @@
         on_cpython = (option.runappdirect and
                             not hasattr(sys, 'pypy_translation_info'))
         cls.w_on_cpython = cls.space.wrap(on_cpython)
+        cls.w_runappdirect = cls.space.wrap(option.runappdirect)
 
     def test_getstrategyfromlist_w(self):
         l0 = ["a", "2", "a", True]
@@ -898,6 +899,18 @@
         l[::-1] = l
         assert l == [6,5,4,3,2,1]
 
+    def test_setitem_slice_performance(self):
+        # because of a complexity bug, this used to take forever on a
+        # translated pypy.  On CPython2.6 -A, it takes around 5 seconds.
+        if self.runappdirect:
+            count = 16*1024*1024
+        else:
+            count = 1024
+        b = [None] * count
+        for i in range(count):
+            b[i:i+1] = ['y']
+        assert b == ['y'] * count
+
     def test_recursive_repr(self):
         l = []
         assert repr(l) == '[]'

From noreply at buildbot.pypy.org  Sat Jan 21 11:22:08 2012
From: noreply at buildbot.pypy.org (bivab)
Date: Sat, 21 Jan 2012 11:22:08 +0100 (CET)
Subject: [pypy-commit] extradoc extradoc: planning for today
Message-ID: <20120121102208.4A8AF82CB2@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider 
Branch: extradoc
Changeset: r4048:e398e08fdd6b
Date: 2012-01-21 11:21 +0100
http://bitbucket.org/pypy/extradoc/changeset/e398e08fdd6b/

Log:	planning for today

diff --git a/sprintinfo/leysin-winter-2012/planning.txt b/sprintinfo/leysin-winter-2012/planning.txt
--- a/sprintinfo/leysin-winter-2012/planning.txt
+++ b/sprintinfo/leysin-winter-2012/planning.txt
@@ -2,7 +2,6 @@
 People present
 --------------
 
-* Antonio Cuni
 * Armin Rigo
 * Romain Guillebert
 * David Schneider
@@ -14,19 +13,13 @@
 
 * review the JVM backend pull request (DONE)
 
-* py3k (romain, anto) (some progress)
+* py3k (romain) (little bit of progress)
 
-* ffistruct
-
-* Cython backend
-
-* Debug the ARM backend (bivab, armin around) (some progress)
+* Debug the ARM backend (bivab, armin around) (some more progress)
 
 * STM
   - refactored the RPython API (DONE)
   - app-level transaction module (DONE)
-  - start work on the GC
+  - start work on the GC (armin)
 
-* concurrent-marksweep GC
-
-* lightweight tracing experiment (everyone)
+* lightweight tracing experiment (DONE but still experimental)

From noreply at buildbot.pypy.org  Sat Jan 21 11:38:02 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 21 Jan 2012 11:38:02 +0100 (CET)
Subject: [pypy-commit] pypy default: Special-case in RPython the case of "x
	** 2.0".
Message-ID: <20120121103802.0FBD582CB2@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r51565:c5d041657831
Date: 2012-01-21 11:37 +0100
http://bitbucket.org/pypy/pypy/changeset/c5d041657831/

Log:	Special-case in RPython the case of "x ** 2.0".

diff --git a/pypy/rpython/lltypesystem/module/ll_math.py b/pypy/rpython/lltypesystem/module/ll_math.py
--- a/pypy/rpython/lltypesystem/module/ll_math.py
+++ b/pypy/rpython/lltypesystem/module/ll_math.py
@@ -292,6 +292,10 @@
     # deal directly with IEEE specials, to cope with problems on various
     # platforms whose semantics don't exactly match C99
 
+    if y == 2.0:
+        return x * x     # this is always a correct answer, and is relatively
+                         # common in user programs.
+
     if isnan(y):
         if x == 1.0:
             return 1.0   # 1**Nan = 1

From noreply at buildbot.pypy.org  Sat Jan 21 12:34:04 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 21 Jan 2012 12:34:04 +0100 (CET)
Subject: [pypy-commit] pypy default: Disable the error about "pypy
 translate.py" overwriting the same "pypy"
Message-ID: <20120121113404.7209D82CB2@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r51566:75795d0a00d2
Date: 2012-01-21 12:33 +0100
http://bitbucket.org/pypy/pypy/changeset/75795d0a00d2/

Log:	Disable the error about "pypy translate.py" overwriting the same
	"pypy" if we are actually only running it partly, e.g. "pypy
	--annotate translate.py".

diff --git a/pypy/translator/goal/translate.py b/pypy/translator/goal/translate.py
--- a/pypy/translator/goal/translate.py
+++ b/pypy/translator/goal/translate.py
@@ -293,17 +293,18 @@
             drv.exe_name = targetspec_dic['__name__'] + '-%(backend)s'
 
         # Double check to ensure we are not overwriting the current interpreter
-        try:
-            this_exe = py.path.local(sys.executable).new(ext='')
-            exe_name = drv.compute_exe_name()
-            samefile = this_exe.samefile(exe_name)
-            assert not samefile, (
-                'Output file %s is the currently running '
-                'interpreter (use --output=...)'% exe_name)
-        except EnvironmentError:
-            pass
+        goals = translateconfig.goals
+        if not goals or 'compile' in goals:
+            try:
+                this_exe = py.path.local(sys.executable).new(ext='')
+                exe_name = drv.compute_exe_name()
+                samefile = this_exe.samefile(exe_name)
+                assert not samefile, (
+                    'Output file %s is the currently running '
+                    'interpreter (use --output=...)'% exe_name)
+            except EnvironmentError:
+                pass
 
-        goals = translateconfig.goals
         try:
             drv.proceed(goals)
         finally:

From noreply at buildbot.pypy.org  Sat Jan 21 13:12:06 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Sat, 21 Jan 2012 13:12:06 +0100 (CET)
Subject: [pypy-commit] pypy default: add a few asserts,
	propagate axis where it works
Message-ID: <20120121121206.BA94182CB2@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: 
Changeset: r51567:5a6305c3c894
Date: 2012-01-21 13:58 +0200
http://bitbucket.org/pypy/pypy/changeset/5a6305c3c894/

Log:	add a few asserts, propagate axis where it works

diff --git a/lib_pypy/numpypy/core/fromnumeric.py b/lib_pypy/numpypy/core/fromnumeric.py
--- a/lib_pypy/numpypy/core/fromnumeric.py
+++ b/lib_pypy/numpypy/core/fromnumeric.py
@@ -149,6 +149,7 @@
            [5, 6]])
 
     """
+    assert order == 'C'
     if not hasattr(a, 'reshape'):
        a = numpypy.array(a)
     return a.reshape(newshape)
@@ -690,6 +691,7 @@
     1
 
     """
+    assert axis is None
     if not hasattr(a, 'argmax'):
         a = numpypy.array(a)
     return a.argmax()
@@ -705,6 +707,7 @@
         documentation.
 
     """
+    assert axis is None
     if not hasattr(a, 'argmin'):
         a = numpypy.array(a)
     return a.argmin()
@@ -1354,9 +1357,11 @@
     -128
 
     """
+    assert dtype is None
+    assert out is None
     if not hasattr(a, "sum"):
         a = numpypy.array(a)
-    return a.sum()
+    return a.sum(axis=axis)
 
 
 def product (a, axis=None, dtype=None, out=None):
@@ -1382,6 +1387,8 @@
     any : equivalent function
 
     """
+    assert axis is None
+    assert out is None
     if not hasattr(a, 'any'):
         a = numpypy.array(a)
     return a.any()
@@ -1396,6 +1403,8 @@
     numpy.all : Equivalent function; see for details.
 
     """
+    assert axis is None
+    assert out is None
     if not hasattr(a, 'all'):
         a = numpypy.array(a)
     return a.all()
@@ -1464,6 +1473,8 @@
     (191614240, 191614240)
 
     """
+    assert axis is None
+    assert out is None
     if not hasattr(a, 'any'):
         a = numpypy.array(a)
     return a.any()
@@ -1526,6 +1537,8 @@
     (28293632, 28293632, array([ True], dtype=bool))
 
     """
+    assert axis is None
+    assert out is None
     if not hasattr(a, 'all'):
         a = numpypy.array(a)
     return a.all()
@@ -1705,6 +1718,8 @@
     4.0
 
     """
+    assert axis is None
+    assert out is None
     if not hasattr(a, "max"):
         a = numpypy.array(a)
     return a.max()
@@ -1766,6 +1781,8 @@
 
     """
     # amin() is equivalent to min()
+    assert axis is None
+    assert out is None
     if not hasattr(a, 'min'):
         a = numpypy.array(a)
     return a.min()
@@ -2214,9 +2231,11 @@
     0.55000000074505806
 
     """
+    assert dtype is None
+    assert out is None
     if not hasattr(a, "mean"):
         a = numpypy.array(a)
-    return a.mean()
+    return a.mean(axis=axis)
 
 
 def std(a, axis=None, dtype=None, out=None, ddof=0):
@@ -2305,6 +2324,10 @@
     0.44999999925552653
 
     """
+    assert axis is None
+    assert dtype is None
+    assert out is None
+    assert ddof == 0
     if not hasattr(a, "std"):
         a = numpypy.array(a)
     return a.std()
@@ -2398,6 +2421,10 @@
     0.20250000000000001
 
     """
+    assert axis is None
+    assert dtype is None
+    assert out is None
+    assert ddof == 0
     if not hasattr(a, "var"):
         a = numpypy.array(a)
     return a.var()
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -728,7 +728,7 @@
         assert d[1] == 12
 
     def test_mean(self):
-        from _numpypy import array
+        from _numpypy import array, arange
         a = array(range(5))
         assert a.mean() == 2.0
         assert a[:4].mean() == 1.5
@@ -738,6 +738,7 @@
         assert a.mean(axis=0)[0, 0] == 35
         assert (b == array(range(35, 70), dtype=float).reshape(5, 7)).all()
         assert (a.mean(2) == array(range(0, 15), dtype=float).reshape(3, 5) * 7 + 3).all()
+        assert (arange(10).reshape(5, 2).mean(axis=1) == [0.5, 2.5, 4.5, 6.5, 8.5]).all()
 
     def test_sum(self):
         from _numpypy import array
@@ -1021,11 +1022,15 @@
         assert a[0].tolist() == [17.1, 27.2]
 
     def test_var(self):
-        from _numpypy import array
+        from _numpypy import array, arange
         a = array(range(10))
         assert a.var() == 8.25
         a = array([5.0])
         assert a.var() == 0.0
+        a = arange(10).reshape(5, 2)
+        assert a.var() == 8.25
+        #assert (a.var(0) == [8, 8]).all()
+        #assert (a.var(1) == [.25] * 5).all()
 
     def test_std(self):
         from _numpypy import array
diff --git a/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
--- a/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
+++ b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
@@ -18,8 +18,8 @@
         from numpypy import array, arange, argmin
         a = arange(6).reshape((2,3))
         assert argmin(a) == 0
-        assert (argmin(a, axis=0) == array([0, 0, 0])).all()
-        assert (argmin(a, axis=1) == array([0, 0])).all()
+        #assert (argmin(a, axis=0) == array([0, 0, 0])).all()
+        #assert (argmin(a, axis=1) == array([0, 0])).all()
         b = arange(6)
         b[1] = 0
         assert argmin(b) == 0
@@ -40,8 +40,8 @@
         assert sum([0.5, 1.5])== 2.0
         assert sum([[0, 1], [0, 5]]) == 6
         # assert sum([0.5, 0.7, 0.2, 1.5], dtype=int32) == 1
-        # assert (sum([[0, 1], [0, 5]], axis=0) == array([0, 6])).all()
-        # assert (sum([[0, 1], [0, 5]], axis=1) == array([1, 5])).all()
+        assert (sum([[0, 1], [0, 5]], axis=0) == array([0, 6])).all()
+        assert (sum([[0, 1], [0, 5]], axis=1) == array([1, 5])).all()
         # If the accumulator is too small, overflow occurs:
         # assert ones(128, dtype=int8).sum(dtype=int8) == -128
 
@@ -98,20 +98,22 @@
         from numpypy import array, var
         a = array([[1,2],[3,4]])
         assert var(a) == 1.25
-        # assert (np.var(a,0) == array([ 1.,  1.])).all()
-        # assert (np.var(a,1) == array([ 0.25,  0.25])).all()
+        #assert (var(a,0) == array([ 1.,  1.])).all()
+        #assert (var(a,1) == array([ 0.25,  0.25])).all()
 
     def test_std(self):
         from numpypy import array, std
         a = array([[1, 2], [3, 4]])
         assert std(a) ==  1.1180339887498949
-        # assert (std(a, axis=0) == array([ 1.,  1.])).all()
-        # assert (std(a, axis=1) == array([ 0.5,  0.5]).all()
+        #assert (std(a, axis=0) == array([ 1.,  1.])).all()
+        #assert (std(a, axis=1) == array([ 0.5,  0.5])).all()
 
     def test_mean(self):
-        from numpypy import array, mean
+        from numpypy import array, mean, arange
         assert mean(array(range(5))) == 2.0
         assert mean(range(5)) == 2.0
+        assert (mean(arange(10).reshape(5, 2), axis=0) == [4, 5]).all()
+        assert (mean(arange(10).reshape(5, 2), axis=1) == [0.5, 2.5, 4.5, 6.5, 8.5]).all()
 
     def test_reshape(self):
         from numpypy import arange, array, dtype, reshape

From noreply at buildbot.pypy.org  Sat Jan 21 13:12:07 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Sat, 21 Jan 2012 13:12:07 +0100 (CET)
Subject: [pypy-commit] pypy default: merge
Message-ID: <20120121121208.0029982CB2@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: 
Changeset: r51568:6a0c4cca12c4
Date: 2012-01-21 14:11 +0200
http://bitbucket.org/pypy/pypy/changeset/6a0c4cca12c4/

Log:	merge

diff --git a/pypy/translator/goal/translate.py b/pypy/translator/goal/translate.py
--- a/pypy/translator/goal/translate.py
+++ b/pypy/translator/goal/translate.py
@@ -293,17 +293,18 @@
             drv.exe_name = targetspec_dic['__name__'] + '-%(backend)s'
 
         # Double check to ensure we are not overwriting the current interpreter
-        try:
-            this_exe = py.path.local(sys.executable).new(ext='')
-            exe_name = drv.compute_exe_name()
-            samefile = this_exe.samefile(exe_name)
-            assert not samefile, (
-                'Output file %s is the currently running '
-                'interpreter (use --output=...)'% exe_name)
-        except EnvironmentError:
-            pass
+        goals = translateconfig.goals
+        if not goals or 'compile' in goals:
+            try:
+                this_exe = py.path.local(sys.executable).new(ext='')
+                exe_name = drv.compute_exe_name()
+                samefile = this_exe.samefile(exe_name)
+                assert not samefile, (
+                    'Output file %s is the currently running '
+                    'interpreter (use --output=...)'% exe_name)
+            except EnvironmentError:
+                pass
 
-        goals = translateconfig.goals
         try:
             drv.proceed(goals)
         finally:

From noreply at buildbot.pypy.org  Sat Jan 21 13:19:20 2012
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sat, 21 Jan 2012 13:19:20 +0100 (CET)
Subject: [pypy-commit] pypy py3k: hg merge default
Message-ID: <20120121121920.0480B82CB2@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: py3k
Changeset: r51569:2c98a32d1484
Date: 2012-01-20 20:53 +0100
http://bitbucket.org/pypy/pypy/changeset/2c98a32d1484/

Log:	hg merge default

diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -2,6 +2,9 @@
 *.py[co]
 *~
 .*.swp
+.idea
+.project
+.pydevproject
 
 syntax: regexp
 ^testresult$
diff --git a/lib_pypy/numpypy/__init__.py b/lib_pypy/numpypy/__init__.py
--- a/lib_pypy/numpypy/__init__.py
+++ b/lib_pypy/numpypy/__init__.py
@@ -1,2 +1,2 @@
 from _numpypy import *
-from .fromnumeric import *
+from .core import *
diff --git a/lib_pypy/numpypy/core/__init__.py b/lib_pypy/numpypy/core/__init__.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/numpypy/core/__init__.py
@@ -0,0 +1,1 @@
+from .fromnumeric import *
diff --git a/lib_pypy/numpypy/fromnumeric.py b/lib_pypy/numpypy/core/fromnumeric.py
rename from lib_pypy/numpypy/fromnumeric.py
rename to lib_pypy/numpypy/core/fromnumeric.py
--- a/lib_pypy/numpypy/fromnumeric.py
+++ b/lib_pypy/numpypy/core/fromnumeric.py
@@ -85,7 +85,7 @@
     array([4, 3, 6])
 
     """
-    raise NotImplemented('Waiting on interp level method')
+    raise NotImplementedError('Waiting on interp level method')
 
 
 # not deprecated --- copy if necessary, view otherwise
@@ -273,7 +273,7 @@
             [-1, -2, -3, -4, -5]]])
 
     """
-    raise NotImplemented('Waiting on interp level method')
+    raise NotImplementedError('Waiting on interp level method')
 
 
 def repeat(a, repeats, axis=None):
@@ -315,7 +315,7 @@
            [3, 4]])
 
     """
-    raise NotImplemented('Waiting on interp level method')
+    raise NotImplementedError('Waiting on interp level method')
 
 
 def put(a, ind, v, mode='raise'):
@@ -366,7 +366,7 @@
     array([ 0,  1,  2,  3, -5])
 
     """
-    raise NotImplemented('Waiting on interp level method')
+    raise NotImplementedError('Waiting on interp level method')
 
 
 def swapaxes(a, axis1, axis2):
@@ -410,7 +410,7 @@
             [3, 7]]])
 
     """
-    raise NotImplemented('Waiting on interp level method')
+    raise NotImplementedError('Waiting on interp level method')
 
 
 def transpose(a, axes=None):
@@ -451,8 +451,11 @@
     (2, 1, 3)
 
     """
-    raise NotImplemented('Waiting on interp level method')
-
+    if axes is not None:
+        raise NotImplementedError('No "axes" arg yet.')
+    if not hasattr(a, 'T'):
+        a = numpypy.array(a)
+    return a.T
 
 def sort(a, axis=-1, kind='quicksort', order=None):
     """
@@ -553,7 +556,7 @@
           dtype=[('name', '|S10'), ('height', '= self.size
 
@@ -157,6 +177,8 @@
                 offset += self.strides[i]
                 break
             else:
+                if i == self.dim:
+                    first_line = True
                 indices[i] = 0
                 offset -= self.backstrides[i]
         else:
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -2,14 +2,15 @@
 from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.interpreter.gateway import interp2app, NoneNotWrapped
 from pypy.interpreter.typedef import TypeDef, GetSetProperty
-from pypy.module.micronumpy import interp_ufuncs, interp_dtype, signature
+from pypy.module.micronumpy import interp_ufuncs, interp_dtype, signature,\
+     interp_boxes
 from pypy.module.micronumpy.strides import calculate_slice_strides
 from pypy.rlib import jit
 from pypy.rpython.lltypesystem import lltype, rffi
 from pypy.tool.sourcetools import func_with_new_name
 from pypy.rlib.rstring import StringBuilder
 from pypy.module.micronumpy.interp_iter import ArrayIterator, OneDimIterator,\
-     SkipLastAxisIterator
+     SkipLastAxisIterator, Chunk, ViewIterator
 
 numpy_driver = jit.JitDriver(
     greens=['shapelen', 'sig'],
@@ -39,7 +40,24 @@
     get_printable_location=signature.new_printable_location('slice'),
     name='numpy_slice',
 )
-
+count_driver = jit.JitDriver(
+    greens=['shapelen'],
+    virtualizables=['frame'],
+    reds=['s', 'frame', 'iter', 'arr'],
+    name='numpy_count'
+)
+filter_driver = jit.JitDriver(
+    greens=['shapelen', 'sig'],
+    virtualizables=['frame'],
+    reds=['concr', 'argi', 'ri', 'frame', 'v', 'res', 'self'],
+    name='numpy_filter',
+)
+filter_set_driver = jit.JitDriver(
+    greens=['shapelen', 'sig'],
+    virtualizables=['frame'],
+    reds=['idx', 'idxi', 'frame', 'arr'],
+    name='numpy_filterset',
+)
 
 def _find_shape_and_elems(space, w_iterable):
     shape = [space.len_w(w_iterable)]
@@ -270,6 +288,9 @@
     descr_gt = _binop_impl("greater")
     descr_ge = _binop_impl("greater_equal")
 
+    descr_and = _binop_impl("bitwise_and")
+    descr_or = _binop_impl("bitwise_or")
+
     def _binop_right_impl(ufunc_name):
         def impl(self, space, w_other):
             w_other = scalar_w(space,
@@ -287,11 +308,11 @@
     descr_rmod = _binop_right_impl("mod")
 
     def _reduce_ufunc_impl(ufunc_name, promote_to_largest=False):
-        def impl(self, space, w_dim=None):
-            if space.is_w(w_dim, space.w_None):
-                w_dim = space.wrap(-1)
+        def impl(self, space, w_axis=None):
+            if space.is_w(w_axis, space.w_None):
+                w_axis = space.wrap(-1)
             return getattr(interp_ufuncs.get(space), ufunc_name).reduce(space,
-                                        self, True, promote_to_largest, w_dim)
+                                        self, True, promote_to_largest, w_axis)
         return func_with_new_name(impl, "reduce_%s_impl" % ufunc_name)
 
     descr_sum = _reduce_ufunc_impl("add")
@@ -479,11 +500,69 @@
     def _prepare_slice_args(self, space, w_idx):
         if (space.isinstance_w(w_idx, space.w_int) or
             space.isinstance_w(w_idx, space.w_slice)):
-            return [space.decode_index4(w_idx, self.shape[0])]
-        return [space.decode_index4(w_item, self.shape[i]) for i, w_item in
+            return [Chunk(*space.decode_index4(w_idx, self.shape[0]))]
+        return [Chunk(*space.decode_index4(w_item, self.shape[i])) for i, w_item in
                 enumerate(space.fixedview(w_idx))]
 
+    def count_all_true(self, arr):
+        sig = arr.find_sig()
+        frame = sig.create_frame(self)
+        shapelen = len(arr.shape)
+        s = 0
+        iter = None
+        while not frame.done():
+            count_driver.jit_merge_point(arr=arr, frame=frame, iter=iter, s=s,
+                                         shapelen=shapelen)
+            iter = frame.get_final_iter()
+            s += arr.dtype.getitem_bool(arr.storage, iter.offset)
+            frame.next(shapelen)
+        return s
+
+    def getitem_filter(self, space, arr):
+        concr = arr.get_concrete()
+        size = self.count_all_true(concr)
+        res = W_NDimArray(size, [size], self.find_dtype())
+        ri = ArrayIterator(size)
+        shapelen = len(self.shape)
+        argi = concr.create_iter()
+        sig = self.find_sig()
+        frame = sig.create_frame(self)
+        v = None
+        while not frame.done():
+            filter_driver.jit_merge_point(concr=concr, argi=argi, ri=ri,
+                                          frame=frame, v=v, res=res, sig=sig,
+                                          shapelen=shapelen, self=self)
+            if concr.dtype.getitem_bool(concr.storage, argi.offset):
+                v = sig.eval(frame, self)
+                res.setitem(ri.offset, v)
+                ri = ri.next(1)
+            else:
+                ri = ri.next_no_increase(1)
+            argi = argi.next(shapelen)
+            frame.next(shapelen)
+        return res
+
+    def setitem_filter(self, space, idx, val):
+        size = self.count_all_true(idx)
+        arr = SliceArray([size], self.dtype, self, val)
+        sig = arr.find_sig()
+        shapelen = len(self.shape)
+        frame = sig.create_frame(arr)
+        idxi = idx.create_iter()
+        while not frame.done():
+            filter_set_driver.jit_merge_point(idx=idx, idxi=idxi, sig=sig,
+                                              frame=frame, arr=arr,
+                                              shapelen=shapelen)
+            if idx.dtype.getitem_bool(idx.storage, idxi.offset):
+                sig.eval(frame, arr)
+                frame.next_from_second(1)
+            frame.next_first(shapelen)
+            idxi = idxi.next(shapelen)
+
     def descr_getitem(self, space, w_idx):
+        if (isinstance(w_idx, BaseArray) and w_idx.shape == self.shape and
+            w_idx.find_dtype().is_bool_type()):
+            return self.getitem_filter(space, w_idx)
         if self._single_item_result(space, w_idx):
             concrete = self.get_concrete()
             item = concrete._index_of_single_item(space, w_idx)
@@ -493,6 +572,11 @@
 
     def descr_setitem(self, space, w_idx, w_value):
         self.invalidated()
+        if (isinstance(w_idx, BaseArray) and w_idx.shape == self.shape and
+            w_idx.find_dtype().is_bool_type()):
+            return self.get_concrete().setitem_filter(space,
+                                                      w_idx.get_concrete(),
+                                             convert_to_array(space, w_value))
         if self._single_item_result(space, w_idx):
             concrete = self.get_concrete()
             item = concrete._index_of_single_item(space, w_idx)
@@ -509,9 +593,8 @@
     def create_slice(self, chunks):
         shape = []
         i = -1
-        for i, (start_, stop, step, lgt) in enumerate(chunks):
-            if step != 0:
-                shape.append(lgt)
+        for i, chunk in enumerate(chunks):
+            chunk.extend_shape(shape)
         s = i + 1
         assert s >= 0
         shape += self.shape[s:]
@@ -569,19 +652,19 @@
             )
         return w_result
 
-    def descr_mean(self, space, w_dim=None):
-        if space.is_w(w_dim, space.w_None):
-            w_dim = space.wrap(-1)
+    def descr_mean(self, space, w_axis=None):
+        if space.is_w(w_axis, space.w_None):
+            w_axis = space.wrap(-1)
             w_denom = space.wrap(self.size)
         else:
-            dim = space.int_w(w_dim)
+            dim = space.int_w(w_axis)
             w_denom = space.wrap(self.shape[dim])
-        return space.div(self.descr_sum_promote(space, w_dim), w_denom)
+        return space.div(self.descr_sum_promote(space, w_axis), w_denom)
 
     def descr_var(self, space):
         # var = mean((values - mean(values)) ** 2)
         w_res = self.descr_sub(space, self.descr_mean(space, space.w_None))
-        assert isinstance(w_res, BaseArray) 
+        assert isinstance(w_res, BaseArray)
         w_res = w_res.descr_pow(space, space.wrap(2))
         assert isinstance(w_res, BaseArray)
         return w_res.descr_mean(space, space.w_None)
@@ -590,6 +673,10 @@
         # std(v) = sqrt(var(v))
         return interp_ufuncs.get(space).sqrt.call(space, [self.descr_var(space)])
 
+    def descr_fill(self, space, w_value):
+        concr = self.get_concrete_or_scalar()
+        concr.fill(space, w_value)
+
     def descr_nonzero(self, space):
         if self.size > 1:
             raise OperationError(space.w_ValueError, space.wrap(
@@ -682,6 +769,9 @@
     def copy(self, space):
         return Scalar(self.dtype, self.value)
 
+    def fill(self, space, w_value):
+        self.value = self.dtype.coerce(space, w_value)
+
     def create_sig(self):
         return signature.ScalarSignature(self.dtype)
 
@@ -717,8 +807,7 @@
                                          frame=frame,
                                          ri=ri,
                                          self=self, result=result)
-            result.dtype.setitem(result.storage, ri.offset,
-                                 sig.eval(frame, self))
+            result.setitem(ri.offset, sig.eval(frame, self))
             frame.next(shapelen)
             ri = ri.next(shapelen)
         return result
@@ -788,7 +877,7 @@
     Intermediate class for performing binary operations.
     """
     _immutable_fields_ = ['left', 'right']
-    
+
     def __init__(self, ufunc, name, shape, calc_dtype, res_dtype, left, right):
         VirtualArray.__init__(self, name, shape, res_dtype)
         self.ufunc = ufunc
@@ -828,7 +917,7 @@
     def __init__(self, shape, dtype, left, right):
         Call2.__init__(self, None, 'sliceloop', shape, dtype, dtype, left,
                        right)
-    
+
     def create_sig(self):
         lsig = self.left.create_sig()
         rsig = self.right.create_sig()
@@ -847,7 +936,7 @@
     when we'll make AxisReduce lazy
     """
     _immutable_fields_ = ['left', 'right']
-    
+
     def __init__(self, ufunc, name, shape, dtype, left, right, dim):
         Call2.__init__(self, ufunc, name, shape, dtype, dtype,
                        left, right)
@@ -919,14 +1008,14 @@
         if size < 1:
             builder.append('[]')
             return
-        elif size == 1:
-            builder.append(dtype.itemtype.str_format(self.getitem(0)))
-            return
         if size > 1000:
             # Once this goes True it does not go back to False for recursive
             # calls
             use_ellipsis = True
         ndims = len(self.shape)
+        if ndims == 0:
+            builder.append(dtype.itemtype.str_format(self.getitem(0)))
+            return
         i = 0
         builder.append('[')
         if ndims > 1:
@@ -938,7 +1027,7 @@
                             builder.append('\n' + indent)
                         else:
                             builder.append(indent)
-                    view = self.create_slice([(i, 0, 0, 1)]).get_concrete()
+                    view = self.create_slice([Chunk(i, 0, 0, 1)]).get_concrete()
                     view.to_str(space, comma, builder, indent=indent + ' ',
                                                     use_ellipsis=use_ellipsis)
                 if i < self.shape[0] - 1:
@@ -955,7 +1044,7 @@
                         builder.append(indent)
                 # create_slice requires len(chunks) > 1 in order to reduce
                 # shape
-                view = self.create_slice([(i, 0, 0, 1)]).get_concrete()
+                view = self.create_slice([Chunk(i, 0, 0, 1)]).get_concrete()
                 view.to_str(space, comma, builder, indent=indent + ' ',
                                                     use_ellipsis=use_ellipsis)
                 i += 1
@@ -1061,6 +1150,9 @@
         array.setslice(space, self)
         return array
 
+    def fill(self, space, w_value):
+        self.setslice(space, scalar_w(space, self.dtype, w_value))
+
 
 class ViewArray(ConcreteArray):
     def create_sig(self):
@@ -1081,6 +1173,10 @@
                                parent)
         self.start = start
 
+    def create_iter(self):
+        return ViewIterator(self.start, self.strides, self.backstrides,
+                            self.shape)
+
     def setshape(self, space, new_shape):
         if len(self.shape) < 1:
             return
@@ -1127,6 +1223,9 @@
         self.shape = new_shape
         self.calc_strides(new_shape)
 
+    def create_iter(self):
+        return ArrayIterator(self.size)
+
     def create_sig(self):
         return signature.ArraySignature(self.dtype)
 
@@ -1181,6 +1280,7 @@
     arr = W_NDimArray(size, shape[:], dtype=dtype, order=order)
     shapelen = len(shape)
     arr_iter = ArrayIterator(arr.size)
+    # XXX we might want to have a jitdriver here
     for i in range(len(elems_w)):
         w_elem = elems_w[i]
         dtype.setitem(arr.storage, arr_iter.offset,
@@ -1247,6 +1347,9 @@
     __gt__ = interp2app(BaseArray.descr_gt),
     __ge__ = interp2app(BaseArray.descr_ge),
 
+    __and__ = interp2app(BaseArray.descr_and),
+    __or__ = interp2app(BaseArray.descr_or),
+
     __repr__ = interp2app(BaseArray.descr_repr),
     __str__ = interp2app(BaseArray.descr_str),
     __array_interface__ = GetSetProperty(BaseArray.descr_array_iface),
@@ -1273,6 +1376,8 @@
     var = interp2app(BaseArray.descr_var),
     std = interp2app(BaseArray.descr_std),
 
+    fill = interp2app(BaseArray.descr_fill),
+
     copy = interp2app(BaseArray.descr_copy),
     reshape = interp2app(BaseArray.descr_reshape),
     tolist = interp2app(BaseArray.descr_tolist),
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -249,15 +249,16 @@
 
 
 class W_Ufunc2(W_Ufunc):
-    _immutable_fields_ = ["comparison_func", "func", "name"]
+    _immutable_fields_ = ["comparison_func", "func", "name", "int_only"]
     argcount = 2
 
     def __init__(self, func, name, promote_to_float=False, promote_bools=False,
-        identity=None, comparison_func=False):
+        identity=None, comparison_func=False, int_only=False):
 
         W_Ufunc.__init__(self, name, promote_to_float, promote_bools, identity)
         self.func = func
         self.comparison_func = comparison_func
+        self.int_only = int_only
 
     def call(self, space, args_w):
         from pypy.module.micronumpy.interp_numarray import (Call2,
@@ -268,6 +269,7 @@
         w_rhs = convert_to_array(space, w_rhs)
         calc_dtype = find_binop_result_dtype(space,
             w_lhs.find_dtype(), w_rhs.find_dtype(),
+            int_only=self.int_only,
             promote_to_float=self.promote_to_float,
             promote_bools=self.promote_bools,
         )
@@ -304,10 +306,12 @@
 
 
 def find_binop_result_dtype(space, dt1, dt2, promote_to_float=False,
-    promote_bools=False):
+    promote_bools=False, int_only=False):
     # dt1.num should be <= dt2.num
     if dt1.num > dt2.num:
         dt1, dt2 = dt2, dt1
+    if int_only and (not dt1.is_int_type() or not dt2.is_int_type()):
+        raise OperationError(space.w_TypeError, space.wrap("Unsupported types"))
     # Some operations promote op(bool, bool) to return int8, rather than bool
     if promote_bools and (dt1.kind == dt2.kind == interp_dtype.BOOLLTR):
         return interp_dtype.get_dtype_cache(space).w_int8dtype
@@ -420,6 +424,10 @@
             ("add", "add", 2, {"identity": 0}),
             ("subtract", "sub", 2),
             ("multiply", "mul", 2, {"identity": 1}),
+            ("bitwise_and", "bitwise_and", 2, {"identity": 1,
+                                               'int_only': True}),
+            ("bitwise_or", "bitwise_or", 2, {"identity": 0,
+                                             'int_only': True}),
             ("divide", "div", 2, {"promote_bools": True}),
             ("mod", "mod", 2, {"promote_bools": True}),
             ("power", "pow", 2, {"promote_bools": True}),
@@ -444,6 +452,7 @@
 
             ("fabs", "fabs", 1, {"promote_to_float": True}),
             ("floor", "floor", 1, {"promote_to_float": True}),
+            ("ceil", "ceil", 1, {"promote_to_float": True}),
             ("exp", "exp", 1, {"promote_to_float": True}),
 
             ('sqrt', 'sqrt', 1, {'promote_to_float': True}),
@@ -470,7 +479,7 @@
         extra_kwargs["identity"] = identity
 
         func = ufunc_dtype_caller(space, ufunc_name, op_name, argcount,
-            comparison_func=extra_kwargs.get("comparison_func", False)
+            comparison_func=extra_kwargs.get("comparison_func", False),
         )
         if argcount == 1:
             ufunc = W_Ufunc1(func, ufunc_name, **extra_kwargs)
diff --git a/pypy/module/micronumpy/signature.py b/pypy/module/micronumpy/signature.py
--- a/pypy/module/micronumpy/signature.py
+++ b/pypy/module/micronumpy/signature.py
@@ -82,6 +82,16 @@
         for i in range(len(self.iterators)):
             self.iterators[i] = self.iterators[i].next(shapelen)
 
+    @unroll_safe
+    def next_from_second(self, shapelen):
+        """ Don't increase the first iterator
+        """
+        for i in range(1, len(self.iterators)):
+            self.iterators[i] = self.iterators[i].next(shapelen)
+
+    def next_first(self, shapelen):
+        self.iterators[0] = self.iterators[0].next(shapelen)
+
     def get_final_iter(self):
         final_iter = promote(self.final_iter)
         if final_iter < 0:
diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py
--- a/pypy/module/micronumpy/strides.py
+++ b/pypy/module/micronumpy/strides.py
@@ -10,12 +10,12 @@
     rstart = start
     rshape = []
     i = -1
-    for i, (start_, stop, step, lgt) in enumerate(chunks):
-        if step != 0:
-            rstrides.append(strides[i] * step)
-            rbackstrides.append(strides[i] * (lgt - 1) * step)
-            rshape.append(lgt)
-        rstart += strides[i] * start_
+    for i, chunk in enumerate(chunks):
+        if chunk.step != 0:
+            rstrides.append(strides[i] * chunk.step)
+            rbackstrides.append(strides[i] * (chunk.lgt - 1) * chunk.step)
+            rshape.append(chunk.lgt)
+        rstart += strides[i] * chunk.start
     # add a reminder
     s = i + 1
     assert s >= 0
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -166,14 +166,11 @@
         # You can't subclass dtype
         raises(TypeError, type, "Foo", (dtype,), {})
 
-    def test_new(self):
-        import _numpypy as np
-        assert np.int_(4) == 4
-        assert np.float_(3.4) == 3.4
+    def test_aliases(self):
+        from _numpypy import dtype
 
-    def test_pow(self):
-        from _numpypy import int_
-        assert int_(4) ** 2 == 16
+        assert dtype("float") is dtype(float)
+
 
 class AppTestTypes(BaseNumpyAppTest):
     def test_abstract_types(self):
@@ -189,6 +186,15 @@
         raises(TypeError, numpy.floating, 0)
         raises(TypeError, numpy.inexact, 0)
 
+    def test_new(self):
+        import _numpypy as np
+        assert np.int_(4) == 4
+        assert np.float_(3.4) == 3.4
+
+    def test_pow(self):
+        from _numpypy import int_
+        assert int_(4) ** 2 == 16
+
     def test_bool(self):
         import _numpypy as numpy
 
@@ -318,7 +324,7 @@
         else:
             raises(OverflowError, numpy.int64, 9223372036854775807)
             raises(OverflowError, numpy.int64, '9223372036854775807')
-        
+
         raises(OverflowError, numpy.int64, 9223372036854775808)
         raises(OverflowError, numpy.int64, '9223372036854775808')
 
diff --git a/pypy/module/micronumpy/test/test_module.py b/pypy/module/micronumpy/test/test_module.py
--- a/pypy/module/micronumpy/test/test_module.py
+++ b/pypy/module/micronumpy/test/test_module.py
@@ -2,16 +2,11 @@
 
 
 class AppTestNumPyModule(BaseNumpyAppTest):
-    def test_mean(self):
-        from _numpypy import array, mean
-        assert mean(array(range(5))) == 2.0
-        assert mean(range(5)) == 2.0
-
     def test_average(self):
         from _numpypy import array, average
         assert average(range(10)) == 4.5
         assert average(array(range(10))) == 4.5
-        
+
     def test_sum(self):
         from _numpypy import array, sum
         assert sum(range(10)) == 45
@@ -21,7 +16,7 @@
         from _numpypy import array, min
         assert min(range(10)) == 0
         assert min(array(range(10))) == 0
-        
+
     def test_max(self):
         from _numpypy import array, max
         assert max(range(10)) == 9
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -2,6 +2,7 @@
 import py
 from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
 from pypy.module.micronumpy.interp_numarray import W_NDimArray, shape_agreement
+from pypy.module.micronumpy.interp_iter import Chunk
 from pypy.module.micronumpy import signature
 from pypy.interpreter.error import OperationError
 from pypy.conftest import gettestobjspace
@@ -37,53 +38,54 @@
 
     def test_create_slice_f(self):
         a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'F')
-        s = a.create_slice([(3, 0, 0, 1)])
+        s = a.create_slice([Chunk(3, 0, 0, 1)])
         assert s.start == 3
         assert s.strides == [10, 50]
         assert s.backstrides == [40, 100]
-        s = a.create_slice([(1, 9, 2, 4)])
+        s = a.create_slice([Chunk(1, 9, 2, 4)])
         assert s.start == 1
         assert s.strides == [2, 10, 50]
         assert s.backstrides == [6, 40, 100]
-        s = a.create_slice([(1, 5, 3, 2), (1, 2, 1, 1), (1, 0, 0, 1)])
+        s = a.create_slice([Chunk(1, 5, 3, 2), Chunk(1, 2, 1, 1), Chunk(1, 0, 0, 1)])
         assert s.shape == [2, 1]
         assert s.strides == [3, 10]
         assert s.backstrides == [3, 0]
-        s = a.create_slice([(0, 10, 1, 10), (2, 0, 0, 1)])
+        s = a.create_slice([Chunk(0, 10, 1, 10), Chunk(2, 0, 0, 1)])
         assert s.start == 20
         assert s.shape == [10, 3]
 
     def test_create_slice_c(self):
         a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'C')
-        s = a.create_slice([(3, 0, 0, 1)])
+        s = a.create_slice([Chunk(3, 0, 0, 1)])
         assert s.start == 45
         assert s.strides == [3, 1]
         assert s.backstrides == [12, 2]
-        s = a.create_slice([(1, 9, 2, 4)])
+        s = a.create_slice([Chunk(1, 9, 2, 4)])
         assert s.start == 15
         assert s.strides == [30, 3, 1]
         assert s.backstrides == [90, 12, 2]
-        s = a.create_slice([(1, 5, 3, 2), (1, 2, 1, 1), (1, 0, 0, 1)])
+        s = a.create_slice([Chunk(1, 5, 3, 2), Chunk(1, 2, 1, 1),
+                            Chunk(1, 0, 0, 1)])
         assert s.start == 19
         assert s.shape == [2, 1]
         assert s.strides == [45, 3]
         assert s.backstrides == [45, 0]
-        s = a.create_slice([(0, 10, 1, 10), (2, 0, 0, 1)])
+        s = a.create_slice([Chunk(0, 10, 1, 10), Chunk(2, 0, 0, 1)])
         assert s.start == 6
         assert s.shape == [10, 3]
 
     def test_slice_of_slice_f(self):
         a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'F')
-        s = a.create_slice([(5, 0, 0, 1)])
+        s = a.create_slice([Chunk(5, 0, 0, 1)])
         assert s.start == 5
-        s2 = s.create_slice([(3, 0, 0, 1)])
+        s2 = s.create_slice([Chunk(3, 0, 0, 1)])
         assert s2.shape == [3]
         assert s2.strides == [50]
         assert s2.parent is a
         assert s2.backstrides == [100]
         assert s2.start == 35
-        s = a.create_slice([(1, 5, 3, 2)])
-        s2 = s.create_slice([(0, 2, 1, 2), (2, 0, 0, 1)])
+        s = a.create_slice([Chunk(1, 5, 3, 2)])
+        s2 = s.create_slice([Chunk(0, 2, 1, 2), Chunk(2, 0, 0, 1)])
         assert s2.shape == [2, 3]
         assert s2.strides == [3, 50]
         assert s2.backstrides == [3, 100]
@@ -91,16 +93,16 @@
 
     def test_slice_of_slice_c(self):
         a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), order='C')
-        s = a.create_slice([(5, 0, 0, 1)])
+        s = a.create_slice([Chunk(5, 0, 0, 1)])
         assert s.start == 15 * 5
-        s2 = s.create_slice([(3, 0, 0, 1)])
+        s2 = s.create_slice([Chunk(3, 0, 0, 1)])
         assert s2.shape == [3]
         assert s2.strides == [1]
         assert s2.parent is a
         assert s2.backstrides == [2]
         assert s2.start == 5 * 15 + 3 * 3
-        s = a.create_slice([(1, 5, 3, 2)])
-        s2 = s.create_slice([(0, 2, 1, 2), (2, 0, 0, 1)])
+        s = a.create_slice([Chunk(1, 5, 3, 2)])
+        s2 = s.create_slice([Chunk(0, 2, 1, 2), Chunk(2, 0, 0, 1)])
         assert s2.shape == [2, 3]
         assert s2.strides == [45, 1]
         assert s2.backstrides == [45, 2]
@@ -108,14 +110,14 @@
 
     def test_negative_step_f(self):
         a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'F')
-        s = a.create_slice([(9, -1, -2, 5)])
+        s = a.create_slice([Chunk(9, -1, -2, 5)])
         assert s.start == 9
         assert s.strides == [-2, 10, 50]
         assert s.backstrides == [-8, 40, 100]
 
     def test_negative_step_c(self):
         a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), order='C')
-        s = a.create_slice([(9, -1, -2, 5)])
+        s = a.create_slice([Chunk(9, -1, -2, 5)])
         assert s.start == 135
         assert s.strides == [-30, 3, 1]
         assert s.backstrides == [-120, 12, 2]
@@ -124,7 +126,7 @@
         a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'F')
         r = a._index_of_single_item(self.space, self.newtuple(1, 2, 2))
         assert r == 1 + 2 * 10 + 2 * 50
-        s = a.create_slice([(0, 10, 1, 10), (2, 0, 0, 1)])
+        s = a.create_slice([Chunk(0, 10, 1, 10), Chunk(2, 0, 0, 1)])
         r = s._index_of_single_item(self.space, self.newtuple(1, 0))
         assert r == a._index_of_single_item(self.space, self.newtuple(1, 2, 0))
         r = s._index_of_single_item(self.space, self.newtuple(1, 1))
@@ -134,7 +136,7 @@
         a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'C')
         r = a._index_of_single_item(self.space, self.newtuple(1, 2, 2))
         assert r == 1 * 3 * 5 + 2 * 3 + 2
-        s = a.create_slice([(0, 10, 1, 10), (2, 0, 0, 1)])
+        s = a.create_slice([Chunk(0, 10, 1, 10), Chunk(2, 0, 0, 1)])
         r = s._index_of_single_item(self.space, self.newtuple(1, 0))
         assert r == a._index_of_single_item(self.space, self.newtuple(1, 2, 0))
         r = s._index_of_single_item(self.space, self.newtuple(1, 1))
@@ -726,15 +728,16 @@
         assert d[1] == 12
 
     def test_mean(self):
-        from _numpypy import array, mean
+        from _numpypy import array
         a = array(range(5))
         assert a.mean() == 2.0
         assert a[:4].mean() == 1.5
         a = array(range(105)).reshape(3, 5, 7)
-        b = mean(a, axis=0)
-        b[0,0]==35.
+        b = a.mean(axis=0)
+        b[0, 0]==35.
+        assert a.mean(axis=0)[0, 0] == 35
         assert (b == array(range(35, 70), dtype=float).reshape(5, 7)).all()
-        assert (mean(a, 2) == array(range(0, 15), dtype=float).reshape(3, 5) * 7 + 3).all()
+        assert (a.mean(2) == array(range(0, 15), dtype=float).reshape(3, 5) * 7 + 3).all()
 
     def test_sum(self):
         from _numpypy import array
@@ -755,6 +758,7 @@
         assert array([]).sum() == 0.0
         raises(ValueError, 'array([]).max()')
         assert (a.sum(0) == [30, 35, 40]).all()
+        assert (a.sum(axis=0) == [30, 35, 40]).all()
         assert (a.sum(1) == [3, 12, 21, 30, 39]).all()
         assert (a.max(0) == [12, 13, 14]).all()
         assert (a.max(1) == [2, 5, 8, 11, 14]).all()
@@ -769,6 +773,8 @@
         assert ((a + a).T.sum(2).T == (a + a).sum(0)).all()
         assert (a.reshape(1,-1).sum(0) == range(105)).all()
         assert (a.reshape(1,-1).sum(1) == 5460)
+        assert (array([[1,2],[3,4]]).prod(0) == [3, 8]).all()
+        assert (array([[1,2],[3,4]]).prod(1) == [2, 12]).all()
 
     def test_identity(self):
         from _numpypy import identity, array
@@ -1298,6 +1304,56 @@
         assert isinstance(i['data'][0], int)
         raises(TypeError, getattr, array(3), '__array_interface__')
 
+    def test_array_indexing_one_elem(self):
+        skip("not yet")
+        from _numpypy import array, arange
+        raises(IndexError, 'arange(3)[array([3.5])]')
+        a = arange(3)[array([1])]
+        assert a == 1
+        assert a[0] == 1
+        raises(IndexError,'arange(3)[array([15])]')
+        assert arange(3)[array([-3])] == 0
+        raises(IndexError,'arange(3)[array([-15])]')
+        assert arange(3)[array(1)] == 1
+
+    def test_fill(self):
+        from _numpypy import array
+        a = array([1, 2, 3])
+        a.fill(10)
+        assert (a == [10, 10, 10]).all()
+        a.fill(False)
+        assert (a == [0, 0, 0]).all()
+        b = a[:1]
+        b.fill(4)
+        assert (b == [4]).all()
+        assert (a == [4, 0, 0]).all()
+
+        c = b + b
+        c.fill(27)
+        assert (c == [27]).all()
+
+        d = array(10)
+        d.fill(100)
+        assert d == 100
+
+    def test_array_indexing_bool(self):
+        from _numpypy import arange
+        a = arange(10)
+        assert (a[a > 3] == [4, 5, 6, 7, 8, 9]).all()
+        a = arange(10).reshape(5, 2)
+        assert (a[a > 3] == [4, 5, 6, 7, 8, 9]).all()
+        assert (a[a & 1 == 1] == [1, 3, 5, 7, 9]).all()
+
+    def test_array_indexing_bool_setitem(self):
+        from _numpypy import arange, array
+        a = arange(6)
+        a[a > 3] = 15
+        assert (a == [0, 1, 2, 3, 15, 15]).all()
+        a = arange(6).reshape(3, 2)
+        a[a & 1 == 1] = array([8, 9, 10])
+        assert (a == [[0, 8], [2, 9], [4, 10]]).all()
+
+
 
 class AppTestSupport(BaseNumpyAppTest):
     def setup_class(cls):
@@ -1437,9 +1493,11 @@
         assert repr(a) == "array(0.0)"
         a = array(0.2)
         assert repr(a) == "array(0.2)"
+        a = array([2])
+        assert repr(a) == "array([2])"
 
     def test_repr_multi(self):
-        from _numpypy import arange, zeros
+        from _numpypy import arange, zeros, array
         a = zeros((3, 4))
         assert repr(a) == '''array([[0.0, 0.0, 0.0, 0.0],
        [0.0, 0.0, 0.0, 0.0],
@@ -1462,6 +1520,9 @@
        [498, 999],
        [499, 1000],
        [500, 1001]])'''
+        a = arange(2).reshape((2,1))
+        assert repr(a) == '''array([[0],
+       [1]])'''
 
     def test_repr_slice(self):
         from _numpypy import array, zeros
@@ -1546,18 +1607,3 @@
         a = arange(0, 0.8, 0.1)
         assert len(a) == 8
         assert arange(False, True, True).dtype is dtype(int)
-
-
-class AppTestRanges(BaseNumpyAppTest):
-    def test_app_reshape(self):
-        from _numpypy import arange, array, dtype, reshape
-        a = arange(12)
-        b = reshape(a, (3, 4))
-        assert b.shape == (3, 4)
-        a = range(12)
-        b = reshape(a, (3, 4))
-        assert b.shape == (3, 4)
-        a = array(range(105)).reshape(3, 5, 7)
-        assert a.reshape(1, -1).shape == (1, 105)
-        assert a.reshape(1, 1, -1).shape == (1, 1, 105)
-        assert a.reshape(-1, 1, 1).shape == (105, 1, 1)
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -190,14 +190,24 @@
         for i in range(3):
             assert c[i] == a[i] - b[i]
 
-    def test_floor(self):
-        from _numpypy import array, floor
-
-        reference = [-2.0, -1.0, 0.0, 1.0, 1.0]
-        a = array([-1.4, -1.0, 0.0, 1.0, 1.4])
+    def test_floorceil(self):
+        from _numpypy import array, floor, ceil
+        import math
+        reference = [-2.0, -2.0, -1.0, 0.0, 1.0, 1.0, 0]
+        a = array([-1.4, -1.5, -1.0, 0.0, 1.0, 1.4, 0.5])
         b = floor(a)
         for i in range(5):
             assert b[i] == reference[i]
+        reference = [-1.0, -1.0, -1.0, 0.0, 1.0, 2.0, 1.0]
+        a = array([-1.4, -1.5, -1.0, 0.0, 1.0, 1.4, 0.5])
+        b = ceil(a)
+        assert (reference == b).all()
+        inf = float("inf")
+        data = [1.5, 2.9999, -1.999, inf]
+        results = [math.floor(x) for x in data]
+        assert (floor(data) == results).all()
+        results = [math.ceil(x) for x in data]
+        assert (ceil(data) == results).all()
 
     def test_copysign(self):
         from _numpypy import array, copysign
@@ -238,7 +248,7 @@
             assert b[i] == math.sin(a[i])
 
         a = sin(array([True, False], dtype=bool))
-        assert abs(a[0] - sin(1)) < 1e-7 # a[0] will be less precise
+        assert abs(a[0] - sin(1)) < 1e-7  # a[0] will be less precise
         assert a[1] == 0.0
 
     def test_cos(self):
@@ -259,7 +269,6 @@
         for i in range(len(a)):
             assert b[i] == math.tan(a[i])
 
-
     def test_arcsin(self):
         import math
         from _numpypy import array, arcsin
@@ -283,7 +292,6 @@
         for i in range(len(a)):
             assert b[i] == math.acos(a[i])
 
-
         a = array([-10, -1.5, -1.01, 1.01, 1.5, 10, float('nan'), float('inf'), float('-inf')])
         b = arccos(a)
         for f in b:
@@ -347,11 +355,20 @@
         raises(ValueError, maximum.reduce, [])
 
     def test_reduceND(self):
-        from numpypy import add, arange
+        from _numpypy import add, arange
         a = arange(12).reshape(3, 4)
         assert (add.reduce(a, 0) == [12, 15, 18, 21]).all()
         assert (add.reduce(a, 1) == [6.0, 22.0, 38.0]).all()
 
+    def test_bitwise(self):
+        from _numpypy import bitwise_and, bitwise_or, arange, array
+        a = arange(6).reshape(2, 3)
+        assert (a & 1 == [[0, 1, 0], [1, 0, 1]]).all()
+        assert (a & 1 == bitwise_and(a, 1)).all()
+        assert (a | 1 == [[1, 1, 3], [3, 5, 5]]).all()
+        assert (a | 1 == bitwise_or(a, 1)).all()
+        raises(TypeError, 'array([1.0]) & 1')
+
     def test_comparisons(self):
         import operator
         from _numpypy import equal, not_equal, less, less_equal, greater, greater_equal
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -217,6 +217,7 @@
         # This is the sum of the ops for both loops, however if you remove the
         # optimization then you end up with 2 float_adds, so we can still be
         # sure it was optimized correctly.
+        py.test.skip("too fragile")
         self.check_resops({'setinteriorfield_raw': 4, 'getfield_gc': 22,
                            'getarrayitem_gc': 4, 'getarrayitem_gc_pure': 2,
                            'getfield_gc_pure': 8,
@@ -349,7 +350,8 @@
         self.check_trace_count(1)
         self.check_simple_loop({'getinteriorfield_raw': 2, 'float_add': 1,
                                 'setinteriorfield_raw': 1, 'int_add': 2,
-                                'int_eq': 1, 'guard_false': 1, 'jump': 1})
+                                'int_eq': 1, 'guard_false': 1, 'jump': 1,
+                                'arraylen_gc': 1})
 
     def define_virtual_slice():
         return """
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -94,6 +94,9 @@
             width, storage, i, offset
         ))
 
+    def read_bool(self, storage, width, i, offset):
+        raise NotImplementedError
+
     def store(self, storage, width, i, offset, box):
         value = self.unbox(box)
         libffi.array_setitem(clibffi.cast_type_to_ffitype(self.T),
@@ -168,6 +171,7 @@
     @simple_binary_op
     def min(self, v1, v2):
         return min(v1, v2)
+    
 
 class Bool(BaseType, Primitive):
     T = lltype.Bool
@@ -185,6 +189,11 @@
         else:
             return self.False
 
+
+    def read_bool(self, storage, width, i, offset):
+        return libffi.array_getitem(clibffi.cast_type_to_ffitype(self.T),
+                                    width, storage, i, offset)
+
     def coerce_subtype(self, space, w_subtype, w_item):
         # Doesn't return subclasses so it can return the constants.
         return self._coerce(space, w_item)
@@ -253,6 +262,14 @@
             assert v == 0
             return 0
 
+    @simple_binary_op
+    def bitwise_and(self, v1, v2):
+        return v1 & v2
+
+    @simple_binary_op
+    def bitwise_or(self, v1, v2):
+        return v1 | v2
+
 class Int8(BaseType, Integer):
     T = rffi.SIGNEDCHAR
     BoxType = interp_boxes.W_Int8Box
@@ -374,6 +391,10 @@
         return math.floor(v)
 
     @simple_unary_op
+    def ceil(self, v):
+        return math.ceil(v)
+
+    @simple_unary_op
     def exp(self, v):
         try:
             return math.exp(v)
@@ -436,4 +457,4 @@
 class Float64(BaseType, Float):
     T = rffi.DOUBLE
     BoxType = interp_boxes.W_Float64Box
-    format_code = "d"
\ No newline at end of file
+    format_code = "d"
diff --git a/pypy/module/pypyjit/__init__.py b/pypy/module/pypyjit/__init__.py
--- a/pypy/module/pypyjit/__init__.py
+++ b/pypy/module/pypyjit/__init__.py
@@ -11,6 +11,7 @@
         'set_optimize_hook': 'interp_resop.set_optimize_hook',
         'set_abort_hook': 'interp_resop.set_abort_hook',
         'ResOperation': 'interp_resop.WrappedOp',
+        'DebugMergePoint': 'interp_resop.DebugMergePoint',
         'Box': 'interp_resop.WrappedBox',
     }
 
diff --git a/pypy/module/pypyjit/interp_resop.py b/pypy/module/pypyjit/interp_resop.py
--- a/pypy/module/pypyjit/interp_resop.py
+++ b/pypy/module/pypyjit/interp_resop.py
@@ -1,5 +1,6 @@
 
-from pypy.interpreter.typedef import TypeDef, GetSetProperty
+from pypy.interpreter.typedef import (TypeDef, GetSetProperty,
+     interp_attrproperty, interp_attrproperty_w)
 from pypy.interpreter.baseobjspace import Wrappable
 from pypy.interpreter.gateway import unwrap_spec, interp2app, NoneNotWrapped
 from pypy.interpreter.pycode import PyCode
@@ -10,6 +11,7 @@
 from pypy.jit.metainterp.resoperation import rop, AbstractResOp
 from pypy.rlib.nonconst import NonConstant
 from pypy.rlib import jit_hooks
+from pypy.module.pypyjit.interp_jit import pypyjitdriver
 
 class Cache(object):
     in_recursion = False
@@ -111,13 +113,24 @@
 
 def wrap_oplist(space, logops, operations, ops_offset=None):
     l_w = []
+    jitdrivers_sd = logops.metainterp_sd.jitdrivers_sd
     for op in operations:
         if ops_offset is None:
             ofs = -1
         else:
             ofs = ops_offset.get(op, 0)
-        l_w.append(WrappedOp(jit_hooks._cast_to_gcref(op), ofs,
-                             logops.repr_of_resop(op)))
+        if op.opnum == rop.DEBUG_MERGE_POINT:
+            jd_sd = jitdrivers_sd[op.getarg(0).getint()]
+            greenkey = op.getarglist()[2:]
+            repr = jd_sd.warmstate.get_location_str(greenkey)
+            w_greenkey = wrap_greenkey(space, jd_sd.jitdriver, greenkey, repr)
+            l_w.append(DebugMergePoint(space, jit_hooks._cast_to_gcref(op),
+                                       logops.repr_of_resop(op),
+                                       jd_sd.jitdriver.name,
+                                       w_greenkey))
+        else:
+            l_w.append(WrappedOp(jit_hooks._cast_to_gcref(op), ofs,
+                                 logops.repr_of_resop(op)))
     return l_w
 
 class WrappedBox(Wrappable):
@@ -150,6 +163,15 @@
         llres = res.llbox
     return WrappedOp(jit_hooks.resop_new(num, args, llres), offset, repr)
 
+ at unwrap_spec(repr=str, jd_name=str)
+def descr_new_dmp(space, w_tp, w_args, repr, jd_name, w_greenkey):
+    args = [space.interp_w(WrappedBox, w_arg).llbox for w_arg in
+            space.listview(w_args)]
+    num = rop.DEBUG_MERGE_POINT
+    return DebugMergePoint(space,
+                           jit_hooks.resop_new(num, args, jit_hooks.emptyval()),
+                           repr, jd_name, w_greenkey)
+
 class WrappedOp(Wrappable):
     """ A class representing a single ResOperation, wrapped nicely
     """
@@ -182,6 +204,25 @@
         box = space.interp_w(WrappedBox, w_box)
         jit_hooks.resop_setresult(self.op, box.llbox)
 
+class DebugMergePoint(WrappedOp):
+    def __init__(self, space, op, repr_of_resop, jd_name, w_greenkey):
+        WrappedOp.__init__(self, op, -1, repr_of_resop)
+        self.w_greenkey = w_greenkey
+        self.jd_name = jd_name
+
+    def get_pycode(self, space):
+        if self.jd_name == pypyjitdriver.name:
+            return space.getitem(self.w_greenkey, space.wrap(0))
+        raise OperationError(space.w_AttributeError, space.wrap("This DebugMergePoint doesn't belong to the main Python JitDriver"))
+
+    def get_bytecode_no(self, space):
+        if self.jd_name == pypyjitdriver.name:
+            return space.getitem(self.w_greenkey, space.wrap(1))
+        raise OperationError(space.w_AttributeError, space.wrap("This DebugMergePoint doesn't belong to the main Python JitDriver"))
+
+    def get_jitdriver_name(self, space):
+        return space.wrap(self.jd_name)
+
 WrappedOp.typedef = TypeDef(
     'ResOperation',
     __doc__ = WrappedOp.__doc__,
@@ -195,3 +236,15 @@
                             WrappedOp.descr_setresult)
 )
 WrappedOp.acceptable_as_base_class = False
+
+DebugMergePoint.typedef = TypeDef(
+    'DebugMergePoint', WrappedOp.typedef,
+    __new__ = interp2app(descr_new_dmp),
+    greenkey = interp_attrproperty_w("w_greenkey", cls=DebugMergePoint),
+    pycode = GetSetProperty(DebugMergePoint.get_pycode),
+    bytecode_no = GetSetProperty(DebugMergePoint.get_bytecode_no),
+    jitdriver_name = GetSetProperty(DebugMergePoint.get_jitdriver_name),
+)
+DebugMergePoint.acceptable_as_base_class = False
+
+
diff --git a/pypy/module/pypyjit/policy.py b/pypy/module/pypyjit/policy.py
--- a/pypy/module/pypyjit/policy.py
+++ b/pypy/module/pypyjit/policy.py
@@ -127,7 +127,7 @@
                        'imp', 'sys', 'array', '_ffi', 'itertools', 'operator',
                        'posix', '_socket', '_sre', '_lsprof', '_weakref',
                        '__pypy__', 'cStringIO', '_collections', 'struct',
-                       'mmap', 'marshal']:
+                       'mmap', 'marshal', '_codecs']:
             if modname == 'pypyjit' and 'interp_resop' in rest:
                 return False
             return True
diff --git a/pypy/module/pypyjit/test/test_jit_hook.py b/pypy/module/pypyjit/test/test_jit_hook.py
--- a/pypy/module/pypyjit/test/test_jit_hook.py
+++ b/pypy/module/pypyjit/test/test_jit_hook.py
@@ -92,6 +92,7 @@
         cls.w_on_compile_bridge = space.wrap(interp2app(interp_on_compile_bridge))
         cls.w_on_abort = space.wrap(interp2app(interp_on_abort))
         cls.w_int_add_num = space.wrap(rop.INT_ADD)
+        cls.w_dmp_num = space.wrap(rop.DEBUG_MERGE_POINT)
         cls.w_on_optimize = space.wrap(interp2app(interp_on_optimize))
         cls.orig_oplist = oplist
 
@@ -117,6 +118,10 @@
         assert elem[2][2] == False
         assert len(elem[3]) == 4
         int_add = elem[3][0]
+        dmp = elem[3][1]
+        assert isinstance(dmp, pypyjit.DebugMergePoint)
+        assert dmp.pycode is self.f.func_code
+        assert dmp.greenkey == (self.f.func_code, 0, False)
         #assert int_add.name == 'int_add'
         assert int_add.num == self.int_add_num
         self.on_compile_bridge()
@@ -211,3 +216,18 @@
         assert op.getarg(0).getint() == 4
         op.result = box
         assert op.result.getint() == 1
+
+    def test_creation_dmp(self):
+        from pypyjit import DebugMergePoint, Box
+
+        def f():
+            pass
+
+        op = DebugMergePoint([Box(0)], 'repr', 'pypyjit', (f.func_code, 0, 0))
+        assert op.bytecode_no == 0
+        assert op.pycode is f.func_code
+        assert repr(op) == 'repr'
+        assert op.jitdriver_name == 'pypyjit'
+        assert op.num == self.dmp_num
+        op = DebugMergePoint([Box(0)], 'repr', 'notmain', ('str',))
+        raises(AttributeError, 'op.pycode')
diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py
--- a/pypy/module/sys/version.py
+++ b/pypy/module/sys/version.py
@@ -10,7 +10,7 @@
 CPYTHON_VERSION            = (3, 2, 2, "final", 0)   #XXX # sync patchlevel.h
 CPYTHON_API_VERSION        = 1013   #XXX # sync with include/modsupport.h
 
-PYPY_VERSION               = (1, 7, 1, "dev", 0)    #XXX # sync patchlevel.h
+PYPY_VERSION               = (1, 8, 1, "dev", 0)    #XXX # sync patchlevel.h
 
 if platform.name == 'msvc':
     COMPILER_INFO = 'MSC v.%d 32 bit' % (platform.version * 10 + 600)
diff --git a/lib_pypy/numpypy/test/test_fromnumeric.py b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
rename from lib_pypy/numpypy/test/test_fromnumeric.py
rename to pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
--- a/lib_pypy/numpypy/test/test_fromnumeric.py
+++ b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
@@ -1,7 +1,7 @@
-
 from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
 
-class AppTestFromNumeric(BaseNumpyAppTest):     
+
+class AppTestFromNumeric(BaseNumpyAppTest):
     def test_argmax(self):
         # tests taken from numpy/core/fromnumeric.py docstring
         from numpypy import array, arange, argmax
@@ -18,12 +18,12 @@
         from numpypy import array, arange, argmin
         a = arange(6).reshape((2,3))
         assert argmin(a) == 0
-        # assert (argmax(a, axis=0) == array([0, 0, 0])).all()
-        # assert (argmax(a, axis=1) == array([0, 0])).all()
+        assert (argmin(a, axis=0) == array([0, 0, 0])).all()
+        assert (argmin(a, axis=1) == array([0, 0])).all()
         b = arange(6)
         b[1] = 0
         assert argmin(b) == 0
-   
+
     def test_shape(self):
         # tests taken from numpy/core/fromnumeric.py docstring
         from numpypy import array, identity, shape
@@ -44,7 +44,7 @@
         # assert (sum([[0, 1], [0, 5]], axis=1) == array([1, 5])).all()
         # If the accumulator is too small, overflow occurs:
         # assert ones(128, dtype=int8).sum(dtype=int8) == -128
-                                 
+
     def test_amin(self):
         # tests taken from numpy/core/fromnumeric.py docstring
         from numpypy import array, arange, amin
@@ -86,14 +86,14 @@
         assert ndim([[1,2,3],[4,5,6]]) == 2
         assert ndim(array([[1,2,3],[4,5,6]])) == 2
         assert ndim(1) == 0
-    
+
     def test_rank(self):
         # tests taken from numpy/core/fromnumeric.py docstring
         from numpypy import array, rank
         assert rank([[1,2,3],[4,5,6]]) == 2
         assert rank(array([[1,2,3],[4,5,6]])) == 2
         assert rank(1) == 0
-    
+
     def test_var(self):
         from numpypy import array, var
         a = array([[1,2],[3,4]])
@@ -107,3 +107,31 @@
         assert std(a) ==  1.1180339887498949
         # assert (std(a, axis=0) == array([ 1.,  1.])).all()
         # assert (std(a, axis=1) == array([ 0.5,  0.5]).all()
+
+    def test_mean(self):
+        from numpypy import array, mean
+        assert mean(array(range(5))) == 2.0
+        assert mean(range(5)) == 2.0
+
+    def test_reshape(self):
+        from numpypy import arange, array, dtype, reshape
+        a = arange(12)
+        b = reshape(a, (3, 4))
+        assert b.shape == (3, 4)
+        a = range(12)
+        b = reshape(a, (3, 4))
+        assert b.shape == (3, 4)
+        a = array(range(105)).reshape(3, 5, 7)
+        assert reshape(a, (1, -1)).shape == (1, 105)
+        assert reshape(a, (1, 1, -1)).shape == (1, 1, 105)
+        assert reshape(a, (-1, 1, 1)).shape == (105, 1, 1)
+    
+    def test_transpose(self):   
+        from numpypy import arange, array, transpose, ones
+        x = arange(4).reshape((2,2))
+        assert (transpose(x) == array([[0, 2],[1, 3]])).all()
+        # Once axes argument is implemented, add more tests
+        raises(NotImplementedError, "transpose(x, axes=(1, 0, 2))")
+        # x = ones((1, 2, 3))
+        # assert transpose(x, (1, 0, 2)).shape == (2, 1, 3)
+
diff --git a/pypy/objspace/std/test/test_unicodeobject.py b/pypy/objspace/std/test/test_unicodeobject.py
--- a/pypy/objspace/std/test/test_unicodeobject.py
+++ b/pypy/objspace/std/test/test_unicodeobject.py
@@ -29,6 +29,8 @@
             assert type(a) == type(b)
         check(', '.join(['a']), 'a')
         raises(TypeError, ','.join, [b'a']) 
+        exc = raises(TypeError, ''.join, ['a', 2, 3])
+        assert 'sequence item 1' in str(e.value)
 
     def test_contains(self):
         assert '' in 'abc'
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -129,7 +129,7 @@
             return space.wrap(l[0])
         return space.wrap(w_self._value.join(l))
 
-    list_w = space.unpackiterable(w_list)
+    list_w = space.listview(w_list)
     size = len(list_w)
 
     if size == 0:
@@ -144,17 +144,23 @@
 
 def _unicode_join_many_items(space, w_self, list_w, size):
     self = w_self._value
-    sb = UnicodeBuilder()
+    prealloc_size = len(self) * (size - 1)
+    for i in range(size):
+        w_s = list_w[i]
+        try:
+            prealloc_size += len(space.unicode_w(w_s))
+        except OperationError, e:
+            if not e.match(space, space.w_TypeError):
+                raise
+            raise operationerrfmt(space.w_TypeError,
+                                  "sequence item %d: expected string, %s "
+                                  "found", i, space.type(w_s).getname(space))
+    sb = UnicodeBuilder(prealloc_size)
     for i in range(size):
         if self and i != 0:
             sb.append(self)
         w_s = list_w[i]
-        if not isinstance(w_s, W_UnicodeObject):
-            raise operationerrfmt(
-                space.w_TypeError,
-                "sequence item %d: expected string, %s "
-                "found", i, space.type(w_s).getname(space))
-        sb.append(w_s._value)
+        sb.append(space.unicode_w(w_s))
     return space.wrap(sb.build())
 
 def hash__Unicode(space, w_uni):
diff --git a/pypy/rlib/jit_hooks.py b/pypy/rlib/jit_hooks.py
--- a/pypy/rlib/jit_hooks.py
+++ b/pypy/rlib/jit_hooks.py
@@ -52,7 +52,10 @@
     from pypy.jit.metainterp.history import ResOperation
 
     args = [_cast_to_box(llargs[i]) for i in range(len(llargs))]
-    res = _cast_to_box(llres)
+    if llres:
+        res = _cast_to_box(llres)
+    else:
+        res = None
     return _cast_to_gcref(ResOperation(no, args, res))
 
 @register_helper(annmodel.SomePtr(llmemory.GCREF))
diff --git a/pypy/rlib/longlong2float.py b/pypy/rlib/longlong2float.py
--- a/pypy/rlib/longlong2float.py
+++ b/pypy/rlib/longlong2float.py
@@ -79,19 +79,23 @@
 longlong2float = rffi.llexternal(
     "pypy__longlong2float", [rffi.LONGLONG], rffi.DOUBLE,
     _callable=longlong2float_emulator, compilation_info=eci,
-    _nowrapper=True, elidable_function=True, sandboxsafe=True)
+    _nowrapper=True, elidable_function=True, sandboxsafe=True,
+    oo_primitive="pypy__longlong2float")
 
 float2longlong = rffi.llexternal(
     "pypy__float2longlong", [rffi.DOUBLE], rffi.LONGLONG,
     _callable=float2longlong_emulator, compilation_info=eci,
-    _nowrapper=True, elidable_function=True, sandboxsafe=True)
+    _nowrapper=True, elidable_function=True, sandboxsafe=True,
+    oo_primitive="pypy__float2longlong")
 
 uint2singlefloat = rffi.llexternal(
     "pypy__uint2singlefloat", [rffi.UINT], rffi.FLOAT,
     _callable=uint2singlefloat_emulator, compilation_info=eci,
-    _nowrapper=True, elidable_function=True, sandboxsafe=True)
+    _nowrapper=True, elidable_function=True, sandboxsafe=True,
+    oo_primitive="pypy__uint2singlefloat")
 
 singlefloat2uint = rffi.llexternal(
     "pypy__singlefloat2uint", [rffi.FLOAT], rffi.UINT,
     _callable=singlefloat2uint_emulator, compilation_info=eci,
-    _nowrapper=True, elidable_function=True, sandboxsafe=True)
+    _nowrapper=True, elidable_function=True, sandboxsafe=True,
+    oo_primitive="pypy__singlefloat2uint")
diff --git a/pypy/rlib/objectmodel.py b/pypy/rlib/objectmodel.py
--- a/pypy/rlib/objectmodel.py
+++ b/pypy/rlib/objectmodel.py
@@ -424,7 +424,7 @@
                   vobj.concretetype.TO._gckind == 'gc')
         else:
             from pypy.rpython.ootypesystem import ootype
-            ok = isinstance(vobj.concretetype, ootype.Instance)
+            ok = isinstance(vobj.concretetype, (ootype.Instance, ootype.BuiltinType))
         if not ok:
             from pypy.rpython.error import TyperError
             raise TyperError("compute_unique_id() cannot be applied to"
diff --git a/pypy/rpython/lltypesystem/rclass.py b/pypy/rpython/lltypesystem/rclass.py
--- a/pypy/rpython/lltypesystem/rclass.py
+++ b/pypy/rpython/lltypesystem/rclass.py
@@ -510,7 +510,13 @@
         ctype = inputconst(Void, self.object_type)
         cflags = inputconst(Void, flags)
         vlist = [ctype, cflags]
-        vptr = llops.genop('malloc', vlist,
+        cnonmovable = self.classdef.classdesc.read_attribute(
+            '_alloc_nonmovable_', Constant(False))
+        if cnonmovable.value:
+            opname = 'malloc_nonmovable'
+        else:
+            opname = 'malloc'
+        vptr = llops.genop(opname, vlist,
                            resulttype = Ptr(self.object_type))
         ctypeptr = inputconst(CLASSTYPE, self.rclass.getvtable())
         self.setfield(vptr, '__class__', ctypeptr, llops)
diff --git a/pypy/rpython/ootypesystem/ootype.py b/pypy/rpython/ootypesystem/ootype.py
--- a/pypy/rpython/ootypesystem/ootype.py
+++ b/pypy/rpython/ootypesystem/ootype.py
@@ -512,6 +512,7 @@
             "ll_append_char": Meth([CHARTP], Void),
             "ll_append": Meth([STRINGTP], Void),
             "ll_build": Meth([], STRINGTP),
+            "ll_getlength": Meth([], Signed),
             })
         self._setup_methods({})
 
@@ -1376,6 +1377,9 @@
     def _cast_to_object(self):
         return make_object(self)
 
+    def _identityhash(self):
+        return object.__hash__(self)
+
 class _string(_builtin_type):
 
     def __init__(self, STRING, value = ''):
@@ -1543,6 +1547,9 @@
         else:
             return make_unicode(u''.join(self._buf))
 
+    def ll_getlength(self):
+        return self.ll_build().ll_strlen()
+
 class _null_string_builder(_null_mixin(_string_builder), _string_builder):
     def __init__(self, STRING_BUILDER):
         self.__dict__["_TYPE"] = STRING_BUILDER
diff --git a/pypy/rpython/ootypesystem/rbuilder.py b/pypy/rpython/ootypesystem/rbuilder.py
--- a/pypy/rpython/ootypesystem/rbuilder.py
+++ b/pypy/rpython/ootypesystem/rbuilder.py
@@ -21,6 +21,10 @@
         builder.ll_append_char(char)
 
     @staticmethod
+    def ll_getlength(builder):
+        return builder.ll_getlength()
+
+    @staticmethod
     def ll_append(builder, string):
         builder.ll_append(string)
 
diff --git a/pypy/rpython/test/test_rbuilder.py b/pypy/rpython/test/test_rbuilder.py
--- a/pypy/rpython/test/test_rbuilder.py
+++ b/pypy/rpython/test/test_rbuilder.py
@@ -124,9 +124,5 @@
     pass
 
 class TestOOtype(BaseTestStringBuilder, OORtypeMixin):
-    def test_string_getlength(self):
-        py.test.skip("getlength(): not implemented on ootype")
-    def test_unicode_getlength(self):
-        py.test.skip("getlength(): not implemented on ootype")
     def test_append_charpsize(self):
         py.test.skip("append_charpsize(): not implemented on ootype")
diff --git a/pypy/rpython/test/test_rbuiltin.py b/pypy/rpython/test/test_rbuiltin.py
--- a/pypy/rpython/test/test_rbuiltin.py
+++ b/pypy/rpython/test/test_rbuiltin.py
@@ -463,6 +463,31 @@
             assert x1 == intmask(x0)
             assert x3 == intmask(x2)
 
+    def test_id_on_builtins(self):
+        from pypy.rlib.objectmodel import compute_unique_id
+        from pypy.rlib.rstring import StringBuilder, UnicodeBuilder
+        def fn():
+            return (compute_unique_id("foo"),
+                    compute_unique_id(u"bar"),
+                    compute_unique_id([1]),
+                    compute_unique_id({"foo": 3}),
+                    compute_unique_id(StringBuilder()),
+                    compute_unique_id(UnicodeBuilder()))
+        res = self.interpret(fn, [])
+        for id in self.ll_unpack_tuple(res, 6):
+            assert isinstance(id, (int, r_longlong))
+
+    def test_uniqueness_of_id_on_strings(self):
+        from pypy.rlib.objectmodel import compute_unique_id
+        def fn(s1, s2):
+            return (compute_unique_id(s1), compute_unique_id(s2))
+
+        s1 = "foo"
+        s2 = ''.join(['f','oo'])
+        res = self.interpret(fn, [self.string_to_ll(s1), self.string_to_ll(s2)])
+        i1, i2 = self.ll_unpack_tuple(res, 2)
+        assert i1 != i2
+
     def test_cast_primitive(self):
         from pypy.rpython.annlowlevel import LowLevelAnnotatorPolicy
         def llf(u):
diff --git a/pypy/rpython/test/test_rclass.py b/pypy/rpython/test/test_rclass.py
--- a/pypy/rpython/test/test_rclass.py
+++ b/pypy/rpython/test/test_rclass.py
@@ -1130,6 +1130,18 @@
             assert sorted([u]) == [6]                    # 32-bit types
             assert sorted([i, r, d, l]) == [2, 3, 4, 5]  # 64-bit types
 
+    def test_nonmovable(self):
+        for (nonmovable, opname) in [(True, 'malloc_nonmovable'),
+                                     (False, 'malloc')]:
+            class A(object):
+                _alloc_nonmovable_ = nonmovable
+            def f():
+                return A()
+            t, typer, graph = self.gengraph(f, [])
+            assert summary(graph) == {opname: 1,
+                                      'cast_pointer': 1,
+                                      'setfield': 1}
+
 
 class TestOOtype(BaseTestRclass, OORtypeMixin):
 
diff --git a/pypy/tool/jitlogparser/parser.py b/pypy/tool/jitlogparser/parser.py
--- a/pypy/tool/jitlogparser/parser.py
+++ b/pypy/tool/jitlogparser/parser.py
@@ -140,13 +140,15 @@
     bytecode_name = None
     is_bytecode = True
     inline_level = None
+    has_dmp = False
 
     def parse_code_data(self, arg):
         m = re.search('\w]+)[\.,] file \'(.+?)\'[\.,] line (\d+)> #(\d+) (\w+)',
                       arg)
         if m is None:
             # a non-code loop, like StrLiteralSearch or something
-            self.bytecode_name = arg
+            if arg:
+                self.bytecode_name = arg
         else:
             self.name, self.filename, lineno, bytecode_no, self.bytecode_name = m.groups()
             self.startlineno = int(lineno)
@@ -218,7 +220,7 @@
         self.inputargs = inputargs
         self.chunks = chunks
         for chunk in self.chunks:
-            if chunk.filename is not None:
+            if chunk.bytecode_name is not None:
                 self.startlineno = chunk.startlineno
                 self.filename = chunk.filename
                 self.name = chunk.name
diff --git a/pypy/tool/jitlogparser/test/test_parser.py b/pypy/tool/jitlogparser/test/test_parser.py
--- a/pypy/tool/jitlogparser/test/test_parser.py
+++ b/pypy/tool/jitlogparser/test/test_parser.py
@@ -283,3 +283,13 @@
     assert loops[-1].count == 1234
     assert loops[1].count == 123
     assert loops[2].count == 12
+
+def test_parse_nonpython():
+    loop = parse("""
+    []
+    debug_merge_point(0, 'random')
+    debug_merge_point(0, ' #15 COMPARE_OP')
+    """)
+    f = Function.from_operations(loop.operations, LoopStorage())
+    assert f.chunks[-1].filename == 'x.py'
+    assert f.filename is None
diff --git a/pypy/tool/test/test_version.py b/pypy/tool/test/test_version.py
--- a/pypy/tool/test/test_version.py
+++ b/pypy/tool/test/test_version.py
@@ -1,6 +1,22 @@
 import os, sys
 import py
-from pypy.tool.version import get_repo_version_info
+from pypy.tool.version import get_repo_version_info, _get_hg_archive_version
+
+def test_hg_archival_version(tmpdir):
+    def version_for(name, **kw):
+        path = tmpdir.join(name)
+        path.write('\n'.join('%s: %s' % x for x in kw.items()))
+        return _get_hg_archive_version(str(path))
+
+    assert version_for('release',
+                       tag='release-123',
+                       node='000',
+                      ) == ('PyPy', 'release-123', '000')
+    assert version_for('somebranch',
+                       node='000',
+                       branch='something',
+                      ) == ('PyPy', 'something', '000')
+
 
 def test_get_repo_version_info():
     assert get_repo_version_info(None)
diff --git a/pypy/tool/version.py b/pypy/tool/version.py
--- a/pypy/tool/version.py
+++ b/pypy/tool/version.py
@@ -3,111 +3,139 @@
 from subprocess import Popen, PIPE
 import pypy
 pypydir = os.path.dirname(os.path.abspath(pypy.__file__))
+pypyroot = os.path.dirname(pypydir)
+default_retval = 'PyPy', '?', '?'
+
+def maywarn(err, repo_type='Mercurial'):
+    if not err:
+        return
+
+    from pypy.tool.ansi_print import ansi_log
+    log = py.log.Producer("version")
+    py.log.setconsumer("version", ansi_log)
+    log.WARNING('Errors getting %s information: %s' % (repo_type, err))
 
 def get_repo_version_info(hgexe=None):
     '''Obtain version information by invoking the 'hg' or 'git' commands.'''
-    # TODO: support extracting from .hg_archival.txt
-
-    default_retval = 'PyPy', '?', '?'
-    pypyroot = os.path.abspath(os.path.join(pypydir, '..'))
-
-    def maywarn(err, repo_type='Mercurial'):
-        if not err:
-            return
-
-        from pypy.tool.ansi_print import ansi_log
-        log = py.log.Producer("version")
-        py.log.setconsumer("version", ansi_log)
-        log.WARNING('Errors getting %s information: %s' % (repo_type, err))
 
     # Try to see if we can get info from Git if hgexe is not specified.
     if not hgexe:
         if os.path.isdir(os.path.join(pypyroot, '.git')):
-            gitexe = py.path.local.sysfind('git')
-            if gitexe:
-                try:
-                    p = Popen(
-                        [str(gitexe), 'rev-parse', 'HEAD'],
-                        stdout=PIPE, stderr=PIPE, cwd=pypyroot
-                        )
-                except OSError, e:
-                    maywarn(e, 'Git')
-                    return default_retval
-                if p.wait() != 0:
-                    maywarn(p.stderr.read(), 'Git')
-                    return default_retval
-                revision_id = p.stdout.read().strip()[:12]
-                p = Popen(
-                    [str(gitexe), 'describe', '--tags', '--exact-match'],
-                    stdout=PIPE, stderr=PIPE, cwd=pypyroot
-                    )
-                if p.wait() != 0:
-                    p = Popen(
-                        [str(gitexe), 'branch'], stdout=PIPE, stderr=PIPE,
-                        cwd=pypyroot
-                        )
-                    if p.wait() != 0:
-                        maywarn(p.stderr.read(), 'Git')
-                        return 'PyPy', '?', revision_id
-                    branch = '?'
-                    for line in p.stdout.read().strip().split('\n'):
-                        if line.startswith('* '):
-                            branch = line[1:].strip()
-                            if branch == '(no branch)':
-                                branch = '?'
-                            break
-                    return 'PyPy', branch, revision_id
-                return 'PyPy', p.stdout.read().strip(), revision_id
+            return _get_git_version()
 
     # Fallback to trying Mercurial.
     if hgexe is None:
         hgexe = py.path.local.sysfind('hg')
 
-    if not os.path.isdir(os.path.join(pypyroot, '.hg')):
+    if os.path.isfile(os.path.join(pypyroot, '.hg_archival.txt')):
+        return _get_hg_archive_version(os.path.join(pypyroot, '.hg_archival.txt'))
+    elif not os.path.isdir(os.path.join(pypyroot, '.hg')):
         maywarn('Not running from a Mercurial repository!')
         return default_retval
     elif not hgexe:
         maywarn('Cannot find Mercurial command!')
         return default_retval
     else:
-        env = dict(os.environ)
-        # get Mercurial into scripting mode
-        env['HGPLAIN'] = '1'
-        # disable user configuration, extensions, etc.
-        env['HGRCPATH'] = os.devnull
+        return _get_hg_version(hgexe)
 
-        try:
-            p = Popen([str(hgexe), 'version', '-q'],
-                      stdout=PIPE, stderr=PIPE, env=env)
-        except OSError, e:
-            maywarn(e)
-            return default_retval
 
-        if not p.stdout.read().startswith('Mercurial Distributed SCM'):
-            maywarn('command does not identify itself as Mercurial')
-            return default_retval
+def _get_hg_version(hgexe):
+    env = dict(os.environ)
+    # get Mercurial into scripting mode
+    env['HGPLAIN'] = '1'
+    # disable user configuration, extensions, etc.
+    env['HGRCPATH'] = os.devnull
 
-        p = Popen([str(hgexe), 'id', '-i', pypyroot],
+    try:
+        p = Popen([str(hgexe), 'version', '-q'],
                   stdout=PIPE, stderr=PIPE, env=env)
-        hgid = p.stdout.read().strip()
+    except OSError, e:
+        maywarn(e)
+        return default_retval
+
+    if not p.stdout.read().startswith('Mercurial Distributed SCM'):
+        maywarn('command does not identify itself as Mercurial')
+        return default_retval
+
+    p = Popen([str(hgexe), 'id', '-i', pypyroot],
+              stdout=PIPE, stderr=PIPE, env=env)
+    hgid = p.stdout.read().strip()
+    maywarn(p.stderr.read())
+    if p.wait() != 0:
+        hgid = '?'
+
+    p = Popen([str(hgexe), 'id', '-t', pypyroot],
+              stdout=PIPE, stderr=PIPE, env=env)
+    hgtags = [t for t in p.stdout.read().strip().split() if t != 'tip']
+    maywarn(p.stderr.read())
+    if p.wait() != 0:
+        hgtags = ['?']
+
+    if hgtags:
+        return 'PyPy', hgtags[0], hgid
+    else:
+        # use the branch instead
+        p = Popen([str(hgexe), 'id', '-b', pypyroot],
+                  stdout=PIPE, stderr=PIPE, env=env)
+        hgbranch = p.stdout.read().strip()
         maywarn(p.stderr.read())
+
+        return 'PyPy', hgbranch, hgid
+
+
+def _get_hg_archive_version(path):
+    fp = open(path)
+    try:
+        data = dict(x.split(': ', 1) for x in fp.read().splitlines())
+    finally:
+        fp.close()
+    if 'tag' in data:
+        return 'PyPy', data['tag'], data['node']
+    else:
+        return 'PyPy', data['branch'], data['node']
+
+
+def _get_git_version():
+    #XXX: this function is a untested hack,
+    #     so the git mirror tav made will work
+    gitexe = py.path.local.sysfind('git')
+    if not gitexe:
+        return default_retval
+
+    try:
+        p = Popen(
+            [str(gitexe), 'rev-parse', 'HEAD'],
+            stdout=PIPE, stderr=PIPE, cwd=pypyroot
+            )
+    except OSError, e:
+        maywarn(e, 'Git')
+        return default_retval
+    if p.wait() != 0:
+        maywarn(p.stderr.read(), 'Git')
+        return default_retval
+    revision_id = p.stdout.read().strip()[:12]
+    p = Popen(
+        [str(gitexe), 'describe', '--tags', '--exact-match'],
+        stdout=PIPE, stderr=PIPE, cwd=pypyroot
+        )
+    if p.wait() != 0:
+        p = Popen(
+            [str(gitexe), 'branch'], stdout=PIPE, stderr=PIPE,
+            cwd=pypyroot
+            )
         if p.wait() != 0:
-            hgid = '?'
+            maywarn(p.stderr.read(), 'Git')
+            return 'PyPy', '?', revision_id
+        branch = '?'
+        for line in p.stdout.read().strip().split('\n'):
+            if line.startswith('* '):
+                branch = line[1:].strip()
+                if branch == '(no branch)':
+                    branch = '?'
+                break
+        return 'PyPy', branch, revision_id
+    return 'PyPy', p.stdout.read().strip(), revision_id
 
-        p = Popen([str(hgexe), 'id', '-t', pypyroot],
-                  stdout=PIPE, stderr=PIPE, env=env)
-        hgtags = [t for t in p.stdout.read().strip().split() if t != 'tip']
-        maywarn(p.stderr.read())
-        if p.wait() != 0:
-            hgtags = ['?']
 
-        if hgtags:
-            return 'PyPy', hgtags[0], hgid
-        else:
-            # use the branch instead
-            p = Popen([str(hgexe), 'id', '-b', pypyroot],
-                      stdout=PIPE, stderr=PIPE, env=env)
-            hgbranch = p.stdout.read().strip()
-            maywarn(p.stderr.read())
-
-            return 'PyPy', hgbranch, hgid
+if __name__ == '__main__':
+    print get_repo_version_info()
diff --git a/pypy/translator/jvm/builtin.py b/pypy/translator/jvm/builtin.py
--- a/pypy/translator/jvm/builtin.py
+++ b/pypy/translator/jvm/builtin.py
@@ -84,6 +84,9 @@
     (ootype.StringBuilder.__class__, "ll_build"):
     jvm.Method.v(jStringBuilder, "toString", (), jString),
 
+    (ootype.StringBuilder.__class__, "ll_getlength"):
+    jvm.Method.v(jStringBuilder, "length", (), jInt),
+
     (ootype.String.__class__, "ll_hash"):
     jvm.Method.v(jString, "hashCode", (), jInt),
 
diff --git a/pypy/translator/jvm/database.py b/pypy/translator/jvm/database.py
--- a/pypy/translator/jvm/database.py
+++ b/pypy/translator/jvm/database.py
@@ -358,7 +358,7 @@
         ootype.Unsigned:jvm.PYPYSERIALIZEUINT,
         ootype.SignedLongLong:jvm.LONGTOSTRINGL,
         ootype.UnsignedLongLong: jvm.PYPYSERIALIZEULONG,
-        ootype.Float:jvm.DOUBLETOSTRINGD,
+        ootype.Float:jvm.PYPYSERIALIZEDOUBLE,
         ootype.Bool:jvm.PYPYSERIALIZEBOOLEAN,
         ootype.Void:jvm.PYPYSERIALIZEVOID,
         ootype.Char:jvm.PYPYESCAPEDCHAR,
diff --git a/pypy/translator/jvm/metavm.py b/pypy/translator/jvm/metavm.py
--- a/pypy/translator/jvm/metavm.py
+++ b/pypy/translator/jvm/metavm.py
@@ -92,6 +92,7 @@
 CASTS = {
 #   FROM                      TO
     (ootype.Signed,           ootype.UnsignedLongLong): jvm.I2L,
+    (ootype.Unsigned,         ootype.UnsignedLongLong): jvm.I2L,
     (ootype.SignedLongLong,   ootype.Signed):           jvm.L2I,
     (ootype.UnsignedLongLong, ootype.Unsigned):         jvm.L2I,
     (ootype.UnsignedLongLong, ootype.Signed):           jvm.L2I,
diff --git a/pypy/translator/jvm/opcodes.py b/pypy/translator/jvm/opcodes.py
--- a/pypy/translator/jvm/opcodes.py
+++ b/pypy/translator/jvm/opcodes.py
@@ -101,6 +101,7 @@
     'jit_force_virtualizable':  Ignore,
     'jit_force_virtual':        DoNothing,
     'jit_force_quasi_immutable': Ignore,
+    'jit_is_virtual':           PushPrimitive(ootype.Bool, False),
 
     'debug_assert':              [], # TODO: implement?
     'debug_start_traceback':    Ignore,
diff --git a/pypy/translator/jvm/src/pypy/PyPy.java b/pypy/translator/jvm/src/pypy/PyPy.java
--- a/pypy/translator/jvm/src/pypy/PyPy.java
+++ b/pypy/translator/jvm/src/pypy/PyPy.java
@@ -283,6 +283,14 @@
         }
     }
 
+    public double pypy__longlong2float(long l) {
+        return Double.longBitsToDouble(l);
+    }
+
+    public long pypy__float2longlong(double d) {
+        return Double.doubleToRawLongBits(d);
+    }
+
     public double ooparse_float(String s) {
         try {
             return Double.parseDouble(s);
@@ -353,6 +361,19 @@
             return "False";
     }
 
+    public static String serialize_double(double d) {
+        if (Double.isNaN(d)) {
+            return "float(\"nan\")";
+        } else if (Double.isInfinite(d)) {
+            if (d > 0)
+                return "float(\"inf\")";
+            else
+                return "float(\"-inf\")";
+        } else {
+            return Double.toString(d);
+        }
+    }
+
     private static String format_char(char c) {
         String res = "\\x";
         if (c <= 0x0F) res = res + "0";
diff --git a/pypy/translator/jvm/test/runtest.py b/pypy/translator/jvm/test/runtest.py
--- a/pypy/translator/jvm/test/runtest.py
+++ b/pypy/translator/jvm/test/runtest.py
@@ -56,6 +56,7 @@
 
 # CLI could-be duplicate
 class JvmGeneratedSourceWrapper(object):
+
     def __init__(self, gensrc):
         """ gensrc is an instance of JvmGeneratedSource """
         self.gensrc = gensrc
diff --git a/pypy/translator/jvm/test/test_builder.py b/pypy/translator/jvm/test/test_builder.py
new file mode 100644
--- /dev/null
+++ b/pypy/translator/jvm/test/test_builder.py
@@ -0,0 +1,7 @@
+from pypy.translator.jvm.test.runtest import JvmTest
+from pypy.rpython.test.test_rbuilder import BaseTestStringBuilder
+import py
+
+class TestJvmStringBuilder(JvmTest, BaseTestStringBuilder):
+    def test_append_charpsize(self):
+        py.test.skip("append_charpsize(): not implemented on ootype")
diff --git a/pypy/translator/jvm/test/test_longlong2float.py b/pypy/translator/jvm/test/test_longlong2float.py
new file mode 100644
--- /dev/null
+++ b/pypy/translator/jvm/test/test_longlong2float.py
@@ -0,0 +1,20 @@
+from pypy.translator.jvm.test.runtest import JvmTest
+from pypy.rlib.longlong2float import *
+from pypy.rlib.test.test_longlong2float import enum_floats
+from pypy.rlib.test.test_longlong2float import fn as float2longlong2float
+import py
+
+class TestLongLong2Float(JvmTest):
+
+    def test_float2longlong_and_longlong2float(self):
+        def func(f):
+            return float2longlong2float(f)
+
+        for f in enum_floats():
+            assert repr(f) == repr(self.interpret(func, [f]))
+
+    def test_uint2singlefloat(self):
+        py.test.skip("uint2singlefloat is not implemented in ootype")
+
+    def test_singlefloat2uint(self):
+        py.test.skip("singlefloat2uint is not implemented in ootype")
diff --git a/pypy/translator/jvm/typesystem.py b/pypy/translator/jvm/typesystem.py
--- a/pypy/translator/jvm/typesystem.py
+++ b/pypy/translator/jvm/typesystem.py
@@ -955,6 +955,7 @@
 PYPYSERIALIZEUINT  =    Method.s(jPyPy, 'serialize_uint', (jInt,), jString)
 PYPYSERIALIZEULONG =    Method.s(jPyPy, 'serialize_ulonglong', (jLong,),jString)
 PYPYSERIALIZEVOID =     Method.s(jPyPy, 'serialize_void', (), jString)
+PYPYSERIALIZEDOUBLE =   Method.s(jPyPy, 'serialize_double', (jDouble,), jString)
 PYPYESCAPEDCHAR =       Method.s(jPyPy, 'escaped_char', (jChar,), jString)
 PYPYESCAPEDUNICHAR =    Method.s(jPyPy, 'escaped_unichar', (jChar,), jString)
 PYPYESCAPEDSTRING =     Method.s(jPyPy, 'escaped_string', (jString,), jString)
diff --git a/pypy/translator/oosupport/test_template/cast.py b/pypy/translator/oosupport/test_template/cast.py
--- a/pypy/translator/oosupport/test_template/cast.py
+++ b/pypy/translator/oosupport/test_template/cast.py
@@ -13,6 +13,9 @@
 def to_longlong(x):
     return r_longlong(x)
 
+def to_ulonglong(x):
+    return r_ulonglong(x)
+
 def uint_to_int(x):
     return intmask(x)
 
@@ -56,6 +59,9 @@
     def test_unsignedlonglong_to_unsigned4(self):
         self.check(to_uint, [r_ulonglong(18446744073709551615l)]) # max 64 bit num
 
+    def test_unsigned_to_usignedlonglong(self):
+        self.check(to_ulonglong, [r_uint(42)])
+
     def test_uint_to_int(self):
         self.check(uint_to_int, [r_uint(sys.maxint+1)])
 

From noreply at buildbot.pypy.org  Sat Jan 21 13:19:21 2012
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sat, 21 Jan 2012 13:19:21 +0100 (CET)
Subject: [pypy-commit] pypy py3k: pyrepl has a lot of SyntaxErrors at the
	moment.
Message-ID: <20120121121921.3CA8482CB2@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: py3k
Changeset: r51570:e613284ce7f5
Date: 2012-01-21 11:22 +0100
http://bitbucket.org/pypy/pypy/changeset/e613284ce7f5/

Log:	pyrepl has a lot of SyntaxErrors at the moment. gracefully raise an
	ImportError in readline.py

diff --git a/lib_pypy/readline.py b/lib_pypy/readline.py
--- a/lib_pypy/readline.py
+++ b/lib_pypy/readline.py
@@ -6,4 +6,7 @@
 are only stubs at the moment.
 """
 
-from pyrepl.readline import *
+try:
+    from pyrepl.readline import *
+except SyntaxError:
+    raise ImportError

From noreply at buildbot.pypy.org  Sat Jan 21 13:19:22 2012
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sat, 21 Jan 2012 13:19:22 +0100 (CET)
Subject: [pypy-commit] pypy py3k: Merge heads
Message-ID: <20120121121922.E00CE82CB2@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: py3k
Changeset: r51571:af3dee783792
Date: 2012-01-21 11:36 +0100
http://bitbucket.org/pypy/pypy/changeset/af3dee783792/

Log:	Merge heads

diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -2,6 +2,9 @@
 *.py[co]
 *~
 .*.swp
+.idea
+.project
+.pydevproject
 
 syntax: regexp
 ^testresult$
diff --git a/lib_pypy/numpypy/__init__.py b/lib_pypy/numpypy/__init__.py
--- a/lib_pypy/numpypy/__init__.py
+++ b/lib_pypy/numpypy/__init__.py
@@ -1,2 +1,2 @@
 from _numpypy import *
-from .fromnumeric import *
+from .core import *
diff --git a/lib_pypy/numpypy/core/__init__.py b/lib_pypy/numpypy/core/__init__.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/numpypy/core/__init__.py
@@ -0,0 +1,1 @@
+from .fromnumeric import *
diff --git a/lib_pypy/numpypy/fromnumeric.py b/lib_pypy/numpypy/core/fromnumeric.py
rename from lib_pypy/numpypy/fromnumeric.py
rename to lib_pypy/numpypy/core/fromnumeric.py
--- a/lib_pypy/numpypy/fromnumeric.py
+++ b/lib_pypy/numpypy/core/fromnumeric.py
@@ -85,7 +85,7 @@
     array([4, 3, 6])
 
     """
-    raise NotImplemented('Waiting on interp level method')
+    raise NotImplementedError('Waiting on interp level method')
 
 
 # not deprecated --- copy if necessary, view otherwise
@@ -273,7 +273,7 @@
             [-1, -2, -3, -4, -5]]])
 
     """
-    raise NotImplemented('Waiting on interp level method')
+    raise NotImplementedError('Waiting on interp level method')
 
 
 def repeat(a, repeats, axis=None):
@@ -315,7 +315,7 @@
            [3, 4]])
 
     """
-    raise NotImplemented('Waiting on interp level method')
+    raise NotImplementedError('Waiting on interp level method')
 
 
 def put(a, ind, v, mode='raise'):
@@ -366,7 +366,7 @@
     array([ 0,  1,  2,  3, -5])
 
     """
-    raise NotImplemented('Waiting on interp level method')
+    raise NotImplementedError('Waiting on interp level method')
 
 
 def swapaxes(a, axis1, axis2):
@@ -410,7 +410,7 @@
             [3, 7]]])
 
     """
-    raise NotImplemented('Waiting on interp level method')
+    raise NotImplementedError('Waiting on interp level method')
 
 
 def transpose(a, axes=None):
@@ -451,8 +451,11 @@
     (2, 1, 3)
 
     """
-    raise NotImplemented('Waiting on interp level method')
-
+    if axes is not None:
+        raise NotImplementedError('No "axes" arg yet.')
+    if not hasattr(a, 'T'):
+        a = numpypy.array(a)
+    return a.T
 
 def sort(a, axis=-1, kind='quicksort', order=None):
     """
@@ -553,7 +556,7 @@
           dtype=[('name', '|S10'), ('height', '= self.size
 
@@ -157,6 +177,8 @@
                 offset += self.strides[i]
                 break
             else:
+                if i == self.dim:
+                    first_line = True
                 indices[i] = 0
                 offset -= self.backstrides[i]
         else:
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -2,14 +2,15 @@
 from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.interpreter.gateway import interp2app, NoneNotWrapped
 from pypy.interpreter.typedef import TypeDef, GetSetProperty
-from pypy.module.micronumpy import interp_ufuncs, interp_dtype, signature
+from pypy.module.micronumpy import interp_ufuncs, interp_dtype, signature,\
+     interp_boxes
 from pypy.module.micronumpy.strides import calculate_slice_strides
 from pypy.rlib import jit
 from pypy.rpython.lltypesystem import lltype, rffi
 from pypy.tool.sourcetools import func_with_new_name
 from pypy.rlib.rstring import StringBuilder
 from pypy.module.micronumpy.interp_iter import ArrayIterator, OneDimIterator,\
-     SkipLastAxisIterator
+     SkipLastAxisIterator, Chunk, ViewIterator
 
 numpy_driver = jit.JitDriver(
     greens=['shapelen', 'sig'],
@@ -39,7 +40,24 @@
     get_printable_location=signature.new_printable_location('slice'),
     name='numpy_slice',
 )
-
+count_driver = jit.JitDriver(
+    greens=['shapelen'],
+    virtualizables=['frame'],
+    reds=['s', 'frame', 'iter', 'arr'],
+    name='numpy_count'
+)
+filter_driver = jit.JitDriver(
+    greens=['shapelen', 'sig'],
+    virtualizables=['frame'],
+    reds=['concr', 'argi', 'ri', 'frame', 'v', 'res', 'self'],
+    name='numpy_filter',
+)
+filter_set_driver = jit.JitDriver(
+    greens=['shapelen', 'sig'],
+    virtualizables=['frame'],
+    reds=['idx', 'idxi', 'frame', 'arr'],
+    name='numpy_filterset',
+)
 
 def _find_shape_and_elems(space, w_iterable):
     shape = [space.len_w(w_iterable)]
@@ -270,6 +288,9 @@
     descr_gt = _binop_impl("greater")
     descr_ge = _binop_impl("greater_equal")
 
+    descr_and = _binop_impl("bitwise_and")
+    descr_or = _binop_impl("bitwise_or")
+
     def _binop_right_impl(ufunc_name):
         def impl(self, space, w_other):
             w_other = scalar_w(space,
@@ -287,11 +308,11 @@
     descr_rmod = _binop_right_impl("mod")
 
     def _reduce_ufunc_impl(ufunc_name, promote_to_largest=False):
-        def impl(self, space, w_dim=None):
-            if space.is_w(w_dim, space.w_None):
-                w_dim = space.wrap(-1)
+        def impl(self, space, w_axis=None):
+            if space.is_w(w_axis, space.w_None):
+                w_axis = space.wrap(-1)
             return getattr(interp_ufuncs.get(space), ufunc_name).reduce(space,
-                                        self, True, promote_to_largest, w_dim)
+                                        self, True, promote_to_largest, w_axis)
         return func_with_new_name(impl, "reduce_%s_impl" % ufunc_name)
 
     descr_sum = _reduce_ufunc_impl("add")
@@ -479,11 +500,69 @@
     def _prepare_slice_args(self, space, w_idx):
         if (space.isinstance_w(w_idx, space.w_int) or
             space.isinstance_w(w_idx, space.w_slice)):
-            return [space.decode_index4(w_idx, self.shape[0])]
-        return [space.decode_index4(w_item, self.shape[i]) for i, w_item in
+            return [Chunk(*space.decode_index4(w_idx, self.shape[0]))]
+        return [Chunk(*space.decode_index4(w_item, self.shape[i])) for i, w_item in
                 enumerate(space.fixedview(w_idx))]
 
+    def count_all_true(self, arr):
+        sig = arr.find_sig()
+        frame = sig.create_frame(self)
+        shapelen = len(arr.shape)
+        s = 0
+        iter = None
+        while not frame.done():
+            count_driver.jit_merge_point(arr=arr, frame=frame, iter=iter, s=s,
+                                         shapelen=shapelen)
+            iter = frame.get_final_iter()
+            s += arr.dtype.getitem_bool(arr.storage, iter.offset)
+            frame.next(shapelen)
+        return s
+
+    def getitem_filter(self, space, arr):
+        concr = arr.get_concrete()
+        size = self.count_all_true(concr)
+        res = W_NDimArray(size, [size], self.find_dtype())
+        ri = ArrayIterator(size)
+        shapelen = len(self.shape)
+        argi = concr.create_iter()
+        sig = self.find_sig()
+        frame = sig.create_frame(self)
+        v = None
+        while not frame.done():
+            filter_driver.jit_merge_point(concr=concr, argi=argi, ri=ri,
+                                          frame=frame, v=v, res=res, sig=sig,
+                                          shapelen=shapelen, self=self)
+            if concr.dtype.getitem_bool(concr.storage, argi.offset):
+                v = sig.eval(frame, self)
+                res.setitem(ri.offset, v)
+                ri = ri.next(1)
+            else:
+                ri = ri.next_no_increase(1)
+            argi = argi.next(shapelen)
+            frame.next(shapelen)
+        return res
+
+    def setitem_filter(self, space, idx, val):
+        size = self.count_all_true(idx)
+        arr = SliceArray([size], self.dtype, self, val)
+        sig = arr.find_sig()
+        shapelen = len(self.shape)
+        frame = sig.create_frame(arr)
+        idxi = idx.create_iter()
+        while not frame.done():
+            filter_set_driver.jit_merge_point(idx=idx, idxi=idxi, sig=sig,
+                                              frame=frame, arr=arr,
+                                              shapelen=shapelen)
+            if idx.dtype.getitem_bool(idx.storage, idxi.offset):
+                sig.eval(frame, arr)
+                frame.next_from_second(1)
+            frame.next_first(shapelen)
+            idxi = idxi.next(shapelen)
+
     def descr_getitem(self, space, w_idx):
+        if (isinstance(w_idx, BaseArray) and w_idx.shape == self.shape and
+            w_idx.find_dtype().is_bool_type()):
+            return self.getitem_filter(space, w_idx)
         if self._single_item_result(space, w_idx):
             concrete = self.get_concrete()
             item = concrete._index_of_single_item(space, w_idx)
@@ -493,6 +572,11 @@
 
     def descr_setitem(self, space, w_idx, w_value):
         self.invalidated()
+        if (isinstance(w_idx, BaseArray) and w_idx.shape == self.shape and
+            w_idx.find_dtype().is_bool_type()):
+            return self.get_concrete().setitem_filter(space,
+                                                      w_idx.get_concrete(),
+                                             convert_to_array(space, w_value))
         if self._single_item_result(space, w_idx):
             concrete = self.get_concrete()
             item = concrete._index_of_single_item(space, w_idx)
@@ -509,9 +593,8 @@
     def create_slice(self, chunks):
         shape = []
         i = -1
-        for i, (start_, stop, step, lgt) in enumerate(chunks):
-            if step != 0:
-                shape.append(lgt)
+        for i, chunk in enumerate(chunks):
+            chunk.extend_shape(shape)
         s = i + 1
         assert s >= 0
         shape += self.shape[s:]
@@ -569,19 +652,19 @@
             )
         return w_result
 
-    def descr_mean(self, space, w_dim=None):
-        if space.is_w(w_dim, space.w_None):
-            w_dim = space.wrap(-1)
+    def descr_mean(self, space, w_axis=None):
+        if space.is_w(w_axis, space.w_None):
+            w_axis = space.wrap(-1)
             w_denom = space.wrap(self.size)
         else:
-            dim = space.int_w(w_dim)
+            dim = space.int_w(w_axis)
             w_denom = space.wrap(self.shape[dim])
-        return space.div(self.descr_sum_promote(space, w_dim), w_denom)
+        return space.div(self.descr_sum_promote(space, w_axis), w_denom)
 
     def descr_var(self, space):
         # var = mean((values - mean(values)) ** 2)
         w_res = self.descr_sub(space, self.descr_mean(space, space.w_None))
-        assert isinstance(w_res, BaseArray) 
+        assert isinstance(w_res, BaseArray)
         w_res = w_res.descr_pow(space, space.wrap(2))
         assert isinstance(w_res, BaseArray)
         return w_res.descr_mean(space, space.w_None)
@@ -590,6 +673,10 @@
         # std(v) = sqrt(var(v))
         return interp_ufuncs.get(space).sqrt.call(space, [self.descr_var(space)])
 
+    def descr_fill(self, space, w_value):
+        concr = self.get_concrete_or_scalar()
+        concr.fill(space, w_value)
+
     def descr_nonzero(self, space):
         if self.size > 1:
             raise OperationError(space.w_ValueError, space.wrap(
@@ -682,6 +769,9 @@
     def copy(self, space):
         return Scalar(self.dtype, self.value)
 
+    def fill(self, space, w_value):
+        self.value = self.dtype.coerce(space, w_value)
+
     def create_sig(self):
         return signature.ScalarSignature(self.dtype)
 
@@ -717,8 +807,7 @@
                                          frame=frame,
                                          ri=ri,
                                          self=self, result=result)
-            result.dtype.setitem(result.storage, ri.offset,
-                                 sig.eval(frame, self))
+            result.setitem(ri.offset, sig.eval(frame, self))
             frame.next(shapelen)
             ri = ri.next(shapelen)
         return result
@@ -788,7 +877,7 @@
     Intermediate class for performing binary operations.
     """
     _immutable_fields_ = ['left', 'right']
-    
+
     def __init__(self, ufunc, name, shape, calc_dtype, res_dtype, left, right):
         VirtualArray.__init__(self, name, shape, res_dtype)
         self.ufunc = ufunc
@@ -828,7 +917,7 @@
     def __init__(self, shape, dtype, left, right):
         Call2.__init__(self, None, 'sliceloop', shape, dtype, dtype, left,
                        right)
-    
+
     def create_sig(self):
         lsig = self.left.create_sig()
         rsig = self.right.create_sig()
@@ -847,7 +936,7 @@
     when we'll make AxisReduce lazy
     """
     _immutable_fields_ = ['left', 'right']
-    
+
     def __init__(self, ufunc, name, shape, dtype, left, right, dim):
         Call2.__init__(self, ufunc, name, shape, dtype, dtype,
                        left, right)
@@ -919,14 +1008,14 @@
         if size < 1:
             builder.append('[]')
             return
-        elif size == 1:
-            builder.append(dtype.itemtype.str_format(self.getitem(0)))
-            return
         if size > 1000:
             # Once this goes True it does not go back to False for recursive
             # calls
             use_ellipsis = True
         ndims = len(self.shape)
+        if ndims == 0:
+            builder.append(dtype.itemtype.str_format(self.getitem(0)))
+            return
         i = 0
         builder.append('[')
         if ndims > 1:
@@ -938,7 +1027,7 @@
                             builder.append('\n' + indent)
                         else:
                             builder.append(indent)
-                    view = self.create_slice([(i, 0, 0, 1)]).get_concrete()
+                    view = self.create_slice([Chunk(i, 0, 0, 1)]).get_concrete()
                     view.to_str(space, comma, builder, indent=indent + ' ',
                                                     use_ellipsis=use_ellipsis)
                 if i < self.shape[0] - 1:
@@ -955,7 +1044,7 @@
                         builder.append(indent)
                 # create_slice requires len(chunks) > 1 in order to reduce
                 # shape
-                view = self.create_slice([(i, 0, 0, 1)]).get_concrete()
+                view = self.create_slice([Chunk(i, 0, 0, 1)]).get_concrete()
                 view.to_str(space, comma, builder, indent=indent + ' ',
                                                     use_ellipsis=use_ellipsis)
                 i += 1
@@ -1061,6 +1150,9 @@
         array.setslice(space, self)
         return array
 
+    def fill(self, space, w_value):
+        self.setslice(space, scalar_w(space, self.dtype, w_value))
+
 
 class ViewArray(ConcreteArray):
     def create_sig(self):
@@ -1081,6 +1173,10 @@
                                parent)
         self.start = start
 
+    def create_iter(self):
+        return ViewIterator(self.start, self.strides, self.backstrides,
+                            self.shape)
+
     def setshape(self, space, new_shape):
         if len(self.shape) < 1:
             return
@@ -1127,6 +1223,9 @@
         self.shape = new_shape
         self.calc_strides(new_shape)
 
+    def create_iter(self):
+        return ArrayIterator(self.size)
+
     def create_sig(self):
         return signature.ArraySignature(self.dtype)
 
@@ -1181,6 +1280,7 @@
     arr = W_NDimArray(size, shape[:], dtype=dtype, order=order)
     shapelen = len(shape)
     arr_iter = ArrayIterator(arr.size)
+    # XXX we might want to have a jitdriver here
     for i in range(len(elems_w)):
         w_elem = elems_w[i]
         dtype.setitem(arr.storage, arr_iter.offset,
@@ -1247,6 +1347,9 @@
     __gt__ = interp2app(BaseArray.descr_gt),
     __ge__ = interp2app(BaseArray.descr_ge),
 
+    __and__ = interp2app(BaseArray.descr_and),
+    __or__ = interp2app(BaseArray.descr_or),
+
     __repr__ = interp2app(BaseArray.descr_repr),
     __str__ = interp2app(BaseArray.descr_str),
     __array_interface__ = GetSetProperty(BaseArray.descr_array_iface),
@@ -1273,6 +1376,8 @@
     var = interp2app(BaseArray.descr_var),
     std = interp2app(BaseArray.descr_std),
 
+    fill = interp2app(BaseArray.descr_fill),
+
     copy = interp2app(BaseArray.descr_copy),
     reshape = interp2app(BaseArray.descr_reshape),
     tolist = interp2app(BaseArray.descr_tolist),
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -249,15 +249,16 @@
 
 
 class W_Ufunc2(W_Ufunc):
-    _immutable_fields_ = ["comparison_func", "func", "name"]
+    _immutable_fields_ = ["comparison_func", "func", "name", "int_only"]
     argcount = 2
 
     def __init__(self, func, name, promote_to_float=False, promote_bools=False,
-        identity=None, comparison_func=False):
+        identity=None, comparison_func=False, int_only=False):
 
         W_Ufunc.__init__(self, name, promote_to_float, promote_bools, identity)
         self.func = func
         self.comparison_func = comparison_func
+        self.int_only = int_only
 
     def call(self, space, args_w):
         from pypy.module.micronumpy.interp_numarray import (Call2,
@@ -268,6 +269,7 @@
         w_rhs = convert_to_array(space, w_rhs)
         calc_dtype = find_binop_result_dtype(space,
             w_lhs.find_dtype(), w_rhs.find_dtype(),
+            int_only=self.int_only,
             promote_to_float=self.promote_to_float,
             promote_bools=self.promote_bools,
         )
@@ -304,10 +306,12 @@
 
 
 def find_binop_result_dtype(space, dt1, dt2, promote_to_float=False,
-    promote_bools=False):
+    promote_bools=False, int_only=False):
     # dt1.num should be <= dt2.num
     if dt1.num > dt2.num:
         dt1, dt2 = dt2, dt1
+    if int_only and (not dt1.is_int_type() or not dt2.is_int_type()):
+        raise OperationError(space.w_TypeError, space.wrap("Unsupported types"))
     # Some operations promote op(bool, bool) to return int8, rather than bool
     if promote_bools and (dt1.kind == dt2.kind == interp_dtype.BOOLLTR):
         return interp_dtype.get_dtype_cache(space).w_int8dtype
@@ -420,6 +424,10 @@
             ("add", "add", 2, {"identity": 0}),
             ("subtract", "sub", 2),
             ("multiply", "mul", 2, {"identity": 1}),
+            ("bitwise_and", "bitwise_and", 2, {"identity": 1,
+                                               'int_only': True}),
+            ("bitwise_or", "bitwise_or", 2, {"identity": 0,
+                                             'int_only': True}),
             ("divide", "div", 2, {"promote_bools": True}),
             ("mod", "mod", 2, {"promote_bools": True}),
             ("power", "pow", 2, {"promote_bools": True}),
@@ -444,6 +452,7 @@
 
             ("fabs", "fabs", 1, {"promote_to_float": True}),
             ("floor", "floor", 1, {"promote_to_float": True}),
+            ("ceil", "ceil", 1, {"promote_to_float": True}),
             ("exp", "exp", 1, {"promote_to_float": True}),
 
             ('sqrt', 'sqrt', 1, {'promote_to_float': True}),
@@ -470,7 +479,7 @@
         extra_kwargs["identity"] = identity
 
         func = ufunc_dtype_caller(space, ufunc_name, op_name, argcount,
-            comparison_func=extra_kwargs.get("comparison_func", False)
+            comparison_func=extra_kwargs.get("comparison_func", False),
         )
         if argcount == 1:
             ufunc = W_Ufunc1(func, ufunc_name, **extra_kwargs)
diff --git a/pypy/module/micronumpy/signature.py b/pypy/module/micronumpy/signature.py
--- a/pypy/module/micronumpy/signature.py
+++ b/pypy/module/micronumpy/signature.py
@@ -82,6 +82,16 @@
         for i in range(len(self.iterators)):
             self.iterators[i] = self.iterators[i].next(shapelen)
 
+    @unroll_safe
+    def next_from_second(self, shapelen):
+        """ Don't increase the first iterator
+        """
+        for i in range(1, len(self.iterators)):
+            self.iterators[i] = self.iterators[i].next(shapelen)
+
+    def next_first(self, shapelen):
+        self.iterators[0] = self.iterators[0].next(shapelen)
+
     def get_final_iter(self):
         final_iter = promote(self.final_iter)
         if final_iter < 0:
diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py
--- a/pypy/module/micronumpy/strides.py
+++ b/pypy/module/micronumpy/strides.py
@@ -10,12 +10,12 @@
     rstart = start
     rshape = []
     i = -1
-    for i, (start_, stop, step, lgt) in enumerate(chunks):
-        if step != 0:
-            rstrides.append(strides[i] * step)
-            rbackstrides.append(strides[i] * (lgt - 1) * step)
-            rshape.append(lgt)
-        rstart += strides[i] * start_
+    for i, chunk in enumerate(chunks):
+        if chunk.step != 0:
+            rstrides.append(strides[i] * chunk.step)
+            rbackstrides.append(strides[i] * (chunk.lgt - 1) * chunk.step)
+            rshape.append(chunk.lgt)
+        rstart += strides[i] * chunk.start
     # add a reminder
     s = i + 1
     assert s >= 0
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -166,14 +166,11 @@
         # You can't subclass dtype
         raises(TypeError, type, "Foo", (dtype,), {})
 
-    def test_new(self):
-        import _numpypy as np
-        assert np.int_(4) == 4
-        assert np.float_(3.4) == 3.4
+    def test_aliases(self):
+        from _numpypy import dtype
 
-    def test_pow(self):
-        from _numpypy import int_
-        assert int_(4) ** 2 == 16
+        assert dtype("float") is dtype(float)
+
 
 class AppTestTypes(BaseNumpyAppTest):
     def test_abstract_types(self):
@@ -189,6 +186,15 @@
         raises(TypeError, numpy.floating, 0)
         raises(TypeError, numpy.inexact, 0)
 
+    def test_new(self):
+        import _numpypy as np
+        assert np.int_(4) == 4
+        assert np.float_(3.4) == 3.4
+
+    def test_pow(self):
+        from _numpypy import int_
+        assert int_(4) ** 2 == 16
+
     def test_bool(self):
         import _numpypy as numpy
 
@@ -318,7 +324,7 @@
         else:
             raises(OverflowError, numpy.int64, 9223372036854775807)
             raises(OverflowError, numpy.int64, '9223372036854775807')
-        
+
         raises(OverflowError, numpy.int64, 9223372036854775808)
         raises(OverflowError, numpy.int64, '9223372036854775808')
 
diff --git a/pypy/module/micronumpy/test/test_module.py b/pypy/module/micronumpy/test/test_module.py
--- a/pypy/module/micronumpy/test/test_module.py
+++ b/pypy/module/micronumpy/test/test_module.py
@@ -2,16 +2,11 @@
 
 
 class AppTestNumPyModule(BaseNumpyAppTest):
-    def test_mean(self):
-        from _numpypy import array, mean
-        assert mean(array(range(5))) == 2.0
-        assert mean(range(5)) == 2.0
-
     def test_average(self):
         from _numpypy import array, average
         assert average(range(10)) == 4.5
         assert average(array(range(10))) == 4.5
-        
+
     def test_sum(self):
         from _numpypy import array, sum
         assert sum(range(10)) == 45
@@ -21,7 +16,7 @@
         from _numpypy import array, min
         assert min(range(10)) == 0
         assert min(array(range(10))) == 0
-        
+
     def test_max(self):
         from _numpypy import array, max
         assert max(range(10)) == 9
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -2,6 +2,7 @@
 import py
 from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
 from pypy.module.micronumpy.interp_numarray import W_NDimArray, shape_agreement
+from pypy.module.micronumpy.interp_iter import Chunk
 from pypy.module.micronumpy import signature
 from pypy.interpreter.error import OperationError
 from pypy.conftest import gettestobjspace
@@ -37,53 +38,54 @@
 
     def test_create_slice_f(self):
         a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'F')
-        s = a.create_slice([(3, 0, 0, 1)])
+        s = a.create_slice([Chunk(3, 0, 0, 1)])
         assert s.start == 3
         assert s.strides == [10, 50]
         assert s.backstrides == [40, 100]
-        s = a.create_slice([(1, 9, 2, 4)])
+        s = a.create_slice([Chunk(1, 9, 2, 4)])
         assert s.start == 1
         assert s.strides == [2, 10, 50]
         assert s.backstrides == [6, 40, 100]
-        s = a.create_slice([(1, 5, 3, 2), (1, 2, 1, 1), (1, 0, 0, 1)])
+        s = a.create_slice([Chunk(1, 5, 3, 2), Chunk(1, 2, 1, 1), Chunk(1, 0, 0, 1)])
         assert s.shape == [2, 1]
         assert s.strides == [3, 10]
         assert s.backstrides == [3, 0]
-        s = a.create_slice([(0, 10, 1, 10), (2, 0, 0, 1)])
+        s = a.create_slice([Chunk(0, 10, 1, 10), Chunk(2, 0, 0, 1)])
         assert s.start == 20
         assert s.shape == [10, 3]
 
     def test_create_slice_c(self):
         a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'C')
-        s = a.create_slice([(3, 0, 0, 1)])
+        s = a.create_slice([Chunk(3, 0, 0, 1)])
         assert s.start == 45
         assert s.strides == [3, 1]
         assert s.backstrides == [12, 2]
-        s = a.create_slice([(1, 9, 2, 4)])
+        s = a.create_slice([Chunk(1, 9, 2, 4)])
         assert s.start == 15
         assert s.strides == [30, 3, 1]
         assert s.backstrides == [90, 12, 2]
-        s = a.create_slice([(1, 5, 3, 2), (1, 2, 1, 1), (1, 0, 0, 1)])
+        s = a.create_slice([Chunk(1, 5, 3, 2), Chunk(1, 2, 1, 1),
+                            Chunk(1, 0, 0, 1)])
         assert s.start == 19
         assert s.shape == [2, 1]
         assert s.strides == [45, 3]
         assert s.backstrides == [45, 0]
-        s = a.create_slice([(0, 10, 1, 10), (2, 0, 0, 1)])
+        s = a.create_slice([Chunk(0, 10, 1, 10), Chunk(2, 0, 0, 1)])
         assert s.start == 6
         assert s.shape == [10, 3]
 
     def test_slice_of_slice_f(self):
         a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'F')
-        s = a.create_slice([(5, 0, 0, 1)])
+        s = a.create_slice([Chunk(5, 0, 0, 1)])
         assert s.start == 5
-        s2 = s.create_slice([(3, 0, 0, 1)])
+        s2 = s.create_slice([Chunk(3, 0, 0, 1)])
         assert s2.shape == [3]
         assert s2.strides == [50]
         assert s2.parent is a
         assert s2.backstrides == [100]
         assert s2.start == 35
-        s = a.create_slice([(1, 5, 3, 2)])
-        s2 = s.create_slice([(0, 2, 1, 2), (2, 0, 0, 1)])
+        s = a.create_slice([Chunk(1, 5, 3, 2)])
+        s2 = s.create_slice([Chunk(0, 2, 1, 2), Chunk(2, 0, 0, 1)])
         assert s2.shape == [2, 3]
         assert s2.strides == [3, 50]
         assert s2.backstrides == [3, 100]
@@ -91,16 +93,16 @@
 
     def test_slice_of_slice_c(self):
         a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), order='C')
-        s = a.create_slice([(5, 0, 0, 1)])
+        s = a.create_slice([Chunk(5, 0, 0, 1)])
         assert s.start == 15 * 5
-        s2 = s.create_slice([(3, 0, 0, 1)])
+        s2 = s.create_slice([Chunk(3, 0, 0, 1)])
         assert s2.shape == [3]
         assert s2.strides == [1]
         assert s2.parent is a
         assert s2.backstrides == [2]
         assert s2.start == 5 * 15 + 3 * 3
-        s = a.create_slice([(1, 5, 3, 2)])
-        s2 = s.create_slice([(0, 2, 1, 2), (2, 0, 0, 1)])
+        s = a.create_slice([Chunk(1, 5, 3, 2)])
+        s2 = s.create_slice([Chunk(0, 2, 1, 2), Chunk(2, 0, 0, 1)])
         assert s2.shape == [2, 3]
         assert s2.strides == [45, 1]
         assert s2.backstrides == [45, 2]
@@ -108,14 +110,14 @@
 
     def test_negative_step_f(self):
         a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'F')
-        s = a.create_slice([(9, -1, -2, 5)])
+        s = a.create_slice([Chunk(9, -1, -2, 5)])
         assert s.start == 9
         assert s.strides == [-2, 10, 50]
         assert s.backstrides == [-8, 40, 100]
 
     def test_negative_step_c(self):
         a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), order='C')
-        s = a.create_slice([(9, -1, -2, 5)])
+        s = a.create_slice([Chunk(9, -1, -2, 5)])
         assert s.start == 135
         assert s.strides == [-30, 3, 1]
         assert s.backstrides == [-120, 12, 2]
@@ -124,7 +126,7 @@
         a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'F')
         r = a._index_of_single_item(self.space, self.newtuple(1, 2, 2))
         assert r == 1 + 2 * 10 + 2 * 50
-        s = a.create_slice([(0, 10, 1, 10), (2, 0, 0, 1)])
+        s = a.create_slice([Chunk(0, 10, 1, 10), Chunk(2, 0, 0, 1)])
         r = s._index_of_single_item(self.space, self.newtuple(1, 0))
         assert r == a._index_of_single_item(self.space, self.newtuple(1, 2, 0))
         r = s._index_of_single_item(self.space, self.newtuple(1, 1))
@@ -134,7 +136,7 @@
         a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'C')
         r = a._index_of_single_item(self.space, self.newtuple(1, 2, 2))
         assert r == 1 * 3 * 5 + 2 * 3 + 2
-        s = a.create_slice([(0, 10, 1, 10), (2, 0, 0, 1)])
+        s = a.create_slice([Chunk(0, 10, 1, 10), Chunk(2, 0, 0, 1)])
         r = s._index_of_single_item(self.space, self.newtuple(1, 0))
         assert r == a._index_of_single_item(self.space, self.newtuple(1, 2, 0))
         r = s._index_of_single_item(self.space, self.newtuple(1, 1))
@@ -726,15 +728,16 @@
         assert d[1] == 12
 
     def test_mean(self):
-        from _numpypy import array, mean
+        from _numpypy import array
         a = array(range(5))
         assert a.mean() == 2.0
         assert a[:4].mean() == 1.5
         a = array(range(105)).reshape(3, 5, 7)
-        b = mean(a, axis=0)
-        b[0,0]==35.
+        b = a.mean(axis=0)
+        b[0, 0]==35.
+        assert a.mean(axis=0)[0, 0] == 35
         assert (b == array(range(35, 70), dtype=float).reshape(5, 7)).all()
-        assert (mean(a, 2) == array(range(0, 15), dtype=float).reshape(3, 5) * 7 + 3).all()
+        assert (a.mean(2) == array(range(0, 15), dtype=float).reshape(3, 5) * 7 + 3).all()
 
     def test_sum(self):
         from _numpypy import array
@@ -755,6 +758,7 @@
         assert array([]).sum() == 0.0
         raises(ValueError, 'array([]).max()')
         assert (a.sum(0) == [30, 35, 40]).all()
+        assert (a.sum(axis=0) == [30, 35, 40]).all()
         assert (a.sum(1) == [3, 12, 21, 30, 39]).all()
         assert (a.max(0) == [12, 13, 14]).all()
         assert (a.max(1) == [2, 5, 8, 11, 14]).all()
@@ -769,6 +773,8 @@
         assert ((a + a).T.sum(2).T == (a + a).sum(0)).all()
         assert (a.reshape(1,-1).sum(0) == range(105)).all()
         assert (a.reshape(1,-1).sum(1) == 5460)
+        assert (array([[1,2],[3,4]]).prod(0) == [3, 8]).all()
+        assert (array([[1,2],[3,4]]).prod(1) == [2, 12]).all()
 
     def test_identity(self):
         from _numpypy import identity, array
@@ -1298,6 +1304,56 @@
         assert isinstance(i['data'][0], int)
         raises(TypeError, getattr, array(3), '__array_interface__')
 
+    def test_array_indexing_one_elem(self):
+        skip("not yet")
+        from _numpypy import array, arange
+        raises(IndexError, 'arange(3)[array([3.5])]')
+        a = arange(3)[array([1])]
+        assert a == 1
+        assert a[0] == 1
+        raises(IndexError,'arange(3)[array([15])]')
+        assert arange(3)[array([-3])] == 0
+        raises(IndexError,'arange(3)[array([-15])]')
+        assert arange(3)[array(1)] == 1
+
+    def test_fill(self):
+        from _numpypy import array
+        a = array([1, 2, 3])
+        a.fill(10)
+        assert (a == [10, 10, 10]).all()
+        a.fill(False)
+        assert (a == [0, 0, 0]).all()
+        b = a[:1]
+        b.fill(4)
+        assert (b == [4]).all()
+        assert (a == [4, 0, 0]).all()
+
+        c = b + b
+        c.fill(27)
+        assert (c == [27]).all()
+
+        d = array(10)
+        d.fill(100)
+        assert d == 100
+
+    def test_array_indexing_bool(self):
+        from _numpypy import arange
+        a = arange(10)
+        assert (a[a > 3] == [4, 5, 6, 7, 8, 9]).all()
+        a = arange(10).reshape(5, 2)
+        assert (a[a > 3] == [4, 5, 6, 7, 8, 9]).all()
+        assert (a[a & 1 == 1] == [1, 3, 5, 7, 9]).all()
+
+    def test_array_indexing_bool_setitem(self):
+        from _numpypy import arange, array
+        a = arange(6)
+        a[a > 3] = 15
+        assert (a == [0, 1, 2, 3, 15, 15]).all()
+        a = arange(6).reshape(3, 2)
+        a[a & 1 == 1] = array([8, 9, 10])
+        assert (a == [[0, 8], [2, 9], [4, 10]]).all()
+
+
 
 class AppTestSupport(BaseNumpyAppTest):
     def setup_class(cls):
@@ -1437,9 +1493,11 @@
         assert repr(a) == "array(0.0)"
         a = array(0.2)
         assert repr(a) == "array(0.2)"
+        a = array([2])
+        assert repr(a) == "array([2])"
 
     def test_repr_multi(self):
-        from _numpypy import arange, zeros
+        from _numpypy import arange, zeros, array
         a = zeros((3, 4))
         assert repr(a) == '''array([[0.0, 0.0, 0.0, 0.0],
        [0.0, 0.0, 0.0, 0.0],
@@ -1462,6 +1520,9 @@
        [498, 999],
        [499, 1000],
        [500, 1001]])'''
+        a = arange(2).reshape((2,1))
+        assert repr(a) == '''array([[0],
+       [1]])'''
 
     def test_repr_slice(self):
         from _numpypy import array, zeros
@@ -1546,18 +1607,3 @@
         a = arange(0, 0.8, 0.1)
         assert len(a) == 8
         assert arange(False, True, True).dtype is dtype(int)
-
-
-class AppTestRanges(BaseNumpyAppTest):
-    def test_app_reshape(self):
-        from _numpypy import arange, array, dtype, reshape
-        a = arange(12)
-        b = reshape(a, (3, 4))
-        assert b.shape == (3, 4)
-        a = range(12)
-        b = reshape(a, (3, 4))
-        assert b.shape == (3, 4)
-        a = array(range(105)).reshape(3, 5, 7)
-        assert a.reshape(1, -1).shape == (1, 105)
-        assert a.reshape(1, 1, -1).shape == (1, 1, 105)
-        assert a.reshape(-1, 1, 1).shape == (105, 1, 1)
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -190,14 +190,24 @@
         for i in range(3):
             assert c[i] == a[i] - b[i]
 
-    def test_floor(self):
-        from _numpypy import array, floor
-
-        reference = [-2.0, -1.0, 0.0, 1.0, 1.0]
-        a = array([-1.4, -1.0, 0.0, 1.0, 1.4])
+    def test_floorceil(self):
+        from _numpypy import array, floor, ceil
+        import math
+        reference = [-2.0, -2.0, -1.0, 0.0, 1.0, 1.0, 0]
+        a = array([-1.4, -1.5, -1.0, 0.0, 1.0, 1.4, 0.5])
         b = floor(a)
         for i in range(5):
             assert b[i] == reference[i]
+        reference = [-1.0, -1.0, -1.0, 0.0, 1.0, 2.0, 1.0]
+        a = array([-1.4, -1.5, -1.0, 0.0, 1.0, 1.4, 0.5])
+        b = ceil(a)
+        assert (reference == b).all()
+        inf = float("inf")
+        data = [1.5, 2.9999, -1.999, inf]
+        results = [math.floor(x) for x in data]
+        assert (floor(data) == results).all()
+        results = [math.ceil(x) for x in data]
+        assert (ceil(data) == results).all()
 
     def test_copysign(self):
         from _numpypy import array, copysign
@@ -238,7 +248,7 @@
             assert b[i] == math.sin(a[i])
 
         a = sin(array([True, False], dtype=bool))
-        assert abs(a[0] - sin(1)) < 1e-7 # a[0] will be less precise
+        assert abs(a[0] - sin(1)) < 1e-7  # a[0] will be less precise
         assert a[1] == 0.0
 
     def test_cos(self):
@@ -259,7 +269,6 @@
         for i in range(len(a)):
             assert b[i] == math.tan(a[i])
 
-
     def test_arcsin(self):
         import math
         from _numpypy import array, arcsin
@@ -283,7 +292,6 @@
         for i in range(len(a)):
             assert b[i] == math.acos(a[i])
 
-
         a = array([-10, -1.5, -1.01, 1.01, 1.5, 10, float('nan'), float('inf'), float('-inf')])
         b = arccos(a)
         for f in b:
@@ -347,11 +355,20 @@
         raises(ValueError, maximum.reduce, [])
 
     def test_reduceND(self):
-        from numpypy import add, arange
+        from _numpypy import add, arange
         a = arange(12).reshape(3, 4)
         assert (add.reduce(a, 0) == [12, 15, 18, 21]).all()
         assert (add.reduce(a, 1) == [6.0, 22.0, 38.0]).all()
 
+    def test_bitwise(self):
+        from _numpypy import bitwise_and, bitwise_or, arange, array
+        a = arange(6).reshape(2, 3)
+        assert (a & 1 == [[0, 1, 0], [1, 0, 1]]).all()
+        assert (a & 1 == bitwise_and(a, 1)).all()
+        assert (a | 1 == [[1, 1, 3], [3, 5, 5]]).all()
+        assert (a | 1 == bitwise_or(a, 1)).all()
+        raises(TypeError, 'array([1.0]) & 1')
+
     def test_comparisons(self):
         import operator
         from _numpypy import equal, not_equal, less, less_equal, greater, greater_equal
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -217,6 +217,7 @@
         # This is the sum of the ops for both loops, however if you remove the
         # optimization then you end up with 2 float_adds, so we can still be
         # sure it was optimized correctly.
+        py.test.skip("too fragile")
         self.check_resops({'setinteriorfield_raw': 4, 'getfield_gc': 22,
                            'getarrayitem_gc': 4, 'getarrayitem_gc_pure': 2,
                            'getfield_gc_pure': 8,
@@ -349,7 +350,8 @@
         self.check_trace_count(1)
         self.check_simple_loop({'getinteriorfield_raw': 2, 'float_add': 1,
                                 'setinteriorfield_raw': 1, 'int_add': 2,
-                                'int_eq': 1, 'guard_false': 1, 'jump': 1})
+                                'int_eq': 1, 'guard_false': 1, 'jump': 1,
+                                'arraylen_gc': 1})
 
     def define_virtual_slice():
         return """
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -94,6 +94,9 @@
             width, storage, i, offset
         ))
 
+    def read_bool(self, storage, width, i, offset):
+        raise NotImplementedError
+
     def store(self, storage, width, i, offset, box):
         value = self.unbox(box)
         libffi.array_setitem(clibffi.cast_type_to_ffitype(self.T),
@@ -168,6 +171,7 @@
     @simple_binary_op
     def min(self, v1, v2):
         return min(v1, v2)
+    
 
 class Bool(BaseType, Primitive):
     T = lltype.Bool
@@ -185,6 +189,11 @@
         else:
             return self.False
 
+
+    def read_bool(self, storage, width, i, offset):
+        return libffi.array_getitem(clibffi.cast_type_to_ffitype(self.T),
+                                    width, storage, i, offset)
+
     def coerce_subtype(self, space, w_subtype, w_item):
         # Doesn't return subclasses so it can return the constants.
         return self._coerce(space, w_item)
@@ -253,6 +262,14 @@
             assert v == 0
             return 0
 
+    @simple_binary_op
+    def bitwise_and(self, v1, v2):
+        return v1 & v2
+
+    @simple_binary_op
+    def bitwise_or(self, v1, v2):
+        return v1 | v2
+
 class Int8(BaseType, Integer):
     T = rffi.SIGNEDCHAR
     BoxType = interp_boxes.W_Int8Box
@@ -374,6 +391,10 @@
         return math.floor(v)
 
     @simple_unary_op
+    def ceil(self, v):
+        return math.ceil(v)
+
+    @simple_unary_op
     def exp(self, v):
         try:
             return math.exp(v)
@@ -436,4 +457,4 @@
 class Float64(BaseType, Float):
     T = rffi.DOUBLE
     BoxType = interp_boxes.W_Float64Box
-    format_code = "d"
\ No newline at end of file
+    format_code = "d"
diff --git a/pypy/module/pypyjit/__init__.py b/pypy/module/pypyjit/__init__.py
--- a/pypy/module/pypyjit/__init__.py
+++ b/pypy/module/pypyjit/__init__.py
@@ -11,6 +11,7 @@
         'set_optimize_hook': 'interp_resop.set_optimize_hook',
         'set_abort_hook': 'interp_resop.set_abort_hook',
         'ResOperation': 'interp_resop.WrappedOp',
+        'DebugMergePoint': 'interp_resop.DebugMergePoint',
         'Box': 'interp_resop.WrappedBox',
     }
 
diff --git a/pypy/module/pypyjit/interp_resop.py b/pypy/module/pypyjit/interp_resop.py
--- a/pypy/module/pypyjit/interp_resop.py
+++ b/pypy/module/pypyjit/interp_resop.py
@@ -1,5 +1,6 @@
 
-from pypy.interpreter.typedef import TypeDef, GetSetProperty
+from pypy.interpreter.typedef import (TypeDef, GetSetProperty,
+     interp_attrproperty, interp_attrproperty_w)
 from pypy.interpreter.baseobjspace import Wrappable
 from pypy.interpreter.gateway import unwrap_spec, interp2app, NoneNotWrapped
 from pypy.interpreter.pycode import PyCode
@@ -10,6 +11,7 @@
 from pypy.jit.metainterp.resoperation import rop, AbstractResOp
 from pypy.rlib.nonconst import NonConstant
 from pypy.rlib import jit_hooks
+from pypy.module.pypyjit.interp_jit import pypyjitdriver
 
 class Cache(object):
     in_recursion = False
@@ -111,13 +113,24 @@
 
 def wrap_oplist(space, logops, operations, ops_offset=None):
     l_w = []
+    jitdrivers_sd = logops.metainterp_sd.jitdrivers_sd
     for op in operations:
         if ops_offset is None:
             ofs = -1
         else:
             ofs = ops_offset.get(op, 0)
-        l_w.append(WrappedOp(jit_hooks._cast_to_gcref(op), ofs,
-                             logops.repr_of_resop(op)))
+        if op.opnum == rop.DEBUG_MERGE_POINT:
+            jd_sd = jitdrivers_sd[op.getarg(0).getint()]
+            greenkey = op.getarglist()[2:]
+            repr = jd_sd.warmstate.get_location_str(greenkey)
+            w_greenkey = wrap_greenkey(space, jd_sd.jitdriver, greenkey, repr)
+            l_w.append(DebugMergePoint(space, jit_hooks._cast_to_gcref(op),
+                                       logops.repr_of_resop(op),
+                                       jd_sd.jitdriver.name,
+                                       w_greenkey))
+        else:
+            l_w.append(WrappedOp(jit_hooks._cast_to_gcref(op), ofs,
+                                 logops.repr_of_resop(op)))
     return l_w
 
 class WrappedBox(Wrappable):
@@ -150,6 +163,15 @@
         llres = res.llbox
     return WrappedOp(jit_hooks.resop_new(num, args, llres), offset, repr)
 
+ at unwrap_spec(repr=str, jd_name=str)
+def descr_new_dmp(space, w_tp, w_args, repr, jd_name, w_greenkey):
+    args = [space.interp_w(WrappedBox, w_arg).llbox for w_arg in
+            space.listview(w_args)]
+    num = rop.DEBUG_MERGE_POINT
+    return DebugMergePoint(space,
+                           jit_hooks.resop_new(num, args, jit_hooks.emptyval()),
+                           repr, jd_name, w_greenkey)
+
 class WrappedOp(Wrappable):
     """ A class representing a single ResOperation, wrapped nicely
     """
@@ -182,6 +204,25 @@
         box = space.interp_w(WrappedBox, w_box)
         jit_hooks.resop_setresult(self.op, box.llbox)
 
+class DebugMergePoint(WrappedOp):
+    def __init__(self, space, op, repr_of_resop, jd_name, w_greenkey):
+        WrappedOp.__init__(self, op, -1, repr_of_resop)
+        self.w_greenkey = w_greenkey
+        self.jd_name = jd_name
+
+    def get_pycode(self, space):
+        if self.jd_name == pypyjitdriver.name:
+            return space.getitem(self.w_greenkey, space.wrap(0))
+        raise OperationError(space.w_AttributeError, space.wrap("This DebugMergePoint doesn't belong to the main Python JitDriver"))
+
+    def get_bytecode_no(self, space):
+        if self.jd_name == pypyjitdriver.name:
+            return space.getitem(self.w_greenkey, space.wrap(1))
+        raise OperationError(space.w_AttributeError, space.wrap("This DebugMergePoint doesn't belong to the main Python JitDriver"))
+
+    def get_jitdriver_name(self, space):
+        return space.wrap(self.jd_name)
+
 WrappedOp.typedef = TypeDef(
     'ResOperation',
     __doc__ = WrappedOp.__doc__,
@@ -195,3 +236,15 @@
                             WrappedOp.descr_setresult)
 )
 WrappedOp.acceptable_as_base_class = False
+
+DebugMergePoint.typedef = TypeDef(
+    'DebugMergePoint', WrappedOp.typedef,
+    __new__ = interp2app(descr_new_dmp),
+    greenkey = interp_attrproperty_w("w_greenkey", cls=DebugMergePoint),
+    pycode = GetSetProperty(DebugMergePoint.get_pycode),
+    bytecode_no = GetSetProperty(DebugMergePoint.get_bytecode_no),
+    jitdriver_name = GetSetProperty(DebugMergePoint.get_jitdriver_name),
+)
+DebugMergePoint.acceptable_as_base_class = False
+
+
diff --git a/pypy/module/pypyjit/policy.py b/pypy/module/pypyjit/policy.py
--- a/pypy/module/pypyjit/policy.py
+++ b/pypy/module/pypyjit/policy.py
@@ -127,7 +127,7 @@
                        'imp', 'sys', 'array', '_ffi', 'itertools', 'operator',
                        'posix', '_socket', '_sre', '_lsprof', '_weakref',
                        '__pypy__', 'cStringIO', '_collections', 'struct',
-                       'mmap', 'marshal']:
+                       'mmap', 'marshal', '_codecs']:
             if modname == 'pypyjit' and 'interp_resop' in rest:
                 return False
             return True
diff --git a/pypy/module/pypyjit/test/test_jit_hook.py b/pypy/module/pypyjit/test/test_jit_hook.py
--- a/pypy/module/pypyjit/test/test_jit_hook.py
+++ b/pypy/module/pypyjit/test/test_jit_hook.py
@@ -92,6 +92,7 @@
         cls.w_on_compile_bridge = space.wrap(interp2app(interp_on_compile_bridge))
         cls.w_on_abort = space.wrap(interp2app(interp_on_abort))
         cls.w_int_add_num = space.wrap(rop.INT_ADD)
+        cls.w_dmp_num = space.wrap(rop.DEBUG_MERGE_POINT)
         cls.w_on_optimize = space.wrap(interp2app(interp_on_optimize))
         cls.orig_oplist = oplist
 
@@ -117,6 +118,10 @@
         assert elem[2][2] == False
         assert len(elem[3]) == 4
         int_add = elem[3][0]
+        dmp = elem[3][1]
+        assert isinstance(dmp, pypyjit.DebugMergePoint)
+        assert dmp.pycode is self.f.func_code
+        assert dmp.greenkey == (self.f.func_code, 0, False)
         #assert int_add.name == 'int_add'
         assert int_add.num == self.int_add_num
         self.on_compile_bridge()
@@ -211,3 +216,18 @@
         assert op.getarg(0).getint() == 4
         op.result = box
         assert op.result.getint() == 1
+
+    def test_creation_dmp(self):
+        from pypyjit import DebugMergePoint, Box
+
+        def f():
+            pass
+
+        op = DebugMergePoint([Box(0)], 'repr', 'pypyjit', (f.func_code, 0, 0))
+        assert op.bytecode_no == 0
+        assert op.pycode is f.func_code
+        assert repr(op) == 'repr'
+        assert op.jitdriver_name == 'pypyjit'
+        assert op.num == self.dmp_num
+        op = DebugMergePoint([Box(0)], 'repr', 'notmain', ('str',))
+        raises(AttributeError, 'op.pycode')
diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py
--- a/pypy/module/sys/version.py
+++ b/pypy/module/sys/version.py
@@ -10,7 +10,7 @@
 CPYTHON_VERSION            = (3, 2, 2, "final", 0)   #XXX # sync patchlevel.h
 CPYTHON_API_VERSION        = 1013   #XXX # sync with include/modsupport.h
 
-PYPY_VERSION               = (1, 7, 1, "dev", 0)    #XXX # sync patchlevel.h
+PYPY_VERSION               = (1, 8, 1, "dev", 0)    #XXX # sync patchlevel.h
 
 if platform.name == 'msvc':
     COMPILER_INFO = 'MSC v.%d 32 bit' % (platform.version * 10 + 600)
diff --git a/lib_pypy/numpypy/test/test_fromnumeric.py b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
rename from lib_pypy/numpypy/test/test_fromnumeric.py
rename to pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
--- a/lib_pypy/numpypy/test/test_fromnumeric.py
+++ b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
@@ -1,7 +1,7 @@
-
 from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
 
-class AppTestFromNumeric(BaseNumpyAppTest):     
+
+class AppTestFromNumeric(BaseNumpyAppTest):
     def test_argmax(self):
         # tests taken from numpy/core/fromnumeric.py docstring
         from numpypy import array, arange, argmax
@@ -18,12 +18,12 @@
         from numpypy import array, arange, argmin
         a = arange(6).reshape((2,3))
         assert argmin(a) == 0
-        # assert (argmax(a, axis=0) == array([0, 0, 0])).all()
-        # assert (argmax(a, axis=1) == array([0, 0])).all()
+        assert (argmin(a, axis=0) == array([0, 0, 0])).all()
+        assert (argmin(a, axis=1) == array([0, 0])).all()
         b = arange(6)
         b[1] = 0
         assert argmin(b) == 0
-   
+
     def test_shape(self):
         # tests taken from numpy/core/fromnumeric.py docstring
         from numpypy import array, identity, shape
@@ -44,7 +44,7 @@
         # assert (sum([[0, 1], [0, 5]], axis=1) == array([1, 5])).all()
         # If the accumulator is too small, overflow occurs:
         # assert ones(128, dtype=int8).sum(dtype=int8) == -128
-                                 
+
     def test_amin(self):
         # tests taken from numpy/core/fromnumeric.py docstring
         from numpypy import array, arange, amin
@@ -86,14 +86,14 @@
         assert ndim([[1,2,3],[4,5,6]]) == 2
         assert ndim(array([[1,2,3],[4,5,6]])) == 2
         assert ndim(1) == 0
-    
+
     def test_rank(self):
         # tests taken from numpy/core/fromnumeric.py docstring
         from numpypy import array, rank
         assert rank([[1,2,3],[4,5,6]]) == 2
         assert rank(array([[1,2,3],[4,5,6]])) == 2
         assert rank(1) == 0
-    
+
     def test_var(self):
         from numpypy import array, var
         a = array([[1,2],[3,4]])
@@ -107,3 +107,31 @@
         assert std(a) ==  1.1180339887498949
         # assert (std(a, axis=0) == array([ 1.,  1.])).all()
         # assert (std(a, axis=1) == array([ 0.5,  0.5]).all()
+
+    def test_mean(self):
+        from numpypy import array, mean
+        assert mean(array(range(5))) == 2.0
+        assert mean(range(5)) == 2.0
+
+    def test_reshape(self):
+        from numpypy import arange, array, dtype, reshape
+        a = arange(12)
+        b = reshape(a, (3, 4))
+        assert b.shape == (3, 4)
+        a = range(12)
+        b = reshape(a, (3, 4))
+        assert b.shape == (3, 4)
+        a = array(range(105)).reshape(3, 5, 7)
+        assert reshape(a, (1, -1)).shape == (1, 105)
+        assert reshape(a, (1, 1, -1)).shape == (1, 1, 105)
+        assert reshape(a, (-1, 1, 1)).shape == (105, 1, 1)
+    
+    def test_transpose(self):   
+        from numpypy import arange, array, transpose, ones
+        x = arange(4).reshape((2,2))
+        assert (transpose(x) == array([[0, 2],[1, 3]])).all()
+        # Once axes argument is implemented, add more tests
+        raises(NotImplementedError, "transpose(x, axes=(1, 0, 2))")
+        # x = ones((1, 2, 3))
+        # assert transpose(x, (1, 0, 2)).shape == (2, 1, 3)
+
diff --git a/pypy/objspace/std/test/test_unicodeobject.py b/pypy/objspace/std/test/test_unicodeobject.py
--- a/pypy/objspace/std/test/test_unicodeobject.py
+++ b/pypy/objspace/std/test/test_unicodeobject.py
@@ -29,6 +29,8 @@
             assert type(a) == type(b)
         check(', '.join(['a']), 'a')
         raises(TypeError, ','.join, [b'a']) 
+        exc = raises(TypeError, ''.join, ['a', 2, 3])
+        assert 'sequence item 1' in str(e.value)
 
     def test_contains(self):
         assert '' in 'abc'
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -129,7 +129,7 @@
             return space.wrap(l[0])
         return space.wrap(w_self._value.join(l))
 
-    list_w = space.unpackiterable(w_list)
+    list_w = space.listview(w_list)
     size = len(list_w)
 
     if size == 0:
@@ -144,17 +144,23 @@
 
 def _unicode_join_many_items(space, w_self, list_w, size):
     self = w_self._value
-    sb = UnicodeBuilder()
+    prealloc_size = len(self) * (size - 1)
+    for i in range(size):
+        w_s = list_w[i]
+        try:
+            prealloc_size += len(space.unicode_w(w_s))
+        except OperationError, e:
+            if not e.match(space, space.w_TypeError):
+                raise
+            raise operationerrfmt(space.w_TypeError,
+                                  "sequence item %d: expected string, %s "
+                                  "found", i, space.type(w_s).getname(space))
+    sb = UnicodeBuilder(prealloc_size)
     for i in range(size):
         if self and i != 0:
             sb.append(self)
         w_s = list_w[i]
-        if not isinstance(w_s, W_UnicodeObject):
-            raise operationerrfmt(
-                space.w_TypeError,
-                "sequence item %d: expected string, %s "
-                "found", i, space.type(w_s).getname(space))
-        sb.append(w_s._value)
+        sb.append(space.unicode_w(w_s))
     return space.wrap(sb.build())
 
 def hash__Unicode(space, w_uni):
diff --git a/pypy/rlib/jit_hooks.py b/pypy/rlib/jit_hooks.py
--- a/pypy/rlib/jit_hooks.py
+++ b/pypy/rlib/jit_hooks.py
@@ -52,7 +52,10 @@
     from pypy.jit.metainterp.history import ResOperation
 
     args = [_cast_to_box(llargs[i]) for i in range(len(llargs))]
-    res = _cast_to_box(llres)
+    if llres:
+        res = _cast_to_box(llres)
+    else:
+        res = None
     return _cast_to_gcref(ResOperation(no, args, res))
 
 @register_helper(annmodel.SomePtr(llmemory.GCREF))
diff --git a/pypy/rlib/longlong2float.py b/pypy/rlib/longlong2float.py
--- a/pypy/rlib/longlong2float.py
+++ b/pypy/rlib/longlong2float.py
@@ -79,19 +79,23 @@
 longlong2float = rffi.llexternal(
     "pypy__longlong2float", [rffi.LONGLONG], rffi.DOUBLE,
     _callable=longlong2float_emulator, compilation_info=eci,
-    _nowrapper=True, elidable_function=True, sandboxsafe=True)
+    _nowrapper=True, elidable_function=True, sandboxsafe=True,
+    oo_primitive="pypy__longlong2float")
 
 float2longlong = rffi.llexternal(
     "pypy__float2longlong", [rffi.DOUBLE], rffi.LONGLONG,
     _callable=float2longlong_emulator, compilation_info=eci,
-    _nowrapper=True, elidable_function=True, sandboxsafe=True)
+    _nowrapper=True, elidable_function=True, sandboxsafe=True,
+    oo_primitive="pypy__float2longlong")
 
 uint2singlefloat = rffi.llexternal(
     "pypy__uint2singlefloat", [rffi.UINT], rffi.FLOAT,
     _callable=uint2singlefloat_emulator, compilation_info=eci,
-    _nowrapper=True, elidable_function=True, sandboxsafe=True)
+    _nowrapper=True, elidable_function=True, sandboxsafe=True,
+    oo_primitive="pypy__uint2singlefloat")
 
 singlefloat2uint = rffi.llexternal(
     "pypy__singlefloat2uint", [rffi.FLOAT], rffi.UINT,
     _callable=singlefloat2uint_emulator, compilation_info=eci,
-    _nowrapper=True, elidable_function=True, sandboxsafe=True)
+    _nowrapper=True, elidable_function=True, sandboxsafe=True,
+    oo_primitive="pypy__singlefloat2uint")
diff --git a/pypy/rlib/objectmodel.py b/pypy/rlib/objectmodel.py
--- a/pypy/rlib/objectmodel.py
+++ b/pypy/rlib/objectmodel.py
@@ -424,7 +424,7 @@
                   vobj.concretetype.TO._gckind == 'gc')
         else:
             from pypy.rpython.ootypesystem import ootype
-            ok = isinstance(vobj.concretetype, ootype.Instance)
+            ok = isinstance(vobj.concretetype, (ootype.Instance, ootype.BuiltinType))
         if not ok:
             from pypy.rpython.error import TyperError
             raise TyperError("compute_unique_id() cannot be applied to"
diff --git a/pypy/rpython/lltypesystem/rclass.py b/pypy/rpython/lltypesystem/rclass.py
--- a/pypy/rpython/lltypesystem/rclass.py
+++ b/pypy/rpython/lltypesystem/rclass.py
@@ -510,7 +510,13 @@
         ctype = inputconst(Void, self.object_type)
         cflags = inputconst(Void, flags)
         vlist = [ctype, cflags]
-        vptr = llops.genop('malloc', vlist,
+        cnonmovable = self.classdef.classdesc.read_attribute(
+            '_alloc_nonmovable_', Constant(False))
+        if cnonmovable.value:
+            opname = 'malloc_nonmovable'
+        else:
+            opname = 'malloc'
+        vptr = llops.genop(opname, vlist,
                            resulttype = Ptr(self.object_type))
         ctypeptr = inputconst(CLASSTYPE, self.rclass.getvtable())
         self.setfield(vptr, '__class__', ctypeptr, llops)
diff --git a/pypy/rpython/ootypesystem/ootype.py b/pypy/rpython/ootypesystem/ootype.py
--- a/pypy/rpython/ootypesystem/ootype.py
+++ b/pypy/rpython/ootypesystem/ootype.py
@@ -512,6 +512,7 @@
             "ll_append_char": Meth([CHARTP], Void),
             "ll_append": Meth([STRINGTP], Void),
             "ll_build": Meth([], STRINGTP),
+            "ll_getlength": Meth([], Signed),
             })
         self._setup_methods({})
 
@@ -1376,6 +1377,9 @@
     def _cast_to_object(self):
         return make_object(self)
 
+    def _identityhash(self):
+        return object.__hash__(self)
+
 class _string(_builtin_type):
 
     def __init__(self, STRING, value = ''):
@@ -1543,6 +1547,9 @@
         else:
             return make_unicode(u''.join(self._buf))
 
+    def ll_getlength(self):
+        return self.ll_build().ll_strlen()
+
 class _null_string_builder(_null_mixin(_string_builder), _string_builder):
     def __init__(self, STRING_BUILDER):
         self.__dict__["_TYPE"] = STRING_BUILDER
diff --git a/pypy/rpython/ootypesystem/rbuilder.py b/pypy/rpython/ootypesystem/rbuilder.py
--- a/pypy/rpython/ootypesystem/rbuilder.py
+++ b/pypy/rpython/ootypesystem/rbuilder.py
@@ -21,6 +21,10 @@
         builder.ll_append_char(char)
 
     @staticmethod
+    def ll_getlength(builder):
+        return builder.ll_getlength()
+
+    @staticmethod
     def ll_append(builder, string):
         builder.ll_append(string)
 
diff --git a/pypy/rpython/test/test_rbuilder.py b/pypy/rpython/test/test_rbuilder.py
--- a/pypy/rpython/test/test_rbuilder.py
+++ b/pypy/rpython/test/test_rbuilder.py
@@ -124,9 +124,5 @@
     pass
 
 class TestOOtype(BaseTestStringBuilder, OORtypeMixin):
-    def test_string_getlength(self):
-        py.test.skip("getlength(): not implemented on ootype")
-    def test_unicode_getlength(self):
-        py.test.skip("getlength(): not implemented on ootype")
     def test_append_charpsize(self):
         py.test.skip("append_charpsize(): not implemented on ootype")
diff --git a/pypy/rpython/test/test_rbuiltin.py b/pypy/rpython/test/test_rbuiltin.py
--- a/pypy/rpython/test/test_rbuiltin.py
+++ b/pypy/rpython/test/test_rbuiltin.py
@@ -463,6 +463,31 @@
             assert x1 == intmask(x0)
             assert x3 == intmask(x2)
 
+    def test_id_on_builtins(self):
+        from pypy.rlib.objectmodel import compute_unique_id
+        from pypy.rlib.rstring import StringBuilder, UnicodeBuilder
+        def fn():
+            return (compute_unique_id("foo"),
+                    compute_unique_id(u"bar"),
+                    compute_unique_id([1]),
+                    compute_unique_id({"foo": 3}),
+                    compute_unique_id(StringBuilder()),
+                    compute_unique_id(UnicodeBuilder()))
+        res = self.interpret(fn, [])
+        for id in self.ll_unpack_tuple(res, 6):
+            assert isinstance(id, (int, r_longlong))
+
+    def test_uniqueness_of_id_on_strings(self):
+        from pypy.rlib.objectmodel import compute_unique_id
+        def fn(s1, s2):
+            return (compute_unique_id(s1), compute_unique_id(s2))
+
+        s1 = "foo"
+        s2 = ''.join(['f','oo'])
+        res = self.interpret(fn, [self.string_to_ll(s1), self.string_to_ll(s2)])
+        i1, i2 = self.ll_unpack_tuple(res, 2)
+        assert i1 != i2
+
     def test_cast_primitive(self):
         from pypy.rpython.annlowlevel import LowLevelAnnotatorPolicy
         def llf(u):
diff --git a/pypy/rpython/test/test_rclass.py b/pypy/rpython/test/test_rclass.py
--- a/pypy/rpython/test/test_rclass.py
+++ b/pypy/rpython/test/test_rclass.py
@@ -1130,6 +1130,18 @@
             assert sorted([u]) == [6]                    # 32-bit types
             assert sorted([i, r, d, l]) == [2, 3, 4, 5]  # 64-bit types
 
+    def test_nonmovable(self):
+        for (nonmovable, opname) in [(True, 'malloc_nonmovable'),
+                                     (False, 'malloc')]:
+            class A(object):
+                _alloc_nonmovable_ = nonmovable
+            def f():
+                return A()
+            t, typer, graph = self.gengraph(f, [])
+            assert summary(graph) == {opname: 1,
+                                      'cast_pointer': 1,
+                                      'setfield': 1}
+
 
 class TestOOtype(BaseTestRclass, OORtypeMixin):
 
diff --git a/pypy/tool/jitlogparser/parser.py b/pypy/tool/jitlogparser/parser.py
--- a/pypy/tool/jitlogparser/parser.py
+++ b/pypy/tool/jitlogparser/parser.py
@@ -140,13 +140,15 @@
     bytecode_name = None
     is_bytecode = True
     inline_level = None
+    has_dmp = False
 
     def parse_code_data(self, arg):
         m = re.search('\w]+)[\.,] file \'(.+?)\'[\.,] line (\d+)> #(\d+) (\w+)',
                       arg)
         if m is None:
             # a non-code loop, like StrLiteralSearch or something
-            self.bytecode_name = arg
+            if arg:
+                self.bytecode_name = arg
         else:
             self.name, self.filename, lineno, bytecode_no, self.bytecode_name = m.groups()
             self.startlineno = int(lineno)
@@ -218,7 +220,7 @@
         self.inputargs = inputargs
         self.chunks = chunks
         for chunk in self.chunks:
-            if chunk.filename is not None:
+            if chunk.bytecode_name is not None:
                 self.startlineno = chunk.startlineno
                 self.filename = chunk.filename
                 self.name = chunk.name
diff --git a/pypy/tool/jitlogparser/test/test_parser.py b/pypy/tool/jitlogparser/test/test_parser.py
--- a/pypy/tool/jitlogparser/test/test_parser.py
+++ b/pypy/tool/jitlogparser/test/test_parser.py
@@ -283,3 +283,13 @@
     assert loops[-1].count == 1234
     assert loops[1].count == 123
     assert loops[2].count == 12
+
+def test_parse_nonpython():
+    loop = parse("""
+    []
+    debug_merge_point(0, 'random')
+    debug_merge_point(0, ' #15 COMPARE_OP')
+    """)
+    f = Function.from_operations(loop.operations, LoopStorage())
+    assert f.chunks[-1].filename == 'x.py'
+    assert f.filename is None
diff --git a/pypy/tool/test/test_version.py b/pypy/tool/test/test_version.py
--- a/pypy/tool/test/test_version.py
+++ b/pypy/tool/test/test_version.py
@@ -1,6 +1,22 @@
 import os, sys
 import py
-from pypy.tool.version import get_repo_version_info
+from pypy.tool.version import get_repo_version_info, _get_hg_archive_version
+
+def test_hg_archival_version(tmpdir):
+    def version_for(name, **kw):
+        path = tmpdir.join(name)
+        path.write('\n'.join('%s: %s' % x for x in kw.items()))
+        return _get_hg_archive_version(str(path))
+
+    assert version_for('release',
+                       tag='release-123',
+                       node='000',
+                      ) == ('PyPy', 'release-123', '000')
+    assert version_for('somebranch',
+                       node='000',
+                       branch='something',
+                      ) == ('PyPy', 'something', '000')
+
 
 def test_get_repo_version_info():
     assert get_repo_version_info(None)
diff --git a/pypy/tool/version.py b/pypy/tool/version.py
--- a/pypy/tool/version.py
+++ b/pypy/tool/version.py
@@ -3,111 +3,139 @@
 from subprocess import Popen, PIPE
 import pypy
 pypydir = os.path.dirname(os.path.abspath(pypy.__file__))
+pypyroot = os.path.dirname(pypydir)
+default_retval = 'PyPy', '?', '?'
+
+def maywarn(err, repo_type='Mercurial'):
+    if not err:
+        return
+
+    from pypy.tool.ansi_print import ansi_log
+    log = py.log.Producer("version")
+    py.log.setconsumer("version", ansi_log)
+    log.WARNING('Errors getting %s information: %s' % (repo_type, err))
 
 def get_repo_version_info(hgexe=None):
     '''Obtain version information by invoking the 'hg' or 'git' commands.'''
-    # TODO: support extracting from .hg_archival.txt
-
-    default_retval = 'PyPy', '?', '?'
-    pypyroot = os.path.abspath(os.path.join(pypydir, '..'))
-
-    def maywarn(err, repo_type='Mercurial'):
-        if not err:
-            return
-
-        from pypy.tool.ansi_print import ansi_log
-        log = py.log.Producer("version")
-        py.log.setconsumer("version", ansi_log)
-        log.WARNING('Errors getting %s information: %s' % (repo_type, err))
 
     # Try to see if we can get info from Git if hgexe is not specified.
     if not hgexe:
         if os.path.isdir(os.path.join(pypyroot, '.git')):
-            gitexe = py.path.local.sysfind('git')
-            if gitexe:
-                try:
-                    p = Popen(
-                        [str(gitexe), 'rev-parse', 'HEAD'],
-                        stdout=PIPE, stderr=PIPE, cwd=pypyroot
-                        )
-                except OSError, e:
-                    maywarn(e, 'Git')
-                    return default_retval
-                if p.wait() != 0:
-                    maywarn(p.stderr.read(), 'Git')
-                    return default_retval
-                revision_id = p.stdout.read().strip()[:12]
-                p = Popen(
-                    [str(gitexe), 'describe', '--tags', '--exact-match'],
-                    stdout=PIPE, stderr=PIPE, cwd=pypyroot
-                    )
-                if p.wait() != 0:
-                    p = Popen(
-                        [str(gitexe), 'branch'], stdout=PIPE, stderr=PIPE,
-                        cwd=pypyroot
-                        )
-                    if p.wait() != 0:
-                        maywarn(p.stderr.read(), 'Git')
-                        return 'PyPy', '?', revision_id
-                    branch = '?'
-                    for line in p.stdout.read().strip().split('\n'):
-                        if line.startswith('* '):
-                            branch = line[1:].strip()
-                            if branch == '(no branch)':
-                                branch = '?'
-                            break
-                    return 'PyPy', branch, revision_id
-                return 'PyPy', p.stdout.read().strip(), revision_id
+            return _get_git_version()
 
     # Fallback to trying Mercurial.
     if hgexe is None:
         hgexe = py.path.local.sysfind('hg')
 
-    if not os.path.isdir(os.path.join(pypyroot, '.hg')):
+    if os.path.isfile(os.path.join(pypyroot, '.hg_archival.txt')):
+        return _get_hg_archive_version(os.path.join(pypyroot, '.hg_archival.txt'))
+    elif not os.path.isdir(os.path.join(pypyroot, '.hg')):
         maywarn('Not running from a Mercurial repository!')
         return default_retval
     elif not hgexe:
         maywarn('Cannot find Mercurial command!')
         return default_retval
     else:
-        env = dict(os.environ)
-        # get Mercurial into scripting mode
-        env['HGPLAIN'] = '1'
-        # disable user configuration, extensions, etc.
-        env['HGRCPATH'] = os.devnull
+        return _get_hg_version(hgexe)
 
-        try:
-            p = Popen([str(hgexe), 'version', '-q'],
-                      stdout=PIPE, stderr=PIPE, env=env)
-        except OSError, e:
-            maywarn(e)
-            return default_retval
 
-        if not p.stdout.read().startswith('Mercurial Distributed SCM'):
-            maywarn('command does not identify itself as Mercurial')
-            return default_retval
+def _get_hg_version(hgexe):
+    env = dict(os.environ)
+    # get Mercurial into scripting mode
+    env['HGPLAIN'] = '1'
+    # disable user configuration, extensions, etc.
+    env['HGRCPATH'] = os.devnull
 
-        p = Popen([str(hgexe), 'id', '-i', pypyroot],
+    try:
+        p = Popen([str(hgexe), 'version', '-q'],
                   stdout=PIPE, stderr=PIPE, env=env)
-        hgid = p.stdout.read().strip()
+    except OSError, e:
+        maywarn(e)
+        return default_retval
+
+    if not p.stdout.read().startswith('Mercurial Distributed SCM'):
+        maywarn('command does not identify itself as Mercurial')
+        return default_retval
+
+    p = Popen([str(hgexe), 'id', '-i', pypyroot],
+              stdout=PIPE, stderr=PIPE, env=env)
+    hgid = p.stdout.read().strip()
+    maywarn(p.stderr.read())
+    if p.wait() != 0:
+        hgid = '?'
+
+    p = Popen([str(hgexe), 'id', '-t', pypyroot],
+              stdout=PIPE, stderr=PIPE, env=env)
+    hgtags = [t for t in p.stdout.read().strip().split() if t != 'tip']
+    maywarn(p.stderr.read())
+    if p.wait() != 0:
+        hgtags = ['?']
+
+    if hgtags:
+        return 'PyPy', hgtags[0], hgid
+    else:
+        # use the branch instead
+        p = Popen([str(hgexe), 'id', '-b', pypyroot],
+                  stdout=PIPE, stderr=PIPE, env=env)
+        hgbranch = p.stdout.read().strip()
         maywarn(p.stderr.read())
+
+        return 'PyPy', hgbranch, hgid
+
+
+def _get_hg_archive_version(path):
+    fp = open(path)
+    try:
+        data = dict(x.split(': ', 1) for x in fp.read().splitlines())
+    finally:
+        fp.close()
+    if 'tag' in data:
+        return 'PyPy', data['tag'], data['node']
+    else:
+        return 'PyPy', data['branch'], data['node']
+
+
+def _get_git_version():
+    #XXX: this function is a untested hack,
+    #     so the git mirror tav made will work
+    gitexe = py.path.local.sysfind('git')
+    if not gitexe:
+        return default_retval
+
+    try:
+        p = Popen(
+            [str(gitexe), 'rev-parse', 'HEAD'],
+            stdout=PIPE, stderr=PIPE, cwd=pypyroot
+            )
+    except OSError, e:
+        maywarn(e, 'Git')
+        return default_retval
+    if p.wait() != 0:
+        maywarn(p.stderr.read(), 'Git')
+        return default_retval
+    revision_id = p.stdout.read().strip()[:12]
+    p = Popen(
+        [str(gitexe), 'describe', '--tags', '--exact-match'],
+        stdout=PIPE, stderr=PIPE, cwd=pypyroot
+        )
+    if p.wait() != 0:
+        p = Popen(
+            [str(gitexe), 'branch'], stdout=PIPE, stderr=PIPE,
+            cwd=pypyroot
+            )
         if p.wait() != 0:
-            hgid = '?'
+            maywarn(p.stderr.read(), 'Git')
+            return 'PyPy', '?', revision_id
+        branch = '?'
+        for line in p.stdout.read().strip().split('\n'):
+            if line.startswith('* '):
+                branch = line[1:].strip()
+                if branch == '(no branch)':
+                    branch = '?'
+                break
+        return 'PyPy', branch, revision_id
+    return 'PyPy', p.stdout.read().strip(), revision_id
 
-        p = Popen([str(hgexe), 'id', '-t', pypyroot],
-                  stdout=PIPE, stderr=PIPE, env=env)
-        hgtags = [t for t in p.stdout.read().strip().split() if t != 'tip']
-        maywarn(p.stderr.read())
-        if p.wait() != 0:
-            hgtags = ['?']
 
-        if hgtags:
-            return 'PyPy', hgtags[0], hgid
-        else:
-            # use the branch instead
-            p = Popen([str(hgexe), 'id', '-b', pypyroot],
-                      stdout=PIPE, stderr=PIPE, env=env)
-            hgbranch = p.stdout.read().strip()
-            maywarn(p.stderr.read())
-
-            return 'PyPy', hgbranch, hgid
+if __name__ == '__main__':
+    print get_repo_version_info()
diff --git a/pypy/translator/jvm/builtin.py b/pypy/translator/jvm/builtin.py
--- a/pypy/translator/jvm/builtin.py
+++ b/pypy/translator/jvm/builtin.py
@@ -84,6 +84,9 @@
     (ootype.StringBuilder.__class__, "ll_build"):
     jvm.Method.v(jStringBuilder, "toString", (), jString),
 
+    (ootype.StringBuilder.__class__, "ll_getlength"):
+    jvm.Method.v(jStringBuilder, "length", (), jInt),
+
     (ootype.String.__class__, "ll_hash"):
     jvm.Method.v(jString, "hashCode", (), jInt),
 
diff --git a/pypy/translator/jvm/database.py b/pypy/translator/jvm/database.py
--- a/pypy/translator/jvm/database.py
+++ b/pypy/translator/jvm/database.py
@@ -358,7 +358,7 @@
         ootype.Unsigned:jvm.PYPYSERIALIZEUINT,
         ootype.SignedLongLong:jvm.LONGTOSTRINGL,
         ootype.UnsignedLongLong: jvm.PYPYSERIALIZEULONG,
-        ootype.Float:jvm.DOUBLETOSTRINGD,
+        ootype.Float:jvm.PYPYSERIALIZEDOUBLE,
         ootype.Bool:jvm.PYPYSERIALIZEBOOLEAN,
         ootype.Void:jvm.PYPYSERIALIZEVOID,
         ootype.Char:jvm.PYPYESCAPEDCHAR,
diff --git a/pypy/translator/jvm/metavm.py b/pypy/translator/jvm/metavm.py
--- a/pypy/translator/jvm/metavm.py
+++ b/pypy/translator/jvm/metavm.py
@@ -92,6 +92,7 @@
 CASTS = {
 #   FROM                      TO
     (ootype.Signed,           ootype.UnsignedLongLong): jvm.I2L,
+    (ootype.Unsigned,         ootype.UnsignedLongLong): jvm.I2L,
     (ootype.SignedLongLong,   ootype.Signed):           jvm.L2I,
     (ootype.UnsignedLongLong, ootype.Unsigned):         jvm.L2I,
     (ootype.UnsignedLongLong, ootype.Signed):           jvm.L2I,
diff --git a/pypy/translator/jvm/opcodes.py b/pypy/translator/jvm/opcodes.py
--- a/pypy/translator/jvm/opcodes.py
+++ b/pypy/translator/jvm/opcodes.py
@@ -101,6 +101,7 @@
     'jit_force_virtualizable':  Ignore,
     'jit_force_virtual':        DoNothing,
     'jit_force_quasi_immutable': Ignore,
+    'jit_is_virtual':           PushPrimitive(ootype.Bool, False),
 
     'debug_assert':              [], # TODO: implement?
     'debug_start_traceback':    Ignore,
diff --git a/pypy/translator/jvm/src/pypy/PyPy.java b/pypy/translator/jvm/src/pypy/PyPy.java
--- a/pypy/translator/jvm/src/pypy/PyPy.java
+++ b/pypy/translator/jvm/src/pypy/PyPy.java
@@ -283,6 +283,14 @@
         }
     }
 
+    public double pypy__longlong2float(long l) {
+        return Double.longBitsToDouble(l);
+    }
+
+    public long pypy__float2longlong(double d) {
+        return Double.doubleToRawLongBits(d);
+    }
+
     public double ooparse_float(String s) {
         try {
             return Double.parseDouble(s);
@@ -353,6 +361,19 @@
             return "False";
     }
 
+    public static String serialize_double(double d) {
+        if (Double.isNaN(d)) {
+            return "float(\"nan\")";
+        } else if (Double.isInfinite(d)) {
+            if (d > 0)
+                return "float(\"inf\")";
+            else
+                return "float(\"-inf\")";
+        } else {
+            return Double.toString(d);
+        }
+    }
+
     private static String format_char(char c) {
         String res = "\\x";
         if (c <= 0x0F) res = res + "0";
diff --git a/pypy/translator/jvm/test/runtest.py b/pypy/translator/jvm/test/runtest.py
--- a/pypy/translator/jvm/test/runtest.py
+++ b/pypy/translator/jvm/test/runtest.py
@@ -56,6 +56,7 @@
 
 # CLI could-be duplicate
 class JvmGeneratedSourceWrapper(object):
+
     def __init__(self, gensrc):
         """ gensrc is an instance of JvmGeneratedSource """
         self.gensrc = gensrc
diff --git a/pypy/translator/jvm/test/test_builder.py b/pypy/translator/jvm/test/test_builder.py
new file mode 100644
--- /dev/null
+++ b/pypy/translator/jvm/test/test_builder.py
@@ -0,0 +1,7 @@
+from pypy.translator.jvm.test.runtest import JvmTest
+from pypy.rpython.test.test_rbuilder import BaseTestStringBuilder
+import py
+
+class TestJvmStringBuilder(JvmTest, BaseTestStringBuilder):
+    def test_append_charpsize(self):
+        py.test.skip("append_charpsize(): not implemented on ootype")
diff --git a/pypy/translator/jvm/test/test_longlong2float.py b/pypy/translator/jvm/test/test_longlong2float.py
new file mode 100644
--- /dev/null
+++ b/pypy/translator/jvm/test/test_longlong2float.py
@@ -0,0 +1,20 @@
+from pypy.translator.jvm.test.runtest import JvmTest
+from pypy.rlib.longlong2float import *
+from pypy.rlib.test.test_longlong2float import enum_floats
+from pypy.rlib.test.test_longlong2float import fn as float2longlong2float
+import py
+
+class TestLongLong2Float(JvmTest):
+
+    def test_float2longlong_and_longlong2float(self):
+        def func(f):
+            return float2longlong2float(f)
+
+        for f in enum_floats():
+            assert repr(f) == repr(self.interpret(func, [f]))
+
+    def test_uint2singlefloat(self):
+        py.test.skip("uint2singlefloat is not implemented in ootype")
+
+    def test_singlefloat2uint(self):
+        py.test.skip("singlefloat2uint is not implemented in ootype")
diff --git a/pypy/translator/jvm/typesystem.py b/pypy/translator/jvm/typesystem.py
--- a/pypy/translator/jvm/typesystem.py
+++ b/pypy/translator/jvm/typesystem.py
@@ -955,6 +955,7 @@
 PYPYSERIALIZEUINT  =    Method.s(jPyPy, 'serialize_uint', (jInt,), jString)
 PYPYSERIALIZEULONG =    Method.s(jPyPy, 'serialize_ulonglong', (jLong,),jString)
 PYPYSERIALIZEVOID =     Method.s(jPyPy, 'serialize_void', (), jString)
+PYPYSERIALIZEDOUBLE =   Method.s(jPyPy, 'serialize_double', (jDouble,), jString)
 PYPYESCAPEDCHAR =       Method.s(jPyPy, 'escaped_char', (jChar,), jString)
 PYPYESCAPEDUNICHAR =    Method.s(jPyPy, 'escaped_unichar', (jChar,), jString)
 PYPYESCAPEDSTRING =     Method.s(jPyPy, 'escaped_string', (jString,), jString)
diff --git a/pypy/translator/oosupport/test_template/cast.py b/pypy/translator/oosupport/test_template/cast.py
--- a/pypy/translator/oosupport/test_template/cast.py
+++ b/pypy/translator/oosupport/test_template/cast.py
@@ -13,6 +13,9 @@
 def to_longlong(x):
     return r_longlong(x)
 
+def to_ulonglong(x):
+    return r_ulonglong(x)
+
 def uint_to_int(x):
     return intmask(x)
 
@@ -56,6 +59,9 @@
     def test_unsignedlonglong_to_unsigned4(self):
         self.check(to_uint, [r_ulonglong(18446744073709551615l)]) # max 64 bit num
 
+    def test_unsigned_to_usignedlonglong(self):
+        self.check(to_ulonglong, [r_uint(42)])
+
     def test_uint_to_int(self):
         self.check(uint_to_int, [r_uint(sys.maxint+1)])
 

From noreply at buildbot.pypy.org  Sat Jan 21 13:19:24 2012
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sat, 21 Jan 2012 13:19:24 +0100 (CET)
Subject: [pypy-commit] pypy py3k: Fix translation
Message-ID: <20120121121924.2C00A82CB2@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: py3k
Changeset: r51572:a718c93124eb
Date: 2012-01-21 12:47 +0100
http://bitbucket.org/pypy/pypy/changeset/a718c93124eb/

Log:	Fix translation

diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -9,7 +9,7 @@
 
 class Signature(object):
     _immutable_ = True
-    _immutable_fields_ = ["argnames[*]"]
+    _immutable_fields_ = ["argnames[*]", "kwonlyargnames[*]"]
     __slots__ = ("argnames", "kwonlyargnames", "varargname", "kwargname")
 
     def __init__(self, argnames, varargname=None, kwargname=None, kwonlyargnames=None):
@@ -51,7 +51,7 @@
         argnames = self.argnames
         if self.varargname is not None:
             argnames = argnames + [self.varargname]
-        argnames += self.kwonlyargnames
+        argnames = argnames + self.kwonlyargnames
         if self.kwargname is not None:
             argnames = argnames + [self.kwargname]
         return argnames

From noreply at buildbot.pypy.org  Sat Jan 21 13:39:53 2012
From: noreply at buildbot.pypy.org (bivab)
Date: Sat, 21 Jan 2012 13:39:53 +0100 (CET)
Subject: [pypy-commit] pypy arm-backend-2: off by one error when extracting
 call arguments in guard_call_xxx operations
Message-ID: <20120121123953.4662A82CB2@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider 
Branch: arm-backend-2
Changeset: r51573:ce782a82bacb
Date: 2012-01-21 12:13 +0100
http://bitbucket.org/pypy/pypy/changeset/ce782a82bacb/

Log:	off by one error when extracting call arguments in guard_call_xxx
	operations

diff --git a/pypy/jit/backend/arm/opassembler.py b/pypy/jit/backend/arm/opassembler.py
--- a/pypy/jit/backend/arm/opassembler.py
+++ b/pypy/jit/backend/arm/opassembler.py
@@ -1118,7 +1118,7 @@
         fail_index = self.cpu.get_fail_descr_number(faildescr)
         self._write_fail_index(fail_index)
         numargs = op.numargs()
-        callargs = arglocs[2:numargs]
+        callargs = arglocs[2:numargs + 1]  # extract the arguments to the call
         adr = arglocs[1]
         resloc = arglocs[0]
         self._emit_call(fail_index, adr, callargs, fcond, resloc)
@@ -1134,9 +1134,9 @@
         # first, close the stack in the sense of the asmgcc GC root tracker
         gcrootmap = self.cpu.gc_ll_descr.gcrootmap
         numargs = op.numargs()
+        callargs = arglocs[2:numargs + 1]  # extract the arguments to the call
+        adr = arglocs[1]
         resloc = arglocs[0]
-        adr = arglocs[1]
-        callargs = arglocs[2:numargs]
 
         if gcrootmap:
             self.call_release_gil(gcrootmap, arglocs, fcond)

From noreply at buildbot.pypy.org  Sat Jan 21 13:39:54 2012
From: noreply at buildbot.pypy.org (bivab)
Date: Sat, 21 Jan 2012 13:39:54 +0100 (CET)
Subject: [pypy-commit] pypy arm-backend-2: merge default
Message-ID: <20120121123954.9BBC882CB2@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider 
Branch: arm-backend-2
Changeset: r51574:f81818811f73
Date: 2012-01-21 12:32 +0100
http://bitbucket.org/pypy/pypy/changeset/f81818811f73/

Log:	merge default

diff --git a/pypy/doc/translation.rst b/pypy/doc/translation.rst
--- a/pypy/doc/translation.rst
+++ b/pypy/doc/translation.rst
@@ -155,7 +155,7 @@
                    function.  The two input variables are the exception class
                    and the exception value, respectively.  (No other block will
                    actually link to the exceptblock if the function does not
-                   explicitely raise exceptions.)
+                   explicitly raise exceptions.)
 
 
 ``Block``
@@ -325,7 +325,7 @@
 Mutable objects need special treatment during annotation, because
 the annotation of contained values needs to be possibly updated to account
 for mutation operations, and consequently the annotation information
-reflown through the relevant parts of the flow the graphs.
+reflown through the relevant parts of the flow graphs.
 
 * ``SomeList`` stands for a list of homogeneous type (i.e. all the
   elements of the list are represented by a single common ``SomeXxx``
@@ -503,8 +503,8 @@
 
 Since RPython is a garbage collected language there is a lot of heap memory
 allocation going on all the time, which would either not occur at all in a more
-traditional explicitely managed language or results in an object which dies at
-a time known in advance and can thus be explicitely deallocated. For example a
+traditional explicitly managed language or results in an object which dies at
+a time known in advance and can thus be explicitly deallocated. For example a
 loop of the following form::
 
     for i in range(n):
@@ -696,7 +696,7 @@
 
 So far it is the second most mature high level backend after GenCLI:
 it still can't translate the full Standard Interpreter, but after the
-Leysin sprint we were able to compile and run the rpytstone and
+Leysin sprint we were able to compile and run the rpystone and
 richards benchmarks.
 
 GenJVM is almost entirely the work of Niko Matsakis, who worked on it
diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -1118,6 +1118,12 @@
             for src, dst in singlefloats:
                 self.mc.MOVD(dst, src)
         # Finally remap the arguments in the main regs
+        # If x is a register and is in dst_locs, then oups, it needs to
+        # be moved away:
+        if x in dst_locs:
+            src_locs.append(x)
+            dst_locs.append(r10)
+            x = r10
         remap_frame_layout(self, src_locs, dst_locs, X86_64_SCRATCH_REG)
 
         self._regalloc.reserve_param(len(pass_on_stack))
@@ -2042,10 +2048,7 @@
         size = sizeloc.value
         signloc = arglocs[1]
 
-        if isinstance(op.getarg(0), Const):
-            x = imm(op.getarg(0).getint())
-        else:
-            x = arglocs[2]
+        x = arglocs[2]     # the function address
         if x is eax:
             tmp = ecx
         else:
diff --git a/pypy/jit/backend/x86/runner.py b/pypy/jit/backend/x86/runner.py
--- a/pypy/jit/backend/x86/runner.py
+++ b/pypy/jit/backend/x86/runner.py
@@ -6,7 +6,7 @@
 from pypy.jit.codewriter import longlong
 from pypy.jit.metainterp import history, compile
 from pypy.jit.backend.x86.assembler import Assembler386
-from pypy.jit.backend.x86.arch import FORCE_INDEX_OFS
+from pypy.jit.backend.x86.arch import FORCE_INDEX_OFS, IS_X86_32
 from pypy.jit.backend.x86.profagent import ProfileAgent
 from pypy.jit.backend.llsupport.llmodel import AbstractLLCPU
 from pypy.jit.backend.x86 import regloc
@@ -142,7 +142,9 @@
     cast_ptr_to_int._annspecialcase_ = 'specialize:arglltype(0)'
     cast_ptr_to_int = staticmethod(cast_ptr_to_int)
 
-    all_null_registers = lltype.malloc(rffi.LONGP.TO, 24,
+    all_null_registers = lltype.malloc(rffi.LONGP.TO,
+                                       IS_X86_32 and (16+8)  # 16 + 8 regs
+                                                 or (16+16), # 16 + 16 regs
                                        flavor='raw', zero=True,
                                        immortal=True)
 
diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -70,6 +70,7 @@
         ("exp", "exp"),
         ("fabs", "fabs"),
         ("floor", "floor"),
+        ("ceil", "ceil"),
         ("greater", "greater"),
         ("greater_equal", "greater_equal"),
         ("less", "less"),
@@ -85,6 +86,8 @@
         ("subtract", "subtract"),
         ('sqrt', 'sqrt'),
         ("tan", "tan"),
+        ('bitwise_and', 'bitwise_and'),
+        ('bitwise_or', 'bitwise_or'),
     ]:
         interpleveldefs[exposed] = "interp_ufuncs.get(space).%s" % impl
 
diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -47,6 +47,10 @@
     def getitem(self, storage, i):
         return self.itemtype.read(storage, self.itemtype.get_element_size(), i, 0)
 
+    def getitem_bool(self, storage, i):
+        isize = self.itemtype.get_element_size()
+        return self.itemtype.read_bool(storage, isize, i, 0)
+
     def setitem(self, storage, i, box):
         self.itemtype.store(storage, self.itemtype.get_element_size(), i, 0, box)
 
@@ -85,6 +89,12 @@
     def descr_get_shape(self, space):
         return space.newtuple([])
 
+    def is_int_type(self):
+        return self.kind == SIGNEDLTR or self.kind == UNSIGNEDLTR
+
+    def is_bool_type(self):
+        return self.kind == BOOLLTR
+
 W_Dtype.typedef = TypeDef("dtype",
     __module__ = "numpypy",
     __new__ = interp2app(W_Dtype.descr__new__.im_func),
diff --git a/pypy/module/micronumpy/interp_iter.py b/pypy/module/micronumpy/interp_iter.py
--- a/pypy/module/micronumpy/interp_iter.py
+++ b/pypy/module/micronumpy/interp_iter.py
@@ -4,6 +4,19 @@
 from pypy.module.micronumpy.strides import calculate_broadcast_strides,\
      calculate_slice_strides
 
+# structures to describe slicing
+
+class Chunk(object):
+    def __init__(self, start, stop, step, lgt):
+        self.start = start
+        self.stop = stop
+        self.step = step
+        self.lgt = lgt
+
+    def extend_shape(self, shape):
+        if self.step != 0:
+            shape.append(self.lgt)
+
 class BaseTransform(object):
     pass
 
@@ -38,11 +51,18 @@
         self.size = size
 
     def next(self, shapelen):
+        return self._next(1)
+
+    def _next(self, ofs):
         arr = instantiate(ArrayIterator)
         arr.size = self.size
-        arr.offset = self.offset + 1
+        arr.offset = self.offset + ofs
         return arr
 
+    def next_no_increase(self, shapelen):
+        # a hack to make JIT believe this is always virtual
+        return self._next(0)
+
     def done(self):
         return self.offset >= self.size
 
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -2,14 +2,15 @@
 from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.interpreter.gateway import interp2app, NoneNotWrapped
 from pypy.interpreter.typedef import TypeDef, GetSetProperty
-from pypy.module.micronumpy import interp_ufuncs, interp_dtype, signature
+from pypy.module.micronumpy import interp_ufuncs, interp_dtype, signature,\
+     interp_boxes
 from pypy.module.micronumpy.strides import calculate_slice_strides
 from pypy.rlib import jit
 from pypy.rpython.lltypesystem import lltype, rffi
 from pypy.tool.sourcetools import func_with_new_name
 from pypy.rlib.rstring import StringBuilder
 from pypy.module.micronumpy.interp_iter import ArrayIterator, OneDimIterator,\
-     SkipLastAxisIterator
+     SkipLastAxisIterator, Chunk, ViewIterator
 
 numpy_driver = jit.JitDriver(
     greens=['shapelen', 'sig'],
@@ -39,7 +40,24 @@
     get_printable_location=signature.new_printable_location('slice'),
     name='numpy_slice',
 )
-
+count_driver = jit.JitDriver(
+    greens=['shapelen'],
+    virtualizables=['frame'],
+    reds=['s', 'frame', 'iter', 'arr'],
+    name='numpy_count'
+)
+filter_driver = jit.JitDriver(
+    greens=['shapelen', 'sig'],
+    virtualizables=['frame'],
+    reds=['concr', 'argi', 'ri', 'frame', 'v', 'res', 'self'],
+    name='numpy_filter',
+)
+filter_set_driver = jit.JitDriver(
+    greens=['shapelen', 'sig'],
+    virtualizables=['frame'],
+    reds=['idx', 'idxi', 'frame', 'arr'],
+    name='numpy_filterset',
+)
 
 def _find_shape_and_elems(space, w_iterable):
     shape = [space.len_w(w_iterable)]
@@ -270,6 +288,9 @@
     descr_gt = _binop_impl("greater")
     descr_ge = _binop_impl("greater_equal")
 
+    descr_and = _binop_impl("bitwise_and")
+    descr_or = _binop_impl("bitwise_or")
+
     def _binop_right_impl(ufunc_name):
         def impl(self, space, w_other):
             w_other = scalar_w(space,
@@ -479,11 +500,69 @@
     def _prepare_slice_args(self, space, w_idx):
         if (space.isinstance_w(w_idx, space.w_int) or
             space.isinstance_w(w_idx, space.w_slice)):
-            return [space.decode_index4(w_idx, self.shape[0])]
-        return [space.decode_index4(w_item, self.shape[i]) for i, w_item in
+            return [Chunk(*space.decode_index4(w_idx, self.shape[0]))]
+        return [Chunk(*space.decode_index4(w_item, self.shape[i])) for i, w_item in
                 enumerate(space.fixedview(w_idx))]
 
+    def count_all_true(self, arr):
+        sig = arr.find_sig()
+        frame = sig.create_frame(self)
+        shapelen = len(arr.shape)
+        s = 0
+        iter = None
+        while not frame.done():
+            count_driver.jit_merge_point(arr=arr, frame=frame, iter=iter, s=s,
+                                         shapelen=shapelen)
+            iter = frame.get_final_iter()
+            s += arr.dtype.getitem_bool(arr.storage, iter.offset)
+            frame.next(shapelen)
+        return s
+
+    def getitem_filter(self, space, arr):
+        concr = arr.get_concrete()
+        size = self.count_all_true(concr)
+        res = W_NDimArray(size, [size], self.find_dtype())
+        ri = ArrayIterator(size)
+        shapelen = len(self.shape)
+        argi = concr.create_iter()
+        sig = self.find_sig()
+        frame = sig.create_frame(self)
+        v = None
+        while not frame.done():
+            filter_driver.jit_merge_point(concr=concr, argi=argi, ri=ri,
+                                          frame=frame, v=v, res=res, sig=sig,
+                                          shapelen=shapelen, self=self)
+            if concr.dtype.getitem_bool(concr.storage, argi.offset):
+                v = sig.eval(frame, self)
+                res.setitem(ri.offset, v)
+                ri = ri.next(1)
+            else:
+                ri = ri.next_no_increase(1)
+            argi = argi.next(shapelen)
+            frame.next(shapelen)
+        return res
+
+    def setitem_filter(self, space, idx, val):
+        size = self.count_all_true(idx)
+        arr = SliceArray([size], self.dtype, self, val)
+        sig = arr.find_sig()
+        shapelen = len(self.shape)
+        frame = sig.create_frame(arr)
+        idxi = idx.create_iter()
+        while not frame.done():
+            filter_set_driver.jit_merge_point(idx=idx, idxi=idxi, sig=sig,
+                                              frame=frame, arr=arr,
+                                              shapelen=shapelen)
+            if idx.dtype.getitem_bool(idx.storage, idxi.offset):
+                sig.eval(frame, arr)
+                frame.next_from_second(1)
+            frame.next_first(shapelen)
+            idxi = idxi.next(shapelen)
+
     def descr_getitem(self, space, w_idx):
+        if (isinstance(w_idx, BaseArray) and w_idx.shape == self.shape and
+            w_idx.find_dtype().is_bool_type()):
+            return self.getitem_filter(space, w_idx)
         if self._single_item_result(space, w_idx):
             concrete = self.get_concrete()
             item = concrete._index_of_single_item(space, w_idx)
@@ -493,6 +572,11 @@
 
     def descr_setitem(self, space, w_idx, w_value):
         self.invalidated()
+        if (isinstance(w_idx, BaseArray) and w_idx.shape == self.shape and
+            w_idx.find_dtype().is_bool_type()):
+            return self.get_concrete().setitem_filter(space,
+                                                      w_idx.get_concrete(),
+                                             convert_to_array(space, w_value))
         if self._single_item_result(space, w_idx):
             concrete = self.get_concrete()
             item = concrete._index_of_single_item(space, w_idx)
@@ -509,9 +593,8 @@
     def create_slice(self, chunks):
         shape = []
         i = -1
-        for i, (start_, stop, step, lgt) in enumerate(chunks):
-            if step != 0:
-                shape.append(lgt)
+        for i, chunk in enumerate(chunks):
+            chunk.extend_shape(shape)
         s = i + 1
         assert s >= 0
         shape += self.shape[s:]
@@ -724,8 +807,7 @@
                                          frame=frame,
                                          ri=ri,
                                          self=self, result=result)
-            result.dtype.setitem(result.storage, ri.offset,
-                                 sig.eval(frame, self))
+            result.setitem(ri.offset, sig.eval(frame, self))
             frame.next(shapelen)
             ri = ri.next(shapelen)
         return result
@@ -945,7 +1027,7 @@
                             builder.append('\n' + indent)
                         else:
                             builder.append(indent)
-                    view = self.create_slice([(i, 0, 0, 1)]).get_concrete()
+                    view = self.create_slice([Chunk(i, 0, 0, 1)]).get_concrete()
                     view.to_str(space, comma, builder, indent=indent + ' ',
                                                     use_ellipsis=use_ellipsis)
                 if i < self.shape[0] - 1:
@@ -962,7 +1044,7 @@
                         builder.append(indent)
                 # create_slice requires len(chunks) > 1 in order to reduce
                 # shape
-                view = self.create_slice([(i, 0, 0, 1)]).get_concrete()
+                view = self.create_slice([Chunk(i, 0, 0, 1)]).get_concrete()
                 view.to_str(space, comma, builder, indent=indent + ' ',
                                                     use_ellipsis=use_ellipsis)
                 i += 1
@@ -1091,6 +1173,10 @@
                                parent)
         self.start = start
 
+    def create_iter(self):
+        return ViewIterator(self.start, self.strides, self.backstrides,
+                            self.shape)
+
     def setshape(self, space, new_shape):
         if len(self.shape) < 1:
             return
@@ -1137,6 +1223,9 @@
         self.shape = new_shape
         self.calc_strides(new_shape)
 
+    def create_iter(self):
+        return ArrayIterator(self.size)
+
     def create_sig(self):
         return signature.ArraySignature(self.dtype)
 
@@ -1191,6 +1280,7 @@
     arr = W_NDimArray(size, shape[:], dtype=dtype, order=order)
     shapelen = len(shape)
     arr_iter = ArrayIterator(arr.size)
+    # XXX we might want to have a jitdriver here
     for i in range(len(elems_w)):
         w_elem = elems_w[i]
         dtype.setitem(arr.storage, arr_iter.offset,
@@ -1257,6 +1347,9 @@
     __gt__ = interp2app(BaseArray.descr_gt),
     __ge__ = interp2app(BaseArray.descr_ge),
 
+    __and__ = interp2app(BaseArray.descr_and),
+    __or__ = interp2app(BaseArray.descr_or),
+
     __repr__ = interp2app(BaseArray.descr_repr),
     __str__ = interp2app(BaseArray.descr_str),
     __array_interface__ = GetSetProperty(BaseArray.descr_array_iface),
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -249,15 +249,16 @@
 
 
 class W_Ufunc2(W_Ufunc):
-    _immutable_fields_ = ["comparison_func", "func", "name"]
+    _immutable_fields_ = ["comparison_func", "func", "name", "int_only"]
     argcount = 2
 
     def __init__(self, func, name, promote_to_float=False, promote_bools=False,
-        identity=None, comparison_func=False):
+        identity=None, comparison_func=False, int_only=False):
 
         W_Ufunc.__init__(self, name, promote_to_float, promote_bools, identity)
         self.func = func
         self.comparison_func = comparison_func
+        self.int_only = int_only
 
     def call(self, space, args_w):
         from pypy.module.micronumpy.interp_numarray import (Call2,
@@ -268,6 +269,7 @@
         w_rhs = convert_to_array(space, w_rhs)
         calc_dtype = find_binop_result_dtype(space,
             w_lhs.find_dtype(), w_rhs.find_dtype(),
+            int_only=self.int_only,
             promote_to_float=self.promote_to_float,
             promote_bools=self.promote_bools,
         )
@@ -304,10 +306,12 @@
 
 
 def find_binop_result_dtype(space, dt1, dt2, promote_to_float=False,
-    promote_bools=False):
+    promote_bools=False, int_only=False):
     # dt1.num should be <= dt2.num
     if dt1.num > dt2.num:
         dt1, dt2 = dt2, dt1
+    if int_only and (not dt1.is_int_type() or not dt2.is_int_type()):
+        raise OperationError(space.w_TypeError, space.wrap("Unsupported types"))
     # Some operations promote op(bool, bool) to return int8, rather than bool
     if promote_bools and (dt1.kind == dt2.kind == interp_dtype.BOOLLTR):
         return interp_dtype.get_dtype_cache(space).w_int8dtype
@@ -425,6 +429,10 @@
             ("add", "add", 2, {"identity": 0}),
             ("subtract", "sub", 2),
             ("multiply", "mul", 2, {"identity": 1}),
+            ("bitwise_and", "bitwise_and", 2, {"identity": 1,
+                                               'int_only': True}),
+            ("bitwise_or", "bitwise_or", 2, {"identity": 0,
+                                             'int_only': True}),
             ("divide", "div", 2, {"promote_bools": True}),
             ("mod", "mod", 2, {"promote_bools": True}),
             ("power", "pow", 2, {"promote_bools": True}),
@@ -449,6 +457,7 @@
 
             ("fabs", "fabs", 1, {"promote_to_float": True}),
             ("floor", "floor", 1, {"promote_to_float": True}),
+            ("ceil", "ceil", 1, {"promote_to_float": True}),
             ("exp", "exp", 1, {"promote_to_float": True}),
 
             ('sqrt', 'sqrt', 1, {'promote_to_float': True}),
@@ -475,7 +484,7 @@
         extra_kwargs["identity"] = identity
 
         func = ufunc_dtype_caller(space, ufunc_name, op_name, argcount,
-            comparison_func=extra_kwargs.get("comparison_func", False)
+            comparison_func=extra_kwargs.get("comparison_func", False),
         )
         if argcount == 1:
             ufunc = W_Ufunc1(func, ufunc_name, **extra_kwargs)
diff --git a/pypy/module/micronumpy/signature.py b/pypy/module/micronumpy/signature.py
--- a/pypy/module/micronumpy/signature.py
+++ b/pypy/module/micronumpy/signature.py
@@ -82,6 +82,16 @@
         for i in range(len(self.iterators)):
             self.iterators[i] = self.iterators[i].next(shapelen)
 
+    @unroll_safe
+    def next_from_second(self, shapelen):
+        """ Don't increase the first iterator
+        """
+        for i in range(1, len(self.iterators)):
+            self.iterators[i] = self.iterators[i].next(shapelen)
+
+    def next_first(self, shapelen):
+        self.iterators[0] = self.iterators[0].next(shapelen)
+
     def get_final_iter(self):
         final_iter = promote(self.final_iter)
         if final_iter < 0:
diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py
--- a/pypy/module/micronumpy/strides.py
+++ b/pypy/module/micronumpy/strides.py
@@ -10,12 +10,12 @@
     rstart = start
     rshape = []
     i = -1
-    for i, (start_, stop, step, lgt) in enumerate(chunks):
-        if step != 0:
-            rstrides.append(strides[i] * step)
-            rbackstrides.append(strides[i] * (lgt - 1) * step)
-            rshape.append(lgt)
-        rstart += strides[i] * start_
+    for i, chunk in enumerate(chunks):
+        if chunk.step != 0:
+            rstrides.append(strides[i] * chunk.step)
+            rbackstrides.append(strides[i] * (chunk.lgt - 1) * chunk.step)
+            rshape.append(chunk.lgt)
+        rstart += strides[i] * chunk.start
     # add a reminder
     s = i + 1
     assert s >= 0
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -2,6 +2,7 @@
 import py
 from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
 from pypy.module.micronumpy.interp_numarray import W_NDimArray, shape_agreement
+from pypy.module.micronumpy.interp_iter import Chunk
 from pypy.module.micronumpy import signature
 from pypy.interpreter.error import OperationError
 from pypy.conftest import gettestobjspace
@@ -37,53 +38,54 @@
 
     def test_create_slice_f(self):
         a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'F')
-        s = a.create_slice([(3, 0, 0, 1)])
+        s = a.create_slice([Chunk(3, 0, 0, 1)])
         assert s.start == 3
         assert s.strides == [10, 50]
         assert s.backstrides == [40, 100]
-        s = a.create_slice([(1, 9, 2, 4)])
+        s = a.create_slice([Chunk(1, 9, 2, 4)])
         assert s.start == 1
         assert s.strides == [2, 10, 50]
         assert s.backstrides == [6, 40, 100]
-        s = a.create_slice([(1, 5, 3, 2), (1, 2, 1, 1), (1, 0, 0, 1)])
+        s = a.create_slice([Chunk(1, 5, 3, 2), Chunk(1, 2, 1, 1), Chunk(1, 0, 0, 1)])
         assert s.shape == [2, 1]
         assert s.strides == [3, 10]
         assert s.backstrides == [3, 0]
-        s = a.create_slice([(0, 10, 1, 10), (2, 0, 0, 1)])
+        s = a.create_slice([Chunk(0, 10, 1, 10), Chunk(2, 0, 0, 1)])
         assert s.start == 20
         assert s.shape == [10, 3]
 
     def test_create_slice_c(self):
         a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'C')
-        s = a.create_slice([(3, 0, 0, 1)])
+        s = a.create_slice([Chunk(3, 0, 0, 1)])
         assert s.start == 45
         assert s.strides == [3, 1]
         assert s.backstrides == [12, 2]
-        s = a.create_slice([(1, 9, 2, 4)])
+        s = a.create_slice([Chunk(1, 9, 2, 4)])
         assert s.start == 15
         assert s.strides == [30, 3, 1]
         assert s.backstrides == [90, 12, 2]
-        s = a.create_slice([(1, 5, 3, 2), (1, 2, 1, 1), (1, 0, 0, 1)])
+        s = a.create_slice([Chunk(1, 5, 3, 2), Chunk(1, 2, 1, 1),
+                            Chunk(1, 0, 0, 1)])
         assert s.start == 19
         assert s.shape == [2, 1]
         assert s.strides == [45, 3]
         assert s.backstrides == [45, 0]
-        s = a.create_slice([(0, 10, 1, 10), (2, 0, 0, 1)])
+        s = a.create_slice([Chunk(0, 10, 1, 10), Chunk(2, 0, 0, 1)])
         assert s.start == 6
         assert s.shape == [10, 3]
 
     def test_slice_of_slice_f(self):
         a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'F')
-        s = a.create_slice([(5, 0, 0, 1)])
+        s = a.create_slice([Chunk(5, 0, 0, 1)])
         assert s.start == 5
-        s2 = s.create_slice([(3, 0, 0, 1)])
+        s2 = s.create_slice([Chunk(3, 0, 0, 1)])
         assert s2.shape == [3]
         assert s2.strides == [50]
         assert s2.parent is a
         assert s2.backstrides == [100]
         assert s2.start == 35
-        s = a.create_slice([(1, 5, 3, 2)])
-        s2 = s.create_slice([(0, 2, 1, 2), (2, 0, 0, 1)])
+        s = a.create_slice([Chunk(1, 5, 3, 2)])
+        s2 = s.create_slice([Chunk(0, 2, 1, 2), Chunk(2, 0, 0, 1)])
         assert s2.shape == [2, 3]
         assert s2.strides == [3, 50]
         assert s2.backstrides == [3, 100]
@@ -91,16 +93,16 @@
 
     def test_slice_of_slice_c(self):
         a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), order='C')
-        s = a.create_slice([(5, 0, 0, 1)])
+        s = a.create_slice([Chunk(5, 0, 0, 1)])
         assert s.start == 15 * 5
-        s2 = s.create_slice([(3, 0, 0, 1)])
+        s2 = s.create_slice([Chunk(3, 0, 0, 1)])
         assert s2.shape == [3]
         assert s2.strides == [1]
         assert s2.parent is a
         assert s2.backstrides == [2]
         assert s2.start == 5 * 15 + 3 * 3
-        s = a.create_slice([(1, 5, 3, 2)])
-        s2 = s.create_slice([(0, 2, 1, 2), (2, 0, 0, 1)])
+        s = a.create_slice([Chunk(1, 5, 3, 2)])
+        s2 = s.create_slice([Chunk(0, 2, 1, 2), Chunk(2, 0, 0, 1)])
         assert s2.shape == [2, 3]
         assert s2.strides == [45, 1]
         assert s2.backstrides == [45, 2]
@@ -108,14 +110,14 @@
 
     def test_negative_step_f(self):
         a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'F')
-        s = a.create_slice([(9, -1, -2, 5)])
+        s = a.create_slice([Chunk(9, -1, -2, 5)])
         assert s.start == 9
         assert s.strides == [-2, 10, 50]
         assert s.backstrides == [-8, 40, 100]
 
     def test_negative_step_c(self):
         a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), order='C')
-        s = a.create_slice([(9, -1, -2, 5)])
+        s = a.create_slice([Chunk(9, -1, -2, 5)])
         assert s.start == 135
         assert s.strides == [-30, 3, 1]
         assert s.backstrides == [-120, 12, 2]
@@ -124,7 +126,7 @@
         a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'F')
         r = a._index_of_single_item(self.space, self.newtuple(1, 2, 2))
         assert r == 1 + 2 * 10 + 2 * 50
-        s = a.create_slice([(0, 10, 1, 10), (2, 0, 0, 1)])
+        s = a.create_slice([Chunk(0, 10, 1, 10), Chunk(2, 0, 0, 1)])
         r = s._index_of_single_item(self.space, self.newtuple(1, 0))
         assert r == a._index_of_single_item(self.space, self.newtuple(1, 2, 0))
         r = s._index_of_single_item(self.space, self.newtuple(1, 1))
@@ -134,7 +136,7 @@
         a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'C')
         r = a._index_of_single_item(self.space, self.newtuple(1, 2, 2))
         assert r == 1 * 3 * 5 + 2 * 3 + 2
-        s = a.create_slice([(0, 10, 1, 10), (2, 0, 0, 1)])
+        s = a.create_slice([Chunk(0, 10, 1, 10), Chunk(2, 0, 0, 1)])
         r = s._index_of_single_item(self.space, self.newtuple(1, 0))
         assert r == a._index_of_single_item(self.space, self.newtuple(1, 2, 0))
         r = s._index_of_single_item(self.space, self.newtuple(1, 1))
@@ -1302,15 +1304,25 @@
         assert isinstance(i['data'][0], int)
         raises(TypeError, getattr, array(3), '__array_interface__')
 
+    def test_array_indexing_one_elem(self):
+        skip("not yet")
+        from _numpypy import array, arange
+        raises(IndexError, 'arange(3)[array([3.5])]')
+        a = arange(3)[array([1])]
+        assert a == 1
+        assert a[0] == 1
+        raises(IndexError,'arange(3)[array([15])]')
+        assert arange(3)[array([-3])] == 0
+        raises(IndexError,'arange(3)[array([-15])]')
+        assert arange(3)[array(1)] == 1
+
     def test_fill(self):
         from _numpypy import array
-
         a = array([1, 2, 3])
         a.fill(10)
         assert (a == [10, 10, 10]).all()
         a.fill(False)
         assert (a == [0, 0, 0]).all()
-
         b = a[:1]
         b.fill(4)
         assert (b == [4]).all()
@@ -1324,6 +1336,24 @@
         d.fill(100)
         assert d == 100
 
+    def test_array_indexing_bool(self):
+        from _numpypy import arange
+        a = arange(10)
+        assert (a[a > 3] == [4, 5, 6, 7, 8, 9]).all()
+        a = arange(10).reshape(5, 2)
+        assert (a[a > 3] == [4, 5, 6, 7, 8, 9]).all()
+        assert (a[a & 1 == 1] == [1, 3, 5, 7, 9]).all()
+
+    def test_array_indexing_bool_setitem(self):
+        from _numpypy import arange, array
+        a = arange(6)
+        a[a > 3] = 15
+        assert (a == [0, 1, 2, 3, 15, 15]).all()
+        a = arange(6).reshape(3, 2)
+        a[a & 1 == 1] = array([8, 9, 10])
+        assert (a == [[0, 8], [2, 9], [4, 10]]).all()
+
+
 
 class AppTestSupport(BaseNumpyAppTest):
     def setup_class(cls):
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -190,14 +190,24 @@
         for i in range(3):
             assert c[i] == a[i] - b[i]
 
-    def test_floor(self):
-        from _numpypy import array, floor
-
-        reference = [-2.0, -1.0, 0.0, 1.0, 1.0]
-        a = array([-1.4, -1.0, 0.0, 1.0, 1.4])
+    def test_floorceil(self):
+        from _numpypy import array, floor, ceil
+        import math
+        reference = [-2.0, -2.0, -1.0, 0.0, 1.0, 1.0, 0]
+        a = array([-1.4, -1.5, -1.0, 0.0, 1.0, 1.4, 0.5])
         b = floor(a)
         for i in range(5):
             assert b[i] == reference[i]
+        reference = [-1.0, -1.0, -1.0, 0.0, 1.0, 2.0, 1.0]
+        a = array([-1.4, -1.5, -1.0, 0.0, 1.0, 1.4, 0.5])
+        b = ceil(a)
+        assert (reference == b).all()
+        inf = float("inf")
+        data = [1.5, 2.9999, -1.999, inf]
+        results = [math.floor(x) for x in data]
+        assert (floor(data) == results).all()
+        results = [math.ceil(x) for x in data]
+        assert (ceil(data) == results).all()
 
     def test_copysign(self):
         from _numpypy import array, copysign
@@ -238,7 +248,7 @@
             assert b[i] == math.sin(a[i])
 
         a = sin(array([True, False], dtype=bool))
-        assert abs(a[0] - sin(1)) < 1e-7 # a[0] will be less precise
+        assert abs(a[0] - sin(1)) < 1e-7  # a[0] will be less precise
         assert a[1] == 0.0
 
     def test_cos(self):
@@ -259,7 +269,6 @@
         for i in range(len(a)):
             assert b[i] == math.tan(a[i])
 
-
     def test_arcsin(self):
         import math
         from _numpypy import array, arcsin
@@ -283,7 +292,6 @@
         for i in range(len(a)):
             assert b[i] == math.acos(a[i])
 
-
         a = array([-10, -1.5, -1.01, 1.01, 1.5, 10, float('nan'), float('inf'), float('-inf')])
         b = arccos(a)
         for f in b:
@@ -347,11 +355,20 @@
         raises(ValueError, maximum.reduce, [])
 
     def test_reduceND(self):
-        from numpypy import add, arange
+        from _numpypy import add, arange
         a = arange(12).reshape(3, 4)
         assert (add.reduce(a, 0) == [12, 15, 18, 21]).all()
         assert (add.reduce(a, 1) == [6.0, 22.0, 38.0]).all()
 
+    def test_bitwise(self):
+        from _numpypy import bitwise_and, bitwise_or, arange, array
+        a = arange(6).reshape(2, 3)
+        assert (a & 1 == [[0, 1, 0], [1, 0, 1]]).all()
+        assert (a & 1 == bitwise_and(a, 1)).all()
+        assert (a | 1 == [[1, 1, 3], [3, 5, 5]]).all()
+        assert (a | 1 == bitwise_or(a, 1)).all()
+        raises(TypeError, 'array([1.0]) & 1')
+
     def test_comparisons(self):
         import operator
         from _numpypy import equal, not_equal, less, less_equal, greater, greater_equal
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -217,6 +217,7 @@
         # This is the sum of the ops for both loops, however if you remove the
         # optimization then you end up with 2 float_adds, so we can still be
         # sure it was optimized correctly.
+        py.test.skip("too fragile")
         self.check_resops({'setinteriorfield_raw': 4, 'getfield_gc': 22,
                            'getarrayitem_gc': 4, 'getarrayitem_gc_pure': 2,
                            'getfield_gc_pure': 8,
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -94,6 +94,9 @@
             width, storage, i, offset
         ))
 
+    def read_bool(self, storage, width, i, offset):
+        raise NotImplementedError
+
     def store(self, storage, width, i, offset, box):
         value = self.unbox(box)
         libffi.array_setitem(clibffi.cast_type_to_ffitype(self.T),
@@ -168,6 +171,7 @@
     @simple_binary_op
     def min(self, v1, v2):
         return min(v1, v2)
+    
 
 class Bool(BaseType, Primitive):
     T = lltype.Bool
@@ -185,6 +189,11 @@
         else:
             return self.False
 
+
+    def read_bool(self, storage, width, i, offset):
+        return libffi.array_getitem(clibffi.cast_type_to_ffitype(self.T),
+                                    width, storage, i, offset)
+
     def coerce_subtype(self, space, w_subtype, w_item):
         # Doesn't return subclasses so it can return the constants.
         return self._coerce(space, w_item)
@@ -253,6 +262,14 @@
             assert v == 0
             return 0
 
+    @simple_binary_op
+    def bitwise_and(self, v1, v2):
+        return v1 & v2
+
+    @simple_binary_op
+    def bitwise_or(self, v1, v2):
+        return v1 | v2
+
 class Int8(BaseType, Integer):
     T = rffi.SIGNEDCHAR
     BoxType = interp_boxes.W_Int8Box
@@ -374,6 +391,10 @@
         return math.floor(v)
 
     @simple_unary_op
+    def ceil(self, v):
+        return math.ceil(v)
+
+    @simple_unary_op
     def exp(self, v):
         try:
             return math.exp(v)
@@ -436,4 +457,4 @@
 class Float64(BaseType, Float):
     T = rffi.DOUBLE
     BoxType = interp_boxes.W_Float64Box
-    format_code = "d"
\ No newline at end of file
+    format_code = "d"
diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -624,8 +624,8 @@
 
     if step == 1:
         assert start >= 0
-        assert slicelength >= 0
-        del items[start:start+slicelength]
+        if slicelength > 0:
+            del items[start:start+slicelength]
     else:
         n = len(items)
         i = start
@@ -662,10 +662,11 @@
             while i >= lim:
                 items[i] = items[i-delta]
                 i -= 1
-        elif start >= 0:
+        elif delta == 0:
+            pass
+        else:
+            assert start >= 0   # start<0 is only possible with slicelength==0
             del items[start:start+delta]
-        else:
-            assert delta==0   # start<0 is only possible with slicelength==0
     elif len2 != slicelength:  # No resize for extended slices
         raise operationerrfmt(space.w_ValueError, "attempt to "
               "assign sequence of size %d to extended slice of size %d",
diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -804,10 +804,11 @@
                 while i >= lim:
                     items[i] = items[i-delta]
                     i -= 1
-            elif start >= 0:
+            elif delta == 0:
+                pass
+            else:
+                assert start >= 0 # start<0 is only possible with slicelength==0
                 del items[start:start+delta]
-            else:
-                assert delta==0   # start<0 is only possible with slicelength==0
         elif len2 != slicelength:  # No resize for extended slices
             raise operationerrfmt(self.space.w_ValueError, "attempt to "
                   "assign sequence of size %d to extended slice of size %d",
@@ -851,8 +852,8 @@
 
         if step == 1:
             assert start >= 0
-            assert slicelength >= 0
-            del items[start:start+slicelength]
+            if slicelength > 0:
+                del items[start:start+slicelength]
         else:
             n = len(items)
             i = start
diff --git a/pypy/objspace/std/test/test_bytearrayobject.py b/pypy/objspace/std/test/test_bytearrayobject.py
--- a/pypy/objspace/std/test/test_bytearrayobject.py
+++ b/pypy/objspace/std/test/test_bytearrayobject.py
@@ -1,5 +1,9 @@
+from pypy import conftest
 
 class AppTestBytesArray:
+    def setup_class(cls):
+        cls.w_runappdirect = cls.space.wrap(conftest.option.runappdirect)
+
     def test_basics(self):
         b = bytearray()
         assert type(b) is bytearray
@@ -439,3 +443,15 @@
     def test_reduce(self):
         assert bytearray('caf\xe9').__reduce__() == (
             bytearray, (u'caf\xe9', 'latin-1'), None)
+
+    def test_setitem_slice_performance(self):
+        # because of a complexity bug, this used to take forever on a
+        # translated pypy.  On CPython2.6 -A, it takes around 8 seconds.
+        if self.runappdirect:
+            count = 16*1024*1024
+        else:
+            count = 1024
+        b = bytearray(count)
+        for i in range(count):
+            b[i:i+1] = 'y'
+        assert str(b) == 'y' * count
diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py
--- a/pypy/objspace/std/test/test_listobject.py
+++ b/pypy/objspace/std/test/test_listobject.py
@@ -397,6 +397,7 @@
         on_cpython = (option.runappdirect and
                             not hasattr(sys, 'pypy_translation_info'))
         cls.w_on_cpython = cls.space.wrap(on_cpython)
+        cls.w_runappdirect = cls.space.wrap(option.runappdirect)
 
     def test_getstrategyfromlist_w(self):
         l0 = ["a", "2", "a", True]
@@ -898,6 +899,18 @@
         l[::-1] = l
         assert l == [6,5,4,3,2,1]
 
+    def test_setitem_slice_performance(self):
+        # because of a complexity bug, this used to take forever on a
+        # translated pypy.  On CPython2.6 -A, it takes around 5 seconds.
+        if self.runappdirect:
+            count = 16*1024*1024
+        else:
+            count = 1024
+        b = [None] * count
+        for i in range(count):
+            b[i:i+1] = ['y']
+        assert b == ['y'] * count
+
     def test_recursive_repr(self):
         l = []
         assert repr(l) == '[]'
diff --git a/pypy/rpython/lltypesystem/module/ll_math.py b/pypy/rpython/lltypesystem/module/ll_math.py
--- a/pypy/rpython/lltypesystem/module/ll_math.py
+++ b/pypy/rpython/lltypesystem/module/ll_math.py
@@ -292,6 +292,10 @@
     # deal directly with IEEE specials, to cope with problems on various
     # platforms whose semantics don't exactly match C99
 
+    if y == 2.0:
+        return x * x     # this is always a correct answer, and is relatively
+                         # common in user programs.
+
     if isnan(y):
         if x == 1.0:
             return 1.0   # 1**Nan = 1
diff --git a/pypy/tool/test/test_version.py b/pypy/tool/test/test_version.py
--- a/pypy/tool/test/test_version.py
+++ b/pypy/tool/test/test_version.py
@@ -1,6 +1,22 @@
 import os, sys
 import py
-from pypy.tool.version import get_repo_version_info
+from pypy.tool.version import get_repo_version_info, _get_hg_archive_version
+
+def test_hg_archival_version(tmpdir):
+    def version_for(name, **kw):
+        path = tmpdir.join(name)
+        path.write('\n'.join('%s: %s' % x for x in kw.items()))
+        return _get_hg_archive_version(str(path))
+
+    assert version_for('release',
+                       tag='release-123',
+                       node='000',
+                      ) == ('PyPy', 'release-123', '000')
+    assert version_for('somebranch',
+                       node='000',
+                       branch='something',
+                      ) == ('PyPy', 'something', '000')
+
 
 def test_get_repo_version_info():
     assert get_repo_version_info(None)
diff --git a/pypy/tool/version.py b/pypy/tool/version.py
--- a/pypy/tool/version.py
+++ b/pypy/tool/version.py
@@ -3,111 +3,139 @@
 from subprocess import Popen, PIPE
 import pypy
 pypydir = os.path.dirname(os.path.abspath(pypy.__file__))
+pypyroot = os.path.dirname(pypydir)
+default_retval = 'PyPy', '?', '?'
+
+def maywarn(err, repo_type='Mercurial'):
+    if not err:
+        return
+
+    from pypy.tool.ansi_print import ansi_log
+    log = py.log.Producer("version")
+    py.log.setconsumer("version", ansi_log)
+    log.WARNING('Errors getting %s information: %s' % (repo_type, err))
 
 def get_repo_version_info(hgexe=None):
     '''Obtain version information by invoking the 'hg' or 'git' commands.'''
-    # TODO: support extracting from .hg_archival.txt
-
-    default_retval = 'PyPy', '?', '?'
-    pypyroot = os.path.abspath(os.path.join(pypydir, '..'))
-
-    def maywarn(err, repo_type='Mercurial'):
-        if not err:
-            return
-
-        from pypy.tool.ansi_print import ansi_log
-        log = py.log.Producer("version")
-        py.log.setconsumer("version", ansi_log)
-        log.WARNING('Errors getting %s information: %s' % (repo_type, err))
 
     # Try to see if we can get info from Git if hgexe is not specified.
     if not hgexe:
         if os.path.isdir(os.path.join(pypyroot, '.git')):
-            gitexe = py.path.local.sysfind('git')
-            if gitexe:
-                try:
-                    p = Popen(
-                        [str(gitexe), 'rev-parse', 'HEAD'],
-                        stdout=PIPE, stderr=PIPE, cwd=pypyroot
-                        )
-                except OSError, e:
-                    maywarn(e, 'Git')
-                    return default_retval
-                if p.wait() != 0:
-                    maywarn(p.stderr.read(), 'Git')
-                    return default_retval
-                revision_id = p.stdout.read().strip()[:12]
-                p = Popen(
-                    [str(gitexe), 'describe', '--tags', '--exact-match'],
-                    stdout=PIPE, stderr=PIPE, cwd=pypyroot
-                    )
-                if p.wait() != 0:
-                    p = Popen(
-                        [str(gitexe), 'branch'], stdout=PIPE, stderr=PIPE,
-                        cwd=pypyroot
-                        )
-                    if p.wait() != 0:
-                        maywarn(p.stderr.read(), 'Git')
-                        return 'PyPy', '?', revision_id
-                    branch = '?'
-                    for line in p.stdout.read().strip().split('\n'):
-                        if line.startswith('* '):
-                            branch = line[1:].strip()
-                            if branch == '(no branch)':
-                                branch = '?'
-                            break
-                    return 'PyPy', branch, revision_id
-                return 'PyPy', p.stdout.read().strip(), revision_id
+            return _get_git_version()
 
     # Fallback to trying Mercurial.
     if hgexe is None:
         hgexe = py.path.local.sysfind('hg')
 
-    if not os.path.isdir(os.path.join(pypyroot, '.hg')):
+    if os.path.isfile(os.path.join(pypyroot, '.hg_archival.txt')):
+        return _get_hg_archive_version(os.path.join(pypyroot, '.hg_archival.txt'))
+    elif not os.path.isdir(os.path.join(pypyroot, '.hg')):
         maywarn('Not running from a Mercurial repository!')
         return default_retval
     elif not hgexe:
         maywarn('Cannot find Mercurial command!')
         return default_retval
     else:
-        env = dict(os.environ)
-        # get Mercurial into scripting mode
-        env['HGPLAIN'] = '1'
-        # disable user configuration, extensions, etc.
-        env['HGRCPATH'] = os.devnull
+        return _get_hg_version(hgexe)
 
-        try:
-            p = Popen([str(hgexe), 'version', '-q'],
-                      stdout=PIPE, stderr=PIPE, env=env)
-        except OSError, e:
-            maywarn(e)
-            return default_retval
 
-        if not p.stdout.read().startswith('Mercurial Distributed SCM'):
-            maywarn('command does not identify itself as Mercurial')
-            return default_retval
+def _get_hg_version(hgexe):
+    env = dict(os.environ)
+    # get Mercurial into scripting mode
+    env['HGPLAIN'] = '1'
+    # disable user configuration, extensions, etc.
+    env['HGRCPATH'] = os.devnull
 
-        p = Popen([str(hgexe), 'id', '-i', pypyroot],
+    try:
+        p = Popen([str(hgexe), 'version', '-q'],
                   stdout=PIPE, stderr=PIPE, env=env)
-        hgid = p.stdout.read().strip()
+    except OSError, e:
+        maywarn(e)
+        return default_retval
+
+    if not p.stdout.read().startswith('Mercurial Distributed SCM'):
+        maywarn('command does not identify itself as Mercurial')
+        return default_retval
+
+    p = Popen([str(hgexe), 'id', '-i', pypyroot],
+              stdout=PIPE, stderr=PIPE, env=env)
+    hgid = p.stdout.read().strip()
+    maywarn(p.stderr.read())
+    if p.wait() != 0:
+        hgid = '?'
+
+    p = Popen([str(hgexe), 'id', '-t', pypyroot],
+              stdout=PIPE, stderr=PIPE, env=env)
+    hgtags = [t for t in p.stdout.read().strip().split() if t != 'tip']
+    maywarn(p.stderr.read())
+    if p.wait() != 0:
+        hgtags = ['?']
+
+    if hgtags:
+        return 'PyPy', hgtags[0], hgid
+    else:
+        # use the branch instead
+        p = Popen([str(hgexe), 'id', '-b', pypyroot],
+                  stdout=PIPE, stderr=PIPE, env=env)
+        hgbranch = p.stdout.read().strip()
         maywarn(p.stderr.read())
+
+        return 'PyPy', hgbranch, hgid
+
+
+def _get_hg_archive_version(path):
+    fp = open(path)
+    try:
+        data = dict(x.split(': ', 1) for x in fp.read().splitlines())
+    finally:
+        fp.close()
+    if 'tag' in data:
+        return 'PyPy', data['tag'], data['node']
+    else:
+        return 'PyPy', data['branch'], data['node']
+
+
+def _get_git_version():
+    #XXX: this function is a untested hack,
+    #     so the git mirror tav made will work
+    gitexe = py.path.local.sysfind('git')
+    if not gitexe:
+        return default_retval
+
+    try:
+        p = Popen(
+            [str(gitexe), 'rev-parse', 'HEAD'],
+            stdout=PIPE, stderr=PIPE, cwd=pypyroot
+            )
+    except OSError, e:
+        maywarn(e, 'Git')
+        return default_retval
+    if p.wait() != 0:
+        maywarn(p.stderr.read(), 'Git')
+        return default_retval
+    revision_id = p.stdout.read().strip()[:12]
+    p = Popen(
+        [str(gitexe), 'describe', '--tags', '--exact-match'],
+        stdout=PIPE, stderr=PIPE, cwd=pypyroot
+        )
+    if p.wait() != 0:
+        p = Popen(
+            [str(gitexe), 'branch'], stdout=PIPE, stderr=PIPE,
+            cwd=pypyroot
+            )
         if p.wait() != 0:
-            hgid = '?'
+            maywarn(p.stderr.read(), 'Git')
+            return 'PyPy', '?', revision_id
+        branch = '?'
+        for line in p.stdout.read().strip().split('\n'):
+            if line.startswith('* '):
+                branch = line[1:].strip()
+                if branch == '(no branch)':
+                    branch = '?'
+                break
+        return 'PyPy', branch, revision_id
+    return 'PyPy', p.stdout.read().strip(), revision_id
 
-        p = Popen([str(hgexe), 'id', '-t', pypyroot],
-                  stdout=PIPE, stderr=PIPE, env=env)
-        hgtags = [t for t in p.stdout.read().strip().split() if t != 'tip']
-        maywarn(p.stderr.read())
-        if p.wait() != 0:
-            hgtags = ['?']
 
-        if hgtags:
-            return 'PyPy', hgtags[0], hgid
-        else:
-            # use the branch instead
-            p = Popen([str(hgexe), 'id', '-b', pypyroot],
-                      stdout=PIPE, stderr=PIPE, env=env)
-            hgbranch = p.stdout.read().strip()
-            maywarn(p.stderr.read())
-
-            return 'PyPy', hgbranch, hgid
+if __name__ == '__main__':
+    print get_repo_version_info()

From noreply at buildbot.pypy.org  Sat Jan 21 14:12:06 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Sat, 21 Jan 2012 14:12:06 +0100 (CET)
Subject: [pypy-commit] pypy numpy-back-to-applevel: start working on a
 branch that moves back stuff from interp-level to app-level
Message-ID: <20120121131206.B683482D03@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: numpy-back-to-applevel
Changeset: r51575:78fdc8d84157
Date: 2012-01-21 15:10 +0200
http://bitbucket.org/pypy/pypy/changeset/78fdc8d84157/

Log:	start working on a branch that moves back stuff from interp-level to
	app-level

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -1,6 +1,6 @@
 from pypy.interpreter.baseobjspace import Wrappable
 from pypy.interpreter.error import OperationError, operationerrfmt
-from pypy.interpreter.gateway import interp2app, NoneNotWrapped
+from pypy.interpreter.gateway import interp2app, NoneNotWrapped, unwrap_spec
 from pypy.interpreter.typedef import TypeDef, GetSetProperty
 from pypy.module.micronumpy import interp_ufuncs, interp_dtype, signature,\
      interp_boxes
@@ -11,6 +11,7 @@
 from pypy.rlib.rstring import StringBuilder
 from pypy.module.micronumpy.interp_iter import ArrayIterator, OneDimIterator,\
      SkipLastAxisIterator, Chunk, ViewIterator
+from pypy.module.micronumpy.appbridge import get_appbridge_cache
 
 numpy_driver = jit.JitDriver(
     greens=['shapelen', 'sig'],
@@ -661,17 +662,13 @@
             w_denom = space.wrap(self.shape[dim])
         return space.div(self.descr_sum_promote(space, w_axis), w_denom)
 
-    def descr_var(self, space):
-        # var = mean((values - mean(values)) ** 2)
-        w_res = self.descr_sub(space, self.descr_mean(space, space.w_None))
-        assert isinstance(w_res, BaseArray)
-        w_res = w_res.descr_pow(space, space.wrap(2))
-        assert isinstance(w_res, BaseArray)
-        return w_res.descr_mean(space, space.w_None)
+    def descr_var(self, space, w_axis=None):
+        return get_appbridge_cache(space).call_method(space, '_var', self,
+                                                      w_axis)
 
-    def descr_std(self, space):
-        # std(v) = sqrt(var(v))
-        return interp_ufuncs.get(space).sqrt.call(space, [self.descr_var(space)])
+    def descr_std(self, space, w_axis=None):
+        return get_appbridge_cache(space).call_method(space, '_std', self,
+                                                      w_axis)
 
     def descr_fill(self, space, w_value):
         concr = self.get_concrete_or_scalar()
@@ -1245,8 +1242,13 @@
             shape.append(item)
     return size, shape
 
-def array(space, w_item_or_iterable, w_dtype=None, w_order=NoneNotWrapped):
+ at unwrap_spec(subok=bool, copy=bool, ownmaskna=bool)
+def array(space, w_item_or_iterable, w_dtype=None, w_order=None,
+          subok=True, copy=False, w_maskna=None, ownmaskna=False):
     # find scalar
+    if (not subok or copy or not space.is_w(w_maskna, space.w_None) or
+        ownmaskna):
+        raise OperationError(space.w_NotImplementedError, space.wrap("Unsupported args"))
     if not space.issequence_w(w_item_or_iterable):
         if space.is_w(w_dtype, space.w_None):
             w_dtype = interp_ufuncs.find_dtype_for_scalar(space,
@@ -1255,7 +1257,7 @@
             space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype)
         )
         return scalar_w(space, dtype, w_item_or_iterable)
-    if w_order is None:
+    if space.is_w(w_order, space.w_None):
         order = 'C'
     else:
         order = space.str_w(w_order)
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1029,8 +1029,8 @@
         assert a.var() == 0.0
         a = arange(10).reshape(5, 2)
         assert a.var() == 8.25
-        #assert (a.var(0) == [8, 8]).all()
-        #assert (a.var(1) == [.25] * 5).all()
+        assert (a.var(0) == [8, 8]).all()
+        assert (a.var(1) == [.25] * 5).all()
 
     def test_std(self):
         from _numpypy import array

From noreply at buildbot.pypy.org  Sat Jan 21 14:34:28 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Sat, 21 Jan 2012 14:34:28 +0100 (CET)
Subject: [pypy-commit] pypy numpy-back-to-applevel: add count_reduce_numbers
Message-ID: <20120121133428.990F9710653@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: numpy-back-to-applevel
Changeset: r51576:f53826f6f84a
Date: 2012-01-21 15:33 +0200
http://bitbucket.org/pypy/pypy/changeset/f53826f6f84a/

Log:	add count_reduce_numbers

diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -28,6 +28,8 @@
         'fromstring': 'interp_support.fromstring',
         'flatiter': 'interp_numarray.W_FlatIterator',
 
+        'count_reduce_items': 'interp_numarray.count_reduce_items',
+
         'True_': 'types.Bool.True',
         'False_': 'types.Bool.False',
 
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -1308,6 +1308,23 @@
     arr.dtype.fill(arr.storage, one, 0, size)
     return space.wrap(arr)
 
+ at unwrap_spec(arr=BaseArray, skipna=bool, keepdims=bool)
+def count_reduce_items(space, arr, w_axis=None, skipna=False, keepdims=True):
+    if not keepdims:
+        raise OperationError(space.w_NotImplementedError, space.wrap("unsupported"))
+    if space.is_w(w_axis, space.w_None):
+        s = 1
+        for elem in arr.shape:
+            s *= elem
+        return space.wrap(s)
+    if space.isinstance_w(w_axis, space.w_int):
+        return space.wrap(arr.shape[space.int_w(w_axis)])
+    s = 1
+    elems = space.fixedview(w_axis)
+    for w_elem in elems:
+        s *= arr.shape[space.int_w(w_elem)]
+    return space.wrap(s)
+
 def dot(space, w_obj, w_obj2):
     w_arr = convert_to_array(space, w_obj)
     if isinstance(w_arr, Scalar):
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -1,6 +1,6 @@
 from pypy.interpreter.baseobjspace import Wrappable
 from pypy.interpreter.error import OperationError, operationerrfmt
-from pypy.interpreter.gateway import interp2app
+from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.typedef import TypeDef, GetSetProperty, interp_attrproperty
 from pypy.module.micronumpy import interp_boxes, interp_dtype
 from pypy.module.micronumpy.signature import ReduceSignature,\
@@ -58,7 +58,9 @@
             )
         return self.call(space, __args__.arguments_w)
 
-    def descr_reduce(self, space, w_obj, w_dim=0):
+    @unwrap_spec(skipna=bool, keepdims=bool)
+    def descr_reduce(self, space, w_obj, w_axis=None, w_dtype=None,
+                     skipna=False, keepdims=True):
         """reduce(...)
         reduce(a, axis=0)
 
@@ -111,15 +113,18 @@
         array([[ 1,  5],
                [ 9, 13]])
         """
-        return self.reduce(space, w_obj, False, False, w_dim)
+        if space.is_w(w_axis, space.w_None):
+            axis = -1
+        else:
+            axis = space.int_w(w_axis)
+        return self.reduce(space, w_obj, False, False, axis)
 
-    def reduce(self, space, w_obj, multidim, promote_to_largest, w_dim):
+    def reduce(self, space, w_obj, multidim, promote_to_largest, dim):
         from pypy.module.micronumpy.interp_numarray import convert_to_array, \
                                                            Scalar
         if self.argcount != 2:
             raise OperationError(space.w_ValueError, space.wrap("reduce only "
                 "supported for binary functions"))
-        dim = space.int_w(w_dim)
         assert isinstance(self, W_Ufunc2)
         obj = convert_to_array(space, w_obj)
         if dim >= len(obj.shape):
@@ -494,3 +499,4 @@
 
 def get(space):
     return space.fromcache(UfuncState)
+
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -394,3 +394,11 @@
                 (3, 3.5),
             ]:
                 assert ufunc(a, b) == func(a, b)
+
+    def test_count_reduce_items(self):
+        from _numpypy import count_reduce_items, arange
+        a = arange(24).reshape(2, 3, 4)
+        assert count_reduce_items(a) == 24
+        assert count_reduce_items(a, 1) == 3
+        assert count_reduce_items(a, (1, 2)) == 3 * 4
+        

From noreply at buildbot.pypy.org  Sat Jan 21 14:42:10 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Sat, 21 Jan 2012 14:42:10 +0100 (CET)
Subject: [pypy-commit] pypy numpy-back-to-applevel: Add missing files,
 to progress, we need out attribute (Which is pointless
Message-ID: <20120121134210.2FE65710653@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: numpy-back-to-applevel
Changeset: r51577:1ff91b2e045c
Date: 2012-01-21 15:41 +0200
http://bitbucket.org/pypy/pypy/changeset/1ff91b2e045c/

Log:	Add missing files, to progress, we need out attribute (Which is
	pointless on pypy btw)

diff --git a/lib_pypy/numpypy/core/_methods.py b/lib_pypy/numpypy/core/_methods.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/numpypy/core/_methods.py
@@ -0,0 +1,98 @@
+# Array methods which are called by the both the C-code for the method
+# and the Python code for the NumPy-namespace function
+
+import _numpypy as mu
+um = mu
+#from numpypy.core import umath as um
+from numpypy.core.numeric import asanyarray
+
+def _amax(a, axis=None, out=None, skipna=False, keepdims=False):
+    return um.maximum.reduce(a, axis=axis,
+                            out=out, skipna=skipna, keepdims=keepdims)
+
+def _amin(a, axis=None, out=None, skipna=False, keepdims=False):
+    return um.minimum.reduce(a, axis=axis,
+                            out=out, skipna=skipna, keepdims=keepdims)
+
+def _sum(a, axis=None, dtype=None, out=None, skipna=False, keepdims=False):
+    return um.add.reduce(a, axis=axis, dtype=dtype,
+                            out=out, skipna=skipna, keepdims=keepdims)
+
+def _prod(a, axis=None, dtype=None, out=None, skipna=False, keepdims=False):
+    return um.multiply.reduce(a, axis=axis, dtype=dtype,
+                            out=out, skipna=skipna, keepdims=keepdims)
+
+def _mean(a, axis=None, dtype=None, out=None, skipna=False, keepdims=False):
+    arr = asanyarray(a)
+
+    # Upgrade bool, unsigned int, and int to float64
+    if dtype is None and arr.dtype.kind in ['b','u','i']:
+        ret = um.add.reduce(arr, axis=axis, dtype='f8',
+                            out=out, skipna=skipna, keepdims=keepdims)
+    else:
+        ret = um.add.reduce(arr, axis=axis, dtype=dtype,
+                            out=out, skipna=skipna, keepdims=keepdims)
+    rcount = mu.count_reduce_items(arr, axis=axis,
+                            skipna=skipna, keepdims=keepdims)
+    if isinstance(ret, mu.ndarray):
+        ret = um.true_divide(ret, rcount,
+                        out=ret, casting='unsafe', subok=False)
+    else:
+        ret = ret / float(rcount)
+    return ret
+
+def _var(a, axis=None, dtype=None, out=None, ddof=0,
+                            skipna=False, keepdims=False):
+    arr = asanyarray(a)
+
+    # First compute the mean, saving 'rcount' for reuse later
+    if dtype is None and arr.dtype.kind in ['b','u','i']:
+        arrmean = um.add.reduce(arr, axis=axis, dtype='f8',
+                            skipna=skipna, keepdims=True)
+    else:
+        arrmean = um.add.reduce(arr, axis=axis, dtype=dtype,
+                            skipna=skipna, keepdims=True)
+    rcount = mu.count_reduce_items(arr, axis=axis,
+                            skipna=skipna, keepdims=True)
+    if isinstance(arrmean, mu.ndarray):
+        arrmean = um.true_divide(arrmean, rcount,
+                            out=arrmean, casting='unsafe', subok=False)
+    else:
+        arrmean = arrmean / float(rcount)
+
+    # arr - arrmean
+    x = arr - arrmean
+
+    # (arr - arrmean) ** 2
+    if arr.dtype.kind == 'c':
+        x = um.multiply(x, um.conjugate(x), out=x).real
+    else:
+        x = um.multiply(x, x, out=x)
+
+    # add.reduce((arr - arrmean) ** 2, axis)
+    ret = um.add.reduce(x, axis=axis, dtype=dtype, out=out,
+                                skipna=skipna, keepdims=keepdims)
+
+    # add.reduce((arr - arrmean) ** 2, axis) / (n - ddof)
+    if not keepdims and isinstance(rcount, mu.ndarray):
+        rcount = rcount.squeeze(axis=axis)
+    rcount -= ddof
+    if isinstance(ret, mu.ndarray):
+        ret = um.true_divide(ret, rcount,
+                        out=ret, casting='unsafe', subok=False)
+    else:
+        ret = ret / float(rcount)
+
+    return ret
+
+def _std(a, axis=None, dtype=None, out=None, ddof=0,
+                            skipna=False, keepdims=False):
+    ret = _var(a, axis=axis, dtype=dtype, out=out, ddof=ddof,
+                                skipna=skipna, keepdims=keepdims)
+
+    if isinstance(ret, mu.ndarray):
+        ret = um.sqrt(ret, out=ret)
+    else:
+        ret = um.sqrt(ret)
+
+    return ret
diff --git a/lib_pypy/numpypy/core/numeric.py b/lib_pypy/numpypy/core/numeric.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/numpypy/core/numeric.py
@@ -0,0 +1,62 @@
+
+from _numpypy import array
+
+def asanyarray(a, dtype=None, order=None, maskna=None, ownmaskna=False):
+    """
+    Convert the input to an ndarray, but pass ndarray subclasses through.
+
+    Parameters
+    ----------
+    a : array_like
+        Input data, in any form that can be converted to an array.  This
+        includes scalars, lists, lists of tuples, tuples, tuples of tuples,
+        tuples of lists, and ndarrays.
+    dtype : data-type, optional
+        By default, the data-type is inferred from the input data.
+    order : {'C', 'F'}, optional
+        Whether to use row-major ('C') or column-major ('F') memory
+        representation.  Defaults to 'C'.
+   maskna : bool or None, optional
+        If this is set to True, it forces the array to have an NA mask.
+        If this is set to False, it forces the array to not have an NA
+        mask.
+    ownmaskna : bool, optional
+        If this is set to True, forces the array to have a mask which
+        it owns.
+
+    Returns
+    -------
+    out : ndarray or an ndarray subclass
+        Array interpretation of `a`.  If `a` is an ndarray or a subclass
+        of ndarray, it is returned as-is and no copy is performed.
+
+    See Also
+    --------
+    asarray : Similar function which always returns ndarrays.
+    ascontiguousarray : Convert input to a contiguous array.
+    asfarray : Convert input to a floating point ndarray.
+    asfortranarray : Convert input to an ndarray with column-major
+                     memory order.
+    asarray_chkfinite : Similar function which checks input for NaNs and
+                        Infs.
+    fromiter : Create an array from an iterator.
+    fromfunction : Construct an array by executing a function on grid
+                   positions.
+
+    Examples
+    --------
+    Convert a list into an array:
+
+    >>> a = [1, 2]
+    >>> np.asanyarray(a)
+    array([1, 2])
+
+    Instances of `ndarray` subclasses are passed through as-is:
+
+    >>> a = np.matrix([1, 2])
+    >>> np.asanyarray(a) is a
+    True
+
+    """
+    return array(a, dtype, copy=False, order=order, subok=True,
+                                maskna=maskna, ownmaskna=ownmaskna)
diff --git a/pypy/module/micronumpy/appbridge.py b/pypy/module/micronumpy/appbridge.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/micronumpy/appbridge.py
@@ -0,0 +1,28 @@
+
+from pypy.rlib.objectmodel import specialize
+
+class AppBridgeCache(object):
+    w__var = None
+    w__std = None
+    w_module = None
+
+    def __init__(self, space):
+        self.w_import = space.appexec([], """():
+        def f():
+           mod = __import__('numpypy.core._methods', {}, {}, [''])
+           return mod
+        return f
+        """)
+    
+    @specialize.arg(2)
+    def call_method(self, space, name, *args):
+        w_meth = getattr(self, 'w_' + name)
+        if w_meth is None:
+            if self.w_module is None:
+                self.w_module = space.call_function(self.w_import)
+            w_meth = space.getattr(self.w_module, space.wrap(name))
+            setattr(self, 'w_' + name, w_meth)
+        return space.call_function(w_meth, *args)
+
+def get_appbridge_cache(space):
+    return space.fromcache(AppBridgeCache)

From noreply at buildbot.pypy.org  Sat Jan 21 14:48:41 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Sat, 21 Jan 2012 14:48:41 +0100 (CET)
Subject: [pypy-commit] pypy numpy-back-to-applevel: rebinding is a bit
 harmful for now. Let's not do it
Message-ID: <20120121134841.397EC710653@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: numpy-back-to-applevel
Changeset: r51578:d6c604ffffa0
Date: 2012-01-21 15:48 +0200
http://bitbucket.org/pypy/pypy/changeset/d6c604ffffa0/

Log:	rebinding is a bit harmful for now. Let's not do it

diff --git a/lib_pypy/numpypy/core/_methods.py b/lib_pypy/numpypy/core/_methods.py
--- a/lib_pypy/numpypy/core/_methods.py
+++ b/lib_pypy/numpypy/core/_methods.py
@@ -55,23 +55,23 @@
     rcount = mu.count_reduce_items(arr, axis=axis,
                             skipna=skipna, keepdims=True)
     if isinstance(arrmean, mu.ndarray):
-        arrmean = um.true_divide(arrmean, rcount,
-                            out=arrmean, casting='unsafe', subok=False)
+        arrmean2 = um.true_divide(arrmean, rcount,
+                                  casting='unsafe', subok=False)
     else:
-        arrmean = arrmean / float(rcount)
+        arrmean2 = arrmean / float(rcount)
 
     # arr - arrmean
-    x = arr - arrmean
+    x = arr - arrmean2
 
     # (arr - arrmean) ** 2
     if arr.dtype.kind == 'c':
-        x = um.multiply(x, um.conjugate(x), out=x).real
+        y = um.multiply(x, um.conjugate(x)).real
     else:
-        x = um.multiply(x, x, out=x)
+        y = um.multiply(x, x)
 
     # add.reduce((arr - arrmean) ** 2, axis)
-    ret = um.add.reduce(x, axis=axis, dtype=dtype, out=out,
-                                skipna=skipna, keepdims=keepdims)
+    ret = um.add.reduce(y, axis=axis, dtype=dtype, out=out,
+                        skipna=skipna, keepdims=keepdims)
 
     # add.reduce((arr - arrmean) ** 2, axis) / (n - ddof)
     if not keepdims and isinstance(rcount, mu.ndarray):
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -60,7 +60,7 @@
 
     @unwrap_spec(skipna=bool, keepdims=bool)
     def descr_reduce(self, space, w_obj, w_axis=None, w_dtype=None,
-                     skipna=False, keepdims=True):
+                     skipna=False, keepdims=True, w_out=None):
         """reduce(...)
         reduce(a, axis=0)
 
@@ -113,6 +113,9 @@
         array([[ 1,  5],
                [ 9, 13]])
         """
+        if not space.is_w(w_out, space.w_None):
+            raise OperationError(space.w_NotImplementedError, space.wrap(
+                "out not supported"))
         if space.is_w(w_axis, space.w_None):
             axis = -1
         else:

From noreply at buildbot.pypy.org  Sat Jan 21 14:52:30 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Sat, 21 Jan 2012 14:52:30 +0100 (CET)
Subject: [pypy-commit] pypy numpy-back-to-applevel: change rebindings,
	nonsense
Message-ID: <20120121135230.EDE48710653@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: numpy-back-to-applevel
Changeset: r51579:ac6129336b42
Date: 2012-01-21 15:51 +0200
http://bitbucket.org/pypy/pypy/changeset/ac6129336b42/

Log:	change rebindings, nonsense

diff --git a/lib_pypy/numpypy/core/_methods.py b/lib_pypy/numpypy/core/_methods.py
--- a/lib_pypy/numpypy/core/_methods.py
+++ b/lib_pypy/numpypy/core/_methods.py
@@ -55,22 +55,22 @@
     rcount = mu.count_reduce_items(arr, axis=axis,
                             skipna=skipna, keepdims=True)
     if isinstance(arrmean, mu.ndarray):
-        arrmean2 = um.true_divide(arrmean, rcount,
+        arrmean = um.true_divide(arrmean, rcount,
                                   casting='unsafe', subok=False)
     else:
-        arrmean2 = arrmean / float(rcount)
+        arrmean = arrmean / float(rcount)
 
     # arr - arrmean
-    x = arr - arrmean2
+    x = arr - arrmean
 
     # (arr - arrmean) ** 2
     if arr.dtype.kind == 'c':
-        y = um.multiply(x, um.conjugate(x)).real
+        x = um.multiply(x, um.conjugate(x)).real
     else:
-        y = um.multiply(x, x)
+        x = um.multiply(x, x)
 
     # add.reduce((arr - arrmean) ** 2, axis)
-    ret = um.add.reduce(y, axis=axis, dtype=dtype, out=out,
+    ret = um.add.reduce(x, axis=axis, dtype=dtype, out=out,
                         skipna=skipna, keepdims=keepdims)
 
     # add.reduce((arr - arrmean) ** 2, axis) / (n - ddof)

From noreply at buildbot.pypy.org  Sat Jan 21 15:54:37 2012
From: noreply at buildbot.pypy.org (rguillebert)
Date: Sat, 21 Jan 2012 15:54:37 +0100 (CET)
Subject: [pypy-commit] pypy py3k: Start to port test_itertools' apptests to
	py3k
Message-ID: <20120121145437.1553A82D3C@wyvern.cs.uni-duesseldorf.de>

Author: Romain Guillebert 
Branch: py3k
Changeset: r51580:250aef313572
Date: 2012-01-21 15:51 +0100
http://bitbucket.org/pypy/pypy/changeset/250aef313572/

Log:	Start to port test_itertools' apptests to py3k

diff --git a/pypy/module/itertools/test/test_itertools.py b/pypy/module/itertools/test/test_itertools.py
--- a/pypy/module/itertools/test/test_itertools.py
+++ b/pypy/module/itertools/test/test_itertools.py
@@ -11,21 +11,21 @@
 
         it = itertools.count()
         for x in range(10):
-            assert it.next() == x
+            assert next(it) == x
 
     def test_count_firstval(self):
         import itertools
 
         it = itertools.count(3)
         for x in range(10):
-            assert it.next() == x + 3
+            assert next(it) == x + 3
 
     def test_count_repr(self):
         import itertools
 
         it = itertools.count(123)
         assert repr(it) == 'count(123)'
-        it.next()
+        next(it)
         assert repr(it) == 'count(124)'
         it = itertools.count(12.1, 1.0)
         assert repr(it) == 'count(12.1, 1.0)'
@@ -43,7 +43,7 @@
         it = itertools.repeat(o)
 
         for x in range(10):
-            assert o is it.next()
+            assert o is next(it)
 
     def test_repeat_times(self):
         import itertools
@@ -51,8 +51,8 @@
         times = 10
         it = itertools.repeat(None, times)
         for i in range(times):
-            it.next()
-        raises(StopIteration, it.next)
+            next(it)
+        raises(StopIteration, next, it)
 
         #---does not work in CPython 2.5
         #it = itertools.repeat(None, None)
@@ -60,12 +60,12 @@
         #    it.next()    # Should be no StopIteration
 
         it = itertools.repeat(None, 0)
-        raises(StopIteration, it.next)
-        raises(StopIteration, it.next)
+        raises(StopIteration, next, it)
+        raises(StopIteration, next, it)
 
         it = itertools.repeat(None, -1)
-        raises(StopIteration, it.next)
-        raises(StopIteration, it.next)
+        raises(StopIteration, next, it)
+        raises(StopIteration, next, it)
 
     def test_repeat_overflow(self):
         import itertools
@@ -78,12 +78,12 @@
 
         it = itertools.repeat('foobar')
         assert repr(it) == "repeat('foobar')"
-        it.next()
+        next(it)
         assert repr(it) == "repeat('foobar')"
 
         it = itertools.repeat('foobar', 10)
         assert repr(it) == "repeat('foobar', 10)"
-        it.next()
+        next(it)
         assert repr(it) == "repeat('foobar', 9)"
         list(it)
         assert repr(it) == "repeat('foobar', 0)"
@@ -92,22 +92,22 @@
         import itertools
 
         it = itertools.takewhile(bool, [])
-        raises(StopIteration, it.next)
+        raises(StopIteration, next, it)
 
         it = itertools.takewhile(bool, [False, True, True])
-        raises(StopIteration, it.next)
+        raises(StopIteration, next, it)
 
         it = itertools.takewhile(bool, [1, 2, 3, 0, 1, 1])
         for x in [1, 2, 3]:
-            assert it.next() == x
+            assert next(it) == x
 
-        raises(StopIteration, it.next)
+        raises(StopIteration, next, it)
 
     def test_takewhile_wrongargs(self):
         import itertools
 
         it = itertools.takewhile(None, [1])
-        raises(TypeError, it.next)
+        raises(TypeError, next, it)
 
         raises(TypeError, itertools.takewhile, bool, None)
 
@@ -115,25 +115,25 @@
         import itertools
 
         it = itertools.dropwhile(bool, [])
-        raises(StopIteration, it.next)
+        raises(StopIteration, next, it)
 
         it = itertools.dropwhile(bool, [True, True, True])
-        raises(StopIteration, it.next)
+        raises(StopIteration, next, it)
 
         def is_odd(arg):
             return (arg % 2 == 1)
 
         it = itertools.dropwhile(is_odd, [1, 3, 5, 2, 4, 6])
         for x in [2, 4, 6]:
-            assert it.next() == x
+            assert next(it) == x
 
-        raises(StopIteration, it.next)
+        raises(StopIteration, next, it)
 
     def test_dropwhile_wrongargs(self):
         import itertools
 
         it = itertools.dropwhile(None, [1])
-        raises(TypeError, it.next)
+        raises(TypeError, next, it)
 
         raises(TypeError, itertools.dropwhile, bool, None)
 
@@ -141,26 +141,26 @@
         import itertools
 
         it = itertools.ifilter(None, [])
-        raises(StopIteration, it.next)
+        raises(StopIteration, next, it)
 
         it = itertools.ifilter(None, [1, 0, 2, 3, 0])
         for x in [1, 2, 3]:
-            assert it.next() == x
-        raises(StopIteration, it.next)
+            assert next(it) == x
+        raises(StopIteration, next, it)
 
         def is_odd(arg):
             return (arg % 2 == 1)
 
         it = itertools.ifilter(is_odd, [1, 2, 3, 4, 5, 6])
         for x in [1, 3, 5]:
-            assert it.next() == x
-        raises(StopIteration, it.next)
+            assert next(it) == x
+        raises(StopIteration, next, it)
 
     def test_ifilter_wrongargs(self):
         import itertools
 
         it = itertools.ifilter(0, [1])
-        raises(TypeError, it.next)
+        raises(TypeError, next, it)
 
         raises(TypeError, itertools.ifilter, bool, None)
 
@@ -168,26 +168,26 @@
         import itertools
 
         it = itertools.ifilterfalse(None, [])
-        raises(StopIteration, it.next)
+        raises(StopIteration, next, it)
 
         it = itertools.ifilterfalse(None, [1, 0, 2, 3, 0])
         for x in [0, 0]:
-            assert it.next() == x
-        raises(StopIteration, it.next)
+            assert next(it) == x
+        raises(StopIteration, next, it)
 
         def is_odd(arg):
             return (arg % 2 == 1)
 
         it = itertools.ifilterfalse(is_odd, [1, 2, 3, 4, 5, 6])
         for x in [2, 4, 6]:
-            assert it.next() == x
-        raises(StopIteration, it.next)
+            assert next(it) == x
+        raises(StopIteration, next, it)
 
     def test_ifilterfalse_wrongargs(self):
         import itertools
 
         it = itertools.ifilterfalse(0, [1])
-        raises(TypeError, it.next)
+        raises(TypeError, next, it)
 
         raises(TypeError, itertools.ifilterfalse, bool, None)
 
@@ -195,83 +195,83 @@
         import itertools
 
         it = itertools.islice([], 0)
-        raises(StopIteration, it.next)
+        raises(StopIteration, next, it)
 
         it = itertools.islice([1, 2, 3], 0)
-        raises(StopIteration, it.next)
+        raises(StopIteration, next, it)
 
         it = itertools.islice([1, 2, 3, 4, 5], 3)
         for x in [1, 2, 3]:
-            assert it.next() == x
-        raises(StopIteration, it.next)
+            assert next(it) == x
+        raises(StopIteration, next, it)
 
         it = itertools.islice([1, 2, 3, 4, 5], 1, 3)
         for x in [2, 3]:
-            assert it.next() == x
-        raises(StopIteration, it.next)
+            assert next(it) == x
+        raises(StopIteration, next, it)
 
         it = itertools.islice([1, 2, 3, 4, 5], 0, 3, 2)
         for x in [1, 3]:
-            assert it.next() == x
-        raises(StopIteration, it.next)
+            assert next(it) == x
+        raises(StopIteration, next, it)
 
         it = itertools.islice([1, 2, 3], 0, None)
         for x in [1, 2, 3]:
-            assert it.next() == x
-        raises(StopIteration, it.next)
+            assert next(it) == x
+        raises(StopIteration, next, it)
 
-        assert list(itertools.islice(xrange(100), 10, 3)) == []
+        assert list(itertools.islice(range(100), 10, 3)) == []
 
         # new in 2.5: start=None or step=None
-        assert list(itertools.islice(xrange(10), None)) == range(10)
-        assert list(itertools.islice(xrange(10), None,None)) == range(10)
-        assert list(itertools.islice(xrange(10), None,None,None)) == range(10)
+        assert list(itertools.islice(range(10), None)) == range(10)
+        assert list(itertools.islice(range(10), None,None)) == range(10)
+        assert list(itertools.islice(range(10), None,None,None)) == range(10)
 
     def test_islice_dropitems_exact(self):
         import itertools
 
         it = iter("abcdefghij")
         itertools.islice(it, 2, 2)    # doesn't eagerly drop anything
-        assert it.next() == "a"
+        assert next(it) == "a"
         itertools.islice(it, 3, 8, 2)    # doesn't eagerly drop anything
-        assert it.next() == "b"
-        assert it.next() == "c"
+        assert next(it) == "b"
+        assert next(it) == "c"
 
         it = iter("abcdefghij")
         x = next(itertools.islice(it, 2, 3), None)    # drops 2 items
         assert x == "c"
-        assert it.next() == "d"
+        assert next(it) == "d"
 
         it = iter("abcdefghij")
         x = next(itertools.islice(it, 3, 8, 2), None)    # drops 3 items
         assert x == "d"
-        assert it.next() == "e"
+        assert next(it) == "e"
 
         it = iter("abcdefghij")
         x = next(itertools.islice(it, None, 8), None)    # drops 0 items
         assert x == "a"
-        assert it.next() == "b"
+        assert next(it) == "b"
 
         it = iter("abcdefghij")
         x = next(itertools.islice(it, 3, 2), None)    # drops 3 items
         assert x is None
-        assert it.next() == "d"
+        assert next(it) == "d"
 
         it = iter("abcdefghij")
         islc = itertools.islice(it, 3, 7, 2)
-        assert islc.next() == "d"    # drops 0, 1, 2, returns item #3
-        assert it.next() == "e"
-        assert islc.next() == "g"    # drops the 4th and return item #5
-        assert it.next() == "h"
-        raises(StopIteration, islc.next)  # drops the 6th and raise
-        assert it.next() == "j"
+        assert next(islc) == "d"    # drops 0, 1, 2, returns item #3
+        assert next(it) == "e"
+        assert next(islc) == "g"    # drops the 4th and return item #5
+        assert next(it) == "h"
+        raises(StopIteration, next, islc)  # drops the 6th and raise
+        assert next(it) == "j"
 
         it = iter("abcdefghij")
         islc = itertools.islice(it, 3, 4, 3)
-        assert islc.next() == "d"    # drops 0, 1, 2, returns item #3
-        assert it.next() == "e"
-        raises(StopIteration, islc.next)  # item #4 is 'stop', so just raise
-        assert it.next() == "f"
+        assert next(islc) == "d"    # drops 0, 1, 2, returns item #3
+        assert next(it) == "e"
+        raises(StopIteration, next, islc)  # item #4 is 'stop', so just raise
+        assert next(it) == "f"
 
     def test_islice_overflow(self):
         import itertools
@@ -298,22 +298,22 @@
         import itertools
         
         it = itertools.chain()
-        raises(StopIteration, it.next)
-        raises(StopIteration, it.next)
+        raises(StopIteration, next, it)
+        raises(StopIteration, next, it)
         
         it = itertools.chain([1, 2, 3])
         for x in [1, 2, 3]:
-            assert it.next() == x
-        raises(StopIteration, it.next)
+            assert next(it) == x
+        raises(StopIteration, next, it)
 
         it = itertools.chain([1, 2, 3], [4], [5, 6])
         for x in [1, 2, 3, 4, 5, 6]:
-            assert it.next() == x
-        raises(StopIteration, it.next)
+            assert next(it) == x
+        raises(StopIteration, next, it)
 
         it = itertools.chain([], [], [1], [])
-        assert it.next() == 1
-        raises(StopIteration, it.next)
+        assert next(it) == 1
+        raises(StopIteration, next, it)
 
     def test_imap(self):
         import itertools
@@ -321,35 +321,35 @@
         obj_list = [object(), object(), object()]
         it = itertools.imap(None, obj_list)
         for x in obj_list:
-            assert it.next() == (x, )
-        raises(StopIteration, it.next)
+            assert next(it) == (x, )
+        raises(StopIteration, next, it)
 
         it = itertools.imap(None, [1, 2, 3], [4], [5, 6])
-        assert it.next() == (1, 4, 5)
-        raises(StopIteration, it.next)
+        assert next(it) == (1, 4, 5)
+        raises(StopIteration, next, it)
 
         it = itertools.imap(None, [], [], [1], [])
-        raises(StopIteration, it.next)
+        raises(StopIteration, next, it)
 
         it = itertools.imap(str, [0, 1, 0, 1])
         for x in ['0', '1', '0', '1']:
-            assert it.next() == x
-        raises(StopIteration, it.next)
+            assert next(it) == x
+        raises(StopIteration, next, it)
 
         import operator
         it = itertools.imap(operator.add, [1, 2, 3], [4, 5, 6])
         for x in [5, 7, 9]:
-            assert it.next() == x
-        raises(StopIteration, it.next)
+            assert next(it) == x
+        raises(StopIteration, next, it)
 
     def test_imap_wrongargs(self):
         import itertools
         
         # Duplicate python 2.4 behaviour for invalid arguments
         it = itertools.imap(0, [])
-        raises(StopIteration, it.next)
+        raises(StopIteration, next, it)
         it = itertools.imap(0, [0])
-        raises(TypeError, it.next)
+        raises(TypeError, next, it)
         raises(TypeError, itertools.imap, None, 0)
 
         raises(TypeError, itertools.imap, None)
@@ -360,29 +360,29 @@
         import itertools
 
         it = itertools.izip()
-        raises(StopIteration, it.next)
+        raises(StopIteration, next, it)
 
         obj_list = [object(), object(), object()]
         it = itertools.izip(obj_list)
         for x in obj_list:
-            assert it.next() == (x, )
-        raises(StopIteration, it.next)
+            assert next(it) == (x, )
+        raises(StopIteration, next, it)
         
         it = itertools.izip([1, 2, 3], [4], [5, 6])
-        assert it.next() == (1, 4, 5)
-        raises(StopIteration, it.next)
+        assert next(it) == (1, 4, 5)
+        raises(StopIteration, next, it)
         
         it = itertools.izip([], [], [1], [])
-        raises(StopIteration, it.next)
+        raises(StopIteration, next, it)
 
         # Up to one additional item may be consumed per iterable, as per python docs
         it1 = iter([1, 2, 3, 4, 5, 6])
         it2 = iter([5, 6])
         it = itertools.izip(it1, it2)
         for x in [(1, 5), (2, 6)]:
-            assert it.next() == x
-        raises(StopIteration, it.next)
-        assert it1.next() in [3, 4]
+            assert next(it) == x
+        raises(StopIteration, next, it)
+        assert next(it1) in [3, 4]
         #---does not work in CPython 2.5
         #raises(StopIteration, it.next)
         #assert it1.next() in [4, 5]
@@ -398,7 +398,7 @@
             args = [()] * x + [None] + [()] * (9 - x)
             try:
                 itertools.izip(*args)
-            except TypeError, e:
+            except TypeError as e:
                 assert str(e).find("#" + str(x + 1) + " ") >= 0
             else:
                 fail("TypeError expected")
@@ -407,22 +407,22 @@
         import itertools
 
         it = itertools.cycle([])
-        raises(StopIteration, it.next)
+        raises(StopIteration, next, it)
         
         it = itertools.cycle([1, 2, 3])
         for x in [1, 2, 3, 1, 2, 3, 1, 2, 3]:
-            assert it.next() == x
+            assert next(it) == x
 
     def test_starmap(self):
         import itertools, operator
 
         it = itertools.starmap(operator.add, [])
-        raises(StopIteration, it.next)
+        raises(StopIteration, next, it)
 
         it = itertools.starmap(operator.add, [(0, 1), (2, 3), (4, 5)])
         for x in [1, 5, 9]:
-            assert it.next() == x
-        raises(StopIteration, it.next)
+            assert next(it) == x
+        raises(StopIteration, next, it)
 
         assert list(itertools.starmap(operator.add, [iter((40,2))])) == [42]
 
@@ -430,37 +430,37 @@
         import itertools
 
         it = itertools.starmap(None, [(1, )])
-        raises(TypeError, it.next)
+        raises(TypeError, next, it)
 
         it = itertools.starmap(None, [])
-        raises(StopIteration, it.next)
+        raises(StopIteration, next, it)
 
         it = itertools.starmap(bool, [0])
-        raises(TypeError, it.next)
+        raises(TypeError, next, it)
 
     def test_tee(self):
         import itertools
 
         it1, it2 = itertools.tee([])
-        raises(StopIteration, it1.next)
-        raises(StopIteration, it2.next)
+        raises(StopIteration, next, it1)
+        raises(StopIteration, next, it2)
 
         it1, it2 = itertools.tee([1, 2, 3])
         for x in [1, 2]:
-            assert it1.next() == x
+            assert next(it1) == x
         for x in [1, 2, 3]:
-            assert it2.next() == x
-        assert it1.next() == 3
-        raises(StopIteration, it1.next)
-        raises(StopIteration, it2.next)
+            assert next(it2) == x
+        assert next(it1) == 3
+        raises(StopIteration, next, it1)
+        raises(StopIteration, next, it2)
 
         assert itertools.tee([], 0) == ()
 
         iterators = itertools.tee([1, 2, 3], 10)
         for it in iterators:
             for x in [1, 2, 3]:
-                assert it.next() == x
-            raises(StopIteration, it.next)
+                assert next(it) == x
+            raises(StopIteration, next, it)
 
     def test_tee_wrongargs(self):
         import itertools
@@ -504,67 +504,67 @@
         import itertools
         
         it = itertools.groupby([])
-        raises(StopIteration, it.next)
+        raises(StopIteration, next, it)
 
         it = itertools.groupby([1, 2, 2, 3, 3, 3, 4, 4, 4, 4])
         for x in [1, 2, 3, 4]:
-            k, g = it.next()
+            k, g = next(it)
             assert k == x
             assert len(list(g)) == x
-            raises(StopIteration, g.next)
-        raises(StopIteration, it.next)
+            raises(StopIteration, next, g)
+        raises(StopIteration, next ,it)
 
         it = itertools.groupby([0, 1, 2, 3, 4, 5], None)
         for x in [0, 1, 2, 3, 4, 5]:
-            k, g = it.next()
+            k, g = next(it)
             assert k == x
-            assert g.next() == x
-            raises(StopIteration, g.next)
-        raises(StopIteration, it.next)
+            assert next(g) == x
+            raises(StopIteration, next, g)
+        raises(StopIteration, next, it)
 
         # consumes after group started
         it = itertools.groupby([0, 0, 0, 0, 1])
-        k1, g1 = it.next()
-        assert g1.next() == 0
-        k2, g2 = it.next()
-        raises(StopIteration, g1.next)
-        assert g2.next() == 1
-        raises(StopIteration, g2.next)
+        k1, g1 = next(it)
+        assert next(g1) == 0
+        k2, g2 = next(it)
+        raises(StopIteration, next, g1)
+        assert next(g2) == 1
+        raises(StopIteration, next, g2)
 
         # skips with not started group
         it = itertools.groupby([0, 0, 1])
-        k1, g1 = it.next()
-        k2, g2 = it.next()
-        raises(StopIteration, g1.next)
-        assert g2.next() == 1
-        raises(StopIteration, g2.next)
+        k1, g1 = next(it)
+        k2, g2 = next(it)
+        raises(StopIteration, next, g1)
+        assert next(g2) == 1
+        raises(StopIteration, next, g2)
 
         it = itertools.groupby([0, 1, 2])
-        k1, g1 = it.next()
-        k2, g2 = it.next()
-        k2, g3 = it.next()
-        raises(StopIteration, g1.next)
-        raises(StopIteration, g2.next)
-        assert g3.next() == 2
+        k1, g1 = next(it)
+        k2, g2 = next(it)
+        k2, g3 = next(it)
+        raises(StopIteration, next, g1)
+        raises(StopIteration, next, g2)
+        assert next(g3) == 2
 
         def half_floor(x):
             return x // 2
         it = itertools.groupby([0, 1, 2, 3, 4, 5], half_floor)
         for x in [0, 1, 2]:
-            k, g = it.next()
+            k, g = next(it)
             assert k == x
-            assert half_floor(g.next()) == x
-            assert half_floor(g.next()) == x
-            raises(StopIteration, g.next)
-        raises(StopIteration, it.next)
+            assert half_floor(next(g)) == x
+            assert half_floor(next(g)) == x
+            raises(StopIteration, next, g)
+        raises(StopIteration, next, it)
 
         # keyword argument
         it = itertools.groupby([0, 1, 2, 3, 4, 5], key = half_floor)
         for x in [0, 1, 2]:
-            k, g = it.next()
+            k, g = next(it)
             assert k == x
             assert list(g) == [x*2, x*2+1]
-        raises(StopIteration, it.next)
+        raises(StopIteration, next, it)
 
         # Grouping is not based on key identity
         class NeverEqual(object):
@@ -573,12 +573,12 @@
         objects = [NeverEqual(), NeverEqual(), NeverEqual()]
         it = itertools.groupby(objects)
         for x in objects:
-            print "Trying", x
-            k, g = it.next()
+            print("Trying", x)
+            k, g = next(it)
             assert k is x
-            assert g.next() is x
-            raises(StopIteration, g.next)
-        raises(StopIteration, it.next)
+            assert next(g) is x
+            raises(StopIteration, next, g)
+        raises(StopIteration, next, it)
         
         # Grouping is based on key equality
         class AlwaysEqual(object):
@@ -586,19 +586,19 @@
                 return True
         objects = [AlwaysEqual(), AlwaysEqual(), AlwaysEqual()]
         it = itertools.groupby(objects)
-        k, g = it.next()
+        k, g = next(it)
         assert k is objects[0]
         for x in objects:
-            assert g.next() is x
-        raises(StopIteration, g.next)
-        raises(StopIteration, it.next)
+            assert next(g) is x
+        raises(StopIteration, next, g)
+        raises(StopIteration, next, it)
 
     def test_groupby_wrongargs(self):
         import itertools
 
         raises(TypeError, itertools.groupby, 0)
         it = itertools.groupby([0], 1)
-        raises(TypeError, it.next)
+        raises(TypeError, next, it)
 
     def test_iterables(self):
         import itertools
@@ -624,8 +624,8 @@
         for it in iterables:
             assert hasattr(it, '__iter__')
             assert iter(it) is it
-            assert hasattr(it, 'next')
-            assert callable(it.next)
+            assert hasattr(it, '__next__')
+            assert callable(it.__next__)
 
     def test_docstrings(self):
         import itertools
@@ -669,25 +669,25 @@
     def test_count_overflow(self):
         import itertools, sys
         it = itertools.count(sys.maxint - 1)
-        assert it.next() == sys.maxint - 1
-        assert it.next() == sys.maxint
-        assert it.next() == sys.maxint + 1
+        assert next(it) == sys.maxint - 1
+        assert next(it) == sys.maxint
+        assert next(it) == sys.maxint + 1
         it = itertools.count(sys.maxint + 1)
-        assert it.next() == sys.maxint + 1
-        assert it.next() == sys.maxint + 2
+        assert next(it) == sys.maxint + 1
+        assert next(it) == sys.maxint + 2
         it = itertools.count(-sys.maxint-2)
-        assert it.next() == -sys.maxint - 2
-        assert it.next() == -sys.maxint - 1
-        assert it.next() == -sys.maxint
-        assert it.next() == -sys.maxint + 1
+        assert next(it) == -sys.maxint - 2
+        assert next(it) == -sys.maxint - 1
+        assert next(it) == -sys.maxint
+        assert next(it) == -sys.maxint + 1
         it = itertools.count(0, sys.maxint)
-        assert it.next() == sys.maxint * 0
-        assert it.next() == sys.maxint * 1
-        assert it.next() == sys.maxint * 2
+        assert next(it) == sys.maxint * 0
+        assert next(it) == sys.maxint * 1
+        assert next(it) == sys.maxint * 2
         it = itertools.count(0, sys.maxint + 1)
-        assert it.next() == (sys.maxint + 1) * 0
-        assert it.next() == (sys.maxint + 1) * 1
-        assert it.next() == (sys.maxint + 1) * 2
+        assert next(it) == (sys.maxint + 1) * 0
+        assert next(it) == (sys.maxint + 1) * 1
+        assert next(it) == (sys.maxint + 1) * 2
 
     def test_chain_fromiterable(self):
         import itertools
@@ -832,8 +832,8 @@
     def test_product_empty(self):
         from itertools import product
         prod = product('abc', repeat=0)
-        assert prod.next() == ()
-        raises (StopIteration, prod.next)
+        assert next(prod) == ()
+        raises (StopIteration, next, prod)
 
     def test_permutations(self):
         from itertools import permutations
@@ -882,7 +882,7 @@
     def test_permutations_r_gt_n(self):
         from itertools import permutations
         perm = permutations([1, 2], 3)
-        raises(StopIteration, perm.next)
+        raises(StopIteration, next, perm)
 
     def test_permutations_neg_r(self):
         from itertools import permutations
@@ -905,14 +905,14 @@
     def test_compress_diff_len(self):
         import itertools
         it = itertools.compress(['a'], [])
-        raises(StopIteration, it.next)
+        raises(StopIteration, next, it)
 
     def test_count_kwargs(self):
         import itertools
         it = itertools.count(start=2, step=3)
-        assert it.next() == 2
-        assert it.next() == 5
-        assert it.next() == 8
+        assert next(it) == 2
+        assert next(it) == 5
+        assert next(it) == 8
 
     def test_repeat_kwargs(self):
         import itertools
@@ -961,10 +961,10 @@
         r1 = Repeater(1, 3, RuntimeError)
         r2 = Repeater(2, 4, StopIteration)
         it = itertools.izip_longest(r1, r2, fillvalue=0)
-        assert it.next() == (1, 2)
-        assert it.next() == (1, 2)
-        assert it.next()== (1, 2)
-        raises(RuntimeError, it.next)
+        assert next(it) == (1, 2)
+        assert next(it) == (1, 2)
+        assert next(it)== (1, 2)
+        raises(RuntimeError, next, it)
 
     def test_subclassing(self):
         import itertools

From noreply at buildbot.pypy.org  Sat Jan 21 15:54:38 2012
From: noreply at buildbot.pypy.org (rguillebert)
Date: Sat, 21 Jan 2012 15:54:38 +0100 (CET)
Subject: [pypy-commit] pypy py3k: Add a test for keyword-only arguments
Message-ID: <20120121145438.54A1482D3D@wyvern.cs.uni-duesseldorf.de>

Author: Romain Guillebert 
Branch: py3k
Changeset: r51581:39eb4a258003
Date: 2012-01-21 15:53 +0100
http://bitbucket.org/pypy/pypy/changeset/39eb4a258003/

Log:	Add a test for keyword-only arguments

diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -810,6 +810,14 @@
         """
         yield self.st, func, "f()", (1, [2, 3, 4], 5)
 
+    def test_kwonlyargs_default_parameters(self):
+        """ This actually test an interpreter bug, but since we can't parse
+        py3k only code in the interpreter tests right now, it's there waiting
+        for this feature"""
+        func = """ def f(a, b, c=3, *, d=4):
+            return a, b, c, d
+        """
+        yield self.st, func, "f(1, 2)", (1, 2, 3, 4)
 
 class AppTestCompiler:
 

From noreply at buildbot.pypy.org  Sat Jan 21 17:03:23 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Sat, 21 Jan 2012 17:03:23 +0100 (CET)
Subject: [pypy-commit] pypy numpy-back-to-applevel: shuffle stuff around
Message-ID: <20120121160323.F17E482D45@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: numpy-back-to-applevel
Changeset: r51582:b31c0d563d03
Date: 2012-01-21 18:02 +0200
http://bitbucket.org/pypy/pypy/changeset/b31c0d563d03/

Log:	shuffle stuff around

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -1,10 +1,11 @@
 from pypy.interpreter.baseobjspace import Wrappable
 from pypy.interpreter.error import OperationError, operationerrfmt
-from pypy.interpreter.gateway import interp2app, NoneNotWrapped, unwrap_spec
+from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.typedef import TypeDef, GetSetProperty
-from pypy.module.micronumpy import interp_ufuncs, interp_dtype, signature,\
-     interp_boxes
-from pypy.module.micronumpy.strides import calculate_slice_strides
+from pypy.module.micronumpy import interp_ufuncs, interp_dtype, signature
+from pypy.module.micronumpy.strides import calculate_slice_strides,\
+     shape_agreement, find_shape_and_elems, get_shape_from_iterable,\
+     calc_new_strides
 from pypy.rlib import jit
 from pypy.rpython.lltypesystem import lltype, rffi
 from pypy.tool.sourcetools import func_with_new_name
@@ -60,176 +61,6 @@
     name='numpy_filterset',
 )
 
-def _find_shape_and_elems(space, w_iterable):
-    shape = [space.len_w(w_iterable)]
-    batch = space.listview(w_iterable)
-    while True:
-        new_batch = []
-        if not batch:
-            return shape, []
-        if not space.issequence_w(batch[0]):
-            for elem in batch:
-                if space.issequence_w(elem):
-                    raise OperationError(space.w_ValueError, space.wrap(
-                        "setting an array element with a sequence"))
-            return shape, batch
-        size = space.len_w(batch[0])
-        for w_elem in batch:
-            if not space.issequence_w(w_elem) or space.len_w(w_elem) != size:
-                raise OperationError(space.w_ValueError, space.wrap(
-                    "setting an array element with a sequence"))
-            new_batch += space.listview(w_elem)
-        shape.append(size)
-        batch = new_batch
-
-def shape_agreement(space, shape1, shape2):
-    ret = _shape_agreement(shape1, shape2)
-    if len(ret) < max(len(shape1), len(shape2)):
-        raise OperationError(space.w_ValueError,
-            space.wrap("operands could not be broadcast together with shapes (%s) (%s)" % (
-                ",".join([str(x) for x in shape1]),
-                ",".join([str(x) for x in shape2]),
-            ))
-        )
-    return ret
-
-def _shape_agreement(shape1, shape2):
-    """ Checks agreement about two shapes with respect to broadcasting. Returns
-    the resulting shape.
-    """
-    lshift = 0
-    rshift = 0
-    if len(shape1) > len(shape2):
-        m = len(shape1)
-        n = len(shape2)
-        rshift = len(shape2) - len(shape1)
-        remainder = shape1
-    else:
-        m = len(shape2)
-        n = len(shape1)
-        lshift = len(shape1) - len(shape2)
-        remainder = shape2
-    endshape = [0] * m
-    indices1 = [True] * m
-    indices2 = [True] * m
-    for i in range(m - 1, m - n - 1, -1):
-        left = shape1[i + lshift]
-        right = shape2[i + rshift]
-        if left == right:
-            endshape[i] = left
-        elif left == 1:
-            endshape[i] = right
-            indices1[i + lshift] = False
-        elif right == 1:
-            endshape[i] = left
-            indices2[i + rshift] = False
-        else:
-            return []
-            #raise OperationError(space.w_ValueError, space.wrap(
-            #    "frames are not aligned"))
-    for i in range(m - n):
-        endshape[i] = remainder[i]
-    return endshape
-
-def get_shape_from_iterable(space, old_size, w_iterable):
-    new_size = 0
-    new_shape = []
-    if space.isinstance_w(w_iterable, space.w_int):
-        new_size = space.int_w(w_iterable)
-        if new_size < 0:
-            new_size = old_size
-        new_shape = [new_size]
-    else:
-        neg_dim = -1
-        batch = space.listview(w_iterable)
-        new_size = 1
-        if len(batch) < 1:
-            if old_size == 1:
-                # Scalars can have an empty size.
-                new_size = 1
-            else:
-                new_size = 0
-        new_shape = []
-        i = 0
-        for elem in batch:
-            s = space.int_w(elem)
-            if s < 0:
-                if neg_dim >= 0:
-                    raise OperationError(space.w_ValueError, space.wrap(
-                             "can only specify one unknown dimension"))
-                s = 1
-                neg_dim = i
-            new_size *= s
-            new_shape.append(s)
-            i += 1
-        if neg_dim >= 0:
-            new_shape[neg_dim] = old_size / new_size
-            new_size *= new_shape[neg_dim]
-    if new_size != old_size:
-        raise OperationError(space.w_ValueError,
-                space.wrap("total size of new array must be unchanged"))
-    return new_shape
-
-# Recalculating strides. Find the steps that the iteration does for each
-# dimension, given the stride and shape. Then try to create a new stride that
-# fits the new shape, using those steps. If there is a shape/step mismatch
-# (meaning that the realignment of elements crosses from one step into another)
-# return None so that the caller can raise an exception.
-def calc_new_strides(new_shape, old_shape, old_strides):
-    # Assumes that prod(old_shape) == prod(new_shape), len(old_shape) > 1, and
-    # len(new_shape) > 0
-    steps = []
-    last_step = 1
-    oldI = 0
-    new_strides = []
-    if old_strides[0] < old_strides[-1]:
-        #Start at old_shape[0], old_stides[0]
-        for i in range(len(old_shape)):
-            steps.append(old_strides[i] / last_step)
-            last_step *= old_shape[i]
-        cur_step = steps[0]
-        n_new_elems_used = 1
-        n_old_elems_to_use = old_shape[0]
-        for s in new_shape:
-            new_strides.append(cur_step * n_new_elems_used)
-            n_new_elems_used *= s
-            while n_new_elems_used > n_old_elems_to_use:
-                oldI += 1
-                if steps[oldI] != steps[oldI - 1]:
-                    return None
-                n_old_elems_to_use *= old_shape[oldI]
-            if n_new_elems_used == n_old_elems_to_use:
-                oldI += 1
-                if oldI >= len(old_shape):
-                    continue
-                cur_step = steps[oldI]
-                n_old_elems_to_use *= old_shape[oldI]
-    else:
-        #Start at old_shape[-1], old_strides[-1]
-        for i in range(len(old_shape) - 1, -1, -1):
-            steps.insert(0, old_strides[i] / last_step)
-            last_step *= old_shape[i]
-        cur_step = steps[-1]
-        n_new_elems_used = 1
-        oldI = -1
-        n_old_elems_to_use = old_shape[-1]
-        for i in range(len(new_shape) - 1, -1, -1):
-            s = new_shape[i]
-            new_strides.insert(0, cur_step * n_new_elems_used)
-            n_new_elems_used *= s
-            while n_new_elems_used > n_old_elems_to_use:
-                oldI -= 1
-                if steps[oldI] != steps[oldI + 1]:
-                    return None
-                n_old_elems_to_use *= old_shape[oldI]
-            if n_new_elems_used == n_old_elems_to_use:
-                oldI -= 1
-                if oldI < -len(old_shape):
-                    continue
-                cur_step = steps[oldI]
-                n_old_elems_to_use *= old_shape[oldI]
-    return new_strides
-
 class BaseArray(Wrappable):
     _attrs_ = ["invalidates", "shape", 'size']
 
@@ -1264,7 +1095,7 @@
         if order != 'C':  # or order != 'F':
             raise operationerrfmt(space.w_ValueError, "Unknown order: %s",
                                   order)
-    shape, elems_w = _find_shape_and_elems(space, w_item_or_iterable)
+    shape, elems_w = find_shape_and_elems(space, w_item_or_iterable)
     # they come back in C order
     size = len(elems_w)
     if space.is_w(w_dtype, space.w_None):
diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py
--- a/pypy/module/micronumpy/strides.py
+++ b/pypy/module/micronumpy/strides.py
@@ -37,3 +37,174 @@
     rstrides = [0] * (len(res_shape) - len(orig_shape)) + rstrides
     rbackstrides = [0] * (len(res_shape) - len(orig_shape)) + rbackstrides
     return rstrides, rbackstrides
+
+
+def find_shape_and_elems(space, w_iterable):
+    shape = [space.len_w(w_iterable)]
+    batch = space.listview(w_iterable)
+    while True:
+        new_batch = []
+        if not batch:
+            return shape, []
+        if not space.issequence_w(batch[0]):
+            for elem in batch:
+                if space.issequence_w(elem):
+                    raise OperationError(space.w_ValueError, space.wrap(
+                        "setting an array element with a sequence"))
+            return shape, batch
+        size = space.len_w(batch[0])
+        for w_elem in batch:
+            if not space.issequence_w(w_elem) or space.len_w(w_elem) != size:
+                raise OperationError(space.w_ValueError, space.wrap(
+                    "setting an array element with a sequence"))
+            new_batch += space.listview(w_elem)
+        shape.append(size)
+        batch = new_batch
+
+def shape_agreement(space, shape1, shape2):
+    ret = _shape_agreement(shape1, shape2)
+    if len(ret) < max(len(shape1), len(shape2)):
+        raise OperationError(space.w_ValueError,
+            space.wrap("operands could not be broadcast together with shapes (%s) (%s)" % (
+                ",".join([str(x) for x in shape1]),
+                ",".join([str(x) for x in shape2]),
+            ))
+        )
+    return ret
+
+def _shape_agreement(shape1, shape2):
+    """ Checks agreement about two shapes with respect to broadcasting. Returns
+    the resulting shape.
+    """
+    lshift = 0
+    rshift = 0
+    if len(shape1) > len(shape2):
+        m = len(shape1)
+        n = len(shape2)
+        rshift = len(shape2) - len(shape1)
+        remainder = shape1
+    else:
+        m = len(shape2)
+        n = len(shape1)
+        lshift = len(shape1) - len(shape2)
+        remainder = shape2
+    endshape = [0] * m
+    indices1 = [True] * m
+    indices2 = [True] * m
+    for i in range(m - 1, m - n - 1, -1):
+        left = shape1[i + lshift]
+        right = shape2[i + rshift]
+        if left == right:
+            endshape[i] = left
+        elif left == 1:
+            endshape[i] = right
+            indices1[i + lshift] = False
+        elif right == 1:
+            endshape[i] = left
+            indices2[i + rshift] = False
+        else:
+            return []
+            #raise OperationError(space.w_ValueError, space.wrap(
+            #    "frames are not aligned"))
+    for i in range(m - n):
+        endshape[i] = remainder[i]
+    return endshape
+
+def get_shape_from_iterable(space, old_size, w_iterable):
+    new_size = 0
+    new_shape = []
+    if space.isinstance_w(w_iterable, space.w_int):
+        new_size = space.int_w(w_iterable)
+        if new_size < 0:
+            new_size = old_size
+        new_shape = [new_size]
+    else:
+        neg_dim = -1
+        batch = space.listview(w_iterable)
+        new_size = 1
+        if len(batch) < 1:
+            if old_size == 1:
+                # Scalars can have an empty size.
+                new_size = 1
+            else:
+                new_size = 0
+        new_shape = []
+        i = 0
+        for elem in batch:
+            s = space.int_w(elem)
+            if s < 0:
+                if neg_dim >= 0:
+                    raise OperationError(space.w_ValueError, space.wrap(
+                             "can only specify one unknown dimension"))
+                s = 1
+                neg_dim = i
+            new_size *= s
+            new_shape.append(s)
+            i += 1
+        if neg_dim >= 0:
+            new_shape[neg_dim] = old_size / new_size
+            new_size *= new_shape[neg_dim]
+    if new_size != old_size:
+        raise OperationError(space.w_ValueError,
+                space.wrap("total size of new array must be unchanged"))
+    return new_shape
+
+# Recalculating strides. Find the steps that the iteration does for each
+# dimension, given the stride and shape. Then try to create a new stride that
+# fits the new shape, using those steps. If there is a shape/step mismatch
+# (meaning that the realignment of elements crosses from one step into another)
+# return None so that the caller can raise an exception.
+def calc_new_strides(new_shape, old_shape, old_strides):
+    # Assumes that prod(old_shape) == prod(new_shape), len(old_shape) > 1, and
+    # len(new_shape) > 0
+    steps = []
+    last_step = 1
+    oldI = 0
+    new_strides = []
+    if old_strides[0] < old_strides[-1]:
+        #Start at old_shape[0], old_stides[0]
+        for i in range(len(old_shape)):
+            steps.append(old_strides[i] / last_step)
+            last_step *= old_shape[i]
+        cur_step = steps[0]
+        n_new_elems_used = 1
+        n_old_elems_to_use = old_shape[0]
+        for s in new_shape:
+            new_strides.append(cur_step * n_new_elems_used)
+            n_new_elems_used *= s
+            while n_new_elems_used > n_old_elems_to_use:
+                oldI += 1
+                if steps[oldI] != steps[oldI - 1]:
+                    return None
+                n_old_elems_to_use *= old_shape[oldI]
+            if n_new_elems_used == n_old_elems_to_use:
+                oldI += 1
+                if oldI >= len(old_shape):
+                    continue
+                cur_step = steps[oldI]
+                n_old_elems_to_use *= old_shape[oldI]
+    else:
+        #Start at old_shape[-1], old_strides[-1]
+        for i in range(len(old_shape) - 1, -1, -1):
+            steps.insert(0, old_strides[i] / last_step)
+            last_step *= old_shape[i]
+        cur_step = steps[-1]
+        n_new_elems_used = 1
+        oldI = -1
+        n_old_elems_to_use = old_shape[-1]
+        for i in range(len(new_shape) - 1, -1, -1):
+            s = new_shape[i]
+            new_strides.insert(0, cur_step * n_new_elems_used)
+            n_new_elems_used *= s
+            while n_new_elems_used > n_old_elems_to_use:
+                oldI -= 1
+                if steps[oldI] != steps[oldI + 1]:
+                    return None
+                n_old_elems_to_use *= old_shape[oldI]
+            if n_new_elems_used == n_old_elems_to_use:
+                oldI -= 1
+                if oldI < -len(old_shape):
+                    continue
+                cur_step = steps[oldI]
+                n_old_elems_to_use *= old_shape[oldI]
+    return new_strides

From noreply at buildbot.pypy.org  Sat Jan 21 17:04:44 2012
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Sat, 21 Jan 2012 17:04:44 +0100 (CET)
Subject: [pypy-commit] pypy pytest: syncronize pylib and pytest with current
	hg versions
Message-ID: <20120121160444.32ECD82D45@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt 
Branch: pytest
Changeset: r51583:513060cd54e5
Date: 2012-01-21 17:03 +0100
http://bitbucket.org/pypy/pypy/changeset/513060cd54e5/

Log:	syncronize pylib and pytest with current hg versions

diff --git a/_pytest/__init__.py b/_pytest/__init__.py
--- a/_pytest/__init__.py
+++ b/_pytest/__init__.py
@@ -1,2 +1,2 @@
 #
-__version__ = '2.1.0.dev4'
+__version__ = '2.2.2.dev6'
diff --git a/_pytest/assertion/__init__.py b/_pytest/assertion/__init__.py
--- a/_pytest/assertion/__init__.py
+++ b/_pytest/assertion/__init__.py
@@ -2,35 +2,25 @@
 support for presenting detailed information in failing assertions.
 """
 import py
-import imp
-import marshal
-import struct
 import sys
 import pytest
 from _pytest.monkeypatch import monkeypatch
-from _pytest.assertion import reinterpret, util
-
-try:
-    from _pytest.assertion.rewrite import rewrite_asserts
-except ImportError:
-    rewrite_asserts = None
-else:
-    import ast
+from _pytest.assertion import util
 
 def pytest_addoption(parser):
     group = parser.getgroup("debugconfig")
-    group.addoption('--assertmode', action="store", dest="assertmode",
-                    choices=("on", "old", "off", "default"), default="default",
-                    metavar="on|old|off",
+    group.addoption('--assert', action="store", dest="assertmode",
+                    choices=("rewrite", "reinterp", "plain",),
+                    default="rewrite", metavar="MODE",
                     help="""control assertion debugging tools.
-'off' performs no assertion debugging.
-'old' reinterprets the expressions in asserts to glean information.
-'on' (the default) rewrites the assert statements in test modules to provide
-sub-expression results.""")
+'plain' performs no assertion debugging.
+'reinterp' reinterprets assert statements after they failed to provide assertion expression information.
+'rewrite' (the default) rewrites assert statements in test modules on import
+to provide assert expression information. """)
     group.addoption('--no-assert', action="store_true", default=False,
-        dest="noassert", help="DEPRECATED equivalent to --assertmode=off")
+        dest="noassert", help="DEPRECATED equivalent to --assert=plain")
     group.addoption('--nomagic', action="store_true", default=False,
-        dest="nomagic", help="DEPRECATED equivalent to --assertmode=off")
+        dest="nomagic", help="DEPRECATED equivalent to --assert=plain")
 
 class AssertionState:
     """State for the assertion plugin."""
@@ -40,89 +30,90 @@
         self.trace = config.trace.root.get("assertion")
 
 def pytest_configure(config):
-    warn_about_missing_assertion()
     mode = config.getvalue("assertmode")
     if config.getvalue("noassert") or config.getvalue("nomagic"):
-        if mode not in ("off", "default"):
-            raise pytest.UsageError("assertion options conflict")
-        mode = "off"
-    elif mode == "default":
-        mode = "on"
-    if mode != "off":
-        def callbinrepr(op, left, right):
-            hook_result = config.hook.pytest_assertrepr_compare(
-                config=config, op=op, left=left, right=right)
-            for new_expl in hook_result:
-                if new_expl:
-                    return '\n~'.join(new_expl)
+        mode = "plain"
+    if mode == "rewrite":
+        try:
+            import ast
+        except ImportError:
+            mode = "reinterp"
+        else:
+            if sys.platform.startswith('java'):
+                mode = "reinterp"
+    if mode != "plain":
+        _load_modules(mode)
         m = monkeypatch()
         config._cleanup.append(m.undo)
         m.setattr(py.builtin.builtins, 'AssertionError',
                   reinterpret.AssertionError)
-        m.setattr(util, '_reprcompare', callbinrepr)
-    if mode == "on" and rewrite_asserts is None:
-        mode = "old"
+    hook = None
+    if mode == "rewrite":
+        hook = rewrite.AssertionRewritingHook()
+        sys.meta_path.append(hook)
+    warn_about_missing_assertion(mode)
     config._assertstate = AssertionState(config, mode)
+    config._assertstate.hook = hook
     config._assertstate.trace("configured with mode set to %r" % (mode,))
 
-def _write_pyc(co, source_path):
-    if hasattr(imp, "cache_from_source"):
-        # Handle PEP 3147 pycs.
-        pyc = py.path.local(imp.cache_from_source(str(source_path)))
-        pyc.ensure()
-    else:
-        pyc = source_path + "c"
-    mtime = int(source_path.mtime())
-    fp = pyc.open("wb")
-    try:
-        fp.write(imp.get_magic())
-        fp.write(struct.pack(" 0 and
-                  item.identifier != "__future__"):
+            elif (not isinstance(item, ast.ImportFrom) or item.level > 0 or
+                  item.module != "__future__"):
                 lineno = item.lineno
                 break
             pos += 1
@@ -118,9 +358,9 @@
                    for alias in aliases]
         mod.body[pos:pos] = imports
         # Collect asserts.
-        nodes = collections.deque([mod])
+        nodes = [mod]
         while nodes:
-            node = nodes.popleft()
+            node = nodes.pop()
             for name, field in ast.iter_fields(node):
                 if isinstance(field, list):
                     new = []
@@ -143,7 +383,7 @@
         """Get a new variable."""
         # Use a character invalid in python identifiers to avoid clashing.
         name = "@py_assert" + str(next(self.variable_counter))
-        self.variables.add(name)
+        self.variables.append(name)
         return name
 
     def assign(self, expr):
@@ -198,7 +438,8 @@
             # There's already a message. Don't mess with it.
             return [assert_]
         self.statements = []
-        self.variables = set()
+        self.cond_chain = ()
+        self.variables = []
         self.variable_counter = itertools.count()
         self.stack = []
         self.on_failure = []
@@ -220,11 +461,11 @@
         else:
             raise_ = ast.Raise(exc, None, None)
         body.append(raise_)
-        # Delete temporary variables.
-        names = [ast.Name(name, ast.Del()) for name in self.variables]
-        if names:
-            delete = ast.Delete(names)
-            self.statements.append(delete)
+        # Clear temporary variables by setting them to None.
+        if self.variables:
+            variables = [ast.Name(name, ast.Store()) for name in self.variables]
+            clear = ast.Assign(variables, ast.Name("None", ast.Load()))
+            self.statements.append(clear)
         # Fix line numbers.
         for stmt in self.statements:
             set_location(stmt, assert_.lineno, assert_.col_offset)
@@ -240,21 +481,38 @@
         return name, self.explanation_param(expr)
 
     def visit_BoolOp(self, boolop):
-        operands = []
-        explanations = []
+        res_var = self.variable()
+        expl_list = self.assign(ast.List([], ast.Load()))
+        app = ast.Attribute(expl_list, "append", ast.Load())
+        is_or = int(isinstance(boolop.op, ast.Or))
+        body = save = self.statements
+        fail_save = self.on_failure
+        levels = len(boolop.values) - 1
         self.push_format_context()
-        for operand in boolop.values:
-            res, explanation = self.visit(operand)
-            operands.append(res)
-            explanations.append(explanation)
-        expls = ast.Tuple([ast.Str(expl) for expl in explanations], ast.Load())
-        is_or = ast.Num(isinstance(boolop.op, ast.Or))
-        expl_template = self.helper("format_boolop",
-                                    ast.Tuple(operands, ast.Load()), expls,
-                                    is_or)
+        # Process each operand, short-circuting if needed.
+        for i, v in enumerate(boolop.values):
+            if i:
+                fail_inner = []
+                self.on_failure.append(ast.If(cond, fail_inner, []))
+                self.on_failure = fail_inner
+            self.push_format_context()
+            res, expl = self.visit(v)
+            body.append(ast.Assign([ast.Name(res_var, ast.Store())], res))
+            expl_format = self.pop_format_context(ast.Str(expl))
+            call = ast.Call(app, [expl_format], [], None, None)
+            self.on_failure.append(ast.Expr(call))
+            if i < levels:
+                cond = res
+                if is_or:
+                    cond = ast.UnaryOp(ast.Not(), cond)
+                inner = []
+                self.statements.append(ast.If(cond, inner, []))
+                self.statements = body = inner
+        self.statements = save
+        self.on_failure = fail_save
+        expl_template = self.helper("format_boolop", expl_list, ast.Num(is_or))
         expl = self.pop_format_context(expl_template)
-        res = self.assign(ast.BoolOp(boolop.op, operands))
-        return res, self.explanation_param(expl)
+        return ast.Name(res_var, ast.Load()), self.explanation_param(expl)
 
     def visit_UnaryOp(self, unary):
         pattern = unary_map[unary.op.__class__]
@@ -288,7 +546,7 @@
             new_star, expl = self.visit(call.starargs)
             arg_expls.append("*" + expl)
         if call.kwargs:
-            new_kwarg, expl = self.visit(call.kwarg)
+            new_kwarg, expl = self.visit(call.kwargs)
             arg_expls.append("**" + expl)
         expl = "%s(%s)" % (func_expl, ', '.join(arg_expls))
         new_call = ast.Call(new_func, new_args, new_kwargs, new_star, new_kwarg)
diff --git a/_pytest/capture.py b/_pytest/capture.py
--- a/_pytest/capture.py
+++ b/_pytest/capture.py
@@ -11,22 +11,22 @@
     group._addoption('-s', action="store_const", const="no", dest="capture",
         help="shortcut for --capture=no.")
 
+ at pytest.mark.tryfirst
+def pytest_cmdline_parse(pluginmanager, args):
+    # we want to perform capturing already for plugin/conftest loading
+    if '-s' in args or "--capture=no" in args:
+        method = "no"
+    elif hasattr(os, 'dup') and '--capture=sys' not in args:
+        method = "fd"
+    else:
+        method = "sys"
+    capman = CaptureManager(method)
+    pluginmanager.register(capman, "capturemanager")
+
 def addouterr(rep, outerr):
-    repr = getattr(rep, 'longrepr', None)
-    if not hasattr(repr, 'addsection'):
-        return
     for secname, content in zip(["out", "err"], outerr):
         if content:
-            repr.addsection("Captured std%s" % secname, content.rstrip())
-
-def pytest_unconfigure(config):
-    # registered in config.py during early conftest.py loading
-    capman = config.pluginmanager.getplugin('capturemanager')
-    while capman._method2capture:
-        name, cap = capman._method2capture.popitem()
-        # XXX logging module may wants to close it itself on process exit
-        # otherwise we could do finalization here and call "reset()".
-        cap.suspend()
+            rep.sections.append(("Captured std%s" % secname, content))
 
 class NoCapture:
     def startall(self):
@@ -39,8 +39,9 @@
         return "", ""
 
 class CaptureManager:
-    def __init__(self):
+    def __init__(self, defaultmethod=None):
         self._method2capture = {}
+        self._defaultmethod = defaultmethod
 
     def _maketempfile(self):
         f = py.std.tempfile.TemporaryFile()
@@ -65,14 +66,6 @@
         else:
             raise ValueError("unknown capturing method: %r" % method)
 
-    def _getmethod_preoptionparse(self, args):
-        if '-s' in args or "--capture=no" in args:
-            return "no"
-        elif hasattr(os, 'dup') and '--capture=sys' not in args:
-            return "fd"
-        else:
-            return "sys"
-
     def _getmethod(self, config, fspath):
         if config.option.capture:
             method = config.option.capture
@@ -85,16 +78,22 @@
             method = "sys"
         return method
 
+    def reset_capturings(self):
+        for name, cap in self._method2capture.items():
+            cap.reset()
+
     def resumecapture_item(self, item):
         method = self._getmethod(item.config, item.fspath)
         if not hasattr(item, 'outerr'):
             item.outerr = ('', '') # we accumulate outerr on the item
         return self.resumecapture(method)
 
-    def resumecapture(self, method):
+    def resumecapture(self, method=None):
         if hasattr(self, '_capturing'):
             raise ValueError("cannot resume, already capturing with %r" %
                 (self._capturing,))
+        if method is None:
+            method = self._defaultmethod
         cap = self._method2capture.get(method)
         self._capturing = method
         if cap is None:
@@ -164,17 +163,6 @@
     def pytest_runtest_teardown(self, item):
         self.resumecapture_item(item)
 
-    def pytest__teardown_final(self, __multicall__, session):
-        method = self._getmethod(session.config, None)
-        self.resumecapture(method)
-        try:
-            rep = __multicall__.execute()
-        finally:
-            outerr = self.suspendcapture()
-        if rep:
-            addouterr(rep, outerr)
-        return rep
-
     def pytest_keyboard_interrupt(self, excinfo):
         if hasattr(self, '_capturing'):
             self.suspendcapture()
diff --git a/_pytest/config.py b/_pytest/config.py
--- a/_pytest/config.py
+++ b/_pytest/config.py
@@ -8,13 +8,15 @@
 def pytest_cmdline_parse(pluginmanager, args):
     config = Config(pluginmanager)
     config.parse(args)
-    if config.option.debug:
-        config.trace.root.setwriter(sys.stderr.write)
     return config
 
 def pytest_unconfigure(config):
-    for func in config._cleanup:
-        func()
+    while 1:
+        try:
+            fin = config._cleanup.pop()
+        except IndexError:
+            break
+        fin()
 
 class Parser:
     """ Parser for command line arguments. """
@@ -81,6 +83,7 @@
         self._inidict[name] = (help, type, default)
         self._ininames.append(name)
 
+
 class OptionGroup:
     def __init__(self, name, description="", parser=None):
         self.name = name
@@ -256,11 +259,14 @@
         self.hook = self.pluginmanager.hook
         self._inicache = {}
         self._cleanup = []
-    
+
     @classmethod
     def fromdictargs(cls, option_dict, args):
         """ constructor useable for subprocesses. """
         config = cls()
+        # XXX slightly crude way to initialize capturing
+        import _pytest.capture
+        _pytest.capture.pytest_cmdline_parse(config.pluginmanager, args)
         config._preparse(args, addopts=False)
         config.option.__dict__.update(option_dict)
         for x in config.option.plugins:
@@ -285,11 +291,10 @@
 
     def _setinitialconftest(self, args):
         # capture output during conftest init (#issue93)
-        from _pytest.capture import CaptureManager
-        capman = CaptureManager()
-        self.pluginmanager.register(capman, 'capturemanager')
-        # will be unregistered in capture.py's unconfigure()
-        capman.resumecapture(capman._getmethod_preoptionparse(args))
+        # XXX introduce load_conftest hook to avoid needing to know
+        # about capturing plugin here
+        capman = self.pluginmanager.getplugin("capturemanager")
+        capman.resumecapture()
         try:
             try:
                 self._conftest.setinitial(args)
@@ -334,6 +339,7 @@
         # Note that this can only be called once per testing process.
         assert not hasattr(self, 'args'), (
                 "can only parse cmdline args at most once per Config object")
+        self._origargs = args
         self._preparse(args)
         self._parser.hints.extend(self.pluginmanager._hints)
         args = self._parser.parse_setoption(args, self.option)
@@ -341,6 +347,14 @@
             args.append(py.std.os.getcwd())
         self.args = args
 
+    def addinivalue_line(self, name, line):
+        """ add a line to an ini-file option. The option must have been
+        declared but might not yet be set in which case the line becomes the
+        the first line in its value. """
+        x = self.getini(name)
+        assert isinstance(x, list)
+        x.append(line) # modifies the cached list inline
+
     def getini(self, name):
         """ return configuration value from an ini file. If the
         specified name hasn't been registered through a prior ``parse.addini``
@@ -422,7 +436,7 @@
 
 
 def getcfg(args, inibasenames):
-    args = [x for x in args if str(x)[0] != "-"]
+    args = [x for x in args if not str(x).startswith("-")]
     if not args:
         args = [py.path.local()]
     for arg in args:
diff --git a/_pytest/core.py b/_pytest/core.py
--- a/_pytest/core.py
+++ b/_pytest/core.py
@@ -16,11 +16,10 @@
  "junitxml resultlog doctest").split()
 
 class TagTracer:
-    def __init__(self, prefix="[pytest] "):
+    def __init__(self):
         self._tag2proc = {}
         self.writer = None
         self.indent = 0
-        self.prefix = prefix
 
     def get(self, name):
         return TagTracerSub(self, (name,))
@@ -30,7 +29,7 @@
             if args:
                 indent = "  " * self.indent
                 content = " ".join(map(str, args))
-                self.writer("%s%s%s\n" %(self.prefix, indent, content))
+                self.writer("%s%s [%s]\n" %(indent, content, ":".join(tags)))
         try:
             self._tag2proc[tags](tags, args)
         except KeyError:
@@ -212,6 +211,14 @@
             self.register(mod, modname)
             self.consider_module(mod)
 
+    def pytest_configure(self, config):
+        config.addinivalue_line("markers",
+            "tryfirst: mark a hook implementation function such that the "
+            "plugin machinery will try to call it first/as early as possible.")
+        config.addinivalue_line("markers",
+            "trylast: mark a hook implementation function such that the "
+            "plugin machinery will try to call it last/as late as possible.")
+
     def pytest_plugin_registered(self, plugin):
         import pytest
         dic = self.call_plugin(plugin, "pytest_namespace", {}) or {}
@@ -432,10 +439,7 @@
 def _preloadplugins():
     _preinit.append(PluginManager(load=True))
 
-def main(args=None, plugins=None):
-    """ returned exit code integer, after an in-process testing run
-    with the given command line arguments, preloading an optional list
-    of passed in plugin objects. """
+def _prepareconfig(args=None, plugins=None):
     if args is None:
         args = sys.argv[1:]
     elif isinstance(args, py.path.local):
@@ -449,13 +453,19 @@
     else: # subsequent calls to main will create a fresh instance
         _pluginmanager = PluginManager(load=True)
     hook = _pluginmanager.hook
+    if plugins:
+        for plugin in plugins:
+            _pluginmanager.register(plugin)
+    return hook.pytest_cmdline_parse(
+            pluginmanager=_pluginmanager, args=args)
+
+def main(args=None, plugins=None):
+    """ returned exit code integer, after an in-process testing run
+    with the given command line arguments, preloading an optional list
+    of passed in plugin objects. """
     try:
-        if plugins:
-            for plugin in plugins:
-                _pluginmanager.register(plugin)
-        config = hook.pytest_cmdline_parse(
-                pluginmanager=_pluginmanager, args=args)
-        exitstatus = hook.pytest_cmdline_main(config=config)
+        config = _prepareconfig(args, plugins)
+        exitstatus = config.hook.pytest_cmdline_main(config=config)
     except UsageError:
         e = sys.exc_info()[1]
         sys.stderr.write("ERROR: %s\n" %(e.args[0],))
diff --git a/_pytest/helpconfig.py b/_pytest/helpconfig.py
--- a/_pytest/helpconfig.py
+++ b/_pytest/helpconfig.py
@@ -1,7 +1,7 @@
 """ version info, help messages, tracing configuration.  """
 import py
 import pytest
-import inspect, sys
+import os, inspect, sys
 from _pytest.core import varnames
 
 def pytest_addoption(parser):
@@ -18,7 +18,29 @@
                help="trace considerations of conftest.py files."),
     group.addoption('--debug',
                action="store_true", dest="debug", default=False,
-               help="generate and show internal debugging information.")
+               help="store internal tracing debug information in 'pytestdebug.log'.")
+
+
+def pytest_cmdline_parse(__multicall__):
+    config = __multicall__.execute()
+    if config.option.debug:
+        path = os.path.abspath("pytestdebug.log")
+        f = open(path, 'w')
+        config._debugfile = f
+        f.write("versions pytest-%s, py-%s, python-%s\ncwd=%s\nargs=%s\n\n" %(
+            pytest.__version__, py.__version__, ".".join(map(str, sys.version_info)),
+            os.getcwd(), config._origargs))
+        config.trace.root.setwriter(f.write)
+        sys.stderr.write("writing pytestdebug information to %s\n" % path)
+    return config
+
+ at pytest.mark.trylast
+def pytest_unconfigure(config):
+    if hasattr(config, '_debugfile'):
+        config._debugfile.close()
+        sys.stderr.write("wrote pytestdebug information to %s\n" %
+            config._debugfile.name)
+        config.trace.root.setwriter(None)
 
 
 def pytest_cmdline_main(config):
@@ -34,6 +56,7 @@
     elif config.option.help:
         config.pluginmanager.do_configure(config)
         showhelp(config)
+        config.pluginmanager.do_unconfigure(config)
         return 0
 
 def showhelp(config):
@@ -91,7 +114,7 @@
         verinfo = getpluginversioninfo(config)
         if verinfo:
             lines.extend(verinfo)
-            
+
     if config.option.traceconfig:
         lines.append("active plugins:")
         plugins = []
diff --git a/_pytest/hookspec.py b/_pytest/hookspec.py
--- a/_pytest/hookspec.py
+++ b/_pytest/hookspec.py
@@ -121,16 +121,23 @@
 def pytest_itemstart(item, node=None):
     """ (deprecated, use pytest_runtest_logstart). """
 
-def pytest_runtest_protocol(item):
-    """ implements the standard runtest_setup/call/teardown protocol including
-    capturing exceptions and calling reporting hooks on the results accordingly.
+def pytest_runtest_protocol(item, nextitem):
+    """ implements the runtest_setup/call/teardown protocol for
+    the given test item, including capturing exceptions and calling
+    reporting hooks.
+
+    :arg item: test item for which the runtest protocol is performed.
+
+    :arg nexitem: the scheduled-to-be-next test item (or None if this
+                  is the end my friend).  This argument is passed on to
+                  :py:func:`pytest_runtest_teardown`.
 
     :return boolean: True if no further hook implementations should be invoked.
     """
 pytest_runtest_protocol.firstresult = True
 
 def pytest_runtest_logstart(nodeid, location):
-    """ signal the start of a test run. """
+    """ signal the start of running a single test item. """
 
 def pytest_runtest_setup(item):
     """ called before ``pytest_runtest_call(item)``. """
@@ -138,8 +145,14 @@
 def pytest_runtest_call(item):
     """ called to execute the test ``item``. """
 
-def pytest_runtest_teardown(item):
-    """ called after ``pytest_runtest_call``. """
+def pytest_runtest_teardown(item, nextitem):
+    """ called after ``pytest_runtest_call``.
+
+    :arg nexitem: the scheduled-to-be-next test item (None if no further
+                  test item is scheduled).  This argument can be used to
+                  perform exact teardowns, i.e. calling just enough finalizers
+                  so that nextitem only needs to call setup-functions.
+    """
 
 def pytest_runtest_makereport(item, call):
     """ return a :py:class:`_pytest.runner.TestReport` object
@@ -149,15 +162,8 @@
 pytest_runtest_makereport.firstresult = True
 
 def pytest_runtest_logreport(report):
-    """ process item test report. """
-
-# special handling for final teardown - somewhat internal for now
-def pytest__teardown_final(session):
-    """ called before test session finishes. """
-pytest__teardown_final.firstresult = True
-
-def pytest__teardown_final_logerror(report, session):
-    """ called if runtest_teardown_final failed. """
+    """ process a test setup/call/teardown report relating to
+    the respective phase of executing a test. """
 
 # -------------------------------------------------------------------------
 # test session related hooks
diff --git a/_pytest/junitxml.py b/_pytest/junitxml.py
--- a/_pytest/junitxml.py
+++ b/_pytest/junitxml.py
@@ -25,6 +25,10 @@
     long = int
 
 
+class Junit(py.xml.Namespace):
+    pass
+
+
 # We need to get the subset of the invalid unicode ranges according to
 # XML 1.0 which are valid in this python build.  Hence we calculate
 # this dynamically instead of hardcoding it.  The spec range of valid
@@ -40,6 +44,14 @@
 del _illegal_unichrs
 del _illegal_ranges
 
+def bin_xml_escape(arg):
+    def repl(matchobj):
+        i = ord(matchobj.group())
+        if i <= 0xFF:
+            return unicode('#x%02X') % i
+        else:
+            return unicode('#x%04X') % i
+    return illegal_xml_re.sub(repl, py.xml.escape(arg))
 
 def pytest_addoption(parser):
     group = parser.getgroup("terminal reporting")
@@ -68,117 +80,97 @@
         logfile = os.path.expanduser(os.path.expandvars(logfile))
         self.logfile = os.path.normpath(logfile)
         self.prefix = prefix
-        self.test_logs = []
+        self.tests = []
         self.passed = self.skipped = 0
         self.failed = self.errors = 0
-        self._durations = {}
 
     def _opentestcase(self, report):
         names = report.nodeid.split("::")
         names[0] = names[0].replace("/", '.')
-        names = tuple(names)
-        d = {'time': self._durations.pop(report.nodeid, "0")}
         names = [x.replace(".py", "") for x in names if x != "()"]
         classnames = names[:-1]
         if self.prefix:
             classnames.insert(0, self.prefix)
-        d['classname'] = ".".join(classnames)
-        d['name'] = py.xml.escape(names[-1])
-        attrs = ['%s="%s"' % item for item in sorted(d.items())]
-        self.test_logs.append("\n" % " ".join(attrs))
+        self.tests.append(Junit.testcase(
+            classname=".".join(classnames),
+            name=names[-1],
+            time=getattr(report, 'duration', 0)
+        ))
 
-    def _closetestcase(self):
-        self.test_logs.append("")
-
-    def appendlog(self, fmt, *args):
-        def repl(matchobj):
-            i = ord(matchobj.group())
-            if i <= 0xFF:
-                return unicode('#x%02X') % i
-            else:
-                return unicode('#x%04X') % i
-        args = tuple([illegal_xml_re.sub(repl, py.xml.escape(arg))
-                      for arg in args])
-        self.test_logs.append(fmt % args)
+    def append(self, obj):
+        self.tests[-1].append(obj)
 
     def append_pass(self, report):
         self.passed += 1
-        self._opentestcase(report)
-        self._closetestcase()
 
     def append_failure(self, report):
-        self._opentestcase(report)
         #msg = str(report.longrepr.reprtraceback.extraline)
         if "xfail" in report.keywords:
-            self.appendlog(
-                '')
+            self.append(
+                Junit.skipped(message="xfail-marked test passes unexpectedly"))
             self.skipped += 1
         else:
-            self.appendlog('%s',
-                report.longrepr)
+            sec = dict(report.sections)
+            fail = Junit.failure(message="test failure")
+            fail.append(str(report.longrepr))
+            self.append(fail)
+            for name in ('out', 'err'):
+                content = sec.get("Captured std%s" % name)
+                if content:
+                    tag = getattr(Junit, 'system-'+name)
+                    self.append(tag(bin_xml_escape(content)))
             self.failed += 1
-        self._closetestcase()
 
     def append_collect_failure(self, report):
-        self._opentestcase(report)
         #msg = str(report.longrepr.reprtraceback.extraline)
-        self.appendlog('%s',
-            report.longrepr)
-        self._closetestcase()
+        self.append(Junit.failure(str(report.longrepr),
+                                  message="collection failure"))
         self.errors += 1
 
     def append_collect_skipped(self, report):
-        self._opentestcase(report)
         #msg = str(report.longrepr.reprtraceback.extraline)
-        self.appendlog('%s',
-            report.longrepr)
-        self._closetestcase()
+        self.append(Junit.skipped(str(report.longrepr),
+                                  message="collection skipped"))
         self.skipped += 1
 
     def append_error(self, report):
-        self._opentestcase(report)
-        self.appendlog('%s',
-            report.longrepr)
-        self._closetestcase()
+        self.append(Junit.error(str(report.longrepr),
+                                message="test setup failure"))
         self.errors += 1
 
     def append_skipped(self, report):
-        self._opentestcase(report)
         if "xfail" in report.keywords:
-            self.appendlog(
-                '%s',
-                report.keywords['xfail'])
+            self.append(Junit.skipped(str(report.keywords['xfail']),
+                                      message="expected test failure"))
         else:
             filename, lineno, skipreason = report.longrepr
             if skipreason.startswith("Skipped: "):
                 skipreason = skipreason[9:]
-            self.appendlog('%s',
-                skipreason, "%s:%s: %s" % report.longrepr,
-                )
-        self._closetestcase()
+            self.append(
+                Junit.skipped("%s:%s: %s" % report.longrepr,
+                              type="pytest.skip",
+                              message=skipreason
+                ))
         self.skipped += 1
 
     def pytest_runtest_logreport(self, report):
         if report.passed:
-            self.append_pass(report)
+            if report.when == "call": # ignore setup/teardown
+                self._opentestcase(report)
+                self.append_pass(report)
         elif report.failed:
+            self._opentestcase(report)
             if report.when != "call":
                 self.append_error(report)
             else:
                 self.append_failure(report)
         elif report.skipped:
+            self._opentestcase(report)
             self.append_skipped(report)
 
-    def pytest_runtest_call(self, item, __multicall__):
-        start = time.time()
-        try:
-            return __multicall__.execute()
-        finally:
-            self._durations[item.nodeid] = time.time() - start
-
     def pytest_collectreport(self, report):
         if not report.passed:
+            self._opentestcase(report)
             if report.failed:
                 self.append_collect_failure(report)
             else:
@@ -187,10 +179,11 @@
     def pytest_internalerror(self, excrepr):
         self.errors += 1
         data = py.xml.escape(excrepr)
-        self.test_logs.append(
-            '\n'
-            '    '
-            '%s' % data)
+        self.tests.append(
+            Junit.testcase(
+                    Junit.error(data, message="internal error"),
+                    classname="pytest",
+                    name="internal"))
 
     def pytest_sessionstart(self, session):
         self.suite_start_time = time.time()
@@ -204,17 +197,17 @@
         suite_stop_time = time.time()
         suite_time_delta = suite_stop_time - self.suite_start_time
         numtests = self.passed + self.failed
+
         logfile.write('')
-        logfile.write('')
-        logfile.writelines(self.test_logs)
-        logfile.write('')
+        logfile.write(Junit.testsuite(
+            self.tests,
+            name="",
+            errors=self.errors,
+            failures=self.failed,
+            skips=self.skipped,
+            tests=numtests,
+            time="%.3f" % suite_time_delta,
+        ).unicode(indent=0))
         logfile.close()
 
     def pytest_terminal_summary(self, terminalreporter):
diff --git a/_pytest/main.py b/_pytest/main.py
--- a/_pytest/main.py
+++ b/_pytest/main.py
@@ -2,7 +2,7 @@
 
 import py
 import pytest, _pytest
-import os, sys
+import os, sys, imp
 tracebackcutdir = py.path.local(_pytest.__file__).dirpath()
 
 # exitcodes for the command line
@@ -11,6 +11,8 @@
 EXIT_INTERRUPTED = 2
 EXIT_INTERNALERROR = 3
 
+name_re = py.std.re.compile("^[a-zA-Z_]\w*$")
+
 def pytest_addoption(parser):
     parser.addini("norecursedirs", "directory patterns to avoid for recursion",
         type="args", default=('.*', 'CVS', '_darcs', '{arch}'))
@@ -27,6 +29,9 @@
                action="store", type="int", dest="maxfail", default=0,
                help="exit after first num failures or errors.")
 
+    group._addoption('--strict', action="store_true",
+               help="run pytest in strict mode, warnings become errors.")
+
     group = parser.getgroup("collect", "collection")
     group.addoption('--collectonly',
         action="store_true", dest="collectonly",
@@ -48,7 +53,7 @@
 def pytest_namespace():
     collect = dict(Item=Item, Collector=Collector, File=File, Session=Session)
     return dict(collect=collect)
-        
+
 def pytest_configure(config):
     py.test.config = config # compatibiltiy
     if config.option.exitfirst:
@@ -77,11 +82,11 @@
         session.exitstatus = EXIT_INTERNALERROR
         if excinfo.errisinstance(SystemExit):
             sys.stderr.write("mainloop: caught Spurious SystemExit!\n")
+    if initstate >= 2:
+        config.hook.pytest_sessionfinish(session=session,
+            exitstatus=session.exitstatus or (session._testsfailed and 1))
     if not session.exitstatus and session._testsfailed:
         session.exitstatus = EXIT_TESTSFAILED
-    if initstate >= 2:
-        config.hook.pytest_sessionfinish(session=session,
-            exitstatus=session.exitstatus)
     if initstate >= 1:
         config.pluginmanager.do_unconfigure(config)
     return session.exitstatus
@@ -101,8 +106,12 @@
 def pytest_runtestloop(session):
     if session.config.option.collectonly:
         return True
-    for item in session.session.items:
-        item.config.hook.pytest_runtest_protocol(item=item)
+    for i, item in enumerate(session.items):
+        try:
+            nextitem = session.items[i+1]
+        except IndexError:
+            nextitem = None
+        item.config.hook.pytest_runtest_protocol(item=item, nextitem=nextitem)
         if session.shouldstop:
             raise session.Interrupted(session.shouldstop)
     return True
@@ -132,7 +141,7 @@
         return getattr(pytest, name)
     return property(fget, None, None,
         "deprecated attribute %r, use pytest.%s" % (name,name))
-    
+
 class Node(object):
     """ base class for all Nodes in the collection tree.
     Collector subclasses have children, Items are terminal nodes."""
@@ -143,13 +152,13 @@
 
         #: the parent collector node.
         self.parent = parent
-        
+
         #: the test config object
         self.config = config or parent.config
 
         #: the collection this node is part of
         self.session = session or parent.session
-        
+
         #: filesystem path where this node was collected from
         self.fspath = getattr(parent, 'fspath', None)
         self.ihook = self.session.gethookproxy(self.fspath)
@@ -224,13 +233,13 @@
     def listchain(self):
         """ return list of all parent collectors up to self,
             starting from root of collection tree. """
-        l = [self]
-        while 1:
-            x = l[0]
-            if x.parent is not None: # and x.parent.parent is not None:
-                l.insert(0, x.parent)
-            else:
-                return l
+        chain = []
+        item = self
+        while item is not None:
+            chain.append(item)
+            item = item.parent
+        chain.reverse()
+        return chain
 
     def listnames(self):
         return [x.name for x in self.listchain()]
@@ -325,6 +334,8 @@
     """ a basic test invocation item. Note that for a single function
     there might be multiple test invocation items.
     """
+    nextitem = None
+
     def reportinfo(self):
         return self.fspath, None, ""
 
@@ -469,16 +480,29 @@
         return True
 
     def _tryconvertpyarg(self, x):
-        try:
-            mod = __import__(x, None, None, ['__doc__'])
-        except (ValueError, ImportError):
-            return x
-        p = py.path.local(mod.__file__)
-        if p.purebasename == "__init__":
-            p = p.dirpath()
-        else:
-            p = p.new(basename=p.purebasename+".py")
-        return str(p)
+        mod = None
+        path = [os.path.abspath('.')] + sys.path
+        for name in x.split('.'):
+            # ignore anything that's not a proper name here
+            # else something like --pyargs will mess up '.'
+            # since imp.find_module will actually sometimes work for it
+            # but it's supposed to be considered a filesystem path
+            # not a package
+            if name_re.match(name) is None:
+                return x
+            try:
+                fd, mod, type_ = imp.find_module(name, path)
+            except ImportError:
+                return x
+            else:
+                if fd is not None:
+                    fd.close()
+
+            if type_[2] != imp.PKG_DIRECTORY:
+                path = [os.path.dirname(mod)]
+            else:
+                path = [mod]
+        return mod
 
     def _parsearg(self, arg):
         """ return (fspath, names) tuple after checking the file exists. """
@@ -496,7 +520,7 @@
             raise pytest.UsageError(msg + arg)
         parts[0] = path
         return parts
-   
+
     def matchnodes(self, matching, names):
         self.trace("matchnodes", matching, names)
         self.trace.root.indent += 1
diff --git a/_pytest/mark.py b/_pytest/mark.py
--- a/_pytest/mark.py
+++ b/_pytest/mark.py
@@ -14,12 +14,37 @@
              "Terminate expression with ':' to make the first match match "
              "all subsequent tests (usually file-order). ")
 
+    group._addoption("-m",
+        action="store", dest="markexpr", default="", metavar="MARKEXPR",
+        help="only run tests matching given mark expression.  "
+             "example: -m 'mark1 and not mark2'."
+             )
+
+    group.addoption("--markers", action="store_true", help=
+        "show markers (builtin, plugin and per-project ones).")
+
+    parser.addini("markers", "markers for test functions", 'linelist')
+
+def pytest_cmdline_main(config):
+    if config.option.markers:
+        config.pluginmanager.do_configure(config)
+        tw = py.io.TerminalWriter()
+        for line in config.getini("markers"):
+            name, rest = line.split(":", 1)
+            tw.write("@pytest.mark.%s:" %  name, bold=True)
+            tw.line(rest)
+            tw.line()
+        config.pluginmanager.do_unconfigure(config)
+        return 0
+pytest_cmdline_main.tryfirst = True
+
 def pytest_collection_modifyitems(items, config):
     keywordexpr = config.option.keyword
-    if not keywordexpr:
+    matchexpr = config.option.markexpr
+    if not keywordexpr and not matchexpr:
         return
     selectuntil = False
-    if keywordexpr[-1] == ":":
+    if keywordexpr[-1:] == ":":
         selectuntil = True
         keywordexpr = keywordexpr[:-1]
 
@@ -29,21 +54,38 @@
         if keywordexpr and skipbykeyword(colitem, keywordexpr):
             deselected.append(colitem)
         else:
-            remaining.append(colitem)
             if selectuntil:
                 keywordexpr = None
+            if matchexpr:
+                if not matchmark(colitem, matchexpr):
+                    deselected.append(colitem)
+                    continue
+            remaining.append(colitem)
 
     if deselected:
         config.hook.pytest_deselected(items=deselected)
         items[:] = remaining
 
+class BoolDict:
+    def __init__(self, mydict):
+        self._mydict = mydict
+    def __getitem__(self, name):
+        return name in self._mydict
+
+def matchmark(colitem, matchexpr):
+    return eval(matchexpr, {}, BoolDict(colitem.obj.__dict__))
+
+def pytest_configure(config):
+    if config.option.strict:
+        pytest.mark._config = config
+
 def skipbykeyword(colitem, keywordexpr):
     """ return True if they given keyword expression means to
         skip this collector/item.
     """
     if not keywordexpr:
         return
-    
+
     itemkeywords = getkeywords(colitem)
     for key in filter(None, keywordexpr.split()):
         eor = key[:1] == '-'
@@ -77,15 +119,31 @@
          @py.test.mark.slowtest
          def test_function():
             pass
-  
+
     will set a 'slowtest' :class:`MarkInfo` object
     on the ``test_function`` object. """
 
     def __getattr__(self, name):
         if name[0] == "_":
             raise AttributeError(name)
+        if hasattr(self, '_config'):
+            self._check(name)
         return MarkDecorator(name)
 
+    def _check(self, name):
+        try:
+            if name in self._markers:
+                return
+        except AttributeError:
+            pass
+        self._markers = l = set()
+        for line in self._config.getini("markers"):
+            beginning = line.split(":", 1)
+            x = beginning[0].split("(", 1)[0]
+            l.add(x)
+        if name not in self._markers:
+            raise AttributeError("%r not a registered marker" % (name,))
+
 class MarkDecorator:
     """ A decorator for test functions and test classes.  When applied
     it will create :class:`MarkInfo` objects which may be
@@ -133,8 +191,7 @@
                         holder = MarkInfo(self.markname, self.args, self.kwargs)
                         setattr(func, self.markname, holder)
                     else:
-                        holder.kwargs.update(self.kwargs)
-                        holder.args += self.args
+                        holder.add(self.args, self.kwargs)
                 return func
         kw = self.kwargs.copy()
         kw.update(kwargs)
@@ -150,27 +207,20 @@
         self.args = args
         #: keyword argument dictionary, empty if nothing specified
         self.kwargs = kwargs
+        self._arglist = [(args, kwargs.copy())]
 
     def __repr__(self):
         return "" % (
                 self.name, self.args, self.kwargs)
 
-def pytest_itemcollected(item):
-    if not isinstance(item, pytest.Function):
-        return
-    try:
-        func = item.obj.__func__
-    except AttributeError:
-        func = getattr(item.obj, 'im_func', item.obj)
-    pyclasses = (pytest.Class, pytest.Module)
-    for node in item.listchain():
-        if isinstance(node, pyclasses):
-            marker = getattr(node.obj, 'pytestmark', None)
-            if marker is not None:
-                if isinstance(marker, list):
-                    for mark in marker:
-                        mark(func)
-                else:
-                    marker(func)
-        node = node.parent
-    item.keywords.update(py.builtin._getfuncdict(func))
+    def add(self, args, kwargs):
+        """ add a MarkInfo with the given args and kwargs. """
+        self._arglist.append((args, kwargs))
+        self.args += args
+        self.kwargs.update(kwargs)
+
+    def __iter__(self):
+        """ yield MarkInfo objects each relating to a marking-call. """
+        for args, kwargs in self._arglist:
+            yield MarkInfo(self.name, args, kwargs)
+
diff --git a/_pytest/monkeypatch.py b/_pytest/monkeypatch.py
--- a/_pytest/monkeypatch.py
+++ b/_pytest/monkeypatch.py
@@ -13,6 +13,7 @@
         monkeypatch.setenv(name, value, prepend=False)
         monkeypatch.delenv(name, value, raising=True)
         monkeypatch.syspath_prepend(path)
+        monkeypatch.chdir(path)
 
     All modifications will be undone after the requesting
     test function has finished. The ``raising``
@@ -30,6 +31,7 @@
     def __init__(self):
         self._setattr = []
         self._setitem = []
+        self._cwd = None
 
     def setattr(self, obj, name, value, raising=True):
         """ set attribute ``name`` on ``obj`` to ``value``, by default
@@ -83,6 +85,17 @@
             self._savesyspath = sys.path[:]
         sys.path.insert(0, str(path))
 
+    def chdir(self, path):
+        """ change the current working directory to the specified path
+        path can be a string or a py.path.local object
+        """
+        if self._cwd is None:
+            self._cwd = os.getcwd()
+        if hasattr(path, "chdir"):
+            path.chdir()
+        else:
+            os.chdir(path)
+
     def undo(self):
         """ undo previous changes.  This call consumes the
         undo stack.  Calling it a second time has no effect unless
@@ -95,9 +108,17 @@
         self._setattr[:] = []
         for dictionary, name, value in self._setitem:
             if value is notset:
-                del dictionary[name]
+                try:
+                    del dictionary[name]
+                except KeyError:
+                    pass # was already deleted, so we have the desired state
             else:
                 dictionary[name] = value
         self._setitem[:] = []
         if hasattr(self, '_savesyspath'):
             sys.path[:] = self._savesyspath
+            del self._savesyspath
+
+        if self._cwd is not None:
+            os.chdir(self._cwd)
+            self._cwd = None
diff --git a/_pytest/nose.py b/_pytest/nose.py
--- a/_pytest/nose.py
+++ b/_pytest/nose.py
@@ -13,6 +13,7 @@
             call.excinfo = call2.excinfo
 
 
+ at pytest.mark.trylast
 def pytest_runtest_setup(item):
     if isinstance(item, (pytest.Function)):
         if isinstance(item.parent, pytest.Generator):
diff --git a/_pytest/pastebin.py b/_pytest/pastebin.py
--- a/_pytest/pastebin.py
+++ b/_pytest/pastebin.py
@@ -38,7 +38,11 @@
         del tr._tw.__dict__['write']
 
 def getproxy():
-    return py.std.xmlrpclib.ServerProxy(url.xmlrpc).pastes
+    if sys.version_info < (3, 0):
+        from xmlrpclib import ServerProxy
+    else:
+        from xmlrpc.client import ServerProxy
+    return ServerProxy(url.xmlrpc).pastes
 
 def pytest_terminal_summary(terminalreporter):
     if terminalreporter.config.option.pastebin != "failed":
diff --git a/_pytest/pdb.py b/_pytest/pdb.py
--- a/_pytest/pdb.py
+++ b/_pytest/pdb.py
@@ -19,11 +19,13 @@
 class pytestPDB:
     """ Pseudo PDB that defers to the real pdb. """
     item = None
+    collector = None
 
     def set_trace(self):
         """ invoke PDB set_trace debugging, dropping any IO capturing. """
         frame = sys._getframe().f_back
-        item = getattr(self, 'item', None)
+        item = self.item or self.collector
+
         if item is not None:
             capman = item.config.pluginmanager.getplugin("capturemanager")
             out, err = capman.suspendcapture()
@@ -38,6 +40,14 @@
     pytestPDB.item = item
 pytest_runtest_setup = pytest_runtest_call = pytest_runtest_teardown = pdbitem
 
+ at pytest.mark.tryfirst
+def pytest_make_collect_report(__multicall__, collector):
+    try:
+        pytestPDB.collector = collector
+        return __multicall__.execute()
+    finally:
+        pytestPDB.collector = None
+
 def pytest_runtest_makereport():
     pytestPDB.item = None
     
@@ -60,7 +70,13 @@
         tw.sep(">", "traceback")
         rep.toterminal(tw)
         tw.sep(">", "entering PDB")
-        post_mortem(call.excinfo._excinfo[2])
+        # A doctest.UnexpectedException is not useful for post_mortem.
+        # Use the underlying exception instead:
+        if isinstance(call.excinfo.value, py.std.doctest.UnexpectedException):
+            tb = call.excinfo.value.exc_info[2]
+        else:
+            tb = call.excinfo._excinfo[2]
+        post_mortem(tb)
         rep._pdbshown = True
         return rep
 
diff --git a/_pytest/pytester.py b/_pytest/pytester.py
--- a/_pytest/pytester.py
+++ b/_pytest/pytester.py
@@ -25,6 +25,7 @@
         _pytest_fullpath
     except NameError:
         _pytest_fullpath = os.path.abspath(pytest.__file__.rstrip("oc"))
+        _pytest_fullpath = _pytest_fullpath.replace("$py.class", ".py")
 
 def pytest_funcarg___pytest(request):
     return PytestArg(request)
@@ -313,16 +314,6 @@
             result.extend(session.genitems(colitem))
         return result
 
-    def inline_genitems(self, *args):
-        #config = self.parseconfig(*args)
-        config = self.parseconfigure(*args)
-        rec = self.getreportrecorder(config)
-        session = Session(config)
-        config.hook.pytest_sessionstart(session=session)
-        session.perform_collect()
-        config.hook.pytest_sessionfinish(session=session, exitstatus=EXIT_OK)
-        return session.items, rec
-
     def runitem(self, source):
         # used from runner functional tests
         item = self.getitem(source)
@@ -343,64 +334,57 @@
         l = list(args) + [p]
         reprec = self.inline_run(*l)
         reports = reprec.getreports("pytest_runtest_logreport")
-        assert len(reports) == 1, reports
-        return reports[0]
+        assert len(reports) == 3, reports # setup/call/teardown
+        return reports[1]
+
+    def inline_genitems(self, *args):
+        return self.inprocess_run(list(args) + ['--collectonly'])
 
     def inline_run(self, *args):
-        args = ("-s", ) + args # otherwise FD leakage
-        config = self.parseconfig(*args)
-        reprec = self.getreportrecorder(config)
-        #config.pluginmanager.do_configure(config)
-        config.hook.pytest_cmdline_main(config=config)
-        #config.pluginmanager.do_unconfigure(config)
-        return reprec
+        items, rec = self.inprocess_run(args)
+        return rec
 
-    def config_preparse(self):
-        config = self.Config()
-        for plugin in self.plugins:
-            if isinstance(plugin, str):
-                config.pluginmanager.import_plugin(plugin)
-            else:
-                if isinstance(plugin, dict):
-                    plugin = PseudoPlugin(plugin)
-                if not config.pluginmanager.isregistered(plugin):
-                    config.pluginmanager.register(plugin)
-        return config
+    def inprocess_run(self, args, plugins=None):
+        rec = []
+        items = []
+        class Collect:
+            def pytest_configure(x, config):
+                rec.append(self.getreportrecorder(config))
+            def pytest_itemcollected(self, item):
+                items.append(item)
+        if not plugins:
+            plugins = []
+        plugins.append(Collect())
+        ret = self.pytestmain(list(args), plugins=[Collect()])
+        reprec = rec[0]
+        reprec.ret = ret
+        assert len(rec) == 1
+        return items, reprec
 
     def parseconfig(self, *args):
-        if not args:
-            args = (self.tmpdir,)
-        config = self.config_preparse()
-        args = list(args)
+        args = [str(x) for x in args]
         for x in args:
             if str(x).startswith('--basetemp'):
                 break
         else:
             args.append("--basetemp=%s" % self.tmpdir.dirpath('basetemp'))
-        config.parse(args)
+        import _pytest.core
+        config = _pytest.core._prepareconfig(args, self.plugins)
+        # the in-process pytest invocation needs to avoid leaking FDs
+        # so we register a "reset_capturings" callmon the capturing manager
+        # and make sure it gets called
+        config._cleanup.append(
+            config.pluginmanager.getplugin("capturemanager").reset_capturings)
+        import _pytest.config
+        self.request.addfinalizer(
+            lambda: _pytest.config.pytest_unconfigure(config))
         return config
 
-    def reparseconfig(self, args=None):
-        """ this is used from tests that want to re-invoke parse(). """
-        if not args:
-            args = [self.tmpdir]
-        oldconfig = getattr(py.test, 'config', None)
-        try:
-            c = py.test.config = self.Config()
-            c.basetemp = py.path.local.make_numbered_dir(prefix="reparse",
-                keep=0, rootdir=self.tmpdir, lock_timeout=None)
-            c.parse(args)
-            c.pluginmanager.do_configure(c)
-            self.request.addfinalizer(lambda: c.pluginmanager.do_unconfigure(c))
-            return c
-        finally:
-            py.test.config = oldconfig
-
     def parseconfigure(self, *args):
         config = self.parseconfig(*args)
         config.pluginmanager.do_configure(config)
         self.request.addfinalizer(lambda:
-            config.pluginmanager.do_unconfigure(config))
+        config.pluginmanager.do_unconfigure(config))
         return config
 
     def getitem(self,  source, funcname="test_func"):
@@ -420,7 +404,6 @@
             self.makepyfile(__init__ = "#")
         self.config = config = self.parseconfigure(path, *configargs)
         node = self.getnode(config, path)
-        #config.pluginmanager.do_unconfigure(config)
         return node
 
     def collect_by_name(self, modcol, name):
@@ -437,9 +420,16 @@
         return py.std.subprocess.Popen(cmdargs, stdout=stdout, stderr=stderr, **kw)
 
     def pytestmain(self, *args, **kwargs):
-        ret = pytest.main(*args, **kwargs)
-        if ret == 2:
-            raise KeyboardInterrupt()
+        class ResetCapturing:
+            @pytest.mark.trylast
+            def pytest_unconfigure(self, config):
+                capman = config.pluginmanager.getplugin("capturemanager")
+                capman.reset_capturings()
+        plugins = kwargs.setdefault("plugins", [])
+        rc = ResetCapturing()
+        plugins.append(rc)
+        return pytest.main(*args, **kwargs)
+
     def run(self, *cmdargs):
         return self._run(*cmdargs)
 
@@ -528,6 +518,8 @@
         pexpect = py.test.importorskip("pexpect", "2.4")
         if hasattr(sys, 'pypy_version_info') and '64' in py.std.platform.machine():
             pytest.skip("pypy-64 bit not supported")
+        if sys.platform == "darwin":
+            pytest.xfail("pexpect does not work reliably on darwin?!")
         logfile = self.tmpdir.join("spawn.out")
         child = pexpect.spawn(cmd, logfile=logfile.open("w"))
         child.timeout = expect_timeout
@@ -540,10 +532,6 @@
             return "INTERNAL not-utf8-decodeable, truncated string:\n%s" % (
                     py.io.saferepr(out),)
 
-class PseudoPlugin:
-    def __init__(self, vars):
-        self.__dict__.update(vars)
-
 class ReportRecorder(object):
     def __init__(self, hook):
         self.hook = hook
@@ -565,10 +553,17 @@
     def getreports(self, names="pytest_runtest_logreport pytest_collectreport"):
         return [x.report for x in self.getcalls(names)]
 
-    def matchreport(self, inamepart="", names="pytest_runtest_logreport pytest_collectreport", when=None):
+    def matchreport(self, inamepart="",
+        names="pytest_runtest_logreport pytest_collectreport", when=None):
         """ return a testreport whose dotted import path matches """
         l = []
         for rep in self.getreports(names=names):
+            try:
+                if not when and rep.when != "call" and rep.passed:
+                    # setup/teardown passing reports - let's ignore those
+                    continue
+            except AttributeError:
+                pass
             if when and getattr(rep, 'when', None) != when:
                 continue
             if not inamepart or inamepart in rep.nodeid.split("::"):
diff --git a/_pytest/python.py b/_pytest/python.py
--- a/_pytest/python.py
+++ b/_pytest/python.py
@@ -4,6 +4,7 @@
 import sys
 import pytest
 from py._code.code import TerminalRepr
+from _pytest.monkeypatch import monkeypatch
 
 import _pytest
 cutdir = py.path.local(_pytest.__file__).dirpath()
@@ -26,6 +27,24 @@
         showfuncargs(config)
         return 0
 
+
+def pytest_generate_tests(metafunc):
+    try:
+        param = metafunc.function.parametrize
+    except AttributeError:
+        return
+    for p in param:
+        metafunc.parametrize(*p.args, **p.kwargs)
+
+def pytest_configure(config):
+    config.addinivalue_line("markers",
+        "parametrize(argnames, argvalues): call a test function multiple "
+        "times passing in multiple different argument value sets. Example: "
+        "@parametrize('arg1', [1,2]) would lead to two calls of the decorated "
+        "test function, one with arg1=1 and another with arg1=2."
+    )
+
+
 @pytest.mark.trylast
 def pytest_namespace():
     raises.Exception = pytest.fail.Exception
@@ -138,6 +157,7 @@
             obj = obj.place_as
 
         self._fslineno = py.code.getfslineno(obj)
+        assert isinstance(self._fslineno[1], int), obj
         return self._fslineno
 
     def reportinfo(self):
@@ -155,6 +175,7 @@
         else:
             fspath, lineno = self._getfslineno()
             modpath = self.getmodpath()
+        assert isinstance(lineno, int)
         return fspath, lineno, modpath
 
 class PyCollectorMixin(PyobjMixin, pytest.Collector):
@@ -200,6 +221,7 @@
         module = self.getparent(Module).obj
         clscol = self.getparent(Class)
         cls = clscol and clscol.obj or None
+        transfer_markers(funcobj, cls, module)
         metafunc = Metafunc(funcobj, config=self.config,
             cls=cls, module=module)
         gentesthook = self.config.hook.pytest_generate_tests
@@ -219,6 +241,19 @@
             l.append(function)
         return l
 
+def transfer_markers(funcobj, cls, mod):
+    # XXX this should rather be code in the mark plugin or the mark
+    # plugin should merge with the python plugin.
+    for holder in (cls, mod):
+        try:
+            pytestmark = holder.pytestmark
+        except AttributeError:
+            continue
+        if isinstance(pytestmark, list):
+            for mark in pytestmark:
+                mark(funcobj)
+        else:
+            pytestmark(funcobj)
 
 class Module(pytest.File, PyCollectorMixin):
     def _getobj(self):
@@ -226,13 +261,8 @@
 
     def _importtestmodule(self):
         # we assume we are only called once per module
-        from _pytest import assertion
-        assertion.before_module_import(self)
         try:
-            try:
-                mod = self.fspath.pyimport(ensuresyspath=True)
-            finally:
-                assertion.after_module_import(self)
+            mod = self.fspath.pyimport(ensuresyspath=True)
         except SyntaxError:
             excinfo = py.code.ExceptionInfo()
             raise self.CollectError(excinfo.getrepr(style="short"))
@@ -244,7 +274,8 @@
                 "  %s\n"
                 "which is not the same as the test file we want to collect:\n"
                 "  %s\n"
-                "HINT: use a unique basename for your test file modules"
+                "HINT: remove __pycache__ / .pyc files and/or use a "
+                "unique basename for your test file modules"
                  % e.args
             )
         #print "imported test module", mod
@@ -374,6 +405,7 @@
         tw.line()
         tw.line("%s:%d" % (self.filename, self.firstlineno+1))
 
+
 class Generator(FunctionMixin, PyCollectorMixin, pytest.Collector):
     def collect(self):
         # test generators are seen as collectors but they also
@@ -430,6 +462,7 @@
                 "yielded functions (deprecated) cannot have funcargs")
         else:
             if callspec is not None:
+                self.callspec = callspec
                 self.funcargs = callspec.funcargs or {}
                 self._genid = callspec.id
                 if hasattr(callspec, "param"):
@@ -506,15 +539,59 @@
     request._fillfuncargs()
 
 _notexists = object()
-class CallSpec:
-    def __init__(self, funcargs, id, param):
-        self.funcargs = funcargs
-        self.id = id
+
+class CallSpec2(object):
+    def __init__(self, metafunc):
+        self.metafunc = metafunc
+        self.funcargs = {}
+        self._idlist = []
+        self.params = {}
+        self._globalid = _notexists
+        self._globalid_args = set()
+        self._globalparam = _notexists
+
+    def copy(self, metafunc):
+        cs = CallSpec2(self.metafunc)
+        cs.funcargs.update(self.funcargs)
+        cs.params.update(self.params)
+        cs._idlist = list(self._idlist)
+        cs._globalid = self._globalid
+        cs._globalid_args = self._globalid_args
+        cs._globalparam = self._globalparam
+        return cs
+
+    def _checkargnotcontained(self, arg):
+        if arg in self.params or arg in self.funcargs:
+            raise ValueError("duplicate %r" %(arg,))
+
+    def getparam(self, name):
+        try:
+            return self.params[name]
+        except KeyError:
+            if self._globalparam is _notexists:
+                raise ValueError(name)
+            return self._globalparam
+
+    @property
+    def id(self):
+        return "-".join(map(str, filter(None, self._idlist)))
+
+    def setmulti(self, valtype, argnames, valset, id):
+        for arg,val in zip(argnames, valset):
+            self._checkargnotcontained(arg)
+            getattr(self, valtype)[arg] = val
+        self._idlist.append(id)
+
+    def setall(self, funcargs, id, param):
+        for x in funcargs:
+            self._checkargnotcontained(x)
+        self.funcargs.update(funcargs)
+        if id is not _notexists:
+            self._idlist.append(id)
         if param is not _notexists:
-            self.param = param
-    def __repr__(self):
-        return "" %(
-            self.id, getattr(self, 'param', '?'), self.funcargs)
+            assert self._globalparam is _notexists
+            self._globalparam = param
+
 
 class Metafunc:
     def __init__(self, function, config=None, cls=None, module=None):
@@ -528,31 +605,69 @@
         self._calls = []
         self._ids = py.builtin.set()
 
+    def parametrize(self, argnames, argvalues, indirect=False, ids=None):
+        """ Add new invocations to the underlying test function using the list
+        of argvalues for the given argnames.  Parametrization is performed
+        during the collection phase.  If you need to setup expensive resources
+        you may pass indirect=True and implement a funcarg factory which can
+        perform the expensive setup just before a test is actually run.
+
+        :arg argnames: an argument name or a list of argument names
+
+        :arg argvalues: a list of values for the argname or a list of tuples of
+            values for the list of argument names.
+
+        :arg indirect: if True each argvalue corresponding to an argument will
+            be passed as request.param to its respective funcarg factory so
+            that it can perform more expensive setups during the setup phase of
+            a test rather than at collection time.
+
+        :arg ids: list of string ids each corresponding to the argvalues so
+            that they are part of the test id. If no ids are provided they will
+            be generated automatically from the argvalues.
+        """
+        if not isinstance(argnames, (tuple, list)):
+            argnames = (argnames,)
+            argvalues = [(val,) for val in argvalues]
+        for arg in argnames:
+            if arg not in self.funcargnames:
+                raise ValueError("%r has no argument %r" %(self.function, arg))
+        valtype = indirect and "params" or "funcargs"
+        if not ids:
+            idmaker = IDMaker()
+            ids = list(map(idmaker, argvalues))
+        newcalls = []
+        for callspec in self._calls or [CallSpec2(self)]:
+            for i, valset in enumerate(argvalues):
+                assert len(valset) == len(argnames)
+                newcallspec = callspec.copy(self)
+                newcallspec.setmulti(valtype, argnames, valset, ids[i])
+                newcalls.append(newcallspec)
+        self._calls = newcalls
+
     def addcall(self, funcargs=None, id=_notexists, param=_notexists):
-        """ add a new call to the underlying test function during the
-        collection phase of a test run.  Note that request.addcall() is
-        called during the test collection phase prior and independently
-        to actual test execution.  Therefore you should perform setup
-        of resources in a funcarg factory which can be instrumented
-        with the ``param``.
+        """ (deprecated, use parametrize) Add a new call to the underlying
+        test function during the collection phase of a test run.  Note that
+        request.addcall() is called during the test collection phase prior and
+        independently to actual test execution.  You should only use addcall()
+        if you need to specify multiple arguments of a test function.
 
         :arg funcargs: argument keyword dictionary used when invoking
             the test function.
 
         :arg id: used for reporting and identification purposes.  If you
-            don't supply an `id` the length of the currently
-            list of calls to the test function will be used.
+            don't supply an `id` an automatic unique id will be generated.
 
-        :arg param: will be exposed to a later funcarg factory invocation
-            through the ``request.param`` attribute.  It allows to
-            defer test fixture setup activities to when an actual
-            test is run.
+        :arg param: a parameter which will be exposed to a later funcarg factory
+            invocation through the ``request.param`` attribute.
         """
         assert funcargs is None or isinstance(funcargs, dict)
         if funcargs is not None:
             for name in funcargs:
                 if name not in self.funcargnames:
                     pytest.fail("funcarg %r not used in this function." % name)
+        else:
+            funcargs = {}
         if id is None:
             raise ValueError("id=None not allowed")
         if id is _notexists:
@@ -561,11 +676,26 @@
         if id in self._ids:
             raise ValueError("duplicate id %r" % id)
         self._ids.add(id)
-        self._calls.append(CallSpec(funcargs, id, param))
+
+        cs = CallSpec2(self)
+        cs.setall(funcargs, id, param)
+        self._calls.append(cs)
+
+class IDMaker:
+    def __init__(self):
+        self.counter = 0
+    def __call__(self, valset):
+        l = []
+        for val in valset:
+            if not isinstance(val, (int, str)):
+                val = "."+str(self.counter)
+            self.counter += 1
+            l.append(str(val))
+        return "-".join(l)
 
 class FuncargRequest:
     """ A request for function arguments from a test function.
-        
+
         Note that there is an optional ``param`` attribute in case
         there was an invocation to metafunc.addcall(param=...).
         If no such call was done in a ``pytest_generate_tests``
@@ -637,7 +767,7 @@
 
 
     def applymarker(self, marker):
-        """ apply a marker to a single test function invocation.
+        """ Apply a marker to a single test function invocation.
         This method is useful if you don't want to have a keyword/marker
         on all function invocations.
 
@@ -649,7 +779,7 @@
         self._pyfuncitem.keywords[marker.markname] = marker
 
     def cached_setup(self, setup, teardown=None, scope="module", extrakey=None):
-        """ return a testing resource managed by ``setup`` &
+        """ Return a testing resource managed by ``setup`` &
         ``teardown`` calls.  ``scope`` and ``extrakey`` determine when the
         ``teardown`` function will be called so that subsequent calls to
         ``setup`` would recreate the resource.
@@ -698,11 +828,18 @@
             self._raiselookupfailed(argname)
         funcargfactory = self._name2factory[argname].pop()
         oldarg = self._currentarg
-        self._currentarg = argname
+        mp = monkeypatch()
+        mp.setattr(self, '_currentarg', argname)
+        try:
+            param = self._pyfuncitem.callspec.getparam(argname)
+        except (AttributeError, ValueError):
+            pass
+        else:
+            mp.setattr(self, 'param', param, raising=False)
         try:
             self._funcargs[argname] = res = funcargfactory(request=self)
         finally:
-            self._currentarg = oldarg
+            mp.undo()
         return res
 
     def _getscopeitem(self, scope):
@@ -817,8 +954,7 @@
         >>> raises(ZeroDivisionError, f, x=0)
         
 
-    A third possibility is to use a string which which will
-    be executed::
+    A third possibility is to use a string to be executed::
 
         >>> raises(ZeroDivisionError, "f(0)")
         
diff --git a/_pytest/resultlog.py b/_pytest/resultlog.py
--- a/_pytest/resultlog.py
+++ b/_pytest/resultlog.py
@@ -63,6 +63,8 @@
         self.write_log_entry(testpath, lettercode, longrepr)
 
     def pytest_runtest_logreport(self, report):
+        if report.when != "call" and report.passed:
+            return
         res = self.config.hook.pytest_report_teststatus(report=report)
         code = res[1]
         if code == 'x':
@@ -89,5 +91,8 @@
             self.log_outcome(report, code, longrepr)
 
     def pytest_internalerror(self, excrepr):
-        path = excrepr.reprcrash.path
+        reprcrash = getattr(excrepr, 'reprcrash', None)
+        path = getattr(reprcrash, "path", None)
+        if path is None:
+            path = "cwd:%s" % py.path.local()
         self.write_log_entry(path, '!', str(excrepr))
diff --git a/_pytest/runner.py b/_pytest/runner.py
--- a/_pytest/runner.py
+++ b/_pytest/runner.py
@@ -1,6 +1,6 @@
 """ basic collect and runtest protocol implementations """
 
-import py, sys
+import py, sys, time
 from py._code.code import TerminalRepr
 
 def pytest_namespace():
@@ -14,33 +14,58 @@
 #
 # pytest plugin hooks
 
+def pytest_addoption(parser):
+    group = parser.getgroup("terminal reporting", "reporting", after="general")
+    group.addoption('--durations',
+         action="store", type="int", default=None, metavar="N",
+         help="show N slowest setup/test durations (N=0 for all)."),
+
+def pytest_terminal_summary(terminalreporter):
+    durations = terminalreporter.config.option.durations
+    if durations is None:
+        return
+    tr = terminalreporter
+    dlist = []
+    for replist in tr.stats.values():
+        for rep in replist:
+            if hasattr(rep, 'duration'):
+                dlist.append(rep)
+    if not dlist:
+        return
+    dlist.sort(key=lambda x: x.duration)
+    dlist.reverse()
+    if not durations:
+        tr.write_sep("=", "slowest test durations")
+    else:
+        tr.write_sep("=", "slowest %s test durations" % durations)
+        dlist = dlist[:durations]
+
+    for rep in dlist:
+        nodeid = rep.nodeid.replace("::()::", "::")
+        tr.write_line("%02.2fs %-8s %s" %
+            (rep.duration, rep.when, nodeid))
+
 def pytest_sessionstart(session):
     session._setupstate = SetupState()
 
-def pytest_sessionfinish(session, exitstatus):
-    hook = session.config.hook
-    rep = hook.pytest__teardown_final(session=session)
-    if rep:
-        hook.pytest__teardown_final_logerror(session=session, report=rep)
-        session.exitstatus = 1
-
 class NodeInfo:
     def __init__(self, location):
         self.location = location
 
-def pytest_runtest_protocol(item):
+def pytest_runtest_protocol(item, nextitem):
     item.ihook.pytest_runtest_logstart(
         nodeid=item.nodeid, location=item.location,
     )
-    runtestprotocol(item)
+    runtestprotocol(item, nextitem=nextitem)
     return True
 
-def runtestprotocol(item, log=True):
+def runtestprotocol(item, log=True, nextitem=None):
     rep = call_and_report(item, "setup", log)
     reports = [rep]
     if rep.passed:
         reports.append(call_and_report(item, "call", log))
-    reports.append(call_and_report(item, "teardown", log))
+    reports.append(call_and_report(item, "teardown", log,
+        nextitem=nextitem))
     return reports
 
 def pytest_runtest_setup(item):
@@ -49,16 +74,8 @@
 def pytest_runtest_call(item):
     item.runtest()
 
-def pytest_runtest_teardown(item):
-    item.session._setupstate.teardown_exact(item)
-
-def pytest__teardown_final(session):
-    call = CallInfo(session._setupstate.teardown_all, when="teardown")
-    if call.excinfo:
-        ntraceback = call.excinfo.traceback .cut(excludepath=py._pydir)
-        call.excinfo.traceback = ntraceback.filter()
-        longrepr = call.excinfo.getrepr(funcargs=True)
-        return TeardownErrorReport(longrepr)
+def pytest_runtest_teardown(item, nextitem):
+    item.session._setupstate.teardown_exact(item, nextitem)
 
 def pytest_report_teststatus(report):
     if report.when in ("setup", "teardown"):
@@ -74,18 +91,18 @@
 #
 # Implementation
 
-def call_and_report(item, when, log=True):
-    call = call_runtest_hook(item, when)
+def call_and_report(item, when, log=True, **kwds):
+    call = call_runtest_hook(item, when, **kwds)
     hook = item.ihook
     report = hook.pytest_runtest_makereport(item=item, call=call)
-    if log and (when == "call" or not report.passed):
+    if log:
         hook.pytest_runtest_logreport(report=report)
     return report
 
-def call_runtest_hook(item, when):
+def call_runtest_hook(item, when, **kwds):
     hookname = "pytest_runtest_" + when
     ihook = getattr(item.ihook, hookname)
-    return CallInfo(lambda: ihook(item=item), when=when)
+    return CallInfo(lambda: ihook(item=item, **kwds), when=when)
 
 class CallInfo:
     """ Result/Exception info a function invocation. """
@@ -95,12 +112,16 @@
         #: context of invocation: one of "setup", "call",
         #: "teardown", "memocollect"
         self.when = when
+        self.start = time.time()
         try:
-            self.result = func()
-        except KeyboardInterrupt:
-            raise
-        except:
-            self.excinfo = py.code.ExceptionInfo()
+            try:
+                self.result = func()
+            except KeyboardInterrupt:
+                raise
+            except:
+                self.excinfo = py.code.ExceptionInfo()
+        finally:
+            self.stop = time.time()
 
     def __repr__(self):
         if self.excinfo:
@@ -120,6 +141,10 @@
         return s
 
 class BaseReport(object):
+
+    def __init__(self, **kw):
+        self.__dict__.update(kw)
+
     def toterminal(self, out):
         longrepr = self.longrepr
         if hasattr(self, 'node'):
@@ -139,6 +164,7 @@
 
 def pytest_runtest_makereport(item, call):
     when = call.when
+    duration = call.stop-call.start
     keywords = dict([(x,1) for x in item.keywords])
     excinfo = call.excinfo
     if not call.excinfo:
@@ -160,14 +186,15 @@
             else: # exception in setup or teardown
                 longrepr = item._repr_failure_py(excinfo)
     return TestReport(item.nodeid, item.location,
-        keywords, outcome, longrepr, when)
+                      keywords, outcome, longrepr, when,
+                      duration=duration)
 
 class TestReport(BaseReport):
     """ Basic test report object (also used for setup and teardown calls if
     they fail).
     """
     def __init__(self, nodeid, location,
-            keywords, outcome, longrepr, when):
+            keywords, outcome, longrepr, when, sections=(), duration=0, **extra):
         #: normalized collection node id
         self.nodeid = nodeid
 
@@ -179,16 +206,25 @@
         #: a name -> value dictionary containing all keywords and
         #: markers associated with a test invocation.
         self.keywords = keywords
-        
+
         #: test outcome, always one of "passed", "failed", "skipped".
         self.outcome = outcome
 
         #: None or a failure representation.
         self.longrepr = longrepr
-        
+
         #: one of 'setup', 'call', 'teardown' to indicate runtest phase.
         self.when = when
 
+        #: list of (secname, data) extra information which needs to
+        #: marshallable
+        self.sections = list(sections)
+
+        #: time it took to run just the test
+        self.duration = duration
+
+        self.__dict__.update(extra)
+
     def __repr__(self):
         return "" % (
             self.nodeid, self.when, self.outcome)
@@ -196,8 +232,10 @@
 class TeardownErrorReport(BaseReport):
     outcome = "failed"
     when = "teardown"
-    def __init__(self, longrepr):
+    def __init__(self, longrepr, **extra):
         self.longrepr = longrepr
+        self.sections = []
+        self.__dict__.update(extra)
 
 def pytest_make_collect_report(collector):
     call = CallInfo(collector._memocollect, "memocollect")
@@ -219,11 +257,13 @@
         getattr(call, 'result', None))
 
 class CollectReport(BaseReport):
-    def __init__(self, nodeid, outcome, longrepr, result):
+    def __init__(self, nodeid, outcome, longrepr, result, sections=(), **extra):
         self.nodeid = nodeid
         self.outcome = outcome
         self.longrepr = longrepr
         self.result = result or []
+        self.sections = list(sections)
+        self.__dict__.update(extra)
 
     @property
     def location(self):
@@ -277,20 +317,22 @@
         self._teardown_with_finalization(None)
         assert not self._finalizers
 
-    def teardown_exact(self, item):
-        if self.stack and item == self.stack[-1]:
+    def teardown_exact(self, item, nextitem):
+        needed_collectors = nextitem and nextitem.listchain() or []
+        self._teardown_towards(needed_collectors)
+
+    def _teardown_towards(self, needed_collectors):
+        while self.stack:
+            if self.stack == needed_collectors[:len(self.stack)]:
+                break
             self._pop_and_teardown()
-        else:
-            self._callfinalizers(item)
 
     def prepare(self, colitem):
         """ setup objects along the collector chain to the test-method
             and teardown previously setup objects."""
         needed_collectors = colitem.listchain()
-        while self.stack:
-            if self.stack == needed_collectors[:len(self.stack)]:
-                break
-            self._pop_and_teardown()
+        self._teardown_towards(needed_collectors)
+
         # check if the last collection node has raised an error
         for col in self.stack:
             if hasattr(col, '_prepare_exc'):
diff --git a/_pytest/skipping.py b/_pytest/skipping.py
--- a/_pytest/skipping.py
+++ b/_pytest/skipping.py
@@ -9,6 +9,21 @@
            action="store_true", dest="runxfail", default=False,
            help="run tests even if they are marked xfail")
 
+def pytest_configure(config):
+    config.addinivalue_line("markers",
+        "skipif(*conditions): skip the given test function if evaluation "
+        "of all conditions has a True value.  Evaluation happens within the "
+        "module global context. Example: skipif('sys.platform == \"win32\"') "
+        "skips the test if we are on the win32 platform. "
+    )
+    config.addinivalue_line("markers",
+        "xfail(*conditions, reason=None, run=True): mark the the test function "
+        "as an expected failure. Optionally specify a reason and run=False "
+        "if you don't even want to execute the test function. Any positional "
+        "condition strings will be evaluated (like with skipif) and if one is "
+        "False the marker will not be applied."
+    )
+
 def pytest_namespace():
     return dict(xfail=xfail)
 
@@ -169,21 +184,23 @@
         elif char == "X":
             show_xpassed(terminalreporter, lines)
         elif char in "fF":
-            show_failed(terminalreporter, lines)
+            show_simple(terminalreporter, lines, 'failed', "FAIL %s")
         elif char in "sS":
             show_skipped(terminalreporter, lines)
+        elif char == "E":
+            show_simple(terminalreporter, lines, 'error', "ERROR %s")
     if lines:
         tr._tw.sep("=", "short test summary info")
         for line in lines:
             tr._tw.line(line)
 
-def show_failed(terminalreporter, lines):
+def show_simple(terminalreporter, lines, stat, format):
     tw = terminalreporter._tw
-    failed = terminalreporter.stats.get("failed")
+    failed = terminalreporter.stats.get(stat)
     if failed:
         for rep in failed:
             pos = rep.nodeid
-            lines.append("FAIL %s" %(pos, ))
+            lines.append(format %(pos, ))
 
 def show_xfailed(terminalreporter, lines):
     xfailed = terminalreporter.stats.get("xfailed")
diff --git a/_pytest/terminal.py b/_pytest/terminal.py
--- a/_pytest/terminal.py
+++ b/_pytest/terminal.py
@@ -15,7 +15,7 @@
     group._addoption('-r',
          action="store", dest="reportchars", default=None, metavar="chars",
          help="show extra test summary info as specified by chars (f)ailed, "
-              "(s)skipped, (x)failed, (X)passed.")
+              "(E)error, (s)skipped, (x)failed, (X)passed.")
     group._addoption('-l', '--showlocals',
          action="store_true", dest="showlocals", default=False,
          help="show locals in tracebacks (disabled by default).")
@@ -43,7 +43,8 @@
             pass
         else:
             stdout = os.fdopen(newfd, stdout.mode, 1)
-            config._toclose = stdout
+            config._cleanup.append(lambda: stdout.close())
+
     reporter = TerminalReporter(config, stdout)
     config.pluginmanager.register(reporter, 'terminalreporter')
     if config.option.debug or config.option.traceconfig:
@@ -52,11 +53,6 @@
             reporter.write_line("[traceconfig] " + msg)
         config.trace.root.setprocessor("pytest:config", mywriter)
 
-def pytest_unconfigure(config):
-    if hasattr(config, '_toclose'):
-        #print "closing", config._toclose, config._toclose.fileno()
-        config._toclose.close()
-
 def getreportopt(config):
     reportopts = ""
     optvalue = config.option.report
@@ -165,9 +161,6 @@
     def pytest_deselected(self, items):
         self.stats.setdefault('deselected', []).extend(items)
 
-    def pytest__teardown_final_logerror(self, report):
-        self.stats.setdefault("error", []).append(report)
-
     def pytest_runtest_logstart(self, nodeid, location):
         # ensure that the path is printed before the
         # 1st test of a module starts running
@@ -259,7 +252,7 @@
         msg = "platform %s -- Python %s" % (sys.platform, verinfo)
         if hasattr(sys, 'pypy_version_info'):
             verinfo = ".".join(map(str, sys.pypy_version_info[:3]))
-            msg += "[pypy-%s]" % verinfo
+            msg += "[pypy-%s-%s]" % (verinfo, sys.pypy_version_info[3])
         msg += " -- pytest-%s" % (py.test.__version__)
         if self.verbosity > 0 or self.config.option.debug or \
            getattr(self.config.option, 'pastebin', None):
@@ -318,12 +311,17 @@
             self.config.hook.pytest_terminal_summary(terminalreporter=self)
         if exitstatus == 2:
             self._report_keyboardinterrupt()
+            del self._keyboardinterrupt_memo
         self.summary_deselected()
         self.summary_stats()
 
     def pytest_keyboard_interrupt(self, excinfo):
         self._keyboardinterrupt_memo = excinfo.getrepr(funcargs=True)
 
+    def pytest_unconfigure(self):
+        if hasattr(self, '_keyboardinterrupt_memo'):
+            self._report_keyboardinterrupt()
+
     def _report_keyboardinterrupt(self):
         excrepr = self._keyboardinterrupt_memo
         msg = excrepr.reprcrash.message
@@ -388,7 +386,7 @@
                 else:
                     msg = self._getfailureheadline(rep)
                     self.write_sep("_", msg)
-                    rep.toterminal(self._tw)
+                    self._outrep_summary(rep)
 
     def summary_errors(self):
         if self.config.option.tbstyle != "no":
@@ -406,7 +404,15 @@
                 elif rep.when == "teardown":
                     msg = "ERROR at teardown of " + msg
                 self.write_sep("_", msg)
-                rep.toterminal(self._tw)
+                self._outrep_summary(rep)
+
+    def _outrep_summary(self, rep):
+        rep.toterminal(self._tw)
+        for secname, content in rep.sections:
+            self._tw.sep("-", secname)
+            if content[-1:] == "\n":
+                content = content[:-1]
+            self._tw.line(content)
 
     def summary_stats(self):
         session_duration = py.std.time.time() - self._sessionstarttime
@@ -417,9 +423,10 @@
                 keys.append(key)
         parts = []
         for key in keys:
-            val = self.stats.get(key, None)
-            if val:
-                parts.append("%d %s" %(len(val), key))
+            if key: # setup/teardown reports have an empty key, ignore them
+                val = self.stats.get(key, None)
+                if val:
+                    parts.append("%d %s" %(len(val), key))
         line = ", ".join(parts)
         # XXX coloring
         msg = "%s in %.2f seconds" %(line, session_duration)
@@ -430,8 +437,15 @@
 
     def summary_deselected(self):
         if 'deselected' in self.stats:
+            l = []
+            k = self.config.option.keyword
+            if k:
+                l.append("-k%s" % k)
+            m = self.config.option.markexpr
+            if m:
+                l.append("-m %r" % m)
             self.write_sep("=", "%d tests deselected by %r" %(
-                len(self.stats['deselected']), self.config.option.keyword), bold=True)
+                len(self.stats['deselected']), " ".join(l)), bold=True)
 
 def repr_pythonversion(v=None):
     if v is None:
diff --git a/_pytest/tmpdir.py b/_pytest/tmpdir.py
--- a/_pytest/tmpdir.py
+++ b/_pytest/tmpdir.py
@@ -46,7 +46,7 @@
 
     def finish(self):
         self.trace("finish")
-        
+
 def pytest_configure(config):
     mp = monkeypatch()
     t = TempdirHandler(config)
@@ -64,5 +64,5 @@
     name = request._pyfuncitem.name
     name = py.std.re.sub("[\W]", "_", name)
     x = request.config._tmpdirhandler.mktemp(name, numbered=True)
-    return x.realpath()
+    return x
 
diff --git a/_pytest/unittest.py b/_pytest/unittest.py
--- a/_pytest/unittest.py
+++ b/_pytest/unittest.py
@@ -2,6 +2,9 @@
 import pytest, py
 import sys, pdb
 
+# for transfering markers
+from _pytest.python import transfer_markers
+
 def pytest_pycollect_makeitem(collector, name, obj):
     unittest = sys.modules.get('unittest')
     if unittest is None:
@@ -19,7 +22,14 @@
 class UnitTestCase(pytest.Class):
     def collect(self):
         loader = py.std.unittest.TestLoader()
+        module = self.getparent(pytest.Module).obj
+        cls = self.obj
         for name in loader.getTestCaseNames(self.obj):
+            x = getattr(self.obj, name)
+            funcobj = getattr(x, 'im_func', x)
+            transfer_markers(funcobj, cls, module)
+            if hasattr(funcobj, 'todo'):
+                pytest.mark.xfail(reason=str(funcobj.todo))(funcobj)
             yield TestCaseFunction(name, parent=self)
 
     def setup(self):
@@ -37,15 +47,13 @@
 class TestCaseFunction(pytest.Function):
     _excinfo = None
 
-    def __init__(self, name, parent):
-        super(TestCaseFunction, self).__init__(name, parent)
-        if hasattr(self._obj, 'todo'):
-            getattr(self._obj, 'im_func', self._obj).xfail = \
-                pytest.mark.xfail(reason=str(self._obj.todo))
-
     def setup(self):
         self._testcase = self.parent.obj(self.name)
         self._obj = getattr(self._testcase, self.name)
+        if hasattr(self._testcase, 'skip'):
+            pytest.skip(self._testcase.skip)
+        if hasattr(self._obj, 'skip'):
+            pytest.skip(self._obj.skip)
         if hasattr(self._testcase, 'setup_method'):
             self._testcase.setup_method(self._obj)
 
@@ -104,7 +112,10 @@
 
     def _prunetraceback(self, excinfo):
         pytest.Function._prunetraceback(self, excinfo)
-        excinfo.traceback = excinfo.traceback.filter(lambda x:not x.frame.f_globals.get('__unittest'))
+        traceback = excinfo.traceback.filter(
+            lambda x:not x.frame.f_globals.get('__unittest'))
+        if traceback:
+            excinfo.traceback = traceback
 
 @pytest.mark.tryfirst
 def pytest_runtest_makereport(item, call):
@@ -120,14 +131,19 @@
             ut = sys.modules['twisted.python.failure']
             Failure__init__ = ut.Failure.__init__.im_func
             check_testcase_implements_trial_reporter()
-            def excstore(self, exc_value=None, exc_type=None, exc_tb=None):
+            def excstore(self, exc_value=None, exc_type=None, exc_tb=None,
+                captureVars=None):
                 if exc_value is None:
                     self._rawexcinfo = sys.exc_info()
                 else:
                     if exc_type is None:
                         exc_type = type(exc_value)
                     self._rawexcinfo = (exc_type, exc_value, exc_tb)
-                Failure__init__(self, exc_value, exc_type, exc_tb)
+                try:
+                    Failure__init__(self, exc_value, exc_type, exc_tb,
+                        captureVars=captureVars)
+                except TypeError:
+                    Failure__init__(self, exc_value, exc_type, exc_tb)
             ut.Failure.__init__ = excstore
             try:
                 return __multicall__.execute()
diff --git a/py/__init__.py b/py/__init__.py
--- a/py/__init__.py
+++ b/py/__init__.py
@@ -8,7 +8,7 @@
 
 (c) Holger Krekel and others, 2004-2010
 """
-__version__ = '1.4.4.dev1'
+__version__ = '1.4.7.dev3'
 
 from py import _apipkg
 
@@ -70,6 +70,11 @@
         'getrawcode'        : '._code.code:getrawcode',
         'patch_builtins'    : '._code.code:patch_builtins',
         'unpatch_builtins'  : '._code.code:unpatch_builtins',
+        '_AssertionError'   : '._code.assertion:AssertionError',
+        '_reinterpret_old'  : '._code.assertion:reinterpret_old',
+        '_reinterpret'      : '._code.assertion:reinterpret',
+        '_reprcompare'      : '._code.assertion:_reprcompare',
+        '_format_explanation' : '._code.assertion:_format_explanation',
     },
 
     # backports and additions of builtins
diff --git a/py/_builtin.py b/py/_builtin.py
--- a/py/_builtin.py
+++ b/py/_builtin.py
@@ -142,7 +142,7 @@
             del back
         elif locs is None:
             locs = globs
-        fp = open(fn, "rb")
+        fp = open(fn, "r")
         try:
             source = fp.read()
         finally:
diff --git a/py/_code/_assertionnew.py b/py/_code/_assertionnew.py
new file mode 100644
--- /dev/null
+++ b/py/_code/_assertionnew.py
@@ -0,0 +1,339 @@
+"""
+Find intermediate evalutation results in assert statements through builtin AST.
+This should replace _assertionold.py eventually.
+"""
+
+import sys
+import ast
+
+import py
+from py._code.assertion import _format_explanation, BuiltinAssertionError
+
+
+if sys.platform.startswith("java") and sys.version_info < (2, 5, 2):
+    # See http://bugs.jython.org/issue1497
+    _exprs = ("BoolOp", "BinOp", "UnaryOp", "Lambda", "IfExp", "Dict",
+              "ListComp", "GeneratorExp", "Yield", "Compare", "Call",
+              "Repr", "Num", "Str", "Attribute", "Subscript", "Name",
+              "List", "Tuple")
+    _stmts = ("FunctionDef", "ClassDef", "Return", "Delete", "Assign",
+              "AugAssign", "Print", "For", "While", "If", "With", "Raise",
+              "TryExcept", "TryFinally", "Assert", "Import", "ImportFrom",
+              "Exec", "Global", "Expr", "Pass", "Break", "Continue")
+    _expr_nodes = set(getattr(ast, name) for name in _exprs)
+    _stmt_nodes = set(getattr(ast, name) for name in _stmts)
+    def _is_ast_expr(node):
+        return node.__class__ in _expr_nodes
+    def _is_ast_stmt(node):
+        return node.__class__ in _stmt_nodes
+else:
+    def _is_ast_expr(node):
+        return isinstance(node, ast.expr)
+    def _is_ast_stmt(node):
+        return isinstance(node, ast.stmt)
+
+
+class Failure(Exception):
+    """Error found while interpreting AST."""
+
+    def __init__(self, explanation=""):
+        self.cause = sys.exc_info()
+        self.explanation = explanation
+
+
+def interpret(source, frame, should_fail=False):
+    mod = ast.parse(source)
+    visitor = DebugInterpreter(frame)
+    try:
+        visitor.visit(mod)
+    except Failure:
+        failure = sys.exc_info()[1]
+        return getfailure(failure)
+    if should_fail:
+        return ("(assertion failed, but when it was re-run for "
+                "printing intermediate values, it did not fail.  Suggestions: "
+                "compute assert expression before the assert or use --no-assert)")
+
+def run(offending_line, frame=None):
+    if frame is None:
+        frame = py.code.Frame(sys._getframe(1))
+    return interpret(offending_line, frame)
+
+def getfailure(failure):
+    explanation = _format_explanation(failure.explanation)
+    value = failure.cause[1]
+    if str(value):
+        lines = explanation.splitlines()
+        if not lines:
+            lines.append("")
+        lines[0] += " << %s" % (value,)
+        explanation = "\n".join(lines)
+    text = "%s: %s" % (failure.cause[0].__name__, explanation)
+    if text.startswith("AssertionError: assert "):
+        text = text[16:]
+    return text
+
+
+operator_map = {
+    ast.BitOr : "|",
+    ast.BitXor : "^",
+    ast.BitAnd : "&",
+    ast.LShift : "<<",
+    ast.RShift : ">>",
+    ast.Add : "+",
+    ast.Sub : "-",
+    ast.Mult : "*",
+    ast.Div : "/",
+    ast.FloorDiv : "//",
+    ast.Mod : "%",
+    ast.Eq : "==",
+    ast.NotEq : "!=",
+    ast.Lt : "<",
+    ast.LtE : "<=",
+    ast.Gt : ">",
+    ast.GtE : ">=",
+    ast.Pow : "**",
+    ast.Is : "is",
+    ast.IsNot : "is not",
+    ast.In : "in",
+    ast.NotIn : "not in"
+}
+
+unary_map = {
+    ast.Not : "not %s",
+    ast.Invert : "~%s",
+    ast.USub : "-%s",
+    ast.UAdd : "+%s"
+}
+
+
+class DebugInterpreter(ast.NodeVisitor):
+    """Interpret AST nodes to gleam useful debugging information. """
+
+    def __init__(self, frame):
+        self.frame = frame
+
+    def generic_visit(self, node):
+        # Fallback when we don't have a special implementation.
+        if _is_ast_expr(node):
+            mod = ast.Expression(node)
+            co = self._compile(mod)
+            try:
+                result = self.frame.eval(co)
+            except Exception:
+                raise Failure()
+            explanation = self.frame.repr(result)
+            return explanation, result
+        elif _is_ast_stmt(node):
+            mod = ast.Module([node])
+            co = self._compile(mod, "exec")
+            try:
+                self.frame.exec_(co)
+            except Exception:
+                raise Failure()
+            return None, None
+        else:
+            raise AssertionError("can't handle %s" %(node,))
+
+    def _compile(self, source, mode="eval"):
+        return compile(source, "", mode)
+
+    def visit_Expr(self, expr):
+        return self.visit(expr.value)
+
+    def visit_Module(self, mod):
+        for stmt in mod.body:
+            self.visit(stmt)
+
+    def visit_Name(self, name):
+        explanation, result = self.generic_visit(name)
+        # See if the name is local.
+        source = "%r in locals() is not globals()" % (name.id,)
+        co = self._compile(source)
+        try:
+            local = self.frame.eval(co)
+        except Exception:
+            # have to assume it isn't
+            local = False
+        if not local:
+            return name.id, result
+        return explanation, result
+
+    def visit_Compare(self, comp):
+        left = comp.left
+        left_explanation, left_result = self.visit(left)
+        for op, next_op in zip(comp.ops, comp.comparators):
+            next_explanation, next_result = self.visit(next_op)
+            op_symbol = operator_map[op.__class__]
+            explanation = "%s %s %s" % (left_explanation, op_symbol,
+                                        next_explanation)
+            source = "__exprinfo_left %s __exprinfo_right" % (op_symbol,)
+            co = self._compile(source)
+            try:
+                result = self.frame.eval(co, __exprinfo_left=left_result,
+                                         __exprinfo_right=next_result)
+            except Exception:
+                raise Failure(explanation)
+            try:
+                if not result:
+                    break
+            except KeyboardInterrupt:
+                raise
+            except:
+                break
+            left_explanation, left_result = next_explanation, next_result
+
+        rcomp = py.code._reprcompare
+        if rcomp:
+            res = rcomp(op_symbol, left_result, next_result)
+            if res:
+                explanation = res
+        return explanation, result
+
+    def visit_BoolOp(self, boolop):
+        is_or = isinstance(boolop.op, ast.Or)
+        explanations = []
+        for operand in boolop.values:
+            explanation, result = self.visit(operand)
+            explanations.append(explanation)
+            if result == is_or:
+                break
+        name = is_or and " or " or " and "
+        explanation = "(" + name.join(explanations) + ")"
+        return explanation, result
+
+    def visit_UnaryOp(self, unary):
+        pattern = unary_map[unary.op.__class__]
+        operand_explanation, operand_result = self.visit(unary.operand)
+        explanation = pattern % (operand_explanation,)
+        co = self._compile(pattern % ("__exprinfo_expr",))
+        try:
+            result = self.frame.eval(co, __exprinfo_expr=operand_result)
+        except Exception:
+            raise Failure(explanation)
+        return explanation, result
+
+    def visit_BinOp(self, binop):
+        left_explanation, left_result = self.visit(binop.left)
+        right_explanation, right_result = self.visit(binop.right)
+        symbol = operator_map[binop.op.__class__]
+        explanation = "(%s %s %s)" % (left_explanation, symbol,
+                                      right_explanation)
+        source = "__exprinfo_left %s __exprinfo_right" % (symbol,)
+        co = self._compile(source)
+        try:
+            result = self.frame.eval(co, __exprinfo_left=left_result,
+                                     __exprinfo_right=right_result)
+        except Exception:
+            raise Failure(explanation)
+        return explanation, result
+
+    def visit_Call(self, call):
+        func_explanation, func = self.visit(call.func)
+        arg_explanations = []
+        ns = {"__exprinfo_func" : func}
+        arguments = []
+        for arg in call.args:
+            arg_explanation, arg_result = self.visit(arg)
+            arg_name = "__exprinfo_%s" % (len(ns),)
+            ns[arg_name] = arg_result
+            arguments.append(arg_name)
+            arg_explanations.append(arg_explanation)
+        for keyword in call.keywords:
+            arg_explanation, arg_result = self.visit(keyword.value)
+            arg_name = "__exprinfo_%s" % (len(ns),)
+            ns[arg_name] = arg_result
+            keyword_source = "%s=%%s" % (keyword.arg)
+            arguments.append(keyword_source % (arg_name,))
+            arg_explanations.append(keyword_source % (arg_explanation,))
+        if call.starargs:
+            arg_explanation, arg_result = self.visit(call.starargs)
+            arg_name = "__exprinfo_star"
+            ns[arg_name] = arg_result
+            arguments.append("*%s" % (arg_name,))
+            arg_explanations.append("*%s" % (arg_explanation,))
+        if call.kwargs:
+            arg_explanation, arg_result = self.visit(call.kwargs)
+            arg_name = "__exprinfo_kwds"
+            ns[arg_name] = arg_result
+            arguments.append("**%s" % (arg_name,))
+            arg_explanations.append("**%s" % (arg_explanation,))
+        args_explained = ", ".join(arg_explanations)
+        explanation = "%s(%s)" % (func_explanation, args_explained)
+        args = ", ".join(arguments)
+        source = "__exprinfo_func(%s)" % (args,)
+        co = self._compile(source)
+        try:
+            result = self.frame.eval(co, **ns)
+        except Exception:
+            raise Failure(explanation)
+        pattern = "%s\n{%s = %s\n}"
+        rep = self.frame.repr(result)
+        explanation = pattern % (rep, rep, explanation)
+        return explanation, result
+
+    def _is_builtin_name(self, name):
+        pattern = "%r not in globals() and %r not in locals()"
+        source = pattern % (name.id, name.id)
+        co = self._compile(source)
+        try:
+            return self.frame.eval(co)
+        except Exception:
+            return False
+
+    def visit_Attribute(self, attr):
+        if not isinstance(attr.ctx, ast.Load):
+            return self.generic_visit(attr)
+        source_explanation, source_result = self.visit(attr.value)
+        explanation = "%s.%s" % (source_explanation, attr.attr)
+        source = "__exprinfo_expr.%s" % (attr.attr,)
+        co = self._compile(source)
+        try:
+            result = self.frame.eval(co, __exprinfo_expr=source_result)
+        except Exception:
+            raise Failure(explanation)
+        explanation = "%s\n{%s = %s.%s\n}" % (self.frame.repr(result),
+                                              self.frame.repr(result),
+                                              source_explanation, attr.attr)
+        # Check if the attr is from an instance.
+        source = "%r in getattr(__exprinfo_expr, '__dict__', {})"
+        source = source % (attr.attr,)
+        co = self._compile(source)
+        try:
+            from_instance = self.frame.eval(co, __exprinfo_expr=source_result)
+        except Exception:
+            from_instance = True
+        if from_instance:
+            rep = self.frame.repr(result)
+            pattern = "%s\n{%s = %s\n}"
+            explanation = pattern % (rep, rep, explanation)
+        return explanation, result
+
+    def visit_Assert(self, assrt):
+        test_explanation, test_result = self.visit(assrt.test)
+        if test_explanation.startswith("False\n{False =") and \
+                test_explanation.endswith("\n"):
+            test_explanation = test_explanation[15:-2]
+        explanation = "assert %s" % (test_explanation,)
+        if not test_result:
+            try:
+                raise BuiltinAssertionError
+            except Exception:
+                raise Failure(explanation)
+        return explanation, test_result
+
+    def visit_Assign(self, assign):
+        value_explanation, value_result = self.visit(assign.value)
+        explanation = "... = %s" % (value_explanation,)
+        name = ast.Name("__exprinfo_expr", ast.Load(),
+                        lineno=assign.value.lineno,
+                        col_offset=assign.value.col_offset)
+        new_assign = ast.Assign(assign.targets, name, lineno=assign.lineno,
+                                col_offset=assign.col_offset)
+        mod = ast.Module([new_assign])
+        co = self._compile(mod, "exec")
+        try:
+            self.frame.exec_(co, __exprinfo_expr=value_result)
+        except Exception:
+            raise Failure(explanation)
+        return explanation, value_result
diff --git a/py/_code/_assertionold.py b/py/_code/_assertionold.py
new file mode 100644
--- /dev/null
+++ b/py/_code/_assertionold.py
@@ -0,0 +1,555 @@
+import py
+import sys, inspect
+from compiler import parse, ast, pycodegen
+from py._code.assertion import BuiltinAssertionError, _format_explanation
+
+passthroughex = py.builtin._sysex
+
+class Failure:
+    def __init__(self, node):
+        self.exc, self.value, self.tb = sys.exc_info()
+        self.node = node
+
+class View(object):
+    """View base class.
+
+    If C is a subclass of View, then C(x) creates a proxy object around
+    the object x.  The actual class of the proxy is not C in general,
+    but a *subclass* of C determined by the rules below.  To avoid confusion
+    we call view class the class of the proxy (a subclass of C, so of View)
+    and object class the class of x.
+
+    Attributes and methods not found in the proxy are automatically read on x.
+    Other operations like setting attributes are performed on the proxy, as
+    determined by its view class.  The object x is available from the proxy
+    as its __obj__ attribute.
+
+    The view class selection is determined by the __view__ tuples and the
+    optional __viewkey__ method.  By default, the selected view class is the
+    most specific subclass of C whose __view__ mentions the class of x.
+    If no such subclass is found, the search proceeds with the parent
+    object classes.  For example, C(True) will first look for a subclass
+    of C with __view__ = (..., bool, ...) and only if it doesn't find any
+    look for one with __view__ = (..., int, ...), and then ..., object,...
+    If everything fails the class C itself is considered to be the default.
+
+    Alternatively, the view class selection can be driven by another aspect
+    of the object x, instead of the class of x, by overriding __viewkey__.
+    See last example at the end of this module.
+    """
+
+    _viewcache = {}
+    __view__ = ()
+
+    def __new__(rootclass, obj, *args, **kwds):
+        self = object.__new__(rootclass)
+        self.__obj__ = obj
+        self.__rootclass__ = rootclass
+        key = self.__viewkey__()
+        try:
+            self.__class__ = self._viewcache[key]
+        except KeyError:
+            self.__class__ = self._selectsubclass(key)
+        return self
+
+    def __getattr__(self, attr):
+        # attributes not found in the normal hierarchy rooted on View
+        # are looked up in the object's real class
+        return getattr(self.__obj__, attr)
+
+    def __viewkey__(self):
+        return self.__obj__.__class__
+
+    def __matchkey__(self, key, subclasses):
+        if inspect.isclass(key):
+            keys = inspect.getmro(key)
+        else:
+            keys = [key]
+        for key in keys:
+            result = [C for C in subclasses if key in C.__view__]
+            if result:
+                return result
+        return []
+
+    def _selectsubclass(self, key):
+        subclasses = list(enumsubclasses(self.__rootclass__))
+        for C in subclasses:
+            if not isinstance(C.__view__, tuple):
+                C.__view__ = (C.__view__,)
+        choices = self.__matchkey__(key, subclasses)
+        if not choices:
+            return self.__rootclass__
+        elif len(choices) == 1:
+            return choices[0]
+        else:
+            # combine the multiple choices
+            return type('?', tuple(choices), {})
+
+    def __repr__(self):
+        return '%s(%r)' % (self.__rootclass__.__name__, self.__obj__)
+
+
+def enumsubclasses(cls):
+    for subcls in cls.__subclasses__():
+        for subsubclass in enumsubclasses(subcls):
+            yield subsubclass
+    yield cls
+
+
+class Interpretable(View):
+    """A parse tree node with a few extra methods."""
+    explanation = None
+
+    def is_builtin(self, frame):
+        return False
+
+    def eval(self, frame):
+        # fall-back for unknown expression nodes
+        try:
+            expr = ast.Expression(self.__obj__)
+            expr.filename = ''
+            self.__obj__.filename = ''
+            co = pycodegen.ExpressionCodeGenerator(expr).getCode()
+            result = frame.eval(co)
+        except passthroughex:
+            raise
+        except:
+            raise Failure(self)
+        self.result = result
+        self.explanation = self.explanation or frame.repr(self.result)
+
+    def run(self, frame):
+        # fall-back for unknown statement nodes
+        try:
+            expr = ast.Module(None, ast.Stmt([self.__obj__]))
+            expr.filename = ''
+            co = pycodegen.ModuleCodeGenerator(expr).getCode()
+            frame.exec_(co)
+        except passthroughex:
+            raise
+        except:
+            raise Failure(self)
+
+    def nice_explanation(self):
+        return _format_explanation(self.explanation)
+
+
+class Name(Interpretable):
+    __view__ = ast.Name
+
+    def is_local(self, frame):
+        source = '%r in locals() is not globals()' % self.name
+        try:
+            return frame.is_true(frame.eval(source))
+        except passthroughex:
+            raise
+        except:
+            return False
+
+    def is_global(self, frame):
+        source = '%r in globals()' % self.name
+        try:
+            return frame.is_true(frame.eval(source))
+        except passthroughex:
+            raise
+        except:
+            return False
+
+    def is_builtin(self, frame):
+        source = '%r not in locals() and %r not in globals()' % (
+            self.name, self.name)
+        try:
+            return frame.is_true(frame.eval(source))
+        except passthroughex:
+            raise
+        except:
+            return False
+
+    def eval(self, frame):
+        super(Name, self).eval(frame)
+        if not self.is_local(frame):
+            self.explanation = self.name
+
+class Compare(Interpretable):
+    __view__ = ast.Compare
+
+    def eval(self, frame):
+        expr = Interpretable(self.expr)
+        expr.eval(frame)
+        for operation, expr2 in self.ops:
+            if hasattr(self, 'result'):
+                # shortcutting in chained expressions
+                if not frame.is_true(self.result):
+                    break
+            expr2 = Interpretable(expr2)
+            expr2.eval(frame)
+            self.explanation = "%s %s %s" % (
+                expr.explanation, operation, expr2.explanation)
+            source = "__exprinfo_left %s __exprinfo_right" % operation
+            try:
+                self.result = frame.eval(source,
+                                         __exprinfo_left=expr.result,
+                                         __exprinfo_right=expr2.result)
+            except passthroughex:
+                raise
+            except:
+                raise Failure(self)
+            expr = expr2
+
+class And(Interpretable):
+    __view__ = ast.And
+
+    def eval(self, frame):
+        explanations = []
+        for expr in self.nodes:
+            expr = Interpretable(expr)
+            expr.eval(frame)
+            explanations.append(expr.explanation)
+            self.result = expr.result
+            if not frame.is_true(expr.result):
+                break
+        self.explanation = '(' + ' and '.join(explanations) + ')'
+
+class Or(Interpretable):
+    __view__ = ast.Or
+
+    def eval(self, frame):
+        explanations = []
+        for expr in self.nodes:
+            expr = Interpretable(expr)
+            expr.eval(frame)
+            explanations.append(expr.explanation)
+            self.result = expr.result
+            if frame.is_true(expr.result):
+                break
+        self.explanation = '(' + ' or '.join(explanations) + ')'
+
+
+# == Unary operations ==
+keepalive = []
+for astclass, astpattern in {
+    ast.Not    : 'not __exprinfo_expr',
+    ast.Invert : '(~__exprinfo_expr)',
+    }.items():
+
+    class UnaryArith(Interpretable):
+        __view__ = astclass
+
+        def eval(self, frame, astpattern=astpattern):
+            expr = Interpretable(self.expr)
+            expr.eval(frame)
+            self.explanation = astpattern.replace('__exprinfo_expr',
+                                                  expr.explanation)
+            try:
+                self.result = frame.eval(astpattern,
+                                         __exprinfo_expr=expr.result)
+            except passthroughex:
+                raise
+            except:
+                raise Failure(self)
+
+    keepalive.append(UnaryArith)
+
+# == Binary operations ==
+for astclass, astpattern in {
+    ast.Add    : '(__exprinfo_left + __exprinfo_right)',
+    ast.Sub    : '(__exprinfo_left - __exprinfo_right)',
+    ast.Mul    : '(__exprinfo_left * __exprinfo_right)',
+    ast.Div    : '(__exprinfo_left / __exprinfo_right)',
+    ast.Mod    : '(__exprinfo_left % __exprinfo_right)',
+    ast.Power  : '(__exprinfo_left ** __exprinfo_right)',
+    }.items():
+
+    class BinaryArith(Interpretable):
+        __view__ = astclass
+
+        def eval(self, frame, astpattern=astpattern):
+            left = Interpretable(self.left)
+            left.eval(frame)
+            right = Interpretable(self.right)
+            right.eval(frame)
+            self.explanation = (astpattern
+                                .replace('__exprinfo_left',  left .explanation)
+                                .replace('__exprinfo_right', right.explanation))
+            try:
+                self.result = frame.eval(astpattern,
+                                         __exprinfo_left=left.result,
+                                         __exprinfo_right=right.result)
+            except passthroughex:
+                raise
+            except:
+                raise Failure(self)
+
+    keepalive.append(BinaryArith)
+
+
+class CallFunc(Interpretable):
+    __view__ = ast.CallFunc
+
+    def is_bool(self, frame):
+        source = 'isinstance(__exprinfo_value, bool)'
+        try:
+            return frame.is_true(frame.eval(source,
+                                            __exprinfo_value=self.result))
+        except passthroughex:
+            raise
+        except:
+            return False
+
+    def eval(self, frame):
+        node = Interpretable(self.node)
+        node.eval(frame)
+        explanations = []
+        vars = {'__exprinfo_fn': node.result}
+        source = '__exprinfo_fn('
+        for a in self.args:
+            if isinstance(a, ast.Keyword):
+                keyword = a.name
+                a = a.expr
+            else:
+                keyword = None
+            a = Interpretable(a)
+            a.eval(frame)
+            argname = '__exprinfo_%d' % len(vars)
+            vars[argname] = a.result
+            if keyword is None:
+                source += argname + ','
+                explanations.append(a.explanation)
+            else:
+                source += '%s=%s,' % (keyword, argname)
+                explanations.append('%s=%s' % (keyword, a.explanation))
+        if self.star_args:
+            star_args = Interpretable(self.star_args)
+            star_args.eval(frame)
+            argname = '__exprinfo_star'
+            vars[argname] = star_args.result
+            source += '*' + argname + ','
+            explanations.append('*' + star_args.explanation)
+        if self.dstar_args:
+            dstar_args = Interpretable(self.dstar_args)
+            dstar_args.eval(frame)
+            argname = '__exprinfo_kwds'
+            vars[argname] = dstar_args.result
+            source += '**' + argname + ','
+            explanations.append('**' + dstar_args.explanation)
+        self.explanation = "%s(%s)" % (
+            node.explanation, ', '.join(explanations))
+        if source.endswith(','):
+            source = source[:-1]
+        source += ')'
+        try:
+            self.result = frame.eval(source, **vars)
+        except passthroughex:
+            raise
+        except:
+            raise Failure(self)
+        if not node.is_builtin(frame) or not self.is_bool(frame):
+            r = frame.repr(self.result)
+            self.explanation = '%s\n{%s = %s\n}' % (r, r, self.explanation)
+
+class Getattr(Interpretable):
+    __view__ = ast.Getattr
+
+    def eval(self, frame):
+        expr = Interpretable(self.expr)
+        expr.eval(frame)
+        source = '__exprinfo_expr.%s' % self.attrname
+        try:
+            self.result = frame.eval(source, __exprinfo_expr=expr.result)
+        except passthroughex:
+            raise
+        except:
+            raise Failure(self)
+        self.explanation = '%s.%s' % (expr.explanation, self.attrname)
+        # if the attribute comes from the instance, its value is interesting
+        source = ('hasattr(__exprinfo_expr, "__dict__") and '
+                  '%r in __exprinfo_expr.__dict__' % self.attrname)
+        try:
+            from_instance = frame.is_true(
+                frame.eval(source, __exprinfo_expr=expr.result))
+        except passthroughex:
+            raise
+        except:
+            from_instance = True
+        if from_instance:
+            r = frame.repr(self.result)
+            self.explanation = '%s\n{%s = %s\n}' % (r, r, self.explanation)
+
+# == Re-interpretation of full statements ==
+
+class Assert(Interpretable):
+    __view__ = ast.Assert
+
+    def run(self, frame):
+        test = Interpretable(self.test)
+        test.eval(frame)
+        # simplify 'assert False where False = ...'
+        if (test.explanation.startswith('False\n{False = ') and
+            test.explanation.endswith('\n}')):
+            test.explanation = test.explanation[15:-2]
+        # print the result as  'assert '
+        self.result = test.result
+        self.explanation = 'assert ' + test.explanation
+        if not frame.is_true(test.result):
+            try:
+                raise BuiltinAssertionError
+            except passthroughex:
+                raise
+            except:
+                raise Failure(self)
+
+class Assign(Interpretable):
+    __view__ = ast.Assign
+
+    def run(self, frame):
+        expr = Interpretable(self.expr)
+        expr.eval(frame)
+        self.result = expr.result
+        self.explanation = '... = ' + expr.explanation
+        # fall-back-run the rest of the assignment
+        ass = ast.Assign(self.nodes, ast.Name('__exprinfo_expr'))
+        mod = ast.Module(None, ast.Stmt([ass]))
+        mod.filename = ''
+        co = pycodegen.ModuleCodeGenerator(mod).getCode()
+        try:
+            frame.exec_(co, __exprinfo_expr=expr.result)
+        except passthroughex:
+            raise
+        except:
+            raise Failure(self)
+
+class Discard(Interpretable):
+    __view__ = ast.Discard
+
+    def run(self, frame):
+        expr = Interpretable(self.expr)
+        expr.eval(frame)
+        self.result = expr.result
+        self.explanation = expr.explanation
+
+class Stmt(Interpretable):
+    __view__ = ast.Stmt
+
+    def run(self, frame):
+        for stmt in self.nodes:
+            stmt = Interpretable(stmt)
+            stmt.run(frame)
+
+
+def report_failure(e):
+    explanation = e.node.nice_explanation()
+    if explanation:
+        explanation = ", in: " + explanation
+    else:
+        explanation = ""
+    sys.stdout.write("%s: %s%s\n" % (e.exc.__name__, e.value, explanation))
+
+def check(s, frame=None):
+    if frame is None:
+        frame = sys._getframe(1)
+        frame = py.code.Frame(frame)
+    expr = parse(s, 'eval')
+    assert isinstance(expr, ast.Expression)
+    node = Interpretable(expr.node)
+    try:
+        node.eval(frame)
+    except passthroughex:
+        raise
+    except Failure:
+        e = sys.exc_info()[1]
+        report_failure(e)
+    else:
+        if not frame.is_true(node.result):
+            sys.stderr.write("assertion failed: %s\n" % node.nice_explanation())
+
+
+###########################################################
+# API / Entry points
+# #########################################################
+
+def interpret(source, frame, should_fail=False):
+    module = Interpretable(parse(source, 'exec').node)
+    #print "got module", module
+    if isinstance(frame, py.std.types.FrameType):
+        frame = py.code.Frame(frame)
+    try:
+        module.run(frame)
+    except Failure:
+        e = sys.exc_info()[1]
+        return getfailure(e)
+    except passthroughex:
+        raise
+    except:
+        import traceback
+        traceback.print_exc()
+    if should_fail:
+        return ("(assertion failed, but when it was re-run for "
+                "printing intermediate values, it did not fail.  Suggestions: "
+                "compute assert expression before the assert or use --nomagic)")
+    else:
+        return None
+
+def getmsg(excinfo):
+    if isinstance(excinfo, tuple):
+        excinfo = py.code.ExceptionInfo(excinfo)
+    #frame, line = gettbline(tb)
+    #frame = py.code.Frame(frame)
+    #return interpret(line, frame)
+
+    tb = excinfo.traceback[-1]
+    source = str(tb.statement).strip()
+    x = interpret(source, tb.frame, should_fail=True)
+    if not isinstance(x, str):
+        raise TypeError("interpret returned non-string %r" % (x,))
+    return x
+
+def getfailure(e):
+    explanation = e.node.nice_explanation()
+    if str(e.value):
+        lines = explanation.split('\n')
+        lines[0] += "  << %s" % (e.value,)
+        explanation = '\n'.join(lines)
+    text = "%s: %s" % (e.exc.__name__, explanation)
+    if text.startswith('AssertionError: assert '):
+        text = text[16:]
+    return text
+
+def run(s, frame=None):
+    if frame is None:
+        frame = sys._getframe(1)
+        frame = py.code.Frame(frame)
+    module = Interpretable(parse(s, 'exec').node)
+    try:
+        module.run(frame)
+    except Failure:
+        e = sys.exc_info()[1]
+        report_failure(e)
+
+
+if __name__ == '__main__':
+    # example:
+    def f():
+        return 5
+    def g():
+        return 3
+    def h(x):
+        return 'never'
+    check("f() * g() == 5")
+    check("not f()")
+    check("not (f() and g() or 0)")
+    check("f() == g()")
+    i = 4
+    check("i == f()")
+    check("len(f()) == 0")
+    check("isinstance(2+3+4, float)")
+
+    run("x = i")
+    check("x == 5")
+
+    run("assert not f(), 'oops'")
+    run("a, b, c = 1, 2")
+    run("a, b, c = f()")
+
+    check("max([f(),g()]) == 4")
+    check("'hello'[g()] == 'h'")
+    run("'guk%d' % h(f())")
diff --git a/py/_code/assertion.py b/py/_code/assertion.py
new file mode 100644
--- /dev/null
+++ b/py/_code/assertion.py
@@ -0,0 +1,94 @@
+import sys
+import py
+
+BuiltinAssertionError = py.builtin.builtins.AssertionError
+
+_reprcompare = None # if set, will be called by assert reinterp for comparison ops
+
+def _format_explanation(explanation):
+    """This formats an explanation
+
+    Normally all embedded newlines are escaped, however there are
+    three exceptions: \n{, \n} and \n~.  The first two are intended
+    cover nested explanations, see function and attribute explanations
+    for examples (.visit_Call(), visit_Attribute()).  The last one is
+    for when one explanation needs to span multiple lines, e.g. when
+    displaying diffs.
+    """
+    raw_lines = (explanation or '').split('\n')
+    # escape newlines not followed by {, } and ~
+    lines = [raw_lines[0]]
+    for l in raw_lines[1:]:
+        if l.startswith('{') or l.startswith('}') or l.startswith('~'):
+            lines.append(l)
+        else:
+            lines[-1] += '\\n' + l
+
+    result = lines[:1]
+    stack = [0]
+    stackcnt = [0]
+    for line in lines[1:]:
+        if line.startswith('{'):
+            if stackcnt[-1]:
+                s = 'and   '
+            else:
+                s = 'where '
+            stack.append(len(result))
+            stackcnt[-1] += 1
+            stackcnt.append(0)
+            result.append(' +' + '  '*(len(stack)-1) + s + line[1:])
+        elif line.startswith('}'):
+            assert line.startswith('}')
+            stack.pop()
+            stackcnt.pop()
+            result[stack[-1]] += line[1:]
+        else:
+            assert line.startswith('~')
+            result.append('  '*len(stack) + line[1:])
+    assert len(stack) == 1
+    return '\n'.join(result)
+
+
+class AssertionError(BuiltinAssertionError):
+    def __init__(self, *args):
+        BuiltinAssertionError.__init__(self, *args)
+        if args:
+            try:
+                self.msg = str(args[0])
+            except py.builtin._sysex:
+                raise
+            except:
+                self.msg = "<[broken __repr__] %s at %0xd>" %(
+                    args[0].__class__, id(args[0]))
+        else:
+            f = py.code.Frame(sys._getframe(1))
+            try:
+                source = f.code.fullsource
+                if source is not None:
+                    try:
+                        source = source.getstatement(f.lineno, assertion=True)
+                    except IndexError:
+                        source = None
+                    else:
+                        source = str(source.deindent()).strip()
+            except py.error.ENOENT:
+                source = None
+                # this can also occur during reinterpretation, when the
+                # co_filename is set to "".
+            if source:
+                self.msg = reinterpret(source, f, should_fail=True)
+            else:
+                self.msg = ""
+            if not self.args:
+                self.args = (self.msg,)
+
+if sys.version_info > (3, 0):
+    AssertionError.__module__ = "builtins"
+    reinterpret_old = "old reinterpretation not available for py3"
+else:
+    from py._code._assertionold import interpret as reinterpret_old
+if sys.version_info >= (2, 6) or (sys.platform.startswith("java")):
+    from py._code._assertionnew import interpret as reinterpret
+else:
+    reinterpret = reinterpret_old
+
diff --git a/py/_code/code.py b/py/_code/code.py
--- a/py/_code/code.py
+++ b/py/_code/code.py
@@ -145,6 +145,17 @@
         return self.frame.f_locals
     locals = property(getlocals, None, None, "locals of underlaying frame")
 
+    def reinterpret(self):
+        """Reinterpret the failing statement and returns a detailed information
+           about what operations are performed."""
+        if self.exprinfo is None:
+            source = str(self.statement).strip()
+            x = py.code._reinterpret(source, self.frame, should_fail=True)
+            if not isinstance(x, str):
+                raise TypeError("interpret returned non-string %r" % (x,))
+            self.exprinfo = x
+        return self.exprinfo
+
     def getfirstlinesource(self):
         # on Jython this firstlineno can be -1 apparently
         return max(self.frame.code.firstlineno, 0)
@@ -158,13 +169,12 @@
         end = self.lineno
         try:
             _, end = source.getstatementrange(end)
-        except IndexError:
+        except (IndexError, ValueError):
             end = self.lineno + 1
         # heuristic to stop displaying source on e.g.
         #   if something:  # assume this causes a NameError
         #      # _this_ lines and the one
                #        below we don't want from entry.getsource()
-        end = min(end, len(source))
         for i in range(self.lineno, end):
             if source[i].rstrip().endswith(':'):
                 end = i + 1
@@ -273,7 +283,11 @@
         """
         cache = {}
         for i, entry in enumerate(self):
-            key = entry.frame.code.path, entry.lineno
+            # id for the code.raw is needed to work around
+            # the strange metaprogramming in the decorator lib from pypi
+            # which generates code objects that have hash/value equality
+            #XXX needs a test
+            key = entry.frame.code.path, id(entry.frame.code.raw), entry.lineno
             #print "checking for recursion at", key
             l = cache.setdefault(key, [])
             if l:
@@ -308,7 +322,7 @@
                     self._striptext = 'AssertionError: '
         self._excinfo = tup
         self.type, self.value, tb = self._excinfo
-        self.typename = getattr(self.type, "__name__", "???")
+        self.typename = self.type.__name__
         self.traceback = py.code.Traceback(tb)
 
     def __repr__(self):
@@ -347,14 +361,16 @@
             showlocals: show locals per traceback entry
             style: long|short|no|native traceback style
             tbfilter: hide entries (where __tracebackhide__ is true)
+
+            in case of style==native, tbfilter and showlocals is ignored.
         """
         if style == 'native':
-            import traceback
-            return ''.join(traceback.format_exception(
-                self.type,
-                self.value,
-                self.traceback[0]._rawentry,
-                ))
+            return ReprExceptionInfo(ReprTracebackNative(
+                py.std.traceback.format_exception(
+                    self.type,
+                    self.value,
+                    self.traceback[0]._rawentry,
+                )), self._getreprcrash())
 
         fmt = FormattedExcinfo(showlocals=showlocals, style=style,
             abspath=abspath, tbfilter=tbfilter, funcargs=funcargs)
@@ -452,7 +468,7 @@
     def repr_locals(self, locals):
         if self.showlocals:
             lines = []
-            keys = list(locals)
+            keys = [loc for loc in locals if loc[0] != "@"]
             keys.sort()
             for name in keys:
                 value = locals[name]
@@ -506,7 +522,10 @@
 
     def _makepath(self, path):
         if not self.abspath:
-            np = py.path.local().bestrelpath(path)
+            try:
+                np = py.path.local().bestrelpath(path)
+            except OSError:
+                return path
             if len(np) < len(str(path)):
                 path = np
         return path
@@ -595,6 +614,19 @@
         if self.extraline:
             tw.line(self.extraline)
 
+class ReprTracebackNative(ReprTraceback):
+    def __init__(self, tblines):
+        self.style = "native"
+        self.reprentries = [ReprEntryNative(tblines)]
+        self.extraline = None
+
+class ReprEntryNative(TerminalRepr):
+    def __init__(self, tblines):
+        self.lines = tblines
+
+    def toterminal(self, tw):
+        tw.write("".join(self.lines))
+
 class ReprEntry(TerminalRepr):
     localssep = "_ "
 
@@ -680,19 +712,26 @@
 
 oldbuiltins = {}
 
-def patch_builtins(compile=True):
-    """ put compile builtins to Python's builtins. """
+def patch_builtins(assertion=True, compile=True):
+    """ put compile and AssertionError builtins to Python's builtins. """
+    if assertion:
+        from py._code import assertion
+        l = oldbuiltins.setdefault('AssertionError', [])
+        l.append(py.builtin.builtins.AssertionError)
+        py.builtin.builtins.AssertionError = assertion.AssertionError
     if compile:
         l = oldbuiltins.setdefault('compile', [])
         l.append(py.builtin.builtins.compile)
         py.builtin.builtins.compile = py.code.compile
 
-def unpatch_builtins(compile=True):
+def unpatch_builtins(assertion=True, compile=True):
     """ remove compile and AssertionError builtins from Python builtins. """
+    if assertion:
+        py.builtin.builtins.AssertionError = oldbuiltins['AssertionError'].pop()
     if compile:
         py.builtin.builtins.compile = oldbuiltins['compile'].pop()
 
-def getrawcode(obj):
+def getrawcode(obj, trycall=True):
     """ return code object for given function. """
     try:
         return obj.__code__
@@ -701,5 +740,10 @@
         obj = getattr(obj, 'func_code', obj)
         obj = getattr(obj, 'f_code', obj)
         obj = getattr(obj, '__code__', obj)
+        if trycall and not hasattr(obj, 'co_firstlineno'):
+            if hasattr(obj, '__call__') and not py.std.inspect.isclass(obj):
+                x = getrawcode(obj.__call__, trycall=False)
+                if hasattr(x, 'co_firstlineno'):
+                    return x
         return obj
 
diff --git a/py/_code/source.py b/py/_code/source.py
--- a/py/_code/source.py
+++ b/py/_code/source.py
@@ -108,6 +108,7 @@
     def getstatementrange(self, lineno, assertion=False):
         """ return (start, end) tuple which spans the minimal
             statement region which containing the given lineno.
+            raise an IndexError if no such statementrange can be found.
         """
         # XXX there must be a better than these heuristic ways ...
         # XXX there may even be better heuristics :-)
@@ -116,6 +117,7 @@
 
         # 1. find the start of the statement
         from codeop import compile_command
+        end = None
         for start in range(lineno, -1, -1):
             if assertion:
                 line = self.lines[start]
@@ -139,7 +141,9 @@
                 trysource = self[start:end]
                 if trysource.isparseable():
                     return start, end
-        return start, len(self)
+        if end is None:
+            raise IndexError("no valid source range around line %d " % (lineno,))
+        return start, end
 
     def getblockend(self, lineno):
         # XXX
@@ -257,23 +261,29 @@
 
 
 def getfslineno(obj):
+    """ Return source location (path, lineno) for the given object.
+    If the source cannot be determined return ("", -1)
+    """
     try:
         code = py.code.Code(obj)
     except TypeError:
-        # fallback to
-        fn = (py.std.inspect.getsourcefile(obj) or
-              py.std.inspect.getfile(obj))
+        try:
+            fn = (py.std.inspect.getsourcefile(obj) or
+                  py.std.inspect.getfile(obj))
+        except TypeError:
+            return "", -1
+
         fspath = fn and py.path.local(fn) or None
+        lineno = -1
         if fspath:
             try:
                 _, lineno = findsource(obj)
             except IOError:
-                lineno = None
-        else:
-            lineno = None
+                pass
     else:
         fspath = code.path
         lineno = code.firstlineno
+    assert isinstance(lineno, int)
     return fspath, lineno
 
 #
@@ -286,7 +296,7 @@
     except py.builtin._sysex:
         raise
     except:
-        return None, None
+        return None, -1
     source = Source()
     source.lines = [line.rstrip() for line in sourcelines]
     return source, lineno
diff --git a/py/_iniconfig.py b/py/_iniconfig.py
--- a/py/_iniconfig.py
+++ b/py/_iniconfig.py
@@ -103,6 +103,7 @@
     def _parseline(self, line, lineno):
         # comments
         line = line.split('#')[0].rstrip()
+        line = line.split(';')[0].rstrip()
         # blank lines
         if not line:
             return None, None
diff --git a/py/_io/capture.py b/py/_io/capture.py
--- a/py/_io/capture.py
+++ b/py/_io/capture.py
@@ -258,6 +258,9 @@
                 f = getattr(self, name).tmpfile
                 f.seek(0)
                 res = f.read()
+                enc = getattr(f, 'encoding', None)
+                if enc:
+                    res = py.builtin._totext(res, enc)
                 f.truncate(0)
                 f.seek(0)
             l.append(res)
diff --git a/py/_io/terminalwriter.py b/py/_io/terminalwriter.py
--- a/py/_io/terminalwriter.py
+++ b/py/_io/terminalwriter.py
@@ -105,6 +105,8 @@
                      Blue=44, Purple=45, Cyan=46, White=47,
                      bold=1, light=2, blink=5, invert=7)
 
+    _newline = None   # the last line printed
+
     # XXX deprecate stringio argument
     def __init__(self, file=None, stringio=False, encoding=None):
         if file is None:
@@ -112,11 +114,9 @@
                 self.stringio = file = py.io.TextIO()
             else:
                 file = py.std.sys.stdout
-                if hasattr(file, 'encoding'):
-                    encoding = file.encoding
         elif hasattr(file, '__call__'):
             file = WriteFile(file, encoding=encoding)
-        self.encoding = encoding
+        self.encoding = encoding or getattr(file, 'encoding', "utf-8")
         self._file = file
         self.fullwidth = get_terminal_width()
         self.hasmarkup = should_do_markup(file)
@@ -182,8 +182,24 @@
         return s
 
     def line(self, s='', **kw):
+        if self._newline == False:
+            self.write("\n")
         self.write(s, **kw)
         self.write('\n')
+        self._newline = True
+
+    def reline(self, line, **opts):
+        if not self.hasmarkup:
+            raise ValueError("cannot use rewrite-line without terminal")
+        if not self._newline:
+            self.write("\r")
+        self.write(line, **opts)
+        lastlen = getattr(self, '_lastlinelen', None)
+        self._lastlinelen = lenlastline = len(line)
+        if lenlastline < lastlen:
+            self.write(" " * (lastlen - lenlastline + 1))
+        self._newline = False
+
 
 class Win32ConsoleWriter(TerminalWriter):
     def write(self, s, **kw):
diff --git a/py/_path/common.py b/py/_path/common.py
--- a/py/_path/common.py
+++ b/py/_path/common.py
@@ -368,6 +368,5 @@
         else:
             name = str(path) # path.strpath # XXX svn?
             pattern = '*' + path.sep + pattern
-        from fnmatch import fnmatch
-        return fnmatch(name, pattern)
+        return py.std.fnmatch.fnmatch(name, pattern)
 
diff --git a/py/_path/local.py b/py/_path/local.py
--- a/py/_path/local.py
+++ b/py/_path/local.py
@@ -157,14 +157,16 @@
         return str(self) < str(other)
 
     def samefile(self, other):
-        """ return True if 'other' references the same file as 'self'. """
-        if not iswin32:
-            return py.error.checked_call(
-                    os.path.samefile, str(self), str(other))
+        """ return True if 'other' references the same file as 'self'.
+        """
+        if not isinstance(other, py.path.local):
+            other = os.path.abspath(str(other))
         if self == other:
             return True
-        other = os.path.abspath(str(other))
-        return self == other
+        if iswin32:
+            return False # ther is no samefile
+        return py.error.checked_call(
+                os.path.samefile, str(self), str(other))
 
     def remove(self, rec=1, ignore_errors=False):
         """ remove a file or directory (or a directory tree if rec=1).
@@ -539,7 +541,11 @@
                 if self.basename != "__init__.py":
                     modfile = modfile[:-12]
 
-            if not self.samefile(modfile):
+            try:
+                issame = self.samefile(modfile)
+            except py.error.ENOENT:
+                issame = False
+            if not issame:
                 raise self.ImportMismatchError(modname, modfile, self)
             return mod
         else:
diff --git a/py/_path/svnurl.py b/py/_path/svnurl.py
--- a/py/_path/svnurl.py
+++ b/py/_path/svnurl.py
@@ -233,6 +233,8 @@
                 e = sys.exc_info()[1]
                 if e.err.find('non-existent in that revision') != -1:
                     raise py.error.ENOENT(self, e.err)
+                elif e.err.find("E200009:") != -1:
+                    raise py.error.ENOENT(self, e.err)
                 elif e.err.find('File not found') != -1:
                     raise py.error.ENOENT(self, e.err)
                 elif e.err.find('not part of a repository')!=-1:
diff --git a/py/_path/svnwc.py b/py/_path/svnwc.py
--- a/py/_path/svnwc.py
+++ b/py/_path/svnwc.py
@@ -482,10 +482,13 @@
         except py.process.cmdexec.Error:
             e = sys.exc_info()[1]
             strerr = e.err.lower()
-            if strerr.find('file not found') != -1:
+            if strerr.find('not found') != -1:
+                raise py.error.ENOENT(self)
+            elif strerr.find("E200009:") != -1:
                 raise py.error.ENOENT(self)
             if (strerr.find('file exists') != -1 or
                 strerr.find('file already exists') != -1 or
+                strerr.find('w150002:') != -1 or
                 strerr.find("can't create directory") != -1):
                 raise py.error.EEXIST(self)
             raise
@@ -593,7 +596,7 @@
         out = self._authsvn('lock').strip()
         if not out:
             # warning or error, raise exception
-            raise Exception(out[4:])
+            raise ValueError("unknown error in svn lock command")
 
     def unlock(self):
         """ unset a previously set lock """
@@ -1066,6 +1069,8 @@
                 modrev = '?'
                 author = '?'
                 date = ''
+            elif itemstatus == "replaced":
+                pass
             else:
                 #print entryel.toxml()
                 commitel = entryel.getElementsByTagName('commit')[0]
@@ -1148,7 +1153,11 @@
             raise  ValueError("Not a versioned resource")
             #raise ValueError, "Not a versioned resource %r" % path
         self.kind = d['nodekind'] == 'directory' and 'dir' or d['nodekind']
-        self.rev = int(d['revision'])
+        try:
+            self.rev = int(d['revision'])
+        except KeyError:
+            self.rev = None
+
         self.path = py.path.local(d['path'])
         self.size = self.path.size()
         if 'lastchangedrev' in d:
diff --git a/py/_xmlgen.py b/py/_xmlgen.py
--- a/py/_xmlgen.py
+++ b/py/_xmlgen.py
@@ -136,7 +136,8 @@
     def list(self, obj):
         assert id(obj) not in self.visited
         self.visited[id(obj)] = 1
-        map(self.visit, obj)
+        for elem in obj:
+            self.visit(elem)
 
     def Tag(self, tag):
         assert id(tag) not in self.visited
diff --git a/py/bin/_findpy.py b/py/bin/_findpy.py
deleted file mode 100644
--- a/py/bin/_findpy.py
+++ /dev/null
@@ -1,38 +0,0 @@
-#!/usr/bin/env python
-
-#
-# find and import a version of 'py'
-#
-import sys
-import os
-from os.path import dirname as opd, exists, join, basename, abspath
-
-def searchpy(current):
-    while 1:
-        last = current
-        initpy = join(current, '__init__.py')
-        if not exists(initpy):
-            pydir = join(current, 'py')
-            # recognize py-package and ensure it is importable
-            if exists(pydir) and exists(join(pydir, '__init__.py')):
-                #for p in sys.path:
-                #    if p == current:
-                #        return True
-                if current != sys.path[0]:  # if we are already first, then ok
-                    sys.stderr.write("inserting into sys.path: %s\n" % current)
-                    sys.path.insert(0, current)
-                return True
-        current = opd(current)
-        if last == current:
-            return False
-
-if not searchpy(abspath(os.curdir)):
-    if not searchpy(opd(abspath(sys.argv[0]))):
-        if not searchpy(opd(__file__)):
-            pass # let's hope it is just on sys.path
-
-import py
-import pytest
-
-if __name__ == '__main__':
-    print ("py lib is at %s" % py.__file__)
diff --git a/py/bin/py.test b/py/bin/py.test
deleted file mode 100755
--- a/py/bin/py.test
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/usr/bin/env python
-from _findpy import pytest
-raise SystemExit(pytest.main())

From pullrequests-noreply at bitbucket.org  Sat Jan 21 17:05:48 2012
From: pullrequests-noreply at bitbucket.org (Bitbucket)
Date: Sat, 21 Jan 2012 16:05:48 -0000
Subject: [pypy-commit] [OPEN] Pull request #21 for pypy/pypy: datetime.py
 fix for issue972, with unit test
In-Reply-To: 
References: 
Message-ID: <20120121160548.14690.86236@bitbucket03.managed.contegix.com>

Pull request #21 has been updated by David Ripton to include new changes.

https://bitbucket.org/pypy/pypy/pull-request/21/datetimepy-fix-for-issue972-with-unit-test

Title: datetime.py fix for issue972, with unit test

Creator: David Ripton

Copying utcfromtimestamp from the CPython3.2 version of datetime.py seems to fix the roundoff errors in time zone calculations.

Updated list of changes:

07e667cb75a4 by David Ripton: "Add a test to prove that datetime issue972 (and dupe issue986) are fixed."
235d8b8434a8 by David Ripton: "Copy function utcfromtimestamp from CPython 3.2's datetime.py

Fixes issue972, w?"


--
This is an issue notification from bitbucket.org.
You are receiving this either because you are the participating
in a pull request, or you are following it.


From pullrequests-noreply at bitbucket.org  Sat Jan 21 17:16:09 2012
From: pullrequests-noreply at bitbucket.org (Bitbucket)
Date: Sat, 21 Jan 2012 16:16:09 -0000
Subject: [pypy-commit] [OPEN] Pull request #21 for pypy/pypy: datetime.py
 fix for issue972, with unit test
In-Reply-To: 
References: 
Message-ID: <20120121161609.29507.44926@bitbucket01.managed.contegix.com>

Pull request #21 has been updated by David Ripton to include new changes.

https://bitbucket.org/pypy/pypy/pull-request/21/datetimepy-fix-for-issue972-with-unit-test

Title: datetime.py fix for issue972, with unit test

Creator: David Ripton

Copying utcfromtimestamp from the CPython3.2 version of datetime.py seems to fix the roundoff errors in time zone calculations.

Updated list of changes:

eeedeffe6525 by David Ripton: "Clean up os.environ["TZ"] when we're done, in case other tests follow."
07e667cb75a4 by David Ripton: "Add a test to prove that datetime issue972 (and dupe issue986) are fixed."
235d8b8434a8 by David Ripton: "Copy function utcfromtimestamp from CPython 3.2's datetime.py

Fixes issue972, w?"


--
This is an issue notification from bitbucket.org.
You are receiving this either because you are the participating
in a pull request, or you are following it.


From noreply at buildbot.pypy.org  Sat Jan 21 17:26:08 2012
From: noreply at buildbot.pypy.org (dripton)
Date: Sat, 21 Jan 2012 17:26:08 +0100 (CET)
Subject: [pypy-commit] pypy default: Copy function utcfromtimestamp from
 CPython 3.2's datetime.py
Message-ID: <20120121162608.C116C82D45@wyvern.cs.uni-duesseldorf.de>

Author: David Ripton 
Branch: 
Changeset: r51584:235d8b8434a8
Date: 2012-01-18 20:07 -0500
http://bitbucket.org/pypy/pypy/changeset/235d8b8434a8/

Log:	Copy function utcfromtimestamp from CPython 3.2's datetime.py

	Fixes issue972, which was caused by rounding differences between
	fromtimestamp and utcfromtimestamp.

diff --git a/lib_pypy/datetime.py b/lib_pypy/datetime.py
--- a/lib_pypy/datetime.py
+++ b/lib_pypy/datetime.py
@@ -1440,17 +1440,22 @@
         return result
     fromtimestamp = classmethod(fromtimestamp)
 
+    @classmethod
     def utcfromtimestamp(cls, t):
         "Construct a UTC datetime from a POSIX timestamp (like time.time())."
-        if 1 - (t % 1.0) < 0.0000005:
-            t = float(int(t)) + 1
-        if t < 0:
-            t -= 1
+        t, frac = divmod(t, 1.0)
+        us = round(frac * 1e6)
+
+        # If timestamp is less than one microsecond smaller than a
+        # full second, us can be rounded up to 1000000.  In this case,
+        # roll over to seconds, otherwise, ValueError is raised
+        # by the constructor.
+        if us == 1000000:
+            t += 1
+            us = 0
         y, m, d, hh, mm, ss, weekday, jday, dst = _time.gmtime(t)
-        us = int((t % 1.0) * 1000000)
         ss = min(ss, 59)    # clamp out leap seconds if the platform has them
         return cls(y, m, d, hh, mm, ss, us)
-    utcfromtimestamp = classmethod(utcfromtimestamp)
 
     # XXX This is supposed to do better than we *can* do by using time.time(),
     # XXX if the platform supports a more accurate way.  The C implementation

From noreply at buildbot.pypy.org  Sat Jan 21 17:26:10 2012
From: noreply at buildbot.pypy.org (dripton)
Date: Sat, 21 Jan 2012 17:26:10 +0100 (CET)
Subject: [pypy-commit] pypy default: Add a test to prove that datetime
 issue972 (and dupe issue986) are fixed.
Message-ID: <20120121162610.00FCD82D45@wyvern.cs.uni-duesseldorf.de>

Author: David Ripton 
Branch: 
Changeset: r51585:07e667cb75a4
Date: 2012-01-21 11:01 -0500
http://bitbucket.org/pypy/pypy/changeset/07e667cb75a4/

Log:	Add a test to prove that datetime issue972 (and dupe issue986) are
	fixed.

diff --git a/pypy/module/test_lib_pypy/test_datetime.py b/pypy/module/test_lib_pypy/test_datetime.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/test_lib_pypy/test_datetime.py
@@ -0,0 +1,17 @@
+"""Additional tests for datetime."""
+
+import time
+import datetime
+import os
+
+def test_utcfromtimestamp():
+    """Confirm that utcfromtimestamp and fromtimestamp give consistent results.
+
+    Based on danchr's test script in https://bugs.pypy.org/issue986
+    """
+    os.putenv("TZ", "GMT")
+    for unused in xrange(100):
+        now = time.time()
+        delta = (datetime.datetime.utcfromtimestamp(now) -
+                 datetime.datetime.fromtimestamp(now))
+        assert delta.days * 86400 + delta.seconds == 0

From pullrequests-noreply at bitbucket.org  Sat Jan 21 17:26:10 2012
From: pullrequests-noreply at bitbucket.org (Bitbucket)
Date: Sat, 21 Jan 2012 16:26:10 -0000
Subject: [pypy-commit] [ACCEPTED] Pull request #21 for pypy/pypy:
 datetime.py fix for issue972, with unit test
In-Reply-To: 
References: 
Message-ID: <20120121162610.23241.5787@bitbucket05.managed.contegix.com>

Pull request #21 has been accepted by Maciej Fijalkowski.

Changes in dripton/pypy have been pulled into pypy/pypy.

https://bitbucket.org/pypy/pypy/pull-request/21/datetimepy-fix-for-issue972-with-unit-test
--
This is an issue notification from bitbucket.org.
You are receiving this either because you are the participating
in a pull request, or you are following it.


From noreply at buildbot.pypy.org  Sat Jan 21 17:26:11 2012
From: noreply at buildbot.pypy.org (dripton)
Date: Sat, 21 Jan 2012 17:26:11 +0100 (CET)
Subject: [pypy-commit] pypy default: Clean up os.environ["TZ"] when we're
 done, in case other tests follow.
Message-ID: <20120121162611.3064682D45@wyvern.cs.uni-duesseldorf.de>

Author: David Ripton 
Branch: 
Changeset: r51586:eeedeffe6525
Date: 2012-01-21 11:13 -0500
http://bitbucket.org/pypy/pypy/changeset/eeedeffe6525/

Log:	Clean up os.environ["TZ"] when we're done, in case other tests
	follow.

diff --git a/pypy/module/test_lib_pypy/test_datetime.py b/pypy/module/test_lib_pypy/test_datetime.py
--- a/pypy/module/test_lib_pypy/test_datetime.py
+++ b/pypy/module/test_lib_pypy/test_datetime.py
@@ -9,9 +9,16 @@
 
     Based on danchr's test script in https://bugs.pypy.org/issue986
     """
-    os.putenv("TZ", "GMT")
-    for unused in xrange(100):
-        now = time.time()
-        delta = (datetime.datetime.utcfromtimestamp(now) -
-                 datetime.datetime.fromtimestamp(now))
-        assert delta.days * 86400 + delta.seconds == 0
+    try:
+        prev_tz = os.environ.get("TZ")
+        os.environ["TZ"] = "GMT"
+        for unused in xrange(100):
+            now = time.time()
+            delta = (datetime.datetime.utcfromtimestamp(now) -
+                     datetime.datetime.fromtimestamp(now))
+            assert delta.days * 86400 + delta.seconds == 0
+    finally:
+        if prev_tz is None:
+            del os.environ["TZ"]
+        else:
+            os.environ["TZ"] = prev_tz

From noreply at buildbot.pypy.org  Sat Jan 21 17:47:19 2012
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Sat, 21 Jan 2012 17:47:19 +0100 (CET)
Subject: [pypy-commit] pypy pytest: resuffle the code in pypy/test_all.py,
 so a ./pytest.py run will not confuse at collection time
Message-ID: <20120121164719.CB4C882D45@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt 
Branch: pytest
Changeset: r51588:8237f969655e
Date: 2012-01-21 17:23 +0100
http://bitbucket.org/pypy/pypy/changeset/8237f969655e/

Log:	resuffle the code in pypy/test_all.py, so a ./pytest.py run will not
	confuse at collection time

diff --git a/pypy/test_all.py b/pypy/test_all.py
--- a/pypy/test_all.py
+++ b/pypy/test_all.py
@@ -11,11 +11,12 @@
 """
 import sys, os
 
-if len(sys.argv) == 1 and os.path.dirname(sys.argv[0]) in '.':
-    print >> sys.stderr, __doc__
-    sys.exit(2)
 
 if __name__ == '__main__':
+    if len(sys.argv) == 1 and os.path.dirname(sys.argv[0]) in '.':
+        print >> sys.stderr, __doc__
+        sys.exit(2)
+
     import tool.autopath
     import pytest
     import pytest_cov

From noreply at buildbot.pypy.org  Sat Jan 21 17:47:21 2012
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Sat, 21 Jan 2012 17:47:21 +0100 (CET)
Subject: [pypy-commit] pypy pytest: switch pytest.ini over new option name
Message-ID: <20120121164721.0D38E82D45@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt 
Branch: pytest
Changeset: r51589:e17af4cf9825
Date: 2012-01-21 17:33 +0100
http://bitbucket.org/pypy/pypy/changeset/e17af4cf9825/

Log:	switch pytest.ini over new option name

diff --git a/pypy/pytest.ini b/pypy/pytest.ini
--- a/pypy/pytest.ini
+++ b/pypy/pytest.ini
@@ -1,2 +1,2 @@
 [pytest]
-addopts = --assertmode=old -rf
+addopts = --assert=plain -rf

From noreply at buildbot.pypy.org  Sat Jan 21 17:47:22 2012
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Sat, 21 Jan 2012 17:47:22 +0100 (CET)
Subject: [pypy-commit] pypy pytest: add 2 missing __init__.py files so those
 test files can be discovered correctly
Message-ID: <20120121164722.3FE2082D45@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt 
Branch: pytest
Changeset: r51590:5a86baac8343
Date: 2012-01-21 17:39 +0100
http://bitbucket.org/pypy/pypy/changeset/5a86baac8343/

Log:	add 2 missing __init__.py files so those test files can be
	discovered correctly

diff --git a/pypy/module/pyexpat/test/__init__.py b/pypy/module/pyexpat/test/__init__.py
new file mode 100644
diff --git a/pypy/tool/jitlogparser/test/__init__.py b/pypy/tool/jitlogparser/test/__init__.py
new file mode 100644

From noreply at buildbot.pypy.org  Sat Jan 21 17:56:34 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 21 Jan 2012 17:56:34 +0100 (CET)
Subject: [pypy-commit] pypy stm: Adding a libitm wrapper.  Not used so far,
 and crashes anyway
Message-ID: <20120121165634.031A882D45@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stm
Changeset: r51591:bee9a6779dd0
Date: 2012-01-21 17:56 +0100
http://bitbucket.org/pypy/pypy/changeset/bee9a6779dd0/

Log:	Adding a libitm wrapper. Not used so far, and crashes anyway
	because libitm doesn't expect _ITM_RU?() calls when not in a
	transaction at all.

diff --git a/pypy/translator/stm/using_libitm/stm2itm.h b/pypy/translator/stm/using_libitm/stm2itm.h
new file mode 100644
--- /dev/null
+++ b/pypy/translator/stm/using_libitm/stm2itm.h
@@ -0,0 +1,74 @@
+#include 
+#include 
+#include 
+
+
+static void stm_descriptor_init(void) { /* nothing */ }
+static void stm_descriptor_done(void) { /* nothing */ }
+
+static void* stm_perform_transaction(void*(*f)(void*), void* arg)
+{
+    void *result;
+    int _i = _ITM_beginTransaction(pr_instrumentedCode);
+    assert(_i & a_runInstrumentedCode);
+    /**/
+    result = f(arg);
+    /**/
+    _ITM_commitTransaction();
+    return result;
+}
+
+#define STM_CCHARP1(arg)    void
+#define STM_EXPLAIN1(info)  /* nothing */
+
+static void stm_try_inevitable(STM_CCHARP1(why))
+{
+    _ITM_changeTransactionMode(modeSerialIrrevocable);
+}
+
+static void stm_abort_and_retry(void)
+{
+    abort();   /* XXX */
+}
+
+static long stm_debug_get_state(void)
+{
+    return _ITM_inTransaction();
+}
+
+
+#if PYPY_LONG_BIT == 32
+#  define stm_read_word(addr)        _ITM_RU4(addr)
+#  define stm_write_word(addr, val)  _ITM_WU4(addr, val)
+#else
+#  define stm_read_word(addr)        _ITM_RU8(addr)
+#  define stm_write_word(addr, val)  _ITM_WU8(addr, val)
+#endif
+
+// XXX little-endian only!
+/* this macro is used if 'base' is a word-aligned pointer and 'offset'
+   is a compile-time constant */
+#define stm_fx_read_partial(base, offset)                               \
+       (stm_read_word(                                                  \
+           (long*)(((char*)(base)) + ((offset) & ~(sizeof(void*)-1))))  \
+        >> (8 * ((offset) & (sizeof(void*)-1))))
+
+#define stm_read_partial_1(addr)          _ITM_RU1(addr)
+#define stm_read_partial_2(addr)          _ITM_RU2(addr)
+#define stm_write_partial_1(addr, nval)   _ITM_WU1(addr, nval)
+#define stm_write_partial_2(addr, nval)   _ITM_WU2(addr, nval)
+#if PYPY_LONG_BIT == 64
+#define stm_read_partial_4(addr)          _ITM_RU4(addr)
+#define stm_write_partial_4(addr, nval)   _ITM_WU4(addr, nval)
+#endif
+
+#define stm_read_double(addr)             _ITM_RD(addr)
+#define stm_write_double(addr, val)       _ITM_WD(addr, val)
+
+#define stm_read_float(addr)              _ITM_RF(addr)
+#define stm_write_float(addr, val)        _ITM_WF(addr, val)
+
+#if PYPY_LONG_BIT == 32
+#define stm_read_doubleword(addr)         _ITM_RU8(addr)
+#define stm_write_doubleword(addr, val)   _ITM_WU8(addr, val)
+#endif

From noreply at buildbot.pypy.org  Sat Jan 21 18:02:28 2012
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sat, 21 Jan 2012 18:02:28 +0100 (CET)
Subject: [pypy-commit] pypy default: Implement numpypy.ndarray.flatten
Message-ID: <20120121170228.C277C82D45@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor 
Branch: 
Changeset: r51592:6eb004a15a12
Date: 2012-01-21 11:02 -0600
http://bitbucket.org/pypy/pypy/changeset/6eb004a15a12/

Log:	Implement numpypy.ndarray.flatten

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -430,9 +430,15 @@
     def descr_copy(self, space):
         return self.copy(space)
 
+    def descr_flatten(self, space):
+        return self.flatten(space)
+
     def copy(self, space):
         return self.get_concrete().copy(space)
 
+    def flatten(self, space):
+        return self.get_concrete().flatten(space)
+
     def descr_len(self, space):
         if len(self.shape):
             return space.wrap(self.shape[0])
@@ -769,6 +775,11 @@
     def copy(self, space):
         return Scalar(self.dtype, self.value)
 
+    def flatten(self, space):
+        array = W_NDimArray(self.size, [self.size], self.dtype)
+        array.setitem(0, self.value)
+        return array
+
     def fill(self, space, w_value):
         self.value = self.dtype.coerce(space, w_value)
 
@@ -1150,6 +1161,15 @@
         array.setslice(space, self)
         return array
 
+    def flatten(self, space):
+        array = W_NDimArray(self.size, [self.size], self.dtype, self.order)
+        if self.supports_fast_slicing():
+            array._fast_setslice(space, self)
+        else:
+            arr = SliceArray(array.shape, array.dtype, array, self)
+            array._sliceloop(arr)
+        return array
+
     def fill(self, space, w_value):
         self.setslice(space, scalar_w(space, self.dtype, w_value))
 
@@ -1379,6 +1399,7 @@
     fill = interp2app(BaseArray.descr_fill),
 
     copy = interp2app(BaseArray.descr_copy),
+    flatten = interp2app(BaseArray.descr_flatten),
     reshape = interp2app(BaseArray.descr_reshape),
     tolist = interp2app(BaseArray.descr_tolist),
 )
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1039,6 +1039,20 @@
         a = array([5.0])
         assert a.std() == 0.0
 
+    def test_flatten(self):
+        from _numpypy import array
+
+        a = array([[1, 2, 3], [4, 5, 6]])
+        assert (a.flatten() == [1, 2, 3, 4, 5, 6]).all()
+        a = array([[[1, 2], [3, 4]], [[5, 6], [7, 8]]])
+        assert (a.flatten() == [1, 2, 3, 4, 5, 6, 7, 8]).all()
+        a = array([1, 2, 3, 4, 5, 6, 7, 8])
+        assert (a[::2].flatten() == [1, 3, 5, 7]).all()
+        a = array([1, 2, 3])
+        assert ((a + a).flatten() == [2, 4, 6]).all()
+        a = array(2)
+        assert (a.flatten() == [2]).all()
+
 
 class AppTestMultiDim(BaseNumpyAppTest):
     def test_init(self):

From noreply at buildbot.pypy.org  Sat Jan 21 18:13:26 2012
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sat, 21 Jan 2012 18:13:26 +0100 (CET)
Subject: [pypy-commit] pypy default: A failing test for flatten
Message-ID: <20120121171326.C88B782D45@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor 
Branch: 
Changeset: r51593:637b4cfa4f65
Date: 2012-01-21 11:13 -0600
http://bitbucket.org/pypy/pypy/changeset/637b4cfa4f65/

Log:	A failing test for flatten

diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1052,6 +1052,8 @@
         assert ((a + a).flatten() == [2, 4, 6]).all()
         a = array(2)
         assert (a.flatten() == [2]).all()
+        a = array([[1, 2], [3, 4]])
+        assert (a.T.flatten() == [1, 3, 2, 4]).all()
 
 
 class AppTestMultiDim(BaseNumpyAppTest):

From noreply at buildbot.pypy.org  Sat Jan 21 18:28:42 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 21 Jan 2012 18:28:42 +0100 (CET)
Subject: [pypy-commit] pypy default: Special-case for "complex ** 2" too.
Message-ID: <20120121172842.A6DEE82D45@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r51594:76e23aa1ef63
Date: 2012-01-21 18:28 +0100
http://bitbucket.org/pypy/pypy/changeset/76e23aa1ef63/

Log:	Special-case for "complex ** 2" too.

diff --git a/pypy/objspace/std/complexobject.py b/pypy/objspace/std/complexobject.py
--- a/pypy/objspace/std/complexobject.py
+++ b/pypy/objspace/std/complexobject.py
@@ -8,6 +8,7 @@
 from pypy.rlib.rbigint import rbigint
 from pypy.rlib.rfloat import (
     formatd, DTSF_STR_PRECISION, isinf, isnan, copysign)
+from pypy.rlib import jit
 
 import math
 
@@ -129,10 +130,10 @@
             ir = len * math.sin(phase)
         return W_ComplexObject(rr, ir)
 
-    def pow_int(self, n):
-        if n > 100 or n < -100:
-            return self.pow(W_ComplexObject(1.0 * n, 0.0))
-        elif n > 0:
+    def pow_small_int(self, n):
+        if n >= 0:
+            if jit.isconstant(n) and n == 2:
+                return self.mul(self)
             return self.pow_positive_int(n)
         else:
             return w_one.div(self.pow_positive_int(-n))
@@ -217,10 +218,10 @@
 def pow__Complex_Complex_ANY(space, w_complex, w_exponent, thirdArg):
     if not space.is_w(thirdArg, space.w_None):
         raise OperationError(space.w_ValueError, space.wrap('complex modulo'))
-    int_exponent = int(w_exponent.realval)
     try:
-        if w_exponent.imagval == 0.0 and w_exponent.realval == int_exponent:
-            w_p = w_complex.pow_int(int_exponent)
+        r = w_exponent.realval
+        if w_exponent.imagval == 0.0 and -100.0 <= r <= 100.0 and r == int(r):
+            w_p = w_complex.pow_small_int(int(r))
         else:
             w_p = w_complex.pow(w_exponent)
     except ZeroDivisionError:
diff --git a/pypy/objspace/std/test/test_complexobject.py b/pypy/objspace/std/test/test_complexobject.py
--- a/pypy/objspace/std/test/test_complexobject.py
+++ b/pypy/objspace/std/test/test_complexobject.py
@@ -71,7 +71,7 @@
         assert _powu((0.0,1.0),2) == (-1.0,0.0)
 
         def _powi((r1, i1), n):
-            w_res = W_ComplexObject(r1, i1).pow_int(n)
+            w_res = W_ComplexObject(r1, i1).pow_small_int(n)
             return w_res.realval, w_res.imagval
         assert _powi((0.0,2.0),0) == (1.0,0.0)
         assert _powi((0.0,0.0),2) == (0.0,0.0)
@@ -213,6 +213,7 @@
         assert a ** 105 == a ** 105
         assert a ** -105 == a ** -105
         assert a ** -30 == a ** -30
+        assert a ** 2 == a * a
 
         assert 0.0j ** 0 == 1
 

From noreply at buildbot.pypy.org  Sat Jan 21 18:33:33 2012
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sat, 21 Jan 2012 18:33:33 +0100 (CET)
Subject: [pypy-commit] pypy default: Fix for flatten() with certain types of
	arrays.
Message-ID: <20120121173333.5780382D45@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor 
Branch: 
Changeset: r51595:fe8a844cd0d0
Date: 2012-01-21 11:30 -0600
http://bitbucket.org/pypy/pypy/changeset/fe8a844cd0d0/

Log:	Fix for flatten() with certain types of arrays.

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -925,14 +925,15 @@
                                self.left.create_sig(), self.right.create_sig())
 
 class SliceArray(Call2):
-    def __init__(self, shape, dtype, left, right):
+    def __init__(self, shape, dtype, left, right, no_broadcast=False):
+        self.no_broadcast = no_broadcast
         Call2.__init__(self, None, 'sliceloop', shape, dtype, dtype, left,
                        right)
 
     def create_sig(self):
         lsig = self.left.create_sig()
         rsig = self.right.create_sig()
-        if self.shape != self.right.shape:
+        if not self.no_broadcast and self.shape != self.right.shape:
             return signature.SliceloopBroadcastSignature(self.ufunc,
                                                          self.name,
                                                          self.calc_dtype,
@@ -1166,7 +1167,7 @@
         if self.supports_fast_slicing():
             array._fast_setslice(space, self)
         else:
-            arr = SliceArray(array.shape, array.dtype, array, self)
+            arr = SliceArray(array.shape, array.dtype, array, self, no_broadcast=True)
             array._sliceloop(arr)
         return array
 

From noreply at buildbot.pypy.org  Sat Jan 21 18:33:34 2012
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sat, 21 Jan 2012 18:33:34 +0100 (CET)
Subject: [pypy-commit] pypy default: merged upstream
Message-ID: <20120121173334.A31E182D45@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor 
Branch: 
Changeset: r51596:1ebae6842fb2
Date: 2012-01-21 11:33 -0600
http://bitbucket.org/pypy/pypy/changeset/1ebae6842fb2/

Log:	merged upstream

diff --git a/pypy/objspace/std/complexobject.py b/pypy/objspace/std/complexobject.py
--- a/pypy/objspace/std/complexobject.py
+++ b/pypy/objspace/std/complexobject.py
@@ -8,6 +8,7 @@
 from pypy.rlib.rbigint import rbigint
 from pypy.rlib.rfloat import (
     formatd, DTSF_STR_PRECISION, isinf, isnan, copysign)
+from pypy.rlib import jit
 
 import math
 
@@ -129,10 +130,10 @@
             ir = len * math.sin(phase)
         return W_ComplexObject(rr, ir)
 
-    def pow_int(self, n):
-        if n > 100 or n < -100:
-            return self.pow(W_ComplexObject(1.0 * n, 0.0))
-        elif n > 0:
+    def pow_small_int(self, n):
+        if n >= 0:
+            if jit.isconstant(n) and n == 2:
+                return self.mul(self)
             return self.pow_positive_int(n)
         else:
             return w_one.div(self.pow_positive_int(-n))
@@ -217,10 +218,10 @@
 def pow__Complex_Complex_ANY(space, w_complex, w_exponent, thirdArg):
     if not space.is_w(thirdArg, space.w_None):
         raise OperationError(space.w_ValueError, space.wrap('complex modulo'))
-    int_exponent = int(w_exponent.realval)
     try:
-        if w_exponent.imagval == 0.0 and w_exponent.realval == int_exponent:
-            w_p = w_complex.pow_int(int_exponent)
+        r = w_exponent.realval
+        if w_exponent.imagval == 0.0 and -100.0 <= r <= 100.0 and r == int(r):
+            w_p = w_complex.pow_small_int(int(r))
         else:
             w_p = w_complex.pow(w_exponent)
     except ZeroDivisionError:
diff --git a/pypy/objspace/std/test/test_complexobject.py b/pypy/objspace/std/test/test_complexobject.py
--- a/pypy/objspace/std/test/test_complexobject.py
+++ b/pypy/objspace/std/test/test_complexobject.py
@@ -71,7 +71,7 @@
         assert _powu((0.0,1.0),2) == (-1.0,0.0)
 
         def _powi((r1, i1), n):
-            w_res = W_ComplexObject(r1, i1).pow_int(n)
+            w_res = W_ComplexObject(r1, i1).pow_small_int(n)
             return w_res.realval, w_res.imagval
         assert _powi((0.0,2.0),0) == (1.0,0.0)
         assert _powi((0.0,0.0),2) == (0.0,0.0)
@@ -213,6 +213,7 @@
         assert a ** 105 == a ** 105
         assert a ** -105 == a ** -105
         assert a ** -30 == a ** -30
+        assert a ** 2 == a * a
 
         assert 0.0j ** 0 == 1
 

From noreply at buildbot.pypy.org  Sat Jan 21 18:35:20 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Sat, 21 Jan 2012 18:35:20 +0100 (CET)
Subject: [pypy-commit] pypy numpy-back-to-applevel: add true divide
Message-ID: <20120121173520.E11F082D45@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: numpy-back-to-applevel
Changeset: r51597:a045db574996
Date: 2012-01-21 18:31 +0200
http://bitbucket.org/pypy/pypy/changeset/a045db574996/

Log:	add true divide

diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -68,6 +68,7 @@
         ("copysign", "copysign"),
         ("cos", "cos"),
         ("divide", "divide"),
+        ("true_divide", "true_divide"),
         ("equal", "equal"),
         ("exp", "exp"),
         ("fabs", "fabs"),
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -442,6 +442,7 @@
             ("bitwise_or", "bitwise_or", 2, {"identity": 0,
                                              'int_only': True}),
             ("divide", "div", 2, {"promote_bools": True}),
+            ("true_divide", "div", 2, {"promote_to_float": True}),
             ("mod", "mod", 2, {"promote_bools": True}),
             ("power", "pow", 2, {"promote_bools": True}),
 
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -402,3 +402,6 @@
         assert count_reduce_items(a, 1) == 3
         assert count_reduce_items(a, (1, 2)) == 3 * 4
         
+    def test_true_divide(self):
+        from _numpypy import arange, array, true_divide
+        assert (true_divide(arange(3), array([2, 2, 2])) == array([0, 0.5, 1])).all()

From noreply at buildbot.pypy.org  Sat Jan 21 18:35:22 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Sat, 21 Jan 2012 18:35:22 +0100 (CET)
Subject: [pypy-commit] pypy numpy-back-to-applevel: implement keepdims=True
Message-ID: <20120121173522.282CF82D45@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: numpy-back-to-applevel
Changeset: r51598:0fcad0cba011
Date: 2012-01-21 19:34 +0200
http://bitbucket.org/pypy/pypy/changeset/0fcad0cba011/

Log:	implement keepdims=True

diff --git a/pypy/module/micronumpy/interp_iter.py b/pypy/module/micronumpy/interp_iter.py
--- a/pypy/module/micronumpy/interp_iter.py
+++ b/pypy/module/micronumpy/interp_iter.py
@@ -153,8 +153,13 @@
 class AxisIterator(BaseIterator):
     def __init__(self, start, dim, shape, strides, backstrides):
         self.res_shape = shape[:]
-        self.strides = strides[:dim] + [0] + strides[dim:]
-        self.backstrides = backstrides[:dim] + [0] + backstrides[dim:]
+        if len(shape) == len(strides):
+            # keepdims = True
+            self.strides = strides[:dim] + [0] + strides[dim + 1:]
+            self.backstrides = backstrides[:dim] + [0] + backstrides[dim + 1:]
+        else:
+            self.strides = strides[:dim] + [0] + strides[dim:]
+            self.backstrides = backstrides[:dim] + [0] + backstrides[dim:]
         self.first_line = True
         self.indices = [0] * len(shape)
         self._done = False
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -1077,6 +1077,8 @@
 def array(space, w_item_or_iterable, w_dtype=None, w_order=None,
           subok=True, copy=False, w_maskna=None, ownmaskna=False):
     # find scalar
+    if w_maskna is None:
+        w_maskna = space.w_None
     if (not subok or copy or not space.is_w(w_maskna, space.w_None) or
         ownmaskna):
         raise OperationError(space.w_NotImplementedError, space.wrap("Unsupported args"))
@@ -1088,7 +1090,7 @@
             space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype)
         )
         return scalar_w(space, dtype, w_item_or_iterable)
-    if space.is_w(w_order, space.w_None):
+    if space.is_w(w_order, space.w_None) or w_order is None:
         order = 'C'
     else:
         order = space.str_w(w_order)
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -46,7 +46,8 @@
         return self.identity
 
     def descr_call(self, space, __args__):
-        if __args__.keywords or len(__args__.arguments_w) < self.argcount:
+        # XXX do something with strange keywords
+        if len(__args__.arguments_w) < self.argcount:
             raise OperationError(space.w_ValueError,
                 space.wrap("invalid number of arguments")
             )
@@ -60,7 +61,7 @@
 
     @unwrap_spec(skipna=bool, keepdims=bool)
     def descr_reduce(self, space, w_obj, w_axis=None, w_dtype=None,
-                     skipna=False, keepdims=True, w_out=None):
+                     skipna=False, keepdims=False, w_out=None):
         """reduce(...)
         reduce(a, axis=0)
 
@@ -120,9 +121,9 @@
             axis = -1
         else:
             axis = space.int_w(w_axis)
-        return self.reduce(space, w_obj, False, False, axis)
+        return self.reduce(space, w_obj, False, False, axis, keepdims)
 
-    def reduce(self, space, w_obj, multidim, promote_to_largest, dim):
+    def reduce(self, space, w_obj, multidim, promote_to_largest, dim, keepdims):
         from pypy.module.micronumpy.interp_numarray import convert_to_array, \
                                                            Scalar
         if self.argcount != 2:
@@ -148,7 +149,7 @@
             raise operationerrfmt(space.w_ValueError, "zero-size array to "
                     "%s.reduce without identity", self.name)
         if shapelen > 1 and dim >= 0:
-            res = self.do_axis_reduce(obj, dtype, dim)
+            res = self.do_axis_reduce(obj, dtype, dim, keepdims)
             return space.wrap(res)
         scalarsig = ScalarSignature(dtype)
         sig = find_sig(ReduceSignature(self.func, self.name, dtype,
@@ -162,11 +163,14 @@
             value = self.identity.convert_to(dtype)
         return self.reduce_loop(shapelen, sig, frame, value, obj, dtype)
 
-    def do_axis_reduce(self, obj, dtype, dim):
+    def do_axis_reduce(self, obj, dtype, dim, keepdims):
         from pypy.module.micronumpy.interp_numarray import AxisReduce,\
              W_NDimArray
-        
-        shape = obj.shape[0:dim] + obj.shape[dim + 1:len(obj.shape)]
+
+        if keepdims:
+            shape = obj.shape[:dim] + [1] + obj.shape[dim + 1:]
+        else:
+            shape = obj.shape[:dim] + obj.shape[dim + 1:]
         size = 1
         for s in shape:
             size *= s
diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py
--- a/pypy/module/micronumpy/strides.py
+++ b/pypy/module/micronumpy/strides.py
@@ -1,5 +1,5 @@
 from pypy.rlib import jit
-
+from pypy.interpreter.error import OperationError
 
 @jit.look_inside_iff(lambda shape, start, strides, backstrides, chunks:
     jit.isconstant(len(chunks))
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -344,7 +344,7 @@
         from _numpypy import sin, add
 
         raises(ValueError, sin.reduce, [1, 2, 3])
-        raises(ValueError, add.reduce, 1)
+        raises(TypeError, add.reduce, 1)
 
     def test_reduce_1d(self):
         from _numpypy import add, maximum
@@ -360,6 +360,14 @@
         assert (add.reduce(a, 0) == [12, 15, 18, 21]).all()
         assert (add.reduce(a, 1) == [6.0, 22.0, 38.0]).all()
 
+    def test_reduce_keepdims(self):
+        from _numpypy import add, arange
+        a = arange(12).reshape(3, 4)
+        b = add.reduce(a, 0, keepdims=True)
+        assert b.shape == (1, 4)
+        assert (add.reduce(a, 0, keepdims=True) == [12, 15, 18, 21]).all()
+        
+
     def test_bitwise(self):
         from _numpypy import bitwise_and, bitwise_or, arange, array
         a = arange(6).reshape(2, 3)

From noreply at buildbot.pypy.org  Sat Jan 21 18:40:15 2012
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sat, 21 Jan 2012 18:40:15 +0100 (CET)
Subject: [pypy-commit] extradoc extradoc: note that .T is forcing arrays
Message-ID: <20120121174015.0872082D45@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor 
Branch: extradoc
Changeset: r4049:dcdfbead3f5f
Date: 2012-01-21 11:40 -0600
http://bitbucket.org/pypy/extradoc/changeset/dcdfbead3f5f/

Log:	note that .T is forcing arrays

diff --git a/planning/micronumpy.txt b/planning/micronumpy.txt
--- a/planning/micronumpy.txt
+++ b/planning/micronumpy.txt
@@ -19,6 +19,8 @@
 
 - expose ndarray.ctypes
 
+- Make .T not force a lazy array.
+
 - subclassing ndarray (instantiating subcalsses curently returns the wrong type)
 
   * keep subclass type when slicing, __array_finalize__

From noreply at buildbot.pypy.org  Sat Jan 21 18:51:03 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Sat, 21 Jan 2012 18:51:03 +0100 (CET)
Subject: [pypy-commit] pypy numpy-back-to-applevel: a bit of shuffling until
 all the tests pass
Message-ID: <20120121175103.660F682D45@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: numpy-back-to-applevel
Changeset: r51599:890ac4e99e93
Date: 2012-01-21 19:50 +0200
http://bitbucket.org/pypy/pypy/changeset/890ac4e99e93/

Log:	a bit of shuffling until all the tests pass

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -142,9 +142,11 @@
     def _reduce_ufunc_impl(ufunc_name, promote_to_largest=False):
         def impl(self, space, w_axis=None):
             if space.is_w(w_axis, space.w_None):
-                w_axis = space.wrap(-1)
+                axis = -1
+            else:
+                axis = space.int_w(w_axis)
             return getattr(interp_ufuncs.get(space), ufunc_name).reduce(space,
-                                        self, True, promote_to_largest, w_axis)
+                                        self, True, promote_to_largest, axis)
         return func_with_new_name(impl, "reduce_%s_impl" % ufunc_name)
 
     descr_sum = _reduce_ufunc_impl("add")
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -1,6 +1,6 @@
 from pypy.interpreter.baseobjspace import Wrappable
 from pypy.interpreter.error import OperationError, operationerrfmt
-from pypy.interpreter.gateway import interp2app, unwrap_spec
+from pypy.interpreter.gateway import interp2app, unwrap_spec, NoneNotWrapped
 from pypy.interpreter.typedef import TypeDef, GetSetProperty, interp_attrproperty
 from pypy.module.micronumpy import interp_boxes, interp_dtype
 from pypy.module.micronumpy.signature import ReduceSignature,\
@@ -60,7 +60,7 @@
         return self.call(space, __args__.arguments_w)
 
     @unwrap_spec(skipna=bool, keepdims=bool)
-    def descr_reduce(self, space, w_obj, w_axis=None, w_dtype=None,
+    def descr_reduce(self, space, w_obj, w_axis=NoneNotWrapped, w_dtype=None,
                      skipna=False, keepdims=False, w_out=None):
         """reduce(...)
         reduce(a, axis=0)
@@ -117,13 +117,16 @@
         if not space.is_w(w_out, space.w_None):
             raise OperationError(space.w_NotImplementedError, space.wrap(
                 "out not supported"))
-        if space.is_w(w_axis, space.w_None):
+        if w_axis is None:
+            axis = 0
+        elif space.is_w(w_axis, space.w_None):
             axis = -1
         else:
             axis = space.int_w(w_axis)
         return self.reduce(space, w_obj, False, False, axis, keepdims)
 
-    def reduce(self, space, w_obj, multidim, promote_to_largest, dim, keepdims):
+    def reduce(self, space, w_obj, multidim, promote_to_largest, dim,
+               keepdims=False):
         from pypy.module.micronumpy.interp_numarray import convert_to_array, \
                                                            Scalar
         if self.argcount != 2:

From noreply at buildbot.pypy.org  Sat Jan 21 18:51:47 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Sat, 21 Jan 2012 18:51:47 +0100 (CET)
Subject: [pypy-commit] pypy numpy-back-to-applevel: fix one more test
Message-ID: <20120121175147.2E7B082D45@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: numpy-back-to-applevel
Changeset: r51600:92d15f3d3e45
Date: 2012-01-21 19:51 +0200
http://bitbucket.org/pypy/pypy/changeset/92d15f3d3e45/

Log:	fix one more test

diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -344,7 +344,7 @@
         from _numpypy import sin, add
 
         raises(ValueError, sin.reduce, [1, 2, 3])
-        raises(TypeError, add.reduce, 1)
+        raises((ValueError, TypeError), add.reduce, 1)
 
     def test_reduce_1d(self):
         from _numpypy import add, maximum

From noreply at buildbot.pypy.org  Sat Jan 21 18:56:56 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Sat, 21 Jan 2012 18:56:56 +0100 (CET)
Subject: [pypy-commit] pypy numpy-back-to-applevel: translation fix for test
	zjit
Message-ID: <20120121175656.CBFDF82D45@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: numpy-back-to-applevel
Changeset: r51601:220d6c657b35
Date: 2012-01-21 19:55 +0200
http://bitbucket.org/pypy/pypy/changeset/220d6c657b35/

Log:	translation fix for test zjit

diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -39,6 +39,7 @@
     w_TypeError = None
     w_IndexError = None
     w_OverflowError = None
+    w_NotImplementedError = None
     w_None = None
 
     w_bool = "bool"

From noreply at buildbot.pypy.org  Sat Jan 21 19:00:14 2012
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Sat, 21 Jan 2012 19:00:14 +0100 (CET)
Subject: [pypy-commit] pypy pytest: teach testsupport tests about reports
 now coming in unconditionally
Message-ID: <20120121180014.9EB0A82D45@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt 
Branch: pytest
Changeset: r51602:ab8337ed12e8
Date: 2012-01-21 18:59 +0100
http://bitbucket.org/pypy/pypy/changeset/ab8337ed12e8/

Log:	teach testsupport tests about reports now coming in unconditionally

diff --git a/pypy/tool/pytest/test/test_pytestsupport.py b/pypy/tool/pytest/test/test_pytestsupport.py
--- a/pypy/tool/pytest/test/test_pytestsupport.py
+++ b/pypy/tool/pytest/test/test_pytestsupport.py
@@ -165,7 +165,10 @@
     def test_one(self): exec 'blow'
     """)
 
-    ev, = sorter.getreports("pytest_runtest_logreport")
+    reports = sorter.getreports("pytest_runtest_logreport")
+    setup, ev, teardown = reports
     assert ev.failed
+    assert setup.passed
+    assert teardown.passed
     assert 'NameError' in ev.longrepr.reprcrash.message
     assert 'blow' in ev.longrepr.reprcrash.message

From noreply at buildbot.pypy.org  Sat Jan 21 19:28:30 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Sat, 21 Jan 2012 19:28:30 +0100 (CET)
Subject: [pypy-commit] pypy numpy-back-to-applevel: implement copy kwarg
Message-ID: <20120121182830.5967682D45@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: numpy-back-to-applevel
Changeset: r51603:cc80fb4d3ab9
Date: 2012-01-21 20:23 +0200
http://bitbucket.org/pypy/pypy/changeset/cc80fb4d3ab9/

Log:	implement copy kwarg

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -1077,11 +1077,11 @@
 
 @unwrap_spec(subok=bool, copy=bool, ownmaskna=bool)
 def array(space, w_item_or_iterable, w_dtype=None, w_order=None,
-          subok=True, copy=False, w_maskna=None, ownmaskna=False):
+          subok=True, copy=True, w_maskna=None, ownmaskna=False):
     # find scalar
     if w_maskna is None:
         w_maskna = space.w_None
-    if (not subok or copy or not space.is_w(w_maskna, space.w_None) or
+    if (not subok or not space.is_w(w_maskna, space.w_None) or
         ownmaskna):
         raise OperationError(space.w_NotImplementedError, space.wrap("Unsupported args"))
     if not space.issequence_w(w_item_or_iterable):
@@ -1099,6 +1099,14 @@
         if order != 'C':  # or order != 'F':
             raise operationerrfmt(space.w_ValueError, "Unknown order: %s",
                                   order)
+    if isinstance(w_item_or_iterable, BaseArray):
+        if (not space.is_w(w_dtype, space.w_None) and
+            w_item_or_iterable.find_dtype() is not w_dtype):
+            raise OperationError(space.w_NotImplementedError, space.wrap(
+                "copying over different dtypes unsupported"))
+        if copy:
+            return w_item_or_iterable.copy(space)
+        return w_item_or_iterable
     shape, elems_w = find_shape_and_elems(space, w_item_or_iterable)
     # they come back in C order
     size = len(elems_w)
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1358,7 +1358,13 @@
         a[a & 1 == 1] = array([8, 9, 10])
         assert (a == [[0, 8], [2, 9], [4, 10]]).all()
 
-
+    def test_copy_kwarg(self):
+        from _numpypy import array
+        x = array([1, 2, 3])
+        assert (array(x) == x).all()
+        assert array(x) is not x
+        assert array(x, copy=False) is x
+        assert array(x, copy=True) is not x
 
 class AppTestSupport(BaseNumpyAppTest):
     def setup_class(cls):

From noreply at buildbot.pypy.org  Sat Jan 21 20:04:52 2012
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sat, 21 Jan 2012 20:04:52 +0100 (CET)
Subject: [pypy-commit] pypy numpy-back-to-applevel: merged default
Message-ID: <20120121190452.B0BB782D45@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor 
Branch: numpy-back-to-applevel
Changeset: r51604:f49b248ef393
Date: 2012-01-21 11:57 -0600
http://bitbucket.org/pypy/pypy/changeset/f49b248ef393/

Log:	merged default

diff --git a/lib_pypy/datetime.py b/lib_pypy/datetime.py
--- a/lib_pypy/datetime.py
+++ b/lib_pypy/datetime.py
@@ -1440,17 +1440,22 @@
         return result
     fromtimestamp = classmethod(fromtimestamp)
 
+    @classmethod
     def utcfromtimestamp(cls, t):
         "Construct a UTC datetime from a POSIX timestamp (like time.time())."
-        if 1 - (t % 1.0) < 0.0000005:
-            t = float(int(t)) + 1
-        if t < 0:
-            t -= 1
+        t, frac = divmod(t, 1.0)
+        us = round(frac * 1e6)
+
+        # If timestamp is less than one microsecond smaller than a
+        # full second, us can be rounded up to 1000000.  In this case,
+        # roll over to seconds, otherwise, ValueError is raised
+        # by the constructor.
+        if us == 1000000:
+            t += 1
+            us = 0
         y, m, d, hh, mm, ss, weekday, jday, dst = _time.gmtime(t)
-        us = int((t % 1.0) * 1000000)
         ss = min(ss, 59)    # clamp out leap seconds if the platform has them
         return cls(y, m, d, hh, mm, ss, us)
-    utcfromtimestamp = classmethod(utcfromtimestamp)
 
     # XXX This is supposed to do better than we *can* do by using time.time(),
     # XXX if the platform supports a more accurate way.  The C implementation
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -264,9 +264,15 @@
     def descr_copy(self, space):
         return self.copy(space)
 
+    def descr_flatten(self, space):
+        return self.flatten(space)
+
     def copy(self, space):
         return self.get_concrete().copy(space)
 
+    def flatten(self, space):
+        return self.get_concrete().flatten(space)
+
     def descr_len(self, space):
         if len(self.shape):
             return space.wrap(self.shape[0])
@@ -599,6 +605,11 @@
     def copy(self, space):
         return Scalar(self.dtype, self.value)
 
+    def flatten(self, space):
+        array = W_NDimArray(self.size, [self.size], self.dtype)
+        array.setitem(0, self.value)
+        return array
+
     def fill(self, space, w_value):
         self.value = self.dtype.coerce(space, w_value)
 
@@ -744,14 +755,15 @@
                                self.left.create_sig(), self.right.create_sig())
 
 class SliceArray(Call2):
-    def __init__(self, shape, dtype, left, right):
+    def __init__(self, shape, dtype, left, right, no_broadcast=False):
+        self.no_broadcast = no_broadcast
         Call2.__init__(self, None, 'sliceloop', shape, dtype, dtype, left,
                        right)
 
     def create_sig(self):
         lsig = self.left.create_sig()
         rsig = self.right.create_sig()
-        if self.shape != self.right.shape:
+        if not self.no_broadcast and self.shape != self.right.shape:
             return signature.SliceloopBroadcastSignature(self.ufunc,
                                                          self.name,
                                                          self.calc_dtype,
@@ -980,6 +992,15 @@
         array.setslice(space, self)
         return array
 
+    def flatten(self, space):
+        array = W_NDimArray(self.size, [self.size], self.dtype, self.order)
+        if self.supports_fast_slicing():
+            array._fast_setslice(space, self)
+        else:
+            arr = SliceArray(array.shape, array.dtype, array, self, no_broadcast=True)
+            array._sliceloop(arr)
+        return array
+
     def fill(self, space, w_value):
         self.setslice(space, scalar_w(space, self.dtype, w_value))
 
@@ -1233,6 +1254,7 @@
     fill = interp2app(BaseArray.descr_fill),
 
     copy = interp2app(BaseArray.descr_copy),
+    flatten = interp2app(BaseArray.descr_flatten),
     reshape = interp2app(BaseArray.descr_reshape),
     tolist = interp2app(BaseArray.descr_tolist),
 )
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1039,6 +1039,22 @@
         a = array([5.0])
         assert a.std() == 0.0
 
+    def test_flatten(self):
+        from _numpypy import array
+
+        a = array([[1, 2, 3], [4, 5, 6]])
+        assert (a.flatten() == [1, 2, 3, 4, 5, 6]).all()
+        a = array([[[1, 2], [3, 4]], [[5, 6], [7, 8]]])
+        assert (a.flatten() == [1, 2, 3, 4, 5, 6, 7, 8]).all()
+        a = array([1, 2, 3, 4, 5, 6, 7, 8])
+        assert (a[::2].flatten() == [1, 3, 5, 7]).all()
+        a = array([1, 2, 3])
+        assert ((a + a).flatten() == [2, 4, 6]).all()
+        a = array(2)
+        assert (a.flatten() == [2]).all()
+        a = array([[1, 2], [3, 4]])
+        assert (a.T.flatten() == [1, 3, 2, 4]).all()
+
 
 class AppTestMultiDim(BaseNumpyAppTest):
     def test_init(self):
diff --git a/pypy/module/test_lib_pypy/test_datetime.py b/pypy/module/test_lib_pypy/test_datetime.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/test_lib_pypy/test_datetime.py
@@ -0,0 +1,24 @@
+"""Additional tests for datetime."""
+
+import time
+import datetime
+import os
+
+def test_utcfromtimestamp():
+    """Confirm that utcfromtimestamp and fromtimestamp give consistent results.
+
+    Based on danchr's test script in https://bugs.pypy.org/issue986
+    """
+    try:
+        prev_tz = os.environ.get("TZ")
+        os.environ["TZ"] = "GMT"
+        for unused in xrange(100):
+            now = time.time()
+            delta = (datetime.datetime.utcfromtimestamp(now) -
+                     datetime.datetime.fromtimestamp(now))
+            assert delta.days * 86400 + delta.seconds == 0
+    finally:
+        if prev_tz is None:
+            del os.environ["TZ"]
+        else:
+            os.environ["TZ"] = prev_tz
diff --git a/pypy/objspace/std/complexobject.py b/pypy/objspace/std/complexobject.py
--- a/pypy/objspace/std/complexobject.py
+++ b/pypy/objspace/std/complexobject.py
@@ -8,6 +8,7 @@
 from pypy.rlib.rbigint import rbigint
 from pypy.rlib.rfloat import (
     formatd, DTSF_STR_PRECISION, isinf, isnan, copysign)
+from pypy.rlib import jit
 
 import math
 
@@ -129,10 +130,10 @@
             ir = len * math.sin(phase)
         return W_ComplexObject(rr, ir)
 
-    def pow_int(self, n):
-        if n > 100 or n < -100:
-            return self.pow(W_ComplexObject(1.0 * n, 0.0))
-        elif n > 0:
+    def pow_small_int(self, n):
+        if n >= 0:
+            if jit.isconstant(n) and n == 2:
+                return self.mul(self)
             return self.pow_positive_int(n)
         else:
             return w_one.div(self.pow_positive_int(-n))
@@ -217,10 +218,10 @@
 def pow__Complex_Complex_ANY(space, w_complex, w_exponent, thirdArg):
     if not space.is_w(thirdArg, space.w_None):
         raise OperationError(space.w_ValueError, space.wrap('complex modulo'))
-    int_exponent = int(w_exponent.realval)
     try:
-        if w_exponent.imagval == 0.0 and w_exponent.realval == int_exponent:
-            w_p = w_complex.pow_int(int_exponent)
+        r = w_exponent.realval
+        if w_exponent.imagval == 0.0 and -100.0 <= r <= 100.0 and r == int(r):
+            w_p = w_complex.pow_small_int(int(r))
         else:
             w_p = w_complex.pow(w_exponent)
     except ZeroDivisionError:
diff --git a/pypy/objspace/std/test/test_complexobject.py b/pypy/objspace/std/test/test_complexobject.py
--- a/pypy/objspace/std/test/test_complexobject.py
+++ b/pypy/objspace/std/test/test_complexobject.py
@@ -71,7 +71,7 @@
         assert _powu((0.0,1.0),2) == (-1.0,0.0)
 
         def _powi((r1, i1), n):
-            w_res = W_ComplexObject(r1, i1).pow_int(n)
+            w_res = W_ComplexObject(r1, i1).pow_small_int(n)
             return w_res.realval, w_res.imagval
         assert _powi((0.0,2.0),0) == (1.0,0.0)
         assert _powi((0.0,0.0),2) == (0.0,0.0)
@@ -213,6 +213,7 @@
         assert a ** 105 == a ** 105
         assert a ** -105 == a ** -105
         assert a ** -30 == a ** -30
+        assert a ** 2 == a * a
 
         assert 0.0j ** 0 == 1
 

From noreply at buildbot.pypy.org  Sat Jan 21 20:04:53 2012
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sat, 21 Jan 2012 20:04:53 +0100 (CET)
Subject: [pypy-commit] pypy numpy-back-to-applevel: merged upstream
Message-ID: <20120121190453.E9A0F82D45@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor 
Branch: numpy-back-to-applevel
Changeset: r51605:8b5bfccf07e6
Date: 2012-01-21 12:56 -0600
http://bitbucket.org/pypy/pypy/changeset/8b5bfccf07e6/

Log:	merged upstream

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -1098,11 +1098,11 @@
 
 @unwrap_spec(subok=bool, copy=bool, ownmaskna=bool)
 def array(space, w_item_or_iterable, w_dtype=None, w_order=None,
-          subok=True, copy=False, w_maskna=None, ownmaskna=False):
+          subok=True, copy=True, w_maskna=None, ownmaskna=False):
     # find scalar
     if w_maskna is None:
         w_maskna = space.w_None
-    if (not subok or copy or not space.is_w(w_maskna, space.w_None) or
+    if (not subok or not space.is_w(w_maskna, space.w_None) or
         ownmaskna):
         raise OperationError(space.w_NotImplementedError, space.wrap("Unsupported args"))
     if not space.issequence_w(w_item_or_iterable):
@@ -1120,6 +1120,14 @@
         if order != 'C':  # or order != 'F':
             raise operationerrfmt(space.w_ValueError, "Unknown order: %s",
                                   order)
+    if isinstance(w_item_or_iterable, BaseArray):
+        if (not space.is_w(w_dtype, space.w_None) and
+            w_item_or_iterable.find_dtype() is not w_dtype):
+            raise OperationError(space.w_NotImplementedError, space.wrap(
+                "copying over different dtypes unsupported"))
+        if copy:
+            return w_item_or_iterable.copy(space)
+        return w_item_or_iterable
     shape, elems_w = find_shape_and_elems(space, w_item_or_iterable)
     # they come back in C order
     size = len(elems_w)
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1374,7 +1374,13 @@
         a[a & 1 == 1] = array([8, 9, 10])
         assert (a == [[0, 8], [2, 9], [4, 10]]).all()
 
-
+    def test_copy_kwarg(self):
+        from _numpypy import array
+        x = array([1, 2, 3])
+        assert (array(x) == x).all()
+        assert array(x) is not x
+        assert array(x, copy=False) is x
+        assert array(x, copy=True) is not x
 
 class AppTestSupport(BaseNumpyAppTest):
     def setup_class(cls):

From noreply at buildbot.pypy.org  Sat Jan 21 20:12:14 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Sat, 21 Jan 2012 20:12:14 +0100 (CET)
Subject: [pypy-commit] pypy numpy-back-to-applevel: import var/std,
 figure out extra kwargs for reduce
Message-ID: <20120121191214.2175582D45@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: numpy-back-to-applevel
Changeset: r51606:1c6f3a721515
Date: 2012-01-21 21:02 +0200
http://bitbucket.org/pypy/pypy/changeset/1c6f3a721515/

Log:	import var/std, figure out extra kwargs for reduce

diff --git a/lib_pypy/numpypy/core/_methods.py b/lib_pypy/numpypy/core/_methods.py
--- a/lib_pypy/numpypy/core/_methods.py
+++ b/lib_pypy/numpypy/core/_methods.py
@@ -36,7 +36,7 @@
                             skipna=skipna, keepdims=keepdims)
     if isinstance(ret, mu.ndarray):
         ret = um.true_divide(ret, rcount,
-                        out=ret, casting='unsafe', subok=False)
+                        casting='unsafe', subok=False)
     else:
         ret = ret / float(rcount)
     return ret
@@ -79,7 +79,7 @@
     rcount -= ddof
     if isinstance(ret, mu.ndarray):
         ret = um.true_divide(ret, rcount,
-                        out=ret, casting='unsafe', subok=False)
+                        casting='unsafe', subok=False)
     else:
         ret = ret / float(rcount)
 
@@ -91,7 +91,7 @@
                                 skipna=skipna, keepdims=keepdims)
 
     if isinstance(ret, mu.ndarray):
-        ret = um.sqrt(ret, out=ret)
+        ret = um.sqrt(ret)
     else:
         ret = um.sqrt(ret)
 
diff --git a/lib_pypy/numpypy/core/fromnumeric.py b/lib_pypy/numpypy/core/fromnumeric.py
--- a/lib_pypy/numpypy/core/fromnumeric.py
+++ b/lib_pypy/numpypy/core/fromnumeric.py
@@ -2324,13 +2324,12 @@
     0.44999999925552653
 
     """
-    assert axis is None
     assert dtype is None
     assert out is None
     assert ddof == 0
     if not hasattr(a, "std"):
         a = numpypy.array(a)
-    return a.std()
+    return a.std(axis=axis)
 
 
 def var(a, axis=None, dtype=None, out=None, ddof=0):
@@ -2421,10 +2420,9 @@
     0.20250000000000001
 
     """
-    assert axis is None
     assert dtype is None
     assert out is None
     assert ddof == 0
     if not hasattr(a, "var"):
         a = numpypy.array(a)
-    return a.var()
+    return a.var(axis=axis)
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -46,18 +46,27 @@
         return self.identity
 
     def descr_call(self, space, __args__):
-        # XXX do something with strange keywords
-        if len(__args__.arguments_w) < self.argcount:
+        args_w, kwds_w = __args__.unpack()
+        # it occurs to me that we don't support any datatypes that
+        # require casting, change it later when we do
+        kwds_w.pop('casting', None)
+        w_subok = kwds_w.pop('subok', None)
+        w_out = kwds_w.pop('out', space.w_None)
+        if ((w_subok is not None and space.is_true(w_subok)) or
+            not space.is_w(w_out, space.w_None)):
+            raise OperationError(space.w_NotImplementedError,
+                                 space.wrap("parameters unsupported"))
+        if kwds_w or len(args_w) < self.argcount:
             raise OperationError(space.w_ValueError,
                 space.wrap("invalid number of arguments")
             )
-        elif len(__args__.arguments_w) > self.argcount:
+        elif len(args_w) > self.argcount:
             # The extra arguments should actually be the output array, but we
             # don't support that yet.
             raise OperationError(space.w_TypeError,
                 space.wrap("invalid number of arguments")
             )
-        return self.call(space, __args__.arguments_w)
+        return self.call(space, args_w)
 
     @unwrap_spec(skipna=bool, keepdims=bool)
     def descr_reduce(self, space, w_obj, w_axis=NoneNotWrapped, w_dtype=None,
diff --git a/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
--- a/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
+++ b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
@@ -98,15 +98,15 @@
         from numpypy import array, var
         a = array([[1,2],[3,4]])
         assert var(a) == 1.25
-        #assert (var(a,0) == array([ 1.,  1.])).all()
-        #assert (var(a,1) == array([ 0.25,  0.25])).all()
+        assert (var(a,0) == array([ 1.,  1.])).all()
+        assert (var(a,1) == array([ 0.25,  0.25])).all()
 
     def test_std(self):
         from numpypy import array, std
         a = array([[1, 2], [3, 4]])
         assert std(a) ==  1.1180339887498949
-        #assert (std(a, axis=0) == array([ 1.,  1.])).all()
-        #assert (std(a, axis=1) == array([ 0.5,  0.5])).all()
+        assert (std(a, axis=0) == array([ 1.,  1.])).all()
+        assert (std(a, axis=1) == array([ 0.5,  0.5])).all()
 
     def test_mean(self):
         from numpypy import array, mean, arange

From noreply at buildbot.pypy.org  Sat Jan 21 20:12:15 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Sat, 21 Jan 2012 20:12:15 +0100 (CET)
Subject: [pypy-commit] pypy numpy-back-to-applevel: merge
Message-ID: <20120121191215.6160382D45@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: numpy-back-to-applevel
Changeset: r51607:33707327eb21
Date: 2012-01-21 21:11 +0200
http://bitbucket.org/pypy/pypy/changeset/33707327eb21/

Log:	merge

diff --git a/lib_pypy/datetime.py b/lib_pypy/datetime.py
--- a/lib_pypy/datetime.py
+++ b/lib_pypy/datetime.py
@@ -1440,17 +1440,22 @@
         return result
     fromtimestamp = classmethod(fromtimestamp)
 
+    @classmethod
     def utcfromtimestamp(cls, t):
         "Construct a UTC datetime from a POSIX timestamp (like time.time())."
-        if 1 - (t % 1.0) < 0.0000005:
-            t = float(int(t)) + 1
-        if t < 0:
-            t -= 1
+        t, frac = divmod(t, 1.0)
+        us = round(frac * 1e6)
+
+        # If timestamp is less than one microsecond smaller than a
+        # full second, us can be rounded up to 1000000.  In this case,
+        # roll over to seconds, otherwise, ValueError is raised
+        # by the constructor.
+        if us == 1000000:
+            t += 1
+            us = 0
         y, m, d, hh, mm, ss, weekday, jday, dst = _time.gmtime(t)
-        us = int((t % 1.0) * 1000000)
         ss = min(ss, 59)    # clamp out leap seconds if the platform has them
         return cls(y, m, d, hh, mm, ss, us)
-    utcfromtimestamp = classmethod(utcfromtimestamp)
 
     # XXX This is supposed to do better than we *can* do by using time.time(),
     # XXX if the platform supports a more accurate way.  The C implementation
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -264,9 +264,15 @@
     def descr_copy(self, space):
         return self.copy(space)
 
+    def descr_flatten(self, space):
+        return self.flatten(space)
+
     def copy(self, space):
         return self.get_concrete().copy(space)
 
+    def flatten(self, space):
+        return self.get_concrete().flatten(space)
+
     def descr_len(self, space):
         if len(self.shape):
             return space.wrap(self.shape[0])
@@ -599,6 +605,11 @@
     def copy(self, space):
         return Scalar(self.dtype, self.value)
 
+    def flatten(self, space):
+        array = W_NDimArray(self.size, [self.size], self.dtype)
+        array.setitem(0, self.value)
+        return array
+
     def fill(self, space, w_value):
         self.value = self.dtype.coerce(space, w_value)
 
@@ -744,14 +755,15 @@
                                self.left.create_sig(), self.right.create_sig())
 
 class SliceArray(Call2):
-    def __init__(self, shape, dtype, left, right):
+    def __init__(self, shape, dtype, left, right, no_broadcast=False):
+        self.no_broadcast = no_broadcast
         Call2.__init__(self, None, 'sliceloop', shape, dtype, dtype, left,
                        right)
 
     def create_sig(self):
         lsig = self.left.create_sig()
         rsig = self.right.create_sig()
-        if self.shape != self.right.shape:
+        if not self.no_broadcast and self.shape != self.right.shape:
             return signature.SliceloopBroadcastSignature(self.ufunc,
                                                          self.name,
                                                          self.calc_dtype,
@@ -980,6 +992,15 @@
         array.setslice(space, self)
         return array
 
+    def flatten(self, space):
+        array = W_NDimArray(self.size, [self.size], self.dtype, self.order)
+        if self.supports_fast_slicing():
+            array._fast_setslice(space, self)
+        else:
+            arr = SliceArray(array.shape, array.dtype, array, self, no_broadcast=True)
+            array._sliceloop(arr)
+        return array
+
     def fill(self, space, w_value):
         self.setslice(space, scalar_w(space, self.dtype, w_value))
 
@@ -1241,6 +1262,7 @@
     fill = interp2app(BaseArray.descr_fill),
 
     copy = interp2app(BaseArray.descr_copy),
+    flatten = interp2app(BaseArray.descr_flatten),
     reshape = interp2app(BaseArray.descr_reshape),
     tolist = interp2app(BaseArray.descr_tolist),
 )
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1039,6 +1039,22 @@
         a = array([5.0])
         assert a.std() == 0.0
 
+    def test_flatten(self):
+        from _numpypy import array
+
+        a = array([[1, 2, 3], [4, 5, 6]])
+        assert (a.flatten() == [1, 2, 3, 4, 5, 6]).all()
+        a = array([[[1, 2], [3, 4]], [[5, 6], [7, 8]]])
+        assert (a.flatten() == [1, 2, 3, 4, 5, 6, 7, 8]).all()
+        a = array([1, 2, 3, 4, 5, 6, 7, 8])
+        assert (a[::2].flatten() == [1, 3, 5, 7]).all()
+        a = array([1, 2, 3])
+        assert ((a + a).flatten() == [2, 4, 6]).all()
+        a = array(2)
+        assert (a.flatten() == [2]).all()
+        a = array([[1, 2], [3, 4]])
+        assert (a.T.flatten() == [1, 3, 2, 4]).all()
+
 
 class AppTestMultiDim(BaseNumpyAppTest):
     def test_init(self):
diff --git a/pypy/module/test_lib_pypy/test_datetime.py b/pypy/module/test_lib_pypy/test_datetime.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/test_lib_pypy/test_datetime.py
@@ -0,0 +1,24 @@
+"""Additional tests for datetime."""
+
+import time
+import datetime
+import os
+
+def test_utcfromtimestamp():
+    """Confirm that utcfromtimestamp and fromtimestamp give consistent results.
+
+    Based on danchr's test script in https://bugs.pypy.org/issue986
+    """
+    try:
+        prev_tz = os.environ.get("TZ")
+        os.environ["TZ"] = "GMT"
+        for unused in xrange(100):
+            now = time.time()
+            delta = (datetime.datetime.utcfromtimestamp(now) -
+                     datetime.datetime.fromtimestamp(now))
+            assert delta.days * 86400 + delta.seconds == 0
+    finally:
+        if prev_tz is None:
+            del os.environ["TZ"]
+        else:
+            os.environ["TZ"] = prev_tz
diff --git a/pypy/objspace/std/complexobject.py b/pypy/objspace/std/complexobject.py
--- a/pypy/objspace/std/complexobject.py
+++ b/pypy/objspace/std/complexobject.py
@@ -8,6 +8,7 @@
 from pypy.rlib.rbigint import rbigint
 from pypy.rlib.rfloat import (
     formatd, DTSF_STR_PRECISION, isinf, isnan, copysign)
+from pypy.rlib import jit
 
 import math
 
@@ -129,10 +130,10 @@
             ir = len * math.sin(phase)
         return W_ComplexObject(rr, ir)
 
-    def pow_int(self, n):
-        if n > 100 or n < -100:
-            return self.pow(W_ComplexObject(1.0 * n, 0.0))
-        elif n > 0:
+    def pow_small_int(self, n):
+        if n >= 0:
+            if jit.isconstant(n) and n == 2:
+                return self.mul(self)
             return self.pow_positive_int(n)
         else:
             return w_one.div(self.pow_positive_int(-n))
@@ -217,10 +218,10 @@
 def pow__Complex_Complex_ANY(space, w_complex, w_exponent, thirdArg):
     if not space.is_w(thirdArg, space.w_None):
         raise OperationError(space.w_ValueError, space.wrap('complex modulo'))
-    int_exponent = int(w_exponent.realval)
     try:
-        if w_exponent.imagval == 0.0 and w_exponent.realval == int_exponent:
-            w_p = w_complex.pow_int(int_exponent)
+        r = w_exponent.realval
+        if w_exponent.imagval == 0.0 and -100.0 <= r <= 100.0 and r == int(r):
+            w_p = w_complex.pow_small_int(int(r))
         else:
             w_p = w_complex.pow(w_exponent)
     except ZeroDivisionError:
diff --git a/pypy/objspace/std/test/test_complexobject.py b/pypy/objspace/std/test/test_complexobject.py
--- a/pypy/objspace/std/test/test_complexobject.py
+++ b/pypy/objspace/std/test/test_complexobject.py
@@ -71,7 +71,7 @@
         assert _powu((0.0,1.0),2) == (-1.0,0.0)
 
         def _powi((r1, i1), n):
-            w_res = W_ComplexObject(r1, i1).pow_int(n)
+            w_res = W_ComplexObject(r1, i1).pow_small_int(n)
             return w_res.realval, w_res.imagval
         assert _powi((0.0,2.0),0) == (1.0,0.0)
         assert _powi((0.0,0.0),2) == (0.0,0.0)
@@ -213,6 +213,7 @@
         assert a ** 105 == a ** 105
         assert a ** -105 == a ** -105
         assert a ** -30 == a ** -30
+        assert a ** 2 == a * a
 
         assert 0.0j ** 0 == 1
 

From noreply at buildbot.pypy.org  Sat Jan 21 20:48:48 2012
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sat, 21 Jan 2012 20:48:48 +0100 (CET)
Subject: [pypy-commit] pypy numpy-back-to-applevel: Make dict.pop RPython.
Message-ID: <20120121194848.81E1C82D45@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor 
Branch: numpy-back-to-applevel
Changeset: r51608:81d4c62723ed
Date: 2012-01-21 13:48 -0600
http://bitbucket.org/pypy/pypy/changeset/81d4c62723ed/

Log:	Make dict.pop RPython.

diff --git a/pypy/annotation/unaryop.py b/pypy/annotation/unaryop.py
--- a/pypy/annotation/unaryop.py
+++ b/pypy/annotation/unaryop.py
@@ -440,6 +440,12 @@
     def method_popitem(dct):
         return dct.getanyitem('items')
 
+    def method_pop(dct, s_key, s_dfl=None):
+        dct.dictdef.generalize_key(s_key)
+        if s_dfl is not None:
+            dct.dictdef.generalize_value(s_dfl)
+        return dct.dictdef.read_value()
+
     def _can_only_throw(dic, *ignore):
         if dic1.dictdef.dictkey.custom_eq_hash:
             return None    # r_dict: can throw anything
diff --git a/pypy/rpython/lltypesystem/rdict.py b/pypy/rpython/lltypesystem/rdict.py
--- a/pypy/rpython/lltypesystem/rdict.py
+++ b/pypy/rpython/lltypesystem/rdict.py
@@ -323,6 +323,16 @@
         hop.exception_is_here()
         return hop.gendirectcall(ll_popitem, cTUPLE, v_dict)
 
+    def rtype_method_pop(self, hop):
+        if hop.nb_args == 2:
+            v_args = hop.inputargs(self, self.key_repr)
+            target = ll_pop
+        elif hop.nb_args == 3:
+            v_args = hop.inputargs(self, self.key_repr, self.value_repr)
+            target = ll_pop_default
+        hop.exception_is_here()
+        return hop.gendirectcall(target, *v_args)
+
 class __extend__(pairtype(DictRepr, rmodel.Repr)):
 
     def rtype_getitem((r_dict, r_key), hop):
@@ -874,3 +884,18 @@
     r.item1 = recast(ELEM.TO.item1, entry.value)
     _ll_dict_del(dic, i)
     return r
+
+def ll_pop(dic, key):
+    i = ll_dict_lookup(dic, key, dic.keyhash(key))
+    if not i & HIGHEST_BIT:
+        value = ll_get_value(dic, i)
+        _ll_dict_del(dic, i)
+        return value
+    else:
+        raise KeyError
+
+def ll_pop_default(dic, key, dfl):
+    try:
+        return ll_pop(dic, key)
+    except KeyError:
+        return dfl
diff --git a/pypy/rpython/ootypesystem/rdict.py b/pypy/rpython/ootypesystem/rdict.py
--- a/pypy/rpython/ootypesystem/rdict.py
+++ b/pypy/rpython/ootypesystem/rdict.py
@@ -160,6 +160,16 @@
         hop.exception_is_here()
         return hop.gendirectcall(ll_popitem, cTUPLE, v_dict)
 
+    def rtype_method_pop(self, hop):
+        if hop.nb_args == 2:
+            v_args = hop.inputargs(self, self.key_repr)
+            target = ll_pop
+        elif hop.nb_args == 3:
+            v_args = hop.inputargs(self, self.key_repr, self.value_repr)
+            target = ll_pop_default
+        hop.exception_is_here()
+        return hop.gendirectcall(target, *v_args)
+
     def __get_func(self, interp, r_func, fn, TYPE):
         if isinstance(r_func, MethodOfFrozenPBCRepr):
             obj = r_func.r_im_self.convert_const(fn.im_self)
@@ -370,6 +380,20 @@
         return res
     raise KeyError
 
+def ll_pop(d, key):
+    if d.ll_contains(key):
+        value = d.ll_get(key)
+        d.ll_remove(key)
+        return value
+    else:
+        raise KeyError
+
+def ll_pop_default(d, key, dfl):
+    try:
+        return ll_pop(d, key)
+    except KeyError:
+        return dfl
+
 # ____________________________________________________________
 #
 #  Iteration.
diff --git a/pypy/rpython/test/test_rdict.py b/pypy/rpython/test/test_rdict.py
--- a/pypy/rpython/test/test_rdict.py
+++ b/pypy/rpython/test/test_rdict.py
@@ -622,6 +622,26 @@
         res = self.interpret(func, [])
         assert res in [5263, 6352]
 
+    def test_dict_pop(self):
+        def f(n, default):
+            d = {}
+            d[2] = 3
+            d[4] = 5
+            if default == -1:
+                try:
+                    x = d.pop(n)
+                except KeyError:
+                    x = -1
+            else:
+                x = d.pop(n, default)
+            return x * 10 + len(d)
+        res = self.interpret(f, [2, -1])
+        assert res == 31
+        res = self.interpret(f, [3, -1])
+        assert res == -8
+        res = self.interpret(f, [2, 5])
+        assert res == 31
+
 class TestLLtype(BaseTestRdict, LLRtypeMixin):
     def test_dict_but_not_with_char_keys(self):
         def func(i):

From noreply at buildbot.pypy.org  Sat Jan 21 21:40:08 2012
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Sat, 21 Jan 2012 21:40:08 +0100 (CET)
Subject: [pypy-commit] pypy pytest: use funcargs in the testrunner test to
 ease having recoverable output
Message-ID: <20120121204008.0B52782D45@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt 
Branch: pytest
Changeset: r51609:a696d22b84c6
Date: 2012-01-21 20:08 +0100
http://bitbucket.org/pypy/pypy/changeset/a696d22b84c6/

Log:	use funcargs in the testrunner test to ease having recoverable
	output

diff --git a/testrunner/test/conftest.py b/testrunner/test/conftest.py
new file mode 100644
--- /dev/null
+++ b/testrunner/test/conftest.py
@@ -0,0 +1,6 @@
+
+def pytest_runtest_makereport(__multicall__, item):
+    report = __multicall__.execute()
+    if 'out' in item.funcargs:
+        report.sections.append(('out', item.funcargs['out'].read()))
+    return report
diff --git a/testrunner/test/test_runner.py b/testrunner/test/test_runner.py
--- a/testrunner/test/test_runner.py
+++ b/testrunner/test/test_runner.py
@@ -53,49 +53,44 @@
     assert not should_report_failure("F Def\n. Ghi\n. Jkl\n")
 
 
+
 class TestRunHelper(object):
+    def pytest_funcarg__out(self, request):
+        tmpdir = request.getfuncargvalue('tmpdir')
+        return tmpdir.ensure('out')
 
-    def setup_method(self, meth):
-        h, self.fn = tempfile.mkstemp()
-        os.close(h)
+    def test_run(self, out):
+        res = runner.run([sys.executable, "-c", "print 42"], '.', out)
+        assert res == 0
+        assert out.read() == "42\n"
 
-    def teardown_method(self, meth):
-        os.unlink(self.fn)
-
-    def test_run(self):
-        res = runner.run([sys.executable, "-c", "print 42"], '.',
-                         py.path.local(self.fn))
-        assert res == 0
-        out = py.path.local(self.fn).read('r')
-        assert out == "42\n"
-
-    def test_error(self):
-        res = runner.run([sys.executable, "-c", "import sys; sys.exit(3)"], '.', py.path.local(self.fn))
+    def test_error(self, out):
+        res = runner.run([sys.executable, "-c", "import sys; sys.exit(3)"], '.', out)
         assert res == 3
 
-    def test_signal(self):
+    def test_signal(self, out):
         if sys.platform == 'win32':
             py.test.skip("no death by signal on windows")
-        res = runner.run([sys.executable, "-c", "import os; os.kill(os.getpid(), 9)"], '.', py.path.local(self.fn))
+        res = runner.run([sys.executable, "-c", "import os; os.kill(os.getpid(), 9)"], '.', out)
         assert res == -9
 
-    def test_timeout(self):
-        res = runner.run([sys.executable, "-c", "while True: pass"], '.', py.path.local(self.fn), timeout=3)
+    def test_timeout(self, out):
+        res = runner.run([sys.executable, "-c", "while True: pass"], '.', out, timeout=3)
         assert res == -999
 
-    def test_timeout_lock(self):
-        res = runner.run([sys.executable, "-c", "import threading; l=threading.Lock(); l.acquire(); l.acquire()"], '.', py.path.local(self.fn), timeout=3)
+    def test_timeout_lock(self, out):
+        res = runner.run([sys.executable, "-c", "import threading; l=threading.Lock(); l.acquire(); l.acquire()"], '.', out, timeout=3)
         assert res == -999
 
-    def test_timeout_syscall(self):
-        res = runner.run([sys.executable, "-c", "import socket; s=s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM); s.bind(('', 0)); s.recv(1000)"], '.', py.path.local(self.fn), timeout=3)
+    def test_timeout_syscall(self, out):
+        res = runner.run([sys.executable, "-c", "import socket; s=s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM); s.bind(('', 0)); s.recv(1000)"], '.', out, timeout=3)
         assert res == -999        
 
-    def test_timeout_success(self):
+    def test_timeout_success(self, out):
         res = runner.run([sys.executable, "-c", "print 42"], '.',
-                         py.path.local(self.fn), timeout=2)
+                         out, timeout=2)
         assert res == 0
-        out = py.path.local(self.fn).read('r')
+        out = out.read()
         assert out == "42\n"        
 
 

From noreply at buildbot.pypy.org  Sat Jan 21 21:40:09 2012
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Sat, 21 Jan 2012 21:40:09 +0100 (CET)
Subject: [pypy-commit] pypy pytest: fix the remaining testrunner issues
Message-ID: <20120121204009.418FA82D45@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt 
Branch: pytest
Changeset: r51610:d423bf345f99
Date: 2012-01-21 21:39 +0100
http://bitbucket.org/pypy/pypy/changeset/d423bf345f99/

Log:	fix the remaining testrunner issues

diff --git a/testrunner/test/test_runner.py b/testrunner/test/test_runner.py
--- a/testrunner/test/test_runner.py
+++ b/testrunner/test/test_runner.py
@@ -117,6 +117,7 @@
 
         expected = ['INTERP', 'IARG',
                     'driver', 'darg',
+                    '-p', 'resultlog',
                     '--resultlog=LOGFILE',
                     'test_one']
 
@@ -133,6 +134,7 @@
 
         expected = ['/wd' + os.sep + './INTERP', 'IARG',
                     'driver', 'darg',
+                    '-p', 'resultlog',
                     '--resultlog=LOGFILE',
                     'test_one']
 
@@ -246,7 +248,7 @@
         assert '\n' in log        
         log_lines = log.splitlines()
 
-        assert log_lines[0] == ". test_normal/test_example.py:test_one"
+        assert ". test_normal/test_example.py::test_one" in log_lines
         nfailures = 0
         noutcomes = 0
         for line in log_lines:

From noreply at buildbot.pypy.org  Sat Jan 21 22:58:57 2012
From: noreply at buildbot.pypy.org (rguillebert)
Date: Sat, 21 Jan 2012 22:58:57 +0100 (CET)
Subject: [pypy-commit] pypy py3k: Move the kwonly arguments test to the
	interpreter
Message-ID: <20120121215857.475F582D45@wyvern.cs.uni-duesseldorf.de>

Author: Romain Guillebert 
Branch: py3k
Changeset: r51611:31e2e6510fba
Date: 2012-01-21 22:58 +0100
http://bitbucket.org/pypy/pypy/changeset/31e2e6510fba/

Log:	Move the kwonly arguments test to the interpreter

diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -810,15 +810,6 @@
         """
         yield self.st, func, "f()", (1, [2, 3, 4], 5)
 
-    def test_kwonlyargs_default_parameters(self):
-        """ This actually test an interpreter bug, but since we can't parse
-        py3k only code in the interpreter tests right now, it's there waiting
-        for this feature"""
-        func = """ def f(a, b, c=3, *, d=4):
-            return a, b, c, d
-        """
-        yield self.st, func, "f(1, 2)", (1, 2, 3, 4)
-
 class AppTestCompiler:
 
     def test_docstring_not_loaded(self):
diff --git a/pypy/interpreter/test/test_interpreter.py b/pypy/interpreter/test/test_interpreter.py
--- a/pypy/interpreter/test/test_interpreter.py
+++ b/pypy/interpreter/test/test_interpreter.py
@@ -240,6 +240,12 @@
             '''
         assert self.codetest(code, 'f', []) == os.name
 
+    def test_kwonlyargs_default_parameters(self):
+        code = """ def f(a, b, c=3, *, d=4):
+            return a, b, c, d
+        """
+        assert self.codetest(code, "f", [1, 2]) == (1, 2, 3, 4)
+
 
 class AppTestInterpreter: 
     def test_trivial(self):

From noreply at buildbot.pypy.org  Sat Jan 21 23:11:15 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Sat, 21 Jan 2012 23:11:15 +0100 (CET)
Subject: [pypy-commit] pypy numpy-back-to-applevel: start adding necessary
 things for using applevel __str__ and __repr__
Message-ID: <20120121221115.6A78582D45@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: numpy-back-to-applevel
Changeset: r51612:616ba3da7de7
Date: 2012-01-21 21:46 +0200
http://bitbucket.org/pypy/pypy/changeset/616ba3da7de7/

Log:	start adding necessary things for using applevel __str__ and
	__repr__

diff --git a/lib_pypy/numpypy/core/__init__.py b/lib_pypy/numpypy/core/__init__.py
--- a/lib_pypy/numpypy/core/__init__.py
+++ b/lib_pypy/numpypy/core/__init__.py
@@ -1,1 +1,2 @@
 from .fromnumeric import *
+from .numeric import *
diff --git a/lib_pypy/numpypy/core/arrayprint.py b/lib_pypy/numpypy/core/arrayprint.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/numpypy/core/arrayprint.py
@@ -0,0 +1,785 @@
+"""Array printing function
+
+$Id: arrayprint.py,v 1.9 2005/09/13 13:58:44 teoliphant Exp $
+"""
+__all__ = ["array2string", "set_printoptions", "get_printoptions"]
+__docformat__ = 'restructuredtext'
+
+#
+# Written by Konrad Hinsen 
+# last revision: 1996-3-13
+# modified by Jim Hugunin 1997-3-3 for repr's and str's (and other details)
+# and by Perry Greenfield 2000-4-1 for numarray
+# and by Travis Oliphant  2005-8-22 for numpy
+
+import sys
+import _numpypy as _nt
+from _numpypy import maximum, minimum, absolute, not_equal #, isnan, isinf
+#from _numpypy import format_longfloat, datetime_as_string, datetime_data, isna
+from fromnumeric import ravel
+
+
+def product(x, y): return x*y
+
+_summaryEdgeItems = 3     # repr N leading and trailing items of each dimension
+_summaryThreshold = 1000 # total items > triggers array summarization
+
+_float_output_precision = 8
+_float_output_suppress_small = False
+_line_width = 75
+_nan_str = 'nan'
+_inf_str = 'inf'
+_na_str = 'NA'
+_formatter = None  # formatting function for array elements
+
+if sys.version_info[0] >= 3:
+    from functools import reduce
+
+def set_printoptions(precision=None, threshold=None, edgeitems=None,
+                     linewidth=None, suppress=None,
+                     nanstr=None, infstr=None, nastr=None,
+                     formatter=None):
+    """
+    Set printing options.
+
+    These options determine the way floating point numbers, arrays and
+    other NumPy objects are displayed.
+
+    Parameters
+    ----------
+    precision : int, optional
+        Number of digits of precision for floating point output (default 8).
+    threshold : int, optional
+        Total number of array elements which trigger summarization
+        rather than full repr (default 1000).
+    edgeitems : int, optional
+        Number of array items in summary at beginning and end of
+        each dimension (default 3).
+    linewidth : int, optional
+        The number of characters per line for the purpose of inserting
+        line breaks (default 75).
+    suppress : bool, optional
+        Whether or not suppress printing of small floating point values
+        using scientific notation (default False).
+    nanstr : str, optional
+        String representation of floating point not-a-number (default nan).
+    infstr : str, optional
+        String representation of floating point infinity (default inf).
+    nastr : str, optional
+        String representation of NA missing value (default NA).
+    formatter : dict of callables, optional
+        If not None, the keys should indicate the type(s) that the respective
+        formatting function applies to.  Callables should return a string.
+        Types that are not specified (by their corresponding keys) are handled
+        by the default formatters.  Individual types for which a formatter
+        can be set are::
+
+            - 'bool'
+            - 'int'
+            - 'timedelta' : a `numpy.timedelta64`
+            - 'datetime' : a `numpy.datetime64`
+            - 'float'
+            - 'longfloat' : 128-bit floats
+            - 'complexfloat'
+            - 'longcomplexfloat' : composed of two 128-bit floats
+            - 'numpy_str' : types `numpy.string_` and `numpy.unicode_`
+            - 'str' : all other strings
+
+        Other keys that can be used to set a group of types at once are::
+
+            - 'all' : sets all types
+            - 'int_kind' : sets 'int'
+            - 'float_kind' : sets 'float' and 'longfloat'
+            - 'complex_kind' : sets 'complexfloat' and 'longcomplexfloat'
+            - 'str_kind' : sets 'str' and 'numpystr'
+
+    See Also
+    --------
+    get_printoptions, set_string_function, array2string
+
+    Notes
+    -----
+    `formatter` is always reset with a call to `set_printoptions`.
+
+    Examples
+    --------
+    Floating point precision can be set:
+
+    >>> np.set_printoptions(precision=4)
+    >>> print np.array([1.123456789])
+    [ 1.1235]
+
+    Long arrays can be summarised:
+
+    >>> np.set_printoptions(threshold=5)
+    >>> print np.arange(10)
+    [0 1 2 ..., 7 8 9]
+
+    Small results can be suppressed:
+
+    >>> eps = np.finfo(float).eps
+    >>> x = np.arange(4.)
+    >>> x**2 - (x + eps)**2
+    array([ -4.9304e-32,  -4.4409e-16,   0.0000e+00,   0.0000e+00])
+    >>> np.set_printoptions(suppress=True)
+    >>> x**2 - (x + eps)**2
+    array([-0., -0.,  0.,  0.])
+
+    A custom formatter can be used to display array elements as desired:
+
+    >>> np.set_printoptions(formatter={'all':lambda x: 'int: '+str(-x)})
+    >>> x = np.arange(3)
+    >>> x
+    array([int: 0, int: -1, int: -2])
+    >>> np.set_printoptions()  # formatter gets reset
+    >>> x
+    array([0, 1, 2])
+
+    To put back the default options, you can use:
+
+    >>> np.set_printoptions(edgeitems=3,infstr='inf',
+    ... linewidth=75, nanstr='nan', precision=8,
+    ... suppress=False, threshold=1000, formatter=None)
+    """
+
+    global _summaryThreshold, _summaryEdgeItems, _float_output_precision, \
+           _line_width, _float_output_suppress_small, _nan_str, _inf_str, \
+           _na_str, _formatter
+    if linewidth is not None:
+        _line_width = linewidth
+    if threshold is not None:
+        _summaryThreshold = threshold
+    if edgeitems is not None:
+        _summaryEdgeItems = edgeitems
+    if precision is not None:
+        _float_output_precision = precision
+    if suppress is not None:
+        _float_output_suppress_small = not not suppress
+    if nanstr is not None:
+        _nan_str = nanstr
+    if infstr is not None:
+        _inf_str = infstr
+    if nastr is not None:
+        _na_str = nastr
+    _formatter = formatter
+
+def get_printoptions():
+    """
+    Return the current print options.
+
+    Returns
+    -------
+    print_opts : dict
+        Dictionary of current print options with keys
+
+          - precision : int
+          - threshold : int
+          - edgeitems : int
+          - linewidth : int
+          - suppress : bool
+          - nanstr : str
+          - infstr : str
+          - formatter : dict of callables
+
+        For a full description of these options, see `set_printoptions`.
+
+    See Also
+    --------
+    set_printoptions, set_string_function
+
+    """
+    d = dict(precision=_float_output_precision,
+             threshold=_summaryThreshold,
+             edgeitems=_summaryEdgeItems,
+             linewidth=_line_width,
+             suppress=_float_output_suppress_small,
+             nanstr=_nan_str,
+             infstr=_inf_str,
+             nastr=_na_str,
+             formatter=_formatter)
+    return d
+
+def _leading_trailing(a):
+    import numeric as _nc
+    if a.ndim == 1:
+        if len(a) > 2*_summaryEdgeItems:
+            b = _nc.concatenate((a[:_summaryEdgeItems],
+                                     a[-_summaryEdgeItems:]))
+        else:
+            b = a
+    else:
+        if len(a) > 2*_summaryEdgeItems:
+            l = [_leading_trailing(a[i]) for i in range(
+                min(len(a), _summaryEdgeItems))]
+            l.extend([_leading_trailing(a[-i]) for i in range(
+                min(len(a), _summaryEdgeItems),0,-1)])
+        else:
+            l = [_leading_trailing(a[i]) for i in range(0, len(a))]
+        b = _nc.concatenate(tuple(l))
+    return b
+
+def _boolFormatter(x):
+    if isna(x):
+        return str(x).replace('NA', _na_str, 1)
+    elif x:
+        return ' True'
+    else:
+        return 'False'
+
+
+def repr_format(x):
+    if isna(x):
+        return str(x).replace('NA', _na_str, 1)
+    else:
+        return repr(x)
+
+def _array2string(a, max_line_width, precision, suppress_small, separator=' ',
+                  prefix="", formatter=None):
+
+    if max_line_width is None:
+        max_line_width = _line_width
+
+    if precision is None:
+        precision = _float_output_precision
+
+    if suppress_small is None:
+        suppress_small = _float_output_suppress_small
+
+    if formatter is None:
+        formatter = _formatter
+
+    if a.size > _summaryThreshold:
+        summary_insert = "..., "
+        data = _leading_trailing(a)
+    else:
+        summary_insert = ""
+        data = ravel(a)
+
+    formatdict = {'bool' : _boolFormatter,
+                  'int' : IntegerFormat(data),
+                  'float' : FloatFormat(data, precision, suppress_small),
+                  'longfloat' : LongFloatFormat(precision),
+                  'complexfloat' : ComplexFormat(data, precision,
+                                                 suppress_small),
+                  'longcomplexfloat' : LongComplexFormat(precision),
+                  'datetime' : DatetimeFormat(data),
+                  'timedelta' : TimedeltaFormat(data),
+                  'numpystr' : repr_format,
+                  'str' : str}
+
+    if formatter is not None:
+        fkeys = [k for k in formatter.keys() if formatter[k] is not None]
+        if 'all' in fkeys:
+            for key in formatdict.keys():
+                formatdict[key] = formatter['all']
+        if 'int_kind' in fkeys:
+            for key in ['int']:
+                formatdict[key] = formatter['int_kind']
+        if 'float_kind' in fkeys:
+            for key in ['float', 'longfloat']:
+                formatdict[key] = formatter['float_kind']
+        if 'complex_kind' in fkeys:
+            for key in ['complexfloat', 'longcomplexfloat']:
+                formatdict[key] = formatter['complex_kind']
+        if 'str_kind' in fkeys:
+            for key in ['numpystr', 'str']:
+                formatdict[key] = formatter['str_kind']
+        for key in formatdict.keys():
+            if key in fkeys:
+                formatdict[key] = formatter[key]
+
+    try:
+        format_function = a._format
+        msg = "The `_format` attribute is deprecated in Numpy 2.0 and " \
+              "will be removed in 2.1. Use the `formatter` kw instead."
+        import warnings
+        warnings.warn(msg, DeprecationWarning)
+    except AttributeError:
+        # find the right formatting function for the array
+        dtypeobj = a.dtype.type
+        if issubclass(dtypeobj, _nt.bool_):
+            format_function = formatdict['bool']
+        elif issubclass(dtypeobj, _nt.integer):
+            if issubclass(dtypeobj, _nt.timedelta64):
+                format_function = formatdict['timedelta']
+            else:
+                format_function = formatdict['int']
+        elif issubclass(dtypeobj, _nt.floating):
+            if issubclass(dtypeobj, _nt.longfloat):
+                format_function = formatdict['longfloat']
+            else:
+                format_function = formatdict['float']
+        elif issubclass(dtypeobj, _nt.complexfloating):
+            if issubclass(dtypeobj, _nt.clongfloat):
+                format_function = formatdict['longcomplexfloat']
+            else:
+                format_function = formatdict['complexfloat']
+        elif issubclass(dtypeobj, (_nt.unicode_, _nt.string_)):
+            format_function = formatdict['numpystr']
+        elif issubclass(dtypeobj, _nt.datetime64):
+            format_function = formatdict['datetime']
+        else:
+            format_function = formatdict['str']
+
+    # skip over "["
+    next_line_prefix = " "
+    # skip over array(
+    next_line_prefix += " "*len(prefix)
+
+    lst = _formatArray(a, format_function, len(a.shape), max_line_width,
+                       next_line_prefix, separator,
+                       _summaryEdgeItems, summary_insert)[:-1]
+    return lst
+
+def _convert_arrays(obj):
+    import numeric as _nc
+    newtup = []
+    for k in obj:
+        if isinstance(k, _nc.ndarray):
+            k = k.tolist()
+        elif isinstance(k, tuple):
+            k = _convert_arrays(k)
+        newtup.append(k)
+    return tuple(newtup)
+
+
+def array2string(a, max_line_width=None, precision=None,
+                 suppress_small=None, separator=' ', prefix="",
+                 style=repr, formatter=None):
+    """
+    Return a string representation of an array.
+
+    Parameters
+    ----------
+    a : ndarray
+        Input array.
+    max_line_width : int, optional
+        The maximum number of columns the string should span. Newline
+        characters splits the string appropriately after array elements.
+    precision : int, optional
+        Floating point precision. Default is the current printing
+        precision (usually 8), which can be altered using `set_printoptions`.
+    suppress_small : bool, optional
+        Represent very small numbers as zero. A number is "very small" if it
+        is smaller than the current printing precision.
+    separator : str, optional
+        Inserted between elements.
+    prefix : str, optional
+        An array is typically printed as::
+
+          'prefix(' + array2string(a) + ')'
+
+        The length of the prefix string is used to align the
+        output correctly.
+    style : function, optional
+        A function that accepts an ndarray and returns a string.  Used only
+        when the shape of `a` is equal to ``()``, i.e. for 0-D arrays.
+    formatter : dict of callables, optional
+        If not None, the keys should indicate the type(s) that the respective
+        formatting function applies to.  Callables should return a string.
+        Types that are not specified (by their corresponding keys) are handled
+        by the default formatters.  Individual types for which a formatter
+        can be set are::
+
+            - 'bool'
+            - 'int'
+            - 'timedelta' : a `numpy.timedelta64`
+            - 'datetime' : a `numpy.datetime64`
+            - 'float'
+            - 'longfloat' : 128-bit floats
+            - 'complexfloat'
+            - 'longcomplexfloat' : composed of two 128-bit floats
+            - 'numpy_str' : types `numpy.string_` and `numpy.unicode_`
+            - 'str' : all other strings
+
+        Other keys that can be used to set a group of types at once are::
+
+            - 'all' : sets all types
+            - 'int_kind' : sets 'int'
+            - 'float_kind' : sets 'float' and 'longfloat'
+            - 'complex_kind' : sets 'complexfloat' and 'longcomplexfloat'
+            - 'str_kind' : sets 'str' and 'numpystr'
+
+    Returns
+    -------
+    array_str : str
+        String representation of the array.
+
+    Raises
+    ------
+    TypeError : if a callable in `formatter` does not return a string.
+
+    See Also
+    --------
+    array_str, array_repr, set_printoptions, get_printoptions
+
+    Notes
+    -----
+    If a formatter is specified for a certain type, the `precision` keyword is
+    ignored for that type.
+
+    Examples
+    --------
+    >>> x = np.array([1e-16,1,2,3])
+    >>> print np.array2string(x, precision=2, separator=',',
+    ...                       suppress_small=True)
+    [ 0., 1., 2., 3.]
+
+    >>> x  = np.arange(3.)
+    >>> np.array2string(x, formatter={'float_kind':lambda x: "%.2f" % x})
+    '[0.00 1.00 2.00]'
+
+    >>> x  = np.arange(3)
+    >>> np.array2string(x, formatter={'int':lambda x: hex(x)})
+    '[0x0L 0x1L 0x2L]'
+
+    """
+
+    if a.shape == ():
+        x = a.item()
+        if isna(x):
+            lst = str(x).replace('NA', _na_str, 1)
+        else:
+            try:
+                lst = a._format(x)
+                msg = "The `_format` attribute is deprecated in Numpy " \
+                      "2.0 and will be removed in 2.1. Use the " \
+                      "`formatter` kw instead."
+                import warnings
+                warnings.warn(msg, DeprecationWarning)
+            except AttributeError:
+                if isinstance(x, tuple):
+                    x = _convert_arrays(x)
+                lst = style(x)
+    elif reduce(product, a.shape) == 0:
+        # treat as a null array if any of shape elements == 0
+        lst = "[]"
+    else:
+        lst = _array2string(a, max_line_width, precision, suppress_small,
+                            separator, prefix, formatter=formatter)
+    return lst
+
+def _extendLine(s, line, word, max_line_len, next_line_prefix):
+    if len(line.rstrip()) + len(word.rstrip()) >= max_line_len:
+        s += line.rstrip() + "\n"
+        line = next_line_prefix
+    line += word
+    return s, line
+
+
+def _formatArray(a, format_function, rank, max_line_len,
+                 next_line_prefix, separator, edge_items, summary_insert):
+    """formatArray is designed for two modes of operation:
+
+    1. Full output
+
+    2. Summarized output
+
+    """
+    if rank == 0:
+        obj = a.item()
+        if isinstance(obj, tuple):
+            obj = _convert_arrays(obj)
+        return str(obj)
+
+    if summary_insert and 2*edge_items < len(a):
+        leading_items, trailing_items, summary_insert1 = \
+                       edge_items, edge_items, summary_insert
+    else:
+        leading_items, trailing_items, summary_insert1 = 0, len(a), ""
+
+    if rank == 1:
+        s = ""
+        line = next_line_prefix
+        for i in xrange(leading_items):
+            word = format_function(a[i]) + separator
+            s, line = _extendLine(s, line, word, max_line_len, next_line_prefix)
+
+        if summary_insert1:
+            s, line = _extendLine(s, line, summary_insert1, max_line_len, next_line_prefix)
+
+        for i in xrange(trailing_items, 1, -1):
+            word = format_function(a[-i]) + separator
+            s, line = _extendLine(s, line, word, max_line_len, next_line_prefix)
+
+        word = format_function(a[-1])
+        s, line = _extendLine(s, line, word, max_line_len, next_line_prefix)
+        s += line + "]\n"
+        s = '[' + s[len(next_line_prefix):]
+    else:
+        s = '['
+        sep = separator.rstrip()
+        for i in xrange(leading_items):
+            if i > 0:
+                s += next_line_prefix
+            s += _formatArray(a[i], format_function, rank-1, max_line_len,
+                              " " + next_line_prefix, separator, edge_items,
+                              summary_insert)
+            s = s.rstrip() + sep.rstrip() + '\n'*max(rank-1,1)
+
+        if summary_insert1:
+            s += next_line_prefix + summary_insert1 + "\n"
+
+        for i in xrange(trailing_items, 1, -1):
+            if leading_items or i != trailing_items:
+                s += next_line_prefix
+            s += _formatArray(a[-i], format_function, rank-1, max_line_len,
+                              " " + next_line_prefix, separator, edge_items,
+                              summary_insert)
+            s = s.rstrip() + sep.rstrip() + '\n'*max(rank-1,1)
+        if leading_items or trailing_items > 1:
+            s += next_line_prefix
+        s += _formatArray(a[-1], format_function, rank-1, max_line_len,
+                          " " + next_line_prefix, separator, edge_items,
+                          summary_insert).rstrip()+']\n'
+    return s
+
+class FloatFormat(object):
+    def __init__(self, data, precision, suppress_small, sign=False):
+        self.precision = precision
+        self.suppress_small = suppress_small
+        self.sign = sign
+        self.exp_format = False
+        self.large_exponent = False
+        self.max_str_len = 0
+        try:
+            self.fillFormat(data)
+        except (TypeError, NotImplementedError):
+            # if reduce(data) fails, this instance will not be called, just
+            # instantiated in formatdict.
+            pass
+
+    def fillFormat(self, data):
+        import numeric as _nc
+        errstate = _nc.seterr(all='ignore')
+        try:
+            special = isnan(data) | isinf(data) | isna(data)
+            special[isna(data)] = False
+            valid = not_equal(data, 0) & ~special
+            valid[isna(data)] = False
+            non_zero = absolute(data.compress(valid))
+            if len(non_zero) == 0:
+                max_val = 0.
+                min_val = 0.
+            else:
+                max_val = maximum.reduce(non_zero, skipna=True)
+                min_val = minimum.reduce(non_zero, skipna=True)
+                if max_val >= 1.e8:
+                    self.exp_format = True
+                if not self.suppress_small and (min_val < 0.0001
+                                           or max_val/min_val > 1000.):
+                    self.exp_format = True
+        finally:
+            _nc.seterr(**errstate)
+
+        if self.exp_format:
+            self.large_exponent = 0 < min_val < 1e-99 or max_val >= 1e100
+            self.max_str_len = 8 + self.precision
+            if self.large_exponent:
+                self.max_str_len += 1
+            if self.sign:
+                format = '%+'
+            else:
+                format = '%'
+            format = format + '%d.%de' % (self.max_str_len, self.precision)
+        else:
+            format = '%%.%df' % (self.precision,)
+            if len(non_zero):
+                precision = max([_digits(x, self.precision, format)
+                                 for x in non_zero])
+            else:
+                precision = 0
+            precision = min(self.precision, precision)
+            self.max_str_len = len(str(int(max_val))) + precision + 2
+            if _nc.any(special):
+                self.max_str_len = max(self.max_str_len,
+                                       len(_nan_str),
+                                       len(_inf_str)+1,
+                                       len(_na_str))
+            if self.sign:
+                format = '%#+'
+            else:
+                format = '%#'
+            format = format + '%d.%df' % (self.max_str_len, precision)
+
+        self.special_fmt = '%%%ds' % (self.max_str_len,)
+        self.format = format
+
+    def __call__(self, x, strip_zeros=True):
+        import numeric as _nc
+        err = _nc.seterr(invalid='ignore')
+        try:
+            if isna(x):
+                return self.special_fmt % (str(x).replace('NA', _na_str, 1),)
+            elif isnan(x):
+                if self.sign:
+                    return self.special_fmt % ('+' + _nan_str,)
+                else:
+                    return self.special_fmt % (_nan_str,)
+            elif isinf(x):
+                if x > 0:
+                    if self.sign:
+                        return self.special_fmt % ('+' + _inf_str,)
+                    else:
+                        return self.special_fmt % (_inf_str,)
+                else:
+                    return self.special_fmt % ('-' + _inf_str,)
+        finally:
+            _nc.seterr(**err)
+
+        s = self.format % x
+        if self.large_exponent:
+            # 3-digit exponent
+            expsign = s[-3]
+            if expsign == '+' or expsign == '-':
+                s = s[1:-2] + '0' + s[-2:]
+        elif self.exp_format:
+            # 2-digit exponent
+            if s[-3] == '0':
+                s = ' ' + s[:-3] + s[-2:]
+        elif strip_zeros:
+            z = s.rstrip('0')
+            s = z + ' '*(len(s)-len(z))
+        return s
+
+
+def _digits(x, precision, format):
+    s = format % x
+    z = s.rstrip('0')
+    return precision - len(s) + len(z)
+
+
+_MAXINT = sys.maxint
+_MININT = -sys.maxint-1
+class IntegerFormat(object):
+    def __init__(self, data):
+        try:
+            max_str_len = max(len(str(maximum.reduce(data, skipna=True))),
+                              len(str(minimum.reduce(data, skipna=True))))
+            self.format = '%' + str(max_str_len) + 'd'
+        except TypeError, NotImplementedError:
+            # if reduce(data) fails, this instance will not be called, just
+            # instantiated in formatdict.
+            pass
+        except ValueError:
+            # this occurs when everything is NA
+            pass
+
+    def __call__(self, x):
+        if isna(x):
+            return str(x).replace('NA', _na_str, 1)
+        elif _MININT < x < _MAXINT:
+            return self.format % x
+        else:
+            return "%s" % x
+
+class LongFloatFormat(object):
+    # XXX Have to add something to determine the width to use a la FloatFormat
+    # Right now, things won't line up properly
+    def __init__(self, precision, sign=False):
+        self.precision = precision
+        self.sign = sign
+
+    def __call__(self, x):
+        if isna(x):
+            return str(x).replace('NA', _na_str, 1)
+        elif isnan(x):
+            if self.sign:
+                return '+' + _nan_str
+            else:
+                return ' ' + _nan_str
+        elif isinf(x):
+            if x > 0:
+                if self.sign:
+                    return '+' + _inf_str
+                else:
+                    return ' ' + _inf_str
+            else:
+                return '-' + _inf_str
+        elif x >= 0:
+            if self.sign:
+                return '+' + format_longfloat(x, self.precision)
+            else:
+                return ' ' + format_longfloat(x, self.precision)
+        else:
+            return format_longfloat(x, self.precision)
+
+
+class LongComplexFormat(object):
+    def __init__(self, precision):
+        self.real_format = LongFloatFormat(precision)
+        self.imag_format = LongFloatFormat(precision, sign=True)
+
+    def __call__(self, x):
+        if isna(x):
+            return str(x).replace('NA', _na_str, 1)
+        else:
+            r = self.real_format(x.real)
+            i = self.imag_format(x.imag)
+            return r + i + 'j'
+
+
+class ComplexFormat(object):
+    def __init__(self, x, precision, suppress_small):
+        self.real_format = FloatFormat(x.real, precision, suppress_small)
+        self.imag_format = FloatFormat(x.imag, precision, suppress_small,
+                                       sign=True)
+
+    def __call__(self, x):
+        if isna(x):
+            return str(x).replace('NA', _na_str, 1)
+        else:
+            r = self.real_format(x.real, strip_zeros=False)
+            i = self.imag_format(x.imag, strip_zeros=False)
+            if not self.imag_format.exp_format:
+                z = i.rstrip('0')
+                i = z + 'j' + ' '*(len(i)-len(z))
+            else:
+                i = i + 'j'
+            return r + i
+
+class DatetimeFormat(object):
+    def __init__(self, x, unit=None,
+                timezone=None, casting='same_kind'):
+        # Get the unit from the dtype
+        if unit is None:
+            if x.dtype.kind == 'M':
+                unit = datetime_data(x.dtype)[0]
+            else:
+                unit = 's'
+
+        # If timezone is default, make it 'local' or 'UTC' based on the unit
+        if timezone is None:
+            # Date units -> UTC, time units -> local
+            if unit in ('Y', 'M', 'W', 'D'):
+                self.timezone = 'UTC'
+            else:
+                self.timezone = 'local'
+        else:
+            self.timezone = timezone
+        self.unit = unit
+        self.casting = casting
+
+    def __call__(self, x):
+        if isna(x):
+            return str(x).replace('NA', _na_str, 1)
+        else:
+            return "'%s'" % datetime_as_string(x,
+                                        unit=self.unit,
+                                        timezone=self.timezone,
+                                        casting=self.casting)
+
+class TimedeltaFormat(object):
+    def __init__(self, data):
+        if data.dtype.kind == 'm':
+            v = data.view('i8')
+            max_str_len = max(len(str(maximum.reduce(v))),
+                              len(str(minimum.reduce(v))))
+            self.format = '%' + str(max_str_len) + 'd'
+
+    def __call__(self, x):
+        if isna(x):
+            return str(x).replace('NA', _na_str, 1)
+        else:
+            return self.format % x.astype('i8')
+
diff --git a/lib_pypy/numpypy/core/numeric.py b/lib_pypy/numpypy/core/numeric.py
--- a/lib_pypy/numpypy/core/numeric.py
+++ b/lib_pypy/numpypy/core/numeric.py
@@ -1,5 +1,8 @@
 
-from _numpypy import array
+from _numpypy import array, ndarray, int_, float_ #, complex_# , longlong
+import sys
+import _numpypy as multiarray # ARGH
+from numpypy.core.arrayprint import array2string
 
 def asanyarray(a, dtype=None, order=None, maskna=None, ownmaskna=False):
     """
@@ -60,3 +63,245 @@
     """
     return array(a, dtype, copy=False, order=order, subok=True,
                                 maskna=maskna, ownmaskna=ownmaskna)
+
+def base_repr(number, base=2, padding=0):
+    """
+    Return a string representation of a number in the given base system.
+
+    Parameters
+    ----------
+    number : int
+        The value to convert. Only positive values are handled.
+    base : int, optional
+        Convert `number` to the `base` number system. The valid range is 2-36,
+        the default value is 2.
+    padding : int, optional
+        Number of zeros padded on the left. Default is 0 (no padding).
+
+    Returns
+    -------
+    out : str
+        String representation of `number` in `base` system.
+
+    See Also
+    --------
+    binary_repr : Faster version of `base_repr` for base 2.
+
+    Examples
+    --------
+    >>> np.base_repr(5)
+    '101'
+    >>> np.base_repr(6, 5)
+    '11'
+    >>> np.base_repr(7, base=5, padding=3)
+    '00012'
+
+    >>> np.base_repr(10, base=16)
+    'A'
+    >>> np.base_repr(32, base=16)
+    '20'
+
+    """
+    digits = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ'
+    if base > len(digits):
+        raise ValueError("Bases greater than 36 not handled in base_repr.")
+
+    num = abs(number)
+    res = []
+    while num:
+        res.append(digits[num % base])
+        num //= base
+    if padding:
+        res.append('0' * padding)
+    if number < 0:
+        res.append('-')
+    return ''.join(reversed(res or '0'))
+
+_typelessdata = [int_, float_]#, complex_]
+# XXX
+#if issubclass(intc, int):
+#    _typelessdata.append(intc)
+
+#if issubclass(longlong, int):
+#    _typelessdata.append(longlong)
+
+def array_repr(arr, max_line_width=None, precision=None, suppress_small=None):
+    """
+    Return the string representation of an array.
+
+    Parameters
+    ----------
+    arr : ndarray
+        Input array.
+    max_line_width : int, optional
+        The maximum number of columns the string should span. Newline
+        characters split the string appropriately after array elements.
+    precision : int, optional
+        Floating point precision. Default is the current printing precision
+        (usually 8), which can be altered using `set_printoptions`.
+    suppress_small : bool, optional
+        Represent very small numbers as zero, default is False. Very small
+        is defined by `precision`, if the precision is 8 then
+        numbers smaller than 5e-9 are represented as zero.
+
+    Returns
+    -------
+    string : str
+      The string representation of an array.
+
+    See Also
+    --------
+    array_str, array2string, set_printoptions
+
+    Examples
+    --------
+    >>> np.array_repr(np.array([1,2]))
+    'array([1, 2])'
+    >>> np.array_repr(np.ma.array([0.]))
+    'MaskedArray([ 0.])'
+    >>> np.array_repr(np.array([], np.int32))
+    'array([], dtype=int32)'
+
+    >>> x = np.array([1e-6, 4e-7, 2, 3])
+    >>> np.array_repr(x, precision=6, suppress_small=True)
+    'array([ 0.000001,  0.      ,  2.      ,  3.      ])'
+
+    """
+    if arr.size > 0 or arr.shape==(0,):
+        lst = array2string(arr, max_line_width, precision, suppress_small,
+                           ', ', "array(")
+    else: # show zero-length shape unless it is (0,)
+        lst = "[], shape=%s" % (repr(arr.shape),)
+
+    if arr.__class__ is not ndarray:
+        cName= arr.__class__.__name__
+    else:
+        cName = "array"
+
+    skipdtype = (arr.dtype.type in _typelessdata) and arr.size > 0
+
+    if arr.flags.maskna:
+        whichna = isna(arr)
+        # If nothing is NA, explicitly signal the NA-mask
+        if not any(whichna):
+            lst += ", maskna=True"
+        # If everything is NA, can't skip the dtype
+        if skipdtype and all(whichna):
+            skipdtype = False
+
+    if skipdtype:
+        return "%s(%s)" % (cName, lst)
+    else:
+        typename = arr.dtype.name
+        # Quote typename in the output if it is "complex".
+        if typename and not (typename[0].isalpha() and typename.isalnum()):
+            typename = "'%s'" % typename
+
+        lf = ''
+        if 0 or issubclass(arr.dtype.type, flexible):
+            if arr.dtype.names:
+                typename = "%s" % str(arr.dtype)
+            else:
+                typename = "'%s'" % str(arr.dtype)
+            lf = '\n'+' '*len("array(")
+        return cName + "(%s, %sdtype=%s)" % (lst, lf, typename)
+
+def array_str(a, max_line_width=None, precision=None, suppress_small=None):
+    """
+    Return a string representation of the data in an array.
+
+    The data in the array is returned as a single string.  This function is
+    similar to `array_repr`, the difference being that `array_repr` also
+    returns information on the kind of array and its data type.
+
+    Parameters
+    ----------
+    a : ndarray
+        Input array.
+    max_line_width : int, optional
+        Inserts newlines if text is longer than `max_line_width`.  The
+        default is, indirectly, 75.
+    precision : int, optional
+        Floating point precision.  Default is the current printing precision
+        (usually 8), which can be altered using `set_printoptions`.
+    suppress_small : bool, optional
+        Represent numbers "very close" to zero as zero; default is False.
+        Very close is defined by precision: if the precision is 8, e.g.,
+        numbers smaller (in absolute value) than 5e-9 are represented as
+        zero.
+
+    See Also
+    --------
+    array2string, array_repr, set_printoptions
+
+    Examples
+    --------
+    >>> np.array_str(np.arange(3))
+    '[0 1 2]'
+
+    """
+    return array2string(a, max_line_width, precision, suppress_small, ' ', "", str)
+
+def set_string_function(f, repr=True):
+    """
+    Set a Python function to be used when pretty printing arrays.
+
+    Parameters
+    ----------
+    f : function or None
+        Function to be used to pretty print arrays. The function should expect
+        a single array argument and return a string of the representation of
+        the array. If None, the function is reset to the default NumPy function
+        to print arrays.
+    repr : bool, optional
+        If True (default), the function for pretty printing (``__repr__``)
+        is set, if False the function that returns the default string
+        representation (``__str__``) is set.
+
+    See Also
+    --------
+    set_printoptions, get_printoptions
+
+    Examples
+    --------
+    >>> def pprint(arr):
+    ...     return 'HA! - What are you going to do now?'
+    ...
+    >>> np.set_string_function(pprint)
+    >>> a = np.arange(10)
+    >>> a
+    HA! - What are you going to do now?
+    >>> print a
+    [0 1 2 3 4 5 6 7 8 9]
+
+    We can reset the function to the default:
+
+    >>> np.set_string_function(None)
+    >>> a
+    array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])
+
+    `repr` affects either pretty printing or normal string representation.
+    Note that ``__repr__`` is still affected by setting ``__str__``
+    because the width of each array element in the returned string becomes
+    equal to the length of the result of ``__str__()``.
+
+    >>> x = np.arange(4)
+    >>> np.set_string_function(lambda x:'random', repr=False)
+    >>> x.__str__()
+    'random'
+    >>> x.__repr__()
+    'array([     0,      1,      2,      3])'
+
+    """
+    if f is None:
+        if repr:
+            return multiarray.set_string_function(array_repr, 1)
+        else:
+            return multiarray.set_string_function(array_str, 0)
+    else:
+        return multiarray.set_string_function(f, repr)
+
+set_string_function(array_str, 0)
+set_string_function(array_repr, 1)
+
+little_endian = (sys.byteorder == 'little')
diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -28,6 +28,8 @@
         'fromstring': 'interp_support.fromstring',
         'flatiter': 'interp_numarray.W_FlatIterator',
 
+        'set_string_function': 'appbridge.set_string_function',
+        
         'count_reduce_items': 'interp_numarray.count_reduce_items',
 
         'True_': 'types.Bool.True',
diff --git a/pypy/module/micronumpy/appbridge.py b/pypy/module/micronumpy/appbridge.py
--- a/pypy/module/micronumpy/appbridge.py
+++ b/pypy/module/micronumpy/appbridge.py
@@ -5,6 +5,8 @@
     w__var = None
     w__std = None
     w_module = None
+    w_array_repr = None
+    w_array_str = None
 
     def __init__(self, space):
         self.w_import = space.appexec([], """():
@@ -24,5 +26,12 @@
             setattr(self, 'w_' + name, w_meth)
         return space.call_function(w_meth, *args)
 
+def set_string_function(space, w_f, w_repr):
+    cache = get_appbridge_cache(space)
+    if space.is_true(w_repr):
+        cache.w_array_repr = w_f
+    else:
+        cache.w_array_str = w_f
+
 def get_appbridge_cache(space):
     return space.fromcache(AppBridgeCache)
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -303,6 +303,12 @@
         return space.wrap(res.build())
 
     def descr_str(self, space):
+        cache = get_appbridge_cache(space)
+        if cache.w_array_str is None:
+            raise OperationError(space.w_RuntimeError, space.wrap(
+                "str function not set"))
+        return space.call_function(cache.w_array_str, self)
+        
         ret = StringBuilder()
         concrete = self.get_concrete_or_scalar()
         concrete.to_str(space, 0, ret, ' ')
diff --git a/pypy/module/test_lib_pypy/numpypy/core/test_numeric.py b/pypy/module/test_lib_pypy/numpypy/core/test_numeric.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/test_lib_pypy/numpypy/core/test_numeric.py
@@ -0,0 +1,21 @@
+
+from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
+
+
+class AppTestBaseRepr(BaseNumpyAppTest):
+    def test_base3(self):
+        from numpypy import base_repr
+        assert base_repr(3**5, 3) == '100000'
+
+    def test_positive(self):
+        from numpypy import base_repr
+        assert base_repr(12, 10) == '12'
+        assert base_repr(12, 10, 4) == '000012'
+        assert base_repr(12, 4) == '30'
+        assert base_repr(3731624803700888, 36) == '10QR0ROFCEW'
+
+    def test_negative(self):
+        from numpypy import base_repr
+        assert base_repr(-12, 10) == '-12'
+        assert base_repr(-12, 10, 4) == '-000012'
+        assert base_repr(-12, 4) == '-30'

From noreply at buildbot.pypy.org  Sat Jan 21 23:11:16 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Sat, 21 Jan 2012 23:11:16 +0100 (CET)
Subject: [pypy-commit] pypy numpy-back-to-applevel: Start working towards
 ravel and applevel array_string, nothing works!
Message-ID: <20120121221116.AE00A82D45@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: numpy-back-to-applevel
Changeset: r51613:eb5dcdf724f3
Date: 2012-01-21 22:44 +0200
http://bitbucket.org/pypy/pypy/changeset/eb5dcdf724f3/

Log:	Start working towards ravel and applevel array_string, nothing
	works!

diff --git a/lib_pypy/numpypy/core/fromnumeric.py b/lib_pypy/numpypy/core/fromnumeric.py
--- a/lib_pypy/numpypy/core/fromnumeric.py
+++ b/lib_pypy/numpypy/core/fromnumeric.py
@@ -1054,8 +1054,9 @@
     array([ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
-
+    if not hasattr(a, 'ravel'):
+        a = numpypy.array(a)
+    return a.ravel(order=order)
 
 def nonzero(a):
     """
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -280,6 +280,11 @@
             "len() of unsized object"))
 
     def descr_repr(self, space):
+        cache = get_appbridge_cache(space)
+        if cache.w_array_repr is None:
+            return space.wrap(self.dump_data())
+        return space.call_function(cache.w_array_repr, self)
+
         res = StringBuilder()
         res.append("array(")
         concrete = self.get_concrete_or_scalar()
@@ -302,11 +307,26 @@
         res.append(")")
         return space.wrap(res.build())
 
+    def dump_data(self):
+        concr = self.get_concrete()
+        i = concr.create_iter()
+        first = True
+        s = StringBuilder()
+        s.append('array([')
+        while not i.done():
+            if first:
+                first = False
+            else:
+                s.append(', ')
+            s.append(concr.dtype.itemtype.str_format(concr.getitem(i.offset)))
+            i = i.next(len(concr.shape))
+        s.append('])')
+        return s.build()
+
     def descr_str(self, space):
         cache = get_appbridge_cache(space)
         if cache.w_array_str is None:
-            raise OperationError(space.w_RuntimeError, space.wrap(
-                "str function not set"))
+            return space.wrap(self.dump_data())
         return space.call_function(cache.w_array_str, self)
         
         ret = StringBuilder()
@@ -546,6 +566,12 @@
         return space.wrap(W_NDimSlice(concrete.start, strides,
                                       backstrides, shape, concrete))
 
+    def descr_ravel(self, space, w_order=None):
+        if not space.is_w(w_order, space.w_None):
+            raise OperationError(space.w_NotImplementedError, space.wrap(
+                "order not implemented"))
+        return self.descr_reshape(space, [space.wrap(-1)])
+
     def descr_get_flatiter(self, space):
         return space.wrap(W_FlatIterator(self))
 
@@ -1251,6 +1277,7 @@
 
     T = GetSetProperty(BaseArray.descr_get_transpose),
     flat = GetSetProperty(BaseArray.descr_get_flatiter),
+    ravel = interp2app(BaseArray.descr_ravel),
 
     mean = interp2app(BaseArray.descr_mean),
     sum = interp2app(BaseArray.descr_sum),
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1382,6 +1382,14 @@
         assert array(x, copy=False) is x
         assert array(x, copy=True) is not x
 
+    def test_ravel(self):
+        from _numpypy import arange
+        assert (arange(3).ravel() == arange(3)).all()
+        assert (arange(6).reshape(2, 3).ravel() == arange(6)).all()
+        print arange(6).reshape(2, 3).T.ravel()
+        assert (arange(6).reshape(2, 3).T.ravel() == [0, 3, 1, 4, 2, 5]).all()
+        
+
 class AppTestSupport(BaseNumpyAppTest):
     def setup_class(cls):
         import struct
diff --git a/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
--- a/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
+++ b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
@@ -137,3 +137,6 @@
         # x = ones((1, 2, 3))
         # assert transpose(x, (1, 0, 2)).shape == (2, 1, 3)
 
+    def test_ravel(self):
+        from numpypy import ravel
+        
diff --git a/pypy/module/test_lib_pypy/numpypy/core/test_numeric.py b/pypy/module/test_lib_pypy/numpypy/core/test_numeric.py
--- a/pypy/module/test_lib_pypy/numpypy/core/test_numeric.py
+++ b/pypy/module/test_lib_pypy/numpypy/core/test_numeric.py
@@ -19,3 +19,8 @@
         assert base_repr(-12, 10) == '-12'
         assert base_repr(-12, 10, 4) == '-000012'
         assert base_repr(-12, 4) == '-30'
+
+class AppTestRepr(BaseNumpyAppTest):
+    def test_repr(self):
+        from numpypy import array
+        assert repr(array([1, 2, 3, 4])) == 'array([1, 2, 3, 4])'

From noreply at buildbot.pypy.org  Sat Jan 21 23:11:17 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Sat, 21 Jan 2012 23:11:17 +0100 (CET)
Subject: [pypy-commit] pypy numpy-back-to-applevel: merge
Message-ID: <20120121221117.EB7F982D45@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: numpy-back-to-applevel
Changeset: r51614:da00788e9d17
Date: 2012-01-22 00:10 +0200
http://bitbucket.org/pypy/pypy/changeset/da00788e9d17/

Log:	merge

diff --git a/pypy/annotation/unaryop.py b/pypy/annotation/unaryop.py
--- a/pypy/annotation/unaryop.py
+++ b/pypy/annotation/unaryop.py
@@ -440,6 +440,12 @@
     def method_popitem(dct):
         return dct.getanyitem('items')
 
+    def method_pop(dct, s_key, s_dfl=None):
+        dct.dictdef.generalize_key(s_key)
+        if s_dfl is not None:
+            dct.dictdef.generalize_value(s_dfl)
+        return dct.dictdef.read_value()
+
     def _can_only_throw(dic, *ignore):
         if dic1.dictdef.dictkey.custom_eq_hash:
             return None    # r_dict: can throw anything
diff --git a/pypy/rpython/lltypesystem/rdict.py b/pypy/rpython/lltypesystem/rdict.py
--- a/pypy/rpython/lltypesystem/rdict.py
+++ b/pypy/rpython/lltypesystem/rdict.py
@@ -323,6 +323,16 @@
         hop.exception_is_here()
         return hop.gendirectcall(ll_popitem, cTUPLE, v_dict)
 
+    def rtype_method_pop(self, hop):
+        if hop.nb_args == 2:
+            v_args = hop.inputargs(self, self.key_repr)
+            target = ll_pop
+        elif hop.nb_args == 3:
+            v_args = hop.inputargs(self, self.key_repr, self.value_repr)
+            target = ll_pop_default
+        hop.exception_is_here()
+        return hop.gendirectcall(target, *v_args)
+
 class __extend__(pairtype(DictRepr, rmodel.Repr)):
 
     def rtype_getitem((r_dict, r_key), hop):
@@ -874,3 +884,18 @@
     r.item1 = recast(ELEM.TO.item1, entry.value)
     _ll_dict_del(dic, i)
     return r
+
+def ll_pop(dic, key):
+    i = ll_dict_lookup(dic, key, dic.keyhash(key))
+    if not i & HIGHEST_BIT:
+        value = ll_get_value(dic, i)
+        _ll_dict_del(dic, i)
+        return value
+    else:
+        raise KeyError
+
+def ll_pop_default(dic, key, dfl):
+    try:
+        return ll_pop(dic, key)
+    except KeyError:
+        return dfl
diff --git a/pypy/rpython/ootypesystem/rdict.py b/pypy/rpython/ootypesystem/rdict.py
--- a/pypy/rpython/ootypesystem/rdict.py
+++ b/pypy/rpython/ootypesystem/rdict.py
@@ -160,6 +160,16 @@
         hop.exception_is_here()
         return hop.gendirectcall(ll_popitem, cTUPLE, v_dict)
 
+    def rtype_method_pop(self, hop):
+        if hop.nb_args == 2:
+            v_args = hop.inputargs(self, self.key_repr)
+            target = ll_pop
+        elif hop.nb_args == 3:
+            v_args = hop.inputargs(self, self.key_repr, self.value_repr)
+            target = ll_pop_default
+        hop.exception_is_here()
+        return hop.gendirectcall(target, *v_args)
+
     def __get_func(self, interp, r_func, fn, TYPE):
         if isinstance(r_func, MethodOfFrozenPBCRepr):
             obj = r_func.r_im_self.convert_const(fn.im_self)
@@ -370,6 +380,20 @@
         return res
     raise KeyError
 
+def ll_pop(d, key):
+    if d.ll_contains(key):
+        value = d.ll_get(key)
+        d.ll_remove(key)
+        return value
+    else:
+        raise KeyError
+
+def ll_pop_default(d, key, dfl):
+    try:
+        return ll_pop(d, key)
+    except KeyError:
+        return dfl
+
 # ____________________________________________________________
 #
 #  Iteration.
diff --git a/pypy/rpython/test/test_rdict.py b/pypy/rpython/test/test_rdict.py
--- a/pypy/rpython/test/test_rdict.py
+++ b/pypy/rpython/test/test_rdict.py
@@ -622,6 +622,26 @@
         res = self.interpret(func, [])
         assert res in [5263, 6352]
 
+    def test_dict_pop(self):
+        def f(n, default):
+            d = {}
+            d[2] = 3
+            d[4] = 5
+            if default == -1:
+                try:
+                    x = d.pop(n)
+                except KeyError:
+                    x = -1
+            else:
+                x = d.pop(n, default)
+            return x * 10 + len(d)
+        res = self.interpret(f, [2, -1])
+        assert res == 31
+        res = self.interpret(f, [3, -1])
+        assert res == -8
+        res = self.interpret(f, [2, 5])
+        assert res == 31
+
 class TestLLtype(BaseTestRdict, LLRtypeMixin):
     def test_dict_but_not_with_char_keys(self):
         def func(i):

From noreply at buildbot.pypy.org  Sat Jan 21 23:17:05 2012
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sat, 21 Jan 2012 23:17:05 +0100 (CET)
Subject: [pypy-commit] extradoc extradoc: english and a conclusion
Message-ID: <20120121221705.B951B82D45@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor 
Branch: extradoc
Changeset: r4050:1b7919a194e1
Date: 2012-01-21 16:16 -0600
http://bitbucket.org/pypy/extradoc/changeset/1b7919a194e1/

Log:	english and a conclusion

diff --git a/blog/draft/pypy-2011.rst b/blog/draft/pypy-2011.rst
--- a/blog/draft/pypy-2011.rst
+++ b/blog/draft/pypy-2011.rst
@@ -3,35 +3,44 @@
 
 Hello everyone.
 
-PyPy development is fast, sometimes very fast. We try to do 3-4 releases a year
-and yet a lot of time when someone complains about the last release being slow
-we usually say "oh, but it's the release, that was AGES ago". That makes
-something as big as summarizing a year next to impossible, but using my
-external IQ amplifiers like internet and hg log, I will try to provide you
-with some semi-serious statistics:
+PyPy's development is fast, sometimes very fast. We try to do 3-4 releases a
+year and still many times when someone complains about the latest release being
+slow we usually say "oh, that release was AGES ago". This makes something as
+big as summarizing a year next to impossible, but using my external IQ
+amplifiers, like internet and hg log, I will try to provide you with some
+semi-serious statistics:
 
-* We made 3 releases last year, boringly, 1.5, 1.6 and 1.7
+* We did 3 releases last year, boringly named, 1.5, 1.6 and 1.7
 
-* Number of visitors on pypy.org grew from about 200 to 800 per day (averaged
-  weekly)
+* The number of visitors on pypy.org grew from about 200 to 800 per day
+  (averaged weekly)
 
 * We spoke at 15 conferences/meetups
+  (XXX: can we put a list in a comment to be sure it's correct)
 
-* We made 10660 commits, or 29 per day (mostly typos ;-)
+* We made 10660 commits, or 29 per day (mostly typos ;-))
 
 * We published 43 blog posts, keeping you entertained almost weekly
 
-* We made PyPy over 2x faster (on a select set of benchmarks),
+* We made PyPy over 2x faster (on a select set of benchmarks), see the
   `1.4 and nightly`_ comparison
 
-* We got 71 new people which did at least one commit in the PyPy repository
+* 71 brand new people contributed at least one commit to the PyPy repository.
   (XXX: this is what I got by doing wc on contributors.txt at 2011-01-01 and
   the current contributors.rst, but we still need to update the latter)
 
-* We made PyPy 17x more compatible (of course this is unmeasurable, so why
-  not just claim it)
+* We made PyPy 17x more compatible (of course this is not measurable, so we're
+  going to claim it!)
 
-* We probably have [XXX insert unicode for infinity] infinitely many times more users now, since noone used
-  1.4 in production
+* We have [XXX insert unicode for infinity] infinitely many times more users
+  now, since no one used 1.4 in production
+
+2011 was a very exciting year for us. But we're pretty sure 2012 is going to be
+even more exciting! So watch this space for more exciting news, try PyPy out on
+your projects, and, as always, we invite you to join us and contribute to PyPy
+in any way you can.
+
+Signed,
+The PyPy Developers
 
 .. _`1.4 and nightly` http://speed.pypy.org/comparison/?exe=1%2B172%2C1%2BL%2Bdefault&ben=1%2C34%2C27%2C2%2C25%2C3%2C4%2C5%2C22%2C6%2C39%2C7%2C8%2C23%2C24%2C9%2C10%2C11%2C12%2C13%2C14%2C15%2C35%2C36%2C37%2C38%2C16%2C28%2C30%2C32%2C29%2C33%2C17%2C18%2C19%2C20&env=1%2C2&hor=true&bas=1%2B172&chart=normal+bars

From noreply at buildbot.pypy.org  Sat Jan 21 23:27:10 2012
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sat, 21 Jan 2012 23:27:10 +0100 (CET)
Subject: [pypy-commit] pypy default: Cosmetic changes in datetime.py,
 to reduce differences with the version in CPython 3.2
Message-ID: <20120121222710.91D3E82D45@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: 
Changeset: r51615:c5fa49a9207e
Date: 2012-01-18 22:38 +0100
http://bitbucket.org/pypy/pypy/changeset/c5fa49a9207e/

Log:	Cosmetic changes in datetime.py, to reduce differences with the
	version in CPython 3.2

diff --git a/lib_pypy/datetime.py b/lib_pypy/datetime.py
--- a/lib_pypy/datetime.py
+++ b/lib_pypy/datetime.py
@@ -13,7 +13,7 @@
 Sources for time zone and DST data: http://www.twinsun.com/tz/tz-link.htm
 
 This was originally copied from the sandbox of the CPython CVS repository.
-Thanks to Tim Peters for suggesting using it. 
+Thanks to Tim Peters for suggesting using it.
 """
 
 import time as _time
@@ -271,6 +271,8 @@
     raise ValueError("%s()=%d, must be in -1439..1439" % (name, offset))
 
 def _check_date_fields(year, month, day):
+    if not isinstance(year, (int, long)):
+        raise TypeError('int expected')
     if not MINYEAR <= year <= MAXYEAR:
         raise ValueError('year must be in %d..%d' % (MINYEAR, MAXYEAR), year)
     if not 1 <= month <= 12:
@@ -280,6 +282,8 @@
         raise ValueError('day must be in 1..%d' % dim, day)
 
 def _check_time_fields(hour, minute, second, microsecond):
+    if not isinstance(hour, (int, long)):
+        raise TypeError('int expected')
     if not 0 <= hour <= 23:
         raise ValueError('hour must be in 0..23', hour)
     if not 0 <= minute <= 59:
@@ -575,15 +579,26 @@
             s = s + ".%06d" % self.__microseconds
         return s
 
-    days = property(lambda self: self.__days, doc="days")
-    seconds = property(lambda self: self.__seconds, doc="seconds")
-    microseconds = property(lambda self: self.__microseconds,
-                            doc="microseconds")
-
     def total_seconds(self):
         return ((self.days * 86400 + self.seconds) * 10**6
                 + self.microseconds) / 1e6
 
+    # Read-only field accessors
+    @property
+    def days(self):
+        """days"""
+        return self.__days
+
+    @property
+    def seconds(self):
+        """seconds"""
+        return self.__seconds
+
+    @property
+    def microseconds(self):
+        """microseconds"""
+        return self.__microseconds
+
     def __add__(self, other):
         if isinstance(other, timedelta):
             # for CPython compatibility, we cannot use
@@ -756,18 +771,19 @@
 
     # Additional constructors
 
+    @classmethod
     def fromtimestamp(cls, t):
         "Construct a date from a POSIX timestamp (like time.time())."
         y, m, d, hh, mm, ss, weekday, jday, dst = _time.localtime(t)
         return cls(y, m, d)
-    fromtimestamp = classmethod(fromtimestamp)
 
+    @classmethod
     def today(cls):
         "Construct a date from time.time()."
         t = _time.time()
         return cls.fromtimestamp(t)
-    today = classmethod(today)
 
+    @classmethod
     def fromordinal(cls, n):
         """Contruct a date from a proleptic Gregorian ordinal.
 
@@ -776,7 +792,6 @@
         """
         y, m, d = _ord2ymd(n)
         return cls(y, m, d)
-    fromordinal = classmethod(fromordinal)
 
     # Conversions to string
 
@@ -799,6 +814,14 @@
         "Format using strftime()."
         return _wrap_strftime(self, fmt, self.timetuple())
 
+    def __format__(self, format):
+        if not isinstance(format, (str, unicode)):
+            raise ValueError("__format__ excepts str or unicode, not %s" %
+                             format.__class__.__name__)
+        if not format:
+            return str(self)
+        return self.strftime(format)
+
     def isoformat(self):
         """Return the date formatted according to ISO.
 
@@ -812,19 +835,21 @@
 
     __str__ = isoformat
 
-    def __format__(self, format):
-        if not isinstance(format, (str, unicode)):
-            raise ValueError("__format__ excepts str or unicode, not %s" %
-                             format.__class__.__name__)
-        if not format:
-            return str(self)
-        return self.strftime(format)
+    # Read-only field accessors
+    @property
+    def year(self):
+        """year (1-9999)"""
+        return self.__year
 
-    # Read-only field accessors
-    year = property(lambda self: self.__year,
-                    doc="year (%d-%d)" % (MINYEAR, MAXYEAR))
-    month = property(lambda self: self.__month, doc="month (1-12)")
-    day = property(lambda self: self.__day, doc="day (1-31)")
+    @property
+    def month(self):
+        """month (1-12)"""
+        return self.__month
+
+    @property
+    def day(self):
+        """day (1-31)"""
+        return self.__day
 
     # Standard conversions, __cmp__, __hash__ (and helpers)
 
@@ -852,7 +877,7 @@
         _check_date_fields(year, month, day)
         return date(year, month, day)
 
-    # Comparisons.
+    # Comparisons of date objects with other.
 
     def __eq__(self, other):
         if isinstance(other, date):
@@ -1126,16 +1151,34 @@
         return self
 
     # Read-only field accessors
-    hour = property(lambda self: self.__hour, doc="hour (0-23)")
-    minute = property(lambda self: self.__minute, doc="minute (0-59)")
-    second = property(lambda self: self.__second, doc="second (0-59)")
-    microsecond = property(lambda self: self.__microsecond,
-                           doc="microsecond (0-999999)")
-    tzinfo = property(lambda self: self._tzinfo, doc="timezone info object")
+    @property
+    def hour(self):
+        """hour (0-23)"""
+        return self.__hour
+
+    @property
+    def minute(self):
+        """minute (0-59)"""
+        return self.__minute
+
+    @property
+    def second(self):
+        """second (0-59)"""
+        return self.__second
+
+    @property
+    def microsecond(self):
+        """microsecond (0-999999)"""
+        return self.__microsecond
+
+    @property
+    def tzinfo(self):
+        """timezone info object"""
+        return self._tzinfo
 
     # Standard conversions, __hash__ (and helpers)
 
-    # Comparisons.
+    # Comparisons of time objects with other.
 
     def __eq__(self, other):
         if isinstance(other, time):
@@ -1255,14 +1298,6 @@
 
     __str__ = isoformat
 
-    def __format__(self, format):
-        if not isinstance(format, (str, unicode)):
-            raise ValueError("__format__ excepts str or unicode, not %s" %
-                             format.__class__.__name__)
-        if not format:
-            return str(self)
-        return self.strftime(format)
-
     def strftime(self, fmt):
         """Format using strftime().  The date part of the timestamp passed
         to underlying strftime should not be used.
@@ -1274,6 +1309,14 @@
                      0, 1, -1)
         return _wrap_strftime(self, fmt, timetuple)
 
+    def __format__(self, format):
+        if not isinstance(format, (str, unicode)):
+            raise ValueError("__format__ excepts str or unicode, not %s" %
+                             format.__class__.__name__)
+        if not format:
+            return str(self)
+        return self.strftime(format)
+
     # Timezone functions
 
     def utcoffset(self):
@@ -1378,9 +1421,11 @@
 time.resolution = timedelta(microseconds=1)
 
 class datetime(date):
+    """datetime(year, month, day[, hour[, minute[, second[, microsecond[,tzinfo]]]]])
 
-    # XXX needs docstrings
-    # See http://www.zope.org/Members/fdrake/DateTimeWiki/TimeZoneInfo
+    The year, month and day arguments are required. tzinfo may be None, or an
+    instance of a tzinfo subclass. The remaining arguments may be ints or longs.
+    """
 
     def __new__(cls, year, month=None, day=None, hour=0, minute=0, second=0,
                 microsecond=0, tzinfo=None):
@@ -1404,13 +1449,32 @@
         return self
 
     # Read-only field accessors
-    hour = property(lambda self: self.__hour, doc="hour (0-23)")
-    minute = property(lambda self: self.__minute, doc="minute (0-59)")
-    second = property(lambda self: self.__second, doc="second (0-59)")
-    microsecond = property(lambda self: self.__microsecond,
-                           doc="microsecond (0-999999)")
-    tzinfo = property(lambda self: self._tzinfo, doc="timezone info object")
+    @property
+    def hour(self):
+        """hour (0-23)"""
+        return self.__hour
 
+    @property
+    def minute(self):
+        """minute (0-59)"""
+        return self.__minute
+
+    @property
+    def second(self):
+        """second (0-59)"""
+        return self.__second
+
+    @property
+    def microsecond(self):
+        """microsecond (0-999999)"""
+        return self.__microsecond
+
+    @property
+    def tzinfo(self):
+        """timezone info object"""
+        return self._tzinfo
+
+    @classmethod
     def fromtimestamp(cls, t, tz=None):
         """Construct a datetime from a POSIX timestamp (like time.time()).
 
@@ -1438,8 +1502,8 @@
         if tz is not None:
             result = tz.fromutc(result)
         return result
-    fromtimestamp = classmethod(fromtimestamp)
 
+    @classmethod
     def utcfromtimestamp(cls, t):
         "Construct a UTC datetime from a POSIX timestamp (like time.time())."
         if 1 - (t % 1.0) < 0.0000005:
@@ -1450,25 +1514,25 @@
         us = int((t % 1.0) * 1000000)
         ss = min(ss, 59)    # clamp out leap seconds if the platform has them
         return cls(y, m, d, hh, mm, ss, us)
-    utcfromtimestamp = classmethod(utcfromtimestamp)
 
     # XXX This is supposed to do better than we *can* do by using time.time(),
     # XXX if the platform supports a more accurate way.  The C implementation
     # XXX uses gettimeofday on platforms that have it, but that isn't
     # XXX available from Python.  So now() may return different results
     # XXX across the implementations.
+    @classmethod
     def now(cls, tz=None):
         "Construct a datetime from time.time() and optional time zone info."
         t = _time.time()
         return cls.fromtimestamp(t, tz)
-    now = classmethod(now)
 
+    @classmethod
     def utcnow(cls):
         "Construct a UTC datetime from time.time()."
         t = _time.time()
         return cls.utcfromtimestamp(t)
-    utcnow = classmethod(utcnow)
 
+    @classmethod
     def combine(cls, date, time):
         "Construct a datetime from a given date and a given time."
         if not isinstance(date, _date_class):
@@ -1478,7 +1542,6 @@
         return cls(date.year, date.month, date.day,
                    time.hour, time.minute, time.second, time.microsecond,
                    time.tzinfo)
-    combine = classmethod(combine)
 
     def timetuple(self):
         "Return local time tuple compatible with time.localtime()."
@@ -1596,13 +1659,13 @@
         return s
 
     def __repr__(self):
-        "Convert to formal string, for repr()."
+        """Convert to formal string, for repr()."""
         L = [self.__year, self.__month, self.__day, # These are never zero
              self.__hour, self.__minute, self.__second, self.__microsecond]
         if L[-1] == 0:
             del L[-1]
         if L[-1] == 0:
-            del L[-1]            
+            del L[-1]
         s = ", ".join(map(str, L))
         s = "%s(%s)" % ('datetime.' + self.__class__.__name__, s)
         if self._tzinfo is not None:
@@ -2009,7 +2072,7 @@
 
 Because we know z.d said z was in daylight time (else [5] would have held and
 we would have stopped then), and we know z.d != z'.d (else [8] would have held
-and we we have stopped then), and there are only 2 possible values dst() can
+and we have stopped then), and there are only 2 possible values dst() can
 return in Eastern, it follows that z'.d must be 0 (which it is in the example,
 but the reasoning doesn't depend on the example -- it depends on there being
 two possible dst() outcomes, one zero and the other non-zero).  Therefore

From noreply at buildbot.pypy.org  Sat Jan 21 23:27:11 2012
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sat, 21 Jan 2012 23:27:11 +0100 (CET)
Subject: [pypy-commit] pypy default: Slowly merge datetime.py from upstream:
 don't use "private" attributes.
Message-ID: <20120121222711.DBB0A82D45@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: 
Changeset: r51616:5aa09e8483d3
Date: 2012-01-18 23:04 +0100
http://bitbucket.org/pypy/pypy/changeset/5aa09e8483d3/

Log:	Slowly merge datetime.py from upstream: don't use "private"
	attributes.

diff --git a/lib_pypy/datetime.py b/lib_pypy/datetime.py
--- a/lib_pypy/datetime.py
+++ b/lib_pypy/datetime.py
@@ -547,36 +547,36 @@
 
         self = object.__new__(cls)
 
-        self.__days = d
-        self.__seconds = s
-        self.__microseconds = us
+        self._days = d
+        self._seconds = s
+        self._microseconds = us
         if abs(d) > 999999999:
             raise OverflowError("timedelta # of days is too large: %d" % d)
 
         return self
 
     def __repr__(self):
-        if self.__microseconds:
+        if self._microseconds:
             return "%s(%d, %d, %d)" % ('datetime.' + self.__class__.__name__,
-                                       self.__days,
-                                       self.__seconds,
-                                       self.__microseconds)
-        if self.__seconds:
+                                       self._days,
+                                       self._seconds,
+                                       self._microseconds)
+        if self._seconds:
             return "%s(%d, %d)" % ('datetime.' + self.__class__.__name__,
-                                   self.__days,
-                                   self.__seconds)
-        return "%s(%d)" % ('datetime.' + self.__class__.__name__, self.__days)
+                                   self._days,
+                                   self._seconds)
+        return "%s(%d)" % ('datetime.' + self.__class__.__name__, self._days)
 
     def __str__(self):
-        mm, ss = divmod(self.__seconds, 60)
+        mm, ss = divmod(self._seconds, 60)
         hh, mm = divmod(mm, 60)
         s = "%d:%02d:%02d" % (hh, mm, ss)
-        if self.__days:
+        if self._days:
             def plural(n):
                 return n, abs(n) != 1 and "s" or ""
-            s = ("%d day%s, " % plural(self.__days)) + s
-        if self.__microseconds:
-            s = s + ".%06d" % self.__microseconds
+            s = ("%d day%s, " % plural(self._days)) + s
+        if self._microseconds:
+            s = s + ".%06d" % self._microseconds
         return s
 
     def total_seconds(self):
@@ -587,32 +587,36 @@
     @property
     def days(self):
         """days"""
-        return self.__days
+        return self._days
 
     @property
     def seconds(self):
         """seconds"""
-        return self.__seconds
+        return self._seconds
 
     @property
     def microseconds(self):
         """microseconds"""
-        return self.__microseconds
+        return self._microseconds
 
     def __add__(self, other):
         if isinstance(other, timedelta):
             # for CPython compatibility, we cannot use
             # our __class__ here, but need a real timedelta
-            return timedelta(self.__days + other.__days,
-                             self.__seconds + other.__seconds,
-                             self.__microseconds + other.__microseconds)
+            return timedelta(self._days + other._days,
+                             self._seconds + other._seconds,
+                             self._microseconds + other._microseconds)
         return NotImplemented
 
     __radd__ = __add__
 
     def __sub__(self, other):
         if isinstance(other, timedelta):
-            return self + -other
+            # for CPython compatibility, we cannot use
+            # our __class__ here, but need a real timedelta
+            return timedelta(self._days - other._days,
+                             self._seconds - other._seconds,
+                             self._microseconds - other._microseconds)
         return NotImplemented
 
     def __rsub__(self, other):
@@ -621,17 +625,17 @@
         return NotImplemented
 
     def __neg__(self):
-            # for CPython compatibility, we cannot use
-            # our __class__ here, but need a real timedelta
-            return timedelta(-self.__days,
-                             -self.__seconds,
-                             -self.__microseconds)
+        # for CPython compatibility, we cannot use
+        # our __class__ here, but need a real timedelta
+        return timedelta(-self._days,
+                         -self._seconds,
+                         -self._microseconds)
 
     def __pos__(self):
         return self
 
     def __abs__(self):
-        if self.__days < 0:
+        if self._days < 0:
             return -self
         else:
             return self
@@ -640,81 +644,81 @@
         if isinstance(other, (int, long)):
             # for CPython compatibility, we cannot use
             # our __class__ here, but need a real timedelta
-            return timedelta(self.__days * other,
-                             self.__seconds * other,
-                             self.__microseconds * other)
+            return timedelta(self._days * other,
+                             self._seconds * other,
+                             self._microseconds * other)
         return NotImplemented
 
     __rmul__ = __mul__
 
     def __div__(self, other):
         if isinstance(other, (int, long)):
-            usec = ((self.__days * (24*3600L) + self.__seconds) * 1000000 +
-                    self.__microseconds)
+            usec = ((self._days * (24*3600L) + self._seconds) * 1000000 +
+                    self._microseconds)
             return timedelta(0, 0, usec // other)
         return NotImplemented
 
     __floordiv__ = __div__
 
-    # Comparisons.
+    # Comparisons of timedelta objects with other.
 
     def __eq__(self, other):
         if isinstance(other, timedelta):
-            return self.__cmp(other) == 0
+            return self._cmp(other) == 0
         else:
             return False
 
     def __ne__(self, other):
         if isinstance(other, timedelta):
-            return self.__cmp(other) != 0
+            return self._cmp(other) != 0
         else:
             return True
 
     def __le__(self, other):
         if isinstance(other, timedelta):
-            return self.__cmp(other) <= 0
+            return self._cmp(other) <= 0
         else:
             _cmperror(self, other)
 
     def __lt__(self, other):
         if isinstance(other, timedelta):
-            return self.__cmp(other) < 0
+            return self._cmp(other) < 0
         else:
             _cmperror(self, other)
 
     def __ge__(self, other):
         if isinstance(other, timedelta):
-            return self.__cmp(other) >= 0
+            return self._cmp(other) >= 0
         else:
             _cmperror(self, other)
 
     def __gt__(self, other):
         if isinstance(other, timedelta):
-            return self.__cmp(other) > 0
+            return self._cmp(other) > 0
         else:
             _cmperror(self, other)
 
-    def __cmp(self, other):
+    def _cmp(self, other):
         assert isinstance(other, timedelta)
-        return cmp(self.__getstate(), other.__getstate())
+        return cmp(self._getstate(), other._getstate())
 
     def __hash__(self):
-        return hash(self.__getstate())
+        return hash(self._getstate())
 
     def __nonzero__(self):
-        return (self.__days != 0 or
-                self.__seconds != 0 or
-                self.__microseconds != 0)
+        return (self._days != 0 or
+                self._seconds != 0 or
+                self._microseconds != 0)
 
     # Pickle support.
 
     __safe_for_unpickling__ = True      # For Python 2.2
 
-    def __getstate(self):
-        return (self.__days, self.__seconds, self.__microseconds)
+    def _getstate(self):
+        return (self._days, self._seconds, self._microseconds)
 
     def __reduce__(self):
-        return (self.__class__, self.__getstate())
+        return (self.__class__, self._getstate())
 
 timedelta.min = timedelta(-999999999)
 timedelta.max = timedelta(days=999999999, hours=23, minutes=59, seconds=59,
@@ -764,9 +768,9 @@
             return self
         _check_date_fields(year, month, day)
         self = object.__new__(cls)
-        self.__year = year
-        self.__month = month
-        self.__day = day
+        self._year = year
+        self._month = month
+        self._day = day
         return self
 
     # Additional constructors
@@ -796,11 +800,20 @@
     # Conversions to string
 
     def __repr__(self):
-        "Convert to formal string, for repr()."
+        """Convert to formal string, for repr().
+
+        >>> dt = datetime(2010, 1, 1)
+        >>> repr(dt)
+        'datetime.datetime(2010, 1, 1, 0, 0)'
+
+        >>> dt = datetime(2010, 1, 1, tzinfo=timezone.utc)
+        >>> repr(dt)
+        'datetime.datetime(2010, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)'
+        """
         return "%s(%d, %d, %d)" % ('datetime.' + self.__class__.__name__,
-                                   self.__year,
-                                   self.__month,
-                                   self.__day)
+                                   self._year,
+                                   self._month,
+                                   self._day)
     # XXX These shouldn't depend on time.localtime(), because that
     # clips the usable dates to [1970 .. 2038).  At least ctime() is
     # easily done without using strftime() -- that's better too because
@@ -808,19 +821,19 @@
 
     def ctime(self):
         "Format a la ctime()."
-        return tmxxx(self.__year, self.__month, self.__day).ctime()
+        return tmxxx(self._year, self._month, self._day).ctime()
 
     def strftime(self, fmt):
         "Format using strftime()."
         return _wrap_strftime(self, fmt, self.timetuple())
 
-    def __format__(self, format):
-        if not isinstance(format, (str, unicode)):
+    def __format__(self, fmt):
+        if not isinstance(fmt, (str, unicode)):
             raise ValueError("__format__ excepts str or unicode, not %s" %
-                             format.__class__.__name__)
-        if not format:
-            return str(self)
-        return self.strftime(format)
+                             fmt.__class__.__name__)
+        if len(fmt) != 0:
+            return self.strftime(fmt)
+        return str(self)
 
     def isoformat(self):
         """Return the date formatted according to ISO.
@@ -831,7 +844,7 @@
         - http://www.w3.org/TR/NOTE-datetime
         - http://www.cl.cam.ac.uk/~mgk25/iso-time.html
         """
-        return "%04d-%02d-%02d" % (self.__year, self.__month, self.__day)
+        return "%04d-%02d-%02d" % (self._year, self._month, self._day)
 
     __str__ = isoformat
 
@@ -839,23 +852,23 @@
     @property
     def year(self):
         """year (1-9999)"""
-        return self.__year
+        return self._year
 
     @property
     def month(self):
         """month (1-12)"""
-        return self.__month
+        return self._month
 
     @property
     def day(self):
         """day (1-31)"""
-        return self.__day
+        return self._day
 
     # Standard conversions, __cmp__, __hash__ (and helpers)
 
     def timetuple(self):
         "Return local time tuple compatible with time.localtime()."
-        return _build_struct_time(self.__year, self.__month, self.__day,
+        return _build_struct_time(self._year, self._month, self._day,
                                   0, 0, 0, -1)
 
     def toordinal(self):
@@ -864,16 +877,16 @@
         January 1 of year 1 is day 1.  Only the year, month and day values
         contribute to the result.
         """
-        return _ymd2ord(self.__year, self.__month, self.__day)
+        return _ymd2ord(self._year, self._month, self._day)
 
     def replace(self, year=None, month=None, day=None):
         """Return a new date with new values for the specified fields."""
         if year is None:
-            year = self.__year
+            year = self._year
         if month is None:
-            month = self.__month
+            month = self._month
         if day is None:
-            day = self.__day
+            day = self._day
         _check_date_fields(year, month, day)
         return date(year, month, day)
 
@@ -881,7 +894,7 @@
 
     def __eq__(self, other):
         if isinstance(other, date):
-            return self.__cmp(other) == 0
+            return self._cmp(other) == 0
         elif hasattr(other, "timetuple"):
             return NotImplemented
         else:
@@ -889,7 +902,7 @@
 
     def __ne__(self, other):
         if isinstance(other, date):
-            return self.__cmp(other) != 0
+            return self._cmp(other) != 0
         elif hasattr(other, "timetuple"):
             return NotImplemented
         else:
@@ -897,7 +910,7 @@
 
     def __le__(self, other):
         if isinstance(other, date):
-            return self.__cmp(other) <= 0
+            return self._cmp(other) <= 0
         elif hasattr(other, "timetuple"):
             return NotImplemented
         else:
@@ -905,7 +918,7 @@
 
     def __lt__(self, other):
         if isinstance(other, date):
-            return self.__cmp(other) < 0
+            return self._cmp(other) < 0
         elif hasattr(other, "timetuple"):
             return NotImplemented
         else:
@@ -913,7 +926,7 @@
 
     def __ge__(self, other):
         if isinstance(other, date):
-            return self.__cmp(other) >= 0
+            return self._cmp(other) >= 0
         elif hasattr(other, "timetuple"):
             return NotImplemented
         else:
@@ -921,21 +934,21 @@
 
     def __gt__(self, other):
         if isinstance(other, date):
-            return self.__cmp(other) > 0
+            return self._cmp(other) > 0
         elif hasattr(other, "timetuple"):
             return NotImplemented
         else:
             _cmperror(self, other)
 
-    def __cmp(self, other):
+    def _cmp(self, other):
         assert isinstance(other, date)
-        y, m, d = self.__year, self.__month, self.__day
-        y2, m2, d2 = other.__year, other.__month, other.__day
+        y, m, d = self._year, self._month, self._day
+        y2, m2, d2 = other._year, other._month, other._day
         return cmp((y, m, d), (y2, m2, d2))
 
     def __hash__(self):
         "Hash."
-        return hash(self.__getstate())
+        return hash(self._getstate())
 
     # Computations
 
@@ -947,9 +960,9 @@
     def __add__(self, other):
         "Add a date to a timedelta."
         if isinstance(other, timedelta):
-            t = tmxxx(self.__year,
-                      self.__month,
-                      self.__day + other.days)
+            t = tmxxx(self._year,
+                      self._month,
+                      self._day + other.days)
             self._checkOverflow(t.year)
             result = date(t.year, t.month, t.day)
             return result
@@ -991,9 +1004,9 @@
         ISO calendar algorithm taken from
         http://www.phys.uu.nl/~vgent/calendar/isocalendar.htm
         """
-        year = self.__year
+        year = self._year
         week1monday = _isoweek1monday(year)
-        today = _ymd2ord(self.__year, self.__month, self.__day)
+        today = _ymd2ord(self._year, self._month, self._day)
         # Internally, week and day have origin 0
         week, day = divmod(today - week1monday, 7)
         if week < 0:
@@ -1010,18 +1023,18 @@
 
     __safe_for_unpickling__ = True      # For Python 2.2
 
-    def __getstate(self):
-        yhi, ylo = divmod(self.__year, 256)
-        return ("%c%c%c%c" % (yhi, ylo, self.__month, self.__day), )
+    def _getstate(self):
+        yhi, ylo = divmod(self._year, 256)
+        return ("%c%c%c%c" % (yhi, ylo, self._month, self._day), )
 
     def __setstate(self, string):
         if len(string) != 4 or not (1 <= ord(string[2]) <= 12):
             raise TypeError("not enough arguments")
-        yhi, ylo, self.__month, self.__day = map(ord, string)
-        self.__year = yhi * 256 + ylo
+        yhi, ylo, self._month, self._day = map(ord, string)
+        self._year = yhi * 256 + ylo
 
     def __reduce__(self):
-        return (self.__class__, self.__getstate())
+        return (self.__class__, self._getstate())
 
 _date_class = date  # so functions w/ args named "date" can get at the class
 
@@ -1143,10 +1156,10 @@
             return self
         _check_tzinfo_arg(tzinfo)
         _check_time_fields(hour, minute, second, microsecond)
-        self.__hour = hour
-        self.__minute = minute
-        self.__second = second
-        self.__microsecond = microsecond
+        self._hour = hour
+        self._minute = minute
+        self._second = second
+        self._microsecond = microsecond
         self._tzinfo = tzinfo
         return self
 
@@ -1154,22 +1167,22 @@
     @property
     def hour(self):
         """hour (0-23)"""
-        return self.__hour
+        return self._hour
 
     @property
     def minute(self):
         """minute (0-59)"""
-        return self.__minute
+        return self._minute
 
     @property
     def second(self):
         """second (0-59)"""
-        return self.__second
+        return self._second
 
     @property
     def microsecond(self):
         """microsecond (0-999999)"""
-        return self.__microsecond
+        return self._microsecond
 
     @property
     def tzinfo(self):
@@ -1182,41 +1195,41 @@
 
     def __eq__(self, other):
         if isinstance(other, time):
-            return self.__cmp(other) == 0
+            return self._cmp(other) == 0
         else:
             return False
 
     def __ne__(self, other):
         if isinstance(other, time):
-            return self.__cmp(other) != 0
+            return self._cmp(other) != 0
         else:
             return True
 
     def __le__(self, other):
         if isinstance(other, time):
-            return self.__cmp(other) <= 0
+            return self._cmp(other) <= 0
         else:
             _cmperror(self, other)
 
     def __lt__(self, other):
         if isinstance(other, time):
-            return self.__cmp(other) < 0
+            return self._cmp(other) < 0
         else:
             _cmperror(self, other)
 
     def __ge__(self, other):
         if isinstance(other, time):
-            return self.__cmp(other) >= 0
+            return self._cmp(other) >= 0
         else:
             _cmperror(self, other)
 
     def __gt__(self, other):
         if isinstance(other, time):
-            return self.__cmp(other) > 0
+            return self._cmp(other) > 0
         else:
             _cmperror(self, other)
 
-    def __cmp(self, other):
+    def _cmp(self, other):
         assert isinstance(other, time)
         mytz = self._tzinfo
         ottz = other._tzinfo
@@ -1230,23 +1243,23 @@
             base_compare = myoff == otoff
 
         if base_compare:
-            return cmp((self.__hour, self.__minute, self.__second,
-                        self.__microsecond),
-                       (other.__hour, other.__minute, other.__second,
-                        other.__microsecond))
+            return cmp((self._hour, self._minute, self._second,
+                        self._microsecond),
+                       (other._hour, other._minute, other._second,
+                        other._microsecond))
         if myoff is None or otoff is None:
             # XXX Buggy in 2.2.2.
             raise TypeError("cannot compare naive and aware times")
-        myhhmm = self.__hour * 60 + self.__minute - myoff
-        othhmm = other.__hour * 60 + other.__minute - otoff
-        return cmp((myhhmm, self.__second, self.__microsecond),
-                   (othhmm, other.__second, other.__microsecond))
+        myhhmm = self._hour * 60 + self._minute - myoff
+        othhmm = other._hour * 60 + other._minute - otoff
+        return cmp((myhhmm, self._second, self._microsecond),
+                   (othhmm, other._second, other._microsecond))
 
     def __hash__(self):
         """Hash."""
         tzoff = self._utcoffset()
         if not tzoff: # zero or None
-            return hash(self.__getstate()[0])
+            return hash(self._getstate()[0])
         h, m = divmod(self.hour * 60 + self.minute - tzoff, 60)
         if 0 <= h < 24:
             return hash(time(h, m, self.second, self.microsecond))
@@ -1270,14 +1283,14 @@
 
     def __repr__(self):
         """Convert to formal string, for repr()."""
-        if self.__microsecond != 0:
-            s = ", %d, %d" % (self.__second, self.__microsecond)
-        elif self.__second != 0:
-            s = ", %d" % self.__second
+        if self._microsecond != 0:
+            s = ", %d, %d" % (self._second, self._microsecond)
+        elif self._second != 0:
+            s = ", %d" % self._second
         else:
             s = ""
         s= "%s(%d, %d%s)" % ('datetime.' + self.__class__.__name__,
-                             self.__hour, self.__minute, s)
+                             self._hour, self._minute, s)
         if self._tzinfo is not None:
             assert s[-1:] == ")"
             s = s[:-1] + ", tzinfo=%r" % self._tzinfo + ")"
@@ -1289,8 +1302,8 @@
         This is 'HH:MM:SS.mmmmmm+zz:zz', or 'HH:MM:SS+zz:zz' if
         self.microsecond == 0.
         """
-        s = _format_time(self.__hour, self.__minute, self.__second,
-                         self.__microsecond)
+        s = _format_time(self._hour, self._minute, self._second,
+                         self._microsecond)
         tz = self._tzstr()
         if tz:
             s += tz
@@ -1305,17 +1318,17 @@
         # The year must be >= 1900 else Python's strftime implementation
         # can raise a bogus exception.
         timetuple = (1900, 1, 1,
-                     self.__hour, self.__minute, self.__second,
+                     self._hour, self._minute, self._second,
                      0, 1, -1)
         return _wrap_strftime(self, fmt, timetuple)
 
-    def __format__(self, format):
-        if not isinstance(format, (str, unicode)):
+    def __format__(self, fmt):
+        if not isinstance(fmt, (str, unicode)):
             raise ValueError("__format__ excepts str or unicode, not %s" %
-                             format.__class__.__name__)
-        if not format:
-            return str(self)
-        return self.strftime(format)
+                             fmt.__class__.__name__)
+        if len(fmt) != 0:
+            return self.strftime(fmt)
+        return str(self)
 
     # Timezone functions
 
@@ -1393,10 +1406,10 @@
 
     __safe_for_unpickling__ = True      # For Python 2.2
 
-    def __getstate(self):
-        us2, us3 = divmod(self.__microsecond, 256)
+    def _getstate(self):
+        us2, us3 = divmod(self._microsecond, 256)
         us1, us2 = divmod(us2, 256)
-        basestate = ("%c" * 6) % (self.__hour, self.__minute, self.__second,
+        basestate = ("%c" * 6) % (self._hour, self._minute, self._second,
                                   us1, us2, us3)
         if self._tzinfo is None:
             return (basestate,)
@@ -1406,13 +1419,13 @@
     def __setstate(self, string, tzinfo):
         if len(string) != 6 or ord(string[0]) >= 24:
             raise TypeError("an integer is required")
-        self.__hour, self.__minute, self.__second, us1, us2, us3 = \
+        self._hour, self._minute, self._second, us1, us2, us3 = \
                                                             map(ord, string)
-        self.__microsecond = (((us1 << 8) | us2) << 8) | us3
+        self._microsecond = (((us1 << 8) | us2) << 8) | us3
         self._tzinfo = tzinfo
 
     def __reduce__(self):
-        return (time, self.__getstate())
+        return (time, self._getstate())
 
 _time_class = time  # so functions w/ args named "time" can get at the class
 
@@ -1438,13 +1451,13 @@
         _check_time_fields(hour, minute, second, microsecond)
         self = date.__new__(cls, year, month, day)
         # XXX This duplicates __year, __month, __day for convenience :-(
-        self.__year = year
-        self.__month = month
-        self.__day = day
-        self.__hour = hour
-        self.__minute = minute
-        self.__second = second
-        self.__microsecond = microsecond
+        self._year = year
+        self._month = month
+        self._day = day
+        self._hour = hour
+        self._minute = minute
+        self._second = second
+        self._microsecond = microsecond
         self._tzinfo = tzinfo
         return self
 
@@ -1452,22 +1465,22 @@
     @property
     def hour(self):
         """hour (0-23)"""
-        return self.__hour
+        return self._hour
 
     @property
     def minute(self):
         """minute (0-59)"""
-        return self.__minute
+        return self._minute
 
     @property
     def second(self):
         """second (0-59)"""
-        return self.__second
+        return self._second
 
     @property
     def microsecond(self):
         """microsecond (0-999999)"""
-        return self.__microsecond
+        return self._microsecond
 
     @property
     def tzinfo(self):
@@ -1567,7 +1580,7 @@
 
     def date(self):
         "Return the date part."
-        return date(self.__year, self.__month, self.__day)
+        return date(self._year, self._month, self._day)
 
     def time(self):
         "Return the time part, with tzinfo None."
@@ -1627,8 +1640,8 @@
 
     def ctime(self):
         "Format a la ctime()."
-        t = tmxxx(self.__year, self.__month, self.__day, self.__hour,
-                  self.__minute, self.__second)
+        t = tmxxx(self._year, self._month, self._day, self._hour,
+                  self._minute, self._second)
         return t.ctime()
 
     def isoformat(self, sep='T'):
@@ -1643,10 +1656,10 @@
         Optional argument sep specifies the separator between date and
         time, default 'T'.
         """
-        s = ("%04d-%02d-%02d%c" % (self.__year, self.__month, self.__day,
+        s = ("%04d-%02d-%02d%c" % (self._year, self._month, self._day,
                                   sep) +
-                _format_time(self.__hour, self.__minute, self.__second,
-                             self.__microsecond))
+                _format_time(self._hour, self._minute, self._second,
+                             self._microsecond))
         off = self._utcoffset()
         if off is not None:
             if off < 0:
@@ -1660,8 +1673,8 @@
 
     def __repr__(self):
         """Convert to formal string, for repr()."""
-        L = [self.__year, self.__month, self.__day, # These are never zero
-             self.__hour, self.__minute, self.__second, self.__microsecond]
+        L = [self._year, self._month, self._day, # These are never zero
+             self._hour, self._minute, self._second, self._microsecond]
         if L[-1] == 0:
             del L[-1]
         if L[-1] == 0:
@@ -1738,7 +1751,7 @@
 
     def __eq__(self, other):
         if isinstance(other, datetime):
-            return self.__cmp(other) == 0
+            return self._cmp(other) == 0
         elif hasattr(other, "timetuple") and not isinstance(other, date):
             return NotImplemented
         else:
@@ -1746,7 +1759,7 @@
 
     def __ne__(self, other):
         if isinstance(other, datetime):
-            return self.__cmp(other) != 0
+            return self._cmp(other) != 0
         elif hasattr(other, "timetuple") and not isinstance(other, date):
             return NotImplemented
         else:
@@ -1754,7 +1767,7 @@
 
     def __le__(self, other):
         if isinstance(other, datetime):
-            return self.__cmp(other) <= 0
+            return self._cmp(other) <= 0
         elif hasattr(other, "timetuple") and not isinstance(other, date):
             return NotImplemented
         else:
@@ -1762,7 +1775,7 @@
 
     def __lt__(self, other):
         if isinstance(other, datetime):
-            return self.__cmp(other) < 0
+            return self._cmp(other) < 0
         elif hasattr(other, "timetuple") and not isinstance(other, date):
             return NotImplemented
         else:
@@ -1770,7 +1783,7 @@
 
     def __ge__(self, other):
         if isinstance(other, datetime):
-            return self.__cmp(other) >= 0
+            return self._cmp(other) >= 0
         elif hasattr(other, "timetuple") and not isinstance(other, date):
             return NotImplemented
         else:
@@ -1778,13 +1791,13 @@
 
     def __gt__(self, other):
         if isinstance(other, datetime):
-            return self.__cmp(other) > 0
+            return self._cmp(other) > 0
         elif hasattr(other, "timetuple") and not isinstance(other, date):
             return NotImplemented
         else:
             _cmperror(self, other)
 
-    def __cmp(self, other):
+    def _cmp(self, other):
         assert isinstance(other, datetime)
         mytz = self._tzinfo
         ottz = other._tzinfo
@@ -1800,12 +1813,12 @@
             base_compare = myoff == otoff
 
         if base_compare:
-            return cmp((self.__year, self.__month, self.__day,
-                        self.__hour, self.__minute, self.__second,
-                        self.__microsecond),
-                       (other.__year, other.__month, other.__day,
-                        other.__hour, other.__minute, other.__second,
-                        other.__microsecond))
+            return cmp((self._year, self._month, self._day,
+                        self._hour, self._minute, self._second,
+                        self._microsecond),
+                       (other._year, other._month, other._day,
+                        other._hour, other._minute, other._second,
+                        other._microsecond))
         if myoff is None or otoff is None:
             # XXX Buggy in 2.2.2.
             raise TypeError("cannot compare naive and aware datetimes")
@@ -1819,13 +1832,13 @@
         "Add a datetime and a timedelta."
         if not isinstance(other, timedelta):
             return NotImplemented
-        t = tmxxx(self.__year,
-                  self.__month,
-                  self.__day + other.days,
-                  self.__hour,
-                  self.__minute,
-                  self.__second + other.seconds,
-                  self.__microsecond + other.microseconds)
+        t = tmxxx(self._year,
+                  self._month,
+                  self._day + other.days,
+                  self._hour,
+                  self._minute,
+                  self._second + other.seconds,
+                  self._microsecond + other.microseconds)
         self._checkOverflow(t.year)
         result = datetime(t.year, t.month, t.day,
                                 t.hour, t.minute, t.second,
@@ -1843,11 +1856,11 @@
 
         days1 = self.toordinal()
         days2 = other.toordinal()
-        secs1 = self.__second + self.__minute * 60 + self.__hour * 3600
-        secs2 = other.__second + other.__minute * 60 + other.__hour * 3600
+        secs1 = self._second + self._minute * 60 + self._hour * 3600
+        secs2 = other._second + other._minute * 60 + other._hour * 3600
         base = timedelta(days1 - days2,
                          secs1 - secs2,
-                         self.__microsecond - other.__microsecond)
+                         self._microsecond - other._microsecond)
         if self._tzinfo is other._tzinfo:
             return base
         myoff = self._utcoffset()
@@ -1855,13 +1868,13 @@
         if myoff == otoff:
             return base
         if myoff is None or otoff is None:
-            raise TypeError, "cannot mix naive and timezone-aware time"
+            raise TypeError("cannot mix naive and timezone-aware time")
         return base + timedelta(minutes = otoff-myoff)
 
     def __hash__(self):
         tzoff = self._utcoffset()
         if tzoff is None:
-            return hash(self.__getstate()[0])
+            return hash(self._getstate()[0])
         days = _ymd2ord(self.year, self.month, self.day)
         seconds = self.hour * 3600 + (self.minute - tzoff) * 60 + self.second
         return hash(timedelta(days, seconds, self.microsecond))
@@ -1870,12 +1883,12 @@
 
     __safe_for_unpickling__ = True      # For Python 2.2
 
-    def __getstate(self):
-        yhi, ylo = divmod(self.__year, 256)
-        us2, us3 = divmod(self.__microsecond, 256)
+    def _getstate(self):
+        yhi, ylo = divmod(self._year, 256)
+        us2, us3 = divmod(self._microsecond, 256)
         us1, us2 = divmod(us2, 256)
-        basestate = ("%c" * 10) % (yhi, ylo, self.__month, self.__day,
-                                   self.__hour, self.__minute, self.__second,
+        basestate = ("%c" * 10) % (yhi, ylo, self._month, self._day,
+                                   self._hour, self._minute, self._second,
                                    us1, us2, us3)
         if self._tzinfo is None:
             return (basestate,)
@@ -1883,14 +1896,14 @@
             return (basestate, self._tzinfo)
 
     def __setstate(self, string, tzinfo):
-        (yhi, ylo, self.__month, self.__day, self.__hour,
-         self.__minute, self.__second, us1, us2, us3) = map(ord, string)
-        self.__year = yhi * 256 + ylo
-        self.__microsecond = (((us1 << 8) | us2) << 8) | us3
+        (yhi, ylo, self._month, self._day, self._hour,
+         self._minute, self._second, us1, us2, us3) = map(ord, string)
+        self._year = yhi * 256 + ylo
+        self._microsecond = (((us1 << 8) | us2) << 8) | us3
         self._tzinfo = tzinfo
 
     def __reduce__(self):
-        return (self.__class__, self.__getstate())
+        return (self.__class__, self._getstate())
 
 
 datetime.min = datetime(1, 1, 1)

From noreply at buildbot.pypy.org  Sat Jan 21 23:27:13 2012
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sat, 21 Jan 2012 23:27:13 +0100 (CET)
Subject: [pypy-commit] pypy merge-2.7.2: A branch to merge CPython 2.7.2
Message-ID: <20120121222713.2E40282D45@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: merge-2.7.2
Changeset: r51617:37357d7dabda
Date: 2012-01-21 23:25 +0100
http://bitbucket.org/pypy/pypy/changeset/37357d7dabda/

Log:	A branch to merge CPython 2.7.2


From noreply at buildbot.pypy.org  Sun Jan 22 00:51:42 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Sun, 22 Jan 2012 00:51:42 +0100 (CET)
Subject: [pypy-commit] pypy matrixmath-dot: remvoe debug cruft
Message-ID: <20120121235142.A9D7B82D45@wyvern.cs.uni-duesseldorf.de>

Author: mattip
Branch: matrixmath-dot
Changeset: r51618:a8252a946ad8
Date: 2012-01-21 20:01 +0200
http://bitbucket.org/pypy/pypy/changeset/a8252a946ad8/

Log:	remvoe debug cruft

diff --git a/pypy/module/micronumpy/dot.py b/pypy/module/micronumpy/dot.py
--- a/pypy/module/micronumpy/dot.py
+++ b/pypy/module/micronumpy/dot.py
@@ -69,8 +69,6 @@
     _r = calculate_dot_strides(right.strides, right.backstrides,
                                   broadcast_shape, right_skip)
     righti = ViewIterator(right.start, _r[0], _r[1], broadcast_shape)
-    if right.size==4:
-        xxx
     while not outi.done():
         '''
         dot_driver.jit_merge_point(left=left,

From noreply at buildbot.pypy.org  Sun Jan 22 00:51:43 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Sun, 22 Jan 2012 00:51:43 +0100 (CET)
Subject: [pypy-commit] pypy numppy-flatitter: need to rethink getitem,
	setitem
Message-ID: <20120121235143.E54E982D45@wyvern.cs.uni-duesseldorf.de>

Author: mattip
Branch: numppy-flatitter
Changeset: r51619:aa7917ad8e16
Date: 2012-01-21 23:14 +0200
http://bitbucket.org/pypy/pypy/changeset/aa7917ad8e16/

Log:	need to rethink getitem, setitem

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -1299,10 +1299,13 @@
         size = 1
         for sh in arr.shape:
             size *= sh
-        if arr.strides[-1] < arr.strides[0]:
+        if arr.strides[-1] <= arr.strides[0]:
             self.strides = [arr.strides[-1]]
             self.backstrides = [arr.backstrides[-1]]
         else:
+            XXX
+            # This will not work: getitem and setitem will
+            # fail. Need to be smarter: calculate the indices from the int
             self.strides = [arr.strides[0]]
             self.backstrides = [arr.backstrides[0]]
         ViewArray.__init__(self, size, [size], arr.dtype, order=arr.order,

From noreply at buildbot.pypy.org  Sun Jan 22 00:51:45 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Sun, 22 Jan 2012 00:51:45 +0100 (CET)
Subject: [pypy-commit] pypy matrixmath-dot: fix bad test
Message-ID: <20120121235145.28E6E82D45@wyvern.cs.uni-duesseldorf.de>

Author: mattip
Branch: matrixmath-dot
Changeset: r51620:0125f74ace80
Date: 2012-01-22 01:17 +0200
http://bitbucket.org/pypy/pypy/changeset/0125f74ace80/

Log:	fix bad test

diff --git a/pypy/module/micronumpy/dot.py b/pypy/module/micronumpy/dot.py
--- a/pypy/module/micronumpy/dot.py
+++ b/pypy/module/micronumpy/dot.py
@@ -70,7 +70,6 @@
                                   broadcast_shape, right_skip)
     righti = ViewIterator(right.start, _r[0], _r[1], broadcast_shape)
     while not outi.done():
-        '''
         dot_driver.jit_merge_point(left=left,
                                    right=right,
                                    shape_len=shape_len,
@@ -81,18 +80,14 @@
                                    dtype=dtype,
                                    sig=None, #For get_printable_location
                                   )
-        '''
         lval = left.getitem(lefti.offset).convert_to(dtype) 
         rval = right.getitem(righti.offset).convert_to(dtype) 
         outval = result.getitem(outi.offset).convert_to(dtype) 
         v = dtype.itemtype.mul(lval, rval)
         value = dtype.itemtype.add(v, outval)
         #Do I need to convert it to result.dtype or does settiem do that?
-        assert outi.offset < result.size
         result.setitem(outi.offset, value)
         outi = outi.next(shape_len)
         righti = righti.next(shape_len)
         lefti = lefti.next(shape_len)
-    assert lefti.done()
-    assert righti.done()
     return result
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -891,7 +891,7 @@
         assert (c == [[[14, 38, 62], [38, 126, 214], [62, 214, 366]],
                    [[86, 302, 518], [110, 390, 670], [134, 478, 822]]]).all()
         c = dot(a, b[:, 2])
-        assert (c == [[38, 126, 214], [302, 390, 478]]).all()
+        assert (c == [[62, 214, 366], [518, 670, 822]]).all()
 
     def test_dot_constant(self):
         from _numpypy import array
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -379,7 +379,17 @@
     def test_dot(self):
         result = self.run("dot")
         assert result == 184
-        self.check_simple_loop({})
+        self.check_simple_loop({{'arraylen_gc': 9,
+                                'float_add': 1,
+                                'float_mul': 1,
+                                'getinteriorfield_raw': 3,
+                                'guard_false': 3,
+                                'guard_true': 3,
+                                'int_add': 6,
+                                'int_lt': 6,
+                                'int_sub': 3,
+                                'jump': 1,
+                                'setinteriorfield_raw': 1}})
 
 
 class TestNumpyOld(LLJitMixin):

From noreply at buildbot.pypy.org  Sun Jan 22 00:51:46 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Sun, 22 Jan 2012 00:51:46 +0100 (CET)
Subject: [pypy-commit] pypy numpypy-shape-bug: add failing setshape test,
	fix for test
Message-ID: <20120121235146.6225282D45@wyvern.cs.uni-duesseldorf.de>

Author: mattip
Branch: numpypy-shape-bug
Changeset: r51621:e11082ff75b9
Date: 2012-01-22 01:50 +0200
http://bitbucket.org/pypy/pypy/changeset/e11082ff75b9/

Log:	add failing setshape test, fix for test

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -156,7 +156,7 @@
 # fits the new shape, using those steps. If there is a shape/step mismatch
 # (meaning that the realignment of elements crosses from one step into another)
 # return None so that the caller can raise an exception.
-def calc_new_strides(new_shape, old_shape, old_strides):
+def calc_new_strides(new_shape, old_shape, old_strides, order):
     # Return the proper strides for new_shape, or None if the mapping crosses
     # stepping boundaries
 
@@ -166,7 +166,7 @@
     last_step = 1
     oldI = 0
     new_strides = []
-    if old_strides[0] < old_strides[-1]:
+    if order == 'F':
         for i in range(len(old_shape)):
             steps.append(old_strides[i] / last_step)
             last_step *= old_shape[i]
@@ -187,7 +187,7 @@
                     break
                 cur_step = steps[oldI]
                 n_old_elems_to_use *= old_shape[oldI]
-    else:
+    elif order == 'C':
         for i in range(len(old_shape) - 1, -1, -1):
             steps.insert(0, old_strides[i] / last_step)
             last_step *= old_shape[i]
@@ -543,8 +543,8 @@
         concrete = self.get_concrete()
         new_shape = get_shape_from_iterable(space, concrete.size, w_shape)
         # Since we got to here, prod(new_shape) == self.size
-        new_strides = calc_new_strides(new_shape,
-                                       concrete.shape, concrete.strides)
+        new_strides = calc_new_strides(new_shape, concrete.shape,
+                                     concrete.strides, concrete.order)
         if new_strides:
             # We can create a view, strides somehow match up.
             ndims = len(new_shape)
@@ -1105,7 +1105,8 @@
             self.backstrides = backstrides
             self.shape = new_shape
             return
-        new_strides = calc_new_strides(new_shape, self.shape, self.strides)
+        new_strides = calc_new_strides(new_shape, self.shape, self.strides,
+                                                   self.order)
         if new_strides is None:
             raise OperationError(space.w_AttributeError, space.wrap(
                           "incompatible shape for a non-contiguous array"))
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -152,11 +152,11 @@
 
     def test_calc_new_strides(self):
         from pypy.module.micronumpy.interp_numarray import calc_new_strides
-        assert calc_new_strides([2, 4], [4, 2], [4, 2]) == [8, 2]
-        assert calc_new_strides([2, 4, 3], [8, 3], [1, 16]) == [1, 2, 16]
-        assert calc_new_strides([2, 3, 4], [8, 3], [1, 16]) is None
-        assert calc_new_strides([24], [2, 4, 3], [48, 6, 1]) is None
-        assert calc_new_strides([24], [2, 4, 3], [24, 6, 2]) == [2]
+        assert calc_new_strides([2, 4], [4, 2], [4, 2], "C") == [8, 2]
+        assert calc_new_strides([2, 4, 3], [8, 3], [1, 16], 'F') == [1, 2, 16]
+        assert calc_new_strides([2, 3, 4], [8, 3], [1, 16], 'F') is None
+        assert calc_new_strides([24], [2, 4, 3], [48, 6, 1], 'C') is None
+        assert calc_new_strides([24], [2, 4, 3], [24, 6, 2], 'C') == [2]
 
 
 class AppTestNumArray(BaseNumpyAppTest):
@@ -381,6 +381,8 @@
         a.shape = ()
         #numpy allows this
         a.shape = (1,)
+        a = array(range(6)).reshape(2,3).T
+        raises(AttributeError, 'a.shape = 6')
 
     def test_reshape(self):
         from _numpypy import array, zeros
@@ -765,7 +767,7 @@
         assert (a[:, 1, :].sum(1) == [70, 315, 560]).all()
         raises (ValueError, 'a[:, 1, :].sum(2)')
         assert ((a + a).T.sum(2).T == (a + a).sum(0)).all()
-        skip("Those are broken on reshape, fix!")
+        skip("Those are broken, fix after removing Scalar")
         assert (a.reshape(1,-1).sum(0) == range(105)).all()
         assert (a.reshape(1,-1).sum(1) == 5460)
 

From noreply at buildbot.pypy.org  Sun Jan 22 01:02:19 2012
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sun, 22 Jan 2012 01:02:19 +0100 (CET)
Subject: [pypy-commit] pypy merge-2.7.2: Install CPython 2.7.2 version of
	the std library
Message-ID: <20120122000219.73C0382D45@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: merge-2.7.2
Changeset: r51622:d74ec4c815b1
Date: 2012-01-22 00:07 +0100
http://bitbucket.org/pypy/pypy/changeset/d74ec4c815b1/

Log:	Install CPython 2.7.2 version of the std library

diff too long, truncating to 10000 out of 120278 lines

diff --git a/lib-python/2.7/BaseHTTPServer.py b/lib-python/2.7/BaseHTTPServer.py
--- a/lib-python/2.7/BaseHTTPServer.py
+++ b/lib-python/2.7/BaseHTTPServer.py
@@ -310,7 +310,13 @@
 
         """
         try:
-            self.raw_requestline = self.rfile.readline()
+            self.raw_requestline = self.rfile.readline(65537)
+            if len(self.raw_requestline) > 65536:
+                self.requestline = ''
+                self.request_version = ''
+                self.command = ''
+                self.send_error(414)
+                return
             if not self.raw_requestline:
                 self.close_connection = 1
                 return
diff --git a/lib-python/2.7/ConfigParser.py b/lib-python/2.7/ConfigParser.py
--- a/lib-python/2.7/ConfigParser.py
+++ b/lib-python/2.7/ConfigParser.py
@@ -545,6 +545,38 @@
                 if isinstance(val, list):
                     options[name] = '\n'.join(val)
 
+import UserDict as _UserDict
+
+class _Chainmap(_UserDict.DictMixin):
+    """Combine multiple mappings for successive lookups.
+
+    For example, to emulate Python's normal lookup sequence:
+
+        import __builtin__
+        pylookup = _Chainmap(locals(), globals(), vars(__builtin__))
+    """
+
+    def __init__(self, *maps):
+        self._maps = maps
+
+    def __getitem__(self, key):
+        for mapping in self._maps:
+            try:
+                return mapping[key]
+            except KeyError:
+                pass
+        raise KeyError(key)
+
+    def keys(self):
+        result = []
+        seen = set()
+        for mapping in self_maps:
+            for key in mapping:
+                if key not in seen:
+                    result.append(key)
+                    seen.add(key)
+        return result
+
 class ConfigParser(RawConfigParser):
 
     def get(self, section, option, raw=False, vars=None):
@@ -559,16 +591,18 @@
 
         The section DEFAULT is special.
         """
-        d = self._defaults.copy()
+        sectiondict = {}
         try:
-            d.update(self._sections[section])
+            sectiondict = self._sections[section]
         except KeyError:
             if section != DEFAULTSECT:
                 raise NoSectionError(section)
         # Update with the entry specific variables
+        vardict = {}
         if vars:
             for key, value in vars.items():
-                d[self.optionxform(key)] = value
+                vardict[self.optionxform(key)] = value
+        d = _Chainmap(vardict, sectiondict, self._defaults)
         option = self.optionxform(option)
         try:
             value = d[option]
diff --git a/lib-python/2.7/Cookie.py b/lib-python/2.7/Cookie.py
--- a/lib-python/2.7/Cookie.py
+++ b/lib-python/2.7/Cookie.py
@@ -258,6 +258,11 @@
     '\033' : '\\033',  '\034' : '\\034',  '\035' : '\\035',
     '\036' : '\\036',  '\037' : '\\037',
 
+    # Because of the way browsers really handle cookies (as opposed
+    # to what the RFC says) we also encode , and ;
+
+    ',' : '\\054', ';' : '\\073',
+
     '"' : '\\"',       '\\' : '\\\\',
 
     '\177' : '\\177',  '\200' : '\\200',  '\201' : '\\201',
diff --git a/lib-python/2.7/HTMLParser.py b/lib-python/2.7/HTMLParser.py
--- a/lib-python/2.7/HTMLParser.py
+++ b/lib-python/2.7/HTMLParser.py
@@ -26,7 +26,7 @@
 tagfind = re.compile('[a-zA-Z][-.a-zA-Z0-9:_]*')
 attrfind = re.compile(
     r'\s*([a-zA-Z_][-.:a-zA-Z_0-9]*)(\s*=\s*'
-    r'(\'[^\']*\'|"[^"]*"|[-a-zA-Z0-9./,:;+*%?!&$\(\)_#=~@]*))?')
+    r'(\'[^\']*\'|"[^"]*"|[^\s"\'=<>`]*))?')
 
 locatestarttagend = re.compile(r"""
   <[a-zA-Z][-.a-zA-Z0-9:_]*          # tag name
@@ -99,7 +99,7 @@
         markupbase.ParserBase.reset(self)
 
     def feed(self, data):
-        """Feed data to the parser.
+        r"""Feed data to the parser.
 
         Call this as often as you want, with as little or as much text
         as you want (may include '\n').
@@ -367,13 +367,16 @@
             return s
         def replaceEntities(s):
             s = s.groups()[0]
-            if s[0] == "#":
-                s = s[1:]
-                if s[0] in ['x','X']:
-                    c = int(s[1:], 16)
-                else:
-                    c = int(s)
-                return unichr(c)
+            try:
+                if s[0] == "#":
+                    s = s[1:]
+                    if s[0] in ['x','X']:
+                        c = int(s[1:], 16)
+                    else:
+                        c = int(s)
+                    return unichr(c)
+            except ValueError:
+                return '&#'+s+';'
             else:
                 # Cannot use name2codepoint directly, because HTMLParser supports apos,
                 # which is not part of HTML 4
diff --git a/lib-python/2.7/SimpleHTTPServer.py b/lib-python/2.7/SimpleHTTPServer.py
--- a/lib-python/2.7/SimpleHTTPServer.py
+++ b/lib-python/2.7/SimpleHTTPServer.py
@@ -15,6 +15,7 @@
 import BaseHTTPServer
 import urllib
 import cgi
+import sys
 import shutil
 import mimetypes
 try:
@@ -131,7 +132,8 @@
         length = f.tell()
         f.seek(0)
         self.send_response(200)
-        self.send_header("Content-type", "text/html")
+        encoding = sys.getfilesystemencoding()
+        self.send_header("Content-type", "text/html; charset=%s" % encoding)
         self.send_header("Content-Length", str(length))
         self.end_headers()
         return f
diff --git a/lib-python/2.7/SimpleXMLRPCServer.py b/lib-python/2.7/SimpleXMLRPCServer.py
--- a/lib-python/2.7/SimpleXMLRPCServer.py
+++ b/lib-python/2.7/SimpleXMLRPCServer.py
@@ -246,7 +246,7 @@
         marshalled data. For backwards compatibility, a dispatch
         function can be provided as an argument (see comment in
         SimpleXMLRPCRequestHandler.do_POST) but overriding the
-        existing method through subclassing is the prefered means
+        existing method through subclassing is the preferred means
         of changing method dispatch behavior.
         """
 
diff --git a/lib-python/2.7/SocketServer.py b/lib-python/2.7/SocketServer.py
--- a/lib-python/2.7/SocketServer.py
+++ b/lib-python/2.7/SocketServer.py
@@ -675,7 +675,7 @@
     # A timeout to apply to the request socket, if not None.
     timeout = None
 
-    # Disable nagle algoritm for this socket, if True.
+    # Disable nagle algorithm for this socket, if True.
     # Use only when wbufsize != 0, to avoid small packets.
     disable_nagle_algorithm = False
 
diff --git a/lib-python/2.7/StringIO.py b/lib-python/2.7/StringIO.py
--- a/lib-python/2.7/StringIO.py
+++ b/lib-python/2.7/StringIO.py
@@ -266,6 +266,7 @@
         8th bit) will cause a UnicodeError to be raised when getvalue()
         is called.
         """
+        _complain_ifclosed(self.closed)
         if self.buflist:
             self.buf += ''.join(self.buflist)
             self.buflist = []
diff --git a/lib-python/2.7/_abcoll.py b/lib-python/2.7/_abcoll.py
--- a/lib-python/2.7/_abcoll.py
+++ b/lib-python/2.7/_abcoll.py
@@ -82,7 +82,7 @@
     @classmethod
     def __subclasshook__(cls, C):
         if cls is Iterator:
-            if _hasattr(C, "next"):
+            if _hasattr(C, "next") and _hasattr(C, "__iter__"):
                 return True
         return NotImplemented
 
diff --git a/lib-python/2.7/_pyio.py b/lib-python/2.7/_pyio.py
--- a/lib-python/2.7/_pyio.py
+++ b/lib-python/2.7/_pyio.py
@@ -16,6 +16,7 @@
 
 import io
 from io import (__all__, SEEK_SET, SEEK_CUR, SEEK_END)
+from errno import EINTR
 
 __metaclass__ = type
 
@@ -559,7 +560,11 @@
             if not data:
                 break
             res += data
-        return bytes(res)
+        if res:
+            return bytes(res)
+        else:
+            # b'' or None
+            return data
 
     def readinto(self, b):
         """Read up to len(b) bytes into b.
@@ -678,7 +683,7 @@
     """
 
     def __init__(self, raw):
-        self.raw = raw
+        self._raw = raw
 
     ### Positioning ###
 
@@ -722,8 +727,8 @@
         if self.raw is None:
             raise ValueError("raw stream already detached")
         self.flush()
-        raw = self.raw
-        self.raw = None
+        raw = self._raw
+        self._raw = None
         return raw
 
     ### Inquiries ###
@@ -738,6 +743,10 @@
         return self.raw.writable()
 
     @property
+    def raw(self):
+        return self._raw
+
+    @property
     def closed(self):
         return self.raw.closed
 
@@ -933,7 +942,12 @@
             current_size = 0
             while True:
                 # Read until EOF or until read() would block.
-                chunk = self.raw.read()
+                try:
+                    chunk = self.raw.read()
+                except IOError as e:
+                    if e.errno != EINTR:
+                        raise
+                    continue
                 if chunk in empty_values:
                     nodata_val = chunk
                     break
@@ -952,7 +966,12 @@
         chunks = [buf[pos:]]
         wanted = max(self.buffer_size, n)
         while avail < n:
-            chunk = self.raw.read(wanted)
+            try:
+                chunk = self.raw.read(wanted)
+            except IOError as e:
+                if e.errno != EINTR:
+                    raise
+                continue
             if chunk in empty_values:
                 nodata_val = chunk
                 break
@@ -981,7 +1000,14 @@
         have = len(self._read_buf) - self._read_pos
         if have < want or have <= 0:
             to_read = self.buffer_size - have
-            current = self.raw.read(to_read)
+            while True:
+                try:
+                    current = self.raw.read(to_read)
+                except IOError as e:
+                    if e.errno != EINTR:
+                        raise
+                    continue
+                break
             if current:
                 self._read_buf = self._read_buf[self._read_pos:] + current
                 self._read_pos = 0
@@ -1088,7 +1114,12 @@
         written = 0
         try:
             while self._write_buf:
-                n = self.raw.write(self._write_buf)
+                try:
+                    n = self.raw.write(self._write_buf)
+                except IOError as e:
+                    if e.errno != EINTR:
+                        raise
+                    continue
                 if n > len(self._write_buf) or n < 0:
                     raise IOError("write() returned incorrect number of bytes")
                 del self._write_buf[:n]
@@ -1456,7 +1487,7 @@
             if not isinstance(errors, basestring):
                 raise ValueError("invalid errors: %r" % errors)
 
-        self.buffer = buffer
+        self._buffer = buffer
         self._line_buffering = line_buffering
         self._encoding = encoding
         self._errors = errors
@@ -1511,6 +1542,10 @@
     def line_buffering(self):
         return self._line_buffering
 
+    @property
+    def buffer(self):
+        return self._buffer
+
     def seekable(self):
         return self._seekable
 
@@ -1724,8 +1759,8 @@
         if self.buffer is None:
             raise ValueError("buffer is already detached")
         self.flush()
-        buffer = self.buffer
-        self.buffer = None
+        buffer = self._buffer
+        self._buffer = None
         return buffer
 
     def seek(self, cookie, whence=0):
diff --git a/lib-python/2.7/_weakrefset.py b/lib-python/2.7/_weakrefset.py
--- a/lib-python/2.7/_weakrefset.py
+++ b/lib-python/2.7/_weakrefset.py
@@ -66,7 +66,11 @@
         return sum(x() is not None for x in self.data)
 
     def __contains__(self, item):
-        return ref(item) in self.data
+        try:
+            wr = ref(item)
+        except TypeError:
+            return False
+        return wr in self.data
 
     def __reduce__(self):
         return (self.__class__, (list(self),),
diff --git a/lib-python/2.7/anydbm.py b/lib-python/2.7/anydbm.py
--- a/lib-python/2.7/anydbm.py
+++ b/lib-python/2.7/anydbm.py
@@ -29,17 +29,8 @@
         list = d.keys() # return a list of all existing keys (slow!)
 
 Future versions may change the order in which implementations are
-tested for existence, add interfaces to other dbm-like
+tested for existence, and add interfaces to other dbm-like
 implementations.
-
-The open function has an optional second argument.  This can be 'r',
-for read-only access, 'w', for read-write access of an existing
-database, 'c' for read-write access to a new or existing database, and
-'n' for read-write access to a new database.  The default is 'r'.
-
-Note: 'r' and 'w' fail if the database doesn't exist; 'c' creates it
-only if it doesn't exist; and 'n' always creates a new database.
-
 """
 
 class error(Exception):
@@ -63,7 +54,18 @@
 
 error = tuple(_errors)
 
-def open(file, flag = 'r', mode = 0666):
+def open(file, flag='r', mode=0666):
+    """Open or create database at path given by *file*.
+
+    Optional argument *flag* can be 'r' (default) for read-only access, 'w'
+    for read-write access of an existing database, 'c' for read-write access
+    to a new or existing database, and 'n' for read-write access to a new
+    database.
+
+    Note: 'r' and 'w' fail if the database doesn't exist; 'c' creates it
+    only if it doesn't exist; and 'n' always creates a new database.
+    """
+
     # guess the type of an existing database
     from whichdb import whichdb
     result=whichdb(file)
diff --git a/lib-python/2.7/argparse.py b/lib-python/2.7/argparse.py
--- a/lib-python/2.7/argparse.py
+++ b/lib-python/2.7/argparse.py
@@ -82,6 +82,7 @@
 ]
 
 
+import collections as _collections
 import copy as _copy
 import os as _os
 import re as _re
@@ -1037,7 +1038,7 @@
 
         self._prog_prefix = prog
         self._parser_class = parser_class
-        self._name_parser_map = {}
+        self._name_parser_map = _collections.OrderedDict()
         self._choices_actions = []
 
         super(_SubParsersAction, self).__init__(
@@ -1080,7 +1081,7 @@
             parser = self._name_parser_map[parser_name]
         except KeyError:
             tup = parser_name, ', '.join(self._name_parser_map)
-            msg = _('unknown parser %r (choices: %s)' % tup)
+            msg = _('unknown parser %r (choices: %s)') % tup
             raise ArgumentError(self, msg)
 
         # parse all the remaining options into the namespace
@@ -1109,7 +1110,7 @@
             the builtin open() function.
     """
 
-    def __init__(self, mode='r', bufsize=None):
+    def __init__(self, mode='r', bufsize=-1):
         self._mode = mode
         self._bufsize = bufsize
 
@@ -1121,18 +1122,19 @@
             elif 'w' in self._mode:
                 return _sys.stdout
             else:
-                msg = _('argument "-" with mode %r' % self._mode)
+                msg = _('argument "-" with mode %r') % self._mode
                 raise ValueError(msg)
 
         # all other arguments are used as file names
-        if self._bufsize:
+        try:
             return open(string, self._mode, self._bufsize)
-        else:
-            return open(string, self._mode)
+        except IOError as e:
+            message = _("can't open '%s': %s")
+            raise ArgumentTypeError(message % (string, e))
 
     def __repr__(self):
-        args = [self._mode, self._bufsize]
-        args_str = ', '.join([repr(arg) for arg in args if arg is not None])
+        args = self._mode, self._bufsize
+        args_str = ', '.join(repr(arg) for arg in args if arg != -1)
         return '%s(%s)' % (type(self).__name__, args_str)
 
 # ===========================
@@ -1275,13 +1277,20 @@
         # create the action object, and add it to the parser
         action_class = self._pop_action_class(kwargs)
         if not _callable(action_class):
-            raise ValueError('unknown action "%s"' % action_class)
+            raise ValueError('unknown action "%s"' % (action_class,))
         action = action_class(**kwargs)
 
         # raise an error if the action type is not callable
         type_func = self._registry_get('type', action.type, action.type)
         if not _callable(type_func):
-            raise ValueError('%r is not callable' % type_func)
+            raise ValueError('%r is not callable' % (type_func,))
+
+        # raise an error if the metavar does not match the type
+        if hasattr(self, "_get_formatter"):
+            try:
+                self._get_formatter()._format_args(action, None)
+            except TypeError:
+                raise ValueError("length of metavar tuple does not match nargs")
 
         return self._add_action(action)
 
@@ -1481,6 +1490,7 @@
         self._defaults = container._defaults
         self._has_negative_number_optionals = \
             container._has_negative_number_optionals
+        self._mutually_exclusive_groups = container._mutually_exclusive_groups
 
     def _add_action(self, action):
         action = super(_ArgumentGroup, self)._add_action(action)
diff --git a/lib-python/2.7/ast.py b/lib-python/2.7/ast.py
--- a/lib-python/2.7/ast.py
+++ b/lib-python/2.7/ast.py
@@ -29,12 +29,12 @@
 from _ast import __version__
 
 
-def parse(expr, filename='', mode='exec'):
+def parse(source, filename='', mode='exec'):
     """
-    Parse an expression into an AST node.
-    Equivalent to compile(expr, filename, mode, PyCF_ONLY_AST).
+    Parse the source into an AST node.
+    Equivalent to compile(source, filename, mode, PyCF_ONLY_AST).
     """
-    return compile(expr, filename, mode, PyCF_ONLY_AST)
+    return compile(source, filename, mode, PyCF_ONLY_AST)
 
 
 def literal_eval(node_or_string):
@@ -152,8 +152,6 @@
     Increment the line number of each node in the tree starting at *node* by *n*.
     This is useful to "move code" to a different location in a file.
     """
-    if 'lineno' in node._attributes:
-        node.lineno = getattr(node, 'lineno', 0) + n
     for child in walk(node):
         if 'lineno' in child._attributes:
             child.lineno = getattr(child, 'lineno', 0) + n
@@ -204,9 +202,9 @@
 
 def walk(node):
     """
-    Recursively yield all child nodes of *node*, in no specified order.  This is
-    useful if you only want to modify nodes in place and don't care about the
-    context.
+    Recursively yield all descendant nodes in the tree starting at *node*
+    (including *node* itself), in no specified order.  This is useful if you
+    only want to modify nodes in place and don't care about the context.
     """
     from collections import deque
     todo = deque([node])
diff --git a/lib-python/2.7/asyncore.py b/lib-python/2.7/asyncore.py
--- a/lib-python/2.7/asyncore.py
+++ b/lib-python/2.7/asyncore.py
@@ -54,7 +54,11 @@
 
 import os
 from errno import EALREADY, EINPROGRESS, EWOULDBLOCK, ECONNRESET, EINVAL, \
-     ENOTCONN, ESHUTDOWN, EINTR, EISCONN, EBADF, ECONNABORTED, errorcode
+     ENOTCONN, ESHUTDOWN, EINTR, EISCONN, EBADF, ECONNABORTED, EPIPE, EAGAIN, \
+     errorcode
+
+_DISCONNECTED = frozenset((ECONNRESET, ENOTCONN, ESHUTDOWN, ECONNABORTED, EPIPE,
+                           EBADF))
 
 try:
     socket_map
@@ -109,7 +113,7 @@
         if flags & (select.POLLHUP | select.POLLERR | select.POLLNVAL):
             obj.handle_close()
     except socket.error, e:
-        if e.args[0] not in (EBADF, ECONNRESET, ENOTCONN, ESHUTDOWN, ECONNABORTED):
+        if e.args[0] not in _DISCONNECTED:
             obj.handle_error()
         else:
             obj.handle_close()
@@ -353,7 +357,7 @@
         except TypeError:
             return None
         except socket.error as why:
-            if why.args[0] in (EWOULDBLOCK, ECONNABORTED):
+            if why.args[0] in (EWOULDBLOCK, ECONNABORTED, EAGAIN):
                 return None
             else:
                 raise
@@ -367,7 +371,7 @@
         except socket.error, why:
             if why.args[0] == EWOULDBLOCK:
                 return 0
-            elif why.args[0] in (ECONNRESET, ENOTCONN, ESHUTDOWN, ECONNABORTED):
+            elif why.args[0] in _DISCONNECTED:
                 self.handle_close()
                 return 0
             else:
@@ -385,7 +389,7 @@
                 return data
         except socket.error, why:
             # winsock sometimes throws ENOTCONN
-            if why.args[0] in [ECONNRESET, ENOTCONN, ESHUTDOWN, ECONNABORTED]:
+            if why.args[0] in _DISCONNECTED:
                 self.handle_close()
                 return ''
             else:
diff --git a/lib-python/2.7/bdb.py b/lib-python/2.7/bdb.py
--- a/lib-python/2.7/bdb.py
+++ b/lib-python/2.7/bdb.py
@@ -250,6 +250,12 @@
             list.append(lineno)
         bp = Breakpoint(filename, lineno, temporary, cond, funcname)
 
+    def _prune_breaks(self, filename, lineno):
+        if (filename, lineno) not in Breakpoint.bplist:
+            self.breaks[filename].remove(lineno)
+        if not self.breaks[filename]:
+            del self.breaks[filename]
+
     def clear_break(self, filename, lineno):
         filename = self.canonic(filename)
         if not filename in self.breaks:
@@ -261,10 +267,7 @@
         # pair, then remove the breaks entry
         for bp in Breakpoint.bplist[filename, lineno][:]:
             bp.deleteMe()
-        if (filename, lineno) not in Breakpoint.bplist:
-            self.breaks[filename].remove(lineno)
-        if not self.breaks[filename]:
-            del self.breaks[filename]
+        self._prune_breaks(filename, lineno)
 
     def clear_bpbynumber(self, arg):
         try:
@@ -277,7 +280,8 @@
             return 'Breakpoint number (%d) out of range' % number
         if not bp:
             return 'Breakpoint (%d) already deleted' % number
-        self.clear_break(bp.file, bp.line)
+        bp.deleteMe()
+        self._prune_breaks(bp.file, bp.line)
 
     def clear_all_file_breaks(self, filename):
         filename = self.canonic(filename)
diff --git a/lib-python/2.7/collections.py b/lib-python/2.7/collections.py
--- a/lib-python/2.7/collections.py
+++ b/lib-python/2.7/collections.py
@@ -6,59 +6,38 @@
 __all__ += _abcoll.__all__
 
 from _collections import deque, defaultdict
-from operator import itemgetter as _itemgetter, eq as _eq
+from operator import itemgetter as _itemgetter
 from keyword import iskeyword as _iskeyword
 import sys as _sys
 import heapq as _heapq
-from itertools import repeat as _repeat, chain as _chain, starmap as _starmap, \
-                      ifilter as _ifilter, imap as _imap
+from itertools import repeat as _repeat, chain as _chain, starmap as _starmap
+
 try:
-    from thread import get_ident
+    from thread import get_ident as _get_ident
 except ImportError:
-    from dummy_thread import get_ident
-
-def _recursive_repr(user_function):
-    'Decorator to make a repr function return "..." for a recursive call'
-    repr_running = set()
-
-    def wrapper(self):
-        key = id(self), get_ident()
-        if key in repr_running:
-            return '...'
-        repr_running.add(key)
-        try:
-            result = user_function(self)
-        finally:
-            repr_running.discard(key)
-        return result
-
-    # Can't use functools.wraps() here because of bootstrap issues
-    wrapper.__module__ = getattr(user_function, '__module__')
-    wrapper.__doc__ = getattr(user_function, '__doc__')
-    wrapper.__name__ = getattr(user_function, '__name__')
-    return wrapper
+    from dummy_thread import get_ident as _get_ident
 
 
 ################################################################################
 ### OrderedDict
 ################################################################################
 
-class OrderedDict(dict, MutableMapping):
+class OrderedDict(dict):
     'Dictionary that remembers insertion order'
     # An inherited dict maps keys to values.
     # The inherited dict provides __getitem__, __len__, __contains__, and get.
     # The remaining methods are order-aware.
-    # Big-O running times for all methods are the same as for regular dictionaries.
+    # Big-O running times for all methods are the same as regular dictionaries.
 
-    # The internal self.__map dictionary maps keys to links in a doubly linked list.
+    # The internal self.__map dict maps keys to links in a doubly linked list.
     # The circular doubly linked list starts and ends with a sentinel element.
     # The sentinel element never gets deleted (this simplifies the algorithm).
     # Each link is stored as a list of length three:  [PREV, NEXT, KEY].
 
     def __init__(self, *args, **kwds):
-        '''Initialize an ordered dictionary.  Signature is the same as for
-        regular dictionaries, but keyword arguments are not recommended
-        because their insertion order is arbitrary.
+        '''Initialize an ordered dictionary.  The signature is the same as
+        regular dictionaries, but keyword arguments are not recommended because
+        their insertion order is arbitrary.
 
         '''
         if len(args) > 1:
@@ -66,17 +45,15 @@
         try:
             self.__root
         except AttributeError:
-            self.__root = root = [None, None, None]     # sentinel node
-            PREV = 0
-            NEXT = 1
-            root[PREV] = root[NEXT] = root
+            self.__root = root = []                     # sentinel node
+            root[:] = [root, root, None]
             self.__map = {}
-        self.update(*args, **kwds)
+        self.__update(*args, **kwds)
 
     def __setitem__(self, key, value, PREV=0, NEXT=1, dict_setitem=dict.__setitem__):
         'od.__setitem__(i, y) <==> od[i]=y'
-        # Setting a new item creates a new link which goes at the end of the linked
-        # list, and the inherited dictionary is updated with the new key/value pair.
+        # Setting a new item creates a new link at the end of the linked list,
+        # and the inherited dictionary is updated with the new key/value pair.
         if key not in self:
             root = self.__root
             last = root[PREV]
@@ -85,65 +62,160 @@
 
     def __delitem__(self, key, PREV=0, NEXT=1, dict_delitem=dict.__delitem__):
         'od.__delitem__(y) <==> del od[y]'
-        # Deleting an existing item uses self.__map to find the link which is
-        # then removed by updating the links in the predecessor and successor nodes.
+        # Deleting an existing item uses self.__map to find the link which gets
+        # removed by updating the links in the predecessor and successor nodes.
         dict_delitem(self, key)
-        link = self.__map.pop(key)
-        link_prev = link[PREV]
-        link_next = link[NEXT]
+        link_prev, link_next, key = self.__map.pop(key)
         link_prev[NEXT] = link_next
         link_next[PREV] = link_prev
 
-    def __iter__(self, NEXT=1, KEY=2):
+    def __iter__(self):
         'od.__iter__() <==> iter(od)'
         # Traverse the linked list in order.
+        NEXT, KEY = 1, 2
         root = self.__root
         curr = root[NEXT]
         while curr is not root:
             yield curr[KEY]
             curr = curr[NEXT]
 
-    def __reversed__(self, PREV=0, KEY=2):
+    def __reversed__(self):
         'od.__reversed__() <==> reversed(od)'
         # Traverse the linked list in reverse order.
+        PREV, KEY = 0, 2
         root = self.__root
         curr = root[PREV]
         while curr is not root:
             yield curr[KEY]
             curr = curr[PREV]
 
+    def clear(self):
+        'od.clear() -> None.  Remove all items from od.'
+        for node in self.__map.itervalues():
+            del node[:]
+        root = self.__root
+        root[:] = [root, root, None]
+        self.__map.clear()
+        dict.clear(self)
+
+    # -- the following methods do not depend on the internal structure --
+
+    def keys(self):
+        'od.keys() -> list of keys in od'
+        return list(self)
+
+    def values(self):
+        'od.values() -> list of values in od'
+        return [self[key] for key in self]
+
+    def items(self):
+        'od.items() -> list of (key, value) pairs in od'
+        return [(key, self[key]) for key in self]
+
+    def iterkeys(self):
+        'od.iterkeys() -> an iterator over the keys in od'
+        return iter(self)
+
+    def itervalues(self):
+        'od.itervalues -> an iterator over the values in od'
+        for k in self:
+            yield self[k]
+
+    def iteritems(self):
+        'od.iteritems -> an iterator over the (key, value) pairs in od'
+        for k in self:
+            yield (k, self[k])
+
+    update = MutableMapping.update
+
+    __update = update # let subclasses override update without breaking __init__
+
+    __marker = object()
+
+    def pop(self, key, default=__marker):
+        '''od.pop(k[,d]) -> v, remove specified key and return the corresponding
+        value.  If key is not found, d is returned if given, otherwise KeyError
+        is raised.
+
+        '''
+        if key in self:
+            result = self[key]
+            del self[key]
+            return result
+        if default is self.__marker:
+            raise KeyError(key)
+        return default
+
+    def setdefault(self, key, default=None):
+        'od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od'
+        if key in self:
+            return self[key]
+        self[key] = default
+        return default
+
+    def popitem(self, last=True):
+        '''od.popitem() -> (k, v), return and remove a (key, value) pair.
+        Pairs are returned in LIFO order if last is true or FIFO order if false.
+
+        '''
+        if not self:
+            raise KeyError('dictionary is empty')
+        key = next(reversed(self) if last else iter(self))
+        value = self.pop(key)
+        return key, value
+
+    def __repr__(self, _repr_running={}):
+        'od.__repr__() <==> repr(od)'
+        call_key = id(self), _get_ident()
+        if call_key in _repr_running:
+            return '...'
+        _repr_running[call_key] = 1
+        try:
+            if not self:
+                return '%s()' % (self.__class__.__name__,)
+            return '%s(%r)' % (self.__class__.__name__, self.items())
+        finally:
+            del _repr_running[call_key]
+
     def __reduce__(self):
         'Return state information for pickling'
         items = [[k, self[k]] for k in self]
-        tmp = self.__map, self.__root
-        del self.__map, self.__root
         inst_dict = vars(self).copy()
-        self.__map, self.__root = tmp
+        for k in vars(OrderedDict()):
+            inst_dict.pop(k, None)
         if inst_dict:
             return (self.__class__, (items,), inst_dict)
         return self.__class__, (items,)
 
-    def clear(self):
-        'od.clear() -> None.  Remove all items from od.'
-        try:
-            for node in self.__map.itervalues():
-                del node[:]
-            self.__root[:] = [self.__root, self.__root, None]
-            self.__map.clear()
-        except AttributeError:
-            pass
-        dict.clear(self)
+    def copy(self):
+        'od.copy() -> a shallow copy of od'
+        return self.__class__(self)
 
-    setdefault = MutableMapping.setdefault
-    update = MutableMapping.update
-    pop = MutableMapping.pop
-    keys = MutableMapping.keys
-    values = MutableMapping.values
-    items = MutableMapping.items
-    iterkeys = MutableMapping.iterkeys
-    itervalues = MutableMapping.itervalues
-    iteritems = MutableMapping.iteritems
-    __ne__ = MutableMapping.__ne__
+    @classmethod
+    def fromkeys(cls, iterable, value=None):
+        '''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S.
+        If not specified, the value defaults to None.
+
+        '''
+        self = cls()
+        for key in iterable:
+            self[key] = value
+        return self
+
+    def __eq__(self, other):
+        '''od.__eq__(y) <==> od==y.  Comparison to another OD is order-sensitive
+        while comparison to a regular mapping is order-insensitive.
+
+        '''
+        if isinstance(other, OrderedDict):
+            return len(self)==len(other) and self.items() == other.items()
+        return dict.__eq__(self, other)
+
+    def __ne__(self, other):
+        'od.__ne__(y) <==> od!=y'
+        return not self == other
+
+    # -- the following methods support python 3.x style dictionary views --
 
     def viewkeys(self):
         "od.viewkeys() -> a set-like object providing a view on od's keys"
@@ -157,49 +229,6 @@
         "od.viewitems() -> a set-like object providing a view on od's items"
         return ItemsView(self)
 
-    def popitem(self, last=True):
-        '''od.popitem() -> (k, v), return and remove a (key, value) pair.
-        Pairs are returned in LIFO order if last is true or FIFO order if false.
-
-        '''
-        if not self:
-            raise KeyError('dictionary is empty')
-        key = next(reversed(self) if last else iter(self))
-        value = self.pop(key)
-        return key, value
-
-    @_recursive_repr
-    def __repr__(self):
-        'od.__repr__() <==> repr(od)'
-        if not self:
-            return '%s()' % (self.__class__.__name__,)
-        return '%s(%r)' % (self.__class__.__name__, self.items())
-
-    def copy(self):
-        'od.copy() -> a shallow copy of od'
-        return self.__class__(self)
-
-    @classmethod
-    def fromkeys(cls, iterable, value=None):
-        '''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S
-        and values equal to v (which defaults to None).
-
-        '''
-        d = cls()
-        for key in iterable:
-            d[key] = value
-        return d
-
-    def __eq__(self, other):
-        '''od.__eq__(y) <==> od==y.  Comparison to another OD is order-sensitive
-        while comparison to a regular mapping is order-insensitive.
-
-        '''
-        if isinstance(other, OrderedDict):
-            return len(self)==len(other) and \
-                   all(_imap(_eq, self.iteritems(), other.iteritems()))
-        return dict.__eq__(self, other)
-
 
 ################################################################################
 ### namedtuple
@@ -328,16 +357,16 @@
     or multiset.  Elements are stored as dictionary keys and their counts
     are stored as dictionary values.
 
-    >>> c = Counter('abracadabra')      # count elements from a string
+    >>> c = Counter('abcdeabcdabcaba')  # count elements from a string
 
     >>> c.most_common(3)                # three most common elements
-    [('a', 5), ('r', 2), ('b', 2)]
+    [('a', 5), ('b', 4), ('c', 3)]
     >>> sorted(c)                       # list all unique elements
-    ['a', 'b', 'c', 'd', 'r']
+    ['a', 'b', 'c', 'd', 'e']
     >>> ''.join(sorted(c.elements()))   # list elements with repetitions
-    'aaaaabbcdrr'
+    'aaaaabbbbcccdde'
     >>> sum(c.values())                 # total of all counts
-    11
+    15
 
     >>> c['a']                          # count of letter 'a'
     5
@@ -345,8 +374,8 @@
     ...     c[elem] += 1                # by adding 1 to each element's count
     >>> c['a']                          # now there are seven 'a'
     7
-    >>> del c['r']                      # remove all 'r'
-    >>> c['r']                          # now there are zero 'r'
+    >>> del c['b']                      # remove all 'b'
+    >>> c['b']                          # now there are zero 'b'
     0
 
     >>> d = Counter('simsalabim')       # make another counter
@@ -385,6 +414,7 @@
         >>> c = Counter(a=4, b=2)                   # a new counter from keyword args
 
         '''
+        super(Counter, self).__init__()
         self.update(iterable, **kwds)
 
     def __missing__(self, key):
@@ -396,8 +426,8 @@
         '''List the n most common elements and their counts from the most
         common to the least.  If n is None, then list all element counts.
 
-        >>> Counter('abracadabra').most_common(3)
-        [('a', 5), ('r', 2), ('b', 2)]
+        >>> Counter('abcdeabcdabcaba').most_common(3)
+        [('a', 5), ('b', 4), ('c', 3)]
 
         '''
         # Emulate Bag.sortedByCount from Smalltalk
@@ -463,7 +493,7 @@
                     for elem, count in iterable.iteritems():
                         self[elem] = self_get(elem, 0) + count
                 else:
-                    dict.update(self, iterable) # fast path when counter is empty
+                    super(Counter, self).update(iterable) # fast path when counter is empty
             else:
                 self_get = self.get
                 for elem in iterable:
@@ -499,13 +529,16 @@
             self.subtract(kwds)
 
     def copy(self):
-        'Like dict.copy() but returns a Counter instance instead of a dict.'
-        return Counter(self)
+        'Return a shallow copy.'
+        return self.__class__(self)
+
+    def __reduce__(self):
+        return self.__class__, (dict(self),)
 
     def __delitem__(self, elem):
         'Like dict.__delitem__() but does not raise KeyError for missing values.'
         if elem in self:
-            dict.__delitem__(self, elem)
+            super(Counter, self).__delitem__(elem)
 
     def __repr__(self):
         if not self:
@@ -532,10 +565,13 @@
         if not isinstance(other, Counter):
             return NotImplemented
         result = Counter()
-        for elem in set(self) | set(other):
-            newcount = self[elem] + other[elem]
+        for elem, count in self.items():
+            newcount = count + other[elem]
             if newcount > 0:
                 result[elem] = newcount
+        for elem, count in other.items():
+            if elem not in self and count > 0:
+                result[elem] = count
         return result
 
     def __sub__(self, other):
@@ -548,10 +584,13 @@
         if not isinstance(other, Counter):
             return NotImplemented
         result = Counter()
-        for elem in set(self) | set(other):
-            newcount = self[elem] - other[elem]
+        for elem, count in self.items():
+            newcount = count - other[elem]
             if newcount > 0:
                 result[elem] = newcount
+        for elem, count in other.items():
+            if elem not in self and count < 0:
+                result[elem] = 0 - count
         return result
 
     def __or__(self, other):
@@ -564,11 +603,14 @@
         if not isinstance(other, Counter):
             return NotImplemented
         result = Counter()
-        for elem in set(self) | set(other):
-            p, q = self[elem], other[elem]
-            newcount = q if p < q else p
+        for elem, count in self.items():
+            other_count = other[elem]
+            newcount = other_count if count < other_count else count
             if newcount > 0:
                 result[elem] = newcount
+        for elem, count in other.items():
+            if elem not in self and count > 0:
+                result[elem] = count
         return result
 
     def __and__(self, other):
@@ -581,11 +623,9 @@
         if not isinstance(other, Counter):
             return NotImplemented
         result = Counter()
-        if len(self) < len(other):
-            self, other = other, self
-        for elem in _ifilter(self.__contains__, other):
-            p, q = self[elem], other[elem]
-            newcount = p if p < q else q
+        for elem, count in self.items():
+            other_count = other[elem]
+            newcount = count if count < other_count else other_count
             if newcount > 0:
                 result[elem] = newcount
         return result
diff --git a/lib-python/2.7/compileall.py b/lib-python/2.7/compileall.py
--- a/lib-python/2.7/compileall.py
+++ b/lib-python/2.7/compileall.py
@@ -9,7 +9,6 @@
 packages -- for now, you'll have to deal with packages separately.)
 
 See module py_compile for details of the actual byte-compilation.
-
 """
 import os
 import sys
@@ -31,7 +30,6 @@
                directory name that will show up in error messages)
     force:     if 1, force compilation, even if timestamps are up-to-date
     quiet:     if 1, be quiet during compilation
-
     """
     if not quiet:
         print 'Listing', dir, '...'
@@ -61,15 +59,16 @@
     return success
 
 def compile_file(fullname, ddir=None, force=0, rx=None, quiet=0):
-    """Byte-compile file.
-    file:      the file to byte-compile
+    """Byte-compile one file.
+
+    Arguments (only fullname is required):
+
+    fullname:  the file to byte-compile
     ddir:      if given, purported directory name (this is the
                directory name that will show up in error messages)
     force:     if 1, force compilation, even if timestamps are up-to-date
     quiet:     if 1, be quiet during compilation
-
     """
-
     success = 1
     name = os.path.basename(fullname)
     if ddir is not None:
@@ -120,7 +119,6 @@
     maxlevels:   max recursion level (default 0)
     force: as for compile_dir() (default 0)
     quiet: as for compile_dir() (default 0)
-
     """
     success = 1
     for dir in sys.path:
diff --git a/lib-python/2.7/csv.py b/lib-python/2.7/csv.py
--- a/lib-python/2.7/csv.py
+++ b/lib-python/2.7/csv.py
@@ -281,7 +281,7 @@
         an all or nothing approach, so we allow for small variations in this
         number.
           1) build a table of the frequency of each character on every line.
-          2) build a table of freqencies of this frequency (meta-frequency?),
+          2) build a table of frequencies of this frequency (meta-frequency?),
              e.g.  'x occurred 5 times in 10 rows, 6 times in 1000 rows,
              7 times in 2 rows'
           3) use the mode of the meta-frequency to determine the /expected/
diff --git a/lib-python/2.7/ctypes/test/test_arrays.py b/lib-python/2.7/ctypes/test/test_arrays.py
--- a/lib-python/2.7/ctypes/test/test_arrays.py
+++ b/lib-python/2.7/ctypes/test/test_arrays.py
@@ -37,7 +37,7 @@
             values = [ia[i] for i in range(len(init))]
             self.assertEqual(values, [0] * len(init))
 
-            # Too many in itializers should be caught
+            # Too many initializers should be caught
             self.assertRaises(IndexError, int_array, *range(alen*2))
 
         CharArray = ARRAY(c_char, 3)
diff --git a/lib-python/2.7/ctypes/test/test_as_parameter.py b/lib-python/2.7/ctypes/test/test_as_parameter.py
--- a/lib-python/2.7/ctypes/test/test_as_parameter.py
+++ b/lib-python/2.7/ctypes/test/test_as_parameter.py
@@ -187,6 +187,18 @@
         self.assertEqual((s8i.a, s8i.b, s8i.c, s8i.d, s8i.e, s8i.f, s8i.g, s8i.h),
                              (9*2, 8*3, 7*4, 6*5, 5*6, 4*7, 3*8, 2*9))
 
+    def test_recursive_as_param(self):
+        from ctypes import c_int
+
+        class A(object):
+            pass
+
+        a = A()
+        a._as_parameter_ = a
+        with self.assertRaises(RuntimeError):
+            c_int.from_param(a)
+
+
 #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 class AsParamWrapper(object):
diff --git a/lib-python/2.7/ctypes/test/test_callbacks.py b/lib-python/2.7/ctypes/test/test_callbacks.py
--- a/lib-python/2.7/ctypes/test/test_callbacks.py
+++ b/lib-python/2.7/ctypes/test/test_callbacks.py
@@ -206,6 +206,42 @@
 
             windll.user32.EnumWindows(EnumWindowsCallbackFunc, 0)
 
+    def test_callback_register_int(self):
+        # Issue #8275: buggy handling of callback args under Win64
+        # NOTE: should be run on release builds as well
+        dll = CDLL(_ctypes_test.__file__)
+        CALLBACK = CFUNCTYPE(c_int, c_int, c_int, c_int, c_int, c_int)
+        # All this function does is call the callback with its args squared
+        func = dll._testfunc_cbk_reg_int
+        func.argtypes = (c_int, c_int, c_int, c_int, c_int, CALLBACK)
+        func.restype = c_int
+
+        def callback(a, b, c, d, e):
+            return a + b + c + d + e
+
+        result = func(2, 3, 4, 5, 6, CALLBACK(callback))
+        self.assertEqual(result, callback(2*2, 3*3, 4*4, 5*5, 6*6))
+
+    def test_callback_register_double(self):
+        # Issue #8275: buggy handling of callback args under Win64
+        # NOTE: should be run on release builds as well
+        dll = CDLL(_ctypes_test.__file__)
+        CALLBACK = CFUNCTYPE(c_double, c_double, c_double, c_double,
+                             c_double, c_double)
+        # All this function does is call the callback with its args squared
+        func = dll._testfunc_cbk_reg_double
+        func.argtypes = (c_double, c_double, c_double,
+                         c_double, c_double, CALLBACK)
+        func.restype = c_double
+
+        def callback(a, b, c, d, e):
+            return a + b + c + d + e
+
+        result = func(1.1, 2.2, 3.3, 4.4, 5.5, CALLBACK(callback))
+        self.assertEqual(result,
+                         callback(1.1*1.1, 2.2*2.2, 3.3*3.3, 4.4*4.4, 5.5*5.5))
+
+
 ################################################################
 
 if __name__ == '__main__':
diff --git a/lib-python/2.7/ctypes/test/test_functions.py b/lib-python/2.7/ctypes/test/test_functions.py
--- a/lib-python/2.7/ctypes/test/test_functions.py
+++ b/lib-python/2.7/ctypes/test/test_functions.py
@@ -116,7 +116,7 @@
         self.assertEqual(result, 21)
         self.assertEqual(type(result), int)
 
-        # You cannot assing character format codes as restype any longer
+        # You cannot assign character format codes as restype any longer
         self.assertRaises(TypeError, setattr, f, "restype", "i")
 
     def test_floatresult(self):
diff --git a/lib-python/2.7/ctypes/test/test_init.py b/lib-python/2.7/ctypes/test/test_init.py
--- a/lib-python/2.7/ctypes/test/test_init.py
+++ b/lib-python/2.7/ctypes/test/test_init.py
@@ -27,7 +27,7 @@
         self.assertEqual((y.x.a, y.x.b), (0, 0))
         self.assertEqual(y.x.new_was_called, False)
 
-        # But explicitely creating an X structure calls __new__ and __init__, of course.
+        # But explicitly creating an X structure calls __new__ and __init__, of course.
         x = X()
         self.assertEqual((x.a, x.b), (9, 12))
         self.assertEqual(x.new_was_called, True)
diff --git a/lib-python/2.7/ctypes/test/test_numbers.py b/lib-python/2.7/ctypes/test/test_numbers.py
--- a/lib-python/2.7/ctypes/test/test_numbers.py
+++ b/lib-python/2.7/ctypes/test/test_numbers.py
@@ -157,7 +157,7 @@
     def test_int_from_address(self):
         from array import array
         for t in signed_types + unsigned_types:
-            # the array module doesn't suppport all format codes
+            # the array module doesn't support all format codes
             # (no 'q' or 'Q')
             try:
                 array(t._type_)
diff --git a/lib-python/2.7/ctypes/test/test_win32.py b/lib-python/2.7/ctypes/test/test_win32.py
--- a/lib-python/2.7/ctypes/test/test_win32.py
+++ b/lib-python/2.7/ctypes/test/test_win32.py
@@ -17,7 +17,7 @@
             # ValueError: Procedure probably called with not enough arguments (4 bytes missing)
             self.assertRaises(ValueError, IsWindow)
 
-            # This one should succeeed...
+            # This one should succeed...
             self.assertEqual(0, IsWindow(0))
 
             # ValueError: Procedure probably called with too many arguments (8 bytes in excess)
diff --git a/lib-python/2.7/curses/wrapper.py b/lib-python/2.7/curses/wrapper.py
--- a/lib-python/2.7/curses/wrapper.py
+++ b/lib-python/2.7/curses/wrapper.py
@@ -43,7 +43,8 @@
         return func(stdscr, *args, **kwds)
     finally:
         # Set everything back to normal
-        stdscr.keypad(0)
-        curses.echo()
-        curses.nocbreak()
-        curses.endwin()
+        if 'stdscr' in locals():
+            stdscr.keypad(0)
+            curses.echo()
+            curses.nocbreak()
+            curses.endwin()
diff --git a/lib-python/2.7/decimal.py b/lib-python/2.7/decimal.py
--- a/lib-python/2.7/decimal.py
+++ b/lib-python/2.7/decimal.py
@@ -1068,14 +1068,16 @@
             if ans:
                 return ans
 
-        if not self:
-            # -Decimal('0') is Decimal('0'), not Decimal('-0')
+        if context is None:
+            context = getcontext()
+
+        if not self and context.rounding != ROUND_FLOOR:
+            # -Decimal('0') is Decimal('0'), not Decimal('-0'), except
+            # in ROUND_FLOOR rounding mode.
             ans = self.copy_abs()
         else:
             ans = self.copy_negate()
 
-        if context is None:
-            context = getcontext()
         return ans._fix(context)
 
     def __pos__(self, context=None):
@@ -1088,14 +1090,15 @@
             if ans:
                 return ans
 
-        if not self:
-            # + (-0) = 0
+        if context is None:
+            context = getcontext()
+
+        if not self and context.rounding != ROUND_FLOOR:
+            # + (-0) = 0, except in ROUND_FLOOR rounding mode.
             ans = self.copy_abs()
         else:
             ans = Decimal(self)
 
-        if context is None:
-            context = getcontext()
         return ans._fix(context)
 
     def __abs__(self, round=True, context=None):
@@ -1680,7 +1683,7 @@
                 self = _dec_from_triple(self._sign, '1', exp_min-1)
                 digits = 0
             rounding_method = self._pick_rounding_function[context.rounding]
-            changed = getattr(self, rounding_method)(digits)
+            changed = rounding_method(self, digits)
             coeff = self._int[:digits] or '0'
             if changed > 0:
                 coeff = str(int(coeff)+1)
@@ -1720,8 +1723,6 @@
         # here self was representable to begin with; return unchanged
         return Decimal(self)
 
-    _pick_rounding_function = {}
-
     # for each of the rounding functions below:
     #   self is a finite, nonzero Decimal
     #   prec is an integer satisfying 0 <= prec < len(self._int)
@@ -1788,6 +1789,17 @@
         else:
             return -self._round_down(prec)
 
+    _pick_rounding_function = dict(
+        ROUND_DOWN = _round_down,
+        ROUND_UP = _round_up,
+        ROUND_HALF_UP = _round_half_up,
+        ROUND_HALF_DOWN = _round_half_down,
+        ROUND_HALF_EVEN = _round_half_even,
+        ROUND_CEILING = _round_ceiling,
+        ROUND_FLOOR = _round_floor,
+        ROUND_05UP = _round_05up,
+    )
+
     def fma(self, other, third, context=None):
         """Fused multiply-add.
 
@@ -2492,8 +2504,8 @@
         if digits < 0:
             self = _dec_from_triple(self._sign, '1', exp-1)
             digits = 0
-        this_function = getattr(self, self._pick_rounding_function[rounding])
-        changed = this_function(digits)
+        this_function = self._pick_rounding_function[rounding]
+        changed = this_function(self, digits)
         coeff = self._int[:digits] or '0'
         if changed == 1:
             coeff = str(int(coeff)+1)
@@ -3705,18 +3717,6 @@
 
 ##### Context class #######################################################
 
-
-# get rounding method function:
-rounding_functions = [name for name in Decimal.__dict__.keys()
-                                    if name.startswith('_round_')]
-for name in rounding_functions:
-    # name is like _round_half_even, goes to the global ROUND_HALF_EVEN value.
-    globalname = name[1:].upper()
-    val = globals()[globalname]
-    Decimal._pick_rounding_function[val] = name
-
-del name, val, globalname, rounding_functions
-
 class _ContextManager(object):
     """Context manager class to support localcontext().
 
@@ -5990,7 +5990,7 @@
 
 def _format_align(sign, body, spec):
     """Given an unpadded, non-aligned numeric string 'body' and sign
-    string 'sign', add padding and aligment conforming to the given
+    string 'sign', add padding and alignment conforming to the given
     format specifier dictionary 'spec' (as produced by
     parse_format_specifier).
 
diff --git a/lib-python/2.7/difflib.py b/lib-python/2.7/difflib.py
--- a/lib-python/2.7/difflib.py
+++ b/lib-python/2.7/difflib.py
@@ -1140,6 +1140,21 @@
     return ch in ws
 
 
+########################################################################
+###  Unified Diff
+########################################################################
+
+def _format_range_unified(start, stop):
+    'Convert range to the "ed" format'
+    # Per the diff spec at http://www.unix.org/single_unix_specification/
+    beginning = start + 1     # lines start numbering with one
+    length = stop - start
+    if length == 1:
+        return '{}'.format(beginning)
+    if not length:
+        beginning -= 1        # empty ranges begin at line just before the range
+    return '{},{}'.format(beginning, length)
+
 def unified_diff(a, b, fromfile='', tofile='', fromfiledate='',
                  tofiledate='', n=3, lineterm='\n'):
     r"""
@@ -1184,25 +1199,45 @@
     started = False
     for group in SequenceMatcher(None,a,b).get_grouped_opcodes(n):
         if not started:
-            fromdate = '\t%s' % fromfiledate if fromfiledate else ''
-            todate = '\t%s' % tofiledate if tofiledate else ''
-            yield '--- %s%s%s' % (fromfile, fromdate, lineterm)
-            yield '+++ %s%s%s' % (tofile, todate, lineterm)
             started = True
-        i1, i2, j1, j2 = group[0][1], group[-1][2], group[0][3], group[-1][4]
-        yield "@@ -%d,%d +%d,%d @@%s" % (i1+1, i2-i1, j1+1, j2-j1, lineterm)
+            fromdate = '\t{}'.format(fromfiledate) if fromfiledate else ''
+            todate = '\t{}'.format(tofiledate) if tofiledate else ''
+            yield '--- {}{}{}'.format(fromfile, fromdate, lineterm)
+            yield '+++ {}{}{}'.format(tofile, todate, lineterm)
+
+        first, last = group[0], group[-1]
+        file1_range = _format_range_unified(first[1], last[2])
+        file2_range = _format_range_unified(first[3], last[4])
+        yield '@@ -{} +{} @@{}'.format(file1_range, file2_range, lineterm)
+
         for tag, i1, i2, j1, j2 in group:
             if tag == 'equal':
                 for line in a[i1:i2]:
                     yield ' ' + line
                 continue
-            if tag == 'replace' or tag == 'delete':
+            if tag in ('replace', 'delete'):
                 for line in a[i1:i2]:
                     yield '-' + line
-            if tag == 'replace' or tag == 'insert':
+            if tag in ('replace', 'insert'):
                 for line in b[j1:j2]:
                     yield '+' + line
 
+
+########################################################################
+###  Context Diff
+########################################################################
+
+def _format_range_context(start, stop):
+    'Convert range to the "ed" format'
+    # Per the diff spec at http://www.unix.org/single_unix_specification/
+    beginning = start + 1     # lines start numbering with one
+    length = stop - start
+    if not length:
+        beginning -= 1        # empty ranges begin at line just before the range
+    if length <= 1:
+        return '{}'.format(beginning)
+    return '{},{}'.format(beginning, beginning + length - 1)
+
 # See http://www.unix.org/single_unix_specification/
 def context_diff(a, b, fromfile='', tofile='',
                  fromfiledate='', tofiledate='', n=3, lineterm='\n'):
@@ -1247,38 +1282,36 @@
       four
     """
 
+    prefix = dict(insert='+ ', delete='- ', replace='! ', equal='  ')
     started = False
-    prefixmap = {'insert':'+ ', 'delete':'- ', 'replace':'! ', 'equal':'  '}
     for group in SequenceMatcher(None,a,b).get_grouped_opcodes(n):
         if not started:
-            fromdate = '\t%s' % fromfiledate if fromfiledate else ''
-            todate = '\t%s' % tofiledate if tofiledate else ''
-            yield '*** %s%s%s' % (fromfile, fromdate, lineterm)
-            yield '--- %s%s%s' % (tofile, todate, lineterm)
             started = True
+            fromdate = '\t{}'.format(fromfiledate) if fromfiledate else ''
+            todate = '\t{}'.format(tofiledate) if tofiledate else ''
+            yield '*** {}{}{}'.format(fromfile, fromdate, lineterm)
+            yield '--- {}{}{}'.format(tofile, todate, lineterm)
 
-        yield '***************%s' % (lineterm,)
-        if group[-1][2] - group[0][1] >= 2:
-            yield '*** %d,%d ****%s' % (group[0][1]+1, group[-1][2], lineterm)
-        else:
-            yield '*** %d ****%s' % (group[-1][2], lineterm)
-        visiblechanges = [e for e in group if e[0] in ('replace', 'delete')]
-        if visiblechanges:
+        first, last = group[0], group[-1]
+        yield '***************' + lineterm
+
+        file1_range = _format_range_context(first[1], last[2])
+        yield '*** {} ****{}'.format(file1_range, lineterm)
+
+        if any(tag in ('replace', 'delete') for tag, _, _, _, _ in group):
             for tag, i1, i2, _, _ in group:
                 if tag != 'insert':
                     for line in a[i1:i2]:
-                        yield prefixmap[tag] + line
+                        yield prefix[tag] + line
 
-        if group[-1][4] - group[0][3] >= 2:
-            yield '--- %d,%d ----%s' % (group[0][3]+1, group[-1][4], lineterm)
-        else:
-            yield '--- %d ----%s' % (group[-1][4], lineterm)
-        visiblechanges = [e for e in group if e[0] in ('replace', 'insert')]
-        if visiblechanges:
+        file2_range = _format_range_context(first[3], last[4])
+        yield '--- {} ----{}'.format(file2_range, lineterm)
+
+        if any(tag in ('replace', 'insert') for tag, _, _, _, _ in group):
             for tag, _, _, j1, j2 in group:
                 if tag != 'delete':
                     for line in b[j1:j2]:
-                        yield prefixmap[tag] + line
+                        yield prefix[tag] + line
 
 def ndiff(a, b, linejunk=None, charjunk=IS_CHARACTER_JUNK):
     r"""
@@ -1714,7 +1747,7 @@
             line = line.replace(' ','\0')
             # expand tabs into spaces
             line = line.expandtabs(self._tabsize)
-            # relace spaces from expanded tabs back into tab characters
+            # replace spaces from expanded tabs back into tab characters
             # (we'll replace them with markup after we do differencing)
             line = line.replace(' ','\t')
             return line.replace('\0',' ').rstrip('\n')
diff --git a/lib-python/2.7/distutils/__init__.py b/lib-python/2.7/distutils/__init__.py
--- a/lib-python/2.7/distutils/__init__.py
+++ b/lib-python/2.7/distutils/__init__.py
@@ -15,5 +15,5 @@
 # Updated automatically by the Python release process.
 #
 #--start constants--
-__version__ = "2.7.1"
+__version__ = "2.7.2"
 #--end constants--
diff --git a/lib-python/2.7/distutils/archive_util.py b/lib-python/2.7/distutils/archive_util.py
--- a/lib-python/2.7/distutils/archive_util.py
+++ b/lib-python/2.7/distutils/archive_util.py
@@ -121,7 +121,7 @@
 def make_zipfile(base_name, base_dir, verbose=0, dry_run=0):
     """Create a zip file from all the files under 'base_dir'.
 
-    The output zip file will be named 'base_dir' + ".zip".  Uses either the
+    The output zip file will be named 'base_name' + ".zip".  Uses either the
     "zipfile" Python module (if available) or the InfoZIP "zip" utility
     (if installed and found on the default search path).  If neither tool is
     available, raises DistutilsExecError.  Returns the name of the output zip
diff --git a/lib-python/2.7/distutils/cmd.py b/lib-python/2.7/distutils/cmd.py
--- a/lib-python/2.7/distutils/cmd.py
+++ b/lib-python/2.7/distutils/cmd.py
@@ -377,7 +377,7 @@
             dry_run=self.dry_run)
 
     def move_file (self, src, dst, level=1):
-        """Move a file respectin dry-run flag."""
+        """Move a file respecting dry-run flag."""
         return file_util.move_file(src, dst, dry_run = self.dry_run)
 
     def spawn (self, cmd, search_path=1, level=1):
diff --git a/lib-python/2.7/distutils/command/build_ext.py b/lib-python/2.7/distutils/command/build_ext.py
--- a/lib-python/2.7/distutils/command/build_ext.py
+++ b/lib-python/2.7/distutils/command/build_ext.py
@@ -207,7 +207,7 @@
 
             elif MSVC_VERSION == 8:
                 self.library_dirs.append(os.path.join(sys.exec_prefix,
-                                         'PC', 'VS8.0', 'win32release'))
+                                         'PC', 'VS8.0'))
             elif MSVC_VERSION == 7:
                 self.library_dirs.append(os.path.join(sys.exec_prefix,
                                          'PC', 'VS7.1'))
diff --git a/lib-python/2.7/distutils/command/sdist.py b/lib-python/2.7/distutils/command/sdist.py
--- a/lib-python/2.7/distutils/command/sdist.py
+++ b/lib-python/2.7/distutils/command/sdist.py
@@ -306,17 +306,20 @@
                             rstrip_ws=1,
                             collapse_join=1)
 
-        while 1:
-            line = template.readline()
-            if line is None:            # end of file
-                break
+        try:
+            while 1:
+                line = template.readline()
+                if line is None:            # end of file
+                    break
 
-            try:
-                self.filelist.process_template_line(line)
-            except DistutilsTemplateError, msg:
-                self.warn("%s, line %d: %s" % (template.filename,
-                                               template.current_line,
-                                               msg))
+                try:
+                    self.filelist.process_template_line(line)
+                except DistutilsTemplateError, msg:
+                    self.warn("%s, line %d: %s" % (template.filename,
+                                                   template.current_line,
+                                                   msg))
+        finally:
+            template.close()
 
     def prune_file_list(self):
         """Prune off branches that might slip into the file list as created
diff --git a/lib-python/2.7/distutils/command/upload.py b/lib-python/2.7/distutils/command/upload.py
--- a/lib-python/2.7/distutils/command/upload.py
+++ b/lib-python/2.7/distutils/command/upload.py
@@ -176,6 +176,9 @@
             result = urlopen(request)
             status = result.getcode()
             reason = result.msg
+            if self.show_response:
+                msg = '\n'.join(('-' * 75, r.read(), '-' * 75))
+                self.announce(msg, log.INFO)
         except socket.error, e:
             self.announce(str(e), log.ERROR)
             return
@@ -189,6 +192,3 @@
         else:
             self.announce('Upload failed (%s): %s' % (status, reason),
                           log.ERROR)
-        if self.show_response:
-            msg = '\n'.join(('-' * 75, r.read(), '-' * 75))
-            self.announce(msg, log.INFO)
diff --git a/lib-python/2.7/distutils/sysconfig.py b/lib-python/2.7/distutils/sysconfig.py
--- a/lib-python/2.7/distutils/sysconfig.py
+++ b/lib-python/2.7/distutils/sysconfig.py
@@ -389,7 +389,7 @@
         cur_target = os.getenv('MACOSX_DEPLOYMENT_TARGET', '')
         if cur_target == '':
             cur_target = cfg_target
-            os.putenv('MACOSX_DEPLOYMENT_TARGET', cfg_target)
+            os.environ['MACOSX_DEPLOYMENT_TARGET'] = cfg_target
         elif map(int, cfg_target.split('.')) > map(int, cur_target.split('.')):
             my_msg = ('$MACOSX_DEPLOYMENT_TARGET mismatch: now "%s" but "%s" during configure'
                 % (cur_target, cfg_target))
diff --git a/lib-python/2.7/distutils/tests/__init__.py b/lib-python/2.7/distutils/tests/__init__.py
--- a/lib-python/2.7/distutils/tests/__init__.py
+++ b/lib-python/2.7/distutils/tests/__init__.py
@@ -15,9 +15,10 @@
 import os
 import sys
 import unittest
+from test.test_support import run_unittest
 
 
-here = os.path.dirname(__file__)
+here = os.path.dirname(__file__) or os.curdir
 
 
 def test_suite():
@@ -32,4 +33,4 @@
 
 
 if __name__ == "__main__":
-    unittest.main(defaultTest="test_suite")
+    run_unittest(test_suite())
diff --git a/lib-python/2.7/distutils/tests/test_archive_util.py b/lib-python/2.7/distutils/tests/test_archive_util.py
--- a/lib-python/2.7/distutils/tests/test_archive_util.py
+++ b/lib-python/2.7/distutils/tests/test_archive_util.py
@@ -12,7 +12,7 @@
                                     ARCHIVE_FORMATS)
 from distutils.spawn import find_executable, spawn
 from distutils.tests import support
-from test.test_support import check_warnings
+from test.test_support import check_warnings, run_unittest
 
 try:
     import grp
@@ -281,4 +281,4 @@
     return unittest.makeSuite(ArchiveUtilTestCase)
 
 if __name__ == "__main__":
-    unittest.main(defaultTest="test_suite")
+    run_unittest(test_suite())
diff --git a/lib-python/2.7/distutils/tests/test_bdist_msi.py b/lib-python/2.7/distutils/tests/test_bdist_msi.py
--- a/lib-python/2.7/distutils/tests/test_bdist_msi.py
+++ b/lib-python/2.7/distutils/tests/test_bdist_msi.py
@@ -11,7 +11,7 @@
                        support.LoggingSilencer,
                        unittest.TestCase):
 
-    def test_minial(self):
+    def test_minimal(self):
         # minimal test XXX need more tests
         from distutils.command.bdist_msi import bdist_msi
         pkg_pth, dist = self.create_dist()
diff --git a/lib-python/2.7/distutils/tests/test_build.py b/lib-python/2.7/distutils/tests/test_build.py
--- a/lib-python/2.7/distutils/tests/test_build.py
+++ b/lib-python/2.7/distutils/tests/test_build.py
@@ -2,6 +2,7 @@
 import unittest
 import os
 import sys
+from test.test_support import run_unittest
 
 from distutils.command.build import build
 from distutils.tests import support
@@ -51,4 +52,4 @@
     return unittest.makeSuite(BuildTestCase)
 
 if __name__ == "__main__":
-    unittest.main(defaultTest="test_suite")
+    run_unittest(test_suite())
diff --git a/lib-python/2.7/distutils/tests/test_build_clib.py b/lib-python/2.7/distutils/tests/test_build_clib.py
--- a/lib-python/2.7/distutils/tests/test_build_clib.py
+++ b/lib-python/2.7/distutils/tests/test_build_clib.py
@@ -3,6 +3,8 @@
 import os
 import sys
 
+from test.test_support import run_unittest
+
 from distutils.command.build_clib import build_clib
 from distutils.errors import DistutilsSetupError
 from distutils.tests import support
@@ -140,4 +142,4 @@
     return unittest.makeSuite(BuildCLibTestCase)
 
 if __name__ == "__main__":
-    unittest.main(defaultTest="test_suite")
+    run_unittest(test_suite())
diff --git a/lib-python/2.7/distutils/tests/test_build_ext.py b/lib-python/2.7/distutils/tests/test_build_ext.py
--- a/lib-python/2.7/distutils/tests/test_build_ext.py
+++ b/lib-python/2.7/distutils/tests/test_build_ext.py
@@ -3,12 +3,13 @@
 import tempfile
 import shutil
 from StringIO import StringIO
+import textwrap
 
 from distutils.core import Extension, Distribution
 from distutils.command.build_ext import build_ext
 from distutils import sysconfig
 from distutils.tests import support
-from distutils.errors import DistutilsSetupError
+from distutils.errors import DistutilsSetupError, CompileError
 
 import unittest
 from test import test_support
@@ -430,6 +431,59 @@
         wanted = os.path.join(cmd.build_lib, 'UpdateManager', 'fdsend' + ext)
         self.assertEqual(ext_path, wanted)
 
+    @unittest.skipUnless(sys.platform == 'darwin', 'test only relevant for MacOSX')
+    def test_deployment_target(self):
+        self._try_compile_deployment_target()
+
+        orig_environ = os.environ
+        os.environ = orig_environ.copy()
+        self.addCleanup(setattr, os, 'environ', orig_environ)
+
+        os.environ['MACOSX_DEPLOYMENT_TARGET']='10.1'
+        self._try_compile_deployment_target()
+
+
+    def _try_compile_deployment_target(self):
+        deptarget_c = os.path.join(self.tmp_dir, 'deptargetmodule.c')
+
+        with open(deptarget_c, 'w') as fp:
+            fp.write(textwrap.dedent('''\
+                #include 
+
+                int dummy;
+
+                #if TARGET != MAC_OS_X_VERSION_MIN_REQUIRED
+                #error "Unexpected target"
+               #endif
+
+            '''))
+
+        target = sysconfig.get_config_var('MACOSX_DEPLOYMENT_TARGET')
+        target = tuple(map(int, target.split('.')))
+        target = '%02d%01d0' % target
+
+        deptarget_ext = Extension(
+            'deptarget',
+            [deptarget_c],
+            extra_compile_args=['-DTARGET=%s'%(target,)],
+        )
+        dist = Distribution({
+            'name': 'deptarget',
+            'ext_modules': [deptarget_ext]
+        })
+        dist.package_dir = self.tmp_dir
+        cmd = build_ext(dist)
+        cmd.build_lib = self.tmp_dir
+        cmd.build_temp = self.tmp_dir
+
+        try:
+            old_stdout = sys.stdout
+            cmd.ensure_finalized()
+            cmd.run()
+
+        except CompileError:
+            self.fail("Wrong deployment target during compilation")
+
 def test_suite():
     return unittest.makeSuite(BuildExtTestCase)
 
diff --git a/lib-python/2.7/distutils/tests/test_build_py.py b/lib-python/2.7/distutils/tests/test_build_py.py
--- a/lib-python/2.7/distutils/tests/test_build_py.py
+++ b/lib-python/2.7/distutils/tests/test_build_py.py
@@ -10,13 +10,14 @@
 from distutils.errors import DistutilsFileError
 
 from distutils.tests import support
+from test.test_support import run_unittest
 
 
 class BuildPyTestCase(support.TempdirManager,
                       support.LoggingSilencer,
                       unittest.TestCase):
 
-    def _setup_package_data(self):
+    def test_package_data(self):
         sources = self.mkdtemp()
         f = open(os.path.join(sources, "__init__.py"), "w")
         try:
@@ -56,20 +57,15 @@
         self.assertEqual(len(cmd.get_outputs()), 3)
         pkgdest = os.path.join(destination, "pkg")
         files = os.listdir(pkgdest)
-        return files
+        self.assertIn("__init__.py", files)
+        self.assertIn("README.txt", files)
+        # XXX even with -O, distutils writes pyc, not pyo; bug?
+        if sys.dont_write_bytecode:
+            self.assertNotIn("__init__.pyc", files)
+        else:
+            self.assertIn("__init__.pyc", files)
 
-    def test_package_data(self):
-        files = self._setup_package_data()
-        self.assertTrue("__init__.py" in files)
-        self.assertTrue("README.txt" in files)
-
-    @unittest.skipIf(sys.flags.optimize >= 2,
-                     "pyc files are not written with -O2 and above")
-    def test_package_data_pyc(self):
-        files = self._setup_package_data()
-        self.assertTrue("__init__.pyc" in files)
-
-    def test_empty_package_dir (self):
+    def test_empty_package_dir(self):
         # See SF 1668596/1720897.
         cwd = os.getcwd()
 
@@ -117,10 +113,10 @@
         finally:
             sys.dont_write_bytecode = old_dont_write_bytecode
 
-        self.assertTrue('byte-compiling is disabled' in self.logs[0][1])
+        self.assertIn('byte-compiling is disabled', self.logs[0][1])
 
 def test_suite():
     return unittest.makeSuite(BuildPyTestCase)
 
 if __name__ == "__main__":
-    unittest.main(defaultTest="test_suite")
+    run_unittest(test_suite())
diff --git a/lib-python/2.7/distutils/tests/test_build_scripts.py b/lib-python/2.7/distutils/tests/test_build_scripts.py
--- a/lib-python/2.7/distutils/tests/test_build_scripts.py
+++ b/lib-python/2.7/distutils/tests/test_build_scripts.py
@@ -8,6 +8,7 @@
 import sysconfig
 
 from distutils.tests import support
+from test.test_support import run_unittest
 
 
 class BuildScriptsTestCase(support.TempdirManager,
@@ -108,4 +109,4 @@
     return unittest.makeSuite(BuildScriptsTestCase)
 
 if __name__ == "__main__":
-    unittest.main(defaultTest="test_suite")
+    run_unittest(test_suite())
diff --git a/lib-python/2.7/distutils/tests/test_check.py b/lib-python/2.7/distutils/tests/test_check.py
--- a/lib-python/2.7/distutils/tests/test_check.py
+++ b/lib-python/2.7/distutils/tests/test_check.py
@@ -1,5 +1,6 @@
 """Tests for distutils.command.check."""
 import unittest
+from test.test_support import run_unittest
 
 from distutils.command.check import check, HAS_DOCUTILS
 from distutils.tests import support
@@ -95,4 +96,4 @@
     return unittest.makeSuite(CheckTestCase)
 
 if __name__ == "__main__":
-    unittest.main(defaultTest="test_suite")
+    run_unittest(test_suite())
diff --git a/lib-python/2.7/distutils/tests/test_clean.py b/lib-python/2.7/distutils/tests/test_clean.py
--- a/lib-python/2.7/distutils/tests/test_clean.py
+++ b/lib-python/2.7/distutils/tests/test_clean.py
@@ -6,6 +6,7 @@
 
 from distutils.command.clean import clean
 from distutils.tests import support
+from test.test_support import run_unittest
 
 class cleanTestCase(support.TempdirManager,
                     support.LoggingSilencer,
@@ -38,7 +39,7 @@
             self.assertTrue(not os.path.exists(path),
                          '%s was not removed' % path)
 
-        # let's run the command again (should spit warnings but suceed)
+        # let's run the command again (should spit warnings but succeed)
         cmd.all = 1
         cmd.ensure_finalized()
         cmd.run()
@@ -47,4 +48,4 @@
     return unittest.makeSuite(cleanTestCase)
 
 if __name__ == "__main__":
-    unittest.main(defaultTest="test_suite")
+    run_unittest(test_suite())
diff --git a/lib-python/2.7/distutils/tests/test_cmd.py b/lib-python/2.7/distutils/tests/test_cmd.py
--- a/lib-python/2.7/distutils/tests/test_cmd.py
+++ b/lib-python/2.7/distutils/tests/test_cmd.py
@@ -99,7 +99,7 @@
 
     def test_ensure_dirname(self):
         cmd = self.cmd
-        cmd.option1 = os.path.dirname(__file__)
+        cmd.option1 = os.path.dirname(__file__) or os.curdir
         cmd.ensure_dirname('option1')
         cmd.option2 = 'xxx'
         self.assertRaises(DistutilsOptionError, cmd.ensure_dirname, 'option2')
diff --git a/lib-python/2.7/distutils/tests/test_config.py b/lib-python/2.7/distutils/tests/test_config.py
--- a/lib-python/2.7/distutils/tests/test_config.py
+++ b/lib-python/2.7/distutils/tests/test_config.py
@@ -11,6 +11,7 @@
 from distutils.log import WARN
 
 from distutils.tests import support
+from test.test_support import run_unittest
 
 PYPIRC = """\
 [distutils]
@@ -119,4 +120,4 @@
     return unittest.makeSuite(PyPIRCCommandTestCase)
 
 if __name__ == "__main__":
-    unittest.main(defaultTest="test_suite")
+    run_unittest(test_suite())
diff --git a/lib-python/2.7/distutils/tests/test_config_cmd.py b/lib-python/2.7/distutils/tests/test_config_cmd.py
--- a/lib-python/2.7/distutils/tests/test_config_cmd.py
+++ b/lib-python/2.7/distutils/tests/test_config_cmd.py
@@ -2,6 +2,7 @@
 import unittest
 import os
 import sys
+from test.test_support import run_unittest
 
 from distutils.command.config import dump_file, config
 from distutils.tests import support
@@ -86,4 +87,4 @@
     return unittest.makeSuite(ConfigTestCase)
 
 if __name__ == "__main__":
-    unittest.main(defaultTest="test_suite")
+    run_unittest(test_suite())
diff --git a/lib-python/2.7/distutils/tests/test_core.py b/lib-python/2.7/distutils/tests/test_core.py
--- a/lib-python/2.7/distutils/tests/test_core.py
+++ b/lib-python/2.7/distutils/tests/test_core.py
@@ -6,7 +6,7 @@
 import shutil
 import sys
 import test.test_support
-from test.test_support import captured_stdout
+from test.test_support import captured_stdout, run_unittest
 import unittest
 from distutils.tests import support
 
@@ -105,4 +105,4 @@
     return unittest.makeSuite(CoreTestCase)
 
 if __name__ == "__main__":
-    unittest.main(defaultTest="test_suite")
+    run_unittest(test_suite())
diff --git a/lib-python/2.7/distutils/tests/test_dep_util.py b/lib-python/2.7/distutils/tests/test_dep_util.py
--- a/lib-python/2.7/distutils/tests/test_dep_util.py
+++ b/lib-python/2.7/distutils/tests/test_dep_util.py
@@ -6,6 +6,7 @@
 from distutils.dep_util import newer, newer_pairwise, newer_group
 from distutils.errors import DistutilsFileError
 from distutils.tests import support
+from test.test_support import run_unittest
 
 class DepUtilTestCase(support.TempdirManager, unittest.TestCase):
 
@@ -77,4 +78,4 @@
     return unittest.makeSuite(DepUtilTestCase)
 
 if __name__ == "__main__":
-    unittest.main(defaultTest="test_suite")
+    run_unittest(test_suite())
diff --git a/lib-python/2.7/distutils/tests/test_dir_util.py b/lib-python/2.7/distutils/tests/test_dir_util.py
--- a/lib-python/2.7/distutils/tests/test_dir_util.py
+++ b/lib-python/2.7/distutils/tests/test_dir_util.py
@@ -10,6 +10,7 @@
 
 from distutils import log
 from distutils.tests import support
+from test.test_support import run_unittest
 
 class DirUtilTestCase(support.TempdirManager, unittest.TestCase):
 
@@ -112,4 +113,4 @@
     return unittest.makeSuite(DirUtilTestCase)
 
 if __name__ == "__main__":
-    unittest.main(defaultTest="test_suite")
+    run_unittest(test_suite())
diff --git a/lib-python/2.7/distutils/tests/test_dist.py b/lib-python/2.7/distutils/tests/test_dist.py
--- a/lib-python/2.7/distutils/tests/test_dist.py
+++ b/lib-python/2.7/distutils/tests/test_dist.py
@@ -11,7 +11,7 @@
 from distutils.dist import Distribution, fix_help_options, DistributionMetadata
 from distutils.cmd import Command
 import distutils.dist
-from test.test_support import TESTFN, captured_stdout
+from test.test_support import TESTFN, captured_stdout, run_unittest
 from distutils.tests import support
 
 class test_dist(Command):
@@ -433,4 +433,4 @@
     return suite
 
 if __name__ == "__main__":
-    unittest.main(defaultTest="test_suite")
+    run_unittest(test_suite())
diff --git a/lib-python/2.7/distutils/tests/test_file_util.py b/lib-python/2.7/distutils/tests/test_file_util.py
--- a/lib-python/2.7/distutils/tests/test_file_util.py
+++ b/lib-python/2.7/distutils/tests/test_file_util.py
@@ -6,6 +6,7 @@
 from distutils.file_util import move_file, write_file, copy_file
 from distutils import log
 from distutils.tests import support
+from test.test_support import run_unittest
 
 class FileUtilTestCase(support.TempdirManager, unittest.TestCase):
 
@@ -77,4 +78,4 @@
     return unittest.makeSuite(FileUtilTestCase)
 
 if __name__ == "__main__":
-    unittest.main(defaultTest="test_suite")
+    run_unittest(test_suite())
diff --git a/lib-python/2.7/distutils/tests/test_filelist.py b/lib-python/2.7/distutils/tests/test_filelist.py
--- a/lib-python/2.7/distutils/tests/test_filelist.py
+++ b/lib-python/2.7/distutils/tests/test_filelist.py
@@ -1,7 +1,7 @@
 """Tests for distutils.filelist."""
 from os.path import join
 import unittest
-from test.test_support import captured_stdout
+from test.test_support import captured_stdout, run_unittest
 
 from distutils.filelist import glob_to_re, FileList
 from distutils import debug
@@ -82,4 +82,4 @@
     return unittest.makeSuite(FileListTestCase)
 
 if __name__ == "__main__":
-    unittest.main(defaultTest="test_suite")
+    run_unittest(test_suite())
diff --git a/lib-python/2.7/distutils/tests/test_install.py b/lib-python/2.7/distutils/tests/test_install.py
--- a/lib-python/2.7/distutils/tests/test_install.py
+++ b/lib-python/2.7/distutils/tests/test_install.py
@@ -3,6 +3,8 @@
 import os
 import unittest
 
+from test.test_support import run_unittest
+
 from distutils.command.install import install
 from distutils.core import Distribution
 
@@ -52,4 +54,4 @@
     return unittest.makeSuite(InstallTestCase)
 
 if __name__ == "__main__":
-    unittest.main(defaultTest="test_suite")
+    run_unittest(test_suite())
diff --git a/lib-python/2.7/distutils/tests/test_install_data.py b/lib-python/2.7/distutils/tests/test_install_data.py
--- a/lib-python/2.7/distutils/tests/test_install_data.py
+++ b/lib-python/2.7/distutils/tests/test_install_data.py
@@ -6,6 +6,7 @@
 
 from distutils.command.install_data import install_data
 from distutils.tests import support
+from test.test_support import run_unittest
 
 class InstallDataTestCase(support.TempdirManager,
                           support.LoggingSilencer,
@@ -73,4 +74,4 @@
     return unittest.makeSuite(InstallDataTestCase)
 
 if __name__ == "__main__":
-    unittest.main(defaultTest="test_suite")
+    run_unittest(test_suite())
diff --git a/lib-python/2.7/distutils/tests/test_install_headers.py b/lib-python/2.7/distutils/tests/test_install_headers.py
--- a/lib-python/2.7/distutils/tests/test_install_headers.py
+++ b/lib-python/2.7/distutils/tests/test_install_headers.py
@@ -6,6 +6,7 @@
 
 from distutils.command.install_headers import install_headers
 from distutils.tests import support
+from test.test_support import run_unittest
 
 class InstallHeadersTestCase(support.TempdirManager,
                              support.LoggingSilencer,
@@ -37,4 +38,4 @@
     return unittest.makeSuite(InstallHeadersTestCase)
 
 if __name__ == "__main__":
-    unittest.main(defaultTest="test_suite")
+    run_unittest(test_suite())
diff --git a/lib-python/2.7/distutils/tests/test_install_lib.py b/lib-python/2.7/distutils/tests/test_install_lib.py
--- a/lib-python/2.7/distutils/tests/test_install_lib.py
+++ b/lib-python/2.7/distutils/tests/test_install_lib.py
@@ -7,6 +7,7 @@
 from distutils.extension import Extension
 from distutils.tests import support
 from distutils.errors import DistutilsOptionError
+from test.test_support import run_unittest
 
 class InstallLibTestCase(support.TempdirManager,
                          support.LoggingSilencer,
@@ -103,4 +104,4 @@
     return unittest.makeSuite(InstallLibTestCase)
 
 if __name__ == "__main__":
-    unittest.main(defaultTest="test_suite")
+    run_unittest(test_suite())
diff --git a/lib-python/2.7/distutils/tests/test_install_scripts.py b/lib-python/2.7/distutils/tests/test_install_scripts.py
--- a/lib-python/2.7/distutils/tests/test_install_scripts.py
+++ b/lib-python/2.7/distutils/tests/test_install_scripts.py
@@ -7,6 +7,7 @@
 from distutils.core import Distribution
 
 from distutils.tests import support
+from test.test_support import run_unittest
 
 
 class InstallScriptsTestCase(support.TempdirManager,
@@ -78,4 +79,4 @@
     return unittest.makeSuite(InstallScriptsTestCase)
 
 if __name__ == "__main__":
-    unittest.main(defaultTest="test_suite")
+    run_unittest(test_suite())
diff --git a/lib-python/2.7/distutils/tests/test_msvc9compiler.py b/lib-python/2.7/distutils/tests/test_msvc9compiler.py
--- a/lib-python/2.7/distutils/tests/test_msvc9compiler.py
+++ b/lib-python/2.7/distutils/tests/test_msvc9compiler.py
@@ -5,6 +5,7 @@
 
 from distutils.errors import DistutilsPlatformError
 from distutils.tests import support
+from test.test_support import run_unittest
 
 _MANIFEST = """\
 
@@ -137,4 +138,4 @@
     return unittest.makeSuite(msvc9compilerTestCase)
 
 if __name__ == "__main__":
-    unittest.main(defaultTest="test_suite")
+    run_unittest(test_suite())
diff --git a/lib-python/2.7/distutils/tests/test_register.py b/lib-python/2.7/distutils/tests/test_register.py
--- a/lib-python/2.7/distutils/tests/test_register.py
+++ b/lib-python/2.7/distutils/tests/test_register.py
@@ -7,7 +7,7 @@
 import urllib2
 import warnings
 
-from test.test_support import check_warnings
+from test.test_support import check_warnings, run_unittest
 
 from distutils.command import register as register_module
 from distutils.command.register import register
@@ -138,7 +138,7 @@
 
         # let's see what the server received : we should
         # have 2 similar requests
-        self.assertTrue(self.conn.reqs, 2)
+        self.assertEqual(len(self.conn.reqs), 2)
         req1 = dict(self.conn.reqs[0].headers)
         req2 = dict(self.conn.reqs[1].headers)
         self.assertEqual(req2['Content-length'], req1['Content-length'])
@@ -168,7 +168,7 @@
             del register_module.raw_input
 
         # we should have send a request
-        self.assertTrue(self.conn.reqs, 1)
+        self.assertEqual(len(self.conn.reqs), 1)
         req = self.conn.reqs[0]
         headers = dict(req.headers)
         self.assertEqual(headers['Content-length'], '608')
@@ -186,7 +186,7 @@
             del register_module.raw_input
 
         # we should have send a request
-        self.assertTrue(self.conn.reqs, 1)
+        self.assertEqual(len(self.conn.reqs), 1)
         req = self.conn.reqs[0]
         headers = dict(req.headers)
         self.assertEqual(headers['Content-length'], '290')
@@ -258,4 +258,4 @@
     return unittest.makeSuite(RegisterTestCase)
 
 if __name__ == "__main__":
-    unittest.main(defaultTest="test_suite")
+    run_unittest(test_suite())
diff --git a/lib-python/2.7/distutils/tests/test_sdist.py b/lib-python/2.7/distutils/tests/test_sdist.py
--- a/lib-python/2.7/distutils/tests/test_sdist.py
+++ b/lib-python/2.7/distutils/tests/test_sdist.py
@@ -24,11 +24,9 @@
 import tempfile
 import warnings
 
-from test.test_support import check_warnings
-from test.test_support import captured_stdout
+from test.test_support import captured_stdout, check_warnings, run_unittest
 
-from distutils.command.sdist import sdist
-from distutils.command.sdist import show_formats
+from distutils.command.sdist import sdist, show_formats
 from distutils.core import Distribution
 from distutils.tests.test_config import PyPIRCCommandTestCase
 from distutils.errors import DistutilsExecError, DistutilsOptionError
@@ -372,7 +370,7 @@
         # adding a file
         self.write_file((self.tmp_dir, 'somecode', 'doc2.txt'), '#')
 
-        # make sure build_py is reinitinialized, like a fresh run
+        # make sure build_py is reinitialized, like a fresh run
         build_py = dist.get_command_obj('build_py')
         build_py.finalized = False
         build_py.ensure_finalized()
@@ -390,6 +388,7 @@
         self.assertEqual(len(manifest2), 6)
         self.assertIn('doc2.txt', manifest2[-1])
 
+    @unittest.skipUnless(zlib, "requires zlib")
     def test_manifest_marker(self):
         # check that autogenerated MANIFESTs have a marker
         dist, cmd = self.get_cmd()
@@ -406,6 +405,7 @@
         self.assertEqual(manifest[0],
                          '# file GENERATED by distutils, do NOT edit')
 
+    @unittest.skipUnless(zlib, "requires zlib")
     def test_manual_manifest(self):
         # check that a MANIFEST without a marker is left alone
         dist, cmd = self.get_cmd()
@@ -426,4 +426,4 @@
     return unittest.makeSuite(SDistTestCase)
 
 if __name__ == "__main__":
-    unittest.main(defaultTest="test_suite")
+    run_unittest(test_suite())
diff --git a/lib-python/2.7/distutils/tests/test_spawn.py b/lib-python/2.7/distutils/tests/test_spawn.py
--- a/lib-python/2.7/distutils/tests/test_spawn.py
+++ b/lib-python/2.7/distutils/tests/test_spawn.py
@@ -2,7 +2,7 @@
 import unittest
 import os
 import time
-from test.test_support import captured_stdout
+from test.test_support import captured_stdout, run_unittest
 
 from distutils.spawn import _nt_quote_args
 from distutils.spawn import spawn, find_executable
@@ -57,4 +57,4 @@
     return unittest.makeSuite(SpawnTestCase)
 
 if __name__ == "__main__":
-    unittest.main(defaultTest="test_suite")
+    run_unittest(test_suite())
diff --git a/lib-python/2.7/distutils/tests/test_text_file.py b/lib-python/2.7/distutils/tests/test_text_file.py
--- a/lib-python/2.7/distutils/tests/test_text_file.py
+++ b/lib-python/2.7/distutils/tests/test_text_file.py
@@ -3,6 +3,7 @@
 import unittest
 from distutils.text_file import TextFile
 from distutils.tests import support
+from test.test_support import run_unittest
 
 TEST_DATA = """# test file
 
@@ -103,4 +104,4 @@
     return unittest.makeSuite(TextFileTestCase)
 
 if __name__ == "__main__":
-    unittest.main(defaultTest="test_suite")
+    run_unittest(test_suite())
diff --git a/lib-python/2.7/distutils/tests/test_unixccompiler.py b/lib-python/2.7/distutils/tests/test_unixccompiler.py
--- a/lib-python/2.7/distutils/tests/test_unixccompiler.py
+++ b/lib-python/2.7/distutils/tests/test_unixccompiler.py
@@ -1,6 +1,7 @@
 """Tests for distutils.unixccompiler."""
 import sys
 import unittest
+from test.test_support import run_unittest
 
 from distutils import sysconfig
 from distutils.unixccompiler import UnixCCompiler
@@ -126,4 +127,4 @@
     return unittest.makeSuite(UnixCCompilerTestCase)
 
 if __name__ == "__main__":
-    unittest.main(defaultTest="test_suite")
+    run_unittest(test_suite())
diff --git a/lib-python/2.7/distutils/tests/test_upload.py b/lib-python/2.7/distutils/tests/test_upload.py
--- a/lib-python/2.7/distutils/tests/test_upload.py
+++ b/lib-python/2.7/distutils/tests/test_upload.py
@@ -1,14 +1,13 @@
+# -*- encoding: utf8 -*-
 """Tests for distutils.command.upload."""
-# -*- encoding: utf8 -*-
-import sys
 import os
 import unittest
+from test.test_support import run_unittest
 
 from distutils.command import upload as upload_mod
 from distutils.command.upload import upload
 from distutils.core import Distribution
 
-from distutils.tests import support
 from distutils.tests.test_config import PYPIRC, PyPIRCCommandTestCase
 
 PYPIRC_LONG_PASSWORD = """\
@@ -129,4 +128,4 @@
     return unittest.makeSuite(uploadTestCase)
 
 if __name__ == "__main__":
-    unittest.main(defaultTest="test_suite")
+    run_unittest(test_suite())
diff --git a/lib-python/2.7/distutils/tests/test_util.py b/lib-python/2.7/distutils/tests/test_util.py
--- a/lib-python/2.7/distutils/tests/test_util.py
+++ b/lib-python/2.7/distutils/tests/test_util.py
@@ -1,6 +1,7 @@
 """Tests for distutils.util."""
 import sys
 import unittest
+from test.test_support import run_unittest
 
 from distutils.errors import DistutilsPlatformError, DistutilsByteCompileError
 from distutils.util import byte_compile
@@ -21,4 +22,4 @@
     return unittest.makeSuite(UtilTestCase)
 
 if __name__ == "__main__":
-    unittest.main(defaultTest="test_suite")
+    run_unittest(test_suite())
diff --git a/lib-python/2.7/distutils/tests/test_version.py b/lib-python/2.7/distutils/tests/test_version.py
--- a/lib-python/2.7/distutils/tests/test_version.py
+++ b/lib-python/2.7/distutils/tests/test_version.py
@@ -2,6 +2,7 @@
 import unittest
 from distutils.version import LooseVersion
 from distutils.version import StrictVersion
+from test.test_support import run_unittest
 
 class VersionTestCase(unittest.TestCase):
 
@@ -67,4 +68,4 @@
     return unittest.makeSuite(VersionTestCase)
 
 if __name__ == "__main__":
-    unittest.main(defaultTest="test_suite")
+    run_unittest(test_suite())
diff --git a/lib-python/2.7/distutils/tests/test_versionpredicate.py b/lib-python/2.7/distutils/tests/test_versionpredicate.py
--- a/lib-python/2.7/distutils/tests/test_versionpredicate.py
+++ b/lib-python/2.7/distutils/tests/test_versionpredicate.py
@@ -4,6 +4,10 @@
 
 import distutils.versionpredicate
 import doctest
+from test.test_support import run_unittest
 
 def test_suite():
     return doctest.DocTestSuite(distutils.versionpredicate)
+
+if __name__ == '__main__':
+    run_unittest(test_suite())
diff --git a/lib-python/2.7/distutils/util.py b/lib-python/2.7/distutils/util.py
--- a/lib-python/2.7/distutils/util.py
+++ b/lib-python/2.7/distutils/util.py
@@ -97,9 +97,7 @@
         from distutils.sysconfig import get_config_vars
         cfgvars = get_config_vars()
 
-        macver = os.environ.get('MACOSX_DEPLOYMENT_TARGET')
-        if not macver:
-            macver = cfgvars.get('MACOSX_DEPLOYMENT_TARGET')
+        macver = cfgvars.get('MACOSX_DEPLOYMENT_TARGET')
 
         if 1:
             # Always calculate the release of the running machine,
diff --git a/lib-python/2.7/doctest.py b/lib-python/2.7/doctest.py
--- a/lib-python/2.7/doctest.py
+++ b/lib-python/2.7/doctest.py
@@ -1217,7 +1217,7 @@
         # Process each example.
         for examplenum, example in enumerate(test.examples):
 
-            # If REPORT_ONLY_FIRST_FAILURE is set, then supress
+            # If REPORT_ONLY_FIRST_FAILURE is set, then suppress
             # reporting after the first failure.
             quiet = (self.optionflags & REPORT_ONLY_FIRST_FAILURE and
                      failures > 0)
@@ -2186,7 +2186,7 @@
            caller can catch the errors and initiate post-mortem debugging.
 
            The DocTestCase provides a debug method that raises
-           UnexpectedException errors if there is an unexepcted
+           UnexpectedException errors if there is an unexpected
            exception:
 
              >>> test = DocTestParser().get_doctest('>>> raise KeyError\n42',
diff --git a/lib-python/2.7/email/charset.py b/lib-python/2.7/email/charset.py
--- a/lib-python/2.7/email/charset.py
+++ b/lib-python/2.7/email/charset.py
@@ -209,7 +209,7 @@
                 input_charset = unicode(input_charset, 'ascii')
         except UnicodeError:
             raise errors.CharsetError(input_charset)
-        input_charset = input_charset.lower()
+        input_charset = input_charset.lower().encode('ascii')
         # Set the input charset after filtering through the aliases and/or codecs
         if not (input_charset in ALIASES or input_charset in CHARSETS):
             try:
diff --git a/lib-python/2.7/email/generator.py b/lib-python/2.7/email/generator.py
--- a/lib-python/2.7/email/generator.py
+++ b/lib-python/2.7/email/generator.py
@@ -202,18 +202,13 @@
             g = self.clone(s)
             g.flatten(part, unixfrom=False)
             msgtexts.append(s.getvalue())
-        # Now make sure the boundary we've selected doesn't appear in any of
-        # the message texts.
-        alltext = NL.join(msgtexts)
         # BAW: What about boundaries that are wrapped in double-quotes?
-        boundary = msg.get_boundary(failobj=_make_boundary(alltext))
-        # If we had to calculate a new boundary because the body text
-        # contained that string, set the new boundary.  We don't do it
-        # unconditionally because, while set_boundary() preserves order, it
-        # doesn't preserve newlines/continuations in headers.  This is no big
-        # deal in practice, but turns out to be inconvenient for the unittest
-        # suite.
-        if msg.get_boundary() != boundary:
+        boundary = msg.get_boundary()
+        if not boundary:
+            # Create a boundary that doesn't appear in any of the
+            # message texts.
+            alltext = NL.join(msgtexts)
+            boundary = _make_boundary(alltext)
             msg.set_boundary(boundary)
         # If there's a preamble, write it out, with a trailing CRLF
         if msg.preamble is not None:
@@ -292,7 +287,7 @@
 _FMT = '[Non-text (%(type)s) part of message omitted, filename %(filename)s]'
 
 class DecodedGenerator(Generator):
-    """Generator a text representation of a message.
+    """Generates a text representation of a message.
 
     Like the Generator base class, except that non-text parts are substituted
     with a format string representing the part.
diff --git a/lib-python/2.7/email/header.py b/lib-python/2.7/email/header.py
--- a/lib-python/2.7/email/header.py
+++ b/lib-python/2.7/email/header.py
@@ -47,6 +47,10 @@
 # For use with .match()
 fcre = re.compile(r'[\041-\176]+:$')
 
+# Find a header embedded in a putative header value.  Used to check for
+# header injection attack.
+_embeded_header = re.compile(r'\n[^ \t]+:')
+
 
 
 # Helpers
@@ -403,7 +407,11 @@
             newchunks += self._split(s, charset, targetlen, splitchars)
             lastchunk, lastcharset = newchunks[-1]
             lastlen = lastcharset.encoded_header_len(lastchunk)
-        return self._encode_chunks(newchunks, maxlinelen)
+        value = self._encode_chunks(newchunks, maxlinelen)
+        if _embeded_header.search(value):
+            raise HeaderParseError("header value appears to contain "
+                "an embedded header: {!r}".format(value))
+        return value
 
 
 
diff --git a/lib-python/2.7/email/message.py b/lib-python/2.7/email/message.py
--- a/lib-python/2.7/email/message.py
+++ b/lib-python/2.7/email/message.py
@@ -38,7 +38,9 @@
 def _formatparam(param, value=None, quote=True):
     """Convenience function to format and return a key=value pair.
 
-    This will quote the value if needed or if quote is true.
+    This will quote the value if needed or if quote is true.  If value is a
+    three tuple (charset, language, value), it will be encoded according
+    to RFC2231 rules.
     """
     if value is not None and len(value) > 0:
         # A tuple is used for RFC 2231 encoded parameter values where items
@@ -97,7 +99,7 @@
     objects, otherwise it is a string.
 
     Message objects implement part of the `mapping' interface, which assumes
-    there is exactly one occurrance of the header per message.  Some headers
+    there is exactly one occurrence of the header per message.  Some headers
     do in fact appear multiple times (e.g. Received) and for those headers,
     you must use the explicit API to set or get all the headers.  Not all of
     the mapping methods are implemented.
@@ -286,7 +288,7 @@
         Return None if the header is missing instead of raising an exception.
 
         Note that if the header appeared multiple times, exactly which
-        occurrance gets returned is undefined.  Use get_all() to get all
+        occurrence gets returned is undefined.  Use get_all() to get all
         the values matching a header field name.
         """
         return self.get(name)
@@ -389,7 +391,10 @@
         name is the header field to add.  keyword arguments can be used to set
         additional parameters for the header field, with underscores converted
         to dashes.  Normally the parameter will be added as key="value" unless
-        value is None, in which case only the key will be added.
+        value is None, in which case only the key will be added.  If a
+        parameter value contains non-ASCII characters it must be specified as a
+        three-tuple of (charset, language, value), in which case it will be
+        encoded according to RFC2231 rules.
 
         Example:
 
diff --git a/lib-python/2.7/email/mime/application.py b/lib-python/2.7/email/mime/application.py
--- a/lib-python/2.7/email/mime/application.py
+++ b/lib-python/2.7/email/mime/application.py
@@ -17,7 +17,7 @@
                  _encoder=encoders.encode_base64, **_params):
         """Create an application/* type MIME document.
 
-        _data is a string containing the raw applicatoin data.
+        _data is a string containing the raw application data.
 
         _subtype is the MIME content type subtype, defaulting to
         'octet-stream'.
diff --git a/lib-python/2.7/email/test/data/msg_26.txt b/lib-python/2.7/email/test/data/msg_26.txt
--- a/lib-python/2.7/email/test/data/msg_26.txt
+++ b/lib-python/2.7/email/test/data/msg_26.txt
@@ -42,4 +42,4 @@
 MzMAAAAACH97tzAAAAALu3c3gAAAAAAL+7tzDABAu7f7cAAAAAAACA+3MA7EQAv/sIAA
 AAAAAAAIAAAAAAAAAIAAAAAA
 
---1618492860--2051301190--113853680--
+--1618492860--2051301190--113853680--
\ No newline at end of file
diff --git a/lib-python/2.7/email/test/test_email.py b/lib-python/2.7/email/test/test_email.py
--- a/lib-python/2.7/email/test/test_email.py
+++ b/lib-python/2.7/email/test/test_email.py
@@ -179,6 +179,17 @@
         self.assertRaises(Errors.HeaderParseError,
                           msg.set_boundary, 'BOUNDARY')
 
+    def test_make_boundary(self):
+        msg = MIMEMultipart('form-data')
+        # Note that when the boundary gets created is an implementation
+        # detail and might change.
+        self.assertEqual(msg.items()[0][1], 'multipart/form-data')
+        # Trigger creation of boundary
+        msg.as_string()
+        self.assertEqual(msg.items()[0][1][:33],
+                        'multipart/form-data; boundary="==')
+        # XXX: there ought to be tests of the uniqueness of the boundary, too.
+
     def test_message_rfc822_only(self):
         # Issue 7970: message/rfc822 not in multipart parsed by
         # HeaderParser caused an exception when flattened.
@@ -542,6 +553,17 @@
         msg.set_charset(u'us-ascii')
         self.assertEqual('us-ascii', msg.get_content_charset())
 
+    # Issue 5871: reject an attempt to embed a header inside a header value
+    # (header injection attack).
+    def test_embeded_header_via_Header_rejected(self):
+        msg = Message()
+        msg['Dummy'] = Header('dummy\nX-Injected-Header: test')
+        self.assertRaises(Errors.HeaderParseError, msg.as_string)
+
+    def test_embeded_header_via_string_rejected(self):
+        msg = Message()
+        msg['Dummy'] = 'dummy\nX-Injected-Header: test'
+        self.assertRaises(Errors.HeaderParseError, msg.as_string)
 
 
 # Test the email.Encoders module
@@ -3113,6 +3135,28 @@
         s = 'Subject: =?EUC-KR?B?CSixpLDtKSC/7Liuvsax4iC6uLmwMcijIKHaILzSwd/H0SC8+LCjwLsgv7W/+Mj3I ?='
         raises(Errors.HeaderParseError, decode_header, s)
 
+    # Issue 1078919
+    def test_ascii_add_header(self):
+        msg = Message()
+        msg.add_header('Content-Disposition', 'attachment',
+                       filename='bud.gif')
+        self.assertEqual('attachment; filename="bud.gif"',
+            msg['Content-Disposition'])
+
+    def test_nonascii_add_header_via_triple(self):
+        msg = Message()
+        msg.add_header('Content-Disposition', 'attachment',
+            filename=('iso-8859-1', '', 'Fu\xdfballer.ppt'))
+        self.assertEqual(
+            'attachment; filename*="iso-8859-1\'\'Fu%DFballer.ppt"',
+            msg['Content-Disposition'])
+
+    def test_encode_unaliased_charset(self):
+        # Issue 1379416: when the charset has no output conversion,
+        # output was accidentally getting coerced to unicode.
+        res = Header('abc','iso-8859-2').encode()
+        self.assertEqual(res, '=?iso-8859-2?q?abc?=')
+        self.assertIsInstance(res, str)
 
 
 # Test RFC 2231 header parameters (en/de)coding
diff --git a/lib-python/2.7/ftplib.py b/lib-python/2.7/ftplib.py
--- a/lib-python/2.7/ftplib.py
+++ b/lib-python/2.7/ftplib.py
@@ -599,7 +599,7 @@
         Usage example:
         >>> from ftplib import FTP_TLS
         >>> ftps = FTP_TLS('ftp.python.org')
-        >>> ftps.login()  # login anonimously previously securing control channel
+        >>> ftps.login()  # login anonymously previously securing control channel
         '230 Guest login ok, access restrictions apply.'
         >>> ftps.prot_p()  # switch to secure data connection
         '200 Protection level set to P'
diff --git a/lib-python/2.7/functools.py b/lib-python/2.7/functools.py
--- a/lib-python/2.7/functools.py
+++ b/lib-python/2.7/functools.py
@@ -53,17 +53,17 @@
 def total_ordering(cls):
     """Class decorator that fills in missing ordering methods"""
     convert = {
-        '__lt__': [('__gt__', lambda self, other: other < self),
-                   ('__le__', lambda self, other: not other < self),
+        '__lt__': [('__gt__', lambda self, other: not (self < other or self == other)),
+                   ('__le__', lambda self, other: self < other or self == other),
                    ('__ge__', lambda self, other: not self < other)],
-        '__le__': [('__ge__', lambda self, other: other <= self),
-                   ('__lt__', lambda self, other: not other <= self),
+        '__le__': [('__ge__', lambda self, other: not self <= other or self == other),
+                   ('__lt__', lambda self, other: self <= other and not self == other),
                    ('__gt__', lambda self, other: not self <= other)],
-        '__gt__': [('__lt__', lambda self, other: other > self),
-                   ('__ge__', lambda self, other: not other > self),
+        '__gt__': [('__lt__', lambda self, other: not (self > other or self == other)),
+                   ('__ge__', lambda self, other: self > other or self == other),
                    ('__le__', lambda self, other: not self > other)],
-        '__ge__': [('__le__', lambda self, other: other >= self),
-                   ('__gt__', lambda self, other: not other >= self),
+        '__ge__': [('__le__', lambda self, other: (not self >= other) or self == other),
+                   ('__gt__', lambda self, other: self >= other and not self == other),
                    ('__lt__', lambda self, other: not self >= other)]
     }
     roots = set(dir(cls)) & set(convert)
@@ -80,6 +80,7 @@
 def cmp_to_key(mycmp):
     """Convert a cmp= function into a key= function"""
     class K(object):
+        __slots__ = ['obj']
         def __init__(self, obj, *args):
             self.obj = obj
         def __lt__(self, other):
diff --git a/lib-python/2.7/getpass.py b/lib-python/2.7/getpass.py
--- a/lib-python/2.7/getpass.py
+++ b/lib-python/2.7/getpass.py
@@ -62,7 +62,7 @@
         try:
             old = termios.tcgetattr(fd)     # a copy to save
             new = old[:]
-            new[3] &= ~(termios.ECHO|termios.ISIG)  # 3 == 'lflags'
+            new[3] &= ~termios.ECHO  # 3 == 'lflags'
             tcsetattr_flags = termios.TCSAFLUSH
             if hasattr(termios, 'TCSASOFT'):
                 tcsetattr_flags |= termios.TCSASOFT
diff --git a/lib-python/2.7/gettext.py b/lib-python/2.7/gettext.py
--- a/lib-python/2.7/gettext.py
+++ b/lib-python/2.7/gettext.py
@@ -316,7 +316,7 @@
             # Note: we unconditionally convert both msgids and msgstrs to
             # Unicode using the character encoding specified in the charset
             # parameter of the Content-Type header.  The gettext documentation
-            # strongly encourages msgids to be us-ascii, but some appliations
+            # strongly encourages msgids to be us-ascii, but some applications
             # require alternative encodings (e.g. Zope's ZCML and ZPT).  For
             # traditional gettext applications, the msgid conversion will
             # cause no problems since us-ascii should always be a subset of
diff --git a/lib-python/2.7/hashlib.py b/lib-python/2.7/hashlib.py
--- a/lib-python/2.7/hashlib.py
+++ b/lib-python/2.7/hashlib.py
@@ -64,26 +64,29 @@
 
 
 def __get_builtin_constructor(name):
-    if name in ('SHA1', 'sha1'):
-        import _sha
-        return _sha.new
-    elif name in ('MD5', 'md5'):
-        import _md5
-        return _md5.new
-    elif name in ('SHA256', 'sha256', 'SHA224', 'sha224'):
-        import _sha256
-        bs = name[3:]
-        if bs == '256':
-            return _sha256.sha256
-        elif bs == '224':
-            return _sha256.sha224
-    elif name in ('SHA512', 'sha512', 'SHA384', 'sha384'):
-        import _sha512
-        bs = name[3:]
-        if bs == '512':
-            return _sha512.sha512
-        elif bs == '384':
-            return _sha512.sha384
+    try:
+        if name in ('SHA1', 'sha1'):
+            import _sha
+            return _sha.new
+        elif name in ('MD5', 'md5'):
+            import _md5
+            return _md5.new
+        elif name in ('SHA256', 'sha256', 'SHA224', 'sha224'):
+            import _sha256
+            bs = name[3:]
+            if bs == '256':
+                return _sha256.sha256
+            elif bs == '224':
+                return _sha256.sha224
+        elif name in ('SHA512', 'sha512', 'SHA384', 'sha384'):
+            import _sha512
+            bs = name[3:]
+            if bs == '512':
+                return _sha512.sha512
+            elif bs == '384':
+                return _sha512.sha384
+    except ImportError:
+        pass  # no extension module, this hash is unsupported.
 
     raise ValueError('unsupported hash type %s' % name)
 
diff --git a/lib-python/2.7/heapq.py b/lib-python/2.7/heapq.py
--- a/lib-python/2.7/heapq.py
+++ b/lib-python/2.7/heapq.py
@@ -133,6 +133,11 @@
 from operator import itemgetter
 import bisect
 
+def cmp_lt(x, y):
+    # Use __lt__ if available; otherwise, try __le__.
+    # In Py3.x, only __lt__ will be called.
+    return (x < y) if hasattr(x, '__lt__') else (not y <= x)
+
 def heappush(heap, item):
     """Push item onto heap, maintaining the heap invariant."""
     heap.append(item)
@@ -167,13 +172,13 @@
 
 def heappushpop(heap, item):
     """Fast version of a heappush followed by a heappop."""
-    if heap and heap[0] < item:
+    if heap and cmp_lt(heap[0], item):
         item, heap[0] = heap[0], item
         _siftup(heap, 0)
     return item
 
 def heapify(x):
-    """Transform list into a heap, in-place, in O(len(heap)) time."""
+    """Transform list into a heap, in-place, in O(len(x)) time."""
     n = len(x)
     # Transform bottom-up.  The largest index there's any point to looking at
     # is the largest with a child index in-range, so must have 2*i + 1 < n,
@@ -215,11 +220,10 @@
         pop = result.pop
         los = result[-1]    # los --> Largest of the nsmallest
         for elem in it:
-            if los <= elem:
-                continue
-            insort(result, elem)
-            pop()
-            los = result[-1]
+            if cmp_lt(elem, los):
+                insort(result, elem)
+                pop()
+                los = result[-1]
         return result
     # An alternative approach manifests the whole iterable in memory but
     # saves comparisons by heapifying all at once.  Also, saves time
@@ -240,7 +244,7 @@
     while pos > startpos:
         parentpos = (pos - 1) >> 1
         parent = heap[parentpos]
-        if newitem < parent:
+        if cmp_lt(newitem, parent):
             heap[pos] = parent
             pos = parentpos
             continue
@@ -295,7 +299,7 @@
     while childpos < endpos:
         # Set childpos to index of smaller child.
         rightpos = childpos + 1
-        if rightpos < endpos and not heap[childpos] < heap[rightpos]:
+        if rightpos < endpos and not cmp_lt(heap[childpos], heap[rightpos]):
             childpos = rightpos
         # Move the smaller child up.
         heap[pos] = heap[childpos]
@@ -364,7 +368,7 @@
             return [min(chain(head, it))]
         return [min(chain(head, it), key=key)]
 
-    # When n>=size, it's faster to use sort()
+    # When n>=size, it's faster to use sorted()
     try:
         size = len(iterable)
     except (TypeError, AttributeError):
@@ -402,7 +406,7 @@
             return [max(chain(head, it))]
         return [max(chain(head, it), key=key)]
 
-    # When n>=size, it's faster to use sort()
+    # When n>=size, it's faster to use sorted()
     try:
         size = len(iterable)
     except (TypeError, AttributeError):
diff --git a/lib-python/2.7/httplib.py b/lib-python/2.7/httplib.py
--- a/lib-python/2.7/httplib.py
+++ b/lib-python/2.7/httplib.py
@@ -212,6 +212,9 @@
 # maximal amount of data to read at one time in _safe_read
 MAXAMOUNT = 1048576
 
+# maximal line length when calling readline().
+_MAXLINE = 65536
+
 class HTTPMessage(mimetools.Message):
 
     def addheader(self, key, value):
@@ -274,7 +277,9 @@
                 except IOError:
                     startofline = tell = None
                     self.seekable = 0
-            line = self.fp.readline()
+            line = self.fp.readline(_MAXLINE + 1)
+            if len(line) > _MAXLINE:
+                raise LineTooLong("header line")
             if not line:
                 self.status = 'EOF in headers'
                 break
@@ -404,7 +409,10 @@
                 break
             # skip the header from the 100 response
             while True:
-                skip = self.fp.readline().strip()
+                skip = self.fp.readline(_MAXLINE + 1)
+                if len(skip) > _MAXLINE:
+                    raise LineTooLong("header line")
+                skip = skip.strip()
                 if not skip:
                     break
                 if self.debuglevel > 0:
@@ -563,7 +571,9 @@
         value = []
         while True:
             if chunk_left is None:
-                line = self.fp.readline()
+                line = self.fp.readline(_MAXLINE + 1)
+                if len(line) > _MAXLINE:
+                    raise LineTooLong("chunk size")
                 i = line.find(';')
                 if i >= 0:
                     line = line[:i] # strip chunk-extensions
@@ -598,7 +608,9 @@
         # read and discard trailer up to the CRLF terminator
         ### note: we shouldn't have any trailers!
         while True:
-            line = self.fp.readline()
+            line = self.fp.readline(_MAXLINE + 1)
+            if len(line) > _MAXLINE:
+                raise LineTooLong("trailer line")
             if not line:
                 # a vanishingly small number of sites EOF without
                 # sending the trailer
@@ -730,7 +742,9 @@
             raise socket.error("Tunnel connection failed: %d %s" % (code,
                                                                     message.strip()))
         while True:
-            line = response.fp.readline()
+            line = response.fp.readline(_MAXLINE + 1)
+            if len(line) > _MAXLINE:
+                raise LineTooLong("header line")
             if line == '\r\n': break
 
 
@@ -790,7 +804,7 @@
         del self._buffer[:]
         # If msg and message_body are sent in a single send() call,
         # it will avoid performance problems caused by the interaction
-        # between delayed ack and the Nagle algorithim.
+        # between delayed ack and the Nagle algorithm.
         if isinstance(message_body, str):
             msg += message_body
             message_body = None
@@ -1233,6 +1247,11 @@
         self.args = line,
         self.line = line
 
+class LineTooLong(HTTPException):
+    def __init__(self, line_type):
+        HTTPException.__init__(self, "got more than %d bytes when reading %s"
+                                     % (_MAXLINE, line_type))
+
 # for backwards compatibility
 error = HTTPException
 
diff --git a/lib-python/2.7/idlelib/Bindings.py b/lib-python/2.7/idlelib/Bindings.py
--- a/lib-python/2.7/idlelib/Bindings.py
+++ b/lib-python/2.7/idlelib/Bindings.py
@@ -98,14 +98,6 @@
     # menu
     del menudefs[-1][1][0:2]
 
-    menudefs.insert(0,
-            ('application', [
-                ('About IDLE', '<>'),
-                None,
-                ('_Preferences....', '<>'),
-            ]))
-
-
 default_keydefs = idleConf.GetCurrentKeySet()
 
 del sys
diff --git a/lib-python/2.7/idlelib/EditorWindow.py b/lib-python/2.7/idlelib/EditorWindow.py
--- a/lib-python/2.7/idlelib/EditorWindow.py
+++ b/lib-python/2.7/idlelib/EditorWindow.py
@@ -48,6 +48,21 @@
             path = module.__path__
         except AttributeError:
             raise ImportError, 'No source for module ' + module.__name__
+    if descr[2] != imp.PY_SOURCE:
+        # If all of the above fails and didn't raise an exception,fallback
+        # to a straight import which can find __init__.py in a package.
+        m = __import__(fullname)
+        try:
+            filename = m.__file__
+        except AttributeError:
+            pass
+        else:
+            file = None
+            base, ext = os.path.splitext(filename)
+            if ext == '.pyc':
+                ext = '.py'
+            filename = base + ext
+            descr = filename, None, imp.PY_SOURCE
     return file, filename, descr
 
 class EditorWindow(object):
@@ -102,8 +117,8 @@
         self.top = top = WindowList.ListedToplevel(root, menu=self.menubar)
         if flist:
             self.tkinter_vars = flist.vars
-            #self.top.instance_dict makes flist.inversedict avalable to
-            #configDialog.py so it can access all EditorWindow instaces
+            #self.top.instance_dict makes flist.inversedict available to
+            #configDialog.py so it can access all EditorWindow instances
             self.top.instance_dict = flist.inversedict
         else:
             self.tkinter_vars = {}  # keys: Tkinter event names
@@ -136,6 +151,14 @@
         if macosxSupport.runningAsOSXApp():
             # Command-W on editorwindows doesn't work without this.
             text.bind('<>', self.close_event)
+            # Some OS X systems have only one mouse button,
+            # so use control-click for pulldown menus there.
+            #  (Note, AquaTk defines <2> as the right button if
+            #   present and the Tk Text widget already binds <2>.)
+            text.bind("",self.right_menu_event)
+        else:
+            # Elsewhere, use right-click for pulldown menus.
+            text.bind("<3>",self.right_menu_event)
         text.bind("<>", self.cut)
         text.bind("<>", self.copy)
         text.bind("<>", self.paste)
@@ -154,7 +177,6 @@
         text.bind("<>", self.find_selection_event)
         text.bind("<>", self.replace_event)
         text.bind("<>", self.goto_line_event)
-        text.bind("<3>", self.right_menu_event)
         text.bind("<>",self.smart_backspace_event)
         text.bind("<>",self.newline_and_indent_event)
         text.bind("<>",self.smart_indent_event)
@@ -300,13 +322,13 @@
         return "break"
 
     def home_callback(self, event):
-        if (event.state & 12) != 0 and event.keysym == "Home":
-            # state&1==shift, state&4==control, state&8==alt
-            return # ; fall back to class binding
-
+        if (event.state & 4) != 0 and event.keysym == "Home":
+            # state&4==Control. If , use the Tk binding.
+            return
         if self.text.index("iomark") and \
            self.text.compare("iomark", "<=", "insert lineend") and \
            self.text.compare("insert linestart", "<=", "iomark"):
+            # In Shell on input line, go to just after prompt
             insertpt = int(self.text.index("iomark").split(".")[1])
         else:
             line = self.text.get("insert linestart", "insert lineend")
@@ -315,30 +337,27 @@
                     break
             else:
                 insertpt=len(line)
-
         lineat = int(self.text.index("insert").split('.')[1])
-
         if insertpt == lineat:
             insertpt = 0
-
         dest = "insert linestart+"+str(insertpt)+"c"
-
         if (event.state&1) == 0:
-            # shift not pressed
+            # shift was not pressed
             self.text.tag_remove("sel", "1.0", "end")
         else:
             if not self.text.index("sel.first"):
-                self.text.mark_set("anchor","insert")
-
+                self.text.mark_set("my_anchor", "insert")  # there was no previous selection
+            else:
+                if self.text.compare(self.text.index("sel.first"), "<", self.text.index("insert")):
+                    self.text.mark_set("my_anchor", "sel.first") # extend back
+                else:
+                    self.text.mark_set("my_anchor", "sel.last") # extend forward
             first = self.text.index(dest)
-            last = self.text.index("anchor")
-
+            last = self.text.index("my_anchor")
             if self.text.compare(first,">",last):
                 first,last = last,first
-
             self.text.tag_remove("sel", "1.0", "end")
             self.text.tag_add("sel", first, last)
-
         self.text.mark_set("insert", dest)
         self.text.see("insert")
         return "break"
@@ -385,7 +404,7 @@
             menudict[name] = menu = Menu(mbar, name=name)
             mbar.add_cascade(label=label, menu=menu, underline=underline)
 
-        if macosxSupport.runningAsOSXApp():
+        if macosxSupport.isCarbonAquaTk(self.root):
             # Insert the application menu
             menudict['application'] = menu = Menu(mbar, name='apple')
             mbar.add_cascade(label='IDLE', menu=menu)
@@ -445,7 +464,11 @@
 
     def python_docs(self, event=None):
         if sys.platform[:3] == 'win':
-            os.startfile(self.help_url)
+            try:
+                os.startfile(self.help_url)
+            except WindowsError as why:
+                tkMessageBox.showerror(title='Document Start Failure',
+                    message=str(why), parent=self.text)
         else:
             webbrowser.open(self.help_url)
         return "break"
@@ -740,9 +763,13 @@
         "Create a callback with the helpfile value frozen at definition time"
         def display_extra_help(helpfile=helpfile):
             if not helpfile.startswith(('www', 'http')):
-                url = os.path.normpath(helpfile)
+                helpfile = os.path.normpath(helpfile)
             if sys.platform[:3] == 'win':
-                os.startfile(helpfile)
+                try:
+                    os.startfile(helpfile)
+                except WindowsError as why:
+                    tkMessageBox.showerror(title='Document Start Failure',
+                        message=str(why), parent=self.text)
             else:
                 webbrowser.open(helpfile)
         return display_extra_help
@@ -1526,7 +1553,12 @@
 
 def get_accelerator(keydefs, eventname):
     keylist = keydefs.get(eventname)
-    if not keylist:
+    # issue10940: temporary workaround to prevent hang with OS X Cocoa Tk 8.5
+    # if not keylist:
+    if (not keylist) or (macosxSupport.runningAsOSXApp() and eventname in {
+                            "<>",
+                            "<>",
+                            "<>"}):
         return ""
     s = keylist[0]
     s = re.sub(r"-[a-z]\b", lambda m: m.group().upper(), s)
diff --git a/lib-python/2.7/idlelib/FileList.py b/lib-python/2.7/idlelib/FileList.py
--- a/lib-python/2.7/idlelib/FileList.py
+++ b/lib-python/2.7/idlelib/FileList.py
@@ -43,7 +43,7 @@
     def new(self, filename=None):
         return self.EditorWindow(self, filename)
 
-    def close_all_callback(self, event):
+    def close_all_callback(self, *args, **kwds):
         for edit in self.inversedict.keys():
             reply = edit.close()
             if reply == "cancel":
diff --git a/lib-python/2.7/idlelib/FormatParagraph.py b/lib-python/2.7/idlelib/FormatParagraph.py
--- a/lib-python/2.7/idlelib/FormatParagraph.py
+++ b/lib-python/2.7/idlelib/FormatParagraph.py
@@ -54,7 +54,7 @@
             # If the block ends in a \n, we dont want the comment
             # prefix inserted after it. (Im not sure it makes sense to
             # reformat a comment block that isnt made of complete
-            # lines, but whatever!)  Can't think of a clean soltution,
+            # lines, but whatever!)  Can't think of a clean solution,
             # so we hack away
             block_suffix = ""
             if not newdata[-1]:
diff --git a/lib-python/2.7/idlelib/HISTORY.txt b/lib-python/2.7/idlelib/HISTORY.txt
--- a/lib-python/2.7/idlelib/HISTORY.txt
+++ b/lib-python/2.7/idlelib/HISTORY.txt
@@ -13,7 +13,7 @@
 - New tarball released as a result of the 'revitalisation' of the IDLEfork
   project. 
 
-- This release requires python 2.1 or better. Compatability with earlier
+- This release requires python 2.1 or better. Compatibility with earlier
   versions of python (especially ancient ones like 1.5x) is no longer a
   priority in IDLEfork development.
 
diff --git a/lib-python/2.7/idlelib/IOBinding.py b/lib-python/2.7/idlelib/IOBinding.py
--- a/lib-python/2.7/idlelib/IOBinding.py
+++ b/lib-python/2.7/idlelib/IOBinding.py
@@ -320,17 +320,20 @@
             return "yes"
         message = "Do you want to save %s before closing?" % (
             self.filename or "this untitled document")
-        m = tkMessageBox.Message(
-            title="Save On Close",
-            message=message,
-            icon=tkMessageBox.QUESTION,
-            type=tkMessageBox.YESNOCANCEL,
-            master=self.text)
-        reply = m.show()
-        if reply == "yes":
+        confirm = tkMessageBox.askyesnocancel(
+                  title="Save On Close",
+                  message=message,
+                  default=tkMessageBox.YES,
+                  master=self.text)
+        if confirm:
+            reply = "yes"
             self.save(None)
             if not self.get_saved():
                 reply = "cancel"
+        elif confirm is None:
+            reply = "cancel"
+        else:
+            reply = "no"
         self.text.focus_set()
         return reply
 
@@ -339,7 +342,7 @@
             self.save_as(event)
         else:
             if self.writefile(self.filename):
-                self.set_saved(1)
+                self.set_saved(True)
                 try:
                     self.editwin.store_file_breaks()
                 except AttributeError:  # may be a PyShell
@@ -465,15 +468,12 @@
             self.text.insert("end-1c", "\n")
 
     def print_window(self, event):
-        m = tkMessageBox.Message(
-            title="Print",
-            message="Print to Default Printer",
-            icon=tkMessageBox.QUESTION,
-            type=tkMessageBox.OKCANCEL,
-            default=tkMessageBox.OK,
-            master=self.text)
-        reply = m.show()
-        if reply != tkMessageBox.OK:
+        confirm = tkMessageBox.askokcancel(
+                  title="Print",
+                  message="Print to Default Printer",
+                  default=tkMessageBox.OK,
+                  master=self.text)
+        if not confirm:
             self.text.focus_set()
             return "break"
         tempfilename = None
@@ -488,8 +488,8 @@
             if not self.writefile(tempfilename):
                 os.unlink(tempfilename)
                 return "break"
-        platform=os.name
-        printPlatform=1
+        platform = os.name
+        printPlatform = True
         if platform == 'posix': #posix platform
             command = idleConf.GetOption('main','General',
                                          'print-command-posix')
@@ -497,7 +497,7 @@
         elif platform == 'nt': #win32 platform
             command = idleConf.GetOption('main','General','print-command-win')
         else: #no printing for this platform
-            printPlatform=0
+            printPlatform = False
         if printPlatform:  #we can try to print for this platform
             command = command % filename
             pipe = os.popen(command, "r")
@@ -511,7 +511,7 @@
                 output = "Printing command: %s\n" % repr(command) + output
                 tkMessageBox.showerror("Print status", output, master=self.text)
         else:  #no printing for this platform
-            message="Printing is not enabled for this platform: %s" % platform
+            message = "Printing is not enabled for this platform: %s" % platform
             tkMessageBox.showinfo("Print status", message, master=self.text)
         if tempfilename:
             os.unlink(tempfilename)
diff --git a/lib-python/2.7/idlelib/NEWS.txt b/lib-python/2.7/idlelib/NEWS.txt
--- a/lib-python/2.7/idlelib/NEWS.txt
+++ b/lib-python/2.7/idlelib/NEWS.txt
@@ -1,3 +1,18 @@
+What's New in IDLE 2.7.2?
+=======================
+
+*Release date: 29-May-2011*
+
+- Issue #6378: Further adjust idle.bat to start associated Python
+
+- Issue #11896: Save on Close failed despite selecting "Yes" in dialog.
+
+-  toggle failing on Tk 8.5, causing IDLE exits and strange selection
+  behavior. Issue 4676.  Improve selection extension behaviour.
+
+-  toggle non-functional when NumLock set on Windows.  Issue 3851.
+
+
 What's New in IDLE 2.7?
 =======================
 
@@ -21,7 +36,7 @@
 
 - Tk 8.5 Text widget requires 'wordprocessor' tabstyle attr to handle
   mixed space/tab properly. Issue 5129, patch by Guilherme Polo.
-  
+
 - Issue #3549: On MacOS the preferences menu was not present
 
 
diff --git a/lib-python/2.7/idlelib/PyShell.py b/lib-python/2.7/idlelib/PyShell.py
--- a/lib-python/2.7/idlelib/PyShell.py
+++ b/lib-python/2.7/idlelib/PyShell.py
@@ -1432,6 +1432,13 @@
             shell.interp.prepend_syspath(script)
             shell.interp.execfile(script)
 
+    # Check for problematic OS X Tk versions and print a warning message
+    # in the IDLE shell window; this is less intrusive than always opening
+    # a separate window.
+    tkversionwarning = macosxSupport.tkVersionWarning(root)
+    if tkversionwarning:
+        shell.interp.runcommand(''.join(("print('", tkversionwarning, "')")))
+
     root.mainloop()
     root.destroy()
 
diff --git a/lib-python/2.7/idlelib/ScriptBinding.py b/lib-python/2.7/idlelib/ScriptBinding.py
--- a/lib-python/2.7/idlelib/ScriptBinding.py
+++ b/lib-python/2.7/idlelib/ScriptBinding.py
@@ -26,6 +26,7 @@
 from idlelib import PyShell
 
 from idlelib.configHandler import idleConf
+from idlelib import macosxSupport
 
 IDENTCHARS = string.ascii_letters + string.digits + "_"
 
@@ -53,6 +54,9 @@
         self.flist = self.editwin.flist
         self.root = self.editwin.root
 
+        if macosxSupport.runningAsOSXApp():
+            self.editwin.text_frame.bind('<>', self._run_module_event)
+
     def check_module_event(self, event):
         filename = self.getfilename()
         if not filename:
@@ -166,6 +170,19 @@
         interp.runcode(code)
         return 'break'
 
+    if macosxSupport.runningAsOSXApp():
+        # Tk-Cocoa in MacOSX is broken until at least
+        # Tk 8.5.9, and without this rather
+        # crude workaround IDLE would hang when a user
+        # tries to run a module using the keyboard shortcut
+        # (the menu item works fine).
+        _run_module_event = run_module_event
+
+        def run_module_event(self, event):
+            self.editwin.text_frame.after(200,
+                lambda: self.editwin.text_frame.event_generate('<>'))
+            return 'break'
+
     def getfilename(self):
         """Get source filename.  If not saved, offer to save (or create) file
 
@@ -184,9 +201,9 @@
             if autosave and filename:
                 self.editwin.io.save(None)
             else:
-                reply = self.ask_save_dialog()
+                confirm = self.ask_save_dialog()
                 self.editwin.text.focus_set()
-                if reply == "ok":
+                if confirm:
                     self.editwin.io.save(None)
                     filename = self.editwin.io.filename
                 else:
@@ -195,13 +212,11 @@
 
     def ask_save_dialog(self):
         msg = "Source Must Be Saved\n" + 5*' ' + "OK to Save?"
-        mb = tkMessageBox.Message(title="Save Before Run or Check",
-                                  message=msg,
-                                  icon=tkMessageBox.QUESTION,
-                                  type=tkMessageBox.OKCANCEL,
-                                  default=tkMessageBox.OK,
-                                  master=self.editwin.text)
-        return mb.show()
+        confirm = tkMessageBox.askokcancel(title="Save Before Run or Check",
+                                           message=msg,
+                                           default=tkMessageBox.OK,
+                                           master=self.editwin.text)
+        return confirm
 
     def errorbox(self, title, message):
         # XXX This should really be a function of EditorWindow...
diff --git a/lib-python/2.7/idlelib/config-keys.def b/lib-python/2.7/idlelib/config-keys.def
--- a/lib-python/2.7/idlelib/config-keys.def
+++ b/lib-python/2.7/idlelib/config-keys.def
@@ -176,7 +176,7 @@
 redo = 
 close-window = 
 restart-shell = 
-save-window-as-file = 
+save-window-as-file = 
 close-all-windows = 
 view-restart = 
 tabify-region = 
@@ -208,7 +208,7 @@
 open-module = 
 find-selection = 
 python-context-help = 
-save-copy-of-window-as-file = 
+save-copy-of-window-as-file = 
 open-window-from-file = 
 python-docs = 
 
diff --git a/lib-python/2.7/idlelib/extend.txt b/lib-python/2.7/idlelib/extend.txt
--- a/lib-python/2.7/idlelib/extend.txt
+++ b/lib-python/2.7/idlelib/extend.txt
@@ -18,7 +18,7 @@
 
 An IDLE extension class is instantiated with a single argument,
 `editwin', an EditorWindow instance. The extension cannot assume much
-about this argument, but it is guarateed to have the following instance
+about this argument, but it is guaranteed to have the following instance
 variables:
 
     text	a Text instance (a widget)
diff --git a/lib-python/2.7/idlelib/idle.bat b/lib-python/2.7/idlelib/idle.bat
--- a/lib-python/2.7/idlelib/idle.bat
+++ b/lib-python/2.7/idlelib/idle.bat
@@ -1,4 +1,4 @@
 @echo off
 rem Start IDLE using the appropriate Python interpreter
 set CURRDIR=%~dp0
-start "%CURRDIR%..\..\pythonw.exe" "%CURRDIR%idle.pyw" %1 %2 %3 %4 %5 %6 %7 %8 %9
+start "IDLE" "%CURRDIR%..\..\pythonw.exe" "%CURRDIR%idle.pyw" %1 %2 %3 %4 %5 %6 %7 %8 %9
diff --git a/lib-python/2.7/idlelib/idlever.py b/lib-python/2.7/idlelib/idlever.py
--- a/lib-python/2.7/idlelib/idlever.py
+++ b/lib-python/2.7/idlelib/idlever.py
@@ -1,1 +1,1 @@
-IDLE_VERSION = "2.7.1"
+IDLE_VERSION = "2.7.2"
diff --git a/lib-python/2.7/idlelib/macosxSupport.py b/lib-python/2.7/idlelib/macosxSupport.py
--- a/lib-python/2.7/idlelib/macosxSupport.py
+++ b/lib-python/2.7/idlelib/macosxSupport.py
@@ -4,6 +4,7 @@
 """
 import sys
 import Tkinter
+from os import path
 
 
 _appbundle = None
@@ -19,10 +20,41 @@
         _appbundle = (sys.platform == 'darwin' and '.app' in sys.executable)
     return _appbundle
 
+_carbonaquatk = None
+
+def isCarbonAquaTk(root):
+    """
+    Returns True if IDLE is using a Carbon Aqua Tk (instead of the
+    newer Cocoa Aqua Tk).
+    """
+    global _carbonaquatk
+    if _carbonaquatk is None:
+        _carbonaquatk = (runningAsOSXApp() and
+                         'aqua' in root.tk.call('tk', 'windowingsystem') and
+                         'AppKit' not in root.tk.call('winfo', 'server', '.'))
+    return _carbonaquatk
+
+def tkVersionWarning(root):
+    """
+    Returns a string warning message if the Tk version in use appears to
+    be one known to cause problems with IDLE.  The Apple Cocoa-based Tk 8.5
+    that was shipped with Mac OS X 10.6.
+    """
+
+    if (runningAsOSXApp() and
+            ('AppKit' in root.tk.call('winfo', 'server', '.')) and
+            (root.tk.call('info', 'patchlevel') == '8.5.7') ):
+        return (r"WARNING: The version of Tcl/Tk (8.5.7) in use may"
+                r" be unstable.\n"
+                r"Visit http://www.python.org/download/mac/tcltk/"
+                r" for current information.")
+    else:
+        return False
+
 def addOpenEventSupport(root, flist):
     """
-    This ensures that the application will respont to open AppleEvents, which
-    makes is feaseable to use IDLE as the default application for python files.
+    This ensures that the application will respond to open AppleEvents, which
+    makes is feasible to use IDLE as the default application for python files.
     """
     def doOpenFile(*args):
         for fn in args:
@@ -79,9 +111,6 @@
         WindowList.add_windows_to_menu(menu)
     WindowList.register_callback(postwindowsmenu)
 
-    menudict['application'] = menu = Menu(menubar, name='apple')
-    menubar.add_cascade(label='IDLE', menu=menu)
-
     def about_dialog(event=None):
         from idlelib import aboutDialog
         aboutDialog.AboutDialog(root, 'About IDLE')
@@ -91,41 +120,45 @@
         root.instance_dict = flist.inversedict
         configDialog.ConfigDialog(root, 'Settings')
 
+    def help_dialog(event=None):
+        from idlelib import textView
+        fn = path.join(path.abspath(path.dirname(__file__)), 'help.txt')
+        textView.view_file(root, 'Help', fn)
 
     root.bind('<>', about_dialog)
     root.bind('<>', config_dialog)
+    root.createcommand('::tk::mac::ShowPreferences', config_dialog)
     if flist:
         root.bind('<>', flist.close_all_callback)
 
+        # The binding above doesn't reliably work on all versions of Tk
+        # on MacOSX. Adding command definition below does seem to do the
+        # right thing for now.
+        root.createcommand('exit', flist.close_all_callback)
 
-    ###check if Tk version >= 8.4.14; if so, use hard-coded showprefs binding
-    tkversion = root.tk.eval('info patchlevel')
-    # Note: we cannot check if the string tkversion >= '8.4.14', because
-    # the string '8.4.7' is greater than the string '8.4.14'.
-    if tuple(map(int, tkversion.split('.'))) >= (8, 4, 14):
-        Bindings.menudefs[0] =  ('application', [
+    if isCarbonAquaTk(root):
+        # for Carbon AquaTk, replace the default Tk apple menu
+        menudict['application'] = menu = Menu(menubar, name='apple')
+        menubar.add_cascade(label='IDLE', menu=menu)
+        Bindings.menudefs.insert(0,
+            ('application', [
                 ('About IDLE', '<>'),
-                None,
-            ])
-        root.createcommand('::tk::mac::ShowPreferences', config_dialog)
+                    None,
+                ]))
+        tkversion = root.tk.eval('info patchlevel')
+        if tuple(map(int, tkversion.split('.'))) < (8, 4, 14):
+            # for earlier AquaTk versions, supply a Preferences menu item
+            Bindings.menudefs[0][1].append(
+                    ('_Preferences....', '<>'),
+                )
     else:
-        for mname, entrylist in Bindings.menudefs:
-            menu = menudict.get(mname)
-            if not menu:
-                continue
-            else:
-                for entry in entrylist:
-                    if not entry:
-                        menu.add_separator()
-                    else:
-                        label, eventname = entry
-                        underline, label = prepstr(label)
-                        accelerator = get_accelerator(Bindings.default_keydefs,
-                        eventname)
-                        def command(text=root, eventname=eventname):
-                            text.event_generate(eventname)
-                        menu.add_command(label=label, underline=underline,
-                        command=command, accelerator=accelerator)
+        # assume Cocoa AquaTk
+        # replace default About dialog with About IDLE one
+        root.createcommand('tkAboutDialog', about_dialog)
+        # replace default "Help" item in Help menu
+        root.createcommand('::tk::mac::ShowHelp', help_dialog)
+        # remove redundant "IDLE Help" from menu
+        del Bindings.menudefs[-1][1][0]
 
 def setupApp(root, flist):
     """
diff --git a/lib-python/2.7/imaplib.py b/lib-python/2.7/imaplib.py
--- a/lib-python/2.7/imaplib.py
+++ b/lib-python/2.7/imaplib.py
@@ -1158,28 +1158,17 @@
             self.port = port
             self.sock = socket.create_connection((host, port))
             self.sslobj = ssl.wrap_socket(self.sock, self.keyfile, self.certfile)
+            self.file = self.sslobj.makefile('rb')
 
 
         def read(self, size):
             """Read 'size' bytes from remote."""
-            # sslobj.read() sometimes returns < size bytes
-            chunks = []
-            read = 0
-            while read < size:
-                data = self.sslobj.read(min(size-read, 16384))
-                read += len(data)
-                chunks.append(data)
-
-            return ''.join(chunks)
+            return self.file.read(size)
 
 
         def readline(self):
             """Read line from remote."""
-            line = []
-            while 1:
-                char = self.sslobj.read(1)
-                line.append(char)
-                if char in ("\n", ""): return ''.join(line)
+            return self.file.readline()
 
 
         def send(self, data):
@@ -1195,6 +1184,7 @@
 
         def shutdown(self):
             """Close I/O established in "open"."""
+            self.file.close()
             self.sock.close()
 
 
@@ -1321,9 +1311,10 @@
         'Jul': 7, 'Aug': 8, 'Sep': 9, 'Oct': 10, 'Nov': 11, 'Dec': 12}
 
 def Internaldate2tuple(resp):
-    """Convert IMAP4 INTERNALDATE to UT.
+    """Parse an IMAP4 INTERNALDATE string.
 
-    Returns Python time module tuple.
+    Return corresponding local time.  The return value is a
+    time.struct_time instance or None if the string has wrong format.
     """
 
     mo = InternalDate.match(resp)
@@ -1390,9 +1381,14 @@
 
 def Time2Internaldate(date_time):
 
-    """Convert 'date_time' to IMAP4 INTERNALDATE representation.
+    """Convert date_time to IMAP4 INTERNALDATE representation.
 
-    Return string in form: '"DD-Mmm-YYYY HH:MM:SS +HHMM"'
+    Return string in form: '"DD-Mmm-YYYY HH:MM:SS +HHMM"'.  The
+    date_time argument can be a number (int or float) representing
+    seconds since epoch (as returned by time.time()), a 9-tuple
+    representing local time (as returned by time.localtime()), or a
+    double-quoted string.  In the last case, it is assumed to already
+    be in the correct format.
     """
 
     if isinstance(date_time, (int, float)):
diff --git a/lib-python/2.7/inspect.py b/lib-python/2.7/inspect.py
--- a/lib-python/2.7/inspect.py
+++ b/lib-python/2.7/inspect.py
@@ -943,8 +943,14 @@
             f_name, 'at most' if defaults else 'exactly', num_args,
             'arguments' if num_args > 1 else 'argument', num_total))
     elif num_args == 0 and num_total:
-        raise TypeError('%s() takes no arguments (%d given)' %
-                        (f_name, num_total))
+        if varkw:
+            if num_pos:
+                # XXX: We should use num_pos, but Python also uses num_total:
+                raise TypeError('%s() takes exactly 0 arguments '
+                                '(%d given)' % (f_name, num_total))
+        else:
+            raise TypeError('%s() takes no arguments (%d given)' %
+                            (f_name, num_total))
     for arg in args:
         if isinstance(arg, str) and arg in named:
             if is_assigned(arg):
diff --git a/lib-python/2.7/json/decoder.py b/lib-python/2.7/json/decoder.py
--- a/lib-python/2.7/json/decoder.py
+++ b/lib-python/2.7/json/decoder.py
@@ -4,7 +4,7 @@
 import sys
 import struct
 
-from json.scanner import make_scanner
+from json import scanner
 try:
     from _json import scanstring as c_scanstring
 except ImportError:
@@ -161,6 +161,12 @@
             nextchar = s[end:end + 1]
         # Trivial empty object
         if nextchar == '}':
+            if object_pairs_hook is not None:
+                result = object_pairs_hook(pairs)
+                return result, end
+            pairs = {}
+            if object_hook is not None:
+                pairs = object_hook(pairs)
             return pairs, end + 1
         elif nextchar != '"':
             raise ValueError(errmsg("Expecting property name", s, end))
@@ -350,7 +356,7 @@
         self.parse_object = JSONObject
         self.parse_array = JSONArray
         self.parse_string = scanstring
-        self.scan_once = make_scanner(self)
+        self.scan_once = scanner.make_scanner(self)
 
     def decode(self, s, _w=WHITESPACE.match):
         """Return the Python representation of ``s`` (a ``str`` or ``unicode``
diff --git a/lib-python/2.7/json/encoder.py b/lib-python/2.7/json/encoder.py
--- a/lib-python/2.7/json/encoder.py
+++ b/lib-python/2.7/json/encoder.py
@@ -251,7 +251,7 @@
 
 
         if (_one_shot and c_make_encoder is not None
-                and not self.indent and not self.sort_keys):
+                and self.indent is None and not self.sort_keys):
             _iterencode = c_make_encoder(
                 markers, self.default, _encoder, self.indent,
                 self.key_separator, self.item_separator, self.sort_keys,
diff --git a/lib-python/2.7/json/tests/__init__.py b/lib-python/2.7/json/tests/__init__.py
--- a/lib-python/2.7/json/tests/__init__.py
+++ b/lib-python/2.7/json/tests/__init__.py
@@ -1,7 +1,46 @@
 import os
 import sys
+import json
+import doctest
 import unittest
-import doctest
+
+from test import test_support
+
+# import json with and without accelerations
+cjson = test_support.import_fresh_module('json', fresh=['_json'])
+pyjson = test_support.import_fresh_module('json', blocked=['_json'])
+
+# create two base classes that will be used by the other tests
+class PyTest(unittest.TestCase):
+    json = pyjson
+    loads = staticmethod(pyjson.loads)
+    dumps = staticmethod(pyjson.dumps)
+
+ at unittest.skipUnless(cjson, 'requires _json')
+class CTest(unittest.TestCase):
+    if cjson is not None:
+        json = cjson
+        loads = staticmethod(cjson.loads)
+        dumps = staticmethod(cjson.dumps)
+
+# test PyTest and CTest checking if the functions come from the right module
+class TestPyTest(PyTest):
+    def test_pyjson(self):
+        self.assertEqual(self.json.scanner.make_scanner.__module__,
+                         'json.scanner')
+        self.assertEqual(self.json.decoder.scanstring.__module__,
+                         'json.decoder')
+        self.assertEqual(self.json.encoder.encode_basestring_ascii.__module__,
+                         'json.encoder')
+
+class TestCTest(CTest):
+    def test_cjson(self):
+        self.assertEqual(self.json.scanner.make_scanner.__module__, '_json')
+        self.assertEqual(self.json.decoder.scanstring.__module__, '_json')
+        self.assertEqual(self.json.encoder.c_make_encoder.__module__, '_json')
+        self.assertEqual(self.json.encoder.encode_basestring_ascii.__module__,
+                         '_json')
+
 
 here = os.path.dirname(__file__)
 
@@ -17,12 +56,11 @@
     return suite
 
 def additional_tests():
-    import json
-    import json.encoder
-    import json.decoder
     suite = unittest.TestSuite()
     for mod in (json, json.encoder, json.decoder):
         suite.addTest(doctest.DocTestSuite(mod))
+    suite.addTest(TestPyTest('test_pyjson'))
+    suite.addTest(TestCTest('test_cjson'))
     return suite
 
 def main():
diff --git a/lib-python/2.7/json/tests/test_check_circular.py b/lib-python/2.7/json/tests/test_check_circular.py
--- a/lib-python/2.7/json/tests/test_check_circular.py
+++ b/lib-python/2.7/json/tests/test_check_circular.py
@@ -1,30 +1,34 @@
-from unittest import TestCase
-import json
+from json.tests import PyTest, CTest
+
 
 def default_iterable(obj):
     return list(obj)
 
-class TestCheckCircular(TestCase):
+class TestCheckCircular(object):
     def test_circular_dict(self):
         dct = {}
         dct['a'] = dct
-        self.assertRaises(ValueError, json.dumps, dct)
+        self.assertRaises(ValueError, self.dumps, dct)
 
     def test_circular_list(self):
         lst = []
         lst.append(lst)
-        self.assertRaises(ValueError, json.dumps, lst)
+        self.assertRaises(ValueError, self.dumps, lst)
 
     def test_circular_composite(self):
         dct2 = {}
         dct2['a'] = []
         dct2['a'].append(dct2)
-        self.assertRaises(ValueError, json.dumps, dct2)
+        self.assertRaises(ValueError, self.dumps, dct2)
 
     def test_circular_default(self):
-        json.dumps([set()], default=default_iterable)
-        self.assertRaises(TypeError, json.dumps, [set()])
+        self.dumps([set()], default=default_iterable)
+        self.assertRaises(TypeError, self.dumps, [set()])
 
     def test_circular_off_default(self):
-        json.dumps([set()], default=default_iterable, check_circular=False)
-        self.assertRaises(TypeError, json.dumps, [set()], check_circular=False)
+        self.dumps([set()], default=default_iterable, check_circular=False)
+        self.assertRaises(TypeError, self.dumps, [set()], check_circular=False)
+
+
+class TestPyCheckCircular(TestCheckCircular, PyTest): pass
+class TestCCheckCircular(TestCheckCircular, CTest): pass
diff --git a/lib-python/2.7/json/tests/test_decode.py b/lib-python/2.7/json/tests/test_decode.py
--- a/lib-python/2.7/json/tests/test_decode.py
+++ b/lib-python/2.7/json/tests/test_decode.py
@@ -1,18 +1,17 @@
 import decimal
-from unittest import TestCase
 from StringIO import StringIO
+from collections import OrderedDict
+from json.tests import PyTest, CTest
 
-import json
-from collections import OrderedDict
 
-class TestDecode(TestCase):
+class TestDecode(object):
     def test_decimal(self):
-        rval = json.loads('1.1', parse_float=decimal.Decimal)
+        rval = self.loads('1.1', parse_float=decimal.Decimal)
         self.assertTrue(isinstance(rval, decimal.Decimal))
         self.assertEqual(rval, decimal.Decimal('1.1'))
 
     def test_float(self):
-        rval = json.loads('1', parse_int=float)
+        rval = self.loads('1', parse_int=float)
         self.assertTrue(isinstance(rval, float))
         self.assertEqual(rval, 1.0)
 
@@ -20,22 +19,32 @@
         # Several optimizations were made that skip over calls to
         # the whitespace regex, so this test is designed to try and
         # exercise the uncommon cases. The array cases are already covered.
-        rval = json.loads('{   "key"    :    "value"    ,  "k":"v"    }')
+        rval = self.loads('{   "key"    :    "value"    ,  "k":"v"    }')
         self.assertEqual(rval, {"key":"value", "k":"v"})
 
+    def test_empty_objects(self):
+        self.assertEqual(self.loads('{}'), {})
+        self.assertEqual(self.loads('[]'), [])
+        self.assertEqual(self.loads('""'), u"")
+        self.assertIsInstance(self.loads('""'), unicode)
+
     def test_object_pairs_hook(self):
         s = '{"xkd":1, "kcw":2, "art":3, "hxm":4, "qrt":5, "pad":6, "hoy":7}'
         p = [("xkd", 1), ("kcw", 2), ("art", 3), ("hxm", 4),
              ("qrt", 5), ("pad", 6), ("hoy", 7)]
-        self.assertEqual(json.loads(s), eval(s))
-        self.assertEqual(json.loads(s, object_pairs_hook=lambda x: x), p)
-        self.assertEqual(json.load(StringIO(s),
-                                   object_pairs_hook=lambda x: x), p)
-        od = json.loads(s, object_pairs_hook=OrderedDict)
+        self.assertEqual(self.loads(s), eval(s))
+        self.assertEqual(self.loads(s, object_pairs_hook=lambda x: x), p)
+        self.assertEqual(self.json.load(StringIO(s),
+                                        object_pairs_hook=lambda x: x), p)
+        od = self.loads(s, object_pairs_hook=OrderedDict)
         self.assertEqual(od, OrderedDict(p))
         self.assertEqual(type(od), OrderedDict)
         # the object_pairs_hook takes priority over the object_hook
-        self.assertEqual(json.loads(s,
+        self.assertEqual(self.loads(s,
                                     object_pairs_hook=OrderedDict,
                                     object_hook=lambda x: None),
                          OrderedDict(p))
+
+
+class TestPyDecode(TestDecode, PyTest): pass
+class TestCDecode(TestDecode, CTest): pass
diff --git a/lib-python/2.7/json/tests/test_default.py b/lib-python/2.7/json/tests/test_default.py
--- a/lib-python/2.7/json/tests/test_default.py
+++ b/lib-python/2.7/json/tests/test_default.py
@@ -1,9 +1,12 @@
-from unittest import TestCase
+from json.tests import PyTest, CTest
 
-import json
 
-class TestDefault(TestCase):
+class TestDefault(object):
     def test_default(self):
         self.assertEqual(
-            json.dumps(type, default=repr),
-            json.dumps(repr(type)))
+            self.dumps(type, default=repr),
+            self.dumps(repr(type)))
+
+
+class TestPyDefault(TestDefault, PyTest): pass
+class TestCDefault(TestDefault, CTest): pass
diff --git a/lib-python/2.7/json/tests/test_dump.py b/lib-python/2.7/json/tests/test_dump.py
--- a/lib-python/2.7/json/tests/test_dump.py
+++ b/lib-python/2.7/json/tests/test_dump.py
@@ -1,21 +1,23 @@
-from unittest import TestCase
 from cStringIO import StringIO
+from json.tests import PyTest, CTest
 
-import json
 
-class TestDump(TestCase):
+class TestDump(object):
     def test_dump(self):
         sio = StringIO()
-        json.dump({}, sio)
+        self.json.dump({}, sio)
         self.assertEqual(sio.getvalue(), '{}')
 
     def test_dumps(self):
-        self.assertEqual(json.dumps({}), '{}')
+        self.assertEqual(self.dumps({}), '{}')
 
     def test_encode_truefalse(self):
-        self.assertEqual(json.dumps(
+        self.assertEqual(self.dumps(
                  {True: False, False: True}, sort_keys=True),
                  '{"false": true, "true": false}')
-        self.assertEqual(json.dumps(
+        self.assertEqual(self.dumps(
                 {2: 3.0, 4.0: 5L, False: 1, 6L: True}, sort_keys=True),
                 '{"false": 1, "2": 3.0, "4.0": 5, "6": true}')
+
+class TestPyDump(TestDump, PyTest): pass
+class TestCDump(TestDump, CTest): pass
diff --git a/lib-python/2.7/json/tests/test_encode_basestring_ascii.py b/lib-python/2.7/json/tests/test_encode_basestring_ascii.py
--- a/lib-python/2.7/json/tests/test_encode_basestring_ascii.py
+++ b/lib-python/2.7/json/tests/test_encode_basestring_ascii.py
@@ -1,8 +1,6 @@
-from unittest import TestCase
+from collections import OrderedDict
+from json.tests import PyTest, CTest
 
-import json.encoder
-from json import dumps
-from collections import OrderedDict
 
 CASES = [
     (u'/\\"\ucafe\ubabe\uab98\ufcde\ubcda\uef4a\x08\x0c\n\r\t`1~!@#$%^&*()_+-=[]{}|;:\',./<>?', '"/\\\\\\"\\ucafe\\ubabe\\uab98\\ufcde\\ubcda\\uef4a\\b\\f\\n\\r\\t`1~!@#$%^&*()_+-=[]{}|;:\',./<>?"'),
@@ -23,19 +21,11 @@
     (u'\u0123\u4567\u89ab\ucdef\uabcd\uef4a', '"\\u0123\\u4567\\u89ab\\ucdef\\uabcd\\uef4a"'),
 ]
 
-class TestEncodeBaseStringAscii(TestCase):
-    def test_py_encode_basestring_ascii(self):
-        self._test_encode_basestring_ascii(json.encoder.py_encode_basestring_ascii)
-
-    def test_c_encode_basestring_ascii(self):
-        if not json.encoder.c_encode_basestring_ascii:
-            return
-        self._test_encode_basestring_ascii(json.encoder.c_encode_basestring_ascii)
-
-    def _test_encode_basestring_ascii(self, encode_basestring_ascii):
-        fname = encode_basestring_ascii.__name__
+class TestEncodeBasestringAscii(object):
+    def test_encode_basestring_ascii(self):
+        fname = self.json.encoder.encode_basestring_ascii.__name__
         for input_string, expect in CASES:
-            result = encode_basestring_ascii(input_string)
+            result = self.json.encoder.encode_basestring_ascii(input_string)
             self.assertEqual(result, expect,
                 '{0!r} != {1!r} for {2}({3!r})'.format(
                     result, expect, fname, input_string))
@@ -43,5 +33,9 @@
     def test_ordered_dict(self):
         # See issue 6105
         items = [('one', 1), ('two', 2), ('three', 3), ('four', 4), ('five', 5)]
-        s = json.dumps(OrderedDict(items))
+        s = self.dumps(OrderedDict(items))
         self.assertEqual(s, '{"one": 1, "two": 2, "three": 3, "four": 4, "five": 5}')
+
+
+class TestPyEncodeBasestringAscii(TestEncodeBasestringAscii, PyTest): pass
+class TestCEncodeBasestringAscii(TestEncodeBasestringAscii, CTest): pass
diff --git a/lib-python/2.7/json/tests/test_fail.py b/lib-python/2.7/json/tests/test_fail.py
--- a/lib-python/2.7/json/tests/test_fail.py
+++ b/lib-python/2.7/json/tests/test_fail.py
@@ -1,6 +1,4 @@
-from unittest import TestCase
-
-import json
+from json.tests import PyTest, CTest
 
 # Fri Dec 30 18:57:26 2005
 JSONDOCS = [
@@ -61,15 +59,15 @@
     18: "spec doesn't specify any nesting limitations",
 }
 
-class TestFail(TestCase):
+class TestFail(object):
     def test_failures(self):
         for idx, doc in enumerate(JSONDOCS):
             idx = idx + 1
             if idx in SKIPS:
-                json.loads(doc)
+                self.loads(doc)
                 continue
             try:
-                json.loads(doc)
+                self.loads(doc)
             except ValueError:
                 pass
             else:
@@ -79,7 +77,11 @@
         data = {'a' : 1, (1, 2) : 2}
 
         #This is for c encoder
-        self.assertRaises(TypeError, json.dumps, data)
+        self.assertRaises(TypeError, self.dumps, data)
 
         #This is for python encoder
-        self.assertRaises(TypeError, json.dumps, data, indent=True)
+        self.assertRaises(TypeError, self.dumps, data, indent=True)
+
+
+class TestPyFail(TestFail, PyTest): pass
+class TestCFail(TestFail, CTest): pass
diff --git a/lib-python/2.7/json/tests/test_float.py b/lib-python/2.7/json/tests/test_float.py
--- a/lib-python/2.7/json/tests/test_float.py
+++ b/lib-python/2.7/json/tests/test_float.py
@@ -1,19 +1,22 @@
 import math
-from unittest import TestCase
+from json.tests import PyTest, CTest
 
-import json
 
-class TestFloat(TestCase):
+class TestFloat(object):
     def test_floats(self):
         for num in [1617161771.7650001, math.pi, math.pi**100,
                     math.pi**-100, 3.1]:
-            self.assertEqual(float(json.dumps(num)), num)
-            self.assertEqual(json.loads(json.dumps(num)), num)
-            self.assertEqual(json.loads(unicode(json.dumps(num))), num)
+            self.assertEqual(float(self.dumps(num)), num)
+            self.assertEqual(self.loads(self.dumps(num)), num)
+            self.assertEqual(self.loads(unicode(self.dumps(num))), num)
 
     def test_ints(self):
         for num in [1, 1L, 1<<32, 1<<64]:
-            self.assertEqual(json.dumps(num), str(num))
-            self.assertEqual(int(json.dumps(num)), num)
-            self.assertEqual(json.loads(json.dumps(num)), num)
-            self.assertEqual(json.loads(unicode(json.dumps(num))), num)
+            self.assertEqual(self.dumps(num), str(num))
+            self.assertEqual(int(self.dumps(num)), num)
+            self.assertEqual(self.loads(self.dumps(num)), num)
+            self.assertEqual(self.loads(unicode(self.dumps(num))), num)
+
+
+class TestPyFloat(TestFloat, PyTest): pass
+class TestCFloat(TestFloat, CTest): pass
diff --git a/lib-python/2.7/json/tests/test_indent.py b/lib-python/2.7/json/tests/test_indent.py
--- a/lib-python/2.7/json/tests/test_indent.py
+++ b/lib-python/2.7/json/tests/test_indent.py
@@ -1,9 +1,9 @@
-from unittest import TestCase
+import textwrap
+from StringIO import StringIO
+from json.tests import PyTest, CTest
 
-import json
-import textwrap
 
-class TestIndent(TestCase):
+class TestIndent(object):
     def test_indent(self):
         h = [['blorpie'], ['whoops'], [], 'd-shtaeou', 'd-nthiouh', 'i-vhbjkhnth',
              {'nifty': 87}, {'field': 'yes', 'morefield': False} ]
@@ -30,12 +30,31 @@
         ]""")
 
 
-        d1 = json.dumps(h)
-        d2 = json.dumps(h, indent=2, sort_keys=True, separators=(',', ': '))
+        d1 = self.dumps(h)
+        d2 = self.dumps(h, indent=2, sort_keys=True, separators=(',', ': '))
 
-        h1 = json.loads(d1)
-        h2 = json.loads(d2)
+        h1 = self.loads(d1)
+        h2 = self.loads(d2)
 
         self.assertEqual(h1, h)
         self.assertEqual(h2, h)
         self.assertEqual(d2, expect)
+
+    def test_indent0(self):
+        h = {3: 1}
+        def check(indent, expected):
+            d1 = self.dumps(h, indent=indent)
+            self.assertEqual(d1, expected)
+
+            sio = StringIO()
+            self.json.dump(h, sio, indent=indent)
+            self.assertEqual(sio.getvalue(), expected)
+
+        # indent=0 should emit newlines
+        check(0, '{\n"3": 1\n}')
+        # indent=None is more compact
+        check(None, '{"3": 1}')
+
+
+class TestPyIndent(TestIndent, PyTest): pass
+class TestCIndent(TestIndent, CTest): pass
diff --git a/lib-python/2.7/json/tests/test_pass1.py b/lib-python/2.7/json/tests/test_pass1.py
--- a/lib-python/2.7/json/tests/test_pass1.py
+++ b/lib-python/2.7/json/tests/test_pass1.py
@@ -1,6 +1,5 @@
-from unittest import TestCase
+from json.tests import PyTest, CTest
 
-import json
 
 # from http://json.org/JSON_checker/test/pass1.json
 JSON = r'''
@@ -62,15 +61,19 @@
 ,"rosebud"]
 '''
 
-class TestPass1(TestCase):
+class TestPass1(object):
     def test_parse(self):
         # test in/out equivalence and parsing
-        res = json.loads(JSON)
-        out = json.dumps(res)
-        self.assertEqual(res, json.loads(out))
+        res = self.loads(JSON)
+        out = self.dumps(res)
+        self.assertEqual(res, self.loads(out))
         try:
-            json.dumps(res, allow_nan=False)
+            self.dumps(res, allow_nan=False)
         except ValueError:
             pass
         else:
             self.fail("23456789012E666 should be out of range")
+
+
+class TestPyPass1(TestPass1, PyTest): pass
+class TestCPass1(TestPass1, CTest): pass
diff --git a/lib-python/2.7/json/tests/test_pass2.py b/lib-python/2.7/json/tests/test_pass2.py
--- a/lib-python/2.7/json/tests/test_pass2.py
+++ b/lib-python/2.7/json/tests/test_pass2.py
@@ -1,14 +1,18 @@
-from unittest import TestCase
-import json
+from json.tests import PyTest, CTest
+
 
 # from http://json.org/JSON_checker/test/pass2.json
 JSON = r'''
 [[[[[[[[[[[[[[[[[[["Not too deep"]]]]]]]]]]]]]]]]]]]
 '''
 
-class TestPass2(TestCase):
+class TestPass2(object):
     def test_parse(self):
         # test in/out equivalence and parsing
-        res = json.loads(JSON)
-        out = json.dumps(res)
-        self.assertEqual(res, json.loads(out))
+        res = self.loads(JSON)
+        out = self.dumps(res)
+        self.assertEqual(res, self.loads(out))
+
+
+class TestPyPass2(TestPass2, PyTest): pass
+class TestCPass2(TestPass2, CTest): pass
diff --git a/lib-python/2.7/json/tests/test_pass3.py b/lib-python/2.7/json/tests/test_pass3.py
--- a/lib-python/2.7/json/tests/test_pass3.py
+++ b/lib-python/2.7/json/tests/test_pass3.py
@@ -1,6 +1,5 @@
-from unittest import TestCase
+from json.tests import PyTest, CTest
 
-import json
 
 # from http://json.org/JSON_checker/test/pass3.json
 JSON = r'''
@@ -12,9 +11,14 @@
 }
 '''
 
-class TestPass3(TestCase):
+
+class TestPass3(object):
     def test_parse(self):
         # test in/out equivalence and parsing
-        res = json.loads(JSON)
-        out = json.dumps(res)
-        self.assertEqual(res, json.loads(out))
+        res = self.loads(JSON)
+        out = self.dumps(res)
+        self.assertEqual(res, self.loads(out))
+
+
+class TestPyPass3(TestPass3, PyTest): pass
+class TestCPass3(TestPass3, CTest): pass
diff --git a/lib-python/2.7/json/tests/test_recursion.py b/lib-python/2.7/json/tests/test_recursion.py
--- a/lib-python/2.7/json/tests/test_recursion.py
+++ b/lib-python/2.7/json/tests/test_recursion.py
@@ -1,28 +1,16 @@
-from unittest import TestCase
+from json.tests import PyTest, CTest
 
-import json
 
 class JSONTestObject:
     pass
 
 
-class RecursiveJSONEncoder(json.JSONEncoder):
-    recurse = False
-    def default(self, o):
-        if o is JSONTestObject:
-            if self.recurse:
-                return [JSONTestObject]
-            else:
-                return 'JSONTestObject'
-        return json.JSONEncoder.default(o)
-
-
-class TestRecursion(TestCase):
+class TestRecursion(object):
     def test_listrecursion(self):
         x = []
         x.append(x)
         try:
-            json.dumps(x)
+            self.dumps(x)
         except ValueError:
             pass
         else:
@@ -31,7 +19,7 @@
         y = [x]
         x.append(y)
         try:
-            json.dumps(x)
+            self.dumps(x)
         except ValueError:
             pass
         else:
@@ -39,13 +27,13 @@
         y = []
         x = [y, y]
         # ensure that the marker is cleared
-        json.dumps(x)
+        self.dumps(x)
 
     def test_dictrecursion(self):
         x = {}
         x["test"] = x
         try:
-            json.dumps(x)
+            self.dumps(x)
         except ValueError:
             pass
         else:
@@ -53,9 +41,19 @@
         x = {}
         y = {"a": x, "b": x}
         # ensure that the marker is cleared
-        json.dumps(x)
+        self.dumps(x)
 
     def test_defaultrecursion(self):
+        class RecursiveJSONEncoder(self.json.JSONEncoder):
+            recurse = False
+            def default(self, o):
+                if o is JSONTestObject:
+                    if self.recurse:
+                        return [JSONTestObject]
+                    else:
+                        return 'JSONTestObject'
+                return pyjson.JSONEncoder.default(o)
+
         enc = RecursiveJSONEncoder()
         self.assertEqual(enc.encode(JSONTestObject), '"JSONTestObject"')
         enc.recurse = True
@@ -65,3 +63,46 @@
             pass
         else:
             self.fail("didn't raise ValueError on default recursion")
+
+
+    def test_highly_nested_objects_decoding(self):
+        # test that loading highly-nested objects doesn't segfault when C
+        # accelerations are used. See #12017
+        # str
+        with self.assertRaises(RuntimeError):
+            self.loads('{"a":' * 100000 + '1' + '}' * 100000)
+        with self.assertRaises(RuntimeError):
+            self.loads('{"a":' * 100000 + '[1]' + '}' * 100000)
+        with self.assertRaises(RuntimeError):
+            self.loads('[' * 100000 + '1' + ']' * 100000)
+        # unicode
+        with self.assertRaises(RuntimeError):
+            self.loads(u'{"a":' * 100000 + u'1' + u'}' * 100000)
+        with self.assertRaises(RuntimeError):
+            self.loads(u'{"a":' * 100000 + u'[1]' + u'}' * 100000)
+        with self.assertRaises(RuntimeError):
+            self.loads(u'[' * 100000 + u'1' + u']' * 100000)
+
+    def test_highly_nested_objects_encoding(self):
+        # See #12051
+        l, d = [], {}
+        for x in xrange(100000):
+            l, d = [l], {'k':d}
+        with self.assertRaises(RuntimeError):
+            self.dumps(l)
+        with self.assertRaises(RuntimeError):
+            self.dumps(d)
+
+    def test_endless_recursion(self):
+        # See #12051
+        class EndlessJSONEncoder(self.json.JSONEncoder):
+            def default(self, o):
+                """If check_circular is False, this will keep adding another list."""
+                return [o]
+
+        with self.assertRaises(RuntimeError):
+            EndlessJSONEncoder(check_circular=False).encode(5j)
+
+
+class TestPyRecursion(TestRecursion, PyTest): pass
+class TestCRecursion(TestRecursion, CTest): pass
diff --git a/lib-python/2.7/json/tests/test_scanstring.py b/lib-python/2.7/json/tests/test_scanstring.py
--- a/lib-python/2.7/json/tests/test_scanstring.py
+++ b/lib-python/2.7/json/tests/test_scanstring.py
@@ -1,18 +1,10 @@
 import sys
-import decimal
-from unittest import TestCase
+from json.tests import PyTest, CTest
 
-import json
-import json.decoder
 
-class TestScanString(TestCase):
-    def test_py_scanstring(self):
-        self._test_scanstring(json.decoder.py_scanstring)
-
-    def test_c_scanstring(self):
-        self._test_scanstring(json.decoder.c_scanstring)
-
-    def _test_scanstring(self, scanstring):
+class TestScanstring(object):
+    def test_scanstring(self):
+        scanstring = self.json.decoder.scanstring
         self.assertEqual(
             scanstring('"z\\ud834\\udd20x"', 1, None, True),
             (u'z\U0001d120x', 16))
@@ -103,10 +95,15 @@
             (u'Bad value', 12))
 
     def test_issue3623(self):
-        self.assertRaises(ValueError, json.decoder.scanstring, b"xxx", 1,
+        self.assertRaises(ValueError, self.json.decoder.scanstring, b"xxx", 1,
                           "xxx")
         self.assertRaises(UnicodeDecodeError,
-                          json.encoder.encode_basestring_ascii, b"xx\xff")
+                          self.json.encoder.encode_basestring_ascii, b"xx\xff")
 
     def test_overflow(self):
-        self.assertRaises(OverflowError, json.decoder.scanstring, b"xxx", sys.maxsize+1)
+        with self.assertRaises(OverflowError):
+            self.json.decoder.scanstring(b"xxx", sys.maxsize+1)
+
+
+class TestPyScanstring(TestScanstring, PyTest): pass
+class TestCScanstring(TestScanstring, CTest): pass
diff --git a/lib-python/2.7/json/tests/test_separators.py b/lib-python/2.7/json/tests/test_separators.py
--- a/lib-python/2.7/json/tests/test_separators.py
+++ b/lib-python/2.7/json/tests/test_separators.py
@@ -1,10 +1,8 @@
 import textwrap
-from unittest import TestCase
+from json.tests import PyTest, CTest
 
-import json
 
-
-class TestSeparators(TestCase):
+class TestSeparators(object):
     def test_separators(self):
         h = [['blorpie'], ['whoops'], [], 'd-shtaeou', 'd-nthiouh', 'i-vhbjkhnth',
              {'nifty': 87}, {'field': 'yes', 'morefield': False} ]
@@ -31,12 +29,16 @@
         ]""")
 
 
-        d1 = json.dumps(h)
-        d2 = json.dumps(h, indent=2, sort_keys=True, separators=(' ,', ' : '))
+        d1 = self.dumps(h)
+        d2 = self.dumps(h, indent=2, sort_keys=True, separators=(' ,', ' : '))
 
-        h1 = json.loads(d1)
-        h2 = json.loads(d2)
+        h1 = self.loads(d1)
+        h2 = self.loads(d2)
 
         self.assertEqual(h1, h)
         self.assertEqual(h2, h)
         self.assertEqual(d2, expect)
+
+
+class TestPySeparators(TestSeparators, PyTest): pass
+class TestCSeparators(TestSeparators, CTest): pass
diff --git a/lib-python/2.7/json/tests/test_speedups.py b/lib-python/2.7/json/tests/test_speedups.py
--- a/lib-python/2.7/json/tests/test_speedups.py
+++ b/lib-python/2.7/json/tests/test_speedups.py
@@ -1,24 +1,23 @@
-import decimal
-from unittest import TestCase
+from json.tests import CTest
 
-from json import decoder, encoder, scanner
 
-class TestSpeedups(TestCase):
+class TestSpeedups(CTest):
     def test_scanstring(self):
-        self.assertEqual(decoder.scanstring.__module__, "_json")
-        self.assertTrue(decoder.scanstring is decoder.c_scanstring)
+        self.assertEqual(self.json.decoder.scanstring.__module__, "_json")
+        self.assertIs(self.json.decoder.scanstring, self.json.decoder.c_scanstring)
 
     def test_encode_basestring_ascii(self):
-        self.assertEqual(encoder.encode_basestring_ascii.__module__, "_json")
-        self.assertTrue(encoder.encode_basestring_ascii is
-                          encoder.c_encode_basestring_ascii)
+        self.assertEqual(self.json.encoder.encode_basestring_ascii.__module__,
+                         "_json")
+        self.assertIs(self.json.encoder.encode_basestring_ascii,
+                      self.json.encoder.c_encode_basestring_ascii)
 
-class TestDecode(TestCase):
+class TestDecode(CTest):
     def test_make_scanner(self):
-        self.assertRaises(AttributeError, scanner.c_make_scanner, 1)
+        self.assertRaises(AttributeError, self.json.scanner.c_make_scanner, 1)
 
     def test_make_encoder(self):
-        self.assertRaises(TypeError, encoder.c_make_encoder,
+        self.assertRaises(TypeError, self.json.encoder.c_make_encoder,
             None,
             "\xCD\x7D\x3D\x4E\x12\x4C\xF9\x79\xD7\x52\xBA\x82\xF2\x27\x4A\x7D\xA0\xCA\x75",
             None)
diff --git a/lib-python/2.7/json/tests/test_unicode.py b/lib-python/2.7/json/tests/test_unicode.py
--- a/lib-python/2.7/json/tests/test_unicode.py
+++ b/lib-python/2.7/json/tests/test_unicode.py
@@ -1,11 +1,10 @@
-from unittest import TestCase
+from collections import OrderedDict
+from json.tests import PyTest, CTest
 
-import json
-from collections import OrderedDict
 
-class TestUnicode(TestCase):
+class TestUnicode(object):
     def test_encoding1(self):
-        encoder = json.JSONEncoder(encoding='utf-8')
+        encoder = self.json.JSONEncoder(encoding='utf-8')
         u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}'
         s = u.encode('utf-8')
         ju = encoder.encode(u)
@@ -15,68 +14,72 @@
     def test_encoding2(self):
         u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}'
         s = u.encode('utf-8')
-        ju = json.dumps(u, encoding='utf-8')
-        js = json.dumps(s, encoding='utf-8')
+        ju = self.dumps(u, encoding='utf-8')
+        js = self.dumps(s, encoding='utf-8')
         self.assertEqual(ju, js)
 
     def test_encoding3(self):
         u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}'
-        j = json.dumps(u)
+        j = self.dumps(u)
         self.assertEqual(j, '"\\u03b1\\u03a9"')
 
     def test_encoding4(self):
         u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}'
-        j = json.dumps([u])
+        j = self.dumps([u])
         self.assertEqual(j, '["\\u03b1\\u03a9"]')
 
     def test_encoding5(self):
         u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}'
-        j = json.dumps(u, ensure_ascii=False)
+        j = self.dumps(u, ensure_ascii=False)
         self.assertEqual(j, u'"{0}"'.format(u))
 
     def test_encoding6(self):
         u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}'
-        j = json.dumps([u], ensure_ascii=False)
+        j = self.dumps([u], ensure_ascii=False)
         self.assertEqual(j, u'["{0}"]'.format(u))
 
     def test_big_unicode_encode(self):
         u = u'\U0001d120'
-        self.assertEqual(json.dumps(u), '"\\ud834\\udd20"')
-        self.assertEqual(json.dumps(u, ensure_ascii=False), u'"\U0001d120"')
+        self.assertEqual(self.dumps(u), '"\\ud834\\udd20"')
+        self.assertEqual(self.dumps(u, ensure_ascii=False), u'"\U0001d120"')
 
     def test_big_unicode_decode(self):
         u = u'z\U0001d120x'
-        self.assertEqual(json.loads('"' + u + '"'), u)
-        self.assertEqual(json.loads('"z\\ud834\\udd20x"'), u)
+        self.assertEqual(self.loads('"' + u + '"'), u)
+        self.assertEqual(self.loads('"z\\ud834\\udd20x"'), u)
 
     def test_unicode_decode(self):
         for i in range(0, 0xd7ff):
             u = unichr(i)
             s = '"\\u{0:04x}"'.format(i)
-            self.assertEqual(json.loads(s), u)
+            self.assertEqual(self.loads(s), u)
 
     def test_object_pairs_hook_with_unicode(self):
         s = u'{"xkd":1, "kcw":2, "art":3, "hxm":4, "qrt":5, "pad":6, "hoy":7}'
         p = [(u"xkd", 1), (u"kcw", 2), (u"art", 3), (u"hxm", 4),
              (u"qrt", 5), (u"pad", 6), (u"hoy", 7)]
-        self.assertEqual(json.loads(s), eval(s))
-        self.assertEqual(json.loads(s, object_pairs_hook = lambda x: x), p)
-        od = json.loads(s, object_pairs_hook = OrderedDict)
+        self.assertEqual(self.loads(s), eval(s))
+        self.assertEqual(self.loads(s, object_pairs_hook = lambda x: x), p)
+        od = self.loads(s, object_pairs_hook = OrderedDict)
         self.assertEqual(od, OrderedDict(p))
         self.assertEqual(type(od), OrderedDict)
         # the object_pairs_hook takes priority over the object_hook
-        self.assertEqual(json.loads(s,
+        self.assertEqual(self.loads(s,
                                     object_pairs_hook = OrderedDict,
                                     object_hook = lambda x: None),
                          OrderedDict(p))
 
     def test_default_encoding(self):
-        self.assertEqual(json.loads(u'{"a": "\xe9"}'.encode('utf-8')),
+        self.assertEqual(self.loads(u'{"a": "\xe9"}'.encode('utf-8')),
             {'a': u'\xe9'})
 
     def test_unicode_preservation(self):
-        self.assertEqual(type(json.loads(u'""')), unicode)
-        self.assertEqual(type(json.loads(u'"a"')), unicode)
-        self.assertEqual(type(json.loads(u'["a"]')[0]), unicode)
+        self.assertEqual(type(self.loads(u'""')), unicode)
+        self.assertEqual(type(self.loads(u'"a"')), unicode)
+        self.assertEqual(type(self.loads(u'["a"]')[0]), unicode)
         # Issue 10038.
-        self.assertEqual(type(json.loads('"foo"')), unicode)
+        self.assertEqual(type(self.loads('"foo"')), unicode)
+
+
+class TestPyUnicode(TestUnicode, PyTest): pass
+class TestCUnicode(TestUnicode, CTest): pass
diff --git a/lib-python/2.7/lib-tk/Tix.py b/lib-python/2.7/lib-tk/Tix.py
--- a/lib-python/2.7/lib-tk/Tix.py
+++ b/lib-python/2.7/lib-tk/Tix.py
@@ -163,7 +163,7 @@
         extensions) exist, then the image type is chosen according to the
         depth of the X display: xbm images are chosen on monochrome
         displays and color images are chosen on color displays. By using
-        tix_ getimage, you can advoid hard coding the pathnames of the
+        tix_ getimage, you can avoid hard coding the pathnames of the
         image files in your application. When successful, this command
         returns the name of the newly created image, which can be used to
         configure the -image option of the Tk and Tix widgets.
@@ -171,7 +171,7 @@
         return self.tk.call('tix', 'getimage', name)
 
     def tix_option_get(self, name):
-        """Gets  the options  manitained  by  the  Tix
+        """Gets  the options  maintained  by  the  Tix
         scheme mechanism. Available options include:
 
             active_bg       active_fg      bg
@@ -576,7 +576,7 @@
 
 class ComboBox(TixWidget):
     """ComboBox - an Entry field with a dropdown menu. The user can select a
-    choice by either typing in the entry subwdget or selecting from the
+    choice by either typing in the entry subwidget or selecting from the
     listbox subwidget.
 
     Subwidget       Class
@@ -869,7 +869,7 @@
     """HList - Hierarchy display  widget can be used to display any data
     that have a hierarchical structure, for example, file system directory
     trees. The list entries are indented and connected by branch lines
-    according to their places in the hierachy.
+    according to their places in the hierarchy.
 
     Subwidgets - None"""
 
@@ -1520,7 +1520,7 @@
         self.tk.call(self._w, 'selection', 'set', first, last)
 
 class Tree(TixWidget):
-    """Tree - The tixTree widget can be used to display hierachical
+    """Tree - The tixTree widget can be used to display hierarchical
     data in a tree form. The user can adjust
     the view of the tree by opening or closing parts of the tree."""
 
diff --git a/lib-python/2.7/lib-tk/Tkinter.py b/lib-python/2.7/lib-tk/Tkinter.py
--- a/lib-python/2.7/lib-tk/Tkinter.py
+++ b/lib-python/2.7/lib-tk/Tkinter.py
@@ -1660,7 +1660,7 @@
 
 class Tk(Misc, Wm):
     """Toplevel widget of Tk which represents mostly the main window
-    of an appliation. It has an associated Tcl interpreter."""
+    of an application. It has an associated Tcl interpreter."""
     _w = '.'
     def __init__(self, screenName=None, baseName=None, className='Tk',
                  useTk=1, sync=0, use=None):
diff --git a/lib-python/2.7/lib-tk/test/test_ttk/test_functions.py b/lib-python/2.7/lib-tk/test/test_ttk/test_functions.py
--- a/lib-python/2.7/lib-tk/test/test_ttk/test_functions.py
+++ b/lib-python/2.7/lib-tk/test/test_ttk/test_functions.py
@@ -136,7 +136,7 @@
         # minimum acceptable for image type
         self.assertEqual(ttk._format_elemcreate('image', False, 'test'),
             ("test ", ()))
-        # specifiyng a state spec
+        # specifying a state spec
         self.assertEqual(ttk._format_elemcreate('image', False, 'test',
             ('', 'a')), ("test {} a", ()))
         # state spec with multiple states
diff --git a/lib-python/2.7/lib-tk/ttk.py b/lib-python/2.7/lib-tk/ttk.py
--- a/lib-python/2.7/lib-tk/ttk.py
+++ b/lib-python/2.7/lib-tk/ttk.py
@@ -707,7 +707,7 @@
             textvariable, values, width
         """
         # The "values" option may need special formatting, so leave to
-        # _format_optdict the responsability to format it
+        # _format_optdict the responsibility to format it
         if "values" in kw:
             kw["values"] = _format_optdict({'v': kw["values"]})[1]
 
@@ -993,7 +993,7 @@
         pane is either an integer index or the name of a managed subwindow.
         If kw is not given, returns a dict of the pane option values. If
         option is specified then the value for that option is returned.
-        Otherwise, sets the options to the correspoding values."""
+        Otherwise, sets the options to the corresponding values."""
         if option is not None:
             kw[option] = None
         return _val_or_dict(kw, self.tk.call, self._w, "pane", pane)
diff --git a/lib-python/2.7/lib-tk/turtle.py b/lib-python/2.7/lib-tk/turtle.py
--- a/lib-python/2.7/lib-tk/turtle.py
+++ b/lib-python/2.7/lib-tk/turtle.py
@@ -1385,7 +1385,7 @@
         Optional argument:
         picname -- a string, name of a gif-file or "nopic".
 
-        If picname is a filename, set the corresponing image as background.
+        If picname is a filename, set the corresponding image as background.
         If picname is "nopic", delete backgroundimage, if present.
         If picname is None, return the filename of the current backgroundimage.
 
@@ -1409,7 +1409,7 @@
         Optional arguments:
         canvwidth -- positive integer, new width of canvas in pixels
         canvheight --  positive integer, new height of canvas in pixels
-        bg -- colorstring or color-tupel, new backgroundcolor
+        bg -- colorstring or color-tuple, new backgroundcolor
         If no arguments are given, return current (canvaswidth, canvasheight)
 
         Do not alter the drawing window. To observe hidden parts of
@@ -3079,9 +3079,9 @@
                                                fill="", width=ps)
         # Turtle now at position old,
         self._position = old
-        ##  if undo is done during crating a polygon, the last vertex
-        ##  will be deleted. if the polygon is entirel deleted,
-        ##  creatigPoly will be set to False.
+        ##  if undo is done during creating a polygon, the last vertex
+        ##  will be deleted. if the polygon is entirely deleted,
+        ##  creatingPoly will be set to False.
         ##  Polygons created before the last one will not be affected by undo()
         if self._creatingPoly:
             if len(self._poly) > 0:
@@ -3221,7 +3221,7 @@
     def dot(self, size=None, *color):
         """Draw a dot with diameter size, using color.
 
-        Optional argumentS:
+        Optional arguments:
         size -- an integer >= 1 (if given)
         color -- a colorstring or a numeric color tuple
 
@@ -3691,7 +3691,7 @@
 
 
 class Turtle(RawTurtle):
-    """RawTurtle auto-crating (scrolled) canvas.
+    """RawTurtle auto-creating (scrolled) canvas.
 
     When a Turtle object is created or a function derived from some
     Turtle method is called a TurtleScreen object is automatically created.
@@ -3731,7 +3731,7 @@
     filename -- a string, used as filename
                 default value is turtle_docstringdict
 
-    Has to be called explicitely, (not used by the turtle-graphics classes)
+    Has to be called explicitly, (not used by the turtle-graphics classes)
     The docstring dictionary will be written to the Python script .py
     It is intended to serve as a template for translation of the docstrings
     into different languages.
diff --git a/lib-python/2.7/lib2to3/__main__.py b/lib-python/2.7/lib2to3/__main__.py
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/lib2to3/__main__.py
@@ -0,0 +1,4 @@
+import sys
+from .main import main
+
+sys.exit(main("lib2to3.fixes"))
diff --git a/lib-python/2.7/lib2to3/fixes/fix_itertools.py b/lib-python/2.7/lib2to3/fixes/fix_itertools.py
--- a/lib-python/2.7/lib2to3/fixes/fix_itertools.py
+++ b/lib-python/2.7/lib2to3/fixes/fix_itertools.py
@@ -13,7 +13,7 @@
 
 class FixItertools(fixer_base.BaseFix):
     BM_compatible = True
-    it_funcs = "('imap'|'ifilter'|'izip'|'ifilterfalse')"
+    it_funcs = "('imap'|'ifilter'|'izip'|'izip_longest'|'ifilterfalse')"
     PATTERN = """
               power< it='itertools'
                   trailer<
@@ -28,7 +28,8 @@
     def transform(self, node, results):
         prefix = None
         func = results['func'][0]
-        if 'it' in results and func.value != u'ifilterfalse':
+        if ('it' in results and
+            func.value not in (u'ifilterfalse', u'izip_longest')):
             dot, it = (results['dot'], results['it'])
             # Remove the 'itertools'
             prefix = it.prefix
diff --git a/lib-python/2.7/lib2to3/fixes/fix_itertools_imports.py b/lib-python/2.7/lib2to3/fixes/fix_itertools_imports.py
--- a/lib-python/2.7/lib2to3/fixes/fix_itertools_imports.py
+++ b/lib-python/2.7/lib2to3/fixes/fix_itertools_imports.py
@@ -31,9 +31,10 @@
             if member_name in (u'imap', u'izip', u'ifilter'):
                 child.value = None
                 child.remove()
-            elif member_name == u'ifilterfalse':
+            elif member_name in (u'ifilterfalse', u'izip_longest'):
                 node.changed()
-                name_node.value = u'filterfalse'
+                name_node.value = (u'filterfalse' if member_name[1] == u'f'
+                                   else u'zip_longest')
 
         # Make sure the import statement is still sane
         children = imports.children[:] or [imports]
diff --git a/lib-python/2.7/lib2to3/fixes/fix_metaclass.py b/lib-python/2.7/lib2to3/fixes/fix_metaclass.py
--- a/lib-python/2.7/lib2to3/fixes/fix_metaclass.py
+++ b/lib-python/2.7/lib2to3/fixes/fix_metaclass.py
@@ -48,7 +48,7 @@
     """
     for node in cls_node.children:
         if node.type == syms.suite:
-            # already in the prefered format, do nothing
+            # already in the preferred format, do nothing
             return
 
     # !%@#! oneliners have no suite node, we have to fake one up
diff --git a/lib-python/2.7/lib2to3/fixes/fix_urllib.py b/lib-python/2.7/lib2to3/fixes/fix_urllib.py
--- a/lib-python/2.7/lib2to3/fixes/fix_urllib.py
+++ b/lib-python/2.7/lib2to3/fixes/fix_urllib.py
@@ -12,7 +12,7 @@
 
 MAPPING = {"urllib":  [
                 ("urllib.request",
-                    ["URLOpener", "FancyURLOpener", "urlretrieve",
+                    ["URLopener", "FancyURLopener", "urlretrieve",
                      "_urlopener", "urlopen", "urlcleanup",
                      "pathname2url", "url2pathname"]),
                 ("urllib.parse",
diff --git a/lib-python/2.7/lib2to3/main.py b/lib-python/2.7/lib2to3/main.py
--- a/lib-python/2.7/lib2to3/main.py
+++ b/lib-python/2.7/lib2to3/main.py
@@ -101,7 +101,7 @@
     parser.add_option("-j", "--processes", action="store", default=1,
                       type="int", help="Run 2to3 concurrently")
     parser.add_option("-x", "--nofix", action="append", default=[],
-                      help="Prevent a fixer from being run.")
+                      help="Prevent a transformation from being run")
     parser.add_option("-l", "--list-fixes", action="store_true",
                       help="List available transformations")
     parser.add_option("-p", "--print-function", action="store_true",
@@ -113,7 +113,7 @@
     parser.add_option("-w", "--write", action="store_true",
                       help="Write back modified files")
     parser.add_option("-n", "--nobackups", action="store_true", default=False,
-                      help="Don't write backups for modified files.")
+                      help="Don't write backups for modified files")
 
     # Parse command line arguments
     refactor_stdin = False
diff --git a/lib-python/2.7/lib2to3/patcomp.py b/lib-python/2.7/lib2to3/patcomp.py
--- a/lib-python/2.7/lib2to3/patcomp.py
+++ b/lib-python/2.7/lib2to3/patcomp.py
@@ -12,6 +12,7 @@
 
 # Python imports
 import os
+import StringIO
 
 # Fairly local imports
 from .pgen2 import driver, literals, token, tokenize, parse, grammar
@@ -32,7 +33,7 @@
 def tokenize_wrapper(input):
     """Tokenizes a string suppressing significant whitespace."""
     skip = set((token.NEWLINE, token.INDENT, token.DEDENT))
-    tokens = tokenize.generate_tokens(driver.generate_lines(input).next)
+    tokens = tokenize.generate_tokens(StringIO.StringIO(input).readline)
     for quintuple in tokens:
         type, value, start, end, line_text = quintuple
         if type not in skip:
diff --git a/lib-python/2.7/lib2to3/pgen2/conv.py b/lib-python/2.7/lib2to3/pgen2/conv.py
--- a/lib-python/2.7/lib2to3/pgen2/conv.py
+++ b/lib-python/2.7/lib2to3/pgen2/conv.py
@@ -51,7 +51,7 @@
         self.finish_off()
 
     def parse_graminit_h(self, filename):
-        """Parse the .h file writen by pgen.  (Internal)
+        """Parse the .h file written by pgen.  (Internal)
 
         This file is a sequence of #define statements defining the
         nonterminals of the grammar as numbers.  We build two tables
@@ -82,7 +82,7 @@
         return True
 
     def parse_graminit_c(self, filename):
-        """Parse the .c file writen by pgen.  (Internal)
+        """Parse the .c file written by pgen.  (Internal)
 
         The file looks as follows.  The first two lines are always this:
 
diff --git a/lib-python/2.7/lib2to3/pgen2/driver.py b/lib-python/2.7/lib2to3/pgen2/driver.py
--- a/lib-python/2.7/lib2to3/pgen2/driver.py
+++ b/lib-python/2.7/lib2to3/pgen2/driver.py
@@ -19,6 +19,7 @@
 import codecs
 import os
 import logging
+import StringIO
 import sys
 
 # Pgen imports
@@ -101,18 +102,10 @@
 
     def parse_string(self, text, debug=False):
         """Parse a string and return the syntax tree."""
-        tokens = tokenize.generate_tokens(generate_lines(text).next)
+        tokens = tokenize.generate_tokens(StringIO.StringIO(text).readline)
         return self.parse_tokens(tokens, debug)
 
 
-def generate_lines(text):
-    """Generator that behaves like readline without using StringIO."""
-    for line in text.splitlines(True):
-        yield line
-    while True:
-        yield ""
-
-
 def load_grammar(gt="Grammar.txt", gp=None,
                  save=True, force=False, logger=None):
     """Load the grammar (maybe from a pickle)."""
diff --git a/lib-python/2.7/lib2to3/pytree.py b/lib-python/2.7/lib2to3/pytree.py
--- a/lib-python/2.7/lib2to3/pytree.py
+++ b/lib-python/2.7/lib2to3/pytree.py
@@ -658,8 +658,8 @@
             content: optional sequence of subsequences of patterns;
                      if absent, matches one node;
                      if present, each subsequence is an alternative [*]
-            min: optinal minumum number of times to match, default 0
-            max: optional maximum number of times tro match, default HUGE
+            min: optional minimum number of times to match, default 0
+            max: optional maximum number of times to match, default HUGE
             name: optional name assigned to this match
 
         [*] Thus, if content is [[a, b, c], [d, e], [f, g, h]] this is
@@ -743,9 +743,11 @@
         else:
             # The reason for this is that hitting the recursion limit usually
             # results in some ugly messages about how RuntimeErrors are being
-            # ignored.
-            save_stderr = sys.stderr
-            sys.stderr = StringIO()
+            # ignored. We don't do this on non-CPython implementation because
+            # they don't have this problem.
+            if hasattr(sys, "getrefcount"):
+                save_stderr = sys.stderr
+                sys.stderr = StringIO()
             try:
                 for count, r in self._recursive_matches(nodes, 0):
                     if self.name:
@@ -759,7 +761,8 @@
                         r[self.name] = nodes[:count]
                     yield count, r
             finally:
-                sys.stderr = save_stderr
+                if hasattr(sys, "getrefcount"):
+                    sys.stderr = save_stderr
 
     def _iterative_matches(self, nodes):
         """Helper to iteratively yield the matches."""
diff --git a/lib-python/2.7/lib2to3/refactor.py b/lib-python/2.7/lib2to3/refactor.py
--- a/lib-python/2.7/lib2to3/refactor.py
+++ b/lib-python/2.7/lib2to3/refactor.py
@@ -302,13 +302,14 @@
 
         Files and subdirectories starting with '.' are skipped.
         """
+        py_ext = os.extsep + "py"
         for dirpath, dirnames, filenames in os.walk(dir_name):
             self.log_debug("Descending into %s", dirpath)
             dirnames.sort()
             filenames.sort()
             for name in filenames:
-                if not name.startswith(".") and \
-                        os.path.splitext(name)[1].endswith("py"):
+                if (not name.startswith(".") and
+                    os.path.splitext(name)[1] == py_ext):
                     fullname = os.path.join(dirpath, name)
                     self.refactor_file(fullname, write, doctests_only)
             # Modify dirnames in-place to remove subdirs with leading dots
diff --git a/lib-python/2.7/lib2to3/tests/data/py2_test_grammar.py b/lib-python/2.7/lib2to3/tests/data/py2_test_grammar.py
--- a/lib-python/2.7/lib2to3/tests/data/py2_test_grammar.py
+++ b/lib-python/2.7/lib2to3/tests/data/py2_test_grammar.py
@@ -316,7 +316,7 @@
         ### simple_stmt: small_stmt (';' small_stmt)* [';']
         x = 1; pass; del x
         def foo():
-            # verify statments that end with semi-colons
+            # verify statements that end with semi-colons
             x = 1; pass; del x;
         foo()
 
diff --git a/lib-python/2.7/lib2to3/tests/data/py3_test_grammar.py b/lib-python/2.7/lib2to3/tests/data/py3_test_grammar.py
--- a/lib-python/2.7/lib2to3/tests/data/py3_test_grammar.py
+++ b/lib-python/2.7/lib2to3/tests/data/py3_test_grammar.py
@@ -356,7 +356,7 @@
         ### simple_stmt: small_stmt (';' small_stmt)* [';']
         x = 1; pass; del x
         def foo():
-            # verify statments that end with semi-colons
+            # verify statements that end with semi-colons
             x = 1; pass; del x;
         foo()
 
diff --git a/lib-python/2.7/lib2to3/tests/test_fixers.py b/lib-python/2.7/lib2to3/tests/test_fixers.py
--- a/lib-python/2.7/lib2to3/tests/test_fixers.py
+++ b/lib-python/2.7/lib2to3/tests/test_fixers.py
@@ -3623,16 +3623,24 @@
         a = """%s(f, a)"""
         self.checkall(b, a)
 
-    def test_2(self):
+    def test_qualified(self):
         b = """itertools.ifilterfalse(a, b)"""
         a = """itertools.filterfalse(a, b)"""
         self.check(b, a)
 
-    def test_4(self):
+        b = """itertools.izip_longest(a, b)"""
+        a = """itertools.zip_longest(a, b)"""
+        self.check(b, a)
+
+    def test_2(self):
         b = """ifilterfalse(a, b)"""
         a = """filterfalse(a, b)"""
         self.check(b, a)
 
+        b = """izip_longest(a, b)"""
+        a = """zip_longest(a, b)"""
+        self.check(b, a)
+
     def test_space_1(self):
         b = """    %s(f, a)"""
         a = """    %s(f, a)"""
@@ -3643,9 +3651,14 @@
         a = """    itertools.filterfalse(a, b)"""
         self.check(b, a)
 
+        b = """    itertools.izip_longest(a, b)"""
+        a = """    itertools.zip_longest(a, b)"""
+        self.check(b, a)
+
     def test_run_order(self):
         self.assert_runs_after('map', 'zip', 'filter')
 
+
 class Test_itertools_imports(FixerTestCase):
     fixer = 'itertools_imports'
 
@@ -3696,18 +3709,19 @@
         s = "from itertools import bar as bang"
         self.unchanged(s)
 
-    def test_ifilter(self):
-        b = "from itertools import ifilterfalse"
-        a = "from itertools import filterfalse"
-        self.check(b, a)
-
-        b = "from itertools import imap, ifilterfalse, foo"
-        a = "from itertools import filterfalse, foo"
-        self.check(b, a)
-
-        b = "from itertools import bar, ifilterfalse, foo"
-        a = "from itertools import bar, filterfalse, foo"
-        self.check(b, a)
+    def test_ifilter_and_zip_longest(self):
+        for name in "filterfalse", "zip_longest":
+            b = "from itertools import i%s" % (name,)
+            a = "from itertools import %s" % (name,)
+            self.check(b, a)
+
+            b = "from itertools import imap, i%s, foo" % (name,)
+            a = "from itertools import %s, foo" % (name,)
+            self.check(b, a)
+
+            b = "from itertools import bar, i%s, foo" % (name,)
+            a = "from itertools import bar, %s, foo" % (name,)
+            self.check(b, a)
 
     def test_import_star(self):
         s = "from itertools import *"
diff --git a/lib-python/2.7/lib2to3/tests/test_parser.py b/lib-python/2.7/lib2to3/tests/test_parser.py
--- a/lib-python/2.7/lib2to3/tests/test_parser.py
+++ b/lib-python/2.7/lib2to3/tests/test_parser.py
@@ -19,6 +19,16 @@
 # Local imports
 from lib2to3.pgen2 import tokenize
 from ..pgen2.parse import ParseError
+from lib2to3.pygram import python_symbols as syms
+
+
+class TestDriver(support.TestCase):
+
+    def test_formfeed(self):
+        s = """print 1\n\x0Cprint 2\n"""
+        t = driver.parse_string(s)
+        self.assertEqual(t.children[0].children[0].type, syms.print_stmt)
+        self.assertEqual(t.children[1].children[0].type, syms.print_stmt)
 
 
 class GrammarTest(support.TestCase):
diff --git a/lib-python/2.7/lib2to3/tests/test_refactor.py b/lib-python/2.7/lib2to3/tests/test_refactor.py
--- a/lib-python/2.7/lib2to3/tests/test_refactor.py
+++ b/lib-python/2.7/lib2to3/tests/test_refactor.py
@@ -223,6 +223,7 @@
                 "hi.py",
                 ".dumb",
                 ".after.py",
+                "notpy.npy",
                 "sappy"]
         expected = ["hi.py"]
         check(tree, expected)
diff --git a/lib-python/2.7/lib2to3/tests/test_util.py b/lib-python/2.7/lib2to3/tests/test_util.py
--- a/lib-python/2.7/lib2to3/tests/test_util.py
+++ b/lib-python/2.7/lib2to3/tests/test_util.py
@@ -568,8 +568,8 @@
 
     def test_from_import(self):
         node = parse('bar()')
-        fixer_util.touch_import("cgi", "escape", node)
-        self.assertEqual(str(node), 'from cgi import escape\nbar()\n\n')
+        fixer_util.touch_import("html", "escape", node)
+        self.assertEqual(str(node), 'from html import escape\nbar()\n\n')
 
     def test_name_import(self):
         node = parse('bar()')
diff --git a/lib-python/2.7/locale.py b/lib-python/2.7/locale.py
--- a/lib-python/2.7/locale.py
+++ b/lib-python/2.7/locale.py
@@ -621,7 +621,7 @@
     'tactis':                       'TACTIS',
     'euc_jp':                       'eucJP',
     'euc_kr':                       'eucKR',
-    'utf_8':                        'UTF8',
+    'utf_8':                        'UTF-8',
     'koi8_r':                       'KOI8-R',
     'koi8_u':                       'KOI8-U',
     # XXX This list is still incomplete. If you know more
diff --git a/lib-python/2.7/logging/__init__.py b/lib-python/2.7/logging/__init__.py
--- a/lib-python/2.7/logging/__init__.py
+++ b/lib-python/2.7/logging/__init__.py
@@ -1627,6 +1627,7 @@
             h = wr()
             if h:
                 try:
+                    h.acquire()
                     h.flush()
                     h.close()
                 except (IOError, ValueError):
@@ -1635,6 +1636,8 @@
                     # references to them are still around at
                     # application exit.
                     pass
+                finally:
+                    h.release()
         except:
             if raiseExceptions:
                 raise
diff --git a/lib-python/2.7/logging/config.py b/lib-python/2.7/logging/config.py
--- a/lib-python/2.7/logging/config.py
+++ b/lib-python/2.7/logging/config.py
@@ -226,14 +226,14 @@
             propagate = 1
         logger = logging.getLogger(qn)
         if qn in existing:
-            i = existing.index(qn)
+            i = existing.index(qn) + 1 # start with the entry after qn
             prefixed = qn + "."
             pflen = len(prefixed)
             num_existing = len(existing)
-            i = i + 1 # look at the entry after qn
-            while (i < num_existing) and (existing[i][:pflen] == prefixed):
-                child_loggers.append(existing[i])
-                i = i + 1
+            while i < num_existing:
+                if existing[i][:pflen] == prefixed:
+                    child_loggers.append(existing[i])
+                i += 1
             existing.remove(qn)
         if "level" in opts:
             level = cp.get(sectname, "level")
diff --git a/lib-python/2.7/logging/handlers.py b/lib-python/2.7/logging/handlers.py
--- a/lib-python/2.7/logging/handlers.py
+++ b/lib-python/2.7/logging/handlers.py
@@ -125,6 +125,7 @@
         """
         if self.stream:
             self.stream.close()
+            self.stream = None
         if self.backupCount > 0:
             for i in range(self.backupCount - 1, 0, -1):
                 sfn = "%s.%d" % (self.baseFilename, i)
@@ -324,6 +325,7 @@
         """
         if self.stream:
             self.stream.close()
+            self.stream = None
         # get the time that this sequence started at and make it a TimeTuple
         t = self.rolloverAt - self.interval
         if self.utc:
diff --git a/lib-python/2.7/mailbox.py b/lib-python/2.7/mailbox.py
--- a/lib-python/2.7/mailbox.py
+++ b/lib-python/2.7/mailbox.py
@@ -234,27 +234,35 @@
     def __init__(self, dirname, factory=rfc822.Message, create=True):
         """Initialize a Maildir instance."""
         Mailbox.__init__(self, dirname, factory, create)
+        self._paths = {
+            'tmp': os.path.join(self._path, 'tmp'),
+            'new': os.path.join(self._path, 'new'),
+            'cur': os.path.join(self._path, 'cur'),
+            }
         if not os.path.exists(self._path):
             if create:
                 os.mkdir(self._path, 0700)
-                os.mkdir(os.path.join(self._path, 'tmp'), 0700)
-                os.mkdir(os.path.join(self._path, 'new'), 0700)
-                os.mkdir(os.path.join(self._path, 'cur'), 0700)
+                for path in self._paths.values():
+                    os.mkdir(path, 0o700)
             else:
                 raise NoSuchMailboxError(self._path)
         self._toc = {}
-        self._last_read = None          # Records last time we read cur/new
-        # NOTE: we manually invalidate _last_read each time we do any
-        # modifications ourselves, otherwise we might get tripped up by
-        # bogus mtime behaviour on some systems (see issue #6896).
+        self._toc_mtimes = {}
+        for subdir in ('cur', 'new'):
+            self._toc_mtimes[subdir] = os.path.getmtime(self._paths[subdir])
+        self._last_read = time.time()  # Records last time we read cur/new
+        self._skewfactor = 0.1         # Adjust if os/fs clocks are skewing
 
     def add(self, message):
         """Add message and return assigned key."""
         tmp_file = self._create_tmp()
         try:
             self._dump_message(message, tmp_file)
-        finally:
-            _sync_close(tmp_file)
+        except BaseException:
+            tmp_file.close()
+            os.remove(tmp_file.name)
+            raise
+        _sync_close(tmp_file)
         if isinstance(message, MaildirMessage):
             subdir = message.get_subdir()
             suffix = self.colon + message.get_info()
@@ -280,15 +288,11 @@
                 raise
         if isinstance(message, MaildirMessage):
             os.utime(dest, (os.path.getatime(dest), message.get_date()))
-        # Invalidate cached toc
-        self._last_read = None
         return uniq
 
     def remove(self, key):
         """Remove the keyed message; raise KeyError if it doesn't exist."""
         os.remove(os.path.join(self._path, self._lookup(key)))
-        # Invalidate cached toc (only on success)
-        self._last_read = None
 
     def discard(self, key):
         """If the keyed message exists, remove it."""
@@ -323,8 +327,6 @@
         if isinstance(message, MaildirMessage):
             os.utime(new_path, (os.path.getatime(new_path),
                                 message.get_date()))
-        # Invalidate cached toc
-        self._last_read = None
 
     def get_message(self, key):
         """Return a Message representation or raise a KeyError."""
@@ -380,8 +382,8 @@
     def flush(self):
         """Write any pending changes to disk."""
         # Maildir changes are always written immediately, so there's nothing
-        # to do except invalidate our cached toc.
-        self._last_read = None
+        # to do.
+        pass
 
     def lock(self):
         """Lock the mailbox."""
@@ -479,36 +481,39 @@
 
     def _refresh(self):
         """Update table of contents mapping."""
-        if self._last_read is not None:
-            for subdir in ('new', 'cur'):
-                mtime = os.path.getmtime(os.path.join(self._path, subdir))
-                if mtime > self._last_read:
-                    break
-            else:
+        # If it has been less than two seconds since the last _refresh() call,
+        # we have to unconditionally re-read the mailbox just in case it has
+        # been modified, because os.path.mtime() has a 2 sec resolution in the
+        # most common worst case (FAT) and a 1 sec resolution typically.  This
+        # results in a few unnecessary re-reads when _refresh() is called
+        # multiple times in that interval, but once the clock ticks over, we
+        # will only re-read as needed.  Because the filesystem might be being
+        # served by an independent system with its own clock, we record and
+        # compare with the mtimes from the filesystem.  Because the other
+        # system's clock might be skewing relative to our clock, we add an
+        # extra delta to our wait.  The default is one tenth second, but is an
+        # instance variable and so can be adjusted if dealing with a
+        # particularly skewed or irregular system.
+        if time.time() - self._last_read > 2 + self._skewfactor:
+            refresh = False
+            for subdir in self._toc_mtimes:
+                mtime = os.path.getmtime(self._paths[subdir])
+                if mtime > self._toc_mtimes[subdir]:
+                    refresh = True
+                self._toc_mtimes[subdir] = mtime
+            if not refresh:
                 return
-
-        # We record the current time - 1sec so that, if _refresh() is called
-        # again in the same second, we will always re-read the mailbox
-        # just in case it's been modified.  (os.path.mtime() only has
-        # 1sec resolution.)  This results in a few unnecessary re-reads
-        # when _refresh() is called multiple times in the same second,
-        # but once the clock ticks over, we will only re-read as needed.
-        now = time.time() - 1
-
+        # Refresh toc
         self._toc = {}
-        def update_dir (subdir):
-            path = os.path.join(self._path, subdir)
+        for subdir in self._toc_mtimes:
+            path = self._paths[subdir]
             for entry in os.listdir(path):
                 p = os.path.join(path, entry)
                 if os.path.isdir(p):
                     continue
                 uniq = entry.split(self.colon)[0]
                 self._toc[uniq] = os.path.join(subdir, entry)
-
-        update_dir('new')
-        update_dir('cur')
-
-        self._last_read = now
+        self._last_read = time.time()
 
     def _lookup(self, key):
         """Use TOC to return subpath for given key, or raise a KeyError."""
@@ -551,7 +556,7 @@
                     f = open(self._path, 'wb+')
                 else:
                     raise NoSuchMailboxError(self._path)
-            elif e.errno == errno.EACCES:
+            elif e.errno in (errno.EACCES, errno.EROFS):
                 f = open(self._path, 'rb')
             else:
                 raise
@@ -700,9 +705,14 @@
     def _append_message(self, message):
         """Append message to mailbox and return (start, stop) offsets."""
         self._file.seek(0, 2)
-        self._pre_message_hook(self._file)
-        offsets = self._install_message(message)
-        self._post_message_hook(self._file)
+        before = self._file.tell()
+        try:
+            self._pre_message_hook(self._file)
+            offsets = self._install_message(message)
+            self._post_message_hook(self._file)
+        except BaseException:
+            self._file.truncate(before)
+            raise
         self._file.flush()
         self._file_length = self._file.tell()  # Record current length of mailbox
         return offsets
@@ -868,18 +878,29 @@
             new_key = max(keys) + 1
         new_path = os.path.join(self._path, str(new_key))
         f = _create_carefully(new_path)
+        closed = False
         try:
             if self._locked:
                 _lock_file(f)
             try:
-                self._dump_message(message, f)
+                try:
+                    self._dump_message(message, f)
+                except BaseException:
+                    # Unlock and close so it can be deleted on Windows
+                    if self._locked:
+                        _unlock_file(f)
+                    _sync_close(f)
+                    closed = True
+                    os.remove(new_path)
+                    raise
                 if isinstance(message, MHMessage):
                     self._dump_sequences(message, new_key)
             finally:
                 if self._locked:
                     _unlock_file(f)
         finally:
-            _sync_close(f)
+            if not closed:
+                _sync_close(f)
         return new_key
 
     def remove(self, key):
@@ -1886,7 +1907,7 @@
             try:
                 fcntl.lockf(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
             except IOError, e:
-                if e.errno in (errno.EAGAIN, errno.EACCES):
+                if e.errno in (errno.EAGAIN, errno.EACCES, errno.EROFS):
                     raise ExternalClashError('lockf: lock unavailable: %s' %
                                              f.name)
                 else:
@@ -1896,7 +1917,7 @@
                 pre_lock = _create_temporary(f.name + '.lock')
                 pre_lock.close()
             except IOError, e:
-                if e.errno == errno.EACCES:
+                if e.errno in (errno.EACCES, errno.EROFS):
                     return  # Without write access, just skip dotlocking.
                 else:
                     raise
diff --git a/lib-python/2.7/msilib/__init__.py b/lib-python/2.7/msilib/__init__.py
--- a/lib-python/2.7/msilib/__init__.py
+++ b/lib-python/2.7/msilib/__init__.py
@@ -173,11 +173,10 @@
         add_data(db, table, getattr(module, table))
 
 def make_id(str):
-    #str = str.replace(".", "_") # colons are allowed
-    str = str.replace(" ", "_")
-    str = str.replace("-", "_")
-    if str[0] in string.digits:
-        str = "_"+str
+    identifier_chars = string.ascii_letters + string.digits + "._"
+    str = "".join([c if c in identifier_chars else "_" for c in str])
+    if str[0] in (string.digits + "."):
+        str = "_" + str
     assert re.match("^[A-Za-z_][A-Za-z0-9_.]*$", str), "FILE"+str
     return str
 
@@ -285,19 +284,28 @@
                         [(feature.id, component)])
 
     def make_short(self, file):
+        oldfile = file
+        file = file.replace('+', '_')
+        file = ''.join(c for c in file if not c in ' "/\[]:;=,')
         parts = file.split(".")
-        if len(parts)>1:
+        if len(parts) > 1:
+            prefix = "".join(parts[:-1]).upper()
             suffix = parts[-1].upper()
+            if not prefix:
+                prefix = suffix
+                suffix = None
         else:
+            prefix = file.upper()
             suffix = None
-        prefix = parts[0].upper()
-        if len(prefix) <= 8 and (not suffix or len(suffix)<=3):
+        if len(parts) < 3 and len(prefix) <= 8 and file == oldfile and (
+                                                not suffix or len(suffix) <= 3):
             if suffix:
                 file = prefix+"."+suffix
             else:
                 file = prefix
-            assert file not in self.short_names
         else:
+            file = None
+        if file is None or file in self.short_names:
             prefix = prefix[:6]
             if suffix:
                 suffix = suffix[:3]
diff --git a/lib-python/2.7/multiprocessing/__init__.py b/lib-python/2.7/multiprocessing/__init__.py
--- a/lib-python/2.7/multiprocessing/__init__.py
+++ b/lib-python/2.7/multiprocessing/__init__.py
@@ -38,6 +38,7 @@
 # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
 #
 
 __version__ = '0.70a1'
@@ -115,8 +116,11 @@
         except (ValueError, KeyError):
             num = 0
     elif 'bsd' in sys.platform or sys.platform == 'darwin':
+        comm = '/sbin/sysctl -n hw.ncpu'
+        if sys.platform == 'darwin':
+            comm = '/usr' + comm
         try:
-            with os.popen('sysctl -n hw.ncpu') as p:
+            with os.popen(comm) as p:
                 num = int(p.read())
         except ValueError:
             num = 0
diff --git a/lib-python/2.7/multiprocessing/connection.py b/lib-python/2.7/multiprocessing/connection.py
--- a/lib-python/2.7/multiprocessing/connection.py
+++ b/lib-python/2.7/multiprocessing/connection.py
@@ -3,7 +3,33 @@
 #
 # multiprocessing/connection.py
 #
-# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt
+# Copyright (c) 2006-2008, R Oudkerk
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+# 3. Neither the name of author nor the names of any contributors may be
+#    used to endorse or promote products derived from this software
+#    without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
 #
 
 __all__ = [ 'Client', 'Listener', 'Pipe' ]
diff --git a/lib-python/2.7/multiprocessing/dummy/__init__.py b/lib-python/2.7/multiprocessing/dummy/__init__.py
--- a/lib-python/2.7/multiprocessing/dummy/__init__.py
+++ b/lib-python/2.7/multiprocessing/dummy/__init__.py
@@ -3,7 +3,33 @@
 #
 # multiprocessing/dummy/__init__.py
 #
-# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt
+# Copyright (c) 2006-2008, R Oudkerk
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+# 3. Neither the name of author nor the names of any contributors may be
+#    used to endorse or promote products derived from this software
+#    without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
 #
 
 __all__ = [
diff --git a/lib-python/2.7/multiprocessing/dummy/connection.py b/lib-python/2.7/multiprocessing/dummy/connection.py
--- a/lib-python/2.7/multiprocessing/dummy/connection.py
+++ b/lib-python/2.7/multiprocessing/dummy/connection.py
@@ -3,7 +3,33 @@
 #
 # multiprocessing/dummy/connection.py
 #
-# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt
+# Copyright (c) 2006-2008, R Oudkerk
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+# 3. Neither the name of author nor the names of any contributors may be
+#    used to endorse or promote products derived from this software
+#    without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
 #
 
 __all__ = [ 'Client', 'Listener', 'Pipe' ]
diff --git a/lib-python/2.7/multiprocessing/forking.py b/lib-python/2.7/multiprocessing/forking.py
--- a/lib-python/2.7/multiprocessing/forking.py
+++ b/lib-python/2.7/multiprocessing/forking.py
@@ -3,7 +3,33 @@
 #
 # multiprocessing/forking.py
 #
-# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt
+# Copyright (c) 2006-2008, R Oudkerk
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+# 3. Neither the name of author nor the names of any contributors may be
+#    used to endorse or promote products derived from this software
+#    without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
 #
 
 import os
@@ -172,6 +198,7 @@
 
     TERMINATE = 0x10000
     WINEXE = (sys.platform == 'win32' and getattr(sys, 'frozen', False))
+    WINSERVICE = sys.executable.lower().endswith("pythonservice.exe")
 
     exit = win32.ExitProcess
     close = win32.CloseHandle
@@ -181,7 +208,7 @@
     # People embedding Python want to modify it.
     #
 
-    if sys.executable.lower().endswith('pythonservice.exe'):
+    if WINSERVICE:
         _python_exe = os.path.join(sys.exec_prefix, 'python.exe')
     else:
         _python_exe = sys.executable
@@ -371,7 +398,7 @@
         if _logger is not None:
             d['log_level'] = _logger.getEffectiveLevel()
 
-        if not WINEXE:
+        if not WINEXE and not WINSERVICE:
             main_path = getattr(sys.modules['__main__'], '__file__', None)
             if not main_path and sys.argv[0] not in ('', '-c'):
                 main_path = sys.argv[0]
diff --git a/lib-python/2.7/multiprocessing/heap.py b/lib-python/2.7/multiprocessing/heap.py
--- a/lib-python/2.7/multiprocessing/heap.py
+++ b/lib-python/2.7/multiprocessing/heap.py
@@ -3,7 +3,33 @@
 #
 # multiprocessing/heap.py
 #
-# Copyright (c) 2007-2008, R Oudkerk --- see COPYING.txt
+# Copyright (c) 2006-2008, R Oudkerk
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+# 3. Neither the name of author nor the names of any contributors may be
+#    used to endorse or promote products derived from this software
+#    without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
 #
 
 import bisect
diff --git a/lib-python/2.7/multiprocessing/managers.py b/lib-python/2.7/multiprocessing/managers.py
--- a/lib-python/2.7/multiprocessing/managers.py
+++ b/lib-python/2.7/multiprocessing/managers.py
@@ -4,7 +4,33 @@
 #
 # multiprocessing/managers.py
 #
-# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt
+# Copyright (c) 2006-2008, R Oudkerk
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+# 3. Neither the name of author nor the names of any contributors may be
+#    used to endorse or promote products derived from this software
+#    without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
 #
 
 __all__ = [ 'BaseManager', 'SyncManager', 'BaseProxy', 'Token' ]
diff --git a/lib-python/2.7/multiprocessing/pool.py b/lib-python/2.7/multiprocessing/pool.py
--- a/lib-python/2.7/multiprocessing/pool.py
+++ b/lib-python/2.7/multiprocessing/pool.py
@@ -3,7 +3,33 @@
 #
 # multiprocessing/pool.py
 #
-# Copyright (c) 2007-2008, R Oudkerk --- see COPYING.txt
+# Copyright (c) 2006-2008, R Oudkerk
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+# 3. Neither the name of author nor the names of any contributors may be
+#    used to endorse or promote products derived from this software
+#    without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
 #
 
 __all__ = ['Pool']
@@ -269,6 +295,8 @@
         while pool._worker_handler._state == RUN and pool._state == RUN:
             pool._maintain_pool()
             time.sleep(0.1)
+        # send sentinel to stop workers
+        pool._taskqueue.put(None)
         debug('worker handler exiting')
 
     @staticmethod
@@ -387,7 +415,6 @@
         if self._state == RUN:
             self._state = CLOSE
             self._worker_handler._state = CLOSE
-            self._taskqueue.put(None)
 
     def terminate(self):
         debug('terminating pool')
@@ -421,7 +448,6 @@
 
         worker_handler._state = TERMINATE
         task_handler._state = TERMINATE
-        taskqueue.put(None)                 # sentinel
 
         debug('helping task handler/workers to finish')
         cls._help_stuff_finish(inqueue, task_handler, len(pool))
@@ -431,6 +457,11 @@
         result_handler._state = TERMINATE
         outqueue.put(None)                  # sentinel
 
+        # We must wait for the worker handler to exit before terminating
+        # workers because we don't want workers to be restarted behind our back.
+        debug('joining worker handler')
+        worker_handler.join()
+
         # Terminate workers which haven't already finished.
         if pool and hasattr(pool[0], 'terminate'):
             debug('terminating workers')
diff --git a/lib-python/2.7/multiprocessing/process.py b/lib-python/2.7/multiprocessing/process.py
--- a/lib-python/2.7/multiprocessing/process.py
+++ b/lib-python/2.7/multiprocessing/process.py
@@ -3,7 +3,33 @@
 #
 # multiprocessing/process.py
 #
-# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt
+# Copyright (c) 2006-2008, R Oudkerk
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+# 3. Neither the name of author nor the names of any contributors may be
+#    used to endorse or promote products derived from this software
+#    without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
 #
 
 __all__ = ['Process', 'current_process', 'active_children']
diff --git a/lib-python/2.7/multiprocessing/queues.py b/lib-python/2.7/multiprocessing/queues.py
--- a/lib-python/2.7/multiprocessing/queues.py
+++ b/lib-python/2.7/multiprocessing/queues.py
@@ -3,7 +3,33 @@
 #
 # multiprocessing/queues.py
 #
-# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt
+# Copyright (c) 2006-2008, R Oudkerk
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+# 3. Neither the name of author nor the names of any contributors may be
+#    used to endorse or promote products derived from this software
+#    without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
 #
 
 __all__ = ['Queue', 'SimpleQueue', 'JoinableQueue']
diff --git a/lib-python/2.7/multiprocessing/reduction.py b/lib-python/2.7/multiprocessing/reduction.py
--- a/lib-python/2.7/multiprocessing/reduction.py
+++ b/lib-python/2.7/multiprocessing/reduction.py
@@ -4,7 +4,33 @@
 #
 # multiprocessing/reduction.py
 #
-# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt
+# Copyright (c) 2006-2008, R Oudkerk
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+# 3. Neither the name of author nor the names of any contributors may be
+#    used to endorse or promote products derived from this software
+#    without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
 #
 
 __all__ = []
diff --git a/lib-python/2.7/multiprocessing/sharedctypes.py b/lib-python/2.7/multiprocessing/sharedctypes.py
--- a/lib-python/2.7/multiprocessing/sharedctypes.py
+++ b/lib-python/2.7/multiprocessing/sharedctypes.py
@@ -3,7 +3,33 @@
 #
 # multiprocessing/sharedctypes.py
 #
-# Copyright (c) 2007-2008, R Oudkerk --- see COPYING.txt
+# Copyright (c) 2006-2008, R Oudkerk
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+# 3. Neither the name of author nor the names of any contributors may be
+#    used to endorse or promote products derived from this software
+#    without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
 #
 
 import sys
@@ -52,9 +78,11 @@
     Returns a ctypes array allocated from shared memory
     '''
     type_ = typecode_to_type.get(typecode_or_type, typecode_or_type)
-    if isinstance(size_or_initializer, int):
+    if isinstance(size_or_initializer, (int, long)):
         type_ = type_ * size_or_initializer
-        return _new_value(type_)
+        obj = _new_value(type_)
+        ctypes.memset(ctypes.addressof(obj), 0, ctypes.sizeof(obj))
+        return obj
     else:
         type_ = type_ * len(size_or_initializer)
         result = _new_value(type_)
diff --git a/lib-python/2.7/multiprocessing/synchronize.py b/lib-python/2.7/multiprocessing/synchronize.py
--- a/lib-python/2.7/multiprocessing/synchronize.py
+++ b/lib-python/2.7/multiprocessing/synchronize.py
@@ -3,7 +3,33 @@
 #
 # multiprocessing/synchronize.py
 #
-# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt
+# Copyright (c) 2006-2008, R Oudkerk
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+# 3. Neither the name of author nor the names of any contributors may be
+#    used to endorse or promote products derived from this software
+#    without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
 #
 
 __all__ = [
diff --git a/lib-python/2.7/multiprocessing/util.py b/lib-python/2.7/multiprocessing/util.py
--- a/lib-python/2.7/multiprocessing/util.py
+++ b/lib-python/2.7/multiprocessing/util.py
@@ -3,7 +3,33 @@
 #
 # multiprocessing/util.py
 #
-# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt
+# Copyright (c) 2006-2008, R Oudkerk
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+# 3. Neither the name of author nor the names of any contributors may be
+#    used to endorse or promote products derived from this software
+#    without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
 #
 
 import itertools
diff --git a/lib-python/2.7/netrc.py b/lib-python/2.7/netrc.py
--- a/lib-python/2.7/netrc.py
+++ b/lib-python/2.7/netrc.py
@@ -34,11 +34,19 @@
     def _parse(self, file, fp):
         lexer = shlex.shlex(fp)
         lexer.wordchars += r"""!"#$%&'()*+,-./:;<=>?@[\]^_`{|}~"""
+        lexer.commenters = lexer.commenters.replace('#', '')
         while 1:
             # Look for a machine, default, or macdef top-level keyword
             toplevel = tt = lexer.get_token()
             if not tt:
                 break
+            elif tt[0] == '#':
+                # seek to beginning of comment, in case reading the token put
+                # us on a new line, and then skip the rest of the line.
+                pos = len(tt) + 1
+                lexer.instream.seek(-pos, 1)
+                lexer.instream.readline()
+                continue
             elif tt == 'machine':
                 entryname = lexer.get_token()
             elif tt == 'default':
@@ -64,8 +72,8 @@
             self.hosts[entryname] = {}
             while 1:
                 tt = lexer.get_token()
-                if (tt=='' or tt == 'machine' or
-                    tt == 'default' or tt =='macdef'):
+                if (tt.startswith('#') or
+                    tt in {'', 'machine', 'default', 'macdef'}):
                     if password:
                         self.hosts[entryname] = (login, account, password)
                         lexer.push_token(tt)
diff --git a/lib-python/2.7/nntplib.py b/lib-python/2.7/nntplib.py
--- a/lib-python/2.7/nntplib.py
+++ b/lib-python/2.7/nntplib.py
@@ -103,7 +103,7 @@
 
         readermode is sometimes necessary if you are connecting to an
         NNTP server on the local machine and intend to call
-        reader-specific comamnds, such as `group'.  If you get
+        reader-specific commands, such as `group'.  If you get
         unexpected NNTPPermanentErrors, you might need to set
         readermode.
         """
diff --git a/lib-python/2.7/ntpath.py b/lib-python/2.7/ntpath.py
--- a/lib-python/2.7/ntpath.py
+++ b/lib-python/2.7/ntpath.py
@@ -310,7 +310,7 @@
 #       - $varname is accepted.
 #       - %varname% is accepted.
 #       - varnames can be made out of letters, digits and the characters '_-'
-#         (though is not verifed in the ${varname} and %varname% cases)
+#         (though is not verified in the ${varname} and %varname% cases)
 # XXX With COMMAND.COM you can use any characters in a variable name,
 # XXX except '^|<>='.
 
diff --git a/lib-python/2.7/nturl2path.py b/lib-python/2.7/nturl2path.py
--- a/lib-python/2.7/nturl2path.py
+++ b/lib-python/2.7/nturl2path.py
@@ -25,11 +25,14 @@
         error = 'Bad URL: ' + url
         raise IOError, error
     drive = comp[0][-1].upper()
+    path = drive + ':'
     components = comp[1].split('/')
-    path = drive + ':'
-    for  comp in components:
+    for comp in components:
         if comp:
             path = path + '\\' + urllib.unquote(comp)
+    # Issue #11474: url like '/C|/' should convert into 'C:\\'
+    if path.endswith(':') and url.endswith('/'):
+        path += '\\'
     return path
 
 def pathname2url(p):
diff --git a/lib-python/2.7/numbers.py b/lib-python/2.7/numbers.py
--- a/lib-python/2.7/numbers.py
+++ b/lib-python/2.7/numbers.py
@@ -63,7 +63,7 @@
 
     @abstractproperty
     def imag(self):
-        """Retrieve the real component of this number.
+        """Retrieve the imaginary component of this number.
 
         This should subclass Real.
         """
diff --git a/lib-python/2.7/optparse.py b/lib-python/2.7/optparse.py
--- a/lib-python/2.7/optparse.py
+++ b/lib-python/2.7/optparse.py
@@ -1131,6 +1131,11 @@
       prog : string
         the name of the current program (to override
         os.path.basename(sys.argv[0])).
+      description : string
+        A paragraph of text giving a brief overview of your program.
+        optparse reformats this paragraph to fit the current terminal
+        width and prints it when the user requests help (after usage,
+        but before the list of options).
       epilog : string
         paragraph of help text to print after option help
 
diff --git a/lib-python/2.7/pickletools.py b/lib-python/2.7/pickletools.py
--- a/lib-python/2.7/pickletools.py
+++ b/lib-python/2.7/pickletools.py
@@ -1370,7 +1370,7 @@
       proto=0,
       doc="""Read an object from the memo and push it on the stack.
 
-      The index of the memo object to push is given by the newline-teriminated
+      The index of the memo object to push is given by the newline-terminated
       decimal string following.  BINGET and LONG_BINGET are space-optimized
       versions.
       """),
diff --git a/lib-python/2.7/pkgutil.py b/lib-python/2.7/pkgutil.py
--- a/lib-python/2.7/pkgutil.py
+++ b/lib-python/2.7/pkgutil.py
@@ -11,7 +11,7 @@
 
 __all__ = [
     'get_importer', 'iter_importers', 'get_loader', 'find_loader',
-    'walk_packages', 'iter_modules',
+    'walk_packages', 'iter_modules', 'get_data',
     'ImpImporter', 'ImpLoader', 'read_code', 'extend_path',
 ]
 
diff --git a/lib-python/2.7/platform.py b/lib-python/2.7/platform.py
--- a/lib-python/2.7/platform.py
+++ b/lib-python/2.7/platform.py
@@ -503,7 +503,7 @@
             info = pipe.read()
             if pipe.close():
                 raise os.error,'command failed'
-            # XXX How can I supress shell errors from being written
+            # XXX How can I suppress shell errors from being written
             #     to stderr ?
         except os.error,why:
             #print 'Command %s failed: %s' % (cmd,why)
@@ -1448,9 +1448,10 @@
     """ Returns a string identifying the Python implementation.
 
         Currently, the following implementations are identified:
-        'CPython' (C implementation of Python),
-        'IronPython' (.NET implementation of Python),
-        'Jython' (Java implementation of Python).
+          'CPython' (C implementation of Python),
+          'IronPython' (.NET implementation of Python),
+          'Jython' (Java implementation of Python),
+          'PyPy' (Python implementation of Python).
 
     """
     return _sys_version()[0]
diff --git a/lib-python/2.7/pydoc.py b/lib-python/2.7/pydoc.py
--- a/lib-python/2.7/pydoc.py
+++ b/lib-python/2.7/pydoc.py
@@ -156,7 +156,7 @@
             no.append(x)
     return yes, no
 
-def visiblename(name, all=None):
+def visiblename(name, all=None, obj=None):
     """Decide whether to show documentation on a variable."""
     # Certain special names are redundant.
     _hidden_names = ('__builtins__', '__doc__', '__file__', '__path__',
@@ -164,6 +164,9 @@
     if name in _hidden_names: return 0
     # Private names are hidden, but special names are displayed.
     if name.startswith('__') and name.endswith('__'): return 1
+    # Namedtuples have public fields and methods with a single leading underscore
+    if name.startswith('_') and hasattr(obj, '_fields'):
+        return 1
     if all is not None:
         # only document that which the programmer exported in __all__
         return name in all
@@ -475,9 +478,9 @@
     def multicolumn(self, list, format, cols=4):
         """Format a list of items into a multi-column list."""
         result = ''
-        rows = (len(list)+cols-1)/cols
+        rows = (len(list)+cols-1)//cols
         for col in range(cols):
-            result = result + '' % (100/cols)
+            result = result + '' % (100//cols)
             for i in range(rows*col, rows*col+rows):
                 if i < len(list):
                     result = result + format(list[i]) + '
\n'
@@ -627,7 +630,7 @@
             # if __all__ exists, believe it.  Otherwise use old heuristic.
             if (all is not None or
                 (inspect.getmodule(value) or object) is object):
-                if visiblename(key, all):
+                if visiblename(key, all, object):
                     classes.append((key, value))
                     cdict[key] = cdict[value] = '#' + key
         for key, value in classes:
@@ -643,13 +646,13 @@
             # if __all__ exists, believe it.  Otherwise use old heuristic.
             if (all is not None or
                 inspect.isbuiltin(value) or inspect.getmodule(value) is object):
-                if visiblename(key, all):
+                if visiblename(key, all, object):
                     funcs.append((key, value))
                     fdict[key] = '#-' + key
                     if inspect.isfunction(value): fdict[value] = fdict[key]
         data = []
         for key, value in inspect.getmembers(object, isdata):
-            if visiblename(key, all):
+            if visiblename(key, all, object):
                 data.append((key, value))
 
         doc = self.markup(getdoc(object), self.preformat, fdict, cdict)
@@ -773,7 +776,7 @@
                     push('\n')
             return attrs
 
-        attrs = filter(lambda data: visiblename(data[0]),
+        attrs = filter(lambda data: visiblename(data[0], obj=object),
                        classify_class_attrs(object))
         mdict = {}
         for key, kind, homecls, value in attrs:
@@ -1042,18 +1045,18 @@
             # if __all__ exists, believe it.  Otherwise use old heuristic.
             if (all is not None
                 or (inspect.getmodule(value) or object) is object):
-                if visiblename(key, all):
+                if visiblename(key, all, object):
                     classes.append((key, value))
         funcs = []
         for key, value in inspect.getmembers(object, inspect.isroutine):
             # if __all__ exists, believe it.  Otherwise use old heuristic.
             if (all is not None or
                 inspect.isbuiltin(value) or inspect.getmodule(value) is object):
-                if visiblename(key, all):
+                if visiblename(key, all, object):
                     funcs.append((key, value))
         data = []
         for key, value in inspect.getmembers(object, isdata):
-            if visiblename(key, all):
+            if visiblename(key, all, object):
                 data.append((key, value))
 
         modpkgs = []
@@ -1113,7 +1116,7 @@
             result = result + self.section('CREDITS', str(object.__credits__))
         return result
 
-    def docclass(self, object, name=None, mod=None):
+    def docclass(self, object, name=None, mod=None, *ignored):
         """Produce text documentation for a given class object."""
         realname = object.__name__
         name = name or realname
@@ -1186,7 +1189,7 @@
                                        name, mod, maxlen=70, doc=doc) + '\n')
             return attrs
 
-        attrs = filter(lambda data: visiblename(data[0]),
+        attrs = filter(lambda data: visiblename(data[0], obj=object),
                        classify_class_attrs(object))
         while attrs:
             if mro:
@@ -1718,8 +1721,9 @@
             return ''
         return ''
 
-    def __call__(self, request=None):
-        if request is not None:
+    _GoInteractive = object()
+    def __call__(self, request=_GoInteractive):
+        if request is not self._GoInteractive:
             self.help(request)
         else:
             self.intro()
diff --git a/lib-python/2.7/pydoc_data/topics.py b/lib-python/2.7/pydoc_data/topics.py
--- a/lib-python/2.7/pydoc_data/topics.py
+++ b/lib-python/2.7/pydoc_data/topics.py
@@ -1,16 +1,16 @@
-# Autogenerated by Sphinx on Sat Jul  3 08:52:04 2010
+# Autogenerated by Sphinx on Sat Jun 11 09:49:30 2011
 topics = {'assert': u'\nThe ``assert`` statement\n************************\n\nAssert statements are a convenient way to insert debugging assertions\ninto a program:\n\n   assert_stmt ::= "assert" expression ["," expression]\n\nThe simple form, ``assert expression``, is equivalent to\n\n   if __debug__:\n      if not expression: raise AssertionError\n\nThe extended form, ``assert expression1, expression2``, is equivalent\nto\n\n   if __debug__:\n      if not expression1: raise AssertionError(expression2)\n\nThese equivalences assume that ``__debug__`` and ``AssertionError``\nrefer to the built-in variables with those names.  In the current\nimplementation, the built-in variable ``__debug__`` is ``True`` under\nnormal circumstances, ``False`` when optimization is requested\n(command line option -O).  The current code generator emits no code\nfor an assert statement when optimization is requested at compile\ntime.  Note that it is unnecessary to include the source code for the\nexpression that failed in the error message; it will be displayed as\npart of the stack trace.\n\nAssignments to ``__debug__`` are illegal.  The value for the built-in\nvariable is determined when the interpreter starts.\n',
- 'assignment': u'\nAssignment statements\n*********************\n\nAssignment statements are used to (re)bind names to values and to\nmodify attributes or items of mutable objects:\n\n   assignment_stmt ::= (target_list "=")+ (expression_list | yield_expression)\n   target_list     ::= target ("," target)* [","]\n   target          ::= identifier\n              | "(" target_list ")"\n              | "[" target_list "]"\n              | attributeref\n              | subscription\n              | slicing\n\n(See section *Primaries* for the syntax definitions for the last three\nsymbols.)\n\nAn assignment statement evaluates the expression list (remember that\nthis can be a single expression or a comma-separated list, the latter\nyielding a tuple) and assigns the single resulting object to each of\nthe target lists, from left to right.\n\nAssignment is defined recursively depending on the form of the target\n(list). When a target is part of a mutable object (an attribute\nreference, subscription or slicing), the mutable object must\nultimately perform the assignment and decide about its validity, and\nmay raise an exception if the assignment is unacceptable.  The rules\nobserved by various types and the exceptions raised are given with the\ndefinition of the object types (see section *The standard type\nhierarchy*).\n\nAssignment of an object to a target list is recursively defined as\nfollows.\n\n* If the target list is a single target: The object is assigned to\n  that target.\n\n* If the target list is a comma-separated list of targets: The object\n  must be an iterable with the same number of items as there are\n  targets in the target list, and the items are assigned, from left to\n  right, to the corresponding targets. (This rule is relaxed as of\n  Python 1.5; in earlier versions, the object had to be a tuple.\n  Since strings are sequences, an assignment like ``a, b = "xy"`` is\n  now legal as long as the string has the right length.)\n\nAssignment of an object to a single target is recursively defined as\nfollows.\n\n* If the target is an identifier (name):\n\n  * If the name does not occur in a ``global`` statement in the\n    current code block: the name is bound to the object in the current\n    local namespace.\n\n  * Otherwise: the name is bound to the object in the current global\n    namespace.\n\n  The name is rebound if it was already bound.  This may cause the\n  reference count for the object previously bound to the name to reach\n  zero, causing the object to be deallocated and its destructor (if it\n  has one) to be called.\n\n* If the target is a target list enclosed in parentheses or in square\n  brackets: The object must be an iterable with the same number of\n  items as there are targets in the target list, and its items are\n  assigned, from left to right, to the corresponding targets.\n\n* If the target is an attribute reference: The primary expression in\n  the reference is evaluated.  It should yield an object with\n  assignable attributes; if this is not the case, ``TypeError`` is\n  raised.  That object is then asked to assign the assigned object to\n  the given attribute; if it cannot perform the assignment, it raises\n  an exception (usually but not necessarily ``AttributeError``).\n\n  Note: If the object is a class instance and the attribute reference\n  occurs on both sides of the assignment operator, the RHS expression,\n  ``a.x`` can access either an instance attribute or (if no instance\n  attribute exists) a class attribute.  The LHS target ``a.x`` is\n  always set as an instance attribute, creating it if necessary.\n  Thus, the two occurrences of ``a.x`` do not necessarily refer to the\n  same attribute: if the RHS expression refers to a class attribute,\n  the LHS creates a new instance attribute as the target of the\n  assignment:\n\n     class Cls:\n         x = 3             # class variable\n     inst = Cls()\n     inst.x = inst.x + 1   # writes inst.x as 4 leaving Cls.x as 3\n\n  This description does not necessarily apply to descriptor\n  attributes, such as properties created with ``property()``.\n\n* If the target is a subscription: The primary expression in the\n  reference is evaluated.  It should yield either a mutable sequence\n  object (such as a list) or a mapping object (such as a dictionary).\n  Next, the subscript expression is evaluated.\n\n  If the primary is a mutable sequence object (such as a list), the\n  subscript must yield a plain integer.  If it is negative, the\n  sequence\'s length is added to it. The resulting value must be a\n  nonnegative integer less than the sequence\'s length, and the\n  sequence is asked to assign the assigned object to its item with\n  that index.  If the index is out of range, ``IndexError`` is raised\n  (assignment to a subscripted sequence cannot add new items to a\n  list).\n\n  If the primary is a mapping object (such as a dictionary), the\n  subscript must have a type compatible with the mapping\'s key type,\n  and the mapping is then asked to create a key/datum pair which maps\n  the subscript to the assigned object.  This can either replace an\n  existing key/value pair with the same key value, or insert a new\n  key/value pair (if no key with the same value existed).\n\n* If the target is a slicing: The primary expression in the reference\n  is evaluated.  It should yield a mutable sequence object (such as a\n  list).  The assigned object should be a sequence object of the same\n  type.  Next, the lower and upper bound expressions are evaluated,\n  insofar they are present; defaults are zero and the sequence\'s\n  length.  The bounds should evaluate to (small) integers.  If either\n  bound is negative, the sequence\'s length is added to it. The\n  resulting bounds are clipped to lie between zero and the sequence\'s\n  length, inclusive.  Finally, the sequence object is asked to replace\n  the slice with the items of the assigned sequence.  The length of\n  the slice may be different from the length of the assigned sequence,\n  thus changing the length of the target sequence, if the object\n  allows it.\n\n**CPython implementation detail:** In the current implementation, the\nsyntax for targets is taken to be the same as for expressions, and\ninvalid syntax is rejected during the code generation phase, causing\nless detailed error messages.\n\nWARNING: Although the definition of assignment implies that overlaps\nbetween the left-hand side and the right-hand side are \'safe\' (for\nexample ``a, b = b, a`` swaps two variables), overlaps *within* the\ncollection of assigned-to variables are not safe!  For instance, the\nfollowing program prints ``[0, 2]``:\n\n   x = [0, 1]\n   i = 0\n   i, x[i] = 1, 2\n   print x\n\n\nAugmented assignment statements\n===============================\n\nAugmented assignment is the combination, in a single statement, of a\nbinary operation and an assignment statement:\n\n   augmented_assignment_stmt ::= augtarget augop (expression_list | yield_expression)\n   augtarget                 ::= identifier | attributeref | subscription | slicing\n   augop                     ::= "+=" | "-=" | "*=" | "/=" | "//=" | "%=" | "**="\n             | ">>=" | "<<=" | "&=" | "^=" | "|="\n\n(See section *Primaries* for the syntax definitions for the last three\nsymbols.)\n\nAn augmented assignment evaluates the target (which, unlike normal\nassignment statements, cannot be an unpacking) and the expression\nlist, performs the binary operation specific to the type of assignment\non the two operands, and assigns the result to the original target.\nThe target is only evaluated once.\n\nAn augmented assignment expression like ``x += 1`` can be rewritten as\n``x = x + 1`` to achieve a similar, but not exactly equal effect. In\nthe augmented version, ``x`` is only evaluated once. Also, when\npossible, the actual operation is performed *in-place*, meaning that\nrather than creating a new object and assigning that to the target,\nthe old object is modified instead.\n\nWith the exception of assigning to tuples and multiple targets in a\nsingle statement, the assignment done by augmented assignment\nstatements is handled the same way as normal assignments. Similarly,\nwith the exception of the possible *in-place* behavior, the binary\noperation performed by augmented assignment is the same as the normal\nbinary operations.\n\nFor targets which are attribute references, the same *caveat about\nclass and instance attributes* applies as for regular assignments.\n',
+ 'assignment': u'\nAssignment statements\n*********************\n\nAssignment statements are used to (re)bind names to values and to\nmodify attributes or items of mutable objects:\n\n   assignment_stmt ::= (target_list "=")+ (expression_list | yield_expression)\n   target_list     ::= target ("," target)* [","]\n   target          ::= identifier\n              | "(" target_list ")"\n              | "[" target_list "]"\n              | attributeref\n              | subscription\n              | slicing\n\n(See section *Primaries* for the syntax definitions for the last three\nsymbols.)\n\nAn assignment statement evaluates the expression list (remember that\nthis can be a single expression or a comma-separated list, the latter\nyielding a tuple) and assigns the single resulting object to each of\nthe target lists, from left to right.\n\nAssignment is defined recursively depending on the form of the target\n(list). When a target is part of a mutable object (an attribute\nreference, subscription or slicing), the mutable object must\nultimately perform the assignment and decide about its validity, and\nmay raise an exception if the assignment is unacceptable.  The rules\nobserved by various types and the exceptions raised are given with the\ndefinition of the object types (see section *The standard type\nhierarchy*).\n\nAssignment of an object to a target list is recursively defined as\nfollows.\n\n* If the target list is a single target: The object is assigned to\n  that target.\n\n* If the target list is a comma-separated list of targets: The object\n  must be an iterable with the same number of items as there are\n  targets in the target list, and the items are assigned, from left to\n  right, to the corresponding targets.\n\nAssignment of an object to a single target is recursively defined as\nfollows.\n\n* If the target is an identifier (name):\n\n  * If the name does not occur in a ``global`` statement in the\n    current code block: the name is bound to the object in the current\n    local namespace.\n\n  * Otherwise: the name is bound to the object in the current global\n    namespace.\n\n  The name is rebound if it was already bound.  This may cause the\n  reference count for the object previously bound to the name to reach\n  zero, causing the object to be deallocated and its destructor (if it\n  has one) to be called.\n\n* If the target is a target list enclosed in parentheses or in square\n  brackets: The object must be an iterable with the same number of\n  items as there are targets in the target list, and its items are\n  assigned, from left to right, to the corresponding targets.\n\n* If the target is an attribute reference: The primary expression in\n  the reference is evaluated.  It should yield an object with\n  assignable attributes; if this is not the case, ``TypeError`` is\n  raised.  That object is then asked to assign the assigned object to\n  the given attribute; if it cannot perform the assignment, it raises\n  an exception (usually but not necessarily ``AttributeError``).\n\n  Note: If the object is a class instance and the attribute reference\n  occurs on both sides of the assignment operator, the RHS expression,\n  ``a.x`` can access either an instance attribute or (if no instance\n  attribute exists) a class attribute.  The LHS target ``a.x`` is\n  always set as an instance attribute, creating it if necessary.\n  Thus, the two occurrences of ``a.x`` do not necessarily refer to the\n  same attribute: if the RHS expression refers to a class attribute,\n  the LHS creates a new instance attribute as the target of the\n  assignment:\n\n     class Cls:\n         x = 3             # class variable\n     inst = Cls()\n     inst.x = inst.x + 1   # writes inst.x as 4 leaving Cls.x as 3\n\n  This description does not necessarily apply to descriptor\n  attributes, such as properties created with ``property()``.\n\n* If the target is a subscription: The primary expression in the\n  reference is evaluated.  It should yield either a mutable sequence\n  object (such as a list) or a mapping object (such as a dictionary).\n  Next, the subscript expression is evaluated.\n\n  If the primary is a mutable sequence object (such as a list), the\n  subscript must yield a plain integer.  If it is negative, the\n  sequence\'s length is added to it. The resulting value must be a\n  nonnegative integer less than the sequence\'s length, and the\n  sequence is asked to assign the assigned object to its item with\n  that index.  If the index is out of range, ``IndexError`` is raised\n  (assignment to a subscripted sequence cannot add new items to a\n  list).\n\n  If the primary is a mapping object (such as a dictionary), the\n  subscript must have a type compatible with the mapping\'s key type,\n  and the mapping is then asked to create a key/datum pair which maps\n  the subscript to the assigned object.  This can either replace an\n  existing key/value pair with the same key value, or insert a new\n  key/value pair (if no key with the same value existed).\n\n* If the target is a slicing: The primary expression in the reference\n  is evaluated.  It should yield a mutable sequence object (such as a\n  list).  The assigned object should be a sequence object of the same\n  type.  Next, the lower and upper bound expressions are evaluated,\n  insofar they are present; defaults are zero and the sequence\'s\n  length.  The bounds should evaluate to (small) integers.  If either\n  bound is negative, the sequence\'s length is added to it. The\n  resulting bounds are clipped to lie between zero and the sequence\'s\n  length, inclusive.  Finally, the sequence object is asked to replace\n  the slice with the items of the assigned sequence.  The length of\n  the slice may be different from the length of the assigned sequence,\n  thus changing the length of the target sequence, if the object\n  allows it.\n\n**CPython implementation detail:** In the current implementation, the\nsyntax for targets is taken to be the same as for expressions, and\ninvalid syntax is rejected during the code generation phase, causing\nless detailed error messages.\n\nWARNING: Although the definition of assignment implies that overlaps\nbetween the left-hand side and the right-hand side are \'safe\' (for\nexample ``a, b = b, a`` swaps two variables), overlaps *within* the\ncollection of assigned-to variables are not safe!  For instance, the\nfollowing program prints ``[0, 2]``:\n\n   x = [0, 1]\n   i = 0\n   i, x[i] = 1, 2\n   print x\n\n\nAugmented assignment statements\n===============================\n\nAugmented assignment is the combination, in a single statement, of a\nbinary operation and an assignment statement:\n\n   augmented_assignment_stmt ::= augtarget augop (expression_list | yield_expression)\n   augtarget                 ::= identifier | attributeref | subscription | slicing\n   augop                     ::= "+=" | "-=" | "*=" | "/=" | "//=" | "%=" | "**="\n             | ">>=" | "<<=" | "&=" | "^=" | "|="\n\n(See section *Primaries* for the syntax definitions for the last three\nsymbols.)\n\nAn augmented assignment evaluates the target (which, unlike normal\nassignment statements, cannot be an unpacking) and the expression\nlist, performs the binary operation specific to the type of assignment\non the two operands, and assigns the result to the original target.\nThe target is only evaluated once.\n\nAn augmented assignment expression like ``x += 1`` can be rewritten as\n``x = x + 1`` to achieve a similar, but not exactly equal effect. In\nthe augmented version, ``x`` is only evaluated once. Also, when\npossible, the actual operation is performed *in-place*, meaning that\nrather than creating a new object and assigning that to the target,\nthe old object is modified instead.\n\nWith the exception of assigning to tuples and multiple targets in a\nsingle statement, the assignment done by augmented assignment\nstatements is handled the same way as normal assignments. Similarly,\nwith the exception of the possible *in-place* behavior, the binary\noperation performed by augmented assignment is the same as the normal\nbinary operations.\n\nFor targets which are attribute references, the same *caveat about\nclass and instance attributes* applies as for regular assignments.\n',
  'atom-identifiers': u'\nIdentifiers (Names)\n*******************\n\nAn identifier occurring as an atom is a name.  See section\n*Identifiers and keywords* for lexical definition and section *Naming\nand binding* for documentation of naming and binding.\n\nWhen the name is bound to an object, evaluation of the atom yields\nthat object. When a name is not bound, an attempt to evaluate it\nraises a ``NameError`` exception.\n\n**Private name mangling:** When an identifier that textually occurs in\na class definition begins with two or more underscore characters and\ndoes not end in two or more underscores, it is considered a *private\nname* of that class. Private names are transformed to a longer form\nbefore code is generated for them.  The transformation inserts the\nclass name in front of the name, with leading underscores removed, and\na single underscore inserted in front of the class name.  For example,\nthe identifier ``__spam`` occurring in a class named ``Ham`` will be\ntransformed to ``_Ham__spam``.  This transformation is independent of\nthe syntactical context in which the identifier is used.  If the\ntransformed name is extremely long (longer than 255 characters),\nimplementation defined truncation may happen.  If the class name\nconsists only of underscores, no transformation is done.\n',
  'atom-literals': u"\nLiterals\n********\n\nPython supports string literals and various numeric literals:\n\n   literal ::= stringliteral | integer | longinteger\n               | floatnumber | imagnumber\n\nEvaluation of a literal yields an object of the given type (string,\ninteger, long integer, floating point number, complex number) with the\ngiven value.  The value may be approximated in the case of floating\npoint and imaginary (complex) literals.  See section *Literals* for\ndetails.\n\nAll literals correspond to immutable data types, and hence the\nobject's identity is less important than its value.  Multiple\nevaluations of literals with the same value (either the same\noccurrence in the program text or a different occurrence) may obtain\nthe same object or a different object with the same value.\n",
- 'attribute-access': u'\nCustomizing attribute access\n****************************\n\nThe following methods can be defined to customize the meaning of\nattribute access (use of, assignment to, or deletion of ``x.name``)\nfor class instances.\n\nobject.__getattr__(self, name)\n\n   Called when an attribute lookup has not found the attribute in the\n   usual places (i.e. it is not an instance attribute nor is it found\n   in the class tree for ``self``).  ``name`` is the attribute name.\n   This method should return the (computed) attribute value or raise\n   an ``AttributeError`` exception.\n\n   Note that if the attribute is found through the normal mechanism,\n   ``__getattr__()`` is not called.  (This is an intentional asymmetry\n   between ``__getattr__()`` and ``__setattr__()``.) This is done both\n   for efficiency reasons and because otherwise ``__getattr__()``\n   would have no way to access other attributes of the instance.  Note\n   that at least for instance variables, you can fake total control by\n   not inserting any values in the instance attribute dictionary (but\n   instead inserting them in another object).  See the\n   ``__getattribute__()`` method below for a way to actually get total\n   control in new-style classes.\n\nobject.__setattr__(self, name, value)\n\n   Called when an attribute assignment is attempted.  This is called\n   instead of the normal mechanism (i.e. store the value in the\n   instance dictionary).  *name* is the attribute name, *value* is the\n   value to be assigned to it.\n\n   If ``__setattr__()`` wants to assign to an instance attribute, it\n   should not simply execute ``self.name = value`` --- this would\n   cause a recursive call to itself.  Instead, it should insert the\n   value in the dictionary of instance attributes, e.g.,\n   ``self.__dict__[name] = value``.  For new-style classes, rather\n   than accessing the instance dictionary, it should call the base\n   class method with the same name, for example,\n   ``object.__setattr__(self, name, value)``.\n\nobject.__delattr__(self, name)\n\n   Like ``__setattr__()`` but for attribute deletion instead of\n   assignment.  This should only be implemented if ``del obj.name`` is\n   meaningful for the object.\n\n\nMore attribute access for new-style classes\n===========================================\n\nThe following methods only apply to new-style classes.\n\nobject.__getattribute__(self, name)\n\n   Called unconditionally to implement attribute accesses for\n   instances of the class. If the class also defines\n   ``__getattr__()``, the latter will not be called unless\n   ``__getattribute__()`` either calls it explicitly or raises an\n   ``AttributeError``. This method should return the (computed)\n   attribute value or raise an ``AttributeError`` exception. In order\n   to avoid infinite recursion in this method, its implementation\n   should always call the base class method with the same name to\n   access any attributes it needs, for example,\n   ``object.__getattribute__(self, name)``.\n\n   Note: This method may still be bypassed when looking up special methods\n     as the result of implicit invocation via language syntax or\n     built-in functions. See *Special method lookup for new-style\n     classes*.\n\n\nImplementing Descriptors\n========================\n\nThe following methods only apply when an instance of the class\ncontaining the method (a so-called *descriptor* class) appears in the\nclass dictionary of another new-style class, known as the *owner*\nclass. In the examples below, "the attribute" refers to the attribute\nwhose name is the key of the property in the owner class\'\n``__dict__``.  Descriptors can only be implemented as new-style\nclasses themselves.\n\nobject.__get__(self, instance, owner)\n\n   Called to get the attribute of the owner class (class attribute\n   access) or of an instance of that class (instance attribute\n   access). *owner* is always the owner class, while *instance* is the\n   instance that the attribute was accessed through, or ``None`` when\n   the attribute is accessed through the *owner*.  This method should\n   return the (computed) attribute value or raise an\n   ``AttributeError`` exception.\n\nobject.__set__(self, instance, value)\n\n   Called to set the attribute on an instance *instance* of the owner\n   class to a new value, *value*.\n\nobject.__delete__(self, instance)\n\n   Called to delete the attribute on an instance *instance* of the\n   owner class.\n\n\nInvoking Descriptors\n====================\n\nIn general, a descriptor is an object attribute with "binding\nbehavior", one whose attribute access has been overridden by methods\nin the descriptor protocol:  ``__get__()``, ``__set__()``, and\n``__delete__()``. If any of those methods are defined for an object,\nit is said to be a descriptor.\n\nThe default behavior for attribute access is to get, set, or delete\nthe attribute from an object\'s dictionary. For instance, ``a.x`` has a\nlookup chain starting with ``a.__dict__[\'x\']``, then\n``type(a).__dict__[\'x\']``, and continuing through the base classes of\n``type(a)`` excluding metaclasses.\n\nHowever, if the looked-up value is an object defining one of the\ndescriptor methods, then Python may override the default behavior and\ninvoke the descriptor method instead.  Where this occurs in the\nprecedence chain depends on which descriptor methods were defined and\nhow they were called.  Note that descriptors are only invoked for new\nstyle objects or classes (ones that subclass ``object()`` or\n``type()``).\n\nThe starting point for descriptor invocation is a binding, ``a.x``.\nHow the arguments are assembled depends on ``a``:\n\nDirect Call\n   The simplest and least common call is when user code directly\n   invokes a descriptor method:    ``x.__get__(a)``.\n\nInstance Binding\n   If binding to a new-style object instance, ``a.x`` is transformed\n   into the call: ``type(a).__dict__[\'x\'].__get__(a, type(a))``.\n\nClass Binding\n   If binding to a new-style class, ``A.x`` is transformed into the\n   call: ``A.__dict__[\'x\'].__get__(None, A)``.\n\nSuper Binding\n   If ``a`` is an instance of ``super``, then the binding ``super(B,\n   obj).m()`` searches ``obj.__class__.__mro__`` for the base class\n   ``A`` immediately preceding ``B`` and then invokes the descriptor\n   with the call: ``A.__dict__[\'m\'].__get__(obj, A)``.\n\nFor instance bindings, the precedence of descriptor invocation depends\non the which descriptor methods are defined.  A descriptor can define\nany combination of ``__get__()``, ``__set__()`` and ``__delete__()``.\nIf it does not define ``__get__()``, then accessing the attribute will\nreturn the descriptor object itself unless there is a value in the\nobject\'s instance dictionary.  If the descriptor defines ``__set__()``\nand/or ``__delete__()``, it is a data descriptor; if it defines\nneither, it is a non-data descriptor.  Normally, data descriptors\ndefine both ``__get__()`` and ``__set__()``, while non-data\ndescriptors have just the ``__get__()`` method.  Data descriptors with\n``__set__()`` and ``__get__()`` defined always override a redefinition\nin an instance dictionary.  In contrast, non-data descriptors can be\noverridden by instances.\n\nPython methods (including ``staticmethod()`` and ``classmethod()``)\nare implemented as non-data descriptors.  Accordingly, instances can\nredefine and override methods.  This allows individual instances to\nacquire behaviors that differ from other instances of the same class.\n\nThe ``property()`` function is implemented as a data descriptor.\nAccordingly, instances cannot override the behavior of a property.\n\n\n__slots__\n=========\n\nBy default, instances of both old and new-style classes have a\ndictionary for attribute storage.  This wastes space for objects\nhaving very few instance variables.  The space consumption can become\nacute when creating large numbers of instances.\n\nThe default can be overridden by defining *__slots__* in a new-style\nclass definition.  The *__slots__* declaration takes a sequence of\ninstance variables and reserves just enough space in each instance to\nhold a value for each variable.  Space is saved because *__dict__* is\nnot created for each instance.\n\n__slots__\n\n   This class variable can be assigned a string, iterable, or sequence\n   of strings with variable names used by instances.  If defined in a\n   new-style class, *__slots__* reserves space for the declared\n   variables and prevents the automatic creation of *__dict__* and\n   *__weakref__* for each instance.\n\n   New in version 2.2.\n\nNotes on using *__slots__*\n\n* When inheriting from a class without *__slots__*, the *__dict__*\n  attribute of that class will always be accessible, so a *__slots__*\n  definition in the subclass is meaningless.\n\n* Without a *__dict__* variable, instances cannot be assigned new\n  variables not listed in the *__slots__* definition.  Attempts to\n  assign to an unlisted variable name raises ``AttributeError``. If\n  dynamic assignment of new variables is desired, then add\n  ``\'__dict__\'`` to the sequence of strings in the *__slots__*\n  declaration.\n\n  Changed in version 2.3: Previously, adding ``\'__dict__\'`` to the\n  *__slots__* declaration would not enable the assignment of new\n  attributes not specifically listed in the sequence of instance\n  variable names.\n\n* Without a *__weakref__* variable for each instance, classes defining\n  *__slots__* do not support weak references to its instances. If weak\n  reference support is needed, then add ``\'__weakref__\'`` to the\n  sequence of strings in the *__slots__* declaration.\n\n  Changed in version 2.3: Previously, adding ``\'__weakref__\'`` to the\n  *__slots__* declaration would not enable support for weak\n  references.\n\n* *__slots__* are implemented at the class level by creating\n  descriptors (*Implementing Descriptors*) for each variable name.  As\n  a result, class attributes cannot be used to set default values for\n  instance variables defined by *__slots__*; otherwise, the class\n  attribute would overwrite the descriptor assignment.\n\n* The action of a *__slots__* declaration is limited to the class\n  where it is defined.  As a result, subclasses will have a *__dict__*\n  unless they also define *__slots__* (which must only contain names\n  of any *additional* slots).\n\n* If a class defines a slot also defined in a base class, the instance\n  variable defined by the base class slot is inaccessible (except by\n  retrieving its descriptor directly from the base class). This\n  renders the meaning of the program undefined.  In the future, a\n  check may be added to prevent this.\n\n* Nonempty *__slots__* does not work for classes derived from\n  "variable-length" built-in types such as ``long``, ``str`` and\n  ``tuple``.\n\n* Any non-string iterable may be assigned to *__slots__*. Mappings may\n  also be used; however, in the future, special meaning may be\n  assigned to the values corresponding to each key.\n\n* *__class__* assignment works only if both classes have the same\n  *__slots__*.\n\n  Changed in version 2.6: Previously, *__class__* assignment raised an\n  error if either new or old class had *__slots__*.\n',
+ 'attribute-access': u'\nCustomizing attribute access\n****************************\n\nThe following methods can be defined to customize the meaning of\nattribute access (use of, assignment to, or deletion of ``x.name``)\nfor class instances.\n\nobject.__getattr__(self, name)\n\n   Called when an attribute lookup has not found the attribute in the\n   usual places (i.e. it is not an instance attribute nor is it found\n   in the class tree for ``self``).  ``name`` is the attribute name.\n   This method should return the (computed) attribute value or raise\n   an ``AttributeError`` exception.\n\n   Note that if the attribute is found through the normal mechanism,\n   ``__getattr__()`` is not called.  (This is an intentional asymmetry\n   between ``__getattr__()`` and ``__setattr__()``.) This is done both\n   for efficiency reasons and because otherwise ``__getattr__()``\n   would have no way to access other attributes of the instance.  Note\n   that at least for instance variables, you can fake total control by\n   not inserting any values in the instance attribute dictionary (but\n   instead inserting them in another object).  See the\n   ``__getattribute__()`` method below for a way to actually get total\n   control in new-style classes.\n\nobject.__setattr__(self, name, value)\n\n   Called when an attribute assignment is attempted.  This is called\n   instead of the normal mechanism (i.e. store the value in the\n   instance dictionary).  *name* is the attribute name, *value* is the\n   value to be assigned to it.\n\n   If ``__setattr__()`` wants to assign to an instance attribute, it\n   should not simply execute ``self.name = value`` --- this would\n   cause a recursive call to itself.  Instead, it should insert the\n   value in the dictionary of instance attributes, e.g.,\n   ``self.__dict__[name] = value``.  For new-style classes, rather\n   than accessing the instance dictionary, it should call the base\n   class method with the same name, for example,\n   ``object.__setattr__(self, name, value)``.\n\nobject.__delattr__(self, name)\n\n   Like ``__setattr__()`` but for attribute deletion instead of\n   assignment.  This should only be implemented if ``del obj.name`` is\n   meaningful for the object.\n\n\nMore attribute access for new-style classes\n===========================================\n\nThe following methods only apply to new-style classes.\n\nobject.__getattribute__(self, name)\n\n   Called unconditionally to implement attribute accesses for\n   instances of the class. If the class also defines\n   ``__getattr__()``, the latter will not be called unless\n   ``__getattribute__()`` either calls it explicitly or raises an\n   ``AttributeError``. This method should return the (computed)\n   attribute value or raise an ``AttributeError`` exception. In order\n   to avoid infinite recursion in this method, its implementation\n   should always call the base class method with the same name to\n   access any attributes it needs, for example,\n   ``object.__getattribute__(self, name)``.\n\n   Note: This method may still be bypassed when looking up special methods\n     as the result of implicit invocation via language syntax or\n     built-in functions. See *Special method lookup for new-style\n     classes*.\n\n\nImplementing Descriptors\n========================\n\nThe following methods only apply when an instance of the class\ncontaining the method (a so-called *descriptor* class) appears in an\n*owner* class (the descriptor must be in either the owner\'s class\ndictionary or in the class dictionary for one of its parents).  In the\nexamples below, "the attribute" refers to the attribute whose name is\nthe key of the property in the owner class\' ``__dict__``.\n\nobject.__get__(self, instance, owner)\n\n   Called to get the attribute of the owner class (class attribute\n   access) or of an instance of that class (instance attribute\n   access). *owner* is always the owner class, while *instance* is the\n   instance that the attribute was accessed through, or ``None`` when\n   the attribute is accessed through the *owner*.  This method should\n   return the (computed) attribute value or raise an\n   ``AttributeError`` exception.\n\nobject.__set__(self, instance, value)\n\n   Called to set the attribute on an instance *instance* of the owner\n   class to a new value, *value*.\n\nobject.__delete__(self, instance)\n\n   Called to delete the attribute on an instance *instance* of the\n   owner class.\n\n\nInvoking Descriptors\n====================\n\nIn general, a descriptor is an object attribute with "binding\nbehavior", one whose attribute access has been overridden by methods\nin the descriptor protocol:  ``__get__()``, ``__set__()``, and\n``__delete__()``. If any of those methods are defined for an object,\nit is said to be a descriptor.\n\nThe default behavior for attribute access is to get, set, or delete\nthe attribute from an object\'s dictionary. For instance, ``a.x`` has a\nlookup chain starting with ``a.__dict__[\'x\']``, then\n``type(a).__dict__[\'x\']``, and continuing through the base classes of\n``type(a)`` excluding metaclasses.\n\nHowever, if the looked-up value is an object defining one of the\ndescriptor methods, then Python may override the default behavior and\ninvoke the descriptor method instead.  Where this occurs in the\nprecedence chain depends on which descriptor methods were defined and\nhow they were called.  Note that descriptors are only invoked for new\nstyle objects or classes (ones that subclass ``object()`` or\n``type()``).\n\nThe starting point for descriptor invocation is a binding, ``a.x``.\nHow the arguments are assembled depends on ``a``:\n\nDirect Call\n   The simplest and least common call is when user code directly\n   invokes a descriptor method:    ``x.__get__(a)``.\n\nInstance Binding\n   If binding to a new-style object instance, ``a.x`` is transformed\n   into the call: ``type(a).__dict__[\'x\'].__get__(a, type(a))``.\n\nClass Binding\n   If binding to a new-style class, ``A.x`` is transformed into the\n   call: ``A.__dict__[\'x\'].__get__(None, A)``.\n\nSuper Binding\n   If ``a`` is an instance of ``super``, then the binding ``super(B,\n   obj).m()`` searches ``obj.__class__.__mro__`` for the base class\n   ``A`` immediately preceding ``B`` and then invokes the descriptor\n   with the call: ``A.__dict__[\'m\'].__get__(obj, obj.__class__)``.\n\nFor instance bindings, the precedence of descriptor invocation depends\non the which descriptor methods are defined.  A descriptor can define\nany combination of ``__get__()``, ``__set__()`` and ``__delete__()``.\nIf it does not define ``__get__()``, then accessing the attribute will\nreturn the descriptor object itself unless there is a value in the\nobject\'s instance dictionary.  If the descriptor defines ``__set__()``\nand/or ``__delete__()``, it is a data descriptor; if it defines\nneither, it is a non-data descriptor.  Normally, data descriptors\ndefine both ``__get__()`` and ``__set__()``, while non-data\ndescriptors have just the ``__get__()`` method.  Data descriptors with\n``__set__()`` and ``__get__()`` defined always override a redefinition\nin an instance dictionary.  In contrast, non-data descriptors can be\noverridden by instances.\n\nPython methods (including ``staticmethod()`` and ``classmethod()``)\nare implemented as non-data descriptors.  Accordingly, instances can\nredefine and override methods.  This allows individual instances to\nacquire behaviors that differ from other instances of the same class.\n\nThe ``property()`` function is implemented as a data descriptor.\nAccordingly, instances cannot override the behavior of a property.\n\n\n__slots__\n=========\n\nBy default, instances of both old and new-style classes have a\ndictionary for attribute storage.  This wastes space for objects\nhaving very few instance variables.  The space consumption can become\nacute when creating large numbers of instances.\n\nThe default can be overridden by defining *__slots__* in a new-style\nclass definition.  The *__slots__* declaration takes a sequence of\ninstance variables and reserves just enough space in each instance to\nhold a value for each variable.  Space is saved because *__dict__* is\nnot created for each instance.\n\n__slots__\n\n   This class variable can be assigned a string, iterable, or sequence\n   of strings with variable names used by instances.  If defined in a\n   new-style class, *__slots__* reserves space for the declared\n   variables and prevents the automatic creation of *__dict__* and\n   *__weakref__* for each instance.\n\n   New in version 2.2.\n\nNotes on using *__slots__*\n\n* When inheriting from a class without *__slots__*, the *__dict__*\n  attribute of that class will always be accessible, so a *__slots__*\n  definition in the subclass is meaningless.\n\n* Without a *__dict__* variable, instances cannot be assigned new\n  variables not listed in the *__slots__* definition.  Attempts to\n  assign to an unlisted variable name raises ``AttributeError``. If\n  dynamic assignment of new variables is desired, then add\n  ``\'__dict__\'`` to the sequence of strings in the *__slots__*\n  declaration.\n\n  Changed in version 2.3: Previously, adding ``\'__dict__\'`` to the\n  *__slots__* declaration would not enable the assignment of new\n  attributes not specifically listed in the sequence of instance\n  variable names.\n\n* Without a *__weakref__* variable for each instance, classes defining\n  *__slots__* do not support weak references to its instances. If weak\n  reference support is needed, then add ``\'__weakref__\'`` to the\n  sequence of strings in the *__slots__* declaration.\n\n  Changed in version 2.3: Previously, adding ``\'__weakref__\'`` to the\n  *__slots__* declaration would not enable support for weak\n  references.\n\n* *__slots__* are implemented at the class level by creating\n  descriptors (*Implementing Descriptors*) for each variable name.  As\n  a result, class attributes cannot be used to set default values for\n  instance variables defined by *__slots__*; otherwise, the class\n  attribute would overwrite the descriptor assignment.\n\n* The action of a *__slots__* declaration is limited to the class\n  where it is defined.  As a result, subclasses will have a *__dict__*\n  unless they also define *__slots__* (which must only contain names\n  of any *additional* slots).\n\n* If a class defines a slot also defined in a base class, the instance\n  variable defined by the base class slot is inaccessible (except by\n  retrieving its descriptor directly from the base class). This\n  renders the meaning of the program undefined.  In the future, a\n  check may be added to prevent this.\n\n* Nonempty *__slots__* does not work for classes derived from\n  "variable-length" built-in types such as ``long``, ``str`` and\n  ``tuple``.\n\n* Any non-string iterable may be assigned to *__slots__*. Mappings may\n  also be used; however, in the future, special meaning may be\n  assigned to the values corresponding to each key.\n\n* *__class__* assignment works only if both classes have the same\n  *__slots__*.\n\n  Changed in version 2.6: Previously, *__class__* assignment raised an\n  error if either new or old class had *__slots__*.\n',
  'attribute-references': u'\nAttribute references\n********************\n\nAn attribute reference is a primary followed by a period and a name:\n\n   attributeref ::= primary "." identifier\n\nThe primary must evaluate to an object of a type that supports\nattribute references, e.g., a module, list, or an instance.  This\nobject is then asked to produce the attribute whose name is the\nidentifier.  If this attribute is not available, the exception\n``AttributeError`` is raised. Otherwise, the type and value of the\nobject produced is determined by the object.  Multiple evaluations of\nthe same attribute reference may yield different objects.\n',
  'augassign': u'\nAugmented assignment statements\n*******************************\n\nAugmented assignment is the combination, in a single statement, of a\nbinary operation and an assignment statement:\n\n   augmented_assignment_stmt ::= augtarget augop (expression_list | yield_expression)\n   augtarget                 ::= identifier | attributeref | subscription | slicing\n   augop                     ::= "+=" | "-=" | "*=" | "/=" | "//=" | "%=" | "**="\n             | ">>=" | "<<=" | "&=" | "^=" | "|="\n\n(See section *Primaries* for the syntax definitions for the last three\nsymbols.)\n\nAn augmented assignment evaluates the target (which, unlike normal\nassignment statements, cannot be an unpacking) and the expression\nlist, performs the binary operation specific to the type of assignment\non the two operands, and assigns the result to the original target.\nThe target is only evaluated once.\n\nAn augmented assignment expression like ``x += 1`` can be rewritten as\n``x = x + 1`` to achieve a similar, but not exactly equal effect. In\nthe augmented version, ``x`` is only evaluated once. Also, when\npossible, the actual operation is performed *in-place*, meaning that\nrather than creating a new object and assigning that to the target,\nthe old object is modified instead.\n\nWith the exception of assigning to tuples and multiple targets in a\nsingle statement, the assignment done by augmented assignment\nstatements is handled the same way as normal assignments. Similarly,\nwith the exception of the possible *in-place* behavior, the binary\noperation performed by augmented assignment is the same as the normal\nbinary operations.\n\nFor targets which are attribute references, the same *caveat about\nclass and instance attributes* applies as for regular assignments.\n',
  'binary': u'\nBinary arithmetic operations\n****************************\n\nThe binary arithmetic operations have the conventional priority\nlevels.  Note that some of these operations also apply to certain non-\nnumeric types.  Apart from the power operator, there are only two\nlevels, one for multiplicative operators and one for additive\noperators:\n\n   m_expr ::= u_expr | m_expr "*" u_expr | m_expr "//" u_expr | m_expr "/" u_expr\n              | m_expr "%" u_expr\n   a_expr ::= m_expr | a_expr "+" m_expr | a_expr "-" m_expr\n\nThe ``*`` (multiplication) operator yields the product of its\narguments.  The arguments must either both be numbers, or one argument\nmust be an integer (plain or long) and the other must be a sequence.\nIn the former case, the numbers are converted to a common type and\nthen multiplied together.  In the latter case, sequence repetition is\nperformed; a negative repetition factor yields an empty sequence.\n\nThe ``/`` (division) and ``//`` (floor division) operators yield the\nquotient of their arguments.  The numeric arguments are first\nconverted to a common type. Plain or long integer division yields an\ninteger of the same type; the result is that of mathematical division\nwith the \'floor\' function applied to the result. Division by zero\nraises the ``ZeroDivisionError`` exception.\n\nThe ``%`` (modulo) operator yields the remainder from the division of\nthe first argument by the second.  The numeric arguments are first\nconverted to a common type.  A zero right argument raises the\n``ZeroDivisionError`` exception.  The arguments may be floating point\nnumbers, e.g., ``3.14%0.7`` equals ``0.34`` (since ``3.14`` equals\n``4*0.7 + 0.34``.)  The modulo operator always yields a result with\nthe same sign as its second operand (or zero); the absolute value of\nthe result is strictly smaller than the absolute value of the second\noperand [2].\n\nThe integer division and modulo operators are connected by the\nfollowing identity: ``x == (x/y)*y + (x%y)``.  Integer division and\nmodulo are also connected with the built-in function ``divmod()``:\n``divmod(x, y) == (x/y, x%y)``.  These identities don\'t hold for\nfloating point numbers; there similar identities hold approximately\nwhere ``x/y`` is replaced by ``floor(x/y)`` or ``floor(x/y) - 1`` [3].\n\nIn addition to performing the modulo operation on numbers, the ``%``\noperator is also overloaded by string and unicode objects to perform\nstring formatting (also known as interpolation). The syntax for string\nformatting is described in the Python Library Reference, section\n*String Formatting Operations*.\n\nDeprecated since version 2.3: The floor division operator, the modulo\noperator, and the ``divmod()`` function are no longer defined for\ncomplex numbers.  Instead, convert to a floating point number using\nthe ``abs()`` function if appropriate.\n\nThe ``+`` (addition) operator yields the sum of its arguments. The\narguments must either both be numbers or both sequences of the same\ntype.  In the former case, the numbers are converted to a common type\nand then added together.  In the latter case, the sequences are\nconcatenated.\n\nThe ``-`` (subtraction) operator yields the difference of its\narguments.  The numeric arguments are first converted to a common\ntype.\n',
  'bitwise': u'\nBinary bitwise operations\n*************************\n\nEach of the three bitwise operations has a different priority level:\n\n   and_expr ::= shift_expr | and_expr "&" shift_expr\n   xor_expr ::= and_expr | xor_expr "^" and_expr\n   or_expr  ::= xor_expr | or_expr "|" xor_expr\n\nThe ``&`` operator yields the bitwise AND of its arguments, which must\nbe plain or long integers.  The arguments are converted to a common\ntype.\n\nThe ``^`` operator yields the bitwise XOR (exclusive OR) of its\narguments, which must be plain or long integers.  The arguments are\nconverted to a common type.\n\nThe ``|`` operator yields the bitwise (inclusive) OR of its arguments,\nwhich must be plain or long integers.  The arguments are converted to\na common type.\n',
  'bltin-code-objects': u'\nCode Objects\n************\n\nCode objects are used by the implementation to represent "pseudo-\ncompiled" executable Python code such as a function body. They differ\nfrom function objects because they don\'t contain a reference to their\nglobal execution environment.  Code objects are returned by the built-\nin ``compile()`` function and can be extracted from function objects\nthrough their ``func_code`` attribute. See also the ``code`` module.\n\nA code object can be executed or evaluated by passing it (instead of a\nsource string) to the ``exec`` statement or the built-in ``eval()``\nfunction.\n\nSee *The standard type hierarchy* for more information.\n',
  'bltin-ellipsis-object': u'\nThe Ellipsis Object\n*******************\n\nThis object is used by extended slice notation (see *Slicings*).  It\nsupports no special operations.  There is exactly one ellipsis object,\nnamed ``Ellipsis`` (a built-in name).\n\nIt is written as ``Ellipsis``.\n',
- 'bltin-file-objects': u'\nFile Objects\n************\n\nFile objects are implemented using C\'s ``stdio`` package and can be\ncreated with the built-in ``open()`` function.  File objects are also\nreturned by some other built-in functions and methods, such as\n``os.popen()`` and ``os.fdopen()`` and the ``makefile()`` method of\nsocket objects. Temporary files can be created using the ``tempfile``\nmodule, and high-level file operations such as copying, moving, and\ndeleting files and directories can be achieved with the ``shutil``\nmodule.\n\nWhen a file operation fails for an I/O-related reason, the exception\n``IOError`` is raised.  This includes situations where the operation\nis not defined for some reason, like ``seek()`` on a tty device or\nwriting a file opened for reading.\n\nFiles have the following methods:\n\nfile.close()\n\n   Close the file.  A closed file cannot be read or written any more.\n   Any operation which requires that the file be open will raise a\n   ``ValueError`` after the file has been closed.  Calling ``close()``\n   more than once is allowed.\n\n   As of Python 2.5, you can avoid having to call this method\n   explicitly if you use the ``with`` statement.  For example, the\n   following code will automatically close *f* when the ``with`` block\n   is exited:\n\n      from __future__ import with_statement # This isn\'t required in Python 2.6\n\n      with open("hello.txt") as f:\n          for line in f:\n              print line\n\n   In older versions of Python, you would have needed to do this to\n   get the same effect:\n\n      f = open("hello.txt")\n      try:\n          for line in f:\n              print line\n      finally:\n          f.close()\n\n   Note: Not all "file-like" types in Python support use as a context\n     manager for the ``with`` statement.  If your code is intended to\n     work with any file-like object, you can use the function\n     ``contextlib.closing()`` instead of using the object directly.\n\nfile.flush()\n\n   Flush the internal buffer, like ``stdio``\'s ``fflush()``.  This may\n   be a no-op on some file-like objects.\n\n   Note: ``flush()`` does not necessarily write the file\'s data to disk.\n     Use ``flush()`` followed by ``os.fsync()`` to ensure this\n     behavior.\n\nfile.fileno()\n\n   Return the integer "file descriptor" that is used by the underlying\n   implementation to request I/O operations from the operating system.\n   This can be useful for other, lower level interfaces that use file\n   descriptors, such as the ``fcntl`` module or ``os.read()`` and\n   friends.\n\n   Note: File-like objects which do not have a real file descriptor should\n     *not* provide this method!\n\nfile.isatty()\n\n   Return ``True`` if the file is connected to a tty(-like) device,\n   else ``False``.\n\n   Note: If a file-like object is not associated with a real file, this\n     method should *not* be implemented.\n\nfile.next()\n\n   A file object is its own iterator, for example ``iter(f)`` returns\n   *f* (unless *f* is closed).  When a file is used as an iterator,\n   typically in a ``for`` loop (for example, ``for line in f: print\n   line``), the ``next()`` method is called repeatedly.  This method\n   returns the next input line, or raises ``StopIteration`` when EOF\n   is hit when the file is open for reading (behavior is undefined\n   when the file is open for writing).  In order to make a ``for``\n   loop the most efficient way of looping over the lines of a file (a\n   very common operation), the ``next()`` method uses a hidden read-\n   ahead buffer.  As a consequence of using a read-ahead buffer,\n   combining ``next()`` with other file methods (like ``readline()``)\n   does not work right.  However, using ``seek()`` to reposition the\n   file to an absolute position will flush the read-ahead buffer.\n\n   New in version 2.3.\n\nfile.read([size])\n\n   Read at most *size* bytes from the file (less if the read hits EOF\n   before obtaining *size* bytes).  If the *size* argument is negative\n   or omitted, read all data until EOF is reached.  The bytes are\n   returned as a string object.  An empty string is returned when EOF\n   is encountered immediately.  (For certain files, like ttys, it\n   makes sense to continue reading after an EOF is hit.)  Note that\n   this method may call the underlying C function ``fread()`` more\n   than once in an effort to acquire as close to *size* bytes as\n   possible. Also note that when in non-blocking mode, less data than\n   was requested may be returned, even if no *size* parameter was\n   given.\n\n   Note: This function is simply a wrapper for the underlying ``fread()``\n     C function, and will behave the same in corner cases, such as\n     whether the EOF value is cached.\n\nfile.readline([size])\n\n   Read one entire line from the file.  A trailing newline character\n   is kept in the string (but may be absent when a file ends with an\n   incomplete line). [5]  If the *size* argument is present and non-\n   negative, it is a maximum byte count (including the trailing\n   newline) and an incomplete line may be returned. An empty string is\n   returned *only* when EOF is encountered immediately.\n\n   Note: Unlike ``stdio``\'s ``fgets()``, the returned string contains null\n     characters (``\'\\0\'``) if they occurred in the input.\n\nfile.readlines([sizehint])\n\n   Read until EOF using ``readline()`` and return a list containing\n   the lines thus read.  If the optional *sizehint* argument is\n   present, instead of reading up to EOF, whole lines totalling\n   approximately *sizehint* bytes (possibly after rounding up to an\n   internal buffer size) are read.  Objects implementing a file-like\n   interface may choose to ignore *sizehint* if it cannot be\n   implemented, or cannot be implemented efficiently.\n\nfile.xreadlines()\n\n   This method returns the same thing as ``iter(f)``.\n\n   New in version 2.1.\n\n   Deprecated since version 2.3: Use ``for line in file`` instead.\n\nfile.seek(offset[, whence])\n\n   Set the file\'s current position, like ``stdio``\'s ``fseek()``. The\n   *whence* argument is optional and defaults to  ``os.SEEK_SET`` or\n   ``0`` (absolute file positioning); other values are ``os.SEEK_CUR``\n   or ``1`` (seek relative to the current position) and\n   ``os.SEEK_END`` or ``2``  (seek relative to the file\'s end).  There\n   is no return value.\n\n   For example, ``f.seek(2, os.SEEK_CUR)`` advances the position by\n   two and ``f.seek(-3, os.SEEK_END)`` sets the position to the third\n   to last.\n\n   Note that if the file is opened for appending (mode ``\'a\'`` or\n   ``\'a+\'``), any ``seek()`` operations will be undone at the next\n   write.  If the file is only opened for writing in append mode (mode\n   ``\'a\'``), this method is essentially a no-op, but it remains useful\n   for files opened in append mode with reading enabled (mode\n   ``\'a+\'``).  If the file is opened in text mode (without ``\'b\'``),\n   only offsets returned by ``tell()`` are legal.  Use of other\n   offsets causes undefined behavior.\n\n   Note that not all file objects are seekable.\n\n   Changed in version 2.6: Passing float values as offset has been\n   deprecated.\n\nfile.tell()\n\n   Return the file\'s current position, like ``stdio``\'s ``ftell()``.\n\n   Note: On Windows, ``tell()`` can return illegal values (after an\n     ``fgets()``) when reading files with Unix-style line-endings. Use\n     binary mode (``\'rb\'``) to circumvent this problem.\n\nfile.truncate([size])\n\n   Truncate the file\'s size.  If the optional *size* argument is\n   present, the file is truncated to (at most) that size.  The size\n   defaults to the current position. The current file position is not\n   changed.  Note that if a specified size exceeds the file\'s current\n   size, the result is platform-dependent:  possibilities include that\n   the file may remain unchanged, increase to the specified size as if\n   zero-filled, or increase to the specified size with undefined new\n   content. Availability:  Windows, many Unix variants.\n\nfile.write(str)\n\n   Write a string to the file.  There is no return value.  Due to\n   buffering, the string may not actually show up in the file until\n   the ``flush()`` or ``close()`` method is called.\n\nfile.writelines(sequence)\n\n   Write a sequence of strings to the file.  The sequence can be any\n   iterable object producing strings, typically a list of strings.\n   There is no return value. (The name is intended to match\n   ``readlines()``; ``writelines()`` does not add line separators.)\n\nFiles support the iterator protocol.  Each iteration returns the same\nresult as ``file.readline()``, and iteration ends when the\n``readline()`` method returns an empty string.\n\nFile objects also offer a number of other interesting attributes.\nThese are not required for file-like objects, but should be\nimplemented if they make sense for the particular object.\n\nfile.closed\n\n   bool indicating the current state of the file object.  This is a\n   read-only attribute; the ``close()`` method changes the value. It\n   may not be available on all file-like objects.\n\nfile.encoding\n\n   The encoding that this file uses. When Unicode strings are written\n   to a file, they will be converted to byte strings using this\n   encoding. In addition, when the file is connected to a terminal,\n   the attribute gives the encoding that the terminal is likely to use\n   (that  information might be incorrect if the user has misconfigured\n   the  terminal). The attribute is read-only and may not be present\n   on all file-like objects. It may also be ``None``, in which case\n   the file uses the system default encoding for converting Unicode\n   strings.\n\n   New in version 2.3.\n\nfile.errors\n\n   The Unicode error handler used along with the encoding.\n\n   New in version 2.6.\n\nfile.mode\n\n   The I/O mode for the file.  If the file was created using the\n   ``open()`` built-in function, this will be the value of the *mode*\n   parameter.  This is a read-only attribute and may not be present on\n   all file-like objects.\n\nfile.name\n\n   If the file object was created using ``open()``, the name of the\n   file. Otherwise, some string that indicates the source of the file\n   object, of the form ``<...>``.  This is a read-only attribute and\n   may not be present on all file-like objects.\n\nfile.newlines\n\n   If Python was built with the *--with-universal-newlines* option to\n   **configure** (the default) this read-only attribute exists, and\n   for files opened in universal newline read mode it keeps track of\n   the types of newlines encountered while reading the file. The\n   values it can take are ``\'\\r\'``, ``\'\\n\'``, ``\'\\r\\n\'``, ``None``\n   (unknown, no newlines read yet) or a tuple containing all the\n   newline types seen, to indicate that multiple newline conventions\n   were encountered. For files not opened in universal newline read\n   mode the value of this attribute will be ``None``.\n\nfile.softspace\n\n   Boolean that indicates whether a space character needs to be\n   printed before another value when using the ``print`` statement.\n   Classes that are trying to simulate a file object should also have\n   a writable ``softspace`` attribute, which should be initialized to\n   zero.  This will be automatic for most classes implemented in\n   Python (care may be needed for objects that override attribute\n   access); types implemented in C will have to provide a writable\n   ``softspace`` attribute.\n\n   Note: This attribute is not used to control the ``print`` statement,\n     but to allow the implementation of ``print`` to keep track of its\n     internal state.\n',
+ 'bltin-file-objects': u'\nFile Objects\n************\n\nFile objects are implemented using C\'s ``stdio`` package and can be\ncreated with the built-in ``open()`` function.  File objects are also\nreturned by some other built-in functions and methods, such as\n``os.popen()`` and ``os.fdopen()`` and the ``makefile()`` method of\nsocket objects. Temporary files can be created using the ``tempfile``\nmodule, and high-level file operations such as copying, moving, and\ndeleting files and directories can be achieved with the ``shutil``\nmodule.\n\nWhen a file operation fails for an I/O-related reason, the exception\n``IOError`` is raised.  This includes situations where the operation\nis not defined for some reason, like ``seek()`` on a tty device or\nwriting a file opened for reading.\n\nFiles have the following methods:\n\nfile.close()\n\n   Close the file.  A closed file cannot be read or written any more.\n   Any operation which requires that the file be open will raise a\n   ``ValueError`` after the file has been closed.  Calling ``close()``\n   more than once is allowed.\n\n   As of Python 2.5, you can avoid having to call this method\n   explicitly if you use the ``with`` statement.  For example, the\n   following code will automatically close *f* when the ``with`` block\n   is exited:\n\n      from __future__ import with_statement # This isn\'t required in Python 2.6\n\n      with open("hello.txt") as f:\n          for line in f:\n              print line\n\n   In older versions of Python, you would have needed to do this to\n   get the same effect:\n\n      f = open("hello.txt")\n      try:\n          for line in f:\n              print line\n      finally:\n          f.close()\n\n   Note: Not all "file-like" types in Python support use as a context\n     manager for the ``with`` statement.  If your code is intended to\n     work with any file-like object, you can use the function\n     ``contextlib.closing()`` instead of using the object directly.\n\nfile.flush()\n\n   Flush the internal buffer, like ``stdio``\'s ``fflush()``.  This may\n   be a no-op on some file-like objects.\n\n   Note: ``flush()`` does not necessarily write the file\'s data to disk.\n     Use ``flush()`` followed by ``os.fsync()`` to ensure this\n     behavior.\n\nfile.fileno()\n\n   Return the integer "file descriptor" that is used by the underlying\n   implementation to request I/O operations from the operating system.\n   This can be useful for other, lower level interfaces that use file\n   descriptors, such as the ``fcntl`` module or ``os.read()`` and\n   friends.\n\n   Note: File-like objects which do not have a real file descriptor should\n     *not* provide this method!\n\nfile.isatty()\n\n   Return ``True`` if the file is connected to a tty(-like) device,\n   else ``False``.\n\n   Note: If a file-like object is not associated with a real file, this\n     method should *not* be implemented.\n\nfile.next()\n\n   A file object is its own iterator, for example ``iter(f)`` returns\n   *f* (unless *f* is closed).  When a file is used as an iterator,\n   typically in a ``for`` loop (for example, ``for line in f: print\n   line``), the ``next()`` method is called repeatedly.  This method\n   returns the next input line, or raises ``StopIteration`` when EOF\n   is hit when the file is open for reading (behavior is undefined\n   when the file is open for writing).  In order to make a ``for``\n   loop the most efficient way of looping over the lines of a file (a\n   very common operation), the ``next()`` method uses a hidden read-\n   ahead buffer.  As a consequence of using a read-ahead buffer,\n   combining ``next()`` with other file methods (like ``readline()``)\n   does not work right.  However, using ``seek()`` to reposition the\n   file to an absolute position will flush the read-ahead buffer.\n\n   New in version 2.3.\n\nfile.read([size])\n\n   Read at most *size* bytes from the file (less if the read hits EOF\n   before obtaining *size* bytes).  If the *size* argument is negative\n   or omitted, read all data until EOF is reached.  The bytes are\n   returned as a string object.  An empty string is returned when EOF\n   is encountered immediately.  (For certain files, like ttys, it\n   makes sense to continue reading after an EOF is hit.)  Note that\n   this method may call the underlying C function ``fread()`` more\n   than once in an effort to acquire as close to *size* bytes as\n   possible. Also note that when in non-blocking mode, less data than\n   was requested may be returned, even if no *size* parameter was\n   given.\n\n   Note: This function is simply a wrapper for the underlying ``fread()``\n     C function, and will behave the same in corner cases, such as\n     whether the EOF value is cached.\n\nfile.readline([size])\n\n   Read one entire line from the file.  A trailing newline character\n   is kept in the string (but may be absent when a file ends with an\n   incomplete line). [5] If the *size* argument is present and non-\n   negative, it is a maximum byte count (including the trailing\n   newline) and an incomplete line may be returned. When *size* is not\n   0, an empty string is returned *only* when EOF is encountered\n   immediately.\n\n   Note: Unlike ``stdio``\'s ``fgets()``, the returned string contains null\n     characters (``\'\\0\'``) if they occurred in the input.\n\nfile.readlines([sizehint])\n\n   Read until EOF using ``readline()`` and return a list containing\n   the lines thus read.  If the optional *sizehint* argument is\n   present, instead of reading up to EOF, whole lines totalling\n   approximately *sizehint* bytes (possibly after rounding up to an\n   internal buffer size) are read.  Objects implementing a file-like\n   interface may choose to ignore *sizehint* if it cannot be\n   implemented, or cannot be implemented efficiently.\n\nfile.xreadlines()\n\n   This method returns the same thing as ``iter(f)``.\n\n   New in version 2.1.\n\n   Deprecated since version 2.3: Use ``for line in file`` instead.\n\nfile.seek(offset[, whence])\n\n   Set the file\'s current position, like ``stdio``\'s ``fseek()``. The\n   *whence* argument is optional and defaults to  ``os.SEEK_SET`` or\n   ``0`` (absolute file positioning); other values are ``os.SEEK_CUR``\n   or ``1`` (seek relative to the current position) and\n   ``os.SEEK_END`` or ``2``  (seek relative to the file\'s end).  There\n   is no return value.\n\n   For example, ``f.seek(2, os.SEEK_CUR)`` advances the position by\n   two and ``f.seek(-3, os.SEEK_END)`` sets the position to the third\n   to last.\n\n   Note that if the file is opened for appending (mode ``\'a\'`` or\n   ``\'a+\'``), any ``seek()`` operations will be undone at the next\n   write.  If the file is only opened for writing in append mode (mode\n   ``\'a\'``), this method is essentially a no-op, but it remains useful\n   for files opened in append mode with reading enabled (mode\n   ``\'a+\'``).  If the file is opened in text mode (without ``\'b\'``),\n   only offsets returned by ``tell()`` are legal.  Use of other\n   offsets causes undefined behavior.\n\n   Note that not all file objects are seekable.\n\n   Changed in version 2.6: Passing float values as offset has been\n   deprecated.\n\nfile.tell()\n\n   Return the file\'s current position, like ``stdio``\'s ``ftell()``.\n\n   Note: On Windows, ``tell()`` can return illegal values (after an\n     ``fgets()``) when reading files with Unix-style line-endings. Use\n     binary mode (``\'rb\'``) to circumvent this problem.\n\nfile.truncate([size])\n\n   Truncate the file\'s size.  If the optional *size* argument is\n   present, the file is truncated to (at most) that size.  The size\n   defaults to the current position. The current file position is not\n   changed.  Note that if a specified size exceeds the file\'s current\n   size, the result is platform-dependent:  possibilities include that\n   the file may remain unchanged, increase to the specified size as if\n   zero-filled, or increase to the specified size with undefined new\n   content. Availability:  Windows, many Unix variants.\n\nfile.write(str)\n\n   Write a string to the file.  There is no return value.  Due to\n   buffering, the string may not actually show up in the file until\n   the ``flush()`` or ``close()`` method is called.\n\nfile.writelines(sequence)\n\n   Write a sequence of strings to the file.  The sequence can be any\n   iterable object producing strings, typically a list of strings.\n   There is no return value. (The name is intended to match\n   ``readlines()``; ``writelines()`` does not add line separators.)\n\nFiles support the iterator protocol.  Each iteration returns the same\nresult as ``file.readline()``, and iteration ends when the\n``readline()`` method returns an empty string.\n\nFile objects also offer a number of other interesting attributes.\nThese are not required for file-like objects, but should be\nimplemented if they make sense for the particular object.\n\nfile.closed\n\n   bool indicating the current state of the file object.  This is a\n   read-only attribute; the ``close()`` method changes the value. It\n   may not be available on all file-like objects.\n\nfile.encoding\n\n   The encoding that this file uses. When Unicode strings are written\n   to a file, they will be converted to byte strings using this\n   encoding. In addition, when the file is connected to a terminal,\n   the attribute gives the encoding that the terminal is likely to use\n   (that  information might be incorrect if the user has misconfigured\n   the  terminal). The attribute is read-only and may not be present\n   on all file-like objects. It may also be ``None``, in which case\n   the file uses the system default encoding for converting Unicode\n   strings.\n\n   New in version 2.3.\n\nfile.errors\n\n   The Unicode error handler used along with the encoding.\n\n   New in version 2.6.\n\nfile.mode\n\n   The I/O mode for the file.  If the file was created using the\n   ``open()`` built-in function, this will be the value of the *mode*\n   parameter.  This is a read-only attribute and may not be present on\n   all file-like objects.\n\nfile.name\n\n   If the file object was created using ``open()``, the name of the\n   file. Otherwise, some string that indicates the source of the file\n   object, of the form ``<...>``.  This is a read-only attribute and\n   may not be present on all file-like objects.\n\nfile.newlines\n\n   If Python was built with universal newlines enabled (the default)\n   this read-only attribute exists, and for files opened in universal\n   newline read mode it keeps track of the types of newlines\n   encountered while reading the file. The values it can take are\n   ``\'\\r\'``, ``\'\\n\'``, ``\'\\r\\n\'``, ``None`` (unknown, no newlines read\n   yet) or a tuple containing all the newline types seen, to indicate\n   that multiple newline conventions were encountered. For files not\n   opened in universal newline read mode the value of this attribute\n   will be ``None``.\n\nfile.softspace\n\n   Boolean that indicates whether a space character needs to be\n   printed before another value when using the ``print`` statement.\n   Classes that are trying to simulate a file object should also have\n   a writable ``softspace`` attribute, which should be initialized to\n   zero.  This will be automatic for most classes implemented in\n   Python (care may be needed for objects that override attribute\n   access); types implemented in C will have to provide a writable\n   ``softspace`` attribute.\n\n   Note: This attribute is not used to control the ``print`` statement,\n     but to allow the implementation of ``print`` to keep track of its\n     internal state.\n',
  'bltin-null-object': u"\nThe Null Object\n***************\n\nThis object is returned by functions that don't explicitly return a\nvalue.  It supports no special operations.  There is exactly one null\nobject, named ``None`` (a built-in name).\n\nIt is written as ``None``.\n",
  'bltin-type-objects': u"\nType Objects\n************\n\nType objects represent the various object types.  An object's type is\naccessed by the built-in function ``type()``.  There are no special\noperations on types.  The standard module ``types`` defines names for\nall standard built-in types.\n\nTypes are written like this: ````.\n",
  'booleans': u'\nBoolean operations\n******************\n\n   or_test  ::= and_test | or_test "or" and_test\n   and_test ::= not_test | and_test "and" not_test\n   not_test ::= comparison | "not" not_test\n\nIn the context of Boolean operations, and also when expressions are\nused by control flow statements, the following values are interpreted\nas false: ``False``, ``None``, numeric zero of all types, and empty\nstrings and containers (including strings, tuples, lists,\ndictionaries, sets and frozensets).  All other values are interpreted\nas true.  (See the ``__nonzero__()`` special method for a way to\nchange this.)\n\nThe operator ``not`` yields ``True`` if its argument is false,\n``False`` otherwise.\n\nThe expression ``x and y`` first evaluates *x*; if *x* is false, its\nvalue is returned; otherwise, *y* is evaluated and the resulting value\nis returned.\n\nThe expression ``x or y`` first evaluates *x*; if *x* is true, its\nvalue is returned; otherwise, *y* is evaluated and the resulting value\nis returned.\n\n(Note that neither ``and`` nor ``or`` restrict the value and type they\nreturn to ``False`` and ``True``, but rather return the last evaluated\nargument. This is sometimes useful, e.g., if ``s`` is a string that\nshould be replaced by a default value if it is empty, the expression\n``s or \'foo\'`` yields the desired value.  Because ``not`` has to\ninvent a value anyway, it does not bother to return a value of the\nsame type as its argument, so e.g., ``not \'foo\'`` yields ``False``,\nnot ``\'\'``.)\n',
@@ -20,39 +20,39 @@
  'class': u'\nClass definitions\n*****************\n\nA class definition defines a class object (see section *The standard\ntype hierarchy*):\n\n   classdef    ::= "class" classname [inheritance] ":" suite\n   inheritance ::= "(" [expression_list] ")"\n   classname   ::= identifier\n\nA class definition is an executable statement.  It first evaluates the\ninheritance list, if present.  Each item in the inheritance list\nshould evaluate to a class object or class type which allows\nsubclassing.  The class\'s suite is then executed in a new execution\nframe (see section *Naming and binding*), using a newly created local\nnamespace and the original global namespace. (Usually, the suite\ncontains only function definitions.)  When the class\'s suite finishes\nexecution, its execution frame is discarded but its local namespace is\nsaved. [4] A class object is then created using the inheritance list\nfor the base classes and the saved local namespace for the attribute\ndictionary.  The class name is bound to this class object in the\noriginal local namespace.\n\n**Programmer\'s note:** Variables defined in the class definition are\nclass variables; they are shared by all instances.  To create instance\nvariables, they can be set in a method with ``self.name = value``.\nBoth class and instance variables are accessible through the notation\n"``self.name``", and an instance variable hides a class variable with\nthe same name when accessed in this way. Class variables can be used\nas defaults for instance variables, but using mutable values there can\nlead to unexpected results.  For *new-style class*es, descriptors can\nbe used to create instance variables with different implementation\ndetails.\n\nClass definitions, like function definitions, may be wrapped by one or\nmore *decorator* expressions.  The evaluation rules for the decorator\nexpressions are the same as for functions.  The result must be a class\nobject, which is then bound to the class name.\n\n-[ Footnotes ]-\n\n[1] The exception is propagated to the invocation stack only if there\n    is no ``finally`` clause that negates the exception.\n\n[2] Currently, control "flows off the end" except in the case of an\n    exception or the execution of a ``return``, ``continue``, or\n    ``break`` statement.\n\n[3] A string literal appearing as the first statement in the function\n    body is transformed into the function\'s ``__doc__`` attribute and\n    therefore the function\'s *docstring*.\n\n[4] A string literal appearing as the first statement in the class\n    body is transformed into the namespace\'s ``__doc__`` item and\n    therefore the class\'s *docstring*.\n',
  'coercion-rules': u"\nCoercion rules\n**************\n\nThis section used to document the rules for coercion.  As the language\nhas evolved, the coercion rules have become hard to document\nprecisely; documenting what one version of one particular\nimplementation does is undesirable.  Instead, here are some informal\nguidelines regarding coercion.  In Python 3.0, coercion will not be\nsupported.\n\n* If the left operand of a % operator is a string or Unicode object,\n  no coercion takes place and the string formatting operation is\n  invoked instead.\n\n* It is no longer recommended to define a coercion operation. Mixed-\n  mode operations on types that don't define coercion pass the\n  original arguments to the operation.\n\n* New-style classes (those derived from ``object``) never invoke the\n  ``__coerce__()`` method in response to a binary operator; the only\n  time ``__coerce__()`` is invoked is when the built-in function\n  ``coerce()`` is called.\n\n* For most intents and purposes, an operator that returns\n  ``NotImplemented`` is treated the same as one that is not\n  implemented at all.\n\n* Below, ``__op__()`` and ``__rop__()`` are used to signify the\n  generic method names corresponding to an operator; ``__iop__()`` is\n  used for the corresponding in-place operator.  For example, for the\n  operator '``+``', ``__add__()`` and ``__radd__()`` are used for the\n  left and right variant of the binary operator, and ``__iadd__()``\n  for the in-place variant.\n\n* For objects *x* and *y*, first ``x.__op__(y)`` is tried.  If this is\n  not implemented or returns ``NotImplemented``, ``y.__rop__(x)`` is\n  tried.  If this is also not implemented or returns\n  ``NotImplemented``, a ``TypeError`` exception is raised.  But see\n  the following exception:\n\n* Exception to the previous item: if the left operand is an instance\n  of a built-in type or a new-style class, and the right operand is an\n  instance of a proper subclass of that type or class and overrides\n  the base's ``__rop__()`` method, the right operand's ``__rop__()``\n  method is tried *before* the left operand's ``__op__()`` method.\n\n  This is done so that a subclass can completely override binary\n  operators. Otherwise, the left operand's ``__op__()`` method would\n  always accept the right operand: when an instance of a given class\n  is expected, an instance of a subclass of that class is always\n  acceptable.\n\n* When either operand type defines a coercion, this coercion is called\n  before that type's ``__op__()`` or ``__rop__()`` method is called,\n  but no sooner.  If the coercion returns an object of a different\n  type for the operand whose coercion is invoked, part of the process\n  is redone using the new object.\n\n* When an in-place operator (like '``+=``') is used, if the left\n  operand implements ``__iop__()``, it is invoked without any\n  coercion.  When the operation falls back to ``__op__()`` and/or\n  ``__rop__()``, the normal coercion rules apply.\n\n* In ``x + y``, if *x* is a sequence that implements sequence\n  concatenation, sequence concatenation is invoked.\n\n* In ``x * y``, if one operator is a sequence that implements sequence\n  repetition, and the other is an integer (``int`` or ``long``),\n  sequence repetition is invoked.\n\n* Rich comparisons (implemented by methods ``__eq__()`` and so on)\n  never use coercion.  Three-way comparison (implemented by\n  ``__cmp__()``) does use coercion under the same conditions as other\n  binary operations use it.\n\n* In the current implementation, the built-in numeric types ``int``,\n  ``long``, ``float``, and ``complex`` do not use coercion. All these\n  types implement a ``__coerce__()`` method, for use by the built-in\n  ``coerce()`` function.\n\n  Changed in version 2.7.\n",
  'comparisons': u'\nComparisons\n***********\n\nUnlike C, all comparison operations in Python have the same priority,\nwhich is lower than that of any arithmetic, shifting or bitwise\noperation.  Also unlike C, expressions like ``a < b < c`` have the\ninterpretation that is conventional in mathematics:\n\n   comparison    ::= or_expr ( comp_operator or_expr )*\n   comp_operator ::= "<" | ">" | "==" | ">=" | "<=" | "<>" | "!="\n                     | "is" ["not"] | ["not"] "in"\n\nComparisons yield boolean values: ``True`` or ``False``.\n\nComparisons can be chained arbitrarily, e.g., ``x < y <= z`` is\nequivalent to ``x < y and y <= z``, except that ``y`` is evaluated\nonly once (but in both cases ``z`` is not evaluated at all when ``x <\ny`` is found to be false).\n\nFormally, if *a*, *b*, *c*, ..., *y*, *z* are expressions and *op1*,\n*op2*, ..., *opN* are comparison operators, then ``a op1 b op2 c ... y\nopN z`` is equivalent to ``a op1 b and b op2 c and ... y opN z``,\nexcept that each expression is evaluated at most once.\n\nNote that ``a op1 b op2 c`` doesn\'t imply any kind of comparison\nbetween *a* and *c*, so that, e.g., ``x < y > z`` is perfectly legal\n(though perhaps not pretty).\n\nThe forms ``<>`` and ``!=`` are equivalent; for consistency with C,\n``!=`` is preferred; where ``!=`` is mentioned below ``<>`` is also\naccepted.  The ``<>`` spelling is considered obsolescent.\n\nThe operators ``<``, ``>``, ``==``, ``>=``, ``<=``, and ``!=`` compare\nthe values of two objects.  The objects need not have the same type.\nIf both are numbers, they are converted to a common type.  Otherwise,\nobjects of different types *always* compare unequal, and are ordered\nconsistently but arbitrarily. You can control comparison behavior of\nobjects of non-built-in types by defining a ``__cmp__`` method or rich\ncomparison methods like ``__gt__``, described in section *Special\nmethod names*.\n\n(This unusual definition of comparison was used to simplify the\ndefinition of operations like sorting and the ``in`` and ``not in``\noperators. In the future, the comparison rules for objects of\ndifferent types are likely to change.)\n\nComparison of objects of the same type depends on the type:\n\n* Numbers are compared arithmetically.\n\n* Strings are compared lexicographically using the numeric equivalents\n  (the result of the built-in function ``ord()``) of their characters.\n  Unicode and 8-bit strings are fully interoperable in this behavior.\n  [4]\n\n* Tuples and lists are compared lexicographically using comparison of\n  corresponding elements.  This means that to compare equal, each\n  element must compare equal and the two sequences must be of the same\n  type and have the same length.\n\n  If not equal, the sequences are ordered the same as their first\n  differing elements.  For example, ``cmp([1,2,x], [1,2,y])`` returns\n  the same as ``cmp(x,y)``.  If the corresponding element does not\n  exist, the shorter sequence is ordered first (for example, ``[1,2] <\n  [1,2,3]``).\n\n* Mappings (dictionaries) compare equal if and only if their sorted\n  (key, value) lists compare equal. [5] Outcomes other than equality\n  are resolved consistently, but are not otherwise defined. [6]\n\n* Most other objects of built-in types compare unequal unless they are\n  the same object; the choice whether one object is considered smaller\n  or larger than another one is made arbitrarily but consistently\n  within one execution of a program.\n\nThe operators ``in`` and ``not in`` test for collection membership.\n``x in s`` evaluates to true if *x* is a member of the collection *s*,\nand false otherwise.  ``x not in s`` returns the negation of ``x in\ns``. The collection membership test has traditionally been bound to\nsequences; an object is a member of a collection if the collection is\na sequence and contains an element equal to that object.  However, it\nmake sense for many other object types to support membership tests\nwithout being a sequence.  In particular, dictionaries (for keys) and\nsets support membership testing.\n\nFor the list and tuple types, ``x in y`` is true if and only if there\nexists an index *i* such that ``x == y[i]`` is true.\n\nFor the Unicode and string types, ``x in y`` is true if and only if\n*x* is a substring of *y*.  An equivalent test is ``y.find(x) != -1``.\nNote, *x* and *y* need not be the same type; consequently, ``u\'ab\' in\n\'abc\'`` will return ``True``. Empty strings are always considered to\nbe a substring of any other string, so ``"" in "abc"`` will return\n``True``.\n\nChanged in version 2.3: Previously, *x* was required to be a string of\nlength ``1``.\n\nFor user-defined classes which define the ``__contains__()`` method,\n``x in y`` is true if and only if ``y.__contains__(x)`` is true.\n\nFor user-defined classes which do not define ``__contains__()`` but do\ndefine ``__iter__()``, ``x in y`` is true if some value ``z`` with ``x\n== z`` is produced while iterating over ``y``.  If an exception is\nraised during the iteration, it is as if ``in`` raised that exception.\n\nLastly, the old-style iteration protocol is tried: if a class defines\n``__getitem__()``, ``x in y`` is true if and only if there is a non-\nnegative integer index *i* such that ``x == y[i]``, and all lower\ninteger indices do not raise ``IndexError`` exception. (If any other\nexception is raised, it is as if ``in`` raised that exception).\n\nThe operator ``not in`` is defined to have the inverse true value of\n``in``.\n\nThe operators ``is`` and ``is not`` test for object identity: ``x is\ny`` is true if and only if *x* and *y* are the same object.  ``x is\nnot y`` yields the inverse truth value. [7]\n',
- 'compound': u'\nCompound statements\n*******************\n\nCompound statements contain (groups of) other statements; they affect\nor control the execution of those other statements in some way.  In\ngeneral, compound statements span multiple lines, although in simple\nincarnations a whole compound statement may be contained in one line.\n\nThe ``if``, ``while`` and ``for`` statements implement traditional\ncontrol flow constructs.  ``try`` specifies exception handlers and/or\ncleanup code for a group of statements.  Function and class\ndefinitions are also syntactically compound statements.\n\nCompound statements consist of one or more \'clauses.\'  A clause\nconsists of a header and a \'suite.\'  The clause headers of a\nparticular compound statement are all at the same indentation level.\nEach clause header begins with a uniquely identifying keyword and ends\nwith a colon.  A suite is a group of statements controlled by a\nclause.  A suite can be one or more semicolon-separated simple\nstatements on the same line as the header, following the header\'s\ncolon, or it can be one or more indented statements on subsequent\nlines.  Only the latter form of suite can contain nested compound\nstatements; the following is illegal, mostly because it wouldn\'t be\nclear to which ``if`` clause a following ``else`` clause would belong:\n\n   if test1: if test2: print x\n\nAlso note that the semicolon binds tighter than the colon in this\ncontext, so that in the following example, either all or none of the\n``print`` statements are executed:\n\n   if x < y < z: print x; print y; print z\n\nSummarizing:\n\n   compound_stmt ::= if_stmt\n                     | while_stmt\n                     | for_stmt\n                     | try_stmt\n                     | with_stmt\n                     | funcdef\n                     | classdef\n                     | decorated\n   suite         ::= stmt_list NEWLINE | NEWLINE INDENT statement+ DEDENT\n   statement     ::= stmt_list NEWLINE | compound_stmt\n   stmt_list     ::= simple_stmt (";" simple_stmt)* [";"]\n\nNote that statements always end in a ``NEWLINE`` possibly followed by\na ``DEDENT``. Also note that optional continuation clauses always\nbegin with a keyword that cannot start a statement, thus there are no\nambiguities (the \'dangling ``else``\' problem is solved in Python by\nrequiring nested ``if`` statements to be indented).\n\nThe formatting of the grammar rules in the following sections places\neach clause on a separate line for clarity.\n\n\nThe ``if`` statement\n====================\n\nThe ``if`` statement is used for conditional execution:\n\n   if_stmt ::= "if" expression ":" suite\n               ( "elif" expression ":" suite )*\n               ["else" ":" suite]\n\nIt selects exactly one of the suites by evaluating the expressions one\nby one until one is found to be true (see section *Boolean operations*\nfor the definition of true and false); then that suite is executed\n(and no other part of the ``if`` statement is executed or evaluated).\nIf all expressions are false, the suite of the ``else`` clause, if\npresent, is executed.\n\n\nThe ``while`` statement\n=======================\n\nThe ``while`` statement is used for repeated execution as long as an\nexpression is true:\n\n   while_stmt ::= "while" expression ":" suite\n                  ["else" ":" suite]\n\nThis repeatedly tests the expression and, if it is true, executes the\nfirst suite; if the expression is false (which may be the first time\nit is tested) the suite of the ``else`` clause, if present, is\nexecuted and the loop terminates.\n\nA ``break`` statement executed in the first suite terminates the loop\nwithout executing the ``else`` clause\'s suite.  A ``continue``\nstatement executed in the first suite skips the rest of the suite and\ngoes back to testing the expression.\n\n\nThe ``for`` statement\n=====================\n\nThe ``for`` statement is used to iterate over the elements of a\nsequence (such as a string, tuple or list) or other iterable object:\n\n   for_stmt ::= "for" target_list "in" expression_list ":" suite\n                ["else" ":" suite]\n\nThe expression list is evaluated once; it should yield an iterable\nobject.  An iterator is created for the result of the\n``expression_list``.  The suite is then executed once for each item\nprovided by the iterator, in the order of ascending indices.  Each\nitem in turn is assigned to the target list using the standard rules\nfor assignments, and then the suite is executed.  When the items are\nexhausted (which is immediately when the sequence is empty), the suite\nin the ``else`` clause, if present, is executed, and the loop\nterminates.\n\nA ``break`` statement executed in the first suite terminates the loop\nwithout executing the ``else`` clause\'s suite.  A ``continue``\nstatement executed in the first suite skips the rest of the suite and\ncontinues with the next item, or with the ``else`` clause if there was\nno next item.\n\nThe suite may assign to the variable(s) in the target list; this does\nnot affect the next item assigned to it.\n\nThe target list is not deleted when the loop is finished, but if the\nsequence is empty, it will not have been assigned to at all by the\nloop.  Hint: the built-in function ``range()`` returns a sequence of\nintegers suitable to emulate the effect of Pascal\'s ``for i := a to b\ndo``; e.g., ``range(3)`` returns the list ``[0, 1, 2]``.\n\nNote: There is a subtlety when the sequence is being modified by the loop\n  (this can only occur for mutable sequences, i.e. lists). An internal\n  counter is used to keep track of which item is used next, and this\n  is incremented on each iteration.  When this counter has reached the\n  length of the sequence the loop terminates.  This means that if the\n  suite deletes the current (or a previous) item from the sequence,\n  the next item will be skipped (since it gets the index of the\n  current item which has already been treated).  Likewise, if the\n  suite inserts an item in the sequence before the current item, the\n  current item will be treated again the next time through the loop.\n  This can lead to nasty bugs that can be avoided by making a\n  temporary copy using a slice of the whole sequence, e.g.,\n\n     for x in a[:]:\n         if x < 0: a.remove(x)\n\n\nThe ``try`` statement\n=====================\n\nThe ``try`` statement specifies exception handlers and/or cleanup code\nfor a group of statements:\n\n   try_stmt  ::= try1_stmt | try2_stmt\n   try1_stmt ::= "try" ":" suite\n                 ("except" [expression [("as" | ",") target]] ":" suite)+\n                 ["else" ":" suite]\n                 ["finally" ":" suite]\n   try2_stmt ::= "try" ":" suite\n                 "finally" ":" suite\n\nChanged in version 2.5: In previous versions of Python,\n``try``...``except``...``finally`` did not work. ``try``...``except``\nhad to be nested in ``try``...``finally``.\n\nThe ``except`` clause(s) specify one or more exception handlers. When\nno exception occurs in the ``try`` clause, no exception handler is\nexecuted. When an exception occurs in the ``try`` suite, a search for\nan exception handler is started.  This search inspects the except\nclauses in turn until one is found that matches the exception.  An\nexpression-less except clause, if present, must be last; it matches\nany exception.  For an except clause with an expression, that\nexpression is evaluated, and the clause matches the exception if the\nresulting object is "compatible" with the exception.  An object is\ncompatible with an exception if it is the class or a base class of the\nexception object, a tuple containing an item compatible with the\nexception, or, in the (deprecated) case of string exceptions, is the\nraised string itself (note that the object identities must match, i.e.\nit must be the same string object, not just a string with the same\nvalue).\n\nIf no except clause matches the exception, the search for an exception\nhandler continues in the surrounding code and on the invocation stack.\n[1]\n\nIf the evaluation of an expression in the header of an except clause\nraises an exception, the original search for a handler is canceled and\na search starts for the new exception in the surrounding code and on\nthe call stack (it is treated as if the entire ``try`` statement\nraised the exception).\n\nWhen a matching except clause is found, the exception is assigned to\nthe target specified in that except clause, if present, and the except\nclause\'s suite is executed.  All except clauses must have an\nexecutable block.  When the end of this block is reached, execution\ncontinues normally after the entire try statement.  (This means that\nif two nested handlers exist for the same exception, and the exception\noccurs in the try clause of the inner handler, the outer handler will\nnot handle the exception.)\n\nBefore an except clause\'s suite is executed, details about the\nexception are assigned to three variables in the ``sys`` module:\n``sys.exc_type`` receives the object identifying the exception;\n``sys.exc_value`` receives the exception\'s parameter;\n``sys.exc_traceback`` receives a traceback object (see section *The\nstandard type hierarchy*) identifying the point in the program where\nthe exception occurred. These details are also available through the\n``sys.exc_info()`` function, which returns a tuple ``(exc_type,\nexc_value, exc_traceback)``.  Use of the corresponding variables is\ndeprecated in favor of this function, since their use is unsafe in a\nthreaded program.  As of Python 1.5, the variables are restored to\ntheir previous values (before the call) when returning from a function\nthat handled an exception.\n\nThe optional ``else`` clause is executed if and when control flows off\nthe end of the ``try`` clause. [2] Exceptions in the ``else`` clause\nare not handled by the preceding ``except`` clauses.\n\nIf ``finally`` is present, it specifies a \'cleanup\' handler.  The\n``try`` clause is executed, including any ``except`` and ``else``\nclauses.  If an exception occurs in any of the clauses and is not\nhandled, the exception is temporarily saved. The ``finally`` clause is\nexecuted.  If there is a saved exception, it is re-raised at the end\nof the ``finally`` clause. If the ``finally`` clause raises another\nexception or executes a ``return`` or ``break`` statement, the saved\nexception is lost.  The exception information is not available to the\nprogram during execution of the ``finally`` clause.\n\nWhen a ``return``, ``break`` or ``continue`` statement is executed in\nthe ``try`` suite of a ``try``...``finally`` statement, the\n``finally`` clause is also executed \'on the way out.\' A ``continue``\nstatement is illegal in the ``finally`` clause. (The reason is a\nproblem with the current implementation --- this restriction may be\nlifted in the future).\n\nAdditional information on exceptions can be found in section\n*Exceptions*, and information on using the ``raise`` statement to\ngenerate exceptions may be found in section *The raise statement*.\n\n\nThe ``with`` statement\n======================\n\nNew in version 2.5.\n\nThe ``with`` statement is used to wrap the execution of a block with\nmethods defined by a context manager (see section *With Statement\nContext Managers*). This allows common\n``try``...``except``...``finally`` usage patterns to be encapsulated\nfor convenient reuse.\n\n   with_stmt ::= "with" with_item ("," with_item)* ":" suite\n   with_item ::= expression ["as" target]\n\nThe execution of the ``with`` statement with one "item" proceeds as\nfollows:\n\n1. The context expression is evaluated to obtain a context manager.\n\n2. The context manager\'s ``__exit__()`` is loaded for later use.\n\n3. The context manager\'s ``__enter__()`` method is invoked.\n\n4. If a target was included in the ``with`` statement, the return\n   value from ``__enter__()`` is assigned to it.\n\n   Note: The ``with`` statement guarantees that if the ``__enter__()``\n     method returns without an error, then ``__exit__()`` will always\n     be called. Thus, if an error occurs during the assignment to the\n     target list, it will be treated the same as an error occurring\n     within the suite would be. See step 6 below.\n\n5. The suite is executed.\n\n6. The context manager\'s ``__exit__()`` method is invoked. If an\n   exception caused the suite to be exited, its type, value, and\n   traceback are passed as arguments to ``__exit__()``. Otherwise,\n   three ``None`` arguments are supplied.\n\n   If the suite was exited due to an exception, and the return value\n   from the ``__exit__()`` method was false, the exception is\n   reraised. If the return value was true, the exception is\n   suppressed, and execution continues with the statement following\n   the ``with`` statement.\n\n   If the suite was exited for any reason other than an exception, the\n   return value from ``__exit__()`` is ignored, and execution proceeds\n   at the normal location for the kind of exit that was taken.\n\nWith more than one item, the context managers are processed as if\nmultiple ``with`` statements were nested:\n\n   with A() as a, B() as b:\n       suite\n\nis equivalent to\n\n   with A() as a:\n       with B() as b:\n           suite\n\nNote: In Python 2.5, the ``with`` statement is only allowed when the\n  ``with_statement`` feature has been enabled.  It is always enabled\n  in Python 2.6.\n\nChanged in version 2.7: Support for multiple context expressions.\n\nSee also:\n\n   **PEP 0343** - The "with" statement\n      The specification, background, and examples for the Python\n      ``with`` statement.\n\n\nFunction definitions\n====================\n\nA function definition defines a user-defined function object (see\nsection *The standard type hierarchy*):\n\n   decorated      ::= decorators (classdef | funcdef)\n   decorators     ::= decorator+\n   decorator      ::= "@" dotted_name ["(" [argument_list [","]] ")"] NEWLINE\n   funcdef        ::= "def" funcname "(" [parameter_list] ")" ":" suite\n   dotted_name    ::= identifier ("." identifier)*\n   parameter_list ::= (defparameter ",")*\n                      (  "*" identifier [, "**" identifier]\n                      | "**" identifier\n                      | defparameter [","] )\n   defparameter   ::= parameter ["=" expression]\n   sublist        ::= parameter ("," parameter)* [","]\n   parameter      ::= identifier | "(" sublist ")"\n   funcname       ::= identifier\n\nA function definition is an executable statement.  Its execution binds\nthe function name in the current local namespace to a function object\n(a wrapper around the executable code for the function).  This\nfunction object contains a reference to the current global namespace\nas the global namespace to be used when the function is called.\n\nThe function definition does not execute the function body; this gets\nexecuted only when the function is called. [3]\n\nA function definition may be wrapped by one or more *decorator*\nexpressions. Decorator expressions are evaluated when the function is\ndefined, in the scope that contains the function definition.  The\nresult must be a callable, which is invoked with the function object\nas the only argument. The returned value is bound to the function name\ninstead of the function object.  Multiple decorators are applied in\nnested fashion. For example, the following code:\n\n   @f1(arg)\n   @f2\n   def func(): pass\n\nis equivalent to:\n\n   def func(): pass\n   func = f1(arg)(f2(func))\n\nWhen one or more top-level parameters have the form *parameter* ``=``\n*expression*, the function is said to have "default parameter values."\nFor a parameter with a default value, the corresponding argument may\nbe omitted from a call, in which case the parameter\'s default value is\nsubstituted.  If a parameter has a default value, all following\nparameters must also have a default value --- this is a syntactic\nrestriction that is not expressed by the grammar.\n\n**Default parameter values are evaluated when the function definition\nis executed.**  This means that the expression is evaluated once, when\nthe function is defined, and that that same "pre-computed" value is\nused for each call.  This is especially important to understand when a\ndefault parameter is a mutable object, such as a list or a dictionary:\nif the function modifies the object (e.g. by appending an item to a\nlist), the default value is in effect modified. This is generally not\nwhat was intended.  A way around this  is to use ``None`` as the\ndefault, and explicitly test for it in the body of the function, e.g.:\n\n   def whats_on_the_telly(penguin=None):\n       if penguin is None:\n           penguin = []\n       penguin.append("property of the zoo")\n       return penguin\n\nFunction call semantics are described in more detail in section\n*Calls*. A function call always assigns values to all parameters\nmentioned in the parameter list, either from position arguments, from\nkeyword arguments, or from default values.  If the form\n"``*identifier``" is present, it is initialized to a tuple receiving\nany excess positional parameters, defaulting to the empty tuple.  If\nthe form "``**identifier``" is present, it is initialized to a new\ndictionary receiving any excess keyword arguments, defaulting to a new\nempty dictionary.\n\nIt is also possible to create anonymous functions (functions not bound\nto a name), for immediate use in expressions.  This uses lambda forms,\ndescribed in section *Lambdas*.  Note that the lambda form is merely a\nshorthand for a simplified function definition; a function defined in\na "``def``" statement can be passed around or assigned to another name\njust like a function defined by a lambda form.  The "``def``" form is\nactually more powerful since it allows the execution of multiple\nstatements.\n\n**Programmer\'s note:** Functions are first-class objects.  A "``def``"\nform executed inside a function definition defines a local function\nthat can be returned or passed around.  Free variables used in the\nnested function can access the local variables of the function\ncontaining the def.  See section *Naming and binding* for details.\n\n\nClass definitions\n=================\n\nA class definition defines a class object (see section *The standard\ntype hierarchy*):\n\n   classdef    ::= "class" classname [inheritance] ":" suite\n   inheritance ::= "(" [expression_list] ")"\n   classname   ::= identifier\n\nA class definition is an executable statement.  It first evaluates the\ninheritance list, if present.  Each item in the inheritance list\nshould evaluate to a class object or class type which allows\nsubclassing.  The class\'s suite is then executed in a new execution\nframe (see section *Naming and binding*), using a newly created local\nnamespace and the original global namespace. (Usually, the suite\ncontains only function definitions.)  When the class\'s suite finishes\nexecution, its execution frame is discarded but its local namespace is\nsaved. [4] A class object is then created using the inheritance list\nfor the base classes and the saved local namespace for the attribute\ndictionary.  The class name is bound to this class object in the\noriginal local namespace.\n\n**Programmer\'s note:** Variables defined in the class definition are\nclass variables; they are shared by all instances.  To create instance\nvariables, they can be set in a method with ``self.name = value``.\nBoth class and instance variables are accessible through the notation\n"``self.name``", and an instance variable hides a class variable with\nthe same name when accessed in this way. Class variables can be used\nas defaults for instance variables, but using mutable values there can\nlead to unexpected results.  For *new-style class*es, descriptors can\nbe used to create instance variables with different implementation\ndetails.\n\nClass definitions, like function definitions, may be wrapped by one or\nmore *decorator* expressions.  The evaluation rules for the decorator\nexpressions are the same as for functions.  The result must be a class\nobject, which is then bound to the class name.\n\n-[ Footnotes ]-\n\n[1] The exception is propagated to the invocation stack only if there\n    is no ``finally`` clause that negates the exception.\n\n[2] Currently, control "flows off the end" except in the case of an\n    exception or the execution of a ``return``, ``continue``, or\n    ``break`` statement.\n\n[3] A string literal appearing as the first statement in the function\n    body is transformed into the function\'s ``__doc__`` attribute and\n    therefore the function\'s *docstring*.\n\n[4] A string literal appearing as the first statement in the class\n    body is transformed into the namespace\'s ``__doc__`` item and\n    therefore the class\'s *docstring*.\n',
+ 'compound': u'\nCompound statements\n*******************\n\nCompound statements contain (groups of) other statements; they affect\nor control the execution of those other statements in some way.  In\ngeneral, compound statements span multiple lines, although in simple\nincarnations a whole compound statement may be contained in one line.\n\nThe ``if``, ``while`` and ``for`` statements implement traditional\ncontrol flow constructs.  ``try`` specifies exception handlers and/or\ncleanup code for a group of statements.  Function and class\ndefinitions are also syntactically compound statements.\n\nCompound statements consist of one or more \'clauses.\'  A clause\nconsists of a header and a \'suite.\'  The clause headers of a\nparticular compound statement are all at the same indentation level.\nEach clause header begins with a uniquely identifying keyword and ends\nwith a colon.  A suite is a group of statements controlled by a\nclause.  A suite can be one or more semicolon-separated simple\nstatements on the same line as the header, following the header\'s\ncolon, or it can be one or more indented statements on subsequent\nlines.  Only the latter form of suite can contain nested compound\nstatements; the following is illegal, mostly because it wouldn\'t be\nclear to which ``if`` clause a following ``else`` clause would belong:\n\n   if test1: if test2: print x\n\nAlso note that the semicolon binds tighter than the colon in this\ncontext, so that in the following example, either all or none of the\n``print`` statements are executed:\n\n   if x < y < z: print x; print y; print z\n\nSummarizing:\n\n   compound_stmt ::= if_stmt\n                     | while_stmt\n                     | for_stmt\n                     | try_stmt\n                     | with_stmt\n                     | funcdef\n                     | classdef\n                     | decorated\n   suite         ::= stmt_list NEWLINE | NEWLINE INDENT statement+ DEDENT\n   statement     ::= stmt_list NEWLINE | compound_stmt\n   stmt_list     ::= simple_stmt (";" simple_stmt)* [";"]\n\nNote that statements always end in a ``NEWLINE`` possibly followed by\na ``DEDENT``. Also note that optional continuation clauses always\nbegin with a keyword that cannot start a statement, thus there are no\nambiguities (the \'dangling ``else``\' problem is solved in Python by\nrequiring nested ``if`` statements to be indented).\n\nThe formatting of the grammar rules in the following sections places\neach clause on a separate line for clarity.\n\n\nThe ``if`` statement\n====================\n\nThe ``if`` statement is used for conditional execution:\n\n   if_stmt ::= "if" expression ":" suite\n               ( "elif" expression ":" suite )*\n               ["else" ":" suite]\n\nIt selects exactly one of the suites by evaluating the expressions one\nby one until one is found to be true (see section *Boolean operations*\nfor the definition of true and false); then that suite is executed\n(and no other part of the ``if`` statement is executed or evaluated).\nIf all expressions are false, the suite of the ``else`` clause, if\npresent, is executed.\n\n\nThe ``while`` statement\n=======================\n\nThe ``while`` statement is used for repeated execution as long as an\nexpression is true:\n\n   while_stmt ::= "while" expression ":" suite\n                  ["else" ":" suite]\n\nThis repeatedly tests the expression and, if it is true, executes the\nfirst suite; if the expression is false (which may be the first time\nit is tested) the suite of the ``else`` clause, if present, is\nexecuted and the loop terminates.\n\nA ``break`` statement executed in the first suite terminates the loop\nwithout executing the ``else`` clause\'s suite.  A ``continue``\nstatement executed in the first suite skips the rest of the suite and\ngoes back to testing the expression.\n\n\nThe ``for`` statement\n=====================\n\nThe ``for`` statement is used to iterate over the elements of a\nsequence (such as a string, tuple or list) or other iterable object:\n\n   for_stmt ::= "for" target_list "in" expression_list ":" suite\n                ["else" ":" suite]\n\nThe expression list is evaluated once; it should yield an iterable\nobject.  An iterator is created for the result of the\n``expression_list``.  The suite is then executed once for each item\nprovided by the iterator, in the order of ascending indices.  Each\nitem in turn is assigned to the target list using the standard rules\nfor assignments, and then the suite is executed.  When the items are\nexhausted (which is immediately when the sequence is empty), the suite\nin the ``else`` clause, if present, is executed, and the loop\nterminates.\n\nA ``break`` statement executed in the first suite terminates the loop\nwithout executing the ``else`` clause\'s suite.  A ``continue``\nstatement executed in the first suite skips the rest of the suite and\ncontinues with the next item, or with the ``else`` clause if there was\nno next item.\n\nThe suite may assign to the variable(s) in the target list; this does\nnot affect the next item assigned to it.\n\nThe target list is not deleted when the loop is finished, but if the\nsequence is empty, it will not have been assigned to at all by the\nloop.  Hint: the built-in function ``range()`` returns a sequence of\nintegers suitable to emulate the effect of Pascal\'s ``for i := a to b\ndo``; e.g., ``range(3)`` returns the list ``[0, 1, 2]``.\n\nNote: There is a subtlety when the sequence is being modified by the loop\n  (this can only occur for mutable sequences, i.e. lists). An internal\n  counter is used to keep track of which item is used next, and this\n  is incremented on each iteration.  When this counter has reached the\n  length of the sequence the loop terminates.  This means that if the\n  suite deletes the current (or a previous) item from the sequence,\n  the next item will be skipped (since it gets the index of the\n  current item which has already been treated).  Likewise, if the\n  suite inserts an item in the sequence before the current item, the\n  current item will be treated again the next time through the loop.\n  This can lead to nasty bugs that can be avoided by making a\n  temporary copy using a slice of the whole sequence, e.g.,\n\n     for x in a[:]:\n         if x < 0: a.remove(x)\n\n\nThe ``try`` statement\n=====================\n\nThe ``try`` statement specifies exception handlers and/or cleanup code\nfor a group of statements:\n\n   try_stmt  ::= try1_stmt | try2_stmt\n   try1_stmt ::= "try" ":" suite\n                 ("except" [expression [("as" | ",") target]] ":" suite)+\n                 ["else" ":" suite]\n                 ["finally" ":" suite]\n   try2_stmt ::= "try" ":" suite\n                 "finally" ":" suite\n\nChanged in version 2.5: In previous versions of Python,\n``try``...``except``...``finally`` did not work. ``try``...``except``\nhad to be nested in ``try``...``finally``.\n\nThe ``except`` clause(s) specify one or more exception handlers. When\nno exception occurs in the ``try`` clause, no exception handler is\nexecuted. When an exception occurs in the ``try`` suite, a search for\nan exception handler is started.  This search inspects the except\nclauses in turn until one is found that matches the exception.  An\nexpression-less except clause, if present, must be last; it matches\nany exception.  For an except clause with an expression, that\nexpression is evaluated, and the clause matches the exception if the\nresulting object is "compatible" with the exception.  An object is\ncompatible with an exception if it is the class or a base class of the\nexception object, a tuple containing an item compatible with the\nexception, or, in the (deprecated) case of string exceptions, is the\nraised string itself (note that the object identities must match, i.e.\nit must be the same string object, not just a string with the same\nvalue).\n\nIf no except clause matches the exception, the search for an exception\nhandler continues in the surrounding code and on the invocation stack.\n[1]\n\nIf the evaluation of an expression in the header of an except clause\nraises an exception, the original search for a handler is canceled and\na search starts for the new exception in the surrounding code and on\nthe call stack (it is treated as if the entire ``try`` statement\nraised the exception).\n\nWhen a matching except clause is found, the exception is assigned to\nthe target specified in that except clause, if present, and the except\nclause\'s suite is executed.  All except clauses must have an\nexecutable block.  When the end of this block is reached, execution\ncontinues normally after the entire try statement.  (This means that\nif two nested handlers exist for the same exception, and the exception\noccurs in the try clause of the inner handler, the outer handler will\nnot handle the exception.)\n\nBefore an except clause\'s suite is executed, details about the\nexception are assigned to three variables in the ``sys`` module:\n``sys.exc_type`` receives the object identifying the exception;\n``sys.exc_value`` receives the exception\'s parameter;\n``sys.exc_traceback`` receives a traceback object (see section *The\nstandard type hierarchy*) identifying the point in the program where\nthe exception occurred. These details are also available through the\n``sys.exc_info()`` function, which returns a tuple ``(exc_type,\nexc_value, exc_traceback)``.  Use of the corresponding variables is\ndeprecated in favor of this function, since their use is unsafe in a\nthreaded program.  As of Python 1.5, the variables are restored to\ntheir previous values (before the call) when returning from a function\nthat handled an exception.\n\nThe optional ``else`` clause is executed if and when control flows off\nthe end of the ``try`` clause. [2] Exceptions in the ``else`` clause\nare not handled by the preceding ``except`` clauses.\n\nIf ``finally`` is present, it specifies a \'cleanup\' handler.  The\n``try`` clause is executed, including any ``except`` and ``else``\nclauses.  If an exception occurs in any of the clauses and is not\nhandled, the exception is temporarily saved. The ``finally`` clause is\nexecuted.  If there is a saved exception, it is re-raised at the end\nof the ``finally`` clause. If the ``finally`` clause raises another\nexception or executes a ``return`` or ``break`` statement, the saved\nexception is lost.  The exception information is not available to the\nprogram during execution of the ``finally`` clause.\n\nWhen a ``return``, ``break`` or ``continue`` statement is executed in\nthe ``try`` suite of a ``try``...``finally`` statement, the\n``finally`` clause is also executed \'on the way out.\' A ``continue``\nstatement is illegal in the ``finally`` clause. (The reason is a\nproblem with the current implementation --- this restriction may be\nlifted in the future).\n\nAdditional information on exceptions can be found in section\n*Exceptions*, and information on using the ``raise`` statement to\ngenerate exceptions may be found in section *The raise statement*.\n\n\nThe ``with`` statement\n======================\n\nNew in version 2.5.\n\nThe ``with`` statement is used to wrap the execution of a block with\nmethods defined by a context manager (see section *With Statement\nContext Managers*). This allows common\n``try``...``except``...``finally`` usage patterns to be encapsulated\nfor convenient reuse.\n\n   with_stmt ::= "with" with_item ("," with_item)* ":" suite\n   with_item ::= expression ["as" target]\n\nThe execution of the ``with`` statement with one "item" proceeds as\nfollows:\n\n1. The context expression (the expression given in the **with_item**)\n   is evaluated to obtain a context manager.\n\n2. The context manager\'s ``__exit__()`` is loaded for later use.\n\n3. The context manager\'s ``__enter__()`` method is invoked.\n\n4. If a target was included in the ``with`` statement, the return\n   value from ``__enter__()`` is assigned to it.\n\n   Note: The ``with`` statement guarantees that if the ``__enter__()``\n     method returns without an error, then ``__exit__()`` will always\n     be called. Thus, if an error occurs during the assignment to the\n     target list, it will be treated the same as an error occurring\n     within the suite would be. See step 6 below.\n\n5. The suite is executed.\n\n6. The context manager\'s ``__exit__()`` method is invoked. If an\n   exception caused the suite to be exited, its type, value, and\n   traceback are passed as arguments to ``__exit__()``. Otherwise,\n   three ``None`` arguments are supplied.\n\n   If the suite was exited due to an exception, and the return value\n   from the ``__exit__()`` method was false, the exception is\n   reraised. If the return value was true, the exception is\n   suppressed, and execution continues with the statement following\n   the ``with`` statement.\n\n   If the suite was exited for any reason other than an exception, the\n   return value from ``__exit__()`` is ignored, and execution proceeds\n   at the normal location for the kind of exit that was taken.\n\nWith more than one item, the context managers are processed as if\nmultiple ``with`` statements were nested:\n\n   with A() as a, B() as b:\n       suite\n\nis equivalent to\n\n   with A() as a:\n       with B() as b:\n           suite\n\nNote: In Python 2.5, the ``with`` statement is only allowed when the\n  ``with_statement`` feature has been enabled.  It is always enabled\n  in Python 2.6.\n\nChanged in version 2.7: Support for multiple context expressions.\n\nSee also:\n\n   **PEP 0343** - The "with" statement\n      The specification, background, and examples for the Python\n      ``with`` statement.\n\n\nFunction definitions\n====================\n\nA function definition defines a user-defined function object (see\nsection *The standard type hierarchy*):\n\n   decorated      ::= decorators (classdef | funcdef)\n   decorators     ::= decorator+\n   decorator      ::= "@" dotted_name ["(" [argument_list [","]] ")"] NEWLINE\n   funcdef        ::= "def" funcname "(" [parameter_list] ")" ":" suite\n   dotted_name    ::= identifier ("." identifier)*\n   parameter_list ::= (defparameter ",")*\n                      (  "*" identifier [, "**" identifier]\n                      | "**" identifier\n                      | defparameter [","] )\n   defparameter   ::= parameter ["=" expression]\n   sublist        ::= parameter ("," parameter)* [","]\n   parameter      ::= identifier | "(" sublist ")"\n   funcname       ::= identifier\n\nA function definition is an executable statement.  Its execution binds\nthe function name in the current local namespace to a function object\n(a wrapper around the executable code for the function).  This\nfunction object contains a reference to the current global namespace\nas the global namespace to be used when the function is called.\n\nThe function definition does not execute the function body; this gets\nexecuted only when the function is called. [3]\n\nA function definition may be wrapped by one or more *decorator*\nexpressions. Decorator expressions are evaluated when the function is\ndefined, in the scope that contains the function definition.  The\nresult must be a callable, which is invoked with the function object\nas the only argument. The returned value is bound to the function name\ninstead of the function object.  Multiple decorators are applied in\nnested fashion. For example, the following code:\n\n   @f1(arg)\n   @f2\n   def func(): pass\n\nis equivalent to:\n\n   def func(): pass\n   func = f1(arg)(f2(func))\n\nWhen one or more top-level parameters have the form *parameter* ``=``\n*expression*, the function is said to have "default parameter values."\nFor a parameter with a default value, the corresponding argument may\nbe omitted from a call, in which case the parameter\'s default value is\nsubstituted.  If a parameter has a default value, all following\nparameters must also have a default value --- this is a syntactic\nrestriction that is not expressed by the grammar.\n\n**Default parameter values are evaluated when the function definition\nis executed.**  This means that the expression is evaluated once, when\nthe function is defined, and that that same "pre-computed" value is\nused for each call.  This is especially important to understand when a\ndefault parameter is a mutable object, such as a list or a dictionary:\nif the function modifies the object (e.g. by appending an item to a\nlist), the default value is in effect modified. This is generally not\nwhat was intended.  A way around this  is to use ``None`` as the\ndefault, and explicitly test for it in the body of the function, e.g.:\n\n   def whats_on_the_telly(penguin=None):\n       if penguin is None:\n           penguin = []\n       penguin.append("property of the zoo")\n       return penguin\n\nFunction call semantics are described in more detail in section\n*Calls*. A function call always assigns values to all parameters\nmentioned in the parameter list, either from position arguments, from\nkeyword arguments, or from default values.  If the form\n"``*identifier``" is present, it is initialized to a tuple receiving\nany excess positional parameters, defaulting to the empty tuple.  If\nthe form "``**identifier``" is present, it is initialized to a new\ndictionary receiving any excess keyword arguments, defaulting to a new\nempty dictionary.\n\nIt is also possible to create anonymous functions (functions not bound\nto a name), for immediate use in expressions.  This uses lambda forms,\ndescribed in section *Lambdas*.  Note that the lambda form is merely a\nshorthand for a simplified function definition; a function defined in\na "``def``" statement can be passed around or assigned to another name\njust like a function defined by a lambda form.  The "``def``" form is\nactually more powerful since it allows the execution of multiple\nstatements.\n\n**Programmer\'s note:** Functions are first-class objects.  A "``def``"\nform executed inside a function definition defines a local function\nthat can be returned or passed around.  Free variables used in the\nnested function can access the local variables of the function\ncontaining the def.  See section *Naming and binding* for details.\n\n\nClass definitions\n=================\n\nA class definition defines a class object (see section *The standard\ntype hierarchy*):\n\n   classdef    ::= "class" classname [inheritance] ":" suite\n   inheritance ::= "(" [expression_list] ")"\n   classname   ::= identifier\n\nA class definition is an executable statement.  It first evaluates the\ninheritance list, if present.  Each item in the inheritance list\nshould evaluate to a class object or class type which allows\nsubclassing.  The class\'s suite is then executed in a new execution\nframe (see section *Naming and binding*), using a newly created local\nnamespace and the original global namespace. (Usually, the suite\ncontains only function definitions.)  When the class\'s suite finishes\nexecution, its execution frame is discarded but its local namespace is\nsaved. [4] A class object is then created using the inheritance list\nfor the base classes and the saved local namespace for the attribute\ndictionary.  The class name is bound to this class object in the\noriginal local namespace.\n\n**Programmer\'s note:** Variables defined in the class definition are\nclass variables; they are shared by all instances.  To create instance\nvariables, they can be set in a method with ``self.name = value``.\nBoth class and instance variables are accessible through the notation\n"``self.name``", and an instance variable hides a class variable with\nthe same name when accessed in this way. Class variables can be used\nas defaults for instance variables, but using mutable values there can\nlead to unexpected results.  For *new-style class*es, descriptors can\nbe used to create instance variables with different implementation\ndetails.\n\nClass definitions, like function definitions, may be wrapped by one or\nmore *decorator* expressions.  The evaluation rules for the decorator\nexpressions are the same as for functions.  The result must be a class\nobject, which is then bound to the class name.\n\n-[ Footnotes ]-\n\n[1] The exception is propagated to the invocation stack only if there\n    is no ``finally`` clause that negates the exception.\n\n[2] Currently, control "flows off the end" except in the case of an\n    exception or the execution of a ``return``, ``continue``, or\n    ``break`` statement.\n\n[3] A string literal appearing as the first statement in the function\n    body is transformed into the function\'s ``__doc__`` attribute and\n    therefore the function\'s *docstring*.\n\n[4] A string literal appearing as the first statement in the class\n    body is transformed into the namespace\'s ``__doc__`` item and\n    therefore the class\'s *docstring*.\n',
  'context-managers': u'\nWith Statement Context Managers\n*******************************\n\nNew in version 2.5.\n\nA *context manager* is an object that defines the runtime context to\nbe established when executing a ``with`` statement. The context\nmanager handles the entry into, and the exit from, the desired runtime\ncontext for the execution of the block of code.  Context managers are\nnormally invoked using the ``with`` statement (described in section\n*The with statement*), but can also be used by directly invoking their\nmethods.\n\nTypical uses of context managers include saving and restoring various\nkinds of global state, locking and unlocking resources, closing opened\nfiles, etc.\n\nFor more information on context managers, see *Context Manager Types*.\n\nobject.__enter__(self)\n\n   Enter the runtime context related to this object. The ``with``\n   statement will bind this method\'s return value to the target(s)\n   specified in the ``as`` clause of the statement, if any.\n\nobject.__exit__(self, exc_type, exc_value, traceback)\n\n   Exit the runtime context related to this object. The parameters\n   describe the exception that caused the context to be exited. If the\n   context was exited without an exception, all three arguments will\n   be ``None``.\n\n   If an exception is supplied, and the method wishes to suppress the\n   exception (i.e., prevent it from being propagated), it should\n   return a true value. Otherwise, the exception will be processed\n   normally upon exit from this method.\n\n   Note that ``__exit__()`` methods should not reraise the passed-in\n   exception; this is the caller\'s responsibility.\n\nSee also:\n\n   **PEP 0343** - The "with" statement\n      The specification, background, and examples for the Python\n      ``with`` statement.\n',
  'continue': u'\nThe ``continue`` statement\n**************************\n\n   continue_stmt ::= "continue"\n\n``continue`` may only occur syntactically nested in a ``for`` or\n``while`` loop, but not nested in a function or class definition or\n``finally`` clause within that loop.  It continues with the next cycle\nof the nearest enclosing loop.\n\nWhen ``continue`` passes control out of a ``try`` statement with a\n``finally`` clause, that ``finally`` clause is executed before really\nstarting the next loop cycle.\n',
  'conversions': u'\nArithmetic conversions\n**********************\n\nWhen a description of an arithmetic operator below uses the phrase\n"the numeric arguments are converted to a common type," the arguments\nare coerced using the coercion rules listed at  *Coercion rules*.  If\nboth arguments are standard numeric types, the following coercions are\napplied:\n\n* If either argument is a complex number, the other is converted to\n  complex;\n\n* otherwise, if either argument is a floating point number, the other\n  is converted to floating point;\n\n* otherwise, if either argument is a long integer, the other is\n  converted to long integer;\n\n* otherwise, both must be plain integers and no conversion is\n  necessary.\n\nSome additional rules apply for certain operators (e.g., a string left\nargument to the \'%\' operator). Extensions can define their own\ncoercions.\n',
  'customization': u'\nBasic customization\n*******************\n\nobject.__new__(cls[, ...])\n\n   Called to create a new instance of class *cls*.  ``__new__()`` is a\n   static method (special-cased so you need not declare it as such)\n   that takes the class of which an instance was requested as its\n   first argument.  The remaining arguments are those passed to the\n   object constructor expression (the call to the class).  The return\n   value of ``__new__()`` should be the new object instance (usually\n   an instance of *cls*).\n\n   Typical implementations create a new instance of the class by\n   invoking the superclass\'s ``__new__()`` method using\n   ``super(currentclass, cls).__new__(cls[, ...])`` with appropriate\n   arguments and then modifying the newly-created instance as\n   necessary before returning it.\n\n   If ``__new__()`` returns an instance of *cls*, then the new\n   instance\'s ``__init__()`` method will be invoked like\n   ``__init__(self[, ...])``, where *self* is the new instance and the\n   remaining arguments are the same as were passed to ``__new__()``.\n\n   If ``__new__()`` does not return an instance of *cls*, then the new\n   instance\'s ``__init__()`` method will not be invoked.\n\n   ``__new__()`` is intended mainly to allow subclasses of immutable\n   types (like int, str, or tuple) to customize instance creation.  It\n   is also commonly overridden in custom metaclasses in order to\n   customize class creation.\n\nobject.__init__(self[, ...])\n\n   Called when the instance is created.  The arguments are those\n   passed to the class constructor expression.  If a base class has an\n   ``__init__()`` method, the derived class\'s ``__init__()`` method,\n   if any, must explicitly call it to ensure proper initialization of\n   the base class part of the instance; for example:\n   ``BaseClass.__init__(self, [args...])``.  As a special constraint\n   on constructors, no value may be returned; doing so will cause a\n   ``TypeError`` to be raised at runtime.\n\nobject.__del__(self)\n\n   Called when the instance is about to be destroyed.  This is also\n   called a destructor.  If a base class has a ``__del__()`` method,\n   the derived class\'s ``__del__()`` method, if any, must explicitly\n   call it to ensure proper deletion of the base class part of the\n   instance.  Note that it is possible (though not recommended!) for\n   the ``__del__()`` method to postpone destruction of the instance by\n   creating a new reference to it.  It may then be called at a later\n   time when this new reference is deleted.  It is not guaranteed that\n   ``__del__()`` methods are called for objects that still exist when\n   the interpreter exits.\n\n   Note: ``del x`` doesn\'t directly call ``x.__del__()`` --- the former\n     decrements the reference count for ``x`` by one, and the latter\n     is only called when ``x``\'s reference count reaches zero.  Some\n     common situations that may prevent the reference count of an\n     object from going to zero include: circular references between\n     objects (e.g., a doubly-linked list or a tree data structure with\n     parent and child pointers); a reference to the object on the\n     stack frame of a function that caught an exception (the traceback\n     stored in ``sys.exc_traceback`` keeps the stack frame alive); or\n     a reference to the object on the stack frame that raised an\n     unhandled exception in interactive mode (the traceback stored in\n     ``sys.last_traceback`` keeps the stack frame alive).  The first\n     situation can only be remedied by explicitly breaking the cycles;\n     the latter two situations can be resolved by storing ``None`` in\n     ``sys.exc_traceback`` or ``sys.last_traceback``.  Circular\n     references which are garbage are detected when the option cycle\n     detector is enabled (it\'s on by default), but can only be cleaned\n     up if there are no Python-level ``__del__()`` methods involved.\n     Refer to the documentation for the ``gc`` module for more\n     information about how ``__del__()`` methods are handled by the\n     cycle detector, particularly the description of the ``garbage``\n     value.\n\n   Warning: Due to the precarious circumstances under which ``__del__()``\n     methods are invoked, exceptions that occur during their execution\n     are ignored, and a warning is printed to ``sys.stderr`` instead.\n     Also, when ``__del__()`` is invoked in response to a module being\n     deleted (e.g., when execution of the program is done), other\n     globals referenced by the ``__del__()`` method may already have\n     been deleted or in the process of being torn down (e.g. the\n     import machinery shutting down).  For this reason, ``__del__()``\n     methods should do the absolute minimum needed to maintain\n     external invariants.  Starting with version 1.5, Python\n     guarantees that globals whose name begins with a single\n     underscore are deleted from their module before other globals are\n     deleted; if no other references to such globals exist, this may\n     help in assuring that imported modules are still available at the\n     time when the ``__del__()`` method is called.\n\nobject.__repr__(self)\n\n   Called by the ``repr()`` built-in function and by string\n   conversions (reverse quotes) to compute the "official" string\n   representation of an object.  If at all possible, this should look\n   like a valid Python expression that could be used to recreate an\n   object with the same value (given an appropriate environment).  If\n   this is not possible, a string of the form ``<...some useful\n   description...>`` should be returned.  The return value must be a\n   string object. If a class defines ``__repr__()`` but not\n   ``__str__()``, then ``__repr__()`` is also used when an "informal"\n   string representation of instances of that class is required.\n\n   This is typically used for debugging, so it is important that the\n   representation is information-rich and unambiguous.\n\nobject.__str__(self)\n\n   Called by the ``str()`` built-in function and by the ``print``\n   statement to compute the "informal" string representation of an\n   object.  This differs from ``__repr__()`` in that it does not have\n   to be a valid Python expression: a more convenient or concise\n   representation may be used instead. The return value must be a\n   string object.\n\nobject.__lt__(self, other)\nobject.__le__(self, other)\nobject.__eq__(self, other)\nobject.__ne__(self, other)\nobject.__gt__(self, other)\nobject.__ge__(self, other)\n\n   New in version 2.1.\n\n   These are the so-called "rich comparison" methods, and are called\n   for comparison operators in preference to ``__cmp__()`` below. The\n   correspondence between operator symbols and method names is as\n   follows: ``xy`` call ``x.__ne__(y)``, ``x>y`` calls ``x.__gt__(y)``, and\n   ``x>=y`` calls ``x.__ge__(y)``.\n\n   A rich comparison method may return the singleton\n   ``NotImplemented`` if it does not implement the operation for a\n   given pair of arguments. By convention, ``False`` and ``True`` are\n   returned for a successful comparison. However, these methods can\n   return any value, so if the comparison operator is used in a\n   Boolean context (e.g., in the condition of an ``if`` statement),\n   Python will call ``bool()`` on the value to determine if the result\n   is true or false.\n\n   There are no implied relationships among the comparison operators.\n   The truth of ``x==y`` does not imply that ``x!=y`` is false.\n   Accordingly, when defining ``__eq__()``, one should also define\n   ``__ne__()`` so that the operators will behave as expected.  See\n   the paragraph on ``__hash__()`` for some important notes on\n   creating *hashable* objects which support custom comparison\n   operations and are usable as dictionary keys.\n\n   There are no swapped-argument versions of these methods (to be used\n   when the left argument does not support the operation but the right\n   argument does); rather, ``__lt__()`` and ``__gt__()`` are each\n   other\'s reflection, ``__le__()`` and ``__ge__()`` are each other\'s\n   reflection, and ``__eq__()`` and ``__ne__()`` are their own\n   reflection.\n\n   Arguments to rich comparison methods are never coerced.\n\n   To automatically generate ordering operations from a single root\n   operation, see ``functools.total_ordering()``.\n\nobject.__cmp__(self, other)\n\n   Called by comparison operations if rich comparison (see above) is\n   not defined.  Should return a negative integer if ``self < other``,\n   zero if ``self == other``, a positive integer if ``self > other``.\n   If no ``__cmp__()``, ``__eq__()`` or ``__ne__()`` operation is\n   defined, class instances are compared by object identity\n   ("address").  See also the description of ``__hash__()`` for some\n   important notes on creating *hashable* objects which support custom\n   comparison operations and are usable as dictionary keys. (Note: the\n   restriction that exceptions are not propagated by ``__cmp__()`` has\n   been removed since Python 1.5.)\n\nobject.__rcmp__(self, other)\n\n   Changed in version 2.1: No longer supported.\n\nobject.__hash__(self)\n\n   Called by built-in function ``hash()`` and for operations on\n   members of hashed collections including ``set``, ``frozenset``, and\n   ``dict``.  ``__hash__()`` should return an integer.  The only\n   required property is that objects which compare equal have the same\n   hash value; it is advised to somehow mix together (e.g. using\n   exclusive or) the hash values for the components of the object that\n   also play a part in comparison of objects.\n\n   If a class does not define a ``__cmp__()`` or ``__eq__()`` method\n   it should not define a ``__hash__()`` operation either; if it\n   defines ``__cmp__()`` or ``__eq__()`` but not ``__hash__()``, its\n   instances will not be usable in hashed collections.  If a class\n   defines mutable objects and implements a ``__cmp__()`` or\n   ``__eq__()`` method, it should not implement ``__hash__()``, since\n   hashable collection implementations require that a object\'s hash\n   value is immutable (if the object\'s hash value changes, it will be\n   in the wrong hash bucket).\n\n   User-defined classes have ``__cmp__()`` and ``__hash__()`` methods\n   by default; with them, all objects compare unequal (except with\n   themselves) and ``x.__hash__()`` returns ``id(x)``.\n\n   Classes which inherit a ``__hash__()`` method from a parent class\n   but change the meaning of ``__cmp__()`` or ``__eq__()`` such that\n   the hash value returned is no longer appropriate (e.g. by switching\n   to a value-based concept of equality instead of the default\n   identity based equality) can explicitly flag themselves as being\n   unhashable by setting ``__hash__ = None`` in the class definition.\n   Doing so means that not only will instances of the class raise an\n   appropriate ``TypeError`` when a program attempts to retrieve their\n   hash value, but they will also be correctly identified as\n   unhashable when checking ``isinstance(obj, collections.Hashable)``\n   (unlike classes which define their own ``__hash__()`` to explicitly\n   raise ``TypeError``).\n\n   Changed in version 2.5: ``__hash__()`` may now also return a long\n   integer object; the 32-bit integer is then derived from the hash of\n   that object.\n\n   Changed in version 2.6: ``__hash__`` may now be set to ``None`` to\n   explicitly flag instances of a class as unhashable.\n\nobject.__nonzero__(self)\n\n   Called to implement truth value testing and the built-in operation\n   ``bool()``; should return ``False`` or ``True``, or their integer\n   equivalents ``0`` or ``1``.  When this method is not defined,\n   ``__len__()`` is called, if it is defined, and the object is\n   considered true if its result is nonzero. If a class defines\n   neither ``__len__()`` nor ``__nonzero__()``, all its instances are\n   considered true.\n\nobject.__unicode__(self)\n\n   Called to implement ``unicode()`` built-in; should return a Unicode\n   object. When this method is not defined, string conversion is\n   attempted, and the result of string conversion is converted to\n   Unicode using the system default encoding.\n',
- 'debugger': u'\n``pdb`` --- The Python Debugger\n*******************************\n\nThe module ``pdb`` defines an interactive source code debugger for\nPython programs.  It supports setting (conditional) breakpoints and\nsingle stepping at the source line level, inspection of stack frames,\nsource code listing, and evaluation of arbitrary Python code in the\ncontext of any stack frame.  It also supports post-mortem debugging\nand can be called under program control.\n\nThe debugger is extensible --- it is actually defined as the class\n``Pdb``. This is currently undocumented but easily understood by\nreading the source.  The extension interface uses the modules ``bdb``\nand ``cmd``.\n\nThe debugger\'s prompt is ``(Pdb)``. Typical usage to run a program\nunder control of the debugger is:\n\n   >>> import pdb\n   >>> import mymodule\n   >>> pdb.run(\'mymodule.test()\')\n   > (0)?()\n   (Pdb) continue\n   > (1)?()\n   (Pdb) continue\n   NameError: \'spam\'\n   > (1)?()\n   (Pdb)\n\n``pdb.py`` can also be invoked as a script to debug other scripts.\nFor example:\n\n   python -m pdb myscript.py\n\nWhen invoked as a script, pdb will automatically enter post-mortem\ndebugging if the program being debugged exits abnormally. After post-\nmortem debugging (or after normal exit of the program), pdb will\nrestart the program. Automatic restarting preserves pdb\'s state (such\nas breakpoints) and in most cases is more useful than quitting the\ndebugger upon program\'s exit.\n\nNew in version 2.4: Restarting post-mortem behavior added.\n\nThe typical usage to break into the debugger from a running program is\nto insert\n\n   import pdb; pdb.set_trace()\n\nat the location you want to break into the debugger.  You can then\nstep through the code following this statement, and continue running\nwithout the debugger using the ``c`` command.\n\nThe typical usage to inspect a crashed program is:\n\n   >>> import pdb\n   >>> import mymodule\n   >>> mymodule.test()\n   Traceback (most recent call last):\n     File "", line 1, in ?\n     File "./mymodule.py", line 4, in test\n       test2()\n     File "./mymodule.py", line 3, in test2\n       print spam\n   NameError: spam\n   >>> pdb.pm()\n   > ./mymodule.py(3)test2()\n   -> print spam\n   (Pdb)\n\nThe module defines the following functions; each enters the debugger\nin a slightly different way:\n\npdb.run(statement[, globals[, locals]])\n\n   Execute the *statement* (given as a string) under debugger control.\n   The debugger prompt appears before any code is executed; you can\n   set breakpoints and type ``continue``, or you can step through the\n   statement using ``step`` or ``next`` (all these commands are\n   explained below).  The optional *globals* and *locals* arguments\n   specify the environment in which the code is executed; by default\n   the dictionary of the module ``__main__`` is used.  (See the\n   explanation of the ``exec`` statement or the ``eval()`` built-in\n   function.)\n\npdb.runeval(expression[, globals[, locals]])\n\n   Evaluate the *expression* (given as a string) under debugger\n   control.  When ``runeval()`` returns, it returns the value of the\n   expression.  Otherwise this function is similar to ``run()``.\n\npdb.runcall(function[, argument, ...])\n\n   Call the *function* (a function or method object, not a string)\n   with the given arguments.  When ``runcall()`` returns, it returns\n   whatever the function call returned.  The debugger prompt appears\n   as soon as the function is entered.\n\npdb.set_trace()\n\n   Enter the debugger at the calling stack frame.  This is useful to\n   hard-code a breakpoint at a given point in a program, even if the\n   code is not otherwise being debugged (e.g. when an assertion\n   fails).\n\npdb.post_mortem([traceback])\n\n   Enter post-mortem debugging of the given *traceback* object.  If no\n   *traceback* is given, it uses the one of the exception that is\n   currently being handled (an exception must be being handled if the\n   default is to be used).\n\npdb.pm()\n\n   Enter post-mortem debugging of the traceback found in\n   ``sys.last_traceback``.\n\nThe ``run_*`` functions and ``set_trace()`` are aliases for\ninstantiating the ``Pdb`` class and calling the method of the same\nname.  If you want to access further features, you have to do this\nyourself:\n\nclass class pdb.Pdb(completekey=\'tab\', stdin=None, stdout=None, skip=None)\n\n   ``Pdb`` is the debugger class.\n\n   The *completekey*, *stdin* and *stdout* arguments are passed to the\n   underlying ``cmd.Cmd`` class; see the description there.\n\n   The *skip* argument, if given, must be an iterable of glob-style\n   module name patterns.  The debugger will not step into frames that\n   originate in a module that matches one of these patterns. [1]\n\n   Example call to enable tracing with *skip*:\n\n      import pdb; pdb.Pdb(skip=[\'django.*\']).set_trace()\n\n   New in version 2.7: The *skip* argument.\n\n   run(statement[, globals[, locals]])\n   runeval(expression[, globals[, locals]])\n   runcall(function[, argument, ...])\n   set_trace()\n\n      See the documentation for the functions explained above.\n',
+ 'debugger': u'\n``pdb`` --- The Python Debugger\n*******************************\n\nThe module ``pdb`` defines an interactive source code debugger for\nPython programs.  It supports setting (conditional) breakpoints and\nsingle stepping at the source line level, inspection of stack frames,\nsource code listing, and evaluation of arbitrary Python code in the\ncontext of any stack frame.  It also supports post-mortem debugging\nand can be called under program control.\n\nThe debugger is extensible --- it is actually defined as the class\n``Pdb``. This is currently undocumented but easily understood by\nreading the source.  The extension interface uses the modules ``bdb``\nand ``cmd``.\n\nThe debugger\'s prompt is ``(Pdb)``. Typical usage to run a program\nunder control of the debugger is:\n\n   >>> import pdb\n   >>> import mymodule\n   >>> pdb.run(\'mymodule.test()\')\n   > (0)?()\n   (Pdb) continue\n   > (1)?()\n   (Pdb) continue\n   NameError: \'spam\'\n   > (1)?()\n   (Pdb)\n\n``pdb.py`` can also be invoked as a script to debug other scripts.\nFor example:\n\n   python -m pdb myscript.py\n\nWhen invoked as a script, pdb will automatically enter post-mortem\ndebugging if the program being debugged exits abnormally. After post-\nmortem debugging (or after normal exit of the program), pdb will\nrestart the program. Automatic restarting preserves pdb\'s state (such\nas breakpoints) and in most cases is more useful than quitting the\ndebugger upon program\'s exit.\n\nNew in version 2.4: Restarting post-mortem behavior added.\n\nThe typical usage to break into the debugger from a running program is\nto insert\n\n   import pdb; pdb.set_trace()\n\nat the location you want to break into the debugger.  You can then\nstep through the code following this statement, and continue running\nwithout the debugger using the ``c`` command.\n\nThe typical usage to inspect a crashed program is:\n\n   >>> import pdb\n   >>> import mymodule\n   >>> mymodule.test()\n   Traceback (most recent call last):\n     File "", line 1, in ?\n     File "./mymodule.py", line 4, in test\n       test2()\n     File "./mymodule.py", line 3, in test2\n       print spam\n   NameError: spam\n   >>> pdb.pm()\n   > ./mymodule.py(3)test2()\n   -> print spam\n   (Pdb)\n\nThe module defines the following functions; each enters the debugger\nin a slightly different way:\n\npdb.run(statement[, globals[, locals]])\n\n   Execute the *statement* (given as a string) under debugger control.\n   The debugger prompt appears before any code is executed; you can\n   set breakpoints and type ``continue``, or you can step through the\n   statement using ``step`` or ``next`` (all these commands are\n   explained below).  The optional *globals* and *locals* arguments\n   specify the environment in which the code is executed; by default\n   the dictionary of the module ``__main__`` is used.  (See the\n   explanation of the ``exec`` statement or the ``eval()`` built-in\n   function.)\n\npdb.runeval(expression[, globals[, locals]])\n\n   Evaluate the *expression* (given as a string) under debugger\n   control.  When ``runeval()`` returns, it returns the value of the\n   expression.  Otherwise this function is similar to ``run()``.\n\npdb.runcall(function[, argument, ...])\n\n   Call the *function* (a function or method object, not a string)\n   with the given arguments.  When ``runcall()`` returns, it returns\n   whatever the function call returned.  The debugger prompt appears\n   as soon as the function is entered.\n\npdb.set_trace()\n\n   Enter the debugger at the calling stack frame.  This is useful to\n   hard-code a breakpoint at a given point in a program, even if the\n   code is not otherwise being debugged (e.g. when an assertion\n   fails).\n\npdb.post_mortem([traceback])\n\n   Enter post-mortem debugging of the given *traceback* object.  If no\n   *traceback* is given, it uses the one of the exception that is\n   currently being handled (an exception must be being handled if the\n   default is to be used).\n\npdb.pm()\n\n   Enter post-mortem debugging of the traceback found in\n   ``sys.last_traceback``.\n\nThe ``run*`` functions and ``set_trace()`` are aliases for\ninstantiating the ``Pdb`` class and calling the method of the same\nname.  If you want to access further features, you have to do this\nyourself:\n\nclass class pdb.Pdb(completekey=\'tab\', stdin=None, stdout=None, skip=None)\n\n   ``Pdb`` is the debugger class.\n\n   The *completekey*, *stdin* and *stdout* arguments are passed to the\n   underlying ``cmd.Cmd`` class; see the description there.\n\n   The *skip* argument, if given, must be an iterable of glob-style\n   module name patterns.  The debugger will not step into frames that\n   originate in a module that matches one of these patterns. [1]\n\n   Example call to enable tracing with *skip*:\n\n      import pdb; pdb.Pdb(skip=[\'django.*\']).set_trace()\n\n   New in version 2.7: The *skip* argument.\n\n   run(statement[, globals[, locals]])\n   runeval(expression[, globals[, locals]])\n   runcall(function[, argument, ...])\n   set_trace()\n\n      See the documentation for the functions explained above.\n',
  'del': u'\nThe ``del`` statement\n*********************\n\n   del_stmt ::= "del" target_list\n\nDeletion is recursively defined very similar to the way assignment is\ndefined. Rather that spelling it out in full details, here are some\nhints.\n\nDeletion of a target list recursively deletes each target, from left\nto right.\n\nDeletion of a name removes the binding of that name  from the local or\nglobal namespace, depending on whether the name occurs in a ``global``\nstatement in the same code block.  If the name is unbound, a\n``NameError`` exception will be raised.\n\nIt is illegal to delete a name from the local namespace if it occurs\nas a free variable in a nested block.\n\nDeletion of attribute references, subscriptions and slicings is passed\nto the primary object involved; deletion of a slicing is in general\nequivalent to assignment of an empty slice of the right type (but even\nthis is determined by the sliced object).\n',
  'dict': u'\nDictionary displays\n*******************\n\nA dictionary display is a possibly empty series of key/datum pairs\nenclosed in curly braces:\n\n   dict_display       ::= "{" [key_datum_list | dict_comprehension] "}"\n   key_datum_list     ::= key_datum ("," key_datum)* [","]\n   key_datum          ::= expression ":" expression\n   dict_comprehension ::= expression ":" expression comp_for\n\nA dictionary display yields a new dictionary object.\n\nIf a comma-separated sequence of key/datum pairs is given, they are\nevaluated from left to right to define the entries of the dictionary:\neach key object is used as a key into the dictionary to store the\ncorresponding datum.  This means that you can specify the same key\nmultiple times in the key/datum list, and the final dictionary\'s value\nfor that key will be the last one given.\n\nA dict comprehension, in contrast to list and set comprehensions,\nneeds two expressions separated with a colon followed by the usual\n"for" and "if" clauses. When the comprehension is run, the resulting\nkey and value elements are inserted in the new dictionary in the order\nthey are produced.\n\nRestrictions on the types of the key values are listed earlier in\nsection *The standard type hierarchy*.  (To summarize, the key type\nshould be *hashable*, which excludes all mutable objects.)  Clashes\nbetween duplicate keys are not detected; the last datum (textually\nrightmost in the display) stored for a given key value prevails.\n',
  'dynamic-features': u'\nInteraction with dynamic features\n*********************************\n\nThere are several cases where Python statements are illegal when used\nin conjunction with nested scopes that contain free variables.\n\nIf a variable is referenced in an enclosing scope, it is illegal to\ndelete the name.  An error will be reported at compile time.\n\nIf the wild card form of import --- ``import *`` --- is used in a\nfunction and the function contains or is a nested block with free\nvariables, the compiler will raise a ``SyntaxError``.\n\nIf ``exec`` is used in a function and the function contains or is a\nnested block with free variables, the compiler will raise a\n``SyntaxError`` unless the exec explicitly specifies the local\nnamespace for the ``exec``.  (In other words, ``exec obj`` would be\nillegal, but ``exec obj in ns`` would be legal.)\n\nThe ``eval()``, ``execfile()``, and ``input()`` functions and the\n``exec`` statement do not have access to the full environment for\nresolving names.  Names may be resolved in the local and global\nnamespaces of the caller.  Free variables are not resolved in the\nnearest enclosing namespace, but in the global namespace. [1] The\n``exec`` statement and the ``eval()`` and ``execfile()`` functions\nhave optional arguments to override the global and local namespace.\nIf only one namespace is specified, it is used for both.\n',
  'else': u'\nThe ``if`` statement\n********************\n\nThe ``if`` statement is used for conditional execution:\n\n   if_stmt ::= "if" expression ":" suite\n               ( "elif" expression ":" suite )*\n               ["else" ":" suite]\n\nIt selects exactly one of the suites by evaluating the expressions one\nby one until one is found to be true (see section *Boolean operations*\nfor the definition of true and false); then that suite is executed\n(and no other part of the ``if`` statement is executed or evaluated).\nIf all expressions are false, the suite of the ``else`` clause, if\npresent, is executed.\n',
  'exceptions': u'\nExceptions\n**********\n\nExceptions are a means of breaking out of the normal flow of control\nof a code block in order to handle errors or other exceptional\nconditions.  An exception is *raised* at the point where the error is\ndetected; it may be *handled* by the surrounding code block or by any\ncode block that directly or indirectly invoked the code block where\nthe error occurred.\n\nThe Python interpreter raises an exception when it detects a run-time\nerror (such as division by zero).  A Python program can also\nexplicitly raise an exception with the ``raise`` statement. Exception\nhandlers are specified with the ``try`` ... ``except`` statement.  The\n``finally`` clause of such a statement can be used to specify cleanup\ncode which does not handle the exception, but is executed whether an\nexception occurred or not in the preceding code.\n\nPython uses the "termination" model of error handling: an exception\nhandler can find out what happened and continue execution at an outer\nlevel, but it cannot repair the cause of the error and retry the\nfailing operation (except by re-entering the offending piece of code\nfrom the top).\n\nWhen an exception is not handled at all, the interpreter terminates\nexecution of the program, or returns to its interactive main loop.  In\neither case, it prints a stack backtrace, except when the exception is\n``SystemExit``.\n\nExceptions are identified by class instances.  The ``except`` clause\nis selected depending on the class of the instance: it must reference\nthe class of the instance or a base class thereof.  The instance can\nbe received by the handler and can carry additional information about\nthe exceptional condition.\n\nExceptions can also be identified by strings, in which case the\n``except`` clause is selected by object identity.  An arbitrary value\ncan be raised along with the identifying string which can be passed to\nthe handler.\n\nNote: Messages to exceptions are not part of the Python API.  Their\n  contents may change from one version of Python to the next without\n  warning and should not be relied on by code which will run under\n  multiple versions of the interpreter.\n\nSee also the description of the ``try`` statement in section *The try\nstatement* and ``raise`` statement in section *The raise statement*.\n\n-[ Footnotes ]-\n\n[1] This limitation occurs because the code that is executed by these\n    operations is not available at the time the module is compiled.\n',
  'exec': u'\nThe ``exec`` statement\n**********************\n\n   exec_stmt ::= "exec" or_expr ["in" expression ["," expression]]\n\nThis statement supports dynamic execution of Python code.  The first\nexpression should evaluate to either a string, an open file object, or\na code object.  If it is a string, the string is parsed as a suite of\nPython statements which is then executed (unless a syntax error\noccurs). [1]  If it is an open file, the file is parsed until EOF and\nexecuted.  If it is a code object, it is simply executed.  In all\ncases, the code that\'s executed is expected to be valid as file input\n(see section *File input*).  Be aware that the ``return`` and\n``yield`` statements may not be used outside of function definitions\neven within the context of code passed to the ``exec`` statement.\n\nIn all cases, if the optional parts are omitted, the code is executed\nin the current scope.  If only the first expression after ``in`` is\nspecified, it should be a dictionary, which will be used for both the\nglobal and the local variables.  If two expressions are given, they\nare used for the global and local variables, respectively. If\nprovided, *locals* can be any mapping object.\n\nChanged in version 2.4: Formerly, *locals* was required to be a\ndictionary.\n\nAs a side effect, an implementation may insert additional keys into\nthe dictionaries given besides those corresponding to variable names\nset by the executed code.  For example, the current implementation may\nadd a reference to the dictionary of the built-in module\n``__builtin__`` under the key ``__builtins__`` (!).\n\n**Programmer\'s hints:** dynamic evaluation of expressions is supported\nby the built-in function ``eval()``.  The built-in functions\n``globals()`` and ``locals()`` return the current global and local\ndictionary, respectively, which may be useful to pass around for use\nby ``exec``.\n\n-[ Footnotes ]-\n\n[1] Note that the parser only accepts the Unix-style end of line\n    convention. If you are reading the code from a file, make sure to\n    use universal newline mode to convert Windows or Mac-style\n    newlines.\n',
- 'execmodel': u'\nExecution model\n***************\n\n\nNaming and binding\n==================\n\n*Names* refer to objects.  Names are introduced by name binding\noperations. Each occurrence of a name in the program text refers to\nthe *binding* of that name established in the innermost function block\ncontaining the use.\n\nA *block* is a piece of Python program text that is executed as a\nunit. The following are blocks: a module, a function body, and a class\ndefinition. Each command typed interactively is a block.  A script\nfile (a file given as standard input to the interpreter or specified\non the interpreter command line the first argument) is a code block.\nA script command (a command specified on the interpreter command line\nwith the \'**-c**\' option) is a code block.  The file read by the\nbuilt-in function ``execfile()`` is a code block.  The string argument\npassed to the built-in function ``eval()`` and to the ``exec``\nstatement is a code block. The expression read and evaluated by the\nbuilt-in function ``input()`` is a code block.\n\nA code block is executed in an *execution frame*.  A frame contains\nsome administrative information (used for debugging) and determines\nwhere and how execution continues after the code block\'s execution has\ncompleted.\n\nA *scope* defines the visibility of a name within a block.  If a local\nvariable is defined in a block, its scope includes that block.  If the\ndefinition occurs in a function block, the scope extends to any blocks\ncontained within the defining one, unless a contained block introduces\na different binding for the name.  The scope of names defined in a\nclass block is limited to the class block; it does not extend to the\ncode blocks of methods -- this includes generator expressions since\nthey are implemented using a function scope.  This means that the\nfollowing will fail:\n\n   class A:\n       a = 42\n       b = list(a + i for i in range(10))\n\nWhen a name is used in a code block, it is resolved using the nearest\nenclosing scope.  The set of all such scopes visible to a code block\nis called the block\'s *environment*.\n\nIf a name is bound in a block, it is a local variable of that block.\nIf a name is bound at the module level, it is a global variable.  (The\nvariables of the module code block are local and global.)  If a\nvariable is used in a code block but not defined there, it is a *free\nvariable*.\n\nWhen a name is not found at all, a ``NameError`` exception is raised.\nIf the name refers to a local variable that has not been bound, a\n``UnboundLocalError`` exception is raised.  ``UnboundLocalError`` is a\nsubclass of ``NameError``.\n\nThe following constructs bind names: formal parameters to functions,\n``import`` statements, class and function definitions (these bind the\nclass or function name in the defining block), and targets that are\nidentifiers if occurring in an assignment, ``for`` loop header, in the\nsecond position of an ``except`` clause header or after ``as`` in a\n``with`` statement.  The ``import`` statement of the form ``from ...\nimport *`` binds all names defined in the imported module, except\nthose beginning with an underscore.  This form may only be used at the\nmodule level.\n\nA target occurring in a ``del`` statement is also considered bound for\nthis purpose (though the actual semantics are to unbind the name).  It\nis illegal to unbind a name that is referenced by an enclosing scope;\nthe compiler will report a ``SyntaxError``.\n\nEach assignment or import statement occurs within a block defined by a\nclass or function definition or at the module level (the top-level\ncode block).\n\nIf a name binding operation occurs anywhere within a code block, all\nuses of the name within the block are treated as references to the\ncurrent block.  This can lead to errors when a name is used within a\nblock before it is bound. This rule is subtle.  Python lacks\ndeclarations and allows name binding operations to occur anywhere\nwithin a code block.  The local variables of a code block can be\ndetermined by scanning the entire text of the block for name binding\noperations.\n\nIf the global statement occurs within a block, all uses of the name\nspecified in the statement refer to the binding of that name in the\ntop-level namespace. Names are resolved in the top-level namespace by\nsearching the global namespace, i.e. the namespace of the module\ncontaining the code block, and the builtins namespace, the namespace\nof the module ``__builtin__``.  The global namespace is searched\nfirst.  If the name is not found there, the builtins namespace is\nsearched.  The global statement must precede all uses of the name.\n\nThe builtins namespace associated with the execution of a code block\nis actually found by looking up the name ``__builtins__`` in its\nglobal namespace; this should be a dictionary or a module (in the\nlatter case the module\'s dictionary is used).  By default, when in the\n``__main__`` module, ``__builtins__`` is the built-in module\n``__builtin__`` (note: no \'s\'); when in any other module,\n``__builtins__`` is an alias for the dictionary of the ``__builtin__``\nmodule itself.  ``__builtins__`` can be set to a user-created\ndictionary to create a weak form of restricted execution.\n\n**CPython implementation detail:** Users should not touch\n``__builtins__``; it is strictly an implementation detail.  Users\nwanting to override values in the builtins namespace should ``import``\nthe ``__builtin__`` (no \'s\') module and modify its attributes\nappropriately.\n\nThe namespace for a module is automatically created the first time a\nmodule is imported.  The main module for a script is always called\n``__main__``.\n\nThe global statement has the same scope as a name binding operation in\nthe same block.  If the nearest enclosing scope for a free variable\ncontains a global statement, the free variable is treated as a global.\n\nA class definition is an executable statement that may use and define\nnames. These references follow the normal rules for name resolution.\nThe namespace of the class definition becomes the attribute dictionary\nof the class.  Names defined at the class scope are not visible in\nmethods.\n\n\nInteraction with dynamic features\n---------------------------------\n\nThere are several cases where Python statements are illegal when used\nin conjunction with nested scopes that contain free variables.\n\nIf a variable is referenced in an enclosing scope, it is illegal to\ndelete the name.  An error will be reported at compile time.\n\nIf the wild card form of import --- ``import *`` --- is used in a\nfunction and the function contains or is a nested block with free\nvariables, the compiler will raise a ``SyntaxError``.\n\nIf ``exec`` is used in a function and the function contains or is a\nnested block with free variables, the compiler will raise a\n``SyntaxError`` unless the exec explicitly specifies the local\nnamespace for the ``exec``.  (In other words, ``exec obj`` would be\nillegal, but ``exec obj in ns`` would be legal.)\n\nThe ``eval()``, ``execfile()``, and ``input()`` functions and the\n``exec`` statement do not have access to the full environment for\nresolving names.  Names may be resolved in the local and global\nnamespaces of the caller.  Free variables are not resolved in the\nnearest enclosing namespace, but in the global namespace. [1] The\n``exec`` statement and the ``eval()`` and ``execfile()`` functions\nhave optional arguments to override the global and local namespace.\nIf only one namespace is specified, it is used for both.\n\n\nExceptions\n==========\n\nExceptions are a means of breaking out of the normal flow of control\nof a code block in order to handle errors or other exceptional\nconditions.  An exception is *raised* at the point where the error is\ndetected; it may be *handled* by the surrounding code block or by any\ncode block that directly or indirectly invoked the code block where\nthe error occurred.\n\nThe Python interpreter raises an exception when it detects a run-time\nerror (such as division by zero).  A Python program can also\nexplicitly raise an exception with the ``raise`` statement. Exception\nhandlers are specified with the ``try`` ... ``except`` statement.  The\n``finally`` clause of such a statement can be used to specify cleanup\ncode which does not handle the exception, but is executed whether an\nexception occurred or not in the preceding code.\n\nPython uses the "termination" model of error handling: an exception\nhandler can find out what happened and continue execution at an outer\nlevel, but it cannot repair the cause of the error and retry the\nfailing operation (except by re-entering the offending piece of code\nfrom the top).\n\nWhen an exception is not handled at all, the interpreter terminates\nexecution of the program, or returns to its interactive main loop.  In\neither case, it prints a stack backtrace, except when the exception is\n``SystemExit``.\n\nExceptions are identified by class instances.  The ``except`` clause\nis selected depending on the class of the instance: it must reference\nthe class of the instance or a base class thereof.  The instance can\nbe received by the handler and can carry additional information about\nthe exceptional condition.\n\nExceptions can also be identified by strings, in which case the\n``except`` clause is selected by object identity.  An arbitrary value\ncan be raised along with the identifying string which can be passed to\nthe handler.\n\nNote: Messages to exceptions are not part of the Python API.  Their\n  contents may change from one version of Python to the next without\n  warning and should not be relied on by code which will run under\n  multiple versions of the interpreter.\n\nSee also the description of the ``try`` statement in section *The try\nstatement* and ``raise`` statement in section *The raise statement*.\n\n-[ Footnotes ]-\n\n[1] This limitation occurs because the code that is executed by these\n    operations is not available at the time the module is compiled.\n',
+ 'execmodel': u'\nExecution model\n***************\n\n\nNaming and binding\n==================\n\n*Names* refer to objects.  Names are introduced by name binding\noperations. Each occurrence of a name in the program text refers to\nthe *binding* of that name established in the innermost function block\ncontaining the use.\n\nA *block* is a piece of Python program text that is executed as a\nunit. The following are blocks: a module, a function body, and a class\ndefinition. Each command typed interactively is a block.  A script\nfile (a file given as standard input to the interpreter or specified\non the interpreter command line the first argument) is a code block.\nA script command (a command specified on the interpreter command line\nwith the \'**-c**\' option) is a code block.  The file read by the\nbuilt-in function ``execfile()`` is a code block.  The string argument\npassed to the built-in function ``eval()`` and to the ``exec``\nstatement is a code block. The expression read and evaluated by the\nbuilt-in function ``input()`` is a code block.\n\nA code block is executed in an *execution frame*.  A frame contains\nsome administrative information (used for debugging) and determines\nwhere and how execution continues after the code block\'s execution has\ncompleted.\n\nA *scope* defines the visibility of a name within a block.  If a local\nvariable is defined in a block, its scope includes that block.  If the\ndefinition occurs in a function block, the scope extends to any blocks\ncontained within the defining one, unless a contained block introduces\na different binding for the name.  The scope of names defined in a\nclass block is limited to the class block; it does not extend to the\ncode blocks of methods -- this includes generator expressions since\nthey are implemented using a function scope.  This means that the\nfollowing will fail:\n\n   class A:\n       a = 42\n       b = list(a + i for i in range(10))\n\nWhen a name is used in a code block, it is resolved using the nearest\nenclosing scope.  The set of all such scopes visible to a code block\nis called the block\'s *environment*.\n\nIf a name is bound in a block, it is a local variable of that block.\nIf a name is bound at the module level, it is a global variable.  (The\nvariables of the module code block are local and global.)  If a\nvariable is used in a code block but not defined there, it is a *free\nvariable*.\n\nWhen a name is not found at all, a ``NameError`` exception is raised.\nIf the name refers to a local variable that has not been bound, a\n``UnboundLocalError`` exception is raised.  ``UnboundLocalError`` is a\nsubclass of ``NameError``.\n\nThe following constructs bind names: formal parameters to functions,\n``import`` statements, class and function definitions (these bind the\nclass or function name in the defining block), and targets that are\nidentifiers if occurring in an assignment, ``for`` loop header, in the\nsecond position of an ``except`` clause header or after ``as`` in a\n``with`` statement.  The ``import`` statement of the form ``from ...\nimport *`` binds all names defined in the imported module, except\nthose beginning with an underscore.  This form may only be used at the\nmodule level.\n\nA target occurring in a ``del`` statement is also considered bound for\nthis purpose (though the actual semantics are to unbind the name).  It\nis illegal to unbind a name that is referenced by an enclosing scope;\nthe compiler will report a ``SyntaxError``.\n\nEach assignment or import statement occurs within a block defined by a\nclass or function definition or at the module level (the top-level\ncode block).\n\nIf a name binding operation occurs anywhere within a code block, all\nuses of the name within the block are treated as references to the\ncurrent block.  This can lead to errors when a name is used within a\nblock before it is bound. This rule is subtle.  Python lacks\ndeclarations and allows name binding operations to occur anywhere\nwithin a code block.  The local variables of a code block can be\ndetermined by scanning the entire text of the block for name binding\noperations.\n\nIf the global statement occurs within a block, all uses of the name\nspecified in the statement refer to the binding of that name in the\ntop-level namespace. Names are resolved in the top-level namespace by\nsearching the global namespace, i.e. the namespace of the module\ncontaining the code block, and the builtins namespace, the namespace\nof the module ``__builtin__``.  The global namespace is searched\nfirst.  If the name is not found there, the builtins namespace is\nsearched.  The global statement must precede all uses of the name.\n\nThe builtins namespace associated with the execution of a code block\nis actually found by looking up the name ``__builtins__`` in its\nglobal namespace; this should be a dictionary or a module (in the\nlatter case the module\'s dictionary is used).  By default, when in the\n``__main__`` module, ``__builtins__`` is the built-in module\n``__builtin__`` (note: no \'s\'); when in any other module,\n``__builtins__`` is an alias for the dictionary of the ``__builtin__``\nmodule itself.  ``__builtins__`` can be set to a user-created\ndictionary to create a weak form of restricted execution.\n\n**CPython implementation detail:** Users should not touch\n``__builtins__``; it is strictly an implementation detail.  Users\nwanting to override values in the builtins namespace should ``import``\nthe ``__builtin__`` (no \'s\') module and modify its attributes\nappropriately.\n\nThe namespace for a module is automatically created the first time a\nmodule is imported.  The main module for a script is always called\n``__main__``.\n\nThe ``global`` statement has the same scope as a name binding\noperation in the same block.  If the nearest enclosing scope for a\nfree variable contains a global statement, the free variable is\ntreated as a global.\n\nA class definition is an executable statement that may use and define\nnames. These references follow the normal rules for name resolution.\nThe namespace of the class definition becomes the attribute dictionary\nof the class.  Names defined at the class scope are not visible in\nmethods.\n\n\nInteraction with dynamic features\n---------------------------------\n\nThere are several cases where Python statements are illegal when used\nin conjunction with nested scopes that contain free variables.\n\nIf a variable is referenced in an enclosing scope, it is illegal to\ndelete the name.  An error will be reported at compile time.\n\nIf the wild card form of import --- ``import *`` --- is used in a\nfunction and the function contains or is a nested block with free\nvariables, the compiler will raise a ``SyntaxError``.\n\nIf ``exec`` is used in a function and the function contains or is a\nnested block with free variables, the compiler will raise a\n``SyntaxError`` unless the exec explicitly specifies the local\nnamespace for the ``exec``.  (In other words, ``exec obj`` would be\nillegal, but ``exec obj in ns`` would be legal.)\n\nThe ``eval()``, ``execfile()``, and ``input()`` functions and the\n``exec`` statement do not have access to the full environment for\nresolving names.  Names may be resolved in the local and global\nnamespaces of the caller.  Free variables are not resolved in the\nnearest enclosing namespace, but in the global namespace. [1] The\n``exec`` statement and the ``eval()`` and ``execfile()`` functions\nhave optional arguments to override the global and local namespace.\nIf only one namespace is specified, it is used for both.\n\n\nExceptions\n==========\n\nExceptions are a means of breaking out of the normal flow of control\nof a code block in order to handle errors or other exceptional\nconditions.  An exception is *raised* at the point where the error is\ndetected; it may be *handled* by the surrounding code block or by any\ncode block that directly or indirectly invoked the code block where\nthe error occurred.\n\nThe Python interpreter raises an exception when it detects a run-time\nerror (such as division by zero).  A Python program can also\nexplicitly raise an exception with the ``raise`` statement. Exception\nhandlers are specified with the ``try`` ... ``except`` statement.  The\n``finally`` clause of such a statement can be used to specify cleanup\ncode which does not handle the exception, but is executed whether an\nexception occurred or not in the preceding code.\n\nPython uses the "termination" model of error handling: an exception\nhandler can find out what happened and continue execution at an outer\nlevel, but it cannot repair the cause of the error and retry the\nfailing operation (except by re-entering the offending piece of code\nfrom the top).\n\nWhen an exception is not handled at all, the interpreter terminates\nexecution of the program, or returns to its interactive main loop.  In\neither case, it prints a stack backtrace, except when the exception is\n``SystemExit``.\n\nExceptions are identified by class instances.  The ``except`` clause\nis selected depending on the class of the instance: it must reference\nthe class of the instance or a base class thereof.  The instance can\nbe received by the handler and can carry additional information about\nthe exceptional condition.\n\nExceptions can also be identified by strings, in which case the\n``except`` clause is selected by object identity.  An arbitrary value\ncan be raised along with the identifying string which can be passed to\nthe handler.\n\nNote: Messages to exceptions are not part of the Python API.  Their\n  contents may change from one version of Python to the next without\n  warning and should not be relied on by code which will run under\n  multiple versions of the interpreter.\n\nSee also the description of the ``try`` statement in section *The try\nstatement* and ``raise`` statement in section *The raise statement*.\n\n-[ Footnotes ]-\n\n[1] This limitation occurs because the code that is executed by these\n    operations is not available at the time the module is compiled.\n',
  'exprlists': u'\nExpression lists\n****************\n\n   expression_list ::= expression ( "," expression )* [","]\n\nAn expression list containing at least one comma yields a tuple.  The\nlength of the tuple is the number of expressions in the list.  The\nexpressions are evaluated from left to right.\n\nThe trailing comma is required only to create a single tuple (a.k.a. a\n*singleton*); it is optional in all other cases.  A single expression\nwithout a trailing comma doesn\'t create a tuple, but rather yields the\nvalue of that expression. (To create an empty tuple, use an empty pair\nof parentheses: ``()``.)\n',
  'floating': u'\nFloating point literals\n***********************\n\nFloating point literals are described by the following lexical\ndefinitions:\n\n   floatnumber   ::= pointfloat | exponentfloat\n   pointfloat    ::= [intpart] fraction | intpart "."\n   exponentfloat ::= (intpart | pointfloat) exponent\n   intpart       ::= digit+\n   fraction      ::= "." digit+\n   exponent      ::= ("e" | "E") ["+" | "-"] digit+\n\nNote that the integer and exponent parts of floating point numbers can\nlook like octal integers, but are interpreted using radix 10.  For\nexample, ``077e010`` is legal, and denotes the same number as\n``77e10``. The allowed range of floating point literals is\nimplementation-dependent. Some examples of floating point literals:\n\n   3.14    10.    .001    1e100    3.14e-10    0e0\n\nNote that numeric literals do not include a sign; a phrase like ``-1``\nis actually an expression composed of the unary operator ``-`` and the\nliteral ``1``.\n',
  'for': u'\nThe ``for`` statement\n*********************\n\nThe ``for`` statement is used to iterate over the elements of a\nsequence (such as a string, tuple or list) or other iterable object:\n\n   for_stmt ::= "for" target_list "in" expression_list ":" suite\n                ["else" ":" suite]\n\nThe expression list is evaluated once; it should yield an iterable\nobject.  An iterator is created for the result of the\n``expression_list``.  The suite is then executed once for each item\nprovided by the iterator, in the order of ascending indices.  Each\nitem in turn is assigned to the target list using the standard rules\nfor assignments, and then the suite is executed.  When the items are\nexhausted (which is immediately when the sequence is empty), the suite\nin the ``else`` clause, if present, is executed, and the loop\nterminates.\n\nA ``break`` statement executed in the first suite terminates the loop\nwithout executing the ``else`` clause\'s suite.  A ``continue``\nstatement executed in the first suite skips the rest of the suite and\ncontinues with the next item, or with the ``else`` clause if there was\nno next item.\n\nThe suite may assign to the variable(s) in the target list; this does\nnot affect the next item assigned to it.\n\nThe target list is not deleted when the loop is finished, but if the\nsequence is empty, it will not have been assigned to at all by the\nloop.  Hint: the built-in function ``range()`` returns a sequence of\nintegers suitable to emulate the effect of Pascal\'s ``for i := a to b\ndo``; e.g., ``range(3)`` returns the list ``[0, 1, 2]``.\n\nNote: There is a subtlety when the sequence is being modified by the loop\n  (this can only occur for mutable sequences, i.e. lists). An internal\n  counter is used to keep track of which item is used next, and this\n  is incremented on each iteration.  When this counter has reached the\n  length of the sequence the loop terminates.  This means that if the\n  suite deletes the current (or a previous) item from the sequence,\n  the next item will be skipped (since it gets the index of the\n  current item which has already been treated).  Likewise, if the\n  suite inserts an item in the sequence before the current item, the\n  current item will be treated again the next time through the loop.\n  This can lead to nasty bugs that can be avoided by making a\n  temporary copy using a slice of the whole sequence, e.g.,\n\n     for x in a[:]:\n         if x < 0: a.remove(x)\n',
- 'formatstrings': u'\nFormat String Syntax\n********************\n\nThe ``str.format()`` method and the ``Formatter`` class share the same\nsyntax for format strings (although in the case of ``Formatter``,\nsubclasses can define their own format string syntax).\n\nFormat strings contain "replacement fields" surrounded by curly braces\n``{}``. Anything that is not contained in braces is considered literal\ntext, which is copied unchanged to the output.  If you need to include\na brace character in the literal text, it can be escaped by doubling:\n``{{`` and ``}}``.\n\nThe grammar for a replacement field is as follows:\n\n      replacement_field ::= "{" [field_name] ["!" conversion] [":" format_spec] "}"\n      field_name        ::= arg_name ("." attribute_name | "[" element_index "]")*\n      arg_name          ::= [identifier | integer]\n      attribute_name    ::= identifier\n      element_index     ::= integer | index_string\n      index_string      ::=  +\n      conversion        ::= "r" | "s"\n      format_spec       ::= \n\nIn less formal terms, the replacement field can start with a\n*field_name* that specifies the object whose value is to be formatted\nand inserted into the output instead of the replacement field. The\n*field_name* is optionally followed by a  *conversion* field, which is\npreceded by an exclamation point ``\'!\'``, and a *format_spec*, which\nis preceded by a colon ``\':\'``.  These specify a non-default format\nfor the replacement value.\n\nSee also the *Format Specification Mini-Language* section.\n\nThe *field_name* itself begins with an *arg_name* that is either\neither a number or a keyword.  If it\'s a number, it refers to a\npositional argument, and if it\'s a keyword, it refers to a named\nkeyword argument.  If the numerical arg_names in a format string are\n0, 1, 2, ... in sequence, they can all be omitted (not just some) and\nthe numbers 0, 1, 2, ... will be automatically inserted in that order.\nThe *arg_name* can be followed by any number of index or attribute\nexpressions. An expression of the form ``\'.name\'`` selects the named\nattribute using ``getattr()``, while an expression of the form\n``\'[index]\'`` does an index lookup using ``__getitem__()``.\n\nChanged in version 2.7: The positional argument specifiers can be\nomitted, so ``\'{} {}\'`` is equivalent to ``\'{0} {1}\'``.\n\nSome simple format string examples:\n\n   "First, thou shalt count to {0}" # References first positional argument\n   "Bring me a {}"                  # Implicitly references the first positional argument\n   "From {} to {}"                  # Same as "From {0} to {1}"\n   "My quest is {name}"             # References keyword argument \'name\'\n   "Weight in tons {0.weight}"      # \'weight\' attribute of first positional arg\n   "Units destroyed: {players[0]}"  # First element of keyword argument \'players\'.\n\nThe *conversion* field causes a type coercion before formatting.\nNormally, the job of formatting a value is done by the\n``__format__()`` method of the value itself.  However, in some cases\nit is desirable to force a type to be formatted as a string,\noverriding its own definition of formatting.  By converting the value\nto a string before calling ``__format__()``, the normal formatting\nlogic is bypassed.\n\nTwo conversion flags are currently supported: ``\'!s\'`` which calls\n``str()`` on the value, and ``\'!r\'`` which calls ``repr()``.\n\nSome examples:\n\n   "Harold\'s a clever {0!s}"        # Calls str() on the argument first\n   "Bring out the holy {name!r}"    # Calls repr() on the argument first\n\nThe *format_spec* field contains a specification of how the value\nshould be presented, including such details as field width, alignment,\npadding, decimal precision and so on.  Each value type can define its\nown "formatting mini-language" or interpretation of the *format_spec*.\n\nMost built-in types support a common formatting mini-language, which\nis described in the next section.\n\nA *format_spec* field can also include nested replacement fields\nwithin it. These nested replacement fields can contain only a field\nname; conversion flags and format specifications are not allowed.  The\nreplacement fields within the format_spec are substituted before the\n*format_spec* string is interpreted. This allows the formatting of a\nvalue to be dynamically specified.\n\nSee the *Format examples* section for some examples.\n\n\nFormat Specification Mini-Language\n==================================\n\n"Format specifications" are used within replacement fields contained\nwithin a format string to define how individual values are presented\n(see *Format String Syntax*).  They can also be passed directly to the\nbuilt-in ``format()`` function.  Each formattable type may define how\nthe format specification is to be interpreted.\n\nMost built-in types implement the following options for format\nspecifications, although some of the formatting options are only\nsupported by the numeric types.\n\nA general convention is that an empty format string (``""``) produces\nthe same result as if you had called ``str()`` on the value. A non-\nempty format string typically modifies the result.\n\nThe general form of a *standard format specifier* is:\n\n   format_spec ::= [[fill]align][sign][#][0][width][,][.precision][type]\n   fill        ::= \n   align       ::= "<" | ">" | "=" | "^"\n   sign        ::= "+" | "-" | " "\n   width       ::= integer\n   precision   ::= integer\n   type        ::= "b" | "c" | "d" | "e" | "E" | "f" | "F" | "g" | "G" | "n" | "o" | "s" | "x" | "X" | "%"\n\nThe *fill* character can be any character other than \'}\' (which\nsignifies the end of the field).  The presence of a fill character is\nsignaled by the *next* character, which must be one of the alignment\noptions. If the second character of *format_spec* is not a valid\nalignment option, then it is assumed that both the fill character and\nthe alignment option are absent.\n\nThe meaning of the various alignment options is as follows:\n\n   +-----------+------------------------------------------------------------+\n   | Option    | Meaning                                                    |\n   +===========+============================================================+\n   | ``\'<\'``   | Forces the field to be left-aligned within the available   |\n   |           | space (this is the default).                               |\n   +-----------+------------------------------------------------------------+\n   | ``\'>\'``   | Forces the field to be right-aligned within the available  |\n   |           | space.                                                     |\n   +-----------+------------------------------------------------------------+\n   | ``\'=\'``   | Forces the padding to be placed after the sign (if any)    |\n   |           | but before the digits.  This is used for printing fields   |\n   |           | in the form \'+000000120\'. This alignment option is only    |\n   |           | valid for numeric types.                                   |\n   +-----------+------------------------------------------------------------+\n   | ``\'^\'``   | Forces the field to be centered within the available       |\n   |           | space.                                                     |\n   +-----------+------------------------------------------------------------+\n\nNote that unless a minimum field width is defined, the field width\nwill always be the same size as the data to fill it, so that the\nalignment option has no meaning in this case.\n\nThe *sign* option is only valid for number types, and can be one of\nthe following:\n\n   +-----------+------------------------------------------------------------+\n   | Option    | Meaning                                                    |\n   +===========+============================================================+\n   | ``\'+\'``   | indicates that a sign should be used for both positive as  |\n   |           | well as negative numbers.                                  |\n   +-----------+------------------------------------------------------------+\n   | ``\'-\'``   | indicates that a sign should be used only for negative     |\n   |           | numbers (this is the default behavior).                    |\n   +-----------+------------------------------------------------------------+\n   | space     | indicates that a leading space should be used on positive  |\n   |           | numbers, and a minus sign on negative numbers.             |\n   +-----------+------------------------------------------------------------+\n\nThe ``\'#\'`` option is only valid for integers, and only for binary,\noctal, or hexadecimal output.  If present, it specifies that the\noutput will be prefixed by ``\'0b\'``, ``\'0o\'``, or ``\'0x\'``,\nrespectively.\n\nThe ``\',\'`` option signals the use of a comma for a thousands\nseparator. For a locale aware separator, use the ``\'n\'`` integer\npresentation type instead.\n\nChanged in version 2.7: Added the ``\',\'`` option (see also **PEP\n378**).\n\n*width* is a decimal integer defining the minimum field width.  If not\nspecified, then the field width will be determined by the content.\n\nIf the *width* field is preceded by a zero (``\'0\'``) character, this\nenables zero-padding.  This is equivalent to an *alignment* type of\n``\'=\'`` and a *fill* character of ``\'0\'``.\n\nThe *precision* is a decimal number indicating how many digits should\nbe displayed after the decimal point for a floating point value\nformatted with ``\'f\'`` and ``\'F\'``, or before and after the decimal\npoint for a floating point value formatted with ``\'g\'`` or ``\'G\'``.\nFor non-number types the field indicates the maximum field size - in\nother words, how many characters will be used from the field content.\nThe *precision* is not allowed for integer values.\n\nFinally, the *type* determines how the data should be presented.\n\nThe available string presentation types are:\n\n   +-----------+------------------------------------------------------------+\n   | Type      | Meaning                                                    |\n   +===========+============================================================+\n   | ``\'s\'``   | String format. This is the default type for strings and    |\n   |           | may be omitted.                                            |\n   +-----------+------------------------------------------------------------+\n   | None      | The same as ``\'s\'``.                                       |\n   +-----------+------------------------------------------------------------+\n\nThe available integer presentation types are:\n\n   +-----------+------------------------------------------------------------+\n   | Type      | Meaning                                                    |\n   +===========+============================================================+\n   | ``\'b\'``   | Binary format. Outputs the number in base 2.               |\n   +-----------+------------------------------------------------------------+\n   | ``\'c\'``   | Character. Converts the integer to the corresponding       |\n   |           | unicode character before printing.                         |\n   +-----------+------------------------------------------------------------+\n   | ``\'d\'``   | Decimal Integer. Outputs the number in base 10.            |\n   +-----------+------------------------------------------------------------+\n   | ``\'o\'``   | Octal format. Outputs the number in base 8.                |\n   +-----------+------------------------------------------------------------+\n   | ``\'x\'``   | Hex format. Outputs the number in base 16, using lower-    |\n   |           | case letters for the digits above 9.                       |\n   +-----------+------------------------------------------------------------+\n   | ``\'X\'``   | Hex format. Outputs the number in base 16, using upper-    |\n   |           | case letters for the digits above 9.                       |\n   +-----------+------------------------------------------------------------+\n   | ``\'n\'``   | Number. This is the same as ``\'d\'``, except that it uses   |\n   |           | the current locale setting to insert the appropriate       |\n   |           | number separator characters.                               |\n   +-----------+------------------------------------------------------------+\n   | None      | The same as ``\'d\'``.                                       |\n   +-----------+------------------------------------------------------------+\n\nIn addition to the above presentation types, integers can be formatted\nwith the floating point presentation types listed below (except\n``\'n\'`` and None). When doing so, ``float()`` is used to convert the\ninteger to a floating point number before formatting.\n\nThe available presentation types for floating point and decimal values\nare:\n\n   +-----------+------------------------------------------------------------+\n   | Type      | Meaning                                                    |\n   +===========+============================================================+\n   | ``\'e\'``   | Exponent notation. Prints the number in scientific         |\n   |           | notation using the letter \'e\' to indicate the exponent.    |\n   +-----------+------------------------------------------------------------+\n   | ``\'E\'``   | Exponent notation. Same as ``\'e\'`` except it uses an upper |\n   |           | case \'E\' as the separator character.                       |\n   +-----------+------------------------------------------------------------+\n   | ``\'f\'``   | Fixed point. Displays the number as a fixed-point number.  |\n   +-----------+------------------------------------------------------------+\n   | ``\'F\'``   | Fixed point. Same as ``\'f\'``.                              |\n   +-----------+------------------------------------------------------------+\n   | ``\'g\'``   | General format.  For a given precision ``p >= 1``, this    |\n   |           | rounds the number to ``p`` significant digits and then     |\n   |           | formats the result in either fixed-point format or in      |\n   |           | scientific notation, depending on its magnitude.  The      |\n   |           | precise rules are as follows: suppose that the result      |\n   |           | formatted with presentation type ``\'e\'`` and precision     |\n   |           | ``p-1`` would have exponent ``exp``.  Then if ``-4 <= exp  |\n   |           | < p``, the number is formatted with presentation type      |\n   |           | ``\'f\'`` and precision ``p-1-exp``. Otherwise, the number   |\n   |           | is formatted with presentation type ``\'e\'`` and precision  |\n   |           | ``p-1``. In both cases insignificant trailing zeros are    |\n   |           | removed from the significand, and the decimal point is     |\n   |           | also removed if there are no remaining digits following    |\n   |           | it.  Postive and negative infinity, positive and negative  |\n   |           | zero, and nans, are formatted as ``inf``, ``-inf``, ``0``, |\n   |           | ``-0`` and ``nan`` respectively, regardless of the         |\n   |           | precision.  A precision of ``0`` is treated as equivalent  |\n   |           | to a precision of ``1``.                                   |\n   +-----------+------------------------------------------------------------+\n   | ``\'G\'``   | General format. Same as ``\'g\'`` except switches to ``\'E\'`` |\n   |           | if the number gets too large. The representations of       |\n   |           | infinity and NaN are uppercased, too.                      |\n   +-----------+------------------------------------------------------------+\n   | ``\'n\'``   | Number. This is the same as ``\'g\'``, except that it uses   |\n   |           | the current locale setting to insert the appropriate       |\n   |           | number separator characters.                               |\n   +-----------+------------------------------------------------------------+\n   | ``\'%\'``   | Percentage. Multiplies the number by 100 and displays in   |\n   |           | fixed (``\'f\'``) format, followed by a percent sign.        |\n   +-----------+------------------------------------------------------------+\n   | None      | The same as ``\'g\'``.                                       |\n   +-----------+------------------------------------------------------------+\n\n\nFormat examples\n===============\n\nThis section contains examples of the new format syntax and comparison\nwith the old ``%``-formatting.\n\nIn most of the cases the syntax is similar to the old\n``%``-formatting, with the addition of the ``{}`` and with ``:`` used\ninstead of ``%``. For example, ``\'%03.2f\'`` can be translated to\n``\'{:03.2f}\'``.\n\nThe new format syntax also supports new and different options, shown\nin the follow examples.\n\nAccessing arguments by position:\n\n   >>> \'{0}, {1}, {2}\'.format(\'a\', \'b\', \'c\')\n   \'a, b, c\'\n   >>> \'{}, {}, {}\'.format(\'a\', \'b\', \'c\')  # 2.7+ only\n   \'a, b, c\'\n   >>> \'{2}, {1}, {0}\'.format(\'a\', \'b\', \'c\')\n   \'c, b, a\'\n   >>> \'{2}, {1}, {0}\'.format(*\'abc\')      # unpacking argument sequence\n   \'c, b, a\'\n   >>> \'{0}{1}{0}\'.format(\'abra\', \'cad\')   # arguments\' indices can be repeated\n   \'abracadabra\'\n\nAccessing arguments by name:\n\n   >>> \'Coordinates: {latitude}, {longitude}\'.format(latitude=\'37.24N\', longitude=\'-115.81W\')\n   \'Coordinates: 37.24N, -115.81W\'\n   >>> coord = {\'latitude\': \'37.24N\', \'longitude\': \'-115.81W\'}\n   >>> \'Coordinates: {latitude}, {longitude}\'.format(**coord)\n   \'Coordinates: 37.24N, -115.81W\'\n\nAccessing arguments\' attributes:\n\n   >>> c = 3-5j\n   >>> (\'The complex number {0} is formed from the real part {0.real} \'\n   ...  \'and the imaginary part {0.imag}.\').format(c)\n   \'The complex number (3-5j) is formed from the real part 3.0 and the imaginary part -5.0.\'\n   >>> class Point(object):\n   ...     def __init__(self, x, y):\n   ...         self.x, self.y = x, y\n   ...     def __str__(self):\n   ...         return \'Point({self.x}, {self.y})\'.format(self=self)\n   ...\n   >>> str(Point(4, 2))\n   \'Point(4, 2)\'\n\nAccessing arguments\' items:\n\n   >>> coord = (3, 5)\n   >>> \'X: {0[0]};  Y: {0[1]}\'.format(coord)\n   \'X: 3;  Y: 5\'\n\nReplacing ``%s`` and ``%r``:\n\n   >>> "repr() shows quotes: {!r}; str() doesn\'t: {!s}".format(\'test1\', \'test2\')\n   "repr() shows quotes: \'test1\'; str() doesn\'t: test2"\n\nAligning the text and specifying a width:\n\n   >>> \'{:<30}\'.format(\'left aligned\')\n   \'left aligned                  \'\n   >>> \'{:>30}\'.format(\'right aligned\')\n   \'                 right aligned\'\n   >>> \'{:^30}\'.format(\'centered\')\n   \'           centered           \'\n   >>> \'{:*^30}\'.format(\'centered\')  # use \'*\' as a fill char\n   \'***********centered***********\'\n\nReplacing ``%+f``, ``%-f``, and ``% f`` and specifying a sign:\n\n   >>> \'{:+f}; {:+f}\'.format(3.14, -3.14)  # show it always\n   \'+3.140000; -3.140000\'\n   >>> \'{: f}; {: f}\'.format(3.14, -3.14)  # show a space for positive numbers\n   \' 3.140000; -3.140000\'\n   >>> \'{:-f}; {:-f}\'.format(3.14, -3.14)  # show only the minus -- same as \'{:f}; {:f}\'\n   \'3.140000; -3.140000\'\n\nReplacing ``%x`` and ``%o`` and converting the value to different\nbases:\n\n   >>> # format also supports binary numbers\n   >>> "int: {0:d};  hex: {0:x};  oct: {0:o};  bin: {0:b}".format(42)\n   \'int: 42;  hex: 2a;  oct: 52;  bin: 101010\'\n   >>> # with 0x, 0o, or 0b as prefix:\n   >>> "int: {0:d};  hex: {0:#x};  oct: {0:#o};  bin: {0:#b}".format(42)\n   \'int: 42;  hex: 0x2a;  oct: 0o52;  bin: 0b101010\'\n\nUsing the comma as a thousands separator:\n\n   >>> \'{:,}\'.format(1234567890)\n   \'1,234,567,890\'\n\nExpressing a percentage:\n\n   >>> points = 19.5\n   >>> total = 22\n   >>> \'Correct answers: {:.2%}.\'.format(points/total)\n   \'Correct answers: 88.64%\'\n\nUsing type-specific formatting:\n\n   >>> import datetime\n   >>> d = datetime.datetime(2010, 7, 4, 12, 15, 58)\n   >>> \'{:%Y-%m-%d %H:%M:%S}\'.format(d)\n   \'2010-07-04 12:15:58\'\n\nNesting arguments and more complex examples:\n\n   >>> for align, text in zip(\'<^>\', [\'left\', \'center\', \'right\']):\n   ...     \'{0:{align}{fill}16}\'.format(text, fill=align, align=align)\n   ...\n   \'left<<<<<<<<<<<<\'\n   \'^^^^^center^^^^^\'\n   \'>>>>>>>>>>>right\'\n   >>>\n   >>> octets = [192, 168, 0, 1]\n   >>> \'{:02X}{:02X}{:02X}{:02X}\'.format(*octets)\n   \'C0A80001\'\n   >>> int(_, 16)\n   3232235521\n   >>>\n   >>> width = 5\n   >>> for num in range(5,12):\n   ...     for base in \'dXob\':\n   ...         print \'{0:{width}{base}}\'.format(num, base=base, width=width),\n   ...     print\n   ...\n       5     5     5   101\n       6     6     6   110\n       7     7     7   111\n       8     8    10  1000\n       9     9    11  1001\n      10     A    12  1010\n      11     B    13  1011\n',
+ 'formatstrings': u'\nFormat String Syntax\n********************\n\nThe ``str.format()`` method and the ``Formatter`` class share the same\nsyntax for format strings (although in the case of ``Formatter``,\nsubclasses can define their own format string syntax).\n\nFormat strings contain "replacement fields" surrounded by curly braces\n``{}``. Anything that is not contained in braces is considered literal\ntext, which is copied unchanged to the output.  If you need to include\na brace character in the literal text, it can be escaped by doubling:\n``{{`` and ``}}``.\n\nThe grammar for a replacement field is as follows:\n\n      replacement_field ::= "{" [field_name] ["!" conversion] [":" format_spec] "}"\n      field_name        ::= arg_name ("." attribute_name | "[" element_index "]")*\n      arg_name          ::= [identifier | integer]\n      attribute_name    ::= identifier\n      element_index     ::= integer | index_string\n      index_string      ::=  +\n      conversion        ::= "r" | "s"\n      format_spec       ::= \n\nIn less formal terms, the replacement field can start with a\n*field_name* that specifies the object whose value is to be formatted\nand inserted into the output instead of the replacement field. The\n*field_name* is optionally followed by a  *conversion* field, which is\npreceded by an exclamation point ``\'!\'``, and a *format_spec*, which\nis preceded by a colon ``\':\'``.  These specify a non-default format\nfor the replacement value.\n\nSee also the *Format Specification Mini-Language* section.\n\nThe *field_name* itself begins with an *arg_name* that is either\neither a number or a keyword.  If it\'s a number, it refers to a\npositional argument, and if it\'s a keyword, it refers to a named\nkeyword argument.  If the numerical arg_names in a format string are\n0, 1, 2, ... in sequence, they can all be omitted (not just some) and\nthe numbers 0, 1, 2, ... will be automatically inserted in that order.\nThe *arg_name* can be followed by any number of index or attribute\nexpressions. An expression of the form ``\'.name\'`` selects the named\nattribute using ``getattr()``, while an expression of the form\n``\'[index]\'`` does an index lookup using ``__getitem__()``.\n\nChanged in version 2.7: The positional argument specifiers can be\nomitted, so ``\'{} {}\'`` is equivalent to ``\'{0} {1}\'``.\n\nSome simple format string examples:\n\n   "First, thou shalt count to {0}" # References first positional argument\n   "Bring me a {}"                  # Implicitly references the first positional argument\n   "From {} to {}"                  # Same as "From {0} to {1}"\n   "My quest is {name}"             # References keyword argument \'name\'\n   "Weight in tons {0.weight}"      # \'weight\' attribute of first positional arg\n   "Units destroyed: {players[0]}"  # First element of keyword argument \'players\'.\n\nThe *conversion* field causes a type coercion before formatting.\nNormally, the job of formatting a value is done by the\n``__format__()`` method of the value itself.  However, in some cases\nit is desirable to force a type to be formatted as a string,\noverriding its own definition of formatting.  By converting the value\nto a string before calling ``__format__()``, the normal formatting\nlogic is bypassed.\n\nTwo conversion flags are currently supported: ``\'!s\'`` which calls\n``str()`` on the value, and ``\'!r\'`` which calls ``repr()``.\n\nSome examples:\n\n   "Harold\'s a clever {0!s}"        # Calls str() on the argument first\n   "Bring out the holy {name!r}"    # Calls repr() on the argument first\n\nThe *format_spec* field contains a specification of how the value\nshould be presented, including such details as field width, alignment,\npadding, decimal precision and so on.  Each value type can define its\nown "formatting mini-language" or interpretation of the *format_spec*.\n\nMost built-in types support a common formatting mini-language, which\nis described in the next section.\n\nA *format_spec* field can also include nested replacement fields\nwithin it. These nested replacement fields can contain only a field\nname; conversion flags and format specifications are not allowed.  The\nreplacement fields within the format_spec are substituted before the\n*format_spec* string is interpreted. This allows the formatting of a\nvalue to be dynamically specified.\n\nSee the *Format examples* section for some examples.\n\n\nFormat Specification Mini-Language\n==================================\n\n"Format specifications" are used within replacement fields contained\nwithin a format string to define how individual values are presented\n(see *Format String Syntax*).  They can also be passed directly to the\nbuilt-in ``format()`` function.  Each formattable type may define how\nthe format specification is to be interpreted.\n\nMost built-in types implement the following options for format\nspecifications, although some of the formatting options are only\nsupported by the numeric types.\n\nA general convention is that an empty format string (``""``) produces\nthe same result as if you had called ``str()`` on the value. A non-\nempty format string typically modifies the result.\n\nThe general form of a *standard format specifier* is:\n\n   format_spec ::= [[fill]align][sign][#][0][width][,][.precision][type]\n   fill        ::= \n   align       ::= "<" | ">" | "=" | "^"\n   sign        ::= "+" | "-" | " "\n   width       ::= integer\n   precision   ::= integer\n   type        ::= "b" | "c" | "d" | "e" | "E" | "f" | "F" | "g" | "G" | "n" | "o" | "s" | "x" | "X" | "%"\n\nThe *fill* character can be any character other than \'{\' or \'}\'.  The\npresence of a fill character is signaled by the character following\nit, which must be one of the alignment options.  If the second\ncharacter of *format_spec* is not a valid alignment option, then it is\nassumed that both the fill character and the alignment option are\nabsent.\n\nThe meaning of the various alignment options is as follows:\n\n   +-----------+------------------------------------------------------------+\n   | Option    | Meaning                                                    |\n   +===========+============================================================+\n   | ``\'<\'``   | Forces the field to be left-aligned within the available   |\n   |           | space (this is the default for most objects).              |\n   +-----------+------------------------------------------------------------+\n   | ``\'>\'``   | Forces the field to be right-aligned within the available  |\n   |           | space (this is the default for numbers).                   |\n   +-----------+------------------------------------------------------------+\n   | ``\'=\'``   | Forces the padding to be placed after the sign (if any)    |\n   |           | but before the digits.  This is used for printing fields   |\n   |           | in the form \'+000000120\'. This alignment option is only    |\n   |           | valid for numeric types.                                   |\n   +-----------+------------------------------------------------------------+\n   | ``\'^\'``   | Forces the field to be centered within the available       |\n   |           | space.                                                     |\n   +-----------+------------------------------------------------------------+\n\nNote that unless a minimum field width is defined, the field width\nwill always be the same size as the data to fill it, so that the\nalignment option has no meaning in this case.\n\nThe *sign* option is only valid for number types, and can be one of\nthe following:\n\n   +-----------+------------------------------------------------------------+\n   | Option    | Meaning                                                    |\n   +===========+============================================================+\n   | ``\'+\'``   | indicates that a sign should be used for both positive as  |\n   |           | well as negative numbers.                                  |\n   +-----------+------------------------------------------------------------+\n   | ``\'-\'``   | indicates that a sign should be used only for negative     |\n   |           | numbers (this is the default behavior).                    |\n   +-----------+------------------------------------------------------------+\n   | space     | indicates that a leading space should be used on positive  |\n   |           | numbers, and a minus sign on negative numbers.             |\n   +-----------+------------------------------------------------------------+\n\nThe ``\'#\'`` option is only valid for integers, and only for binary,\noctal, or hexadecimal output.  If present, it specifies that the\noutput will be prefixed by ``\'0b\'``, ``\'0o\'``, or ``\'0x\'``,\nrespectively.\n\nThe ``\',\'`` option signals the use of a comma for a thousands\nseparator. For a locale aware separator, use the ``\'n\'`` integer\npresentation type instead.\n\nChanged in version 2.7: Added the ``\',\'`` option (see also **PEP\n378**).\n\n*width* is a decimal integer defining the minimum field width.  If not\nspecified, then the field width will be determined by the content.\n\nIf the *width* field is preceded by a zero (``\'0\'``) character, this\nenables zero-padding.  This is equivalent to an *alignment* type of\n``\'=\'`` and a *fill* character of ``\'0\'``.\n\nThe *precision* is a decimal number indicating how many digits should\nbe displayed after the decimal point for a floating point value\nformatted with ``\'f\'`` and ``\'F\'``, or before and after the decimal\npoint for a floating point value formatted with ``\'g\'`` or ``\'G\'``.\nFor non-number types the field indicates the maximum field size - in\nother words, how many characters will be used from the field content.\nThe *precision* is not allowed for integer values.\n\nFinally, the *type* determines how the data should be presented.\n\nThe available string presentation types are:\n\n   +-----------+------------------------------------------------------------+\n   | Type      | Meaning                                                    |\n   +===========+============================================================+\n   | ``\'s\'``   | String format. This is the default type for strings and    |\n   |           | may be omitted.                                            |\n   +-----------+------------------------------------------------------------+\n   | None      | The same as ``\'s\'``.                                       |\n   +-----------+------------------------------------------------------------+\n\nThe available integer presentation types are:\n\n   +-----------+------------------------------------------------------------+\n   | Type      | Meaning                                                    |\n   +===========+============================================================+\n   | ``\'b\'``   | Binary format. Outputs the number in base 2.               |\n   +-----------+------------------------------------------------------------+\n   | ``\'c\'``   | Character. Converts the integer to the corresponding       |\n   |           | unicode character before printing.                         |\n   +-----------+------------------------------------------------------------+\n   | ``\'d\'``   | Decimal Integer. Outputs the number in base 10.            |\n   +-----------+------------------------------------------------------------+\n   | ``\'o\'``   | Octal format. Outputs the number in base 8.                |\n   +-----------+------------------------------------------------------------+\n   | ``\'x\'``   | Hex format. Outputs the number in base 16, using lower-    |\n   |           | case letters for the digits above 9.                       |\n   +-----------+------------------------------------------------------------+\n   | ``\'X\'``   | Hex format. Outputs the number in base 16, using upper-    |\n   |           | case letters for the digits above 9.                       |\n   +-----------+------------------------------------------------------------+\n   | ``\'n\'``   | Number. This is the same as ``\'d\'``, except that it uses   |\n   |           | the current locale setting to insert the appropriate       |\n   |           | number separator characters.                               |\n   +-----------+------------------------------------------------------------+\n   | None      | The same as ``\'d\'``.                                       |\n   +-----------+------------------------------------------------------------+\n\nIn addition to the above presentation types, integers can be formatted\nwith the floating point presentation types listed below (except\n``\'n\'`` and None). When doing so, ``float()`` is used to convert the\ninteger to a floating point number before formatting.\n\nThe available presentation types for floating point and decimal values\nare:\n\n   +-----------+------------------------------------------------------------+\n   | Type      | Meaning                                                    |\n   +===========+============================================================+\n   | ``\'e\'``   | Exponent notation. Prints the number in scientific         |\n   |           | notation using the letter \'e\' to indicate the exponent.    |\n   +-----------+------------------------------------------------------------+\n   | ``\'E\'``   | Exponent notation. Same as ``\'e\'`` except it uses an upper |\n   |           | case \'E\' as the separator character.                       |\n   +-----------+------------------------------------------------------------+\n   | ``\'f\'``   | Fixed point. Displays the number as a fixed-point number.  |\n   +-----------+------------------------------------------------------------+\n   | ``\'F\'``   | Fixed point. Same as ``\'f\'``.                              |\n   +-----------+------------------------------------------------------------+\n   | ``\'g\'``   | General format.  For a given precision ``p >= 1``, this    |\n   |           | rounds the number to ``p`` significant digits and then     |\n   |           | formats the result in either fixed-point format or in      |\n   |           | scientific notation, depending on its magnitude.  The      |\n   |           | precise rules are as follows: suppose that the result      |\n   |           | formatted with presentation type ``\'e\'`` and precision     |\n   |           | ``p-1`` would have exponent ``exp``.  Then if ``-4 <= exp  |\n   |           | < p``, the number is formatted with presentation type      |\n   |           | ``\'f\'`` and precision ``p-1-exp``. Otherwise, the number   |\n   |           | is formatted with presentation type ``\'e\'`` and precision  |\n   |           | ``p-1``. In both cases insignificant trailing zeros are    |\n   |           | removed from the significand, and the decimal point is     |\n   |           | also removed if there are no remaining digits following    |\n   |           | it.  Positive and negative infinity, positive and negative |\n   |           | zero, and nans, are formatted as ``inf``, ``-inf``, ``0``, |\n   |           | ``-0`` and ``nan`` respectively, regardless of the         |\n   |           | precision.  A precision of ``0`` is treated as equivalent  |\n   |           | to a precision of ``1``.                                   |\n   +-----------+------------------------------------------------------------+\n   | ``\'G\'``   | General format. Same as ``\'g\'`` except switches to ``\'E\'`` |\n   |           | if the number gets too large. The representations of       |\n   |           | infinity and NaN are uppercased, too.                      |\n   +-----------+------------------------------------------------------------+\n   | ``\'n\'``   | Number. This is the same as ``\'g\'``, except that it uses   |\n   |           | the current locale setting to insert the appropriate       |\n   |           | number separator characters.                               |\n   +-----------+------------------------------------------------------------+\n   | ``\'%\'``   | Percentage. Multiplies the number by 100 and displays in   |\n   |           | fixed (``\'f\'``) format, followed by a percent sign.        |\n   +-----------+------------------------------------------------------------+\n   | None      | The same as ``\'g\'``.                                       |\n   +-----------+------------------------------------------------------------+\n\n\nFormat examples\n===============\n\nThis section contains examples of the new format syntax and comparison\nwith the old ``%``-formatting.\n\nIn most of the cases the syntax is similar to the old\n``%``-formatting, with the addition of the ``{}`` and with ``:`` used\ninstead of ``%``. For example, ``\'%03.2f\'`` can be translated to\n``\'{:03.2f}\'``.\n\nThe new format syntax also supports new and different options, shown\nin the follow examples.\n\nAccessing arguments by position:\n\n   >>> \'{0}, {1}, {2}\'.format(\'a\', \'b\', \'c\')\n   \'a, b, c\'\n   >>> \'{}, {}, {}\'.format(\'a\', \'b\', \'c\')  # 2.7+ only\n   \'a, b, c\'\n   >>> \'{2}, {1}, {0}\'.format(\'a\', \'b\', \'c\')\n   \'c, b, a\'\n   >>> \'{2}, {1}, {0}\'.format(*\'abc\')      # unpacking argument sequence\n   \'c, b, a\'\n   >>> \'{0}{1}{0}\'.format(\'abra\', \'cad\')   # arguments\' indices can be repeated\n   \'abracadabra\'\n\nAccessing arguments by name:\n\n   >>> \'Coordinates: {latitude}, {longitude}\'.format(latitude=\'37.24N\', longitude=\'-115.81W\')\n   \'Coordinates: 37.24N, -115.81W\'\n   >>> coord = {\'latitude\': \'37.24N\', \'longitude\': \'-115.81W\'}\n   >>> \'Coordinates: {latitude}, {longitude}\'.format(**coord)\n   \'Coordinates: 37.24N, -115.81W\'\n\nAccessing arguments\' attributes:\n\n   >>> c = 3-5j\n   >>> (\'The complex number {0} is formed from the real part {0.real} \'\n   ...  \'and the imaginary part {0.imag}.\').format(c)\n   \'The complex number (3-5j) is formed from the real part 3.0 and the imaginary part -5.0.\'\n   >>> class Point(object):\n   ...     def __init__(self, x, y):\n   ...         self.x, self.y = x, y\n   ...     def __str__(self):\n   ...         return \'Point({self.x}, {self.y})\'.format(self=self)\n   ...\n   >>> str(Point(4, 2))\n   \'Point(4, 2)\'\n\nAccessing arguments\' items:\n\n   >>> coord = (3, 5)\n   >>> \'X: {0[0]};  Y: {0[1]}\'.format(coord)\n   \'X: 3;  Y: 5\'\n\nReplacing ``%s`` and ``%r``:\n\n   >>> "repr() shows quotes: {!r}; str() doesn\'t: {!s}".format(\'test1\', \'test2\')\n   "repr() shows quotes: \'test1\'; str() doesn\'t: test2"\n\nAligning the text and specifying a width:\n\n   >>> \'{:<30}\'.format(\'left aligned\')\n   \'left aligned                  \'\n   >>> \'{:>30}\'.format(\'right aligned\')\n   \'                 right aligned\'\n   >>> \'{:^30}\'.format(\'centered\')\n   \'           centered           \'\n   >>> \'{:*^30}\'.format(\'centered\')  # use \'*\' as a fill char\n   \'***********centered***********\'\n\nReplacing ``%+f``, ``%-f``, and ``% f`` and specifying a sign:\n\n   >>> \'{:+f}; {:+f}\'.format(3.14, -3.14)  # show it always\n   \'+3.140000; -3.140000\'\n   >>> \'{: f}; {: f}\'.format(3.14, -3.14)  # show a space for positive numbers\n   \' 3.140000; -3.140000\'\n   >>> \'{:-f}; {:-f}\'.format(3.14, -3.14)  # show only the minus -- same as \'{:f}; {:f}\'\n   \'3.140000; -3.140000\'\n\nReplacing ``%x`` and ``%o`` and converting the value to different\nbases:\n\n   >>> # format also supports binary numbers\n   >>> "int: {0:d};  hex: {0:x};  oct: {0:o};  bin: {0:b}".format(42)\n   \'int: 42;  hex: 2a;  oct: 52;  bin: 101010\'\n   >>> # with 0x, 0o, or 0b as prefix:\n   >>> "int: {0:d};  hex: {0:#x};  oct: {0:#o};  bin: {0:#b}".format(42)\n   \'int: 42;  hex: 0x2a;  oct: 0o52;  bin: 0b101010\'\n\nUsing the comma as a thousands separator:\n\n   >>> \'{:,}\'.format(1234567890)\n   \'1,234,567,890\'\n\nExpressing a percentage:\n\n   >>> points = 19.5\n   >>> total = 22\n   >>> \'Correct answers: {:.2%}.\'.format(points/total)\n   \'Correct answers: 88.64%\'\n\nUsing type-specific formatting:\n\n   >>> import datetime\n   >>> d = datetime.datetime(2010, 7, 4, 12, 15, 58)\n   >>> \'{:%Y-%m-%d %H:%M:%S}\'.format(d)\n   \'2010-07-04 12:15:58\'\n\nNesting arguments and more complex examples:\n\n   >>> for align, text in zip(\'<^>\', [\'left\', \'center\', \'right\']):\n   ...     \'{0:{fill}{align}16}\'.format(text, fill=align, align=align)\n   ...\n   \'left<<<<<<<<<<<<\'\n   \'^^^^^center^^^^^\'\n   \'>>>>>>>>>>>right\'\n   >>>\n   >>> octets = [192, 168, 0, 1]\n   >>> \'{:02X}{:02X}{:02X}{:02X}\'.format(*octets)\n   \'C0A80001\'\n   >>> int(_, 16)\n   3232235521\n   >>>\n   >>> width = 5\n   >>> for num in range(5,12):\n   ...     for base in \'dXob\':\n   ...         print \'{0:{width}{base}}\'.format(num, base=base, width=width),\n   ...     print\n   ...\n       5     5     5   101\n       6     6     6   110\n       7     7     7   111\n       8     8    10  1000\n       9     9    11  1001\n      10     A    12  1010\n      11     B    13  1011\n',
  'function': u'\nFunction definitions\n********************\n\nA function definition defines a user-defined function object (see\nsection *The standard type hierarchy*):\n\n   decorated      ::= decorators (classdef | funcdef)\n   decorators     ::= decorator+\n   decorator      ::= "@" dotted_name ["(" [argument_list [","]] ")"] NEWLINE\n   funcdef        ::= "def" funcname "(" [parameter_list] ")" ":" suite\n   dotted_name    ::= identifier ("." identifier)*\n   parameter_list ::= (defparameter ",")*\n                      (  "*" identifier [, "**" identifier]\n                      | "**" identifier\n                      | defparameter [","] )\n   defparameter   ::= parameter ["=" expression]\n   sublist        ::= parameter ("," parameter)* [","]\n   parameter      ::= identifier | "(" sublist ")"\n   funcname       ::= identifier\n\nA function definition is an executable statement.  Its execution binds\nthe function name in the current local namespace to a function object\n(a wrapper around the executable code for the function).  This\nfunction object contains a reference to the current global namespace\nas the global namespace to be used when the function is called.\n\nThe function definition does not execute the function body; this gets\nexecuted only when the function is called. [3]\n\nA function definition may be wrapped by one or more *decorator*\nexpressions. Decorator expressions are evaluated when the function is\ndefined, in the scope that contains the function definition.  The\nresult must be a callable, which is invoked with the function object\nas the only argument. The returned value is bound to the function name\ninstead of the function object.  Multiple decorators are applied in\nnested fashion. For example, the following code:\n\n   @f1(arg)\n   @f2\n   def func(): pass\n\nis equivalent to:\n\n   def func(): pass\n   func = f1(arg)(f2(func))\n\nWhen one or more top-level parameters have the form *parameter* ``=``\n*expression*, the function is said to have "default parameter values."\nFor a parameter with a default value, the corresponding argument may\nbe omitted from a call, in which case the parameter\'s default value is\nsubstituted.  If a parameter has a default value, all following\nparameters must also have a default value --- this is a syntactic\nrestriction that is not expressed by the grammar.\n\n**Default parameter values are evaluated when the function definition\nis executed.**  This means that the expression is evaluated once, when\nthe function is defined, and that that same "pre-computed" value is\nused for each call.  This is especially important to understand when a\ndefault parameter is a mutable object, such as a list or a dictionary:\nif the function modifies the object (e.g. by appending an item to a\nlist), the default value is in effect modified. This is generally not\nwhat was intended.  A way around this  is to use ``None`` as the\ndefault, and explicitly test for it in the body of the function, e.g.:\n\n   def whats_on_the_telly(penguin=None):\n       if penguin is None:\n           penguin = []\n       penguin.append("property of the zoo")\n       return penguin\n\nFunction call semantics are described in more detail in section\n*Calls*. A function call always assigns values to all parameters\nmentioned in the parameter list, either from position arguments, from\nkeyword arguments, or from default values.  If the form\n"``*identifier``" is present, it is initialized to a tuple receiving\nany excess positional parameters, defaulting to the empty tuple.  If\nthe form "``**identifier``" is present, it is initialized to a new\ndictionary receiving any excess keyword arguments, defaulting to a new\nempty dictionary.\n\nIt is also possible to create anonymous functions (functions not bound\nto a name), for immediate use in expressions.  This uses lambda forms,\ndescribed in section *Lambdas*.  Note that the lambda form is merely a\nshorthand for a simplified function definition; a function defined in\na "``def``" statement can be passed around or assigned to another name\njust like a function defined by a lambda form.  The "``def``" form is\nactually more powerful since it allows the execution of multiple\nstatements.\n\n**Programmer\'s note:** Functions are first-class objects.  A "``def``"\nform executed inside a function definition defines a local function\nthat can be returned or passed around.  Free variables used in the\nnested function can access the local variables of the function\ncontaining the def.  See section *Naming and binding* for details.\n',
  'global': u'\nThe ``global`` statement\n************************\n\n   global_stmt ::= "global" identifier ("," identifier)*\n\nThe ``global`` statement is a declaration which holds for the entire\ncurrent code block.  It means that the listed identifiers are to be\ninterpreted as globals.  It would be impossible to assign to a global\nvariable without ``global``, although free variables may refer to\nglobals without being declared global.\n\nNames listed in a ``global`` statement must not be used in the same\ncode block textually preceding that ``global`` statement.\n\nNames listed in a ``global`` statement must not be defined as formal\nparameters or in a ``for`` loop control target, ``class`` definition,\nfunction definition, or ``import`` statement.\n\n**CPython implementation detail:** The current implementation does not\nenforce the latter two restrictions, but programs should not abuse\nthis freedom, as future implementations may enforce them or silently\nchange the meaning of the program.\n\n**Programmer\'s note:** the ``global`` is a directive to the parser.\nIt applies only to code parsed at the same time as the ``global``\nstatement. In particular, a ``global`` statement contained in an\n``exec`` statement does not affect the code block *containing* the\n``exec`` statement, and code contained in an ``exec`` statement is\nunaffected by ``global`` statements in the code containing the\n``exec`` statement.  The same applies to the ``eval()``,\n``execfile()`` and ``compile()`` functions.\n',
- 'id-classes': u'\nReserved classes of identifiers\n*******************************\n\nCertain classes of identifiers (besides keywords) have special\nmeanings.  These classes are identified by the patterns of leading and\ntrailing underscore characters:\n\n``_*``\n   Not imported by ``from module import *``.  The special identifier\n   ``_`` is used in the interactive interpreter to store the result of\n   the last evaluation; it is stored in the ``__builtin__`` module.\n   When not in interactive mode, ``_`` has no special meaning and is\n   not defined. See section *The import statement*.\n\n   Note: The name ``_`` is often used in conjunction with\n     internationalization; refer to the documentation for the\n     ``gettext`` module for more information on this convention.\n\n``__*__``\n   System-defined names.  These names are defined by the interpreter\n   and its implementation (including the standard library);\n   applications should not expect to define additional names using\n   this convention.  The set of names of this class defined by Python\n   may be extended in future versions. See section *Special method\n   names*.\n\n``__*``\n   Class-private names.  Names in this category, when used within the\n   context of a class definition, are re-written to use a mangled form\n   to help avoid name clashes between "private" attributes of base and\n   derived classes. See section *Identifiers (Names)*.\n',
- 'identifiers': u'\nIdentifiers and keywords\n************************\n\nIdentifiers (also referred to as *names*) are described by the\nfollowing lexical definitions:\n\n   identifier ::= (letter|"_") (letter | digit | "_")*\n   letter     ::= lowercase | uppercase\n   lowercase  ::= "a"..."z"\n   uppercase  ::= "A"..."Z"\n   digit      ::= "0"..."9"\n\nIdentifiers are unlimited in length.  Case is significant.\n\n\nKeywords\n========\n\nThe following identifiers are used as reserved words, or *keywords* of\nthe language, and cannot be used as ordinary identifiers.  They must\nbe spelled exactly as written here:\n\n   and       del       from      not       while\n   as        elif      global    or        with\n   assert    else      if        pass      yield\n   break     except    import    print\n   class     exec      in        raise\n   continue  finally   is        return\n   def       for       lambda    try\n\nChanged in version 2.4: ``None`` became a constant and is now\nrecognized by the compiler as a name for the built-in object ``None``.\nAlthough it is not a keyword, you cannot assign a different object to\nit.\n\nChanged in version 2.5: Both ``as`` and ``with`` are only recognized\nwhen the ``with_statement`` future feature has been enabled. It will\nalways be enabled in Python 2.6.  See section *The with statement* for\ndetails.  Note that using ``as`` and ``with`` as identifiers will\nalways issue a warning, even when the ``with_statement`` future\ndirective is not in effect.\n\n\nReserved classes of identifiers\n===============================\n\nCertain classes of identifiers (besides keywords) have special\nmeanings.  These classes are identified by the patterns of leading and\ntrailing underscore characters:\n\n``_*``\n   Not imported by ``from module import *``.  The special identifier\n   ``_`` is used in the interactive interpreter to store the result of\n   the last evaluation; it is stored in the ``__builtin__`` module.\n   When not in interactive mode, ``_`` has no special meaning and is\n   not defined. See section *The import statement*.\n\n   Note: The name ``_`` is often used in conjunction with\n     internationalization; refer to the documentation for the\n     ``gettext`` module for more information on this convention.\n\n``__*__``\n   System-defined names.  These names are defined by the interpreter\n   and its implementation (including the standard library);\n   applications should not expect to define additional names using\n   this convention.  The set of names of this class defined by Python\n   may be extended in future versions. See section *Special method\n   names*.\n\n``__*``\n   Class-private names.  Names in this category, when used within the\n   context of a class definition, are re-written to use a mangled form\n   to help avoid name clashes between "private" attributes of base and\n   derived classes. See section *Identifiers (Names)*.\n',
+ 'id-classes': u'\nReserved classes of identifiers\n*******************************\n\nCertain classes of identifiers (besides keywords) have special\nmeanings.  These classes are identified by the patterns of leading and\ntrailing underscore characters:\n\n``_*``\n   Not imported by ``from module import *``.  The special identifier\n   ``_`` is used in the interactive interpreter to store the result of\n   the last evaluation; it is stored in the ``__builtin__`` module.\n   When not in interactive mode, ``_`` has no special meaning and is\n   not defined. See section *The import statement*.\n\n   Note: The name ``_`` is often used in conjunction with\n     internationalization; refer to the documentation for the\n     ``gettext`` module for more information on this convention.\n\n``__*__``\n   System-defined names. These names are defined by the interpreter\n   and its implementation (including the standard library).  Current\n   system names are discussed in the *Special method names* section\n   and elsewhere.  More will likely be defined in future versions of\n   Python.  *Any* use of ``__*__`` names, in any context, that does\n   not follow explicitly documented use, is subject to breakage\n   without warning.\n\n``__*``\n   Class-private names.  Names in this category, when used within the\n   context of a class definition, are re-written to use a mangled form\n   to help avoid name clashes between "private" attributes of base and\n   derived classes. See section *Identifiers (Names)*.\n',
+ 'identifiers': u'\nIdentifiers and keywords\n************************\n\nIdentifiers (also referred to as *names*) are described by the\nfollowing lexical definitions:\n\n   identifier ::= (letter|"_") (letter | digit | "_")*\n   letter     ::= lowercase | uppercase\n   lowercase  ::= "a"..."z"\n   uppercase  ::= "A"..."Z"\n   digit      ::= "0"..."9"\n\nIdentifiers are unlimited in length.  Case is significant.\n\n\nKeywords\n========\n\nThe following identifiers are used as reserved words, or *keywords* of\nthe language, and cannot be used as ordinary identifiers.  They must\nbe spelled exactly as written here:\n\n   and       del       from      not       while\n   as        elif      global    or        with\n   assert    else      if        pass      yield\n   break     except    import    print\n   class     exec      in        raise\n   continue  finally   is        return\n   def       for       lambda    try\n\nChanged in version 2.4: ``None`` became a constant and is now\nrecognized by the compiler as a name for the built-in object ``None``.\nAlthough it is not a keyword, you cannot assign a different object to\nit.\n\nChanged in version 2.5: Both ``as`` and ``with`` are only recognized\nwhen the ``with_statement`` future feature has been enabled. It will\nalways be enabled in Python 2.6.  See section *The with statement* for\ndetails.  Note that using ``as`` and ``with`` as identifiers will\nalways issue a warning, even when the ``with_statement`` future\ndirective is not in effect.\n\n\nReserved classes of identifiers\n===============================\n\nCertain classes of identifiers (besides keywords) have special\nmeanings.  These classes are identified by the patterns of leading and\ntrailing underscore characters:\n\n``_*``\n   Not imported by ``from module import *``.  The special identifier\n   ``_`` is used in the interactive interpreter to store the result of\n   the last evaluation; it is stored in the ``__builtin__`` module.\n   When not in interactive mode, ``_`` has no special meaning and is\n   not defined. See section *The import statement*.\n\n   Note: The name ``_`` is often used in conjunction with\n     internationalization; refer to the documentation for the\n     ``gettext`` module for more information on this convention.\n\n``__*__``\n   System-defined names. These names are defined by the interpreter\n   and its implementation (including the standard library).  Current\n   system names are discussed in the *Special method names* section\n   and elsewhere.  More will likely be defined in future versions of\n   Python.  *Any* use of ``__*__`` names, in any context, that does\n   not follow explicitly documented use, is subject to breakage\n   without warning.\n\n``__*``\n   Class-private names.  Names in this category, when used within the\n   context of a class definition, are re-written to use a mangled form\n   to help avoid name clashes between "private" attributes of base and\n   derived classes. See section *Identifiers (Names)*.\n',
  'if': u'\nThe ``if`` statement\n********************\n\nThe ``if`` statement is used for conditional execution:\n\n   if_stmt ::= "if" expression ":" suite\n               ( "elif" expression ":" suite )*\n               ["else" ":" suite]\n\nIt selects exactly one of the suites by evaluating the expressions one\nby one until one is found to be true (see section *Boolean operations*\nfor the definition of true and false); then that suite is executed\n(and no other part of the ``if`` statement is executed or evaluated).\nIf all expressions are false, the suite of the ``else`` clause, if\npresent, is executed.\n',
  'imaginary': u'\nImaginary literals\n******************\n\nImaginary literals are described by the following lexical definitions:\n\n   imagnumber ::= (floatnumber | intpart) ("j" | "J")\n\nAn imaginary literal yields a complex number with a real part of 0.0.\nComplex numbers are represented as a pair of floating point numbers\nand have the same restrictions on their range.  To create a complex\nnumber with a nonzero real part, add a floating point number to it,\ne.g., ``(3+4j)``.  Some examples of imaginary literals:\n\n   3.14j   10.j    10j     .001j   1e100j  3.14e-10j\n',
- 'import': u'\nThe ``import`` statement\n************************\n\n   import_stmt     ::= "import" module ["as" name] ( "," module ["as" name] )*\n                   | "from" relative_module "import" identifier ["as" name]\n                   ( "," identifier ["as" name] )*\n                   | "from" relative_module "import" "(" identifier ["as" name]\n                   ( "," identifier ["as" name] )* [","] ")"\n                   | "from" module "import" "*"\n   module          ::= (identifier ".")* identifier\n   relative_module ::= "."* module | "."+\n   name            ::= identifier\n\nImport statements are executed in two steps: (1) find a module, and\ninitialize it if necessary; (2) define a name or names in the local\nnamespace (of the scope where the ``import`` statement occurs). The\nstatement comes in two forms differing on whether it uses the ``from``\nkeyword. The first form (without ``from``) repeats these steps for\neach identifier in the list. The form with ``from`` performs step (1)\nonce, and then performs step (2) repeatedly.\n\nTo understand how step (1) occurs, one must first understand how\nPython handles hierarchical naming of modules. To help organize\nmodules and provide a hierarchy in naming, Python has a concept of\npackages. A package can contain other packages and modules while\nmodules cannot contain other modules or packages. From a file system\nperspective, packages are directories and modules are files. The\noriginal specification for packages is still available to read,\nalthough minor details have changed since the writing of that\ndocument.\n\nOnce the name of the module is known (unless otherwise specified, the\nterm "module" will refer to both packages and modules), searching for\nthe module or package can begin. The first place checked is\n``sys.modules``, the cache of all modules that have been imported\npreviously. If the module is found there then it is used in step (2)\nof import.\n\nIf the module is not found in the cache, then ``sys.meta_path`` is\nsearched (the specification for ``sys.meta_path`` can be found in\n**PEP 302**). The object is a list of *finder* objects which are\nqueried in order as to whether they know how to load the module by\ncalling their ``find_module()`` method with the name of the module. If\nthe module happens to be contained within a package (as denoted by the\nexistence of a dot in the name), then a second argument to\n``find_module()`` is given as the value of the ``__path__`` attribute\nfrom the parent package (everything up to the last dot in the name of\nthe module being imported). If a finder can find the module it returns\na *loader* (discussed later) or returns ``None``.\n\nIf none of the finders on ``sys.meta_path`` are able to find the\nmodule then some implicitly defined finders are queried.\nImplementations of Python vary in what implicit meta path finders are\ndefined. The one they all do define, though, is one that handles\n``sys.path_hooks``, ``sys.path_importer_cache``, and ``sys.path``.\n\nThe implicit finder searches for the requested module in the "paths"\nspecified in one of two places ("paths" do not have to be file system\npaths). If the module being imported is supposed to be contained\nwithin a package then the second argument passed to ``find_module()``,\n``__path__`` on the parent package, is used as the source of paths. If\nthe module is not contained in a package then ``sys.path`` is used as\nthe source of paths.\n\nOnce the source of paths is chosen it is iterated over to find a\nfinder that can handle that path. The dict at\n``sys.path_importer_cache`` caches finders for paths and is checked\nfor a finder. If the path does not have a finder cached then\n``sys.path_hooks`` is searched by calling each object in the list with\na single argument of the path, returning a finder or raises\n``ImportError``. If a finder is returned then it is cached in\n``sys.path_importer_cache`` and then used for that path entry. If no\nfinder can be found but the path exists then a value of ``None`` is\nstored in ``sys.path_importer_cache`` to signify that an implicit,\nfile-based finder that handles modules stored as individual files\nshould be used for that path. If the path does not exist then a finder\nwhich always returns ``None`` is placed in the cache for the path.\n\nIf no finder can find the module then ``ImportError`` is raised.\nOtherwise some finder returned a loader whose ``load_module()`` method\nis called with the name of the module to load (see **PEP 302** for the\noriginal definition of loaders). A loader has several responsibilities\nto perform on a module it loads. First, if the module already exists\nin ``sys.modules`` (a possibility if the loader is called outside of\nthe import machinery) then it is to use that module for initialization\nand not a new module. But if the module does not exist in\n``sys.modules`` then it is to be added to that dict before\ninitialization begins. If an error occurs during loading of the module\nand it was added to ``sys.modules`` it is to be removed from the dict.\nIf an error occurs but the module was already in ``sys.modules`` it is\nleft in the dict.\n\nThe loader must set several attributes on the module. ``__name__`` is\nto be set to the name of the module. ``__file__`` is to be the "path"\nto the file unless the module is built-in (and thus listed in\n``sys.builtin_module_names``) in which case the attribute is not set.\nIf what is being imported is a package then ``__path__`` is to be set\nto a list of paths to be searched when looking for modules and\npackages contained within the package being imported. ``__package__``\nis optional but should be set to the name of package that contains the\nmodule or package (the empty string is used for module not contained\nin a package). ``__loader__`` is also optional but should be set to\nthe loader object that is loading the module.\n\nIf an error occurs during loading then the loader raises\n``ImportError`` if some other exception is not already being\npropagated. Otherwise the loader returns the module that was loaded\nand initialized.\n\nWhen step (1) finishes without raising an exception, step (2) can\nbegin.\n\nThe first form of ``import`` statement binds the module name in the\nlocal namespace to the module object, and then goes on to import the\nnext identifier, if any.  If the module name is followed by ``as``,\nthe name following ``as`` is used as the local name for the module.\n\nThe ``from`` form does not bind the module name: it goes through the\nlist of identifiers, looks each one of them up in the module found in\nstep (1), and binds the name in the local namespace to the object thus\nfound.  As with the first form of ``import``, an alternate local name\ncan be supplied by specifying "``as`` localname".  If a name is not\nfound, ``ImportError`` is raised.  If the list of identifiers is\nreplaced by a star (``\'*\'``), all public names defined in the module\nare bound in the local namespace of the ``import`` statement..\n\nThe *public names* defined by a module are determined by checking the\nmodule\'s namespace for a variable named ``__all__``; if defined, it\nmust be a sequence of strings which are names defined or imported by\nthat module.  The names given in ``__all__`` are all considered public\nand are required to exist.  If ``__all__`` is not defined, the set of\npublic names includes all names found in the module\'s namespace which\ndo not begin with an underscore character (``\'_\'``). ``__all__``\nshould contain the entire public API. It is intended to avoid\naccidentally exporting items that are not part of the API (such as\nlibrary modules which were imported and used within the module).\n\nThe ``from`` form with ``*`` may only occur in a module scope.  If the\nwild card form of import --- ``import *`` --- is used in a function\nand the function contains or is a nested block with free variables,\nthe compiler will raise a ``SyntaxError``.\n\nWhen specifying what module to import you do not have to specify the\nabsolute name of the module. When a module or package is contained\nwithin another package it is possible to make a relative import within\nthe same top package without having to mention the package name. By\nusing leading dots in the specified module or package after ``from``\nyou can specify how high to traverse up the current package hierarchy\nwithout specifying exact names. One leading dot means the current\npackage where the module making the import exists. Two dots means up\none package level. Three dots is up two levels, etc. So if you execute\n``from . import mod`` from a module in the ``pkg`` package then you\nwill end up importing ``pkg.mod``. If you execute ``from ..subpkg2\nimprt mod`` from within ``pkg.subpkg1`` you will import\n``pkg.subpkg2.mod``. The specification for relative imports is\ncontained within **PEP 328**.\n\n``importlib.import_module()`` is provided to support applications that\ndetermine which modules need to be loaded dynamically.\n\n\nFuture statements\n=================\n\nA *future statement* is a directive to the compiler that a particular\nmodule should be compiled using syntax or semantics that will be\navailable in a specified future release of Python.  The future\nstatement is intended to ease migration to future versions of Python\nthat introduce incompatible changes to the language.  It allows use of\nthe new features on a per-module basis before the release in which the\nfeature becomes standard.\n\n   future_statement ::= "from" "__future__" "import" feature ["as" name]\n                        ("," feature ["as" name])*\n                        | "from" "__future__" "import" "(" feature ["as" name]\n                        ("," feature ["as" name])* [","] ")"\n   feature          ::= identifier\n   name             ::= identifier\n\nA future statement must appear near the top of the module.  The only\nlines that can appear before a future statement are:\n\n* the module docstring (if any),\n\n* comments,\n\n* blank lines, and\n\n* other future statements.\n\nThe features recognized by Python 2.6 are ``unicode_literals``,\n``print_function``, ``absolute_import``, ``division``, ``generators``,\n``nested_scopes`` and ``with_statement``.  ``generators``,\n``with_statement``, ``nested_scopes`` are redundant in Python version\n2.6 and above because they are always enabled.\n\nA future statement is recognized and treated specially at compile\ntime: Changes to the semantics of core constructs are often\nimplemented by generating different code.  It may even be the case\nthat a new feature introduces new incompatible syntax (such as a new\nreserved word), in which case the compiler may need to parse the\nmodule differently.  Such decisions cannot be pushed off until\nruntime.\n\nFor any given release, the compiler knows which feature names have\nbeen defined, and raises a compile-time error if a future statement\ncontains a feature not known to it.\n\nThe direct runtime semantics are the same as for any import statement:\nthere is a standard module ``__future__``, described later, and it\nwill be imported in the usual way at the time the future statement is\nexecuted.\n\nThe interesting runtime semantics depend on the specific feature\nenabled by the future statement.\n\nNote that there is nothing special about the statement:\n\n   import __future__ [as name]\n\nThat is not a future statement; it\'s an ordinary import statement with\nno special semantics or syntax restrictions.\n\nCode compiled by an ``exec`` statement or calls to the built-in\nfunctions ``compile()`` and ``execfile()`` that occur in a module\n``M`` containing a future statement will, by default, use the new\nsyntax or semantics associated with the future statement.  This can,\nstarting with Python 2.2 be controlled by optional arguments to\n``compile()`` --- see the documentation of that function for details.\n\nA future statement typed at an interactive interpreter prompt will\ntake effect for the rest of the interpreter session.  If an\ninterpreter is started with the *-i* option, is passed a script name\nto execute, and the script includes a future statement, it will be in\neffect in the interactive session started after the script is\nexecuted.\n\nSee also:\n\n   **PEP 236** - Back to the __future__\n      The original proposal for the __future__ mechanism.\n',
+ 'import': u'\nThe ``import`` statement\n************************\n\n   import_stmt     ::= "import" module ["as" name] ( "," module ["as" name] )*\n                   | "from" relative_module "import" identifier ["as" name]\n                   ( "," identifier ["as" name] )*\n                   | "from" relative_module "import" "(" identifier ["as" name]\n                   ( "," identifier ["as" name] )* [","] ")"\n                   | "from" module "import" "*"\n   module          ::= (identifier ".")* identifier\n   relative_module ::= "."* module | "."+\n   name            ::= identifier\n\nImport statements are executed in two steps: (1) find a module, and\ninitialize it if necessary; (2) define a name or names in the local\nnamespace (of the scope where the ``import`` statement occurs). The\nstatement comes in two forms differing on whether it uses the ``from``\nkeyword. The first form (without ``from``) repeats these steps for\neach identifier in the list. The form with ``from`` performs step (1)\nonce, and then performs step (2) repeatedly.\n\nTo understand how step (1) occurs, one must first understand how\nPython handles hierarchical naming of modules. To help organize\nmodules and provide a hierarchy in naming, Python has a concept of\npackages. A package can contain other packages and modules while\nmodules cannot contain other modules or packages. From a file system\nperspective, packages are directories and modules are files. The\noriginal specification for packages is still available to read,\nalthough minor details have changed since the writing of that\ndocument.\n\nOnce the name of the module is known (unless otherwise specified, the\nterm "module" will refer to both packages and modules), searching for\nthe module or package can begin. The first place checked is\n``sys.modules``, the cache of all modules that have been imported\npreviously. If the module is found there then it is used in step (2)\nof import.\n\nIf the module is not found in the cache, then ``sys.meta_path`` is\nsearched (the specification for ``sys.meta_path`` can be found in\n**PEP 302**). The object is a list of *finder* objects which are\nqueried in order as to whether they know how to load the module by\ncalling their ``find_module()`` method with the name of the module. If\nthe module happens to be contained within a package (as denoted by the\nexistence of a dot in the name), then a second argument to\n``find_module()`` is given as the value of the ``__path__`` attribute\nfrom the parent package (everything up to the last dot in the name of\nthe module being imported). If a finder can find the module it returns\na *loader* (discussed later) or returns ``None``.\n\nIf none of the finders on ``sys.meta_path`` are able to find the\nmodule then some implicitly defined finders are queried.\nImplementations of Python vary in what implicit meta path finders are\ndefined. The one they all do define, though, is one that handles\n``sys.path_hooks``, ``sys.path_importer_cache``, and ``sys.path``.\n\nThe implicit finder searches for the requested module in the "paths"\nspecified in one of two places ("paths" do not have to be file system\npaths). If the module being imported is supposed to be contained\nwithin a package then the second argument passed to ``find_module()``,\n``__path__`` on the parent package, is used as the source of paths. If\nthe module is not contained in a package then ``sys.path`` is used as\nthe source of paths.\n\nOnce the source of paths is chosen it is iterated over to find a\nfinder that can handle that path. The dict at\n``sys.path_importer_cache`` caches finders for paths and is checked\nfor a finder. If the path does not have a finder cached then\n``sys.path_hooks`` is searched by calling each object in the list with\na single argument of the path, returning a finder or raises\n``ImportError``. If a finder is returned then it is cached in\n``sys.path_importer_cache`` and then used for that path entry. If no\nfinder can be found but the path exists then a value of ``None`` is\nstored in ``sys.path_importer_cache`` to signify that an implicit,\nfile-based finder that handles modules stored as individual files\nshould be used for that path. If the path does not exist then a finder\nwhich always returns ``None`` is placed in the cache for the path.\n\nIf no finder can find the module then ``ImportError`` is raised.\nOtherwise some finder returned a loader whose ``load_module()`` method\nis called with the name of the module to load (see **PEP 302** for the\noriginal definition of loaders). A loader has several responsibilities\nto perform on a module it loads. First, if the module already exists\nin ``sys.modules`` (a possibility if the loader is called outside of\nthe import machinery) then it is to use that module for initialization\nand not a new module. But if the module does not exist in\n``sys.modules`` then it is to be added to that dict before\ninitialization begins. If an error occurs during loading of the module\nand it was added to ``sys.modules`` it is to be removed from the dict.\nIf an error occurs but the module was already in ``sys.modules`` it is\nleft in the dict.\n\nThe loader must set several attributes on the module. ``__name__`` is\nto be set to the name of the module. ``__file__`` is to be the "path"\nto the file unless the module is built-in (and thus listed in\n``sys.builtin_module_names``) in which case the attribute is not set.\nIf what is being imported is a package then ``__path__`` is to be set\nto a list of paths to be searched when looking for modules and\npackages contained within the package being imported. ``__package__``\nis optional but should be set to the name of package that contains the\nmodule or package (the empty string is used for module not contained\nin a package). ``__loader__`` is also optional but should be set to\nthe loader object that is loading the module.\n\nIf an error occurs during loading then the loader raises\n``ImportError`` if some other exception is not already being\npropagated. Otherwise the loader returns the module that was loaded\nand initialized.\n\nWhen step (1) finishes without raising an exception, step (2) can\nbegin.\n\nThe first form of ``import`` statement binds the module name in the\nlocal namespace to the module object, and then goes on to import the\nnext identifier, if any.  If the module name is followed by ``as``,\nthe name following ``as`` is used as the local name for the module.\n\nThe ``from`` form does not bind the module name: it goes through the\nlist of identifiers, looks each one of them up in the module found in\nstep (1), and binds the name in the local namespace to the object thus\nfound.  As with the first form of ``import``, an alternate local name\ncan be supplied by specifying "``as`` localname".  If a name is not\nfound, ``ImportError`` is raised.  If the list of identifiers is\nreplaced by a star (``\'*\'``), all public names defined in the module\nare bound in the local namespace of the ``import`` statement..\n\nThe *public names* defined by a module are determined by checking the\nmodule\'s namespace for a variable named ``__all__``; if defined, it\nmust be a sequence of strings which are names defined or imported by\nthat module.  The names given in ``__all__`` are all considered public\nand are required to exist.  If ``__all__`` is not defined, the set of\npublic names includes all names found in the module\'s namespace which\ndo not begin with an underscore character (``\'_\'``). ``__all__``\nshould contain the entire public API. It is intended to avoid\naccidentally exporting items that are not part of the API (such as\nlibrary modules which were imported and used within the module).\n\nThe ``from`` form with ``*`` may only occur in a module scope.  If the\nwild card form of import --- ``import *`` --- is used in a function\nand the function contains or is a nested block with free variables,\nthe compiler will raise a ``SyntaxError``.\n\nWhen specifying what module to import you do not have to specify the\nabsolute name of the module. When a module or package is contained\nwithin another package it is possible to make a relative import within\nthe same top package without having to mention the package name. By\nusing leading dots in the specified module or package after ``from``\nyou can specify how high to traverse up the current package hierarchy\nwithout specifying exact names. One leading dot means the current\npackage where the module making the import exists. Two dots means up\none package level. Three dots is up two levels, etc. So if you execute\n``from . import mod`` from a module in the ``pkg`` package then you\nwill end up importing ``pkg.mod``. If you execute ``from ..subpkg2\nimport mod`` from within ``pkg.subpkg1`` you will import\n``pkg.subpkg2.mod``. The specification for relative imports is\ncontained within **PEP 328**.\n\n``importlib.import_module()`` is provided to support applications that\ndetermine which modules need to be loaded dynamically.\n\n\nFuture statements\n=================\n\nA *future statement* is a directive to the compiler that a particular\nmodule should be compiled using syntax or semantics that will be\navailable in a specified future release of Python.  The future\nstatement is intended to ease migration to future versions of Python\nthat introduce incompatible changes to the language.  It allows use of\nthe new features on a per-module basis before the release in which the\nfeature becomes standard.\n\n   future_statement ::= "from" "__future__" "import" feature ["as" name]\n                        ("," feature ["as" name])*\n                        | "from" "__future__" "import" "(" feature ["as" name]\n                        ("," feature ["as" name])* [","] ")"\n   feature          ::= identifier\n   name             ::= identifier\n\nA future statement must appear near the top of the module.  The only\nlines that can appear before a future statement are:\n\n* the module docstring (if any),\n\n* comments,\n\n* blank lines, and\n\n* other future statements.\n\nThe features recognized by Python 2.6 are ``unicode_literals``,\n``print_function``, ``absolute_import``, ``division``, ``generators``,\n``nested_scopes`` and ``with_statement``.  ``generators``,\n``with_statement``, ``nested_scopes`` are redundant in Python version\n2.6 and above because they are always enabled.\n\nA future statement is recognized and treated specially at compile\ntime: Changes to the semantics of core constructs are often\nimplemented by generating different code.  It may even be the case\nthat a new feature introduces new incompatible syntax (such as a new\nreserved word), in which case the compiler may need to parse the\nmodule differently.  Such decisions cannot be pushed off until\nruntime.\n\nFor any given release, the compiler knows which feature names have\nbeen defined, and raises a compile-time error if a future statement\ncontains a feature not known to it.\n\nThe direct runtime semantics are the same as for any import statement:\nthere is a standard module ``__future__``, described later, and it\nwill be imported in the usual way at the time the future statement is\nexecuted.\n\nThe interesting runtime semantics depend on the specific feature\nenabled by the future statement.\n\nNote that there is nothing special about the statement:\n\n   import __future__ [as name]\n\nThat is not a future statement; it\'s an ordinary import statement with\nno special semantics or syntax restrictions.\n\nCode compiled by an ``exec`` statement or calls to the built-in\nfunctions ``compile()`` and ``execfile()`` that occur in a module\n``M`` containing a future statement will, by default, use the new\nsyntax or semantics associated with the future statement.  This can,\nstarting with Python 2.2 be controlled by optional arguments to\n``compile()`` --- see the documentation of that function for details.\n\nA future statement typed at an interactive interpreter prompt will\ntake effect for the rest of the interpreter session.  If an\ninterpreter is started with the *-i* option, is passed a script name\nto execute, and the script includes a future statement, it will be in\neffect in the interactive session started after the script is\nexecuted.\n\nSee also:\n\n   **PEP 236** - Back to the __future__\n      The original proposal for the __future__ mechanism.\n',
  'in': u'\nComparisons\n***********\n\nUnlike C, all comparison operations in Python have the same priority,\nwhich is lower than that of any arithmetic, shifting or bitwise\noperation.  Also unlike C, expressions like ``a < b < c`` have the\ninterpretation that is conventional in mathematics:\n\n   comparison    ::= or_expr ( comp_operator or_expr )*\n   comp_operator ::= "<" | ">" | "==" | ">=" | "<=" | "<>" | "!="\n                     | "is" ["not"] | ["not"] "in"\n\nComparisons yield boolean values: ``True`` or ``False``.\n\nComparisons can be chained arbitrarily, e.g., ``x < y <= z`` is\nequivalent to ``x < y and y <= z``, except that ``y`` is evaluated\nonly once (but in both cases ``z`` is not evaluated at all when ``x <\ny`` is found to be false).\n\nFormally, if *a*, *b*, *c*, ..., *y*, *z* are expressions and *op1*,\n*op2*, ..., *opN* are comparison operators, then ``a op1 b op2 c ... y\nopN z`` is equivalent to ``a op1 b and b op2 c and ... y opN z``,\nexcept that each expression is evaluated at most once.\n\nNote that ``a op1 b op2 c`` doesn\'t imply any kind of comparison\nbetween *a* and *c*, so that, e.g., ``x < y > z`` is perfectly legal\n(though perhaps not pretty).\n\nThe forms ``<>`` and ``!=`` are equivalent; for consistency with C,\n``!=`` is preferred; where ``!=`` is mentioned below ``<>`` is also\naccepted.  The ``<>`` spelling is considered obsolescent.\n\nThe operators ``<``, ``>``, ``==``, ``>=``, ``<=``, and ``!=`` compare\nthe values of two objects.  The objects need not have the same type.\nIf both are numbers, they are converted to a common type.  Otherwise,\nobjects of different types *always* compare unequal, and are ordered\nconsistently but arbitrarily. You can control comparison behavior of\nobjects of non-built-in types by defining a ``__cmp__`` method or rich\ncomparison methods like ``__gt__``, described in section *Special\nmethod names*.\n\n(This unusual definition of comparison was used to simplify the\ndefinition of operations like sorting and the ``in`` and ``not in``\noperators. In the future, the comparison rules for objects of\ndifferent types are likely to change.)\n\nComparison of objects of the same type depends on the type:\n\n* Numbers are compared arithmetically.\n\n* Strings are compared lexicographically using the numeric equivalents\n  (the result of the built-in function ``ord()``) of their characters.\n  Unicode and 8-bit strings are fully interoperable in this behavior.\n  [4]\n\n* Tuples and lists are compared lexicographically using comparison of\n  corresponding elements.  This means that to compare equal, each\n  element must compare equal and the two sequences must be of the same\n  type and have the same length.\n\n  If not equal, the sequences are ordered the same as their first\n  differing elements.  For example, ``cmp([1,2,x], [1,2,y])`` returns\n  the same as ``cmp(x,y)``.  If the corresponding element does not\n  exist, the shorter sequence is ordered first (for example, ``[1,2] <\n  [1,2,3]``).\n\n* Mappings (dictionaries) compare equal if and only if their sorted\n  (key, value) lists compare equal. [5] Outcomes other than equality\n  are resolved consistently, but are not otherwise defined. [6]\n\n* Most other objects of built-in types compare unequal unless they are\n  the same object; the choice whether one object is considered smaller\n  or larger than another one is made arbitrarily but consistently\n  within one execution of a program.\n\nThe operators ``in`` and ``not in`` test for collection membership.\n``x in s`` evaluates to true if *x* is a member of the collection *s*,\nand false otherwise.  ``x not in s`` returns the negation of ``x in\ns``. The collection membership test has traditionally been bound to\nsequences; an object is a member of a collection if the collection is\na sequence and contains an element equal to that object.  However, it\nmake sense for many other object types to support membership tests\nwithout being a sequence.  In particular, dictionaries (for keys) and\nsets support membership testing.\n\nFor the list and tuple types, ``x in y`` is true if and only if there\nexists an index *i* such that ``x == y[i]`` is true.\n\nFor the Unicode and string types, ``x in y`` is true if and only if\n*x* is a substring of *y*.  An equivalent test is ``y.find(x) != -1``.\nNote, *x* and *y* need not be the same type; consequently, ``u\'ab\' in\n\'abc\'`` will return ``True``. Empty strings are always considered to\nbe a substring of any other string, so ``"" in "abc"`` will return\n``True``.\n\nChanged in version 2.3: Previously, *x* was required to be a string of\nlength ``1``.\n\nFor user-defined classes which define the ``__contains__()`` method,\n``x in y`` is true if and only if ``y.__contains__(x)`` is true.\n\nFor user-defined classes which do not define ``__contains__()`` but do\ndefine ``__iter__()``, ``x in y`` is true if some value ``z`` with ``x\n== z`` is produced while iterating over ``y``.  If an exception is\nraised during the iteration, it is as if ``in`` raised that exception.\n\nLastly, the old-style iteration protocol is tried: if a class defines\n``__getitem__()``, ``x in y`` is true if and only if there is a non-\nnegative integer index *i* such that ``x == y[i]``, and all lower\ninteger indices do not raise ``IndexError`` exception. (If any other\nexception is raised, it is as if ``in`` raised that exception).\n\nThe operator ``not in`` is defined to have the inverse true value of\n``in``.\n\nThe operators ``is`` and ``is not`` test for object identity: ``x is\ny`` is true if and only if *x* and *y* are the same object.  ``x is\nnot y`` yields the inverse truth value. [7]\n',
  'integers': u'\nInteger and long integer literals\n*********************************\n\nInteger and long integer literals are described by the following\nlexical definitions:\n\n   longinteger    ::= integer ("l" | "L")\n   integer        ::= decimalinteger | octinteger | hexinteger | bininteger\n   decimalinteger ::= nonzerodigit digit* | "0"\n   octinteger     ::= "0" ("o" | "O") octdigit+ | "0" octdigit+\n   hexinteger     ::= "0" ("x" | "X") hexdigit+\n   bininteger     ::= "0" ("b" | "B") bindigit+\n   nonzerodigit   ::= "1"..."9"\n   octdigit       ::= "0"..."7"\n   bindigit       ::= "0" | "1"\n   hexdigit       ::= digit | "a"..."f" | "A"..."F"\n\nAlthough both lower case ``\'l\'`` and upper case ``\'L\'`` are allowed as\nsuffix for long integers, it is strongly recommended to always use\n``\'L\'``, since the letter ``\'l\'`` looks too much like the digit\n``\'1\'``.\n\nPlain integer literals that are above the largest representable plain\ninteger (e.g., 2147483647 when using 32-bit arithmetic) are accepted\nas if they were long integers instead. [1]  There is no limit for long\ninteger literals apart from what can be stored in available memory.\n\nSome examples of plain integer literals (first row) and long integer\nliterals (second and third rows):\n\n   7     2147483647                        0177\n   3L    79228162514264337593543950336L    0377L   0x100000000L\n         79228162514264337593543950336             0xdeadbeef\n',
  'lambda': u'\nLambdas\n*******\n\n   lambda_form     ::= "lambda" [parameter_list]: expression\n   old_lambda_form ::= "lambda" [parameter_list]: old_expression\n\nLambda forms (lambda expressions) have the same syntactic position as\nexpressions.  They are a shorthand to create anonymous functions; the\nexpression ``lambda arguments: expression`` yields a function object.\nThe unnamed object behaves like a function object defined with\n\n   def name(arguments):\n       return expression\n\nSee section *Function definitions* for the syntax of parameter lists.\nNote that functions created with lambda forms cannot contain\nstatements.\n',
  'lists': u'\nList displays\n*************\n\nA list display is a possibly empty series of expressions enclosed in\nsquare brackets:\n\n   list_display        ::= "[" [expression_list | list_comprehension] "]"\n   list_comprehension  ::= expression list_for\n   list_for            ::= "for" target_list "in" old_expression_list [list_iter]\n   old_expression_list ::= old_expression [("," old_expression)+ [","]]\n   old_expression      ::= or_test | old_lambda_form\n   list_iter           ::= list_for | list_if\n   list_if             ::= "if" old_expression [list_iter]\n\nA list display yields a new list object.  Its contents are specified\nby providing either a list of expressions or a list comprehension.\nWhen a comma-separated list of expressions is supplied, its elements\nare evaluated from left to right and placed into the list object in\nthat order.  When a list comprehension is supplied, it consists of a\nsingle expression followed by at least one ``for`` clause and zero or\nmore ``for`` or ``if`` clauses.  In this case, the elements of the new\nlist are those that would be produced by considering each of the\n``for`` or ``if`` clauses a block, nesting from left to right, and\nevaluating the expression to produce a list element each time the\ninnermost block is reached [1].\n',
- 'naming': u"\nNaming and binding\n******************\n\n*Names* refer to objects.  Names are introduced by name binding\noperations. Each occurrence of a name in the program text refers to\nthe *binding* of that name established in the innermost function block\ncontaining the use.\n\nA *block* is a piece of Python program text that is executed as a\nunit. The following are blocks: a module, a function body, and a class\ndefinition. Each command typed interactively is a block.  A script\nfile (a file given as standard input to the interpreter or specified\non the interpreter command line the first argument) is a code block.\nA script command (a command specified on the interpreter command line\nwith the '**-c**' option) is a code block.  The file read by the\nbuilt-in function ``execfile()`` is a code block.  The string argument\npassed to the built-in function ``eval()`` and to the ``exec``\nstatement is a code block. The expression read and evaluated by the\nbuilt-in function ``input()`` is a code block.\n\nA code block is executed in an *execution frame*.  A frame contains\nsome administrative information (used for debugging) and determines\nwhere and how execution continues after the code block's execution has\ncompleted.\n\nA *scope* defines the visibility of a name within a block.  If a local\nvariable is defined in a block, its scope includes that block.  If the\ndefinition occurs in a function block, the scope extends to any blocks\ncontained within the defining one, unless a contained block introduces\na different binding for the name.  The scope of names defined in a\nclass block is limited to the class block; it does not extend to the\ncode blocks of methods -- this includes generator expressions since\nthey are implemented using a function scope.  This means that the\nfollowing will fail:\n\n   class A:\n       a = 42\n       b = list(a + i for i in range(10))\n\nWhen a name is used in a code block, it is resolved using the nearest\nenclosing scope.  The set of all such scopes visible to a code block\nis called the block's *environment*.\n\nIf a name is bound in a block, it is a local variable of that block.\nIf a name is bound at the module level, it is a global variable.  (The\nvariables of the module code block are local and global.)  If a\nvariable is used in a code block but not defined there, it is a *free\nvariable*.\n\nWhen a name is not found at all, a ``NameError`` exception is raised.\nIf the name refers to a local variable that has not been bound, a\n``UnboundLocalError`` exception is raised.  ``UnboundLocalError`` is a\nsubclass of ``NameError``.\n\nThe following constructs bind names: formal parameters to functions,\n``import`` statements, class and function definitions (these bind the\nclass or function name in the defining block), and targets that are\nidentifiers if occurring in an assignment, ``for`` loop header, in the\nsecond position of an ``except`` clause header or after ``as`` in a\n``with`` statement.  The ``import`` statement of the form ``from ...\nimport *`` binds all names defined in the imported module, except\nthose beginning with an underscore.  This form may only be used at the\nmodule level.\n\nA target occurring in a ``del`` statement is also considered bound for\nthis purpose (though the actual semantics are to unbind the name).  It\nis illegal to unbind a name that is referenced by an enclosing scope;\nthe compiler will report a ``SyntaxError``.\n\nEach assignment or import statement occurs within a block defined by a\nclass or function definition or at the module level (the top-level\ncode block).\n\nIf a name binding operation occurs anywhere within a code block, all\nuses of the name within the block are treated as references to the\ncurrent block.  This can lead to errors when a name is used within a\nblock before it is bound. This rule is subtle.  Python lacks\ndeclarations and allows name binding operations to occur anywhere\nwithin a code block.  The local variables of a code block can be\ndetermined by scanning the entire text of the block for name binding\noperations.\n\nIf the global statement occurs within a block, all uses of the name\nspecified in the statement refer to the binding of that name in the\ntop-level namespace. Names are resolved in the top-level namespace by\nsearching the global namespace, i.e. the namespace of the module\ncontaining the code block, and the builtins namespace, the namespace\nof the module ``__builtin__``.  The global namespace is searched\nfirst.  If the name is not found there, the builtins namespace is\nsearched.  The global statement must precede all uses of the name.\n\nThe builtins namespace associated with the execution of a code block\nis actually found by looking up the name ``__builtins__`` in its\nglobal namespace; this should be a dictionary or a module (in the\nlatter case the module's dictionary is used).  By default, when in the\n``__main__`` module, ``__builtins__`` is the built-in module\n``__builtin__`` (note: no 's'); when in any other module,\n``__builtins__`` is an alias for the dictionary of the ``__builtin__``\nmodule itself.  ``__builtins__`` can be set to a user-created\ndictionary to create a weak form of restricted execution.\n\n**CPython implementation detail:** Users should not touch\n``__builtins__``; it is strictly an implementation detail.  Users\nwanting to override values in the builtins namespace should ``import``\nthe ``__builtin__`` (no 's') module and modify its attributes\nappropriately.\n\nThe namespace for a module is automatically created the first time a\nmodule is imported.  The main module for a script is always called\n``__main__``.\n\nThe global statement has the same scope as a name binding operation in\nthe same block.  If the nearest enclosing scope for a free variable\ncontains a global statement, the free variable is treated as a global.\n\nA class definition is an executable statement that may use and define\nnames. These references follow the normal rules for name resolution.\nThe namespace of the class definition becomes the attribute dictionary\nof the class.  Names defined at the class scope are not visible in\nmethods.\n\n\nInteraction with dynamic features\n=================================\n\nThere are several cases where Python statements are illegal when used\nin conjunction with nested scopes that contain free variables.\n\nIf a variable is referenced in an enclosing scope, it is illegal to\ndelete the name.  An error will be reported at compile time.\n\nIf the wild card form of import --- ``import *`` --- is used in a\nfunction and the function contains or is a nested block with free\nvariables, the compiler will raise a ``SyntaxError``.\n\nIf ``exec`` is used in a function and the function contains or is a\nnested block with free variables, the compiler will raise a\n``SyntaxError`` unless the exec explicitly specifies the local\nnamespace for the ``exec``.  (In other words, ``exec obj`` would be\nillegal, but ``exec obj in ns`` would be legal.)\n\nThe ``eval()``, ``execfile()``, and ``input()`` functions and the\n``exec`` statement do not have access to the full environment for\nresolving names.  Names may be resolved in the local and global\nnamespaces of the caller.  Free variables are not resolved in the\nnearest enclosing namespace, but in the global namespace. [1] The\n``exec`` statement and the ``eval()`` and ``execfile()`` functions\nhave optional arguments to override the global and local namespace.\nIf only one namespace is specified, it is used for both.\n",
+ 'naming': u"\nNaming and binding\n******************\n\n*Names* refer to objects.  Names are introduced by name binding\noperations. Each occurrence of a name in the program text refers to\nthe *binding* of that name established in the innermost function block\ncontaining the use.\n\nA *block* is a piece of Python program text that is executed as a\nunit. The following are blocks: a module, a function body, and a class\ndefinition. Each command typed interactively is a block.  A script\nfile (a file given as standard input to the interpreter or specified\non the interpreter command line the first argument) is a code block.\nA script command (a command specified on the interpreter command line\nwith the '**-c**' option) is a code block.  The file read by the\nbuilt-in function ``execfile()`` is a code block.  The string argument\npassed to the built-in function ``eval()`` and to the ``exec``\nstatement is a code block. The expression read and evaluated by the\nbuilt-in function ``input()`` is a code block.\n\nA code block is executed in an *execution frame*.  A frame contains\nsome administrative information (used for debugging) and determines\nwhere and how execution continues after the code block's execution has\ncompleted.\n\nA *scope* defines the visibility of a name within a block.  If a local\nvariable is defined in a block, its scope includes that block.  If the\ndefinition occurs in a function block, the scope extends to any blocks\ncontained within the defining one, unless a contained block introduces\na different binding for the name.  The scope of names defined in a\nclass block is limited to the class block; it does not extend to the\ncode blocks of methods -- this includes generator expressions since\nthey are implemented using a function scope.  This means that the\nfollowing will fail:\n\n   class A:\n       a = 42\n       b = list(a + i for i in range(10))\n\nWhen a name is used in a code block, it is resolved using the nearest\nenclosing scope.  The set of all such scopes visible to a code block\nis called the block's *environment*.\n\nIf a name is bound in a block, it is a local variable of that block.\nIf a name is bound at the module level, it is a global variable.  (The\nvariables of the module code block are local and global.)  If a\nvariable is used in a code block but not defined there, it is a *free\nvariable*.\n\nWhen a name is not found at all, a ``NameError`` exception is raised.\nIf the name refers to a local variable that has not been bound, a\n``UnboundLocalError`` exception is raised.  ``UnboundLocalError`` is a\nsubclass of ``NameError``.\n\nThe following constructs bind names: formal parameters to functions,\n``import`` statements, class and function definitions (these bind the\nclass or function name in the defining block), and targets that are\nidentifiers if occurring in an assignment, ``for`` loop header, in the\nsecond position of an ``except`` clause header or after ``as`` in a\n``with`` statement.  The ``import`` statement of the form ``from ...\nimport *`` binds all names defined in the imported module, except\nthose beginning with an underscore.  This form may only be used at the\nmodule level.\n\nA target occurring in a ``del`` statement is also considered bound for\nthis purpose (though the actual semantics are to unbind the name).  It\nis illegal to unbind a name that is referenced by an enclosing scope;\nthe compiler will report a ``SyntaxError``.\n\nEach assignment or import statement occurs within a block defined by a\nclass or function definition or at the module level (the top-level\ncode block).\n\nIf a name binding operation occurs anywhere within a code block, all\nuses of the name within the block are treated as references to the\ncurrent block.  This can lead to errors when a name is used within a\nblock before it is bound. This rule is subtle.  Python lacks\ndeclarations and allows name binding operations to occur anywhere\nwithin a code block.  The local variables of a code block can be\ndetermined by scanning the entire text of the block for name binding\noperations.\n\nIf the global statement occurs within a block, all uses of the name\nspecified in the statement refer to the binding of that name in the\ntop-level namespace. Names are resolved in the top-level namespace by\nsearching the global namespace, i.e. the namespace of the module\ncontaining the code block, and the builtins namespace, the namespace\nof the module ``__builtin__``.  The global namespace is searched\nfirst.  If the name is not found there, the builtins namespace is\nsearched.  The global statement must precede all uses of the name.\n\nThe builtins namespace associated with the execution of a code block\nis actually found by looking up the name ``__builtins__`` in its\nglobal namespace; this should be a dictionary or a module (in the\nlatter case the module's dictionary is used).  By default, when in the\n``__main__`` module, ``__builtins__`` is the built-in module\n``__builtin__`` (note: no 's'); when in any other module,\n``__builtins__`` is an alias for the dictionary of the ``__builtin__``\nmodule itself.  ``__builtins__`` can be set to a user-created\ndictionary to create a weak form of restricted execution.\n\n**CPython implementation detail:** Users should not touch\n``__builtins__``; it is strictly an implementation detail.  Users\nwanting to override values in the builtins namespace should ``import``\nthe ``__builtin__`` (no 's') module and modify its attributes\nappropriately.\n\nThe namespace for a module is automatically created the first time a\nmodule is imported.  The main module for a script is always called\n``__main__``.\n\nThe ``global`` statement has the same scope as a name binding\noperation in the same block.  If the nearest enclosing scope for a\nfree variable contains a global statement, the free variable is\ntreated as a global.\n\nA class definition is an executable statement that may use and define\nnames. These references follow the normal rules for name resolution.\nThe namespace of the class definition becomes the attribute dictionary\nof the class.  Names defined at the class scope are not visible in\nmethods.\n\n\nInteraction with dynamic features\n=================================\n\nThere are several cases where Python statements are illegal when used\nin conjunction with nested scopes that contain free variables.\n\nIf a variable is referenced in an enclosing scope, it is illegal to\ndelete the name.  An error will be reported at compile time.\n\nIf the wild card form of import --- ``import *`` --- is used in a\nfunction and the function contains or is a nested block with free\nvariables, the compiler will raise a ``SyntaxError``.\n\nIf ``exec`` is used in a function and the function contains or is a\nnested block with free variables, the compiler will raise a\n``SyntaxError`` unless the exec explicitly specifies the local\nnamespace for the ``exec``.  (In other words, ``exec obj`` would be\nillegal, but ``exec obj in ns`` would be legal.)\n\nThe ``eval()``, ``execfile()``, and ``input()`` functions and the\n``exec`` statement do not have access to the full environment for\nresolving names.  Names may be resolved in the local and global\nnamespaces of the caller.  Free variables are not resolved in the\nnearest enclosing namespace, but in the global namespace. [1] The\n``exec`` statement and the ``eval()`` and ``execfile()`` functions\nhave optional arguments to override the global and local namespace.\nIf only one namespace is specified, it is used for both.\n",
  'numbers': u"\nNumeric literals\n****************\n\nThere are four types of numeric literals: plain integers, long\nintegers, floating point numbers, and imaginary numbers.  There are no\ncomplex literals (complex numbers can be formed by adding a real\nnumber and an imaginary number).\n\nNote that numeric literals do not include a sign; a phrase like ``-1``\nis actually an expression composed of the unary operator '``-``' and\nthe literal ``1``.\n",
  'numeric-types': u'\nEmulating numeric types\n***********************\n\nThe following methods can be defined to emulate numeric objects.\nMethods corresponding to operations that are not supported by the\nparticular kind of number implemented (e.g., bitwise operations for\nnon-integral numbers) should be left undefined.\n\nobject.__add__(self, other)\nobject.__sub__(self, other)\nobject.__mul__(self, other)\nobject.__floordiv__(self, other)\nobject.__mod__(self, other)\nobject.__divmod__(self, other)\nobject.__pow__(self, other[, modulo])\nobject.__lshift__(self, other)\nobject.__rshift__(self, other)\nobject.__and__(self, other)\nobject.__xor__(self, other)\nobject.__or__(self, other)\n\n   These methods are called to implement the binary arithmetic\n   operations (``+``, ``-``, ``*``, ``//``, ``%``, ``divmod()``,\n   ``pow()``, ``**``, ``<<``, ``>>``, ``&``, ``^``, ``|``).  For\n   instance, to evaluate the expression ``x + y``, where *x* is an\n   instance of a class that has an ``__add__()`` method,\n   ``x.__add__(y)`` is called.  The ``__divmod__()`` method should be\n   the equivalent to using ``__floordiv__()`` and ``__mod__()``; it\n   should not be related to ``__truediv__()`` (described below).  Note\n   that ``__pow__()`` should be defined to accept an optional third\n   argument if the ternary version of the built-in ``pow()`` function\n   is to be supported.\n\n   If one of those methods does not support the operation with the\n   supplied arguments, it should return ``NotImplemented``.\n\nobject.__div__(self, other)\nobject.__truediv__(self, other)\n\n   The division operator (``/``) is implemented by these methods.  The\n   ``__truediv__()`` method is used when ``__future__.division`` is in\n   effect, otherwise ``__div__()`` is used.  If only one of these two\n   methods is defined, the object will not support division in the\n   alternate context; ``TypeError`` will be raised instead.\n\nobject.__radd__(self, other)\nobject.__rsub__(self, other)\nobject.__rmul__(self, other)\nobject.__rdiv__(self, other)\nobject.__rtruediv__(self, other)\nobject.__rfloordiv__(self, other)\nobject.__rmod__(self, other)\nobject.__rdivmod__(self, other)\nobject.__rpow__(self, other)\nobject.__rlshift__(self, other)\nobject.__rrshift__(self, other)\nobject.__rand__(self, other)\nobject.__rxor__(self, other)\nobject.__ror__(self, other)\n\n   These methods are called to implement the binary arithmetic\n   operations (``+``, ``-``, ``*``, ``/``, ``%``, ``divmod()``,\n   ``pow()``, ``**``, ``<<``, ``>>``, ``&``, ``^``, ``|``) with\n   reflected (swapped) operands.  These functions are only called if\n   the left operand does not support the corresponding operation and\n   the operands are of different types. [2] For instance, to evaluate\n   the expression ``x - y``, where *y* is an instance of a class that\n   has an ``__rsub__()`` method, ``y.__rsub__(x)`` is called if\n   ``x.__sub__(y)`` returns *NotImplemented*.\n\n   Note that ternary ``pow()`` will not try calling ``__rpow__()``\n   (the coercion rules would become too complicated).\n\n   Note: If the right operand\'s type is a subclass of the left operand\'s\n     type and that subclass provides the reflected method for the\n     operation, this method will be called before the left operand\'s\n     non-reflected method.  This behavior allows subclasses to\n     override their ancestors\' operations.\n\nobject.__iadd__(self, other)\nobject.__isub__(self, other)\nobject.__imul__(self, other)\nobject.__idiv__(self, other)\nobject.__itruediv__(self, other)\nobject.__ifloordiv__(self, other)\nobject.__imod__(self, other)\nobject.__ipow__(self, other[, modulo])\nobject.__ilshift__(self, other)\nobject.__irshift__(self, other)\nobject.__iand__(self, other)\nobject.__ixor__(self, other)\nobject.__ior__(self, other)\n\n   These methods are called to implement the augmented arithmetic\n   assignments (``+=``, ``-=``, ``*=``, ``/=``, ``//=``, ``%=``,\n   ``**=``, ``<<=``, ``>>=``, ``&=``, ``^=``, ``|=``).  These methods\n   should attempt to do the operation in-place (modifying *self*) and\n   return the result (which could be, but does not have to be,\n   *self*).  If a specific method is not defined, the augmented\n   assignment falls back to the normal methods.  For instance, to\n   execute the statement ``x += y``, where *x* is an instance of a\n   class that has an ``__iadd__()`` method, ``x.__iadd__(y)`` is\n   called.  If *x* is an instance of a class that does not define a\n   ``__iadd__()`` method, ``x.__add__(y)`` and ``y.__radd__(x)`` are\n   considered, as with the evaluation of ``x + y``.\n\nobject.__neg__(self)\nobject.__pos__(self)\nobject.__abs__(self)\nobject.__invert__(self)\n\n   Called to implement the unary arithmetic operations (``-``, ``+``,\n   ``abs()`` and ``~``).\n\nobject.__complex__(self)\nobject.__int__(self)\nobject.__long__(self)\nobject.__float__(self)\n\n   Called to implement the built-in functions ``complex()``,\n   ``int()``, ``long()``, and ``float()``.  Should return a value of\n   the appropriate type.\n\nobject.__oct__(self)\nobject.__hex__(self)\n\n   Called to implement the built-in functions ``oct()`` and ``hex()``.\n   Should return a string value.\n\nobject.__index__(self)\n\n   Called to implement ``operator.index()``.  Also called whenever\n   Python needs an integer object (such as in slicing).  Must return\n   an integer (int or long).\n\n   New in version 2.5.\n\nobject.__coerce__(self, other)\n\n   Called to implement "mixed-mode" numeric arithmetic.  Should either\n   return a 2-tuple containing *self* and *other* converted to a\n   common numeric type, or ``None`` if conversion is impossible.  When\n   the common type would be the type of ``other``, it is sufficient to\n   return ``None``, since the interpreter will also ask the other\n   object to attempt a coercion (but sometimes, if the implementation\n   of the other type cannot be changed, it is useful to do the\n   conversion to the other type here).  A return value of\n   ``NotImplemented`` is equivalent to returning ``None``.\n',
- 'objects': u'\nObjects, values and types\n*************************\n\n*Objects* are Python\'s abstraction for data.  All data in a Python\nprogram is represented by objects or by relations between objects. (In\na sense, and in conformance to Von Neumann\'s model of a "stored\nprogram computer," code is also represented by objects.)\n\nEvery object has an identity, a type and a value.  An object\'s\n*identity* never changes once it has been created; you may think of it\nas the object\'s address in memory.  The \'``is``\' operator compares the\nidentity of two objects; the ``id()`` function returns an integer\nrepresenting its identity (currently implemented as its address). An\nobject\'s *type* is also unchangeable. [1] An object\'s type determines\nthe operations that the object supports (e.g., "does it have a\nlength?") and also defines the possible values for objects of that\ntype.  The ``type()`` function returns an object\'s type (which is an\nobject itself).  The *value* of some objects can change.  Objects\nwhose value can change are said to be *mutable*; objects whose value\nis unchangeable once they are created are called *immutable*. (The\nvalue of an immutable container object that contains a reference to a\nmutable object can change when the latter\'s value is changed; however\nthe container is still considered immutable, because the collection of\nobjects it contains cannot be changed.  So, immutability is not\nstrictly the same as having an unchangeable value, it is more subtle.)\nAn object\'s mutability is determined by its type; for instance,\nnumbers, strings and tuples are immutable, while dictionaries and\nlists are mutable.\n\nObjects are never explicitly destroyed; however, when they become\nunreachable they may be garbage-collected.  An implementation is\nallowed to postpone garbage collection or omit it altogether --- it is\na matter of implementation quality how garbage collection is\nimplemented, as long as no objects are collected that are still\nreachable.\n\n**CPython implementation detail:** CPython currently uses a reference-\ncounting scheme with (optional) delayed detection of cyclically linked\ngarbage, which collects most objects as soon as they become\nunreachable, but is not guaranteed to collect garbage containing\ncircular references.  See the documentation of the ``gc`` module for\ninformation on controlling the collection of cyclic garbage. Other\nimplementations act differently and CPython may change.\n\nNote that the use of the implementation\'s tracing or debugging\nfacilities may keep objects alive that would normally be collectable.\nAlso note that catching an exception with a \'``try``...``except``\'\nstatement may keep objects alive.\n\nSome objects contain references to "external" resources such as open\nfiles or windows.  It is understood that these resources are freed\nwhen the object is garbage-collected, but since garbage collection is\nnot guaranteed to happen, such objects also provide an explicit way to\nrelease the external resource, usually a ``close()`` method. Programs\nare strongly recommended to explicitly close such objects.  The\n\'``try``...``finally``\' statement provides a convenient way to do\nthis.\n\nSome objects contain references to other objects; these are called\n*containers*. Examples of containers are tuples, lists and\ndictionaries.  The references are part of a container\'s value.  In\nmost cases, when we talk about the value of a container, we imply the\nvalues, not the identities of the contained objects; however, when we\ntalk about the mutability of a container, only the identities of the\nimmediately contained objects are implied.  So, if an immutable\ncontainer (like a tuple) contains a reference to a mutable object, its\nvalue changes if that mutable object is changed.\n\nTypes affect almost all aspects of object behavior.  Even the\nimportance of object identity is affected in some sense: for immutable\ntypes, operations that compute new values may actually return a\nreference to any existing object with the same type and value, while\nfor mutable objects this is not allowed.  E.g., after ``a = 1; b =\n1``, ``a`` and ``b`` may or may not refer to the same object with the\nvalue one, depending on the implementation, but after ``c = []; d =\n[]``, ``c`` and ``d`` are guaranteed to refer to two different,\nunique, newly created empty lists. (Note that ``c = d = []`` assigns\nthe same object to both ``c`` and ``d``.)\n',
- 'operator-summary': u'\nSummary\n*******\n\nThe following table summarizes the operator precedences in Python,\nfrom lowest precedence (least binding) to highest precedence (most\nbinding). Operators in the same box have the same precedence.  Unless\nthe syntax is explicitly given, operators are binary.  Operators in\nthe same box group left to right (except for comparisons, including\ntests, which all have the same precedence and chain from left to right\n--- see section *Comparisons* --- and exponentiation, which groups\nfrom right to left).\n\n+-------------------------------------------------+---------------------------------------+\n| Operator                                        | Description                           |\n+=================================================+=======================================+\n| ``lambda``                                      | Lambda expression                     |\n+-------------------------------------------------+---------------------------------------+\n| ``if`` -- ``else``                              | Conditional expression                |\n+-------------------------------------------------+---------------------------------------+\n| ``or``                                          | Boolean OR                            |\n+-------------------------------------------------+---------------------------------------+\n| ``and``                                         | Boolean AND                           |\n+-------------------------------------------------+---------------------------------------+\n| ``not`` *x*                                     | Boolean NOT                           |\n+-------------------------------------------------+---------------------------------------+\n| ``in``, ``not`` ``in``, ``is``, ``is not``,     | Comparisons, including membership     |\n| ``<``, ``<=``, ``>``, ``>=``, ``<>``, ``!=``,   | tests and identity tests,             |\n| ``==``                                          |                                       |\n+-------------------------------------------------+---------------------------------------+\n| ``|``                                           | Bitwise OR                            |\n+-------------------------------------------------+---------------------------------------+\n| ``^``                                           | Bitwise XOR                           |\n+-------------------------------------------------+---------------------------------------+\n| ``&``                                           | Bitwise AND                           |\n+-------------------------------------------------+---------------------------------------+\n| ``<<``, ``>>``                                  | Shifts                                |\n+-------------------------------------------------+---------------------------------------+\n| ``+``, ``-``                                    | Addition and subtraction              |\n+-------------------------------------------------+---------------------------------------+\n| ``*``, ``/``, ``//``, ``%``                     | Multiplication, division, remainder   |\n+-------------------------------------------------+---------------------------------------+\n| ``+x``, ``-x``, ``~x``                          | Positive, negative, bitwise NOT       |\n+-------------------------------------------------+---------------------------------------+\n| ``**``                                          | Exponentiation [8]                    |\n+-------------------------------------------------+---------------------------------------+\n| ``x[index]``, ``x[index:index]``,               | Subscription, slicing, call,          |\n| ``x(arguments...)``, ``x.attribute``            | attribute reference                   |\n+-------------------------------------------------+---------------------------------------+\n| ``(expressions...)``, ``[expressions...]``,     | Binding or tuple display, list        |\n| ``{key:datum...}``, ```expressions...```        | display, dictionary display, string   |\n|                                                 | conversion                            |\n+-------------------------------------------------+---------------------------------------+\n\n-[ Footnotes ]-\n\n[1] In Python 2.3 and later releases, a list comprehension "leaks" the\n    control variables of each ``for`` it contains into the containing\n    scope.  However, this behavior is deprecated, and relying on it\n    will not work in Python 3.0\n\n[2] While ``abs(x%y) < abs(y)`` is true mathematically, for floats it\n    may not be true numerically due to roundoff.  For example, and\n    assuming a platform on which a Python float is an IEEE 754 double-\n    precision number, in order that ``-1e-100 % 1e100`` have the same\n    sign as ``1e100``, the computed result is ``-1e-100 + 1e100``,\n    which is numerically exactly equal to ``1e100``.  Function\n    ``fmod()`` in the ``math`` module returns a result whose sign\n    matches the sign of the first argument instead, and so returns\n    ``-1e-100`` in this case. Which approach is more appropriate\n    depends on the application.\n\n[3] If x is very close to an exact integer multiple of y, it\'s\n    possible for ``floor(x/y)`` to be one larger than ``(x-x%y)/y``\n    due to rounding.  In such cases, Python returns the latter result,\n    in order to preserve that ``divmod(x,y)[0] * y + x % y`` be very\n    close to ``x``.\n\n[4] While comparisons between unicode strings make sense at the byte\n    level, they may be counter-intuitive to users. For example, the\n    strings ``u"\\u00C7"`` and ``u"\\u0043\\u0327"`` compare differently,\n    even though they both represent the same unicode character (LATIN\n    CAPITAL LETTER C WITH CEDILLA). To compare strings in a human\n    recognizable way, compare using ``unicodedata.normalize()``.\n\n[5] The implementation computes this efficiently, without constructing\n    lists or sorting.\n\n[6] Earlier versions of Python used lexicographic comparison of the\n    sorted (key, value) lists, but this was very expensive for the\n    common case of comparing for equality.  An even earlier version of\n    Python compared dictionaries by identity only, but this caused\n    surprises because people expected to be able to test a dictionary\n    for emptiness by comparing it to ``{}``.\n\n[7] Due to automatic garbage-collection, free lists, and the dynamic\n    nature of descriptors, you may notice seemingly unusual behaviour\n    in certain uses of the ``is`` operator, like those involving\n    comparisons between instance methods, or constants.  Check their\n    documentation for more info.\n\n[8] The power operator ``**`` binds less tightly than an arithmetic or\n    bitwise unary operator on its right, that is, ``2**-1`` is\n    ``0.5``.\n',
+ 'objects': u'\nObjects, values and types\n*************************\n\n*Objects* are Python\'s abstraction for data.  All data in a Python\nprogram is represented by objects or by relations between objects. (In\na sense, and in conformance to Von Neumann\'s model of a "stored\nprogram computer," code is also represented by objects.)\n\nEvery object has an identity, a type and a value.  An object\'s\n*identity* never changes once it has been created; you may think of it\nas the object\'s address in memory.  The \'``is``\' operator compares the\nidentity of two objects; the ``id()`` function returns an integer\nrepresenting its identity (currently implemented as its address). An\nobject\'s *type* is also unchangeable. [1] An object\'s type determines\nthe operations that the object supports (e.g., "does it have a\nlength?") and also defines the possible values for objects of that\ntype.  The ``type()`` function returns an object\'s type (which is an\nobject itself).  The *value* of some objects can change.  Objects\nwhose value can change are said to be *mutable*; objects whose value\nis unchangeable once they are created are called *immutable*. (The\nvalue of an immutable container object that contains a reference to a\nmutable object can change when the latter\'s value is changed; however\nthe container is still considered immutable, because the collection of\nobjects it contains cannot be changed.  So, immutability is not\nstrictly the same as having an unchangeable value, it is more subtle.)\nAn object\'s mutability is determined by its type; for instance,\nnumbers, strings and tuples are immutable, while dictionaries and\nlists are mutable.\n\nObjects are never explicitly destroyed; however, when they become\nunreachable they may be garbage-collected.  An implementation is\nallowed to postpone garbage collection or omit it altogether --- it is\na matter of implementation quality how garbage collection is\nimplemented, as long as no objects are collected that are still\nreachable.\n\n**CPython implementation detail:** CPython currently uses a reference-\ncounting scheme with (optional) delayed detection of cyclically linked\ngarbage, which collects most objects as soon as they become\nunreachable, but is not guaranteed to collect garbage containing\ncircular references.  See the documentation of the ``gc`` module for\ninformation on controlling the collection of cyclic garbage. Other\nimplementations act differently and CPython may change. Do not depend\non immediate finalization of objects when they become unreachable (ex:\nalways close files).\n\nNote that the use of the implementation\'s tracing or debugging\nfacilities may keep objects alive that would normally be collectable.\nAlso note that catching an exception with a \'``try``...``except``\'\nstatement may keep objects alive.\n\nSome objects contain references to "external" resources such as open\nfiles or windows.  It is understood that these resources are freed\nwhen the object is garbage-collected, but since garbage collection is\nnot guaranteed to happen, such objects also provide an explicit way to\nrelease the external resource, usually a ``close()`` method. Programs\nare strongly recommended to explicitly close such objects.  The\n\'``try``...``finally``\' statement provides a convenient way to do\nthis.\n\nSome objects contain references to other objects; these are called\n*containers*. Examples of containers are tuples, lists and\ndictionaries.  The references are part of a container\'s value.  In\nmost cases, when we talk about the value of a container, we imply the\nvalues, not the identities of the contained objects; however, when we\ntalk about the mutability of a container, only the identities of the\nimmediately contained objects are implied.  So, if an immutable\ncontainer (like a tuple) contains a reference to a mutable object, its\nvalue changes if that mutable object is changed.\n\nTypes affect almost all aspects of object behavior.  Even the\nimportance of object identity is affected in some sense: for immutable\ntypes, operations that compute new values may actually return a\nreference to any existing object with the same type and value, while\nfor mutable objects this is not allowed.  E.g., after ``a = 1; b =\n1``, ``a`` and ``b`` may or may not refer to the same object with the\nvalue one, depending on the implementation, but after ``c = []; d =\n[]``, ``c`` and ``d`` are guaranteed to refer to two different,\nunique, newly created empty lists. (Note that ``c = d = []`` assigns\nthe same object to both ``c`` and ``d``.)\n',
+ 'operator-summary': u'\nSummary\n*******\n\nThe following table summarizes the operator precedences in Python,\nfrom lowest precedence (least binding) to highest precedence (most\nbinding). Operators in the same box have the same precedence.  Unless\nthe syntax is explicitly given, operators are binary.  Operators in\nthe same box group left to right (except for comparisons, including\ntests, which all have the same precedence and chain from left to right\n--- see section *Comparisons* --- and exponentiation, which groups\nfrom right to left).\n\n+-------------------------------------------------+---------------------------------------+\n| Operator                                        | Description                           |\n+=================================================+=======================================+\n| ``lambda``                                      | Lambda expression                     |\n+-------------------------------------------------+---------------------------------------+\n| ``if`` -- ``else``                              | Conditional expression                |\n+-------------------------------------------------+---------------------------------------+\n| ``or``                                          | Boolean OR                            |\n+-------------------------------------------------+---------------------------------------+\n| ``and``                                         | Boolean AND                           |\n+-------------------------------------------------+---------------------------------------+\n| ``not`` *x*                                     | Boolean NOT                           |\n+-------------------------------------------------+---------------------------------------+\n| ``in``, ``not`` ``in``, ``is``, ``is not``,     | Comparisons, including membership     |\n| ``<``, ``<=``, ``>``, ``>=``, ``<>``, ``!=``,   | tests and identity tests,             |\n| ``==``                                          |                                       |\n+-------------------------------------------------+---------------------------------------+\n| ``|``                                           | Bitwise OR                            |\n+-------------------------------------------------+---------------------------------------+\n| ``^``                                           | Bitwise XOR                           |\n+-------------------------------------------------+---------------------------------------+\n| ``&``                                           | Bitwise AND                           |\n+-------------------------------------------------+---------------------------------------+\n| ``<<``, ``>>``                                  | Shifts                                |\n+-------------------------------------------------+---------------------------------------+\n| ``+``, ``-``                                    | Addition and subtraction              |\n+-------------------------------------------------+---------------------------------------+\n| ``*``, ``/``, ``//``, ``%``                     | Multiplication, division, remainder   |\n|                                                 | [8]                                   |\n+-------------------------------------------------+---------------------------------------+\n| ``+x``, ``-x``, ``~x``                          | Positive, negative, bitwise NOT       |\n+-------------------------------------------------+---------------------------------------+\n| ``**``                                          | Exponentiation [9]                    |\n+-------------------------------------------------+---------------------------------------+\n| ``x[index]``, ``x[index:index]``,               | Subscription, slicing, call,          |\n| ``x(arguments...)``, ``x.attribute``            | attribute reference                   |\n+-------------------------------------------------+---------------------------------------+\n| ``(expressions...)``, ``[expressions...]``,     | Binding or tuple display, list        |\n| ``{key:datum...}``, ```expressions...```        | display, dictionary display, string   |\n|                                                 | conversion                            |\n+-------------------------------------------------+---------------------------------------+\n\n-[ Footnotes ]-\n\n[1] In Python 2.3 and later releases, a list comprehension "leaks" the\n    control variables of each ``for`` it contains into the containing\n    scope.  However, this behavior is deprecated, and relying on it\n    will not work in Python 3.0\n\n[2] While ``abs(x%y) < abs(y)`` is true mathematically, for floats it\n    may not be true numerically due to roundoff.  For example, and\n    assuming a platform on which a Python float is an IEEE 754 double-\n    precision number, in order that ``-1e-100 % 1e100`` have the same\n    sign as ``1e100``, the computed result is ``-1e-100 + 1e100``,\n    which is numerically exactly equal to ``1e100``.  The function\n    ``math.fmod()`` returns a result whose sign matches the sign of\n    the first argument instead, and so returns ``-1e-100`` in this\n    case. Which approach is more appropriate depends on the\n    application.\n\n[3] If x is very close to an exact integer multiple of y, it\'s\n    possible for ``floor(x/y)`` to be one larger than ``(x-x%y)/y``\n    due to rounding.  In such cases, Python returns the latter result,\n    in order to preserve that ``divmod(x,y)[0] * y + x % y`` be very\n    close to ``x``.\n\n[4] While comparisons between unicode strings make sense at the byte\n    level, they may be counter-intuitive to users. For example, the\n    strings ``u"\\u00C7"`` and ``u"\\u0043\\u0327"`` compare differently,\n    even though they both represent the same unicode character (LATIN\n    CAPITAL LETTER C WITH CEDILLA). To compare strings in a human\n    recognizable way, compare using ``unicodedata.normalize()``.\n\n[5] The implementation computes this efficiently, without constructing\n    lists or sorting.\n\n[6] Earlier versions of Python used lexicographic comparison of the\n    sorted (key, value) lists, but this was very expensive for the\n    common case of comparing for equality.  An even earlier version of\n    Python compared dictionaries by identity only, but this caused\n    surprises because people expected to be able to test a dictionary\n    for emptiness by comparing it to ``{}``.\n\n[7] Due to automatic garbage-collection, free lists, and the dynamic\n    nature of descriptors, you may notice seemingly unusual behaviour\n    in certain uses of the ``is`` operator, like those involving\n    comparisons between instance methods, or constants.  Check their\n    documentation for more info.\n\n[8] The ``%`` operator is also used for string formatting; the same\n    precedence applies.\n\n[9] The power operator ``**`` binds less tightly than an arithmetic or\n    bitwise unary operator on its right, that is, ``2**-1`` is\n    ``0.5``.\n',
  'pass': u'\nThe ``pass`` statement\n**********************\n\n   pass_stmt ::= "pass"\n\n``pass`` is a null operation --- when it is executed, nothing happens.\nIt is useful as a placeholder when a statement is required\nsyntactically, but no code needs to be executed, for example:\n\n   def f(arg): pass    # a function that does nothing (yet)\n\n   class C: pass       # a class with no methods (yet)\n',
  'power': u'\nThe power operator\n******************\n\nThe power operator binds more tightly than unary operators on its\nleft; it binds less tightly than unary operators on its right.  The\nsyntax is:\n\n   power ::= primary ["**" u_expr]\n\nThus, in an unparenthesized sequence of power and unary operators, the\noperators are evaluated from right to left (this does not constrain\nthe evaluation order for the operands): ``-1**2`` results in ``-1``.\n\nThe power operator has the same semantics as the built-in ``pow()``\nfunction, when called with two arguments: it yields its left argument\nraised to the power of its right argument.  The numeric arguments are\nfirst converted to a common type.  The result type is that of the\narguments after coercion.\n\nWith mixed operand types, the coercion rules for binary arithmetic\noperators apply. For int and long int operands, the result has the\nsame type as the operands (after coercion) unless the second argument\nis negative; in that case, all arguments are converted to float and a\nfloat result is delivered. For example, ``10**2`` returns ``100``, but\n``10**-2`` returns ``0.01``. (This last feature was added in Python\n2.2. In Python 2.1 and before, if both arguments were of integer types\nand the second argument was negative, an exception was raised).\n\nRaising ``0.0`` to a negative power results in a\n``ZeroDivisionError``. Raising a negative number to a fractional power\nresults in a ``ValueError``.\n',
  'print': u'\nThe ``print`` statement\n***********************\n\n   print_stmt ::= "print" ([expression ("," expression)* [","]]\n                  | ">>" expression [("," expression)+ [","]])\n\n``print`` evaluates each expression in turn and writes the resulting\nobject to standard output (see below).  If an object is not a string,\nit is first converted to a string using the rules for string\nconversions.  The (resulting or original) string is then written.  A\nspace is written before each object is (converted and) written, unless\nthe output system believes it is positioned at the beginning of a\nline.  This is the case (1) when no characters have yet been written\nto standard output, (2) when the last character written to standard\noutput is a whitespace character except ``\' \'``, or (3) when the last\nwrite operation on standard output was not a ``print`` statement. (In\nsome cases it may be functional to write an empty string to standard\noutput for this reason.)\n\nNote: Objects which act like file objects but which are not the built-in\n  file objects often do not properly emulate this aspect of the file\n  object\'s behavior, so it is best not to rely on this.\n\nA ``\'\\n\'`` character is written at the end, unless the ``print``\nstatement ends with a comma.  This is the only action if the statement\ncontains just the keyword ``print``.\n\nStandard output is defined as the file object named ``stdout`` in the\nbuilt-in module ``sys``.  If no such object exists, or if it does not\nhave a ``write()`` method, a ``RuntimeError`` exception is raised.\n\n``print`` also has an extended form, defined by the second portion of\nthe syntax described above. This form is sometimes referred to as\n"``print`` chevron." In this form, the first expression after the\n``>>`` must evaluate to a "file-like" object, specifically an object\nthat has a ``write()`` method as described above.  With this extended\nform, the subsequent expressions are printed to this file object.  If\nthe first expression evaluates to ``None``, then ``sys.stdout`` is\nused as the file for output.\n',
@@ -63,21 +63,21 @@
  'shifting': u'\nShifting operations\n*******************\n\nThe shifting operations have lower priority than the arithmetic\noperations:\n\n   shift_expr ::= a_expr | shift_expr ( "<<" | ">>" ) a_expr\n\nThese operators accept plain or long integers as arguments.  The\narguments are converted to a common type.  They shift the first\nargument to the left or right by the number of bits given by the\nsecond argument.\n\nA right shift by *n* bits is defined as division by ``pow(2, n)``.  A\nleft shift by *n* bits is defined as multiplication with ``pow(2,\nn)``.  Negative shift counts raise a ``ValueError`` exception.\n\nNote: In the current implementation, the right-hand operand is required to\n  be at most ``sys.maxsize``.  If the right-hand operand is larger\n  than ``sys.maxsize`` an ``OverflowError`` exception is raised.\n',
  'slicings': u'\nSlicings\n********\n\nA slicing selects a range of items in a sequence object (e.g., a\nstring, tuple or list).  Slicings may be used as expressions or as\ntargets in assignment or ``del`` statements.  The syntax for a\nslicing:\n\n   slicing          ::= simple_slicing | extended_slicing\n   simple_slicing   ::= primary "[" short_slice "]"\n   extended_slicing ::= primary "[" slice_list "]"\n   slice_list       ::= slice_item ("," slice_item)* [","]\n   slice_item       ::= expression | proper_slice | ellipsis\n   proper_slice     ::= short_slice | long_slice\n   short_slice      ::= [lower_bound] ":" [upper_bound]\n   long_slice       ::= short_slice ":" [stride]\n   lower_bound      ::= expression\n   upper_bound      ::= expression\n   stride           ::= expression\n   ellipsis         ::= "..."\n\nThere is ambiguity in the formal syntax here: anything that looks like\nan expression list also looks like a slice list, so any subscription\ncan be interpreted as a slicing.  Rather than further complicating the\nsyntax, this is disambiguated by defining that in this case the\ninterpretation as a subscription takes priority over the\ninterpretation as a slicing (this is the case if the slice list\ncontains no proper slice nor ellipses).  Similarly, when the slice\nlist has exactly one short slice and no trailing comma, the\ninterpretation as a simple slicing takes priority over that as an\nextended slicing.\n\nThe semantics for a simple slicing are as follows.  The primary must\nevaluate to a sequence object.  The lower and upper bound expressions,\nif present, must evaluate to plain integers; defaults are zero and the\n``sys.maxint``, respectively.  If either bound is negative, the\nsequence\'s length is added to it.  The slicing now selects all items\nwith index *k* such that ``i <= k < j`` where *i* and *j* are the\nspecified lower and upper bounds.  This may be an empty sequence.  It\nis not an error if *i* or *j* lie outside the range of valid indexes\n(such items don\'t exist so they aren\'t selected).\n\nThe semantics for an extended slicing are as follows.  The primary\nmust evaluate to a mapping object, and it is indexed with a key that\nis constructed from the slice list, as follows.  If the slice list\ncontains at least one comma, the key is a tuple containing the\nconversion of the slice items; otherwise, the conversion of the lone\nslice item is the key.  The conversion of a slice item that is an\nexpression is that expression.  The conversion of an ellipsis slice\nitem is the built-in ``Ellipsis`` object.  The conversion of a proper\nslice is a slice object (see section *The standard type hierarchy*)\nwhose ``start``, ``stop`` and ``step`` attributes are the values of\nthe expressions given as lower bound, upper bound and stride,\nrespectively, substituting ``None`` for missing expressions.\n',
  'specialattrs': u"\nSpecial Attributes\n******************\n\nThe implementation adds a few special read-only attributes to several\nobject types, where they are relevant.  Some of these are not reported\nby the ``dir()`` built-in function.\n\nobject.__dict__\n\n   A dictionary or other mapping object used to store an object's\n   (writable) attributes.\n\nobject.__methods__\n\n   Deprecated since version 2.2: Use the built-in function ``dir()``\n   to get a list of an object's attributes. This attribute is no\n   longer available.\n\nobject.__members__\n\n   Deprecated since version 2.2: Use the built-in function ``dir()``\n   to get a list of an object's attributes. This attribute is no\n   longer available.\n\ninstance.__class__\n\n   The class to which a class instance belongs.\n\nclass.__bases__\n\n   The tuple of base classes of a class object.\n\nclass.__name__\n\n   The name of the class or type.\n\nThe following attributes are only supported by *new-style class*es.\n\nclass.__mro__\n\n   This attribute is a tuple of classes that are considered when\n   looking for base classes during method resolution.\n\nclass.mro()\n\n   This method can be overridden by a metaclass to customize the\n   method resolution order for its instances.  It is called at class\n   instantiation, and its result is stored in ``__mro__``.\n\nclass.__subclasses__()\n\n   Each new-style class keeps a list of weak references to its\n   immediate subclasses.  This method returns a list of all those\n   references still alive. Example:\n\n      >>> int.__subclasses__()\n      []\n\n-[ Footnotes ]-\n\n[1] Additional information on these special methods may be found in\n    the Python Reference Manual (*Basic customization*).\n\n[2] As a consequence, the list ``[1, 2]`` is considered equal to\n    ``[1.0, 2.0]``, and similarly for tuples.\n\n[3] They must have since the parser can't tell the type of the\n    operands.\n\n[4] To format only a tuple you should therefore provide a singleton\n    tuple whose only element is the tuple to be formatted.\n\n[5] The advantage of leaving the newline on is that returning an empty\n    string is then an unambiguous EOF indication.  It is also possible\n    (in cases where it might matter, for example, if you want to make\n    an exact copy of a file while scanning its lines) to tell whether\n    the last line of a file ended in a newline or not (yes this\n    happens!).\n",
- 'specialnames': u'\nSpecial method names\n********************\n\nA class can implement certain operations that are invoked by special\nsyntax (such as arithmetic operations or subscripting and slicing) by\ndefining methods with special names. This is Python\'s approach to\n*operator overloading*, allowing classes to define their own behavior\nwith respect to language operators.  For instance, if a class defines\na method named ``__getitem__()``, and ``x`` is an instance of this\nclass, then ``x[i]`` is roughly equivalent to ``x.__getitem__(i)`` for\nold-style classes and ``type(x).__getitem__(x, i)`` for new-style\nclasses.  Except where mentioned, attempts to execute an operation\nraise an exception when no appropriate method is defined (typically\n``AttributeError`` or ``TypeError``).\n\nWhen implementing a class that emulates any built-in type, it is\nimportant that the emulation only be implemented to the degree that it\nmakes sense for the object being modelled.  For example, some\nsequences may work well with retrieval of individual elements, but\nextracting a slice may not make sense.  (One example of this is the\n``NodeList`` interface in the W3C\'s Document Object Model.)\n\n\nBasic customization\n===================\n\nobject.__new__(cls[, ...])\n\n   Called to create a new instance of class *cls*.  ``__new__()`` is a\n   static method (special-cased so you need not declare it as such)\n   that takes the class of which an instance was requested as its\n   first argument.  The remaining arguments are those passed to the\n   object constructor expression (the call to the class).  The return\n   value of ``__new__()`` should be the new object instance (usually\n   an instance of *cls*).\n\n   Typical implementations create a new instance of the class by\n   invoking the superclass\'s ``__new__()`` method using\n   ``super(currentclass, cls).__new__(cls[, ...])`` with appropriate\n   arguments and then modifying the newly-created instance as\n   necessary before returning it.\n\n   If ``__new__()`` returns an instance of *cls*, then the new\n   instance\'s ``__init__()`` method will be invoked like\n   ``__init__(self[, ...])``, where *self* is the new instance and the\n   remaining arguments are the same as were passed to ``__new__()``.\n\n   If ``__new__()`` does not return an instance of *cls*, then the new\n   instance\'s ``__init__()`` method will not be invoked.\n\n   ``__new__()`` is intended mainly to allow subclasses of immutable\n   types (like int, str, or tuple) to customize instance creation.  It\n   is also commonly overridden in custom metaclasses in order to\n   customize class creation.\n\nobject.__init__(self[, ...])\n\n   Called when the instance is created.  The arguments are those\n   passed to the class constructor expression.  If a base class has an\n   ``__init__()`` method, the derived class\'s ``__init__()`` method,\n   if any, must explicitly call it to ensure proper initialization of\n   the base class part of the instance; for example:\n   ``BaseClass.__init__(self, [args...])``.  As a special constraint\n   on constructors, no value may be returned; doing so will cause a\n   ``TypeError`` to be raised at runtime.\n\nobject.__del__(self)\n\n   Called when the instance is about to be destroyed.  This is also\n   called a destructor.  If a base class has a ``__del__()`` method,\n   the derived class\'s ``__del__()`` method, if any, must explicitly\n   call it to ensure proper deletion of the base class part of the\n   instance.  Note that it is possible (though not recommended!) for\n   the ``__del__()`` method to postpone destruction of the instance by\n   creating a new reference to it.  It may then be called at a later\n   time when this new reference is deleted.  It is not guaranteed that\n   ``__del__()`` methods are called for objects that still exist when\n   the interpreter exits.\n\n   Note: ``del x`` doesn\'t directly call ``x.__del__()`` --- the former\n     decrements the reference count for ``x`` by one, and the latter\n     is only called when ``x``\'s reference count reaches zero.  Some\n     common situations that may prevent the reference count of an\n     object from going to zero include: circular references between\n     objects (e.g., a doubly-linked list or a tree data structure with\n     parent and child pointers); a reference to the object on the\n     stack frame of a function that caught an exception (the traceback\n     stored in ``sys.exc_traceback`` keeps the stack frame alive); or\n     a reference to the object on the stack frame that raised an\n     unhandled exception in interactive mode (the traceback stored in\n     ``sys.last_traceback`` keeps the stack frame alive).  The first\n     situation can only be remedied by explicitly breaking the cycles;\n     the latter two situations can be resolved by storing ``None`` in\n     ``sys.exc_traceback`` or ``sys.last_traceback``.  Circular\n     references which are garbage are detected when the option cycle\n     detector is enabled (it\'s on by default), but can only be cleaned\n     up if there are no Python-level ``__del__()`` methods involved.\n     Refer to the documentation for the ``gc`` module for more\n     information about how ``__del__()`` methods are handled by the\n     cycle detector, particularly the description of the ``garbage``\n     value.\n\n   Warning: Due to the precarious circumstances under which ``__del__()``\n     methods are invoked, exceptions that occur during their execution\n     are ignored, and a warning is printed to ``sys.stderr`` instead.\n     Also, when ``__del__()`` is invoked in response to a module being\n     deleted (e.g., when execution of the program is done), other\n     globals referenced by the ``__del__()`` method may already have\n     been deleted or in the process of being torn down (e.g. the\n     import machinery shutting down).  For this reason, ``__del__()``\n     methods should do the absolute minimum needed to maintain\n     external invariants.  Starting with version 1.5, Python\n     guarantees that globals whose name begins with a single\n     underscore are deleted from their module before other globals are\n     deleted; if no other references to such globals exist, this may\n     help in assuring that imported modules are still available at the\n     time when the ``__del__()`` method is called.\n\nobject.__repr__(self)\n\n   Called by the ``repr()`` built-in function and by string\n   conversions (reverse quotes) to compute the "official" string\n   representation of an object.  If at all possible, this should look\n   like a valid Python expression that could be used to recreate an\n   object with the same value (given an appropriate environment).  If\n   this is not possible, a string of the form ``<...some useful\n   description...>`` should be returned.  The return value must be a\n   string object. If a class defines ``__repr__()`` but not\n   ``__str__()``, then ``__repr__()`` is also used when an "informal"\n   string representation of instances of that class is required.\n\n   This is typically used for debugging, so it is important that the\n   representation is information-rich and unambiguous.\n\nobject.__str__(self)\n\n   Called by the ``str()`` built-in function and by the ``print``\n   statement to compute the "informal" string representation of an\n   object.  This differs from ``__repr__()`` in that it does not have\n   to be a valid Python expression: a more convenient or concise\n   representation may be used instead. The return value must be a\n   string object.\n\nobject.__lt__(self, other)\nobject.__le__(self, other)\nobject.__eq__(self, other)\nobject.__ne__(self, other)\nobject.__gt__(self, other)\nobject.__ge__(self, other)\n\n   New in version 2.1.\n\n   These are the so-called "rich comparison" methods, and are called\n   for comparison operators in preference to ``__cmp__()`` below. The\n   correspondence between operator symbols and method names is as\n   follows: ``xy`` call ``x.__ne__(y)``, ``x>y`` calls ``x.__gt__(y)``, and\n   ``x>=y`` calls ``x.__ge__(y)``.\n\n   A rich comparison method may return the singleton\n   ``NotImplemented`` if it does not implement the operation for a\n   given pair of arguments. By convention, ``False`` and ``True`` are\n   returned for a successful comparison. However, these methods can\n   return any value, so if the comparison operator is used in a\n   Boolean context (e.g., in the condition of an ``if`` statement),\n   Python will call ``bool()`` on the value to determine if the result\n   is true or false.\n\n   There are no implied relationships among the comparison operators.\n   The truth of ``x==y`` does not imply that ``x!=y`` is false.\n   Accordingly, when defining ``__eq__()``, one should also define\n   ``__ne__()`` so that the operators will behave as expected.  See\n   the paragraph on ``__hash__()`` for some important notes on\n   creating *hashable* objects which support custom comparison\n   operations and are usable as dictionary keys.\n\n   There are no swapped-argument versions of these methods (to be used\n   when the left argument does not support the operation but the right\n   argument does); rather, ``__lt__()`` and ``__gt__()`` are each\n   other\'s reflection, ``__le__()`` and ``__ge__()`` are each other\'s\n   reflection, and ``__eq__()`` and ``__ne__()`` are their own\n   reflection.\n\n   Arguments to rich comparison methods are never coerced.\n\n   To automatically generate ordering operations from a single root\n   operation, see ``functools.total_ordering()``.\n\nobject.__cmp__(self, other)\n\n   Called by comparison operations if rich comparison (see above) is\n   not defined.  Should return a negative integer if ``self < other``,\n   zero if ``self == other``, a positive integer if ``self > other``.\n   If no ``__cmp__()``, ``__eq__()`` or ``__ne__()`` operation is\n   defined, class instances are compared by object identity\n   ("address").  See also the description of ``__hash__()`` for some\n   important notes on creating *hashable* objects which support custom\n   comparison operations and are usable as dictionary keys. (Note: the\n   restriction that exceptions are not propagated by ``__cmp__()`` has\n   been removed since Python 1.5.)\n\nobject.__rcmp__(self, other)\n\n   Changed in version 2.1: No longer supported.\n\nobject.__hash__(self)\n\n   Called by built-in function ``hash()`` and for operations on\n   members of hashed collections including ``set``, ``frozenset``, and\n   ``dict``.  ``__hash__()`` should return an integer.  The only\n   required property is that objects which compare equal have the same\n   hash value; it is advised to somehow mix together (e.g. using\n   exclusive or) the hash values for the components of the object that\n   also play a part in comparison of objects.\n\n   If a class does not define a ``__cmp__()`` or ``__eq__()`` method\n   it should not define a ``__hash__()`` operation either; if it\n   defines ``__cmp__()`` or ``__eq__()`` but not ``__hash__()``, its\n   instances will not be usable in hashed collections.  If a class\n   defines mutable objects and implements a ``__cmp__()`` or\n   ``__eq__()`` method, it should not implement ``__hash__()``, since\n   hashable collection implementations require that a object\'s hash\n   value is immutable (if the object\'s hash value changes, it will be\n   in the wrong hash bucket).\n\n   User-defined classes have ``__cmp__()`` and ``__hash__()`` methods\n   by default; with them, all objects compare unequal (except with\n   themselves) and ``x.__hash__()`` returns ``id(x)``.\n\n   Classes which inherit a ``__hash__()`` method from a parent class\n   but change the meaning of ``__cmp__()`` or ``__eq__()`` such that\n   the hash value returned is no longer appropriate (e.g. by switching\n   to a value-based concept of equality instead of the default\n   identity based equality) can explicitly flag themselves as being\n   unhashable by setting ``__hash__ = None`` in the class definition.\n   Doing so means that not only will instances of the class raise an\n   appropriate ``TypeError`` when a program attempts to retrieve their\n   hash value, but they will also be correctly identified as\n   unhashable when checking ``isinstance(obj, collections.Hashable)``\n   (unlike classes which define their own ``__hash__()`` to explicitly\n   raise ``TypeError``).\n\n   Changed in version 2.5: ``__hash__()`` may now also return a long\n   integer object; the 32-bit integer is then derived from the hash of\n   that object.\n\n   Changed in version 2.6: ``__hash__`` may now be set to ``None`` to\n   explicitly flag instances of a class as unhashable.\n\nobject.__nonzero__(self)\n\n   Called to implement truth value testing and the built-in operation\n   ``bool()``; should return ``False`` or ``True``, or their integer\n   equivalents ``0`` or ``1``.  When this method is not defined,\n   ``__len__()`` is called, if it is defined, and the object is\n   considered true if its result is nonzero. If a class defines\n   neither ``__len__()`` nor ``__nonzero__()``, all its instances are\n   considered true.\n\nobject.__unicode__(self)\n\n   Called to implement ``unicode()`` built-in; should return a Unicode\n   object. When this method is not defined, string conversion is\n   attempted, and the result of string conversion is converted to\n   Unicode using the system default encoding.\n\n\nCustomizing attribute access\n============================\n\nThe following methods can be defined to customize the meaning of\nattribute access (use of, assignment to, or deletion of ``x.name``)\nfor class instances.\n\nobject.__getattr__(self, name)\n\n   Called when an attribute lookup has not found the attribute in the\n   usual places (i.e. it is not an instance attribute nor is it found\n   in the class tree for ``self``).  ``name`` is the attribute name.\n   This method should return the (computed) attribute value or raise\n   an ``AttributeError`` exception.\n\n   Note that if the attribute is found through the normal mechanism,\n   ``__getattr__()`` is not called.  (This is an intentional asymmetry\n   between ``__getattr__()`` and ``__setattr__()``.) This is done both\n   for efficiency reasons and because otherwise ``__getattr__()``\n   would have no way to access other attributes of the instance.  Note\n   that at least for instance variables, you can fake total control by\n   not inserting any values in the instance attribute dictionary (but\n   instead inserting them in another object).  See the\n   ``__getattribute__()`` method below for a way to actually get total\n   control in new-style classes.\n\nobject.__setattr__(self, name, value)\n\n   Called when an attribute assignment is attempted.  This is called\n   instead of the normal mechanism (i.e. store the value in the\n   instance dictionary).  *name* is the attribute name, *value* is the\n   value to be assigned to it.\n\n   If ``__setattr__()`` wants to assign to an instance attribute, it\n   should not simply execute ``self.name = value`` --- this would\n   cause a recursive call to itself.  Instead, it should insert the\n   value in the dictionary of instance attributes, e.g.,\n   ``self.__dict__[name] = value``.  For new-style classes, rather\n   than accessing the instance dictionary, it should call the base\n   class method with the same name, for example,\n   ``object.__setattr__(self, name, value)``.\n\nobject.__delattr__(self, name)\n\n   Like ``__setattr__()`` but for attribute deletion instead of\n   assignment.  This should only be implemented if ``del obj.name`` is\n   meaningful for the object.\n\n\nMore attribute access for new-style classes\n-------------------------------------------\n\nThe following methods only apply to new-style classes.\n\nobject.__getattribute__(self, name)\n\n   Called unconditionally to implement attribute accesses for\n   instances of the class. If the class also defines\n   ``__getattr__()``, the latter will not be called unless\n   ``__getattribute__()`` either calls it explicitly or raises an\n   ``AttributeError``. This method should return the (computed)\n   attribute value or raise an ``AttributeError`` exception. In order\n   to avoid infinite recursion in this method, its implementation\n   should always call the base class method with the same name to\n   access any attributes it needs, for example,\n   ``object.__getattribute__(self, name)``.\n\n   Note: This method may still be bypassed when looking up special methods\n     as the result of implicit invocation via language syntax or\n     built-in functions. See *Special method lookup for new-style\n     classes*.\n\n\nImplementing Descriptors\n------------------------\n\nThe following methods only apply when an instance of the class\ncontaining the method (a so-called *descriptor* class) appears in the\nclass dictionary of another new-style class, known as the *owner*\nclass. In the examples below, "the attribute" refers to the attribute\nwhose name is the key of the property in the owner class\'\n``__dict__``.  Descriptors can only be implemented as new-style\nclasses themselves.\n\nobject.__get__(self, instance, owner)\n\n   Called to get the attribute of the owner class (class attribute\n   access) or of an instance of that class (instance attribute\n   access). *owner* is always the owner class, while *instance* is the\n   instance that the attribute was accessed through, or ``None`` when\n   the attribute is accessed through the *owner*.  This method should\n   return the (computed) attribute value or raise an\n   ``AttributeError`` exception.\n\nobject.__set__(self, instance, value)\n\n   Called to set the attribute on an instance *instance* of the owner\n   class to a new value, *value*.\n\nobject.__delete__(self, instance)\n\n   Called to delete the attribute on an instance *instance* of the\n   owner class.\n\n\nInvoking Descriptors\n--------------------\n\nIn general, a descriptor is an object attribute with "binding\nbehavior", one whose attribute access has been overridden by methods\nin the descriptor protocol:  ``__get__()``, ``__set__()``, and\n``__delete__()``. If any of those methods are defined for an object,\nit is said to be a descriptor.\n\nThe default behavior for attribute access is to get, set, or delete\nthe attribute from an object\'s dictionary. For instance, ``a.x`` has a\nlookup chain starting with ``a.__dict__[\'x\']``, then\n``type(a).__dict__[\'x\']``, and continuing through the base classes of\n``type(a)`` excluding metaclasses.\n\nHowever, if the looked-up value is an object defining one of the\ndescriptor methods, then Python may override the default behavior and\ninvoke the descriptor method instead.  Where this occurs in the\nprecedence chain depends on which descriptor methods were defined and\nhow they were called.  Note that descriptors are only invoked for new\nstyle objects or classes (ones that subclass ``object()`` or\n``type()``).\n\nThe starting point for descriptor invocation is a binding, ``a.x``.\nHow the arguments are assembled depends on ``a``:\n\nDirect Call\n   The simplest and least common call is when user code directly\n   invokes a descriptor method:    ``x.__get__(a)``.\n\nInstance Binding\n   If binding to a new-style object instance, ``a.x`` is transformed\n   into the call: ``type(a).__dict__[\'x\'].__get__(a, type(a))``.\n\nClass Binding\n   If binding to a new-style class, ``A.x`` is transformed into the\n   call: ``A.__dict__[\'x\'].__get__(None, A)``.\n\nSuper Binding\n   If ``a`` is an instance of ``super``, then the binding ``super(B,\n   obj).m()`` searches ``obj.__class__.__mro__`` for the base class\n   ``A`` immediately preceding ``B`` and then invokes the descriptor\n   with the call: ``A.__dict__[\'m\'].__get__(obj, A)``.\n\nFor instance bindings, the precedence of descriptor invocation depends\non the which descriptor methods are defined.  A descriptor can define\nany combination of ``__get__()``, ``__set__()`` and ``__delete__()``.\nIf it does not define ``__get__()``, then accessing the attribute will\nreturn the descriptor object itself unless there is a value in the\nobject\'s instance dictionary.  If the descriptor defines ``__set__()``\nand/or ``__delete__()``, it is a data descriptor; if it defines\nneither, it is a non-data descriptor.  Normally, data descriptors\ndefine both ``__get__()`` and ``__set__()``, while non-data\ndescriptors have just the ``__get__()`` method.  Data descriptors with\n``__set__()`` and ``__get__()`` defined always override a redefinition\nin an instance dictionary.  In contrast, non-data descriptors can be\noverridden by instances.\n\nPython methods (including ``staticmethod()`` and ``classmethod()``)\nare implemented as non-data descriptors.  Accordingly, instances can\nredefine and override methods.  This allows individual instances to\nacquire behaviors that differ from other instances of the same class.\n\nThe ``property()`` function is implemented as a data descriptor.\nAccordingly, instances cannot override the behavior of a property.\n\n\n__slots__\n---------\n\nBy default, instances of both old and new-style classes have a\ndictionary for attribute storage.  This wastes space for objects\nhaving very few instance variables.  The space consumption can become\nacute when creating large numbers of instances.\n\nThe default can be overridden by defining *__slots__* in a new-style\nclass definition.  The *__slots__* declaration takes a sequence of\ninstance variables and reserves just enough space in each instance to\nhold a value for each variable.  Space is saved because *__dict__* is\nnot created for each instance.\n\n__slots__\n\n   This class variable can be assigned a string, iterable, or sequence\n   of strings with variable names used by instances.  If defined in a\n   new-style class, *__slots__* reserves space for the declared\n   variables and prevents the automatic creation of *__dict__* and\n   *__weakref__* for each instance.\n\n   New in version 2.2.\n\nNotes on using *__slots__*\n\n* When inheriting from a class without *__slots__*, the *__dict__*\n  attribute of that class will always be accessible, so a *__slots__*\n  definition in the subclass is meaningless.\n\n* Without a *__dict__* variable, instances cannot be assigned new\n  variables not listed in the *__slots__* definition.  Attempts to\n  assign to an unlisted variable name raises ``AttributeError``. If\n  dynamic assignment of new variables is desired, then add\n  ``\'__dict__\'`` to the sequence of strings in the *__slots__*\n  declaration.\n\n  Changed in version 2.3: Previously, adding ``\'__dict__\'`` to the\n  *__slots__* declaration would not enable the assignment of new\n  attributes not specifically listed in the sequence of instance\n  variable names.\n\n* Without a *__weakref__* variable for each instance, classes defining\n  *__slots__* do not support weak references to its instances. If weak\n  reference support is needed, then add ``\'__weakref__\'`` to the\n  sequence of strings in the *__slots__* declaration.\n\n  Changed in version 2.3: Previously, adding ``\'__weakref__\'`` to the\n  *__slots__* declaration would not enable support for weak\n  references.\n\n* *__slots__* are implemented at the class level by creating\n  descriptors (*Implementing Descriptors*) for each variable name.  As\n  a result, class attributes cannot be used to set default values for\n  instance variables defined by *__slots__*; otherwise, the class\n  attribute would overwrite the descriptor assignment.\n\n* The action of a *__slots__* declaration is limited to the class\n  where it is defined.  As a result, subclasses will have a *__dict__*\n  unless they also define *__slots__* (which must only contain names\n  of any *additional* slots).\n\n* If a class defines a slot also defined in a base class, the instance\n  variable defined by the base class slot is inaccessible (except by\n  retrieving its descriptor directly from the base class). This\n  renders the meaning of the program undefined.  In the future, a\n  check may be added to prevent this.\n\n* Nonempty *__slots__* does not work for classes derived from\n  "variable-length" built-in types such as ``long``, ``str`` and\n  ``tuple``.\n\n* Any non-string iterable may be assigned to *__slots__*. Mappings may\n  also be used; however, in the future, special meaning may be\n  assigned to the values corresponding to each key.\n\n* *__class__* assignment works only if both classes have the same\n  *__slots__*.\n\n  Changed in version 2.6: Previously, *__class__* assignment raised an\n  error if either new or old class had *__slots__*.\n\n\nCustomizing class creation\n==========================\n\nBy default, new-style classes are constructed using ``type()``. A\nclass definition is read into a separate namespace and the value of\nclass name is bound to the result of ``type(name, bases, dict)``.\n\nWhen the class definition is read, if *__metaclass__* is defined then\nthe callable assigned to it will be called instead of ``type()``. This\nallows classes or functions to be written which monitor or alter the\nclass creation process:\n\n* Modifying the class dictionary prior to the class being created.\n\n* Returning an instance of another class -- essentially performing the\n  role of a factory function.\n\nThese steps will have to be performed in the metaclass\'s ``__new__()``\nmethod -- ``type.__new__()`` can then be called from this method to\ncreate a class with different properties.  This example adds a new\nelement to the class dictionary before creating the class:\n\n   class metacls(type):\n       def __new__(mcs, name, bases, dict):\n           dict[\'foo\'] = \'metacls was here\'\n           return type.__new__(mcs, name, bases, dict)\n\nYou can of course also override other class methods (or add new\nmethods); for example defining a custom ``__call__()`` method in the\nmetaclass allows custom behavior when the class is called, e.g. not\nalways creating a new instance.\n\n__metaclass__\n\n   This variable can be any callable accepting arguments for ``name``,\n   ``bases``, and ``dict``.  Upon class creation, the callable is used\n   instead of the built-in ``type()``.\n\n   New in version 2.2.\n\nThe appropriate metaclass is determined by the following precedence\nrules:\n\n* If ``dict[\'__metaclass__\']`` exists, it is used.\n\n* Otherwise, if there is at least one base class, its metaclass is\n  used (this looks for a *__class__* attribute first and if not found,\n  uses its type).\n\n* Otherwise, if a global variable named __metaclass__ exists, it is\n  used.\n\n* Otherwise, the old-style, classic metaclass (types.ClassType) is\n  used.\n\nThe potential uses for metaclasses are boundless. Some ideas that have\nbeen explored including logging, interface checking, automatic\ndelegation, automatic property creation, proxies, frameworks, and\nautomatic resource locking/synchronization.\n\n\nCustomizing instance and subclass checks\n========================================\n\nNew in version 2.6.\n\nThe following methods are used to override the default behavior of the\n``isinstance()`` and ``issubclass()`` built-in functions.\n\nIn particular, the metaclass ``abc.ABCMeta`` implements these methods\nin order to allow the addition of Abstract Base Classes (ABCs) as\n"virtual base classes" to any class or type (including built-in\ntypes), including other ABCs.\n\nclass.__instancecheck__(self, instance)\n\n   Return true if *instance* should be considered a (direct or\n   indirect) instance of *class*. If defined, called to implement\n   ``isinstance(instance, class)``.\n\nclass.__subclasscheck__(self, subclass)\n\n   Return true if *subclass* should be considered a (direct or\n   indirect) subclass of *class*.  If defined, called to implement\n   ``issubclass(subclass, class)``.\n\nNote that these methods are looked up on the type (metaclass) of a\nclass.  They cannot be defined as class methods in the actual class.\nThis is consistent with the lookup of special methods that are called\non instances, only in this case the instance is itself a class.\n\nSee also:\n\n   **PEP 3119** - Introducing Abstract Base Classes\n      Includes the specification for customizing ``isinstance()`` and\n      ``issubclass()`` behavior through ``__instancecheck__()`` and\n      ``__subclasscheck__()``, with motivation for this functionality\n      in the context of adding Abstract Base Classes (see the ``abc``\n      module) to the language.\n\n\nEmulating callable objects\n==========================\n\nobject.__call__(self[, args...])\n\n   Called when the instance is "called" as a function; if this method\n   is defined, ``x(arg1, arg2, ...)`` is a shorthand for\n   ``x.__call__(arg1, arg2, ...)``.\n\n\nEmulating container types\n=========================\n\nThe following methods can be defined to implement container objects.\nContainers usually are sequences (such as lists or tuples) or mappings\n(like dictionaries), but can represent other containers as well.  The\nfirst set of methods is used either to emulate a sequence or to\nemulate a mapping; the difference is that for a sequence, the\nallowable keys should be the integers *k* for which ``0 <= k < N``\nwhere *N* is the length of the sequence, or slice objects, which\ndefine a range of items. (For backwards compatibility, the method\n``__getslice__()`` (see below) can also be defined to handle simple,\nbut not extended slices.) It is also recommended that mappings provide\nthe methods ``keys()``, ``values()``, ``items()``, ``has_key()``,\n``get()``, ``clear()``, ``setdefault()``, ``iterkeys()``,\n``itervalues()``, ``iteritems()``, ``pop()``, ``popitem()``,\n``copy()``, and ``update()`` behaving similar to those for Python\'s\nstandard dictionary objects.  The ``UserDict`` module provides a\n``DictMixin`` class to help create those methods from a base set of\n``__getitem__()``, ``__setitem__()``, ``__delitem__()``, and\n``keys()``. Mutable sequences should provide methods ``append()``,\n``count()``, ``index()``, ``extend()``, ``insert()``, ``pop()``,\n``remove()``, ``reverse()`` and ``sort()``, like Python standard list\nobjects.  Finally, sequence types should implement addition (meaning\nconcatenation) and multiplication (meaning repetition) by defining the\nmethods ``__add__()``, ``__radd__()``, ``__iadd__()``, ``__mul__()``,\n``__rmul__()`` and ``__imul__()`` described below; they should not\ndefine ``__coerce__()`` or other numerical operators.  It is\nrecommended that both mappings and sequences implement the\n``__contains__()`` method to allow efficient use of the ``in``\noperator; for mappings, ``in`` should be equivalent of ``has_key()``;\nfor sequences, it should search through the values.  It is further\nrecommended that both mappings and sequences implement the\n``__iter__()`` method to allow efficient iteration through the\ncontainer; for mappings, ``__iter__()`` should be the same as\n``iterkeys()``; for sequences, it should iterate through the values.\n\nobject.__len__(self)\n\n   Called to implement the built-in function ``len()``.  Should return\n   the length of the object, an integer ``>=`` 0.  Also, an object\n   that doesn\'t define a ``__nonzero__()`` method and whose\n   ``__len__()`` method returns zero is considered to be false in a\n   Boolean context.\n\nobject.__getitem__(self, key)\n\n   Called to implement evaluation of ``self[key]``. For sequence\n   types, the accepted keys should be integers and slice objects.\n   Note that the special interpretation of negative indexes (if the\n   class wishes to emulate a sequence type) is up to the\n   ``__getitem__()`` method. If *key* is of an inappropriate type,\n   ``TypeError`` may be raised; if of a value outside the set of\n   indexes for the sequence (after any special interpretation of\n   negative values), ``IndexError`` should be raised. For mapping\n   types, if *key* is missing (not in the container), ``KeyError``\n   should be raised.\n\n   Note: ``for`` loops expect that an ``IndexError`` will be raised for\n     illegal indexes to allow proper detection of the end of the\n     sequence.\n\nobject.__setitem__(self, key, value)\n\n   Called to implement assignment to ``self[key]``.  Same note as for\n   ``__getitem__()``.  This should only be implemented for mappings if\n   the objects support changes to the values for keys, or if new keys\n   can be added, or for sequences if elements can be replaced.  The\n   same exceptions should be raised for improper *key* values as for\n   the ``__getitem__()`` method.\n\nobject.__delitem__(self, key)\n\n   Called to implement deletion of ``self[key]``.  Same note as for\n   ``__getitem__()``.  This should only be implemented for mappings if\n   the objects support removal of keys, or for sequences if elements\n   can be removed from the sequence.  The same exceptions should be\n   raised for improper *key* values as for the ``__getitem__()``\n   method.\n\nobject.__iter__(self)\n\n   This method is called when an iterator is required for a container.\n   This method should return a new iterator object that can iterate\n   over all the objects in the container.  For mappings, it should\n   iterate over the keys of the container, and should also be made\n   available as the method ``iterkeys()``.\n\n   Iterator objects also need to implement this method; they are\n   required to return themselves.  For more information on iterator\n   objects, see *Iterator Types*.\n\nobject.__reversed__(self)\n\n   Called (if present) by the ``reversed()`` built-in to implement\n   reverse iteration.  It should return a new iterator object that\n   iterates over all the objects in the container in reverse order.\n\n   If the ``__reversed__()`` method is not provided, the\n   ``reversed()`` built-in will fall back to using the sequence\n   protocol (``__len__()`` and ``__getitem__()``).  Objects that\n   support the sequence protocol should only provide\n   ``__reversed__()`` if they can provide an implementation that is\n   more efficient than the one provided by ``reversed()``.\n\n   New in version 2.6.\n\nThe membership test operators (``in`` and ``not in``) are normally\nimplemented as an iteration through a sequence.  However, container\nobjects can supply the following special method with a more efficient\nimplementation, which also does not require the object be a sequence.\n\nobject.__contains__(self, item)\n\n   Called to implement membership test operators.  Should return true\n   if *item* is in *self*, false otherwise.  For mapping objects, this\n   should consider the keys of the mapping rather than the values or\n   the key-item pairs.\n\n   For objects that don\'t define ``__contains__()``, the membership\n   test first tries iteration via ``__iter__()``, then the old\n   sequence iteration protocol via ``__getitem__()``, see *this\n   section in the language reference*.\n\n\nAdditional methods for emulation of sequence types\n==================================================\n\nThe following optional methods can be defined to further emulate\nsequence objects.  Immutable sequences methods should at most only\ndefine ``__getslice__()``; mutable sequences might define all three\nmethods.\n\nobject.__getslice__(self, i, j)\n\n   Deprecated since version 2.0: Support slice objects as parameters\n   to the ``__getitem__()`` method. (However, built-in types in\n   CPython currently still implement ``__getslice__()``.  Therefore,\n   you have to override it in derived classes when implementing\n   slicing.)\n\n   Called to implement evaluation of ``self[i:j]``. The returned\n   object should be of the same type as *self*.  Note that missing *i*\n   or *j* in the slice expression are replaced by zero or\n   ``sys.maxint``, respectively.  If negative indexes are used in the\n   slice, the length of the sequence is added to that index. If the\n   instance does not implement the ``__len__()`` method, an\n   ``AttributeError`` is raised. No guarantee is made that indexes\n   adjusted this way are not still negative.  Indexes which are\n   greater than the length of the sequence are not modified. If no\n   ``__getslice__()`` is found, a slice object is created instead, and\n   passed to ``__getitem__()`` instead.\n\nobject.__setslice__(self, i, j, sequence)\n\n   Called to implement assignment to ``self[i:j]``. Same notes for *i*\n   and *j* as for ``__getslice__()``.\n\n   This method is deprecated. If no ``__setslice__()`` is found, or\n   for extended slicing of the form ``self[i:j:k]``, a slice object is\n   created, and passed to ``__setitem__()``, instead of\n   ``__setslice__()`` being called.\n\nobject.__delslice__(self, i, j)\n\n   Called to implement deletion of ``self[i:j]``. Same notes for *i*\n   and *j* as for ``__getslice__()``. This method is deprecated. If no\n   ``__delslice__()`` is found, or for extended slicing of the form\n   ``self[i:j:k]``, a slice object is created, and passed to\n   ``__delitem__()``, instead of ``__delslice__()`` being called.\n\nNotice that these methods are only invoked when a single slice with a\nsingle colon is used, and the slice method is available.  For slice\noperations involving extended slice notation, or in absence of the\nslice methods, ``__getitem__()``, ``__setitem__()`` or\n``__delitem__()`` is called with a slice object as argument.\n\nThe following example demonstrate how to make your program or module\ncompatible with earlier versions of Python (assuming that methods\n``__getitem__()``, ``__setitem__()`` and ``__delitem__()`` support\nslice objects as arguments):\n\n   class MyClass:\n       ...\n       def __getitem__(self, index):\n           ...\n       def __setitem__(self, index, value):\n           ...\n       def __delitem__(self, index):\n           ...\n\n       if sys.version_info < (2, 0):\n           # They won\'t be defined if version is at least 2.0 final\n\n           def __getslice__(self, i, j):\n               return self[max(0, i):max(0, j):]\n           def __setslice__(self, i, j, seq):\n               self[max(0, i):max(0, j):] = seq\n           def __delslice__(self, i, j):\n               del self[max(0, i):max(0, j):]\n       ...\n\nNote the calls to ``max()``; these are necessary because of the\nhandling of negative indices before the ``__*slice__()`` methods are\ncalled.  When negative indexes are used, the ``__*item__()`` methods\nreceive them as provided, but the ``__*slice__()`` methods get a\n"cooked" form of the index values.  For each negative index value, the\nlength of the sequence is added to the index before calling the method\n(which may still result in a negative index); this is the customary\nhandling of negative indexes by the built-in sequence types, and the\n``__*item__()`` methods are expected to do this as well.  However,\nsince they should already be doing that, negative indexes cannot be\npassed in; they must be constrained to the bounds of the sequence\nbefore being passed to the ``__*item__()`` methods. Calling ``max(0,\ni)`` conveniently returns the proper value.\n\n\nEmulating numeric types\n=======================\n\nThe following methods can be defined to emulate numeric objects.\nMethods corresponding to operations that are not supported by the\nparticular kind of number implemented (e.g., bitwise operations for\nnon-integral numbers) should be left undefined.\n\nobject.__add__(self, other)\nobject.__sub__(self, other)\nobject.__mul__(self, other)\nobject.__floordiv__(self, other)\nobject.__mod__(self, other)\nobject.__divmod__(self, other)\nobject.__pow__(self, other[, modulo])\nobject.__lshift__(self, other)\nobject.__rshift__(self, other)\nobject.__and__(self, other)\nobject.__xor__(self, other)\nobject.__or__(self, other)\n\n   These methods are called to implement the binary arithmetic\n   operations (``+``, ``-``, ``*``, ``//``, ``%``, ``divmod()``,\n   ``pow()``, ``**``, ``<<``, ``>>``, ``&``, ``^``, ``|``).  For\n   instance, to evaluate the expression ``x + y``, where *x* is an\n   instance of a class that has an ``__add__()`` method,\n   ``x.__add__(y)`` is called.  The ``__divmod__()`` method should be\n   the equivalent to using ``__floordiv__()`` and ``__mod__()``; it\n   should not be related to ``__truediv__()`` (described below).  Note\n   that ``__pow__()`` should be defined to accept an optional third\n   argument if the ternary version of the built-in ``pow()`` function\n   is to be supported.\n\n   If one of those methods does not support the operation with the\n   supplied arguments, it should return ``NotImplemented``.\n\nobject.__div__(self, other)\nobject.__truediv__(self, other)\n\n   The division operator (``/``) is implemented by these methods.  The\n   ``__truediv__()`` method is used when ``__future__.division`` is in\n   effect, otherwise ``__div__()`` is used.  If only one of these two\n   methods is defined, the object will not support division in the\n   alternate context; ``TypeError`` will be raised instead.\n\nobject.__radd__(self, other)\nobject.__rsub__(self, other)\nobject.__rmul__(self, other)\nobject.__rdiv__(self, other)\nobject.__rtruediv__(self, other)\nobject.__rfloordiv__(self, other)\nobject.__rmod__(self, other)\nobject.__rdivmod__(self, other)\nobject.__rpow__(self, other)\nobject.__rlshift__(self, other)\nobject.__rrshift__(self, other)\nobject.__rand__(self, other)\nobject.__rxor__(self, other)\nobject.__ror__(self, other)\n\n   These methods are called to implement the binary arithmetic\n   operations (``+``, ``-``, ``*``, ``/``, ``%``, ``divmod()``,\n   ``pow()``, ``**``, ``<<``, ``>>``, ``&``, ``^``, ``|``) with\n   reflected (swapped) operands.  These functions are only called if\n   the left operand does not support the corresponding operation and\n   the operands are of different types. [2] For instance, to evaluate\n   the expression ``x - y``, where *y* is an instance of a class that\n   has an ``__rsub__()`` method, ``y.__rsub__(x)`` is called if\n   ``x.__sub__(y)`` returns *NotImplemented*.\n\n   Note that ternary ``pow()`` will not try calling ``__rpow__()``\n   (the coercion rules would become too complicated).\n\n   Note: If the right operand\'s type is a subclass of the left operand\'s\n     type and that subclass provides the reflected method for the\n     operation, this method will be called before the left operand\'s\n     non-reflected method.  This behavior allows subclasses to\n     override their ancestors\' operations.\n\nobject.__iadd__(self, other)\nobject.__isub__(self, other)\nobject.__imul__(self, other)\nobject.__idiv__(self, other)\nobject.__itruediv__(self, other)\nobject.__ifloordiv__(self, other)\nobject.__imod__(self, other)\nobject.__ipow__(self, other[, modulo])\nobject.__ilshift__(self, other)\nobject.__irshift__(self, other)\nobject.__iand__(self, other)\nobject.__ixor__(self, other)\nobject.__ior__(self, other)\n\n   These methods are called to implement the augmented arithmetic\n   assignments (``+=``, ``-=``, ``*=``, ``/=``, ``//=``, ``%=``,\n   ``**=``, ``<<=``, ``>>=``, ``&=``, ``^=``, ``|=``).  These methods\n   should attempt to do the operation in-place (modifying *self*) and\n   return the result (which could be, but does not have to be,\n   *self*).  If a specific method is not defined, the augmented\n   assignment falls back to the normal methods.  For instance, to\n   execute the statement ``x += y``, where *x* is an instance of a\n   class that has an ``__iadd__()`` method, ``x.__iadd__(y)`` is\n   called.  If *x* is an instance of a class that does not define a\n   ``__iadd__()`` method, ``x.__add__(y)`` and ``y.__radd__(x)`` are\n   considered, as with the evaluation of ``x + y``.\n\nobject.__neg__(self)\nobject.__pos__(self)\nobject.__abs__(self)\nobject.__invert__(self)\n\n   Called to implement the unary arithmetic operations (``-``, ``+``,\n   ``abs()`` and ``~``).\n\nobject.__complex__(self)\nobject.__int__(self)\nobject.__long__(self)\nobject.__float__(self)\n\n   Called to implement the built-in functions ``complex()``,\n   ``int()``, ``long()``, and ``float()``.  Should return a value of\n   the appropriate type.\n\nobject.__oct__(self)\nobject.__hex__(self)\n\n   Called to implement the built-in functions ``oct()`` and ``hex()``.\n   Should return a string value.\n\nobject.__index__(self)\n\n   Called to implement ``operator.index()``.  Also called whenever\n   Python needs an integer object (such as in slicing).  Must return\n   an integer (int or long).\n\n   New in version 2.5.\n\nobject.__coerce__(self, other)\n\n   Called to implement "mixed-mode" numeric arithmetic.  Should either\n   return a 2-tuple containing *self* and *other* converted to a\n   common numeric type, or ``None`` if conversion is impossible.  When\n   the common type would be the type of ``other``, it is sufficient to\n   return ``None``, since the interpreter will also ask the other\n   object to attempt a coercion (but sometimes, if the implementation\n   of the other type cannot be changed, it is useful to do the\n   conversion to the other type here).  A return value of\n   ``NotImplemented`` is equivalent to returning ``None``.\n\n\nCoercion rules\n==============\n\nThis section used to document the rules for coercion.  As the language\nhas evolved, the coercion rules have become hard to document\nprecisely; documenting what one version of one particular\nimplementation does is undesirable.  Instead, here are some informal\nguidelines regarding coercion.  In Python 3.0, coercion will not be\nsupported.\n\n* If the left operand of a % operator is a string or Unicode object,\n  no coercion takes place and the string formatting operation is\n  invoked instead.\n\n* It is no longer recommended to define a coercion operation. Mixed-\n  mode operations on types that don\'t define coercion pass the\n  original arguments to the operation.\n\n* New-style classes (those derived from ``object``) never invoke the\n  ``__coerce__()`` method in response to a binary operator; the only\n  time ``__coerce__()`` is invoked is when the built-in function\n  ``coerce()`` is called.\n\n* For most intents and purposes, an operator that returns\n  ``NotImplemented`` is treated the same as one that is not\n  implemented at all.\n\n* Below, ``__op__()`` and ``__rop__()`` are used to signify the\n  generic method names corresponding to an operator; ``__iop__()`` is\n  used for the corresponding in-place operator.  For example, for the\n  operator \'``+``\', ``__add__()`` and ``__radd__()`` are used for the\n  left and right variant of the binary operator, and ``__iadd__()``\n  for the in-place variant.\n\n* For objects *x* and *y*, first ``x.__op__(y)`` is tried.  If this is\n  not implemented or returns ``NotImplemented``, ``y.__rop__(x)`` is\n  tried.  If this is also not implemented or returns\n  ``NotImplemented``, a ``TypeError`` exception is raised.  But see\n  the following exception:\n\n* Exception to the previous item: if the left operand is an instance\n  of a built-in type or a new-style class, and the right operand is an\n  instance of a proper subclass of that type or class and overrides\n  the base\'s ``__rop__()`` method, the right operand\'s ``__rop__()``\n  method is tried *before* the left operand\'s ``__op__()`` method.\n\n  This is done so that a subclass can completely override binary\n  operators. Otherwise, the left operand\'s ``__op__()`` method would\n  always accept the right operand: when an instance of a given class\n  is expected, an instance of a subclass of that class is always\n  acceptable.\n\n* When either operand type defines a coercion, this coercion is called\n  before that type\'s ``__op__()`` or ``__rop__()`` method is called,\n  but no sooner.  If the coercion returns an object of a different\n  type for the operand whose coercion is invoked, part of the process\n  is redone using the new object.\n\n* When an in-place operator (like \'``+=``\') is used, if the left\n  operand implements ``__iop__()``, it is invoked without any\n  coercion.  When the operation falls back to ``__op__()`` and/or\n  ``__rop__()``, the normal coercion rules apply.\n\n* In ``x + y``, if *x* is a sequence that implements sequence\n  concatenation, sequence concatenation is invoked.\n\n* In ``x * y``, if one operator is a sequence that implements sequence\n  repetition, and the other is an integer (``int`` or ``long``),\n  sequence repetition is invoked.\n\n* Rich comparisons (implemented by methods ``__eq__()`` and so on)\n  never use coercion.  Three-way comparison (implemented by\n  ``__cmp__()``) does use coercion under the same conditions as other\n  binary operations use it.\n\n* In the current implementation, the built-in numeric types ``int``,\n  ``long``, ``float``, and ``complex`` do not use coercion. All these\n  types implement a ``__coerce__()`` method, for use by the built-in\n  ``coerce()`` function.\n\n  Changed in version 2.7.\n\n\nWith Statement Context Managers\n===============================\n\nNew in version 2.5.\n\nA *context manager* is an object that defines the runtime context to\nbe established when executing a ``with`` statement. The context\nmanager handles the entry into, and the exit from, the desired runtime\ncontext for the execution of the block of code.  Context managers are\nnormally invoked using the ``with`` statement (described in section\n*The with statement*), but can also be used by directly invoking their\nmethods.\n\nTypical uses of context managers include saving and restoring various\nkinds of global state, locking and unlocking resources, closing opened\nfiles, etc.\n\nFor more information on context managers, see *Context Manager Types*.\n\nobject.__enter__(self)\n\n   Enter the runtime context related to this object. The ``with``\n   statement will bind this method\'s return value to the target(s)\n   specified in the ``as`` clause of the statement, if any.\n\nobject.__exit__(self, exc_type, exc_value, traceback)\n\n   Exit the runtime context related to this object. The parameters\n   describe the exception that caused the context to be exited. If the\n   context was exited without an exception, all three arguments will\n   be ``None``.\n\n   If an exception is supplied, and the method wishes to suppress the\n   exception (i.e., prevent it from being propagated), it should\n   return a true value. Otherwise, the exception will be processed\n   normally upon exit from this method.\n\n   Note that ``__exit__()`` methods should not reraise the passed-in\n   exception; this is the caller\'s responsibility.\n\nSee also:\n\n   **PEP 0343** - The "with" statement\n      The specification, background, and examples for the Python\n      ``with`` statement.\n\n\nSpecial method lookup for old-style classes\n===========================================\n\nFor old-style classes, special methods are always looked up in exactly\nthe same way as any other method or attribute. This is the case\nregardless of whether the method is being looked up explicitly as in\n``x.__getitem__(i)`` or implicitly as in ``x[i]``.\n\nThis behaviour means that special methods may exhibit different\nbehaviour for different instances of a single old-style class if the\nappropriate special attributes are set differently:\n\n   >>> class C:\n   ...     pass\n   ...\n   >>> c1 = C()\n   >>> c2 = C()\n   >>> c1.__len__ = lambda: 5\n   >>> c2.__len__ = lambda: 9\n   >>> len(c1)\n   5\n   >>> len(c2)\n   9\n\n\nSpecial method lookup for new-style classes\n===========================================\n\nFor new-style classes, implicit invocations of special methods are\nonly guaranteed to work correctly if defined on an object\'s type, not\nin the object\'s instance dictionary.  That behaviour is the reason why\nthe following code raises an exception (unlike the equivalent example\nwith old-style classes):\n\n   >>> class C(object):\n   ...     pass\n   ...\n   >>> c = C()\n   >>> c.__len__ = lambda: 5\n   >>> len(c)\n   Traceback (most recent call last):\n     File "", line 1, in \n   TypeError: object of type \'C\' has no len()\n\nThe rationale behind this behaviour lies with a number of special\nmethods such as ``__hash__()`` and ``__repr__()`` that are implemented\nby all objects, including type objects. If the implicit lookup of\nthese methods used the conventional lookup process, they would fail\nwhen invoked on the type object itself:\n\n   >>> 1 .__hash__() == hash(1)\n   True\n   >>> int.__hash__() == hash(int)\n   Traceback (most recent call last):\n     File "", line 1, in \n   TypeError: descriptor \'__hash__\' of \'int\' object needs an argument\n\nIncorrectly attempting to invoke an unbound method of a class in this\nway is sometimes referred to as \'metaclass confusion\', and is avoided\nby bypassing the instance when looking up special methods:\n\n   >>> type(1).__hash__(1) == hash(1)\n   True\n   >>> type(int).__hash__(int) == hash(int)\n   True\n\nIn addition to bypassing any instance attributes in the interest of\ncorrectness, implicit special method lookup generally also bypasses\nthe ``__getattribute__()`` method even of the object\'s metaclass:\n\n   >>> class Meta(type):\n   ...    def __getattribute__(*args):\n   ...       print "Metaclass getattribute invoked"\n   ...       return type.__getattribute__(*args)\n   ...\n   >>> class C(object):\n   ...     __metaclass__ = Meta\n   ...     def __len__(self):\n   ...         return 10\n   ...     def __getattribute__(*args):\n   ...         print "Class getattribute invoked"\n   ...         return object.__getattribute__(*args)\n   ...\n   >>> c = C()\n   >>> c.__len__()                 # Explicit lookup via instance\n   Class getattribute invoked\n   10\n   >>> type(c).__len__(c)          # Explicit lookup via type\n   Metaclass getattribute invoked\n   10\n   >>> len(c)                      # Implicit lookup\n   10\n\nBypassing the ``__getattribute__()`` machinery in this fashion\nprovides significant scope for speed optimisations within the\ninterpreter, at the cost of some flexibility in the handling of\nspecial methods (the special method *must* be set on the class object\nitself in order to be consistently invoked by the interpreter).\n\n-[ Footnotes ]-\n\n[1] It *is* possible in some cases to change an object\'s type, under\n    certain controlled conditions. It generally isn\'t a good idea\n    though, since it can lead to some very strange behaviour if it is\n    handled incorrectly.\n\n[2] For operands of the same type, it is assumed that if the non-\n    reflected method (such as ``__add__()``) fails the operation is\n    not supported, which is why the reflected method is not called.\n',
+ 'specialnames': u'\nSpecial method names\n********************\n\nA class can implement certain operations that are invoked by special\nsyntax (such as arithmetic operations or subscripting and slicing) by\ndefining methods with special names. This is Python\'s approach to\n*operator overloading*, allowing classes to define their own behavior\nwith respect to language operators.  For instance, if a class defines\na method named ``__getitem__()``, and ``x`` is an instance of this\nclass, then ``x[i]`` is roughly equivalent to ``x.__getitem__(i)`` for\nold-style classes and ``type(x).__getitem__(x, i)`` for new-style\nclasses.  Except where mentioned, attempts to execute an operation\nraise an exception when no appropriate method is defined (typically\n``AttributeError`` or ``TypeError``).\n\nWhen implementing a class that emulates any built-in type, it is\nimportant that the emulation only be implemented to the degree that it\nmakes sense for the object being modelled.  For example, some\nsequences may work well with retrieval of individual elements, but\nextracting a slice may not make sense.  (One example of this is the\n``NodeList`` interface in the W3C\'s Document Object Model.)\n\n\nBasic customization\n===================\n\nobject.__new__(cls[, ...])\n\n   Called to create a new instance of class *cls*.  ``__new__()`` is a\n   static method (special-cased so you need not declare it as such)\n   that takes the class of which an instance was requested as its\n   first argument.  The remaining arguments are those passed to the\n   object constructor expression (the call to the class).  The return\n   value of ``__new__()`` should be the new object instance (usually\n   an instance of *cls*).\n\n   Typical implementations create a new instance of the class by\n   invoking the superclass\'s ``__new__()`` method using\n   ``super(currentclass, cls).__new__(cls[, ...])`` with appropriate\n   arguments and then modifying the newly-created instance as\n   necessary before returning it.\n\n   If ``__new__()`` returns an instance of *cls*, then the new\n   instance\'s ``__init__()`` method will be invoked like\n   ``__init__(self[, ...])``, where *self* is the new instance and the\n   remaining arguments are the same as were passed to ``__new__()``.\n\n   If ``__new__()`` does not return an instance of *cls*, then the new\n   instance\'s ``__init__()`` method will not be invoked.\n\n   ``__new__()`` is intended mainly to allow subclasses of immutable\n   types (like int, str, or tuple) to customize instance creation.  It\n   is also commonly overridden in custom metaclasses in order to\n   customize class creation.\n\nobject.__init__(self[, ...])\n\n   Called when the instance is created.  The arguments are those\n   passed to the class constructor expression.  If a base class has an\n   ``__init__()`` method, the derived class\'s ``__init__()`` method,\n   if any, must explicitly call it to ensure proper initialization of\n   the base class part of the instance; for example:\n   ``BaseClass.__init__(self, [args...])``.  As a special constraint\n   on constructors, no value may be returned; doing so will cause a\n   ``TypeError`` to be raised at runtime.\n\nobject.__del__(self)\n\n   Called when the instance is about to be destroyed.  This is also\n   called a destructor.  If a base class has a ``__del__()`` method,\n   the derived class\'s ``__del__()`` method, if any, must explicitly\n   call it to ensure proper deletion of the base class part of the\n   instance.  Note that it is possible (though not recommended!) for\n   the ``__del__()`` method to postpone destruction of the instance by\n   creating a new reference to it.  It may then be called at a later\n   time when this new reference is deleted.  It is not guaranteed that\n   ``__del__()`` methods are called for objects that still exist when\n   the interpreter exits.\n\n   Note: ``del x`` doesn\'t directly call ``x.__del__()`` --- the former\n     decrements the reference count for ``x`` by one, and the latter\n     is only called when ``x``\'s reference count reaches zero.  Some\n     common situations that may prevent the reference count of an\n     object from going to zero include: circular references between\n     objects (e.g., a doubly-linked list or a tree data structure with\n     parent and child pointers); a reference to the object on the\n     stack frame of a function that caught an exception (the traceback\n     stored in ``sys.exc_traceback`` keeps the stack frame alive); or\n     a reference to the object on the stack frame that raised an\n     unhandled exception in interactive mode (the traceback stored in\n     ``sys.last_traceback`` keeps the stack frame alive).  The first\n     situation can only be remedied by explicitly breaking the cycles;\n     the latter two situations can be resolved by storing ``None`` in\n     ``sys.exc_traceback`` or ``sys.last_traceback``.  Circular\n     references which are garbage are detected when the option cycle\n     detector is enabled (it\'s on by default), but can only be cleaned\n     up if there are no Python-level ``__del__()`` methods involved.\n     Refer to the documentation for the ``gc`` module for more\n     information about how ``__del__()`` methods are handled by the\n     cycle detector, particularly the description of the ``garbage``\n     value.\n\n   Warning: Due to the precarious circumstances under which ``__del__()``\n     methods are invoked, exceptions that occur during their execution\n     are ignored, and a warning is printed to ``sys.stderr`` instead.\n     Also, when ``__del__()`` is invoked in response to a module being\n     deleted (e.g., when execution of the program is done), other\n     globals referenced by the ``__del__()`` method may already have\n     been deleted or in the process of being torn down (e.g. the\n     import machinery shutting down).  For this reason, ``__del__()``\n     methods should do the absolute minimum needed to maintain\n     external invariants.  Starting with version 1.5, Python\n     guarantees that globals whose name begins with a single\n     underscore are deleted from their module before other globals are\n     deleted; if no other references to such globals exist, this may\n     help in assuring that imported modules are still available at the\n     time when the ``__del__()`` method is called.\n\nobject.__repr__(self)\n\n   Called by the ``repr()`` built-in function and by string\n   conversions (reverse quotes) to compute the "official" string\n   representation of an object.  If at all possible, this should look\n   like a valid Python expression that could be used to recreate an\n   object with the same value (given an appropriate environment).  If\n   this is not possible, a string of the form ``<...some useful\n   description...>`` should be returned.  The return value must be a\n   string object. If a class defines ``__repr__()`` but not\n   ``__str__()``, then ``__repr__()`` is also used when an "informal"\n   string representation of instances of that class is required.\n\n   This is typically used for debugging, so it is important that the\n   representation is information-rich and unambiguous.\n\nobject.__str__(self)\n\n   Called by the ``str()`` built-in function and by the ``print``\n   statement to compute the "informal" string representation of an\n   object.  This differs from ``__repr__()`` in that it does not have\n   to be a valid Python expression: a more convenient or concise\n   representation may be used instead. The return value must be a\n   string object.\n\nobject.__lt__(self, other)\nobject.__le__(self, other)\nobject.__eq__(self, other)\nobject.__ne__(self, other)\nobject.__gt__(self, other)\nobject.__ge__(self, other)\n\n   New in version 2.1.\n\n   These are the so-called "rich comparison" methods, and are called\n   for comparison operators in preference to ``__cmp__()`` below. The\n   correspondence between operator symbols and method names is as\n   follows: ``xy`` call ``x.__ne__(y)``, ``x>y`` calls ``x.__gt__(y)``, and\n   ``x>=y`` calls ``x.__ge__(y)``.\n\n   A rich comparison method may return the singleton\n   ``NotImplemented`` if it does not implement the operation for a\n   given pair of arguments. By convention, ``False`` and ``True`` are\n   returned for a successful comparison. However, these methods can\n   return any value, so if the comparison operator is used in a\n   Boolean context (e.g., in the condition of an ``if`` statement),\n   Python will call ``bool()`` on the value to determine if the result\n   is true or false.\n\n   There are no implied relationships among the comparison operators.\n   The truth of ``x==y`` does not imply that ``x!=y`` is false.\n   Accordingly, when defining ``__eq__()``, one should also define\n   ``__ne__()`` so that the operators will behave as expected.  See\n   the paragraph on ``__hash__()`` for some important notes on\n   creating *hashable* objects which support custom comparison\n   operations and are usable as dictionary keys.\n\n   There are no swapped-argument versions of these methods (to be used\n   when the left argument does not support the operation but the right\n   argument does); rather, ``__lt__()`` and ``__gt__()`` are each\n   other\'s reflection, ``__le__()`` and ``__ge__()`` are each other\'s\n   reflection, and ``__eq__()`` and ``__ne__()`` are their own\n   reflection.\n\n   Arguments to rich comparison methods are never coerced.\n\n   To automatically generate ordering operations from a single root\n   operation, see ``functools.total_ordering()``.\n\nobject.__cmp__(self, other)\n\n   Called by comparison operations if rich comparison (see above) is\n   not defined.  Should return a negative integer if ``self < other``,\n   zero if ``self == other``, a positive integer if ``self > other``.\n   If no ``__cmp__()``, ``__eq__()`` or ``__ne__()`` operation is\n   defined, class instances are compared by object identity\n   ("address").  See also the description of ``__hash__()`` for some\n   important notes on creating *hashable* objects which support custom\n   comparison operations and are usable as dictionary keys. (Note: the\n   restriction that exceptions are not propagated by ``__cmp__()`` has\n   been removed since Python 1.5.)\n\nobject.__rcmp__(self, other)\n\n   Changed in version 2.1: No longer supported.\n\nobject.__hash__(self)\n\n   Called by built-in function ``hash()`` and for operations on\n   members of hashed collections including ``set``, ``frozenset``, and\n   ``dict``.  ``__hash__()`` should return an integer.  The only\n   required property is that objects which compare equal have the same\n   hash value; it is advised to somehow mix together (e.g. using\n   exclusive or) the hash values for the components of the object that\n   also play a part in comparison of objects.\n\n   If a class does not define a ``__cmp__()`` or ``__eq__()`` method\n   it should not define a ``__hash__()`` operation either; if it\n   defines ``__cmp__()`` or ``__eq__()`` but not ``__hash__()``, its\n   instances will not be usable in hashed collections.  If a class\n   defines mutable objects and implements a ``__cmp__()`` or\n   ``__eq__()`` method, it should not implement ``__hash__()``, since\n   hashable collection implementations require that a object\'s hash\n   value is immutable (if the object\'s hash value changes, it will be\n   in the wrong hash bucket).\n\n   User-defined classes have ``__cmp__()`` and ``__hash__()`` methods\n   by default; with them, all objects compare unequal (except with\n   themselves) and ``x.__hash__()`` returns ``id(x)``.\n\n   Classes which inherit a ``__hash__()`` method from a parent class\n   but change the meaning of ``__cmp__()`` or ``__eq__()`` such that\n   the hash value returned is no longer appropriate (e.g. by switching\n   to a value-based concept of equality instead of the default\n   identity based equality) can explicitly flag themselves as being\n   unhashable by setting ``__hash__ = None`` in the class definition.\n   Doing so means that not only will instances of the class raise an\n   appropriate ``TypeError`` when a program attempts to retrieve their\n   hash value, but they will also be correctly identified as\n   unhashable when checking ``isinstance(obj, collections.Hashable)``\n   (unlike classes which define their own ``__hash__()`` to explicitly\n   raise ``TypeError``).\n\n   Changed in version 2.5: ``__hash__()`` may now also return a long\n   integer object; the 32-bit integer is then derived from the hash of\n   that object.\n\n   Changed in version 2.6: ``__hash__`` may now be set to ``None`` to\n   explicitly flag instances of a class as unhashable.\n\nobject.__nonzero__(self)\n\n   Called to implement truth value testing and the built-in operation\n   ``bool()``; should return ``False`` or ``True``, or their integer\n   equivalents ``0`` or ``1``.  When this method is not defined,\n   ``__len__()`` is called, if it is defined, and the object is\n   considered true if its result is nonzero. If a class defines\n   neither ``__len__()`` nor ``__nonzero__()``, all its instances are\n   considered true.\n\nobject.__unicode__(self)\n\n   Called to implement ``unicode()`` built-in; should return a Unicode\n   object. When this method is not defined, string conversion is\n   attempted, and the result of string conversion is converted to\n   Unicode using the system default encoding.\n\n\nCustomizing attribute access\n============================\n\nThe following methods can be defined to customize the meaning of\nattribute access (use of, assignment to, or deletion of ``x.name``)\nfor class instances.\n\nobject.__getattr__(self, name)\n\n   Called when an attribute lookup has not found the attribute in the\n   usual places (i.e. it is not an instance attribute nor is it found\n   in the class tree for ``self``).  ``name`` is the attribute name.\n   This method should return the (computed) attribute value or raise\n   an ``AttributeError`` exception.\n\n   Note that if the attribute is found through the normal mechanism,\n   ``__getattr__()`` is not called.  (This is an intentional asymmetry\n   between ``__getattr__()`` and ``__setattr__()``.) This is done both\n   for efficiency reasons and because otherwise ``__getattr__()``\n   would have no way to access other attributes of the instance.  Note\n   that at least for instance variables, you can fake total control by\n   not inserting any values in the instance attribute dictionary (but\n   instead inserting them in another object).  See the\n   ``__getattribute__()`` method below for a way to actually get total\n   control in new-style classes.\n\nobject.__setattr__(self, name, value)\n\n   Called when an attribute assignment is attempted.  This is called\n   instead of the normal mechanism (i.e. store the value in the\n   instance dictionary).  *name* is the attribute name, *value* is the\n   value to be assigned to it.\n\n   If ``__setattr__()`` wants to assign to an instance attribute, it\n   should not simply execute ``self.name = value`` --- this would\n   cause a recursive call to itself.  Instead, it should insert the\n   value in the dictionary of instance attributes, e.g.,\n   ``self.__dict__[name] = value``.  For new-style classes, rather\n   than accessing the instance dictionary, it should call the base\n   class method with the same name, for example,\n   ``object.__setattr__(self, name, value)``.\n\nobject.__delattr__(self, name)\n\n   Like ``__setattr__()`` but for attribute deletion instead of\n   assignment.  This should only be implemented if ``del obj.name`` is\n   meaningful for the object.\n\n\nMore attribute access for new-style classes\n-------------------------------------------\n\nThe following methods only apply to new-style classes.\n\nobject.__getattribute__(self, name)\n\n   Called unconditionally to implement attribute accesses for\n   instances of the class. If the class also defines\n   ``__getattr__()``, the latter will not be called unless\n   ``__getattribute__()`` either calls it explicitly or raises an\n   ``AttributeError``. This method should return the (computed)\n   attribute value or raise an ``AttributeError`` exception. In order\n   to avoid infinite recursion in this method, its implementation\n   should always call the base class method with the same name to\n   access any attributes it needs, for example,\n   ``object.__getattribute__(self, name)``.\n\n   Note: This method may still be bypassed when looking up special methods\n     as the result of implicit invocation via language syntax or\n     built-in functions. See *Special method lookup for new-style\n     classes*.\n\n\nImplementing Descriptors\n------------------------\n\nThe following methods only apply when an instance of the class\ncontaining the method (a so-called *descriptor* class) appears in an\n*owner* class (the descriptor must be in either the owner\'s class\ndictionary or in the class dictionary for one of its parents).  In the\nexamples below, "the attribute" refers to the attribute whose name is\nthe key of the property in the owner class\' ``__dict__``.\n\nobject.__get__(self, instance, owner)\n\n   Called to get the attribute of the owner class (class attribute\n   access) or of an instance of that class (instance attribute\n   access). *owner* is always the owner class, while *instance* is the\n   instance that the attribute was accessed through, or ``None`` when\n   the attribute is accessed through the *owner*.  This method should\n   return the (computed) attribute value or raise an\n   ``AttributeError`` exception.\n\nobject.__set__(self, instance, value)\n\n   Called to set the attribute on an instance *instance* of the owner\n   class to a new value, *value*.\n\nobject.__delete__(self, instance)\n\n   Called to delete the attribute on an instance *instance* of the\n   owner class.\n\n\nInvoking Descriptors\n--------------------\n\nIn general, a descriptor is an object attribute with "binding\nbehavior", one whose attribute access has been overridden by methods\nin the descriptor protocol:  ``__get__()``, ``__set__()``, and\n``__delete__()``. If any of those methods are defined for an object,\nit is said to be a descriptor.\n\nThe default behavior for attribute access is to get, set, or delete\nthe attribute from an object\'s dictionary. For instance, ``a.x`` has a\nlookup chain starting with ``a.__dict__[\'x\']``, then\n``type(a).__dict__[\'x\']``, and continuing through the base classes of\n``type(a)`` excluding metaclasses.\n\nHowever, if the looked-up value is an object defining one of the\ndescriptor methods, then Python may override the default behavior and\ninvoke the descriptor method instead.  Where this occurs in the\nprecedence chain depends on which descriptor methods were defined and\nhow they were called.  Note that descriptors are only invoked for new\nstyle objects or classes (ones that subclass ``object()`` or\n``type()``).\n\nThe starting point for descriptor invocation is a binding, ``a.x``.\nHow the arguments are assembled depends on ``a``:\n\nDirect Call\n   The simplest and least common call is when user code directly\n   invokes a descriptor method:    ``x.__get__(a)``.\n\nInstance Binding\n   If binding to a new-style object instance, ``a.x`` is transformed\n   into the call: ``type(a).__dict__[\'x\'].__get__(a, type(a))``.\n\nClass Binding\n   If binding to a new-style class, ``A.x`` is transformed into the\n   call: ``A.__dict__[\'x\'].__get__(None, A)``.\n\nSuper Binding\n   If ``a`` is an instance of ``super``, then the binding ``super(B,\n   obj).m()`` searches ``obj.__class__.__mro__`` for the base class\n   ``A`` immediately preceding ``B`` and then invokes the descriptor\n   with the call: ``A.__dict__[\'m\'].__get__(obj, obj.__class__)``.\n\nFor instance bindings, the precedence of descriptor invocation depends\non the which descriptor methods are defined.  A descriptor can define\nany combination of ``__get__()``, ``__set__()`` and ``__delete__()``.\nIf it does not define ``__get__()``, then accessing the attribute will\nreturn the descriptor object itself unless there is a value in the\nobject\'s instance dictionary.  If the descriptor defines ``__set__()``\nand/or ``__delete__()``, it is a data descriptor; if it defines\nneither, it is a non-data descriptor.  Normally, data descriptors\ndefine both ``__get__()`` and ``__set__()``, while non-data\ndescriptors have just the ``__get__()`` method.  Data descriptors with\n``__set__()`` and ``__get__()`` defined always override a redefinition\nin an instance dictionary.  In contrast, non-data descriptors can be\noverridden by instances.\n\nPython methods (including ``staticmethod()`` and ``classmethod()``)\nare implemented as non-data descriptors.  Accordingly, instances can\nredefine and override methods.  This allows individual instances to\nacquire behaviors that differ from other instances of the same class.\n\nThe ``property()`` function is implemented as a data descriptor.\nAccordingly, instances cannot override the behavior of a property.\n\n\n__slots__\n---------\n\nBy default, instances of both old and new-style classes have a\ndictionary for attribute storage.  This wastes space for objects\nhaving very few instance variables.  The space consumption can become\nacute when creating large numbers of instances.\n\nThe default can be overridden by defining *__slots__* in a new-style\nclass definition.  The *__slots__* declaration takes a sequence of\ninstance variables and reserves just enough space in each instance to\nhold a value for each variable.  Space is saved because *__dict__* is\nnot created for each instance.\n\n__slots__\n\n   This class variable can be assigned a string, iterable, or sequence\n   of strings with variable names used by instances.  If defined in a\n   new-style class, *__slots__* reserves space for the declared\n   variables and prevents the automatic creation of *__dict__* and\n   *__weakref__* for each instance.\n\n   New in version 2.2.\n\nNotes on using *__slots__*\n\n* When inheriting from a class without *__slots__*, the *__dict__*\n  attribute of that class will always be accessible, so a *__slots__*\n  definition in the subclass is meaningless.\n\n* Without a *__dict__* variable, instances cannot be assigned new\n  variables not listed in the *__slots__* definition.  Attempts to\n  assign to an unlisted variable name raises ``AttributeError``. If\n  dynamic assignment of new variables is desired, then add\n  ``\'__dict__\'`` to the sequence of strings in the *__slots__*\n  declaration.\n\n  Changed in version 2.3: Previously, adding ``\'__dict__\'`` to the\n  *__slots__* declaration would not enable the assignment of new\n  attributes not specifically listed in the sequence of instance\n  variable names.\n\n* Without a *__weakref__* variable for each instance, classes defining\n  *__slots__* do not support weak references to its instances. If weak\n  reference support is needed, then add ``\'__weakref__\'`` to the\n  sequence of strings in the *__slots__* declaration.\n\n  Changed in version 2.3: Previously, adding ``\'__weakref__\'`` to the\n  *__slots__* declaration would not enable support for weak\n  references.\n\n* *__slots__* are implemented at the class level by creating\n  descriptors (*Implementing Descriptors*) for each variable name.  As\n  a result, class attributes cannot be used to set default values for\n  instance variables defined by *__slots__*; otherwise, the class\n  attribute would overwrite the descriptor assignment.\n\n* The action of a *__slots__* declaration is limited to the class\n  where it is defined.  As a result, subclasses will have a *__dict__*\n  unless they also define *__slots__* (which must only contain names\n  of any *additional* slots).\n\n* If a class defines a slot also defined in a base class, the instance\n  variable defined by the base class slot is inaccessible (except by\n  retrieving its descriptor directly from the base class). This\n  renders the meaning of the program undefined.  In the future, a\n  check may be added to prevent this.\n\n* Nonempty *__slots__* does not work for classes derived from\n  "variable-length" built-in types such as ``long``, ``str`` and\n  ``tuple``.\n\n* Any non-string iterable may be assigned to *__slots__*. Mappings may\n  also be used; however, in the future, special meaning may be\n  assigned to the values corresponding to each key.\n\n* *__class__* assignment works only if both classes have the same\n  *__slots__*.\n\n  Changed in version 2.6: Previously, *__class__* assignment raised an\n  error if either new or old class had *__slots__*.\n\n\nCustomizing class creation\n==========================\n\nBy default, new-style classes are constructed using ``type()``. A\nclass definition is read into a separate namespace and the value of\nclass name is bound to the result of ``type(name, bases, dict)``.\n\nWhen the class definition is read, if *__metaclass__* is defined then\nthe callable assigned to it will be called instead of ``type()``. This\nallows classes or functions to be written which monitor or alter the\nclass creation process:\n\n* Modifying the class dictionary prior to the class being created.\n\n* Returning an instance of another class -- essentially performing the\n  role of a factory function.\n\nThese steps will have to be performed in the metaclass\'s ``__new__()``\nmethod -- ``type.__new__()`` can then be called from this method to\ncreate a class with different properties.  This example adds a new\nelement to the class dictionary before creating the class:\n\n   class metacls(type):\n       def __new__(mcs, name, bases, dict):\n           dict[\'foo\'] = \'metacls was here\'\n           return type.__new__(mcs, name, bases, dict)\n\nYou can of course also override other class methods (or add new\nmethods); for example defining a custom ``__call__()`` method in the\nmetaclass allows custom behavior when the class is called, e.g. not\nalways creating a new instance.\n\n__metaclass__\n\n   This variable can be any callable accepting arguments for ``name``,\n   ``bases``, and ``dict``.  Upon class creation, the callable is used\n   instead of the built-in ``type()``.\n\n   New in version 2.2.\n\nThe appropriate metaclass is determined by the following precedence\nrules:\n\n* If ``dict[\'__metaclass__\']`` exists, it is used.\n\n* Otherwise, if there is at least one base class, its metaclass is\n  used (this looks for a *__class__* attribute first and if not found,\n  uses its type).\n\n* Otherwise, if a global variable named __metaclass__ exists, it is\n  used.\n\n* Otherwise, the old-style, classic metaclass (types.ClassType) is\n  used.\n\nThe potential uses for metaclasses are boundless. Some ideas that have\nbeen explored including logging, interface checking, automatic\ndelegation, automatic property creation, proxies, frameworks, and\nautomatic resource locking/synchronization.\n\n\nCustomizing instance and subclass checks\n========================================\n\nNew in version 2.6.\n\nThe following methods are used to override the default behavior of the\n``isinstance()`` and ``issubclass()`` built-in functions.\n\nIn particular, the metaclass ``abc.ABCMeta`` implements these methods\nin order to allow the addition of Abstract Base Classes (ABCs) as\n"virtual base classes" to any class or type (including built-in\ntypes), including other ABCs.\n\nclass.__instancecheck__(self, instance)\n\n   Return true if *instance* should be considered a (direct or\n   indirect) instance of *class*. If defined, called to implement\n   ``isinstance(instance, class)``.\n\nclass.__subclasscheck__(self, subclass)\n\n   Return true if *subclass* should be considered a (direct or\n   indirect) subclass of *class*.  If defined, called to implement\n   ``issubclass(subclass, class)``.\n\nNote that these methods are looked up on the type (metaclass) of a\nclass.  They cannot be defined as class methods in the actual class.\nThis is consistent with the lookup of special methods that are called\non instances, only in this case the instance is itself a class.\n\nSee also:\n\n   **PEP 3119** - Introducing Abstract Base Classes\n      Includes the specification for customizing ``isinstance()`` and\n      ``issubclass()`` behavior through ``__instancecheck__()`` and\n      ``__subclasscheck__()``, with motivation for this functionality\n      in the context of adding Abstract Base Classes (see the ``abc``\n      module) to the language.\n\n\nEmulating callable objects\n==========================\n\nobject.__call__(self[, args...])\n\n   Called when the instance is "called" as a function; if this method\n   is defined, ``x(arg1, arg2, ...)`` is a shorthand for\n   ``x.__call__(arg1, arg2, ...)``.\n\n\nEmulating container types\n=========================\n\nThe following methods can be defined to implement container objects.\nContainers usually are sequences (such as lists or tuples) or mappings\n(like dictionaries), but can represent other containers as well.  The\nfirst set of methods is used either to emulate a sequence or to\nemulate a mapping; the difference is that for a sequence, the\nallowable keys should be the integers *k* for which ``0 <= k < N``\nwhere *N* is the length of the sequence, or slice objects, which\ndefine a range of items. (For backwards compatibility, the method\n``__getslice__()`` (see below) can also be defined to handle simple,\nbut not extended slices.) It is also recommended that mappings provide\nthe methods ``keys()``, ``values()``, ``items()``, ``has_key()``,\n``get()``, ``clear()``, ``setdefault()``, ``iterkeys()``,\n``itervalues()``, ``iteritems()``, ``pop()``, ``popitem()``,\n``copy()``, and ``update()`` behaving similar to those for Python\'s\nstandard dictionary objects.  The ``UserDict`` module provides a\n``DictMixin`` class to help create those methods from a base set of\n``__getitem__()``, ``__setitem__()``, ``__delitem__()``, and\n``keys()``. Mutable sequences should provide methods ``append()``,\n``count()``, ``index()``, ``extend()``, ``insert()``, ``pop()``,\n``remove()``, ``reverse()`` and ``sort()``, like Python standard list\nobjects.  Finally, sequence types should implement addition (meaning\nconcatenation) and multiplication (meaning repetition) by defining the\nmethods ``__add__()``, ``__radd__()``, ``__iadd__()``, ``__mul__()``,\n``__rmul__()`` and ``__imul__()`` described below; they should not\ndefine ``__coerce__()`` or other numerical operators.  It is\nrecommended that both mappings and sequences implement the\n``__contains__()`` method to allow efficient use of the ``in``\noperator; for mappings, ``in`` should be equivalent of ``has_key()``;\nfor sequences, it should search through the values.  It is further\nrecommended that both mappings and sequences implement the\n``__iter__()`` method to allow efficient iteration through the\ncontainer; for mappings, ``__iter__()`` should be the same as\n``iterkeys()``; for sequences, it should iterate through the values.\n\nobject.__len__(self)\n\n   Called to implement the built-in function ``len()``.  Should return\n   the length of the object, an integer ``>=`` 0.  Also, an object\n   that doesn\'t define a ``__nonzero__()`` method and whose\n   ``__len__()`` method returns zero is considered to be false in a\n   Boolean context.\n\nobject.__getitem__(self, key)\n\n   Called to implement evaluation of ``self[key]``. For sequence\n   types, the accepted keys should be integers and slice objects.\n   Note that the special interpretation of negative indexes (if the\n   class wishes to emulate a sequence type) is up to the\n   ``__getitem__()`` method. If *key* is of an inappropriate type,\n   ``TypeError`` may be raised; if of a value outside the set of\n   indexes for the sequence (after any special interpretation of\n   negative values), ``IndexError`` should be raised. For mapping\n   types, if *key* is missing (not in the container), ``KeyError``\n   should be raised.\n\n   Note: ``for`` loops expect that an ``IndexError`` will be raised for\n     illegal indexes to allow proper detection of the end of the\n     sequence.\n\nobject.__setitem__(self, key, value)\n\n   Called to implement assignment to ``self[key]``.  Same note as for\n   ``__getitem__()``.  This should only be implemented for mappings if\n   the objects support changes to the values for keys, or if new keys\n   can be added, or for sequences if elements can be replaced.  The\n   same exceptions should be raised for improper *key* values as for\n   the ``__getitem__()`` method.\n\nobject.__delitem__(self, key)\n\n   Called to implement deletion of ``self[key]``.  Same note as for\n   ``__getitem__()``.  This should only be implemented for mappings if\n   the objects support removal of keys, or for sequences if elements\n   can be removed from the sequence.  The same exceptions should be\n   raised for improper *key* values as for the ``__getitem__()``\n   method.\n\nobject.__iter__(self)\n\n   This method is called when an iterator is required for a container.\n   This method should return a new iterator object that can iterate\n   over all the objects in the container.  For mappings, it should\n   iterate over the keys of the container, and should also be made\n   available as the method ``iterkeys()``.\n\n   Iterator objects also need to implement this method; they are\n   required to return themselves.  For more information on iterator\n   objects, see *Iterator Types*.\n\nobject.__reversed__(self)\n\n   Called (if present) by the ``reversed()`` built-in to implement\n   reverse iteration.  It should return a new iterator object that\n   iterates over all the objects in the container in reverse order.\n\n   If the ``__reversed__()`` method is not provided, the\n   ``reversed()`` built-in will fall back to using the sequence\n   protocol (``__len__()`` and ``__getitem__()``).  Objects that\n   support the sequence protocol should only provide\n   ``__reversed__()`` if they can provide an implementation that is\n   more efficient than the one provided by ``reversed()``.\n\n   New in version 2.6.\n\nThe membership test operators (``in`` and ``not in``) are normally\nimplemented as an iteration through a sequence.  However, container\nobjects can supply the following special method with a more efficient\nimplementation, which also does not require the object be a sequence.\n\nobject.__contains__(self, item)\n\n   Called to implement membership test operators.  Should return true\n   if *item* is in *self*, false otherwise.  For mapping objects, this\n   should consider the keys of the mapping rather than the values or\n   the key-item pairs.\n\n   For objects that don\'t define ``__contains__()``, the membership\n   test first tries iteration via ``__iter__()``, then the old\n   sequence iteration protocol via ``__getitem__()``, see *this\n   section in the language reference*.\n\n\nAdditional methods for emulation of sequence types\n==================================================\n\nThe following optional methods can be defined to further emulate\nsequence objects.  Immutable sequences methods should at most only\ndefine ``__getslice__()``; mutable sequences might define all three\nmethods.\n\nobject.__getslice__(self, i, j)\n\n   Deprecated since version 2.0: Support slice objects as parameters\n   to the ``__getitem__()`` method. (However, built-in types in\n   CPython currently still implement ``__getslice__()``.  Therefore,\n   you have to override it in derived classes when implementing\n   slicing.)\n\n   Called to implement evaluation of ``self[i:j]``. The returned\n   object should be of the same type as *self*.  Note that missing *i*\n   or *j* in the slice expression are replaced by zero or\n   ``sys.maxint``, respectively.  If negative indexes are used in the\n   slice, the length of the sequence is added to that index. If the\n   instance does not implement the ``__len__()`` method, an\n   ``AttributeError`` is raised. No guarantee is made that indexes\n   adjusted this way are not still negative.  Indexes which are\n   greater than the length of the sequence are not modified. If no\n   ``__getslice__()`` is found, a slice object is created instead, and\n   passed to ``__getitem__()`` instead.\n\nobject.__setslice__(self, i, j, sequence)\n\n   Called to implement assignment to ``self[i:j]``. Same notes for *i*\n   and *j* as for ``__getslice__()``.\n\n   This method is deprecated. If no ``__setslice__()`` is found, or\n   for extended slicing of the form ``self[i:j:k]``, a slice object is\n   created, and passed to ``__setitem__()``, instead of\n   ``__setslice__()`` being called.\n\nobject.__delslice__(self, i, j)\n\n   Called to implement deletion of ``self[i:j]``. Same notes for *i*\n   and *j* as for ``__getslice__()``. This method is deprecated. If no\n   ``__delslice__()`` is found, or for extended slicing of the form\n   ``self[i:j:k]``, a slice object is created, and passed to\n   ``__delitem__()``, instead of ``__delslice__()`` being called.\n\nNotice that these methods are only invoked when a single slice with a\nsingle colon is used, and the slice method is available.  For slice\noperations involving extended slice notation, or in absence of the\nslice methods, ``__getitem__()``, ``__setitem__()`` or\n``__delitem__()`` is called with a slice object as argument.\n\nThe following example demonstrate how to make your program or module\ncompatible with earlier versions of Python (assuming that methods\n``__getitem__()``, ``__setitem__()`` and ``__delitem__()`` support\nslice objects as arguments):\n\n   class MyClass:\n       ...\n       def __getitem__(self, index):\n           ...\n       def __setitem__(self, index, value):\n           ...\n       def __delitem__(self, index):\n           ...\n\n       if sys.version_info < (2, 0):\n           # They won\'t be defined if version is at least 2.0 final\n\n           def __getslice__(self, i, j):\n               return self[max(0, i):max(0, j):]\n           def __setslice__(self, i, j, seq):\n               self[max(0, i):max(0, j):] = seq\n           def __delslice__(self, i, j):\n               del self[max(0, i):max(0, j):]\n       ...\n\nNote the calls to ``max()``; these are necessary because of the\nhandling of negative indices before the ``__*slice__()`` methods are\ncalled.  When negative indexes are used, the ``__*item__()`` methods\nreceive them as provided, but the ``__*slice__()`` methods get a\n"cooked" form of the index values.  For each negative index value, the\nlength of the sequence is added to the index before calling the method\n(which may still result in a negative index); this is the customary\nhandling of negative indexes by the built-in sequence types, and the\n``__*item__()`` methods are expected to do this as well.  However,\nsince they should already be doing that, negative indexes cannot be\npassed in; they must be constrained to the bounds of the sequence\nbefore being passed to the ``__*item__()`` methods. Calling ``max(0,\ni)`` conveniently returns the proper value.\n\n\nEmulating numeric types\n=======================\n\nThe following methods can be defined to emulate numeric objects.\nMethods corresponding to operations that are not supported by the\nparticular kind of number implemented (e.g., bitwise operations for\nnon-integral numbers) should be left undefined.\n\nobject.__add__(self, other)\nobject.__sub__(self, other)\nobject.__mul__(self, other)\nobject.__floordiv__(self, other)\nobject.__mod__(self, other)\nobject.__divmod__(self, other)\nobject.__pow__(self, other[, modulo])\nobject.__lshift__(self, other)\nobject.__rshift__(self, other)\nobject.__and__(self, other)\nobject.__xor__(self, other)\nobject.__or__(self, other)\n\n   These methods are called to implement the binary arithmetic\n   operations (``+``, ``-``, ``*``, ``//``, ``%``, ``divmod()``,\n   ``pow()``, ``**``, ``<<``, ``>>``, ``&``, ``^``, ``|``).  For\n   instance, to evaluate the expression ``x + y``, where *x* is an\n   instance of a class that has an ``__add__()`` method,\n   ``x.__add__(y)`` is called.  The ``__divmod__()`` method should be\n   the equivalent to using ``__floordiv__()`` and ``__mod__()``; it\n   should not be related to ``__truediv__()`` (described below).  Note\n   that ``__pow__()`` should be defined to accept an optional third\n   argument if the ternary version of the built-in ``pow()`` function\n   is to be supported.\n\n   If one of those methods does not support the operation with the\n   supplied arguments, it should return ``NotImplemented``.\n\nobject.__div__(self, other)\nobject.__truediv__(self, other)\n\n   The division operator (``/``) is implemented by these methods.  The\n   ``__truediv__()`` method is used when ``__future__.division`` is in\n   effect, otherwise ``__div__()`` is used.  If only one of these two\n   methods is defined, the object will not support division in the\n   alternate context; ``TypeError`` will be raised instead.\n\nobject.__radd__(self, other)\nobject.__rsub__(self, other)\nobject.__rmul__(self, other)\nobject.__rdiv__(self, other)\nobject.__rtruediv__(self, other)\nobject.__rfloordiv__(self, other)\nobject.__rmod__(self, other)\nobject.__rdivmod__(self, other)\nobject.__rpow__(self, other)\nobject.__rlshift__(self, other)\nobject.__rrshift__(self, other)\nobject.__rand__(self, other)\nobject.__rxor__(self, other)\nobject.__ror__(self, other)\n\n   These methods are called to implement the binary arithmetic\n   operations (``+``, ``-``, ``*``, ``/``, ``%``, ``divmod()``,\n   ``pow()``, ``**``, ``<<``, ``>>``, ``&``, ``^``, ``|``) with\n   reflected (swapped) operands.  These functions are only called if\n   the left operand does not support the corresponding operation and\n   the operands are of different types. [2] For instance, to evaluate\n   the expression ``x - y``, where *y* is an instance of a class that\n   has an ``__rsub__()`` method, ``y.__rsub__(x)`` is called if\n   ``x.__sub__(y)`` returns *NotImplemented*.\n\n   Note that ternary ``pow()`` will not try calling ``__rpow__()``\n   (the coercion rules would become too complicated).\n\n   Note: If the right operand\'s type is a subclass of the left operand\'s\n     type and that subclass provides the reflected method for the\n     operation, this method will be called before the left operand\'s\n     non-reflected method.  This behavior allows subclasses to\n     override their ancestors\' operations.\n\nobject.__iadd__(self, other)\nobject.__isub__(self, other)\nobject.__imul__(self, other)\nobject.__idiv__(self, other)\nobject.__itruediv__(self, other)\nobject.__ifloordiv__(self, other)\nobject.__imod__(self, other)\nobject.__ipow__(self, other[, modulo])\nobject.__ilshift__(self, other)\nobject.__irshift__(self, other)\nobject.__iand__(self, other)\nobject.__ixor__(self, other)\nobject.__ior__(self, other)\n\n   These methods are called to implement the augmented arithmetic\n   assignments (``+=``, ``-=``, ``*=``, ``/=``, ``//=``, ``%=``,\n   ``**=``, ``<<=``, ``>>=``, ``&=``, ``^=``, ``|=``).  These methods\n   should attempt to do the operation in-place (modifying *self*) and\n   return the result (which could be, but does not have to be,\n   *self*).  If a specific method is not defined, the augmented\n   assignment falls back to the normal methods.  For instance, to\n   execute the statement ``x += y``, where *x* is an instance of a\n   class that has an ``__iadd__()`` method, ``x.__iadd__(y)`` is\n   called.  If *x* is an instance of a class that does not define a\n   ``__iadd__()`` method, ``x.__add__(y)`` and ``y.__radd__(x)`` are\n   considered, as with the evaluation of ``x + y``.\n\nobject.__neg__(self)\nobject.__pos__(self)\nobject.__abs__(self)\nobject.__invert__(self)\n\n   Called to implement the unary arithmetic operations (``-``, ``+``,\n   ``abs()`` and ``~``).\n\nobject.__complex__(self)\nobject.__int__(self)\nobject.__long__(self)\nobject.__float__(self)\n\n   Called to implement the built-in functions ``complex()``,\n   ``int()``, ``long()``, and ``float()``.  Should return a value of\n   the appropriate type.\n\nobject.__oct__(self)\nobject.__hex__(self)\n\n   Called to implement the built-in functions ``oct()`` and ``hex()``.\n   Should return a string value.\n\nobject.__index__(self)\n\n   Called to implement ``operator.index()``.  Also called whenever\n   Python needs an integer object (such as in slicing).  Must return\n   an integer (int or long).\n\n   New in version 2.5.\n\nobject.__coerce__(self, other)\n\n   Called to implement "mixed-mode" numeric arithmetic.  Should either\n   return a 2-tuple containing *self* and *other* converted to a\n   common numeric type, or ``None`` if conversion is impossible.  When\n   the common type would be the type of ``other``, it is sufficient to\n   return ``None``, since the interpreter will also ask the other\n   object to attempt a coercion (but sometimes, if the implementation\n   of the other type cannot be changed, it is useful to do the\n   conversion to the other type here).  A return value of\n   ``NotImplemented`` is equivalent to returning ``None``.\n\n\nCoercion rules\n==============\n\nThis section used to document the rules for coercion.  As the language\nhas evolved, the coercion rules have become hard to document\nprecisely; documenting what one version of one particular\nimplementation does is undesirable.  Instead, here are some informal\nguidelines regarding coercion.  In Python 3.0, coercion will not be\nsupported.\n\n* If the left operand of a % operator is a string or Unicode object,\n  no coercion takes place and the string formatting operation is\n  invoked instead.\n\n* It is no longer recommended to define a coercion operation. Mixed-\n  mode operations on types that don\'t define coercion pass the\n  original arguments to the operation.\n\n* New-style classes (those derived from ``object``) never invoke the\n  ``__coerce__()`` method in response to a binary operator; the only\n  time ``__coerce__()`` is invoked is when the built-in function\n  ``coerce()`` is called.\n\n* For most intents and purposes, an operator that returns\n  ``NotImplemented`` is treated the same as one that is not\n  implemented at all.\n\n* Below, ``__op__()`` and ``__rop__()`` are used to signify the\n  generic method names corresponding to an operator; ``__iop__()`` is\n  used for the corresponding in-place operator.  For example, for the\n  operator \'``+``\', ``__add__()`` and ``__radd__()`` are used for the\n  left and right variant of the binary operator, and ``__iadd__()``\n  for the in-place variant.\n\n* For objects *x* and *y*, first ``x.__op__(y)`` is tried.  If this is\n  not implemented or returns ``NotImplemented``, ``y.__rop__(x)`` is\n  tried.  If this is also not implemented or returns\n  ``NotImplemented``, a ``TypeError`` exception is raised.  But see\n  the following exception:\n\n* Exception to the previous item: if the left operand is an instance\n  of a built-in type or a new-style class, and the right operand is an\n  instance of a proper subclass of that type or class and overrides\n  the base\'s ``__rop__()`` method, the right operand\'s ``__rop__()``\n  method is tried *before* the left operand\'s ``__op__()`` method.\n\n  This is done so that a subclass can completely override binary\n  operators. Otherwise, the left operand\'s ``__op__()`` method would\n  always accept the right operand: when an instance of a given class\n  is expected, an instance of a subclass of that class is always\n  acceptable.\n\n* When either operand type defines a coercion, this coercion is called\n  before that type\'s ``__op__()`` or ``__rop__()`` method is called,\n  but no sooner.  If the coercion returns an object of a different\n  type for the operand whose coercion is invoked, part of the process\n  is redone using the new object.\n\n* When an in-place operator (like \'``+=``\') is used, if the left\n  operand implements ``__iop__()``, it is invoked without any\n  coercion.  When the operation falls back to ``__op__()`` and/or\n  ``__rop__()``, the normal coercion rules apply.\n\n* In ``x + y``, if *x* is a sequence that implements sequence\n  concatenation, sequence concatenation is invoked.\n\n* In ``x * y``, if one operator is a sequence that implements sequence\n  repetition, and the other is an integer (``int`` or ``long``),\n  sequence repetition is invoked.\n\n* Rich comparisons (implemented by methods ``__eq__()`` and so on)\n  never use coercion.  Three-way comparison (implemented by\n  ``__cmp__()``) does use coercion under the same conditions as other\n  binary operations use it.\n\n* In the current implementation, the built-in numeric types ``int``,\n  ``long``, ``float``, and ``complex`` do not use coercion. All these\n  types implement a ``__coerce__()`` method, for use by the built-in\n  ``coerce()`` function.\n\n  Changed in version 2.7.\n\n\nWith Statement Context Managers\n===============================\n\nNew in version 2.5.\n\nA *context manager* is an object that defines the runtime context to\nbe established when executing a ``with`` statement. The context\nmanager handles the entry into, and the exit from, the desired runtime\ncontext for the execution of the block of code.  Context managers are\nnormally invoked using the ``with`` statement (described in section\n*The with statement*), but can also be used by directly invoking their\nmethods.\n\nTypical uses of context managers include saving and restoring various\nkinds of global state, locking and unlocking resources, closing opened\nfiles, etc.\n\nFor more information on context managers, see *Context Manager Types*.\n\nobject.__enter__(self)\n\n   Enter the runtime context related to this object. The ``with``\n   statement will bind this method\'s return value to the target(s)\n   specified in the ``as`` clause of the statement, if any.\n\nobject.__exit__(self, exc_type, exc_value, traceback)\n\n   Exit the runtime context related to this object. The parameters\n   describe the exception that caused the context to be exited. If the\n   context was exited without an exception, all three arguments will\n   be ``None``.\n\n   If an exception is supplied, and the method wishes to suppress the\n   exception (i.e., prevent it from being propagated), it should\n   return a true value. Otherwise, the exception will be processed\n   normally upon exit from this method.\n\n   Note that ``__exit__()`` methods should not reraise the passed-in\n   exception; this is the caller\'s responsibility.\n\nSee also:\n\n   **PEP 0343** - The "with" statement\n      The specification, background, and examples for the Python\n      ``with`` statement.\n\n\nSpecial method lookup for old-style classes\n===========================================\n\nFor old-style classes, special methods are always looked up in exactly\nthe same way as any other method or attribute. This is the case\nregardless of whether the method is being looked up explicitly as in\n``x.__getitem__(i)`` or implicitly as in ``x[i]``.\n\nThis behaviour means that special methods may exhibit different\nbehaviour for different instances of a single old-style class if the\nappropriate special attributes are set differently:\n\n   >>> class C:\n   ...     pass\n   ...\n   >>> c1 = C()\n   >>> c2 = C()\n   >>> c1.__len__ = lambda: 5\n   >>> c2.__len__ = lambda: 9\n   >>> len(c1)\n   5\n   >>> len(c2)\n   9\n\n\nSpecial method lookup for new-style classes\n===========================================\n\nFor new-style classes, implicit invocations of special methods are\nonly guaranteed to work correctly if defined on an object\'s type, not\nin the object\'s instance dictionary.  That behaviour is the reason why\nthe following code raises an exception (unlike the equivalent example\nwith old-style classes):\n\n   >>> class C(object):\n   ...     pass\n   ...\n   >>> c = C()\n   >>> c.__len__ = lambda: 5\n   >>> len(c)\n   Traceback (most recent call last):\n     File "", line 1, in \n   TypeError: object of type \'C\' has no len()\n\nThe rationale behind this behaviour lies with a number of special\nmethods such as ``__hash__()`` and ``__repr__()`` that are implemented\nby all objects, including type objects. If the implicit lookup of\nthese methods used the conventional lookup process, they would fail\nwhen invoked on the type object itself:\n\n   >>> 1 .__hash__() == hash(1)\n   True\n   >>> int.__hash__() == hash(int)\n   Traceback (most recent call last):\n     File "", line 1, in \n   TypeError: descriptor \'__hash__\' of \'int\' object needs an argument\n\nIncorrectly attempting to invoke an unbound method of a class in this\nway is sometimes referred to as \'metaclass confusion\', and is avoided\nby bypassing the instance when looking up special methods:\n\n   >>> type(1).__hash__(1) == hash(1)\n   True\n   >>> type(int).__hash__(int) == hash(int)\n   True\n\nIn addition to bypassing any instance attributes in the interest of\ncorrectness, implicit special method lookup generally also bypasses\nthe ``__getattribute__()`` method even of the object\'s metaclass:\n\n   >>> class Meta(type):\n   ...    def __getattribute__(*args):\n   ...       print "Metaclass getattribute invoked"\n   ...       return type.__getattribute__(*args)\n   ...\n   >>> class C(object):\n   ...     __metaclass__ = Meta\n   ...     def __len__(self):\n   ...         return 10\n   ...     def __getattribute__(*args):\n   ...         print "Class getattribute invoked"\n   ...         return object.__getattribute__(*args)\n   ...\n   >>> c = C()\n   >>> c.__len__()                 # Explicit lookup via instance\n   Class getattribute invoked\n   10\n   >>> type(c).__len__(c)          # Explicit lookup via type\n   Metaclass getattribute invoked\n   10\n   >>> len(c)                      # Implicit lookup\n   10\n\nBypassing the ``__getattribute__()`` machinery in this fashion\nprovides significant scope for speed optimisations within the\ninterpreter, at the cost of some flexibility in the handling of\nspecial methods (the special method *must* be set on the class object\nitself in order to be consistently invoked by the interpreter).\n\n-[ Footnotes ]-\n\n[1] It *is* possible in some cases to change an object\'s type, under\n    certain controlled conditions. It generally isn\'t a good idea\n    though, since it can lead to some very strange behaviour if it is\n    handled incorrectly.\n\n[2] For operands of the same type, it is assumed that if the non-\n    reflected method (such as ``__add__()``) fails the operation is\n    not supported, which is why the reflected method is not called.\n',
  'string-conversions': u'\nString conversions\n******************\n\nA string conversion is an expression list enclosed in reverse (a.k.a.\nbackward) quotes:\n\n   string_conversion ::= "\'" expression_list "\'"\n\nA string conversion evaluates the contained expression list and\nconverts the resulting object into a string according to rules\nspecific to its type.\n\nIf the object is a string, a number, ``None``, or a tuple, list or\ndictionary containing only objects whose type is one of these, the\nresulting string is a valid Python expression which can be passed to\nthe built-in function ``eval()`` to yield an expression with the same\nvalue (or an approximation, if floating point numbers are involved).\n\n(In particular, converting a string adds quotes around it and converts\n"funny" characters to escape sequences that are safe to print.)\n\nRecursive objects (for example, lists or dictionaries that contain a\nreference to themselves, directly or indirectly) use ``...`` to\nindicate a recursive reference, and the result cannot be passed to\n``eval()`` to get an equal value (``SyntaxError`` will be raised\ninstead).\n\nThe built-in function ``repr()`` performs exactly the same conversion\nin its argument as enclosing it in parentheses and reverse quotes\ndoes.  The built-in function ``str()`` performs a similar but more\nuser-friendly conversion.\n',
- 'string-methods': u'\nString Methods\n**************\n\nBelow are listed the string methods which both 8-bit strings and\nUnicode objects support.\n\nIn addition, Python\'s strings support the sequence type methods\ndescribed in the *Sequence Types --- str, unicode, list, tuple,\nbuffer, xrange* section. To output formatted strings use template\nstrings or the ``%`` operator described in the *String Formatting\nOperations* section. Also, see the ``re`` module for string functions\nbased on regular expressions.\n\nstr.capitalize()\n\n   Return a copy of the string with only its first character\n   capitalized.\n\n   For 8-bit strings, this method is locale-dependent.\n\nstr.center(width[, fillchar])\n\n   Return centered in a string of length *width*. Padding is done\n   using the specified *fillchar* (default is a space).\n\n   Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.count(sub[, start[, end]])\n\n   Return the number of non-overlapping occurrences of substring *sub*\n   in the range [*start*, *end*].  Optional arguments *start* and\n   *end* are interpreted as in slice notation.\n\nstr.decode([encoding[, errors]])\n\n   Decodes the string using the codec registered for *encoding*.\n   *encoding* defaults to the default string encoding.  *errors* may\n   be given to set a different error handling scheme.  The default is\n   ``\'strict\'``, meaning that encoding errors raise ``UnicodeError``.\n   Other possible values are ``\'ignore\'``, ``\'replace\'`` and any other\n   name registered via ``codecs.register_error()``, see section *Codec\n   Base Classes*.\n\n   New in version 2.2.\n\n   Changed in version 2.3: Support for other error handling schemes\n   added.\n\n   Changed in version 2.7: Support for keyword arguments added.\n\nstr.encode([encoding[, errors]])\n\n   Return an encoded version of the string.  Default encoding is the\n   current default string encoding.  *errors* may be given to set a\n   different error handling scheme.  The default for *errors* is\n   ``\'strict\'``, meaning that encoding errors raise a\n   ``UnicodeError``.  Other possible values are ``\'ignore\'``,\n   ``\'replace\'``, ``\'xmlcharrefreplace\'``, ``\'backslashreplace\'`` and\n   any other name registered via ``codecs.register_error()``, see\n   section *Codec Base Classes*. For a list of possible encodings, see\n   section *Standard Encodings*.\n\n   New in version 2.0.\n\n   Changed in version 2.3: Support for ``\'xmlcharrefreplace\'`` and\n   ``\'backslashreplace\'`` and other error handling schemes added.\n\n   Changed in version 2.7: Support for keyword arguments added.\n\nstr.endswith(suffix[, start[, end]])\n\n   Return ``True`` if the string ends with the specified *suffix*,\n   otherwise return ``False``.  *suffix* can also be a tuple of\n   suffixes to look for.  With optional *start*, test beginning at\n   that position.  With optional *end*, stop comparing at that\n   position.\n\n   Changed in version 2.5: Accept tuples as *suffix*.\n\nstr.expandtabs([tabsize])\n\n   Return a copy of the string where all tab characters are replaced\n   by one or more spaces, depending on the current column and the\n   given tab size.  The column number is reset to zero after each\n   newline occurring in the string. If *tabsize* is not given, a tab\n   size of ``8`` characters is assumed.  This doesn\'t understand other\n   non-printing characters or escape sequences.\n\nstr.find(sub[, start[, end]])\n\n   Return the lowest index in the string where substring *sub* is\n   found, such that *sub* is contained in the slice ``s[start:end]``.\n   Optional arguments *start* and *end* are interpreted as in slice\n   notation.  Return ``-1`` if *sub* is not found.\n\nstr.format(*args, **kwargs)\n\n   Perform a string formatting operation.  The string on which this\n   method is called can contain literal text or replacement fields\n   delimited by braces ``{}``.  Each replacement field contains either\n   the numeric index of a positional argument, or the name of a\n   keyword argument.  Returns a copy of the string where each\n   replacement field is replaced with the string value of the\n   corresponding argument.\n\n   >>> "The sum of 1 + 2 is {0}".format(1+2)\n   \'The sum of 1 + 2 is 3\'\n\n   See *Format String Syntax* for a description of the various\n   formatting options that can be specified in format strings.\n\n   This method of string formatting is the new standard in Python 3.0,\n   and should be preferred to the ``%`` formatting described in\n   *String Formatting Operations* in new code.\n\n   New in version 2.6.\n\nstr.index(sub[, start[, end]])\n\n   Like ``find()``, but raise ``ValueError`` when the substring is not\n   found.\n\nstr.isalnum()\n\n   Return true if all characters in the string are alphanumeric and\n   there is at least one character, false otherwise.\n\n   For 8-bit strings, this method is locale-dependent.\n\nstr.isalpha()\n\n   Return true if all characters in the string are alphabetic and\n   there is at least one character, false otherwise.\n\n   For 8-bit strings, this method is locale-dependent.\n\nstr.isdigit()\n\n   Return true if all characters in the string are digits and there is\n   at least one character, false otherwise.\n\n   For 8-bit strings, this method is locale-dependent.\n\nstr.islower()\n\n   Return true if all cased characters in the string are lowercase and\n   there is at least one cased character, false otherwise.\n\n   For 8-bit strings, this method is locale-dependent.\n\nstr.isspace()\n\n   Return true if there are only whitespace characters in the string\n   and there is at least one character, false otherwise.\n\n   For 8-bit strings, this method is locale-dependent.\n\nstr.istitle()\n\n   Return true if the string is a titlecased string and there is at\n   least one character, for example uppercase characters may only\n   follow uncased characters and lowercase characters only cased ones.\n   Return false otherwise.\n\n   For 8-bit strings, this method is locale-dependent.\n\nstr.isupper()\n\n   Return true if all cased characters in the string are uppercase and\n   there is at least one cased character, false otherwise.\n\n   For 8-bit strings, this method is locale-dependent.\n\nstr.join(iterable)\n\n   Return a string which is the concatenation of the strings in the\n   *iterable* *iterable*.  The separator between elements is the\n   string providing this method.\n\nstr.ljust(width[, fillchar])\n\n   Return the string left justified in a string of length *width*.\n   Padding is done using the specified *fillchar* (default is a\n   space).  The original string is returned if *width* is less than\n   ``len(s)``.\n\n   Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.lower()\n\n   Return a copy of the string converted to lowercase.\n\n   For 8-bit strings, this method is locale-dependent.\n\nstr.lstrip([chars])\n\n   Return a copy of the string with leading characters removed.  The\n   *chars* argument is a string specifying the set of characters to be\n   removed.  If omitted or ``None``, the *chars* argument defaults to\n   removing whitespace.  The *chars* argument is not a prefix; rather,\n   all combinations of its values are stripped:\n\n   >>> \'   spacious   \'.lstrip()\n   \'spacious   \'\n   >>> \'www.example.com\'.lstrip(\'cmowz.\')\n   \'example.com\'\n\n   Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.partition(sep)\n\n   Split the string at the first occurrence of *sep*, and return a\n   3-tuple containing the part before the separator, the separator\n   itself, and the part after the separator.  If the separator is not\n   found, return a 3-tuple containing the string itself, followed by\n   two empty strings.\n\n   New in version 2.5.\n\nstr.replace(old, new[, count])\n\n   Return a copy of the string with all occurrences of substring *old*\n   replaced by *new*.  If the optional argument *count* is given, only\n   the first *count* occurrences are replaced.\n\nstr.rfind(sub[, start[, end]])\n\n   Return the highest index in the string where substring *sub* is\n   found, such that *sub* is contained within ``s[start:end]``.\n   Optional arguments *start* and *end* are interpreted as in slice\n   notation.  Return ``-1`` on failure.\n\nstr.rindex(sub[, start[, end]])\n\n   Like ``rfind()`` but raises ``ValueError`` when the substring *sub*\n   is not found.\n\nstr.rjust(width[, fillchar])\n\n   Return the string right justified in a string of length *width*.\n   Padding is done using the specified *fillchar* (default is a\n   space). The original string is returned if *width* is less than\n   ``len(s)``.\n\n   Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.rpartition(sep)\n\n   Split the string at the last occurrence of *sep*, and return a\n   3-tuple containing the part before the separator, the separator\n   itself, and the part after the separator.  If the separator is not\n   found, return a 3-tuple containing two empty strings, followed by\n   the string itself.\n\n   New in version 2.5.\n\nstr.rsplit([sep[, maxsplit]])\n\n   Return a list of the words in the string, using *sep* as the\n   delimiter string. If *maxsplit* is given, at most *maxsplit* splits\n   are done, the *rightmost* ones.  If *sep* is not specified or\n   ``None``, any whitespace string is a separator.  Except for\n   splitting from the right, ``rsplit()`` behaves like ``split()``\n   which is described in detail below.\n\n   New in version 2.4.\n\nstr.rstrip([chars])\n\n   Return a copy of the string with trailing characters removed.  The\n   *chars* argument is a string specifying the set of characters to be\n   removed.  If omitted or ``None``, the *chars* argument defaults to\n   removing whitespace.  The *chars* argument is not a suffix; rather,\n   all combinations of its values are stripped:\n\n   >>> \'   spacious   \'.rstrip()\n   \'   spacious\'\n   >>> \'mississippi\'.rstrip(\'ipz\')\n   \'mississ\'\n\n   Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.split([sep[, maxsplit]])\n\n   Return a list of the words in the string, using *sep* as the\n   delimiter string.  If *maxsplit* is given, at most *maxsplit*\n   splits are done (thus, the list will have at most ``maxsplit+1``\n   elements).  If *maxsplit* is not specified, then there is no limit\n   on the number of splits (all possible splits are made).\n\n   If *sep* is given, consecutive delimiters are not grouped together\n   and are deemed to delimit empty strings (for example,\n   ``\'1,,2\'.split(\',\')`` returns ``[\'1\', \'\', \'2\']``).  The *sep*\n   argument may consist of multiple characters (for example,\n   ``\'1<>2<>3\'.split(\'<>\')`` returns ``[\'1\', \'2\', \'3\']``). Splitting\n   an empty string with a specified separator returns ``[\'\']``.\n\n   If *sep* is not specified or is ``None``, a different splitting\n   algorithm is applied: runs of consecutive whitespace are regarded\n   as a single separator, and the result will contain no empty strings\n   at the start or end if the string has leading or trailing\n   whitespace.  Consequently, splitting an empty string or a string\n   consisting of just whitespace with a ``None`` separator returns\n   ``[]``.\n\n   For example, ``\' 1  2   3  \'.split()`` returns ``[\'1\', \'2\', \'3\']``,\n   and ``\'  1  2   3  \'.split(None, 1)`` returns ``[\'1\', \'2   3  \']``.\n\nstr.splitlines([keepends])\n\n   Return a list of the lines in the string, breaking at line\n   boundaries.  Line breaks are not included in the resulting list\n   unless *keepends* is given and true.\n\nstr.startswith(prefix[, start[, end]])\n\n   Return ``True`` if string starts with the *prefix*, otherwise\n   return ``False``. *prefix* can also be a tuple of prefixes to look\n   for.  With optional *start*, test string beginning at that\n   position.  With optional *end*, stop comparing string at that\n   position.\n\n   Changed in version 2.5: Accept tuples as *prefix*.\n\nstr.strip([chars])\n\n   Return a copy of the string with the leading and trailing\n   characters removed. The *chars* argument is a string specifying the\n   set of characters to be removed. If omitted or ``None``, the\n   *chars* argument defaults to removing whitespace. The *chars*\n   argument is not a prefix or suffix; rather, all combinations of its\n   values are stripped:\n\n   >>> \'   spacious   \'.strip()\n   \'spacious\'\n   >>> \'www.example.com\'.strip(\'cmowz.\')\n   \'example\'\n\n   Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.swapcase()\n\n   Return a copy of the string with uppercase characters converted to\n   lowercase and vice versa.\n\n   For 8-bit strings, this method is locale-dependent.\n\nstr.title()\n\n   Return a titlecased version of the string where words start with an\n   uppercase character and the remaining characters are lowercase.\n\n   The algorithm uses a simple language-independent definition of a\n   word as groups of consecutive letters.  The definition works in\n   many contexts but it means that apostrophes in contractions and\n   possessives form word boundaries, which may not be the desired\n   result:\n\n      >>> "they\'re bill\'s friends from the UK".title()\n      "They\'Re Bill\'S Friends From The Uk"\n\n   A workaround for apostrophes can be constructed using regular\n   expressions:\n\n      >>> import re\n      >>> def titlecase(s):\n              return re.sub(r"[A-Za-z]+(\'[A-Za-z]+)?",\n                            lambda mo: mo.group(0)[0].upper() +\n                                       mo.group(0)[1:].lower(),\n                            s)\n\n      >>> titlecase("they\'re bill\'s friends.")\n      "They\'re Bill\'s Friends."\n\n   For 8-bit strings, this method is locale-dependent.\n\nstr.translate(table[, deletechars])\n\n   Return a copy of the string where all characters occurring in the\n   optional argument *deletechars* are removed, and the remaining\n   characters have been mapped through the given translation table,\n   which must be a string of length 256.\n\n   You can use the ``maketrans()`` helper function in the ``string``\n   module to create a translation table. For string objects, set the\n   *table* argument to ``None`` for translations that only delete\n   characters:\n\n   >>> \'read this short text\'.translate(None, \'aeiou\')\n   \'rd ths shrt txt\'\n\n   New in version 2.6: Support for a ``None`` *table* argument.\n\n   For Unicode objects, the ``translate()`` method does not accept the\n   optional *deletechars* argument.  Instead, it returns a copy of the\n   *s* where all characters have been mapped through the given\n   translation table which must be a mapping of Unicode ordinals to\n   Unicode ordinals, Unicode strings or ``None``. Unmapped characters\n   are left untouched. Characters mapped to ``None`` are deleted.\n   Note, a more flexible approach is to create a custom character\n   mapping codec using the ``codecs`` module (see ``encodings.cp1251``\n   for an example).\n\nstr.upper()\n\n   Return a copy of the string converted to uppercase.\n\n   For 8-bit strings, this method is locale-dependent.\n\nstr.zfill(width)\n\n   Return the numeric string left filled with zeros in a string of\n   length *width*.  A sign prefix is handled correctly.  The original\n   string is returned if *width* is less than ``len(s)``.\n\n   New in version 2.2.2.\n\nThe following methods are present only on unicode objects:\n\nunicode.isnumeric()\n\n   Return ``True`` if there are only numeric characters in S,\n   ``False`` otherwise. Numeric characters include digit characters,\n   and all characters that have the Unicode numeric value property,\n   e.g. U+2155, VULGAR FRACTION ONE FIFTH.\n\nunicode.isdecimal()\n\n   Return ``True`` if there are only decimal characters in S,\n   ``False`` otherwise. Decimal characters include digit characters,\n   and all characters that that can be used to form decimal-radix\n   numbers, e.g. U+0660, ARABIC-INDIC DIGIT ZERO.\n',
- 'strings': u'\nString literals\n***************\n\nString literals are described by the following lexical definitions:\n\n   stringliteral   ::= [stringprefix](shortstring | longstring)\n   stringprefix    ::= "r" | "u" | "ur" | "R" | "U" | "UR" | "Ur" | "uR"\n   shortstring     ::= "\'" shortstringitem* "\'" | \'"\' shortstringitem* \'"\'\n   longstring      ::= "\'\'\'" longstringitem* "\'\'\'"\n                  | \'"""\' longstringitem* \'"""\'\n   shortstringitem ::= shortstringchar | escapeseq\n   longstringitem  ::= longstringchar | escapeseq\n   shortstringchar ::= \n   longstringchar  ::= \n   escapeseq       ::= "\\" \n\nOne syntactic restriction not indicated by these productions is that\nwhitespace is not allowed between the **stringprefix** and the rest of\nthe string literal. The source character set is defined by the\nencoding declaration; it is ASCII if no encoding declaration is given\nin the source file; see section *Encoding declarations*.\n\nIn plain English: String literals can be enclosed in matching single\nquotes (``\'``) or double quotes (``"``).  They can also be enclosed in\nmatching groups of three single or double quotes (these are generally\nreferred to as *triple-quoted strings*).  The backslash (``\\``)\ncharacter is used to escape characters that otherwise have a special\nmeaning, such as newline, backslash itself, or the quote character.\nString literals may optionally be prefixed with a letter ``\'r\'`` or\n``\'R\'``; such strings are called *raw strings* and use different rules\nfor interpreting backslash escape sequences.  A prefix of ``\'u\'`` or\n``\'U\'`` makes the string a Unicode string.  Unicode strings use the\nUnicode character set as defined by the Unicode Consortium and ISO\n10646.  Some additional escape sequences, described below, are\navailable in Unicode strings. The two prefix characters may be\ncombined; in this case, ``\'u\'`` must appear before ``\'r\'``.\n\nIn triple-quoted strings, unescaped newlines and quotes are allowed\n(and are retained), except that three unescaped quotes in a row\nterminate the string.  (A "quote" is the character used to open the\nstring, i.e. either ``\'`` or ``"``.)\n\nUnless an ``\'r\'`` or ``\'R\'`` prefix is present, escape sequences in\nstrings are interpreted according to rules similar to those used by\nStandard C.  The recognized escape sequences are:\n\n+-------------------+-----------------------------------+---------+\n| Escape Sequence   | Meaning                           | Notes   |\n+===================+===================================+=========+\n| ``\\newline``      | Ignored                           |         |\n+-------------------+-----------------------------------+---------+\n| ``\\\\``            | Backslash (``\\``)                 |         |\n+-------------------+-----------------------------------+---------+\n| ``\\\'``            | Single quote (``\'``)              |         |\n+-------------------+-----------------------------------+---------+\n| ``\\"``            | Double quote (``"``)              |         |\n+-------------------+-----------------------------------+---------+\n| ``\\a``            | ASCII Bell (BEL)                  |         |\n+-------------------+-----------------------------------+---------+\n| ``\\b``            | ASCII Backspace (BS)              |         |\n+-------------------+-----------------------------------+---------+\n| ``\\f``            | ASCII Formfeed (FF)               |         |\n+-------------------+-----------------------------------+---------+\n| ``\\n``            | ASCII Linefeed (LF)               |         |\n+-------------------+-----------------------------------+---------+\n| ``\\N{name}``      | Character named *name* in the     |         |\n|                   | Unicode database (Unicode only)   |         |\n+-------------------+-----------------------------------+---------+\n| ``\\r``            | ASCII Carriage Return (CR)        |         |\n+-------------------+-----------------------------------+---------+\n| ``\\t``            | ASCII Horizontal Tab (TAB)        |         |\n+-------------------+-----------------------------------+---------+\n| ``\\uxxxx``        | Character with 16-bit hex value   | (1)     |\n|                   | *xxxx* (Unicode only)             |         |\n+-------------------+-----------------------------------+---------+\n| ``\\Uxxxxxxxx``    | Character with 32-bit hex value   | (2)     |\n|                   | *xxxxxxxx* (Unicode only)         |         |\n+-------------------+-----------------------------------+---------+\n| ``\\v``            | ASCII Vertical Tab (VT)           |         |\n+-------------------+-----------------------------------+---------+\n| ``\\ooo``          | Character with octal value *ooo*  | (3,5)   |\n+-------------------+-----------------------------------+---------+\n| ``\\xhh``          | Character with hex value *hh*     | (4,5)   |\n+-------------------+-----------------------------------+---------+\n\nNotes:\n\n1. Individual code units which form parts of a surrogate pair can be\n   encoded using this escape sequence.\n\n2. Any Unicode character can be encoded this way, but characters\n   outside the Basic Multilingual Plane (BMP) will be encoded using a\n   surrogate pair if Python is compiled to use 16-bit code units (the\n   default).  Individual code units which form parts of a surrogate\n   pair can be encoded using this escape sequence.\n\n3. As in Standard C, up to three octal digits are accepted.\n\n4. Unlike in Standard C, exactly two hex digits are required.\n\n5. In a string literal, hexadecimal and octal escapes denote the byte\n   with the given value; it is not necessary that the byte encodes a\n   character in the source character set. In a Unicode literal, these\n   escapes denote a Unicode character with the given value.\n\nUnlike Standard C, all unrecognized escape sequences are left in the\nstring unchanged, i.e., *the backslash is left in the string*.  (This\nbehavior is useful when debugging: if an escape sequence is mistyped,\nthe resulting output is more easily recognized as broken.)  It is also\nimportant to note that the escape sequences marked as "(Unicode only)"\nin the table above fall into the category of unrecognized escapes for\nnon-Unicode string literals.\n\nWhen an ``\'r\'`` or ``\'R\'`` prefix is present, a character following a\nbackslash is included in the string without change, and *all\nbackslashes are left in the string*.  For example, the string literal\n``r"\\n"`` consists of two characters: a backslash and a lowercase\n``\'n\'``.  String quotes can be escaped with a backslash, but the\nbackslash remains in the string; for example, ``r"\\""`` is a valid\nstring literal consisting of two characters: a backslash and a double\nquote; ``r"\\"`` is not a valid string literal (even a raw string\ncannot end in an odd number of backslashes).  Specifically, *a raw\nstring cannot end in a single backslash* (since the backslash would\nescape the following quote character).  Note also that a single\nbackslash followed by a newline is interpreted as those two characters\nas part of the string, *not* as a line continuation.\n\nWhen an ``\'r\'`` or ``\'R\'`` prefix is used in conjunction with a\n``\'u\'`` or ``\'U\'`` prefix, then the ``\\uXXXX`` and ``\\UXXXXXXXX``\nescape sequences are processed while  *all other backslashes are left\nin the string*. For example, the string literal ``ur"\\u0062\\n"``\nconsists of three Unicode characters: \'LATIN SMALL LETTER B\', \'REVERSE\nSOLIDUS\', and \'LATIN SMALL LETTER N\'. Backslashes can be escaped with\na preceding backslash; however, both remain in the string.  As a\nresult, ``\\uXXXX`` escape sequences are only recognized when there are\nan odd number of backslashes.\n',
+ 'string-methods': u'\nString Methods\n**************\n\nBelow are listed the string methods which both 8-bit strings and\nUnicode objects support.  Some of them are also available on\n``bytearray`` objects.\n\nIn addition, Python\'s strings support the sequence type methods\ndescribed in the *Sequence Types --- str, unicode, list, tuple,\nbytearray, buffer, xrange* section. To output formatted strings use\ntemplate strings or the ``%`` operator described in the *String\nFormatting Operations* section. Also, see the ``re`` module for string\nfunctions based on regular expressions.\n\nstr.capitalize()\n\n   Return a copy of the string with its first character capitalized\n   and the rest lowercased.\n\n   For 8-bit strings, this method is locale-dependent.\n\nstr.center(width[, fillchar])\n\n   Return centered in a string of length *width*. Padding is done\n   using the specified *fillchar* (default is a space).\n\n   Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.count(sub[, start[, end]])\n\n   Return the number of non-overlapping occurrences of substring *sub*\n   in the range [*start*, *end*].  Optional arguments *start* and\n   *end* are interpreted as in slice notation.\n\nstr.decode([encoding[, errors]])\n\n   Decodes the string using the codec registered for *encoding*.\n   *encoding* defaults to the default string encoding.  *errors* may\n   be given to set a different error handling scheme.  The default is\n   ``\'strict\'``, meaning that encoding errors raise ``UnicodeError``.\n   Other possible values are ``\'ignore\'``, ``\'replace\'`` and any other\n   name registered via ``codecs.register_error()``, see section *Codec\n   Base Classes*.\n\n   New in version 2.2.\n\n   Changed in version 2.3: Support for other error handling schemes\n   added.\n\n   Changed in version 2.7: Support for keyword arguments added.\n\nstr.encode([encoding[, errors]])\n\n   Return an encoded version of the string.  Default encoding is the\n   current default string encoding.  *errors* may be given to set a\n   different error handling scheme.  The default for *errors* is\n   ``\'strict\'``, meaning that encoding errors raise a\n   ``UnicodeError``.  Other possible values are ``\'ignore\'``,\n   ``\'replace\'``, ``\'xmlcharrefreplace\'``, ``\'backslashreplace\'`` and\n   any other name registered via ``codecs.register_error()``, see\n   section *Codec Base Classes*. For a list of possible encodings, see\n   section *Standard Encodings*.\n\n   New in version 2.0.\n\n   Changed in version 2.3: Support for ``\'xmlcharrefreplace\'`` and\n   ``\'backslashreplace\'`` and other error handling schemes added.\n\n   Changed in version 2.7: Support for keyword arguments added.\n\nstr.endswith(suffix[, start[, end]])\n\n   Return ``True`` if the string ends with the specified *suffix*,\n   otherwise return ``False``.  *suffix* can also be a tuple of\n   suffixes to look for.  With optional *start*, test beginning at\n   that position.  With optional *end*, stop comparing at that\n   position.\n\n   Changed in version 2.5: Accept tuples as *suffix*.\n\nstr.expandtabs([tabsize])\n\n   Return a copy of the string where all tab characters are replaced\n   by one or more spaces, depending on the current column and the\n   given tab size.  The column number is reset to zero after each\n   newline occurring in the string. If *tabsize* is not given, a tab\n   size of ``8`` characters is assumed.  This doesn\'t understand other\n   non-printing characters or escape sequences.\n\nstr.find(sub[, start[, end]])\n\n   Return the lowest index in the string where substring *sub* is\n   found, such that *sub* is contained in the slice ``s[start:end]``.\n   Optional arguments *start* and *end* are interpreted as in slice\n   notation.  Return ``-1`` if *sub* is not found.\n\n   Note: The ``find()`` method should be used only if you need to know the\n     position of *sub*.  To check if *sub* is a substring or not, use\n     the ``in`` operator:\n\n        >>> \'Py\' in \'Python\'\n        True\n\nstr.format(*args, **kwargs)\n\n   Perform a string formatting operation.  The string on which this\n   method is called can contain literal text or replacement fields\n   delimited by braces ``{}``.  Each replacement field contains either\n   the numeric index of a positional argument, or the name of a\n   keyword argument.  Returns a copy of the string where each\n   replacement field is replaced with the string value of the\n   corresponding argument.\n\n   >>> "The sum of 1 + 2 is {0}".format(1+2)\n   \'The sum of 1 + 2 is 3\'\n\n   See *Format String Syntax* for a description of the various\n   formatting options that can be specified in format strings.\n\n   This method of string formatting is the new standard in Python 3.0,\n   and should be preferred to the ``%`` formatting described in\n   *String Formatting Operations* in new code.\n\n   New in version 2.6.\n\nstr.index(sub[, start[, end]])\n\n   Like ``find()``, but raise ``ValueError`` when the substring is not\n   found.\n\nstr.isalnum()\n\n   Return true if all characters in the string are alphanumeric and\n   there is at least one character, false otherwise.\n\n   For 8-bit strings, this method is locale-dependent.\n\nstr.isalpha()\n\n   Return true if all characters in the string are alphabetic and\n   there is at least one character, false otherwise.\n\n   For 8-bit strings, this method is locale-dependent.\n\nstr.isdigit()\n\n   Return true if all characters in the string are digits and there is\n   at least one character, false otherwise.\n\n   For 8-bit strings, this method is locale-dependent.\n\nstr.islower()\n\n   Return true if all cased characters in the string are lowercase and\n   there is at least one cased character, false otherwise.\n\n   For 8-bit strings, this method is locale-dependent.\n\nstr.isspace()\n\n   Return true if there are only whitespace characters in the string\n   and there is at least one character, false otherwise.\n\n   For 8-bit strings, this method is locale-dependent.\n\nstr.istitle()\n\n   Return true if the string is a titlecased string and there is at\n   least one character, for example uppercase characters may only\n   follow uncased characters and lowercase characters only cased ones.\n   Return false otherwise.\n\n   For 8-bit strings, this method is locale-dependent.\n\nstr.isupper()\n\n   Return true if all cased characters in the string are uppercase and\n   there is at least one cased character, false otherwise.\n\n   For 8-bit strings, this method is locale-dependent.\n\nstr.join(iterable)\n\n   Return a string which is the concatenation of the strings in the\n   *iterable* *iterable*.  The separator between elements is the\n   string providing this method.\n\nstr.ljust(width[, fillchar])\n\n   Return the string left justified in a string of length *width*.\n   Padding is done using the specified *fillchar* (default is a\n   space).  The original string is returned if *width* is less than\n   ``len(s)``.\n\n   Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.lower()\n\n   Return a copy of the string converted to lowercase.\n\n   For 8-bit strings, this method is locale-dependent.\n\nstr.lstrip([chars])\n\n   Return a copy of the string with leading characters removed.  The\n   *chars* argument is a string specifying the set of characters to be\n   removed.  If omitted or ``None``, the *chars* argument defaults to\n   removing whitespace.  The *chars* argument is not a prefix; rather,\n   all combinations of its values are stripped:\n\n   >>> \'   spacious   \'.lstrip()\n   \'spacious   \'\n   >>> \'www.example.com\'.lstrip(\'cmowz.\')\n   \'example.com\'\n\n   Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.partition(sep)\n\n   Split the string at the first occurrence of *sep*, and return a\n   3-tuple containing the part before the separator, the separator\n   itself, and the part after the separator.  If the separator is not\n   found, return a 3-tuple containing the string itself, followed by\n   two empty strings.\n\n   New in version 2.5.\n\nstr.replace(old, new[, count])\n\n   Return a copy of the string with all occurrences of substring *old*\n   replaced by *new*.  If the optional argument *count* is given, only\n   the first *count* occurrences are replaced.\n\nstr.rfind(sub[, start[, end]])\n\n   Return the highest index in the string where substring *sub* is\n   found, such that *sub* is contained within ``s[start:end]``.\n   Optional arguments *start* and *end* are interpreted as in slice\n   notation.  Return ``-1`` on failure.\n\nstr.rindex(sub[, start[, end]])\n\n   Like ``rfind()`` but raises ``ValueError`` when the substring *sub*\n   is not found.\n\nstr.rjust(width[, fillchar])\n\n   Return the string right justified in a string of length *width*.\n   Padding is done using the specified *fillchar* (default is a\n   space). The original string is returned if *width* is less than\n   ``len(s)``.\n\n   Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.rpartition(sep)\n\n   Split the string at the last occurrence of *sep*, and return a\n   3-tuple containing the part before the separator, the separator\n   itself, and the part after the separator.  If the separator is not\n   found, return a 3-tuple containing two empty strings, followed by\n   the string itself.\n\n   New in version 2.5.\n\nstr.rsplit([sep[, maxsplit]])\n\n   Return a list of the words in the string, using *sep* as the\n   delimiter string. If *maxsplit* is given, at most *maxsplit* splits\n   are done, the *rightmost* ones.  If *sep* is not specified or\n   ``None``, any whitespace string is a separator.  Except for\n   splitting from the right, ``rsplit()`` behaves like ``split()``\n   which is described in detail below.\n\n   New in version 2.4.\n\nstr.rstrip([chars])\n\n   Return a copy of the string with trailing characters removed.  The\n   *chars* argument is a string specifying the set of characters to be\n   removed.  If omitted or ``None``, the *chars* argument defaults to\n   removing whitespace.  The *chars* argument is not a suffix; rather,\n   all combinations of its values are stripped:\n\n   >>> \'   spacious   \'.rstrip()\n   \'   spacious\'\n   >>> \'mississippi\'.rstrip(\'ipz\')\n   \'mississ\'\n\n   Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.split([sep[, maxsplit]])\n\n   Return a list of the words in the string, using *sep* as the\n   delimiter string.  If *maxsplit* is given, at most *maxsplit*\n   splits are done (thus, the list will have at most ``maxsplit+1``\n   elements).  If *maxsplit* is not specified, then there is no limit\n   on the number of splits (all possible splits are made).\n\n   If *sep* is given, consecutive delimiters are not grouped together\n   and are deemed to delimit empty strings (for example,\n   ``\'1,,2\'.split(\',\')`` returns ``[\'1\', \'\', \'2\']``).  The *sep*\n   argument may consist of multiple characters (for example,\n   ``\'1<>2<>3\'.split(\'<>\')`` returns ``[\'1\', \'2\', \'3\']``). Splitting\n   an empty string with a specified separator returns ``[\'\']``.\n\n   If *sep* is not specified or is ``None``, a different splitting\n   algorithm is applied: runs of consecutive whitespace are regarded\n   as a single separator, and the result will contain no empty strings\n   at the start or end if the string has leading or trailing\n   whitespace.  Consequently, splitting an empty string or a string\n   consisting of just whitespace with a ``None`` separator returns\n   ``[]``.\n\n   For example, ``\' 1  2   3  \'.split()`` returns ``[\'1\', \'2\', \'3\']``,\n   and ``\'  1  2   3  \'.split(None, 1)`` returns ``[\'1\', \'2   3  \']``.\n\nstr.splitlines([keepends])\n\n   Return a list of the lines in the string, breaking at line\n   boundaries.  Line breaks are not included in the resulting list\n   unless *keepends* is given and true.\n\nstr.startswith(prefix[, start[, end]])\n\n   Return ``True`` if string starts with the *prefix*, otherwise\n   return ``False``. *prefix* can also be a tuple of prefixes to look\n   for.  With optional *start*, test string beginning at that\n   position.  With optional *end*, stop comparing string at that\n   position.\n\n   Changed in version 2.5: Accept tuples as *prefix*.\n\nstr.strip([chars])\n\n   Return a copy of the string with the leading and trailing\n   characters removed. The *chars* argument is a string specifying the\n   set of characters to be removed. If omitted or ``None``, the\n   *chars* argument defaults to removing whitespace. The *chars*\n   argument is not a prefix or suffix; rather, all combinations of its\n   values are stripped:\n\n   >>> \'   spacious   \'.strip()\n   \'spacious\'\n   >>> \'www.example.com\'.strip(\'cmowz.\')\n   \'example\'\n\n   Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.swapcase()\n\n   Return a copy of the string with uppercase characters converted to\n   lowercase and vice versa.\n\n   For 8-bit strings, this method is locale-dependent.\n\nstr.title()\n\n   Return a titlecased version of the string where words start with an\n   uppercase character and the remaining characters are lowercase.\n\n   The algorithm uses a simple language-independent definition of a\n   word as groups of consecutive letters.  The definition works in\n   many contexts but it means that apostrophes in contractions and\n   possessives form word boundaries, which may not be the desired\n   result:\n\n      >>> "they\'re bill\'s friends from the UK".title()\n      "They\'Re Bill\'S Friends From The Uk"\n\n   A workaround for apostrophes can be constructed using regular\n   expressions:\n\n      >>> import re\n      >>> def titlecase(s):\n              return re.sub(r"[A-Za-z]+(\'[A-Za-z]+)?",\n                            lambda mo: mo.group(0)[0].upper() +\n                                       mo.group(0)[1:].lower(),\n                            s)\n\n      >>> titlecase("they\'re bill\'s friends.")\n      "They\'re Bill\'s Friends."\n\n   For 8-bit strings, this method is locale-dependent.\n\nstr.translate(table[, deletechars])\n\n   Return a copy of the string where all characters occurring in the\n   optional argument *deletechars* are removed, and the remaining\n   characters have been mapped through the given translation table,\n   which must be a string of length 256.\n\n   You can use the ``maketrans()`` helper function in the ``string``\n   module to create a translation table. For string objects, set the\n   *table* argument to ``None`` for translations that only delete\n   characters:\n\n   >>> \'read this short text\'.translate(None, \'aeiou\')\n   \'rd ths shrt txt\'\n\n   New in version 2.6: Support for a ``None`` *table* argument.\n\n   For Unicode objects, the ``translate()`` method does not accept the\n   optional *deletechars* argument.  Instead, it returns a copy of the\n   *s* where all characters have been mapped through the given\n   translation table which must be a mapping of Unicode ordinals to\n   Unicode ordinals, Unicode strings or ``None``. Unmapped characters\n   are left untouched. Characters mapped to ``None`` are deleted.\n   Note, a more flexible approach is to create a custom character\n   mapping codec using the ``codecs`` module (see ``encodings.cp1251``\n   for an example).\n\nstr.upper()\n\n   Return a copy of the string converted to uppercase.\n\n   For 8-bit strings, this method is locale-dependent.\n\nstr.zfill(width)\n\n   Return the numeric string left filled with zeros in a string of\n   length *width*.  A sign prefix is handled correctly.  The original\n   string is returned if *width* is less than ``len(s)``.\n\n   New in version 2.2.2.\n\nThe following methods are present only on unicode objects:\n\nunicode.isnumeric()\n\n   Return ``True`` if there are only numeric characters in S,\n   ``False`` otherwise. Numeric characters include digit characters,\n   and all characters that have the Unicode numeric value property,\n   e.g. U+2155, VULGAR FRACTION ONE FIFTH.\n\nunicode.isdecimal()\n\n   Return ``True`` if there are only decimal characters in S,\n   ``False`` otherwise. Decimal characters include digit characters,\n   and all characters that that can be used to form decimal-radix\n   numbers, e.g. U+0660, ARABIC-INDIC DIGIT ZERO.\n',
+ 'strings': u'\nString literals\n***************\n\nString literals are described by the following lexical definitions:\n\n   stringliteral   ::= [stringprefix](shortstring | longstring)\n   stringprefix    ::= "r" | "u" | "ur" | "R" | "U" | "UR" | "Ur" | "uR"\n                    | "b" | "B" | "br" | "Br" | "bR" | "BR"\n   shortstring     ::= "\'" shortstringitem* "\'" | \'"\' shortstringitem* \'"\'\n   longstring      ::= "\'\'\'" longstringitem* "\'\'\'"\n                  | \'"""\' longstringitem* \'"""\'\n   shortstringitem ::= shortstringchar | escapeseq\n   longstringitem  ::= longstringchar | escapeseq\n   shortstringchar ::= \n   longstringchar  ::= \n   escapeseq       ::= "\\" \n\nOne syntactic restriction not indicated by these productions is that\nwhitespace is not allowed between the **stringprefix** and the rest of\nthe string literal. The source character set is defined by the\nencoding declaration; it is ASCII if no encoding declaration is given\nin the source file; see section *Encoding declarations*.\n\nIn plain English: String literals can be enclosed in matching single\nquotes (``\'``) or double quotes (``"``).  They can also be enclosed in\nmatching groups of three single or double quotes (these are generally\nreferred to as *triple-quoted strings*).  The backslash (``\\``)\ncharacter is used to escape characters that otherwise have a special\nmeaning, such as newline, backslash itself, or the quote character.\nString literals may optionally be prefixed with a letter ``\'r\'`` or\n``\'R\'``; such strings are called *raw strings* and use different rules\nfor interpreting backslash escape sequences.  A prefix of ``\'u\'`` or\n``\'U\'`` makes the string a Unicode string.  Unicode strings use the\nUnicode character set as defined by the Unicode Consortium and ISO\n10646.  Some additional escape sequences, described below, are\navailable in Unicode strings. A prefix of ``\'b\'`` or ``\'B\'`` is\nignored in Python 2; it indicates that the literal should become a\nbytes literal in Python 3 (e.g. when code is automatically converted\nwith 2to3).  A ``\'u\'`` or ``\'b\'`` prefix may be followed by an ``\'r\'``\nprefix.\n\nIn triple-quoted strings, unescaped newlines and quotes are allowed\n(and are retained), except that three unescaped quotes in a row\nterminate the string.  (A "quote" is the character used to open the\nstring, i.e. either ``\'`` or ``"``.)\n\nUnless an ``\'r\'`` or ``\'R\'`` prefix is present, escape sequences in\nstrings are interpreted according to rules similar to those used by\nStandard C.  The recognized escape sequences are:\n\n+-------------------+-----------------------------------+---------+\n| Escape Sequence   | Meaning                           | Notes   |\n+===================+===================================+=========+\n| ``\\newline``      | Ignored                           |         |\n+-------------------+-----------------------------------+---------+\n| ``\\\\``            | Backslash (``\\``)                 |         |\n+-------------------+-----------------------------------+---------+\n| ``\\\'``            | Single quote (``\'``)              |         |\n+-------------------+-----------------------------------+---------+\n| ``\\"``            | Double quote (``"``)              |         |\n+-------------------+-----------------------------------+---------+\n| ``\\a``            | ASCII Bell (BEL)                  |         |\n+-------------------+-----------------------------------+---------+\n| ``\\b``            | ASCII Backspace (BS)              |         |\n+-------------------+-----------------------------------+---------+\n| ``\\f``            | ASCII Formfeed (FF)               |         |\n+-------------------+-----------------------------------+---------+\n| ``\\n``            | ASCII Linefeed (LF)               |         |\n+-------------------+-----------------------------------+---------+\n| ``\\N{name}``      | Character named *name* in the     |         |\n|                   | Unicode database (Unicode only)   |         |\n+-------------------+-----------------------------------+---------+\n| ``\\r``            | ASCII Carriage Return (CR)        |         |\n+-------------------+-----------------------------------+---------+\n| ``\\t``            | ASCII Horizontal Tab (TAB)        |         |\n+-------------------+-----------------------------------+---------+\n| ``\\uxxxx``        | Character with 16-bit hex value   | (1)     |\n|                   | *xxxx* (Unicode only)             |         |\n+-------------------+-----------------------------------+---------+\n| ``\\Uxxxxxxxx``    | Character with 32-bit hex value   | (2)     |\n|                   | *xxxxxxxx* (Unicode only)         |         |\n+-------------------+-----------------------------------+---------+\n| ``\\v``            | ASCII Vertical Tab (VT)           |         |\n+-------------------+-----------------------------------+---------+\n| ``\\ooo``          | Character with octal value *ooo*  | (3,5)   |\n+-------------------+-----------------------------------+---------+\n| ``\\xhh``          | Character with hex value *hh*     | (4,5)   |\n+-------------------+-----------------------------------+---------+\n\nNotes:\n\n1. Individual code units which form parts of a surrogate pair can be\n   encoded using this escape sequence.\n\n2. Any Unicode character can be encoded this way, but characters\n   outside the Basic Multilingual Plane (BMP) will be encoded using a\n   surrogate pair if Python is compiled to use 16-bit code units (the\n   default).  Individual code units which form parts of a surrogate\n   pair can be encoded using this escape sequence.\n\n3. As in Standard C, up to three octal digits are accepted.\n\n4. Unlike in Standard C, exactly two hex digits are required.\n\n5. In a string literal, hexadecimal and octal escapes denote the byte\n   with the given value; it is not necessary that the byte encodes a\n   character in the source character set. In a Unicode literal, these\n   escapes denote a Unicode character with the given value.\n\nUnlike Standard C, all unrecognized escape sequences are left in the\nstring unchanged, i.e., *the backslash is left in the string*.  (This\nbehavior is useful when debugging: if an escape sequence is mistyped,\nthe resulting output is more easily recognized as broken.)  It is also\nimportant to note that the escape sequences marked as "(Unicode only)"\nin the table above fall into the category of unrecognized escapes for\nnon-Unicode string literals.\n\nWhen an ``\'r\'`` or ``\'R\'`` prefix is present, a character following a\nbackslash is included in the string without change, and *all\nbackslashes are left in the string*.  For example, the string literal\n``r"\\n"`` consists of two characters: a backslash and a lowercase\n``\'n\'``.  String quotes can be escaped with a backslash, but the\nbackslash remains in the string; for example, ``r"\\""`` is a valid\nstring literal consisting of two characters: a backslash and a double\nquote; ``r"\\"`` is not a valid string literal (even a raw string\ncannot end in an odd number of backslashes).  Specifically, *a raw\nstring cannot end in a single backslash* (since the backslash would\nescape the following quote character).  Note also that a single\nbackslash followed by a newline is interpreted as those two characters\nas part of the string, *not* as a line continuation.\n\nWhen an ``\'r\'`` or ``\'R\'`` prefix is used in conjunction with a\n``\'u\'`` or ``\'U\'`` prefix, then the ``\\uXXXX`` and ``\\UXXXXXXXX``\nescape sequences are processed while  *all other backslashes are left\nin the string*. For example, the string literal ``ur"\\u0062\\n"``\nconsists of three Unicode characters: \'LATIN SMALL LETTER B\', \'REVERSE\nSOLIDUS\', and \'LATIN SMALL LETTER N\'. Backslashes can be escaped with\na preceding backslash; however, both remain in the string.  As a\nresult, ``\\uXXXX`` escape sequences are only recognized when there are\nan odd number of backslashes.\n',
  'subscriptions': u'\nSubscriptions\n*************\n\nA subscription selects an item of a sequence (string, tuple or list)\nor mapping (dictionary) object:\n\n   subscription ::= primary "[" expression_list "]"\n\nThe primary must evaluate to an object of a sequence or mapping type.\n\nIf the primary is a mapping, the expression list must evaluate to an\nobject whose value is one of the keys of the mapping, and the\nsubscription selects the value in the mapping that corresponds to that\nkey.  (The expression list is a tuple except if it has exactly one\nitem.)\n\nIf the primary is a sequence, the expression (list) must evaluate to a\nplain integer.  If this value is negative, the length of the sequence\nis added to it (so that, e.g., ``x[-1]`` selects the last item of\n``x``.)  The resulting value must be a nonnegative integer less than\nthe number of items in the sequence, and the subscription selects the\nitem whose index is that value (counting from zero).\n\nA string\'s items are characters.  A character is not a separate data\ntype but a string of exactly one character.\n',
  'truth': u"\nTruth Value Testing\n*******************\n\nAny object can be tested for truth value, for use in an ``if`` or\n``while`` condition or as operand of the Boolean operations below. The\nfollowing values are considered false:\n\n* ``None``\n\n* ``False``\n\n* zero of any numeric type, for example, ``0``, ``0L``, ``0.0``,\n  ``0j``.\n\n* any empty sequence, for example, ``''``, ``()``, ``[]``.\n\n* any empty mapping, for example, ``{}``.\n\n* instances of user-defined classes, if the class defines a\n  ``__nonzero__()`` or ``__len__()`` method, when that method returns\n  the integer zero or ``bool`` value ``False``. [1]\n\nAll other values are considered true --- so objects of many types are\nalways true.\n\nOperations and built-in functions that have a Boolean result always\nreturn ``0`` or ``False`` for false and ``1`` or ``True`` for true,\nunless otherwise stated. (Important exception: the Boolean operations\n``or`` and ``and`` always return one of their operands.)\n",
  'try': u'\nThe ``try`` statement\n*********************\n\nThe ``try`` statement specifies exception handlers and/or cleanup code\nfor a group of statements:\n\n   try_stmt  ::= try1_stmt | try2_stmt\n   try1_stmt ::= "try" ":" suite\n                 ("except" [expression [("as" | ",") target]] ":" suite)+\n                 ["else" ":" suite]\n                 ["finally" ":" suite]\n   try2_stmt ::= "try" ":" suite\n                 "finally" ":" suite\n\nChanged in version 2.5: In previous versions of Python,\n``try``...``except``...``finally`` did not work. ``try``...``except``\nhad to be nested in ``try``...``finally``.\n\nThe ``except`` clause(s) specify one or more exception handlers. When\nno exception occurs in the ``try`` clause, no exception handler is\nexecuted. When an exception occurs in the ``try`` suite, a search for\nan exception handler is started.  This search inspects the except\nclauses in turn until one is found that matches the exception.  An\nexpression-less except clause, if present, must be last; it matches\nany exception.  For an except clause with an expression, that\nexpression is evaluated, and the clause matches the exception if the\nresulting object is "compatible" with the exception.  An object is\ncompatible with an exception if it is the class or a base class of the\nexception object, a tuple containing an item compatible with the\nexception, or, in the (deprecated) case of string exceptions, is the\nraised string itself (note that the object identities must match, i.e.\nit must be the same string object, not just a string with the same\nvalue).\n\nIf no except clause matches the exception, the search for an exception\nhandler continues in the surrounding code and on the invocation stack.\n[1]\n\nIf the evaluation of an expression in the header of an except clause\nraises an exception, the original search for a handler is canceled and\na search starts for the new exception in the surrounding code and on\nthe call stack (it is treated as if the entire ``try`` statement\nraised the exception).\n\nWhen a matching except clause is found, the exception is assigned to\nthe target specified in that except clause, if present, and the except\nclause\'s suite is executed.  All except clauses must have an\nexecutable block.  When the end of this block is reached, execution\ncontinues normally after the entire try statement.  (This means that\nif two nested handlers exist for the same exception, and the exception\noccurs in the try clause of the inner handler, the outer handler will\nnot handle the exception.)\n\nBefore an except clause\'s suite is executed, details about the\nexception are assigned to three variables in the ``sys`` module:\n``sys.exc_type`` receives the object identifying the exception;\n``sys.exc_value`` receives the exception\'s parameter;\n``sys.exc_traceback`` receives a traceback object (see section *The\nstandard type hierarchy*) identifying the point in the program where\nthe exception occurred. These details are also available through the\n``sys.exc_info()`` function, which returns a tuple ``(exc_type,\nexc_value, exc_traceback)``.  Use of the corresponding variables is\ndeprecated in favor of this function, since their use is unsafe in a\nthreaded program.  As of Python 1.5, the variables are restored to\ntheir previous values (before the call) when returning from a function\nthat handled an exception.\n\nThe optional ``else`` clause is executed if and when control flows off\nthe end of the ``try`` clause. [2] Exceptions in the ``else`` clause\nare not handled by the preceding ``except`` clauses.\n\nIf ``finally`` is present, it specifies a \'cleanup\' handler.  The\n``try`` clause is executed, including any ``except`` and ``else``\nclauses.  If an exception occurs in any of the clauses and is not\nhandled, the exception is temporarily saved. The ``finally`` clause is\nexecuted.  If there is a saved exception, it is re-raised at the end\nof the ``finally`` clause. If the ``finally`` clause raises another\nexception or executes a ``return`` or ``break`` statement, the saved\nexception is lost.  The exception information is not available to the\nprogram during execution of the ``finally`` clause.\n\nWhen a ``return``, ``break`` or ``continue`` statement is executed in\nthe ``try`` suite of a ``try``...``finally`` statement, the\n``finally`` clause is also executed \'on the way out.\' A ``continue``\nstatement is illegal in the ``finally`` clause. (The reason is a\nproblem with the current implementation --- this restriction may be\nlifted in the future).\n\nAdditional information on exceptions can be found in section\n*Exceptions*, and information on using the ``raise`` statement to\ngenerate exceptions may be found in section *The raise statement*.\n',
- 'types': u'\nThe standard type hierarchy\n***************************\n\nBelow is a list of the types that are built into Python.  Extension\nmodules (written in C, Java, or other languages, depending on the\nimplementation) can define additional types.  Future versions of\nPython may add types to the type hierarchy (e.g., rational numbers,\nefficiently stored arrays of integers, etc.).\n\nSome of the type descriptions below contain a paragraph listing\n\'special attributes.\'  These are attributes that provide access to the\nimplementation and are not intended for general use.  Their definition\nmay change in the future.\n\nNone\n   This type has a single value.  There is a single object with this\n   value. This object is accessed through the built-in name ``None``.\n   It is used to signify the absence of a value in many situations,\n   e.g., it is returned from functions that don\'t explicitly return\n   anything. Its truth value is false.\n\nNotImplemented\n   This type has a single value.  There is a single object with this\n   value. This object is accessed through the built-in name\n   ``NotImplemented``. Numeric methods and rich comparison methods may\n   return this value if they do not implement the operation for the\n   operands provided.  (The interpreter will then try the reflected\n   operation, or some other fallback, depending on the operator.)  Its\n   truth value is true.\n\nEllipsis\n   This type has a single value.  There is a single object with this\n   value. This object is accessed through the built-in name\n   ``Ellipsis``. It is used to indicate the presence of the ``...``\n   syntax in a slice.  Its truth value is true.\n\n``numbers.Number``\n   These are created by numeric literals and returned as results by\n   arithmetic operators and arithmetic built-in functions.  Numeric\n   objects are immutable; once created their value never changes.\n   Python numbers are of course strongly related to mathematical\n   numbers, but subject to the limitations of numerical representation\n   in computers.\n\n   Python distinguishes between integers, floating point numbers, and\n   complex numbers:\n\n   ``numbers.Integral``\n      These represent elements from the mathematical set of integers\n      (positive and negative).\n\n      There are three types of integers:\n\n      Plain integers\n         These represent numbers in the range -2147483648 through\n         2147483647. (The range may be larger on machines with a\n         larger natural word size, but not smaller.)  When the result\n         of an operation would fall outside this range, the result is\n         normally returned as a long integer (in some cases, the\n         exception ``OverflowError`` is raised instead).  For the\n         purpose of shift and mask operations, integers are assumed to\n         have a binary, 2\'s complement notation using 32 or more bits,\n         and hiding no bits from the user (i.e., all 4294967296\n         different bit patterns correspond to different values).\n\n      Long integers\n         These represent numbers in an unlimited range, subject to\n         available (virtual) memory only.  For the purpose of shift\n         and mask operations, a binary representation is assumed, and\n         negative numbers are represented in a variant of 2\'s\n         complement which gives the illusion of an infinite string of\n         sign bits extending to the left.\n\n      Booleans\n         These represent the truth values False and True.  The two\n         objects representing the values False and True are the only\n         Boolean objects. The Boolean type is a subtype of plain\n         integers, and Boolean values behave like the values 0 and 1,\n         respectively, in almost all contexts, the exception being\n         that when converted to a string, the strings ``"False"`` or\n         ``"True"`` are returned, respectively.\n\n      The rules for integer representation are intended to give the\n      most meaningful interpretation of shift and mask operations\n      involving negative integers and the least surprises when\n      switching between the plain and long integer domains.  Any\n      operation, if it yields a result in the plain integer domain,\n      will yield the same result in the long integer domain or when\n      using mixed operands.  The switch between domains is transparent\n      to the programmer.\n\n   ``numbers.Real`` (``float``)\n      These represent machine-level double precision floating point\n      numbers. You are at the mercy of the underlying machine\n      architecture (and C or Java implementation) for the accepted\n      range and handling of overflow. Python does not support single-\n      precision floating point numbers; the savings in processor and\n      memory usage that are usually the reason for using these is\n      dwarfed by the overhead of using objects in Python, so there is\n      no reason to complicate the language with two kinds of floating\n      point numbers.\n\n   ``numbers.Complex``\n      These represent complex numbers as a pair of machine-level\n      double precision floating point numbers.  The same caveats apply\n      as for floating point numbers. The real and imaginary parts of a\n      complex number ``z`` can be retrieved through the read-only\n      attributes ``z.real`` and ``z.imag``.\n\nSequences\n   These represent finite ordered sets indexed by non-negative\n   numbers. The built-in function ``len()`` returns the number of\n   items of a sequence. When the length of a sequence is *n*, the\n   index set contains the numbers 0, 1, ..., *n*-1.  Item *i* of\n   sequence *a* is selected by ``a[i]``.\n\n   Sequences also support slicing: ``a[i:j]`` selects all items with\n   index *k* such that *i* ``<=`` *k* ``<`` *j*.  When used as an\n   expression, a slice is a sequence of the same type.  This implies\n   that the index set is renumbered so that it starts at 0.\n\n   Some sequences also support "extended slicing" with a third "step"\n   parameter: ``a[i:j:k]`` selects all items of *a* with index *x*\n   where ``x = i + n*k``, *n* ``>=`` ``0`` and *i* ``<=`` *x* ``<``\n   *j*.\n\n   Sequences are distinguished according to their mutability:\n\n   Immutable sequences\n      An object of an immutable sequence type cannot change once it is\n      created.  (If the object contains references to other objects,\n      these other objects may be mutable and may be changed; however,\n      the collection of objects directly referenced by an immutable\n      object cannot change.)\n\n      The following types are immutable sequences:\n\n      Strings\n         The items of a string are characters.  There is no separate\n         character type; a character is represented by a string of one\n         item. Characters represent (at least) 8-bit bytes.  The\n         built-in functions ``chr()`` and ``ord()`` convert between\n         characters and nonnegative integers representing the byte\n         values.  Bytes with the values 0-127 usually represent the\n         corresponding ASCII values, but the interpretation of values\n         is up to the program.  The string data type is also used to\n         represent arrays of bytes, e.g., to hold data read from a\n         file.\n\n         (On systems whose native character set is not ASCII, strings\n         may use EBCDIC in their internal representation, provided the\n         functions ``chr()`` and ``ord()`` implement a mapping between\n         ASCII and EBCDIC, and string comparison preserves the ASCII\n         order. Or perhaps someone can propose a better rule?)\n\n      Unicode\n         The items of a Unicode object are Unicode code units.  A\n         Unicode code unit is represented by a Unicode object of one\n         item and can hold either a 16-bit or 32-bit value\n         representing a Unicode ordinal (the maximum value for the\n         ordinal is given in ``sys.maxunicode``, and depends on how\n         Python is configured at compile time).  Surrogate pairs may\n         be present in the Unicode object, and will be reported as two\n         separate items.  The built-in functions ``unichr()`` and\n         ``ord()`` convert between code units and nonnegative integers\n         representing the Unicode ordinals as defined in the Unicode\n         Standard 3.0. Conversion from and to other encodings are\n         possible through the Unicode method ``encode()`` and the\n         built-in function ``unicode()``.\n\n      Tuples\n         The items of a tuple are arbitrary Python objects. Tuples of\n         two or more items are formed by comma-separated lists of\n         expressions.  A tuple of one item (a \'singleton\') can be\n         formed by affixing a comma to an expression (an expression by\n         itself does not create a tuple, since parentheses must be\n         usable for grouping of expressions).  An empty tuple can be\n         formed by an empty pair of parentheses.\n\n   Mutable sequences\n      Mutable sequences can be changed after they are created.  The\n      subscription and slicing notations can be used as the target of\n      assignment and ``del`` (delete) statements.\n\n      There are currently two intrinsic mutable sequence types:\n\n      Lists\n         The items of a list are arbitrary Python objects.  Lists are\n         formed by placing a comma-separated list of expressions in\n         square brackets. (Note that there are no special cases needed\n         to form lists of length 0 or 1.)\n\n      Byte Arrays\n         A bytearray object is a mutable array. They are created by\n         the built-in ``bytearray()`` constructor.  Aside from being\n         mutable (and hence unhashable), byte arrays otherwise provide\n         the same interface and functionality as immutable bytes\n         objects.\n\n      The extension module ``array`` provides an additional example of\n      a mutable sequence type.\n\nSet types\n   These represent unordered, finite sets of unique, immutable\n   objects. As such, they cannot be indexed by any subscript. However,\n   they can be iterated over, and the built-in function ``len()``\n   returns the number of items in a set. Common uses for sets are fast\n   membership testing, removing duplicates from a sequence, and\n   computing mathematical operations such as intersection, union,\n   difference, and symmetric difference.\n\n   For set elements, the same immutability rules apply as for\n   dictionary keys. Note that numeric types obey the normal rules for\n   numeric comparison: if two numbers compare equal (e.g., ``1`` and\n   ``1.0``), only one of them can be contained in a set.\n\n   There are currently two intrinsic set types:\n\n   Sets\n      These represent a mutable set. They are created by the built-in\n      ``set()`` constructor and can be modified afterwards by several\n      methods, such as ``add()``.\n\n   Frozen sets\n      These represent an immutable set.  They are created by the\n      built-in ``frozenset()`` constructor.  As a frozenset is\n      immutable and *hashable*, it can be used again as an element of\n      another set, or as a dictionary key.\n\nMappings\n   These represent finite sets of objects indexed by arbitrary index\n   sets. The subscript notation ``a[k]`` selects the item indexed by\n   ``k`` from the mapping ``a``; this can be used in expressions and\n   as the target of assignments or ``del`` statements. The built-in\n   function ``len()`` returns the number of items in a mapping.\n\n   There is currently a single intrinsic mapping type:\n\n   Dictionaries\n      These represent finite sets of objects indexed by nearly\n      arbitrary values.  The only types of values not acceptable as\n      keys are values containing lists or dictionaries or other\n      mutable types that are compared by value rather than by object\n      identity, the reason being that the efficient implementation of\n      dictionaries requires a key\'s hash value to remain constant.\n      Numeric types used for keys obey the normal rules for numeric\n      comparison: if two numbers compare equal (e.g., ``1`` and\n      ``1.0``) then they can be used interchangeably to index the same\n      dictionary entry.\n\n      Dictionaries are mutable; they can be created by the ``{...}``\n      notation (see section *Dictionary displays*).\n\n      The extension modules ``dbm``, ``gdbm``, and ``bsddb`` provide\n      additional examples of mapping types.\n\nCallable types\n   These are the types to which the function call operation (see\n   section *Calls*) can be applied:\n\n   User-defined functions\n      A user-defined function object is created by a function\n      definition (see section *Function definitions*).  It should be\n      called with an argument list containing the same number of items\n      as the function\'s formal parameter list.\n\n      Special attributes:\n\n      +-------------------------+---------------------------------+-------------+\n      | Attribute               | Meaning                         |             |\n      +=========================+=================================+=============+\n      | ``func_doc``            | The function\'s documentation    | Writable    |\n      |                         | string, or ``None`` if          |             |\n      |                         | unavailable                     |             |\n      +-------------------------+---------------------------------+-------------+\n      | ``__doc__``             | Another way of spelling         | Writable    |\n      |                         | ``func_doc``                    |             |\n      +-------------------------+---------------------------------+-------------+\n      | ``func_name``           | The function\'s name             | Writable    |\n      +-------------------------+---------------------------------+-------------+\n      | ``__name__``            | Another way of spelling         | Writable    |\n      |                         | ``func_name``                   |             |\n      +-------------------------+---------------------------------+-------------+\n      | ``__module__``          | The name of the module the      | Writable    |\n      |                         | function was defined in, or     |             |\n      |                         | ``None`` if unavailable.        |             |\n      +-------------------------+---------------------------------+-------------+\n      | ``func_defaults``       | A tuple containing default      | Writable    |\n      |                         | argument values for those       |             |\n      |                         | arguments that have defaults,   |             |\n      |                         | or ``None`` if no arguments     |             |\n      |                         | have a default value            |             |\n      +-------------------------+---------------------------------+-------------+\n      | ``func_code``           | The code object representing    | Writable    |\n      |                         | the compiled function body.     |             |\n      +-------------------------+---------------------------------+-------------+\n      | ``func_globals``        | A reference to the dictionary   | Read-only   |\n      |                         | that holds the function\'s       |             |\n      |                         | global variables --- the global |             |\n      |                         | namespace of the module in      |             |\n      |                         | which the function was defined. |             |\n      +-------------------------+---------------------------------+-------------+\n      | ``func_dict``           | The namespace supporting        | Writable    |\n      |                         | arbitrary function attributes.  |             |\n      +-------------------------+---------------------------------+-------------+\n      | ``func_closure``        | ``None`` or a tuple of cells    | Read-only   |\n      |                         | that contain bindings for the   |             |\n      |                         | function\'s free variables.      |             |\n      +-------------------------+---------------------------------+-------------+\n\n      Most of the attributes labelled "Writable" check the type of the\n      assigned value.\n\n      Changed in version 2.4: ``func_name`` is now writable.\n\n      Function objects also support getting and setting arbitrary\n      attributes, which can be used, for example, to attach metadata\n      to functions.  Regular attribute dot-notation is used to get and\n      set such attributes. *Note that the current implementation only\n      supports function attributes on user-defined functions. Function\n      attributes on built-in functions may be supported in the\n      future.*\n\n      Additional information about a function\'s definition can be\n      retrieved from its code object; see the description of internal\n      types below.\n\n   User-defined methods\n      A user-defined method object combines a class, a class instance\n      (or ``None``) and any callable object (normally a user-defined\n      function).\n\n      Special read-only attributes: ``im_self`` is the class instance\n      object, ``im_func`` is the function object; ``im_class`` is the\n      class of ``im_self`` for bound methods or the class that asked\n      for the method for unbound methods; ``__doc__`` is the method\'s\n      documentation (same as ``im_func.__doc__``); ``__name__`` is the\n      method name (same as ``im_func.__name__``); ``__module__`` is\n      the name of the module the method was defined in, or ``None`` if\n      unavailable.\n\n      Changed in version 2.2: ``im_self`` used to refer to the class\n      that defined the method.\n\n      Changed in version 2.6: For 3.0 forward-compatibility,\n      ``im_func`` is also available as ``__func__``, and ``im_self``\n      as ``__self__``.\n\n      Methods also support accessing (but not setting) the arbitrary\n      function attributes on the underlying function object.\n\n      User-defined method objects may be created when getting an\n      attribute of a class (perhaps via an instance of that class), if\n      that attribute is a user-defined function object, an unbound\n      user-defined method object, or a class method object. When the\n      attribute is a user-defined method object, a new method object\n      is only created if the class from which it is being retrieved is\n      the same as, or a derived class of, the class stored in the\n      original method object; otherwise, the original method object is\n      used as it is.\n\n      When a user-defined method object is created by retrieving a\n      user-defined function object from a class, its ``im_self``\n      attribute is ``None`` and the method object is said to be\n      unbound. When one is created by retrieving a user-defined\n      function object from a class via one of its instances, its\n      ``im_self`` attribute is the instance, and the method object is\n      said to be bound. In either case, the new method\'s ``im_class``\n      attribute is the class from which the retrieval takes place, and\n      its ``im_func`` attribute is the original function object.\n\n      When a user-defined method object is created by retrieving\n      another method object from a class or instance, the behaviour is\n      the same as for a function object, except that the ``im_func``\n      attribute of the new instance is not the original method object\n      but its ``im_func`` attribute.\n\n      When a user-defined method object is created by retrieving a\n      class method object from a class or instance, its ``im_self``\n      attribute is the class itself (the same as the ``im_class``\n      attribute), and its ``im_func`` attribute is the function object\n      underlying the class method.\n\n      When an unbound user-defined method object is called, the\n      underlying function (``im_func``) is called, with the\n      restriction that the first argument must be an instance of the\n      proper class (``im_class``) or of a derived class thereof.\n\n      When a bound user-defined method object is called, the\n      underlying function (``im_func``) is called, inserting the class\n      instance (``im_self``) in front of the argument list.  For\n      instance, when ``C`` is a class which contains a definition for\n      a function ``f()``, and ``x`` is an instance of ``C``, calling\n      ``x.f(1)`` is equivalent to calling ``C.f(x, 1)``.\n\n      When a user-defined method object is derived from a class method\n      object, the "class instance" stored in ``im_self`` will actually\n      be the class itself, so that calling either ``x.f(1)`` or\n      ``C.f(1)`` is equivalent to calling ``f(C,1)`` where ``f`` is\n      the underlying function.\n\n      Note that the transformation from function object to (unbound or\n      bound) method object happens each time the attribute is\n      retrieved from the class or instance. In some cases, a fruitful\n      optimization is to assign the attribute to a local variable and\n      call that local variable. Also notice that this transformation\n      only happens for user-defined functions; other callable objects\n      (and all non-callable objects) are retrieved without\n      transformation.  It is also important to note that user-defined\n      functions which are attributes of a class instance are not\n      converted to bound methods; this *only* happens when the\n      function is an attribute of the class.\n\n   Generator functions\n      A function or method which uses the ``yield`` statement (see\n      section *The yield statement*) is called a *generator function*.\n      Such a function, when called, always returns an iterator object\n      which can be used to execute the body of the function:  calling\n      the iterator\'s ``next()`` method will cause the function to\n      execute until it provides a value using the ``yield`` statement.\n      When the function executes a ``return`` statement or falls off\n      the end, a ``StopIteration`` exception is raised and the\n      iterator will have reached the end of the set of values to be\n      returned.\n\n   Built-in functions\n      A built-in function object is a wrapper around a C function.\n      Examples of built-in functions are ``len()`` and ``math.sin()``\n      (``math`` is a standard built-in module). The number and type of\n      the arguments are determined by the C function. Special read-\n      only attributes: ``__doc__`` is the function\'s documentation\n      string, or ``None`` if unavailable; ``__name__`` is the\n      function\'s name; ``__self__`` is set to ``None`` (but see the\n      next item); ``__module__`` is the name of the module the\n      function was defined in or ``None`` if unavailable.\n\n   Built-in methods\n      This is really a different disguise of a built-in function, this\n      time containing an object passed to the C function as an\n      implicit extra argument.  An example of a built-in method is\n      ``alist.append()``, assuming *alist* is a list object. In this\n      case, the special read-only attribute ``__self__`` is set to the\n      object denoted by *list*.\n\n   Class Types\n      Class types, or "new-style classes," are callable.  These\n      objects normally act as factories for new instances of\n      themselves, but variations are possible for class types that\n      override ``__new__()``.  The arguments of the call are passed to\n      ``__new__()`` and, in the typical case, to ``__init__()`` to\n      initialize the new instance.\n\n   Classic Classes\n      Class objects are described below.  When a class object is\n      called, a new class instance (also described below) is created\n      and returned.  This implies a call to the class\'s ``__init__()``\n      method if it has one.  Any arguments are passed on to the\n      ``__init__()`` method.  If there is no ``__init__()`` method,\n      the class must be called without arguments.\n\n   Class instances\n      Class instances are described below.  Class instances are\n      callable only when the class has a ``__call__()`` method;\n      ``x(arguments)`` is a shorthand for ``x.__call__(arguments)``.\n\nModules\n   Modules are imported by the ``import`` statement (see section *The\n   import statement*). A module object has a namespace implemented by\n   a dictionary object (this is the dictionary referenced by the\n   func_globals attribute of functions defined in the module).\n   Attribute references are translated to lookups in this dictionary,\n   e.g., ``m.x`` is equivalent to ``m.__dict__["x"]``. A module object\n   does not contain the code object used to initialize the module\n   (since it isn\'t needed once the initialization is done).\n\n   Attribute assignment updates the module\'s namespace dictionary,\n   e.g., ``m.x = 1`` is equivalent to ``m.__dict__["x"] = 1``.\n\n   Special read-only attribute: ``__dict__`` is the module\'s namespace\n   as a dictionary object.\n\n   Predefined (writable) attributes: ``__name__`` is the module\'s\n   name; ``__doc__`` is the module\'s documentation string, or ``None``\n   if unavailable; ``__file__`` is the pathname of the file from which\n   the module was loaded, if it was loaded from a file. The\n   ``__file__`` attribute is not present for C modules that are\n   statically linked into the interpreter; for extension modules\n   loaded dynamically from a shared library, it is the pathname of the\n   shared library file.\n\nClasses\n   Both class types (new-style classes) and class objects (old-\n   style/classic classes) are typically created by class definitions\n   (see section *Class definitions*).  A class has a namespace\n   implemented by a dictionary object. Class attribute references are\n   translated to lookups in this dictionary, e.g., ``C.x`` is\n   translated to ``C.__dict__["x"]`` (although for new-style classes\n   in particular there are a number of hooks which allow for other\n   means of locating attributes). When the attribute name is not found\n   there, the attribute search continues in the base classes.  For\n   old-style classes, the search is depth-first, left-to-right in the\n   order of occurrence in the base class list. New-style classes use\n   the more complex C3 method resolution order which behaves correctly\n   even in the presence of \'diamond\' inheritance structures where\n   there are multiple inheritance paths leading back to a common\n   ancestor. Additional details on the C3 MRO used by new-style\n   classes can be found in the documentation accompanying the 2.3\n   release at http://www.python.org/download/releases/2.3/mro/.\n\n   When a class attribute reference (for class ``C``, say) would yield\n   a user-defined function object or an unbound user-defined method\n   object whose associated class is either ``C`` or one of its base\n   classes, it is transformed into an unbound user-defined method\n   object whose ``im_class`` attribute is ``C``. When it would yield a\n   class method object, it is transformed into a bound user-defined\n   method object whose ``im_class`` and ``im_self`` attributes are\n   both ``C``.  When it would yield a static method object, it is\n   transformed into the object wrapped by the static method object.\n   See section *Implementing Descriptors* for another way in which\n   attributes retrieved from a class may differ from those actually\n   contained in its ``__dict__`` (note that only new-style classes\n   support descriptors).\n\n   Class attribute assignments update the class\'s dictionary, never\n   the dictionary of a base class.\n\n   A class object can be called (see above) to yield a class instance\n   (see below).\n\n   Special attributes: ``__name__`` is the class name; ``__module__``\n   is the module name in which the class was defined; ``__dict__`` is\n   the dictionary containing the class\'s namespace; ``__bases__`` is a\n   tuple (possibly empty or a singleton) containing the base classes,\n   in the order of their occurrence in the base class list;\n   ``__doc__`` is the class\'s documentation string, or None if\n   undefined.\n\nClass instances\n   A class instance is created by calling a class object (see above).\n   A class instance has a namespace implemented as a dictionary which\n   is the first place in which attribute references are searched.\n   When an attribute is not found there, and the instance\'s class has\n   an attribute by that name, the search continues with the class\n   attributes.  If a class attribute is found that is a user-defined\n   function object or an unbound user-defined method object whose\n   associated class is the class (call it ``C``) of the instance for\n   which the attribute reference was initiated or one of its bases, it\n   is transformed into a bound user-defined method object whose\n   ``im_class`` attribute is ``C`` and whose ``im_self`` attribute is\n   the instance. Static method and class method objects are also\n   transformed, as if they had been retrieved from class ``C``; see\n   above under "Classes". See section *Implementing Descriptors* for\n   another way in which attributes of a class retrieved via its\n   instances may differ from the objects actually stored in the\n   class\'s ``__dict__``. If no class attribute is found, and the\n   object\'s class has a ``__getattr__()`` method, that is called to\n   satisfy the lookup.\n\n   Attribute assignments and deletions update the instance\'s\n   dictionary, never a class\'s dictionary.  If the class has a\n   ``__setattr__()`` or ``__delattr__()`` method, this is called\n   instead of updating the instance dictionary directly.\n\n   Class instances can pretend to be numbers, sequences, or mappings\n   if they have methods with certain special names.  See section\n   *Special method names*.\n\n   Special attributes: ``__dict__`` is the attribute dictionary;\n   ``__class__`` is the instance\'s class.\n\nFiles\n   A file object represents an open file.  File objects are created by\n   the ``open()`` built-in function, and also by ``os.popen()``,\n   ``os.fdopen()``, and the ``makefile()`` method of socket objects\n   (and perhaps by other functions or methods provided by extension\n   modules).  The objects ``sys.stdin``, ``sys.stdout`` and\n   ``sys.stderr`` are initialized to file objects corresponding to the\n   interpreter\'s standard input, output and error streams.  See *File\n   Objects* for complete documentation of file objects.\n\nInternal types\n   A few types used internally by the interpreter are exposed to the\n   user. Their definitions may change with future versions of the\n   interpreter, but they are mentioned here for completeness.\n\n   Code objects\n      Code objects represent *byte-compiled* executable Python code,\n      or *bytecode*. The difference between a code object and a\n      function object is that the function object contains an explicit\n      reference to the function\'s globals (the module in which it was\n      defined), while a code object contains no context; also the\n      default argument values are stored in the function object, not\n      in the code object (because they represent values calculated at\n      run-time).  Unlike function objects, code objects are immutable\n      and contain no references (directly or indirectly) to mutable\n      objects.\n\n      Special read-only attributes: ``co_name`` gives the function\n      name; ``co_argcount`` is the number of positional arguments\n      (including arguments with default values); ``co_nlocals`` is the\n      number of local variables used by the function (including\n      arguments); ``co_varnames`` is a tuple containing the names of\n      the local variables (starting with the argument names);\n      ``co_cellvars`` is a tuple containing the names of local\n      variables that are referenced by nested functions;\n      ``co_freevars`` is a tuple containing the names of free\n      variables; ``co_code`` is a string representing the sequence of\n      bytecode instructions; ``co_consts`` is a tuple containing the\n      literals used by the bytecode; ``co_names`` is a tuple\n      containing the names used by the bytecode; ``co_filename`` is\n      the filename from which the code was compiled;\n      ``co_firstlineno`` is the first line number of the function;\n      ``co_lnotab`` is a string encoding the mapping from bytecode\n      offsets to line numbers (for details see the source code of the\n      interpreter); ``co_stacksize`` is the required stack size\n      (including local variables); ``co_flags`` is an integer encoding\n      a number of flags for the interpreter.\n\n      The following flag bits are defined for ``co_flags``: bit\n      ``0x04`` is set if the function uses the ``*arguments`` syntax\n      to accept an arbitrary number of positional arguments; bit\n      ``0x08`` is set if the function uses the ``**keywords`` syntax\n      to accept arbitrary keyword arguments; bit ``0x20`` is set if\n      the function is a generator.\n\n      Future feature declarations (``from __future__ import\n      division``) also use bits in ``co_flags`` to indicate whether a\n      code object was compiled with a particular feature enabled: bit\n      ``0x2000`` is set if the function was compiled with future\n      division enabled; bits ``0x10`` and ``0x1000`` were used in\n      earlier versions of Python.\n\n      Other bits in ``co_flags`` are reserved for internal use.\n\n      If a code object represents a function, the first item in\n      ``co_consts`` is the documentation string of the function, or\n      ``None`` if undefined.\n\n   Frame objects\n      Frame objects represent execution frames.  They may occur in\n      traceback objects (see below).\n\n      Special read-only attributes: ``f_back`` is to the previous\n      stack frame (towards the caller), or ``None`` if this is the\n      bottom stack frame; ``f_code`` is the code object being executed\n      in this frame; ``f_locals`` is the dictionary used to look up\n      local variables; ``f_globals`` is used for global variables;\n      ``f_builtins`` is used for built-in (intrinsic) names;\n      ``f_restricted`` is a flag indicating whether the function is\n      executing in restricted execution mode; ``f_lasti`` gives the\n      precise instruction (this is an index into the bytecode string\n      of the code object).\n\n      Special writable attributes: ``f_trace``, if not ``None``, is a\n      function called at the start of each source code line (this is\n      used by the debugger); ``f_exc_type``, ``f_exc_value``,\n      ``f_exc_traceback`` represent the last exception raised in the\n      parent frame provided another exception was ever raised in the\n      current frame (in all other cases they are None); ``f_lineno``\n      is the current line number of the frame --- writing to this from\n      within a trace function jumps to the given line (only for the\n      bottom-most frame).  A debugger can implement a Jump command\n      (aka Set Next Statement) by writing to f_lineno.\n\n   Traceback objects\n      Traceback objects represent a stack trace of an exception.  A\n      traceback object is created when an exception occurs.  When the\n      search for an exception handler unwinds the execution stack, at\n      each unwound level a traceback object is inserted in front of\n      the current traceback.  When an exception handler is entered,\n      the stack trace is made available to the program. (See section\n      *The try statement*.) It is accessible as ``sys.exc_traceback``,\n      and also as the third item of the tuple returned by\n      ``sys.exc_info()``.  The latter is the preferred interface,\n      since it works correctly when the program is using multiple\n      threads. When the program contains no suitable handler, the\n      stack trace is written (nicely formatted) to the standard error\n      stream; if the interpreter is interactive, it is also made\n      available to the user as ``sys.last_traceback``.\n\n      Special read-only attributes: ``tb_next`` is the next level in\n      the stack trace (towards the frame where the exception\n      occurred), or ``None`` if there is no next level; ``tb_frame``\n      points to the execution frame of the current level;\n      ``tb_lineno`` gives the line number where the exception\n      occurred; ``tb_lasti`` indicates the precise instruction.  The\n      line number and last instruction in the traceback may differ\n      from the line number of its frame object if the exception\n      occurred in a ``try`` statement with no matching except clause\n      or with a finally clause.\n\n   Slice objects\n      Slice objects are used to represent slices when *extended slice\n      syntax* is used. This is a slice using two colons, or multiple\n      slices or ellipses separated by commas, e.g., ``a[i:j:step]``,\n      ``a[i:j, k:l]``, or ``a[..., i:j]``.  They are also created by\n      the built-in ``slice()`` function.\n\n      Special read-only attributes: ``start`` is the lower bound;\n      ``stop`` is the upper bound; ``step`` is the step value; each is\n      ``None`` if omitted. These attributes can have any type.\n\n      Slice objects support one method:\n\n      slice.indices(self, length)\n\n         This method takes a single integer argument *length* and\n         computes information about the extended slice that the slice\n         object would describe if applied to a sequence of *length*\n         items.  It returns a tuple of three integers; respectively\n         these are the *start* and *stop* indices and the *step* or\n         stride length of the slice. Missing or out-of-bounds indices\n         are handled in a manner consistent with regular slices.\n\n         New in version 2.3.\n\n   Static method objects\n      Static method objects provide a way of defeating the\n      transformation of function objects to method objects described\n      above. A static method object is a wrapper around any other\n      object, usually a user-defined method object. When a static\n      method object is retrieved from a class or a class instance, the\n      object actually returned is the wrapped object, which is not\n      subject to any further transformation. Static method objects are\n      not themselves callable, although the objects they wrap usually\n      are. Static method objects are created by the built-in\n      ``staticmethod()`` constructor.\n\n   Class method objects\n      A class method object, like a static method object, is a wrapper\n      around another object that alters the way in which that object\n      is retrieved from classes and class instances. The behaviour of\n      class method objects upon such retrieval is described above,\n      under "User-defined methods". Class method objects are created\n      by the built-in ``classmethod()`` constructor.\n',
+ 'types': u'\nThe standard type hierarchy\n***************************\n\nBelow is a list of the types that are built into Python.  Extension\nmodules (written in C, Java, or other languages, depending on the\nimplementation) can define additional types.  Future versions of\nPython may add types to the type hierarchy (e.g., rational numbers,\nefficiently stored arrays of integers, etc.).\n\nSome of the type descriptions below contain a paragraph listing\n\'special attributes.\'  These are attributes that provide access to the\nimplementation and are not intended for general use.  Their definition\nmay change in the future.\n\nNone\n   This type has a single value.  There is a single object with this\n   value. This object is accessed through the built-in name ``None``.\n   It is used to signify the absence of a value in many situations,\n   e.g., it is returned from functions that don\'t explicitly return\n   anything. Its truth value is false.\n\nNotImplemented\n   This type has a single value.  There is a single object with this\n   value. This object is accessed through the built-in name\n   ``NotImplemented``. Numeric methods and rich comparison methods may\n   return this value if they do not implement the operation for the\n   operands provided.  (The interpreter will then try the reflected\n   operation, or some other fallback, depending on the operator.)  Its\n   truth value is true.\n\nEllipsis\n   This type has a single value.  There is a single object with this\n   value. This object is accessed through the built-in name\n   ``Ellipsis``. It is used to indicate the presence of the ``...``\n   syntax in a slice.  Its truth value is true.\n\n``numbers.Number``\n   These are created by numeric literals and returned as results by\n   arithmetic operators and arithmetic built-in functions.  Numeric\n   objects are immutable; once created their value never changes.\n   Python numbers are of course strongly related to mathematical\n   numbers, but subject to the limitations of numerical representation\n   in computers.\n\n   Python distinguishes between integers, floating point numbers, and\n   complex numbers:\n\n   ``numbers.Integral``\n      These represent elements from the mathematical set of integers\n      (positive and negative).\n\n      There are three types of integers:\n\n      Plain integers\n         These represent numbers in the range -2147483648 through\n         2147483647. (The range may be larger on machines with a\n         larger natural word size, but not smaller.)  When the result\n         of an operation would fall outside this range, the result is\n         normally returned as a long integer (in some cases, the\n         exception ``OverflowError`` is raised instead).  For the\n         purpose of shift and mask operations, integers are assumed to\n         have a binary, 2\'s complement notation using 32 or more bits,\n         and hiding no bits from the user (i.e., all 4294967296\n         different bit patterns correspond to different values).\n\n      Long integers\n         These represent numbers in an unlimited range, subject to\n         available (virtual) memory only.  For the purpose of shift\n         and mask operations, a binary representation is assumed, and\n         negative numbers are represented in a variant of 2\'s\n         complement which gives the illusion of an infinite string of\n         sign bits extending to the left.\n\n      Booleans\n         These represent the truth values False and True.  The two\n         objects representing the values False and True are the only\n         Boolean objects. The Boolean type is a subtype of plain\n         integers, and Boolean values behave like the values 0 and 1,\n         respectively, in almost all contexts, the exception being\n         that when converted to a string, the strings ``"False"`` or\n         ``"True"`` are returned, respectively.\n\n      The rules for integer representation are intended to give the\n      most meaningful interpretation of shift and mask operations\n      involving negative integers and the least surprises when\n      switching between the plain and long integer domains.  Any\n      operation, if it yields a result in the plain integer domain,\n      will yield the same result in the long integer domain or when\n      using mixed operands.  The switch between domains is transparent\n      to the programmer.\n\n   ``numbers.Real`` (``float``)\n      These represent machine-level double precision floating point\n      numbers. You are at the mercy of the underlying machine\n      architecture (and C or Java implementation) for the accepted\n      range and handling of overflow. Python does not support single-\n      precision floating point numbers; the savings in processor and\n      memory usage that are usually the reason for using these is\n      dwarfed by the overhead of using objects in Python, so there is\n      no reason to complicate the language with two kinds of floating\n      point numbers.\n\n   ``numbers.Complex``\n      These represent complex numbers as a pair of machine-level\n      double precision floating point numbers.  The same caveats apply\n      as for floating point numbers. The real and imaginary parts of a\n      complex number ``z`` can be retrieved through the read-only\n      attributes ``z.real`` and ``z.imag``.\n\nSequences\n   These represent finite ordered sets indexed by non-negative\n   numbers. The built-in function ``len()`` returns the number of\n   items of a sequence. When the length of a sequence is *n*, the\n   index set contains the numbers 0, 1, ..., *n*-1.  Item *i* of\n   sequence *a* is selected by ``a[i]``.\n\n   Sequences also support slicing: ``a[i:j]`` selects all items with\n   index *k* such that *i* ``<=`` *k* ``<`` *j*.  When used as an\n   expression, a slice is a sequence of the same type.  This implies\n   that the index set is renumbered so that it starts at 0.\n\n   Some sequences also support "extended slicing" with a third "step"\n   parameter: ``a[i:j:k]`` selects all items of *a* with index *x*\n   where ``x = i + n*k``, *n* ``>=`` ``0`` and *i* ``<=`` *x* ``<``\n   *j*.\n\n   Sequences are distinguished according to their mutability:\n\n   Immutable sequences\n      An object of an immutable sequence type cannot change once it is\n      created.  (If the object contains references to other objects,\n      these other objects may be mutable and may be changed; however,\n      the collection of objects directly referenced by an immutable\n      object cannot change.)\n\n      The following types are immutable sequences:\n\n      Strings\n         The items of a string are characters.  There is no separate\n         character type; a character is represented by a string of one\n         item. Characters represent (at least) 8-bit bytes.  The\n         built-in functions ``chr()`` and ``ord()`` convert between\n         characters and nonnegative integers representing the byte\n         values.  Bytes with the values 0-127 usually represent the\n         corresponding ASCII values, but the interpretation of values\n         is up to the program.  The string data type is also used to\n         represent arrays of bytes, e.g., to hold data read from a\n         file.\n\n         (On systems whose native character set is not ASCII, strings\n         may use EBCDIC in their internal representation, provided the\n         functions ``chr()`` and ``ord()`` implement a mapping between\n         ASCII and EBCDIC, and string comparison preserves the ASCII\n         order. Or perhaps someone can propose a better rule?)\n\n      Unicode\n         The items of a Unicode object are Unicode code units.  A\n         Unicode code unit is represented by a Unicode object of one\n         item and can hold either a 16-bit or 32-bit value\n         representing a Unicode ordinal (the maximum value for the\n         ordinal is given in ``sys.maxunicode``, and depends on how\n         Python is configured at compile time).  Surrogate pairs may\n         be present in the Unicode object, and will be reported as two\n         separate items.  The built-in functions ``unichr()`` and\n         ``ord()`` convert between code units and nonnegative integers\n         representing the Unicode ordinals as defined in the Unicode\n         Standard 3.0. Conversion from and to other encodings are\n         possible through the Unicode method ``encode()`` and the\n         built-in function ``unicode()``.\n\n      Tuples\n         The items of a tuple are arbitrary Python objects. Tuples of\n         two or more items are formed by comma-separated lists of\n         expressions.  A tuple of one item (a \'singleton\') can be\n         formed by affixing a comma to an expression (an expression by\n         itself does not create a tuple, since parentheses must be\n         usable for grouping of expressions).  An empty tuple can be\n         formed by an empty pair of parentheses.\n\n   Mutable sequences\n      Mutable sequences can be changed after they are created.  The\n      subscription and slicing notations can be used as the target of\n      assignment and ``del`` (delete) statements.\n\n      There are currently two intrinsic mutable sequence types:\n\n      Lists\n         The items of a list are arbitrary Python objects.  Lists are\n         formed by placing a comma-separated list of expressions in\n         square brackets. (Note that there are no special cases needed\n         to form lists of length 0 or 1.)\n\n      Byte Arrays\n         A bytearray object is a mutable array. They are created by\n         the built-in ``bytearray()`` constructor.  Aside from being\n         mutable (and hence unhashable), byte arrays otherwise provide\n         the same interface and functionality as immutable bytes\n         objects.\n\n      The extension module ``array`` provides an additional example of\n      a mutable sequence type.\n\nSet types\n   These represent unordered, finite sets of unique, immutable\n   objects. As such, they cannot be indexed by any subscript. However,\n   they can be iterated over, and the built-in function ``len()``\n   returns the number of items in a set. Common uses for sets are fast\n   membership testing, removing duplicates from a sequence, and\n   computing mathematical operations such as intersection, union,\n   difference, and symmetric difference.\n\n   For set elements, the same immutability rules apply as for\n   dictionary keys. Note that numeric types obey the normal rules for\n   numeric comparison: if two numbers compare equal (e.g., ``1`` and\n   ``1.0``), only one of them can be contained in a set.\n\n   There are currently two intrinsic set types:\n\n   Sets\n      These represent a mutable set. They are created by the built-in\n      ``set()`` constructor and can be modified afterwards by several\n      methods, such as ``add()``.\n\n   Frozen sets\n      These represent an immutable set.  They are created by the\n      built-in ``frozenset()`` constructor.  As a frozenset is\n      immutable and *hashable*, it can be used again as an element of\n      another set, or as a dictionary key.\n\nMappings\n   These represent finite sets of objects indexed by arbitrary index\n   sets. The subscript notation ``a[k]`` selects the item indexed by\n   ``k`` from the mapping ``a``; this can be used in expressions and\n   as the target of assignments or ``del`` statements. The built-in\n   function ``len()`` returns the number of items in a mapping.\n\n   There is currently a single intrinsic mapping type:\n\n   Dictionaries\n      These represent finite sets of objects indexed by nearly\n      arbitrary values.  The only types of values not acceptable as\n      keys are values containing lists or dictionaries or other\n      mutable types that are compared by value rather than by object\n      identity, the reason being that the efficient implementation of\n      dictionaries requires a key\'s hash value to remain constant.\n      Numeric types used for keys obey the normal rules for numeric\n      comparison: if two numbers compare equal (e.g., ``1`` and\n      ``1.0``) then they can be used interchangeably to index the same\n      dictionary entry.\n\n      Dictionaries are mutable; they can be created by the ``{...}``\n      notation (see section *Dictionary displays*).\n\n      The extension modules ``dbm``, ``gdbm``, and ``bsddb`` provide\n      additional examples of mapping types.\n\nCallable types\n   These are the types to which the function call operation (see\n   section *Calls*) can be applied:\n\n   User-defined functions\n      A user-defined function object is created by a function\n      definition (see section *Function definitions*).  It should be\n      called with an argument list containing the same number of items\n      as the function\'s formal parameter list.\n\n      Special attributes:\n\n      +-------------------------+---------------------------------+-------------+\n      | Attribute               | Meaning                         |             |\n      +=========================+=================================+=============+\n      | ``func_doc``            | The function\'s documentation    | Writable    |\n      |                         | string, or ``None`` if          |             |\n      |                         | unavailable                     |             |\n      +-------------------------+---------------------------------+-------------+\n      | ``__doc__``             | Another way of spelling         | Writable    |\n      |                         | ``func_doc``                    |             |\n      +-------------------------+---------------------------------+-------------+\n      | ``func_name``           | The function\'s name             | Writable    |\n      +-------------------------+---------------------------------+-------------+\n      | ``__name__``            | Another way of spelling         | Writable    |\n      |                         | ``func_name``                   |             |\n      +-------------------------+---------------------------------+-------------+\n      | ``__module__``          | The name of the module the      | Writable    |\n      |                         | function was defined in, or     |             |\n      |                         | ``None`` if unavailable.        |             |\n      +-------------------------+---------------------------------+-------------+\n      | ``func_defaults``       | A tuple containing default      | Writable    |\n      |                         | argument values for those       |             |\n      |                         | arguments that have defaults,   |             |\n      |                         | or ``None`` if no arguments     |             |\n      |                         | have a default value            |             |\n      +-------------------------+---------------------------------+-------------+\n      | ``func_code``           | The code object representing    | Writable    |\n      |                         | the compiled function body.     |             |\n      +-------------------------+---------------------------------+-------------+\n      | ``func_globals``        | A reference to the dictionary   | Read-only   |\n      |                         | that holds the function\'s       |             |\n      |                         | global variables --- the global |             |\n      |                         | namespace of the module in      |             |\n      |                         | which the function was defined. |             |\n      +-------------------------+---------------------------------+-------------+\n      | ``func_dict``           | The namespace supporting        | Writable    |\n      |                         | arbitrary function attributes.  |             |\n      +-------------------------+---------------------------------+-------------+\n      | ``func_closure``        | ``None`` or a tuple of cells    | Read-only   |\n      |                         | that contain bindings for the   |             |\n      |                         | function\'s free variables.      |             |\n      +-------------------------+---------------------------------+-------------+\n\n      Most of the attributes labelled "Writable" check the type of the\n      assigned value.\n\n      Changed in version 2.4: ``func_name`` is now writable.\n\n      Function objects also support getting and setting arbitrary\n      attributes, which can be used, for example, to attach metadata\n      to functions.  Regular attribute dot-notation is used to get and\n      set such attributes. *Note that the current implementation only\n      supports function attributes on user-defined functions. Function\n      attributes on built-in functions may be supported in the\n      future.*\n\n      Additional information about a function\'s definition can be\n      retrieved from its code object; see the description of internal\n      types below.\n\n   User-defined methods\n      A user-defined method object combines a class, a class instance\n      (or ``None``) and any callable object (normally a user-defined\n      function).\n\n      Special read-only attributes: ``im_self`` is the class instance\n      object, ``im_func`` is the function object; ``im_class`` is the\n      class of ``im_self`` for bound methods or the class that asked\n      for the method for unbound methods; ``__doc__`` is the method\'s\n      documentation (same as ``im_func.__doc__``); ``__name__`` is the\n      method name (same as ``im_func.__name__``); ``__module__`` is\n      the name of the module the method was defined in, or ``None`` if\n      unavailable.\n\n      Changed in version 2.2: ``im_self`` used to refer to the class\n      that defined the method.\n\n      Changed in version 2.6: For 3.0 forward-compatibility,\n      ``im_func`` is also available as ``__func__``, and ``im_self``\n      as ``__self__``.\n\n      Methods also support accessing (but not setting) the arbitrary\n      function attributes on the underlying function object.\n\n      User-defined method objects may be created when getting an\n      attribute of a class (perhaps via an instance of that class), if\n      that attribute is a user-defined function object, an unbound\n      user-defined method object, or a class method object. When the\n      attribute is a user-defined method object, a new method object\n      is only created if the class from which it is being retrieved is\n      the same as, or a derived class of, the class stored in the\n      original method object; otherwise, the original method object is\n      used as it is.\n\n      When a user-defined method object is created by retrieving a\n      user-defined function object from a class, its ``im_self``\n      attribute is ``None`` and the method object is said to be\n      unbound. When one is created by retrieving a user-defined\n      function object from a class via one of its instances, its\n      ``im_self`` attribute is the instance, and the method object is\n      said to be bound. In either case, the new method\'s ``im_class``\n      attribute is the class from which the retrieval takes place, and\n      its ``im_func`` attribute is the original function object.\n\n      When a user-defined method object is created by retrieving\n      another method object from a class or instance, the behaviour is\n      the same as for a function object, except that the ``im_func``\n      attribute of the new instance is not the original method object\n      but its ``im_func`` attribute.\n\n      When a user-defined method object is created by retrieving a\n      class method object from a class or instance, its ``im_self``\n      attribute is the class itself (the same as the ``im_class``\n      attribute), and its ``im_func`` attribute is the function object\n      underlying the class method.\n\n      When an unbound user-defined method object is called, the\n      underlying function (``im_func``) is called, with the\n      restriction that the first argument must be an instance of the\n      proper class (``im_class``) or of a derived class thereof.\n\n      When a bound user-defined method object is called, the\n      underlying function (``im_func``) is called, inserting the class\n      instance (``im_self``) in front of the argument list.  For\n      instance, when ``C`` is a class which contains a definition for\n      a function ``f()``, and ``x`` is an instance of ``C``, calling\n      ``x.f(1)`` is equivalent to calling ``C.f(x, 1)``.\n\n      When a user-defined method object is derived from a class method\n      object, the "class instance" stored in ``im_self`` will actually\n      be the class itself, so that calling either ``x.f(1)`` or\n      ``C.f(1)`` is equivalent to calling ``f(C,1)`` where ``f`` is\n      the underlying function.\n\n      Note that the transformation from function object to (unbound or\n      bound) method object happens each time the attribute is\n      retrieved from the class or instance. In some cases, a fruitful\n      optimization is to assign the attribute to a local variable and\n      call that local variable. Also notice that this transformation\n      only happens for user-defined functions; other callable objects\n      (and all non-callable objects) are retrieved without\n      transformation.  It is also important to note that user-defined\n      functions which are attributes of a class instance are not\n      converted to bound methods; this *only* happens when the\n      function is an attribute of the class.\n\n   Generator functions\n      A function or method which uses the ``yield`` statement (see\n      section *The yield statement*) is called a *generator function*.\n      Such a function, when called, always returns an iterator object\n      which can be used to execute the body of the function:  calling\n      the iterator\'s ``next()`` method will cause the function to\n      execute until it provides a value using the ``yield`` statement.\n      When the function executes a ``return`` statement or falls off\n      the end, a ``StopIteration`` exception is raised and the\n      iterator will have reached the end of the set of values to be\n      returned.\n\n   Built-in functions\n      A built-in function object is a wrapper around a C function.\n      Examples of built-in functions are ``len()`` and ``math.sin()``\n      (``math`` is a standard built-in module). The number and type of\n      the arguments are determined by the C function. Special read-\n      only attributes: ``__doc__`` is the function\'s documentation\n      string, or ``None`` if unavailable; ``__name__`` is the\n      function\'s name; ``__self__`` is set to ``None`` (but see the\n      next item); ``__module__`` is the name of the module the\n      function was defined in or ``None`` if unavailable.\n\n   Built-in methods\n      This is really a different disguise of a built-in function, this\n      time containing an object passed to the C function as an\n      implicit extra argument.  An example of a built-in method is\n      ``alist.append()``, assuming *alist* is a list object. In this\n      case, the special read-only attribute ``__self__`` is set to the\n      object denoted by *alist*.\n\n   Class Types\n      Class types, or "new-style classes," are callable.  These\n      objects normally act as factories for new instances of\n      themselves, but variations are possible for class types that\n      override ``__new__()``.  The arguments of the call are passed to\n      ``__new__()`` and, in the typical case, to ``__init__()`` to\n      initialize the new instance.\n\n   Classic Classes\n      Class objects are described below.  When a class object is\n      called, a new class instance (also described below) is created\n      and returned.  This implies a call to the class\'s ``__init__()``\n      method if it has one.  Any arguments are passed on to the\n      ``__init__()`` method.  If there is no ``__init__()`` method,\n      the class must be called without arguments.\n\n   Class instances\n      Class instances are described below.  Class instances are\n      callable only when the class has a ``__call__()`` method;\n      ``x(arguments)`` is a shorthand for ``x.__call__(arguments)``.\n\nModules\n   Modules are imported by the ``import`` statement (see section *The\n   import statement*). A module object has a namespace implemented by\n   a dictionary object (this is the dictionary referenced by the\n   func_globals attribute of functions defined in the module).\n   Attribute references are translated to lookups in this dictionary,\n   e.g., ``m.x`` is equivalent to ``m.__dict__["x"]``. A module object\n   does not contain the code object used to initialize the module\n   (since it isn\'t needed once the initialization is done).\n\n   Attribute assignment updates the module\'s namespace dictionary,\n   e.g., ``m.x = 1`` is equivalent to ``m.__dict__["x"] = 1``.\n\n   Special read-only attribute: ``__dict__`` is the module\'s namespace\n   as a dictionary object.\n\n   **CPython implementation detail:** Because of the way CPython\n   clears module dictionaries, the module dictionary will be cleared\n   when the module falls out of scope even if the dictionary still has\n   live references.  To avoid this, copy the dictionary or keep the\n   module around while using its dictionary directly.\n\n   Predefined (writable) attributes: ``__name__`` is the module\'s\n   name; ``__doc__`` is the module\'s documentation string, or ``None``\n   if unavailable; ``__file__`` is the pathname of the file from which\n   the module was loaded, if it was loaded from a file. The\n   ``__file__`` attribute is not present for C modules that are\n   statically linked into the interpreter; for extension modules\n   loaded dynamically from a shared library, it is the pathname of the\n   shared library file.\n\nClasses\n   Both class types (new-style classes) and class objects (old-\n   style/classic classes) are typically created by class definitions\n   (see section *Class definitions*).  A class has a namespace\n   implemented by a dictionary object. Class attribute references are\n   translated to lookups in this dictionary, e.g., ``C.x`` is\n   translated to ``C.__dict__["x"]`` (although for new-style classes\n   in particular there are a number of hooks which allow for other\n   means of locating attributes). When the attribute name is not found\n   there, the attribute search continues in the base classes.  For\n   old-style classes, the search is depth-first, left-to-right in the\n   order of occurrence in the base class list. New-style classes use\n   the more complex C3 method resolution order which behaves correctly\n   even in the presence of \'diamond\' inheritance structures where\n   there are multiple inheritance paths leading back to a common\n   ancestor. Additional details on the C3 MRO used by new-style\n   classes can be found in the documentation accompanying the 2.3\n   release at http://www.python.org/download/releases/2.3/mro/.\n\n   When a class attribute reference (for class ``C``, say) would yield\n   a user-defined function object or an unbound user-defined method\n   object whose associated class is either ``C`` or one of its base\n   classes, it is transformed into an unbound user-defined method\n   object whose ``im_class`` attribute is ``C``. When it would yield a\n   class method object, it is transformed into a bound user-defined\n   method object whose ``im_class`` and ``im_self`` attributes are\n   both ``C``.  When it would yield a static method object, it is\n   transformed into the object wrapped by the static method object.\n   See section *Implementing Descriptors* for another way in which\n   attributes retrieved from a class may differ from those actually\n   contained in its ``__dict__`` (note that only new-style classes\n   support descriptors).\n\n   Class attribute assignments update the class\'s dictionary, never\n   the dictionary of a base class.\n\n   A class object can be called (see above) to yield a class instance\n   (see below).\n\n   Special attributes: ``__name__`` is the class name; ``__module__``\n   is the module name in which the class was defined; ``__dict__`` is\n   the dictionary containing the class\'s namespace; ``__bases__`` is a\n   tuple (possibly empty or a singleton) containing the base classes,\n   in the order of their occurrence in the base class list;\n   ``__doc__`` is the class\'s documentation string, or None if\n   undefined.\n\nClass instances\n   A class instance is created by calling a class object (see above).\n   A class instance has a namespace implemented as a dictionary which\n   is the first place in which attribute references are searched.\n   When an attribute is not found there, and the instance\'s class has\n   an attribute by that name, the search continues with the class\n   attributes.  If a class attribute is found that is a user-defined\n   function object or an unbound user-defined method object whose\n   associated class is the class (call it ``C``) of the instance for\n   which the attribute reference was initiated or one of its bases, it\n   is transformed into a bound user-defined method object whose\n   ``im_class`` attribute is ``C`` and whose ``im_self`` attribute is\n   the instance. Static method and class method objects are also\n   transformed, as if they had been retrieved from class ``C``; see\n   above under "Classes". See section *Implementing Descriptors* for\n   another way in which attributes of a class retrieved via its\n   instances may differ from the objects actually stored in the\n   class\'s ``__dict__``. If no class attribute is found, and the\n   object\'s class has a ``__getattr__()`` method, that is called to\n   satisfy the lookup.\n\n   Attribute assignments and deletions update the instance\'s\n   dictionary, never a class\'s dictionary.  If the class has a\n   ``__setattr__()`` or ``__delattr__()`` method, this is called\n   instead of updating the instance dictionary directly.\n\n   Class instances can pretend to be numbers, sequences, or mappings\n   if they have methods with certain special names.  See section\n   *Special method names*.\n\n   Special attributes: ``__dict__`` is the attribute dictionary;\n   ``__class__`` is the instance\'s class.\n\nFiles\n   A file object represents an open file.  File objects are created by\n   the ``open()`` built-in function, and also by ``os.popen()``,\n   ``os.fdopen()``, and the ``makefile()`` method of socket objects\n   (and perhaps by other functions or methods provided by extension\n   modules).  The objects ``sys.stdin``, ``sys.stdout`` and\n   ``sys.stderr`` are initialized to file objects corresponding to the\n   interpreter\'s standard input, output and error streams.  See *File\n   Objects* for complete documentation of file objects.\n\nInternal types\n   A few types used internally by the interpreter are exposed to the\n   user. Their definitions may change with future versions of the\n   interpreter, but they are mentioned here for completeness.\n\n   Code objects\n      Code objects represent *byte-compiled* executable Python code,\n      or *bytecode*. The difference between a code object and a\n      function object is that the function object contains an explicit\n      reference to the function\'s globals (the module in which it was\n      defined), while a code object contains no context; also the\n      default argument values are stored in the function object, not\n      in the code object (because they represent values calculated at\n      run-time).  Unlike function objects, code objects are immutable\n      and contain no references (directly or indirectly) to mutable\n      objects.\n\n      Special read-only attributes: ``co_name`` gives the function\n      name; ``co_argcount`` is the number of positional arguments\n      (including arguments with default values); ``co_nlocals`` is the\n      number of local variables used by the function (including\n      arguments); ``co_varnames`` is a tuple containing the names of\n      the local variables (starting with the argument names);\n      ``co_cellvars`` is a tuple containing the names of local\n      variables that are referenced by nested functions;\n      ``co_freevars`` is a tuple containing the names of free\n      variables; ``co_code`` is a string representing the sequence of\n      bytecode instructions; ``co_consts`` is a tuple containing the\n      literals used by the bytecode; ``co_names`` is a tuple\n      containing the names used by the bytecode; ``co_filename`` is\n      the filename from which the code was compiled;\n      ``co_firstlineno`` is the first line number of the function;\n      ``co_lnotab`` is a string encoding the mapping from bytecode\n      offsets to line numbers (for details see the source code of the\n      interpreter); ``co_stacksize`` is the required stack size\n      (including local variables); ``co_flags`` is an integer encoding\n      a number of flags for the interpreter.\n\n      The following flag bits are defined for ``co_flags``: bit\n      ``0x04`` is set if the function uses the ``*arguments`` syntax\n      to accept an arbitrary number of positional arguments; bit\n      ``0x08`` is set if the function uses the ``**keywords`` syntax\n      to accept arbitrary keyword arguments; bit ``0x20`` is set if\n      the function is a generator.\n\n      Future feature declarations (``from __future__ import\n      division``) also use bits in ``co_flags`` to indicate whether a\n      code object was compiled with a particular feature enabled: bit\n      ``0x2000`` is set if the function was compiled with future\n      division enabled; bits ``0x10`` and ``0x1000`` were used in\n      earlier versions of Python.\n\n      Other bits in ``co_flags`` are reserved for internal use.\n\n      If a code object represents a function, the first item in\n      ``co_consts`` is the documentation string of the function, or\n      ``None`` if undefined.\n\n   Frame objects\n      Frame objects represent execution frames.  They may occur in\n      traceback objects (see below).\n\n      Special read-only attributes: ``f_back`` is to the previous\n      stack frame (towards the caller), or ``None`` if this is the\n      bottom stack frame; ``f_code`` is the code object being executed\n      in this frame; ``f_locals`` is the dictionary used to look up\n      local variables; ``f_globals`` is used for global variables;\n      ``f_builtins`` is used for built-in (intrinsic) names;\n      ``f_restricted`` is a flag indicating whether the function is\n      executing in restricted execution mode; ``f_lasti`` gives the\n      precise instruction (this is an index into the bytecode string\n      of the code object).\n\n      Special writable attributes: ``f_trace``, if not ``None``, is a\n      function called at the start of each source code line (this is\n      used by the debugger); ``f_exc_type``, ``f_exc_value``,\n      ``f_exc_traceback`` represent the last exception raised in the\n      parent frame provided another exception was ever raised in the\n      current frame (in all other cases they are None); ``f_lineno``\n      is the current line number of the frame --- writing to this from\n      within a trace function jumps to the given line (only for the\n      bottom-most frame).  A debugger can implement a Jump command\n      (aka Set Next Statement) by writing to f_lineno.\n\n   Traceback objects\n      Traceback objects represent a stack trace of an exception.  A\n      traceback object is created when an exception occurs.  When the\n      search for an exception handler unwinds the execution stack, at\n      each unwound level a traceback object is inserted in front of\n      the current traceback.  When an exception handler is entered,\n      the stack trace is made available to the program. (See section\n      *The try statement*.) It is accessible as ``sys.exc_traceback``,\n      and also as the third item of the tuple returned by\n      ``sys.exc_info()``.  The latter is the preferred interface,\n      since it works correctly when the program is using multiple\n      threads. When the program contains no suitable handler, the\n      stack trace is written (nicely formatted) to the standard error\n      stream; if the interpreter is interactive, it is also made\n      available to the user as ``sys.last_traceback``.\n\n      Special read-only attributes: ``tb_next`` is the next level in\n      the stack trace (towards the frame where the exception\n      occurred), or ``None`` if there is no next level; ``tb_frame``\n      points to the execution frame of the current level;\n      ``tb_lineno`` gives the line number where the exception\n      occurred; ``tb_lasti`` indicates the precise instruction.  The\n      line number and last instruction in the traceback may differ\n      from the line number of its frame object if the exception\n      occurred in a ``try`` statement with no matching except clause\n      or with a finally clause.\n\n   Slice objects\n      Slice objects are used to represent slices when *extended slice\n      syntax* is used. This is a slice using two colons, or multiple\n      slices or ellipses separated by commas, e.g., ``a[i:j:step]``,\n      ``a[i:j, k:l]``, or ``a[..., i:j]``.  They are also created by\n      the built-in ``slice()`` function.\n\n      Special read-only attributes: ``start`` is the lower bound;\n      ``stop`` is the upper bound; ``step`` is the step value; each is\n      ``None`` if omitted. These attributes can have any type.\n\n      Slice objects support one method:\n\n      slice.indices(self, length)\n\n         This method takes a single integer argument *length* and\n         computes information about the extended slice that the slice\n         object would describe if applied to a sequence of *length*\n         items.  It returns a tuple of three integers; respectively\n         these are the *start* and *stop* indices and the *step* or\n         stride length of the slice. Missing or out-of-bounds indices\n         are handled in a manner consistent with regular slices.\n\n         New in version 2.3.\n\n   Static method objects\n      Static method objects provide a way of defeating the\n      transformation of function objects to method objects described\n      above. A static method object is a wrapper around any other\n      object, usually a user-defined method object. When a static\n      method object is retrieved from a class or a class instance, the\n      object actually returned is the wrapped object, which is not\n      subject to any further transformation. Static method objects are\n      not themselves callable, although the objects they wrap usually\n      are. Static method objects are created by the built-in\n      ``staticmethod()`` constructor.\n\n   Class method objects\n      A class method object, like a static method object, is a wrapper\n      around another object that alters the way in which that object\n      is retrieved from classes and class instances. The behaviour of\n      class method objects upon such retrieval is described above,\n      under "User-defined methods". Class method objects are created\n      by the built-in ``classmethod()`` constructor.\n',
  'typesfunctions': u'\nFunctions\n*********\n\nFunction objects are created by function definitions.  The only\noperation on a function object is to call it: ``func(argument-list)``.\n\nThere are really two flavors of function objects: built-in functions\nand user-defined functions.  Both support the same operation (to call\nthe function), but the implementation is different, hence the\ndifferent object types.\n\nSee *Function definitions* for more information.\n',
- 'typesmapping': u'\nMapping Types --- ``dict``\n**************************\n\nA *mapping* object maps *hashable* values to arbitrary objects.\nMappings are mutable objects.  There is currently only one standard\nmapping type, the *dictionary*.  (For other containers see the built\nin ``list``, ``set``, and ``tuple`` classes, and the ``collections``\nmodule.)\n\nA dictionary\'s keys are *almost* arbitrary values.  Values that are\nnot *hashable*, that is, values containing lists, dictionaries or\nother mutable types (that are compared by value rather than by object\nidentity) may not be used as keys.  Numeric types used for keys obey\nthe normal rules for numeric comparison: if two numbers compare equal\n(such as ``1`` and ``1.0``) then they can be used interchangeably to\nindex the same dictionary entry.  (Note however, that since computers\nstore floating-point numbers as approximations it is usually unwise to\nuse them as dictionary keys.)\n\nDictionaries can be created by placing a comma-separated list of\n``key: value`` pairs within braces, for example: ``{\'jack\': 4098,\n\'sjoerd\': 4127}`` or ``{4098: \'jack\', 4127: \'sjoerd\'}``, or by the\n``dict`` constructor.\n\nclass class dict([arg])\n\n   Return a new dictionary initialized from an optional positional\n   argument or from a set of keyword arguments. If no arguments are\n   given, return a new empty dictionary. If the positional argument\n   *arg* is a mapping object, return a dictionary mapping the same\n   keys to the same values as does the mapping object. Otherwise the\n   positional argument must be a sequence, a container that supports\n   iteration, or an iterator object.  The elements of the argument\n   must each also be of one of those kinds, and each must in turn\n   contain exactly two objects. The first is used as a key in the new\n   dictionary, and the second as the key\'s value.  If a given key is\n   seen more than once, the last value associated with it is retained\n   in the new dictionary.\n\n   If keyword arguments are given, the keywords themselves with their\n   associated values are added as items to the dictionary. If a key is\n   specified both in the positional argument and as a keyword\n   argument, the value associated with the keyword is retained in the\n   dictionary. For example, these all return a dictionary equal to\n   ``{"one": 2, "two": 3}``:\n\n   * ``dict(one=2, two=3)``\n\n   * ``dict({\'one\': 2, \'two\': 3})``\n\n   * ``dict(zip((\'one\', \'two\'), (2, 3)))``\n\n   * ``dict([[\'two\', 3], [\'one\', 2]])``\n\n   The first example only works for keys that are valid Python\n   identifiers; the others work with any valid keys.\n\n   New in version 2.2.\n\n   Changed in version 2.3: Support for building a dictionary from\n   keyword arguments added.\n\n   These are the operations that dictionaries support (and therefore,\n   custom mapping types should support too):\n\n   len(d)\n\n      Return the number of items in the dictionary *d*.\n\n   d[key]\n\n      Return the item of *d* with key *key*.  Raises a ``KeyError`` if\n      *key* is not in the map.\n\n      New in version 2.5: If a subclass of dict defines a method\n      ``__missing__()``, if the key *key* is not present, the\n      ``d[key]`` operation calls that method with the key *key* as\n      argument.  The ``d[key]`` operation then returns or raises\n      whatever is returned or raised by the ``__missing__(key)`` call\n      if the key is not present. No other operations or methods invoke\n      ``__missing__()``. If ``__missing__()`` is not defined,\n      ``KeyError`` is raised.  ``__missing__()`` must be a method; it\n      cannot be an instance variable. For an example, see\n      ``collections.defaultdict``.\n\n   d[key] = value\n\n      Set ``d[key]`` to *value*.\n\n   del d[key]\n\n      Remove ``d[key]`` from *d*.  Raises a ``KeyError`` if *key* is\n      not in the map.\n\n   key in d\n\n      Return ``True`` if *d* has a key *key*, else ``False``.\n\n      New in version 2.2.\n\n   key not in d\n\n      Equivalent to ``not key in d``.\n\n      New in version 2.2.\n\n   iter(d)\n\n      Return an iterator over the keys of the dictionary.  This is a\n      shortcut for ``iterkeys()``.\n\n   clear()\n\n      Remove all items from the dictionary.\n\n   copy()\n\n      Return a shallow copy of the dictionary.\n\n   fromkeys(seq[, value])\n\n      Create a new dictionary with keys from *seq* and values set to\n      *value*.\n\n      ``fromkeys()`` is a class method that returns a new dictionary.\n      *value* defaults to ``None``.\n\n      New in version 2.3.\n\n   get(key[, default])\n\n      Return the value for *key* if *key* is in the dictionary, else\n      *default*. If *default* is not given, it defaults to ``None``,\n      so that this method never raises a ``KeyError``.\n\n   has_key(key)\n\n      Test for the presence of *key* in the dictionary.  ``has_key()``\n      is deprecated in favor of ``key in d``.\n\n   items()\n\n      Return a copy of the dictionary\'s list of ``(key, value)``\n      pairs.\n\n      **CPython implementation detail:** Keys and values are listed in\n      an arbitrary order which is non-random, varies across Python\n      implementations, and depends on the dictionary\'s history of\n      insertions and deletions.\n\n      If ``items()``, ``keys()``, ``values()``, ``iteritems()``,\n      ``iterkeys()``, and ``itervalues()`` are called with no\n      intervening modifications to the dictionary, the lists will\n      directly correspond.  This allows the creation of ``(value,\n      key)`` pairs using ``zip()``: ``pairs = zip(d.values(),\n      d.keys())``.  The same relationship holds for the ``iterkeys()``\n      and ``itervalues()`` methods: ``pairs = zip(d.itervalues(),\n      d.iterkeys())`` provides the same value for ``pairs``. Another\n      way to create the same list is ``pairs = [(v, k) for (k, v) in\n      d.iteritems()]``.\n\n   iteritems()\n\n      Return an iterator over the dictionary\'s ``(key, value)`` pairs.\n      See the note for ``dict.items()``.\n\n      Using ``iteritems()`` while adding or deleting entries in the\n      dictionary may raise a ``RuntimeError`` or fail to iterate over\n      all entries.\n\n      New in version 2.2.\n\n   iterkeys()\n\n      Return an iterator over the dictionary\'s keys.  See the note for\n      ``dict.items()``.\n\n      Using ``iterkeys()`` while adding or deleting entries in the\n      dictionary may raise a ``RuntimeError`` or fail to iterate over\n      all entries.\n\n      New in version 2.2.\n\n   itervalues()\n\n      Return an iterator over the dictionary\'s values.  See the note\n      for ``dict.items()``.\n\n      Using ``itervalues()`` while adding or deleting entries in the\n      dictionary may raise a ``RuntimeError`` or fail to iterate over\n      all entries.\n\n      New in version 2.2.\n\n   keys()\n\n      Return a copy of the dictionary\'s list of keys.  See the note\n      for ``dict.items()``.\n\n   pop(key[, default])\n\n      If *key* is in the dictionary, remove it and return its value,\n      else return *default*.  If *default* is not given and *key* is\n      not in the dictionary, a ``KeyError`` is raised.\n\n      New in version 2.3.\n\n   popitem()\n\n      Remove and return an arbitrary ``(key, value)`` pair from the\n      dictionary.\n\n      ``popitem()`` is useful to destructively iterate over a\n      dictionary, as often used in set algorithms.  If the dictionary\n      is empty, calling ``popitem()`` raises a ``KeyError``.\n\n   setdefault(key[, default])\n\n      If *key* is in the dictionary, return its value.  If not, insert\n      *key* with a value of *default* and return *default*.  *default*\n      defaults to ``None``.\n\n   update([other])\n\n      Update the dictionary with the key/value pairs from *other*,\n      overwriting existing keys.  Return ``None``.\n\n      ``update()`` accepts either another dictionary object or an\n      iterable of key/value pairs (as a tuple or other iterable of\n      length two).  If keyword arguments are specified, the dictionary\n      is then updated with those key/value pairs: ``d.update(red=1,\n      blue=2)``.\n\n      Changed in version 2.4: Allowed the argument to be an iterable\n      of key/value pairs and allowed keyword arguments.\n\n   values()\n\n      Return a copy of the dictionary\'s list of values.  See the note\n      for ``dict.items()``.\n\n   viewitems()\n\n      Return a new view of the dictionary\'s items (``(key, value)``\n      pairs).  See below for documentation of view objects.\n\n      New in version 2.7.\n\n   viewkeys()\n\n      Return a new view of the dictionary\'s keys.  See below for\n      documentation of view objects.\n\n      New in version 2.7.\n\n   viewvalues()\n\n      Return a new view of the dictionary\'s values.  See below for\n      documentation of view objects.\n\n      New in version 2.7.\n\n\nDictionary view objects\n=======================\n\nThe objects returned by ``dict.viewkeys()``, ``dict.viewvalues()`` and\n``dict.viewitems()`` are *view objects*.  They provide a dynamic view\non the dictionary\'s entries, which means that when the dictionary\nchanges, the view reflects these changes.\n\nDictionary views can be iterated over to yield their respective data,\nand support membership tests:\n\nlen(dictview)\n\n   Return the number of entries in the dictionary.\n\niter(dictview)\n\n   Return an iterator over the keys, values or items (represented as\n   tuples of ``(key, value)``) in the dictionary.\n\n   Keys and values are iterated over in an arbitrary order which is\n   non-random, varies across Python implementations, and depends on\n   the dictionary\'s history of insertions and deletions. If keys,\n   values and items views are iterated over with no intervening\n   modifications to the dictionary, the order of items will directly\n   correspond.  This allows the creation of ``(value, key)`` pairs\n   using ``zip()``: ``pairs = zip(d.values(), d.keys())``.  Another\n   way to create the same list is ``pairs = [(v, k) for (k, v) in\n   d.items()]``.\n\n   Iterating views while adding or deleting entries in the dictionary\n   may raise a ``RuntimeError`` or fail to iterate over all entries.\n\nx in dictview\n\n   Return ``True`` if *x* is in the underlying dictionary\'s keys,\n   values or items (in the latter case, *x* should be a ``(key,\n   value)`` tuple).\n\nKeys views are set-like since their entries are unique and hashable.\nIf all values are hashable, so that (key, value) pairs are unique and\nhashable, then the items view is also set-like.  (Values views are not\ntreated as set-like since the entries are generally not unique.)  Then\nthese set operations are available ("other" refers either to another\nview or a set):\n\ndictview & other\n\n   Return the intersection of the dictview and the other object as a\n   new set.\n\ndictview | other\n\n   Return the union of the dictview and the other object as a new set.\n\ndictview - other\n\n   Return the difference between the dictview and the other object\n   (all elements in *dictview* that aren\'t in *other*) as a new set.\n\ndictview ^ other\n\n   Return the symmetric difference (all elements either in *dictview*\n   or *other*, but not in both) of the dictview and the other object\n   as a new set.\n\nAn example of dictionary view usage:\n\n   >>> dishes = {\'eggs\': 2, \'sausage\': 1, \'bacon\': 1, \'spam\': 500}\n   >>> keys = dishes.viewkeys()\n   >>> values = dishes.viewvalues()\n\n   >>> # iteration\n   >>> n = 0\n   >>> for val in values:\n   ...     n += val\n   >>> print(n)\n   504\n\n   >>> # keys and values are iterated over in the same order\n   >>> list(keys)\n   [\'eggs\', \'bacon\', \'sausage\', \'spam\']\n   >>> list(values)\n   [2, 1, 1, 500]\n\n   >>> # view objects are dynamic and reflect dict changes\n   >>> del dishes[\'eggs\']\n   >>> del dishes[\'sausage\']\n   >>> list(keys)\n   [\'spam\', \'bacon\']\n\n   >>> # set operations\n   >>> keys & {\'eggs\', \'bacon\', \'salad\'}\n   {\'bacon\'}\n',
+ 'typesmapping': u'\nMapping Types --- ``dict``\n**************************\n\nA *mapping* object maps *hashable* values to arbitrary objects.\nMappings are mutable objects.  There is currently only one standard\nmapping type, the *dictionary*.  (For other containers see the built\nin ``list``, ``set``, and ``tuple`` classes, and the ``collections``\nmodule.)\n\nA dictionary\'s keys are *almost* arbitrary values.  Values that are\nnot *hashable*, that is, values containing lists, dictionaries or\nother mutable types (that are compared by value rather than by object\nidentity) may not be used as keys.  Numeric types used for keys obey\nthe normal rules for numeric comparison: if two numbers compare equal\n(such as ``1`` and ``1.0``) then they can be used interchangeably to\nindex the same dictionary entry.  (Note however, that since computers\nstore floating-point numbers as approximations it is usually unwise to\nuse them as dictionary keys.)\n\nDictionaries can be created by placing a comma-separated list of\n``key: value`` pairs within braces, for example: ``{\'jack\': 4098,\n\'sjoerd\': 4127}`` or ``{4098: \'jack\', 4127: \'sjoerd\'}``, or by the\n``dict`` constructor.\n\nclass class dict([arg])\n\n   Return a new dictionary initialized from an optional positional\n   argument or from a set of keyword arguments. If no arguments are\n   given, return a new empty dictionary. If the positional argument\n   *arg* is a mapping object, return a dictionary mapping the same\n   keys to the same values as does the mapping object. Otherwise the\n   positional argument must be a sequence, a container that supports\n   iteration, or an iterator object.  The elements of the argument\n   must each also be of one of those kinds, and each must in turn\n   contain exactly two objects. The first is used as a key in the new\n   dictionary, and the second as the key\'s value.  If a given key is\n   seen more than once, the last value associated with it is retained\n   in the new dictionary.\n\n   If keyword arguments are given, the keywords themselves with their\n   associated values are added as items to the dictionary. If a key is\n   specified both in the positional argument and as a keyword\n   argument, the value associated with the keyword is retained in the\n   dictionary. For example, these all return a dictionary equal to\n   ``{"one": 1, "two": 2}``:\n\n   * ``dict(one=1, two=2)``\n\n   * ``dict({\'one\': 1, \'two\': 2})``\n\n   * ``dict(zip((\'one\', \'two\'), (1, 2)))``\n\n   * ``dict([[\'two\', 2], [\'one\', 1]])``\n\n   The first example only works for keys that are valid Python\n   identifiers; the others work with any valid keys.\n\n   New in version 2.2.\n\n   Changed in version 2.3: Support for building a dictionary from\n   keyword arguments added.\n\n   These are the operations that dictionaries support (and therefore,\n   custom mapping types should support too):\n\n   len(d)\n\n      Return the number of items in the dictionary *d*.\n\n   d[key]\n\n      Return the item of *d* with key *key*.  Raises a ``KeyError`` if\n      *key* is not in the map.\n\n      New in version 2.5: If a subclass of dict defines a method\n      ``__missing__()``, if the key *key* is not present, the\n      ``d[key]`` operation calls that method with the key *key* as\n      argument.  The ``d[key]`` operation then returns or raises\n      whatever is returned or raised by the ``__missing__(key)`` call\n      if the key is not present. No other operations or methods invoke\n      ``__missing__()``. If ``__missing__()`` is not defined,\n      ``KeyError`` is raised.  ``__missing__()`` must be a method; it\n      cannot be an instance variable. For an example, see\n      ``collections.defaultdict``.\n\n   d[key] = value\n\n      Set ``d[key]`` to *value*.\n\n   del d[key]\n\n      Remove ``d[key]`` from *d*.  Raises a ``KeyError`` if *key* is\n      not in the map.\n\n   key in d\n\n      Return ``True`` if *d* has a key *key*, else ``False``.\n\n      New in version 2.2.\n\n   key not in d\n\n      Equivalent to ``not key in d``.\n\n      New in version 2.2.\n\n   iter(d)\n\n      Return an iterator over the keys of the dictionary.  This is a\n      shortcut for ``iterkeys()``.\n\n   clear()\n\n      Remove all items from the dictionary.\n\n   copy()\n\n      Return a shallow copy of the dictionary.\n\n   fromkeys(seq[, value])\n\n      Create a new dictionary with keys from *seq* and values set to\n      *value*.\n\n      ``fromkeys()`` is a class method that returns a new dictionary.\n      *value* defaults to ``None``.\n\n      New in version 2.3.\n\n   get(key[, default])\n\n      Return the value for *key* if *key* is in the dictionary, else\n      *default*. If *default* is not given, it defaults to ``None``,\n      so that this method never raises a ``KeyError``.\n\n   has_key(key)\n\n      Test for the presence of *key* in the dictionary.  ``has_key()``\n      is deprecated in favor of ``key in d``.\n\n   items()\n\n      Return a copy of the dictionary\'s list of ``(key, value)``\n      pairs.\n\n      **CPython implementation detail:** Keys and values are listed in\n      an arbitrary order which is non-random, varies across Python\n      implementations, and depends on the dictionary\'s history of\n      insertions and deletions.\n\n      If ``items()``, ``keys()``, ``values()``, ``iteritems()``,\n      ``iterkeys()``, and ``itervalues()`` are called with no\n      intervening modifications to the dictionary, the lists will\n      directly correspond.  This allows the creation of ``(value,\n      key)`` pairs using ``zip()``: ``pairs = zip(d.values(),\n      d.keys())``.  The same relationship holds for the ``iterkeys()``\n      and ``itervalues()`` methods: ``pairs = zip(d.itervalues(),\n      d.iterkeys())`` provides the same value for ``pairs``. Another\n      way to create the same list is ``pairs = [(v, k) for (k, v) in\n      d.iteritems()]``.\n\n   iteritems()\n\n      Return an iterator over the dictionary\'s ``(key, value)`` pairs.\n      See the note for ``dict.items()``.\n\n      Using ``iteritems()`` while adding or deleting entries in the\n      dictionary may raise a ``RuntimeError`` or fail to iterate over\n      all entries.\n\n      New in version 2.2.\n\n   iterkeys()\n\n      Return an iterator over the dictionary\'s keys.  See the note for\n      ``dict.items()``.\n\n      Using ``iterkeys()`` while adding or deleting entries in the\n      dictionary may raise a ``RuntimeError`` or fail to iterate over\n      all entries.\n\n      New in version 2.2.\n\n   itervalues()\n\n      Return an iterator over the dictionary\'s values.  See the note\n      for ``dict.items()``.\n\n      Using ``itervalues()`` while adding or deleting entries in the\n      dictionary may raise a ``RuntimeError`` or fail to iterate over\n      all entries.\n\n      New in version 2.2.\n\n   keys()\n\n      Return a copy of the dictionary\'s list of keys.  See the note\n      for ``dict.items()``.\n\n   pop(key[, default])\n\n      If *key* is in the dictionary, remove it and return its value,\n      else return *default*.  If *default* is not given and *key* is\n      not in the dictionary, a ``KeyError`` is raised.\n\n      New in version 2.3.\n\n   popitem()\n\n      Remove and return an arbitrary ``(key, value)`` pair from the\n      dictionary.\n\n      ``popitem()`` is useful to destructively iterate over a\n      dictionary, as often used in set algorithms.  If the dictionary\n      is empty, calling ``popitem()`` raises a ``KeyError``.\n\n   setdefault(key[, default])\n\n      If *key* is in the dictionary, return its value.  If not, insert\n      *key* with a value of *default* and return *default*.  *default*\n      defaults to ``None``.\n\n   update([other])\n\n      Update the dictionary with the key/value pairs from *other*,\n      overwriting existing keys.  Return ``None``.\n\n      ``update()`` accepts either another dictionary object or an\n      iterable of key/value pairs (as tuples or other iterables of\n      length two).  If keyword arguments are specified, the dictionary\n      is then updated with those key/value pairs: ``d.update(red=1,\n      blue=2)``.\n\n      Changed in version 2.4: Allowed the argument to be an iterable\n      of key/value pairs and allowed keyword arguments.\n\n   values()\n\n      Return a copy of the dictionary\'s list of values.  See the note\n      for ``dict.items()``.\n\n   viewitems()\n\n      Return a new view of the dictionary\'s items (``(key, value)``\n      pairs).  See below for documentation of view objects.\n\n      New in version 2.7.\n\n   viewkeys()\n\n      Return a new view of the dictionary\'s keys.  See below for\n      documentation of view objects.\n\n      New in version 2.7.\n\n   viewvalues()\n\n      Return a new view of the dictionary\'s values.  See below for\n      documentation of view objects.\n\n      New in version 2.7.\n\n\nDictionary view objects\n=======================\n\nThe objects returned by ``dict.viewkeys()``, ``dict.viewvalues()`` and\n``dict.viewitems()`` are *view objects*.  They provide a dynamic view\non the dictionary\'s entries, which means that when the dictionary\nchanges, the view reflects these changes.\n\nDictionary views can be iterated over to yield their respective data,\nand support membership tests:\n\nlen(dictview)\n\n   Return the number of entries in the dictionary.\n\niter(dictview)\n\n   Return an iterator over the keys, values or items (represented as\n   tuples of ``(key, value)``) in the dictionary.\n\n   Keys and values are iterated over in an arbitrary order which is\n   non-random, varies across Python implementations, and depends on\n   the dictionary\'s history of insertions and deletions. If keys,\n   values and items views are iterated over with no intervening\n   modifications to the dictionary, the order of items will directly\n   correspond.  This allows the creation of ``(value, key)`` pairs\n   using ``zip()``: ``pairs = zip(d.values(), d.keys())``.  Another\n   way to create the same list is ``pairs = [(v, k) for (k, v) in\n   d.items()]``.\n\n   Iterating views while adding or deleting entries in the dictionary\n   may raise a ``RuntimeError`` or fail to iterate over all entries.\n\nx in dictview\n\n   Return ``True`` if *x* is in the underlying dictionary\'s keys,\n   values or items (in the latter case, *x* should be a ``(key,\n   value)`` tuple).\n\nKeys views are set-like since their entries are unique and hashable.\nIf all values are hashable, so that (key, value) pairs are unique and\nhashable, then the items view is also set-like.  (Values views are not\ntreated as set-like since the entries are generally not unique.)  Then\nthese set operations are available ("other" refers either to another\nview or a set):\n\ndictview & other\n\n   Return the intersection of the dictview and the other object as a\n   new set.\n\ndictview | other\n\n   Return the union of the dictview and the other object as a new set.\n\ndictview - other\n\n   Return the difference between the dictview and the other object\n   (all elements in *dictview* that aren\'t in *other*) as a new set.\n\ndictview ^ other\n\n   Return the symmetric difference (all elements either in *dictview*\n   or *other*, but not in both) of the dictview and the other object\n   as a new set.\n\nAn example of dictionary view usage:\n\n   >>> dishes = {\'eggs\': 2, \'sausage\': 1, \'bacon\': 1, \'spam\': 500}\n   >>> keys = dishes.viewkeys()\n   >>> values = dishes.viewvalues()\n\n   >>> # iteration\n   >>> n = 0\n   >>> for val in values:\n   ...     n += val\n   >>> print(n)\n   504\n\n   >>> # keys and values are iterated over in the same order\n   >>> list(keys)\n   [\'eggs\', \'bacon\', \'sausage\', \'spam\']\n   >>> list(values)\n   [2, 1, 1, 500]\n\n   >>> # view objects are dynamic and reflect dict changes\n   >>> del dishes[\'eggs\']\n   >>> del dishes[\'sausage\']\n   >>> list(keys)\n   [\'spam\', \'bacon\']\n\n   >>> # set operations\n   >>> keys & {\'eggs\', \'bacon\', \'salad\'}\n   {\'bacon\'}\n',
  'typesmethods': u"\nMethods\n*******\n\nMethods are functions that are called using the attribute notation.\nThere are two flavors: built-in methods (such as ``append()`` on\nlists) and class instance methods.  Built-in methods are described\nwith the types that support them.\n\nThe implementation adds two special read-only attributes to class\ninstance methods: ``m.im_self`` is the object on which the method\noperates, and ``m.im_func`` is the function implementing the method.\nCalling ``m(arg-1, arg-2, ..., arg-n)`` is completely equivalent to\ncalling ``m.im_func(m.im_self, arg-1, arg-2, ..., arg-n)``.\n\nClass instance methods are either *bound* or *unbound*, referring to\nwhether the method was accessed through an instance or a class,\nrespectively.  When a method is unbound, its ``im_self`` attribute\nwill be ``None`` and if called, an explicit ``self`` object must be\npassed as the first argument.  In this case, ``self`` must be an\ninstance of the unbound method's class (or a subclass of that class),\notherwise a ``TypeError`` is raised.\n\nLike function objects, methods objects support getting arbitrary\nattributes. However, since method attributes are actually stored on\nthe underlying function object (``meth.im_func``), setting method\nattributes on either bound or unbound methods is disallowed.\nAttempting to set a method attribute results in a ``TypeError`` being\nraised.  In order to set a method attribute, you need to explicitly\nset it on the underlying function object:\n\n   class C:\n       def method(self):\n           pass\n\n   c = C()\n   c.method.im_func.whoami = 'my name is c'\n\nSee *The standard type hierarchy* for more information.\n",
  'typesmodules': u"\nModules\n*******\n\nThe only special operation on a module is attribute access:\n``m.name``, where *m* is a module and *name* accesses a name defined\nin *m*'s symbol table. Module attributes can be assigned to.  (Note\nthat the ``import`` statement is not, strictly speaking, an operation\non a module object; ``import foo`` does not require a module object\nnamed *foo* to exist, rather it requires an (external) *definition*\nfor a module named *foo* somewhere.)\n\nA special member of every module is ``__dict__``. This is the\ndictionary containing the module's symbol table. Modifying this\ndictionary will actually change the module's symbol table, but direct\nassignment to the ``__dict__`` attribute is not possible (you can\nwrite ``m.__dict__['a'] = 1``, which defines ``m.a`` to be ``1``, but\nyou can't write ``m.__dict__ = {}``).  Modifying ``__dict__`` directly\nis not recommended.\n\nModules built into the interpreter are written like this: ````.  If loaded from a file, they are written as\n````.\n",
- 'typesseq': u'\nSequence Types --- ``str``, ``unicode``, ``list``, ``tuple``, ``buffer``, ``xrange``\n************************************************************************************\n\nThere are six sequence types: strings, Unicode strings, lists, tuples,\nbuffers, and xrange objects.\n\nFor other containers see the built in ``dict`` and ``set`` classes,\nand the ``collections`` module.\n\nString literals are written in single or double quotes: ``\'xyzzy\'``,\n``"frobozz"``.  See *String literals* for more about string literals.\nUnicode strings are much like strings, but are specified in the syntax\nusing a preceding ``\'u\'`` character: ``u\'abc\'``, ``u"def"``. In\naddition to the functionality described here, there are also string-\nspecific methods described in the *String Methods* section. Lists are\nconstructed with square brackets, separating items with commas: ``[a,\nb, c]``. Tuples are constructed by the comma operator (not within\nsquare brackets), with or without enclosing parentheses, but an empty\ntuple must have the enclosing parentheses, such as ``a, b, c`` or\n``()``.  A single item tuple must have a trailing comma, such as\n``(d,)``.\n\nBuffer objects are not directly supported by Python syntax, but can be\ncreated by calling the built-in function ``buffer()``.  They don\'t\nsupport concatenation or repetition.\n\nObjects of type xrange are similar to buffers in that there is no\nspecific syntax to create them, but they are created using the\n``xrange()`` function.  They don\'t support slicing, concatenation or\nrepetition, and using ``in``, ``not in``, ``min()`` or ``max()`` on\nthem is inefficient.\n\nMost sequence types support the following operations.  The ``in`` and\n``not in`` operations have the same priorities as the comparison\noperations.  The ``+`` and ``*`` operations have the same priority as\nthe corresponding numeric operations. [3] Additional methods are\nprovided for *Mutable Sequence Types*.\n\nThis table lists the sequence operations sorted in ascending priority\n(operations in the same box have the same priority).  In the table,\n*s* and *t* are sequences of the same type; *n*, *i* and *j* are\nintegers:\n\n+--------------------+----------------------------------+------------+\n| Operation          | Result                           | Notes      |\n+====================+==================================+============+\n| ``x in s``         | ``True`` if an item of *s* is    | (1)        |\n|                    | equal to *x*, else ``False``     |            |\n+--------------------+----------------------------------+------------+\n| ``x not in s``     | ``False`` if an item of *s* is   | (1)        |\n|                    | equal to *x*, else ``True``      |            |\n+--------------------+----------------------------------+------------+\n| ``s + t``          | the concatenation of *s* and *t* | (6)        |\n+--------------------+----------------------------------+------------+\n| ``s * n, n * s``   | *n* shallow copies of *s*        | (2)        |\n|                    | concatenated                     |            |\n+--------------------+----------------------------------+------------+\n| ``s[i]``           | *i*\'th item of *s*, origin 0     | (3)        |\n+--------------------+----------------------------------+------------+\n| ``s[i:j]``         | slice of *s* from *i* to *j*     | (3)(4)     |\n+--------------------+----------------------------------+------------+\n| ``s[i:j:k]``       | slice of *s* from *i* to *j*     | (3)(5)     |\n|                    | with step *k*                    |            |\n+--------------------+----------------------------------+------------+\n| ``len(s)``         | length of *s*                    |            |\n+--------------------+----------------------------------+------------+\n| ``min(s)``         | smallest item of *s*             |            |\n+--------------------+----------------------------------+------------+\n| ``max(s)``         | largest item of *s*              |            |\n+--------------------+----------------------------------+------------+\n\nSequence types also support comparisons. In particular, tuples and\nlists are compared lexicographically by comparing corresponding\nelements. This means that to compare equal, every element must compare\nequal and the two sequences must be of the same type and have the same\nlength. (For full details see *Comparisons* in the language\nreference.)\n\nNotes:\n\n1. When *s* is a string or Unicode string object the ``in`` and ``not\n   in`` operations act like a substring test.  In Python versions\n   before 2.3, *x* had to be a string of length 1. In Python 2.3 and\n   beyond, *x* may be a string of any length.\n\n2. Values of *n* less than ``0`` are treated as ``0`` (which yields an\n   empty sequence of the same type as *s*).  Note also that the copies\n   are shallow; nested structures are not copied.  This often haunts\n   new Python programmers; consider:\n\n   >>> lists = [[]] * 3\n   >>> lists\n   [[], [], []]\n   >>> lists[0].append(3)\n   >>> lists\n   [[3], [3], [3]]\n\n   What has happened is that ``[[]]`` is a one-element list containing\n   an empty list, so all three elements of ``[[]] * 3`` are (pointers\n   to) this single empty list.  Modifying any of the elements of\n   ``lists`` modifies this single list. You can create a list of\n   different lists this way:\n\n   >>> lists = [[] for i in range(3)]\n   >>> lists[0].append(3)\n   >>> lists[1].append(5)\n   >>> lists[2].append(7)\n   >>> lists\n   [[3], [5], [7]]\n\n3. If *i* or *j* is negative, the index is relative to the end of the\n   string: ``len(s) + i`` or ``len(s) + j`` is substituted.  But note\n   that ``-0`` is still ``0``.\n\n4. The slice of *s* from *i* to *j* is defined as the sequence of\n   items with index *k* such that ``i <= k < j``.  If *i* or *j* is\n   greater than ``len(s)``, use ``len(s)``.  If *i* is omitted or\n   ``None``, use ``0``.  If *j* is omitted or ``None``, use\n   ``len(s)``.  If *i* is greater than or equal to *j*, the slice is\n   empty.\n\n5. The slice of *s* from *i* to *j* with step *k* is defined as the\n   sequence of items with index  ``x = i + n*k`` such that ``0 <= n <\n   (j-i)/k``.  In other words, the indices are ``i``, ``i+k``,\n   ``i+2*k``, ``i+3*k`` and so on, stopping when *j* is reached (but\n   never including *j*).  If *i* or *j* is greater than ``len(s)``,\n   use ``len(s)``.  If *i* or *j* are omitted or ``None``, they become\n   "end" values (which end depends on the sign of *k*).  Note, *k*\n   cannot be zero. If *k* is ``None``, it is treated like ``1``.\n\n6. **CPython implementation detail:** If *s* and *t* are both strings,\n   some Python implementations such as CPython can usually perform an\n   in-place optimization for assignments of the form ``s = s + t`` or\n   ``s += t``.  When applicable, this optimization makes quadratic\n   run-time much less likely.  This optimization is both version and\n   implementation dependent.  For performance sensitive code, it is\n   preferable to use the ``str.join()`` method which assures\n   consistent linear concatenation performance across versions and\n   implementations.\n\n   Changed in version 2.4: Formerly, string concatenation never\n   occurred in-place.\n\n\nString Methods\n==============\n\nBelow are listed the string methods which both 8-bit strings and\nUnicode objects support.\n\nIn addition, Python\'s strings support the sequence type methods\ndescribed in the *Sequence Types --- str, unicode, list, tuple,\nbuffer, xrange* section. To output formatted strings use template\nstrings or the ``%`` operator described in the *String Formatting\nOperations* section. Also, see the ``re`` module for string functions\nbased on regular expressions.\n\nstr.capitalize()\n\n   Return a copy of the string with only its first character\n   capitalized.\n\n   For 8-bit strings, this method is locale-dependent.\n\nstr.center(width[, fillchar])\n\n   Return centered in a string of length *width*. Padding is done\n   using the specified *fillchar* (default is a space).\n\n   Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.count(sub[, start[, end]])\n\n   Return the number of non-overlapping occurrences of substring *sub*\n   in the range [*start*, *end*].  Optional arguments *start* and\n   *end* are interpreted as in slice notation.\n\nstr.decode([encoding[, errors]])\n\n   Decodes the string using the codec registered for *encoding*.\n   *encoding* defaults to the default string encoding.  *errors* may\n   be given to set a different error handling scheme.  The default is\n   ``\'strict\'``, meaning that encoding errors raise ``UnicodeError``.\n   Other possible values are ``\'ignore\'``, ``\'replace\'`` and any other\n   name registered via ``codecs.register_error()``, see section *Codec\n   Base Classes*.\n\n   New in version 2.2.\n\n   Changed in version 2.3: Support for other error handling schemes\n   added.\n\n   Changed in version 2.7: Support for keyword arguments added.\n\nstr.encode([encoding[, errors]])\n\n   Return an encoded version of the string.  Default encoding is the\n   current default string encoding.  *errors* may be given to set a\n   different error handling scheme.  The default for *errors* is\n   ``\'strict\'``, meaning that encoding errors raise a\n   ``UnicodeError``.  Other possible values are ``\'ignore\'``,\n   ``\'replace\'``, ``\'xmlcharrefreplace\'``, ``\'backslashreplace\'`` and\n   any other name registered via ``codecs.register_error()``, see\n   section *Codec Base Classes*. For a list of possible encodings, see\n   section *Standard Encodings*.\n\n   New in version 2.0.\n\n   Changed in version 2.3: Support for ``\'xmlcharrefreplace\'`` and\n   ``\'backslashreplace\'`` and other error handling schemes added.\n\n   Changed in version 2.7: Support for keyword arguments added.\n\nstr.endswith(suffix[, start[, end]])\n\n   Return ``True`` if the string ends with the specified *suffix*,\n   otherwise return ``False``.  *suffix* can also be a tuple of\n   suffixes to look for.  With optional *start*, test beginning at\n   that position.  With optional *end*, stop comparing at that\n   position.\n\n   Changed in version 2.5: Accept tuples as *suffix*.\n\nstr.expandtabs([tabsize])\n\n   Return a copy of the string where all tab characters are replaced\n   by one or more spaces, depending on the current column and the\n   given tab size.  The column number is reset to zero after each\n   newline occurring in the string. If *tabsize* is not given, a tab\n   size of ``8`` characters is assumed.  This doesn\'t understand other\n   non-printing characters or escape sequences.\n\nstr.find(sub[, start[, end]])\n\n   Return the lowest index in the string where substring *sub* is\n   found, such that *sub* is contained in the slice ``s[start:end]``.\n   Optional arguments *start* and *end* are interpreted as in slice\n   notation.  Return ``-1`` if *sub* is not found.\n\nstr.format(*args, **kwargs)\n\n   Perform a string formatting operation.  The string on which this\n   method is called can contain literal text or replacement fields\n   delimited by braces ``{}``.  Each replacement field contains either\n   the numeric index of a positional argument, or the name of a\n   keyword argument.  Returns a copy of the string where each\n   replacement field is replaced with the string value of the\n   corresponding argument.\n\n   >>> "The sum of 1 + 2 is {0}".format(1+2)\n   \'The sum of 1 + 2 is 3\'\n\n   See *Format String Syntax* for a description of the various\n   formatting options that can be specified in format strings.\n\n   This method of string formatting is the new standard in Python 3.0,\n   and should be preferred to the ``%`` formatting described in\n   *String Formatting Operations* in new code.\n\n   New in version 2.6.\n\nstr.index(sub[, start[, end]])\n\n   Like ``find()``, but raise ``ValueError`` when the substring is not\n   found.\n\nstr.isalnum()\n\n   Return true if all characters in the string are alphanumeric and\n   there is at least one character, false otherwise.\n\n   For 8-bit strings, this method is locale-dependent.\n\nstr.isalpha()\n\n   Return true if all characters in the string are alphabetic and\n   there is at least one character, false otherwise.\n\n   For 8-bit strings, this method is locale-dependent.\n\nstr.isdigit()\n\n   Return true if all characters in the string are digits and there is\n   at least one character, false otherwise.\n\n   For 8-bit strings, this method is locale-dependent.\n\nstr.islower()\n\n   Return true if all cased characters in the string are lowercase and\n   there is at least one cased character, false otherwise.\n\n   For 8-bit strings, this method is locale-dependent.\n\nstr.isspace()\n\n   Return true if there are only whitespace characters in the string\n   and there is at least one character, false otherwise.\n\n   For 8-bit strings, this method is locale-dependent.\n\nstr.istitle()\n\n   Return true if the string is a titlecased string and there is at\n   least one character, for example uppercase characters may only\n   follow uncased characters and lowercase characters only cased ones.\n   Return false otherwise.\n\n   For 8-bit strings, this method is locale-dependent.\n\nstr.isupper()\n\n   Return true if all cased characters in the string are uppercase and\n   there is at least one cased character, false otherwise.\n\n   For 8-bit strings, this method is locale-dependent.\n\nstr.join(iterable)\n\n   Return a string which is the concatenation of the strings in the\n   *iterable* *iterable*.  The separator between elements is the\n   string providing this method.\n\nstr.ljust(width[, fillchar])\n\n   Return the string left justified in a string of length *width*.\n   Padding is done using the specified *fillchar* (default is a\n   space).  The original string is returned if *width* is less than\n   ``len(s)``.\n\n   Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.lower()\n\n   Return a copy of the string converted to lowercase.\n\n   For 8-bit strings, this method is locale-dependent.\n\nstr.lstrip([chars])\n\n   Return a copy of the string with leading characters removed.  The\n   *chars* argument is a string specifying the set of characters to be\n   removed.  If omitted or ``None``, the *chars* argument defaults to\n   removing whitespace.  The *chars* argument is not a prefix; rather,\n   all combinations of its values are stripped:\n\n   >>> \'   spacious   \'.lstrip()\n   \'spacious   \'\n   >>> \'www.example.com\'.lstrip(\'cmowz.\')\n   \'example.com\'\n\n   Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.partition(sep)\n\n   Split the string at the first occurrence of *sep*, and return a\n   3-tuple containing the part before the separator, the separator\n   itself, and the part after the separator.  If the separator is not\n   found, return a 3-tuple containing the string itself, followed by\n   two empty strings.\n\n   New in version 2.5.\n\nstr.replace(old, new[, count])\n\n   Return a copy of the string with all occurrences of substring *old*\n   replaced by *new*.  If the optional argument *count* is given, only\n   the first *count* occurrences are replaced.\n\nstr.rfind(sub[, start[, end]])\n\n   Return the highest index in the string where substring *sub* is\n   found, such that *sub* is contained within ``s[start:end]``.\n   Optional arguments *start* and *end* are interpreted as in slice\n   notation.  Return ``-1`` on failure.\n\nstr.rindex(sub[, start[, end]])\n\n   Like ``rfind()`` but raises ``ValueError`` when the substring *sub*\n   is not found.\n\nstr.rjust(width[, fillchar])\n\n   Return the string right justified in a string of length *width*.\n   Padding is done using the specified *fillchar* (default is a\n   space). The original string is returned if *width* is less than\n   ``len(s)``.\n\n   Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.rpartition(sep)\n\n   Split the string at the last occurrence of *sep*, and return a\n   3-tuple containing the part before the separator, the separator\n   itself, and the part after the separator.  If the separator is not\n   found, return a 3-tuple containing two empty strings, followed by\n   the string itself.\n\n   New in version 2.5.\n\nstr.rsplit([sep[, maxsplit]])\n\n   Return a list of the words in the string, using *sep* as the\n   delimiter string. If *maxsplit* is given, at most *maxsplit* splits\n   are done, the *rightmost* ones.  If *sep* is not specified or\n   ``None``, any whitespace string is a separator.  Except for\n   splitting from the right, ``rsplit()`` behaves like ``split()``\n   which is described in detail below.\n\n   New in version 2.4.\n\nstr.rstrip([chars])\n\n   Return a copy of the string with trailing characters removed.  The\n   *chars* argument is a string specifying the set of characters to be\n   removed.  If omitted or ``None``, the *chars* argument defaults to\n   removing whitespace.  The *chars* argument is not a suffix; rather,\n   all combinations of its values are stripped:\n\n   >>> \'   spacious   \'.rstrip()\n   \'   spacious\'\n   >>> \'mississippi\'.rstrip(\'ipz\')\n   \'mississ\'\n\n   Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.split([sep[, maxsplit]])\n\n   Return a list of the words in the string, using *sep* as the\n   delimiter string.  If *maxsplit* is given, at most *maxsplit*\n   splits are done (thus, the list will have at most ``maxsplit+1``\n   elements).  If *maxsplit* is not specified, then there is no limit\n   on the number of splits (all possible splits are made).\n\n   If *sep* is given, consecutive delimiters are not grouped together\n   and are deemed to delimit empty strings (for example,\n   ``\'1,,2\'.split(\',\')`` returns ``[\'1\', \'\', \'2\']``).  The *sep*\n   argument may consist of multiple characters (for example,\n   ``\'1<>2<>3\'.split(\'<>\')`` returns ``[\'1\', \'2\', \'3\']``). Splitting\n   an empty string with a specified separator returns ``[\'\']``.\n\n   If *sep* is not specified or is ``None``, a different splitting\n   algorithm is applied: runs of consecutive whitespace are regarded\n   as a single separator, and the result will contain no empty strings\n   at the start or end if the string has leading or trailing\n   whitespace.  Consequently, splitting an empty string or a string\n   consisting of just whitespace with a ``None`` separator returns\n   ``[]``.\n\n   For example, ``\' 1  2   3  \'.split()`` returns ``[\'1\', \'2\', \'3\']``,\n   and ``\'  1  2   3  \'.split(None, 1)`` returns ``[\'1\', \'2   3  \']``.\n\nstr.splitlines([keepends])\n\n   Return a list of the lines in the string, breaking at line\n   boundaries.  Line breaks are not included in the resulting list\n   unless *keepends* is given and true.\n\nstr.startswith(prefix[, start[, end]])\n\n   Return ``True`` if string starts with the *prefix*, otherwise\n   return ``False``. *prefix* can also be a tuple of prefixes to look\n   for.  With optional *start*, test string beginning at that\n   position.  With optional *end*, stop comparing string at that\n   position.\n\n   Changed in version 2.5: Accept tuples as *prefix*.\n\nstr.strip([chars])\n\n   Return a copy of the string with the leading and trailing\n   characters removed. The *chars* argument is a string specifying the\n   set of characters to be removed. If omitted or ``None``, the\n   *chars* argument defaults to removing whitespace. The *chars*\n   argument is not a prefix or suffix; rather, all combinations of its\n   values are stripped:\n\n   >>> \'   spacious   \'.strip()\n   \'spacious\'\n   >>> \'www.example.com\'.strip(\'cmowz.\')\n   \'example\'\n\n   Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.swapcase()\n\n   Return a copy of the string with uppercase characters converted to\n   lowercase and vice versa.\n\n   For 8-bit strings, this method is locale-dependent.\n\nstr.title()\n\n   Return a titlecased version of the string where words start with an\n   uppercase character and the remaining characters are lowercase.\n\n   The algorithm uses a simple language-independent definition of a\n   word as groups of consecutive letters.  The definition works in\n   many contexts but it means that apostrophes in contractions and\n   possessives form word boundaries, which may not be the desired\n   result:\n\n      >>> "they\'re bill\'s friends from the UK".title()\n      "They\'Re Bill\'S Friends From The Uk"\n\n   A workaround for apostrophes can be constructed using regular\n   expressions:\n\n      >>> import re\n      >>> def titlecase(s):\n              return re.sub(r"[A-Za-z]+(\'[A-Za-z]+)?",\n                            lambda mo: mo.group(0)[0].upper() +\n                                       mo.group(0)[1:].lower(),\n                            s)\n\n      >>> titlecase("they\'re bill\'s friends.")\n      "They\'re Bill\'s Friends."\n\n   For 8-bit strings, this method is locale-dependent.\n\nstr.translate(table[, deletechars])\n\n   Return a copy of the string where all characters occurring in the\n   optional argument *deletechars* are removed, and the remaining\n   characters have been mapped through the given translation table,\n   which must be a string of length 256.\n\n   You can use the ``maketrans()`` helper function in the ``string``\n   module to create a translation table. For string objects, set the\n   *table* argument to ``None`` for translations that only delete\n   characters:\n\n   >>> \'read this short text\'.translate(None, \'aeiou\')\n   \'rd ths shrt txt\'\n\n   New in version 2.6: Support for a ``None`` *table* argument.\n\n   For Unicode objects, the ``translate()`` method does not accept the\n   optional *deletechars* argument.  Instead, it returns a copy of the\n   *s* where all characters have been mapped through the given\n   translation table which must be a mapping of Unicode ordinals to\n   Unicode ordinals, Unicode strings or ``None``. Unmapped characters\n   are left untouched. Characters mapped to ``None`` are deleted.\n   Note, a more flexible approach is to create a custom character\n   mapping codec using the ``codecs`` module (see ``encodings.cp1251``\n   for an example).\n\nstr.upper()\n\n   Return a copy of the string converted to uppercase.\n\n   For 8-bit strings, this method is locale-dependent.\n\nstr.zfill(width)\n\n   Return the numeric string left filled with zeros in a string of\n   length *width*.  A sign prefix is handled correctly.  The original\n   string is returned if *width* is less than ``len(s)``.\n\n   New in version 2.2.2.\n\nThe following methods are present only on unicode objects:\n\nunicode.isnumeric()\n\n   Return ``True`` if there are only numeric characters in S,\n   ``False`` otherwise. Numeric characters include digit characters,\n   and all characters that have the Unicode numeric value property,\n   e.g. U+2155, VULGAR FRACTION ONE FIFTH.\n\nunicode.isdecimal()\n\n   Return ``True`` if there are only decimal characters in S,\n   ``False`` otherwise. Decimal characters include digit characters,\n   and all characters that that can be used to form decimal-radix\n   numbers, e.g. U+0660, ARABIC-INDIC DIGIT ZERO.\n\n\nString Formatting Operations\n============================\n\nString and Unicode objects have one unique built-in operation: the\n``%`` operator (modulo).  This is also known as the string\n*formatting* or *interpolation* operator.  Given ``format % values``\n(where *format* is a string or Unicode object), ``%`` conversion\nspecifications in *format* are replaced with zero or more elements of\n*values*.  The effect is similar to the using ``sprintf()`` in the C\nlanguage.  If *format* is a Unicode object, or if any of the objects\nbeing converted using the ``%s`` conversion are Unicode objects, the\nresult will also be a Unicode object.\n\nIf *format* requires a single argument, *values* may be a single non-\ntuple object. [4]  Otherwise, *values* must be a tuple with exactly\nthe number of items specified by the format string, or a single\nmapping object (for example, a dictionary).\n\nA conversion specifier contains two or more characters and has the\nfollowing components, which must occur in this order:\n\n1. The ``\'%\'`` character, which marks the start of the specifier.\n\n2. Mapping key (optional), consisting of a parenthesised sequence of\n   characters (for example, ``(somename)``).\n\n3. Conversion flags (optional), which affect the result of some\n   conversion types.\n\n4. Minimum field width (optional).  If specified as an ``\'*\'``\n   (asterisk), the actual width is read from the next element of the\n   tuple in *values*, and the object to convert comes after the\n   minimum field width and optional precision.\n\n5. Precision (optional), given as a ``\'.\'`` (dot) followed by the\n   precision.  If specified as ``\'*\'`` (an asterisk), the actual width\n   is read from the next element of the tuple in *values*, and the\n   value to convert comes after the precision.\n\n6. Length modifier (optional).\n\n7. Conversion type.\n\nWhen the right argument is a dictionary (or other mapping type), then\nthe formats in the string *must* include a parenthesised mapping key\ninto that dictionary inserted immediately after the ``\'%\'`` character.\nThe mapping key selects the value to be formatted from the mapping.\nFor example:\n\n>>> print \'%(language)s has %(#)03d quote types.\' % \\\n...       {\'language\': "Python", "#": 2}\nPython has 002 quote types.\n\nIn this case no ``*`` specifiers may occur in a format (since they\nrequire a sequential parameter list).\n\nThe conversion flag characters are:\n\n+-----------+-----------------------------------------------------------------------+\n| Flag      | Meaning                                                               |\n+===========+=======================================================================+\n| ``\'#\'``   | The value conversion will use the "alternate form" (where defined     |\n|           | below).                                                               |\n+-----------+-----------------------------------------------------------------------+\n| ``\'0\'``   | The conversion will be zero padded for numeric values.                |\n+-----------+-----------------------------------------------------------------------+\n| ``\'-\'``   | The converted value is left adjusted (overrides the ``\'0\'``           |\n|           | conversion if both are given).                                        |\n+-----------+-----------------------------------------------------------------------+\n| ``\' \'``   | (a space) A blank should be left before a positive number (or empty   |\n|           | string) produced by a signed conversion.                              |\n+-----------+-----------------------------------------------------------------------+\n| ``\'+\'``   | A sign character (``\'+\'`` or ``\'-\'``) will precede the conversion     |\n|           | (overrides a "space" flag).                                           |\n+-----------+-----------------------------------------------------------------------+\n\nA length modifier (``h``, ``l``, or ``L``) may be present, but is\nignored as it is not necessary for Python -- so e.g. ``%ld`` is\nidentical to ``%d``.\n\nThe conversion types are:\n\n+--------------+-------------------------------------------------------+---------+\n| Conversion   | Meaning                                               | Notes   |\n+==============+=======================================================+=========+\n| ``\'d\'``      | Signed integer decimal.                               |         |\n+--------------+-------------------------------------------------------+---------+\n| ``\'i\'``      | Signed integer decimal.                               |         |\n+--------------+-------------------------------------------------------+---------+\n| ``\'o\'``      | Signed octal value.                                   | (1)     |\n+--------------+-------------------------------------------------------+---------+\n| ``\'u\'``      | Obsolete type -- it is identical to ``\'d\'``.          | (7)     |\n+--------------+-------------------------------------------------------+---------+\n| ``\'x\'``      | Signed hexadecimal (lowercase).                       | (2)     |\n+--------------+-------------------------------------------------------+---------+\n| ``\'X\'``      | Signed hexadecimal (uppercase).                       | (2)     |\n+--------------+-------------------------------------------------------+---------+\n| ``\'e\'``      | Floating point exponential format (lowercase).        | (3)     |\n+--------------+-------------------------------------------------------+---------+\n| ``\'E\'``      | Floating point exponential format (uppercase).        | (3)     |\n+--------------+-------------------------------------------------------+---------+\n| ``\'f\'``      | Floating point decimal format.                        | (3)     |\n+--------------+-------------------------------------------------------+---------+\n| ``\'F\'``      | Floating point decimal format.                        | (3)     |\n+--------------+-------------------------------------------------------+---------+\n| ``\'g\'``      | Floating point format. Uses lowercase exponential     | (4)     |\n|              | format if exponent is less than -4 or not less than   |         |\n|              | precision, decimal format otherwise.                  |         |\n+--------------+-------------------------------------------------------+---------+\n| ``\'G\'``      | Floating point format. Uses uppercase exponential     | (4)     |\n|              | format if exponent is less than -4 or not less than   |         |\n|              | precision, decimal format otherwise.                  |         |\n+--------------+-------------------------------------------------------+---------+\n| ``\'c\'``      | Single character (accepts integer or single character |         |\n|              | string).                                              |         |\n+--------------+-------------------------------------------------------+---------+\n| ``\'r\'``      | String (converts any Python object using ``repr()``). | (5)     |\n+--------------+-------------------------------------------------------+---------+\n| ``\'s\'``      | String (converts any Python object using ``str()``).  | (6)     |\n+--------------+-------------------------------------------------------+---------+\n| ``\'%\'``      | No argument is converted, results in a ``\'%\'``        |         |\n|              | character in the result.                              |         |\n+--------------+-------------------------------------------------------+---------+\n\nNotes:\n\n1. The alternate form causes a leading zero (``\'0\'``) to be inserted\n   between left-hand padding and the formatting of the number if the\n   leading character of the result is not already a zero.\n\n2. The alternate form causes a leading ``\'0x\'`` or ``\'0X\'`` (depending\n   on whether the ``\'x\'`` or ``\'X\'`` format was used) to be inserted\n   between left-hand padding and the formatting of the number if the\n   leading character of the result is not already a zero.\n\n3. The alternate form causes the result to always contain a decimal\n   point, even if no digits follow it.\n\n   The precision determines the number of digits after the decimal\n   point and defaults to 6.\n\n4. The alternate form causes the result to always contain a decimal\n   point, and trailing zeroes are not removed as they would otherwise\n   be.\n\n   The precision determines the number of significant digits before\n   and after the decimal point and defaults to 6.\n\n5. The ``%r`` conversion was added in Python 2.0.\n\n   The precision determines the maximal number of characters used.\n\n6. If the object or format provided is a ``unicode`` string, the\n   resulting string will also be ``unicode``.\n\n   The precision determines the maximal number of characters used.\n\n7. See **PEP 237**.\n\nSince Python strings have an explicit length, ``%s`` conversions do\nnot assume that ``\'\\0\'`` is the end of the string.\n\nChanged in version 2.7: ``%f`` conversions for numbers whose absolute\nvalue is over 1e50 are no longer replaced by ``%g`` conversions.\n\nAdditional string operations are defined in standard modules\n``string`` and ``re``.\n\n\nXRange Type\n===========\n\nThe ``xrange`` type is an immutable sequence which is commonly used\nfor looping.  The advantage of the ``xrange`` type is that an\n``xrange`` object will always take the same amount of memory, no\nmatter the size of the range it represents.  There are no consistent\nperformance advantages.\n\nXRange objects have very little behavior: they only support indexing,\niteration, and the ``len()`` function.\n\n\nMutable Sequence Types\n======================\n\nList objects support additional operations that allow in-place\nmodification of the object. Other mutable sequence types (when added\nto the language) should also support these operations. Strings and\ntuples are immutable sequence types: such objects cannot be modified\nonce created. The following operations are defined on mutable sequence\ntypes (where *x* is an arbitrary object):\n\n+--------------------------------+----------------------------------+-----------------------+\n| Operation                      | Result                           | Notes                 |\n+================================+==================================+=======================+\n| ``s[i] = x``                   | item *i* of *s* is replaced by   |                       |\n|                                | *x*                              |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s[i:j] = t``                 | slice of *s* from *i* to *j* is  |                       |\n|                                | replaced by the contents of the  |                       |\n|                                | iterable *t*                     |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``del s[i:j]``                 | same as ``s[i:j] = []``          |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s[i:j:k] = t``               | the elements of ``s[i:j:k]`` are | (1)                   |\n|                                | replaced by those of *t*         |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``del s[i:j:k]``               | removes the elements of          |                       |\n|                                | ``s[i:j:k]`` from the list       |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.append(x)``                | same as ``s[len(s):len(s)] =     | (2)                   |\n|                                | [x]``                            |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.extend(x)``                | same as ``s[len(s):len(s)] = x`` | (3)                   |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.count(x)``                 | return number of *i*\'s for which |                       |\n|                                | ``s[i] == x``                    |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.index(x[, i[, j]])``       | return smallest *k* such that    | (4)                   |\n|                                | ``s[k] == x`` and ``i <= k < j`` |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.insert(i, x)``             | same as ``s[i:i] = [x]``         | (5)                   |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.pop([i])``                 | same as ``x = s[i]; del s[i];    | (6)                   |\n|                                | return x``                       |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.remove(x)``                | same as ``del s[s.index(x)]``    | (4)                   |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.reverse()``                | reverses the items of *s* in     | (7)                   |\n|                                | place                            |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.sort([cmp[, key[,          | sort the items of *s* in place   | (7)(8)(9)(10)         |\n| reverse]]])``                  |                                  |                       |\n+--------------------------------+----------------------------------+-----------------------+\n\nNotes:\n\n1. *t* must have the same length as the slice it is  replacing.\n\n2. The C implementation of Python has historically accepted multiple\n   parameters and implicitly joined them into a tuple; this no longer\n   works in Python 2.0.  Use of this misfeature has been deprecated\n   since Python 1.4.\n\n3. *x* can be any iterable object.\n\n4. Raises ``ValueError`` when *x* is not found in *s*. When a negative\n   index is passed as the second or third parameter to the ``index()``\n   method, the list length is added, as for slice indices.  If it is\n   still negative, it is truncated to zero, as for slice indices.\n\n   Changed in version 2.3: Previously, ``index()`` didn\'t have\n   arguments for specifying start and stop positions.\n\n5. When a negative index is passed as the first parameter to the\n   ``insert()`` method, the list length is added, as for slice\n   indices.  If it is still negative, it is truncated to zero, as for\n   slice indices.\n\n   Changed in version 2.3: Previously, all negative indices were\n   truncated to zero.\n\n6. The ``pop()`` method is only supported by the list and array types.\n   The optional argument *i* defaults to ``-1``, so that by default\n   the last item is removed and returned.\n\n7. The ``sort()`` and ``reverse()`` methods modify the list in place\n   for economy of space when sorting or reversing a large list.  To\n   remind you that they operate by side effect, they don\'t return the\n   sorted or reversed list.\n\n8. The ``sort()`` method takes optional arguments for controlling the\n   comparisons.\n\n   *cmp* specifies a custom comparison function of two arguments (list\n   items) which should return a negative, zero or positive number\n   depending on whether the first argument is considered smaller than,\n   equal to, or larger than the second argument: ``cmp=lambda x,y:\n   cmp(x.lower(), y.lower())``.  The default value is ``None``.\n\n   *key* specifies a function of one argument that is used to extract\n   a comparison key from each list element: ``key=str.lower``.  The\n   default value is ``None``.\n\n   *reverse* is a boolean value.  If set to ``True``, then the list\n   elements are sorted as if each comparison were reversed.\n\n   In general, the *key* and *reverse* conversion processes are much\n   faster than specifying an equivalent *cmp* function.  This is\n   because *cmp* is called multiple times for each list element while\n   *key* and *reverse* touch each element only once.  Use\n   ``functools.cmp_to_key()`` to convert an old-style *cmp* function\n   to a *key* function.\n\n   Changed in version 2.3: Support for ``None`` as an equivalent to\n   omitting *cmp* was added.\n\n   Changed in version 2.4: Support for *key* and *reverse* was added.\n\n9. Starting with Python 2.3, the ``sort()`` method is guaranteed to be\n   stable.  A sort is stable if it guarantees not to change the\n   relative order of elements that compare equal --- this is helpful\n   for sorting in multiple passes (for example, sort by department,\n   then by salary grade).\n\n10. **CPython implementation detail:** While a list is being sorted,\n    the effect of attempting to mutate, or even inspect, the list is\n    undefined.  The C implementation of Python 2.3 and newer makes the\n    list appear empty for the duration, and raises ``ValueError`` if\n    it can detect that the list has been mutated during a sort.\n',
- 'typesseq-mutable': u"\nMutable Sequence Types\n**********************\n\nList objects support additional operations that allow in-place\nmodification of the object. Other mutable sequence types (when added\nto the language) should also support these operations. Strings and\ntuples are immutable sequence types: such objects cannot be modified\nonce created. The following operations are defined on mutable sequence\ntypes (where *x* is an arbitrary object):\n\n+--------------------------------+----------------------------------+-----------------------+\n| Operation                      | Result                           | Notes                 |\n+================================+==================================+=======================+\n| ``s[i] = x``                   | item *i* of *s* is replaced by   |                       |\n|                                | *x*                              |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s[i:j] = t``                 | slice of *s* from *i* to *j* is  |                       |\n|                                | replaced by the contents of the  |                       |\n|                                | iterable *t*                     |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``del s[i:j]``                 | same as ``s[i:j] = []``          |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s[i:j:k] = t``               | the elements of ``s[i:j:k]`` are | (1)                   |\n|                                | replaced by those of *t*         |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``del s[i:j:k]``               | removes the elements of          |                       |\n|                                | ``s[i:j:k]`` from the list       |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.append(x)``                | same as ``s[len(s):len(s)] =     | (2)                   |\n|                                | [x]``                            |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.extend(x)``                | same as ``s[len(s):len(s)] = x`` | (3)                   |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.count(x)``                 | return number of *i*'s for which |                       |\n|                                | ``s[i] == x``                    |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.index(x[, i[, j]])``       | return smallest *k* such that    | (4)                   |\n|                                | ``s[k] == x`` and ``i <= k < j`` |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.insert(i, x)``             | same as ``s[i:i] = [x]``         | (5)                   |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.pop([i])``                 | same as ``x = s[i]; del s[i];    | (6)                   |\n|                                | return x``                       |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.remove(x)``                | same as ``del s[s.index(x)]``    | (4)                   |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.reverse()``                | reverses the items of *s* in     | (7)                   |\n|                                | place                            |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.sort([cmp[, key[,          | sort the items of *s* in place   | (7)(8)(9)(10)         |\n| reverse]]])``                  |                                  |                       |\n+--------------------------------+----------------------------------+-----------------------+\n\nNotes:\n\n1. *t* must have the same length as the slice it is  replacing.\n\n2. The C implementation of Python has historically accepted multiple\n   parameters and implicitly joined them into a tuple; this no longer\n   works in Python 2.0.  Use of this misfeature has been deprecated\n   since Python 1.4.\n\n3. *x* can be any iterable object.\n\n4. Raises ``ValueError`` when *x* is not found in *s*. When a negative\n   index is passed as the second or third parameter to the ``index()``\n   method, the list length is added, as for slice indices.  If it is\n   still negative, it is truncated to zero, as for slice indices.\n\n   Changed in version 2.3: Previously, ``index()`` didn't have\n   arguments for specifying start and stop positions.\n\n5. When a negative index is passed as the first parameter to the\n   ``insert()`` method, the list length is added, as for slice\n   indices.  If it is still negative, it is truncated to zero, as for\n   slice indices.\n\n   Changed in version 2.3: Previously, all negative indices were\n   truncated to zero.\n\n6. The ``pop()`` method is only supported by the list and array types.\n   The optional argument *i* defaults to ``-1``, so that by default\n   the last item is removed and returned.\n\n7. The ``sort()`` and ``reverse()`` methods modify the list in place\n   for economy of space when sorting or reversing a large list.  To\n   remind you that they operate by side effect, they don't return the\n   sorted or reversed list.\n\n8. The ``sort()`` method takes optional arguments for controlling the\n   comparisons.\n\n   *cmp* specifies a custom comparison function of two arguments (list\n   items) which should return a negative, zero or positive number\n   depending on whether the first argument is considered smaller than,\n   equal to, or larger than the second argument: ``cmp=lambda x,y:\n   cmp(x.lower(), y.lower())``.  The default value is ``None``.\n\n   *key* specifies a function of one argument that is used to extract\n   a comparison key from each list element: ``key=str.lower``.  The\n   default value is ``None``.\n\n   *reverse* is a boolean value.  If set to ``True``, then the list\n   elements are sorted as if each comparison were reversed.\n\n   In general, the *key* and *reverse* conversion processes are much\n   faster than specifying an equivalent *cmp* function.  This is\n   because *cmp* is called multiple times for each list element while\n   *key* and *reverse* touch each element only once.  Use\n   ``functools.cmp_to_key()`` to convert an old-style *cmp* function\n   to a *key* function.\n\n   Changed in version 2.3: Support for ``None`` as an equivalent to\n   omitting *cmp* was added.\n\n   Changed in version 2.4: Support for *key* and *reverse* was added.\n\n9. Starting with Python 2.3, the ``sort()`` method is guaranteed to be\n   stable.  A sort is stable if it guarantees not to change the\n   relative order of elements that compare equal --- this is helpful\n   for sorting in multiple passes (for example, sort by department,\n   then by salary grade).\n\n10. **CPython implementation detail:** While a list is being sorted,\n    the effect of attempting to mutate, or even inspect, the list is\n    undefined.  The C implementation of Python 2.3 and newer makes the\n    list appear empty for the duration, and raises ``ValueError`` if\n    it can detect that the list has been mutated during a sort.\n",
+ 'typesseq': u'\nSequence Types --- ``str``, ``unicode``, ``list``, ``tuple``, ``bytearray``, ``buffer``, ``xrange``\n***************************************************************************************************\n\nThere are seven sequence types: strings, Unicode strings, lists,\ntuples, bytearrays, buffers, and xrange objects.\n\nFor other containers see the built in ``dict`` and ``set`` classes,\nand the ``collections`` module.\n\nString literals are written in single or double quotes: ``\'xyzzy\'``,\n``"frobozz"``.  See *String literals* for more about string literals.\nUnicode strings are much like strings, but are specified in the syntax\nusing a preceding ``\'u\'`` character: ``u\'abc\'``, ``u"def"``. In\naddition to the functionality described here, there are also string-\nspecific methods described in the *String Methods* section. Lists are\nconstructed with square brackets, separating items with commas: ``[a,\nb, c]``. Tuples are constructed by the comma operator (not within\nsquare brackets), with or without enclosing parentheses, but an empty\ntuple must have the enclosing parentheses, such as ``a, b, c`` or\n``()``.  A single item tuple must have a trailing comma, such as\n``(d,)``.\n\nBytearray objects are created with the built-in function\n``bytearray()``.\n\nBuffer objects are not directly supported by Python syntax, but can be\ncreated by calling the built-in function ``buffer()``.  They don\'t\nsupport concatenation or repetition.\n\nObjects of type xrange are similar to buffers in that there is no\nspecific syntax to create them, but they are created using the\n``xrange()`` function.  They don\'t support slicing, concatenation or\nrepetition, and using ``in``, ``not in``, ``min()`` or ``max()`` on\nthem is inefficient.\n\nMost sequence types support the following operations.  The ``in`` and\n``not in`` operations have the same priorities as the comparison\noperations.  The ``+`` and ``*`` operations have the same priority as\nthe corresponding numeric operations. [3] Additional methods are\nprovided for *Mutable Sequence Types*.\n\nThis table lists the sequence operations sorted in ascending priority\n(operations in the same box have the same priority).  In the table,\n*s* and *t* are sequences of the same type; *n*, *i* and *j* are\nintegers:\n\n+--------------------+----------------------------------+------------+\n| Operation          | Result                           | Notes      |\n+====================+==================================+============+\n| ``x in s``         | ``True`` if an item of *s* is    | (1)        |\n|                    | equal to *x*, else ``False``     |            |\n+--------------------+----------------------------------+------------+\n| ``x not in s``     | ``False`` if an item of *s* is   | (1)        |\n|                    | equal to *x*, else ``True``      |            |\n+--------------------+----------------------------------+------------+\n| ``s + t``          | the concatenation of *s* and *t* | (6)        |\n+--------------------+----------------------------------+------------+\n| ``s * n, n * s``   | *n* shallow copies of *s*        | (2)        |\n|                    | concatenated                     |            |\n+--------------------+----------------------------------+------------+\n| ``s[i]``           | *i*\'th item of *s*, origin 0     | (3)        |\n+--------------------+----------------------------------+------------+\n| ``s[i:j]``         | slice of *s* from *i* to *j*     | (3)(4)     |\n+--------------------+----------------------------------+------------+\n| ``s[i:j:k]``       | slice of *s* from *i* to *j*     | (3)(5)     |\n|                    | with step *k*                    |            |\n+--------------------+----------------------------------+------------+\n| ``len(s)``         | length of *s*                    |            |\n+--------------------+----------------------------------+------------+\n| ``min(s)``         | smallest item of *s*             |            |\n+--------------------+----------------------------------+------------+\n| ``max(s)``         | largest item of *s*              |            |\n+--------------------+----------------------------------+------------+\n| ``s.index(i)``     | index of the first occurence of  |            |\n|                    | *i* in *s*                       |            |\n+--------------------+----------------------------------+------------+\n| ``s.count(i)``     | total number of occurences of    |            |\n|                    | *i* in *s*                       |            |\n+--------------------+----------------------------------+------------+\n\nSequence types also support comparisons. In particular, tuples and\nlists are compared lexicographically by comparing corresponding\nelements. This means that to compare equal, every element must compare\nequal and the two sequences must be of the same type and have the same\nlength. (For full details see *Comparisons* in the language\nreference.)\n\nNotes:\n\n1. When *s* is a string or Unicode string object the ``in`` and ``not\n   in`` operations act like a substring test.  In Python versions\n   before 2.3, *x* had to be a string of length 1. In Python 2.3 and\n   beyond, *x* may be a string of any length.\n\n2. Values of *n* less than ``0`` are treated as ``0`` (which yields an\n   empty sequence of the same type as *s*).  Note also that the copies\n   are shallow; nested structures are not copied.  This often haunts\n   new Python programmers; consider:\n\n   >>> lists = [[]] * 3\n   >>> lists\n   [[], [], []]\n   >>> lists[0].append(3)\n   >>> lists\n   [[3], [3], [3]]\n\n   What has happened is that ``[[]]`` is a one-element list containing\n   an empty list, so all three elements of ``[[]] * 3`` are (pointers\n   to) this single empty list.  Modifying any of the elements of\n   ``lists`` modifies this single list. You can create a list of\n   different lists this way:\n\n   >>> lists = [[] for i in range(3)]\n   >>> lists[0].append(3)\n   >>> lists[1].append(5)\n   >>> lists[2].append(7)\n   >>> lists\n   [[3], [5], [7]]\n\n3. If *i* or *j* is negative, the index is relative to the end of the\n   string: ``len(s) + i`` or ``len(s) + j`` is substituted.  But note\n   that ``-0`` is still ``0``.\n\n4. The slice of *s* from *i* to *j* is defined as the sequence of\n   items with index *k* such that ``i <= k < j``.  If *i* or *j* is\n   greater than ``len(s)``, use ``len(s)``.  If *i* is omitted or\n   ``None``, use ``0``.  If *j* is omitted or ``None``, use\n   ``len(s)``.  If *i* is greater than or equal to *j*, the slice is\n   empty.\n\n5. The slice of *s* from *i* to *j* with step *k* is defined as the\n   sequence of items with index  ``x = i + n*k`` such that ``0 <= n <\n   (j-i)/k``.  In other words, the indices are ``i``, ``i+k``,\n   ``i+2*k``, ``i+3*k`` and so on, stopping when *j* is reached (but\n   never including *j*).  If *i* or *j* is greater than ``len(s)``,\n   use ``len(s)``.  If *i* or *j* are omitted or ``None``, they become\n   "end" values (which end depends on the sign of *k*).  Note, *k*\n   cannot be zero. If *k* is ``None``, it is treated like ``1``.\n\n6. **CPython implementation detail:** If *s* and *t* are both strings,\n   some Python implementations such as CPython can usually perform an\n   in-place optimization for assignments of the form ``s = s + t`` or\n   ``s += t``.  When applicable, this optimization makes quadratic\n   run-time much less likely.  This optimization is both version and\n   implementation dependent.  For performance sensitive code, it is\n   preferable to use the ``str.join()`` method which assures\n   consistent linear concatenation performance across versions and\n   implementations.\n\n   Changed in version 2.4: Formerly, string concatenation never\n   occurred in-place.\n\n\nString Methods\n==============\n\nBelow are listed the string methods which both 8-bit strings and\nUnicode objects support.  Some of them are also available on\n``bytearray`` objects.\n\nIn addition, Python\'s strings support the sequence type methods\ndescribed in the *Sequence Types --- str, unicode, list, tuple,\nbytearray, buffer, xrange* section. To output formatted strings use\ntemplate strings or the ``%`` operator described in the *String\nFormatting Operations* section. Also, see the ``re`` module for string\nfunctions based on regular expressions.\n\nstr.capitalize()\n\n   Return a copy of the string with its first character capitalized\n   and the rest lowercased.\n\n   For 8-bit strings, this method is locale-dependent.\n\nstr.center(width[, fillchar])\n\n   Return centered in a string of length *width*. Padding is done\n   using the specified *fillchar* (default is a space).\n\n   Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.count(sub[, start[, end]])\n\n   Return the number of non-overlapping occurrences of substring *sub*\n   in the range [*start*, *end*].  Optional arguments *start* and\n   *end* are interpreted as in slice notation.\n\nstr.decode([encoding[, errors]])\n\n   Decodes the string using the codec registered for *encoding*.\n   *encoding* defaults to the default string encoding.  *errors* may\n   be given to set a different error handling scheme.  The default is\n   ``\'strict\'``, meaning that encoding errors raise ``UnicodeError``.\n   Other possible values are ``\'ignore\'``, ``\'replace\'`` and any other\n   name registered via ``codecs.register_error()``, see section *Codec\n   Base Classes*.\n\n   New in version 2.2.\n\n   Changed in version 2.3: Support for other error handling schemes\n   added.\n\n   Changed in version 2.7: Support for keyword arguments added.\n\nstr.encode([encoding[, errors]])\n\n   Return an encoded version of the string.  Default encoding is the\n   current default string encoding.  *errors* may be given to set a\n   different error handling scheme.  The default for *errors* is\n   ``\'strict\'``, meaning that encoding errors raise a\n   ``UnicodeError``.  Other possible values are ``\'ignore\'``,\n   ``\'replace\'``, ``\'xmlcharrefreplace\'``, ``\'backslashreplace\'`` and\n   any other name registered via ``codecs.register_error()``, see\n   section *Codec Base Classes*. For a list of possible encodings, see\n   section *Standard Encodings*.\n\n   New in version 2.0.\n\n   Changed in version 2.3: Support for ``\'xmlcharrefreplace\'`` and\n   ``\'backslashreplace\'`` and other error handling schemes added.\n\n   Changed in version 2.7: Support for keyword arguments added.\n\nstr.endswith(suffix[, start[, end]])\n\n   Return ``True`` if the string ends with the specified *suffix*,\n   otherwise return ``False``.  *suffix* can also be a tuple of\n   suffixes to look for.  With optional *start*, test beginning at\n   that position.  With optional *end*, stop comparing at that\n   position.\n\n   Changed in version 2.5: Accept tuples as *suffix*.\n\nstr.expandtabs([tabsize])\n\n   Return a copy of the string where all tab characters are replaced\n   by one or more spaces, depending on the current column and the\n   given tab size.  The column number is reset to zero after each\n   newline occurring in the string. If *tabsize* is not given, a tab\n   size of ``8`` characters is assumed.  This doesn\'t understand other\n   non-printing characters or escape sequences.\n\nstr.find(sub[, start[, end]])\n\n   Return the lowest index in the string where substring *sub* is\n   found, such that *sub* is contained in the slice ``s[start:end]``.\n   Optional arguments *start* and *end* are interpreted as in slice\n   notation.  Return ``-1`` if *sub* is not found.\n\n   Note: The ``find()`` method should be used only if you need to know the\n     position of *sub*.  To check if *sub* is a substring or not, use\n     the ``in`` operator:\n\n        >>> \'Py\' in \'Python\'\n        True\n\nstr.format(*args, **kwargs)\n\n   Perform a string formatting operation.  The string on which this\n   method is called can contain literal text or replacement fields\n   delimited by braces ``{}``.  Each replacement field contains either\n   the numeric index of a positional argument, or the name of a\n   keyword argument.  Returns a copy of the string where each\n   replacement field is replaced with the string value of the\n   corresponding argument.\n\n   >>> "The sum of 1 + 2 is {0}".format(1+2)\n   \'The sum of 1 + 2 is 3\'\n\n   See *Format String Syntax* for a description of the various\n   formatting options that can be specified in format strings.\n\n   This method of string formatting is the new standard in Python 3.0,\n   and should be preferred to the ``%`` formatting described in\n   *String Formatting Operations* in new code.\n\n   New in version 2.6.\n\nstr.index(sub[, start[, end]])\n\n   Like ``find()``, but raise ``ValueError`` when the substring is not\n   found.\n\nstr.isalnum()\n\n   Return true if all characters in the string are alphanumeric and\n   there is at least one character, false otherwise.\n\n   For 8-bit strings, this method is locale-dependent.\n\nstr.isalpha()\n\n   Return true if all characters in the string are alphabetic and\n   there is at least one character, false otherwise.\n\n   For 8-bit strings, this method is locale-dependent.\n\nstr.isdigit()\n\n   Return true if all characters in the string are digits and there is\n   at least one character, false otherwise.\n\n   For 8-bit strings, this method is locale-dependent.\n\nstr.islower()\n\n   Return true if all cased characters in the string are lowercase and\n   there is at least one cased character, false otherwise.\n\n   For 8-bit strings, this method is locale-dependent.\n\nstr.isspace()\n\n   Return true if there are only whitespace characters in the string\n   and there is at least one character, false otherwise.\n\n   For 8-bit strings, this method is locale-dependent.\n\nstr.istitle()\n\n   Return true if the string is a titlecased string and there is at\n   least one character, for example uppercase characters may only\n   follow uncased characters and lowercase characters only cased ones.\n   Return false otherwise.\n\n   For 8-bit strings, this method is locale-dependent.\n\nstr.isupper()\n\n   Return true if all cased characters in the string are uppercase and\n   there is at least one cased character, false otherwise.\n\n   For 8-bit strings, this method is locale-dependent.\n\nstr.join(iterable)\n\n   Return a string which is the concatenation of the strings in the\n   *iterable* *iterable*.  The separator between elements is the\n   string providing this method.\n\nstr.ljust(width[, fillchar])\n\n   Return the string left justified in a string of length *width*.\n   Padding is done using the specified *fillchar* (default is a\n   space).  The original string is returned if *width* is less than\n   ``len(s)``.\n\n   Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.lower()\n\n   Return a copy of the string converted to lowercase.\n\n   For 8-bit strings, this method is locale-dependent.\n\nstr.lstrip([chars])\n\n   Return a copy of the string with leading characters removed.  The\n   *chars* argument is a string specifying the set of characters to be\n   removed.  If omitted or ``None``, the *chars* argument defaults to\n   removing whitespace.  The *chars* argument is not a prefix; rather,\n   all combinations of its values are stripped:\n\n   >>> \'   spacious   \'.lstrip()\n   \'spacious   \'\n   >>> \'www.example.com\'.lstrip(\'cmowz.\')\n   \'example.com\'\n\n   Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.partition(sep)\n\n   Split the string at the first occurrence of *sep*, and return a\n   3-tuple containing the part before the separator, the separator\n   itself, and the part after the separator.  If the separator is not\n   found, return a 3-tuple containing the string itself, followed by\n   two empty strings.\n\n   New in version 2.5.\n\nstr.replace(old, new[, count])\n\n   Return a copy of the string with all occurrences of substring *old*\n   replaced by *new*.  If the optional argument *count* is given, only\n   the first *count* occurrences are replaced.\n\nstr.rfind(sub[, start[, end]])\n\n   Return the highest index in the string where substring *sub* is\n   found, such that *sub* is contained within ``s[start:end]``.\n   Optional arguments *start* and *end* are interpreted as in slice\n   notation.  Return ``-1`` on failure.\n\nstr.rindex(sub[, start[, end]])\n\n   Like ``rfind()`` but raises ``ValueError`` when the substring *sub*\n   is not found.\n\nstr.rjust(width[, fillchar])\n\n   Return the string right justified in a string of length *width*.\n   Padding is done using the specified *fillchar* (default is a\n   space). The original string is returned if *width* is less than\n   ``len(s)``.\n\n   Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.rpartition(sep)\n\n   Split the string at the last occurrence of *sep*, and return a\n   3-tuple containing the part before the separator, the separator\n   itself, and the part after the separator.  If the separator is not\n   found, return a 3-tuple containing two empty strings, followed by\n   the string itself.\n\n   New in version 2.5.\n\nstr.rsplit([sep[, maxsplit]])\n\n   Return a list of the words in the string, using *sep* as the\n   delimiter string. If *maxsplit* is given, at most *maxsplit* splits\n   are done, the *rightmost* ones.  If *sep* is not specified or\n   ``None``, any whitespace string is a separator.  Except for\n   splitting from the right, ``rsplit()`` behaves like ``split()``\n   which is described in detail below.\n\n   New in version 2.4.\n\nstr.rstrip([chars])\n\n   Return a copy of the string with trailing characters removed.  The\n   *chars* argument is a string specifying the set of characters to be\n   removed.  If omitted or ``None``, the *chars* argument defaults to\n   removing whitespace.  The *chars* argument is not a suffix; rather,\n   all combinations of its values are stripped:\n\n   >>> \'   spacious   \'.rstrip()\n   \'   spacious\'\n   >>> \'mississippi\'.rstrip(\'ipz\')\n   \'mississ\'\n\n   Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.split([sep[, maxsplit]])\n\n   Return a list of the words in the string, using *sep* as the\n   delimiter string.  If *maxsplit* is given, at most *maxsplit*\n   splits are done (thus, the list will have at most ``maxsplit+1``\n   elements).  If *maxsplit* is not specified, then there is no limit\n   on the number of splits (all possible splits are made).\n\n   If *sep* is given, consecutive delimiters are not grouped together\n   and are deemed to delimit empty strings (for example,\n   ``\'1,,2\'.split(\',\')`` returns ``[\'1\', \'\', \'2\']``).  The *sep*\n   argument may consist of multiple characters (for example,\n   ``\'1<>2<>3\'.split(\'<>\')`` returns ``[\'1\', \'2\', \'3\']``). Splitting\n   an empty string with a specified separator returns ``[\'\']``.\n\n   If *sep* is not specified or is ``None``, a different splitting\n   algorithm is applied: runs of consecutive whitespace are regarded\n   as a single separator, and the result will contain no empty strings\n   at the start or end if the string has leading or trailing\n   whitespace.  Consequently, splitting an empty string or a string\n   consisting of just whitespace with a ``None`` separator returns\n   ``[]``.\n\n   For example, ``\' 1  2   3  \'.split()`` returns ``[\'1\', \'2\', \'3\']``,\n   and ``\'  1  2   3  \'.split(None, 1)`` returns ``[\'1\', \'2   3  \']``.\n\nstr.splitlines([keepends])\n\n   Return a list of the lines in the string, breaking at line\n   boundaries.  Line breaks are not included in the resulting list\n   unless *keepends* is given and true.\n\nstr.startswith(prefix[, start[, end]])\n\n   Return ``True`` if string starts with the *prefix*, otherwise\n   return ``False``. *prefix* can also be a tuple of prefixes to look\n   for.  With optional *start*, test string beginning at that\n   position.  With optional *end*, stop comparing string at that\n   position.\n\n   Changed in version 2.5: Accept tuples as *prefix*.\n\nstr.strip([chars])\n\n   Return a copy of the string with the leading and trailing\n   characters removed. The *chars* argument is a string specifying the\n   set of characters to be removed. If omitted or ``None``, the\n   *chars* argument defaults to removing whitespace. The *chars*\n   argument is not a prefix or suffix; rather, all combinations of its\n   values are stripped:\n\n   >>> \'   spacious   \'.strip()\n   \'spacious\'\n   >>> \'www.example.com\'.strip(\'cmowz.\')\n   \'example\'\n\n   Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.swapcase()\n\n   Return a copy of the string with uppercase characters converted to\n   lowercase and vice versa.\n\n   For 8-bit strings, this method is locale-dependent.\n\nstr.title()\n\n   Return a titlecased version of the string where words start with an\n   uppercase character and the remaining characters are lowercase.\n\n   The algorithm uses a simple language-independent definition of a\n   word as groups of consecutive letters.  The definition works in\n   many contexts but it means that apostrophes in contractions and\n   possessives form word boundaries, which may not be the desired\n   result:\n\n      >>> "they\'re bill\'s friends from the UK".title()\n      "They\'Re Bill\'S Friends From The Uk"\n\n   A workaround for apostrophes can be constructed using regular\n   expressions:\n\n      >>> import re\n      >>> def titlecase(s):\n              return re.sub(r"[A-Za-z]+(\'[A-Za-z]+)?",\n                            lambda mo: mo.group(0)[0].upper() +\n                                       mo.group(0)[1:].lower(),\n                            s)\n\n      >>> titlecase("they\'re bill\'s friends.")\n      "They\'re Bill\'s Friends."\n\n   For 8-bit strings, this method is locale-dependent.\n\nstr.translate(table[, deletechars])\n\n   Return a copy of the string where all characters occurring in the\n   optional argument *deletechars* are removed, and the remaining\n   characters have been mapped through the given translation table,\n   which must be a string of length 256.\n\n   You can use the ``maketrans()`` helper function in the ``string``\n   module to create a translation table. For string objects, set the\n   *table* argument to ``None`` for translations that only delete\n   characters:\n\n   >>> \'read this short text\'.translate(None, \'aeiou\')\n   \'rd ths shrt txt\'\n\n   New in version 2.6: Support for a ``None`` *table* argument.\n\n   For Unicode objects, the ``translate()`` method does not accept the\n   optional *deletechars* argument.  Instead, it returns a copy of the\n   *s* where all characters have been mapped through the given\n   translation table which must be a mapping of Unicode ordinals to\n   Unicode ordinals, Unicode strings or ``None``. Unmapped characters\n   are left untouched. Characters mapped to ``None`` are deleted.\n   Note, a more flexible approach is to create a custom character\n   mapping codec using the ``codecs`` module (see ``encodings.cp1251``\n   for an example).\n\nstr.upper()\n\n   Return a copy of the string converted to uppercase.\n\n   For 8-bit strings, this method is locale-dependent.\n\nstr.zfill(width)\n\n   Return the numeric string left filled with zeros in a string of\n   length *width*.  A sign prefix is handled correctly.  The original\n   string is returned if *width* is less than ``len(s)``.\n\n   New in version 2.2.2.\n\nThe following methods are present only on unicode objects:\n\nunicode.isnumeric()\n\n   Return ``True`` if there are only numeric characters in S,\n   ``False`` otherwise. Numeric characters include digit characters,\n   and all characters that have the Unicode numeric value property,\n   e.g. U+2155, VULGAR FRACTION ONE FIFTH.\n\nunicode.isdecimal()\n\n   Return ``True`` if there are only decimal characters in S,\n   ``False`` otherwise. Decimal characters include digit characters,\n   and all characters that that can be used to form decimal-radix\n   numbers, e.g. U+0660, ARABIC-INDIC DIGIT ZERO.\n\n\nString Formatting Operations\n============================\n\nString and Unicode objects have one unique built-in operation: the\n``%`` operator (modulo).  This is also known as the string\n*formatting* or *interpolation* operator.  Given ``format % values``\n(where *format* is a string or Unicode object), ``%`` conversion\nspecifications in *format* are replaced with zero or more elements of\n*values*.  The effect is similar to the using ``sprintf()`` in the C\nlanguage.  If *format* is a Unicode object, or if any of the objects\nbeing converted using the ``%s`` conversion are Unicode objects, the\nresult will also be a Unicode object.\n\nIf *format* requires a single argument, *values* may be a single non-\ntuple object. [4]  Otherwise, *values* must be a tuple with exactly\nthe number of items specified by the format string, or a single\nmapping object (for example, a dictionary).\n\nA conversion specifier contains two or more characters and has the\nfollowing components, which must occur in this order:\n\n1. The ``\'%\'`` character, which marks the start of the specifier.\n\n2. Mapping key (optional), consisting of a parenthesised sequence of\n   characters (for example, ``(somename)``).\n\n3. Conversion flags (optional), which affect the result of some\n   conversion types.\n\n4. Minimum field width (optional).  If specified as an ``\'*\'``\n   (asterisk), the actual width is read from the next element of the\n   tuple in *values*, and the object to convert comes after the\n   minimum field width and optional precision.\n\n5. Precision (optional), given as a ``\'.\'`` (dot) followed by the\n   precision.  If specified as ``\'*\'`` (an asterisk), the actual width\n   is read from the next element of the tuple in *values*, and the\n   value to convert comes after the precision.\n\n6. Length modifier (optional).\n\n7. Conversion type.\n\nWhen the right argument is a dictionary (or other mapping type), then\nthe formats in the string *must* include a parenthesised mapping key\ninto that dictionary inserted immediately after the ``\'%\'`` character.\nThe mapping key selects the value to be formatted from the mapping.\nFor example:\n\n>>> print \'%(language)s has %(number)03d quote types.\' % \\\n...       {"language": "Python", "number": 2}\nPython has 002 quote types.\n\nIn this case no ``*`` specifiers may occur in a format (since they\nrequire a sequential parameter list).\n\nThe conversion flag characters are:\n\n+-----------+-----------------------------------------------------------------------+\n| Flag      | Meaning                                                               |\n+===========+=======================================================================+\n| ``\'#\'``   | The value conversion will use the "alternate form" (where defined     |\n|           | below).                                                               |\n+-----------+-----------------------------------------------------------------------+\n| ``\'0\'``   | The conversion will be zero padded for numeric values.                |\n+-----------+-----------------------------------------------------------------------+\n| ``\'-\'``   | The converted value is left adjusted (overrides the ``\'0\'``           |\n|           | conversion if both are given).                                        |\n+-----------+-----------------------------------------------------------------------+\n| ``\' \'``   | (a space) A blank should be left before a positive number (or empty   |\n|           | string) produced by a signed conversion.                              |\n+-----------+-----------------------------------------------------------------------+\n| ``\'+\'``   | A sign character (``\'+\'`` or ``\'-\'``) will precede the conversion     |\n|           | (overrides a "space" flag).                                           |\n+-----------+-----------------------------------------------------------------------+\n\nA length modifier (``h``, ``l``, or ``L``) may be present, but is\nignored as it is not necessary for Python -- so e.g. ``%ld`` is\nidentical to ``%d``.\n\nThe conversion types are:\n\n+--------------+-------------------------------------------------------+---------+\n| Conversion   | Meaning                                               | Notes   |\n+==============+=======================================================+=========+\n| ``\'d\'``      | Signed integer decimal.                               |         |\n+--------------+-------------------------------------------------------+---------+\n| ``\'i\'``      | Signed integer decimal.                               |         |\n+--------------+-------------------------------------------------------+---------+\n| ``\'o\'``      | Signed octal value.                                   | (1)     |\n+--------------+-------------------------------------------------------+---------+\n| ``\'u\'``      | Obsolete type -- it is identical to ``\'d\'``.          | (7)     |\n+--------------+-------------------------------------------------------+---------+\n| ``\'x\'``      | Signed hexadecimal (lowercase).                       | (2)     |\n+--------------+-------------------------------------------------------+---------+\n| ``\'X\'``      | Signed hexadecimal (uppercase).                       | (2)     |\n+--------------+-------------------------------------------------------+---------+\n| ``\'e\'``      | Floating point exponential format (lowercase).        | (3)     |\n+--------------+-------------------------------------------------------+---------+\n| ``\'E\'``      | Floating point exponential format (uppercase).        | (3)     |\n+--------------+-------------------------------------------------------+---------+\n| ``\'f\'``      | Floating point decimal format.                        | (3)     |\n+--------------+-------------------------------------------------------+---------+\n| ``\'F\'``      | Floating point decimal format.                        | (3)     |\n+--------------+-------------------------------------------------------+---------+\n| ``\'g\'``      | Floating point format. Uses lowercase exponential     | (4)     |\n|              | format if exponent is less than -4 or not less than   |         |\n|              | precision, decimal format otherwise.                  |         |\n+--------------+-------------------------------------------------------+---------+\n| ``\'G\'``      | Floating point format. Uses uppercase exponential     | (4)     |\n|              | format if exponent is less than -4 or not less than   |         |\n|              | precision, decimal format otherwise.                  |         |\n+--------------+-------------------------------------------------------+---------+\n| ``\'c\'``      | Single character (accepts integer or single character |         |\n|              | string).                                              |         |\n+--------------+-------------------------------------------------------+---------+\n| ``\'r\'``      | String (converts any Python object using ``repr()``). | (5)     |\n+--------------+-------------------------------------------------------+---------+\n| ``\'s\'``      | String (converts any Python object using ``str()``).  | (6)     |\n+--------------+-------------------------------------------------------+---------+\n| ``\'%\'``      | No argument is converted, results in a ``\'%\'``        |         |\n|              | character in the result.                              |         |\n+--------------+-------------------------------------------------------+---------+\n\nNotes:\n\n1. The alternate form causes a leading zero (``\'0\'``) to be inserted\n   between left-hand padding and the formatting of the number if the\n   leading character of the result is not already a zero.\n\n2. The alternate form causes a leading ``\'0x\'`` or ``\'0X\'`` (depending\n   on whether the ``\'x\'`` or ``\'X\'`` format was used) to be inserted\n   between left-hand padding and the formatting of the number if the\n   leading character of the result is not already a zero.\n\n3. The alternate form causes the result to always contain a decimal\n   point, even if no digits follow it.\n\n   The precision determines the number of digits after the decimal\n   point and defaults to 6.\n\n4. The alternate form causes the result to always contain a decimal\n   point, and trailing zeroes are not removed as they would otherwise\n   be.\n\n   The precision determines the number of significant digits before\n   and after the decimal point and defaults to 6.\n\n5. The ``%r`` conversion was added in Python 2.0.\n\n   The precision determines the maximal number of characters used.\n\n6. If the object or format provided is a ``unicode`` string, the\n   resulting string will also be ``unicode``.\n\n   The precision determines the maximal number of characters used.\n\n7. See **PEP 237**.\n\nSince Python strings have an explicit length, ``%s`` conversions do\nnot assume that ``\'\\0\'`` is the end of the string.\n\nChanged in version 2.7: ``%f`` conversions for numbers whose absolute\nvalue is over 1e50 are no longer replaced by ``%g`` conversions.\n\nAdditional string operations are defined in standard modules\n``string`` and ``re``.\n\n\nXRange Type\n===========\n\nThe ``xrange`` type is an immutable sequence which is commonly used\nfor looping.  The advantage of the ``xrange`` type is that an\n``xrange`` object will always take the same amount of memory, no\nmatter the size of the range it represents.  There are no consistent\nperformance advantages.\n\nXRange objects have very little behavior: they only support indexing,\niteration, and the ``len()`` function.\n\n\nMutable Sequence Types\n======================\n\nList and ``bytearray`` objects support additional operations that\nallow in-place modification of the object. Other mutable sequence\ntypes (when added to the language) should also support these\noperations. Strings and tuples are immutable sequence types: such\nobjects cannot be modified once created. The following operations are\ndefined on mutable sequence types (where *x* is an arbitrary object):\n\n+--------------------------------+----------------------------------+-----------------------+\n| Operation                      | Result                           | Notes                 |\n+================================+==================================+=======================+\n| ``s[i] = x``                   | item *i* of *s* is replaced by   |                       |\n|                                | *x*                              |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s[i:j] = t``                 | slice of *s* from *i* to *j* is  |                       |\n|                                | replaced by the contents of the  |                       |\n|                                | iterable *t*                     |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``del s[i:j]``                 | same as ``s[i:j] = []``          |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s[i:j:k] = t``               | the elements of ``s[i:j:k]`` are | (1)                   |\n|                                | replaced by those of *t*         |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``del s[i:j:k]``               | removes the elements of          |                       |\n|                                | ``s[i:j:k]`` from the list       |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.append(x)``                | same as ``s[len(s):len(s)] =     | (2)                   |\n|                                | [x]``                            |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.extend(x)``                | same as ``s[len(s):len(s)] = x`` | (3)                   |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.count(x)``                 | return number of *i*\'s for which |                       |\n|                                | ``s[i] == x``                    |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.index(x[, i[, j]])``       | return smallest *k* such that    | (4)                   |\n|                                | ``s[k] == x`` and ``i <= k < j`` |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.insert(i, x)``             | same as ``s[i:i] = [x]``         | (5)                   |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.pop([i])``                 | same as ``x = s[i]; del s[i];    | (6)                   |\n|                                | return x``                       |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.remove(x)``                | same as ``del s[s.index(x)]``    | (4)                   |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.reverse()``                | reverses the items of *s* in     | (7)                   |\n|                                | place                            |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.sort([cmp[, key[,          | sort the items of *s* in place   | (7)(8)(9)(10)         |\n| reverse]]])``                  |                                  |                       |\n+--------------------------------+----------------------------------+-----------------------+\n\nNotes:\n\n1. *t* must have the same length as the slice it is  replacing.\n\n2. The C implementation of Python has historically accepted multiple\n   parameters and implicitly joined them into a tuple; this no longer\n   works in Python 2.0.  Use of this misfeature has been deprecated\n   since Python 1.4.\n\n3. *x* can be any iterable object.\n\n4. Raises ``ValueError`` when *x* is not found in *s*. When a negative\n   index is passed as the second or third parameter to the ``index()``\n   method, the list length is added, as for slice indices.  If it is\n   still negative, it is truncated to zero, as for slice indices.\n\n   Changed in version 2.3: Previously, ``index()`` didn\'t have\n   arguments for specifying start and stop positions.\n\n5. When a negative index is passed as the first parameter to the\n   ``insert()`` method, the list length is added, as for slice\n   indices.  If it is still negative, it is truncated to zero, as for\n   slice indices.\n\n   Changed in version 2.3: Previously, all negative indices were\n   truncated to zero.\n\n6. The ``pop()`` method is only supported by the list and array types.\n   The optional argument *i* defaults to ``-1``, so that by default\n   the last item is removed and returned.\n\n7. The ``sort()`` and ``reverse()`` methods modify the list in place\n   for economy of space when sorting or reversing a large list.  To\n   remind you that they operate by side effect, they don\'t return the\n   sorted or reversed list.\n\n8. The ``sort()`` method takes optional arguments for controlling the\n   comparisons.\n\n   *cmp* specifies a custom comparison function of two arguments (list\n   items) which should return a negative, zero or positive number\n   depending on whether the first argument is considered smaller than,\n   equal to, or larger than the second argument: ``cmp=lambda x,y:\n   cmp(x.lower(), y.lower())``.  The default value is ``None``.\n\n   *key* specifies a function of one argument that is used to extract\n   a comparison key from each list element: ``key=str.lower``.  The\n   default value is ``None``.\n\n   *reverse* is a boolean value.  If set to ``True``, then the list\n   elements are sorted as if each comparison were reversed.\n\n   In general, the *key* and *reverse* conversion processes are much\n   faster than specifying an equivalent *cmp* function.  This is\n   because *cmp* is called multiple times for each list element while\n   *key* and *reverse* touch each element only once.  Use\n   ``functools.cmp_to_key()`` to convert an old-style *cmp* function\n   to a *key* function.\n\n   Changed in version 2.3: Support for ``None`` as an equivalent to\n   omitting *cmp* was added.\n\n   Changed in version 2.4: Support for *key* and *reverse* was added.\n\n9. Starting with Python 2.3, the ``sort()`` method is guaranteed to be\n   stable.  A sort is stable if it guarantees not to change the\n   relative order of elements that compare equal --- this is helpful\n   for sorting in multiple passes (for example, sort by department,\n   then by salary grade).\n\n10. **CPython implementation detail:** While a list is being sorted,\n    the effect of attempting to mutate, or even inspect, the list is\n    undefined.  The C implementation of Python 2.3 and newer makes the\n    list appear empty for the duration, and raises ``ValueError`` if\n    it can detect that the list has been mutated during a sort.\n',
+ 'typesseq-mutable': u"\nMutable Sequence Types\n**********************\n\nList and ``bytearray`` objects support additional operations that\nallow in-place modification of the object. Other mutable sequence\ntypes (when added to the language) should also support these\noperations. Strings and tuples are immutable sequence types: such\nobjects cannot be modified once created. The following operations are\ndefined on mutable sequence types (where *x* is an arbitrary object):\n\n+--------------------------------+----------------------------------+-----------------------+\n| Operation                      | Result                           | Notes                 |\n+================================+==================================+=======================+\n| ``s[i] = x``                   | item *i* of *s* is replaced by   |                       |\n|                                | *x*                              |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s[i:j] = t``                 | slice of *s* from *i* to *j* is  |                       |\n|                                | replaced by the contents of the  |                       |\n|                                | iterable *t*                     |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``del s[i:j]``                 | same as ``s[i:j] = []``          |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s[i:j:k] = t``               | the elements of ``s[i:j:k]`` are | (1)                   |\n|                                | replaced by those of *t*         |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``del s[i:j:k]``               | removes the elements of          |                       |\n|                                | ``s[i:j:k]`` from the list       |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.append(x)``                | same as ``s[len(s):len(s)] =     | (2)                   |\n|                                | [x]``                            |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.extend(x)``                | same as ``s[len(s):len(s)] = x`` | (3)                   |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.count(x)``                 | return number of *i*'s for which |                       |\n|                                | ``s[i] == x``                    |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.index(x[, i[, j]])``       | return smallest *k* such that    | (4)                   |\n|                                | ``s[k] == x`` and ``i <= k < j`` |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.insert(i, x)``             | same as ``s[i:i] = [x]``         | (5)                   |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.pop([i])``                 | same as ``x = s[i]; del s[i];    | (6)                   |\n|                                | return x``                       |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.remove(x)``                | same as ``del s[s.index(x)]``    | (4)                   |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.reverse()``                | reverses the items of *s* in     | (7)                   |\n|                                | place                            |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.sort([cmp[, key[,          | sort the items of *s* in place   | (7)(8)(9)(10)         |\n| reverse]]])``                  |                                  |                       |\n+--------------------------------+----------------------------------+-----------------------+\n\nNotes:\n\n1. *t* must have the same length as the slice it is  replacing.\n\n2. The C implementation of Python has historically accepted multiple\n   parameters and implicitly joined them into a tuple; this no longer\n   works in Python 2.0.  Use of this misfeature has been deprecated\n   since Python 1.4.\n\n3. *x* can be any iterable object.\n\n4. Raises ``ValueError`` when *x* is not found in *s*. When a negative\n   index is passed as the second or third parameter to the ``index()``\n   method, the list length is added, as for slice indices.  If it is\n   still negative, it is truncated to zero, as for slice indices.\n\n   Changed in version 2.3: Previously, ``index()`` didn't have\n   arguments for specifying start and stop positions.\n\n5. When a negative index is passed as the first parameter to the\n   ``insert()`` method, the list length is added, as for slice\n   indices.  If it is still negative, it is truncated to zero, as for\n   slice indices.\n\n   Changed in version 2.3: Previously, all negative indices were\n   truncated to zero.\n\n6. The ``pop()`` method is only supported by the list and array types.\n   The optional argument *i* defaults to ``-1``, so that by default\n   the last item is removed and returned.\n\n7. The ``sort()`` and ``reverse()`` methods modify the list in place\n   for economy of space when sorting or reversing a large list.  To\n   remind you that they operate by side effect, they don't return the\n   sorted or reversed list.\n\n8. The ``sort()`` method takes optional arguments for controlling the\n   comparisons.\n\n   *cmp* specifies a custom comparison function of two arguments (list\n   items) which should return a negative, zero or positive number\n   depending on whether the first argument is considered smaller than,\n   equal to, or larger than the second argument: ``cmp=lambda x,y:\n   cmp(x.lower(), y.lower())``.  The default value is ``None``.\n\n   *key* specifies a function of one argument that is used to extract\n   a comparison key from each list element: ``key=str.lower``.  The\n   default value is ``None``.\n\n   *reverse* is a boolean value.  If set to ``True``, then the list\n   elements are sorted as if each comparison were reversed.\n\n   In general, the *key* and *reverse* conversion processes are much\n   faster than specifying an equivalent *cmp* function.  This is\n   because *cmp* is called multiple times for each list element while\n   *key* and *reverse* touch each element only once.  Use\n   ``functools.cmp_to_key()`` to convert an old-style *cmp* function\n   to a *key* function.\n\n   Changed in version 2.3: Support for ``None`` as an equivalent to\n   omitting *cmp* was added.\n\n   Changed in version 2.4: Support for *key* and *reverse* was added.\n\n9. Starting with Python 2.3, the ``sort()`` method is guaranteed to be\n   stable.  A sort is stable if it guarantees not to change the\n   relative order of elements that compare equal --- this is helpful\n   for sorting in multiple passes (for example, sort by department,\n   then by salary grade).\n\n10. **CPython implementation detail:** While a list is being sorted,\n    the effect of attempting to mutate, or even inspect, the list is\n    undefined.  The C implementation of Python 2.3 and newer makes the\n    list appear empty for the duration, and raises ``ValueError`` if\n    it can detect that the list has been mutated during a sort.\n",
  'unary': u'\nUnary arithmetic and bitwise operations\n***************************************\n\nAll unary arithmetic and bitwise operations have the same priority:\n\n   u_expr ::= power | "-" u_expr | "+" u_expr | "~" u_expr\n\nThe unary ``-`` (minus) operator yields the negation of its numeric\nargument.\n\nThe unary ``+`` (plus) operator yields its numeric argument unchanged.\n\nThe unary ``~`` (invert) operator yields the bitwise inversion of its\nplain or long integer argument.  The bitwise inversion of ``x`` is\ndefined as ``-(x+1)``.  It only applies to integral numbers.\n\nIn all three cases, if the argument does not have the proper type, a\n``TypeError`` exception is raised.\n',
  'while': u'\nThe ``while`` statement\n***********************\n\nThe ``while`` statement is used for repeated execution as long as an\nexpression is true:\n\n   while_stmt ::= "while" expression ":" suite\n                  ["else" ":" suite]\n\nThis repeatedly tests the expression and, if it is true, executes the\nfirst suite; if the expression is false (which may be the first time\nit is tested) the suite of the ``else`` clause, if present, is\nexecuted and the loop terminates.\n\nA ``break`` statement executed in the first suite terminates the loop\nwithout executing the ``else`` clause\'s suite.  A ``continue``\nstatement executed in the first suite skips the rest of the suite and\ngoes back to testing the expression.\n',
- 'with': u'\nThe ``with`` statement\n**********************\n\nNew in version 2.5.\n\nThe ``with`` statement is used to wrap the execution of a block with\nmethods defined by a context manager (see section *With Statement\nContext Managers*). This allows common\n``try``...``except``...``finally`` usage patterns to be encapsulated\nfor convenient reuse.\n\n   with_stmt ::= "with" with_item ("," with_item)* ":" suite\n   with_item ::= expression ["as" target]\n\nThe execution of the ``with`` statement with one "item" proceeds as\nfollows:\n\n1. The context expression is evaluated to obtain a context manager.\n\n2. The context manager\'s ``__exit__()`` is loaded for later use.\n\n3. The context manager\'s ``__enter__()`` method is invoked.\n\n4. If a target was included in the ``with`` statement, the return\n   value from ``__enter__()`` is assigned to it.\n\n   Note: The ``with`` statement guarantees that if the ``__enter__()``\n     method returns without an error, then ``__exit__()`` will always\n     be called. Thus, if an error occurs during the assignment to the\n     target list, it will be treated the same as an error occurring\n     within the suite would be. See step 6 below.\n\n5. The suite is executed.\n\n6. The context manager\'s ``__exit__()`` method is invoked. If an\n   exception caused the suite to be exited, its type, value, and\n   traceback are passed as arguments to ``__exit__()``. Otherwise,\n   three ``None`` arguments are supplied.\n\n   If the suite was exited due to an exception, and the return value\n   from the ``__exit__()`` method was false, the exception is\n   reraised. If the return value was true, the exception is\n   suppressed, and execution continues with the statement following\n   the ``with`` statement.\n\n   If the suite was exited for any reason other than an exception, the\n   return value from ``__exit__()`` is ignored, and execution proceeds\n   at the normal location for the kind of exit that was taken.\n\nWith more than one item, the context managers are processed as if\nmultiple ``with`` statements were nested:\n\n   with A() as a, B() as b:\n       suite\n\nis equivalent to\n\n   with A() as a:\n       with B() as b:\n           suite\n\nNote: In Python 2.5, the ``with`` statement is only allowed when the\n  ``with_statement`` feature has been enabled.  It is always enabled\n  in Python 2.6.\n\nChanged in version 2.7: Support for multiple context expressions.\n\nSee also:\n\n   **PEP 0343** - The "with" statement\n      The specification, background, and examples for the Python\n      ``with`` statement.\n',
+ 'with': u'\nThe ``with`` statement\n**********************\n\nNew in version 2.5.\n\nThe ``with`` statement is used to wrap the execution of a block with\nmethods defined by a context manager (see section *With Statement\nContext Managers*). This allows common\n``try``...``except``...``finally`` usage patterns to be encapsulated\nfor convenient reuse.\n\n   with_stmt ::= "with" with_item ("," with_item)* ":" suite\n   with_item ::= expression ["as" target]\n\nThe execution of the ``with`` statement with one "item" proceeds as\nfollows:\n\n1. The context expression (the expression given in the **with_item**)\n   is evaluated to obtain a context manager.\n\n2. The context manager\'s ``__exit__()`` is loaded for later use.\n\n3. The context manager\'s ``__enter__()`` method is invoked.\n\n4. If a target was included in the ``with`` statement, the return\n   value from ``__enter__()`` is assigned to it.\n\n   Note: The ``with`` statement guarantees that if the ``__enter__()``\n     method returns without an error, then ``__exit__()`` will always\n     be called. Thus, if an error occurs during the assignment to the\n     target list, it will be treated the same as an error occurring\n     within the suite would be. See step 6 below.\n\n5. The suite is executed.\n\n6. The context manager\'s ``__exit__()`` method is invoked. If an\n   exception caused the suite to be exited, its type, value, and\n   traceback are passed as arguments to ``__exit__()``. Otherwise,\n   three ``None`` arguments are supplied.\n\n   If the suite was exited due to an exception, and the return value\n   from the ``__exit__()`` method was false, the exception is\n   reraised. If the return value was true, the exception is\n   suppressed, and execution continues with the statement following\n   the ``with`` statement.\n\n   If the suite was exited for any reason other than an exception, the\n   return value from ``__exit__()`` is ignored, and execution proceeds\n   at the normal location for the kind of exit that was taken.\n\nWith more than one item, the context managers are processed as if\nmultiple ``with`` statements were nested:\n\n   with A() as a, B() as b:\n       suite\n\nis equivalent to\n\n   with A() as a:\n       with B() as b:\n           suite\n\nNote: In Python 2.5, the ``with`` statement is only allowed when the\n  ``with_statement`` feature has been enabled.  It is always enabled\n  in Python 2.6.\n\nChanged in version 2.7: Support for multiple context expressions.\n\nSee also:\n\n   **PEP 0343** - The "with" statement\n      The specification, background, and examples for the Python\n      ``with`` statement.\n',
  'yield': u'\nThe ``yield`` statement\n***********************\n\n   yield_stmt ::= yield_expression\n\nThe ``yield`` statement is only used when defining a generator\nfunction, and is only used in the body of the generator function.\nUsing a ``yield`` statement in a function definition is sufficient to\ncause that definition to create a generator function instead of a\nnormal function.\n\nWhen a generator function is called, it returns an iterator known as a\ngenerator iterator, or more commonly, a generator.  The body of the\ngenerator function is executed by calling the generator\'s ``next()``\nmethod repeatedly until it raises an exception.\n\nWhen a ``yield`` statement is executed, the state of the generator is\nfrozen and the value of **expression_list** is returned to\n``next()``\'s caller.  By "frozen" we mean that all local state is\nretained, including the current bindings of local variables, the\ninstruction pointer, and the internal evaluation stack: enough\ninformation is saved so that the next time ``next()`` is invoked, the\nfunction can proceed exactly as if the ``yield`` statement were just\nanother external call.\n\nAs of Python version 2.5, the ``yield`` statement is now allowed in\nthe ``try`` clause of a ``try`` ...  ``finally`` construct.  If the\ngenerator is not resumed before it is finalized (by reaching a zero\nreference count or by being garbage collected), the generator-\niterator\'s ``close()`` method will be called, allowing any pending\n``finally`` clauses to execute.\n\nNote: In Python 2.2, the ``yield`` statement was only allowed when the\n  ``generators`` feature has been enabled.  This ``__future__`` import\n  statement was used to enable the feature:\n\n     from __future__ import generators\n\nSee also:\n\n   **PEP 0255** - Simple Generators\n      The proposal for adding generators and the ``yield`` statement\n      to Python.\n\n   **PEP 0342** - Coroutines via Enhanced Generators\n      The proposal that, among other generator enhancements, proposed\n      allowing ``yield`` to appear inside a ``try`` ... ``finally``\n      block.\n'}
diff --git a/lib-python/2.7/random.py b/lib-python/2.7/random.py
--- a/lib-python/2.7/random.py
+++ b/lib-python/2.7/random.py
@@ -317,7 +317,7 @@
 
         n = len(population)
         if not 0 <= k <= n:
-            raise ValueError, "sample larger than population"
+            raise ValueError("sample larger than population")
         random = self.random
         _int = int
         result = [None] * k
@@ -490,6 +490,12 @@
 
         Conditions on the parameters are alpha > 0 and beta > 0.
 
+        The probability distribution function is:
+
+                    x ** (alpha - 1) * math.exp(-x / beta)
+          pdf(x) =  --------------------------------------
+                      math.gamma(alpha) * beta ** alpha
+
         """
 
         # alpha > 0, beta > 0, mean is alpha*beta, variance is alpha*beta**2
@@ -592,7 +598,7 @@
 
 ## -------------------- beta --------------------
 ## See
-## http://sourceforge.net/bugs/?func=detailbug&bug_id=130030&group_id=5470
+## http://mail.python.org/pipermail/python-bugs-list/2001-January/003752.html
 ## for Ivan Frohne's insightful analysis of why the original implementation:
 ##
 ##    def betavariate(self, alpha, beta):
diff --git a/lib-python/2.7/re.py b/lib-python/2.7/re.py
--- a/lib-python/2.7/re.py
+++ b/lib-python/2.7/re.py
@@ -207,8 +207,7 @@
     "Escape all non-alphanumeric characters in pattern."
     s = list(pattern)
     alphanum = _alphanum
-    for i in range(len(pattern)):
-        c = pattern[i]
+    for i, c in enumerate(pattern):
         if c not in alphanum:
             if c == "\000":
                 s[i] = "\\000"
diff --git a/lib-python/2.7/shutil.py b/lib-python/2.7/shutil.py
--- a/lib-python/2.7/shutil.py
+++ b/lib-python/2.7/shutil.py
@@ -277,6 +277,12 @@
     """
     real_dst = dst
     if os.path.isdir(dst):
+        if _samefile(src, dst):
+            # We might be on a case insensitive filesystem,
+            # perform the rename anyway.
+            os.rename(src, dst)
+            return
+
         real_dst = os.path.join(dst, _basename(src))
         if os.path.exists(real_dst):
             raise Error, "Destination path '%s' already exists" % real_dst
@@ -336,7 +342,7 @@
     archive that is being built. If not provided, the current owner and group
     will be used.
 
-    The output tar file will be named 'base_dir' +  ".tar", possibly plus
+    The output tar file will be named 'base_name' +  ".tar", possibly plus
     the appropriate compression extension (".gz", or ".bz2").
 
     Returns the output filename.
@@ -406,7 +412,7 @@
 def _make_zipfile(base_name, base_dir, verbose=0, dry_run=0, logger=None):
     """Create a zip file from all the files under 'base_dir'.
 
-    The output zip file will be named 'base_dir' + ".zip".  Uses either the
+    The output zip file will be named 'base_name' + ".zip".  Uses either the
     "zipfile" Python module (if available) or the InfoZIP "zip" utility
     (if installed and found on the default search path).  If neither tool is
     available, raises ExecError.  Returns the name of the output zip
diff --git a/lib-python/2.7/site.py b/lib-python/2.7/site.py
--- a/lib-python/2.7/site.py
+++ b/lib-python/2.7/site.py
@@ -61,6 +61,7 @@
 import sys
 import os
 import __builtin__
+import traceback
 
 # Prefixes for site-packages; add additional prefixes like /usr/local here
 PREFIXES = [sys.prefix, sys.exec_prefix]
@@ -155,17 +156,26 @@
     except IOError:
         return
     with f:
-        for line in f:
+        for n, line in enumerate(f):
             if line.startswith("#"):
                 continue
-            if line.startswith(("import ", "import\t")):
-                exec line
-                continue
-            line = line.rstrip()
-            dir, dircase = makepath(sitedir, line)
-            if not dircase in known_paths and os.path.exists(dir):
-                sys.path.append(dir)
-                known_paths.add(dircase)
+            try:
+                if line.startswith(("import ", "import\t")):
+                    exec line
+                    continue
+                line = line.rstrip()
+                dir, dircase = makepath(sitedir, line)
+                if not dircase in known_paths and os.path.exists(dir):
+                    sys.path.append(dir)
+                    known_paths.add(dircase)
+            except Exception as err:
+                print >>sys.stderr, "Error processing line {:d} of {}:\n".format(
+                    n+1, fullname)
+                for record in traceback.format_exception(*sys.exc_info()):
+                    for line in record.splitlines():
+                        print >>sys.stderr, '  '+line
+                print >>sys.stderr, "\nRemainder of file ignored"
+                break
     if reset:
         known_paths = None
     return known_paths
diff --git a/lib-python/2.7/smtplib.py b/lib-python/2.7/smtplib.py
--- a/lib-python/2.7/smtplib.py
+++ b/lib-python/2.7/smtplib.py
@@ -49,17 +49,18 @@
 from email.base64mime import encode as encode_base64
 from sys import stderr
 
-__all__ = ["SMTPException","SMTPServerDisconnected","SMTPResponseException",
-           "SMTPSenderRefused","SMTPRecipientsRefused","SMTPDataError",
-           "SMTPConnectError","SMTPHeloError","SMTPAuthenticationError",
-           "quoteaddr","quotedata","SMTP"]
+__all__ = ["SMTPException", "SMTPServerDisconnected", "SMTPResponseException",
+           "SMTPSenderRefused", "SMTPRecipientsRefused", "SMTPDataError",
+           "SMTPConnectError", "SMTPHeloError", "SMTPAuthenticationError",
+           "quoteaddr", "quotedata", "SMTP"]
 
 SMTP_PORT = 25
 SMTP_SSL_PORT = 465
-CRLF="\r\n"
+CRLF = "\r\n"
 
 OLDSTYLE_AUTH = re.compile(r"auth=(.*)", re.I)
 
+
 # Exception classes used by this module.
 class SMTPException(Exception):
     """Base class for all exceptions raised by this module."""
@@ -109,7 +110,7 @@
 
     def __init__(self, recipients):
         self.recipients = recipients
-        self.args = ( recipients,)
+        self.args = (recipients,)
 
 
 class SMTPDataError(SMTPResponseException):
@@ -128,6 +129,7 @@
     combination provided.
     """
 
+
 def quoteaddr(addr):
     """Quote a subset of the email addresses defined by RFC 821.
 
@@ -138,7 +140,7 @@
         m = email.utils.parseaddr(addr)[1]
     except AttributeError:
         pass
-    if m == (None, None): # Indicates parse failure or AttributeError
+    if m == (None, None):  # Indicates parse failure or AttributeError
         # something weird here.. punt -ddm
         return "<%s>" % addr
     elif m is None:
@@ -175,7 +177,8 @@
             chr = None
             while chr != "\n":
                 chr = self.sslobj.read(1)
-                if not chr: break
+                if not chr:
+                    break
                 str += chr
             return str
 
@@ -219,6 +222,7 @@
     ehlo_msg = "ehlo"
     ehlo_resp = None
     does_esmtp = 0
+    default_port = SMTP_PORT
 
     def __init__(self, host='', port=0, local_hostname=None,
                  timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
@@ -234,7 +238,6 @@
         """
         self.timeout = timeout
         self.esmtp_features = {}
-        self.default_port = SMTP_PORT
         if host:
             (code, msg) = self.connect(host, port)
             if code != 220:
@@ -269,10 +272,11 @@
     def _get_socket(self, port, host, timeout):
         # This makes it simpler for SMTP_SSL to use the SMTP connect code
         # and just alter the socket connection bit.
-        if self.debuglevel > 0: print>>stderr, 'connect:', (host, port)
+        if self.debuglevel > 0:
+            print>>stderr, 'connect:', (host, port)
         return socket.create_connection((port, host), timeout)
 
-    def connect(self, host='localhost', port = 0):
+    def connect(self, host='localhost', port=0):
         """Connect to a host on a given port.
 
         If the hostname ends with a colon (`:') followed by a number, and
@@ -286,20 +290,25 @@
         if not port and (host.find(':') == host.rfind(':')):
             i = host.rfind(':')
             if i >= 0:
-                host, port = host[:i], host[i+1:]
-                try: port = int(port)
+                host, port = host[:i], host[i + 1:]
+                try:
+                    port = int(port)
                 except ValueError:
                     raise socket.error, "nonnumeric port"
-        if not port: port = self.default_port
-        if self.debuglevel > 0: print>>stderr, 'connect:', (host, port)
+        if not port:
+            port = self.default_port
+        if self.debuglevel > 0:
+            print>>stderr, 'connect:', (host, port)
         self.sock = self._get_socket(host, port, self.timeout)
         (code, msg) = self.getreply()
-        if self.debuglevel > 0: print>>stderr, "connect:", msg
+        if self.debuglevel > 0:
+            print>>stderr, "connect:", msg
         return (code, msg)
 
     def send(self, str):
         """Send `str' to the server."""
-        if self.debuglevel > 0: print>>stderr, 'send:', repr(str)
+        if self.debuglevel > 0:
+            print>>stderr, 'send:', repr(str)
         if hasattr(self, 'sock') and self.sock:
             try:
                 self.sock.sendall(str)
@@ -330,7 +339,7 @@
 
         Raises SMTPServerDisconnected if end-of-file is reached.
         """
-        resp=[]
+        resp = []
         if self.file is None:
             self.file = self.sock.makefile('rb')
         while 1:
@@ -341,9 +350,10 @@
             if line == '':
                 self.close()
                 raise SMTPServerDisconnected("Connection unexpectedly closed")
-            if self.debuglevel > 0: print>>stderr, 'reply:', repr(line)
+            if self.debuglevel > 0:
+                print>>stderr, 'reply:', repr(line)
             resp.append(line[4:].strip())
-            code=line[:3]
+            code = line[:3]
             # Check that the error code is syntactically correct.
             # Don't attempt to read a continuation line if it is broken.
             try:
@@ -352,17 +362,17 @@
                 errcode = -1
                 break
             # Check if multiline response.
-            if line[3:4]!="-":
+            if line[3:4] != "-":
                 break
 
         errmsg = "\n".join(resp)
         if self.debuglevel > 0:
-            print>>stderr, 'reply: retcode (%s); Msg: %s' % (errcode,errmsg)
+            print>>stderr, 'reply: retcode (%s); Msg: %s' % (errcode, errmsg)
         return errcode, errmsg
 
     def docmd(self, cmd, args=""):
         """Send a command, and return its response code."""
-        self.putcmd(cmd,args)
+        self.putcmd(cmd, args)
         return self.getreply()
 
     # std smtp commands
@@ -372,9 +382,9 @@
         host.
         """
         self.putcmd("helo", name or self.local_hostname)
-        (code,msg)=self.getreply()
-        self.helo_resp=msg
-        return (code,msg)
+        (code, msg) = self.getreply()
+        self.helo_resp = msg
+        return (code, msg)
 
     def ehlo(self, name=''):
         """ SMTP 'ehlo' command.
@@ -383,19 +393,19 @@
         """
         self.esmtp_features = {}
         self.putcmd(self.ehlo_msg, name or self.local_hostname)
-        (code,msg)=self.getreply()
+        (code, msg) = self.getreply()
         # According to RFC1869 some (badly written)
         # MTA's will disconnect on an ehlo. Toss an exception if
         # that happens -ddm
         if code == -1 and len(msg) == 0:
             self.close()
             raise SMTPServerDisconnected("Server not connected")
-        self.ehlo_resp=msg
+        self.ehlo_resp = msg
         if code != 250:
-            return (code,msg)
-        self.does_esmtp=1
+            return (code, msg)
+        self.does_esmtp = 1
         #parse the ehlo response -ddm
-        resp=self.ehlo_resp.split('\n')
+        resp = self.ehlo_resp.split('\n')
         del resp[0]
         for each in resp:
             # To be able to communicate with as many SMTP servers as possible,
@@ -415,16 +425,16 @@
             # It's actually stricter, in that only spaces are allowed between
             # parameters, but were not going to check for that here.  Note
             # that the space isn't present if there are no parameters.
-            m=re.match(r'(?P[A-Za-z0-9][A-Za-z0-9\-]*) ?',each)
+            m = re.match(r'(?P[A-Za-z0-9][A-Za-z0-9\-]*) ?', each)
             if m:
-                feature=m.group("feature").lower()
-                params=m.string[m.end("feature"):].strip()
+                feature = m.group("feature").lower()
+                params = m.string[m.end("feature"):].strip()
                 if feature == "auth":
                     self.esmtp_features[feature] = self.esmtp_features.get(feature, "") \
                             + " " + params
                 else:
-                    self.esmtp_features[feature]=params
-        return (code,msg)
+                    self.esmtp_features[feature] = params
+        return (code, msg)
 
     def has_extn(self, opt):
         """Does the server support a given SMTP service extension?"""
@@ -444,23 +454,23 @@
         """SMTP 'noop' command -- doesn't do anything :>"""
         return self.docmd("noop")
 
-    def mail(self,sender,options=[]):
+    def mail(self, sender, options=[]):
         """SMTP 'mail' command -- begins mail xfer session."""
         optionlist = ''
         if options and self.does_esmtp:
             optionlist = ' ' + ' '.join(options)
-        self.putcmd("mail", "FROM:%s%s" % (quoteaddr(sender) ,optionlist))
+        self.putcmd("mail", "FROM:%s%s" % (quoteaddr(sender), optionlist))
         return self.getreply()
 
-    def rcpt(self,recip,options=[]):
+    def rcpt(self, recip, options=[]):
         """SMTP 'rcpt' command -- indicates 1 recipient for this mail."""
         optionlist = ''
         if options and self.does_esmtp:
             optionlist = ' ' + ' '.join(options)
-        self.putcmd("rcpt","TO:%s%s" % (quoteaddr(recip),optionlist))
+        self.putcmd("rcpt", "TO:%s%s" % (quoteaddr(recip), optionlist))
         return self.getreply()
 
-    def data(self,msg):
+    def data(self, msg):
         """SMTP 'DATA' command -- sends message data to server.
 
         Automatically quotes lines beginning with a period per rfc821.
@@ -469,26 +479,28 @@
         response code received when the all data is sent.
         """
         self.putcmd("data")
-        (code,repl)=self.getreply()
-        if self.debuglevel >0 : print>>stderr, "data:", (code,repl)
+        (code, repl) = self.getreply()
+        if self.debuglevel > 0:
+            print>>stderr, "data:", (code, repl)
         if code != 354:
-            raise SMTPDataError(code,repl)
+            raise SMTPDataError(code, repl)
         else:
             q = quotedata(msg)
             if q[-2:] != CRLF:
                 q = q + CRLF
             q = q + "." + CRLF
             self.send(q)
-            (code,msg)=self.getreply()
-            if self.debuglevel >0 : print>>stderr, "data:", (code,msg)
-            return (code,msg)
+            (code, msg) = self.getreply()
+            if self.debuglevel > 0:
+                print>>stderr, "data:", (code, msg)
+            return (code, msg)
 
     def verify(self, address):
         """SMTP 'verify' command -- checks for address validity."""
         self.putcmd("vrfy", quoteaddr(address))
         return self.getreply()
     # a.k.a.
-    vrfy=verify
+    vrfy = verify
 
     def expn(self, address):
         """SMTP 'expn' command -- expands a mailing list."""
@@ -592,7 +604,7 @@
             raise SMTPAuthenticationError(code, resp)
         return (code, resp)
 
-    def starttls(self, keyfile = None, certfile = None):
+    def starttls(self, keyfile=None, certfile=None):
         """Puts the connection to the SMTP server into TLS mode.
 
         If there has been no previous EHLO or HELO command this session, this
@@ -695,22 +707,22 @@
             for option in mail_options:
                 esmtp_opts.append(option)
 
-        (code,resp) = self.mail(from_addr, esmtp_opts)
+        (code, resp) = self.mail(from_addr, esmtp_opts)
         if code != 250:
             self.rset()
             raise SMTPSenderRefused(code, resp, from_addr)
-        senderrs={}
+        senderrs = {}
         if isinstance(to_addrs, basestring):
             to_addrs = [to_addrs]
         for each in to_addrs:
-            (code,resp)=self.rcpt(each, rcpt_options)
+            (code, resp) = self.rcpt(each, rcpt_options)
             if (code != 250) and (code != 251):
-                senderrs[each]=(code,resp)
-        if len(senderrs)==len(to_addrs):
+                senderrs[each] = (code, resp)
+        if len(senderrs) == len(to_addrs):
             # the server refused all our recipients
             self.rset()
             raise SMTPRecipientsRefused(senderrs)
-        (code,resp) = self.data(msg)
+        (code, resp) = self.data(msg)
         if code != 250:
             self.rset()
             raise SMTPDataError(code, resp)
@@ -744,16 +756,19 @@
         are also optional - they can contain a PEM formatted private key and
         certificate chain file for the SSL connection.
         """
+
+        default_port = SMTP_SSL_PORT
+
         def __init__(self, host='', port=0, local_hostname=None,
                      keyfile=None, certfile=None,
                      timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
             self.keyfile = keyfile
             self.certfile = certfile
             SMTP.__init__(self, host, port, local_hostname, timeout)
-            self.default_port = SMTP_SSL_PORT
 
         def _get_socket(self, host, port, timeout):
-            if self.debuglevel > 0: print>>stderr, 'connect:', (host, port)
+            if self.debuglevel > 0:
+                print>>stderr, 'connect:', (host, port)
             new_socket = socket.create_connection((host, port), timeout)
             new_socket = ssl.wrap_socket(new_socket, self.keyfile, self.certfile)
             self.file = SSLFakeFile(new_socket)
@@ -781,11 +796,11 @@
 
     ehlo_msg = "lhlo"
 
-    def __init__(self, host = '', port = LMTP_PORT, local_hostname = None):
+    def __init__(self, host='', port=LMTP_PORT, local_hostname=None):
         """Initialize a new instance."""
         SMTP.__init__(self, host, port, local_hostname)
 
-    def connect(self, host = 'localhost', port = 0):
+    def connect(self, host='localhost', port=0):
         """Connect to the LMTP daemon, on either a Unix or a TCP socket."""
         if host[0] != '/':
             return SMTP.connect(self, host, port)
@@ -795,13 +810,15 @@
             self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
             self.sock.connect(host)
         except socket.error, msg:
-            if self.debuglevel > 0: print>>stderr, 'connect fail:', host
+            if self.debuglevel > 0:
+                print>>stderr, 'connect fail:', host
             if self.sock:
                 self.sock.close()
             self.sock = None
             raise socket.error, msg
         (code, msg) = self.getreply()
-        if self.debuglevel > 0: print>>stderr, "connect:", msg
+        if self.debuglevel > 0:
+            print>>stderr, "connect:", msg
         return (code, msg)
 
 
@@ -815,7 +832,7 @@
         return sys.stdin.readline().strip()
 
     fromaddr = prompt("From")
-    toaddrs  = prompt("To").split(',')
+    toaddrs = prompt("To").split(',')
     print "Enter message, end with ^D:"
     msg = ''
     while 1:
diff --git a/lib-python/2.7/ssl.py b/lib-python/2.7/ssl.py
--- a/lib-python/2.7/ssl.py
+++ b/lib-python/2.7/ssl.py
@@ -121,9 +121,11 @@
             if e.errno != errno.ENOTCONN:
                 raise
             # no, no connection yet
+            self._connected = False
             self._sslobj = None
         else:
             # yes, create the SSL object
+            self._connected = True
             self._sslobj = _ssl.sslwrap(self._sock, server_side,
                                         keyfile, certfile,
                                         cert_reqs, ssl_version, ca_certs,
@@ -293,21 +295,36 @@
 
         self._sslobj.do_handshake()
 
-    def connect(self, addr):
-
-        """Connects to remote ADDR, and then wraps the connection in
-        an SSL channel."""
-
+    def _real_connect(self, addr, return_errno):
         # Here we assume that the socket is client-side, and not
         # connected at the time of the call.  We connect it, then wrap it.
-        if self._sslobj:
+        if self._connected:
             raise ValueError("attempt to connect already-connected SSLSocket!")
-        socket.connect(self, addr)
         self._sslobj = _ssl.sslwrap(self._sock, False, self.keyfile, self.certfile,
                                     self.cert_reqs, self.ssl_version,
                                     self.ca_certs, self.ciphers)
-        if self.do_handshake_on_connect:
-            self.do_handshake()
+        try:
+            socket.connect(self, addr)
+            if self.do_handshake_on_connect:
+                self.do_handshake()
+        except socket_error as e:
+            if return_errno:
+                return e.errno
+            else:
+                self._sslobj = None
+                raise e
+        self._connected = True
+        return 0
+
+    def connect(self, addr):
+        """Connects to remote ADDR, and then wraps the connection in
+        an SSL channel."""
+        self._real_connect(addr, False)
+
+    def connect_ex(self, addr):
+        """Connects to remote ADDR, and then wraps the connection in
+        an SSL channel."""
+        return self._real_connect(addr, True)
 
     def accept(self):
 
diff --git a/lib-python/2.7/subprocess.py b/lib-python/2.7/subprocess.py
--- a/lib-python/2.7/subprocess.py
+++ b/lib-python/2.7/subprocess.py
@@ -396,6 +396,7 @@
 import traceback
 import gc
 import signal
+import errno
 
 # Exception classes used by this module.
 class CalledProcessError(Exception):
@@ -427,7 +428,6 @@
 else:
     import select
     _has_poll = hasattr(select, 'poll')
-    import errno
     import fcntl
     import pickle
 
@@ -441,8 +441,15 @@
            "check_output", "CalledProcessError"]
 
 if mswindows:
-    from _subprocess import CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP
-    __all__.extend(["CREATE_NEW_CONSOLE", "CREATE_NEW_PROCESS_GROUP"])
+    from _subprocess import (CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP,
+                             STD_INPUT_HANDLE, STD_OUTPUT_HANDLE,
+                             STD_ERROR_HANDLE, SW_HIDE,
+                             STARTF_USESTDHANDLES, STARTF_USESHOWWINDOW)
+
+    __all__.extend(["CREATE_NEW_CONSOLE", "CREATE_NEW_PROCESS_GROUP",
+                    "STD_INPUT_HANDLE", "STD_OUTPUT_HANDLE",
+                    "STD_ERROR_HANDLE", "SW_HIDE",
+                    "STARTF_USESTDHANDLES", "STARTF_USESHOWWINDOW"])
 try:
     MAXFD = os.sysconf("SC_OPEN_MAX")
 except:
@@ -726,7 +733,11 @@
             stderr = None
             if self.stdin:
                 if input:
-                    self.stdin.write(input)
+                    try:
+                        self.stdin.write(input)
+                    except IOError as e:
+                        if e.errno != errno.EPIPE and e.errno != errno.EINVAL:
+                            raise
                 self.stdin.close()
             elif self.stdout:
                 stdout = self.stdout.read()
@@ -883,7 +894,7 @@
             except pywintypes.error, e:
                 # Translate pywintypes.error to WindowsError, which is
                 # a subclass of OSError.  FIXME: We should really
-                # translate errno using _sys_errlist (or simliar), but
+                # translate errno using _sys_errlist (or similar), but
                 # how can this be done from Python?
                 raise WindowsError(*e.args)
             finally:
@@ -956,7 +967,11 @@
 
             if self.stdin:
                 if input is not None:
-                    self.stdin.write(input)
+                    try:
+                        self.stdin.write(input)
+                    except IOError as e:
+                        if e.errno != errno.EPIPE:
+                            raise
                 self.stdin.close()
 
             if self.stdout:
@@ -1051,14 +1066,17 @@
                     errread, errwrite)
 
 
-        def _set_cloexec_flag(self, fd):
+        def _set_cloexec_flag(self, fd, cloexec=True):
             try:
                 cloexec_flag = fcntl.FD_CLOEXEC
             except AttributeError:
                 cloexec_flag = 1
 
             old = fcntl.fcntl(fd, fcntl.F_GETFD)
-            fcntl.fcntl(fd, fcntl.F_SETFD, old | cloexec_flag)
+            if cloexec:
+                fcntl.fcntl(fd, fcntl.F_SETFD, old | cloexec_flag)
+            else:
+                fcntl.fcntl(fd, fcntl.F_SETFD, old & ~cloexec_flag)
 
 
         def _close_fds(self, but):
@@ -1128,21 +1146,25 @@
                             os.close(errpipe_read)
 
                             # Dup fds for child
-                            if p2cread is not None:
-                                os.dup2(p2cread, 0)
-                            if c2pwrite is not None:
-                                os.dup2(c2pwrite, 1)
-                            if errwrite is not None:
-                                os.dup2(errwrite, 2)
+                            def _dup2(a, b):
+                                # dup2() removes the CLOEXEC flag but
+                                # we must do it ourselves if dup2()
+                                # would be a no-op (issue #10806).
+                                if a == b:
+                                    self._set_cloexec_flag(a, False)
+                                elif a is not None:
+                                    os.dup2(a, b)
+                            _dup2(p2cread, 0)
+                            _dup2(c2pwrite, 1)
+                            _dup2(errwrite, 2)
 
-                            # Close pipe fds.  Make sure we don't close the same
-                            # fd more than once, or standard fds.
-                            if p2cread is not None and p2cread not in (0,):
-                                os.close(p2cread)
-                            if c2pwrite is not None and c2pwrite not in (p2cread, 1):
-                                os.close(c2pwrite)
-                            if errwrite is not None and errwrite not in (p2cread, c2pwrite, 2):
-                                os.close(errwrite)
+                            # Close pipe fds.  Make sure we don't close the
+                            # same fd more than once, or standard fds.
+                            closed = { None }
+                            for fd in [p2cread, c2pwrite, errwrite]:
+                                if fd not in closed and fd > 2:
+                                    os.close(fd)
+                                    closed.add(fd)
 
                             # Close all other fds, if asked for
                             if close_fds:
@@ -1194,7 +1216,11 @@
                 os.close(errpipe_read)
 
             if data != "":
-                _eintr_retry_call(os.waitpid, self.pid, 0)
+                try:
+                    _eintr_retry_call(os.waitpid, self.pid, 0)
+                except OSError as e:
+                    if e.errno != errno.ECHILD:
+                        raise
                 child_exception = pickle.loads(data)
                 for fd in (p2cwrite, c2pread, errread):
                     if fd is not None:
@@ -1240,7 +1266,15 @@
             """Wait for child process to terminate.  Returns returncode
             attribute."""
             if self.returncode is None:
-                pid, sts = _eintr_retry_call(os.waitpid, self.pid, 0)
+                try:
+                    pid, sts = _eintr_retry_call(os.waitpid, self.pid, 0)
+                except OSError as e:
+                    if e.errno != errno.ECHILD:
+                        raise
+                    # This happens if SIGCLD is set to be ignored or waiting
+                    # for child processes has otherwise been disabled for our
+                    # process.  This child is dead, we can't get the status.
+                    sts = 0
                 self._handle_exitstatus(sts)
             return self.returncode
 
@@ -1317,9 +1351,16 @@
                 for fd, mode in ready:
                     if mode & select.POLLOUT:
                         chunk = input[input_offset : input_offset + _PIPE_BUF]
-                        input_offset += os.write(fd, chunk)
-                        if input_offset >= len(input):
-                            close_unregister_and_remove(fd)
+                        try:
+                            input_offset += os.write(fd, chunk)
+                        except OSError as e:
+                            if e.errno == errno.EPIPE:
+                                close_unregister_and_remove(fd)
+                            else:
+                                raise
+                        else:
+                            if input_offset >= len(input):
+                                close_unregister_and_remove(fd)
                     elif mode & select_POLLIN_POLLPRI:
                         data = os.read(fd, 4096)
                         if not data:
@@ -1358,11 +1399,19 @@
 
                 if self.stdin in wlist:
                     chunk = input[input_offset : input_offset + _PIPE_BUF]
-                    bytes_written = os.write(self.stdin.fileno(), chunk)
-                    input_offset += bytes_written
-                    if input_offset >= len(input):
-                        self.stdin.close()
-                        write_set.remove(self.stdin)
+                    try:
+                        bytes_written = os.write(self.stdin.fileno(), chunk)
+                    except OSError as e:
+                        if e.errno == errno.EPIPE:
+                            self.stdin.close()
+                            write_set.remove(self.stdin)
+                        else:
+                            raise
+                    else:
+                        input_offset += bytes_written
+                        if input_offset >= len(input):
+                            self.stdin.close()
+                            write_set.remove(self.stdin)
 
                 if self.stdout in rlist:
                     data = os.read(self.stdout.fileno(), 1024)
diff --git a/lib-python/2.7/symbol.py b/lib-python/2.7/symbol.py
--- a/lib-python/2.7/symbol.py
+++ b/lib-python/2.7/symbol.py
@@ -82,20 +82,19 @@
 sliceop = 325
 exprlist = 326
 testlist = 327
-dictmaker = 328
-dictorsetmaker = 329
-classdef = 330
-arglist = 331
-argument = 332
-list_iter = 333
-list_for = 334
-list_if = 335
-comp_iter = 336
-comp_for = 337
-comp_if = 338
-testlist1 = 339
-encoding_decl = 340
-yield_expr = 341
+dictorsetmaker = 328
+classdef = 329
+arglist = 330
+argument = 331
+list_iter = 332
+list_for = 333
+list_if = 334
+comp_iter = 335
+comp_for = 336
+comp_if = 337
+testlist1 = 338
+encoding_decl = 339
+yield_expr = 340
 #--end constants--
 
 sym_name = {}
diff --git a/lib-python/2.7/sysconfig.py b/lib-python/2.7/sysconfig.py
--- a/lib-python/2.7/sysconfig.py
+++ b/lib-python/2.7/sysconfig.py
@@ -271,7 +271,7 @@
 def _get_makefile_filename():
     if _PYTHON_BUILD:
         return os.path.join(_PROJECT_BASE, "Makefile")
-    return os.path.join(get_path('stdlib'), "config", "Makefile")
+    return os.path.join(get_path('platstdlib'), "config", "Makefile")
 
 
 def _init_posix(vars):
@@ -297,21 +297,6 @@
             msg = msg + " (%s)" % e.strerror
         raise IOError(msg)
 
-    # On MacOSX we need to check the setting of the environment variable
-    # MACOSX_DEPLOYMENT_TARGET: configure bases some choices on it so
-    # it needs to be compatible.
-    # If it isn't set we set it to the configure-time value
-    if sys.platform == 'darwin' and 'MACOSX_DEPLOYMENT_TARGET' in vars:
-        cfg_target = vars['MACOSX_DEPLOYMENT_TARGET']
-        cur_target = os.getenv('MACOSX_DEPLOYMENT_TARGET', '')
-        if cur_target == '':
-            cur_target = cfg_target
-            os.putenv('MACOSX_DEPLOYMENT_TARGET', cfg_target)
-        elif map(int, cfg_target.split('.')) > map(int, cur_target.split('.')):
-            msg = ('$MACOSX_DEPLOYMENT_TARGET mismatch: now "%s" but "%s" '
-                   'during configure' % (cur_target, cfg_target))
-            raise IOError(msg)
-
     # On AIX, there are wrong paths to the linker scripts in the Makefile
     # -- these paths are relative to the Python source, but when installed
     # the scripts are in another directory.
@@ -616,9 +601,7 @@
         # machine is going to compile and link as if it were
         # MACOSX_DEPLOYMENT_TARGET.
         cfgvars = get_config_vars()
-        macver = os.environ.get('MACOSX_DEPLOYMENT_TARGET')
-        if not macver:
-            macver = cfgvars.get('MACOSX_DEPLOYMENT_TARGET')
+        macver = cfgvars.get('MACOSX_DEPLOYMENT_TARGET')
 
         if 1:
             # Always calculate the release of the running machine,
@@ -639,7 +622,6 @@
                     m = re.search(
                             r'ProductUserVisibleVersion\s*' +
                             r'(.*?)', f.read())
-                    f.close()
                     if m is not None:
                         macrelease = '.'.join(m.group(1).split('.')[:2])
                     # else: fall back to the default behaviour
diff --git a/lib-python/2.7/tarfile.py b/lib-python/2.7/tarfile.py
--- a/lib-python/2.7/tarfile.py
+++ b/lib-python/2.7/tarfile.py
@@ -2239,10 +2239,14 @@
         if hasattr(os, "symlink") and hasattr(os, "link"):
             # For systems that support symbolic and hard links.
             if tarinfo.issym():
+                if os.path.lexists(targetpath):
+                    os.unlink(targetpath)
                 os.symlink(tarinfo.linkname, targetpath)
             else:
                 # See extract().
                 if os.path.exists(tarinfo._link_target):
+                    if os.path.lexists(targetpath):
+                        os.unlink(targetpath)
                     os.link(tarinfo._link_target, targetpath)
                 else:
                     self._extract_member(self._find_link_target(tarinfo), targetpath)
diff --git a/lib-python/2.7/telnetlib.py b/lib-python/2.7/telnetlib.py
--- a/lib-python/2.7/telnetlib.py
+++ b/lib-python/2.7/telnetlib.py
@@ -236,7 +236,7 @@
 
         """
         if self.debuglevel > 0:
-            print 'Telnet(%s,%d):' % (self.host, self.port),
+            print 'Telnet(%s,%s):' % (self.host, self.port),
             if args:
                 print msg % args
             else:
diff --git a/lib-python/2.7/test/cjkencodings/big5-utf8.txt b/lib-python/2.7/test/cjkencodings/big5-utf8.txt
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/test/cjkencodings/big5-utf8.txt
@@ -0,0 +1,9 @@
+如何在 Python 中使用既有的 C library?
+　在資訊科技快速發展的今天, 開發及測試軟體的速度是不容忽視的
+課題. 為加快開發及測試的速度, 我們便常希望能利用一些已開發好的
+library, 並有一個 fast prototyping 的 programming language 可
+供使用. 目前有許許多多的 library 是以 C 寫成, 而 Python 是一個
+fast prototyping 的 programming language. 故我們希望能將既有的
+C library 拿到 Python 的環境中測試及整合. 其中最主要也是我們所
+要討論的問題就是:
+
diff --git a/lib-python/2.7/test/cjkencodings/big5.txt b/lib-python/2.7/test/cjkencodings/big5.txt
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/test/cjkencodings/big5.txt
@@ -0,0 +1,9 @@
+�p��b Python ���ϥάJ���� C library?
+�@�b��T��ާֳt�o�i������, �}�o�δ��ճn�骺�t�׬O���e������
+���D. ���[�ֶ}�o�δ��ժ��t��, �ڭ̫K�`�Ʊ��Q�Τ@�Ǥw�}�o�n��
+library, �æ��@�� fast prototyping �� programming language �i
+�Ѩϥ�. �ثe���\�\�h�h�� library �O�H C �g��, �� Python �O�@��
+fast prototyping �� programming language. �G�ڭ̧Ʊ��N�J����
+C library ���� Python �����Ҥ����դξ�X. �䤤�̥D�n�]�O�ڭ̩�
+�n�Q�ת����D�N�O:
+
diff --git a/lib-python/2.7/test/cjkencodings/big5hkscs-utf8.txt b/lib-python/2.7/test/cjkencodings/big5hkscs-utf8.txt
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/test/cjkencodings/big5hkscs-utf8.txt
@@ -0,0 +1,2 @@
+𠄌Ě鵮罓洆
+ÊÊ̄ê êê̄
diff --git a/lib-python/2.7/test/cjkencodings/big5hkscs.txt b/lib-python/2.7/test/cjkencodings/big5hkscs.txt
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/test/cjkencodings/big5hkscs.txt
@@ -0,0 +1,2 @@
+�E�\�s�ڍ�
+�f�b�� ����
diff --git a/lib-python/2.7/test/cjkencodings/cp949-utf8.txt b/lib-python/2.7/test/cjkencodings/cp949-utf8.txt
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/test/cjkencodings/cp949-utf8.txt
@@ -0,0 +1,9 @@
+똠방각하 펲시콜라
+
+㉯㉯납!! 因九月패믤릔궈 ⓡⓖ훀¿¿¿ 긍뒙 ⓔ뎨 ㉯. .
+亞영ⓔ능횹 . . . . 서울뤄 뎐학乙 家훀 ! ! !ㅠ.ㅠ
+흐흐흐 ㄱㄱㄱ☆ㅠ_ㅠ 어릨 탸콰긐 뎌응 칑九들乙 ㉯드긐
+설릌 家훀 . . . . 굴애쉌 ⓔ궈 ⓡ릘㉱긐 因仁川女中까즼
+와쒀훀 ! ! 亞영ⓔ 家능궈 ☆上관 없능궈능 亞능뒈훀 글애듴
+ⓡ려듀九 싀풔숴훀 어릨 因仁川女中싁⑨들앜!! ㉯㉯납♡ ⌒⌒*
+
diff --git a/lib-python/2.7/test/cjkencodings/cp949.txt b/lib-python/2.7/test/cjkencodings/cp949.txt
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/test/cjkencodings/cp949.txt
@@ -0,0 +1,9 @@
+�c�氢�� �����ݶ�
+
+������!! �������В�p�� �ި��R������ ���� �ѵ� ��. .
+䬿��Ѵ��� . . . . ����� ������ ʫ�R ! ! !��.��
+������ �������٤�_�� � ����O ���� �h������ ����O
+���j ʫ�R . . . . ���֚f �ѱ� �ސt�ƒO ����������
+�;��R ! ! 䬿��� ʫ�ɱ� ��߾�� ���ɱŴ� 䬴ɵ��R �۾֊�
+�޷����� ��Ǵ���R � ����������Ĩ���!! �������� �ҡ�*
+
diff --git a/lib-python/2.7/test/cjkencodings/euc_jisx0213-utf8.txt b/lib-python/2.7/test/cjkencodings/euc_jisx0213-utf8.txt
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/test/cjkencodings/euc_jisx0213-utf8.txt
@@ -0,0 +1,8 @@
+Python の開発は、1990 年ごろから開始されています。
+開発者の Guido van Rossum は教育用のプログラミング言語「ABC」の開発に参加していましたが、ABC は実用上の目的にはあまり適していませんでした。
+このため、Guido はより実用的なプログラミング言語の開発を開始し、英国 BBS 放送のコメディ番組「モンティ パイソン」のファンである Guido はこの言語を「Python」と名づけました。
+このような背景から生まれた Python の言語設計は、「シンプル」で「習得が容易」という目標に重点が置かれています。
+多くのスクリプト系言語ではユーザの目先の利便性を優先して色々な機能を言語要素として取り入れる場合が多いのですが、Python ではそういった小細工が追加されることはあまりありません。
+言語自体の機能は最小限に押さえ、必要な機能は拡張モジュールとして追加する、というのが Python のポリシーです。
+
+ノか゚ ト゚ トキ喝塀 𡚴𪎌 麀齁𩛰
diff --git a/lib-python/2.7/test/cjkencodings/euc_jisx0213.txt b/lib-python/2.7/test/cjkencodings/euc_jisx0213.txt
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/test/cjkencodings/euc_jisx0213.txt
@@ -0,0 +1,8 @@
+Python �γ�ȯ�ϡ�1990 ǯ�����鳫�Ϥ���Ƥ��ޤ���
+��ȯ�Ԥ� Guido van Rossum �϶����ѤΥץ���ߥ󥰸����ABC�פγ�ȯ�˻��ä��Ƥ��ޤ�������ABC �ϼ��Ѿ����Ū�ˤϤ��ޤ�Ŭ���Ƥ��ޤ���Ǥ�����
+���Τ��ᡢGuido �Ϥ�����Ū�ʥץ���ߥ󥰸���γ�ȯ�򳫻Ϥ����ѹ� BBS �����Υ���ǥ����ȡ֥��ƥ� �ѥ�����פΥե���Ǥ��� Guido �Ϥ��θ�����Python�פ�̾�Ť��ޤ�����
+���Τ褦���طʤ������ޤ줿 Python �θ����߷פϡ��֥���ץ�פǡֽ������ưספȤ�����ɸ�˽������֤���Ƥ��ޤ���
+¿���Υ�����ץȷϸ���Ǥϥ桼�����������������ͥ�褷�ƿ����ʵ�ǽ��������ǤȤ��Ƽ��������礬¿���ΤǤ�����Python �ǤϤ������ä����ٹ����ɲä���뤳�ȤϤ��ޤꤢ��ޤ���
+���켫�Τε�ǽ�ϺǾ��¤˲�������ɬ�פʵ�ǽ�ϳ�ĥ�⥸�塼��Ȥ����ɲä��롢�Ȥ����Τ� Python �Υݥꥷ���Ǥ���
+
+�Τ� �� �ȥ����� ���� ��ԏ����
diff --git a/lib-python/2.7/test/cjkencodings/euc_jp-utf8.txt b/lib-python/2.7/test/cjkencodings/euc_jp-utf8.txt
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/test/cjkencodings/euc_jp-utf8.txt
@@ -0,0 +1,7 @@
+Python の開発は、1990 年ごろから開始されています。
+開発者の Guido van Rossum は教育用のプログラミング言語「ABC」の開発に参加していましたが、ABC は実用上の目的にはあまり適していませんでした。
+このため、Guido はより実用的なプログラミング言語の開発を開始し、英国 BBS 放送のコメディ番組「モンティ パイソン」のファンである Guido はこの言語を「Python」と名づけました。
+このような背景から生まれた Python の言語設計は、「シンプル」で「習得が容易」という目標に重点が置かれています。
+多くのスクリプト系言語ではユーザの目先の利便性を優先して色々な機能を言語要素として取り入れる場合が多いのですが、Python ではそういった小細工が追加されることはあまりありません。
+言語自体の機能は最小限に押さえ、必要な機能は拡張モジュールとして追加する、というのが Python のポリシーです。
+
diff --git a/lib-python/2.7/test/cjkencodings/euc_jp.txt b/lib-python/2.7/test/cjkencodings/euc_jp.txt
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/test/cjkencodings/euc_jp.txt
@@ -0,0 +1,7 @@
+Python �γ�ȯ�ϡ�1990 ǯ�����鳫�Ϥ���Ƥ��ޤ���
+��ȯ�Ԥ� Guido van Rossum �϶����ѤΥץ���ߥ󥰸����ABC�פγ�ȯ�˻��ä��Ƥ��ޤ�������ABC �ϼ��Ѿ����Ū�ˤϤ��ޤ�Ŭ���Ƥ��ޤ���Ǥ�����
+���Τ��ᡢGuido �Ϥ�����Ū�ʥץ���ߥ󥰸���γ�ȯ�򳫻Ϥ����ѹ� BBS �����Υ���ǥ����ȡ֥��ƥ� �ѥ�����פΥե���Ǥ��� Guido �Ϥ��θ�����Python�פ�̾�Ť��ޤ�����
+���Τ褦���طʤ������ޤ줿 Python �θ����߷פϡ��֥���ץ�פǡֽ������ưספȤ�����ɸ�˽������֤���Ƥ��ޤ���
+¿���Υ�����ץȷϸ���Ǥϥ桼�����������������ͥ�褷�ƿ����ʵ�ǽ��������ǤȤ��Ƽ��������礬¿���ΤǤ�����Python �ǤϤ������ä����ٹ����ɲä���뤳�ȤϤ��ޤꤢ��ޤ���
+���켫�Τε�ǽ�ϺǾ��¤˲�������ɬ�פʵ�ǽ�ϳ�ĥ�⥸�塼��Ȥ����ɲä��롢�Ȥ����Τ� Python �Υݥꥷ���Ǥ���
+
diff --git a/lib-python/2.7/test/cjkencodings/euc_kr-utf8.txt b/lib-python/2.7/test/cjkencodings/euc_kr-utf8.txt
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/test/cjkencodings/euc_kr-utf8.txt
@@ -0,0 +1,7 @@
+◎ 파이썬(Python)은 배우기 쉽고, 강력한 프로그래밍 언어입니다. 파이썬은
+효율적인 고수준 데이터 구조와 간단하지만 효율적인 객체지향프로그래밍을
+지원합니다. 파이썬의 우아(優雅)한 문법과 동적 타이핑, 그리고 인터프리팅
+환경은 파이썬을 스크립팅과 여러 분야에서와 대부분의 플랫폼에서의 빠른
+애플리케이션 개발을 할 수 있는 이상적인 언어로 만들어줍니다.
+
+☆첫가끝: 날아라 쓔쓔쓩~ 닁큼! 뜽금없이 전홥니다. 뷁. 그런거 읎다.
diff --git a/lib-python/2.7/test/cjkencodings/euc_kr.txt b/lib-python/2.7/test/cjkencodings/euc_kr.txt
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/test/cjkencodings/euc_kr.txt
@@ -0,0 +1,7 @@
+�� ���̽�(Python)�� ���� ����, ������ ���α׷��� ����Դϴ�. ���̽���
+ȿ������ ����� ������ ������ ���������� ȿ������ ��ü�������α׷�����
+�����մϴ�. ���̽��� ���(���)�� ������ ���� Ÿ����, �׸��� ����������
+ȯ���� ���̽��� ��ũ���ð� ���� �о߿����� ��κ��� �÷��������� ����
+���ø����̼� ������ �� �� �ִ� �̻����� ���� ������ݴϴ�.
+
+��ù����: ���ƶ� �Ԥ��ФԤԤ��ФԾ�~ �Ԥ��Ҥ�ŭ! �Ԥ��Ѥ��ݾ��� ���Ԥ��Ȥ��ϴ�. �Ԥ��Τ�. �׷��� �Ԥ��Ѥ���.
diff --git a/lib-python/2.7/test/cjkencodings/gb18030-utf8.txt b/lib-python/2.7/test/cjkencodings/gb18030-utf8.txt
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/test/cjkencodings/gb18030-utf8.txt
@@ -0,0 +1,15 @@
+Python（派森）语言是一种功能强大而完善的通用型计算机程序设计语言，
+已经具有十多年的发展历史，成熟且稳定。这种语言具有非常简捷而清晰
+的语法特点，适合完成各种高层任务，几乎可以在所有的操作系统中
+运行。这种语言简单而强大，适合各种人士学习使用。目前，基于这
+种语言的相关技术正在飞速的发展，用户数量急剧扩大，相关的资源非常多。
+如何在 Python 中使用既有的 C library?
+　在資訊科技快速發展的今天, 開發及測試軟體的速度是不容忽視的
+課題. 為加快開發及測試的速度, 我們便常希望能利用一些已開發好的
+library, 並有一個 fast prototyping 的 programming language 可
+供使用. 目前有許許多多的 library 是以 C 寫成, 而 Python 是一個
+fast prototyping 的 programming language. 故我們希望能將既有的
+C library 拿到 Python 的環境中測試及整合. 其中最主要也是我們所
+要討論的問題就是:
+파이썬은 강력한 기능을 지닌 범용 컴퓨터 프로그래밍 언어다.
+
diff --git a/lib-python/2.7/test/cjkencodings/gb18030.txt b/lib-python/2.7/test/cjkencodings/gb18030.txt
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/test/cjkencodings/gb18030.txt
@@ -0,0 +1,15 @@
+Python����ɭ��������һ�ֹ���ǿ������Ƶ�ͨ���ͼ��������������ԣ�
+�Ѿ�����ʮ����ķ�չ��ʷ���������ȶ����������Ծ��зǳ���ݶ�����
+���﷨�ص㣬�ʺ���ɸ��ָ߲����񣬼������������еĲ���ϵͳ��
+���С��������Լ򵥶�ǿ���ʺϸ�����ʿѧϰʹ�á�Ŀǰ��������
+�����Ե���ؼ������ڷ��ٵķ�չ���û���������������ص���Դ�ǳ��ࡣ
+����� Python ��ʹ�ü��е� C library?
+�����YӍ�Ƽ����ٰlչ�Ľ���, �_�l���yԇܛ�w���ٶ��ǲ��ݺ�ҕ��
+�n�}. ��ӿ��_�l���yԇ���ٶ�, �҂��㳣ϣ��������һЩ���_�l�õ�
+library, �K��һ�� fast prototyping �� programming language ��
+��ʹ��. Ŀǰ���S�S���� library ���� C ����, �� Python ��һ��
+fast prototyping �� programming language. ���҂�ϣ���܌����е�
+C library �õ� Python �ĭh���Мyԇ������. ��������ҪҲ���҂���
+ҪӑՓ�Ć��}����:
+�5�1�3�3�2�1�3�1 �7�6�0�4�6�3 �8�5�8�6�3�5 �3�1�9�5 �0�9�3�0 �4�3�5�7�5�5 �5�5�0�9�8�9�9�3�0�4 �2�9�2�5�9�9.
+
diff --git a/lib-python/2.7/test/cjkencodings/gb2312-utf8.txt b/lib-python/2.7/test/cjkencodings/gb2312-utf8.txt
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/test/cjkencodings/gb2312-utf8.txt
@@ -0,0 +1,6 @@
+Python（派森）语言是一种功能强大而完善的通用型计算机程序设计语言，
+已经具有十多年的发展历史，成熟且稳定。这种语言具有非常简捷而清晰
+的语法特点，适合完成各种高层任务，几乎可以在所有的操作系统中
+运行。这种语言简单而强大，适合各种人士学习使用。目前，基于这
+种语言的相关技术正在飞速的发展，用户数量急剧扩大，相关的资源非常多。
+
diff --git a/lib-python/2.7/test/cjkencodings/gb2312.txt b/lib-python/2.7/test/cjkencodings/gb2312.txt
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/test/cjkencodings/gb2312.txt
@@ -0,0 +1,6 @@
+Python����ɭ��������һ�ֹ���ǿ������Ƶ�ͨ���ͼ��������������ԣ�
+�Ѿ�����ʮ����ķ�չ��ʷ���������ȶ����������Ծ��зǳ���ݶ�����
+���﷨�ص㣬�ʺ���ɸ��ָ߲����񣬼������������еĲ���ϵͳ��
+���С��������Լ򵥶�ǿ���ʺϸ�����ʿѧϰʹ�á�Ŀǰ��������
+�����Ե���ؼ������ڷ��ٵķ�չ���û���������������ص���Դ�ǳ��ࡣ
+
diff --git a/lib-python/2.7/test/cjkencodings/gbk-utf8.txt b/lib-python/2.7/test/cjkencodings/gbk-utf8.txt
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/test/cjkencodings/gbk-utf8.txt
@@ -0,0 +1,14 @@
+Python（派森）语言是一种功能强大而完善的通用型计算机程序设计语言，
+已经具有十多年的发展历史，成熟且稳定。这种语言具有非常简捷而清晰
+的语法特点，适合完成各种高层任务，几乎可以在所有的操作系统中
+运行。这种语言简单而强大，适合各种人士学习使用。目前，基于这
+种语言的相关技术正在飞速的发展，用户数量急剧扩大，相关的资源非常多。
+如何在 Python 中使用既有的 C library?
+　在資訊科技快速發展的今天, 開發及測試軟體的速度是不容忽視的
+課題. 為加快開發及測試的速度, 我們便常希望能利用一些已開發好的
+library, 並有一個 fast prototyping 的 programming language 可
+供使用. 目前有許許多多的 library 是以 C 寫成, 而 Python 是一個
+fast prototyping 的 programming language. 故我們希望能將既有的
+C library 拿到 Python 的環境中測試及整合. 其中最主要也是我們所
+要討論的問題就是:
+
diff --git a/lib-python/2.7/test/cjkencodings/gbk.txt b/lib-python/2.7/test/cjkencodings/gbk.txt
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/test/cjkencodings/gbk.txt
@@ -0,0 +1,14 @@
+Python����ɭ��������һ�ֹ���ǿ������Ƶ�ͨ���ͼ��������������ԣ�
+�Ѿ�����ʮ����ķ�չ��ʷ���������ȶ����������Ծ��зǳ���ݶ�����
+���﷨�ص㣬�ʺ���ɸ��ָ߲����񣬼������������еĲ���ϵͳ��
+���С��������Լ򵥶�ǿ���ʺϸ�����ʿѧϰʹ�á�Ŀǰ��������
+�����Ե���ؼ������ڷ��ٵķ�չ���û���������������ص���Դ�ǳ��ࡣ
+����� Python ��ʹ�ü��е� C library?
+�����YӍ�Ƽ����ٰlչ�Ľ���, �_�l���yԇܛ�w���ٶ��ǲ��ݺ�ҕ��
+�n�}. ��ӿ��_�l���yԇ���ٶ�, �҂��㳣ϣ��������һЩ���_�l�õ�
+library, �K��һ�� fast prototyping �� programming language ��
+��ʹ��. Ŀǰ���S�S���� library ���� C ����, �� Python ��һ��
+fast prototyping �� programming language. ���҂�ϣ���܌����е�
+C library �õ� Python �ĭh���Мyԇ������. ��������ҪҲ���҂���
+ҪӑՓ�Ć��}����:
+
diff --git a/lib-python/2.7/test/cjkencodings/hz-utf8.txt b/lib-python/2.7/test/cjkencodings/hz-utf8.txt
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/test/cjkencodings/hz-utf8.txt
@@ -0,0 +1,2 @@
+This sentence is in ASCII.
+The next sentence is in GB.己所不欲，勿施於人。Bye.
diff --git a/lib-python/2.7/test/cjkencodings/hz.txt b/lib-python/2.7/test/cjkencodings/hz.txt
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/test/cjkencodings/hz.txt
@@ -0,0 +1,2 @@
+This sentence is in ASCII.
+The next sentence is in GB.~{<:Ky2;S{#,NpJ)l6HK!#~}Bye.
diff --git a/lib-python/2.7/test/cjkencodings/johab-utf8.txt b/lib-python/2.7/test/cjkencodings/johab-utf8.txt
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/test/cjkencodings/johab-utf8.txt
@@ -0,0 +1,9 @@
+똠방각하 펲시콜라
+
+㉯㉯납!! 因九月패믤릔궈 ⓡⓖ훀¿¿¿ 긍뒙 ⓔ뎨 ㉯. .
+亞영ⓔ능횹 . . . . 서울뤄 뎐학乙 家훀 ! ! !ㅠ.ㅠ
+흐흐흐 ㄱㄱㄱ☆ㅠ_ㅠ 어릨 탸콰긐 뎌응 칑九들乙 ㉯드긐
+설릌 家훀 . . . . 굴애쉌 ⓔ궈 ⓡ릘㉱긐 因仁川女中까즼
+와쒀훀 ! ! 亞영ⓔ 家능궈 ☆上관 없능궈능 亞능뒈훀 글애듴
+ⓡ려듀九 싀풔숴훀 어릨 因仁川女中싁⑨들앜!! ㉯㉯납♡ ⌒⌒*
+
diff --git a/lib-python/2.7/test/cjkencodings/johab.txt b/lib-python/2.7/test/cjkencodings/johab.txt
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/test/cjkencodings/johab.txt
@@ -0,0 +1,9 @@
+���w�b�a �\��ũ�a
+
+�����s!! �g��Ú������ �����zٯٯٯ �w�� �ѕ� ��. .
+�<�w�ѓw�s . . . . �ᶉ�� �e�b�� �;�z ! ! !�A.�A
+�a�a�a �A�A�A�i�A_�A �៚ ȡ���z �a�w ×✗i�� ���a�z
+��z �;�z . . . . ������ �ъ� �ޟ��‹z �g�b�I����a��
+�����z ! ! �<�w�� �;�w�� �i꾉� ���w���w �<�w���z �i���z
+�ޝa�A� ��Ρ���z �៚ �g�b�I���鯂��i�z!! �����sٽ �b�b*
+
diff --git a/lib-python/2.7/test/cjkencodings/shift_jis-utf8.txt b/lib-python/2.7/test/cjkencodings/shift_jis-utf8.txt
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/test/cjkencodings/shift_jis-utf8.txt
@@ -0,0 +1,7 @@
+Python の開発は、1990 年ごろから開始されています。
+開発者の Guido van Rossum は教育用のプログラミング言語「ABC」の開発に参加していましたが、ABC は実用上の目的にはあまり適していませんでした。
+このため、Guido はより実用的なプログラミング言語の開発を開始し、英国 BBS 放送のコメディ番組「モンティ パイソン」のファンである Guido はこの言語を「Python」と名づけました。
+このような背景から生まれた Python の言語設計は、「シンプル」で「習得が容易」という目標に重点が置かれています。
+多くのスクリプト系言語ではユーザの目先の利便性を優先して色々な機能を言語要素として取り入れる場合が多いのですが、Python ではそういった小細工が追加されることはあまりありません。
+言語自体の機能は最小限に押さえ、必要な機能は拡張モジュールとして追加する、というのが Python のポリシーです。
+
diff --git a/lib-python/2.7/test/cjkencodings/shift_jis.txt b/lib-python/2.7/test/cjkencodings/shift_jis.txt
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/test/cjkencodings/shift_jis.txt
@@ -0,0 +1,7 @@
+Python �̊J���́A1990 �N���납��J�n����Ă��܂��B
+�J���҂� Guido van Rossum �͋���p�̃v���O���~���O����uABC�v�̊J���ɎQ�����Ă��܂������AABC �͎��p��̖ړI�ɂ͂��܂�K���Ă��܂���ł����B
+���̂��߁AGuido �͂����p�I�ȃv���O���~���O����̊J�����J�n���A�p�� BBS �����̃R���f�B�ԑg�u�����e�B �p�C�\���v�̃t�@���ł��� Guido �͂��̌�����uPython�v�Ɩ��Â��܂����B
+���̂悤�Ȕw�i���琶�܂ꂽ Python �̌���݌v�́A�u�V���v���v�Łu�K�����e�Ձv�Ƃ����ڕW�ɏd�_���u����Ă��܂��B
+�����̃X�N���v�g�n����ł̓��[�U�̖ڐ�̗��֐���D�悵�ĐF�X�ȋ@�\������v�f�Ƃ��Ď������ꍇ�������̂ł����APython �ł͂������������׍H���ǉ�����邱�Ƃ͂��܂肠��܂���B
+���ꎩ�̂̋@�\�͍ŏ����ɉ������A�K�v�ȋ@�\�͊g�����W���[���Ƃ��Ēǉ�����A�Ƃ����̂� Python �̃|���V�[�ł��B
+
diff --git a/lib-python/2.7/test/cjkencodings/shift_jisx0213-utf8.txt b/lib-python/2.7/test/cjkencodings/shift_jisx0213-utf8.txt
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/test/cjkencodings/shift_jisx0213-utf8.txt
@@ -0,0 +1,8 @@
+Python の開発は、1990 年ごろから開始されています。
+開発者の Guido van Rossum は教育用のプログラミング言語「ABC」の開発に参加していましたが、ABC は実用上の目的にはあまり適していませんでした。
+このため、Guido はより実用的なプログラミング言語の開発を開始し、英国 BBS 放送のコメディ番組「モンティ パイソン」のファンである Guido はこの言語を「Python」と名づけました。
+このような背景から生まれた Python の言語設計は、「シンプル」で「習得が容易」という目標に重点が置かれています。
+多くのスクリプト系言語ではユーザの目先の利便性を優先して色々な機能を言語要素として取り入れる場合が多いのですが、Python ではそういった小細工が追加されることはあまりありません。
+言語自体の機能は最小限に押さえ、必要な機能は拡張モジュールとして追加する、というのが Python のポリシーです。
+
+ノか゚ ト゚ トキ喝塀 𡚴𪎌 麀齁𩛰
diff --git a/lib-python/2.7/test/cjkencodings/shift_jisx0213.txt b/lib-python/2.7/test/cjkencodings/shift_jisx0213.txt
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/test/cjkencodings/shift_jisx0213.txt
@@ -0,0 +1,8 @@
+Python �̊J���́A1990 �N���납��J�n����Ă��܂��B
+�J���҂� Guido van Rossum �͋���p�̃v���O���~���O����uABC�v�̊J���ɎQ�����Ă��܂������AABC �͎��p��̖ړI�ɂ͂��܂�K���Ă��܂���ł����B
+���̂��߁AGuido �͂����p�I�ȃv���O���~���O����̊J�����J�n���A�p�� BBS �����̃R���f�B�ԑg�u�����e�B �p�C�\���v�̃t�@���ł��� Guido �͂��̌�����uPython�v�Ɩ��Â��܂����B
+���̂悤�Ȕw�i���琶�܂ꂽ Python �̌���݌v�́A�u�V���v���v�Łu�K�����e�Ձv�Ƃ����ڕW�ɏd�_���u����Ă��܂��B
+�����̃X�N���v�g�n����ł̓��[�U�̖ڐ�̗��֐���D�悵�ĐF�X�ȋ@�\������v�f�Ƃ��Ď������ꍇ�������̂ł����APython �ł͂������������׍H���ǉ�����邱�Ƃ͂��܂肠��܂���B
+���ꎩ�̂̋@�\�͍ŏ����ɉ������A�K�v�ȋ@�\�͊g�����W���[���Ƃ��Ēǉ�����A�Ƃ����̂� Python �̃|���V�[�ł��B
+
+�m�� �� �g�L�K�y ���� ������
diff --git a/lib-python/2.7/test/cjkencodings_test.py b/lib-python/2.7/test/cjkencodings_test.py
deleted file mode 100644
--- a/lib-python/2.7/test/cjkencodings_test.py
+++ /dev/null
@@ -1,1019 +0,0 @@
-teststring = {
-'big5': (
-"\xa6\x70\xa6\xf3\xa6\x62\x20\x50\x79\x74\x68\x6f\x6e\x20\xa4\xa4"
-"\xa8\xcf\xa5\xce\xac\x4a\xa6\xb3\xaa\xba\x20\x43\x20\x6c\x69\x62"
-"\x72\x61\x72\x79\x3f\x0a\xa1\x40\xa6\x62\xb8\xea\xb0\x54\xac\xec"
-"\xa7\xde\xa7\xd6\xb3\x74\xb5\x6f\xae\x69\xaa\xba\xa4\xb5\xa4\xd1"
-"\x2c\x20\xb6\x7d\xb5\x6f\xa4\xce\xb4\xfa\xb8\xd5\xb3\x6e\xc5\xe9"
-"\xaa\xba\xb3\x74\xab\xd7\xac\x4f\xa4\xa3\xae\x65\xa9\xbf\xb5\xf8"
-"\xaa\xba\x0a\xbd\xd2\xc3\x44\x2e\x20\xac\xb0\xa5\x5b\xa7\xd6\xb6"
-"\x7d\xb5\x6f\xa4\xce\xb4\xfa\xb8\xd5\xaa\xba\xb3\x74\xab\xd7\x2c"
-"\x20\xa7\xda\xad\xcc\xab\x4b\xb1\x60\xa7\xc6\xb1\xe6\xaf\xe0\xa7"
-"\x51\xa5\xce\xa4\x40\xa8\xc7\xa4\x77\xb6\x7d\xb5\x6f\xa6\x6e\xaa"
-"\xba\x0a\x6c\x69\x62\x72\x61\x72\x79\x2c\x20\xa8\xc3\xa6\xb3\xa4"
-"\x40\xad\xd3\x20\x66\x61\x73\x74\x20\x70\x72\x6f\x74\x6f\x74\x79"
-"\x70\x69\x6e\x67\x20\xaa\xba\x20\x70\x72\x6f\x67\x72\x61\x6d\x6d"
-"\x69\x6e\x67\x20\x6c\x61\x6e\x67\x75\x61\x67\x65\x20\xa5\x69\x0a"
-"\xa8\xd1\xa8\xcf\xa5\xce\x2e\x20\xa5\xd8\xab\x65\xa6\xb3\xb3\x5c"
-"\xb3\x5c\xa6\x68\xa6\x68\xaa\xba\x20\x6c\x69\x62\x72\x61\x72\x79"
-"\x20\xac\x4f\xa5\x48\x20\x43\x20\xbc\x67\xa6\xa8\x2c\x20\xa6\xd3"
-"\x20\x50\x79\x74\x68\x6f\x6e\x20\xac\x4f\xa4\x40\xad\xd3\x0a\x66"
-"\x61\x73\x74\x20\x70\x72\x6f\x74\x6f\x74\x79\x70\x69\x6e\x67\x20"
-"\xaa\xba\x20\x70\x72\x6f\x67\x72\x61\x6d\x6d\x69\x6e\x67\x20\x6c"
-"\x61\x6e\x67\x75\x61\x67\x65\x2e\x20\xac\x47\xa7\xda\xad\xcc\xa7"
-"\xc6\xb1\xe6\xaf\xe0\xb1\x4e\xac\x4a\xa6\xb3\xaa\xba\x0a\x43\x20"
-"\x6c\x69\x62\x72\x61\x72\x79\x20\xae\xb3\xa8\xec\x20\x50\x79\x74"
-"\x68\x6f\x6e\x20\xaa\xba\xc0\xf4\xb9\xd2\xa4\xa4\xb4\xfa\xb8\xd5"
-"\xa4\xce\xbe\xe3\xa6\x58\x2e\x20\xa8\xe4\xa4\xa4\xb3\xcc\xa5\x44"
-"\xad\x6e\xa4\x5d\xac\x4f\xa7\xda\xad\xcc\xa9\xd2\x0a\xad\x6e\xb0"
-"\x51\xbd\xd7\xaa\xba\xb0\xdd\xc3\x44\xb4\x4e\xac\x4f\x3a\x0a\x0a",
-"\xe5\xa6\x82\xe4\xbd\x95\xe5\x9c\xa8\x20\x50\x79\x74\x68\x6f\x6e"
-"\x20\xe4\xb8\xad\xe4\xbd\xbf\xe7\x94\xa8\xe6\x97\xa2\xe6\x9c\x89"
-"\xe7\x9a\x84\x20\x43\x20\x6c\x69\x62\x72\x61\x72\x79\x3f\x0a\xe3"
-"\x80\x80\xe5\x9c\xa8\xe8\xb3\x87\xe8\xa8\x8a\xe7\xa7\x91\xe6\x8a"
-"\x80\xe5\xbf\xab\xe9\x80\x9f\xe7\x99\xbc\xe5\xb1\x95\xe7\x9a\x84"
-"\xe4\xbb\x8a\xe5\xa4\xa9\x2c\x20\xe9\x96\x8b\xe7\x99\xbc\xe5\x8f"
-"\x8a\xe6\xb8\xac\xe8\xa9\xa6\xe8\xbb\x9f\xe9\xab\x94\xe7\x9a\x84"
-"\xe9\x80\x9f\xe5\xba\xa6\xe6\x98\xaf\xe4\xb8\x8d\xe5\xae\xb9\xe5"
-"\xbf\xbd\xe8\xa6\x96\xe7\x9a\x84\x0a\xe8\xaa\xb2\xe9\xa1\x8c\x2e"
-"\x20\xe7\x82\xba\xe5\x8a\xa0\xe5\xbf\xab\xe9\x96\x8b\xe7\x99\xbc"
-"\xe5\x8f\x8a\xe6\xb8\xac\xe8\xa9\xa6\xe7\x9a\x84\xe9\x80\x9f\xe5"
-"\xba\xa6\x2c\x20\xe6\x88\x91\xe5\x80\x91\xe4\xbe\xbf\xe5\xb8\xb8"
-"\xe5\xb8\x8c\xe6\x9c\x9b\xe8\x83\xbd\xe5\x88\xa9\xe7\x94\xa8\xe4"
-"\xb8\x80\xe4\xba\x9b\xe5\xb7\xb2\xe9\x96\x8b\xe7\x99\xbc\xe5\xa5"
-"\xbd\xe7\x9a\x84\x0a\x6c\x69\x62\x72\x61\x72\x79\x2c\x20\xe4\xb8"
-"\xa6\xe6\x9c\x89\xe4\xb8\x80\xe5\x80\x8b\x20\x66\x61\x73\x74\x20"
-"\x70\x72\x6f\x74\x6f\x74\x79\x70\x69\x6e\x67\x20\xe7\x9a\x84\x20"
-"\x70\x72\x6f\x67\x72\x61\x6d\x6d\x69\x6e\x67\x20\x6c\x61\x6e\x67"
-"\x75\x61\x67\x65\x20\xe5\x8f\xaf\x0a\xe4\xbe\x9b\xe4\xbd\xbf\xe7"
-"\x94\xa8\x2e\x20\xe7\x9b\xae\xe5\x89\x8d\xe6\x9c\x89\xe8\xa8\xb1"
-"\xe8\xa8\xb1\xe5\xa4\x9a\xe5\xa4\x9a\xe7\x9a\x84\x20\x6c\x69\x62"
-"\x72\x61\x72\x79\x20\xe6\x98\xaf\xe4\xbb\xa5\x20\x43\x20\xe5\xaf"
-"\xab\xe6\x88\x90\x2c\x20\xe8\x80\x8c\x20\x50\x79\x74\x68\x6f\x6e"
-"\x20\xe6\x98\xaf\xe4\xb8\x80\xe5\x80\x8b\x0a\x66\x61\x73\x74\x20"
-"\x70\x72\x6f\x74\x6f\x74\x79\x70\x69\x6e\x67\x20\xe7\x9a\x84\x20"
-"\x70\x72\x6f\x67\x72\x61\x6d\x6d\x69\x6e\x67\x20\x6c\x61\x6e\x67"
-"\x75\x61\x67\x65\x2e\x20\xe6\x95\x85\xe6\x88\x91\xe5\x80\x91\xe5"
-"\xb8\x8c\xe6\x9c\x9b\xe8\x83\xbd\xe5\xb0\x87\xe6\x97\xa2\xe6\x9c"
-"\x89\xe7\x9a\x84\x0a\x43\x20\x6c\x69\x62\x72\x61\x72\x79\x20\xe6"
-"\x8b\xbf\xe5\x88\xb0\x20\x50\x79\x74\x68\x6f\x6e\x20\xe7\x9a\x84"
-"\xe7\x92\xb0\xe5\xa2\x83\xe4\xb8\xad\xe6\xb8\xac\xe8\xa9\xa6\xe5"
-"\x8f\x8a\xe6\x95\xb4\xe5\x90\x88\x2e\x20\xe5\x85\xb6\xe4\xb8\xad"
-"\xe6\x9c\x80\xe4\xb8\xbb\xe8\xa6\x81\xe4\xb9\x9f\xe6\x98\xaf\xe6"
-"\x88\x91\xe5\x80\x91\xe6\x89\x80\x0a\xe8\xa6\x81\xe8\xa8\x8e\xe8"
-"\xab\x96\xe7\x9a\x84\xe5\x95\x8f\xe9\xa1\x8c\xe5\xb0\xb1\xe6\x98"
-"\xaf\x3a\x0a\x0a"),
-'big5hkscs': (
-"\x88\x45\x88\x5c\x8a\x73\x8b\xda\x8d\xd8\x0a\x88\x66\x88\x62\x88"
-"\xa7\x20\x88\xa7\x88\xa3\x0a",
-"\xf0\xa0\x84\x8c\xc4\x9a\xe9\xb5\xae\xe7\xbd\x93\xe6\xb4\x86\x0a"
-"\xc3\x8a\xc3\x8a\xcc\x84\xc3\xaa\x20\xc3\xaa\xc3\xaa\xcc\x84\x0a"),
-'cp949': (
-"\x8c\x63\xb9\xe6\xb0\xa2\xc7\xcf\x20\xbc\x84\xbd\xc3\xc4\xdd\xb6"
-"\xf3\x0a\x0a\xa8\xc0\xa8\xc0\xb3\xb3\x21\x21\x20\xec\xd7\xce\xfa"
-"\xea\xc5\xc6\xd0\x92\xe6\x90\x70\xb1\xc5\x20\xa8\xde\xa8\xd3\xc4"
-"\x52\xa2\xaf\xa2\xaf\xa2\xaf\x20\xb1\xe0\x8a\x96\x20\xa8\xd1\xb5"
-"\xb3\x20\xa8\xc0\x2e\x20\x2e\x0a\xe4\xac\xbf\xb5\xa8\xd1\xb4\xc9"
-"\xc8\xc2\x20\x2e\x20\x2e\x20\x2e\x20\x2e\x20\xbc\xad\xbf\xef\xb7"
-"\xef\x20\xb5\xaf\xc7\xd0\xeb\xe0\x20\xca\xab\xc4\x52\x20\x21\x20"
-"\x21\x20\x21\xa4\xd0\x2e\xa4\xd0\x0a\xc8\xe5\xc8\xe5\xc8\xe5\x20"
-"\xa4\xa1\xa4\xa1\xa4\xa1\xa1\xd9\xa4\xd0\x5f\xa4\xd0\x20\xbe\xee"
-"\x90\x8a\x20\xc5\xcb\xc4\xe2\x83\x4f\x20\xb5\xae\xc0\xc0\x20\xaf"
-"\x68\xce\xfa\xb5\xe9\xeb\xe0\x20\xa8\xc0\xb5\xe5\x83\x4f\x0a\xbc"
-"\xb3\x90\x6a\x20\xca\xab\xc4\x52\x20\x2e\x20\x2e\x20\x2e\x20\x2e"
-"\x20\xb1\xbc\xbe\xd6\x9a\x66\x20\xa8\xd1\xb1\xc5\x20\xa8\xde\x90"
-"\x74\xa8\xc2\x83\x4f\x20\xec\xd7\xec\xd2\xf4\xb9\xe5\xfc\xf1\xe9"
-"\xb1\xee\xa3\x8e\x0a\xbf\xcd\xbe\xac\xc4\x52\x20\x21\x20\x21\x20"
-"\xe4\xac\xbf\xb5\xa8\xd1\x20\xca\xab\xb4\xc9\xb1\xc5\x20\xa1\xd9"
-"\xdf\xbe\xb0\xfc\x20\xbe\xf8\xb4\xc9\xb1\xc5\xb4\xc9\x20\xe4\xac"
-"\xb4\xc9\xb5\xd8\xc4\x52\x20\xb1\xdb\xbe\xd6\x8a\xdb\x0a\xa8\xde"
-"\xb7\xc1\xb5\xe0\xce\xfa\x20\x9a\xc3\xc7\xb4\xbd\xa4\xc4\x52\x20"
-"\xbe\xee\x90\x8a\x20\xec\xd7\xec\xd2\xf4\xb9\xe5\xfc\xf1\xe9\x9a"
-"\xc4\xa8\xef\xb5\xe9\x9d\xda\x21\x21\x20\xa8\xc0\xa8\xc0\xb3\xb3"
-"\xa2\xbd\x20\xa1\xd2\xa1\xd2\x2a\x0a\x0a",
-"\xeb\x98\xa0\xeb\xb0\xa9\xea\xb0\x81\xed\x95\x98\x20\xed\x8e\xb2"
-"\xec\x8b\x9c\xec\xbd\x9c\xeb\x9d\xbc\x0a\x0a\xe3\x89\xaf\xe3\x89"
-"\xaf\xeb\x82\xa9\x21\x21\x20\xe5\x9b\xa0\xe4\xb9\x9d\xe6\x9c\x88"
-"\xed\x8c\xa8\xeb\xaf\xa4\xeb\xa6\x94\xea\xb6\x88\x20\xe2\x93\xa1"
-"\xe2\x93\x96\xed\x9b\x80\xc2\xbf\xc2\xbf\xc2\xbf\x20\xea\xb8\x8d"
-"\xeb\x92\x99\x20\xe2\x93\x94\xeb\x8e\xa8\x20\xe3\x89\xaf\x2e\x20"
-"\x2e\x0a\xe4\xba\x9e\xec\x98\x81\xe2\x93\x94\xeb\x8a\xa5\xed\x9a"
-"\xb9\x20\x2e\x20\x2e\x20\x2e\x20\x2e\x20\xec\x84\x9c\xec\x9a\xb8"
-"\xeb\xa4\x84\x20\xeb\x8e\x90\xed\x95\x99\xe4\xb9\x99\x20\xe5\xae"
-"\xb6\xed\x9b\x80\x20\x21\x20\x21\x20\x21\xe3\x85\xa0\x2e\xe3\x85"
-"\xa0\x0a\xed\x9d\x90\xed\x9d\x90\xed\x9d\x90\x20\xe3\x84\xb1\xe3"
-"\x84\xb1\xe3\x84\xb1\xe2\x98\x86\xe3\x85\xa0\x5f\xe3\x85\xa0\x20"
-"\xec\x96\xb4\xeb\xa6\xa8\x20\xed\x83\xb8\xec\xbd\xb0\xea\xb8\x90"
-"\x20\xeb\x8e\x8c\xec\x9d\x91\x20\xec\xb9\x91\xe4\xb9\x9d\xeb\x93"
-"\xa4\xe4\xb9\x99\x20\xe3\x89\xaf\xeb\x93\x9c\xea\xb8\x90\x0a\xec"
-"\x84\xa4\xeb\xa6\x8c\x20\xe5\xae\xb6\xed\x9b\x80\x20\x2e\x20\x2e"
-"\x20\x2e\x20\x2e\x20\xea\xb5\xb4\xec\x95\xa0\xec\x89\x8c\x20\xe2"
-"\x93\x94\xea\xb6\x88\x20\xe2\x93\xa1\xeb\xa6\x98\xe3\x89\xb1\xea"
-"\xb8\x90\x20\xe5\x9b\xa0\xe4\xbb\x81\xe5\xb7\x9d\xef\xa6\x81\xe4"
-"\xb8\xad\xea\xb9\x8c\xec\xa6\xbc\x0a\xec\x99\x80\xec\x92\x80\xed"
-"\x9b\x80\x20\x21\x20\x21\x20\xe4\xba\x9e\xec\x98\x81\xe2\x93\x94"
-"\x20\xe5\xae\xb6\xeb\x8a\xa5\xea\xb6\x88\x20\xe2\x98\x86\xe4\xb8"
-"\x8a\xea\xb4\x80\x20\xec\x97\x86\xeb\x8a\xa5\xea\xb6\x88\xeb\x8a"
-"\xa5\x20\xe4\xba\x9e\xeb\x8a\xa5\xeb\x92\x88\xed\x9b\x80\x20\xea"
-"\xb8\x80\xec\x95\xa0\xeb\x93\xb4\x0a\xe2\x93\xa1\xeb\xa0\xa4\xeb"
-"\x93\x80\xe4\xb9\x9d\x20\xec\x8b\x80\xed\x92\x94\xec\x88\xb4\xed"
-"\x9b\x80\x20\xec\x96\xb4\xeb\xa6\xa8\x20\xe5\x9b\xa0\xe4\xbb\x81"
-"\xe5\xb7\x9d\xef\xa6\x81\xe4\xb8\xad\xec\x8b\x81\xe2\x91\xa8\xeb"
-"\x93\xa4\xec\x95\x9c\x21\x21\x20\xe3\x89\xaf\xe3\x89\xaf\xeb\x82"
-"\xa9\xe2\x99\xa1\x20\xe2\x8c\x92\xe2\x8c\x92\x2a\x0a\x0a"),
-'euc_jisx0213': (
-"\x50\x79\x74\x68\x6f\x6e\x20\xa4\xce\xb3\xab\xc8\xaf\xa4\xcf\xa1"
-"\xa2\x31\x39\x39\x30\x20\xc7\xaf\xa4\xb4\xa4\xed\xa4\xab\xa4\xe9"
-"\xb3\xab\xbb\xcf\xa4\xb5\xa4\xec\xa4\xc6\xa4\xa4\xa4\xde\xa4\xb9"
-"\xa1\xa3\x0a\xb3\xab\xc8\xaf\xbc\xd4\xa4\xce\x20\x47\x75\x69\x64"
-"\x6f\x20\x76\x61\x6e\x20\x52\x6f\x73\x73\x75\x6d\x20\xa4\xcf\xb6"
-"\xb5\xb0\xe9\xcd\xd1\xa4\xce\xa5\xd7\xa5\xed\xa5\xb0\xa5\xe9\xa5"
-"\xdf\xa5\xf3\xa5\xb0\xb8\xc0\xb8\xec\xa1\xd6\x41\x42\x43\xa1\xd7"
-"\xa4\xce\xb3\xab\xc8\xaf\xa4\xcb\xbb\xb2\xb2\xc3\xa4\xb7\xa4\xc6"
-"\xa4\xa4\xa4\xde\xa4\xb7\xa4\xbf\xa4\xac\xa1\xa2\x41\x42\x43\x20"
-"\xa4\xcf\xbc\xc2\xcd\xd1\xbe\xe5\xa4\xce\xcc\xdc\xc5\xaa\xa4\xcb"
-"\xa4\xcf\xa4\xa2\xa4\xde\xa4\xea\xc5\xac\xa4\xb7\xa4\xc6\xa4\xa4"
-"\xa4\xde\xa4\xbb\xa4\xf3\xa4\xc7\xa4\xb7\xa4\xbf\xa1\xa3\x0a\xa4"
-"\xb3\xa4\xce\xa4\xbf\xa4\xe1\xa1\xa2\x47\x75\x69\x64\x6f\x20\xa4"
-"\xcf\xa4\xe8\xa4\xea\xbc\xc2\xcd\xd1\xc5\xaa\xa4\xca\xa5\xd7\xa5"
-"\xed\xa5\xb0\xa5\xe9\xa5\xdf\xa5\xf3\xa5\xb0\xb8\xc0\xb8\xec\xa4"
-"\xce\xb3\xab\xc8\xaf\xa4\xf2\xb3\xab\xbb\xcf\xa4\xb7\xa1\xa2\xb1"
-"\xd1\xb9\xf1\x20\x42\x42\x53\x20\xca\xfc\xc1\xf7\xa4\xce\xa5\xb3"
-"\xa5\xe1\xa5\xc7\xa5\xa3\xc8\xd6\xc1\xc8\xa1\xd6\xa5\xe2\xa5\xf3"
-"\xa5\xc6\xa5\xa3\x20\xa5\xd1\xa5\xa4\xa5\xbd\xa5\xf3\xa1\xd7\xa4"
-"\xce\xa5\xd5\xa5\xa1\xa5\xf3\xa4\xc7\xa4\xa2\xa4\xeb\x20\x47\x75"
-"\x69\x64\x6f\x20\xa4\xcf\xa4\xb3\xa4\xce\xb8\xc0\xb8\xec\xa4\xf2"
-"\xa1\xd6\x50\x79\x74\x68\x6f\x6e\xa1\xd7\xa4\xc8\xcc\xbe\xa4\xc5"
-"\xa4\xb1\xa4\xde\xa4\xb7\xa4\xbf\xa1\xa3\x0a\xa4\xb3\xa4\xce\xa4"
-"\xe8\xa4\xa6\xa4\xca\xc7\xd8\xb7\xca\xa4\xab\xa4\xe9\xc0\xb8\xa4"
-"\xde\xa4\xec\xa4\xbf\x20\x50\x79\x74\x68\x6f\x6e\x20\xa4\xce\xb8"
-"\xc0\xb8\xec\xc0\xdf\xb7\xd7\xa4\xcf\xa1\xa2\xa1\xd6\xa5\xb7\xa5"
-"\xf3\xa5\xd7\xa5\xeb\xa1\xd7\xa4\xc7\xa1\xd6\xbd\xac\xc6\xc0\xa4"
-"\xac\xcd\xc6\xb0\xd7\xa1\xd7\xa4\xc8\xa4\xa4\xa4\xa6\xcc\xdc\xc9"
-"\xb8\xa4\xcb\xbd\xc5\xc5\xc0\xa4\xac\xc3\xd6\xa4\xab\xa4\xec\xa4"
-"\xc6\xa4\xa4\xa4\xde\xa4\xb9\xa1\xa3\x0a\xc2\xbf\xa4\xaf\xa4\xce"
-"\xa5\xb9\xa5\xaf\xa5\xea\xa5\xd7\xa5\xc8\xb7\xcf\xb8\xc0\xb8\xec"
-"\xa4\xc7\xa4\xcf\xa5\xe6\xa1\xbc\xa5\xb6\xa4\xce\xcc\xdc\xc0\xe8"
-"\xa4\xce\xcd\xf8\xca\xd8\xc0\xad\xa4\xf2\xcd\xa5\xc0\xe8\xa4\xb7"
-"\xa4\xc6\xbf\xa7\xa1\xb9\xa4\xca\xb5\xa1\xc7\xbd\xa4\xf2\xb8\xc0"
-"\xb8\xec\xcd\xd7\xc1\xc7\xa4\xc8\xa4\xb7\xa4\xc6\xbc\xe8\xa4\xea"
-"\xc6\xfe\xa4\xec\xa4\xeb\xbe\xec\xb9\xe7\xa4\xac\xc2\xbf\xa4\xa4"
-"\xa4\xce\xa4\xc7\xa4\xb9\xa4\xac\xa1\xa2\x50\x79\x74\x68\x6f\x6e"
-"\x20\xa4\xc7\xa4\xcf\xa4\xbd\xa4\xa6\xa4\xa4\xa4\xc3\xa4\xbf\xbe"
-"\xae\xba\xd9\xb9\xa9\xa4\xac\xc4\xc9\xb2\xc3\xa4\xb5\xa4\xec\xa4"
-"\xeb\xa4\xb3\xa4\xc8\xa4\xcf\xa4\xa2\xa4\xde\xa4\xea\xa4\xa2\xa4"
-"\xea\xa4\xde\xa4\xbb\xa4\xf3\xa1\xa3\x0a\xb8\xc0\xb8\xec\xbc\xab"
-"\xc2\xce\xa4\xce\xb5\xa1\xc7\xbd\xa4\xcf\xba\xc7\xbe\xae\xb8\xc2"
-"\xa4\xcb\xb2\xa1\xa4\xb5\xa4\xa8\xa1\xa2\xc9\xac\xcd\xd7\xa4\xca"
-"\xb5\xa1\xc7\xbd\xa4\xcf\xb3\xc8\xc4\xa5\xa5\xe2\xa5\xb8\xa5\xe5"
-"\xa1\xbc\xa5\xeb\xa4\xc8\xa4\xb7\xa4\xc6\xc4\xc9\xb2\xc3\xa4\xb9"
-"\xa4\xeb\xa1\xa2\xa4\xc8\xa4\xa4\xa4\xa6\xa4\xce\xa4\xac\x20\x50"
-"\x79\x74\x68\x6f\x6e\x20\xa4\xce\xa5\xdd\xa5\xea\xa5\xb7\xa1\xbc"
-"\xa4\xc7\xa4\xb9\xa1\xa3\x0a\x0a\xa5\xce\xa4\xf7\x20\xa5\xfe\x20"
-"\xa5\xc8\xa5\xad\xaf\xac\xaf\xda\x20\xcf\xe3\x8f\xfe\xd8\x20\x8f"
-"\xfe\xd4\x8f\xfe\xe8\x8f\xfc\xd6\x0a",
-"\x50\x79\x74\x68\x6f\x6e\x20\xe3\x81\xae\xe9\x96\x8b\xe7\x99\xba"
-"\xe3\x81\xaf\xe3\x80\x81\x31\x39\x39\x30\x20\xe5\xb9\xb4\xe3\x81"
-"\x94\xe3\x82\x8d\xe3\x81\x8b\xe3\x82\x89\xe9\x96\x8b\xe5\xa7\x8b"
-"\xe3\x81\x95\xe3\x82\x8c\xe3\x81\xa6\xe3\x81\x84\xe3\x81\xbe\xe3"
-"\x81\x99\xe3\x80\x82\x0a\xe9\x96\x8b\xe7\x99\xba\xe8\x80\x85\xe3"
-"\x81\xae\x20\x47\x75\x69\x64\x6f\x20\x76\x61\x6e\x20\x52\x6f\x73"
-"\x73\x75\x6d\x20\xe3\x81\xaf\xe6\x95\x99\xe8\x82\xb2\xe7\x94\xa8"
-"\xe3\x81\xae\xe3\x83\x97\xe3\x83\xad\xe3\x82\xb0\xe3\x83\xa9\xe3"
-"\x83\x9f\xe3\x83\xb3\xe3\x82\xb0\xe8\xa8\x80\xe8\xaa\x9e\xe3\x80"
-"\x8c\x41\x42\x43\xe3\x80\x8d\xe3\x81\xae\xe9\x96\x8b\xe7\x99\xba"
-"\xe3\x81\xab\xe5\x8f\x82\xe5\x8a\xa0\xe3\x81\x97\xe3\x81\xa6\xe3"
-"\x81\x84\xe3\x81\xbe\xe3\x81\x97\xe3\x81\x9f\xe3\x81\x8c\xe3\x80"
-"\x81\x41\x42\x43\x20\xe3\x81\xaf\xe5\xae\x9f\xe7\x94\xa8\xe4\xb8"
-"\x8a\xe3\x81\xae\xe7\x9b\xae\xe7\x9a\x84\xe3\x81\xab\xe3\x81\xaf"
-"\xe3\x81\x82\xe3\x81\xbe\xe3\x82\x8a\xe9\x81\xa9\xe3\x81\x97\xe3"
-"\x81\xa6\xe3\x81\x84\xe3\x81\xbe\xe3\x81\x9b\xe3\x82\x93\xe3\x81"
-"\xa7\xe3\x81\x97\xe3\x81\x9f\xe3\x80\x82\x0a\xe3\x81\x93\xe3\x81"
-"\xae\xe3\x81\x9f\xe3\x82\x81\xe3\x80\x81\x47\x75\x69\x64\x6f\x20"
-"\xe3\x81\xaf\xe3\x82\x88\xe3\x82\x8a\xe5\xae\x9f\xe7\x94\xa8\xe7"
-"\x9a\x84\xe3\x81\xaa\xe3\x83\x97\xe3\x83\xad\xe3\x82\xb0\xe3\x83"
-"\xa9\xe3\x83\x9f\xe3\x83\xb3\xe3\x82\xb0\xe8\xa8\x80\xe8\xaa\x9e"
-"\xe3\x81\xae\xe9\x96\x8b\xe7\x99\xba\xe3\x82\x92\xe9\x96\x8b\xe5"
-"\xa7\x8b\xe3\x81\x97\xe3\x80\x81\xe8\x8b\xb1\xe5\x9b\xbd\x20\x42"
-"\x42\x53\x20\xe6\x94\xbe\xe9\x80\x81\xe3\x81\xae\xe3\x82\xb3\xe3"
-"\x83\xa1\xe3\x83\x87\xe3\x82\xa3\xe7\x95\xaa\xe7\xb5\x84\xe3\x80"
-"\x8c\xe3\x83\xa2\xe3\x83\xb3\xe3\x83\x86\xe3\x82\xa3\x20\xe3\x83"
-"\x91\xe3\x82\xa4\xe3\x82\xbd\xe3\x83\xb3\xe3\x80\x8d\xe3\x81\xae"
-"\xe3\x83\x95\xe3\x82\xa1\xe3\x83\xb3\xe3\x81\xa7\xe3\x81\x82\xe3"
-"\x82\x8b\x20\x47\x75\x69\x64\x6f\x20\xe3\x81\xaf\xe3\x81\x93\xe3"
-"\x81\xae\xe8\xa8\x80\xe8\xaa\x9e\xe3\x82\x92\xe3\x80\x8c\x50\x79"
-"\x74\x68\x6f\x6e\xe3\x80\x8d\xe3\x81\xa8\xe5\x90\x8d\xe3\x81\xa5"
-"\xe3\x81\x91\xe3\x81\xbe\xe3\x81\x97\xe3\x81\x9f\xe3\x80\x82\x0a"
-"\xe3\x81\x93\xe3\x81\xae\xe3\x82\x88\xe3\x81\x86\xe3\x81\xaa\xe8"
-"\x83\x8c\xe6\x99\xaf\xe3\x81\x8b\xe3\x82\x89\xe7\x94\x9f\xe3\x81"
-"\xbe\xe3\x82\x8c\xe3\x81\x9f\x20\x50\x79\x74\x68\x6f\x6e\x20\xe3"
-"\x81\xae\xe8\xa8\x80\xe8\xaa\x9e\xe8\xa8\xad\xe8\xa8\x88\xe3\x81"
-"\xaf\xe3\x80\x81\xe3\x80\x8c\xe3\x82\xb7\xe3\x83\xb3\xe3\x83\x97"
-"\xe3\x83\xab\xe3\x80\x8d\xe3\x81\xa7\xe3\x80\x8c\xe7\xbf\x92\xe5"
-"\xbe\x97\xe3\x81\x8c\xe5\xae\xb9\xe6\x98\x93\xe3\x80\x8d\xe3\x81"
-"\xa8\xe3\x81\x84\xe3\x81\x86\xe7\x9b\xae\xe6\xa8\x99\xe3\x81\xab"
-"\xe9\x87\x8d\xe7\x82\xb9\xe3\x81\x8c\xe7\xbd\xae\xe3\x81\x8b\xe3"
-"\x82\x8c\xe3\x81\xa6\xe3\x81\x84\xe3\x81\xbe\xe3\x81\x99\xe3\x80"
-"\x82\x0a\xe5\xa4\x9a\xe3\x81\x8f\xe3\x81\xae\xe3\x82\xb9\xe3\x82"
-"\xaf\xe3\x83\xaa\xe3\x83\x97\xe3\x83\x88\xe7\xb3\xbb\xe8\xa8\x80"
-"\xe8\xaa\x9e\xe3\x81\xa7\xe3\x81\xaf\xe3\x83\xa6\xe3\x83\xbc\xe3"
-"\x82\xb6\xe3\x81\xae\xe7\x9b\xae\xe5\x85\x88\xe3\x81\xae\xe5\x88"
-"\xa9\xe4\xbe\xbf\xe6\x80\xa7\xe3\x82\x92\xe5\x84\xaa\xe5\x85\x88"
-"\xe3\x81\x97\xe3\x81\xa6\xe8\x89\xb2\xe3\x80\x85\xe3\x81\xaa\xe6"
-"\xa9\x9f\xe8\x83\xbd\xe3\x82\x92\xe8\xa8\x80\xe8\xaa\x9e\xe8\xa6"
-"\x81\xe7\xb4\xa0\xe3\x81\xa8\xe3\x81\x97\xe3\x81\xa6\xe5\x8f\x96"
-"\xe3\x82\x8a\xe5\x85\xa5\xe3\x82\x8c\xe3\x82\x8b\xe5\xa0\xb4\xe5"
-"\x90\x88\xe3\x81\x8c\xe5\xa4\x9a\xe3\x81\x84\xe3\x81\xae\xe3\x81"
-"\xa7\xe3\x81\x99\xe3\x81\x8c\xe3\x80\x81\x50\x79\x74\x68\x6f\x6e"
-"\x20\xe3\x81\xa7\xe3\x81\xaf\xe3\x81\x9d\xe3\x81\x86\xe3\x81\x84"
-"\xe3\x81\xa3\xe3\x81\x9f\xe5\xb0\x8f\xe7\xb4\xb0\xe5\xb7\xa5\xe3"
-"\x81\x8c\xe8\xbf\xbd\xe5\x8a\xa0\xe3\x81\x95\xe3\x82\x8c\xe3\x82"
-"\x8b\xe3\x81\x93\xe3\x81\xa8\xe3\x81\xaf\xe3\x81\x82\xe3\x81\xbe"
-"\xe3\x82\x8a\xe3\x81\x82\xe3\x82\x8a\xe3\x81\xbe\xe3\x81\x9b\xe3"
-"\x82\x93\xe3\x80\x82\x0a\xe8\xa8\x80\xe8\xaa\x9e\xe8\x87\xaa\xe4"
-"\xbd\x93\xe3\x81\xae\xe6\xa9\x9f\xe8\x83\xbd\xe3\x81\xaf\xe6\x9c"
-"\x80\xe5\xb0\x8f\xe9\x99\x90\xe3\x81\xab\xe6\x8a\xbc\xe3\x81\x95"
-"\xe3\x81\x88\xe3\x80\x81\xe5\xbf\x85\xe8\xa6\x81\xe3\x81\xaa\xe6"
-"\xa9\x9f\xe8\x83\xbd\xe3\x81\xaf\xe6\x8b\xa1\xe5\xbc\xb5\xe3\x83"
-"\xa2\xe3\x82\xb8\xe3\x83\xa5\xe3\x83\xbc\xe3\x83\xab\xe3\x81\xa8"
-"\xe3\x81\x97\xe3\x81\xa6\xe8\xbf\xbd\xe5\x8a\xa0\xe3\x81\x99\xe3"
-"\x82\x8b\xe3\x80\x81\xe3\x81\xa8\xe3\x81\x84\xe3\x81\x86\xe3\x81"
-"\xae\xe3\x81\x8c\x20\x50\x79\x74\x68\x6f\x6e\x20\xe3\x81\xae\xe3"
-"\x83\x9d\xe3\x83\xaa\xe3\x82\xb7\xe3\x83\xbc\xe3\x81\xa7\xe3\x81"
-"\x99\xe3\x80\x82\x0a\x0a\xe3\x83\x8e\xe3\x81\x8b\xe3\x82\x9a\x20"
-"\xe3\x83\x88\xe3\x82\x9a\x20\xe3\x83\x88\xe3\x82\xad\xef\xa8\xb6"
-"\xef\xa8\xb9\x20\xf0\xa1\x9a\xb4\xf0\xaa\x8e\x8c\x20\xe9\xba\x80"
-"\xe9\xbd\x81\xf0\xa9\x9b\xb0\x0a"),
-'euc_jp': (
-"\x50\x79\x74\x68\x6f\x6e\x20\xa4\xce\xb3\xab\xc8\xaf\xa4\xcf\xa1"
-"\xa2\x31\x39\x39\x30\x20\xc7\xaf\xa4\xb4\xa4\xed\xa4\xab\xa4\xe9"
-"\xb3\xab\xbb\xcf\xa4\xb5\xa4\xec\xa4\xc6\xa4\xa4\xa4\xde\xa4\xb9"
-"\xa1\xa3\x0a\xb3\xab\xc8\xaf\xbc\xd4\xa4\xce\x20\x47\x75\x69\x64"
-"\x6f\x20\x76\x61\x6e\x20\x52\x6f\x73\x73\x75\x6d\x20\xa4\xcf\xb6"
-"\xb5\xb0\xe9\xcd\xd1\xa4\xce\xa5\xd7\xa5\xed\xa5\xb0\xa5\xe9\xa5"
-"\xdf\xa5\xf3\xa5\xb0\xb8\xc0\xb8\xec\xa1\xd6\x41\x42\x43\xa1\xd7"
-"\xa4\xce\xb3\xab\xc8\xaf\xa4\xcb\xbb\xb2\xb2\xc3\xa4\xb7\xa4\xc6"
-"\xa4\xa4\xa4\xde\xa4\xb7\xa4\xbf\xa4\xac\xa1\xa2\x41\x42\x43\x20"
-"\xa4\xcf\xbc\xc2\xcd\xd1\xbe\xe5\xa4\xce\xcc\xdc\xc5\xaa\xa4\xcb"
-"\xa4\xcf\xa4\xa2\xa4\xde\xa4\xea\xc5\xac\xa4\xb7\xa4\xc6\xa4\xa4"
-"\xa4\xde\xa4\xbb\xa4\xf3\xa4\xc7\xa4\xb7\xa4\xbf\xa1\xa3\x0a\xa4"
-"\xb3\xa4\xce\xa4\xbf\xa4\xe1\xa1\xa2\x47\x75\x69\x64\x6f\x20\xa4"
-"\xcf\xa4\xe8\xa4\xea\xbc\xc2\xcd\xd1\xc5\xaa\xa4\xca\xa5\xd7\xa5"
-"\xed\xa5\xb0\xa5\xe9\xa5\xdf\xa5\xf3\xa5\xb0\xb8\xc0\xb8\xec\xa4"
-"\xce\xb3\xab\xc8\xaf\xa4\xf2\xb3\xab\xbb\xcf\xa4\xb7\xa1\xa2\xb1"
-"\xd1\xb9\xf1\x20\x42\x42\x53\x20\xca\xfc\xc1\xf7\xa4\xce\xa5\xb3"
-"\xa5\xe1\xa5\xc7\xa5\xa3\xc8\xd6\xc1\xc8\xa1\xd6\xa5\xe2\xa5\xf3"
-"\xa5\xc6\xa5\xa3\x20\xa5\xd1\xa5\xa4\xa5\xbd\xa5\xf3\xa1\xd7\xa4"
-"\xce\xa5\xd5\xa5\xa1\xa5\xf3\xa4\xc7\xa4\xa2\xa4\xeb\x20\x47\x75"
-"\x69\x64\x6f\x20\xa4\xcf\xa4\xb3\xa4\xce\xb8\xc0\xb8\xec\xa4\xf2"
-"\xa1\xd6\x50\x79\x74\x68\x6f\x6e\xa1\xd7\xa4\xc8\xcc\xbe\xa4\xc5"
-"\xa4\xb1\xa4\xde\xa4\xb7\xa4\xbf\xa1\xa3\x0a\xa4\xb3\xa4\xce\xa4"
-"\xe8\xa4\xa6\xa4\xca\xc7\xd8\xb7\xca\xa4\xab\xa4\xe9\xc0\xb8\xa4"
-"\xde\xa4\xec\xa4\xbf\x20\x50\x79\x74\x68\x6f\x6e\x20\xa4\xce\xb8"
-"\xc0\xb8\xec\xc0\xdf\xb7\xd7\xa4\xcf\xa1\xa2\xa1\xd6\xa5\xb7\xa5"
-"\xf3\xa5\xd7\xa5\xeb\xa1\xd7\xa4\xc7\xa1\xd6\xbd\xac\xc6\xc0\xa4"
-"\xac\xcd\xc6\xb0\xd7\xa1\xd7\xa4\xc8\xa4\xa4\xa4\xa6\xcc\xdc\xc9"
-"\xb8\xa4\xcb\xbd\xc5\xc5\xc0\xa4\xac\xc3\xd6\xa4\xab\xa4\xec\xa4"
-"\xc6\xa4\xa4\xa4\xde\xa4\xb9\xa1\xa3\x0a\xc2\xbf\xa4\xaf\xa4\xce"
-"\xa5\xb9\xa5\xaf\xa5\xea\xa5\xd7\xa5\xc8\xb7\xcf\xb8\xc0\xb8\xec"
-"\xa4\xc7\xa4\xcf\xa5\xe6\xa1\xbc\xa5\xb6\xa4\xce\xcc\xdc\xc0\xe8"
-"\xa4\xce\xcd\xf8\xca\xd8\xc0\xad\xa4\xf2\xcd\xa5\xc0\xe8\xa4\xb7"
-"\xa4\xc6\xbf\xa7\xa1\xb9\xa4\xca\xb5\xa1\xc7\xbd\xa4\xf2\xb8\xc0"
-"\xb8\xec\xcd\xd7\xc1\xc7\xa4\xc8\xa4\xb7\xa4\xc6\xbc\xe8\xa4\xea"
-"\xc6\xfe\xa4\xec\xa4\xeb\xbe\xec\xb9\xe7\xa4\xac\xc2\xbf\xa4\xa4"
-"\xa4\xce\xa4\xc7\xa4\xb9\xa4\xac\xa1\xa2\x50\x79\x74\x68\x6f\x6e"
-"\x20\xa4\xc7\xa4\xcf\xa4\xbd\xa4\xa6\xa4\xa4\xa4\xc3\xa4\xbf\xbe"
-"\xae\xba\xd9\xb9\xa9\xa4\xac\xc4\xc9\xb2\xc3\xa4\xb5\xa4\xec\xa4"
-"\xeb\xa4\xb3\xa4\xc8\xa4\xcf\xa4\xa2\xa4\xde\xa4\xea\xa4\xa2\xa4"
-"\xea\xa4\xde\xa4\xbb\xa4\xf3\xa1\xa3\x0a\xb8\xc0\xb8\xec\xbc\xab"
-"\xc2\xce\xa4\xce\xb5\xa1\xc7\xbd\xa4\xcf\xba\xc7\xbe\xae\xb8\xc2"
-"\xa4\xcb\xb2\xa1\xa4\xb5\xa4\xa8\xa1\xa2\xc9\xac\xcd\xd7\xa4\xca"
-"\xb5\xa1\xc7\xbd\xa4\xcf\xb3\xc8\xc4\xa5\xa5\xe2\xa5\xb8\xa5\xe5"
-"\xa1\xbc\xa5\xeb\xa4\xc8\xa4\xb7\xa4\xc6\xc4\xc9\xb2\xc3\xa4\xb9"
-"\xa4\xeb\xa1\xa2\xa4\xc8\xa4\xa4\xa4\xa6\xa4\xce\xa4\xac\x20\x50"
-"\x79\x74\x68\x6f\x6e\x20\xa4\xce\xa5\xdd\xa5\xea\xa5\xb7\xa1\xbc"
-"\xa4\xc7\xa4\xb9\xa1\xa3\x0a\x0a",
-"\x50\x79\x74\x68\x6f\x6e\x20\xe3\x81\xae\xe9\x96\x8b\xe7\x99\xba"
-"\xe3\x81\xaf\xe3\x80\x81\x31\x39\x39\x30\x20\xe5\xb9\xb4\xe3\x81"
-"\x94\xe3\x82\x8d\xe3\x81\x8b\xe3\x82\x89\xe9\x96\x8b\xe5\xa7\x8b"
-"\xe3\x81\x95\xe3\x82\x8c\xe3\x81\xa6\xe3\x81\x84\xe3\x81\xbe\xe3"
-"\x81\x99\xe3\x80\x82\x0a\xe9\x96\x8b\xe7\x99\xba\xe8\x80\x85\xe3"
-"\x81\xae\x20\x47\x75\x69\x64\x6f\x20\x76\x61\x6e\x20\x52\x6f\x73"
-"\x73\x75\x6d\x20\xe3\x81\xaf\xe6\x95\x99\xe8\x82\xb2\xe7\x94\xa8"
-"\xe3\x81\xae\xe3\x83\x97\xe3\x83\xad\xe3\x82\xb0\xe3\x83\xa9\xe3"
-"\x83\x9f\xe3\x83\xb3\xe3\x82\xb0\xe8\xa8\x80\xe8\xaa\x9e\xe3\x80"
-"\x8c\x41\x42\x43\xe3\x80\x8d\xe3\x81\xae\xe9\x96\x8b\xe7\x99\xba"
-"\xe3\x81\xab\xe5\x8f\x82\xe5\x8a\xa0\xe3\x81\x97\xe3\x81\xa6\xe3"
-"\x81\x84\xe3\x81\xbe\xe3\x81\x97\xe3\x81\x9f\xe3\x81\x8c\xe3\x80"
-"\x81\x41\x42\x43\x20\xe3\x81\xaf\xe5\xae\x9f\xe7\x94\xa8\xe4\xb8"
-"\x8a\xe3\x81\xae\xe7\x9b\xae\xe7\x9a\x84\xe3\x81\xab\xe3\x81\xaf"
-"\xe3\x81\x82\xe3\x81\xbe\xe3\x82\x8a\xe9\x81\xa9\xe3\x81\x97\xe3"
-"\x81\xa6\xe3\x81\x84\xe3\x81\xbe\xe3\x81\x9b\xe3\x82\x93\xe3\x81"
-"\xa7\xe3\x81\x97\xe3\x81\x9f\xe3\x80\x82\x0a\xe3\x81\x93\xe3\x81"
-"\xae\xe3\x81\x9f\xe3\x82\x81\xe3\x80\x81\x47\x75\x69\x64\x6f\x20"
-"\xe3\x81\xaf\xe3\x82\x88\xe3\x82\x8a\xe5\xae\x9f\xe7\x94\xa8\xe7"
-"\x9a\x84\xe3\x81\xaa\xe3\x83\x97\xe3\x83\xad\xe3\x82\xb0\xe3\x83"
-"\xa9\xe3\x83\x9f\xe3\x83\xb3\xe3\x82\xb0\xe8\xa8\x80\xe8\xaa\x9e"
-"\xe3\x81\xae\xe9\x96\x8b\xe7\x99\xba\xe3\x82\x92\xe9\x96\x8b\xe5"
-"\xa7\x8b\xe3\x81\x97\xe3\x80\x81\xe8\x8b\xb1\xe5\x9b\xbd\x20\x42"
-"\x42\x53\x20\xe6\x94\xbe\xe9\x80\x81\xe3\x81\xae\xe3\x82\xb3\xe3"
-"\x83\xa1\xe3\x83\x87\xe3\x82\xa3\xe7\x95\xaa\xe7\xb5\x84\xe3\x80"
-"\x8c\xe3\x83\xa2\xe3\x83\xb3\xe3\x83\x86\xe3\x82\xa3\x20\xe3\x83"
-"\x91\xe3\x82\xa4\xe3\x82\xbd\xe3\x83\xb3\xe3\x80\x8d\xe3\x81\xae"
-"\xe3\x83\x95\xe3\x82\xa1\xe3\x83\xb3\xe3\x81\xa7\xe3\x81\x82\xe3"
-"\x82\x8b\x20\x47\x75\x69\x64\x6f\x20\xe3\x81\xaf\xe3\x81\x93\xe3"
-"\x81\xae\xe8\xa8\x80\xe8\xaa\x9e\xe3\x82\x92\xe3\x80\x8c\x50\x79"
-"\x74\x68\x6f\x6e\xe3\x80\x8d\xe3\x81\xa8\xe5\x90\x8d\xe3\x81\xa5"
-"\xe3\x81\x91\xe3\x81\xbe\xe3\x81\x97\xe3\x81\x9f\xe3\x80\x82\x0a"
-"\xe3\x81\x93\xe3\x81\xae\xe3\x82\x88\xe3\x81\x86\xe3\x81\xaa\xe8"
-"\x83\x8c\xe6\x99\xaf\xe3\x81\x8b\xe3\x82\x89\xe7\x94\x9f\xe3\x81"
-"\xbe\xe3\x82\x8c\xe3\x81\x9f\x20\x50\x79\x74\x68\x6f\x6e\x20\xe3"
-"\x81\xae\xe8\xa8\x80\xe8\xaa\x9e\xe8\xa8\xad\xe8\xa8\x88\xe3\x81"
-"\xaf\xe3\x80\x81\xe3\x80\x8c\xe3\x82\xb7\xe3\x83\xb3\xe3\x83\x97"
-"\xe3\x83\xab\xe3\x80\x8d\xe3\x81\xa7\xe3\x80\x8c\xe7\xbf\x92\xe5"
-"\xbe\x97\xe3\x81\x8c\xe5\xae\xb9\xe6\x98\x93\xe3\x80\x8d\xe3\x81"
-"\xa8\xe3\x81\x84\xe3\x81\x86\xe7\x9b\xae\xe6\xa8\x99\xe3\x81\xab"
-"\xe9\x87\x8d\xe7\x82\xb9\xe3\x81\x8c\xe7\xbd\xae\xe3\x81\x8b\xe3"
-"\x82\x8c\xe3\x81\xa6\xe3\x81\x84\xe3\x81\xbe\xe3\x81\x99\xe3\x80"
-"\x82\x0a\xe5\xa4\x9a\xe3\x81\x8f\xe3\x81\xae\xe3\x82\xb9\xe3\x82"
-"\xaf\xe3\x83\xaa\xe3\x83\x97\xe3\x83\x88\xe7\xb3\xbb\xe8\xa8\x80"
-"\xe8\xaa\x9e\xe3\x81\xa7\xe3\x81\xaf\xe3\x83\xa6\xe3\x83\xbc\xe3"
-"\x82\xb6\xe3\x81\xae\xe7\x9b\xae\xe5\x85\x88\xe3\x81\xae\xe5\x88"
-"\xa9\xe4\xbe\xbf\xe6\x80\xa7\xe3\x82\x92\xe5\x84\xaa\xe5\x85\x88"
-"\xe3\x81\x97\xe3\x81\xa6\xe8\x89\xb2\xe3\x80\x85\xe3\x81\xaa\xe6"
-"\xa9\x9f\xe8\x83\xbd\xe3\x82\x92\xe8\xa8\x80\xe8\xaa\x9e\xe8\xa6"
-"\x81\xe7\xb4\xa0\xe3\x81\xa8\xe3\x81\x97\xe3\x81\xa6\xe5\x8f\x96"
-"\xe3\x82\x8a\xe5\x85\xa5\xe3\x82\x8c\xe3\x82\x8b\xe5\xa0\xb4\xe5"
-"\x90\x88\xe3\x81\x8c\xe5\xa4\x9a\xe3\x81\x84\xe3\x81\xae\xe3\x81"
-"\xa7\xe3\x81\x99\xe3\x81\x8c\xe3\x80\x81\x50\x79\x74\x68\x6f\x6e"
-"\x20\xe3\x81\xa7\xe3\x81\xaf\xe3\x81\x9d\xe3\x81\x86\xe3\x81\x84"
-"\xe3\x81\xa3\xe3\x81\x9f\xe5\xb0\x8f\xe7\xb4\xb0\xe5\xb7\xa5\xe3"
-"\x81\x8c\xe8\xbf\xbd\xe5\x8a\xa0\xe3\x81\x95\xe3\x82\x8c\xe3\x82"
-"\x8b\xe3\x81\x93\xe3\x81\xa8\xe3\x81\xaf\xe3\x81\x82\xe3\x81\xbe"
-"\xe3\x82\x8a\xe3\x81\x82\xe3\x82\x8a\xe3\x81\xbe\xe3\x81\x9b\xe3"
-"\x82\x93\xe3\x80\x82\x0a\xe8\xa8\x80\xe8\xaa\x9e\xe8\x87\xaa\xe4"
-"\xbd\x93\xe3\x81\xae\xe6\xa9\x9f\xe8\x83\xbd\xe3\x81\xaf\xe6\x9c"
-"\x80\xe5\xb0\x8f\xe9\x99\x90\xe3\x81\xab\xe6\x8a\xbc\xe3\x81\x95"
-"\xe3\x81\x88\xe3\x80\x81\xe5\xbf\x85\xe8\xa6\x81\xe3\x81\xaa\xe6"
-"\xa9\x9f\xe8\x83\xbd\xe3\x81\xaf\xe6\x8b\xa1\xe5\xbc\xb5\xe3\x83"
-"\xa2\xe3\x82\xb8\xe3\x83\xa5\xe3\x83\xbc\xe3\x83\xab\xe3\x81\xa8"
-"\xe3\x81\x97\xe3\x81\xa6\xe8\xbf\xbd\xe5\x8a\xa0\xe3\x81\x99\xe3"
-"\x82\x8b\xe3\x80\x81\xe3\x81\xa8\xe3\x81\x84\xe3\x81\x86\xe3\x81"
-"\xae\xe3\x81\x8c\x20\x50\x79\x74\x68\x6f\x6e\x20\xe3\x81\xae\xe3"
-"\x83\x9d\xe3\x83\xaa\xe3\x82\xb7\xe3\x83\xbc\xe3\x81\xa7\xe3\x81"
-"\x99\xe3\x80\x82\x0a\x0a"),
-'euc_kr': (
-"\xa1\xdd\x20\xc6\xc4\xc0\xcc\xbd\xe3\x28\x50\x79\x74\x68\x6f\x6e"
-"\x29\xc0\xba\x20\xb9\xe8\xbf\xec\xb1\xe2\x20\xbd\xb1\xb0\xed\x2c"
-"\x20\xb0\xad\xb7\xc2\xc7\xd1\x20\xc7\xc1\xb7\xce\xb1\xd7\xb7\xa1"
-"\xb9\xd6\x20\xbe\xf0\xbe\xee\xc0\xd4\xb4\xcf\xb4\xd9\x2e\x20\xc6"
-"\xc4\xc0\xcc\xbd\xe3\xc0\xba\x0a\xc8\xbf\xc0\xb2\xc0\xfb\xc0\xce"
-"\x20\xb0\xed\xbc\xf6\xc1\xd8\x20\xb5\xa5\xc0\xcc\xc5\xcd\x20\xb1"
-"\xb8\xc1\xb6\xbf\xcd\x20\xb0\xa3\xb4\xdc\xc7\xcf\xc1\xf6\xb8\xb8"
-"\x20\xc8\xbf\xc0\xb2\xc0\xfb\xc0\xce\x20\xb0\xb4\xc3\xbc\xc1\xf6"
-"\xc7\xe2\xc7\xc1\xb7\xce\xb1\xd7\xb7\xa1\xb9\xd6\xc0\xbb\x0a\xc1"
-"\xf6\xbf\xf8\xc7\xd5\xb4\xcf\xb4\xd9\x2e\x20\xc6\xc4\xc0\xcc\xbd"
-"\xe3\xc0\xc7\x20\xbf\xec\xbe\xc6\x28\xe9\xd0\xe4\xba\x29\xc7\xd1"
-"\x20\xb9\xae\xb9\xfd\xb0\xfa\x20\xb5\xbf\xc0\xfb\x20\xc5\xb8\xc0"
-"\xcc\xc7\xce\x2c\x20\xb1\xd7\xb8\xae\xb0\xed\x20\xc0\xce\xc5\xcd"
-"\xc7\xc1\xb8\xae\xc6\xc3\x0a\xc8\xaf\xb0\xe6\xc0\xba\x20\xc6\xc4"
-"\xc0\xcc\xbd\xe3\xc0\xbb\x20\xbd\xba\xc5\xa9\xb8\xb3\xc6\xc3\xb0"
-"\xfa\x20\xbf\xa9\xb7\xaf\x20\xba\xd0\xbe\xdf\xbf\xa1\xbc\xad\xbf"
-"\xcd\x20\xb4\xeb\xba\xce\xba\xd0\xc0\xc7\x20\xc7\xc3\xb7\xa7\xc6"
-"\xfb\xbf\xa1\xbc\xad\xc0\xc7\x20\xba\xfc\xb8\xa5\x0a\xbe\xd6\xc7"
-"\xc3\xb8\xae\xc4\xc9\xc0\xcc\xbc\xc7\x20\xb0\xb3\xb9\xdf\xc0\xbb"
-"\x20\xc7\xd2\x20\xbc\xf6\x20\xc0\xd6\xb4\xc2\x20\xc0\xcc\xbb\xf3"
-"\xc0\xfb\xc0\xce\x20\xbe\xf0\xbe\xee\xb7\xce\x20\xb8\xb8\xb5\xe9"
-"\xbe\xee\xc1\xdd\xb4\xcf\xb4\xd9\x2e\x0a\x0a\xa1\xd9\xc3\xb9\xb0"
-"\xa1\xb3\xa1\x3a\x20\xb3\xaf\xbe\xc6\xb6\xf3\x20\xa4\xd4\xa4\xb6"
-"\xa4\xd0\xa4\xd4\xa4\xd4\xa4\xb6\xa4\xd0\xa4\xd4\xbe\xb1\x7e\x20"
-"\xa4\xd4\xa4\xa4\xa4\xd2\xa4\xb7\xc5\xad\x21\x20\xa4\xd4\xa4\xa8"
-"\xa4\xd1\xa4\xb7\xb1\xdd\xbe\xf8\xc0\xcc\x20\xc0\xfc\xa4\xd4\xa4"
-"\xbe\xa4\xc8\xa4\xb2\xb4\xcf\xb4\xd9\x2e\x20\xa4\xd4\xa4\xb2\xa4"
-"\xce\xa4\xaa\x2e\x20\xb1\xd7\xb7\xb1\xb0\xc5\x20\xa4\xd4\xa4\xb7"
-"\xa4\xd1\xa4\xb4\xb4\xd9\x2e\x0a",
-"\xe2\x97\x8e\x20\xed\x8c\x8c\xec\x9d\xb4\xec\x8d\xac\x28\x50\x79"
-"\x74\x68\x6f\x6e\x29\xec\x9d\x80\x20\xeb\xb0\xb0\xec\x9a\xb0\xea"
-"\xb8\xb0\x20\xec\x89\xbd\xea\xb3\xa0\x2c\x20\xea\xb0\x95\xeb\xa0"
-"\xa5\xed\x95\x9c\x20\xed\x94\x84\xeb\xa1\x9c\xea\xb7\xb8\xeb\x9e"
-"\x98\xeb\xb0\x8d\x20\xec\x96\xb8\xec\x96\xb4\xec\x9e\x85\xeb\x8b"
-"\x88\xeb\x8b\xa4\x2e\x20\xed\x8c\x8c\xec\x9d\xb4\xec\x8d\xac\xec"
-"\x9d\x80\x0a\xed\x9a\xa8\xec\x9c\xa8\xec\xa0\x81\xec\x9d\xb8\x20"
-"\xea\xb3\xa0\xec\x88\x98\xec\xa4\x80\x20\xeb\x8d\xb0\xec\x9d\xb4"
-"\xed\x84\xb0\x20\xea\xb5\xac\xec\xa1\xb0\xec\x99\x80\x20\xea\xb0"
-"\x84\xeb\x8b\xa8\xed\x95\x98\xec\xa7\x80\xeb\xa7\x8c\x20\xed\x9a"
-"\xa8\xec\x9c\xa8\xec\xa0\x81\xec\x9d\xb8\x20\xea\xb0\x9d\xec\xb2"
-"\xb4\xec\xa7\x80\xed\x96\xa5\xed\x94\x84\xeb\xa1\x9c\xea\xb7\xb8"
-"\xeb\x9e\x98\xeb\xb0\x8d\xec\x9d\x84\x0a\xec\xa7\x80\xec\x9b\x90"
-"\xed\x95\xa9\xeb\x8b\x88\xeb\x8b\xa4\x2e\x20\xed\x8c\x8c\xec\x9d"
-"\xb4\xec\x8d\xac\xec\x9d\x98\x20\xec\x9a\xb0\xec\x95\x84\x28\xe5"
-"\x84\xaa\xe9\x9b\x85\x29\xed\x95\x9c\x20\xeb\xac\xb8\xeb\xb2\x95"
-"\xea\xb3\xbc\x20\xeb\x8f\x99\xec\xa0\x81\x20\xed\x83\x80\xec\x9d"
-"\xb4\xed\x95\x91\x2c\x20\xea\xb7\xb8\xeb\xa6\xac\xea\xb3\xa0\x20"
-"\xec\x9d\xb8\xed\x84\xb0\xed\x94\x84\xeb\xa6\xac\xed\x8c\x85\x0a"
-"\xed\x99\x98\xea\xb2\xbd\xec\x9d\x80\x20\xed\x8c\x8c\xec\x9d\xb4"
-"\xec\x8d\xac\xec\x9d\x84\x20\xec\x8a\xa4\xed\x81\xac\xeb\xa6\xbd"
-"\xed\x8c\x85\xea\xb3\xbc\x20\xec\x97\xac\xeb\x9f\xac\x20\xeb\xb6"
-"\x84\xec\x95\xbc\xec\x97\x90\xec\x84\x9c\xec\x99\x80\x20\xeb\x8c"
-"\x80\xeb\xb6\x80\xeb\xb6\x84\xec\x9d\x98\x20\xed\x94\x8c\xeb\x9e"
-"\xab\xed\x8f\xbc\xec\x97\x90\xec\x84\x9c\xec\x9d\x98\x20\xeb\xb9"
-"\xa0\xeb\xa5\xb8\x0a\xec\x95\xa0\xed\x94\x8c\xeb\xa6\xac\xec\xbc"
-"\x80\xec\x9d\xb4\xec\x85\x98\x20\xea\xb0\x9c\xeb\xb0\x9c\xec\x9d"
-"\x84\x20\xed\x95\xa0\x20\xec\x88\x98\x20\xec\x9e\x88\xeb\x8a\x94"
-"\x20\xec\x9d\xb4\xec\x83\x81\xec\xa0\x81\xec\x9d\xb8\x20\xec\x96"
-"\xb8\xec\x96\xb4\xeb\xa1\x9c\x20\xeb\xa7\x8c\xeb\x93\xa4\xec\x96"
-"\xb4\xec\xa4\x8d\xeb\x8b\x88\xeb\x8b\xa4\x2e\x0a\x0a\xe2\x98\x86"
-"\xec\xb2\xab\xea\xb0\x80\xeb\x81\x9d\x3a\x20\xeb\x82\xa0\xec\x95"
-"\x84\xeb\x9d\xbc\x20\xec\x93\x94\xec\x93\x94\xec\x93\xa9\x7e\x20"
-"\xeb\x8b\x81\xed\x81\xbc\x21\x20\xeb\x9c\xbd\xea\xb8\x88\xec\x97"
-"\x86\xec\x9d\xb4\x20\xec\xa0\x84\xed\x99\xa5\xeb\x8b\x88\xeb\x8b"
-"\xa4\x2e\x20\xeb\xb7\x81\x2e\x20\xea\xb7\xb8\xeb\x9f\xb0\xea\xb1"
-"\xb0\x20\xec\x9d\x8e\xeb\x8b\xa4\x2e\x0a"),
-'gb18030': (
-"\x50\x79\x74\x68\x6f\x6e\xa3\xa8\xc5\xc9\xc9\xad\xa3\xa9\xd3\xef"
-"\xd1\xd4\xca\xc7\xd2\xbb\xd6\xd6\xb9\xa6\xc4\xdc\xc7\xbf\xb4\xf3"
-"\xb6\xf8\xcd\xea\xc9\xc6\xb5\xc4\xcd\xa8\xd3\xc3\xd0\xcd\xbc\xc6"
-"\xcb\xe3\xbb\xfa\xb3\xcc\xd0\xf2\xc9\xe8\xbc\xc6\xd3\xef\xd1\xd4"
-"\xa3\xac\x0a\xd2\xd1\xbe\xad\xbe\xdf\xd3\xd0\xca\xae\xb6\xe0\xc4"
-"\xea\xb5\xc4\xb7\xa2\xd5\xb9\xc0\xfa\xca\xb7\xa3\xac\xb3\xc9\xca"
-"\xec\xc7\xd2\xce\xc8\xb6\xa8\xa1\xa3\xd5\xe2\xd6\xd6\xd3\xef\xd1"
-"\xd4\xbe\xdf\xd3\xd0\xb7\xc7\xb3\xa3\xbc\xf2\xbd\xdd\xb6\xf8\xc7"
-"\xe5\xce\xfa\x0a\xb5\xc4\xd3\xef\xb7\xa8\xcc\xd8\xb5\xe3\xa3\xac"
-"\xca\xca\xba\xcf\xcd\xea\xb3\xc9\xb8\xf7\xd6\xd6\xb8\xdf\xb2\xe3"
-"\xc8\xce\xce\xf1\xa3\xac\xbc\xb8\xba\xf5\xbf\xc9\xd2\xd4\xd4\xda"
-"\xcb\xf9\xd3\xd0\xb5\xc4\xb2\xd9\xd7\xf7\xcf\xb5\xcd\xb3\xd6\xd0"
-"\x0a\xd4\xcb\xd0\xd0\xa1\xa3\xd5\xe2\xd6\xd6\xd3\xef\xd1\xd4\xbc"
-"\xf2\xb5\xa5\xb6\xf8\xc7\xbf\xb4\xf3\xa3\xac\xca\xca\xba\xcf\xb8"
-"\xf7\xd6\xd6\xc8\xcb\xca\xbf\xd1\xa7\xcf\xb0\xca\xb9\xd3\xc3\xa1"
-"\xa3\xc4\xbf\xc7\xb0\xa3\xac\xbb\xf9\xd3\xda\xd5\xe2\x0a\xd6\xd6"
-"\xd3\xef\xd1\xd4\xb5\xc4\xcf\xe0\xb9\xd8\xbc\xbc\xca\xf5\xd5\xfd"
-"\xd4\xda\xb7\xc9\xcb\xd9\xb5\xc4\xb7\xa2\xd5\xb9\xa3\xac\xd3\xc3"
-"\xbb\xa7\xca\xfd\xc1\xbf\xbc\xb1\xbe\xe7\xc0\xa9\xb4\xf3\xa3\xac"
-"\xcf\xe0\xb9\xd8\xb5\xc4\xd7\xca\xd4\xb4\xb7\xc7\xb3\xa3\xb6\xe0"
-"\xa1\xa3\x0a\xc8\xe7\xba\xce\xd4\xda\x20\x50\x79\x74\x68\x6f\x6e"
-"\x20\xd6\xd0\xca\xb9\xd3\xc3\xbc\xc8\xd3\xd0\xb5\xc4\x20\x43\x20"
-"\x6c\x69\x62\x72\x61\x72\x79\x3f\x0a\xa1\xa1\xd4\xda\xd9\x59\xd3"
-"\x8d\xbf\xc6\xbc\xbc\xbf\xec\xcb\xd9\xb0\x6c\xd5\xb9\xb5\xc4\xbd"
-"\xf1\xcc\xec\x2c\x20\xe9\x5f\xb0\x6c\xbc\xb0\x9c\x79\xd4\x87\xdc"
-"\x9b\xf3\x77\xb5\xc4\xcb\xd9\xb6\xc8\xca\xc7\xb2\xbb\xc8\xdd\xba"
-"\xf6\xd2\x95\xb5\xc4\x0a\xd5\x6e\xee\x7d\x2e\x20\x9e\xe9\xbc\xd3"
-"\xbf\xec\xe9\x5f\xb0\x6c\xbc\xb0\x9c\x79\xd4\x87\xb5\xc4\xcb\xd9"
-"\xb6\xc8\x2c\x20\xce\xd2\x82\x83\xb1\xe3\xb3\xa3\xcf\xa3\xcd\xfb"
-"\xc4\xdc\xc0\xfb\xd3\xc3\xd2\xbb\xd0\xa9\xd2\xd1\xe9\x5f\xb0\x6c"
-"\xba\xc3\xb5\xc4\x0a\x6c\x69\x62\x72\x61\x72\x79\x2c\x20\x81\x4b"
-"\xd3\xd0\xd2\xbb\x82\x80\x20\x66\x61\x73\x74\x20\x70\x72\x6f\x74"
-"\x6f\x74\x79\x70\x69\x6e\x67\x20\xb5\xc4\x20\x70\x72\x6f\x67\x72"
-"\x61\x6d\x6d\x69\x6e\x67\x20\x6c\x61\x6e\x67\x75\x61\x67\x65\x20"
-"\xbf\xc9\x0a\xb9\xa9\xca\xb9\xd3\xc3\x2e\x20\xc4\xbf\xc7\xb0\xd3"
-"\xd0\xd4\x53\xd4\x53\xb6\xe0\xb6\xe0\xb5\xc4\x20\x6c\x69\x62\x72"
-"\x61\x72\x79\x20\xca\xc7\xd2\xd4\x20\x43\x20\x8c\x91\xb3\xc9\x2c"
-"\x20\xb6\xf8\x20\x50\x79\x74\x68\x6f\x6e\x20\xca\xc7\xd2\xbb\x82"
-"\x80\x0a\x66\x61\x73\x74\x20\x70\x72\x6f\x74\x6f\x74\x79\x70\x69"
-"\x6e\x67\x20\xb5\xc4\x20\x70\x72\x6f\x67\x72\x61\x6d\x6d\x69\x6e"
-"\x67\x20\x6c\x61\x6e\x67\x75\x61\x67\x65\x2e\x20\xb9\xca\xce\xd2"
-"\x82\x83\xcf\xa3\xcd\xfb\xc4\xdc\x8c\xa2\xbc\xc8\xd3\xd0\xb5\xc4"
-"\x0a\x43\x20\x6c\x69\x62\x72\x61\x72\x79\x20\xc4\xc3\xb5\xbd\x20"
-"\x50\x79\x74\x68\x6f\x6e\x20\xb5\xc4\xad\x68\xbe\xb3\xd6\xd0\x9c"
-"\x79\xd4\x87\xbc\xb0\xd5\xfb\xba\xcf\x2e\x20\xc6\xe4\xd6\xd0\xd7"
-"\xee\xd6\xf7\xd2\xaa\xd2\xb2\xca\xc7\xce\xd2\x82\x83\xcb\xf9\x0a"
-"\xd2\xaa\xd3\x91\xd5\x93\xb5\xc4\x86\x96\xee\x7d\xbe\xcd\xca\xc7"
-"\x3a\x0a\x83\x35\xc7\x31\x83\x33\x9a\x33\x83\x32\xb1\x31\x83\x33"
-"\x95\x31\x20\x82\x37\xd1\x36\x83\x30\x8c\x34\x83\x36\x84\x33\x20"
-"\x82\x38\x89\x35\x82\x38\xfb\x36\x83\x33\x95\x35\x20\x83\x33\xd5"
-"\x31\x82\x39\x81\x35\x20\x83\x30\xfd\x39\x83\x33\x86\x30\x20\x83"
-"\x34\xdc\x33\x83\x35\xf6\x37\x83\x35\x97\x35\x20\x83\x35\xf9\x35"
-"\x83\x30\x91\x39\x82\x38\x83\x39\x82\x39\xfc\x33\x83\x30\xf0\x34"
-"\x20\x83\x32\xeb\x39\x83\x32\xeb\x35\x82\x39\x83\x39\x2e\x0a\x0a",
-"\x50\x79\x74\x68\x6f\x6e\xef\xbc\x88\xe6\xb4\xbe\xe6\xa3\xae\xef"
-"\xbc\x89\xe8\xaf\xad\xe8\xa8\x80\xe6\x98\xaf\xe4\xb8\x80\xe7\xa7"
-"\x8d\xe5\x8a\x9f\xe8\x83\xbd\xe5\xbc\xba\xe5\xa4\xa7\xe8\x80\x8c"
-"\xe5\xae\x8c\xe5\x96\x84\xe7\x9a\x84\xe9\x80\x9a\xe7\x94\xa8\xe5"
-"\x9e\x8b\xe8\xae\xa1\xe7\xae\x97\xe6\x9c\xba\xe7\xa8\x8b\xe5\xba"
-"\x8f\xe8\xae\xbe\xe8\xae\xa1\xe8\xaf\xad\xe8\xa8\x80\xef\xbc\x8c"
-"\x0a\xe5\xb7\xb2\xe7\xbb\x8f\xe5\x85\xb7\xe6\x9c\x89\xe5\x8d\x81"
-"\xe5\xa4\x9a\xe5\xb9\xb4\xe7\x9a\x84\xe5\x8f\x91\xe5\xb1\x95\xe5"
-"\x8e\x86\xe5\x8f\xb2\xef\xbc\x8c\xe6\x88\x90\xe7\x86\x9f\xe4\xb8"
-"\x94\xe7\xa8\xb3\xe5\xae\x9a\xe3\x80\x82\xe8\xbf\x99\xe7\xa7\x8d"
-"\xe8\xaf\xad\xe8\xa8\x80\xe5\x85\xb7\xe6\x9c\x89\xe9\x9d\x9e\xe5"
-"\xb8\xb8\xe7\xae\x80\xe6\x8d\xb7\xe8\x80\x8c\xe6\xb8\x85\xe6\x99"
-"\xb0\x0a\xe7\x9a\x84\xe8\xaf\xad\xe6\xb3\x95\xe7\x89\xb9\xe7\x82"
-"\xb9\xef\xbc\x8c\xe9\x80\x82\xe5\x90\x88\xe5\xae\x8c\xe6\x88\x90"
-"\xe5\x90\x84\xe7\xa7\x8d\xe9\xab\x98\xe5\xb1\x82\xe4\xbb\xbb\xe5"
-"\x8a\xa1\xef\xbc\x8c\xe5\x87\xa0\xe4\xb9\x8e\xe5\x8f\xaf\xe4\xbb"
-"\xa5\xe5\x9c\xa8\xe6\x89\x80\xe6\x9c\x89\xe7\x9a\x84\xe6\x93\x8d"
-"\xe4\xbd\x9c\xe7\xb3\xbb\xe7\xbb\x9f\xe4\xb8\xad\x0a\xe8\xbf\x90"
-"\xe8\xa1\x8c\xe3\x80\x82\xe8\xbf\x99\xe7\xa7\x8d\xe8\xaf\xad\xe8"
-"\xa8\x80\xe7\xae\x80\xe5\x8d\x95\xe8\x80\x8c\xe5\xbc\xba\xe5\xa4"
-"\xa7\xef\xbc\x8c\xe9\x80\x82\xe5\x90\x88\xe5\x90\x84\xe7\xa7\x8d"
-"\xe4\xba\xba\xe5\xa3\xab\xe5\xad\xa6\xe4\xb9\xa0\xe4\xbd\xbf\xe7"
-"\x94\xa8\xe3\x80\x82\xe7\x9b\xae\xe5\x89\x8d\xef\xbc\x8c\xe5\x9f"
-"\xba\xe4\xba\x8e\xe8\xbf\x99\x0a\xe7\xa7\x8d\xe8\xaf\xad\xe8\xa8"
-"\x80\xe7\x9a\x84\xe7\x9b\xb8\xe5\x85\xb3\xe6\x8a\x80\xe6\x9c\xaf"
-"\xe6\xad\xa3\xe5\x9c\xa8\xe9\xa3\x9e\xe9\x80\x9f\xe7\x9a\x84\xe5"
-"\x8f\x91\xe5\xb1\x95\xef\xbc\x8c\xe7\x94\xa8\xe6\x88\xb7\xe6\x95"
-"\xb0\xe9\x87\x8f\xe6\x80\xa5\xe5\x89\xa7\xe6\x89\xa9\xe5\xa4\xa7"
-"\xef\xbc\x8c\xe7\x9b\xb8\xe5\x85\xb3\xe7\x9a\x84\xe8\xb5\x84\xe6"
-"\xba\x90\xe9\x9d\x9e\xe5\xb8\xb8\xe5\xa4\x9a\xe3\x80\x82\x0a\xe5"
-"\xa6\x82\xe4\xbd\x95\xe5\x9c\xa8\x20\x50\x79\x74\x68\x6f\x6e\x20"
-"\xe4\xb8\xad\xe4\xbd\xbf\xe7\x94\xa8\xe6\x97\xa2\xe6\x9c\x89\xe7"
-"\x9a\x84\x20\x43\x20\x6c\x69\x62\x72\x61\x72\x79\x3f\x0a\xe3\x80"
-"\x80\xe5\x9c\xa8\xe8\xb3\x87\xe8\xa8\x8a\xe7\xa7\x91\xe6\x8a\x80"
-"\xe5\xbf\xab\xe9\x80\x9f\xe7\x99\xbc\xe5\xb1\x95\xe7\x9a\x84\xe4"
-"\xbb\x8a\xe5\xa4\xa9\x2c\x20\xe9\x96\x8b\xe7\x99\xbc\xe5\x8f\x8a"
-"\xe6\xb8\xac\xe8\xa9\xa6\xe8\xbb\x9f\xe9\xab\x94\xe7\x9a\x84\xe9"
-"\x80\x9f\xe5\xba\xa6\xe6\x98\xaf\xe4\xb8\x8d\xe5\xae\xb9\xe5\xbf"
-"\xbd\xe8\xa6\x96\xe7\x9a\x84\x0a\xe8\xaa\xb2\xe9\xa1\x8c\x2e\x20"
-"\xe7\x82\xba\xe5\x8a\xa0\xe5\xbf\xab\xe9\x96\x8b\xe7\x99\xbc\xe5"
-"\x8f\x8a\xe6\xb8\xac\xe8\xa9\xa6\xe7\x9a\x84\xe9\x80\x9f\xe5\xba"
-"\xa6\x2c\x20\xe6\x88\x91\xe5\x80\x91\xe4\xbe\xbf\xe5\xb8\xb8\xe5"
-"\xb8\x8c\xe6\x9c\x9b\xe8\x83\xbd\xe5\x88\xa9\xe7\x94\xa8\xe4\xb8"
-"\x80\xe4\xba\x9b\xe5\xb7\xb2\xe9\x96\x8b\xe7\x99\xbc\xe5\xa5\xbd"
-"\xe7\x9a\x84\x0a\x6c\x69\x62\x72\x61\x72\x79\x2c\x20\xe4\xb8\xa6"
-"\xe6\x9c\x89\xe4\xb8\x80\xe5\x80\x8b\x20\x66\x61\x73\x74\x20\x70"
-"\x72\x6f\x74\x6f\x74\x79\x70\x69\x6e\x67\x20\xe7\x9a\x84\x20\x70"
-"\x72\x6f\x67\x72\x61\x6d\x6d\x69\x6e\x67\x20\x6c\x61\x6e\x67\x75"
-"\x61\x67\x65\x20\xe5\x8f\xaf\x0a\xe4\xbe\x9b\xe4\xbd\xbf\xe7\x94"
-"\xa8\x2e\x20\xe7\x9b\xae\xe5\x89\x8d\xe6\x9c\x89\xe8\xa8\xb1\xe8"
-"\xa8\xb1\xe5\xa4\x9a\xe5\xa4\x9a\xe7\x9a\x84\x20\x6c\x69\x62\x72"
-"\x61\x72\x79\x20\xe6\x98\xaf\xe4\xbb\xa5\x20\x43\x20\xe5\xaf\xab"
-"\xe6\x88\x90\x2c\x20\xe8\x80\x8c\x20\x50\x79\x74\x68\x6f\x6e\x20"
-"\xe6\x98\xaf\xe4\xb8\x80\xe5\x80\x8b\x0a\x66\x61\x73\x74\x20\x70"
-"\x72\x6f\x74\x6f\x74\x79\x70\x69\x6e\x67\x20\xe7\x9a\x84\x20\x70"
-"\x72\x6f\x67\x72\x61\x6d\x6d\x69\x6e\x67\x20\x6c\x61\x6e\x67\x75"
-"\x61\x67\x65\x2e\x20\xe6\x95\x85\xe6\x88\x91\xe5\x80\x91\xe5\xb8"
-"\x8c\xe6\x9c\x9b\xe8\x83\xbd\xe5\xb0\x87\xe6\x97\xa2\xe6\x9c\x89"
-"\xe7\x9a\x84\x0a\x43\x20\x6c\x69\x62\x72\x61\x72\x79\x20\xe6\x8b"
-"\xbf\xe5\x88\xb0\x20\x50\x79\x74\x68\x6f\x6e\x20\xe7\x9a\x84\xe7"
-"\x92\xb0\xe5\xa2\x83\xe4\xb8\xad\xe6\xb8\xac\xe8\xa9\xa6\xe5\x8f"
-"\x8a\xe6\x95\xb4\xe5\x90\x88\x2e\x20\xe5\x85\xb6\xe4\xb8\xad\xe6"
-"\x9c\x80\xe4\xb8\xbb\xe8\xa6\x81\xe4\xb9\x9f\xe6\x98\xaf\xe6\x88"
-"\x91\xe5\x80\x91\xe6\x89\x80\x0a\xe8\xa6\x81\xe8\xa8\x8e\xe8\xab"
-"\x96\xe7\x9a\x84\xe5\x95\x8f\xe9\xa1\x8c\xe5\xb0\xb1\xe6\x98\xaf"
-"\x3a\x0a\xed\x8c\x8c\xec\x9d\xb4\xec\x8d\xac\xec\x9d\x80\x20\xea"
-"\xb0\x95\xeb\xa0\xa5\xed\x95\x9c\x20\xea\xb8\xb0\xeb\x8a\xa5\xec"
-"\x9d\x84\x20\xec\xa7\x80\xeb\x8b\x8c\x20\xeb\xb2\x94\xec\x9a\xa9"
-"\x20\xec\xbb\xb4\xed\x93\xa8\xed\x84\xb0\x20\xed\x94\x84\xeb\xa1"
-"\x9c\xea\xb7\xb8\xeb\x9e\x98\xeb\xb0\x8d\x20\xec\x96\xb8\xec\x96"
-"\xb4\xeb\x8b\xa4\x2e\x0a\x0a"),
-'gb2312': (
-"\x50\x79\x74\x68\x6f\x6e\xa3\xa8\xc5\xc9\xc9\xad\xa3\xa9\xd3\xef"
-"\xd1\xd4\xca\xc7\xd2\xbb\xd6\xd6\xb9\xa6\xc4\xdc\xc7\xbf\xb4\xf3"
-"\xb6\xf8\xcd\xea\xc9\xc6\xb5\xc4\xcd\xa8\xd3\xc3\xd0\xcd\xbc\xc6"
-"\xcb\xe3\xbb\xfa\xb3\xcc\xd0\xf2\xc9\xe8\xbc\xc6\xd3\xef\xd1\xd4"
-"\xa3\xac\x0a\xd2\xd1\xbe\xad\xbe\xdf\xd3\xd0\xca\xae\xb6\xe0\xc4"
-"\xea\xb5\xc4\xb7\xa2\xd5\xb9\xc0\xfa\xca\xb7\xa3\xac\xb3\xc9\xca"
-"\xec\xc7\xd2\xce\xc8\xb6\xa8\xa1\xa3\xd5\xe2\xd6\xd6\xd3\xef\xd1"
-"\xd4\xbe\xdf\xd3\xd0\xb7\xc7\xb3\xa3\xbc\xf2\xbd\xdd\xb6\xf8\xc7"
-"\xe5\xce\xfa\x0a\xb5\xc4\xd3\xef\xb7\xa8\xcc\xd8\xb5\xe3\xa3\xac"
-"\xca\xca\xba\xcf\xcd\xea\xb3\xc9\xb8\xf7\xd6\xd6\xb8\xdf\xb2\xe3"
-"\xc8\xce\xce\xf1\xa3\xac\xbc\xb8\xba\xf5\xbf\xc9\xd2\xd4\xd4\xda"
-"\xcb\xf9\xd3\xd0\xb5\xc4\xb2\xd9\xd7\xf7\xcf\xb5\xcd\xb3\xd6\xd0"
-"\x0a\xd4\xcb\xd0\xd0\xa1\xa3\xd5\xe2\xd6\xd6\xd3\xef\xd1\xd4\xbc"
-"\xf2\xb5\xa5\xb6\xf8\xc7\xbf\xb4\xf3\xa3\xac\xca\xca\xba\xcf\xb8"
-"\xf7\xd6\xd6\xc8\xcb\xca\xbf\xd1\xa7\xcf\xb0\xca\xb9\xd3\xc3\xa1"
-"\xa3\xc4\xbf\xc7\xb0\xa3\xac\xbb\xf9\xd3\xda\xd5\xe2\x0a\xd6\xd6"
-"\xd3\xef\xd1\xd4\xb5\xc4\xcf\xe0\xb9\xd8\xbc\xbc\xca\xf5\xd5\xfd"
-"\xd4\xda\xb7\xc9\xcb\xd9\xb5\xc4\xb7\xa2\xd5\xb9\xa3\xac\xd3\xc3"
-"\xbb\xa7\xca\xfd\xc1\xbf\xbc\xb1\xbe\xe7\xc0\xa9\xb4\xf3\xa3\xac"
-"\xcf\xe0\xb9\xd8\xb5\xc4\xd7\xca\xd4\xb4\xb7\xc7\xb3\xa3\xb6\xe0"
-"\xa1\xa3\x0a\x0a",
-"\x50\x79\x74\x68\x6f\x6e\xef\xbc\x88\xe6\xb4\xbe\xe6\xa3\xae\xef"
-"\xbc\x89\xe8\xaf\xad\xe8\xa8\x80\xe6\x98\xaf\xe4\xb8\x80\xe7\xa7"
-"\x8d\xe5\x8a\x9f\xe8\x83\xbd\xe5\xbc\xba\xe5\xa4\xa7\xe8\x80\x8c"
-"\xe5\xae\x8c\xe5\x96\x84\xe7\x9a\x84\xe9\x80\x9a\xe7\x94\xa8\xe5"
-"\x9e\x8b\xe8\xae\xa1\xe7\xae\x97\xe6\x9c\xba\xe7\xa8\x8b\xe5\xba"
-"\x8f\xe8\xae\xbe\xe8\xae\xa1\xe8\xaf\xad\xe8\xa8\x80\xef\xbc\x8c"
-"\x0a\xe5\xb7\xb2\xe7\xbb\x8f\xe5\x85\xb7\xe6\x9c\x89\xe5\x8d\x81"
-"\xe5\xa4\x9a\xe5\xb9\xb4\xe7\x9a\x84\xe5\x8f\x91\xe5\xb1\x95\xe5"
-"\x8e\x86\xe5\x8f\xb2\xef\xbc\x8c\xe6\x88\x90\xe7\x86\x9f\xe4\xb8"
-"\x94\xe7\xa8\xb3\xe5\xae\x9a\xe3\x80\x82\xe8\xbf\x99\xe7\xa7\x8d"
-"\xe8\xaf\xad\xe8\xa8\x80\xe5\x85\xb7\xe6\x9c\x89\xe9\x9d\x9e\xe5"
-"\xb8\xb8\xe7\xae\x80\xe6\x8d\xb7\xe8\x80\x8c\xe6\xb8\x85\xe6\x99"
-"\xb0\x0a\xe7\x9a\x84\xe8\xaf\xad\xe6\xb3\x95\xe7\x89\xb9\xe7\x82"
-"\xb9\xef\xbc\x8c\xe9\x80\x82\xe5\x90\x88\xe5\xae\x8c\xe6\x88\x90"
-"\xe5\x90\x84\xe7\xa7\x8d\xe9\xab\x98\xe5\xb1\x82\xe4\xbb\xbb\xe5"
-"\x8a\xa1\xef\xbc\x8c\xe5\x87\xa0\xe4\xb9\x8e\xe5\x8f\xaf\xe4\xbb"
-"\xa5\xe5\x9c\xa8\xe6\x89\x80\xe6\x9c\x89\xe7\x9a\x84\xe6\x93\x8d"
-"\xe4\xbd\x9c\xe7\xb3\xbb\xe7\xbb\x9f\xe4\xb8\xad\x0a\xe8\xbf\x90"
-"\xe8\xa1\x8c\xe3\x80\x82\xe8\xbf\x99\xe7\xa7\x8d\xe8\xaf\xad\xe8"
-"\xa8\x80\xe7\xae\x80\xe5\x8d\x95\xe8\x80\x8c\xe5\xbc\xba\xe5\xa4"
-"\xa7\xef\xbc\x8c\xe9\x80\x82\xe5\x90\x88\xe5\x90\x84\xe7\xa7\x8d"
-"\xe4\xba\xba\xe5\xa3\xab\xe5\xad\xa6\xe4\xb9\xa0\xe4\xbd\xbf\xe7"
-"\x94\xa8\xe3\x80\x82\xe7\x9b\xae\xe5\x89\x8d\xef\xbc\x8c\xe5\x9f"
-"\xba\xe4\xba\x8e\xe8\xbf\x99\x0a\xe7\xa7\x8d\xe8\xaf\xad\xe8\xa8"
-"\x80\xe7\x9a\x84\xe7\x9b\xb8\xe5\x85\xb3\xe6\x8a\x80\xe6\x9c\xaf"
-"\xe6\xad\xa3\xe5\x9c\xa8\xe9\xa3\x9e\xe9\x80\x9f\xe7\x9a\x84\xe5"
-"\x8f\x91\xe5\xb1\x95\xef\xbc\x8c\xe7\x94\xa8\xe6\x88\xb7\xe6\x95"
-"\xb0\xe9\x87\x8f\xe6\x80\xa5\xe5\x89\xa7\xe6\x89\xa9\xe5\xa4\xa7"
-"\xef\xbc\x8c\xe7\x9b\xb8\xe5\x85\xb3\xe7\x9a\x84\xe8\xb5\x84\xe6"
-"\xba\x90\xe9\x9d\x9e\xe5\xb8\xb8\xe5\xa4\x9a\xe3\x80\x82\x0a\x0a"),
-'gbk': (
-"\x50\x79\x74\x68\x6f\x6e\xa3\xa8\xc5\xc9\xc9\xad\xa3\xa9\xd3\xef"
-"\xd1\xd4\xca\xc7\xd2\xbb\xd6\xd6\xb9\xa6\xc4\xdc\xc7\xbf\xb4\xf3"
-"\xb6\xf8\xcd\xea\xc9\xc6\xb5\xc4\xcd\xa8\xd3\xc3\xd0\xcd\xbc\xc6"
-"\xcb\xe3\xbb\xfa\xb3\xcc\xd0\xf2\xc9\xe8\xbc\xc6\xd3\xef\xd1\xd4"
-"\xa3\xac\x0a\xd2\xd1\xbe\xad\xbe\xdf\xd3\xd0\xca\xae\xb6\xe0\xc4"
-"\xea\xb5\xc4\xb7\xa2\xd5\xb9\xc0\xfa\xca\xb7\xa3\xac\xb3\xc9\xca"
-"\xec\xc7\xd2\xce\xc8\xb6\xa8\xa1\xa3\xd5\xe2\xd6\xd6\xd3\xef\xd1"
-"\xd4\xbe\xdf\xd3\xd0\xb7\xc7\xb3\xa3\xbc\xf2\xbd\xdd\xb6\xf8\xc7"
-"\xe5\xce\xfa\x0a\xb5\xc4\xd3\xef\xb7\xa8\xcc\xd8\xb5\xe3\xa3\xac"
-"\xca\xca\xba\xcf\xcd\xea\xb3\xc9\xb8\xf7\xd6\xd6\xb8\xdf\xb2\xe3"
-"\xc8\xce\xce\xf1\xa3\xac\xbc\xb8\xba\xf5\xbf\xc9\xd2\xd4\xd4\xda"
-"\xcb\xf9\xd3\xd0\xb5\xc4\xb2\xd9\xd7\xf7\xcf\xb5\xcd\xb3\xd6\xd0"
-"\x0a\xd4\xcb\xd0\xd0\xa1\xa3\xd5\xe2\xd6\xd6\xd3\xef\xd1\xd4\xbc"
-"\xf2\xb5\xa5\xb6\xf8\xc7\xbf\xb4\xf3\xa3\xac\xca\xca\xba\xcf\xb8"
-"\xf7\xd6\xd6\xc8\xcb\xca\xbf\xd1\xa7\xcf\xb0\xca\xb9\xd3\xc3\xa1"
-"\xa3\xc4\xbf\xc7\xb0\xa3\xac\xbb\xf9\xd3\xda\xd5\xe2\x0a\xd6\xd6"
-"\xd3\xef\xd1\xd4\xb5\xc4\xcf\xe0\xb9\xd8\xbc\xbc\xca\xf5\xd5\xfd"
-"\xd4\xda\xb7\xc9\xcb\xd9\xb5\xc4\xb7\xa2\xd5\xb9\xa3\xac\xd3\xc3"
-"\xbb\xa7\xca\xfd\xc1\xbf\xbc\xb1\xbe\xe7\xc0\xa9\xb4\xf3\xa3\xac"
-"\xcf\xe0\xb9\xd8\xb5\xc4\xd7\xca\xd4\xb4\xb7\xc7\xb3\xa3\xb6\xe0"
-"\xa1\xa3\x0a\xc8\xe7\xba\xce\xd4\xda\x20\x50\x79\x74\x68\x6f\x6e"
-"\x20\xd6\xd0\xca\xb9\xd3\xc3\xbc\xc8\xd3\xd0\xb5\xc4\x20\x43\x20"
-"\x6c\x69\x62\x72\x61\x72\x79\x3f\x0a\xa1\xa1\xd4\xda\xd9\x59\xd3"
-"\x8d\xbf\xc6\xbc\xbc\xbf\xec\xcb\xd9\xb0\x6c\xd5\xb9\xb5\xc4\xbd"
-"\xf1\xcc\xec\x2c\x20\xe9\x5f\xb0\x6c\xbc\xb0\x9c\x79\xd4\x87\xdc"
-"\x9b\xf3\x77\xb5\xc4\xcb\xd9\xb6\xc8\xca\xc7\xb2\xbb\xc8\xdd\xba"
-"\xf6\xd2\x95\xb5\xc4\x0a\xd5\x6e\xee\x7d\x2e\x20\x9e\xe9\xbc\xd3"
-"\xbf\xec\xe9\x5f\xb0\x6c\xbc\xb0\x9c\x79\xd4\x87\xb5\xc4\xcb\xd9"
-"\xb6\xc8\x2c\x20\xce\xd2\x82\x83\xb1\xe3\xb3\xa3\xcf\xa3\xcd\xfb"
-"\xc4\xdc\xc0\xfb\xd3\xc3\xd2\xbb\xd0\xa9\xd2\xd1\xe9\x5f\xb0\x6c"
-"\xba\xc3\xb5\xc4\x0a\x6c\x69\x62\x72\x61\x72\x79\x2c\x20\x81\x4b"
-"\xd3\xd0\xd2\xbb\x82\x80\x20\x66\x61\x73\x74\x20\x70\x72\x6f\x74"
-"\x6f\x74\x79\x70\x69\x6e\x67\x20\xb5\xc4\x20\x70\x72\x6f\x67\x72"
-"\x61\x6d\x6d\x69\x6e\x67\x20\x6c\x61\x6e\x67\x75\x61\x67\x65\x20"
-"\xbf\xc9\x0a\xb9\xa9\xca\xb9\xd3\xc3\x2e\x20\xc4\xbf\xc7\xb0\xd3"
-"\xd0\xd4\x53\xd4\x53\xb6\xe0\xb6\xe0\xb5\xc4\x20\x6c\x69\x62\x72"
-"\x61\x72\x79\x20\xca\xc7\xd2\xd4\x20\x43\x20\x8c\x91\xb3\xc9\x2c"
-"\x20\xb6\xf8\x20\x50\x79\x74\x68\x6f\x6e\x20\xca\xc7\xd2\xbb\x82"
-"\x80\x0a\x66\x61\x73\x74\x20\x70\x72\x6f\x74\x6f\x74\x79\x70\x69"
-"\x6e\x67\x20\xb5\xc4\x20\x70\x72\x6f\x67\x72\x61\x6d\x6d\x69\x6e"
-"\x67\x20\x6c\x61\x6e\x67\x75\x61\x67\x65\x2e\x20\xb9\xca\xce\xd2"
-"\x82\x83\xcf\xa3\xcd\xfb\xc4\xdc\x8c\xa2\xbc\xc8\xd3\xd0\xb5\xc4"
-"\x0a\x43\x20\x6c\x69\x62\x72\x61\x72\x79\x20\xc4\xc3\xb5\xbd\x20"
-"\x50\x79\x74\x68\x6f\x6e\x20\xb5\xc4\xad\x68\xbe\xb3\xd6\xd0\x9c"
-"\x79\xd4\x87\xbc\xb0\xd5\xfb\xba\xcf\x2e\x20\xc6\xe4\xd6\xd0\xd7"
-"\xee\xd6\xf7\xd2\xaa\xd2\xb2\xca\xc7\xce\xd2\x82\x83\xcb\xf9\x0a"
-"\xd2\xaa\xd3\x91\xd5\x93\xb5\xc4\x86\x96\xee\x7d\xbe\xcd\xca\xc7"
-"\x3a\x0a\x0a",
-"\x50\x79\x74\x68\x6f\x6e\xef\xbc\x88\xe6\xb4\xbe\xe6\xa3\xae\xef"
-"\xbc\x89\xe8\xaf\xad\xe8\xa8\x80\xe6\x98\xaf\xe4\xb8\x80\xe7\xa7"
-"\x8d\xe5\x8a\x9f\xe8\x83\xbd\xe5\xbc\xba\xe5\xa4\xa7\xe8\x80\x8c"
-"\xe5\xae\x8c\xe5\x96\x84\xe7\x9a\x84\xe9\x80\x9a\xe7\x94\xa8\xe5"
-"\x9e\x8b\xe8\xae\xa1\xe7\xae\x97\xe6\x9c\xba\xe7\xa8\x8b\xe5\xba"
-"\x8f\xe8\xae\xbe\xe8\xae\xa1\xe8\xaf\xad\xe8\xa8\x80\xef\xbc\x8c"
-"\x0a\xe5\xb7\xb2\xe7\xbb\x8f\xe5\x85\xb7\xe6\x9c\x89\xe5\x8d\x81"
-"\xe5\xa4\x9a\xe5\xb9\xb4\xe7\x9a\x84\xe5\x8f\x91\xe5\xb1\x95\xe5"
-"\x8e\x86\xe5\x8f\xb2\xef\xbc\x8c\xe6\x88\x90\xe7\x86\x9f\xe4\xb8"
-"\x94\xe7\xa8\xb3\xe5\xae\x9a\xe3\x80\x82\xe8\xbf\x99\xe7\xa7\x8d"
-"\xe8\xaf\xad\xe8\xa8\x80\xe5\x85\xb7\xe6\x9c\x89\xe9\x9d\x9e\xe5"
-"\xb8\xb8\xe7\xae\x80\xe6\x8d\xb7\xe8\x80\x8c\xe6\xb8\x85\xe6\x99"
-"\xb0\x0a\xe7\x9a\x84\xe8\xaf\xad\xe6\xb3\x95\xe7\x89\xb9\xe7\x82"
-"\xb9\xef\xbc\x8c\xe9\x80\x82\xe5\x90\x88\xe5\xae\x8c\xe6\x88\x90"
-"\xe5\x90\x84\xe7\xa7\x8d\xe9\xab\x98\xe5\xb1\x82\xe4\xbb\xbb\xe5"
-"\x8a\xa1\xef\xbc\x8c\xe5\x87\xa0\xe4\xb9\x8e\xe5\x8f\xaf\xe4\xbb"
-"\xa5\xe5\x9c\xa8\xe6\x89\x80\xe6\x9c\x89\xe7\x9a\x84\xe6\x93\x8d"
-"\xe4\xbd\x9c\xe7\xb3\xbb\xe7\xbb\x9f\xe4\xb8\xad\x0a\xe8\xbf\x90"
-"\xe8\xa1\x8c\xe3\x80\x82\xe8\xbf\x99\xe7\xa7\x8d\xe8\xaf\xad\xe8"
-"\xa8\x80\xe7\xae\x80\xe5\x8d\x95\xe8\x80\x8c\xe5\xbc\xba\xe5\xa4"
-"\xa7\xef\xbc\x8c\xe9\x80\x82\xe5\x90\x88\xe5\x90\x84\xe7\xa7\x8d"
-"\xe4\xba\xba\xe5\xa3\xab\xe5\xad\xa6\xe4\xb9\xa0\xe4\xbd\xbf\xe7"
-"\x94\xa8\xe3\x80\x82\xe7\x9b\xae\xe5\x89\x8d\xef\xbc\x8c\xe5\x9f"
-"\xba\xe4\xba\x8e\xe8\xbf\x99\x0a\xe7\xa7\x8d\xe8\xaf\xad\xe8\xa8"
-"\x80\xe7\x9a\x84\xe7\x9b\xb8\xe5\x85\xb3\xe6\x8a\x80\xe6\x9c\xaf"
-"\xe6\xad\xa3\xe5\x9c\xa8\xe9\xa3\x9e\xe9\x80\x9f\xe7\x9a\x84\xe5"
-"\x8f\x91\xe5\xb1\x95\xef\xbc\x8c\xe7\x94\xa8\xe6\x88\xb7\xe6\x95"
-"\xb0\xe9\x87\x8f\xe6\x80\xa5\xe5\x89\xa7\xe6\x89\xa9\xe5\xa4\xa7"
-"\xef\xbc\x8c\xe7\x9b\xb8\xe5\x85\xb3\xe7\x9a\x84\xe8\xb5\x84\xe6"
-"\xba\x90\xe9\x9d\x9e\xe5\xb8\xb8\xe5\xa4\x9a\xe3\x80\x82\x0a\xe5"
-"\xa6\x82\xe4\xbd\x95\xe5\x9c\xa8\x20\x50\x79\x74\x68\x6f\x6e\x20"
-"\xe4\xb8\xad\xe4\xbd\xbf\xe7\x94\xa8\xe6\x97\xa2\xe6\x9c\x89\xe7"
-"\x9a\x84\x20\x43\x20\x6c\x69\x62\x72\x61\x72\x79\x3f\x0a\xe3\x80"
-"\x80\xe5\x9c\xa8\xe8\xb3\x87\xe8\xa8\x8a\xe7\xa7\x91\xe6\x8a\x80"
-"\xe5\xbf\xab\xe9\x80\x9f\xe7\x99\xbc\xe5\xb1\x95\xe7\x9a\x84\xe4"
-"\xbb\x8a\xe5\xa4\xa9\x2c\x20\xe9\x96\x8b\xe7\x99\xbc\xe5\x8f\x8a"
-"\xe6\xb8\xac\xe8\xa9\xa6\xe8\xbb\x9f\xe9\xab\x94\xe7\x9a\x84\xe9"
-"\x80\x9f\xe5\xba\xa6\xe6\x98\xaf\xe4\xb8\x8d\xe5\xae\xb9\xe5\xbf"
-"\xbd\xe8\xa6\x96\xe7\x9a\x84\x0a\xe8\xaa\xb2\xe9\xa1\x8c\x2e\x20"
-"\xe7\x82\xba\xe5\x8a\xa0\xe5\xbf\xab\xe9\x96\x8b\xe7\x99\xbc\xe5"
-"\x8f\x8a\xe6\xb8\xac\xe8\xa9\xa6\xe7\x9a\x84\xe9\x80\x9f\xe5\xba"
-"\xa6\x2c\x20\xe6\x88\x91\xe5\x80\x91\xe4\xbe\xbf\xe5\xb8\xb8\xe5"
-"\xb8\x8c\xe6\x9c\x9b\xe8\x83\xbd\xe5\x88\xa9\xe7\x94\xa8\xe4\xb8"
-"\x80\xe4\xba\x9b\xe5\xb7\xb2\xe9\x96\x8b\xe7\x99\xbc\xe5\xa5\xbd"
-"\xe7\x9a\x84\x0a\x6c\x69\x62\x72\x61\x72\x79\x2c\x20\xe4\xb8\xa6"
-"\xe6\x9c\x89\xe4\xb8\x80\xe5\x80\x8b\x20\x66\x61\x73\x74\x20\x70"
-"\x72\x6f\x74\x6f\x74\x79\x70\x69\x6e\x67\x20\xe7\x9a\x84\x20\x70"
-"\x72\x6f\x67\x72\x61\x6d\x6d\x69\x6e\x67\x20\x6c\x61\x6e\x67\x75"
-"\x61\x67\x65\x20\xe5\x8f\xaf\x0a\xe4\xbe\x9b\xe4\xbd\xbf\xe7\x94"
-"\xa8\x2e\x20\xe7\x9b\xae\xe5\x89\x8d\xe6\x9c\x89\xe8\xa8\xb1\xe8"
-"\xa8\xb1\xe5\xa4\x9a\xe5\xa4\x9a\xe7\x9a\x84\x20\x6c\x69\x62\x72"
-"\x61\x72\x79\x20\xe6\x98\xaf\xe4\xbb\xa5\x20\x43\x20\xe5\xaf\xab"
-"\xe6\x88\x90\x2c\x20\xe8\x80\x8c\x20\x50\x79\x74\x68\x6f\x6e\x20"
-"\xe6\x98\xaf\xe4\xb8\x80\xe5\x80\x8b\x0a\x66\x61\x73\x74\x20\x70"
-"\x72\x6f\x74\x6f\x74\x79\x70\x69\x6e\x67\x20\xe7\x9a\x84\x20\x70"
-"\x72\x6f\x67\x72\x61\x6d\x6d\x69\x6e\x67\x20\x6c\x61\x6e\x67\x75"
-"\x61\x67\x65\x2e\x20\xe6\x95\x85\xe6\x88\x91\xe5\x80\x91\xe5\xb8"
-"\x8c\xe6\x9c\x9b\xe8\x83\xbd\xe5\xb0\x87\xe6\x97\xa2\xe6\x9c\x89"
-"\xe7\x9a\x84\x0a\x43\x20\x6c\x69\x62\x72\x61\x72\x79\x20\xe6\x8b"
-"\xbf\xe5\x88\xb0\x20\x50\x79\x74\x68\x6f\x6e\x20\xe7\x9a\x84\xe7"
-"\x92\xb0\xe5\xa2\x83\xe4\xb8\xad\xe6\xb8\xac\xe8\xa9\xa6\xe5\x8f"
-"\x8a\xe6\x95\xb4\xe5\x90\x88\x2e\x20\xe5\x85\xb6\xe4\xb8\xad\xe6"
-"\x9c\x80\xe4\xb8\xbb\xe8\xa6\x81\xe4\xb9\x9f\xe6\x98\xaf\xe6\x88"
-"\x91\xe5\x80\x91\xe6\x89\x80\x0a\xe8\xa6\x81\xe8\xa8\x8e\xe8\xab"
-"\x96\xe7\x9a\x84\xe5\x95\x8f\xe9\xa1\x8c\xe5\xb0\xb1\xe6\x98\xaf"
-"\x3a\x0a\x0a"),
-'johab': (
-"\x99\xb1\xa4\x77\x88\x62\xd0\x61\x20\xcd\x5c\xaf\xa1\xc5\xa9\x9c"
-"\x61\x0a\x0a\xdc\xc0\xdc\xc0\x90\x73\x21\x21\x20\xf1\x67\xe2\x9c"
-"\xf0\x55\xcc\x81\xa3\x89\x9f\x85\x8a\xa1\x20\xdc\xde\xdc\xd3\xd2"
-"\x7a\xd9\xaf\xd9\xaf\xd9\xaf\x20\x8b\x77\x96\xd3\x20\xdc\xd1\x95"
-"\x81\x20\xdc\xc0\x2e\x20\x2e\x0a\xed\x3c\xb5\x77\xdc\xd1\x93\x77"
-"\xd2\x73\x20\x2e\x20\x2e\x20\x2e\x20\x2e\x20\xac\xe1\xb6\x89\x9e"
-"\xa1\x20\x95\x65\xd0\x62\xf0\xe0\x20\xe0\x3b\xd2\x7a\x20\x21\x20"
-"\x21\x20\x21\x87\x41\x2e\x87\x41\x0a\xd3\x61\xd3\x61\xd3\x61\x20"
-"\x88\x41\x88\x41\x88\x41\xd9\x69\x87\x41\x5f\x87\x41\x20\xb4\xe1"
-"\x9f\x9a\x20\xc8\xa1\xc5\xc1\x8b\x7a\x20\x95\x61\xb7\x77\x20\xc3"
-"\x97\xe2\x9c\x97\x69\xf0\xe0\x20\xdc\xc0\x97\x61\x8b\x7a\x0a\xac"
-"\xe9\x9f\x7a\x20\xe0\x3b\xd2\x7a\x20\x2e\x20\x2e\x20\x2e\x20\x2e"
-"\x20\x8a\x89\xb4\x81\xae\xba\x20\xdc\xd1\x8a\xa1\x20\xdc\xde\x9f"
-"\x89\xdc\xc2\x8b\x7a\x20\xf1\x67\xf1\x62\xf5\x49\xed\xfc\xf3\xe9"
-"\x8c\x61\xbb\x9a\x0a\xb5\xc1\xb2\xa1\xd2\x7a\x20\x21\x20\x21\x20"
-"\xed\x3c\xb5\x77\xdc\xd1\x20\xe0\x3b\x93\x77\x8a\xa1\x20\xd9\x69"
-"\xea\xbe\x89\xc5\x20\xb4\xf4\x93\x77\x8a\xa1\x93\x77\x20\xed\x3c"
-"\x93\x77\x96\xc1\xd2\x7a\x20\x8b\x69\xb4\x81\x97\x7a\x0a\xdc\xde"
-"\x9d\x61\x97\x41\xe2\x9c\x20\xaf\x81\xce\xa1\xae\xa1\xd2\x7a\x20"
-"\xb4\xe1\x9f\x9a\x20\xf1\x67\xf1\x62\xf5\x49\xed\xfc\xf3\xe9\xaf"
-"\x82\xdc\xef\x97\x69\xb4\x7a\x21\x21\x20\xdc\xc0\xdc\xc0\x90\x73"
-"\xd9\xbd\x20\xd9\x62\xd9\x62\x2a\x0a\x0a",
-"\xeb\x98\xa0\xeb\xb0\xa9\xea\xb0\x81\xed\x95\x98\x20\xed\x8e\xb2"
-"\xec\x8b\x9c\xec\xbd\x9c\xeb\x9d\xbc\x0a\x0a\xe3\x89\xaf\xe3\x89"
-"\xaf\xeb\x82\xa9\x21\x21\x20\xe5\x9b\xa0\xe4\xb9\x9d\xe6\x9c\x88"
-"\xed\x8c\xa8\xeb\xaf\xa4\xeb\xa6\x94\xea\xb6\x88\x20\xe2\x93\xa1"
-"\xe2\x93\x96\xed\x9b\x80\xc2\xbf\xc2\xbf\xc2\xbf\x20\xea\xb8\x8d"
-"\xeb\x92\x99\x20\xe2\x93\x94\xeb\x8e\xa8\x20\xe3\x89\xaf\x2e\x20"
-"\x2e\x0a\xe4\xba\x9e\xec\x98\x81\xe2\x93\x94\xeb\x8a\xa5\xed\x9a"
-"\xb9\x20\x2e\x20\x2e\x20\x2e\x20\x2e\x20\xec\x84\x9c\xec\x9a\xb8"
-"\xeb\xa4\x84\x20\xeb\x8e\x90\xed\x95\x99\xe4\xb9\x99\x20\xe5\xae"
-"\xb6\xed\x9b\x80\x20\x21\x20\x21\x20\x21\xe3\x85\xa0\x2e\xe3\x85"
-"\xa0\x0a\xed\x9d\x90\xed\x9d\x90\xed\x9d\x90\x20\xe3\x84\xb1\xe3"
-"\x84\xb1\xe3\x84\xb1\xe2\x98\x86\xe3\x85\xa0\x5f\xe3\x85\xa0\x20"
-"\xec\x96\xb4\xeb\xa6\xa8\x20\xed\x83\xb8\xec\xbd\xb0\xea\xb8\x90"
-"\x20\xeb\x8e\x8c\xec\x9d\x91\x20\xec\xb9\x91\xe4\xb9\x9d\xeb\x93"
-"\xa4\xe4\xb9\x99\x20\xe3\x89\xaf\xeb\x93\x9c\xea\xb8\x90\x0a\xec"
-"\x84\xa4\xeb\xa6\x8c\x20\xe5\xae\xb6\xed\x9b\x80\x20\x2e\x20\x2e"
-"\x20\x2e\x20\x2e\x20\xea\xb5\xb4\xec\x95\xa0\xec\x89\x8c\x20\xe2"
-"\x93\x94\xea\xb6\x88\x20\xe2\x93\xa1\xeb\xa6\x98\xe3\x89\xb1\xea"
-"\xb8\x90\x20\xe5\x9b\xa0\xe4\xbb\x81\xe5\xb7\x9d\xef\xa6\x81\xe4"
-"\xb8\xad\xea\xb9\x8c\xec\xa6\xbc\x0a\xec\x99\x80\xec\x92\x80\xed"
-"\x9b\x80\x20\x21\x20\x21\x20\xe4\xba\x9e\xec\x98\x81\xe2\x93\x94"
-"\x20\xe5\xae\xb6\xeb\x8a\xa5\xea\xb6\x88\x20\xe2\x98\x86\xe4\xb8"
-"\x8a\xea\xb4\x80\x20\xec\x97\x86\xeb\x8a\xa5\xea\xb6\x88\xeb\x8a"
-"\xa5\x20\xe4\xba\x9e\xeb\x8a\xa5\xeb\x92\x88\xed\x9b\x80\x20\xea"
-"\xb8\x80\xec\x95\xa0\xeb\x93\xb4\x0a\xe2\x93\xa1\xeb\xa0\xa4\xeb"
-"\x93\x80\xe4\xb9\x9d\x20\xec\x8b\x80\xed\x92\x94\xec\x88\xb4\xed"
-"\x9b\x80\x20\xec\x96\xb4\xeb\xa6\xa8\x20\xe5\x9b\xa0\xe4\xbb\x81"
-"\xe5\xb7\x9d\xef\xa6\x81\xe4\xb8\xad\xec\x8b\x81\xe2\x91\xa8\xeb"
-"\x93\xa4\xec\x95\x9c\x21\x21\x20\xe3\x89\xaf\xe3\x89\xaf\xeb\x82"
-"\xa9\xe2\x99\xa1\x20\xe2\x8c\x92\xe2\x8c\x92\x2a\x0a\x0a"),
-'shift_jis': (
-"\x50\x79\x74\x68\x6f\x6e\x20\x82\xcc\x8a\x4a\x94\xad\x82\xcd\x81"
-"\x41\x31\x39\x39\x30\x20\x94\x4e\x82\xb2\x82\xeb\x82\xa9\x82\xe7"
-"\x8a\x4a\x8e\x6e\x82\xb3\x82\xea\x82\xc4\x82\xa2\x82\xdc\x82\xb7"
-"\x81\x42\x0a\x8a\x4a\x94\xad\x8e\xd2\x82\xcc\x20\x47\x75\x69\x64"
-"\x6f\x20\x76\x61\x6e\x20\x52\x6f\x73\x73\x75\x6d\x20\x82\xcd\x8b"
-"\xb3\x88\xe7\x97\x70\x82\xcc\x83\x76\x83\x8d\x83\x4f\x83\x89\x83"
-"\x7e\x83\x93\x83\x4f\x8c\xbe\x8c\xea\x81\x75\x41\x42\x43\x81\x76"
-"\x82\xcc\x8a\x4a\x94\xad\x82\xc9\x8e\x51\x89\xc1\x82\xb5\x82\xc4"
-"\x82\xa2\x82\xdc\x82\xb5\x82\xbd\x82\xaa\x81\x41\x41\x42\x43\x20"
-"\x82\xcd\x8e\xc0\x97\x70\x8f\xe3\x82\xcc\x96\xda\x93\x49\x82\xc9"
-"\x82\xcd\x82\xa0\x82\xdc\x82\xe8\x93\x4b\x82\xb5\x82\xc4\x82\xa2"
-"\x82\xdc\x82\xb9\x82\xf1\x82\xc5\x82\xb5\x82\xbd\x81\x42\x0a\x82"
-"\xb1\x82\xcc\x82\xbd\x82\xdf\x81\x41\x47\x75\x69\x64\x6f\x20\x82"
-"\xcd\x82\xe6\x82\xe8\x8e\xc0\x97\x70\x93\x49\x82\xc8\x83\x76\x83"
-"\x8d\x83\x4f\x83\x89\x83\x7e\x83\x93\x83\x4f\x8c\xbe\x8c\xea\x82"
-"\xcc\x8a\x4a\x94\xad\x82\xf0\x8a\x4a\x8e\x6e\x82\xb5\x81\x41\x89"
-"\x70\x8d\x91\x20\x42\x42\x53\x20\x95\xfa\x91\x97\x82\xcc\x83\x52"
-"\x83\x81\x83\x66\x83\x42\x94\xd4\x91\x67\x81\x75\x83\x82\x83\x93"
-"\x83\x65\x83\x42\x20\x83\x70\x83\x43\x83\x5c\x83\x93\x81\x76\x82"
-"\xcc\x83\x74\x83\x40\x83\x93\x82\xc5\x82\xa0\x82\xe9\x20\x47\x75"
-"\x69\x64\x6f\x20\x82\xcd\x82\xb1\x82\xcc\x8c\xbe\x8c\xea\x82\xf0"
-"\x81\x75\x50\x79\x74\x68\x6f\x6e\x81\x76\x82\xc6\x96\xbc\x82\xc3"
-"\x82\xaf\x82\xdc\x82\xb5\x82\xbd\x81\x42\x0a\x82\xb1\x82\xcc\x82"
-"\xe6\x82\xa4\x82\xc8\x94\x77\x8c\x69\x82\xa9\x82\xe7\x90\xb6\x82"
-"\xdc\x82\xea\x82\xbd\x20\x50\x79\x74\x68\x6f\x6e\x20\x82\xcc\x8c"
-"\xbe\x8c\xea\x90\xdd\x8c\x76\x82\xcd\x81\x41\x81\x75\x83\x56\x83"
-"\x93\x83\x76\x83\x8b\x81\x76\x82\xc5\x81\x75\x8f\x4b\x93\xbe\x82"
-"\xaa\x97\x65\x88\xd5\x81\x76\x82\xc6\x82\xa2\x82\xa4\x96\xda\x95"
-"\x57\x82\xc9\x8f\x64\x93\x5f\x82\xaa\x92\x75\x82\xa9\x82\xea\x82"
-"\xc4\x82\xa2\x82\xdc\x82\xb7\x81\x42\x0a\x91\xbd\x82\xad\x82\xcc"
-"\x83\x58\x83\x4e\x83\x8a\x83\x76\x83\x67\x8c\x6e\x8c\xbe\x8c\xea"
-"\x82\xc5\x82\xcd\x83\x86\x81\x5b\x83\x55\x82\xcc\x96\xda\x90\xe6"
-"\x82\xcc\x97\x98\x95\xd6\x90\xab\x82\xf0\x97\x44\x90\xe6\x82\xb5"
-"\x82\xc4\x90\x46\x81\x58\x82\xc8\x8b\x40\x94\x5c\x82\xf0\x8c\xbe"
-"\x8c\xea\x97\x76\x91\x66\x82\xc6\x82\xb5\x82\xc4\x8e\xe6\x82\xe8"
-"\x93\xfc\x82\xea\x82\xe9\x8f\xea\x8d\x87\x82\xaa\x91\xbd\x82\xa2"
-"\x82\xcc\x82\xc5\x82\xb7\x82\xaa\x81\x41\x50\x79\x74\x68\x6f\x6e"
-"\x20\x82\xc5\x82\xcd\x82\xbb\x82\xa4\x82\xa2\x82\xc1\x82\xbd\x8f"
-"\xac\x8d\xd7\x8d\x48\x82\xaa\x92\xc7\x89\xc1\x82\xb3\x82\xea\x82"
-"\xe9\x82\xb1\x82\xc6\x82\xcd\x82\xa0\x82\xdc\x82\xe8\x82\xa0\x82"
-"\xe8\x82\xdc\x82\xb9\x82\xf1\x81\x42\x0a\x8c\xbe\x8c\xea\x8e\xa9"
-"\x91\xcc\x82\xcc\x8b\x40\x94\x5c\x82\xcd\x8d\xc5\x8f\xac\x8c\xc0"
-"\x82\xc9\x89\x9f\x82\xb3\x82\xa6\x81\x41\x95\x4b\x97\x76\x82\xc8"
-"\x8b\x40\x94\x5c\x82\xcd\x8a\x67\x92\xa3\x83\x82\x83\x57\x83\x85"
-"\x81\x5b\x83\x8b\x82\xc6\x82\xb5\x82\xc4\x92\xc7\x89\xc1\x82\xb7"
-"\x82\xe9\x81\x41\x82\xc6\x82\xa2\x82\xa4\x82\xcc\x82\xaa\x20\x50"
-"\x79\x74\x68\x6f\x6e\x20\x82\xcc\x83\x7c\x83\x8a\x83\x56\x81\x5b"
-"\x82\xc5\x82\xb7\x81\x42\x0a\x0a",
-"\x50\x79\x74\x68\x6f\x6e\x20\xe3\x81\xae\xe9\x96\x8b\xe7\x99\xba"
-"\xe3\x81\xaf\xe3\x80\x81\x31\x39\x39\x30\x20\xe5\xb9\xb4\xe3\x81"
-"\x94\xe3\x82\x8d\xe3\x81\x8b\xe3\x82\x89\xe9\x96\x8b\xe5\xa7\x8b"
-"\xe3\x81\x95\xe3\x82\x8c\xe3\x81\xa6\xe3\x81\x84\xe3\x81\xbe\xe3"
-"\x81\x99\xe3\x80\x82\x0a\xe9\x96\x8b\xe7\x99\xba\xe8\x80\x85\xe3"
-"\x81\xae\x20\x47\x75\x69\x64\x6f\x20\x76\x61\x6e\x20\x52\x6f\x73"
-"\x73\x75\x6d\x20\xe3\x81\xaf\xe6\x95\x99\xe8\x82\xb2\xe7\x94\xa8"
-"\xe3\x81\xae\xe3\x83\x97\xe3\x83\xad\xe3\x82\xb0\xe3\x83\xa9\xe3"
-"\x83\x9f\xe3\x83\xb3\xe3\x82\xb0\xe8\xa8\x80\xe8\xaa\x9e\xe3\x80"
-"\x8c\x41\x42\x43\xe3\x80\x8d\xe3\x81\xae\xe9\x96\x8b\xe7\x99\xba"
-"\xe3\x81\xab\xe5\x8f\x82\xe5\x8a\xa0\xe3\x81\x97\xe3\x81\xa6\xe3"
-"\x81\x84\xe3\x81\xbe\xe3\x81\x97\xe3\x81\x9f\xe3\x81\x8c\xe3\x80"
-"\x81\x41\x42\x43\x20\xe3\x81\xaf\xe5\xae\x9f\xe7\x94\xa8\xe4\xb8"
-"\x8a\xe3\x81\xae\xe7\x9b\xae\xe7\x9a\x84\xe3\x81\xab\xe3\x81\xaf"
-"\xe3\x81\x82\xe3\x81\xbe\xe3\x82\x8a\xe9\x81\xa9\xe3\x81\x97\xe3"
-"\x81\xa6\xe3\x81\x84\xe3\x81\xbe\xe3\x81\x9b\xe3\x82\x93\xe3\x81"
-"\xa7\xe3\x81\x97\xe3\x81\x9f\xe3\x80\x82\x0a\xe3\x81\x93\xe3\x81"
-"\xae\xe3\x81\x9f\xe3\x82\x81\xe3\x80\x81\x47\x75\x69\x64\x6f\x20"
-"\xe3\x81\xaf\xe3\x82\x88\xe3\x82\x8a\xe5\xae\x9f\xe7\x94\xa8\xe7"
-"\x9a\x84\xe3\x81\xaa\xe3\x83\x97\xe3\x83\xad\xe3\x82\xb0\xe3\x83"
-"\xa9\xe3\x83\x9f\xe3\x83\xb3\xe3\x82\xb0\xe8\xa8\x80\xe8\xaa\x9e"
-"\xe3\x81\xae\xe9\x96\x8b\xe7\x99\xba\xe3\x82\x92\xe9\x96\x8b\xe5"
-"\xa7\x8b\xe3\x81\x97\xe3\x80\x81\xe8\x8b\xb1\xe5\x9b\xbd\x20\x42"
-"\x42\x53\x20\xe6\x94\xbe\xe9\x80\x81\xe3\x81\xae\xe3\x82\xb3\xe3"
-"\x83\xa1\xe3\x83\x87\xe3\x82\xa3\xe7\x95\xaa\xe7\xb5\x84\xe3\x80"
-"\x8c\xe3\x83\xa2\xe3\x83\xb3\xe3\x83\x86\xe3\x82\xa3\x20\xe3\x83"
-"\x91\xe3\x82\xa4\xe3\x82\xbd\xe3\x83\xb3\xe3\x80\x8d\xe3\x81\xae"
-"\xe3\x83\x95\xe3\x82\xa1\xe3\x83\xb3\xe3\x81\xa7\xe3\x81\x82\xe3"
-"\x82\x8b\x20\x47\x75\x69\x64\x6f\x20\xe3\x81\xaf\xe3\x81\x93\xe3"
-"\x81\xae\xe8\xa8\x80\xe8\xaa\x9e\xe3\x82\x92\xe3\x80\x8c\x50\x79"
-"\x74\x68\x6f\x6e\xe3\x80\x8d\xe3\x81\xa8\xe5\x90\x8d\xe3\x81\xa5"
-"\xe3\x81\x91\xe3\x81\xbe\xe3\x81\x97\xe3\x81\x9f\xe3\x80\x82\x0a"
-"\xe3\x81\x93\xe3\x81\xae\xe3\x82\x88\xe3\x81\x86\xe3\x81\xaa\xe8"
-"\x83\x8c\xe6\x99\xaf\xe3\x81\x8b\xe3\x82\x89\xe7\x94\x9f\xe3\x81"
-"\xbe\xe3\x82\x8c\xe3\x81\x9f\x20\x50\x79\x74\x68\x6f\x6e\x20\xe3"
-"\x81\xae\xe8\xa8\x80\xe8\xaa\x9e\xe8\xa8\xad\xe8\xa8\x88\xe3\x81"
-"\xaf\xe3\x80\x81\xe3\x80\x8c\xe3\x82\xb7\xe3\x83\xb3\xe3\x83\x97"
-"\xe3\x83\xab\xe3\x80\x8d\xe3\x81\xa7\xe3\x80\x8c\xe7\xbf\x92\xe5"
-"\xbe\x97\xe3\x81\x8c\xe5\xae\xb9\xe6\x98\x93\xe3\x80\x8d\xe3\x81"
-"\xa8\xe3\x81\x84\xe3\x81\x86\xe7\x9b\xae\xe6\xa8\x99\xe3\x81\xab"
-"\xe9\x87\x8d\xe7\x82\xb9\xe3\x81\x8c\xe7\xbd\xae\xe3\x81\x8b\xe3"
-"\x82\x8c\xe3\x81\xa6\xe3\x81\x84\xe3\x81\xbe\xe3\x81\x99\xe3\x80"
-"\x82\x0a\xe5\xa4\x9a\xe3\x81\x8f\xe3\x81\xae\xe3\x82\xb9\xe3\x82"
-"\xaf\xe3\x83\xaa\xe3\x83\x97\xe3\x83\x88\xe7\xb3\xbb\xe8\xa8\x80"
-"\xe8\xaa\x9e\xe3\x81\xa7\xe3\x81\xaf\xe3\x83\xa6\xe3\x83\xbc\xe3"
-"\x82\xb6\xe3\x81\xae\xe7\x9b\xae\xe5\x85\x88\xe3\x81\xae\xe5\x88"
-"\xa9\xe4\xbe\xbf\xe6\x80\xa7\xe3\x82\x92\xe5\x84\xaa\xe5\x85\x88"
-"\xe3\x81\x97\xe3\x81\xa6\xe8\x89\xb2\xe3\x80\x85\xe3\x81\xaa\xe6"
-"\xa9\x9f\xe8\x83\xbd\xe3\x82\x92\xe8\xa8\x80\xe8\xaa\x9e\xe8\xa6"
-"\x81\xe7\xb4\xa0\xe3\x81\xa8\xe3\x81\x97\xe3\x81\xa6\xe5\x8f\x96"
-"\xe3\x82\x8a\xe5\x85\xa5\xe3\x82\x8c\xe3\x82\x8b\xe5\xa0\xb4\xe5"
-"\x90\x88\xe3\x81\x8c\xe5\xa4\x9a\xe3\x81\x84\xe3\x81\xae\xe3\x81"
-"\xa7\xe3\x81\x99\xe3\x81\x8c\xe3\x80\x81\x50\x79\x74\x68\x6f\x6e"
-"\x20\xe3\x81\xa7\xe3\x81\xaf\xe3\x81\x9d\xe3\x81\x86\xe3\x81\x84"
-"\xe3\x81\xa3\xe3\x81\x9f\xe5\xb0\x8f\xe7\xb4\xb0\xe5\xb7\xa5\xe3"
-"\x81\x8c\xe8\xbf\xbd\xe5\x8a\xa0\xe3\x81\x95\xe3\x82\x8c\xe3\x82"
-"\x8b\xe3\x81\x93\xe3\x81\xa8\xe3\x81\xaf\xe3\x81\x82\xe3\x81\xbe"
-"\xe3\x82\x8a\xe3\x81\x82\xe3\x82\x8a\xe3\x81\xbe\xe3\x81\x9b\xe3"
-"\x82\x93\xe3\x80\x82\x0a\xe8\xa8\x80\xe8\xaa\x9e\xe8\x87\xaa\xe4"
-"\xbd\x93\xe3\x81\xae\xe6\xa9\x9f\xe8\x83\xbd\xe3\x81\xaf\xe6\x9c"
-"\x80\xe5\xb0\x8f\xe9\x99\x90\xe3\x81\xab\xe6\x8a\xbc\xe3\x81\x95"
-"\xe3\x81\x88\xe3\x80\x81\xe5\xbf\x85\xe8\xa6\x81\xe3\x81\xaa\xe6"
-"\xa9\x9f\xe8\x83\xbd\xe3\x81\xaf\xe6\x8b\xa1\xe5\xbc\xb5\xe3\x83"
-"\xa2\xe3\x82\xb8\xe3\x83\xa5\xe3\x83\xbc\xe3\x83\xab\xe3\x81\xa8"
-"\xe3\x81\x97\xe3\x81\xa6\xe8\xbf\xbd\xe5\x8a\xa0\xe3\x81\x99\xe3"
-"\x82\x8b\xe3\x80\x81\xe3\x81\xa8\xe3\x81\x84\xe3\x81\x86\xe3\x81"
-"\xae\xe3\x81\x8c\x20\x50\x79\x74\x68\x6f\x6e\x20\xe3\x81\xae\xe3"
-"\x83\x9d\xe3\x83\xaa\xe3\x82\xb7\xe3\x83\xbc\xe3\x81\xa7\xe3\x81"
-"\x99\xe3\x80\x82\x0a\x0a"),
-'shift_jisx0213': (
-"\x50\x79\x74\x68\x6f\x6e\x20\x82\xcc\x8a\x4a\x94\xad\x82\xcd\x81"
-"\x41\x31\x39\x39\x30\x20\x94\x4e\x82\xb2\x82\xeb\x82\xa9\x82\xe7"
-"\x8a\x4a\x8e\x6e\x82\xb3\x82\xea\x82\xc4\x82\xa2\x82\xdc\x82\xb7"
-"\x81\x42\x0a\x8a\x4a\x94\xad\x8e\xd2\x82\xcc\x20\x47\x75\x69\x64"
-"\x6f\x20\x76\x61\x6e\x20\x52\x6f\x73\x73\x75\x6d\x20\x82\xcd\x8b"
-"\xb3\x88\xe7\x97\x70\x82\xcc\x83\x76\x83\x8d\x83\x4f\x83\x89\x83"
-"\x7e\x83\x93\x83\x4f\x8c\xbe\x8c\xea\x81\x75\x41\x42\x43\x81\x76"
-"\x82\xcc\x8a\x4a\x94\xad\x82\xc9\x8e\x51\x89\xc1\x82\xb5\x82\xc4"
-"\x82\xa2\x82\xdc\x82\xb5\x82\xbd\x82\xaa\x81\x41\x41\x42\x43\x20"
-"\x82\xcd\x8e\xc0\x97\x70\x8f\xe3\x82\xcc\x96\xda\x93\x49\x82\xc9"
-"\x82\xcd\x82\xa0\x82\xdc\x82\xe8\x93\x4b\x82\xb5\x82\xc4\x82\xa2"
-"\x82\xdc\x82\xb9\x82\xf1\x82\xc5\x82\xb5\x82\xbd\x81\x42\x0a\x82"
-"\xb1\x82\xcc\x82\xbd\x82\xdf\x81\x41\x47\x75\x69\x64\x6f\x20\x82"
-"\xcd\x82\xe6\x82\xe8\x8e\xc0\x97\x70\x93\x49\x82\xc8\x83\x76\x83"
-"\x8d\x83\x4f\x83\x89\x83\x7e\x83\x93\x83\x4f\x8c\xbe\x8c\xea\x82"
-"\xcc\x8a\x4a\x94\xad\x82\xf0\x8a\x4a\x8e\x6e\x82\xb5\x81\x41\x89"
-"\x70\x8d\x91\x20\x42\x42\x53\x20\x95\xfa\x91\x97\x82\xcc\x83\x52"
-"\x83\x81\x83\x66\x83\x42\x94\xd4\x91\x67\x81\x75\x83\x82\x83\x93"
-"\x83\x65\x83\x42\x20\x83\x70\x83\x43\x83\x5c\x83\x93\x81\x76\x82"
-"\xcc\x83\x74\x83\x40\x83\x93\x82\xc5\x82\xa0\x82\xe9\x20\x47\x75"
-"\x69\x64\x6f\x20\x82\xcd\x82\xb1\x82\xcc\x8c\xbe\x8c\xea\x82\xf0"
-"\x81\x75\x50\x79\x74\x68\x6f\x6e\x81\x76\x82\xc6\x96\xbc\x82\xc3"
-"\x82\xaf\x82\xdc\x82\xb5\x82\xbd\x81\x42\x0a\x82\xb1\x82\xcc\x82"
-"\xe6\x82\xa4\x82\xc8\x94\x77\x8c\x69\x82\xa9\x82\xe7\x90\xb6\x82"
-"\xdc\x82\xea\x82\xbd\x20\x50\x79\x74\x68\x6f\x6e\x20\x82\xcc\x8c"
-"\xbe\x8c\xea\x90\xdd\x8c\x76\x82\xcd\x81\x41\x81\x75\x83\x56\x83"
-"\x93\x83\x76\x83\x8b\x81\x76\x82\xc5\x81\x75\x8f\x4b\x93\xbe\x82"
-"\xaa\x97\x65\x88\xd5\x81\x76\x82\xc6\x82\xa2\x82\xa4\x96\xda\x95"
-"\x57\x82\xc9\x8f\x64\x93\x5f\x82\xaa\x92\x75\x82\xa9\x82\xea\x82"
-"\xc4\x82\xa2\x82\xdc\x82\xb7\x81\x42\x0a\x91\xbd\x82\xad\x82\xcc"
-"\x83\x58\x83\x4e\x83\x8a\x83\x76\x83\x67\x8c\x6e\x8c\xbe\x8c\xea"
-"\x82\xc5\x82\xcd\x83\x86\x81\x5b\x83\x55\x82\xcc\x96\xda\x90\xe6"
-"\x82\xcc\x97\x98\x95\xd6\x90\xab\x82\xf0\x97\x44\x90\xe6\x82\xb5"
-"\x82\xc4\x90\x46\x81\x58\x82\xc8\x8b\x40\x94\x5c\x82\xf0\x8c\xbe"
-"\x8c\xea\x97\x76\x91\x66\x82\xc6\x82\xb5\x82\xc4\x8e\xe6\x82\xe8"
-"\x93\xfc\x82\xea\x82\xe9\x8f\xea\x8d\x87\x82\xaa\x91\xbd\x82\xa2"
-"\x82\xcc\x82\xc5\x82\xb7\x82\xaa\x81\x41\x50\x79\x74\x68\x6f\x6e"
-"\x20\x82\xc5\x82\xcd\x82\xbb\x82\xa4\x82\xa2\x82\xc1\x82\xbd\x8f"
-"\xac\x8d\xd7\x8d\x48\x82\xaa\x92\xc7\x89\xc1\x82\xb3\x82\xea\x82"
-"\xe9\x82\xb1\x82\xc6\x82\xcd\x82\xa0\x82\xdc\x82\xe8\x82\xa0\x82"
-"\xe8\x82\xdc\x82\xb9\x82\xf1\x81\x42\x0a\x8c\xbe\x8c\xea\x8e\xa9"
-"\x91\xcc\x82\xcc\x8b\x40\x94\x5c\x82\xcd\x8d\xc5\x8f\xac\x8c\xc0"
-"\x82\xc9\x89\x9f\x82\xb3\x82\xa6\x81\x41\x95\x4b\x97\x76\x82\xc8"
-"\x8b\x40\x94\x5c\x82\xcd\x8a\x67\x92\xa3\x83\x82\x83\x57\x83\x85"
-"\x81\x5b\x83\x8b\x82\xc6\x82\xb5\x82\xc4\x92\xc7\x89\xc1\x82\xb7"
-"\x82\xe9\x81\x41\x82\xc6\x82\xa2\x82\xa4\x82\xcc\x82\xaa\x20\x50"
-"\x79\x74\x68\x6f\x6e\x20\x82\xcc\x83\x7c\x83\x8a\x83\x56\x81\x5b"
-"\x82\xc5\x82\xb7\x81\x42\x0a\x0a\x83\x6d\x82\xf5\x20\x83\x9e\x20"
-"\x83\x67\x83\x4c\x88\x4b\x88\x79\x20\x98\x83\xfc\xd6\x20\xfc\xd2"
-"\xfc\xe6\xfb\xd4\x0a",
-"\x50\x79\x74\x68\x6f\x6e\x20\xe3\x81\xae\xe9\x96\x8b\xe7\x99\xba"
-"\xe3\x81\xaf\xe3\x80\x81\x31\x39\x39\x30\x20\xe5\xb9\xb4\xe3\x81"
-"\x94\xe3\x82\x8d\xe3\x81\x8b\xe3\x82\x89\xe9\x96\x8b\xe5\xa7\x8b"
-"\xe3\x81\x95\xe3\x82\x8c\xe3\x81\xa6\xe3\x81\x84\xe3\x81\xbe\xe3"
-"\x81\x99\xe3\x80\x82\x0a\xe9\x96\x8b\xe7\x99\xba\xe8\x80\x85\xe3"
-"\x81\xae\x20\x47\x75\x69\x64\x6f\x20\x76\x61\x6e\x20\x52\x6f\x73"
-"\x73\x75\x6d\x20\xe3\x81\xaf\xe6\x95\x99\xe8\x82\xb2\xe7\x94\xa8"
-"\xe3\x81\xae\xe3\x83\x97\xe3\x83\xad\xe3\x82\xb0\xe3\x83\xa9\xe3"
-"\x83\x9f\xe3\x83\xb3\xe3\x82\xb0\xe8\xa8\x80\xe8\xaa\x9e\xe3\x80"
-"\x8c\x41\x42\x43\xe3\x80\x8d\xe3\x81\xae\xe9\x96\x8b\xe7\x99\xba"
-"\xe3\x81\xab\xe5\x8f\x82\xe5\x8a\xa0\xe3\x81\x97\xe3\x81\xa6\xe3"
-"\x81\x84\xe3\x81\xbe\xe3\x81\x97\xe3\x81\x9f\xe3\x81\x8c\xe3\x80"
-"\x81\x41\x42\x43\x20\xe3\x81\xaf\xe5\xae\x9f\xe7\x94\xa8\xe4\xb8"
-"\x8a\xe3\x81\xae\xe7\x9b\xae\xe7\x9a\x84\xe3\x81\xab\xe3\x81\xaf"
-"\xe3\x81\x82\xe3\x81\xbe\xe3\x82\x8a\xe9\x81\xa9\xe3\x81\x97\xe3"
-"\x81\xa6\xe3\x81\x84\xe3\x81\xbe\xe3\x81\x9b\xe3\x82\x93\xe3\x81"
-"\xa7\xe3\x81\x97\xe3\x81\x9f\xe3\x80\x82\x0a\xe3\x81\x93\xe3\x81"
-"\xae\xe3\x81\x9f\xe3\x82\x81\xe3\x80\x81\x47\x75\x69\x64\x6f\x20"
-"\xe3\x81\xaf\xe3\x82\x88\xe3\x82\x8a\xe5\xae\x9f\xe7\x94\xa8\xe7"
-"\x9a\x84\xe3\x81\xaa\xe3\x83\x97\xe3\x83\xad\xe3\x82\xb0\xe3\x83"
-"\xa9\xe3\x83\x9f\xe3\x83\xb3\xe3\x82\xb0\xe8\xa8\x80\xe8\xaa\x9e"
-"\xe3\x81\xae\xe9\x96\x8b\xe7\x99\xba\xe3\x82\x92\xe9\x96\x8b\xe5"
-"\xa7\x8b\xe3\x81\x97\xe3\x80\x81\xe8\x8b\xb1\xe5\x9b\xbd\x20\x42"
-"\x42\x53\x20\xe6\x94\xbe\xe9\x80\x81\xe3\x81\xae\xe3\x82\xb3\xe3"
-"\x83\xa1\xe3\x83\x87\xe3\x82\xa3\xe7\x95\xaa\xe7\xb5\x84\xe3\x80"
-"\x8c\xe3\x83\xa2\xe3\x83\xb3\xe3\x83\x86\xe3\x82\xa3\x20\xe3\x83"
-"\x91\xe3\x82\xa4\xe3\x82\xbd\xe3\x83\xb3\xe3\x80\x8d\xe3\x81\xae"
-"\xe3\x83\x95\xe3\x82\xa1\xe3\x83\xb3\xe3\x81\xa7\xe3\x81\x82\xe3"
-"\x82\x8b\x20\x47\x75\x69\x64\x6f\x20\xe3\x81\xaf\xe3\x81\x93\xe3"
-"\x81\xae\xe8\xa8\x80\xe8\xaa\x9e\xe3\x82\x92\xe3\x80\x8c\x50\x79"
-"\x74\x68\x6f\x6e\xe3\x80\x8d\xe3\x81\xa8\xe5\x90\x8d\xe3\x81\xa5"
-"\xe3\x81\x91\xe3\x81\xbe\xe3\x81\x97\xe3\x81\x9f\xe3\x80\x82\x0a"
-"\xe3\x81\x93\xe3\x81\xae\xe3\x82\x88\xe3\x81\x86\xe3\x81\xaa\xe8"
-"\x83\x8c\xe6\x99\xaf\xe3\x81\x8b\xe3\x82\x89\xe7\x94\x9f\xe3\x81"
-"\xbe\xe3\x82\x8c\xe3\x81\x9f\x20\x50\x79\x74\x68\x6f\x6e\x20\xe3"
-"\x81\xae\xe8\xa8\x80\xe8\xaa\x9e\xe8\xa8\xad\xe8\xa8\x88\xe3\x81"
-"\xaf\xe3\x80\x81\xe3\x80\x8c\xe3\x82\xb7\xe3\x83\xb3\xe3\x83\x97"
-"\xe3\x83\xab\xe3\x80\x8d\xe3\x81\xa7\xe3\x80\x8c\xe7\xbf\x92\xe5"
-"\xbe\x97\xe3\x81\x8c\xe5\xae\xb9\xe6\x98\x93\xe3\x80\x8d\xe3\x81"
-"\xa8\xe3\x81\x84\xe3\x81\x86\xe7\x9b\xae\xe6\xa8\x99\xe3\x81\xab"
-"\xe9\x87\x8d\xe7\x82\xb9\xe3\x81\x8c\xe7\xbd\xae\xe3\x81\x8b\xe3"
-"\x82\x8c\xe3\x81\xa6\xe3\x81\x84\xe3\x81\xbe\xe3\x81\x99\xe3\x80"
-"\x82\x0a\xe5\xa4\x9a\xe3\x81\x8f\xe3\x81\xae\xe3\x82\xb9\xe3\x82"
-"\xaf\xe3\x83\xaa\xe3\x83\x97\xe3\x83\x88\xe7\xb3\xbb\xe8\xa8\x80"
-"\xe8\xaa\x9e\xe3\x81\xa7\xe3\x81\xaf\xe3\x83\xa6\xe3\x83\xbc\xe3"
-"\x82\xb6\xe3\x81\xae\xe7\x9b\xae\xe5\x85\x88\xe3\x81\xae\xe5\x88"
-"\xa9\xe4\xbe\xbf\xe6\x80\xa7\xe3\x82\x92\xe5\x84\xaa\xe5\x85\x88"
-"\xe3\x81\x97\xe3\x81\xa6\xe8\x89\xb2\xe3\x80\x85\xe3\x81\xaa\xe6"
-"\xa9\x9f\xe8\x83\xbd\xe3\x82\x92\xe8\xa8\x80\xe8\xaa\x9e\xe8\xa6"
-"\x81\xe7\xb4\xa0\xe3\x81\xa8\xe3\x81\x97\xe3\x81\xa6\xe5\x8f\x96"
-"\xe3\x82\x8a\xe5\x85\xa5\xe3\x82\x8c\xe3\x82\x8b\xe5\xa0\xb4\xe5"
-"\x90\x88\xe3\x81\x8c\xe5\xa4\x9a\xe3\x81\x84\xe3\x81\xae\xe3\x81"
-"\xa7\xe3\x81\x99\xe3\x81\x8c\xe3\x80\x81\x50\x79\x74\x68\x6f\x6e"
-"\x20\xe3\x81\xa7\xe3\x81\xaf\xe3\x81\x9d\xe3\x81\x86\xe3\x81\x84"
-"\xe3\x81\xa3\xe3\x81\x9f\xe5\xb0\x8f\xe7\xb4\xb0\xe5\xb7\xa5\xe3"
-"\x81\x8c\xe8\xbf\xbd\xe5\x8a\xa0\xe3\x81\x95\xe3\x82\x8c\xe3\x82"
-"\x8b\xe3\x81\x93\xe3\x81\xa8\xe3\x81\xaf\xe3\x81\x82\xe3\x81\xbe"
-"\xe3\x82\x8a\xe3\x81\x82\xe3\x82\x8a\xe3\x81\xbe\xe3\x81\x9b\xe3"
-"\x82\x93\xe3\x80\x82\x0a\xe8\xa8\x80\xe8\xaa\x9e\xe8\x87\xaa\xe4"
-"\xbd\x93\xe3\x81\xae\xe6\xa9\x9f\xe8\x83\xbd\xe3\x81\xaf\xe6\x9c"
-"\x80\xe5\xb0\x8f\xe9\x99\x90\xe3\x81\xab\xe6\x8a\xbc\xe3\x81\x95"
-"\xe3\x81\x88\xe3\x80\x81\xe5\xbf\x85\xe8\xa6\x81\xe3\x81\xaa\xe6"
-"\xa9\x9f\xe8\x83\xbd\xe3\x81\xaf\xe6\x8b\xa1\xe5\xbc\xb5\xe3\x83"
-"\xa2\xe3\x82\xb8\xe3\x83\xa5\xe3\x83\xbc\xe3\x83\xab\xe3\x81\xa8"
-"\xe3\x81\x97\xe3\x81\xa6\xe8\xbf\xbd\xe5\x8a\xa0\xe3\x81\x99\xe3"
-"\x82\x8b\xe3\x80\x81\xe3\x81\xa8\xe3\x81\x84\xe3\x81\x86\xe3\x81"
-"\xae\xe3\x81\x8c\x20\x50\x79\x74\x68\x6f\x6e\x20\xe3\x81\xae\xe3"
-"\x83\x9d\xe3\x83\xaa\xe3\x82\xb7\xe3\x83\xbc\xe3\x81\xa7\xe3\x81"
-"\x99\xe3\x80\x82\x0a\x0a\xe3\x83\x8e\xe3\x81\x8b\xe3\x82\x9a\x20"
-"\xe3\x83\x88\xe3\x82\x9a\x20\xe3\x83\x88\xe3\x82\xad\xef\xa8\xb6"
-"\xef\xa8\xb9\x20\xf0\xa1\x9a\xb4\xf0\xaa\x8e\x8c\x20\xe9\xba\x80"
-"\xe9\xbd\x81\xf0\xa9\x9b\xb0\x0a"),
-}
diff --git a/lib-python/2.7/test/crashers/README b/lib-python/2.7/test/crashers/README
--- a/lib-python/2.7/test/crashers/README
+++ b/lib-python/2.7/test/crashers/README
@@ -1,20 +1,16 @@
-This directory only contains tests for outstanding bugs that cause
-the interpreter to segfault.  Ideally this directory should always
-be empty.  Sometimes it may not be easy to fix the underlying cause.
+This directory only contains tests for outstanding bugs that cause the
+interpreter to segfault.  Ideally this directory should always be empty, but
+sometimes it may not be easy to fix the underlying cause and the bug is deemed
+too obscure to invest the effort.
 
 Each test should fail when run from the command line:
 
 	./python Lib/test/crashers/weakref_in_del.py
 
-Each test should have a link to the bug report:
+Put as much info into a docstring or comments to help determine the cause of the
+failure, as well as a bugs.python.org issue number if it exists.  Particularly
+note if the cause is system or environment dependent and what the variables are.
 
-	# http://python.org/sf/BUG#
-
-Put as much info into a docstring or comments to help determine
-the cause of the failure.  Particularly note if the cause is
-system or environment dependent and what the variables are.
-
-Once the crash is fixed, the test case should be moved into an appropriate
-test (even if it was originally from the test suite).  This ensures the
-regression doesn't happen again.  And if it does, it should be easier
-to track down.
+Once the crash is fixed, the test case should be moved into an appropriate test
+(even if it was originally from the test suite).  This ensures the regression
+doesn't happen again.  And if it does, it should be easier to track down.
diff --git a/lib-python/2.7/test/crashers/recursion_limit_too_high.py b/lib-python/2.7/test/crashers/recursion_limit_too_high.py
--- a/lib-python/2.7/test/crashers/recursion_limit_too_high.py
+++ b/lib-python/2.7/test/crashers/recursion_limit_too_high.py
@@ -5,7 +5,7 @@
 # file handles.
 
 # The point of this example is to show that sys.setrecursionlimit() is a
-# hack, and not a robust solution.  This example simply exercices a path
+# hack, and not a robust solution.  This example simply exercises a path
 # where it takes many C-level recursions, consuming a lot of stack
 # space, for each Python-level recursion.  So 1000 times this amount of
 # stack space may be too much for standard platforms already.
diff --git a/lib-python/2.7/test/decimaltestdata/and.decTest b/lib-python/2.7/test/decimaltestdata/and.decTest
--- a/lib-python/2.7/test/decimaltestdata/and.decTest
+++ b/lib-python/2.7/test/decimaltestdata/and.decTest
@@ -1,338 +1,338 @@
-------------------------------------------------------------------------
--- and.decTest -- digitwise logical AND                               --
--- Copyright (c) IBM Corporation, 1981, 2008.  All rights reserved.   --
-------------------------------------------------------------------------
--- Please see the document "General Decimal Arithmetic Testcases"     --
--- at http://www2.hursley.ibm.com/decimal for the description of      --
--- these testcases.                                                   --
---                                                                    --
--- These testcases are experimental ('beta' versions), and they       --
--- may contain errors.  They are offered on an as-is basis.  In       --
--- particular, achieving the same results as the tests here is not    --
--- a guarantee that an implementation complies with any Standard      --
--- or specification.  The tests are not exhaustive.                   --
---                                                                    --
--- Please send comments, suggestions, and corrections to the author:  --
---   Mike Cowlishaw, IBM Fellow                                       --
---   IBM UK, PO Box 31, Birmingham Road, Warwick CV34 5JL, UK         --
---   mfc at uk.ibm.com                                                   --
-------------------------------------------------------------------------
-version: 2.59
-
-extended:    1
-precision:   9
-rounding:    half_up
-maxExponent: 999
-minExponent: -999
-
--- Sanity check (truth table)
-andx001 and             0    0 ->    0
-andx002 and             0    1 ->    0
-andx003 and             1    0 ->    0
-andx004 and             1    1 ->    1
-andx005 and          1100 1010 -> 1000
-andx006 and          1111   10 ->   10
-andx007 and          1111 1010 -> 1010
-
--- and at msd and msd-1
-andx010 and 000000000 000000000 ->           0
-andx011 and 000000000 100000000 ->           0
-andx012 and 100000000 000000000 ->           0
-andx013 and 100000000 100000000 ->   100000000
-andx014 and 000000000 000000000 ->           0
-andx015 and 000000000 010000000 ->           0
-andx016 and 010000000 000000000 ->           0
-andx017 and 010000000 010000000 ->    10000000
-
--- Various lengths
---          123456789     123456789      123456789
-andx021 and 111111111     111111111  ->  111111111
-andx022 and 111111111111  111111111  ->  111111111
-andx023 and 111111111111   11111111  ->   11111111
-andx024 and 111111111      11111111  ->   11111111
-andx025 and 111111111       1111111  ->    1111111
-andx026 and 111111111111     111111  ->     111111
-andx027 and 111111111111      11111  ->      11111
-andx028 and 111111111111       1111  ->       1111
-andx029 and 111111111111        111  ->        111
-andx031 and 111111111111         11  ->         11
-andx032 and 111111111111          1  ->          1
-andx033 and 111111111111 1111111111  ->  111111111
-andx034 and 11111111111 11111111111  ->  111111111
-andx035 and 1111111111 111111111111  ->  111111111
-andx036 and 111111111 1111111111111  ->  111111111
-
-andx040 and 111111111  111111111111  ->  111111111
-andx041 and  11111111  111111111111  ->   11111111
-andx042 and  11111111     111111111  ->   11111111
-andx043 and   1111111     111111111  ->    1111111
-andx044 and    111111     111111111  ->     111111
-andx045 and     11111     111111111  ->      11111
-andx046 and      1111     111111111  ->       1111
-andx047 and       111     111111111  ->        111
-andx048 and        11     111111111  ->         11
-andx049 and         1     111111111  ->          1
-
-andx050 and 1111111111  1  ->  1
-andx051 and  111111111  1  ->  1
-andx052 and   11111111  1  ->  1
-andx053 and    1111111  1  ->  1
-andx054 and     111111  1  ->  1
-andx055 and      11111  1  ->  1
-andx056 and       1111  1  ->  1
-andx057 and        111  1  ->  1
-andx058 and         11  1  ->  1
-andx059 and          1  1  ->  1
-
-andx060 and 1111111111  0  ->  0
-andx061 and  111111111  0  ->  0
-andx062 and   11111111  0  ->  0
-andx063 and    1111111  0  ->  0
-andx064 and     111111  0  ->  0
-andx065 and      11111  0  ->  0
-andx066 and       1111  0  ->  0
-andx067 and        111  0  ->  0
-andx068 and         11  0  ->  0
-andx069 and          1  0  ->  0
-
-andx070 and 1  1111111111  ->  1
-andx071 and 1   111111111  ->  1
-andx072 and 1    11111111  ->  1
-andx073 and 1     1111111  ->  1
-andx074 and 1      111111  ->  1
-andx075 and 1       11111  ->  1
-andx076 and 1        1111  ->  1
-andx077 and 1         111  ->  1
-andx078 and 1          11  ->  1
-andx079 and 1           1  ->  1
-
-andx080 and 0  1111111111  ->  0
-andx081 and 0   111111111  ->  0
-andx082 and 0    11111111  ->  0
-andx083 and 0     1111111  ->  0
-andx084 and 0      111111  ->  0
-andx085 and 0       11111  ->  0
-andx086 and 0        1111  ->  0
-andx087 and 0         111  ->  0
-andx088 and 0          11  ->  0
-andx089 and 0           1  ->  0
-
-andx090 and 011111111  111111111  ->   11111111
-andx091 and 101111111  111111111  ->  101111111
-andx092 and 110111111  111111111  ->  110111111
-andx093 and 111011111  111111111  ->  111011111
-andx094 and 111101111  111111111  ->  111101111
-andx095 and 111110111  111111111  ->  111110111
-andx096 and 111111011  111111111  ->  111111011
-andx097 and 111111101  111111111  ->  111111101
-andx098 and 111111110  111111111  ->  111111110
-
-andx100 and 111111111  011111111  ->   11111111
-andx101 and 111111111  101111111  ->  101111111
-andx102 and 111111111  110111111  ->  110111111
-andx103 and 111111111  111011111  ->  111011111
-andx104 and 111111111  111101111  ->  111101111
-andx105 and 111111111  111110111  ->  111110111
-andx106 and 111111111  111111011  ->  111111011
-andx107 and 111111111  111111101  ->  111111101
-andx108 and 111111111  111111110  ->  111111110
-
--- non-0/1 should not be accepted, nor should signs
-andx220 and 111111112  111111111  ->  NaN Invalid_operation
-andx221 and 333333333  333333333  ->  NaN Invalid_operation
-andx222 and 555555555  555555555  ->  NaN Invalid_operation
-andx223 and 777777777  777777777  ->  NaN Invalid_operation
-andx224 and 999999999  999999999  ->  NaN Invalid_operation
-andx225 and 222222222  999999999  ->  NaN Invalid_operation
-andx226 and 444444444  999999999  ->  NaN Invalid_operation
-andx227 and 666666666  999999999  ->  NaN Invalid_operation
-andx228 and 888888888  999999999  ->  NaN Invalid_operation
-andx229 and 999999999  222222222  ->  NaN Invalid_operation
-andx230 and 999999999  444444444  ->  NaN Invalid_operation
-andx231 and 999999999  666666666  ->  NaN Invalid_operation
-andx232 and 999999999  888888888  ->  NaN Invalid_operation
--- a few randoms
-andx240 and  567468689 -934981942 ->  NaN Invalid_operation
-andx241 and  567367689  934981942 ->  NaN Invalid_operation
-andx242 and -631917772 -706014634 ->  NaN Invalid_operation
-andx243 and -756253257  138579234 ->  NaN Invalid_operation
-andx244 and  835590149  567435400 ->  NaN Invalid_operation
--- test MSD
-andx250 and  200000000 100000000 ->  NaN Invalid_operation
-andx251 and  700000000 100000000 ->  NaN Invalid_operation
-andx252 and  800000000 100000000 ->  NaN Invalid_operation
-andx253 and  900000000 100000000 ->  NaN Invalid_operation
-andx254 and  200000000 000000000 ->  NaN Invalid_operation
-andx255 and  700000000 000000000 ->  NaN Invalid_operation
-andx256 and  800000000 000000000 ->  NaN Invalid_operation
-andx257 and  900000000 000000000 ->  NaN Invalid_operation
-andx258 and  100000000 200000000 ->  NaN Invalid_operation
-andx259 and  100000000 700000000 ->  NaN Invalid_operation
-andx260 and  100000000 800000000 ->  NaN Invalid_operation
-andx261 and  100000000 900000000 ->  NaN Invalid_operation
-andx262 and  000000000 200000000 ->  NaN Invalid_operation
-andx263 and  000000000 700000000 ->  NaN Invalid_operation
-andx264 and  000000000 800000000 ->  NaN Invalid_operation
-andx265 and  000000000 900000000 ->  NaN Invalid_operation
--- test MSD-1
-andx270 and  020000000 100000000 ->  NaN Invalid_operation
-andx271 and  070100000 100000000 ->  NaN Invalid_operation
-andx272 and  080010000 100000001 ->  NaN Invalid_operation
-andx273 and  090001000 100000010 ->  NaN Invalid_operation
-andx274 and  100000100 020010100 ->  NaN Invalid_operation
-andx275 and  100000000 070001000 ->  NaN Invalid_operation
-andx276 and  100000010 080010100 ->  NaN Invalid_operation
-andx277 and  100000000 090000010 ->  NaN Invalid_operation
--- test LSD
-andx280 and  001000002 100000000 ->  NaN Invalid_operation
-andx281 and  000000007 100000000 ->  NaN Invalid_operation
-andx282 and  000000008 100000000 ->  NaN Invalid_operation
-andx283 and  000000009 100000000 ->  NaN Invalid_operation
-andx284 and  100000000 000100002 ->  NaN Invalid_operation
-andx285 and  100100000 001000007 ->  NaN Invalid_operation
-andx286 and  100010000 010000008 ->  NaN Invalid_operation
-andx287 and  100001000 100000009 ->  NaN Invalid_operation
--- test Middie
-andx288 and  001020000 100000000 ->  NaN Invalid_operation
-andx289 and  000070001 100000000 ->  NaN Invalid_operation
-andx290 and  000080000 100010000 ->  NaN Invalid_operation
-andx291 and  000090000 100001000 ->  NaN Invalid_operation
-andx292 and  100000010 000020100 ->  NaN Invalid_operation
-andx293 and  100100000 000070010 ->  NaN Invalid_operation
-andx294 and  100010100 000080001 ->  NaN Invalid_operation
-andx295 and  100001000 000090000 ->  NaN Invalid_operation
--- signs
-andx296 and -100001000 -000000000 ->  NaN Invalid_operation
-andx297 and -100001000  000010000 ->  NaN Invalid_operation
-andx298 and  100001000 -000000000 ->  NaN Invalid_operation
-andx299 and  100001000  000011000 ->  1000
-
--- Nmax, Nmin, Ntiny
-andx331 and  2   9.99999999E+999     -> NaN Invalid_operation
-andx332 and  3   1E-999              -> NaN Invalid_operation
-andx333 and  4   1.00000000E-999     -> NaN Invalid_operation
-andx334 and  5   1E-1007             -> NaN Invalid_operation
-andx335 and  6   -1E-1007            -> NaN Invalid_operation
-andx336 and  7   -1.00000000E-999    -> NaN Invalid_operation
-andx337 and  8   -1E-999             -> NaN Invalid_operation
-andx338 and  9   -9.99999999E+999    -> NaN Invalid_operation
-andx341 and  9.99999999E+999     -18 -> NaN Invalid_operation
-andx342 and  1E-999               01 -> NaN Invalid_operation
-andx343 and  1.00000000E-999     -18 -> NaN Invalid_operation
-andx344 and  1E-1007              18 -> NaN Invalid_operation
-andx345 and  -1E-1007            -10 -> NaN Invalid_operation
-andx346 and  -1.00000000E-999     18 -> NaN Invalid_operation
-andx347 and  -1E-999              10 -> NaN Invalid_operation
-andx348 and  -9.99999999E+999    -18 -> NaN Invalid_operation
-
--- A few other non-integers
-andx361 and  1.0                  1  -> NaN Invalid_operation
-andx362 and  1E+1                 1  -> NaN Invalid_operation
-andx363 and  0.0                  1  -> NaN Invalid_operation
-andx364 and  0E+1                 1  -> NaN Invalid_operation
-andx365 and  9.9                  1  -> NaN Invalid_operation
-andx366 and  9E+1                 1  -> NaN Invalid_operation
-andx371 and  0 1.0                   -> NaN Invalid_operation
-andx372 and  0 1E+1                  -> NaN Invalid_operation
-andx373 and  0 0.0                   -> NaN Invalid_operation
-andx374 and  0 0E+1                  -> NaN Invalid_operation
-andx375 and  0 9.9                   -> NaN Invalid_operation
-andx376 and  0 9E+1                  -> NaN Invalid_operation
-
--- All Specials are in error
-andx780 and -Inf  -Inf   -> NaN Invalid_operation
-andx781 and -Inf  -1000  -> NaN Invalid_operation
-andx782 and -Inf  -1     -> NaN Invalid_operation
-andx783 and -Inf  -0     -> NaN Invalid_operation
-andx784 and -Inf   0     -> NaN Invalid_operation
-andx785 and -Inf   1     -> NaN Invalid_operation
-andx786 and -Inf   1000  -> NaN Invalid_operation
-andx787 and -1000 -Inf   -> NaN Invalid_operation
-andx788 and -Inf  -Inf   -> NaN Invalid_operation
-andx789 and -1    -Inf   -> NaN Invalid_operation
-andx790 and -0    -Inf   -> NaN Invalid_operation
-andx791 and  0    -Inf   -> NaN Invalid_operation
-andx792 and  1    -Inf   -> NaN Invalid_operation
-andx793 and  1000 -Inf   -> NaN Invalid_operation
-andx794 and  Inf  -Inf   -> NaN Invalid_operation
-
-andx800 and  Inf  -Inf   -> NaN Invalid_operation
-andx801 and  Inf  -1000  -> NaN Invalid_operation
-andx802 and  Inf  -1     -> NaN Invalid_operation
-andx803 and  Inf  -0     -> NaN Invalid_operation
-andx804 and  Inf   0     -> NaN Invalid_operation
-andx805 and  Inf   1     -> NaN Invalid_operation
-andx806 and  Inf   1000  -> NaN Invalid_operation
-andx807 and  Inf   Inf   -> NaN Invalid_operation
-andx808 and -1000  Inf   -> NaN Invalid_operation
-andx809 and -Inf   Inf   -> NaN Invalid_operation
-andx810 and -1     Inf   -> NaN Invalid_operation
-andx811 and -0     Inf   -> NaN Invalid_operation
-andx812 and  0     Inf   -> NaN Invalid_operation
-andx813 and  1     Inf   -> NaN Invalid_operation
-andx814 and  1000  Inf   -> NaN Invalid_operation
-andx815 and  Inf   Inf   -> NaN Invalid_operation
-
-andx821 and  NaN -Inf    -> NaN Invalid_operation
-andx822 and  NaN -1000   -> NaN Invalid_operation
-andx823 and  NaN -1      -> NaN Invalid_operation
-andx824 and  NaN -0      -> NaN Invalid_operation
-andx825 and  NaN  0      -> NaN Invalid_operation
-andx826 and  NaN  1      -> NaN Invalid_operation
-andx827 and  NaN  1000   -> NaN Invalid_operation
-andx828 and  NaN  Inf    -> NaN Invalid_operation
-andx829 and  NaN  NaN    -> NaN Invalid_operation
-andx830 and -Inf  NaN    -> NaN Invalid_operation
-andx831 and -1000 NaN    -> NaN Invalid_operation
-andx832 and -1    NaN    -> NaN Invalid_operation
-andx833 and -0    NaN    -> NaN Invalid_operation
-andx834 and  0    NaN    -> NaN Invalid_operation
-andx835 and  1    NaN    -> NaN Invalid_operation
-andx836 and  1000 NaN    -> NaN Invalid_operation
-andx837 and  Inf  NaN    -> NaN Invalid_operation
-
-andx841 and  sNaN -Inf   ->  NaN  Invalid_operation
-andx842 and  sNaN -1000  ->  NaN  Invalid_operation
-andx843 and  sNaN -1     ->  NaN  Invalid_operation
-andx844 and  sNaN -0     ->  NaN  Invalid_operation
-andx845 and  sNaN  0     ->  NaN  Invalid_operation
-andx846 and  sNaN  1     ->  NaN  Invalid_operation
-andx847 and  sNaN  1000  ->  NaN  Invalid_operation
-andx848 and  sNaN  NaN   ->  NaN  Invalid_operation
-andx849 and  sNaN sNaN   ->  NaN  Invalid_operation
-andx850 and  NaN  sNaN   ->  NaN  Invalid_operation
-andx851 and -Inf  sNaN   ->  NaN  Invalid_operation
-andx852 and -1000 sNaN   ->  NaN  Invalid_operation
-andx853 and -1    sNaN   ->  NaN  Invalid_operation
-andx854 and -0    sNaN   ->  NaN  Invalid_operation
-andx855 and  0    sNaN   ->  NaN  Invalid_operation
-andx856 and  1    sNaN   ->  NaN  Invalid_operation
-andx857 and  1000 sNaN   ->  NaN  Invalid_operation
-andx858 and  Inf  sNaN   ->  NaN  Invalid_operation
-andx859 and  NaN  sNaN   ->  NaN  Invalid_operation
-
--- propagating NaNs
-andx861 and  NaN1   -Inf    -> NaN Invalid_operation
-andx862 and +NaN2   -1000   -> NaN Invalid_operation
-andx863 and  NaN3    1000   -> NaN Invalid_operation
-andx864 and  NaN4    Inf    -> NaN Invalid_operation
-andx865 and  NaN5   +NaN6   -> NaN Invalid_operation
-andx866 and -Inf     NaN7   -> NaN Invalid_operation
-andx867 and -1000    NaN8   -> NaN Invalid_operation
-andx868 and  1000    NaN9   -> NaN Invalid_operation
-andx869 and  Inf    +NaN10  -> NaN Invalid_operation
-andx871 and  sNaN11  -Inf   -> NaN Invalid_operation
-andx872 and  sNaN12  -1000  -> NaN Invalid_operation
-andx873 and  sNaN13   1000  -> NaN Invalid_operation
-andx874 and  sNaN14   NaN17 -> NaN Invalid_operation
-andx875 and  sNaN15  sNaN18 -> NaN Invalid_operation
-andx876 and  NaN16   sNaN19 -> NaN Invalid_operation
-andx877 and -Inf    +sNaN20 -> NaN Invalid_operation
-andx878 and -1000    sNaN21 -> NaN Invalid_operation
-andx879 and  1000    sNaN22 -> NaN Invalid_operation
-andx880 and  Inf     sNaN23 -> NaN Invalid_operation
-andx881 and +NaN25  +sNaN24 -> NaN Invalid_operation
-andx882 and -NaN26    NaN28 -> NaN Invalid_operation
-andx883 and -sNaN27  sNaN29 -> NaN Invalid_operation
-andx884 and  1000    -NaN30 -> NaN Invalid_operation
-andx885 and  1000   -sNaN31 -> NaN Invalid_operation
+------------------------------------------------------------------------
+-- and.decTest -- digitwise logical AND                               --
+-- Copyright (c) IBM Corporation, 1981, 2008.  All rights reserved.   --
+------------------------------------------------------------------------
+-- Please see the document "General Decimal Arithmetic Testcases"     --
+-- at http://www2.hursley.ibm.com/decimal for the description of      --
+-- these testcases.                                                   --
+--                                                                    --
+-- These testcases are experimental ('beta' versions), and they       --
+-- may contain errors.  They are offered on an as-is basis.  In       --
+-- particular, achieving the same results as the tests here is not    --
+-- a guarantee that an implementation complies with any Standard      --
+-- or specification.  The tests are not exhaustive.                   --
+--                                                                    --
+-- Please send comments, suggestions, and corrections to the author:  --
+--   Mike Cowlishaw, IBM Fellow                                       --
+--   IBM UK, PO Box 31, Birmingham Road, Warwick CV34 5JL, UK         --
+--   mfc at uk.ibm.com                                                   --
+------------------------------------------------------------------------
+version: 2.59
+
+extended:    1
+precision:   9
+rounding:    half_up
+maxExponent: 999
+minExponent: -999
+
+-- Sanity check (truth table)
+andx001 and             0    0 ->    0
+andx002 and             0    1 ->    0
+andx003 and             1    0 ->    0
+andx004 and             1    1 ->    1
+andx005 and          1100 1010 -> 1000
+andx006 and          1111   10 ->   10
+andx007 and          1111 1010 -> 1010
+
+-- and at msd and msd-1
+andx010 and 000000000 000000000 ->           0
+andx011 and 000000000 100000000 ->           0
+andx012 and 100000000 000000000 ->           0
+andx013 and 100000000 100000000 ->   100000000
+andx014 and 000000000 000000000 ->           0
+andx015 and 000000000 010000000 ->           0
+andx016 and 010000000 000000000 ->           0
+andx017 and 010000000 010000000 ->    10000000
+
+-- Various lengths
+--          123456789     123456789      123456789
+andx021 and 111111111     111111111  ->  111111111
+andx022 and 111111111111  111111111  ->  111111111
+andx023 and 111111111111   11111111  ->   11111111
+andx024 and 111111111      11111111  ->   11111111
+andx025 and 111111111       1111111  ->    1111111
+andx026 and 111111111111     111111  ->     111111
+andx027 and 111111111111      11111  ->      11111
+andx028 and 111111111111       1111  ->       1111
+andx029 and 111111111111        111  ->        111
+andx031 and 111111111111         11  ->         11
+andx032 and 111111111111          1  ->          1
+andx033 and 111111111111 1111111111  ->  111111111
+andx034 and 11111111111 11111111111  ->  111111111
+andx035 and 1111111111 111111111111  ->  111111111
+andx036 and 111111111 1111111111111  ->  111111111
+
+andx040 and 111111111  111111111111  ->  111111111
+andx041 and  11111111  111111111111  ->   11111111
+andx042 and  11111111     111111111  ->   11111111
+andx043 and   1111111     111111111  ->    1111111
+andx044 and    111111     111111111  ->     111111
+andx045 and     11111     111111111  ->      11111
+andx046 and      1111     111111111  ->       1111
+andx047 and       111     111111111  ->        111
+andx048 and        11     111111111  ->         11
+andx049 and         1     111111111  ->          1
+
+andx050 and 1111111111  1  ->  1
+andx051 and  111111111  1  ->  1
+andx052 and   11111111  1  ->  1
+andx053 and    1111111  1  ->  1
+andx054 and     111111  1  ->  1
+andx055 and      11111  1  ->  1
+andx056 and       1111  1  ->  1
+andx057 and        111  1  ->  1
+andx058 and         11  1  ->  1
+andx059 and          1  1  ->  1
+
+andx060 and 1111111111  0  ->  0
+andx061 and  111111111  0  ->  0
+andx062 and   11111111  0  ->  0
+andx063 and    1111111  0  ->  0
+andx064 and     111111  0  ->  0
+andx065 and      11111  0  ->  0
+andx066 and       1111  0  ->  0
+andx067 and        111  0  ->  0
+andx068 and         11  0  ->  0
+andx069 and          1  0  ->  0
+
+andx070 and 1  1111111111  ->  1
+andx071 and 1   111111111  ->  1
+andx072 and 1    11111111  ->  1
+andx073 and 1     1111111  ->  1
+andx074 and 1      111111  ->  1
+andx075 and 1       11111  ->  1
+andx076 and 1        1111  ->  1
+andx077 and 1         111  ->  1
+andx078 and 1          11  ->  1
+andx079 and 1           1  ->  1
+
+andx080 and 0  1111111111  ->  0
+andx081 and 0   111111111  ->  0
+andx082 and 0    11111111  ->  0
+andx083 and 0     1111111  ->  0
+andx084 and 0      111111  ->  0
+andx085 and 0       11111  ->  0
+andx086 and 0        1111  ->  0
+andx087 and 0         111  ->  0
+andx088 and 0          11  ->  0
+andx089 and 0           1  ->  0
+
+andx090 and 011111111  111111111  ->   11111111
+andx091 and 101111111  111111111  ->  101111111
+andx092 and 110111111  111111111  ->  110111111
+andx093 and 111011111  111111111  ->  111011111
+andx094 and 111101111  111111111  ->  111101111
+andx095 and 111110111  111111111  ->  111110111
+andx096 and 111111011  111111111  ->  111111011
+andx097 and 111111101  111111111  ->  111111101
+andx098 and 111111110  111111111  ->  111111110
+
+andx100 and 111111111  011111111  ->   11111111
+andx101 and 111111111  101111111  ->  101111111
+andx102 and 111111111  110111111  ->  110111111
+andx103 and 111111111  111011111  ->  111011111
+andx104 and 111111111  111101111  ->  111101111
+andx105 and 111111111  111110111  ->  111110111
+andx106 and 111111111  111111011  ->  111111011
+andx107 and 111111111  111111101  ->  111111101
+andx108 and 111111111  111111110  ->  111111110
+
+-- non-0/1 should not be accepted, nor should signs
+andx220 and 111111112  111111111  ->  NaN Invalid_operation
+andx221 and 333333333  333333333  ->  NaN Invalid_operation
+andx222 and 555555555  555555555  ->  NaN Invalid_operation
+andx223 and 777777777  777777777  ->  NaN Invalid_operation
+andx224 and 999999999  999999999  ->  NaN Invalid_operation
+andx225 and 222222222  999999999  ->  NaN Invalid_operation
+andx226 and 444444444  999999999  ->  NaN Invalid_operation
+andx227 and 666666666  999999999  ->  NaN Invalid_operation
+andx228 and 888888888  999999999  ->  NaN Invalid_operation
+andx229 and 999999999  222222222  ->  NaN Invalid_operation
+andx230 and 999999999  444444444  ->  NaN Invalid_operation
+andx231 and 999999999  666666666  ->  NaN Invalid_operation
+andx232 and 999999999  888888888  ->  NaN Invalid_operation
+-- a few randoms
+andx240 and  567468689 -934981942 ->  NaN Invalid_operation
+andx241 and  567367689  934981942 ->  NaN Invalid_operation
+andx242 and -631917772 -706014634 ->  NaN Invalid_operation
+andx243 and -756253257  138579234 ->  NaN Invalid_operation
+andx244 and  835590149  567435400 ->  NaN Invalid_operation
+-- test MSD
+andx250 and  200000000 100000000 ->  NaN Invalid_operation
+andx251 and  700000000 100000000 ->  NaN Invalid_operation
+andx252 and  800000000 100000000 ->  NaN Invalid_operation
+andx253 and  900000000 100000000 ->  NaN Invalid_operation
+andx254 and  200000000 000000000 ->  NaN Invalid_operation
+andx255 and  700000000 000000000 ->  NaN Invalid_operation
+andx256 and  800000000 000000000 ->  NaN Invalid_operation
+andx257 and  900000000 000000000 ->  NaN Invalid_operation
+andx258 and  100000000 200000000 ->  NaN Invalid_operation
+andx259 and  100000000 700000000 ->  NaN Invalid_operation
+andx260 and  100000000 800000000 ->  NaN Invalid_operation
+andx261 and  100000000 900000000 ->  NaN Invalid_operation
+andx262 and  000000000 200000000 ->  NaN Invalid_operation
+andx263 and  000000000 700000000 ->  NaN Invalid_operation
+andx264 and  000000000 800000000 ->  NaN Invalid_operation
+andx265 and  000000000 900000000 ->  NaN Invalid_operation
+-- test MSD-1
+andx270 and  020000000 100000000 ->  NaN Invalid_operation
+andx271 and  070100000 100000000 ->  NaN Invalid_operation
+andx272 and  080010000 100000001 ->  NaN Invalid_operation
+andx273 and  090001000 100000010 ->  NaN Invalid_operation
+andx274 and  100000100 020010100 ->  NaN Invalid_operation
+andx275 and  100000000 070001000 ->  NaN Invalid_operation
+andx276 and  100000010 080010100 ->  NaN Invalid_operation
+andx277 and  100000000 090000010 ->  NaN Invalid_operation
+-- test LSD
+andx280 and  001000002 100000000 ->  NaN Invalid_operation
+andx281 and  000000007 100000000 ->  NaN Invalid_operation
+andx282 and  000000008 100000000 ->  NaN Invalid_operation
+andx283 and  000000009 100000000 ->  NaN Invalid_operation
+andx284 and  100000000 000100002 ->  NaN Invalid_operation
+andx285 and  100100000 001000007 ->  NaN Invalid_operation
+andx286 and  100010000 010000008 ->  NaN Invalid_operation
+andx287 and  100001000 100000009 ->  NaN Invalid_operation
+-- test Middie
+andx288 and  001020000 100000000 ->  NaN Invalid_operation
+andx289 and  000070001 100000000 ->  NaN Invalid_operation
+andx290 and  000080000 100010000 ->  NaN Invalid_operation
+andx291 and  000090000 100001000 ->  NaN Invalid_operation
+andx292 and  100000010 000020100 ->  NaN Invalid_operation
+andx293 and  100100000 000070010 ->  NaN Invalid_operation
+andx294 and  100010100 000080001 ->  NaN Invalid_operation
+andx295 and  100001000 000090000 ->  NaN Invalid_operation
+-- signs
+andx296 and -100001000 -000000000 ->  NaN Invalid_operation
+andx297 and -100001000  000010000 ->  NaN Invalid_operation
+andx298 and  100001000 -000000000 ->  NaN Invalid_operation
+andx299 and  100001000  000011000 ->  1000
+
+-- Nmax, Nmin, Ntiny
+andx331 and  2   9.99999999E+999     -> NaN Invalid_operation
+andx332 and  3   1E-999              -> NaN Invalid_operation
+andx333 and  4   1.00000000E-999     -> NaN Invalid_operation
+andx334 and  5   1E-1007             -> NaN Invalid_operation
+andx335 and  6   -1E-1007            -> NaN Invalid_operation
+andx336 and  7   -1.00000000E-999    -> NaN Invalid_operation
+andx337 and  8   -1E-999             -> NaN Invalid_operation
+andx338 and  9   -9.99999999E+999    -> NaN Invalid_operation
+andx341 and  9.99999999E+999     -18 -> NaN Invalid_operation
+andx342 and  1E-999               01 -> NaN Invalid_operation
+andx343 and  1.00000000E-999     -18 -> NaN Invalid_operation
+andx344 and  1E-1007              18 -> NaN Invalid_operation
+andx345 and  -1E-1007            -10 -> NaN Invalid_operation
+andx346 and  -1.00000000E-999     18 -> NaN Invalid_operation
+andx347 and  -1E-999              10 -> NaN Invalid_operation
+andx348 and  -9.99999999E+999    -18 -> NaN Invalid_operation
+
+-- A few other non-integers
+andx361 and  1.0                  1  -> NaN Invalid_operation
+andx362 and  1E+1                 1  -> NaN Invalid_operation
+andx363 and  0.0                  1  -> NaN Invalid_operation
+andx364 and  0E+1                 1  -> NaN Invalid_operation
+andx365 and  9.9                  1  -> NaN Invalid_operation
+andx366 and  9E+1                 1  -> NaN Invalid_operation
+andx371 and  0 1.0                   -> NaN Invalid_operation
+andx372 and  0 1E+1                  -> NaN Invalid_operation
+andx373 and  0 0.0                   -> NaN Invalid_operation
+andx374 and  0 0E+1                  -> NaN Invalid_operation
+andx375 and  0 9.9                   -> NaN Invalid_operation
+andx376 and  0 9E+1                  -> NaN Invalid_operation
+
+-- All Specials are in error
+andx780 and -Inf  -Inf   -> NaN Invalid_operation
+andx781 and -Inf  -1000  -> NaN Invalid_operation
+andx782 and -Inf  -1     -> NaN Invalid_operation
+andx783 and -Inf  -0     -> NaN Invalid_operation
+andx784 and -Inf   0     -> NaN Invalid_operation
+andx785 and -Inf   1     -> NaN Invalid_operation
+andx786 and -Inf   1000  -> NaN Invalid_operation
+andx787 and -1000 -Inf   -> NaN Invalid_operation
+andx788 and -Inf  -Inf   -> NaN Invalid_operation
+andx789 and -1    -Inf   -> NaN Invalid_operation
+andx790 and -0    -Inf   -> NaN Invalid_operation
+andx791 and  0    -Inf   -> NaN Invalid_operation
+andx792 and  1    -Inf   -> NaN Invalid_operation
+andx793 and  1000 -Inf   -> NaN Invalid_operation
+andx794 and  Inf  -Inf   -> NaN Invalid_operation
+
+andx800 and  Inf  -Inf   -> NaN Invalid_operation
+andx801 and  Inf  -1000  -> NaN Invalid_operation
+andx802 and  Inf  -1     -> NaN Invalid_operation
+andx803 and  Inf  -0     -> NaN Invalid_operation
+andx804 and  Inf   0     -> NaN Invalid_operation
+andx805 and  Inf   1     -> NaN Invalid_operation
+andx806 and  Inf   1000  -> NaN Invalid_operation
+andx807 and  Inf   Inf   -> NaN Invalid_operation
+andx808 and -1000  Inf   -> NaN Invalid_operation
+andx809 and -Inf   Inf   -> NaN Invalid_operation
+andx810 and -1     Inf   -> NaN Invalid_operation
+andx811 and -0     Inf   -> NaN Invalid_operation
+andx812 and  0     Inf   -> NaN Invalid_operation
+andx813 and  1     Inf   -> NaN Invalid_operation
+andx814 and  1000  Inf   -> NaN Invalid_operation
+andx815 and  Inf   Inf   -> NaN Invalid_operation
+
+andx821 and  NaN -Inf    -> NaN Invalid_operation
+andx822 and  NaN -1000   -> NaN Invalid_operation
+andx823 and  NaN -1      -> NaN Invalid_operation
+andx824 and  NaN -0      -> NaN Invalid_operation
+andx825 and  NaN  0      -> NaN Invalid_operation
+andx826 and  NaN  1      -> NaN Invalid_operation
+andx827 and  NaN  1000   -> NaN Invalid_operation
+andx828 and  NaN  Inf    -> NaN Invalid_operation
+andx829 and  NaN  NaN    -> NaN Invalid_operation
+andx830 and -Inf  NaN    -> NaN Invalid_operation
+andx831 and -1000 NaN    -> NaN Invalid_operation
+andx832 and -1    NaN    -> NaN Invalid_operation
+andx833 and -0    NaN    -> NaN Invalid_operation
+andx834 and  0    NaN    -> NaN Invalid_operation
+andx835 and  1    NaN    -> NaN Invalid_operation
+andx836 and  1000 NaN    -> NaN Invalid_operation
+andx837 and  Inf  NaN    -> NaN Invalid_operation
+
+andx841 and  sNaN -Inf   ->  NaN  Invalid_operation
+andx842 and  sNaN -1000  ->  NaN  Invalid_operation
+andx843 and  sNaN -1     ->  NaN  Invalid_operation
+andx844 and  sNaN -0     ->  NaN  Invalid_operation
+andx845 and  sNaN  0     ->  NaN  Invalid_operation
+andx846 and  sNaN  1     ->  NaN  Invalid_operation
+andx847 and  sNaN  1000  ->  NaN  Invalid_operation
+andx848 and  sNaN  NaN   ->  NaN  Invalid_operation
+andx849 and  sNaN sNaN   ->  NaN  Invalid_operation
+andx850 and  NaN  sNaN   ->  NaN  Invalid_operation
+andx851 and -Inf  sNaN   ->  NaN  Invalid_operation
+andx852 and -1000 sNaN   ->  NaN  Invalid_operation
+andx853 and -1    sNaN   ->  NaN  Invalid_operation
+andx854 and -0    sNaN   ->  NaN  Invalid_operation
+andx855 and  0    sNaN   ->  NaN  Invalid_operation
+andx856 and  1    sNaN   ->  NaN  Invalid_operation
+andx857 and  1000 sNaN   ->  NaN  Invalid_operation
+andx858 and  Inf  sNaN   ->  NaN  Invalid_operation
+andx859 and  NaN  sNaN   ->  NaN  Invalid_operation
+
+-- propagating NaNs
+andx861 and  NaN1   -Inf    -> NaN Invalid_operation
+andx862 and +NaN2   -1000   -> NaN Invalid_operation
+andx863 and  NaN3    1000   -> NaN Invalid_operation
+andx864 and  NaN4    Inf    -> NaN Invalid_operation
+andx865 and  NaN5   +NaN6   -> NaN Invalid_operation
+andx866 and -Inf     NaN7   -> NaN Invalid_operation
+andx867 and -1000    NaN8   -> NaN Invalid_operation
+andx868 and  1000    NaN9   -> NaN Invalid_operation
+andx869 and  Inf    +NaN10  -> NaN Invalid_operation
+andx871 and  sNaN11  -Inf   -> NaN Invalid_operation
+andx872 and  sNaN12  -1000  -> NaN Invalid_operation
+andx873 and  sNaN13   1000  -> NaN Invalid_operation
+andx874 and  sNaN14   NaN17 -> NaN Invalid_operation
+andx875 and  sNaN15  sNaN18 -> NaN Invalid_operation
+andx876 and  NaN16   sNaN19 -> NaN Invalid_operation
+andx877 and -Inf    +sNaN20 -> NaN Invalid_operation
+andx878 and -1000    sNaN21 -> NaN Invalid_operation
+andx879 and  1000    sNaN22 -> NaN Invalid_operation
+andx880 and  Inf     sNaN23 -> NaN Invalid_operation
+andx881 and +NaN25  +sNaN24 -> NaN Invalid_operation
+andx882 and -NaN26    NaN28 -> NaN Invalid_operation
+andx883 and -sNaN27  sNaN29 -> NaN Invalid_operation
+andx884 and  1000    -NaN30 -> NaN Invalid_operation
+andx885 and  1000   -sNaN31 -> NaN Invalid_operation
diff --git a/lib-python/2.7/test/decimaltestdata/class.decTest b/lib-python/2.7/test/decimaltestdata/class.decTest
--- a/lib-python/2.7/test/decimaltestdata/class.decTest
+++ b/lib-python/2.7/test/decimaltestdata/class.decTest
@@ -1,131 +1,131 @@
-------------------------------------------------------------------------
--- class.decTest -- Class operations                                  --
--- Copyright (c) IBM Corporation, 1981, 2008.  All rights reserved.   --
-------------------------------------------------------------------------
--- Please see the document "General Decimal Arithmetic Testcases"     --
--- at http://www2.hursley.ibm.com/decimal for the description of      --
--- these testcases.                                                   --
---                                                                    --
--- These testcases are experimental ('beta' versions), and they       --
--- may contain errors.  They are offered on an as-is basis.  In       --
--- particular, achieving the same results as the tests here is not    --
--- a guarantee that an implementation complies with any Standard      --
--- or specification.  The tests are not exhaustive.                   --
---                                                                    --
--- Please send comments, suggestions, and corrections to the author:  --
---   Mike Cowlishaw, IBM Fellow                                       --
---   IBM UK, PO Box 31, Birmingham Road, Warwick CV34 5JL, UK         --
---   mfc at uk.ibm.com                                                   --
-------------------------------------------------------------------------
-version: 2.59
-
--- [New 2006.11.27]
-
-precision:   9
-maxExponent: 999
-minExponent: -999
-extended:    1
-clamp:       1
-rounding:    half_even
-
-clasx001  class    0                        -> +Zero
-clasx002  class    0.00                     -> +Zero
-clasx003  class    0E+5                     -> +Zero
-clasx004  class    1E-1007                  -> +Subnormal
-clasx005  class  0.1E-999                   -> +Subnormal
-clasx006  class  0.99999999E-999            -> +Subnormal
-clasx007  class  1.00000000E-999            -> +Normal
-clasx008  class   1E-999                    -> +Normal
-clasx009  class   1E-100                    -> +Normal
-clasx010  class   1E-10                     -> +Normal
-clasx012  class   1E-1                      -> +Normal
-clasx013  class   1                         -> +Normal
-clasx014  class   2.50                      -> +Normal
-clasx015  class   100.100                   -> +Normal
-clasx016  class   1E+30                     -> +Normal
-clasx017  class   1E+999                    -> +Normal
-clasx018  class   9.99999999E+999           -> +Normal
-clasx019  class   Inf                       -> +Infinity
-
-clasx021  class   -0                        -> -Zero
-clasx022  class   -0.00                     -> -Zero
-clasx023  class   -0E+5                     -> -Zero
-clasx024  class   -1E-1007                  -> -Subnormal
-clasx025  class  -0.1E-999                  -> -Subnormal
-clasx026  class  -0.99999999E-999           -> -Subnormal
-clasx027  class  -1.00000000E-999           -> -Normal
-clasx028  class  -1E-999                    -> -Normal
-clasx029  class  -1E-100                    -> -Normal
-clasx030  class  -1E-10                     -> -Normal
-clasx032  class  -1E-1                      -> -Normal
-clasx033  class  -1                         -> -Normal
-clasx034  class  -2.50                      -> -Normal
-clasx035  class  -100.100                   -> -Normal
-clasx036  class  -1E+30                     -> -Normal
-clasx037  class  -1E+999                    -> -Normal
-clasx038  class  -9.99999999E+999           -> -Normal
-clasx039  class  -Inf                       -> -Infinity
-
-clasx041  class   NaN                       -> NaN
-clasx042  class  -NaN                       -> NaN
-clasx043  class  +NaN12345                  -> NaN
-clasx044  class   sNaN                      -> sNaN
-clasx045  class  -sNaN                      -> sNaN
-clasx046  class  +sNaN12345                 -> sNaN
-
-
--- decimal64 bounds
-
-precision:   16
-maxExponent: 384
-minExponent: -383
-clamp:       1
-rounding:    half_even
-
-clasx201  class    0                        -> +Zero
-clasx202  class    0.00                     -> +Zero
-clasx203  class    0E+5                     -> +Zero
-clasx204  class    1E-396                   -> +Subnormal
-clasx205  class  0.1E-383                   -> +Subnormal
-clasx206  class  0.999999999999999E-383     -> +Subnormal
-clasx207  class  1.000000000000000E-383     -> +Normal
-clasx208  class   1E-383                    -> +Normal
-clasx209  class   1E-100                    -> +Normal
-clasx210  class   1E-10                     -> +Normal
-clasx212  class   1E-1                      -> +Normal
-clasx213  class   1                         -> +Normal
-clasx214  class   2.50                      -> +Normal
-clasx215  class   100.100                   -> +Normal
-clasx216  class   1E+30                     -> +Normal
-clasx217  class   1E+384                    -> +Normal
-clasx218  class   9.999999999999999E+384    -> +Normal
-clasx219  class   Inf                       -> +Infinity
-
-clasx221  class   -0                        -> -Zero
-clasx222  class   -0.00                     -> -Zero
-clasx223  class   -0E+5                     -> -Zero
-clasx224  class   -1E-396                   -> -Subnormal
-clasx225  class  -0.1E-383                  -> -Subnormal
-clasx226  class  -0.999999999999999E-383    -> -Subnormal
-clasx227  class  -1.000000000000000E-383    -> -Normal
-clasx228  class  -1E-383                    -> -Normal
-clasx229  class  -1E-100                    -> -Normal
-clasx230  class  -1E-10                     -> -Normal
-clasx232  class  -1E-1                      -> -Normal
-clasx233  class  -1                         -> -Normal
-clasx234  class  -2.50                      -> -Normal
-clasx235  class  -100.100                   -> -Normal
-clasx236  class  -1E+30                     -> -Normal
-clasx237  class  -1E+384                    -> -Normal
-clasx238  class  -9.999999999999999E+384    -> -Normal
-clasx239  class  -Inf                       -> -Infinity
-
-clasx241  class   NaN                       -> NaN
-clasx242  class  -NaN                       -> NaN
-clasx243  class  +NaN12345                  -> NaN
-clasx244  class   sNaN                      -> sNaN
-clasx245  class  -sNaN                      -> sNaN
-clasx246  class  +sNaN12345                 -> sNaN
-
-
-
+------------------------------------------------------------------------
+-- class.decTest -- Class operations                                  --
+-- Copyright (c) IBM Corporation, 1981, 2008.  All rights reserved.   --
+------------------------------------------------------------------------
+-- Please see the document "General Decimal Arithmetic Testcases"     --
+-- at http://www2.hursley.ibm.com/decimal for the description of      --
+-- these testcases.                                                   --
+--                                                                    --
+-- These testcases are experimental ('beta' versions), and they       --
+-- may contain errors.  They are offered on an as-is basis.  In       --
+-- particular, achieving the same results as the tests here is not    --
+-- a guarantee that an implementation complies with any Standard      --
+-- or specification.  The tests are not exhaustive.                   --
+--                                                                    --
+-- Please send comments, suggestions, and corrections to the author:  --
+--   Mike Cowlishaw, IBM Fellow                                       --
+--   IBM UK, PO Box 31, Birmingham Road, Warwick CV34 5JL, UK         --
+--   mfc at uk.ibm.com                                                   --
+------------------------------------------------------------------------
+version: 2.59
+
+-- [New 2006.11.27]
+
+precision:   9
+maxExponent: 999
+minExponent: -999
+extended:    1
+clamp:       1
+rounding:    half_even
+
+clasx001  class    0                        -> +Zero
+clasx002  class    0.00                     -> +Zero
+clasx003  class    0E+5                     -> +Zero
+clasx004  class    1E-1007                  -> +Subnormal
+clasx005  class  0.1E-999                   -> +Subnormal
+clasx006  class  0.99999999E-999            -> +Subnormal
+clasx007  class  1.00000000E-999            -> +Normal
+clasx008  class   1E-999                    -> +Normal
+clasx009  class   1E-100                    -> +Normal
+clasx010  class   1E-10                     -> +Normal
+clasx012  class   1E-1                      -> +Normal
+clasx013  class   1                         -> +Normal
+clasx014  class   2.50                      -> +Normal
+clasx015  class   100.100                   -> +Normal
+clasx016  class   1E+30                     -> +Normal
+clasx017  class   1E+999                    -> +Normal
+clasx018  class   9.99999999E+999           -> +Normal
+clasx019  class   Inf                       -> +Infinity
+
+clasx021  class   -0                        -> -Zero
+clasx022  class   -0.00                     -> -Zero
+clasx023  class   -0E+5                     -> -Zero
+clasx024  class   -1E-1007                  -> -Subnormal
+clasx025  class  -0.1E-999                  -> -Subnormal
+clasx026  class  -0.99999999E-999           -> -Subnormal
+clasx027  class  -1.00000000E-999           -> -Normal
+clasx028  class  -1E-999                    -> -Normal
+clasx029  class  -1E-100                    -> -Normal
+clasx030  class  -1E-10                     -> -Normal
+clasx032  class  -1E-1                      -> -Normal
+clasx033  class  -1                         -> -Normal
+clasx034  class  -2.50                      -> -Normal
+clasx035  class  -100.100                   -> -Normal
+clasx036  class  -1E+30                     -> -Normal
+clasx037  class  -1E+999                    -> -Normal
+clasx038  class  -9.99999999E+999           -> -Normal
+clasx039  class  -Inf                       -> -Infinity
+
+clasx041  class   NaN                       -> NaN
+clasx042  class  -NaN                       -> NaN
+clasx043  class  +NaN12345                  -> NaN
+clasx044  class   sNaN                      -> sNaN
+clasx045  class  -sNaN                      -> sNaN
+clasx046  class  +sNaN12345                 -> sNaN
+
+
+-- decimal64 bounds
+
+precision:   16
+maxExponent: 384
+minExponent: -383
+clamp:       1
+rounding:    half_even
+
+clasx201  class    0                        -> +Zero
+clasx202  class    0.00                     -> +Zero
+clasx203  class    0E+5                     -> +Zero
+clasx204  class    1E-396                   -> +Subnormal
+clasx205  class  0.1E-383                   -> +Subnormal
+clasx206  class  0.999999999999999E-383     -> +Subnormal
+clasx207  class  1.000000000000000E-383     -> +Normal
+clasx208  class   1E-383                    -> +Normal
+clasx209  class   1E-100                    -> +Normal
+clasx210  class   1E-10                     -> +Normal
+clasx212  class   1E-1                      -> +Normal
+clasx213  class   1                         -> +Normal
+clasx214  class   2.50                      -> +Normal
+clasx215  class   100.100                   -> +Normal
+clasx216  class   1E+30                     -> +Normal
+clasx217  class   1E+384                    -> +Normal
+clasx218  class   9.999999999999999E+384    -> +Normal
+clasx219  class   Inf                       -> +Infinity
+
+clasx221  class   -0                        -> -Zero
+clasx222  class   -0.00                     -> -Zero
+clasx223  class   -0E+5                     -> -Zero
+clasx224  class   -1E-396                   -> -Subnormal
+clasx225  class  -0.1E-383                  -> -Subnormal
+clasx226  class  -0.999999999999999E-383    -> -Subnormal
+clasx227  class  -1.000000000000000E-383    -> -Normal
+clasx228  class  -1E-383                    -> -Normal
+clasx229  class  -1E-100                    -> -Normal
+clasx230  class  -1E-10                     -> -Normal
+clasx232  class  -1E-1                      -> -Normal
+clasx233  class  -1                         -> -Normal
+clasx234  class  -2.50                      -> -Normal
+clasx235  class  -100.100                   -> -Normal
+clasx236  class  -1E+30                     -> -Normal
+clasx237  class  -1E+384                    -> -Normal
+clasx238  class  -9.999999999999999E+384    -> -Normal
+clasx239  class  -Inf                       -> -Infinity
+
+clasx241  class   NaN                       -> NaN
+clasx242  class  -NaN                       -> NaN
+clasx243  class  +NaN12345                  -> NaN
+clasx244  class   sNaN                      -> sNaN
+clasx245  class  -sNaN                      -> sNaN
+clasx246  class  +sNaN12345                 -> sNaN
+
+
+
diff --git a/lib-python/2.7/test/decimaltestdata/comparetotal.decTest b/lib-python/2.7/test/decimaltestdata/comparetotal.decTest
--- a/lib-python/2.7/test/decimaltestdata/comparetotal.decTest
+++ b/lib-python/2.7/test/decimaltestdata/comparetotal.decTest
@@ -1,798 +1,798 @@
-------------------------------------------------------------------------
--- comparetotal.decTest -- decimal comparison using total ordering    --
--- Copyright (c) IBM Corporation, 1981, 2008.  All rights reserved.   --
-------------------------------------------------------------------------
--- Please see the document "General Decimal Arithmetic Testcases"     --
--- at http://www2.hursley.ibm.com/decimal for the description of      --
--- these testcases.                                                   --
---                                                                    --
--- These testcases are experimental ('beta' versions), and they       --
--- may contain errors.  They are offered on an as-is basis.  In       --
--- particular, achieving the same results as the tests here is not    --
--- a guarantee that an implementation complies with any Standard      --
--- or specification.  The tests are not exhaustive.                   --
---                                                                    --
--- Please send comments, suggestions, and corrections to the author:  --
---   Mike Cowlishaw, IBM Fellow                                       --
---   IBM UK, PO Box 31, Birmingham Road, Warwick CV34 5JL, UK         --
---   mfc at uk.ibm.com                                                   --
-------------------------------------------------------------------------
-version: 2.59
-
--- Note that we cannot assume add/subtract tests cover paths adequately,
--- here, because the code might be quite different (comparison cannot
--- overflow or underflow, so actual subtractions are not necessary).
--- Similarly, comparetotal will have some radically different paths
--- than compare.
-
-extended:    1
-precision:   16
-rounding:    half_up
-maxExponent: 384
-minExponent: -383
-
--- sanity checks
-cotx001 comparetotal  -2  -2  -> 0
-cotx002 comparetotal  -2  -1  -> -1
-cotx003 comparetotal  -2   0  -> -1
-cotx004 comparetotal  -2   1  -> -1
-cotx005 comparetotal  -2   2  -> -1
-cotx006 comparetotal  -1  -2  -> 1
-cotx007 comparetotal  -1  -1  -> 0
-cotx008 comparetotal  -1   0  -> -1
-cotx009 comparetotal  -1   1  -> -1
-cotx010 comparetotal  -1   2  -> -1
-cotx011 comparetotal   0  -2  -> 1
-cotx012 comparetotal   0  -1  -> 1
-cotx013 comparetotal   0   0  -> 0
-cotx014 comparetotal   0   1  -> -1
-cotx015 comparetotal   0   2  -> -1
-cotx016 comparetotal   1  -2  -> 1
-cotx017 comparetotal   1  -1  -> 1
-cotx018 comparetotal   1   0  -> 1
-cotx019 comparetotal   1   1  -> 0
-cotx020 comparetotal   1   2  -> -1
-cotx021 comparetotal   2  -2  -> 1
-cotx022 comparetotal   2  -1  -> 1
-cotx023 comparetotal   2   0  -> 1
-cotx025 comparetotal   2   1  -> 1
-cotx026 comparetotal   2   2  -> 0
-
-cotx031 comparetotal  -20  -20  -> 0
-cotx032 comparetotal  -20  -10  -> -1
-cotx033 comparetotal  -20   00  -> -1
-cotx034 comparetotal  -20   10  -> -1
-cotx035 comparetotal  -20   20  -> -1
-cotx036 comparetotal  -10  -20  -> 1
-cotx037 comparetotal  -10  -10  -> 0
-cotx038 comparetotal  -10   00  -> -1
-cotx039 comparetotal  -10   10  -> -1
-cotx040 comparetotal  -10   20  -> -1
-cotx041 comparetotal   00  -20  -> 1
-cotx042 comparetotal   00  -10  -> 1
-cotx043 comparetotal   00   00  -> 0
-cotx044 comparetotal   00   10  -> -1
-cotx045 comparetotal   00   20  -> -1
-cotx046 comparetotal   10  -20  -> 1
-cotx047 comparetotal   10  -10  -> 1
-cotx048 comparetotal   10   00  -> 1
-cotx049 comparetotal   10   10  -> 0
-cotx050 comparetotal   10   20  -> -1
-cotx051 comparetotal   20  -20  -> 1
-cotx052 comparetotal   20  -10  -> 1
-cotx053 comparetotal   20   00  -> 1
-cotx055 comparetotal   20   10  -> 1
-cotx056 comparetotal   20   20  -> 0
-
-cotx061 comparetotal  -2.0  -2.0  -> 0
-cotx062 comparetotal  -2.0  -1.0  -> -1
-cotx063 comparetotal  -2.0   0.0  -> -1
-cotx064 comparetotal  -2.0   1.0  -> -1
-cotx065 comparetotal  -2.0   2.0  -> -1
-cotx066 comparetotal  -1.0  -2.0  -> 1
-cotx067 comparetotal  -1.0  -1.0  -> 0
-cotx068 comparetotal  -1.0   0.0  -> -1
-cotx069 comparetotal  -1.0   1.0  -> -1
-cotx070 comparetotal  -1.0   2.0  -> -1
-cotx071 comparetotal   0.0  -2.0  -> 1
-cotx072 comparetotal   0.0  -1.0  -> 1
-cotx073 comparetotal   0.0   0.0  -> 0
-cotx074 comparetotal   0.0   1.0  -> -1
-cotx075 comparetotal   0.0   2.0  -> -1
-cotx076 comparetotal   1.0  -2.0  -> 1
-cotx077 comparetotal   1.0  -1.0  -> 1
-cotx078 comparetotal   1.0   0.0  -> 1
-cotx079 comparetotal   1.0   1.0  -> 0
-cotx080 comparetotal   1.0   2.0  -> -1
-cotx081 comparetotal   2.0  -2.0  -> 1
-cotx082 comparetotal   2.0  -1.0  -> 1
-cotx083 comparetotal   2.0   0.0  -> 1
-cotx085 comparetotal   2.0   1.0  -> 1
-cotx086 comparetotal   2.0   2.0  -> 0
-
--- now some cases which might overflow if subtract were used
-maxexponent: 999999999
-minexponent: -999999999
-cotx090 comparetotal  9.99999999E+999999999 9.99999999E+999999999  -> 0
-cotx091 comparetotal -9.99999999E+999999999 9.99999999E+999999999  -> -1
-cotx092 comparetotal  9.99999999E+999999999 -9.99999999E+999999999 -> 1
-cotx093 comparetotal -9.99999999E+999999999 -9.99999999E+999999999 -> 0
-
--- Examples
-cotx094 comparetotal  12.73  127.9  -> -1
-cotx095 comparetotal  -127   12     -> -1
-cotx096 comparetotal  12.30  12.3   -> -1
-cotx097 comparetotal  12.30  12.30  ->  0
-cotx098 comparetotal  12.3   12.300 ->  1
-cotx099 comparetotal  12.3   NaN    -> -1
-
--- some differing length/exponent cases
--- in this first group, compare would compare all equal
-cotx100 comparetotal   7.0    7.0    -> 0
-cotx101 comparetotal   7.0    7      -> -1
-cotx102 comparetotal   7      7.0    -> 1
-cotx103 comparetotal   7E+0   7.0    -> 1
-cotx104 comparetotal   70E-1  7.0    -> 0
-cotx105 comparetotal   0.7E+1 7      -> 0
-cotx106 comparetotal   70E-1  7      -> -1
-cotx107 comparetotal   7.0    7E+0   -> -1
-cotx108 comparetotal   7.0    70E-1  -> 0
-cotx109 comparetotal   7      0.7E+1 -> 0
-cotx110 comparetotal   7      70E-1  -> 1
-
-cotx120 comparetotal   8.0    7.0    -> 1
-cotx121 comparetotal   8.0    7      -> 1
-cotx122 comparetotal   8      7.0    -> 1
-cotx123 comparetotal   8E+0   7.0    -> 1
-cotx124 comparetotal   80E-1  7.0    -> 1
-cotx125 comparetotal   0.8E+1 7      -> 1
-cotx126 comparetotal   80E-1  7      -> 1
-cotx127 comparetotal   8.0    7E+0   -> 1
-cotx128 comparetotal   8.0    70E-1  -> 1
-cotx129 comparetotal   8      0.7E+1  -> 1
-cotx130 comparetotal   8      70E-1  -> 1
-
-cotx140 comparetotal   8.0    9.0    -> -1
-cotx141 comparetotal   8.0    9      -> -1
-cotx142 comparetotal   8      9.0    -> -1
-cotx143 comparetotal   8E+0   9.0    -> -1
-cotx144 comparetotal   80E-1  9.0    -> -1
-cotx145 comparetotal   0.8E+1 9      -> -1
-cotx146 comparetotal   80E-1  9      -> -1
-cotx147 comparetotal   8.0    9E+0   -> -1
-cotx148 comparetotal   8.0    90E-1  -> -1
-cotx149 comparetotal   8      0.9E+1 -> -1
-cotx150 comparetotal   8      90E-1  -> -1
-
--- and again, with sign changes -+ ..
-cotx200 comparetotal  -7.0    7.0    -> -1
-cotx201 comparetotal  -7.0    7      -> -1
-cotx202 comparetotal  -7      7.0    -> -1
-cotx203 comparetotal  -7E+0   7.0    -> -1
-cotx204 comparetotal  -70E-1  7.0    -> -1
-cotx205 comparetotal  -0.7E+1 7      -> -1
-cotx206 comparetotal  -70E-1  7      -> -1
-cotx207 comparetotal  -7.0    7E+0   -> -1
-cotx208 comparetotal  -7.0    70E-1  -> -1
-cotx209 comparetotal  -7      0.7E+1 -> -1
-cotx210 comparetotal  -7      70E-1  -> -1
-
-cotx220 comparetotal  -8.0    7.0    -> -1
-cotx221 comparetotal  -8.0    7      -> -1
-cotx222 comparetotal  -8      7.0    -> -1
-cotx223 comparetotal  -8E+0   7.0    -> -1
-cotx224 comparetotal  -80E-1  7.0    -> -1
-cotx225 comparetotal  -0.8E+1 7      -> -1
-cotx226 comparetotal  -80E-1  7      -> -1
-cotx227 comparetotal  -8.0    7E+0   -> -1
-cotx228 comparetotal  -8.0    70E-1  -> -1
-cotx229 comparetotal  -8      0.7E+1 -> -1
-cotx230 comparetotal  -8      70E-1  -> -1
-
-cotx240 comparetotal  -8.0    9.0    -> -1
-cotx241 comparetotal  -8.0    9      -> -1
-cotx242 comparetotal  -8      9.0    -> -1
-cotx243 comparetotal  -8E+0   9.0    -> -1
-cotx244 comparetotal  -80E-1  9.0    -> -1
-cotx245 comparetotal  -0.8E+1 9      -> -1
-cotx246 comparetotal  -80E-1  9      -> -1
-cotx247 comparetotal  -8.0    9E+0   -> -1
-cotx248 comparetotal  -8.0    90E-1  -> -1
-cotx249 comparetotal  -8      0.9E+1 -> -1
-cotx250 comparetotal  -8      90E-1  -> -1
-
--- and again, with sign changes +- ..
-cotx300 comparetotal   7.0    -7.0    -> 1
-cotx301 comparetotal   7.0    -7      -> 1
-cotx302 comparetotal   7      -7.0    -> 1
-cotx303 comparetotal   7E+0   -7.0    -> 1
-cotx304 comparetotal   70E-1  -7.0    -> 1
-cotx305 comparetotal   .7E+1  -7      -> 1
-cotx306 comparetotal   70E-1  -7      -> 1
-cotx307 comparetotal   7.0    -7E+0   -> 1
-cotx308 comparetotal   7.0    -70E-1  -> 1
-cotx309 comparetotal   7      -.7E+1  -> 1
-cotx310 comparetotal   7      -70E-1  -> 1
-
-cotx320 comparetotal   8.0    -7.0    -> 1
-cotx321 comparetotal   8.0    -7      -> 1
-cotx322 comparetotal   8      -7.0    -> 1
-cotx323 comparetotal   8E+0   -7.0    -> 1
-cotx324 comparetotal   80E-1  -7.0    -> 1
-cotx325 comparetotal   .8E+1  -7      -> 1
-cotx326 comparetotal   80E-1  -7      -> 1
-cotx327 comparetotal   8.0    -7E+0   -> 1
-cotx328 comparetotal   8.0    -70E-1  -> 1
-cotx329 comparetotal   8      -.7E+1  -> 1
-cotx330 comparetotal   8      -70E-1  -> 1
-
-cotx340 comparetotal   8.0    -9.0    -> 1
-cotx341 comparetotal   8.0    -9      -> 1
-cotx342 comparetotal   8      -9.0    -> 1
-cotx343 comparetotal   8E+0   -9.0    -> 1
-cotx344 comparetotal   80E-1  -9.0    -> 1
-cotx345 comparetotal   .8E+1  -9      -> 1
-cotx346 comparetotal   80E-1  -9      -> 1
-cotx347 comparetotal   8.0    -9E+0   -> 1
-cotx348 comparetotal   8.0    -90E-1  -> 1
-cotx349 comparetotal   8      -.9E+1  -> 1
-cotx350 comparetotal   8      -90E-1  -> 1
-
--- and again, with sign changes -- ..
-cotx400 comparetotal   -7.0    -7.0    -> 0
-cotx401 comparetotal   -7.0    -7      -> 1
-cotx402 comparetotal   -7      -7.0    -> -1
-cotx403 comparetotal   -7E+0   -7.0    -> -1
-cotx404 comparetotal   -70E-1  -7.0    -> 0
-cotx405 comparetotal   -.7E+1  -7      -> 0
-cotx406 comparetotal   -70E-1  -7      -> 1
-cotx407 comparetotal   -7.0    -7E+0   -> 1
-cotx408 comparetotal   -7.0    -70E-1  -> 0
-cotx409 comparetotal   -7      -.7E+1  -> 0

From noreply at buildbot.pypy.org  Sun Jan 22 01:02:21 2012
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sun, 22 Jan 2012 01:02:21 +0100 (CET)
Subject: [pypy-commit] pypy merge-2.7.2: Merge 2.7/ recent changes to
	modified-2.7/
Message-ID: <20120122000221.6086E82D45@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: merge-2.7.2
Changeset: r51623:725c5aa57558
Date: 2012-01-22 01:00 +0100
http://bitbucket.org/pypy/pypy/changeset/725c5aa57558/

Log:	Merge 2.7/ recent changes to modified-2.7/

diff --git a/lib-python/modified-2.7/ctypes/test/test_arrays.py b/lib-python/modified-2.7/ctypes/test/test_arrays.py
--- a/lib-python/modified-2.7/ctypes/test/test_arrays.py
+++ b/lib-python/modified-2.7/ctypes/test/test_arrays.py
@@ -37,7 +37,7 @@
             values = [ia[i] for i in range(len(init))]
             self.assertEqual(values, [0] * len(init))
 
-            # Too many in itializers should be caught
+            # Too many initializers should be caught
             self.assertRaises(IndexError, int_array, *range(alen*2))
 
         CharArray = ARRAY(c_char, 3)
diff --git a/lib-python/modified-2.7/ctypes/test/test_as_parameter.py b/lib-python/modified-2.7/ctypes/test/test_as_parameter.py
--- a/lib-python/modified-2.7/ctypes/test/test_as_parameter.py
+++ b/lib-python/modified-2.7/ctypes/test/test_as_parameter.py
@@ -187,6 +187,18 @@
         self.assertEqual((s8i.a, s8i.b, s8i.c, s8i.d, s8i.e, s8i.f, s8i.g, s8i.h),
                              (9*2, 8*3, 7*4, 6*5, 5*6, 4*7, 3*8, 2*9))
 
+    def test_recursive_as_param(self):
+        from ctypes import c_int
+
+        class A(object):
+            pass
+
+        a = A()
+        a._as_parameter_ = a
+        with self.assertRaises(RuntimeError):
+            c_int.from_param(a)
+
+
 #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 class AsParamWrapper(object):
diff --git a/lib-python/modified-2.7/ctypes/test/test_callbacks.py b/lib-python/modified-2.7/ctypes/test/test_callbacks.py
--- a/lib-python/modified-2.7/ctypes/test/test_callbacks.py
+++ b/lib-python/modified-2.7/ctypes/test/test_callbacks.py
@@ -208,6 +208,42 @@
 
             windll.user32.EnumWindows(EnumWindowsCallbackFunc, 0)
 
+    def test_callback_register_int(self):
+        # Issue #8275: buggy handling of callback args under Win64
+        # NOTE: should be run on release builds as well
+        dll = CDLL(_ctypes_test.__file__)
+        CALLBACK = CFUNCTYPE(c_int, c_int, c_int, c_int, c_int, c_int)
+        # All this function does is call the callback with its args squared
+        func = dll._testfunc_cbk_reg_int
+        func.argtypes = (c_int, c_int, c_int, c_int, c_int, CALLBACK)
+        func.restype = c_int
+
+        def callback(a, b, c, d, e):
+            return a + b + c + d + e
+
+        result = func(2, 3, 4, 5, 6, CALLBACK(callback))
+        self.assertEqual(result, callback(2*2, 3*3, 4*4, 5*5, 6*6))
+
+    def test_callback_register_double(self):
+        # Issue #8275: buggy handling of callback args under Win64
+        # NOTE: should be run on release builds as well
+        dll = CDLL(_ctypes_test.__file__)
+        CALLBACK = CFUNCTYPE(c_double, c_double, c_double, c_double,
+                             c_double, c_double)
+        # All this function does is call the callback with its args squared
+        func = dll._testfunc_cbk_reg_double
+        func.argtypes = (c_double, c_double, c_double,
+                         c_double, c_double, CALLBACK)
+        func.restype = c_double
+
+        def callback(a, b, c, d, e):
+            return a + b + c + d + e
+
+        result = func(1.1, 2.2, 3.3, 4.4, 5.5, CALLBACK(callback))
+        self.assertEqual(result,
+                         callback(1.1*1.1, 2.2*2.2, 3.3*3.3, 4.4*4.4, 5.5*5.5))
+
+
 ################################################################
 
 if __name__ == '__main__':
diff --git a/lib-python/modified-2.7/ctypes/test/test_functions.py b/lib-python/modified-2.7/ctypes/test/test_functions.py
--- a/lib-python/modified-2.7/ctypes/test/test_functions.py
+++ b/lib-python/modified-2.7/ctypes/test/test_functions.py
@@ -118,7 +118,7 @@
         self.assertEqual(result, 21)
         self.assertEqual(type(result), int)
 
-        # You cannot assing character format codes as restype any longer
+        # You cannot assign character format codes as restype any longer
         self.assertRaises(TypeError, setattr, f, "restype", "i")
 
     def test_floatresult(self):
diff --git a/lib-python/modified-2.7/ctypes/test/test_init.py b/lib-python/modified-2.7/ctypes/test/test_init.py
--- a/lib-python/modified-2.7/ctypes/test/test_init.py
+++ b/lib-python/modified-2.7/ctypes/test/test_init.py
@@ -27,7 +27,7 @@
         self.assertEqual((y.x.a, y.x.b), (0, 0))
         self.assertEqual(y.x.new_was_called, False)
 
-        # But explicitely creating an X structure calls __new__ and __init__, of course.
+        # But explicitly creating an X structure calls __new__ and __init__, of course.
         x = X()
         self.assertEqual((x.a, x.b), (9, 12))
         self.assertEqual(x.new_was_called, True)
diff --git a/lib-python/modified-2.7/ctypes/test/test_numbers.py b/lib-python/modified-2.7/ctypes/test/test_numbers.py
--- a/lib-python/modified-2.7/ctypes/test/test_numbers.py
+++ b/lib-python/modified-2.7/ctypes/test/test_numbers.py
@@ -162,7 +162,7 @@
     def test_int_from_address(self):
         from array import array
         for t in signed_types + unsigned_types:
-            # the array module doesn't suppport all format codes
+            # the array module doesn't support all format codes
             # (no 'q' or 'Q')
             try:
                 array(t._type_)
diff --git a/lib-python/modified-2.7/ctypes/test/test_win32.py b/lib-python/modified-2.7/ctypes/test/test_win32.py
--- a/lib-python/modified-2.7/ctypes/test/test_win32.py
+++ b/lib-python/modified-2.7/ctypes/test/test_win32.py
@@ -17,7 +17,7 @@
             # ValueError: Procedure probably called with not enough arguments (4 bytes missing)
             self.assertRaises(ValueError, IsWindow)
 
-            # This one should succeeed...
+            # This one should succeed...
             self.assertEqual(0, IsWindow(0))
 
             # ValueError: Procedure probably called with too many arguments (8 bytes in excess)
diff --git a/lib-python/modified-2.7/distutils/__init__.py b/lib-python/modified-2.7/distutils/__init__.py
--- a/lib-python/modified-2.7/distutils/__init__.py
+++ b/lib-python/modified-2.7/distutils/__init__.py
@@ -15,5 +15,5 @@
 # Updated automatically by the Python release process.
 #
 #--start constants--
-__version__ = "2.7.1"
+__version__ = "2.7.2"
 #--end constants--
diff --git a/lib-python/modified-2.7/distutils/archive_util.py b/lib-python/modified-2.7/distutils/archive_util.py
--- a/lib-python/modified-2.7/distutils/archive_util.py
+++ b/lib-python/modified-2.7/distutils/archive_util.py
@@ -121,7 +121,7 @@
 def make_zipfile(base_name, base_dir, verbose=0, dry_run=0):
     """Create a zip file from all the files under 'base_dir'.
 
-    The output zip file will be named 'base_dir' + ".zip".  Uses either the
+    The output zip file will be named 'base_name' + ".zip".  Uses either the
     "zipfile" Python module (if available) or the InfoZIP "zip" utility
     (if installed and found on the default search path).  If neither tool is
     available, raises DistutilsExecError.  Returns the name of the output zip
diff --git a/lib-python/modified-2.7/distutils/cmd.py b/lib-python/modified-2.7/distutils/cmd.py
--- a/lib-python/modified-2.7/distutils/cmd.py
+++ b/lib-python/modified-2.7/distutils/cmd.py
@@ -377,7 +377,7 @@
             dry_run=self.dry_run)
 
     def move_file (self, src, dst, level=1):
-        """Move a file respectin dry-run flag."""
+        """Move a file respecting dry-run flag."""
         return file_util.move_file(src, dst, dry_run = self.dry_run)
 
     def spawn (self, cmd, search_path=1, level=1):
diff --git a/lib-python/modified-2.7/distutils/command/build_ext.py b/lib-python/modified-2.7/distutils/command/build_ext.py
--- a/lib-python/modified-2.7/distutils/command/build_ext.py
+++ b/lib-python/modified-2.7/distutils/command/build_ext.py
@@ -212,7 +212,7 @@
 
             elif MSVC_VERSION == 8:
                 self.library_dirs.append(os.path.join(sys.exec_prefix,
-                                         'PC', 'VS8.0', 'win32release'))
+                                         'PC', 'VS8.0'))
             elif MSVC_VERSION == 7:
                 self.library_dirs.append(os.path.join(sys.exec_prefix,
                                          'PC', 'VS7.1'))
diff --git a/lib-python/modified-2.7/distutils/command/sdist.py b/lib-python/modified-2.7/distutils/command/sdist.py
--- a/lib-python/modified-2.7/distutils/command/sdist.py
+++ b/lib-python/modified-2.7/distutils/command/sdist.py
@@ -306,17 +306,20 @@
                             rstrip_ws=1,
                             collapse_join=1)
 
-        while 1:
-            line = template.readline()
-            if line is None:            # end of file
-                break
+        try:
+            while 1:
+                line = template.readline()
+                if line is None:            # end of file
+                    break
 
-            try:
-                self.filelist.process_template_line(line)
-            except DistutilsTemplateError, msg:
-                self.warn("%s, line %d: %s" % (template.filename,
-                                               template.current_line,
-                                               msg))
+                try:
+                    self.filelist.process_template_line(line)
+                except DistutilsTemplateError, msg:
+                    self.warn("%s, line %d: %s" % (template.filename,
+                                                   template.current_line,
+                                                   msg))
+        finally:
+            template.close()
 
     def prune_file_list(self):
         """Prune off branches that might slip into the file list as created
diff --git a/lib-python/modified-2.7/distutils/command/upload.py b/lib-python/modified-2.7/distutils/command/upload.py
--- a/lib-python/modified-2.7/distutils/command/upload.py
+++ b/lib-python/modified-2.7/distutils/command/upload.py
@@ -176,6 +176,9 @@
             result = urlopen(request)
             status = result.getcode()
             reason = result.msg
+            if self.show_response:
+                msg = '\n'.join(('-' * 75, r.read(), '-' * 75))
+                self.announce(msg, log.INFO)
         except socket.error, e:
             self.announce(str(e), log.ERROR)
             return
@@ -189,6 +192,3 @@
         else:
             self.announce('Upload failed (%s): %s' % (status, reason),
                           log.ERROR)
-        if self.show_response:
-            msg = '\n'.join(('-' * 75, r.read(), '-' * 75))
-            self.announce(msg, log.INFO)
diff --git a/lib-python/modified-2.7/distutils/sysconfig.py b/lib-python/modified-2.7/distutils/sysconfig.py
--- a/lib-python/modified-2.7/distutils/sysconfig.py
+++ b/lib-python/modified-2.7/distutils/sysconfig.py
@@ -9,21 +9,563 @@
 Email:        
 """
 
-__revision__ = "$Id: sysconfig.py 85358 2010-10-10 09:54:59Z antoine.pitrou $"
+__revision__ = "$Id$"
 
+import os
+import re
+import string
 import sys
 
+from distutils.errors import DistutilsPlatformError
 
-# The content of this file is redirected from
-# sysconfig_cpython or sysconfig_pypy.
+# These are needed in a couple of spots, so just compute them once.
+PREFIX = os.path.normpath(sys.prefix)
+EXEC_PREFIX = os.path.normpath(sys.exec_prefix)
 
-if '__pypy__' in sys.builtin_module_names:
-    from distutils.sysconfig_pypy import *
-    from distutils.sysconfig_pypy import _config_vars # needed by setuptools
-    from distutils.sysconfig_pypy import _variable_rx # read_setup_file()
-else:
-    from distutils.sysconfig_cpython import *
-    from distutils.sysconfig_cpython import _config_vars # needed by setuptools
-    from distutils.sysconfig_cpython import _variable_rx # read_setup_file()
+# Path to the base directory of the project. On Windows the binary may
+# live in project/PCBuild9.  If we're dealing with an x64 Windows build,
+# it'll live in project/PCbuild/amd64.
+project_base = os.path.dirname(os.path.abspath(sys.executable))
+if os.name == "nt" and "pcbuild" in project_base[-8:].lower():
+    project_base = os.path.abspath(os.path.join(project_base, os.path.pardir))
+# PC/VS7.1
+if os.name == "nt" and "\\pc\\v" in project_base[-10:].lower():
+    project_base = os.path.abspath(os.path.join(project_base, os.path.pardir,
+                                                os.path.pardir))
+# PC/AMD64
+if os.name == "nt" and "\\pcbuild\\amd64" in project_base[-14:].lower():
+    project_base = os.path.abspath(os.path.join(project_base, os.path.pardir,
+                                                os.path.pardir))
 
+# python_build: (Boolean) if true, we're either building Python or
+# building an extension with an un-installed Python, so we use
+# different (hard-wired) directories.
+# Setup.local is available for Makefile builds including VPATH builds,
+# Setup.dist is available on Windows
+def _python_build():
+    for fn in ("Setup.dist", "Setup.local"):
+        if os.path.isfile(os.path.join(project_base, "Modules", fn)):
+            return True
+    return False
+python_build = _python_build()
 
+
+def get_python_version():
+    """Return a string containing the major and minor Python version,
+    leaving off the patchlevel.  Sample return values could be '1.5'
+    or '2.2'.
+    """
+    return sys.version[:3]
+
+
+def get_python_inc(plat_specific=0, prefix=None):
+    """Return the directory containing installed Python header files.
+
+    If 'plat_specific' is false (the default), this is the path to the
+    non-platform-specific header files, i.e. Python.h and so on;
+    otherwise, this is the path to platform-specific header files
+    (namely pyconfig.h).
+
+    If 'prefix' is supplied, use it instead of sys.prefix or
+    sys.exec_prefix -- i.e., ignore 'plat_specific'.
+    """
+    if prefix is None:
+        prefix = plat_specific and EXEC_PREFIX or PREFIX
+
+    if os.name == "posix":
+        if python_build:
+            buildir = os.path.dirname(sys.executable)
+            if plat_specific:
+                # python.h is located in the buildir
+                inc_dir = buildir
+            else:
+                # the source dir is relative to the buildir
+                srcdir = os.path.abspath(os.path.join(buildir,
+                                         get_config_var('srcdir')))
+                # Include is located in the srcdir
+                inc_dir = os.path.join(srcdir, "Include")
+            return inc_dir
+        return os.path.join(prefix, "include", "python" + get_python_version())
+    elif os.name == "nt":
+        return os.path.join(prefix, "include")
+    elif os.name == "os2":
+        return os.path.join(prefix, "Include")
+    else:
+        raise DistutilsPlatformError(
+            "I don't know where Python installs its C header files "
+            "on platform '%s'" % os.name)
+
+
+def get_python_lib(plat_specific=0, standard_lib=0, prefix=None):
+    """Return the directory containing the Python library (standard or
+    site additions).
+
+    If 'plat_specific' is true, return the directory containing
+    platform-specific modules, i.e. any module from a non-pure-Python
+    module distribution; otherwise, return the platform-shared library
+    directory.  If 'standard_lib' is true, return the directory
+    containing standard Python library modules; otherwise, return the
+    directory for site-specific modules.
+
+    If 'prefix' is supplied, use it instead of sys.prefix or
+    sys.exec_prefix -- i.e., ignore 'plat_specific'.
+    """
+    if prefix is None:
+        prefix = plat_specific and EXEC_PREFIX or PREFIX
+
+    if os.name == "posix":
+        libpython = os.path.join(prefix,
+                                 "lib", "python" + get_python_version())
+        if standard_lib:
+            return libpython
+        else:
+            return os.path.join(libpython, "site-packages")
+
+    elif os.name == "nt":
+        if standard_lib:
+            return os.path.join(prefix, "Lib")
+        else:
+            if get_python_version() < "2.2":
+                return prefix
+            else:
+                return os.path.join(prefix, "Lib", "site-packages")
+
+    elif os.name == "os2":
+        if standard_lib:
+            return os.path.join(prefix, "Lib")
+        else:
+            return os.path.join(prefix, "Lib", "site-packages")
+
+    else:
+        raise DistutilsPlatformError(
+            "I don't know where Python installs its library "
+            "on platform '%s'" % os.name)
+
+
+def customize_compiler(compiler):
+    """Do any platform-specific customization of a CCompiler instance.
+
+    Mainly needed on Unix, so we can plug in the information that
+    varies across Unices and is stored in Python's Makefile.
+    """
+    if compiler.compiler_type == "unix":
+        (cc, cxx, opt, cflags, ccshared, ldshared, so_ext) = \
+            get_config_vars('CC', 'CXX', 'OPT', 'CFLAGS',
+                            'CCSHARED', 'LDSHARED', 'SO')
+
+        if 'CC' in os.environ:
+            cc = os.environ['CC']
+        if 'CXX' in os.environ:
+            cxx = os.environ['CXX']
+        if 'LDSHARED' in os.environ:
+            ldshared = os.environ['LDSHARED']
+        if 'CPP' in os.environ:
+            cpp = os.environ['CPP']
+        else:
+            cpp = cc + " -E"           # not always
+        if 'LDFLAGS' in os.environ:
+            ldshared = ldshared + ' ' + os.environ['LDFLAGS']
+        if 'CFLAGS' in os.environ:
+            cflags = opt + ' ' + os.environ['CFLAGS']
+            ldshared = ldshared + ' ' + os.environ['CFLAGS']
+        if 'CPPFLAGS' in os.environ:
+            cpp = cpp + ' ' + os.environ['CPPFLAGS']
+            cflags = cflags + ' ' + os.environ['CPPFLAGS']
+            ldshared = ldshared + ' ' + os.environ['CPPFLAGS']
+
+        cc_cmd = cc + ' ' + cflags
+        compiler.set_executables(
+            preprocessor=cpp,
+            compiler=cc_cmd,
+            compiler_so=cc_cmd + ' ' + ccshared,
+            compiler_cxx=cxx,
+            linker_so=ldshared,
+            linker_exe=cc)
+
+        compiler.shared_lib_extension = so_ext
+
+
+def get_config_h_filename():
+    """Return full pathname of installed pyconfig.h file."""
+    if python_build:
+        if os.name == "nt":
+            inc_dir = os.path.join(project_base, "PC")
+        else:
+            inc_dir = project_base
+    else:
+        inc_dir = get_python_inc(plat_specific=1)
+    if get_python_version() < '2.2':
+        config_h = 'config.h'
+    else:
+        # The name of the config.h file changed in 2.2
+        config_h = 'pyconfig.h'
+    return os.path.join(inc_dir, config_h)
+
+
+def get_makefile_filename():
+    """Return full pathname of installed Makefile from the Python build."""
+    if python_build:
+        return os.path.join(os.path.dirname(sys.executable), "Makefile")
+    lib_dir = get_python_lib(plat_specific=1, standard_lib=1)
+    return os.path.join(lib_dir, "config", "Makefile")
+
+
+def parse_config_h(fp, g=None):
+    """Parse a config.h-style file.
+
+    A dictionary containing name/value pairs is returned.  If an
+    optional dictionary is passed in as the second argument, it is
+    used instead of a new dictionary.
+    """
+    if g is None:
+        g = {}
+    define_rx = re.compile("#define ([A-Z][A-Za-z0-9_]+) (.*)\n")
+    undef_rx = re.compile("/[*] #undef ([A-Z][A-Za-z0-9_]+) [*]/\n")
+    #
+    while 1:
+        line = fp.readline()
+        if not line:
+            break
+        m = define_rx.match(line)
+        if m:
+            n, v = m.group(1, 2)
+            try: v = int(v)
+            except ValueError: pass
+            g[n] = v
+        else:
+            m = undef_rx.match(line)
+            if m:
+                g[m.group(1)] = 0
+    return g
+
+
+# Regexes needed for parsing Makefile (and similar syntaxes,
+# like old-style Setup files).
+_variable_rx = re.compile("([a-zA-Z][a-zA-Z0-9_]+)\s*=\s*(.*)")
+_findvar1_rx = re.compile(r"\$\(([A-Za-z][A-Za-z0-9_]*)\)")
+_findvar2_rx = re.compile(r"\${([A-Za-z][A-Za-z0-9_]*)}")
+
+def parse_makefile(fn, g=None):
+    """Parse a Makefile-style file.
+
+    A dictionary containing name/value pairs is returned.  If an
+    optional dictionary is passed in as the second argument, it is
+    used instead of a new dictionary.
+    """
+    from distutils.text_file import TextFile
+    fp = TextFile(fn, strip_comments=1, skip_blanks=1, join_lines=1)
+
+    if g is None:
+        g = {}
+    done = {}
+    notdone = {}
+
+    while 1:
+        line = fp.readline()
+        if line is None:  # eof
+            break
+        m = _variable_rx.match(line)
+        if m:
+            n, v = m.group(1, 2)
+            v = v.strip()
+            # `$$' is a literal `$' in make
+            tmpv = v.replace('$$', '')
+
+            if "$" in tmpv:
+                notdone[n] = v
+            else:
+                try:
+                    v = int(v)
+                except ValueError:
+                    # insert literal `$'
+                    done[n] = v.replace('$$', '$')
+                else:
+                    done[n] = v
+
+    # do variable interpolation here
+    while notdone:
+        for name in notdone.keys():
+            value = notdone[name]
+            m = _findvar1_rx.search(value) or _findvar2_rx.search(value)
+            if m:
+                n = m.group(1)
+                found = True
+                if n in done:
+                    item = str(done[n])
+                elif n in notdone:
+                    # get it on a subsequent round
+                    found = False
+                elif n in os.environ:
+                    # do it like make: fall back to environment
+                    item = os.environ[n]
+                else:
+                    done[n] = item = ""
+                if found:
+                    after = value[m.end():]
+                    value = value[:m.start()] + item + after
+                    if "$" in after:
+                        notdone[name] = value
+                    else:
+                        try: value = int(value)
+                        except ValueError:
+                            done[name] = value.strip()
+                        else:
+                            done[name] = value
+                        del notdone[name]
+            else:
+                # bogus variable reference; just drop it since we can't deal
+                del notdone[name]
+
+    fp.close()
+
+    # strip spurious spaces
+    for k, v in done.items():
+        if isinstance(v, str):
+            done[k] = v.strip()
+
+    # save the results in the global dictionary
+    g.update(done)
+    return g
+
+
+def expand_makefile_vars(s, vars):
+    """Expand Makefile-style variables -- "${foo}" or "$(foo)" -- in
+    'string' according to 'vars' (a dictionary mapping variable names to
+    values).  Variables not present in 'vars' are silently expanded to the
+    empty string.  The variable values in 'vars' should not contain further
+    variable expansions; if 'vars' is the output of 'parse_makefile()',
+    you're fine.  Returns a variable-expanded version of 's'.
+    """
+
+    # This algorithm does multiple expansion, so if vars['foo'] contains
+    # "${bar}", it will expand ${foo} to ${bar}, and then expand
+    # ${bar}... and so forth.  This is fine as long as 'vars' comes from
+    # 'parse_makefile()', which takes care of such expansions eagerly,
+    # according to make's variable expansion semantics.
+
+    while 1:
+        m = _findvar1_rx.search(s) or _findvar2_rx.search(s)
+        if m:
+            (beg, end) = m.span()
+            s = s[0:beg] + vars.get(m.group(1)) + s[end:]
+        else:
+            break
+    return s
+
+
+_config_vars = None
+
+def _init_posix():
+    """Initialize the module as appropriate for POSIX systems."""
+    g = {}
+    # load the installed Makefile:
+    try:
+        filename = get_makefile_filename()
+        parse_makefile(filename, g)
+    except IOError, msg:
+        my_msg = "invalid Python installation: unable to open %s" % filename
+        if hasattr(msg, "strerror"):
+            my_msg = my_msg + " (%s)" % msg.strerror
+
+        raise DistutilsPlatformError(my_msg)
+
+    # load the installed pyconfig.h:
+    try:
+        filename = get_config_h_filename()
+        parse_config_h(file(filename), g)
+    except IOError, msg:
+        my_msg = "invalid Python installation: unable to open %s" % filename
+        if hasattr(msg, "strerror"):
+            my_msg = my_msg + " (%s)" % msg.strerror
+
+        raise DistutilsPlatformError(my_msg)
+
+    # On MacOSX we need to check the setting of the environment variable
+    # MACOSX_DEPLOYMENT_TARGET: configure bases some choices on it so
+    # it needs to be compatible.
+    # If it isn't set we set it to the configure-time value
+    if sys.platform == 'darwin' and 'MACOSX_DEPLOYMENT_TARGET' in g:
+        cfg_target = g['MACOSX_DEPLOYMENT_TARGET']
+        cur_target = os.getenv('MACOSX_DEPLOYMENT_TARGET', '')
+        if cur_target == '':
+            cur_target = cfg_target
+            os.environ['MACOSX_DEPLOYMENT_TARGET'] = cfg_target
+        elif map(int, cfg_target.split('.')) > map(int, cur_target.split('.')):
+            my_msg = ('$MACOSX_DEPLOYMENT_TARGET mismatch: now "%s" but "%s" during configure'
+                % (cur_target, cfg_target))
+            raise DistutilsPlatformError(my_msg)
+
+    # On AIX, there are wrong paths to the linker scripts in the Makefile
+    # -- these paths are relative to the Python source, but when installed
+    # the scripts are in another directory.
+    if python_build:
+        g['LDSHARED'] = g['BLDSHARED']
+
+    elif get_python_version() < '2.1':
+        # The following two branches are for 1.5.2 compatibility.
+        if sys.platform == 'aix4':          # what about AIX 3.x ?
+            # Linker script is in the config directory, not in Modules as the
+            # Makefile says.
+            python_lib = get_python_lib(standard_lib=1)
+            ld_so_aix = os.path.join(python_lib, 'config', 'ld_so_aix')
+            python_exp = os.path.join(python_lib, 'config', 'python.exp')
+
+            g['LDSHARED'] = "%s %s -bI:%s" % (ld_so_aix, g['CC'], python_exp)
+
+        elif sys.platform == 'beos':
+            # Linker script is in the config directory.  In the Makefile it is
+            # relative to the srcdir, which after installation no longer makes
+            # sense.
+            python_lib = get_python_lib(standard_lib=1)
+            linkerscript_path = string.split(g['LDSHARED'])[0]
+            linkerscript_name = os.path.basename(linkerscript_path)
+            linkerscript = os.path.join(python_lib, 'config',
+                                        linkerscript_name)
+
+            # XXX this isn't the right place to do this: adding the Python
+            # library to the link, if needed, should be in the "build_ext"
+            # command.  (It's also needed for non-MS compilers on Windows, and
+            # it's taken care of for them by the 'build_ext.get_libraries()'
+            # method.)
+            g['LDSHARED'] = ("%s -L%s/lib -lpython%s" %
+                             (linkerscript, PREFIX, get_python_version()))
+
+    global _config_vars
+    _config_vars = g
+
+
+def _init_nt():
+    """Initialize the module as appropriate for NT"""
+    g = {}
+    # set basic install directories
+    g['LIBDEST'] = get_python_lib(plat_specific=0, standard_lib=1)
+    g['BINLIBDEST'] = get_python_lib(plat_specific=1, standard_lib=1)
+
+    # XXX hmmm.. a normal install puts include files here
+    g['INCLUDEPY'] = get_python_inc(plat_specific=0)
+
+    g['SO'] = '.pyd'
+    g['EXE'] = ".exe"
+    g['VERSION'] = get_python_version().replace(".", "")
+    g['BINDIR'] = os.path.dirname(os.path.abspath(sys.executable))
+
+    global _config_vars
+    _config_vars = g
+
+
+def _init_os2():
+    """Initialize the module as appropriate for OS/2"""
+    g = {}
+    # set basic install directories
+    g['LIBDEST'] = get_python_lib(plat_specific=0, standard_lib=1)
+    g['BINLIBDEST'] = get_python_lib(plat_specific=1, standard_lib=1)
+
+    # XXX hmmm.. a normal install puts include files here
+    g['INCLUDEPY'] = get_python_inc(plat_specific=0)
+
+    g['SO'] = '.pyd'
+    g['EXE'] = ".exe"
+
+    global _config_vars
+    _config_vars = g
+
+
+def get_config_vars(*args):
+    """With no arguments, return a dictionary of all configuration
+    variables relevant for the current platform.  Generally this includes
+    everything needed to build extensions and install both pure modules and
+    extensions.  On Unix, this means every variable defined in Python's
+    installed Makefile; on Windows and Mac OS it's a much smaller set.
+
+    With arguments, return a list of values that result from looking up
+    each argument in the configuration variable dictionary.
+    """
+    global _config_vars
+    if _config_vars is None:
+        func = globals().get("_init_" + os.name)
+        if func:
+            func()
+        else:
+            _config_vars = {}
+
+        # Normalized versions of prefix and exec_prefix are handy to have;
+        # in fact, these are the standard versions used most places in the
+        # Distutils.
+        _config_vars['prefix'] = PREFIX
+        _config_vars['exec_prefix'] = EXEC_PREFIX
+
+        if sys.platform == 'darwin':
+            kernel_version = os.uname()[2] # Kernel version (8.4.3)
+            major_version = int(kernel_version.split('.')[0])
+
+            if major_version < 8:
+                # On Mac OS X before 10.4, check if -arch and -isysroot
+                # are in CFLAGS or LDFLAGS and remove them if they are.
+                # This is needed when building extensions on a 10.3 system
+                # using a universal build of python.
+                for key in ('LDFLAGS', 'BASECFLAGS', 'LDSHARED',
+                        # a number of derived variables. These need to be
+                        # patched up as well.
+                        'CFLAGS', 'PY_CFLAGS', 'BLDSHARED'):
+                    flags = _config_vars[key]
+                    flags = re.sub('-arch\s+\w+\s', ' ', flags)
+                    flags = re.sub('-isysroot [^ \t]*', ' ', flags)
+                    _config_vars[key] = flags
+
+            else:
+
+                # Allow the user to override the architecture flags using
+                # an environment variable.
+                # NOTE: This name was introduced by Apple in OSX 10.5 and
+                # is used by several scripting languages distributed with
+                # that OS release.
+
+                if 'ARCHFLAGS' in os.environ:
+                    arch = os.environ['ARCHFLAGS']
+                    for key in ('LDFLAGS', 'BASECFLAGS', 'LDSHARED',
+                        # a number of derived variables. These need to be
+                        # patched up as well.
+                        'CFLAGS', 'PY_CFLAGS', 'BLDSHARED'):
+
+                        flags = _config_vars[key]
+                        flags = re.sub('-arch\s+\w+\s', ' ', flags)
+                        flags = flags + ' ' + arch
+                        _config_vars[key] = flags
+
+                # If we're on OSX 10.5 or later and the user tries to
+                # compiles an extension using an SDK that is not present
+                # on the current machine it is better to not use an SDK
+                # than to fail.
+                #
+                # The major usecase for this is users using a Python.org
+                # binary installer  on OSX 10.6: that installer uses
+                # the 10.4u SDK, but that SDK is not installed by default
+                # when you install Xcode.
+                #
+                m = re.search('-isysroot\s+(\S+)', _config_vars['CFLAGS'])
+                if m is not None:
+                    sdk = m.group(1)
+                    if not os.path.exists(sdk):
+                        for key in ('LDFLAGS', 'BASECFLAGS', 'LDSHARED',
+                             # a number of derived variables. These need to be
+                             # patched up as well.
+                            'CFLAGS', 'PY_CFLAGS', 'BLDSHARED'):
+
+                            flags = _config_vars[key]
+                            flags = re.sub('-isysroot\s+\S+(\s|$)', ' ', flags)
+                            _config_vars[key] = flags
+
+    if args:
+        vals = []
+        for name in args:
+            vals.append(_config_vars.get(name))
+        return vals
+    else:
+        return _config_vars
+
+def get_config_var(name):
+    """Return the value of a single variable using the dictionary
+    returned by 'get_config_vars()'.  Equivalent to
+    get_config_vars().get(name)
+    """
+    return get_config_vars().get(name)
diff --git a/lib-python/modified-2.7/distutils/sysconfig_cpython.py b/lib-python/modified-2.7/distutils/sysconfig_cpython.py
--- a/lib-python/modified-2.7/distutils/sysconfig_cpython.py
+++ b/lib-python/modified-2.7/distutils/sysconfig_cpython.py
@@ -389,7 +389,7 @@
         cur_target = os.getenv('MACOSX_DEPLOYMENT_TARGET', '')
         if cur_target == '':
             cur_target = cfg_target
-            os.putenv('MACOSX_DEPLOYMENT_TARGET', cfg_target)
+            os.environ['MACOSX_DEPLOYMENT_TARGET'] = cfg_target
         elif map(int, cfg_target.split('.')) > map(int, cur_target.split('.')):
             my_msg = ('$MACOSX_DEPLOYMENT_TARGET mismatch: now "%s" but "%s" during configure'
                 % (cur_target, cfg_target))
diff --git a/lib-python/modified-2.7/distutils/tests/__init__.py b/lib-python/modified-2.7/distutils/tests/__init__.py
--- a/lib-python/modified-2.7/distutils/tests/__init__.py
+++ b/lib-python/modified-2.7/distutils/tests/__init__.py
@@ -15,9 +15,10 @@
 import os
 import sys
 import unittest
+from test.test_support import run_unittest
 
 
-here = os.path.dirname(__file__)
+here = os.path.dirname(__file__) or os.curdir
 
 
 def test_suite():
@@ -32,4 +33,4 @@
 
 
 if __name__ == "__main__":
-    unittest.main(defaultTest="test_suite")
+    run_unittest(test_suite())
diff --git a/lib-python/modified-2.7/distutils/tests/test_archive_util.py b/lib-python/modified-2.7/distutils/tests/test_archive_util.py
--- a/lib-python/modified-2.7/distutils/tests/test_archive_util.py
+++ b/lib-python/modified-2.7/distutils/tests/test_archive_util.py
@@ -12,7 +12,7 @@
                                     ARCHIVE_FORMATS)
 from distutils.spawn import find_executable, spawn
 from distutils.tests import support
-from test.test_support import check_warnings
+from test.test_support import check_warnings, run_unittest
 
 try:
     import grp
@@ -281,4 +281,4 @@
     return unittest.makeSuite(ArchiveUtilTestCase)
 
 if __name__ == "__main__":
-    unittest.main(defaultTest="test_suite")
+    run_unittest(test_suite())
diff --git a/lib-python/modified-2.7/distutils/tests/test_bdist_msi.py b/lib-python/modified-2.7/distutils/tests/test_bdist_msi.py
--- a/lib-python/modified-2.7/distutils/tests/test_bdist_msi.py
+++ b/lib-python/modified-2.7/distutils/tests/test_bdist_msi.py
@@ -11,7 +11,7 @@
                        support.LoggingSilencer,
                        unittest.TestCase):
 
-    def test_minial(self):
+    def test_minimal(self):
         # minimal test XXX need more tests
         from distutils.command.bdist_msi import bdist_msi
         pkg_pth, dist = self.create_dist()
diff --git a/lib-python/modified-2.7/distutils/tests/test_build.py b/lib-python/modified-2.7/distutils/tests/test_build.py
--- a/lib-python/modified-2.7/distutils/tests/test_build.py
+++ b/lib-python/modified-2.7/distutils/tests/test_build.py
@@ -2,6 +2,7 @@
 import unittest
 import os
 import sys
+from test.test_support import run_unittest
 
 from distutils.command.build import build
 from distutils.tests import support
@@ -51,4 +52,4 @@
     return unittest.makeSuite(BuildTestCase)
 
 if __name__ == "__main__":
-    unittest.main(defaultTest="test_suite")
+    run_unittest(test_suite())
diff --git a/lib-python/modified-2.7/distutils/tests/test_build_clib.py b/lib-python/modified-2.7/distutils/tests/test_build_clib.py
--- a/lib-python/modified-2.7/distutils/tests/test_build_clib.py
+++ b/lib-python/modified-2.7/distutils/tests/test_build_clib.py
@@ -3,6 +3,8 @@
 import os
 import sys
 
+from test.test_support import run_unittest
+
 from distutils.command.build_clib import build_clib
 from distutils.errors import DistutilsSetupError
 from distutils.tests import support
@@ -140,4 +142,4 @@
     return unittest.makeSuite(BuildCLibTestCase)
 
 if __name__ == "__main__":
-    unittest.main(defaultTest="test_suite")
+    run_unittest(test_suite())
diff --git a/lib-python/modified-2.7/distutils/tests/test_build_ext.py b/lib-python/modified-2.7/distutils/tests/test_build_ext.py
--- a/lib-python/modified-2.7/distutils/tests/test_build_ext.py
+++ b/lib-python/modified-2.7/distutils/tests/test_build_ext.py
@@ -3,12 +3,13 @@
 import tempfile
 import shutil
 from StringIO import StringIO
+import textwrap
 
 from distutils.core import Extension, Distribution
 from distutils.command.build_ext import build_ext
 from distutils import sysconfig
 from distutils.tests import support
-from distutils.errors import DistutilsSetupError
+from distutils.errors import DistutilsSetupError, CompileError
 
 import unittest
 from test import test_support
@@ -430,6 +431,59 @@
         wanted = os.path.join(cmd.build_lib, 'UpdateManager', 'fdsend' + ext)
         self.assertEqual(ext_path, wanted)
 
+    @unittest.skipUnless(sys.platform == 'darwin', 'test only relevant for MacOSX')
+    def test_deployment_target(self):
+        self._try_compile_deployment_target()
+
+        orig_environ = os.environ
+        os.environ = orig_environ.copy()
+        self.addCleanup(setattr, os, 'environ', orig_environ)
+
+        os.environ['MACOSX_DEPLOYMENT_TARGET']='10.1'
+        self._try_compile_deployment_target()
+
+
+    def _try_compile_deployment_target(self):
+        deptarget_c = os.path.join(self.tmp_dir, 'deptargetmodule.c')
+
+        with open(deptarget_c, 'w') as fp:
+            fp.write(textwrap.dedent('''\
+                #include 
+
+                int dummy;
+
+                #if TARGET != MAC_OS_X_VERSION_MIN_REQUIRED
+                #error "Unexpected target"
+               #endif
+
+            '''))
+
+        target = sysconfig.get_config_var('MACOSX_DEPLOYMENT_TARGET')
+        target = tuple(map(int, target.split('.')))
+        target = '%02d%01d0' % target
+
+        deptarget_ext = Extension(
+            'deptarget',
+            [deptarget_c],
+            extra_compile_args=['-DTARGET=%s'%(target,)],
+        )
+        dist = Distribution({
+            'name': 'deptarget',
+            'ext_modules': [deptarget_ext]
+        })
+        dist.package_dir = self.tmp_dir
+        cmd = build_ext(dist)
+        cmd.build_lib = self.tmp_dir
+        cmd.build_temp = self.tmp_dir
+
+        try:
+            old_stdout = sys.stdout
+            cmd.ensure_finalized()
+            cmd.run()
+
+        except CompileError:
+            self.fail("Wrong deployment target during compilation")
+
 def test_suite():
     return unittest.makeSuite(BuildExtTestCase)
 
diff --git a/lib-python/modified-2.7/distutils/tests/test_build_py.py b/lib-python/modified-2.7/distutils/tests/test_build_py.py
--- a/lib-python/modified-2.7/distutils/tests/test_build_py.py
+++ b/lib-python/modified-2.7/distutils/tests/test_build_py.py
@@ -10,13 +10,14 @@
 from distutils.errors import DistutilsFileError
 
 from distutils.tests import support
+from test.test_support import run_unittest
 
 
 class BuildPyTestCase(support.TempdirManager,
                       support.LoggingSilencer,
                       unittest.TestCase):
 
-    def _setup_package_data(self):
+    def test_package_data(self):
         sources = self.mkdtemp()
         f = open(os.path.join(sources, "__init__.py"), "w")
         try:
@@ -56,20 +57,15 @@
         self.assertEqual(len(cmd.get_outputs()), 3)
         pkgdest = os.path.join(destination, "pkg")
         files = os.listdir(pkgdest)
-        return files
+        self.assertIn("__init__.py", files)
+        self.assertIn("README.txt", files)
+        # XXX even with -O, distutils writes pyc, not pyo; bug?
+        if sys.dont_write_bytecode:
+            self.assertNotIn("__init__.pyc", files)
+        else:
+            self.assertIn("__init__.pyc", files)
 
-    def test_package_data(self):
-        files = self._setup_package_data()
-        self.assertTrue("__init__.py" in files)
-        self.assertTrue("README.txt" in files)
-
-    @unittest.skipIf(sys.flags.optimize >= 2,
-                     "pyc files are not written with -O2 and above")
-    def test_package_data_pyc(self):
-        files = self._setup_package_data()
-        self.assertTrue("__init__.pyc" in files)
-
-    def test_empty_package_dir (self):
+    def test_empty_package_dir(self):
         # See SF 1668596/1720897.
         cwd = os.getcwd()
 
@@ -117,10 +113,10 @@
         finally:
             sys.dont_write_bytecode = old_dont_write_bytecode
 
-        self.assertTrue('byte-compiling is disabled' in self.logs[0][1])
+        self.assertIn('byte-compiling is disabled', self.logs[0][1])
 
 def test_suite():
     return unittest.makeSuite(BuildPyTestCase)
 
 if __name__ == "__main__":
-    unittest.main(defaultTest="test_suite")
+    run_unittest(test_suite())
diff --git a/lib-python/modified-2.7/distutils/tests/test_build_scripts.py b/lib-python/modified-2.7/distutils/tests/test_build_scripts.py
--- a/lib-python/modified-2.7/distutils/tests/test_build_scripts.py
+++ b/lib-python/modified-2.7/distutils/tests/test_build_scripts.py
@@ -8,6 +8,7 @@
 import sysconfig
 
 from distutils.tests import support
+from test.test_support import run_unittest
 
 
 class BuildScriptsTestCase(support.TempdirManager,
@@ -108,4 +109,4 @@
     return unittest.makeSuite(BuildScriptsTestCase)
 
 if __name__ == "__main__":
-    unittest.main(defaultTest="test_suite")
+    run_unittest(test_suite())
diff --git a/lib-python/modified-2.7/distutils/tests/test_check.py b/lib-python/modified-2.7/distutils/tests/test_check.py
--- a/lib-python/modified-2.7/distutils/tests/test_check.py
+++ b/lib-python/modified-2.7/distutils/tests/test_check.py
@@ -1,5 +1,6 @@
 """Tests for distutils.command.check."""
 import unittest
+from test.test_support import run_unittest
 
 from distutils.command.check import check, HAS_DOCUTILS
 from distutils.tests import support
@@ -95,4 +96,4 @@
     return unittest.makeSuite(CheckTestCase)
 
 if __name__ == "__main__":
-    unittest.main(defaultTest="test_suite")
+    run_unittest(test_suite())
diff --git a/lib-python/modified-2.7/distutils/tests/test_clean.py b/lib-python/modified-2.7/distutils/tests/test_clean.py
--- a/lib-python/modified-2.7/distutils/tests/test_clean.py
+++ b/lib-python/modified-2.7/distutils/tests/test_clean.py
@@ -6,6 +6,7 @@
 
 from distutils.command.clean import clean
 from distutils.tests import support
+from test.test_support import run_unittest
 
 class cleanTestCase(support.TempdirManager,
                     support.LoggingSilencer,
@@ -38,7 +39,7 @@
             self.assertTrue(not os.path.exists(path),
                          '%s was not removed' % path)
 
-        # let's run the command again (should spit warnings but suceed)
+        # let's run the command again (should spit warnings but succeed)
         cmd.all = 1
         cmd.ensure_finalized()
         cmd.run()
@@ -47,4 +48,4 @@
     return unittest.makeSuite(cleanTestCase)
 
 if __name__ == "__main__":
-    unittest.main(defaultTest="test_suite")
+    run_unittest(test_suite())
diff --git a/lib-python/modified-2.7/distutils/tests/test_cmd.py b/lib-python/modified-2.7/distutils/tests/test_cmd.py
--- a/lib-python/modified-2.7/distutils/tests/test_cmd.py
+++ b/lib-python/modified-2.7/distutils/tests/test_cmd.py
@@ -99,7 +99,7 @@
 
     def test_ensure_dirname(self):
         cmd = self.cmd
-        cmd.option1 = os.path.dirname(__file__)
+        cmd.option1 = os.path.dirname(__file__) or os.curdir
         cmd.ensure_dirname('option1')
         cmd.option2 = 'xxx'
         self.assertRaises(DistutilsOptionError, cmd.ensure_dirname, 'option2')
diff --git a/lib-python/modified-2.7/distutils/tests/test_config.py b/lib-python/modified-2.7/distutils/tests/test_config.py
--- a/lib-python/modified-2.7/distutils/tests/test_config.py
+++ b/lib-python/modified-2.7/distutils/tests/test_config.py
@@ -11,6 +11,7 @@
 from distutils.log import WARN
 
 from distutils.tests import support
+from test.test_support import run_unittest
 
 PYPIRC = """\
 [distutils]
@@ -119,4 +120,4 @@
     return unittest.makeSuite(PyPIRCCommandTestCase)
 
 if __name__ == "__main__":
-    unittest.main(defaultTest="test_suite")
+    run_unittest(test_suite())
diff --git a/lib-python/modified-2.7/distutils/tests/test_config_cmd.py b/lib-python/modified-2.7/distutils/tests/test_config_cmd.py
--- a/lib-python/modified-2.7/distutils/tests/test_config_cmd.py
+++ b/lib-python/modified-2.7/distutils/tests/test_config_cmd.py
@@ -2,6 +2,7 @@
 import unittest
 import os
 import sys
+from test.test_support import run_unittest
 
 from distutils.command.config import dump_file, config
 from distutils.tests import support
@@ -86,4 +87,4 @@
     return unittest.makeSuite(ConfigTestCase)
 
 if __name__ == "__main__":
-    unittest.main(defaultTest="test_suite")
+    run_unittest(test_suite())
diff --git a/lib-python/modified-2.7/distutils/tests/test_core.py b/lib-python/modified-2.7/distutils/tests/test_core.py
--- a/lib-python/modified-2.7/distutils/tests/test_core.py
+++ b/lib-python/modified-2.7/distutils/tests/test_core.py
@@ -6,7 +6,7 @@
 import shutil
 import sys
 import test.test_support
-from test.test_support import captured_stdout
+from test.test_support import captured_stdout, run_unittest
 import unittest
 from distutils.tests import support
 
@@ -105,4 +105,4 @@
     return unittest.makeSuite(CoreTestCase)
 
 if __name__ == "__main__":
-    unittest.main(defaultTest="test_suite")
+    run_unittest(test_suite())
diff --git a/lib-python/modified-2.7/distutils/tests/test_dep_util.py b/lib-python/modified-2.7/distutils/tests/test_dep_util.py
--- a/lib-python/modified-2.7/distutils/tests/test_dep_util.py
+++ b/lib-python/modified-2.7/distutils/tests/test_dep_util.py
@@ -6,6 +6,7 @@
 from distutils.dep_util import newer, newer_pairwise, newer_group
 from distutils.errors import DistutilsFileError
 from distutils.tests import support
+from test.test_support import run_unittest
 
 class DepUtilTestCase(support.TempdirManager, unittest.TestCase):
 
@@ -77,4 +78,4 @@
     return unittest.makeSuite(DepUtilTestCase)
 
 if __name__ == "__main__":
-    unittest.main(defaultTest="test_suite")
+    run_unittest(test_suite())
diff --git a/lib-python/modified-2.7/distutils/tests/test_dir_util.py b/lib-python/modified-2.7/distutils/tests/test_dir_util.py
--- a/lib-python/modified-2.7/distutils/tests/test_dir_util.py
+++ b/lib-python/modified-2.7/distutils/tests/test_dir_util.py
@@ -10,6 +10,7 @@
 
 from distutils import log
 from distutils.tests import support
+from test.test_support import run_unittest
 
 class DirUtilTestCase(support.TempdirManager, unittest.TestCase):
 
@@ -112,4 +113,4 @@
     return unittest.makeSuite(DirUtilTestCase)
 
 if __name__ == "__main__":
-    unittest.main(defaultTest="test_suite")
+    run_unittest(test_suite())
diff --git a/lib-python/modified-2.7/distutils/tests/test_dist.py b/lib-python/modified-2.7/distutils/tests/test_dist.py
--- a/lib-python/modified-2.7/distutils/tests/test_dist.py
+++ b/lib-python/modified-2.7/distutils/tests/test_dist.py
@@ -11,7 +11,7 @@
 from distutils.dist import Distribution, fix_help_options, DistributionMetadata
 from distutils.cmd import Command
 import distutils.dist
-from test.test_support import TESTFN, captured_stdout
+from test.test_support import TESTFN, captured_stdout, run_unittest
 from distutils.tests import support
 
 class test_dist(Command):
@@ -433,4 +433,4 @@
     return suite
 
 if __name__ == "__main__":
-    unittest.main(defaultTest="test_suite")
+    run_unittest(test_suite())
diff --git a/lib-python/modified-2.7/distutils/tests/test_file_util.py b/lib-python/modified-2.7/distutils/tests/test_file_util.py
--- a/lib-python/modified-2.7/distutils/tests/test_file_util.py
+++ b/lib-python/modified-2.7/distutils/tests/test_file_util.py
@@ -6,6 +6,7 @@
 from distutils.file_util import move_file, write_file, copy_file
 from distutils import log
 from distutils.tests import support
+from test.test_support import run_unittest
 
 class FileUtilTestCase(support.TempdirManager, unittest.TestCase):
 
@@ -77,4 +78,4 @@
     return unittest.makeSuite(FileUtilTestCase)
 
 if __name__ == "__main__":
-    unittest.main(defaultTest="test_suite")
+    run_unittest(test_suite())
diff --git a/lib-python/modified-2.7/distutils/tests/test_filelist.py b/lib-python/modified-2.7/distutils/tests/test_filelist.py
--- a/lib-python/modified-2.7/distutils/tests/test_filelist.py
+++ b/lib-python/modified-2.7/distutils/tests/test_filelist.py
@@ -1,7 +1,7 @@
 """Tests for distutils.filelist."""
 from os.path import join
 import unittest
-from test.test_support import captured_stdout
+from test.test_support import captured_stdout, run_unittest
 
 from distutils.filelist import glob_to_re, FileList
 from distutils import debug
@@ -82,4 +82,4 @@
     return unittest.makeSuite(FileListTestCase)
 
 if __name__ == "__main__":
-    unittest.main(defaultTest="test_suite")
+    run_unittest(test_suite())
diff --git a/lib-python/modified-2.7/distutils/tests/test_install.py b/lib-python/modified-2.7/distutils/tests/test_install.py
--- a/lib-python/modified-2.7/distutils/tests/test_install.py
+++ b/lib-python/modified-2.7/distutils/tests/test_install.py
@@ -4,6 +4,8 @@
 import unittest
 from test import test_support
 
+from test.test_support import run_unittest
+
 from distutils.command.install import install
 from distutils.core import Distribution
 
@@ -54,4 +56,4 @@
     return unittest.makeSuite(InstallTestCase)
 
 if __name__ == "__main__":
-    unittest.main(defaultTest="test_suite")
+    run_unittest(test_suite())
diff --git a/lib-python/modified-2.7/distutils/tests/test_install_data.py b/lib-python/modified-2.7/distutils/tests/test_install_data.py
--- a/lib-python/modified-2.7/distutils/tests/test_install_data.py
+++ b/lib-python/modified-2.7/distutils/tests/test_install_data.py
@@ -6,6 +6,7 @@
 
 from distutils.command.install_data import install_data
 from distutils.tests import support
+from test.test_support import run_unittest
 
 class InstallDataTestCase(support.TempdirManager,
                           support.LoggingSilencer,
@@ -73,4 +74,4 @@
     return unittest.makeSuite(InstallDataTestCase)
 
 if __name__ == "__main__":
-    unittest.main(defaultTest="test_suite")
+    run_unittest(test_suite())
diff --git a/lib-python/modified-2.7/distutils/tests/test_install_headers.py b/lib-python/modified-2.7/distutils/tests/test_install_headers.py
--- a/lib-python/modified-2.7/distutils/tests/test_install_headers.py
+++ b/lib-python/modified-2.7/distutils/tests/test_install_headers.py
@@ -6,6 +6,7 @@
 
 from distutils.command.install_headers import install_headers
 from distutils.tests import support
+from test.test_support import run_unittest
 
 class InstallHeadersTestCase(support.TempdirManager,
                              support.LoggingSilencer,
@@ -37,4 +38,4 @@
     return unittest.makeSuite(InstallHeadersTestCase)
 
 if __name__ == "__main__":
-    unittest.main(defaultTest="test_suite")
+    run_unittest(test_suite())
diff --git a/lib-python/modified-2.7/distutils/tests/test_install_lib.py b/lib-python/modified-2.7/distutils/tests/test_install_lib.py
--- a/lib-python/modified-2.7/distutils/tests/test_install_lib.py
+++ b/lib-python/modified-2.7/distutils/tests/test_install_lib.py
@@ -7,6 +7,7 @@
 from distutils.extension import Extension
 from distutils.tests import support
 from distutils.errors import DistutilsOptionError
+from test.test_support import run_unittest
 
 class InstallLibTestCase(support.TempdirManager,
                          support.LoggingSilencer,
@@ -103,4 +104,4 @@
     return unittest.makeSuite(InstallLibTestCase)
 
 if __name__ == "__main__":
-    unittest.main(defaultTest="test_suite")
+    run_unittest(test_suite())
diff --git a/lib-python/modified-2.7/distutils/tests/test_install_scripts.py b/lib-python/modified-2.7/distutils/tests/test_install_scripts.py
--- a/lib-python/modified-2.7/distutils/tests/test_install_scripts.py
+++ b/lib-python/modified-2.7/distutils/tests/test_install_scripts.py
@@ -7,6 +7,7 @@
 from distutils.core import Distribution
 
 from distutils.tests import support
+from test.test_support import run_unittest
 
 
 class InstallScriptsTestCase(support.TempdirManager,
@@ -78,4 +79,4 @@
     return unittest.makeSuite(InstallScriptsTestCase)
 
 if __name__ == "__main__":
-    unittest.main(defaultTest="test_suite")
+    run_unittest(test_suite())
diff --git a/lib-python/modified-2.7/distutils/tests/test_msvc9compiler.py b/lib-python/modified-2.7/distutils/tests/test_msvc9compiler.py
--- a/lib-python/modified-2.7/distutils/tests/test_msvc9compiler.py
+++ b/lib-python/modified-2.7/distutils/tests/test_msvc9compiler.py
@@ -5,6 +5,7 @@
 
 from distutils.errors import DistutilsPlatformError
 from distutils.tests import support
+from test.test_support import run_unittest
 
 _MANIFEST = """\
 
@@ -137,4 +138,4 @@
     return unittest.makeSuite(msvc9compilerTestCase)
 
 if __name__ == "__main__":
-    unittest.main(defaultTest="test_suite")
+    run_unittest(test_suite())
diff --git a/lib-python/modified-2.7/distutils/tests/test_register.py b/lib-python/modified-2.7/distutils/tests/test_register.py
--- a/lib-python/modified-2.7/distutils/tests/test_register.py
+++ b/lib-python/modified-2.7/distutils/tests/test_register.py
@@ -7,7 +7,7 @@
 import urllib2
 import warnings
 
-from test.test_support import check_warnings
+from test.test_support import check_warnings, run_unittest
 
 from distutils.command import register as register_module
 from distutils.command.register import register
@@ -138,7 +138,7 @@
 
         # let's see what the server received : we should
         # have 2 similar requests
-        self.assertTrue(self.conn.reqs, 2)
+        self.assertEqual(len(self.conn.reqs), 2)
         req1 = dict(self.conn.reqs[0].headers)
         req2 = dict(self.conn.reqs[1].headers)
         self.assertEqual(req2['Content-length'], req1['Content-length'])
@@ -168,7 +168,7 @@
             del register_module.raw_input
 
         # we should have send a request
-        self.assertTrue(self.conn.reqs, 1)
+        self.assertEqual(len(self.conn.reqs), 1)
         req = self.conn.reqs[0]
         headers = dict(req.headers)
         self.assertEqual(headers['Content-length'], '608')
@@ -186,7 +186,7 @@
             del register_module.raw_input
 
         # we should have send a request
-        self.assertTrue(self.conn.reqs, 1)
+        self.assertEqual(len(self.conn.reqs), 1)
         req = self.conn.reqs[0]
         headers = dict(req.headers)
         self.assertEqual(headers['Content-length'], '290')
@@ -258,4 +258,4 @@
     return unittest.makeSuite(RegisterTestCase)
 
 if __name__ == "__main__":
-    unittest.main(defaultTest="test_suite")
+    run_unittest(test_suite())
diff --git a/lib-python/modified-2.7/distutils/tests/test_sdist.py b/lib-python/modified-2.7/distutils/tests/test_sdist.py
--- a/lib-python/modified-2.7/distutils/tests/test_sdist.py
+++ b/lib-python/modified-2.7/distutils/tests/test_sdist.py
@@ -24,11 +24,9 @@
 import tempfile
 import warnings
 
-from test.test_support import check_warnings
-from test.test_support import captured_stdout
+from test.test_support import captured_stdout, check_warnings, run_unittest
 
-from distutils.command.sdist import sdist
-from distutils.command.sdist import show_formats
+from distutils.command.sdist import sdist, show_formats
 from distutils.core import Distribution
 from distutils.tests.test_config import PyPIRCCommandTestCase
 from distutils.errors import DistutilsExecError, DistutilsOptionError
@@ -372,7 +370,7 @@
         # adding a file
         self.write_file((self.tmp_dir, 'somecode', 'doc2.txt'), '#')
 
-        # make sure build_py is reinitinialized, like a fresh run
+        # make sure build_py is reinitialized, like a fresh run
         build_py = dist.get_command_obj('build_py')
         build_py.finalized = False
         build_py.ensure_finalized()
@@ -390,6 +388,7 @@
         self.assertEqual(len(manifest2), 6)
         self.assertIn('doc2.txt', manifest2[-1])
 
+    @unittest.skipUnless(zlib, "requires zlib")
     def test_manifest_marker(self):
         # check that autogenerated MANIFESTs have a marker
         dist, cmd = self.get_cmd()
@@ -406,6 +405,7 @@
         self.assertEqual(manifest[0],
                          '# file GENERATED by distutils, do NOT edit')
 
+    @unittest.skipUnless(zlib, "requires zlib")
     def test_manual_manifest(self):
         # check that a MANIFEST without a marker is left alone
         dist, cmd = self.get_cmd()
@@ -426,4 +426,4 @@
     return unittest.makeSuite(SDistTestCase)
 
 if __name__ == "__main__":
-    unittest.main(defaultTest="test_suite")
+    run_unittest(test_suite())
diff --git a/lib-python/modified-2.7/distutils/tests/test_spawn.py b/lib-python/modified-2.7/distutils/tests/test_spawn.py
--- a/lib-python/modified-2.7/distutils/tests/test_spawn.py
+++ b/lib-python/modified-2.7/distutils/tests/test_spawn.py
@@ -2,7 +2,7 @@
 import unittest
 import os
 import time
-from test.test_support import captured_stdout
+from test.test_support import captured_stdout, run_unittest
 
 from distutils.spawn import _nt_quote_args
 from distutils.spawn import spawn, find_executable
@@ -57,4 +57,4 @@
     return unittest.makeSuite(SpawnTestCase)
 
 if __name__ == "__main__":
-    unittest.main(defaultTest="test_suite")
+    run_unittest(test_suite())
diff --git a/lib-python/modified-2.7/distutils/tests/test_text_file.py b/lib-python/modified-2.7/distutils/tests/test_text_file.py
--- a/lib-python/modified-2.7/distutils/tests/test_text_file.py
+++ b/lib-python/modified-2.7/distutils/tests/test_text_file.py
@@ -3,6 +3,7 @@
 import unittest
 from distutils.text_file import TextFile
 from distutils.tests import support
+from test.test_support import run_unittest
 
 TEST_DATA = """# test file
 
@@ -103,4 +104,4 @@
     return unittest.makeSuite(TextFileTestCase)
 
 if __name__ == "__main__":
-    unittest.main(defaultTest="test_suite")
+    run_unittest(test_suite())
diff --git a/lib-python/modified-2.7/distutils/tests/test_unixccompiler.py b/lib-python/modified-2.7/distutils/tests/test_unixccompiler.py
--- a/lib-python/modified-2.7/distutils/tests/test_unixccompiler.py
+++ b/lib-python/modified-2.7/distutils/tests/test_unixccompiler.py
@@ -1,6 +1,7 @@
 """Tests for distutils.unixccompiler."""
 import sys
 import unittest
+from test.test_support import run_unittest
 
 from distutils import sysconfig
 from distutils.unixccompiler import UnixCCompiler
@@ -126,4 +127,4 @@
     return unittest.makeSuite(UnixCCompilerTestCase)
 
 if __name__ == "__main__":
-    unittest.main(defaultTest="test_suite")
+    run_unittest(test_suite())
diff --git a/lib-python/modified-2.7/distutils/tests/test_upload.py b/lib-python/modified-2.7/distutils/tests/test_upload.py
--- a/lib-python/modified-2.7/distutils/tests/test_upload.py
+++ b/lib-python/modified-2.7/distutils/tests/test_upload.py
@@ -1,14 +1,13 @@
+# -*- encoding: utf8 -*-
 """Tests for distutils.command.upload."""
-# -*- encoding: utf8 -*-
-import sys
 import os
 import unittest
+from test.test_support import run_unittest
 
 from distutils.command import upload as upload_mod
 from distutils.command.upload import upload
 from distutils.core import Distribution
 
-from distutils.tests import support
 from distutils.tests.test_config import PYPIRC, PyPIRCCommandTestCase
 
 PYPIRC_LONG_PASSWORD = """\
@@ -129,4 +128,4 @@
     return unittest.makeSuite(uploadTestCase)
 
 if __name__ == "__main__":
-    unittest.main(defaultTest="test_suite")
+    run_unittest(test_suite())
diff --git a/lib-python/modified-2.7/distutils/tests/test_util.py b/lib-python/modified-2.7/distutils/tests/test_util.py
--- a/lib-python/modified-2.7/distutils/tests/test_util.py
+++ b/lib-python/modified-2.7/distutils/tests/test_util.py
@@ -1,6 +1,7 @@
 """Tests for distutils.util."""
 import sys
 import unittest
+from test.test_support import run_unittest
 
 from distutils.errors import DistutilsPlatformError, DistutilsByteCompileError
 from distutils.util import byte_compile
@@ -21,4 +22,4 @@
     return unittest.makeSuite(UtilTestCase)
 
 if __name__ == "__main__":
-    unittest.main(defaultTest="test_suite")
+    run_unittest(test_suite())
diff --git a/lib-python/modified-2.7/distutils/tests/test_version.py b/lib-python/modified-2.7/distutils/tests/test_version.py
--- a/lib-python/modified-2.7/distutils/tests/test_version.py
+++ b/lib-python/modified-2.7/distutils/tests/test_version.py
@@ -2,6 +2,7 @@
 import unittest
 from distutils.version import LooseVersion
 from distutils.version import StrictVersion
+from test.test_support import run_unittest
 
 class VersionTestCase(unittest.TestCase):
 
@@ -67,4 +68,4 @@
     return unittest.makeSuite(VersionTestCase)
 
 if __name__ == "__main__":
-    unittest.main(defaultTest="test_suite")
+    run_unittest(test_suite())
diff --git a/lib-python/modified-2.7/distutils/tests/test_versionpredicate.py b/lib-python/modified-2.7/distutils/tests/test_versionpredicate.py
--- a/lib-python/modified-2.7/distutils/tests/test_versionpredicate.py
+++ b/lib-python/modified-2.7/distutils/tests/test_versionpredicate.py
@@ -4,6 +4,10 @@
 
 import distutils.versionpredicate
 import doctest
+from test.test_support import run_unittest
 
 def test_suite():
     return doctest.DocTestSuite(distutils.versionpredicate)
+
+if __name__ == '__main__':
+    run_unittest(test_suite())
diff --git a/lib-python/modified-2.7/distutils/util.py b/lib-python/modified-2.7/distutils/util.py
--- a/lib-python/modified-2.7/distutils/util.py
+++ b/lib-python/modified-2.7/distutils/util.py
@@ -97,9 +97,7 @@
         from distutils.sysconfig import get_config_vars
         cfgvars = get_config_vars()
 
-        macver = os.environ.get('MACOSX_DEPLOYMENT_TARGET')
-        if not macver:
-            macver = cfgvars.get('MACOSX_DEPLOYMENT_TARGET')
+        macver = cfgvars.get('MACOSX_DEPLOYMENT_TARGET')
 
         if 1:
             # Always calculate the release of the running machine,
diff --git a/lib-python/modified-2.7/email/charset.py b/lib-python/modified-2.7/email/charset.py
--- a/lib-python/modified-2.7/email/charset.py
+++ b/lib-python/modified-2.7/email/charset.py
@@ -209,7 +209,7 @@
                 input_charset = unicode(input_charset, 'ascii')
         except UnicodeError:
             raise errors.CharsetError(input_charset)
-        input_charset = input_charset.lower()
+        input_charset = input_charset.lower().encode('ascii')
         # Set the input charset after filtering through the aliases and/or codecs
         if not (input_charset in ALIASES or input_charset in CHARSETS):
             try:
diff --git a/lib-python/modified-2.7/email/generator.py b/lib-python/modified-2.7/email/generator.py
--- a/lib-python/modified-2.7/email/generator.py
+++ b/lib-python/modified-2.7/email/generator.py
@@ -202,18 +202,13 @@
             g = self.clone(s)
             g.flatten(part, unixfrom=False)
             msgtexts.append(s.getvalue())
-        # Now make sure the boundary we've selected doesn't appear in any of
-        # the message texts.
-        alltext = NL.join(msgtexts)
         # BAW: What about boundaries that are wrapped in double-quotes?
-        boundary = msg.get_boundary(failobj=_make_boundary(alltext))
-        # If we had to calculate a new boundary because the body text
-        # contained that string, set the new boundary.  We don't do it
-        # unconditionally because, while set_boundary() preserves order, it
-        # doesn't preserve newlines/continuations in headers.  This is no big
-        # deal in practice, but turns out to be inconvenient for the unittest
-        # suite.
-        if msg.get_boundary() != boundary:
+        boundary = msg.get_boundary()
+        if not boundary:
+            # Create a boundary that doesn't appear in any of the
+            # message texts.
+            alltext = NL.join(msgtexts)
+            boundary = _make_boundary(alltext)
             msg.set_boundary(boundary)
         # If there's a preamble, write it out, with a trailing CRLF
         if msg.preamble is not None:
@@ -292,7 +287,7 @@
 _FMT = '[Non-text (%(type)s) part of message omitted, filename %(filename)s]'
 
 class DecodedGenerator(Generator):
-    """Generator a text representation of a message.
+    """Generates a text representation of a message.
 
     Like the Generator base class, except that non-text parts are substituted
     with a format string representing the part.
diff --git a/lib-python/modified-2.7/email/header.py b/lib-python/modified-2.7/email/header.py
--- a/lib-python/modified-2.7/email/header.py
+++ b/lib-python/modified-2.7/email/header.py
@@ -47,6 +47,10 @@
 # For use with .match()
 fcre = re.compile(r'[\041-\176]+:$')
 
+# Find a header embedded in a putative header value.  Used to check for
+# header injection attack.
+_embeded_header = re.compile(r'\n[^ \t]+:')
+
 
 
 # Helpers
@@ -403,7 +407,11 @@
             newchunks += self._split(s, charset, targetlen, splitchars)
             lastchunk, lastcharset = newchunks[-1]
             lastlen = lastcharset.encoded_header_len(lastchunk)
-        return self._encode_chunks(newchunks, maxlinelen)
+        value = self._encode_chunks(newchunks, maxlinelen)
+        if _embeded_header.search(value):
+            raise HeaderParseError("header value appears to contain "
+                "an embedded header: {!r}".format(value))
+        return value
 
 
 
diff --git a/lib-python/modified-2.7/email/message.py b/lib-python/modified-2.7/email/message.py
--- a/lib-python/modified-2.7/email/message.py
+++ b/lib-python/modified-2.7/email/message.py
@@ -38,7 +38,9 @@
 def _formatparam(param, value=None, quote=True):
     """Convenience function to format and return a key=value pair.
 
-    This will quote the value if needed or if quote is true.
+    This will quote the value if needed or if quote is true.  If value is a
+    three tuple (charset, language, value), it will be encoded according
+    to RFC2231 rules.
     """
     if value is not None and len(value) > 0:
         # A tuple is used for RFC 2231 encoded parameter values where items
@@ -97,7 +99,7 @@
     objects, otherwise it is a string.
 
     Message objects implement part of the `mapping' interface, which assumes
-    there is exactly one occurrance of the header per message.  Some headers
+    there is exactly one occurrence of the header per message.  Some headers
     do in fact appear multiple times (e.g. Received) and for those headers,
     you must use the explicit API to set or get all the headers.  Not all of
     the mapping methods are implemented.
@@ -286,7 +288,7 @@
         Return None if the header is missing instead of raising an exception.
 
         Note that if the header appeared multiple times, exactly which
-        occurrance gets returned is undefined.  Use get_all() to get all
+        occurrence gets returned is undefined.  Use get_all() to get all
         the values matching a header field name.
         """
         return self.get(name)
@@ -389,7 +391,10 @@
         name is the header field to add.  keyword arguments can be used to set
         additional parameters for the header field, with underscores converted
         to dashes.  Normally the parameter will be added as key="value" unless
-        value is None, in which case only the key will be added.
+        value is None, in which case only the key will be added.  If a
+        parameter value contains non-ASCII characters it must be specified as a
+        three-tuple of (charset, language, value), in which case it will be
+        encoded according to RFC2231 rules.
 
         Example:
 
diff --git a/lib-python/modified-2.7/email/mime/application.py b/lib-python/modified-2.7/email/mime/application.py
--- a/lib-python/modified-2.7/email/mime/application.py
+++ b/lib-python/modified-2.7/email/mime/application.py
@@ -17,7 +17,7 @@
                  _encoder=encoders.encode_base64, **_params):
         """Create an application/* type MIME document.
 
-        _data is a string containing the raw applicatoin data.
+        _data is a string containing the raw application data.
 
         _subtype is the MIME content type subtype, defaulting to
         'octet-stream'.
diff --git a/lib-python/modified-2.7/email/test/data/msg_26.txt b/lib-python/modified-2.7/email/test/data/msg_26.txt
--- a/lib-python/modified-2.7/email/test/data/msg_26.txt
+++ b/lib-python/modified-2.7/email/test/data/msg_26.txt
@@ -42,4 +42,4 @@
 MzMAAAAACH97tzAAAAALu3c3gAAAAAAL+7tzDABAu7f7cAAAAAAACA+3MA7EQAv/sIAA
 AAAAAAAIAAAAAAAAAIAAAAAA
 
---1618492860--2051301190--113853680--
+--1618492860--2051301190--113853680--
\ No newline at end of file
diff --git a/lib-python/modified-2.7/email/test/test_email.py b/lib-python/modified-2.7/email/test/test_email.py
--- a/lib-python/modified-2.7/email/test/test_email.py
+++ b/lib-python/modified-2.7/email/test/test_email.py
@@ -179,6 +179,17 @@
         self.assertRaises(Errors.HeaderParseError,
                           msg.set_boundary, 'BOUNDARY')
 
+    def test_make_boundary(self):
+        msg = MIMEMultipart('form-data')
+        # Note that when the boundary gets created is an implementation
+        # detail and might change.
+        self.assertEqual(msg.items()[0][1], 'multipart/form-data')
+        # Trigger creation of boundary
+        msg.as_string()
+        self.assertEqual(msg.items()[0][1][:33],
+                        'multipart/form-data; boundary="==')
+        # XXX: there ought to be tests of the uniqueness of the boundary, too.
+
     def test_message_rfc822_only(self):
         # Issue 7970: message/rfc822 not in multipart parsed by
         # HeaderParser caused an exception when flattened.
@@ -542,6 +553,17 @@
         msg.set_charset(u'us-ascii')
         self.assertEqual('us-ascii', msg.get_content_charset())
 
+    # Issue 5871: reject an attempt to embed a header inside a header value
+    # (header injection attack).
+    def test_embeded_header_via_Header_rejected(self):
+        msg = Message()
+        msg['Dummy'] = Header('dummy\nX-Injected-Header: test')
+        self.assertRaises(Errors.HeaderParseError, msg.as_string)
+
+    def test_embeded_header_via_string_rejected(self):
+        msg = Message()
+        msg['Dummy'] = 'dummy\nX-Injected-Header: test'
+        self.assertRaises(Errors.HeaderParseError, msg.as_string)
 
 
 # Test the email.Encoders module
@@ -3114,6 +3136,28 @@
         s = 'Subject: =?EUC-KR?B?CSixpLDtKSC/7Liuvsax4iC6uLmwMcijIKHaILzSwd/H0SC8+LCjwLsgv7W/+Mj3I ?='
         raises(Errors.HeaderParseError, decode_header, s)
 
+    # Issue 1078919
+    def test_ascii_add_header(self):
+        msg = Message()
+        msg.add_header('Content-Disposition', 'attachment',
+                       filename='bud.gif')
+        self.assertEqual('attachment; filename="bud.gif"',
+            msg['Content-Disposition'])
+
+    def test_nonascii_add_header_via_triple(self):
+        msg = Message()
+        msg.add_header('Content-Disposition', 'attachment',
+            filename=('iso-8859-1', '', 'Fu\xdfballer.ppt'))
+        self.assertEqual(
+            'attachment; filename*="iso-8859-1\'\'Fu%DFballer.ppt"',
+            msg['Content-Disposition'])
+
+    def test_encode_unaliased_charset(self):
+        # Issue 1379416: when the charset has no output conversion,
+        # output was accidentally getting coerced to unicode.
+        res = Header('abc','iso-8859-2').encode()
+        self.assertEqual(res, '=?iso-8859-2?q?abc?=')
+        self.assertIsInstance(res, str)
 
 
 # Test RFC 2231 header parameters (en/de)coding
diff --git a/lib-python/modified-2.7/heapq.py b/lib-python/modified-2.7/heapq.py
--- a/lib-python/modified-2.7/heapq.py
+++ b/lib-python/modified-2.7/heapq.py
@@ -133,6 +133,11 @@
 from operator import itemgetter
 import bisect
 
+def cmp_lt(x, y):
+    # Use __lt__ if available; otherwise, try __le__.
+    # In Py3.x, only __lt__ will be called.
+    return (x < y) if hasattr(x, '__lt__') else (not y <= x)
+
 def heappush(heap, item):
     """Push item onto heap, maintaining the heap invariant."""
     heap.append(item)
@@ -167,13 +172,13 @@
 
 def heappushpop(heap, item):
     """Fast version of a heappush followed by a heappop."""
-    if heap and heap[0] < item:
+    if heap and cmp_lt(heap[0], item):
         item, heap[0] = heap[0], item
         _siftup(heap, 0)
     return item
 
 def heapify(x):
-    """Transform list into a heap, in-place, in O(len(heap)) time."""
+    """Transform list into a heap, in-place, in O(len(x)) time."""
     n = len(x)
     # Transform bottom-up.  The largest index there's any point to looking at
     # is the largest with a child index in-range, so must have 2*i + 1 < n,
@@ -219,11 +224,10 @@
         pop = result.pop
         los = result[-1]    # los --> Largest of the nsmallest
         for elem in it:
-            if los <= elem:
-                continue
-            insort(result, elem)
-            pop()
-            los = result[-1]
+            if cmp_lt(elem, los):
+                insort(result, elem)
+                pop()
+                los = result[-1]
         return result
     # An alternative approach manifests the whole iterable in memory but
     # saves comparisons by heapifying all at once.  Also, saves time
@@ -244,7 +248,7 @@
     while pos > startpos:
         parentpos = (pos - 1) >> 1
         parent = heap[parentpos]
-        if newitem < parent:
+        if cmp_lt(newitem, parent):
             heap[pos] = parent
             pos = parentpos
             continue
@@ -299,7 +303,7 @@
     while childpos < endpos:
         # Set childpos to index of smaller child.
         rightpos = childpos + 1
-        if rightpos < endpos and not heap[childpos] < heap[rightpos]:
+        if rightpos < endpos and not cmp_lt(heap[childpos], heap[rightpos]):
             childpos = rightpos
         # Move the smaller child up.
         heap[pos] = heap[childpos]
@@ -368,7 +372,7 @@
             return [min(chain(head, it))]
         return [min(chain(head, it), key=key)]
 
-    # When n>=size, it's faster to use sort()
+    # When n>=size, it's faster to use sorted()
     try:
         size = len(iterable)
     except (TypeError, AttributeError):
@@ -406,7 +410,7 @@
             return [max(chain(head, it))]
         return [max(chain(head, it), key=key)]
 
-    # When n>=size, it's faster to use sort()
+    # When n>=size, it's faster to use sorted()
     try:
         size = len(iterable)
     except (TypeError, AttributeError):
diff --git a/lib-python/modified-2.7/httplib.py b/lib-python/modified-2.7/httplib.py
--- a/lib-python/modified-2.7/httplib.py
+++ b/lib-python/modified-2.7/httplib.py
@@ -212,6 +212,9 @@
 # maximal amount of data to read at one time in _safe_read
 MAXAMOUNT = 1048576
 
+# maximal line length when calling readline().
+_MAXLINE = 65536
+
 class HTTPMessage(mimetools.Message):
 
     def addheader(self, key, value):
@@ -274,7 +277,9 @@
                 except IOError:
                     startofline = tell = None
                     self.seekable = 0
-            line = self.fp.readline()
+            line = self.fp.readline(_MAXLINE + 1)
+            if len(line) > _MAXLINE:
+                raise LineTooLong("header line")
             if not line:
                 self.status = 'EOF in headers'
                 break
@@ -404,7 +409,10 @@
                 break
             # skip the header from the 100 response
             while True:
-                skip = self.fp.readline().strip()
+                skip = self.fp.readline(_MAXLINE + 1)
+                if len(skip) > _MAXLINE:
+                    raise LineTooLong("header line")
+                skip = skip.strip()
                 if not skip:
                     break
                 if self.debuglevel > 0:
@@ -563,7 +571,9 @@
         value = []
         while True:
             if chunk_left is None:
-                line = self.fp.readline()
+                line = self.fp.readline(_MAXLINE + 1)
+                if len(line) > _MAXLINE:
+                    raise LineTooLong("chunk size")
                 i = line.find(';')
                 if i >= 0:
                     line = line[:i] # strip chunk-extensions
@@ -598,7 +608,9 @@
         # read and discard trailer up to the CRLF terminator
         ### note: we shouldn't have any trailers!
         while True:
-            line = self.fp.readline()
+            line = self.fp.readline(_MAXLINE + 1)
+            if len(line) > _MAXLINE:
+                raise LineTooLong("trailer line")
             if not line:
                 # a vanishingly small number of sites EOF without
                 # sending the trailer
@@ -730,7 +742,9 @@
             raise socket.error("Tunnel connection failed: %d %s" % (code,
                                                                     message.strip()))
         while True:
-            line = response.fp.readline()
+            line = response.fp.readline(_MAXLINE + 1)
+            if len(line) > _MAXLINE:
+                raise LineTooLong("header line")
             if line == '\r\n': break
 
 
@@ -790,7 +804,7 @@
         del self._buffer[:]
         # If msg and message_body are sent in a single send() call,
         # it will avoid performance problems caused by the interaction
-        # between delayed ack and the Nagle algorithim.
+        # between delayed ack and the Nagle algorithm.
         if isinstance(message_body, str):
             msg += message_body
             message_body = None
@@ -1237,6 +1251,11 @@
         self.args = line,
         self.line = line
 
+class LineTooLong(HTTPException):
+    def __init__(self, line_type):
+        HTTPException.__init__(self, "got more than %d bytes when reading %s"
+                                     % (_MAXLINE, line_type))
+
 # for backwards compatibility
 error = HTTPException
 
diff --git a/lib-python/modified-2.7/idlelib/Bindings.py b/lib-python/modified-2.7/idlelib/Bindings.py
--- a/lib-python/modified-2.7/idlelib/Bindings.py
+++ b/lib-python/modified-2.7/idlelib/Bindings.py
@@ -98,14 +98,6 @@
     # menu
     del menudefs[-1][1][0:2]
 
-    menudefs.insert(0,
-            ('application', [
-                ('About IDLE', '<>'),
-                None,
-                ('_Preferences....', '<>'),
-            ]))
-
-
 default_keydefs = idleConf.GetCurrentKeySet()
 
 del sys
diff --git a/lib-python/modified-2.7/idlelib/EditorWindow.py b/lib-python/modified-2.7/idlelib/EditorWindow.py
--- a/lib-python/modified-2.7/idlelib/EditorWindow.py
+++ b/lib-python/modified-2.7/idlelib/EditorWindow.py
@@ -48,6 +48,21 @@
             path = module.__path__
         except AttributeError:
             raise ImportError, 'No source for module ' + module.__name__
+    if descr[2] != imp.PY_SOURCE:
+        # If all of the above fails and didn't raise an exception,fallback
+        # to a straight import which can find __init__.py in a package.
+        m = __import__(fullname)
+        try:
+            filename = m.__file__
+        except AttributeError:
+            pass
+        else:
+            file = None
+            base, ext = os.path.splitext(filename)
+            if ext == '.pyc':
+                ext = '.py'
+            filename = base + ext
+            descr = filename, None, imp.PY_SOURCE
     return file, filename, descr
 
 class EditorWindow(object):
@@ -102,8 +117,8 @@
         self.top = top = WindowList.ListedToplevel(root, menu=self.menubar)
         if flist:
             self.tkinter_vars = flist.vars
-            #self.top.instance_dict makes flist.inversedict avalable to
-            #configDialog.py so it can access all EditorWindow instaces
+            #self.top.instance_dict makes flist.inversedict available to
+            #configDialog.py so it can access all EditorWindow instances
             self.top.instance_dict = flist.inversedict
         else:
             self.tkinter_vars = {}  # keys: Tkinter event names
@@ -136,6 +151,14 @@
         if macosxSupport.runningAsOSXApp():
             # Command-W on editorwindows doesn't work without this.
             text.bind('<>', self.close_event)
+            # Some OS X systems have only one mouse button,
+            # so use control-click for pulldown menus there.
+            #  (Note, AquaTk defines <2> as the right button if
+            #   present and the Tk Text widget already binds <2>.)
+            text.bind("",self.right_menu_event)
+        else:
+            # Elsewhere, use right-click for pulldown menus.
+            text.bind("<3>",self.right_menu_event)
         text.bind("<>", self.cut)
         text.bind("<>", self.copy)
         text.bind("<>", self.paste)
@@ -154,7 +177,6 @@
         text.bind("<>", self.find_selection_event)
         text.bind("<>", self.replace_event)
         text.bind("<>", self.goto_line_event)
-        text.bind("<3>", self.right_menu_event)
         text.bind("<>",self.smart_backspace_event)
         text.bind("<>",self.newline_and_indent_event)
         text.bind("<>",self.smart_indent_event)
@@ -300,13 +322,13 @@
         return "break"
 
     def home_callback(self, event):
-        if (event.state & 12) != 0 and event.keysym == "Home":
-            # state&1==shift, state&4==control, state&8==alt
-            return # ; fall back to class binding
-
+        if (event.state & 4) != 0 and event.keysym == "Home":
+            # state&4==Control. If , use the Tk binding.
+            return
         if self.text.index("iomark") and \
            self.text.compare("iomark", "<=", "insert lineend") and \
            self.text.compare("insert linestart", "<=", "iomark"):
+            # In Shell on input line, go to just after prompt
             insertpt = int(self.text.index("iomark").split(".")[1])
         else:
             line = self.text.get("insert linestart", "insert lineend")
@@ -315,30 +337,27 @@
                     break
             else:
                 insertpt=len(line)
-
         lineat = int(self.text.index("insert").split('.')[1])
-
         if insertpt == lineat:
             insertpt = 0
-
         dest = "insert linestart+"+str(insertpt)+"c"
-
         if (event.state&1) == 0:
-            # shift not pressed
+            # shift was not pressed
             self.text.tag_remove("sel", "1.0", "end")
         else:
             if not self.text.index("sel.first"):
-                self.text.mark_set("anchor","insert")
-
+                self.text.mark_set("my_anchor", "insert")  # there was no previous selection
+            else:
+                if self.text.compare(self.text.index("sel.first"), "<", self.text.index("insert")):
+                    self.text.mark_set("my_anchor", "sel.first") # extend back
+                else:
+                    self.text.mark_set("my_anchor", "sel.last") # extend forward
             first = self.text.index(dest)
-            last = self.text.index("anchor")
-
+            last = self.text.index("my_anchor")
             if self.text.compare(first,">",last):
                 first,last = last,first
-
             self.text.tag_remove("sel", "1.0", "end")
             self.text.tag_add("sel", first, last)
-
         self.text.mark_set("insert", dest)
         self.text.see("insert")
         return "break"
@@ -385,7 +404,7 @@
             menudict[name] = menu = Menu(mbar, name=name)
             mbar.add_cascade(label=label, menu=menu, underline=underline)
 
-        if macosxSupport.runningAsOSXApp():
+        if macosxSupport.isCarbonAquaTk(self.root):
             # Insert the application menu
             menudict['application'] = menu = Menu(mbar, name='apple')
             mbar.add_cascade(label='IDLE', menu=menu)
@@ -445,7 +464,11 @@
 
     def python_docs(self, event=None):
         if sys.platform[:3] == 'win':
-            os.startfile(self.help_url)
+            try:
+                os.startfile(self.help_url)
+            except WindowsError as why:
+                tkMessageBox.showerror(title='Document Start Failure',
+                    message=str(why), parent=self.text)
         else:
             webbrowser.open(self.help_url)
         return "break"
@@ -740,9 +763,13 @@
         "Create a callback with the helpfile value frozen at definition time"
         def display_extra_help(helpfile=helpfile):
             if not helpfile.startswith(('www', 'http')):
-                url = os.path.normpath(helpfile)
+                helpfile = os.path.normpath(helpfile)
             if sys.platform[:3] == 'win':
-                os.startfile(helpfile)
+                try:
+                    os.startfile(helpfile)
+                except WindowsError as why:
+                    tkMessageBox.showerror(title='Document Start Failure',
+                        message=str(why), parent=self.text)
             else:
                 webbrowser.open(helpfile)
         return display_extra_help
@@ -1526,7 +1553,12 @@
 
 def get_accelerator(keydefs, eventname):
     keylist = keydefs.get(eventname)
-    if not keylist:
+    # issue10940: temporary workaround to prevent hang with OS X Cocoa Tk 8.5
+    # if not keylist:
+    if (not keylist) or (macosxSupport.runningAsOSXApp() and eventname in {
+                            "<>",
+                            "<>",
+                            "<>"}):
         return ""
     s = keylist[0]
     s = re.sub(r"-[a-z]\b", lambda m: m.group().upper(), s)
diff --git a/lib-python/modified-2.7/idlelib/FileList.py b/lib-python/modified-2.7/idlelib/FileList.py
--- a/lib-python/modified-2.7/idlelib/FileList.py
+++ b/lib-python/modified-2.7/idlelib/FileList.py
@@ -43,7 +43,7 @@
     def new(self, filename=None):
         return self.EditorWindow(self, filename)
 
-    def close_all_callback(self, event):
+    def close_all_callback(self, *args, **kwds):
         for edit in self.inversedict.keys():
             reply = edit.close()
             if reply == "cancel":
diff --git a/lib-python/modified-2.7/idlelib/FormatParagraph.py b/lib-python/modified-2.7/idlelib/FormatParagraph.py
--- a/lib-python/modified-2.7/idlelib/FormatParagraph.py
+++ b/lib-python/modified-2.7/idlelib/FormatParagraph.py
@@ -54,7 +54,7 @@
             # If the block ends in a \n, we dont want the comment
             # prefix inserted after it. (Im not sure it makes sense to
             # reformat a comment block that isnt made of complete
-            # lines, but whatever!)  Can't think of a clean soltution,
+            # lines, but whatever!)  Can't think of a clean solution,
             # so we hack away
             block_suffix = ""
             if not newdata[-1]:
diff --git a/lib-python/modified-2.7/idlelib/HISTORY.txt b/lib-python/modified-2.7/idlelib/HISTORY.txt
--- a/lib-python/modified-2.7/idlelib/HISTORY.txt
+++ b/lib-python/modified-2.7/idlelib/HISTORY.txt
@@ -13,7 +13,7 @@
 - New tarball released as a result of the 'revitalisation' of the IDLEfork
   project. 
 
-- This release requires python 2.1 or better. Compatability with earlier
+- This release requires python 2.1 or better. Compatibility with earlier
   versions of python (especially ancient ones like 1.5x) is no longer a
   priority in IDLEfork development.
 
diff --git a/lib-python/modified-2.7/idlelib/IOBinding.py b/lib-python/modified-2.7/idlelib/IOBinding.py
--- a/lib-python/modified-2.7/idlelib/IOBinding.py
+++ b/lib-python/modified-2.7/idlelib/IOBinding.py
@@ -320,17 +320,20 @@
             return "yes"
         message = "Do you want to save %s before closing?" % (
             self.filename or "this untitled document")
-        m = tkMessageBox.Message(
-            title="Save On Close",
-            message=message,
-            icon=tkMessageBox.QUESTION,
-            type=tkMessageBox.YESNOCANCEL,
-            master=self.text)
-        reply = m.show()
-        if reply == "yes":
+        confirm = tkMessageBox.askyesnocancel(
+                  title="Save On Close",
+                  message=message,
+                  default=tkMessageBox.YES,
+                  master=self.text)
+        if confirm:
+            reply = "yes"
             self.save(None)
             if not self.get_saved():
                 reply = "cancel"
+        elif confirm is None:
+            reply = "cancel"
+        else:
+            reply = "no"
         self.text.focus_set()
         return reply
 
@@ -339,7 +342,7 @@
             self.save_as(event)
         else:
             if self.writefile(self.filename):
-                self.set_saved(1)
+                self.set_saved(True)
                 try:
                     self.editwin.store_file_breaks()
                 except AttributeError:  # may be a PyShell
@@ -465,15 +468,12 @@
             self.text.insert("end-1c", "\n")
 
     def print_window(self, event):
-        m = tkMessageBox.Message(
-            title="Print",
-            message="Print to Default Printer",
-            icon=tkMessageBox.QUESTION,
-            type=tkMessageBox.OKCANCEL,
-            default=tkMessageBox.OK,
-            master=self.text)
-        reply = m.show()
-        if reply != tkMessageBox.OK:
+        confirm = tkMessageBox.askokcancel(
+                  title="Print",
+                  message="Print to Default Printer",
+                  default=tkMessageBox.OK,
+                  master=self.text)
+        if not confirm:
             self.text.focus_set()
             return "break"
         tempfilename = None
@@ -488,8 +488,8 @@
             if not self.writefile(tempfilename):
                 os.unlink(tempfilename)
                 return "break"
-        platform=os.name
-        printPlatform=1
+        platform = os.name
+        printPlatform = True
         if platform == 'posix': #posix platform
             command = idleConf.GetOption('main','General',
                                          'print-command-posix')
@@ -497,7 +497,7 @@
         elif platform == 'nt': #win32 platform
             command = idleConf.GetOption('main','General','print-command-win')
         else: #no printing for this platform
-            printPlatform=0
+            printPlatform = False
         if printPlatform:  #we can try to print for this platform
             command = command % filename
             pipe = os.popen(command, "r")
@@ -511,7 +511,7 @@
                 output = "Printing command: %s\n" % repr(command) + output
                 tkMessageBox.showerror("Print status", output, master=self.text)
         else:  #no printing for this platform
-            message="Printing is not enabled for this platform: %s" % platform
+            message = "Printing is not enabled for this platform: %s" % platform
             tkMessageBox.showinfo("Print status", message, master=self.text)
         if tempfilename:
             os.unlink(tempfilename)
diff --git a/lib-python/modified-2.7/idlelib/NEWS.txt b/lib-python/modified-2.7/idlelib/NEWS.txt
--- a/lib-python/modified-2.7/idlelib/NEWS.txt
+++ b/lib-python/modified-2.7/idlelib/NEWS.txt
@@ -1,3 +1,18 @@
+What's New in IDLE 2.7.2?
+=======================
+
+*Release date: 29-May-2011*
+
+- Issue #6378: Further adjust idle.bat to start associated Python
+
+- Issue #11896: Save on Close failed despite selecting "Yes" in dialog.
+
+-  toggle failing on Tk 8.5, causing IDLE exits and strange selection
+  behavior. Issue 4676.  Improve selection extension behaviour.
+
+-  toggle non-functional when NumLock set on Windows.  Issue 3851.
+
+
 What's New in IDLE 2.7?
 =======================
 
@@ -21,7 +36,7 @@
 
 - Tk 8.5 Text widget requires 'wordprocessor' tabstyle attr to handle
   mixed space/tab properly. Issue 5129, patch by Guilherme Polo.
-  
+
 - Issue #3549: On MacOS the preferences menu was not present
 
 
diff --git a/lib-python/modified-2.7/idlelib/PyShell.py b/lib-python/modified-2.7/idlelib/PyShell.py
--- a/lib-python/modified-2.7/idlelib/PyShell.py
+++ b/lib-python/modified-2.7/idlelib/PyShell.py
@@ -1432,6 +1432,13 @@
             shell.interp.prepend_syspath(script)
             shell.interp.execfile(script)
 
+    # Check for problematic OS X Tk versions and print a warning message
+    # in the IDLE shell window; this is less intrusive than always opening
+    # a separate window.
+    tkversionwarning = macosxSupport.tkVersionWarning(root)
+    if tkversionwarning:
+        shell.interp.runcommand(''.join(("print('", tkversionwarning, "')")))
+
     root.mainloop()
     root.destroy()
 
diff --git a/lib-python/modified-2.7/idlelib/ScriptBinding.py b/lib-python/modified-2.7/idlelib/ScriptBinding.py
--- a/lib-python/modified-2.7/idlelib/ScriptBinding.py
+++ b/lib-python/modified-2.7/idlelib/ScriptBinding.py
@@ -26,6 +26,7 @@
 from idlelib import PyShell
 
 from idlelib.configHandler import idleConf
+from idlelib import macosxSupport
 
 IDENTCHARS = string.ascii_letters + string.digits + "_"
 
@@ -53,6 +54,9 @@
         self.flist = self.editwin.flist
         self.root = self.editwin.root
 
+        if macosxSupport.runningAsOSXApp():
+            self.editwin.text_frame.bind('<>', self._run_module_event)
+
     def check_module_event(self, event):
         filename = self.getfilename()
         if not filename:
@@ -166,6 +170,19 @@
         interp.runcode(code)
         return 'break'
 
+    if macosxSupport.runningAsOSXApp():
+        # Tk-Cocoa in MacOSX is broken until at least
+        # Tk 8.5.9, and without this rather
+        # crude workaround IDLE would hang when a user
+        # tries to run a module using the keyboard shortcut
+        # (the menu item works fine).
+        _run_module_event = run_module_event
+
+        def run_module_event(self, event):
+            self.editwin.text_frame.after(200,
+                lambda: self.editwin.text_frame.event_generate('<>'))
+            return 'break'
+
     def getfilename(self):
         """Get source filename.  If not saved, offer to save (or create) file
 
@@ -184,9 +201,9 @@
             if autosave and filename:
                 self.editwin.io.save(None)
             else:
-                reply = self.ask_save_dialog()
+                confirm = self.ask_save_dialog()
                 self.editwin.text.focus_set()
-                if reply == "ok":
+                if confirm:
                     self.editwin.io.save(None)
                     filename = self.editwin.io.filename
                 else:
@@ -195,13 +212,11 @@
 
     def ask_save_dialog(self):
         msg = "Source Must Be Saved\n" + 5*' ' + "OK to Save?"
-        mb = tkMessageBox.Message(title="Save Before Run or Check",
-                                  message=msg,
-                                  icon=tkMessageBox.QUESTION,
-                                  type=tkMessageBox.OKCANCEL,
-                                  default=tkMessageBox.OK,
-                                  master=self.editwin.text)
-        return mb.show()
+        confirm = tkMessageBox.askokcancel(title="Save Before Run or Check",
+                                           message=msg,
+                                           default=tkMessageBox.OK,
+                                           master=self.editwin.text)
+        return confirm
 
     def errorbox(self, title, message):
         # XXX This should really be a function of EditorWindow...
diff --git a/lib-python/modified-2.7/idlelib/config-keys.def b/lib-python/modified-2.7/idlelib/config-keys.def
--- a/lib-python/modified-2.7/idlelib/config-keys.def
+++ b/lib-python/modified-2.7/idlelib/config-keys.def
@@ -176,7 +176,7 @@
 redo = 
 close-window = 
 restart-shell = 
-save-window-as-file = 
+save-window-as-file = 
 close-all-windows = 
 view-restart = 
 tabify-region = 
@@ -208,7 +208,7 @@
 open-module = 
 find-selection = 
 python-context-help = 
-save-copy-of-window-as-file = 
+save-copy-of-window-as-file = 
 open-window-from-file = 
 python-docs = 
 
diff --git a/lib-python/modified-2.7/idlelib/extend.txt b/lib-python/modified-2.7/idlelib/extend.txt
--- a/lib-python/modified-2.7/idlelib/extend.txt
+++ b/lib-python/modified-2.7/idlelib/extend.txt
@@ -18,7 +18,7 @@
 
 An IDLE extension class is instantiated with a single argument,
 `editwin', an EditorWindow instance. The extension cannot assume much
-about this argument, but it is guarateed to have the following instance
+about this argument, but it is guaranteed to have the following instance
 variables:
 
     text	a Text instance (a widget)
diff --git a/lib-python/modified-2.7/idlelib/idle.bat b/lib-python/modified-2.7/idlelib/idle.bat
--- a/lib-python/modified-2.7/idlelib/idle.bat
+++ b/lib-python/modified-2.7/idlelib/idle.bat
@@ -1,4 +1,4 @@
 @echo off
 rem Start IDLE using the appropriate Python interpreter
 set CURRDIR=%~dp0
-start "%CURRDIR%..\..\pythonw.exe" "%CURRDIR%idle.pyw" %1 %2 %3 %4 %5 %6 %7 %8 %9
+start "IDLE" "%CURRDIR%..\..\pythonw.exe" "%CURRDIR%idle.pyw" %1 %2 %3 %4 %5 %6 %7 %8 %9
diff --git a/lib-python/modified-2.7/idlelib/idlever.py b/lib-python/modified-2.7/idlelib/idlever.py
--- a/lib-python/modified-2.7/idlelib/idlever.py
+++ b/lib-python/modified-2.7/idlelib/idlever.py
@@ -1,1 +1,1 @@
-IDLE_VERSION = "2.7.1"
+IDLE_VERSION = "2.7.2"
diff --git a/lib-python/modified-2.7/idlelib/macosxSupport.py b/lib-python/modified-2.7/idlelib/macosxSupport.py
--- a/lib-python/modified-2.7/idlelib/macosxSupport.py
+++ b/lib-python/modified-2.7/idlelib/macosxSupport.py
@@ -4,6 +4,7 @@
 """
 import sys
 import Tkinter
+from os import path
 
 
 _appbundle = None
@@ -19,10 +20,41 @@
         _appbundle = (sys.platform == 'darwin' and '.app' in sys.executable)
     return _appbundle
 
+_carbonaquatk = None
+
+def isCarbonAquaTk(root):
+    """
+    Returns True if IDLE is using a Carbon Aqua Tk (instead of the
+    newer Cocoa Aqua Tk).
+    """
+    global _carbonaquatk
+    if _carbonaquatk is None:
+        _carbonaquatk = (runningAsOSXApp() and
+                         'aqua' in root.tk.call('tk', 'windowingsystem') and
+                         'AppKit' not in root.tk.call('winfo', 'server', '.'))
+    return _carbonaquatk
+
+def tkVersionWarning(root):
+    """
+    Returns a string warning message if the Tk version in use appears to
+    be one known to cause problems with IDLE.  The Apple Cocoa-based Tk 8.5
+    that was shipped with Mac OS X 10.6.
+    """
+
+    if (runningAsOSXApp() and
+            ('AppKit' in root.tk.call('winfo', 'server', '.')) and
+            (root.tk.call('info', 'patchlevel') == '8.5.7') ):
+        return (r"WARNING: The version of Tcl/Tk (8.5.7) in use may"
+                r" be unstable.\n"
+                r"Visit http://www.python.org/download/mac/tcltk/"
+                r" for current information.")
+    else:
+        return False
+
 def addOpenEventSupport(root, flist):
     """
-    This ensures that the application will respont to open AppleEvents, which
-    makes is feaseable to use IDLE as the default application for python files.
+    This ensures that the application will respond to open AppleEvents, which
+    makes is feasible to use IDLE as the default application for python files.
     """
     def doOpenFile(*args):
         for fn in args:
@@ -79,9 +111,6 @@
         WindowList.add_windows_to_menu(menu)
     WindowList.register_callback(postwindowsmenu)
 
-    menudict['application'] = menu = Menu(menubar, name='apple')
-    menubar.add_cascade(label='IDLE', menu=menu)
-
     def about_dialog(event=None):
         from idlelib import aboutDialog
         aboutDialog.AboutDialog(root, 'About IDLE')
@@ -91,41 +120,45 @@
         root.instance_dict = flist.inversedict
         configDialog.ConfigDialog(root, 'Settings')
 
+    def help_dialog(event=None):
+        from idlelib import textView
+        fn = path.join(path.abspath(path.dirname(__file__)), 'help.txt')
+        textView.view_file(root, 'Help', fn)
 
     root.bind('<>', about_dialog)
     root.bind('<>', config_dialog)
+    root.createcommand('::tk::mac::ShowPreferences', config_dialog)
     if flist:
         root.bind('<>', flist.close_all_callback)
 
+        # The binding above doesn't reliably work on all versions of Tk
+        # on MacOSX. Adding command definition below does seem to do the
+        # right thing for now.
+        root.createcommand('exit', flist.close_all_callback)
 
-    ###check if Tk version >= 8.4.14; if so, use hard-coded showprefs binding
-    tkversion = root.tk.eval('info patchlevel')
-    # Note: we cannot check if the string tkversion >= '8.4.14', because
-    # the string '8.4.7' is greater than the string '8.4.14'.
-    if tuple(map(int, tkversion.split('.'))) >= (8, 4, 14):
-        Bindings.menudefs[0] =  ('application', [
+    if isCarbonAquaTk(root):
+        # for Carbon AquaTk, replace the default Tk apple menu
+        menudict['application'] = menu = Menu(menubar, name='apple')
+        menubar.add_cascade(label='IDLE', menu=menu)
+        Bindings.menudefs.insert(0,
+            ('application', [
                 ('About IDLE', '<>'),
-                None,
-            ])
-        root.createcommand('::tk::mac::ShowPreferences', config_dialog)
+                    None,
+                ]))
+        tkversion = root.tk.eval('info patchlevel')
+        if tuple(map(int, tkversion.split('.'))) < (8, 4, 14):
+            # for earlier AquaTk versions, supply a Preferences menu item
+            Bindings.menudefs[0][1].append(
+                    ('_Preferences....', '<>'),
+                )
     else:
-        for mname, entrylist in Bindings.menudefs:
-            menu = menudict.get(mname)
-            if not menu:
-                continue
-            else:
-                for entry in entrylist:
-                    if not entry:
-                        menu.add_separator()
-                    else:
-                        label, eventname = entry
-                        underline, label = prepstr(label)
-                        accelerator = get_accelerator(Bindings.default_keydefs,
-                        eventname)
-                        def command(text=root, eventname=eventname):
-                            text.event_generate(eventname)
-                        menu.add_command(label=label, underline=underline,
-                        command=command, accelerator=accelerator)
+        # assume Cocoa AquaTk
+        # replace default About dialog with About IDLE one
+        root.createcommand('tkAboutDialog', about_dialog)
+        # replace default "Help" item in Help menu
+        root.createcommand('::tk::mac::ShowHelp', help_dialog)
+        # remove redundant "IDLE Help" from menu
+        del Bindings.menudefs[-1][1][0]
 
 def setupApp(root, flist):
     """
diff --git a/lib-python/modified-2.7/inspect.py b/lib-python/modified-2.7/inspect.py
--- a/lib-python/modified-2.7/inspect.py
+++ b/lib-python/modified-2.7/inspect.py
@@ -952,8 +952,14 @@
             f_name, 'at most' if defaults else 'exactly', num_args,
             'arguments' if num_args > 1 else 'argument', num_total))
     elif num_args == 0 and num_total:
-        raise TypeError('%s() takes no argument (%d given)' %
-                        (f_name, num_total))
+        if varkw:
+            if num_pos:
+                # XXX: We should use num_pos, but Python also uses num_total:
+                raise TypeError('%s() takes exactly 0 arguments '
+                                '(%d given)' % (f_name, num_total))
+        else:
+            raise TypeError('%s() takes no argument (%d given)' %
+                            (f_name, num_total))
     for arg in args:
         if isinstance(arg, str) and arg in named:
             if is_assigned(arg):
diff --git a/lib-python/modified-2.7/json/decoder.py b/lib-python/modified-2.7/json/decoder.py
--- a/lib-python/modified-2.7/json/decoder.py
+++ b/lib-python/modified-2.7/json/decoder.py
@@ -4,7 +4,7 @@
 import sys
 import struct
 
-from json.scanner import make_scanner
+from json import scanner
 try:
     from _json import scanstring as c_scanstring
 except ImportError:
@@ -356,7 +356,7 @@
         self.parse_object = JSONObject
         self.parse_array = JSONArray
         self.parse_string = scanstring
-        self.scan_once = make_scanner(self)
+        self.scan_once = scanner.make_scanner(self)
 
     def decode(self, s, _w=WHITESPACE.match):
         """Return the Python representation of ``s`` (a ``str`` or ``unicode``
diff --git a/lib-python/modified-2.7/json/tests/__init__.py b/lib-python/modified-2.7/json/tests/__init__.py
--- a/lib-python/modified-2.7/json/tests/__init__.py
+++ b/lib-python/modified-2.7/json/tests/__init__.py
@@ -1,7 +1,46 @@
 import os
 import sys
+import json
+import doctest
 import unittest
-import doctest
+
+from test import test_support
+
+# import json with and without accelerations
+cjson = test_support.import_fresh_module('json', fresh=['_json'])
+pyjson = test_support.import_fresh_module('json', blocked=['_json'])
+
+# create two base classes that will be used by the other tests
+class PyTest(unittest.TestCase):
+    json = pyjson
+    loads = staticmethod(pyjson.loads)
+    dumps = staticmethod(pyjson.dumps)
+
+ at unittest.skipUnless(cjson, 'requires _json')
+class CTest(unittest.TestCase):
+    if cjson is not None:
+        json = cjson
+        loads = staticmethod(cjson.loads)
+        dumps = staticmethod(cjson.dumps)
+
+# test PyTest and CTest checking if the functions come from the right module
+class TestPyTest(PyTest):
+    def test_pyjson(self):
+        self.assertEqual(self.json.scanner.make_scanner.__module__,
+                         'json.scanner')
+        self.assertEqual(self.json.decoder.scanstring.__module__,
+                         'json.decoder')
+        self.assertEqual(self.json.encoder.encode_basestring_ascii.__module__,
+                         'json.encoder')
+
+class TestCTest(CTest):
+    def test_cjson(self):
+        self.assertEqual(self.json.scanner.make_scanner.__module__, '_json')
+        self.assertEqual(self.json.decoder.scanstring.__module__, '_json')
+        self.assertEqual(self.json.encoder.c_make_encoder.__module__, '_json')
+        self.assertEqual(self.json.encoder.encode_basestring_ascii.__module__,
+                         '_json')
+
 
 here = os.path.dirname(__file__)
 
@@ -17,12 +56,11 @@
     return suite
 
 def additional_tests():
-    import json
-    import json.encoder
-    import json.decoder
     suite = unittest.TestSuite()
     for mod in (json, json.encoder, json.decoder):
         suite.addTest(doctest.DocTestSuite(mod))
+    suite.addTest(TestPyTest('test_pyjson'))
+    suite.addTest(TestCTest('test_cjson'))
     return suite
 
 def main():
diff --git a/lib-python/modified-2.7/json/tests/test_check_circular.py b/lib-python/modified-2.7/json/tests/test_check_circular.py
--- a/lib-python/modified-2.7/json/tests/test_check_circular.py
+++ b/lib-python/modified-2.7/json/tests/test_check_circular.py
@@ -1,30 +1,34 @@
-from unittest import TestCase
-import json
+from json.tests import PyTest, CTest
+
 
 def default_iterable(obj):
     return list(obj)
 
-class TestCheckCircular(TestCase):
+class TestCheckCircular(object):
     def test_circular_dict(self):
         dct = {}
         dct['a'] = dct
-        self.assertRaises(ValueError, json.dumps, dct)
+        self.assertRaises(ValueError, self.dumps, dct)
 
     def test_circular_list(self):
         lst = []
         lst.append(lst)
-        self.assertRaises(ValueError, json.dumps, lst)
+        self.assertRaises(ValueError, self.dumps, lst)
 
     def test_circular_composite(self):
         dct2 = {}
         dct2['a'] = []
         dct2['a'].append(dct2)
-        self.assertRaises(ValueError, json.dumps, dct2)
+        self.assertRaises(ValueError, self.dumps, dct2)
 
     def test_circular_default(self):
-        json.dumps([set()], default=default_iterable)
-        self.assertRaises(TypeError, json.dumps, [set()])
+        self.dumps([set()], default=default_iterable)
+        self.assertRaises(TypeError, self.dumps, [set()])
 
     def test_circular_off_default(self):
-        json.dumps([set()], default=default_iterable, check_circular=False)
-        self.assertRaises(TypeError, json.dumps, [set()], check_circular=False)
+        self.dumps([set()], default=default_iterable, check_circular=False)
+        self.assertRaises(TypeError, self.dumps, [set()], check_circular=False)
+
+
+class TestPyCheckCircular(TestCheckCircular, PyTest): pass
+class TestCCheckCircular(TestCheckCircular, CTest): pass
diff --git a/lib-python/modified-2.7/json/tests/test_decode.py b/lib-python/modified-2.7/json/tests/test_decode.py
--- a/lib-python/modified-2.7/json/tests/test_decode.py
+++ b/lib-python/modified-2.7/json/tests/test_decode.py
@@ -1,18 +1,17 @@
 import decimal
-from unittest import TestCase
 from StringIO import StringIO
+from collections import OrderedDict
+from json.tests import PyTest, CTest
 
-import json
-from collections import OrderedDict
 
-class TestDecode(TestCase):
+class TestDecode(object):
     def test_decimal(self):
-        rval = json.loads('1.1', parse_float=decimal.Decimal)
+        rval = self.loads('1.1', parse_float=decimal.Decimal)
         self.assertTrue(isinstance(rval, decimal.Decimal))
         self.assertEqual(rval, decimal.Decimal('1.1'))
 
     def test_float(self):
-        rval = json.loads('1', parse_int=float)
+        rval = self.loads('1', parse_int=float)
         self.assertTrue(isinstance(rval, float))
         self.assertEqual(rval, 1.0)
 
@@ -20,7 +19,7 @@
         # Several optimizations were made that skip over calls to
         # the whitespace regex, so this test is designed to try and
         # exercise the uncommon cases. The array cases are already covered.
-        rval = json.loads('{   "key"    :    "value"    ,  "k":"v"    }')
+        rval = self.loads('{   "key"    :    "value"    ,  "k":"v"    }')
         self.assertEqual(rval, {"key":"value", "k":"v"})
 
     def test_empty_objects(self):
@@ -35,15 +34,19 @@
         s = '{"xkd":1, "kcw":2, "art":3, "hxm":4, "qrt":5, "pad":6, "hoy":7}'
         p = [("xkd", 1), ("kcw", 2), ("art", 3), ("hxm", 4),
              ("qrt", 5), ("pad", 6), ("hoy", 7)]
-        self.assertEqual(json.loads(s), eval(s))
-        self.assertEqual(json.loads(s, object_pairs_hook=lambda x: x), p)
-        self.assertEqual(json.load(StringIO(s),
-                                   object_pairs_hook=lambda x: x), p)
-        od = json.loads(s, object_pairs_hook=OrderedDict)
+        self.assertEqual(self.loads(s), eval(s))
+        self.assertEqual(self.loads(s, object_pairs_hook=lambda x: x), p)
+        self.assertEqual(self.json.load(StringIO(s),
+                                        object_pairs_hook=lambda x: x), p)
+        od = self.loads(s, object_pairs_hook=OrderedDict)
         self.assertEqual(od, OrderedDict(p))
         self.assertEqual(type(od), OrderedDict)
         # the object_pairs_hook takes priority over the object_hook
-        self.assertEqual(json.loads(s,
+        self.assertEqual(self.loads(s,
                                     object_pairs_hook=OrderedDict,
                                     object_hook=lambda x: None),
                          OrderedDict(p))
+
+
+class TestPyDecode(TestDecode, PyTest): pass
+class TestCDecode(TestDecode, CTest): pass
diff --git a/lib-python/modified-2.7/json/tests/test_default.py b/lib-python/modified-2.7/json/tests/test_default.py
--- a/lib-python/modified-2.7/json/tests/test_default.py
+++ b/lib-python/modified-2.7/json/tests/test_default.py
@@ -1,9 +1,12 @@
-from unittest import TestCase
+from json.tests import PyTest, CTest
 
-import json
 
-class TestDefault(TestCase):
+class TestDefault(object):
     def test_default(self):
         self.assertEqual(
-            json.dumps(type, default=repr),
-            json.dumps(repr(type)))
+            self.dumps(type, default=repr),
+            self.dumps(repr(type)))
+
+
+class TestPyDefault(TestDefault, PyTest): pass
+class TestCDefault(TestDefault, CTest): pass
diff --git a/lib-python/modified-2.7/json/tests/test_dump.py b/lib-python/modified-2.7/json/tests/test_dump.py
--- a/lib-python/modified-2.7/json/tests/test_dump.py
+++ b/lib-python/modified-2.7/json/tests/test_dump.py
@@ -1,21 +1,23 @@
-from unittest import TestCase
 from cStringIO import StringIO
+from json.tests import PyTest, CTest
 
-import json
 
-class TestDump(TestCase):
+class TestDump(object):
     def test_dump(self):
         sio = StringIO()
-        json.dump({}, sio)
+        self.json.dump({}, sio)
         self.assertEqual(sio.getvalue(), '{}')
 
     def test_dumps(self):
-        self.assertEqual(json.dumps({}), '{}')
+        self.assertEqual(self.dumps({}), '{}')
 
     def test_encode_truefalse(self):
-        self.assertEqual(json.dumps(
+        self.assertEqual(self.dumps(
                  {True: False, False: True}, sort_keys=True),
                  '{"false": true, "true": false}')
-        self.assertEqual(json.dumps(
+        self.assertEqual(self.dumps(
                 {2: 3.0, 4.0: 5L, False: 1, 6L: True}, sort_keys=True),
                 '{"false": 1, "2": 3.0, "4.0": 5, "6": true}')
+
+class TestPyDump(TestDump, PyTest): pass
+class TestCDump(TestDump, CTest): pass
diff --git a/lib-python/modified-2.7/json/tests/test_encode_basestring_ascii.py b/lib-python/modified-2.7/json/tests/test_encode_basestring_ascii.py
--- a/lib-python/modified-2.7/json/tests/test_encode_basestring_ascii.py
+++ b/lib-python/modified-2.7/json/tests/test_encode_basestring_ascii.py
@@ -1,8 +1,6 @@
-from unittest import TestCase
+from collections import OrderedDict
+from json.tests import PyTest, CTest
 
-import json.encoder
-from json import dumps
-from collections import OrderedDict
 
 CASES = [
     (u'/\\"\ucafe\ubabe\uab98\ufcde\ubcda\uef4a\x08\x0c\n\r\t`1~!@#$%^&*()_+-=[]{}|;:\',./<>?', '"/\\\\\\"\\ucafe\\ubabe\\uab98\\ufcde\\ubcda\\uef4a\\b\\f\\n\\r\\t`1~!@#$%^&*()_+-=[]{}|;:\',./<>?"'),
@@ -23,19 +21,11 @@
     (u'\u0123\u4567\u89ab\ucdef\uabcd\uef4a', '"\\u0123\\u4567\\u89ab\\ucdef\\uabcd\\uef4a"'),
 ]
 
-class TestEncodeBaseStringAscii(TestCase):
-    def test_py_encode_basestring_ascii(self):
-        self._test_encode_basestring_ascii(json.encoder.py_encode_basestring_ascii)
-
-    def test_c_encode_basestring_ascii(self):
-        if not json.encoder.c_encode_basestring_ascii:
-            return
-        self._test_encode_basestring_ascii(json.encoder.c_encode_basestring_ascii)
-
-    def _test_encode_basestring_ascii(self, encode_basestring_ascii):
-        fname = encode_basestring_ascii.__name__
+class TestEncodeBasestringAscii(object):
+    def test_encode_basestring_ascii(self):
+        fname = self.json.encoder.encode_basestring_ascii.__name__
         for input_string, expect in CASES:
-            result = encode_basestring_ascii(input_string)
+            result = self.json.encoder.encode_basestring_ascii(input_string)
             self.assertEqual(result, expect,
                 '{0!r} != {1!r} for {2}({3!r})'.format(
                     result, expect, fname, input_string))
@@ -43,5 +33,9 @@
     def test_ordered_dict(self):
         # See issue 6105
         items = [('one', 1), ('two', 2), ('three', 3), ('four', 4), ('five', 5)]
-        s = json.dumps(OrderedDict(items))
+        s = self.dumps(OrderedDict(items))
         self.assertEqual(s, '{"one": 1, "two": 2, "three": 3, "four": 4, "five": 5}')
+
+
+class TestPyEncodeBasestringAscii(TestEncodeBasestringAscii, PyTest): pass
+class TestCEncodeBasestringAscii(TestEncodeBasestringAscii, CTest): pass
diff --git a/lib-python/modified-2.7/json/tests/test_fail.py b/lib-python/modified-2.7/json/tests/test_fail.py
--- a/lib-python/modified-2.7/json/tests/test_fail.py
+++ b/lib-python/modified-2.7/json/tests/test_fail.py
@@ -1,6 +1,4 @@
-from unittest import TestCase
-
-import json
+from json.tests import PyTest, CTest
 
 # Fri Dec 30 18:57:26 2005
 JSONDOCS = [
@@ -61,15 +59,15 @@
     18: "spec doesn't specify any nesting limitations",
 }
 
-class TestFail(TestCase):
+class TestFail(object):
     def test_failures(self):
         for idx, doc in enumerate(JSONDOCS):
             idx = idx + 1
             if idx in SKIPS:
-                json.loads(doc)
+                self.loads(doc)
                 continue
             try:
-                json.loads(doc)
+                self.loads(doc)
             except ValueError:
                 pass
             else:
@@ -79,7 +77,11 @@
         data = {'a' : 1, (1, 2) : 2}
 
         #This is for c encoder
-        self.assertRaises(TypeError, json.dumps, data)
+        self.assertRaises(TypeError, self.dumps, data)
 
         #This is for python encoder
-        self.assertRaises(TypeError, json.dumps, data, indent=True)
+        self.assertRaises(TypeError, self.dumps, data, indent=True)
+
+
+class TestPyFail(TestFail, PyTest): pass
+class TestCFail(TestFail, CTest): pass
diff --git a/lib-python/modified-2.7/json/tests/test_float.py b/lib-python/modified-2.7/json/tests/test_float.py
--- a/lib-python/modified-2.7/json/tests/test_float.py
+++ b/lib-python/modified-2.7/json/tests/test_float.py
@@ -1,19 +1,22 @@
 import math
-from unittest import TestCase
+from json.tests import PyTest, CTest
 
-import json
 
-class TestFloat(TestCase):
+class TestFloat(object):
     def test_floats(self):
         for num in [1617161771.7650001, math.pi, math.pi**100,
                     math.pi**-100, 3.1]:
-            self.assertEqual(float(json.dumps(num)), num)
-            self.assertEqual(json.loads(json.dumps(num)), num)
-            self.assertEqual(json.loads(unicode(json.dumps(num))), num)
+            self.assertEqual(float(self.dumps(num)), num)
+            self.assertEqual(self.loads(self.dumps(num)), num)
+            self.assertEqual(self.loads(unicode(self.dumps(num))), num)
 
     def test_ints(self):
         for num in [1, 1L, 1<<32, 1<<64]:
-            self.assertEqual(json.dumps(num), str(num))
-            self.assertEqual(int(json.dumps(num)), num)
-            self.assertEqual(json.loads(json.dumps(num)), num)
-            self.assertEqual(json.loads(unicode(json.dumps(num))), num)
+            self.assertEqual(self.dumps(num), str(num))
+            self.assertEqual(int(self.dumps(num)), num)
+            self.assertEqual(self.loads(self.dumps(num)), num)
+            self.assertEqual(self.loads(unicode(self.dumps(num))), num)
+
+
+class TestPyFloat(TestFloat, PyTest): pass
+class TestCFloat(TestFloat, CTest): pass
diff --git a/lib-python/modified-2.7/json/tests/test_indent.py b/lib-python/modified-2.7/json/tests/test_indent.py
--- a/lib-python/modified-2.7/json/tests/test_indent.py
+++ b/lib-python/modified-2.7/json/tests/test_indent.py
@@ -1,9 +1,9 @@
-from unittest import TestCase
+import textwrap
+from StringIO import StringIO
+from json.tests import PyTest, CTest
 
-import json
-import textwrap
 
-class TestIndent(TestCase):
+class TestIndent(object):
     def test_indent(self):
         h = [['blorpie'], ['whoops'], [], 'd-shtaeou', 'd-nthiouh', 'i-vhbjkhnth',
              {'nifty': 87}, {'field': 'yes', 'morefield': False} ]
@@ -30,12 +30,31 @@
         ]""")
 
 
-        d1 = json.dumps(h)
-        d2 = json.dumps(h, indent=2, sort_keys=True, separators=(',', ': '))
+        d1 = self.dumps(h)
+        d2 = self.dumps(h, indent=2, sort_keys=True, separators=(',', ': '))
 
-        h1 = json.loads(d1)
-        h2 = json.loads(d2)
+        h1 = self.loads(d1)
+        h2 = self.loads(d2)
 
         self.assertEqual(h1, h)
         self.assertEqual(h2, h)
         self.assertEqual(d2, expect)
+
+    def test_indent0(self):
+        h = {3: 1}
+        def check(indent, expected):
+            d1 = self.dumps(h, indent=indent)
+            self.assertEqual(d1, expected)
+
+            sio = StringIO()
+            self.json.dump(h, sio, indent=indent)
+            self.assertEqual(sio.getvalue(), expected)
+
+        # indent=0 should emit newlines
+        check(0, '{\n"3": 1\n}')
+        # indent=None is more compact
+        check(None, '{"3": 1}')
+
+
+class TestPyIndent(TestIndent, PyTest): pass
+class TestCIndent(TestIndent, CTest): pass
diff --git a/lib-python/modified-2.7/json/tests/test_pass1.py b/lib-python/modified-2.7/json/tests/test_pass1.py
--- a/lib-python/modified-2.7/json/tests/test_pass1.py
+++ b/lib-python/modified-2.7/json/tests/test_pass1.py
@@ -1,6 +1,5 @@
-from unittest import TestCase
+from json.tests import PyTest, CTest
 
-import json
 
 # from http://json.org/JSON_checker/test/pass1.json
 JSON = r'''
@@ -62,15 +61,19 @@
 ,"rosebud"]
 '''
 
-class TestPass1(TestCase):
+class TestPass1(object):
     def test_parse(self):
         # test in/out equivalence and parsing
-        res = json.loads(JSON)
-        out = json.dumps(res)
-        self.assertEqual(res, json.loads(out))
+        res = self.loads(JSON)
+        out = self.dumps(res)
+        self.assertEqual(res, self.loads(out))
         try:
-            json.dumps(res, allow_nan=False)
+            self.dumps(res, allow_nan=False)
         except ValueError:
             pass
         else:
             self.fail("23456789012E666 should be out of range")
+
+
+class TestPyPass1(TestPass1, PyTest): pass
+class TestCPass1(TestPass1, CTest): pass
diff --git a/lib-python/modified-2.7/json/tests/test_pass2.py b/lib-python/modified-2.7/json/tests/test_pass2.py
--- a/lib-python/modified-2.7/json/tests/test_pass2.py
+++ b/lib-python/modified-2.7/json/tests/test_pass2.py
@@ -1,14 +1,18 @@
-from unittest import TestCase
-import json
+from json.tests import PyTest, CTest
+
 
 # from http://json.org/JSON_checker/test/pass2.json
 JSON = r'''
 [[[[[[[[[[[[[[[[[[["Not too deep"]]]]]]]]]]]]]]]]]]]
 '''
 
-class TestPass2(TestCase):
+class TestPass2(object):
     def test_parse(self):
         # test in/out equivalence and parsing
-        res = json.loads(JSON)
-        out = json.dumps(res)
-        self.assertEqual(res, json.loads(out))
+        res = self.loads(JSON)
+        out = self.dumps(res)
+        self.assertEqual(res, self.loads(out))
+
+
+class TestPyPass2(TestPass2, PyTest): pass
+class TestCPass2(TestPass2, CTest): pass
diff --git a/lib-python/modified-2.7/json/tests/test_pass3.py b/lib-python/modified-2.7/json/tests/test_pass3.py
--- a/lib-python/modified-2.7/json/tests/test_pass3.py
+++ b/lib-python/modified-2.7/json/tests/test_pass3.py
@@ -1,6 +1,5 @@
-from unittest import TestCase
+from json.tests import PyTest, CTest
 
-import json
 
 # from http://json.org/JSON_checker/test/pass3.json
 JSON = r'''
@@ -12,9 +11,14 @@
 }
 '''
 
-class TestPass3(TestCase):
+
+class TestPass3(object):
     def test_parse(self):
         # test in/out equivalence and parsing
-        res = json.loads(JSON)
-        out = json.dumps(res)
-        self.assertEqual(res, json.loads(out))
+        res = self.loads(JSON)
+        out = self.dumps(res)
+        self.assertEqual(res, self.loads(out))
+
+
+class TestPyPass3(TestPass3, PyTest): pass
+class TestCPass3(TestPass3, CTest): pass
diff --git a/lib-python/modified-2.7/json/tests/test_recursion.py b/lib-python/modified-2.7/json/tests/test_recursion.py
--- a/lib-python/modified-2.7/json/tests/test_recursion.py
+++ b/lib-python/modified-2.7/json/tests/test_recursion.py
@@ -1,28 +1,16 @@
-from unittest import TestCase
+from json.tests import PyTest, CTest
 
-import json
 
 class JSONTestObject:
     pass
 
 
-class RecursiveJSONEncoder(json.JSONEncoder):
-    recurse = False
-    def default(self, o):
-        if o is JSONTestObject:
-            if self.recurse:
-                return [JSONTestObject]
-            else:
-                return 'JSONTestObject'
-        return json.JSONEncoder.default(o)
-
-
-class TestRecursion(TestCase):
+class TestRecursion(object):
     def test_listrecursion(self):
         x = []
         x.append(x)
         try:
-            json.dumps(x)
+            self.dumps(x)
         except ValueError:
             pass
         else:
@@ -31,7 +19,7 @@
         y = [x]
         x.append(y)
         try:
-            json.dumps(x)
+            self.dumps(x)
         except ValueError:
             pass
         else:
@@ -39,13 +27,13 @@
         y = []
         x = [y, y]
         # ensure that the marker is cleared
-        json.dumps(x)
+        self.dumps(x)
 
     def test_dictrecursion(self):
         x = {}
         x["test"] = x
         try:
-            json.dumps(x)
+            self.dumps(x)
         except ValueError:
             pass
         else:
@@ -53,9 +41,19 @@
         x = {}
         y = {"a": x, "b": x}
         # ensure that the marker is cleared
-        json.dumps(x)
+        self.dumps(x)
 
     def test_defaultrecursion(self):
+        class RecursiveJSONEncoder(self.json.JSONEncoder):
+            recurse = False
+            def default(self, o):
+                if o is JSONTestObject:
+                    if self.recurse:
+                        return [JSONTestObject]
+                    else:
+                        return 'JSONTestObject'
+                return pyjson.JSONEncoder.default(o)
+
         enc = RecursiveJSONEncoder()
         self.assertEqual(enc.encode(JSONTestObject), '"JSONTestObject"')
         enc.recurse = True
@@ -65,3 +63,46 @@
             pass
         else:
             self.fail("didn't raise ValueError on default recursion")
+
+
+    def test_highly_nested_objects_decoding(self):
+        # test that loading highly-nested objects doesn't segfault when C
+        # accelerations are used. See #12017
+        # str
+        with self.assertRaises(RuntimeError):
+            self.loads('{"a":' * 100000 + '1' + '}' * 100000)
+        with self.assertRaises(RuntimeError):
+            self.loads('{"a":' * 100000 + '[1]' + '}' * 100000)
+        with self.assertRaises(RuntimeError):
+            self.loads('[' * 100000 + '1' + ']' * 100000)
+        # unicode
+        with self.assertRaises(RuntimeError):
+            self.loads(u'{"a":' * 100000 + u'1' + u'}' * 100000)
+        with self.assertRaises(RuntimeError):
+            self.loads(u'{"a":' * 100000 + u'[1]' + u'}' * 100000)
+        with self.assertRaises(RuntimeError):
+            self.loads(u'[' * 100000 + u'1' + u']' * 100000)
+
+    def test_highly_nested_objects_encoding(self):
+        # See #12051
+        l, d = [], {}
+        for x in xrange(100000):
+            l, d = [l], {'k':d}
+        with self.assertRaises(RuntimeError):
+            self.dumps(l)
+        with self.assertRaises(RuntimeError):
+            self.dumps(d)
+
+    def test_endless_recursion(self):
+        # See #12051
+        class EndlessJSONEncoder(self.json.JSONEncoder):
+            def default(self, o):
+                """If check_circular is False, this will keep adding another list."""
+                return [o]
+
+        with self.assertRaises(RuntimeError):
+            EndlessJSONEncoder(check_circular=False).encode(5j)
+
+
+class TestPyRecursion(TestRecursion, PyTest): pass
+class TestCRecursion(TestRecursion, CTest): pass
diff --git a/lib-python/modified-2.7/json/tests/test_scanstring.py b/lib-python/modified-2.7/json/tests/test_scanstring.py
--- a/lib-python/modified-2.7/json/tests/test_scanstring.py
+++ b/lib-python/modified-2.7/json/tests/test_scanstring.py
@@ -1,20 +1,10 @@
 import sys
-import decimal
-from unittest import TestCase
-from test import test_support
+from json.tests import PyTest, CTest
 
-import json
-import json.decoder
 
-class TestScanString(TestCase):
-    def test_py_scanstring(self):
-        self._test_scanstring(json.decoder.py_scanstring)
-
-    @test_support.impl_detail()
-    def test_c_scanstring(self):
-        self._test_scanstring(json.decoder.c_scanstring)
-
-    def _test_scanstring(self, scanstring):
+class TestScanstring(object):
+    def test_scanstring(self):
+        scanstring = self.json.decoder.scanstring
         self.assertEqual(
             scanstring('"z\\ud834\\udd20x"', 1, None, True),
             (u'z\U0001d120x', 16))
@@ -105,10 +95,15 @@
             (u'Bad value', 12))
 
     def test_issue3623(self):
-        self.assertRaises(ValueError, json.decoder.scanstring, b"xxx", 1,
+        self.assertRaises(ValueError, self.json.decoder.scanstring, b"xxx", 1,
                           "xxx")
         self.assertRaises(UnicodeDecodeError,
-                          json.encoder.encode_basestring_ascii, b"xx\xff")
+                          self.json.encoder.encode_basestring_ascii, b"xx\xff")
 
     def test_overflow(self):
-        self.assertRaises(OverflowError, json.decoder.scanstring, b"xxx", sys.maxsize+1)
+        with self.assertRaises(OverflowError):
+            self.json.decoder.scanstring(b"xxx", sys.maxsize+1)
+
+
+class TestPyScanstring(TestScanstring, PyTest): pass
+class TestCScanstring(TestScanstring, CTest): pass
diff --git a/lib-python/modified-2.7/json/tests/test_separators.py b/lib-python/modified-2.7/json/tests/test_separators.py
--- a/lib-python/modified-2.7/json/tests/test_separators.py
+++ b/lib-python/modified-2.7/json/tests/test_separators.py
@@ -1,10 +1,8 @@
 import textwrap
-from unittest import TestCase
+from json.tests import PyTest, CTest
 
-import json
 
-
-class TestSeparators(TestCase):
+class TestSeparators(object):
     def test_separators(self):
         h = [['blorpie'], ['whoops'], [], 'd-shtaeou', 'd-nthiouh', 'i-vhbjkhnth',
              {'nifty': 87}, {'field': 'yes', 'morefield': False} ]
@@ -31,12 +29,16 @@
         ]""")
 
 
-        d1 = json.dumps(h)
-        d2 = json.dumps(h, indent=2, sort_keys=True, separators=(' ,', ' : '))
+        d1 = self.dumps(h)
+        d2 = self.dumps(h, indent=2, sort_keys=True, separators=(' ,', ' : '))
 
-        h1 = json.loads(d1)
-        h2 = json.loads(d2)
+        h1 = self.loads(d1)
+        h2 = self.loads(d2)
 
         self.assertEqual(h1, h)
         self.assertEqual(h2, h)
         self.assertEqual(d2, expect)
+
+
+class TestPySeparators(TestSeparators, PyTest): pass
+class TestCSeparators(TestSeparators, CTest): pass
diff --git a/lib-python/modified-2.7/json/tests/test_speedups.py b/lib-python/modified-2.7/json/tests/test_speedups.py
--- a/lib-python/modified-2.7/json/tests/test_speedups.py
+++ b/lib-python/modified-2.7/json/tests/test_speedups.py
@@ -1,29 +1,23 @@
-import decimal
-from unittest import TestCase
-from test import test_support
+from json.tests import CTest
 
-from json import decoder, encoder, scanner
 
-class TestSpeedups(TestCase):
-    @test_support.impl_detail()
+class TestSpeedups(CTest):
     def test_scanstring(self):
-        self.assertEqual(decoder.scanstring.__module__, "_json")
-        self.assertTrue(decoder.scanstring is decoder.c_scanstring)
+        self.assertEqual(self.json.decoder.scanstring.__module__, "_json")
+        self.assertIs(self.json.decoder.scanstring, self.json.decoder.c_scanstring)
 
-    @test_support.impl_detail()
     def test_encode_basestring_ascii(self):
-        self.assertEqual(encoder.encode_basestring_ascii.__module__, "_json")
-        self.assertTrue(encoder.encode_basestring_ascii is
-                          encoder.c_encode_basestring_ascii)
+        self.assertEqual(self.json.encoder.encode_basestring_ascii.__module__,
+                         "_json")
+        self.assertIs(self.json.encoder.encode_basestring_ascii,
+                      self.json.encoder.c_encode_basestring_ascii)
 
-class TestDecode(TestCase):
-    @test_support.impl_detail()
+class TestDecode(CTest):
     def test_make_scanner(self):
-        self.assertRaises(AttributeError, scanner.c_make_scanner, 1)
+        self.assertRaises(AttributeError, self.json.scanner.c_make_scanner, 1)
 
-    @test_support.impl_detail()
     def test_make_encoder(self):
-        self.assertRaises(TypeError, encoder.c_make_encoder,
+        self.assertRaises(TypeError, self.json.encoder.c_make_encoder,
             None,
             "\xCD\x7D\x3D\x4E\x12\x4C\xF9\x79\xD7\x52\xBA\x82\xF2\x27\x4A\x7D\xA0\xCA\x75",
             None)
diff --git a/lib-python/modified-2.7/json/tests/test_unicode.py b/lib-python/modified-2.7/json/tests/test_unicode.py
--- a/lib-python/modified-2.7/json/tests/test_unicode.py
+++ b/lib-python/modified-2.7/json/tests/test_unicode.py
@@ -1,11 +1,10 @@
-from unittest import TestCase
+from collections import OrderedDict
+from json.tests import PyTest, CTest
 
-import json
-from collections import OrderedDict
 
-class TestUnicode(TestCase):
+class TestUnicode(object):
     def test_encoding1(self):
-        encoder = json.JSONEncoder(encoding='utf-8')
+        encoder = self.json.JSONEncoder(encoding='utf-8')
         u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}'
         s = u.encode('utf-8')
         ju = encoder.encode(u)
@@ -15,74 +14,78 @@
     def test_encoding2(self):
         u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}'
         s = u.encode('utf-8')
-        ju = json.dumps(u, encoding='utf-8')
-        js = json.dumps(s, encoding='utf-8')
+        ju = self.dumps(u, encoding='utf-8')
+        js = self.dumps(s, encoding='utf-8')
         self.assertEqual(ju, js)
 
     def test_encoding3(self):
         u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}'
-        j = json.dumps(u)
+        j = self.dumps(u)
         self.assertEqual(j, '"\\u03b1\\u03a9"')
 
     def test_encoding4(self):
         u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}'
-        j = json.dumps([u])
+        j = self.dumps([u])
         self.assertEqual(j, '["\\u03b1\\u03a9"]')
 
     def test_encoding5(self):
         u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}'
-        j = json.dumps(u, ensure_ascii=False)
+        j = self.dumps(u, ensure_ascii=False)
         self.assertEqual(j, u'"{0}"'.format(u))
 
     def test_encoding6(self):
         u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}'
-        j = json.dumps([u], ensure_ascii=False)
+        j = self.dumps([u], ensure_ascii=False)
         self.assertEqual(j, u'["{0}"]'.format(u))
 
     def test_big_unicode_encode(self):
         u = u'\U0001d120'
-        self.assertEqual(json.dumps(u), '"\\ud834\\udd20"')
-        self.assertEqual(json.dumps(u, ensure_ascii=False), u'"\U0001d120"')
+        self.assertEqual(self.dumps(u), '"\\ud834\\udd20"')
+        self.assertEqual(self.dumps(u, ensure_ascii=False), u'"\U0001d120"')
 
     def test_big_unicode_decode(self):
         u = u'z\U0001d120x'
-        self.assertEqual(json.loads('"' + u + '"'), u)
-        self.assertEqual(json.loads('"z\\ud834\\udd20x"'), u)
+        self.assertEqual(self.loads('"' + u + '"'), u)
+        self.assertEqual(self.loads('"z\\ud834\\udd20x"'), u)
 
     def test_unicode_decode(self):
         for i in range(0, 0xd7ff):
             u = unichr(i)
             s = '"\\u{0:04x}"'.format(i)
-            self.assertEqual(json.loads(s), u)
+            self.assertEqual(self.loads(s), u)
 
     def test_object_pairs_hook_with_unicode(self):
         s = u'{"xkd":1, "kcw":2, "art":3, "hxm":4, "qrt":5, "pad":6, "hoy":7}'
         p = [(u"xkd", 1), (u"kcw", 2), (u"art", 3), (u"hxm", 4),
              (u"qrt", 5), (u"pad", 6), (u"hoy", 7)]
-        self.assertEqual(json.loads(s), eval(s))
-        self.assertEqual(json.loads(s, object_pairs_hook = lambda x: x), p)
-        od = json.loads(s, object_pairs_hook = OrderedDict)
+        self.assertEqual(self.loads(s), eval(s))
+        self.assertEqual(self.loads(s, object_pairs_hook = lambda x: x), p)
+        od = self.loads(s, object_pairs_hook = OrderedDict)
         self.assertEqual(od, OrderedDict(p))
         self.assertEqual(type(od), OrderedDict)
         # the object_pairs_hook takes priority over the object_hook
-        self.assertEqual(json.loads(s,
+        self.assertEqual(self.loads(s,
                                     object_pairs_hook = OrderedDict,
                                     object_hook = lambda x: None),
                          OrderedDict(p))
 
     def test_default_encoding(self):
-        self.assertEqual(json.loads(u'{"a": "\xe9"}'.encode('utf-8')),
+        self.assertEqual(self.loads(u'{"a": "\xe9"}'.encode('utf-8')),
             {'a': u'\xe9'})
 
     def test_unicode_preservation(self):
-        self.assertEqual(type(json.loads(u'""')), unicode)
-        self.assertEqual(type(json.loads(u'"a"')), unicode)
-        self.assertEqual(type(json.loads(u'["a"]')[0]), unicode)
+        self.assertEqual(type(self.loads(u'""')), unicode)
+        self.assertEqual(type(self.loads(u'"a"')), unicode)
+        self.assertEqual(type(self.loads(u'["a"]')[0]), unicode)
         # Issue 10038.
-        self.assertEqual(type(json.loads('"foo"')), unicode)
+        self.assertEqual(type(self.loads('"foo"')), unicode)
 
     def test_encode_not_utf_8(self):
         self.assertEqual(json.dumps('\xb1\xe6', encoding='iso8859-2'),
                          '"\\u0105\\u0107"')
         self.assertEqual(json.dumps(['\xb1\xe6'], encoding='iso8859-2'),
                          '["\\u0105\\u0107"]')
+
+
+class TestPyUnicode(TestUnicode, PyTest): pass
+class TestCUnicode(TestUnicode, CTest): pass
diff --git a/lib-python/modified-2.7/lib2to3/fixes/fix_itertools.py b/lib-python/modified-2.7/lib2to3/fixes/fix_itertools.py
--- a/lib-python/modified-2.7/lib2to3/fixes/fix_itertools.py
+++ b/lib-python/modified-2.7/lib2to3/fixes/fix_itertools.py
@@ -13,7 +13,7 @@
 
 class FixItertools(fixer_base.BaseFix):
     BM_compatible = True
-    it_funcs = "('imap'|'ifilter'|'izip'|'ifilterfalse')"
+    it_funcs = "('imap'|'ifilter'|'izip'|'izip_longest'|'ifilterfalse')"
     PATTERN = """
               power< it='itertools'
                   trailer<
@@ -28,7 +28,8 @@
     def transform(self, node, results):
         prefix = None
         func = results['func'][0]
-        if 'it' in results and func.value != u'ifilterfalse':
+        if ('it' in results and
+            func.value not in (u'ifilterfalse', u'izip_longest')):
             dot, it = (results['dot'], results['it'])
             # Remove the 'itertools'
             prefix = it.prefix
diff --git a/lib-python/modified-2.7/lib2to3/fixes/fix_itertools_imports.py b/lib-python/modified-2.7/lib2to3/fixes/fix_itertools_imports.py
--- a/lib-python/modified-2.7/lib2to3/fixes/fix_itertools_imports.py
+++ b/lib-python/modified-2.7/lib2to3/fixes/fix_itertools_imports.py
@@ -31,9 +31,10 @@
             if member_name in (u'imap', u'izip', u'ifilter'):
                 child.value = None
                 child.remove()
-            elif member_name == u'ifilterfalse':
+            elif member_name in (u'ifilterfalse', u'izip_longest'):
                 node.changed()
-                name_node.value = u'filterfalse'
+                name_node.value = (u'filterfalse' if member_name[1] == u'f'
+                                   else u'zip_longest')
 
         # Make sure the import statement is still sane
         children = imports.children[:] or [imports]
diff --git a/lib-python/modified-2.7/lib2to3/fixes/fix_metaclass.py b/lib-python/modified-2.7/lib2to3/fixes/fix_metaclass.py
--- a/lib-python/modified-2.7/lib2to3/fixes/fix_metaclass.py
+++ b/lib-python/modified-2.7/lib2to3/fixes/fix_metaclass.py
@@ -48,7 +48,7 @@
     """
     for node in cls_node.children:
         if node.type == syms.suite:
-            # already in the prefered format, do nothing
+            # already in the preferred format, do nothing
             return
 
     # !%@#! oneliners have no suite node, we have to fake one up
diff --git a/lib-python/modified-2.7/lib2to3/fixes/fix_urllib.py b/lib-python/modified-2.7/lib2to3/fixes/fix_urllib.py
--- a/lib-python/modified-2.7/lib2to3/fixes/fix_urllib.py
+++ b/lib-python/modified-2.7/lib2to3/fixes/fix_urllib.py
@@ -12,7 +12,7 @@
 
 MAPPING = {"urllib":  [
                 ("urllib.request",
-                    ["URLOpener", "FancyURLOpener", "urlretrieve",
+                    ["URLopener", "FancyURLopener", "urlretrieve",
                      "_urlopener", "urlopen", "urlcleanup",
                      "pathname2url", "url2pathname"]),
                 ("urllib.parse",
diff --git a/lib-python/modified-2.7/lib2to3/main.py b/lib-python/modified-2.7/lib2to3/main.py
--- a/lib-python/modified-2.7/lib2to3/main.py
+++ b/lib-python/modified-2.7/lib2to3/main.py
@@ -101,7 +101,7 @@
     parser.add_option("-j", "--processes", action="store", default=1,
                       type="int", help="Run 2to3 concurrently")
     parser.add_option("-x", "--nofix", action="append", default=[],
-                      help="Prevent a fixer from being run.")
+                      help="Prevent a transformation from being run")
     parser.add_option("-l", "--list-fixes", action="store_true",
                       help="List available transformations")
     parser.add_option("-p", "--print-function", action="store_true",
@@ -113,7 +113,7 @@
     parser.add_option("-w", "--write", action="store_true",
                       help="Write back modified files")
     parser.add_option("-n", "--nobackups", action="store_true", default=False,
-                      help="Don't write backups for modified files.")
+                      help="Don't write backups for modified files")
 
     # Parse command line arguments
     refactor_stdin = False
diff --git a/lib-python/modified-2.7/lib2to3/patcomp.py b/lib-python/modified-2.7/lib2to3/patcomp.py
--- a/lib-python/modified-2.7/lib2to3/patcomp.py
+++ b/lib-python/modified-2.7/lib2to3/patcomp.py
@@ -12,6 +12,7 @@
 
 # Python imports
 import os
+import StringIO
 
 # Fairly local imports
 from .pgen2 import driver, literals, token, tokenize, parse, grammar
@@ -32,7 +33,7 @@
 def tokenize_wrapper(input):
     """Tokenizes a string suppressing significant whitespace."""
     skip = set((token.NEWLINE, token.INDENT, token.DEDENT))
-    tokens = tokenize.generate_tokens(driver.generate_lines(input).next)
+    tokens = tokenize.generate_tokens(StringIO.StringIO(input).readline)
     for quintuple in tokens:
         type, value, start, end, line_text = quintuple
         if type not in skip:
diff --git a/lib-python/modified-2.7/lib2to3/pgen2/conv.py b/lib-python/modified-2.7/lib2to3/pgen2/conv.py
--- a/lib-python/modified-2.7/lib2to3/pgen2/conv.py
+++ b/lib-python/modified-2.7/lib2to3/pgen2/conv.py
@@ -51,7 +51,7 @@
         self.finish_off()
 
     def parse_graminit_h(self, filename):
-        """Parse the .h file writen by pgen.  (Internal)
+        """Parse the .h file written by pgen.  (Internal)
 
         This file is a sequence of #define statements defining the
         nonterminals of the grammar as numbers.  We build two tables
@@ -82,7 +82,7 @@
         return True
 
     def parse_graminit_c(self, filename):
-        """Parse the .c file writen by pgen.  (Internal)
+        """Parse the .c file written by pgen.  (Internal)
 
         The file looks as follows.  The first two lines are always this:
 
diff --git a/lib-python/modified-2.7/lib2to3/pgen2/driver.py b/lib-python/modified-2.7/lib2to3/pgen2/driver.py
--- a/lib-python/modified-2.7/lib2to3/pgen2/driver.py
+++ b/lib-python/modified-2.7/lib2to3/pgen2/driver.py
@@ -19,6 +19,7 @@
 import codecs
 import os
 import logging
+import StringIO
 import sys
 
 # Pgen imports
@@ -101,18 +102,10 @@
 
     def parse_string(self, text, debug=False):
         """Parse a string and return the syntax tree."""
-        tokens = tokenize.generate_tokens(generate_lines(text).next)
+        tokens = tokenize.generate_tokens(StringIO.StringIO(text).readline)
         return self.parse_tokens(tokens, debug)
 
 
-def generate_lines(text):
-    """Generator that behaves like readline without using StringIO."""
-    for line in text.splitlines(True):
-        yield line
-    while True:
-        yield ""
-
-
 def load_grammar(gt="Grammar.txt", gp=None,
                  save=True, force=False, logger=None):
     """Load the grammar (maybe from a pickle)."""
diff --git a/lib-python/modified-2.7/lib2to3/pytree.py b/lib-python/modified-2.7/lib2to3/pytree.py
--- a/lib-python/modified-2.7/lib2to3/pytree.py
+++ b/lib-python/modified-2.7/lib2to3/pytree.py
@@ -658,8 +658,8 @@
             content: optional sequence of subsequences of patterns;
                      if absent, matches one node;
                      if present, each subsequence is an alternative [*]
-            min: optinal minumum number of times to match, default 0
-            max: optional maximum number of times tro match, default HUGE
+            min: optional minimum number of times to match, default 0
+            max: optional maximum number of times to match, default HUGE
             name: optional name assigned to this match
 
         [*] Thus, if content is [[a, b, c], [d, e], [f, g, h]] this is
@@ -741,12 +741,13 @@
         elif self.name == "bare_name":
             yield self._bare_name_matches(nodes)
         else:
-            # There used to be some monkey patching of sys.stderr here, to
-            # silence the error message from the RuntimError, PyPy has removed
-            # this because it relied on reference counting.  This is because the
-            # caller of this function doesn't consume this generator fully, so
-            # the finally statement that used to be here would only be executed
-            # when the gc happened to run.
+            # The reason for this is that hitting the recursion limit usually
+            # results in some ugly messages about how RuntimeErrors are being
+            # ignored. We don't do this on non-CPython implementation because
+            # they don't have this problem.
+            if hasattr(sys, "getrefcount"):
+                save_stderr = sys.stderr
+                sys.stderr = StringIO()
             try:
                 for count, r in self._recursive_matches(nodes, 0):
                     if self.name:
@@ -759,6 +760,9 @@
                     if self.name:
                         r[self.name] = nodes[:count]
                     yield count, r
+            finally:
+                if hasattr(sys, "getrefcount"):
+                    sys.stderr = save_stderr
 
     def _iterative_matches(self, nodes):
         """Helper to iteratively yield the matches."""
diff --git a/lib-python/modified-2.7/lib2to3/refactor.py b/lib-python/modified-2.7/lib2to3/refactor.py
--- a/lib-python/modified-2.7/lib2to3/refactor.py
+++ b/lib-python/modified-2.7/lib2to3/refactor.py
@@ -302,13 +302,14 @@
 
         Files and subdirectories starting with '.' are skipped.
         """
+        py_ext = os.extsep + "py"
         for dirpath, dirnames, filenames in os.walk(dir_name):
             self.log_debug("Descending into %s", dirpath)
             dirnames.sort()
             filenames.sort()
             for name in filenames:
-                if not name.startswith(".") and \
-                        os.path.splitext(name)[1].endswith("py"):
+                if (not name.startswith(".") and
+                    os.path.splitext(name)[1] == py_ext):
                     fullname = os.path.join(dirpath, name)
                     self.refactor_file(fullname, write, doctests_only)
             # Modify dirnames in-place to remove subdirs with leading dots
diff --git a/lib-python/modified-2.7/lib2to3/tests/data/py2_test_grammar.py b/lib-python/modified-2.7/lib2to3/tests/data/py2_test_grammar.py
--- a/lib-python/modified-2.7/lib2to3/tests/data/py2_test_grammar.py
+++ b/lib-python/modified-2.7/lib2to3/tests/data/py2_test_grammar.py
@@ -316,7 +316,7 @@
         ### simple_stmt: small_stmt (';' small_stmt)* [';']
         x = 1; pass; del x
         def foo():
-            # verify statments that end with semi-colons
+            # verify statements that end with semi-colons
             x = 1; pass; del x;
         foo()
 
diff --git a/lib-python/modified-2.7/lib2to3/tests/data/py3_test_grammar.py b/lib-python/modified-2.7/lib2to3/tests/data/py3_test_grammar.py
--- a/lib-python/modified-2.7/lib2to3/tests/data/py3_test_grammar.py
+++ b/lib-python/modified-2.7/lib2to3/tests/data/py3_test_grammar.py
@@ -356,7 +356,7 @@
         ### simple_stmt: small_stmt (';' small_stmt)* [';']
         x = 1; pass; del x
         def foo():
-            # verify statments that end with semi-colons
+            # verify statements that end with semi-colons
             x = 1; pass; del x;
         foo()
 
diff --git a/lib-python/modified-2.7/lib2to3/tests/test_fixers.py b/lib-python/modified-2.7/lib2to3/tests/test_fixers.py
--- a/lib-python/modified-2.7/lib2to3/tests/test_fixers.py
+++ b/lib-python/modified-2.7/lib2to3/tests/test_fixers.py
@@ -3623,16 +3623,24 @@
         a = """%s(f, a)"""
         self.checkall(b, a)
 
-    def test_2(self):
+    def test_qualified(self):
         b = """itertools.ifilterfalse(a, b)"""
         a = """itertools.filterfalse(a, b)"""
         self.check(b, a)
 
-    def test_4(self):
+        b = """itertools.izip_longest(a, b)"""
+        a = """itertools.zip_longest(a, b)"""
+        self.check(b, a)
+
+    def test_2(self):
         b = """ifilterfalse(a, b)"""
         a = """filterfalse(a, b)"""
         self.check(b, a)
 
+        b = """izip_longest(a, b)"""
+        a = """zip_longest(a, b)"""
+        self.check(b, a)
+
     def test_space_1(self):
         b = """    %s(f, a)"""
         a = """    %s(f, a)"""
@@ -3643,9 +3651,14 @@
         a = """    itertools.filterfalse(a, b)"""
         self.check(b, a)
 
+        b = """    itertools.izip_longest(a, b)"""
+        a = """    itertools.zip_longest(a, b)"""
+        self.check(b, a)
+
     def test_run_order(self):
         self.assert_runs_after('map', 'zip', 'filter')
 
+
 class Test_itertools_imports(FixerTestCase):
     fixer = 'itertools_imports'
 
@@ -3696,18 +3709,19 @@
         s = "from itertools import bar as bang"
         self.unchanged(s)
 
-    def test_ifilter(self):
-        b = "from itertools import ifilterfalse"
-        a = "from itertools import filterfalse"
-        self.check(b, a)
-
-        b = "from itertools import imap, ifilterfalse, foo"
-        a = "from itertools import filterfalse, foo"
-        self.check(b, a)
-
-        b = "from itertools import bar, ifilterfalse, foo"
-        a = "from itertools import bar, filterfalse, foo"
-        self.check(b, a)
+    def test_ifilter_and_zip_longest(self):
+        for name in "filterfalse", "zip_longest":
+            b = "from itertools import i%s" % (name,)
+            a = "from itertools import %s" % (name,)
+            self.check(b, a)
+
+            b = "from itertools import imap, i%s, foo" % (name,)
+            a = "from itertools import %s, foo" % (name,)
+            self.check(b, a)
+
+            b = "from itertools import bar, i%s, foo" % (name,)
+            a = "from itertools import bar, %s, foo" % (name,)
+            self.check(b, a)
 
     def test_import_star(self):
         s = "from itertools import *"
diff --git a/lib-python/modified-2.7/lib2to3/tests/test_parser.py b/lib-python/modified-2.7/lib2to3/tests/test_parser.py
--- a/lib-python/modified-2.7/lib2to3/tests/test_parser.py
+++ b/lib-python/modified-2.7/lib2to3/tests/test_parser.py
@@ -19,6 +19,16 @@
 # Local imports
 from lib2to3.pgen2 import tokenize
 from ..pgen2.parse import ParseError
+from lib2to3.pygram import python_symbols as syms
+
+
+class TestDriver(support.TestCase):
+
+    def test_formfeed(self):
+        s = """print 1\n\x0Cprint 2\n"""
+        t = driver.parse_string(s)
+        self.assertEqual(t.children[0].children[0].type, syms.print_stmt)
+        self.assertEqual(t.children[1].children[0].type, syms.print_stmt)
 
 
 class GrammarTest(support.TestCase):
diff --git a/lib-python/modified-2.7/lib2to3/tests/test_refactor.py b/lib-python/modified-2.7/lib2to3/tests/test_refactor.py
--- a/lib-python/modified-2.7/lib2to3/tests/test_refactor.py
+++ b/lib-python/modified-2.7/lib2to3/tests/test_refactor.py
@@ -223,6 +223,7 @@
                 "hi.py",
                 ".dumb",
                 ".after.py",
+                "notpy.npy",
                 "sappy"]
         expected = ["hi.py"]
         check(tree, expected)
diff --git a/lib-python/modified-2.7/lib2to3/tests/test_util.py b/lib-python/modified-2.7/lib2to3/tests/test_util.py
--- a/lib-python/modified-2.7/lib2to3/tests/test_util.py
+++ b/lib-python/modified-2.7/lib2to3/tests/test_util.py
@@ -568,8 +568,8 @@
 
     def test_from_import(self):
         node = parse('bar()')
-        fixer_util.touch_import("cgi", "escape", node)
-        self.assertEqual(str(node), 'from cgi import escape\nbar()\n\n')
+        fixer_util.touch_import("html", "escape", node)
+        self.assertEqual(str(node), 'from html import escape\nbar()\n\n')
 
     def test_name_import(self):
         node = parse('bar()')
diff --git a/lib-python/modified-2.7/multiprocessing/__init__.py b/lib-python/modified-2.7/multiprocessing/__init__.py
--- a/lib-python/modified-2.7/multiprocessing/__init__.py
+++ b/lib-python/modified-2.7/multiprocessing/__init__.py
@@ -38,6 +38,7 @@
 # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
 #
 
 __version__ = '0.70a1'
@@ -115,8 +116,11 @@
         except (ValueError, KeyError):
             num = 0
     elif 'bsd' in sys.platform or sys.platform == 'darwin':
+        comm = '/sbin/sysctl -n hw.ncpu'
+        if sys.platform == 'darwin':
+            comm = '/usr' + comm
         try:
-            with os.popen('sysctl -n hw.ncpu') as p:
+            with os.popen(comm) as p:
                 num = int(p.read())
         except ValueError:
             num = 0
diff --git a/lib-python/modified-2.7/multiprocessing/connection.py b/lib-python/modified-2.7/multiprocessing/connection.py
--- a/lib-python/modified-2.7/multiprocessing/connection.py
+++ b/lib-python/modified-2.7/multiprocessing/connection.py
@@ -3,7 +3,33 @@
 #
 # multiprocessing/connection.py
 #
-# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt
+# Copyright (c) 2006-2008, R Oudkerk
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+# 3. Neither the name of author nor the names of any contributors may be
+#    used to endorse or promote products derived from this software
+#    without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
 #
 
 __all__ = [ 'Client', 'Listener', 'Pipe' ]
diff --git a/lib-python/modified-2.7/multiprocessing/dummy/__init__.py b/lib-python/modified-2.7/multiprocessing/dummy/__init__.py
--- a/lib-python/modified-2.7/multiprocessing/dummy/__init__.py
+++ b/lib-python/modified-2.7/multiprocessing/dummy/__init__.py
@@ -3,7 +3,33 @@
 #
 # multiprocessing/dummy/__init__.py
 #
-# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt
+# Copyright (c) 2006-2008, R Oudkerk
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+# 3. Neither the name of author nor the names of any contributors may be
+#    used to endorse or promote products derived from this software
+#    without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
 #
 
 __all__ = [
diff --git a/lib-python/modified-2.7/multiprocessing/dummy/connection.py b/lib-python/modified-2.7/multiprocessing/dummy/connection.py
--- a/lib-python/modified-2.7/multiprocessing/dummy/connection.py
+++ b/lib-python/modified-2.7/multiprocessing/dummy/connection.py
@@ -3,7 +3,33 @@
 #
 # multiprocessing/dummy/connection.py
 #
-# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt
+# Copyright (c) 2006-2008, R Oudkerk
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+# 3. Neither the name of author nor the names of any contributors may be
+#    used to endorse or promote products derived from this software
+#    without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
 #
 
 __all__ = [ 'Client', 'Listener', 'Pipe' ]
diff --git a/lib-python/modified-2.7/multiprocessing/forking.py b/lib-python/modified-2.7/multiprocessing/forking.py
--- a/lib-python/modified-2.7/multiprocessing/forking.py
+++ b/lib-python/modified-2.7/multiprocessing/forking.py
@@ -3,7 +3,33 @@
 #
 # multiprocessing/forking.py
 #
-# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt
+# Copyright (c) 2006-2008, R Oudkerk
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+# 3. Neither the name of author nor the names of any contributors may be
+#    used to endorse or promote products derived from this software
+#    without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
 #
 
 import os
@@ -169,6 +195,7 @@
 
     TERMINATE = 0x10000
     WINEXE = (sys.platform == 'win32' and getattr(sys, 'frozen', False))
+    WINSERVICE = sys.executable.lower().endswith("pythonservice.exe")
 
     exit = win32.ExitProcess
     close = win32.CloseHandle
@@ -178,7 +205,7 @@
     # People embedding Python want to modify it.
     #
 
-    if sys.executable.lower().endswith('pythonservice.exe'):
+    if WINSERVICE:
         _python_exe = os.path.join(sys.exec_prefix, 'python.exe')
     else:
         _python_exe = sys.executable
@@ -368,7 +395,7 @@
         if _logger is not None:
             d['log_level'] = _logger.getEffectiveLevel()
 
-        if not WINEXE:
+        if not WINEXE and not WINSERVICE:
             main_path = getattr(sys.modules['__main__'], '__file__', None)
             if not main_path and sys.argv[0] not in ('', '-c'):
                 main_path = sys.argv[0]
diff --git a/lib-python/modified-2.7/multiprocessing/heap.py b/lib-python/modified-2.7/multiprocessing/heap.py
--- a/lib-python/modified-2.7/multiprocessing/heap.py
+++ b/lib-python/modified-2.7/multiprocessing/heap.py
@@ -3,7 +3,33 @@
 #
 # multiprocessing/heap.py
 #
-# Copyright (c) 2007-2008, R Oudkerk --- see COPYING.txt
+# Copyright (c) 2006-2008, R Oudkerk
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+# 3. Neither the name of author nor the names of any contributors may be
+#    used to endorse or promote products derived from this software
+#    without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
 #
 
 import bisect
diff --git a/lib-python/modified-2.7/multiprocessing/managers.py b/lib-python/modified-2.7/multiprocessing/managers.py
--- a/lib-python/modified-2.7/multiprocessing/managers.py
+++ b/lib-python/modified-2.7/multiprocessing/managers.py
@@ -4,7 +4,33 @@
 #
 # multiprocessing/managers.py
 #
-# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt
+# Copyright (c) 2006-2008, R Oudkerk
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+# 3. Neither the name of author nor the names of any contributors may be
+#    used to endorse or promote products derived from this software
+#    without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
 #
 
 __all__ = [ 'BaseManager', 'SyncManager', 'BaseProxy', 'Token' ]
diff --git a/lib-python/modified-2.7/multiprocessing/pool.py b/lib-python/modified-2.7/multiprocessing/pool.py
--- a/lib-python/modified-2.7/multiprocessing/pool.py
+++ b/lib-python/modified-2.7/multiprocessing/pool.py
@@ -3,7 +3,33 @@
 #
 # multiprocessing/pool.py
 #
-# Copyright (c) 2007-2008, R Oudkerk --- see COPYING.txt
+# Copyright (c) 2006-2008, R Oudkerk
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+# 3. Neither the name of author nor the names of any contributors may be
+#    used to endorse or promote products derived from this software
+#    without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
 #
 
 __all__ = ['Pool']
@@ -269,6 +295,8 @@
         while pool._worker_handler._state == RUN and pool._state == RUN:
             pool._maintain_pool()
             time.sleep(0.1)
+        # send sentinel to stop workers
+        pool._taskqueue.put(None)
         debug('worker handler exiting')
 
     @staticmethod
@@ -387,7 +415,6 @@
         if self._state == RUN:
             self._state = CLOSE
             self._worker_handler._state = CLOSE
-            self._taskqueue.put(None)
 
     def terminate(self):
         debug('terminating pool')
@@ -421,7 +448,6 @@
 
         worker_handler._state = TERMINATE
         task_handler._state = TERMINATE
-        taskqueue.put(None)                 # sentinel
 
         debug('helping task handler/workers to finish')
         cls._help_stuff_finish(inqueue, task_handler, len(pool))
@@ -431,6 +457,11 @@
         result_handler._state = TERMINATE
         outqueue.put(None)                  # sentinel
 
+        # We must wait for the worker handler to exit before terminating
+        # workers because we don't want workers to be restarted behind our back.
+        debug('joining worker handler')
+        worker_handler.join()
+
         # Terminate workers which haven't already finished.
         if pool and hasattr(pool[0], 'terminate'):
             debug('terminating workers')
diff --git a/lib-python/modified-2.7/multiprocessing/process.py b/lib-python/modified-2.7/multiprocessing/process.py
--- a/lib-python/modified-2.7/multiprocessing/process.py
+++ b/lib-python/modified-2.7/multiprocessing/process.py
@@ -3,7 +3,33 @@
 #
 # multiprocessing/process.py
 #
-# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt
+# Copyright (c) 2006-2008, R Oudkerk
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+# 3. Neither the name of author nor the names of any contributors may be
+#    used to endorse or promote products derived from this software
+#    without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
 #
 
 __all__ = ['Process', 'current_process', 'active_children']
diff --git a/lib-python/modified-2.7/multiprocessing/queues.py b/lib-python/modified-2.7/multiprocessing/queues.py
--- a/lib-python/modified-2.7/multiprocessing/queues.py
+++ b/lib-python/modified-2.7/multiprocessing/queues.py
@@ -3,7 +3,33 @@
 #
 # multiprocessing/queues.py
 #
-# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt
+# Copyright (c) 2006-2008, R Oudkerk
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+# 3. Neither the name of author nor the names of any contributors may be
+#    used to endorse or promote products derived from this software
+#    without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
 #
 
 __all__ = ['Queue', 'SimpleQueue', 'JoinableQueue']
diff --git a/lib-python/modified-2.7/multiprocessing/reduction.py b/lib-python/modified-2.7/multiprocessing/reduction.py
--- a/lib-python/modified-2.7/multiprocessing/reduction.py
+++ b/lib-python/modified-2.7/multiprocessing/reduction.py
@@ -4,7 +4,33 @@
 #
 # multiprocessing/reduction.py
 #
-# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt
+# Copyright (c) 2006-2008, R Oudkerk
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+# 3. Neither the name of author nor the names of any contributors may be
+#    used to endorse or promote products derived from this software
+#    without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
 #
 
 __all__ = []
diff --git a/lib-python/modified-2.7/multiprocessing/sharedctypes.py b/lib-python/modified-2.7/multiprocessing/sharedctypes.py
--- a/lib-python/modified-2.7/multiprocessing/sharedctypes.py
+++ b/lib-python/modified-2.7/multiprocessing/sharedctypes.py
@@ -3,7 +3,33 @@
 #
 # multiprocessing/sharedctypes.py
 #
-# Copyright (c) 2007-2008, R Oudkerk --- see COPYING.txt
+# Copyright (c) 2006-2008, R Oudkerk
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+# 3. Neither the name of author nor the names of any contributors may be
+#    used to endorse or promote products derived from this software
+#    without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
 #
 
 import sys
@@ -52,9 +78,11 @@
     Returns a ctypes array allocated from shared memory
     '''
     type_ = typecode_to_type.get(typecode_or_type, typecode_or_type)
-    if isinstance(size_or_initializer, int):
+    if isinstance(size_or_initializer, (int, long)):
         type_ = type_ * size_or_initializer
-        return _new_value(type_)
+        obj = _new_value(type_)
+        ctypes.memset(ctypes.addressof(obj), 0, ctypes.sizeof(obj))
+        return obj
     else:
         type_ = type_ * len(size_or_initializer)
         result = _new_value(type_)
diff --git a/lib-python/modified-2.7/multiprocessing/synchronize.py b/lib-python/modified-2.7/multiprocessing/synchronize.py
--- a/lib-python/modified-2.7/multiprocessing/synchronize.py
+++ b/lib-python/modified-2.7/multiprocessing/synchronize.py
@@ -3,7 +3,33 @@
 #
 # multiprocessing/synchronize.py
 #
-# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt
+# Copyright (c) 2006-2008, R Oudkerk
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+# 3. Neither the name of author nor the names of any contributors may be
+#    used to endorse or promote products derived from this software
+#    without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
 #
 
 __all__ = [
diff --git a/lib-python/modified-2.7/multiprocessing/util.py b/lib-python/modified-2.7/multiprocessing/util.py
--- a/lib-python/modified-2.7/multiprocessing/util.py
+++ b/lib-python/modified-2.7/multiprocessing/util.py
@@ -3,7 +3,33 @@
 #
 # multiprocessing/util.py
 #
-# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt
+# Copyright (c) 2006-2008, R Oudkerk
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+# 3. Neither the name of author nor the names of any contributors may be
+#    used to endorse or promote products derived from this software
+#    without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
 #
 
 import itertools
diff --git a/lib-python/modified-2.7/pkgutil.py b/lib-python/modified-2.7/pkgutil.py
--- a/lib-python/modified-2.7/pkgutil.py
+++ b/lib-python/modified-2.7/pkgutil.py
@@ -11,7 +11,7 @@
 
 __all__ = [
     'get_importer', 'iter_importers', 'get_loader', 'find_loader',
-    'walk_packages', 'iter_modules',
+    'walk_packages', 'iter_modules', 'get_data',
     'ImpImporter', 'ImpLoader', 'read_code', 'extend_path',
 ]
 
diff --git a/lib-python/modified-2.7/pydoc.py b/lib-python/modified-2.7/pydoc.py
--- a/lib-python/modified-2.7/pydoc.py
+++ b/lib-python/modified-2.7/pydoc.py
@@ -156,7 +156,7 @@
             no.append(x)
     return yes, no
 
-def visiblename(name, all=None):
+def visiblename(name, all=None, obj=None):
     """Decide whether to show documentation on a variable."""
     # Certain special names are redundant.
     _hidden_names = ('__builtins__', '__doc__', '__file__', '__path__',
@@ -164,6 +164,9 @@
     if name in _hidden_names: return 0
     # Private names are hidden, but special names are displayed.
     if name.startswith('__') and name.endswith('__'): return 1
+    # Namedtuples have public fields and methods with a single leading underscore
+    if name.startswith('_') and hasattr(obj, '_fields'):
+        return 1
     if all is not None:
         # only document that which the programmer exported in __all__
         return name in all
@@ -475,9 +478,9 @@
     def multicolumn(self, list, format, cols=4):
         """Format a list of items into a multi-column list."""
         result = ''
-        rows = (len(list)+cols-1)/cols
+        rows = (len(list)+cols-1)//cols
         for col in range(cols):
-            result = result + '' % (100/cols)
+            result = result + '' % (100//cols)
             for i in range(rows*col, rows*col+rows):
                 if i < len(list):
                     result = result + format(list[i]) + '
\n'
@@ -629,7 +632,7 @@
             # if __all__ exists, believe it.  Otherwise use old heuristic.
             if (all is not None or
                 (inspect.getmodule(value) or object) is object):
-                if visiblename(key, all):
+                if visiblename(key, all, object):
                     classes.append((key, value))
                     cdict[key] = cdict[value] = '#' + key
         for key, value in classes:
@@ -645,13 +648,13 @@
             # if __all__ exists, believe it.  Otherwise use old heuristic.
             if (all is not None or
                 inspect.isbuiltin(value) or inspect.getmodule(value) is object):
-                if visiblename(key, all):
+                if visiblename(key, all, object):
                     funcs.append((key, value))
                     fdict[key] = '#-' + key
                     if inspect.isfunction(value): fdict[value] = fdict[key]
         data = []
         for key, value in inspect.getmembers(object, isdata):
-            if visiblename(key, all):
+            if visiblename(key, all, object):
                 data.append((key, value))
 
         doc = self.markup(getdoc(object), self.preformat, fdict, cdict)
@@ -775,7 +778,7 @@
                     push('\n')
             return attrs
 
-        attrs = filter(lambda data: visiblename(data[0]),
+        attrs = filter(lambda data: visiblename(data[0], obj=object),
                        classify_class_attrs(object))
         mdict = {}
         for key, kind, homecls, value in attrs:
@@ -1044,18 +1047,18 @@
             # if __all__ exists, believe it.  Otherwise use old heuristic.
             if (all is not None
                 or (inspect.getmodule(value) or object) is object):
-                if visiblename(key, all):
+                if visiblename(key, all, object):
                     classes.append((key, value))
         funcs = []
         for key, value in inspect.getmembers(object, inspect.isroutine):
             # if __all__ exists, believe it.  Otherwise use old heuristic.
             if (all is not None or
                 inspect.isbuiltin(value) or inspect.getmodule(value) is object):
-                if visiblename(key, all):
+                if visiblename(key, all, object):
                     funcs.append((key, value))
         data = []
         for key, value in inspect.getmembers(object, isdata):
-            if visiblename(key, all):
+            if visiblename(key, all, object):
                 data.append((key, value))
 
         modpkgs = []
@@ -1115,7 +1118,7 @@
             result = result + self.section('CREDITS', str(object.__credits__))
         return result
 
-    def docclass(self, object, name=None, mod=None):
+    def docclass(self, object, name=None, mod=None, *ignored):
         """Produce text documentation for a given class object."""
         realname = object.__name__
         name = name or realname
@@ -1188,7 +1191,7 @@
                                        name, mod, maxlen=70, doc=doc) + '\n')
             return attrs
 
-        attrs = filter(lambda data: visiblename(data[0]),
+        attrs = filter(lambda data: visiblename(data[0], obj=object),
                        classify_class_attrs(object))
         while attrs:
             if mro:
@@ -1720,8 +1723,9 @@
             return ''
         return ''
 
-    def __call__(self, request=None):
-        if request is not None:
+    _GoInteractive = object()
+    def __call__(self, request=_GoInteractive):
+        if request is not self._GoInteractive:
             self.help(request)
         else:
             self.intro()
diff --git a/lib-python/modified-2.7/random.py b/lib-python/modified-2.7/random.py
--- a/lib-python/modified-2.7/random.py
+++ b/lib-python/modified-2.7/random.py
@@ -316,7 +316,7 @@
 
         n = len(population)
         if not 0 <= k <= n:
-            raise ValueError, "sample larger than population"
+            raise ValueError("sample larger than population")
         random = self.random
         _int = int
         result = [None] * k
@@ -489,6 +489,12 @@
 
         Conditions on the parameters are alpha > 0 and beta > 0.
 
+        The probability distribution function is:
+
+                    x ** (alpha - 1) * math.exp(-x / beta)
+          pdf(x) =  --------------------------------------
+                      math.gamma(alpha) * beta ** alpha
+
         """
 
         # alpha > 0, beta > 0, mean is alpha*beta, variance is alpha*beta**2
@@ -591,7 +597,7 @@
 
 ## -------------------- beta --------------------
 ## See
-## http://sourceforge.net/bugs/?func=detailbug&bug_id=130030&group_id=5470
+## http://mail.python.org/pipermail/python-bugs-list/2001-January/003752.html
 ## for Ivan Frohne's insightful analysis of why the original implementation:
 ##
 ##    def betavariate(self, alpha, beta):
diff --git a/lib-python/modified-2.7/site.py b/lib-python/modified-2.7/site.py
--- a/lib-python/modified-2.7/site.py
+++ b/lib-python/modified-2.7/site.py
@@ -61,6 +61,7 @@
 import sys
 import os
 import __builtin__
+import traceback
 
 # Prefixes for site-packages; add additional prefixes like /usr/local here
 PREFIXES = [sys.prefix, sys.exec_prefix]
@@ -157,17 +158,26 @@
     except IOError:
         return
     with f:
-        for line in f:
+        for n, line in enumerate(f):
             if line.startswith("#"):
                 continue
-            if line.startswith(("import ", "import\t")):
-                exec line
-                continue
-            line = line.rstrip()
-            dir, dircase = makepath(sitedir, line)
-            if not dircase in known_paths and os.path.exists(dir):
-                sys.path.append(dir)
-                known_paths.add(dircase)
+            try:
+                if line.startswith(("import ", "import\t")):
+                    exec line
+                    continue
+                line = line.rstrip()
+                dir, dircase = makepath(sitedir, line)
+                if not dircase in known_paths and os.path.exists(dir):
+                    sys.path.append(dir)
+                    known_paths.add(dircase)
+            except Exception as err:
+                print >>sys.stderr, "Error processing line {:d} of {}:\n".format(
+                    n+1, fullname)
+                for record in traceback.format_exception(*sys.exc_info()):
+                    for line in record.splitlines():
+                        print >>sys.stderr, '  '+line
+                print >>sys.stderr, "\nRemainder of file ignored"
+                break
     if reset:
         known_paths = None
     return known_paths
diff --git a/lib-python/modified-2.7/ssl.py b/lib-python/modified-2.7/ssl.py
--- a/lib-python/modified-2.7/ssl.py
+++ b/lib-python/modified-2.7/ssl.py
@@ -113,9 +113,11 @@
             if e.errno != errno.ENOTCONN:
                 raise
             # no, no connection yet
+            self._connected = False
             self._sslobj = None
         else:
             # yes, create the SSL object
+            self._connected = True
             self._sslobj = _ssl.sslwrap(self._sock, server_side,
                                         keyfile, certfile,
                                         cert_reqs, ssl_version, ca_certs,
@@ -285,21 +287,36 @@
 
         self._sslobj.do_handshake()
 
-    def connect(self, addr):
-
-        """Connects to remote ADDR, and then wraps the connection in
-        an SSL channel."""
-
+    def _real_connect(self, addr, return_errno):
         # Here we assume that the socket is client-side, and not
         # connected at the time of the call.  We connect it, then wrap it.
-        if self._sslobj:
+        if self._connected:
             raise ValueError("attempt to connect already-connected SSLSocket!")
-        socket.connect(self, addr)
         self._sslobj = _ssl.sslwrap(self._sock, False, self.keyfile, self.certfile,
                                     self.cert_reqs, self.ssl_version,
                                     self.ca_certs, self.ciphers)
-        if self.do_handshake_on_connect:
-            self.do_handshake()
+        try:
+            socket.connect(self, addr)
+            if self.do_handshake_on_connect:
+                self.do_handshake()
+        except socket_error as e:
+            if return_errno:
+                return e.errno
+            else:
+                self._sslobj = None
+                raise e
+        self._connected = True
+        return 0
+
+    def connect(self, addr):
+        """Connects to remote ADDR, and then wraps the connection in
+        an SSL channel."""
+        self._real_connect(addr, False)
+
+    def connect_ex(self, addr):
+        """Connects to remote ADDR, and then wraps the connection in
+        an SSL channel."""
+        return self._real_connect(addr, True)
 
     def accept(self):
 
diff --git a/lib-python/modified-2.7/subprocess.py b/lib-python/modified-2.7/subprocess.py
--- a/lib-python/modified-2.7/subprocess.py
+++ b/lib-python/modified-2.7/subprocess.py
@@ -396,6 +396,7 @@
 import traceback
 import gc
 import signal
+import errno
 
 # Exception classes used by this module.
 class CalledProcessError(Exception):
@@ -427,7 +428,6 @@
 else:
     import select
     _has_poll = hasattr(select, 'poll')
-    import errno
     import fcntl
     import pickle
 
@@ -441,8 +441,15 @@
            "check_output", "CalledProcessError"]
 
 if mswindows:
-    from _subprocess import CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP
-    __all__.extend(["CREATE_NEW_CONSOLE", "CREATE_NEW_PROCESS_GROUP"])
+    from _subprocess import (CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP,
+                             STD_INPUT_HANDLE, STD_OUTPUT_HANDLE,
+                             STD_ERROR_HANDLE, SW_HIDE,
+                             STARTF_USESTDHANDLES, STARTF_USESHOWWINDOW)
+
+    __all__.extend(["CREATE_NEW_CONSOLE", "CREATE_NEW_PROCESS_GROUP",
+                    "STD_INPUT_HANDLE", "STD_OUTPUT_HANDLE",
+                    "STD_ERROR_HANDLE", "SW_HIDE",
+                    "STARTF_USESTDHANDLES", "STARTF_USESHOWWINDOW"])
 try:
     MAXFD = os.sysconf("SC_OPEN_MAX")
 except:
@@ -726,7 +733,11 @@
             stderr = None
             if self.stdin:
                 if input:
-                    self.stdin.write(input)
+                    try:
+                        self.stdin.write(input)
+                    except IOError as e:
+                        if e.errno != errno.EPIPE and e.errno != errno.EINVAL:
+                            raise
                 self.stdin.close()
             elif self.stdout:
                 stdout = self.stdout.read()
@@ -887,7 +898,7 @@
             except pywintypes.error, e:
                 # Translate pywintypes.error to WindowsError, which is
                 # a subclass of OSError.  FIXME: We should really
-                # translate errno using _sys_errlist (or simliar), but
+                # translate errno using _sys_errlist (or similar), but
                 # how can this be done from Python?
                 raise WindowsError(*e.args)
             finally:
@@ -960,7 +971,11 @@
 
             if self.stdin:
                 if input is not None:
-                    self.stdin.write(input)
+                    try:
+                        self.stdin.write(input)
+                    except IOError as e:
+                        if e.errno != errno.EPIPE:
+                            raise
                 self.stdin.close()
 
             if self.stdout:
@@ -1055,14 +1070,17 @@
                     errread, errwrite)
 
 
-        def _set_cloexec_flag(self, fd):
+        def _set_cloexec_flag(self, fd, cloexec=True):
             try:
                 cloexec_flag = fcntl.FD_CLOEXEC
             except AttributeError:
                 cloexec_flag = 1
 
             old = fcntl.fcntl(fd, fcntl.F_GETFD)
-            fcntl.fcntl(fd, fcntl.F_SETFD, old | cloexec_flag)
+            if cloexec:
+                fcntl.fcntl(fd, fcntl.F_SETFD, old | cloexec_flag)
+            else:
+                fcntl.fcntl(fd, fcntl.F_SETFD, old & ~cloexec_flag)
 
 
         def _close_fds(self, but):
@@ -1132,21 +1150,25 @@
                             os.close(errpipe_read)
 
                             # Dup fds for child
-                            if p2cread is not None:
-                                os.dup2(p2cread, 0)
-                            if c2pwrite is not None:
-                                os.dup2(c2pwrite, 1)
-                            if errwrite is not None:
-                                os.dup2(errwrite, 2)
+                            def _dup2(a, b):
+                                # dup2() removes the CLOEXEC flag but
+                                # we must do it ourselves if dup2()
+                                # would be a no-op (issue #10806).
+                                if a == b:
+                                    self._set_cloexec_flag(a, False)
+                                elif a is not None:
+                                    os.dup2(a, b)
+                            _dup2(p2cread, 0)
+                            _dup2(c2pwrite, 1)
+                            _dup2(errwrite, 2)
 
-                            # Close pipe fds.  Make sure we don't close the same
-                            # fd more than once, or standard fds.
-                            if p2cread is not None and p2cread not in (0,):
-                                os.close(p2cread)
-                            if c2pwrite is not None and c2pwrite not in (p2cread, 1):
-                                os.close(c2pwrite)
-                            if errwrite is not None and errwrite not in (p2cread, c2pwrite, 2):
-                                os.close(errwrite)
+                            # Close pipe fds.  Make sure we don't close the
+                            # same fd more than once, or standard fds.
+                            closed = { None }
+                            for fd in [p2cread, c2pwrite, errwrite]:
+                                if fd not in closed and fd > 2:
+                                    os.close(fd)
+                                    closed.add(fd)
 
                             # Close all other fds, if asked for
                             if close_fds:
@@ -1198,7 +1220,11 @@
                 os.close(errpipe_read)
 
             if data != "":
-                _eintr_retry_call(os.waitpid, self.pid, 0)
+                try:
+                    _eintr_retry_call(os.waitpid, self.pid, 0)
+                except OSError as e:
+                    if e.errno != errno.ECHILD:
+                        raise
                 child_exception = pickle.loads(data)
                 for fd in (p2cwrite, c2pread, errread):
                     if fd is not None:
@@ -1244,7 +1270,15 @@
             """Wait for child process to terminate.  Returns returncode
             attribute."""
             if self.returncode is None:
-                pid, sts = _eintr_retry_call(os.waitpid, self.pid, 0)
+                try:
+                    pid, sts = _eintr_retry_call(os.waitpid, self.pid, 0)
+                except OSError as e:
+                    if e.errno != errno.ECHILD:
+                        raise
+                    # This happens if SIGCLD is set to be ignored or waiting
+                    # for child processes has otherwise been disabled for our
+                    # process.  This child is dead, we can't get the status.
+                    sts = 0
                 self._handle_exitstatus(sts)
             return self.returncode
 
@@ -1321,9 +1355,16 @@
                 for fd, mode in ready:
                     if mode & select.POLLOUT:
                         chunk = input[input_offset : input_offset + _PIPE_BUF]
-                        input_offset += os.write(fd, chunk)
-                        if input_offset >= len(input):
-                            close_unregister_and_remove(fd)
+                        try:
+                            input_offset += os.write(fd, chunk)
+                        except OSError as e:
+                            if e.errno == errno.EPIPE:
+                                close_unregister_and_remove(fd)
+                            else:
+                                raise
+                        else:
+                            if input_offset >= len(input):
+                                close_unregister_and_remove(fd)
                     elif mode & select_POLLIN_POLLPRI:
                         data = os.read(fd, 4096)
                         if not data:
@@ -1362,11 +1403,19 @@
 
                 if self.stdin in wlist:
                     chunk = input[input_offset : input_offset + _PIPE_BUF]
-                    bytes_written = os.write(self.stdin.fileno(), chunk)
-                    input_offset += bytes_written
-                    if input_offset >= len(input):
-                        self.stdin.close()
-                        write_set.remove(self.stdin)
+                    try:
+                        bytes_written = os.write(self.stdin.fileno(), chunk)
+                    except OSError as e:
+                        if e.errno == errno.EPIPE:
+                            self.stdin.close()
+                            write_set.remove(self.stdin)
+                        else:
+                            raise
+                    else:
+                        input_offset += bytes_written
+                        if input_offset >= len(input):
+                            self.stdin.close()
+                            write_set.remove(self.stdin)
 
                 if self.stdout in rlist:
                     data = os.read(self.stdout.fileno(), 1024)
diff --git a/lib-python/modified-2.7/sysconfig.py b/lib-python/modified-2.7/sysconfig.py
--- a/lib-python/modified-2.7/sysconfig.py
+++ b/lib-python/modified-2.7/sysconfig.py
@@ -497,9 +497,7 @@
         # machine is going to compile and link as if it were
         # MACOSX_DEPLOYMENT_TARGET.
         cfgvars = get_config_vars()
-        macver = os.environ.get('MACOSX_DEPLOYMENT_TARGET')
-        if not macver:
-            macver = cfgvars.get('MACOSX_DEPLOYMENT_TARGET')
+        macver = cfgvars.get('MACOSX_DEPLOYMENT_TARGET')
 
         if 1:
             # Always calculate the release of the running machine,
@@ -520,7 +518,6 @@
                     m = re.search(
                             r'ProductUserVisibleVersion\s*' +
                             r'(.*?)', f.read())
-                    f.close()
                     if m is not None:
                         macrelease = '.'.join(m.group(1).split('.')[:2])
                     # else: fall back to the default behaviour
diff --git a/lib-python/modified-2.7/tarfile.py b/lib-python/modified-2.7/tarfile.py
--- a/lib-python/modified-2.7/tarfile.py
+++ b/lib-python/modified-2.7/tarfile.py
@@ -2236,10 +2236,14 @@
         if hasattr(os, "symlink") and hasattr(os, "link"):
             # For systems that support symbolic and hard links.
             if tarinfo.issym():
+                if os.path.lexists(targetpath):
+                    os.unlink(targetpath)
                 os.symlink(tarinfo.linkname, targetpath)
             else:
                 # See extract().
                 if os.path.exists(tarinfo._link_target):
+                    if os.path.lexists(targetpath):
+                        os.unlink(targetpath)
                     os.link(tarinfo._link_target, targetpath)
                 else:
                     self._extract_member(self._find_link_target(tarinfo), targetpath)
diff --git a/lib-python/modified-2.7/test/pyclbr_input.py b/lib-python/modified-2.7/test/pyclbr_input.py
--- a/lib-python/modified-2.7/test/pyclbr_input.py
+++ b/lib-python/modified-2.7/test/pyclbr_input.py
@@ -19,7 +19,7 @@
 
     # XXX: This causes test_pyclbr.py to fail, but only because the
     #      introspection-based is_method() code in the test can't
-    #      distinguish between this and a geniune method function like m().
+    #      distinguish between this and a genuine method function like m().
     #      The pyclbr.py module gets this right as it parses the text.
     #
     #f = f
diff --git a/lib-python/modified-2.7/test/regrtest.py b/lib-python/modified-2.7/test/regrtest.py
--- a/lib-python/modified-2.7/test/regrtest.py
+++ b/lib-python/modified-2.7/test/regrtest.py
@@ -28,10 +28,12 @@
 -W/--verbose3   -- re-run failed tests in verbose mode immediately
 -q/--quiet      -- no output unless one or more tests fail
 -S/--slow       -- print the slowest 10 tests
+   --header     -- print header with interpreter info
 
 Selecting tests
 
 -r/--random     -- randomize test execution order (see below)
+   --randseed   -- pass a random seed to reproduce a previous random run
 -f/--fromfile   -- read names of tests to run from a file (see below)
 -x/--exclude    -- arguments are tests to *exclude*
 -s/--single     -- single step through a set of tests (see below)
@@ -227,7 +229,8 @@
          exclude=False, single=False, randomize=False, fromfile=None,
          findleaks=False, use_resources=None, trace=False, coverdir='coverage',
          runleaks=False, huntrleaks=False, verbose2=False, print_slow=False,
-         random_seed=None, use_mp=None, verbose3=False, forever=False):
+         random_seed=None, use_mp=None, verbose3=False, forever=False,
+         header=False):
     """Execute a test suite.
 
     This also parses command-line options and modifies its behavior
@@ -258,7 +261,7 @@
              'exclude', 'single', 'slow', 'random', 'fromfile', 'findleaks',
              'use=', 'threshold=', 'trace', 'coverdir=', 'nocoverdir',
              'runleaks', 'huntrleaks=', 'memlimit=', 'randseed=',
-             'multiprocess=', 'slaveargs=', 'forever'])
+             'multiprocess=', 'slaveargs=', 'forever', 'header'])
     except getopt.error, msg:
         usage(2, msg)
 
@@ -342,6 +345,8 @@
             forever = True
         elif o in ('-j', '--multiprocess'):
             use_mp = int(a)
+        elif o == '--header':
+            header = True
         elif o == '--slaveargs':
             args, kwargs = json.loads(a)
             try:
@@ -415,13 +420,14 @@
         args = []
 
     # For a partial run, we do not need to clutter the output.
-    if verbose or not (quiet or single or tests or args):
+    if verbose or header or not (quiet or single or tests or args):
         # Print basic platform information
         print "==", platform.python_implementation(), \
                     " ".join(sys.version.split())
         print "==  ", platform.platform(aliased=True), \
                       "%s-endian" % sys.byteorder
         print "==  ", os.getcwd()
+        print "Testing with flags:", sys.flags
 
     alltests = findtests(testdir, stdtests, nottests)
     selected = tests or args or alltests
@@ -484,7 +490,7 @@
         def tests_and_args():
             for test in tests:
                 args_tuple = (
-                    (test, verbose, quiet, testdir),
+                    (test, verbose, quiet),
                     dict(huntrleaks=huntrleaks, use_resources=use_resources)
                 )
                 yield (test, args_tuple)
@@ -551,16 +557,15 @@
             if trace:
                 # If we're tracing code coverage, then we don't exit with status
                 # if on a false return value from main.
-                tracer.runctx('runtest(test, verbose, quiet, testdir)',
+                tracer.runctx('runtest(test, verbose, quiet)',
                               globals=globals(), locals=vars())
             else:
                 try:
-                    result = runtest(test, verbose, quiet,
-                                     testdir, huntrleaks)
+                    result = runtest(test, verbose, quiet, huntrleaks)
                     accumulate_result(test, result)
                     if verbose3 and result[0] == FAILED:
                         print "Re-running test %r in verbose mode" % test
-                        runtest(test, True, quiet, testdir, huntrleaks)
+                        runtest(test, True, quiet, huntrleaks)
                 except KeyboardInterrupt:
                     interrupted = True
                     break
@@ -630,8 +635,7 @@
             sys.stdout.flush()
             try:
                 test_support.verbose = True
-                ok = runtest(test, True, quiet, testdir,
-                             huntrleaks)
+                ok = runtest(test, True, quiet, huntrleaks)
             except KeyboardInterrupt:
                 # print a newline separate from the ^C
                 print
@@ -692,14 +696,13 @@
     return stdtests + sorted(tests)
 
 def runtest(test, verbose, quiet,
-            testdir=None, huntrleaks=False, use_resources=None):
+            huntrleaks=False, use_resources=None):
     """Run a single test.
 
     test -- the name of the test
     verbose -- if true, print more messages
     quiet -- if true, don't print 'skipped' messages (probably redundant)
     test_times -- a list of (time, test_name) pairs
-    testdir -- test directory
     huntrleaks -- run multiple times to test for leaks; requires a debug
                   build; a triple corresponding to -R's three arguments
     Returns one of the test result constants:
@@ -715,8 +718,7 @@
     if use_resources is not None:
         test_support.use_resources = use_resources
     try:
-        return runtest_inner(test, verbose, quiet,
-                             testdir, huntrleaks)
+        return runtest_inner(test, verbose, quiet, huntrleaks)
     finally:
         cleanup_test_droppings(test, verbose)
 
@@ -849,8 +851,7 @@
         return False
 
 
-def runtest_inner(test, verbose, quiet,
-                  testdir=None, huntrleaks=False):
+def runtest_inner(test, verbose, quiet, huntrleaks=False):
     test_support.unload(test)
     if verbose:
         capture_stdout = None
@@ -898,16 +899,16 @@
     except KeyboardInterrupt:
         raise
     except test_support.TestFailed, msg:
-        print "test", test, "failed --", msg
-        sys.stdout.flush()
+        print >>sys.stderr, "test", test, "failed --", msg
+        sys.stderr.flush()
         return FAILED, test_time
     except:
         type, value = sys.exc_info()[:2]
-        print "test", test, "crashed --", str(type) + ":", value
-        sys.stdout.flush()
+        print >>sys.stderr, "test", test, "crashed --", str(type) + ":", value
+        sys.stderr.flush()
         if verbose:
-            traceback.print_exc(file=sys.stdout)
-            sys.stdout.flush()
+            traceback.print_exc(file=sys.stderr)
+            sys.stderr.flush()
         return FAILED, test_time
     else:
         if refleak:
@@ -1504,7 +1505,7 @@
                 # is distributed with Python
                 WIN_ONLY = ["test_unicode_file", "test_winreg",
                             "test_winsound", "test_startfile",
-                            "test_sqlite"]
+                            "test_sqlite", "test_msilib"]
                 for skip in WIN_ONLY:
                     self.expected.add(skip)
 
diff --git a/lib-python/modified-2.7/test/string_tests.py b/lib-python/modified-2.7/test/string_tests.py
--- a/lib-python/modified-2.7/test/string_tests.py
+++ b/lib-python/modified-2.7/test/string_tests.py
@@ -1180,6 +1180,63 @@
         # mixed use of str and unicode
         self.assertEqual('a/b/c'.rpartition(u'/'), ('a/b', '/', 'c'))
 
+    def test_none_arguments(self):
+        # issue 11828
+        s = 'hello'
+        self.checkequal(2, s, 'find', 'l', None)
+        self.checkequal(3, s, 'find', 'l', -2, None)
+        self.checkequal(2, s, 'find', 'l', None, -2)
+        self.checkequal(0, s, 'find', 'h', None, None)
+
+        self.checkequal(3, s, 'rfind', 'l', None)
+        self.checkequal(3, s, 'rfind', 'l', -2, None)
+        self.checkequal(2, s, 'rfind', 'l', None, -2)
+        self.checkequal(0, s, 'rfind', 'h', None, None)
+
+        self.checkequal(2, s, 'index', 'l', None)
+        self.checkequal(3, s, 'index', 'l', -2, None)
+        self.checkequal(2, s, 'index', 'l', None, -2)
+        self.checkequal(0, s, 'index', 'h', None, None)
+
+        self.checkequal(3, s, 'rindex', 'l', None)
+        self.checkequal(3, s, 'rindex', 'l', -2, None)
+        self.checkequal(2, s, 'rindex', 'l', None, -2)
+        self.checkequal(0, s, 'rindex', 'h', None, None)
+
+        self.checkequal(2, s, 'count', 'l', None)
+        self.checkequal(1, s, 'count', 'l', -2, None)
+        self.checkequal(1, s, 'count', 'l', None, -2)
+        self.checkequal(0, s, 'count', 'x', None, None)
+
+        self.checkequal(True, s, 'endswith', 'o', None)
+        self.checkequal(True, s, 'endswith', 'lo', -2, None)
+        self.checkequal(True, s, 'endswith', 'l', None, -2)
+        self.checkequal(False, s, 'endswith', 'x', None, None)
+
+        self.checkequal(True, s, 'startswith', 'h', None)
+        self.checkequal(True, s, 'startswith', 'l', -2, None)
+        self.checkequal(True, s, 'startswith', 'h', None, -2)
+        self.checkequal(False, s, 'startswith', 'x', None, None)
+
+    def test_find_etc_raise_correct_error_messages(self):
+        # issue 11828
+        s = 'hello'
+        x = 'x'
+        self.assertRaisesRegexp(TypeError, r'\bfind\b', s.find,
+                                x, None, None, None)
+        self.assertRaisesRegexp(TypeError, r'\brfind\b', s.rfind,
+                                x, None, None, None)
+        self.assertRaisesRegexp(TypeError, r'\bindex\b', s.index,
+                                x, None, None, None)
+        self.assertRaisesRegexp(TypeError, r'\brindex\b', s.rindex,
+                                x, None, None, None)
+        self.assertRaisesRegexp(TypeError, r'^count\(', s.count,
+                                x, None, None, None)
+        self.assertRaisesRegexp(TypeError, r'^startswith\(', s.startswith,
+                                x, None, None, None)
+        self.assertRaisesRegexp(TypeError, r'^endswith\(', s.endswith,
+                                x, None, None, None)
+
 class MixinStrStringUserStringTest:
     # Additional tests for 8bit strings, i.e. str, UserString and
     # the string module
diff --git a/lib-python/modified-2.7/test/test_argparse.py b/lib-python/modified-2.7/test/test_argparse.py
--- a/lib-python/modified-2.7/test/test_argparse.py
+++ b/lib-python/modified-2.7/test/test_argparse.py
@@ -4,6 +4,7 @@
 import inspect
 import os
 import shutil
+import stat
 import sys
 import textwrap
 import tempfile
@@ -46,15 +47,13 @@
 
     def tearDown(self):
         os.chdir(self.old_dir)
-        while True:
-            try:
-                shutil.rmtree(self.temp_dir)
-            except WindowsError:
-                test_support.gc_collect()
-                continue
-            else:
-                break
-
+        shutil.rmtree(self.temp_dir, True)
+
+    def create_readonly_file(self, filename):
+        file_path = os.path.join(self.temp_dir, filename)
+        with open(file_path, 'w') as file:
+            file.write(filename)
+        os.chmod(file_path, stat.S_IREAD)
 
 class Sig(object):
 
@@ -1452,17 +1451,19 @@
             file = open(os.path.join(self.temp_dir, file_name), 'w')
             file.write(file_name)
             file.close()
+        self.create_readonly_file('readonly')
 
     argument_signatures = [
         Sig('-x', type=argparse.FileType()),
         Sig('spam', type=argparse.FileType('r')),
     ]
-    failures = ['-x', '-x bar']
+    failures = ['-x', '-x bar', 'non-existent-file.txt']
     successes = [
         ('foo', NS(x=None, spam=RFile('foo'))),
         ('-x foo bar', NS(x=RFile('foo'), spam=RFile('bar'))),
         ('bar -x foo', NS(x=RFile('foo'), spam=RFile('bar'))),
         ('-x - -', NS(x=sys.stdin, spam=sys.stdin)),
+        ('readonly', NS(x=None, spam=RFile('readonly'))),
     ]
 
 
@@ -1511,11 +1512,16 @@
 class TestFileTypeW(TempDirMixin, ParserTestCase):
     """Test the FileType option/argument type for writing files"""
 
+    def setUp(self):
+        super(TestFileTypeW, self).setUp()
+        self.create_readonly_file('readonly')
+
     argument_signatures = [
         Sig('-x', type=argparse.FileType('w')),
         Sig('spam', type=argparse.FileType('w')),
     ]
     failures = ['-x', '-x bar']
+    failures = ['-x', '-x bar', 'readonly']
     successes = [
         ('foo', NS(x=None, spam=WFile('foo'))),
         ('-x foo bar', NS(x=WFile('foo'), spam=WFile('bar'))),
@@ -2499,6 +2505,46 @@
         '''
 
 
+class TestMutuallyExclusiveInGroup(MEMixin, TestCase):
+
+    def get_parser(self, required=None):
+        parser = ErrorRaisingArgumentParser(prog='PROG')
+        titled_group = parser.add_argument_group(
+            title='Titled group', description='Group description')
+        mutex_group = \
+            titled_group.add_mutually_exclusive_group(required=required)
+        mutex_group.add_argument('--bar', help='bar help')
+        mutex_group.add_argument('--baz', help='baz help')
+        return parser
+
+    failures = ['--bar X --baz Y', '--baz X --bar Y']
+    successes = [
+        ('--bar X', NS(bar='X', baz=None)),
+        ('--baz Y', NS(bar=None, baz='Y')),
+    ]
+    successes_when_not_required = [
+        ('', NS(bar=None, baz=None)),
+    ]
+
+    usage_when_not_required = '''\
+        usage: PROG [-h] [--bar BAR | --baz BAZ]
+        '''
+    usage_when_required = '''\
+        usage: PROG [-h] (--bar BAR | --baz BAZ)
+        '''
+    help = '''\
+
+        optional arguments:
+          -h, --help  show this help message and exit
+
+        Titled group:
+          Group description
+
+          --bar BAR   bar help
+          --baz BAZ   baz help
+        '''
+
+
 class TestMutuallyExclusiveOptionalsAndPositionalsMixed(MEMixin, TestCase):
 
     def get_parser(self, required):
@@ -2756,16 +2802,22 @@
                 parser = argparse.ArgumentParser(
                     *tester.parser_signature.args,
                     **tester.parser_signature.kwargs)
-                for argument_sig in tester.argument_signatures:
+                for argument_sig in getattr(tester, 'argument_signatures', []):
                     parser.add_argument(*argument_sig.args,
                                         **argument_sig.kwargs)
-                group_signatures = tester.argument_group_signatures
-                for group_sig, argument_sigs in group_signatures:
+                group_sigs = getattr(tester, 'argument_group_signatures', [])
+                for group_sig, argument_sigs in group_sigs:
                     group = parser.add_argument_group(*group_sig.args,
                                                       **group_sig.kwargs)
                     for argument_sig in argument_sigs:
                         group.add_argument(*argument_sig.args,
                                            **argument_sig.kwargs)
+                subparsers_sigs = getattr(tester, 'subparsers_signatures', [])
+                if subparsers_sigs:
+                    subparsers = parser.add_subparsers()
+                    for subparser_sig in subparsers_sigs:
+                        subparsers.add_parser(*subparser_sig.args,
+                                               **subparser_sig.kwargs)
                 return parser
 
             def _test(self, tester, parser_text):
@@ -3859,6 +3911,77 @@
         '''
     version = ''
 
+class TestHelpSubparsersOrdering(HelpTestCase):
+    """Test ordering of subcommands in help matches the code"""
+    parser_signature = Sig(prog='PROG',
+                           description='display some subcommands',
+                           version='0.1')
+
+    subparsers_signatures = [Sig(name=name)
+                             for name in ('a', 'b', 'c', 'd', 'e')]
+
+    usage = '''\
+        usage: PROG [-h] [-v] {a,b,c,d,e} ...
+        '''
+
+    help = usage + '''\
+
+        display some subcommands
+
+        positional arguments:
+          {a,b,c,d,e}
+
+        optional arguments:
+          -h, --help     show this help message and exit
+          -v, --version  show program's version number and exit
+        '''
+
+    version = '''\
+        0.1
+        '''
+
+class TestHelpSubparsersWithHelpOrdering(HelpTestCase):
+    """Test ordering of subcommands in help matches the code"""
+    parser_signature = Sig(prog='PROG',
+                           description='display some subcommands',
+                           version='0.1')
+
+    subcommand_data = (('a', 'a subcommand help'),
+                       ('b', 'b subcommand help'),
+                       ('c', 'c subcommand help'),
+                       ('d', 'd subcommand help'),
+                       ('e', 'e subcommand help'),
+                       )
+
+    subparsers_signatures = [Sig(name=name, help=help)
+                             for name, help in subcommand_data]
+
+    usage = '''\
+        usage: PROG [-h] [-v] {a,b,c,d,e} ...
+        '''
+
+    help = usage + '''\
+
+        display some subcommands
+
+        positional arguments:
+          {a,b,c,d,e}
+            a            a subcommand help
+            b            b subcommand help
+            c            c subcommand help
+            d            d subcommand help
+            e            e subcommand help
+
+        optional arguments:
+          -h, --help     show this help message and exit
+          -v, --version  show program's version number and exit
+        '''
+
+    version = '''\
+        0.1
+        '''
+
+
 # =====================================
 # Optional/Positional constructor tests
 # =====================================
@@ -3893,10 +4016,12 @@
 
     def test_invalid_type(self):
         self.assertValueError('--foo', type='int')
+        self.assertValueError('--foo', type=(int, float))
 
     def test_invalid_action(self):
         self.assertValueError('-x', action='foo')
         self.assertValueError('foo', action='baz')
+        self.assertValueError('--foo', action=('store', 'append'))
         parser = argparse.ArgumentParser()
         try:
             parser.add_argument("--foo", action="store-true")
@@ -4271,7 +4396,7 @@
 # ArgumentTypeError tests
 # =======================
 
-class TestArgumentError(TestCase):
+class TestArgumentTypeError(TestCase):
 
     def test_argument_type_error(self):
 
@@ -4313,6 +4438,177 @@
         self.assertEqual(NS(v=3, spam=True, badger="B"), args)
         self.assertEqual(["C", "--foo", "4"], extras)
 
+# ==========================
+# add_argument metavar tests
+# ==========================
+
+class TestAddArgumentMetavar(TestCase):
+
+    EXPECTED_MESSAGE = "length of metavar tuple does not match nargs"
+
+    def do_test_no_exception(self, nargs, metavar):
+        parser = argparse.ArgumentParser()
+        parser.add_argument("--foo", nargs=nargs, metavar=metavar)
+
+    def do_test_exception(self, nargs, metavar):
+        parser = argparse.ArgumentParser()
+        with self.assertRaises(ValueError) as cm:
+            parser.add_argument("--foo", nargs=nargs, metavar=metavar)
+        self.assertEqual(cm.exception.args[0], self.EXPECTED_MESSAGE)
+
+    # Unit tests for different values of metavar when nargs=None
+
+    def test_nargs_None_metavar_string(self):
+        self.do_test_no_exception(nargs=None, metavar="1")
+
+    def test_nargs_None_metavar_length0(self):
+        self.do_test_exception(nargs=None, metavar=tuple())
+
+    def test_nargs_None_metavar_length1(self):
+        self.do_test_no_exception(nargs=None, metavar=("1"))
+
+    def test_nargs_None_metavar_length2(self):
+        self.do_test_exception(nargs=None, metavar=("1", "2"))
+
+    def test_nargs_None_metavar_length3(self):
+        self.do_test_exception(nargs=None, metavar=("1", "2", "3"))
+
+    # Unit tests for different values of metavar when nargs=?
+
+    def test_nargs_optional_metavar_string(self):
+        self.do_test_no_exception(nargs="?", metavar="1")
+
+    def test_nargs_optional_metavar_length0(self):
+        self.do_test_exception(nargs="?", metavar=tuple())
+
+    def test_nargs_optional_metavar_length1(self):
+        self.do_test_no_exception(nargs="?", metavar=("1"))
+
+    def test_nargs_optional_metavar_length2(self):
+        self.do_test_exception(nargs="?", metavar=("1", "2"))
+
+    def test_nargs_optional_metavar_length3(self):
+        self.do_test_exception(nargs="?", metavar=("1", "2", "3"))
+
+    # Unit tests for different values of metavar when nargs=*
+
+    def test_nargs_zeroormore_metavar_string(self):
+        self.do_test_no_exception(nargs="*", metavar="1")
+
+    def test_nargs_zeroormore_metavar_length0(self):
+        self.do_test_exception(nargs="*", metavar=tuple())
+
+    def test_nargs_zeroormore_metavar_length1(self):
+        self.do_test_no_exception(nargs="*", metavar=("1"))
+
+    def test_nargs_zeroormore_metavar_length2(self):
+        self.do_test_no_exception(nargs="*", metavar=("1", "2"))
+
+    def test_nargs_zeroormore_metavar_length3(self):
+        self.do_test_exception(nargs="*", metavar=("1", "2", "3"))
+
+    # Unit tests for different values of metavar when nargs=+
+
+    def test_nargs_oneormore_metavar_string(self):
+        self.do_test_no_exception(nargs="+", metavar="1")
+
+    def test_nargs_oneormore_metavar_length0(self):
+        self.do_test_exception(nargs="+", metavar=tuple())
+
+    def test_nargs_oneormore_metavar_length1(self):
+        self.do_test_no_exception(nargs="+", metavar=("1"))
+
+    def test_nargs_oneormore_metavar_length2(self):
+        self.do_test_no_exception(nargs="+", metavar=("1", "2"))
+
+    def test_nargs_oneormore_metavar_length3(self):
+        self.do_test_exception(nargs="+", metavar=("1", "2", "3"))
+
+    # Unit tests for different values of metavar when nargs=...
+
+    def test_nargs_remainder_metavar_string(self):
+        self.do_test_no_exception(nargs="...", metavar="1")
+
+    def test_nargs_remainder_metavar_length0(self):
+        self.do_test_no_exception(nargs="...", metavar=tuple())
+
+    def test_nargs_remainder_metavar_length1(self):
+        self.do_test_no_exception(nargs="...", metavar=("1"))
+
+    def test_nargs_remainder_metavar_length2(self):
+        self.do_test_no_exception(nargs="...", metavar=("1", "2"))
+
+    def test_nargs_remainder_metavar_length3(self):
+        self.do_test_no_exception(nargs="...", metavar=("1", "2", "3"))
+
+    # Unit tests for different values of metavar when nargs=A...
+
+    def test_nargs_parser_metavar_string(self):
+        self.do_test_no_exception(nargs="A...", metavar="1")
+
+    def test_nargs_parser_metavar_length0(self):
+        self.do_test_exception(nargs="A...", metavar=tuple())
+
+    def test_nargs_parser_metavar_length1(self):
+        self.do_test_no_exception(nargs="A...", metavar=("1"))
+
+    def test_nargs_parser_metavar_length2(self):
+        self.do_test_exception(nargs="A...", metavar=("1", "2"))
+
+    def test_nargs_parser_metavar_length3(self):
+        self.do_test_exception(nargs="A...", metavar=("1", "2", "3"))
+
+    # Unit tests for different values of metavar when nargs=1
+
+    def test_nargs_1_metavar_string(self):
+        self.do_test_no_exception(nargs=1, metavar="1")
+
+    def test_nargs_1_metavar_length0(self):
+        self.do_test_exception(nargs=1, metavar=tuple())
+
+    def test_nargs_1_metavar_length1(self):
+        self.do_test_no_exception(nargs=1, metavar=("1"))
+
+    def test_nargs_1_metavar_length2(self):
+        self.do_test_exception(nargs=1, metavar=("1", "2"))
+
+    def test_nargs_1_metavar_length3(self):
+        self.do_test_exception(nargs=1, metavar=("1", "2", "3"))
+
+    # Unit tests for different values of metavar when nargs=2
+
+    def test_nargs_2_metavar_string(self):
+        self.do_test_no_exception(nargs=2, metavar="1")
+
+    def test_nargs_2_metavar_length0(self):
+        self.do_test_exception(nargs=2, metavar=tuple())
+
+    def test_nargs_2_metavar_length1(self):
+        self.do_test_no_exception(nargs=2, metavar=("1"))
+
+    def test_nargs_2_metavar_length2(self):
+        self.do_test_no_exception(nargs=2, metavar=("1", "2"))
+
+    def test_nargs_2_metavar_length3(self):
+        self.do_test_exception(nargs=2, metavar=("1", "2", "3"))
+
+    # Unit tests for different values of metavar when nargs=3
+
+    def test_nargs_3_metavar_string(self):
+        self.do_test_no_exception(nargs=3, metavar="1")
+
+    def test_nargs_3_metavar_length0(self):
+        self.do_test_exception(nargs=3, metavar=tuple())
+
+    def test_nargs_3_metavar_length1(self):
+        self.do_test_no_exception(nargs=3, metavar=("1"))
+
+    def test_nargs_3_metavar_length2(self):
+        self.do_test_exception(nargs=3, metavar=("1", "2"))
+
+    def test_nargs_3_metavar_length3(self):
+        self.do_test_no_exception(nargs=3, metavar=("1", "2", "3"))
+
 # ============================
 # from argparse import * tests
 # ============================
diff --git a/lib-python/modified-2.7/test/test_ast.py b/lib-python/modified-2.7/test/test_ast.py
--- a/lib-python/modified-2.7/test/test_ast.py
+++ b/lib-python/modified-2.7/test/test_ast.py
@@ -466,6 +466,14 @@
             'op=Add(), right=Num(n=1, lineno=4, col_offset=4), lineno=4, '
             'col_offset=0))'
         )
+        # issue10869: do not increment lineno of root twice
+        src = ast.parse('1 + 1', mode='eval')
+        self.assertEqual(ast.increment_lineno(src.body, n=3), src.body)
+        self.assertEqual(ast.dump(src, include_attributes=True),
+            'Expression(body=BinOp(left=Num(n=1, lineno=4, col_offset=0), '
+            'op=Add(), right=Num(n=1, lineno=4, col_offset=4), lineno=4, '
+            'col_offset=0))'
+        )
 
     def test_iter_fields(self):
         node = ast.parse('foo()', mode='eval')
diff --git a/lib-python/modified-2.7/test/test_asyncore.py b/lib-python/modified-2.7/test/test_asyncore.py
--- a/lib-python/modified-2.7/test/test_asyncore.py
+++ b/lib-python/modified-2.7/test/test_asyncore.py
@@ -330,7 +330,7 @@
         if hasattr(os, 'strerror'):
             self.assertEqual(err, os.strerror(errno.EPERM))
         err = asyncore._strerror(-1)
-        self.assertIn("unknown error", err.lower())
+        self.assertTrue(err != "")
 
 
 class dispatcherwithsend_noread(asyncore.dispatcher_with_send):
@@ -398,8 +398,6 @@
 class FileWrapperTest(unittest.TestCase):
     def setUp(self):
         self.d = "It's not dead, it's sleeping!"
-        # Fixed in CPython 2.7.2 (release27-maint branch
-        # revision 88046.)
         with file(TESTFN, 'w') as h:
             h.write(self.d)
 
diff --git a/lib-python/modified-2.7/test/test_builtin.py b/lib-python/modified-2.7/test/test_builtin.py
--- a/lib-python/modified-2.7/test/test_builtin.py
+++ b/lib-python/modified-2.7/test/test_builtin.py
@@ -701,7 +701,7 @@
         # provide too much opportunity for insane things to happen.
         # We don't want them in the interned dict and if they aren't
         # actually interned, we don't want to create the appearance
-        # that they are by allowing intern() to succeeed.
+        # that they are by allowing intern() to succeed.
         class S(str):
             def __hash__(self):
                 return 123
diff --git a/lib-python/modified-2.7/test/test_bytes.py b/lib-python/modified-2.7/test/test_bytes.py
--- a/lib-python/modified-2.7/test/test_bytes.py
+++ b/lib-python/modified-2.7/test/test_bytes.py
@@ -456,6 +456,68 @@
         self.assertEqual([ord(b[i:i+1]) for i in range(len(b))],
                          [0, 65, 127, 128, 255])
 
+    def test_none_arguments(self):
+        # issue 11828
+        b = self.type2test(b'hello')
+        l = self.type2test(b'l')
+        h = self.type2test(b'h')
+        x = self.type2test(b'x')
+        o = self.type2test(b'o')
+
+        self.assertEqual(2, b.find(l, None))
+        self.assertEqual(3, b.find(l, -2, None))
+        self.assertEqual(2, b.find(l, None, -2))
+        self.assertEqual(0, b.find(h, None, None))
+
+        self.assertEqual(3, b.rfind(l, None))
+        self.assertEqual(3, b.rfind(l, -2, None))
+        self.assertEqual(2, b.rfind(l, None, -2))
+        self.assertEqual(0, b.rfind(h, None, None))
+
+        self.assertEqual(2, b.index(l, None))
+        self.assertEqual(3, b.index(l, -2, None))
+        self.assertEqual(2, b.index(l, None, -2))
+        self.assertEqual(0, b.index(h, None, None))
+
+        self.assertEqual(3, b.rindex(l, None))
+        self.assertEqual(3, b.rindex(l, -2, None))
+        self.assertEqual(2, b.rindex(l, None, -2))
+        self.assertEqual(0, b.rindex(h, None, None))
+
+        self.assertEqual(2, b.count(l, None))
+        self.assertEqual(1, b.count(l, -2, None))
+        self.assertEqual(1, b.count(l, None, -2))
+        self.assertEqual(0, b.count(x, None, None))
+
+        self.assertEqual(True, b.endswith(o, None))
+        self.assertEqual(True, b.endswith(o, -2, None))
+        self.assertEqual(True, b.endswith(l, None, -2))
+        self.assertEqual(False, b.endswith(x, None, None))
+
+        self.assertEqual(True, b.startswith(h, None))
+        self.assertEqual(True, b.startswith(l, -2, None))
+        self.assertEqual(True, b.startswith(h, None, -2))
+        self.assertEqual(False, b.startswith(x, None, None))
+
+    def test_find_etc_raise_correct_error_messages(self):
+        # issue 11828
+        b = self.type2test(b'hello')
+        x = self.type2test(b'x')
+        self.assertRaisesRegexp(TypeError, r'\bfind\b', b.find,
+                                x, None, None, None)
+        self.assertRaisesRegexp(TypeError, r'\brfind\b', b.rfind,
+                                x, None, None, None)
+        self.assertRaisesRegexp(TypeError, r'\bindex\b', b.index,
+                                x, None, None, None)
+        self.assertRaisesRegexp(TypeError, r'\brindex\b', b.rindex,
+                                x, None, None, None)
+        self.assertRaisesRegexp(TypeError, r'\bcount\b', b.count,
+                                x, None, None, None)
+        self.assertRaisesRegexp(TypeError, r'\bstartswith\b', b.startswith,
+                                x, None, None, None)
+        self.assertRaisesRegexp(TypeError, r'\bendswith\b', b.endswith,
+                                x, None, None, None)
+
 
 class ByteArrayTest(BaseBytesTest):
     type2test = bytearray
@@ -696,7 +758,7 @@
         self.assertEqual(b.pop(0), ord('w'))
         self.assertEqual(b.pop(-2), ord('r'))
         self.assertRaises(IndexError, lambda: b.pop(10))
-        self.assertRaises(OverflowError, lambda: bytearray().pop())
+        self.assertRaises(IndexError, lambda: bytearray().pop())
         # test for issue #6846
         self.assertEqual(bytearray(b'\xff').pop(), 0xff)
 
diff --git a/lib-python/modified-2.7/test/test_bz2.py b/lib-python/modified-2.7/test/test_bz2.py
--- a/lib-python/modified-2.7/test/test_bz2.py
+++ b/lib-python/modified-2.7/test/test_bz2.py
@@ -86,7 +86,7 @@
                 if not str:
                     break
                 text += str
-            self.assertEqual(text, text)
+            self.assertEqual(text, self.TEXT)
 
     def testRead100(self):
         # "Test BZ2File.read(100)"
diff --git a/lib-python/modified-2.7/test/test_compile.py b/lib-python/modified-2.7/test/test_compile.py
--- a/lib-python/modified-2.7/test/test_compile.py
+++ b/lib-python/modified-2.7/test/test_compile.py
@@ -254,7 +254,7 @@
             self.assertEqual(eval("-" + all_one_bits), -18446744073709551615L)
         else:
             self.fail("How many bits *does* this machine have???")
-        # Verify treatment of contant folding on -(sys.maxint+1)
+        # Verify treatment of constant folding on -(sys.maxint+1)
         # i.e. -2147483648 on 32 bit platforms.  Should return int, not long.
         self.assertIsInstance(eval("%s" % (-sys.maxint - 1)), int)
         self.assertIsInstance(eval("%s" % (-sys.maxint - 2)), long)
diff --git a/lib-python/modified-2.7/test/test_csv.py b/lib-python/modified-2.7/test/test_csv.py
--- a/lib-python/modified-2.7/test/test_csv.py
+++ b/lib-python/modified-2.7/test/test_csv.py
@@ -328,22 +328,17 @@
         expected_dialects = csv.list_dialects() + [name]
         expected_dialects.sort()
         csv.register_dialect(name, myexceltsv)
-        try:
-            self.assertTrue(csv.get_dialect(name).delimiter, '\t')
-            got_dialects = csv.list_dialects()
-            got_dialects.sort()
-            self.assertEqual(expected_dialects, got_dialects)
-        finally:
-            csv.unregister_dialect(name)
+        self.addCleanup(csv.unregister_dialect, name)
+        self.assertEqual(csv.get_dialect(name).delimiter, '\t')
+        got_dialects = sorted(csv.list_dialects())
+        self.assertEqual(expected_dialects, got_dialects)
 
     def test_register_kwargs(self):
         name = 'fedcba'
         csv.register_dialect(name, delimiter=';')
-        try:
-            self.assertTrue(csv.get_dialect(name).delimiter, '\t')
-            self.assertTrue(list(csv.reader('X;Y;Z', name)), ['X', 'Y', 'Z'])
-        finally:
-            csv.unregister_dialect(name)
+        self.addCleanup(csv.unregister_dialect, name)
+        self.assertEqual(csv.get_dialect(name).delimiter, ';')
+        self.assertEqual([['X', 'Y', 'Z']], list(csv.reader(['X;Y;Z'], name)))
 
     def test_incomplete_dialect(self):
         class myexceltsv(csv.Dialect):
diff --git a/lib-python/modified-2.7/test/test_deque.py b/lib-python/modified-2.7/test/test_deque.py
--- a/lib-python/modified-2.7/test/test_deque.py
+++ b/lib-python/modified-2.7/test/test_deque.py
@@ -137,6 +137,15 @@
         m.d = d
         self.assertRaises(RuntimeError, d.count, 3)
 
+        # test issue11004
+        # block advance failed after rotation aligned elements on right side of block
+        d = deque([None]*16)
+        for i in range(len(d)):
+            d.rotate(-1)
+        d.rotate(1)
+        self.assertEqual(d.count(1), 0)
+        self.assertEqual(d.count(None), 16)
+
     def test_comparisons(self):
         d = deque('xabc'); d.popleft()
         for e in [d, deque('abc'), deque('ab'), deque(), list(d)]:
diff --git a/lib-python/modified-2.7/test/test_descr.py b/lib-python/modified-2.7/test/test_descr.py
--- a/lib-python/modified-2.7/test/test_descr.py
+++ b/lib-python/modified-2.7/test/test_descr.py
@@ -877,7 +877,7 @@
     # see "A Monotonic Superclass Linearization for Dylan",
     # by Kim Barrett et al. (OOPSLA 1996)
     def test_consistency_with_epg(self):
-        # Testing consistentcy with EPG...
+        # Testing consistency with EPG...
         class Pane(object): pass
         class ScrollingMixin(object): pass
         class EditingMixin(object): pass
@@ -1719,6 +1719,7 @@
             ("__exit__", run_context, swallow, set(), {"__enter__" : iden}),
             ("__complex__", complex, complex_num, set(), {}),
             ("__format__", format, format_impl, set(), {}),
+            ("__dir__", dir, empty_seq, set(), {}),
             ]
 
         class Checker(object):
@@ -4299,7 +4300,7 @@
         except TypeError:
             pass
         else:
-            self.fail("Carlo Verre __setattr__ suceeded!")
+            self.fail("Carlo Verre __setattr__ succeeded!")
         try:
             object.__delattr__(str, "lower")
         except TypeError:
@@ -4567,6 +4568,33 @@
 
         self.assertRaises(AttributeError, getattr, EvilGetattribute(), "attr")
 
+    def test_abstractmethods(self):
+        # type pretends not to have __abstractmethods__.
+        self.assertRaises(AttributeError, getattr, type, "__abstractmethods__")
+        class meta(type):
+            pass
+        self.assertRaises(AttributeError, getattr, meta, "__abstractmethods__")
+        class X(object):
+            pass
+        with self.assertRaises(AttributeError):
+            del X.__abstractmethods__
+
+    def test_proxy_call(self):
+        class FakeStr(object):
+            __class__ = str
+
+        fake_str = FakeStr()
+        # isinstance() reads __class__ on new style classes
+        self.assertTrue(isinstance(fake_str, str))
+
+        # call a method descriptor
+        with self.assertRaises(TypeError):
+            str.split(fake_str)
+
+        # call a slot wrapper descriptor
+        with self.assertRaises(TypeError):
+            str.__add__(fake_str, "abc")
+
 
 class DictProxyTests(unittest.TestCase):
     def setUp(self):
diff --git a/lib-python/modified-2.7/test/test_doctest.py b/lib-python/modified-2.7/test/test_doctest.py
--- a/lib-python/modified-2.7/test/test_doctest.py
+++ b/lib-python/modified-2.7/test/test_doctest.py
@@ -1292,7 +1292,7 @@
         ?     +              ++    ^
     TestResults(failed=1, attempted=1)
 
-The REPORT_ONLY_FIRST_FAILURE supresses result output after the first
+The REPORT_ONLY_FIRST_FAILURE suppresses result output after the first
 failing example:
 
     >>> def f(x):
@@ -1322,7 +1322,7 @@
         2
     TestResults(failed=3, attempted=5)
 
-However, output from `report_start` is not supressed:
+However, output from `report_start` is not suppressed:
 
     >>> doctest.DocTestRunner(verbose=True, optionflags=flags).run(test)
     ... # doctest: +ELLIPSIS
@@ -2334,7 +2334,7 @@
     TestResults(failed=0, attempted=2)
     >>> doctest.master = None  # Reset master.
 
-Verbosity can be increased with the optional `verbose` paremter:
+Verbosity can be increased with the optional `verbose` parameter:
 
     >>> doctest.testfile('test_doctest.txt', globs=globs, verbose=True)
     Trying:
@@ -2371,7 +2371,7 @@
     TestResults(failed=1, attempted=2)
     >>> doctest.master = None  # Reset master.
 
-The summary report may be supressed with the optional `report`
+The summary report may be suppressed with the optional `report`
 parameter:
 
     >>> doctest.testfile('test_doctest.txt', report=False)
diff --git a/lib-python/modified-2.7/test/test_extcall.py b/lib-python/modified-2.7/test/test_extcall.py
--- a/lib-python/modified-2.7/test/test_extcall.py
+++ b/lib-python/modified-2.7/test/test_extcall.py
@@ -234,7 +234,7 @@
     TypeError: unbound method method() must be called with Foo instance as \
 first argument (got int instance instead)
 
-A PyCFunction that takes only positional parameters shoud allow an
+A PyCFunction that takes only positional parameters should allow an
 empty keyword dictionary to pass without a complaint, but raise a
 TypeError if te dictionary is not empty
 
diff --git a/lib-python/modified-2.7/test/test_functools.py b/lib-python/modified-2.7/test/test_functools.py
--- a/lib-python/modified-2.7/test/test_functools.py
+++ b/lib-python/modified-2.7/test/test_functools.py
@@ -364,6 +364,8 @@
                 self.value = value
             def __lt__(self, other):
                 return self.value < other.value
+            def __eq__(self, other):
+                return self.value == other.value
         self.assertTrue(A(1) < A(2))
         self.assertTrue(A(2) > A(1))
         self.assertTrue(A(1) <= A(2))
@@ -378,6 +380,8 @@
                 self.value = value
             def __le__(self, other):
                 return self.value <= other.value
+            def __eq__(self, other):
+                return self.value == other.value
         self.assertTrue(A(1) < A(2))
         self.assertTrue(A(2) > A(1))
         self.assertTrue(A(1) <= A(2))
@@ -392,6 +396,8 @@
                 self.value = value
             def __gt__(self, other):
                 return self.value > other.value
+            def __eq__(self, other):
+                return self.value == other.value
         self.assertTrue(A(1) < A(2))
         self.assertTrue(A(2) > A(1))
         self.assertTrue(A(1) <= A(2))
@@ -406,6 +412,8 @@
                 self.value = value
             def __ge__(self, other):
                 return self.value >= other.value
+            def __eq__(self, other):
+                return self.value == other.value
         self.assertTrue(A(1) < A(2))
         self.assertTrue(A(2) > A(1))
         self.assertTrue(A(1) <= A(2))
@@ -431,6 +439,22 @@
             class A:
                 pass
 
+    def test_bug_10042(self):
+        @functools.total_ordering
+        class TestTO:
+            def __init__(self, value):
+                self.value = value
+            def __eq__(self, other):
+                if isinstance(other, TestTO):
+                    return self.value == other.value
+                return False
+            def __lt__(self, other):
+                if isinstance(other, TestTO):
+                    return self.value < other.value
+                raise TypeError
+        with self.assertRaises(TypeError):
+            TestTO(8) <= ()
+
 def test_main(verbose=None):
     test_classes = (
         TestPartial,
diff --git a/lib-python/modified-2.7/test/test_heapq.py b/lib-python/modified-2.7/test/test_heapq.py
--- a/lib-python/modified-2.7/test/test_heapq.py
+++ b/lib-python/modified-2.7/test/test_heapq.py
@@ -1,18 +1,31 @@
 """Unittests for heapq."""
 
+import sys
 import random
-import unittest
+
 from test import test_support
-import sys
+from unittest import TestCase, skipUnless
 
-# We do a bit of trickery here to be able to test both the C implementation
-# and the Python implementation of the module.
-import heapq as c_heapq
-if '_heapq' not in sys.modules:
-    c_heapq = None    # we don't have a _heapq module to test at all
 py_heapq = test_support.import_fresh_module('heapq', blocked=['_heapq'])
+c_heapq = test_support.import_fresh_module('heapq', fresh=['_heapq'])
 
-class TestHeap(unittest.TestCase):
+# _heapq.nlargest/nsmallest are saved in heapq._nlargest/_smallest when
+# _heapq is imported, so check them there
+func_names = ['heapify', 'heappop', 'heappush', 'heappushpop',
+              'heapreplace', '_nlargest', '_nsmallest']
+
+class TestModules(TestCase):
+    def test_py_functions(self):
+        for fname in func_names:
+            self.assertEqual(getattr(py_heapq, fname).__module__, 'heapq')
+
+    @skipUnless(c_heapq, 'requires _heapq')
+    def test_c_functions(self):
+        for fname in func_names:
+            self.assertEqual(getattr(c_heapq, fname).__module__, '_heapq')
+
+
+class TestHeap(TestCase):
     module = None
 
     def test_push_pop(self):
@@ -177,26 +190,8 @@
                 self.assertEqual(self.module.nlargest(n, data, key=f),
                                  sorted(data, key=f, reverse=True)[:n])
 
-class TestHeapPython(TestHeap):
-    module = py_heapq
-
-    # As an early adopter, we sanity check the
-    # test_support.import_fresh_module utility function
-    def test_pure_python(self):
-        self.assertFalse(sys.modules['heapq'] is self.module)
-        self.assertTrue(hasattr(self.module.heapify, 'func_code'))
-
-    def test_islice_protection(self):
-        m = self.module
-        self.assertFalse(m.nsmallest(-1, [1]))
-        self.assertFalse(m.nlargest(-1, [1]))
-
-
-class TestHeapC(TestHeap):
-    module = c_heapq
-
     def test_comparison_operator(self):
-        # Issue 3501: Make sure heapq works with both __lt__ and __le__
+        # Issue 3051: Make sure heapq works with both __lt__ and __le__
         def hsort(data, comp):
             data = map(comp, data)
             self.module.heapify(data)
@@ -216,11 +211,19 @@
         self.assertEqual(hsort(data, LT), target)
         self.assertEqual(hsort(data, LE), target)
 
-    # As an early adopter, we sanity check the
-    # test_support.import_fresh_module utility function
-    def test_accelerated(self):
-        self.assertTrue(sys.modules['heapq'] is self.module)
-        self.assertFalse(hasattr(self.module.heapify, 'func_code'))
+
+class TestHeapPython(TestHeap):
+    module = py_heapq
+
+    def test_islice_protection(self):
+        m = self.module
+        self.assertFalse(m.nsmallest(-1, [1]))
+        self.assertFalse(m.nlargest(-1, [1]))
+
+
+ at skipUnless(c_heapq, 'requires _heapq')
+class TestHeapC(TestHeap):
+    module = c_heapq
 
 
 #==============================================================================
@@ -317,22 +320,21 @@
     'Test multiple tiers of iterators'
     return chain(imap(lambda x:x, R(Ig(G(seqn)))))
 
-class TestErrorHandling(unittest.TestCase):
-    # only for C implementation
-    module = c_heapq
+class TestErrorHandling(TestCase):
+    module = None
 
     def test_non_sequence(self):
         for f in (self.module.heapify, self.module.heappop):
-            self.assertRaises(TypeError, f, 10)
+            self.assertRaises((TypeError, AttributeError), f, 10)
         for f in (self.module.heappush, self.module.heapreplace,
                   self.module.nlargest, self.module.nsmallest):
-            self.assertRaises(TypeError, f, 10, 10)
+            self.assertRaises((TypeError, AttributeError), f, 10, 10)
 
     def test_len_only(self):
         for f in (self.module.heapify, self.module.heappop):
-            self.assertRaises(TypeError, f, LenOnly())
+            self.assertRaises((TypeError, AttributeError), f, LenOnly())
         for f in (self.module.heappush, self.module.heapreplace):
-            self.assertRaises(TypeError, f, LenOnly(), 10)
+            self.assertRaises((TypeError, AttributeError), f, LenOnly(), 10)
         for f in (self.module.nlargest, self.module.nsmallest):
             self.assertRaises(TypeError, f, 2, LenOnly())
 
@@ -349,7 +351,7 @@
         for f in (self.module.heapify, self.module.heappop,
                   self.module.heappush, self.module.heapreplace,
                   self.module.nlargest, self.module.nsmallest):
-            self.assertRaises(TypeError, f, 10)
+            self.assertRaises((TypeError, AttributeError), f, 10)
 
     def test_iterable_args(self):
         for f in (self.module.nlargest, self.module.nsmallest):
@@ -365,13 +367,21 @@
                 self.assertRaises(ZeroDivisionError, f, 2, E(s))
 
 
+class TestErrorHandlingPython(TestErrorHandling):
+    module = py_heapq
+
+
+ at skipUnless(c_heapq, 'requires _heapq')
+class TestErrorHandlingC(TestErrorHandling):
+    module = c_heapq
+
+
 #==============================================================================
 
 
 def test_main(verbose=None):
-    test_classes = [TestHeapPython]
-    if c_heapq is not None:
-        test_classes += [TestHeapC, TestErrorHandling]
+    test_classes = [TestModules, TestHeapPython, TestHeapC,
+                    TestErrorHandlingPython, TestErrorHandlingC]
     test_support.run_unittest(*test_classes)
 
     # verify reference counting
diff --git a/lib-python/modified-2.7/test/test_import.py b/lib-python/modified-2.7/test/test_import.py
--- a/lib-python/modified-2.7/test/test_import.py
+++ b/lib-python/modified-2.7/test/test_import.py
@@ -9,7 +9,8 @@
 from test.test_support import (unlink, TESTFN, unload, run_unittest, rmtree,
                                is_jython, check_warnings, EnvironmentVarGuard,
                                impl_detail, check_impl_detail)
-
+import textwrap
+from test import script_helper
 
 def remove_files(name):
     for f in (name + os.extsep + "py",
@@ -64,7 +65,6 @@
             except ImportError, err:
                 self.fail("import from %s failed: %s" % (ext, err))
             else:
-                # XXX importing .pyw is missing on Windows
                 self.assertEqual(mod.a, a,
                     "module loaded (%s) but contents invalid" % mod)
                 self.assertEqual(mod.b, b,
@@ -259,6 +259,17 @@
         self.assertEqual("Import by filename is not supported.",
                          c.exception.args[0])
 
+    def test_import_in_del_does_not_crash(self):
+        # Issue 4236
+        testfn = script_helper.make_script('', TESTFN, textwrap.dedent("""\
+            import sys
+            class C:
+               def __del__(self):
+                  import imp
+            sys.argv.insert(0, C())
+            """))
+        script_helper.assert_python_ok(testfn)
+
 
 class PycRewritingTests(unittest.TestCase):
     # Test that the `co_filename` attribute on code objects always points
diff --git a/lib-python/modified-2.7/test/test_inspect.py b/lib-python/modified-2.7/test/test_inspect.py
--- a/lib-python/modified-2.7/test/test_inspect.py
+++ b/lib-python/modified-2.7/test/test_inspect.py
@@ -634,6 +634,16 @@
         self.assertEqualCallArgs(f, '2, c=4, **{u"b":3}')
         self.assertEqualCallArgs(f, 'b=2, **{u"a":3, u"c":4}')
 
+    def test_varkw_only(self):
+        # issue11256:
+        f = self.makeCallable('**c')
+        self.assertEqualCallArgs(f, '')
+        self.assertEqualCallArgs(f, 'a=1')
+        self.assertEqualCallArgs(f, 'a=1, b=2')
+        self.assertEqualCallArgs(f, 'c=3, **{"a": 1, "b": 2}')
+        self.assertEqualCallArgs(f, '**UserDict(a=1, b=2)')
+        self.assertEqualCallArgs(f, 'c=3, **UserDict(a=1, b=2)')
+
     def test_tupleargs(self):
         f = self.makeCallable('(b,c), (d,(e,f))=(0,[1,2])')
         self.assertEqualCallArgs(f, '(2,3)')
@@ -695,6 +705,10 @@
         self.assertEqualException(f, '1')
         self.assertEqualException(f, '[1]')
         self.assertEqualException(f, '(1,2,3)')
+        # issue11256:
+        f3 = self.makeCallable('**c')
+        self.assertEqualException(f3, '1, 2')
+        self.assertEqualException(f3, '1, 2, a=1, b=2')
 
 class TestGetcallargsMethods(TestGetcallargsFunctions):
 
diff --git a/lib-python/modified-2.7/test/test_io.py b/lib-python/modified-2.7/test/test_io.py
--- a/lib-python/modified-2.7/test/test_io.py
+++ b/lib-python/modified-2.7/test/test_io.py
@@ -701,6 +701,13 @@
         b.close()
         self.assertRaises(ValueError, b.flush)
 
+    def test_readonly_attributes(self):
+        raw = self.MockRawIO()
+        buf = self.tp(raw)
+        x = self.MockRawIO()
+        with self.assertRaises((AttributeError, TypeError)):
+            buf.raw = x
+
 
 class BufferedReaderTest(unittest.TestCase, CommonBufferedTests):
     read_mode = "rb"
@@ -791,14 +798,17 @@
         # Inject some None's in there to simulate EWOULDBLOCK
         rawio = self.MockRawIO((b"abc", b"d", None, b"efg", None, None, None))
         bufio = self.tp(rawio)
-
         self.assertEqual(b"abcd", bufio.read(6))
         self.assertEqual(b"e", bufio.read(1))
         self.assertEqual(b"fg", bufio.read())
         self.assertEqual(b"", bufio.peek(1))
-        self.assertTrue(None is bufio.read())
+        self.assertIsNone(bufio.read())
         self.assertEqual(b"", bufio.read())
 
+        rawio = self.MockRawIO((b"a", None, None))
+        self.assertEqual(b"a", rawio.readall())
+        self.assertIsNone(rawio.readall())
+
     def test_read_past_eof(self):
         rawio = self.MockRawIO((b"abc", b"d", b"efg"))
         bufio = self.tp(rawio)
@@ -1455,6 +1465,32 @@
             self.assertEqual(s,
                 b"A" + b"B" * overwrite_size + b"A" * (9 - overwrite_size))
 
+    def test_write_rewind_write(self):
+        # Various combinations of reading / writing / seeking backwards / writing again
+        def mutate(bufio, pos1, pos2):
+            assert pos2 >= pos1
+            # Fill the buffer
+            bufio.seek(pos1)
+            bufio.read(pos2 - pos1)
+            bufio.write(b'\x02')
+            # This writes earlier than the previous write, but still inside
+            # the buffer.
+            bufio.seek(pos1)
+            bufio.write(b'\x01')
+
+        b = b"\x80\x81\x82\x83\x84"
+        for i in range(0, len(b)):
+            for j in range(i, len(b)):
+                raw = self.BytesIO(b)
+                bufio = self.tp(raw, 100)
+                mutate(bufio, i, j)
+                bufio.flush()
+                expected = bytearray(b)
+                expected[j] = 2
+                expected[i] = 1
+                self.assertEqual(raw.getvalue(), expected,
+                                 "failed result for i=%d, j=%d" % (i, j))
+
     def test_truncate_after_read_or_write(self):
         raw = self.BytesIO(b"A" * 10)
         bufio = self.tp(raw, 100)
@@ -2211,6 +2247,12 @@
         txt.close()
         self.assertRaises(ValueError, txt.flush)
 
+    def test_readonly_attributes(self):
+        txt = self.TextIOWrapper(self.BytesIO(self.testdata), encoding="ascii")
+        buf = self.BytesIO(self.testdata)
+        with self.assertRaises((AttributeError, TypeError)):
+            txt.buffer = buf
+
 class CTextIOWrapperTest(TextIOWrapperTest):
 
     def test_initialization(self):
@@ -2429,6 +2471,8 @@
             self.assertRaises(ValueError, f.read)
             if hasattr(f, "read1"):
                 self.assertRaises(ValueError, f.read1, 1024)
+            if hasattr(f, "readall"):
+                self.assertRaises(ValueError, f.readall)
             if hasattr(f, "readinto"):
                 self.assertRaises(ValueError, f.readinto, bytearray(1024))
             self.assertRaises(ValueError, f.readline)
@@ -2515,7 +2559,8 @@
     @unittest.skipUnless(threading, 'Threading required for this test.')
     def check_interrupted_write(self, item, bytes, **fdopen_kwargs):
         """Check that a partial write, when it gets interrupted, properly
-        invokes the signal handler."""
+        invokes the signal handler, and bubbles up the exception raised
+        in the latter."""
 
         # XXX This test has three flaws that appear when objects are
         # XXX not reference counted.
@@ -2586,12 +2631,142 @@
     def test_interrupted_write_text(self):
         self.check_interrupted_write("xy", b"xy", mode="w", encoding="ascii")
 
+    def check_reentrant_write(self, data, **fdopen_kwargs):
+        def on_alarm(*args):
+            # Will be called reentrantly from the same thread
+            wio.write(data)
+            1/0
+        signal.signal(signal.SIGALRM, on_alarm)
+        r, w = os.pipe()
+        wio = self.io.open(w, **fdopen_kwargs)
+        try:
+            signal.alarm(1)
+            # Either the reentrant call to wio.write() fails with RuntimeError,
+            # or the signal handler raises ZeroDivisionError.
+            with self.assertRaises((ZeroDivisionError, RuntimeError)) as cm:
+                while 1:
+                    for i in range(100):
+                        wio.write(data)
+                        wio.flush()
+                    # Make sure the buffer doesn't fill up and block further writes
+                    os.read(r, len(data) * 100)
+            exc = cm.exception
+            if isinstance(exc, RuntimeError):
+                self.assertTrue(str(exc).startswith("reentrant call"), str(exc))
+        finally:
+            wio.close()
+            os.close(r)
+
+    def test_reentrant_write_buffered(self):
+        self.check_reentrant_write(b"xy", mode="wb")
+
+    def test_reentrant_write_text(self):
+        self.check_reentrant_write("xy", mode="w", encoding="ascii")
+
+    def check_interrupted_read_retry(self, decode, **fdopen_kwargs):
+        """Check that a buffered read, when it gets interrupted (either
+        returning a partial result or EINTR), properly invokes the signal
+        handler and retries if the latter returned successfully."""
+        r, w = os.pipe()
+        fdopen_kwargs["closefd"] = False
+        def alarm_handler(sig, frame):
+            os.write(w, b"bar")
+        signal.signal(signal.SIGALRM, alarm_handler)
+        try:
+            rio = self.io.open(r, **fdopen_kwargs)
+            os.write(w, b"foo")
+            signal.alarm(1)
+            # Expected behaviour:
+            # - first raw read() returns partial b"foo"
+            # - second raw read() returns EINTR
+            # - third raw read() returns b"bar"
+            self.assertEqual(decode(rio.read(6)), "foobar")
+        finally:
+            rio.close()
+            os.close(w)
+            os.close(r)
+
+    def test_interrupterd_read_retry_buffered(self):
+        self.check_interrupted_read_retry(lambda x: x.decode('latin1'),
+                                          mode="rb")
+
+    def test_interrupterd_read_retry_text(self):
+        self.check_interrupted_read_retry(lambda x: x,
+                                          mode="r")
+
+    @unittest.skipUnless(threading, 'Threading required for this test.')
+    def check_interrupted_write_retry(self, item, **fdopen_kwargs):
+        """Check that a buffered write, when it gets interrupted (either
+        returning a partial result or EINTR), properly invokes the signal
+        handler and retries if the latter returned successfully."""
+        select = support.import_module("select")
+        # A quantity that exceeds the buffer size of an anonymous pipe's
+        # write end.
+        N = 1024 * 1024
+        r, w = os.pipe()
+        fdopen_kwargs["closefd"] = False
+        # We need a separate thread to read from the pipe and allow the
+        # write() to finish.  This thread is started after the SIGALRM is
+        # received (forcing a first EINTR in write()).
+        read_results = []
+        write_finished = False
+        def _read():
+            while not write_finished:
+                while r in select.select([r], [], [], 1.0)[0]:
+                    s = os.read(r, 1024)
+                    read_results.append(s)
+        t = threading.Thread(target=_read)
+        t.daemon = True
+        def alarm1(sig, frame):
+            signal.signal(signal.SIGALRM, alarm2)
+            signal.alarm(1)
+        def alarm2(sig, frame):
+            t.start()
+        signal.signal(signal.SIGALRM, alarm1)
+        try:
+            wio = self.io.open(w, **fdopen_kwargs)
+            signal.alarm(1)
+            # Expected behaviour:
+            # - first raw write() is partial (because of the limited pipe buffer
+            #   and the first alarm)
+            # - second raw write() returns EINTR (because of the second alarm)
+            # - subsequent write()s are successful (either partial or complete)
+            self.assertEqual(N, wio.write(item * N))
+            wio.flush()
+            write_finished = True
+            t.join()
+            self.assertEqual(N, sum(len(x) for x in read_results))
+        finally:
+            write_finished = True
+            os.close(w)
+            os.close(r)
+            # This is deliberate. If we didn't close the file descriptor
+            # before closing wio, wio would try to flush its internal
+            # buffer, and could block (in case of failure).
+            try:
+                wio.close()
+            except IOError as e:
+                if e.errno != errno.EBADF:
+                    raise
+
+    def test_interrupterd_write_retry_buffered(self):
+        self.check_interrupted_write_retry(b"x", mode="wb")
+
+    def test_interrupterd_write_retry_text(self):
+        self.check_interrupted_write_retry("x", mode="w", encoding="latin1")
+
+
 class CSignalsTest(SignalsTest):
     io = io
 
 class PySignalsTest(SignalsTest):
     io = pyio
 
+    # Handling reentrancy issues would slow down _pyio even more, so the
+    # tests are disabled.
+    test_reentrant_write_buffered = None
+    test_reentrant_write_text = None
+
 
 def test_main():
     tests = (CIOTest, PyIOTest,
diff --git a/lib-python/modified-2.7/test/test_itertools.py b/lib-python/modified-2.7/test/test_itertools.py
--- a/lib-python/modified-2.7/test/test_itertools.py
+++ b/lib-python/modified-2.7/test/test_itertools.py
@@ -785,13 +785,18 @@
         self.assertRaises(ValueError, islice, xrange(10), 1, -5, -1)
         self.assertRaises(ValueError, islice, xrange(10), 1, 10, -1)
         self.assertRaises(ValueError, islice, xrange(10), 1, 10, 0)
-        self.assertRaises((ValueError,TypeError), islice, xrange(10), 'a')
-        self.assertRaises((ValueError,TypeError), islice, xrange(10), 'a', 1)
-        self.assertRaises((ValueError,TypeError), islice, xrange(10), 1, 'a')
-        self.assertRaises((ValueError,TypeError), islice, xrange(10), 'a', 1, 1)
-        self.assertRaises((ValueError,TypeError), islice, xrange(10), 1, 'a', 1)
+        self.assertRaises((ValueError, TypeError), islice, xrange(10), 'a')
+        self.assertRaises((ValueError, TypeError), islice, xrange(10), 'a', 1)
+        self.assertRaises((ValueError, TypeError), islice, xrange(10), 1, 'a')
+        self.assertRaises((ValueError, TypeError), islice, xrange(10), 'a', 1, 1)
+        self.assertRaises((ValueError, TypeError), islice, xrange(10), 1, 'a', 1)
         self.assertEqual(len(list(islice(count(), 1, 10, maxsize))), 1)
 
+        # Issue #10323:  Less islice in a predictable state
+        c = count()
+        self.assertEqual(list(islice(c, 1, 3, 50)), [1])
+        self.assertEqual(next(c), 3)
+
     def test_takewhile(self):
         data = [1, 3, 5, 20, 2, 4, 6, 8]
         underten = lambda x: x<10
@@ -1494,7 +1499,7 @@
 ...     return chain(iterable, repeat(None))
 
 >>> def ncycles(iterable, n):
-...     "Returns the seqeuence elements n times"
+...     "Returns the sequence elements n times"
 ...     return chain(*repeat(iterable, n))
 
 >>> def dotproduct(vec1, vec2):
diff --git a/lib-python/modified-2.7/test/test_linecache.py b/lib-python/modified-2.7/test/test_linecache.py
--- a/lib-python/modified-2.7/test/test_linecache.py
+++ b/lib-python/modified-2.7/test/test_linecache.py
@@ -9,7 +9,7 @@
 FILENAME = linecache.__file__
 INVALID_NAME = '!@$)(!@#_1'
 EMPTY = ''
-TESTS = 'cjkencodings_test inspect_fodder inspect_fodder2 mapping_tests'
+TESTS = 'inspect_fodder inspect_fodder2 mapping_tests'
 TESTS = TESTS.split()
 TEST_PATH = os.path.dirname(support.__file__)
 MODULES = "linecache abc".split()
diff --git a/lib-python/modified-2.7/test/test_marshal.py b/lib-python/modified-2.7/test/test_marshal.py
--- a/lib-python/modified-2.7/test/test_marshal.py
+++ b/lib-python/modified-2.7/test/test_marshal.py
@@ -196,7 +196,7 @@
         #   >>> type(loads(dumps(Int())))
         #   
         for typ in (int, long, float, complex, tuple, list, dict, set, frozenset):
-            # Note: str and unicode sublclasses are not tested because they get handled
+            # Note: str and unicode subclasses are not tested because they get handled
             # by marshal's routines for objects supporting the buffer API.
             subtyp = type('subtyp', (typ,), {})
             self.assertRaises(ValueError, marshal.dumps, subtyp())
diff --git a/lib-python/modified-2.7/test/test_memoryio.py b/lib-python/modified-2.7/test/test_memoryio.py
--- a/lib-python/modified-2.7/test/test_memoryio.py
+++ b/lib-python/modified-2.7/test/test_memoryio.py
@@ -372,7 +372,7 @@
 
         # Pickle expects the class to be on the module level. Here we use a
         # little hack to allow the PickleTestMemIO class to derive from
-        # self.ioclass without having to define all combinations explictly on
+        # self.ioclass without having to define all combinations explicitly on
         # the module-level.
         import __main__
         PickleTestMemIO.__module__ = '__main__'
diff --git a/lib-python/modified-2.7/test/test_memoryview.py b/lib-python/modified-2.7/test/test_memoryview.py
--- a/lib-python/modified-2.7/test/test_memoryview.py
+++ b/lib-python/modified-2.7/test/test_memoryview.py
@@ -9,6 +9,7 @@
 import weakref
 import array
 from test import test_support
+import io
 
 
 class AbstractMemoryTests:
@@ -236,6 +237,16 @@
             gc.collect()
             self.assertTrue(wr() is None, wr())
 
+    def test_writable_readonly(self):
+        # Issue #10451: memoryview incorrectly exposes a readonly
+        # buffer as writable causing a segfault if using mmap
+        tp = self.ro_type
+        if tp is None:
+            return
+        b = tp(self._source)
+        m = self._view(b)
+        i = io.BytesIO(b'ZZZZ')
+        self.assertRaises(TypeError, i.readinto, m)
 
 # Variations on source objects for the buffer: bytes-like objects, then arrays
 # with itemsize > 1.
diff --git a/lib-python/modified-2.7/test/test_mmap.py b/lib-python/modified-2.7/test/test_mmap.py
--- a/lib-python/modified-2.7/test/test_mmap.py
+++ b/lib-python/modified-2.7/test/test_mmap.py
@@ -1,6 +1,7 @@
-from test.test_support import TESTFN, run_unittest, import_module
+from test.test_support import (TESTFN, run_unittest, import_module, unlink,
+                               requires, _2G, _4G)
 import unittest
-import os, re, itertools, socket
+import os, re, itertools, socket, sys
 
 mmap = import_module('mmap')
 
@@ -244,6 +245,14 @@
                               prot=mmap.PROT_READ, access=mmap.ACCESS_WRITE)
             f.close()
 
+            # Try writing with PROT_EXEC and without PROT_WRITE
+            prot = mmap.PROT_READ | getattr(mmap, 'PROT_EXEC', 0)
+            with open(TESTFN, "r+b") as f:
+                m = mmap.mmap(f.fileno(), mapsize, prot=prot)
+                self.assertRaises(TypeError, m.write, b"abcdef")
+                self.assertRaises(TypeError, m.write_byte, 0)
+                m.close()
+
     def test_bad_file_desc(self):
         # Try opening a bad file descriptor...
         self.assertRaises(mmap.error, mmap.mmap, -2, 4096)
@@ -333,6 +342,36 @@
             mf.close()
             f.close()
 
+    def test_length_0_offset(self):
+        # Issue #10916: test mapping of remainder of file by passing 0 for
+        # map length with an offset doesn't cause a segfault.
+        if not hasattr(os, "stat"):
+            self.skipTest("needs os.stat")
+        # NOTE: allocation granularity is currently 65536 under Win64,
+        # and therefore the minimum offset alignment.
+        with open(TESTFN, "wb") as f:
+            f.write((65536 * 2) * b'm') # Arbitrary character
+
+        with open(TESTFN, "rb") as f:
+            mf = mmap.mmap(f.fileno(), 0, offset=65536, access=mmap.ACCESS_READ)
+            try:
+                self.assertRaises(IndexError, mf.__getitem__, 80000)
+            finally:
+                mf.close()
+
+    def test_length_0_large_offset(self):
+        # Issue #10959: test mapping of a file by passing 0 for
+        # map length with a large offset doesn't cause a segfault.
+        if not hasattr(os, "stat"):
+            self.skipTest("needs os.stat")
+
+        with open(TESTFN, "wb") as f:
+            f.write(115699 * b'm') # Arbitrary character
+
+        with open(TESTFN, "w+b") as f:
+            self.assertRaises(ValueError, mmap.mmap, f.fileno(), 0,
+                              offset=2147418112)
+
     def test_move(self):
         # make move works everywhere (64-bit format problem earlier)
         f = open(TESTFN, 'w+')
@@ -562,7 +601,7 @@
             m2.close()
             m1.close()
 
-            # Test differnt tag
+            # Test different tag
             m1 = mmap.mmap(-1, len(data1), tagname="foo")
             m1[:] = data1
             m2 = mmap.mmap(-1, len(data2), tagname="boo")
@@ -608,8 +647,69 @@
             finally:
                 s.close()
 
+
+class LargeMmapTests(unittest.TestCase):
+
+    def setUp(self):
+        unlink(TESTFN)
+
+    def tearDown(self):
+        unlink(TESTFN)
+
+    def _make_test_file(self, num_zeroes, tail):
+        if sys.platform[:3] == 'win' or sys.platform == 'darwin':
+            requires('largefile',
+                'test requires %s bytes and a long time to run' % str(0x180000000))
+        f = open(TESTFN, 'w+b')
+        try:
+            f.seek(num_zeroes)
+            f.write(tail)
+            f.flush()
+        except (IOError, OverflowError):
+            f.close()
+            raise unittest.SkipTest("filesystem does not have largefile support")
+        return f
+
+    def test_large_offset(self):
+        with self._make_test_file(0x14FFFFFFF, b" ") as f:
+            m = mmap.mmap(f.fileno(), 0, offset=0x140000000, access=mmap.ACCESS_READ)
+            try:
+                self.assertEqual(m[0xFFFFFFF], b" ")
+            finally:
+                m.close()
+
+    def test_large_filesize(self):
+        with self._make_test_file(0x17FFFFFFF, b" ") as f:
+            m = mmap.mmap(f.fileno(), 0x10000, access=mmap.ACCESS_READ)
+            try:
+                self.assertEqual(m.size(), 0x180000000)
+            finally:
+                m.close()
+
+    # Issue 11277: mmap() with large (~4GB) sparse files crashes on OS X.
+
+    def _test_around_boundary(self, boundary):
+        tail = b'  DEARdear  '
+        start = boundary - len(tail) // 2
+        end = start + len(tail)
+        with self._make_test_file(start, tail) as f:
+            m = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ)
+            try:
+                self.assertEqual(m[start:end], tail)
+            finally:
+                m.close()
+
+    @unittest.skipUnless(sys.maxsize > _4G, "test cannot run on 32-bit systems")
+    def test_around_2GB(self):
+        self._test_around_boundary(_2G)
+
+    @unittest.skipUnless(sys.maxsize > _4G, "test cannot run on 32-bit systems")
+    def test_around_4GB(self):
+        self._test_around_boundary(_4G)
+
+
 def test_main():
-    run_unittest(MmapTests)
+    run_unittest(MmapTests, LargeMmapTests)
 
 if __name__ == '__main__':
     test_main()
diff --git a/lib-python/modified-2.7/test/test_multibytecodec.py b/lib-python/modified-2.7/test/test_multibytecodec.py
--- a/lib-python/modified-2.7/test/test_multibytecodec.py
+++ b/lib-python/modified-2.7/test/test_multibytecodec.py
@@ -238,6 +238,36 @@
             # Any ISO 2022 codec will cause the segfault
             myunichr(x).encode('iso_2022_jp', 'ignore')
 
+class TestStateful(unittest.TestCase):
+    text = u'\u4E16\u4E16'
+    encoding = 'iso-2022-jp'
+    expected = b'\x1b$B@$@$'
+    expected_reset = b'\x1b$B@$@$\x1b(B'
+
+    def test_encode(self):
+        self.assertEqual(self.text.encode(self.encoding), self.expected_reset)
+
+    def test_incrementalencoder(self):
+        encoder = codecs.getincrementalencoder(self.encoding)()
+        output = b''.join(
+            encoder.encode(char)
+            for char in self.text)
+        self.assertEqual(output, self.expected)
+
+    def test_incrementalencoder_final(self):
+        encoder = codecs.getincrementalencoder(self.encoding)()
+        last_index = len(self.text) - 1
+        output = b''.join(
+            encoder.encode(char, index == last_index)
+            for index, char in enumerate(self.text))
+        self.assertEqual(output, self.expected_reset)
+
+class TestHZStateful(TestStateful):
+    text = u'\u804a\u804a'
+    encoding = 'hz'
+    expected = b'~{ADAD'
+    expected_reset = b'~{ADAD~}'
+
 def test_main():
     test_support.run_unittest(__name__)
 
diff --git a/lib-python/modified-2.7/test/test_multibytecodec_support.py b/lib-python/modified-2.7/test/test_multibytecodec_support.py
--- a/lib-python/modified-2.7/test/test_multibytecodec_support.py
+++ b/lib-python/modified-2.7/test/test_multibytecodec_support.py
@@ -4,8 +4,11 @@
 #   Common Unittest Routines for CJK codecs
 #
 
-import sys, codecs
-import unittest, re
+import codecs
+import os
+import re
+import sys
+import unittest
 from httplib import HTTPException
 from test import test_support
 from StringIO import StringIO
@@ -326,6 +329,10 @@
                 self.fail('Decoding failed while testing %s -> %s: %s' % (
                             repr(csetch), repr(unich), exc.reason))
 
-def load_teststring(encoding):
-    from test import cjkencodings_test
-    return cjkencodings_test.teststring[encoding]
+def load_teststring(name):
+    dir = os.path.join(os.path.dirname(__file__), 'cjkencodings')
+    with open(os.path.join(dir, name + '.txt'), 'rb') as f:
+        encoded = f.read()
+    with open(os.path.join(dir, name + '-utf8.txt'), 'rb') as f:
+        utf8 = f.read()
+    return encoded, utf8
diff --git a/lib-python/modified-2.7/test/test_multiprocessing.py b/lib-python/modified-2.7/test/test_multiprocessing.py
--- a/lib-python/modified-2.7/test/test_multiprocessing.py
+++ b/lib-python/modified-2.7/test/test_multiprocessing.py
@@ -18,7 +18,7 @@
 from test import test_support
 from StringIO import StringIO
 _multiprocessing = test_support.import_module('_multiprocessing')
-# import threading after _multiprocessing to raise a more revelant error
+# import threading after _multiprocessing to raise a more relevant error
 # message: "No module named _multiprocessing". _multiprocessing is not compiled
 # without thread support.
 import threading
@@ -780,7 +780,7 @@
         event = self.Event()
         wait = TimingWrapper(event.wait)
 
-        # Removed temporaily, due to API shear, this does not
+        # Removed temporarily, due to API shear, this does not
         # work with threading._Event objects. is_set == isSet
         self.assertEqual(event.is_set(), False)
 
@@ -914,10 +914,32 @@
         self.assertEqual(list(arr[:]), seq)
 
     @unittest.skipIf(c_int is None, "requires _ctypes")
+    def test_array_from_size(self):
+        size = 10
+        # Test for zeroing (see issue #11675).
+        # The repetition below strengthens the test by increasing the chances
+        # of previously allocated non-zero memory being used for the new array
+        # on the 2nd and 3rd loops.
+        for _ in range(3):
+            arr = self.Array('i', size)
+            self.assertEqual(len(arr), size)
+            self.assertEqual(list(arr), [0] * size)
+            arr[:] = range(10)
+            self.assertEqual(list(arr), range(10))
+            del arr
+
+    @unittest.skipIf(c_int is None, "requires _ctypes")
     def test_rawarray(self):
         self.test_array(raw=True)
 
     @unittest.skipIf(c_int is None, "requires _ctypes")
+    def test_array_accepts_long(self):
+        arr = self.Array('i', 10L)
+        self.assertEqual(len(arr), 10)
+        raw_arr = self.RawArray('i', 10L)
+        self.assertEqual(len(raw_arr), 10)
+
+    @unittest.skipIf(c_int is None, "requires _ctypes")
     def test_getobj_getlock_obj(self):
         arr1 = self.Array('i', range(10))
         lock1 = arr1.get_lock()
@@ -1104,7 +1126,8 @@
         # Refill the pool
         p._repopulate_pool()
         # Wait until all workers are alive
-        countdown = 5
+        # (countdown * DELTA = 5 seconds max startup process time)
+        countdown = 50
         while countdown and not all(w.is_alive() for w in p._pool):
             countdown -= 1
             time.sleep(DELTA)
@@ -1711,7 +1734,7 @@
 
         util.Finalize(None, conn.send, args=('STOP',), exitpriority=-100)
 
-        # call mutliprocessing's cleanup function then exit process without
+        # call multiprocessing's cleanup function then exit process without
         # garbage collecting locals
         util._exit_function()
         conn.close()
diff --git a/lib-python/modified-2.7/test/test_peepholer.py b/lib-python/modified-2.7/test/test_peepholer.py
--- a/lib-python/modified-2.7/test/test_peepholer.py
+++ b/lib-python/modified-2.7/test/test_peepholer.py
@@ -143,6 +143,24 @@
         asm = dis_single('a="x"*1000')
         self.assertIn('(1000)', asm)
 
+    def test_binary_subscr_on_unicode(self):
+        # valid code get optimized
+        asm = dis_single('u"foo"[0]')
+        self.assertIn("(u'f')", asm)
+        self.assertNotIn('BINARY_SUBSCR', asm)
+        asm = dis_single('u"\u0061\uffff"[1]')
+        self.assertIn("(u'\\uffff')", asm)
+        self.assertNotIn('BINARY_SUBSCR', asm)
+
+        # invalid code doesn't get optimized
+        # out of range
+        asm = dis_single('u"fuu"[10]')
+        self.assertIn('BINARY_SUBSCR', asm)
+        # non-BMP char (see #5057)
+        asm = dis_single('u"\U00012345"[0]')
+        self.assertIn('BINARY_SUBSCR', asm)
+
+
     def test_folding_of_unaryops_on_constants(self):
         for line, elem in (
             ('`1`', "('1')"),                       # unary convert
diff --git a/lib-python/modified-2.7/test/test_pydoc.py b/lib-python/modified-2.7/test/test_pydoc.py
--- a/lib-python/modified-2.7/test/test_pydoc.py
+++ b/lib-python/modified-2.7/test/test_pydoc.py
@@ -6,12 +6,14 @@
 import re
 import pydoc
 import inspect
+import keyword
 import unittest
 import xml.etree
 import test.test_support
 from contextlib import contextmanager
+from collections import namedtuple
 from test.test_support import (
-    TESTFN, forget, rmtree, EnvironmentVarGuard, reap_children)
+    TESTFN, forget, rmtree, EnvironmentVarGuard, reap_children, captured_stdout)
 
 from test import pydoc_mod
 
@@ -340,10 +342,26 @@
         expected = 'C in module %s object' % __name__
         self.assertIn(expected, pydoc.render_doc(c))
 
+    def test_namedtuple_public_underscore(self):
+        NT = namedtuple('NT', ['abc', 'def'], rename=True)
+        with captured_stdout() as help_io:
+            help(NT)
+        helptext = help_io.getvalue()
+        self.assertIn('_1', helptext)
+        self.assertIn('_replace', helptext)
+        self.assertIn('_asdict', helptext)
+
+
+class TestHelper(unittest.TestCase):
+    def test_keywords(self):
+        self.assertEqual(sorted(pydoc.Helper.keywords),
+                         sorted(keyword.kwlist))
+
 
 def test_main():
     test.test_support.run_unittest(PyDocDocTest,
-                                   TestDescriptions)
+                                   TestDescriptions,
+                                   TestHelper)
 
 if __name__ == "__main__":
     test_main()
diff --git a/lib-python/modified-2.7/test/test_pyexpat.py b/lib-python/modified-2.7/test/test_pyexpat.py
--- a/lib-python/modified-2.7/test/test_pyexpat.py
+++ b/lib-python/modified-2.7/test/test_pyexpat.py
@@ -6,6 +6,7 @@
 
 from xml.parsers import expat
 
+from test import test_support
 from test.test_support import sortdict, run_unittest
 
 
@@ -217,6 +218,16 @@
         self.assertEqual(op[15], "External entity ref: (None, u'entity.file', None)")
         self.assertEqual(op[16], "End element: u'root'")
 
+        # Issue 4877: expat.ParseFile causes segfault on a closed file.
+        fp = open(test_support.TESTFN, 'wb')
+        try:
+            fp.close()
+            parser = expat.ParserCreate()
+            with self.assertRaises(ValueError):
+                parser.ParseFile(fp)
+        finally:
+            test_support.unlink(test_support.TESTFN)
+
 
 class NamespaceSeparatorTest(unittest.TestCase):
     def test_legal(self):
diff --git a/lib-python/modified-2.7/test/test_set.py b/lib-python/modified-2.7/test/test_set.py
--- a/lib-python/modified-2.7/test/test_set.py
+++ b/lib-python/modified-2.7/test/test_set.py
@@ -1592,6 +1592,39 @@
                 self.assertRaises(TypeError, getattr(set('january'), methname), N(data))
                 self.assertRaises(ZeroDivisionError, getattr(set('january'), methname), E(data))
 
+class bad_eq:
+    def __eq__(self, other):
+        if be_bad:
+            set2.clear()
+            raise ZeroDivisionError
+        return self is other
+    def __hash__(self):
+        return 0
+
+class bad_dict_clear:
+    def __eq__(self, other):
+        if be_bad:
+            dict2.clear()
+        return self is other
+    def __hash__(self):
+        return 0
+
+class TestWeirdBugs(unittest.TestCase):
+    def test_8420_set_merge(self):
+        # This used to segfault
+        global be_bad, set2, dict2
+        be_bad = False
+        set1 = {bad_eq()}
+        set2 = {bad_eq() for i in range(75)}
+        be_bad = True
+        self.assertRaises(ZeroDivisionError, set1.update, set2)
+
+        be_bad = False
+        set1 = {bad_dict_clear()}
+        dict2 = {bad_dict_clear(): None}
+        be_bad = True
+        set1.symmetric_difference_update(dict2)
+
 # Application tests (based on David Eppstein's graph recipes ====================================
 
 def powerset(U):
@@ -1733,6 +1766,7 @@
         TestIdentities,
         TestVariousIteratorArgs,
         TestGraphs,
+        TestWeirdBugs,
         )
 
     test_support.run_unittest(*test_classes)
diff --git a/lib-python/modified-2.7/test/test_site.py b/lib-python/modified-2.7/test/test_site.py
--- a/lib-python/modified-2.7/test/test_site.py
+++ b/lib-python/modified-2.7/test/test_site.py
@@ -6,9 +6,11 @@
 """
 import unittest
 from test.test_support import run_unittest, TESTFN, EnvironmentVarGuard
+from test.test_support import captured_output
 import __builtin__
 import os
 import sys
+import re
 import encodings
 import subprocess
 import sysconfig
@@ -94,6 +96,58 @@
         finally:
             pth_file.cleanup()
 
+    def make_pth(self, contents, pth_dir='.', pth_name=TESTFN):
+        # Create a .pth file and return its (abspath, basename).
+        pth_dir = os.path.abspath(pth_dir)
+        pth_basename = pth_name + '.pth'
+        pth_fn = os.path.join(pth_dir, pth_basename)
+        pth_file = open(pth_fn, 'w')
+        self.addCleanup(lambda: os.remove(pth_fn))
+        pth_file.write(contents)
+        pth_file.close()
+        return pth_dir, pth_basename
+
+    def test_addpackage_import_bad_syntax(self):
+        # Issue 10642
+        pth_dir, pth_fn = self.make_pth("import bad)syntax\n")
+        with captured_output("stderr") as err_out:
+            site.addpackage(pth_dir, pth_fn, set())
+        self.assertRegexpMatches(err_out.getvalue(), "line 1")
+        self.assertRegexpMatches(err_out.getvalue(),
+            re.escape(os.path.join(pth_dir, pth_fn)))
+        # XXX: the previous two should be independent checks so that the
+        # order doesn't matter.  The next three could be a single check
+        # but my regex foo isn't good enough to write it.
+        self.assertRegexpMatches(err_out.getvalue(), 'Traceback')
+        self.assertRegexpMatches(err_out.getvalue(), r'import bad\)syntax')
+        self.assertRegexpMatches(err_out.getvalue(), 'SyntaxError')
+
+    def test_addpackage_import_bad_exec(self):
+        # Issue 10642
+        pth_dir, pth_fn = self.make_pth("randompath\nimport nosuchmodule\n")
+        with captured_output("stderr") as err_out:
+            site.addpackage(pth_dir, pth_fn, set())
+        self.assertRegexpMatches(err_out.getvalue(), "line 2")
+        self.assertRegexpMatches(err_out.getvalue(),
+            re.escape(os.path.join(pth_dir, pth_fn)))
+        # XXX: ditto previous XXX comment.
+        self.assertRegexpMatches(err_out.getvalue(), 'Traceback')
+        self.assertRegexpMatches(err_out.getvalue(), 'ImportError')
+
+    @unittest.skipIf(sys.platform == "win32", "Windows does not raise an "
+                      "error for file paths containing null characters")
+    def test_addpackage_import_bad_pth_file(self):
+        # Issue 5258
+        pth_dir, pth_fn = self.make_pth("abc\x00def\n")
+        with captured_output("stderr") as err_out:
+            site.addpackage(pth_dir, pth_fn, set())
+        self.assertRegexpMatches(err_out.getvalue(), "line 1")
+        self.assertRegexpMatches(err_out.getvalue(),
+            re.escape(os.path.join(pth_dir, pth_fn)))
+        # XXX: ditto previous XXX comment.
+        self.assertRegexpMatches(err_out.getvalue(), 'Traceback')
+        self.assertRegexpMatches(err_out.getvalue(), 'TypeError')
+
     def test_addsitedir(self):
         # Same tests for test_addpackage since addsitedir() essentially just
         # calls addpackage() for every .pth file in the directory
@@ -111,13 +165,17 @@
         usersite = site.USER_SITE
         self.assertIn(usersite, sys.path)
 
+        env = os.environ.copy()
         rc = subprocess.call([sys.executable, '-c',
-            'import sys; sys.exit(%r in sys.path)' % usersite])
+            'import sys; sys.exit(%r in sys.path)' % usersite],
+            env=env)
         self.assertEqual(rc, 1, "%r is not in sys.path (sys.exit returned %r)"
                 % (usersite, rc))
 
+        env = os.environ.copy()
         rc = subprocess.call([sys.executable, '-s', '-c',
-            'import sys; sys.exit(%r in sys.path)' % usersite])
+            'import sys; sys.exit(%r in sys.path)' % usersite],
+            env=env)
         self.assertEqual(rc, 0)
 
         env = os.environ.copy()
diff --git a/lib-python/modified-2.7/test/test_socket.py b/lib-python/modified-2.7/test/test_socket.py
--- a/lib-python/modified-2.7/test/test_socket.py
+++ b/lib-python/modified-2.7/test/test_socket.py
@@ -275,6 +275,45 @@
         self.assertRaises(socket.error, raise_gaierror,
                               "Error raising socket exception.")
 
+    def testSendtoErrors(self):
+        # Testing that sendto doens't masks failures. See #10169.
+        s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+        self.addCleanup(s.close)
+        s.bind(('', 0))
+        sockname = s.getsockname()
+        # 2 args
+        with self.assertRaises(UnicodeEncodeError):
+            s.sendto(u'\u2620', sockname)
+        with self.assertRaises(TypeError) as cm:
+            s.sendto(5j, sockname)
+        self.assertIn('not complex', str(cm.exception))
+        with self.assertRaises(TypeError) as cm:
+            s.sendto('foo', None)
+        self.assertIn('not NoneType', str(cm.exception))
+        # 3 args
+        with self.assertRaises(UnicodeEncodeError):
+            s.sendto(u'\u2620', 0, sockname)
+        with self.assertRaises(TypeError) as cm:
+            s.sendto(5j, 0, sockname)
+        self.assertIn('not complex', str(cm.exception))
+        with self.assertRaises(TypeError) as cm:
+            s.sendto('foo', 0, None)
+        self.assertIn('not NoneType', str(cm.exception))
+        with self.assertRaises(TypeError) as cm:
+            s.sendto('foo', 'bar', sockname)
+        self.assertIn('an integer is required', str(cm.exception))
+        with self.assertRaises(TypeError) as cm:
+            s.sendto('foo', None, None)
+        self.assertIn('an integer is required', str(cm.exception))
+        # wrong number of args
+        with self.assertRaises(TypeError) as cm:
+            s.sendto('foo')
+        self.assertIn('(1 given)', str(cm.exception))
+        with self.assertRaises(TypeError) as cm:
+            s.sendto('foo', 0, sockname, 4)
+        self.assertIn('(4 given)', str(cm.exception))
+
+
     def testCrucialConstants(self):
         # Testing for mission critical constants
         socket.AF_INET
@@ -662,6 +701,13 @@
     def test_sendall_interrupted_with_timeout(self):
         self.check_sendall_interrupted(True)
 
+    def testListenBacklog0(self):
+        srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        srv.bind((HOST, 0))
+        # backlog = 0
+        srv.listen(0)
+        srv.close()
+
 
 @unittest.skipUnless(thread, 'Threading required for this test.')
 class BasicTCPTest(SocketConnectedTest):
diff --git a/lib-python/modified-2.7/test/test_ssl.py b/lib-python/modified-2.7/test/test_ssl.py
--- a/lib-python/modified-2.7/test/test_ssl.py
+++ b/lib-python/modified-2.7/test/test_ssl.py
@@ -20,12 +20,7 @@
 from BaseHTTPServer import HTTPServer
 from SimpleHTTPServer import SimpleHTTPRequestHandler
 
-# Optionally test SSL support, if we have it in the tested platform
-skip_expected = False
-try:
-    import ssl
-except ImportError:
-    skip_expected = True
+ssl = test_support.import_module("ssl")
 
 HOST = test_support.HOST
 CERTFILE = None
@@ -228,6 +223,50 @@
             finally:
                 s.close()
 
+    def test_connect_ex(self):
+        # Issue #11326: check connect_ex() implementation
+        with test_support.transient_internet("svn.python.org"):
+            s = ssl.wrap_socket(socket.socket(socket.AF_INET),
+                                cert_reqs=ssl.CERT_REQUIRED,
+                                ca_certs=SVN_PYTHON_ORG_ROOT_CERT)
+            try:
+                self.assertEqual(0, s.connect_ex(("svn.python.org", 443)))
+                self.assertTrue(s.getpeercert())
+            finally:
+                s.close()
+
+    def test_non_blocking_connect_ex(self):
+        # Issue #11326: non-blocking connect_ex() should allow handshake
+        # to proceed after the socket gets ready.
+        with test_support.transient_internet("svn.python.org"):
+            s = ssl.wrap_socket(socket.socket(socket.AF_INET),
+                                cert_reqs=ssl.CERT_REQUIRED,
+                                ca_certs=SVN_PYTHON_ORG_ROOT_CERT,
+                                do_handshake_on_connect=False)
+            try:
+                s.setblocking(False)
+                rc = s.connect_ex(('svn.python.org', 443))
+                # EWOULDBLOCK under Windows, EINPROGRESS elsewhere
+                self.assertIn(rc, (0, errno.EINPROGRESS, errno.EWOULDBLOCK))
+                # Wait for connect to finish
+                select.select([], [s], [], 5.0)
+                # Non-blocking handshake
+                while True:
+                    try:
+                        s.do_handshake()
+                        break
+                    except ssl.SSLError as err:
+                        if err.args[0] == ssl.SSL_ERROR_WANT_READ:
+                            select.select([s], [], [], 5.0)
+                        elif err.args[0] == ssl.SSL_ERROR_WANT_WRITE:
+                            select.select([], [s], [], 5.0)
+                        else:
+                            raise
+                # SSL established
+                self.assertTrue(s.getpeercert())
+            finally:
+                s.close()
+
     @unittest.skipIf(os.name == "nt", "Can't use a socket as a file under Windows")
     def test_makefile_close(self):
         # Issue #5238: creating a file-like object with makefile() shouldn't
@@ -300,9 +339,9 @@
         if ssl.OPENSSL_VERSION_INFO < (0, 9, 8, 0, 15):
             self.skipTest("SHA256 not available on %r" % ssl.OPENSSL_VERSION)
         # NOTE: https://sha256.tbs-internet.com is another possible test host
-        remote = ("sha2.hboeck.de", 443)
+        remote = ("sha256.tbs-internet.com", 443)
         sha256_cert = os.path.join(os.path.dirname(__file__), "sha256.pem")
-        with test_support.transient_internet("sha2.hboeck.de"):
+        with test_support.transient_internet("sha256.tbs-internet.com"):
             s = ssl.wrap_socket(socket.socket(socket.AF_INET),
                                 cert_reqs=ssl.CERT_REQUIRED,
                                 ca_certs=sha256_cert,)
@@ -1292,12 +1331,10 @@
 
 
 def test_main(verbose=False):
-    if skip_expected:
-        raise unittest.SkipTest("No SSL support")
-
     global CERTFILE, SVN_PYTHON_ORG_ROOT_CERT
     CERTFILE = test_support.findfile("keycert.pem")
-    SVN_PYTHON_ORG_ROOT_CERT =  test_support.findfile(
+    SVN_PYTHON_ORG_ROOT_CERT = test_support.findfile(
+        os.path.dirname(__file__) or os.curdir,
         "https_svn_python_org_root.pem")
 
     if (not os.path.exists(CERTFILE) or
diff --git a/lib-python/modified-2.7/test/test_subprocess.py b/lib-python/modified-2.7/test/test_subprocess.py
--- a/lib-python/modified-2.7/test/test_subprocess.py
+++ b/lib-python/modified-2.7/test/test_subprocess.py
@@ -575,7 +575,8 @@
                 subprocess.Popen(['nonexisting_i_hope'],
                                  stdout=subprocess.PIPE,
                                  stderr=subprocess.PIPE)
-            if c.exception.errno != errno.ENOENT:  # ignore "no such file"
+            # ignore errors that indicate the command was not found
+            if c.exception.errno not in (errno.ENOENT, errno.EACCES):
                 raise c.exception
 
     def test_handles_closed_on_exception(self):
@@ -598,6 +599,24 @@
         self.assertFalse(os.path.exists(ofname))
         self.assertFalse(os.path.exists(efname))
 
+    def test_communicate_epipe(self):
+        # Issue 10963: communicate() should hide EPIPE
+        p = subprocess.Popen([sys.executable, "-c", 'pass'],
+                             stdin=subprocess.PIPE,
+                             stdout=subprocess.PIPE,
+                             stderr=subprocess.PIPE)
+        self.addCleanup(p.stdout.close)
+        self.addCleanup(p.stderr.close)
+        self.addCleanup(p.stdin.close)
+        p.communicate("x" * 2**20)
+
+    def test_communicate_epipe_only_stdin(self):
+        # Issue 10963: communicate() should hide EPIPE
+        p = subprocess.Popen([sys.executable, "-c", 'pass'],
+                             stdin=subprocess.PIPE)
+        self.addCleanup(p.stdin.close)
+        time.sleep(2)
+        p.communicate("x" * 2**20)
 
 # context manager
 class _SuppressCoreFiles(object):
@@ -780,6 +799,68 @@
         self.assertStderrEqual(stderr, '')
         self.assertEqual(p.wait(), -signal.SIGTERM)
 
+    def check_close_std_fds(self, fds):
+        # Issue #9905: test that subprocess pipes still work properly with
+        # some standard fds closed
+        stdin = 0
+        newfds = []
+        for a in fds:
+            b = os.dup(a)
+            newfds.append(b)
+            if a == 0:
+                stdin = b
+        try:
+            for fd in fds:
+                os.close(fd)
+            out, err = subprocess.Popen([sys.executable, "-c",
+                              'import sys;'
+                              'sys.stdout.write("apple");'
+                              'sys.stdout.flush();'
+                              'sys.stderr.write("orange")'],
+                       stdin=stdin,
+                       stdout=subprocess.PIPE,
+                       stderr=subprocess.PIPE).communicate()
+            err = test_support.strip_python_stderr(err)
+            self.assertEqual((out, err), (b'apple', b'orange'))
+        finally:
+            for b, a in zip(newfds, fds):
+                os.dup2(b, a)
+            for b in newfds:
+                os.close(b)
+
+    def test_close_fd_0(self):
+        self.check_close_std_fds([0])
+
+    def test_close_fd_1(self):
+        self.check_close_std_fds([1])
+
+    def test_close_fd_2(self):
+        self.check_close_std_fds([2])
+
+    def test_close_fds_0_1(self):
+        self.check_close_std_fds([0, 1])
+
+    def test_close_fds_0_2(self):
+        self.check_close_std_fds([0, 2])
+
+    def test_close_fds_1_2(self):
+        self.check_close_std_fds([1, 2])
+
+    def test_close_fds_0_1_2(self):
+        # Issue #10806: test that subprocess pipes still work properly with
+        # all standard fds closed.
+        self.check_close_std_fds([0, 1, 2])
+
+    def test_wait_when_sigchild_ignored(self):
+        # NOTE: sigchild_ignore.py may not be an effective test on all OSes.
+        sigchild_ignore = test_support.findfile("sigchild_ignore.py",
+                                                subdir="subprocessdata")
+        p = subprocess.Popen([sys.executable, sigchild_ignore],
+                             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        stdout, stderr = p.communicate()
+        self.assertEqual(0, p.returncode, "sigchild_ignore.py exited"
+                         " non-zero with this error:\n%s" % stderr)
+
 
 @unittest.skipUnless(mswindows, "Windows specific tests")
 class Win32ProcessTestCase(BaseTestCase):
diff --git a/lib-python/modified-2.7/test/test_support.py b/lib-python/modified-2.7/test/test_support.py
--- a/lib-python/modified-2.7/test/test_support.py
+++ b/lib-python/modified-2.7/test/test_support.py
@@ -35,7 +35,8 @@
            "run_with_locale", "set_memlimit", "bigmemtest", "bigaddrspacetest",
            "BasicTestRunner", "run_unittest", "run_doctest", "threading_setup",
            "threading_cleanup", "reap_children", "cpython_only",
-           "check_impl_detail", "get_attribute", "py3k_bytes"]
+           "check_impl_detail", "get_attribute", "py3k_bytes",
+           "import_fresh_module"]
 
 
 class Error(Exception):
@@ -83,23 +84,20 @@
 def _save_and_remove_module(name, orig_modules):
     """Helper function to save and remove a module from sys.modules
 
-       Return value is True if the module was in sys.modules and
-       False otherwise."""
-    saved = True
-    try:
-        orig_modules[name] = sys.modules[name]
-    except KeyError:
-        saved = False
-    else:
+       Raise ImportError if the module can't be imported."""
+    # try to import the module and raise an error if it can't be imported
+    if name not in sys.modules:
+        __import__(name)
         del sys.modules[name]
-    return saved
-
+    for modname in list(sys.modules):
+        if modname == name or modname.startswith(name + '.'):
+            orig_modules[modname] = sys.modules[modname]
+            del sys.modules[modname]
 
 def _save_and_block_module(name, orig_modules):
     """Helper function to save and block a module in sys.modules
 
-       Return value is True if the module was in sys.modules and
-       False otherwise."""
+       Return True if the module was in sys.modules, False otherwise."""
     saved = True
     try:
         orig_modules[name] = sys.modules[name]
@@ -115,14 +113,15 @@
     the sys.modules cache is restored to its original state.
 
     Modules named in fresh are also imported anew if needed by the import.
+    If one of these modules can't be imported, None is returned.
 
     Importing of modules named in blocked is prevented while the fresh import
     takes place.
 
     If deprecated is True, any module or package deprecation messages
     will be suppressed."""
-    # NOTE: test_heapq and test_warnings include extra sanity checks to make
-    # sure that this utility function is working as expected
+    # NOTE: test_heapq, test_json, and test_warnings include extra sanity
+    # checks to make sure that this utility function is working as expected
     with _ignore_deprecated_imports(deprecated):
         # Keep track of modules saved for later restoration as well
         # as those which just need a blocking entry removed
@@ -136,6 +135,8 @@
                 if not _save_and_block_module(blocked_name, orig_modules):
                     names_to_remove.append(blocked_name)
             fresh_module = importlib.import_module(name)
+        except ImportError:
+            fresh_module = None
         finally:
             for orig_name, module in orig_modules.items():
                 sys.modules[orig_name] = module
@@ -761,6 +762,7 @@
     default_errnos = [
         ('ECONNREFUSED', 111),
         ('ECONNRESET', 104),
+        ('EHOSTUNREACH', 113),
         ('ENETUNREACH', 101),
         ('ETIMEDOUT', 110),
     ]
@@ -816,14 +818,8 @@
 
 @contextlib.contextmanager
 def captured_output(stream_name):
-    """Run the 'with' statement body using a StringIO object in place of a
-    specific attribute on the sys module.
-    Example use (with 'stream_name=stdout')::
-
-       with captured_stdout() as s:
-           print "hello"
-       assert s.getvalue() == "hello"
-    """
+    """Return a context manager used by captured_stdout and captured_stdin
+    that temporarily replaces the sys stream *stream_name* with a StringIO."""
     import StringIO
     orig_stdout = getattr(sys, stream_name)
     setattr(sys, stream_name, StringIO.StringIO())
@@ -833,6 +829,12 @@
         setattr(sys, stream_name, orig_stdout)
 
 def captured_stdout():
+    """Capture the output of sys.stdout:
+
+       with captured_stdout() as s:
+           print "hello"
+       self.assertEqual(s.getvalue(), "hello")
+    """
     return captured_output("stdout")
 
 def captured_stdin():
@@ -1270,3 +1272,13 @@
         if v > 0:
             args.append('-' + opt * v)
     return args
+
+def strip_python_stderr(stderr):
+    """Strip the stderr of a Python process from potential debug output
+    emitted by the interpreter.
+
+    This will typically be run on the result of the communicate() method
+    of a subprocess.Popen object.
+    """
+    stderr = re.sub(br"\[\d+ refs\]\r?\n?$", b"", stderr).strip()
+    return stderr
diff --git a/lib-python/modified-2.7/test/test_syntax.py b/lib-python/modified-2.7/test/test_syntax.py
--- a/lib-python/modified-2.7/test/test_syntax.py
+++ b/lib-python/modified-2.7/test/test_syntax.py
@@ -267,7 +267,7 @@
 
 Test continue in finally in weird combinations.
 
-continue in for loop under finally shouuld be ok.
+continue in for loop under finally should be ok.
 
     >>> def test():
     ...     try:
diff --git a/lib-python/modified-2.7/test/test_sysconfig.py b/lib-python/modified-2.7/test/test_sysconfig.py
--- a/lib-python/modified-2.7/test/test_sysconfig.py
+++ b/lib-python/modified-2.7/test/test_sysconfig.py
@@ -141,7 +141,7 @@
                    ('Darwin Kernel Version 8.11.1: '
                     'Wed Oct 10 18:23:28 PDT 2007; '
                     'root:xnu-792.25.20~1/RELEASE_I386'), 'PowerPC'))
-        os.environ['MACOSX_DEPLOYMENT_TARGET'] = '10.3'
+        get_config_vars()['MACOSX_DEPLOYMENT_TARGET'] = '10.3'
 
         get_config_vars()['CFLAGS'] = ('-fno-strict-aliasing -DNDEBUG -g '
                                        '-fwrapv -O3 -Wall -Wstrict-prototypes')
@@ -161,7 +161,6 @@
                     'Wed Oct 10 18:23:28 PDT 2007; '
                     'root:xnu-792.25.20~1/RELEASE_I386'), 'i386'))
         get_config_vars()['MACOSX_DEPLOYMENT_TARGET'] = '10.3'
-        os.environ['MACOSX_DEPLOYMENT_TARGET'] = '10.3'
 
         get_config_vars()['CFLAGS'] = ('-fno-strict-aliasing -DNDEBUG -g '
                                        '-fwrapv -O3 -Wall -Wstrict-prototypes')
@@ -176,7 +175,7 @@
             sys.maxint = maxint
 
         # macbook with fat binaries (fat, universal or fat64)
-        os.environ['MACOSX_DEPLOYMENT_TARGET'] = '10.4'
+        get_config_vars()['MACOSX_DEPLOYMENT_TARGET'] = '10.4'
         get_config_vars()['CFLAGS'] = ('-arch ppc -arch i386 -isysroot '
                                        '/Developer/SDKs/MacOSX10.4u.sdk  '
                                        '-fno-strict-aliasing -fno-common '
@@ -274,6 +273,51 @@
             user_path = get_path(name, 'posix_user')
             self.assertEqual(user_path, global_path.replace(base, user))
 
+    @unittest.skipUnless(sys.platform == "darwin", "test only relevant on MacOSX")
+    def test_platform_in_subprocess(self):
+        my_platform = sysconfig.get_platform()
+
+        # Test without MACOSX_DEPLOYMENT_TARGET in the environment
+
+        env = os.environ.copy()
+        if 'MACOSX_DEPLOYMENT_TARGET' in env:
+            del env['MACOSX_DEPLOYMENT_TARGET']
+
+        with open('/dev/null', 'w') as devnull_fp:
+            p = subprocess.Popen([
+                    sys.executable, '-c',
+                   'import sysconfig; print(sysconfig.get_platform())',
+                ],
+                stdout=subprocess.PIPE,
+                stderr=devnull_fp,
+                env=env)
+        test_platform = p.communicate()[0].strip()
+        test_platform = test_platform.decode('utf-8')
+        status = p.wait()
+
+        self.assertEqual(status, 0)
+        self.assertEqual(my_platform, test_platform)
+
+
+        # Test with MACOSX_DEPLOYMENT_TARGET in the environment, and
+        # using a value that is unlikely to be the default one.
+        env = os.environ.copy()
+        env['MACOSX_DEPLOYMENT_TARGET'] = '10.1'
+
+        p = subprocess.Popen([
+                sys.executable, '-c',
+                'import sysconfig; print(sysconfig.get_platform())',
+            ],
+            stdout=subprocess.PIPE,
+            stderr=open('/dev/null'),
+            env=env)
+        test_platform = p.communicate()[0].strip()
+        test_platform = test_platform.decode('utf-8')
+        status = p.wait()
+
+        self.assertEqual(status, 0)
+        self.assertEqual(my_platform, test_platform)
+
 def test_main():
     run_unittest(TestSysConfig)
 
diff --git a/lib-python/modified-2.7/test/test_tarfile.py b/lib-python/modified-2.7/test/test_tarfile.py
--- a/lib-python/modified-2.7/test/test_tarfile.py
+++ b/lib-python/modified-2.7/test/test_tarfile.py
@@ -855,6 +855,94 @@
         finally:
             os.chdir(cwd)
 
+    @unittest.skipUnless(hasattr(os, 'symlink'), "needs os.symlink")
+    def test_extractall_symlinks(self):
+        # Test if extractall works properly when tarfile contains symlinks
+        tempdir = os.path.join(TEMPDIR, "testsymlinks")
+        temparchive = os.path.join(TEMPDIR, "testsymlinks.tar")
+        os.mkdir(tempdir)
+        try:
+            source_file = os.path.join(tempdir,'source')
+            target_file = os.path.join(tempdir,'symlink')
+            with open(source_file,'w') as f:
+                f.write('something\n')
+            os.symlink(source_file, target_file)
+            tar = tarfile.open(temparchive,'w')
+            tar.add(source_file, arcname=os.path.basename(source_file))
+            tar.add(target_file, arcname=os.path.basename(target_file))
+            tar.close()
+            # Let's extract it to the location which contains the symlink
+            tar = tarfile.open(temparchive,'r')
+            # this should not raise OSError: [Errno 17] File exists
+            try:
+                tar.extractall(path=tempdir)
+            except OSError:
+                self.fail("extractall failed with symlinked files")
+            finally:
+                tar.close()
+        finally:
+            os.unlink(temparchive)
+            shutil.rmtree(tempdir)
+
+    @unittest.skipUnless(hasattr(os, 'symlink'), "needs os.symlink")
+    def test_extractall_broken_symlinks(self):
+        # Test if extractall works properly when tarfile contains broken
+        # symlinks
+        tempdir = os.path.join(TEMPDIR, "testsymlinks")
+        temparchive = os.path.join(TEMPDIR, "testsymlinks.tar")
+        os.mkdir(tempdir)
+        try:
+            source_file = os.path.join(tempdir,'source')
+            target_file = os.path.join(tempdir,'symlink')
+            with open(source_file,'w') as f:
+                f.write('something\n')
+            os.symlink(source_file, target_file)
+            tar = tarfile.open(temparchive,'w')
+            tar.add(target_file, arcname=os.path.basename(target_file))
+            tar.close()
+            # remove the real file
+            os.unlink(source_file)
+            # Let's extract it to the location which contains the symlink
+            tar = tarfile.open(temparchive,'r')
+            # this should not raise OSError: [Errno 17] File exists
+            try:
+                tar.extractall(path=tempdir)
+            except OSError:
+                self.fail("extractall failed with broken symlinked files")
+            finally:
+                tar.close()
+        finally:
+            os.unlink(temparchive)
+            shutil.rmtree(tempdir)
+
+    @unittest.skipUnless(hasattr(os, 'link'), "needs os.link")
+    def test_extractall_hardlinks(self):
+        # Test if extractall works properly when tarfile contains symlinks
+        tempdir = os.path.join(TEMPDIR, "testsymlinks")
+        temparchive = os.path.join(TEMPDIR, "testsymlinks.tar")
+        os.mkdir(tempdir)
+        try:
+            source_file = os.path.join(tempdir,'source')
+            target_file = os.path.join(tempdir,'symlink')
+            with open(source_file,'w') as f:
+                f.write('something\n')
+            os.link(source_file, target_file)
+            tar = tarfile.open(temparchive,'w')
+            tar.add(source_file, arcname=os.path.basename(source_file))
+            tar.add(target_file, arcname=os.path.basename(target_file))
+            tar.close()
+            # Let's extract it to the location which contains the symlink
+            tar = tarfile.open(temparchive,'r')
+            # this should not raise OSError: [Errno 17] File exists
+            try:
+                tar.extractall(path=tempdir)
+            except OSError:
+                self.fail("extractall failed with linked files")
+            finally:
+                tar.close()
+        finally:
+            os.unlink(temparchive)
+            shutil.rmtree(tempdir)
 
 class StreamWriteTest(WriteTestBase):
 
diff --git a/lib-python/modified-2.7/test/test_telnetlib.py b/lib-python/modified-2.7/test/test_telnetlib.py
--- a/lib-python/modified-2.7/test/test_telnetlib.py
+++ b/lib-python/modified-2.7/test/test_telnetlib.py
@@ -34,12 +34,12 @@
                     data += item
                 written = conn.send(data)
                 data = data[written:]
-        conn.close()
     except socket.timeout:
         pass
+    else:
+        conn.close()
     finally:
         serv.close()
-        conn.close()
         evt.set()
 
 class GeneralTests(TestCase):
diff --git a/lib-python/modified-2.7/test/test_tempfile.py b/lib-python/modified-2.7/test/test_tempfile.py
--- a/lib-python/modified-2.7/test/test_tempfile.py
+++ b/lib-python/modified-2.7/test/test_tempfile.py
@@ -693,6 +693,23 @@
         f.write('x')
         self.assertTrue(f._rolled)
 
+    def test_writelines(self):
+        # Verify writelines with a SpooledTemporaryFile
+        f = self.do_create()
+        f.writelines((b'x', b'y', b'z'))
+        f.seek(0)
+        buf = f.read()
+        self.assertEqual(buf, b'xyz')
+
+    def test_writelines_sequential(self):
+        # A SpooledTemporaryFile should hold exactly max_size bytes, and roll
+        # over afterward
+        f = self.do_create(max_size=35)
+        f.writelines((b'x' * 20, b'x' * 10, b'x' * 5))
+        self.assertFalse(f._rolled)
+        f.write(b'x')
+        self.assertTrue(f._rolled)
+
     def test_sparse(self):
         # A SpooledTemporaryFile that is written late in the file will extend
         # when that occurs
diff --git a/lib-python/modified-2.7/test/test_threading.py b/lib-python/modified-2.7/test/test_threading.py
--- a/lib-python/modified-2.7/test/test_threading.py
+++ b/lib-python/modified-2.7/test/test_threading.py
@@ -10,6 +10,8 @@
 import time
 import unittest
 import weakref
+import os
+import subprocess
 
 from test import lock_tests
 
@@ -277,7 +279,6 @@
                 print("test_finalize_with_runnning_thread can't import ctypes")
             return  # can't do anything
 
-        import subprocess
         rc = subprocess.call([sys.executable, "-c", """if 1:
             import ctypes, sys, time, thread
 
@@ -308,7 +309,6 @@
     def test_finalize_with_trace(self):
         # Issue1733757
         # Avoid a deadlock when sys.settrace steps into threading._shutdown
-        import subprocess
         p = subprocess.Popen([sys.executable, "-c", """if 1:
             import sys, threading
 
@@ -343,7 +343,6 @@
     def test_join_nondaemon_on_shutdown(self):
         # Issue 1722344
         # Raising SystemExit skipped threading._shutdown
-        import subprocess
         p = subprocess.Popen([sys.executable, "-c", """if 1:
                 import threading
                 from time import sleep
@@ -434,7 +433,6 @@
                 sys.stdout.flush()
         \n""" + script
 
-        import subprocess
         p = subprocess.Popen([sys.executable, "-c", script], stdout=subprocess.PIPE)
         rc = p.wait()
         data = p.stdout.read().replace('\r', '')
@@ -506,6 +504,152 @@
             """
         self._run_and_join(script)
 
+    def assertScriptHasOutput(self, script, expected_output):
+        p = subprocess.Popen([sys.executable, "-c", script],
+                             stdout=subprocess.PIPE)
+        rc = p.wait()
+        data = p.stdout.read().decode().replace('\r', '')
+        self.assertEqual(rc, 0, "Unexpected error")
+        self.assertEqual(data, expected_output)
+
+    @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()")
+    def test_4_joining_across_fork_in_worker_thread(self):
+        # There used to be a possible deadlock when forking from a child
+        # thread.  See http://bugs.python.org/issue6643.
+
+        # Skip platforms with known problems forking from a worker thread.
+        # See http://bugs.python.org/issue3863.
+        if sys.platform in ('freebsd4', 'freebsd5', 'freebsd6', 'os2emx'):
+            raise unittest.SkipTest('due to known OS bugs on ' + sys.platform)
+
+        # The script takes the following steps:
+        # - The main thread in the parent process starts a new thread and then
+        #   tries to join it.
+        # - The join operation acquires the Lock inside the thread's _block
+        #   Condition.  (See threading.py:Thread.join().)
+        # - We stub out the acquire method on the condition to force it to wait
+        #   until the child thread forks.  (See LOCK ACQUIRED HERE)
+        # - The child thread forks.  (See LOCK HELD and WORKER THREAD FORKS
+        #   HERE)
+        # - The main thread of the parent process enters Condition.wait(),
+        #   which releases the lock on the child thread.
+        # - The child process returns.  Without the necessary fix, when the
+        #   main thread of the child process (which used to be the child thread
+        #   in the parent process) attempts to exit, it will try to acquire the
+        #   lock in the Thread._block Condition object and hang, because the
+        #   lock was held across the fork.
+
+        script = """if 1:
+            import os, time, threading
+
+            finish_join = False
+            start_fork = False
+
+            def worker():
+                # Wait until this thread's lock is acquired before forking to
+                # create the deadlock.
+                global finish_join
+                while not start_fork:
+                    time.sleep(0.01)
+                # LOCK HELD: Main thread holds lock across this call.
+                childpid = os.fork()
+                finish_join = True
+                if childpid != 0:
+                    # Parent process just waits for child.
+                    os.waitpid(childpid, 0)
+                # Child process should just return.
+
+            w = threading.Thread(target=worker)
+
+            # Stub out the private condition variable's lock acquire method.
+            # This acquires the lock and then waits until the child has forked
+            # before returning, which will release the lock soon after.  If
+            # someone else tries to fix this test case by acquiring this lock
+            # before forking instead of resetting it, the test case will
+            # deadlock when it shouldn't.
+            condition = w._block
+            orig_acquire = condition.acquire
+            call_count_lock = threading.Lock()
+            call_count = 0
+            def my_acquire():
+                global call_count
+                global start_fork
+                orig_acquire()  # LOCK ACQUIRED HERE
+                start_fork = True
+                if call_count == 0:
+                    while not finish_join:
+                        time.sleep(0.01)  # WORKER THREAD FORKS HERE
+                with call_count_lock:
+                    call_count += 1
+            condition.acquire = my_acquire
+
+            w.start()
+            w.join()
+            print('end of main')
+            """
+        self.assertScriptHasOutput(script, "end of main\n")
+
+    @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()")
+    def test_5_clear_waiter_locks_to_avoid_crash(self):
+        # Check that a spawned thread that forks doesn't segfault on certain
+        # platforms, namely OS X.  This used to happen if there was a waiter
+        # lock in the thread's condition variable's waiters list.  Even though
+        # we know the lock will be held across the fork, it is not safe to
+        # release locks held across forks on all platforms, so releasing the
+        # waiter lock caused a segfault on OS X.  Furthermore, since locks on
+        # OS X are (as of this writing) implemented with a mutex + condition
+        # variable instead of a semaphore, while we know that the Python-level
+        # lock will be acquired, we can't know if the internal mutex will be
+        # acquired at the time of the fork.
+
+        # Skip platforms with known problems forking from a worker thread.
+        # See http://bugs.python.org/issue3863.
+        if sys.platform in ('freebsd4', 'freebsd5', 'freebsd6', 'os2emx'):
+            raise unittest.SkipTest('due to known OS bugs on ' + sys.platform)
+        script = """if True:
+            import os, time, threading
+
+            start_fork = False
+
+            def worker():
+                # Wait until the main thread has attempted to join this thread
+                # before continuing.
+                while not start_fork:
+                    time.sleep(0.01)
+                childpid = os.fork()
+                if childpid != 0:
+                    # Parent process just waits for child.
+                    (cpid, rc) = os.waitpid(childpid, 0)
+                    assert cpid == childpid
+                    assert rc == 0
+                    print('end of worker thread')
+                else:
+                    # Child process should just return.
+                    pass
+
+            w = threading.Thread(target=worker)
+
+            # Stub out the private condition variable's _release_save method.
+            # This releases the condition's lock and flips the global that
+            # causes the worker to fork.  At this point, the problematic waiter
+            # lock has been acquired once by the waiter and has been put onto
+            # the waiters list.
+            condition = w._block
+            orig_release_save = condition._release_save
+            def my_release_save():
+                global start_fork
+                orig_release_save()
+                # Waiter lock held here, condition lock released.
+                start_fork = True
+            condition._release_save = my_release_save
+
+            w.start()
+            w.join()
+            print('end of main thread')
+            """
+        output = "end of worker thread\nend of main thread\n"
+        self.assertScriptHasOutput(script, output)
+
 
 class ThreadingExceptionTests(BaseTestCase):
     # A RuntimeError should be raised if Thread.start() is called
@@ -551,6 +695,37 @@
 class BoundedSemaphoreTests(lock_tests.BoundedSemaphoreTests):
     semtype = staticmethod(threading.BoundedSemaphore)
 
+    @unittest.skipUnless(sys.platform == 'darwin', 'test macosx problem')
+    def test_recursion_limit(self):
+        # Issue 9670
+        # test that excessive recursion within a non-main thread causes
+        # an exception rather than crashing the interpreter on platforms
+        # like Mac OS X or FreeBSD which have small default stack sizes
+        # for threads
+        script = """if True:
+            import threading
+
+            def recurse():
+                return recurse()
+
+            def outer():
+                try:
+                    recurse()
+                except RuntimeError:
+                    pass
+
+            w = threading.Thread(target=outer)
+            w.start()
+            w.join()
+            print('end of main thread')
+            """
+        expected_output = "end of main thread\n"
+        p = subprocess.Popen([sys.executable, "-c", script],
+                             stdout=subprocess.PIPE)
+        stdout, stderr = p.communicate()
+        data = stdout.decode().replace('\r', '')
+        self.assertEqual(p.returncode, 0, "Unexpected error")
+        self.assertEqual(data, expected_output)
 
 def test_main():
     test.test_support.run_unittest(LockTests, RLockTests, EventTests,
diff --git a/lib-python/modified-2.7/test/test_unicode.py b/lib-python/modified-2.7/test/test_unicode.py
--- a/lib-python/modified-2.7/test/test_unicode.py
+++ b/lib-python/modified-2.7/test/test_unicode.py
@@ -442,6 +442,17 @@
                 return u'\u1234'
         self.assertEqual('%s' % Wrapper(), u'\u1234')
 
+    def test_startswith_endswith_errors(self):
+        for meth in (u'foo'.startswith, u'foo'.endswith):
+            with self.assertRaises(UnicodeDecodeError):
+                meth('\xff')
+            with self.assertRaises(TypeError) as cm:
+                meth(['f'])
+            exc = str(cm.exception)
+            self.assertIn('unicode', exc)
+            self.assertIn('str', exc)
+            self.assertIn('tuple', exc)
+
     @test_support.run_with_locale('LC_ALL', 'de_DE', 'fr_FR')
     def test_format_float(self):
         # should not format with a comma, but always with C locale
@@ -667,11 +678,17 @@
         # see http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf
         # (table 3-7) and http://www.rfc-editor.org/rfc/rfc3629.txt
         #for cb in map(chr, range(0xA0, 0xC0)):
-            #sys.__stdout__.write('\\xED\\x%02x\\x80\n' % ord(cb))
             #self.assertRaises(UnicodeDecodeError,
                               #('\xED'+cb+'\x80').decode, 'utf-8')
             #self.assertRaises(UnicodeDecodeError,
                               #('\xED'+cb+'\xBF').decode, 'utf-8')
+        # but since they are valid on Python 2 add a test for that:
+        for cb, surrogate in zip(map(chr, range(0xA0, 0xC0)),
+                                 map(unichr, range(0xd800, 0xe000, 64))):
+            encoded = '\xED'+cb+'\x80'
+            self.assertEqual(encoded.decode('utf-8'), surrogate)
+            self.assertEqual(surrogate.encode('utf-8'), encoded)
+
         for cb in map(chr, range(0x80, 0x90)):
             self.assertRaises(UnicodeDecodeError,
                               ('\xF0'+cb+'\x80\x80').decode, 'utf-8')
diff --git a/lib-python/modified-2.7/test/test_unicodedata.py b/lib-python/modified-2.7/test/test_unicodedata.py
--- a/lib-python/modified-2.7/test/test_unicodedata.py
+++ b/lib-python/modified-2.7/test/test_unicodedata.py
@@ -188,9 +188,22 @@
 
     def test_pr29(self):
         # http://www.unicode.org/review/pr-29.html
-        for text in (u"\u0b47\u0300\u0b3e", u"\u1100\u0300\u1161"):
+        # See issues #1054943 and #10254.
+        composed = (u"\u0b47\u0300\u0b3e", u"\u1100\u0300\u1161",
+                    u'Li\u030dt-s\u1e73\u0301',
+                    u'\u092e\u093e\u0930\u094d\u0915 \u091c\u093c'
+                    + u'\u0941\u0915\u0947\u0930\u092c\u0930\u094d\u0917',
+                    u'\u0915\u093f\u0930\u094d\u0917\u093f\u091c\u093c'
+                    + 'u\u0938\u094d\u0924\u093e\u0928')
+        for text in composed:
             self.assertEqual(self.db.normalize('NFC', text), text)
 
+    def test_issue10254(self):
+        # Crash reported in #10254
+        a = u'C\u0338' * 20  + u'C\u0327'
+        b = u'C\u0338' * 20  + u'\xC7'
+        self.assertEqual(self.db.normalize('NFC', a), b)
+
     def test_east_asian_width(self):
         eaw = self.db.east_asian_width
         self.assertRaises(TypeError, eaw, 'a')
diff --git a/lib-python/modified-2.7/test/test_urllib2.py b/lib-python/modified-2.7/test/test_urllib2.py
--- a/lib-python/modified-2.7/test/test_urllib2.py
+++ b/lib-python/modified-2.7/test/test_urllib2.py
@@ -972,6 +972,28 @@
             self.assertEqual(count,
                              urllib2.HTTPRedirectHandler.max_redirections)
 
+    def test_invalid_redirect(self):
+        from_url = "http://example.com/a.html"
+        valid_schemes = ['http', 'https', 'ftp']
+        invalid_schemes = ['file', 'imap', 'ldap']
+        schemeless_url = "example.com/b.html"
+        h = urllib2.HTTPRedirectHandler()
+        o = h.parent = MockOpener()
+        req = Request(from_url)
+        req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT
+
+        for scheme in invalid_schemes:
+            invalid_url = scheme + '://' + schemeless_url
+            self.assertRaises(urllib2.HTTPError, h.http_error_302,
+                              req, MockFile(), 302, "Security Loophole",
+                              MockHeaders({"location": invalid_url}))
+
+        for scheme in valid_schemes:
+            valid_url = scheme + '://' + schemeless_url
+            h.http_error_302(req, MockFile(), 302, "That's fine",
+                MockHeaders({"location": valid_url}))
+            self.assertEqual(o.req.get_full_url(), valid_url)
+
     def test_cookie_redirect(self):
         # cookies shouldn't leak into redirected requests
         from cookielib import CookieJar
@@ -988,6 +1010,15 @@
         o.open("http://www.example.com/")
         self.assertTrue(not hh.req.has_header("Cookie"))
 
+    def test_redirect_fragment(self):
+        redirected_url = 'http://www.example.com/index.html#OK\r\n\r\n'
+        hh = MockHTTPHandler(302, 'Location: ' + redirected_url)
+        hdeh = urllib2.HTTPDefaultErrorHandler()
+        hrh = urllib2.HTTPRedirectHandler()
+        o = build_test_opener(hh, hdeh, hrh)
+        fp = o.open('http://www.example.com')
+        self.assertEqual(fp.geturl(), redirected_url.strip())
+
     def test_proxy(self):
         o = OpenerDirector()
         ph = urllib2.ProxyHandler(dict(http="proxy.example.com:3128"))
@@ -1273,12 +1304,16 @@
         req = Request("")
         self.assertEqual("www.python.org", req.get_host())
 
-    def test_urlwith_fragment(self):
+    def test_url_fragment(self):
         req = Request("http://www.python.org/?qs=query#fragment=true")
         self.assertEqual("/?qs=query", req.get_selector())
         req = Request("http://www.python.org/#fun=true")
         self.assertEqual("/", req.get_selector())
 
+        # Issue 11703: geturl() omits fragment in the original URL.
+        url = 'http://docs.python.org/library/urllib2.html#OK'
+        req = Request(url)
+        self.assertEqual(req.get_full_url(), url)
 
 def test_main(verbose=None):
     from test import test_urllib2
diff --git a/lib-python/modified-2.7/test/test_warnings.py b/lib-python/modified-2.7/test/test_warnings.py
--- a/lib-python/modified-2.7/test/test_warnings.py
+++ b/lib-python/modified-2.7/test/test_warnings.py
@@ -320,7 +320,7 @@
             sys.argv = argv
 
     def test_warn_explicit_type_errors(self):
-        # warn_explicit() shoud error out gracefully if it is given objects
+        # warn_explicit() should error out gracefully if it is given objects
         # of the wrong types.
         # lineno is expected to be an integer.
         self.assertRaises(TypeError, self.module.warn_explicit,
diff --git a/lib-python/modified-2.7/test/test_weakset.py b/lib-python/modified-2.7/test/test_weakset.py
--- a/lib-python/modified-2.7/test/test_weakset.py
+++ b/lib-python/modified-2.7/test/test_weakset.py
@@ -63,7 +63,8 @@
     def test_contains(self):
         for c in self.letters:
             self.assertEqual(c in self.s, c in self.d)
-        self.assertRaises(TypeError, self.s.__contains__, [[]])
+        # 1 is not weakref'able, but that TypeError is caught by __contains__
+        self.assertNotIn(1, self.s)
         self.assertIn(self.obj, self.fs)
         del self.obj
         test_support.gc_collect()
diff --git a/lib-python/modified-2.7/test/test_zlib.py b/lib-python/modified-2.7/test/test_zlib.py
--- a/lib-python/modified-2.7/test/test_zlib.py
+++ b/lib-python/modified-2.7/test/test_zlib.py
@@ -1,11 +1,17 @@
 import unittest
-from test import test_support
+from test.test_support import TESTFN, run_unittest, import_module, unlink, requires
 import binascii
 import os
 import random
-from test.test_support import precisionbigmemtest, _1G
+from test.test_support import precisionbigmemtest, _1G, _4G
+import sys
 
-zlib = test_support.import_module('zlib')
+try:
+    import mmap
+except ImportError:
+    mmap = None
+
+zlib = import_module('zlib')
 
 
 class ChecksumTestCase(unittest.TestCase):
@@ -298,6 +304,15 @@
         self.assertRaises(ValueError, dco.decompress, "", -1)
         self.assertEqual('', dco.unconsumed_tail)
 
+    def test_clear_unconsumed_tail(self):
+        # Issue #12050: calling decompress() without providing max_length
+        # should clear the unconsumed_tail attribute.
+        cdata = "x\x9cKLJ\x06\x00\x02M\x01"     # "abc"
+        dco = zlib.decompressobj()
+        ddata = dco.decompress(cdata, 1)
+        ddata += dco.decompress(dco.unconsumed_tail)
+        self.assertEqual(dco.unconsumed_tail, "")
+
     def test_flushes(self):
         # Test flush() with the various options, using all the
         # different levels in order to provide more variations.
@@ -540,7 +555,7 @@
 
 
 def test_main():
-    test_support.run_unittest(
+    run_unittest(
         ChecksumTestCase,
         ExceptionTestCase,
         CompressTestCase,
diff --git a/lib-python/modified-2.7/trace.py b/lib-python/modified-2.7/trace.py
--- a/lib-python/modified-2.7/trace.py
+++ b/lib-python/modified-2.7/trace.py
@@ -335,7 +335,7 @@
                                                       lnotab, count)
 
             if summary and n_lines:
-                percent = int(100 * n_hits / n_lines)
+                percent = 100 * n_hits // n_lines
                 sums[modulename] = n_lines, percent, modulename, filename
 
         if summary and sums:
diff --git a/lib-python/modified-2.7/urllib2.py b/lib-python/modified-2.7/urllib2.py
--- a/lib-python/modified-2.7/urllib2.py
+++ b/lib-python/modified-2.7/urllib2.py
@@ -190,7 +190,7 @@
                  origin_req_host=None, unverifiable=False):
         # unwrap('') --> 'type://host/path'
         self.__original = unwrap(url)
-        self.__original, fragment = splittag(self.__original)
+        self.__original, self.__fragment = splittag(self.__original)
         self.type = None
         # self.__r_type is what's left after doing the splittype
         self.host = None
@@ -236,7 +236,10 @@
         return self.data
 
     def get_full_url(self):
-        return self.__original
+        if self.__fragment:
+            return '%s#%s' % (self.__original, self.__fragment)
+        else:
+            return self.__original
 
     def get_type(self):
         if self.type is None:
@@ -299,8 +302,9 @@
     def __init__(self):
         client_version = "Python-urllib/%s" % __version__
         self.addheaders = [('User-agent', client_version)]
+        # self.handlers is retained only for backward compatibility
+        self.handlers = []
         # manage the individual handlers
-        self.handlers = []
         self.handle_open = {}
         self.handle_error = {}
         self.process_response = {}
@@ -350,8 +354,6 @@
             added = True
 
         if added:
-            # the handlers must work in an specific order, the order
-            # is specified in a Handler attribute
             bisect.insort(self.handlers, handler)
             handler.add_parent(self)
 
@@ -579,6 +581,17 @@
 
         newurl = urlparse.urljoin(req.get_full_url(), newurl)
 
+        # For security reasons we do not allow redirects to protocols
+        # other than HTTP, HTTPS or FTP.
+        newurl_lower = newurl.lower()
+        if not (newurl_lower.startswith('http://') or
+                newurl_lower.startswith('https://') or
+                newurl_lower.startswith('ftp://')):
+            raise HTTPError(newurl, code,
+                            msg + " - Redirection to url '%s' is not allowed" %
+                            newurl,
+                            headers, fp)
+
         # XXX Probably want to forget about the state of the current
         # request, although that might interact poorly with other
         # handlers that also use handler-specific request attributes

From noreply at buildbot.pypy.org  Sun Jan 22 10:17:14 2012
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sun, 22 Jan 2012 10:17:14 +0100 (CET)
Subject: [pypy-commit] pypy merge-2.7.2: List a test that was recently added,
 otherwise the test suite won't run :(
Message-ID: <20120122091714.EEFB8821FA@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: merge-2.7.2
Changeset: r51624:ff154856cb9b
Date: 2012-01-22 10:16 +0100
http://bitbucket.org/pypy/pypy/changeset/ff154856cb9b/

Log:	List a test that was recently added, otherwise the test suite won't
	run :(

diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -314,6 +314,7 @@
     RegrTest('test_mmap.py'),
     RegrTest('test_module.py', core=True),
     RegrTest('test_modulefinder.py'),
+    RegrTest('test_msilib.py', skip=only_win32),
     RegrTest('test_multibytecodec.py', usemodules='_multibytecodec'),
     RegrTest('test_multibytecodec_support.py', skip="not a test"),
     RegrTest('test_multifile.py'),

From noreply at buildbot.pypy.org  Sun Jan 22 10:33:50 2012
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sun, 22 Jan 2012 10:33:50 +0100 (CET)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <20120122093350.D8630821FA@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: 
Changeset: r51625:01999b68d9cc
Date: 2012-01-22 10:33 +0100
http://bitbucket.org/pypy/pypy/changeset/01999b68d9cc/

Log:	merge heads

diff --git a/lib_pypy/datetime.py b/lib_pypy/datetime.py
--- a/lib_pypy/datetime.py
+++ b/lib_pypy/datetime.py
@@ -13,7 +13,7 @@
 Sources for time zone and DST data: http://www.twinsun.com/tz/tz-link.htm
 
 This was originally copied from the sandbox of the CPython CVS repository.
-Thanks to Tim Peters for suggesting using it. 
+Thanks to Tim Peters for suggesting using it.
 """
 
 import time as _time
@@ -271,6 +271,8 @@
     raise ValueError("%s()=%d, must be in -1439..1439" % (name, offset))
 
 def _check_date_fields(year, month, day):
+    if not isinstance(year, (int, long)):
+        raise TypeError('int expected')
     if not MINYEAR <= year <= MAXYEAR:
         raise ValueError('year must be in %d..%d' % (MINYEAR, MAXYEAR), year)
     if not 1 <= month <= 12:
@@ -280,6 +282,8 @@
         raise ValueError('day must be in 1..%d' % dim, day)
 
 def _check_time_fields(hour, minute, second, microsecond):
+    if not isinstance(hour, (int, long)):
+        raise TypeError('int expected')
     if not 0 <= hour <= 23:
         raise ValueError('hour must be in 0..23', hour)
     if not 0 <= minute <= 59:
@@ -543,61 +547,76 @@
 
         self = object.__new__(cls)
 
-        self.__days = d
-        self.__seconds = s
-        self.__microseconds = us
+        self._days = d
+        self._seconds = s
+        self._microseconds = us
         if abs(d) > 999999999:
             raise OverflowError("timedelta # of days is too large: %d" % d)
 
         return self
 
     def __repr__(self):
-        if self.__microseconds:
+        if self._microseconds:
             return "%s(%d, %d, %d)" % ('datetime.' + self.__class__.__name__,
-                                       self.__days,
-                                       self.__seconds,
-                                       self.__microseconds)
-        if self.__seconds:
+                                       self._days,
+                                       self._seconds,
+                                       self._microseconds)
+        if self._seconds:
             return "%s(%d, %d)" % ('datetime.' + self.__class__.__name__,
-                                   self.__days,
-                                   self.__seconds)
-        return "%s(%d)" % ('datetime.' + self.__class__.__name__, self.__days)
+                                   self._days,
+                                   self._seconds)
+        return "%s(%d)" % ('datetime.' + self.__class__.__name__, self._days)
 
     def __str__(self):
-        mm, ss = divmod(self.__seconds, 60)
+        mm, ss = divmod(self._seconds, 60)
         hh, mm = divmod(mm, 60)
         s = "%d:%02d:%02d" % (hh, mm, ss)
-        if self.__days:
+        if self._days:
             def plural(n):
                 return n, abs(n) != 1 and "s" or ""
-            s = ("%d day%s, " % plural(self.__days)) + s
-        if self.__microseconds:
-            s = s + ".%06d" % self.__microseconds
+            s = ("%d day%s, " % plural(self._days)) + s
+        if self._microseconds:
+            s = s + ".%06d" % self._microseconds
         return s
 
-    days = property(lambda self: self.__days, doc="days")
-    seconds = property(lambda self: self.__seconds, doc="seconds")
-    microseconds = property(lambda self: self.__microseconds,
-                            doc="microseconds")
-
     def total_seconds(self):
         return ((self.days * 86400 + self.seconds) * 10**6
                 + self.microseconds) / 1e6
 
+    # Read-only field accessors
+    @property
+    def days(self):
+        """days"""
+        return self._days
+
+    @property
+    def seconds(self):
+        """seconds"""
+        return self._seconds
+
+    @property
+    def microseconds(self):
+        """microseconds"""
+        return self._microseconds
+
     def __add__(self, other):
         if isinstance(other, timedelta):
             # for CPython compatibility, we cannot use
             # our __class__ here, but need a real timedelta
-            return timedelta(self.__days + other.__days,
-                             self.__seconds + other.__seconds,
-                             self.__microseconds + other.__microseconds)
+            return timedelta(self._days + other._days,
+                             self._seconds + other._seconds,
+                             self._microseconds + other._microseconds)
         return NotImplemented
 
     __radd__ = __add__
 
     def __sub__(self, other):
         if isinstance(other, timedelta):
-            return self + -other
+            # for CPython compatibility, we cannot use
+            # our __class__ here, but need a real timedelta
+            return timedelta(self._days - other._days,
+                             self._seconds - other._seconds,
+                             self._microseconds - other._microseconds)
         return NotImplemented
 
     def __rsub__(self, other):
@@ -606,17 +625,17 @@
         return NotImplemented
 
     def __neg__(self):
-            # for CPython compatibility, we cannot use
-            # our __class__ here, but need a real timedelta
-            return timedelta(-self.__days,
-                             -self.__seconds,
-                             -self.__microseconds)
+        # for CPython compatibility, we cannot use
+        # our __class__ here, but need a real timedelta
+        return timedelta(-self._days,
+                         -self._seconds,
+                         -self._microseconds)
 
     def __pos__(self):
         return self
 
     def __abs__(self):
-        if self.__days < 0:
+        if self._days < 0:
             return -self
         else:
             return self
@@ -625,81 +644,81 @@
         if isinstance(other, (int, long)):
             # for CPython compatibility, we cannot use
             # our __class__ here, but need a real timedelta
-            return timedelta(self.__days * other,
-                             self.__seconds * other,
-                             self.__microseconds * other)
+            return timedelta(self._days * other,
+                             self._seconds * other,
+                             self._microseconds * other)
         return NotImplemented
 
     __rmul__ = __mul__
 
     def __div__(self, other):
         if isinstance(other, (int, long)):
-            usec = ((self.__days * (24*3600L) + self.__seconds) * 1000000 +
-                    self.__microseconds)
+            usec = ((self._days * (24*3600L) + self._seconds) * 1000000 +
+                    self._microseconds)
             return timedelta(0, 0, usec // other)
         return NotImplemented
 
     __floordiv__ = __div__
 
-    # Comparisons.
+    # Comparisons of timedelta objects with other.
 
     def __eq__(self, other):
         if isinstance(other, timedelta):
-            return self.__cmp(other) == 0
+            return self._cmp(other) == 0
         else:
             return False
 
     def __ne__(self, other):
         if isinstance(other, timedelta):
-            return self.__cmp(other) != 0
+            return self._cmp(other) != 0
         else:
             return True
 
     def __le__(self, other):
         if isinstance(other, timedelta):
-            return self.__cmp(other) <= 0
+            return self._cmp(other) <= 0
         else:
             _cmperror(self, other)
 
     def __lt__(self, other):
         if isinstance(other, timedelta):
-            return self.__cmp(other) < 0
+            return self._cmp(other) < 0
         else:
             _cmperror(self, other)
 
     def __ge__(self, other):
         if isinstance(other, timedelta):
-            return self.__cmp(other) >= 0
+            return self._cmp(other) >= 0
         else:
             _cmperror(self, other)
 
     def __gt__(self, other):
         if isinstance(other, timedelta):
-            return self.__cmp(other) > 0
+            return self._cmp(other) > 0
         else:
             _cmperror(self, other)
 
-    def __cmp(self, other):
+    def _cmp(self, other):
         assert isinstance(other, timedelta)
-        return cmp(self.__getstate(), other.__getstate())
+        return cmp(self._getstate(), other._getstate())
 
     def __hash__(self):
-        return hash(self.__getstate())
+        return hash(self._getstate())
 
     def __nonzero__(self):
-        return (self.__days != 0 or
-                self.__seconds != 0 or
-                self.__microseconds != 0)
+        return (self._days != 0 or
+                self._seconds != 0 or
+                self._microseconds != 0)
 
     # Pickle support.
 
     __safe_for_unpickling__ = True      # For Python 2.2
 
-    def __getstate(self):
-        return (self.__days, self.__seconds, self.__microseconds)
+    def _getstate(self):
+        return (self._days, self._seconds, self._microseconds)
 
     def __reduce__(self):
-        return (self.__class__, self.__getstate())
+        return (self.__class__, self._getstate())
 
 timedelta.min = timedelta(-999999999)
 timedelta.max = timedelta(days=999999999, hours=23, minutes=59, seconds=59,
@@ -749,25 +768,26 @@
             return self
         _check_date_fields(year, month, day)
         self = object.__new__(cls)
-        self.__year = year
-        self.__month = month
-        self.__day = day
+        self._year = year
+        self._month = month
+        self._day = day
         return self
 
     # Additional constructors
 
+    @classmethod
     def fromtimestamp(cls, t):
         "Construct a date from a POSIX timestamp (like time.time())."
         y, m, d, hh, mm, ss, weekday, jday, dst = _time.localtime(t)
         return cls(y, m, d)
-    fromtimestamp = classmethod(fromtimestamp)
 
+    @classmethod
     def today(cls):
         "Construct a date from time.time()."
         t = _time.time()
         return cls.fromtimestamp(t)
-    today = classmethod(today)
 
+    @classmethod
     def fromordinal(cls, n):
         """Contruct a date from a proleptic Gregorian ordinal.
 
@@ -776,16 +796,24 @@
         """
         y, m, d = _ord2ymd(n)
         return cls(y, m, d)
-    fromordinal = classmethod(fromordinal)
 
     # Conversions to string
 
     def __repr__(self):
-        "Convert to formal string, for repr()."
+        """Convert to formal string, for repr().
+
+        >>> dt = datetime(2010, 1, 1)
+        >>> repr(dt)
+        'datetime.datetime(2010, 1, 1, 0, 0)'
+
+        >>> dt = datetime(2010, 1, 1, tzinfo=timezone.utc)
+        >>> repr(dt)
+        'datetime.datetime(2010, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)'
+        """
         return "%s(%d, %d, %d)" % ('datetime.' + self.__class__.__name__,
-                                   self.__year,
-                                   self.__month,
-                                   self.__day)
+                                   self._year,
+                                   self._month,
+                                   self._day)
     # XXX These shouldn't depend on time.localtime(), because that
     # clips the usable dates to [1970 .. 2038).  At least ctime() is
     # easily done without using strftime() -- that's better too because
@@ -793,12 +821,20 @@
 
     def ctime(self):
         "Format a la ctime()."
-        return tmxxx(self.__year, self.__month, self.__day).ctime()
+        return tmxxx(self._year, self._month, self._day).ctime()
 
     def strftime(self, fmt):
         "Format using strftime()."
         return _wrap_strftime(self, fmt, self.timetuple())
 
+    def __format__(self, fmt):
+        if not isinstance(fmt, (str, unicode)):
+            raise ValueError("__format__ excepts str or unicode, not %s" %
+                             fmt.__class__.__name__)
+        if len(fmt) != 0:
+            return self.strftime(fmt)
+        return str(self)
+
     def isoformat(self):
         """Return the date formatted according to ISO.
 
@@ -808,29 +844,31 @@
         - http://www.w3.org/TR/NOTE-datetime
         - http://www.cl.cam.ac.uk/~mgk25/iso-time.html
         """
-        return "%04d-%02d-%02d" % (self.__year, self.__month, self.__day)
+        return "%04d-%02d-%02d" % (self._year, self._month, self._day)
 
     __str__ = isoformat
 
-    def __format__(self, format):
-        if not isinstance(format, (str, unicode)):
-            raise ValueError("__format__ excepts str or unicode, not %s" %
-                             format.__class__.__name__)
-        if not format:
-            return str(self)
-        return self.strftime(format)
+    # Read-only field accessors
+    @property
+    def year(self):
+        """year (1-9999)"""
+        return self._year
 
-    # Read-only field accessors
-    year = property(lambda self: self.__year,
-                    doc="year (%d-%d)" % (MINYEAR, MAXYEAR))
-    month = property(lambda self: self.__month, doc="month (1-12)")
-    day = property(lambda self: self.__day, doc="day (1-31)")
+    @property
+    def month(self):
+        """month (1-12)"""
+        return self._month
+
+    @property
+    def day(self):
+        """day (1-31)"""
+        return self._day
 
     # Standard conversions, __cmp__, __hash__ (and helpers)
 
     def timetuple(self):
         "Return local time tuple compatible with time.localtime()."
-        return _build_struct_time(self.__year, self.__month, self.__day,
+        return _build_struct_time(self._year, self._month, self._day,
                                   0, 0, 0, -1)
 
     def toordinal(self):
@@ -839,24 +877,24 @@
         January 1 of year 1 is day 1.  Only the year, month and day values
         contribute to the result.
         """
-        return _ymd2ord(self.__year, self.__month, self.__day)
+        return _ymd2ord(self._year, self._month, self._day)
 
     def replace(self, year=None, month=None, day=None):
         """Return a new date with new values for the specified fields."""
         if year is None:
-            year = self.__year
+            year = self._year
         if month is None:
-            month = self.__month
+            month = self._month
         if day is None:
-            day = self.__day
+            day = self._day
         _check_date_fields(year, month, day)
         return date(year, month, day)
 
-    # Comparisons.
+    # Comparisons of date objects with other.
 
     def __eq__(self, other):
         if isinstance(other, date):
-            return self.__cmp(other) == 0
+            return self._cmp(other) == 0
         elif hasattr(other, "timetuple"):
             return NotImplemented
         else:
@@ -864,7 +902,7 @@
 
     def __ne__(self, other):
         if isinstance(other, date):
-            return self.__cmp(other) != 0
+            return self._cmp(other) != 0
         elif hasattr(other, "timetuple"):
             return NotImplemented
         else:
@@ -872,7 +910,7 @@
 
     def __le__(self, other):
         if isinstance(other, date):
-            return self.__cmp(other) <= 0
+            return self._cmp(other) <= 0
         elif hasattr(other, "timetuple"):
             return NotImplemented
         else:
@@ -880,7 +918,7 @@
 
     def __lt__(self, other):
         if isinstance(other, date):
-            return self.__cmp(other) < 0
+            return self._cmp(other) < 0
         elif hasattr(other, "timetuple"):
             return NotImplemented
         else:
@@ -888,7 +926,7 @@
 
     def __ge__(self, other):
         if isinstance(other, date):
-            return self.__cmp(other) >= 0
+            return self._cmp(other) >= 0
         elif hasattr(other, "timetuple"):
             return NotImplemented
         else:
@@ -896,21 +934,21 @@
 
     def __gt__(self, other):
         if isinstance(other, date):
-            return self.__cmp(other) > 0
+            return self._cmp(other) > 0
         elif hasattr(other, "timetuple"):
             return NotImplemented
         else:
             _cmperror(self, other)
 
-    def __cmp(self, other):
+    def _cmp(self, other):
         assert isinstance(other, date)
-        y, m, d = self.__year, self.__month, self.__day
-        y2, m2, d2 = other.__year, other.__month, other.__day
+        y, m, d = self._year, self._month, self._day
+        y2, m2, d2 = other._year, other._month, other._day
         return cmp((y, m, d), (y2, m2, d2))
 
     def __hash__(self):
         "Hash."
-        return hash(self.__getstate())
+        return hash(self._getstate())
 
     # Computations
 
@@ -922,9 +960,9 @@
     def __add__(self, other):
         "Add a date to a timedelta."
         if isinstance(other, timedelta):
-            t = tmxxx(self.__year,
-                      self.__month,
-                      self.__day + other.days)
+            t = tmxxx(self._year,
+                      self._month,
+                      self._day + other.days)
             self._checkOverflow(t.year)
             result = date(t.year, t.month, t.day)
             return result
@@ -966,9 +1004,9 @@
         ISO calendar algorithm taken from
         http://www.phys.uu.nl/~vgent/calendar/isocalendar.htm
         """
-        year = self.__year
+        year = self._year
         week1monday = _isoweek1monday(year)
-        today = _ymd2ord(self.__year, self.__month, self.__day)
+        today = _ymd2ord(self._year, self._month, self._day)
         # Internally, week and day have origin 0
         week, day = divmod(today - week1monday, 7)
         if week < 0:
@@ -985,18 +1023,18 @@
 
     __safe_for_unpickling__ = True      # For Python 2.2
 
-    def __getstate(self):
-        yhi, ylo = divmod(self.__year, 256)
-        return ("%c%c%c%c" % (yhi, ylo, self.__month, self.__day), )
+    def _getstate(self):
+        yhi, ylo = divmod(self._year, 256)
+        return ("%c%c%c%c" % (yhi, ylo, self._month, self._day), )
 
     def __setstate(self, string):
         if len(string) != 4 or not (1 <= ord(string[2]) <= 12):
             raise TypeError("not enough arguments")
-        yhi, ylo, self.__month, self.__day = map(ord, string)
-        self.__year = yhi * 256 + ylo
+        yhi, ylo, self._month, self._day = map(ord, string)
+        self._year = yhi * 256 + ylo
 
     def __reduce__(self):
-        return (self.__class__, self.__getstate())
+        return (self.__class__, self._getstate())
 
 _date_class = date  # so functions w/ args named "date" can get at the class
 
@@ -1118,62 +1156,80 @@
             return self
         _check_tzinfo_arg(tzinfo)
         _check_time_fields(hour, minute, second, microsecond)
-        self.__hour = hour
-        self.__minute = minute
-        self.__second = second
-        self.__microsecond = microsecond
+        self._hour = hour
+        self._minute = minute
+        self._second = second
+        self._microsecond = microsecond
         self._tzinfo = tzinfo
         return self
 
     # Read-only field accessors
-    hour = property(lambda self: self.__hour, doc="hour (0-23)")
-    minute = property(lambda self: self.__minute, doc="minute (0-59)")
-    second = property(lambda self: self.__second, doc="second (0-59)")
-    microsecond = property(lambda self: self.__microsecond,
-                           doc="microsecond (0-999999)")
-    tzinfo = property(lambda self: self._tzinfo, doc="timezone info object")
+    @property
+    def hour(self):
+        """hour (0-23)"""
+        return self._hour
+
+    @property
+    def minute(self):
+        """minute (0-59)"""
+        return self._minute
+
+    @property
+    def second(self):
+        """second (0-59)"""
+        return self._second
+
+    @property
+    def microsecond(self):
+        """microsecond (0-999999)"""
+        return self._microsecond
+
+    @property
+    def tzinfo(self):
+        """timezone info object"""
+        return self._tzinfo
 
     # Standard conversions, __hash__ (and helpers)
 
-    # Comparisons.
+    # Comparisons of time objects with other.
 
     def __eq__(self, other):
         if isinstance(other, time):
-            return self.__cmp(other) == 0
+            return self._cmp(other) == 0
         else:
             return False
 
     def __ne__(self, other):
         if isinstance(other, time):
-            return self.__cmp(other) != 0
+            return self._cmp(other) != 0
         else:
             return True
 
     def __le__(self, other):
         if isinstance(other, time):
-            return self.__cmp(other) <= 0
+            return self._cmp(other) <= 0
         else:
             _cmperror(self, other)
 
     def __lt__(self, other):
         if isinstance(other, time):
-            return self.__cmp(other) < 0
+            return self._cmp(other) < 0
         else:
             _cmperror(self, other)
 
     def __ge__(self, other):
         if isinstance(other, time):
-            return self.__cmp(other) >= 0
+            return self._cmp(other) >= 0
         else:
             _cmperror(self, other)
 
     def __gt__(self, other):
         if isinstance(other, time):
-            return self.__cmp(other) > 0
+            return self._cmp(other) > 0
         else:
             _cmperror(self, other)
 
-    def __cmp(self, other):
+    def _cmp(self, other):
         assert isinstance(other, time)
         mytz = self._tzinfo
         ottz = other._tzinfo
@@ -1187,23 +1243,23 @@
             base_compare = myoff == otoff
 
         if base_compare:
-            return cmp((self.__hour, self.__minute, self.__second,
-                        self.__microsecond),
-                       (other.__hour, other.__minute, other.__second,
-                        other.__microsecond))
+            return cmp((self._hour, self._minute, self._second,
+                        self._microsecond),
+                       (other._hour, other._minute, other._second,
+                        other._microsecond))
         if myoff is None or otoff is None:
             # XXX Buggy in 2.2.2.
             raise TypeError("cannot compare naive and aware times")
-        myhhmm = self.__hour * 60 + self.__minute - myoff
-        othhmm = other.__hour * 60 + other.__minute - otoff
-        return cmp((myhhmm, self.__second, self.__microsecond),
-                   (othhmm, other.__second, other.__microsecond))
+        myhhmm = self._hour * 60 + self._minute - myoff
+        othhmm = other._hour * 60 + other._minute - otoff
+        return cmp((myhhmm, self._second, self._microsecond),
+                   (othhmm, other._second, other._microsecond))
 
     def __hash__(self):
         """Hash."""
         tzoff = self._utcoffset()
         if not tzoff: # zero or None
-            return hash(self.__getstate()[0])
+            return hash(self._getstate()[0])
         h, m = divmod(self.hour * 60 + self.minute - tzoff, 60)
         if 0 <= h < 24:
             return hash(time(h, m, self.second, self.microsecond))
@@ -1227,14 +1283,14 @@
 
     def __repr__(self):
         """Convert to formal string, for repr()."""
-        if self.__microsecond != 0:
-            s = ", %d, %d" % (self.__second, self.__microsecond)
-        elif self.__second != 0:
-            s = ", %d" % self.__second
+        if self._microsecond != 0:
+            s = ", %d, %d" % (self._second, self._microsecond)
+        elif self._second != 0:
+            s = ", %d" % self._second
         else:
             s = ""
         s= "%s(%d, %d%s)" % ('datetime.' + self.__class__.__name__,
-                             self.__hour, self.__minute, s)
+                             self._hour, self._minute, s)
         if self._tzinfo is not None:
             assert s[-1:] == ")"
             s = s[:-1] + ", tzinfo=%r" % self._tzinfo + ")"
@@ -1246,8 +1302,8 @@
         This is 'HH:MM:SS.mmmmmm+zz:zz', or 'HH:MM:SS+zz:zz' if
         self.microsecond == 0.
         """
-        s = _format_time(self.__hour, self.__minute, self.__second,
-                         self.__microsecond)
+        s = _format_time(self._hour, self._minute, self._second,
+                         self._microsecond)
         tz = self._tzstr()
         if tz:
             s += tz
@@ -1255,14 +1311,6 @@
 
     __str__ = isoformat
 
-    def __format__(self, format):
-        if not isinstance(format, (str, unicode)):
-            raise ValueError("__format__ excepts str or unicode, not %s" %
-                             format.__class__.__name__)
-        if not format:
-            return str(self)
-        return self.strftime(format)
-
     def strftime(self, fmt):
         """Format using strftime().  The date part of the timestamp passed
         to underlying strftime should not be used.
@@ -1270,10 +1318,18 @@
         # The year must be >= 1900 else Python's strftime implementation
         # can raise a bogus exception.
         timetuple = (1900, 1, 1,
-                     self.__hour, self.__minute, self.__second,
+                     self._hour, self._minute, self._second,
                      0, 1, -1)
         return _wrap_strftime(self, fmt, timetuple)
 
+    def __format__(self, fmt):
+        if not isinstance(fmt, (str, unicode)):
+            raise ValueError("__format__ excepts str or unicode, not %s" %
+                             fmt.__class__.__name__)
+        if len(fmt) != 0:
+            return self.strftime(fmt)
+        return str(self)
+
     # Timezone functions
 
     def utcoffset(self):
@@ -1350,10 +1406,10 @@
 
     __safe_for_unpickling__ = True      # For Python 2.2
 
-    def __getstate(self):
-        us2, us3 = divmod(self.__microsecond, 256)
+    def _getstate(self):
+        us2, us3 = divmod(self._microsecond, 256)
         us1, us2 = divmod(us2, 256)
-        basestate = ("%c" * 6) % (self.__hour, self.__minute, self.__second,
+        basestate = ("%c" * 6) % (self._hour, self._minute, self._second,
                                   us1, us2, us3)
         if self._tzinfo is None:
             return (basestate,)
@@ -1363,13 +1419,13 @@
     def __setstate(self, string, tzinfo):
         if len(string) != 6 or ord(string[0]) >= 24:
             raise TypeError("an integer is required")
-        self.__hour, self.__minute, self.__second, us1, us2, us3 = \
+        self._hour, self._minute, self._second, us1, us2, us3 = \
                                                             map(ord, string)
-        self.__microsecond = (((us1 << 8) | us2) << 8) | us3
+        self._microsecond = (((us1 << 8) | us2) << 8) | us3
         self._tzinfo = tzinfo
 
     def __reduce__(self):
-        return (time, self.__getstate())
+        return (time, self._getstate())
 
 _time_class = time  # so functions w/ args named "time" can get at the class
 
@@ -1378,9 +1434,11 @@
 time.resolution = timedelta(microseconds=1)
 
 class datetime(date):
+    """datetime(year, month, day[, hour[, minute[, second[, microsecond[,tzinfo]]]]])
 
-    # XXX needs docstrings
-    # See http://www.zope.org/Members/fdrake/DateTimeWiki/TimeZoneInfo
+    The year, month and day arguments are required. tzinfo may be None, or an
+    instance of a tzinfo subclass. The remaining arguments may be ints or longs.
+    """
 
     def __new__(cls, year, month=None, day=None, hour=0, minute=0, second=0,
                 microsecond=0, tzinfo=None):
@@ -1393,24 +1451,43 @@
         _check_time_fields(hour, minute, second, microsecond)
         self = date.__new__(cls, year, month, day)
         # XXX This duplicates __year, __month, __day for convenience :-(
-        self.__year = year
-        self.__month = month
-        self.__day = day
-        self.__hour = hour
-        self.__minute = minute
-        self.__second = second
-        self.__microsecond = microsecond
+        self._year = year
+        self._month = month
+        self._day = day
+        self._hour = hour
+        self._minute = minute
+        self._second = second
+        self._microsecond = microsecond
         self._tzinfo = tzinfo
         return self
 
     # Read-only field accessors
-    hour = property(lambda self: self.__hour, doc="hour (0-23)")
-    minute = property(lambda self: self.__minute, doc="minute (0-59)")
-    second = property(lambda self: self.__second, doc="second (0-59)")
-    microsecond = property(lambda self: self.__microsecond,
-                           doc="microsecond (0-999999)")
-    tzinfo = property(lambda self: self._tzinfo, doc="timezone info object")
+    @property
+    def hour(self):
+        """hour (0-23)"""
+        return self._hour
 
+    @property
+    def minute(self):
+        """minute (0-59)"""
+        return self._minute
+
+    @property
+    def second(self):
+        """second (0-59)"""
+        return self._second
+
+    @property
+    def microsecond(self):
+        """microsecond (0-999999)"""
+        return self._microsecond
+
+    @property
+    def tzinfo(self):
+        """timezone info object"""
+        return self._tzinfo
+
+    @classmethod
     def fromtimestamp(cls, t, tz=None):
         """Construct a datetime from a POSIX timestamp (like time.time()).
 
@@ -1438,7 +1515,6 @@
         if tz is not None:
             result = tz.fromutc(result)
         return result
-    fromtimestamp = classmethod(fromtimestamp)
 
     @classmethod
     def utcfromtimestamp(cls, t):
@@ -1462,18 +1538,19 @@
     # XXX uses gettimeofday on platforms that have it, but that isn't
     # XXX available from Python.  So now() may return different results
     # XXX across the implementations.
+    @classmethod
     def now(cls, tz=None):
         "Construct a datetime from time.time() and optional time zone info."
         t = _time.time()
         return cls.fromtimestamp(t, tz)
-    now = classmethod(now)
 
+    @classmethod
     def utcnow(cls):
         "Construct a UTC datetime from time.time()."
         t = _time.time()
         return cls.utcfromtimestamp(t)
-    utcnow = classmethod(utcnow)
 
+    @classmethod
     def combine(cls, date, time):
         "Construct a datetime from a given date and a given time."
         if not isinstance(date, _date_class):
@@ -1483,7 +1560,6 @@
         return cls(date.year, date.month, date.day,
                    time.hour, time.minute, time.second, time.microsecond,
                    time.tzinfo)
-    combine = classmethod(combine)
 
     def timetuple(self):
         "Return local time tuple compatible with time.localtime()."
@@ -1509,7 +1585,7 @@
 
     def date(self):
         "Return the date part."
-        return date(self.__year, self.__month, self.__day)
+        return date(self._year, self._month, self._day)
 
     def time(self):
         "Return the time part, with tzinfo None."
@@ -1569,8 +1645,8 @@
 
     def ctime(self):
         "Format a la ctime()."
-        t = tmxxx(self.__year, self.__month, self.__day, self.__hour,
-                  self.__minute, self.__second)
+        t = tmxxx(self._year, self._month, self._day, self._hour,
+                  self._minute, self._second)
         return t.ctime()
 
     def isoformat(self, sep='T'):
@@ -1585,10 +1661,10 @@
         Optional argument sep specifies the separator between date and
         time, default 'T'.
         """
-        s = ("%04d-%02d-%02d%c" % (self.__year, self.__month, self.__day,
+        s = ("%04d-%02d-%02d%c" % (self._year, self._month, self._day,
                                   sep) +
-                _format_time(self.__hour, self.__minute, self.__second,
-                             self.__microsecond))
+                _format_time(self._hour, self._minute, self._second,
+                             self._microsecond))
         off = self._utcoffset()
         if off is not None:
             if off < 0:
@@ -1601,13 +1677,13 @@
         return s
 
     def __repr__(self):
-        "Convert to formal string, for repr()."
-        L = [self.__year, self.__month, self.__day, # These are never zero
-             self.__hour, self.__minute, self.__second, self.__microsecond]
+        """Convert to formal string, for repr()."""
+        L = [self._year, self._month, self._day, # These are never zero
+             self._hour, self._minute, self._second, self._microsecond]
         if L[-1] == 0:
             del L[-1]
         if L[-1] == 0:
-            del L[-1]            
+            del L[-1]
         s = ", ".join(map(str, L))
         s = "%s(%s)" % ('datetime.' + self.__class__.__name__, s)
         if self._tzinfo is not None:
@@ -1680,7 +1756,7 @@
 
     def __eq__(self, other):
         if isinstance(other, datetime):
-            return self.__cmp(other) == 0
+            return self._cmp(other) == 0
         elif hasattr(other, "timetuple") and not isinstance(other, date):
             return NotImplemented
         else:
@@ -1688,7 +1764,7 @@
 
     def __ne__(self, other):
         if isinstance(other, datetime):
-            return self.__cmp(other) != 0
+            return self._cmp(other) != 0
         elif hasattr(other, "timetuple") and not isinstance(other, date):
             return NotImplemented
         else:
@@ -1696,7 +1772,7 @@
 
     def __le__(self, other):
         if isinstance(other, datetime):
-            return self.__cmp(other) <= 0
+            return self._cmp(other) <= 0
         elif hasattr(other, "timetuple") and not isinstance(other, date):
             return NotImplemented
         else:
@@ -1704,7 +1780,7 @@
 
     def __lt__(self, other):
         if isinstance(other, datetime):
-            return self.__cmp(other) < 0
+            return self._cmp(other) < 0
         elif hasattr(other, "timetuple") and not isinstance(other, date):
             return NotImplemented
         else:
@@ -1712,7 +1788,7 @@
 
     def __ge__(self, other):
         if isinstance(other, datetime):
-            return self.__cmp(other) >= 0
+            return self._cmp(other) >= 0
         elif hasattr(other, "timetuple") and not isinstance(other, date):
             return NotImplemented
         else:
@@ -1720,13 +1796,13 @@
 
     def __gt__(self, other):
         if isinstance(other, datetime):
-            return self.__cmp(other) > 0
+            return self._cmp(other) > 0
         elif hasattr(other, "timetuple") and not isinstance(other, date):
             return NotImplemented
         else:
             _cmperror(self, other)
 
-    def __cmp(self, other):
+    def _cmp(self, other):
         assert isinstance(other, datetime)
         mytz = self._tzinfo
         ottz = other._tzinfo
@@ -1742,12 +1818,12 @@
             base_compare = myoff == otoff
 
         if base_compare:
-            return cmp((self.__year, self.__month, self.__day,
-                        self.__hour, self.__minute, self.__second,
-                        self.__microsecond),
-                       (other.__year, other.__month, other.__day,
-                        other.__hour, other.__minute, other.__second,
-                        other.__microsecond))
+            return cmp((self._year, self._month, self._day,
+                        self._hour, self._minute, self._second,
+                        self._microsecond),
+                       (other._year, other._month, other._day,
+                        other._hour, other._minute, other._second,
+                        other._microsecond))
         if myoff is None or otoff is None:
             # XXX Buggy in 2.2.2.
             raise TypeError("cannot compare naive and aware datetimes")
@@ -1761,13 +1837,13 @@
         "Add a datetime and a timedelta."
         if not isinstance(other, timedelta):
             return NotImplemented
-        t = tmxxx(self.__year,
-                  self.__month,
-                  self.__day + other.days,
-                  self.__hour,
-                  self.__minute,
-                  self.__second + other.seconds,
-                  self.__microsecond + other.microseconds)
+        t = tmxxx(self._year,
+                  self._month,
+                  self._day + other.days,
+                  self._hour,
+                  self._minute,
+                  self._second + other.seconds,
+                  self._microsecond + other.microseconds)
         self._checkOverflow(t.year)
         result = datetime(t.year, t.month, t.day,
                                 t.hour, t.minute, t.second,
@@ -1785,11 +1861,11 @@
 
         days1 = self.toordinal()
         days2 = other.toordinal()
-        secs1 = self.__second + self.__minute * 60 + self.__hour * 3600
-        secs2 = other.__second + other.__minute * 60 + other.__hour * 3600
+        secs1 = self._second + self._minute * 60 + self._hour * 3600
+        secs2 = other._second + other._minute * 60 + other._hour * 3600
         base = timedelta(days1 - days2,
                          secs1 - secs2,
-                         self.__microsecond - other.__microsecond)
+                         self._microsecond - other._microsecond)
         if self._tzinfo is other._tzinfo:
             return base
         myoff = self._utcoffset()
@@ -1797,13 +1873,13 @@
         if myoff == otoff:
             return base
         if myoff is None or otoff is None:
-            raise TypeError, "cannot mix naive and timezone-aware time"
+            raise TypeError("cannot mix naive and timezone-aware time")
         return base + timedelta(minutes = otoff-myoff)
 
     def __hash__(self):
         tzoff = self._utcoffset()
         if tzoff is None:
-            return hash(self.__getstate()[0])
+            return hash(self._getstate()[0])
         days = _ymd2ord(self.year, self.month, self.day)
         seconds = self.hour * 3600 + (self.minute - tzoff) * 60 + self.second
         return hash(timedelta(days, seconds, self.microsecond))
@@ -1812,12 +1888,12 @@
 
     __safe_for_unpickling__ = True      # For Python 2.2
 
-    def __getstate(self):
-        yhi, ylo = divmod(self.__year, 256)
-        us2, us3 = divmod(self.__microsecond, 256)
+    def _getstate(self):
+        yhi, ylo = divmod(self._year, 256)
+        us2, us3 = divmod(self._microsecond, 256)
         us1, us2 = divmod(us2, 256)
-        basestate = ("%c" * 10) % (yhi, ylo, self.__month, self.__day,
-                                   self.__hour, self.__minute, self.__second,
+        basestate = ("%c" * 10) % (yhi, ylo, self._month, self._day,
+                                   self._hour, self._minute, self._second,
                                    us1, us2, us3)
         if self._tzinfo is None:
             return (basestate,)
@@ -1825,14 +1901,14 @@
             return (basestate, self._tzinfo)
 
     def __setstate(self, string, tzinfo):
-        (yhi, ylo, self.__month, self.__day, self.__hour,
-         self.__minute, self.__second, us1, us2, us3) = map(ord, string)
-        self.__year = yhi * 256 + ylo
-        self.__microsecond = (((us1 << 8) | us2) << 8) | us3
+        (yhi, ylo, self._month, self._day, self._hour,
+         self._minute, self._second, us1, us2, us3) = map(ord, string)
+        self._year = yhi * 256 + ylo
+        self._microsecond = (((us1 << 8) | us2) << 8) | us3
         self._tzinfo = tzinfo
 
     def __reduce__(self):
-        return (self.__class__, self.__getstate())
+        return (self.__class__, self._getstate())
 
 
 datetime.min = datetime(1, 1, 1)
@@ -2014,7 +2090,7 @@
 
 Because we know z.d said z was in daylight time (else [5] would have held and
 we would have stopped then), and we know z.d != z'.d (else [8] would have held
-and we we have stopped then), and there are only 2 possible values dst() can
+and we have stopped then), and there are only 2 possible values dst() can
 return in Eastern, it follows that z'.d must be 0 (which it is in the example,
 but the reasoning doesn't depend on the example -- it depends on there being
 two possible dst() outcomes, one zero and the other non-zero).  Therefore

From noreply at buildbot.pypy.org  Sun Jan 22 10:53:05 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 22 Jan 2012 10:53:05 +0100 (CET)
Subject: [pypy-commit] pypy stm: Add an argument to the callback invoked by
 stm_perform_transaction:
Message-ID: <20120122095305.5D298821FA@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stm
Changeset: r51626:b8a1df61795a
Date: 2012-01-22 10:51 +0100
http://bitbucket.org/pypy/pypy/changeset/b8a1df61795a/

Log:	Add an argument to the callback invoked by stm_perform_transaction:
	a retry counter starting at 0.

diff --git a/pypy/module/transaction/interp_transaction.py b/pypy/module/transaction/interp_transaction.py
--- a/pypy/module/transaction/interp_transaction.py
+++ b/pypy/module/transaction/interp_transaction.py
@@ -84,11 +84,18 @@
         self.w_callback = w_callback
         self.args = args
 
+    def register(self):
+        id = threadintf.thread_id()
+        state.pending_lists[id].append(self)
+
     def run(self):
         rstm.perform_transaction(Pending._run_in_transaction, Pending, self)
 
     @staticmethod
-    def _run_in_transaction(pending):
+    def _run_in_transaction(pending, retry_counter):
+        if retry_counter > 0:
+            self.register()   # retrying: will be done later, try others first
+            return
         if state.got_exception is not None:
             return   # return early if there is already a 'got_exception'
         try:
@@ -99,8 +106,7 @@
 
 
 def add(space, w_callback, __args__):
-    id = threadintf.thread_id()
-    state.pending_lists[id].append(Pending(w_callback, __args__))
+    Pending(w_callback, __args__).register()
 
 
 def add_list(new_pending_list):
diff --git a/pypy/rlib/rstm.py b/pypy/rlib/rstm.py
--- a/pypy/rlib/rstm.py
+++ b/pypy/rlib/rstm.py
@@ -10,13 +10,13 @@
 
 @specialize.memo()
 def _get_stm_callback(func, argcls):
-    def _stm_callback(llarg):
+    def _stm_callback(llarg, retry_counter):
         if we_are_translated():
             llarg = rffi.cast(rclass.OBJECTPTR, llarg)
             arg = cast_base_ptr_to_instance(argcls, llarg)
         else:
             arg = lltype.TLS.stm_callback_arg
-        res = func(arg)
+        res = func(arg, retry_counter)
         assert res is None
         return lltype.nullptr(rffi.VOIDP.TO)
     return _stm_callback
diff --git a/pypy/rlib/test/test_rstm.py b/pypy/rlib/test/test_rstm.py
--- a/pypy/rlib/test/test_rstm.py
+++ b/pypy/rlib/test/test_rstm.py
@@ -7,7 +7,7 @@
 class Arg(object):
     _alloc_nonmovable_ = True
 
-def setx(arg):
+def setx(arg, retry_counter):
     debug_print(arg.x)
     assert rstm.debug_get_state() == 1
     if arg.x == 303:
diff --git a/pypy/translator/stm/_rffi_stm.py b/pypy/translator/stm/_rffi_stm.py
--- a/pypy/translator/stm/_rffi_stm.py
+++ b/pypy/translator/stm/_rffi_stm.py
@@ -32,7 +32,7 @@
 stm_write_word = llexternal('stm_write_word', [SignedP, lltype.Signed],
                             lltype.Void)
 
-CALLBACK = lltype.Ptr(lltype.FuncType([rffi.VOIDP], rffi.VOIDP))
+CALLBACK = lltype.Ptr(lltype.FuncType([rffi.VOIDP, lltype.Signed], rffi.VOIDP))
 stm_perform_transaction = llexternal('stm_perform_transaction',
                                      [CALLBACK, rffi.VOIDP], rffi.VOIDP)
 
diff --git a/pypy/translator/stm/src_stm/et.c b/pypy/translator/stm/src_stm/et.c
--- a/pypy/translator/stm/src_stm/et.c
+++ b/pypy/translator/stm/src_stm/et.c
@@ -622,16 +622,20 @@
   return d->end_time;
 }
 
-void* stm_perform_transaction(void*(*callback)(void*), void *arg)
+void* stm_perform_transaction(void*(*callback)(void*, long), void *arg)
 {
   void *result;
   jmp_buf _jmpbuf;
+  volatile long v_counter = 0;
+  long counter;
   /* you need to call descriptor_init() before calling
      stm_perform_transaction() */
   assert(thread_descriptor != NULL_TX);
   setjmp(_jmpbuf);
   begin_transaction(&_jmpbuf);
-  result = callback(arg);
+  counter = v_counter;
+  v_counter = counter + 1;
+  result = callback(arg, counter);
   commit_transaction();
   return result;
 }
diff --git a/pypy/translator/stm/src_stm/et.h b/pypy/translator/stm/src_stm/et.h
--- a/pypy/translator/stm/src_stm/et.h
+++ b/pypy/translator/stm/src_stm/et.h
@@ -22,7 +22,7 @@
 
 void stm_descriptor_init(void);
 void stm_descriptor_done(void);
-void* stm_perform_transaction(void*(*)(void*), void*);
+void* stm_perform_transaction(void*(*)(void*, long), void*);
 long stm_read_word(long* addr);
 void stm_write_word(long* addr, long val);
 void stm_try_inevitable(STM_CCHARP1(why));
diff --git a/pypy/translator/stm/test/targetdemo.py b/pypy/translator/stm/test/targetdemo.py
--- a/pypy/translator/stm/test/targetdemo.py
+++ b/pypy/translator/stm/test/targetdemo.py
@@ -20,7 +20,7 @@
     _alloc_nonmovable_ = True
 
 
-def add_at_end_of_chained_list(arg):
+def add_at_end_of_chained_list(arg, retry_counter):
     node = arg.anchor
     value = arg.value
     x = Node(value)
@@ -52,7 +52,7 @@
     print "check ok!"
 
 
-def increment_done(arg):
+def increment_done(arg, retry_counter):
     print "thread done."
     glob.done += 1
 
diff --git a/pypy/translator/stm/test/test_funcgen.py b/pypy/translator/stm/test/test_funcgen.py
--- a/pypy/translator/stm/test/test_funcgen.py
+++ b/pypy/translator/stm/test/test_funcgen.py
@@ -93,7 +93,7 @@
     return a
 a_prebuilt = make_a_1()
 
-def _play_with_getfield(dummy_arg):
+def _play_with_getfield(dummy_arg, retry_counter):
     a = a_prebuilt
     assert a.x == -611
     assert a.c1 == '/'
@@ -106,7 +106,7 @@
     assert float(a.sb) == float(rs1b)
     return NULL
     
-def _play_with_setfields(dummy_arg):
+def _play_with_setfields(dummy_arg, retry_counter):
     a = a_prebuilt
     #
     a.x = 12871981
@@ -125,10 +125,10 @@
     a.sb = rs2b
     # read the values which have not been commited yet, but are local to the
     # transaction
-    _check_values_of_fields(dummy_arg)
+    _check_values_of_fields(dummy_arg, retry_counter)
     return NULL
 
-def _check_values_of_fields(dummy_arg):
+def _check_values_of_fields(dummy_arg, retry_counter):
     a = a_prebuilt
     assert a.x == 12871981
     assert a.c1 == '('
@@ -162,14 +162,14 @@
         array[i] = rffi.cast(lltype.typeOf(array).TO.OF, newvalues[i])
 change._annspecialcase_ = 'specialize:ll'
 
-def _play_with_getarrayitem(dummy_arg):
+def _play_with_getarrayitem(dummy_arg, retry_counter):
     check(prebuilt_array_signed, [1, 10, -1, -10, 42])
     check(prebuilt_array_char,   [chr(1), chr(10), chr(255),
                                   chr(246), chr(42)])
     return NULL
 
 
-def _play_with_setarrayitem_1(dummy_arg):
+def _play_with_setarrayitem_1(dummy_arg, retry_counter):
     change(prebuilt_array_signed, [500000, -10000000, 3])
     check(prebuilt_array_signed,  [500000, -10000000, 3, -10, 42])
     prebuilt_array_char[0] = 'A'
@@ -180,7 +180,7 @@
     check(prebuilt_array_char,    ['A', chr(10), chr(255), 'B', 'C'])
     return NULL
 
-def _play_with_setarrayitem_2(dummy_arg):
+def _play_with_setarrayitem_2(dummy_arg, retry_counter):
     check(prebuilt_array_char,    ['A', chr(10), chr(255), 'B', 'C'])
     prebuilt_array_char[1] = 'D'
     check(prebuilt_array_char,    ['A', 'D', chr(255), 'B', 'C'])
@@ -188,7 +188,7 @@
     check(prebuilt_array_char,    ['A', 'D', 'E', 'B', 'C'])
     return NULL
 
-def _play_with_setarrayitem_3(dummy_arg):
+def _play_with_setarrayitem_3(dummy_arg, retry_counter):
     check(prebuilt_array_char,    ['A', 'D', 'E', 'B', 'C'])
     return NULL
 
@@ -222,14 +222,14 @@
         array[i].y = rffi.cast(lltype.typeOf(array).TO.OF.y, newvalues2[i])
 change2._annspecialcase_ = 'specialize:ll'
 
-def _play_with_getinteriorfield(dummy_arg):
+def _play_with_getinteriorfield(dummy_arg, retry_counter):
     check2(prebuilt_array_signed_signed, [1, -1, -50], [10, 20, -30])
     check2(prebuilt_array_char_char, [chr(1), chr(255), chr(206)],
                                      [chr(10), chr(20), chr(226)])
     return NULL
 
 
-def _play_with_setinteriorfield_1(dummy_arg):
+def _play_with_setinteriorfield_1(dummy_arg, retry_counter):
     change2(prebuilt_array_signed_signed, [500000, -10000000], [102101202])
     check2(prebuilt_array_signed_signed, [500000, -10000000, -50],
                                          [102101202, 20, -30])
@@ -238,7 +238,7 @@
                                      ['b', chr(20), chr(226)])
     return NULL
 
-def _play_with_setinteriorfield_2(dummy_arg):
+def _play_with_setinteriorfield_2(dummy_arg, retry_counter):
     check2(prebuilt_array_signed_signed, [500000, -10000000, -50],
                                          [102101202, 20, -30])
     check2(prebuilt_array_char_char, ['a', chr(255), chr(206)],
@@ -253,7 +253,7 @@
 
     def test_getfield_all_sizes(self):
         def do_stm_getfield(argv):
-            _play_with_getfield(None)
+            _play_with_getfield(None, 0)
             return 0
         t, cbuilder = self.compile(do_stm_getfield)
         cbuilder.cmdexec('')
@@ -272,7 +272,7 @@
         def do_stm_getfield(argv):
             stm_descriptor_init()
             # we have a descriptor, but we don't call it in a transaction
-            _play_with_getfield(None)
+            _play_with_getfield(None, 0)
             stm_descriptor_done()
             return 0
         t, cbuilder = self.compile(do_stm_getfield)
@@ -280,7 +280,7 @@
 
     def test_setfield_all_sizes(self):
         def do_stm_setfield(argv):
-            _play_with_setfields(None)
+            _play_with_setfields(None, 0)
             return 0
         t, cbuilder = self.compile(do_stm_setfield)
         cbuilder.cmdexec('')
@@ -301,7 +301,7 @@
     def test_setfield_all_sizes_outside_transaction(self):
         def do_stm_setfield(argv):
             stm_descriptor_init()
-            _play_with_setfields(None)
+            _play_with_setfields(None, 0)
             stm_descriptor_done()
             return 0
         t, cbuilder = self.compile(do_stm_setfield)
@@ -309,7 +309,7 @@
 
     def test_getarrayitem_all_sizes(self):
         def do_stm_getarrayitem(argv):
-            _play_with_getarrayitem(None)
+            _play_with_getarrayitem(None, 0)
             return 0
         t, cbuilder = self.compile(do_stm_getarrayitem)
         cbuilder.cmdexec('')
@@ -327,9 +327,9 @@
 
     def test_setarrayitem_all_sizes(self):
         def do_stm_setarrayitem(argv):
-            _play_with_setarrayitem_1(None)
-            _play_with_setarrayitem_2(None)
-            _play_with_setarrayitem_3(None)
+            _play_with_setarrayitem_1(None, 0)
+            _play_with_setarrayitem_2(None, 0)
+            _play_with_setarrayitem_3(None, 0)
             return 0
         t, cbuilder = self.compile(do_stm_setarrayitem)
         cbuilder.cmdexec('')
@@ -351,7 +351,7 @@
 
     def test_getinteriorfield_all_sizes(self):
         def do_stm_getinteriorfield(argv):
-            _play_with_getinteriorfield(None)
+            _play_with_getinteriorfield(None, 0)
             return 0
         t, cbuilder = self.compile(do_stm_getinteriorfield)
         cbuilder.cmdexec('')
@@ -369,8 +369,8 @@
 
     def test_setinteriorfield_all_sizes(self):
         def do_stm_setinteriorfield(argv):
-            _play_with_setinteriorfield_1(None)
-            _play_with_setinteriorfield_2(None)
+            _play_with_setinteriorfield_1(None, 0)
+            _play_with_setinteriorfield_2(None, 0)
             return 0
         t, cbuilder = self.compile(do_stm_setinteriorfield)
         cbuilder.cmdexec('')
diff --git a/pypy/translator/stm/test/test_llstm.py b/pypy/translator/stm/test/test_llstm.py
--- a/pypy/translator/stm/test/test_llstm.py
+++ b/pypy/translator/stm/test/test_llstm.py
@@ -19,8 +19,9 @@
 rs1b = r_singlefloat(40.121)
 rs2b = r_singlefloat(-9e9)
 
-def callback1(a):
+def callback1(a, retry_counter):
     a = rffi.cast(lltype.Ptr(A), a)
+    assert retry_counter == a.y      # non-transactionally
     assert a.x == -611
     assert a.c1 == '/'
     assert a.c2 == '\\'
@@ -76,7 +77,7 @@
     assert a.y == 10
     lltype.free(a, flavor='raw')
 
-def callback2(a):
+def callback2(a, retry_counter):
     a = rffi.cast(lltype.Ptr(A), a)
     assert a.x == -611
     assert a.c1 == '&'
diff --git a/pypy/translator/stm/test/test_rffi_stm.py b/pypy/translator/stm/test/test_rffi_stm.py
--- a/pypy/translator/stm/test/test_rffi_stm.py
+++ b/pypy/translator/stm/test/test_rffi_stm.py
@@ -6,7 +6,7 @@
     stm_descriptor_done()
 
 def test_stm_perform_transaction():
-    def callback1(x):
+    def callback1(x, retry_counter):
         return lltype.nullptr(rffi.VOIDP.TO)
     stm_descriptor_init()
     stm_perform_transaction(llhelper(CALLBACK, callback1),
@@ -17,7 +17,8 @@
     A = lltype.Struct('A', ('x', lltype.Signed), ('y', lltype.Signed))
     a = lltype.malloc(A, immortal=True, flavor='raw')
     a.y = 0
-    def callback1(x):
+    def callback1(x, retry_counter):
+        assert retry_counter == a.y
         if a.y < 10:
             a.y += 1    # non-transactionally
             stm_abort_and_retry()
@@ -35,7 +36,8 @@
     a = lltype.malloc(A, immortal=True, flavor='raw')
     a.x = -611
     a.y = 0
-    def callback1(x):
+    def callback1(x, retry_counter):
+        assert retry_counter == a.y
         assert a.x == -611
         p = lltype.direct_fieldptr(a, 'x')
         p = rffi.cast(SignedP, p)
@@ -55,7 +57,7 @@
     assert a.x == 420
 
 def test_stm_debug_get_state():
-    def callback1(x):
+    def callback1(x, retry_counter):
         assert stm_debug_get_state() == 1
         stm_try_inevitable()
         assert stm_debug_get_state() == 2

From noreply at buildbot.pypy.org  Sun Jan 22 10:53:06 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 22 Jan 2012 10:53:06 +0100 (CET)
Subject: [pypy-commit] pypy stm: Fix test_interp_transaction.
Message-ID: <20120122095306.A0119821FA@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stm
Changeset: r51627:a18852e15aae
Date: 2012-01-22 10:52 +0100
http://bitbucket.org/pypy/pypy/changeset/a18852e15aae/

Log:	Fix test_interp_transaction.

diff --git a/pypy/module/transaction/test/test_interp_transaction.py b/pypy/module/transaction/test/test_interp_transaction.py
--- a/pypy/module/transaction/test/test_interp_transaction.py
+++ b/pypy/module/transaction/test/test_interp_transaction.py
@@ -3,8 +3,14 @@
 
 
 class FakeSpace:
-    def new_exception_class(self, name):
-        return "some error class"
+    def getbuiltinmodule(self, name):
+        assert name == 'transaction'
+        return 'transaction module'
+    def getattr(self, w_obj, w_name):
+        assert w_obj == 'transaction module'
+        return 'some stuff from the transaction module'
+    def wrap(self, x):
+        return 'wrapped stuff'
     def call_args(self, w_callback, args):
         w_callback(*args)
 

From noreply at buildbot.pypy.org  Sun Jan 22 10:58:15 2012
From: noreply at buildbot.pypy.org (rguillebert)
Date: Sun, 22 Jan 2012 10:58:15 +0100 (CET)
Subject: [pypy-commit] pypy py3k: Fixes the combined use of kwonly arguments
 and default parameters
Message-ID: <20120122095815.5BD38821FA@wyvern.cs.uni-duesseldorf.de>

Author: Romain Guillebert 
Branch: py3k
Changeset: r51628:99083d2bfef1
Date: 2012-01-22 10:57 +0100
http://bitbucket.org/pypy/pypy/changeset/99083d2bfef1/

Log:	Fixes the combined use of kwonly arguments and default parameters

diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -35,6 +35,9 @@
     def num_argnames(self):
         return len(self.argnames)
 
+    def num_kwonlyargnames(self):
+        return len(self.kwonlyargnames)
+
     def has_vararg(self):
         return self.varargname is not None
 
@@ -291,6 +294,7 @@
         # so all values coming from there can be assumed constant. It assumes
         # that the length of the defaults_w does not vary too much.
         co_argcount = signature.num_argnames() # expected formal arguments, without */**
+        co_kwonlyargcount = signature.num_kwonlyargnames()
         has_vararg = signature.has_vararg()
         has_kwarg = signature.has_kwarg()
         extravarargs = None
@@ -379,9 +383,9 @@
                     used_keywords[i] = True # mark as used
                     num_remainingkwds -= 1
         missing = 0
-        if input_argcount < co_argcount:
-            def_first = co_argcount - (0 if defaults_w is None else len(defaults_w))
-            for i in range(input_argcount, co_argcount):
+        if input_argcount < co_argcount + co_kwonlyargcount:
+            def_first = co_argcount + co_kwonlyargcount - (0 if defaults_w is None else len(defaults_w))
+            for i in range(input_argcount, co_argcount + co_kwonlyargcount):
                 if scope_w[i] is not None:
                     continue
                 defnum = i - def_first
@@ -393,6 +397,10 @@
                     # keyword arguments, which will be checked for below.
                     missing += 1
 
+        # TODO: Put a nice error message
+        #if co_kwonlyargcount:
+        #    assert co_kwonlyargcount == len(signature.kwonlyargnames)
+
         # collect extra keyword arguments into the **kwarg
         if has_kwarg:
             w_kwds = self.space.newdict()
@@ -423,7 +431,7 @@
                               co_argcount, has_vararg, has_kwarg,
                               defaults_w, missing)
 
-        return co_argcount + has_vararg + has_kwarg
+        return co_argcount + has_vararg + has_kwarg + co_kwonlyargcount
 
 
 
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -106,6 +106,7 @@
 
         if self.co_cellvars:
             argcount = self.co_argcount
+            argcount += self.co_kwonlyargcount
             assert argcount >= 0     # annotator hint
             if self.co_flags & CO_VARARGS:
                 argcount += 1
@@ -208,6 +209,8 @@
             return
         if len(self._args_as_cellvars) > 0:
             return
+        if self.co_kwonlyargcount > 0:
+            return
         if self.co_argcount > 0xff:
             return
 

From noreply at buildbot.pypy.org  Sun Jan 22 11:11:46 2012
From: noreply at buildbot.pypy.org (rguillebert)
Date: Sun, 22 Jan 2012 11:11:46 +0100 (CET)
Subject: [pypy-commit] pypy py3k: Add support for keyword arguments in the
 test suite, add a test for the order
Message-ID: <20120122101146.05355821FA@wyvern.cs.uni-duesseldorf.de>

Author: Romain Guillebert 
Branch: py3k
Changeset: r51629:2457af5955f1
Date: 2012-01-22 11:11 +0100
http://bitbucket.org/pypy/pypy/changeset/2457af5955f1/

Log:	Add support for keyword arguments in the test suite, add a test for
	the order of kwonly arguments

diff --git a/pypy/interpreter/test/test_interpreter.py b/pypy/interpreter/test/test_interpreter.py
--- a/pypy/interpreter/test/test_interpreter.py
+++ b/pypy/interpreter/test/test_interpreter.py
@@ -4,7 +4,7 @@
 
 class TestInterpreter: 
 
-    def codetest(self, source, functionname, args):
+    def codetest(self, source, functionname, args, kwargs={}):
         """Compile and run the given code string, and then call its function
         named by 'functionname' with arguments 'args'."""
         space = self.space
@@ -22,10 +22,11 @@
         code = space.unwrap(w_code)
         code.exec_code(space, w_glob, w_glob)
 
-        wrappedargs = [w(a) for a in args]
+        wrappedargs = w(args)
+        wrappedkwargs = w(kwargs)
         wrappedfunc = space.getitem(w_glob, w(functionname))
         try:
-            w_output = space.call_function(wrappedfunc, *wrappedargs)
+            w_output = space.call(wrappedfunc, wrappedargs, wrappedkwargs)
         except error.OperationError, e:
             #e.print_detailed_traceback(space)
             return '<<<%s>>>' % e.errorstr(space)
@@ -246,6 +247,13 @@
         """
         assert self.codetest(code, "f", [1, 2]) == (1, 2, 3, 4)
 
+    def test_kwonlyargs_order(self):
+        code = """ def f(a, b, *, c, d):
+            return a, b, c, d
+        """
+        assert self.codetest(code, "f", [1, 2], {"d" : 3, "c" : 4}) == (1, 2, 3, 4)
+
+
 
 class AppTestInterpreter: 
     def test_trivial(self):

From noreply at buildbot.pypy.org  Sun Jan 22 11:18:45 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 22 Jan 2012 11:18:45 +0100 (CET)
Subject: [pypy-commit] pypy default: fix newlines
Message-ID: <20120122101845.3DE6A821FA@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: 
Changeset: r51630:444a1d6d447d
Date: 2012-01-22 12:04 +0200
http://bitbucket.org/pypy/pypy/changeset/444a1d6d447d/

Log:	fix newlines

diff --git a/pypy/translator/c/src/profiling.c b/pypy/translator/c/src/profiling.c
--- a/pypy/translator/c/src/profiling.c
+++ b/pypy/translator/c/src/profiling.c
@@ -29,33 +29,33 @@
     profiling_setup = 0;
   }
 }
-
-#elif defined(_WIN32)
-#include 
-
-DWORD_PTR base_affinity_mask;
-int profiling_setup = 0;
-
-void pypy_setup_profiling() { 
-    if (!profiling_setup) {
-        DWORD_PTR affinity_mask, system_affinity_mask;
-        GetProcessAffinityMask(GetCurrentProcess(),
-            &base_affinity_mask, &system_affinity_mask);
-        affinity_mask = 1;
-        /* Pick one cpu allowed by the system */
-        if (system_affinity_mask)
-            while ((affinity_mask & system_affinity_mask) == 0)
-                affinity_mask <<= 1;
-        SetProcessAffinityMask(GetCurrentProcess(), affinity_mask);
-        profiling_setup = 1;
-    }
-}
-
-void pypy_teardown_profiling() {
-    if (profiling_setup) {
-        SetProcessAffinityMask(GetCurrentProcess(), base_affinity_mask);
-        profiling_setup = 0;
-    }
+
+#elif defined(_WIN32)
+#include 
+
+DWORD_PTR base_affinity_mask;
+int profiling_setup = 0;
+
+void pypy_setup_profiling() { 
+    if (!profiling_setup) {
+        DWORD_PTR affinity_mask, system_affinity_mask;
+        GetProcessAffinityMask(GetCurrentProcess(),
+            &base_affinity_mask, &system_affinity_mask);
+        affinity_mask = 1;
+        /* Pick one cpu allowed by the system */
+        if (system_affinity_mask)
+            while ((affinity_mask & system_affinity_mask) == 0)
+                affinity_mask <<= 1;
+        SetProcessAffinityMask(GetCurrentProcess(), affinity_mask);
+        profiling_setup = 1;
+    }
+}
+
+void pypy_teardown_profiling() {
+    if (profiling_setup) {
+        SetProcessAffinityMask(GetCurrentProcess(), base_affinity_mask);
+        profiling_setup = 0;
+    }
 }
 
 #else

From noreply at buildbot.pypy.org  Sun Jan 22 11:18:46 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 22 Jan 2012 11:18:46 +0100 (CET)
Subject: [pypy-commit] pypy default: merge
Message-ID: <20120122101846.7CA9C821FA@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: 
Changeset: r51631:eeb977421a88
Date: 2012-01-22 12:18 +0200
http://bitbucket.org/pypy/pypy/changeset/eeb977421a88/

Log:	merge

diff --git a/lib_pypy/datetime.py b/lib_pypy/datetime.py
--- a/lib_pypy/datetime.py
+++ b/lib_pypy/datetime.py
@@ -13,7 +13,7 @@
 Sources for time zone and DST data: http://www.twinsun.com/tz/tz-link.htm
 
 This was originally copied from the sandbox of the CPython CVS repository.
-Thanks to Tim Peters for suggesting using it. 
+Thanks to Tim Peters for suggesting using it.
 """
 
 import time as _time
@@ -271,6 +271,8 @@
     raise ValueError("%s()=%d, must be in -1439..1439" % (name, offset))
 
 def _check_date_fields(year, month, day):
+    if not isinstance(year, (int, long)):
+        raise TypeError('int expected')
     if not MINYEAR <= year <= MAXYEAR:
         raise ValueError('year must be in %d..%d' % (MINYEAR, MAXYEAR), year)
     if not 1 <= month <= 12:
@@ -280,6 +282,8 @@
         raise ValueError('day must be in 1..%d' % dim, day)
 
 def _check_time_fields(hour, minute, second, microsecond):
+    if not isinstance(hour, (int, long)):
+        raise TypeError('int expected')
     if not 0 <= hour <= 23:
         raise ValueError('hour must be in 0..23', hour)
     if not 0 <= minute <= 59:
@@ -543,61 +547,76 @@
 
         self = object.__new__(cls)
 
-        self.__days = d
-        self.__seconds = s
-        self.__microseconds = us
+        self._days = d
+        self._seconds = s
+        self._microseconds = us
         if abs(d) > 999999999:
             raise OverflowError("timedelta # of days is too large: %d" % d)
 
         return self
 
     def __repr__(self):
-        if self.__microseconds:
+        if self._microseconds:
             return "%s(%d, %d, %d)" % ('datetime.' + self.__class__.__name__,
-                                       self.__days,
-                                       self.__seconds,
-                                       self.__microseconds)
-        if self.__seconds:
+                                       self._days,
+                                       self._seconds,
+                                       self._microseconds)
+        if self._seconds:
             return "%s(%d, %d)" % ('datetime.' + self.__class__.__name__,
-                                   self.__days,
-                                   self.__seconds)
-        return "%s(%d)" % ('datetime.' + self.__class__.__name__, self.__days)
+                                   self._days,
+                                   self._seconds)
+        return "%s(%d)" % ('datetime.' + self.__class__.__name__, self._days)
 
     def __str__(self):
-        mm, ss = divmod(self.__seconds, 60)
+        mm, ss = divmod(self._seconds, 60)
         hh, mm = divmod(mm, 60)
         s = "%d:%02d:%02d" % (hh, mm, ss)
-        if self.__days:
+        if self._days:
             def plural(n):
                 return n, abs(n) != 1 and "s" or ""
-            s = ("%d day%s, " % plural(self.__days)) + s
-        if self.__microseconds:
-            s = s + ".%06d" % self.__microseconds
+            s = ("%d day%s, " % plural(self._days)) + s
+        if self._microseconds:
+            s = s + ".%06d" % self._microseconds
         return s
 
-    days = property(lambda self: self.__days, doc="days")
-    seconds = property(lambda self: self.__seconds, doc="seconds")
-    microseconds = property(lambda self: self.__microseconds,
-                            doc="microseconds")
-
     def total_seconds(self):
         return ((self.days * 86400 + self.seconds) * 10**6
                 + self.microseconds) / 1e6
 
+    # Read-only field accessors
+    @property
+    def days(self):
+        """days"""
+        return self._days
+
+    @property
+    def seconds(self):
+        """seconds"""
+        return self._seconds
+
+    @property
+    def microseconds(self):
+        """microseconds"""
+        return self._microseconds
+
     def __add__(self, other):
         if isinstance(other, timedelta):
             # for CPython compatibility, we cannot use
             # our __class__ here, but need a real timedelta
-            return timedelta(self.__days + other.__days,
-                             self.__seconds + other.__seconds,
-                             self.__microseconds + other.__microseconds)
+            return timedelta(self._days + other._days,
+                             self._seconds + other._seconds,
+                             self._microseconds + other._microseconds)
         return NotImplemented
 
     __radd__ = __add__
 
     def __sub__(self, other):
         if isinstance(other, timedelta):
-            return self + -other
+            # for CPython compatibility, we cannot use
+            # our __class__ here, but need a real timedelta
+            return timedelta(self._days - other._days,
+                             self._seconds - other._seconds,
+                             self._microseconds - other._microseconds)
         return NotImplemented
 
     def __rsub__(self, other):
@@ -606,17 +625,17 @@
         return NotImplemented
 
     def __neg__(self):
-            # for CPython compatibility, we cannot use
-            # our __class__ here, but need a real timedelta
-            return timedelta(-self.__days,
-                             -self.__seconds,
-                             -self.__microseconds)
+        # for CPython compatibility, we cannot use
+        # our __class__ here, but need a real timedelta
+        return timedelta(-self._days,
+                         -self._seconds,
+                         -self._microseconds)
 
     def __pos__(self):
         return self
 
     def __abs__(self):
-        if self.__days < 0:
+        if self._days < 0:
             return -self
         else:
             return self
@@ -625,81 +644,81 @@
         if isinstance(other, (int, long)):
             # for CPython compatibility, we cannot use
             # our __class__ here, but need a real timedelta
-            return timedelta(self.__days * other,
-                             self.__seconds * other,
-                             self.__microseconds * other)
+            return timedelta(self._days * other,
+                             self._seconds * other,
+                             self._microseconds * other)
         return NotImplemented
 
     __rmul__ = __mul__
 
     def __div__(self, other):
         if isinstance(other, (int, long)):
-            usec = ((self.__days * (24*3600L) + self.__seconds) * 1000000 +
-                    self.__microseconds)
+            usec = ((self._days * (24*3600L) + self._seconds) * 1000000 +
+                    self._microseconds)
             return timedelta(0, 0, usec // other)
         return NotImplemented
 
     __floordiv__ = __div__
 
-    # Comparisons.
+    # Comparisons of timedelta objects with other.
 
     def __eq__(self, other):
         if isinstance(other, timedelta):
-            return self.__cmp(other) == 0
+            return self._cmp(other) == 0
         else:
             return False
 
     def __ne__(self, other):
         if isinstance(other, timedelta):
-            return self.__cmp(other) != 0
+            return self._cmp(other) != 0
         else:
             return True
 
     def __le__(self, other):
         if isinstance(other, timedelta):
-            return self.__cmp(other) <= 0
+            return self._cmp(other) <= 0
         else:
             _cmperror(self, other)
 
     def __lt__(self, other):
         if isinstance(other, timedelta):
-            return self.__cmp(other) < 0
+            return self._cmp(other) < 0
         else:
             _cmperror(self, other)
 
     def __ge__(self, other):
         if isinstance(other, timedelta):
-            return self.__cmp(other) >= 0
+            return self._cmp(other) >= 0
         else:
             _cmperror(self, other)
 
     def __gt__(self, other):
         if isinstance(other, timedelta):
-            return self.__cmp(other) > 0
+            return self._cmp(other) > 0
         else:
             _cmperror(self, other)
 
-    def __cmp(self, other):
+    def _cmp(self, other):
         assert isinstance(other, timedelta)
-        return cmp(self.__getstate(), other.__getstate())
+        return cmp(self._getstate(), other._getstate())
 
     def __hash__(self):
-        return hash(self.__getstate())
+        return hash(self._getstate())
 
     def __nonzero__(self):
-        return (self.__days != 0 or
-                self.__seconds != 0 or
-                self.__microseconds != 0)
+        return (self._days != 0 or
+                self._seconds != 0 or
+                self._microseconds != 0)
 
     # Pickle support.
 
     __safe_for_unpickling__ = True      # For Python 2.2
 
-    def __getstate(self):
-        return (self.__days, self.__seconds, self.__microseconds)
+    def _getstate(self):
+        return (self._days, self._seconds, self._microseconds)
 
     def __reduce__(self):
-        return (self.__class__, self.__getstate())
+        return (self.__class__, self._getstate())
 
 timedelta.min = timedelta(-999999999)
 timedelta.max = timedelta(days=999999999, hours=23, minutes=59, seconds=59,
@@ -749,25 +768,26 @@
             return self
         _check_date_fields(year, month, day)
         self = object.__new__(cls)
-        self.__year = year
-        self.__month = month
-        self.__day = day
+        self._year = year
+        self._month = month
+        self._day = day
         return self
 
     # Additional constructors
 
+    @classmethod
     def fromtimestamp(cls, t):
         "Construct a date from a POSIX timestamp (like time.time())."
         y, m, d, hh, mm, ss, weekday, jday, dst = _time.localtime(t)
         return cls(y, m, d)
-    fromtimestamp = classmethod(fromtimestamp)
 
+    @classmethod
     def today(cls):
         "Construct a date from time.time()."
         t = _time.time()
         return cls.fromtimestamp(t)
-    today = classmethod(today)
 
+    @classmethod
     def fromordinal(cls, n):
         """Contruct a date from a proleptic Gregorian ordinal.
 
@@ -776,16 +796,24 @@
         """
         y, m, d = _ord2ymd(n)
         return cls(y, m, d)
-    fromordinal = classmethod(fromordinal)
 
     # Conversions to string
 
     def __repr__(self):
-        "Convert to formal string, for repr()."
+        """Convert to formal string, for repr().
+
+        >>> dt = datetime(2010, 1, 1)
+        >>> repr(dt)
+        'datetime.datetime(2010, 1, 1, 0, 0)'
+
+        >>> dt = datetime(2010, 1, 1, tzinfo=timezone.utc)
+        >>> repr(dt)
+        'datetime.datetime(2010, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)'
+        """
         return "%s(%d, %d, %d)" % ('datetime.' + self.__class__.__name__,
-                                   self.__year,
-                                   self.__month,
-                                   self.__day)
+                                   self._year,
+                                   self._month,
+                                   self._day)
     # XXX These shouldn't depend on time.localtime(), because that
     # clips the usable dates to [1970 .. 2038).  At least ctime() is
     # easily done without using strftime() -- that's better too because
@@ -793,12 +821,20 @@
 
     def ctime(self):
         "Format a la ctime()."
-        return tmxxx(self.__year, self.__month, self.__day).ctime()
+        return tmxxx(self._year, self._month, self._day).ctime()
 
     def strftime(self, fmt):
         "Format using strftime()."
         return _wrap_strftime(self, fmt, self.timetuple())
 
+    def __format__(self, fmt):
+        if not isinstance(fmt, (str, unicode)):
+            raise ValueError("__format__ excepts str or unicode, not %s" %
+                             fmt.__class__.__name__)
+        if len(fmt) != 0:
+            return self.strftime(fmt)
+        return str(self)
+
     def isoformat(self):
         """Return the date formatted according to ISO.
 
@@ -808,29 +844,31 @@
         - http://www.w3.org/TR/NOTE-datetime
         - http://www.cl.cam.ac.uk/~mgk25/iso-time.html
         """
-        return "%04d-%02d-%02d" % (self.__year, self.__month, self.__day)
+        return "%04d-%02d-%02d" % (self._year, self._month, self._day)
 
     __str__ = isoformat
 
-    def __format__(self, format):
-        if not isinstance(format, (str, unicode)):
-            raise ValueError("__format__ excepts str or unicode, not %s" %
-                             format.__class__.__name__)
-        if not format:
-            return str(self)
-        return self.strftime(format)
+    # Read-only field accessors
+    @property
+    def year(self):
+        """year (1-9999)"""
+        return self._year
 
-    # Read-only field accessors
-    year = property(lambda self: self.__year,
-                    doc="year (%d-%d)" % (MINYEAR, MAXYEAR))
-    month = property(lambda self: self.__month, doc="month (1-12)")
-    day = property(lambda self: self.__day, doc="day (1-31)")
+    @property
+    def month(self):
+        """month (1-12)"""
+        return self._month
+
+    @property
+    def day(self):
+        """day (1-31)"""
+        return self._day
 
     # Standard conversions, __cmp__, __hash__ (and helpers)
 
     def timetuple(self):
         "Return local time tuple compatible with time.localtime()."
-        return _build_struct_time(self.__year, self.__month, self.__day,
+        return _build_struct_time(self._year, self._month, self._day,
                                   0, 0, 0, -1)
 
     def toordinal(self):
@@ -839,24 +877,24 @@
         January 1 of year 1 is day 1.  Only the year, month and day values
         contribute to the result.
         """
-        return _ymd2ord(self.__year, self.__month, self.__day)
+        return _ymd2ord(self._year, self._month, self._day)
 
     def replace(self, year=None, month=None, day=None):
         """Return a new date with new values for the specified fields."""
         if year is None:
-            year = self.__year
+            year = self._year
         if month is None:
-            month = self.__month
+            month = self._month
         if day is None:
-            day = self.__day
+            day = self._day
         _check_date_fields(year, month, day)
         return date(year, month, day)
 
-    # Comparisons.
+    # Comparisons of date objects with other.
 
     def __eq__(self, other):
         if isinstance(other, date):
-            return self.__cmp(other) == 0
+            return self._cmp(other) == 0
         elif hasattr(other, "timetuple"):
             return NotImplemented
         else:
@@ -864,7 +902,7 @@
 
     def __ne__(self, other):
         if isinstance(other, date):
-            return self.__cmp(other) != 0
+            return self._cmp(other) != 0
         elif hasattr(other, "timetuple"):
             return NotImplemented
         else:
@@ -872,7 +910,7 @@
 
     def __le__(self, other):
         if isinstance(other, date):
-            return self.__cmp(other) <= 0
+            return self._cmp(other) <= 0
         elif hasattr(other, "timetuple"):
             return NotImplemented
         else:
@@ -880,7 +918,7 @@
 
     def __lt__(self, other):
         if isinstance(other, date):
-            return self.__cmp(other) < 0
+            return self._cmp(other) < 0
         elif hasattr(other, "timetuple"):
             return NotImplemented
         else:
@@ -888,7 +926,7 @@
 
     def __ge__(self, other):
         if isinstance(other, date):
-            return self.__cmp(other) >= 0
+            return self._cmp(other) >= 0
         elif hasattr(other, "timetuple"):
             return NotImplemented
         else:
@@ -896,21 +934,21 @@
 
     def __gt__(self, other):
         if isinstance(other, date):
-            return self.__cmp(other) > 0
+            return self._cmp(other) > 0
         elif hasattr(other, "timetuple"):
             return NotImplemented
         else:
             _cmperror(self, other)
 
-    def __cmp(self, other):
+    def _cmp(self, other):
         assert isinstance(other, date)
-        y, m, d = self.__year, self.__month, self.__day
-        y2, m2, d2 = other.__year, other.__month, other.__day
+        y, m, d = self._year, self._month, self._day
+        y2, m2, d2 = other._year, other._month, other._day
         return cmp((y, m, d), (y2, m2, d2))
 
     def __hash__(self):
         "Hash."
-        return hash(self.__getstate())
+        return hash(self._getstate())
 
     # Computations
 
@@ -922,9 +960,9 @@
     def __add__(self, other):
         "Add a date to a timedelta."
         if isinstance(other, timedelta):
-            t = tmxxx(self.__year,
-                      self.__month,
-                      self.__day + other.days)
+            t = tmxxx(self._year,
+                      self._month,
+                      self._day + other.days)
             self._checkOverflow(t.year)
             result = date(t.year, t.month, t.day)
             return result
@@ -966,9 +1004,9 @@
         ISO calendar algorithm taken from
         http://www.phys.uu.nl/~vgent/calendar/isocalendar.htm
         """
-        year = self.__year
+        year = self._year
         week1monday = _isoweek1monday(year)
-        today = _ymd2ord(self.__year, self.__month, self.__day)
+        today = _ymd2ord(self._year, self._month, self._day)
         # Internally, week and day have origin 0
         week, day = divmod(today - week1monday, 7)
         if week < 0:
@@ -985,18 +1023,18 @@
 
     __safe_for_unpickling__ = True      # For Python 2.2
 
-    def __getstate(self):
-        yhi, ylo = divmod(self.__year, 256)
-        return ("%c%c%c%c" % (yhi, ylo, self.__month, self.__day), )
+    def _getstate(self):
+        yhi, ylo = divmod(self._year, 256)
+        return ("%c%c%c%c" % (yhi, ylo, self._month, self._day), )
 
     def __setstate(self, string):
         if len(string) != 4 or not (1 <= ord(string[2]) <= 12):
             raise TypeError("not enough arguments")
-        yhi, ylo, self.__month, self.__day = map(ord, string)
-        self.__year = yhi * 256 + ylo
+        yhi, ylo, self._month, self._day = map(ord, string)
+        self._year = yhi * 256 + ylo
 
     def __reduce__(self):
-        return (self.__class__, self.__getstate())
+        return (self.__class__, self._getstate())
 
 _date_class = date  # so functions w/ args named "date" can get at the class
 
@@ -1118,62 +1156,80 @@
             return self
         _check_tzinfo_arg(tzinfo)
         _check_time_fields(hour, minute, second, microsecond)
-        self.__hour = hour
-        self.__minute = minute
-        self.__second = second
-        self.__microsecond = microsecond
+        self._hour = hour
+        self._minute = minute
+        self._second = second
+        self._microsecond = microsecond
         self._tzinfo = tzinfo
         return self
 
     # Read-only field accessors
-    hour = property(lambda self: self.__hour, doc="hour (0-23)")
-    minute = property(lambda self: self.__minute, doc="minute (0-59)")
-    second = property(lambda self: self.__second, doc="second (0-59)")
-    microsecond = property(lambda self: self.__microsecond,
-                           doc="microsecond (0-999999)")
-    tzinfo = property(lambda self: self._tzinfo, doc="timezone info object")
+    @property
+    def hour(self):
+        """hour (0-23)"""
+        return self._hour
+
+    @property
+    def minute(self):
+        """minute (0-59)"""
+        return self._minute
+
+    @property
+    def second(self):
+        """second (0-59)"""
+        return self._second
+
+    @property
+    def microsecond(self):
+        """microsecond (0-999999)"""
+        return self._microsecond
+
+    @property
+    def tzinfo(self):
+        """timezone info object"""
+        return self._tzinfo
 
     # Standard conversions, __hash__ (and helpers)
 
-    # Comparisons.
+    # Comparisons of time objects with other.
 
     def __eq__(self, other):
         if isinstance(other, time):
-            return self.__cmp(other) == 0
+            return self._cmp(other) == 0
         else:
             return False
 
     def __ne__(self, other):
         if isinstance(other, time):
-            return self.__cmp(other) != 0
+            return self._cmp(other) != 0
         else:
             return True
 
     def __le__(self, other):
         if isinstance(other, time):
-            return self.__cmp(other) <= 0
+            return self._cmp(other) <= 0
         else:
             _cmperror(self, other)
 
     def __lt__(self, other):
         if isinstance(other, time):
-            return self.__cmp(other) < 0
+            return self._cmp(other) < 0
         else:
             _cmperror(self, other)
 
     def __ge__(self, other):
         if isinstance(other, time):
-            return self.__cmp(other) >= 0
+            return self._cmp(other) >= 0
         else:
             _cmperror(self, other)
 
     def __gt__(self, other):
         if isinstance(other, time):
-            return self.__cmp(other) > 0
+            return self._cmp(other) > 0
         else:
             _cmperror(self, other)
 
-    def __cmp(self, other):
+    def _cmp(self, other):
         assert isinstance(other, time)
         mytz = self._tzinfo
         ottz = other._tzinfo
@@ -1187,23 +1243,23 @@
             base_compare = myoff == otoff
 
         if base_compare:
-            return cmp((self.__hour, self.__minute, self.__second,
-                        self.__microsecond),
-                       (other.__hour, other.__minute, other.__second,
-                        other.__microsecond))
+            return cmp((self._hour, self._minute, self._second,
+                        self._microsecond),
+                       (other._hour, other._minute, other._second,
+                        other._microsecond))
         if myoff is None or otoff is None:
             # XXX Buggy in 2.2.2.
             raise TypeError("cannot compare naive and aware times")
-        myhhmm = self.__hour * 60 + self.__minute - myoff
-        othhmm = other.__hour * 60 + other.__minute - otoff
-        return cmp((myhhmm, self.__second, self.__microsecond),
-                   (othhmm, other.__second, other.__microsecond))
+        myhhmm = self._hour * 60 + self._minute - myoff
+        othhmm = other._hour * 60 + other._minute - otoff
+        return cmp((myhhmm, self._second, self._microsecond),
+                   (othhmm, other._second, other._microsecond))
 
     def __hash__(self):
         """Hash."""
         tzoff = self._utcoffset()
         if not tzoff: # zero or None
-            return hash(self.__getstate()[0])
+            return hash(self._getstate()[0])
         h, m = divmod(self.hour * 60 + self.minute - tzoff, 60)
         if 0 <= h < 24:
             return hash(time(h, m, self.second, self.microsecond))
@@ -1227,14 +1283,14 @@
 
     def __repr__(self):
         """Convert to formal string, for repr()."""
-        if self.__microsecond != 0:
-            s = ", %d, %d" % (self.__second, self.__microsecond)
-        elif self.__second != 0:
-            s = ", %d" % self.__second
+        if self._microsecond != 0:
+            s = ", %d, %d" % (self._second, self._microsecond)
+        elif self._second != 0:
+            s = ", %d" % self._second
         else:
             s = ""
         s= "%s(%d, %d%s)" % ('datetime.' + self.__class__.__name__,
-                             self.__hour, self.__minute, s)
+                             self._hour, self._minute, s)
         if self._tzinfo is not None:
             assert s[-1:] == ")"
             s = s[:-1] + ", tzinfo=%r" % self._tzinfo + ")"
@@ -1246,8 +1302,8 @@
         This is 'HH:MM:SS.mmmmmm+zz:zz', or 'HH:MM:SS+zz:zz' if
         self.microsecond == 0.
         """
-        s = _format_time(self.__hour, self.__minute, self.__second,
-                         self.__microsecond)
+        s = _format_time(self._hour, self._minute, self._second,
+                         self._microsecond)
         tz = self._tzstr()
         if tz:
             s += tz
@@ -1255,14 +1311,6 @@
 
     __str__ = isoformat
 
-    def __format__(self, format):
-        if not isinstance(format, (str, unicode)):
-            raise ValueError("__format__ excepts str or unicode, not %s" %
-                             format.__class__.__name__)
-        if not format:
-            return str(self)
-        return self.strftime(format)
-
     def strftime(self, fmt):
         """Format using strftime().  The date part of the timestamp passed
         to underlying strftime should not be used.
@@ -1270,10 +1318,18 @@
         # The year must be >= 1900 else Python's strftime implementation
         # can raise a bogus exception.
         timetuple = (1900, 1, 1,
-                     self.__hour, self.__minute, self.__second,
+                     self._hour, self._minute, self._second,
                      0, 1, -1)
         return _wrap_strftime(self, fmt, timetuple)
 
+    def __format__(self, fmt):
+        if not isinstance(fmt, (str, unicode)):
+            raise ValueError("__format__ excepts str or unicode, not %s" %
+                             fmt.__class__.__name__)
+        if len(fmt) != 0:
+            return self.strftime(fmt)
+        return str(self)
+
     # Timezone functions
 
     def utcoffset(self):
@@ -1350,10 +1406,10 @@
 
     __safe_for_unpickling__ = True      # For Python 2.2
 
-    def __getstate(self):
-        us2, us3 = divmod(self.__microsecond, 256)
+    def _getstate(self):
+        us2, us3 = divmod(self._microsecond, 256)
         us1, us2 = divmod(us2, 256)
-        basestate = ("%c" * 6) % (self.__hour, self.__minute, self.__second,
+        basestate = ("%c" * 6) % (self._hour, self._minute, self._second,
                                   us1, us2, us3)
         if self._tzinfo is None:
             return (basestate,)
@@ -1363,13 +1419,13 @@
     def __setstate(self, string, tzinfo):
         if len(string) != 6 or ord(string[0]) >= 24:
             raise TypeError("an integer is required")
-        self.__hour, self.__minute, self.__second, us1, us2, us3 = \
+        self._hour, self._minute, self._second, us1, us2, us3 = \
                                                             map(ord, string)
-        self.__microsecond = (((us1 << 8) | us2) << 8) | us3
+        self._microsecond = (((us1 << 8) | us2) << 8) | us3
         self._tzinfo = tzinfo
 
     def __reduce__(self):
-        return (time, self.__getstate())
+        return (time, self._getstate())
 
 _time_class = time  # so functions w/ args named "time" can get at the class
 
@@ -1378,9 +1434,11 @@
 time.resolution = timedelta(microseconds=1)
 
 class datetime(date):
+    """datetime(year, month, day[, hour[, minute[, second[, microsecond[,tzinfo]]]]])
 
-    # XXX needs docstrings
-    # See http://www.zope.org/Members/fdrake/DateTimeWiki/TimeZoneInfo
+    The year, month and day arguments are required. tzinfo may be None, or an
+    instance of a tzinfo subclass. The remaining arguments may be ints or longs.
+    """
 
     def __new__(cls, year, month=None, day=None, hour=0, minute=0, second=0,
                 microsecond=0, tzinfo=None):
@@ -1393,24 +1451,43 @@
         _check_time_fields(hour, minute, second, microsecond)
         self = date.__new__(cls, year, month, day)
         # XXX This duplicates __year, __month, __day for convenience :-(
-        self.__year = year
-        self.__month = month
-        self.__day = day
-        self.__hour = hour
-        self.__minute = minute
-        self.__second = second
-        self.__microsecond = microsecond
+        self._year = year
+        self._month = month
+        self._day = day
+        self._hour = hour
+        self._minute = minute
+        self._second = second
+        self._microsecond = microsecond
         self._tzinfo = tzinfo
         return self
 
     # Read-only field accessors
-    hour = property(lambda self: self.__hour, doc="hour (0-23)")
-    minute = property(lambda self: self.__minute, doc="minute (0-59)")
-    second = property(lambda self: self.__second, doc="second (0-59)")
-    microsecond = property(lambda self: self.__microsecond,
-                           doc="microsecond (0-999999)")
-    tzinfo = property(lambda self: self._tzinfo, doc="timezone info object")
+    @property
+    def hour(self):
+        """hour (0-23)"""
+        return self._hour
 
+    @property
+    def minute(self):
+        """minute (0-59)"""
+        return self._minute
+
+    @property
+    def second(self):
+        """second (0-59)"""
+        return self._second
+
+    @property
+    def microsecond(self):
+        """microsecond (0-999999)"""
+        return self._microsecond
+
+    @property
+    def tzinfo(self):
+        """timezone info object"""
+        return self._tzinfo
+
+    @classmethod
     def fromtimestamp(cls, t, tz=None):
         """Construct a datetime from a POSIX timestamp (like time.time()).
 
@@ -1438,7 +1515,6 @@
         if tz is not None:
             result = tz.fromutc(result)
         return result
-    fromtimestamp = classmethod(fromtimestamp)
 
     @classmethod
     def utcfromtimestamp(cls, t):
@@ -1462,18 +1538,19 @@
     # XXX uses gettimeofday on platforms that have it, but that isn't
     # XXX available from Python.  So now() may return different results
     # XXX across the implementations.
+    @classmethod
     def now(cls, tz=None):
         "Construct a datetime from time.time() and optional time zone info."
         t = _time.time()
         return cls.fromtimestamp(t, tz)
-    now = classmethod(now)
 
+    @classmethod
     def utcnow(cls):
         "Construct a UTC datetime from time.time()."
         t = _time.time()
         return cls.utcfromtimestamp(t)
-    utcnow = classmethod(utcnow)
 
+    @classmethod
     def combine(cls, date, time):
         "Construct a datetime from a given date and a given time."
         if not isinstance(date, _date_class):
@@ -1483,7 +1560,6 @@
         return cls(date.year, date.month, date.day,
                    time.hour, time.minute, time.second, time.microsecond,
                    time.tzinfo)
-    combine = classmethod(combine)
 
     def timetuple(self):
         "Return local time tuple compatible with time.localtime()."
@@ -1509,7 +1585,7 @@
 
     def date(self):
         "Return the date part."
-        return date(self.__year, self.__month, self.__day)
+        return date(self._year, self._month, self._day)
 
     def time(self):
         "Return the time part, with tzinfo None."
@@ -1569,8 +1645,8 @@
 
     def ctime(self):
         "Format a la ctime()."
-        t = tmxxx(self.__year, self.__month, self.__day, self.__hour,
-                  self.__minute, self.__second)
+        t = tmxxx(self._year, self._month, self._day, self._hour,
+                  self._minute, self._second)
         return t.ctime()
 
     def isoformat(self, sep='T'):
@@ -1585,10 +1661,10 @@
         Optional argument sep specifies the separator between date and
         time, default 'T'.
         """
-        s = ("%04d-%02d-%02d%c" % (self.__year, self.__month, self.__day,
+        s = ("%04d-%02d-%02d%c" % (self._year, self._month, self._day,
                                   sep) +
-                _format_time(self.__hour, self.__minute, self.__second,
-                             self.__microsecond))
+                _format_time(self._hour, self._minute, self._second,
+                             self._microsecond))
         off = self._utcoffset()
         if off is not None:
             if off < 0:
@@ -1601,13 +1677,13 @@
         return s
 
     def __repr__(self):
-        "Convert to formal string, for repr()."
-        L = [self.__year, self.__month, self.__day, # These are never zero
-             self.__hour, self.__minute, self.__second, self.__microsecond]
+        """Convert to formal string, for repr()."""
+        L = [self._year, self._month, self._day, # These are never zero
+             self._hour, self._minute, self._second, self._microsecond]
         if L[-1] == 0:
             del L[-1]
         if L[-1] == 0:
-            del L[-1]            
+            del L[-1]
         s = ", ".join(map(str, L))
         s = "%s(%s)" % ('datetime.' + self.__class__.__name__, s)
         if self._tzinfo is not None:
@@ -1680,7 +1756,7 @@
 
     def __eq__(self, other):
         if isinstance(other, datetime):
-            return self.__cmp(other) == 0
+            return self._cmp(other) == 0
         elif hasattr(other, "timetuple") and not isinstance(other, date):
             return NotImplemented
         else:
@@ -1688,7 +1764,7 @@
 
     def __ne__(self, other):
         if isinstance(other, datetime):
-            return self.__cmp(other) != 0
+            return self._cmp(other) != 0
         elif hasattr(other, "timetuple") and not isinstance(other, date):
             return NotImplemented
         else:
@@ -1696,7 +1772,7 @@
 
     def __le__(self, other):
         if isinstance(other, datetime):
-            return self.__cmp(other) <= 0
+            return self._cmp(other) <= 0
         elif hasattr(other, "timetuple") and not isinstance(other, date):
             return NotImplemented
         else:
@@ -1704,7 +1780,7 @@
 
     def __lt__(self, other):
         if isinstance(other, datetime):
-            return self.__cmp(other) < 0
+            return self._cmp(other) < 0
         elif hasattr(other, "timetuple") and not isinstance(other, date):
             return NotImplemented
         else:
@@ -1712,7 +1788,7 @@
 
     def __ge__(self, other):
         if isinstance(other, datetime):
-            return self.__cmp(other) >= 0
+            return self._cmp(other) >= 0
         elif hasattr(other, "timetuple") and not isinstance(other, date):
             return NotImplemented
         else:
@@ -1720,13 +1796,13 @@
 
     def __gt__(self, other):
         if isinstance(other, datetime):
-            return self.__cmp(other) > 0
+            return self._cmp(other) > 0
         elif hasattr(other, "timetuple") and not isinstance(other, date):
             return NotImplemented
         else:
             _cmperror(self, other)
 
-    def __cmp(self, other):
+    def _cmp(self, other):
         assert isinstance(other, datetime)
         mytz = self._tzinfo
         ottz = other._tzinfo
@@ -1742,12 +1818,12 @@
             base_compare = myoff == otoff
 
         if base_compare:
-            return cmp((self.__year, self.__month, self.__day,
-                        self.__hour, self.__minute, self.__second,
-                        self.__microsecond),
-                       (other.__year, other.__month, other.__day,
-                        other.__hour, other.__minute, other.__second,
-                        other.__microsecond))
+            return cmp((self._year, self._month, self._day,
+                        self._hour, self._minute, self._second,
+                        self._microsecond),
+                       (other._year, other._month, other._day,
+                        other._hour, other._minute, other._second,
+                        other._microsecond))
         if myoff is None or otoff is None:
             # XXX Buggy in 2.2.2.
             raise TypeError("cannot compare naive and aware datetimes")
@@ -1761,13 +1837,13 @@
         "Add a datetime and a timedelta."
         if not isinstance(other, timedelta):
             return NotImplemented
-        t = tmxxx(self.__year,
-                  self.__month,
-                  self.__day + other.days,
-                  self.__hour,
-                  self.__minute,
-                  self.__second + other.seconds,
-                  self.__microsecond + other.microseconds)
+        t = tmxxx(self._year,
+                  self._month,
+                  self._day + other.days,
+                  self._hour,
+                  self._minute,
+                  self._second + other.seconds,
+                  self._microsecond + other.microseconds)
         self._checkOverflow(t.year)
         result = datetime(t.year, t.month, t.day,
                                 t.hour, t.minute, t.second,
@@ -1785,11 +1861,11 @@
 
         days1 = self.toordinal()
         days2 = other.toordinal()
-        secs1 = self.__second + self.__minute * 60 + self.__hour * 3600
-        secs2 = other.__second + other.__minute * 60 + other.__hour * 3600
+        secs1 = self._second + self._minute * 60 + self._hour * 3600
+        secs2 = other._second + other._minute * 60 + other._hour * 3600
         base = timedelta(days1 - days2,
                          secs1 - secs2,
-                         self.__microsecond - other.__microsecond)
+                         self._microsecond - other._microsecond)
         if self._tzinfo is other._tzinfo:
             return base
         myoff = self._utcoffset()
@@ -1797,13 +1873,13 @@
         if myoff == otoff:
             return base
         if myoff is None or otoff is None:
-            raise TypeError, "cannot mix naive and timezone-aware time"
+            raise TypeError("cannot mix naive and timezone-aware time")
         return base + timedelta(minutes = otoff-myoff)
 
     def __hash__(self):
         tzoff = self._utcoffset()
         if tzoff is None:
-            return hash(self.__getstate()[0])
+            return hash(self._getstate()[0])
         days = _ymd2ord(self.year, self.month, self.day)
         seconds = self.hour * 3600 + (self.minute - tzoff) * 60 + self.second
         return hash(timedelta(days, seconds, self.microsecond))
@@ -1812,12 +1888,12 @@
 
     __safe_for_unpickling__ = True      # For Python 2.2
 
-    def __getstate(self):
-        yhi, ylo = divmod(self.__year, 256)
-        us2, us3 = divmod(self.__microsecond, 256)
+    def _getstate(self):
+        yhi, ylo = divmod(self._year, 256)
+        us2, us3 = divmod(self._microsecond, 256)
         us1, us2 = divmod(us2, 256)
-        basestate = ("%c" * 10) % (yhi, ylo, self.__month, self.__day,
-                                   self.__hour, self.__minute, self.__second,
+        basestate = ("%c" * 10) % (yhi, ylo, self._month, self._day,
+                                   self._hour, self._minute, self._second,
                                    us1, us2, us3)
         if self._tzinfo is None:
             return (basestate,)
@@ -1825,14 +1901,14 @@
             return (basestate, self._tzinfo)
 
     def __setstate(self, string, tzinfo):
-        (yhi, ylo, self.__month, self.__day, self.__hour,
-         self.__minute, self.__second, us1, us2, us3) = map(ord, string)
-        self.__year = yhi * 256 + ylo
-        self.__microsecond = (((us1 << 8) | us2) << 8) | us3
+        (yhi, ylo, self._month, self._day, self._hour,
+         self._minute, self._second, us1, us2, us3) = map(ord, string)
+        self._year = yhi * 256 + ylo
+        self._microsecond = (((us1 << 8) | us2) << 8) | us3
         self._tzinfo = tzinfo
 
     def __reduce__(self):
-        return (self.__class__, self.__getstate())
+        return (self.__class__, self._getstate())
 
 
 datetime.min = datetime(1, 1, 1)
@@ -2014,7 +2090,7 @@
 
 Because we know z.d said z was in daylight time (else [5] would have held and
 we would have stopped then), and we know z.d != z'.d (else [8] would have held
-and we we have stopped then), and there are only 2 possible values dst() can
+and we have stopped then), and there are only 2 possible values dst() can
 return in Eastern, it follows that z'.d must be 0 (which it is in the example,
 but the reasoning doesn't depend on the example -- it depends on there being
 two possible dst() outcomes, one zero and the other non-zero).  Therefore

From noreply at buildbot.pypy.org  Sun Jan 22 11:29:32 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 22 Jan 2012 11:29:32 +0100 (CET)
Subject: [pypy-commit] pypy stm: Oups.
Message-ID: <20120122102932.52A1D821FA@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stm
Changeset: r51632:4ae4d737bd18
Date: 2012-01-22 11:16 +0100
http://bitbucket.org/pypy/pypy/changeset/4ae4d737bd18/

Log:	Oups.

diff --git a/pypy/module/transaction/interp_transaction.py b/pypy/module/transaction/interp_transaction.py
--- a/pypy/module/transaction/interp_transaction.py
+++ b/pypy/module/transaction/interp_transaction.py
@@ -94,7 +94,7 @@
     @staticmethod
     def _run_in_transaction(pending, retry_counter):
         if retry_counter > 0:
-            self.register()   # retrying: will be done later, try others first
+            pending.register() # retrying: will be done later, try others first
             return
         if state.got_exception is not None:
             return   # return early if there is already a 'got_exception'

From noreply at buildbot.pypy.org  Sun Jan 22 11:29:33 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 22 Jan 2012 11:29:33 +0100 (CET)
Subject: [pypy-commit] pypy stm: Add an explicit fifo queue implementation,
	instead of using
Message-ID: <20120122102933.90B8A821FA@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stm
Changeset: r51633:5ab80750fca1
Date: 2012-01-22 11:17 +0100
http://bitbucket.org/pypy/pypy/changeset/5ab80750fca1/

Log:	Add an explicit fifo queue implementation, instead of using
	list.pop(0).

diff --git a/pypy/module/transaction/fifo.py b/pypy/module/transaction/fifo.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/transaction/fifo.py
@@ -0,0 +1,34 @@
+
+class Fifo(object):
+    def __init__(self):
+        self.first = None
+        self.last = None
+
+    def append(self, newitem):
+        newitem.next = None
+        if self.last is None:
+            self.first = newitem
+        else:
+            self.last.next = newitem
+        self.last = newitem
+
+    def is_empty(self):
+        assert (self.first is None) == (self.last is None)
+        return self.first is None
+
+    def popleft(self):
+        item = self.first
+        self.first = item.next
+        if self.first is None:
+            self.last = None
+        return item
+
+    def steal(self, otherfifo):
+        if otherfifo.last is not None:
+            if self.last is None:
+                self.first = otherfifo.first
+            else:
+                self.last.next = otherfifo.first
+            self.last = otherfifo.last
+            otherfifo.first = None
+            otherfifo.last = None
diff --git a/pypy/module/transaction/interp_transaction.py b/pypy/module/transaction/interp_transaction.py
--- a/pypy/module/transaction/interp_transaction.py
+++ b/pypy/module/transaction/interp_transaction.py
@@ -1,6 +1,7 @@
 from pypy.interpreter.error import OperationError
 from pypy.interpreter.gateway import unwrap_spec
 from pypy.module.transaction import threadintf
+from pypy.module.transaction.fifo import Fifo
 from pypy.rlib import rstm
 
 
@@ -13,7 +14,7 @@
         self.__dict__.clear()
         self.running = False
         self.num_threads = NUM_THREADS_DEFAULT
-        self.pending = []
+        self.pending = Fifo()
         self.pending_lists = {0: self.pending}
         self.ll_lock = threadintf.null_ll_lock
         self.ll_no_tasks_pending_lock = threadintf.null_ll_lock
@@ -110,24 +111,23 @@
 
 
 def add_list(new_pending_list):
-    if len(new_pending_list) == 0:
+    if new_pending_list.is_empty():
         return
-    was_empty = len(state.pending) == 0
-    state.pending += new_pending_list
-    del new_pending_list[:]
+    was_empty = state.pending.is_empty()
+    state.pending.steal(new_pending_list)
     if was_empty:
         state.unlock_no_tasks_pending()
 
 
 def _run_thread():
     state.lock()
-    my_pending_list = []
+    my_pending_list = Fifo()
     my_thread_id = threadintf.thread_id()
     state.pending_lists[my_thread_id] = my_pending_list
     rstm.descriptor_init()
     #
     while True:
-        if len(state.pending) == 0:
+        if state.pending.is_empty():
             assert state.is_locked_no_tasks_pending()
             state.num_waiting_threads += 1
             if state.num_waiting_threads == state.num_threads:
@@ -143,8 +143,8 @@
             if state.finished:
                 break
         else:
-            pending = state.pending.pop(0)
-            if len(state.pending) == 0:
+            pending = state.pending.popleft()
+            if state.pending.is_empty():
                 state.lock_no_tasks_pending()
             state.unlock()
             pending.run()
@@ -164,7 +164,7 @@
             state.w_error,
             space.wrap("recursive invocation of transaction.run()"))
     assert not state.is_locked_no_tasks_pending()
-    if len(state.pending) == 0:
+    if state.pending.is_empty():
         return
     state.num_waiting_threads = 0
     state.finished = False
@@ -177,7 +177,7 @@
     state.lock_unfinished()  # wait for all threads to finish
     #
     assert state.num_waiting_threads == 0
-    assert len(state.pending) == 0
+    assert state.pending.is_empty()
     assert state.pending_lists.keys() == [state.main_thread_id]
     assert not state.is_locked_no_tasks_pending()
     state.running = False
diff --git a/pypy/module/transaction/test/test_fifo.py b/pypy/module/transaction/test/test_fifo.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/transaction/test/test_fifo.py
@@ -0,0 +1,40 @@
+from pypy.module.transaction.fifo import Fifo
+
+class Item:
+    def __init__(self, value):
+        self.value = value
+
+def test_one_item():
+    f = Fifo()
+    assert f.is_empty()
+    f.append(Item(123))
+    assert not f.is_empty()
+    item = f.popleft()
+    assert f.is_empty()
+    assert item.value == 123
+
+def test_three_items():
+    f = Fifo()
+    for i in [10, 20, 30]:
+        f.append(Item(i))
+        assert not f.is_empty()
+    for i in [10, 20, 30]:
+        assert not f.is_empty()
+        item = f.popleft()
+        assert item.value == i
+    assert f.is_empty()
+
+def test_steal():
+    for n1 in range(3):
+        for n2 in range(3):
+            f1 = Fifo()
+            f2 = Fifo()
+            for i in range(n1): f1.append(Item(10 + i))
+            for i in range(n2): f2.append(Item(20 + i))
+            f1.steal(f2)
+            assert f2.is_empty()
+            for x in range(10, 10+n1) + range(20, 20+n2):
+                assert not f1.is_empty()
+                item = f1.popleft()
+                assert item.value == x
+            assert f1.is_empty()

From noreply at buildbot.pypy.org  Sun Jan 22 11:29:34 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 22 Jan 2012 11:29:34 +0100 (CET)
Subject: [pypy-commit] pypy stm: Hacks to make checkmodule() work.
Message-ID: <20120122102934.CB7C4821FA@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stm
Changeset: r51634:69e39e76c0d2
Date: 2012-01-22 11:29 +0100
http://bitbucket.org/pypy/pypy/changeset/69e39e76c0d2/

Log:	Hacks to make checkmodule() work.

diff --git a/pypy/module/transaction/__init__.py b/pypy/module/transaction/__init__.py
--- a/pypy/module/transaction/__init__.py
+++ b/pypy/module/transaction/__init__.py
@@ -18,3 +18,8 @@
     def startup(self, space):
         from pypy.module.transaction import interp_transaction
         interp_transaction.state.startup(space)
+
+    def translating_for_checkmodule(self, space):
+        from pypy.module.transaction import interp_transaction
+        interp_transaction.state.space = space
+        interp_transaction.state._freeze_ = lambda: None   # hack!
diff --git a/pypy/module/transaction/interp_transaction.py b/pypy/module/transaction/interp_transaction.py
--- a/pypy/module/transaction/interp_transaction.py
+++ b/pypy/module/transaction/interp_transaction.py
@@ -19,6 +19,8 @@
         self.ll_lock = threadintf.null_ll_lock
         self.ll_no_tasks_pending_lock = threadintf.null_ll_lock
         self.ll_unfinished_lock = threadintf.null_ll_lock
+        self.main_thread_id = 0
+        self.w_error = None
 
     def startup(self, space):
         self.space = space
diff --git a/pypy/objspace/fake/checkmodule.py b/pypy/objspace/fake/checkmodule.py
--- a/pypy/objspace/fake/checkmodule.py
+++ b/pypy/objspace/fake/checkmodule.py
@@ -10,5 +10,7 @@
     module = mod.Module(space, W_Root())
     for name in module.loaders:
         module._load_lazily(space, name)
+    if hasattr(module, 'translating_for_checkmodule'):
+        module.translating_for_checkmodule(space)
     #
     space.translates(**{'translation.list_comprehension_operations':True})

From noreply at buildbot.pypy.org  Sun Jan 22 11:56:45 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 22 Jan 2012 11:56:45 +0100 (CET)
Subject: [pypy-commit] pypy stm: print the numeric reason when aborting.
Message-ID: <20120122105645.57BD1821FA@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stm
Changeset: r51635:3c073f6abf1a
Date: 2012-01-22 11:56 +0100
http://bitbucket.org/pypy/pypy/changeset/3c073f6abf1a/

Log:	print the numeric reason when aborting.

diff --git a/pypy/translator/stm/src_stm/et.c b/pypy/translator/stm/src_stm/et.c
--- a/pypy/translator/stm/src_stm/et.c
+++ b/pypy/translator/stm/src_stm/et.c
@@ -271,8 +271,9 @@
   d->num_aborts[reason]++;
 #ifdef RPY_STM_DEBUG_PRINT
   PYPY_DEBUG_START("stm-abort");
-  if (PYPY_HAVE_DEBUG_PRINTS) fprintf(PYPY_DEBUG_FILE, "thread %lx aborting\n",
-                                      (long)pthread_self());
+  if (PYPY_HAVE_DEBUG_PRINTS)
+      fprintf(PYPY_DEBUG_FILE, "thread %lx aborting %d\n",
+                               (long)pthread_self(), reason);
   PYPY_DEBUG_STOP("stm-abort");
 #endif
   tx_restart(d);

From noreply at buildbot.pypy.org  Sun Jan 22 12:06:39 2012
From: noreply at buildbot.pypy.org (rguillebert)
Date: Sun, 22 Jan 2012 12:06:39 +0100 (CET)
Subject: [pypy-commit] pypy py3k: The test was wrong...
Message-ID: <20120122110639.D3F05821FA@wyvern.cs.uni-duesseldorf.de>

Author: Romain Guillebert 
Branch: py3k
Changeset: r51636:0b136e560f89
Date: 2012-01-22 12:05 +0100
http://bitbucket.org/pypy/pypy/changeset/0b136e560f89/

Log:	The test was wrong...

diff --git a/pypy/interpreter/test/test_interpreter.py b/pypy/interpreter/test/test_interpreter.py
--- a/pypy/interpreter/test/test_interpreter.py
+++ b/pypy/interpreter/test/test_interpreter.py
@@ -251,7 +251,7 @@
         code = """ def f(a, b, *, c, d):
             return a, b, c, d
         """
-        assert self.codetest(code, "f", [1, 2], {"d" : 3, "c" : 4}) == (1, 2, 3, 4)
+        assert self.codetest(code, "f", [1, 2], {"d" : 4, "c" : 3}) == (1, 2, 3, 4)
 
 
 

From noreply at buildbot.pypy.org  Sun Jan 22 12:09:59 2012
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sun, 22 Jan 2012 12:09:59 +0100 (CET)
Subject: [pypy-commit] pypy py3k: Easy fixes in test_compiler
Message-ID: <20120122110959.29806821FA@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: py3k
Changeset: r51637:52eaf273d8ed
Date: 2012-01-21 22:36 +0100
http://bitbucket.org/pypy/pypy/changeset/52eaf273d8ed/

Log:	Easy fixes in test_compiler

diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -226,8 +226,7 @@
     def test_funccalls(self):
         decl = py.code.Source("""
             def f(*args, **kwds):
-                kwds = kwds.items()
-                kwds.sort()
+                kwds = sorted(kwds.items())
                 return list(args) + kwds
         """)
         decl = str(decl) + '\n'
@@ -328,7 +327,7 @@
                     from __foo__.bar import x
             try:
                 A().m()
-            except ImportError, e:
+            except ImportError as e:
                 msg = str(e)
             ''', "msg", "No module named __foo__")
 
@@ -519,8 +518,8 @@
         """, 'x', [True, 3, 4, 6]
 
     def test_type_of_constants(self):
-        yield self.simple_test, "x=[0, 0L]", 'type(x[1])', long
-        yield self.simple_test, "x=[(1,0), (1,0L)]", 'type(x[1][1])', long
+        yield self.simple_test, "x=[0, 0.]", 'type(x[1])', float
+        yield self.simple_test, "x=[(1,0), (1,0.)]", 'type(x[1][1])', float
         yield self.simple_test, "x=['2?-', '2?-']", 'id(x[0])==id(x[1])', True
 
     def test_pprint(self):
@@ -646,17 +645,15 @@
                 #Indexing
                 for key, value in self.reference.items():
                     self.assertEqual(d[key], value)
-                knownkey = self.other.keys()[0]
+                knownkey = next(iter(self.other))
                 self.failUnlessRaises(KeyError, lambda:d[knownkey])
                 #len
                 self.assertEqual(len(p), 0)
                 self.assertEqual(len(d), len(self.reference))
                 #has_key
                 for k in self.reference:
-                    self.assert_(d.has_key(k))
                     self.assert_(k in d)
                 for k in self.other:
-                    self.failIf(d.has_key(k))
                     self.failIf(k in d)
                 #cmp
                 self.assertEqual(cmp(p,p), 0)
diff --git a/pypy/interpreter/pyparser/test/expressions.py b/pypy/interpreter/pyparser/test/expressions.py
--- a/pypy/interpreter/pyparser/test/expressions.py
+++ b/pypy/interpreter/pyparser/test/expressions.py
@@ -4,11 +4,11 @@
 
 constants = [
     "0",
+    "00",
     "7",
     "-3",
-    "053",
+    "0o53",
     "0x18",
-    "14L",
     "1.0",
     "3.9",
     "-3.6",

From noreply at buildbot.pypy.org  Sun Jan 22 12:10:00 2012
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sun, 22 Jan 2012 12:10:00 +0100 (CET)
Subject: [pypy-commit] pypy py3k: One more fix for test_compiler
Message-ID: <20120122111000.67318821FA@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: py3k
Changeset: r51638:e27203d55679
Date: 2012-01-21 22:39 +0100
http://bitbucket.org/pypy/pypy/changeset/e27203d55679/

Log:	One more fix for test_compiler

diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -588,7 +588,7 @@
         w_iterable = self.popvalue()
         items = self.space.fixedview(w_iterable)
         itemcount = len(items)
-        if right < itemcount:
+        if right > itemcount:
             count = left + right
             if count == 1:
                 plural = ''

From noreply at buildbot.pypy.org  Sun Jan 22 12:10:01 2012
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sun, 22 Jan 2012 12:10:01 +0100 (CET)
Subject: [pypy-commit] pypy py3k: The exception handler target "except
 ValueError as exc" was always compiled as a global variable. Test and fix.
Message-ID: <20120122111001.A2D7B821FA@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: py3k
Changeset: r51639:e325e4d3227a
Date: 2012-01-22 12:02 +0100
http://bitbucket.org/pypy/pypy/changeset/e325e4d3227a/

Log:	The exception handler target "except ValueError as exc" was always
	compiled as a global variable. Test and fix.

diff --git a/pypy/interpreter/astcompiler/symtable.py b/pypy/interpreter/astcompiler/symtable.py
--- a/pypy/interpreter/astcompiler/symtable.py
+++ b/pypy/interpreter/astcompiler/symtable.py
@@ -417,6 +417,11 @@
     def visit_alias(self, alias):
         self._visit_alias(alias)
 
+    def visit_ExceptHandler(self, handler):
+        if handler.name:
+            self.note_symbol(handler.name, SYM_ASSIGNED)
+        ast.GenericASTVisitor.visit_ExceptHandler(self, handler)
+
     def visit_Yield(self, yie):
         self.scope.note_yield(yie)
         ast.GenericASTVisitor.visit_Yield(self, yie)
diff --git a/pypy/interpreter/astcompiler/test/test_symtable.py b/pypy/interpreter/astcompiler/test/test_symtable.py
--- a/pypy/interpreter/astcompiler/test/test_symtable.py
+++ b/pypy/interpreter/astcompiler/test/test_symtable.py
@@ -142,6 +142,10 @@
         scp = self.func_scope("def f(): x")
         assert scp.lookup("x") == symtable.SCOPE_GLOBAL_IMPLICIT
 
+    def test_exception_variable(self):
+        scp = self.mod_scope("try: pass\nexcept ValueError as e: pass")
+        assert scp.lookup("e") == symtable.SCOPE_LOCAL
+
     def test_nested_scopes(self):
         def nested_scope(*bodies):
             names = enumerate("f" + string.ascii_letters)

From noreply at buildbot.pypy.org  Sun Jan 22 12:10:02 2012
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sun, 22 Jan 2012 12:10:02 +0100 (CET)
Subject: [pypy-commit] pypy py3k: Fixes in test_interpreter
Message-ID: <20120122111002.DE1F4821FA@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: py3k
Changeset: r51640:4ce0e0c33cd2
Date: 2012-01-22 12:04 +0100
http://bitbucket.org/pypy/pypy/changeset/4ce0e0c33cd2/

Log:	Fixes in test_interpreter

diff --git a/pypy/interpreter/test/test_interpreter.py b/pypy/interpreter/test/test_interpreter.py
--- a/pypy/interpreter/test/test_interpreter.py
+++ b/pypy/interpreter/test/test_interpreter.py
@@ -38,7 +38,7 @@
                 def f():
                     try:
                         raise Exception()
-                    except Exception, e:
+                    except Exception as e:
                         return 1
                     return 2
             ''', 'f', [])
@@ -48,8 +48,8 @@
         x = self.codetest('''
             def f():
                 try:
-                    raise Exception, 1
-                except Exception, e:
+                    raise Exception(1)
+                except Exception as e:
                     return e.args[0]
             ''', 'f', [])
         assert x == 1
@@ -83,7 +83,7 @@
                     z = 0
                     try:
                         "x"+1
-                    except TypeError, e:
+                    except TypeError as e:
                         z = 5
                         raise e
                 except TypeError:
@@ -97,7 +97,7 @@
                     z = 0
                     try:
                         z = 1//v
-                    except ZeroDivisionError, e:
+                    except ZeroDivisionError as e:
                         z = "infinite result"
                     return z
                 '''
@@ -295,20 +295,23 @@
                 self.data.append((type(x), x))
         sys.stdout = out = Out()
         try:
-            print(unichr(0xa2))
-            assert out.data == [(unicode, unichr(0xa2)), (str, "\n")]
+            print(chr(0xa2))
+            assert out.data == [(str, chr(0xa2)), (str, "\n")]
             out.data = []
             out.encoding = "cp424"     # ignored!
-            print(unichr(0xa2))
-            assert out.data == [(unicode, unichr(0xa2)), (str, "\n")]
+            print(chr(0xa2))
+            assert out.data == [(str, chr(0xa2)), (str, "\n")]
             del out.data[:]
             del out.encoding
-            print "foo\t", "bar\n", "trick", "baz\n"  # softspace handling
-            assert out.data == [(unicode, "foo\t"),
-                                (unicode, "bar\n"),
-                                (unicode, "trick"),
+            # we don't care about softspace anymore
+            print("foo\t", "bar\n", "trick", "baz\n")
+            assert out.data == [(str, "foo\t"),
                                 (str, " "),
-                                (unicode, "baz\n"),
+                                (str, "bar\n"),
+                                (str, " "),
+                                (str, "trick"),
+                                (str, " "),
+                                (str, "baz\n"),
                                 (str, "\n")]
         finally:
             sys.stdout = save

From noreply at buildbot.pypy.org  Sun Jan 22 12:22:34 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 22 Jan 2012 12:22:34 +0100 (CET)
Subject: [pypy-commit] pypy default: hg backout c5d041657831: I will move it
 to the stdobjspace instead, 
Message-ID: <20120122112234.8F7BE821FA@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r51641:937fb53e76ac
Date: 2012-01-22 12:19 +0100
http://bitbucket.org/pypy/pypy/changeset/937fb53e76ac/

Log:	hg backout c5d041657831: I will move it to the stdobjspace instead,
	where it can be special-cased before we do all the "isinf()" and
	"isnan()" checks.

diff --git a/pypy/rpython/lltypesystem/module/ll_math.py b/pypy/rpython/lltypesystem/module/ll_math.py
--- a/pypy/rpython/lltypesystem/module/ll_math.py
+++ b/pypy/rpython/lltypesystem/module/ll_math.py
@@ -292,10 +292,6 @@
     # deal directly with IEEE specials, to cope with problems on various
     # platforms whose semantics don't exactly match C99
 
-    if y == 2.0:
-        return x * x     # this is always a correct answer, and is relatively
-                         # common in user programs.
-
     if isnan(y):
         if x == 1.0:
             return 1.0   # 1**Nan = 1

From noreply at buildbot.pypy.org  Sun Jan 22 12:22:35 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 22 Jan 2012 12:22:35 +0100 (CET)
Subject: [pypy-commit] pypy default: Special-case "x ** 2" here instead.
Message-ID: <20120122112235.CD675821FA@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r51642:f1fb181c13a3
Date: 2012-01-22 12:22 +0100
http://bitbucket.org/pypy/pypy/changeset/f1fb181c13a3/

Log:	Special-case "x ** 2" here instead.

diff --git a/pypy/objspace/std/floatobject.py b/pypy/objspace/std/floatobject.py
--- a/pypy/objspace/std/floatobject.py
+++ b/pypy/objspace/std/floatobject.py
@@ -443,6 +443,8 @@
     y = w_float2.floatval
 
     # Sort out special cases here instead of relying on pow()
+    if y == 2.0:                      # special case for performance:
+        return W_FloatObject(x * x)   # x * x is always correct
     if y == 0.0:
         # x**0 is 1, even 0**0
         return W_FloatObject(1.0)

From noreply at buildbot.pypy.org  Sun Jan 22 12:31:00 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 22 Jan 2012 12:31:00 +0100 (CET)
Subject: [pypy-commit] pypy default: Write a test_pypy_c checking that "f **
 2" gets us just a float_mul
Message-ID: <20120122113100.2D5EF821FA@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r51643:e82f6594d309
Date: 2012-01-22 12:30 +0100
http://bitbucket.org/pypy/pypy/changeset/e82f6594d309/

Log:	Write a test_pypy_c checking that "f ** 2" gets us just a float_mul
	operation. (Needs to check if the test really passes or if we are
	getting a few extra stuff too.)

diff --git a/pypy/module/pypyjit/test_pypy_c/test_math.py b/pypy/module/pypyjit/test_pypy_c/test_math.py
--- a/pypy/module/pypyjit/test_pypy_c/test_math.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_math.py
@@ -90,3 +90,18 @@
             --TICK--
             jump(..., descr=)
         """)
+
+    def test_pow_two(self):
+        def main(n):
+            s = 0.123
+            while n > 0:
+                s -= s ** 2   # ID: pow
+                n -= 1
+            return s
+        log = self.run(main, [500])
+        assert abs(log.result - main(500)) < 1e-9
+        loop, = log.loops_by_filename(self.filepath)
+        assert loop.match_by_id("pow", """
+            f2 = float_mul(f1, f1)
+            f3 = float_sub(f1, f2)
+        """)

From noreply at buildbot.pypy.org  Sun Jan 22 12:38:02 2012
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sun, 22 Jan 2012 12:38:02 +0100 (CET)
Subject: [pypy-commit] pypy merge-2.7.2: Fix method used to find
	cjkencodings data files
Message-ID: <20120122113802.78872821FA@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: merge-2.7.2
Changeset: r51644:70241d66bf5f
Date: 2012-01-22 12:36 +0100
http://bitbucket.org/pypy/pypy/changeset/70241d66bf5f/

Log:	Fix method used to find cjkencodings data files

diff --git a/lib-python/modified-2.7/test/test_multibytecodec_support.py b/lib-python/modified-2.7/test/test_multibytecodec_support.py
--- a/lib-python/modified-2.7/test/test_multibytecodec_support.py
+++ b/lib-python/modified-2.7/test/test_multibytecodec_support.py
@@ -330,7 +330,7 @@
                             repr(csetch), repr(unich), exc.reason))
 
 def load_teststring(name):
-    dir = os.path.join(os.path.dirname(__file__), 'cjkencodings')
+    dir = test_support.findfile('cjkencodings')
     with open(os.path.join(dir, name + '.txt'), 'rb') as f:
         encoded = f.read()
     with open(os.path.join(dir, name + '-utf8.txt'), 'rb') as f:

From noreply at buildbot.pypy.org  Sun Jan 22 14:35:38 2012
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sun, 22 Jan 2012 14:35:38 +0100 (CET)
Subject: [pypy-commit] pypy merge-2.7.2: Respect __subclasscheck__ when
 rebinding a method to a more specific class.
Message-ID: <20120122133538.B1CB6821FA@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: merge-2.7.2
Changeset: r51645:24cea2c24401
Date: 2012-01-22 14:33 +0100
http://bitbucket.org/pypy/pypy/changeset/24cea2c24401/

Log:	Respect __subclasscheck__ when rebinding a method to a more specific
	class. This change is needed to make OrderedDict() work at all, but
	it may have unexpected performance impact.

diff --git a/pypy/interpreter/function.py b/pypy/interpreter/function.py
--- a/pypy/interpreter/function.py
+++ b/pypy/interpreter/function.py
@@ -497,7 +497,8 @@
             # only allow binding to a more specific class than before
             if (w_cls is not None and
                 not space.is_w(w_cls, space.w_None) and
-                not space.abstract_issubclass_w(w_cls, self.w_class)):
+                not space.is_true(
+                    space.issubtype_allow_override(w_cls, self.w_class))):
                 return space.wrap(self)    # subclass test failed
             else:
                 return descr_function_get(space, self.w_function, w_obj, w_cls)
diff --git a/pypy/objspace/test/test_descroperation.py b/pypy/objspace/test/test_descroperation.py
--- a/pypy/objspace/test/test_descroperation.py
+++ b/pypy/objspace/test/test_descroperation.py
@@ -641,6 +641,19 @@
         assert issubclass(B, B)
         assert issubclass(23, B)
 
+    def test_issubclass_and_method(self):
+        class Meta(type):
+            def __subclasscheck__(cls, sub):
+                if sub is Dict:
+                    return True
+        class A:
+            __metaclass__ = Meta
+            def method(self):
+                return 42
+        class Dict:
+            method = A.method
+        assert Dict().method() == 42
+
     def test_truth_of_long(self):
         class X(object):
             def __len__(self): return 1L

From notifications-noreply at bitbucket.org  Sun Jan 22 16:33:00 2012
From: notifications-noreply at bitbucket.org (Bitbucket)
Date: Sun, 22 Jan 2012 15:33:00 -0000
Subject: [pypy-commit] Notification: benchmarks
Message-ID: <20120122153300.3632.18640@bitbucket13.managed.contegix.com>

You have received a notification from Carsten Senger.

Hi, I forked benchmarks. My fork is at https://bitbucket.org/csenger/benchmarks.

--
Disable notifications at https://bitbucket.org/account/notifications/

From noreply at buildbot.pypy.org  Sun Jan 22 17:27:09 2012
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sun, 22 Jan 2012 17:27:09 +0100 (CET)
Subject: [pypy-commit] pypy merge-2.7.2: Undo the previous change,
 and just add "allow_override=True".
Message-ID: <20120122162709.9CB7D821FA@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: merge-2.7.2
Changeset: r51646:89f238f17ecb
Date: 2012-01-22 17:25 +0100
http://bitbucket.org/pypy/pypy/changeset/89f238f17ecb/

Log:	Undo the previous change, and just add "allow_override=True". This
	fixes the cases when an old-style class in involved.

diff --git a/pypy/interpreter/function.py b/pypy/interpreter/function.py
--- a/pypy/interpreter/function.py
+++ b/pypy/interpreter/function.py
@@ -497,8 +497,8 @@
             # only allow binding to a more specific class than before
             if (w_cls is not None and
                 not space.is_w(w_cls, space.w_None) and
-                not space.is_true(
-                    space.issubtype_allow_override(w_cls, self.w_class))):
+                not space.abstract_issubclass_w(w_cls, self.w_class,
+                                                allow_override=True)):
                 return space.wrap(self)    # subclass test failed
             else:
                 return descr_function_get(space, self.w_function, w_obj, w_cls)
diff --git a/pypy/interpreter/test/test_function.py b/pypy/interpreter/test/test_function.py
--- a/pypy/interpreter/test/test_function.py
+++ b/pypy/interpreter/test/test_function.py
@@ -597,6 +597,17 @@
         # --- with an incompatible class
         w_meth5 = meth3.descr_method_get(space.wrap('hello'), space.w_str)
         assert space.is_w(w_meth5, w_meth3)
+        # Same thing, with an old-style class
+        w_oldclass = space.call_function(
+            space.builtin.get('__metaclass__'),
+            space.wrap('OldClass'), space.newtuple([]), space.newdict())
+        w_meth6 = meth3.descr_method_get(space.wrap('hello'), w_oldclass)
+        assert space.is_w(w_meth6, w_meth3)
+        # Reverse order of old/new styles
+        w_meth7 = descr_function_get(space, func, space.w_None, w_oldclass)
+        meth7 = space.unwrap(w_meth7)
+        w_meth8 = meth7.descr_method_get(space.wrap('hello'), space.w_str)
+        assert space.is_w(w_meth8, w_meth7)
 
 class TestShortcuts(object):
 

From noreply at buildbot.pypy.org  Sun Jan 22 17:30:01 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 22 Jan 2012 17:30:01 +0100 (CET)
Subject: [pypy-commit] pypy default: fix the test.
Message-ID: <20120122163001.DEBCC821FA@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r51647:7cd209e0414e
Date: 2012-01-22 17:29 +0100
http://bitbucket.org/pypy/pypy/changeset/7cd209e0414e/

Log:	fix the test.

diff --git a/pypy/module/pypyjit/test_pypy_c/test_math.py b/pypy/module/pypyjit/test_pypy_c/test_math.py
--- a/pypy/module/pypyjit/test_pypy_c/test_math.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_math.py
@@ -102,6 +102,7 @@
         assert abs(log.result - main(500)) < 1e-9
         loop, = log.loops_by_filename(self.filepath)
         assert loop.match_by_id("pow", """
-            f2 = float_mul(f1, f1)
-            f3 = float_sub(f1, f2)
+            guard_not_invalidated(descr=...)
+            f38 = float_mul(f30, f30)
+            f39 = float_sub(f30, f38)
         """)

From noreply at buildbot.pypy.org  Sun Jan 22 18:53:27 2012
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sun, 22 Jan 2012 18:53:27 +0100 (CET)
Subject: [pypy-commit] pypy merge-2.7.2: fix JSON tests
Message-ID: <20120122175327.2E399821FA@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor 
Branch: merge-2.7.2
Changeset: r51648:e5417ef18814
Date: 2012-01-22 11:53 -0600
http://bitbucket.org/pypy/pypy/changeset/e5417ef18814/

Log:	fix JSON tests

diff --git a/lib-python/modified-2.7/json/tests/test_decode.py b/lib-python/modified-2.7/json/tests/test_decode.py
--- a/lib-python/modified-2.7/json/tests/test_decode.py
+++ b/lib-python/modified-2.7/json/tests/test_decode.py
@@ -23,12 +23,10 @@
         self.assertEqual(rval, {"key":"value", "k":"v"})
 
     def test_empty_objects(self):
-        s = '{}'
-        self.assertEqual(json.loads(s), eval(s))
-        s = '[]'
-        self.assertEqual(json.loads(s), eval(s))
-        s = '""'
-        self.assertEqual(json.loads(s), eval(s))
+        self.assertEqual(self.loads('{}'), {})
+        self.assertEqual(self.loads('[]'), [])
+        self.assertEqual(self.loads('""'), u"")
+        self.assertIsInstance(self.loads('""'), unicode)
 
     def test_object_pairs_hook(self):
         s = '{"xkd":1, "kcw":2, "art":3, "hxm":4, "qrt":5, "pad":6, "hoy":7}'
diff --git a/lib-python/modified-2.7/json/tests/test_unicode.py b/lib-python/modified-2.7/json/tests/test_unicode.py
--- a/lib-python/modified-2.7/json/tests/test_unicode.py
+++ b/lib-python/modified-2.7/json/tests/test_unicode.py
@@ -81,9 +81,9 @@
         self.assertEqual(type(self.loads('"foo"')), unicode)
 
     def test_encode_not_utf_8(self):
-        self.assertEqual(json.dumps('\xb1\xe6', encoding='iso8859-2'),
+        self.assertEqual(self.dumps('\xb1\xe6', encoding='iso8859-2'),
                          '"\\u0105\\u0107"')
-        self.assertEqual(json.dumps(['\xb1\xe6'], encoding='iso8859-2'),
+        self.assertEqual(self.dumps(['\xb1\xe6'], encoding='iso8859-2'),
                          '["\\u0105\\u0107"]')
 
 

From noreply at buildbot.pypy.org  Sun Jan 22 19:09:51 2012
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sun, 22 Jan 2012 19:09:51 +0100 (CET)
Subject: [pypy-commit] pypy merge-2.7.2: Fix for time.mktime,
 -1 is a valid return value in some cases.
Message-ID: <20120122180951.60E1F821FA@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor 
Branch: merge-2.7.2
Changeset: r51649:a0e53cbd4f7c
Date: 2012-01-22 12:09 -0600
http://bitbucket.org/pypy/pypy/changeset/a0e53cbd4f7c/

Log:	Fix for time.mktime, -1 is a valid return value in some cases.

diff --git a/pypy/module/rctime/interp_time.py b/pypy/module/rctime/interp_time.py
--- a/pypy/module/rctime/interp_time.py
+++ b/pypy/module/rctime/interp_time.py
@@ -498,8 +498,11 @@
     Convert a time tuple in local time to seconds since the Epoch."""
 
     buf = _gettmarg(space, w_tup, allowNone=False)
+    rffi.setintfield(buf, "c_tm_wday", -1)
     tt = c_mktime(buf)
-    if tt == -1:
+    # A return value of -1 does not necessarily mean an error, but tm_wday
+    # cannot remain set to -1 if mktime succeeds.
+    if tt == -1 and rffi.getintfield(buf, "c_tm_wday") == -1:
         raise OperationError(space.w_OverflowError,
             space.wrap("mktime argument out of range"))
 
diff --git a/pypy/module/rctime/test/test_rctime.py b/pypy/module/rctime/test/test_rctime.py
--- a/pypy/module/rctime/test/test_rctime.py
+++ b/pypy/module/rctime/test/test_rctime.py
@@ -14,7 +14,7 @@
         assert isinstance(rctime.timezone, int)
         assert isinstance(rctime.tzname, tuple)
         assert isinstance(rctime.__doc__, str)
-    
+
     def test_sleep(self):
         import time as rctime
         import sys
@@ -46,7 +46,7 @@
         assert isinstance(res, str)
         rctime.ctime(rctime.time())
         raises(ValueError, rctime.ctime, 1E200)
-    
+
     def test_gmtime(self):
         import time as rctime
         raises(TypeError, rctime.gmtime, "foo")
@@ -75,7 +75,7 @@
         assert 0 <= (t1 - t0) < 1.2
         t = rctime.time()
         assert rctime.localtime(t) == rctime.localtime(t)
-    
+
     def test_mktime(self):
         import time as rctime
         import os, sys
@@ -85,30 +85,32 @@
         raises(TypeError, rctime.mktime, (1, 2, 3, 4, 5, 6, 'f', 8, 9))
         res = rctime.mktime(rctime.localtime())
         assert isinstance(res, float)
-        
+
         ltime = rctime.localtime()
         rctime.accept2dyear == 0
         ltime = list(ltime)
         ltime[0] = 1899
         raises(ValueError, rctime.mktime, tuple(ltime))
         rctime.accept2dyear == 1
-    
+
         ltime = list(ltime)
         ltime[0] = 67
         ltime = tuple(ltime)
         if os.name != "nt" and sys.maxint < 1<<32:   # time_t may be 64bit
             raises(OverflowError, rctime.mktime, ltime)
-    
+
         ltime = list(ltime)
         ltime[0] = 100
         raises(ValueError, rctime.mktime, tuple(ltime))
-    
+
         t = rctime.time()
         assert long(rctime.mktime(rctime.localtime(t))) == long(t)
         assert long(rctime.mktime(rctime.gmtime(t))) - rctime.timezone == long(t)
         ltime = rctime.localtime()
         assert rctime.mktime(tuple(ltime)) == rctime.mktime(ltime)
-    
+
+        assert rctime.mktime(rctime.localtime(-1)) == -1
+
     def test_asctime(self):
         import time as rctime
         rctime.asctime()
@@ -138,18 +140,18 @@
         st_time = rctime.struct_time(tup)
         assert str(st_time).startswith('time.struct_time(tm_year=1, ')
         assert len(st_time) == len(tup)
-    
+
     def test_tzset(self):
         import time as rctime
         import os
-        
+
         if not os.name == "posix":
             skip("tzset available only under Unix")
-    
+
         # epoch time of midnight Dec 25th 2002. Never DST in northern
         # hemisphere.
         xmas2002 = 1040774400.0
-    
+
         # these formats are correct for 2002, and possibly future years
         # this format is the 'standard' as documented at:
         # http://www.opengroup.org/onlinepubs/007904975/basedefs/xbd_chap08.html
@@ -158,7 +160,7 @@
         eastern = 'EST+05EDT,M4.1.0,M10.5.0'
         victoria = 'AEST-10AEDT-11,M10.5.0,M3.5.0'
         utc = 'UTC+0'
-    
+
         org_TZ = os.environ.get('TZ', None)
         try:
             # Make sure we can switch to UTC time and results are correct
@@ -172,7 +174,7 @@
             assert rctime.daylight == 0
             assert rctime.timezone == 0
             assert rctime.localtime(xmas2002).tm_isdst == 0
-            
+
             # make sure we can switch to US/Eastern
             os.environ['TZ'] = eastern
             rctime.tzset()
@@ -183,7 +185,7 @@
             assert rctime.timezone == 18000
             assert rctime.altzone == 14400
             assert rctime.localtime(xmas2002).tm_isdst == 0
-            
+
             # now go to the southern hemisphere.
             os.environ['TZ'] = victoria
             rctime.tzset()
@@ -206,7 +208,7 @@
 
     def test_strftime(self):
         import time as rctime
-        
+
         t = rctime.time()
         tt = rctime.gmtime(t)
         for directive in ('a', 'A', 'b', 'B', 'c', 'd', 'H', 'I',
@@ -214,7 +216,7 @@
                           'U', 'w', 'W', 'x', 'X', 'y', 'Y', 'Z', '%'):
             format = ' %' + directive
             rctime.strftime(format, tt)
-        
+
         raises(TypeError, rctime.strftime, ())
         raises(TypeError, rctime.strftime, (1,))
         raises(TypeError, rctime.strftime, range(8))
@@ -234,10 +236,10 @@
 
     def test_strftime_bounds_checking(self):
         import time as rctime
-        
+
         # make sure that strftime() checks the bounds of the various parts
         # of the time tuple.
-    
+
         # check year
         raises(ValueError, rctime.strftime, '', (1899, 1, 1, 0, 0, 0, 0, 1, -1))
         if rctime.accept2dyear:

From noreply at buildbot.pypy.org  Sun Jan 22 19:16:41 2012
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sun, 22 Jan 2012 19:16:41 +0100 (CET)
Subject: [pypy-commit] pypy merge-2.7.2: Change to the new exception for
	bytearray().pop()
Message-ID: <20120122181641.8E37A821FA@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor 
Branch: merge-2.7.2
Changeset: r51650:f17c2e5a8629
Date: 2012-01-22 12:16 -0600
http://bitbucket.org/pypy/pypy/changeset/f17c2e5a8629/

Log:	Change to the new exception for bytearray().pop()

diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -414,8 +414,8 @@
         result = w_bytearray.data.pop(index)
     except IndexError:
         if not w_bytearray.data:
-            raise OperationError(space.w_OverflowError, space.wrap(
-                "cannot pop an empty bytearray"))
+            raise OperationError(space.w_IndexError, space.wrap(
+                "pop from empty bytearray"))
         raise OperationError(space.w_IndexError, space.wrap(
             "pop index out of range"))
     return space.wrap(ord(result))
diff --git a/pypy/objspace/std/test/test_bytearrayobject.py b/pypy/objspace/std/test/test_bytearrayobject.py
--- a/pypy/objspace/std/test/test_bytearrayobject.py
+++ b/pypy/objspace/std/test/test_bytearrayobject.py
@@ -266,7 +266,7 @@
         assert b.pop(0) == ord('w')
         assert b.pop(-2) == ord('r')
         raises(IndexError, b.pop, 10)
-        raises(OverflowError, bytearray().pop)
+        raises(IndexError, bytearray().pop)
         assert bytearray('\xff').pop() == 0xff
 
     def test_remove(self):

From noreply at buildbot.pypy.org  Sun Jan 22 19:23:34 2012
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sun, 22 Jan 2012 19:23:34 +0100 (CET)
Subject: [pypy-commit] pypy merge-2.7.2: CPython issue11391: Fix a mmap
	crasher
Message-ID: <20120122182334.75D16821FA@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: merge-2.7.2
Changeset: r51651:9def34947826
Date: 2012-01-22 17:55 +0100
http://bitbucket.org/pypy/pypy/changeset/9def34947826/

Log:	CPython issue11391: Fix a mmap crasher

diff --git a/pypy/rlib/rmmap.py b/pypy/rlib/rmmap.py
--- a/pypy/rlib/rmmap.py
+++ b/pypy/rlib/rmmap.py
@@ -625,13 +625,16 @@
             flags = MAP_PRIVATE
             prot = PROT_READ | PROT_WRITE
         elif access == _ACCESS_DEFAULT:
-            pass
+            # map prot to access type
+            if prot & PROT_READ and prot & PROT_WRITE:
+                pass  # _ACCESS_DEFAULT
+            elif prot & PROT_WRITE:
+                access = ACCESS_WRITE
+            else:
+                access = ACCESS_READ
         else:
             raise RValueError("mmap invalid access parameter.")
 
-        if prot == PROT_READ:
-            access = ACCESS_READ
-
         # check file size
         try:
             st = os.fstat(fd)
diff --git a/pypy/rlib/test/test_rmmap.py b/pypy/rlib/test/test_rmmap.py
--- a/pypy/rlib/test/test_rmmap.py
+++ b/pypy/rlib/test/test_rmmap.py
@@ -263,10 +263,23 @@
         f.flush()
         m = mmap.mmap(f.fileno(), 6, prot=mmap.PROT_READ)
         raises(RTypeError, m.write, "foo")
+        m.close()
+        f.close()
+
+    def test_write_without_protwrite(self):
+        if os.name == "nt":
+            skip("Needs PROT_WRITE")
+        f = open(self.tmpname + "l2", "w+")
+        f.write("foobar")
+        f.flush()
+        m = mmap.mmap(f.fileno(), 6, prot=~mmap.PROT_WRITE)
+        raises(RTypeError, m.write_byte, 'a')
+        raises(RTypeError, m.write, "foo")
+        m.close()
         f.close()
 
     def test_size(self):
-        f = open(self.tmpname + "l", "w+")
+        f = open(self.tmpname + "l3", "w+")
         
         f.write("foobar")
         f.flush()

From noreply at buildbot.pypy.org  Sun Jan 22 19:23:35 2012
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sun, 22 Jan 2012 19:23:35 +0100 (CET)
Subject: [pypy-commit] pypy merge-2.7.2: test_ssl: Correctly use the helper
 function to find data files
Message-ID: <20120122182335.A9480821FA@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: merge-2.7.2
Changeset: r51652:a49166fe832a
Date: 2012-01-22 18:01 +0100
http://bitbucket.org/pypy/pypy/changeset/a49166fe832a/

Log:	test_ssl: Correctly use the helper function to find data files

diff --git a/lib-python/modified-2.7/test/test_ssl.py b/lib-python/modified-2.7/test/test_ssl.py
--- a/lib-python/modified-2.7/test/test_ssl.py
+++ b/lib-python/modified-2.7/test/test_ssl.py
@@ -1334,7 +1334,6 @@
     global CERTFILE, SVN_PYTHON_ORG_ROOT_CERT
     CERTFILE = test_support.findfile("keycert.pem")
     SVN_PYTHON_ORG_ROOT_CERT = test_support.findfile(
-        os.path.dirname(__file__) or os.curdir,
         "https_svn_python_org_root.pem")
 
     if (not os.path.exists(CERTFILE) or

From noreply at buildbot.pypy.org  Sun Jan 22 19:23:36 2012
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sun, 22 Jan 2012 19:23:36 +0100 (CET)
Subject: [pypy-commit] pypy merge-2.7.2: Fix our copy of "sysconfig.py",
 which I broke during the merge
Message-ID: <20120122182336.DE394821FA@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: merge-2.7.2
Changeset: r51653:9f554debdd92
Date: 2012-01-22 18:03 +0100
http://bitbucket.org/pypy/pypy/changeset/9f554debdd92/

Log:	Fix our copy of "sysconfig.py", which I broke during the merge

diff --git a/lib-python/modified-2.7/distutils/sysconfig.py b/lib-python/modified-2.7/distutils/sysconfig.py
--- a/lib-python/modified-2.7/distutils/sysconfig.py
+++ b/lib-python/modified-2.7/distutils/sysconfig.py
@@ -9,563 +9,21 @@
 Email:        
 """
 
-__revision__ = "$Id$"
+__revision__ = "$Id: sysconfig.py 85358 2010-10-10 09:54:59Z antoine.pitrou $"
 
-import os
-import re
-import string
 import sys
 
-from distutils.errors import DistutilsPlatformError
 
-# These are needed in a couple of spots, so just compute them once.
-PREFIX = os.path.normpath(sys.prefix)
-EXEC_PREFIX = os.path.normpath(sys.exec_prefix)
+# The content of this file is redirected from
+# sysconfig_cpython or sysconfig_pypy.
 
-# Path to the base directory of the project. On Windows the binary may
-# live in project/PCBuild9.  If we're dealing with an x64 Windows build,
-# it'll live in project/PCbuild/amd64.
-project_base = os.path.dirname(os.path.abspath(sys.executable))
-if os.name == "nt" and "pcbuild" in project_base[-8:].lower():
-    project_base = os.path.abspath(os.path.join(project_base, os.path.pardir))
-# PC/VS7.1
-if os.name == "nt" and "\\pc\\v" in project_base[-10:].lower():
-    project_base = os.path.abspath(os.path.join(project_base, os.path.pardir,
-                                                os.path.pardir))
-# PC/AMD64
-if os.name == "nt" and "\\pcbuild\\amd64" in project_base[-14:].lower():
-    project_base = os.path.abspath(os.path.join(project_base, os.path.pardir,
-                                                os.path.pardir))
+if '__pypy__' in sys.builtin_module_names:
+    from distutils.sysconfig_pypy import *
+    from distutils.sysconfig_pypy import _config_vars # needed by setuptools
+    from distutils.sysconfig_pypy import _variable_rx # read_setup_file()
+else:
+    from distutils.sysconfig_cpython import *
+    from distutils.sysconfig_cpython import _config_vars # needed by setuptools
+    from distutils.sysconfig_cpython import _variable_rx # read_setup_file()
 
-# python_build: (Boolean) if true, we're either building Python or
-# building an extension with an un-installed Python, so we use
-# different (hard-wired) directories.
-# Setup.local is available for Makefile builds including VPATH builds,
-# Setup.dist is available on Windows
-def _python_build():
-    for fn in ("Setup.dist", "Setup.local"):
-        if os.path.isfile(os.path.join(project_base, "Modules", fn)):
-            return True
-    return False
-python_build = _python_build()
 
-
-def get_python_version():
-    """Return a string containing the major and minor Python version,
-    leaving off the patchlevel.  Sample return values could be '1.5'
-    or '2.2'.
-    """
-    return sys.version[:3]
-
-
-def get_python_inc(plat_specific=0, prefix=None):
-    """Return the directory containing installed Python header files.
-
-    If 'plat_specific' is false (the default), this is the path to the
-    non-platform-specific header files, i.e. Python.h and so on;
-    otherwise, this is the path to platform-specific header files
-    (namely pyconfig.h).
-
-    If 'prefix' is supplied, use it instead of sys.prefix or
-    sys.exec_prefix -- i.e., ignore 'plat_specific'.
-    """
-    if prefix is None:
-        prefix = plat_specific and EXEC_PREFIX or PREFIX
-
-    if os.name == "posix":
-        if python_build:
-            buildir = os.path.dirname(sys.executable)
-            if plat_specific:
-                # python.h is located in the buildir
-                inc_dir = buildir
-            else:
-                # the source dir is relative to the buildir
-                srcdir = os.path.abspath(os.path.join(buildir,
-                                         get_config_var('srcdir')))
-                # Include is located in the srcdir
-                inc_dir = os.path.join(srcdir, "Include")
-            return inc_dir
-        return os.path.join(prefix, "include", "python" + get_python_version())
-    elif os.name == "nt":
-        return os.path.join(prefix, "include")
-    elif os.name == "os2":
-        return os.path.join(prefix, "Include")
-    else:
-        raise DistutilsPlatformError(
-            "I don't know where Python installs its C header files "
-            "on platform '%s'" % os.name)
-
-
-def get_python_lib(plat_specific=0, standard_lib=0, prefix=None):
-    """Return the directory containing the Python library (standard or
-    site additions).
-
-    If 'plat_specific' is true, return the directory containing
-    platform-specific modules, i.e. any module from a non-pure-Python
-    module distribution; otherwise, return the platform-shared library
-    directory.  If 'standard_lib' is true, return the directory
-    containing standard Python library modules; otherwise, return the
-    directory for site-specific modules.
-
-    If 'prefix' is supplied, use it instead of sys.prefix or
-    sys.exec_prefix -- i.e., ignore 'plat_specific'.
-    """
-    if prefix is None:
-        prefix = plat_specific and EXEC_PREFIX or PREFIX
-
-    if os.name == "posix":
-        libpython = os.path.join(prefix,
-                                 "lib", "python" + get_python_version())
-        if standard_lib:
-            return libpython
-        else:
-            return os.path.join(libpython, "site-packages")
-
-    elif os.name == "nt":
-        if standard_lib:
-            return os.path.join(prefix, "Lib")
-        else:
-            if get_python_version() < "2.2":
-                return prefix
-            else:
-                return os.path.join(prefix, "Lib", "site-packages")
-
-    elif os.name == "os2":
-        if standard_lib:
-            return os.path.join(prefix, "Lib")
-        else:
-            return os.path.join(prefix, "Lib", "site-packages")
-
-    else:
-        raise DistutilsPlatformError(
-            "I don't know where Python installs its library "
-            "on platform '%s'" % os.name)
-
-
-def customize_compiler(compiler):
-    """Do any platform-specific customization of a CCompiler instance.
-
-    Mainly needed on Unix, so we can plug in the information that
-    varies across Unices and is stored in Python's Makefile.
-    """
-    if compiler.compiler_type == "unix":
-        (cc, cxx, opt, cflags, ccshared, ldshared, so_ext) = \
-            get_config_vars('CC', 'CXX', 'OPT', 'CFLAGS',
-                            'CCSHARED', 'LDSHARED', 'SO')
-
-        if 'CC' in os.environ:
-            cc = os.environ['CC']
-        if 'CXX' in os.environ:
-            cxx = os.environ['CXX']
-        if 'LDSHARED' in os.environ:
-            ldshared = os.environ['LDSHARED']
-        if 'CPP' in os.environ:
-            cpp = os.environ['CPP']
-        else:
-            cpp = cc + " -E"           # not always
-        if 'LDFLAGS' in os.environ:
-            ldshared = ldshared + ' ' + os.environ['LDFLAGS']
-        if 'CFLAGS' in os.environ:
-            cflags = opt + ' ' + os.environ['CFLAGS']
-            ldshared = ldshared + ' ' + os.environ['CFLAGS']
-        if 'CPPFLAGS' in os.environ:
-            cpp = cpp + ' ' + os.environ['CPPFLAGS']
-            cflags = cflags + ' ' + os.environ['CPPFLAGS']
-            ldshared = ldshared + ' ' + os.environ['CPPFLAGS']
-
-        cc_cmd = cc + ' ' + cflags
-        compiler.set_executables(
-            preprocessor=cpp,
-            compiler=cc_cmd,
-            compiler_so=cc_cmd + ' ' + ccshared,
-            compiler_cxx=cxx,
-            linker_so=ldshared,
-            linker_exe=cc)
-
-        compiler.shared_lib_extension = so_ext
-
-
-def get_config_h_filename():
-    """Return full pathname of installed pyconfig.h file."""
-    if python_build:
-        if os.name == "nt":
-            inc_dir = os.path.join(project_base, "PC")
-        else:
-            inc_dir = project_base
-    else:
-        inc_dir = get_python_inc(plat_specific=1)
-    if get_python_version() < '2.2':
-        config_h = 'config.h'
-    else:
-        # The name of the config.h file changed in 2.2
-        config_h = 'pyconfig.h'
-    return os.path.join(inc_dir, config_h)
-
-
-def get_makefile_filename():
-    """Return full pathname of installed Makefile from the Python build."""
-    if python_build:
-        return os.path.join(os.path.dirname(sys.executable), "Makefile")
-    lib_dir = get_python_lib(plat_specific=1, standard_lib=1)
-    return os.path.join(lib_dir, "config", "Makefile")
-
-
-def parse_config_h(fp, g=None):
-    """Parse a config.h-style file.
-
-    A dictionary containing name/value pairs is returned.  If an
-    optional dictionary is passed in as the second argument, it is
-    used instead of a new dictionary.
-    """
-    if g is None:
-        g = {}
-    define_rx = re.compile("#define ([A-Z][A-Za-z0-9_]+) (.*)\n")
-    undef_rx = re.compile("/[*] #undef ([A-Z][A-Za-z0-9_]+) [*]/\n")
-    #
-    while 1:
-        line = fp.readline()
-        if not line:
-            break
-        m = define_rx.match(line)
-        if m:
-            n, v = m.group(1, 2)
-            try: v = int(v)
-            except ValueError: pass
-            g[n] = v
-        else:
-            m = undef_rx.match(line)
-            if m:
-                g[m.group(1)] = 0
-    return g
-
-
-# Regexes needed for parsing Makefile (and similar syntaxes,
-# like old-style Setup files).
-_variable_rx = re.compile("([a-zA-Z][a-zA-Z0-9_]+)\s*=\s*(.*)")
-_findvar1_rx = re.compile(r"\$\(([A-Za-z][A-Za-z0-9_]*)\)")
-_findvar2_rx = re.compile(r"\${([A-Za-z][A-Za-z0-9_]*)}")
-
-def parse_makefile(fn, g=None):
-    """Parse a Makefile-style file.
-
-    A dictionary containing name/value pairs is returned.  If an
-    optional dictionary is passed in as the second argument, it is
-    used instead of a new dictionary.
-    """
-    from distutils.text_file import TextFile
-    fp = TextFile(fn, strip_comments=1, skip_blanks=1, join_lines=1)
-
-    if g is None:
-        g = {}
-    done = {}
-    notdone = {}
-
-    while 1:
-        line = fp.readline()
-        if line is None:  # eof
-            break
-        m = _variable_rx.match(line)
-        if m:
-            n, v = m.group(1, 2)
-            v = v.strip()
-            # `$$' is a literal `$' in make
-            tmpv = v.replace('$$', '')
-
-            if "$" in tmpv:
-                notdone[n] = v
-            else:
-                try:
-                    v = int(v)
-                except ValueError:
-                    # insert literal `$'
-                    done[n] = v.replace('$$', '$')
-                else:
-                    done[n] = v
-
-    # do variable interpolation here
-    while notdone:
-        for name in notdone.keys():
-            value = notdone[name]
-            m = _findvar1_rx.search(value) or _findvar2_rx.search(value)
-            if m:
-                n = m.group(1)
-                found = True
-                if n in done:
-                    item = str(done[n])
-                elif n in notdone:
-                    # get it on a subsequent round
-                    found = False
-                elif n in os.environ:
-                    # do it like make: fall back to environment
-                    item = os.environ[n]
-                else:
-                    done[n] = item = ""
-                if found:
-                    after = value[m.end():]
-                    value = value[:m.start()] + item + after
-                    if "$" in after:
-                        notdone[name] = value
-                    else:
-                        try: value = int(value)
-                        except ValueError:
-                            done[name] = value.strip()
-                        else:
-                            done[name] = value
-                        del notdone[name]
-            else:
-                # bogus variable reference; just drop it since we can't deal
-                del notdone[name]
-
-    fp.close()
-
-    # strip spurious spaces
-    for k, v in done.items():
-        if isinstance(v, str):
-            done[k] = v.strip()
-
-    # save the results in the global dictionary
-    g.update(done)
-    return g
-
-
-def expand_makefile_vars(s, vars):
-    """Expand Makefile-style variables -- "${foo}" or "$(foo)" -- in
-    'string' according to 'vars' (a dictionary mapping variable names to
-    values).  Variables not present in 'vars' are silently expanded to the
-    empty string.  The variable values in 'vars' should not contain further
-    variable expansions; if 'vars' is the output of 'parse_makefile()',
-    you're fine.  Returns a variable-expanded version of 's'.
-    """
-
-    # This algorithm does multiple expansion, so if vars['foo'] contains
-    # "${bar}", it will expand ${foo} to ${bar}, and then expand
-    # ${bar}... and so forth.  This is fine as long as 'vars' comes from
-    # 'parse_makefile()', which takes care of such expansions eagerly,
-    # according to make's variable expansion semantics.
-
-    while 1:
-        m = _findvar1_rx.search(s) or _findvar2_rx.search(s)
-        if m:
-            (beg, end) = m.span()
-            s = s[0:beg] + vars.get(m.group(1)) + s[end:]
-        else:
-            break
-    return s
-
-
-_config_vars = None
-
-def _init_posix():
-    """Initialize the module as appropriate for POSIX systems."""
-    g = {}
-    # load the installed Makefile:
-    try:
-        filename = get_makefile_filename()
-        parse_makefile(filename, g)
-    except IOError, msg:
-        my_msg = "invalid Python installation: unable to open %s" % filename
-        if hasattr(msg, "strerror"):
-            my_msg = my_msg + " (%s)" % msg.strerror
-
-        raise DistutilsPlatformError(my_msg)
-
-    # load the installed pyconfig.h:
-    try:
-        filename = get_config_h_filename()
-        parse_config_h(file(filename), g)
-    except IOError, msg:
-        my_msg = "invalid Python installation: unable to open %s" % filename
-        if hasattr(msg, "strerror"):
-            my_msg = my_msg + " (%s)" % msg.strerror
-
-        raise DistutilsPlatformError(my_msg)
-
-    # On MacOSX we need to check the setting of the environment variable
-    # MACOSX_DEPLOYMENT_TARGET: configure bases some choices on it so
-    # it needs to be compatible.
-    # If it isn't set we set it to the configure-time value
-    if sys.platform == 'darwin' and 'MACOSX_DEPLOYMENT_TARGET' in g:
-        cfg_target = g['MACOSX_DEPLOYMENT_TARGET']
-        cur_target = os.getenv('MACOSX_DEPLOYMENT_TARGET', '')
-        if cur_target == '':
-            cur_target = cfg_target
-            os.environ['MACOSX_DEPLOYMENT_TARGET'] = cfg_target
-        elif map(int, cfg_target.split('.')) > map(int, cur_target.split('.')):
-            my_msg = ('$MACOSX_DEPLOYMENT_TARGET mismatch: now "%s" but "%s" during configure'
-                % (cur_target, cfg_target))
-            raise DistutilsPlatformError(my_msg)
-
-    # On AIX, there are wrong paths to the linker scripts in the Makefile
-    # -- these paths are relative to the Python source, but when installed
-    # the scripts are in another directory.
-    if python_build:
-        g['LDSHARED'] = g['BLDSHARED']
-
-    elif get_python_version() < '2.1':
-        # The following two branches are for 1.5.2 compatibility.
-        if sys.platform == 'aix4':          # what about AIX 3.x ?
-            # Linker script is in the config directory, not in Modules as the
-            # Makefile says.
-            python_lib = get_python_lib(standard_lib=1)
-            ld_so_aix = os.path.join(python_lib, 'config', 'ld_so_aix')
-            python_exp = os.path.join(python_lib, 'config', 'python.exp')
-
-            g['LDSHARED'] = "%s %s -bI:%s" % (ld_so_aix, g['CC'], python_exp)
-
-        elif sys.platform == 'beos':
-            # Linker script is in the config directory.  In the Makefile it is
-            # relative to the srcdir, which after installation no longer makes
-            # sense.
-            python_lib = get_python_lib(standard_lib=1)
-            linkerscript_path = string.split(g['LDSHARED'])[0]
-            linkerscript_name = os.path.basename(linkerscript_path)
-            linkerscript = os.path.join(python_lib, 'config',
-                                        linkerscript_name)
-
-            # XXX this isn't the right place to do this: adding the Python
-            # library to the link, if needed, should be in the "build_ext"
-            # command.  (It's also needed for non-MS compilers on Windows, and
-            # it's taken care of for them by the 'build_ext.get_libraries()'
-            # method.)
-            g['LDSHARED'] = ("%s -L%s/lib -lpython%s" %
-                             (linkerscript, PREFIX, get_python_version()))
-
-    global _config_vars
-    _config_vars = g
-
-
-def _init_nt():
-    """Initialize the module as appropriate for NT"""
-    g = {}
-    # set basic install directories
-    g['LIBDEST'] = get_python_lib(plat_specific=0, standard_lib=1)
-    g['BINLIBDEST'] = get_python_lib(plat_specific=1, standard_lib=1)
-
-    # XXX hmmm.. a normal install puts include files here
-    g['INCLUDEPY'] = get_python_inc(plat_specific=0)
-
-    g['SO'] = '.pyd'
-    g['EXE'] = ".exe"
-    g['VERSION'] = get_python_version().replace(".", "")
-    g['BINDIR'] = os.path.dirname(os.path.abspath(sys.executable))
-
-    global _config_vars
-    _config_vars = g
-
-
-def _init_os2():
-    """Initialize the module as appropriate for OS/2"""
-    g = {}
-    # set basic install directories
-    g['LIBDEST'] = get_python_lib(plat_specific=0, standard_lib=1)
-    g['BINLIBDEST'] = get_python_lib(plat_specific=1, standard_lib=1)
-
-    # XXX hmmm.. a normal install puts include files here
-    g['INCLUDEPY'] = get_python_inc(plat_specific=0)
-
-    g['SO'] = '.pyd'
-    g['EXE'] = ".exe"
-
-    global _config_vars
-    _config_vars = g
-
-
-def get_config_vars(*args):
-    """With no arguments, return a dictionary of all configuration
-    variables relevant for the current platform.  Generally this includes
-    everything needed to build extensions and install both pure modules and
-    extensions.  On Unix, this means every variable defined in Python's
-    installed Makefile; on Windows and Mac OS it's a much smaller set.
-
-    With arguments, return a list of values that result from looking up
-    each argument in the configuration variable dictionary.
-    """
-    global _config_vars
-    if _config_vars is None:
-        func = globals().get("_init_" + os.name)
-        if func:
-            func()
-        else:
-            _config_vars = {}
-
-        # Normalized versions of prefix and exec_prefix are handy to have;
-        # in fact, these are the standard versions used most places in the
-        # Distutils.
-        _config_vars['prefix'] = PREFIX
-        _config_vars['exec_prefix'] = EXEC_PREFIX
-
-        if sys.platform == 'darwin':
-            kernel_version = os.uname()[2] # Kernel version (8.4.3)
-            major_version = int(kernel_version.split('.')[0])
-
-            if major_version < 8:
-                # On Mac OS X before 10.4, check if -arch and -isysroot
-                # are in CFLAGS or LDFLAGS and remove them if they are.
-                # This is needed when building extensions on a 10.3 system
-                # using a universal build of python.
-                for key in ('LDFLAGS', 'BASECFLAGS', 'LDSHARED',
-                        # a number of derived variables. These need to be
-                        # patched up as well.
-                        'CFLAGS', 'PY_CFLAGS', 'BLDSHARED'):
-                    flags = _config_vars[key]
-                    flags = re.sub('-arch\s+\w+\s', ' ', flags)
-                    flags = re.sub('-isysroot [^ \t]*', ' ', flags)
-                    _config_vars[key] = flags
-
-            else:
-
-                # Allow the user to override the architecture flags using
-                # an environment variable.
-                # NOTE: This name was introduced by Apple in OSX 10.5 and
-                # is used by several scripting languages distributed with
-                # that OS release.
-
-                if 'ARCHFLAGS' in os.environ:
-                    arch = os.environ['ARCHFLAGS']
-                    for key in ('LDFLAGS', 'BASECFLAGS', 'LDSHARED',
-                        # a number of derived variables. These need to be
-                        # patched up as well.
-                        'CFLAGS', 'PY_CFLAGS', 'BLDSHARED'):
-
-                        flags = _config_vars[key]
-                        flags = re.sub('-arch\s+\w+\s', ' ', flags)
-                        flags = flags + ' ' + arch
-                        _config_vars[key] = flags
-
-                # If we're on OSX 10.5 or later and the user tries to
-                # compiles an extension using an SDK that is not present
-                # on the current machine it is better to not use an SDK
-                # than to fail.
-                #
-                # The major usecase for this is users using a Python.org
-                # binary installer  on OSX 10.6: that installer uses
-                # the 10.4u SDK, but that SDK is not installed by default
-                # when you install Xcode.
-                #
-                m = re.search('-isysroot\s+(\S+)', _config_vars['CFLAGS'])
-                if m is not None:
-                    sdk = m.group(1)
-                    if not os.path.exists(sdk):
-                        for key in ('LDFLAGS', 'BASECFLAGS', 'LDSHARED',
-                             # a number of derived variables. These need to be
-                             # patched up as well.
-                            'CFLAGS', 'PY_CFLAGS', 'BLDSHARED'):
-
-                            flags = _config_vars[key]
-                            flags = re.sub('-isysroot\s+\S+(\s|$)', ' ', flags)
-                            _config_vars[key] = flags
-
-    if args:
-        vals = []
-        for name in args:
-            vals.append(_config_vars.get(name))
-        return vals
-    else:
-        return _config_vars
-
-def get_config_var(name):
-    """Return the value of a single variable using the dictionary
-    returned by 'get_config_vars()'.  Equivalent to
-    get_config_vars().get(name)
-    """
-    return get_config_vars().get(name)

From noreply at buildbot.pypy.org  Sun Jan 22 19:23:38 2012
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sun, 22 Jan 2012 19:23:38 +0100 (CET)
Subject: [pypy-commit] pypy merge-2.7.2: Fix edge cases in float.__mod__
Message-ID: <20120122182338.2042F821FA@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: merge-2.7.2
Changeset: r51654:40000ebe74d7
Date: 2012-01-22 18:31 +0100
http://bitbucket.org/pypy/pypy/changeset/40000ebe74d7/

Log:	Fix edge cases in float.__mod__

diff --git a/pypy/objspace/std/floatobject.py b/pypy/objspace/std/floatobject.py
--- a/pypy/objspace/std/floatobject.py
+++ b/pypy/objspace/std/floatobject.py
@@ -383,8 +383,16 @@
     except ValueError:
         mod = rfloat.NAN
     else:
-        if (mod and ((y < 0.0) != (mod < 0.0))):
-            mod += y
+        if mod:
+            # ensure the remainder has the same sign as the denominator
+            if (y < 0.0) != (mod < 0.0):
+                mod += y
+        else:
+            # the remainder is zero, and in the presence of signed zeroes
+            # fmod returns different results across platforms; ensure
+            # it has the same sign as the denominator; we'd like to do
+            # "mod = y * 0.0", but that may get optimized away
+            mod = copysign(0.0, y)
 
     return W_FloatObject(mod)
 
diff --git a/pypy/objspace/std/test/test_floatobject.py b/pypy/objspace/std/test/test_floatobject.py
--- a/pypy/objspace/std/test/test_floatobject.py
+++ b/pypy/objspace/std/test/test_floatobject.py
@@ -789,3 +789,26 @@
         raises(ZeroDivisionError, lambda: inf % 0)
         raises(ZeroDivisionError, lambda: inf // 0)
         raises(ZeroDivisionError, divmod, inf, 0)
+
+    def test_modulo_edgecases(self):
+        # Check behaviour of % operator for IEEE 754 special cases.
+        # In particular, check signs of zeros.
+        mod = float.__mod__
+        import math
+
+        def check(a, b):
+            assert (a, math.copysign(1.0, a)) == (b, math.copysign(1.0, b))
+            
+        check(mod(-1.0, 1.0), 0.0)
+        check(mod(-1e-100, 1.0), 1.0)
+        check(mod(-0.0, 1.0), 0.0)
+        check(mod(0.0, 1.0), 0.0)
+        check(mod(1e-100, 1.0), 1e-100)
+        check(mod(1.0, 1.0), 0.0)
+
+        check(mod(-1.0, -1.0), -0.0)
+        check(mod(-1e-100, -1.0), -1e-100)
+        check(mod(-0.0, -1.0), -0.0)
+        check(mod(0.0, -1.0), -0.0)
+        check(mod(1e-100, -1.0), -1.0)
+        check(mod(1.0, -1.0), -0.0)

From noreply at buildbot.pypy.org  Sun Jan 22 19:32:08 2012
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sun, 22 Jan 2012 19:32:08 +0100 (CET)
Subject: [pypy-commit] pypy merge-2.7.2: Added in the new behavior for
 RawIOBase.readall, if read() returns None then readall() also returns None
Message-ID: <20120122183208.DCE18821FA@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor 
Branch: merge-2.7.2
Changeset: r51655:12b899970671
Date: 2012-01-22 12:31 -0600
http://bitbucket.org/pypy/pypy/changeset/12b899970671/

Log:	Added in the new behavior for RawIOBase.readall, if read() returns
	None then readall() also returns None

diff --git a/pypy/module/_io/interp_iobase.py b/pypy/module/_io/interp_iobase.py
--- a/pypy/module/_io/interp_iobase.py
+++ b/pypy/module/_io/interp_iobase.py
@@ -282,6 +282,10 @@
         while True:
             w_data = space.call_method(self, "read",
                                        space.wrap(DEFAULT_BUFFER_SIZE))
+            if space.is_w(w_data, space.w_None):
+                if not builder.getlength():
+                    return w_data
+                break
 
             if not space.isinstance_w(w_data, space.w_str):
                 raise OperationError(space.w_TypeError, space.wrap(
diff --git a/pypy/module/_io/test/test_io.py b/pypy/module/_io/test/test_io.py
--- a/pypy/module/_io/test/test_io.py
+++ b/pypy/module/_io/test/test_io.py
@@ -135,10 +135,27 @@
         assert r.read(2) == 'ab'
         assert r.read(2) == 'c'
         assert r.read(2) == 'de'
-        assert r.read(2) == None
+        assert r.read(2) is None
         assert r.read(2) == 'fg'
         assert r.read(2) == ''
 
+    def test_rawio_readall_none(self):
+        import _io
+        class MockRawIO(_io._RawIOBase):
+            read_stack = [None, None, "a"]
+            def readinto(self, buf):
+                v = self.read_stack.pop()
+                if v is None:
+                    return v
+                buf[:len(v)] = v
+                return len(v)
+
+        r = MockRawIO()
+        s = r.readall()
+        assert s =="a"
+        s = r.readall()
+        assert s is None
+
 class AppTestOpen:
     def setup_class(cls):
         cls.space = gettestobjspace(usemodules=['_io', '_locale'])

From noreply at buildbot.pypy.org  Sun Jan 22 19:32:10 2012
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sun, 22 Jan 2012 19:32:10 +0100 (CET)
Subject: [pypy-commit] pypy merge-2.7.2: merged upstream
Message-ID: <20120122183210.275CB821FA@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor 
Branch: merge-2.7.2
Changeset: r51656:1119c3892bfc
Date: 2012-01-22 12:31 -0600
http://bitbucket.org/pypy/pypy/changeset/1119c3892bfc/

Log:	merged upstream

diff --git a/lib-python/modified-2.7/distutils/sysconfig.py b/lib-python/modified-2.7/distutils/sysconfig.py
--- a/lib-python/modified-2.7/distutils/sysconfig.py
+++ b/lib-python/modified-2.7/distutils/sysconfig.py
@@ -9,563 +9,21 @@
 Email:        
 """
 
-__revision__ = "$Id$"
+__revision__ = "$Id: sysconfig.py 85358 2010-10-10 09:54:59Z antoine.pitrou $"
 
-import os
-import re
-import string
 import sys
 
-from distutils.errors import DistutilsPlatformError
 
-# These are needed in a couple of spots, so just compute them once.
-PREFIX = os.path.normpath(sys.prefix)
-EXEC_PREFIX = os.path.normpath(sys.exec_prefix)
+# The content of this file is redirected from
+# sysconfig_cpython or sysconfig_pypy.
 
-# Path to the base directory of the project. On Windows the binary may
-# live in project/PCBuild9.  If we're dealing with an x64 Windows build,
-# it'll live in project/PCbuild/amd64.
-project_base = os.path.dirname(os.path.abspath(sys.executable))
-if os.name == "nt" and "pcbuild" in project_base[-8:].lower():
-    project_base = os.path.abspath(os.path.join(project_base, os.path.pardir))
-# PC/VS7.1
-if os.name == "nt" and "\\pc\\v" in project_base[-10:].lower():
-    project_base = os.path.abspath(os.path.join(project_base, os.path.pardir,
-                                                os.path.pardir))
-# PC/AMD64
-if os.name == "nt" and "\\pcbuild\\amd64" in project_base[-14:].lower():
-    project_base = os.path.abspath(os.path.join(project_base, os.path.pardir,
-                                                os.path.pardir))
+if '__pypy__' in sys.builtin_module_names:
+    from distutils.sysconfig_pypy import *
+    from distutils.sysconfig_pypy import _config_vars # needed by setuptools
+    from distutils.sysconfig_pypy import _variable_rx # read_setup_file()
+else:
+    from distutils.sysconfig_cpython import *
+    from distutils.sysconfig_cpython import _config_vars # needed by setuptools
+    from distutils.sysconfig_cpython import _variable_rx # read_setup_file()
 
-# python_build: (Boolean) if true, we're either building Python or
-# building an extension with an un-installed Python, so we use
-# different (hard-wired) directories.
-# Setup.local is available for Makefile builds including VPATH builds,
-# Setup.dist is available on Windows
-def _python_build():
-    for fn in ("Setup.dist", "Setup.local"):
-        if os.path.isfile(os.path.join(project_base, "Modules", fn)):
-            return True
-    return False
-python_build = _python_build()
 
-
-def get_python_version():
-    """Return a string containing the major and minor Python version,
-    leaving off the patchlevel.  Sample return values could be '1.5'
-    or '2.2'.
-    """
-    return sys.version[:3]
-
-
-def get_python_inc(plat_specific=0, prefix=None):
-    """Return the directory containing installed Python header files.
-
-    If 'plat_specific' is false (the default), this is the path to the
-    non-platform-specific header files, i.e. Python.h and so on;
-    otherwise, this is the path to platform-specific header files
-    (namely pyconfig.h).
-
-    If 'prefix' is supplied, use it instead of sys.prefix or
-    sys.exec_prefix -- i.e., ignore 'plat_specific'.
-    """
-    if prefix is None:
-        prefix = plat_specific and EXEC_PREFIX or PREFIX
-
-    if os.name == "posix":
-        if python_build:
-            buildir = os.path.dirname(sys.executable)
-            if plat_specific:
-                # python.h is located in the buildir
-                inc_dir = buildir
-            else:
-                # the source dir is relative to the buildir
-                srcdir = os.path.abspath(os.path.join(buildir,
-                                         get_config_var('srcdir')))
-                # Include is located in the srcdir
-                inc_dir = os.path.join(srcdir, "Include")
-            return inc_dir
-        return os.path.join(prefix, "include", "python" + get_python_version())
-    elif os.name == "nt":
-        return os.path.join(prefix, "include")
-    elif os.name == "os2":
-        return os.path.join(prefix, "Include")
-    else:
-        raise DistutilsPlatformError(
-            "I don't know where Python installs its C header files "
-            "on platform '%s'" % os.name)
-
-
-def get_python_lib(plat_specific=0, standard_lib=0, prefix=None):
-    """Return the directory containing the Python library (standard or
-    site additions).
-
-    If 'plat_specific' is true, return the directory containing
-    platform-specific modules, i.e. any module from a non-pure-Python
-    module distribution; otherwise, return the platform-shared library
-    directory.  If 'standard_lib' is true, return the directory
-    containing standard Python library modules; otherwise, return the
-    directory for site-specific modules.
-
-    If 'prefix' is supplied, use it instead of sys.prefix or
-    sys.exec_prefix -- i.e., ignore 'plat_specific'.
-    """
-    if prefix is None:
-        prefix = plat_specific and EXEC_PREFIX or PREFIX
-
-    if os.name == "posix":
-        libpython = os.path.join(prefix,
-                                 "lib", "python" + get_python_version())
-        if standard_lib:
-            return libpython
-        else:
-            return os.path.join(libpython, "site-packages")
-
-    elif os.name == "nt":
-        if standard_lib:
-            return os.path.join(prefix, "Lib")
-        else:
-            if get_python_version() < "2.2":
-                return prefix
-            else:
-                return os.path.join(prefix, "Lib", "site-packages")
-
-    elif os.name == "os2":
-        if standard_lib:
-            return os.path.join(prefix, "Lib")
-        else:
-            return os.path.join(prefix, "Lib", "site-packages")
-
-    else:
-        raise DistutilsPlatformError(
-            "I don't know where Python installs its library "
-            "on platform '%s'" % os.name)
-
-
-def customize_compiler(compiler):
-    """Do any platform-specific customization of a CCompiler instance.
-
-    Mainly needed on Unix, so we can plug in the information that
-    varies across Unices and is stored in Python's Makefile.
-    """
-    if compiler.compiler_type == "unix":
-        (cc, cxx, opt, cflags, ccshared, ldshared, so_ext) = \
-            get_config_vars('CC', 'CXX', 'OPT', 'CFLAGS',
-                            'CCSHARED', 'LDSHARED', 'SO')
-
-        if 'CC' in os.environ:
-            cc = os.environ['CC']
-        if 'CXX' in os.environ:
-            cxx = os.environ['CXX']
-        if 'LDSHARED' in os.environ:
-            ldshared = os.environ['LDSHARED']
-        if 'CPP' in os.environ:
-            cpp = os.environ['CPP']
-        else:
-            cpp = cc + " -E"           # not always
-        if 'LDFLAGS' in os.environ:
-            ldshared = ldshared + ' ' + os.environ['LDFLAGS']
-        if 'CFLAGS' in os.environ:
-            cflags = opt + ' ' + os.environ['CFLAGS']
-            ldshared = ldshared + ' ' + os.environ['CFLAGS']
-        if 'CPPFLAGS' in os.environ:
-            cpp = cpp + ' ' + os.environ['CPPFLAGS']
-            cflags = cflags + ' ' + os.environ['CPPFLAGS']
-            ldshared = ldshared + ' ' + os.environ['CPPFLAGS']
-
-        cc_cmd = cc + ' ' + cflags
-        compiler.set_executables(
-            preprocessor=cpp,
-            compiler=cc_cmd,
-            compiler_so=cc_cmd + ' ' + ccshared,
-            compiler_cxx=cxx,
-            linker_so=ldshared,
-            linker_exe=cc)
-
-        compiler.shared_lib_extension = so_ext
-
-
-def get_config_h_filename():
-    """Return full pathname of installed pyconfig.h file."""
-    if python_build:
-        if os.name == "nt":
-            inc_dir = os.path.join(project_base, "PC")
-        else:
-            inc_dir = project_base
-    else:
-        inc_dir = get_python_inc(plat_specific=1)
-    if get_python_version() < '2.2':
-        config_h = 'config.h'
-    else:
-        # The name of the config.h file changed in 2.2
-        config_h = 'pyconfig.h'
-    return os.path.join(inc_dir, config_h)
-
-
-def get_makefile_filename():
-    """Return full pathname of installed Makefile from the Python build."""
-    if python_build:
-        return os.path.join(os.path.dirname(sys.executable), "Makefile")
-    lib_dir = get_python_lib(plat_specific=1, standard_lib=1)
-    return os.path.join(lib_dir, "config", "Makefile")
-
-
-def parse_config_h(fp, g=None):
-    """Parse a config.h-style file.
-
-    A dictionary containing name/value pairs is returned.  If an
-    optional dictionary is passed in as the second argument, it is
-    used instead of a new dictionary.
-    """
-    if g is None:
-        g = {}
-    define_rx = re.compile("#define ([A-Z][A-Za-z0-9_]+) (.*)\n")
-    undef_rx = re.compile("/[*] #undef ([A-Z][A-Za-z0-9_]+) [*]/\n")
-    #
-    while 1:
-        line = fp.readline()
-        if not line:
-            break
-        m = define_rx.match(line)
-        if m:
-            n, v = m.group(1, 2)
-            try: v = int(v)
-            except ValueError: pass
-            g[n] = v
-        else:
-            m = undef_rx.match(line)
-            if m:
-                g[m.group(1)] = 0
-    return g
-
-
-# Regexes needed for parsing Makefile (and similar syntaxes,
-# like old-style Setup files).
-_variable_rx = re.compile("([a-zA-Z][a-zA-Z0-9_]+)\s*=\s*(.*)")
-_findvar1_rx = re.compile(r"\$\(([A-Za-z][A-Za-z0-9_]*)\)")
-_findvar2_rx = re.compile(r"\${([A-Za-z][A-Za-z0-9_]*)}")
-
-def parse_makefile(fn, g=None):
-    """Parse a Makefile-style file.
-
-    A dictionary containing name/value pairs is returned.  If an
-    optional dictionary is passed in as the second argument, it is
-    used instead of a new dictionary.
-    """
-    from distutils.text_file import TextFile
-    fp = TextFile(fn, strip_comments=1, skip_blanks=1, join_lines=1)
-
-    if g is None:
-        g = {}
-    done = {}
-    notdone = {}
-
-    while 1:
-        line = fp.readline()
-        if line is None:  # eof
-            break
-        m = _variable_rx.match(line)
-        if m:
-            n, v = m.group(1, 2)
-            v = v.strip()
-            # `$$' is a literal `$' in make
-            tmpv = v.replace('$$', '')
-
-            if "$" in tmpv:
-                notdone[n] = v
-            else:
-                try:
-                    v = int(v)
-                except ValueError:
-                    # insert literal `$'
-                    done[n] = v.replace('$$', '$')
-                else:
-                    done[n] = v
-
-    # do variable interpolation here
-    while notdone:
-        for name in notdone.keys():
-            value = notdone[name]
-            m = _findvar1_rx.search(value) or _findvar2_rx.search(value)
-            if m:
-                n = m.group(1)
-                found = True
-                if n in done:
-                    item = str(done[n])
-                elif n in notdone:
-                    # get it on a subsequent round
-                    found = False
-                elif n in os.environ:
-                    # do it like make: fall back to environment
-                    item = os.environ[n]
-                else:
-                    done[n] = item = ""
-                if found:
-                    after = value[m.end():]
-                    value = value[:m.start()] + item + after
-                    if "$" in after:
-                        notdone[name] = value
-                    else:
-                        try: value = int(value)
-                        except ValueError:
-                            done[name] = value.strip()
-                        else:
-                            done[name] = value
-                        del notdone[name]
-            else:
-                # bogus variable reference; just drop it since we can't deal
-                del notdone[name]
-
-    fp.close()
-
-    # strip spurious spaces
-    for k, v in done.items():
-        if isinstance(v, str):
-            done[k] = v.strip()
-
-    # save the results in the global dictionary
-    g.update(done)
-    return g
-
-
-def expand_makefile_vars(s, vars):
-    """Expand Makefile-style variables -- "${foo}" or "$(foo)" -- in
-    'string' according to 'vars' (a dictionary mapping variable names to
-    values).  Variables not present in 'vars' are silently expanded to the
-    empty string.  The variable values in 'vars' should not contain further
-    variable expansions; if 'vars' is the output of 'parse_makefile()',
-    you're fine.  Returns a variable-expanded version of 's'.
-    """
-
-    # This algorithm does multiple expansion, so if vars['foo'] contains
-    # "${bar}", it will expand ${foo} to ${bar}, and then expand
-    # ${bar}... and so forth.  This is fine as long as 'vars' comes from
-    # 'parse_makefile()', which takes care of such expansions eagerly,
-    # according to make's variable expansion semantics.
-
-    while 1:
-        m = _findvar1_rx.search(s) or _findvar2_rx.search(s)
-        if m:
-            (beg, end) = m.span()
-            s = s[0:beg] + vars.get(m.group(1)) + s[end:]
-        else:
-            break
-    return s
-
-
-_config_vars = None
-
-def _init_posix():
-    """Initialize the module as appropriate for POSIX systems."""
-    g = {}
-    # load the installed Makefile:
-    try:
-        filename = get_makefile_filename()
-        parse_makefile(filename, g)
-    except IOError, msg:
-        my_msg = "invalid Python installation: unable to open %s" % filename
-        if hasattr(msg, "strerror"):
-            my_msg = my_msg + " (%s)" % msg.strerror
-
-        raise DistutilsPlatformError(my_msg)
-
-    # load the installed pyconfig.h:
-    try:
-        filename = get_config_h_filename()
-        parse_config_h(file(filename), g)
-    except IOError, msg:
-        my_msg = "invalid Python installation: unable to open %s" % filename
-        if hasattr(msg, "strerror"):
-            my_msg = my_msg + " (%s)" % msg.strerror
-
-        raise DistutilsPlatformError(my_msg)
-
-    # On MacOSX we need to check the setting of the environment variable
-    # MACOSX_DEPLOYMENT_TARGET: configure bases some choices on it so
-    # it needs to be compatible.
-    # If it isn't set we set it to the configure-time value
-    if sys.platform == 'darwin' and 'MACOSX_DEPLOYMENT_TARGET' in g:
-        cfg_target = g['MACOSX_DEPLOYMENT_TARGET']
-        cur_target = os.getenv('MACOSX_DEPLOYMENT_TARGET', '')
-        if cur_target == '':
-            cur_target = cfg_target
-            os.environ['MACOSX_DEPLOYMENT_TARGET'] = cfg_target
-        elif map(int, cfg_target.split('.')) > map(int, cur_target.split('.')):
-            my_msg = ('$MACOSX_DEPLOYMENT_TARGET mismatch: now "%s" but "%s" during configure'
-                % (cur_target, cfg_target))
-            raise DistutilsPlatformError(my_msg)
-
-    # On AIX, there are wrong paths to the linker scripts in the Makefile
-    # -- these paths are relative to the Python source, but when installed
-    # the scripts are in another directory.
-    if python_build:
-        g['LDSHARED'] = g['BLDSHARED']
-
-    elif get_python_version() < '2.1':
-        # The following two branches are for 1.5.2 compatibility.
-        if sys.platform == 'aix4':          # what about AIX 3.x ?
-            # Linker script is in the config directory, not in Modules as the
-            # Makefile says.
-            python_lib = get_python_lib(standard_lib=1)
-            ld_so_aix = os.path.join(python_lib, 'config', 'ld_so_aix')
-            python_exp = os.path.join(python_lib, 'config', 'python.exp')
-
-            g['LDSHARED'] = "%s %s -bI:%s" % (ld_so_aix, g['CC'], python_exp)
-
-        elif sys.platform == 'beos':
-            # Linker script is in the config directory.  In the Makefile it is
-            # relative to the srcdir, which after installation no longer makes
-            # sense.
-            python_lib = get_python_lib(standard_lib=1)
-            linkerscript_path = string.split(g['LDSHARED'])[0]
-            linkerscript_name = os.path.basename(linkerscript_path)
-            linkerscript = os.path.join(python_lib, 'config',
-                                        linkerscript_name)
-
-            # XXX this isn't the right place to do this: adding the Python
-            # library to the link, if needed, should be in the "build_ext"
-            # command.  (It's also needed for non-MS compilers on Windows, and
-            # it's taken care of for them by the 'build_ext.get_libraries()'
-            # method.)
-            g['LDSHARED'] = ("%s -L%s/lib -lpython%s" %
-                             (linkerscript, PREFIX, get_python_version()))
-
-    global _config_vars
-    _config_vars = g
-
-
-def _init_nt():
-    """Initialize the module as appropriate for NT"""
-    g = {}
-    # set basic install directories
-    g['LIBDEST'] = get_python_lib(plat_specific=0, standard_lib=1)
-    g['BINLIBDEST'] = get_python_lib(plat_specific=1, standard_lib=1)
-
-    # XXX hmmm.. a normal install puts include files here
-    g['INCLUDEPY'] = get_python_inc(plat_specific=0)
-
-    g['SO'] = '.pyd'
-    g['EXE'] = ".exe"
-    g['VERSION'] = get_python_version().replace(".", "")
-    g['BINDIR'] = os.path.dirname(os.path.abspath(sys.executable))
-
-    global _config_vars
-    _config_vars = g
-
-
-def _init_os2():
-    """Initialize the module as appropriate for OS/2"""
-    g = {}
-    # set basic install directories
-    g['LIBDEST'] = get_python_lib(plat_specific=0, standard_lib=1)
-    g['BINLIBDEST'] = get_python_lib(plat_specific=1, standard_lib=1)
-
-    # XXX hmmm.. a normal install puts include files here
-    g['INCLUDEPY'] = get_python_inc(plat_specific=0)
-
-    g['SO'] = '.pyd'
-    g['EXE'] = ".exe"
-
-    global _config_vars
-    _config_vars = g
-
-
-def get_config_vars(*args):
-    """With no arguments, return a dictionary of all configuration
-    variables relevant for the current platform.  Generally this includes
-    everything needed to build extensions and install both pure modules and
-    extensions.  On Unix, this means every variable defined in Python's
-    installed Makefile; on Windows and Mac OS it's a much smaller set.
-
-    With arguments, return a list of values that result from looking up
-    each argument in the configuration variable dictionary.
-    """
-    global _config_vars
-    if _config_vars is None:
-        func = globals().get("_init_" + os.name)
-        if func:
-            func()
-        else:
-            _config_vars = {}
-
-        # Normalized versions of prefix and exec_prefix are handy to have;
-        # in fact, these are the standard versions used most places in the
-        # Distutils.
-        _config_vars['prefix'] = PREFIX
-        _config_vars['exec_prefix'] = EXEC_PREFIX
-
-        if sys.platform == 'darwin':
-            kernel_version = os.uname()[2] # Kernel version (8.4.3)
-            major_version = int(kernel_version.split('.')[0])
-
-            if major_version < 8:
-                # On Mac OS X before 10.4, check if -arch and -isysroot
-                # are in CFLAGS or LDFLAGS and remove them if they are.
-                # This is needed when building extensions on a 10.3 system
-                # using a universal build of python.
-                for key in ('LDFLAGS', 'BASECFLAGS', 'LDSHARED',
-                        # a number of derived variables. These need to be
-                        # patched up as well.
-                        'CFLAGS', 'PY_CFLAGS', 'BLDSHARED'):
-                    flags = _config_vars[key]
-                    flags = re.sub('-arch\s+\w+\s', ' ', flags)
-                    flags = re.sub('-isysroot [^ \t]*', ' ', flags)
-                    _config_vars[key] = flags
-
-            else:
-
-                # Allow the user to override the architecture flags using
-                # an environment variable.
-                # NOTE: This name was introduced by Apple in OSX 10.5 and
-                # is used by several scripting languages distributed with
-                # that OS release.
-
-                if 'ARCHFLAGS' in os.environ:
-                    arch = os.environ['ARCHFLAGS']
-                    for key in ('LDFLAGS', 'BASECFLAGS', 'LDSHARED',
-                        # a number of derived variables. These need to be
-                        # patched up as well.
-                        'CFLAGS', 'PY_CFLAGS', 'BLDSHARED'):
-
-                        flags = _config_vars[key]
-                        flags = re.sub('-arch\s+\w+\s', ' ', flags)
-                        flags = flags + ' ' + arch
-                        _config_vars[key] = flags
-
-                # If we're on OSX 10.5 or later and the user tries to
-                # compiles an extension using an SDK that is not present
-                # on the current machine it is better to not use an SDK
-                # than to fail.
-                #
-                # The major usecase for this is users using a Python.org
-                # binary installer  on OSX 10.6: that installer uses
-                # the 10.4u SDK, but that SDK is not installed by default
-                # when you install Xcode.
-                #
-                m = re.search('-isysroot\s+(\S+)', _config_vars['CFLAGS'])
-                if m is not None:
-                    sdk = m.group(1)
-                    if not os.path.exists(sdk):
-                        for key in ('LDFLAGS', 'BASECFLAGS', 'LDSHARED',
-                             # a number of derived variables. These need to be
-                             # patched up as well.
-                            'CFLAGS', 'PY_CFLAGS', 'BLDSHARED'):
-
-                            flags = _config_vars[key]
-                            flags = re.sub('-isysroot\s+\S+(\s|$)', ' ', flags)
-                            _config_vars[key] = flags
-
-    if args:
-        vals = []
-        for name in args:
-            vals.append(_config_vars.get(name))
-        return vals
-    else:
-        return _config_vars
-
-def get_config_var(name):
-    """Return the value of a single variable using the dictionary
-    returned by 'get_config_vars()'.  Equivalent to
-    get_config_vars().get(name)
-    """
-    return get_config_vars().get(name)
diff --git a/lib-python/modified-2.7/test/test_ssl.py b/lib-python/modified-2.7/test/test_ssl.py
--- a/lib-python/modified-2.7/test/test_ssl.py
+++ b/lib-python/modified-2.7/test/test_ssl.py
@@ -1334,7 +1334,6 @@
     global CERTFILE, SVN_PYTHON_ORG_ROOT_CERT
     CERTFILE = test_support.findfile("keycert.pem")
     SVN_PYTHON_ORG_ROOT_CERT = test_support.findfile(
-        os.path.dirname(__file__) or os.curdir,
         "https_svn_python_org_root.pem")
 
     if (not os.path.exists(CERTFILE) or
diff --git a/pypy/objspace/std/floatobject.py b/pypy/objspace/std/floatobject.py
--- a/pypy/objspace/std/floatobject.py
+++ b/pypy/objspace/std/floatobject.py
@@ -383,8 +383,16 @@
     except ValueError:
         mod = rfloat.NAN
     else:
-        if (mod and ((y < 0.0) != (mod < 0.0))):
-            mod += y
+        if mod:
+            # ensure the remainder has the same sign as the denominator
+            if (y < 0.0) != (mod < 0.0):
+                mod += y
+        else:
+            # the remainder is zero, and in the presence of signed zeroes
+            # fmod returns different results across platforms; ensure
+            # it has the same sign as the denominator; we'd like to do
+            # "mod = y * 0.0", but that may get optimized away
+            mod = copysign(0.0, y)
 
     return W_FloatObject(mod)
 
diff --git a/pypy/objspace/std/test/test_floatobject.py b/pypy/objspace/std/test/test_floatobject.py
--- a/pypy/objspace/std/test/test_floatobject.py
+++ b/pypy/objspace/std/test/test_floatobject.py
@@ -789,3 +789,26 @@
         raises(ZeroDivisionError, lambda: inf % 0)
         raises(ZeroDivisionError, lambda: inf // 0)
         raises(ZeroDivisionError, divmod, inf, 0)
+
+    def test_modulo_edgecases(self):
+        # Check behaviour of % operator for IEEE 754 special cases.
+        # In particular, check signs of zeros.
+        mod = float.__mod__
+        import math
+
+        def check(a, b):
+            assert (a, math.copysign(1.0, a)) == (b, math.copysign(1.0, b))
+            
+        check(mod(-1.0, 1.0), 0.0)
+        check(mod(-1e-100, 1.0), 1.0)
+        check(mod(-0.0, 1.0), 0.0)
+        check(mod(0.0, 1.0), 0.0)
+        check(mod(1e-100, 1.0), 1e-100)
+        check(mod(1.0, 1.0), 0.0)
+
+        check(mod(-1.0, -1.0), -0.0)
+        check(mod(-1e-100, -1.0), -1e-100)
+        check(mod(-0.0, -1.0), -0.0)
+        check(mod(0.0, -1.0), -0.0)
+        check(mod(1e-100, -1.0), -1.0)
+        check(mod(1.0, -1.0), -0.0)
diff --git a/pypy/rlib/rmmap.py b/pypy/rlib/rmmap.py
--- a/pypy/rlib/rmmap.py
+++ b/pypy/rlib/rmmap.py
@@ -625,13 +625,16 @@
             flags = MAP_PRIVATE
             prot = PROT_READ | PROT_WRITE
         elif access == _ACCESS_DEFAULT:
-            pass
+            # map prot to access type
+            if prot & PROT_READ and prot & PROT_WRITE:
+                pass  # _ACCESS_DEFAULT
+            elif prot & PROT_WRITE:
+                access = ACCESS_WRITE
+            else:
+                access = ACCESS_READ
         else:
             raise RValueError("mmap invalid access parameter.")
 
-        if prot == PROT_READ:
-            access = ACCESS_READ
-
         # check file size
         try:
             st = os.fstat(fd)
diff --git a/pypy/rlib/test/test_rmmap.py b/pypy/rlib/test/test_rmmap.py
--- a/pypy/rlib/test/test_rmmap.py
+++ b/pypy/rlib/test/test_rmmap.py
@@ -263,10 +263,23 @@
         f.flush()
         m = mmap.mmap(f.fileno(), 6, prot=mmap.PROT_READ)
         raises(RTypeError, m.write, "foo")
+        m.close()
+        f.close()
+
+    def test_write_without_protwrite(self):
+        if os.name == "nt":
+            skip("Needs PROT_WRITE")
+        f = open(self.tmpname + "l2", "w+")
+        f.write("foobar")
+        f.flush()
+        m = mmap.mmap(f.fileno(), 6, prot=~mmap.PROT_WRITE)
+        raises(RTypeError, m.write_byte, 'a')
+        raises(RTypeError, m.write, "foo")
+        m.close()
         f.close()
 
     def test_size(self):
-        f = open(self.tmpname + "l", "w+")
+        f = open(self.tmpname + "l3", "w+")
         
         f.write("foobar")
         f.flush()

From noreply at buildbot.pypy.org  Sun Jan 22 20:27:41 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 22 Jan 2012 20:27:41 +0100 (CET)
Subject: [pypy-commit] pypy concurrent-marksweep: Fix: conflict with global
	name 'error'.
Message-ID: <20120122192741.70CD1821FA@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: concurrent-marksweep
Changeset: r51657:d1c21fed7569
Date: 2012-01-12 14:10 +0100
http://bitbucket.org/pypy/pypy/changeset/d1c21fed7569/

Log:	Fix: conflict with global name 'error'.

diff --git a/pypy/module/thread/ll_thread.py b/pypy/module/thread/ll_thread.py
--- a/pypy/module/thread/ll_thread.py
+++ b/pypy/module/thread/ll_thread.py
@@ -122,9 +122,9 @@
 
     def release(self):
         # Sanity check: the lock must be locked
-        error = self.acquire(False)
+        err = self.acquire(False)
         c_thread_releaselock(self._lock)
-        if error:
+        if err:
             raise error("bad lock")
 
     def __del__(self):

From noreply at buildbot.pypy.org  Sun Jan 22 20:27:43 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 22 Jan 2012 20:27:43 +0100 (CET)
Subject: [pypy-commit] pypy concurrent-marksweep: merge heads
Message-ID: <20120122192743.023E9821FA@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: concurrent-marksweep
Changeset: r51658:edaf872702ab
Date: 2012-01-22 20:26 +0100
http://bitbucket.org/pypy/pypy/changeset/edaf872702ab/

Log:	merge heads

diff --git a/pypy/module/thread/ll_thread.py b/pypy/module/thread/ll_thread.py
--- a/pypy/module/thread/ll_thread.py
+++ b/pypy/module/thread/ll_thread.py
@@ -122,9 +122,9 @@
 
     def release(self):
         # Sanity check: the lock must be locked
-        error = self.acquire(False)
+        err = self.acquire(False)
         c_thread_releaselock(self._lock)
-        if error:
+        if err:
             raise error("bad lock")
 
     def __del__(self):

From noreply at buildbot.pypy.org  Sun Jan 22 20:28:04 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 22 Jan 2012 20:28:04 +0100 (CET)
Subject: [pypy-commit] pypy 32ptr-on-64bit: hg merge default
Message-ID: <20120122192804.7FAC1821FA@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 32ptr-on-64bit
Changeset: r51659:35da68422201
Date: 2012-01-22 20:12 +0100
http://bitbucket.org/pypy/pypy/changeset/35da68422201/

Log:	hg merge default

diff too long, truncating to 10000 out of 192957 lines

diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -1,6 +1,10 @@
 syntax: glob
 *.py[co]
 *~
+.*.swp
+.idea
+.project
+.pydevproject
 
 syntax: regexp
 ^testresult$
@@ -24,6 +28,8 @@
 ^pypy/translator/c/src/libffi_msvc/.+\.dll$
 ^pypy/translator/c/src/libffi_msvc/.+\.lib$
 ^pypy/translator/c/src/libffi_msvc/.+\.exp$
+^pypy/translator/c/src/cjkcodecs/.+\.o$
+^pypy/translator/c/src/cjkcodecs/.+\.obj$
 ^pypy/translator/jvm/\.project$
 ^pypy/translator/jvm/\.classpath$
 ^pypy/translator/jvm/eclipse-bin$
@@ -36,6 +42,8 @@
 ^pypy/translator/benchmark/shootout_benchmarks$
 ^pypy/translator/goal/pypy-translation-snapshot$
 ^pypy/translator/goal/pypy-c
+^pypy/translator/goal/pypy-jvm
+^pypy/translator/goal/pypy-jvm.jar
 ^pypy/translator/goal/.+\.exe$
 ^pypy/translator/goal/.+\.dll$
 ^pypy/translator/goal/target.+-c$
@@ -62,6 +70,7 @@
 ^pypy/doc/image/lattice3\.png$
 ^pypy/doc/image/stackless_informal\.png$
 ^pypy/doc/image/parsing_example.+\.png$
+^pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test\.o$
 ^compiled
 ^.git/
 ^release/
diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -1,1 +1,4 @@
 b590cf6de4190623aad9aa698694c22e614d67b9 release-1.5
+b48df0bf4e75b81d98f19ce89d4a7dc3e1dab5e5 benchmarked
+d8ac7d23d3ec5f9a0fa1264972f74a010dbfd07f release-1.6
+ff4af8f318821f7f5ca998613a60fca09aa137da release-1.7
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -27,7 +27,7 @@
     DEALINGS IN THE SOFTWARE.
 
 
-PyPy Copyright holders 2003-2011
+PyPy Copyright holders 2003-2012
 ----------------------------------- 
 
 Except when otherwise stated (look for LICENSE files or information at
@@ -42,34 +42,39 @@
     Samuele Pedroni
     Michael Hudson
     Holger Krekel
+    Alex Gaynor
     Christian Tismer
+    Hakan Ardo
     Benjamin Peterson
+    David Schneider
     Eric van Riet Paap
-    Anders Chrigström
-    Håkan Ardö
+    Anders Chrigstrom
     Richard Emslie
     Dan Villiom Podlaski Christiansen
     Alexander Schremmer
-    Alex Gaynor
-    David Schneider
-    Aurelién Campeas
+    Lukas Diekmann
+    Aurelien Campeas
     Anders Lehmann
     Camillo Bruni
     Niklaus Haldimann
+    Sven Hager
     Leonardo Santagada
     Toon Verwaest
     Seo Sanghyeon
+    Justin Peel
     Lawrence Oluyede
     Bartosz Skowron
     Jakub Gustak
     Guido Wesdorp
+    Daniel Roberts
+    Laura Creighton
     Adrien Di Mascio
-    Laura Creighton
     Ludovic Aubry
     Niko Matsakis
-    Daniel Roberts
+    Wim Lavrijsen
+    Matti Picus
     Jason Creighton
-    Jacob Hallén
+    Jacob Hallen
     Alex Martelli
     Anders Hammarquist
     Jan de Mooij
@@ -83,29 +88,36 @@
     Alexandre Fayolle
     Marius Gedminas
     Simon Burton
+    David Edelsohn
     Jean-Paul Calderone
     John Witulski
-    Wim Lavrijsen
+    Timo Paulssen
+    holger krekel
+    Dario Bertini
+    Mark Pearse
     Andreas Stührk
     Jean-Philippe St. Pierre
     Guido van Rossum
     Pavel Vinogradov
     Valentino Volonghi
     Paul deGrandis
+    Ilya Osadchiy
+    Ronny Pfannschmidt
     Adrian Kuhn
     tav
     Georg Brandl
+    Philip Jenvey
     Gerald Klix
     Wanja Saatkamp
     Boris Feigin
     Oscar Nierstrasz
-    Dario Bertini
     David Malcolm
     Eugene Oden
     Henry Mason
+    Jeff Terrace
     Lukas Renggli
     Guenter Jantzen
-    Ronny Pfannschmidt
+    Ned Batchelder
     Bert Freudenberg
     Amit Regmi
     Ben Young
@@ -122,8 +134,8 @@
     Jared Grubb
     Karl Bartel
     Gabriel Lavoie
+    Victor Stinner
     Brian Dorsey
-    Victor Stinner
     Stuart Williams
     Toby Watson
     Antoine Pitrou
@@ -134,19 +146,22 @@
     Jonathan David Riehl
     Elmo Mäntynen
     Anders Qvist
-    Beatrice Düring
+    Beatrice During
     Alexander Sedov
+    Corbin Simpson
     Vincent Legoll
+    Romain Guillebert
     Alan McIntyre
-    Romain Guillebert
     Alex Perry
     Jens-Uwe Mager
+    Simon Cross
     Dan Stromberg
-    Lukas Diekmann
+    Guillebert Romain
     Carl Meyer
     Pieter Zieschang
     Alejandro J. Cura
     Sylvain Thenault
+    Christoph Gerum
     Travis Francis Athougies
     Henrik Vendelbo
     Lutz Paelike
@@ -155,8 +170,10 @@
     Lucio Torre
     Lene Wagner
     Miguel de Val Borro
+    Artur Lisiecki
+    Bruno Gola
     Ignas Mikalajunas
-    Artur Lisiecki
+    Stefano Rivera
     Joshua Gilbert
     Godefroid Chappelle
     Yusei Tahara
@@ -165,26 +182,31 @@
     Gustavo Niemeyer
     William Leslie
     Akira Li
-    Kristján Valur Jónsson
+    Kristjan Valur Jonsson
     Bobby Impollonia
+    Michael Hudson-Doyle
+    Laurence Tratt
+    Yasir Suhail
     Andrew Thompson
     Anders Sigfridsson
+    Floris Bruynooghe
     Jacek Generowicz
     Dan Colish
-    Sven Hager
     Zooko Wilcox-O Hearn
-    Anders Hammarquist
+    Dan Loewenherz
+    Chris Lambacher
     Dinu Gherman
-    Dan Colish
+    Brett Cannon
     Daniel Neuhäuser
     Michael Chermside
     Konrad Delong
     Anna Ravencroft
     Greg Price
     Armin Ronacher
+    Christian Muirhead
     Jim Baker
-    Philip Jenvey
     Rodrigo Araújo
+    Romain Guillebert
 
     Heinrich-Heine University, Germany 
     Open End AB (formerly AB Strakt), Sweden
diff --git a/README b/README
--- a/README
+++ b/README
@@ -15,10 +15,10 @@
 
 The getting-started document will help guide you:
 
-    http://codespeak.net/pypy/dist/pypy/doc/getting-started.html
+    http://doc.pypy.org/en/latest/getting-started.html
 
 It will also point you to the rest of the documentation which is generated
 from files in the pypy/doc directory within the source repositories. Enjoy
 and send us feedback!
 
-    the pypy-dev team 
+    the pypy-dev team 
diff --git a/_pytest/__init__.py b/_pytest/__init__.py
--- a/_pytest/__init__.py
+++ b/_pytest/__init__.py
@@ -1,2 +1,2 @@
 #
-__version__ = '2.0.3'
+__version__ = '2.1.0.dev4'
diff --git a/_pytest/assertion.py b/_pytest/assertion.py
deleted file mode 100644
--- a/_pytest/assertion.py
+++ /dev/null
@@ -1,177 +0,0 @@
-"""
-support for presented detailed information in failing assertions.
-"""
-import py
-import sys
-from _pytest.monkeypatch import monkeypatch
-
-def pytest_addoption(parser):
-    group = parser.getgroup("debugconfig")
-    group._addoption('--no-assert', action="store_true", default=False,
-        dest="noassert",
-        help="disable python assert expression reinterpretation."),
-
-def pytest_configure(config):
-    # The _reprcompare attribute on the py.code module is used by
-    # py._code._assertionnew to detect this plugin was loaded and in
-    # turn call the hooks defined here as part of the
-    # DebugInterpreter.
-    m = monkeypatch()
-    config._cleanup.append(m.undo)
-    warn_about_missing_assertion()
-    if not config.getvalue("noassert") and not config.getvalue("nomagic"):
-        def callbinrepr(op, left, right):
-            hook_result = config.hook.pytest_assertrepr_compare(
-                config=config, op=op, left=left, right=right)
-            for new_expl in hook_result:
-                if new_expl:
-                    return '\n~'.join(new_expl)
-        m.setattr(py.builtin.builtins,
-                  'AssertionError', py.code._AssertionError)
-        m.setattr(py.code, '_reprcompare', callbinrepr)
-
-def warn_about_missing_assertion():
-    try:
-        assert False
-    except AssertionError:
-        pass
-    else:
-        sys.stderr.write("WARNING: failing tests may report as passing because "
-        "assertions are turned off!  (are you using python -O?)\n")
-
-# Provide basestring in python3
-try:
-    basestring = basestring
-except NameError:
-    basestring = str
-
-
-def pytest_assertrepr_compare(op, left, right):
-    """return specialised explanations for some operators/operands"""
-    width = 80 - 15 - len(op) - 2 # 15 chars indentation, 1 space around op
-    left_repr = py.io.saferepr(left, maxsize=int(width/2))
-    right_repr = py.io.saferepr(right, maxsize=width-len(left_repr))
-    summary = '%s %s %s' % (left_repr, op, right_repr)
-
-    issequence = lambda x: isinstance(x, (list, tuple))
-    istext = lambda x: isinstance(x, basestring)
-    isdict = lambda x: isinstance(x, dict)
-    isset = lambda x: isinstance(x, set)
-
-    explanation = None
-    try:
-        if op == '==':
-            if istext(left) and istext(right):
-                explanation = _diff_text(left, right)
-            elif issequence(left) and issequence(right):
-                explanation = _compare_eq_sequence(left, right)
-            elif isset(left) and isset(right):
-                explanation = _compare_eq_set(left, right)
-            elif isdict(left) and isdict(right):
-                explanation = _diff_text(py.std.pprint.pformat(left),
-                                         py.std.pprint.pformat(right))
-        elif op == 'not in':
-            if istext(left) and istext(right):
-                explanation = _notin_text(left, right)
-    except py.builtin._sysex:
-        raise
-    except:
-        excinfo = py.code.ExceptionInfo()
-        explanation = ['(pytest_assertion plugin: representation of '
-            'details failed. Probably an object has a faulty __repr__.)',
-            str(excinfo)
-            ]
-
-
-    if not explanation:
-        return None
-
-    # Don't include pageloads of data, should be configurable
-    if len(''.join(explanation)) > 80*8:
-        explanation = ['Detailed information too verbose, truncated']
-
-    return [summary] + explanation
-
-
-def _diff_text(left, right):
-    """Return the explanation for the diff between text
-
-    This will skip leading and trailing characters which are
-    identical to keep the diff minimal.
-    """
-    explanation = []
-    i = 0 # just in case left or right has zero length
-    for i in range(min(len(left), len(right))):
-        if left[i] != right[i]:
-            break
-    if i > 42:
-        i -= 10                 # Provide some context
-        explanation = ['Skipping %s identical '
-                       'leading characters in diff' % i]
-        left = left[i:]
-        right = right[i:]
-    if len(left) == len(right):
-        for i in range(len(left)):
-            if left[-i] != right[-i]:
-                break
-        if i > 42:
-            i -= 10     # Provide some context
-            explanation += ['Skipping %s identical '
-                            'trailing characters in diff' % i]
-            left = left[:-i]
-            right = right[:-i]
-    explanation += [line.strip('\n')
-                    for line in py.std.difflib.ndiff(left.splitlines(),
-                                                     right.splitlines())]
-    return explanation
-
-
-def _compare_eq_sequence(left, right):
-    explanation = []
-    for i in range(min(len(left), len(right))):
-        if left[i] != right[i]:
-            explanation += ['At index %s diff: %r != %r' %
-                            (i, left[i], right[i])]
-            break
-    if len(left) > len(right):
-        explanation += ['Left contains more items, '
-            'first extra item: %s' % py.io.saferepr(left[len(right)],)]
-    elif len(left) < len(right):
-        explanation += ['Right contains more items, '
-            'first extra item: %s' % py.io.saferepr(right[len(left)],)]
-    return explanation # + _diff_text(py.std.pprint.pformat(left),
-                       #             py.std.pprint.pformat(right))
-
-
-def _compare_eq_set(left, right):
-    explanation = []
-    diff_left = left - right
-    diff_right = right - left
-    if diff_left:
-        explanation.append('Extra items in the left set:')
-        for item in diff_left:
-            explanation.append(py.io.saferepr(item))
-    if diff_right:
-        explanation.append('Extra items in the right set:')
-        for item in diff_right:
-            explanation.append(py.io.saferepr(item))
-    return explanation
-
-
-def _notin_text(term, text):
-    index = text.find(term)
-    head = text[:index]
-    tail = text[index+len(term):]
-    correct_text = head + tail
-    diff = _diff_text(correct_text, text)
-    newdiff = ['%s is contained here:' % py.io.saferepr(term, maxsize=42)]
-    for line in diff:
-        if line.startswith('Skipping'):
-            continue
-        if line.startswith('- '):
-            continue
-        if line.startswith('+ '):
-            newdiff.append('  ' + line[2:])
-        else:
-            newdiff.append(line)
-    return newdiff
diff --git a/_pytest/assertion/__init__.py b/_pytest/assertion/__init__.py
new file mode 100644
--- /dev/null
+++ b/_pytest/assertion/__init__.py
@@ -0,0 +1,128 @@
+"""
+support for presenting detailed information in failing assertions.
+"""
+import py
+import imp
+import marshal
+import struct
+import sys
+import pytest
+from _pytest.monkeypatch import monkeypatch
+from _pytest.assertion import reinterpret, util
+
+try:
+    from _pytest.assertion.rewrite import rewrite_asserts
+except ImportError:
+    rewrite_asserts = None
+else:
+    import ast
+
+def pytest_addoption(parser):
+    group = parser.getgroup("debugconfig")
+    group.addoption('--assertmode', action="store", dest="assertmode",
+                    choices=("on", "old", "off", "default"), default="default",
+                    metavar="on|old|off",
+                    help="""control assertion debugging tools.
+'off' performs no assertion debugging.
+'old' reinterprets the expressions in asserts to glean information.
+'on' (the default) rewrites the assert statements in test modules to provide
+sub-expression results.""")
+    group.addoption('--no-assert', action="store_true", default=False,
+        dest="noassert", help="DEPRECATED equivalent to --assertmode=off")
+    group.addoption('--nomagic', action="store_true", default=False,
+        dest="nomagic", help="DEPRECATED equivalent to --assertmode=off")
+
+class AssertionState:
+    """State for the assertion plugin."""
+
+    def __init__(self, config, mode):
+        self.mode = mode
+        self.trace = config.trace.root.get("assertion")
+
+def pytest_configure(config):
+    warn_about_missing_assertion()
+    mode = config.getvalue("assertmode")
+    if config.getvalue("noassert") or config.getvalue("nomagic"):
+        if mode not in ("off", "default"):
+            raise pytest.UsageError("assertion options conflict")
+        mode = "off"
+    elif mode == "default":
+        mode = "on"
+    if mode != "off":
+        def callbinrepr(op, left, right):
+            hook_result = config.hook.pytest_assertrepr_compare(
+                config=config, op=op, left=left, right=right)
+            for new_expl in hook_result:
+                if new_expl:
+                    return '\n~'.join(new_expl)
+        m = monkeypatch()
+        config._cleanup.append(m.undo)
+        m.setattr(py.builtin.builtins, 'AssertionError',
+                  reinterpret.AssertionError)
+        m.setattr(util, '_reprcompare', callbinrepr)
+    if mode == "on" and rewrite_asserts is None:
+        mode = "old"
+    config._assertstate = AssertionState(config, mode)
+    config._assertstate.trace("configured with mode set to %r" % (mode,))
+
+def _write_pyc(co, source_path):
+    if hasattr(imp, "cache_from_source"):
+        # Handle PEP 3147 pycs.
+        pyc = py.path.local(imp.cache_from_source(str(source_path)))
+        pyc.ensure()
+    else:
+        pyc = source_path + "c"
+    mtime = int(source_path.mtime())
+    fp = pyc.open("wb")
+    try:
+        fp.write(imp.get_magic())
+        fp.write(struct.pack(">",
+    ast.Add : "+",
+    ast.Sub : "-",
+    ast.Mult : "*",
+    ast.Div : "/",
+    ast.FloorDiv : "//",
+    ast.Mod : "%",
+    ast.Eq : "==",
+    ast.NotEq : "!=",
+    ast.Lt : "<",
+    ast.LtE : "<=",
+    ast.Gt : ">",
+    ast.GtE : ">=",
+    ast.Pow : "**",
+    ast.Is : "is",
+    ast.IsNot : "is not",
+    ast.In : "in",
+    ast.NotIn : "not in"
+}
+
+unary_map = {
+    ast.Not : "not %s",
+    ast.Invert : "~%s",
+    ast.USub : "-%s",
+    ast.UAdd : "+%s"
+}
+
+
+class DebugInterpreter(ast.NodeVisitor):
+    """Interpret AST nodes to gleam useful debugging information. """
+
+    def __init__(self, frame):
+        self.frame = frame
+
+    def generic_visit(self, node):
+        # Fallback when we don't have a special implementation.
+        if _is_ast_expr(node):
+            mod = ast.Expression(node)
+            co = self._compile(mod)
+            try:
+                result = self.frame.eval(co)
+            except Exception:
+                raise Failure()
+            explanation = self.frame.repr(result)
+            return explanation, result
+        elif _is_ast_stmt(node):
+            mod = ast.Module([node])
+            co = self._compile(mod, "exec")
+            try:
+                self.frame.exec_(co)
+            except Exception:
+                raise Failure()
+            return None, None
+        else:
+            raise AssertionError("can't handle %s" %(node,))
+
+    def _compile(self, source, mode="eval"):
+        return compile(source, "", mode)
+
+    def visit_Expr(self, expr):
+        return self.visit(expr.value)
+
+    def visit_Module(self, mod):
+        for stmt in mod.body:
+            self.visit(stmt)
+
+    def visit_Name(self, name):
+        explanation, result = self.generic_visit(name)
+        # See if the name is local.
+        source = "%r in locals() is not globals()" % (name.id,)
+        co = self._compile(source)
+        try:
+            local = self.frame.eval(co)
+        except Exception:
+            # have to assume it isn't
+            local = None
+        if local is None or not self.frame.is_true(local):
+            return name.id, result
+        return explanation, result
+
+    def visit_Compare(self, comp):
+        left = comp.left
+        left_explanation, left_result = self.visit(left)
+        for op, next_op in zip(comp.ops, comp.comparators):
+            next_explanation, next_result = self.visit(next_op)
+            op_symbol = operator_map[op.__class__]
+            explanation = "%s %s %s" % (left_explanation, op_symbol,
+                                        next_explanation)
+            source = "__exprinfo_left %s __exprinfo_right" % (op_symbol,)
+            co = self._compile(source)
+            try:
+                result = self.frame.eval(co, __exprinfo_left=left_result,
+                                         __exprinfo_right=next_result)
+            except Exception:
+                raise Failure(explanation)
+            try:
+                if not self.frame.is_true(result):
+                    break
+            except KeyboardInterrupt:
+                raise
+            except:
+                break
+            left_explanation, left_result = next_explanation, next_result
+
+        if util._reprcompare is not None:
+            res = util._reprcompare(op_symbol, left_result, next_result)
+            if res:
+                explanation = res
+        return explanation, result
+
+    def visit_BoolOp(self, boolop):
+        is_or = isinstance(boolop.op, ast.Or)
+        explanations = []
+        for operand in boolop.values:
+            explanation, result = self.visit(operand)
+            explanations.append(explanation)
+            if result == is_or:
+                break
+        name = is_or and " or " or " and "
+        explanation = "(" + name.join(explanations) + ")"
+        return explanation, result
+
+    def visit_UnaryOp(self, unary):
+        pattern = unary_map[unary.op.__class__]
+        operand_explanation, operand_result = self.visit(unary.operand)
+        explanation = pattern % (operand_explanation,)
+        co = self._compile(pattern % ("__exprinfo_expr",))
+        try:
+            result = self.frame.eval(co, __exprinfo_expr=operand_result)
+        except Exception:
+            raise Failure(explanation)
+        return explanation, result
+
+    def visit_BinOp(self, binop):
+        left_explanation, left_result = self.visit(binop.left)
+        right_explanation, right_result = self.visit(binop.right)
+        symbol = operator_map[binop.op.__class__]
+        explanation = "(%s %s %s)" % (left_explanation, symbol,
+                                      right_explanation)
+        source = "__exprinfo_left %s __exprinfo_right" % (symbol,)
+        co = self._compile(source)
+        try:
+            result = self.frame.eval(co, __exprinfo_left=left_result,
+                                     __exprinfo_right=right_result)
+        except Exception:
+            raise Failure(explanation)
+        return explanation, result
+
+    def visit_Call(self, call):
+        func_explanation, func = self.visit(call.func)
+        arg_explanations = []
+        ns = {"__exprinfo_func" : func}
+        arguments = []
+        for arg in call.args:
+            arg_explanation, arg_result = self.visit(arg)
+            arg_name = "__exprinfo_%s" % (len(ns),)
+            ns[arg_name] = arg_result
+            arguments.append(arg_name)
+            arg_explanations.append(arg_explanation)
+        for keyword in call.keywords:
+            arg_explanation, arg_result = self.visit(keyword.value)
+            arg_name = "__exprinfo_%s" % (len(ns),)
+            ns[arg_name] = arg_result
+            keyword_source = "%s=%%s" % (keyword.arg)
+            arguments.append(keyword_source % (arg_name,))
+            arg_explanations.append(keyword_source % (arg_explanation,))
+        if call.starargs:
+            arg_explanation, arg_result = self.visit(call.starargs)
+            arg_name = "__exprinfo_star"
+            ns[arg_name] = arg_result
+            arguments.append("*%s" % (arg_name,))
+            arg_explanations.append("*%s" % (arg_explanation,))
+        if call.kwargs:
+            arg_explanation, arg_result = self.visit(call.kwargs)
+            arg_name = "__exprinfo_kwds"
+            ns[arg_name] = arg_result
+            arguments.append("**%s" % (arg_name,))
+            arg_explanations.append("**%s" % (arg_explanation,))
+        args_explained = ", ".join(arg_explanations)
+        explanation = "%s(%s)" % (func_explanation, args_explained)
+        args = ", ".join(arguments)
+        source = "__exprinfo_func(%s)" % (args,)
+        co = self._compile(source)
+        try:
+            result = self.frame.eval(co, **ns)
+        except Exception:
+            raise Failure(explanation)
+        pattern = "%s\n{%s = %s\n}"
+        rep = self.frame.repr(result)
+        explanation = pattern % (rep, rep, explanation)
+        return explanation, result
+
+    def _is_builtin_name(self, name):
+        pattern = "%r not in globals() and %r not in locals()"
+        source = pattern % (name.id, name.id)
+        co = self._compile(source)
+        try:
+            return self.frame.eval(co)
+        except Exception:
+            return False
+
+    def visit_Attribute(self, attr):
+        if not isinstance(attr.ctx, ast.Load):
+            return self.generic_visit(attr)
+        source_explanation, source_result = self.visit(attr.value)
+        explanation = "%s.%s" % (source_explanation, attr.attr)
+        source = "__exprinfo_expr.%s" % (attr.attr,)
+        co = self._compile(source)
+        try:
+            result = self.frame.eval(co, __exprinfo_expr=source_result)
+        except Exception:
+            raise Failure(explanation)
+        explanation = "%s\n{%s = %s.%s\n}" % (self.frame.repr(result),
+                                              self.frame.repr(result),
+                                              source_explanation, attr.attr)
+        # Check if the attr is from an instance.
+        source = "%r in getattr(__exprinfo_expr, '__dict__', {})"
+        source = source % (attr.attr,)
+        co = self._compile(source)
+        try:
+            from_instance = self.frame.eval(co, __exprinfo_expr=source_result)
+        except Exception:
+            from_instance = None
+        if from_instance is None or self.frame.is_true(from_instance):
+            rep = self.frame.repr(result)
+            pattern = "%s\n{%s = %s\n}"
+            explanation = pattern % (rep, rep, explanation)
+        return explanation, result
+
+    def visit_Assert(self, assrt):
+        test_explanation, test_result = self.visit(assrt.test)
+        explanation = "assert %s" % (test_explanation,)
+        if not self.frame.is_true(test_result):
+            try:
+                raise BuiltinAssertionError
+            except Exception:
+                raise Failure(explanation)
+        return explanation, test_result
+
+    def visit_Assign(self, assign):
+        value_explanation, value_result = self.visit(assign.value)
+        explanation = "... = %s" % (value_explanation,)
+        name = ast.Name("__exprinfo_expr", ast.Load(),
+                        lineno=assign.value.lineno,
+                        col_offset=assign.value.col_offset)
+        new_assign = ast.Assign(assign.targets, name, lineno=assign.lineno,
+                                col_offset=assign.col_offset)
+        mod = ast.Module([new_assign])
+        co = self._compile(mod, "exec")
+        try:
+            self.frame.exec_(co, __exprinfo_expr=value_result)
+        except Exception:
+            raise Failure(explanation)
+        return explanation, value_result
diff --git a/_pytest/assertion/oldinterpret.py b/_pytest/assertion/oldinterpret.py
new file mode 100644
--- /dev/null
+++ b/_pytest/assertion/oldinterpret.py
@@ -0,0 +1,552 @@
+import py
+import sys, inspect
+from compiler import parse, ast, pycodegen
+from _pytest.assertion.util import format_explanation
+from _pytest.assertion.reinterpret import BuiltinAssertionError
+
+passthroughex = py.builtin._sysex
+
+class Failure:
+    def __init__(self, node):
+        self.exc, self.value, self.tb = sys.exc_info()
+        self.node = node
+
+class View(object):
+    """View base class.
+
+    If C is a subclass of View, then C(x) creates a proxy object around
+    the object x.  The actual class of the proxy is not C in general,
+    but a *subclass* of C determined by the rules below.  To avoid confusion
+    we call view class the class of the proxy (a subclass of C, so of View)
+    and object class the class of x.
+
+    Attributes and methods not found in the proxy are automatically read on x.
+    Other operations like setting attributes are performed on the proxy, as
+    determined by its view class.  The object x is available from the proxy
+    as its __obj__ attribute.
+
+    The view class selection is determined by the __view__ tuples and the
+    optional __viewkey__ method.  By default, the selected view class is the
+    most specific subclass of C whose __view__ mentions the class of x.
+    If no such subclass is found, the search proceeds with the parent
+    object classes.  For example, C(True) will first look for a subclass
+    of C with __view__ = (..., bool, ...) and only if it doesn't find any
+    look for one with __view__ = (..., int, ...), and then ..., object,...
+    If everything fails the class C itself is considered to be the default.
+
+    Alternatively, the view class selection can be driven by another aspect
+    of the object x, instead of the class of x, by overriding __viewkey__.
+    See last example at the end of this module.
+    """
+
+    _viewcache = {}
+    __view__ = ()
+
+    def __new__(rootclass, obj, *args, **kwds):
+        self = object.__new__(rootclass)
+        self.__obj__ = obj
+        self.__rootclass__ = rootclass
+        key = self.__viewkey__()
+        try:
+            self.__class__ = self._viewcache[key]
+        except KeyError:
+            self.__class__ = self._selectsubclass(key)
+        return self
+
+    def __getattr__(self, attr):
+        # attributes not found in the normal hierarchy rooted on View
+        # are looked up in the object's real class
+        return getattr(self.__obj__, attr)
+
+    def __viewkey__(self):
+        return self.__obj__.__class__
+
+    def __matchkey__(self, key, subclasses):
+        if inspect.isclass(key):
+            keys = inspect.getmro(key)
+        else:
+            keys = [key]
+        for key in keys:
+            result = [C for C in subclasses if key in C.__view__]
+            if result:
+                return result
+        return []
+
+    def _selectsubclass(self, key):
+        subclasses = list(enumsubclasses(self.__rootclass__))
+        for C in subclasses:
+            if not isinstance(C.__view__, tuple):
+                C.__view__ = (C.__view__,)
+        choices = self.__matchkey__(key, subclasses)
+        if not choices:
+            return self.__rootclass__
+        elif len(choices) == 1:
+            return choices[0]
+        else:
+            # combine the multiple choices
+            return type('?', tuple(choices), {})
+
+    def __repr__(self):
+        return '%s(%r)' % (self.__rootclass__.__name__, self.__obj__)
+
+
+def enumsubclasses(cls):
+    for subcls in cls.__subclasses__():
+        for subsubclass in enumsubclasses(subcls):
+            yield subsubclass
+    yield cls
+
+
+class Interpretable(View):
+    """A parse tree node with a few extra methods."""
+    explanation = None
+
+    def is_builtin(self, frame):
+        return False
+
+    def eval(self, frame):
+        # fall-back for unknown expression nodes
+        try:
+            expr = ast.Expression(self.__obj__)
+            expr.filename = ''
+            self.__obj__.filename = ''
+            co = pycodegen.ExpressionCodeGenerator(expr).getCode()
+            result = frame.eval(co)
+        except passthroughex:
+            raise
+        except:
+            raise Failure(self)
+        self.result = result
+        self.explanation = self.explanation or frame.repr(self.result)
+
+    def run(self, frame):
+        # fall-back for unknown statement nodes
+        try:
+            expr = ast.Module(None, ast.Stmt([self.__obj__]))
+            expr.filename = ''
+            co = pycodegen.ModuleCodeGenerator(expr).getCode()
+            frame.exec_(co)
+        except passthroughex:
+            raise
+        except:
+            raise Failure(self)
+
+    def nice_explanation(self):
+        return format_explanation(self.explanation)
+
+
+class Name(Interpretable):
+    __view__ = ast.Name
+
+    def is_local(self, frame):
+        source = '%r in locals() is not globals()' % self.name
+        try:
+            return frame.is_true(frame.eval(source))
+        except passthroughex:
+            raise
+        except:
+            return False
+
+    def is_global(self, frame):
+        source = '%r in globals()' % self.name
+        try:
+            return frame.is_true(frame.eval(source))
+        except passthroughex:
+            raise
+        except:
+            return False
+
+    def is_builtin(self, frame):
+        source = '%r not in locals() and %r not in globals()' % (
+            self.name, self.name)
+        try:
+            return frame.is_true(frame.eval(source))
+        except passthroughex:
+            raise
+        except:
+            return False
+
+    def eval(self, frame):
+        super(Name, self).eval(frame)
+        if not self.is_local(frame):
+            self.explanation = self.name
+
+class Compare(Interpretable):
+    __view__ = ast.Compare
+
+    def eval(self, frame):
+        expr = Interpretable(self.expr)
+        expr.eval(frame)
+        for operation, expr2 in self.ops:
+            if hasattr(self, 'result'):
+                # shortcutting in chained expressions
+                if not frame.is_true(self.result):
+                    break
+            expr2 = Interpretable(expr2)
+            expr2.eval(frame)
+            self.explanation = "%s %s %s" % (
+                expr.explanation, operation, expr2.explanation)
+            source = "__exprinfo_left %s __exprinfo_right" % operation
+            try:
+                self.result = frame.eval(source,
+                                         __exprinfo_left=expr.result,
+                                         __exprinfo_right=expr2.result)
+            except passthroughex:
+                raise
+            except:
+                raise Failure(self)
+            expr = expr2
+
+class And(Interpretable):
+    __view__ = ast.And
+
+    def eval(self, frame):
+        explanations = []
+        for expr in self.nodes:
+            expr = Interpretable(expr)
+            expr.eval(frame)
+            explanations.append(expr.explanation)
+            self.result = expr.result
+            if not frame.is_true(expr.result):
+                break
+        self.explanation = '(' + ' and '.join(explanations) + ')'
+
+class Or(Interpretable):
+    __view__ = ast.Or
+
+    def eval(self, frame):
+        explanations = []
+        for expr in self.nodes:
+            expr = Interpretable(expr)
+            expr.eval(frame)
+            explanations.append(expr.explanation)
+            self.result = expr.result
+            if frame.is_true(expr.result):
+                break
+        self.explanation = '(' + ' or '.join(explanations) + ')'
+
+
+# == Unary operations ==
+keepalive = []
+for astclass, astpattern in {
+    ast.Not    : 'not __exprinfo_expr',
+    ast.Invert : '(~__exprinfo_expr)',
+    }.items():
+
+    class UnaryArith(Interpretable):
+        __view__ = astclass
+
+        def eval(self, frame, astpattern=astpattern):
+            expr = Interpretable(self.expr)
+            expr.eval(frame)
+            self.explanation = astpattern.replace('__exprinfo_expr',
+                                                  expr.explanation)
+            try:
+                self.result = frame.eval(astpattern,
+                                         __exprinfo_expr=expr.result)
+            except passthroughex:
+                raise
+            except:
+                raise Failure(self)
+
+    keepalive.append(UnaryArith)
+
+# == Binary operations ==
+for astclass, astpattern in {
+    ast.Add    : '(__exprinfo_left + __exprinfo_right)',
+    ast.Sub    : '(__exprinfo_left - __exprinfo_right)',
+    ast.Mul    : '(__exprinfo_left * __exprinfo_right)',
+    ast.Div    : '(__exprinfo_left / __exprinfo_right)',
+    ast.Mod    : '(__exprinfo_left % __exprinfo_right)',
+    ast.Power  : '(__exprinfo_left ** __exprinfo_right)',
+    }.items():
+
+    class BinaryArith(Interpretable):
+        __view__ = astclass
+
+        def eval(self, frame, astpattern=astpattern):
+            left = Interpretable(self.left)
+            left.eval(frame)
+            right = Interpretable(self.right)
+            right.eval(frame)
+            self.explanation = (astpattern
+                                .replace('__exprinfo_left',  left .explanation)
+                                .replace('__exprinfo_right', right.explanation))
+            try:
+                self.result = frame.eval(astpattern,
+                                         __exprinfo_left=left.result,
+                                         __exprinfo_right=right.result)
+            except passthroughex:
+                raise
+            except:
+                raise Failure(self)
+
+    keepalive.append(BinaryArith)
+
+
+class CallFunc(Interpretable):
+    __view__ = ast.CallFunc
+
+    def is_bool(self, frame):
+        source = 'isinstance(__exprinfo_value, bool)'
+        try:
+            return frame.is_true(frame.eval(source,
+                                            __exprinfo_value=self.result))
+        except passthroughex:
+            raise
+        except:
+            return False
+
+    def eval(self, frame):
+        node = Interpretable(self.node)
+        node.eval(frame)
+        explanations = []
+        vars = {'__exprinfo_fn': node.result}
+        source = '__exprinfo_fn('
+        for a in self.args:
+            if isinstance(a, ast.Keyword):
+                keyword = a.name
+                a = a.expr
+            else:
+                keyword = None
+            a = Interpretable(a)
+            a.eval(frame)
+            argname = '__exprinfo_%d' % len(vars)
+            vars[argname] = a.result
+            if keyword is None:
+                source += argname + ','
+                explanations.append(a.explanation)
+            else:
+                source += '%s=%s,' % (keyword, argname)
+                explanations.append('%s=%s' % (keyword, a.explanation))
+        if self.star_args:
+            star_args = Interpretable(self.star_args)
+            star_args.eval(frame)
+            argname = '__exprinfo_star'
+            vars[argname] = star_args.result
+            source += '*' + argname + ','
+            explanations.append('*' + star_args.explanation)
+        if self.dstar_args:
+            dstar_args = Interpretable(self.dstar_args)
+            dstar_args.eval(frame)
+            argname = '__exprinfo_kwds'
+            vars[argname] = dstar_args.result
+            source += '**' + argname + ','
+            explanations.append('**' + dstar_args.explanation)
+        self.explanation = "%s(%s)" % (
+            node.explanation, ', '.join(explanations))
+        if source.endswith(','):
+            source = source[:-1]
+        source += ')'
+        try:
+            self.result = frame.eval(source, **vars)
+        except passthroughex:
+            raise
+        except:
+            raise Failure(self)
+        if not node.is_builtin(frame) or not self.is_bool(frame):
+            r = frame.repr(self.result)
+            self.explanation = '%s\n{%s = %s\n}' % (r, r, self.explanation)
+
+class Getattr(Interpretable):
+    __view__ = ast.Getattr
+
+    def eval(self, frame):
+        expr = Interpretable(self.expr)
+        expr.eval(frame)
+        source = '__exprinfo_expr.%s' % self.attrname
+        try:
+            self.result = frame.eval(source, __exprinfo_expr=expr.result)
+        except passthroughex:
+            raise
+        except:
+            raise Failure(self)
+        self.explanation = '%s.%s' % (expr.explanation, self.attrname)
+        # if the attribute comes from the instance, its value is interesting
+        source = ('hasattr(__exprinfo_expr, "__dict__") and '
+                  '%r in __exprinfo_expr.__dict__' % self.attrname)
+        try:
+            from_instance = frame.is_true(
+                frame.eval(source, __exprinfo_expr=expr.result))
+        except passthroughex:
+            raise
+        except:
+            from_instance = True
+        if from_instance:
+            r = frame.repr(self.result)
+            self.explanation = '%s\n{%s = %s\n}' % (r, r, self.explanation)
+
+# == Re-interpretation of full statements ==
+
+class Assert(Interpretable):
+    __view__ = ast.Assert
+
+    def run(self, frame):
+        test = Interpretable(self.test)
+        test.eval(frame)
+        # print the result as  'assert '
+        self.result = test.result
+        self.explanation = 'assert ' + test.explanation
+        if not frame.is_true(test.result):
+            try:
+                raise BuiltinAssertionError
+            except passthroughex:
+                raise
+            except:
+                raise Failure(self)
+
+class Assign(Interpretable):
+    __view__ = ast.Assign
+
+    def run(self, frame):
+        expr = Interpretable(self.expr)
+        expr.eval(frame)
+        self.result = expr.result
+        self.explanation = '... = ' + expr.explanation
+        # fall-back-run the rest of the assignment
+        ass = ast.Assign(self.nodes, ast.Name('__exprinfo_expr'))
+        mod = ast.Module(None, ast.Stmt([ass]))
+        mod.filename = ''
+        co = pycodegen.ModuleCodeGenerator(mod).getCode()
+        try:
+            frame.exec_(co, __exprinfo_expr=expr.result)
+        except passthroughex:
+            raise
+        except:
+            raise Failure(self)
+
+class Discard(Interpretable):
+    __view__ = ast.Discard
+
+    def run(self, frame):
+        expr = Interpretable(self.expr)
+        expr.eval(frame)
+        self.result = expr.result
+        self.explanation = expr.explanation
+
+class Stmt(Interpretable):
+    __view__ = ast.Stmt
+
+    def run(self, frame):
+        for stmt in self.nodes:
+            stmt = Interpretable(stmt)
+            stmt.run(frame)
+
+
+def report_failure(e):
+    explanation = e.node.nice_explanation()
+    if explanation:
+        explanation = ", in: " + explanation
+    else:
+        explanation = ""
+    sys.stdout.write("%s: %s%s\n" % (e.exc.__name__, e.value, explanation))
+
+def check(s, frame=None):
+    if frame is None:
+        frame = sys._getframe(1)
+        frame = py.code.Frame(frame)
+    expr = parse(s, 'eval')
+    assert isinstance(expr, ast.Expression)
+    node = Interpretable(expr.node)
+    try:
+        node.eval(frame)
+    except passthroughex:
+        raise
+    except Failure:
+        e = sys.exc_info()[1]
+        report_failure(e)
+    else:
+        if not frame.is_true(node.result):
+            sys.stderr.write("assertion failed: %s\n" % node.nice_explanation())
+
+
+###########################################################
+# API / Entry points
+# #########################################################
+
+def interpret(source, frame, should_fail=False):
+    module = Interpretable(parse(source, 'exec').node)
+    #print "got module", module
+    if isinstance(frame, py.std.types.FrameType):
+        frame = py.code.Frame(frame)
+    try:
+        module.run(frame)
+    except Failure:
+        e = sys.exc_info()[1]
+        return getfailure(e)
+    except passthroughex:
+        raise
+    except:
+        import traceback
+        traceback.print_exc()
+    if should_fail:
+        return ("(assertion failed, but when it was re-run for "
+                "printing intermediate values, it did not fail.  Suggestions: "
+                "compute assert expression before the assert or use --nomagic)")
+    else:
+        return None
+
+def getmsg(excinfo):
+    if isinstance(excinfo, tuple):
+        excinfo = py.code.ExceptionInfo(excinfo)
+    #frame, line = gettbline(tb)
+    #frame = py.code.Frame(frame)
+    #return interpret(line, frame)
+
+    tb = excinfo.traceback[-1]
+    source = str(tb.statement).strip()
+    x = interpret(source, tb.frame, should_fail=True)
+    if not isinstance(x, str):
+        raise TypeError("interpret returned non-string %r" % (x,))
+    return x
+
+def getfailure(e):
+    explanation = e.node.nice_explanation()
+    if str(e.value):
+        lines = explanation.split('\n')
+        lines[0] += "  << %s" % (e.value,)
+        explanation = '\n'.join(lines)
+    text = "%s: %s" % (e.exc.__name__, explanation)
+    if text.startswith('AssertionError: assert '):
+        text = text[16:]
+    return text
+
+def run(s, frame=None):
+    if frame is None:
+        frame = sys._getframe(1)
+        frame = py.code.Frame(frame)
+    module = Interpretable(parse(s, 'exec').node)
+    try:
+        module.run(frame)
+    except Failure:
+        e = sys.exc_info()[1]
+        report_failure(e)
+
+
+if __name__ == '__main__':
+    # example:
+    def f():
+        return 5
+    def g():
+        return 3
+    def h(x):
+        return 'never'
+    check("f() * g() == 5")
+    check("not f()")
+    check("not (f() and g() or 0)")
+    check("f() == g()")
+    i = 4
+    check("i == f()")
+    check("len(f()) == 0")
+    check("isinstance(2+3+4, float)")
+
+    run("x = i")
+    check("x == 5")
+
+    run("assert not f(), 'oops'")
+    run("a, b, c = 1, 2")
+    run("a, b, c = f()")
+
+    check("max([f(),g()]) == 4")
+    check("'hello'[g()] == 'h'")
+    run("'guk%d' % h(f())")
diff --git a/_pytest/assertion/reinterpret.py b/_pytest/assertion/reinterpret.py
new file mode 100644
--- /dev/null
+++ b/_pytest/assertion/reinterpret.py
@@ -0,0 +1,48 @@
+import sys
+import py
+
+BuiltinAssertionError = py.builtin.builtins.AssertionError
+
+class AssertionError(BuiltinAssertionError):
+    def __init__(self, *args):
+        BuiltinAssertionError.__init__(self, *args)
+        if args:
+            try:
+                self.msg = str(args[0])
+            except py.builtin._sysex:
+                raise
+            except:
+                self.msg = "<[broken __repr__] %s at %0xd>" %(
+                    args[0].__class__, id(args[0]))
+        else:
+            f = py.code.Frame(sys._getframe(1))
+            try:
+                source = f.code.fullsource
+                if source is not None:
+                    try:
+                        source = source.getstatement(f.lineno, assertion=True)
+                    except IndexError:
+                        source = None
+                    else:
+                        source = str(source.deindent()).strip()
+            except py.error.ENOENT:
+                source = None
+                # this can also occur during reinterpretation, when the
+                # co_filename is set to "".
+            if source:
+                self.msg = reinterpret(source, f, should_fail=True)
+            else:
+                self.msg = ""
+            if not self.args:
+                self.args = (self.msg,)
+
+if sys.version_info > (3, 0):
+    AssertionError.__module__ = "builtins"
+    reinterpret_old = "old reinterpretation not available for py3"
+else:
+    from _pytest.assertion.oldinterpret import interpret as reinterpret_old
+if sys.version_info >= (2, 6) or (sys.platform.startswith("java")):
+    from _pytest.assertion.newinterpret import interpret as reinterpret
+else:
+    reinterpret = reinterpret_old
+
diff --git a/_pytest/assertion/rewrite.py b/_pytest/assertion/rewrite.py
new file mode 100644
--- /dev/null
+++ b/_pytest/assertion/rewrite.py
@@ -0,0 +1,340 @@
+"""Rewrite assertion AST to produce nice error messages"""
+
+import ast
+import collections
+import itertools
+import sys
+
+import py
+from _pytest.assertion import util
+
+
+def rewrite_asserts(mod):
+    """Rewrite the assert statements in mod."""
+    AssertionRewriter().run(mod)
+
+
+_saferepr = py.io.saferepr
+from _pytest.assertion.util import format_explanation as _format_explanation
+
+def _format_boolop(operands, explanations, is_or):
+    show_explanations = []
+    for operand, expl in zip(operands, explanations):
+        show_explanations.append(expl)
+        if operand == is_or:
+            break
+    return "(" + (is_or and " or " or " and ").join(show_explanations) + ")"
+
+def _call_reprcompare(ops, results, expls, each_obj):
+    for i, res, expl in zip(range(len(ops)), results, expls):
+        try:
+            done = not res
+        except Exception:
+            done = True
+        if done:
+            break
+    if util._reprcompare is not None:
+        custom = util._reprcompare(ops[i], each_obj[i], each_obj[i + 1])
+        if custom is not None:
+            return custom
+    return expl
+
+
+unary_map = {
+    ast.Not : "not %s",
+    ast.Invert : "~%s",
+    ast.USub : "-%s",
+    ast.UAdd : "+%s"
+}
+
+binop_map = {
+    ast.BitOr : "|",
+    ast.BitXor : "^",
+    ast.BitAnd : "&",
+    ast.LShift : "<<",
+    ast.RShift : ">>",
+    ast.Add : "+",
+    ast.Sub : "-",
+    ast.Mult : "*",
+    ast.Div : "/",
+    ast.FloorDiv : "//",
+    ast.Mod : "%",
+    ast.Eq : "==",
+    ast.NotEq : "!=",
+    ast.Lt : "<",
+    ast.LtE : "<=",
+    ast.Gt : ">",
+    ast.GtE : ">=",
+    ast.Pow : "**",
+    ast.Is : "is",
+    ast.IsNot : "is not",
+    ast.In : "in",
+    ast.NotIn : "not in"
+}
+
+
+def set_location(node, lineno, col_offset):
+    """Set node location information recursively."""
+    def _fix(node, lineno, col_offset):
+        if "lineno" in node._attributes:
+            node.lineno = lineno
+        if "col_offset" in node._attributes:
+            node.col_offset = col_offset
+        for child in ast.iter_child_nodes(node):
+            _fix(child, lineno, col_offset)
+    _fix(node, lineno, col_offset)
+    return node
+
+
+class AssertionRewriter(ast.NodeVisitor):
+
+    def run(self, mod):
+        """Find all assert statements in *mod* and rewrite them."""
+        if not mod.body:
+            # Nothing to do.
+            return
+        # Insert some special imports at the top of the module but after any
+        # docstrings and __future__ imports.
+        aliases = [ast.alias(py.builtin.builtins.__name__, "@py_builtins"),
+                   ast.alias("_pytest.assertion.rewrite", "@pytest_ar")]
+        expect_docstring = True
+        pos = 0
+        lineno = 0
+        for item in mod.body:
+            if (expect_docstring and isinstance(item, ast.Expr) and
+                isinstance(item.value, ast.Str)):
+                doc = item.value.s
+                if "PYTEST_DONT_REWRITE" in doc:
+                    # The module has disabled assertion rewriting.
+                    return
+                lineno += len(doc) - 1
+                expect_docstring = False
+            elif (not isinstance(item, ast.ImportFrom) or item.level > 0 and
+                  item.identifier != "__future__"):
+                lineno = item.lineno
+                break
+            pos += 1
+        imports = [ast.Import([alias], lineno=lineno, col_offset=0)
+                   for alias in aliases]
+        mod.body[pos:pos] = imports
+        # Collect asserts.
+        nodes = collections.deque([mod])
+        while nodes:
+            node = nodes.popleft()
+            for name, field in ast.iter_fields(node):
+                if isinstance(field, list):
+                    new = []
+                    for i, child in enumerate(field):
+                        if isinstance(child, ast.Assert):
+                            # Transform assert.
+                            new.extend(self.visit(child))
+                        else:
+                            new.append(child)
+                            if isinstance(child, ast.AST):
+                                nodes.append(child)
+                    setattr(node, name, new)
+                elif (isinstance(field, ast.AST) and
+                      # Don't recurse into expressions as they can't contain
+                      # asserts.
+                      not isinstance(field, ast.expr)):
+                    nodes.append(field)
+
+    def variable(self):
+        """Get a new variable."""
+        # Use a character invalid in python identifiers to avoid clashing.
+        name = "@py_assert" + str(next(self.variable_counter))
+        self.variables.add(name)
+        return name
+
+    def assign(self, expr):
+        """Give *expr* a name."""
+        name = self.variable()
+        self.statements.append(ast.Assign([ast.Name(name, ast.Store())], expr))
+        return ast.Name(name, ast.Load())
+
+    def display(self, expr):
+        """Call py.io.saferepr on the expression."""
+        return self.helper("saferepr", expr)
+
+    def helper(self, name, *args):
+        """Call a helper in this module."""
+        py_name = ast.Name("@pytest_ar", ast.Load())
+        attr = ast.Attribute(py_name, "_" + name, ast.Load())
+        return ast.Call(attr, list(args), [], None, None)
+
+    def builtin(self, name):
+        """Return the builtin called *name*."""
+        builtin_name = ast.Name("@py_builtins", ast.Load())
+        return ast.Attribute(builtin_name, name, ast.Load())
+
+    def explanation_param(self, expr):
+        specifier = "py" + str(next(self.variable_counter))
+        self.explanation_specifiers[specifier] = expr
+        return "%(" + specifier + ")s"
+
+    def push_format_context(self):
+        self.explanation_specifiers = {}
+        self.stack.append(self.explanation_specifiers)
+
+    def pop_format_context(self, expl_expr):
+        current = self.stack.pop()
+        if self.stack:
+            self.explanation_specifiers = self.stack[-1]
+        keys = [ast.Str(key) for key in current.keys()]
+        format_dict = ast.Dict(keys, list(current.values()))
+        form = ast.BinOp(expl_expr, ast.Mod(), format_dict)
+        name = "@py_format" + str(next(self.variable_counter))
+        self.on_failure.append(ast.Assign([ast.Name(name, ast.Store())], form))
+        return ast.Name(name, ast.Load())
+
+    def generic_visit(self, node):
+        """Handle expressions we don't have custom code for."""
+        assert isinstance(node, ast.expr)
+        res = self.assign(node)
+        return res, self.explanation_param(self.display(res))
+
+    def visit_Assert(self, assert_):
+        if assert_.msg:
+            # There's already a message. Don't mess with it.
+            return [assert_]
+        self.statements = []
+        self.variables = set()
+        self.variable_counter = itertools.count()
+        self.stack = []
+        self.on_failure = []
+        self.push_format_context()
+        # Rewrite assert into a bunch of statements.
+        top_condition, explanation = self.visit(assert_.test)
+        # Create failure message.
+        body = self.on_failure
+        negation = ast.UnaryOp(ast.Not(), top_condition)
+        self.statements.append(ast.If(negation, body, []))
+        explanation = "assert " + explanation
+        template = ast.Str(explanation)
+        msg = self.pop_format_context(template)
+        fmt = self.helper("format_explanation", msg)
+        err_name = ast.Name("AssertionError", ast.Load())
+        exc = ast.Call(err_name, [fmt], [], None, None)
+        if sys.version_info[0] >= 3:
+            raise_ = ast.Raise(exc, None)
+        else:
+            raise_ = ast.Raise(exc, None, None)
+        body.append(raise_)
+        # Delete temporary variables.
+        names = [ast.Name(name, ast.Del()) for name in self.variables]
+        if names:
+            delete = ast.Delete(names)
+            self.statements.append(delete)
+        # Fix line numbers.
+        for stmt in self.statements:
+            set_location(stmt, assert_.lineno, assert_.col_offset)
+        return self.statements
+
+    def visit_Name(self, name):
+        # Check if the name is local or not.
+        locs = ast.Call(self.builtin("locals"), [], [], None, None)
+        globs = ast.Call(self.builtin("globals"), [], [], None, None)
+        ops = [ast.In(), ast.IsNot()]
+        test = ast.Compare(ast.Str(name.id), ops, [locs, globs])
+        expr = ast.IfExp(test, self.display(name), ast.Str(name.id))
+        return name, self.explanation_param(expr)
+
+    def visit_BoolOp(self, boolop):
+        operands = []
+        explanations = []
+        self.push_format_context()
+        for operand in boolop.values:
+            res, explanation = self.visit(operand)
+            operands.append(res)
+            explanations.append(explanation)
+        expls = ast.Tuple([ast.Str(expl) for expl in explanations], ast.Load())
+        is_or = ast.Num(isinstance(boolop.op, ast.Or))
+        expl_template = self.helper("format_boolop",
+                                    ast.Tuple(operands, ast.Load()), expls,
+                                    is_or)
+        expl = self.pop_format_context(expl_template)
+        res = self.assign(ast.BoolOp(boolop.op, operands))
+        return res, self.explanation_param(expl)
+
+    def visit_UnaryOp(self, unary):
+        pattern = unary_map[unary.op.__class__]
+        operand_res, operand_expl = self.visit(unary.operand)
+        res = self.assign(ast.UnaryOp(unary.op, operand_res))
+        return res, pattern % (operand_expl,)
+
+    def visit_BinOp(self, binop):
+        symbol = binop_map[binop.op.__class__]
+        left_expr, left_expl = self.visit(binop.left)
+        right_expr, right_expl = self.visit(binop.right)
+        explanation = "(%s %s %s)" % (left_expl, symbol, right_expl)
+        res = self.assign(ast.BinOp(left_expr, binop.op, right_expr))
+        return res, explanation
+
+    def visit_Call(self, call):
+        new_func, func_expl = self.visit(call.func)
+        arg_expls = []
+        new_args = []
+        new_kwargs = []
+        new_star = new_kwarg = None
+        for arg in call.args:
+            res, expl = self.visit(arg)
+            new_args.append(res)
+            arg_expls.append(expl)
+        for keyword in call.keywords:
+            res, expl = self.visit(keyword.value)
+            new_kwargs.append(ast.keyword(keyword.arg, res))
+            arg_expls.append(keyword.arg + "=" + expl)
+        if call.starargs:
+            new_star, expl = self.visit(call.starargs)
+            arg_expls.append("*" + expl)
+        if call.kwargs:
+            new_kwarg, expl = self.visit(call.kwarg)
+            arg_expls.append("**" + expl)
+        expl = "%s(%s)" % (func_expl, ', '.join(arg_expls))
+        new_call = ast.Call(new_func, new_args, new_kwargs, new_star, new_kwarg)
+        res = self.assign(new_call)
+        res_expl = self.explanation_param(self.display(res))
+        outer_expl = "%s\n{%s = %s\n}" % (res_expl, res_expl, expl)
+        return res, outer_expl
+
+    def visit_Attribute(self, attr):
+        if not isinstance(attr.ctx, ast.Load):
+            return self.generic_visit(attr)
+        value, value_expl = self.visit(attr.value)
+        res = self.assign(ast.Attribute(value, attr.attr, ast.Load()))
+        res_expl = self.explanation_param(self.display(res))
+        pat = "%s\n{%s = %s.%s\n}"
+        expl = pat % (res_expl, res_expl, value_expl, attr.attr)
+        return res, expl
+
+    def visit_Compare(self, comp):
+        self.push_format_context()
+        left_res, left_expl = self.visit(comp.left)
+        res_variables = [self.variable() for i in range(len(comp.ops))]
+        load_names = [ast.Name(v, ast.Load()) for v in res_variables]
+        store_names = [ast.Name(v, ast.Store()) for v in res_variables]
+        it = zip(range(len(comp.ops)), comp.ops, comp.comparators)
+        expls = []
+        syms = []
+        results = [left_res]
+        for i, op, next_operand in it:
+            next_res, next_expl = self.visit(next_operand)
+            results.append(next_res)
+            sym = binop_map[op.__class__]
+            syms.append(ast.Str(sym))
+            expl = "%s %s %s" % (left_expl, sym, next_expl)
+            expls.append(ast.Str(expl))
+            res_expr = ast.Compare(left_res, [op], [next_res])
+            self.statements.append(ast.Assign([store_names[i]], res_expr))
+            left_res, left_expl = next_res, next_expl
+        # Use py.code._reprcompare if that's available.
+        expl_call = self.helper("call_reprcompare",
+                                ast.Tuple(syms, ast.Load()),
+                                ast.Tuple(load_names, ast.Load()),
+                                ast.Tuple(expls, ast.Load()),
+                                ast.Tuple(results, ast.Load()))
+        if len(comp.ops) > 1:
+            res = ast.BoolOp(ast.And(), load_names)
+        else:
+            res = load_names[0]
+        return res, self.explanation_param(self.pop_format_context(expl_call))
diff --git a/_pytest/assertion/util.py b/_pytest/assertion/util.py
new file mode 100644
--- /dev/null
+++ b/_pytest/assertion/util.py
@@ -0,0 +1,213 @@
+"""Utilities for assertion debugging"""
+
+import py
+
+
+# The _reprcompare attribute on the util module is used by the new assertion
+# interpretation code and assertion rewriter to detect this plugin was
+# loaded and in turn call the hooks defined here as part of the
+# DebugInterpreter.
+_reprcompare = None
+
+def format_explanation(explanation):
+    """This formats an explanation
+
+    Normally all embedded newlines are escaped, however there are
+    three exceptions: \n{, \n} and \n~.  The first two are intended
+    cover nested explanations, see function and attribute explanations
+    for examples (.visit_Call(), visit_Attribute()).  The last one is
+    for when one explanation needs to span multiple lines, e.g. when
+    displaying diffs.
+    """
+    # simplify 'assert False where False = ...'
+    where = 0
+    while True:
+        start = where = explanation.find("False\n{False = ", where)
+        if where == -1:
+            break
+        level = 0
+        for i, c in enumerate(explanation[start:]):
+            if c == "{":
+                level += 1
+            elif c == "}":
+                level -= 1
+                if not level:
+                    break
+        else:
+            raise AssertionError("unbalanced braces: %r" % (explanation,))
+        end = start + i
+        where = end
+        if explanation[end - 1] == '\n':
+            explanation = (explanation[:start] + explanation[start+15:end-1] +
+                           explanation[end+1:])
+            where -= 17
+    raw_lines = (explanation or '').split('\n')
+    # escape newlines not followed by {, } and ~
+    lines = [raw_lines[0]]
+    for l in raw_lines[1:]:
+        if l.startswith('{') or l.startswith('}') or l.startswith('~'):
+            lines.append(l)
+        else:
+            lines[-1] += '\\n' + l
+
+    result = lines[:1]
+    stack = [0]
+    stackcnt = [0]
+    for line in lines[1:]:
+        if line.startswith('{'):
+            if stackcnt[-1]:
+                s = 'and   '
+            else:
+                s = 'where '
+            stack.append(len(result))
+            stackcnt[-1] += 1
+            stackcnt.append(0)
+            result.append(' +' + '  '*(len(stack)-1) + s + line[1:])
+        elif line.startswith('}'):
+            assert line.startswith('}')
+            stack.pop()
+            stackcnt.pop()
+            result[stack[-1]] += line[1:]
+        else:
+            assert line.startswith('~')
+            result.append('  '*len(stack) + line[1:])
+    assert len(stack) == 1
+    return '\n'.join(result)
+
+
+# Provide basestring in python3
+try:
+    basestring = basestring
+except NameError:
+    basestring = str
+
+
+def assertrepr_compare(op, left, right):
+    """return specialised explanations for some operators/operands"""
+    width = 80 - 15 - len(op) - 2 # 15 chars indentation, 1 space around op
+    left_repr = py.io.saferepr(left, maxsize=int(width/2))
+    right_repr = py.io.saferepr(right, maxsize=width-len(left_repr))
+    summary = '%s %s %s' % (left_repr, op, right_repr)
+
+    issequence = lambda x: isinstance(x, (list, tuple))
+    istext = lambda x: isinstance(x, basestring)
+    isdict = lambda x: isinstance(x, dict)
+    isset = lambda x: isinstance(x, set)
+
+    explanation = None
+    try:
+        if op == '==':
+            if istext(left) and istext(right):
+                explanation = _diff_text(left, right)
+            elif issequence(left) and issequence(right):
+                explanation = _compare_eq_sequence(left, right)
+            elif isset(left) and isset(right):
+                explanation = _compare_eq_set(left, right)
+            elif isdict(left) and isdict(right):
+                explanation = _diff_text(py.std.pprint.pformat(left),
+                                         py.std.pprint.pformat(right))
+        elif op == 'not in':
+            if istext(left) and istext(right):
+                explanation = _notin_text(left, right)
+    except py.builtin._sysex:
+        raise
+    except:
+        excinfo = py.code.ExceptionInfo()
+        explanation = ['(pytest_assertion plugin: representation of '
+            'details failed. Probably an object has a faulty __repr__.)',
+            str(excinfo)
+            ]
+
+
+    if not explanation:
+        return None
+
+    # Don't include pageloads of data, should be configurable
+    if len(''.join(explanation)) > 80*8:
+        explanation = ['Detailed information too verbose, truncated']
+
+    return [summary] + explanation
+
+
+def _diff_text(left, right):
+    """Return the explanation for the diff between text
+
+    This will skip leading and trailing characters which are
+    identical to keep the diff minimal.
+    """
+    explanation = []
+    i = 0 # just in case left or right has zero length
+    for i in range(min(len(left), len(right))):
+        if left[i] != right[i]:
+            break
+    if i > 42:
+        i -= 10                 # Provide some context
+        explanation = ['Skipping %s identical '
+                       'leading characters in diff' % i]
+        left = left[i:]
+        right = right[i:]
+    if len(left) == len(right):
+        for i in range(len(left)):
+            if left[-i] != right[-i]:
+                break
+        if i > 42:
+            i -= 10     # Provide some context
+            explanation += ['Skipping %s identical '
+                            'trailing characters in diff' % i]
+            left = left[:-i]
+            right = right[:-i]
+    explanation += [line.strip('\n')
+                    for line in py.std.difflib.ndiff(left.splitlines(),
+                                                     right.splitlines())]
+    return explanation
+
+
+def _compare_eq_sequence(left, right):
+    explanation = []
+    for i in range(min(len(left), len(right))):
+        if left[i] != right[i]:
+            explanation += ['At index %s diff: %r != %r' %
+                            (i, left[i], right[i])]
+            break
+    if len(left) > len(right):
+        explanation += ['Left contains more items, '
+            'first extra item: %s' % py.io.saferepr(left[len(right)],)]
+    elif len(left) < len(right):
+        explanation += ['Right contains more items, '
+            'first extra item: %s' % py.io.saferepr(right[len(left)],)]
+    return explanation # + _diff_text(py.std.pprint.pformat(left),
+                       #             py.std.pprint.pformat(right))
+
+
+def _compare_eq_set(left, right):
+    explanation = []
+    diff_left = left - right
+    diff_right = right - left
+    if diff_left:
+        explanation.append('Extra items in the left set:')
+        for item in diff_left:
+            explanation.append(py.io.saferepr(item))
+    if diff_right:
+        explanation.append('Extra items in the right set:')
+        for item in diff_right:
+            explanation.append(py.io.saferepr(item))
+    return explanation
+
+
+def _notin_text(term, text):
+    index = text.find(term)
+    head = text[:index]
+    tail = text[index+len(term):]
+    correct_text = head + tail
+    diff = _diff_text(correct_text, text)
+    newdiff = ['%s is contained here:' % py.io.saferepr(term, maxsize=42)]
+    for line in diff:
+        if line.startswith('Skipping'):
+            continue
+        if line.startswith('- '):
+            continue
+        if line.startswith('+ '):
+            newdiff.append('  ' + line[2:])
+        else:
+            newdiff.append(line)
+    return newdiff
diff --git a/_pytest/doctest.py b/_pytest/doctest.py
--- a/_pytest/doctest.py
+++ b/_pytest/doctest.py
@@ -59,7 +59,7 @@
                 inner_excinfo = py.code.ExceptionInfo(excinfo.value.exc_info)
                 lines += ["UNEXPECTED EXCEPTION: %s" %
                             repr(inner_excinfo.value)]
-
+                lines += py.std.traceback.format_exception(*excinfo.value.exc_info)
             return ReprFailDoctest(reprlocation, lines)
         else:
             return super(DoctestItem, self).repr_failure(excinfo)
diff --git a/_pytest/helpconfig.py b/_pytest/helpconfig.py
--- a/_pytest/helpconfig.py
+++ b/_pytest/helpconfig.py
@@ -16,9 +16,6 @@
     group.addoption('--traceconfig',
                action="store_true", dest="traceconfig", default=False,
                help="trace considerations of conftest.py files."),
-    group._addoption('--nomagic',
-               action="store_true", dest="nomagic", default=False,
-               help="don't reinterpret asserts, no traceback cutting. ")
     group.addoption('--debug',
                action="store_true", dest="debug", default=False,
                help="generate and show internal debugging information.")
diff --git a/_pytest/junitxml.py b/_pytest/junitxml.py
--- a/_pytest/junitxml.py
+++ b/_pytest/junitxml.py
@@ -65,7 +65,8 @@
 
 class LogXML(object):
     def __init__(self, logfile, prefix):
-        self.logfile = logfile
+        logfile = os.path.expanduser(os.path.expandvars(logfile))
+        self.logfile = os.path.normpath(logfile)
         self.prefix = prefix
         self.test_logs = []
         self.passed = self.skipped = 0
@@ -76,7 +77,7 @@
         names = report.nodeid.split("::")
         names[0] = names[0].replace("/", '.')
         names = tuple(names)
-        d = {'time': self._durations.pop(names, "0")}
+        d = {'time': self._durations.pop(report.nodeid, "0")}
         names = [x.replace(".py", "") for x in names if x != "()"]
         classnames = names[:-1]
         if self.prefix:
@@ -170,12 +171,11 @@
             self.append_skipped(report)
 
     def pytest_runtest_call(self, item, __multicall__):
-        names = tuple(item.listnames())
         start = time.time()
         try:
             return __multicall__.execute()
         finally:
-            self._durations[names] = time.time() - start
+            self._durations[item.nodeid] = time.time() - start
 
     def pytest_collectreport(self, report):
         if not report.passed:
diff --git a/_pytest/main.py b/_pytest/main.py
--- a/_pytest/main.py
+++ b/_pytest/main.py
@@ -46,23 +46,25 @@
 
 
 def pytest_namespace():
-    return dict(collect=dict(Item=Item, Collector=Collector, File=File))
+    collect = dict(Item=Item, Collector=Collector, File=File, Session=Session)
+    return dict(collect=collect)
         
 def pytest_configure(config):
     py.test.config = config # compatibiltiy
     if config.option.exitfirst:
         config.option.maxfail = 1
 
-def pytest_cmdline_main(config):
-    """ default command line protocol for initialization, session,
-    running tests and reporting. """
+def wrap_session(config, doit):
+    """Skeleton command line program"""
     session = Session(config)
     session.exitstatus = EXIT_OK
+    initstate = 0
     try:
         config.pluginmanager.do_configure(config)
+        initstate = 1
         config.hook.pytest_sessionstart(session=session)
-        config.hook.pytest_collection(session=session)
-        config.hook.pytest_runtestloop(session=session)
+        initstate = 2
+        doit(config, session)
     except pytest.UsageError:
         raise
     except KeyboardInterrupt:
@@ -77,18 +79,24 @@
             sys.stderr.write("mainloop: caught Spurious SystemExit!\n")
     if not session.exitstatus and session._testsfailed:
         session.exitstatus = EXIT_TESTSFAILED
-    config.hook.pytest_sessionfinish(session=session,
-        exitstatus=session.exitstatus)
-    config.pluginmanager.do_unconfigure(config)
+    if initstate >= 2:
+        config.hook.pytest_sessionfinish(session=session,
+            exitstatus=session.exitstatus)
+    if initstate >= 1:
+        config.pluginmanager.do_unconfigure(config)
     return session.exitstatus
 
+def pytest_cmdline_main(config):
+    return wrap_session(config, _main)
+
+def _main(config, session):
+    """ default command line protocol for initialization, session,
+    running tests and reporting. """
+    config.hook.pytest_collection(session=session)
+    config.hook.pytest_runtestloop(session=session)
+
 def pytest_collection(session):
-    session.perform_collect()
-    hook = session.config.hook
-    hook.pytest_collection_modifyitems(session=session,
-        config=session.config, items=session.items)
-    hook.pytest_collection_finish(session=session)
-    return True
+    return session.perform_collect()
 
 def pytest_runtestloop(session):
     if session.config.option.collectonly:
@@ -374,6 +382,16 @@
         return HookProxy(fspath, self.config)
 
     def perform_collect(self, args=None, genitems=True):
+        hook = self.config.hook
+        try:
+            items = self._perform_collect(args, genitems)
+            hook.pytest_collection_modifyitems(session=self,
+                config=self.config, items=items)
+        finally:
+            hook.pytest_collection_finish(session=self)
+        return items
+
+    def _perform_collect(self, args, genitems):
         if args is None:
             args = self.config.args
         self.trace("perform_collect", self, args)
diff --git a/_pytest/mark.py b/_pytest/mark.py
--- a/_pytest/mark.py
+++ b/_pytest/mark.py
@@ -153,7 +153,7 @@
 
     def __repr__(self):
         return "" % (
-                self._name, self.args, self.kwargs)
+                self.name, self.args, self.kwargs)
 
 def pytest_itemcollected(item):
     if not isinstance(item, pytest.Function):
diff --git a/_pytest/pytester.py b/_pytest/pytester.py
--- a/_pytest/pytester.py
+++ b/_pytest/pytester.py
@@ -6,7 +6,7 @@
 import inspect
 import time
 from fnmatch import fnmatch
-from _pytest.main import Session
+from _pytest.main import Session, EXIT_OK
 from py.builtin import print_
 from _pytest.core import HookRelay
 
@@ -292,13 +292,19 @@
         assert '::' not in str(arg)
         p = py.path.local(arg)
         x = session.fspath.bestrelpath(p)
-        return session.perform_collect([x], genitems=False)[0]
+        config.hook.pytest_sessionstart(session=session)
+        res = session.perform_collect([x], genitems=False)[0]
+        config.hook.pytest_sessionfinish(session=session, exitstatus=EXIT_OK)
+        return res
 
     def getpathnode(self, path):
-        config = self.parseconfig(path)
+        config = self.parseconfigure(path)
         session = Session(config)
         x = session.fspath.bestrelpath(path)
-        return session.perform_collect([x], genitems=False)[0]
+        config.hook.pytest_sessionstart(session=session)
+        res = session.perform_collect([x], genitems=False)[0]
+        config.hook.pytest_sessionfinish(session=session, exitstatus=EXIT_OK)
+        return res
 
     def genitems(self, colitems):
         session = colitems[0].session
@@ -312,7 +318,9 @@
         config = self.parseconfigure(*args)
         rec = self.getreportrecorder(config)
         session = Session(config)
+        config.hook.pytest_sessionstart(session=session)
         session.perform_collect()
+        config.hook.pytest_sessionfinish(session=session, exitstatus=EXIT_OK)
         return session.items, rec
 
     def runitem(self, source):
@@ -382,6 +390,8 @@
             c.basetemp = py.path.local.make_numbered_dir(prefix="reparse",
                 keep=0, rootdir=self.tmpdir, lock_timeout=None)
             c.parse(args)
+            c.pluginmanager.do_configure(c)
+            self.request.addfinalizer(lambda: c.pluginmanager.do_unconfigure(c))
             return c
         finally:
             py.test.config = oldconfig
diff --git a/_pytest/python.py b/_pytest/python.py
--- a/_pytest/python.py
+++ b/_pytest/python.py
@@ -226,8 +226,13 @@
 
     def _importtestmodule(self):
         # we assume we are only called once per module
+        from _pytest import assertion
+        assertion.before_module_import(self)
         try:
-            mod = self.fspath.pyimport(ensuresyspath=True)
+            try:
+                mod = self.fspath.pyimport(ensuresyspath=True)
+            finally:
+                assertion.after_module_import(self)
         except SyntaxError:
             excinfo = py.code.ExceptionInfo()
             raise self.CollectError(excinfo.getrepr(style="short"))
@@ -374,7 +379,7 @@
         # test generators are seen as collectors but they also
         # invoke setup/teardown on popular request
         # (induced by the common "test_*" naming shared with normal tests)
-        self.config._setupstate.prepare(self)
+        self.session._setupstate.prepare(self)
         # see FunctionMixin.setup and test_setupstate_is_preserved_134
         self._preservedparent = self.parent.obj
         l = []
@@ -721,7 +726,7 @@
 
     def _addfinalizer(self, finalizer, scope):
         colitem = self._getscopeitem(scope)
-        self.config._setupstate.addfinalizer(
+        self._pyfuncitem.session._setupstate.addfinalizer(
             finalizer=finalizer, colitem=colitem)
 
     def __repr__(self):
@@ -742,8 +747,10 @@
         raise self.LookupError(msg)
 
 def showfuncargs(config):
-    from _pytest.main import Session
-    session = Session(config)
+    from _pytest.main import wrap_session
+    return wrap_session(config, _showfuncargs_main)
+
+def _showfuncargs_main(config, session):
     session.perform_collect()
     if session.items:
         plugins = session.items[0].getplugins()
diff --git a/_pytest/runner.py b/_pytest/runner.py
--- a/_pytest/runner.py
+++ b/_pytest/runner.py
@@ -14,17 +14,15 @@
 #
 # pytest plugin hooks
 
-# XXX move to pytest_sessionstart and fix py.test owns tests
-def pytest_configure(config):
-    config._setupstate = SetupState()
+def pytest_sessionstart(session):
+    session._setupstate = SetupState()
 
 def pytest_sessionfinish(session, exitstatus):
-    if hasattr(session.config, '_setupstate'):
-        hook = session.config.hook
-        rep = hook.pytest__teardown_final(session=session)
-        if rep:
-            hook.pytest__teardown_final_logerror(session=session, report=rep)
-            session.exitstatus = 1
+    hook = session.config.hook
+    rep = hook.pytest__teardown_final(session=session)
+    if rep:
+        hook.pytest__teardown_final_logerror(session=session, report=rep)
+        session.exitstatus = 1
 
 class NodeInfo:
     def __init__(self, location):
@@ -46,16 +44,16 @@
     return reports
 
 def pytest_runtest_setup(item):
-    item.config._setupstate.prepare(item)
+    item.session._setupstate.prepare(item)
 
 def pytest_runtest_call(item):
     item.runtest()
 
 def pytest_runtest_teardown(item):
-    item.config._setupstate.teardown_exact(item)
+    item.session._setupstate.teardown_exact(item)
 
 def pytest__teardown_final(session):
-    call = CallInfo(session.config._setupstate.teardown_all, when="teardown")
+    call = CallInfo(session._setupstate.teardown_all, when="teardown")
     if call.excinfo:
         ntraceback = call.excinfo.traceback .cut(excludepath=py._pydir)
         call.excinfo.traceback = ntraceback.filter()
diff --git a/ctypes_configure/configure.py b/ctypes_configure/configure.py
--- a/ctypes_configure/configure.py
+++ b/ctypes_configure/configure.py
@@ -559,7 +559,9 @@
 C_HEADER = """
 #include 
 #include    /* for offsetof() */
-#include    /* FreeBSD: for uint64_t */
+#ifndef _WIN32
+#  include    /* FreeBSD: for uint64_t */
+#endif
 
 void dump(char* key, int value) {
     printf("%s: %d\\n", key, value);
diff --git a/ctypes_configure/stdoutcapture.py b/ctypes_configure/stdoutcapture.py
--- a/ctypes_configure/stdoutcapture.py
+++ b/ctypes_configure/stdoutcapture.py
@@ -15,6 +15,15 @@
             not hasattr(os, 'fdopen')):
             self.dummy = 1
         else:
+            try:
+                self.tmpout = os.tmpfile()
+                if mixed_out_err:
+                    self.tmperr = self.tmpout
+                else:
+                    self.tmperr = os.tmpfile()
+            except OSError:     # bah?  on at least one Windows box
+                self.dummy = 1
+                return
             self.dummy = 0
             # make new stdout/stderr files if needed
             self.localoutfd = os.dup(1)
@@ -29,11 +38,6 @@
                 sys.stderr = os.fdopen(self.localerrfd, 'w', 0)
             else:
                 self.saved_stderr = None
-            self.tmpout = os.tmpfile()
-            if mixed_out_err:
-                self.tmperr = self.tmpout
-            else:
-                self.tmperr = os.tmpfile()
             os.dup2(self.tmpout.fileno(), 1)
             os.dup2(self.tmperr.fileno(), 2)
 
diff --git a/dotviewer/graphparse.py b/dotviewer/graphparse.py
--- a/dotviewer/graphparse.py
+++ b/dotviewer/graphparse.py
@@ -36,48 +36,45 @@
     print >> sys.stderr, "Warning: could not guess file type, using 'dot'"
     return 'unknown'
 
-def dot2plain(content, contenttype, use_codespeak=False):
-    if contenttype == 'plain':
-        # already a .plain file
-        return content
+def dot2plain_graphviz(content, contenttype, use_codespeak=False):
+    if contenttype != 'neato':
+        cmdline = 'dot -Tplain'
+    else:
+        cmdline = 'neato -Tplain'
+    #print >> sys.stderr, '* running:', cmdline
+    close_fds = sys.platform != 'win32'
+    p = subprocess.Popen(cmdline, shell=True, close_fds=close_fds,
+                         stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+    (child_in, child_out) = (p.stdin, p.stdout)
+    try:
+        import thread
+    except ImportError:
+        bkgndwrite(child_in, content)
+    else:
+        thread.start_new_thread(bkgndwrite, (child_in, content))
+    plaincontent = child_out.read()
+    child_out.close()
+    if not plaincontent:    # 'dot' is likely not installed
+        raise PlainParseError("no result from running 'dot'")
+    return plaincontent
 
-    if not use_codespeak:
-        if contenttype != 'neato':
-            cmdline = 'dot -Tplain'
-        else:
-            cmdline = 'neato -Tplain'
-        #print >> sys.stderr, '* running:', cmdline
-        close_fds = sys.platform != 'win32'
-        p = subprocess.Popen(cmdline, shell=True, close_fds=close_fds,
-                             stdin=subprocess.PIPE, stdout=subprocess.PIPE)
-        (child_in, child_out) = (p.stdin, p.stdout)
-        try:
-            import thread
-        except ImportError:
-            bkgndwrite(child_in, content)
-        else:
-            thread.start_new_thread(bkgndwrite, (child_in, content))
-        plaincontent = child_out.read()
-        child_out.close()
-        if not plaincontent:    # 'dot' is likely not installed
-            raise PlainParseError("no result from running 'dot'")
-    else:
-        import urllib
-        request = urllib.urlencode({'dot': content})
-        url = 'http://codespeak.net/pypy/convertdot.cgi'
-        print >> sys.stderr, '* posting:', url
-        g = urllib.urlopen(url, data=request)
-        result = []
-        while True:
-            data = g.read(16384)
-            if not data:
-                break
-            result.append(data)
-        g.close()
-        plaincontent = ''.join(result)
-        # very simple-minded way to give a somewhat better error message
-        if plaincontent.startswith('> sys.stderr, '* posting:', url
+    g = urllib.urlopen(url, data=request)
+    result = []
+    while True:
+        data = g.read(16384)
+        if not data:
+            break
+        result.append(data)
+    g.close()
+    plaincontent = ''.join(result)
+    # very simple-minded way to give a somewhat better error message
+    if plaincontent.startswith('"
+    return _PROTOCOL_NAMES.get(protocol_code, '')
 
 
 # a replacement for the old socket.ssl function
diff --git a/lib-python/2.7/test/test_os.py b/lib-python/2.7/test/test_os.py
--- a/lib-python/2.7/test/test_os.py
+++ b/lib-python/2.7/test/test_os.py
@@ -74,7 +74,8 @@
         self.assertFalse(os.path.exists(name),
                     "file already exists for temporary file")
         # make sure we can create the file
-        open(name, "w")
+        f = open(name, "w")
+        f.close()
         self.files.append(name)
 
     def test_tempnam(self):
diff --git a/lib-python/2.7/test/test_ssl.py b/lib-python/2.7/test/test_ssl.py
--- a/lib-python/2.7/test/test_ssl.py
+++ b/lib-python/2.7/test/test_ssl.py
@@ -58,32 +58,35 @@
 
 # Issue #9415: Ubuntu hijacks their OpenSSL and forcefully disables SSLv2
 def skip_if_broken_ubuntu_ssl(func):
-    # We need to access the lower-level wrapper in order to create an
-    # implicit SSL context without trying to connect or listen.
-    try:
-        import _ssl
-    except ImportError:
-        # The returned function won't get executed, just ignore the error
-        pass
-    @functools.wraps(func)
-    def f(*args, **kwargs):
+    if hasattr(ssl, 'PROTOCOL_SSLv2'):
+        # We need to access the lower-level wrapper in order to create an
+        # implicit SSL context without trying to connect or listen.
         try:
-            s = socket.socket(socket.AF_INET)
-            _ssl.sslwrap(s._sock, 0, None, None,
-                         ssl.CERT_NONE, ssl.PROTOCOL_SSLv2, None, None)
-        except ssl.SSLError as e:
-            if (ssl.OPENSSL_VERSION_INFO == (0, 9, 8, 15, 15) and
-                platform.linux_distribution() == ('debian', 'squeeze/sid', '')
-                and 'Invalid SSL protocol variant specified' in str(e)):
-                raise unittest.SkipTest("Patched Ubuntu OpenSSL breaks behaviour")
-        return func(*args, **kwargs)
-    return f
+            import _ssl
+        except ImportError:
+            # The returned function won't get executed, just ignore the error
+            pass
+        @functools.wraps(func)
+        def f(*args, **kwargs):
+            try:
+                s = socket.socket(socket.AF_INET)
+                _ssl.sslwrap(s._sock, 0, None, None,
+                             ssl.CERT_NONE, ssl.PROTOCOL_SSLv2, None, None)
+            except ssl.SSLError as e:
+                if (ssl.OPENSSL_VERSION_INFO == (0, 9, 8, 15, 15) and
+                    platform.linux_distribution() == ('debian', 'squeeze/sid', '')
+                    and 'Invalid SSL protocol variant specified' in str(e)):
+                    raise unittest.SkipTest("Patched Ubuntu OpenSSL breaks behaviour")
+            return func(*args, **kwargs)
+        return f
+    else:
+        return func
 
 
 class BasicSocketTests(unittest.TestCase):
 
     def test_constants(self):
-        ssl.PROTOCOL_SSLv2
+        #ssl.PROTOCOL_SSLv2
         ssl.PROTOCOL_SSLv23
         ssl.PROTOCOL_SSLv3
         ssl.PROTOCOL_TLSv1
@@ -964,7 +967,8 @@
             try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, True)
             try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, True, ssl.CERT_OPTIONAL)
             try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, True, ssl.CERT_REQUIRED)
-            try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv2, False)
+            if hasattr(ssl, 'PROTOCOL_SSLv2'):
+                try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv2, False)
             try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv23, False)
             try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_TLSv1, False)
 
@@ -976,7 +980,8 @@
             try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, True)
             try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, True, ssl.CERT_OPTIONAL)
             try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, True, ssl.CERT_REQUIRED)
-            try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv2, False)
+            if hasattr(ssl, 'PROTOCOL_SSLv2'):
+                try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv2, False)
             try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv3, False)
             try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv23, False)
 
diff --git a/lib-python/TODO b/lib-python/TODO
deleted file mode 100644
--- a/lib-python/TODO
+++ /dev/null
@@ -1,100 +0,0 @@
-TODO list for 2.7.0
-===================
-
-You can find the results of the most recent buildbot run at:
-http://buildbot.pypy.org/
-
-
-Probably easy tasks
--------------------
-
-- (unicode|bytearray).(index|find) should accept None as indices (see
-  test_unicode.py)
-
-- missing posix.confstr and posix.confstr_names
-
-- remove code duplication: bit_length() and _count_bits() in rlib/rbigint.py,
-  objspace/std/longobject.py and objspace/std/longtype.py.
-
-- missing module pyexpat.errors
-
-- support for PYTHONIOENCODING, this needs a way to update file.encoding
-
-- implement format__Complex_ANY() in pypy/objspace/std/complexobject.py
-
-- Code like this does not work, for two reasons::
-
-  \
-  from __future__ import (with_statement,
-                          unicode_literals)
-  assert type("") is unicode
-
-- Code like::
-
-  assert(x is not None, "error message")
-
-  should emit a SyntaxWarning when compiled (the tuple is always true)
-
-
-Medium tasks
-------------
-
-- socket module has a couple of changes (including AF_TIPC packet range)
-
-Longer tasks
-------------
-
-- Fix usage of __cmp__ in subclasses::
-
-    class badint(int):
-        def __cmp__(self, other):
-            raise RuntimeError
-    raises(RuntimeError, cmp, 0, badint(1))
-
-- Fix comparison of objects layout: if two classes have the same __slots__, it
-  should be possible to change the instances __class__::
-
-      class A(object): __slots__ = ('a', 'b')
-      class B(object): __slots__ = ('b', 'a')
-      a = A()
-      a.__class__ = B
-
-- Show a ResourceWarning when a file/socket is not explicitely closed, like
-  CPython did for 3.2: http://svn.python.org/view?view=rev&revision=85920
-  in PyPy this should be enabled by default
-
-Won't do for this release
--------------------------
-
-Note: when you give up with a missing feature, please mention it here, as well
-as the various skips added to the test suite.
-
-- py3k warnings
-
-  * the -3 flag is accepted on the command line, but displays a warning (see
-    `translator/goal/app_main.py`)
-
-- CJK codecs.
-
-  * In `./conftest.py`, skipped all `test_codecencodings_*.py` and
-    `test_codecmaps_*.py`.
-
-  * In test_codecs, commented out various items in `all_unicode_encodings`.
-
-- Error messages about ill-formed calls (like "argument after ** must be a
-  mapping") don't always show the function name.  That's hard to fix for
-  the case of errors raised when the Argument object is created (as opposed
-  to when parsing for a given target function, which occurs later).
-
-  * Some "..." were added to doctests in test_extcall.py
-
-- CPython's builtin methods are both functions and unbound methods (for
-  example, `str.upper is dict(str.__dict__)['upper']`). This is not the case
-  in pypy, and assertions like `object.__str__ is object.__str__` are False
-  with pypy.  Use the `==` operator instead.
-
-  * pprint.py, _threading_local.py
-
-- When importing a nested module fails, the ImportError message mentions the
-  name of the package up to the component that could not be imported (CPython
-  prefers to display the names starting with the failing part).
diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -61,7 +61,7 @@
                                  usemodules = '',
                                  skip=None): 
         self.basename = basename 
-        self._usemodules = usemodules.split()
+        self._usemodules = usemodules.split() + ['signal']
         self._compiler = compiler 
         self.core = core
         self.skip = skip
@@ -154,18 +154,18 @@
     RegrTest('test_cmd.py'),
     RegrTest('test_cmd_line_script.py'),
     RegrTest('test_codeccallbacks.py', core=True),
-    RegrTest('test_codecencodings_cn.py', skip="encodings not available"),
-    RegrTest('test_codecencodings_hk.py', skip="encodings not available"),
-    RegrTest('test_codecencodings_jp.py', skip="encodings not available"),
-    RegrTest('test_codecencodings_kr.py', skip="encodings not available"),
-    RegrTest('test_codecencodings_tw.py', skip="encodings not available"),
+    RegrTest('test_codecencodings_cn.py', usemodules='_multibytecodec'),
+    RegrTest('test_codecencodings_hk.py', usemodules='_multibytecodec'),
+    RegrTest('test_codecencodings_jp.py', usemodules='_multibytecodec'),
+    RegrTest('test_codecencodings_kr.py', usemodules='_multibytecodec'),
+    RegrTest('test_codecencodings_tw.py', usemodules='_multibytecodec'),
 
-    RegrTest('test_codecmaps_cn.py', skip="encodings not available"),
-    RegrTest('test_codecmaps_hk.py', skip="encodings not available"),
-    RegrTest('test_codecmaps_jp.py', skip="encodings not available"),
-    RegrTest('test_codecmaps_kr.py', skip="encodings not available"),
-    RegrTest('test_codecmaps_tw.py', skip="encodings not available"),
-    RegrTest('test_codecs.py', core=True),
+    RegrTest('test_codecmaps_cn.py', usemodules='_multibytecodec'),
+    RegrTest('test_codecmaps_hk.py', usemodules='_multibytecodec'),
+    RegrTest('test_codecmaps_jp.py', usemodules='_multibytecodec'),
+    RegrTest('test_codecmaps_kr.py', usemodules='_multibytecodec'),
+    RegrTest('test_codecmaps_tw.py', usemodules='_multibytecodec'),
+    RegrTest('test_codecs.py', core=True, usemodules='_multibytecodec'),
     RegrTest('test_codeop.py', core=True),
     RegrTest('test_coercion.py', core=True),
     RegrTest('test_collections.py'),
@@ -201,7 +201,7 @@
     RegrTest('test_difflib.py'),
     RegrTest('test_dircache.py', core=True),
     RegrTest('test_dis.py'),
-    RegrTest('test_distutils.py'),
+    RegrTest('test_distutils.py', skip=True),
     RegrTest('test_dl.py', skip=True),
     RegrTest('test_doctest.py', usemodules="thread"),
     RegrTest('test_doctest2.py'),
@@ -314,10 +314,10 @@
     RegrTest('test_mmap.py'),
     RegrTest('test_module.py', core=True),
     RegrTest('test_modulefinder.py'),
-    RegrTest('test_multibytecodec.py', skip="unsupported codecs"),
+    RegrTest('test_multibytecodec.py', usemodules='_multibytecodec'),
     RegrTest('test_multibytecodec_support.py', skip="not a test"),
     RegrTest('test_multifile.py'),
-    RegrTest('test_multiprocessing.py', skip='FIXME leaves subprocesses'),
+    RegrTest('test_multiprocessing.py', skip="FIXME leaves subprocesses"),
     RegrTest('test_mutants.py', core="possibly"),
     RegrTest('test_mutex.py'),
     RegrTest('test_netrc.py'),
@@ -359,7 +359,7 @@
     RegrTest('test_property.py', core=True),
     RegrTest('test_pstats.py'),
     RegrTest('test_pty.py', skip="unsupported extension module"),
-    RegrTest('test_pwd.py', skip=skip_win32),
+    RegrTest('test_pwd.py', usemodules="pwd", skip=skip_win32),
     RegrTest('test_py3kwarn.py'),
     RegrTest('test_pyclbr.py'),
     RegrTest('test_pydoc.py'),
@@ -400,7 +400,7 @@
 
     RegrTest('test_softspace.py', core=True),
     RegrTest('test_sort.py', core=True),
-    RegrTest('test_ssl.py'),
+    RegrTest('test_ssl.py', usemodules='_ssl _socket select'),
     RegrTest('test_str.py', core=True),
 
     RegrTest('test_strftime.py'),
@@ -569,7 +569,6 @@
 #
 import os
 import time
-import socket
 import getpass
 
 class ReallyRunFileExternal(py.test.collect.Item): 
diff --git a/lib-python/modified-2.7/ctypes/__init__.py b/lib-python/modified-2.7/ctypes/__init__.py
--- a/lib-python/modified-2.7/ctypes/__init__.py
+++ b/lib-python/modified-2.7/ctypes/__init__.py
@@ -7,6 +7,7 @@
 
 __version__ = "1.1.0"
 
+import _ffi
 from _ctypes import Union, Structure, Array
 from _ctypes import _Pointer
 from _ctypes import CFuncPtr as _CFuncPtr
@@ -350,7 +351,7 @@
         self._FuncPtr = _FuncPtr
 
         if handle is None:
-            self._handle = _dlopen(self._name, mode)
+            self._handle = _ffi.CDLL(name, mode)
         else:
             self._handle = handle
 
@@ -488,9 +489,12 @@
         _flags_ = _FUNCFLAG_CDECL | _FUNCFLAG_PYTHONAPI
     return CFunctionType
 
-_cast = PYFUNCTYPE(py_object, c_void_p, py_object, py_object)(_cast_addr)
 def cast(obj, typ):
-    return _cast(obj, obj, typ)
+    try:
+        c_void_p.from_param(obj)
+    except TypeError, e:
+        raise ArgumentError(str(e))
+    return _cast_addr(obj, obj, typ)
 
 _string_at = PYFUNCTYPE(py_object, c_void_p, c_int)(_string_at_addr)
 def string_at(ptr, size=-1):
diff --git a/lib-python/modified-2.7/ctypes/test/test_callbacks.py b/lib-python/modified-2.7/ctypes/test/test_callbacks.py
--- a/lib-python/modified-2.7/ctypes/test/test_callbacks.py
+++ b/lib-python/modified-2.7/ctypes/test/test_callbacks.py
@@ -1,5 +1,6 @@
 import unittest
 from ctypes import *
+from ctypes.test import xfail
 import _ctypes_test
 
 class Callbacks(unittest.TestCase):
@@ -98,6 +99,7 @@
 ##        self.check_type(c_char_p, "abc")
 ##        self.check_type(c_char_p, "def")
 
+    @xfail
     def test_pyobject(self):
         o = ()
         from sys import getrefcount as grc
diff --git a/lib-python/modified-2.7/ctypes/test/test_cfuncs.py b/lib-python/modified-2.7/ctypes/test/test_cfuncs.py
--- a/lib-python/modified-2.7/ctypes/test/test_cfuncs.py
+++ b/lib-python/modified-2.7/ctypes/test/test_cfuncs.py
@@ -3,8 +3,8 @@
 
 import unittest
 from ctypes import *
-
 import _ctypes_test
+from test.test_support import impl_detail
 
 class CFunctions(unittest.TestCase):
     _dll = CDLL(_ctypes_test.__file__)
@@ -158,12 +158,14 @@
         self.assertEqual(self._dll.tf_bd(0, 42.), 14.)
         self.assertEqual(self.S(), 42)
 
+    @impl_detail('long double not supported by PyPy', pypy=False)
     def test_longdouble(self):
         self._dll.tf_D.restype = c_longdouble
         self._dll.tf_D.argtypes = (c_longdouble,)
         self.assertEqual(self._dll.tf_D(42.), 14.)
         self.assertEqual(self.S(), 42)
-
+        
+    @impl_detail('long double not supported by PyPy', pypy=False)
     def test_longdouble_plus(self):
         self._dll.tf_bD.restype = c_longdouble
         self._dll.tf_bD.argtypes = (c_byte, c_longdouble)
diff --git a/lib-python/modified-2.7/ctypes/test/test_functions.py b/lib-python/modified-2.7/ctypes/test/test_functions.py
--- a/lib-python/modified-2.7/ctypes/test/test_functions.py
+++ b/lib-python/modified-2.7/ctypes/test/test_functions.py
@@ -8,6 +8,7 @@
 from ctypes import *
 import sys, unittest
 from ctypes.test import xfail
+from test.test_support import impl_detail
 
 try:
     WINFUNCTYPE
@@ -144,6 +145,7 @@
         self.assertEqual(result, -21)
         self.assertEqual(type(result), float)
 
+    @impl_detail('long double not supported by PyPy', pypy=False)
     def test_longdoubleresult(self):
         f = dll._testfunc_D_bhilfD
         f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_longdouble]
diff --git a/lib-python/modified-2.7/ctypes/test/test_libc.py b/lib-python/modified-2.7/ctypes/test/test_libc.py
--- a/lib-python/modified-2.7/ctypes/test/test_libc.py
+++ b/lib-python/modified-2.7/ctypes/test/test_libc.py
@@ -25,7 +25,11 @@
         lib.my_qsort(chars, len(chars)-1, sizeof(c_char), comparefunc(sort))
         self.assertEqual(chars.raw, "   ,,aaaadmmmnpppsss\x00")
 
-    def test_no_more_xfail(self):
+    def SKIPPED_test_no_more_xfail(self):
+        # We decided to not explicitly support the whole ctypes-2.7
+        # and instead go for a case-by-case, demand-driven approach.
+        # So this test is skipped instead of failing.
+        import socket
         import ctypes.test
         self.assertTrue(not hasattr(ctypes.test, 'xfail'),
                         "You should incrementally grep for '@xfail' and remove them, they are real failures")
diff --git a/lib-python/modified-2.7/ctypes/test/test_simplesubclasses.py b/lib-python/modified-2.7/ctypes/test/test_simplesubclasses.py
--- a/lib-python/modified-2.7/ctypes/test/test_simplesubclasses.py
+++ b/lib-python/modified-2.7/ctypes/test/test_simplesubclasses.py
@@ -1,6 +1,5 @@
 import unittest
 from ctypes import *
-from ctypes.test import xfail
 
 class MyInt(c_int):
     def __cmp__(self, other):
@@ -27,7 +26,6 @@
         self.assertEqual(None, cb())
 
 
-    @xfail
     def test_int_callback(self):
         args = []
         def func(arg):
diff --git a/lib-python/modified-2.7/ctypes/util.py b/lib-python/modified-2.7/ctypes/util.py
--- a/lib-python/modified-2.7/ctypes/util.py
+++ b/lib-python/modified-2.7/ctypes/util.py
@@ -72,8 +72,8 @@
         return name
 
 if os.name == "posix" and sys.platform == "darwin":
-    from ctypes.macholib.dyld import dyld_find as _dyld_find
     def find_library(name):
+        from ctypes.macholib.dyld import dyld_find as _dyld_find
         possible = ['lib%s.dylib' % name,
                     '%s.dylib' % name,
                     '%s.framework/%s' % (name, name)]
diff --git a/lib-python/modified-2.7/distutils/cygwinccompiler.py b/lib-python/modified-2.7/distutils/cygwinccompiler.py
--- a/lib-python/modified-2.7/distutils/cygwinccompiler.py
+++ b/lib-python/modified-2.7/distutils/cygwinccompiler.py
@@ -75,6 +75,9 @@
         elif msc_ver == '1500':
             # VS2008 / MSVC 9.0
             return ['msvcr90']
+        elif msc_ver == '1600':
+            # VS2010 / MSVC 10.0
+            return ['msvcr100']
         else:
             raise ValueError("Unknown MS Compiler version %s " % msc_ver)
 
diff --git a/lib-python/modified-2.7/distutils/sysconfig.py b/lib-python/modified-2.7/distutils/sysconfig.py
--- a/lib-python/modified-2.7/distutils/sysconfig.py
+++ b/lib-python/modified-2.7/distutils/sysconfig.py
@@ -20,8 +20,10 @@
 if '__pypy__' in sys.builtin_module_names:
     from distutils.sysconfig_pypy import *
     from distutils.sysconfig_pypy import _config_vars # needed by setuptools
+    from distutils.sysconfig_pypy import _variable_rx # read_setup_file()
 else:
     from distutils.sysconfig_cpython import *
     from distutils.sysconfig_cpython import _config_vars # needed by setuptools
+    from distutils.sysconfig_cpython import _variable_rx # read_setup_file()
 
 
diff --git a/lib-python/modified-2.7/distutils/sysconfig_pypy.py b/lib-python/modified-2.7/distutils/sysconfig_pypy.py
--- a/lib-python/modified-2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/modified-2.7/distutils/sysconfig_pypy.py
@@ -116,3 +116,13 @@
     if compiler.compiler_type == "unix":
         compiler.compiler_so.extend(['-fPIC', '-Wimplicit'])
         compiler.shared_lib_extension = get_config_var('SO')
+        if "CFLAGS" in os.environ:
+            cflags = os.environ["CFLAGS"]
+            compiler.compiler.append(cflags)
+            compiler.compiler_so.append(cflags)
+            compiler.linker_so.append(cflags)
+
+
+from sysconfig_cpython import (
+    parse_makefile, _variable_rx, expand_makefile_vars)
+
diff --git a/lib-python/modified-2.7/distutils/unixccompiler.py b/lib-python/modified-2.7/distutils/unixccompiler.py
--- a/lib-python/modified-2.7/distutils/unixccompiler.py
+++ b/lib-python/modified-2.7/distutils/unixccompiler.py
@@ -324,7 +324,7 @@
             # On OSX users can specify an alternate SDK using
             # '-isysroot', calculate the SDK root if it is specified
             # (and use it further on)
-            cflags = sysconfig.get_config_var('CFLAGS')
+            cflags = sysconfig.get_config_var('CFLAGS') or ''
             m = re.search(r'-isysroot\s+(\S+)', cflags)
             if m is None:
                 sysroot = '/'
diff --git a/lib-python/modified-2.7/heapq.py b/lib-python/modified-2.7/heapq.py
new file mode 100644
--- /dev/null
+++ b/lib-python/modified-2.7/heapq.py
@@ -0,0 +1,442 @@
+# -*- coding: latin-1 -*-
+
+"""Heap queue algorithm (a.k.a. priority queue).
+
+Heaps are arrays for which a[k] <= a[2*k+1] and a[k] <= a[2*k+2] for
+all k, counting elements from 0.  For the sake of comparison,
+non-existing elements are considered to be infinite.  The interesting
+property of a heap is that a[0] is always its smallest element.
+
+Usage:
+
+heap = []            # creates an empty heap
+heappush(heap, item) # pushes a new item on the heap
+item = heappop(heap) # pops the smallest item from the heap
+item = heap[0]       # smallest item on the heap without popping it
+heapify(x)           # transforms list into a heap, in-place, in linear time
+item = heapreplace(heap, item) # pops and returns smallest item, and adds
+                               # new item; the heap size is unchanged
+
+Our API differs from textbook heap algorithms as follows:
+
+- We use 0-based indexing.  This makes the relationship between the
+  index for a node and the indexes for its children slightly less
+  obvious, but is more suitable since Python uses 0-based indexing.
+
+- Our heappop() method returns the smallest item, not the largest.
+
+These two make it possible to view the heap as a regular Python list
+without surprises: heap[0] is the smallest item, and heap.sort()
+maintains the heap invariant!
+"""
+
+# Original code by Kevin O'Connor, augmented by Tim Peters and Raymond Hettinger
+
+__about__ = """Heap queues
+
+[explanation by Fran�ois Pinard]
+
+Heaps are arrays for which a[k] <= a[2*k+1] and a[k] <= a[2*k+2] for
+all k, counting elements from 0.  For the sake of comparison,
+non-existing elements are considered to be infinite.  The interesting
+property of a heap is that a[0] is always its smallest element.
+
+The strange invariant above is meant to be an efficient memory
+representation for a tournament.  The numbers below are `k', not a[k]:
+
+                                   0
+
+                  1                                 2
+
+          3               4                5               6
+
+      7       8       9       10      11      12      13      14
+
+    15 16   17 18   19 20   21 22   23 24   25 26   27 28   29 30
+
+
+In the tree above, each cell `k' is topping `2*k+1' and `2*k+2'.  In
+an usual binary tournament we see in sports, each cell is the winner
+over the two cells it tops, and we can trace the winner down the tree
+to see all opponents s/he had.  However, in many computer applications
+of such tournaments, we do not need to trace the history of a winner.
+To be more memory efficient, when a winner is promoted, we try to
+replace it by something else at a lower level, and the rule becomes
+that a cell and the two cells it tops contain three different items,
+but the top cell "wins" over the two topped cells.
+
+If this heap invariant is protected at all time, index 0 is clearly
+the overall winner.  The simplest algorithmic way to remove it and
+find the "next" winner is to move some loser (let's say cell 30 in the
+diagram above) into the 0 position, and then percolate this new 0 down
+the tree, exchanging values, until the invariant is re-established.
+This is clearly logarithmic on the total number of items in the tree.
+By iterating over all items, you get an O(n ln n) sort.
+
+A nice feature of this sort is that you can efficiently insert new
+items while the sort is going on, provided that the inserted items are
+not "better" than the last 0'th element you extracted.  This is
+especially useful in simulation contexts, where the tree holds all
+incoming events, and the "win" condition means the smallest scheduled
+time.  When an event schedule other events for execution, they are
+scheduled into the future, so they can easily go into the heap.  So, a
+heap is a good structure for implementing schedulers (this is what I
+used for my MIDI sequencer :-).
+
+Various structures for implementing schedulers have been extensively
+studied, and heaps are good for this, as they are reasonably speedy,
+the speed is almost constant, and the worst case is not much different
+than the average case.  However, there are other representations which
+are more efficient overall, yet the worst cases might be terrible.
+
+Heaps are also very useful in big disk sorts.  You most probably all
+know that a big sort implies producing "runs" (which are pre-sorted
+sequences, which size is usually related to the amount of CPU memory),
+followed by a merging passes for these runs, which merging is often
+very cleverly organised[1].  It is very important that the initial
+sort produces the longest runs possible.  Tournaments are a good way
+to that.  If, using all the memory available to hold a tournament, you
+replace and percolate items that happen to fit the current run, you'll
+produce runs which are twice the size of the memory for random input,
+and much better for input fuzzily ordered.
+
+Moreover, if you output the 0'th item on disk and get an input which
+may not fit in the current tournament (because the value "wins" over
+the last output value), it cannot fit in the heap, so the size of the
+heap decreases.  The freed memory could be cleverly reused immediately
+for progressively building a second heap, which grows at exactly the
+same rate the first heap is melting.  When the first heap completely
+vanishes, you switch heaps and start a new run.  Clever and quite
+effective!
+
+In a word, heaps are useful memory structures to know.  I use them in
+a few applications, and I think it is good to keep a `heap' module
+around. :-)
+
+--------------------
+[1] The disk balancing algorithms which are current, nowadays, are
+more annoying than clever, and this is a consequence of the seeking
+capabilities of the disks.  On devices which cannot seek, like big
+tape drives, the story was quite different, and one had to be very
+clever to ensure (far in advance) that each tape movement will be the
+most effective possible (that is, will best participate at
+"progressing" the merge).  Some tapes were even able to read
+backwards, and this was also used to avoid the rewinding time.
+Believe me, real good tape sorts were quite spectacular to watch!
+From all times, sorting has always been a Great Art! :-)
+"""
+
+__all__ = ['heappush', 'heappop', 'heapify', 'heapreplace', 'merge',
+           'nlargest', 'nsmallest', 'heappushpop']
+
+from itertools import islice, repeat, count, imap, izip, tee, chain
+from operator import itemgetter
+import bisect
+
+def heappush(heap, item):
+    """Push item onto heap, maintaining the heap invariant."""
+    heap.append(item)
+    _siftdown(heap, 0, len(heap)-1)
+
+def heappop(heap):
+    """Pop the smallest item off the heap, maintaining the heap invariant."""
+    lastelt = heap.pop()    # raises appropriate IndexError if heap is empty
+    if heap:
+        returnitem = heap[0]
+        heap[0] = lastelt
+        _siftup(heap, 0)
+    else:
+        returnitem = lastelt
+    return returnitem
+
+def heapreplace(heap, item):
+    """Pop and return the current smallest value, and add the new item.
+
+    This is more efficient than heappop() followed by heappush(), and can be
+    more appropriate when using a fixed-size heap.  Note that the value
+    returned may be larger than item!  That constrains reasonable uses of
+    this routine unless written as part of a conditional replacement:
+
+        if item > heap[0]:
+            item = heapreplace(heap, item)
+    """
+    returnitem = heap[0]    # raises appropriate IndexError if heap is empty
+    heap[0] = item
+    _siftup(heap, 0)
+    return returnitem
+
+def heappushpop(heap, item):
+    """Fast version of a heappush followed by a heappop."""
+    if heap and heap[0] < item:
+        item, heap[0] = heap[0], item
+        _siftup(heap, 0)
+    return item
+
+def heapify(x):
+    """Transform list into a heap, in-place, in O(len(heap)) time."""
+    n = len(x)
+    # Transform bottom-up.  The largest index there's any point to looking at
+    # is the largest with a child index in-range, so must have 2*i + 1 < n,
+    # or i < (n-1)/2.  If n is even = 2*j, this is (2*j-1)/2 = j-1/2 so
+    # j-1 is the largest, which is n//2 - 1.  If n is odd = 2*j+1, this is
+    # (2*j+1-1)/2 = j so j-1 is the largest, and that's again n//2-1.
+    for i in reversed(xrange(n//2)):
+        _siftup(x, i)
+
+def nlargest(n, iterable):
+    """Find the n largest elements in a dataset.
+
+    Equivalent to:  sorted(iterable, reverse=True)[:n]
+    """
+    if n < 0: # for consistency with the c impl
+        return []
+    it = iter(iterable)
+    result = list(islice(it, n))
+    if not result:
+        return result
+    heapify(result)
+    _heappushpop = heappushpop
+    for elem in it:
+        _heappushpop(result, elem)
+    result.sort(reverse=True)
+    return result
+
+def nsmallest(n, iterable):
+    """Find the n smallest elements in a dataset.
+
+    Equivalent to:  sorted(iterable)[:n]
+    """
+    if n < 0: # for consistency with the c impl
+        return []
+    if hasattr(iterable, '__len__') and n * 10 <= len(iterable):
+        # For smaller values of n, the bisect method is faster than a minheap.
+        # It is also memory efficient, consuming only n elements of space.
+        it = iter(iterable)
+        result = sorted(islice(it, 0, n))
+        if not result:
+            return result
+        insort = bisect.insort
+        pop = result.pop
+        los = result[-1]    # los --> Largest of the nsmallest
+        for elem in it:
+            if los <= elem:
+                continue
+            insort(result, elem)
+            pop()
+            los = result[-1]
+        return result
+    # An alternative approach manifests the whole iterable in memory but
+    # saves comparisons by heapifying all at once.  Also, saves time
+    # over bisect.insort() which has O(n) data movement time for every
+    # insertion.  Finding the n smallest of an m length iterable requires
+    #    O(m) + O(n log m) comparisons.
+    h = list(iterable)
+    heapify(h)
+    return map(heappop, repeat(h, min(n, len(h))))
+
+# 'heap' is a heap at all indices >= startpos, except possibly for pos.  pos
+# is the index of a leaf with a possibly out-of-order value.  Restore the
+# heap invariant.
+def _siftdown(heap, startpos, pos):
+    newitem = heap[pos]
+    # Follow the path to the root, moving parents down until finding a place
+    # newitem fits.
+    while pos > startpos:
+        parentpos = (pos - 1) >> 1
+        parent = heap[parentpos]
+        if newitem < parent:
+            heap[pos] = parent
+            pos = parentpos
+            continue
+        break
+    heap[pos] = newitem
+
+# The child indices of heap index pos are already heaps, and we want to make
+# a heap at index pos too.  We do this by bubbling the smaller child of
+# pos up (and so on with that child's children, etc) until hitting a leaf,
+# then using _siftdown to move the oddball originally at index pos into place.
+#
+# We *could* break out of the loop as soon as we find a pos where newitem <=
+# both its children, but turns out that's not a good idea, and despite that
+# many books write the algorithm that way.  During a heap pop, the last array
+# element is sifted in, and that tends to be large, so that comparing it
+# against values starting from the root usually doesn't pay (= usually doesn't
+# get us out of the loop early).  See Knuth, Volume 3, where this is
+# explained and quantified in an exercise.
+#
+# Cutting the # of comparisons is important, since these routines have no
+# way to extract "the priority" from an array element, so that intelligence
+# is likely to be hiding in custom __cmp__ methods, or in array elements
+# storing (priority, record) tuples.  Comparisons are thus potentially
+# expensive.
+#
+# On random arrays of length 1000, making this change cut the number of
+# comparisons made by heapify() a little, and those made by exhaustive
+# heappop() a lot, in accord with theory.  Here are typical results from 3
+# runs (3 just to demonstrate how small the variance is):
+#
+# Compares needed by heapify     Compares needed by 1000 heappops
+# --------------------------     --------------------------------
+# 1837 cut to 1663               14996 cut to 8680
+# 1855 cut to 1659               14966 cut to 8678
+# 1847 cut to 1660               15024 cut to 8703
+#
+# Building the heap by using heappush() 1000 times instead required
+# 2198, 2148, and 2219 compares:  heapify() is more efficient, when
+# you can use it.
+#
+# The total compares needed by list.sort() on the same lists were 8627,
+# 8627, and 8632 (this should be compared to the sum of heapify() and
+# heappop() compares):  list.sort() is (unsurprisingly!) more efficient
+# for sorting.
+
+def _siftup(heap, pos):
+    endpos = len(heap)
+    startpos = pos
+    newitem = heap[pos]
+    # Bubble up the smaller child until hitting a leaf.
+    childpos = 2*pos + 1    # leftmost child position
+    while childpos < endpos:
+        # Set childpos to index of smaller child.
+        rightpos = childpos + 1
+        if rightpos < endpos and not heap[childpos] < heap[rightpos]:
+            childpos = rightpos
+        # Move the smaller child up.
+        heap[pos] = heap[childpos]
+        pos = childpos
+        childpos = 2*pos + 1
+    # The leaf at pos is empty now.  Put newitem there, and bubble it up
+    # to its final resting place (by sifting its parents down).
+    heap[pos] = newitem
+    _siftdown(heap, startpos, pos)
+
+# If available, use C implementation
+try:
+    from _heapq import *
+except ImportError:
+    pass
+
+def merge(*iterables):
+    '''Merge multiple sorted inputs into a single sorted output.
+
+    Similar to sorted(itertools.chain(*iterables)) but returns a generator,
+    does not pull the data into memory all at once, and assumes that each of
+    the input streams is already sorted (smallest to largest).
+
+    >>> list(merge([1,3,5,7], [0,2,4,8], [5,10,15,20], [], [25]))
+    [0, 1, 2, 3, 4, 5, 5, 7, 8, 10, 15, 20, 25]
+
+    '''
+    _heappop, _heapreplace, _StopIteration = heappop, heapreplace, StopIteration
+
+    h = []
+    h_append = h.append
+    for itnum, it in enumerate(map(iter, iterables)):
+        try:
+            next = it.next
+            h_append([next(), itnum, next])
+        except _StopIteration:
+            pass
+    heapify(h)
+
+    while 1:
+        try:
+            while 1:
+                v, itnum, next = s = h[0]   # raises IndexError when h is empty
+                yield v
+                s[0] = next()               # raises StopIteration when exhausted
+                _heapreplace(h, s)          # restore heap condition
+        except _StopIteration:
+            _heappop(h)                     # remove empty iterator
+        except IndexError:
+            return
+
+# Extend the implementations of nsmallest and nlargest to use a key= argument
+_nsmallest = nsmallest
+def nsmallest(n, iterable, key=None):
+    """Find the n smallest elements in a dataset.
+
+    Equivalent to:  sorted(iterable, key=key)[:n]
+    """
+    # Short-cut for n==1 is to use min() when len(iterable)>0
+    if n == 1:
+        it = iter(iterable)
+        head = list(islice(it, 1))
+        if not head:
+            return []
+        if key is None:
+            return [min(chain(head, it))]
+        return [min(chain(head, it), key=key)]
+
+    # When n>=size, it's faster to use sort()
+    try:
+        size = len(iterable)
+    except (TypeError, AttributeError):
+        pass
+    else:
+        if n >= size:
+            return sorted(iterable, key=key)[:n]
+
+    # When key is none, use simpler decoration
+    if key is None:
+        it = izip(iterable, count())                        # decorate
+        result = _nsmallest(n, it)
+        return map(itemgetter(0), result)                   # undecorate
+
+    # General case, slowest method
+    in1, in2 = tee(iterable)
+    it = izip(imap(key, in1), count(), in2)                 # decorate
+    result = _nsmallest(n, it)
+    return map(itemgetter(2), result)                       # undecorate
+
+_nlargest = nlargest
+def nlargest(n, iterable, key=None):
+    """Find the n largest elements in a dataset.
+
+    Equivalent to:  sorted(iterable, key=key, reverse=True)[:n]
+    """
+
+    # Short-cut for n==1 is to use max() when len(iterable)>0
+    if n == 1:
+        it = iter(iterable)
+        head = list(islice(it, 1))
+        if not head:
+            return []
+        if key is None:
+            return [max(chain(head, it))]
+        return [max(chain(head, it), key=key)]
+
+    # When n>=size, it's faster to use sort()
+    try:
+        size = len(iterable)
+    except (TypeError, AttributeError):
+        pass
+    else:
+        if n >= size:
+            return sorted(iterable, key=key, reverse=True)[:n]
+
+    # When key is none, use simpler decoration
+    if key is None:
+        it = izip(iterable, count(0,-1))                    # decorate
+        result = _nlargest(n, it)
+        return map(itemgetter(0), result)                   # undecorate
+
+    # General case, slowest method
+    in1, in2 = tee(iterable)
+    it = izip(imap(key, in1), count(0,-1), in2)             # decorate
+    result = _nlargest(n, it)
+    return map(itemgetter(2), result)                       # undecorate
+
+if __name__ == "__main__":
+    # Simple sanity test
+    heap = []
+    data = [1, 3, 5, 7, 9, 2, 4, 6, 8, 0]
+    for item in data:
+        heappush(heap, item)
+    sort = []
+    while heap:
+        sort.append(heappop(heap))
+    print sort
+
+    import doctest
+    doctest.testmod()
diff --git a/lib-python/modified-2.7/httplib.py b/lib-python/modified-2.7/httplib.py
new file mode 100644
--- /dev/null
+++ b/lib-python/modified-2.7/httplib.py
@@ -0,0 +1,1377 @@
+"""HTTP/1.1 client library
+
+
+
+
+HTTPConnection goes through a number of "states", which define when a client
+may legally make another request or fetch the response for a particular
+request. This diagram details these state transitions:
+
+    (null)
+      |
+      | HTTPConnection()
+      v
+    Idle
+      |
+      | putrequest()
+      v
+    Request-started
+      |
+      | ( putheader() )*  endheaders()
+      v
+    Request-sent
+      |
+      | response = getresponse()
+      v
+    Unread-response   [Response-headers-read]
+      |\____________________
+      |                     |
+      | response.read()     | putrequest()
+      v                     v
+    Idle                  Req-started-unread-response
+                     ______/|
+                   /        |
+   response.read() |        | ( putheader() )*  endheaders()
+                   v        v
+       Request-started    Req-sent-unread-response
+                            |
+                            | response.read()
+                            v
+                          Request-sent
+
+This diagram presents the following rules:
+  -- a second request may not be started until {response-headers-read}
+  -- a response [object] cannot be retrieved until {request-sent}
+  -- there is no differentiation between an unread response body and a
+     partially read response body
+
+Note: this enforcement is applied by the HTTPConnection class. The
+      HTTPResponse class does not enforce this state machine, which
+      implies sophisticated clients may accelerate the request/response
+      pipeline. Caution should be taken, though: accelerating the states
+      beyond the above pattern may imply knowledge of the server's
+      connection-close behavior for certain requests. For example, it
+      is impossible to tell whether the server will close the connection
+      UNTIL the response headers have been read; this means that further
+      requests cannot be placed into the pipeline until it is known that
+      the server will NOT be closing the connection.
+
+Logical State                  __state            __response
+-------------                  -------            ----------
+Idle                           _CS_IDLE           None
+Request-started                _CS_REQ_STARTED    None
+Request-sent                   _CS_REQ_SENT       None
+Unread-response                _CS_IDLE           
+Req-started-unread-response    _CS_REQ_STARTED    
+Req-sent-unread-response       _CS_REQ_SENT       
+"""
+
+from array import array
+import os
+import socket
+from sys import py3kwarning
+from urlparse import urlsplit
+import warnings
+with warnings.catch_warnings():
+    if py3kwarning:
+        warnings.filterwarnings("ignore", ".*mimetools has been removed",
+                                DeprecationWarning)
+    import mimetools
+
+try:
+    from cStringIO import StringIO
+except ImportError:
+    from StringIO import StringIO
+
+__all__ = ["HTTP", "HTTPResponse", "HTTPConnection",
+           "HTTPException", "NotConnected", "UnknownProtocol",
+           "UnknownTransferEncoding", "UnimplementedFileMode",
+           "IncompleteRead", "InvalidURL", "ImproperConnectionState",
+           "CannotSendRequest", "CannotSendHeader", "ResponseNotReady",
+           "BadStatusLine", "error", "responses"]
+
+HTTP_PORT = 80
+HTTPS_PORT = 443
+
+_UNKNOWN = 'UNKNOWN'
+
+# connection states
+_CS_IDLE = 'Idle'
+_CS_REQ_STARTED = 'Request-started'
+_CS_REQ_SENT = 'Request-sent'
+
+# status codes
+# informational
+CONTINUE = 100
+SWITCHING_PROTOCOLS = 101
+PROCESSING = 102
+
+# successful
+OK = 200
+CREATED = 201
+ACCEPTED = 202
+NON_AUTHORITATIVE_INFORMATION = 203
+NO_CONTENT = 204
+RESET_CONTENT = 205
+PARTIAL_CONTENT = 206
+MULTI_STATUS = 207
+IM_USED = 226
+
+# redirection
+MULTIPLE_CHOICES = 300
+MOVED_PERMANENTLY = 301
+FOUND = 302
+SEE_OTHER = 303
+NOT_MODIFIED = 304
+USE_PROXY = 305
+TEMPORARY_REDIRECT = 307
+
+# client error
+BAD_REQUEST = 400
+UNAUTHORIZED = 401
+PAYMENT_REQUIRED = 402
+FORBIDDEN = 403
+NOT_FOUND = 404
+METHOD_NOT_ALLOWED = 405
+NOT_ACCEPTABLE = 406
+PROXY_AUTHENTICATION_REQUIRED = 407
+REQUEST_TIMEOUT = 408
+CONFLICT = 409
+GONE = 410
+LENGTH_REQUIRED = 411
+PRECONDITION_FAILED = 412
+REQUEST_ENTITY_TOO_LARGE = 413
+REQUEST_URI_TOO_LONG = 414
+UNSUPPORTED_MEDIA_TYPE = 415
+REQUESTED_RANGE_NOT_SATISFIABLE = 416
+EXPECTATION_FAILED = 417
+UNPROCESSABLE_ENTITY = 422
+LOCKED = 423
+FAILED_DEPENDENCY = 424
+UPGRADE_REQUIRED = 426
+
+# server error
+INTERNAL_SERVER_ERROR = 500
+NOT_IMPLEMENTED = 501
+BAD_GATEWAY = 502
+SERVICE_UNAVAILABLE = 503
+GATEWAY_TIMEOUT = 504
+HTTP_VERSION_NOT_SUPPORTED = 505
+INSUFFICIENT_STORAGE = 507
+NOT_EXTENDED = 510
+
+# Mapping status codes to official W3C names
+responses = {
+    100: 'Continue',
+    101: 'Switching Protocols',
+
+    200: 'OK',
+    201: 'Created',
+    202: 'Accepted',
+    203: 'Non-Authoritative Information',
+    204: 'No Content',
+    205: 'Reset Content',
+    206: 'Partial Content',
+
+    300: 'Multiple Choices',
+    301: 'Moved Permanently',
+    302: 'Found',
+    303: 'See Other',
+    304: 'Not Modified',
+    305: 'Use Proxy',
+    306: '(Unused)',
+    307: 'Temporary Redirect',
+
+    400: 'Bad Request',
+    401: 'Unauthorized',
+    402: 'Payment Required',
+    403: 'Forbidden',
+    404: 'Not Found',
+    405: 'Method Not Allowed',
+    406: 'Not Acceptable',
+    407: 'Proxy Authentication Required',
+    408: 'Request Timeout',
+    409: 'Conflict',
+    410: 'Gone',
+    411: 'Length Required',
+    412: 'Precondition Failed',
+    413: 'Request Entity Too Large',
+    414: 'Request-URI Too Long',
+    415: 'Unsupported Media Type',
+    416: 'Requested Range Not Satisfiable',
+    417: 'Expectation Failed',
+
+    500: 'Internal Server Error',
+    501: 'Not Implemented',
+    502: 'Bad Gateway',
+    503: 'Service Unavailable',
+    504: 'Gateway Timeout',
+    505: 'HTTP Version Not Supported',
+}
+
+# maximal amount of data to read at one time in _safe_read
+MAXAMOUNT = 1048576
+
+class HTTPMessage(mimetools.Message):
+
+    def addheader(self, key, value):
+        """Add header for field key handling repeats."""
+        prev = self.dict.get(key)
+        if prev is None:
+            self.dict[key] = value
+        else:
+            combined = ", ".join((prev, value))
+            self.dict[key] = combined
+
+    def addcontinue(self, key, more):
+        """Add more field data from a continuation line."""
+        prev = self.dict[key]
+        self.dict[key] = prev + "\n " + more
+
+    def readheaders(self):
+        """Read header lines.
+
+        Read header lines up to the entirely blank line that terminates them.
+        The (normally blank) line that ends the headers is skipped, but not
+        included in the returned list.  If a non-header line ends the headers,
+        (which is an error), an attempt is made to backspace over it; it is
+        never included in the returned list.
+
+        The variable self.status is set to the empty string if all went well,
+        otherwise it is an error message.  The variable self.headers is a
+        completely uninterpreted list of lines contained in the header (so
+        printing them will reproduce the header exactly as it appears in the
+        file).
+
+        If multiple header fields with the same name occur, they are combined
+        according to the rules in RFC 2616 sec 4.2:
+
+        Appending each subsequent field-value to the first, each separated
+        by a comma. The order in which header fields with the same field-name
+        are received is significant to the interpretation of the combined
+        field value.
+        """
+        # XXX The implementation overrides the readheaders() method of
+        # rfc822.Message.  The base class design isn't amenable to
+        # customized behavior here so the method here is a copy of the
+        # base class code with a few small changes.
+
+        self.dict = {}
+        self.unixfrom = ''
+        self.headers = hlist = []
+        self.status = ''
+        headerseen = ""
+        firstline = 1
+        startofline = unread = tell = None
+        if hasattr(self.fp, 'unread'):
+            unread = self.fp.unread
+        elif self.seekable:
+            tell = self.fp.tell
+        while True:
+            if tell:
+                try:
+                    startofline = tell()
+                except IOError:
+                    startofline = tell = None
+                    self.seekable = 0
+            line = self.fp.readline()
+            if not line:
+                self.status = 'EOF in headers'
+                break
+            # Skip unix From name time lines
+            if firstline and line.startswith('From '):
+                self.unixfrom = self.unixfrom + line
+                continue
+            firstline = 0
+            if headerseen and line[0] in ' \t':
+                # XXX Not sure if continuation lines are handled properly
+                # for http and/or for repeating headers
+                # It's a continuation line.
+                hlist.append(line)
+                self.addcontinue(headerseen, line.strip())
+                continue
+            elif self.iscomment(line):
+                # It's a comment.  Ignore it.
+                continue
+            elif self.islast(line):
+                # Note! No pushback here!  The delimiter line gets eaten.
+                break
+            headerseen = self.isheader(line)
+            if headerseen:
+                # It's a legal header line, save it.
+                hlist.append(line)
+                self.addheader(headerseen, line[len(headerseen)+1:].strip())
+                continue
+            else:
+                # It's not a header line; throw it back and stop here.
+                if not self.dict:
+                    self.status = 'No headers'
+                else:
+                    self.status = 'Non-header line where header expected'
+                # Try to undo the read.
+                if unread:
+                    unread(line)
+                elif tell:
+                    self.fp.seek(startofline)
+                else:
+                    self.status = self.status + '; bad seek'
+                break
+
+class HTTPResponse:
+
+    # strict: If true, raise BadStatusLine if the status line can't be
+    # parsed as a valid HTTP/1.0 or 1.1 status line.  By default it is
+    # false because it prevents clients from talking to HTTP/0.9
+    # servers.  Note that a response with a sufficiently corrupted
+    # status line will look like an HTTP/0.9 response.
+
+    # See RFC 2616 sec 19.6 and RFC 1945 sec 6 for details.
+
+    def __init__(self, sock, debuglevel=0, strict=0, method=None, buffering=False):
+        if buffering:
+            # The caller won't be using any sock.recv() calls, so buffering
+            # is fine and recommended for performance.
+            self.fp = sock.makefile('rb')
+        else:
+            # The buffer size is specified as zero, because the headers of
+            # the response are read with readline().  If the reads were
+            # buffered the readline() calls could consume some of the
+            # response, which make be read via a recv() on the underlying
+            # socket.
+            self.fp = sock.makefile('rb', 0)
+        self.debuglevel = debuglevel
+        self.strict = strict
+        self._method = method
+
+        self.msg = None
+
+        # from the Status-Line of the response
+        self.version = _UNKNOWN # HTTP-Version
+        self.status = _UNKNOWN  # Status-Code
+        self.reason = _UNKNOWN  # Reason-Phrase
+
+        self.chunked = _UNKNOWN         # is "chunked" being used?
+        self.chunk_left = _UNKNOWN      # bytes left to read in current chunk
+        self.length = _UNKNOWN          # number of bytes left in response
+        self.will_close = _UNKNOWN      # conn will close at end of response
+
+    def _read_status(self):
+        # Initialize with Simple-Response defaults
+        line = self.fp.readline()
+        if self.debuglevel > 0:
+            print "reply:", repr(line)
+        if not line:
+            # Presumably, the server closed the connection before
+            # sending a valid response.
+            raise BadStatusLine(line)
+        try:
+            [version, status, reason] = line.split(None, 2)
+        except ValueError:
+            try:
+                [version, status] = line.split(None, 1)
+                reason = ""
+            except ValueError:
+                # empty version will cause next test to fail and status
+                # will be treated as 0.9 response.
+                version = ""
+        if not version.startswith('HTTP/'):
+            if self.strict:
+                self.close()
+                raise BadStatusLine(line)
+            else:
+                # assume it's a Simple-Response from an 0.9 server
+                self.fp = LineAndFileWrapper(line, self.fp)
+                return "HTTP/0.9", 200, ""
+
+        # The status code is a three-digit number
+        try:
+            status = int(status)
+            if status < 100 or status > 999:
+                raise BadStatusLine(line)
+        except ValueError:
+            raise BadStatusLine(line)
+        return version, status, reason
+
+    def begin(self):
+        if self.msg is not None:
+            # we've already started reading the response
+            return
+
+        # read until we get a non-100 response
+        while True:
+            version, status, reason = self._read_status()
+            if status != CONTINUE:
+                break
+            # skip the header from the 100 response
+            while True:
+                skip = self.fp.readline().strip()
+                if not skip:
+                    break
+                if self.debuglevel > 0:
+                    print "header:", skip
+
+        self.status = status
+        self.reason = reason.strip()
+        if version == 'HTTP/1.0':
+            self.version = 10
+        elif version.startswith('HTTP/1.'):
+            self.version = 11   # use HTTP/1.1 code for HTTP/1.x where x>=1
+        elif version == 'HTTP/0.9':
+            self.version = 9
+        else:
+            raise UnknownProtocol(version)
+
+        if self.version == 9:
+            self.length = None
+            self.chunked = 0
+            self.will_close = 1
+            self.msg = HTTPMessage(StringIO())
+            return
+
+        self.msg = HTTPMessage(self.fp, 0)
+        if self.debuglevel > 0:
+            for hdr in self.msg.headers:
+                print "header:", hdr,
+
+        # don't let the msg keep an fp
+        self.msg.fp = None
+
+        # are we using the chunked-style of transfer encoding?
+        tr_enc = self.msg.getheader('transfer-encoding')
+        if tr_enc and tr_enc.lower() == "chunked":
+            self.chunked = 1
+            self.chunk_left = None
+        else:
+            self.chunked = 0
+
+        # will the connection close at the end of the response?
+        self.will_close = self._check_close()
+
+        # do we have a Content-Length?
+        # NOTE: RFC 2616, S4.4, #3 says we ignore this if tr_enc is "chunked"
+        length = self.msg.getheader('content-length')
+        if length and not self.chunked:
+            try:
+                self.length = int(length)
+            except ValueError:
+                self.length = None
+            else:
+                if self.length < 0:  # ignore nonsensical negative lengths
+                    self.length = None
+        else:
+            self.length = None
+
+        # does the body have a fixed length? (of zero)
+        if (status == NO_CONTENT or status == NOT_MODIFIED or
+            100 <= status < 200 or      # 1xx codes
+            self._method == 'HEAD'):
+            self.length = 0
+
+        # if the connection remains open, and we aren't using chunked, and
+        # a content-length was not provided, then assume that the connection
+        # WILL close.
+        if not self.will_close and \
+           not self.chunked and \
+           self.length is None:
+            self.will_close = 1
+
+    def _check_close(self):
+        conn = self.msg.getheader('connection')
+        if self.version == 11:
+            # An HTTP/1.1 proxy is assumed to stay open unless
+            # explicitly closed.
+            conn = self.msg.getheader('connection')
+            if conn and "close" in conn.lower():
+                return True
+            return False
+
+        # Some HTTP/1.0 implementations have support for persistent
+        # connections, using rules different than HTTP/1.1.
+
+        # For older HTTP, Keep-Alive indicates persistent connection.
+        if self.msg.getheader('keep-alive'):
+            return False
+
+        # At least Akamai returns a "Connection: Keep-Alive" header,
+        # which was supposed to be sent by the client.
+        if conn and "keep-alive" in conn.lower():
+            return False
+
+        # Proxy-Connection is a netscape hack.
+        pconn = self.msg.getheader('proxy-connection')
+        if pconn and "keep-alive" in pconn.lower():
+            return False
+
+        # otherwise, assume it will close
+        return True
+
+    def close(self):
+        if self.fp:
+            self.fp.close()
+            self.fp = None
+
+    def isclosed(self):
+        # NOTE: it is possible that we will not ever call self.close(). This
+        #       case occurs when will_close is TRUE, length is None, and we
+        #       read up to the last byte, but NOT past it.
+        #
+        # IMPLIES: if will_close is FALSE, then self.close() will ALWAYS be
+        #          called, meaning self.isclosed() is meaningful.
+        return self.fp is None
+
+    # XXX It would be nice to have readline and __iter__ for this, too.
+
+    def read(self, amt=None):
+        if self.fp is None:
+            return ''
+
+        if self._method == 'HEAD':
+            self.close()
+            return ''
+
+        if self.chunked:
+            return self._read_chunked(amt)
+
+        if amt is None:
+            # unbounded read
+            if self.length is None:
+                s = self.fp.read()
+            else:
+                s = self._safe_read(self.length)
+                self.length = 0
+            self.close()        # we read everything
+            return s
+
+        if self.length is not None:
+            if amt > self.length:
+                # clip the read to the "end of response"
+                amt = self.length
+
+        # we do not use _safe_read() here because this may be a .will_close
+        # connection, and the user is reading more bytes than will be provided
+        # (for example, reading in 1k chunks)
+        s = self.fp.read(amt)
+        if self.length is not None:
+            self.length -= len(s)
+            if not self.length:
+                self.close()
+        return s
+
+    def _read_chunked(self, amt):
+        assert self.chunked != _UNKNOWN
+        chunk_left = self.chunk_left
+        value = []
+        while True:
+            if chunk_left is None:
+                line = self.fp.readline()
+                i = line.find(';')
+                if i >= 0:
+                    line = line[:i] # strip chunk-extensions
+                try:
+                    chunk_left = int(line, 16)
+                except ValueError:
+                    # close the connection as protocol synchronisation is
+                    # probably lost
+                    self.close()
+                    raise IncompleteRead(''.join(value))
+                if chunk_left == 0:
+                    break
+            if amt is None:
+                value.append(self._safe_read(chunk_left))
+            elif amt < chunk_left:
+                value.append(self._safe_read(amt))
+                self.chunk_left = chunk_left - amt
+                return ''.join(value)
+            elif amt == chunk_left:
+                value.append(self._safe_read(amt))
+                self._safe_read(2)  # toss the CRLF at the end of the chunk
+                self.chunk_left = None
+                return ''.join(value)
+            else:
+                value.append(self._safe_read(chunk_left))
+                amt -= chunk_left
+
+            # we read the whole chunk, get another
+            self._safe_read(2)      # toss the CRLF at the end of the chunk
+            chunk_left = None
+
+        # read and discard trailer up to the CRLF terminator
+        ### note: we shouldn't have any trailers!
+        while True:
+            line = self.fp.readline()
+            if not line:
+                # a vanishingly small number of sites EOF without
+                # sending the trailer
+                break
+            if line == '\r\n':
+                break
+
+        # we read everything; close the "file"
+        self.close()
+
+        return ''.join(value)
+
+    def _safe_read(self, amt):
+        """Read the number of bytes requested, compensating for partial reads.
+
+        Normally, we have a blocking socket, but a read() can be interrupted
+        by a signal (resulting in a partial read).
+
+        Note that we cannot distinguish between EOF and an interrupt when zero
+        bytes have been read. IncompleteRead() will be raised in this
+        situation.
+
+        This function should be used when  bytes "should" be present for
+        reading. If the bytes are truly not available (due to EOF), then the
+        IncompleteRead exception can be used to detect the problem.
+        """
+        # NOTE(gps): As of svn r74426 socket._fileobject.read(x) will never
+        # return less than x bytes unless EOF is encountered.  It now handles
+        # signal interruptions (socket.error EINTR) internally.  This code
+        # never caught that exception anyways.  It seems largely pointless.
+        # self.fp.read(amt) will work fine.
+        s = []
+        while amt > 0:
+            chunk = self.fp.read(min(amt, MAXAMOUNT))
+            if not chunk:
+                raise IncompleteRead(''.join(s), amt)
+            s.append(chunk)
+            amt -= len(chunk)
+        return ''.join(s)
+
+    def fileno(self):
+        return self.fp.fileno()
+
+    def getheader(self, name, default=None):
+        if self.msg is None:
+            raise ResponseNotReady()
+        return self.msg.getheader(name, default)
+
+    def getheaders(self):
+        """Return list of (header, value) tuples."""
+        if self.msg is None:
+            raise ResponseNotReady()
+        return self.msg.items()
+
+
+class HTTPConnection:
+
+    _http_vsn = 11
+    _http_vsn_str = 'HTTP/1.1'
+
+    response_class = HTTPResponse
+    default_port = HTTP_PORT
+    auto_open = 1
+    debuglevel = 0
+    strict = 0
+
+    def __init__(self, host, port=None, strict=None,
+                 timeout=socket._GLOBAL_DEFAULT_TIMEOUT, source_address=None):
+        self.timeout = timeout
+        self.source_address = source_address
+        self.sock = None
+        self._buffer = []
+        self.__response = None
+        self.__state = _CS_IDLE
+        self._method = None
+        self._tunnel_host = None
+        self._tunnel_port = None
+        self._tunnel_headers = {}
+
+        self._set_hostport(host, port)
+        if strict is not None:
+            self.strict = strict
+
+    def set_tunnel(self, host, port=None, headers=None):
+        """ Sets up the host and the port for the HTTP CONNECT Tunnelling.
+
+        The headers argument should be a mapping of extra HTTP headers
+        to send with the CONNECT request.
+        """
+        self._tunnel_host = host
+        self._tunnel_port = port
+        if headers:
+            self._tunnel_headers = headers
+        else:
+            self._tunnel_headers.clear()
+
+    def _set_hostport(self, host, port):
+        if port is None:
+            i = host.rfind(':')
+            j = host.rfind(']')         # ipv6 addresses have [...]
+            if i > j:
+                try:
+                    port = int(host[i+1:])
+                except ValueError:
+                    raise InvalidURL("nonnumeric port: '%s'" % host[i+1:])
+                host = host[:i]
+            else:
+                port = self.default_port
+            if host and host[0] == '[' and host[-1] == ']':
+                host = host[1:-1]
+        self.host = host
+        self.port = port
+
+    def set_debuglevel(self, level):
+        self.debuglevel = level
+
+    def _tunnel(self):
+        self._set_hostport(self._tunnel_host, self._tunnel_port)
+        self.send("CONNECT %s:%d HTTP/1.0\r\n" % (self.host, self.port))
+        for header, value in self._tunnel_headers.iteritems():
+            self.send("%s: %s\r\n" % (header, value))
+        self.send("\r\n")
+        response = self.response_class(self.sock, strict = self.strict,
+                                       method = self._method)
+        (version, code, message) = response._read_status()
+
+        if code != 200:
+            self.close()
+            raise socket.error("Tunnel connection failed: %d %s" % (code,
+                                                                    message.strip()))
+        while True:
+            line = response.fp.readline()
+            if line == '\r\n': break
+
+
+    def connect(self):
+        """Connect to the host and port specified in __init__."""
+        self.sock = socket.create_connection((self.host,self.port),
+                                             self.timeout, self.source_address)
+
+        if self._tunnel_host:
+            self._tunnel()
+
+    def close(self):
+        """Close the connection to the HTTP server."""
+        if self.sock:
+            self.sock.close()   # close it manually... there may be other refs
+            self.sock = None
+        if self.__response:
+            self.__response.close()
+            self.__response = None
+        self.__state = _CS_IDLE
+
+    def send(self, data):
+        """Send `data' to the server."""
+        if self.sock is None:
+            if self.auto_open:
+                self.connect()
+            else:
+                raise NotConnected()
+
+        if self.debuglevel > 0:
+            print "send:", repr(data)
+        blocksize = 8192
+        if hasattr(data,'read') and not isinstance(data, array):
+            if self.debuglevel > 0: print "sendIng a read()able"
+            datablock = data.read(blocksize)
+            while datablock:
+                self.sock.sendall(datablock)
+                datablock = data.read(blocksize)
+        else:
+            self.sock.sendall(data)
+
+    def _output(self, s):
+        """Add a line of output to the current request buffer.
+
+        Assumes that the line does *not* end with \\r\\n.
+        """
+        self._buffer.append(s)
+
+    def _send_output(self, message_body=None):
+        """Send the currently buffered request and clear the buffer.
+
+        Appends an extra \\r\\n to the buffer.
+        A message_body may be specified, to be appended to the request.
+        """
+        self._buffer.extend(("", ""))
+        msg = "\r\n".join(self._buffer)
+        del self._buffer[:]
+        # If msg and message_body are sent in a single send() call,
+        # it will avoid performance problems caused by the interaction
+        # between delayed ack and the Nagle algorithim.
+        if isinstance(message_body, str):
+            msg += message_body
+            message_body = None
+        self.send(msg)
+        if message_body is not None:
+            #message_body was not a string (i.e. it is a file) and
+            #we must run the risk of Nagle
+            self.send(message_body)
+
+    def putrequest(self, method, url, skip_host=0, skip_accept_encoding=0):
+        """Send a request to the server.
+
+        `method' specifies an HTTP request method, e.g. 'GET'.
+        `url' specifies the object being requested, e.g. '/index.html'.
+        `skip_host' if True does not add automatically a 'Host:' header
+        `skip_accept_encoding' if True does not add automatically an
+           'Accept-Encoding:' header
+        """
+
+        # if a prior response has been completed, then forget about it.
+        if self.__response and self.__response.isclosed():
+            self.__response = None
+
+
+        # in certain cases, we cannot issue another request on this connection.
+        # this occurs when:
+        #   1) we are in the process of sending a request.   (_CS_REQ_STARTED)
+        #   2) a response to a previous request has signalled that it is going
+        #      to close the connection upon completion.
+        #   3) the headers for the previous response have not been read, thus
+        #      we cannot determine whether point (2) is true.   (_CS_REQ_SENT)
+        #
+        # if there is no prior response, then we can request at will.
+        #
+        # if point (2) is true, then we will have passed the socket to the
+        # response (effectively meaning, "there is no prior response"), and
+        # will open a new one when a new request is made.
+        #
+        # Note: if a prior response exists, then we *can* start a new request.
+        #       We are not allowed to begin fetching the response to this new
+        #       request, however, until that prior response is complete.
+        #
+        if self.__state == _CS_IDLE:
+            self.__state = _CS_REQ_STARTED
+        else:
+            raise CannotSendRequest()
+
+        # Save the method we use, we need it later in the response phase
+        self._method = method
+        if not url:
+            url = '/'
+        hdr = '%s %s %s' % (method, url, self._http_vsn_str)
+
+        self._output(hdr)
+
+        if self._http_vsn == 11:
+            # Issue some standard headers for better HTTP/1.1 compliance
+
+            if not skip_host:
+                # this header is issued *only* for HTTP/1.1
+                # connections. more specifically, this means it is
+                # only issued when the client uses the new
+                # HTTPConnection() class. backwards-compat clients
+                # will be using HTTP/1.0 and those clients may be
+                # issuing this header themselves. we should NOT issue
+                # it twice; some web servers (such as Apache) barf
+                # when they see two Host: headers
+
+                # If we need a non-standard port,include it in the
+                # header.  If the request is going through a proxy,
+                # but the host of the actual URL, not the host of the
+                # proxy.
+
+                netloc = ''
+                if url.startswith('http'):
+                    nil, netloc, nil, nil, nil = urlsplit(url)
+
+                if netloc:
+                    try:
+                        netloc_enc = netloc.encode("ascii")
+                    except UnicodeEncodeError:
+                        netloc_enc = netloc.encode("idna")
+                    self.putheader('Host', netloc_enc)
+                else:
+                    try:
+                        host_enc = self.host.encode("ascii")
+                    except UnicodeEncodeError:
+                        host_enc = self.host.encode("idna")
+                    # Wrap the IPv6 Host Header with [] (RFC 2732)
+                    if host_enc.find(':') >= 0:
+                        host_enc = "[" + host_enc + "]"
+                    if self.port == self.default_port:
+                        self.putheader('Host', host_enc)
+                    else:
+                        self.putheader('Host', "%s:%s" % (host_enc, self.port))
+
+            # note: we are assuming that clients will not attempt to set these
+            #       headers since *this* library must deal with the
+            #       consequences. this also means that when the supporting
+            #       libraries are updated to recognize other forms, then this
+            #       code should be changed (removed or updated).
+
+            # we only want a Content-Encoding of "identity" since we don't
+            # support encodings such as x-gzip or x-deflate.
+            if not skip_accept_encoding:
+                self.putheader('Accept-Encoding', 'identity')
+
+            # we can accept "chunked" Transfer-Encodings, but no others
+            # NOTE: no TE header implies *only* "chunked"
+            #self.putheader('TE', 'chunked')
+
+            # if TE is supplied in the header, then it must appear in a
+            # Connection header.
+            #self.putheader('Connection', 'TE')
+
+        else:
+            # For HTTP/1.0, the server will assume "not chunked"
+            pass
+
+    def putheader(self, header, *values):
+        """Send a request header line to the server.
+
+        For example: h.putheader('Accept', 'text/html')
+        """
+        if self.__state != _CS_REQ_STARTED:
+            raise CannotSendHeader()
+
+        hdr = '%s: %s' % (header, '\r\n\t'.join([str(v) for v in values]))
+        self._output(hdr)
+
+    def endheaders(self, message_body=None):
+        """Indicate that the last header line has been sent to the server.
+
+        This method sends the request to the server.  The optional
+        message_body argument can be used to pass message body
+        associated with the request.  The message body will be sent in
+        the same packet as the message headers if possible.  The
+        message_body should be a string.
+        """
+        if self.__state == _CS_REQ_STARTED:
+            self.__state = _CS_REQ_SENT
+        else:
+            raise CannotSendHeader()
+        self._send_output(message_body)
+
+    def request(self, method, url, body=None, headers={}):
+        """Send a complete request to the server."""
+        self._send_request(method, url, body, headers)
+
+    def _set_content_length(self, body):
+        # Set the content-length based on the body.
+        thelen = None
+        try:
+            thelen = str(len(body))
+        except TypeError, te:
+            # If this is a file-like object, try to
+            # fstat its file descriptor
+            try:
+                thelen = str(os.fstat(body.fileno()).st_size)
+            except (AttributeError, OSError):
+                # Don't send a length if this failed
+                if self.debuglevel > 0: print "Cannot stat!!"
+
+        if thelen is not None:
+            self.putheader('Content-Length', thelen)
+
+    def _send_request(self, method, url, body, headers):
+        # Honor explicitly requested Host: and Accept-Encoding: headers.
+        header_names = dict.fromkeys([k.lower() for k in headers])
+        skips = {}
+        if 'host' in header_names:
+            skips['skip_host'] = 1
+        if 'accept-encoding' in header_names:
+            skips['skip_accept_encoding'] = 1
+
+        self.putrequest(method, url, **skips)
+
+        if body and ('content-length' not in header_names):
+            self._set_content_length(body)
+        for hdr, value in headers.iteritems():
+            self.putheader(hdr, value)
+        self.endheaders(body)
+
+    def getresponse(self, buffering=False):
+        "Get the response from the server."
+
+        # if a prior response has been completed, then forget about it.
+        if self.__response and self.__response.isclosed():
+            self.__response = None
+
+        #
+        # if a prior response exists, then it must be completed (otherwise, we
+        # cannot read this response's header to determine the connection-close
+        # behavior)
+        #
+        # note: if a prior response existed, but was connection-close, then the
+        # socket and response were made independent of this HTTPConnection
+        # object since a new request requires that we open a whole new
+        # connection
+        #
+        # this means the prior response had one of two states:
+        #   1) will_close: this connection was reset and the prior socket and
+        #                  response operate independently
+        #   2) persistent: the response was retained and we await its
+        #                  isclosed() status to become true.
+        #
+        if self.__state != _CS_REQ_SENT or self.__response:
+            raise ResponseNotReady()
+
+        args = (self.sock,)
+        kwds = {"strict":self.strict, "method":self._method}
+        if self.debuglevel > 0:
+            args += (self.debuglevel,)
+        if buffering:
+            #only add this keyword if non-default, for compatibility with
+            #other response_classes.
+            kwds["buffering"] = True;
+        response = self.response_class(*args, **kwds)
+
+        try:
+            response.begin()
+        except:
+            response.close()
+            raise
+        assert response.will_close != _UNKNOWN
+        self.__state = _CS_IDLE
+
+        if response.will_close:
+            # this effectively passes the connection to the response
+            self.close()
+        else:
+            # remember this, so we can tell when it is complete
+            self.__response = response
+
+        return response
+
+
+class HTTP:
+    "Compatibility class with httplib.py from 1.5."
+
+    _http_vsn = 10
+    _http_vsn_str = 'HTTP/1.0'
+
+    debuglevel = 0
+
+    _connection_class = HTTPConnection
+
+    def __init__(self, host='', port=None, strict=None):
+        "Provide a default host, since the superclass requires one."
+
+        # some joker passed 0 explicitly, meaning default port
+        if port == 0:
+            port = None
+
+        # Note that we may pass an empty string as the host; this will throw
+        # an error when we attempt to connect. Presumably, the client code
+        # will call connect before then, with a proper host.
+        self._setup(self._connection_class(host, port, strict))
+
+    def _setup(self, conn):
+        self._conn = conn
+
+        # set up delegation to flesh out interface
+        self.send = conn.send
+        self.putrequest = conn.putrequest
+        self.putheader = conn.putheader
+        self.endheaders = conn.endheaders
+        self.set_debuglevel = conn.set_debuglevel
+
+        conn._http_vsn = self._http_vsn
+        conn._http_vsn_str = self._http_vsn_str
+
+        self.file = None
+
+    def connect(self, host=None, port=None):
+        "Accept arguments to set the host/port, since the superclass doesn't."
+
+        if host is not None:
+            self._conn._set_hostport(host, port)
+        self._conn.connect()
+
+    def getfile(self):
+        "Provide a getfile, since the superclass' does not use this concept."
+        return self.file
+
+    def getreply(self, buffering=False):
+        """Compat definition since superclass does not define it.
+
+        Returns a tuple consisting of:
+        - server status code (e.g. '200' if all goes well)
+        - server "reason" corresponding to status code
+        - any RFC822 headers in the response from the server
+        """
+        try:
+            if not buffering:
+                response = self._conn.getresponse()
+            else:
+                #only add this keyword if non-default for compatibility
+                #with other connection classes
+                response = self._conn.getresponse(buffering)
+        except BadStatusLine, e:
+            ### hmm. if getresponse() ever closes the socket on a bad request,
+            ### then we are going to have problems with self.sock
+
+            ### should we keep this behavior? do people use it?
+            # keep the socket open (as a file), and return it
+            self.file = self._conn.sock.makefile('rb', 0)
+
+            # close our socket -- we want to restart after any protocol error
+            self.close()
+
+            self.headers = None
+            return -1, e.line, None
+
+        self.headers = response.msg
+        self.file = response.fp
+        return response.status, response.reason, response.msg
+
+    def close(self):
+        self._conn.close()
+
+        # note that self.file == response.fp, which gets closed by the
+        # superclass. just clear the object ref here.
+        ### hmm. messy. if status==-1, then self.file is owned by us.
+        ### well... we aren't explicitly closing, but losing this ref will
+        ### do it
+        self.file = None
+
+try:
+    import ssl
+except ImportError:
+    pass
+else:
+    class HTTPSConnection(HTTPConnection):
+        "This class allows communication via SSL."
+
+        default_port = HTTPS_PORT
+
+        def __init__(self, host, port=None, key_file=None, cert_file=None,
+                     strict=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
+                     source_address=None):
+            HTTPConnection.__init__(self, host, port, strict, timeout,
+                                    source_address)
+            self.key_file = key_file
+            self.cert_file = cert_file
+
+        def connect(self):
+            "Connect to a host on a given (SSL) port."
+
+            sock = socket.create_connection((self.host, self.port),
+                                            self.timeout, self.source_address)
+            if self._tunnel_host:
+                self.sock = sock
+                self._tunnel()
+            self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file)
+
+    __all__.append("HTTPSConnection")
+
+    class HTTPS(HTTP):
+        """Compatibility with 1.5 httplib interface
+
+        Python 1.5.2 did not have an HTTPS class, but it defined an
+        interface for sending http requests that is also useful for
+        https.
+        """
+
+        _connection_class = HTTPSConnection
+
+        def __init__(self, host='', port=None, key_file=None, cert_file=None,
+                     strict=None):
+            # provide a default host, pass the X509 cert info
+
+            # urf. compensate for bad input.
+            if port == 0:
+                port = None
+            self._setup(self._connection_class(host, port, key_file,
+                                               cert_file, strict))
+
+            # we never actually use these for anything, but we keep them
+            # here for compatibility with post-1.5.2 CVS.
+            self.key_file = key_file
+            self.cert_file = cert_file
+
+
+    def FakeSocket (sock, sslobj):
+        warnings.warn("FakeSocket is deprecated, and won't be in 3.x.  " +
+                      "Use the result of ssl.wrap_socket() directly instead.",
+                      DeprecationWarning, stacklevel=2)
+        return sslobj
+
+
+class HTTPException(Exception):
+    # Subclasses that define an __init__ must call Exception.__init__
+    # or define self.args.  Otherwise, str() will fail.
+    pass
+
+class NotConnected(HTTPException):
+    pass
+
+class InvalidURL(HTTPException):
+    pass
+
+class UnknownProtocol(HTTPException):
+    def __init__(self, version):
+        self.args = version,
+        self.version = version
+
+class UnknownTransferEncoding(HTTPException):
+    pass
+
+class UnimplementedFileMode(HTTPException):
+    pass
+
+class IncompleteRead(HTTPException):
+    def __init__(self, partial, expected=None):
+        self.args = partial,
+        self.partial = partial
+        self.expected = expected
+    def __repr__(self):
+        if self.expected is not None:
+            e = ', %i more expected' % self.expected
+        else:
+            e = ''
+        return 'IncompleteRead(%i bytes read%s)' % (len(self.partial), e)
+    def __str__(self):
+        return repr(self)
+
+class ImproperConnectionState(HTTPException):
+    pass
+
+class CannotSendRequest(ImproperConnectionState):
+    pass
+
+class CannotSendHeader(ImproperConnectionState):
+    pass
+
+class ResponseNotReady(ImproperConnectionState):
+    pass
+
+class BadStatusLine(HTTPException):
+    def __init__(self, line):
+        if not line:
+            line = repr(line)
+        self.args = line,
+        self.line = line
+
+# for backwards compatibility
+error = HTTPException
+
+class LineAndFileWrapper:
+    """A limited file-like object for HTTP/0.9 responses."""
+
+    # The status-line parsing code calls readline(), which normally
+    # get the HTTP status line.  For a 0.9 response, however, this is
+    # actually the first line of the body!  Clients need to get a
+    # readable file object that contains that line.
+
+    def __init__(self, line, file):
+        self._line = line
+        self._file = file
+        self._line_consumed = 0
+        self._line_offset = 0
+        self._line_left = len(line)
+
+    def __getattr__(self, attr):
+        return getattr(self._file, attr)
+
+    def _done(self):
+        # called when the last byte is read from the line.  After the
+        # call, all read methods are delegated to the underlying file
+        # object.
+        self._line_consumed = 1
+        self.read = self._file.read
+        self.readline = self._file.readline
+        self.readlines = self._file.readlines
+
+    def read(self, amt=None):
+        if self._line_consumed:
+            return self._file.read(amt)
+        assert self._line_left
+        if amt is None or amt > self._line_left:
+            s = self._line[self._line_offset:]
+            self._done()
+            if amt is None:
+                return s + self._file.read()
+            else:
+                return s + self._file.read(amt - len(s))
+        else:
+            assert amt <= self._line_left
+            i = self._line_offset
+            j = i + amt
+            s = self._line[i:j]
+            self._line_offset = j
+            self._line_left -= amt
+            if self._line_left == 0:
+                self._done()
+            return s
+
+    def readline(self):
+        if self._line_consumed:
+            return self._file.readline()
+        assert self._line_left
+        s = self._line[self._line_offset:]
+        self._done()
+        return s
+
+    def readlines(self, size=None):
+        if self._line_consumed:
+            return self._file.readlines(size)
+        assert self._line_left
+        L = [self._line[self._line_offset:]]
+        self._done()
+        if size is None:
+            return L + self._file.readlines()
+        else:
+            return L + self._file.readlines(size)
+
+def test():
+    """Test this module.
+
+    A hodge podge of tests collected here, because they have too many
+    external dependencies for the regular test suite.
+    """
+
+    import sys
+    import getopt
+    opts, args = getopt.getopt(sys.argv[1:], 'd')
+    dl = 0
+    for o, a in opts:
+        if o == '-d': dl = dl + 1
+    host = 'www.python.org'
+    selector = '/'
+    if args[0:]: host = args[0]
+    if args[1:]: selector = args[1]
+    h = HTTP()
+    h.set_debuglevel(dl)
+    h.connect(host)
+    h.putrequest('GET', selector)
+    h.endheaders()
+    status, reason, headers = h.getreply()
+    print 'status =', status
+    print 'reason =', reason
+    print "read", len(h.getfile().read())
+    print
+    if headers:
+        for header in headers.headers: print header.strip()
+    print
+
+    # minimal test that code to extract host from url works
+    class HTTP11(HTTP):
+        _http_vsn = 11
+        _http_vsn_str = 'HTTP/1.1'
+
+    h = HTTP11('www.python.org')
+    h.putrequest('GET', 'http://www.python.org/~jeremy/')
+    h.endheaders()
+    h.getreply()
+    h.close()
+
+    try:
+        import ssl
+    except ImportError:
+        pass
+    else:
+
+        for host, selector in (('sourceforge.net', '/projects/python'),
+                               ):
+            print "https://%s%s" % (host, selector)
+            hs = HTTPS()
+            hs.set_debuglevel(dl)
+            hs.connect(host)
+            hs.putrequest('GET', selector)
+            hs.endheaders()
+            status, reason, headers = hs.getreply()
+            print 'status =', status
+            print 'reason =', reason
+            print "read", len(hs.getfile().read())
+            print
+            if headers:
+                for header in headers.headers: print header.strip()
+            print
+
+if __name__ == '__main__':
+    test()
diff --git a/lib-python/modified-2.7/json/encoder.py b/lib-python/modified-2.7/json/encoder.py
--- a/lib-python/modified-2.7/json/encoder.py
+++ b/lib-python/modified-2.7/json/encoder.py
@@ -2,14 +2,7 @@
 """
 import re
 
-try:
-    from _json import encode_basestring_ascii as c_encode_basestring_ascii
-except ImportError:
-    c_encode_basestring_ascii = None
-try:
-    from _json import make_encoder as c_make_encoder
-except ImportError:
-    c_make_encoder = None
+from __pypy__.builders import StringBuilder, UnicodeBuilder
 
 ESCAPE = re.compile(r'[\x00-\x1f\\"\b\f\n\r\t]')
 ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])')
@@ -24,8 +17,7 @@
     '\t': '\\t',
 }
 for i in range(0x20):
-    ESCAPE_DCT.setdefault(chr(i), '\\u{0:04x}'.format(i))
-    #ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,))
+    ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,))
 
 # Assume this produces an infinity on all machines (probably not guaranteed)
 INFINITY = float('1e66666')
@@ -37,10 +29,9 @@
     """
     def replace(match):
         return ESCAPE_DCT[match.group(0)]
-    return '"' + ESCAPE.sub(replace, s) + '"'
+    return ESCAPE.sub(replace, s)
 
-
-def py_encode_basestring_ascii(s):
+def encode_basestring_ascii(s):
     """Return an ASCII-only JSON representation of a Python string
 
     """
@@ -53,20 +44,18 @@
         except KeyError:
             n = ord(s)
             if n < 0x10000:
-                return '\\u{0:04x}'.format(n)
-                #return '\\u%04x' % (n,)
+                return '\\u%04x' % (n,)
             else:
                 # surrogate pair
                 n -= 0x10000
                 s1 = 0xd800 | ((n >> 10) & 0x3ff)
                 s2 = 0xdc00 | (n & 0x3ff)
-                return '\\u{0:04x}\\u{1:04x}'.format(s1, s2)
-                #return '\\u%04x\\u%04x' % (s1, s2)
-    return '"' + str(ESCAPE_ASCII.sub(replace, s)) + '"'
-
-
-encode_basestring_ascii = (
-    c_encode_basestring_ascii or py_encode_basestring_ascii)
+                return '\\u%04x\\u%04x' % (s1, s2)
+    if ESCAPE_ASCII.search(s):
+        return str(ESCAPE_ASCII.sub(replace, s))
+    return s
+py_encode_basestring_ascii = lambda s: '"' + encode_basestring_ascii(s) + '"'
+c_encode_basestring_ascii = None
 
 class JSONEncoder(object):
     """Extensible JSON  encoder for Python data structures.
@@ -147,6 +136,17 @@
 
         self.skipkeys = skipkeys
         self.ensure_ascii = ensure_ascii
+        if ensure_ascii:
+            self.encoder = encode_basestring_ascii
+        else:
+            self.encoder = encode_basestring
+        if encoding != 'utf-8':
+            orig_encoder = self.encoder
+            def encoder(o):
+                if isinstance(o, str):
+                    o = o.decode(encoding)
+                return orig_encoder(o)
+            self.encoder = encoder
         self.check_circular = check_circular
         self.allow_nan = allow_nan
         self.sort_keys = sort_keys
@@ -184,24 +184,126 @@
         '{"foo": ["bar", "baz"]}'
 
         """
-        # This is for extremely simple cases and benchmarks.
+        if self.check_circular:
+            markers = {}
+        else:
+            markers = None
+        if self.ensure_ascii:
+            builder = StringBuilder()
+        else:
+            builder = UnicodeBuilder()
+        self._encode(o, markers, builder, 0)
+        return builder.build()
+
+    def _emit_indent(self, builder, _current_indent_level):
+        if self.indent is not None:
+            _current_indent_level += 1
+            newline_indent = '\n' + (' ' * (self.indent *
+                                            _current_indent_level))
+            separator = self.item_separator + newline_indent
+            builder.append(newline_indent)
+        else:
+            separator = self.item_separator
+        return separator, _current_indent_level
+
+    def _emit_unindent(self, builder, _current_indent_level):
+        if self.indent is not None:
+            builder.append('\n')
+            builder.append(' ' * (self.indent * (_current_indent_level - 1)))
+
+    def _encode(self, o, markers, builder, _current_indent_level):
         if isinstance(o, basestring):
-            if isinstance(o, str):
-                _encoding = self.encoding
-                if (_encoding is not None
-                        and not (_encoding == 'utf-8')):
-                    o = o.decode(_encoding)
-            if self.ensure_ascii:
-                return encode_basestring_ascii(o)
+            builder.append('"')
+            builder.append(self.encoder(o))
+            builder.append('"')
+        elif o is None:
+            builder.append('null')
+        elif o is True:
+            builder.append('true')
+        elif o is False:
+            builder.append('false')
+        elif isinstance(o, (int, long)):
+            builder.append(str(o))
+        elif isinstance(o, float):
+            builder.append(self._floatstr(o))
+        elif isinstance(o, (list, tuple)):
+            if not o:
+                builder.append('[]')
+                return
+            self._encode_list(o, markers, builder, _current_indent_level)
+        elif isinstance(o, dict):
+            if not o:
+                builder.append('{}')
+                return
+            self._encode_dict(o, markers, builder, _current_indent_level)
+        else:
+            self._mark_markers(markers, o)
+            res = self.default(o)
+            self._encode(res, markers, builder, _current_indent_level)
+            self._remove_markers(markers, o)
+            return res
+
+    def _encode_list(self, l, markers, builder, _current_indent_level):
+        self._mark_markers(markers, l)
+        builder.append('[')
+        first = True
+        separator, _current_indent_level = self._emit_indent(builder,
+                                                      _current_indent_level)
+        for elem in l:
+            if first:
+                first = False
             else:
-                return encode_basestring(o)
-        # This doesn't pass the iterator directly to ''.join() because the
-        # exceptions aren't as detailed.  The list call should be roughly
-        # equivalent to the PySequence_Fast that ''.join() would do.
-        chunks = self.iterencode(o, _one_shot=True)
-        if not isinstance(chunks, (list, tuple)):
-            chunks = list(chunks)
-        return ''.join(chunks)
+                builder.append(separator)
+            self._encode(elem, markers, builder, _current_indent_level)
+            del elem # XXX grumble
+        self._emit_unindent(builder, _current_indent_level)
+        builder.append(']')
+        self._remove_markers(markers, l)
+
+    def _encode_dict(self, d, markers, builder, _current_indent_level):
+        self._mark_markers(markers, d)
+        first = True
+        builder.append('{')
+        separator, _current_indent_level = self._emit_indent(builder,
+                                                         _current_indent_level)
+        if self.sort_keys:
+            items = sorted(d.items(), key=lambda kv: kv[0])
+        else:
+            items = d.iteritems()
+
+        for key, v in items:
+            if first:
+                first = False
+            else:
+                builder.append(separator)
+            if isinstance(key, basestring):
+                pass
+            # JavaScript is weakly typed for these, so it makes sense to
+            # also allow them.  Many encoders seem to do something like this.
+            elif isinstance(key, float):
+                key = self._floatstr(key)
+            elif key is True:
+                key = 'true'
+            elif key is False:
+                key = 'false'
+            elif key is None:
+                key = 'null'
+            elif isinstance(key, (int, long)):
+                key = str(key)
+            elif self.skipkeys:
+                continue
+            else:
+                raise TypeError("key " + repr(key) + " is not a string")
+            builder.append('"')
+            builder.append(self.encoder(key))
+            builder.append('"')
+            builder.append(self.key_separator)
+            self._encode(v, markers, builder, _current_indent_level)
+            del key
+            del v # XXX grumble
+        self._emit_unindent(builder, _current_indent_level)
+        builder.append('}')
+        self._remove_markers(markers, d)
 
     def iterencode(self, o, _one_shot=False):
         """Encode the given object and yield each string
@@ -217,86 +319,54 @@
             markers = {}
         else:
             markers = None
-        if self.ensure_ascii:
-            _encoder = encode_basestring_ascii
+        return self._iterencode(o, markers, 0)
+
+    def _floatstr(self, o):
+        # Check for specials.  Note that this type of test is processor
+        # and/or platform-specific, so do tests which don't depend on the
+        # internals.
+
+        if o != o:
+            text = 'NaN'
+        elif o == INFINITY:
+            text = 'Infinity'
+        elif o == -INFINITY:
+            text = '-Infinity'
         else:
-            _encoder = encode_basestring
-        if self.encoding != 'utf-8':
-            def _encoder(o, _orig_encoder=_encoder, _encoding=self.encoding):
-                if isinstance(o, str):
-                    o = o.decode(_encoding)
-                return _orig_encoder(o)
+            return FLOAT_REPR(o)
 
-        def floatstr(o, allow_nan=self.allow_nan,
-                _repr=FLOAT_REPR, _inf=INFINITY, _neginf=-INFINITY):
-            # Check for specials.  Note that this type of test is processor
-            # and/or platform-specific, so do tests which don't depend on the
-            # internals.
+        if not self.allow_nan:
+            raise ValueError(
+                "Out of range float values are not JSON compliant: " +
+                repr(o))
 
-            if o != o:
-                text = 'NaN'
-            elif o == _inf:
-                text = 'Infinity'
-            elif o == _neginf:
-                text = '-Infinity'
-            else:
-                return _repr(o)
+        return text
 
-            if not allow_nan:
-                raise ValueError(
-                    "Out of range float values are not JSON compliant: " +
-                    repr(o))
+    def _mark_markers(self, markers, o):
+        if markers is not None:
+            if id(o) in markers:
+                raise ValueError("Circular reference detected")
+            markers[id(o)] = None
 
-            return text
+    def _remove_markers(self, markers, o):
+        if markers is not None:
+            del markers[id(o)]
 
-
-        if (_one_shot and c_make_encoder is not None
-                and not self.indent and not self.sort_keys):
-            _iterencode = c_make_encoder(
-                markers, self.default, _encoder, self.indent,
-                self.key_separator, self.item_separator, self.sort_keys,
-                self.skipkeys, self.allow_nan)
-        else:
-            _iterencode = _make_iterencode(
-                markers, self.default, _encoder, self.indent, floatstr,
-                self.key_separator, self.item_separator, self.sort_keys,
-                self.skipkeys, _one_shot)
-        return _iterencode(o, 0)
-
-def _make_iterencode(markers, _default, _encoder, _indent, _floatstr,
-        _key_separator, _item_separator, _sort_keys, _skipkeys, _one_shot,
-        ## HACK: hand-optimized bytecode; turn globals into locals
-        ValueError=ValueError,
-        basestring=basestring,
-        dict=dict,
-        float=float,
-        id=id,
-        int=int,
-        isinstance=isinstance,
-        list=list,
-        long=long,
-        str=str,
-        tuple=tuple,
-    ):
-
-    def _iterencode_list(lst, _current_indent_level):
+    def _iterencode_list(self, lst, markers, _current_indent_level):
         if not lst:
             yield '[]'
             return
-        if markers is not None:
-            markerid = id(lst)
-            if markerid in markers:
-                raise ValueError("Circular reference detected")
-            markers[markerid] = lst
+        self._mark_markers(markers, lst)
         buf = '['
-        if _indent is not None:
+        if self.indent is not None:
             _current_indent_level += 1
-            newline_indent = '\n' + (' ' * (_indent * _current_indent_level))
-            separator = _item_separator + newline_indent
+            newline_indent = '\n' + (' ' * (self.indent *
+                                            _current_indent_level))
+            separator = self.item_separator + newline_indent
             buf += newline_indent
         else:
             newline_indent = None
-            separator = _item_separator
+            separator = self.item_separator
         first = True
         for value in lst:
             if first:
@@ -304,7 +374,7 @@
             else:
                 buf = separator
             if isinstance(value, basestring):
-                yield buf + _encoder(value)
+                yield buf + '"' + self.encoder(value) + '"'
             elif value is None:
                 yield buf + 'null'
             elif value is True:
@@ -314,44 +384,43 @@
             elif isinstance(value, (int, long)):
                 yield buf + str(value)
             elif isinstance(value, float):
-                yield buf + _floatstr(value)
+                yield buf + self._floatstr(value)
             else:
                 yield buf
                 if isinstance(value, (list, tuple)):
-                    chunks = _iterencode_list(value, _current_indent_level)
+                    chunks = self._iterencode_list(value, markers,
+                                                   _current_indent_level)
                 elif isinstance(value, dict):
-                    chunks = _iterencode_dict(value, _current_indent_level)
+                    chunks = self._iterencode_dict(value, markers,
+                                                   _current_indent_level)
                 else:
-                    chunks = _iterencode(value, _current_indent_level)
+                    chunks = self._iterencode(value, markers,
+                                              _current_indent_level)
                 for chunk in chunks:
                     yield chunk
         if newline_indent is not None:
             _current_indent_level -= 1
-            yield '\n' + (' ' * (_indent * _current_indent_level))
+            yield '\n' + (' ' * (self.indent * _current_indent_level))
         yield ']'
-        if markers is not None:
-            del markers[markerid]
+        self._remove_markers(markers, lst)
 
-    def _iterencode_dict(dct, _current_indent_level):
+    def _iterencode_dict(self, dct, markers, _current_indent_level):
         if not dct:
             yield '{}'
             return
-        if markers is not None:
-            markerid = id(dct)
-            if markerid in markers:
-                raise ValueError("Circular reference detected")
-            markers[markerid] = dct
+        self._mark_markers(markers, dct)
         yield '{'
-        if _indent is not None:
+        if self.indent is not None:
             _current_indent_level += 1
-            newline_indent = '\n' + (' ' * (_indent * _current_indent_level))
-            item_separator = _item_separator + newline_indent
+            newline_indent = '\n' + (' ' * (self.indent *
+                                            _current_indent_level))
+            item_separator = self.item_separator + newline_indent
             yield newline_indent
         else:
             newline_indent = None
-            item_separator = _item_separator
+            item_separator = self.item_separator
         first = True
-        if _sort_keys:
+        if self.sort_keys:
             items = sorted(dct.items(), key=lambda kv: kv[0])
         else:
             items = dct.iteritems()
@@ -361,7 +430,7 @@
             # JavaScript is weakly typed for these, so it makes sense to
             # also allow them.  Many encoders seem to do something like this.
             elif isinstance(key, float):
-                key = _floatstr(key)
+                key = self._floatstr(key)
             elif key is True:
                 key = 'true'
             elif key is False:
@@ -370,7 +439,7 @@
                 key = 'null'
             elif isinstance(key, (int, long)):
                 key = str(key)
-            elif _skipkeys:
+            elif self.skipkeys:
                 continue
             else:
                 raise TypeError("key " + repr(key) + " is not a string")
@@ -378,10 +447,10 @@
                 first = False
             else:
                 yield item_separator
-            yield _encoder(key)
-            yield _key_separator
+            yield '"' + self.encoder(key) + '"'
+            yield self.key_separator
             if isinstance(value, basestring):
-                yield _encoder(value)
+                yield '"' + self.encoder(value) + '"'
             elif value is None:
                 yield 'null'
             elif value is True:
@@ -391,26 +460,28 @@
             elif isinstance(value, (int, long)):
                 yield str(value)
             elif isinstance(value, float):
-                yield _floatstr(value)
+                yield self._floatstr(value)
             else:
                 if isinstance(value, (list, tuple)):
-                    chunks = _iterencode_list(value, _current_indent_level)
+                    chunks = self._iterencode_list(value, markers,
+                                                   _current_indent_level)
                 elif isinstance(value, dict):
-                    chunks = _iterencode_dict(value, _current_indent_level)
+                    chunks = self._iterencode_dict(value, markers,
+                                                   _current_indent_level)
                 else:
-                    chunks = _iterencode(value, _current_indent_level)
+                    chunks = self._iterencode(value, markers,
+                                              _current_indent_level)
                 for chunk in chunks:
                     yield chunk
         if newline_indent is not None:
             _current_indent_level -= 1
-            yield '\n' + (' ' * (_indent * _current_indent_level))
+            yield '\n' + (' ' * (self.indent * _current_indent_level))
         yield '}'
-        if markers is not None:
-            del markers[markerid]
+        self._remove_markers(markers, dct)
 
-    def _iterencode(o, _current_indent_level):
+    def _iterencode(self, o, markers, _current_indent_level):
         if isinstance(o, basestring):
-            yield _encoder(o)
+            yield '"' + self.encoder(o) + '"'
         elif o is None:
             yield 'null'
         elif o is True:
@@ -420,23 +491,19 @@
         elif isinstance(o, (int, long)):
             yield str(o)
         elif isinstance(o, float):
-            yield _floatstr(o)
+            yield self._floatstr(o)
         elif isinstance(o, (list, tuple)):
-            for chunk in _iterencode_list(o, _current_indent_level):
+            for chunk in self._iterencode_list(o, markers,
+                                               _current_indent_level):
                 yield chunk
         elif isinstance(o, dict):
-            for chunk in _iterencode_dict(o, _current_indent_level):
+            for chunk in self._iterencode_dict(o, markers,
+                                               _current_indent_level):
                 yield chunk
         else:
-            if markers is not None:
-                markerid = id(o)
-                if markerid in markers:
-                    raise ValueError("Circular reference detected")
-                markers[markerid] = o
-            o = _default(o)
-            for chunk in _iterencode(o, _current_indent_level):
+            self._mark_markers(markers, o)
+            obj = self.default(o)
+            for chunk in self._iterencode(obj, markers,
+                                          _current_indent_level):
                 yield chunk
-            if markers is not None:
-                del markers[markerid]
-
-    return _iterencode
+            self._remove_markers(markers, o)
diff --git a/lib-python/modified-2.7/json/tests/test_unicode.py b/lib-python/modified-2.7/json/tests/test_unicode.py
--- a/lib-python/modified-2.7/json/tests/test_unicode.py
+++ b/lib-python/modified-2.7/json/tests/test_unicode.py
@@ -80,3 +80,9 @@
         self.assertEqual(type(json.loads(u'["a"]')[0]), unicode)
         # Issue 10038.
         self.assertEqual(type(json.loads('"foo"')), unicode)
+
+    def test_encode_not_utf_8(self):
+        self.assertEqual(json.dumps('\xb1\xe6', encoding='iso8859-2'),
+                         '"\\u0105\\u0107"')
+        self.assertEqual(json.dumps(['\xb1\xe6'], encoding='iso8859-2'),
+                         '["\\u0105\\u0107"]')
diff --git a/lib-python/modified-2.7/opcode.py b/lib-python/modified-2.7/opcode.py
--- a/lib-python/modified-2.7/opcode.py
+++ b/lib-python/modified-2.7/opcode.py
@@ -189,7 +189,6 @@
 def_op('MAP_ADD', 147)
 
 # pypy modification, experimental bytecode
-def_op('CALL_LIKELY_BUILTIN', 200)    # #args + (#kwargs << 8)
 def_op('LOOKUP_METHOD', 201)          # Index in name list
 hasname.append(201)
 def_op('CALL_METHOD', 202)            # #args not including 'self'
diff --git a/lib-python/modified-2.7/pickle.py b/lib-python/modified-2.7/pickle.py
--- a/lib-python/modified-2.7/pickle.py
+++ b/lib-python/modified-2.7/pickle.py
@@ -168,7 +168,7 @@
 
 # Pickling machinery
 
-class Pickler:
+class Pickler(object):
 
     def __init__(self, file, protocol=None):
         """This takes a file-like object for writing a pickle data stream.
@@ -873,7 +873,7 @@
 
 # Unpickling machinery
 
-class Unpickler:
+class Unpickler(object):
 
     def __init__(self, file):
         """This takes a file-like object for reading a pickle data stream.
diff --git a/lib-python/2.7/pkgutil.py b/lib-python/modified-2.7/pkgutil.py
copy from lib-python/2.7/pkgutil.py
copy to lib-python/modified-2.7/pkgutil.py
--- a/lib-python/2.7/pkgutil.py
+++ b/lib-python/modified-2.7/pkgutil.py
@@ -244,7 +244,8 @@
         return mod
 
     def get_data(self, pathname):
-        return open(pathname, "rb").read()
+        with open(pathname, "rb") as f:
+            return f.read()
 
     def _reopen(self):
         if self.file and self.file.closed:
diff --git a/lib-python/modified-2.7/site.py b/lib-python/modified-2.7/site.py
--- a/lib-python/modified-2.7/site.py
+++ b/lib-python/modified-2.7/site.py
@@ -454,10 +454,10 @@
     __builtin__.copyright = _Printer("copyright", sys.copyright)
     __builtin__.credits = _Printer(
         "credits",
-        "PyPy is maintained by the PyPy developers: http://codespeak.net/pypy")
+        "PyPy is maintained by the PyPy developers: http://pypy.org/")
     __builtin__.license = _Printer(
         "license",
-        "See http://codespeak.net/svn/pypy/dist/LICENSE")
+        "See https://bitbucket.org/pypy/pypy/src/default/LICENSE")
 
 
 
diff --git a/lib-python/modified-2.7/sqlite3/test/regression.py b/lib-python/modified-2.7/sqlite3/test/regression.py
--- a/lib-python/modified-2.7/sqlite3/test/regression.py
+++ b/lib-python/modified-2.7/sqlite3/test/regression.py
@@ -274,6 +274,18 @@
         cur.execute("UPDATE foo SET id = 3 WHERE id = 1")
         self.assertEqual(cur.description, None)
 
+    def CheckStatementCache(self):
+        cur = self.con.cursor()
+        cur.execute("CREATE TABLE foo (id INTEGER)")
+        values = [(i,) for i in xrange(5)]
+        cur.executemany("INSERT INTO foo (id) VALUES (?)", values)
+
+        cur.execute("SELECT id FROM foo")
+        self.assertEqual(list(cur), values)
+        self.con.commit()
+        cur.execute("SELECT id FROM foo")
+        self.assertEqual(list(cur), values)
+
 def suite():
     regression_suite = unittest.makeSuite(RegressionTests, "Check")
     return unittest.TestSuite((regression_suite,))
diff --git a/lib-python/modified-2.7/ssl.py b/lib-python/modified-2.7/ssl.py
--- a/lib-python/modified-2.7/ssl.py
+++ b/lib-python/modified-2.7/ssl.py
@@ -62,7 +62,6 @@
 from _ssl import OPENSSL_VERSION_NUMBER, OPENSSL_VERSION_INFO, OPENSSL_VERSION
 from _ssl import SSLError
 from _ssl import CERT_NONE, CERT_OPTIONAL, CERT_REQUIRED
-from _ssl import PROTOCOL_SSLv2, PROTOCOL_SSLv3, PROTOCOL_SSLv23, PROTOCOL_TLSv1
 from _ssl import RAND_status, RAND_egd, RAND_add
 from _ssl import \
      SSL_ERROR_ZERO_RETURN, \
@@ -74,6 +73,18 @@
      SSL_ERROR_WANT_CONNECT, \
      SSL_ERROR_EOF, \
      SSL_ERROR_INVALID_ERROR_CODE
+from _ssl import PROTOCOL_SSLv3, PROTOCOL_SSLv23, PROTOCOL_TLSv1
+_PROTOCOL_NAMES = {
+    PROTOCOL_TLSv1: "TLSv1",
+    PROTOCOL_SSLv23: "SSLv23",
+    PROTOCOL_SSLv3: "SSLv3",
+}
+try:
+    from _ssl import PROTOCOL_SSLv2
+except ImportError:
+    pass
+else:
+    _PROTOCOL_NAMES[PROTOCOL_SSLv2] = "SSLv2"
 
 from socket import socket, _fileobject, error as socket_error
 from socket import getnameinfo as _getnameinfo
@@ -400,16 +411,7 @@
     return DER_cert_to_PEM_cert(dercert)
 
 def get_protocol_name(protocol_code):
-    if protocol_code == PROTOCOL_TLSv1:
-        return "TLSv1"
-    elif protocol_code == PROTOCOL_SSLv23:
-        return "SSLv23"
-    elif protocol_code == PROTOCOL_SSLv2:
-        return "SSLv2"
-    elif protocol_code == PROTOCOL_SSLv3:
-        return "SSLv3"
-    else:
-        return ""
+    return _PROTOCOL_NAMES.get(protocol_code, '')
 
 
 # a replacement for the old socket.ssl function
diff --git a/lib-python/modified-2.7/test/regrtest.py b/lib-python/modified-2.7/test/regrtest.py
--- a/lib-python/modified-2.7/test/regrtest.py
+++ b/lib-python/modified-2.7/test/regrtest.py
@@ -1403,7 +1403,26 @@
         test_zipimport
         test_zlib
         """,
-    'openbsd3':
+    'openbsd4':
+        """
+        test_ascii_formatd
+        test_bsddb
+        test_bsddb3
+        test_ctypes
+        test_dl
+        test_epoll
+        test_gdbm
+        test_locale
+        test_normalization
+        test_ossaudiodev
+        test_pep277
+        test_tcl
+        test_tk
+        test_ttk_guionly
+        test_ttk_textonly
+        test_multiprocessing
+        """,
+    'openbsd5':
         """
         test_ascii_formatd
         test_bsddb
diff --git a/lib-python/modified-2.7/test/test_array.py b/lib-python/modified-2.7/test/test_array.py
--- a/lib-python/modified-2.7/test/test_array.py
+++ b/lib-python/modified-2.7/test/test_array.py
@@ -295,9 +295,10 @@
         )
 
         b = array.array(self.badtypecode())
-        self.assertRaises(TypeError, "a + b")
-
-        self.assertRaises(TypeError, "a + 'bad'")
+        with self.assertRaises(TypeError):
+            a + b
+        with self.assertRaises(TypeError):
+            a + 'bad'
 
     def test_iadd(self):
         a = array.array(self.typecode, self.example[::-1])
@@ -316,9 +317,10 @@
         )
 
         b = array.array(self.badtypecode())
-        self.assertRaises(TypeError, "a += b")
-
-        self.assertRaises(TypeError, "a += 'bad'")
+        with self.assertRaises(TypeError):
+            a += b
+        with self.assertRaises(TypeError):
+            a += 'bad'
 
     def test_mul(self):
         a = 5*array.array(self.typecode, self.example)
@@ -345,7 +347,8 @@
             array.array(self.typecode)
         )
 
-        self.assertRaises(TypeError, "a * 'bad'")
+        with self.assertRaises(TypeError):
+            a * 'bad'
 
     def test_imul(self):
         a = array.array(self.typecode, self.example)
@@ -374,7 +377,8 @@
         a *= -1
         self.assertEqual(a, array.array(self.typecode))
 
-        self.assertRaises(TypeError, "a *= 'bad'")
+        with self.assertRaises(TypeError):
+            a *= 'bad'
 
     def test_getitem(self):
         a = array.array(self.typecode, self.example)
diff --git a/lib-python/modified-2.7/test/test_bz2.py b/lib-python/modified-2.7/test/test_bz2.py
--- a/lib-python/modified-2.7/test/test_bz2.py
+++ b/lib-python/modified-2.7/test/test_bz2.py
@@ -50,6 +50,7 @@
         self.filename = TESTFN
 
     def tearDown(self):
+        test_support.gc_collect()
         if os.path.isfile(self.filename):
             os.unlink(self.filename)
 
diff --git a/lib-python/modified-2.7/test/test_codecs.py b/lib-python/modified-2.7/test/test_codecs.py
deleted file mode 100644
--- a/lib-python/modified-2.7/test/test_codecs.py
+++ /dev/null
@@ -1,1615 +0,0 @@
-from test import test_support
-import unittest
-import codecs
-import sys, StringIO, _testcapi
-
-class Queue(object):
-    """
-    queue: write bytes at one end, read bytes from the other end
-    """
-    def __init__(self):
-        self._buffer = ""
-
-    def write(self, chars):
-        self._buffer += chars
-
-    def read(self, size=-1):
-        if size<0:
-            s = self._buffer
-            self._buffer = ""
-            return s
-        else:
-            s = self._buffer[:size]
-            self._buffer = self._buffer[size:]
-            return s
-
-class ReadTest(unittest.TestCase):
-    def check_partial(self, input, partialresults):
-        # get a StreamReader for the encoding and feed the bytestring version
-        # of input to the reader byte by byte. Read everything available from
-        # the StreamReader and check that the results equal the appropriate
-        # entries from partialresults.
-        q = Queue()
-        r = codecs.getreader(self.encoding)(q)
-        result = u""
-        for (c, partialresult) in zip(input.encode(self.encoding), partialresults):
-            q.write(c)
-            result += r.read()
-            self.assertEqual(result, partialresult)
-        # check that there's nothing left in the buffers
-        self.assertEqual(r.read(), u"")
-        self.assertEqual(r.bytebuffer, "")
-        self.assertEqual(r.charbuffer, u"")
-
-        # do the check again, this time using a incremental decoder
-        d = codecs.getincrementaldecoder(self.encoding)()
-        result = u""
-        for (c, partialresult) in zip(input.encode(self.encoding), partialresults):
-            result += d.decode(c)
-            self.assertEqual(result, partialresult)
-        # check that there's nothing left in the buffers
-        self.assertEqual(d.decode("", True), u"")
-        self.assertEqual(d.buffer, "")
-
-        # Check whether the reset method works properly
-        d.reset()
-        result = u""
-        for (c, partialresult) in zip(input.encode(self.encoding), partialresults):
-            result += d.decode(c)
-            self.assertEqual(result, partialresult)
-        # check that there's nothing left in the buffers
-        self.assertEqual(d.decode("", True), u"")
-        self.assertEqual(d.buffer, "")
-
-        # check iterdecode()
-        encoded = input.encode(self.encoding)
-        self.assertEqual(
-            input,
-            u"".join(codecs.iterdecode(encoded, self.encoding))
-        )
-
-    def test_readline(self):
-        def getreader(input):
-            stream = StringIO.StringIO(input.encode(self.encoding))
-            return codecs.getreader(self.encoding)(stream)
-
-        def readalllines(input, keepends=True, size=None):
-            reader = getreader(input)
-            lines = []
-            while True:
-                line = reader.readline(size=size, keepends=keepends)
-                if not line:
-                    break
-                lines.append(line)
-            return "|".join(lines)
-
-        s = u"foo\nbar\r\nbaz\rspam\u2028eggs"
-        sexpected = u"foo\n|bar\r\n|baz\r|spam\u2028|eggs"
-        sexpectednoends = u"foo|bar|baz|spam|eggs"
-        self.assertEqual(readalllines(s, True), sexpected)
-        self.assertEqual(readalllines(s, False), sexpectednoends)
-        self.assertEqual(readalllines(s, True, 10), sexpected)
-        self.assertEqual(readalllines(s, False, 10), sexpectednoends)
-
-        # Test long lines (multiple calls to read() in readline())
-        vw = []
-        vwo = []
-        for (i, lineend) in enumerate(u"\n \r\n \r \u2028".split()):
-            vw.append((i*200)*u"\3042" + lineend)
-            vwo.append((i*200)*u"\3042")
-        self.assertEqual(readalllines("".join(vw), True), "".join(vw))
-        self.assertEqual(readalllines("".join(vw), False),"".join(vwo))
-
-        # Test lines where the first read might end with \r, so the
-        # reader has to look ahead whether this is a lone \r or a \r\n
-        for size in xrange(80):
-            for lineend in u"\n \r\n \r \u2028".split():
-                s = 10*(size*u"a" + lineend + u"xxx\n")
-                reader = getreader(s)
-                for i in xrange(10):
-                    self.assertEqual(
-                        reader.readline(keepends=True),
-                        size*u"a" + lineend,
-                    )
-                reader = getreader(s)
-                for i in xrange(10):
-                    self.assertEqual(
-                        reader.readline(keepends=False),
-                        size*u"a",
-                    )
-
-    def test_bug1175396(self):
-        s = [
-            '<%!--===================================================\r\n',
-            '    BLOG index page: show recent articles,\r\n',
-            '    today\'s articles, or articles of a specific date.\r\n',
-            '========================================================--%>\r\n',
-            '<%@inputencoding="ISO-8859-1"%>\r\n',
-            '<%@pagetemplate=TEMPLATE.y%>\r\n',
-            '<%@import=import frog.util, frog%>\r\n',
-            '<%@import=import frog.objects%>\r\n',
-            '<%@import=from frog.storageerrors import StorageError%>\r\n',
-            '<%\r\n',
-            '\r\n',
-            'import logging\r\n',
-            'log=logging.getLogger("Snakelets.logger")\r\n',
-            '\r\n',
-            '\r\n',
-            'user=self.SessionCtx.user\r\n',
-            'storageEngine=self.SessionCtx.storageEngine\r\n',
-            '\r\n',
-            '\r\n',
-            'def readArticlesFromDate(date, count=None):\r\n',
-            '    entryids=storageEngine.listBlogEntries(date)\r\n',
-            '    entryids.reverse() # descending\r\n',
-            '    if count:\r\n',
-            '        entryids=entryids[:count]\r\n',
-            '    try:\r\n',
-            '        return [ frog.objects.BlogEntry.load(storageEngine, date, Id) for Id in entryids ]\r\n',
-            '    except StorageError,x:\r\n',
-            '        log.error("Error loading articles: "+str(x))\r\n',
-            '        self.abort("cannot load articles")\r\n',
-            '\r\n',
-            'showdate=None\r\n',
-            '\r\n',
-            'arg=self.Request.getArg()\r\n',
-            'if arg=="today":\r\n',
-            '    #-------------------- TODAY\'S ARTICLES\r\n',
-            '    self.write("Today\'s articles")\r\n',
-            '    showdate = frog.util.isodatestr() \r\n',
-            '    entries = readArticlesFromDate(showdate)\r\n',
-            'elif arg=="active":\r\n',
-            '    #-------------------- ACTIVE ARTICLES redirect\r\n',
-            '    self.Yredirect("active.y")\r\n',
-            'elif arg=="login":\r\n',
-            '    #-------------------- LOGIN PAGE redirect\r\n',
-            '    self.Yredirect("login.y")\r\n',
-            'elif arg=="date":\r\n',
-            '    #-------------------- ARTICLES OF A SPECIFIC DATE\r\n',
-            '    showdate = self.Request.getParameter("date")\r\n',
-            '    self.write("Articles written on %s"% frog.util.mediumdatestr(showdate))\r\n',
-            '    entries = readArticlesFromDate(showdate)\r\n',
-            'else:\r\n',
-            '    #-------------------- RECENT ARTICLES\r\n',
-            '    self.write("Recent articles")\r\n',
-            '    dates=storageEngine.listBlogEntryDates()\r\n',
-            '    if dates:\r\n',
-            '        entries=[]\r\n',
-            '        SHOWAMOUNT=10\r\n',
-            '        for showdate in dates:\r\n',
-            '            entries.extend( readArticlesFromDate(showdate, SHOWAMOUNT-len(entries)) )\r\n',
-            '            if len(entries)>=SHOWAMOUNT:\r\n',
-            '                break\r\n',
-            '                \r\n',
-        ]
-        stream = StringIO.StringIO("".join(s).encode(self.encoding))
-        reader = codecs.getreader(self.encoding)(stream)
-        for (i, line) in enumerate(reader):
-            self.assertEqual(line, s[i])
-
-    def test_readlinequeue(self):
-        q = Queue()
-        writer = codecs.getwriter(self.encoding)(q)
-        reader = codecs.getreader(self.encoding)(q)
-
-        # No lineends
-        writer.write(u"foo\r")
-        self.assertEqual(reader.readline(keepends=False), u"foo")
-        writer.write(u"\nbar\r")
-        self.assertEqual(reader.readline(keepends=False), u"")
-        self.assertEqual(reader.readline(keepends=False), u"bar")
-        writer.write(u"baz")
-        self.assertEqual(reader.readline(keepends=False), u"baz")
-        self.assertEqual(reader.readline(keepends=False), u"")
-
-        # Lineends
-        writer.write(u"foo\r")
-        self.assertEqual(reader.readline(keepends=True), u"foo\r")
-        writer.write(u"\nbar\r")
-        self.assertEqual(reader.readline(keepends=True), u"\n")
-        self.assertEqual(reader.readline(keepends=True), u"bar\r")
-        writer.write(u"baz")
-        self.assertEqual(reader.readline(keepends=True), u"baz")
-        self.assertEqual(reader.readline(keepends=True), u"")
-        writer.write(u"foo\r\n")
-        self.assertEqual(reader.readline(keepends=True), u"foo\r\n")
-
-    def test_bug1098990_a(self):
-        s1 = u"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy\r\n"
-        s2 = u"offending line: ladfj askldfj klasdj fskla dfzaskdj fasklfj laskd fjasklfzzzzaa%whereisthis!!!\r\n"
-        s3 = u"next line.\r\n"
-
-        s = (s1+s2+s3).encode(self.encoding)
-        stream = StringIO.StringIO(s)
-        reader = codecs.getreader(self.encoding)(stream)
-        self.assertEqual(reader.readline(), s1)
-        self.assertEqual(reader.readline(), s2)
-        self.assertEqual(reader.readline(), s3)
-        self.assertEqual(reader.readline(), u"")
-
-    def test_bug1098990_b(self):
-        s1 = u"aaaaaaaaaaaaaaaaaaaaaaaa\r\n"
-        s2 = u"bbbbbbbbbbbbbbbbbbbbbbbb\r\n"
-        s3 = u"stillokay:bbbbxx\r\n"
-        s4 = u"broken!!!!badbad\r\n"
-        s5 = u"againokay.\r\n"
-
-        s = (s1+s2+s3+s4+s5).encode(self.encoding)
-        stream = StringIO.StringIO(s)
-        reader = codecs.getreader(self.encoding)(stream)
-        self.assertEqual(reader.readline(), s1)
-        self.assertEqual(reader.readline(), s2)
-        self.assertEqual(reader.readline(), s3)
-        self.assertEqual(reader.readline(), s4)
-        self.assertEqual(reader.readline(), s5)
-        self.assertEqual(reader.readline(), u"")
-
-class UTF32Test(ReadTest):
-    encoding = "utf-32"
-
-    spamle = ('\xff\xfe\x00\x00'
-              's\x00\x00\x00p\x00\x00\x00a\x00\x00\x00m\x00\x00\x00'
-              's\x00\x00\x00p\x00\x00\x00a\x00\x00\x00m\x00\x00\x00')
-    spambe = ('\x00\x00\xfe\xff'
-              '\x00\x00\x00s\x00\x00\x00p\x00\x00\x00a\x00\x00\x00m'
-              '\x00\x00\x00s\x00\x00\x00p\x00\x00\x00a\x00\x00\x00m')
-
-    def test_only_one_bom(self):
-        _,_,reader,writer = codecs.lookup(self.encoding)
-        # encode some stream
-        s = StringIO.StringIO()
-        f = writer(s)
-        f.write(u"spam")
-        f.write(u"spam")
-        d = s.getvalue()
-        # check whether there is exactly one BOM in it
-        self.assertTrue(d == self.spamle or d == self.spambe)
-        # try to read it back
-        s = StringIO.StringIO(d)
-        f = reader(s)
-        self.assertEqual(f.read(), u"spamspam")
-
-    def test_badbom(self):
-        s = StringIO.StringIO(4*"\xff")
-        f = codecs.getreader(self.encoding)(s)
-        self.assertRaises(UnicodeError, f.read)
-
-        s = StringIO.StringIO(8*"\xff")
-        f = codecs.getreader(self.encoding)(s)
-        self.assertRaises(UnicodeError, f.read)
-
-    def test_partial(self):
-        self.check_partial(
-            u"\x00\xff\u0100\uffff",
-            [
-                u"", # first byte of BOM read
-                u"", # second byte of BOM read
-                u"", # third byte of BOM read
-                u"", # fourth byte of BOM read => byteorder known
-                u"",
-                u"",
-                u"",
-                u"\x00",
-                u"\x00",
-                u"\x00",
-                u"\x00",
-                u"\x00\xff",
-                u"\x00\xff",
-                u"\x00\xff",
-                u"\x00\xff",
-                u"\x00\xff\u0100",
-                u"\x00\xff\u0100",
-                u"\x00\xff\u0100",
-                u"\x00\xff\u0100",
-                u"\x00\xff\u0100\uffff",
-            ]
-        )
-
-    def test_handlers(self):
-        self.assertEqual((u'\ufffd', 1),
-                         codecs.utf_32_decode('\x01', 'replace', True))
-        self.assertEqual((u'', 1),
-                         codecs.utf_32_decode('\x01', 'ignore', True))
-
-    def test_errors(self):
-        self.assertRaises(UnicodeDecodeError, codecs.utf_32_decode,
-                          "\xff", "strict", True)
-
-    def test_issue8941(self):
-        # Issue #8941: insufficient result allocation when decoding into
-        # surrogate pairs on UCS-2 builds.
-        encoded_le = '\xff\xfe\x00\x00' + '\x00\x00\x01\x00' * 1024
-        self.assertEqual(u'\U00010000' * 1024,
-                         codecs.utf_32_decode(encoded_le)[0])
-        encoded_be = '\x00\x00\xfe\xff' + '\x00\x01\x00\x00' * 1024
-        self.assertEqual(u'\U00010000' * 1024,
-                         codecs.utf_32_decode(encoded_be)[0])
-
-class UTF32LETest(ReadTest):
-    encoding = "utf-32-le"
-
-    def test_partial(self):
-        self.check_partial(
-            u"\x00\xff\u0100\uffff",
-            [
-                u"",
-                u"",
-                u"",
-                u"\x00",
-                u"\x00",
-                u"\x00",
-                u"\x00",
-                u"\x00\xff",
-                u"\x00\xff",
-                u"\x00\xff",
-                u"\x00\xff",
-                u"\x00\xff\u0100",
-                u"\x00\xff\u0100",
-                u"\x00\xff\u0100",
-                u"\x00\xff\u0100",
-                u"\x00\xff\u0100\uffff",
-            ]
-        )
-
-    def test_simple(self):
-        self.assertEqual(u"\U00010203".encode(self.encoding), "\x03\x02\x01\x00")
-
-    def test_errors(self):
-        self.assertRaises(UnicodeDecodeError, codecs.utf_32_le_decode,
-                          "\xff", "strict", True)
-
-    def test_issue8941(self):
-        # Issue #8941: insufficient result allocation when decoding into
-        # surrogate pairs on UCS-2 builds.
-        encoded = '\x00\x00\x01\x00' * 1024
-        self.assertEqual(u'\U00010000' * 1024,
-                         codecs.utf_32_le_decode(encoded)[0])
-
-class UTF32BETest(ReadTest):
-    encoding = "utf-32-be"
-
-    def test_partial(self):
-        self.check_partial(
-            u"\x00\xff\u0100\uffff",
-            [
-                u"",
-                u"",
-                u"",
-                u"\x00",
-                u"\x00",
-                u"\x00",
-                u"\x00",
-                u"\x00\xff",
-                u"\x00\xff",
-                u"\x00\xff",
-                u"\x00\xff",
-                u"\x00\xff\u0100",
-                u"\x00\xff\u0100",
-                u"\x00\xff\u0100",
-                u"\x00\xff\u0100",
-                u"\x00\xff\u0100\uffff",
-            ]
-        )
-
-    def test_simple(self):
-        self.assertEqual(u"\U00010203".encode(self.encoding), "\x00\x01\x02\x03")
-
-    def test_errors(self):
-        self.assertRaises(UnicodeDecodeError, codecs.utf_32_be_decode,
-                          "\xff", "strict", True)
-
-    def test_issue8941(self):
-        # Issue #8941: insufficient result allocation when decoding into
-        # surrogate pairs on UCS-2 builds.
-        encoded = '\x00\x01\x00\x00' * 1024
-        self.assertEqual(u'\U00010000' * 1024,
-                         codecs.utf_32_be_decode(encoded)[0])
-
-
-class UTF16Test(ReadTest):
-    encoding = "utf-16"
-
-    spamle = '\xff\xfes\x00p\x00a\x00m\x00s\x00p\x00a\x00m\x00'
-    spambe = '\xfe\xff\x00s\x00p\x00a\x00m\x00s\x00p\x00a\x00m'
-
-    def test_only_one_bom(self):
-        _,_,reader,writer = codecs.lookup(self.encoding)
-        # encode some stream
-        s = StringIO.StringIO()
-        f = writer(s)
-        f.write(u"spam")
-        f.write(u"spam")
-        d = s.getvalue()
-        # check whether there is exactly one BOM in it
-        self.assertTrue(d == self.spamle or d == self.spambe)
-        # try to read it back
-        s = StringIO.StringIO(d)
-        f = reader(s)
-        self.assertEqual(f.read(), u"spamspam")
-
-    def test_badbom(self):
-        s = StringIO.StringIO("\xff\xff")
-        f = codecs.getreader(self.encoding)(s)
-        self.assertRaises(UnicodeError, f.read)
-
-        s = StringIO.StringIO("\xff\xff\xff\xff")
-        f = codecs.getreader(self.encoding)(s)
-        self.assertRaises(UnicodeError, f.read)
-
-    def test_partial(self):
-        self.check_partial(
-            u"\x00\xff\u0100\uffff",
-            [
-                u"", # first byte of BOM read
-                u"", # second byte of BOM read => byteorder known
-                u"",
-                u"\x00",
-                u"\x00",
-                u"\x00\xff",
-                u"\x00\xff",
-                u"\x00\xff\u0100",
-                u"\x00\xff\u0100",
-                u"\x00\xff\u0100\uffff",
-            ]
-        )
-
-    def test_handlers(self):
-        self.assertEqual((u'\ufffd', 1),
-                         codecs.utf_16_decode('\x01', 'replace', True))
-        self.assertEqual((u'', 1),
-                         codecs.utf_16_decode('\x01', 'ignore', True))
-
-    def test_errors(self):
-        self.assertRaises(UnicodeDecodeError, codecs.utf_16_decode, "\xff", "strict", True)
-
-    def test_bug691291(self):
-        # Files are always opened in binary mode, even if no binary mode was
-        # specified.  This means that no automatic conversion of '\n' is done
-        # on reading and writing.
-        s1 = u'Hello\r\nworld\r\n'
-
-        s = s1.encode(self.encoding)
-        try:
-            with open(test_support.TESTFN, 'wb') as fp:
-                fp.write(s)
-            with codecs.open(test_support.TESTFN, 'U', encoding=self.encoding) as reader:
-                self.assertEqual(reader.read(), s1)
-        finally:
-            test_support.unlink(test_support.TESTFN)
-
-class UTF16LETest(ReadTest):
-    encoding = "utf-16-le"
-
-    def test_partial(self):
-        self.check_partial(
-            u"\x00\xff\u0100\uffff",
-            [
-                u"",
-                u"\x00",
-                u"\x00",
-                u"\x00\xff",
-                u"\x00\xff",
-                u"\x00\xff\u0100",
-                u"\x00\xff\u0100",
-                u"\x00\xff\u0100\uffff",
-            ]
-        )
-
-    def test_errors(self):
-        self.assertRaises(UnicodeDecodeError, codecs.utf_16_le_decode, "\xff", "strict", True)
-
-class UTF16BETest(ReadTest):
-    encoding = "utf-16-be"
-
-    def test_partial(self):
-        self.check_partial(
-            u"\x00\xff\u0100\uffff",
-            [
-                u"",
-                u"\x00",
-                u"\x00",
-                u"\x00\xff",
-                u"\x00\xff",
-                u"\x00\xff\u0100",
-                u"\x00\xff\u0100",
-                u"\x00\xff\u0100\uffff",
-            ]
-        )
-
-    def test_errors(self):
-        self.assertRaises(UnicodeDecodeError, codecs.utf_16_be_decode, "\xff", "strict", True)
-
-class UTF8Test(ReadTest):
-    encoding = "utf-8"
-
-    def test_partial(self):
-        self.check_partial(
-            u"\x00\xff\u07ff\u0800\uffff",
-            [
-                u"\x00",
-                u"\x00",
-                u"\x00\xff",
-                u"\x00\xff",
-                u"\x00\xff\u07ff",
-                u"\x00\xff\u07ff",
-                u"\x00\xff\u07ff",
-                u"\x00\xff\u07ff\u0800",
-                u"\x00\xff\u07ff\u0800",
-                u"\x00\xff\u07ff\u0800",
-                u"\x00\xff\u07ff\u0800\uffff",
-            ]
-        )
-
-class UTF7Test(ReadTest):
-    encoding = "utf-7"
-
-    def test_partial(self):
-        self.check_partial(
-            u"a+-b",
-            [
-                u"a",
-                u"a",
-                u"a+",
-                u"a+-",
-                u"a+-b",
-            ]
-        )
-
-class UTF16ExTest(unittest.TestCase):
-
-    def test_errors(self):
-        self.assertRaises(UnicodeDecodeError, codecs.utf_16_ex_decode, "\xff", "strict", 0, True)
-
-    def test_bad_args(self):
-        self.assertRaises(TypeError, codecs.utf_16_ex_decode)
-
-class ReadBufferTest(unittest.TestCase):
-
-    def test_array(self):
-        import array
-        self.assertEqual(
-            codecs.readbuffer_encode(array.array("c", "spam")),
-            ("spam", 4)
-        )
-
-    def test_empty(self):
-        self.assertEqual(codecs.readbuffer_encode(""), ("", 0))
-
-    def test_bad_args(self):
-        self.assertRaises(TypeError, codecs.readbuffer_encode)
-        self.assertRaises(TypeError, codecs.readbuffer_encode, 42)
-
-class CharBufferTest(unittest.TestCase):
-
-    def test_string(self):
-        self.assertEqual(codecs.charbuffer_encode("spam"), ("spam", 4))
-
-    def test_empty(self):
-        self.assertEqual(codecs.charbuffer_encode(""), ("", 0))
-
-    def test_bad_args(self):
-        self.assertRaises(TypeError, codecs.charbuffer_encode)
-        self.assertRaises(TypeError, codecs.charbuffer_encode, 42)
-
-class UTF8SigTest(ReadTest):
-    encoding = "utf-8-sig"
-
-    def test_partial(self):
-        self.check_partial(
-            u"\ufeff\x00\xff\u07ff\u0800\uffff",
-            [
-                u"",
-                u"",
-                u"", # First BOM has been read and skipped
-                u"",
-                u"",
-                u"\ufeff", # Second BOM has been read and emitted
-                u"\ufeff\x00", # "\x00" read and emitted
-                u"\ufeff\x00", # First byte of encoded u"\xff" read
-                u"\ufeff\x00\xff", # Second byte of encoded u"\xff" read
-                u"\ufeff\x00\xff", # First byte of encoded u"\u07ff" read
-                u"\ufeff\x00\xff\u07ff", # Second byte of encoded u"\u07ff" read
-                u"\ufeff\x00\xff\u07ff",
-                u"\ufeff\x00\xff\u07ff",
-                u"\ufeff\x00\xff\u07ff\u0800",
-                u"\ufeff\x00\xff\u07ff\u0800",
-                u"\ufeff\x00\xff\u07ff\u0800",
-                u"\ufeff\x00\xff\u07ff\u0800\uffff",
-            ]
-        )
-
-    def test_bug1601501(self):
-        # SF bug #1601501: check that the codec works with a buffer
-        unicode("\xef\xbb\xbf", "utf-8-sig")
-
-    def test_bom(self):
-        d = codecs.getincrementaldecoder("utf-8-sig")()
-        s = u"spam"
-        self.assertEqual(d.decode(s.encode("utf-8-sig")), s)
-
-    def test_stream_bom(self):
-        unistring = u"ABC\u00A1\u2200XYZ"
-        bytestring = codecs.BOM_UTF8 + "ABC\xC2\xA1\xE2\x88\x80XYZ"
-
-        reader = codecs.getreader("utf-8-sig")
-        for sizehint in [None] + range(1, 11) + \
-                        [64, 128, 256, 512, 1024]:
-            istream = reader(StringIO.StringIO(bytestring))
-            ostream = StringIO.StringIO()
-            while 1:
-                if sizehint is not None:
-                    data = istream.read(sizehint)
-                else:
-                    data = istream.read()
-
-                if not data:
-                    break
-                ostream.write(data)
-
-            got = ostream.getvalue()
-            self.assertEqual(got, unistring)
-
-    def test_stream_bare(self):
-        unistring = u"ABC\u00A1\u2200XYZ"
-        bytestring = "ABC\xC2\xA1\xE2\x88\x80XYZ"
-
-        reader = codecs.getreader("utf-8-sig")
-        for sizehint in [None] + range(1, 11) + \
-                        [64, 128, 256, 512, 1024]:
-            istream = reader(StringIO.StringIO(bytestring))
-            ostream = StringIO.StringIO()
-            while 1:
-                if sizehint is not None:
-                    data = istream.read(sizehint)
-                else:
-                    data = istream.read()
-
-                if not data:
-                    break
-                ostream.write(data)
-
-            got = ostream.getvalue()
-            self.assertEqual(got, unistring)
-
-class EscapeDecodeTest(unittest.TestCase):
-    def test_empty(self):
-        self.assertEqual(codecs.escape_decode(""), ("", 0))
-
-class RecodingTest(unittest.TestCase):
-    def test_recoding(self):
-        f = StringIO.StringIO()
-        f2 = codecs.EncodedFile(f, "unicode_internal", "utf-8")
-        f2.write(u"a")
-        f2.close()
-        # Python used to crash on this at exit because of a refcount
-        # bug in _codecsmodule.c
-
-# From RFC 3492
-punycode_testcases = [
-    # A Arabic (Egyptian):
-    (u"\u0644\u064A\u0647\u0645\u0627\u0628\u062A\u0643\u0644"
-     u"\u0645\u0648\u0634\u0639\u0631\u0628\u064A\u061F",
-     "egbpdaj6bu4bxfgehfvwxn"),
-    # B Chinese (simplified):
-    (u"\u4ED6\u4EEC\u4E3A\u4EC0\u4E48\u4E0D\u8BF4\u4E2D\u6587",
-     "ihqwcrb4cv8a8dqg056pqjye"),
-    # C Chinese (traditional):
-    (u"\u4ED6\u5011\u7232\u4EC0\u9EBD\u4E0D\u8AAA\u4E2D\u6587",
-     "ihqwctvzc91f659drss3x8bo0yb"),
-    # D Czech: Proprostnemluvesky
-    (u"\u0050\u0072\u006F\u010D\u0070\u0072\u006F\u0073\u0074"
-     u"\u011B\u006E\u0065\u006D\u006C\u0075\u0076\u00ED\u010D"
-     u"\u0065\u0073\u006B\u0079",
-     "Proprostnemluvesky-uyb24dma41a"),
-    # E Hebrew:
-    (u"\u05DC\u05DE\u05D4\u05D4\u05DD\u05E4\u05E9\u05D5\u05D8"
-     u"\u05DC\u05D0\u05DE\u05D3\u05D1\u05E8\u05D9\u05DD\u05E2"
-     u"\u05D1\u05E8\u05D9\u05EA",
-     "4dbcagdahymbxekheh6e0a7fei0b"),
-    # F Hindi (Devanagari):
-    (u"\u092F\u0939\u0932\u094B\u0917\u0939\u093F\u0928\u094D"
-    u"\u0926\u0940\u0915\u094D\u092F\u094B\u0902\u0928\u0939"
-    u"\u0940\u0902\u092C\u094B\u0932\u0938\u0915\u0924\u0947"
-    u"\u0939\u0948\u0902",
-    "i1baa7eci9glrd9b2ae1bj0hfcgg6iyaf8o0a1dig0cd"),
-
-    #(G) Japanese (kanji and hiragana):
-    (u"\u306A\u305C\u307F\u3093\u306A\u65E5\u672C\u8A9E\u3092"
-    u"\u8A71\u3057\u3066\u304F\u308C\u306A\u3044\u306E\u304B",
-     "n8jok5ay5dzabd5bym9f0cm5685rrjetr6pdxa"),
-
-    # (H) Korean (Hangul syllables):
-    (u"\uC138\uACC4\uC758\uBAA8\uB4E0\uC0AC\uB78C\uB4E4\uC774"
-     u"\uD55C\uAD6D\uC5B4\uB97C\uC774\uD574\uD55C\uB2E4\uBA74"
-     u"\uC5BC\uB9C8\uB098\uC88B\uC744\uAE4C",
-     "989aomsvi5e83db1d2a355cv1e0vak1dwrv93d5xbh15a0dt30a5j"
-     "psd879ccm6fea98c"),
-
-    # (I) Russian (Cyrillic):
-    (u"\u043F\u043E\u0447\u0435\u043C\u0443\u0436\u0435\u043E"
-     u"\u043D\u0438\u043D\u0435\u0433\u043E\u0432\u043E\u0440"
-     u"\u044F\u0442\u043F\u043E\u0440\u0443\u0441\u0441\u043A"
-     u"\u0438",
-     "b1abfaaepdrnnbgefbaDotcwatmq2g4l"),
-
-    # (J) Spanish: PorqunopuedensimplementehablarenEspaol
-    (u"\u0050\u006F\u0072\u0071\u0075\u00E9\u006E\u006F\u0070"
-     u"\u0075\u0065\u0064\u0065\u006E\u0073\u0069\u006D\u0070"
-     u"\u006C\u0065\u006D\u0065\u006E\u0074\u0065\u0068\u0061"
-     u"\u0062\u006C\u0061\u0072\u0065\u006E\u0045\u0073\u0070"
-     u"\u0061\u00F1\u006F\u006C",
-     "PorqunopuedensimplementehablarenEspaol-fmd56a"),
-
-    # (K) Vietnamese:
-    #  Tisaohkhngthch\
-    #   nitingVit
-    (u"\u0054\u1EA1\u0069\u0073\u0061\u006F\u0068\u1ECD\u006B"
-     u"\u0068\u00F4\u006E\u0067\u0074\u0068\u1EC3\u0063\u0068"
-     u"\u1EC9\u006E\u00F3\u0069\u0074\u0069\u1EBF\u006E\u0067"
-     u"\u0056\u0069\u1EC7\u0074",
-     "TisaohkhngthchnitingVit-kjcr8268qyxafd2f1b9g"),
-
-    #(L) 3B
-    (u"\u0033\u5E74\u0042\u7D44\u91D1\u516B\u5148\u751F",
-     "3B-ww4c5e180e575a65lsy2b"),
-
-    # (M) -with-SUPER-MONKEYS
-    (u"\u5B89\u5BA4\u5948\u7F8E\u6075\u002D\u0077\u0069\u0074"
-     u"\u0068\u002D\u0053\u0055\u0050\u0045\u0052\u002D\u004D"
-     u"\u004F\u004E\u004B\u0045\u0059\u0053",
-     "-with-SUPER-MONKEYS-pc58ag80a8qai00g7n9n"),
-
-    # (N) Hello-Another-Way-
-    (u"\u0048\u0065\u006C\u006C\u006F\u002D\u0041\u006E\u006F"
-     u"\u0074\u0068\u0065\u0072\u002D\u0057\u0061\u0079\u002D"
-     u"\u305D\u308C\u305E\u308C\u306E\u5834\u6240",
-     "Hello-Another-Way--fc4qua05auwb3674vfr0b"),
-
-    # (O) 2
-    (u"\u3072\u3068\u3064\u5C4B\u6839\u306E\u4E0B\u0032",
-     "2-u9tlzr9756bt3uc0v"),
-
-    # (P) MajiKoi5
-    (u"\u004D\u0061\u006A\u0069\u3067\u004B\u006F\u0069\u3059"
-     u"\u308B\u0035\u79D2\u524D",
-     "MajiKoi5-783gue6qz075azm5e"),
-
-     # (Q) de
-    (u"\u30D1\u30D5\u30A3\u30FC\u0064\u0065\u30EB\u30F3\u30D0",
-     "de-jg4avhby1noc0d"),
-
-    # (R) 
-    (u"\u305D\u306E\u30B9\u30D4\u30FC\u30C9\u3067",
-     "d9juau41awczczp"),
-
-    # (S) -> $1.00 <-
-    (u"\u002D\u003E\u0020\u0024\u0031\u002E\u0030\u0030\u0020"
-     u"\u003C\u002D",
-     "-> $1.00 <--")
-    ]
-
-for i in punycode_testcases:
-    if len(i)!=2:
-        print repr(i)
-
-class PunycodeTest(unittest.TestCase):
-    def test_encode(self):
-        for uni, puny in punycode_testcases:
-            # Need to convert both strings to lower case, since
-            # some of the extended encodings use upper case, but our
-            # code produces only lower case. Converting just puny to
-            # lower is also insufficient, since some of the input characters
-            # are upper case.
-            self.assertEqual(uni.encode("punycode").lower(), puny.lower())
-
-    def test_decode(self):
-        for uni, puny in punycode_testcases:
-            self.assertEqual(uni, puny.decode("punycode"))
-
-class UnicodeInternalTest(unittest.TestCase):
-    def test_bug1251300(self):
-        # Decoding with unicode_internal used to not correctly handle "code
-        # points" above 0x10ffff on UCS-4 builds.
-        if sys.maxunicode > 0xffff:
-            ok = [
-                ("\x00\x10\xff\xff", u"\U0010ffff"),
-                ("\x00\x00\x01\x01", u"\U00000101"),
-                ("", u""),
-            ]
-            not_ok = [
-                "\x7f\xff\xff\xff",
-                "\x80\x00\x00\x00",
-                "\x81\x00\x00\x00",
-                "\x00",
-                "\x00\x00\x00\x00\x00",
-            ]
-            for internal, uni in ok:
-                if sys.byteorder == "little":
-                    internal = "".join(reversed(internal))
-                self.assertEqual(uni, internal.decode("unicode_internal"))
-            for internal in not_ok:
-                if sys.byteorder == "little":
-                    internal = "".join(reversed(internal))
-                self.assertRaises(UnicodeDecodeError, internal.decode,
-                    "unicode_internal")
-
-    def test_decode_error_attributes(self):
-        if sys.maxunicode > 0xffff:
-            try:
-                "\x00\x00\x00\x00\x00\x11\x11\x00".decode("unicode_internal")
-            except UnicodeDecodeError, ex:
-                self.assertEqual("unicode_internal", ex.encoding)
-                self.assertEqual("\x00\x00\x00\x00\x00\x11\x11\x00", ex.object)
-                self.assertEqual(4, ex.start)
-                self.assertEqual(8, ex.end)
-            else:
-                self.fail()
-
-    def test_decode_callback(self):
-        if sys.maxunicode > 0xffff:
-            codecs.register_error("UnicodeInternalTest", codecs.ignore_errors)
-            decoder = codecs.getdecoder("unicode_internal")
-            ab = u"ab".encode("unicode_internal")
-            ignored = decoder("%s\x22\x22\x22\x22%s" % (ab[:4], ab[4:]),
-                "UnicodeInternalTest")
-            self.assertEqual((u"ab", 12), ignored)
-
-    def test_encode_length(self):
-        # Issue 3739
-        encoder = codecs.getencoder("unicode_internal")
-        self.assertEqual(encoder(u"a")[1], 1)
-        self.assertEqual(encoder(u"\xe9\u0142")[1], 2)
-
-        encoder = codecs.getencoder("string-escape")
-        self.assertEqual(encoder(r'\x00')[1], 4)
-
-# From http://www.gnu.org/software/libidn/draft-josefsson-idn-test-vectors.html
-nameprep_tests = [
-    # 3.1 Map to nothing.
-    ('foo\xc2\xad\xcd\x8f\xe1\xa0\x86\xe1\xa0\x8bbar'
-     '\xe2\x80\x8b\xe2\x81\xa0baz\xef\xb8\x80\xef\xb8\x88\xef'
-     '\xb8\x8f\xef\xbb\xbf',
-     'foobarbaz'),
-    # 3.2 Case folding ASCII U+0043 U+0041 U+0046 U+0045.
-    ('CAFE',
-     'cafe'),
-    # 3.3 Case folding 8bit U+00DF (german sharp s).
-    # The original test case is bogus; it says \xc3\xdf
-    ('\xc3\x9f',
-     'ss'),
-    # 3.4 Case folding U+0130 (turkish capital I with dot).
-    ('\xc4\xb0',
-     'i\xcc\x87'),
-    # 3.5 Case folding multibyte U+0143 U+037A.
-    ('\xc5\x83\xcd\xba',
-     '\xc5\x84 \xce\xb9'),
-    # 3.6 Case folding U+2121 U+33C6 U+1D7BB.
-    # XXX: skip this as it fails in UCS-2 mode
-    #('\xe2\x84\xa1\xe3\x8f\x86\xf0\x9d\x9e\xbb',
-    # 'telc\xe2\x88\x95kg\xcf\x83'),
-    (None, None),
-    # 3.7 Normalization of U+006a U+030c U+00A0 U+00AA.
-    ('j\xcc\x8c\xc2\xa0\xc2\xaa',
-     '\xc7\xb0 a'),
-    # 3.8 Case folding U+1FB7 and normalization.
-    ('\xe1\xbe\xb7',
-     '\xe1\xbe\xb6\xce\xb9'),
-    # 3.9 Self-reverting case folding U+01F0 and normalization.
-    # The original test case is bogus, it says `\xc7\xf0'
-    ('\xc7\xb0',
-     '\xc7\xb0'),
-    # 3.10 Self-reverting case folding U+0390 and normalization.
-    ('\xce\x90',
-     '\xce\x90'),
-    # 3.11 Self-reverting case folding U+03B0 and normalization.
-    ('\xce\xb0',
-     '\xce\xb0'),
-    # 3.12 Self-reverting case folding U+1E96 and normalization.
-    ('\xe1\xba\x96',
-     '\xe1\xba\x96'),
-    # 3.13 Self-reverting case folding U+1F56 and normalization.
-    ('\xe1\xbd\x96',
-     '\xe1\xbd\x96'),
-    # 3.14 ASCII space character U+0020.
-    (' ',
-     ' '),
-    # 3.15 Non-ASCII 8bit space character U+00A0.
-    ('\xc2\xa0',
-     ' '),
-    # 3.16 Non-ASCII multibyte space character U+1680.
-    ('\xe1\x9a\x80',
-     None),
-    # 3.17 Non-ASCII multibyte space character U+2000.
-    ('\xe2\x80\x80',
-     ' '),
-    # 3.18 Zero Width Space U+200b.
-    ('\xe2\x80\x8b',
-     ''),
-    # 3.19 Non-ASCII multibyte space character U+3000.
-    ('\xe3\x80\x80',
-     ' '),
-    # 3.20 ASCII control characters U+0010 U+007F.
-    ('\x10\x7f',
-     '\x10\x7f'),
-    # 3.21 Non-ASCII 8bit control character U+0085.
-    ('\xc2\x85',
-     None),
-    # 3.22 Non-ASCII multibyte control character U+180E.
-    ('\xe1\xa0\x8e',
-     None),
-    # 3.23 Zero Width No-Break Space U+FEFF.
-    ('\xef\xbb\xbf',
-     ''),
-    # 3.24 Non-ASCII control character U+1D175.
-    ('\xf0\x9d\x85\xb5',
-     None),
-    # 3.25 Plane 0 private use character U+F123.
-    ('\xef\x84\xa3',
-     None),
-    # 3.26 Plane 15 private use character U+F1234.
-    ('\xf3\xb1\x88\xb4',
-     None),
-    # 3.27 Plane 16 private use character U+10F234.
-    ('\xf4\x8f\x88\xb4',
-     None),
-    # 3.28 Non-character code point U+8FFFE.
-    ('\xf2\x8f\xbf\xbe',
-     None),
-    # 3.29 Non-character code point U+10FFFF.
-    ('\xf4\x8f\xbf\xbf',
-     None),
-    # 3.30 Surrogate code U+DF42.
-    ('\xed\xbd\x82',
-     None),
-    # 3.31 Non-plain text character U+FFFD.
-    ('\xef\xbf\xbd',
-     None),
-    # 3.32 Ideographic description character U+2FF5.
-    ('\xe2\xbf\xb5',
-     None),
-    # 3.33 Display property character U+0341.
-    ('\xcd\x81',
-     '\xcc\x81'),
-    # 3.34 Left-to-right mark U+200E.
-    ('\xe2\x80\x8e',
-     None),
-    # 3.35 Deprecated U+202A.
-    ('\xe2\x80\xaa',
-     None),
-    # 3.36 Language tagging character U+E0001.
-    ('\xf3\xa0\x80\x81',
-     None),
-    # 3.37 Language tagging character U+E0042.
-    ('\xf3\xa0\x81\x82',
-     None),
-    # 3.38 Bidi: RandALCat character U+05BE and LCat characters.
-    ('foo\xd6\xbebar',
-     None),
-    # 3.39 Bidi: RandALCat character U+FD50 and LCat characters.
-    ('foo\xef\xb5\x90bar',
-     None),
-    # 3.40 Bidi: RandALCat character U+FB38 and LCat characters.
-    ('foo\xef\xb9\xb6bar',
-     'foo \xd9\x8ebar'),
-    # 3.41 Bidi: RandALCat without trailing RandALCat U+0627 U+0031.
-    ('\xd8\xa71',
-     None),
-    # 3.42 Bidi: RandALCat character U+0627 U+0031 U+0628.
-    ('\xd8\xa71\xd8\xa8',
-     '\xd8\xa71\xd8\xa8'),
-    # 3.43 Unassigned code point U+E0002.
-    # Skip this test as we allow unassigned
-    #('\xf3\xa0\x80\x82',
-    # None),
-    (None, None),
-    # 3.44 Larger test (shrinking).
-    # Original test case reads \xc3\xdf
-    ('X\xc2\xad\xc3\x9f\xc4\xb0\xe2\x84\xa1j\xcc\x8c\xc2\xa0\xc2'
-     '\xaa\xce\xb0\xe2\x80\x80',
-     'xssi\xcc\x87tel\xc7\xb0 a\xce\xb0 '),
-    # 3.45 Larger test (expanding).
-    # Original test case reads \xc3\x9f
-    ('X\xc3\x9f\xe3\x8c\x96\xc4\xb0\xe2\x84\xa1\xe2\x92\x9f\xe3\x8c'
-     '\x80',
-     'xss\xe3\x82\xad\xe3\x83\xad\xe3\x83\xa1\xe3\x83\xbc\xe3'
-     '\x83\x88\xe3\x83\xabi\xcc\x87tel\x28d\x29\xe3\x82'
-     '\xa2\xe3\x83\x91\xe3\x83\xbc\xe3\x83\x88')
-    ]
-
-
-class NameprepTest(unittest.TestCase):
-    def test_nameprep(self):
-        from encodings.idna import nameprep
-        for pos, (orig, prepped) in enumerate(nameprep_tests):
-            if orig is None:
-                # Skipped
-                continue
-            # The Unicode strings are given in UTF-8
-            orig = unicode(orig, "utf-8")
-            if prepped is None:
-                # Input contains prohibited characters
-                self.assertRaises(UnicodeError, nameprep, orig)
-            else:
-                prepped = unicode(prepped, "utf-8")
-                try:
-                    self.assertEqual(nameprep(orig), prepped)
-                except Exception,e:
-                    raise test_support.TestFailed("Test 3.%d: %s" % (pos+1, str(e)))
-
-class IDNACodecTest(unittest.TestCase):
-    def test_builtin_decode(self):
-        self.assertEqual(unicode("python.org", "idna"), u"python.org")
-        self.assertEqual(unicode("python.org.", "idna"), u"python.org.")
-        self.assertEqual(unicode("xn--pythn-mua.org", "idna"), u"pyth\xf6n.org")
-        self.assertEqual(unicode("xn--pythn-mua.org.", "idna"), u"pyth\xf6n.org.")
-
-    def test_builtin_encode(self):
-        self.assertEqual(u"python.org".encode("idna"), "python.org")
-        self.assertEqual("python.org.".encode("idna"), "python.org.")
-        self.assertEqual(u"pyth\xf6n.org".encode("idna"), "xn--pythn-mua.org")
-        self.assertEqual(u"pyth\xf6n.org.".encode("idna"), "xn--pythn-mua.org.")
-
-    def test_stream(self):
-        import StringIO
-        r = codecs.getreader("idna")(StringIO.StringIO("abc"))
-        r.read(3)
-        self.assertEqual(r.read(), u"")
-
-    def test_incremental_decode(self):
-        self.assertEqual(
-            "".join(codecs.iterdecode("python.org", "idna")),
-            u"python.org"
-        )
-        self.assertEqual(
-            "".join(codecs.iterdecode("python.org.", "idna")),
-            u"python.org."
-        )
-        self.assertEqual(
-            "".join(codecs.iterdecode("xn--pythn-mua.org.", "idna")),
-            u"pyth\xf6n.org."
-        )
-        self.assertEqual(
-            "".join(codecs.iterdecode("xn--pythn-mua.org.", "idna")),
-            u"pyth\xf6n.org."
-        )
-
-        decoder = codecs.getincrementaldecoder("idna")()
-        self.assertEqual(decoder.decode("xn--xam", ), u"")
-        self.assertEqual(decoder.decode("ple-9ta.o", ), u"\xe4xample.")
-        self.assertEqual(decoder.decode(u"rg"), u"")
-        self.assertEqual(decoder.decode(u"", True), u"org")
-
-        decoder.reset()
-        self.assertEqual(decoder.decode("xn--xam", ), u"")
-        self.assertEqual(decoder.decode("ple-9ta.o", ), u"\xe4xample.")
-        self.assertEqual(decoder.decode("rg."), u"org.")
-        self.assertEqual(decoder.decode("", True), u"")
-
-    def test_incremental_encode(self):
-        self.assertEqual(
-            "".join(codecs.iterencode(u"python.org", "idna")),
-            "python.org"
-        )
-        self.assertEqual(
-            "".join(codecs.iterencode(u"python.org.", "idna")),
-            "python.org."
-        )
-        self.assertEqual(
-            "".join(codecs.iterencode(u"pyth\xf6n.org.", "idna")),
-            "xn--pythn-mua.org."
-        )
-        self.assertEqual(
-            "".join(codecs.iterencode(u"pyth\xf6n.org.", "idna")),
-            "xn--pythn-mua.org."
-        )
-
-        encoder = codecs.getincrementalencoder("idna")()
-        self.assertEqual(encoder.encode(u"\xe4x"), "")
-        self.assertEqual(encoder.encode(u"ample.org"), "xn--xample-9ta.")
-        self.assertEqual(encoder.encode(u"", True), "org")
-
-        encoder.reset()
-        self.assertEqual(encoder.encode(u"\xe4x"), "")
-        self.assertEqual(encoder.encode(u"ample.org."), "xn--xample-9ta.org.")
-        self.assertEqual(encoder.encode(u"", True), "")
-
-class CodecsModuleTest(unittest.TestCase):
-
-    def test_decode(self):
-        self.assertEqual(codecs.decode('\xe4\xf6\xfc', 'latin-1'),
-                          u'\xe4\xf6\xfc')
-        self.assertRaises(TypeError, codecs.decode)
-        self.assertEqual(codecs.decode('abc'), u'abc')
-        self.assertRaises(UnicodeDecodeError, codecs.decode, '\xff', 'ascii')
-
-    def test_encode(self):
-        self.assertEqual(codecs.encode(u'\xe4\xf6\xfc', 'latin-1'),
-                          '\xe4\xf6\xfc')
-        self.assertRaises(TypeError, codecs.encode)
-        self.assertRaises(LookupError, codecs.encode, "foo", "__spam__")
-        self.assertEqual(codecs.encode(u'abc'), 'abc')
-        self.assertRaises(UnicodeEncodeError, codecs.encode, u'\xffff', 'ascii')
-
-    def test_register(self):
-        self.assertRaises(TypeError, codecs.register)
-        self.assertRaises(TypeError, codecs.register, 42)
-
-    def test_lookup(self):
-        self.assertRaises(TypeError, codecs.lookup)
-        self.assertRaises(LookupError, codecs.lookup, "__spam__")
-        self.assertRaises(LookupError, codecs.lookup, " ")
-
-    def test_getencoder(self):
-        self.assertRaises(TypeError, codecs.getencoder)
-        self.assertRaises(LookupError, codecs.getencoder, "__spam__")
-
-    def test_getdecoder(self):
-        self.assertRaises(TypeError, codecs.getdecoder)
-        self.assertRaises(LookupError, codecs.getdecoder, "__spam__")
-
-    def test_getreader(self):
-        self.assertRaises(TypeError, codecs.getreader)
-        self.assertRaises(LookupError, codecs.getreader, "__spam__")
-
-    def test_getwriter(self):
-        self.assertRaises(TypeError, codecs.getwriter)
-        self.assertRaises(LookupError, codecs.getwriter, "__spam__")
-
-class StreamReaderTest(unittest.TestCase):
-
-    def setUp(self):
-        self.reader = codecs.getreader('utf-8')
-        self.stream = StringIO.StringIO('\xed\x95\x9c\n\xea\xb8\x80')
-
-    def test_readlines(self):
-        f = self.reader(self.stream)
-        self.assertEqual(f.readlines(), [u'\ud55c\n', u'\uae00'])
-
-class EncodedFileTest(unittest.TestCase):
-
-    def test_basic(self):
-        f = StringIO.StringIO('\xed\x95\x9c\n\xea\xb8\x80')
-        ef = codecs.EncodedFile(f, 'utf-16-le', 'utf-8')
-        self.assertEqual(ef.read(), '\\\xd5\n\x00\x00\xae')
-
-        f = StringIO.StringIO()
-        ef = codecs.EncodedFile(f, 'utf-8', 'latin1')
-        ef.write('\xc3\xbc')
-        self.assertEqual(f.getvalue(), '\xfc')
-
-class Str2StrTest(unittest.TestCase):
-
-    def test_read(self):
-        sin = "\x80".encode("base64_codec")
-        reader = codecs.getreader("base64_codec")(StringIO.StringIO(sin))
-        sout = reader.read()
-        self.assertEqual(sout, "\x80")
-        self.assertIsInstance(sout, str)
-
-    def test_readline(self):
-        sin = "\x80".encode("base64_codec")
-        reader = codecs.getreader("base64_codec")(StringIO.StringIO(sin))
-        sout = reader.readline()
-        self.assertEqual(sout, "\x80")
-        self.assertIsInstance(sout, str)
-
-all_unicode_encodings = [
-    "ascii",
-    "base64_codec",
-    ## "big5",
-    ## "big5hkscs",
-    "charmap",
-    "cp037",
-    "cp1006",
-    "cp1026",
-    "cp1140",
-    "cp1250",
-    "cp1251",
-    "cp1252",
-    "cp1253",
-    "cp1254",
-    "cp1255",
-    "cp1256",
-    "cp1257",
-    "cp1258",
-    "cp424",
-    "cp437",
-    "cp500",
-    "cp720",
-    "cp737",
-    "cp775",
-    "cp850",
-    "cp852",
-    "cp855",
-    "cp856",
-    "cp857",
-    "cp858",
-    "cp860",
-    "cp861",
-    "cp862",
-    "cp863",
-    "cp864",
-    "cp865",
-    "cp866",
-    "cp869",
-    "cp874",
-    "cp875",
-    ## "cp932",
-    ## "cp949",
-    ## "cp950",
-    ## "euc_jis_2004",
-    ## "euc_jisx0213",
-    ## "euc_jp",
-    ## "euc_kr",
-    ## "gb18030",
-    ## "gb2312",
-    ## "gbk",
-    "hex_codec",
-    "hp_roman8",
-    ## "hz",
-    "idna",
-    ## "iso2022_jp",
-    ## "iso2022_jp_1",
-    ## "iso2022_jp_2",
-    ## "iso2022_jp_2004",
-    ## "iso2022_jp_3",
-    ## "iso2022_jp_ext",
-    ## "iso2022_kr",
-    "iso8859_1",
-    "iso8859_10",
-    "iso8859_11",
-    "iso8859_13",
-    "iso8859_14",
-    "iso8859_15",
-    "iso8859_16",
-    "iso8859_2",
-    "iso8859_3",
-    "iso8859_4",
-    "iso8859_5",
-    "iso8859_6",
-    "iso8859_7",
-    "iso8859_8",
-    "iso8859_9",
-    ## "johab",
-    "koi8_r",
-    "koi8_u",
-    "latin_1",
-    "mac_cyrillic",
-    "mac_greek",
-    "mac_iceland",
-    "mac_latin2",
-    "mac_roman",
-    "mac_turkish",
-    "palmos",
-    "ptcp154",
-    "punycode",
-    "raw_unicode_escape",
-    "rot_13",
-    ## "shift_jis",
-    ## "shift_jis_2004",
-    ## "shift_jisx0213",
-    "tis_620",
-    "unicode_escape",
-    "unicode_internal",
-    "utf_16",
-    "utf_16_be",
-    "utf_16_le",
-    "utf_7",
-    "utf_8",
-]
-
-if hasattr(codecs, "mbcs_encode"):
-    all_unicode_encodings.append("mbcs")
-
-# The following encodings work only with str, not unicode
-all_string_encodings = [
-    "quopri_codec",
-    "string_escape",
-    "uu_codec",
-]
-
-# The following encoding is not tested, because it's not supposed
-# to work:
-#    "undefined"
-
-# The following encodings don't work in stateful mode
-broken_unicode_with_streams = [
-    "base64_codec",
-    "hex_codec",
-    "punycode",
-    "unicode_internal"
-]
-broken_incremental_coders = broken_unicode_with_streams[:]
-
-# The following encodings only support "strict" mode
-only_strict_mode = [
-    "idna",
-    "zlib_codec",
-    "bz2_codec",
-]
-
-try:
-    import bz2
-except ImportError:
-    pass
-else:
-    all_unicode_encodings.append("bz2_codec")
-    broken_unicode_with_streams.append("bz2_codec")
-
-try:
-    import zlib
-except ImportError:
-    pass
-else:
-    all_unicode_encodings.append("zlib_codec")
-    broken_unicode_with_streams.append("zlib_codec")
-
-class BasicUnicodeTest(unittest.TestCase):
-    def test_basics(self):
-        s = u"abc123" # all codecs should be able to encode these
-        for encoding in all_unicode_encodings:
-            name = codecs.lookup(encoding).name
-            if encoding.endswith("_codec"):
-                name += "_codec"
-            elif encoding == "latin_1":
-                name = "latin_1"
-            self.assertEqual(encoding.replace("_", "-"), name.replace("_", "-"))
-            (bytes, size) = codecs.getencoder(encoding)(s)
-            self.assertEqual(size, len(s), "%r != %r (encoding=%r)" % (size, len(s), encoding))
-            (chars, size) = codecs.getdecoder(encoding)(bytes)
-            self.assertEqual(chars, s, "%r != %r (encoding=%r)" % (chars, s, encoding))
-
-            if encoding not in broken_unicode_with_streams:
-                # check stream reader/writer
-                q = Queue()
-                writer = codecs.getwriter(encoding)(q)
-                encodedresult = ""
-                for c in s:
-                    writer.write(c)
-                    encodedresult += q.read()
-                q = Queue()
-                reader = codecs.getreader(encoding)(q)
-                decodedresult = u""
-                for c in encodedresult:
-                    q.write(c)
-                    decodedresult += reader.read()
-                self.assertEqual(decodedresult, s, "%r != %r (encoding=%r)" % (decodedresult, s, encoding))
-
-            if encoding not in broken_incremental_coders:
-                # check incremental decoder/encoder (fetched via the Python
-                # and C API) and iterencode()/iterdecode()
-                try:
-                    encoder = codecs.getincrementalencoder(encoding)()
-                    cencoder = _testcapi.codec_incrementalencoder(encoding)
-                except LookupError: # no IncrementalEncoder
-                    pass
-                else:
-                    # check incremental decoder/encoder
-                    encodedresult = ""
-                    for c in s:
-                        encodedresult += encoder.encode(c)
-                    encodedresult += encoder.encode(u"", True)
-                    decoder = codecs.getincrementaldecoder(encoding)()
-                    decodedresult = u""
-                    for c in encodedresult:
-                        decodedresult += decoder.decode(c)
-                    decodedresult += decoder.decode("", True)
-                    self.assertEqual(decodedresult, s, "%r != %r (encoding=%r)" % (decodedresult, s, encoding))
-
-                    # check C API
-                    encodedresult = ""
-                    for c in s:
-                        encodedresult += cencoder.encode(c)
-                    encodedresult += cencoder.encode(u"", True)
-                    cdecoder = _testcapi.codec_incrementaldecoder(encoding)
-                    decodedresult = u""
-                    for c in encodedresult:
-                        decodedresult += cdecoder.decode(c)
-                    decodedresult += cdecoder.decode("", True)
-                    self.assertEqual(decodedresult, s, "%r != %r (encoding=%r)" % (decodedresult, s, encoding))
-
-                    # check iterencode()/iterdecode()
-                    result = u"".join(codecs.iterdecode(codecs.iterencode(s, encoding), encoding))
-                    self.assertEqual(result, s, "%r != %r (encoding=%r)" % (result, s, encoding))
-
-                    # check iterencode()/iterdecode() with empty string
-                    result = u"".join(codecs.iterdecode(codecs.iterencode(u"", encoding), encoding))
-                    self.assertEqual(result, u"")
-
-                if encoding not in only_strict_mode:
-                    # check incremental decoder/encoder with errors argument
-                    try:
-                        encoder = codecs.getincrementalencoder(encoding)("ignore")
-                        cencoder = _testcapi.codec_incrementalencoder(encoding, "ignore")
-                    except LookupError: # no IncrementalEncoder
-                        pass
-                    else:
-                        encodedresult = "".join(encoder.encode(c) for c in s)
-                        decoder = codecs.getincrementaldecoder(encoding)("ignore")
-                        decodedresult = u"".join(decoder.decode(c) for c in encodedresult)
-                        self.assertEqual(decodedresult, s, "%r != %r (encoding=%r)" % (decodedresult, s, encoding))
-
-                        encodedresult = "".join(cencoder.encode(c) for c in s)
-                        cdecoder = _testcapi.codec_incrementaldecoder(encoding, "ignore")
-                        decodedresult = u"".join(cdecoder.decode(c) for c in encodedresult)
-                        self.assertEqual(decodedresult, s, "%r != %r (encoding=%r)" % (decodedresult, s, encoding))
-
-    def test_seek(self):
-        # all codecs should be able to encode these
-        s = u"%s\n%s\n" % (100*u"abc123", 100*u"def456")
-        for encoding in all_unicode_encodings:
-            if encoding == "idna": # FIXME: See SF bug #1163178
-                continue
-            if encoding in broken_unicode_with_streams:
-                continue
-            reader = codecs.getreader(encoding)(StringIO.StringIO(s.encode(encoding)))
-            for t in xrange(5):
-                # Test that calling seek resets the internal codec state and buffers
-                reader.seek(0, 0)
-                line = reader.readline()
-                self.assertEqual(s[:len(line)], line)
-
-    def test_bad_decode_args(self):
-        for encoding in all_unicode_encodings:
-            decoder = codecs.getdecoder(encoding)
-            self.assertRaises(TypeError, decoder)
-            if encoding not in ("idna", "punycode"):
-                self.assertRaises(TypeError, decoder, 42)
-
-    def test_bad_encode_args(self):
-        for encoding in all_unicode_encodings:
-            encoder = codecs.getencoder(encoding)
-            self.assertRaises(TypeError, encoder)
-
-    def test_encoding_map_type_initialized(self):
-        from encodings import cp1140
-        # This used to crash, we are only verifying there's no crash.
-        table_type = type(cp1140.encoding_table)
-        self.assertEqual(table_type, table_type)
-
-class BasicStrTest(unittest.TestCase):
-    def test_basics(self):
-        s = "abc123"
-        for encoding in all_string_encodings:
-            (bytes, size) = codecs.getencoder(encoding)(s)
-            self.assertEqual(size, len(s))
-            (chars, size) = codecs.getdecoder(encoding)(bytes)
-            self.assertEqual(chars, s, "%r != %r (encoding=%r)" % (chars, s, encoding))
-
-class CharmapTest(unittest.TestCase):
-    def test_decode_with_string_map(self):
-        self.assertEqual(
-            codecs.charmap_decode("\x00\x01\x02", "strict", u"abc"),
-            (u"abc", 3)
-        )
-
-        self.assertEqual(
-            codecs.charmap_decode("\x00\x01\x02", "replace", u"ab"),
-            (u"ab\ufffd", 3)
-        )
-
-        self.assertEqual(
-            codecs.charmap_decode("\x00\x01\x02", "replace", u"ab\ufffe"),
-            (u"ab\ufffd", 3)
-        )
-
-        self.assertEqual(
-            codecs.charmap_decode("\x00\x01\x02", "ignore", u"ab"),
-            (u"ab", 3)
-        )
-
-        self.assertEqual(
-            codecs.charmap_decode("\x00\x01\x02", "ignore", u"ab\ufffe"),
-            (u"ab", 3)
-        )
-
-        allbytes = "".join(chr(i) for i in xrange(256))
-        self.assertEqual(
-            codecs.charmap_decode(allbytes, "ignore", u""),
-            (u"", len(allbytes))
-        )
-
-class WithStmtTest(unittest.TestCase):
-    def test_encodedfile(self):
-        f = StringIO.StringIO("\xc3\xbc")
-        with codecs.EncodedFile(f, "latin-1", "utf-8") as ef:
-            self.assertEqual(ef.read(), "\xfc")
-
-    def test_streamreaderwriter(self):
-        f = StringIO.StringIO("\xc3\xbc")
-        info = codecs.lookup("utf-8")
-        with codecs.StreamReaderWriter(f, info.streamreader,
-                                       info.streamwriter, 'strict') as srw:
-            self.assertEqual(srw.read(), u"\xfc")
-
-
-class BomTest(unittest.TestCase):
-    def test_seek0(self):
-        data = u"1234567890"
-        tests = ("utf-16",
-                 "utf-16-le",
-                 "utf-16-be",
-                 "utf-32",
-                 "utf-32-le",
-                 "utf-32-be")
-        for encoding in tests:
-            # Check if the BOM is written only once
-            with codecs.open(test_support.TESTFN, 'w+', encoding=encoding) as f:
-                f.write(data)
-                f.write(data)
-                f.seek(0)
-                self.assertEqual(f.read(), data * 2)
-                f.seek(0)
-                self.assertEqual(f.read(), data * 2)
-
-            # Check that the BOM is written after a seek(0)
-            with codecs.open(test_support.TESTFN, 'w+', encoding=encoding) as f:
-                f.write(data[0])
-                self.assertNotEqual(f.tell(), 0)
-                f.seek(0)
-                f.write(data)
-                f.seek(0)
-                self.assertEqual(f.read(), data)
-
-            # (StreamWriter) Check that the BOM is written after a seek(0)
-            with codecs.open(test_support.TESTFN, 'w+', encoding=encoding) as f:
-                f.writer.write(data[0])
-                self.assertNotEqual(f.writer.tell(), 0)
-                f.writer.seek(0)
-                f.writer.write(data)
-                f.seek(0)
-                self.assertEqual(f.read(), data)
-
-            # Check that the BOM is not written after a seek() at a position
-            # different than the start
-            with codecs.open(test_support.TESTFN, 'w+', encoding=encoding) as f:
-                f.write(data)
-                f.seek(f.tell())
-                f.write(data)
-                f.seek(0)
-                self.assertEqual(f.read(), data * 2)
-
-            # (StreamWriter) Check that the BOM is not written after a seek()
-            # at a position different than the start
-            with codecs.open(test_support.TESTFN, 'w+', encoding=encoding) as f:
-                f.writer.write(data)
-                f.writer.seek(f.writer.tell())
-                f.writer.write(data)
-                f.seek(0)
-                self.assertEqual(f.read(), data * 2)
-
-
-def test_main():
-    test_support.run_unittest(
-        UTF32Test,
-        UTF32LETest,
-        UTF32BETest,
-        UTF16Test,
-        UTF16LETest,
-        UTF16BETest,
-        UTF8Test,
-        UTF8SigTest,
-        UTF7Test,
-        UTF16ExTest,
-        ReadBufferTest,
-        CharBufferTest,
-        EscapeDecodeTest,
-        RecodingTest,
-        PunycodeTest,
-        UnicodeInternalTest,
-        NameprepTest,
-        IDNACodecTest,
-        CodecsModuleTest,
-        StreamReaderTest,
-        EncodedFileTest,
-        Str2StrTest,
-        BasicUnicodeTest,
-        BasicStrTest,
-        CharmapTest,
-        WithStmtTest,
-        BomTest,
-    )
-
-
-if __name__ == "__main__":
-    test_main()
diff --git a/lib-python/modified-2.7/test/test_descr.py b/lib-python/modified-2.7/test/test_descr.py
--- a/lib-python/modified-2.7/test/test_descr.py
+++ b/lib-python/modified-2.7/test/test_descr.py
@@ -4399,13 +4399,10 @@
         self.assertTrue(l.__add__ != [5].__add__)
         self.assertTrue(l.__add__ != l.__mul__)
         self.assertTrue(l.__add__.__name__ == '__add__')
-        if hasattr(l.__add__, '__self__'):
-            # CPython
-            self.assertTrue(l.__add__.__self__ is l)
+        self.assertTrue(l.__add__.__self__ is l)
+        if hasattr(l.__add__, '__objclass__'):   # CPython
             self.assertTrue(l.__add__.__objclass__ is list)
-        else:
-            # Python implementations where [].__add__ is a normal bound method
-            self.assertTrue(l.__add__.im_self is l)
+        else:                                    # PyPy
             self.assertTrue(l.__add__.im_class is list)
         self.assertEqual(l.__add__.__doc__, list.__add__.__doc__)
         try:
diff --git a/lib-python/modified-2.7/test/test_dis.py b/lib-python/modified-2.7/test/test_dis.py
deleted file mode 100644
--- a/lib-python/modified-2.7/test/test_dis.py
+++ /dev/null
@@ -1,152 +0,0 @@
-# Minimal tests for dis module
-
-from test.test_support import run_unittest
-import unittest
-import sys
-import dis
-import StringIO
-
-
-def _f(a):
-    print a
-    return 1
-
-dis_f = """\
- %-4d         0 LOAD_FAST                0 (a)
-              3 PRINT_ITEM
-              4 PRINT_NEWLINE
-
- %-4d         5 LOAD_CONST               1 (1)
-              8 RETURN_VALUE
-"""%(_f.func_code.co_firstlineno + 1,
-     _f.func_code.co_firstlineno + 2)
-
-
-# we "call" rangexxx() instead of range() to disable the
-# pypy optimization that turns it into CALL_LIKELY_BUILTIN.
-def bug708901():
-    for res in rangexxx(1,
-                        10):
-        pass
-
-dis_bug708901 = """\
- %-4d         0 SETUP_LOOP              23 (to 26)
-              3 LOAD_GLOBAL              0 (rangexxx)
-              6 LOAD_CONST               1 (1)
-
- %-4d         9 LOAD_CONST               2 (10)
-             12 CALL_FUNCTION            2
-             15 GET_ITER
-        >>   16 FOR_ITER                 6 (to 25)
-             19 STORE_FAST               0 (res)
-
- %-4d        22 JUMP_ABSOLUTE           16
-        >>   25 POP_BLOCK
-        >>   26 LOAD_CONST               0 (None)
-             29 RETURN_VALUE
-"""%(bug708901.func_code.co_firstlineno + 1,
-     bug708901.func_code.co_firstlineno + 2,
-     bug708901.func_code.co_firstlineno + 3)
-
-
-def bug1333982(x=[]):
-    assert 0, ([s for s in x] +
-              1)
-    pass
-
-dis_bug1333982 = """\
- %-4d         0 LOAD_CONST               1 (0)
-              3 POP_JUMP_IF_TRUE        38
-              6 LOAD_GLOBAL              0 (AssertionError)
-              9 BUILD_LIST               0
-             12 LOAD_FAST                0 (x)
-             15 GET_ITER
-        >>   16 FOR_ITER                12 (to 31)
-             19 STORE_FAST               1 (s)
-             22 LOAD_FAST                1 (s)
-             25 LIST_APPEND              2
-             28 JUMP_ABSOLUTE           16
-
- %-4d   >>   31 LOAD_CONST               2 (1)
-             34 BINARY_ADD
-             35 RAISE_VARARGS            2
-
- %-4d   >>   38 LOAD_CONST               0 (None)
-             41 RETURN_VALUE
-"""%(bug1333982.func_code.co_firstlineno + 1,
-     bug1333982.func_code.co_firstlineno + 2,
-     bug1333982.func_code.co_firstlineno + 3)
-
-_BIG_LINENO_FORMAT = """\
-%3d           0 LOAD_GLOBAL              0 (spam)
-              3 POP_TOP
-              4 LOAD_CONST               0 (None)
-              7 RETURN_VALUE
-"""
-
-class DisTests(unittest.TestCase):
-    def do_disassembly_test(self, func, expected):
-        s = StringIO.StringIO()
-        save_stdout = sys.stdout
-        sys.stdout = s
-        dis.dis(func)
-        sys.stdout = save_stdout
-        got = s.getvalue()
-        # Trim trailing blanks (if any).
-        lines = got.split('\n')
-        lines = [line.rstrip() for line in lines]
-        expected = expected.split("\n")
-        import difflib
-        if expected != lines:
-            self.fail(
-                "events did not match expectation:\n" +
-                "\n".join(difflib.ndiff(expected,
-                                        lines)))
-
-    def test_opmap(self):
-        self.assertEqual(dis.opmap["STOP_CODE"], 0)
-        self.assertIn(dis.opmap["LOAD_CONST"], dis.hasconst)
-        self.assertIn(dis.opmap["STORE_NAME"], dis.hasname)
-
-    def test_opname(self):
-        self.assertEqual(dis.opname[dis.opmap["LOAD_FAST"]], "LOAD_FAST")
-
-    def test_boundaries(self):
-        self.assertEqual(dis.opmap["EXTENDED_ARG"], dis.EXTENDED_ARG)
-        self.assertEqual(dis.opmap["STORE_NAME"], dis.HAVE_ARGUMENT)
-
-    def test_dis(self):
-        self.do_disassembly_test(_f, dis_f)
-
-    def test_bug_708901(self):
-        self.do_disassembly_test(bug708901, dis_bug708901)
-
-    def test_bug_1333982(self):
-        # This one is checking bytecodes generated for an `assert` statement,
-        # so fails if the tests are run with -O.  Skip this test then.
-        if __debug__:
-            self.do_disassembly_test(bug1333982, dis_bug1333982)
-
-    def test_big_linenos(self):
-        def func(count):
-            namespace = {}
-            func = "def foo():\n " + "".join(["\n "] * count + ["spam\n"])
-            exec func in namespace
-            return namespace['foo']
-
-        # Test all small ranges
-        for i in xrange(1, 300):
-            expected = _BIG_LINENO_FORMAT % (i + 2)
-            self.do_disassembly_test(func(i), expected)
-
-        # Test some larger ranges too
-        for i in xrange(300, 5000, 10):
-            expected = _BIG_LINENO_FORMAT % (i + 2)
-            self.do_disassembly_test(func(i), expected)
-
-def test_main():
-    run_unittest(DisTests)
-
-
-if __name__ == "__main__":
-    test_main()
diff --git a/lib-python/modified-2.7/test/test_extcall.py b/lib-python/modified-2.7/test/test_extcall.py
--- a/lib-python/modified-2.7/test/test_extcall.py
+++ b/lib-python/modified-2.7/test/test_extcall.py
@@ -299,7 +299,7 @@
         def f(a):
             return a
         self.assertEqual(f(**{u'a': 4}), 4)
-        self.assertRaises(TypeError, lambda: f(**{u'stören': 4}))
+        self.assertRaises(TypeError, f, **{u'stören': 4})
         self.assertRaises(TypeError, f, **{u'someLongString':2})
         try:
             f(a=4, **{u'a': 4})
diff --git a/lib-python/modified-2.7/test/test_fcntl.py b/lib-python/modified-2.7/test/test_fcntl.py
new file mode 100644
--- /dev/null
+++ b/lib-python/modified-2.7/test/test_fcntl.py
@@ -0,0 +1,108 @@
+"""Test program for the fcntl C module.
+
+OS/2+EMX doesn't support the file locking operations.
+
+"""
+import os
+import struct
+import sys
+import unittest
+from test.test_support import (verbose, TESTFN, unlink, run_unittest,
+    import_module)
+
+# Skip test if no fnctl module.
+fcntl = import_module('fcntl')
+
+
+# TODO - Write tests for flock() and lockf().
+
+def get_lockdata():
+    if sys.platform.startswith('atheos'):
+        start_len = "qq"
+    else:
+        try:
+            os.O_LARGEFILE
+        except AttributeError:
+            start_len = "ll"
+        else:
+            start_len = "qq"
+
+    if sys.platform in ('netbsd1', 'netbsd2', 'netbsd3',
+                        'Darwin1.2', 'darwin',
+                        'freebsd2', 'freebsd3', 'freebsd4', 'freebsd5',
+                        'freebsd6', 'freebsd7', 'freebsd8',
+                        'bsdos2', 'bsdos3', 'bsdos4',
+                        'openbsd', 'openbsd2', 'openbsd3', 'openbsd4', 'openbsd5'):
+        if struct.calcsize('l') == 8:
+            off_t = 'l'
+            pid_t = 'i'
+        else:
+            off_t = 'lxxxx'
+            pid_t = 'l'
+        lockdata = struct.pack(off_t + off_t + pid_t + 'hh', 0, 0, 0,
+                               fcntl.F_WRLCK, 0)
+    elif sys.platform in ['aix3', 'aix4', 'hp-uxB', 'unixware7']:
+        lockdata = struct.pack('hhlllii', fcntl.F_WRLCK, 0, 0, 0, 0, 0, 0)
+    elif sys.platform in ['os2emx']:
+        lockdata = None
+    else:
+        lockdata = struct.pack('hh'+start_len+'hh', fcntl.F_WRLCK, 0, 0, 0, 0, 0)
+    if lockdata:
+        if verbose:
+            print 'struct.pack: ', repr(lockdata)
+    return lockdata
+
+lockdata = get_lockdata()
+
+
+class TestFcntl(unittest.TestCase):
+
+    def setUp(self):
+        self.f = None
+
+    def tearDown(self):
+        if self.f and not self.f.closed:
+            self.f.close()
+        unlink(TESTFN)
+
+    def test_fcntl_fileno(self):
+        # the example from the library docs
+        self.f = open(TESTFN, 'w')
+        rv = fcntl.fcntl(self.f.fileno(), fcntl.F_SETFL, os.O_NONBLOCK)
+        if verbose:
+            print 'Status from fcntl with O_NONBLOCK: ', rv
+        if sys.platform not in ['os2emx']:
+            rv = fcntl.fcntl(self.f.fileno(), fcntl.F_SETLKW, lockdata)
+            if verbose:
+                print 'String from fcntl with F_SETLKW: ', repr(rv)
+        self.f.close()
+
+    def test_fcntl_file_descriptor(self):
+        # again, but pass the file rather than numeric descriptor
+        self.f = open(TESTFN, 'w')
+        rv = fcntl.fcntl(self.f, fcntl.F_SETFL, os.O_NONBLOCK)
+        if sys.platform not in ['os2emx']:
+            rv = fcntl.fcntl(self.f, fcntl.F_SETLKW, lockdata)
+        self.f.close()
+
+    def test_fcntl_64_bit(self):
+        # Issue #1309352: fcntl shouldn't fail when the third arg fits in a
+        # C 'long' but not in a C 'int'.
+        try:
+            cmd = fcntl.F_NOTIFY
+            # This flag is larger than 2**31 in 64-bit builds
+            flags = fcntl.DN_MULTISHOT
+        except AttributeError:
+            self.skipTest("F_NOTIFY or DN_MULTISHOT unavailable")
+        fd = os.open(os.path.dirname(os.path.abspath(TESTFN)), os.O_RDONLY)
+        try:
+            fcntl.fcntl(fd, cmd, flags)
+        finally:
+            os.close(fd)
+
+
+def test_main():
+    run_unittest(TestFcntl)
+
+if __name__ == '__main__':
+    test_main()
diff --git a/lib-python/modified-2.7/test/test_heapq.py b/lib-python/modified-2.7/test/test_heapq.py
--- a/lib-python/modified-2.7/test/test_heapq.py
+++ b/lib-python/modified-2.7/test/test_heapq.py
@@ -186,6 +186,11 @@
         self.assertFalse(sys.modules['heapq'] is self.module)
         self.assertTrue(hasattr(self.module.heapify, 'func_code'))
 
+    def test_islice_protection(self):
+        m = self.module
+        self.assertFalse(m.nsmallest(-1, [1]))
+        self.assertFalse(m.nlargest(-1, [1]))
+
 
 class TestHeapC(TestHeap):
     module = c_heapq
diff --git a/lib-python/modified-2.7/test/test_import.py b/lib-python/modified-2.7/test/test_import.py
--- a/lib-python/modified-2.7/test/test_import.py
+++ b/lib-python/modified-2.7/test/test_import.py
@@ -64,6 +64,7 @@
             except ImportError, err:
                 self.fail("import from %s failed: %s" % (ext, err))
             else:
+                # XXX importing .pyw is missing on Windows
                 self.assertEqual(mod.a, a,
                     "module loaded (%s) but contents invalid" % mod)
                 self.assertEqual(mod.b, b,
diff --git a/lib-python/2.7/test/test_multibytecodec.py b/lib-python/modified-2.7/test/test_multibytecodec.py
copy from lib-python/2.7/test/test_multibytecodec.py
copy to lib-python/modified-2.7/test/test_multibytecodec.py
--- a/lib-python/2.7/test/test_multibytecodec.py
+++ b/lib-python/modified-2.7/test/test_multibytecodec.py
@@ -42,7 +42,7 @@
         dec = codecs.getdecoder('euc-kr')
         myreplace  = lambda exc: (u'', sys.maxint+1)
         codecs.register_error('test.cjktest', myreplace)
-        self.assertRaises(IndexError, dec,
+        self.assertRaises((IndexError, OverflowError), dec,
                           'apple\x92ham\x93spam', 'test.cjktest')
 
     def test_codingspec(self):
@@ -148,7 +148,8 @@
 class Test_StreamReader(unittest.TestCase):
     def test_bug1728403(self):
         try:
-            open(TESTFN, 'w').write('\xa1')
+            with open(TESTFN, 'w') as f:
+                f.write('\xa1')
             f = codecs.open(TESTFN, encoding='cp949')
             self.assertRaises(UnicodeDecodeError, f.read, 2)
         finally:
diff --git a/lib-python/2.7/test/test_multibytecodec_support.py b/lib-python/modified-2.7/test/test_multibytecodec_support.py
copy from lib-python/2.7/test/test_multibytecodec_support.py
copy to lib-python/modified-2.7/test/test_multibytecodec_support.py
--- a/lib-python/2.7/test/test_multibytecodec_support.py
+++ b/lib-python/modified-2.7/test/test_multibytecodec_support.py
@@ -107,8 +107,8 @@
         def myreplace(exc):
             return (u'x', sys.maxint + 1)
         codecs.register_error("test.cjktest", myreplace)
-        self.assertRaises(IndexError, self.encode, self.unmappedunicode,
-                          'test.cjktest')
+        self.assertRaises((IndexError, OverflowError), self.encode,
+                          self.unmappedunicode, 'test.cjktest')
 
     def test_callback_None_index(self):
         def myreplace(exc):
diff --git a/lib-python/modified-2.7/test/test_multiprocessing.py b/lib-python/modified-2.7/test/test_multiprocessing.py
--- a/lib-python/modified-2.7/test/test_multiprocessing.py
+++ b/lib-python/modified-2.7/test/test_multiprocessing.py
@@ -510,7 +510,6 @@
 
         p.join()
 
-    @unittest.skipIf(os.name == 'posix', "PYPY: FIXME")
     def test_qsize(self):
         q = self.Queue()
         try:
@@ -532,7 +531,6 @@
             time.sleep(DELTA)
             q.task_done()
 
-    @unittest.skipIf(os.name == 'posix', "PYPY: FIXME")
     def test_task_done(self):
         queue = self.JoinableQueue()
 
@@ -1091,7 +1089,6 @@
 class _TestPoolWorkerLifetime(BaseTestCase):
 
     ALLOWED_TYPES = ('processes', )
-    @unittest.skipIf(os.name == 'posix', "PYPY: FIXME")
     def test_pool_worker_lifetime(self):
         p = multiprocessing.Pool(3, maxtasksperchild=10)
         self.assertEqual(3, len(p._pool))
@@ -1280,7 +1277,6 @@
         queue = manager.get_queue()
         queue.put('hello world')
 
-    @unittest.skipIf(os.name == 'posix', "PYPY: FIXME")
     def test_rapid_restart(self):
         authkey = os.urandom(32)
         manager = QueueManager(
@@ -1297,6 +1293,7 @@
         queue = manager.get_queue()
         self.assertEqual(queue.get(), 'hello world')
         del queue
+        test_support.gc_collect()
         manager.shutdown()
         manager = QueueManager(
             address=addr, authkey=authkey, serializer=SERIALIZER)
@@ -1573,7 +1570,6 @@
 
     ALLOWED_TYPES = ('processes',)
 
-    @unittest.skipIf(os.name == 'posix', "PYPY: FIXME")
     def test_heap(self):
         iterations = 5000
         maxblocks = 50
diff --git a/lib-python/modified-2.7/test/test_repr.py b/lib-python/modified-2.7/test/test_repr.py
--- a/lib-python/modified-2.7/test/test_repr.py
+++ b/lib-python/modified-2.7/test/test_repr.py
@@ -254,8 +254,14 @@
         eq = self.assertEqual
         touch(os.path.join(self.subpkgname, self.pkgname + os.extsep + 'py'))
         from areallylongpackageandmodulenametotestreprtruncation.areallylongpackageandmodulenametotestreprtruncation import areallylongpackageandmodulenametotestreprtruncation
-        eq(repr(areallylongpackageandmodulenametotestreprtruncation),
-           "" % (areallylongpackageandmodulenametotestreprtruncation.__name__, areallylongpackageandmodulenametotestreprtruncation.__file__))
+        # On PyPy, we use %r to format the file name; on CPython it is done
+        # with '%s'.  It seems to me that %r is safer .
+        if '__pypy__' in sys.builtin_module_names:
+            eq(repr(areallylongpackageandmodulenametotestreprtruncation),
+               "" % (areallylongpackageandmodulenametotestreprtruncation.__name__, areallylongpackageandmodulenametotestreprtruncation.__file__))
+        else:
+            eq(repr(areallylongpackageandmodulenametotestreprtruncation),
+               "" % (areallylongpackageandmodulenametotestreprtruncation.__name__, areallylongpackageandmodulenametotestreprtruncation.__file__))
         eq(repr(sys), "")
 
     def test_type(self):
diff --git a/lib-python/2.7/test/test_sets.py b/lib-python/modified-2.7/test/test_sets.py
copy from lib-python/2.7/test/test_sets.py
copy to lib-python/modified-2.7/test/test_sets.py
--- a/lib-python/2.7/test/test_sets.py
+++ b/lib-python/modified-2.7/test/test_sets.py
@@ -686,7 +686,9 @@
         set_list = sorted(self.set)
         self.assertEqual(len(dup_list), len(set_list))
         for i, el in enumerate(dup_list):
-            self.assertIs(el, set_list[i])
+            # Object identity is not guarnteed for immutable objects, so we
+            # can't use assertIs here.
+            self.assertEqual(el, set_list[i])
 
     def test_deep_copy(self):
         dup = copy.deepcopy(self.set)
diff --git a/lib-python/modified-2.7/test/test_ssl.py b/lib-python/modified-2.7/test/test_ssl.py
--- a/lib-python/modified-2.7/test/test_ssl.py
+++ b/lib-python/modified-2.7/test/test_ssl.py
@@ -58,32 +58,35 @@
 
 # Issue #9415: Ubuntu hijacks their OpenSSL and forcefully disables SSLv2
 def skip_if_broken_ubuntu_ssl(func):
-    # We need to access the lower-level wrapper in order to create an
-    # implicit SSL context without trying to connect or listen.
-    try:
-        import _ssl
-    except ImportError:
-        # The returned function won't get executed, just ignore the error
-        pass
-    @functools.wraps(func)
-    def f(*args, **kwargs):
+    if hasattr(ssl, 'PROTOCOL_SSLv2'):
+        # We need to access the lower-level wrapper in order to create an
+        # implicit SSL context without trying to connect or listen.
         try:
-            s = socket.socket(socket.AF_INET)
-            _ssl.sslwrap(s._sock, 0, None, None,
-                         ssl.CERT_NONE, ssl.PROTOCOL_SSLv2, None, None)
-        except ssl.SSLError as e:
-            if (ssl.OPENSSL_VERSION_INFO == (0, 9, 8, 15, 15) and
-                platform.linux_distribution() == ('debian', 'squeeze/sid', '')
-                and 'Invalid SSL protocol variant specified' in str(e)):
-                raise unittest.SkipTest("Patched Ubuntu OpenSSL breaks behaviour")
-        return func(*args, **kwargs)
-    return f
+            import _ssl
+        except ImportError:
+            # The returned function won't get executed, just ignore the error
+            pass
+        @functools.wraps(func)
+        def f(*args, **kwargs):
+            try:
+                s = socket.socket(socket.AF_INET)
+                _ssl.sslwrap(s._sock, 0, None, None,
+                             ssl.CERT_NONE, ssl.PROTOCOL_SSLv2, None, None)
+            except ssl.SSLError as e:
+                if (ssl.OPENSSL_VERSION_INFO == (0, 9, 8, 15, 15) and
+                    platform.linux_distribution() == ('debian', 'squeeze/sid', '')
+                    and 'Invalid SSL protocol variant specified' in str(e)):
+                    raise unittest.SkipTest("Patched Ubuntu OpenSSL breaks behaviour")
+            return func(*args, **kwargs)
+        return f
+    else:
+        return func
 
 
 class BasicSocketTests(unittest.TestCase):
 
     def test_constants(self):
-        ssl.PROTOCOL_SSLv2
+        #ssl.PROTOCOL_SSLv2
         ssl.PROTOCOL_SSLv23
         ssl.PROTOCOL_SSLv3
         ssl.PROTOCOL_TLSv1
@@ -105,7 +108,6 @@
             print "didn't raise TypeError"
         ssl.RAND_add("this is a random string", 75.0)
 
-    @test_support.impl_detail("obscure test")
     def test_parse_cert(self):
         # note that this uses an 'unofficial' function in _ssl.c,
         # provided solely for this test, to exercise the certificate
@@ -840,6 +842,8 @@
                 c = socket.socket()
                 c.connect((HOST, port))
                 listener_gone.wait()
+                # XXX why is it necessary?
+                test_support.gc_collect()
                 try:
                     ssl_sock = ssl.wrap_socket(c)
                 except IOError:
@@ -965,7 +969,8 @@
             try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, True)
             try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, True, ssl.CERT_OPTIONAL)
             try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, True, ssl.CERT_REQUIRED)
-            try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv2, False)
+            if hasattr(ssl, 'PROTOCOL_SSLv2'):
+                try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv2, False)
             try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv23, False)
             try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_TLSv1, False)
 
@@ -977,7 +982,8 @@
             try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, True)
             try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, True, ssl.CERT_OPTIONAL)
             try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, True, ssl.CERT_REQUIRED)
-            try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv2, False)
+            if hasattr(ssl, 'PROTOCOL_SSLv2'):
+                try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv2, False)
             try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv3, False)
             try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv23, False)
 
diff --git a/lib-python/2.7/test/test_subprocess.py b/lib-python/modified-2.7/test/test_subprocess.py
copy from lib-python/2.7/test/test_subprocess.py
copy to lib-python/modified-2.7/test/test_subprocess.py
--- a/lib-python/2.7/test/test_subprocess.py
+++ b/lib-python/modified-2.7/test/test_subprocess.py
@@ -16,11 +16,11 @@
 # Depends on the following external programs: Python
 #
 
-if mswindows:
-    SETBINARY = ('import msvcrt; msvcrt.setmode(sys.stdout.fileno(), '
-                                                'os.O_BINARY);')
-else:
-    SETBINARY = ''
+#if mswindows:
+#    SETBINARY = ('import msvcrt; msvcrt.setmode(sys.stdout.fileno(), '
+#                                                'os.O_BINARY);')
+#else:
+#    SETBINARY = ''
 
 
 try:
@@ -420,8 +420,9 @@
         self.assertStderrEqual(stderr, "")
 
     def test_universal_newlines(self):
-        p = subprocess.Popen([sys.executable, "-c",
-                          'import sys,os;' + SETBINARY +
+        # NB. replaced SETBINARY with the -u flag
+        p = subprocess.Popen([sys.executable, "-u", "-c",
+                          'import sys,os;' + #SETBINARY +
                           'sys.stdout.write("line1\\n");'
                           'sys.stdout.flush();'
                           'sys.stdout.write("line2\\r");'
@@ -448,8 +449,9 @@
 
     def test_universal_newlines_communicate(self):
         # universal newlines through communicate()
-        p = subprocess.Popen([sys.executable, "-c",
-                          'import sys,os;' + SETBINARY +
+        # NB. replaced SETBINARY with the -u flag
+        p = subprocess.Popen([sys.executable, "-u", "-c",
+                          'import sys,os;' + #SETBINARY +
                           'sys.stdout.write("line1\\n");'
                           'sys.stdout.flush();'
                           'sys.stdout.write("line2\\r");'
diff --git a/lib-python/modified-2.7/test/test_support.py b/lib-python/modified-2.7/test/test_support.py
--- a/lib-python/modified-2.7/test/test_support.py
+++ b/lib-python/modified-2.7/test/test_support.py
@@ -1066,7 +1066,7 @@
         if '--pdb' in sys.argv:
             import pdb, traceback
             traceback.print_tb(exc_info[2])
-            pdb.post_mortem(exc_info[2], pdb.Pdb)
+            pdb.post_mortem(exc_info[2])
 
 # ----------------------------------
 
diff --git a/lib-python/modified-2.7/test/test_sys_settrace.py b/lib-python/modified-2.7/test/test_sys_settrace.py
--- a/lib-python/modified-2.7/test/test_sys_settrace.py
+++ b/lib-python/modified-2.7/test/test_sys_settrace.py
@@ -286,11 +286,11 @@
         self.compare_events(func.func_code.co_firstlineno,
                             tracer.events, func.events)
 
-    def set_and_retrieve_none(self):
+    def test_set_and_retrieve_none(self):
         sys.settrace(None)
         assert sys.gettrace() is None
 
-    def set_and_retrieve_func(self):
+    def test_set_and_retrieve_func(self):
         def fn(*args):
             pass
 
diff --git a/lib-python/2.7/test/test_tarfile.py b/lib-python/modified-2.7/test/test_tarfile.py
copy from lib-python/2.7/test/test_tarfile.py
copy to lib-python/modified-2.7/test/test_tarfile.py
--- a/lib-python/2.7/test/test_tarfile.py
+++ b/lib-python/modified-2.7/test/test_tarfile.py
@@ -169,6 +169,7 @@
         except tarfile.ReadError:
             self.fail("tarfile.open() failed on empty archive")
         self.assertListEqual(tar.getmembers(), [])
+        tar.close()
 
     def test_null_tarfile(self):
         # Test for issue6123: Allow opening empty archives.
@@ -207,16 +208,21 @@
         fobj = open(self.tarname, "rb")
         tar = tarfile.open(fileobj=fobj, mode=self.mode)
         self.assertEqual(tar.name, os.path.abspath(fobj.name))
+        tar.close()
 
     def test_no_name_attribute(self):
-        data = open(self.tarname, "rb").read()
+        f = open(self.tarname, "rb")
+        data = f.read()
+        f.close()
         fobj = StringIO.StringIO(data)
         self.assertRaises(AttributeError, getattr, fobj, "name")
         tar = tarfile.open(fileobj=fobj, mode=self.mode)
         self.assertEqual(tar.name, None)
 
     def test_empty_name_attribute(self):
-        data = open(self.tarname, "rb").read()
+        f = open(self.tarname, "rb")
+        data = f.read()
+        f.close()
         fobj = StringIO.StringIO(data)
         fobj.name = ""
         tar = tarfile.open(fileobj=fobj, mode=self.mode)
@@ -515,6 +521,7 @@
         self.tar = tarfile.open(self.tarname, mode=self.mode, encoding="iso8859-1")
         tarinfo = self.tar.getmember("pax/umlauts-�������")
         self._test_member(tarinfo, size=7011, chksum=md5_regtype)
+        self.tar.close()
 
 
 class LongnameTest(ReadTest):
@@ -675,6 +682,7 @@
             tar = tarfile.open(tmpname, self.mode)
             tarinfo = tar.gettarinfo(path)
             self.assertEqual(tarinfo.size, 0)
+            tar.close()
         finally:
             os.rmdir(path)
 
@@ -692,6 +700,7 @@
                 tar.gettarinfo(target)
                 tarinfo = tar.gettarinfo(link)
                 self.assertEqual(tarinfo.size, 0)
+                tar.close()
             finally:
                 os.remove(target)
                 os.remove(link)
@@ -704,6 +713,7 @@
                 tar = tarfile.open(tmpname, self.mode)
                 tarinfo = tar.gettarinfo(path)
                 self.assertEqual(tarinfo.size, 0)
+                tar.close()
             finally:
                 os.remove(path)
 
@@ -722,6 +732,7 @@
         tar.add(dstname)
         os.chdir(cwd)
         self.assertTrue(tar.getnames() == [], "added the archive to itself")
+        tar.close()
 
     def test_exclude(self):
         tempdir = os.path.join(TEMPDIR, "exclude")
@@ -742,6 +753,7 @@
             tar = tarfile.open(tmpname, "r")
             self.assertEqual(len(tar.getmembers()), 1)
             self.assertEqual(tar.getnames()[0], "empty_dir")
+            tar.close()
         finally:
             shutil.rmtree(tempdir)
 
@@ -859,7 +871,9 @@
             fobj.close()
         elif self.mode.endswith("bz2"):
             dec = bz2.BZ2Decompressor()
-            data = open(tmpname, "rb").read()
+            f = open(tmpname, "rb")
+            data = f.read()
+            f.close()
             data = dec.decompress(data)
             self.assertTrue(len(dec.unused_data) == 0,
                     "found trailing data")
@@ -938,6 +952,7 @@
                 "unable to read longname member")
         self.assertEqual(tarinfo.linkname, member.linkname,
                 "unable to read longname member")
+        tar.close()
 
     def test_longname_1023(self):
         self._test(("longnam/" * 127) + "longnam")
@@ -1030,6 +1045,7 @@
         else:
             n = tar.getmembers()[0].name
             self.assertTrue(name == n, "PAX longname creation failed")
+        tar.close()
 
     def test_pax_global_header(self):
         pax_headers = {
@@ -1058,6 +1074,7 @@
                     tarfile.PAX_NUMBER_FIELDS[key](val)
                 except (TypeError, ValueError):
                     self.fail("unable to convert pax header field")
+        tar.close()
 
     def test_pax_extended_header(self):
         # The fields from the pax header have priority over the
@@ -1077,6 +1094,7 @@
         self.assertEqual(t.pax_headers, pax_headers)
         self.assertEqual(t.name, "foo")
         self.assertEqual(t.uid, 123)
+        tar.close()
 
 
 class UstarUnicodeTest(unittest.TestCase):
@@ -1120,6 +1138,7 @@
         tarinfo.name = "foo"
         tarinfo.uname = u"���"
         self.assertRaises(UnicodeError, tar.addfile, tarinfo)
+        tar.close()
 
     def test_unicode_argument(self):
         tar = tarfile.open(tarname, "r", encoding="iso8859-1", errors="strict")
@@ -1174,6 +1193,7 @@
             tar = tarfile.open(tmpname, format=self.format, encoding="ascii",
                     errors=handler)
             self.assertEqual(tar.getnames()[0], name)
+            tar.close()
 
         self.assertRaises(UnicodeError, tarfile.open, tmpname,
                 encoding="ascii", errors="strict")
@@ -1186,6 +1206,7 @@
         tar = tarfile.open(tmpname, format=self.format, encoding="iso8859-1",
                 errors="utf-8")
         self.assertEqual(tar.getnames()[0], "���/" + u"�".encode("utf8"))
+        tar.close()
 
 
 class AppendTest(unittest.TestCase):
@@ -1213,6 +1234,7 @@
     def _test(self, names=["bar"], fileobj=None):
         tar = tarfile.open(self.tarname, fileobj=fileobj)
         self.assertEqual(tar.getnames(), names)
+        tar.close()
 
     def test_non_existing(self):
         self._add_testfile()
@@ -1231,7 +1253,9 @@
 
     def test_fileobj(self):
         self._create_testtar()
-        data = open(self.tarname).read()
+        f = open(self.tarname)
+        data = f.read()
+        f.close()
         fobj = StringIO.StringIO(data)
         self._add_testfile(fobj)
         fobj.seek(0)
@@ -1257,7 +1281,9 @@
     # Append mode is supposed to fail if the tarfile to append to
     # does not end with a zero block.
     def _test_error(self, data):
-        open(self.tarname, "wb").write(data)
+        f = open(self.tarname, "wb")
+        f.write(data)
+        f.close()
         self.assertRaises(tarfile.ReadError, self._add_testfile)
 
     def test_null(self):
diff --git a/lib-python/modified-2.7/test/test_tempfile.py b/lib-python/modified-2.7/test/test_tempfile.py
--- a/lib-python/modified-2.7/test/test_tempfile.py
+++ b/lib-python/modified-2.7/test/test_tempfile.py
@@ -23,8 +23,8 @@
 
 # TEST_FILES may need to be tweaked for systems depending on the maximum
 # number of files that can be opened at one time (see ulimit -n)
-if sys.platform in ('openbsd3', 'openbsd4'):
-    TEST_FILES = 48
+if sys.platform.startswith("openbsd"):
+    TEST_FILES = 64 # ulimit -n defaults to 128 for normal users
 else:
     TEST_FILES = 100
 
diff --git a/lib-python/modified-2.7/test/test_urllib2.py b/lib-python/modified-2.7/test/test_urllib2.py
--- a/lib-python/modified-2.7/test/test_urllib2.py
+++ b/lib-python/modified-2.7/test/test_urllib2.py
@@ -307,6 +307,9 @@
     def getresponse(self):
         return MockHTTPResponse(MockFile(), {}, 200, "OK")
 
+    def close(self):
+        pass
+
 class MockHandler:
     # useful for testing handler machinery
     # see add_ordered_mock_handlers() docstring
diff --git a/lib-python/modified-2.7/test/test_weakref.py b/lib-python/modified-2.7/test/test_weakref.py
--- a/lib-python/modified-2.7/test/test_weakref.py
+++ b/lib-python/modified-2.7/test/test_weakref.py
@@ -993,13 +993,13 @@
         self.assertTrue(len(weakdict) == 2)
         k, v = weakdict.popitem()
         self.assertTrue(len(weakdict) == 1)
-        if k is key1:
+        if k == key1:
             self.assertTrue(v is value1)
         else:
             self.assertTrue(v is value2)
         k, v = weakdict.popitem()
         self.assertTrue(len(weakdict) == 0)
-        if k is key1:
+        if k == key1:
             self.assertTrue(v is value1)
         else:
             self.assertTrue(v is value2)
diff --git a/lib-python/modified-2.7/urllib2.py b/lib-python/modified-2.7/urllib2.py
new file mode 100644
--- /dev/null
+++ b/lib-python/modified-2.7/urllib2.py
@@ -0,0 +1,1436 @@
+"""An extensible library for opening URLs using a variety of protocols
+
+The simplest way to use this module is to call the urlopen function,
+which accepts a string containing a URL or a Request object (described
+below).  It opens the URL and returns the results as file-like
+object; the returned object has some extra methods described below.
+
+The OpenerDirector manages a collection of Handler objects that do
+all the actual work.  Each Handler implements a particular protocol or
+option.  The OpenerDirector is a composite object that invokes the
+Handlers needed to open the requested URL.  For example, the
+HTTPHandler performs HTTP GET and POST requests and deals with
+non-error returns.  The HTTPRedirectHandler automatically deals with
+HTTP 301, 302, 303 and 307 redirect errors, and the HTTPDigestAuthHandler
+deals with digest authentication.
+
+urlopen(url, data=None) -- Basic usage is the same as original
+urllib.  pass the url and optionally data to post to an HTTP URL, and
+get a file-like object back.  One difference is that you can also pass
+a Request instance instead of URL.  Raises a URLError (subclass of
+IOError); for HTTP errors, raises an HTTPError, which can also be
+treated as a valid response.
+
+build_opener -- Function that creates a new OpenerDirector instance.
+Will install the default handlers.  Accepts one or more Handlers as
+arguments, either instances or Handler classes that it will
+instantiate.  If one of the argument is a subclass of the default
+handler, the argument will be installed instead of the default.
+
+install_opener -- Installs a new opener as the default opener.
+
+objects of interest:
+
+OpenerDirector -- Sets up the User Agent as the Python-urllib client and manages
+the Handler classes, while dealing with requests and responses.
+
+Request -- An object that encapsulates the state of a request.  The
+state can be as simple as the URL.  It can also include extra HTTP
+headers, e.g. a User-Agent.
+
+BaseHandler --
+
+exceptions:
+URLError -- A subclass of IOError, individual protocols have their own
+specific subclass.
+
+HTTPError -- Also a valid HTTP response, so you can treat an HTTP error
+as an exceptional event or valid response.
+
+internals:
+BaseHandler and parent
+_call_chain conventions
+
+Example usage:
+
+import urllib2
+
+# set up authentication info
+authinfo = urllib2.HTTPBasicAuthHandler()
+authinfo.add_password(realm='PDQ Application',
+                      uri='https://mahler:8092/site-updates.py',
+                      user='klem',
+                      passwd='geheim$parole')
+
+proxy_support = urllib2.ProxyHandler({"http" : "http://ahad-haam:3128"})
+
+# build a new opener that adds authentication and caching FTP handlers
+opener = urllib2.build_opener(proxy_support, authinfo, urllib2.CacheFTPHandler)
+
+# install it
+urllib2.install_opener(opener)
+
+f = urllib2.urlopen('http://www.python.org/')
+
+
+"""
+
+# XXX issues:
+# If an authentication error handler that tries to perform
+# authentication for some reason but fails, how should the error be
+# signalled?  The client needs to know the HTTP error code.  But if
+# the handler knows that the problem was, e.g., that it didn't know
+# that hash algo that requested in the challenge, it would be good to
+# pass that information along to the client, too.
+# ftp errors aren't handled cleanly
+# check digest against correct (i.e. non-apache) implementation
+
+# Possible extensions:
+# complex proxies  XXX not sure what exactly was meant by this
+# abstract factory for opener
+
+import base64
+import hashlib
+import httplib
+import mimetools
+import os
+import posixpath
+import random
+import re
+import socket
+import sys
+import time
+import urlparse
+import bisect
+
+try:
+    from cStringIO import StringIO
+except ImportError:
+    from StringIO import StringIO
+
+from urllib import (unwrap, unquote, splittype, splithost, quote,
+     addinfourl, splitport, splittag,
+     splitattr, ftpwrapper, splituser, splitpasswd, splitvalue)
+
+# support for FileHandler, proxies via environment variables
+from urllib import localhost, url2pathname, getproxies, proxy_bypass
+
+# used in User-Agent header sent
+__version__ = sys.version[:3]
+
+_opener = None
+def urlopen(url, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
+    global _opener
+    if _opener is None:
+        _opener = build_opener()
+    return _opener.open(url, data, timeout)
+
+def install_opener(opener):
+    global _opener
+    _opener = opener
+
+# do these error classes make sense?
+# make sure all of the IOError stuff is overridden.  we just want to be
+# subtypes.
+
+class URLError(IOError):
+    # URLError is a sub-type of IOError, but it doesn't share any of
+    # the implementation.  need to override __init__ and __str__.
+    # It sets self.args for compatibility with other EnvironmentError
+    # subclasses, but args doesn't have the typical format with errno in
+    # slot 0 and strerror in slot 1.  This may be better than nothing.
+    def __init__(self, reason):
+        self.args = reason,
+        self.reason = reason
+
+    def __str__(self):
+        return '' % self.reason
+
+class HTTPError(URLError, addinfourl):
+    """Raised when HTTP error occurs, but also acts like non-error return"""
+    __super_init = addinfourl.__init__
+
+    def __init__(self, url, code, msg, hdrs, fp):
+        self.code = code
+        self.msg = msg
+        self.hdrs = hdrs
+        self.fp = fp
+        self.filename = url
+        # The addinfourl classes depend on fp being a valid file
+        # object.  In some cases, the HTTPError may not have a valid
+        # file object.  If this happens, the simplest workaround is to
+        # not initialize the base classes.
+        if fp is not None:
+            self.__super_init(fp, hdrs, url, code)
+
+    def __str__(self):
+        return 'HTTP Error %s: %s' % (self.code, self.msg)
+
+# copied from cookielib.py
+_cut_port_re = re.compile(r":\d+$")
+def request_host(request):
+    """Return request-host, as defined by RFC 2965.
+
+    Variation from RFC: returned value is lowercased, for convenient
+    comparison.
+
+    """
+    url = request.get_full_url()
+    host = urlparse.urlparse(url)[1]
+    if host == "":
+        host = request.get_header("Host", "")
+
+    # remove port, if present
+    host = _cut_port_re.sub("", host, 1)
+    return host.lower()
+
+class Request:
+
+    def __init__(self, url, data=None, headers={},
+                 origin_req_host=None, unverifiable=False):
+        # unwrap('') --> 'type://host/path'
+        self.__original = unwrap(url)
+        self.__original, fragment = splittag(self.__original)
+        self.type = None
+        # self.__r_type is what's left after doing the splittype
+        self.host = None
+        self.port = None
+        self._tunnel_host = None
+        self.data = data
+        self.headers = {}
+        for key, value in headers.items():
+            self.add_header(key, value)
+        self.unredirected_hdrs = {}
+        if origin_req_host is None:
+            origin_req_host = request_host(self)
+        self.origin_req_host = origin_req_host
+        self.unverifiable = unverifiable
+
+    def __getattr__(self, attr):
+        # XXX this is a fallback mechanism to guard against these
+        # methods getting called in a non-standard order.  this may be
+        # too complicated and/or unnecessary.
+        # XXX should the __r_XXX attributes be public?
+        if attr[:12] == '_Request__r_':
+            name = attr[12:]
+            if hasattr(Request, 'get_' + name):
+                getattr(self, 'get_' + name)()
+                return getattr(self, attr)
+        raise AttributeError, attr
+
+    def get_method(self):
+        if self.has_data():
+            return "POST"
+        else:
+            return "GET"
+
+    # XXX these helper methods are lame
+
+    def add_data(self, data):
+        self.data = data
+
+    def has_data(self):
+        return self.data is not None
+
+    def get_data(self):
+        return self.data
+
+    def get_full_url(self):
+        return self.__original
+
+    def get_type(self):
+        if self.type is None:
+            self.type, self.__r_type = splittype(self.__original)
+            if self.type is None:
+                raise ValueError, "unknown url type: %s" % self.__original
+        return self.type
+
+    def get_host(self):
+        if self.host is None:
+            self.host, self.__r_host = splithost(self.__r_type)
+            if self.host:
+                self.host = unquote(self.host)
+        return self.host
+
+    def get_selector(self):
+        return self.__r_host
+
+    def set_proxy(self, host, type):
+        if self.type == 'https' and not self._tunnel_host:
+            self._tunnel_host = self.host
+        else:
+            self.type = type
+            self.__r_host = self.__original
+
+        self.host = host
+
+    def has_proxy(self):
+        return self.__r_host == self.__original
+
+    def get_origin_req_host(self):
+        return self.origin_req_host
+
+    def is_unverifiable(self):
+        return self.unverifiable
+
+    def add_header(self, key, val):
+        # useful for something like authentication
+        self.headers[key.capitalize()] = val
+
+    def add_unredirected_header(self, key, val):
+        # will not be added to a redirected request
+        self.unredirected_hdrs[key.capitalize()] = val
+
+    def has_header(self, header_name):
+        return (header_name in self.headers or
+                header_name in self.unredirected_hdrs)
+
+    def get_header(self, header_name, default=None):
+        return self.headers.get(
+            header_name,
+            self.unredirected_hdrs.get(header_name, default))
+
+    def header_items(self):
+        hdrs = self.unredirected_hdrs.copy()
+        hdrs.update(self.headers)
+        return hdrs.items()
+
+class OpenerDirector:
+    def __init__(self):
+        client_version = "Python-urllib/%s" % __version__
+        self.addheaders = [('User-agent', client_version)]
+        # manage the individual handlers
+        self.handlers = []
+        self.handle_open = {}
+        self.handle_error = {}
+        self.process_response = {}
+        self.process_request = {}
+
+    def add_handler(self, handler):
+        if not hasattr(handler, "add_parent"):
+            raise TypeError("expected BaseHandler instance, got %r" %
+                            type(handler))
+
+        added = False
+        for meth in dir(handler):
+            if meth in ["redirect_request", "do_open", "proxy_open"]:
+                # oops, coincidental match
+                continue
+
+            i = meth.find("_")
+            protocol = meth[:i]
+            condition = meth[i+1:]
+
+            if condition.startswith("error"):
+                j = condition.find("_") + i + 1
+                kind = meth[j+1:]
+                try:
+                    kind = int(kind)
+                except ValueError:
+                    pass
+                lookup = self.handle_error.get(protocol, {})
+                self.handle_error[protocol] = lookup
+            elif condition == "open":
+                kind = protocol
+                lookup = self.handle_open
+            elif condition == "response":
+                kind = protocol
+                lookup = self.process_response
+            elif condition == "request":
+                kind = protocol
+                lookup = self.process_request
+            else:
+                continue
+
+            handlers = lookup.setdefault(kind, [])
+            if handlers:
+                bisect.insort(handlers, handler)
+            else:
+                handlers.append(handler)
+            added = True
+
+        if added:
+            # the handlers must work in an specific order, the order
+            # is specified in a Handler attribute
+            bisect.insort(self.handlers, handler)
+            handler.add_parent(self)
+
+    def close(self):
+        # Only exists for backwards compatibility.
+        pass
+
+    def _call_chain(self, chain, kind, meth_name, *args):
+        # Handlers raise an exception if no one else should try to handle
+        # the request, or return None if they can't but another handler
+        # could.  Otherwise, they return the response.
+        handlers = chain.get(kind, ())
+        for handler in handlers:
+            func = getattr(handler, meth_name)
+
+            result = func(*args)
+            if result is not None:
+                return result
+
+    def open(self, fullurl, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
+        # accept a URL or a Request object
+        if isinstance(fullurl, basestring):
+            req = Request(fullurl, data)
+        else:
+            req = fullurl
+            if data is not None:
+                req.add_data(data)
+
+        req.timeout = timeout
+        protocol = req.get_type()
+
+        # pre-process request
+        meth_name = protocol+"_request"
+        for processor in self.process_request.get(protocol, []):
+            meth = getattr(processor, meth_name)
+            req = meth(req)
+
+        response = self._open(req, data)
+
+        # post-process response
+        meth_name = protocol+"_response"
+        for processor in self.process_response.get(protocol, []):
+            meth = getattr(processor, meth_name)
+            response = meth(req, response)
+
+        return response
+
+    def _open(self, req, data=None):
+        result = self._call_chain(self.handle_open, 'default',
+                                  'default_open', req)
+        if result:
+            return result
+
+        protocol = req.get_type()
+        result = self._call_chain(self.handle_open, protocol, protocol +
+                                  '_open', req)
+        if result:
+            return result
+
+        return self._call_chain(self.handle_open, 'unknown',
+                                'unknown_open', req)
+
+    def error(self, proto, *args):
+        if proto in ('http', 'https'):
+            # XXX http[s] protocols are special-cased
+            dict = self.handle_error['http'] # https is not different than http
+            proto = args[2]  # YUCK!
+            meth_name = 'http_error_%s' % proto
+            http_err = 1
+            orig_args = args
+        else:
+            dict = self.handle_error
+            meth_name = proto + '_error'
+            http_err = 0
+        args = (dict, proto, meth_name) + args
+        result = self._call_chain(*args)
+        if result:
+            return result
+
+        if http_err:
+            args = (dict, 'default', 'http_error_default') + orig_args
+            return self._call_chain(*args)
+
+# XXX probably also want an abstract factory that knows when it makes
+# sense to skip a superclass in favor of a subclass and when it might
+# make sense to include both
+
+def build_opener(*handlers):
+    """Create an opener object from a list of handlers.
+
+    The opener will use several default handlers, including support
+    for HTTP, FTP and when applicable, HTTPS.
+
+    If any of the handlers passed as arguments are subclasses of the
+    default handlers, the default handlers will not be used.
+    """
+    import types
+    def isclass(obj):
+        return isinstance(obj, (types.ClassType, type))
+
+    opener = OpenerDirector()
+    default_classes = [ProxyHandler, UnknownHandler, HTTPHandler,
+                       HTTPDefaultErrorHandler, HTTPRedirectHandler,
+                       FTPHandler, FileHandler, HTTPErrorProcessor]
+    if hasattr(httplib, 'HTTPS'):
+        default_classes.append(HTTPSHandler)
+    skip = set()
+    for klass in default_classes:
+        for check in handlers:
+            if isclass(check):
+                if issubclass(check, klass):
+                    skip.add(klass)
+            elif isinstance(check, klass):
+                skip.add(klass)
+    for klass in skip:
+        default_classes.remove(klass)
+
+    for klass in default_classes:
+        opener.add_handler(klass())
+
+    for h in handlers:
+        if isclass(h):
+            h = h()
+        opener.add_handler(h)
+    return opener
+
+class BaseHandler:
+    handler_order = 500
+
+    def add_parent(self, parent):
+        self.parent = parent
+
+    def close(self):
+        # Only exists for backwards compatibility
+        pass
+
+    def __lt__(self, other):
+        if not hasattr(other, "handler_order"):
+            # Try to preserve the old behavior of having custom classes
+            # inserted after default ones (works only for custom user
+            # classes which are not aware of handler_order).
+            return True
+        return self.handler_order < other.handler_order
+
+
+class HTTPErrorProcessor(BaseHandler):
+    """Process HTTP error responses."""
+    handler_order = 1000  # after all other processing
+
+    def http_response(self, request, response):
+        code, msg, hdrs = response.code, response.msg, response.info()
+
+        # According to RFC 2616, "2xx" code indicates that the client's
+        # request was successfully received, understood, and accepted.
+        if not (200 <= code < 300):
+            response = self.parent.error(
+                'http', request, response, code, msg, hdrs)
+
+        return response
+
+    https_response = http_response
+
+class HTTPDefaultErrorHandler(BaseHandler):
+    def http_error_default(self, req, fp, code, msg, hdrs):
+        raise HTTPError(req.get_full_url(), code, msg, hdrs, fp)
+
+class HTTPRedirectHandler(BaseHandler):
+    # maximum number of redirections to any single URL
+    # this is needed because of the state that cookies introduce
+    max_repeats = 4
+    # maximum total number of redirections (regardless of URL) before
+    # assuming we're in a loop
+    max_redirections = 10
+
+    def redirect_request(self, req, fp, code, msg, headers, newurl):
+        """Return a Request or None in response to a redirect.
+
+        This is called by the http_error_30x methods when a
+        redirection response is received.  If a redirection should
+        take place, return a new Request to allow http_error_30x to
+        perform the redirect.  Otherwise, raise HTTPError if no-one
+        else should try to handle this url.  Return None if you can't
+        but another Handler might.
+        """
+        m = req.get_method()
+        if (code in (301, 302, 303, 307) and m in ("GET", "HEAD")
+            or code in (301, 302, 303) and m == "POST"):
+            # Strictly (according to RFC 2616), 301 or 302 in response
+            # to a POST MUST NOT cause a redirection without confirmation
+            # from the user (of urllib2, in this case).  In practice,
+            # essentially all clients do redirect in this case, so we
+            # do the same.
+            # be conciliant with URIs containing a space
+            newurl = newurl.replace(' ', '%20')
+            newheaders = dict((k,v) for k,v in req.headers.items()
+                              if k.lower() not in ("content-length", "content-type")
+                             )
+            return Request(newurl,
+                           headers=newheaders,
+                           origin_req_host=req.get_origin_req_host(),
+                           unverifiable=True)
+        else:
+            raise HTTPError(req.get_full_url(), code, msg, headers, fp)
+
+    # Implementation note: To avoid the server sending us into an
+    # infinite loop, the request object needs to track what URLs we
+    # have already seen.  Do this by adding a handler-specific
+    # attribute to the Request object.
+    def http_error_302(self, req, fp, code, msg, headers):
+        # Some servers (incorrectly) return multiple Location headers
+        # (so probably same goes for URI).  Use first header.
+        if 'location' in headers:
+            newurl = headers.getheaders('location')[0]
+        elif 'uri' in headers:
+            newurl = headers.getheaders('uri')[0]
+        else:
+            return
+
+        # fix a possible malformed URL
+        urlparts = urlparse.urlparse(newurl)
+        if not urlparts.path:
+            urlparts = list(urlparts)
+            urlparts[2] = "/"
+        newurl = urlparse.urlunparse(urlparts)
+
+        newurl = urlparse.urljoin(req.get_full_url(), newurl)
+
+        # XXX Probably want to forget about the state of the current
+        # request, although that might interact poorly with other
+        # handlers that also use handler-specific request attributes
+        new = self.redirect_request(req, fp, code, msg, headers, newurl)
+        if new is None:
+            return
+
+        # loop detection
+        # .redirect_dict has a key url if url was previously visited.
+        if hasattr(req, 'redirect_dict'):
+            visited = new.redirect_dict = req.redirect_dict
+            if (visited.get(newurl, 0) >= self.max_repeats or
+                len(visited) >= self.max_redirections):
+                raise HTTPError(req.get_full_url(), code,
+                                self.inf_msg + msg, headers, fp)
+        else:
+            visited = new.redirect_dict = req.redirect_dict = {}
+        visited[newurl] = visited.get(newurl, 0) + 1
+
+        # Don't close the fp until we are sure that we won't use it
+        # with HTTPError.
+        fp.read()
+        fp.close()
+
+        return self.parent.open(new, timeout=req.timeout)
+
+    http_error_301 = http_error_303 = http_error_307 = http_error_302
+
+    inf_msg = "The HTTP server returned a redirect error that would " \
+              "lead to an infinite loop.\n" \
+              "The last 30x error message was:\n"
+
+
+def _parse_proxy(proxy):
+    """Return (scheme, user, password, host/port) given a URL or an authority.
+
+    If a URL is supplied, it must have an authority (host:port) component.
+    According to RFC 3986, having an authority component means the URL must
+    have two slashes after the scheme:
+
+    >>> _parse_proxy('file:/ftp.example.com/')
+    Traceback (most recent call last):
+    ValueError: proxy URL with no authority: 'file:/ftp.example.com/'
+
+    The first three items of the returned tuple may be None.
+
+    Examples of authority parsing:
+
+    >>> _parse_proxy('proxy.example.com')
+    (None, None, None, 'proxy.example.com')
+    >>> _parse_proxy('proxy.example.com:3128')
+    (None, None, None, 'proxy.example.com:3128')
+
+    The authority component may optionally include userinfo (assumed to be
+    username:password):
+
+    >>> _parse_proxy('joe:password at proxy.example.com')
+    (None, 'joe', 'password', 'proxy.example.com')
+    >>> _parse_proxy('joe:password at proxy.example.com:3128')
+    (None, 'joe', 'password', 'proxy.example.com:3128')
+
+    Same examples, but with URLs instead:
+
+    >>> _parse_proxy('http://proxy.example.com/')
+    ('http', None, None, 'proxy.example.com')
+    >>> _parse_proxy('http://proxy.example.com:3128/')
+    ('http', None, None, 'proxy.example.com:3128')
+    >>> _parse_proxy('http://joe:password at proxy.example.com/')
+    ('http', 'joe', 'password', 'proxy.example.com')
+    >>> _parse_proxy('http://joe:password at proxy.example.com:3128')
+    ('http', 'joe', 'password', 'proxy.example.com:3128')
+
+    Everything after the authority is ignored:
+
+    >>> _parse_proxy('ftp://joe:password at proxy.example.com/rubbish:3128')
+    ('ftp', 'joe', 'password', 'proxy.example.com')
+
+    Test for no trailing '/' case:
+
+    >>> _parse_proxy('http://joe:password at proxy.example.com')
+    ('http', 'joe', 'password', 'proxy.example.com')
+
+    """
+    scheme, r_scheme = splittype(proxy)
+    if not r_scheme.startswith("/"):
+        # authority
+        scheme = None
+        authority = proxy
+    else:
+        # URL
+        if not r_scheme.startswith("//"):
+            raise ValueError("proxy URL with no authority: %r" % proxy)
+        # We have an authority, so for RFC 3986-compliant URLs (by ss 3.
+        # and 3.3.), path is empty or starts with '/'
+        end = r_scheme.find("/", 2)
+        if end == -1:
+            end = None
+        authority = r_scheme[2:end]
+    userinfo, hostport = splituser(authority)
+    if userinfo is not None:
+        user, password = splitpasswd(userinfo)
+    else:
+        user = password = None
+    return scheme, user, password, hostport
+
+class ProxyHandler(BaseHandler):
+    # Proxies must be in front
+    handler_order = 100
+
+    def __init__(self, proxies=None):
+        if proxies is None:
+            proxies = getproxies()
+        assert hasattr(proxies, 'has_key'), "proxies must be a mapping"
+        self.proxies = proxies
+        for type, url in proxies.items():
+            setattr(self, '%s_open' % type,
+                    lambda r, proxy=url, type=type, meth=self.proxy_open: \
+                    meth(r, proxy, type))
+
+    def proxy_open(self, req, proxy, type):
+        orig_type = req.get_type()
+        proxy_type, user, password, hostport = _parse_proxy(proxy)
+
+        if proxy_type is None:
+            proxy_type = orig_type
+
+        if req.host and proxy_bypass(req.host):
+            return None
+
+        if user and password:
+            user_pass = '%s:%s' % (unquote(user), unquote(password))
+            creds = base64.b64encode(user_pass).strip()
+            req.add_header('Proxy-authorization', 'Basic ' + creds)
+        hostport = unquote(hostport)
+        req.set_proxy(hostport, proxy_type)
+
+        if orig_type == proxy_type or orig_type == 'https':
+            # let other handlers take care of it
+            return None
+        else:
+            # need to start over, because the other handlers don't
+            # grok the proxy's URL type
+            # e.g. if we have a constructor arg proxies like so:
+            # {'http': 'ftp://proxy.example.com'}, we may end up turning
+            # a request for http://acme.example.com/a into one for
+            # ftp://proxy.example.com/a
+            return self.parent.open(req, timeout=req.timeout)
+
+class HTTPPasswordMgr:
+
+    def __init__(self):
+        self.passwd = {}
+
+    def add_password(self, realm, uri, user, passwd):
+        # uri could be a single URI or a sequence
+        if isinstance(uri, basestring):
+            uri = [uri]
+        if not realm in self.passwd:
+            self.passwd[realm] = {}
+        for default_port in True, False:
+            reduced_uri = tuple(
+                [self.reduce_uri(u, default_port) for u in uri])
+            self.passwd[realm][reduced_uri] = (user, passwd)
+
+    def find_user_password(self, realm, authuri):
+        domains = self.passwd.get(realm, {})
+        for default_port in True, False:
+            reduced_authuri = self.reduce_uri(authuri, default_port)
+            for uris, authinfo in domains.iteritems():
+                for uri in uris:
+                    if self.is_suburi(uri, reduced_authuri):
+                        return authinfo
+        return None, None
+
+    def reduce_uri(self, uri, default_port=True):
+        """Accept authority or URI and extract only the authority and path."""
+        # note HTTP URLs do not have a userinfo component
+        parts = urlparse.urlsplit(uri)
+        if parts[1]:
+            # URI
+            scheme = parts[0]
+            authority = parts[1]
+            path = parts[2] or '/'
+        else:
+            # host or host:port
+            scheme = None
+            authority = uri
+            path = '/'
+        host, port = splitport(authority)
+        if default_port and port is None and scheme is not None:
+            dport = {"http": 80,
+                     "https": 443,
+                     }.get(scheme)
+            if dport is not None:
+                authority = "%s:%d" % (host, dport)
+        return authority, path
+
+    def is_suburi(self, base, test):
+        """Check if test is below base in a URI tree
+
+        Both args must be URIs in reduced form.
+        """
+        if base == test:
+            return True
+        if base[0] != test[0]:
+            return False
+        common = posixpath.commonprefix((base[1], test[1]))
+        if len(common) == len(base[1]):
+            return True
+        return False
+
+
+class HTTPPasswordMgrWithDefaultRealm(HTTPPasswordMgr):
+
+    def find_user_password(self, realm, authuri):
+        user, password = HTTPPasswordMgr.find_user_password(self, realm,
+                                                            authuri)
+        if user is not None:
+            return user, password
+        return HTTPPasswordMgr.find_user_password(self, None, authuri)
+
+
+class AbstractBasicAuthHandler:
+
+    # XXX this allows for multiple auth-schemes, but will stupidly pick
+    # the last one with a realm specified.
+
+    # allow for double- and single-quoted realm values
+    # (single quotes are a violation of the RFC, but appear in the wild)
+    rx = re.compile('(?:.*,)*[ \t]*([^ \t]+)[ \t]+'
+                    'realm=(["\'])(.*?)\\2', re.I)
+
+    # XXX could pre-emptively send auth info already accepted (RFC 2617,
+    # end of section 2, and section 1.2 immediately after "credentials"
+    # production).
+
+    def __init__(self, password_mgr=None):
+        if password_mgr is None:
+            password_mgr = HTTPPasswordMgr()
+        self.passwd = password_mgr
+        self.add_password = self.passwd.add_password
+        self.retried = 0
+
+    def reset_retry_count(self):
+        self.retried = 0
+
+    def http_error_auth_reqed(self, authreq, host, req, headers):
+        # host may be an authority (without userinfo) or a URL with an
+        # authority
+        # XXX could be multiple headers
+        authreq = headers.get(authreq, None)
+
+        if self.retried > 5:
+            # retry sending the username:password 5 times before failing.
+            raise HTTPError(req.get_full_url(), 401, "basic auth failed",
+                            headers, None)
+        else:
+            self.retried += 1
+
+        if authreq:
+            mo = AbstractBasicAuthHandler.rx.search(authreq)
+            if mo:
+                scheme, quote, realm = mo.groups()
+                if scheme.lower() == 'basic':
+                    response = self.retry_http_basic_auth(host, req, realm)
+                    if response and response.code != 401:
+                        self.retried = 0
+                    return response
+
+    def retry_http_basic_auth(self, host, req, realm):
+        user, pw = self.passwd.find_user_password(realm, host)
+        if pw is not None:
+            raw = "%s:%s" % (user, pw)
+            auth = 'Basic %s' % base64.b64encode(raw).strip()
+            if req.headers.get(self.auth_header, None) == auth:
+                return None
+            req.add_unredirected_header(self.auth_header, auth)
+            return self.parent.open(req, timeout=req.timeout)
+        else:
+            return None
+
+
+class HTTPBasicAuthHandler(AbstractBasicAuthHandler, BaseHandler):
+
+    auth_header = 'Authorization'
+
+    def http_error_401(self, req, fp, code, msg, headers):
+        url = req.get_full_url()
+        response = self.http_error_auth_reqed('www-authenticate',
+                                              url, req, headers)
+        self.reset_retry_count()
+        return response
+
+
+class ProxyBasicAuthHandler(AbstractBasicAuthHandler, BaseHandler):
+
+    auth_header = 'Proxy-authorization'
+
+    def http_error_407(self, req, fp, code, msg, headers):
+        # http_error_auth_reqed requires that there is no userinfo component in
+        # authority.  Assume there isn't one, since urllib2 does not (and
+        # should not, RFC 3986 s. 3.2.1) support requests for URLs containing
+        # userinfo.
+        authority = req.get_host()
+        response = self.http_error_auth_reqed('proxy-authenticate',
+                                          authority, req, headers)
+        self.reset_retry_count()
+        return response
+
+
+def randombytes(n):
+    """Return n random bytes."""
+    # Use /dev/urandom if it is available.  Fall back to random module
+    # if not.  It might be worthwhile to extend this function to use
+    # other platform-specific mechanisms for getting random bytes.
+    if os.path.exists("/dev/urandom"):
+        f = open("/dev/urandom")
+        s = f.read(n)
+        f.close()
+        return s
+    else:
+        L = [chr(random.randrange(0, 256)) for i in range(n)]
+        return "".join(L)
+
+class AbstractDigestAuthHandler:
+    # Digest authentication is specified in RFC 2617.
+
+    # XXX The client does not inspect the Authentication-Info header
+    # in a successful response.
+
+    # XXX It should be possible to test this implementation against
+    # a mock server that just generates a static set of challenges.
+
+    # XXX qop="auth-int" supports is shaky
+
+    def __init__(self, passwd=None):
+        if passwd is None:
+            passwd = HTTPPasswordMgr()
+        self.passwd = passwd
+        self.add_password = self.passwd.add_password
+        self.retried = 0
+        self.nonce_count = 0
+        self.last_nonce = None
+
+    def reset_retry_count(self):
+        self.retried = 0
+
+    def http_error_auth_reqed(self, auth_header, host, req, headers):
+        authreq = headers.get(auth_header, None)
+        if self.retried > 5:
+            # Don't fail endlessly - if we failed once, we'll probably
+            # fail a second time. Hm. Unless the Password Manager is
+            # prompting for the information. Crap. This isn't great
+            # but it's better than the current 'repeat until recursion
+            # depth exceeded' approach 
+            raise HTTPError(req.get_full_url(), 401, "digest auth failed",
+                            headers, None)
+        else:
+            self.retried += 1
+        if authreq:
+            scheme = authreq.split()[0]
+            if scheme.lower() == 'digest':
+                return self.retry_http_digest_auth(req, authreq)
+
+    def retry_http_digest_auth(self, req, auth):
+        token, challenge = auth.split(' ', 1)
+        chal = parse_keqv_list(parse_http_list(challenge))
+        auth = self.get_authorization(req, chal)
+        if auth:
+            auth_val = 'Digest %s' % auth
+            if req.headers.get(self.auth_header, None) == auth_val:
+                return None
+            req.add_unredirected_header(self.auth_header, auth_val)
+            resp = self.parent.open(req, timeout=req.timeout)
+            return resp
+
+    def get_cnonce(self, nonce):
+        # The cnonce-value is an opaque
+        # quoted string value provided by the client and used by both client
+        # and server to avoid chosen plaintext attacks, to provide mutual
+        # authentication, and to provide some message integrity protection.
+        # This isn't a fabulous effort, but it's probably Good Enough.
+        dig = hashlib.sha1("%s:%s:%s:%s" % (self.nonce_count, nonce, time.ctime(),
+                                            randombytes(8))).hexdigest()
+        return dig[:16]
+
+    def get_authorization(self, req, chal):
+        try:
+            realm = chal['realm']
+            nonce = chal['nonce']
+            qop = chal.get('qop')
+            algorithm = chal.get('algorithm', 'MD5')
+            # mod_digest doesn't send an opaque, even though it isn't
+            # supposed to be optional
+            opaque = chal.get('opaque', None)
+        except KeyError:
+            return None
+
+        H, KD = self.get_algorithm_impls(algorithm)
+        if H is None:
+            return None
+
+        user, pw = self.passwd.find_user_password(realm, req.get_full_url())
+        if user is None:
+            return None
+
+        # XXX not implemented yet
+        if req.has_data():
+            entdig = self.get_entity_digest(req.get_data(), chal)
+        else:
+            entdig = None
+
+        A1 = "%s:%s:%s" % (user, realm, pw)
+        A2 = "%s:%s" % (req.get_method(),
+                        # XXX selector: what about proxies and full urls
+                        req.get_selector())
+        if qop == 'auth':
+            if nonce == self.last_nonce:
+                self.nonce_count += 1
+            else:
+                self.nonce_count = 1
+                self.last_nonce = nonce
+
+            ncvalue = '%08x' % self.nonce_count
+            cnonce = self.get_cnonce(nonce)
+            noncebit = "%s:%s:%s:%s:%s" % (nonce, ncvalue, cnonce, qop, H(A2))
+            respdig = KD(H(A1), noncebit)
+        elif qop is None:
+            respdig = KD(H(A1), "%s:%s" % (nonce, H(A2)))
+        else:
+            # XXX handle auth-int.
+            raise URLError("qop '%s' is not supported." % qop)
+
+        # XXX should the partial digests be encoded too?
+
+        base = 'username="%s", realm="%s", nonce="%s", uri="%s", ' \
+               'response="%s"' % (user, realm, nonce, req.get_selector(),
+                                  respdig)
+        if opaque:
+            base += ', opaque="%s"' % opaque
+        if entdig:
+            base += ', digest="%s"' % entdig
+        base += ', algorithm="%s"' % algorithm
+        if qop:
+            base += ', qop=auth, nc=%s, cnonce="%s"' % (ncvalue, cnonce)
+        return base
+
+    def get_algorithm_impls(self, algorithm):
+        # algorithm should be case-insensitive according to RFC2617
+        algorithm = algorithm.upper()
+        # lambdas assume digest modules are imported at the top level
+        if algorithm == 'MD5':
+            H = lambda x: hashlib.md5(x).hexdigest()
+        elif algorithm == 'SHA':
+            H = lambda x: hashlib.sha1(x).hexdigest()
+        # XXX MD5-sess
+        KD = lambda s, d: H("%s:%s" % (s, d))
+        return H, KD
+
+    def get_entity_digest(self, data, chal):
+        # XXX not implemented yet
+        return None
+
+
+class HTTPDigestAuthHandler(BaseHandler, AbstractDigestAuthHandler):
+    """An authentication protocol defined by RFC 2069
+
+    Digest authentication improves on basic authentication because it
+    does not transmit passwords in the clear.
+    """
+
+    auth_header = 'Authorization'
+    handler_order = 490  # before Basic auth
+
+    def http_error_401(self, req, fp, code, msg, headers):
+        host = urlparse.urlparse(req.get_full_url())[1]
+        retry = self.http_error_auth_reqed('www-authenticate',
+                                           host, req, headers)
+        self.reset_retry_count()
+        return retry
+
+
+class ProxyDigestAuthHandler(BaseHandler, AbstractDigestAuthHandler):
+
+    auth_header = 'Proxy-Authorization'
+    handler_order = 490  # before Basic auth
+
+    def http_error_407(self, req, fp, code, msg, headers):
+        host = req.get_host()
+        retry = self.http_error_auth_reqed('proxy-authenticate',
+                                           host, req, headers)
+        self.reset_retry_count()
+        return retry
+
+class AbstractHTTPHandler(BaseHandler):
+
+    def __init__(self, debuglevel=0):
+        self._debuglevel = debuglevel
+
+    def set_http_debuglevel(self, level):
+        self._debuglevel = level
+
+    def do_request_(self, request):
+        host = request.get_host()
+        if not host:
+            raise URLError('no host given')
+
+        if request.has_data():  # POST
+            data = request.get_data()
+            if not request.has_header('Content-type'):
+                request.add_unredirected_header(
+                    'Content-type',
+                    'application/x-www-form-urlencoded')
+            if not request.has_header('Content-length'):
+                request.add_unredirected_header(
+                    'Content-length', '%d' % len(data))
+
+        sel_host = host
+        if request.has_proxy():
+            scheme, sel = splittype(request.get_selector())
+            sel_host, sel_path = splithost(sel)
+
+        if not request.has_header('Host'):
+            request.add_unredirected_header('Host', sel_host)
+        for name, value in self.parent.addheaders:
+            name = name.capitalize()
+            if not request.has_header(name):
+                request.add_unredirected_header(name, value)
+
+        return request
+
+    def do_open(self, http_class, req):
+        """Return an addinfourl object for the request, using http_class.
+
+        http_class must implement the HTTPConnection API from httplib.
+        The addinfourl return value is a file-like object.  It also
+        has methods and attributes including:
+            - info(): return a mimetools.Message object for the headers
+            - geturl(): return the original request URL
+            - code: HTTP status code
+        """
+        host = req.get_host()
+        if not host:
+            raise URLError('no host given')
+
+        h = http_class(host, timeout=req.timeout) # will parse host:port
+        h.set_debuglevel(self._debuglevel)
+
+        headers = dict(req.unredirected_hdrs)
+        headers.update(dict((k, v) for k, v in req.headers.items()
+                            if k not in headers))
+
+        # We want to make an HTTP/1.1 request, but the addinfourl
+        # class isn't prepared to deal with a persistent connection.
+        # It will try to read all remaining data from the socket,
+        # which will block while the server waits for the next request.
+        # So make sure the connection gets closed after the (only)
+        # request.
+        headers["Connection"] = "close"
+        headers = dict(
+            (name.title(), val) for name, val in headers.items())
+
+        if req._tunnel_host:
+            tunnel_headers = {}
+            proxy_auth_hdr = "Proxy-Authorization"
+            if proxy_auth_hdr in headers:
+                tunnel_headers[proxy_auth_hdr] = headers[proxy_auth_hdr]
+                # Proxy-Authorization should not be sent to origin
+                # server.
+                del headers[proxy_auth_hdr]
+            h.set_tunnel(req._tunnel_host, headers=tunnel_headers)
+
+        try:
+            h.request(req.get_method(), req.get_selector(), req.data, headers)
+            try:
+                r = h.getresponse(buffering=True)
+            except TypeError: #buffering kw not supported
+                r = h.getresponse()
+        except socket.error, err: # XXX what error?
+            h.close()
+            raise URLError(err)
+
+        # Pick apart the HTTPResponse object to get the addinfourl
+        # object initialized properly.
+
+        # Wrap the HTTPResponse object in socket's file object adapter
+        # for Windows.  That adapter calls recv(), so delegate recv()
+        # to read().  This weird wrapping allows the returned object to
+        # have readline() and readlines() methods.
+
+        # XXX It might be better to extract the read buffering code
+        # out of socket._fileobject() and into a base class.
+
+        r.recv = r.read
+        fp = socket._fileobject(r, close=True)
+
+        resp = addinfourl(fp, r.msg, req.get_full_url())
+        resp.code = r.status
+        resp.msg = r.reason
+        return resp
+
+
+class HTTPHandler(AbstractHTTPHandler):
+
+    def http_open(self, req):
+        return self.do_open(httplib.HTTPConnection, req)
+
+    http_request = AbstractHTTPHandler.do_request_
+
+if hasattr(httplib, 'HTTPS'):
+    class HTTPSHandler(AbstractHTTPHandler):
+
+        def https_open(self, req):
+            return self.do_open(httplib.HTTPSConnection, req)
+
+        https_request = AbstractHTTPHandler.do_request_
+
+class HTTPCookieProcessor(BaseHandler):
+    def __init__(self, cookiejar=None):
+        import cookielib
+        if cookiejar is None:
+            cookiejar = cookielib.CookieJar()
+        self.cookiejar = cookiejar
+
+    def http_request(self, request):
+        self.cookiejar.add_cookie_header(request)
+        return request
+
+    def http_response(self, request, response):
+        self.cookiejar.extract_cookies(response, request)
+        return response
+
+    https_request = http_request
+    https_response = http_response
+
+class UnknownHandler(BaseHandler):
+    def unknown_open(self, req):
+        type = req.get_type()
+        raise URLError('unknown url type: %s' % type)
+
+def parse_keqv_list(l):
+    """Parse list of key=value strings where keys are not duplicated."""
+    parsed = {}
+    for elt in l:
+        k, v = elt.split('=', 1)
+        if v[0] == '"' and v[-1] == '"':
+            v = v[1:-1]
+        parsed[k] = v
+    return parsed
+
+def parse_http_list(s):
+    """Parse lists as described by RFC 2068 Section 2.
+
+    In particular, parse comma-separated lists where the elements of
+    the list may include quoted-strings.  A quoted-string could
+    contain a comma.  A non-quoted string could have quotes in the
+    middle.  Neither commas nor quotes count if they are escaped.
+    Only double-quotes count, not single-quotes.
+    """
+    res = []
+    part = ''
+
+    escape = quote = False
+    for cur in s:
+        if escape:
+            part += cur
+            escape = False
+            continue
+        if quote:
+            if cur == '\\':
+                escape = True
+                continue
+            elif cur == '"':
+                quote = False
+            part += cur
+            continue
+
+        if cur == ',':
+            res.append(part)
+            part = ''
+            continue
+
+        if cur == '"':
+            quote = True
+
+        part += cur
+
+    # append last part
+    if part:
+        res.append(part)
+
+    return [part.strip() for part in res]
+
+def _safe_gethostbyname(host):
+    try:
+        return socket.gethostbyname(host)
+    except socket.gaierror:
+        return None
+
+class FileHandler(BaseHandler):
+    # Use local file or FTP depending on form of URL
+    def file_open(self, req):
+        url = req.get_selector()
+        if url[:2] == '//' and url[2:3] != '/' and (req.host and
+                req.host != 'localhost'):
+            req.type = 'ftp'
+            return self.parent.open(req)
+        else:
+            return self.open_local_file(req)
+
+    # names for the localhost
+    names = None
+    def get_names(self):
+        if FileHandler.names is None:
+            try:
+                FileHandler.names = tuple(
+                    socket.gethostbyname_ex('localhost')[2] +
+                    socket.gethostbyname_ex(socket.gethostname())[2])
+            except socket.gaierror:
+                FileHandler.names = (socket.gethostbyname('localhost'),)
+        return FileHandler.names
+
+    # not entirely sure what the rules are here
+    def open_local_file(self, req):
+        import email.utils
+        import mimetypes
+        host = req.get_host()
+        filename = req.get_selector()
+        localfile = url2pathname(filename)
+        try:
+            stats = os.stat(localfile)
+            size = stats.st_size
+            modified = email.utils.formatdate(stats.st_mtime, usegmt=True)
+            mtype = mimetypes.guess_type(filename)[0]
+            headers = mimetools.Message(StringIO(
+                'Content-type: %s\nContent-length: %d\nLast-modified: %s\n' %
+                (mtype or 'text/plain', size, modified)))
+            if host:
+                host, port = splitport(host)
+            if not host or \
+                (not port and _safe_gethostbyname(host) in self.get_names()):
+                if host:
+                    origurl = 'file://' + host + filename
+                else:
+                    origurl = 'file://' + filename
+                return addinfourl(open(localfile, 'rb'), headers, origurl)
+        except OSError, msg:
+            # urllib2 users shouldn't expect OSErrors coming from urlopen()
+            raise URLError(msg)
+        raise URLError('file not on local host')
+
+class FTPHandler(BaseHandler):
+    def ftp_open(self, req):
+        import ftplib
+        import mimetypes
+        host = req.get_host()
+        if not host:
+            raise URLError('ftp error: no host given')
+        host, port = splitport(host)
+        if port is None:
+            port = ftplib.FTP_PORT
+        else:
+            port = int(port)
+
+        # username/password handling
+        user, host = splituser(host)
+        if user:
+            user, passwd = splitpasswd(user)
+        else:
+            passwd = None
+        host = unquote(host)
+        user = user or ''
+        passwd = passwd or ''
+
+        try:
+            host = socket.gethostbyname(host)
+        except socket.error, msg:
+            raise URLError(msg)
+        path, attrs = splitattr(req.get_selector())
+        dirs = path.split('/')
+        dirs = map(unquote, dirs)
+        dirs, file = dirs[:-1], dirs[-1]
+        if dirs and not dirs[0]:
+            dirs = dirs[1:]
+        try:
+            fw = self.connect_ftp(user, passwd, host, port, dirs, req.timeout)
+            type = file and 'I' or 'D'
+            for attr in attrs:
+                attr, value = splitvalue(attr)
+                if attr.lower() == 'type' and \
+                   value in ('a', 'A', 'i', 'I', 'd', 'D'):
+                    type = value.upper()
+            fp, retrlen = fw.retrfile(file, type)
+            headers = ""
+            mtype = mimetypes.guess_type(req.get_full_url())[0]
+            if mtype:
+                headers += "Content-type: %s\n" % mtype
+            if retrlen is not None and retrlen >= 0:
+                headers += "Content-length: %d\n" % retrlen
+            sf = StringIO(headers)
+            headers = mimetools.Message(sf)
+            return addinfourl(fp, headers, req.get_full_url())
+        except ftplib.all_errors, msg:
+            raise URLError, ('ftp error: %s' % msg), sys.exc_info()[2]
+
+    def connect_ftp(self, user, passwd, host, port, dirs, timeout):
+        fw = ftpwrapper(user, passwd, host, port, dirs, timeout)
+##        fw.ftp.set_debuglevel(1)
+        return fw
+
+class CacheFTPHandler(FTPHandler):
+    # XXX would be nice to have pluggable cache strategies
+    # XXX this stuff is definitely not thread safe
+    def __init__(self):
+        self.cache = {}
+        self.timeout = {}
+        self.soonest = 0
+        self.delay = 60
+        self.max_conns = 16
+
+    def setTimeout(self, t):
+        self.delay = t
+
+    def setMaxConns(self, m):
+        self.max_conns = m
+
+    def connect_ftp(self, user, passwd, host, port, dirs, timeout):
+        key = user, host, port, '/'.join(dirs), timeout
+        if key in self.cache:
+            self.timeout[key] = time.time() + self.delay
+        else:
+            self.cache[key] = ftpwrapper(user, passwd, host, port, dirs, timeout)
+            self.timeout[key] = time.time() + self.delay
+        self.check_cache()
+        return self.cache[key]
+
+    def check_cache(self):
+        # first check for old ones
+        t = time.time()
+        if self.soonest <= t:
+            for k, v in self.timeout.items():
+                if v < t:
+                    self.cache[k].close()
+                    del self.cache[k]
+                    del self.timeout[k]
+        self.soonest = min(self.timeout.values())
+
+        # then check the size
+        if len(self.cache) == self.max_conns:
+            for k, v in self.timeout.items():
+                if v == self.soonest:
+                    del self.cache[k]
+                    del self.timeout[k]
+                    break
+            self.soonest = min(self.timeout.values())
diff --git a/lib-python/2.7/uuid.py b/lib-python/modified-2.7/uuid.py
copy from lib-python/2.7/uuid.py
copy to lib-python/modified-2.7/uuid.py
--- a/lib-python/2.7/uuid.py
+++ b/lib-python/modified-2.7/uuid.py
@@ -406,8 +406,12 @@
             continue
         if hasattr(lib, 'uuid_generate_random'):
             _uuid_generate_random = lib.uuid_generate_random
+            _uuid_generate_random.argtypes = [ctypes.c_char * 16]
+            _uuid_generate_random.restype = None
         if hasattr(lib, 'uuid_generate_time'):
             _uuid_generate_time = lib.uuid_generate_time
+            _uuid_generate_time.argtypes = [ctypes.c_char * 16]
+            _uuid_generate_time.restype = None
 
     # The uuid_generate_* functions are broken on MacOS X 10.5, as noted
     # in issue #8621 the function generates the same sequence of values
@@ -436,6 +440,9 @@
         lib = None
     _UuidCreate = getattr(lib, 'UuidCreateSequential',
                           getattr(lib, 'UuidCreate', None))
+    if _UuidCreate is not None:
+        _UuidCreate.argtypes = [ctypes.c_char * 16]
+        _UuidCreate.restype = ctypes.c_int
 except:
     pass
 
diff --git a/lib_pypy/_codecs_cn.py b/lib_pypy/_codecs_cn.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_codecs_cn.py
@@ -0,0 +1,7 @@
+# this getcodec() function supports any multibyte codec, although
+# for compatibility with CPython it should only be used for the
+# codecs from this module, i.e.:
+#
+#     'gb2312', 'gbk', 'gb18030', 'hz'
+
+from _multibytecodec import __getcodec as getcodec
diff --git a/lib_pypy/_codecs_hk.py b/lib_pypy/_codecs_hk.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_codecs_hk.py
@@ -0,0 +1,7 @@
+# this getcodec() function supports any multibyte codec, although
+# for compatibility with CPython it should only be used for the
+# codecs from this module, i.e.:
+#
+#    'big5hkscs'
+
+from _multibytecodec import __getcodec as getcodec
diff --git a/lib_pypy/_codecs_iso2022.py b/lib_pypy/_codecs_iso2022.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_codecs_iso2022.py
@@ -0,0 +1,8 @@
+# this getcodec() function supports any multibyte codec, although
+# for compatibility with CPython it should only be used for the
+# codecs from this module, i.e.:
+#
+#    'iso2022_kr', 'iso2022_jp', 'iso2022_jp_1', 'iso2022_jp_2',
+#    'iso2022_jp_2004', 'iso2022_jp_3', 'iso2022_jp_ext'
+
+from _multibytecodec import __getcodec as getcodec
diff --git a/lib_pypy/_codecs_jp.py b/lib_pypy/_codecs_jp.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_codecs_jp.py
@@ -0,0 +1,8 @@
+# this getcodec() function supports any multibyte codec, although
+# for compatibility with CPython it should only be used for the
+# codecs from this module, i.e.:
+#
+#    'shift_jis', 'cp932', 'euc_jp', 'shift_jis_2004',
+#    'euc_jis_2004', 'euc_jisx0213', 'shift_jisx0213'
+
+from _multibytecodec import __getcodec as getcodec
diff --git a/lib_pypy/_codecs_kr.py b/lib_pypy/_codecs_kr.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_codecs_kr.py
@@ -0,0 +1,7 @@
+# this getcodec() function supports any multibyte codec, although
+# for compatibility with CPython it should only be used for the
+# codecs from this module, i.e.:
+#
+#    'euc_kr', 'cp949', 'johab'
+
+from _multibytecodec import __getcodec as getcodec
diff --git a/lib_pypy/_codecs_tw.py b/lib_pypy/_codecs_tw.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_codecs_tw.py
@@ -0,0 +1,7 @@
+# this getcodec() function supports any multibyte codec, although
+# for compatibility with CPython it should only be used for the
+# codecs from this module, i.e.:
+#
+#    'big5', 'cp950'
+
+from _multibytecodec import __getcodec as getcodec
diff --git a/lib_pypy/_collections.py b/lib_pypy/_collections.py
--- a/lib_pypy/_collections.py
+++ b/lib_pypy/_collections.py
@@ -379,12 +379,14 @@
 class defaultdict(dict):
     
     def __init__(self, *args, **kwds):
-        self.default_factory = None
-        if 'default_factory' in kwds:
-            self.default_factory = kwds.pop('default_factory')
-        elif len(args) > 0 and (callable(args[0]) or args[0] is None):
-            self.default_factory = args[0]
+        if len(args) > 0:
+            default_factory = args[0]
             args = args[1:]
+            if not callable(default_factory) and default_factory is not None:
+                raise TypeError("first argument must be callable")
+        else:
+            default_factory = None
+        self.default_factory = default_factory
         super(defaultdict, self).__init__(*args, **kwds)
  
     def __missing__(self, key):
@@ -404,7 +406,7 @@
             recurse.remove(id(self))
 
     def copy(self):
-        return type(self)(self, default_factory=self.default_factory)
+        return type(self)(self.default_factory, self)
     
     def __copy__(self):
         return self.copy()
diff --git a/lib_pypy/_ctypes/__init__.py b/lib_pypy/_ctypes/__init__.py
--- a/lib_pypy/_ctypes/__init__.py
+++ b/lib_pypy/_ctypes/__init__.py
@@ -18,7 +18,16 @@
 if _os.name in ("nt", "ce"):
     from _rawffi import FormatError
     from _rawffi import check_HRESULT as _check_HRESULT
-    CopyComPointer = None # XXX
+
+    def CopyComPointer(src, dst):
+        from ctypes import c_void_p, cast
+        if src:
+            hr = src[0][0].AddRef(src)
+            if hr & 0x80000000:
+                return hr
+        dst[0] = cast(src, c_void_p).value
+        return 0
+
     LoadLibrary = dlopen
 
 from _rawffi import FUNCFLAG_STDCALL, FUNCFLAG_CDECL, FUNCFLAG_PYTHONAPI
diff --git a/lib_pypy/_ctypes/array.py b/lib_pypy/_ctypes/array.py
--- a/lib_pypy/_ctypes/array.py
+++ b/lib_pypy/_ctypes/array.py
@@ -208,6 +208,9 @@
     def _get_buffer_value(self):
         return self._buffer.buffer
 
+    def _to_ffi_param(self):
+        return self._get_buffer_value()
+
 ARRAY_CACHE = {}
 
 def create_array_type(base, length):
diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py
--- a/lib_pypy/_ctypes/basics.py
+++ b/lib_pypy/_ctypes/basics.py
@@ -1,5 +1,6 @@
 
 import _rawffi
+import _ffi
 import sys
 
 keepalive_key = str # XXX fix this when provided with test
@@ -46,6 +47,16 @@
         else:
             return self.from_param(as_parameter)
 
+    def get_ffi_param(self, value):
+        cdata = self.from_param(value)
+        return cdata, cdata._to_ffi_param()
+
+    def get_ffi_argtype(self):
+        if self._ffiargtype:
+            return self._ffiargtype
+        self._ffiargtype = _shape_to_ffi_type(self._ffiargshape)
+        return self._ffiargtype
+
     def _CData_output(self, resbuffer, base=None, index=-1):
         #assert isinstance(resbuffer, _rawffi.ArrayInstance)
         """Used when data exits ctypes and goes into user code.
@@ -99,6 +110,7 @@
     """
     __metaclass__ = _CDataMeta
     _objects = None
+    _ffiargtype = None
 
     def __init__(self, *args, **kwds):
         raise TypeError("%s has no type" % (type(self),))
@@ -119,11 +131,20 @@
     def _get_buffer_value(self):
         return self._buffer[0]
 
+    def _to_ffi_param(self):
+        if self.__class__._is_pointer_like():
+            return self._get_buffer_value()
+        else:
+            return self.value
+
     def __buffer__(self):
         return buffer(self._buffer)
 
     def _get_b_base(self):
-        return self._base
+        try:
+            return self._base
+        except AttributeError:
+            return None
     _b_base_ = property(_get_b_base)
     _b_needsfree_ = False
 
@@ -146,11 +167,12 @@
     return tp._alignmentofinstances()
 
 def byref(cdata):
-    from ctypes import pointer
+    # "pointer" is imported at the end of this module to avoid circular
+    # imports
     return pointer(cdata)
 
 def cdata_from_address(self, address):
-    # fix the address, in case it's unsigned
+    # fix the address: turn it into as unsigned, in case it's a negative number
     address = address & (sys.maxint * 2 + 1)
     instance = self.__new__(self)
     lgt = getattr(self, '_length_', 1)
@@ -159,3 +181,54 @@
 
 def addressof(tp):
     return tp._buffer.buffer
+
+
+# ----------------------------------------------------------------------
+
+def is_struct_shape(shape):
+    # see the corresponding code to set the shape in
+    # _ctypes.structure._set_shape
+    return (isinstance(shape, tuple) and
+            len(shape) == 2 and
+            isinstance(shape[0], _rawffi.Structure) and
+            shape[1] == 1)
+
+def _shape_to_ffi_type(shape):
+    try:
+        return _shape_to_ffi_type.typemap[shape]
+    except KeyError:
+        pass
+    if is_struct_shape(shape):
+        return shape[0].get_ffi_type()
+    #
+    assert False, 'unknown shape %s' % (shape,)
+
+
+_shape_to_ffi_type.typemap =  {
+    'c' : _ffi.types.char,
+    'b' : _ffi.types.sbyte,
+    'B' : _ffi.types.ubyte,
+    'h' : _ffi.types.sshort,
+    'u' : _ffi.types.unichar,
+    'H' : _ffi.types.ushort,
+    'i' : _ffi.types.sint,
+    'I' : _ffi.types.uint,
+    'l' : _ffi.types.slong,
+    'L' : _ffi.types.ulong,
+    'q' : _ffi.types.slonglong,
+    'Q' : _ffi.types.ulonglong,
+    'f' : _ffi.types.float,
+    'd' : _ffi.types.double,
+    's' : _ffi.types.void_p,
+    'P' : _ffi.types.void_p,
+    'z' : _ffi.types.void_p,
+    'O' : _ffi.types.void_p,
+    'Z' : _ffi.types.void_p,
+    'X' : _ffi.types.void_p,
+    'v' : _ffi.types.sshort,
+    '?' : _ffi.types.ubyte,
+    }
+
+
+# used by "byref"
+from _ctypes.pointer import pointer
diff --git a/lib_pypy/_ctypes/function.py b/lib_pypy/_ctypes/function.py
--- a/lib_pypy/_ctypes/function.py
+++ b/lib_pypy/_ctypes/function.py
@@ -1,12 +1,15 @@
+
+from _ctypes.basics import _CData, _CDataMeta, cdata_from_address
+from _ctypes.primitive import SimpleType, _SimpleCData
+from _ctypes.basics import ArgumentError, keepalive_key
+from _ctypes.basics import is_struct_shape
+from _ctypes.builtin import set_errno, set_last_error
 import _rawffi
+import _ffi
 import sys
 import traceback
 import warnings
 
-from _ctypes.basics import ArgumentError, keepalive_key
-from _ctypes.basics import _CData, _CDataMeta, cdata_from_address
-from _ctypes.builtin import set_errno, set_last_error
-from _ctypes.primitive import SimpleType
 
 # XXX this file needs huge refactoring I fear
 
@@ -24,6 +27,7 @@
 
 WIN64 = sys.platform == 'win32' and sys.maxint == 2**63 - 1
 
+
 def get_com_error(errcode, riid, pIunk):
     "Win32 specific: build a COM Error exception"
     # XXX need C support code
@@ -36,6 +40,7 @@
     funcptr.restype = int
     return funcptr(*args)
 
+
 class CFuncPtrType(_CDataMeta):
     # XXX write down here defaults and such things
 
@@ -50,6 +55,7 @@
 
     from_address = cdata_from_address
 
+
 class CFuncPtr(_CData):
     __metaclass__ = CFuncPtrType
 
@@ -65,12 +71,12 @@
     callable = None
     _ptr = None
     _buffer = None
+    _address = None
     # win32 COM properties
     _paramflags = None
     _com_index = None
     _com_iid = None
-
-    __restype_set = False
+    _is_fastpath = False
 
     def _getargtypes(self):
         return self._argtypes_
@@ -85,9 +91,14 @@
                     raise TypeError(
                         "item %d in _argtypes_ has no from_param method" % (
                             i + 1,))
-            self._argtypes_ = argtypes
+            self._argtypes_ = list(argtypes)
+            self._check_argtypes_for_fastpath()
+    argtypes = property(_getargtypes, _setargtypes)
 
-    argtypes = property(_getargtypes, _setargtypes)
+    def _check_argtypes_for_fastpath(self):
+        if all([hasattr(argtype, '_ffiargshape') for argtype in self._argtypes_]):
+            fastpath_cls = make_fastpath_subclass(self.__class__)
+            fastpath_cls.enable_fastpath_maybe(self)
 
     def _getparamflags(self):
         return self._paramflags
@@ -133,11 +144,11 @@
 
     paramflags = property(_getparamflags, _setparamflags)
 
+
     def _getrestype(self):
         return self._restype_
 
     def _setrestype(self, restype):
-        self.__restype_set = True
         self._ptr = None
         if restype is int:
             from ctypes import c_int
@@ -146,27 +157,24 @@
                 callable(restype)):
             raise TypeError("restype must be a type, a callable, or None")
         self._restype_ = restype
-
+        
     def _delrestype(self):
         self._ptr = None
         del self._restype_
-
+        
     restype = property(_getrestype, _setrestype, _delrestype)
 
     def _geterrcheck(self):
         return getattr(self, '_errcheck_', None)
-

From noreply at buildbot.pypy.org  Sun Jan 22 20:28:05 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 22 Jan 2012 20:28:05 +0100 (CET)
Subject: [pypy-commit] pypy 32ptr-on-64bit: This part of the code lost
 during the merge belongs to rewrite.py.
Message-ID: <20120122192805.B8233821FA@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 32ptr-on-64bit
Changeset: r51660:6687f847a628
Date: 2012-01-22 20:16 +0100
http://bitbucket.org/pypy/pypy/changeset/6687f847a628/

Log:	This part of the code lost during the merge belongs to rewrite.py.
	Save it there for now, commented out.

diff --git a/pypy/jit/backend/llsupport/rewrite.py b/pypy/jit/backend/llsupport/rewrite.py
--- a/pypy/jit/backend/llsupport/rewrite.py
+++ b/pypy/jit/backend/llsupport/rewrite.py
@@ -326,3 +326,58 @@
             # assume that "self.gc_ll_descr.minimal_size_in_nursery" is 2 WORDs
             size = max(size, 2 * WORD)
             return (size + WORD-1) & ~(WORD-1)     # round up
+
+
+if 0:
+            # ---------- compressptr support ----------
+            descr = op.getdescr()
+            if (self.supports_compressed_ptrs and
+                (isinstance(descr, GcPtrHidden32FieldDescr) or
+                 isinstance(descr, GcPtrHidden32ArrayDescr))):
+                num = op.getopnum()
+                if (num == rop.GETFIELD_GC or
+                    num == rop.GETFIELD_GC_PURE or
+                    num == rop.GETARRAYITEM_GC or
+                    num == rop.GETARRAYITEM_GC_PURE):
+                    v1 = BoxInt()
+                    v2 = op.result
+                    newops.append(op.copy_and_change(num, result=v1))
+                    op = ResOperation(rop.SHOW_FROM_PTR32, [v1], v2)
+                elif num == rop.SETFIELD_GC or num == rop.SETFIELD_RAW:
+                    v1 = op.getarg(1)
+                    v2 = BoxInt()
+                    newops.append(ResOperation(rop.HIDE_INTO_PTR32, [v1], v2))
+                    op = op.copy_and_change(num, args=[op.getarg(0), v2])
+                elif num == rop.SETARRAYITEM_GC or num == rop.SETARRAYITEM_RAW:
+                    v1 = op.getarg(2)
+                    v2 = BoxInt()
+                    newops.append(ResOperation(rop.HIDE_INTO_PTR32, [v1], v2))
+                    op = op.copy_and_change(num, args=[op.getarg(0),
+                                                       op.getarg(1), v2])
+                elif num == rop.ARRAYLEN_GC or num == rop.NEW_ARRAY:
+                    # although these operations operate on a
+                    # GcArray(HiddenGcRef32), there is no actual
+                    # HiddenGcRef32 argument or result
+                    pass
+                else:
+                    raise AssertionError(op)
+            elif (self.supports_compressed_ptrs and
+                  isinstance(descr, BaseCallDescr)):
+                args = op.getarglist()
+                arg_classes = descr.get_arg_types()
+                fixed = 1
+                assert len(args) == fixed + len(arg_classes)
+                for i in range(len(arg_classes)):
+                    if arg_classes[i] == 'H':
+                        v1 = args[fixed + i]
+                        v2 = BoxInt()
+                        newops.append(ResOperation(rop.HIDE_INTO_PTR32,
+                                                   [v1], v2))
+                        args[fixed + i] = v2
+                        op = op.copy_and_change(op.getopnum(), args=args)
+                if descr.get_return_type() == 'H':
+                    v1 = BoxInt()
+                    v2 = op.result
+                    newops.append(op.copy_and_change(op.getopnum(), result=v1))
+                    op = ResOperation(rop.SHOW_FROM_PTR32, [v1], v2)
+            # ----------

From noreply at buildbot.pypy.org  Sun Jan 22 20:28:06 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 22 Jan 2012 20:28:06 +0100 (CET)
Subject: [pypy-commit] pypy 32ptr-on-64bit: fix.
Message-ID: <20120122192806.EB188821FA@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 32ptr-on-64bit
Changeset: r51661:27de83186d13
Date: 2012-01-22 20:20 +0100
http://bitbucket.org/pypy/pypy/changeset/27de83186d13/

Log:	fix.

diff --git a/pypy/rpython/lltypesystem/opimpl.py b/pypy/rpython/lltypesystem/opimpl.py
--- a/pypy/rpython/lltypesystem/opimpl.py
+++ b/pypy/rpython/lltypesystem/opimpl.py
@@ -622,14 +622,14 @@
     return lltype.cast_opaque_ptr(llmemory.HiddenGcRef32, ptr)
 
 def op_show_from_ptr32(RESTYPE, ptr32):
-    if not ptr32:
-        return lltype.nullptr(RESTYPE.TO)
     if RESTYPE == llmemory.Address:
         if not ptr32:
             return llmemory.NULL
         PTRTYPE = lltype.Ptr(ptr32._obj.container._TYPE)
         ptr = lltype.cast_opaque_ptr(PTRTYPE, ptr32)
         return llmemory.cast_ptr_to_adr(ptr)
+    if not ptr32:
+        return lltype.nullptr(RESTYPE.TO)
     if isinstance(RESTYPE.TO, lltype.GcOpaqueType):
         try:
             ptr32 = ptr32._obj.container._as_ptr()

From noreply at buildbot.pypy.org  Sun Jan 22 20:57:13 2012
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sun, 22 Jan 2012 20:57:13 +0100 (CET)
Subject: [pypy-commit] pypy merge-2.7.2: Implement CPython issue5057: do not
 const-fold a unicode.__getitem__
Message-ID: <20120122195713.336F4821FA@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: merge-2.7.2
Changeset: r51662:693b08144e00
Date: 2012-01-22 20:24 +0100
http://bitbucket.org/pypy/pypy/changeset/693b08144e00/

Log:	Implement CPython issue5057: do not const-fold a unicode.__getitem__
	operation which returns a non-BMP character, this produces .pyc
	files which depends on the unicode width

diff --git a/pypy/interpreter/astcompiler/optimize.py b/pypy/interpreter/astcompiler/optimize.py
--- a/pypy/interpreter/astcompiler/optimize.py
+++ b/pypy/interpreter/astcompiler/optimize.py
@@ -5,6 +5,7 @@
 from pypy.tool import stdlib_opcode as ops
 from pypy.interpreter.error import OperationError
 from pypy.rlib.unroll import unrolling_iterable
+from pypy.rlib.runicode import MAXUNICODE
 
 
 def optimize_ast(space, tree, compile_info):
@@ -289,8 +290,30 @@
                 w_idx = subs.slice.as_constant()
                 if w_idx is not None:
                     try:
-                        return ast.Const(self.space.getitem(w_obj, w_idx), subs.lineno, subs.col_offset)
+                        w_const = self.space.getitem(w_obj, w_idx)
                     except OperationError:
-                        # Let exceptions propgate at runtime.
-                        pass
+                        # Let exceptions propagate at runtime.
+                        return subs
+
+                    # CPython issue5057: if v is unicode, there might
+                    # be differences between wide and narrow builds in
+                    # cases like u'\U00012345'[0].
+                    # Wide builds will return a non-BMP char, whereas
+                    # narrow builds will return a surrogate.  In both
+                    # the cases skip the optimization in order to
+                    # produce compatible pycs.
+                    if (self.space.isinstance_w(w_obj, self.space.w_unicode)
+                        and 
+                        self.space.isinstance_w(w_const, self.space.w_unicode)):
+                        unistr = self.space.unicode_w(w_const)
+                        if len(unistr) == 1:
+                            ch = ord(unistr[0])
+                        else:
+                            ch = 0
+                        if (ch > 0xFFFF or
+                            (MAXUNICODE == 0xFFFF and 0xD800 <= ch <= OxDFFFF)):
+                            return subs
+
+                    return ast.Const(w_const, subs.lineno, subs.col_offset)
+
         return subs
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -838,6 +838,30 @@
         # Just checking this doesn't crash out
         self.count_instructions(source)
 
+    def test_const_fold_unicode_subscr(self):
+        source = """def f():
+        return u"abc"[0]
+        """
+        counts = self.count_instructions(source)
+        assert counts == {ops.LOAD_CONST: 1, ops.RETURN_VALUE: 1}
+
+        # getitem outside of the BMP should not be optimized
+        source = """def f():
+        return u"\U00012345"[0]
+        """
+        counts = self.count_instructions(source)
+        assert counts == {ops.LOAD_CONST: 2, ops.BINARY_SUBSCR: 1,
+                          ops.RETURN_VALUE: 1}
+
+        # getslice is not yet optimized.
+        # Still, check a case which yields the empty string.
+        source = """def f():
+        return u"abc"[:0]
+        """
+        counts = self.count_instructions(source)
+        assert counts == {ops.LOAD_CONST: 2, ops.SLICE+2: 1,
+                          ops.RETURN_VALUE: 1}
+
     def test_remove_dead_code(self):
         source = """def f(x):
             return 5

From noreply at buildbot.pypy.org  Sun Jan 22 20:57:14 2012
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sun, 22 Jan 2012 20:57:14 +0100 (CET)
Subject: [pypy-commit] pypy merge-2.7.2: hg merge default
Message-ID: <20120122195714.C0E18821FA@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: merge-2.7.2
Changeset: r51663:6e17d519d090
Date: 2012-01-22 20:42 +0100
http://bitbucket.org/pypy/pypy/changeset/6e17d519d090/

Log:	hg merge default

diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -2,6 +2,9 @@
 *.py[co]
 *~
 .*.swp
+.idea
+.project
+.pydevproject
 
 syntax: regexp
 ^testresult$
diff --git a/lib_pypy/datetime.py b/lib_pypy/datetime.py
--- a/lib_pypy/datetime.py
+++ b/lib_pypy/datetime.py
@@ -1519,12 +1519,17 @@
     @classmethod
     def utcfromtimestamp(cls, t):
         "Construct a UTC datetime from a POSIX timestamp (like time.time())."
-        if 1 - (t % 1.0) < 0.0000005:
-            t = float(int(t)) + 1
-        if t < 0:
-            t -= 1
+        t, frac = divmod(t, 1.0)
+        us = round(frac * 1e6)
+
+        # If timestamp is less than one microsecond smaller than a
+        # full second, us can be rounded up to 1000000.  In this case,
+        # roll over to seconds, otherwise, ValueError is raised
+        # by the constructor.
+        if us == 1000000:
+            t += 1
+            us = 0
         y, m, d, hh, mm, ss, weekday, jday, dst = _time.gmtime(t)
-        us = int((t % 1.0) * 1000000)
         ss = min(ss, 59)    # clamp out leap seconds if the platform has them
         return cls(y, m, d, hh, mm, ss, us)
 
diff --git a/lib_pypy/numpypy/__init__.py b/lib_pypy/numpypy/__init__.py
--- a/lib_pypy/numpypy/__init__.py
+++ b/lib_pypy/numpypy/__init__.py
@@ -1,2 +1,2 @@
 from _numpypy import *
-from .fromnumeric import *
+from .core import *
diff --git a/lib_pypy/numpypy/core/__init__.py b/lib_pypy/numpypy/core/__init__.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/numpypy/core/__init__.py
@@ -0,0 +1,1 @@
+from .fromnumeric import *
diff --git a/lib_pypy/numpypy/fromnumeric.py b/lib_pypy/numpypy/core/fromnumeric.py
rename from lib_pypy/numpypy/fromnumeric.py
rename to lib_pypy/numpypy/core/fromnumeric.py
--- a/lib_pypy/numpypy/fromnumeric.py
+++ b/lib_pypy/numpypy/core/fromnumeric.py
@@ -85,7 +85,7 @@
     array([4, 3, 6])
 
     """
-    raise NotImplemented('Waiting on interp level method')
+    raise NotImplementedError('Waiting on interp level method')
 
 
 # not deprecated --- copy if necessary, view otherwise
@@ -149,6 +149,7 @@
            [5, 6]])
 
     """
+    assert order == 'C'
     if not hasattr(a, 'reshape'):
        a = numpypy.array(a)
     return a.reshape(newshape)
@@ -273,7 +274,7 @@
             [-1, -2, -3, -4, -5]]])
 
     """
-    raise NotImplemented('Waiting on interp level method')
+    raise NotImplementedError('Waiting on interp level method')
 
 
 def repeat(a, repeats, axis=None):
@@ -315,7 +316,7 @@
            [3, 4]])
 
     """
-    raise NotImplemented('Waiting on interp level method')
+    raise NotImplementedError('Waiting on interp level method')
 
 
 def put(a, ind, v, mode='raise'):
@@ -366,7 +367,7 @@
     array([ 0,  1,  2,  3, -5])
 
     """
-    raise NotImplemented('Waiting on interp level method')
+    raise NotImplementedError('Waiting on interp level method')
 
 
 def swapaxes(a, axis1, axis2):
@@ -410,7 +411,7 @@
             [3, 7]]])
 
     """
-    raise NotImplemented('Waiting on interp level method')
+    raise NotImplementedError('Waiting on interp level method')
 
 
 def transpose(a, axes=None):
@@ -451,8 +452,11 @@
     (2, 1, 3)
 
     """
-    raise NotImplemented('Waiting on interp level method')
-
+    if axes is not None:
+        raise NotImplementedError('No "axes" arg yet.')
+    if not hasattr(a, 'T'):
+        a = numpypy.array(a)
+    return a.T
 
 def sort(a, axis=-1, kind='quicksort', order=None):
     """
@@ -553,7 +557,7 @@
           dtype=[('name', '|S10'), ('height', '= self.size
 
@@ -157,6 +177,8 @@
                 offset += self.strides[i]
                 break
             else:
+                if i == self.dim:
+                    first_line = True
                 indices[i] = 0
                 offset -= self.backstrides[i]
         else:
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -2,14 +2,15 @@
 from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.interpreter.gateway import interp2app, NoneNotWrapped
 from pypy.interpreter.typedef import TypeDef, GetSetProperty
-from pypy.module.micronumpy import interp_ufuncs, interp_dtype, signature
+from pypy.module.micronumpy import interp_ufuncs, interp_dtype, signature,\
+     interp_boxes
 from pypy.module.micronumpy.strides import calculate_slice_strides
 from pypy.rlib import jit
 from pypy.rpython.lltypesystem import lltype, rffi
 from pypy.tool.sourcetools import func_with_new_name
 from pypy.rlib.rstring import StringBuilder
 from pypy.module.micronumpy.interp_iter import ArrayIterator, OneDimIterator,\
-     SkipLastAxisIterator
+     SkipLastAxisIterator, Chunk, ViewIterator
 
 numpy_driver = jit.JitDriver(
     greens=['shapelen', 'sig'],
@@ -39,7 +40,24 @@
     get_printable_location=signature.new_printable_location('slice'),
     name='numpy_slice',
 )
-
+count_driver = jit.JitDriver(
+    greens=['shapelen'],
+    virtualizables=['frame'],
+    reds=['s', 'frame', 'iter', 'arr'],
+    name='numpy_count'
+)
+filter_driver = jit.JitDriver(
+    greens=['shapelen', 'sig'],
+    virtualizables=['frame'],
+    reds=['concr', 'argi', 'ri', 'frame', 'v', 'res', 'self'],
+    name='numpy_filter',
+)
+filter_set_driver = jit.JitDriver(
+    greens=['shapelen', 'sig'],
+    virtualizables=['frame'],
+    reds=['idx', 'idxi', 'frame', 'arr'],
+    name='numpy_filterset',
+)
 
 def _find_shape_and_elems(space, w_iterable):
     shape = [space.len_w(w_iterable)]
@@ -157,9 +175,6 @@
 # (meaning that the realignment of elements crosses from one step into another)
 # return None so that the caller can raise an exception.
 def calc_new_strides(new_shape, old_shape, old_strides):
-    # Return the proper strides for new_shape, or None if the mapping crosses
-    # stepping boundaries
-
     # Assumes that prod(old_shape) == prod(new_shape), len(old_shape) > 1, and
     # len(new_shape) > 0
     steps = []
@@ -167,6 +182,7 @@
     oldI = 0
     new_strides = []
     if old_strides[0] < old_strides[-1]:
+        #Start at old_shape[0], old_stides[0]
         for i in range(len(old_shape)):
             steps.append(old_strides[i] / last_step)
             last_step *= old_shape[i]
@@ -184,10 +200,11 @@
             if n_new_elems_used == n_old_elems_to_use:
                 oldI += 1
                 if oldI >= len(old_shape):
-                    break
+                    continue
                 cur_step = steps[oldI]
                 n_old_elems_to_use *= old_shape[oldI]
     else:
+        #Start at old_shape[-1], old_strides[-1]
         for i in range(len(old_shape) - 1, -1, -1):
             steps.insert(0, old_strides[i] / last_step)
             last_step *= old_shape[i]
@@ -207,7 +224,7 @@
             if n_new_elems_used == n_old_elems_to_use:
                 oldI -= 1
                 if oldI < -len(old_shape):
-                    break
+                    continue
                 cur_step = steps[oldI]
                 n_old_elems_to_use *= old_shape[oldI]
     return new_strides
@@ -271,6 +288,9 @@
     descr_gt = _binop_impl("greater")
     descr_ge = _binop_impl("greater_equal")
 
+    descr_and = _binop_impl("bitwise_and")
+    descr_or = _binop_impl("bitwise_or")
+
     def _binop_right_impl(ufunc_name):
         def impl(self, space, w_other):
             w_other = scalar_w(space,
@@ -288,11 +308,11 @@
     descr_rmod = _binop_right_impl("mod")
 
     def _reduce_ufunc_impl(ufunc_name, promote_to_largest=False):
-        def impl(self, space, w_dim=None):
-            if space.is_w(w_dim, space.w_None):
-                w_dim = space.wrap(-1)
+        def impl(self, space, w_axis=None):
+            if space.is_w(w_axis, space.w_None):
+                w_axis = space.wrap(-1)
             return getattr(interp_ufuncs.get(space), ufunc_name).reduce(space,
-                                        self, True, promote_to_largest, w_dim)
+                                        self, True, promote_to_largest, w_axis)
         return func_with_new_name(impl, "reduce_%s_impl" % ufunc_name)
 
     descr_sum = _reduce_ufunc_impl("add")
@@ -410,9 +430,15 @@
     def descr_copy(self, space):
         return self.copy(space)
 
+    def descr_flatten(self, space):
+        return self.flatten(space)
+
     def copy(self, space):
         return self.get_concrete().copy(space)
 
+    def flatten(self, space):
+        return self.get_concrete().flatten(space)
+
     def descr_len(self, space):
         if len(self.shape):
             return space.wrap(self.shape[0])
@@ -480,11 +506,69 @@
     def _prepare_slice_args(self, space, w_idx):
         if (space.isinstance_w(w_idx, space.w_int) or
             space.isinstance_w(w_idx, space.w_slice)):
-            return [space.decode_index4(w_idx, self.shape[0])]
-        return [space.decode_index4(w_item, self.shape[i]) for i, w_item in
+            return [Chunk(*space.decode_index4(w_idx, self.shape[0]))]
+        return [Chunk(*space.decode_index4(w_item, self.shape[i])) for i, w_item in
                 enumerate(space.fixedview(w_idx))]
 
+    def count_all_true(self, arr):
+        sig = arr.find_sig()
+        frame = sig.create_frame(self)
+        shapelen = len(arr.shape)
+        s = 0
+        iter = None
+        while not frame.done():
+            count_driver.jit_merge_point(arr=arr, frame=frame, iter=iter, s=s,
+                                         shapelen=shapelen)
+            iter = frame.get_final_iter()
+            s += arr.dtype.getitem_bool(arr.storage, iter.offset)
+            frame.next(shapelen)
+        return s
+
+    def getitem_filter(self, space, arr):
+        concr = arr.get_concrete()
+        size = self.count_all_true(concr)
+        res = W_NDimArray(size, [size], self.find_dtype())
+        ri = ArrayIterator(size)
+        shapelen = len(self.shape)
+        argi = concr.create_iter()
+        sig = self.find_sig()
+        frame = sig.create_frame(self)
+        v = None
+        while not frame.done():
+            filter_driver.jit_merge_point(concr=concr, argi=argi, ri=ri,
+                                          frame=frame, v=v, res=res, sig=sig,
+                                          shapelen=shapelen, self=self)
+            if concr.dtype.getitem_bool(concr.storage, argi.offset):
+                v = sig.eval(frame, self)
+                res.setitem(ri.offset, v)
+                ri = ri.next(1)
+            else:
+                ri = ri.next_no_increase(1)
+            argi = argi.next(shapelen)
+            frame.next(shapelen)
+        return res
+
+    def setitem_filter(self, space, idx, val):
+        size = self.count_all_true(idx)
+        arr = SliceArray([size], self.dtype, self, val)
+        sig = arr.find_sig()
+        shapelen = len(self.shape)
+        frame = sig.create_frame(arr)
+        idxi = idx.create_iter()
+        while not frame.done():
+            filter_set_driver.jit_merge_point(idx=idx, idxi=idxi, sig=sig,
+                                              frame=frame, arr=arr,
+                                              shapelen=shapelen)
+            if idx.dtype.getitem_bool(idx.storage, idxi.offset):
+                sig.eval(frame, arr)
+                frame.next_from_second(1)
+            frame.next_first(shapelen)
+            idxi = idxi.next(shapelen)
+
     def descr_getitem(self, space, w_idx):
+        if (isinstance(w_idx, BaseArray) and w_idx.shape == self.shape and
+            w_idx.find_dtype().is_bool_type()):
+            return self.getitem_filter(space, w_idx)
         if self._single_item_result(space, w_idx):
             concrete = self.get_concrete()
             item = concrete._index_of_single_item(space, w_idx)
@@ -494,6 +578,11 @@
 
     def descr_setitem(self, space, w_idx, w_value):
         self.invalidated()
+        if (isinstance(w_idx, BaseArray) and w_idx.shape == self.shape and
+            w_idx.find_dtype().is_bool_type()):
+            return self.get_concrete().setitem_filter(space,
+                                                      w_idx.get_concrete(),
+                                             convert_to_array(space, w_value))
         if self._single_item_result(space, w_idx):
             concrete = self.get_concrete()
             item = concrete._index_of_single_item(space, w_idx)
@@ -510,9 +599,8 @@
     def create_slice(self, chunks):
         shape = []
         i = -1
-        for i, (start_, stop, step, lgt) in enumerate(chunks):
-            if step != 0:
-                shape.append(lgt)
+        for i, chunk in enumerate(chunks):
+            chunk.extend_shape(shape)
         s = i + 1
         assert s >= 0
         shape += self.shape[s:]
@@ -570,19 +658,19 @@
             )
         return w_result
 
-    def descr_mean(self, space, w_dim=None):
-        if space.is_w(w_dim, space.w_None):
-            w_dim = space.wrap(-1)
+    def descr_mean(self, space, w_axis=None):
+        if space.is_w(w_axis, space.w_None):
+            w_axis = space.wrap(-1)
             w_denom = space.wrap(self.size)
         else:
-            dim = space.int_w(w_dim)
+            dim = space.int_w(w_axis)
             w_denom = space.wrap(self.shape[dim])
-        return space.div(self.descr_sum_promote(space, w_dim), w_denom)
+        return space.div(self.descr_sum_promote(space, w_axis), w_denom)
 
     def descr_var(self, space):
         # var = mean((values - mean(values)) ** 2)
         w_res = self.descr_sub(space, self.descr_mean(space, space.w_None))
-        assert isinstance(w_res, BaseArray) 
+        assert isinstance(w_res, BaseArray)
         w_res = w_res.descr_pow(space, space.wrap(2))
         assert isinstance(w_res, BaseArray)
         return w_res.descr_mean(space, space.w_None)
@@ -591,6 +679,10 @@
         # std(v) = sqrt(var(v))
         return interp_ufuncs.get(space).sqrt.call(space, [self.descr_var(space)])
 
+    def descr_fill(self, space, w_value):
+        concr = self.get_concrete_or_scalar()
+        concr.fill(space, w_value)
+
     def descr_nonzero(self, space):
         if self.size > 1:
             raise OperationError(space.w_ValueError, space.wrap(
@@ -683,6 +775,14 @@
     def copy(self, space):
         return Scalar(self.dtype, self.value)
 
+    def flatten(self, space):
+        array = W_NDimArray(self.size, [self.size], self.dtype)
+        array.setitem(0, self.value)
+        return array
+
+    def fill(self, space, w_value):
+        self.value = self.dtype.coerce(space, w_value)
+
     def create_sig(self):
         return signature.ScalarSignature(self.dtype)
 
@@ -718,8 +818,7 @@
                                          frame=frame,
                                          ri=ri,
                                          self=self, result=result)
-            result.dtype.setitem(result.storage, ri.offset,
-                                 sig.eval(frame, self))
+            result.setitem(ri.offset, sig.eval(frame, self))
             frame.next(shapelen)
             ri = ri.next(shapelen)
         return result
@@ -789,7 +888,7 @@
     Intermediate class for performing binary operations.
     """
     _immutable_fields_ = ['left', 'right']
-    
+
     def __init__(self, ufunc, name, shape, calc_dtype, res_dtype, left, right):
         VirtualArray.__init__(self, name, shape, res_dtype)
         self.ufunc = ufunc
@@ -826,14 +925,15 @@
                                self.left.create_sig(), self.right.create_sig())
 
 class SliceArray(Call2):
-    def __init__(self, shape, dtype, left, right):
+    def __init__(self, shape, dtype, left, right, no_broadcast=False):
+        self.no_broadcast = no_broadcast
         Call2.__init__(self, None, 'sliceloop', shape, dtype, dtype, left,
                        right)
-    
+
     def create_sig(self):
         lsig = self.left.create_sig()
         rsig = self.right.create_sig()
-        if self.shape != self.right.shape:
+        if not self.no_broadcast and self.shape != self.right.shape:
             return signature.SliceloopBroadcastSignature(self.ufunc,
                                                          self.name,
                                                          self.calc_dtype,
@@ -848,7 +948,7 @@
     when we'll make AxisReduce lazy
     """
     _immutable_fields_ = ['left', 'right']
-    
+
     def __init__(self, ufunc, name, shape, dtype, left, right, dim):
         Call2.__init__(self, ufunc, name, shape, dtype, dtype,
                        left, right)
@@ -920,14 +1020,14 @@
         if size < 1:
             builder.append('[]')
             return
-        elif size == 1:
-            builder.append(dtype.itemtype.str_format(self.getitem(0)))
-            return
         if size > 1000:
             # Once this goes True it does not go back to False for recursive
             # calls
             use_ellipsis = True
         ndims = len(self.shape)
+        if ndims == 0:
+            builder.append(dtype.itemtype.str_format(self.getitem(0)))
+            return
         i = 0
         builder.append('[')
         if ndims > 1:
@@ -939,7 +1039,7 @@
                             builder.append('\n' + indent)
                         else:
                             builder.append(indent)
-                    view = self.create_slice([(i, 0, 0, 1)]).get_concrete()
+                    view = self.create_slice([Chunk(i, 0, 0, 1)]).get_concrete()
                     view.to_str(space, comma, builder, indent=indent + ' ',
                                                     use_ellipsis=use_ellipsis)
                 if i < self.shape[0] - 1:
@@ -956,7 +1056,7 @@
                         builder.append(indent)
                 # create_slice requires len(chunks) > 1 in order to reduce
                 # shape
-                view = self.create_slice([(i, 0, 0, 1)]).get_concrete()
+                view = self.create_slice([Chunk(i, 0, 0, 1)]).get_concrete()
                 view.to_str(space, comma, builder, indent=indent + ' ',
                                                     use_ellipsis=use_ellipsis)
                 i += 1
@@ -1062,6 +1162,18 @@
         array.setslice(space, self)
         return array
 
+    def flatten(self, space):
+        array = W_NDimArray(self.size, [self.size], self.dtype, self.order)
+        if self.supports_fast_slicing():
+            array._fast_setslice(space, self)
+        else:
+            arr = SliceArray(array.shape, array.dtype, array, self, no_broadcast=True)
+            array._sliceloop(arr)
+        return array
+
+    def fill(self, space, w_value):
+        self.setslice(space, scalar_w(space, self.dtype, w_value))
+
 
 class ViewArray(ConcreteArray):
     def create_sig(self):
@@ -1082,6 +1194,10 @@
                                parent)
         self.start = start
 
+    def create_iter(self):
+        return ViewIterator(self.start, self.strides, self.backstrides,
+                            self.shape)
+
     def setshape(self, space, new_shape):
         if len(self.shape) < 1:
             return
@@ -1128,6 +1244,9 @@
         self.shape = new_shape
         self.calc_strides(new_shape)
 
+    def create_iter(self):
+        return ArrayIterator(self.size)
+
     def create_sig(self):
         return signature.ArraySignature(self.dtype)
 
@@ -1182,6 +1301,7 @@
     arr = W_NDimArray(size, shape[:], dtype=dtype, order=order)
     shapelen = len(shape)
     arr_iter = ArrayIterator(arr.size)
+    # XXX we might want to have a jitdriver here
     for i in range(len(elems_w)):
         w_elem = elems_w[i]
         dtype.setitem(arr.storage, arr_iter.offset,
@@ -1248,6 +1368,9 @@
     __gt__ = interp2app(BaseArray.descr_gt),
     __ge__ = interp2app(BaseArray.descr_ge),
 
+    __and__ = interp2app(BaseArray.descr_and),
+    __or__ = interp2app(BaseArray.descr_or),
+
     __repr__ = interp2app(BaseArray.descr_repr),
     __str__ = interp2app(BaseArray.descr_str),
     __array_interface__ = GetSetProperty(BaseArray.descr_array_iface),
@@ -1274,7 +1397,10 @@
     var = interp2app(BaseArray.descr_var),
     std = interp2app(BaseArray.descr_std),
 
+    fill = interp2app(BaseArray.descr_fill),
+
     copy = interp2app(BaseArray.descr_copy),
+    flatten = interp2app(BaseArray.descr_flatten),
     reshape = interp2app(BaseArray.descr_reshape),
     tolist = interp2app(BaseArray.descr_tolist),
 )
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -249,15 +249,16 @@
 
 
 class W_Ufunc2(W_Ufunc):
-    _immutable_fields_ = ["comparison_func", "func", "name"]
+    _immutable_fields_ = ["comparison_func", "func", "name", "int_only"]
     argcount = 2
 
     def __init__(self, func, name, promote_to_float=False, promote_bools=False,
-        identity=None, comparison_func=False):
+        identity=None, comparison_func=False, int_only=False):
 
         W_Ufunc.__init__(self, name, promote_to_float, promote_bools, identity)
         self.func = func
         self.comparison_func = comparison_func
+        self.int_only = int_only
 
     def call(self, space, args_w):
         from pypy.module.micronumpy.interp_numarray import (Call2,
@@ -268,6 +269,7 @@
         w_rhs = convert_to_array(space, w_rhs)
         calc_dtype = find_binop_result_dtype(space,
             w_lhs.find_dtype(), w_rhs.find_dtype(),
+            int_only=self.int_only,
             promote_to_float=self.promote_to_float,
             promote_bools=self.promote_bools,
         )
@@ -304,10 +306,12 @@
 
 
 def find_binop_result_dtype(space, dt1, dt2, promote_to_float=False,
-    promote_bools=False):
+    promote_bools=False, int_only=False):
     # dt1.num should be <= dt2.num
     if dt1.num > dt2.num:
         dt1, dt2 = dt2, dt1
+    if int_only and (not dt1.is_int_type() or not dt2.is_int_type()):
+        raise OperationError(space.w_TypeError, space.wrap("Unsupported types"))
     # Some operations promote op(bool, bool) to return int8, rather than bool
     if promote_bools and (dt1.kind == dt2.kind == interp_dtype.BOOLLTR):
         return interp_dtype.get_dtype_cache(space).w_int8dtype
@@ -425,6 +429,10 @@
             ("add", "add", 2, {"identity": 0}),
             ("subtract", "sub", 2),
             ("multiply", "mul", 2, {"identity": 1}),
+            ("bitwise_and", "bitwise_and", 2, {"identity": 1,
+                                               'int_only': True}),
+            ("bitwise_or", "bitwise_or", 2, {"identity": 0,
+                                             'int_only': True}),
             ("divide", "div", 2, {"promote_bools": True}),
             ("mod", "mod", 2, {"promote_bools": True}),
             ("power", "pow", 2, {"promote_bools": True}),
@@ -449,6 +457,7 @@
 
             ("fabs", "fabs", 1, {"promote_to_float": True}),
             ("floor", "floor", 1, {"promote_to_float": True}),
+            ("ceil", "ceil", 1, {"promote_to_float": True}),
             ("exp", "exp", 1, {"promote_to_float": True}),
 
             ('sqrt', 'sqrt', 1, {'promote_to_float': True}),
@@ -475,7 +484,7 @@
         extra_kwargs["identity"] = identity
 
         func = ufunc_dtype_caller(space, ufunc_name, op_name, argcount,
-            comparison_func=extra_kwargs.get("comparison_func", False)
+            comparison_func=extra_kwargs.get("comparison_func", False),
         )
         if argcount == 1:
             ufunc = W_Ufunc1(func, ufunc_name, **extra_kwargs)
diff --git a/pypy/module/micronumpy/signature.py b/pypy/module/micronumpy/signature.py
--- a/pypy/module/micronumpy/signature.py
+++ b/pypy/module/micronumpy/signature.py
@@ -82,6 +82,16 @@
         for i in range(len(self.iterators)):
             self.iterators[i] = self.iterators[i].next(shapelen)
 
+    @unroll_safe
+    def next_from_second(self, shapelen):
+        """ Don't increase the first iterator
+        """
+        for i in range(1, len(self.iterators)):
+            self.iterators[i] = self.iterators[i].next(shapelen)
+
+    def next_first(self, shapelen):
+        self.iterators[0] = self.iterators[0].next(shapelen)
+
     def get_final_iter(self):
         final_iter = promote(self.final_iter)
         if final_iter < 0:
diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py
--- a/pypy/module/micronumpy/strides.py
+++ b/pypy/module/micronumpy/strides.py
@@ -10,12 +10,12 @@
     rstart = start
     rshape = []
     i = -1
-    for i, (start_, stop, step, lgt) in enumerate(chunks):
-        if step != 0:
-            rstrides.append(strides[i] * step)
-            rbackstrides.append(strides[i] * (lgt - 1) * step)
-            rshape.append(lgt)
-        rstart += strides[i] * start_
+    for i, chunk in enumerate(chunks):
+        if chunk.step != 0:
+            rstrides.append(strides[i] * chunk.step)
+            rbackstrides.append(strides[i] * (chunk.lgt - 1) * chunk.step)
+            rshape.append(chunk.lgt)
+        rstart += strides[i] * chunk.start
     # add a reminder
     s = i + 1
     assert s >= 0
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -166,14 +166,11 @@
         # You can't subclass dtype
         raises(TypeError, type, "Foo", (dtype,), {})
 
-    def test_new(self):
-        import _numpypy as np
-        assert np.int_(4) == 4
-        assert np.float_(3.4) == 3.4
+    def test_aliases(self):
+        from _numpypy import dtype
 
-    def test_pow(self):
-        from _numpypy import int_
-        assert int_(4) ** 2 == 16
+        assert dtype("float") is dtype(float)
+
 
 class AppTestTypes(BaseNumpyAppTest):
     def test_abstract_types(self):
@@ -189,6 +186,15 @@
         raises(TypeError, numpy.floating, 0)
         raises(TypeError, numpy.inexact, 0)
 
+    def test_new(self):
+        import _numpypy as np
+        assert np.int_(4) == 4
+        assert np.float_(3.4) == 3.4
+
+    def test_pow(self):
+        from _numpypy import int_
+        assert int_(4) ** 2 == 16
+
     def test_bool(self):
         import _numpypy as numpy
 
@@ -318,7 +324,7 @@
         else:
             raises(OverflowError, numpy.int64, 9223372036854775807)
             raises(OverflowError, numpy.int64, '9223372036854775807')
-        
+
         raises(OverflowError, numpy.int64, 9223372036854775808)
         raises(OverflowError, numpy.int64, '9223372036854775808')
 
diff --git a/pypy/module/micronumpy/test/test_module.py b/pypy/module/micronumpy/test/test_module.py
--- a/pypy/module/micronumpy/test/test_module.py
+++ b/pypy/module/micronumpy/test/test_module.py
@@ -2,16 +2,11 @@
 
 
 class AppTestNumPyModule(BaseNumpyAppTest):
-    def test_mean(self):
-        from _numpypy import array, mean
-        assert mean(array(range(5))) == 2.0
-        assert mean(range(5)) == 2.0
-
     def test_average(self):
         from _numpypy import array, average
         assert average(range(10)) == 4.5
         assert average(array(range(10))) == 4.5
-        
+
     def test_sum(self):
         from _numpypy import array, sum
         assert sum(range(10)) == 45
@@ -21,7 +16,7 @@
         from _numpypy import array, min
         assert min(range(10)) == 0
         assert min(array(range(10))) == 0
-        
+
     def test_max(self):
         from _numpypy import array, max
         assert max(range(10)) == 9
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -2,6 +2,7 @@
 import py
 from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
 from pypy.module.micronumpy.interp_numarray import W_NDimArray, shape_agreement
+from pypy.module.micronumpy.interp_iter import Chunk
 from pypy.module.micronumpy import signature
 from pypy.interpreter.error import OperationError
 from pypy.conftest import gettestobjspace
@@ -37,53 +38,54 @@
 
     def test_create_slice_f(self):
         a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'F')
-        s = a.create_slice([(3, 0, 0, 1)])
+        s = a.create_slice([Chunk(3, 0, 0, 1)])
         assert s.start == 3
         assert s.strides == [10, 50]
         assert s.backstrides == [40, 100]
-        s = a.create_slice([(1, 9, 2, 4)])
+        s = a.create_slice([Chunk(1, 9, 2, 4)])
         assert s.start == 1
         assert s.strides == [2, 10, 50]
         assert s.backstrides == [6, 40, 100]
-        s = a.create_slice([(1, 5, 3, 2), (1, 2, 1, 1), (1, 0, 0, 1)])
+        s = a.create_slice([Chunk(1, 5, 3, 2), Chunk(1, 2, 1, 1), Chunk(1, 0, 0, 1)])
         assert s.shape == [2, 1]
         assert s.strides == [3, 10]
         assert s.backstrides == [3, 0]
-        s = a.create_slice([(0, 10, 1, 10), (2, 0, 0, 1)])
+        s = a.create_slice([Chunk(0, 10, 1, 10), Chunk(2, 0, 0, 1)])
         assert s.start == 20
         assert s.shape == [10, 3]
 
     def test_create_slice_c(self):
         a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'C')
-        s = a.create_slice([(3, 0, 0, 1)])
+        s = a.create_slice([Chunk(3, 0, 0, 1)])
         assert s.start == 45
         assert s.strides == [3, 1]
         assert s.backstrides == [12, 2]
-        s = a.create_slice([(1, 9, 2, 4)])
+        s = a.create_slice([Chunk(1, 9, 2, 4)])
         assert s.start == 15
         assert s.strides == [30, 3, 1]
         assert s.backstrides == [90, 12, 2]
-        s = a.create_slice([(1, 5, 3, 2), (1, 2, 1, 1), (1, 0, 0, 1)])
+        s = a.create_slice([Chunk(1, 5, 3, 2), Chunk(1, 2, 1, 1),
+                            Chunk(1, 0, 0, 1)])
         assert s.start == 19
         assert s.shape == [2, 1]
         assert s.strides == [45, 3]
         assert s.backstrides == [45, 0]
-        s = a.create_slice([(0, 10, 1, 10), (2, 0, 0, 1)])
+        s = a.create_slice([Chunk(0, 10, 1, 10), Chunk(2, 0, 0, 1)])
         assert s.start == 6
         assert s.shape == [10, 3]
 
     def test_slice_of_slice_f(self):
         a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'F')
-        s = a.create_slice([(5, 0, 0, 1)])
+        s = a.create_slice([Chunk(5, 0, 0, 1)])
         assert s.start == 5
-        s2 = s.create_slice([(3, 0, 0, 1)])
+        s2 = s.create_slice([Chunk(3, 0, 0, 1)])
         assert s2.shape == [3]
         assert s2.strides == [50]
         assert s2.parent is a
         assert s2.backstrides == [100]
         assert s2.start == 35
-        s = a.create_slice([(1, 5, 3, 2)])
-        s2 = s.create_slice([(0, 2, 1, 2), (2, 0, 0, 1)])
+        s = a.create_slice([Chunk(1, 5, 3, 2)])
+        s2 = s.create_slice([Chunk(0, 2, 1, 2), Chunk(2, 0, 0, 1)])
         assert s2.shape == [2, 3]
         assert s2.strides == [3, 50]
         assert s2.backstrides == [3, 100]
@@ -91,16 +93,16 @@
 
     def test_slice_of_slice_c(self):
         a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), order='C')
-        s = a.create_slice([(5, 0, 0, 1)])
+        s = a.create_slice([Chunk(5, 0, 0, 1)])
         assert s.start == 15 * 5
-        s2 = s.create_slice([(3, 0, 0, 1)])
+        s2 = s.create_slice([Chunk(3, 0, 0, 1)])
         assert s2.shape == [3]
         assert s2.strides == [1]
         assert s2.parent is a
         assert s2.backstrides == [2]
         assert s2.start == 5 * 15 + 3 * 3
-        s = a.create_slice([(1, 5, 3, 2)])
-        s2 = s.create_slice([(0, 2, 1, 2), (2, 0, 0, 1)])
+        s = a.create_slice([Chunk(1, 5, 3, 2)])
+        s2 = s.create_slice([Chunk(0, 2, 1, 2), Chunk(2, 0, 0, 1)])
         assert s2.shape == [2, 3]
         assert s2.strides == [45, 1]
         assert s2.backstrides == [45, 2]
@@ -108,14 +110,14 @@
 
     def test_negative_step_f(self):
         a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'F')
-        s = a.create_slice([(9, -1, -2, 5)])
+        s = a.create_slice([Chunk(9, -1, -2, 5)])
         assert s.start == 9
         assert s.strides == [-2, 10, 50]
         assert s.backstrides == [-8, 40, 100]
 
     def test_negative_step_c(self):
         a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), order='C')
-        s = a.create_slice([(9, -1, -2, 5)])
+        s = a.create_slice([Chunk(9, -1, -2, 5)])
         assert s.start == 135
         assert s.strides == [-30, 3, 1]
         assert s.backstrides == [-120, 12, 2]
@@ -124,7 +126,7 @@
         a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'F')
         r = a._index_of_single_item(self.space, self.newtuple(1, 2, 2))
         assert r == 1 + 2 * 10 + 2 * 50
-        s = a.create_slice([(0, 10, 1, 10), (2, 0, 0, 1)])
+        s = a.create_slice([Chunk(0, 10, 1, 10), Chunk(2, 0, 0, 1)])
         r = s._index_of_single_item(self.space, self.newtuple(1, 0))
         assert r == a._index_of_single_item(self.space, self.newtuple(1, 2, 0))
         r = s._index_of_single_item(self.space, self.newtuple(1, 1))
@@ -134,7 +136,7 @@
         a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'C')
         r = a._index_of_single_item(self.space, self.newtuple(1, 2, 2))
         assert r == 1 * 3 * 5 + 2 * 3 + 2
-        s = a.create_slice([(0, 10, 1, 10), (2, 0, 0, 1)])
+        s = a.create_slice([Chunk(0, 10, 1, 10), Chunk(2, 0, 0, 1)])
         r = s._index_of_single_item(self.space, self.newtuple(1, 0))
         assert r == a._index_of_single_item(self.space, self.newtuple(1, 2, 0))
         r = s._index_of_single_item(self.space, self.newtuple(1, 1))
@@ -157,6 +159,8 @@
         assert calc_new_strides([2, 3, 4], [8, 3], [1, 16]) is None
         assert calc_new_strides([24], [2, 4, 3], [48, 6, 1]) is None
         assert calc_new_strides([24], [2, 4, 3], [24, 6, 2]) == [2]
+        assert calc_new_strides([105, 1], [3, 5, 7], [35, 7, 1]) == [1, 1]
+        assert calc_new_strides([1, 105], [3, 5, 7], [35, 7, 1]) == [105, 1]
 
 
 class AppTestNumArray(BaseNumpyAppTest):
@@ -724,15 +728,17 @@
         assert d[1] == 12
 
     def test_mean(self):
-        from _numpypy import array, mean
+        from _numpypy import array, arange
         a = array(range(5))
         assert a.mean() == 2.0
         assert a[:4].mean() == 1.5
         a = array(range(105)).reshape(3, 5, 7)
-        b = mean(a, axis=0)
-        b[0,0]==35.
+        b = a.mean(axis=0)
+        b[0, 0]==35.
+        assert a.mean(axis=0)[0, 0] == 35
         assert (b == array(range(35, 70), dtype=float).reshape(5, 7)).all()
-        assert (mean(a, 2) == array(range(0, 15), dtype=float).reshape(3, 5) * 7 + 3).all()
+        assert (a.mean(2) == array(range(0, 15), dtype=float).reshape(3, 5) * 7 + 3).all()
+        assert (arange(10).reshape(5, 2).mean(axis=1) == [0.5, 2.5, 4.5, 6.5, 8.5]).all()
 
     def test_sum(self):
         from _numpypy import array
@@ -753,6 +759,7 @@
         assert array([]).sum() == 0.0
         raises(ValueError, 'array([]).max()')
         assert (a.sum(0) == [30, 35, 40]).all()
+        assert (a.sum(axis=0) == [30, 35, 40]).all()
         assert (a.sum(1) == [3, 12, 21, 30, 39]).all()
         assert (a.max(0) == [12, 13, 14]).all()
         assert (a.max(1) == [2, 5, 8, 11, 14]).all()
@@ -765,9 +772,10 @@
         assert (a[:, 1, :].sum(1) == [70, 315, 560]).all()
         raises (ValueError, 'a[:, 1, :].sum(2)')
         assert ((a + a).T.sum(2).T == (a + a).sum(0)).all()
-        skip("Those are broken on reshape, fix!")
         assert (a.reshape(1,-1).sum(0) == range(105)).all()
         assert (a.reshape(1,-1).sum(1) == 5460)
+        assert (array([[1,2],[3,4]]).prod(0) == [3, 8]).all()
+        assert (array([[1,2],[3,4]]).prod(1) == [2, 12]).all()
 
     def test_identity(self):
         from _numpypy import identity, array
@@ -1014,11 +1022,15 @@
         assert a[0].tolist() == [17.1, 27.2]
 
     def test_var(self):
-        from _numpypy import array
+        from _numpypy import array, arange
         a = array(range(10))
         assert a.var() == 8.25
         a = array([5.0])
         assert a.var() == 0.0
+        a = arange(10).reshape(5, 2)
+        assert a.var() == 8.25
+        #assert (a.var(0) == [8, 8]).all()
+        #assert (a.var(1) == [.25] * 5).all()
 
     def test_std(self):
         from _numpypy import array
@@ -1027,6 +1039,22 @@
         a = array([5.0])
         assert a.std() == 0.0
 
+    def test_flatten(self):
+        from _numpypy import array
+
+        a = array([[1, 2, 3], [4, 5, 6]])
+        assert (a.flatten() == [1, 2, 3, 4, 5, 6]).all()
+        a = array([[[1, 2], [3, 4]], [[5, 6], [7, 8]]])
+        assert (a.flatten() == [1, 2, 3, 4, 5, 6, 7, 8]).all()
+        a = array([1, 2, 3, 4, 5, 6, 7, 8])
+        assert (a[::2].flatten() == [1, 3, 5, 7]).all()
+        a = array([1, 2, 3])
+        assert ((a + a).flatten() == [2, 4, 6]).all()
+        a = array(2)
+        assert (a.flatten() == [2]).all()
+        a = array([[1, 2], [3, 4]])
+        assert (a.T.flatten() == [1, 3, 2, 4]).all()
+
 
 class AppTestMultiDim(BaseNumpyAppTest):
     def test_init(self):
@@ -1297,6 +1325,56 @@
         assert isinstance(i['data'][0], int)
         raises(TypeError, getattr, array(3), '__array_interface__')
 
+    def test_array_indexing_one_elem(self):
+        skip("not yet")
+        from _numpypy import array, arange
+        raises(IndexError, 'arange(3)[array([3.5])]')
+        a = arange(3)[array([1])]
+        assert a == 1
+        assert a[0] == 1
+        raises(IndexError,'arange(3)[array([15])]')
+        assert arange(3)[array([-3])] == 0
+        raises(IndexError,'arange(3)[array([-15])]')
+        assert arange(3)[array(1)] == 1
+
+    def test_fill(self):
+        from _numpypy import array
+        a = array([1, 2, 3])
+        a.fill(10)
+        assert (a == [10, 10, 10]).all()
+        a.fill(False)
+        assert (a == [0, 0, 0]).all()
+        b = a[:1]
+        b.fill(4)
+        assert (b == [4]).all()
+        assert (a == [4, 0, 0]).all()
+
+        c = b + b
+        c.fill(27)
+        assert (c == [27]).all()
+
+        d = array(10)
+        d.fill(100)
+        assert d == 100
+
+    def test_array_indexing_bool(self):
+        from _numpypy import arange
+        a = arange(10)
+        assert (a[a > 3] == [4, 5, 6, 7, 8, 9]).all()
+        a = arange(10).reshape(5, 2)
+        assert (a[a > 3] == [4, 5, 6, 7, 8, 9]).all()
+        assert (a[a & 1 == 1] == [1, 3, 5, 7, 9]).all()
+
+    def test_array_indexing_bool_setitem(self):
+        from _numpypy import arange, array
+        a = arange(6)
+        a[a > 3] = 15
+        assert (a == [0, 1, 2, 3, 15, 15]).all()
+        a = arange(6).reshape(3, 2)
+        a[a & 1 == 1] = array([8, 9, 10])
+        assert (a == [[0, 8], [2, 9], [4, 10]]).all()
+
+
 
 class AppTestSupport(BaseNumpyAppTest):
     def setup_class(cls):
@@ -1436,9 +1514,11 @@
         assert repr(a) == "array(0.0)"
         a = array(0.2)
         assert repr(a) == "array(0.2)"
+        a = array([2])
+        assert repr(a) == "array([2])"
 
     def test_repr_multi(self):
-        from _numpypy import arange, zeros
+        from _numpypy import arange, zeros, array
         a = zeros((3, 4))
         assert repr(a) == '''array([[0.0, 0.0, 0.0, 0.0],
        [0.0, 0.0, 0.0, 0.0],
@@ -1461,6 +1541,9 @@
        [498, 999],
        [499, 1000],
        [500, 1001]])'''
+        a = arange(2).reshape((2,1))
+        assert repr(a) == '''array([[0],
+       [1]])'''
 
     def test_repr_slice(self):
         from _numpypy import array, zeros
@@ -1545,14 +1628,3 @@
         a = arange(0, 0.8, 0.1)
         assert len(a) == 8
         assert arange(False, True, True).dtype is dtype(int)
-
-
-class AppTestRanges(BaseNumpyAppTest):
-    def test_app_reshape(self):
-        from _numpypy import arange, array, dtype, reshape
-        a = arange(12)
-        b = reshape(a, (3, 4))
-        assert b.shape == (3, 4)
-        a = range(12)
-        b = reshape(a, (3, 4))
-        assert b.shape == (3, 4)
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -190,14 +190,24 @@
         for i in range(3):
             assert c[i] == a[i] - b[i]
 
-    def test_floor(self):
-        from _numpypy import array, floor
-
-        reference = [-2.0, -1.0, 0.0, 1.0, 1.0]
-        a = array([-1.4, -1.0, 0.0, 1.0, 1.4])
+    def test_floorceil(self):
+        from _numpypy import array, floor, ceil
+        import math
+        reference = [-2.0, -2.0, -1.0, 0.0, 1.0, 1.0, 0]
+        a = array([-1.4, -1.5, -1.0, 0.0, 1.0, 1.4, 0.5])
         b = floor(a)
         for i in range(5):
             assert b[i] == reference[i]
+        reference = [-1.0, -1.0, -1.0, 0.0, 1.0, 2.0, 1.0]
+        a = array([-1.4, -1.5, -1.0, 0.0, 1.0, 1.4, 0.5])
+        b = ceil(a)
+        assert (reference == b).all()
+        inf = float("inf")
+        data = [1.5, 2.9999, -1.999, inf]
+        results = [math.floor(x) for x in data]
+        assert (floor(data) == results).all()
+        results = [math.ceil(x) for x in data]
+        assert (ceil(data) == results).all()
 
     def test_copysign(self):
         from _numpypy import array, copysign
@@ -238,7 +248,7 @@
             assert b[i] == math.sin(a[i])
 
         a = sin(array([True, False], dtype=bool))
-        assert abs(a[0] - sin(1)) < 1e-7 # a[0] will be less precise
+        assert abs(a[0] - sin(1)) < 1e-7  # a[0] will be less precise
         assert a[1] == 0.0
 
     def test_cos(self):
@@ -259,7 +269,6 @@
         for i in range(len(a)):
             assert b[i] == math.tan(a[i])
 
-
     def test_arcsin(self):
         import math
         from _numpypy import array, arcsin
@@ -283,7 +292,6 @@
         for i in range(len(a)):
             assert b[i] == math.acos(a[i])
 
-
         a = array([-10, -1.5, -1.01, 1.01, 1.5, 10, float('nan'), float('inf'), float('-inf')])
         b = arccos(a)
         for f in b:
@@ -347,11 +355,20 @@
         raises(ValueError, maximum.reduce, [])
 
     def test_reduceND(self):
-        from numpypy import add, arange
+        from _numpypy import add, arange
         a = arange(12).reshape(3, 4)
         assert (add.reduce(a, 0) == [12, 15, 18, 21]).all()
         assert (add.reduce(a, 1) == [6.0, 22.0, 38.0]).all()
 
+    def test_bitwise(self):
+        from _numpypy import bitwise_and, bitwise_or, arange, array
+        a = arange(6).reshape(2, 3)
+        assert (a & 1 == [[0, 1, 0], [1, 0, 1]]).all()
+        assert (a & 1 == bitwise_and(a, 1)).all()
+        assert (a | 1 == [[1, 1, 3], [3, 5, 5]]).all()
+        assert (a | 1 == bitwise_or(a, 1)).all()
+        raises(TypeError, 'array([1.0]) & 1')
+
     def test_comparisons(self):
         import operator
         from _numpypy import equal, not_equal, less, less_equal, greater, greater_equal
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -217,6 +217,7 @@
         # This is the sum of the ops for both loops, however if you remove the
         # optimization then you end up with 2 float_adds, so we can still be
         # sure it was optimized correctly.
+        py.test.skip("too fragile")
         self.check_resops({'setinteriorfield_raw': 4, 'getfield_gc': 22,
                            'getarrayitem_gc': 4, 'getarrayitem_gc_pure': 2,
                            'getfield_gc_pure': 8,
@@ -349,7 +350,8 @@
         self.check_trace_count(1)
         self.check_simple_loop({'getinteriorfield_raw': 2, 'float_add': 1,
                                 'setinteriorfield_raw': 1, 'int_add': 2,
-                                'int_eq': 1, 'guard_false': 1, 'jump': 1})
+                                'int_eq': 1, 'guard_false': 1, 'jump': 1,
+                                'arraylen_gc': 1})
 
     def define_virtual_slice():
         return """
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -94,6 +94,9 @@
             width, storage, i, offset
         ))
 
+    def read_bool(self, storage, width, i, offset):
+        raise NotImplementedError
+
     def store(self, storage, width, i, offset, box):
         value = self.unbox(box)
         libffi.array_setitem(clibffi.cast_type_to_ffitype(self.T),
@@ -168,6 +171,7 @@
     @simple_binary_op
     def min(self, v1, v2):
         return min(v1, v2)
+    
 
 class Bool(BaseType, Primitive):
     T = lltype.Bool
@@ -185,6 +189,11 @@
         else:
             return self.False
 
+
+    def read_bool(self, storage, width, i, offset):
+        return libffi.array_getitem(clibffi.cast_type_to_ffitype(self.T),
+                                    width, storage, i, offset)
+
     def coerce_subtype(self, space, w_subtype, w_item):
         # Doesn't return subclasses so it can return the constants.
         return self._coerce(space, w_item)
@@ -253,6 +262,14 @@
             assert v == 0
             return 0
 
+    @simple_binary_op
+    def bitwise_and(self, v1, v2):
+        return v1 & v2
+
+    @simple_binary_op
+    def bitwise_or(self, v1, v2):
+        return v1 | v2
+
 class Int8(BaseType, Integer):
     T = rffi.SIGNEDCHAR
     BoxType = interp_boxes.W_Int8Box
@@ -374,6 +391,10 @@
         return math.floor(v)
 
     @simple_unary_op
+    def ceil(self, v):
+        return math.ceil(v)
+
+    @simple_unary_op
     def exp(self, v):
         try:
             return math.exp(v)
@@ -436,4 +457,4 @@
 class Float64(BaseType, Float):
     T = rffi.DOUBLE
     BoxType = interp_boxes.W_Float64Box
-    format_code = "d"
\ No newline at end of file
+    format_code = "d"
diff --git a/pypy/module/pypyjit/__init__.py b/pypy/module/pypyjit/__init__.py
--- a/pypy/module/pypyjit/__init__.py
+++ b/pypy/module/pypyjit/__init__.py
@@ -11,6 +11,7 @@
         'set_optimize_hook': 'interp_resop.set_optimize_hook',
         'set_abort_hook': 'interp_resop.set_abort_hook',
         'ResOperation': 'interp_resop.WrappedOp',
+        'DebugMergePoint': 'interp_resop.DebugMergePoint',
         'Box': 'interp_resop.WrappedBox',
     }
 
diff --git a/pypy/module/pypyjit/interp_resop.py b/pypy/module/pypyjit/interp_resop.py
--- a/pypy/module/pypyjit/interp_resop.py
+++ b/pypy/module/pypyjit/interp_resop.py
@@ -1,5 +1,6 @@
 
-from pypy.interpreter.typedef import TypeDef, GetSetProperty
+from pypy.interpreter.typedef import (TypeDef, GetSetProperty,
+     interp_attrproperty, interp_attrproperty_w)
 from pypy.interpreter.baseobjspace import Wrappable
 from pypy.interpreter.gateway import unwrap_spec, interp2app, NoneNotWrapped
 from pypy.interpreter.pycode import PyCode
@@ -10,6 +11,7 @@
 from pypy.jit.metainterp.resoperation import rop, AbstractResOp
 from pypy.rlib.nonconst import NonConstant
 from pypy.rlib import jit_hooks
+from pypy.module.pypyjit.interp_jit import pypyjitdriver
 
 class Cache(object):
     in_recursion = False
@@ -111,13 +113,24 @@
 
 def wrap_oplist(space, logops, operations, ops_offset=None):
     l_w = []
+    jitdrivers_sd = logops.metainterp_sd.jitdrivers_sd
     for op in operations:
         if ops_offset is None:
             ofs = -1
         else:
             ofs = ops_offset.get(op, 0)
-        l_w.append(WrappedOp(jit_hooks._cast_to_gcref(op), ofs,
-                             logops.repr_of_resop(op)))
+        if op.opnum == rop.DEBUG_MERGE_POINT:
+            jd_sd = jitdrivers_sd[op.getarg(0).getint()]
+            greenkey = op.getarglist()[2:]
+            repr = jd_sd.warmstate.get_location_str(greenkey)
+            w_greenkey = wrap_greenkey(space, jd_sd.jitdriver, greenkey, repr)
+            l_w.append(DebugMergePoint(space, jit_hooks._cast_to_gcref(op),
+                                       logops.repr_of_resop(op),
+                                       jd_sd.jitdriver.name,
+                                       w_greenkey))
+        else:
+            l_w.append(WrappedOp(jit_hooks._cast_to_gcref(op), ofs,
+                                 logops.repr_of_resop(op)))
     return l_w
 
 class WrappedBox(Wrappable):
@@ -150,6 +163,15 @@
         llres = res.llbox
     return WrappedOp(jit_hooks.resop_new(num, args, llres), offset, repr)
 
+ at unwrap_spec(repr=str, jd_name=str)
+def descr_new_dmp(space, w_tp, w_args, repr, jd_name, w_greenkey):
+    args = [space.interp_w(WrappedBox, w_arg).llbox for w_arg in
+            space.listview(w_args)]
+    num = rop.DEBUG_MERGE_POINT
+    return DebugMergePoint(space,
+                           jit_hooks.resop_new(num, args, jit_hooks.emptyval()),
+                           repr, jd_name, w_greenkey)
+
 class WrappedOp(Wrappable):
     """ A class representing a single ResOperation, wrapped nicely
     """
@@ -182,6 +204,25 @@
         box = space.interp_w(WrappedBox, w_box)
         jit_hooks.resop_setresult(self.op, box.llbox)
 
+class DebugMergePoint(WrappedOp):
+    def __init__(self, space, op, repr_of_resop, jd_name, w_greenkey):
+        WrappedOp.__init__(self, op, -1, repr_of_resop)
+        self.w_greenkey = w_greenkey
+        self.jd_name = jd_name
+
+    def get_pycode(self, space):
+        if self.jd_name == pypyjitdriver.name:
+            return space.getitem(self.w_greenkey, space.wrap(0))
+        raise OperationError(space.w_AttributeError, space.wrap("This DebugMergePoint doesn't belong to the main Python JitDriver"))
+
+    def get_bytecode_no(self, space):
+        if self.jd_name == pypyjitdriver.name:
+            return space.getitem(self.w_greenkey, space.wrap(1))
+        raise OperationError(space.w_AttributeError, space.wrap("This DebugMergePoint doesn't belong to the main Python JitDriver"))
+
+    def get_jitdriver_name(self, space):
+        return space.wrap(self.jd_name)
+
 WrappedOp.typedef = TypeDef(
     'ResOperation',
     __doc__ = WrappedOp.__doc__,
@@ -195,3 +236,15 @@
                             WrappedOp.descr_setresult)
 )
 WrappedOp.acceptable_as_base_class = False
+
+DebugMergePoint.typedef = TypeDef(
+    'DebugMergePoint', WrappedOp.typedef,
+    __new__ = interp2app(descr_new_dmp),
+    greenkey = interp_attrproperty_w("w_greenkey", cls=DebugMergePoint),
+    pycode = GetSetProperty(DebugMergePoint.get_pycode),
+    bytecode_no = GetSetProperty(DebugMergePoint.get_bytecode_no),
+    jitdriver_name = GetSetProperty(DebugMergePoint.get_jitdriver_name),
+)
+DebugMergePoint.acceptable_as_base_class = False
+
+
diff --git a/pypy/module/pypyjit/policy.py b/pypy/module/pypyjit/policy.py
--- a/pypy/module/pypyjit/policy.py
+++ b/pypy/module/pypyjit/policy.py
@@ -127,7 +127,7 @@
                        'imp', 'sys', 'array', '_ffi', 'itertools', 'operator',
                        'posix', '_socket', '_sre', '_lsprof', '_weakref',
                        '__pypy__', 'cStringIO', '_collections', 'struct',
-                       'mmap', 'marshal']:
+                       'mmap', 'marshal', '_codecs']:
             if modname == 'pypyjit' and 'interp_resop' in rest:
                 return False
             return True
diff --git a/pypy/module/pypyjit/test/test_jit_hook.py b/pypy/module/pypyjit/test/test_jit_hook.py
--- a/pypy/module/pypyjit/test/test_jit_hook.py
+++ b/pypy/module/pypyjit/test/test_jit_hook.py
@@ -92,6 +92,7 @@
         cls.w_on_compile_bridge = space.wrap(interp2app(interp_on_compile_bridge))
         cls.w_on_abort = space.wrap(interp2app(interp_on_abort))
         cls.w_int_add_num = space.wrap(rop.INT_ADD)
+        cls.w_dmp_num = space.wrap(rop.DEBUG_MERGE_POINT)
         cls.w_on_optimize = space.wrap(interp2app(interp_on_optimize))
         cls.orig_oplist = oplist
 
@@ -117,6 +118,10 @@
         assert elem[2][2] == False
         assert len(elem[3]) == 4
         int_add = elem[3][0]
+        dmp = elem[3][1]
+        assert isinstance(dmp, pypyjit.DebugMergePoint)
+        assert dmp.pycode is self.f.func_code
+        assert dmp.greenkey == (self.f.func_code, 0, False)
         #assert int_add.name == 'int_add'
         assert int_add.num == self.int_add_num
         self.on_compile_bridge()
@@ -211,3 +216,18 @@
         assert op.getarg(0).getint() == 4
         op.result = box
         assert op.result.getint() == 1
+
+    def test_creation_dmp(self):
+        from pypyjit import DebugMergePoint, Box
+
+        def f():
+            pass
+
+        op = DebugMergePoint([Box(0)], 'repr', 'pypyjit', (f.func_code, 0, 0))
+        assert op.bytecode_no == 0
+        assert op.pycode is f.func_code
+        assert repr(op) == 'repr'
+        assert op.jitdriver_name == 'pypyjit'
+        assert op.num == self.dmp_num
+        op = DebugMergePoint([Box(0)], 'repr', 'notmain', ('str',))
+        raises(AttributeError, 'op.pycode')
diff --git a/pypy/module/pypyjit/test_pypy_c/test_math.py b/pypy/module/pypyjit/test_pypy_c/test_math.py
--- a/pypy/module/pypyjit/test_pypy_c/test_math.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_math.py
@@ -90,3 +90,19 @@
             --TICK--
             jump(..., descr=)
         """)
+
+    def test_pow_two(self):
+        def main(n):
+            s = 0.123
+            while n > 0:
+                s -= s ** 2   # ID: pow
+                n -= 1
+            return s
+        log = self.run(main, [500])
+        assert abs(log.result - main(500)) < 1e-9
+        loop, = log.loops_by_filename(self.filepath)
+        assert loop.match_by_id("pow", """
+            guard_not_invalidated(descr=...)
+            f38 = float_mul(f30, f30)
+            f39 = float_sub(f30, f38)
+        """)
diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py
--- a/pypy/module/sys/version.py
+++ b/pypy/module/sys/version.py
@@ -10,7 +10,7 @@
 CPYTHON_VERSION            = (2, 7, 1, "final", 42)   #XXX # sync patchlevel.h
 CPYTHON_API_VERSION        = 1013   #XXX # sync with include/modsupport.h
 
-PYPY_VERSION               = (1, 7, 1, "dev", 0)    #XXX # sync patchlevel.h
+PYPY_VERSION               = (1, 8, 1, "dev", 0)    #XXX # sync patchlevel.h
 
 if platform.name == 'msvc':
     COMPILER_INFO = 'MSC v.%d 32 bit' % (platform.version * 10 + 600)
diff --git a/lib_pypy/numpypy/test/test_fromnumeric.py b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
rename from lib_pypy/numpypy/test/test_fromnumeric.py
rename to pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
--- a/lib_pypy/numpypy/test/test_fromnumeric.py
+++ b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
@@ -1,7 +1,7 @@
-
 from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
 
-class AppTestFromNumeric(BaseNumpyAppTest):     
+
+class AppTestFromNumeric(BaseNumpyAppTest):
     def test_argmax(self):
         # tests taken from numpy/core/fromnumeric.py docstring
         from numpypy import array, arange, argmax
@@ -18,12 +18,12 @@
         from numpypy import array, arange, argmin
         a = arange(6).reshape((2,3))
         assert argmin(a) == 0
-        # assert (argmax(a, axis=0) == array([0, 0, 0])).all()
-        # assert (argmax(a, axis=1) == array([0, 0])).all()
+        #assert (argmin(a, axis=0) == array([0, 0, 0])).all()
+        #assert (argmin(a, axis=1) == array([0, 0])).all()
         b = arange(6)
         b[1] = 0
         assert argmin(b) == 0
-   
+
     def test_shape(self):
         # tests taken from numpy/core/fromnumeric.py docstring
         from numpypy import array, identity, shape
@@ -40,11 +40,11 @@
         assert sum([0.5, 1.5])== 2.0
         assert sum([[0, 1], [0, 5]]) == 6
         # assert sum([0.5, 0.7, 0.2, 1.5], dtype=int32) == 1
-        # assert (sum([[0, 1], [0, 5]], axis=0) == array([0, 6])).all()
-        # assert (sum([[0, 1], [0, 5]], axis=1) == array([1, 5])).all()
+        assert (sum([[0, 1], [0, 5]], axis=0) == array([0, 6])).all()
+        assert (sum([[0, 1], [0, 5]], axis=1) == array([1, 5])).all()
         # If the accumulator is too small, overflow occurs:
         # assert ones(128, dtype=int8).sum(dtype=int8) == -128
-                                 
+
     def test_amin(self):
         # tests taken from numpy/core/fromnumeric.py docstring
         from numpypy import array, arange, amin
@@ -86,24 +86,54 @@
         assert ndim([[1,2,3],[4,5,6]]) == 2
         assert ndim(array([[1,2,3],[4,5,6]])) == 2
         assert ndim(1) == 0
-    
+
     def test_rank(self):
         # tests taken from numpy/core/fromnumeric.py docstring
         from numpypy import array, rank
         assert rank([[1,2,3],[4,5,6]]) == 2
         assert rank(array([[1,2,3],[4,5,6]])) == 2
         assert rank(1) == 0
-    
+
     def test_var(self):
         from numpypy import array, var
         a = array([[1,2],[3,4]])
         assert var(a) == 1.25
-        # assert (np.var(a,0) == array([ 1.,  1.])).all()
-        # assert (np.var(a,1) == array([ 0.25,  0.25])).all()
+        #assert (var(a,0) == array([ 1.,  1.])).all()
+        #assert (var(a,1) == array([ 0.25,  0.25])).all()
 
     def test_std(self):
         from numpypy import array, std
         a = array([[1, 2], [3, 4]])
         assert std(a) ==  1.1180339887498949
-        # assert (std(a, axis=0) == array([ 1.,  1.])).all()
-        # assert (std(a, axis=1) == array([ 0.5,  0.5]).all()
+        #assert (std(a, axis=0) == array([ 1.,  1.])).all()
+        #assert (std(a, axis=1) == array([ 0.5,  0.5])).all()
+
+    def test_mean(self):
+        from numpypy import array, mean, arange
+        assert mean(array(range(5))) == 2.0
+        assert mean(range(5)) == 2.0
+        assert (mean(arange(10).reshape(5, 2), axis=0) == [4, 5]).all()
+        assert (mean(arange(10).reshape(5, 2), axis=1) == [0.5, 2.5, 4.5, 6.5, 8.5]).all()
+
+    def test_reshape(self):
+        from numpypy import arange, array, dtype, reshape
+        a = arange(12)
+        b = reshape(a, (3, 4))
+        assert b.shape == (3, 4)
+        a = range(12)
+        b = reshape(a, (3, 4))
+        assert b.shape == (3, 4)
+        a = array(range(105)).reshape(3, 5, 7)
+        assert reshape(a, (1, -1)).shape == (1, 105)
+        assert reshape(a, (1, 1, -1)).shape == (1, 1, 105)
+        assert reshape(a, (-1, 1, 1)).shape == (105, 1, 1)
+    
+    def test_transpose(self):   
+        from numpypy import arange, array, transpose, ones
+        x = arange(4).reshape((2,2))
+        assert (transpose(x) == array([[0, 2],[1, 3]])).all()
+        # Once axes argument is implemented, add more tests
+        raises(NotImplementedError, "transpose(x, axes=(1, 0, 2))")
+        # x = ones((1, 2, 3))
+        # assert transpose(x, (1, 0, 2)).shape == (2, 1, 3)
+
diff --git a/pypy/module/test_lib_pypy/test_datetime.py b/pypy/module/test_lib_pypy/test_datetime.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/test_lib_pypy/test_datetime.py
@@ -0,0 +1,24 @@
+"""Additional tests for datetime."""
+
+import time
+import datetime
+import os
+
+def test_utcfromtimestamp():
+    """Confirm that utcfromtimestamp and fromtimestamp give consistent results.
+
+    Based on danchr's test script in https://bugs.pypy.org/issue986
+    """
+    try:
+        prev_tz = os.environ.get("TZ")
+        os.environ["TZ"] = "GMT"
+        for unused in xrange(100):
+            now = time.time()
+            delta = (datetime.datetime.utcfromtimestamp(now) -
+                     datetime.datetime.fromtimestamp(now))
+            assert delta.days * 86400 + delta.seconds == 0
+    finally:
+        if prev_tz is None:
+            del os.environ["TZ"]
+        else:
+            os.environ["TZ"] = prev_tz
diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -624,8 +624,8 @@
 
     if step == 1:
         assert start >= 0
-        assert slicelength >= 0
-        del items[start:start+slicelength]
+        if slicelength > 0:
+            del items[start:start+slicelength]
     else:
         n = len(items)
         i = start
@@ -662,10 +662,11 @@
             while i >= lim:
                 items[i] = items[i-delta]
                 i -= 1
-        elif start >= 0:
+        elif delta == 0:
+            pass
+        else:
+            assert start >= 0   # start<0 is only possible with slicelength==0
             del items[start:start+delta]
-        else:
-            assert delta==0   # start<0 is only possible with slicelength==0
     elif len2 != slicelength:  # No resize for extended slices
         raise operationerrfmt(space.w_ValueError, "attempt to "
               "assign sequence of size %d to extended slice of size %d",
diff --git a/pypy/objspace/std/complexobject.py b/pypy/objspace/std/complexobject.py
--- a/pypy/objspace/std/complexobject.py
+++ b/pypy/objspace/std/complexobject.py
@@ -8,6 +8,7 @@
 from pypy.rlib.rbigint import rbigint
 from pypy.rlib.rfloat import (
     formatd, DTSF_STR_PRECISION, isinf, isnan, copysign)
+from pypy.rlib import jit
 
 import math
 
@@ -129,10 +130,10 @@
             ir = len * math.sin(phase)
         return W_ComplexObject(rr, ir)
 
-    def pow_int(self, n):
-        if n > 100 or n < -100:
-            return self.pow(W_ComplexObject(1.0 * n, 0.0))
-        elif n > 0:
+    def pow_small_int(self, n):
+        if n >= 0:
+            if jit.isconstant(n) and n == 2:
+                return self.mul(self)
             return self.pow_positive_int(n)
         else:
             return w_one.div(self.pow_positive_int(-n))
@@ -217,10 +218,10 @@
 def pow__Complex_Complex_ANY(space, w_complex, w_exponent, thirdArg):
     if not space.is_w(thirdArg, space.w_None):
         raise OperationError(space.w_ValueError, space.wrap('complex modulo'))
-    int_exponent = int(w_exponent.realval)
     try:
-        if w_exponent.imagval == 0.0 and w_exponent.realval == int_exponent:
-            w_p = w_complex.pow_int(int_exponent)
+        r = w_exponent.realval
+        if w_exponent.imagval == 0.0 and -100.0 <= r <= 100.0 and r == int(r):
+            w_p = w_complex.pow_small_int(int(r))
         else:
             w_p = w_complex.pow(w_exponent)
     except ZeroDivisionError:
diff --git a/pypy/objspace/std/floatobject.py b/pypy/objspace/std/floatobject.py
--- a/pypy/objspace/std/floatobject.py
+++ b/pypy/objspace/std/floatobject.py
@@ -451,6 +451,8 @@
     y = w_float2.floatval
 
     # Sort out special cases here instead of relying on pow()
+    if y == 2.0:                      # special case for performance:
+        return W_FloatObject(x * x)   # x * x is always correct
     if y == 0.0:
         # x**0 is 1, even 0**0
         return W_FloatObject(1.0)
diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -804,10 +804,11 @@
                 while i >= lim:
                     items[i] = items[i-delta]
                     i -= 1
-            elif start >= 0:
+            elif delta == 0:
+                pass
+            else:
+                assert start >= 0 # start<0 is only possible with slicelength==0
                 del items[start:start+delta]
-            else:
-                assert delta==0   # start<0 is only possible with slicelength==0
         elif len2 != slicelength:  # No resize for extended slices
             raise operationerrfmt(self.space.w_ValueError, "attempt to "
                   "assign sequence of size %d to extended slice of size %d",
@@ -851,8 +852,8 @@
 
         if step == 1:
             assert start >= 0
-            assert slicelength >= 0
-            del items[start:start+slicelength]
+            if slicelength > 0:
+                del items[start:start+slicelength]
         else:
             n = len(items)
             i = start
diff --git a/pypy/objspace/std/test/test_bytearrayobject.py b/pypy/objspace/std/test/test_bytearrayobject.py
--- a/pypy/objspace/std/test/test_bytearrayobject.py
+++ b/pypy/objspace/std/test/test_bytearrayobject.py
@@ -1,5 +1,9 @@
+from pypy import conftest
 
 class AppTestBytesArray:
+    def setup_class(cls):
+        cls.w_runappdirect = cls.space.wrap(conftest.option.runappdirect)
+
     def test_basics(self):
         b = bytearray()
         assert type(b) is bytearray
@@ -439,3 +443,15 @@
     def test_reduce(self):
         assert bytearray('caf\xe9').__reduce__() == (
             bytearray, (u'caf\xe9', 'latin-1'), None)
+
+    def test_setitem_slice_performance(self):
+        # because of a complexity bug, this used to take forever on a
+        # translated pypy.  On CPython2.6 -A, it takes around 8 seconds.
+        if self.runappdirect:
+            count = 16*1024*1024
+        else:
+            count = 1024
+        b = bytearray(count)
+        for i in range(count):
+            b[i:i+1] = 'y'
+        assert str(b) == 'y' * count
diff --git a/pypy/objspace/std/test/test_complexobject.py b/pypy/objspace/std/test/test_complexobject.py
--- a/pypy/objspace/std/test/test_complexobject.py
+++ b/pypy/objspace/std/test/test_complexobject.py
@@ -71,7 +71,7 @@
         assert _powu((0.0,1.0),2) == (-1.0,0.0)
 
         def _powi((r1, i1), n):
-            w_res = W_ComplexObject(r1, i1).pow_int(n)
+            w_res = W_ComplexObject(r1, i1).pow_small_int(n)
             return w_res.realval, w_res.imagval
         assert _powi((0.0,2.0),0) == (1.0,0.0)
         assert _powi((0.0,0.0),2) == (0.0,0.0)
@@ -213,6 +213,7 @@
         assert a ** 105 == a ** 105
         assert a ** -105 == a ** -105
         assert a ** -30 == a ** -30
+        assert a ** 2 == a * a
 
         assert 0.0j ** 0 == 1
 
diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py
--- a/pypy/objspace/std/test/test_listobject.py
+++ b/pypy/objspace/std/test/test_listobject.py
@@ -397,6 +397,7 @@
         on_cpython = (option.runappdirect and
                             not hasattr(sys, 'pypy_translation_info'))
         cls.w_on_cpython = cls.space.wrap(on_cpython)
+        cls.w_runappdirect = cls.space.wrap(option.runappdirect)
 
     def test_getstrategyfromlist_w(self):
         l0 = ["a", "2", "a", True]
@@ -898,6 +899,18 @@
         l[::-1] = l
         assert l == [6,5,4,3,2,1]
 
+    def test_setitem_slice_performance(self):
+        # because of a complexity bug, this used to take forever on a
+        # translated pypy.  On CPython2.6 -A, it takes around 5 seconds.
+        if self.runappdirect:
+            count = 16*1024*1024
+        else:
+            count = 1024
+        b = [None] * count
+        for i in range(count):
+            b[i:i+1] = ['y']
+        assert b == ['y'] * count
+
     def test_recursive_repr(self):
         l = []
         assert repr(l) == '[]'
diff --git a/pypy/objspace/std/test/test_unicodeobject.py b/pypy/objspace/std/test/test_unicodeobject.py
--- a/pypy/objspace/std/test/test_unicodeobject.py
+++ b/pypy/objspace/std/test/test_unicodeobject.py
@@ -64,6 +64,12 @@
         check(', '.join([u'a']), u'a')
         check(', '.join(['a', u'b']), u'a, b')
         check(u', '.join(['a', 'b']), u'a, b')
+        try:
+            u''.join([u'a', 2, 3])
+        except TypeError, e:
+            assert 'sequence item 1' in str(e)
+        else:
+            raise Exception("DID NOT RAISE")
 
     if sys.version_info >= (2,3):
         def test_contains_ex(self):
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -201,7 +201,7 @@
     return space.newbool(container.find(item) != -1)
 
 def unicode_join__Unicode_ANY(space, w_self, w_list):
-    list_w = space.unpackiterable(w_list)
+    list_w = space.listview(w_list)
     size = len(list_w)
 
     if size == 0:
@@ -216,22 +216,21 @@
 
 def _unicode_join_many_items(space, w_self, list_w, size):
     self = w_self._value
-    sb = UnicodeBuilder()
+    prealloc_size = len(self) * (size - 1)
+    for i in range(size):
+        try:
+            prealloc_size += len(space.unicode_w(list_w[i]))
+        except OperationError, e:
+            if not e.match(space, space.w_TypeError):
+                raise
+            raise operationerrfmt(space.w_TypeError,
+                        "sequence item %d: expected string or Unicode", i)
+    sb = UnicodeBuilder(prealloc_size)
     for i in range(size):
         if self and i != 0:
             sb.append(self)
         w_s = list_w[i]
-        if isinstance(w_s, W_UnicodeObject):
-            # shortcut for performance
-            sb.append(w_s._value)
-        else:
-            try:
-                sb.append(space.unicode_w(w_s))
-            except OperationError, e:
-                if not e.match(space, space.w_TypeError):
-                    raise
-                raise operationerrfmt(space.w_TypeError,
-                    "sequence item %d: expected string or Unicode", i)
+        sb.append(space.unicode_w(w_s))
     return space.wrap(sb.build())
 
 def hash__Unicode(space, w_uni):
diff --git a/pypy/rlib/jit_hooks.py b/pypy/rlib/jit_hooks.py
--- a/pypy/rlib/jit_hooks.py
+++ b/pypy/rlib/jit_hooks.py
@@ -52,7 +52,10 @@
     from pypy.jit.metainterp.history import ResOperation
 
     args = [_cast_to_box(llargs[i]) for i in range(len(llargs))]
-    res = _cast_to_box(llres)
+    if llres:
+        res = _cast_to_box(llres)
+    else:
+        res = None
     return _cast_to_gcref(ResOperation(no, args, res))
 
 @register_helper(annmodel.SomePtr(llmemory.GCREF))
diff --git a/pypy/rlib/longlong2float.py b/pypy/rlib/longlong2float.py
--- a/pypy/rlib/longlong2float.py
+++ b/pypy/rlib/longlong2float.py
@@ -79,19 +79,23 @@
 longlong2float = rffi.llexternal(
     "pypy__longlong2float", [rffi.LONGLONG], rffi.DOUBLE,
     _callable=longlong2float_emulator, compilation_info=eci,
-    _nowrapper=True, elidable_function=True, sandboxsafe=True)
+    _nowrapper=True, elidable_function=True, sandboxsafe=True,
+    oo_primitive="pypy__longlong2float")
 
 float2longlong = rffi.llexternal(
     "pypy__float2longlong", [rffi.DOUBLE], rffi.LONGLONG,
     _callable=float2longlong_emulator, compilation_info=eci,
-    _nowrapper=True, elidable_function=True, sandboxsafe=True)
+    _nowrapper=True, elidable_function=True, sandboxsafe=True,
+    oo_primitive="pypy__float2longlong")
 
 uint2singlefloat = rffi.llexternal(
     "pypy__uint2singlefloat", [rffi.UINT], rffi.FLOAT,
     _callable=uint2singlefloat_emulator, compilation_info=eci,
-    _nowrapper=True, elidable_function=True, sandboxsafe=True)
+    _nowrapper=True, elidable_function=True, sandboxsafe=True,
+    oo_primitive="pypy__uint2singlefloat")
 
 singlefloat2uint = rffi.llexternal(
     "pypy__singlefloat2uint", [rffi.FLOAT], rffi.UINT,
     _callable=singlefloat2uint_emulator, compilation_info=eci,
-    _nowrapper=True, elidable_function=True, sandboxsafe=True)
+    _nowrapper=True, elidable_function=True, sandboxsafe=True,
+    oo_primitive="pypy__singlefloat2uint")
diff --git a/pypy/rlib/objectmodel.py b/pypy/rlib/objectmodel.py
--- a/pypy/rlib/objectmodel.py
+++ b/pypy/rlib/objectmodel.py
@@ -420,7 +420,7 @@
                   vobj.concretetype.TO._gckind == 'gc')
         else:
             from pypy.rpython.ootypesystem import ootype
-            ok = isinstance(vobj.concretetype, ootype.Instance)
+            ok = isinstance(vobj.concretetype, (ootype.Instance, ootype.BuiltinType))
         if not ok:
             from pypy.rpython.error import TyperError
             raise TyperError("compute_unique_id() cannot be applied to"
diff --git a/pypy/rpython/lltypesystem/rclass.py b/pypy/rpython/lltypesystem/rclass.py
--- a/pypy/rpython/lltypesystem/rclass.py
+++ b/pypy/rpython/lltypesystem/rclass.py
@@ -510,7 +510,13 @@
         ctype = inputconst(Void, self.object_type)
         cflags = inputconst(Void, flags)
         vlist = [ctype, cflags]
-        vptr = llops.genop('malloc', vlist,
+        cnonmovable = self.classdef.classdesc.read_attribute(
+            '_alloc_nonmovable_', Constant(False))
+        if cnonmovable.value:
+            opname = 'malloc_nonmovable'
+        else:
+            opname = 'malloc'
+        vptr = llops.genop(opname, vlist,
                            resulttype = Ptr(self.object_type))
         ctypeptr = inputconst(CLASSTYPE, self.rclass.getvtable())
         self.setfield(vptr, '__class__', ctypeptr, llops)
diff --git a/pypy/rpython/lltypesystem/rtuple.py b/pypy/rpython/lltypesystem/rtuple.py
--- a/pypy/rpython/lltypesystem/rtuple.py
+++ b/pypy/rpython/lltypesystem/rtuple.py
@@ -27,6 +27,10 @@
 
     def newtuple(cls, llops, r_tuple, items_v):
         # items_v should have the lowleveltype of the internal reprs
+        assert len(r_tuple.items_r) == len(items_v)
+        for r_item, v_item in zip(r_tuple.items_r, items_v):
+            assert r_item.lowleveltype == v_item.concretetype
+        #
         if len(r_tuple.items_r) == 0:
             return inputconst(Void, ())    # a Void empty tuple
         c1 = inputconst(Void, r_tuple.lowleveltype.TO)
diff --git a/pypy/rpython/ootypesystem/ootype.py b/pypy/rpython/ootypesystem/ootype.py
--- a/pypy/rpython/ootypesystem/ootype.py
+++ b/pypy/rpython/ootypesystem/ootype.py
@@ -512,6 +512,7 @@
             "ll_append_char": Meth([CHARTP], Void),
             "ll_append": Meth([STRINGTP], Void),
             "ll_build": Meth([], STRINGTP),
+            "ll_getlength": Meth([], Signed),
             })
         self._setup_methods({})
 
@@ -1376,6 +1377,9 @@
     def _cast_to_object(self):
         return make_object(self)
 
+    def _identityhash(self):
+        return object.__hash__(self)
+
 class _string(_builtin_type):
 
     def __init__(self, STRING, value = ''):
@@ -1543,6 +1547,9 @@
         else:
             return make_unicode(u''.join(self._buf))
 
+    def ll_getlength(self):
+        return self.ll_build().ll_strlen()
+
 class _null_string_builder(_null_mixin(_string_builder), _string_builder):
     def __init__(self, STRING_BUILDER):
         self.__dict__["_TYPE"] = STRING_BUILDER
diff --git a/pypy/rpython/ootypesystem/rbuilder.py b/pypy/rpython/ootypesystem/rbuilder.py
--- a/pypy/rpython/ootypesystem/rbuilder.py
+++ b/pypy/rpython/ootypesystem/rbuilder.py
@@ -21,6 +21,10 @@
         builder.ll_append_char(char)
 
     @staticmethod
+    def ll_getlength(builder):
+        return builder.ll_getlength()
+
+    @staticmethod
     def ll_append(builder, string):
         builder.ll_append(string)
 
diff --git a/pypy/rpython/rrange.py b/pypy/rpython/rrange.py
--- a/pypy/rpython/rrange.py
+++ b/pypy/rpython/rrange.py
@@ -204,7 +204,10 @@
         v_index = hop.gendirectcall(self.ll_getnextindex, v_enumerate)
         hop2 = hop.copy()
         hop2.args_r = [self.r_baseiter]
+        r_item_src = self.r_baseiter.r_list.external_item_repr
+        r_item_dst = hop.r_result.items_r[1]
         v_item = self.r_baseiter.rtype_next(hop2)
+        v_item = hop.llops.convertvar(v_item, r_item_src, r_item_dst)
         return hop.r_result.newtuple(hop.llops, hop.r_result,
                                      [v_index, v_item])
 
diff --git a/pypy/rpython/test/test_rbuilder.py b/pypy/rpython/test/test_rbuilder.py
--- a/pypy/rpython/test/test_rbuilder.py
+++ b/pypy/rpython/test/test_rbuilder.py
@@ -124,9 +124,5 @@
     pass
 
 class TestOOtype(BaseTestStringBuilder, OORtypeMixin):
-    def test_string_getlength(self):
-        py.test.skip("getlength(): not implemented on ootype")
-    def test_unicode_getlength(self):
-        py.test.skip("getlength(): not implemented on ootype")
     def test_append_charpsize(self):
         py.test.skip("append_charpsize(): not implemented on ootype")
diff --git a/pypy/rpython/test/test_rbuiltin.py b/pypy/rpython/test/test_rbuiltin.py
--- a/pypy/rpython/test/test_rbuiltin.py
+++ b/pypy/rpython/test/test_rbuiltin.py
@@ -463,6 +463,31 @@
             assert x1 == intmask(x0)
             assert x3 == intmask(x2)
 
+    def test_id_on_builtins(self):
+        from pypy.rlib.objectmodel import compute_unique_id
+        from pypy.rlib.rstring import StringBuilder, UnicodeBuilder
+        def fn():
+            return (compute_unique_id("foo"),
+                    compute_unique_id(u"bar"),
+                    compute_unique_id([1]),
+                    compute_unique_id({"foo": 3}),
+                    compute_unique_id(StringBuilder()),
+                    compute_unique_id(UnicodeBuilder()))
+        res = self.interpret(fn, [])
+        for id in self.ll_unpack_tuple(res, 6):
+            assert isinstance(id, (int, r_longlong))
+
+    def test_uniqueness_of_id_on_strings(self):
+        from pypy.rlib.objectmodel import compute_unique_id
+        def fn(s1, s2):
+            return (compute_unique_id(s1), compute_unique_id(s2))
+
+        s1 = "foo"
+        s2 = ''.join(['f','oo'])
+        res = self.interpret(fn, [self.string_to_ll(s1), self.string_to_ll(s2)])
+        i1, i2 = self.ll_unpack_tuple(res, 2)
+        assert i1 != i2
+
     def test_cast_primitive(self):
         from pypy.rpython.annlowlevel import LowLevelAnnotatorPolicy
         def llf(u):
diff --git a/pypy/rpython/test/test_rclass.py b/pypy/rpython/test/test_rclass.py
--- a/pypy/rpython/test/test_rclass.py
+++ b/pypy/rpython/test/test_rclass.py
@@ -1130,6 +1130,18 @@
             assert sorted([u]) == [6]                    # 32-bit types
             assert sorted([i, r, d, l]) == [2, 3, 4, 5]  # 64-bit types
 
+    def test_nonmovable(self):
+        for (nonmovable, opname) in [(True, 'malloc_nonmovable'),
+                                     (False, 'malloc')]:
+            class A(object):
+                _alloc_nonmovable_ = nonmovable
+            def f():
+                return A()
+            t, typer, graph = self.gengraph(f, [])
+            assert summary(graph) == {opname: 1,
+                                      'cast_pointer': 1,
+                                      'setfield': 1}
+
 
 class TestOOtype(BaseTestRclass, OORtypeMixin):
 
diff --git a/pypy/rpython/test/test_rrange.py b/pypy/rpython/test/test_rrange.py
--- a/pypy/rpython/test/test_rrange.py
+++ b/pypy/rpython/test/test_rrange.py
@@ -169,6 +169,22 @@
         res = self.interpret(fn, [2])
         assert res == 789
 
+    def test_enumerate_instances(self):
+        class A:
+            pass
+        def fn(n):
+            a = A()
+            b = A()
+            a.k = 10
+            b.k = 20
+            for i, x in enumerate([a, b]):
+                if i == n:
+                    return x.k
+            return 5
+        res = self.interpret(fn, [1])
+        assert res == 20
+
+
 
 class TestLLtype(BaseTestRrange, LLRtypeMixin):
     from pypy.rpython.lltypesystem import rrange 
diff --git a/pypy/tool/jitlogparser/parser.py b/pypy/tool/jitlogparser/parser.py
--- a/pypy/tool/jitlogparser/parser.py
+++ b/pypy/tool/jitlogparser/parser.py
@@ -140,13 +140,15 @@
     bytecode_name = None
     is_bytecode = True
     inline_level = None
+    has_dmp = False
 
     def parse_code_data(self, arg):
         m = re.search('\w]+)[\.,] file \'(.+?)\'[\.,] line (\d+)> #(\d+) (\w+)',
                       arg)
         if m is None:
             # a non-code loop, like StrLiteralSearch or something
-            self.bytecode_name = arg
+            if arg:
+                self.bytecode_name = arg
         else:
             self.name, self.filename, lineno, bytecode_no, self.bytecode_name = m.groups()
             self.startlineno = int(lineno)
@@ -218,7 +220,7 @@
         self.inputargs = inputargs
         self.chunks = chunks
         for chunk in self.chunks:
-            if chunk.filename is not None:
+            if chunk.bytecode_name is not None:
                 self.startlineno = chunk.startlineno
                 self.filename = chunk.filename
                 self.name = chunk.name
diff --git a/pypy/tool/jitlogparser/test/test_parser.py b/pypy/tool/jitlogparser/test/test_parser.py
--- a/pypy/tool/jitlogparser/test/test_parser.py
+++ b/pypy/tool/jitlogparser/test/test_parser.py
@@ -283,3 +283,13 @@
     assert loops[-1].count == 1234
     assert loops[1].count == 123
     assert loops[2].count == 12
+
+def test_parse_nonpython():
+    loop = parse("""
+    []
+    debug_merge_point(0, 'random')
+    debug_merge_point(0, ' #15 COMPARE_OP')
+    """)
+    f = Function.from_operations(loop.operations, LoopStorage())
+    assert f.chunks[-1].filename == 'x.py'
+    assert f.filename is None
diff --git a/pypy/tool/test/test_version.py b/pypy/tool/test/test_version.py
--- a/pypy/tool/test/test_version.py
+++ b/pypy/tool/test/test_version.py
@@ -1,6 +1,22 @@
 import os, sys
 import py
-from pypy.tool.version import get_repo_version_info
+from pypy.tool.version import get_repo_version_info, _get_hg_archive_version
+
+def test_hg_archival_version(tmpdir):
+    def version_for(name, **kw):
+        path = tmpdir.join(name)
+        path.write('\n'.join('%s: %s' % x for x in kw.items()))
+        return _get_hg_archive_version(str(path))
+
+    assert version_for('release',
+                       tag='release-123',
+                       node='000',
+                      ) == ('PyPy', 'release-123', '000')
+    assert version_for('somebranch',
+                       node='000',
+                       branch='something',
+                      ) == ('PyPy', 'something', '000')
+
 
 def test_get_repo_version_info():
     assert get_repo_version_info(None)
diff --git a/pypy/tool/version.py b/pypy/tool/version.py
--- a/pypy/tool/version.py
+++ b/pypy/tool/version.py
@@ -3,111 +3,139 @@
 from subprocess import Popen, PIPE
 import pypy
 pypydir = os.path.dirname(os.path.abspath(pypy.__file__))
+pypyroot = os.path.dirname(pypydir)
+default_retval = 'PyPy', '?', '?'
+
+def maywarn(err, repo_type='Mercurial'):
+    if not err:
+        return
+
+    from pypy.tool.ansi_print import ansi_log
+    log = py.log.Producer("version")
+    py.log.setconsumer("version", ansi_log)
+    log.WARNING('Errors getting %s information: %s' % (repo_type, err))
 
 def get_repo_version_info(hgexe=None):
     '''Obtain version information by invoking the 'hg' or 'git' commands.'''
-    # TODO: support extracting from .hg_archival.txt
-
-    default_retval = 'PyPy', '?', '?'
-    pypyroot = os.path.abspath(os.path.join(pypydir, '..'))
-
-    def maywarn(err, repo_type='Mercurial'):
-        if not err:
-            return
-
-        from pypy.tool.ansi_print import ansi_log
-        log = py.log.Producer("version")
-        py.log.setconsumer("version", ansi_log)
-        log.WARNING('Errors getting %s information: %s' % (repo_type, err))
 
     # Try to see if we can get info from Git if hgexe is not specified.
     if not hgexe:
         if os.path.isdir(os.path.join(pypyroot, '.git')):
-            gitexe = py.path.local.sysfind('git')
-            if gitexe:
-                try:
-                    p = Popen(
-                        [str(gitexe), 'rev-parse', 'HEAD'],
-                        stdout=PIPE, stderr=PIPE, cwd=pypyroot
-                        )
-                except OSError, e:
-                    maywarn(e, 'Git')
-                    return default_retval
-                if p.wait() != 0:
-                    maywarn(p.stderr.read(), 'Git')
-                    return default_retval
-                revision_id = p.stdout.read().strip()[:12]
-                p = Popen(
-                    [str(gitexe), 'describe', '--tags', '--exact-match'],
-                    stdout=PIPE, stderr=PIPE, cwd=pypyroot
-                    )
-                if p.wait() != 0:
-                    p = Popen(
-                        [str(gitexe), 'branch'], stdout=PIPE, stderr=PIPE,
-                        cwd=pypyroot
-                        )
-                    if p.wait() != 0:
-                        maywarn(p.stderr.read(), 'Git')
-                        return 'PyPy', '?', revision_id
-                    branch = '?'
-                    for line in p.stdout.read().strip().split('\n'):
-                        if line.startswith('* '):
-                            branch = line[1:].strip()
-                            if branch == '(no branch)':
-                                branch = '?'
-                            break
-                    return 'PyPy', branch, revision_id
-                return 'PyPy', p.stdout.read().strip(), revision_id
+            return _get_git_version()
 
     # Fallback to trying Mercurial.
     if hgexe is None:
         hgexe = py.path.local.sysfind('hg')
 
-    if not os.path.isdir(os.path.join(pypyroot, '.hg')):
+    if os.path.isfile(os.path.join(pypyroot, '.hg_archival.txt')):
+        return _get_hg_archive_version(os.path.join(pypyroot, '.hg_archival.txt'))
+    elif not os.path.isdir(os.path.join(pypyroot, '.hg')):
         maywarn('Not running from a Mercurial repository!')
         return default_retval
     elif not hgexe:
         maywarn('Cannot find Mercurial command!')
         return default_retval
     else:
-        env = dict(os.environ)
-        # get Mercurial into scripting mode
-        env['HGPLAIN'] = '1'
-        # disable user configuration, extensions, etc.
-        env['HGRCPATH'] = os.devnull
+        return _get_hg_version(hgexe)
 
-        try:
-            p = Popen([str(hgexe), 'version', '-q'],
-                      stdout=PIPE, stderr=PIPE, env=env)
-        except OSError, e:
-            maywarn(e)
-            return default_retval
 
-        if not p.stdout.read().startswith('Mercurial Distributed SCM'):
-            maywarn('command does not identify itself as Mercurial')
-            return default_retval
+def _get_hg_version(hgexe):
+    env = dict(os.environ)
+    # get Mercurial into scripting mode
+    env['HGPLAIN'] = '1'
+    # disable user configuration, extensions, etc.
+    env['HGRCPATH'] = os.devnull
 
-        p = Popen([str(hgexe), 'id', '-i', pypyroot],
+    try:
+        p = Popen([str(hgexe), 'version', '-q'],
                   stdout=PIPE, stderr=PIPE, env=env)
-        hgid = p.stdout.read().strip()
+    except OSError, e:
+        maywarn(e)
+        return default_retval
+
+    if not p.stdout.read().startswith('Mercurial Distributed SCM'):
+        maywarn('command does not identify itself as Mercurial')
+        return default_retval
+
+    p = Popen([str(hgexe), 'id', '-i', pypyroot],
+              stdout=PIPE, stderr=PIPE, env=env)
+    hgid = p.stdout.read().strip()
+    maywarn(p.stderr.read())
+    if p.wait() != 0:
+        hgid = '?'
+
+    p = Popen([str(hgexe), 'id', '-t', pypyroot],
+              stdout=PIPE, stderr=PIPE, env=env)
+    hgtags = [t for t in p.stdout.read().strip().split() if t != 'tip']
+    maywarn(p.stderr.read())
+    if p.wait() != 0:
+        hgtags = ['?']
+
+    if hgtags:
+        return 'PyPy', hgtags[0], hgid
+    else:
+        # use the branch instead
+        p = Popen([str(hgexe), 'id', '-b', pypyroot],
+                  stdout=PIPE, stderr=PIPE, env=env)
+        hgbranch = p.stdout.read().strip()
         maywarn(p.stderr.read())
+
+        return 'PyPy', hgbranch, hgid
+
+
+def _get_hg_archive_version(path):
+    fp = open(path)
+    try:
+        data = dict(x.split(': ', 1) for x in fp.read().splitlines())
+    finally:
+        fp.close()
+    if 'tag' in data:
+        return 'PyPy', data['tag'], data['node']
+    else:
+        return 'PyPy', data['branch'], data['node']
+
+
+def _get_git_version():
+    #XXX: this function is a untested hack,
+    #     so the git mirror tav made will work
+    gitexe = py.path.local.sysfind('git')
+    if not gitexe:
+        return default_retval
+
+    try:
+        p = Popen(
+            [str(gitexe), 'rev-parse', 'HEAD'],
+            stdout=PIPE, stderr=PIPE, cwd=pypyroot
+            )
+    except OSError, e:
+        maywarn(e, 'Git')
+        return default_retval
+    if p.wait() != 0:
+        maywarn(p.stderr.read(), 'Git')
+        return default_retval
+    revision_id = p.stdout.read().strip()[:12]
+    p = Popen(
+        [str(gitexe), 'describe', '--tags', '--exact-match'],
+        stdout=PIPE, stderr=PIPE, cwd=pypyroot
+        )
+    if p.wait() != 0:
+        p = Popen(
+            [str(gitexe), 'branch'], stdout=PIPE, stderr=PIPE,
+            cwd=pypyroot
+            )
         if p.wait() != 0:
-            hgid = '?'
+            maywarn(p.stderr.read(), 'Git')
+            return 'PyPy', '?', revision_id
+        branch = '?'
+        for line in p.stdout.read().strip().split('\n'):
+            if line.startswith('* '):
+                branch = line[1:].strip()
+                if branch == '(no branch)':
+                    branch = '?'
+                break
+        return 'PyPy', branch, revision_id
+    return 'PyPy', p.stdout.read().strip(), revision_id
 
-        p = Popen([str(hgexe), 'id', '-t', pypyroot],
-                  stdout=PIPE, stderr=PIPE, env=env)
-        hgtags = [t for t in p.stdout.read().strip().split() if t != 'tip']
-        maywarn(p.stderr.read())
-        if p.wait() != 0:
-            hgtags = ['?']
 
-        if hgtags:
-            return 'PyPy', hgtags[0], hgid
-        else:
-            # use the branch instead
-            p = Popen([str(hgexe), 'id', '-b', pypyroot],
-                      stdout=PIPE, stderr=PIPE, env=env)
-            hgbranch = p.stdout.read().strip()
-            maywarn(p.stderr.read())
-
-            return 'PyPy', hgbranch, hgid
+if __name__ == '__main__':
+    print get_repo_version_info()
diff --git a/pypy/translator/c/src/profiling.c b/pypy/translator/c/src/profiling.c
--- a/pypy/translator/c/src/profiling.c
+++ b/pypy/translator/c/src/profiling.c
@@ -29,33 +29,33 @@
     profiling_setup = 0;
   }
 }
-
-#elif defined(_WIN32)
-#include 
-
-DWORD_PTR base_affinity_mask;
-int profiling_setup = 0;
-
-void pypy_setup_profiling() { 
-    if (!profiling_setup) {
-        DWORD_PTR affinity_mask, system_affinity_mask;
-        GetProcessAffinityMask(GetCurrentProcess(),
-            &base_affinity_mask, &system_affinity_mask);
-        affinity_mask = 1;
-        /* Pick one cpu allowed by the system */
-        if (system_affinity_mask)
-            while ((affinity_mask & system_affinity_mask) == 0)
-                affinity_mask <<= 1;
-        SetProcessAffinityMask(GetCurrentProcess(), affinity_mask);
-        profiling_setup = 1;
-    }
-}
-
-void pypy_teardown_profiling() {
-    if (profiling_setup) {
-        SetProcessAffinityMask(GetCurrentProcess(), base_affinity_mask);
-        profiling_setup = 0;
-    }
+
+#elif defined(_WIN32)
+#include 
+
+DWORD_PTR base_affinity_mask;
+int profiling_setup = 0;
+
+void pypy_setup_profiling() { 
+    if (!profiling_setup) {
+        DWORD_PTR affinity_mask, system_affinity_mask;
+        GetProcessAffinityMask(GetCurrentProcess(),
+            &base_affinity_mask, &system_affinity_mask);
+        affinity_mask = 1;
+        /* Pick one cpu allowed by the system */
+        if (system_affinity_mask)
+            while ((affinity_mask & system_affinity_mask) == 0)
+                affinity_mask <<= 1;
+        SetProcessAffinityMask(GetCurrentProcess(), affinity_mask);
+        profiling_setup = 1;
+    }
+}
+
+void pypy_teardown_profiling() {
+    if (profiling_setup) {
+        SetProcessAffinityMask(GetCurrentProcess(), base_affinity_mask);
+        profiling_setup = 0;
+    }
 }
 
 #else
diff --git a/pypy/translator/goal/translate.py b/pypy/translator/goal/translate.py
--- a/pypy/translator/goal/translate.py
+++ b/pypy/translator/goal/translate.py
@@ -293,17 +293,18 @@
             drv.exe_name = targetspec_dic['__name__'] + '-%(backend)s'
 
         # Double check to ensure we are not overwriting the current interpreter
-        try:
-            this_exe = py.path.local(sys.executable).new(ext='')
-            exe_name = drv.compute_exe_name()
-            samefile = this_exe.samefile(exe_name)
-            assert not samefile, (
-                'Output file %s is the currently running '
-                'interpreter (use --output=...)'% exe_name)
-        except EnvironmentError:
-            pass
+        goals = translateconfig.goals
+        if not goals or 'compile' in goals:
+            try:
+                this_exe = py.path.local(sys.executable).new(ext='')
+                exe_name = drv.compute_exe_name()
+                samefile = this_exe.samefile(exe_name)
+                assert not samefile, (
+                    'Output file %s is the currently running '
+                    'interpreter (use --output=...)'% exe_name)
+            except EnvironmentError:
+                pass
 
-        goals = translateconfig.goals
         try:
             drv.proceed(goals)
         finally:
diff --git a/pypy/translator/jvm/builtin.py b/pypy/translator/jvm/builtin.py
--- a/pypy/translator/jvm/builtin.py
+++ b/pypy/translator/jvm/builtin.py
@@ -84,6 +84,9 @@
     (ootype.StringBuilder.__class__, "ll_build"):
     jvm.Method.v(jStringBuilder, "toString", (), jString),
 
+    (ootype.StringBuilder.__class__, "ll_getlength"):
+    jvm.Method.v(jStringBuilder, "length", (), jInt),
+
     (ootype.String.__class__, "ll_hash"):
     jvm.Method.v(jString, "hashCode", (), jInt),
 
diff --git a/pypy/translator/jvm/database.py b/pypy/translator/jvm/database.py
--- a/pypy/translator/jvm/database.py
+++ b/pypy/translator/jvm/database.py
@@ -358,7 +358,7 @@
         ootype.Unsigned:jvm.PYPYSERIALIZEUINT,
         ootype.SignedLongLong:jvm.LONGTOSTRINGL,
         ootype.UnsignedLongLong: jvm.PYPYSERIALIZEULONG,
-        ootype.Float:jvm.DOUBLETOSTRINGD,
+        ootype.Float:jvm.PYPYSERIALIZEDOUBLE,
         ootype.Bool:jvm.PYPYSERIALIZEBOOLEAN,
         ootype.Void:jvm.PYPYSERIALIZEVOID,
         ootype.Char:jvm.PYPYESCAPEDCHAR,
diff --git a/pypy/translator/jvm/metavm.py b/pypy/translator/jvm/metavm.py
--- a/pypy/translator/jvm/metavm.py
+++ b/pypy/translator/jvm/metavm.py
@@ -92,6 +92,7 @@
 CASTS = {
 #   FROM                      TO
     (ootype.Signed,           ootype.UnsignedLongLong): jvm.I2L,
+    (ootype.Unsigned,         ootype.UnsignedLongLong): jvm.I2L,
     (ootype.SignedLongLong,   ootype.Signed):           jvm.L2I,
     (ootype.UnsignedLongLong, ootype.Unsigned):         jvm.L2I,
     (ootype.UnsignedLongLong, ootype.Signed):           jvm.L2I,
diff --git a/pypy/translator/jvm/opcodes.py b/pypy/translator/jvm/opcodes.py
--- a/pypy/translator/jvm/opcodes.py
+++ b/pypy/translator/jvm/opcodes.py
@@ -101,6 +101,7 @@
     'jit_force_virtualizable':  Ignore,
     'jit_force_virtual':        DoNothing,
     'jit_force_quasi_immutable': Ignore,
+    'jit_is_virtual':           PushPrimitive(ootype.Bool, False),
 
     'debug_assert':              [], # TODO: implement?
     'debug_start_traceback':    Ignore,
diff --git a/pypy/translator/jvm/src/pypy/PyPy.java b/pypy/translator/jvm/src/pypy/PyPy.java
--- a/pypy/translator/jvm/src/pypy/PyPy.java
+++ b/pypy/translator/jvm/src/pypy/PyPy.java
@@ -283,6 +283,14 @@
         }
     }
 
+    public double pypy__longlong2float(long l) {
+        return Double.longBitsToDouble(l);
+    }
+
+    public long pypy__float2longlong(double d) {
+        return Double.doubleToRawLongBits(d);
+    }
+
     public double ooparse_float(String s) {
         try {
             return Double.parseDouble(s);
@@ -353,6 +361,19 @@
             return "False";
     }
 
+    public static String serialize_double(double d) {
+        if (Double.isNaN(d)) {
+            return "float(\"nan\")";
+        } else if (Double.isInfinite(d)) {
+            if (d > 0)
+                return "float(\"inf\")";
+            else
+                return "float(\"-inf\")";
+        } else {
+            return Double.toString(d);
+        }
+    }
+
     private static String format_char(char c) {
         String res = "\\x";
         if (c <= 0x0F) res = res + "0";
diff --git a/pypy/translator/jvm/test/runtest.py b/pypy/translator/jvm/test/runtest.py
--- a/pypy/translator/jvm/test/runtest.py
+++ b/pypy/translator/jvm/test/runtest.py
@@ -56,6 +56,7 @@
 
 # CLI could-be duplicate
 class JvmGeneratedSourceWrapper(object):
+
     def __init__(self, gensrc):
         """ gensrc is an instance of JvmGeneratedSource """
         self.gensrc = gensrc
diff --git a/pypy/translator/jvm/test/test_builder.py b/pypy/translator/jvm/test/test_builder.py
new file mode 100644
--- /dev/null
+++ b/pypy/translator/jvm/test/test_builder.py
@@ -0,0 +1,7 @@
+from pypy.translator.jvm.test.runtest import JvmTest
+from pypy.rpython.test.test_rbuilder import BaseTestStringBuilder
+import py
+
+class TestJvmStringBuilder(JvmTest, BaseTestStringBuilder):
+    def test_append_charpsize(self):
+        py.test.skip("append_charpsize(): not implemented on ootype")
diff --git a/pypy/translator/jvm/test/test_longlong2float.py b/pypy/translator/jvm/test/test_longlong2float.py
new file mode 100644
--- /dev/null
+++ b/pypy/translator/jvm/test/test_longlong2float.py
@@ -0,0 +1,20 @@
+from pypy.translator.jvm.test.runtest import JvmTest
+from pypy.rlib.longlong2float import *
+from pypy.rlib.test.test_longlong2float import enum_floats
+from pypy.rlib.test.test_longlong2float import fn as float2longlong2float
+import py
+
+class TestLongLong2Float(JvmTest):
+
+    def test_float2longlong_and_longlong2float(self):
+        def func(f):
+            return float2longlong2float(f)
+
+        for f in enum_floats():
+            assert repr(f) == repr(self.interpret(func, [f]))
+
+    def test_uint2singlefloat(self):
+        py.test.skip("uint2singlefloat is not implemented in ootype")
+
+    def test_singlefloat2uint(self):
+        py.test.skip("singlefloat2uint is not implemented in ootype")
diff --git a/pypy/translator/jvm/typesystem.py b/pypy/translator/jvm/typesystem.py
--- a/pypy/translator/jvm/typesystem.py
+++ b/pypy/translator/jvm/typesystem.py
@@ -955,6 +955,7 @@
 PYPYSERIALIZEUINT  =    Method.s(jPyPy, 'serialize_uint', (jInt,), jString)
 PYPYSERIALIZEULONG =    Method.s(jPyPy, 'serialize_ulonglong', (jLong,),jString)
 PYPYSERIALIZEVOID =     Method.s(jPyPy, 'serialize_void', (), jString)
+PYPYSERIALIZEDOUBLE =   Method.s(jPyPy, 'serialize_double', (jDouble,), jString)
 PYPYESCAPEDCHAR =       Method.s(jPyPy, 'escaped_char', (jChar,), jString)
 PYPYESCAPEDUNICHAR =    Method.s(jPyPy, 'escaped_unichar', (jChar,), jString)
 PYPYESCAPEDSTRING =     Method.s(jPyPy, 'escaped_string', (jString,), jString)
diff --git a/pypy/translator/oosupport/test_template/cast.py b/pypy/translator/oosupport/test_template/cast.py
--- a/pypy/translator/oosupport/test_template/cast.py
+++ b/pypy/translator/oosupport/test_template/cast.py
@@ -13,6 +13,9 @@
 def to_longlong(x):
     return r_longlong(x)
 
+def to_ulonglong(x):
+    return r_ulonglong(x)
+
 def uint_to_int(x):
     return intmask(x)
 
@@ -56,6 +59,9 @@
     def test_unsignedlonglong_to_unsigned4(self):
         self.check(to_uint, [r_ulonglong(18446744073709551615l)]) # max 64 bit num
 
+    def test_unsigned_to_usignedlonglong(self):
+        self.check(to_ulonglong, [r_uint(42)])
+
     def test_uint_to_int(self):
         self.check(uint_to_int, [r_uint(sys.maxint+1)])
 

From noreply at buildbot.pypy.org  Sun Jan 22 20:57:16 2012
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sun, 22 Jan 2012 20:57:16 +0100 (CET)
Subject: [pypy-commit] pypy merge-2.7.2: Update _ctypes_test.c from CPython
	2.7.2
Message-ID: <20120122195716.060E1821FA@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: merge-2.7.2
Changeset: r51664:ead17cd9edbf
Date: 2012-01-22 20:49 +0100
http://bitbucket.org/pypy/pypy/changeset/ead17cd9edbf/

Log:	Update _ctypes_test.c from CPython 2.7.2

diff --git a/lib_pypy/_ctypes_test.c b/lib_pypy/_ctypes_test.c
--- a/lib_pypy/_ctypes_test.c
+++ b/lib_pypy/_ctypes_test.c
@@ -26,6 +26,20 @@
 
 /* some functions handy for testing */
 
+EXPORT(int)
+_testfunc_cbk_reg_int(int a, int b, int c, int d, int e,
+                      int (*func)(int, int, int, int, int))
+{
+    return func(a*a, b*b, c*c, d*d, e*e);
+}
+
+EXPORT(double)
+_testfunc_cbk_reg_double(double a, double b, double c, double d, double e,
+                         double (*func)(double, double, double, double, double))
+{
+    return func(a*a, b*b, c*c, d*d, e*e);
+}
+
 EXPORT(void)testfunc_array(int values[4])
 {
     printf("testfunc_array %d %d %d %d\n",

From noreply at buildbot.pypy.org  Sun Jan 22 20:57:17 2012
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sun, 22 Jan 2012 20:57:17 +0100 (CET)
Subject: [pypy-commit] pypy merge-2.7.2: json: Rename some functions in our
 pure-python implementation:
Message-ID: <20120122195717.38E6E821FA@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: merge-2.7.2
Changeset: r51665:c000c3332cb0
Date: 2012-01-22 20:56 +0100
http://bitbucket.org/pypy/pypy/changeset/c000c3332cb0/

Log:	json: Rename some functions in our pure-python implementation: in
	CPython, encode_basestring() is supposed to add double quotes.

diff --git a/lib-python/modified-2.7/json/encoder.py b/lib-python/modified-2.7/json/encoder.py
--- a/lib-python/modified-2.7/json/encoder.py
+++ b/lib-python/modified-2.7/json/encoder.py
@@ -23,15 +23,16 @@
 INFINITY = float('1e66666')
 FLOAT_REPR = repr
 
-def encode_basestring(s):
+def raw_encode_basestring(s):
     """Return a JSON representation of a Python string
 
     """
     def replace(match):
         return ESCAPE_DCT[match.group(0)]
     return ESCAPE.sub(replace, s)
+encode_basestring = lambda s: '"' + raw_encode_basestring(s) + '"'
 
-def encode_basestring_ascii(s):
+def raw_encode_basestring_ascii(s):
     """Return an ASCII-only JSON representation of a Python string
 
     """
@@ -54,8 +55,8 @@
     if ESCAPE_ASCII.search(s):
         return str(ESCAPE_ASCII.sub(replace, s))
     return s
-py_encode_basestring_ascii = lambda s: '"' + encode_basestring_ascii(s) + '"'
-c_encode_basestring_ascii = None
+encode_basestring_ascii = lambda s: '"' + raw_encode_basestring_ascii(s) + '"'
+
 
 class JSONEncoder(object):
     """Extensible JSON  encoder for Python data structures.
@@ -137,9 +138,9 @@
         self.skipkeys = skipkeys
         self.ensure_ascii = ensure_ascii
         if ensure_ascii:
-            self.encoder = encode_basestring_ascii
+            self.encoder = raw_encode_basestring_ascii
         else:
-            self.encoder = encode_basestring
+            self.encoder = raw_encode_basestring
         if encoding != 'utf-8':
             orig_encoder = self.encoder
             def encoder(o):

From noreply at buildbot.pypy.org  Sun Jan 22 21:54:07 2012
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sun, 22 Jan 2012 21:54:07 +0100 (CET)
Subject: [pypy-commit] pypy merge-2.7.2: Allow both str.__add__ to raise
 TypeError or return NotImplemented.
Message-ID: <20120122205407.9B85C821FA@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: merge-2.7.2
Changeset: r51666:c84666d936d2
Date: 2012-01-22 21:10 +0100
http://bitbucket.org/pypy/pypy/changeset/c84666d936d2/

Log:	Allow both str.__add__ to raise TypeError or return NotImplemented.

	IMO the point of the test is to check that str.__add__ does not
	crash or use str() instead.

diff --git a/lib-python/modified-2.7/test/test_descr.py b/lib-python/modified-2.7/test/test_descr.py
--- a/lib-python/modified-2.7/test/test_descr.py
+++ b/lib-python/modified-2.7/test/test_descr.py
@@ -4592,8 +4592,12 @@
             str.split(fake_str)
 
         # call a slot wrapper descriptor
-        with self.assertRaises(TypeError):
-            str.__add__(fake_str, "abc")
+        try:
+            r = str.__add__(fake_str, "abc")
+        except TypeError:
+            pass
+        else:
+            self.assertEqual(r, NotImplemented)
 
 
 class DictProxyTests(unittest.TestCase):

From noreply at buildbot.pypy.org  Sun Jan 22 22:10:29 2012
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sun, 22 Jan 2012 22:10:29 +0100 (CET)
Subject: [pypy-commit] buildbot default: Added the numready script.
Message-ID: <20120122211029.4DC65821FA@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor 
Branch: 
Changeset: r608:4232c21cfab6
Date: 2012-01-22 15:10 -0600
http://bitbucket.org/pypy/buildbot/changeset/4232c21cfab6/

Log:	Added the numready script.

diff --git a/numready.py b/numready.py
new file mode 100644
--- /dev/null
+++ b/numready.py
@@ -0,0 +1,205 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+This should be run under PyPy.
+"""
+
+import platform
+import subprocess
+import sys
+import tempfile
+import webbrowser
+from collections import OrderedDict
+
+import jinja2
+
+
+MODULE_SEARCH_CODE = """
+import types
+import {modname} as numpy
+
+for name in dir(numpy):
+    if name.startswith("_"):
+        continue
+    obj = getattr(numpy, name)
+    kind = "{kinds[UNKNOWN]}"
+    if isinstance(obj, types.TypeType):
+        kind = "{kinds[TYPE]}"
+    print kind, ":", name
+"""
+
+ATTR_SEARCH_CODE = """
+import types
+import {modname} as numpy
+
+obj = getattr(numpy, "{name}")
+for name in dir(obj):
+    if name.startswith("_"):
+        continue
+    sub_obj = getattr(obj, name)
+    kind = "{kinds[UNKNOWN]}"
+    if isinstance(sub_obj, types.TypeType):
+        kind = "{kinds[TYPE]}"
+    print kind, ":", name
+"""
+
+KINDS = {
+    "UNKNOWN": "U",
+    "TYPE": "T",
+}
+
+PAGE_TEMPLATE = u"""
+
+
+    
+        NumPyPy Status
+        
+        
+    
+    
+        NumPyPy Status
+        
+            
+                
+                    
+                    
+                
+            
+            
+                {% for item in all_items %}
+                    
+                        
+                        
+                    
+                    {% if item.subitems %}
+                        {% for item in item.subitems %}
+                            
+                                
+                                
+                            
+                        {% endfor %}
+                    {% endif %}
+                {% endfor %}
+            
+        PyPy
{{ item.name }} {% if item.pypy_exists %}✔{% else %}✖{% endif %}
   {{ item.name }} {% if item.pypy_exists %}✔{% else %}✖{% endif %}
+    
+
+"""
+
+class SearchableSet(object):
+    def __init__(self, items=()):
+        self._items = {}
+        for item in items:
+            self.add(item)
+
+    def __iter__(self):
+        return iter(self._items)
+
+    def __contains__(self, other):
+        return other in self._items
+
+    def __getitem__(self, idx):
+        return self._items[idx]
+
+    def add(self, item):
+        self._items[item] = item
+
+class Item(object):
+    def __init__(self, name, kind, subitems=None):
+        self.name = name
+        self.kind = kind
+        self.subitems = subitems
+
+    def __hash__(self):
+        return hash(self.name)
+
+    def __eq__(self, other):
+        return self.name == other.name
+
+
+class ItemStatus(object):
+    def __init__(self, name, pypy_exists, subitems=None):
+        self.name = name
+        self.pypy_exists = pypy_exists
+        self.subitems = subitems
+
+    def __lt__(self, other):
+        return self.name < other.name
+
+def find_numpy_attrs(python, modname, name):
+    lines = subprocess.check_output(
+        [python, "-c", ATTR_SEARCH_CODE.format(modname=modname, kinds=KINDS, name=name)]
+    ).splitlines()
+    items = SearchableSet()
+    for line in lines:
+        kind, name = line.split(" : ", 1)
+        items.add(Item(name, kind))
+    return items
+
+def find_numpy_items(python, modname="numpy"):
+    lines = subprocess.check_output(
+        [python, "-c", MODULE_SEARCH_CODE.format(modname=modname, kinds=KINDS)]
+    ).splitlines()
+    items = SearchableSet()
+    for line in lines:
+        kind, name = line.split(" : ", 1)
+        subitems = None
+        if kind == KINDS["TYPE"]:
+            subitems = find_numpy_attrs(python, modname, name)
+        items.add(Item(name, kind, subitems))
+    return items
+
+def main(argv):
+    assert platform.python_implementation() == "PyPy"
+
+    cpy_items = find_numpy_items("/usr/bin/python")
+    pypy_items = find_numpy_items(sys.executable, "numpypy")
+    all_items = []
+
+    for item in cpy_items:
+        pypy_exists = item in pypy_items
+        subitems = None
+        if item.subitems:
+            subitems = []
+            for sub in item.subitems:
+                subitems.append(
+                    ItemStatus(sub.name, pypy_exists=pypy_exists and pypy_items[item].subitems and sub in pypy_items[item].subitems)
+                )
+            subitems = sorted(subitems)
+        all_items.append(
+            ItemStatus(item.name, pypy_exists=item in pypy_items, subitems=subitems)
+        )
+
+    html = jinja2.Template(PAGE_TEMPLATE).render(all_items=sorted(all_items))
+    with tempfile.NamedTemporaryFile(delete=False) as f:
+        f.write(html.encode("utf-8"))
+    webbrowser.open_new_tab(f.name)
+
+
+if __name__ == '__main__':
+    main(sys.argv)
\ No newline at end of file

From noreply at buildbot.pypy.org  Sun Jan 22 22:26:52 2012
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sun, 22 Jan 2012 22:26:52 +0100 (CET)
Subject: [pypy-commit] pypy merge-2.7.2: Fix for mmap when trying to specify
 an offset that's past the end of a file.
Message-ID: <20120122212652.BF356821FA@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor 
Branch: merge-2.7.2
Changeset: r51667:b0b180ee94ea
Date: 2012-01-22 15:26 -0600
http://bitbucket.org/pypy/pypy/changeset/b0b180ee94ea/

Log:	Fix for mmap when trying to specify an offset that's past the end of
	a file.

diff --git a/pypy/module/mmap/test/test_mmap.py b/pypy/module/mmap/test/test_mmap.py
--- a/pypy/module/mmap/test/test_mmap.py
+++ b/pypy/module/mmap/test/test_mmap.py
@@ -8,7 +8,7 @@
         space = gettestobjspace(usemodules=('mmap',))
         cls.space = space
         cls.w_tmpname = space.wrap(str(udir.join('mmap-')))
-    
+
     def test_page_size(self):
         import mmap
         assert mmap.PAGESIZE > 0
@@ -43,7 +43,7 @@
 
         raises(TypeError, mmap, "foo")
         raises(TypeError, mmap, 0, "foo")
-             
+
         if os.name == "posix":
             raises(ValueError, mmap, 0, 1, 2, 3, 4)
             raises(TypeError, mmap, 0, 1, 2, 3, "foo", 5)
@@ -72,7 +72,7 @@
 
         from mmap import mmap
         f = open(self.tmpname + "a", "w+")
-        
+
         f.write("c")
         f.flush()
         raises(ValueError, mmap, f.fileno(), 123)
@@ -81,18 +81,18 @@
     def test_create(self):
         from mmap import mmap
         f = open(self.tmpname + "b", "w+")
-        
+
         f.write("c")
         f.flush()
         m = mmap(f.fileno(), 1)
         assert m.read(99) == "c"
-        
+
         f.close()
 
     def test_close(self):
         from mmap import mmap
         f = open(self.tmpname + "c", "w+")
-        
+
         f.write("c")
         f.flush()
         m = mmap(f.fileno(), 1)
@@ -131,7 +131,7 @@
     def test_read(self):
         from mmap import mmap
         f = open(self.tmpname + "f", "w+")
-        
+
         f.write("foobar")
         f.flush()
         m = mmap(f.fileno(), 6)
@@ -217,7 +217,7 @@
     def test_is_modifiable(self):
         import mmap
         f = open(self.tmpname + "h", "w+")
-        
+
         f.write("foobar")
         f.flush()
         m = mmap.mmap(f.fileno(), 6, access=mmap.ACCESS_READ)
@@ -229,7 +229,7 @@
     def test_seek(self):
         from mmap import mmap
         f = open(self.tmpname + "i", "w+")
-        
+
         f.write("foobar")
         f.flush()
         m = mmap(f.fileno(), 6)
@@ -270,7 +270,7 @@
     def test_write_byte(self):
         import mmap
         f = open(self.tmpname + "k", "w+")
-        
+
         f.write("foobar")
         f.flush()
         m = mmap.mmap(f.fileno(), 6, access=mmap.ACCESS_READ)
@@ -286,7 +286,7 @@
     def test_size(self):
         from mmap import mmap
         f = open(self.tmpname + "l", "w+")
-        
+
         f.write("foobar")
         f.flush()
         m = mmap(f.fileno(), 5)
@@ -297,7 +297,7 @@
     def test_tell(self):
         from mmap import mmap
         f = open(self.tmpname + "m", "w+")
-        
+
         f.write("c")
         f.flush()
         m = mmap(f.fileno(), 1)
@@ -308,7 +308,7 @@
     def test_flush(self):
         from mmap import mmap
         f = open(self.tmpname + "n", "w+")
-        
+
         f.write("foobar")
         f.flush()
         m = mmap(f.fileno(), 6)
@@ -319,10 +319,18 @@
         m.close()
         f.close()
 
+    def test_length_0_large_offset(self):
+        import mmap
+
+        with open(self.tmpname, "wb") as f:
+            f.write(115699 * 'm')
+        with open(self.tmpname, "w+b") as f:
+            raises(ValueError, mmap.mmap, f.fileno(), 0, offset=2147418112)
+
     def test_move(self):
         import mmap
         f = open(self.tmpname + "o", "w+")
-        
+
         f.write("foobar")
         f.flush()
         m = mmap.mmap(f.fileno(), 6, access=mmap.ACCESS_READ)
@@ -339,15 +347,15 @@
         assert a == "frarar"
         m.close()
         f.close()
-    
+
     def test_resize(self):
         import sys
         if ("darwin" in sys.platform) or ("freebsd" in sys.platform):
             skip("resize does not work under OSX or FreeBSD")
-        
+
         import mmap
         import os
-        
+
         f = open(self.tmpname + "p", "w+")
         f.write("foobar")
         f.flush()
@@ -368,10 +376,10 @@
         import sys
         if ("darwin" not in sys.platform) and ("freebsd" not in sys.platform):
             skip("resize works under not OSX or FreeBSD")
-        
+
         import mmap
         import os
-        
+
         f = open(self.tmpname + "p", "w+")
         f.write("foobar")
         f.flush()
@@ -388,7 +396,7 @@
 
     def test_len(self):
         from mmap import mmap
-        
+
         f = open(self.tmpname + "q", "w+")
         f.write("foobar")
         f.flush()
@@ -397,14 +405,14 @@
         assert len(m) == 6
         m.close()
         f.close()
-     
+
     def test_get_item(self):
         from mmap import mmap
-        
+
         f = open(self.tmpname + "r", "w+")
         f.write("foobar")
         f.flush()
-        
+
         m = mmap(f.fileno(), 6)
         fn = lambda: m["foo"]
         raises(TypeError, fn)
@@ -416,10 +424,10 @@
         assert m[4:1:-2] == 'ao'
         m.close()
         f.close()
-    
+
     def test_set_item(self):
         import mmap
-        
+
         f = open(self.tmpname + "s", "w+")
         f.write("foobar")
         f.flush()
@@ -455,14 +463,14 @@
         assert data == "yxxBaR"
         m.close()
         f.close()
-    
+
     def test_del_item(self):
         from mmap import mmap
-        
+
         f = open(self.tmpname + "t", "w+")
         f.write("foobar")
         f.flush()
-        
+
         m = mmap(f.fileno(), 6)
         def fn(): del m["foo"]
         raises(TypeError, fn)
@@ -475,11 +483,11 @@
 
     def test_concatenation(self):
         from mmap import mmap
-        
+
         f = open(self.tmpname + "u", "w+")
         f.write("foobar")
         f.flush()
-        
+
         m = mmap(f.fileno(), 6)
         def fn(): m + 1
         raises((SystemError, TypeError), fn)     # SystemError is in CPython,
@@ -492,11 +500,11 @@
 
     def test_repeatition(self):
         from mmap import mmap
-        
+
         f = open(self.tmpname + "v", "w+")
         f.write("foobar")
         f.flush()
-        
+
         m = mmap(f.fileno(), 6)
         def fn(): m * 1
         raises((SystemError, TypeError), fn)      # SystemError is in CPython,
@@ -506,14 +514,14 @@
         raises((SystemError, TypeError), fn)
         m.close()
         f.close()
-         
+
     def test_slicing(self):
         from mmap import mmap
 
         f = open(self.tmpname + "v", "w+")
         f.write("foobar")
         f.flush()
-        
+
         f.seek(0)
         m = mmap(f.fileno(), 6)
         assert m[-3:7] == "bar"
@@ -589,11 +597,11 @@
         from mmap import PAGESIZE
         import sys
         import os
-        
+
         filename = self.tmpname + "w"
-    
+
         f = open(filename, "w+")
-    
+
         # write 2 pages worth of data to the file
         f.write('\0' * PAGESIZE)
         f.write('foo')
@@ -601,24 +609,24 @@
         f.flush()
         m = mmap.mmap(f.fileno(), 2 * PAGESIZE)
         f.close()
-    
+
         # sanity checks
         assert m.find("foo") == PAGESIZE
         assert len(m) == 2 * PAGESIZE
         assert m[0] == '\0'
         assert m[0:3] == '\0\0\0'
-    
+
         # modify the file's content
         m[0] = '3'
         m[PAGESIZE+3:PAGESIZE+3+3] = 'bar'
-    
+
         # check that the modification worked
         assert m[0] == '3'
         assert m[0:3] == '3\0\0'
         assert m[PAGESIZE-1:PAGESIZE+7] == '\0foobar\0'
 
         m.flush()
-    
+
         # test seeking around
         m.seek(0,0)
         assert m.tell() == 0
@@ -626,28 +634,28 @@
         assert m.tell() == 42
         m.seek(0, 2)
         assert m.tell() == len(m)
-        
+
         raises(ValueError, m.seek, -1)
         raises(ValueError, m.seek, 1, 2)
         raises(ValueError, m.seek, -len(m) - 1, 2)
-        
+
         # try resizing map
         if not (("darwin" in sys.platform) or ("freebsd" in sys.platform)):
             m.resize(512)
-        
+
             assert len(m) == 512
             raises(ValueError, m.seek, 513, 0)
-            
+
             # check that the underlying file is truncated too
             f = open(filename)
             f.seek(0, 2)
             assert f.tell() == 512
             f.close()
             assert m.size() == 512
-        
+
         m.close()
         f.close()
-        
+
         # test access=ACCESS_READ
         mapsize = 10
         f = open(filename, "wb")
@@ -667,21 +675,21 @@
         if not (("darwin" in sys.platform) or ("freebsd" in sys.platform)):
             raises(TypeError, m.resize, 2 * mapsize)
             assert open(filename, "rb").read() == 'a' * mapsize
-        
+
         # opening with size too big
         f = open(filename, "r+b")
         if not os.name == "nt":
             # this should work under windows
             raises(ValueError, mmap.mmap, f.fileno(), mapsize + 1)
         f.close()
-        
+
         # if _MS_WINDOWS:
         #     # repair damage from the resizing test.
         #     f = open(filename, 'r+b')
         #     f.truncate(mapsize)
         #     f.close()
         m.close()
-        
+
         # test access=ACCESS_WRITE"
         f = open(filename, "r+b")
         m = mmap.mmap(f.fileno(), mapsize, access=mmap.ACCESS_WRITE)
@@ -696,7 +704,7 @@
         stuff = f.read()
         f.close()
         assert stuff == 'c' * mapsize
-        
+
         # test access=ACCESS_COPY
         f = open(filename, "r+b")
         m = mmap.mmap(f.fileno(), mapsize, access=mmap.ACCESS_COPY)
@@ -710,12 +718,12 @@
             raises(TypeError, m.resize, 2 * mapsize)
         m.close()
         f.close()
-        
+
         # test invalid access
         f = open(filename, "r+b")
         raises(ValueError, mmap.mmap, f.fileno(), mapsize, access=4)
         f.close()
-        
+
         # test incompatible parameters
         if os.name == "posix":
             f = open(filename, "r+b")
@@ -723,10 +731,10 @@
                 prot=mmap.PROT_READ, access=mmap.ACCESS_WRITE)
             f.close()
 
-        
+
         # bad file descriptor
         raises(EnvironmentError, mmap.mmap, -2, 4096)
-        
+
         # do a tougher .find() test.  SF bug 515943 pointed out that, in 2.2,
         # searching for data with embedded \0 bytes didn't work.
         f = open(filename, 'w+')
@@ -736,14 +744,14 @@
         f.flush()
         m = mmap.mmap(f.fileno(), n)
         f.close()
-        
+
         for start in range(n + 1):
             for finish in range(start, n + 1):
                 sl = data[start:finish]
                 assert m.find(sl) == data.find(sl)
                 assert m.find(sl + 'x') ==  -1
         m.close()
-        
+
         # test mapping of entire file by passing 0 for map length
         f = open(filename, "w+")
         f.write(2**16 * 'm')
@@ -754,7 +762,7 @@
         assert m.read(2**16) == 2**16 * "m"
         m.close()
         f.close()
-        
+
         # make move works everywhere (64-bit format problem earlier)
         f = open(filename, 'w+')
         f.write("ABCDEabcde")
diff --git a/pypy/rlib/rmmap.py b/pypy/rlib/rmmap.py
--- a/pypy/rlib/rmmap.py
+++ b/pypy/rlib/rmmap.py
@@ -43,7 +43,7 @@
 constants = {}
 if _POSIX:
     # constants, look in sys/mman.h and platform docs for the meaning
-    # some constants are linux only so they will be correctly exposed outside 
+    # some constants are linux only so they will be correctly exposed outside
     # depending on the OS
     constant_names = ['MAP_SHARED', 'MAP_PRIVATE',
                       'PROT_READ', 'PROT_WRITE',
@@ -136,7 +136,7 @@
             )
 
         SYSINFO_UNION = rffi.CStruct(
-            'union SYSINFO_UNION', 
+            'union SYSINFO_UNION',
                 ("dwOemId", DWORD),
                 ("_struct_", SYSINFO_STRUCT),
             )
@@ -250,7 +250,7 @@
         elif _POSIX:
             self.fd = -1
             self.closed = False
-    
+
     def check_valid(self):
         if _MS_WINDOWS:
             to_close = self.map_handle == INVALID_HANDLE
@@ -259,11 +259,11 @@
 
         if to_close:
             raise RValueError("map closed or invalid")
-    
+
     def check_writeable(self):
         if not (self.access != ACCESS_READ):
             raise RTypeError("mmap can't modify a readonly memory map.")
-    
+
     def check_resizeable(self):
         if not (self.access == ACCESS_WRITE or self.access == _ACCESS_DEFAULT):
             raise RTypeError("mmap can't resize a readonly or copy-on-write memory map.")
@@ -273,7 +273,7 @@
         assert size >= 0
         self.data = data
         self.size = size
-    
+
     def close(self):
         if _MS_WINDOWS:
             if self.size > 0:
@@ -302,7 +302,7 @@
 
     def unmapview(self):
         UnmapViewOfFile(self.getptr(0))
-    
+
     def read_byte(self):
         self.check_valid()
 
@@ -312,7 +312,7 @@
             return value
         else:
             raise RValueError("read byte out of range")
-    
+
     def readline(self):
         self.check_valid()
 
@@ -327,7 +327,7 @@
         res = "".join([data[i] for i in range(self.pos, eol)])
         self.pos += len(res)
         return res
-    
+
     def read(self, num=-1):
         self.check_valid()
 
@@ -389,10 +389,10 @@
 
     def seek(self, pos, whence=0):
         self.check_valid()
-        
+
         dist = pos
         how = whence
-        
+
         if how == 0: # relative to start
             where = dist
         elif how == 1: # relative to current position
@@ -404,16 +404,16 @@
 
         if not (0 <= where <= self.size):
             raise RValueError("seek out of range")
-        
+
         self.pos = where
-    
+
     def tell(self):
         self.check_valid()
         return self.pos
-    
+
     def file_size(self):
         self.check_valid()
-        
+
         size = self.size
         if _MS_WINDOWS:
             if self.file_handle != INVALID_HANDLE:
@@ -433,11 +433,11 @@
             else:
                 size = int(size)
         return size
-    
+
     def write(self, data):
-        self.check_valid()        
+        self.check_valid()
         self.check_writeable()
-        
+
         data_len = len(data)
         if self.pos + data_len > self.size:
             raise RValueError("data out of range")
@@ -447,10 +447,10 @@
         for i in range(data_len):
             internaldata[start+i] = data[i]
         self.pos = start + data_len
-    
+
     def write_byte(self, byte):
         self.check_valid()
-        
+
         if len(byte) != 1:
             raise RTypeError("write_byte() argument must be char")
 
@@ -491,14 +491,14 @@
                 if res == -1:
                     errno = rposix.get_errno()
                     raise OSError(errno, os.strerror(errno))
-        
+
         return 0
-    
+
     def move(self, dest, src, count):
         self.check_valid()
-        
+
         self.check_writeable()
-        
+
         # check boundings
         if (src < 0 or dest < 0 or count < 0 or
             src + count > self.size or dest + count > self.size):
@@ -507,19 +507,19 @@
         datasrc = self.getptr(src)
         datadest = self.getptr(dest)
         c_memmove(datadest, datasrc, count)
-    
+
     def resize(self, newsize):
         self.check_valid()
-        
+
         self.check_resizeable()
-        
+
         if _POSIX:
             if not has_mremap:
                 raise RValueError("mmap: resizing not available--no mremap()")
-            
+
             # resize the underlying file first
             os.ftruncate(self.fd, self.offset + newsize)
-                
+
             # now resize the mmap
             newdata = c_mremap(self.getptr(0), self.size, newsize,
                                MREMAP_MAYMOVE or 0)
@@ -573,9 +573,9 @@
 
     def len(self):
         self.check_valid()
-        
+
         return self.size
-    
+
     def getitem(self, index):
         self.check_valid()
         # simplified version, for rpython
@@ -650,6 +650,8 @@
                 size = int(size)
             if stat.S_ISREG(mode):
                 if map_size == 0:
+                    if offset > st[stat.ST_SIZE]:
+                        raise RValueError("mmap offset is greater than file size")
                     map_size = size
                 elif map_size > size:
                     raise RValueError("mmap length is greater than file size")
@@ -673,7 +675,7 @@
         if res == rffi.cast(PTR, -1):
             errno = rposix.get_errno()
             raise OSError(errno, os.strerror(errno))
-        
+
         m.setdata(res, map_size)
         return m
 
@@ -704,7 +706,7 @@
     alloc._annenforceargs_ = (int,)
 
     free = c_munmap_safe
-    
+
 elif _MS_WINDOWS:
     def mmap(fileno, length, tagname="", access=_ACCESS_DEFAULT, offset=0):
         # check size boundaries
@@ -712,11 +714,11 @@
         map_size = length
         if offset < 0:
             raise RValueError("negative offset")
-        
+
         flProtect = 0
         dwDesiredAccess = 0
         fh = NULL_HANDLE
-        
+
         if access == ACCESS_READ:
             flProtect = PAGE_READONLY
             dwDesiredAccess = FILE_MAP_READ
@@ -728,7 +730,7 @@
             dwDesiredAccess = FILE_MAP_COPY
         else:
             raise RValueError("mmap invalid access parameter.")
-        
+
         # assume -1 and 0 both mean invalid file descriptor
         # to 'anonymously' map memory.
         if fileno != -1 and fileno != 0:
@@ -739,7 +741,7 @@
             # Win9x appears to need us seeked to zero
             # SEEK_SET = 0
             # libc._lseek(fileno, 0, SEEK_SET)
-        
+
         m = MMap(access, offset)
         m.file_handle = INVALID_HANDLE
         m.map_handle = INVALID_HANDLE
@@ -752,7 +754,7 @@
                 res = DuplicateHandle(GetCurrentProcess(), # source process handle
                                       fh, # handle to be duplicated
                                       GetCurrentProcess(), # target process handle
-                                      handle_ref, # result  
+                                      handle_ref, # result
                                       0, # access - ignored due to options value
                                       False, # inherited by child procs?
                                       DUPLICATE_SAME_ACCESS) # options
@@ -761,7 +763,7 @@
                 m.file_handle = handle_ref[0]
             finally:
                 lltype.free(handle_ref, flavor='raw')
-            
+
             if not map_size:
                 low, high = _get_file_size(fh)
                 if _64BIT:
@@ -775,7 +777,7 @@
 
         if tagname:
             m.tagname = tagname
-        
+
         # DWORD is a 4-byte int. If int > 4-byte it must be divided
         if _64BIT:
             size_hi = (map_size + offset) >> 32
@@ -807,7 +809,7 @@
 
     def alloc(map_size):
         """Allocate memory.  This is intended to be used by the JIT,
-        so the memory has the executable bit set.  
+        so the memory has the executable bit set.
         XXX implement me: it should get allocated internally in
         case of a sandboxed process
         """
@@ -825,5 +827,5 @@
 
     def free(ptr, map_size):
         VirtualFree(ptr, 0, MEM_RELEASE)
-        
+
 # register_external here?

From noreply at buildbot.pypy.org  Sun Jan 22 22:35:50 2012
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sun, 22 Jan 2012 22:35:50 +0100 (CET)
Subject: [pypy-commit] pypy merge-2.7.2: CPython Issue #12100: Don't reset
 incremental encoders of CJK codecs at each call to encode().
Message-ID: <20120122213550.38D19821FA@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: merge-2.7.2
Changeset: r51668:0c697ef6b87f
Date: 2012-01-22 22:32 +0100
http://bitbucket.org/pypy/pypy/changeset/0c697ef6b87f/

Log:	CPython Issue #12100: Don't reset incremental encoders of CJK codecs
	at each call to encode().

diff --git a/pypy/module/_multibytecodec/c_codecs.py b/pypy/module/_multibytecodec/c_codecs.py
--- a/pypy/module/_multibytecodec/c_codecs.py
+++ b/pypy/module/_multibytecodec/c_codecs.py
@@ -230,14 +230,14 @@
         if ignore_error == 0:
             flags = MBENC_FLUSH | MBENC_RESET
         else:
-            flags = MBENC_RESET
+            flags = 0
         while True:
             r = pypy_cjk_enc_chunk(encodebuf, flags)
             if r == 0 or r == ignore_error:
                 break
             multibytecodec_encerror(encodebuf, r, errors,
                                     errorcb, namecb, unicodedata)
-        while True:
+        while flags & MBENC_RESET:
             r = pypy_cjk_enc_reset(encodebuf)
             if r == 0:
                 break
diff --git a/pypy/module/_multibytecodec/test/test_app_incremental.py b/pypy/module/_multibytecodec/test/test_app_incremental.py
--- a/pypy/module/_multibytecodec/test/test_app_incremental.py
+++ b/pypy/module/_multibytecodec/test/test_app_incremental.py
@@ -129,6 +129,15 @@
         r = e.encode(u"xyz\u5f95\u6c85")
         assert r == 'xyz~{abcd~}'
 
+    def test_encode_hz_noreset(self):
+        text = (u'\u5df1\u6240\u4e0d\u6b32\uff0c\u52ff\u65bd\u65bc\u4eba\u3002'
+                u'Bye.')
+        out = ''
+        e = self.IncrementalHzEncoder()
+        for c in text:
+            out += e.encode(c)
+        assert out == b'~{<:Ky2;S{#,NpJ)l6HK!#~}Bye.'
+
     def test_encode_hz_error(self):
         e = self.IncrementalHzEncoder()
         raises(UnicodeEncodeError, e.encode, u"\u4321", True)
diff --git a/pypy/translator/c/src/cjkcodecs/multibytecodec.c b/pypy/translator/c/src/cjkcodecs/multibytecodec.c
--- a/pypy/translator/c/src/cjkcodecs/multibytecodec.c
+++ b/pypy/translator/c/src/cjkcodecs/multibytecodec.c
@@ -187,7 +187,7 @@
       Py_ssize_t r;
       Py_ssize_t inleft = (Py_ssize_t)(d->inbuf_end - d->inbuf);
       Py_ssize_t outleft = (Py_ssize_t)(d->outbuf_end - d->outbuf);
-      if (inleft == 0)
+      if (inleft == 0 && !(flags & MBENC_RESET))
         return 0;
       r = d->codec->encode(&d->state, d->codec->config,
                            &d->inbuf, inleft, &d->outbuf, outleft, flags);
diff --git a/pypy/translator/c/src/cjkcodecs/multibytecodec.h b/pypy/translator/c/src/cjkcodecs/multibytecodec.h
--- a/pypy/translator/c/src/cjkcodecs/multibytecodec.h
+++ b/pypy/translator/c/src/cjkcodecs/multibytecodec.h
@@ -84,6 +84,7 @@
 #define MBERR_NOMEMORY          (-4) /* out of memory */
 
 #define MBENC_FLUSH             0x0001 /* encode all characters encodable */
+#define MBENC_RESET             0x0002 /* reset after an encoding session */
 #define MBENC_MAX               MBENC_FLUSH
 
 

From noreply at buildbot.pypy.org  Sun Jan 22 22:55:28 2012
From: noreply at buildbot.pypy.org (hodgestar)
Date: Sun, 22 Jan 2012 22:55:28 +0100 (CET)
Subject: [pypy-commit] benchmarks default: Add Genshi XML and text template
	benchmarks.
Message-ID: <20120122215528.9F9C071027D@wyvern.cs.uni-duesseldorf.de>

Author: Simon Cross 
Branch: 
Changeset: r161:fad651ead401
Date: 2012-01-22 23:50 +0200
http://bitbucket.org/pypy/benchmarks/changeset/fad651ead401/

Log:	Add Genshi XML and text template benchmarks.

diff too long, truncating to 10000 out of 32654 lines

diff --git a/benchmarks.py b/benchmarks.py
--- a/benchmarks.py
+++ b/benchmarks.py
@@ -52,6 +52,11 @@
                      extra_args=['--benchmark=' + name],
                      iteration_scaling=0.1)
 
+for name in ['xml', 'text']:
+    _register_new_bm('bm_genshi', 'genshi_' + name,
+                     globals(), bm_env={'PYTHONPATH': relative('lib/genshi')},
+                     extra_args=['--benchmark=' + name])
+
 for name in ['float', 'nbody_modified', 'meteor-contest', 'fannkuch',
              'spectral-norm', 'chaos', 'telco', 'go', 'pyflate-fast',
              'raytrace-simple', 'crypto_pyaes', 'bm_mako', 'bm_chameleon',
diff --git a/lib/genshi/COPYING b/lib/genshi/COPYING
new file mode 100644
--- /dev/null
+++ b/lib/genshi/COPYING
@@ -0,0 +1,28 @@
+Copyright (C) 2006-2010 Edgewall Software
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+ 1. Redistributions of source code must retain the above copyright
+    notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+    notice, this list of conditions and the following disclaimer in
+    the documentation and/or other materials provided with the
+    distribution.
+ 3. The name of the author may not be used to endorse or promote
+    products derived from this software without specific prior
+    written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
+OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
+GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
+IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/lib/genshi/ChangeLog b/lib/genshi/ChangeLog
new file mode 100644
--- /dev/null
+++ b/lib/genshi/ChangeLog
@@ -0,0 +1,472 @@
+Version 0.6.1
+http://svn.edgewall.org/repos/genshi/tags/0.6.1/
+(???, from branches/stable/0.6.x)
+
+ * Fix for error in how `HTMLFormFiller` would handle `textarea` elements if
+   no value was not supplied form them.
+ * The `HTMLFormFiller` now correctly handles check boxes and radio buttons
+   with an empty `value` attribute.
+
+
+Version 0.6
+http://svn.edgewall.org/repos/genshi/tags/0.6.0/
+(Apr 22 2010, from branches/stable/0.6.x)
+
+ * Support for Python 2.3 has been dropped.
+ * Rewrite of the XPath evaluation engine for better performance and improved
+   correctness. This is the result of integrating work done by Marcin Kurczych
+   during GSoC 2008.
+ * Updated the Python AST processing for template code evaluation to use the
+   `_ast` module instead of the deprecated `compiler` package, including an
+   adapter layer for Python 2.4. This, too, is the result of integrating work
+   done by  Marcin Kurczych during GSoC 2008.
+ * Added caching in the serialization stage for improved performance in some
+   cases.
+ * Various improvements to the HTML sanitization filter.
+ * Fix problem with I18n filter that would get confused by expressions in
+   attribute values when inside an `i18n:msg` block (ticket #250).
+ * Fix problem with the transformation filter dropping events after the
+   selection (ticket #290).
+ * `for` loops in template code blocks no longer establish their own locals
+   scope, meaning you can now access variables assigned in the loop outside
+   of the loop, just as you can in regular Python code (ticket #259).
+ * Import statements inside function definitions in template code blocks no 
+   longer result in an UndefinedError when the imported name is accessed 
+   (ticket #276).
+ * Fixed handling of relative URLs with fragment identifiers containing colons
+   in the `HTMLSanitizer` (ticket #274).
+ * Added an option to the `HTMLFiller` to also populate password fields.
+ * Match template processing no longer produces unwanted duplicate output in
+   some cases (ticket #254).
+ * Templates instantiated without a loader now get an implicit loader based on
+   their file path, or the current directory as a fallback (ticket #320).
+ * Added documentation for the `TemplateLoader`.
+ * Enhanced documentation for internationalization.
+
+
+Version 0.5.1
+http://svn.edgewall.org/repos/genshi/tags/0.5.1/
+(Jul 9 2008, from branches/stable/0.5.x)
+
+ * Fix problem with nested match templates not being applied when buffering
+   on the outer `py:match` is disabled. Thanks to Erik Bray for reporting the
+   problem and providing a test case!
+ * Fix problem in `Translator` filter that would cause the translation of
+   text nodes to fail if the translation function returned an object that was
+   not directly a string, but rather something like an instance of the
+   `LazyProxy` class in Babel (ticket #145).
+ * Fix problem with match templates incorrectly being applied multiple times.
+ * Includes from templates loaded via an absolute path now include the correct
+   file in nested directories as long if no search path has been configured
+   (ticket #240).
+ * Unbuffered match templates could result in parts of the matched content
+   being included in the output if the match template didn't actually consume
+   it via one or more calls to the `select()` function (ticket #243).
+
+
+Version 0.5
+http://svn.edgewall.org/repos/genshi/tags/0.5.0/
+(Jun 9 2008, from branches/stable/0.5.x)
+
+ * Added #include directive for text templates (ticket #115).
+ * Added new markup transformation filter contributed by Alec Thomas. This
+   provides gorgeous jQuery-inspired stream transformation capabilities based
+   on XPath expressions.
+ * When using HTML or XHTML serialization, the `xml:lang` attribute is
+   automatically translated to the `lang` attribute which HTML user agents
+   understand.
+ * Added support for the XPath 2 `matches()` function in XPath expressions,
+   which allow matching against regular expressions.
+ * Support for Python code blocks in templates can now be disabled
+   (ticket #123).
+ * Includes are now processed when the template is parsed if possible, but
+   only if the template loader is not set to do automatic reloading. Included
+   templates are basically inlined into the including template, which can
+   speed up rendering of that template a bit.
+ * Added new syntax for text templates, which is more powerful and flexible
+   with respect to white-space and line breaks. It also supports Python code
+   blocks. The old syntax is still available and the default for now, but in a
+   future release the new syntax will become the default, and some time after
+   that the old syntax will be removed.
+ * Added support for passing optimization hints to `` directives,
+   which can speed up match templates in many cases, for example when a match
+   template should only be applied once to a stream, or when it should not be
+   applied recursively.
+ * Text templates now default to rendering as plain text; it is no longer
+   necessary to explicitly specify the "text" method to the `render()` or
+   `serialize()` method of the generated markup stream.
+ * XInclude elements in markup templates now support the `parse` attribute;
+   when set to "xml" (the default), the include is processed as before, but
+   when set to "text", the included template is parsed as a text template using
+   the new syntax (ticket #101).
+ * Python code blocks inside match templates are now executed (ticket #155).
+ * The template engine plugin no longer adds the `default_doctype` when the
+   `fragment` parameter is `True`.
+ * The `striptags` function now also removes HTML/XML-style comments (ticket
+   #150).
+ * The `py:replace` directive can now also be used as an element, with an
+   attribute named `value` (ticket #144).
+ * The `TextSerializer` class no longer strips all markup in text by default,
+   so that it is still possible to use the Genshi `escape` function even with
+   text templates. The old behavior is available via the `strip_markup` option
+   of the serializer (ticket #146).
+ * Assigning to a variable named `data` in a Python code block no longer
+   breaks context lookup.
+ * The `Stream.render` now accepts an optional `out` parameter that can be
+   used to pass in a writable file-like object to use for assembling the
+   output, instead of building a big string and returning it.
+ * The XHTML serializer now strips `xml:space` attributes as they are only
+   allowed on very few tags.
+ * Match templates are now applied in a more controlled fashion: in the order
+   they are declared in the template source, all match templates up to (and
+   including) the matching template itself are applied to the matched content,
+   whereas the match templates declared after the matching template are only
+   applied to the generated content (ticket #186).
+ * The `TemplateLoader` class now provides an `_instantiate()` method that can
+   be overridden by subclasses to implement advanced template instantiation
+   logic (ticket #204).
+ * The search path of the `TemplateLoader` class can now contain ''load
+   functions'' in addition to path strings. A load function is passed the
+   name of the requested template file, and should return a file-like object
+   and some metadata. New load functions are supplied for loading from egg
+   package data, and loading from different loaders depending on the path
+   prefix of the requested filename (ticket #182).
+ * Match templates can now be processed without keeping the complete matched
+   content in memory, which could cause excessive memory use on long pages.
+   The buffering can be disabled using the new `buffer` optimization hint on
+   the `` directive.
+ * Improve error reporting when accessing an attribute in a Python expression
+   raises an `AttributeError` (ticket #191).
+ * The `Markup` class now supports mappings for right hand of the `%` (modulo)
+   operator in the same way the Python string classes do, except that the
+   substituted values are escape. Also, the special constructor which took
+   positional arguments that would be substituted was removed. Thus the
+   `Markup` class now supports the same arguments as that of its `unicode`
+   base class (ticket #211).
+ * The `Template` class and its subclasses, as well as the interpolation API,
+   now take an `filepath` parameter instead of `basedir` (ticket #207).
+ * The `XHTMLSerializer` now has a `drop_xml_decl` option that defaults to
+   `True`. Setting it to `False` will cause any XML decl in the serialized
+   stream to be included in the output as it would for XML serialization.
+ * Add support for a protocol that would allow interoperability of different
+   Python packages that generate and/or consume markup, based on the special
+   `__html__()` method (ticket #202).
+
+
+Version 0.4.4
+http://svn.edgewall.org/repos/genshi/tags/0.4.4/
+(Aug 14, 2007, from branches/stable/0.4.x)
+
+ * Fixed augmented assignment to local variables in Python code blocks.
+ * Fixed handling of nested function and class definitions in Python code
+   blocks.
+ * Includes were not raising `TemplateNotFound` exceptions even when no
+   fallback has been specified. That has been corrected.
+ * The template loader now raises a `TemplateNotFound` error when a previously
+   cached template is removed or renamed, where it previously was passing up
+   an `OSError`.
+ * The Genshi I18n filter can be configured to only extract messages found in
+   `gettext` function calls, ignoring any text nodes and attribute values
+   (ticket #138).
+
+
+Version 0.4.3
+http://svn.edgewall.org/repos/genshi/tags/0.4.3/
+(Jul 17 2007, from branches/stable/0.4.x)
+
+ * The I18n filter no longer extracts or translates literal strings in
+   attribute values that also contain expressions.
+ * Added `loader_callback` option to plugin interface, which allows specifying
+   a callback function that the template loader should invoke whenever a new
+   template is loaded (ticket #130). Note that the value for this option can
+   not be specified as a string, only as an actual function object, which means
+   it is not available for use through configuration files.
+ * The I18n filter now extracts messages from gettext functions even inside
+   ignored tags (ticket #132).
+ * The HTML sanitizer now strips any CSS comments in style attributes, which
+   could previously be used to hide malicious property values.
+ * The HTML sanitizer now also removes any HTML comments encountered, as those
+   may be used to hide malicious payloads targetting a certain "innovative"
+   browser that goes and interprets the content of specially prepared comments.
+ * Attribute access in template expressions no longer silently ignores
+   exceptions other than `AttributeError` raised in the attribute accessor.
+
+
+Version 0.4.2
+http://svn.edgewall.org/repos/genshi/tags/0.4.2/
+(Jun 20 2007, from branches/stable/0.4.x)
+
+ * The `doctype` parameter of the markup serializers now also accepts the "name"
+   of the doctype as string, in addition to the `(name, pubid, sysid)` tuple.
+ * The I18n filter was not replacing the original attributes with the
+   translation, but instead adding a second attribute with the same name.
+ * `TextTemplate` can now handle unicode source (ticket #125).
+ * A `` processing instruction containing trailing whitespace no
+   longer causes a syntax error (ticket #127).
+ * The I18n filter now skips the content of elements that have an `xml:lang`
+   attribute with a fixed string value. Basically, `xml:lang` can now be used
+   as a flag to mark specific sections as not needing localization.
+ * Added plugin for message extraction via Babel (http://babel.edgewall.org/).
+
+
+Version 0.4.1
+http://svn.edgewall.org/repos/genshi/tags/0.4.1/
+(May 21 2007, from branches/stable/0.4.x)
+
+ * Fix incorrect reference to translation function in the I18N filter.
+ * The `ET()` function now correctly handles attributes with a namespace.
+ * XML declarations are now processed internally, as well as written to the
+   output when XML serialization is used (ticket #111).
+ * Added the functions `encode()` and `get_serializer()` to the `genshi.output`
+   module, which provide a lower-level API to the functionality previously only
+   available through `Stream.render()` and `Stream.serialize()`.
+ * The `DocType` class now has a `get(name)` function that returns a `DOCTYPE`
+   tuple for a given string.
+ * Added frameset variants to the `DocType` constants for HTML 4.01 and XHTML
+   1.0.
+ * Improved I18n extraction for pluralizable messages: for any translation
+   function with multiple string arguments (such as ``ngettext``), a single
+   item with a tuple of strings is yielded, instead an item for each string
+   argument.
+ * The `HTMLFormFiller` stream filter no longer alters form elements for which
+   the data element contains no corresponding item.
+ * Code in `` processing instructions no longer gets the special
+   treatment as Python code in template expressions, i.e. item and attribute
+   access are no longer interchangeable (which was broken in a number of ways
+   anyway, see ticket #113). This change does not affect expressions.
+ * Numerous fixes for the execution of Python code in `` processing
+   instructions (tickets #113 and #114).
+ * The `py:def` (and `#def`) directive now supports "star args" (i.e. `*args`
+   and `**kwargs`) in the function declaration (ticket #116).
+
+
+Version 0.4
+http://svn.edgewall.org/repos/genshi/tags/0.4.0/
+(Apr 16 2007, from branches/stable/0.4.x)
+
+ * New example applications for CherryPy and web.py.
+ * The template loader now uses a LRU cache to limit the number of cached
+   templates to a configurable maximum. Also, a race condition in the template
+   loader was fixed by adding locking.
+ * A new filter (genshi.filters.HTMLFormFiller) was added, which can populate 
+   HTML forms based on a dictionary of values.
+ * The set of permitted tag and attribute names for the HTMLSanitizer can now
+   be configured per instance.
+ * The template engine plugin now supports a range of options for
+   configuration, for example to set the default serialization method, the
+   default output encoding, or the default DOCTYPE.
+ * The ElementTree adaptation function `ET()` has moved into the `genshi.input`
+   module.
+ * Allow `when` directives to omit the test expression as long as the
+   associated choose directive does have one. In that case, the when branch is
+   followed if the expression of the choose directive evaluates to a truth
+   value.
+ * Unsuccessful attribute or item lookups now return `Undefined` objects for
+   nicer error messages.
+ * Split up the `genshi.template` module into multiple modules inside the new
+   `genshi.template` package.
+ * Results of expression evaluation are no longer implicitly called if they
+   are callable.
+ * Instances of the `genshi.core.Attrs` class are now immutable (they are
+   subclasses of `tuple` instead of `list`).
+ * `MarkupTemplate`s can now be instantiated from markup streams, in addition
+   to strings and file-like objects (ticket #69).
+ * Improve handling of incorrectly nested tags in the HTML parser.
+ * Template includes can now be nested inside fallback content.
+ * Expressions can now contain dict literals (ticket #37).
+ * It is now possible to have one or more escaped dollar signs in front of a 
+   full expression (ticket #92).
+ * The `Markup` class is now available by default in template expressions
+   (ticket #67).
+ * The handling of namespace declarations in XML/XHTML output has been improved.
+ * The `Attrs` class no longer automatically wraps all attribute names in
+   `QName` objects. This is now the responsibility of whoever is instantiating
+   `Attrs` objects (for example, stream filters and generators).
+ * Python code blocks are now supported using the `` processing
+   instruction (ticket #84).
+ * The way errors in template expressions are handled can now be configured. The
+   option `LenientLookup` provides the same forgiving mode used in previous
+   Genshi versions, while `StrictLookup` raises exceptions when undefined
+   variables or members are accessed. The lenient mode is still the default in
+   this version, but that may change in the future. (ticket #88)
+ * If a variable is not necessarily defined at the top level of the template
+   data, the new built-in functions `defined(key)` and `value_of(key, default)`
+   can be used so that the template also works in strict lookup mode. These
+   functions were previously only available when using Genshi via the template
+   engine plugin (for compatibility with Kid).
+ * `style` attributes are no longer allowed by the `HTMLSanitizer` by default.
+   If they are explicitly added to the set of safe attributes, any unicode
+   escapes in the attribute value are now handled properly.
+ * Namespace declarations on conditional elements (for example using a `py:if`
+   directive`) are no longer moved to the following element when the element
+   originally carrying the declaration is removed from the stream (ticket #107).
+ * Added basic built-in support for internationalizing templates by providing
+   a new `Translator` class that can both extract localizable strings from a
+   stream, and replace those strings with their localizations at render time.
+   The code for this was largely taken from previous work done by Matt Good
+   and David Fraser.
+
+
+Version 0.3.6
+http://svn.edgewall.org/repos/genshi/tags/0.3.6/
+(Dec 11 2006, from branches/stable/0.3.x)
+
+ * The builder API now accepts streams as children of elements and fragments.
+
+
+Version 0.3.5
+http://svn.edgewall.org/repos/genshi/tags/0.3.5/
+(Nov 22 2006, from branches/stable/0.3.x)
+
+ * Fix XPath traversal in match templates. Previously, `div/p` would be treated 
+   the same as `div//p`, i.e. it would match all descendants and not just the
+   immediate children.
+ * Preserve whitespace in HTML `` elements also when they contain child
+   elements.
+ * Match templates no longer match their own output (ticket #77).
+ * Blank lines before directives in text templates are now preserved as
+   expected (ticket #62).
+
+
+Version 0.3.4
+http://svn.edgewall.org/repos/genshi/tags/0.3.4/
+(Nov 2 2006, from branches/stable/0.3.x)
+
+ * The encoding of HTML and XML files, as well as markup and text templates,
+   can now be specified. Also, the encoding specified in XML declarations is
+   now respected unless an expiclit encoding is requested.
+ * Expressions used as arguments for `py:with`, `py:def`, and `py:for`
+   directives can now contain non-ASCII strings.
+
+
+Version 0.3.3
+http://svn.edgewall.org/repos/genshi/tags/0.3.3/
+(Oct 16 2006, from branches/stable/0.3.x)
+
+ * Fixed bug introduced in 0.3.2 that broke the parsing of templates which
+   declare the same namespace more than once in a nested fashion.
+ * Fixed the parsing of HTML entity references inside attribute values, both
+   in the `XMLParser` and the `HTMLParser` classes.
+ * Some changes to usage of absolute vs. relative template paths to ensure that
+   the filenamed-keyed cache employed by the TemplateLoader doesn't mix up
+   templates with the same name, but from different subdirectories.
+
+
+Version 0.3.2
+http://svn.edgewall.org/repos/genshi/tags/0.3.2/
+(Oct 12 2006, from branches/stable/0.3.x)
+
+ * Exceptions from templates now contain the absolute path to the template file
+   when a search path is used. This enables tracebacks to display the line in
+   question.
+ * The template engine plugin now provides three different types: "genshi" and
+   "genshi-markup" map to markup templates, while "genshi-text" maps to text
+   templates.
+ * Fixed the namespace context used by XPath patterns in py:match templates.
+   The were erroneously using the namespace context of the elements being
+   matched, where they should rather use the context in which they were
+   defined.
+ * The contents of `
+  ...

	PyPy
{{ item.name }}	{% if item.pypy_exists %}✔{% else %}✖{% endif %}
{{ item.name }}	{% if item.pypy_exists %}✔{% else %}✖{% endif %}